diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/manifest.txt b/tiera-rerun-fixed-moonep/echo/tierbprobe/manifest.txt new file mode 100644 index 0000000000000000000000000000000000000000..dfc35d7ae904f15961b102b317f00ae3dad7627e --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/manifest.txt @@ -0,0 +1,19 @@ +timestamp: 2026-08-30T04:52:53+00:00 +host: sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 +gpu: NVIDIA B200 x8 +driver: 580.95.05 +study_scripts_branch: granularity-echo +study_scripts_commit: 774e9e2 +campaign: echo +backend: echo +phase: tierbprobe +gbs: 16 +seq: 4096 +configs: G1 G2 G3 G4 +capacity: 1.25 +repeats: 3 +nsys: 1 +megatron_repo: /workspace/Megatron-MoE-Echo +megatron_branch_required: token-drop-analysis +megatron_branch_actual: token-drop-analysis +megatron_commit: b099fd84 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/pip_freeze.txt b/tiera-rerun-fixed-moonep/echo/tierbprobe/pip_freeze.txt new file mode 100644 index 0000000000000000000000000000000000000000..2911a706106f0055210d5f3631cc878b909ba134 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/pip_freeze.txt @@ -0,0 +1,317 @@ +absl-py==2.3.0 +aiohappyeyeballs @ file:///rapids/aiohappyeyeballs-2.6.1-py3-none-any.whl#sha256=f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 +aiohttp @ file:///rapids/aiohttp-3.12.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=94f98e0e5a49f89b252e115844f756c04fc8050f38252a32a3dd994ce8121f10 +aiosignal @ file:///rapids/aiosignal-1.3.2-py2.py3-none-any.whl#sha256=45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5 +annotated-types==0.7.0 +anyio==4.9.0 +apex @ file:///opt/pytorch/apex/dist/apex-0.1-cp312-cp312-linux_x86_64.whl#sha256=a1508e4a386362e9c8f46c92ccada9110e421212b964c694235d0fd5a224b170 +argon2-cffi==25.1.0 +argon2-cffi-bindings==21.2.0 +arrow==1.3.0 +asciitree @ file:///rapids/asciitree-0.3.3-py3-none-any.whl#sha256=3e507ccfaeb417823158091fdb9d26b0205f6a59ba680b479a71db65c73d01d4 +asttokens==3.0.0 +astunparse==1.6.3 +async-lru==2.0.5 +attrs==25.3.0 +audioread==3.0.1 +babel==2.17.0 +beautifulsoup4==4.13.4 +black==25.1.0 +bleach==6.2.0 +blis==0.7.11 +cachetools @ file:///rapids/cachetools-6.0.0-py3-none-any.whl#sha256=82e73ba88f7b30228b5507dce1a1f878498fc669d972aef2dde4f3a3c24f103e +catalogue==2.0.10 +certifi==2025.4.26 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.5.0 +cloudpathlib==0.21.1 +cloudpickle @ file:///rapids/cloudpickle-3.1.1-py3-none-any.whl#sha256=c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e +cmake==3.31.6 +comm==0.2.2 +confection==0.1.5 +contourpy==1.3.2 +cuda-bindings @ file:///rapids/cuda_bindings-12.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ff0e28d1e34758654b9c961e1f55e4786e49aee6a4dbceaf3cc24c46c672df7e +cuda-python @ file:///rapids/cuda_python-12.9.0-py3-none-any.whl#sha256=926acba49b2c0a0374c61b7c98f337c085199cf51cdfe4d6423c4129c20547a7 +cudf @ file:///rapids/cudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b779a8e8c0e8577b9b560895958496da211bfac49cdc4f11a49e8d18cb82e348 +cudf-polars @ file:///rapids/cudf_polars-25.4.0-py3-none-any.whl#sha256=227b6c5147e93ea86282c8f7219118f916769fa2ca0cdd931e97a148aa66b869 +cugraph @ file:///rapids/cugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b837c2dc2f7b235c702e4f8df64479b8fcbb342f848516d5faf0cdc4b5f31f29 +cugraph-service-client @ file:///rapids/cugraph_service_client-25.4.0-py3-none-any.whl#sha256=7e568c3e41154b9752f5173555ffcc282ac52d72343feeb79653b2e2b1232470 +cugraph-service-server @ file:///rapids/cugraph_service_server-25.4.0-py3-none-any.whl#sha256=a2830dcffafcccb16e12ebf85b98f8f247869d38c01ab4dd8664a1e7439d31be +cuml @ file:///rapids/cuml-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1e90a464a18b094e24797f1676688c5d7696c4cf97c934504214a53d5aa09aa5 +cupy-cuda12x @ file:///rapids/cupy_cuda12x-13.3.0-cp312-cp312-linux_x86_64.whl#sha256=6a2dfa0aa1e8189a10d1534276c08f1d2f73b747fd28bfbe7b9f4ef66d9af73f +cuvs @ file:///rapids/cuvs-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b12f6b9f80e053a3b38542c2947d2516e468c02cad54a13d8b1a4a61d5aa7914 +cycler==0.12.1 +cymem==2.0.11 +Cython==3.1.1 +dask @ file:///rapids/dask-2025.2.0-py3-none-any.whl#sha256=f0fdeef6ceb0a06569d456c9e704f220f7f54e80f3a6ea42ab98cea6bc642b6e +dask-cuda @ file:///rapids/dask_cuda-25.4.0-py3-none-any.whl#sha256=442d87da0a95d00ca2a262b64c76e785a32ad905c82000fa2227932f81390e19 +dask-cudf @ file:///rapids/dask_cudf-25.4.0-py3-none-any.whl#sha256=5e7cea0b26daea17da6e0bb457435e24460f76ba740358a3a18e239d6015c95c +datasets==5.0.1 +debugpy==1.8.14 +decorator==5.2.1 +-e git+https://github.com/deepseek-ai/DeepEP.git@34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72#egg=deep_ep +defusedxml==0.7.1 +dill==0.4.1 +distributed @ file:///rapids/distributed-2025.2.0-py3-none-any.whl#sha256=368462084be47092d0a835f6c64c9b852da64dc88302b76d5f93bd5131228c67 +distributed-ucxx @ file:///rapids/distributed_ucxx-0.43.0-py3-none-any.whl#sha256=94f60d0ccb0c94ba8dbfa1eaf455874d18aaab17e2ae218c38590e9fdc2d721d +dm-tree==0.1.9 +einops==0.8.1 +execnet==2.1.1 +executing==2.2.0 +expecttest==0.3.0 +fasteners @ file:///rapids/fasteners-0.19-py3-none-any.whl#sha256=758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237 +fastjsonschema==2.21.1 +fastrlock @ file:///rapids/fastrlock-0.8.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl#sha256=5f13ec08f1adb1aa916c384b05ecb7dbebb8df9ea81abd045f60941c6283a670 +filelock==3.18.0 +flash_attn @ file:///opt/pytorch/flash_attn-2.7.4.post1-cp312-cp312-linux_x86_64.whl#sha256=a10455d6c56b387785b7413fd0bbcb392b42b3d4f4b88fda5d86cc8a9ce9182a +fonttools==4.58.1 +fqdn==1.5.1 +frozenlist @ file:///rapids/frozenlist-1.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=4e1be9111cb6756868ac242b3c2bd1f09d9aea09846e4f5c23715e7afb647103 +fsspec==2025.5.1 +gast==0.6.0 +grpcio==1.62.1 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.29.0 +hypothesis==6.130.8 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +intel-openmp==2021.4.0 +ipykernel==6.29.5 +ipython==9.3.0 +ipython_pygments_lexers==1.1.1 +isoduration==20.11.0 +isort==6.0.1 +jedi==0.19.2 +Jinja2==3.1.6 +joblib==1.5.1 +json5==0.12.0 +jsonpointer==3.0.0 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +jupyter-events==0.12.0 +jupyter-lsp==2.2.5 +jupyter_client==8.6.3 +jupyter_core==5.8.1 +jupyter_server==2.16.0 +jupyter_server_terminals==0.5.3 +jupyterlab==4.4.3 +jupyterlab_code_formatter==3.0.2 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.3 +jupyterlab_tensorboard_pro @ file:///builder/jupyterlab_tensorboard_pro-4.0.0-py2.py3-none-any.whl#sha256=6bfcb5ced3358eb72150f4b2ab91312faef2f0ca14d845e8379d8fa52df6902e +jupytext==1.17.2 +kiwisolver==1.4.8 +kvikio @ file:///rapids/kvikio-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=8b40fd8ea9e4a2974c04b50b7fc26b06094ac807f9a233281d0bafbaa71b1d85 +langcodes==3.5.0 +language_data==1.3.0 +lazy_loader==0.4 +libcudf @ file:///rapids/libcudf-25.4.0-py3-none-linux_x86_64.whl#sha256=238320b4958c1737bba2a609673e2539a78216b811b5498a68e00395d9b021a4 +libcugraph @ file:///rapids/libcugraph-25.4.0-py3-none-linux_x86_64.whl#sha256=29085e901b2f7569ff2653bb1355fe6fad20f2fcaa585f5649da9867d8df766d +libcuml @ file:///rapids/libcuml-25.4.0-py3-none-linux_x86_64.whl#sha256=8e2e444c72b0e995e15a48da37ff12fb410a47a2baa69ba86ccc9b1a3db91e8c +libcuvs @ file:///rapids/libcuvs-25.4.0-py3-none-linux_x86_64.whl#sha256=e2f8db61c92b769c590dd752f43439ddd62b2f6592a3c337362b6c1ef220c3b9 +libkvikio @ file:///rapids/libkvikio-25.4.0-py3-none-linux_x86_64.whl#sha256=da8a5713b05fc478bf40529a3e0fb61a2d605bb5a70ca4d461f96a6005dbe8b7 +libraft @ file:///rapids/libraft-25.4.0-py3-none-linux_x86_64.whl#sha256=b921dec005aa4a321dd156776a18376feb47c96a5b67eec125c8a8e19edd1e24 +librmm @ file:///rapids/librmm-25.4.0-py3-none-linux_x86_64.whl#sha256=ff951811bda729d4fa1109ab906225ae1f489a6ec2eeab5d1b674cc34bd4422f +librmm-cu12 @ file:///rapids/librmm_cu12-25.4.0-py3-none-any.whl#sha256=29296ef2b0e90679a0db280e86015be12d6ab9f4e62a181aa085f62dbadd06b3 +librosa==0.11.0 +libucxx @ file:///rapids/libucxx-0.43.0-py3-none-linux_x86_64.whl#sha256=f15849d7ac2fb68f0fd8998838a13321ddcfb0c594f7e6cbc401a3bcc4750566 +lightning-thunder @ file:///opt/pytorch/lightning-thunder +lightning-utilities==0.14.3 +lintrunner==0.12.7 +llvmlite==0.42.0 +locket @ file:///rapids/locket-1.0.0-py2.py3-none-any.whl#sha256=b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3 +looseversion==1.3.0 +marisa-trie==1.2.1 +Markdown==3.8 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.3 +matplotlib-inline==0.1.7 +mdit-py-plugins==0.4.2 +mdurl==0.1.2 +-e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@b099fd8494b09e8c08ad34e26abf91fb6c83636d#egg=megatron_core +mistune==3.1.3 +mkl==2021.1.1 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +ml_dtypes==0.6.0 +mock==5.2.0 +-e git+ssh://git@github.com/sriram-17-17/MoonEP.git@d8bc6667fad7177e6e09b637d773fa67f08bae93#egg=moonep +mpmath==1.3.0 +msgpack==1.1.0 +multidict @ file:///rapids/multidict-6.4.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a920f9cf2abdf6e493c519492d892c362007f113c94da4c239ae88429835bad1 +multiprocess==0.70.19 +murmurhash==1.0.13 +mypy_extensions==1.1.0 +nbclient==0.10.2 +nbconvert==7.16.6 +nbformat==5.10.4 +nest-asyncio==1.6.0 +networkx==3.5 +ninja==1.11.1.4 +notebook==7.4.3 +notebook_shim==0.2.4 +numba==0.59.1 +numba-cuda @ file:///rapids/numba_cuda-0.4.0-py3-none-any.whl#sha256=5b95a5f42068771e37142944d44fe844fa87f1b2f89051461a4cb0af91bcc28a +numcodecs @ file:///rapids/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53 +numpy==1.26.4 +nvdlfw_inspect @ git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git@6b60eb0e675606fb2fbbfcb12667ecbec75eaf87 +nvfuser==0.2.27a0+9bf5aca +nvidia-cuda-cccl-cu12==12.9.27 +nvidia-cudnn-frontend @ file:///opt/pytorch/lightning-thunder/tmp/cudnn_frontend +nvidia-cutlass-dsl==4.4.2 +nvidia-cutlass-dsl-libs-base==4.4.2 +nvidia-dali-cuda120==1.50.0 +nvidia-ml-py @ file:///rapids/nvidia_ml_py-12.575.51-py3-none-any.whl#sha256=eb8641800d98ce40a22f479873f34b482e214a7e80349c63be51c3919845446e +nvidia-modelopt==0.29.0 +nvidia-modelopt-core==0.29.0 +nvidia-nvcomp-cu12==4.2.0.14 +nvidia-nvimgcodec-cu12==0.5.0.13 +nvidia-nvjpeg-cu12==12.4.0.16 +nvidia-nvjpeg2k-cu12==0.8.1.40 +nvidia-nvshmem-cu12==3.7.2 +nvidia-nvtiff-cu12==0.5.0.67 +nvidia-resiliency-ext==0.4.0 +nvtx @ file:///rapids/nvtx-0.2.11-cp312-cp312-linux_x86_64.whl#sha256=73521e37cba42b4db0b3cd68fc5eb814e55c500a61eeed6edcdf6746315ac53e +nx-cugraph @ file:///rapids/nx_cugraph-25.4.0-py3-none-any.whl#sha256=e625444486cc899a77bd48e7da76a2edd95c0006dd95fbf323745feae7f33111 +onnx @ file:///opt/pytorch/pytorch/third_party/onnx +onnx-ir==1.0.0 +onnxscript==0.7.1 +opt_einsum==3.4.0 +optree==0.16.0 +overrides==7.7.0 +packaging==23.2 +pandas @ file:///rapids/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319 +pandocfilters==1.5.1 +parso==0.8.4 +partd @ file:///rapids/partd-1.4.2-py3-none-any.whl#sha256=978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f +pathspec==0.12.1 +pexpect==4.9.0 +pillow==11.2.1 +platformdirs==4.3.8 +pluggy==1.6.0 +ply @ file:///rapids/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce +polars @ file:///rapids/polars-1.25.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f7fcbb4f476784384ccda48757fca4e8c2e2c5a0a3aef3717aaf56aee4e30e09 +polygraphy==0.49.20 +pooch==1.8.2 +preshed==3.0.10 +prometheus_client==0.22.1 +prompt_toolkit==3.0.51 +propcache @ file:///rapids/propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25 +protobuf==4.24.4 +psutil==7.0.0 +ptyprocess==0.7.0 +PuLP==3.2.1 +pure_eval==0.2.3 +pyarrow==25.0.1 +pybind11==2.13.6 +pybind11_global==2.13.6 +pycocotools @ git+https://github.com/nvidia/cocoapi.git@08350a157b41f0755a536441b5abec89ab934c5b#subdirectory=PythonAPI +pycparser==2.22 +pydantic==2.11.5 +pydantic_core==2.33.2 +Pygments==2.19.1 +pylibcudf @ file:///rapids/pylibcudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=401b8bfd022d06f84e120779ca7c3d01a28b68762aaa8f7873b10d6b1975cf6a +pylibcugraph @ file:///rapids/pylibcugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=061c7389a88c815b54fd5d3a3584fb87bf693e5602f5be335260d0ba575ba557 +pylibcugraphops @ file:///rapids/pylibcugraphops-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=a25ef4b22166c80718d5c7e5bddb755e81a2fe340fa5becd0ac36ec3edfb208a +pylibraft @ file:///rapids/pylibraft-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=f2ecd23fe51f9c2caa87bbbe85cb32fcaff4ecfb52ceb9a9991bc50f82782c2e +pylibwholegraph @ file:///rapids/pylibwholegraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b82913d1e38bda76d8c272fbd8cc9072d987209d3d47cc93a464a37c2925aca2 +pynvjitlink @ file:///rapids/pynvjitlink-0.3.0-cp312-cp312-linux_x86_64.whl#sha256=066caf1314730643d3b48c9847bd65789defdd6a7c78993176eb3d8bbe3f4919 +pynvml @ file:///rapids/pynvml-12.0.0-py3-none-any.whl#sha256=fdff84b62a27dbe98e08e1a647eb77342bef1aebe0878bcd15e99a83fcbecb9e +pyparsing==3.2.3 +pytest==8.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==15.1 +pytest-shard==0.1.2 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-hostlist==2.2.1 +python-json-logger==3.3.0 +pytorch-triton @ file:///tmp/dist/pytorch_triton-3.3.0%2Bgit96316ce52.nvinternal-cp312-cp312-linux_x86_64.whl#sha256=6d6921772b9dfd0243c055e0d054de42a113e92cabcfdd1d3e80a5ad9c08cf7d +pytz @ file:///rapids/pytz-2023.4-py2.py3-none-any.whl#sha256=f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a +PyYAML==6.0.2 +pyzmq==26.4.0 +raft-dask @ file:///rapids/raft_dask-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=36cf37ba6ce35454c4729c2cb597a360f22780eefb11d514a16a07137da5eb6c +rapids-dask-dependency @ file:///rapids/rapids_dask_dependency-25.4.0a0-py3-none-any.whl#sha256=484faca626e7760112a7d8ad3426f5bdd720b609006d8dca87c2607076a348bd +rapids-logger @ file:///rapids/rapids_logger-0.1.18-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=3617b2d86291ac6cb957542895f20fc9737ace29773ca90337e6fb04c73bd120 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.3 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rich==14.0.0 +rmm @ file:///rapids/rmm-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1824bcab3836296666980a997ddcb9c4bca24dfb873012cfa0a7ecea583ca712 +rpds-py==0.25.1 +safetensors==0.5.3 +scikit-learn==1.6.1 +scipy==1.15.3 +Send2Trash==1.8.3 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.16.0 +smart-open==7.1.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.13.1 +soupsieve==2.7 +soxr==0.5.0.post1 +spacy==3.7.5 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +srsly==2.5.1 +stack-data==0.6.3 +sympy==1.14.0 +tabulate==0.9.0 +tbb==2021.13.1 +tblib @ file:///rapids/tblib-3.1.0-py3-none-any.whl#sha256=670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorrt @ file:///workspace/TensorRT-10.11.0.33/python/tensorrt-10.11.0.33-cp312-none-linux_x86_64.whl#sha256=c2f18f72758a3d54a71901495c8fda9e7094a8bcdd2a5b6b58967fbf05733b3a +terminado==0.18.1 +thinc==8.2.5 +threadpoolctl==3.6.0 +thriftpy2 @ file:///rapids/thriftpy2-0.5.2-cp312-cp312-linux_x86_64.whl#sha256=c0c7d939bdc5ccf7739154959938445d5fdf72f1d6e12bd563947a36f67e2147 +tinycss2==1.4.0 +toolz @ file:///rapids/toolz-1.0.0-py3-none-any.whl#sha256=292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236 +torch @ file:///opt/transfer/torch-2.8.0a0%2B5228986c39.nv25.6-29946474-cp312-cp312-linux_x86_64.whl#sha256=7a3c9ff5288d8102a31296b3d62491f3f47c8820a5c9aaefc7dc0fe20e1e8ad5 +torch_tensorrt @ file:///opt/pytorch/torch_tensorrt/dist/torch_tensorrt-2.8.0a0-cp312-cp312-linux_x86_64.whl#sha256=26f4d7a04e7e6e017a5b917b8b98b83720bb2bb0195f48d25f52d567d286bfe1 +torchao @ file:///opt/pytorch/ao +torchprofile==0.0.4 +torchvision @ file:///opt/transfer/torchvision-0.22.0a0%2B95f10a4e-29946474-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=746a3c674cc61680a86f4a14b374b21af1b5c21f629c42765b40c9494f6b234d +tornado==6.5.1 +tqdm==4.67.1 +traitlets==5.14.3 +transformer_engine @ file:///pvcdata/aiinfra/b-aghosh/wheels/transformer_engine-2.10.0.dev0%2B10f6128d-cp312-cp312-linux_x86_64.whl#sha256=35cf5f4b83b2baad864e12e512bb422e0929409ac05610812fa521d4942d0189 +treelite @ file:///rapids/treelite-4.4.1-py3-none-linux_x86_64.whl#sha256=174741724205540a1a5b37da81483aeff55d89548f42b3461cf07acac7ae96eb +typer==0.16.0 +types-dataclasses==0.6.6 +types-python-dateutil==2.9.0.20250516 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata @ file:///rapids/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 +ucx-py @ file:///rapids/ucx_py-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=4a452fd8d8c983283ada3d49086e3657dc2ee94fbad08efdd708d15ab6e0b73a +ucxx @ file:///rapids/ucxx-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=f1b4e55cb45c66fe88a2d8f9c2b87751165f3cfec21fbdc9c47c4ddff75e8a5f +-e git+https://github.com/Dots-Infra/UltraEP.git@94cab099b44fffa99a82fea99e7c12d89cf65e4f#egg=ultra_ep +uri-template==1.3.0 +urllib3==2.0.7 +wasabi==1.1.3 +wcwidth==0.2.13 +weasel==0.4.1 +webcolors==24.11.1 +webencodings==0.5.1 +websocket-client==1.8.0 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +xdoctest==1.0.2 +xgboost @ file:///rapids/xgboost-2.1.4-py3-none-manylinux_2_39_x86_64.whl#sha256=25eeca27f4048fc491940b3a0b4b5782ee48c33cb3aec059cef71f8da542930b +xxhash==4.0.1 +yarl @ file:///rapids/yarl-1.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=bf099e2432131093cc611623e0b0bcc399b8cddd9a91eded8bfb50402ec35018 +zarr @ file:///rapids/zarr-2.18.7-py3-none-any.whl#sha256=ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223 +zict @ file:///rapids/zict-3.0.0-py2.py3-none-any.whl#sha256=5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae +zipp==3.22.0 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/provenance.txt b/tiera-rerun-fixed-moonep/echo/tierbprobe/provenance.txt index b0e6d24314c0c4d3a3d740db87b2831ceca54710..53b5174dfc876a90617e5ef259c0ddc7201414ac 100644 --- a/tiera-rerun-fixed-moonep/echo/tierbprobe/provenance.txt +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/provenance.txt @@ -27,3 +27,64 @@ command : run_ladder.sh --phase tierbprobe --gbs 16 --out /workspace/tiera 7, 120 MHz, 1965 MHz, 1000.00 W, 28 NVLink topology: + GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID + GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A + GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A + GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A + GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A + GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A + GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A + NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE + NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE + NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS + NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS + NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS + unavailable + +---- toolchain ----------------------------------------------- + nvcc : release 12.9, V12.9.86 + gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 + python : 3.12.3 + torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3 + torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120'] + transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py] + deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py] + ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py] + moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py] + megatron : 0.14.0rc5 [None] + flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py] + numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py] + huggingface_hub: 1.29.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py] + +---- repositories -------------------------------------------- + Megatron-MoE-Echo token-drop-analysis b099fd8494b0 + Megatron-UltraEP ultraep-bracing-regions-through-kernels 03d57ef7d476 + UltraEP main 94cab099b44f + DeepEP HEAD 34152ae28f80 (+3 untracked: build output) + MoonEP megatron-ddp-symmetric-params d8bc6667fad7 + blackwell_run_scripts granularity-echo 774e9e241ebc + +---- environment --------------------------------------------- + CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 + CUDA_DRIVER_VERSION=575.57.08 + CUDA_HOME=/usr/local/cuda + CUDA_MODULE_LOADING=LAZY + CUDA_VERSION=12.9.1.010 + HF_REMOTE_ROOT=tiera-rerun-fixed-moonep + NCCL_VERSION=2.27.3 + NVSHMEM_VERSION=3.2.5 + PYTORCH_BUILD_NUMBER=0 + PYTORCH_BUILD_VERSION=2.8.0a0+5228986 + PYTORCH_HOME=/opt/pytorch/pytorch + PYTORCH_VERSION=2.8.0a0+5228986 + TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1 + TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX + TORCH_NCCL_USE_COMM_NONBLOCKING=0 + +---- shared library resolution ------------------------------- + libnvshmem_host.so: not in ldconfig + libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2 + lrwxrwxrwx 1 root root 20 Aug 29 16:41 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3 + -rw-r--r-- 1 root root 179045744 Aug 29 16:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..4f4b0d2328a91a4d779f59fde42d8ad2b294aaf7 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv @@ -0,0 +1,256 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 819 +2, 819 +3, 819 +4, 819 +5, 819 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1895 +1, 1895 +2, 1895 +3, 1895 +4, 1895 +5, 1895 +6, 1895 +7, 1895 +0, 39215 +1, 39213 +2, 39213 +3, 39213 +4, 39213 +5, 39213 +6, 39213 +7, 39213 +0, 39617 +1, 39615 +2, 39615 +3, 39615 +4, 39615 +5, 39615 +6, 39615 +7, 39615 +0, 40265 +1, 40265 +2, 40263 +3, 40263 +4, 40263 +5, 40263 +6, 40263 +7, 40263 +0, 42325 +1, 42321 +2, 42321 +3, 42321 +4, 42321 +5, 42321 +6, 42321 +7, 42321 +0, 42551 +1, 42547 +2, 42547 +3, 42547 +4, 42547 +5, 42547 +6, 42547 +7, 42547 +0, 42767 +1, 42741 +2, 42735 +3, 42733 +4, 42749 +5, 42727 +6, 42757 +7, 42765 +0, 43461 +1, 43457 +2, 43457 +3, 43457 +4, 43457 +5, 43457 +6, 43457 +7, 43457 +0, 43461 +1, 43457 +2, 43457 +3, 43457 +4, 43457 +5, 43457 +6, 43457 +7, 43457 +0, 44357 +1, 44353 +2, 44353 +3, 44353 +4, 44353 +5, 44353 +6, 44353 +7, 44353 +0, 44805 +1, 44801 +2, 44801 +3, 44801 +4, 44801 +5, 44801 +6, 44801 +7, 44801 +0, 81803 +1, 81799 +2, 81799 +3, 81799 +4, 81801 +5, 81799 +6, 81799 +7, 81799 +0, 81803 +1, 81799 +2, 81799 +3, 81799 +4, 81801 +5, 81799 +6, 81799 +7, 81799 +0, 82699 +1, 82695 +2, 82695 +3, 82695 +4, 82697 +5, 82695 +6, 82695 +7, 82695 +0, 82699 +1, 82695 +2, 82695 +3, 82695 +4, 82697 +5, 82695 +6, 82695 +7, 82695 +0, 99547 +1, 99545 +2, 99545 +3, 99545 +4, 99545 +5, 99545 +6, 99545 +7, 99545 +0, 100195 +1, 100193 +2, 100193 +3, 100193 +4, 100193 +5, 100193 +6, 100193 +7, 100193 +0, 101605 +1, 101581 +2, 101613 +3, 101577 +4, 101575 +5, 101589 +6, 101585 +7, 101577 +0, 102441 +1, 102479 +2, 102413 +3, 102421 +4, 102499 +5, 102451 +6, 102489 +7, 102499 +0, 114695 +1, 114747 +2, 114747 +3, 114747 +4, 114637 +5, 114747 +6, 114747 +7, 114747 +0, 125477 +1, 125471 +2, 125471 +3, 125471 +4, 125415 +5, 125471 +6, 125471 +7, 125471 +0, 125477 +1, 125471 +2, 125471 +3, 125471 +4, 125415 +5, 125471 +6, 125471 +7, 125471 +0, 125477 +1, 125471 +2, 125471 +3, 125471 +4, 125415 +5, 125471 +6, 125471 +7, 125471 +0, 125093 +1, 124835 +2, 125209 +3, 125067 +4, 125153 +5, 124883 +6, 124741 +7, 125209 +0, 1831 +1, 1834 +2, 1834 +3, 2090 +4, 124507 +5, 3114 +6, 1834 +7, 1834 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1/run.env b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..02ada6af35df6e589356fb4ee15b6e9bf41cdf90 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1/run.env @@ -0,0 +1,70 @@ +backend=echo +config=G1 +family=tierA +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=14336 +num_experts=8 +topk=2 +experts_per_rank=1 +expert_params=176160768 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=8192 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=1 +capacity_mode=static +capacity=2.00 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c348ea639cfaaf5888a36b2bc43560acd7c6fb36 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log @@ -0,0 +1,16347 @@ +[04:52:54] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G1 / echo (slots/rank=1 policy=matched) + E=8 K=2 H'=14336 m=1 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=2.00 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0830 04:52:55.692000 439912 torch/distributed/run.py:766] +W0830 04:52:55.692000 439912 torch/distributed/run.py:766] ***************************************** +W0830 04:52:55.692000 439912 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 04:52:55.692000 439912 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 14336 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 2.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 8 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank [Gloo] Rank [Gloo] Rank 40 is connected to 1 is connected to 7 is connected to 7 peer ranks. 7 peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 7 +7 + +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 75 is connected to +7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank2]:[W830 04:53:13.459435162 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 04:53:13.459576846 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 04:53:13.459710240 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 04:53:13.460120903 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 04:53:13.460187540 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 04:53:13.460199431 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 04:53:13.460336764 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 04:53:13.613292577 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.894 seconds +time to initialize megatron (seconds): 8.627 +[after megatron is initialized] datetime: 2026-08-30 04:53:20 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3902410752 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (41988096 elements, 41988096 padded size): + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight +Params for bucket 3 (41984000 elements, 41984000 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.pre_mlp_layernorm.weight +Params for bucket 4 (41984000 elements, 41984000 padded size): + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 5 (41984000 elements, 41984000 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight +Params for bucket 6 (41984000 elements, 41984000 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight +Params for bucket 7 (41984000 elements, 41984000 padded size): + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 8 (41984000 elements, 41984000 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.pre_mlp_layernorm.weight +Params for bucket 9 (41984000 elements, 41984000 padded size): + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight +Params for bucket 10 (41984000 elements, 41984000 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_qkv.weight +Params for bucket 11 (41984000 elements, 41984000 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.pre_mlp_layernorm.weight +Params for bucket 12 (41984000 elements, 41984000 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight +Params for bucket 13 (41984000 elements, 41984000 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight +Params for bucket 14 (41984000 elements, 41984000 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight +Params for bucket 15 (41984000 elements, 41984000 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.weight +Params for bucket 16 (41984000 elements, 41984000 padded size): + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight +Params for bucket 17 (41984000 elements, 41984000 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.pre_mlp_layernorm.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 32 +Params for bucket 1 (58720256 elements, 58720256 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 +Params for bucket 2 (117440512 elements, 117440512 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 +Params for bucket 3 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 4 (117440512 elements, 117440512 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 5 (58720256 elements, 58720256 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 +Params for bucket 6 (117440512 elements, 117440512 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 +Params for bucket 7 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 +Params for bucket 8 (117440512 elements, 117440512 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 +Params for bucket 9 (58720256 elements, 58720256 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 +Params for bucket 10 (117440512 elements, 117440512 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 +Params for bucket 11 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 12 (117440512 elements, 117440512 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (58720256 elements, 58720256 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 +Params for bucket 14 (117440512 elements, 117440512 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 15 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 +Params for bucket 16 (117440512 elements, 117440512 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 18 (117440512 elements, 117440512 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 19 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 20 (117440512 elements, 117440512 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 +Params for bucket 22 (117440512 elements, 117440512 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 +Params for bucket 23 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 24 (117440512 elements, 117440512 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (58720256 elements, 58720256 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 26 (117440512 elements, 117440512 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 28 (117440512 elements, 117440512 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 30 (117440512 elements, 117440512 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 32 (117440512 elements, 117440512 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 04:53:20 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Build and save the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +INFO:megatron.core.datasets.gpt_dataset:> total number of epochs: 1 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 04:53:20 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (206.35, 208.37) + train/valid/test-data-iterators-setup ..........: (116.56, 123.77) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 04:53:20 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[135, 3278, 551, 493, 1454, 545, 1060, 676] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[162, 3000, 1014, 244, 1218, 479, 1024, 1051] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[112, 3316, 751, 344, 1291, 585, 654, 1139] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[67, 3339, 585, 272, 1396, 435, 832, 1266] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[111, 2812, 847, 472, 1557, 247, 1124, 1022] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[52, 3184, 941, 393, 1421, 493, 745, 963] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[47, 3304, 407, 195, 1691, 545, 1161, 842] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[72, 3231, 639, 272, 1423, 590, 1018, 947] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[52, 929, 1033, 0, 941, 297, 393, 688, 1018, 0, 493, 534, 745, 71, 963, 35] rerouted_tokens_per_ep_rank=[981, 1033, 1238, 1081, 1018, 1027, 816, 998] avg=1024.0000 max=1238 max_rank=2 min=816 min_rank=6 imbalance_ratio=1.208984 max_load_reduction=6954 max_load_reduction_pct=0.848877 moved_to_echo=3859 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[112, 968, 1075, 0, 751, 270, 344, 717, 925, 0, 585, 556, 654, 64, 1139, 32] rerouted_tokens_per_ep_rank=[1080, 1075, 1021, 1061, 925, 1141, 718, 1171] avg=1024.0000 max=1171 max_rank=7 min=718 min_rank=6 imbalance_ratio=1.143555 max_load_reduction=7021 max_load_reduction_pct=0.857056 moved_to_echo=3955 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[135, 956, 1064, 0, 551, 304, 493, 708, 1042, 0, 545, 550, 1060, 72, 676, 36] rerouted_tokens_per_ep_rank=[1091, 1064, 855, 1201, 1042, 1095, 1132, 712] avg=1024.0000 max=1201 max_rank=3 min=712 min_rank=7 imbalance_ratio=1.172852 max_load_reduction=6991 max_load_reduction_pct=0.853394 moved_to_echo=3981 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[67, 974, 1083, 0, 585, 292, 272, 722, 1001, 0, 435, 560, 832, 69, 1266, 34] rerouted_tokens_per_ep_rank=[1041, 1083, 877, 994, 1001, 995, 901, 1300] avg=1024.0000 max=1300 max_rank=7 min=877 min_rank=2 imbalance_ratio=1.269531 max_load_reduction=6892 max_load_reduction_pct=0.841309 moved_to_echo=4197 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[72, 943, 1048, 0, 639, 298, 272, 698, 1019, 0, 590, 542, 1018, 71, 947, 35] rerouted_tokens_per_ep_rank=[1015, 1048, 937, 970, 1019, 1132, 1089, 982] avg=1024.0000 max=1132 max_rank=5 min=937 min_rank=2 imbalance_ratio=1.105469 max_load_reduction=7060 max_load_reduction_pct=0.861816 moved_to_echo=4222 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[162, 880, 962, 0, 1014, 258, 244, 652, 861, 0, 479, 506, 1024, 65, 1051, 34] rerouted_tokens_per_ep_rank=[1042, 962, 1272, 896, 861, 985, 1089, 1085] avg=1024.0000 max=1272 max_rank=2 min=861 min_rank=4 imbalance_ratio=1.242188 max_load_reduction=6920 max_load_reduction_pct=0.844727 moved_to_echo=4020 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[47, 964, 1072, 0, 407, 354, 195, 714, 1211, 0, 545, 554, 1161, 84, 842, 42] rerouted_tokens_per_ep_rank=[1011, 1072, 761, 909, 1211, 1099, 1245, 884] avg=1024.0000 max=1245 max_rank=6 min=761 min_rank=2 imbalance_ratio=1.215820 max_load_reduction=6947 max_load_reduction_pct=0.848022 moved_to_echo=4439 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[111, 820, 913, 0, 847, 326, 472, 608, 1115, 0, 247, 471, 1124, 78, 1022, 38] rerouted_tokens_per_ep_rank=[931, 913, 1173, 1080, 1115, 718, 1202, 1060] avg=1024.0000 max=1202 max_rank=6 min=718 min_rank=5 imbalance_ratio=1.173828 max_load_reduction=6990 max_load_reduction_pct=0.853271 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8250, 8134, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8250, 8134, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8250, 8134, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8250 expected_overflow=0 exact_recv_per_rank=[8192, 8250, 8134, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8134 expected_overflow=0 exact_recv_per_rank=[8192, 8250, 8134, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8250, 8134, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8250, 8134, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8250, 8134, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2688, 5632] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8134 actual_valid_recv=8192 observed_loss=-58 tokens_per_expert=[5760, 2432] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7680, 640] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8250 actual_valid_recv=8320 observed_loss=-70 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3968, 4352] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7936, 384] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1411, 271, 209, 882, 2074, 402, 1606, 1337] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[709, 243, 131, 984, 2024, 831, 1422, 1848] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1473, 210, 207, 839, 1905, 516, 1907, 1135] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1660, 650, 588, 633, 1792, 726, 1316, 827] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1661, 207, 343, 662, 1307, 539, 1672, 1801] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1415, 270, 171, 1322, 1872, 641, 930, 1571] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1207, 325, 132, 1773, 2208, 743, 1010, 794] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1309, 79, 453, 1127, 1879, 442, 1205, 1698] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[912, 0, 325, 262, 132, 873, 1773, 0, 1335, 0, 743, 295, 748, 0, 794, 0] rerouted_tokens_per_ep_rank=[912, 587, 1005, 1773, 1335, 1038, 748, 794] avg=1024.0000 max=1773 max_rank=3 min=587 min_rank=1 imbalance_ratio=1.731445 max_load_reduction=6419 max_load_reduction_pct=0.783569 moved_to_echo=3915 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1113, 0, 210, 495, 207, 753, 839, 0, 1152, 0, 516, 360, 1412, 0, 1135, 0] rerouted_tokens_per_ep_rank=[1113, 705, 960, 839, 1152, 876, 1412, 1135] avg=1024.0000 max=1412 max_rank=6 min=705 min_rank=1 imbalance_ratio=1.378906 max_load_reduction=6780 max_load_reduction_pct=0.827637 moved_to_echo=4575 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1255, 0, 207, 434, 343, 517, 662, 0, 790, 0, 539, 406, 1238, 0, 1801, 0] rerouted_tokens_per_ep_rank=[1255, 641, 860, 662, 790, 945, 1238, 1801] avg=1024.0000 max=1801 max_rank=7 min=641 min_rank=1 imbalance_ratio=1.758789 max_load_reduction=6391 max_load_reduction_pct=0.780151 moved_to_echo=4774 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1066, 0, 271, 417, 209, 820, 882, 0, 1254, 0, 402, 345, 1189, 0, 1337, 0] rerouted_tokens_per_ep_rank=[1066, 688, 1029, 882, 1254, 747, 1189, 1337] avg=1024.0000 max=1337 max_rank=7 min=688 min_rank=1 imbalance_ratio=1.305664 max_load_reduction=6855 max_load_reduction_pct=0.836792 moved_to_echo=4527 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1069, 0, 270, 241, 171, 740, 1322, 0, 1132, 0, 641, 346, 689, 0, 1571, 0] rerouted_tokens_per_ep_rank=[1069, 511, 911, 1322, 1132, 987, 689, 1571] avg=1024.0000 max=1571 max_rank=7 min=511 min_rank=1 imbalance_ratio=1.534180 max_load_reduction=6621 max_load_reduction_pct=0.808228 moved_to_echo=4379 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1252, 0, 650, 345, 588, 712, 633, 0, 1080, 0, 726, 408, 971, 0, 827, 0] rerouted_tokens_per_ep_rank=[1252, 995, 1300, 633, 1080, 1134, 971, 827] avg=1024.0000 max=1300 max_rank=2 min=633 min_rank=3 imbalance_ratio=1.269531 max_load_reduction=6892 max_load_reduction_pct=0.841309 moved_to_echo=4012 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[536, 0, 243, 369, 131, 800, 984, 0, 1224, 0, 831, 173, 1053, 0, 1848, 0] rerouted_tokens_per_ep_rank=[536, 612, 931, 984, 1224, 1004, 1053, 1848] avg=1024.0000 max=1848 max_rank=7 min=536 min_rank=0 imbalance_ratio=1.804688 max_load_reduction=6344 max_load_reduction_pct=0.774414 moved_to_echo=5129 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[989, 0, 79, 313, 453, 743, 1127, 0, 1136, 0, 442, 320, 892, 0, 1698, 0] rerouted_tokens_per_ep_rank=[989, 392, 1196, 1127, 1136, 762, 892, 1698] avg=1024.0000 max=1698 max_rank=7 min=392 min_rank=1 imbalance_ratio=1.658203 max_load_reduction=6494 max_load_reduction_pct=0.792725 moved_to_echo=4488 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5131, 8192, 8222, 9103, 7493, 8192, 11011] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5131 expected_overflow=0 exact_recv_per_rank=[8192, 5131, 8192, 8222, 9103, 7493, 8192, 11011] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7493 expected_overflow=0 exact_recv_per_rank=[8192, 5131, 8192, 8222, 9103, 7493, 8192, 11011] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5131, 8192, 8222, 9103, 7493, 8192, 11011] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8222 expected_overflow=0 exact_recv_per_rank=[8192, 5131, 8192, 8222, 9103, 7493, 8192, 11011] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5131, 8192, 8222, 9103, 7493, 8192, 11011] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9103 expected_overflow=0 exact_recv_per_rank=[8192, 5131, 8192, 8222, 9103, 7493, 8192, 11011] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11011 expected_overflow=0 exact_recv_per_rank=[8192, 5131, 8192, 8222, 9103, 7493, 8192, 11011] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=5131 actual_valid_recv=5248 observed_loss=-117 tokens_per_expert=[2304, 2944] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7493 actual_valid_recv=7552 observed_loss=-59 tokens_per_expert=[4864, 2688] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8222 actual_valid_recv=8320 observed_loss=-98 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=9103 actual_valid_recv=9216 observed_loss=-113 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2304, 6016] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=11011 actual_valid_recv=11136 observed_loss=-125 tokens_per_expert=[11136, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=5248 slack_slots=11136 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=11136 slack_slots=5248 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1897, 212, 2974, 1227, 168, 552, 789, 373] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1417, 19, 569, 2712, 1672, 578, 794, 431] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[2236, 161, 2176, 655, 221, 958, 649, 1136] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[2573, 15, 768, 2184, 115, 1672, 788, 77] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[719, 18, 1558, 2390, 72, 2241, 866, 328] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[591, 187, 263, 2398, 392, 2688, 1309, 364] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[927, 74, 138, 2480, 512, 3022, 605, 434] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[2113, 484, 1307, 1301, 522, 724, 1201, 540] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1246, 0, 212, 561, 2611, 0, 666, 0, 168, 651, 369, 0, 789, 363, 373, 183] rerouted_tokens_per_ep_rank=[1246, 773, 2611, 666, 819, 369, 1152, 556] avg=1024.0000 max=2611 max_rank=2 min=369 min_rank=5 imbalance_ratio=2.549805 max_load_reduction=5581 max_load_reduction_pct=0.681274 moved_to_echo=2896 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1469, 0, 161, 299, 1911, 0, 356, 0, 221, 767, 640, 0, 649, 265, 1136, 318] rerouted_tokens_per_ep_rank=[1469, 460, 1911, 356, 988, 640, 914, 1454] avg=1024.0000 max=1911 max_rank=2 min=356 min_rank=3 imbalance_ratio=1.866211 max_load_reduction=6281 max_load_reduction_pct=0.766724 moved_to_echo=3996 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[931, 0, 19, 1240, 500, 0, 1472, 0, 1672, 486, 386, 0, 794, 69, 431, 192] rerouted_tokens_per_ep_rank=[931, 1259, 500, 1472, 2158, 386, 863, 623] avg=1024.0000 max=2158 max_rank=4 min=386 min_rank=5 imbalance_ratio=2.107422 max_load_reduction=6034 max_load_reduction_pct=0.736572 moved_to_echo=4030 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1690, 0, 15, 999, 675, 0, 1185, 0, 115, 883, 1116, 0, 788, 93, 77, 556] rerouted_tokens_per_ep_rank=[1690, 1014, 675, 1185, 998, 1116, 881, 633] avg=1024.0000 max=1690 max_rank=0 min=633 min_rank=7 imbalance_ratio=1.650391 max_load_reduction=6502 max_load_reduction_pct=0.793701 moved_to_echo=3628 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[606, 0, 74, 1138, 118, 0, 1342, 0, 512, 321, 2011, 0, 605, 20, 434, 1011] rerouted_tokens_per_ep_rank=[606, 1212, 118, 1342, 833, 2011, 625, 1445] avg=1024.0000 max=2011 max_rank=5 min=118 min_rank=2 imbalance_ratio=1.963867 max_load_reduction=6181 max_load_reduction_pct=0.754517 moved_to_echo=4914 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1388, 0, 484, 595, 1148, 0, 706, 0, 522, 725, 484, 0, 1201, 159, 540, 240] rerouted_tokens_per_ep_rank=[1388, 1079, 1148, 706, 1247, 484, 1360, 780] avg=1024.0000 max=1388 max_rank=0 min=484 min_rank=5 imbalance_ratio=1.355469 max_load_reduction=6804 max_load_reduction_pct=0.830566 moved_to_echo=3871 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[389, 0, 187, 1097, 231, 0, 1301, 0, 392, 202, 1794, 0, 1309, 32, 364, 894] rerouted_tokens_per_ep_rank=[389, 1284, 231, 1301, 594, 1794, 1341, 1258] avg=1024.0000 max=1794 max_rank=5 min=231 min_rank=2 imbalance_ratio=1.751953 max_load_reduction=6398 max_load_reduction_pct=0.781006 moved_to_echo=4987 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[473, 0, 18, 1093, 1368, 0, 1297, 0, 72, 246, 1496, 0, 866, 190, 328, 745] rerouted_tokens_per_ep_rank=[473, 1111, 1368, 1297, 318, 1496, 1056, 1073] avg=1024.0000 max=1496 max_rank=5 min=318 min_rank=4 imbalance_ratio=1.460938 max_load_reduction=6696 max_load_reduction_pct=0.817383 moved_to_echo=3943 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8562 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8562, 8325, 7955, 8296, 8192, 7822] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7955 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8562, 8325, 7955, 8296, 8192, 7822] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8562, 8325, 7955, 8296, 8192, 7822] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8296 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8562, 8325, 7955, 8296, 8192, 7822] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8325 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8562, 8325, 7955, 8296, 8192, 7822] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7822 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8562, 8325, 7955, 8296, 8192, 7822] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8562, 8325, 7955, 8296, 8192, 7822] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8562, 8325, 7955, 8296, 8192, 7822] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1280, 7040] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8325 actual_valid_recv=8448 observed_loss=-123 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8562 actual_valid_recv=8576 observed_loss=-14 tokens_per_expert=[8576, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7040, 1280] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=7822 actual_valid_recv=7936 observed_loss=-114 tokens_per_expert=[3712, 4224] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7955 actual_valid_recv=8064 observed_loss=-109 tokens_per_expert=[3712, 4352] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8296 actual_valid_recv=8320 observed_loss=-24 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8576 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=7936 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8064 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[107, 3517, 101, 1151, 1345, 568, 775, 628] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1739, 2298, 981, 411, 507, 611, 1342, 303] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1665, 2958, 64, 559, 704, 1023, 935, 284] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[264, 2200, 201, 949, 1967, 1133, 1454, 24] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2931, 3115, 614, 315, 390, 171, 421, 235] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[551, 2183, 545, 302, 846, 236, 1985, 1544] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1054, 3153, 365, 2031, 898, 155, 237, 299] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[128, 3151, 496, 356, 1354, 1183, 509, 1015] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[105, 0, 1280, 0, 101, 751, 1151, 329, 1345, 2, 568, 484, 775, 72, 628, 601] rerouted_tokens_per_ep_rank=[105, 1280, 852, 1480, 1347, 1052, 847, 1229] avg=1024.0000 max=1480 max_rank=3 min=105 min_rank=0 imbalance_ratio=1.445312 max_load_reduction=6712 max_load_reduction_pct=0.819336 moved_to_echo=4475 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1702, 0, 837, 0, 981, 491, 411, 215, 507, 37, 611, 316, 1342, 47, 303, 392] rerouted_tokens_per_ep_rank=[1702, 837, 1472, 626, 544, 927, 1389, 695] avg=1024.0000 max=1702 max_rank=0 min=544 min_rank=4 imbalance_ratio=1.662109 max_load_reduction=6490 max_load_reduction_pct=0.792236 moved_to_echo=3555 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[126, 0, 1146, 0, 496, 673, 356, 295, 1354, 2, 1183, 434, 509, 65, 1015, 538] rerouted_tokens_per_ep_rank=[126, 1146, 1169, 651, 1356, 1617, 574, 1553] avg=1024.0000 max=1617 max_rank=5 min=126 min_rank=0 imbalance_ratio=1.579102 max_load_reduction=6575 max_load_reduction_pct=0.802612 moved_to_echo=5100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2869, 0, 1133, 0, 614, 665, 315, 292, 390, 62, 171, 429, 421, 64, 235, 532] rerouted_tokens_per_ep_rank=[2869, 1133, 1279, 607, 452, 600, 485, 767] avg=1024.0000 max=2869 max_rank=0 min=452 min_rank=4 imbalance_ratio=2.801758 max_load_reduction=5323 max_load_reduction_pct=0.649780 moved_to_echo=2304 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1032, 0, 1147, 0, 365, 673, 2031, 295, 898, 22, 155, 434, 237, 65, 299, 539] rerouted_tokens_per_ep_rank=[1032, 1147, 1038, 2326, 920, 589, 302, 838] avg=1024.0000 max=2326 max_rank=3 min=302 min_rank=6 imbalance_ratio=2.271484 max_load_reduction=5866 max_load_reduction_pct=0.716064 moved_to_echo=2649 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[535, 0, 773, 0, 545, 470, 302, 209, 846, 16, 236, 305, 1985, 49, 1544, 377] rerouted_tokens_per_ep_rank=[535, 773, 1015, 511, 862, 541, 2034, 1921] avg=1024.0000 max=2034 max_rank=6 min=511 min_rank=3 imbalance_ratio=1.986328 max_load_reduction=6158 max_load_reduction_pct=0.751709 moved_to_echo=5358 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[259, 0, 800, 0, 201, 470, 949, 206, 1967, 5, 1133, 303, 1454, 45, 24, 376] rerouted_tokens_per_ep_rank=[259, 800, 671, 1155, 1972, 1436, 1499, 400] avg=1024.0000 max=1972 max_rank=4 min=259 min_rank=0 imbalance_ratio=1.925781 max_load_reduction=6220 max_load_reduction_pct=0.759277 moved_to_echo=5307 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1630, 0, 1076, 0, 64, 632, 559, 277, 704, 35, 1023, 407, 935, 61, 284, 505] rerouted_tokens_per_ep_rank=[1630, 1076, 696, 836, 739, 1430, 996, 789] avg=1024.0000 max=1630 max_rank=0 min=696 min_rank=2 imbalance_ratio=1.591797 max_load_reduction=6562 max_load_reduction_pct=0.801025 moved_to_echo=3954 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8258, 8192, 8192, 8192, 8192, 8192, 8126, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8258, 8192, 8192, 8192, 8192, 8192, 8126, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8258, 8192, 8192, 8192, 8192, 8192, 8126, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8258 expected_overflow=0 exact_recv_per_rank=[8258, 8192, 8192, 8192, 8192, 8192, 8126, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8258, 8192, 8192, 8192, 8192, 8192, 8126, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8258, 8192, 8192, 8192, 8192, 8192, 8126, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8126 expected_overflow=0 exact_recv_per_rank=[8258, 8192, 8192, 8192, 8192, 8192, 8126, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8258, 8192, 8192, 8192, 8192, 8192, 8126, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5120, 3200] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[8064, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6144, 2176] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4352, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3456, 4864] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8258 actual_valid_recv=8320 observed_loss=-62 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8126 actual_valid_recv=8192 observed_loss=-66 tokens_per_expert=[7680, 512] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[574, 2944, 768, 999, 1034, 279, 114, 1480] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1826, 857, 1503, 552, 1530, 782, 567, 575] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[302, 2110, 29, 3141, 1726, 433, 226, 225] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[2056, 277, 298, 436, 1243, 704, 2283, 895] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[654, 1276, 379, 2511, 1184, 1594, 373, 221] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[2165, 2745, 35, 2219, 267, 242, 355, 164] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[3094, 982, 751, 256, 1807, 1050, 244, 8] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1610, 1059, 537, 2038, 2041, 53, 103, 751] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1473, 0, 1910, 0, 35, 835, 1572, 0, 202, 0, 242, 65, 355, 692, 164, 647] rerouted_tokens_per_ep_rank=[1473, 1910, 870, 1572, 202, 307, 1047, 811] avg=1024.0000 max=1910 max_rank=1 min=202 min_rank=4 imbalance_ratio=1.865234 max_load_reduction=6282 max_load_reduction_pct=0.766846 moved_to_echo=2367 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1243, 0, 597, 0, 1503, 260, 391, 0, 1158, 0, 782, 372, 567, 583, 575, 161] rerouted_tokens_per_ep_rank=[1243, 597, 1763, 391, 1158, 1154, 1150, 736] avg=1024.0000 max=1763 max_rank=2 min=391 min_rank=3 imbalance_ratio=1.721680 max_load_reduction=6429 max_load_reduction_pct=0.784790 moved_to_echo=4198 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[391, 0, 2048, 0, 768, 896, 708, 0, 782, 0, 279, 252, 114, 183, 1480, 291] rerouted_tokens_per_ep_rank=[391, 2048, 1664, 708, 782, 531, 297, 1771] avg=1024.0000 max=2048 max_rank=1 min=297 min_rank=6 imbalance_ratio=2.000000 max_load_reduction=6144 max_load_reduction_pct=0.750000 moved_to_echo=3381 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2105, 0, 683, 0, 751, 299, 182, 0, 1367, 0, 1050, 440, 244, 989, 8, 74] rerouted_tokens_per_ep_rank=[2105, 683, 1050, 182, 1367, 1490, 1233, 82] avg=1024.0000 max=2105 max_rank=0 min=82 min_rank=7 imbalance_ratio=2.055664 max_load_reduction=6087 max_load_reduction_pct=0.743042 moved_to_echo=4172 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1399, 0, 193, 0, 298, 84, 309, 0, 941, 0, 704, 302, 2283, 657, 895, 127] rerouted_tokens_per_ep_rank=[1399, 193, 382, 309, 941, 1006, 2940, 1022] avg=1024.0000 max=2940 max_rank=6 min=193 min_rank=1 imbalance_ratio=2.871094 max_load_reduction=5252 max_load_reduction_pct=0.641113 moved_to_echo=5909 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[206, 0, 1468, 0, 29, 642, 2225, 0, 1306, 0, 433, 420, 226, 96, 225, 916] rerouted_tokens_per_ep_rank=[206, 1468, 671, 2225, 1306, 853, 322, 1141] avg=1024.0000 max=2225 max_rank=3 min=206 min_rank=0 imbalance_ratio=2.172852 max_load_reduction=5967 max_load_reduction_pct=0.728394 moved_to_echo=3622 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[441, 0, 884, 0, 379, 392, 1776, 0, 892, 0, 1594, 292, 373, 213, 221, 735] rerouted_tokens_per_ep_rank=[441, 884, 771, 1776, 892, 1886, 586, 956] avg=1024.0000 max=1886 max_rank=5 min=441 min_rank=0 imbalance_ratio=1.841797 max_load_reduction=6306 max_load_reduction_pct=0.769775 moved_to_echo=4320 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1096, 0, 737, 0, 537, 322, 1444, 0, 1544, 0, 53, 497, 103, 514, 751, 594] rerouted_tokens_per_ep_rank=[1096, 737, 859, 1444, 1544, 550, 617, 1345] avg=1024.0000 max=1544 max_rank=4 min=550 min_rank=5 imbalance_ratio=1.507812 max_load_reduction=6648 max_load_reduction_pct=0.811523 moved_to_echo=4056 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8354, 8520, 8030, 8607, 8192, 7777, 8192, 7864] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8607 expected_overflow=0 exact_recv_per_rank=[8354, 8520, 8030, 8607, 8192, 7777, 8192, 7864] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7777 expected_overflow=0 exact_recv_per_rank=[8354, 8520, 8030, 8607, 8192, 7777, 8192, 7864] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7864 expected_overflow=0 exact_recv_per_rank=[8354, 8520, 8030, 8607, 8192, 7777, 8192, 7864] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8520 expected_overflow=0 exact_recv_per_rank=[8354, 8520, 8030, 8607, 8192, 7777, 8192, 7864] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8354, 8520, 8030, 8607, 8192, 7777, 8192, 7864] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8030 expected_overflow=0 exact_recv_per_rank=[8354, 8520, 8030, 8607, 8192, 7777, 8192, 7864] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8354 expected_overflow=0 exact_recv_per_rank=[8354, 8520, 8030, 8607, 8192, 7777, 8192, 7864] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=7864 actual_valid_recv=7936 observed_loss=-72 tokens_per_expert=[4352, 3584] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8030 actual_valid_recv=8192 observed_loss=-162 tokens_per_expert=[4352, 3840] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4352, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7777 actual_valid_recv=7936 observed_loss=-159 tokens_per_expert=[5248, 2688] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8354 actual_valid_recv=8448 observed_loss=-94 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8607 actual_valid_recv=8704 observed_loss=-97 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8520 actual_valid_recv=8576 observed_loss=-56 tokens_per_expert=[8576, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=7936 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7936 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8576 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[748, 576, 972, 1130, 1500, 710, 78, 2478] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[53, 341, 152, 129, 1817, 2662, 1155, 1883] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1557, 47, 1821, 448, 1084, 814, 859, 1562] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[100, 1841, 36, 343, 1674, 434, 155, 3609] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[188, 678, 61, 2644, 2955, 216, 980, 470] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[335, 2906, 143, 603, 1628, 260, 1358, 959] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1902, 966, 613, 180, 2333, 893, 1244, 61] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1329, 2509, 269, 2795, 941, 89, 181, 79] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[335, 492, 2414, 0, 143, 482, 603, 0, 958, 0, 260, 182, 1358, 188, 777, 0] rerouted_tokens_per_ep_rank=[827, 2414, 625, 603, 958, 442, 1546, 777] avg=1024.0000 max=2414 max_rank=1 min=442 min_rank=5 imbalance_ratio=2.357422 max_load_reduction=5778 max_load_reduction_pct=0.705322 moved_to_echo=3723 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[100, 312, 1529, 0, 36, 495, 343, 0, 985, 0, 434, 687, 155, 194, 2922, 0] rerouted_tokens_per_ep_rank=[412, 1529, 531, 343, 985, 1121, 349, 2922] avg=1024.0000 max=2922 max_rank=7 min=343 min_rank=3 imbalance_ratio=2.853516 max_load_reduction=5270 max_load_reduction_pct=0.643311 moved_to_echo=5377 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1557, 7, 40, 0, 1821, 320, 448, 0, 639, 0, 814, 297, 859, 125, 1265, 0] rerouted_tokens_per_ep_rank=[1564, 40, 2141, 448, 639, 1111, 984, 1265] avg=1024.0000 max=2141 max_rank=2 min=40 min_rank=1 imbalance_ratio=2.090820 max_load_reduction=6051 max_load_reduction_pct=0.738647 moved_to_echo=3999 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[53, 57, 284, 0, 152, 537, 129, 0, 1070, 0, 2662, 358, 1155, 210, 1525, 0] rerouted_tokens_per_ep_rank=[110, 284, 689, 129, 1070, 3020, 1365, 1525] avg=1024.0000 max=3020 max_rank=5 min=110 min_rank=0 imbalance_ratio=2.949219 max_load_reduction=5172 max_load_reduction_pct=0.631348 moved_to_echo=6980 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[748, 97, 479, 0, 972, 444, 1130, 0, 883, 0, 710, 471, 78, 173, 2007, 0] rerouted_tokens_per_ep_rank=[845, 479, 1416, 1130, 883, 1181, 251, 2007] avg=1024.0000 max=2007 max_rank=7 min=251 min_rank=6 imbalance_ratio=1.959961 max_load_reduction=6185 max_load_reduction_pct=0.755005 moved_to_echo=4322 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1329, 430, 2079, 0, 269, 283, 2795, 0, 545, 0, 89, 19, 181, 113, 60, 0] rerouted_tokens_per_ep_rank=[1759, 2079, 552, 2795, 545, 108, 294, 60] avg=1024.0000 max=2795 max_rank=3 min=60 min_rank=7 imbalance_ratio=2.729492 max_load_reduction=5397 max_load_reduction_pct=0.658813 moved_to_echo=1007 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1902, 163, 803, 0, 613, 690, 180, 0, 1373, 0, 893, 11, 1244, 270, 50, 0] rerouted_tokens_per_ep_rank=[2065, 803, 1303, 180, 1373, 904, 1514, 50] avg=1024.0000 max=2065 max_rank=0 min=50 min_rank=7 imbalance_ratio=2.016602 max_load_reduction=6127 max_load_reduction_pct=0.747925 moved_to_echo=3841 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[188, 114, 564, 0, 61, 874, 2644, 0, 1739, 0, 216, 89, 980, 342, 381, 0] rerouted_tokens_per_ep_rank=[302, 564, 935, 2644, 1739, 305, 1322, 381] avg=1024.0000 max=2644 max_rank=3 min=302 min_rank=0 imbalance_ratio=2.582031 max_load_reduction=5548 max_load_reduction_pct=0.677246 moved_to_echo=3747 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7884, 8192, 8192, 8272, 8192, 8192, 7625, 8987] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7884 expected_overflow=0 exact_recv_per_rank=[7884, 8192, 8192, 8272, 8192, 8192, 7625, 8987] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7884, 8192, 8192, 8272, 8192, 8192, 7625, 8987] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8272 expected_overflow=0 exact_recv_per_rank=[7884, 8192, 8192, 8272, 8192, 8192, 7625, 8987] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7884, 8192, 8192, 8272, 8192, 8192, 7625, 8987] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8987 expected_overflow=0 exact_recv_per_rank=[7884, 8192, 8192, 8272, 8192, 8192, 7625, 8987] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7625 expected_overflow=0 exact_recv_per_rank=[7884, 8192, 8192, 8272, 8192, 8192, 7625, 8987] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7884, 8192, 8192, 8272, 8192, 8192, 7625, 8987] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=7884 actual_valid_recv=8064 observed_loss=-180 tokens_per_expert=[6272, 1792] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7625 actual_valid_recv=7680 observed_loss=-55 tokens_per_expert=[6016, 1664] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4096, 4224] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6144, 2176] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8272 actual_valid_recv=8320 observed_loss=-48 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8987 actual_valid_recv=9088 observed_loss=-101 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8064 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[3959, 279, 1050, 93, 417, 402, 1555, 437] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[987, 163, 1496, 1405, 5, 1536, 1262, 1338] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1023, 19, 23, 184, 0, 2561, 1780, 2602] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[2985, 158, 755, 102, 66, 1931, 1676, 519] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[487, 59, 1830, 36, 54, 2084, 2057, 1585] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1803, 134, 623, 176, 6, 34, 2940, 2476] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3520, 10, 1133, 3, 21, 396, 2280, 829] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[387, 3, 2, 361, 25, 1843, 2025, 3546] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[271, 0, 59, 216, 1830, 247, 36, 537, 54, 975, 1837, 0, 1082, 0, 1048, 0] rerouted_tokens_per_ep_rank=[271, 275, 2077, 573, 1029, 1837, 1082, 1048] avg=1024.0000 max=2077 max_rank=2 min=271 min_rank=0 imbalance_ratio=2.028320 max_load_reduction=6115 max_load_reduction_pct=0.746460 moved_to_echo=4996 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[568, 0, 19, 455, 23, 303, 184, 881, 0, 843, 2258, 0, 937, 0, 1721, 0] rerouted_tokens_per_ep_rank=[568, 474, 326, 1065, 843, 2258, 937, 1721] avg=1024.0000 max=2258 max_rank=5 min=326 min_rank=2 imbalance_ratio=2.205078 max_load_reduction=5934 max_load_reduction_pct=0.724365 moved_to_echo=5759 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[548, 0, 163, 439, 1496, 182, 1405, 453, 5, 598, 1354, 0, 664, 0, 885, 0] rerouted_tokens_per_ep_rank=[548, 602, 1678, 1858, 603, 1354, 664, 885] avg=1024.0000 max=1858 max_rank=3 min=548 min_rank=0 imbalance_ratio=1.814453 max_load_reduction=6334 max_load_reduction_pct=0.773193 moved_to_echo=3506 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1657, 0, 158, 1328, 755, 229, 102, 175, 66, 794, 1702, 0, 882, 0, 344, 0] rerouted_tokens_per_ep_rank=[1657, 1486, 984, 277, 860, 1702, 882, 344] avg=1024.0000 max=1702 max_rank=5 min=277 min_rank=3 imbalance_ratio=1.662109 max_load_reduction=6490 max_load_reduction_pct=0.792236 moved_to_echo=3788 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2193, 0, 279, 1766, 1050, 51, 93, 152, 417, 741, 351, 0, 814, 0, 285, 0] rerouted_tokens_per_ep_rank=[2193, 2045, 1101, 245, 1158, 351, 814, 285] avg=1024.0000 max=2193 max_rank=0 min=245 min_rank=3 imbalance_ratio=2.141602 max_load_reduction=5999 max_load_reduction_pct=0.732300 moved_to_echo=2608 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1954, 0, 10, 1566, 1133, 46, 3, 280, 21, 1080, 350, 0, 1200, 0, 549, 0] rerouted_tokens_per_ep_rank=[1954, 1576, 1179, 283, 1101, 350, 1200, 549] avg=1024.0000 max=1954 max_rank=0 min=283 min_rank=3 imbalance_ratio=1.908203 max_load_reduction=6238 max_load_reduction_pct=0.761475 moved_to_echo=3200 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1001, 0, 134, 802, 623, 4, 176, 838, 6, 1393, 30, 0, 1547, 0, 1638, 0] rerouted_tokens_per_ep_rank=[1001, 936, 627, 1014, 1399, 30, 1547, 1638] avg=1024.0000 max=1638 max_rank=7 min=30 min_rank=5 imbalance_ratio=1.599609 max_load_reduction=6554 max_load_reduction_pct=0.800049 moved_to_echo=4614 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[215, 0, 3, 172, 2, 218, 361, 1201, 25, 959, 1625, 0, 1066, 0, 2345, 0] rerouted_tokens_per_ep_rank=[215, 175, 220, 1562, 984, 1625, 1066, 2345] avg=1024.0000 max=2345 max_rank=7 min=175 min_rank=1 imbalance_ratio=2.290039 max_load_reduction=5847 max_load_reduction_pct=0.713745 moved_to_echo=6020 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9507 expected_overflow=0 exact_recv_per_rank=[8407, 7569, 8192, 6877, 7977, 9507, 8192, 8815] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7569 expected_overflow=0 exact_recv_per_rank=[8407, 7569, 8192, 6877, 7977, 9507, 8192, 8815] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8815 expected_overflow=0 exact_recv_per_rank=[8407, 7569, 8192, 6877, 7977, 9507, 8192, 8815] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8407, 7569, 8192, 6877, 7977, 9507, 8192, 8815] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8407 expected_overflow=0 exact_recv_per_rank=[8407, 7569, 8192, 6877, 7977, 9507, 8192, 8815] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6877 expected_overflow=0 exact_recv_per_rank=[8407, 7569, 8192, 6877, 7977, 9507, 8192, 8815] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8407, 7569, 8192, 6877, 7977, 9507, 8192, 8815] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7977 expected_overflow=0 exact_recv_per_rank=[8407, 7569, 8192, 6877, 7977, 9507, 8192, 8815] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=6877 actual_valid_recv=7040 observed_loss=-163 tokens_per_expert=[2432, 4608] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=7569 actual_valid_recv=7680 observed_loss=-111 tokens_per_expert=[896, 6784] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7977 actual_valid_recv=8064 observed_loss=-87 tokens_per_expert=[640, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8815 actual_valid_recv=8832 observed_loss=-17 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8407 actual_valid_recv=8448 observed_loss=-41 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[6912, 1280] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=9507 actual_valid_recv=9600 observed_loss=-93 tokens_per_expert=[9600, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7040 slack_slots=9344 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8064 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=9600 slack_slots=6784 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3237, 1027, 1778, 122, 892, 28, 0, 1108] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1340, 0, 2212, 365, 127, 1900, 3, 2245] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1461, 160, 930, 2484, 144, 479, 368, 2166] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[413, 270, 1890, 3133, 556, 686, 23, 1221] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[691, 1956, 919, 63, 588, 468, 89, 3418] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1964, 465, 2985, 678, 820, 452, 14, 814] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[245, 2320, 948, 563, 1141, 32, 1, 2942] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[87, 334, 2553, 866, 1225, 136, 7, 2984] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1269, 0, 160, 192, 559, 0, 2484, 0, 144, 176, 479, 195, 368, 985, 1181, 0] rerouted_tokens_per_ep_rank=[1269, 352, 559, 2484, 320, 674, 1353, 1181] avg=1024.0000 max=2484 max_rank=3 min=320 min_rank=4 imbalance_ratio=2.425781 max_load_reduction=5708 max_load_reduction_pct=0.696777 moved_to_echo=3528 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[359, 0, 270, 54, 1135, 0, 3133, 0, 556, 358, 686, 397, 23, 555, 666, 0] rerouted_tokens_per_ep_rank=[359, 324, 1135, 3133, 914, 1083, 578, 666] avg=1024.0000 max=3133 max_rank=3 min=324 min_rank=1 imbalance_ratio=3.059570 max_load_reduction=5059 max_load_reduction_pct=0.617554 moved_to_echo=3241 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2806, 0, 1027, 431, 1057, 0, 122, 0, 892, 343, 28, 378, 0, 507, 601, 0] rerouted_tokens_per_ep_rank=[2806, 1458, 1057, 122, 1235, 406, 507, 601] avg=1024.0000 max=2806 max_rank=0 min=122 min_rank=3 imbalance_ratio=2.740234 max_load_reduction=5386 max_load_reduction_pct=0.657471 moved_to_echo=2749 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[213, 0, 2320, 32, 570, 0, 563, 0, 1141, 179, 32, 199, 1, 1338, 1604, 0] rerouted_tokens_per_ep_rank=[213, 2352, 570, 563, 1320, 231, 1339, 1604] avg=1024.0000 max=2352 max_rank=1 min=213 min_rank=0 imbalance_ratio=2.296875 max_load_reduction=5840 max_load_reduction_pct=0.712891 moved_to_echo=4494 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[600, 0, 1956, 91, 552, 0, 63, 0, 588, 174, 468, 193, 89, 1554, 1864, 0] rerouted_tokens_per_ep_rank=[600, 2047, 552, 63, 762, 661, 1643, 1864] avg=1024.0000 max=2047 max_rank=1 min=63 min_rank=3 imbalance_ratio=1.999023 max_load_reduction=6145 max_load_reduction_pct=0.750122 moved_to_echo=4930 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1164, 0, 0, 176, 1328, 0, 365, 0, 127, 419, 1900, 465, 3, 1021, 1224, 0] rerouted_tokens_per_ep_rank=[1164, 176, 1328, 365, 546, 2365, 1024, 1224] avg=1024.0000 max=2365 max_rank=5 min=176 min_rank=1 imbalance_ratio=2.309570 max_load_reduction=5827 max_load_reduction_pct=0.711304 moved_to_echo=5159 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1705, 0, 465, 259, 1791, 0, 678, 0, 820, 566, 452, 628, 14, 370, 444, 0] rerouted_tokens_per_ep_rank=[1705, 724, 1791, 678, 1386, 1080, 384, 444] avg=1024.0000 max=1791 max_rank=2 min=384 min_rank=6 imbalance_ratio=1.749023 max_load_reduction=6401 max_load_reduction_pct=0.781372 moved_to_echo=3294 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[76, 0, 334, 11, 1532, 0, 866, 0, 1225, 484, 136, 537, 7, 1357, 1627, 0] rerouted_tokens_per_ep_rank=[76, 345, 1532, 866, 1709, 673, 1364, 1627] avg=1024.0000 max=1709 max_rank=4 min=76 min_rank=0 imbalance_ratio=1.668945 max_load_reduction=6483 max_load_reduction_pct=0.791382 moved_to_echo=5373 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8274 expected_overflow=0 exact_recv_per_rank=[8192, 7778, 8524, 8274, 8192, 7173, 8192, 9211] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9211 expected_overflow=0 exact_recv_per_rank=[8192, 7778, 8524, 8274, 8192, 7173, 8192, 9211] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7173 expected_overflow=0 exact_recv_per_rank=[8192, 7778, 8524, 8274, 8192, 7173, 8192, 9211] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7778, 8524, 8274, 8192, 7173, 8192, 9211] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7778, 8524, 8274, 8192, 7173, 8192, 9211] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7778 expected_overflow=0 exact_recv_per_rank=[8192, 7778, 8524, 8274, 8192, 7173, 8192, 9211] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7778, 8524, 8274, 8192, 7173, 8192, 9211] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8524 expected_overflow=0 exact_recv_per_rank=[8192, 7778, 8524, 8274, 8192, 7173, 8192, 9211] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7173 actual_valid_recv=7296 observed_loss=-123 tokens_per_expert=[4224, 3072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=7778 actual_valid_recv=7936 observed_loss=-158 tokens_per_expert=[6656, 1280] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8524 actual_valid_recv=8576 observed_loss=-52 tokens_per_expert=[8576, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5504, 2816] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=9211 actual_valid_recv=9216 observed_loss=-5 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8274 actual_valid_recv=8320 observed_loss=-46 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7296 slack_slots=9088 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=7936 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8576 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[2363, 2601, 17, 1060, 116, 508, 24, 1503] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[484, 3614, 15, 3088, 51, 46, 736, 158] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[273, 513, 254, 3446, 856, 142, 2702, 6] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1400, 2530, 1222, 2838, 5, 3, 79, 115] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4, 960, 703, 191, 197, 1710, 2205, 2222] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1295, 268, 355, 1149, 180, 690, 2989, 1266] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1711, 65, 69, 3381, 987, 498, 558, 923] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[2699, 667, 750, 1728, 317, 1648, 296, 87] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2012, 0, 2601, 0, 17, 201, 515, 0, 116, 344, 508, 351, 21, 0, 1503, 3] rerouted_tokens_per_ep_rank=[2012, 2601, 218, 515, 460, 859, 21, 1506] avg=1024.0000 max=2601 max_rank=1 min=21 min_rank=6 imbalance_ratio=2.540039 max_load_reduction=5591 max_load_reduction_pct=0.682495 moved_to_echo=2846 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2298, 0, 667, 0, 750, 328, 839, 0, 317, 561, 1648, 401, 253, 0, 87, 43] rerouted_tokens_per_ep_rank=[2298, 667, 1078, 839, 878, 2049, 253, 130] avg=1024.0000 max=2298 max_rank=0 min=130 min_rank=7 imbalance_ratio=2.244141 max_load_reduction=5894 max_load_reduction_pct=0.719482 moved_to_echo=3310 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1103, 0, 268, 0, 355, 218, 558, 0, 180, 373, 690, 192, 2554, 0, 1266, 435] rerouted_tokens_per_ep_rank=[1103, 268, 573, 558, 553, 882, 2554, 1701] avg=1024.0000 max=2554 max_rank=6 min=268 min_rank=1 imbalance_ratio=2.494141 max_load_reduction=5638 max_load_reduction_pct=0.688232 moved_to_echo=5690 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[233, 0, 513, 0, 254, 654, 1673, 0, 856, 1119, 142, 40, 2309, 0, 6, 393] rerouted_tokens_per_ep_rank=[233, 513, 908, 1673, 1975, 182, 2309, 399] avg=1024.0000 max=2309 max_rank=6 min=182 min_rank=5 imbalance_ratio=2.254883 max_load_reduction=5883 max_load_reduction_pct=0.718140 moved_to_echo=4865 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[412, 0, 3614, 0, 15, 586, 1500, 0, 51, 1002, 46, 72, 629, 0, 158, 107] rerouted_tokens_per_ep_rank=[412, 3614, 601, 1500, 1053, 118, 629, 265] avg=1024.0000 max=3614 max_rank=1 min=118 min_rank=5 imbalance_ratio=3.529297 max_load_reduction=4578 max_load_reduction_pct=0.558838 moved_to_echo=2065 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 960, 0, 703, 39, 87, 0, 197, 65, 1710, 4, 1881, 0, 2222, 324] rerouted_tokens_per_ep_rank=[0, 960, 742, 87, 262, 1714, 1881, 2546] avg=1024.0000 max=2546 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.486328 max_load_reduction=5646 max_load_reduction_pct=0.689209 moved_to_echo=6403 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1192, 0, 2530, 0, 1222, 538, 1379, 0, 5, 921, 3, 208, 68, 0, 115, 11] rerouted_tokens_per_ep_rank=[1192, 2530, 1760, 1379, 926, 211, 68, 126] avg=1024.0000 max=2530 max_rank=1 min=68 min_rank=6 imbalance_ratio=2.470703 max_load_reduction=5662 max_load_reduction_pct=0.691162 moved_to_echo=1331 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1457, 0, 65, 0, 69, 642, 1641, 0, 987, 1098, 498, 254, 477, 0, 923, 81] rerouted_tokens_per_ep_rank=[1457, 65, 711, 1641, 2085, 752, 477, 1004] avg=1024.0000 max=2085 max_rank=4 min=65 min_rank=1 imbalance_ratio=2.036133 max_load_reduction=6107 max_load_reduction_pct=0.745483 moved_to_echo=4318 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6767 expected_overflow=0 exact_recv_per_rank=[8707, 11218, 6591, 8192, 8192, 6767, 8192, 7677] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8707, 11218, 6591, 8192, 8192, 6767, 8192, 7677] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8707 expected_overflow=0 exact_recv_per_rank=[8707, 11218, 6591, 8192, 8192, 6767, 8192, 7677] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8707, 11218, 6591, 8192, 8192, 6767, 8192, 7677] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6591 expected_overflow=0 exact_recv_per_rank=[8707, 11218, 6591, 8192, 8192, 6767, 8192, 7677] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11218 expected_overflow=0 exact_recv_per_rank=[8707, 11218, 6591, 8192, 8192, 6767, 8192, 7677] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7677 expected_overflow=0 exact_recv_per_rank=[8707, 11218, 6591, 8192, 8192, 6767, 8192, 7677] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8707, 11218, 6591, 8192, 8192, 6767, 8192, 7677] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=6767 actual_valid_recv=6784 observed_loss=-17 tokens_per_expert=[5248, 1536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=6591 actual_valid_recv=6784 observed_loss=-193 tokens_per_expert=[3456, 3328] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2816, 5504] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=7677 actual_valid_recv=7808 observed_loss=-131 tokens_per_expert=[6400, 1408] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8707 actual_valid_recv=8832 observed_loss=-125 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=11218 actual_valid_recv=11264 observed_loss=-46 tokens_per_expert=[11264, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=6784 slack_slots=9600 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=6784 slack_slots=9600 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=11264 slack_slots=5120 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1486, 201, 115, 714, 138, 2775, 93, 2670] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1479, 576, 66, 1148, 685, 2590, 1586, 62] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1926, 2772, 2, 337, 56, 85, 2904, 110] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[468, 3975, 90, 570, 266, 1318, 1467, 38] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1082, 2275, 138, 1014, 541, 7, 301, 2834] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[793, 3614, 8, 177, 33, 121, 3364, 82] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[2901, 1726, 187, 377, 840, 2091, 6, 64] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[222, 3410, 118, 495, 535, 682, 1464, 1266] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1176, 0, 90, 0, 115, 80, 714, 310, 138, 31, 2470, 0, 93, 0, 2670, 305] rerouted_tokens_per_ep_rank=[1176, 90, 195, 1024, 169, 2470, 93, 2975] avg=1024.0000 max=2975 max_rank=7 min=90 min_rank=1 imbalance_ratio=2.905273 max_load_reduction=5217 max_load_reduction_pct=0.636841 moved_to_echo=5707 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1170, 0, 256, 0, 66, 231, 1148, 309, 685, 89, 2305, 0, 1586, 0, 62, 285] rerouted_tokens_per_ep_rank=[1170, 256, 297, 1457, 774, 2305, 1586, 347] avg=1024.0000 max=2305 max_rank=5 min=256 min_rank=1 imbalance_ratio=2.250977 max_load_reduction=5887 max_load_reduction_pct=0.718628 moved_to_echo=5012 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1524, 0, 1225, 0, 2, 1116, 337, 402, 56, 431, 76, 0, 2904, 0, 110, 9] rerouted_tokens_per_ep_rank=[1524, 1225, 1118, 739, 487, 76, 2904, 119] avg=1024.0000 max=2904 max_rank=6 min=76 min_rank=5 imbalance_ratio=2.835938 max_load_reduction=5288 max_load_reduction_pct=0.645508 moved_to_echo=3586 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[628, 0, 1597, 0, 8, 1455, 177, 165, 33, 562, 108, 0, 3364, 0, 82, 13] rerouted_tokens_per_ep_rank=[628, 1597, 1463, 342, 595, 108, 3364, 95] avg=1024.0000 max=3364 max_rank=6 min=95 min_rank=7 imbalance_ratio=3.285156 max_load_reduction=4828 max_load_reduction_pct=0.589355 moved_to_echo=4162 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[176, 0, 1507, 0, 118, 1372, 495, 46, 535, 531, 607, 0, 1464, 0, 1266, 75] rerouted_tokens_per_ep_rank=[176, 1507, 1490, 541, 1066, 607, 1464, 1341] avg=1024.0000 max=1507 max_rank=1 min=176 min_rank=0 imbalance_ratio=1.471680 max_load_reduction=6685 max_load_reduction_pct=0.816040 moved_to_echo=4478 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[367, 0, 1747, 0, 90, 1605, 570, 101, 266, 623, 1169, 0, 1467, 0, 38, 149] rerouted_tokens_per_ep_rank=[367, 1747, 1695, 671, 889, 1169, 1467, 187] avg=1024.0000 max=1747 max_rank=1 min=187 min_rank=7 imbalance_ratio=1.706055 max_load_reduction=6445 max_load_reduction_pct=0.786743 moved_to_echo=3712 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[856, 0, 1006, 0, 138, 915, 1014, 226, 541, 354, 7, 0, 301, 0, 2834, 0] rerouted_tokens_per_ep_rank=[856, 1006, 1053, 1240, 895, 7, 301, 2834] avg=1024.0000 max=2834 max_rank=7 min=7 min_rank=5 imbalance_ratio=2.767578 max_load_reduction=5358 max_load_reduction_pct=0.654053 moved_to_echo=4037 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2295, 0, 764, 0, 187, 694, 377, 606, 840, 268, 1861, 0, 6, 0, 64, 230] rerouted_tokens_per_ep_rank=[2295, 764, 881, 983, 1108, 1861, 6, 294] avg=1024.0000 max=2295 max_rank=0 min=6 min_rank=6 imbalance_ratio=2.241211 max_load_reduction=5897 max_load_reduction_pct=0.719849 moved_to_echo=3269 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8603 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6997, 5983, 8603, 11185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6997, 5983, 8603, 11185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11185 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6997, 5983, 8603, 11185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5983 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6997, 5983, 8603, 11185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6997, 5983, 8603, 11185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6997, 5983, 8603, 11185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6997, 5983, 8603, 11185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6997 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6997, 5983, 8603, 11185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=5983 actual_valid_recv=6144 observed_loss=-161 tokens_per_expert=[3200, 2944] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8603 actual_valid_recv=8704 observed_loss=-101 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=6997 actual_valid_recv=7040 observed_loss=-43 tokens_per_expert=[4864, 2176] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=11185 actual_valid_recv=11264 observed_loss=-79 tokens_per_expert=[11264, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7168, 1152] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=6144 slack_slots=10240 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7040 slack_slots=9344 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=11264 slack_slots=5120 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1266, 1640, 81, 410, 1026, 351, 1700, 1718] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[3450, 1257, 160, 131, 59, 65, 4, 3066] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[3104, 37, 17, 1702, 21, 0, 95, 3216] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[2298, 3538, 182, 704, 216, 674, 61, 519] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[2374, 634, 497, 521, 36, 258, 1272, 2600] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4022, 390, 905, 1156, 219, 219, 104, 1177] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[683, 1400, 589, 2194, 255, 3, 676, 2392] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[155, 269, 475, 3254, 50, 0, 2, 3987] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1651, 0, 634, 0, 497, 723, 424, 0, 36, 537, 258, 921, 1272, 97, 1142, 0] rerouted_tokens_per_ep_rank=[1651, 634, 1220, 424, 573, 1179, 1369, 1142] avg=1024.0000 max=1651 max_rank=0 min=424 min_rank=3 imbalance_ratio=1.612305 max_load_reduction=6541 max_load_reduction_pct=0.798462 moved_to_echo=4263 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2400, 0, 1257, 0, 160, 1050, 107, 0, 59, 633, 65, 1087, 4, 24, 1346, 0] rerouted_tokens_per_ep_rank=[2400, 1257, 1210, 107, 692, 1152, 28, 1346] avg=1024.0000 max=2400 max_rank=0 min=28 min_rank=6 imbalance_ratio=2.343750 max_load_reduction=5792 max_load_reduction_pct=0.707031 moved_to_echo=3218 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1598, 0, 3538, 0, 182, 700, 573, 0, 216, 107, 674, 184, 61, 131, 228, 0] rerouted_tokens_per_ep_rank=[1598, 3538, 882, 573, 323, 858, 192, 228] avg=1024.0000 max=3538 max_rank=1 min=192 min_rank=6 imbalance_ratio=3.455078 max_load_reduction=4654 max_load_reduction_pct=0.568115 moved_to_echo=1601 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[881, 0, 1640, 0, 81, 385, 334, 0, 1026, 355, 351, 609, 1700, 76, 754, 0] rerouted_tokens_per_ep_rank=[881, 1640, 466, 334, 1381, 960, 1776, 754] avg=1024.0000 max=1776 max_rank=6 min=334 min_rank=3 imbalance_ratio=1.734375 max_load_reduction=6416 max_load_reduction_pct=0.783203 moved_to_echo=4871 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2797, 0, 390, 0, 905, 1225, 941, 0, 219, 243, 219, 417, 104, 215, 517, 0] rerouted_tokens_per_ep_rank=[2797, 390, 2130, 941, 462, 636, 319, 517] avg=1024.0000 max=2797 max_rank=0 min=319 min_rank=6 imbalance_ratio=2.731445 max_load_reduction=5395 max_load_reduction_pct=0.658569 moved_to_echo=1934 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2159, 0, 37, 0, 17, 945, 1385, 0, 21, 664, 0, 1140, 95, 317, 1412, 0] rerouted_tokens_per_ep_rank=[2159, 37, 962, 1385, 685, 1140, 412, 1412] avg=1024.0000 max=2159 max_rank=0 min=37 min_rank=1 imbalance_ratio=2.108398 max_load_reduction=6033 max_load_reduction_pct=0.736450 moved_to_echo=3649 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[105, 0, 269, 0, 475, 50, 2643, 0, 50, 828, 0, 1416, 2, 611, 1743, 0] rerouted_tokens_per_ep_rank=[105, 269, 525, 2643, 878, 1416, 613, 1743] avg=1024.0000 max=2643 max_rank=3 min=105 min_rank=0 imbalance_ratio=2.581055 max_load_reduction=5549 max_load_reduction_pct=0.677368 moved_to_echo=4650 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[475, 0, 1400, 0, 589, 208, 1785, 0, 255, 494, 3, 848, 676, 409, 1050, 0] rerouted_tokens_per_ep_rank=[475, 1400, 797, 1785, 749, 851, 1085, 1050] avg=1024.0000 max=1785 max_rank=3 min=475 min_rank=0 imbalance_ratio=1.743164 max_load_reduction=6407 max_load_reduction_pct=0.782104 moved_to_echo=3735 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12066, 9165, 8192, 8192, 5743, 8192, 5794, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5794 expected_overflow=0 exact_recv_per_rank=[12066, 9165, 8192, 8192, 5743, 8192, 5794, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12066, 9165, 8192, 8192, 5743, 8192, 5794, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=12066 expected_overflow=0 exact_recv_per_rank=[12066, 9165, 8192, 8192, 5743, 8192, 5794, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12066, 9165, 8192, 8192, 5743, 8192, 5794, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9165 expected_overflow=0 exact_recv_per_rank=[12066, 9165, 8192, 8192, 5743, 8192, 5794, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12066, 9165, 8192, 8192, 5743, 8192, 5794, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5743 expected_overflow=0 exact_recv_per_rank=[12066, 9165, 8192, 8192, 5743, 8192, 5794, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=5794 actual_valid_recv=5888 observed_loss=-94 tokens_per_expert=[3968, 1920] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=5743 actual_valid_recv=5888 observed_loss=-145 tokens_per_expert=[1920, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2944, 5376] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1664, 6656] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=9165 actual_valid_recv=9216 observed_loss=-51 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=12066 actual_valid_recv=12160 observed_loss=-94 tokens_per_expert=[12160, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=5888 slack_slots=10496 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=5888 slack_slots=10496 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=12160 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1808, 85, 128, 2616, 1017, 63, 1381, 1094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[473, 104, 615, 3076, 385, 143, 3199, 197] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3461, 337, 213, 113, 1593, 1954, 520, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[36, 456, 1018, 3502, 1002, 204, 1974, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3287, 0, 9, 1936, 596, 189, 1965, 210] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1646, 2308, 22, 2586, 1103, 513, 13, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[2931, 1214, 91, 1418, 8, 4, 2524, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[3551, 1502, 526, 1034, 62, 1324, 1, 192] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1125, 0, 85, 260, 128, 492, 1317, 0, 1017, 197, 63, 610, 1121, 0, 1094, 683] rerouted_tokens_per_ep_rank=[1125, 345, 620, 1317, 1214, 673, 1121, 1777] avg=1024.0000 max=1777 max_rank=7 min=345 min_rank=1 imbalance_ratio=1.735352 max_load_reduction=6415 max_load_reduction_pct=0.783081 moved_to_echo=4785 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[23, 0, 456, 372, 1018, 659, 1764, 0, 1002, 263, 204, 816, 1602, 0, 0, 13] rerouted_tokens_per_ep_rank=[23, 828, 1677, 1764, 1265, 1020, 1602, 13] avg=1024.0000 max=1764 max_rank=3 min=13 min_rank=7 imbalance_ratio=1.722656 max_load_reduction=6428 max_load_reduction_pct=0.784668 moved_to_echo=3900 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2046, 0, 0, 371, 9, 364, 976, 0, 596, 145, 189, 451, 1594, 0, 210, 1241] rerouted_tokens_per_ep_rank=[2046, 371, 373, 976, 741, 640, 1594, 1451] avg=1024.0000 max=2046 max_rank=0 min=371 min_rank=1 imbalance_ratio=1.998047 max_load_reduction=6146 max_load_reduction_pct=0.750244 moved_to_echo=4426 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[291, 0, 104, 607, 615, 582, 1536, 0, 385, 237, 143, 721, 2592, 0, 197, 182] rerouted_tokens_per_ep_rank=[291, 711, 1197, 1536, 622, 864, 2592, 379] avg=1024.0000 max=2592 max_rank=6 min=291 min_rank=0 imbalance_ratio=2.531250 max_load_reduction=5600 max_load_reduction_pct=0.683594 moved_to_echo=4457 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1824, 0, 1214, 476, 91, 266, 716, 0, 8, 106, 4, 330, 2048, 0, 2, 1107] rerouted_tokens_per_ep_rank=[1824, 1690, 357, 716, 114, 334, 2048, 1109] avg=1024.0000 max=2048 max_rank=6 min=114 min_rank=4 imbalance_ratio=2.000000 max_load_reduction=6144 max_load_reduction_pct=0.750000 moved_to_echo=3605 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 2308, 2, 22, 486, 1303, 0, 1103, 194, 513, 603, 11, 0, 1, 621] rerouted_tokens_per_ep_rank=[1025, 2310, 508, 1303, 1297, 1116, 11, 622] avg=1024.0000 max=2310 max_rank=1 min=11 min_rank=6 imbalance_ratio=2.255859 max_load_reduction=5882 max_load_reduction_pct=0.718018 moved_to_echo=3046 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2154, 0, 337, 98, 213, 21, 58, 0, 1593, 8, 1954, 26, 422, 0, 1, 1307] rerouted_tokens_per_ep_rank=[2154, 435, 234, 58, 1601, 1980, 422, 1308] avg=1024.0000 max=2154 max_rank=0 min=58 min_rank=3 imbalance_ratio=2.103516 max_load_reduction=6038 max_load_reduction_pct=0.737061 moved_to_echo=5311 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2210, 0, 1502, 0, 526, 194, 522, 0, 62, 77, 1324, 241, 1, 0, 192, 1341] rerouted_tokens_per_ep_rank=[2210, 1502, 720, 522, 139, 1565, 1, 1533] avg=1024.0000 max=2210 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.158203 max_load_reduction=5982 max_load_reduction_pct=0.730225 moved_to_echo=3238 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10698, 8192, 5686, 8192, 6993, 8192, 9391, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6993 expected_overflow=0 exact_recv_per_rank=[10698, 8192, 5686, 8192, 6993, 8192, 9391, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10698, 8192, 5686, 8192, 6993, 8192, 9391, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5686 expected_overflow=0 exact_recv_per_rank=[10698, 8192, 5686, 8192, 6993, 8192, 9391, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10698, 8192, 5686, 8192, 6993, 8192, 9391, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9391 expected_overflow=0 exact_recv_per_rank=[10698, 8192, 5686, 8192, 6993, 8192, 9391, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10698 expected_overflow=0 exact_recv_per_rank=[10698, 8192, 5686, 8192, 6993, 8192, 9391, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10698, 8192, 5686, 8192, 6993, 8192, 9391, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=5686 actual_valid_recv=5760 observed_loss=-74 tokens_per_expert=[2688, 3072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=6993 actual_valid_recv=7168 observed_loss=-175 tokens_per_expert=[5888, 1280] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4480, 3840] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1792, 6528] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6016, 2304] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9391 actual_valid_recv=9472 observed_loss=-81 tokens_per_expert=[9472, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=10698 actual_valid_recv=10752 observed_loss=-54 tokens_per_expert=[10752, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=5760 slack_slots=10624 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=7168 slack_slots=9216 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9472 slack_slots=6912 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=10752 slack_slots=5632 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[2850, 2473, 32, 1267, 807, 744, 10, 9] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[3341, 1028, 216, 652, 130, 352, 2024, 449] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[2178, 1361, 365, 691, 1923, 267, 4, 1403] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[816, 2147, 355, 2465, 22, 2339, 48, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[303, 3057, 402, 585, 1426, 67, 2162, 190] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[251, 1917, 2270, 1121, 677, 238, 0, 1718] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[2658, 472, 268, 3940, 11, 1, 103, 739] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1954, 23, 3, 1015, 3665, 9, 878, 645] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2000, 0, 1624, 0, 32, 850, 998, 0, 807, 0, 744, 455, 10, 269, 9, 394] rerouted_tokens_per_ep_rank=[2000, 1624, 882, 998, 807, 1199, 279, 403] avg=1024.0000 max=2000 max_rank=0 min=279 min_rank=6 imbalance_ratio=1.953125 max_load_reduction=6192 max_load_reduction_pct=0.755859 moved_to_echo=2688 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2345, 0, 676, 0, 216, 996, 514, 0, 130, 0, 352, 189, 2024, 138, 449, 163] rerouted_tokens_per_ep_rank=[2345, 676, 1212, 514, 130, 541, 2162, 612] avg=1024.0000 max=2345 max_rank=0 min=130 min_rank=4 imbalance_ratio=2.290039 max_load_reduction=5847 max_load_reduction_pct=0.713745 moved_to_echo=3445 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[213, 0, 2008, 0, 402, 90, 461, 0, 1426, 0, 67, 562, 2162, 124, 190, 487] rerouted_tokens_per_ep_rank=[213, 2008, 492, 461, 1426, 629, 2286, 677] avg=1024.0000 max=2286 max_rank=6 min=213 min_rank=0 imbalance_ratio=2.232422 max_load_reduction=5906 max_load_reduction_pct=0.720947 moved_to_echo=5018 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[573, 0, 1410, 0, 355, 243, 1942, 0, 22, 0, 2339, 395, 48, 523, 0, 342] rerouted_tokens_per_ep_rank=[573, 1410, 598, 1942, 22, 2734, 571, 342] avg=1024.0000 max=2734 max_rank=5 min=22 min_rank=4 imbalance_ratio=2.669922 max_load_reduction=5458 max_load_reduction_pct=0.666260 moved_to_echo=3669 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1529, 0, 895, 0, 365, 649, 545, 0, 1923, 0, 267, 250, 4, 146, 1403, 216] rerouted_tokens_per_ep_rank=[1529, 895, 1014, 545, 1923, 517, 150, 1619] avg=1024.0000 max=1923 max_rank=4 min=150 min_rank=6 imbalance_ratio=1.877930 max_load_reduction=6269 max_load_reduction_pct=0.765259 moved_to_echo=4209 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1866, 0, 311, 0, 268, 792, 3103, 0, 11, 0, 1, 86, 103, 837, 739, 75] rerouted_tokens_per_ep_rank=[1866, 311, 1060, 3103, 11, 87, 940, 814] avg=1024.0000 max=3103 max_rank=3 min=11 min_rank=4 imbalance_ratio=3.030273 max_load_reduction=5089 max_load_reduction_pct=0.621216 moved_to_echo=1852 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1367, 0, 8, 0, 3, 587, 796, 0, 3665, 0, 9, 8, 878, 219, 645, 7] rerouted_tokens_per_ep_rank=[1367, 8, 590, 796, 3665, 17, 1097, 652] avg=1024.0000 max=3665 max_rank=4 min=8 min_rank=1 imbalance_ratio=3.579102 max_load_reduction=4527 max_load_reduction_pct=0.552612 moved_to_echo=5431 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[177, 0, 1260, 0, 2270, 74, 883, 0, 677, 0, 238, 352, 0, 238, 1718, 305] rerouted_tokens_per_ep_rank=[177, 1260, 2344, 883, 677, 590, 238, 2023] avg=1024.0000 max=2344 max_rank=2 min=177 min_rank=0 imbalance_ratio=2.289062 max_load_reduction=5848 max_load_reduction_pct=0.713867 moved_to_echo=3528 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8661 expected_overflow=0 exact_recv_per_rank=[10070, 8192, 8192, 9242, 8661, 6314, 7723, 7142] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9242 expected_overflow=0 exact_recv_per_rank=[10070, 8192, 8192, 9242, 8661, 6314, 7723, 7142] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10070, 8192, 8192, 9242, 8661, 6314, 7723, 7142] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10070 expected_overflow=0 exact_recv_per_rank=[10070, 8192, 8192, 9242, 8661, 6314, 7723, 7142] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7142 expected_overflow=0 exact_recv_per_rank=[10070, 8192, 8192, 9242, 8661, 6314, 7723, 7142] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6314 expected_overflow=0 exact_recv_per_rank=[10070, 8192, 8192, 9242, 8661, 6314, 7723, 7142] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7723 expected_overflow=0 exact_recv_per_rank=[10070, 8192, 8192, 9242, 8661, 6314, 7723, 7142] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10070, 8192, 8192, 9242, 8661, 6314, 7723, 7142] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=6314 actual_valid_recv=6400 observed_loss=-86 tokens_per_expert=[4096, 2304] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=7142 actual_valid_recv=7296 observed_loss=-154 tokens_per_expert=[5248, 2048] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8661 actual_valid_recv=8704 observed_loss=-43 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7723 actual_valid_recv=7808 observed_loss=-85 tokens_per_expert=[5248, 2560] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=10070 actual_valid_recv=10112 observed_loss=-42 tokens_per_expert=[10112, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3968, 4352] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=9242 actual_valid_recv=9344 observed_loss=-102 tokens_per_expert=[9344, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=7296 slack_slots=9088 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=6400 slack_slots=9984 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=10112 slack_slots=6272 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=9344 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[2088, 58, 2963, 1052, 317, 505, 1193, 16] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[174, 1650, 383, 3237, 810, 51, 159, 1728] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[2, 1, 701, 1997, 2488, 393, 1, 2609] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[152, 729, 486, 3008, 2953, 774, 15, 75] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 3, 223, 2979, 2039, 1013, 0, 1935] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1482, 16, 823, 1916, 789, 20, 2935, 211] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[3266, 4, 0, 2452, 768, 983, 363, 356] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[530, 346, 2589, 2949, 96, 10, 1062, 610] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2088, 15, 58, 289, 2963, 0, 441, 0, 282, 0, 505, 238, 1193, 84, 16, 20] rerouted_tokens_per_ep_rank=[2103, 347, 2963, 441, 282, 743, 1277, 36] avg=1024.0000 max=2963 max_rank=2 min=36 min_rank=7 imbalance_ratio=2.893555 max_load_reduction=5229 max_load_reduction_pct=0.638306 moved_to_echo=2338 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[152, 143, 729, 826, 486, 6, 1259, 0, 2617, 0, 774, 682, 15, 241, 75, 187] rerouted_tokens_per_ep_rank=[295, 1555, 492, 1259, 2617, 1456, 256, 262] avg=1024.0000 max=2617 max_rank=4 min=256 min_rank=6 imbalance_ratio=2.555664 max_load_reduction=5575 max_load_reduction_pct=0.680542 moved_to_echo=4591 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3266, 37, 4, 674, 0, 1, 1026, 0, 682, 0, 983, 556, 363, 196, 356, 48] rerouted_tokens_per_ep_rank=[3303, 678, 1, 1026, 682, 1539, 559, 404] avg=1024.0000 max=3303 max_rank=0 min=1 min_rank=2 imbalance_ratio=3.225586 max_load_reduction=4889 max_load_reduction_pct=0.596802 moved_to_echo=3184 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 98, 3, 818, 223, 4, 1248, 0, 1808, 0, 1013, 675, 0, 238, 1935, 129] rerouted_tokens_per_ep_rank=[98, 821, 227, 1248, 1808, 1688, 238, 2064] avg=1024.0000 max=2064 max_rank=7 min=98 min_rank=0 imbalance_ratio=2.015625 max_load_reduction=6128 max_load_reduction_pct=0.748047 moved_to_echo=5798 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[174, 39, 1650, 889, 383, 1, 1355, 0, 719, 0, 51, 734, 159, 259, 1728, 51] rerouted_tokens_per_ep_rank=[213, 2539, 384, 1355, 719, 785, 418, 1779] avg=1024.0000 max=2539 max_rank=1 min=213 min_rank=0 imbalance_ratio=2.479492 max_load_reduction=5653 max_load_reduction_pct=0.690063 moved_to_echo=3701 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 124, 1, 553, 701, 11, 825, 0, 2192, 0, 393, 456, 1, 163, 2609, 161] rerouted_tokens_per_ep_rank=[126, 554, 712, 825, 2192, 849, 164, 2770] avg=1024.0000 max=2770 max_rank=7 min=126 min_rank=0 imbalance_ratio=2.705078 max_load_reduction=5422 max_load_reduction_pct=0.661865 moved_to_echo=5975 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1482, 38, 16, 526, 823, 1, 803, 0, 700, 0, 20, 434, 2935, 153, 211, 50] rerouted_tokens_per_ep_rank=[1520, 542, 824, 803, 700, 454, 3088, 261] avg=1024.0000 max=3088 max_rank=6 min=261 min_rank=7 imbalance_ratio=3.015625 max_load_reduction=5104 max_load_reduction_pct=0.623047 moved_to_echo=4503 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[530, 4, 346, 810, 2589, 0, 1235, 0, 86, 0, 10, 668, 1062, 236, 610, 6] rerouted_tokens_per_ep_rank=[534, 1156, 2589, 1235, 86, 678, 1298, 616] avg=1024.0000 max=2589 max_rank=2 min=86 min_rank=4 imbalance_ratio=2.528320 max_load_reduction=5603 max_load_reduction_pct=0.683960 moved_to_echo=2678 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9086 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9086, 8192, 7298, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9086, 8192, 7298, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9086, 8192, 7298, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9086, 8192, 7298, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9086, 8192, 7298, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7298 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9086, 8192, 7298, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9086, 8192, 7298, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9086, 8192, 7298, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7808, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2816, 5504] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[8192, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7552, 768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7298 actual_valid_recv=7424 observed_loss=-126 tokens_per_expert=[5760, 1664] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3840, 4480] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=9086 actual_valid_recv=9088 observed_loss=-2 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=7424 slack_slots=8960 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[3, 810, 1704, 1026, 78, 3288, 227, 1056] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[311, 530, 1136, 2456, 3275, 241, 148, 95] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 111, 8, 327, 2313, 3718, 893, 822] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[192, 44, 1057, 486, 1163, 3869, 1287, 94] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[30, 373, 159, 1471, 1, 3665, 1951, 542] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[62, 2, 625, 160, 2352, 2197, 1368, 1426] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[3, 270, 89, 144, 1691, 2648, 216, 3131] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[22, 661, 3249, 512, 130, 2870, 76, 672] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[311, 81, 530, 57, 1136, 49, 2456, 479, 2642, 0, 89, 0, 148, 14, 95, 105] rerouted_tokens_per_ep_rank=[392, 587, 1185, 2935, 2642, 89, 162, 200] avg=1024.0000 max=2935 max_rank=3 min=89 min_rank=5 imbalance_ratio=2.866211 max_load_reduction=5257 max_load_reduction_pct=0.641724 moved_to_echo=3093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3, 1106, 810, 787, 1704, 1, 1026, 11, 64, 0, 1199, 0, 227, 196, 1056, 2] rerouted_tokens_per_ep_rank=[1109, 1597, 1705, 1037, 64, 1199, 423, 1058] avg=1024.0000 max=1705 max_rank=2 min=64 min_rank=4 imbalance_ratio=1.665039 max_load_reduction=6487 max_load_reduction_pct=0.791870 moved_to_echo=2744 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[30, 1233, 373, 878, 159, 0, 1471, 0, 1, 0, 1336, 0, 1951, 218, 542, 0] rerouted_tokens_per_ep_rank=[1263, 1251, 159, 1471, 1, 1336, 2169, 542] avg=1024.0000 max=2169 max_rank=6 min=1 min_rank=4 imbalance_ratio=2.118164 max_load_reduction=6023 max_load_reduction_pct=0.735229 moved_to_echo=4048 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1254, 111, 895, 8, 37, 327, 340, 1859, 0, 1344, 0, 893, 225, 822, 77] rerouted_tokens_per_ep_rank=[1254, 1006, 45, 667, 1859, 1344, 1118, 899] avg=1024.0000 max=1859 max_rank=4 min=45 min_rank=2 imbalance_ratio=1.815430 max_load_reduction=6333 max_load_reduction_pct=0.773071 moved_to_echo=5220 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[62, 739, 2, 526, 625, 35, 160, 344, 1898, 0, 801, 0, 1368, 131, 1426, 75] rerouted_tokens_per_ep_rank=[801, 528, 660, 504, 1898, 801, 1499, 1501] avg=1024.0000 max=1898 max_rank=4 min=504 min_rank=3 imbalance_ratio=1.853516 max_load_reduction=6294 max_load_reduction_pct=0.768311 moved_to_echo=5699 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[192, 1301, 44, 927, 1057, 17, 486, 170, 939, 0, 1410, 0, 1287, 231, 94, 37] rerouted_tokens_per_ep_rank=[1493, 971, 1074, 656, 939, 1410, 1518, 131] avg=1024.0000 max=1518 max_rank=6 min=131 min_rank=7 imbalance_ratio=1.482422 max_load_reduction=6674 max_load_reduction_pct=0.814697 moved_to_echo=3998 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3, 890, 270, 634, 89, 25, 144, 247, 1365, 0, 966, 0, 216, 158, 3131, 54] rerouted_tokens_per_ep_rank=[893, 904, 114, 391, 1365, 966, 374, 3185] avg=1024.0000 max=3185 max_rank=7 min=114 min_rank=2 imbalance_ratio=3.110352 max_load_reduction=5007 max_load_reduction_pct=0.611206 moved_to_echo=5890 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[22, 965, 661, 687, 3249, 1, 512, 19, 106, 0, 1047, 0, 76, 171, 672, 4] rerouted_tokens_per_ep_rank=[987, 1348, 3250, 531, 106, 1047, 247, 676] avg=1024.0000 max=3250 max_rank=2 min=106 min_rank=4 imbalance_ratio=3.173828 max_load_reduction=4942 max_load_reduction_pct=0.603271 moved_to_echo=2076 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8874, 8192, 7510, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8874, 8192, 7510, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8874, 8192, 7510, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8874 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8874, 8192, 7510, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8874, 8192, 7510, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8874, 8192, 7510, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8874, 8192, 7510, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7510 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8874, 8192, 7510, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7510 actual_valid_recv=7680 observed_loss=-170 tokens_per_expert=[6272, 1408] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2816, 5504] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6656, 1664] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[640, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7936, 384] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[8064, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8874 actual_valid_recv=8960 observed_loss=-86 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[2428, 573, 1507, 2045, 192, 462, 127, 858] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[6, 2974, 1142, 24, 61, 2, 116, 3867] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[60, 1669, 1, 5, 1086, 3101, 2200, 70] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1, 4009, 63, 352, 540, 478, 2598, 151] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[681, 2124, 47, 30, 14, 3492, 1763, 41] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[101, 256, 1054, 11, 2435, 3, 2700, 1632] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[3771, 224, 370, 103, 389, 382, 293, 2660] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[201, 1002, 1344, 473, 556, 1636, 543, 2437] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2428, 45, 366, 0, 1507, 26, 2045, 207, 192, 213, 417, 0, 101, 0, 645, 0] rerouted_tokens_per_ep_rank=[2473, 366, 1533, 2252, 405, 417, 101, 645] avg=1024.0000 max=2473 max_rank=0 min=101 min_rank=6 imbalance_ratio=2.415039 max_load_reduction=5719 max_load_reduction_pct=0.698120 moved_to_echo=1568 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[101, 0, 164, 0, 1054, 560, 11, 92, 2435, 406, 3, 0, 2140, 0, 1226, 0] rerouted_tokens_per_ep_rank=[101, 164, 1614, 103, 2841, 3, 2140, 1226] avg=1024.0000 max=2841 max_rank=4 min=3 min_rank=5 imbalance_ratio=2.774414 max_load_reduction=5351 max_load_reduction_pct=0.653198 moved_to_echo=6210 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[681, 344, 1357, 0, 47, 366, 30, 767, 14, 10, 3148, 0, 1397, 0, 31, 0] rerouted_tokens_per_ep_rank=[1025, 1357, 413, 797, 24, 3148, 1397, 31] avg=1024.0000 max=3148 max_rank=5 min=24 min_rank=4 imbalance_ratio=3.074219 max_load_reduction=5044 max_load_reduction_pct=0.615723 moved_to_echo=4600 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[6, 2, 1895, 0, 1142, 28, 24, 1079, 61, 967, 0, 0, 88, 0, 2900, 0] rerouted_tokens_per_ep_rank=[8, 1895, 1170, 1103, 1028, 0, 88, 2900] avg=1024.0000 max=2900 max_rank=7 min=0 min_rank=5 imbalance_ratio=2.832031 max_load_reduction=5292 max_load_reduction_pct=0.645996 moved_to_echo=4016 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[60, 306, 1066, 0, 1, 457, 5, 603, 1086, 17, 2795, 0, 1743, 0, 53, 0] rerouted_tokens_per_ep_rank=[366, 1066, 458, 608, 1103, 2795, 1743, 53] avg=1024.0000 max=2795 max_rank=5 min=53 min_rank=7 imbalance_ratio=2.729492 max_load_reduction=5397 max_load_reduction_pct=0.658813 moved_to_echo=5694 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3771, 38, 144, 0, 370, 60, 103, 80, 389, 662, 344, 0, 233, 0, 1998, 0] rerouted_tokens_per_ep_rank=[3809, 144, 430, 183, 1051, 344, 233, 1998] avg=1024.0000 max=3809 max_rank=0 min=144 min_rank=1 imbalance_ratio=3.719727 max_load_reduction=4383 max_load_reduction_pct=0.535034 moved_to_echo=3626 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[201, 161, 640, 0, 1344, 112, 473, 362, 556, 607, 1475, 0, 431, 0, 1830, 0] rerouted_tokens_per_ep_rank=[362, 640, 1456, 835, 1163, 1475, 431, 1830] avg=1024.0000 max=1830 max_rank=7 min=362 min_rank=0 imbalance_ratio=1.787109 max_load_reduction=6362 max_load_reduction_pct=0.776611 moved_to_echo=4899 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 47, 2560, 0, 63, 539, 352, 1449, 540, 37, 431, 0, 2059, 0, 114, 0] rerouted_tokens_per_ep_rank=[48, 2560, 602, 1801, 577, 431, 2059, 114] avg=1024.0000 max=2560 max_rank=1 min=48 min_rank=0 imbalance_ratio=2.500000 max_load_reduction=5632 max_load_reduction_pct=0.687500 moved_to_echo=3181 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8613 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7676, 7682, 8192, 8613, 8192, 8797] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7676, 7682, 8192, 8613, 8192, 8797] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7676 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7676, 7682, 8192, 8613, 8192, 8797] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7676, 7682, 8192, 8613, 8192, 8797] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7682 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7676, 7682, 8192, 8613, 8192, 8797] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8797 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7676, 7682, 8192, 8613, 8192, 8797] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7676, 7682, 8192, 8613, 8192, 8797] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7676, 7682, 8192, 8613, 8192, 8797] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=7682 actual_valid_recv=7808 observed_loss=-126 tokens_per_expert=[3072, 4736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7296, 1024] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=7676 actual_valid_recv=7808 observed_loss=-132 tokens_per_expert=[5632, 2176] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5376, 2944] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8797 actual_valid_recv=8832 observed_loss=-35 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8613 actual_valid_recv=8704 observed_loss=-91 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[64, 2825, 505, 248, 1813, 350, 1234, 1153] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[35, 3086, 520, 415, 2060, 425, 682, 969] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[213, 3392, 725, 387, 1209, 504, 735, 1027] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[40, 3464, 503, 293, 1609, 364, 838, 1081] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[52, 2959, 848, 280, 1716, 414, 1026, 897] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[108, 3149, 801, 258, 1599, 399, 746, 1132] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[384, 3073, 325, 422, 1797, 332, 761, 1098] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[95, 2936, 1014, 266, 1579, 362, 1046, 894] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[384, 892, 1002, 0, 325, 400, 422, 698, 1251, 0, 332, 481, 761, 146, 1098, 0] rerouted_tokens_per_ep_rank=[1276, 1002, 725, 1120, 1251, 813, 907, 1098] avg=1024.0000 max=1276 max_rank=0 min=725 min_rank=2 imbalance_ratio=1.246094 max_load_reduction=6916 max_load_reduction_pct=0.844238 moved_to_echo=4069 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[35, 893, 1017, 0, 520, 454, 415, 697, 1443, 0, 425, 479, 682, 163, 969, 0] rerouted_tokens_per_ep_rank=[928, 1017, 974, 1112, 1443, 904, 845, 969] avg=1024.0000 max=1443 max_rank=4 min=845 min_rank=6 imbalance_ratio=1.409180 max_load_reduction=6749 max_load_reduction_pct=0.823853 moved_to_echo=4161 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[213, 981, 1118, 0, 725, 266, 387, 766, 847, 0, 504, 527, 735, 96, 1027, 0] rerouted_tokens_per_ep_rank=[1194, 1118, 991, 1153, 847, 1031, 831, 1027] avg=1024.0000 max=1194 max_rank=0 min=831 min_rank=6 imbalance_ratio=1.166016 max_load_reduction=6998 max_load_reduction_pct=0.854248 moved_to_echo=3736 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[52, 856, 976, 0, 848, 378, 280, 668, 1202, 0, 414, 459, 1026, 136, 897, 0] rerouted_tokens_per_ep_rank=[908, 976, 1226, 948, 1202, 873, 1162, 897] avg=1024.0000 max=1226 max_rank=2 min=873 min_rank=5 imbalance_ratio=1.197266 max_load_reduction=6966 max_load_reduction_pct=0.850342 moved_to_echo=4134 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[108, 911, 1038, 0, 801, 352, 258, 711, 1120, 0, 399, 489, 746, 127, 1132, 0] rerouted_tokens_per_ep_rank=[1019, 1038, 1153, 969, 1120, 888, 873, 1132] avg=1024.0000 max=1153 max_rank=2 min=873 min_rank=6 imbalance_ratio=1.125977 max_load_reduction=7039 max_load_reduction_pct=0.859253 moved_to_echo=4013 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[64, 817, 931, 0, 505, 399, 248, 638, 1270, 0, 350, 439, 1234, 144, 1153, 0] rerouted_tokens_per_ep_rank=[881, 931, 904, 886, 1270, 789, 1378, 1153] avg=1024.0000 max=1378 max_rank=6 min=789 min_rank=5 imbalance_ratio=1.345703 max_load_reduction=6814 max_load_reduction_pct=0.831787 moved_to_echo=4590 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[40, 1002, 1142, 0, 503, 354, 293, 782, 1127, 0, 364, 538, 838, 128, 1081, 0] rerouted_tokens_per_ep_rank=[1042, 1142, 857, 1075, 1127, 902, 966, 1081] avg=1024.0000 max=1142 max_rank=1 min=857 min_rank=2 imbalance_ratio=1.115234 max_load_reduction=7050 max_load_reduction_pct=0.860596 moved_to_echo=4076 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[95, 849, 968, 0, 1014, 348, 266, 663, 1106, 0, 362, 456, 1046, 125, 894, 0] rerouted_tokens_per_ep_rank=[944, 968, 1362, 929, 1106, 818, 1171, 894] avg=1024.0000 max=1362 max_rank=2 min=818 min_rank=5 imbalance_ratio=1.330078 max_load_reduction=6830 max_load_reduction_pct=0.833740 moved_to_echo=3989 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9366, 7018, 8133, 8251] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9366, 7018, 8133, 8251] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7018 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9366, 7018, 8133, 8251] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8133 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9366, 7018, 8133, 8251] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9366, 7018, 8133, 8251] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9366 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9366, 7018, 8133, 8251] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8251 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9366, 7018, 8133, 8251] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 9366, 7018, 8133, 8251] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7018 actual_valid_recv=7168 observed_loss=-150 tokens_per_expert=[3200, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8133 actual_valid_recv=8320 observed_loss=-187 tokens_per_expert=[7168, 1152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5248, 3072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2688, 5632] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1024, 7296] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=9366 actual_valid_recv=9472 observed_loss=-106 tokens_per_expert=[9472, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8251 actual_valid_recv=8320 observed_loss=-69 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7168 slack_slots=9216 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=9472 slack_slots=6912 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1709, 262, 312, 732, 1234, 832, 1256, 1855] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[398, 80, 136, 1354, 2548, 726, 922, 2028] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[789, 253, 1458, 330, 2250, 549, 1258, 1305] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[748, 53, 667, 1115, 2110, 312, 1664, 1523] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1607, 491, 128, 559, 2633, 406, 1202, 1166] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1384, 302, 586, 1291, 1751, 347, 1277, 1254] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[817, 122, 612, 732, 1827, 745, 2171, 1166] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[935, 128, 171, 1090, 1833, 523, 1938, 1574] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1709, 0, 262, 495, 312, 410, 732, 85, 739, 0, 832, 290, 881, 0, 1445, 0] rerouted_tokens_per_ep_rank=[1709, 757, 722, 817, 739, 1122, 881, 1445] avg=1024.0000 max=1709 max_rank=0 min=722 min_rank=2 imbalance_ratio=1.668945 max_load_reduction=6483 max_load_reduction_pct=0.791382 moved_to_echo=4187 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[748, 0, 53, 847, 667, 337, 1115, 113, 1263, 0, 312, 384, 1167, 0, 1186, 0] rerouted_tokens_per_ep_rank=[748, 900, 1004, 1228, 1263, 696, 1167, 1186] avg=1024.0000 max=1263 max_rank=4 min=696 min_rank=5 imbalance_ratio=1.233398 max_load_reduction=6929 max_load_reduction_pct=0.845825 moved_to_echo=4312 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1384, 0, 302, 703, 586, 277, 1291, 86, 1048, 0, 347, 295, 896, 0, 977, 0] rerouted_tokens_per_ep_rank=[1384, 1005, 863, 1377, 1048, 642, 896, 977] avg=1024.0000 max=1384 max_rank=0 min=642 min_rank=5 imbalance_ratio=1.351562 max_load_reduction=6808 max_load_reduction_pct=0.831055 moved_to_echo=3563 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[789, 0, 253, 903, 1458, 289, 330, 85, 1347, 0, 549, 290, 883, 0, 1016, 0] rerouted_tokens_per_ep_rank=[789, 1156, 1747, 415, 1347, 839, 883, 1016] avg=1024.0000 max=1747 max_rank=2 min=415 min_rank=3 imbalance_ratio=1.706055 max_load_reduction=6445 max_load_reduction_pct=0.786743 moved_to_echo=4085 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[817, 0, 122, 733, 612, 258, 732, 147, 1094, 0, 745, 501, 1523, 0, 908, 0] rerouted_tokens_per_ep_rank=[817, 855, 870, 879, 1094, 1246, 1523, 908] avg=1024.0000 max=1523 max_rank=6 min=817 min_rank=0 imbalance_ratio=1.487305 max_load_reduction=6669 max_load_reduction_pct=0.814087 moved_to_echo=4771 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1607, 0, 491, 1057, 128, 258, 559, 81, 1576, 0, 406, 277, 844, 0, 908, 0] rerouted_tokens_per_ep_rank=[1607, 1548, 386, 640, 1576, 683, 844, 908] avg=1024.0000 max=1607 max_rank=0 min=386 min_rank=2 imbalance_ratio=1.569336 max_load_reduction=6585 max_load_reduction_pct=0.803833 moved_to_echo=4011 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[398, 0, 80, 1027, 136, 452, 1354, 66, 1521, 0, 726, 217, 639, 0, 1576, 0] rerouted_tokens_per_ep_rank=[398, 1107, 588, 1420, 1521, 943, 639, 1576] avg=1024.0000 max=1576 max_rank=7 min=398 min_rank=0 imbalance_ratio=1.539062 max_load_reduction=6616 max_load_reduction_pct=0.807617 moved_to_echo=4679 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[935, 0, 128, 736, 171, 348, 1090, 131, 1097, 0, 523, 448, 1359, 0, 1226, 0] rerouted_tokens_per_ep_rank=[935, 864, 519, 1221, 1097, 971, 1359, 1226] avg=1024.0000 max=1359 max_rank=6 min=519 min_rank=2 imbalance_ratio=1.327148 max_load_reduction=6833 max_load_reduction_pct=0.834106 moved_to_echo=4653 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8387, 8192, 6699, 7997, 9685, 7142, 8192, 9242] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6699 expected_overflow=0 exact_recv_per_rank=[8387, 8192, 6699, 7997, 9685, 7142, 8192, 9242] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9685 expected_overflow=0 exact_recv_per_rank=[8387, 8192, 6699, 7997, 9685, 7142, 8192, 9242] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8387 expected_overflow=0 exact_recv_per_rank=[8387, 8192, 6699, 7997, 9685, 7142, 8192, 9242] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7997 expected_overflow=0 exact_recv_per_rank=[8387, 8192, 6699, 7997, 9685, 7142, 8192, 9242] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7142 expected_overflow=0 exact_recv_per_rank=[8387, 8192, 6699, 7997, 9685, 7142, 8192, 9242] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9242 expected_overflow=0 exact_recv_per_rank=[8387, 8192, 6699, 7997, 9685, 7142, 8192, 9242] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8387, 8192, 6699, 7997, 9685, 7142, 8192, 9242] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=6699 actual_valid_recv=6784 observed_loss=-85 tokens_per_expert=[4096, 2688] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8387 actual_valid_recv=8448 observed_loss=-61 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=7997 actual_valid_recv=8192 observed_loss=-195 tokens_per_expert=[7296, 896] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1792, 6528] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7142 actual_valid_recv=7296 observed_loss=-154 tokens_per_expert=[4480, 2816] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=9242 actual_valid_recv=9344 observed_loss=-102 tokens_per_expert=[9344, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=9685 actual_valid_recv=9728 observed_loss=-43 tokens_per_expert=[9728, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=6784 slack_slots=9600 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7296 slack_slots=9088 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=9344 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=9728 slack_slots=6656 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1113, 57, 392, 2725, 11, 2655, 1064, 175] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1642, 82, 1576, 1922, 57, 1565, 1007, 341] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1686, 418, 1058, 1934, 45, 1129, 1746, 176] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[903, 187, 2560, 1324, 133, 1967, 535, 583] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[665, 113, 1171, 3376, 27, 1889, 306, 645] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1310, 115, 509, 2749, 321, 2208, 767, 213] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[600, 7, 206, 2825, 123, 2394, 1719, 318] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[696, 58, 121, 3244, 185, 1972, 1879, 37] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1562, 0, 82, 441, 1576, 80, 784, 0, 57, 697, 1065, 0, 1007, 0, 341, 500] rerouted_tokens_per_ep_rank=[1562, 523, 1656, 784, 754, 1065, 1007, 841] avg=1024.0000 max=1656 max_rank=2 min=523 min_rank=1 imbalance_ratio=1.617188 max_load_reduction=6536 max_load_reduction_pct=0.797852 moved_to_echo=3667 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[633, 0, 113, 775, 1171, 32, 1377, 0, 27, 1224, 1285, 0, 306, 0, 645, 604] rerouted_tokens_per_ep_rank=[633, 888, 1203, 1377, 1251, 1285, 306, 1249] avg=1024.0000 max=1377 max_rank=3 min=306 min_rank=6 imbalance_ratio=1.344727 max_load_reduction=6815 max_load_reduction_pct=0.831909 moved_to_echo=4091 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[859, 0, 187, 304, 2560, 44, 540, 0, 133, 480, 1338, 0, 535, 0, 583, 629] rerouted_tokens_per_ep_rank=[859, 491, 2604, 540, 613, 1338, 535, 1212] avg=1024.0000 max=2604 max_rank=2 min=491 min_rank=1 imbalance_ratio=2.542969 max_load_reduction=5588 max_load_reduction_pct=0.682129 moved_to_echo=3698 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1604, 0, 418, 444, 1058, 82, 789, 0, 45, 701, 768, 0, 1746, 0, 176, 361] rerouted_tokens_per_ep_rank=[1604, 862, 1140, 789, 746, 768, 1746, 537] avg=1024.0000 max=1746 max_rank=6 min=537 min_rank=7 imbalance_ratio=1.705078 max_load_reduction=6446 max_load_reduction_pct=0.786865 moved_to_echo=3797 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1055, 0, 57, 629, 392, 58, 1105, 0, 11, 991, 1803, 0, 1064, 0, 175, 852] rerouted_tokens_per_ep_rank=[1055, 686, 450, 1105, 1002, 1803, 1064, 1027] avg=1024.0000 max=1803 max_rank=5 min=450 min_rank=2 imbalance_ratio=1.760742 max_load_reduction=6389 max_load_reduction_pct=0.779907 moved_to_echo=4896 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1246, 0, 115, 631, 509, 64, 1121, 0, 321, 997, 1502, 0, 767, 0, 213, 706] rerouted_tokens_per_ep_rank=[1246, 746, 573, 1121, 1318, 1502, 767, 919] avg=1024.0000 max=1502 max_rank=5 min=573 min_rank=2 imbalance_ratio=1.466797 max_load_reduction=6690 max_load_reduction_pct=0.816650 moved_to_echo=4506 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[571, 0, 7, 648, 206, 29, 1153, 0, 123, 1024, 1628, 0, 1719, 0, 318, 766] rerouted_tokens_per_ep_rank=[571, 655, 235, 1153, 1147, 1628, 1719, 1084] avg=1024.0000 max=1719 max_rank=6 min=235 min_rank=2 imbalance_ratio=1.678711 max_load_reduction=6473 max_load_reduction_pct=0.790161 moved_to_echo=5578 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[662, 0, 58, 745, 121, 34, 1323, 0, 185, 1176, 1341, 0, 1879, 0, 37, 631] rerouted_tokens_per_ep_rank=[662, 803, 155, 1323, 1361, 1341, 1879, 668] avg=1024.0000 max=1879 max_rank=6 min=155 min_rank=2 imbalance_ratio=1.834961 max_load_reduction=6313 max_load_reduction_pct=0.770630 moved_to_echo=5249 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10730 expected_overflow=0 exact_recv_per_rank=[8192, 5654, 8016, 8192, 8192, 10730, 9023, 7537] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5654, 8016, 8192, 8192, 10730, 9023, 7537] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5654, 8016, 8192, 8192, 10730, 9023, 7537] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5654, 8016, 8192, 8192, 10730, 9023, 7537] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8016 expected_overflow=0 exact_recv_per_rank=[8192, 5654, 8016, 8192, 8192, 10730, 9023, 7537] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5654 expected_overflow=0 exact_recv_per_rank=[8192, 5654, 8016, 8192, 8192, 10730, 9023, 7537] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7537 expected_overflow=0 exact_recv_per_rank=[8192, 5654, 8016, 8192, 8192, 10730, 9023, 7537] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9023 expected_overflow=0 exact_recv_per_rank=[8192, 5654, 8016, 8192, 8192, 10730, 9023, 7537] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=5654 actual_valid_recv=5888 observed_loss=-234 tokens_per_expert=[1152, 4736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=7537 actual_valid_recv=7680 observed_loss=-143 tokens_per_expert=[2560, 5120] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8016 actual_valid_recv=8192 observed_loss=-176 tokens_per_expert=[7680, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9023 actual_valid_recv=9088 observed_loss=-65 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1024, 7296] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=10730 actual_valid_recv=10752 observed_loss=-22 tokens_per_expert=[10752, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=5888 slack_slots=10496 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=10752 slack_slots=5632 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[780, 3248, 764, 111, 1373, 679, 989, 248] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[603, 1144, 216, 1396, 665, 586, 730, 2852] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1084, 3527, 307, 1877, 832, 196, 199, 170] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1233, 2215, 720, 1341, 993, 477, 1059, 154] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[473, 2324, 1706, 481, 1042, 319, 1662, 185] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[652, 2677, 339, 698, 2207, 232, 668, 719] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[124, 2424, 285, 78, 1598, 35, 3516, 132] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[874, 2739, 22, 481, 1891, 149, 765, 1271] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1084, 162, 1476, 0, 307, 666, 1877, 28, 670, 0, 196, 958, 171, 0, 170, 427] rerouted_tokens_per_ep_rank=[1246, 1476, 973, 1905, 670, 1154, 171, 597] avg=1024.0000 max=1905 max_rank=3 min=171 min_rank=6 imbalance_ratio=1.860352 max_load_reduction=6287 max_load_reduction_pct=0.767456 moved_to_echo=2592 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[780, 268, 1359, 0, 764, 613, 111, 143, 1105, 0, 679, 883, 846, 0, 248, 393] rerouted_tokens_per_ep_rank=[1048, 1359, 1377, 254, 1105, 1562, 846, 641] avg=1024.0000 max=1562 max_rank=5 min=254 min_rank=3 imbalance_ratio=1.525391 max_load_reduction=6630 max_load_reduction_pct=0.809326 moved_to_echo=4154 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1233, 194, 927, 0, 720, 418, 1341, 154, 799, 0, 477, 602, 905, 0, 154, 268] rerouted_tokens_per_ep_rank=[1427, 927, 1138, 1495, 799, 1079, 905, 422] avg=1024.0000 max=1495 max_rank=3 min=422 min_rank=7 imbalance_ratio=1.459961 max_load_reduction=6697 max_load_reduction_pct=0.817505 moved_to_echo=3205 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[603, 130, 479, 0, 216, 216, 1396, 106, 535, 0, 586, 311, 624, 0, 2852, 138] rerouted_tokens_per_ep_rank=[733, 479, 432, 1502, 535, 897, 624, 2990] avg=1024.0000 max=2990 max_rank=7 min=432 min_rank=2 imbalance_ratio=2.919922 max_load_reduction=5202 max_load_reduction_pct=0.635010 moved_to_echo=5046 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[652, 432, 1121, 0, 339, 505, 698, 97, 1775, 0, 232, 727, 571, 0, 719, 324] rerouted_tokens_per_ep_rank=[1084, 1121, 844, 795, 1775, 959, 571, 1043] avg=1024.0000 max=1775 max_rank=4 min=571 min_rank=6 imbalance_ratio=1.733398 max_load_reduction=6417 max_load_reduction_pct=0.783325 moved_to_echo=4348 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[124, 316, 1003, 0, 285, 460, 78, 516, 1282, 0, 35, 663, 3000, 0, 132, 298] rerouted_tokens_per_ep_rank=[440, 1003, 745, 594, 1282, 698, 3000, 430] avg=1024.0000 max=3000 max_rank=6 min=430 min_rank=7 imbalance_ratio=2.929688 max_load_reduction=5192 max_load_reduction_pct=0.633789 moved_to_echo=5410 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[473, 204, 974, 0, 1706, 438, 481, 241, 838, 0, 319, 631, 1421, 0, 185, 281] rerouted_tokens_per_ep_rank=[677, 974, 2144, 722, 838, 950, 1421, 466] avg=1024.0000 max=2144 max_rank=2 min=466 min_rank=7 imbalance_ratio=2.093750 max_load_reduction=6048 max_load_reduction_pct=0.738281 moved_to_echo=3675 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[874, 370, 1146, 0, 22, 517, 481, 111, 1521, 0, 149, 744, 654, 0, 1271, 332] rerouted_tokens_per_ep_rank=[1244, 1146, 539, 592, 1521, 893, 654, 1603] avg=1024.0000 max=1603 max_rank=7 min=539 min_rank=2 imbalance_ratio=1.565430 max_load_reduction=6589 max_load_reduction_pct=0.804321 moved_to_echo=4671 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7859 expected_overflow=0 exact_recv_per_rank=[7899, 8485, 8192, 7859, 8525, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7899 expected_overflow=0 exact_recv_per_rank=[7899, 8485, 8192, 7859, 8525, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8525 expected_overflow=0 exact_recv_per_rank=[7899, 8485, 8192, 7859, 8525, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7899, 8485, 8192, 7859, 8525, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7899, 8485, 8192, 7859, 8525, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7899, 8485, 8192, 7859, 8525, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8485 expected_overflow=0 exact_recv_per_rank=[7899, 8485, 8192, 7859, 8525, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7899, 8485, 8192, 7859, 8525, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=7899 actual_valid_recv=8064 observed_loss=-165 tokens_per_expert=[5888, 2176] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8525 actual_valid_recv=8576 observed_loss=-51 tokens_per_expert=[8576, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8485 actual_valid_recv=8576 observed_loss=-91 tokens_per_expert=[8576, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4480, 3840] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2688, 5632] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5760, 2560] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=7859 actual_valid_recv=7936 observed_loss=-77 tokens_per_expert=[6528, 1408] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8064 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8576 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8576 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7936 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1359, 2816, 144, 1358, 601, 729, 683, 502] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1229, 3020, 589, 1005, 417, 714, 70, 1148] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1307, 1583, 110, 1190, 1977, 1019, 20, 986] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1699, 746, 866, 772, 203, 1237, 930, 1739] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1382, 392, 300, 774, 561, 1793, 2753, 237] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[3123, 2455, 537, 367, 78, 1183, 341, 108] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[889, 1376, 993, 700, 3039, 161, 13, 1021] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1449, 791, 523, 1428, 866, 2688, 292, 155] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[811, 0, 2074, 0, 589, 946, 1005, 44, 417, 33, 637, 0, 70, 220, 1148, 198] rerouted_tokens_per_ep_rank=[811, 2074, 1535, 1049, 450, 637, 290, 1346] avg=1024.0000 max=2074 max_rank=1 min=290 min_rank=6 imbalance_ratio=2.025391 max_load_reduction=6118 max_load_reduction_pct=0.746826 moved_to_echo=2723 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1120, 0, 513, 0, 866, 233, 772, 77, 203, 58, 1102, 0, 930, 305, 1739, 274] rerouted_tokens_per_ep_rank=[1120, 513, 1099, 849, 261, 1102, 1235, 2013] avg=1024.0000 max=2013 max_rank=7 min=261 min_rank=4 imbalance_ratio=1.965820 max_load_reduction=6179 max_load_reduction_pct=0.754272 moved_to_echo=4611 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[862, 0, 1087, 0, 110, 496, 1190, 63, 1977, 48, 908, 0, 20, 234, 986, 211] rerouted_tokens_per_ep_rank=[862, 1087, 606, 1253, 2025, 908, 254, 1197] avg=1024.0000 max=2025 max_rank=4 min=254 min_rank=6 imbalance_ratio=1.977539 max_load_reduction=6167 max_load_reduction_pct=0.752808 moved_to_echo=4384 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[911, 0, 270, 0, 300, 122, 774, 112, 561, 84, 1597, 0, 2753, 248, 237, 223] rerouted_tokens_per_ep_rank=[911, 270, 422, 886, 645, 1597, 3001, 460] avg=1024.0000 max=3001 max_rank=6 min=270 min_rank=1 imbalance_ratio=2.930664 max_load_reduction=5191 max_load_reduction_pct=0.633667 moved_to_echo=5703 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[896, 0, 1934, 0, 144, 882, 1358, 45, 601, 34, 650, 0, 683, 244, 502, 219] rerouted_tokens_per_ep_rank=[896, 1934, 1026, 1403, 635, 650, 927, 721] avg=1024.0000 max=1934 max_rank=1 min=635 min_rank=4 imbalance_ratio=1.888672 max_load_reduction=6258 max_load_reduction_pct=0.763916 moved_to_echo=2933 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2050, 0, 1682, 0, 537, 773, 367, 79, 78, 59, 1045, 0, 341, 563, 108, 510] rerouted_tokens_per_ep_rank=[2050, 1682, 1310, 446, 137, 1045, 904, 618] avg=1024.0000 max=2050 max_rank=0 min=137 min_rank=4 imbalance_ratio=2.001953 max_load_reduction=6142 max_load_reduction_pct=0.749756 moved_to_echo=2704 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[587, 0, 945, 0, 993, 431, 700, 10, 3039, 7, 144, 0, 13, 159, 1021, 143] rerouted_tokens_per_ep_rank=[587, 945, 1424, 710, 3046, 144, 172, 1164] avg=1024.0000 max=3046 max_rank=4 min=144 min_rank=5 imbalance_ratio=2.974609 max_load_reduction=5146 max_load_reduction_pct=0.628174 moved_to_echo=4526 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[955, 0, 544, 0, 523, 247, 1428, 168, 866, 127, 2393, 0, 292, 260, 155, 234] rerouted_tokens_per_ep_rank=[955, 544, 770, 1596, 993, 2393, 552, 389] avg=1024.0000 max=2393 max_rank=5 min=389 min_rank=7 imbalance_ratio=2.336914 max_load_reduction=5799 max_load_reduction_pct=0.707886 moved_to_echo=4327 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9049, 8192, 8192, 8192, 8476, 7335, 7908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9049 expected_overflow=0 exact_recv_per_rank=[8192, 9049, 8192, 8192, 8192, 8476, 7335, 7908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9049, 8192, 8192, 8192, 8476, 7335, 7908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9049, 8192, 8192, 8192, 8476, 7335, 7908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8476 expected_overflow=0 exact_recv_per_rank=[8192, 9049, 8192, 8192, 8192, 8476, 7335, 7908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7335 expected_overflow=0 exact_recv_per_rank=[8192, 9049, 8192, 8192, 8192, 8476, 7335, 7908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9049, 8192, 8192, 8192, 8476, 7335, 7908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7908 expected_overflow=0 exact_recv_per_rank=[8192, 9049, 8192, 8192, 8192, 8476, 7335, 7908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7335 actual_valid_recv=7424 observed_loss=-89 tokens_per_expert=[5120, 2304] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4096, 4224] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7680, 640] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7808, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=7908 actual_valid_recv=8064 observed_loss=-156 tokens_per_expert=[6016, 2048] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8476 actual_valid_recv=8576 observed_loss=-100 tokens_per_expert=[8576, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=9049 actual_valid_recv=9088 observed_loss=-39 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=7424 slack_slots=8960 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8576 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8064 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[355, 717, 76, 129, 2488, 791, 1691, 1945] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[637, 1540, 629, 611, 3223, 243, 308, 1001] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[55, 8, 255, 731, 2901, 114, 3556, 572] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[876, 1704, 19, 581, 2344, 455, 285, 1928] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1927, 127, 0, 38, 2317, 280, 2037, 1466] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[3693, 937, 385, 1108, 819, 220, 18, 1012] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[268, 292, 31, 170, 3519, 143, 1722, 2047] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1456, 1678, 290, 81, 1509, 734, 39, 2405] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[564, 0, 1540, 73, 629, 1096, 611, 274, 1382, 0, 243, 745, 308, 0, 727, 0] rerouted_tokens_per_ep_rank=[564, 1613, 1725, 885, 1382, 988, 308, 727] avg=1024.0000 max=1725 max_rank=2 min=308 min_rank=6 imbalance_ratio=1.684570 max_load_reduction=6467 max_load_reduction_pct=0.789429 moved_to_echo=3405 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[314, 0, 717, 41, 76, 846, 129, 533, 1067, 0, 791, 575, 1691, 0, 1412, 0] rerouted_tokens_per_ep_rank=[314, 758, 922, 662, 1067, 1366, 1691, 1412] avg=1024.0000 max=1691 max_rank=6 min=314 min_rank=0 imbalance_ratio=1.651367 max_load_reduction=6501 max_load_reduction_pct=0.793579 moved_to_echo=5536 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[49, 0, 8, 6, 255, 987, 731, 156, 1244, 0, 114, 670, 3556, 0, 416, 0] rerouted_tokens_per_ep_rank=[49, 14, 1242, 887, 1244, 784, 3556, 416] avg=1024.0000 max=3556 max_rank=6 min=14 min_rank=1 imbalance_ratio=3.472656 max_load_reduction=4636 max_load_reduction_pct=0.565918 moved_to_echo=6000 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1700, 0, 127, 227, 0, 793, 38, 405, 984, 0, 280, 540, 2037, 0, 1061, 0] rerouted_tokens_per_ep_rank=[1700, 354, 793, 443, 984, 820, 2037, 1061] avg=1024.0000 max=2037 max_rank=6 min=354 min_rank=1 imbalance_ratio=1.989258 max_load_reduction=6155 max_load_reduction_pct=0.751343 moved_to_echo=4902 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3265, 0, 937, 428, 385, 278, 1108, 277, 352, 0, 220, 189, 18, 0, 735, 0] rerouted_tokens_per_ep_rank=[3265, 1365, 663, 1385, 352, 409, 18, 735] avg=1024.0000 max=3265 max_rank=0 min=18 min_rank=6 imbalance_ratio=3.188477 max_load_reduction=4927 max_load_reduction_pct=0.601440 moved_to_echo=1514 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1288, 0, 1678, 168, 290, 513, 81, 659, 648, 0, 734, 348, 39, 0, 1746, 0] rerouted_tokens_per_ep_rank=[1288, 1846, 803, 740, 648, 1082, 39, 1746] avg=1024.0000 max=1846 max_rank=1 min=39 min_rank=6 imbalance_ratio=1.802734 max_load_reduction=6346 max_load_reduction_pct=0.774658 moved_to_echo=3515 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[775, 0, 1704, 101, 19, 797, 581, 528, 1006, 0, 455, 541, 285, 0, 1400, 0] rerouted_tokens_per_ep_rank=[775, 1805, 816, 1109, 1006, 996, 285, 1400] avg=1024.0000 max=1805 max_rank=1 min=285 min_rank=6 imbalance_ratio=1.762695 max_load_reduction=6387 max_load_reduction_pct=0.779663 moved_to_echo=3687 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[237, 0, 292, 31, 31, 1197, 170, 561, 1509, 0, 143, 813, 1722, 0, 1486, 0] rerouted_tokens_per_ep_rank=[237, 323, 1228, 731, 1509, 956, 1722, 1486] avg=1024.0000 max=1722 max_rank=6 min=237 min_rank=0 imbalance_ratio=1.681641 max_load_reduction=6470 max_load_reduction_pct=0.789795 moved_to_echo=5673 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7401 expected_overflow=0 exact_recv_per_rank=[8192, 8078, 8192, 6842, 8192, 7401, 9656, 8983] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6842 expected_overflow=0 exact_recv_per_rank=[8192, 8078, 8192, 6842, 8192, 7401, 9656, 8983] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8078, 8192, 6842, 8192, 7401, 9656, 8983] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8078, 8192, 6842, 8192, 7401, 9656, 8983] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8078, 8192, 6842, 8192, 7401, 9656, 8983] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8078 expected_overflow=0 exact_recv_per_rank=[8192, 8078, 8192, 6842, 8192, 7401, 9656, 8983] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8983 expected_overflow=0 exact_recv_per_rank=[8192, 8078, 8192, 6842, 8192, 7401, 9656, 8983] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9656 expected_overflow=0 exact_recv_per_rank=[8192, 8078, 8192, 6842, 8192, 7401, 9656, 8983] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=6842 actual_valid_recv=6912 observed_loss=-70 tokens_per_expert=[3456, 3456] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1792, 6528] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7401 actual_valid_recv=7552 observed_loss=-151 tokens_per_expert=[3072, 4480] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8078 actual_valid_recv=8192 observed_loss=-114 tokens_per_expert=[7040, 1152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8983 actual_valid_recv=9088 observed_loss=-105 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9656 actual_valid_recv=9728 observed_loss=-72 tokens_per_expert=[9728, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=6912 slack_slots=9472 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9728 slack_slots=6656 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[2531, 205, 208, 33, 58, 3011, 938, 1208] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[2551, 42, 112, 2158, 15, 523, 2400, 391] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[2623, 9, 836, 54, 62, 137, 425, 4046] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[918, 213, 338, 124, 808, 1220, 2861, 1710] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[211, 15, 47, 154, 59, 2954, 1154, 3598] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[2921, 43, 44, 294, 2, 910, 990, 2988] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[2661, 907, 871, 481, 120, 580, 1049, 1523] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1905, 118, 964, 160, 98, 1457, 366, 3124] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1791, 0, 205, 222, 208, 740, 33, 725, 58, 452, 2286, 0, 938, 0, 534, 0] rerouted_tokens_per_ep_rank=[1791, 427, 948, 758, 510, 2286, 938, 534] avg=1024.0000 max=2286 max_rank=5 min=427 min_rank=1 imbalance_ratio=2.232422 max_load_reduction=5906 max_load_reduction_pct=0.720947 moved_to_echo=4268 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[150, 0, 15, 663, 47, 61, 154, 711, 59, 1349, 2243, 0, 1154, 0, 1586, 0] rerouted_tokens_per_ep_rank=[150, 678, 108, 865, 1408, 2243, 1154, 1586] avg=1024.0000 max=2243 max_rank=5 min=108 min_rank=2 imbalance_ratio=2.190430 max_load_reduction=5949 max_load_reduction_pct=0.726196 moved_to_echo=6391 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1806, 0, 42, 72, 112, 745, 2158, 126, 15, 146, 397, 0, 2400, 0, 173, 0] rerouted_tokens_per_ep_rank=[1806, 114, 857, 2284, 161, 397, 2400, 173] avg=1024.0000 max=2400 max_rank=6 min=114 min_rank=1 imbalance_ratio=2.343750 max_load_reduction=5792 max_load_reduction_pct=0.707031 moved_to_echo=3131 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1883, 0, 907, 280, 871, 778, 481, 139, 120, 571, 441, 0, 1049, 0, 672, 0] rerouted_tokens_per_ep_rank=[1883, 1187, 1649, 620, 691, 441, 1049, 672] avg=1024.0000 max=1883 max_rank=0 min=441 min_rank=5 imbalance_ratio=1.838867 max_load_reduction=6309 max_load_reduction_pct=0.770142 moved_to_echo=2853 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1857, 0, 9, 745, 836, 766, 54, 33, 62, 1517, 104, 0, 425, 0, 1784, 0] rerouted_tokens_per_ep_rank=[1857, 754, 1602, 87, 1579, 104, 425, 1784] avg=1024.0000 max=1857 max_rank=0 min=87 min_rank=3 imbalance_ratio=1.813477 max_load_reduction=6335 max_load_reduction_pct=0.773315 moved_to_echo=3892 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2063, 0, 43, 554, 44, 858, 294, 222, 2, 1123, 688, 0, 990, 0, 1311, 0] rerouted_tokens_per_ep_rank=[2063, 597, 902, 516, 1125, 688, 990, 1311] avg=1024.0000 max=2063 max_rank=0 min=516 min_rank=3 imbalance_ratio=2.014648 max_load_reduction=6129 max_load_reduction_pct=0.748169 moved_to_echo=4114 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[650, 0, 213, 315, 338, 268, 124, 293, 808, 641, 927, 0, 2861, 0, 754, 0] rerouted_tokens_per_ep_rank=[650, 528, 606, 417, 1449, 927, 2861, 754] avg=1024.0000 max=2861 max_rank=6 min=417 min_rank=3 imbalance_ratio=2.793945 max_load_reduction=5331 max_load_reduction_pct=0.650757 moved_to_echo=5991 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1349, 0, 118, 575, 964, 556, 160, 351, 98, 1171, 1106, 0, 366, 0, 1378, 0] rerouted_tokens_per_ep_rank=[1349, 693, 1520, 511, 1269, 1106, 366, 1378] avg=1024.0000 max=1520 max_rank=2 min=366 min_rank=6 imbalance_ratio=1.484375 max_load_reduction=6672 max_load_reduction_pct=0.814453 moved_to_echo=4119 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11549, 4978, 8192, 6058, 8192, 8192, 10183, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10183 expected_overflow=0 exact_recv_per_rank=[11549, 4978, 8192, 6058, 8192, 8192, 10183, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6058 expected_overflow=0 exact_recv_per_rank=[11549, 4978, 8192, 6058, 8192, 8192, 10183, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11549 expected_overflow=0 exact_recv_per_rank=[11549, 4978, 8192, 6058, 8192, 8192, 10183, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4978 expected_overflow=0 exact_recv_per_rank=[11549, 4978, 8192, 6058, 8192, 8192, 10183, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11549, 4978, 8192, 6058, 8192, 8192, 10183, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11549, 4978, 8192, 6058, 8192, 8192, 10183, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11549, 4978, 8192, 6058, 8192, 8192, 10183, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=4978 actual_valid_recv=5120 observed_loss=-142 tokens_per_expert=[1664, 3456] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=6058 actual_valid_recv=6272 observed_loss=-214 tokens_per_expert=[3584, 2688] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3456, 4864] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1280, 7040] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=10183 actual_valid_recv=10240 observed_loss=-57 tokens_per_expert=[10240, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=11549 actual_valid_recv=11648 observed_loss=-99 tokens_per_expert=[11648, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=5120 slack_slots=11264 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=6272 slack_slots=10112 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=10240 slack_slots=6144 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=11648 slack_slots=4736 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1108, 641, 1508, 2207, 597, 10, 14, 2107] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[2057, 143, 2589, 679, 2068, 157, 86, 413] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[2562, 7, 1955, 853, 1986, 105, 246, 478] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[187, 4042, 2515, 782, 227, 14, 42, 383] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[3038, 272, 235, 895, 1475, 89, 3, 2185] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1288, 446, 2536, 510, 456, 933, 7, 2016] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[221, 9, 3039, 54, 1145, 366, 537, 2821] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[3799, 39, 2290, 763, 784, 33, 17, 467] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[701, 0, 641, 344, 853, 0, 2207, 175, 597, 0, 10, 407, 14, 655, 1588, 0] rerouted_tokens_per_ep_rank=[701, 985, 853, 2382, 597, 417, 669, 1588] avg=1024.0000 max=2382 max_rank=3 min=417 min_rank=5 imbalance_ratio=2.326172 max_load_reduction=5810 max_load_reduction_pct=0.709229 moved_to_echo=3271 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1300, 0, 143, 67, 1465, 0, 679, 34, 2068, 0, 157, 757, 86, 1124, 312, 0] rerouted_tokens_per_ep_rank=[1300, 210, 1465, 713, 2068, 914, 1210, 312] avg=1024.0000 max=2068 max_rank=4 min=210 min_rank=1 imbalance_ratio=2.019531 max_load_reduction=6124 max_load_reduction_pct=0.747559 moved_to_echo=4504 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[814, 0, 446, 329, 1435, 0, 510, 167, 456, 0, 933, 474, 7, 1101, 1520, 0] rerouted_tokens_per_ep_rank=[814, 775, 1435, 677, 456, 1407, 1108, 1520] avg=1024.0000 max=1520 max_rank=7 min=456 min_rank=4 imbalance_ratio=1.484375 max_load_reduction=6672 max_load_reduction_pct=0.814453 moved_to_echo=4491 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[140, 0, 9, 460, 1719, 0, 54, 234, 1145, 0, 366, 81, 537, 1320, 2127, 0] rerouted_tokens_per_ep_rank=[140, 469, 1719, 288, 1145, 447, 1857, 2127] avg=1024.0000 max=2127 max_rank=7 min=140 min_rank=0 imbalance_ratio=2.077148 max_load_reduction=6065 max_load_reduction_pct=0.740356 moved_to_echo=5576 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1920, 0, 272, 356, 133, 0, 895, 181, 1475, 0, 89, 1118, 3, 102, 1648, 0] rerouted_tokens_per_ep_rank=[1920, 628, 133, 1076, 1475, 1207, 105, 1648] avg=1024.0000 max=1920 max_rank=0 min=105 min_rank=6 imbalance_ratio=1.875000 max_load_reduction=6272 max_load_reduction_pct=0.765625 moved_to_echo=4435 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1619, 0, 7, 78, 1106, 0, 853, 39, 1986, 0, 105, 943, 246, 849, 361, 0] rerouted_tokens_per_ep_rank=[1619, 85, 1106, 892, 1986, 1048, 1095, 361] avg=1024.0000 max=1986 max_rank=4 min=85 min_rank=1 imbalance_ratio=1.939453 max_load_reduction=6206 max_load_reduction_pct=0.757568 moved_to_echo=4490 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[115, 0, 4042, 65, 1420, 0, 782, 35, 227, 0, 14, 72, 42, 1095, 283, 0] rerouted_tokens_per_ep_rank=[115, 4107, 1420, 817, 227, 86, 1137, 283] avg=1024.0000 max=4107 max_rank=1 min=86 min_rank=5 imbalance_ratio=4.010742 max_load_reduction=4085 max_load_reduction_pct=0.498657 moved_to_echo=1733 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2401, 0, 39, 76, 1296, 0, 763, 38, 784, 0, 33, 1398, 17, 994, 353, 0] rerouted_tokens_per_ep_rank=[2401, 115, 1296, 801, 784, 1431, 1011, 353] avg=1024.0000 max=2401 max_rank=0 min=115 min_rank=1 imbalance_ratio=2.344727 max_load_reduction=5791 max_load_reduction_pct=0.706909 moved_to_echo=3579 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9427 expected_overflow=0 exact_recv_per_rank=[9010, 7374, 9427, 7646, 8738, 6957, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6957 expected_overflow=0 exact_recv_per_rank=[9010, 7374, 9427, 7646, 8738, 6957, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7374 expected_overflow=0 exact_recv_per_rank=[9010, 7374, 9427, 7646, 8738, 6957, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9010 expected_overflow=0 exact_recv_per_rank=[9010, 7374, 9427, 7646, 8738, 6957, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8738 expected_overflow=0 exact_recv_per_rank=[9010, 7374, 9427, 7646, 8738, 6957, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7646 expected_overflow=0 exact_recv_per_rank=[9010, 7374, 9427, 7646, 8738, 6957, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9010, 7374, 9427, 7646, 8738, 6957, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9010, 7374, 9427, 7646, 8738, 6957, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=6957 actual_valid_recv=7168 observed_loss=-211 tokens_per_expert=[1792, 5376] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=7374 actual_valid_recv=7424 observed_loss=-50 tokens_per_expert=[5632, 1792] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=7646 actual_valid_recv=7808 observed_loss=-162 tokens_per_expert=[6784, 1024] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8738 actual_valid_recv=8832 observed_loss=-94 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1024, 7296] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=9427 actual_valid_recv=9472 observed_loss=-45 tokens_per_expert=[9472, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=9010 actual_valid_recv=9088 observed_loss=-78 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7168 slack_slots=9216 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=7424 slack_slots=8960 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=9472 slack_slots=6912 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[801, 1387, 202, 2151, 1233, 246, 571, 1601] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1279, 58, 1354, 3871, 15, 84, 128, 1403] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1101, 292, 741, 3013, 423, 226, 113, 2283] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3634, 194, 142, 226, 2696, 2, 1064, 234] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[313, 73, 721, 2324, 1133, 385, 501, 2742] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[36, 799, 2936, 968, 268, 97, 2114, 974] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[284, 252, 492, 2454, 34, 764, 770, 3142] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4, 21, 1298, 847, 849, 56, 2211, 2906] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1101, 110, 292, 565, 741, 45, 1810, 0, 423, 230, 226, 1203, 113, 107, 1226, 0] rerouted_tokens_per_ep_rank=[1211, 857, 786, 1810, 653, 1429, 220, 1226] avg=1024.0000 max=1810 max_rank=3 min=220 min_rank=6 imbalance_ratio=1.767578 max_load_reduction=6382 max_load_reduction_pct=0.779053 moved_to_echo=3528 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[801, 77, 1387, 396, 202, 32, 1292, 0, 1233, 161, 246, 859, 571, 75, 860, 0] rerouted_tokens_per_ep_rank=[878, 1783, 234, 1292, 1394, 1105, 646, 860] avg=1024.0000 max=1783 max_rank=1 min=234 min_rank=2 imbalance_ratio=1.741211 max_load_reduction=6409 max_load_reduction_pct=0.782349 moved_to_echo=4005 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3634, 11, 194, 57, 142, 4, 136, 0, 2696, 23, 2, 90, 1064, 11, 128, 0] rerouted_tokens_per_ep_rank=[3645, 251, 146, 136, 2719, 92, 1075, 128] avg=1024.0000 max=3645 max_rank=0 min=92 min_rank=5 imbalance_ratio=3.559570 max_load_reduction=4547 max_load_reduction_pct=0.555054 moved_to_echo=4014 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[36, 47, 799, 241, 2936, 19, 582, 0, 268, 98, 97, 386, 2114, 45, 524, 0] rerouted_tokens_per_ep_rank=[83, 1040, 2955, 582, 366, 483, 2159, 524] avg=1024.0000 max=2955 max_rank=2 min=83 min_rank=0 imbalance_ratio=2.885742 max_load_reduction=5237 max_load_reduction_pct=0.639282 moved_to_echo=3532 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[313, 136, 73, 683, 721, 58, 1394, 0, 1133, 280, 385, 930, 501, 132, 1453, 0] rerouted_tokens_per_ep_rank=[449, 756, 779, 1394, 1413, 1315, 633, 1453] avg=1024.0000 max=1453 max_rank=7 min=449 min_rank=0 imbalance_ratio=1.418945 max_load_reduction=6739 max_load_reduction_pct=0.822632 moved_to_echo=4814 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1279, 67, 58, 347, 1354, 28, 2325, 0, 15, 141, 84, 1546, 128, 66, 754, 0] rerouted_tokens_per_ep_rank=[1346, 405, 1382, 2325, 156, 1630, 194, 754] avg=1024.0000 max=2325 max_rank=3 min=156 min_rank=4 imbalance_ratio=2.270508 max_load_reduction=5867 max_load_reduction_pct=0.716187 moved_to_echo=2734 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[284, 152, 252, 778, 492, 62, 1474, 0, 34, 316, 764, 980, 770, 148, 1686, 0] rerouted_tokens_per_ep_rank=[436, 1030, 554, 1474, 350, 1744, 918, 1686] avg=1024.0000 max=1744 max_rank=5 min=350 min_rank=4 imbalance_ratio=1.703125 max_load_reduction=6448 max_load_reduction_pct=0.787109 moved_to_echo=4698 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 140, 21, 719, 1298, 58, 509, 0, 849, 292, 56, 338, 2211, 136, 1561, 0] rerouted_tokens_per_ep_rank=[144, 740, 1356, 509, 1141, 394, 2347, 1561] avg=1024.0000 max=2347 max_rank=6 min=144 min_rank=0 imbalance_ratio=2.291992 max_load_reduction=5845 max_load_reduction_pct=0.713501 moved_to_echo=5443 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9522 expected_overflow=0 exact_recv_per_rank=[8192, 6862, 8192, 9522, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6862, 8192, 9522, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6862, 8192, 9522, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6862, 8192, 9522, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6862, 8192, 9522, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6862, 8192, 9522, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6862, 8192, 9522, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6862 expected_overflow=0 exact_recv_per_rank=[8192, 6862, 8192, 9522, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=6862 actual_valid_recv=7040 observed_loss=-178 tokens_per_expert=[3200, 3840] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6656, 1664] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7552, 768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7936, 384] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1920, 6400] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7552, 768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=9522 actual_valid_recv=9600 observed_loss=-78 tokens_per_expert=[9600, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=7040 slack_slots=9344 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=9600 slack_slots=6784 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1244, 2141, 1, 3, 2290, 2010, 425, 78] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3474, 3972, 109, 166, 3, 65, 284, 119] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[130, 3206, 5, 123, 57, 258, 2908, 1505] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[2968, 4063, 37, 171, 290, 215, 238, 210] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[690, 612, 195, 774, 1684, 324, 2009, 1904] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[2562, 3050, 9, 45, 3, 0, 67, 2456] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1724, 3469, 577, 85, 46, 164, 1005, 1122] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3458, 509, 0, 0, 665, 537, 3018, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[724, 0, 835, 0, 1, 739, 3, 567, 2290, 167, 2010, 353, 392, 0, 78, 33] rerouted_tokens_per_ep_rank=[724, 835, 740, 570, 2457, 2363, 392, 111] avg=1024.0000 max=2457 max_rank=4 min=111 min_rank=7 imbalance_ratio=2.399414 max_load_reduction=5735 max_load_reduction_pct=0.700073 moved_to_echo=5323 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[402, 0, 239, 0, 195, 211, 774, 162, 1684, 92, 324, 196, 1849, 0, 1904, 160] rerouted_tokens_per_ep_rank=[402, 239, 406, 936, 1776, 520, 1849, 2064] avg=1024.0000 max=2064 max_rank=7 min=239 min_rank=1 imbalance_ratio=2.015625 max_load_reduction=6128 max_load_reduction_pct=0.748047 moved_to_echo=6209 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1726, 0, 1585, 0, 37, 1402, 171, 1076, 290, 399, 215, 843, 220, 0, 210, 18] rerouted_tokens_per_ep_rank=[1726, 1585, 1439, 1247, 689, 1058, 220, 228] avg=1024.0000 max=1726 max_rank=0 min=220 min_rank=6 imbalance_ratio=1.685547 max_load_reduction=6466 max_load_reduction_pct=0.789307 moved_to_echo=2195 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2020, 0, 1549, 0, 109, 1371, 166, 1052, 3, 467, 65, 987, 262, 0, 119, 22] rerouted_tokens_per_ep_rank=[2020, 1549, 1480, 1218, 470, 1052, 262, 141] avg=1024.0000 max=2020 max_rank=0 min=141 min_rank=7 imbalance_ratio=1.972656 max_load_reduction=6172 max_load_reduction_pct=0.753418 moved_to_echo=1925 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1003, 0, 1353, 0, 577, 1197, 85, 919, 46, 231, 164, 490, 925, 0, 1122, 80] rerouted_tokens_per_ep_rank=[1003, 1353, 1774, 1004, 277, 654, 925, 1202] avg=1024.0000 max=1774 max_rank=2 min=277 min_rank=4 imbalance_ratio=1.732422 max_load_reduction=6418 max_load_reduction_pct=0.783447 moved_to_echo=3058 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1482, 0, 1181, 0, 9, 1057, 45, 812, 3, 348, 0, 732, 58, 0, 2456, 9] rerouted_tokens_per_ep_rank=[1482, 1181, 1066, 857, 351, 732, 58, 2465] avg=1024.0000 max=2465 max_rank=7 min=58 min_rank=6 imbalance_ratio=2.407227 max_load_reduction=5727 max_load_reduction_pct=0.699097 moved_to_echo=3606 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[77, 0, 1250, 0, 5, 1107, 123, 849, 57, 17, 258, 36, 2677, 0, 1505, 231] rerouted_tokens_per_ep_rank=[77, 1250, 1112, 972, 74, 294, 2677, 1736] avg=1024.0000 max=2677 max_rank=6 min=74 min_rank=4 imbalance_ratio=2.614258 max_load_reduction=5515 max_load_reduction_pct=0.673218 moved_to_echo=4781 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2012, 0, 200, 0, 0, 175, 0, 134, 665, 464, 537, 982, 2778, 0, 5, 240] rerouted_tokens_per_ep_rank=[2012, 200, 175, 134, 1129, 1519, 2778, 245] avg=1024.0000 max=2778 max_rank=6 min=134 min_rank=3 imbalance_ratio=2.712891 max_load_reduction=5414 max_load_reduction_pct=0.660889 moved_to_echo=5671 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9446 expected_overflow=0 exact_recv_per_rank=[9446, 8192, 8192, 6938, 7223, 8192, 9161, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9161 expected_overflow=0 exact_recv_per_rank=[9446, 8192, 8192, 6938, 7223, 8192, 9161, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9446, 8192, 8192, 6938, 7223, 8192, 9161, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6938 expected_overflow=0 exact_recv_per_rank=[9446, 8192, 8192, 6938, 7223, 8192, 9161, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9446, 8192, 8192, 6938, 7223, 8192, 9161, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7223 expected_overflow=0 exact_recv_per_rank=[9446, 8192, 8192, 6938, 7223, 8192, 9161, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9446, 8192, 8192, 6938, 7223, 8192, 9161, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9446, 8192, 8192, 6938, 7223, 8192, 9161, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7223 actual_valid_recv=7424 observed_loss=-201 tokens_per_expert=[5120, 2304] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=6938 actual_valid_recv=7040 observed_loss=-102 tokens_per_expert=[1408, 5632] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9161 actual_valid_recv=9216 observed_loss=-55 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3584, 4736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1024, 7296] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=9446 actual_valid_recv=9472 observed_loss=-26 tokens_per_expert=[9472, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7424, 896] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=7424 slack_slots=8960 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7040 slack_slots=9344 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=9472 slack_slots=6912 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[2800, 1187, 242, 847, 51, 189, 13, 2863] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[3203, 1451, 28, 409, 28, 98, 259, 2716] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[3758, 3820, 66, 338, 90, 49, 47, 24] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[992, 81, 3173, 2811, 0, 139, 21, 975] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1718, 153, 184, 322, 295, 2109, 1194, 2217] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[2539, 453, 164, 506, 110, 1, 444, 3975] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[3841, 752, 428, 78, 344, 167, 3, 2579] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[3550, 81, 2, 1853, 217, 1, 4, 2484] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[701, 0, 153, 26, 184, 485, 322, 127, 295, 541, 2109, 558, 1194, 476, 1021, 0] rerouted_tokens_per_ep_rank=[701, 179, 669, 449, 836, 2667, 1670, 1021] avg=1024.0000 max=2667 max_rank=5 min=179 min_rank=1 imbalance_ratio=2.604492 max_load_reduction=5525 max_load_reduction_pct=0.674438 moved_to_echo=6194 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1143, 0, 1187, 34, 242, 626, 847, 165, 51, 882, 189, 721, 13, 775, 1317, 0] rerouted_tokens_per_ep_rank=[1143, 1221, 868, 1012, 933, 910, 788, 1317] avg=1024.0000 max=1317 max_rank=7 min=788 min_rank=6 imbalance_ratio=1.286133 max_load_reduction=6875 max_load_reduction_pct=0.839233 moved_to_echo=3948 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1534, 0, 3820, 0, 66, 5, 338, 1, 90, 1183, 49, 6, 47, 1041, 12, 0] rerouted_tokens_per_ep_rank=[1534, 3820, 71, 339, 1273, 55, 1088, 12] avg=1024.0000 max=3820 max_rank=1 min=12 min_rank=7 imbalance_ratio=3.730469 max_load_reduction=4372 max_load_reduction_pct=0.533691 moved_to_echo=2428 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1307, 0, 1451, 32, 28, 594, 409, 156, 28, 1009, 98, 684, 259, 887, 1250, 0] rerouted_tokens_per_ep_rank=[1307, 1483, 622, 565, 1037, 782, 1146, 1250] avg=1024.0000 max=1483 max_rank=1 min=565 min_rank=3 imbalance_ratio=1.448242 max_load_reduction=6709 max_load_reduction_pct=0.818970 moved_to_echo=4215 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1037, 0, 453, 47, 164, 870, 506, 229, 110, 799, 1, 1001, 444, 703, 1828, 0] rerouted_tokens_per_ep_rank=[1037, 500, 1034, 735, 909, 1002, 1147, 1828] avg=1024.0000 max=1828 max_rank=7 min=500 min_rank=1 imbalance_ratio=1.785156 max_load_reduction=6364 max_load_reduction_pct=0.776855 moved_to_echo=4886 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[399, 0, 81, 16, 3173, 218, 2811, 59, 0, 315, 139, 250, 21, 278, 432, 0] rerouted_tokens_per_ep_rank=[399, 97, 3391, 2870, 315, 389, 299, 432] avg=1024.0000 max=3391 max_rank=2 min=97 min_rank=1 imbalance_ratio=3.311523 max_load_reduction=4801 max_load_reduction_pct=0.586060 moved_to_echo=1435 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1567, 0, 752, 30, 428, 564, 78, 148, 344, 1210, 167, 649, 3, 1064, 1188, 0] rerouted_tokens_per_ep_rank=[1567, 782, 992, 226, 1554, 816, 1067, 1188] avg=1024.0000 max=1567 max_rank=0 min=226 min_rank=3 imbalance_ratio=1.530273 max_load_reduction=6625 max_load_reduction_pct=0.808716 moved_to_echo=4625 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1449, 0, 81, 29, 2, 543, 1853, 143, 217, 1118, 1, 625, 4, 983, 1144, 0] rerouted_tokens_per_ep_rank=[1449, 110, 545, 1996, 1335, 626, 987, 1144] avg=1024.0000 max=1996 max_rank=3 min=110 min_rank=1 imbalance_ratio=1.949219 max_load_reduction=6196 max_load_reduction_pct=0.756348 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7247 expected_overflow=0 exact_recv_per_rank=[9137, 8192, 8192, 8192, 8192, 7247, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9137, 8192, 8192, 8192, 8192, 7247, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9137, 8192, 8192, 8192, 8192, 7247, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9137, 8192, 8192, 8192, 8192, 7247, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9137, 8192, 8192, 8192, 8192, 7247, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9137, 8192, 8192, 8192, 8192, 7247, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9137, 8192, 8192, 8192, 8192, 7247, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9137 expected_overflow=0 exact_recv_per_rank=[9137, 8192, 8192, 8192, 8192, 7247, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4352, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2048, 6272] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7247 actual_valid_recv=7424 observed_loss=-177 tokens_per_expert=[2816, 4608] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7168, 1152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=9137 actual_valid_recv=9216 observed_loss=-79 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[8064, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7424 slack_slots=8960 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1355, 0, 50, 3854, 358, 1443, 1132, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1518, 571, 142, 3168, 925, 985, 882, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1059, 44, 97, 2204, 49, 2930, 1696, 113] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[896, 2776, 1314, 1939, 986, 64, 135, 82] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[497, 723, 791, 3492, 1131, 1268, 141, 149] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3311, 385, 243, 1267, 185, 1656, 998, 147] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[858, 336, 204, 938, 1261, 2017, 2067, 511] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[63, 890, 654, 3653, 1151, 1059, 661, 61] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1287, 0, 0, 248, 50, 882, 1633, 0, 358, 159, 1036, 0, 1132, 68, 0, 1339] rerouted_tokens_per_ep_rank=[1287, 248, 932, 1633, 517, 1036, 1200, 1339] avg=1024.0000 max=1633 max_rank=3 min=248 min_rank=1 imbalance_ratio=1.594727 max_load_reduction=6559 max_load_reduction_pct=0.800659 moved_to_echo=4092 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[851, 0, 2776, 11, 1314, 443, 823, 0, 986, 7, 46, 0, 135, 45, 82, 673] rerouted_tokens_per_ep_rank=[851, 2787, 1757, 823, 993, 46, 180, 755] avg=1024.0000 max=2787 max_rank=1 min=46 min_rank=5 imbalance_ratio=2.721680 max_load_reduction=5405 max_load_reduction_pct=0.659790 moved_to_echo=1974 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1004, 0, 44, 509, 97, 508, 927, 0, 49, 328, 2093, 0, 1696, 55, 113, 769] rerouted_tokens_per_ep_rank=[1004, 553, 605, 927, 377, 2093, 1751, 882] avg=1024.0000 max=2093 max_rank=5 min=377 min_rank=4 imbalance_ratio=2.043945 max_load_reduction=6099 max_load_reduction_pct=0.744507 moved_to_echo=5103 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[473, 0, 723, 218, 791, 799, 1480, 0, 1131, 139, 911, 0, 141, 24, 149, 1213] rerouted_tokens_per_ep_rank=[473, 941, 1590, 1480, 1270, 911, 165, 1362] avg=1024.0000 max=1590 max_rank=2 min=165 min_rank=6 imbalance_ratio=1.552734 max_load_reduction=6602 max_load_reduction_pct=0.805908 moved_to_echo=3708 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1442, 0, 571, 169, 142, 725, 1343, 0, 925, 108, 708, 0, 882, 76, 1, 1100] rerouted_tokens_per_ep_rank=[1442, 740, 867, 1343, 1033, 708, 958, 1101] avg=1024.0000 max=1442 max_rank=0 min=708 min_rank=5 imbalance_ratio=1.408203 max_load_reduction=6750 max_load_reduction_pct=0.823975 moved_to_echo=3800 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3145, 0, 385, 285, 243, 290, 537, 0, 185, 182, 1189, 0, 998, 166, 147, 440] rerouted_tokens_per_ep_rank=[3145, 670, 533, 537, 367, 1189, 1164, 587] avg=1024.0000 max=3145 max_rank=0 min=367 min_rank=4 imbalance_ratio=3.071289 max_load_reduction=5047 max_load_reduction_pct=0.616089 moved_to_echo=3307 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[60, 0, 890, 182, 654, 836, 1548, 0, 1151, 116, 761, 0, 661, 3, 61, 1269] rerouted_tokens_per_ep_rank=[60, 1072, 1490, 1548, 1267, 761, 664, 1330] avg=1024.0000 max=1548 max_rank=3 min=60 min_rank=0 imbalance_ratio=1.511719 max_load_reduction=6644 max_load_reduction_pct=0.811035 moved_to_echo=4022 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[815, 0, 336, 347, 204, 214, 399, 0, 1261, 222, 1448, 0, 2067, 43, 511, 325] rerouted_tokens_per_ep_rank=[815, 683, 418, 399, 1483, 1448, 2110, 836] avg=1024.0000 max=2110 max_rank=6 min=399 min_rank=3 imbalance_ratio=2.060547 max_load_reduction=6082 max_load_reduction_pct=0.742432 moved_to_echo=5877 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7307 expected_overflow=0 exact_recv_per_rank=[9077, 7694, 8192, 8690, 7307, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8690 expected_overflow=0 exact_recv_per_rank=[9077, 7694, 8192, 8690, 7307, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9077, 7694, 8192, 8690, 7307, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9077, 7694, 8192, 8690, 7307, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9077, 7694, 8192, 8690, 7307, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7694 expected_overflow=0 exact_recv_per_rank=[9077, 7694, 8192, 8690, 7307, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9077, 7694, 8192, 8690, 7307, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9077 expected_overflow=0 exact_recv_per_rank=[9077, 7694, 8192, 8690, 7307, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3584, 4736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=9077 actual_valid_recv=9088 observed_loss=-11 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7307 actual_valid_recv=7424 observed_loss=-117 tokens_per_expert=[6144, 1280] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=7694 actual_valid_recv=7808 observed_loss=-114 tokens_per_expert=[5760, 2048] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7808, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8690 actual_valid_recv=8704 observed_loss=-14 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=7424 slack_slots=8960 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[109, 199, 605, 2435, 781, 117, 109, 3837] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[2131, 23, 130, 3593, 316, 1564, 27, 408] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[86, 1081, 873, 2208, 316, 749, 110, 2769] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[3636, 399, 8, 3722, 59, 285, 81, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[3388, 2808, 4, 265, 1665, 51, 0, 11] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[2891, 413, 0, 1615, 25, 1042, 387, 1819] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1210, 12, 0, 3578, 236, 0, 63, 3093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[610, 115, 9, 3065, 2249, 437, 370, 1337] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2616, 0, 399, 0, 8, 952, 1490, 0, 59, 0, 285, 1020, 81, 1280, 2, 0] rerouted_tokens_per_ep_rank=[2616, 399, 960, 1490, 59, 1305, 1361, 2] avg=1024.0000 max=2616 max_rank=0 min=2 min_rank=7 imbalance_ratio=2.554688 max_load_reduction=5576 max_load_reduction_pct=0.680664 moved_to_echo=2727 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1533, 0, 23, 78, 130, 919, 1439, 0, 316, 78, 1564, 598, 27, 1235, 252, 0] rerouted_tokens_per_ep_rank=[1533, 101, 1049, 1439, 394, 2162, 1262, 252] avg=1024.0000 max=2162 max_rank=5 min=101 min_rank=1 imbalance_ratio=2.111328 max_load_reduction=6030 max_load_reduction_pct=0.736084 moved_to_echo=4070 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[62, 0, 1081, 529, 873, 565, 884, 0, 316, 530, 749, 24, 110, 759, 1710, 0] rerouted_tokens_per_ep_rank=[62, 1610, 1438, 884, 846, 773, 869, 1710] avg=1024.0000 max=1710 max_rank=7 min=62 min_rank=0 imbalance_ratio=1.669922 max_load_reduction=6482 max_load_reduction_pct=0.791260 moved_to_echo=4198 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[76, 0, 199, 737, 605, 628, 966, 0, 781, 739, 117, 33, 109, 841, 2361, 0] rerouted_tokens_per_ep_rank=[76, 936, 1233, 966, 1520, 150, 950, 2361] avg=1024.0000 max=2361 max_rank=7 min=76 min_rank=0 imbalance_ratio=2.305664 max_load_reduction=5831 max_load_reduction_pct=0.711792 moved_to_echo=4981 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[439, 0, 115, 255, 9, 784, 1227, 0, 2249, 256, 437, 171, 370, 1054, 826, 0] rerouted_tokens_per_ep_rank=[439, 370, 793, 1227, 2505, 608, 1424, 826] avg=1024.0000 max=2505 max_rank=4 min=370 min_rank=1 imbalance_ratio=2.446289 max_load_reduction=5687 max_load_reduction_pct=0.694214 moved_to_echo=5363 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2080, 0, 413, 347, 0, 413, 647, 0, 25, 348, 1042, 811, 387, 555, 1124, 0] rerouted_tokens_per_ep_rank=[2080, 760, 413, 647, 373, 1853, 942, 1124] avg=1024.0000 max=2080 max_rank=0 min=373 min_rank=4 imbalance_ratio=2.031250 max_load_reduction=6112 max_load_reduction_pct=0.746094 moved_to_echo=4292 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2437, 0, 2808, 2, 4, 67, 107, 0, 1665, 2, 51, 951, 0, 91, 7, 0] rerouted_tokens_per_ep_rank=[2437, 2810, 71, 107, 1667, 1002, 91, 7] avg=1024.0000 max=2810 max_rank=1 min=7 min_rank=7 imbalance_ratio=2.744141 max_load_reduction=5382 max_load_reduction_pct=0.656982 moved_to_echo=2767 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[871, 0, 12, 591, 0, 916, 1432, 0, 236, 592, 0, 339, 63, 1230, 1910, 0] rerouted_tokens_per_ep_rank=[871, 603, 916, 1432, 828, 339, 1293, 1910] avg=1024.0000 max=1910 max_rank=7 min=339 min_rank=5 imbalance_ratio=1.865234 max_load_reduction=6282 max_load_reduction_pct=0.766846 moved_to_echo=4370 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10114, 7589, 6873, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6873 expected_overflow=0 exact_recv_per_rank=[10114, 7589, 6873, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10114, 7589, 6873, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10114 expected_overflow=0 exact_recv_per_rank=[10114, 7589, 6873, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10114, 7589, 6873, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7589 expected_overflow=0 exact_recv_per_rank=[10114, 7589, 6873, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10114, 7589, 6873, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10114, 7589, 6873, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=7589 actual_valid_recv=7680 observed_loss=-91 tokens_per_expert=[5120, 2560] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4352, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=6873 actual_valid_recv=6912 observed_loss=-39 tokens_per_expert=[1664, 5248] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5760, 2560] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=10114 actual_valid_recv=10240 observed_loss=-126 tokens_per_expert=[10240, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=6912 slack_slots=9472 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=10240 slack_slots=6144 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[24, 1, 3602, 1876, 189, 6, 2, 2492] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 730, 2904, 860, 1169, 49, 373, 2107] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[51, 3763, 355, 821, 535, 2421, 12, 234] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[2829, 26, 114, 1069, 3670, 367, 1, 116] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[327, 5, 60, 2488, 284, 1740, 864, 2424] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[32, 185, 3669, 3641, 361, 8, 13, 283] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[528, 106, 617, 3293, 902, 2610, 12, 124] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4, 4, 737, 3951, 3050, 222, 155, 69] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[51, 138, 3763, 74, 281, 0, 375, 0, 477, 0, 2421, 40, 12, 308, 234, 18] rerouted_tokens_per_ep_rank=[189, 3837, 281, 375, 477, 2461, 320, 252] avg=1024.0000 max=3837 max_rank=1 min=189 min_rank=0 imbalance_ratio=3.747070 max_load_reduction=4355 max_load_reduction_pct=0.531616 moved_to_echo=3510 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[24, 317, 1, 751, 2851, 0, 855, 0, 169, 0, 6, 14, 2, 704, 2492, 6] rerouted_tokens_per_ep_rank=[341, 752, 2851, 855, 169, 20, 706, 2498] avg=1024.0000 max=2851 max_rank=2 min=20 min_rank=5 imbalance_ratio=2.784180 max_load_reduction=5341 max_load_reduction_pct=0.651978 moved_to_echo=3393 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 145, 730, 605, 2299, 0, 393, 0, 1042, 0, 49, 88, 373, 322, 2107, 39] rerouted_tokens_per_ep_rank=[145, 1335, 2299, 393, 1042, 137, 695, 2146] avg=1024.0000 max=2299 max_rank=2 min=137 min_rank=5 imbalance_ratio=2.245117 max_load_reduction=5893 max_load_reduction_pct=0.719360 moved_to_echo=4020 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[327, 421, 5, 12, 48, 0, 1133, 0, 254, 0, 1740, 21, 864, 934, 2424, 9] rerouted_tokens_per_ep_rank=[748, 17, 48, 1133, 254, 1761, 1798, 2433] avg=1024.0000 max=2433 max_rank=7 min=17 min_rank=1 imbalance_ratio=2.375977 max_load_reduction=5759 max_load_reduction_pct=0.703003 moved_to_echo=6246 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[528, 562, 106, 133, 484, 0, 1490, 0, 796, 0, 2610, 72, 12, 1241, 124, 34] rerouted_tokens_per_ep_rank=[1090, 239, 484, 1490, 796, 2682, 1253, 158] avg=1024.0000 max=2682 max_rank=5 min=158 min_rank=7 imbalance_ratio=2.619141 max_load_reduction=5510 max_load_reduction_pct=0.672607 moved_to_echo=4889 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[32, 616, 185, 765, 2904, 0, 1658, 0, 322, 0, 8, 27, 13, 1367, 283, 12] rerouted_tokens_per_ep_rank=[648, 950, 2904, 1658, 322, 35, 1380, 295] avg=1024.0000 max=2904 max_rank=2 min=35 min_rank=5 imbalance_ratio=2.835938 max_load_reduction=5288 max_load_reduction_pct=0.645508 moved_to_echo=2032 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 668, 4, 153, 584, 0, 1800, 0, 2718, 0, 222, 230, 155, 1483, 69, 102] rerouted_tokens_per_ep_rank=[672, 157, 584, 1800, 2718, 452, 1638, 171] avg=1024.0000 max=2718 max_rank=4 min=157 min_rank=1 imbalance_ratio=2.654297 max_load_reduction=5474 max_load_reduction_pct=0.668213 moved_to_echo=4979 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2829, 180, 26, 23, 91, 0, 488, 0, 3270, 0, 367, 277, 1, 401, 116, 123] rerouted_tokens_per_ep_rank=[3009, 49, 91, 488, 3270, 644, 402, 239] avg=1024.0000 max=3270 max_rank=4 min=49 min_rank=1 imbalance_ratio=3.193359 max_load_reduction=4922 max_load_reduction_pct=0.600830 moved_to_echo=4555 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[6842, 7336, 9542, 8192, 9048, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7336 expected_overflow=0 exact_recv_per_rank=[6842, 7336, 9542, 8192, 9048, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[6842, 7336, 9542, 8192, 9048, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[6842, 7336, 9542, 8192, 9048, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6842 expected_overflow=0 exact_recv_per_rank=[6842, 7336, 9542, 8192, 9048, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[6842, 7336, 9542, 8192, 9048, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9542 expected_overflow=0 exact_recv_per_rank=[6842, 7336, 9542, 8192, 9048, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9048 expected_overflow=0 exact_recv_per_rank=[6842, 7336, 9542, 8192, 9048, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=6842 actual_valid_recv=6912 observed_loss=-70 tokens_per_expert=[3840, 3072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7424, 896] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=7336 actual_valid_recv=7424 observed_loss=-88 tokens_per_expert=[4864, 2560] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=9048 actual_valid_recv=9088 observed_loss=-40 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7936, 384] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=9542 actual_valid_recv=9600 observed_loss=-58 tokens_per_expert=[9600, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1536, 6784] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=6912 slack_slots=9472 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=7424 slack_slots=8960 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=9600 slack_slots=6784 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[275, 75, 1, 3139, 26, 4042, 165, 469] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[16, 525, 4088, 1900, 490, 261, 3, 909] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[9, 4, 2320, 3183, 865, 258, 1414, 139] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 672, 71, 33, 409, 4065, 2891, 51] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[2, 2320, 849, 108, 443, 940, 3351, 179] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[2, 244, 3375, 149, 39, 4021, 0, 362] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[207, 67, 712, 401, 1, 3669, 3060, 75] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[37, 3088, 3468, 311, 71, 248, 12, 957] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[275, 1765, 75, 351, 1, 0, 2788, 0, 26, 0, 2277, 0, 127, 0, 469, 38] rerouted_tokens_per_ep_rank=[2040, 426, 1, 2788, 26, 2277, 127, 507] avg=1024.0000 max=2788 max_rank=3 min=1 min_rank=2 imbalance_ratio=2.722656 max_load_reduction=5404 max_load_reduction_pct=0.659668 moved_to_echo=2937 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[16, 113, 525, 212, 2940, 0, 1688, 0, 490, 1148, 148, 0, 3, 0, 909, 0] rerouted_tokens_per_ep_rank=[129, 737, 2940, 1688, 1638, 148, 3, 909] avg=1024.0000 max=2940 max_rank=2 min=3 min_rank=6 imbalance_ratio=2.871094 max_load_reduction=5252 max_load_reduction_pct=0.641113 moved_to_echo=2698 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1775, 672, 3, 52, 0, 30, 0, 409, 19, 2290, 0, 2218, 0, 51, 673] rerouted_tokens_per_ep_rank=[1775, 675, 52, 30, 428, 2290, 2218, 724] avg=1024.0000 max=2290 max_rank=5 min=30 min_rank=3 imbalance_ratio=2.236328 max_load_reduction=5902 max_load_reduction_pct=0.720459 moved_to_echo=5660 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 410, 2320, 12, 611, 0, 96, 0, 443, 238, 530, 0, 2571, 0, 179, 780] rerouted_tokens_per_ep_rank=[412, 2332, 611, 96, 681, 530, 2571, 959] avg=1024.0000 max=2571 max_rank=6 min=96 min_rank=3 imbalance_ratio=2.510742 max_load_reduction=5621 max_load_reduction_pct=0.686157 moved_to_echo=4741 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[9, 116, 4, 360, 1664, 0, 2823, 0, 865, 656, 142, 0, 1081, 0, 139, 333] rerouted_tokens_per_ep_rank=[125, 364, 1664, 2823, 1521, 142, 1081, 472] avg=1024.0000 max=2823 max_rank=3 min=125 min_rank=0 imbalance_ratio=2.756836 max_load_reduction=5369 max_load_reduction_pct=0.655396 moved_to_echo=3216 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 1755, 244, 16, 2428, 0, 133, 0, 39, 947, 2266, 0, 0, 0, 362, 0] rerouted_tokens_per_ep_rank=[1757, 260, 2428, 133, 986, 2266, 0, 362] avg=1024.0000 max=2428 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.371094 max_load_reduction=5764 max_load_reduction_pct=0.703613 moved_to_echo=3614 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[207, 1602, 67, 44, 513, 0, 357, 0, 1, 199, 2067, 0, 2347, 0, 75, 713] rerouted_tokens_per_ep_rank=[1809, 111, 513, 357, 200, 2067, 2347, 788] avg=1024.0000 max=2347 max_rank=6 min=111 min_rank=1 imbalance_ratio=2.291992 max_load_reduction=5845 max_load_reduction_pct=0.713501 moved_to_echo=5402 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[37, 108, 3088, 34, 2495, 0, 277, 0, 71, 973, 140, 0, 10, 0, 957, 2] rerouted_tokens_per_ep_rank=[145, 3122, 2495, 277, 1044, 140, 10, 959] avg=1024.0000 max=3122 max_rank=1 min=10 min_rank=6 imbalance_ratio=3.048828 max_load_reduction=5070 max_load_reduction_pct=0.618896 moved_to_echo=2153 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9860 expected_overflow=0 exact_recv_per_rank=[8192, 8027, 10704, 8192, 6524, 9860, 8357, 5680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6524 expected_overflow=0 exact_recv_per_rank=[8192, 8027, 10704, 8192, 6524, 9860, 8357, 5680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8027, 10704, 8192, 6524, 9860, 8357, 5680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8027, 10704, 8192, 6524, 9860, 8357, 5680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8357 expected_overflow=0 exact_recv_per_rank=[8192, 8027, 10704, 8192, 6524, 9860, 8357, 5680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10704 expected_overflow=0 exact_recv_per_rank=[8192, 8027, 10704, 8192, 6524, 9860, 8357, 5680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8027 expected_overflow=0 exact_recv_per_rank=[8192, 8027, 10704, 8192, 6524, 9860, 8357, 5680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5680 expected_overflow=0 exact_recv_per_rank=[8192, 8027, 10704, 8192, 6524, 9860, 8357, 5680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=5680 actual_valid_recv=5760 observed_loss=-80 tokens_per_expert=[3200, 2560] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8357 actual_valid_recv=8448 observed_loss=-91 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=6524 actual_valid_recv=6656 observed_loss=-132 tokens_per_expert=[2432, 4224] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[640, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=10704 actual_valid_recv=10752 observed_loss=-48 tokens_per_expert=[10752, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8027 actual_valid_recv=8192 observed_loss=-165 tokens_per_expert=[7040, 1152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=9860 actual_valid_recv=9984 observed_loss=-124 tokens_per_expert=[9984, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=5760 slack_slots=10624 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=6656 slack_slots=9728 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=10752 slack_slots=5632 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=9984 slack_slots=6400 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[10, 3344, 95, 48, 24, 2997, 19, 1655] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1629, 2529, 2686, 264, 240, 14, 736, 94] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[70, 2685, 1, 228, 3086, 50, 2040, 32] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[12, 1199, 1, 83, 180, 3195, 2, 3520] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1955, 243, 1961, 7, 2017, 364, 1215, 430] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[17, 3376, 164, 43, 104, 99, 538, 3851] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1, 4073, 500, 342, 534, 31, 177, 2534] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[72, 3093, 89, 13, 122, 15, 921, 3867] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1629, 544, 1104, 0, 2686, 11, 264, 881, 240, 11, 14, 8, 736, 14, 50, 0] rerouted_tokens_per_ep_rank=[2173, 1104, 2697, 1145, 251, 22, 750, 50] avg=1024.0000 max=2697 max_rank=2 min=22 min_rank=5 imbalance_ratio=2.633789 max_load_reduction=5495 max_load_reduction_pct=0.670776 moved_to_echo=1073 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[10, 720, 1458, 0, 95, 200, 48, 1166, 24, 195, 2997, 147, 19, 263, 850, 0] rerouted_tokens_per_ep_rank=[730, 1458, 295, 1214, 219, 3144, 282, 850] avg=1024.0000 max=3144 max_rank=5 min=219 min_rank=4 imbalance_ratio=3.070312 max_load_reduction=5048 max_load_reduction_pct=0.616211 moved_to_echo=4495 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[17, 727, 1472, 0, 164, 466, 43, 1177, 104, 454, 99, 343, 538, 612, 1976, 0] rerouted_tokens_per_ep_rank=[744, 1472, 630, 1220, 558, 442, 1150, 1976] avg=1024.0000 max=1976 max_rank=7 min=442 min_rank=5 imbalance_ratio=1.929688 max_load_reduction=6216 max_load_reduction_pct=0.758789 moved_to_echo=4126 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 877, 1776, 0, 500, 306, 342, 1420, 534, 298, 31, 226, 177, 403, 1301, 0] rerouted_tokens_per_ep_rank=[878, 1776, 806, 1762, 832, 257, 580, 1301] avg=1024.0000 max=1776 max_rank=1 min=257 min_rank=5 imbalance_ratio=1.734375 max_load_reduction=6416 max_load_reduction_pct=0.783203 moved_to_echo=2970 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[70, 582, 1163, 0, 1, 6, 228, 940, 3086, 6, 50, 6, 2040, 9, 5, 0] rerouted_tokens_per_ep_rank=[652, 1163, 7, 1168, 3092, 56, 2049, 5] avg=1024.0000 max=3092 max_rank=4 min=5 min_rank=7 imbalance_ratio=3.019531 max_load_reduction=5100 max_load_reduction_pct=0.622559 moved_to_echo=5202 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[12, 258, 523, 0, 1, 426, 83, 418, 180, 415, 3195, 314, 2, 560, 1805, 0] rerouted_tokens_per_ep_rank=[270, 523, 427, 501, 595, 3509, 562, 1805] avg=1024.0000 max=3509 max_rank=5 min=270 min_rank=0 imbalance_ratio=3.426758 max_load_reduction=4683 max_load_reduction_pct=0.571655 moved_to_echo=6471 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1955, 52, 107, 0, 1961, 52, 7, 84, 2017, 50, 364, 38, 1215, 68, 222, 0] rerouted_tokens_per_ep_rank=[2007, 107, 2013, 91, 2067, 402, 1283, 222] avg=1024.0000 max=2067 max_rank=4 min=91 min_rank=3 imbalance_ratio=2.018555 max_load_reduction=6125 max_load_reduction_pct=0.747681 moved_to_echo=3974 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[72, 666, 1349, 0, 89, 468, 13, 1078, 122, 456, 15, 345, 921, 615, 1983, 0] rerouted_tokens_per_ep_rank=[738, 1349, 557, 1091, 578, 360, 1536, 1983] avg=1024.0000 max=1983 max_rank=7 min=360 min_rank=5 imbalance_ratio=1.936523 max_load_reduction=6209 max_load_reduction_pct=0.757935 moved_to_echo=4457 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8952, 7432, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8952, 7432, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8952, 7432, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8952, 7432, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8952 expected_overflow=0 exact_recv_per_rank=[8192, 8952, 7432, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8952, 7432, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7432 expected_overflow=0 exact_recv_per_rank=[8192, 8952, 7432, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8952, 7432, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3840, 4480] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5760, 2560] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6784, 1536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=7432 actual_valid_recv=7552 observed_loss=-120 tokens_per_expert=[5504, 2048] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6400, 1920] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8952 actual_valid_recv=8960 observed_loss=-8 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True + [2026-08-30 04:54:00] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 39500.3 | throughput per GPU (TFLOP/s/GPU): 8.4 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.104376E+01 | loss scale: 1.0 | grad norm: 34.297 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 63763.1845703125 | max allocated: 92616.9560546875 | reserved: 94730.0 | max reserved: 94730.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[64, 2716, 607, 203, 2048, 518, 812, 1224] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[96, 3315, 793, 293, 1479, 360, 937, 919] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[52, 3148, 326, 245, 1876, 315, 1165, 1065] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[193, 3358, 527, 421, 1210, 442, 1090, 951] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[121, 3006, 451, 243, 1568, 501, 888, 1414] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[69, 3288, 360, 263, 1705, 195, 1191, 1121] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[136, 3568, 661, 257, 1435, 299, 519, 1317] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[126, 2921, 675, 378, 1740, 506, 971, 875] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[64, 786, 881, 0, 607, 582, 203, 631, 1466, 0, 518, 418, 812, 85, 1139, 0] rerouted_tokens_per_ep_rank=[850, 881, 1189, 834, 1466, 936, 897, 1139] avg=1024.0000 max=1466 max_rank=4 min=834 min_rank=3 imbalance_ratio=1.431641 max_load_reduction=6726 max_load_reduction_pct=0.821045 moved_to_echo=4438 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[52, 911, 1020, 0, 326, 533, 245, 732, 1343, 0, 315, 485, 1165, 74, 991, 0] rerouted_tokens_per_ep_rank=[963, 1020, 859, 977, 1343, 800, 1239, 991] avg=1024.0000 max=1343 max_rank=4 min=800 min_rank=5 imbalance_ratio=1.311523 max_load_reduction=6849 max_load_reduction_pct=0.836060 moved_to_echo=4373 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[193, 972, 1088, 0, 527, 344, 421, 781, 866, 0, 442, 517, 1090, 66, 885, 0] rerouted_tokens_per_ep_rank=[1165, 1088, 871, 1202, 866, 959, 1156, 885] avg=1024.0000 max=1202 max_rank=3 min=866 min_rank=4 imbalance_ratio=1.173828 max_load_reduction=6990 max_load_reduction_pct=0.853271 moved_to_echo=3866 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[69, 952, 1066, 0, 360, 485, 263, 764, 1220, 0, 195, 506, 1191, 78, 1043, 0] rerouted_tokens_per_ep_rank=[1021, 1066, 845, 1027, 1220, 701, 1269, 1043] avg=1024.0000 max=1269 max_rank=6 min=701 min_rank=5 imbalance_ratio=1.239258 max_load_reduction=6923 max_load_reduction_pct=0.845093 moved_to_echo=4233 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[121, 870, 974, 0, 451, 446, 243, 699, 1122, 0, 501, 463, 888, 98, 1316, 0] rerouted_tokens_per_ep_rank=[991, 974, 897, 942, 1122, 964, 986, 1316] avg=1024.0000 max=1316 max_rank=7 min=897 min_rank=2 imbalance_ratio=1.285156 max_load_reduction=6876 max_load_reduction_pct=0.839355 moved_to_echo=4388 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[96, 965, 1061, 0, 793, 424, 293, 774, 1055, 0, 360, 515, 937, 67, 852, 0] rerouted_tokens_per_ep_rank=[1061, 1061, 1217, 1067, 1055, 875, 1004, 852] avg=1024.0000 max=1217 max_rank=2 min=852 min_rank=7 imbalance_ratio=1.188477 max_load_reduction=6975 max_load_reduction_pct=0.851440 moved_to_echo=3786 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[136, 1033, 1156, 0, 661, 408, 257, 829, 1027, 0, 299, 550, 519, 91, 1226, 0] rerouted_tokens_per_ep_rank=[1169, 1156, 1069, 1086, 1027, 849, 610, 1226] avg=1024.0000 max=1226 max_rank=7 min=610 min_rank=6 imbalance_ratio=1.197266 max_load_reduction=6966 max_load_reduction_pct=0.850342 moved_to_echo=3712 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[126, 846, 946, 0, 675, 495, 378, 679, 1245, 0, 506, 450, 971, 60, 815, 0] rerouted_tokens_per_ep_rank=[972, 946, 1170, 1057, 1245, 956, 1031, 815] avg=1024.0000 max=1245 max_rank=4 min=815 min_rank=7 imbalance_ratio=1.215820 max_load_reduction=6947 max_load_reduction_pct=0.848022 moved_to_echo=4047 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8117, 8192, 9344, 7040, 8192, 8267] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7040 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8117, 8192, 9344, 7040, 8192, 8267] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8117, 8192, 9344, 7040, 8192, 8267] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8117 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8117, 8192, 9344, 7040, 8192, 8267] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8267 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8117, 8192, 9344, 7040, 8192, 8267] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8117, 8192, 9344, 7040, 8192, 8267] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8117, 8192, 9344, 7040, 8192, 8267] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9344 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8117, 8192, 9344, 7040, 8192, 8267] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7040 actual_valid_recv=7168 observed_loss=-128 tokens_per_expert=[3200, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[896, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7680, 640] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8117 actual_valid_recv=8320 observed_loss=-203 tokens_per_expert=[4480, 3840] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8267 actual_valid_recv=8320 observed_loss=-53 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2304, 6016] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=9344 actual_valid_recv=9344 observed_loss=0 tokens_per_expert=[9344, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7168 slack_slots=9216 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=9344 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1229, 112, 184, 1604, 1949, 781, 1365, 968] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[614, 96, 63, 1453, 2786, 788, 1811, 581] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[646, 134, 83, 1548, 1408, 177, 1808, 2388] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[922, 352, 586, 920, 1846, 629, 1028, 1909] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1754, 313, 286, 1066, 1510, 812, 1116, 1335] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[557, 57, 196, 1363, 2982, 735, 1108, 1194] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1368, 369, 507, 421, 1754, 641, 1645, 1487] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1768, 27, 53, 2535, 1572, 596, 974, 667] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[614, 0, 96, 1186, 63, 361, 1092, 0, 1600, 0, 788, 128, 1811, 0, 453, 0] rerouted_tokens_per_ep_rank=[614, 1282, 424, 1092, 1600, 916, 1811, 453] avg=1024.0000 max=1811 max_rank=6 min=424 min_rank=2 imbalance_ratio=1.768555 max_load_reduction=6381 max_load_reduction_pct=0.778931 moved_to_echo=4780 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1229, 0, 112, 830, 184, 399, 1205, 0, 1119, 0, 781, 214, 1365, 0, 754, 0] rerouted_tokens_per_ep_rank=[1229, 942, 583, 1205, 1119, 995, 1365, 754] avg=1024.0000 max=1365 max_rank=6 min=583 min_rank=2 imbalance_ratio=1.333008 max_load_reduction=6827 max_load_reduction_pct=0.833374 moved_to_echo=4233 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[646, 0, 134, 599, 83, 385, 1163, 0, 809, 0, 177, 530, 1808, 0, 1858, 0] rerouted_tokens_per_ep_rank=[646, 733, 468, 1163, 809, 707, 1808, 1858] avg=1024.0000 max=1858 max_rank=7 min=468 min_rank=2 imbalance_ratio=1.814453 max_load_reduction=6334 max_load_reduction_pct=0.773193 moved_to_echo=5182 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1768, 0, 27, 669, 53, 631, 1904, 0, 903, 0, 596, 148, 974, 0, 519, 0] rerouted_tokens_per_ep_rank=[1768, 696, 684, 1904, 903, 744, 974, 519] avg=1024.0000 max=1904 max_rank=3 min=519 min_rank=7 imbalance_ratio=1.859375 max_load_reduction=6288 max_load_reduction_pct=0.767578 moved_to_echo=3140 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[557, 0, 57, 1272, 196, 344, 1019, 0, 1710, 0, 735, 268, 1108, 0, 926, 0] rerouted_tokens_per_ep_rank=[557, 1329, 540, 1019, 1710, 1003, 1108, 926] avg=1024.0000 max=1710 max_rank=4 min=540 min_rank=2 imbalance_ratio=1.669922 max_load_reduction=6482 max_load_reduction_pct=0.791260 moved_to_echo=4747 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[922, 0, 352, 786, 586, 229, 691, 0, 1060, 0, 629, 423, 1028, 0, 1486, 0] rerouted_tokens_per_ep_rank=[922, 1138, 815, 691, 1060, 1052, 1028, 1486] avg=1024.0000 max=1486 max_rank=7 min=691 min_rank=3 imbalance_ratio=1.451172 max_load_reduction=6706 max_load_reduction_pct=0.818604 moved_to_echo=4626 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1368, 0, 369, 747, 507, 104, 317, 0, 1007, 0, 641, 330, 1645, 0, 1157, 0] rerouted_tokens_per_ep_rank=[1368, 1116, 611, 317, 1007, 971, 1645, 1157] avg=1024.0000 max=1645 max_rank=6 min=317 min_rank=3 imbalance_ratio=1.606445 max_load_reduction=6547 max_load_reduction_pct=0.799194 moved_to_echo=4780 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1754, 0, 313, 643, 286, 265, 801, 0, 867, 0, 812, 296, 1116, 0, 1039, 0] rerouted_tokens_per_ep_rank=[1754, 956, 551, 801, 867, 1108, 1116, 1039] avg=1024.0000 max=1754 max_rank=0 min=551 min_rank=2 imbalance_ratio=1.712891 max_load_reduction=6438 max_load_reduction_pct=0.785889 moved_to_echo=4130 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8858 expected_overflow=0 exact_recv_per_rank=[8858, 8192, 4676, 8192, 9075, 7496, 10855, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8858, 8192, 4676, 8192, 9075, 7496, 10855, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8858, 8192, 4676, 8192, 9075, 7496, 10855, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7496 expected_overflow=0 exact_recv_per_rank=[8858, 8192, 4676, 8192, 9075, 7496, 10855, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10855 expected_overflow=0 exact_recv_per_rank=[8858, 8192, 4676, 8192, 9075, 7496, 10855, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4676 expected_overflow=0 exact_recv_per_rank=[8858, 8192, 4676, 8192, 9075, 7496, 10855, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9075 expected_overflow=0 exact_recv_per_rank=[8858, 8192, 4676, 8192, 9075, 7496, 10855, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8858, 8192, 4676, 8192, 9075, 7496, 10855, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=4676 actual_valid_recv=4864 observed_loss=-188 tokens_per_expert=[2048, 2816] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7496 actual_valid_recv=7680 observed_loss=-184 tokens_per_expert=[5248, 2432] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1536, 6784] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8858 actual_valid_recv=8960 observed_loss=-102 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=9075 actual_valid_recv=9088 observed_loss=-13 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=10855 actual_valid_recv=10880 observed_loss=-25 tokens_per_expert=[10880, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=4864 slack_slots=11520 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=10880 slack_slots=5504 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1413, 160, 1104, 1231, 316, 1835, 1622, 511] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[485, 20, 810, 2591, 154, 3203, 880, 49] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[702, 169, 898, 1901, 276, 2615, 1040, 591] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[505, 35, 1188, 2735, 65, 2603, 740, 321] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[389, 546, 1501, 973, 61, 743, 3419, 560] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[747, 47, 717, 2205, 1048, 1555, 1585, 288] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[2313, 28, 1624, 2126, 13, 1304, 456, 328] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1026, 707, 1179, 2656, 416, 1327, 530, 351] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1413, 74, 160, 485, 1030, 0, 746, 0, 316, 495, 991, 0, 1622, 0, 511, 349] rerouted_tokens_per_ep_rank=[1487, 645, 1030, 746, 811, 991, 1622, 860] avg=1024.0000 max=1622 max_rank=6 min=645 min_rank=1 imbalance_ratio=1.583984 max_load_reduction=6570 max_load_reduction_pct=0.802002 moved_to_echo=4284 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[702, 60, 169, 750, 838, 0, 1151, 0, 276, 705, 1412, 0, 1040, 0, 591, 498] rerouted_tokens_per_ep_rank=[762, 919, 838, 1151, 981, 1412, 1040, 1089] avg=1024.0000 max=1412 max_rank=5 min=762 min_rank=0 imbalance_ratio=1.378906 max_load_reduction=6780 max_load_reduction_pct=0.827637 moved_to_echo=4522 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[485, 54, 20, 1022, 756, 0, 1569, 0, 154, 864, 1729, 0, 880, 0, 49, 610] rerouted_tokens_per_ep_rank=[539, 1042, 756, 1569, 1018, 1729, 880, 659] avg=1024.0000 max=1729 max_rank=5 min=539 min_rank=0 imbalance_ratio=1.688477 max_load_reduction=6463 max_load_reduction_pct=0.788940 moved_to_echo=4286 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[505, 86, 35, 1082, 1102, 0, 1653, 0, 65, 705, 1397, 0, 740, 0, 321, 501] rerouted_tokens_per_ep_rank=[591, 1117, 1102, 1653, 770, 1397, 740, 822] avg=1024.0000 max=1653 max_rank=3 min=591 min_rank=0 imbalance_ratio=1.614258 max_load_reduction=6539 max_load_reduction_pct=0.798218 moved_to_echo=3729 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[389, 101, 546, 384, 1400, 0, 589, 0, 61, 200, 402, 0, 3419, 0, 560, 141] rerouted_tokens_per_ep_rank=[490, 930, 1400, 589, 261, 402, 3419, 701] avg=1024.0000 max=3419 max_rank=6 min=261 min_rank=4 imbalance_ratio=3.338867 max_load_reduction=4773 max_load_reduction_pct=0.582642 moved_to_echo=4783 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[747, 48, 47, 870, 669, 0, 1335, 0, 1048, 419, 840, 0, 1585, 0, 288, 296] rerouted_tokens_per_ep_rank=[795, 917, 669, 1335, 1467, 840, 1585, 584] avg=1024.0000 max=1585 max_rank=6 min=584 min_rank=7 imbalance_ratio=1.547852 max_load_reduction=6607 max_load_reduction_pct=0.806519 moved_to_echo=4476 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2313, 110, 28, 839, 1514, 0, 1287, 0, 13, 351, 705, 0, 456, 0, 328, 248] rerouted_tokens_per_ep_rank=[2423, 867, 1514, 1287, 364, 705, 456, 576] avg=1024.0000 max=2423 max_rank=0 min=364 min_rank=4 imbalance_ratio=2.366211 max_load_reduction=5769 max_load_reduction_pct=0.704224 moved_to_echo=2101 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1026, 79, 707, 1048, 1100, 0, 1608, 0, 416, 358, 716, 0, 530, 0, 351, 253] rerouted_tokens_per_ep_rank=[1105, 1755, 1100, 1608, 774, 716, 530, 604] avg=1024.0000 max=1755 max_rank=1 min=530 min_rank=6 imbalance_ratio=1.713867 max_load_reduction=6437 max_load_reduction_pct=0.785767 moved_to_echo=2624 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9938 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8409, 9938, 6446, 8192, 10272, 5895] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8409, 9938, 6446, 8192, 10272, 5895] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8409, 9938, 6446, 8192, 10272, 5895] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6446 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8409, 9938, 6446, 8192, 10272, 5895] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10272 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8409, 9938, 6446, 8192, 10272, 5895] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8409, 9938, 6446, 8192, 10272, 5895] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5895 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8409, 9938, 6446, 8192, 10272, 5895] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8409 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8409, 9938, 6446, 8192, 10272, 5895] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=6446 actual_valid_recv=6656 observed_loss=-210 tokens_per_expert=[2432, 4224] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7680, 640] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=5895 actual_valid_recv=6016 observed_loss=-121 tokens_per_expert=[3072, 2944] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1792, 6528] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=10272 actual_valid_recv=10368 observed_loss=-96 tokens_per_expert=[10368, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8409 actual_valid_recv=8448 observed_loss=-39 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=9938 actual_valid_recv=9984 observed_loss=-46 tokens_per_expert=[9984, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=6656 slack_slots=9728 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=6016 slack_slots=10368 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=10368 slack_slots=6016 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=9984 slack_slots=6400 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[2172, 2862, 367, 126, 505, 160, 1397, 603] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1380, 3593, 31, 246, 1009, 245, 1523, 165] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[124, 2884, 439, 823, 854, 259, 1636, 1173] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[649, 3853, 181, 256, 1604, 927, 612, 110] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[398, 2795, 787, 327, 620, 1582, 1673, 10] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[324, 3676, 171, 561, 1861, 141, 1436, 22] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[979, 3284, 124, 222, 594, 219, 2231, 539] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[311, 1897, 148, 1891, 1035, 480, 1265, 1165] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[324, 226, 1214, 0, 171, 879, 561, 314, 1861, 13, 141, 618, 1197, 0, 22, 651] rerouted_tokens_per_ep_rank=[550, 1214, 1050, 875, 1874, 759, 1197, 673] avg=1024.0000 max=1874 max_rank=4 min=550 min_rank=0 imbalance_ratio=1.830078 max_load_reduction=6318 max_load_reduction_pct=0.771240 moved_to_echo=4503 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1380, 239, 1186, 0, 31, 859, 246, 307, 1009, 14, 245, 604, 1270, 0, 165, 637] rerouted_tokens_per_ep_rank=[1619, 1186, 890, 553, 1023, 849, 1270, 802] avg=1024.0000 max=1619 max_rank=0 min=553 min_rank=3 imbalance_ratio=1.581055 max_load_reduction=6573 max_load_reduction_pct=0.802368 moved_to_echo=3944 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[979, 351, 1085, 0, 124, 785, 222, 280, 594, 20, 219, 552, 1860, 0, 539, 582] rerouted_tokens_per_ep_rank=[1330, 1085, 909, 502, 614, 771, 1860, 1121] avg=1024.0000 max=1860 max_rank=6 min=502 min_rank=3 imbalance_ratio=1.816406 max_load_reduction=6332 max_load_reduction_pct=0.772949 moved_to_echo=4366 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[124, 257, 952, 0, 439, 690, 823, 246, 854, 15, 259, 485, 1364, 0, 1173, 511] rerouted_tokens_per_ep_rank=[381, 952, 1129, 1069, 869, 744, 1364, 1684] avg=1024.0000 max=1684 max_rank=7 min=381 min_rank=0 imbalance_ratio=1.644531 max_load_reduction=6508 max_load_reduction_pct=0.794434 moved_to_echo=4661 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[649, 100, 1258, 0, 181, 926, 256, 333, 1604, 9, 927, 650, 503, 0, 110, 686] rerouted_tokens_per_ep_rank=[749, 1258, 1107, 589, 1613, 1577, 503, 796] avg=1024.0000 max=1613 max_rank=4 min=503 min_rank=6 imbalance_ratio=1.575195 max_load_reduction=6579 max_load_reduction_pct=0.803101 moved_to_echo=4489 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[398, 263, 924, 0, 787, 668, 327, 238, 620, 15, 1582, 470, 1395, 0, 10, 495] rerouted_tokens_per_ep_rank=[661, 924, 1455, 565, 635, 2052, 1395, 505] avg=1024.0000 max=2052 max_rank=5 min=505 min_rank=7 imbalance_ratio=2.003906 max_load_reduction=6140 max_load_reduction_pct=0.749512 moved_to_echo=4587 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2172, 220, 946, 0, 367, 684, 126, 244, 505, 13, 160, 481, 1164, 0, 603, 507] rerouted_tokens_per_ep_rank=[2392, 946, 1051, 370, 518, 641, 1164, 1110] avg=1024.0000 max=2392 max_rank=0 min=370 min_rank=3 imbalance_ratio=2.335938 max_load_reduction=5800 max_load_reduction_pct=0.708008 moved_to_echo=3433 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[311, 199, 627, 0, 148, 453, 1891, 162, 1035, 11, 480, 319, 1055, 0, 1165, 336] rerouted_tokens_per_ep_rank=[510, 627, 601, 2053, 1046, 799, 1055, 1501] avg=1024.0000 max=2053 max_rank=3 min=510 min_rank=0 imbalance_ratio=2.004883 max_load_reduction=6139 max_load_reduction_pct=0.749390 moved_to_echo=4401 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6576 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6576, 8192, 8192, 9808, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6576, 8192, 8192, 9808, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6576, 8192, 8192, 9808, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6576, 8192, 8192, 9808, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6576, 8192, 8192, 9808, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6576, 8192, 8192, 9808, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9808 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6576, 8192, 8192, 9808, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 6576, 8192, 8192, 9808, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=6576 actual_valid_recv=6656 observed_loss=-80 tokens_per_expert=[4480, 2176] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6400, 1920] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2304, 6016] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3840, 4480] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4096, 4224] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[8192, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9808 actual_valid_recv=9856 observed_loss=-48 tokens_per_expert=[9856, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=6656 slack_slots=9728 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9856 slack_slots=6528 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[680, 1463, 1689, 1530, 1034, 105, 1104, 587] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1824, 852, 107, 925, 345, 319, 3163, 657] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[761, 601, 903, 947, 1079, 800, 406, 2695] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[401, 1475, 582, 1626, 2033, 386, 114, 1575] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2369, 1357, 824, 983, 164, 1683, 296, 516] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3739, 367, 123, 149, 82, 3584, 119, 29] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[557, 1144, 1028, 736, 2476, 800, 448, 1003] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[2560, 201, 713, 149, 2279, 752, 533, 1005] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[367, 0, 1144, 166, 1028, 99, 736, 181, 2129, 0, 784, 0, 448, 91, 1003, 16] rerouted_tokens_per_ep_rank=[367, 1310, 1127, 917, 2129, 784, 539, 1019] avg=1024.0000 max=2129 max_rank=4 min=367 min_rank=0 imbalance_ratio=2.079102 max_load_reduction=6063 max_load_reduction_pct=0.740112 moved_to_echo=4471 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[458, 0, 1463, 67, 1689, 117, 1530, 74, 893, 0, 104, 0, 1104, 105, 587, 1] rerouted_tokens_per_ep_rank=[458, 1530, 1806, 1604, 893, 104, 1209, 588] avg=1024.0000 max=1806 max_rank=2 min=104 min_rank=5 imbalance_ratio=1.763672 max_load_reduction=6386 max_load_reduction_pct=0.779541 moved_to_echo=2794 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1226, 0, 852, 22, 107, 314, 925, 24, 299, 0, 315, 0, 3163, 284, 657, 4] rerouted_tokens_per_ep_rank=[1226, 874, 421, 949, 299, 315, 3447, 661] avg=1024.0000 max=3447 max_rank=6 min=299 min_rank=4 imbalance_ratio=3.366211 max_load_reduction=4745 max_load_reduction_pct=0.579224 moved_to_echo=4722 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[512, 0, 601, 70, 903, 131, 947, 77, 932, 0, 789, 0, 406, 118, 2695, 11] rerouted_tokens_per_ep_rank=[512, 671, 1034, 1024, 932, 789, 524, 2706] avg=1024.0000 max=2706 max_rank=7 min=512 min_rank=0 imbalance_ratio=2.642578 max_load_reduction=5486 max_load_reduction_pct=0.669678 moved_to_echo=4951 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1721, 0, 201, 148, 713, 441, 149, 163, 1968, 0, 741, 0, 533, 398, 1005, 11] rerouted_tokens_per_ep_rank=[1721, 349, 1154, 312, 1968, 741, 931, 1016] avg=1024.0000 max=1968 max_rank=4 min=312 min_rank=3 imbalance_ratio=1.921875 max_load_reduction=6224 max_load_reduction_pct=0.759766 moved_to_echo=4656 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1592, 0, 1357, 10, 824, 408, 983, 11, 143, 0, 1659, 0, 296, 369, 516, 24] rerouted_tokens_per_ep_rank=[1592, 1367, 1232, 994, 143, 1659, 665, 540] avg=1024.0000 max=1659 max_rank=5 min=143 min_rank=4 imbalance_ratio=1.620117 max_load_reduction=6533 max_load_reduction_pct=0.797485 moved_to_echo=3007 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2513, 0, 367, 5, 123, 644, 149, 5, 72, 0, 3531, 0, 119, 582, 29, 53] rerouted_tokens_per_ep_rank=[2513, 372, 767, 154, 72, 3531, 701, 82] avg=1024.0000 max=3531 max_rank=5 min=72 min_rank=4 imbalance_ratio=3.448242 max_load_reduction=4661 max_load_reduction_pct=0.568970 moved_to_echo=4386 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[270, 0, 1475, 132, 582, 69, 1626, 145, 1756, 0, 381, 0, 114, 62, 1575, 5] rerouted_tokens_per_ep_rank=[270, 1607, 651, 1771, 1756, 381, 176, 1580] avg=1024.0000 max=1771 max_rank=3 min=176 min_rank=6 imbalance_ratio=1.729492 max_load_reduction=6421 max_load_reduction_pct=0.783813 moved_to_echo=3893 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7725 expected_overflow=0 exact_recv_per_rank=[8659, 8080, 8192, 7725, 8192, 8304, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8659, 8080, 8192, 7725, 8192, 8304, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8659, 8080, 8192, 7725, 8192, 8304, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8659 expected_overflow=0 exact_recv_per_rank=[8659, 8080, 8192, 7725, 8192, 8304, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8304 expected_overflow=0 exact_recv_per_rank=[8659, 8080, 8192, 7725, 8192, 8304, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8080 expected_overflow=0 exact_recv_per_rank=[8659, 8080, 8192, 7725, 8192, 8304, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8659, 8080, 8192, 7725, 8192, 8304, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8659, 8080, 8192, 7725, 8192, 8304, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8659 actual_valid_recv=8704 observed_loss=-45 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6016, 2304] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8304 actual_valid_recv=8320 observed_loss=-16 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=7725 actual_valid_recv=7936 observed_loss=-211 tokens_per_expert=[7168, 768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8080 actual_valid_recv=8192 observed_loss=-112 tokens_per_expert=[7552, 640] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[8192, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6272, 2048] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7936 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1114, 100, 718, 2506, 1087, 202, 1422, 1043] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[198, 1049, 127, 296, 2736, 80, 2986, 720] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[8, 2722, 23, 3, 1190, 3116, 27, 1103] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2543, 294, 403, 1002, 3432, 49, 175, 294] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[386, 1681, 15, 999, 3436, 705, 193, 777] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[933, 25, 8, 3057, 2022, 41, 1410, 696] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[190, 1398, 1273, 46, 2542, 48, 209, 2486] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[135, 284, 155, 1242, 2519, 433, 2778, 646] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1114, 102, 100, 145, 718, 313, 2245, 0, 471, 0, 202, 201, 1422, 0, 1043, 116] rerouted_tokens_per_ep_rank=[1216, 245, 1031, 2245, 471, 403, 1422, 1159] avg=1024.0000 max=2245 max_rank=3 min=245 min_rank=1 imbalance_ratio=2.192383 max_load_reduction=5947 max_load_reduction_pct=0.725952 moved_to_echo=3455 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2543, 322, 294, 58, 403, 989, 898, 0, 1485, 0, 49, 636, 175, 0, 294, 46] rerouted_tokens_per_ep_rank=[2865, 352, 1392, 898, 1485, 685, 175, 340] avg=1024.0000 max=2865 max_rank=0 min=175 min_rank=6 imbalance_ratio=2.797852 max_load_reduction=5327 max_load_reduction_pct=0.650269 moved_to_echo=2685 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[933, 190, 25, 177, 8, 583, 2738, 0, 874, 0, 41, 375, 1410, 0, 696, 142] rerouted_tokens_per_ep_rank=[1123, 202, 591, 2738, 874, 416, 1410, 838] avg=1024.0000 max=2738 max_rank=3 min=202 min_rank=1 imbalance_ratio=2.673828 max_load_reduction=5454 max_load_reduction_pct=0.665771 moved_to_echo=3538 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[386, 327, 1681, 61, 15, 994, 887, 0, 1474, 0, 705, 641, 193, 0, 777, 51] rerouted_tokens_per_ep_rank=[713, 1742, 1009, 887, 1474, 1346, 193, 828] avg=1024.0000 max=1742 max_rank=1 min=193 min_rank=6 imbalance_ratio=1.701172 max_load_reduction=6450 max_load_reduction_pct=0.787354 moved_to_echo=3841 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[198, 257, 1049, 17, 127, 789, 266, 0, 1183, 0, 80, 507, 2986, 0, 720, 13] rerouted_tokens_per_ep_rank=[455, 1066, 916, 266, 1183, 587, 2986, 733] avg=1024.0000 max=2986 max_rank=6 min=266 min_rank=3 imbalance_ratio=2.916016 max_load_reduction=5206 max_load_reduction_pct=0.635498 moved_to_echo=5489 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[8, 111, 2722, 0, 23, 343, 3, 0, 516, 0, 3116, 220, 27, 0, 1103, 0] rerouted_tokens_per_ep_rank=[119, 2722, 366, 3, 516, 3336, 27, 1103] avg=1024.0000 max=3336 max_rank=5 min=3 min_rank=3 imbalance_ratio=3.257812 max_load_reduction=4856 max_load_reduction_pct=0.592773 moved_to_echo=4982 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[190, 239, 1398, 2, 1273, 733, 42, 0, 1099, 0, 48, 471, 209, 0, 2486, 2] rerouted_tokens_per_ep_rank=[429, 1400, 2006, 42, 1099, 519, 209, 2488] avg=1024.0000 max=2488 max_rank=7 min=42 min_rank=3 imbalance_ratio=2.429688 max_load_reduction=5704 max_load_reduction_pct=0.696289 moved_to_echo=4315 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[135, 236, 284, 72, 155, 726, 1113, 0, 1090, 0, 433, 467, 2778, 0, 646, 57] rerouted_tokens_per_ep_rank=[371, 356, 881, 1113, 1090, 900, 2778, 703] avg=1024.0000 max=2778 max_rank=6 min=356 min_rank=1 imbalance_ratio=2.712891 max_load_reduction=5414 max_load_reduction_pct=0.660889 moved_to_echo=5471 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7291, 8085, 8192, 8192, 8192, 8192, 9200, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7291, 8085, 8192, 8192, 8192, 8192, 9200, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8085 expected_overflow=0 exact_recv_per_rank=[7291, 8085, 8192, 8192, 8192, 8192, 9200, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7291, 8085, 8192, 8192, 8192, 8192, 9200, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7291 expected_overflow=0 exact_recv_per_rank=[7291, 8085, 8192, 8192, 8192, 8192, 9200, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7291, 8085, 8192, 8192, 8192, 8192, 9200, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7291, 8085, 8192, 8192, 8192, 8192, 9200, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9200 expected_overflow=0 exact_recv_per_rank=[7291, 8085, 8192, 8192, 8192, 8192, 9200, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=7291 actual_valid_recv=7424 observed_loss=-133 tokens_per_expert=[5632, 1792] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4736, 3584] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8085 actual_valid_recv=8320 observed_loss=-235 tokens_per_expert=[7680, 640] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7808, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9200 actual_valid_recv=9216 observed_loss=-16 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2816, 5504] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=7424 slack_slots=8960 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[71, 164, 434, 766, 75, 2175, 965, 3542] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[2494, 91, 15, 1795, 1, 1728, 1986, 82] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[2882, 748, 53, 49, 16, 1726, 322, 2396] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[2594, 199, 169, 87, 457, 585, 2598, 1503] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1781, 117, 21, 227, 678, 469, 1847, 3052] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3099, 310, 327, 184, 910, 1763, 395, 1204] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1220, 1204, 516, 2258, 60, 615, 929, 1390] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3172, 173, 382, 36, 1, 340, 2075, 2013] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[46, 0, 164, 896, 434, 25, 766, 137, 75, 734, 2175, 0, 828, 0, 1912, 0] rerouted_tokens_per_ep_rank=[46, 1060, 459, 903, 809, 2175, 828, 1912] avg=1024.0000 max=2175 max_rank=5 min=46 min_rank=0 imbalance_ratio=2.124023 max_load_reduction=6017 max_load_reduction_pct=0.734497 moved_to_echo=5724 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1654, 0, 199, 380, 169, 940, 87, 369, 457, 311, 585, 0, 2229, 0, 812, 0] rerouted_tokens_per_ep_rank=[1654, 579, 1109, 456, 768, 585, 2229, 812] avg=1024.0000 max=2229 max_rank=6 min=456 min_rank=3 imbalance_ratio=2.176758 max_load_reduction=5963 max_load_reduction_pct=0.727905 moved_to_echo=4394 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1136, 0, 117, 772, 21, 645, 227, 262, 678, 632, 469, 0, 1585, 0, 1648, 0] rerouted_tokens_per_ep_rank=[1136, 889, 666, 489, 1310, 469, 1585, 1648] avg=1024.0000 max=1648 max_rank=7 min=469 min_rank=5 imbalance_ratio=1.609375 max_load_reduction=6544 max_load_reduction_pct=0.798828 moved_to_echo=5012 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1838, 0, 748, 606, 53, 1044, 49, 45, 16, 496, 1726, 0, 277, 0, 1294, 0] rerouted_tokens_per_ep_rank=[1838, 1354, 1097, 94, 512, 1726, 277, 1294] avg=1024.0000 max=1838 max_rank=0 min=94 min_rank=3 imbalance_ratio=1.794922 max_load_reduction=6354 max_load_reduction_pct=0.775635 moved_to_echo=3809 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1587, 0, 91, 24, 15, 907, 1795, 285, 1, 21, 1728, 0, 1701, 0, 37, 0] rerouted_tokens_per_ep_rank=[1587, 115, 922, 2080, 22, 1728, 1701, 37] avg=1024.0000 max=2080 max_rank=3 min=22 min_rank=4 imbalance_ratio=2.031250 max_load_reduction=6112 max_load_reduction_pct=0.746094 moved_to_echo=3488 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[778, 0, 1204, 351, 516, 442, 2258, 132, 60, 288, 615, 0, 797, 0, 751, 0] rerouted_tokens_per_ep_rank=[778, 1555, 958, 2390, 348, 615, 797, 751] avg=1024.0000 max=2390 max_rank=3 min=348 min_rank=4 imbalance_ratio=2.333984 max_load_reduction=5802 max_load_reduction_pct=0.708252 moved_to_echo=2511 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1976, 0, 310, 304, 327, 1123, 184, 56, 910, 249, 1763, 0, 339, 0, 651, 0] rerouted_tokens_per_ep_rank=[1976, 614, 1450, 240, 1159, 1763, 339, 651] avg=1024.0000 max=1976 max_rank=0 min=240 min_rank=3 imbalance_ratio=1.929688 max_load_reduction=6216 max_load_reduction_pct=0.758789 moved_to_echo=3912 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2023, 0, 173, 509, 382, 1149, 36, 295, 1, 417, 340, 0, 1780, 0, 1087, 0] rerouted_tokens_per_ep_rank=[2023, 682, 1531, 331, 418, 340, 1780, 1087] avg=1024.0000 max=2023 max_rank=0 min=331 min_rank=3 imbalance_ratio=1.975586 max_load_reduction=6169 max_load_reduction_pct=0.753052 moved_to_echo=3625 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11038, 6848, 8192, 6983, 5346, 9401, 9536, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5346 expected_overflow=0 exact_recv_per_rank=[11038, 6848, 8192, 6983, 5346, 9401, 9536, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6983 expected_overflow=0 exact_recv_per_rank=[11038, 6848, 8192, 6983, 5346, 9401, 9536, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9401 expected_overflow=0 exact_recv_per_rank=[11038, 6848, 8192, 6983, 5346, 9401, 9536, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9536 expected_overflow=0 exact_recv_per_rank=[11038, 6848, 8192, 6983, 5346, 9401, 9536, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11038, 6848, 8192, 6983, 5346, 9401, 9536, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11038 expected_overflow=0 exact_recv_per_rank=[11038, 6848, 8192, 6983, 5346, 9401, 9536, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6848 expected_overflow=0 exact_recv_per_rank=[11038, 6848, 8192, 6983, 5346, 9401, 9536, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=5346 actual_valid_recv=5504 observed_loss=-158 tokens_per_expert=[2304, 3200] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=6983 actual_valid_recv=7168 observed_loss=-185 tokens_per_expert=[5504, 1664] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=6848 actual_valid_recv=7040 observed_loss=-192 tokens_per_expert=[3072, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1920, 6400] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=11038 actual_valid_recv=11136 observed_loss=-98 tokens_per_expert=[11136, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9536 actual_valid_recv=9600 observed_loss=-64 tokens_per_expert=[9600, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=9401 actual_valid_recv=9472 observed_loss=-71 tokens_per_expert=[9472, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=5504 slack_slots=10880 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7168 slack_slots=9216 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=7040 slack_slots=9344 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=11136 slack_slots=5248 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=9472 slack_slots=6912 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9600 slack_slots=6784 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1593, 1051, 3188, 102, 564, 1526, 3, 165] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[20, 1888, 2691, 130, 177, 0, 12, 3274] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[48, 566, 1231, 330, 3220, 248, 58, 2491] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3188, 555, 3315, 249, 151, 614, 2, 118] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1301, 1049, 2299, 2552, 865, 64, 17, 45] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[45, 23, 1651, 1086, 781, 2973, 807, 826] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[2163, 2297, 207, 368, 2090, 317, 4, 746] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[303, 67, 1970, 1655, 1384, 183, 53, 2577] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[46, 0, 566, 2, 693, 0, 330, 220, 3220, 0, 248, 277, 58, 538, 1994, 0] rerouted_tokens_per_ep_rank=[46, 568, 693, 550, 3220, 525, 596, 1994] avg=1024.0000 max=3220 max_rank=4 min=46 min_rank=0 imbalance_ratio=3.144531 max_load_reduction=4972 max_load_reduction_pct=0.606934 moved_to_echo=6335 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1507, 0, 1051, 86, 1795, 0, 102, 14, 564, 0, 1526, 18, 3, 1393, 133, 0] rerouted_tokens_per_ep_rank=[1507, 1137, 1795, 116, 564, 1544, 1396, 133] avg=1024.0000 max=1795 max_rank=2 min=116 min_rank=3 imbalance_ratio=1.752930 max_load_reduction=6397 max_load_reduction_pct=0.780884 moved_to_echo=3637 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 1888, 1, 1515, 0, 130, 289, 177, 0, 0, 365, 12, 1176, 2620, 0] rerouted_tokens_per_ep_rank=[19, 1889, 1515, 419, 177, 365, 1188, 2620] avg=1024.0000 max=2620 max_rank=7 min=19 min_rank=0 imbalance_ratio=2.558594 max_load_reduction=5572 max_load_reduction_pct=0.680176 moved_to_echo=4350 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3016, 0, 555, 172, 1866, 0, 249, 10, 151, 0, 614, 13, 2, 1449, 95, 0] rerouted_tokens_per_ep_rank=[3016, 727, 1866, 259, 151, 627, 1451, 95] avg=1024.0000 max=3016 max_rank=0 min=95 min_rank=7 imbalance_ratio=2.945312 max_load_reduction=5176 max_load_reduction_pct=0.631836 moved_to_echo=2324 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1228, 0, 1049, 73, 1291, 0, 2552, 7, 865, 0, 64, 8, 17, 1008, 30, 0] rerouted_tokens_per_ep_rank=[1228, 1122, 1291, 2559, 865, 72, 1025, 30] avg=1024.0000 max=2559 max_rank=3 min=30 min_rank=7 imbalance_ratio=2.499023 max_load_reduction=5633 max_load_reduction_pct=0.687622 moved_to_echo=1992 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[43, 0, 23, 2, 930, 0, 1086, 73, 781, 0, 2973, 92, 807, 721, 661, 0] rerouted_tokens_per_ep_rank=[43, 25, 930, 1159, 781, 3065, 1528, 661] avg=1024.0000 max=3065 max_rank=5 min=25 min_rank=1 imbalance_ratio=2.993164 max_load_reduction=5127 max_load_reduction_pct=0.625854 moved_to_echo=6035 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2046, 0, 2297, 117, 117, 0, 368, 66, 2090, 0, 317, 83, 4, 90, 597, 0] rerouted_tokens_per_ep_rank=[2046, 2414, 117, 434, 2090, 400, 94, 597] avg=1024.0000 max=2414 max_rank=1 min=94 min_rank=6 imbalance_ratio=2.357422 max_load_reduction=5778 max_load_reduction_pct=0.705322 moved_to_echo=3181 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[287, 0, 67, 16, 1109, 0, 1655, 228, 1384, 0, 183, 287, 53, 861, 2062, 0] rerouted_tokens_per_ep_rank=[287, 83, 1109, 1883, 1384, 470, 914, 2062] avg=1024.0000 max=2062 max_rank=7 min=83 min_rank=1 imbalance_ratio=2.013672 max_load_reduction=6130 max_load_reduction_pct=0.748291 moved_to_echo=4830 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9232 expected_overflow=0 exact_recv_per_rank=[8192, 7965, 9316, 7379, 9232, 7068, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7068 expected_overflow=0 exact_recv_per_rank=[8192, 7965, 9316, 7379, 9232, 7068, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7379 expected_overflow=0 exact_recv_per_rank=[8192, 7965, 9316, 7379, 9232, 7068, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9316 expected_overflow=0 exact_recv_per_rank=[8192, 7965, 9316, 7379, 9232, 7068, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7965, 9316, 7379, 9232, 7068, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7965, 9316, 7379, 9232, 7068, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7965, 9316, 7379, 9232, 7068, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7965 expected_overflow=0 exact_recv_per_rank=[8192, 7965, 9316, 7379, 9232, 7068, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=7379 actual_valid_recv=7552 observed_loss=-173 tokens_per_expert=[6528, 1024] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=9232 actual_valid_recv=9344 observed_loss=-112 tokens_per_expert=[9344, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=7965 actual_valid_recv=8064 observed_loss=-99 tokens_per_expert=[7552, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7068 actual_valid_recv=7168 observed_loss=-100 tokens_per_expert=[6016, 1152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=9316 actual_valid_recv=9344 observed_loss=-28 tokens_per_expert=[9344, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1024, 7296] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=9344 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8064 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7168 slack_slots=9216 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=9344 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1118, 160, 1256, 40, 570, 660, 2653, 1735] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[236, 210, 548, 3087, 3135, 13, 327, 636] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[2718, 222, 200, 995, 978, 927, 2140, 12] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2381, 637, 0, 2726, 1084, 12, 1312, 40] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[207, 62, 681, 3302, 410, 27, 3166, 337] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1997, 1127, 379, 3404, 5, 147, 311, 822] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2612, 101, 15, 75, 94, 80, 2616, 2599] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[943, 836, 578, 1711, 2706, 1284, 113, 21] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[751, 0, 160, 573, 1256, 258, 27, 0, 570, 0, 660, 13, 2080, 0, 1735, 109] rerouted_tokens_per_ep_rank=[751, 733, 1514, 27, 570, 673, 2080, 1844] avg=1024.0000 max=2080 max_rank=6 min=27 min_rank=3 imbalance_ratio=2.031250 max_load_reduction=6112 max_load_reduction_pct=0.746094 moved_to_echo=5167 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[159, 0, 210, 70, 548, 54, 2073, 0, 3135, 0, 13, 1014, 257, 0, 636, 23] rerouted_tokens_per_ep_rank=[159, 280, 602, 2073, 3135, 1027, 257, 659] avg=1024.0000 max=3135 max_rank=4 min=159 min_rank=0 imbalance_ratio=3.061523 max_load_reduction=5057 max_load_reduction_pct=0.617310 moved_to_echo=5078 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1816, 0, 222, 465, 200, 631, 664, 0, 978, 0, 927, 331, 1675, 0, 12, 271] rerouted_tokens_per_ep_rank=[1816, 687, 831, 664, 978, 1258, 1675, 283] avg=1024.0000 max=1816 max_rank=0 min=283 min_rank=7 imbalance_ratio=1.773438 max_load_reduction=6376 max_load_reduction_pct=0.778320 moved_to_echo=4194 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1753, 0, 101, 565, 15, 603, 51, 0, 94, 0, 80, 24, 2051, 0, 2599, 256] rerouted_tokens_per_ep_rank=[1753, 666, 618, 51, 94, 104, 2051, 2855] avg=1024.0000 max=2855 max_rank=7 min=51 min_rank=3 imbalance_ratio=2.788086 max_load_reduction=5337 max_load_reduction_pct=0.651489 moved_to_echo=5104 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1599, 0, 637, 283, 0, 549, 1831, 0, 1084, 0, 12, 895, 1029, 0, 40, 233] rerouted_tokens_per_ep_rank=[1599, 920, 549, 1831, 1084, 907, 1029, 273] avg=1024.0000 max=1831 max_rank=3 min=273 min_rank=7 imbalance_ratio=1.788086 max_load_reduction=6361 max_load_reduction_pct=0.776489 moved_to_echo=3293 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1340, 0, 1127, 67, 379, 461, 2286, 0, 5, 0, 147, 1118, 244, 0, 822, 196] rerouted_tokens_per_ep_rank=[1340, 1194, 840, 2286, 5, 1265, 244, 1018] avg=1024.0000 max=2286 max_rank=3 min=5 min_rank=4 imbalance_ratio=2.232422 max_load_reduction=5906 max_load_reduction_pct=0.720947 moved_to_echo=2532 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[140, 0, 62, 684, 681, 47, 2217, 0, 410, 0, 27, 1085, 2482, 0, 337, 20] rerouted_tokens_per_ep_rank=[140, 746, 728, 2217, 410, 1112, 2482, 357] avg=1024.0000 max=2482 max_rank=6 min=140 min_rank=0 imbalance_ratio=2.423828 max_load_reduction=5710 max_load_reduction_pct=0.697021 moved_to_echo=4361 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[634, 0, 836, 24, 578, 217, 1149, 0, 2706, 0, 1284, 562, 89, 0, 21, 92] rerouted_tokens_per_ep_rank=[634, 860, 795, 1149, 2706, 1846, 89, 113] avg=1024.0000 max=2706 max_rank=4 min=89 min_rank=6 imbalance_ratio=2.642578 max_load_reduction=5486 max_load_reduction_pct=0.669678 moved_to_echo=4754 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6086, 6477, 10298, 8982, 8192, 9907, 7402] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6086, 6477, 10298, 8982, 8192, 9907, 7402] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7402 expected_overflow=0 exact_recv_per_rank=[8192, 6086, 6477, 10298, 8982, 8192, 9907, 7402] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9907 expected_overflow=0 exact_recv_per_rank=[8192, 6086, 6477, 10298, 8982, 8192, 9907, 7402] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6086 expected_overflow=0 exact_recv_per_rank=[8192, 6086, 6477, 10298, 8982, 8192, 9907, 7402] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10298 expected_overflow=0 exact_recv_per_rank=[8192, 6086, 6477, 10298, 8982, 8192, 9907, 7402] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6477 expected_overflow=0 exact_recv_per_rank=[8192, 6086, 6477, 10298, 8982, 8192, 9907, 7402] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8982 expected_overflow=0 exact_recv_per_rank=[8192, 6086, 6477, 10298, 8982, 8192, 9907, 7402] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=6086 actual_valid_recv=6272 observed_loss=-186 tokens_per_expert=[3456, 2816] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=7402 actual_valid_recv=7552 observed_loss=-150 tokens_per_expert=[6272, 1280] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=6477 actual_valid_recv=6656 observed_loss=-179 tokens_per_expert=[3712, 2944] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9907 actual_valid_recv=9984 observed_loss=-77 tokens_per_expert=[9984, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8982 actual_valid_recv=9088 observed_loss=-106 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3200, 5120] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=10298 actual_valid_recv=10368 observed_loss=-70 tokens_per_expert=[10368, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=6272 slack_slots=10112 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=6656 slack_slots=9728 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9984 slack_slots=6400 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=10368 slack_slots=6016 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[753, 3256, 22, 976, 924, 464, 1797, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3211, 616, 126, 8, 572, 208, 2305, 1146] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[3695, 31, 0, 199, 1684, 15, 2343, 225] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1583, 1626, 90, 52, 1092, 40, 3602, 107] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[2286, 993, 1, 118, 1269, 3209, 84, 232] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[770, 25, 20, 1561, 50, 947, 2362, 2457] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[676, 2659, 543, 10, 1569, 260, 2473, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[365, 3479, 53, 1705, 723, 104, 238, 1525] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2354, 0, 399, 0, 126, 1063, 8, 857, 572, 15, 208, 81, 1242, 0, 1146, 121] rerouted_tokens_per_ep_rank=[2354, 399, 1189, 865, 587, 289, 1242, 1267] avg=1024.0000 max=2354 max_rank=0 min=289 min_rank=5 imbalance_ratio=2.298828 max_load_reduction=5838 max_load_reduction_pct=0.712646 moved_to_echo=3385 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[552, 0, 2104, 0, 22, 828, 976, 201, 924, 79, 464, 432, 969, 0, 0, 641] rerouted_tokens_per_ep_rank=[552, 2104, 850, 1177, 1003, 896, 969, 641] avg=1024.0000 max=2104 max_rank=1 min=552 min_rank=0 imbalance_ratio=2.054688 max_load_reduction=6088 max_load_reduction_pct=0.743164 moved_to_echo=3509 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1161, 0, 1051, 0, 90, 1661, 52, 422, 1092, 39, 40, 216, 1941, 0, 107, 320] rerouted_tokens_per_ep_rank=[1161, 1051, 1751, 474, 1131, 256, 1941, 427] avg=1024.0000 max=1941 max_rank=6 min=256 min_rank=5 imbalance_ratio=1.895508 max_load_reduction=6251 max_load_reduction_pct=0.763062 moved_to_echo=3755 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[565, 0, 18, 0, 20, 1089, 1561, 205, 50, 0, 947, 3, 1273, 0, 2457, 4] rerouted_tokens_per_ep_rank=[565, 18, 1109, 1766, 50, 950, 1273, 2461] avg=1024.0000 max=2461 max_rank=7 min=18 min_rank=1 imbalance_ratio=2.403320 max_load_reduction=5731 max_load_reduction_pct=0.699585 moved_to_echo=4734 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2704, 0, 10, 0, 0, 1084, 199, 991, 1684, 4, 15, 8, 1259, 0, 225, 9] rerouted_tokens_per_ep_rank=[2704, 10, 1084, 1190, 1688, 23, 1259, 234] avg=1024.0000 max=2704 max_rank=0 min=10 min_rank=1 imbalance_ratio=2.640625 max_load_reduction=5488 max_load_reduction_pct=0.669922 moved_to_echo=3204 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1676, 0, 643, 0, 1, 38, 118, 610, 1269, 24, 3209, 131, 46, 0, 232, 195] rerouted_tokens_per_ep_rank=[1676, 643, 39, 728, 1293, 3340, 46, 427] avg=1024.0000 max=3340 max_rank=5 min=39 min_rank=2 imbalance_ratio=3.261719 max_load_reduction=4852 max_load_reduction_pct=0.592285 moved_to_echo=5106 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[496, 0, 1719, 0, 543, 1140, 10, 180, 1569, 64, 260, 353, 1333, 0, 2, 523] rerouted_tokens_per_ep_rank=[496, 1719, 1683, 190, 1633, 613, 1333, 525] avg=1024.0000 max=1719 max_rank=1 min=190 min_rank=3 imbalance_ratio=1.678711 max_load_reduction=6473 max_load_reduction_pct=0.790161 moved_to_echo=4104 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[268, 0, 2248, 0, 53, 109, 1705, 97, 723, 84, 104, 462, 129, 0, 1525, 685] rerouted_tokens_per_ep_rank=[268, 2248, 162, 1802, 807, 566, 129, 2210] avg=1024.0000 max=2248 max_rank=1 min=129 min_rank=6 imbalance_ratio=2.195312 max_load_reduction=5944 max_load_reduction_pct=0.725586 moved_to_echo=3712 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9776, 8192, 7867, 8192, 8192, 6933, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9776, 8192, 7867, 8192, 8192, 6933, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6933 expected_overflow=0 exact_recv_per_rank=[9776, 8192, 7867, 8192, 8192, 6933, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9776, 8192, 7867, 8192, 8192, 6933, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9776 expected_overflow=0 exact_recv_per_rank=[9776, 8192, 7867, 8192, 8192, 6933, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9776, 8192, 7867, 8192, 8192, 6933, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9776, 8192, 7867, 8192, 8192, 6933, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7867 expected_overflow=0 exact_recv_per_rank=[9776, 8192, 7867, 8192, 8192, 6933, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=6933 actual_valid_recv=7040 observed_loss=-107 tokens_per_expert=[5248, 1792] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7936, 384] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4736, 3584] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=7867 actual_valid_recv=7936 observed_loss=-69 tokens_per_expert=[896, 7040] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5760, 2560] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=9776 actual_valid_recv=9856 observed_loss=-80 tokens_per_expert=[9856, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7040 slack_slots=9344 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=7936 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=9856 slack_slots=6528 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1475, 2151, 30, 2005, 28, 22, 170, 2311] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[376, 1219, 234, 2583, 553, 21, 0, 3206] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[3759, 519, 31, 48, 2, 6, 421, 3406] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1869, 3186, 672, 1774, 112, 119, 70, 390] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[3825, 660, 1127, 1361, 17, 0, 249, 953] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[3462, 3789, 38, 506, 6, 207, 0, 184] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[3602, 365, 494, 3063, 222, 2, 14, 430] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[586, 245, 948, 2457, 139, 396, 177, 3244] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[229, 0, 823, 0, 234, 396, 1661, 0, 553, 855, 21, 147, 0, 922, 2351, 0] rerouted_tokens_per_ep_rank=[229, 823, 630, 1661, 1408, 168, 922, 2351] avg=1024.0000 max=2351 max_rank=7 min=168 min_rank=5 imbalance_ratio=2.295898 max_load_reduction=5841 max_load_reduction_pct=0.713013 moved_to_echo=4849 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[898, 0, 1453, 0, 30, 698, 1289, 0, 28, 616, 22, 577, 170, 716, 1695, 0] rerouted_tokens_per_ep_rank=[898, 1453, 728, 1289, 644, 599, 886, 1695] avg=1024.0000 max=1695 max_rank=7 min=599 min_rank=5 imbalance_ratio=1.655273 max_load_reduction=6497 max_load_reduction_pct=0.793091 moved_to_echo=3824 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2328, 0, 446, 0, 1127, 214, 875, 0, 17, 254, 0, 1497, 249, 486, 699, 0] rerouted_tokens_per_ep_rank=[2328, 446, 1341, 875, 271, 1497, 735, 699] avg=1024.0000 max=2328 max_rank=0 min=271 min_rank=4 imbalance_ratio=2.273438 max_load_reduction=5864 max_load_reduction_pct=0.715820 moved_to_echo=3202 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1135, 0, 2147, 0, 672, 1039, 1137, 0, 112, 108, 119, 734, 70, 637, 282, 0] rerouted_tokens_per_ep_rank=[1135, 2147, 1711, 1137, 220, 853, 707, 282] avg=1024.0000 max=2147 max_rank=1 min=220 min_rank=4 imbalance_ratio=2.096680 max_load_reduction=6045 max_load_reduction_pct=0.737915 moved_to_echo=2062 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2107, 0, 2559, 0, 38, 1230, 326, 0, 6, 49, 207, 1355, 0, 180, 135, 0] rerouted_tokens_per_ep_rank=[2107, 2559, 1268, 326, 55, 1562, 180, 135] avg=1024.0000 max=2559 max_rank=1 min=55 min_rank=4 imbalance_ratio=2.499023 max_load_reduction=5633 max_load_reduction_pct=0.687622 moved_to_echo=1932 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2193, 0, 247, 0, 494, 118, 1969, 0, 222, 114, 2, 1409, 14, 1094, 316, 0] rerouted_tokens_per_ep_rank=[2193, 247, 612, 1969, 336, 1411, 1108, 316] avg=1024.0000 max=2193 max_rank=0 min=247 min_rank=1 imbalance_ratio=2.141602 max_load_reduction=5999 max_load_reduction_pct=0.732300 moved_to_echo=3171 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2288, 0, 351, 0, 31, 168, 31, 0, 2, 909, 6, 1471, 421, 17, 2497, 0] rerouted_tokens_per_ep_rank=[2288, 351, 199, 31, 911, 1477, 438, 2497] avg=1024.0000 max=2497 max_rank=7 min=31 min_rank=3 imbalance_ratio=2.438477 max_load_reduction=5695 max_load_reduction_pct=0.695190 moved_to_echo=5323 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[357, 0, 166, 0, 948, 79, 1580, 0, 139, 865, 396, 229, 177, 877, 2379, 0] rerouted_tokens_per_ep_rank=[357, 166, 1027, 1580, 1004, 625, 1054, 2379] avg=1024.0000 max=2379 max_rank=7 min=166 min_rank=1 imbalance_ratio=2.323242 max_load_reduction=5813 max_load_reduction_pct=0.709595 moved_to_echo=5062 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11535, 8192, 7516, 8868, 4849, 8192, 6030, 10354] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8868 expected_overflow=0 exact_recv_per_rank=[11535, 8192, 7516, 8868, 4849, 8192, 6030, 10354] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11535, 8192, 7516, 8868, 4849, 8192, 6030, 10354] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6030 expected_overflow=0 exact_recv_per_rank=[11535, 8192, 7516, 8868, 4849, 8192, 6030, 10354] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7516 expected_overflow=0 exact_recv_per_rank=[11535, 8192, 7516, 8868, 4849, 8192, 6030, 10354] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10354 expected_overflow=0 exact_recv_per_rank=[11535, 8192, 7516, 8868, 4849, 8192, 6030, 10354] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11535 expected_overflow=0 exact_recv_per_rank=[11535, 8192, 7516, 8868, 4849, 8192, 6030, 10354] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4849 expected_overflow=0 exact_recv_per_rank=[11535, 8192, 7516, 8868, 4849, 8192, 6030, 10354] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=4849 actual_valid_recv=4992 observed_loss=-143 tokens_per_expert=[1152, 3840] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=6030 actual_valid_recv=6144 observed_loss=-114 tokens_per_expert=[1152, 4992] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=7516 actual_valid_recv=7552 observed_loss=-36 tokens_per_expert=[3584, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[896, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8868 actual_valid_recv=8960 observed_loss=-92 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=11535 actual_valid_recv=11648 observed_loss=-113 tokens_per_expert=[11648, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=10354 actual_valid_recv=10368 observed_loss=-14 tokens_per_expert=[10368, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=4992 slack_slots=11392 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=6144 slack_slots=10240 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=11648 slack_slots=4736 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=10368 slack_slots=6016 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[81, 12, 1117, 1881, 876, 1846, 2378, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1164, 7, 1037, 1309, 2978, 162, 1535, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3429, 57, 807, 2791, 214, 232, 536, 126] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[907, 282, 473, 1464, 1442, 92, 3530, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3399, 71, 23, 3113, 297, 25, 1263, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1254, 87, 389, 783, 2402, 3173, 104, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[598, 418, 2003, 2015, 2034, 709, 415, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1850, 1, 1326, 908, 1687, 2416, 3, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[907, 0, 282, 451, 473, 200, 841, 0, 991, 0, 92, 0, 3330, 0, 2, 623] rerouted_tokens_per_ep_rank=[907, 733, 673, 841, 991, 92, 3330, 625] avg=1024.0000 max=3330 max_rank=6 min=92 min_rank=5 imbalance_ratio=3.251953 max_load_reduction=4862 max_load_reduction_pct=0.593506 moved_to_echo=5038 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3399, 0, 71, 93, 23, 71, 1788, 0, 204, 0, 25, 0, 1192, 0, 1, 1325] rerouted_tokens_per_ep_rank=[3399, 164, 94, 1788, 204, 25, 1192, 1326] avg=1024.0000 max=3399 max_rank=0 min=25 min_rank=5 imbalance_ratio=3.319336 max_load_reduction=4793 max_load_reduction_pct=0.585083 moved_to_echo=2747 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[81, 0, 12, 274, 1117, 134, 1081, 0, 602, 0, 1846, 0, 2244, 0, 1, 800] rerouted_tokens_per_ep_rank=[81, 286, 1251, 1081, 602, 1846, 2244, 801] avg=1024.0000 max=2244 max_rank=6 min=81 min_rank=0 imbalance_ratio=2.191406 max_load_reduction=5948 max_load_reduction_pct=0.726074 moved_to_echo=5493 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3429, 0, 57, 67, 807, 30, 1603, 0, 147, 0, 232, 0, 506, 0, 126, 1188] rerouted_tokens_per_ep_rank=[3429, 124, 837, 1603, 147, 232, 506, 1314] avg=1024.0000 max=3429 max_rank=0 min=124 min_rank=1 imbalance_ratio=3.348633 max_load_reduction=4763 max_load_reduction_pct=0.581421 moved_to_echo=2199 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[598, 0, 418, 637, 2003, 23, 1158, 0, 1397, 0, 709, 0, 392, 0, 0, 857] rerouted_tokens_per_ep_rank=[598, 1055, 2026, 1158, 1397, 709, 392, 857] avg=1024.0000 max=2026 max_rank=2 min=392 min_rank=6 imbalance_ratio=1.978516 max_load_reduction=6166 max_load_reduction_pct=0.752686 moved_to_echo=3355 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1164, 0, 7, 933, 1037, 87, 752, 0, 2045, 0, 162, 0, 1448, 0, 0, 557] rerouted_tokens_per_ep_rank=[1164, 940, 1124, 752, 2045, 162, 1448, 557] avg=1024.0000 max=2045 max_rank=4 min=162 min_rank=5 imbalance_ratio=1.997070 max_load_reduction=6147 max_load_reduction_pct=0.750366 moved_to_echo=4212 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1254, 0, 87, 755, 389, 9, 447, 0, 1647, 0, 3173, 0, 95, 0, 0, 336] rerouted_tokens_per_ep_rank=[1254, 842, 398, 447, 1647, 3173, 95, 336] avg=1024.0000 max=3173 max_rank=5 min=95 min_rank=6 imbalance_ratio=3.098633 max_load_reduction=5019 max_load_reduction_pct=0.612671 moved_to_echo=5251 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1850, 0, 1, 528, 1326, 0, 522, 0, 1159, 0, 2416, 0, 3, 0, 1, 386] rerouted_tokens_per_ep_rank=[1850, 529, 1326, 522, 1159, 2416, 3, 387] avg=1024.0000 max=2416 max_rank=5 min=3 min_rank=6 imbalance_ratio=2.359375 max_load_reduction=5776 max_load_reduction_pct=0.705078 moved_to_echo=3965 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=12682 expected_overflow=0 exact_recv_per_rank=[12682, 4673, 7729, 8192, 8192, 8655, 9210, 6203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4673 expected_overflow=0 exact_recv_per_rank=[12682, 4673, 7729, 8192, 8192, 8655, 9210, 6203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6203 expected_overflow=0 exact_recv_per_rank=[12682, 4673, 7729, 8192, 8192, 8655, 9210, 6203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9210 expected_overflow=0 exact_recv_per_rank=[12682, 4673, 7729, 8192, 8192, 8655, 9210, 6203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12682, 4673, 7729, 8192, 8192, 8655, 9210, 6203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7729 expected_overflow=0 exact_recv_per_rank=[12682, 4673, 7729, 8192, 8192, 8655, 9210, 6203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8655 expected_overflow=0 exact_recv_per_rank=[12682, 4673, 7729, 8192, 8192, 8655, 9210, 6203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12682, 4673, 7729, 8192, 8192, 8655, 9210, 6203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=4673 actual_valid_recv=4864 observed_loss=-191 tokens_per_expert=[1024, 3840] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=6203 actual_valid_recv=6400 observed_loss=-197 tokens_per_expert=[256, 6144] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=7729 actual_valid_recv=7936 observed_loss=-207 tokens_per_expert=[7296, 640] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8655 actual_valid_recv=8704 observed_loss=-49 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9210 actual_valid_recv=9216 observed_loss=-6 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=12682 actual_valid_recv=12800 observed_loss=-118 tokens_per_expert=[12800, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=4864 slack_slots=11520 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=6400 slack_slots=9984 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=7936 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=12800 slack_slots=3584 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[663, 2519, 0, 2365, 1271, 669, 20, 685] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[509, 2244, 124, 2940, 54, 13, 836, 1472] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1832, 2285, 299, 604, 2529, 486, 11, 146] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[2381, 205, 513, 3851, 247, 19, 33, 943] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[2636, 3174, 52, 259, 1786, 11, 6, 268] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1198, 2329, 15, 80, 1208, 2971, 0, 391] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[5, 1014, 24, 1825, 789, 2, 2552, 1981] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1399, 140, 151, 4028, 353, 8, 2065, 48] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[403, 0, 1322, 0, 124, 1292, 1648, 0, 54, 0, 13, 527, 836, 395, 1472, 106] rerouted_tokens_per_ep_rank=[403, 1322, 1416, 1648, 54, 540, 1231, 1578] avg=1024.0000 max=1648 max_rank=3 min=54 min_rank=4 imbalance_ratio=1.609375 max_load_reduction=6544 max_load_reduction_pct=0.798828 moved_to_echo=3403 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[525, 0, 1485, 0, 0, 1039, 1326, 0, 1271, 0, 669, 591, 20, 443, 685, 138] rerouted_tokens_per_ep_rank=[525, 1485, 1039, 1326, 1271, 1260, 463, 823] avg=1024.0000 max=1485 max_rank=1 min=463 min_rank=6 imbalance_ratio=1.450195 max_load_reduction=6707 max_load_reduction_pct=0.818726 moved_to_echo=3817 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2085, 0, 1863, 0, 52, 117, 142, 0, 1786, 0, 11, 748, 6, 563, 268, 551] rerouted_tokens_per_ep_rank=[2085, 1863, 169, 142, 1786, 759, 569, 819] avg=1024.0000 max=2085 max_rank=0 min=142 min_rank=3 imbalance_ratio=2.036133 max_load_reduction=6107 max_load_reduction_pct=0.745483 moved_to_echo=3933 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 598, 0, 24, 802, 1023, 0, 789, 0, 2, 238, 2552, 178, 1981, 1] rerouted_tokens_per_ep_rank=[4, 598, 826, 1023, 789, 240, 2730, 1982] avg=1024.0000 max=2730 max_rank=6 min=4 min_rank=0 imbalance_ratio=2.666016 max_load_reduction=5462 max_load_reduction_pct=0.666748 moved_to_echo=5741 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1885, 0, 121, 0, 513, 1693, 2158, 0, 247, 0, 19, 48, 33, 36, 943, 496] rerouted_tokens_per_ep_rank=[1885, 121, 2206, 2158, 247, 67, 69, 1439] avg=1024.0000 max=2206 max_rank=2 min=67 min_rank=5 imbalance_ratio=2.154297 max_load_reduction=5986 max_load_reduction_pct=0.730713 moved_to_echo=1822 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1451, 0, 1347, 0, 299, 265, 339, 0, 2529, 0, 486, 536, 11, 402, 146, 381] rerouted_tokens_per_ep_rank=[1451, 1347, 564, 339, 2529, 1022, 413, 527] avg=1024.0000 max=2529 max_rank=4 min=339 min_rank=3 imbalance_ratio=2.469727 max_load_reduction=5663 max_load_reduction_pct=0.691284 moved_to_echo=4491 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[949, 0, 1372, 0, 15, 35, 45, 0, 1208, 0, 2971, 547, 0, 410, 391, 249] rerouted_tokens_per_ep_rank=[949, 1372, 50, 45, 1208, 3518, 410, 640] avg=1024.0000 max=3518 max_rank=5 min=45 min_rank=3 imbalance_ratio=3.435547 max_load_reduction=4674 max_load_reduction_pct=0.570557 moved_to_echo=5776 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1108, 0, 84, 0, 151, 1771, 2257, 0, 353, 0, 8, 32, 2065, 24, 48, 291] rerouted_tokens_per_ep_rank=[1108, 84, 1922, 2257, 353, 40, 2089, 339] avg=1024.0000 max=2257 max_rank=3 min=40 min_rank=5 imbalance_ratio=2.204102 max_load_reduction=5935 max_load_reduction_pct=0.724487 moved_to_echo=2821 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8938 expected_overflow=0 exact_recv_per_rank=[8410, 8192, 8192, 8938, 8237, 7446, 7974, 8147] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8410, 8192, 8192, 8938, 8237, 7446, 7974, 8147] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8410 expected_overflow=0 exact_recv_per_rank=[8410, 8192, 8192, 8938, 8237, 7446, 7974, 8147] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7974 expected_overflow=0 exact_recv_per_rank=[8410, 8192, 8192, 8938, 8237, 7446, 7974, 8147] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7446 expected_overflow=0 exact_recv_per_rank=[8410, 8192, 8192, 8938, 8237, 7446, 7974, 8147] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8147 expected_overflow=0 exact_recv_per_rank=[8410, 8192, 8192, 8938, 8237, 7446, 7974, 8147] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8237 expected_overflow=0 exact_recv_per_rank=[8410, 8192, 8192, 8938, 8237, 7446, 7974, 8147] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8410, 8192, 8192, 8938, 8237, 7446, 7974, 8147] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7446 actual_valid_recv=7552 observed_loss=-106 tokens_per_expert=[4224, 3328] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8410 actual_valid_recv=8448 observed_loss=-38 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7974 actual_valid_recv=8192 observed_loss=-218 tokens_per_expert=[5632, 2560] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1280, 7040] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8938 actual_valid_recv=8960 observed_loss=-22 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8147 actual_valid_recv=8320 observed_loss=-173 tokens_per_expert=[6016, 2304] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8237 actual_valid_recv=8320 observed_loss=-83 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[2918, 27, 252, 927, 4048, 4, 7, 9] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[16, 128, 1842, 3113, 1479, 13, 0, 1601] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[3, 696, 124, 2455, 3232, 42, 154, 1486] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1747, 2166, 1255, 2032, 395, 25, 369, 203] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[346, 300, 2576, 1457, 1728, 6, 201, 1578] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[7, 0, 710, 3711, 621, 983, 0, 2160] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[674, 224, 3850, 923, 896, 64, 967, 594] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[42, 96, 138, 3776, 157, 48, 579, 3356] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 28, 696, 378, 96, 0, 1094, 0, 3232, 0, 42, 935, 154, 426, 1108, 0] rerouted_tokens_per_ep_rank=[31, 1074, 96, 1094, 3232, 977, 580, 1108] avg=1024.0000 max=3232 max_rank=4 min=31 min_rank=0 imbalance_ratio=3.156250 max_load_reduction=4960 max_load_reduction_pct=0.605469 moved_to_echo=5897 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2918, 57, 27, 2, 195, 0, 413, 0, 4048, 0, 4, 353, 7, 161, 7, 0] rerouted_tokens_per_ep_rank=[2975, 29, 195, 413, 4048, 357, 168, 7] avg=1024.0000 max=4048 max_rank=4 min=7 min_rank=7 imbalance_ratio=3.953125 max_load_reduction=4144 max_load_reduction_pct=0.505859 moved_to_echo=4580 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1747, 287, 2166, 54, 968, 0, 899, 0, 395, 0, 25, 777, 369, 356, 149, 0] rerouted_tokens_per_ep_rank=[2034, 2220, 968, 899, 395, 802, 725, 149] avg=1024.0000 max=2220 max_rank=1 min=149 min_rank=7 imbalance_ratio=2.167969 max_load_reduction=5972 max_load_reduction_pct=0.729004 moved_to_echo=2071 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[16, 418, 128, 407, 1424, 0, 1388, 0, 1479, 0, 13, 1185, 0, 540, 1194, 0] rerouted_tokens_per_ep_rank=[434, 535, 1424, 1388, 1479, 1198, 540, 1194] avg=1024.0000 max=1479 max_rank=4 min=434 min_rank=0 imbalance_ratio=1.444336 max_load_reduction=6713 max_load_reduction_pct=0.819458 moved_to_echo=4411 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[346, 584, 300, 401, 1992, 0, 649, 0, 1728, 0, 6, 555, 201, 253, 1177, 0] rerouted_tokens_per_ep_rank=[930, 701, 1992, 649, 1728, 561, 454, 1177] avg=1024.0000 max=1992 max_rank=2 min=454 min_rank=6 imbalance_ratio=1.945312 max_load_reduction=6200 max_load_reduction_pct=0.756836 moved_to_echo=3920 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[42, 31, 96, 853, 107, 0, 1683, 0, 157, 0, 48, 1438, 579, 655, 2503, 0] rerouted_tokens_per_ep_rank=[73, 949, 107, 1683, 157, 1486, 1234, 2503] avg=1024.0000 max=2503 max_rank=7 min=73 min_rank=0 imbalance_ratio=2.444336 max_load_reduction=5689 max_load_reduction_pct=0.694458 moved_to_echo=5380 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[7, 161, 0, 549, 549, 0, 1654, 0, 621, 0, 983, 1413, 0, 644, 1611, 0] rerouted_tokens_per_ep_rank=[168, 549, 549, 1654, 621, 2396, 644, 1611] avg=1024.0000 max=2396 max_rank=5 min=168 min_rank=0 imbalance_ratio=2.339844 max_load_reduction=5796 max_load_reduction_pct=0.707520 moved_to_echo=5272 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[674, 873, 224, 151, 2977, 0, 412, 0, 896, 0, 64, 351, 967, 160, 443, 0] rerouted_tokens_per_ep_rank=[1547, 375, 2977, 412, 896, 415, 1127, 443] avg=1024.0000 max=2977 max_rank=2 min=375 min_rank=1 imbalance_ratio=2.907227 max_load_reduction=5215 max_load_reduction_pct=0.636597 moved_to_echo=2881 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8308 expected_overflow=0 exact_recv_per_rank=[8192, 6432, 8308, 8192, 12556, 8192, 5472, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6432, 8308, 8192, 12556, 8192, 5472, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6432 expected_overflow=0 exact_recv_per_rank=[8192, 6432, 8308, 8192, 12556, 8192, 5472, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6432, 8308, 8192, 12556, 8192, 5472, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6432, 8308, 8192, 12556, 8192, 5472, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=12556 expected_overflow=0 exact_recv_per_rank=[8192, 6432, 8308, 8192, 12556, 8192, 5472, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5472 expected_overflow=0 exact_recv_per_rank=[8192, 6432, 8308, 8192, 12556, 8192, 5472, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6432, 8308, 8192, 12556, 8192, 5472, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=5472 actual_valid_recv=5504 observed_loss=-32 tokens_per_expert=[2304, 3200] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=6432 actual_valid_recv=6528 observed_loss=-96 tokens_per_expert=[3712, 2816] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5760, 2560] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1280, 7040] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8308 actual_valid_recv=8320 observed_loss=-12 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=12556 actual_valid_recv=12672 observed_loss=-116 tokens_per_expert=[12672, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=5504 slack_slots=10880 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=6528 slack_slots=9856 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=12672 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[147, 58, 9, 3353, 67, 2090, 2286, 182] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 1960, 1, 1387, 1407, 532, 2291, 614] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1297, 441, 121, 1577, 2543, 160, 1215, 838] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[25, 1522, 611, 104, 2, 4046, 1560, 322] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[114, 3348, 223, 45, 35, 1052, 133, 3242] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1289, 490, 549, 361, 234, 2392, 624, 2253] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 515, 1, 694, 916, 1503, 3010, 1553] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[5, 61, 67, 755, 1301, 3722, 3, 2278] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 168, 1960, 0, 1, 226, 1387, 0, 1407, 288, 306, 0, 2003, 0, 446, 0] rerouted_tokens_per_ep_rank=[168, 1960, 227, 1387, 1695, 306, 2003, 446] avg=1024.0000 max=2003 max_rank=6 min=168 min_rank=0 imbalance_ratio=1.956055 max_load_reduction=6189 max_load_reduction_pct=0.755493 moved_to_echo=4450 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[147, 53, 58, 0, 9, 895, 3353, 0, 67, 292, 1195, 0, 1994, 0, 129, 0] rerouted_tokens_per_ep_rank=[200, 58, 904, 3353, 359, 1195, 1994, 129] avg=1024.0000 max=3353 max_rank=3 min=58 min_rank=1 imbalance_ratio=3.274414 max_load_reduction=4839 max_load_reduction_pct=0.590698 moved_to_echo=3677 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1297, 229, 441, 0, 121, 68, 1577, 0, 2543, 152, 92, 0, 1063, 0, 609, 0] rerouted_tokens_per_ep_rank=[1526, 441, 189, 1577, 2695, 92, 1063, 609] avg=1024.0000 max=2695 max_rank=4 min=92 min_rank=5 imbalance_ratio=2.631836 max_load_reduction=5497 max_load_reduction_pct=0.671021 moved_to_echo=4459 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[114, 887, 3348, 0, 223, 448, 45, 0, 35, 16, 604, 0, 117, 0, 2355, 0] rerouted_tokens_per_ep_rank=[1001, 3348, 671, 45, 51, 604, 117, 2355] avg=1024.0000 max=3348 max_rank=1 min=45 min_rank=3 imbalance_ratio=3.269531 max_load_reduction=4844 max_load_reduction_pct=0.591309 moved_to_echo=3127 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[25, 88, 1522, 0, 611, 1725, 104, 0, 2, 196, 2321, 0, 1364, 0, 234, 0] rerouted_tokens_per_ep_rank=[113, 1522, 2336, 104, 198, 2321, 1364, 234] avg=1024.0000 max=2336 max_rank=2 min=104 min_rank=3 imbalance_ratio=2.281250 max_load_reduction=5856 max_load_reduction_pct=0.714844 moved_to_echo=4117 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1289, 617, 490, 0, 549, 1020, 361, 0, 234, 78, 1372, 0, 546, 0, 1636, 0] rerouted_tokens_per_ep_rank=[1906, 490, 1569, 361, 312, 1372, 546, 1636] avg=1024.0000 max=1906 max_rank=0 min=312 min_rank=4 imbalance_ratio=1.861328 max_load_reduction=6286 max_load_reduction_pct=0.767334 moved_to_echo=3866 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 425, 515, 0, 1, 641, 694, 0, 916, 378, 862, 0, 2632, 0, 1128, 0] rerouted_tokens_per_ep_rank=[425, 515, 642, 694, 1294, 862, 2632, 1128] avg=1024.0000 max=2632 max_rank=6 min=425 min_rank=0 imbalance_ratio=2.570312 max_load_reduction=5560 max_load_reduction_pct=0.678711 moved_to_echo=5916 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[5, 623, 61, 0, 67, 1587, 755, 0, 1301, 0, 2135, 0, 3, 0, 1655, 0] rerouted_tokens_per_ep_rank=[628, 61, 1654, 755, 1301, 2135, 3, 1655] avg=1024.0000 max=2135 max_rank=5 min=3 min_rank=6 imbalance_ratio=2.084961 max_load_reduction=6057 max_load_reduction_pct=0.739380 moved_to_echo=5094 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8276 expected_overflow=0 exact_recv_per_rank=[5967, 8395, 8192, 8276, 7905, 8887, 9722, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5967 expected_overflow=0 exact_recv_per_rank=[5967, 8395, 8192, 8276, 7905, 8887, 9722, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[5967, 8395, 8192, 8276, 7905, 8887, 9722, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8395 expected_overflow=0 exact_recv_per_rank=[5967, 8395, 8192, 8276, 7905, 8887, 9722, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7905 expected_overflow=0 exact_recv_per_rank=[5967, 8395, 8192, 8276, 7905, 8887, 9722, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8887 expected_overflow=0 exact_recv_per_rank=[5967, 8395, 8192, 8276, 7905, 8887, 9722, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[5967, 8395, 8192, 8276, 7905, 8887, 9722, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9722 expected_overflow=0 exact_recv_per_rank=[5967, 8395, 8192, 8276, 7905, 8887, 9722, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=5967 actual_valid_recv=6144 observed_loss=-177 tokens_per_expert=[2944, 3200] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1664, 6656] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8395 actual_valid_recv=8448 observed_loss=-53 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8276 actual_valid_recv=8320 observed_loss=-44 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8887 actual_valid_recv=8960 observed_loss=-73 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7905 actual_valid_recv=7936 observed_loss=-31 tokens_per_expert=[6528, 1408] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9722 actual_valid_recv=9728 observed_loss=-6 tokens_per_expert=[9728, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=6144 slack_slots=10240 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=7936 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9728 slack_slots=6656 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[172, 3463, 127, 24, 1042, 403, 1695, 1266] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[839, 1379, 62, 108, 263, 4013, 1326, 202] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[28, 3214, 191, 1084, 1481, 925, 1084, 185] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[2, 2122, 23, 36, 425, 2107, 3454, 23] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[3, 3427, 610, 103, 753, 2926, 73, 297] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[51, 1616, 129, 6, 1858, 0, 956, 3576] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[340, 1098, 3443, 0, 1700, 1200, 379, 32] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[102, 3949, 474, 115, 879, 1434, 320, 919] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[172, 915, 1401, 0, 127, 97, 24, 1147, 1042, 0, 306, 0, 1496, 0, 1266, 199] rerouted_tokens_per_ep_rank=[1087, 1401, 224, 1171, 1042, 306, 1496, 1465] avg=1024.0000 max=1496 max_rank=6 min=224 min_rank=2 imbalance_ratio=1.460938 max_load_reduction=6696 max_load_reduction_pct=0.817383 moved_to_echo=4309 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[28, 849, 1301, 0, 191, 222, 1084, 1064, 1481, 0, 703, 0, 957, 0, 185, 127] rerouted_tokens_per_ep_rank=[877, 1301, 413, 2148, 1481, 703, 957, 312] avg=1024.0000 max=2148 max_rank=3 min=312 min_rank=7 imbalance_ratio=2.097656 max_load_reduction=6044 max_load_reduction_pct=0.737793 moved_to_echo=3453 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 565, 849, 0, 23, 510, 36, 708, 425, 0, 1597, 0, 3042, 0, 23, 412] rerouted_tokens_per_ep_rank=[567, 849, 533, 744, 425, 1597, 3042, 435] avg=1024.0000 max=3042 max_rank=6 min=425 min_rank=4 imbalance_ratio=2.970703 max_load_reduction=5150 max_load_reduction_pct=0.628662 moved_to_echo=5499 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[839, 364, 559, 0, 62, 966, 108, 456, 263, 0, 3047, 0, 1170, 0, 202, 156] rerouted_tokens_per_ep_rank=[1203, 559, 1028, 564, 263, 3047, 1170, 358] avg=1024.0000 max=3047 max_rank=5 min=263 min_rank=4 imbalance_ratio=2.975586 max_load_reduction=5145 max_load_reduction_pct=0.628052 moved_to_echo=4838 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[102, 1044, 1597, 0, 474, 345, 115, 1308, 879, 0, 1089, 0, 283, 0, 919, 37] rerouted_tokens_per_ep_rank=[1146, 1597, 819, 1423, 879, 1089, 283, 956] avg=1024.0000 max=1597 max_rank=1 min=283 min_rank=6 imbalance_ratio=1.559570 max_load_reduction=6595 max_load_reduction_pct=0.805054 moved_to_echo=3207 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[340, 290, 445, 0, 3443, 289, 0, 363, 1700, 0, 911, 0, 335, 0, 32, 44] rerouted_tokens_per_ep_rank=[630, 445, 3732, 363, 1700, 911, 335, 76] avg=1024.0000 max=3732 max_rank=2 min=76 min_rank=7 imbalance_ratio=3.644531 max_load_reduction=4460 max_load_reduction_pct=0.544434 moved_to_echo=3022 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[51, 427, 654, 0, 129, 0, 6, 535, 1858, 0, 0, 0, 844, 0, 3576, 112] rerouted_tokens_per_ep_rank=[478, 654, 129, 541, 1858, 0, 844, 3688] avg=1024.0000 max=3688 max_rank=7 min=0 min_rank=5 imbalance_ratio=3.601562 max_load_reduction=4504 max_load_reduction_pct=0.549805 moved_to_echo=6390 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 906, 1386, 0, 610, 704, 103, 1135, 753, 0, 2222, 0, 65, 0, 297, 8] rerouted_tokens_per_ep_rank=[909, 1386, 1314, 1238, 753, 2222, 65, 305] avg=1024.0000 max=2222 max_rank=5 min=65 min_rank=6 imbalance_ratio=2.169922 max_load_reduction=5970 max_load_reduction_pct=0.728760 moved_to_echo=3345 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7595 expected_overflow=0 exact_recv_per_rank=[6897, 8192, 8192, 8192, 8401, 9875, 8192, 7595] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[6897, 8192, 8192, 8192, 8401, 9875, 8192, 7595] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6897 expected_overflow=0 exact_recv_per_rank=[6897, 8192, 8192, 8192, 8401, 9875, 8192, 7595] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9875 expected_overflow=0 exact_recv_per_rank=[6897, 8192, 8192, 8192, 8401, 9875, 8192, 7595] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[6897, 8192, 8192, 8192, 8401, 9875, 8192, 7595] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[6897, 8192, 8192, 8192, 8401, 9875, 8192, 7595] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8401 expected_overflow=0 exact_recv_per_rank=[6897, 8192, 8192, 8192, 8401, 9875, 8192, 7595] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[6897, 8192, 8192, 8192, 8401, 9875, 8192, 7595] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=6897 actual_valid_recv=7040 observed_loss=-143 tokens_per_expert=[1664, 5376] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=7595 actual_valid_recv=7680 observed_loss=-85 tokens_per_expert=[6528, 1152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1536, 6784] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8401 actual_valid_recv=8448 observed_loss=-47 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=9875 actual_valid_recv=9984 observed_loss=-109 tokens_per_expert=[9984, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5120, 3200] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=7040 slack_slots=9344 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=9984 slack_slots=6400 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[105, 3603, 336, 350, 1405, 360, 667, 1366] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[102, 3113, 515, 419, 1694, 389, 671, 1289] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[121, 3058, 907, 175, 1564, 308, 853, 1206] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[78, 3546, 437, 459, 1223, 385, 1090, 974] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[108, 3142, 402, 309, 1785, 542, 492, 1412] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[75, 3296, 487, 344, 1258, 532, 587, 1613] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[208, 3434, 626, 457, 1038, 331, 988, 1110] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[83, 3374, 906, 440, 1191, 392, 694, 1112] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[105, 991, 1231, 0, 336, 340, 350, 710, 1065, 0, 360, 671, 667, 256, 1110, 0] rerouted_tokens_per_ep_rank=[1096, 1231, 676, 1060, 1065, 1031, 923, 1110] avg=1024.0000 max=1231 max_rank=1 min=676 min_rank=2 imbalance_ratio=1.202148 max_load_reduction=6961 max_load_reduction_pct=0.849731 moved_to_echo=4129 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[121, 846, 1032, 0, 907, 384, 175, 607, 1180, 0, 308, 573, 853, 229, 977, 0] rerouted_tokens_per_ep_rank=[967, 1032, 1291, 782, 1180, 881, 1082, 977] avg=1024.0000 max=1291 max_rank=2 min=782 min_rank=3 imbalance_ratio=1.260742 max_load_reduction=6901 max_load_reduction_pct=0.842407 moved_to_echo=4120 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[78, 975, 1211, 0, 437, 296, 459, 699, 927, 0, 385, 661, 1090, 182, 792, 0] rerouted_tokens_per_ep_rank=[1053, 1211, 733, 1158, 927, 1046, 1272, 792] avg=1024.0000 max=1272 max_rank=6 min=733 min_rank=2 imbalance_ratio=1.242188 max_load_reduction=6920 max_load_reduction_pct=0.844727 moved_to_echo=4037 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[102, 856, 1064, 0, 515, 410, 419, 613, 1284, 0, 389, 580, 671, 241, 1048, 0] rerouted_tokens_per_ep_rank=[958, 1064, 925, 1032, 1284, 969, 912, 1048] avg=1024.0000 max=1284 max_rank=4 min=912 min_rank=6 imbalance_ratio=1.253906 max_load_reduction=6908 max_load_reduction_pct=0.843262 moved_to_echo=4213 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[108, 864, 1074, 0, 402, 432, 309, 619, 1353, 0, 542, 585, 492, 264, 1148, 0] rerouted_tokens_per_ep_rank=[972, 1074, 834, 928, 1353, 1127, 756, 1148] avg=1024.0000 max=1353 max_rank=4 min=756 min_rank=6 imbalance_ratio=1.321289 max_load_reduction=6839 max_load_reduction_pct=0.834839 moved_to_echo=4384 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[208, 945, 1172, 0, 626, 251, 457, 677, 787, 0, 331, 640, 988, 208, 902, 0] rerouted_tokens_per_ep_rank=[1153, 1172, 877, 1134, 787, 971, 1196, 902] avg=1024.0000 max=1196 max_rank=6 min=787 min_rank=4 imbalance_ratio=1.167969 max_load_reduction=6996 max_load_reduction_pct=0.854004 moved_to_echo=3856 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[75, 907, 1126, 0, 487, 305, 344, 649, 953, 0, 532, 614, 587, 302, 1311, 0] rerouted_tokens_per_ep_rank=[982, 1126, 792, 993, 953, 1146, 889, 1311] avg=1024.0000 max=1311 max_rank=7 min=792 min_rank=2 imbalance_ratio=1.280273 max_load_reduction=6881 max_load_reduction_pct=0.839966 moved_to_echo=4299 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[83, 928, 1152, 0, 906, 288, 440, 665, 903, 0, 392, 629, 694, 208, 904, 0] rerouted_tokens_per_ep_rank=[1011, 1152, 1194, 1105, 903, 1021, 902, 904] avg=1024.0000 max=1194 max_rank=2 min=902 min_rank=6 imbalance_ratio=1.166016 max_load_reduction=6998 max_load_reduction_pct=0.854248 moved_to_echo=3730 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7322 expected_overflow=0 exact_recv_per_rank=[8192, 9062, 7322, 8192, 8452, 8192, 7932, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9062, 7322, 8192, 8452, 8192, 7932, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9062, 7322, 8192, 8452, 8192, 7932, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9062 expected_overflow=0 exact_recv_per_rank=[8192, 9062, 7322, 8192, 8452, 8192, 7932, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9062, 7322, 8192, 8452, 8192, 7932, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7932 expected_overflow=0 exact_recv_per_rank=[8192, 9062, 7322, 8192, 8452, 8192, 7932, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8452 expected_overflow=0 exact_recv_per_rank=[8192, 9062, 7322, 8192, 8452, 8192, 7932, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9062, 7322, 8192, 8452, 8192, 7932, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=9062 actual_valid_recv=9088 observed_loss=-26 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7932 actual_valid_recv=8064 observed_loss=-132 tokens_per_expert=[6144, 1920] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[896, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3328, 4992] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3072, 5248] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=7322 actual_valid_recv=7552 observed_loss=-230 tokens_per_expert=[4736, 2816] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8452 actual_valid_recv=8576 observed_loss=-124 tokens_per_expert=[8576, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8064 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8576 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1474, 25, 223, 815, 1717, 1241, 1251, 1446] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[752, 138, 397, 946, 1526, 356, 2063, 2014] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[517, 78, 180, 1162, 2550, 709, 1468, 1528] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[851, 178, 332, 916, 1940, 460, 971, 2544] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1044, 177, 158, 1085, 3019, 548, 1048, 1113] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[655, 269, 486, 958, 2447, 476, 1728, 1173] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1216, 81, 267, 950, 1769, 326, 1712, 1871] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1593, 135, 260, 1070, 2098, 242, 994, 1800] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1474, 10, 25, 715, 223, 442, 815, 32, 1002, 0, 1241, 296, 913, 0, 1004, 0] rerouted_tokens_per_ep_rank=[1484, 740, 665, 847, 1002, 1537, 913, 1004] avg=1024.0000 max=1537 max_rank=5 min=665 min_rank=2 imbalance_ratio=1.500977 max_load_reduction=6655 max_load_reduction_pct=0.812378 moved_to_echo=4456 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[517, 11, 78, 1062, 180, 467, 1162, 37, 1488, 0, 709, 347, 1073, 0, 1061, 0] rerouted_tokens_per_ep_rank=[528, 1140, 647, 1199, 1488, 1056, 1073, 1061] avg=1024.0000 max=1488 max_rank=4 min=528 min_rank=0 imbalance_ratio=1.453125 max_load_reduction=6704 max_load_reduction_pct=0.818359 moved_to_echo=4678 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[752, 16, 138, 635, 397, 616, 946, 53, 891, 0, 356, 488, 1506, 0, 1398, 0] rerouted_tokens_per_ep_rank=[768, 773, 1013, 999, 891, 844, 1506, 1398] avg=1024.0000 max=1506 max_rank=6 min=768 min_rank=0 imbalance_ratio=1.470703 max_load_reduction=6686 max_load_reduction_pct=0.816162 moved_to_echo=4639 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[655, 18, 269, 1023, 486, 361, 958, 47, 1424, 0, 476, 414, 1249, 0, 812, 0] rerouted_tokens_per_ep_rank=[673, 1292, 847, 1005, 1424, 890, 1249, 812] avg=1024.0000 max=1424 max_rank=4 min=673 min_rank=0 imbalance_ratio=1.390625 max_load_reduction=6768 max_load_reduction_pct=0.826172 moved_to_echo=4375 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1216, 13, 81, 737, 267, 572, 950, 44, 1032, 0, 326, 405, 1250, 0, 1299, 0] rerouted_tokens_per_ep_rank=[1229, 818, 839, 994, 1032, 731, 1250, 1299] avg=1024.0000 max=1299 max_rank=7 min=731 min_rank=5 imbalance_ratio=1.268555 max_load_reduction=6893 max_load_reduction_pct=0.841431 moved_to_echo=4312 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1044, 8, 177, 1257, 158, 340, 1085, 27, 1762, 0, 548, 248, 765, 0, 773, 0] rerouted_tokens_per_ep_rank=[1052, 1434, 498, 1112, 1762, 796, 765, 773] avg=1024.0000 max=1762 max_rank=4 min=498 min_rank=2 imbalance_ratio=1.720703 max_load_reduction=6430 max_load_reduction_pct=0.784912 moved_to_echo=4096 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[851, 7, 178, 808, 332, 778, 916, 25, 1132, 0, 460, 230, 709, 0, 1766, 0] rerouted_tokens_per_ep_rank=[858, 986, 1110, 941, 1132, 690, 709, 1766] avg=1024.0000 max=1766 max_rank=7 min=690 min_rank=5 imbalance_ratio=1.724609 max_load_reduction=6426 max_load_reduction_pct=0.784424 moved_to_echo=4297 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1593, 7, 135, 874, 260, 550, 1070, 25, 1224, 0, 242, 235, 727, 0, 1250, 0] rerouted_tokens_per_ep_rank=[1600, 1009, 810, 1095, 1224, 477, 727, 1250] avg=1024.0000 max=1600 max_rank=0 min=477 min_rank=5 imbalance_ratio=1.562500 max_load_reduction=6592 max_load_reduction_pct=0.804688 moved_to_echo=3678 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6429, 8192, 9955, 7021, 8192, 9363] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6429, 8192, 9955, 7021, 8192, 9363] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6429, 8192, 9955, 7021, 8192, 9363] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7021 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6429, 8192, 9955, 7021, 8192, 9363] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9363 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6429, 8192, 9955, 7021, 8192, 9363] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6429 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6429, 8192, 9955, 7021, 8192, 9363] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9955 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6429, 8192, 9955, 7021, 8192, 9363] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6429, 8192, 9955, 7021, 8192, 9363] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[8192, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7021 actual_valid_recv=7168 observed_loss=-147 tokens_per_expert=[4480, 2688] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=9955 actual_valid_recv=9984 observed_loss=-29 tokens_per_expert=[9984, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7936, 384] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=6429 actual_valid_recv=6528 observed_loss=-99 tokens_per_expert=[2304, 4224] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=9363 actual_valid_recv=9472 observed_loss=-109 tokens_per_expert=[9472, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7168 slack_slots=9216 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=9984 slack_slots=6400 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=9472 slack_slots=6912 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=6528 slack_slots=9856 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1849, 272, 448, 1702, 125, 2562, 1205, 29] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1389, 125, 774, 1855, 189, 1763, 2006, 91] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[741, 339, 1721, 3017, 228, 1312, 479, 355] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1312, 359, 1727, 1706, 55, 1597, 913, 523] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1303, 251, 853, 1628, 178, 1366, 967, 1646] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1474, 71, 1106, 2415, 265, 1835, 366, 660] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[2484, 120, 1517, 2692, 61, 395, 840, 83] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[621, 306, 812, 2051, 258, 1587, 1660, 897] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1370, 0, 272, 871, 448, 0, 1021, 0, 125, 681, 1691, 0, 1205, 0, 29, 479] rerouted_tokens_per_ep_rank=[1370, 1143, 448, 1021, 806, 1691, 1205, 508] avg=1024.0000 max=1691 max_rank=5 min=448 min_rank=2 imbalance_ratio=1.651367 max_load_reduction=6501 max_load_reduction_pct=0.793579 moved_to_echo=4210 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1029, 0, 125, 599, 774, 0, 1113, 0, 189, 742, 1164, 0, 2006, 0, 91, 360] rerouted_tokens_per_ep_rank=[1029, 724, 774, 1113, 931, 1164, 2006, 451] avg=1024.0000 max=2006 max_rank=6 min=451 min_rank=7 imbalance_ratio=1.958984 max_load_reduction=6186 max_load_reduction_pct=0.755127 moved_to_echo=4552 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[972, 0, 359, 543, 1727, 0, 1023, 0, 55, 683, 1054, 0, 913, 0, 523, 340] rerouted_tokens_per_ep_rank=[972, 902, 1727, 1023, 738, 1054, 913, 863] avg=1024.0000 max=1727 max_rank=2 min=738 min_rank=4 imbalance_ratio=1.686523 max_load_reduction=6465 max_load_reduction_pct=0.789185 moved_to_echo=3568 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[546, 0, 339, 451, 1721, 0, 1805, 0, 228, 1212, 861, 0, 479, 0, 355, 195] rerouted_tokens_per_ep_rank=[546, 790, 1721, 1805, 1440, 861, 479, 550] avg=1024.0000 max=1805 max_rank=3 min=479 min_rank=6 imbalance_ratio=1.762695 max_load_reduction=6387 max_load_reduction_pct=0.779663 moved_to_echo=3330 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1840, 0, 120, 134, 1517, 0, 1615, 0, 61, 1077, 261, 0, 840, 0, 83, 644] rerouted_tokens_per_ep_rank=[1840, 254, 1517, 1615, 1138, 261, 840, 727] avg=1024.0000 max=1840 max_rank=0 min=254 min_rank=1 imbalance_ratio=1.796875 max_load_reduction=6352 max_load_reduction_pct=0.775391 moved_to_echo=2966 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[966, 0, 251, 464, 853, 0, 977, 0, 178, 651, 902, 0, 967, 0, 1646, 337] rerouted_tokens_per_ep_rank=[966, 715, 853, 977, 829, 902, 967, 1983] avg=1024.0000 max=1983 max_rank=7 min=715 min_rank=1 imbalance_ratio=1.936523 max_load_reduction=6209 max_load_reduction_pct=0.757935 moved_to_echo=4681 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1092, 0, 71, 624, 1106, 0, 1449, 0, 265, 966, 1211, 0, 366, 0, 660, 382] rerouted_tokens_per_ep_rank=[1092, 695, 1106, 1449, 1231, 1211, 366, 1042] avg=1024.0000 max=1449 max_rank=3 min=366 min_rank=6 imbalance_ratio=1.415039 max_load_reduction=6743 max_load_reduction_pct=0.823120 moved_to_echo=3850 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[460, 0, 306, 539, 812, 0, 1230, 0, 258, 821, 1048, 0, 1660, 0, 897, 161] rerouted_tokens_per_ep_rank=[460, 845, 812, 1230, 1079, 1048, 1660, 1058] avg=1024.0000 max=1660 max_rank=6 min=460 min_rank=0 imbalance_ratio=1.621094 max_load_reduction=6532 max_load_reduction_pct=0.797363 moved_to_echo=4845 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10233 expected_overflow=0 exact_recv_per_rank=[8275, 6068, 8958, 10233, 8192, 8192, 8436, 7182] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8275, 6068, 8958, 10233, 8192, 8192, 8436, 7182] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8275, 6068, 8958, 10233, 8192, 8192, 8436, 7182] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8958 expected_overflow=0 exact_recv_per_rank=[8275, 6068, 8958, 10233, 8192, 8192, 8436, 7182] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6068 expected_overflow=0 exact_recv_per_rank=[8275, 6068, 8958, 10233, 8192, 8192, 8436, 7182] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8275 expected_overflow=0 exact_recv_per_rank=[8275, 6068, 8958, 10233, 8192, 8192, 8436, 7182] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7182 expected_overflow=0 exact_recv_per_rank=[8275, 6068, 8958, 10233, 8192, 8192, 8436, 7182] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8436 expected_overflow=0 exact_recv_per_rank=[8275, 6068, 8958, 10233, 8192, 8192, 8436, 7182] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8958 actual_valid_recv=8960 observed_loss=-2 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8436 actual_valid_recv=8448 observed_loss=-12 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=6068 actual_valid_recv=6272 observed_loss=-204 tokens_per_expert=[1920, 4352] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8275 actual_valid_recv=8320 observed_loss=-45 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=10233 actual_valid_recv=10240 observed_loss=-7 tokens_per_expert=[10240, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=7182 actual_valid_recv=7296 observed_loss=-114 tokens_per_expert=[4352, 2944] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1408, 6912] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=6272 slack_slots=10112 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=10240 slack_slots=6144 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=7296 slack_slots=9088 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1034, 3684, 36, 1202, 1145, 421, 518, 152] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1469, 3022, 144, 884, 782, 43, 404, 1444] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[437, 3572, 66, 872, 453, 167, 574, 2051] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1138, 1836, 869, 1269, 1040, 559, 1392, 89] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1164, 3252, 95, 186, 1946, 338, 803, 408] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[747, 1631, 73, 1439, 557, 992, 2712, 41] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[406, 3008, 66, 935, 1076, 735, 1088, 878] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[2692, 1871, 372, 278, 820, 705, 200, 1254] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1329, 0, 1134, 0, 144, 893, 884, 152, 782, 60, 43, 584, 404, 80, 1444, 259] rerouted_tokens_per_ep_rank=[1329, 1134, 1037, 1036, 842, 627, 484, 1703] avg=1024.0000 max=1703 max_rank=7 min=484 min_rank=6 imbalance_ratio=1.663086 max_load_reduction=6489 max_load_reduction_pct=0.792114 moved_to_echo=3656 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1030, 0, 689, 0, 869, 543, 1269, 92, 1040, 46, 559, 355, 1392, 62, 89, 157] rerouted_tokens_per_ep_rank=[1030, 689, 1412, 1361, 1086, 914, 1454, 246] avg=1024.0000 max=1454 max_rank=6 min=246 min_rank=7 imbalance_ratio=1.419922 max_load_reduction=6738 max_load_reduction_pct=0.822510 moved_to_echo=3700 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[935, 0, 1382, 0, 36, 1089, 1202, 186, 1145, 42, 421, 712, 518, 57, 152, 315] rerouted_tokens_per_ep_rank=[935, 1382, 1125, 1388, 1187, 1133, 575, 467] avg=1024.0000 max=1388 max_rank=3 min=467 min_rank=7 imbalance_ratio=1.355469 max_load_reduction=6804 max_load_reduction_pct=0.830566 moved_to_echo=3362 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[368, 0, 1129, 0, 66, 889, 935, 152, 1076, 16, 735, 581, 1088, 22, 878, 257] rerouted_tokens_per_ep_rank=[368, 1129, 955, 1087, 1092, 1316, 1110, 1135] avg=1024.0000 max=1316 max_rank=5 min=368 min_rank=0 imbalance_ratio=1.285156 max_load_reduction=6876 max_load_reduction_pct=0.839355 moved_to_echo=4653 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[396, 0, 1339, 0, 66, 1056, 872, 180, 453, 17, 167, 691, 574, 24, 2051, 306] rerouted_tokens_per_ep_rank=[396, 1339, 1122, 1052, 470, 858, 598, 2357] avg=1024.0000 max=2357 max_rank=7 min=396 min_rank=0 imbalance_ratio=2.301758 max_load_reduction=5835 max_load_reduction_pct=0.712280 moved_to_echo=4283 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1053, 0, 1220, 0, 95, 961, 186, 164, 1946, 47, 338, 629, 803, 64, 408, 278] rerouted_tokens_per_ep_rank=[1053, 1220, 1056, 350, 1993, 967, 867, 686] avg=1024.0000 max=1993 max_rank=4 min=350 min_rank=3 imbalance_ratio=1.946289 max_load_reduction=6199 max_load_reduction_pct=0.756714 moved_to_echo=4513 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[668, 0, 596, 0, 73, 487, 1439, 86, 557, 35, 992, 319, 2712, 44, 41, 143] rerouted_tokens_per_ep_rank=[668, 596, 560, 1525, 592, 1311, 2756, 184] avg=1024.0000 max=2756 max_rank=6 min=184 min_rank=7 imbalance_ratio=2.691406 max_load_reduction=5436 max_load_reduction_pct=0.663574 moved_to_echo=4843 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2434, 0, 703, 0, 372, 553, 278, 94, 820, 110, 705, 361, 200, 148, 1254, 160] rerouted_tokens_per_ep_rank=[2434, 703, 925, 372, 930, 1066, 348, 1414] avg=1024.0000 max=2434 max_rank=0 min=348 min_rank=6 imbalance_ratio=2.376953 max_load_reduction=5758 max_load_reduction_pct=0.702881 moved_to_echo=3758 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8171 expected_overflow=0 exact_recv_per_rank=[8213, 8192, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8213, 8192, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8213, 8192, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8213, 8192, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8213, 8192, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8213, 8192, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8213 expected_overflow=0 exact_recv_per_rank=[8213, 8192, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8213, 8192, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6400, 1920] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7808, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3968, 4352] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1792, 6528] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8213 actual_valid_recv=8320 observed_loss=-107 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7936, 384] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8171 actual_valid_recv=8320 observed_loss=-149 tokens_per_expert=[7168, 1152] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[695, 1263, 667, 2068, 1604, 224, 505, 1166] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[2337, 796, 1067, 1191, 285, 85, 278, 2153] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2124, 78, 251, 1485, 117, 1123, 1073, 1941] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1357, 2250, 257, 1583, 930, 591, 256, 968] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1033, 643, 1609, 225, 385, 1921, 2343, 33] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[555, 496, 1267, 541, 1267, 1016, 1575, 1475] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3756, 418, 1033, 1030, 505, 839, 82, 529] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3618, 897, 98, 148, 1145, 1117, 91, 1078] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[372, 0, 1263, 60, 667, 87, 2068, 0, 1604, 87, 224, 143, 505, 89, 1023, 0] rerouted_tokens_per_ep_rank=[372, 1323, 754, 2068, 1691, 367, 594, 1023] avg=1024.0000 max=2068 max_rank=3 min=367 min_rank=5 imbalance_ratio=2.019531 max_load_reduction=6124 max_load_reduction_pct=0.747559 moved_to_echo=3675 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1245, 0, 796, 204, 1067, 293, 1191, 0, 285, 295, 85, 265, 278, 300, 1888, 0] rerouted_tokens_per_ep_rank=[1245, 1000, 1360, 1191, 580, 350, 578, 1888] avg=1024.0000 max=1888 max_rank=7 min=350 min_rank=5 imbalance_ratio=1.843750 max_load_reduction=6304 max_load_reduction_pct=0.769531 moved_to_echo=3396 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[297, 0, 496, 48, 1267, 69, 541, 0, 1267, 70, 1016, 181, 1575, 71, 1294, 0] rerouted_tokens_per_ep_rank=[297, 544, 1336, 541, 1337, 1197, 1646, 1294] avg=1024.0000 max=1646 max_rank=6 min=297 min_rank=0 imbalance_ratio=1.607422 max_load_reduction=6546 max_load_reduction_pct=0.799072 moved_to_echo=5474 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1133, 0, 78, 185, 251, 266, 1485, 0, 117, 268, 1123, 239, 1073, 272, 1702, 0] rerouted_tokens_per_ep_rank=[1133, 263, 517, 1485, 385, 1362, 1345, 1702] avg=1024.0000 max=1702 max_rank=7 min=263 min_rank=1 imbalance_ratio=1.662109 max_load_reduction=6490 max_load_reduction_pct=0.792236 moved_to_echo=4794 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[552, 0, 643, 90, 1609, 129, 225, 0, 385, 130, 1921, 4, 2343, 132, 29, 0] rerouted_tokens_per_ep_rank=[552, 733, 1738, 225, 515, 1925, 2475, 29] avg=1024.0000 max=2475 max_rank=6 min=29 min_rank=7 imbalance_ratio=2.416992 max_load_reduction=5717 max_load_reduction_pct=0.697876 moved_to_echo=4944 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[709, 0, 2250, 122, 257, 174, 1583, 0, 930, 174, 591, 122, 256, 178, 846, 0] rerouted_tokens_per_ep_rank=[709, 2372, 431, 1583, 1104, 713, 434, 846] avg=1024.0000 max=2372 max_rank=1 min=431 min_rank=2 imbalance_ratio=2.316406 max_load_reduction=5820 max_load_reduction_pct=0.710449 moved_to_echo=3097 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2002, 0, 418, 327, 1033, 471, 1030, 0, 505, 474, 839, 65, 82, 482, 464, 0] rerouted_tokens_per_ep_rank=[2002, 745, 1504, 1030, 979, 904, 564, 464] avg=1024.0000 max=2002 max_rank=0 min=464 min_rank=7 imbalance_ratio=1.955078 max_load_reduction=6190 max_load_reduction_pct=0.755615 moved_to_echo=2911 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1928, 0, 897, 315, 98, 454, 148, 0, 1145, 456, 1117, 132, 91, 465, 946, 0] rerouted_tokens_per_ep_rank=[1928, 1212, 552, 148, 1601, 1249, 556, 946] avg=1024.0000 max=1928 max_rank=0 min=148 min_rank=3 imbalance_ratio=1.882812 max_load_reduction=6264 max_load_reduction_pct=0.764648 moved_to_echo=4352 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8271 expected_overflow=0 exact_recv_per_rank=[8238, 8192, 8192, 8271, 8192, 8067, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8238, 8192, 8192, 8271, 8192, 8067, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8238, 8192, 8192, 8271, 8192, 8067, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8238, 8192, 8192, 8271, 8192, 8067, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8067 expected_overflow=0 exact_recv_per_rank=[8238, 8192, 8192, 8271, 8192, 8067, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8238, 8192, 8192, 8271, 8192, 8067, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8238 expected_overflow=0 exact_recv_per_rank=[8238, 8192, 8192, 8271, 8192, 8067, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8238, 8192, 8192, 8271, 8192, 8067, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8067 actual_valid_recv=8192 observed_loss=-125 tokens_per_expert=[7040, 1152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8238 actual_valid_recv=8320 observed_loss=-82 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6272, 2048] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8271 actual_valid_recv=8320 observed_loss=-49 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6912, 1408] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6272, 2048] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6272, 2048] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[660, 577, 364, 1263, 2793, 1761, 75, 699] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[256, 110, 597, 3347, 430, 2600, 823, 29] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1249, 319, 82, 138, 2319, 261, 2240, 1584] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[307, 645, 173, 411, 1365, 816, 2361, 2114] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[420, 889, 2062, 0, 3930, 302, 471, 118] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[19, 1007, 53, 2436, 3110, 698, 654, 215] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1294, 640, 1662, 156, 1821, 1381, 65, 1173] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[887, 1916, 89, 47, 1710, 1809, 10, 1724] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[256, 76, 110, 51, 597, 76, 3347, 106, 203, 0, 2350, 0, 823, 24, 29, 144] rerouted_tokens_per_ep_rank=[332, 161, 673, 3453, 203, 2350, 847, 173] avg=1024.0000 max=3453 max_rank=3 min=161 min_rank=1 imbalance_ratio=3.372070 max_load_reduction=4739 max_load_reduction_pct=0.578491 moved_to_echo=3573 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1249, 411, 319, 277, 82, 412, 138, 10, 1089, 0, 237, 0, 2240, 130, 1584, 14] rerouted_tokens_per_ep_rank=[1660, 596, 494, 148, 1089, 237, 2370, 1598] avg=1024.0000 max=2370 max_rank=6 min=148 min_rank=3 imbalance_ratio=2.314453 max_load_reduction=5822 max_load_reduction_pct=0.710693 moved_to_echo=5294 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[307, 242, 645, 163, 173, 242, 411, 33, 641, 0, 738, 0, 2361, 77, 2114, 45] rerouted_tokens_per_ep_rank=[549, 808, 415, 444, 641, 738, 2438, 2159] avg=1024.0000 max=2438 max_rank=6 min=415 min_rank=2 imbalance_ratio=2.380859 max_load_reduction=5754 max_load_reduction_pct=0.702393 moved_to_echo=5976 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[660, 495, 577, 333, 364, 496, 1263, 72, 1312, 0, 1591, 0, 75, 157, 699, 98] rerouted_tokens_per_ep_rank=[1155, 910, 860, 1335, 1312, 1591, 232, 797] avg=1024.0000 max=1591 max_rank=5 min=232 min_rank=6 imbalance_ratio=1.553711 max_load_reduction=6601 max_load_reduction_pct=0.805786 moved_to_echo=3932 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1294, 325, 640, 221, 1662, 328, 156, 59, 840, 0, 1241, 0, 65, 107, 1173, 81] rerouted_tokens_per_ep_rank=[1619, 861, 1990, 215, 840, 1241, 172, 1254] avg=1024.0000 max=1990 max_rank=2 min=172 min_rank=6 imbalance_ratio=1.943359 max_load_reduction=6202 max_load_reduction_pct=0.757080 moved_to_echo=3507 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[19, 551, 1007, 371, 53, 553, 2436, 28, 1460, 0, 632, 0, 654, 175, 215, 38] rerouted_tokens_per_ep_rank=[570, 1378, 606, 2464, 1460, 632, 829, 253] avg=1024.0000 max=2464 max_rank=3 min=253 min_rank=7 imbalance_ratio=2.406250 max_load_reduction=5728 max_load_reduction_pct=0.699219 moved_to_echo=3174 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[420, 697, 889, 469, 2062, 699, 0, 12, 1844, 0, 274, 0, 471, 221, 118, 16] rerouted_tokens_per_ep_rank=[1117, 1358, 2761, 12, 1844, 274, 692, 134] avg=1024.0000 max=2761 max_rank=2 min=12 min_rank=3 imbalance_ratio=2.696289 max_load_reduction=5431 max_load_reduction_pct=0.662964 moved_to_echo=2944 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[887, 303, 1916, 204, 89, 304, 47, 74, 803, 0, 1635, 0, 10, 96, 1724, 100] rerouted_tokens_per_ep_rank=[1190, 2120, 393, 121, 803, 1635, 106, 1824] avg=1024.0000 max=2120 max_rank=1 min=106 min_rank=6 imbalance_ratio=2.070312 max_load_reduction=6072 max_load_reduction_pct=0.741211 moved_to_echo=4368 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8698, 7686, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8698, 7686, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7686 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8698, 7686, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8698 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8698, 7686, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8698, 7686, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8698, 7686, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8698, 7686, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8698, 7686, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7686 actual_valid_recv=7808 observed_loss=-122 tokens_per_expert=[6784, 1024] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7808, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5120, 3200] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8698 actual_valid_recv=8704 observed_loss=-6 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7680, 640] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5120, 3200] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6144, 2176] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1684, 334, 145, 320, 44, 731, 1057, 3877] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[2566, 2, 65, 795, 772, 1040, 2782, 170] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[407, 141, 67, 243, 4, 393, 2945, 3992] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3597, 28, 51, 22, 200, 210, 176, 3908] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3481, 11, 179, 479, 2, 498, 2529, 1013] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1073, 300, 138, 29, 298, 3885, 324, 2145] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3026, 7, 276, 60, 19, 460, 2998, 1346] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1642, 3, 568, 16, 364, 20, 3599, 1980] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1687, 0, 28, 1561, 51, 788, 22, 66, 200, 1122, 210, 10, 100, 0, 2347, 0] rerouted_tokens_per_ep_rank=[1687, 1589, 839, 88, 1322, 220, 100, 2347] avg=1024.0000 max=2347 max_rank=7 min=88 min_rank=3 imbalance_ratio=2.291992 max_load_reduction=5845 max_load_reduction_pct=0.713501 moved_to_echo=3989 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[783, 0, 334, 1554, 145, 372, 320, 407, 44, 529, 731, 65, 585, 0, 2323, 0] rerouted_tokens_per_ep_rank=[783, 1888, 517, 727, 573, 796, 585, 2323] avg=1024.0000 max=2323 max_rank=7 min=517 min_rank=2 imbalance_ratio=2.268555 max_load_reduction=5869 max_load_reduction_pct=0.716431 moved_to_echo=4277 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1204, 0, 2, 67, 65, 562, 795, 1055, 772, 800, 1040, 161, 1566, 0, 103, 0] rerouted_tokens_per_ep_rank=[1204, 69, 627, 1850, 1572, 1201, 1566, 103] avg=1024.0000 max=1850 max_rank=3 min=69 min_rank=1 imbalance_ratio=1.806641 max_load_reduction=6342 max_load_reduction_pct=0.774170 moved_to_echo=4442 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[771, 0, 3, 791, 568, 359, 16, 1365, 364, 512, 20, 209, 2025, 0, 1189, 0] rerouted_tokens_per_ep_rank=[771, 794, 927, 1381, 876, 229, 2025, 1189] avg=1024.0000 max=2025 max_rank=6 min=229 min_rank=5 imbalance_ratio=1.977539 max_load_reduction=6167 max_load_reduction_pct=0.752808 moved_to_echo=4319 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[191, 0, 141, 1595, 67, 89, 243, 1117, 4, 127, 393, 171, 1657, 0, 2397, 0] rerouted_tokens_per_ep_rank=[191, 1736, 156, 1360, 131, 564, 1657, 2397] avg=1024.0000 max=2397 max_rank=7 min=131 min_rank=4 imbalance_ratio=2.340820 max_load_reduction=5795 max_load_reduction_pct=0.707397 moved_to_echo=4749 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1633, 0, 11, 404, 179, 762, 479, 959, 2, 1086, 498, 147, 1423, 0, 609, 0] rerouted_tokens_per_ep_rank=[1633, 415, 941, 1438, 1088, 645, 1423, 609] avg=1024.0000 max=1633 max_rank=0 min=415 min_rank=1 imbalance_ratio=1.594727 max_load_reduction=6559 max_load_reduction_pct=0.800659 moved_to_echo=3765 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[504, 0, 300, 857, 138, 235, 29, 122, 298, 334, 3885, 18, 184, 0, 1288, 0] rerouted_tokens_per_ep_rank=[504, 1157, 373, 151, 632, 3903, 184, 1288] avg=1024.0000 max=3903 max_rank=5 min=151 min_rank=3 imbalance_ratio=3.811523 max_load_reduction=4289 max_load_reduction_pct=0.523560 moved_to_echo=6007 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1419, 0, 7, 537, 276, 663, 60, 1137, 19, 944, 460, 174, 1687, 0, 809, 0] rerouted_tokens_per_ep_rank=[1419, 544, 939, 1197, 963, 634, 1687, 809] avg=1024.0000 max=1687 max_rank=6 min=544 min_rank=1 imbalance_ratio=1.647461 max_load_reduction=6505 max_load_reduction_pct=0.794067 moved_to_echo=4093 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5319, 8192, 7157, 8192, 9227, 11065] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9227 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5319, 8192, 7157, 8192, 9227, 11065] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5319, 8192, 7157, 8192, 9227, 11065] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5319 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5319, 8192, 7157, 8192, 9227, 11065] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5319, 8192, 7157, 8192, 9227, 11065] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11065 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5319, 8192, 7157, 8192, 9227, 11065] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7157 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5319, 8192, 7157, 8192, 9227, 11065] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5319, 8192, 7157, 8192, 9227, 11065] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=5319 actual_valid_recv=5376 observed_loss=-57 tokens_per_expert=[1536, 3840] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7157 actual_valid_recv=7296 observed_loss=-139 tokens_per_expert=[1792, 5504] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7296, 1024] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[896, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9227 actual_valid_recv=9344 observed_loss=-117 tokens_per_expert=[9344, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2048, 6272] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=11065 actual_valid_recv=11136 observed_loss=-71 tokens_per_expert=[11136, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=5376 slack_slots=11008 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=7296 slack_slots=9088 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9344 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=11136 slack_slots=5248 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[183, 509, 2295, 173, 263, 770, 96, 3903] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1835, 1197, 619, 1165, 3145, 8, 27, 196] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[2547, 38, 1713, 158, 1755, 47, 2, 1932] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[2382, 3311, 898, 763, 594, 21, 89, 134] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[3734, 37, 2283, 543, 898, 108, 307, 282] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[62, 372, 1572, 934, 1878, 1777, 1237, 360] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3106, 27, 3148, 684, 421, 613, 10, 183] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[814, 4, 1842, 397, 1167, 2787, 577, 604] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[111, 0, 509, 430, 1426, 0, 173, 439, 214, 0, 770, 34, 96, 72, 3903, 15] rerouted_tokens_per_ep_rank=[111, 939, 1426, 612, 214, 804, 168, 3918] avg=1024.0000 max=3918 max_rank=7 min=111 min_rank=0 imbalance_ratio=3.826172 max_load_reduction=4274 max_load_reduction_pct=0.521729 moved_to_echo=5104 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1104, 0, 1197, 116, 385, 0, 1165, 118, 2547, 0, 8, 413, 27, 731, 196, 185] rerouted_tokens_per_ep_rank=[1104, 1313, 385, 1283, 2547, 421, 758, 381] avg=1024.0000 max=2547 max_rank=4 min=381 min_rank=7 imbalance_ratio=2.487305 max_load_reduction=5645 max_load_reduction_pct=0.689087 moved_to_echo=4107 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[38, 0, 372, 295, 977, 0, 934, 300, 1522, 0, 1777, 246, 1237, 24, 360, 110] rerouted_tokens_per_ep_rank=[38, 667, 977, 1234, 1522, 2023, 1261, 470] avg=1024.0000 max=2023 max_rank=5 min=38 min_rank=0 imbalance_ratio=1.975586 max_load_reduction=6169 max_load_reduction_pct=0.753052 moved_to_echo=5276 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1526, 0, 38, 325, 1056, 0, 158, 332, 1413, 0, 47, 234, 2, 1021, 1932, 108] rerouted_tokens_per_ep_rank=[1526, 363, 1056, 490, 1413, 281, 1023, 2040] avg=1024.0000 max=2040 max_rank=7 min=281 min_rank=5 imbalance_ratio=1.992188 max_load_reduction=6152 max_load_reduction_pct=0.750977 moved_to_echo=4757 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1433, 0, 3311, 168, 559, 0, 763, 171, 481, 0, 21, 78, 89, 949, 134, 35] rerouted_tokens_per_ep_rank=[1433, 3479, 559, 934, 481, 99, 1038, 169] avg=1024.0000 max=3479 max_rank=1 min=99 min_rank=5 imbalance_ratio=3.397461 max_load_reduction=4713 max_load_reduction_pct=0.575317 moved_to_echo=1787 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2246, 0, 37, 428, 1418, 0, 543, 437, 727, 0, 108, 118, 307, 1488, 282, 53] rerouted_tokens_per_ep_rank=[2246, 465, 1418, 980, 727, 226, 1795, 335] avg=1024.0000 max=2246 max_rank=0 min=226 min_rank=5 imbalance_ratio=2.193359 max_load_reduction=5946 max_load_reduction_pct=0.725830 moved_to_echo=3083 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1868, 0, 27, 590, 1956, 0, 684, 602, 342, 0, 613, 55, 10, 1238, 183, 24] rerouted_tokens_per_ep_rank=[1868, 617, 1956, 1286, 342, 668, 1248, 207] avg=1024.0000 max=1956 max_rank=2 min=207 min_rank=7 imbalance_ratio=1.910156 max_load_reduction=6236 max_load_reduction_pct=0.761230 moved_to_echo=2465 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[490, 0, 4, 345, 1145, 0, 397, 352, 946, 0, 2787, 153, 577, 324, 604, 68] rerouted_tokens_per_ep_rank=[490, 349, 1145, 749, 946, 2940, 901, 672] avg=1024.0000 max=2940 max_rank=5 min=349 min_rank=1 imbalance_ratio=2.871094 max_load_reduction=5252 max_load_reduction_pct=0.641113 moved_to_echo=5459 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8922 expected_overflow=0 exact_recv_per_rank=[8816, 8192, 8922, 7568, 8192, 7462, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8816, 8192, 8922, 7568, 8192, 7462, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8816, 8192, 8922, 7568, 8192, 7462, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7568 expected_overflow=0 exact_recv_per_rank=[8816, 8192, 8922, 7568, 8192, 7462, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8816, 8192, 8922, 7568, 8192, 7462, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8816 expected_overflow=0 exact_recv_per_rank=[8816, 8192, 8922, 7568, 8192, 7462, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7462 expected_overflow=0 exact_recv_per_rank=[8816, 8192, 8922, 7568, 8192, 7462, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8816, 8192, 8922, 7568, 8192, 7462, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=7568 actual_valid_recv=7680 observed_loss=-112 tokens_per_expert=[4864, 2816] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7462 actual_valid_recv=7552 observed_loss=-90 tokens_per_expert=[6144, 1408] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5504, 2816] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2432, 5888] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8816 actual_valid_recv=8832 observed_loss=-16 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7680, 640] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8922 actual_valid_recv=8960 observed_loss=-38 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[923, 858, 51, 48, 485, 11, 3554, 2262] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4024, 10, 19, 2718, 127, 15, 162, 1117] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[501, 11, 1575, 3593, 584, 39, 776, 1113] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[29, 387, 388, 3697, 185, 107, 307, 3092] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[847, 37, 77, 2645, 98, 14, 3467, 1007] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1157, 696, 168, 3480, 96, 1559, 834, 202] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[2053, 720, 1458, 2216, 4, 143, 1379, 219] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[334, 101, 215, 2392, 1408, 2, 1934, 1806] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[923, 0, 858, 12, 51, 536, 22, 0, 485, 1208, 11, 14, 2346, 0, 1726, 0] rerouted_tokens_per_ep_rank=[923, 870, 587, 22, 1693, 25, 2346, 1726] avg=1024.0000 max=2346 max_rank=6 min=22 min_rank=3 imbalance_ratio=2.291016 max_load_reduction=5846 max_load_reduction_pct=0.713623 moved_to_echo=5790 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 387, 955, 388, 733, 1622, 0, 185, 104, 107, 1120, 203, 0, 2359, 0] rerouted_tokens_per_ep_rank=[29, 1342, 1121, 1622, 289, 1227, 203, 2359] avg=1024.0000 max=2359 max_rank=7 min=29 min_rank=0 imbalance_ratio=2.303711 max_load_reduction=5833 max_load_reduction_pct=0.712036 moved_to_echo=4078 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[501, 0, 11, 931, 1575, 268, 1568, 0, 584, 268, 39, 1094, 508, 0, 845, 0] rerouted_tokens_per_ep_rank=[501, 942, 1843, 1568, 852, 1133, 508, 845] avg=1024.0000 max=1843 max_rank=2 min=501 min_rank=0 imbalance_ratio=1.799805 max_load_reduction=6349 max_load_reduction_pct=0.775024 moved_to_echo=3338 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1157, 0, 696, 899, 168, 47, 1527, 0, 96, 283, 1559, 1054, 551, 0, 155, 0] rerouted_tokens_per_ep_rank=[1157, 1595, 215, 1527, 379, 2613, 551, 155] avg=1024.0000 max=2613 max_rank=5 min=155 min_rank=7 imbalance_ratio=2.551758 max_load_reduction=5579 max_load_reduction_pct=0.681030 moved_to_echo=3698 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4024, 0, 10, 702, 19, 264, 1193, 0, 127, 55, 15, 823, 107, 0, 853, 0] rerouted_tokens_per_ep_rank=[4024, 712, 283, 1193, 182, 838, 107, 853] avg=1024.0000 max=4024 max_rank=0 min=107 min_rank=6 imbalance_ratio=3.929688 max_load_reduction=4168 max_load_reduction_pct=0.508789 moved_to_echo=1980 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[847, 0, 37, 683, 77, 238, 1161, 0, 98, 1178, 14, 801, 2289, 0, 769, 0] rerouted_tokens_per_ep_rank=[847, 720, 315, 1161, 1276, 815, 2289, 769] avg=1024.0000 max=2289 max_rank=6 min=315 min_rank=2 imbalance_ratio=2.235352 max_load_reduction=5903 max_load_reduction_pct=0.720581 moved_to_echo=5149 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[334, 0, 101, 618, 215, 428, 1049, 0, 1408, 657, 2, 725, 1277, 0, 1378, 0] rerouted_tokens_per_ep_rank=[334, 719, 643, 1049, 2065, 727, 1277, 1378] avg=1024.0000 max=2065 max_rank=4 min=334 min_rank=0 imbalance_ratio=2.016602 max_load_reduction=6127 max_load_reduction_pct=0.747925 moved_to_echo=5447 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2053, 0, 720, 572, 1458, 51, 973, 0, 4, 468, 143, 671, 911, 0, 168, 0] rerouted_tokens_per_ep_rank=[2053, 1292, 1509, 973, 472, 814, 911, 168] avg=1024.0000 max=2053 max_rank=0 min=168 min_rank=7 imbalance_ratio=2.004883 max_load_reduction=6139 max_load_reduction_pct=0.749390 moved_to_echo=2365 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9115 expected_overflow=0 exact_recv_per_rank=[9868, 8192, 6516, 9115, 7208, 8192, 8192, 8253] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6516 expected_overflow=0 exact_recv_per_rank=[9868, 8192, 6516, 9115, 7208, 8192, 8192, 8253] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9868 expected_overflow=0 exact_recv_per_rank=[9868, 8192, 6516, 9115, 7208, 8192, 8192, 8253] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9868, 8192, 6516, 9115, 7208, 8192, 8192, 8253] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9868, 8192, 6516, 9115, 7208, 8192, 8192, 8253] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8253 expected_overflow=0 exact_recv_per_rank=[9868, 8192, 6516, 9115, 7208, 8192, 8192, 8253] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9868, 8192, 6516, 9115, 7208, 8192, 8192, 8253] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7208 expected_overflow=0 exact_recv_per_rank=[9868, 8192, 6516, 9115, 7208, 8192, 8192, 8253] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8253 actual_valid_recv=8320 observed_loss=-67 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=6516 actual_valid_recv=6656 observed_loss=-140 tokens_per_expert=[3968, 2688] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2944, 5376] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1920, 6400] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7208 actual_valid_recv=7296 observed_loss=-88 tokens_per_expert=[3072, 4224] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=9115 actual_valid_recv=9216 observed_loss=-101 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=9868 actual_valid_recv=9984 observed_loss=-116 tokens_per_expert=[9984, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=6656 slack_slots=9728 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=7296 slack_slots=9088 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=9984 slack_slots=6400 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[655, 3837, 1, 221, 154, 52, 3265, 7] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3630, 2138, 86, 723, 3, 0, 48, 1564] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[126, 2611, 17, 31, 193, 1197, 2759, 1258] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[42, 330, 1373, 218, 1590, 1424, 1890, 1325] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[68, 2627, 9, 1173, 10, 255, 2835, 1215] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[272, 1851, 2, 1755, 235, 468, 1325, 2284] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[2701, 1323, 82, 1034, 12, 88, 2897, 55] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1600, 2911, 34, 1047, 450, 65, 1956, 129] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3489, 0, 994, 0, 86, 799, 723, 4, 3, 345, 0, 13, 31, 0, 1564, 141] rerouted_tokens_per_ep_rank=[3489, 994, 885, 727, 348, 13, 31, 1705] avg=1024.0000 max=3489 max_rank=0 min=13 min_rank=5 imbalance_ratio=3.407227 max_load_reduction=4703 max_load_reduction_pct=0.574097 moved_to_echo=2097 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[625, 0, 1776, 0, 1, 1438, 221, 281, 154, 623, 52, 896, 2088, 0, 7, 30] rerouted_tokens_per_ep_rank=[625, 1776, 1439, 502, 777, 948, 2088, 37] avg=1024.0000 max=2088 max_rank=6 min=37 min_rank=7 imbalance_ratio=2.039062 max_load_reduction=6104 max_load_reduction_pct=0.745117 moved_to_echo=3850 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[66, 0, 1222, 0, 9, 981, 1173, 240, 10, 424, 255, 775, 1820, 0, 1215, 2] rerouted_tokens_per_ep_rank=[66, 1222, 990, 1413, 434, 1030, 1820, 1217] avg=1024.0000 max=1820 max_rank=6 min=66 min_rank=0 imbalance_ratio=1.777344 max_load_reduction=6372 max_load_reduction_pct=0.777832 moved_to_echo=4501 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[122, 0, 1215, 0, 17, 975, 31, 234, 193, 421, 1197, 754, 1771, 0, 1258, 4] rerouted_tokens_per_ep_rank=[122, 1215, 992, 265, 614, 1951, 1771, 1262] avg=1024.0000 max=1951 max_rank=5 min=122 min_rank=0 imbalance_ratio=1.905273 max_load_reduction=6241 max_load_reduction_pct=0.761841 moved_to_echo=5598 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2596, 0, 616, 0, 82, 494, 1034, 246, 12, 213, 88, 792, 1859, 0, 55, 105] rerouted_tokens_per_ep_rank=[2596, 616, 576, 1280, 225, 880, 1859, 160] avg=1024.0000 max=2596 max_rank=0 min=160 min_rank=7 imbalance_ratio=2.535156 max_load_reduction=5596 max_load_reduction_pct=0.683105 moved_to_echo=3124 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[41, 0, 154, 0, 1373, 123, 218, 160, 1590, 53, 1424, 516, 1214, 0, 1325, 1] rerouted_tokens_per_ep_rank=[41, 154, 1496, 378, 1643, 1940, 1214, 1326] avg=1024.0000 max=1940 max_rank=5 min=41 min_rank=0 imbalance_ratio=1.894531 max_load_reduction=6252 max_load_reduction_pct=0.763184 moved_to_echo=6123 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[262, 0, 861, 0, 2, 691, 1755, 112, 235, 299, 468, 362, 851, 0, 2284, 10] rerouted_tokens_per_ep_rank=[262, 861, 693, 1867, 534, 830, 851, 2294] avg=1024.0000 max=2294 max_rank=7 min=262 min_rank=0 imbalance_ratio=2.240234 max_load_reduction=5898 max_load_reduction_pct=0.719971 moved_to_echo=4509 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1538, 0, 1354, 0, 34, 1087, 1047, 166, 450, 470, 65, 535, 1255, 0, 129, 62] rerouted_tokens_per_ep_rank=[1538, 1354, 1121, 1213, 920, 600, 1255, 191] avg=1024.0000 max=1538 max_rank=0 min=191 min_rank=7 imbalance_ratio=1.501953 max_load_reduction=6654 max_load_reduction_pct=0.812256 moved_to_echo=2966 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8739 expected_overflow=0 exact_recv_per_rank=[8739, 8192, 8192, 7645, 5495, 8192, 10889, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8739, 8192, 8192, 7645, 5495, 8192, 10889, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7645 expected_overflow=0 exact_recv_per_rank=[8739, 8192, 8192, 7645, 5495, 8192, 10889, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5495 expected_overflow=0 exact_recv_per_rank=[8739, 8192, 8192, 7645, 5495, 8192, 10889, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8739, 8192, 8192, 7645, 5495, 8192, 10889, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10889 expected_overflow=0 exact_recv_per_rank=[8739, 8192, 8192, 7645, 5495, 8192, 10889, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8739, 8192, 8192, 7645, 5495, 8192, 10889, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8739, 8192, 8192, 7645, 5495, 8192, 10889, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=5495 actual_valid_recv=5632 observed_loss=-137 tokens_per_expert=[2688, 2944] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=7645 actual_valid_recv=7808 observed_loss=-163 tokens_per_expert=[6272, 1536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8739 actual_valid_recv=8832 observed_loss=-93 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7936, 384] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1664, 6656] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3584, 4736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=10889 actual_valid_recv=11008 observed_loss=-119 tokens_per_expert=[11008, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=5632 slack_slots=10752 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=11008 slack_slots=5376 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1358, 1801, 661, 15, 74, 911, 5, 3367] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[3912, 310, 226, 643, 4, 718, 181, 2198] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[3901, 2160, 313, 85, 3, 85, 534, 1111] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1980, 424, 539, 508, 28, 224, 440, 4049] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[3141, 3675, 277, 731, 222, 97, 0, 49] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[2092, 244, 17, 2854, 0, 55, 63, 2867] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[3453, 2553, 1399, 331, 17, 360, 46, 33] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[2419, 1481, 1935, 1712, 8, 156, 56, 425] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1441, 0, 209, 0, 226, 69, 643, 32, 4, 1377, 718, 821, 181, 1094, 1377, 0] rerouted_tokens_per_ep_rank=[1441, 209, 295, 675, 1381, 1539, 1275, 1377] avg=1024.0000 max=1539 max_rank=5 min=209 min_rank=1 imbalance_ratio=1.502930 max_load_reduction=6653 max_load_reduction_pct=0.812134 moved_to_echo=5572 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1158, 0, 2474, 0, 277, 820, 731, 381, 222, 1105, 97, 18, 0, 878, 31, 0] rerouted_tokens_per_ep_rank=[1158, 2474, 1097, 1112, 1327, 115, 878, 31] avg=1024.0000 max=2474 max_rank=1 min=31 min_rank=7 imbalance_ratio=2.416016 max_load_reduction=5718 max_load_reduction_pct=0.697998 moved_to_echo=2351 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[492, 0, 1206, 0, 661, 406, 15, 189, 74, 482, 911, 1261, 5, 384, 2106, 0] rerouted_tokens_per_ep_rank=[492, 1206, 1067, 204, 556, 2172, 389, 2106] avg=1024.0000 max=2172 max_rank=5 min=204 min_rank=3 imbalance_ratio=2.121094 max_load_reduction=6020 max_load_reduction_pct=0.734863 moved_to_echo=5223 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1437, 0, 1454, 0, 313, 482, 85, 224, 3, 1373, 85, 415, 534, 1091, 696, 0] rerouted_tokens_per_ep_rank=[1437, 1454, 795, 309, 1376, 500, 1625, 696] avg=1024.0000 max=1625 max_rank=6 min=309 min_rank=3 imbalance_ratio=1.586914 max_load_reduction=6567 max_load_reduction_pct=0.801636 moved_to_echo=4197 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[729, 0, 286, 0, 539, 94, 508, 44, 28, 697, 224, 1512, 440, 554, 2537, 0] rerouted_tokens_per_ep_rank=[729, 286, 633, 552, 725, 1736, 994, 2537] avg=1024.0000 max=2537 max_rank=7 min=286 min_rank=1 imbalance_ratio=2.477539 max_load_reduction=5655 max_load_reduction_pct=0.690308 moved_to_echo=5992 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[771, 0, 165, 0, 17, 54, 2854, 25, 0, 736, 55, 1071, 63, 585, 1796, 0] rerouted_tokens_per_ep_rank=[771, 165, 71, 2879, 736, 1126, 648, 1796] avg=1024.0000 max=2879 max_rank=3 min=71 min_rank=2 imbalance_ratio=2.811523 max_load_reduction=5313 max_load_reduction_pct=0.648560 moved_to_echo=4306 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1272, 0, 1718, 0, 1399, 570, 331, 265, 17, 1215, 360, 12, 46, 966, 21, 0] rerouted_tokens_per_ep_rank=[1272, 1718, 1969, 596, 1232, 372, 1012, 21] avg=1024.0000 max=1969 max_rank=2 min=21 min_rank=7 imbalance_ratio=1.922852 max_load_reduction=6223 max_load_reduction_pct=0.759644 moved_to_echo=2637 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[892, 0, 998, 0, 1935, 330, 1712, 153, 8, 851, 156, 158, 56, 676, 267, 0] rerouted_tokens_per_ep_rank=[892, 998, 2265, 1865, 859, 314, 732, 267] avg=1024.0000 max=2265 max_rank=2 min=267 min_rank=7 imbalance_ratio=2.211914 max_load_reduction=5927 max_load_reduction_pct=0.723511 moved_to_echo=2172 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7874 expected_overflow=0 exact_recv_per_rank=[8192, 8510, 8192, 8192, 8192, 7874, 7553, 8831] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8510, 8192, 8192, 8192, 7874, 7553, 8831] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8831 expected_overflow=0 exact_recv_per_rank=[8192, 8510, 8192, 8192, 8192, 7874, 7553, 8831] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8510, 8192, 8192, 8192, 7874, 7553, 8831] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8510, 8192, 8192, 8192, 7874, 7553, 8831] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8510, 8192, 8192, 8192, 7874, 7553, 8831] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8510 expected_overflow=0 exact_recv_per_rank=[8192, 8510, 8192, 8192, 8192, 7874, 7553, 8831] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7553 expected_overflow=0 exact_recv_per_rank=[8192, 8510, 8192, 8192, 8192, 7874, 7553, 8831] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7553 actual_valid_recv=7680 observed_loss=-127 tokens_per_expert=[1408, 6272] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7874 actual_valid_recv=8064 observed_loss=-190 tokens_per_expert=[2688, 5376] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6912, 1408] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8510 actual_valid_recv=8576 observed_loss=-66 tokens_per_expert=[8576, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8831 actual_valid_recv=8832 observed_loss=-1 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5376, 2944] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8064 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8576 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3433, 424, 117, 2735, 626, 420, 428, 9] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[48, 278, 35, 3788, 1607, 109, 2246, 81] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[2868, 4, 31, 2419, 310, 2119, 441, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[100, 390, 2199, 2502, 190, 1382, 1427, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[2948, 92, 30, 3890, 5, 243, 182, 802] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[3469, 866, 757, 402, 194, 1443, 858, 203] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[9, 33, 135, 1666, 575, 2232, 3507, 35] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[2054, 356, 1495, 145, 189, 3563, 134, 256] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1885, 0, 424, 735, 117, 86, 1675, 0, 626, 813, 334, 0, 428, 0, 9, 1060] rerouted_tokens_per_ep_rank=[1885, 1159, 203, 1675, 1439, 334, 428, 1069] avg=1024.0000 max=1885 max_rank=0 min=203 min_rank=2 imbalance_ratio=1.840820 max_load_reduction=6307 max_load_reduction_pct=0.769897 moved_to_echo=3270 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[27, 0, 278, 10, 35, 22, 2320, 0, 1607, 11, 87, 0, 2246, 0, 81, 1468] rerouted_tokens_per_ep_rank=[27, 288, 57, 2320, 1618, 87, 2246, 1549] avg=1024.0000 max=2320 max_rank=3 min=27 min_rank=0 imbalance_ratio=2.265625 max_load_reduction=5872 max_load_reduction_pct=0.716797 moved_to_echo=5500 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1575, 0, 4, 614, 31, 434, 1482, 0, 310, 679, 1685, 0, 441, 0, 0, 937] rerouted_tokens_per_ep_rank=[1575, 618, 465, 1482, 989, 1685, 441, 937] avg=1024.0000 max=1685 max_rank=5 min=441 min_rank=6 imbalance_ratio=1.645508 max_load_reduction=6507 max_load_reduction_pct=0.794312 moved_to_echo=4052 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1619, 0, 92, 631, 30, 49, 2382, 0, 5, 698, 194, 0, 182, 0, 802, 1508] rerouted_tokens_per_ep_rank=[1619, 723, 79, 2382, 703, 194, 182, 2310] avg=1024.0000 max=2382 max_rank=3 min=79 min_rank=2 imbalance_ratio=2.326172 max_load_reduction=5810 max_load_reduction_pct=0.709229 moved_to_echo=3389 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[56, 0, 390, 21, 2199, 283, 1532, 0, 190, 23, 1099, 0, 1427, 0, 2, 970] rerouted_tokens_per_ep_rank=[56, 411, 2482, 1532, 213, 1099, 1427, 972] avg=1024.0000 max=2482 max_rank=2 min=56 min_rank=0 imbalance_ratio=2.423828 max_load_reduction=5710 max_load_reduction_pct=0.697021 moved_to_echo=3711 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1895, 0, 866, 747, 757, 300, 243, 0, 194, 827, 1143, 0, 858, 0, 203, 159] rerouted_tokens_per_ep_rank=[1895, 1613, 1057, 243, 1021, 1143, 858, 362] avg=1024.0000 max=1895 max_rank=0 min=243 min_rank=3 imbalance_ratio=1.850586 max_load_reduction=6297 max_load_reduction_pct=0.768677 moved_to_echo=3384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 33, 1, 135, 457, 1020, 0, 575, 2, 1775, 0, 3507, 0, 35, 646] rerouted_tokens_per_ep_rank=[6, 34, 592, 1020, 577, 1775, 3507, 681] avg=1024.0000 max=3507 max_rank=6 min=6 min_rank=0 imbalance_ratio=3.424805 max_load_reduction=4685 max_load_reduction_pct=0.571899 moved_to_echo=6540 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1129, 0, 356, 439, 1495, 731, 89, 0, 189, 486, 2832, 0, 134, 0, 256, 56] rerouted_tokens_per_ep_rank=[1129, 795, 2226, 89, 675, 2832, 134, 312] avg=1024.0000 max=2832 max_rank=5 min=89 min_rank=3 imbalance_ratio=2.765625 max_load_reduction=5360 max_load_reduction_pct=0.654297 moved_to_echo=3953 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10743 expected_overflow=0 exact_recv_per_rank=[8192, 5641, 7161, 10743, 7235, 9149, 9223, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9149 expected_overflow=0 exact_recv_per_rank=[8192, 5641, 7161, 10743, 7235, 9149, 9223, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9223 expected_overflow=0 exact_recv_per_rank=[8192, 5641, 7161, 10743, 7235, 9149, 9223, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7235 expected_overflow=0 exact_recv_per_rank=[8192, 5641, 7161, 10743, 7235, 9149, 9223, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5641, 7161, 10743, 7235, 9149, 9223, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5641, 7161, 10743, 7235, 9149, 9223, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5641 expected_overflow=0 exact_recv_per_rank=[8192, 5641, 7161, 10743, 7235, 9149, 9223, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7161 expected_overflow=0 exact_recv_per_rank=[8192, 5641, 7161, 10743, 7235, 9149, 9223, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=5641 actual_valid_recv=5760 observed_loss=-119 tokens_per_expert=[2560, 3200] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=7161 actual_valid_recv=7296 observed_loss=-135 tokens_per_expert=[4864, 2432] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=9149 actual_valid_recv=9216 observed_loss=-67 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7235 actual_valid_recv=7296 observed_loss=-61 tokens_per_expert=[3712, 3584] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1408, 6912] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9223 actual_valid_recv=9344 observed_loss=-121 tokens_per_expert=[9344, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=10743 actual_valid_recv=10752 observed_loss=-9 tokens_per_expert=[10752, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=7296 slack_slots=9088 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=5760 slack_slots=10624 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=7296 slack_slots=9088 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9344 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=10752 slack_slots=5632 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1566, 1621, 1, 3831, 484, 374, 15, 300] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[250, 1085, 116, 3333, 33, 247, 95, 3033] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[405, 475, 1509, 1242, 2916, 10, 0, 1635] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[304, 934, 423, 3719, 0, 6, 0, 2806] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1857, 287, 217, 3063, 49, 85, 1083, 1551] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[2249, 918, 2415, 2467, 20, 108, 5, 10] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1408, 2680, 146, 1047, 1644, 619, 335, 313] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1321, 4070, 15, 148, 37, 1327, 447, 827] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1566, 0, 1228, 0, 1, 393, 1657, 0, 484, 55, 374, 908, 15, 1266, 245, 0] rerouted_tokens_per_ep_rank=[1566, 1228, 394, 1657, 539, 1282, 1281, 245] avg=1024.0000 max=1657 max_rank=3 min=245 min_rank=7 imbalance_ratio=1.618164 max_load_reduction=6535 max_load_reduction_pct=0.797729 moved_to_echo=3347 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[250, 0, 824, 0, 116, 261, 1449, 0, 33, 533, 247, 786, 95, 1098, 2500, 0] rerouted_tokens_per_ep_rank=[250, 824, 377, 1449, 566, 1033, 1193, 2500] avg=1024.0000 max=2500 max_rank=7 min=250 min_rank=0 imbalance_ratio=2.441406 max_load_reduction=5692 max_load_reduction_pct=0.694824 moved_to_echo=5292 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[405, 0, 361, 0, 1509, 114, 541, 0, 2916, 287, 10, 292, 0, 409, 1348, 0] rerouted_tokens_per_ep_rank=[405, 361, 1623, 541, 3203, 302, 409, 1348] avg=1024.0000 max=3203 max_rank=4 min=302 min_rank=5 imbalance_ratio=3.127930 max_load_reduction=4989 max_load_reduction_pct=0.609009 moved_to_echo=5262 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1321, 0, 3090, 0, 15, 980, 66, 0, 37, 145, 1327, 34, 447, 48, 682, 0] rerouted_tokens_per_ep_rank=[1321, 3090, 995, 66, 182, 1361, 495, 682] avg=1024.0000 max=3090 max_rank=1 min=66 min_rank=3 imbalance_ratio=3.017578 max_load_reduction=5102 max_load_reduction_pct=0.622803 moved_to_echo=2720 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2249, 0, 697, 0, 2415, 221, 1074, 0, 20, 1, 108, 581, 5, 812, 9, 0] rerouted_tokens_per_ep_rank=[2249, 697, 2636, 1074, 21, 689, 817, 9] avg=1024.0000 max=2636 max_rank=2 min=9 min_rank=7 imbalance_ratio=2.574219 max_load_reduction=5556 max_load_reduction_pct=0.678223 moved_to_echo=1536 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1408, 0, 2035, 0, 146, 645, 456, 0, 1644, 55, 619, 246, 335, 345, 258, 0] rerouted_tokens_per_ep_rank=[1408, 2035, 791, 456, 1699, 865, 680, 258] avg=1024.0000 max=2035 max_rank=1 min=258 min_rank=7 imbalance_ratio=1.987305 max_load_reduction=6157 max_load_reduction_pct=0.751587 moved_to_echo=3502 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[304, 0, 709, 0, 423, 225, 1617, 0, 0, 493, 6, 877, 0, 1225, 2313, 0] rerouted_tokens_per_ep_rank=[304, 709, 648, 1617, 493, 883, 1225, 2313] avg=1024.0000 max=2313 max_rank=7 min=304 min_rank=0 imbalance_ratio=2.258789 max_load_reduction=5879 max_load_reduction_pct=0.717651 moved_to_echo=4914 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1857, 0, 218, 0, 217, 69, 1332, 0, 49, 272, 85, 722, 1083, 1009, 1279, 0] rerouted_tokens_per_ep_rank=[1857, 218, 286, 1332, 321, 807, 2092, 1279] avg=1024.0000 max=2092 max_rank=6 min=218 min_rank=1 imbalance_ratio=2.042969 max_load_reduction=6100 max_load_reduction_pct=0.744629 moved_to_echo=4499 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9360, 9162, 7750, 8192, 7024, 7222, 8192, 8634] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9360 expected_overflow=0 exact_recv_per_rank=[9360, 9162, 7750, 8192, 7024, 7222, 8192, 8634] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7024 expected_overflow=0 exact_recv_per_rank=[9360, 9162, 7750, 8192, 7024, 7222, 8192, 8634] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7222 expected_overflow=0 exact_recv_per_rank=[9360, 9162, 7750, 8192, 7024, 7222, 8192, 8634] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9162 expected_overflow=0 exact_recv_per_rank=[9360, 9162, 7750, 8192, 7024, 7222, 8192, 8634] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8634 expected_overflow=0 exact_recv_per_rank=[9360, 9162, 7750, 8192, 7024, 7222, 8192, 8634] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9360, 9162, 7750, 8192, 7024, 7222, 8192, 8634] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7750 expected_overflow=0 exact_recv_per_rank=[9360, 9162, 7750, 8192, 7024, 7222, 8192, 8634] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7222 actual_valid_recv=7296 observed_loss=-74 tokens_per_expert=[2816, 4480] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7024 actual_valid_recv=7168 observed_loss=-144 tokens_per_expert=[5248, 1920] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=7750 actual_valid_recv=7808 observed_loss=-58 tokens_per_expert=[4864, 2944] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2048, 6272] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=9360 actual_valid_recv=9472 observed_loss=-112 tokens_per_expert=[9472, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=9162 actual_valid_recv=9216 observed_loss=-54 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8634 actual_valid_recv=8704 observed_loss=-70 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7296 slack_slots=9088 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=7168 slack_slots=9216 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=9472 slack_slots=6912 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[149, 657, 609, 3020, 590, 16, 2, 3149] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[220, 2, 1630, 3854, 687, 79, 1703, 17] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[362, 60, 214, 3317, 1361, 173, 2434, 271] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[137, 554, 653, 2692, 376, 23, 212, 3545] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[512, 31, 1, 1097, 3941, 10, 95, 2505] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[87, 548, 3719, 219, 857, 1961, 326, 475] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[941, 0, 1485, 1187, 2731, 153, 1, 1694] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[62, 737, 3356, 1062, 2047, 544, 360, 24] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[149, 1050, 657, 143, 450, 0, 1970, 0, 447, 0, 16, 940, 2, 159, 2209, 0] rerouted_tokens_per_ep_rank=[1199, 800, 450, 1970, 447, 956, 161, 2209] avg=1024.0000 max=2209 max_rank=7 min=161 min_rank=6 imbalance_ratio=2.157227 max_load_reduction=5983 max_load_reduction_pct=0.730347 moved_to_echo=3773 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[362, 1153, 60, 331, 158, 0, 2164, 0, 1030, 0, 173, 80, 2434, 56, 191, 0] rerouted_tokens_per_ep_rank=[1515, 391, 158, 2164, 1030, 253, 2490, 191] avg=1024.0000 max=2490 max_rank=6 min=158 min_rank=2 imbalance_ratio=2.431641 max_load_reduction=5702 max_load_reduction_pct=0.696045 moved_to_echo=3964 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[137, 936, 554, 91, 482, 0, 1756, 0, 285, 0, 23, 1058, 212, 171, 2487, 0] rerouted_tokens_per_ep_rank=[1073, 645, 482, 1756, 285, 1081, 383, 2487] avg=1024.0000 max=2487 max_rank=7 min=285 min_rank=4 imbalance_ratio=2.428711 max_load_reduction=5705 max_load_reduction_pct=0.696411 moved_to_echo=4236 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[87, 81, 548, 214, 2741, 0, 138, 0, 643, 0, 1961, 145, 326, 978, 330, 0] rerouted_tokens_per_ep_rank=[168, 762, 2741, 138, 643, 2106, 1304, 330] avg=1024.0000 max=2741 max_rank=2 min=138 min_rank=3 imbalance_ratio=2.676758 max_load_reduction=5451 max_load_reduction_pct=0.665405 moved_to_echo=4383 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[512, 381, 31, 960, 1, 0, 716, 0, 2981, 0, 10, 748, 95, 0, 1757, 0] rerouted_tokens_per_ep_rank=[893, 991, 1, 716, 2981, 758, 95, 1757] avg=1024.0000 max=2981 max_rank=4 min=1 min_rank=2 imbalance_ratio=2.911133 max_load_reduction=5211 max_load_reduction_pct=0.636108 moved_to_echo=5591 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[62, 369, 737, 498, 2477, 0, 693, 0, 1549, 0, 544, 7, 360, 879, 17, 0] rerouted_tokens_per_ep_rank=[431, 1235, 2477, 693, 1549, 551, 1239, 17] avg=1024.0000 max=2477 max_rank=2 min=17 min_rank=7 imbalance_ratio=2.418945 max_load_reduction=5715 max_load_reduction_pct=0.697632 moved_to_echo=3356 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[220, 1340, 2, 167, 1203, 0, 2514, 0, 520, 0, 79, 5, 1703, 427, 12, 0] rerouted_tokens_per_ep_rank=[1560, 169, 1203, 2514, 520, 84, 2130, 12] avg=1024.0000 max=2514 max_rank=3 min=12 min_rank=7 imbalance_ratio=2.455078 max_load_reduction=5678 max_load_reduction_pct=0.693115 moved_to_echo=2746 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[941, 412, 0, 665, 1096, 0, 775, 0, 2066, 0, 153, 505, 1, 389, 1189, 0] rerouted_tokens_per_ep_rank=[1353, 665, 1096, 775, 2066, 658, 390, 1189] avg=1024.0000 max=2066 max_rank=4 min=390 min_rank=6 imbalance_ratio=2.017578 max_load_reduction=6126 max_load_reduction_pct=0.747803 moved_to_echo=4303 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5658 expected_overflow=0 exact_recv_per_rank=[8192, 5658, 8608, 10726, 9521, 6447, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8608 expected_overflow=0 exact_recv_per_rank=[8192, 5658, 8608, 10726, 9521, 6447, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5658, 8608, 10726, 9521, 6447, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5658, 8608, 10726, 9521, 6447, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6447 expected_overflow=0 exact_recv_per_rank=[8192, 5658, 8608, 10726, 9521, 6447, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10726 expected_overflow=0 exact_recv_per_rank=[8192, 5658, 8608, 10726, 9521, 6447, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9521 expected_overflow=0 exact_recv_per_rank=[8192, 5658, 8608, 10726, 9521, 6447, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5658, 8608, 10726, 9521, 6447, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=5658 actual_valid_recv=5760 observed_loss=-102 tokens_per_expert=[2688, 3072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=6447 actual_valid_recv=6656 observed_loss=-209 tokens_per_expert=[3072, 3584] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5248, 3072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2560, 5760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8608 actual_valid_recv=8704 observed_loss=-96 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=9521 actual_valid_recv=9600 observed_loss=-79 tokens_per_expert=[9600, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=10726 actual_valid_recv=10752 observed_loss=-26 tokens_per_expert=[10752, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=5760 slack_slots=10624 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=6656 slack_slots=9728 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=9600 slack_slots=6784 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=10752 slack_slots=5632 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[68, 1090, 1419, 845, 56, 3276, 899, 539] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[6, 622, 106, 2652, 324, 3403, 496, 583] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[8, 166, 143, 257, 2126, 2698, 5, 2789] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[465, 3442, 1857, 1739, 101, 18, 88, 482] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[658, 1, 792, 2919, 134, 241, 1045, 2402] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[50, 237, 78, 2618, 509, 1708, 1321, 1671] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[2, 508, 2693, 827, 3, 2459, 1347, 353] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[3, 1175, 246, 19, 125, 3445, 281, 2898] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[68, 1316, 1090, 43, 1419, 39, 654, 0, 56, 191, 1960, 0, 899, 78, 379, 0] rerouted_tokens_per_ep_rank=[1384, 1133, 1458, 654, 247, 1960, 977, 379] avg=1024.0000 max=1960 max_rank=5 min=247 min_rank=4 imbalance_ratio=1.914062 max_load_reduction=6232 max_load_reduction_pct=0.760742 moved_to_echo=3563 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[50, 686, 237, 135, 78, 122, 2026, 0, 509, 592, 1022, 0, 1321, 244, 1170, 0] rerouted_tokens_per_ep_rank=[736, 372, 200, 2026, 1101, 1022, 1565, 1170] avg=1024.0000 max=2026 max_rank=3 min=200 min_rank=2 imbalance_ratio=1.978516 max_load_reduction=6166 max_load_reduction_pct=0.752686 moved_to_echo=4858 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[465, 7, 3442, 39, 1857, 35, 1346, 0, 101, 393, 11, 0, 88, 70, 338, 0] rerouted_tokens_per_ep_rank=[472, 3481, 1892, 1346, 494, 11, 158, 338] avg=1024.0000 max=3481 max_rank=1 min=11 min_rank=5 imbalance_ratio=3.399414 max_load_reduction=4711 max_load_reduction_pct=0.575073 moved_to_echo=1001 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 1088, 166, 230, 143, 208, 195, 0, 2126, 62, 1610, 0, 5, 413, 1938, 0] rerouted_tokens_per_ep_rank=[1096, 396, 351, 195, 2188, 1610, 418, 1938] avg=1024.0000 max=2188 max_rank=4 min=195 min_rank=3 imbalance_ratio=2.136719 max_load_reduction=6004 max_load_reduction_pct=0.732910 moved_to_echo=6154 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[658, 96, 1, 194, 792, 175, 2258, 0, 134, 661, 145, 0, 1045, 351, 1682, 0] rerouted_tokens_per_ep_rank=[754, 195, 967, 2258, 795, 145, 1396, 1682] avg=1024.0000 max=2258 max_rank=3 min=145 min_rank=5 imbalance_ratio=2.205078 max_load_reduction=5934 max_load_reduction_pct=0.724365 moved_to_echo=4018 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[6, 1367, 622, 47, 106, 42, 2052, 0, 324, 600, 2036, 0, 496, 85, 409, 0] rerouted_tokens_per_ep_rank=[1373, 669, 148, 2052, 924, 2036, 581, 409] avg=1024.0000 max=2052 max_rank=3 min=148 min_rank=2 imbalance_ratio=2.003906 max_load_reduction=6140 max_load_reduction_pct=0.749512 moved_to_echo=3950 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 988, 508, 28, 2693, 25, 640, 0, 3, 187, 1471, 0, 1347, 51, 249, 0] rerouted_tokens_per_ep_rank=[990, 536, 2718, 640, 190, 1471, 1398, 249] avg=1024.0000 max=2718 max_rank=2 min=190 min_rank=4 imbalance_ratio=2.654297 max_load_reduction=5474 max_load_reduction_pct=0.668213 moved_to_echo=3308 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3, 1384, 1175, 235, 246, 212, 15, 0, 125, 4, 2061, 0, 281, 424, 2027, 0] rerouted_tokens_per_ep_rank=[1387, 1410, 458, 15, 129, 2061, 705, 2027] avg=1024.0000 max=2061 max_rank=5 min=15 min_rank=3 imbalance_ratio=2.012695 max_load_reduction=6131 max_load_reduction_pct=0.748413 moved_to_echo=4922 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9186 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 9186, 6068, 10316, 7198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 9186, 6068, 10316, 7198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 9186, 6068, 10316, 7198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 9186, 6068, 10316, 7198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7198 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 9186, 6068, 10316, 7198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10316 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 9186, 6068, 10316, 7198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6068 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 9186, 6068, 10316, 7198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 9186, 6068, 10316, 7198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=6068 actual_valid_recv=6272 observed_loss=-204 tokens_per_expert=[3456, 2816] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7198 actual_valid_recv=7296 observed_loss=-98 tokens_per_expert=[5504, 1792] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=9186 actual_valid_recv=9216 observed_loss=-30 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1280, 7040] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7424, 896] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7296, 1024] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=10316 actual_valid_recv=10368 observed_loss=-52 tokens_per_expert=[10368, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=6272 slack_slots=10112 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=7296 slack_slots=9088 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=10368 slack_slots=6016 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[2438, 2796, 812, 0, 552, 679, 2, 913] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[26, 1595, 1465, 45, 1884, 6, 252, 2919] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[2839, 61, 20, 5, 624, 3468, 1064, 111] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[7, 4035, 1186, 7, 464, 2059, 10, 424] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[72, 134, 65, 5, 9, 3537, 912, 3458] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[151, 3853, 550, 411, 586, 2315, 4, 322] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 1743, 2127, 431, 15, 3747, 6, 123] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[219, 1403, 69, 0, 1326, 1863, 772, 2540] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2438, 307, 1468, 0, 812, 160, 0, 279, 552, 489, 400, 0, 2, 532, 753, 0] rerouted_tokens_per_ep_rank=[2745, 1468, 972, 279, 1041, 400, 534, 753] avg=1024.0000 max=2745 max_rank=0 min=279 min_rank=3 imbalance_ratio=2.680664 max_load_reduction=5447 max_load_reduction_pct=0.664917 moved_to_echo=2728 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[219, 154, 737, 0, 69, 445, 0, 768, 1326, 245, 1095, 0, 772, 267, 2095, 0] rerouted_tokens_per_ep_rank=[373, 737, 514, 768, 1571, 1095, 1039, 2095] avg=1024.0000 max=2095 max_rank=7 min=373 min_rank=0 imbalance_ratio=2.045898 max_load_reduction=6097 max_load_reduction_pct=0.744263 moved_to_echo=5800 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[7, 444, 2118, 0, 1186, 74, 7, 849, 464, 705, 1210, 0, 10, 768, 350, 0] rerouted_tokens_per_ep_rank=[451, 2118, 1260, 856, 1169, 1210, 778, 350] avg=1024.0000 max=2118 max_rank=1 min=350 min_rank=7 imbalance_ratio=2.068359 max_load_reduction=6074 max_load_reduction_pct=0.741455 moved_to_echo=3507 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 191, 916, 0, 2127, 21, 431, 1545, 15, 304, 2202, 0, 6, 332, 102, 0] rerouted_tokens_per_ep_rank=[191, 916, 2148, 1976, 319, 2202, 338, 102] avg=1024.0000 max=2202 max_rank=5 min=102 min_rank=7 imbalance_ratio=2.150391 max_load_reduction=5990 max_load_reduction_pct=0.731201 moved_to_echo=2961 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2839, 6, 34, 0, 20, 19, 5, 1430, 624, 10, 2038, 0, 1064, 11, 92, 0] rerouted_tokens_per_ep_rank=[2845, 34, 39, 1435, 634, 2038, 1075, 92] avg=1024.0000 max=2845 max_rank=0 min=34 min_rank=1 imbalance_ratio=2.778320 max_load_reduction=5347 max_load_reduction_pct=0.652710 moved_to_echo=3839 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[151, 424, 2022, 0, 550, 56, 411, 954, 586, 673, 1361, 0, 4, 734, 266, 0] rerouted_tokens_per_ep_rank=[575, 2022, 606, 1365, 1259, 1361, 738, 266] avg=1024.0000 max=2022 max_rank=1 min=266 min_rank=7 imbalance_ratio=1.974609 max_load_reduction=6170 max_load_reduction_pct=0.753174 moved_to_echo=3624 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[26, 175, 839, 0, 1465, 512, 45, 2, 1884, 278, 4, 0, 252, 303, 2407, 0] rerouted_tokens_per_ep_rank=[201, 839, 1977, 47, 2162, 4, 555, 2407] avg=1024.0000 max=2407 max_rank=7 min=4 min_rank=5 imbalance_ratio=2.350586 max_load_reduction=5785 max_load_reduction_pct=0.706177 moved_to_echo=5128 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[72, 19, 58, 0, 65, 611, 5, 1461, 9, 28, 2076, 0, 912, 29, 2847, 0] rerouted_tokens_per_ep_rank=[91, 58, 676, 1466, 37, 2076, 941, 2847] avg=1024.0000 max=2847 max_rank=7 min=37 min_rank=4 imbalance_ratio=2.780273 max_load_reduction=5345 max_load_reduction_pct=0.652466 moved_to_echo=5901 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7472, 8192, 8192, 8192, 8192, 10386, 5998, 8912] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7472, 8192, 8192, 8192, 8192, 10386, 5998, 8912] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8912 expected_overflow=0 exact_recv_per_rank=[7472, 8192, 8192, 8192, 8192, 10386, 5998, 8912] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10386 expected_overflow=0 exact_recv_per_rank=[7472, 8192, 8192, 8192, 8192, 10386, 5998, 8912] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5998 expected_overflow=0 exact_recv_per_rank=[7472, 8192, 8192, 8192, 8192, 10386, 5998, 8912] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7472, 8192, 8192, 8192, 8192, 10386, 5998, 8912] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7472 expected_overflow=0 exact_recv_per_rank=[7472, 8192, 8192, 8192, 8192, 10386, 5998, 8912] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7472, 8192, 8192, 8192, 8192, 10386, 5998, 8912] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=5998 actual_valid_recv=6144 observed_loss=-146 tokens_per_expert=[3072, 3072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=7472 actual_valid_recv=7552 observed_loss=-80 tokens_per_expert=[5760, 1792] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1024, 7296] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5504, 2816] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6400, 1920] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=10386 actual_valid_recv=10496 observed_loss=-110 tokens_per_expert=[10496, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8912 actual_valid_recv=8960 observed_loss=-48 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=6144 slack_slots=10240 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=10496 slack_slots=5888 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True + [2026-08-30 04:54:01] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1089.7 | throughput per GPU (TFLOP/s/GPU): 306.0 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.104935E+01 | loss scale: 1.0 | grad norm: 35.991 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[31, 338, 1298, 381, 1460, 771, 2085, 1828] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[42, 658, 1518, 459, 1310, 776, 1497, 1932] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[56, 488, 1466, 540, 1663, 558, 1560, 1861] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[108, 749, 1313, 522, 1160, 444, 1916, 1980] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[67, 483, 983, 538, 2299, 729, 1203, 1890] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[134, 440, 710, 325, 2187, 899, 1724, 1773] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[151, 372, 1581, 494, 1380, 554, 1659, 2001] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[54, 333, 1417, 498, 1213, 845, 1644, 2188] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[31, 858, 338, 456, 1034, 0, 381, 695, 1004, 0, 771, 264, 1390, 0, 970, 0] rerouted_tokens_per_ep_rank=[889, 794, 1034, 1076, 1004, 1035, 1390, 970] avg=1024.0000 max=1390 max_rank=6 min=794 min_rank=1 imbalance_ratio=1.357422 max_load_reduction=6802 max_load_reduction_pct=0.830322 moved_to_echo=4399 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[42, 907, 658, 409, 1209, 0, 459, 499, 901, 0, 776, 309, 998, 0, 1025, 0] rerouted_tokens_per_ep_rank=[949, 1067, 1209, 958, 901, 1085, 998, 1025] avg=1024.0000 max=1209 max_rank=2 min=901 min_rank=4 imbalance_ratio=1.180664 max_load_reduction=6983 max_load_reduction_pct=0.852417 moved_to_echo=4009 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[134, 833, 440, 683, 566, 0, 325, 575, 1504, 0, 899, 144, 1149, 0, 940, 0] rerouted_tokens_per_ep_rank=[967, 1123, 566, 900, 1504, 1043, 1149, 940] avg=1024.0000 max=1504 max_rank=4 min=566 min_rank=2 imbalance_ratio=1.468750 max_load_reduction=6688 max_load_reduction_pct=0.816406 moved_to_echo=4636 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[67, 888, 483, 718, 783, 0, 538, 401, 1581, 0, 729, 200, 802, 0, 1002, 0] rerouted_tokens_per_ep_rank=[955, 1201, 783, 939, 1581, 929, 802, 1002] avg=1024.0000 max=1581 max_rank=4 min=783 min_rank=2 imbalance_ratio=1.543945 max_load_reduction=6611 max_load_reduction_pct=0.807007 moved_to_echo=4314 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[108, 930, 749, 362, 1046, 0, 522, 639, 798, 0, 444, 267, 1277, 0, 1050, 0] rerouted_tokens_per_ep_rank=[1038, 1111, 1046, 1161, 798, 711, 1277, 1050] avg=1024.0000 max=1277 max_rank=6 min=711 min_rank=5 imbalance_ratio=1.247070 max_load_reduction=6915 max_load_reduction_pct=0.844116 moved_to_echo=3836 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[151, 943, 372, 433, 1257, 0, 494, 558, 947, 0, 554, 324, 1101, 0, 1058, 0] rerouted_tokens_per_ep_rank=[1094, 805, 1257, 1052, 947, 878, 1101, 1058] avg=1024.0000 max=1257 max_rank=2 min=805 min_rank=1 imbalance_ratio=1.227539 max_load_reduction=6935 max_load_reduction_pct=0.846558 moved_to_echo=3984 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[56, 874, 488, 519, 1168, 0, 540, 520, 1144, 0, 558, 298, 1040, 0, 987, 0] rerouted_tokens_per_ep_rank=[930, 1007, 1168, 1060, 1144, 856, 1040, 987] avg=1024.0000 max=1168 max_rank=2 min=856 min_rank=5 imbalance_ratio=1.140625 max_load_reduction=7024 max_load_reduction_pct=0.857422 moved_to_echo=4027 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[54, 1028, 333, 378, 1129, 0, 498, 548, 835, 0, 845, 288, 1096, 0, 1160, 0] rerouted_tokens_per_ep_rank=[1082, 711, 1129, 1046, 835, 1133, 1096, 1160] avg=1024.0000 max=1160 max_rank=7 min=711 min_rank=1 imbalance_ratio=1.132812 max_load_reduction=7032 max_load_reduction_pct=0.858398 moved_to_echo=4224 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7670 expected_overflow=0 exact_recv_per_rank=[7904, 7819, 8192, 8192, 8714, 7670, 8853, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8714 expected_overflow=0 exact_recv_per_rank=[7904, 7819, 8192, 8192, 8714, 7670, 8853, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7904, 7819, 8192, 8192, 8714, 7670, 8853, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8853 expected_overflow=0 exact_recv_per_rank=[7904, 7819, 8192, 8192, 8714, 7670, 8853, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7819 expected_overflow=0 exact_recv_per_rank=[7904, 7819, 8192, 8192, 8714, 7670, 8853, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7904, 7819, 8192, 8192, 8714, 7670, 8853, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7904 expected_overflow=0 exact_recv_per_rank=[7904, 7819, 8192, 8192, 8714, 7670, 8853, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7904, 7819, 8192, 8192, 8714, 7670, 8853, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=7904 actual_valid_recv=8064 observed_loss=-160 tokens_per_expert=[768, 7296] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3840, 4480] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7670 actual_valid_recv=7808 observed_loss=-138 tokens_per_expert=[5632, 2176] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=7819 actual_valid_recv=7936 observed_loss=-117 tokens_per_expert=[3968, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8714 actual_valid_recv=8832 observed_loss=-118 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8853 actual_valid_recv=8960 observed_loss=-107 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8064 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=7936 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1707, 2443, 172, 3387, 166, 117, 50, 150] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2051, 1727, 231, 3395, 122, 274, 229, 163] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[769, 2355, 339, 3750, 242, 268, 178, 291] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1103, 2232, 142, 3892, 263, 271, 62, 227] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1446, 1976, 166, 3832, 213, 357, 96, 106] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2155, 1571, 219, 3478, 260, 278, 120, 111] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1201, 2209, 135, 3894, 208, 166, 312, 67] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1095, 2904, 116, 3398, 228, 199, 217, 35] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1214, 0, 1560, 0, 172, 778, 980, 0, 166, 883, 117, 493, 50, 808, 150, 821] rerouted_tokens_per_ep_rank=[1214, 1560, 950, 980, 1049, 610, 858, 971] avg=1024.0000 max=1560 max_rank=1 min=610 min_rank=5 imbalance_ratio=1.523438 max_load_reduction=6632 max_load_reduction_pct=0.809570 moved_to_echo=3488 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[547, 0, 1504, 0, 339, 861, 1085, 0, 242, 851, 268, 222, 178, 895, 291, 909] rerouted_tokens_per_ep_rank=[547, 1504, 1200, 1085, 1093, 490, 1073, 1200] avg=1024.0000 max=1504 max_rank=1 min=490 min_rank=5 imbalance_ratio=1.468750 max_load_reduction=6688 max_load_reduction_pct=0.816406 moved_to_echo=3856 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[784, 0, 1425, 0, 142, 894, 1126, 0, 263, 807, 271, 319, 62, 928, 227, 944] rerouted_tokens_per_ep_rank=[784, 1425, 1036, 1126, 1070, 590, 990, 1171] avg=1024.0000 max=1425 max_rank=1 min=590 min_rank=5 imbalance_ratio=1.391602 max_load_reduction=6767 max_load_reduction_pct=0.826050 moved_to_echo=3821 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 1259, 0, 166, 884, 1097, 0, 213, 717, 357, 422, 96, 917, 106, 934] rerouted_tokens_per_ep_rank=[1024, 1259, 1050, 1097, 930, 779, 1013, 1040] avg=1024.0000 max=1259 max_rank=1 min=779 min_rank=5 imbalance_ratio=1.229492 max_load_reduction=6933 max_load_reduction_pct=0.846313 moved_to_echo=3762 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1458, 0, 1103, 0, 231, 780, 982, 0, 122, 624, 274, 593, 229, 810, 163, 823] rerouted_tokens_per_ep_rank=[1458, 1103, 1011, 982, 746, 867, 1039, 986] avg=1024.0000 max=1458 max_rank=0 min=746 min_rank=4 imbalance_ratio=1.423828 max_load_reduction=6734 max_load_reduction_pct=0.822021 moved_to_echo=3638 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[854, 0, 1411, 0, 135, 895, 1126, 0, 208, 798, 166, 347, 312, 929, 67, 944] rerouted_tokens_per_ep_rank=[854, 1411, 1030, 1126, 1006, 513, 1241, 1011] avg=1024.0000 max=1411 max_rank=1 min=513 min_rank=5 imbalance_ratio=1.377930 max_load_reduction=6781 max_load_reduction_pct=0.827759 moved_to_echo=3771 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[779, 0, 1854, 0, 116, 781, 982, 0, 228, 1050, 199, 316, 217, 811, 35, 824] rerouted_tokens_per_ep_rank=[779, 1854, 897, 982, 1278, 515, 1028, 859] avg=1024.0000 max=1854 max_rank=1 min=515 min_rank=5 imbalance_ratio=1.810547 max_load_reduction=6338 max_load_reduction_pct=0.773682 moved_to_echo=3680 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1532, 0, 1003, 0, 219, 799, 1006, 0, 260, 568, 278, 623, 120, 830, 111, 843] rerouted_tokens_per_ep_rank=[1532, 1003, 1018, 1006, 828, 901, 950, 954] avg=1024.0000 max=1532 max_rank=0 min=828 min_rank=4 imbalance_ratio=1.496094 max_load_reduction=6660 max_load_reduction_pct=0.812988 moved_to_echo=3633 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8384 expected_overflow=0 exact_recv_per_rank=[8192, 11119, 8192, 8384, 8000, 5265, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11119 expected_overflow=0 exact_recv_per_rank=[8192, 11119, 8192, 8384, 8000, 5265, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 11119, 8192, 8384, 8000, 5265, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 11119, 8192, 8384, 8000, 5265, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5265 expected_overflow=0 exact_recv_per_rank=[8192, 11119, 8192, 8384, 8000, 5265, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8000 expected_overflow=0 exact_recv_per_rank=[8192, 11119, 8192, 8384, 8000, 5265, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 11119, 8192, 8384, 8000, 5265, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 11119, 8192, 8384, 8000, 5265, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=5265 actual_valid_recv=5504 observed_loss=-239 tokens_per_expert=[2048, 3456] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8000 actual_valid_recv=8192 observed_loss=-192 tokens_per_expert=[1792, 6400] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1280, 7040] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1536, 6784] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8384 actual_valid_recv=8448 observed_loss=-64 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=11119 actual_valid_recv=11136 observed_loss=-17 tokens_per_expert=[11136, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=5504 slack_slots=10880 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=11136 slack_slots=5248 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[222, 123, 33, 68, 2439, 168, 1313, 3826] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[12, 67, 67, 162, 2552, 97, 1414, 3821] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[5, 150, 20, 66, 2150, 87, 1902, 3812] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[272, 30, 133, 204, 1486, 547, 2013, 3507] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[93, 110, 18, 468, 993, 159, 2859, 3492] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[80, 39, 12, 104, 848, 167, 3081, 3861] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[152, 331, 11, 274, 1440, 445, 1973, 3566] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[214, 1049, 8, 121, 2006, 319, 837, 3638] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[222, 925, 123, 536, 33, 1022, 68, 816, 1436, 0, 168, 1003, 777, 0, 1063, 0] rerouted_tokens_per_ep_rank=[1147, 659, 1055, 884, 1436, 1171, 777, 1063] avg=1024.0000 max=1436 max_rank=4 min=659 min_rank=1 imbalance_ratio=1.402344 max_load_reduction=6756 max_load_reduction_pct=0.824707 moved_to_echo=4447 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[12, 924, 67, 578, 67, 1021, 162, 815, 1503, 0, 97, 1049, 836, 0, 1061, 0] rerouted_tokens_per_ep_rank=[936, 645, 1088, 977, 1503, 1146, 836, 1061] avg=1024.0000 max=1503 max_rank=4 min=645 min_rank=1 imbalance_ratio=1.467773 max_load_reduction=6689 max_load_reduction_pct=0.816528 moved_to_echo=4546 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[272, 848, 30, 823, 133, 937, 204, 748, 875, 0, 547, 611, 1190, 0, 974, 0] rerouted_tokens_per_ep_rank=[1120, 853, 1070, 952, 875, 1158, 1190, 974] avg=1024.0000 max=1190 max_rank=6 min=853 min_rank=1 imbalance_ratio=1.162109 max_load_reduction=7002 max_load_reduction_pct=0.854736 moved_to_echo=4197 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 925, 150, 781, 20, 1021, 66, 816, 1263, 0, 87, 887, 1121, 0, 1050, 0] rerouted_tokens_per_ep_rank=[930, 931, 1041, 882, 1263, 974, 1121, 1050] avg=1024.0000 max=1263 max_rank=4 min=882 min_rank=3 imbalance_ratio=1.233398 max_load_reduction=6929 max_load_reduction_pct=0.845825 moved_to_echo=4408 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[80, 934, 39, 1259, 12, 1031, 104, 823, 500, 0, 167, 348, 1822, 0, 1073, 0] rerouted_tokens_per_ep_rank=[1014, 1298, 1043, 927, 500, 515, 1822, 1073] avg=1024.0000 max=1822 max_rank=6 min=500 min_rank=4 imbalance_ratio=1.779297 max_load_reduction=6370 max_load_reduction_pct=0.777588 moved_to_echo=3910 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[152, 862, 331, 806, 11, 953, 274, 760, 848, 0, 445, 592, 1167, 0, 991, 0] rerouted_tokens_per_ep_rank=[1014, 1137, 964, 1034, 848, 1037, 1167, 991] avg=1024.0000 max=1167 max_rank=6 min=848 min_rank=4 imbalance_ratio=1.139648 max_load_reduction=7025 max_load_reduction_pct=0.857544 moved_to_echo=4043 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[93, 844, 110, 1168, 18, 933, 468, 745, 585, 0, 159, 408, 1691, 0, 970, 0] rerouted_tokens_per_ep_rank=[937, 1278, 951, 1213, 585, 567, 1691, 970] avg=1024.0000 max=1691 max_rank=6 min=567 min_rank=5 imbalance_ratio=1.651367 max_load_reduction=6501 max_load_reduction_pct=0.793579 moved_to_echo=3813 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[214, 880, 1049, 342, 8, 972, 121, 776, 1182, 0, 319, 824, 495, 0, 1010, 0] rerouted_tokens_per_ep_rank=[1094, 1391, 980, 897, 1182, 1143, 495, 1010] avg=1024.0000 max=1391 max_rank=1 min=495 min_rank=6 imbalance_ratio=1.358398 max_load_reduction=6801 max_load_reduction_pct=0.830200 moved_to_echo=3830 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9099 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7766, 8192, 7711, 9099, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7711 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7766, 8192, 7711, 9099, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7766 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7766, 8192, 7711, 9099, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7766, 8192, 7711, 9099, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7766, 8192, 7711, 9099, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7766, 8192, 7711, 9099, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7766, 8192, 7711, 9099, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7766, 8192, 7711, 9099, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7711 actual_valid_recv=7808 observed_loss=-97 tokens_per_expert=[2048, 5760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1920, 6400] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=7766 actual_valid_recv=7936 observed_loss=-170 tokens_per_expert=[1536, 6400] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9099 actual_valid_recv=9216 observed_loss=-117 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7936 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[252, 10, 2819, 3, 102, 930, 3912, 164] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[341, 63, 1607, 65, 1378, 700, 3742, 296] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[519, 95, 1875, 21, 206, 1230, 3970, 276] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[504, 27, 1958, 3, 38, 1768, 3598, 296] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1177, 32, 1455, 12, 154, 1755, 3207, 400] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1018, 0, 624, 317, 71, 1535, 3807, 820] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[439, 11, 1701, 40, 77, 1138, 3233, 1553] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[272, 220, 977, 25, 908, 2233, 3191, 366] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[252, 255, 10, 1055, 1949, 0, 3, 1051, 102, 686, 675, 0, 1120, 0, 164, 870] rerouted_tokens_per_ep_rank=[507, 1065, 1949, 1054, 788, 675, 1120, 1034] avg=1024.0000 max=1949 max_rank=2 min=507 min_rank=0 imbalance_ratio=1.903320 max_load_reduction=6243 max_load_reduction_pct=0.762085 moved_to_echo=3617 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[519, 337, 95, 1071, 1296, 0, 21, 1067, 206, 696, 893, 0, 1136, 0, 276, 579] rerouted_tokens_per_ep_rank=[856, 1166, 1296, 1088, 902, 893, 1136, 855] avg=1024.0000 max=1296 max_rank=2 min=855 min_rank=7 imbalance_ratio=1.265625 max_load_reduction=6896 max_load_reduction_pct=0.841797 moved_to_echo=3786 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[504, 485, 27, 970, 1354, 0, 3, 967, 38, 631, 1283, 0, 1030, 0, 296, 604] rerouted_tokens_per_ep_rank=[989, 997, 1354, 970, 669, 1283, 1030, 900] avg=1024.0000 max=1354 max_rank=2 min=669 min_rank=4 imbalance_ratio=1.322266 max_load_reduction=6838 max_load_reduction_pct=0.834717 moved_to_echo=3882 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[341, 194, 63, 1013, 1106, 0, 65, 1010, 1378, 660, 506, 0, 1059, 0, 296, 501] rerouted_tokens_per_ep_rank=[535, 1076, 1106, 1075, 2038, 506, 1059, 797] avg=1024.0000 max=2038 max_rank=4 min=506 min_rank=5 imbalance_ratio=1.990234 max_load_reduction=6154 max_load_reduction_pct=0.751221 moved_to_echo=4400 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1177, 481, 32, 865, 1006, 0, 12, 862, 154, 562, 1274, 0, 918, 0, 400, 449] rerouted_tokens_per_ep_rank=[1658, 897, 1006, 874, 716, 1274, 918, 849] avg=1024.0000 max=1658 max_rank=0 min=716 min_rank=4 imbalance_ratio=1.619141 max_load_reduction=6534 max_load_reduction_pct=0.797607 moved_to_echo=3757 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[439, 312, 11, 872, 1176, 0, 40, 869, 77, 567, 826, 0, 925, 0, 1553, 525] rerouted_tokens_per_ep_rank=[751, 883, 1176, 909, 644, 826, 925, 2078] avg=1024.0000 max=2078 max_rank=7 min=644 min_rank=4 imbalance_ratio=2.029297 max_load_reduction=6114 max_load_reduction_pct=0.746338 moved_to_echo=4473 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1018, 421, 0, 1027, 432, 0, 317, 1023, 71, 667, 1114, 0, 1090, 0, 820, 192] rerouted_tokens_per_ep_rank=[1439, 1027, 432, 1340, 738, 1114, 1090, 1012] avg=1024.0000 max=1439 max_rank=0 min=432 min_rank=2 imbalance_ratio=1.405273 max_load_reduction=6753 max_load_reduction_pct=0.824341 moved_to_echo=3954 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[272, 612, 220, 861, 676, 0, 25, 857, 908, 559, 1621, 0, 914, 0, 366, 301] rerouted_tokens_per_ep_rank=[884, 1081, 676, 882, 1467, 1621, 914, 667] avg=1024.0000 max=1621 max_rank=5 min=667 min_rank=7 imbalance_ratio=1.583008 max_load_reduction=6571 max_load_reduction_pct=0.802124 moved_to_echo=4669 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7619, 8192, 8995, 8192, 7962, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7619 expected_overflow=0 exact_recv_per_rank=[7619, 8192, 8995, 8192, 7962, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7619, 8192, 8995, 8192, 7962, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7619, 8192, 8995, 8192, 7962, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8995 expected_overflow=0 exact_recv_per_rank=[7619, 8192, 8995, 8192, 7962, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7619, 8192, 8995, 8192, 7962, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7962 expected_overflow=0 exact_recv_per_rank=[7619, 8192, 8995, 8192, 7962, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7619, 8192, 8995, 8192, 7962, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=7619 actual_valid_recv=7808 observed_loss=-189 tokens_per_expert=[4608, 3200] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4224, 4096] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7962 actual_valid_recv=8064 observed_loss=-102 tokens_per_expert=[2944, 5120] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8995 actual_valid_recv=9088 observed_loss=-93 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8064 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[107, 185, 114, 3916, 0, 8, 3633, 229] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[516, 452, 278, 2958, 0, 1, 3980, 7] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[396, 151, 832, 3497, 0, 16, 3283, 17] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[110, 47, 391, 3924, 0, 0, 3719, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[48, 161, 313, 3953, 0, 5, 3712, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[61, 83, 180, 3910, 0, 10, 3864, 84] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[165, 77, 507, 3942, 1, 132, 3360, 8] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1766, 475, 1985, 2511, 0, 16, 1437, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[107, 676, 185, 883, 114, 483, 1122, 0, 0, 1121, 8, 1095, 1591, 0, 229, 578] rerouted_tokens_per_ep_rank=[783, 1068, 597, 1122, 1121, 1103, 1591, 807] avg=1024.0000 max=1591 max_rank=6 min=597 min_rank=2 imbalance_ratio=1.553711 max_load_reduction=6601 max_load_reduction_pct=0.805786 moved_to_echo=4622 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[110, 692, 47, 904, 391, 494, 1125, 0, 0, 1123, 0, 1097, 1629, 0, 1, 579] rerouted_tokens_per_ep_rank=[802, 951, 885, 1125, 1123, 1097, 1629, 580] avg=1024.0000 max=1629 max_rank=6 min=580 min_rank=7 imbalance_ratio=1.590820 max_load_reduction=6563 max_load_reduction_pct=0.801147 moved_to_echo=4429 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[48, 690, 161, 902, 313, 494, 1134, 0, 0, 1131, 5, 1105, 1626, 0, 0, 583] rerouted_tokens_per_ep_rank=[738, 1063, 807, 1134, 1131, 1110, 1626, 583] avg=1024.0000 max=1626 max_rank=6 min=583 min_rank=7 imbalance_ratio=1.587891 max_load_reduction=6566 max_load_reduction_pct=0.801514 moved_to_echo=4450 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[396, 614, 151, 801, 832, 440, 989, 0, 0, 1005, 16, 983, 1428, 0, 17, 520] rerouted_tokens_per_ep_rank=[1010, 952, 1272, 989, 1005, 999, 1428, 537] avg=1024.0000 max=1428 max_rank=6 min=537 min_rank=7 imbalance_ratio=1.394531 max_load_reduction=6764 max_load_reduction_pct=0.825684 moved_to_echo=3969 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[61, 719, 83, 939, 180, 514, 1121, 0, 0, 1119, 10, 1093, 1692, 0, 84, 577] rerouted_tokens_per_ep_rank=[780, 1022, 694, 1121, 1119, 1103, 1692, 661] avg=1024.0000 max=1692 max_rank=6 min=661 min_rank=7 imbalance_ratio=1.652344 max_load_reduction=6500 max_load_reduction_pct=0.793457 moved_to_echo=4575 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 740, 452, 967, 278, 529, 849, 0, 0, 846, 1, 827, 1744, 0, 7, 436] rerouted_tokens_per_ep_rank=[1256, 1419, 807, 849, 846, 828, 1744, 443] avg=1024.0000 max=1744 max_rank=6 min=443 min_rank=7 imbalance_ratio=1.703125 max_load_reduction=6448 max_load_reduction_pct=0.787109 moved_to_echo=3861 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[165, 625, 77, 816, 507, 447, 1131, 0, 1, 1128, 132, 1102, 1472, 0, 8, 581] rerouted_tokens_per_ep_rank=[790, 893, 954, 1131, 1129, 1234, 1472, 589] avg=1024.0000 max=1472 max_rank=6 min=589 min_rank=7 imbalance_ratio=1.437500 max_load_reduction=6720 max_load_reduction_pct=0.820312 moved_to_echo=4424 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1766, 267, 475, 349, 1985, 191, 721, 0, 0, 718, 16, 702, 630, 0, 2, 370] rerouted_tokens_per_ep_rank=[2033, 824, 2176, 721, 718, 718, 630, 372] avg=1024.0000 max=2176 max_rank=2 min=372 min_rank=7 imbalance_ratio=2.125000 max_load_reduction=6016 max_load_reduction_pct=0.734375 moved_to_echo=2438 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 11812, 4572] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 11812, 4572] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11812 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 11812, 4572] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 11812, 4572] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 11812, 4572] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 11812, 4572] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 11812, 4572] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4572 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 11812, 4572] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=4572 actual_valid_recv=4608 observed_loss=-36 tokens_per_expert=[384, 4224] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3200, 5120] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4608, 3712] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1664, 6656] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=11812 actual_valid_recv=11904 observed_loss=-92 tokens_per_expert=[11904, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=4608 slack_slots=11776 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=11904 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[24, 4083, 4, 10, 0, 4044, 26, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[5, 4081, 26, 76, 0, 3985, 17, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[3, 4094, 39, 60, 0, 3996, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1, 4094, 208, 204, 0, 3682, 2, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 3, 7, 0, 4086, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[27, 4006, 72, 21, 0, 4059, 0, 7] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 1, 3, 0, 4092, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[68, 4066, 3, 105, 3, 3943, 4, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 1010, 1031, 0, 39, 981, 60, 965, 0, 1027, 1040, 0, 0, 1010, 0, 1026] rerouted_tokens_per_ep_rank=[1013, 1031, 1020, 1025, 1027, 1040, 1010, 1026] avg=1024.0000 max=1040 max_rank=5 min=1010 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[24, 1022, 1026, 0, 4, 993, 10, 977, 0, 1025, 1052, 0, 26, 1008, 1, 1024] rerouted_tokens_per_ep_rank=[1046, 1026, 997, 987, 1025, 1052, 1034, 1025] avg=1024.0000 max=1052 max_rank=5 min=987 min_rank=3 imbalance_ratio=1.027344 max_load_reduction=7140 max_load_reduction_pct=0.871582 moved_to_echo=4136 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[5, 1011, 1013, 0, 26, 984, 76, 968, 0, 1029, 1022, 0, 17, 1011, 2, 1028] rerouted_tokens_per_ep_rank=[1016, 1013, 1010, 1044, 1029, 1022, 1028, 1030] avg=1024.0000 max=1044 max_rank=3 min=1010 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=7148 max_load_reduction_pct=0.872559 moved_to_echo=4109 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 931, 1031, 0, 208, 904, 204, 889, 0, 1027, 958, 0, 2, 1010, 1, 1026] rerouted_tokens_per_ep_rank=[932, 1031, 1112, 1093, 1027, 958, 1012, 1027] avg=1024.0000 max=1112 max_rank=2 min=932 min_rank=0 imbalance_ratio=1.085938 max_load_reduction=7080 max_load_reduction_pct=0.864258 moved_to_echo=4024 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1033, 1029, 0, 3, 1004, 7, 987, 0, 1028, 1062, 0, 1, 1011, 0, 1027] rerouted_tokens_per_ep_rank=[1033, 1029, 1007, 994, 1028, 1062, 1012, 1027] avg=1024.0000 max=1062 max_rank=5 min=994 min_rank=3 imbalance_ratio=1.037109 max_load_reduction=7130 max_load_reduction_pct=0.870361 moved_to_echo=4129 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[27, 1026, 1008, 0, 72, 997, 21, 980, 0, 1005, 1056, 0, 0, 989, 7, 1004] rerouted_tokens_per_ep_rank=[1053, 1008, 1069, 1001, 1005, 1056, 989, 1011] avg=1024.0000 max=1069 max_rank=2 min=989 min_rank=6 imbalance_ratio=1.043945 max_load_reduction=7123 max_load_reduction_pct=0.869507 moved_to_echo=4061 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1034, 1030, 0, 1, 1005, 3, 988, 0, 1028, 1065, 0, 0, 1011, 0, 1027] rerouted_tokens_per_ep_rank=[1034, 1030, 1006, 991, 1028, 1065, 1011, 1027] avg=1024.0000 max=1065 max_rank=5 min=991 min_rank=3 imbalance_ratio=1.040039 max_load_reduction=7127 max_load_reduction_pct=0.869995 moved_to_echo=4131 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[68, 997, 1024, 0, 3, 968, 105, 952, 3, 1020, 1026, 0, 4, 1003, 0, 1019] rerouted_tokens_per_ep_rank=[1065, 1024, 971, 1057, 1023, 1026, 1007, 1019] avg=1024.0000 max=1065 max_rank=0 min=971 min_rank=2 imbalance_ratio=1.040039 max_load_reduction=7127 max_load_reduction_pct=0.869995 moved_to_echo=4075 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8281 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8281, 8103, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8281, 8103, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8281, 8103, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8281, 8103, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8281, 8103, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8103 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8281, 8103, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8281, 8103, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8281, 8103, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8281 actual_valid_recv=8320 observed_loss=-39 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[128, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8103 actual_valid_recv=8192 observed_loss=-89 tokens_per_expert=[128, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 1352, 3152, 1, 48, 4, 0, 3635] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 676, 3299, 4, 8, 108, 14, 4083] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[6, 1073, 3877, 13, 80, 38, 217, 2888] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 508, 4043, 27, 314, 52, 153, 3094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 338, 4085, 7, 388, 26, 35, 3313] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 1114, 3863, 358, 8, 0, 192, 2657] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 1235, 3857, 73, 39, 0, 0, 2988] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[10, 267, 3571, 14, 233, 42, 3, 4052] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 866, 1352, 221, 869, 0, 1, 578, 48, 962, 4, 839, 0, 1031, 1421, 0] rerouted_tokens_per_ep_rank=[866, 1573, 869, 579, 1010, 843, 1031, 1421] avg=1024.0000 max=1573 max_rank=1 min=579 min_rank=3 imbalance_ratio=1.536133 max_load_reduction=6619 max_load_reduction_pct=0.807983 moved_to_echo=4305 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1122, 338, 202, 1127, 0, 7, 749, 388, 877, 26, 1087, 35, 939, 1295, 0] rerouted_tokens_per_ep_rank=[1122, 540, 1127, 756, 1265, 1113, 974, 1295] avg=1024.0000 max=1295 max_rank=7 min=540 min_rank=1 imbalance_ratio=1.264648 max_load_reduction=6897 max_load_reduction_pct=0.841919 moved_to_echo=4647 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1059, 1235, 182, 1064, 0, 73, 707, 39, 791, 0, 1027, 0, 847, 1168, 0] rerouted_tokens_per_ep_rank=[1059, 1417, 1064, 780, 830, 1027, 847, 1168] avg=1024.0000 max=1417 max_rank=1 min=780 min_rank=3 imbalance_ratio=1.383789 max_load_reduction=6775 max_load_reduction_pct=0.827026 moved_to_echo=3872 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 906, 676, 249, 910, 0, 4, 605, 8, 1081, 108, 878, 14, 1158, 1595, 0] rerouted_tokens_per_ep_rank=[906, 925, 910, 609, 1089, 986, 1172, 1595] avg=1024.0000 max=1595 max_rank=7 min=609 min_rank=3 imbalance_ratio=1.557617 max_load_reduction=6597 max_load_reduction_pct=0.805298 moved_to_echo=4842 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 1069, 1073, 178, 1057, 0, 13, 715, 80, 768, 38, 1036, 217, 824, 1118, 0] rerouted_tokens_per_ep_rank=[1075, 1251, 1057, 728, 848, 1074, 1041, 1118] avg=1024.0000 max=1251 max_rank=1 min=728 min_rank=3 imbalance_ratio=1.221680 max_load_reduction=6941 max_load_reduction_pct=0.847290 moved_to_echo=4081 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1061, 1114, 162, 1066, 0, 358, 708, 8, 703, 0, 1028, 192, 753, 1039, 0] rerouted_tokens_per_ep_rank=[1061, 1276, 1066, 1066, 711, 1028, 945, 1039] avg=1024.0000 max=1276 max_rank=1 min=711 min_rank=4 imbalance_ratio=1.246094 max_load_reduction=6916 max_load_reduction_pct=0.844238 moved_to_echo=3723 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[10, 981, 267, 247, 984, 0, 14, 655, 233, 1073, 42, 951, 3, 1149, 1583, 0] rerouted_tokens_per_ep_rank=[991, 514, 984, 669, 1306, 993, 1152, 1583] avg=1024.0000 max=1583 max_rank=7 min=514 min_rank=1 imbalance_ratio=1.545898 max_load_reduction=6609 max_load_reduction_pct=0.806763 moved_to_echo=5034 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 1111, 508, 188, 1115, 0, 27, 741, 314, 819, 52, 1076, 153, 877, 1210, 0] rerouted_tokens_per_ep_rank=[1112, 696, 1115, 768, 1133, 1128, 1030, 1210] avg=1024.0000 max=1210 max_rank=7 min=696 min_rank=1 imbalance_ratio=1.181641 max_load_reduction=6982 max_load_reduction_pct=0.852295 moved_to_echo=4501 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5955 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5955, 8192, 8192, 8192, 10429] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5955, 8192, 8192, 8192, 10429] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5955, 8192, 8192, 8192, 10429] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5955, 8192, 8192, 8192, 10429] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10429 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5955, 8192, 8192, 8192, 10429] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5955, 8192, 8192, 8192, 10429] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5955, 8192, 8192, 8192, 10429] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5955, 8192, 8192, 8192, 10429] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=5955 actual_valid_recv=6016 observed_loss=-61 tokens_per_expert=[512, 5504] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[640, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6656, 1664] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=10429 actual_valid_recv=10496 observed_loss=-67 tokens_per_expert=[10496, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=6016 slack_slots=10368 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=10496 slack_slots=5888 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[39, 3, 3964, 148, 3757, 14, 0, 267] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 0, 4021, 111, 4006, 8, 0, 45] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2222, 882, 4037, 789, 0, 262] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[11, 3, 3906, 110, 4076, 86, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2750, 2164, 2806, 5, 0, 467] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3233, 355, 3798, 86, 0, 720] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[22, 0, 4081, 485, 3438, 133, 1, 32] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[48, 1, 272, 3419, 4052, 114, 0, 286] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 727, 0, 1102, 988, 0, 882, 47, 1105, 0, 789, 632, 0, 1103, 262, 555] rerouted_tokens_per_ep_rank=[727, 1102, 988, 929, 1105, 1421, 1103, 817] avg=1024.0000 max=1421 max_rank=5 min=727 min_rank=0 imbalance_ratio=1.387695 max_load_reduction=6771 max_load_reduction_pct=0.826538 moved_to_echo=4446 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[39, 677, 3, 1026, 1763, 0, 148, 83, 1028, 0, 14, 1127, 0, 1026, 267, 991] rerouted_tokens_per_ep_rank=[716, 1029, 1763, 231, 1028, 1141, 1026, 1258] avg=1024.0000 max=1763 max_rank=2 min=231 min_rank=3 imbalance_ratio=1.721680 max_load_reduction=6429 max_load_reduction_pct=0.784790 moved_to_echo=4453 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 722, 0, 1094, 1787, 0, 111, 85, 1096, 0, 8, 1144, 0, 1094, 45, 1005] rerouted_tokens_per_ep_rank=[723, 1094, 1787, 196, 1096, 1152, 1094, 1050] avg=1024.0000 max=1787 max_rank=2 min=196 min_rank=3 imbalance_ratio=1.745117 max_load_reduction=6405 max_load_reduction_pct=0.781860 moved_to_echo=4392 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 505, 0, 766, 1223, 0, 2164, 58, 769, 0, 5, 782, 0, 766, 467, 687] rerouted_tokens_per_ep_rank=[505, 766, 1223, 2222, 769, 787, 766, 1154] avg=1024.0000 max=2222 max_rank=3 min=505 min_rank=0 imbalance_ratio=2.169922 max_load_reduction=5970 max_load_reduction_pct=0.728760 moved_to_echo=3476 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[11, 738, 3, 1116, 1726, 0, 110, 86, 1104, 0, 86, 1115, 0, 1118, 0, 979] rerouted_tokens_per_ep_rank=[749, 1119, 1726, 196, 1104, 1201, 1118, 979] avg=1024.0000 max=1726 max_rank=2 min=196 min_rank=3 imbalance_ratio=1.685547 max_load_reduction=6466 max_load_reduction_pct=0.789307 moved_to_echo=4402 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[22, 619, 0, 938, 1814, 0, 485, 86, 942, 0, 133, 1161, 1, 939, 32, 1020] rerouted_tokens_per_ep_rank=[641, 938, 1814, 571, 942, 1294, 940, 1052] avg=1024.0000 max=1814 max_rank=2 min=571 min_rank=3 imbalance_ratio=1.771484 max_load_reduction=6378 max_load_reduction_pct=0.778564 moved_to_echo=4228 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 684, 0, 1037, 1438, 0, 355, 68, 1039, 0, 86, 919, 0, 1038, 720, 808] rerouted_tokens_per_ep_rank=[684, 1037, 1438, 423, 1039, 1005, 1038, 1528] avg=1024.0000 max=1528 max_rank=7 min=423 min_rank=3 imbalance_ratio=1.492188 max_load_reduction=6664 max_load_reduction_pct=0.813477 moved_to_echo=4610 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[48, 730, 1, 1106, 122, 0, 3419, 5, 1109, 0, 114, 77, 0, 1107, 286, 68] rerouted_tokens_per_ep_rank=[778, 1107, 122, 3424, 1109, 191, 1107, 354] avg=1024.0000 max=3424 max_rank=3 min=122 min_rank=2 imbalance_ratio=3.343750 max_load_reduction=4768 max_load_reduction_pct=0.582031 moved_to_echo=2761 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10861 expected_overflow=0 exact_recv_per_rank=[5523, 8192, 10861, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[5523, 8192, 10861, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[5523, 8192, 10861, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[5523, 8192, 10861, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5523 expected_overflow=0 exact_recv_per_rank=[5523, 8192, 10861, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[5523, 8192, 10861, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[5523, 8192, 10861, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[5523, 8192, 10861, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=5523 actual_valid_recv=5632 observed_loss=-109 tokens_per_expert=[128, 5504] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2176, 6144] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1280, 7040] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7680, 640] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=10861 actual_valid_recv=10880 observed_loss=-19 tokens_per_expert=[10880, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=5632 slack_slots=10752 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=10880 slack_slots=5504 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[140, 1677, 4, 133, 2359, 8, 2, 3869] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1, 572, 4, 4, 4016, 22, 0, 3573] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[42, 2599, 0, 3, 1859, 441, 0, 3248] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[9, 787, 0, 28, 3808, 80, 9, 3471] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[144, 3194, 26, 38, 2924, 9, 51, 1806] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[267, 2001, 0, 6, 2521, 85, 0, 3312] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[28, 2893, 0, 1, 3681, 4, 1, 1584] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[457, 1641, 2, 7, 2765, 420, 0, 2900] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[9, 363, 424, 0, 0, 1297, 28, 1164, 1305, 0, 80, 1030, 9, 1206, 1277, 0] rerouted_tokens_per_ep_rank=[372, 424, 1297, 1192, 1305, 1110, 1215, 1277] avg=1024.0000 max=1305 max_rank=4 min=372 min_rank=0 imbalance_ratio=1.274414 max_load_reduction=6887 max_load_reduction_pct=0.840698 moved_to_echo=4907 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[140, 780, 897, 0, 4, 807, 133, 1302, 800, 0, 8, 1152, 2, 752, 1415, 0] rerouted_tokens_per_ep_rank=[920, 897, 811, 1435, 800, 1160, 754, 1415] avg=1024.0000 max=1435 max_rank=3 min=754 min_rank=6 imbalance_ratio=1.401367 max_load_reduction=6757 max_load_reduction_pct=0.824829 moved_to_echo=4129 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 264, 308, 0, 4, 1368, 4, 1198, 1376, 0, 22, 1060, 0, 1272, 1315, 0] rerouted_tokens_per_ep_rank=[265, 308, 1372, 1202, 1376, 1082, 1272, 1315] avg=1024.0000 max=1376 max_rank=4 min=265 min_rank=0 imbalance_ratio=1.343750 max_load_reduction=6816 max_load_reduction_pct=0.832031 moved_to_echo=5045 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[42, 1201, 1398, 0, 0, 633, 3, 1089, 637, 0, 441, 964, 0, 589, 1195, 0] rerouted_tokens_per_ep_rank=[1243, 1398, 633, 1092, 637, 1405, 589, 1195] avg=1024.0000 max=1405 max_rank=5 min=589 min_rank=6 imbalance_ratio=1.372070 max_load_reduction=6787 max_load_reduction_pct=0.828491 moved_to_echo=3826 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[267, 925, 1076, 0, 0, 859, 6, 1111, 864, 0, 85, 983, 0, 798, 1218, 0] rerouted_tokens_per_ep_rank=[1192, 1076, 859, 1117, 864, 1068, 798, 1218] avg=1024.0000 max=1218 max_rank=7 min=798 min_rank=6 imbalance_ratio=1.189453 max_load_reduction=6974 max_load_reduction_pct=0.851318 moved_to_echo=3948 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[144, 1476, 1718, 0, 26, 996, 38, 605, 1002, 0, 9, 536, 51, 926, 665, 0] rerouted_tokens_per_ep_rank=[1620, 1718, 1022, 643, 1002, 545, 977, 665] avg=1024.0000 max=1718 max_rank=1 min=545 min_rank=5 imbalance_ratio=1.677734 max_load_reduction=6474 max_load_reduction_pct=0.790283 moved_to_echo=3189 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[28, 1337, 1556, 0, 0, 1254, 1, 531, 1261, 0, 4, 470, 1, 1166, 583, 0] rerouted_tokens_per_ep_rank=[1365, 1556, 1254, 532, 1261, 474, 1167, 583] avg=1024.0000 max=1556 max_rank=1 min=474 min_rank=5 imbalance_ratio=1.519531 max_load_reduction=6636 max_load_reduction_pct=0.810059 moved_to_echo=3485 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[457, 758, 883, 0, 2, 942, 7, 972, 947, 0, 420, 860, 0, 876, 1068, 0] rerouted_tokens_per_ep_rank=[1215, 883, 944, 979, 947, 1280, 876, 1068] avg=1024.0000 max=1280 max_rank=5 min=876 min_rank=6 imbalance_ratio=1.250000 max_load_reduction=6912 max_load_reduction_pct=0.843750 moved_to_echo=4171 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8260, 8192, 8192, 8192, 8124, 7648, 8736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8260, 8192, 8192, 8192, 8124, 7648, 8736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8260, 8192, 8192, 8192, 8124, 7648, 8736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8260, 8192, 8192, 8192, 8124, 7648, 8736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8736 expected_overflow=0 exact_recv_per_rank=[8192, 8260, 8192, 8192, 8192, 8124, 7648, 8736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7648 expected_overflow=0 exact_recv_per_rank=[8192, 8260, 8192, 8192, 8192, 8124, 7648, 8736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8124 expected_overflow=0 exact_recv_per_rank=[8192, 8260, 8192, 8192, 8192, 8124, 7648, 8736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8260 expected_overflow=0 exact_recv_per_rank=[8192, 8260, 8192, 8192, 8192, 8124, 7648, 8736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8124 actual_valid_recv=8320 observed_loss=-196 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7648 actual_valid_recv=7808 observed_loss=-160 tokens_per_expert=[128, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8736 actual_valid_recv=8832 observed_loss=-96 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8260 actual_valid_recv=8320 observed_loss=-60 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 1847, 35, 2033, 3045, 0, 65, 1167] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[5, 197, 2, 2026, 1945, 11, 3, 4003] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 139, 1, 1977, 3966, 15, 0, 2094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 31, 0, 3221, 2966, 0, 0, 1974] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 292, 0, 3376, 1656, 0, 0, 2868] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[10, 74, 0, 3228, 3749, 0, 9, 1122] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[12, 322, 54, 505, 3903, 18, 5, 3373] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[10, 200, 0, 3539, 913, 8, 53, 3469] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1121, 1847, 519, 35, 470, 838, 0, 1127, 0, 0, 797, 65, 676, 697, 0] rerouted_tokens_per_ep_rank=[1121, 2366, 505, 838, 1127, 797, 741, 697] avg=1024.0000 max=2366 max_rank=1 min=505 min_rank=2 imbalance_ratio=2.310547 max_load_reduction=5826 max_load_reduction_pct=0.711182 moved_to_echo=3362 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1460, 139, 505, 1, 845, 815, 0, 1468, 0, 15, 1038, 0, 657, 1249, 0] rerouted_tokens_per_ep_rank=[1460, 644, 846, 815, 1468, 1053, 657, 1249] avg=1024.0000 max=1468 max_rank=4 min=644 min_rank=1 imbalance_ratio=1.433594 max_load_reduction=6724 max_load_reduction_pct=0.820801 moved_to_echo=4427 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1092, 31, 823, 0, 796, 1327, 0, 1098, 0, 0, 776, 0, 1071, 1178, 0] rerouted_tokens_per_ep_rank=[1092, 854, 796, 1327, 1098, 776, 1071, 1178] avg=1024.0000 max=1327 max_rank=3 min=776 min_rank=5 imbalance_ratio=1.295898 max_load_reduction=6865 max_load_reduction_pct=0.838013 moved_to_echo=4123 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[5, 720, 197, 522, 2, 1619, 827, 0, 713, 0, 11, 512, 3, 677, 2384, 0] rerouted_tokens_per_ep_rank=[725, 719, 1621, 827, 713, 523, 680, 2384] avg=1024.0000 max=2384 max_rank=7 min=523 min_rank=5 imbalance_ratio=2.328125 max_load_reduction=5808 max_load_reduction_pct=0.708984 moved_to_echo=4300 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[10, 1380, 74, 825, 0, 452, 1329, 0, 1388, 0, 0, 981, 9, 1074, 670, 0] rerouted_tokens_per_ep_rank=[1390, 899, 452, 1329, 1388, 981, 1083, 670] avg=1024.0000 max=1390 max_rank=0 min=452 min_rank=2 imbalance_ratio=1.357422 max_load_reduction=6802 max_load_reduction_pct=0.830322 moved_to_echo=4122 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[12, 1437, 322, 129, 54, 1361, 208, 0, 1445, 0, 18, 1021, 5, 168, 2012, 0] rerouted_tokens_per_ep_rank=[1449, 451, 1415, 208, 1445, 1039, 173, 2012] avg=1024.0000 max=2012 max_rank=7 min=173 min_rank=6 imbalance_ratio=1.964844 max_load_reduction=6180 max_load_reduction_pct=0.754395 moved_to_echo=4669 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 609, 292, 863, 0, 1157, 1390, 0, 614, 0, 0, 433, 0, 1123, 1711, 0] rerouted_tokens_per_ep_rank=[609, 1155, 1157, 1390, 614, 433, 1123, 1711] avg=1024.0000 max=1711 max_rank=7 min=433 min_rank=5 imbalance_ratio=1.670898 max_load_reduction=6481 max_load_reduction_pct=0.791138 moved_to_echo=3881 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[10, 336, 200, 904, 0, 1400, 1458, 0, 339, 0, 8, 238, 53, 1177, 2069, 0] rerouted_tokens_per_ep_rank=[346, 1104, 1400, 1458, 339, 246, 1230, 2069] avg=1024.0000 max=2069 max_rank=7 min=246 min_rank=5 imbalance_ratio=2.020508 max_load_reduction=6123 max_load_reduction_pct=0.747437 moved_to_echo=3884 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5848, 6758, 11970] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5848, 6758, 11970] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5848 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5848, 6758, 11970] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5848, 6758, 11970] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11970 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5848, 6758, 11970] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5848, 6758, 11970] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5848, 6758, 11970] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6758 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5848, 6758, 11970] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=5848 actual_valid_recv=6016 observed_loss=-168 tokens_per_expert=[128, 5888] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=6758 actual_valid_recv=6912 observed_loss=-154 tokens_per_expert=[256, 6656] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3200, 5120] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=11970 actual_valid_recv=12032 observed_loss=-62 tokens_per_expert=[12032, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=6016 slack_slots=10368 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=6912 slack_slots=9472 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=12032 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 6, 94, 154, 3887, 3980, 28, 43] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 22, 34, 61, 4047, 3976, 29, 23] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[10, 17, 229, 404, 3659, 3863, 7, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1, 1, 1, 107, 2998, 4092, 245, 747] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 72, 1907, 2690, 3412, 69, 42] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[2, 0, 8, 1760, 2302, 3384, 0, 736] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 6, 3157, 2402, 2576, 3, 48] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[19, 6, 5, 2744, 1471, 3891, 30, 26] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1113, 6, 1110, 94, 1283, 154, 0, 1872, 0, 1119, 0, 28, 638, 43, 732] rerouted_tokens_per_ep_rank=[1113, 1116, 1377, 154, 1872, 1119, 666, 775] avg=1024.0000 max=1872 max_rank=4 min=154 min_rank=3 imbalance_ratio=1.828125 max_load_reduction=6320 max_load_reduction_pct=0.771484 moved_to_echo=4432 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1112, 22, 1109, 34, 1335, 61, 0, 1950, 0, 1117, 0, 29, 638, 23, 762] rerouted_tokens_per_ep_rank=[1112, 1131, 1369, 61, 1950, 1117, 667, 785] avg=1024.0000 max=1950 max_rank=4 min=61 min_rank=3 imbalance_ratio=1.904297 max_load_reduction=6242 max_load_reduction_pct=0.761963 moved_to_echo=4519 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 954, 0, 952, 72, 887, 1907, 0, 1296, 0, 959, 0, 69, 547, 42, 507] rerouted_tokens_per_ep_rank=[954, 952, 959, 1907, 1296, 959, 616, 549] avg=1024.0000 max=1907 max_rank=3 min=549 min_rank=7 imbalance_ratio=1.862305 max_load_reduction=6285 max_load_reduction_pct=0.767212 moved_to_echo=3420 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1147, 1, 1145, 1, 995, 107, 0, 1433, 0, 1140, 0, 245, 660, 747, 570] rerouted_tokens_per_ep_rank=[1148, 1146, 996, 107, 1433, 1140, 905, 1317] avg=1024.0000 max=1433 max_rank=4 min=107 min_rank=3 imbalance_ratio=1.399414 max_load_reduction=6759 max_load_reduction_pct=0.825073 moved_to_echo=4795 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[10, 1080, 17, 1077, 229, 1207, 404, 0, 1763, 0, 1087, 0, 7, 619, 3, 689] rerouted_tokens_per_ep_rank=[1090, 1094, 1436, 404, 1763, 1087, 626, 692] avg=1024.0000 max=1763 max_rank=4 min=404 min_rank=3 imbalance_ratio=1.721680 max_load_reduction=6429 max_load_reduction_pct=0.784790 moved_to_echo=4168 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 946, 0, 944, 8, 759, 1760, 0, 1110, 0, 951, 0, 0, 543, 736, 433] rerouted_tokens_per_ep_rank=[948, 944, 767, 1760, 1110, 951, 543, 1169] avg=1024.0000 max=1760 max_rank=3 min=543 min_rank=6 imbalance_ratio=1.718750 max_load_reduction=6432 max_load_reduction_pct=0.785156 moved_to_echo=3773 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 720, 0, 718, 6, 792, 3157, 0, 1158, 0, 725, 0, 3, 413, 48, 452] rerouted_tokens_per_ep_rank=[720, 718, 798, 3157, 1158, 725, 416, 500] avg=1024.0000 max=3157 max_rank=3 min=416 min_rank=6 imbalance_ratio=3.083008 max_load_reduction=5035 max_load_reduction_pct=0.614624 moved_to_echo=2799 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[19, 1088, 6, 1085, 5, 485, 2744, 0, 709, 0, 1094, 0, 30, 624, 26, 277] rerouted_tokens_per_ep_rank=[1107, 1091, 490, 2744, 709, 1094, 654, 303] avg=1024.0000 max=2744 max_rank=3 min=303 min_rank=7 imbalance_ratio=2.679688 max_load_reduction=5448 max_load_reduction_pct=0.665039 moved_to_echo=2760 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10294 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 10294, 11291, 8192, 5093, 6090] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11291 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 10294, 11291, 8192, 5093, 6090] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 10294, 11291, 8192, 5093, 6090] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 10294, 11291, 8192, 5093, 6090] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 10294, 11291, 8192, 5093, 6090] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 10294, 11291, 8192, 5093, 6090] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5093 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 10294, 11291, 8192, 5093, 6090] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6090 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 10294, 11291, 8192, 5093, 6090] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=5093 actual_valid_recv=5248 observed_loss=-155 tokens_per_expert=[512, 4736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=6090 actual_valid_recv=6272 observed_loss=-182 tokens_per_expert=[1792, 4480] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=11291 actual_valid_recv=11392 observed_loss=-101 tokens_per_expert=[11392, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=10294 actual_valid_recv=10368 observed_loss=-74 tokens_per_expert=[10368, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=5248 slack_slots=11136 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=6272 slack_slots=10112 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=11392 slack_slots=4992 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=10368 slack_slots=6016 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[2056, 147, 4088, 0, 27, 0, 1874, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[994, 1339, 4078, 0, 24, 4, 1753, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3591, 77, 4092, 1, 2, 0, 429, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[249, 6, 4039, 0, 180, 0, 3718, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[408, 1877, 4089, 4, 126, 4, 1684, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3096, 97, 3479, 0, 402, 5, 1055, 58] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[374, 3407, 4087, 0, 24, 0, 300, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1426, 396, 3955, 3, 563, 85, 1707, 57] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1914, 0, 147, 142, 1051, 0, 0, 1048, 27, 552, 0, 1037, 1322, 0, 0, 952] rerouted_tokens_per_ep_rank=[1914, 289, 1051, 1048, 579, 1037, 1322, 952] avg=1024.0000 max=1914 max_rank=0 min=289 min_rank=1 imbalance_ratio=1.869141 max_load_reduction=6278 max_load_reduction_pct=0.766357 moved_to_echo=3890 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3342, 0, 77, 249, 1052, 0, 1, 1049, 2, 126, 0, 1038, 303, 0, 0, 953] rerouted_tokens_per_ep_rank=[3342, 326, 1052, 1050, 128, 1038, 303, 953] avg=1024.0000 max=3342 max_rank=0 min=128 min_rank=4 imbalance_ratio=3.263672 max_load_reduction=4850 max_load_reduction_pct=0.592041 moved_to_echo=2422 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[232, 0, 6, 17, 1039, 0, 0, 1035, 180, 1095, 0, 1024, 2623, 0, 0, 941] rerouted_tokens_per_ep_rank=[232, 23, 1039, 1035, 1275, 1024, 2623, 941] avg=1024.0000 max=2623 max_rank=6 min=23 min_rank=1 imbalance_ratio=2.561523 max_load_reduction=5569 max_load_reduction_pct=0.679810 moved_to_echo=5863 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[926, 0, 1339, 68, 1049, 0, 0, 1045, 24, 516, 4, 1034, 1237, 0, 0, 950] rerouted_tokens_per_ep_rank=[926, 1407, 1049, 1045, 540, 1038, 1237, 950] avg=1024.0000 max=1407 max_rank=1 min=540 min_rank=4 imbalance_ratio=1.374023 max_load_reduction=6785 max_load_reduction_pct=0.828247 moved_to_echo=3765 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[375, 0, 1877, 33, 1038, 0, 4, 1053, 126, 499, 4, 1040, 1185, 0, 0, 958] rerouted_tokens_per_ep_rank=[375, 1910, 1038, 1057, 625, 1044, 1185, 958] avg=1024.0000 max=1910 max_rank=1 min=375 min_rank=0 imbalance_ratio=1.865234 max_load_reduction=6282 max_load_reduction_pct=0.766846 moved_to_echo=3812 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2882, 0, 97, 214, 895, 0, 0, 892, 402, 310, 5, 882, 745, 0, 58, 810] rerouted_tokens_per_ep_rank=[2882, 311, 895, 892, 712, 887, 745, 868] avg=1024.0000 max=2882 max_rank=0 min=311 min_rank=1 imbalance_ratio=2.814453 max_load_reduction=5310 max_load_reduction_pct=0.648193 moved_to_echo=3212 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[349, 0, 3407, 25, 1051, 0, 0, 1048, 24, 88, 0, 1036, 212, 0, 0, 952] rerouted_tokens_per_ep_rank=[349, 3432, 1051, 1048, 112, 1036, 212, 952] avg=1024.0000 max=3432 max_rank=1 min=112 min_rank=4 imbalance_ratio=3.351562 max_load_reduction=4760 max_load_reduction_pct=0.581055 moved_to_echo=2312 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1328, 0, 396, 98, 1017, 0, 3, 1014, 563, 502, 85, 1003, 1205, 0, 57, 921] rerouted_tokens_per_ep_rank=[1328, 494, 1017, 1017, 1065, 1088, 1205, 978] avg=1024.0000 max=1328 max_rank=0 min=494 min_rank=1 imbalance_ratio=1.296875 max_load_reduction=6864 max_load_reduction_pct=0.837891 moved_to_echo=4336 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11348, 8192, 8192, 8192, 5036, 8192, 8832, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11348, 8192, 8192, 8192, 5036, 8192, 8832, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11348 expected_overflow=0 exact_recv_per_rank=[11348, 8192, 8192, 8192, 5036, 8192, 8832, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11348, 8192, 8192, 8192, 5036, 8192, 8832, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7552 expected_overflow=0 exact_recv_per_rank=[11348, 8192, 8192, 8192, 5036, 8192, 8832, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5036 expected_overflow=0 exact_recv_per_rank=[11348, 8192, 8192, 8192, 5036, 8192, 8832, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11348, 8192, 8192, 8192, 5036, 8192, 8832, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8832 expected_overflow=0 exact_recv_per_rank=[11348, 8192, 8192, 8192, 5036, 8192, 8832, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=5036 actual_valid_recv=5120 observed_loss=-84 tokens_per_expert=[1408, 3712] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=7552 actual_valid_recv=7680 observed_loss=-128 tokens_per_expert=[128, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7424, 896] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8832 actual_valid_recv=8832 observed_loss=0 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=11348 actual_valid_recv=11392 observed_loss=-44 tokens_per_expert=[11392, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=5120 slack_slots=11264 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=11392 slack_slots=4992 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[3, 4091, 0, 10, 1, 3, 4066, 18] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[9, 1652, 0, 2755, 0, 0, 1404, 2372] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[20, 4057, 3, 2477, 0, 2, 1535, 98] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[8, 3714, 3, 529, 0, 45, 1988, 1905] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[3, 2076, 0, 2700, 1, 0, 3040, 372] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[160, 3833, 40, 64, 25, 10, 4037, 23] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[2, 3859, 0, 16, 0, 3, 4024, 288] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[290, 1182, 0, 2244, 0, 276, 3123, 1077] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[9, 431, 554, 0, 0, 547, 2235, 0, 0, 551, 0, 474, 499, 0, 2372, 520] rerouted_tokens_per_ep_rank=[440, 554, 547, 2235, 551, 474, 499, 2892] avg=1024.0000 max=2892 max_rank=7 min=440 min_rank=0 imbalance_ratio=2.824219 max_load_reduction=5300 max_load_reduction_pct=0.646973 moved_to_echo=4416 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[8, 610, 1245, 0, 3, 1230, 430, 0, 0, 1239, 45, 672, 706, 0, 1905, 99] rerouted_tokens_per_ep_rank=[618, 1245, 1233, 430, 1239, 717, 706, 2004] avg=1024.0000 max=2004 max_rank=7 min=430 min_rank=3 imbalance_ratio=1.957031 max_load_reduction=6188 max_load_reduction_pct=0.755371 moved_to_echo=4666 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 933, 697, 0, 0, 687, 2191, 0, 1, 692, 0, 1028, 1079, 0, 372, 509] rerouted_tokens_per_ep_rank=[936, 697, 687, 2191, 693, 1028, 1079, 881] avg=1024.0000 max=2191 max_rank=3 min=687 min_rank=2 imbalance_ratio=2.139648 max_load_reduction=6001 max_load_reduction_pct=0.732544 moved_to_echo=3681 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[20, 471, 1359, 0, 3, 1344, 2010, 0, 0, 1354, 2, 519, 545, 0, 98, 467] rerouted_tokens_per_ep_rank=[491, 1359, 1347, 2010, 1354, 521, 545, 565] avg=1024.0000 max=2010 max_rank=3 min=491 min_rank=0 imbalance_ratio=1.962891 max_load_reduction=6182 max_load_reduction_pct=0.754639 moved_to_echo=2985 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 1253, 1362, 0, 0, 1360, 4, 0, 1, 1369, 3, 1378, 1435, 0, 18, 6] rerouted_tokens_per_ep_rank=[1256, 1362, 1360, 4, 1370, 1381, 1435, 24] avg=1024.0000 max=1435 max_rank=6 min=4 min_rank=3 imbalance_ratio=1.401367 max_load_reduction=6757 max_load_reduction_pct=0.824829 moved_to_echo=4210 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[160, 1240, 1284, 0, 40, 1270, 52, 0, 25, 1279, 10, 1365, 1432, 0, 23, 12] rerouted_tokens_per_ep_rank=[1400, 1284, 1310, 52, 1304, 1375, 1432, 35] avg=1024.0000 max=1432 max_rank=6 min=35 min_rank=7 imbalance_ratio=1.398438 max_load_reduction=6760 max_load_reduction_pct=0.825195 moved_to_echo=4146 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 1236, 1294, 0, 0, 1278, 13, 0, 0, 1287, 3, 1361, 1427, 0, 288, 3] rerouted_tokens_per_ep_rank=[1238, 1294, 1278, 13, 1287, 1364, 1427, 291] avg=1024.0000 max=1427 max_rank=6 min=13 min_rank=3 imbalance_ratio=1.393555 max_load_reduction=6765 max_load_reduction_pct=0.825806 moved_to_echo=4369 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[290, 959, 397, 0, 0, 391, 1821, 0, 0, 394, 276, 1056, 1108, 0, 1077, 423] rerouted_tokens_per_ep_rank=[1249, 397, 391, 1821, 394, 1332, 1108, 1500] avg=1024.0000 max=1821 max_rank=3 min=391 min_rank=2 imbalance_ratio=1.778320 max_load_reduction=6371 max_load_reduction_pct=0.777710 moved_to_echo=4334 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8756 expected_overflow=0 exact_recv_per_rank=[7628, 8192, 8153, 8756, 8192, 8192, 8231, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7628, 8192, 8153, 8756, 8192, 8192, 8231, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8231 expected_overflow=0 exact_recv_per_rank=[7628, 8192, 8153, 8756, 8192, 8192, 8231, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7628 expected_overflow=0 exact_recv_per_rank=[7628, 8192, 8153, 8756, 8192, 8192, 8231, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7628, 8192, 8153, 8756, 8192, 8192, 8231, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7628, 8192, 8153, 8756, 8192, 8192, 8231, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7628, 8192, 8153, 8756, 8192, 8192, 8231, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8153 expected_overflow=0 exact_recv_per_rank=[7628, 8192, 8153, 8756, 8192, 8192, 8231, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=7628 actual_valid_recv=7680 observed_loss=-52 tokens_per_expert=[512, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6272, 2048] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8153 actual_valid_recv=8320 observed_loss=-167 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8756 actual_valid_recv=8832 observed_loss=-76 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8231 actual_valid_recv=8320 observed_loss=-89 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[2869, 15, 323, 4094, 0, 761, 2, 128] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[3824, 83, 8, 3209, 0, 1053, 0, 15] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1630, 1342, 642, 3798, 21, 759, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[2830, 369, 633, 3839, 3, 512, 0, 6] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1874, 941, 113, 3392, 91, 1492, 9, 280] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[387, 308, 2085, 3861, 0, 1481, 13, 57] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[2857, 884, 4, 3852, 0, 574, 4, 17] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1926, 2025, 35, 4064, 34, 56, 32, 20] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1597, 0, 15, 350, 323, 685, 1116, 0, 0, 1093, 761, 237, 2, 1105, 128, 780] rerouted_tokens_per_ep_rank=[1597, 365, 1008, 1116, 1093, 998, 1107, 908] avg=1024.0000 max=1597 max_rank=0 min=365 min_rank=1 imbalance_ratio=1.559570 max_load_reduction=6595 max_load_reduction_pct=0.805054 moved_to_echo=4106 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2128, 0, 83, 467, 8, 913, 875, 0, 0, 857, 1053, 316, 0, 866, 15, 611] rerouted_tokens_per_ep_rank=[2128, 550, 921, 875, 857, 1369, 866, 626] avg=1024.0000 max=2128 max_rank=0 min=550 min_rank=1 imbalance_ratio=2.078125 max_load_reduction=6064 max_load_reduction_pct=0.740234 moved_to_echo=3718 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1032, 0, 941, 232, 113, 452, 911, 0, 91, 910, 1492, 158, 9, 921, 280, 650] rerouted_tokens_per_ep_rank=[1032, 1173, 565, 911, 1001, 1650, 930, 930] avg=1024.0000 max=1650 max_rank=5 min=565 min_rank=2 imbalance_ratio=1.611328 max_load_reduction=6542 max_load_reduction_pct=0.798584 moved_to_echo=4511 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1575, 0, 369, 346, 633, 676, 1046, 0, 3, 1025, 512, 233, 0, 1036, 6, 732] rerouted_tokens_per_ep_rank=[1575, 715, 1309, 1046, 1028, 745, 1036, 738] avg=1024.0000 max=1575 max_rank=0 min=715 min_rank=1 imbalance_ratio=1.538086 max_load_reduction=6617 max_load_reduction_pct=0.807739 moved_to_echo=3547 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1590, 0, 884, 349, 4, 682, 1050, 0, 0, 1028, 574, 236, 4, 1040, 17, 734] rerouted_tokens_per_ep_rank=[1590, 1233, 686, 1050, 1028, 810, 1044, 751] avg=1024.0000 max=1590 max_rank=0 min=686 min_rank=2 imbalance_ratio=1.552734 max_load_reduction=6602 max_load_reduction_pct=0.805908 moved_to_echo=3633 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[908, 0, 1342, 199, 642, 389, 1035, 0, 21, 1014, 759, 134, 0, 1025, 0, 724] rerouted_tokens_per_ep_rank=[908, 1541, 1031, 1035, 1035, 893, 1025, 724] avg=1024.0000 max=1541 max_rank=1 min=724 min_rank=7 imbalance_ratio=1.504883 max_load_reduction=6651 max_load_reduction_pct=0.811890 moved_to_echo=3677 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[217, 0, 308, 47, 2085, 92, 1052, 0, 0, 1031, 1481, 31, 13, 1042, 57, 736] rerouted_tokens_per_ep_rank=[217, 355, 2177, 1052, 1031, 1512, 1055, 793] avg=1024.0000 max=2177 max_rank=2 min=217 min_rank=0 imbalance_ratio=2.125977 max_load_reduction=6015 max_load_reduction_pct=0.734253 moved_to_echo=4391 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1072, 0, 2025, 235, 35, 460, 1107, 0, 34, 1085, 56, 159, 32, 1097, 20, 775] rerouted_tokens_per_ep_rank=[1072, 2260, 495, 1107, 1119, 215, 1129, 795] avg=1024.0000 max=2260 max_rank=1 min=215 min_rank=5 imbalance_ratio=2.207031 max_load_reduction=5932 max_load_reduction_pct=0.724121 moved_to_echo=3258 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10119, 8192, 8192, 8192, 8192, 8192, 8192, 6265] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10119, 8192, 8192, 8192, 8192, 8192, 8192, 6265] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10119, 8192, 8192, 8192, 8192, 8192, 8192, 6265] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10119, 8192, 8192, 8192, 8192, 8192, 8192, 6265] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10119, 8192, 8192, 8192, 8192, 8192, 8192, 6265] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6265 expected_overflow=0 exact_recv_per_rank=[10119, 8192, 8192, 8192, 8192, 8192, 8192, 6265] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10119 expected_overflow=0 exact_recv_per_rank=[10119, 8192, 8192, 8192, 8192, 8192, 8192, 6265] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10119, 8192, 8192, 8192, 8192, 8192, 8192, 6265] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=6265 actual_valid_recv=6400 observed_loss=-135 tokens_per_expert=[640, 5760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6784, 1536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3968, 4352] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=10119 actual_valid_recv=10240 observed_loss=-121 tokens_per_expert=[10240, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6016, 2304] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=6400 slack_slots=9984 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=10240 slack_slots=6144 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[3947, 11, 1003, 3221, 3, 2, 4, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[2754, 377, 33, 2324, 663, 0, 0, 2041] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[3709, 8, 770, 2904, 116, 1, 5, 679] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1450, 1511, 638, 1227, 3359, 0, 0, 7] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1606, 24, 2774, 3036, 585, 0, 0, 167] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[3942, 61, 1470, 2509, 148, 0, 0, 62] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[315, 922, 1645, 1581, 2778, 0, 0, 951] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1090, 135, 837, 3797, 2203, 130, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2864, 0, 11, 1083, 1003, 0, 1275, 0, 0, 0, 2, 664, 4, 1282, 1, 3] rerouted_tokens_per_ep_rank=[2864, 1094, 1003, 1275, 0, 666, 1286, 4] avg=1024.0000 max=2864 max_rank=0 min=0 min_rank=4 imbalance_ratio=2.796875 max_load_reduction=5328 max_load_reduction_pct=0.650391 moved_to_echo=1956 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1054, 0, 1511, 396, 638, 0, 489, 0, 2793, 0, 0, 251, 0, 487, 7, 566] rerouted_tokens_per_ep_rank=[1054, 1907, 638, 489, 2793, 251, 487, 573] avg=1024.0000 max=2793 max_rank=4 min=251 min_rank=5 imbalance_ratio=2.727539 max_load_reduction=5399 max_load_reduction_pct=0.659058 moved_to_echo=4104 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2696, 0, 8, 1013, 770, 0, 1156, 0, 97, 0, 1, 595, 5, 1153, 679, 19] rerouted_tokens_per_ep_rank=[2696, 1021, 770, 1156, 97, 596, 1158, 698] avg=1024.0000 max=2696 max_rank=0 min=97 min_rank=4 imbalance_ratio=2.632812 max_load_reduction=5496 max_load_reduction_pct=0.670898 moved_to_echo=2549 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1167, 0, 24, 439, 2774, 0, 1208, 0, 487, 0, 0, 622, 0, 1206, 167, 98] rerouted_tokens_per_ep_rank=[1167, 463, 2774, 1208, 487, 622, 1206, 265] avg=1024.0000 max=2774 max_rank=2 min=265 min_rank=7 imbalance_ratio=2.708984 max_load_reduction=5418 max_load_reduction_pct=0.661377 moved_to_echo=2580 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2002, 0, 377, 752, 33, 0, 925, 0, 549, 0, 0, 476, 0, 923, 2041, 114] rerouted_tokens_per_ep_rank=[2002, 1129, 33, 925, 549, 476, 923, 2155] avg=1024.0000 max=2155 max_rank=7 min=33 min_rank=2 imbalance_ratio=2.104492 max_load_reduction=6037 max_load_reduction_pct=0.736938 moved_to_echo=4103 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2865, 0, 61, 1077, 1470, 0, 999, 0, 124, 0, 0, 514, 0, 996, 62, 24] rerouted_tokens_per_ep_rank=[2865, 1138, 1470, 999, 124, 514, 996, 86] avg=1024.0000 max=2865 max_rank=0 min=86 min_rank=7 imbalance_ratio=2.797852 max_load_reduction=5327 max_load_reduction_pct=0.650269 moved_to_echo=1720 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[229, 0, 922, 86, 1645, 0, 629, 0, 2310, 0, 0, 324, 0, 628, 951, 468] rerouted_tokens_per_ep_rank=[229, 1008, 1645, 629, 2310, 324, 628, 1419] avg=1024.0000 max=2310 max_rank=4 min=229 min_rank=0 imbalance_ratio=2.255859 max_load_reduction=5882 max_load_reduction_pct=0.718018 moved_to_echo=4681 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[793, 0, 135, 297, 837, 0, 1511, 0, 1832, 0, 130, 778, 0, 1508, 0, 371] rerouted_tokens_per_ep_rank=[793, 432, 837, 1511, 1832, 908, 1508, 371] avg=1024.0000 max=1832 max_rank=4 min=371 min_rank=7 imbalance_ratio=1.789062 max_load_reduction=6360 max_load_reduction_pct=0.776367 moved_to_echo=4619 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9170 expected_overflow=0 exact_recv_per_rank=[13670, 8192, 9170, 8192, 8192, 4357, 8192, 5571] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=13670 expected_overflow=0 exact_recv_per_rank=[13670, 8192, 9170, 8192, 8192, 4357, 8192, 5571] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4357 expected_overflow=0 exact_recv_per_rank=[13670, 8192, 9170, 8192, 8192, 4357, 8192, 5571] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[13670, 8192, 9170, 8192, 8192, 4357, 8192, 5571] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[13670, 8192, 9170, 8192, 8192, 4357, 8192, 5571] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[13670, 8192, 9170, 8192, 8192, 4357, 8192, 5571] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5571 expected_overflow=0 exact_recv_per_rank=[13670, 8192, 9170, 8192, 8192, 4357, 8192, 5571] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[13670, 8192, 9170, 8192, 8192, 4357, 8192, 5571] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=4357 actual_valid_recv=4480 observed_loss=-123 tokens_per_expert=[256, 4224] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=5571 actual_valid_recv=5632 observed_loss=-61 tokens_per_expert=[3968, 1664] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3072, 5248] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=9170 actual_valid_recv=9216 observed_loss=-46 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=13670 actual_valid_recv=13696 observed_loss=-26 tokens_per_expert=[13696, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=4480 slack_slots=11904 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=5632 slack_slots=10752 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=13696 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[870, 0, 249, 2978, 2, 4090, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[3057, 414, 0, 731, 0, 3985, 5, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[115, 3, 18, 3835, 2, 4094, 9, 116] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4, 4, 3, 3994, 6, 4096, 69, 16] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[38, 0, 3, 4022, 14, 4047, 55, 13] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[34, 2, 314, 3364, 350, 4071, 2, 55] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[123, 0, 0, 3883, 4, 4096, 75, 11] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[11, 20, 152, 2493, 126, 2613, 1060, 1717] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[870, 463, 0, 1019, 249, 980, 966, 0, 2, 1011, 1080, 0, 3, 812, 0, 737] rerouted_tokens_per_ep_rank=[1333, 1019, 1229, 966, 1013, 1080, 815, 737] avg=1024.0000 max=1333 max_rank=0 min=737 min_rank=7 imbalance_ratio=1.301758 max_load_reduction=6859 max_load_reduction_pct=0.837280 moved_to_echo=3645 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3057, 113, 414, 993, 0, 955, 239, 0, 0, 985, 1052, 0, 5, 199, 0, 180] rerouted_tokens_per_ep_rank=[3170, 1407, 955, 239, 985, 1052, 204, 180] avg=1024.0000 max=3170 max_rank=0 min=180 min_rank=7 imbalance_ratio=3.095703 max_load_reduction=5022 max_load_reduction_pct=0.613037 moved_to_echo=2421 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4, 621, 4, 1020, 3, 981, 1296, 0, 6, 1012, 1083, 0, 69, 1089, 16, 988] rerouted_tokens_per_ep_rank=[625, 1024, 984, 1296, 1018, 1083, 1158, 1004] avg=1024.0000 max=1296 max_rank=3 min=625 min_rank=0 imbalance_ratio=1.265625 max_load_reduction=6896 max_load_reduction_pct=0.841797 moved_to_echo=4263 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[115, 597, 3, 1020, 18, 981, 1243, 0, 2, 1012, 1081, 0, 9, 1046, 116, 949] rerouted_tokens_per_ep_rank=[712, 1023, 999, 1243, 1014, 1081, 1055, 1065] avg=1024.0000 max=1243 max_rank=3 min=712 min_rank=0 imbalance_ratio=1.213867 max_load_reduction=6949 max_load_reduction_pct=0.848267 moved_to_echo=4215 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[38, 631, 0, 1012, 3, 974, 1289, 0, 14, 1004, 1057, 0, 55, 1102, 13, 1000] rerouted_tokens_per_ep_rank=[669, 1012, 977, 1289, 1018, 1057, 1157, 1013] avg=1024.0000 max=1289 max_rank=3 min=669 min_rank=0 imbalance_ratio=1.258789 max_load_reduction=6903 max_load_reduction_pct=0.842651 moved_to_echo=4245 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[34, 523, 2, 1014, 314, 975, 1092, 0, 350, 1006, 1076, 0, 2, 917, 55, 832] rerouted_tokens_per_ep_rank=[557, 1016, 1289, 1092, 1356, 1076, 919, 887] avg=1024.0000 max=1356 max_rank=4 min=557 min_rank=0 imbalance_ratio=1.324219 max_load_reduction=6836 max_load_reduction_pct=0.834473 moved_to_echo=4238 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[123, 604, 0, 1020, 0, 981, 1259, 0, 4, 1012, 1083, 0, 75, 1059, 11, 961] rerouted_tokens_per_ep_rank=[727, 1020, 981, 1259, 1016, 1083, 1134, 972] avg=1024.0000 max=1259 max_rank=3 min=727 min_rank=0 imbalance_ratio=1.229492 max_load_reduction=6933 max_load_reduction_pct=0.846313 moved_to_echo=4205 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[11, 388, 20, 651, 152, 626, 808, 0, 126, 646, 690, 0, 1060, 680, 1717, 617] rerouted_tokens_per_ep_rank=[399, 671, 778, 808, 772, 690, 1740, 2334] avg=1024.0000 max=2334 max_rank=7 min=399 min_rank=0 imbalance_ratio=2.279297 max_load_reduction=5858 max_load_reduction_pct=0.715088 moved_to_echo=5536 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8202, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8202, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8202, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8202, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8202, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8202 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8202, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8202, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8182 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8202, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4352, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2048, 6272] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8182 actual_valid_recv=8192 observed_loss=-10 tokens_per_expert=[1280, 6912] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8202 actual_valid_recv=8320 observed_loss=-118 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[182, 898, 1332, 408, 1499, 541, 1499, 1833] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[40, 621, 1518, 145, 1421, 743, 1681, 2023] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[110, 316, 729, 346, 2025, 766, 1449, 2451] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[36, 1009, 1123, 503, 1077, 938, 1816, 1690] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[105, 458, 1003, 358, 2186, 619, 1373, 2090] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[65, 573, 946, 491, 1575, 737, 1887, 1918] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[54, 700, 1356, 344, 1152, 712, 1862, 2012] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[57, 319, 995, 268, 1988, 908, 2134, 1523] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[182, 866, 898, 382, 1332, 0, 408, 581, 953, 0, 541, 164, 918, 0, 967, 0] rerouted_tokens_per_ep_rank=[1048, 1280, 1332, 989, 953, 705, 918, 967] avg=1024.0000 max=1332 max_rank=2 min=705 min_rank=5 imbalance_ratio=1.300781 max_load_reduction=6860 max_load_reduction_pct=0.837402 moved_to_echo=3543 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[40, 956, 621, 362, 1518, 0, 145, 651, 904, 0, 743, 155, 1030, 0, 1067, 0] rerouted_tokens_per_ep_rank=[996, 983, 1518, 796, 904, 898, 1030, 1067] avg=1024.0000 max=1518 max_rank=2 min=796 min_rank=3 imbalance_ratio=1.482422 max_load_reduction=6674 max_load_reduction_pct=0.814697 moved_to_echo=3899 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[110, 1158, 316, 516, 729, 0, 346, 562, 1287, 0, 766, 222, 887, 0, 1293, 0] rerouted_tokens_per_ep_rank=[1268, 832, 729, 908, 1287, 988, 887, 1293] avg=1024.0000 max=1293 max_rank=7 min=729 min_rank=2 imbalance_ratio=1.262695 max_load_reduction=6899 max_load_reduction_pct=0.842163 moved_to_echo=4455 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[105, 988, 458, 557, 1003, 0, 358, 532, 1390, 0, 619, 239, 841, 0, 1102, 0] rerouted_tokens_per_ep_rank=[1093, 1015, 1003, 890, 1390, 858, 841, 1102] avg=1024.0000 max=1390 max_rank=4 min=841 min_rank=6 imbalance_ratio=1.357422 max_load_reduction=6802 max_load_reduction_pct=0.830322 moved_to_echo=4191 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[36, 803, 1009, 280, 1123, 0, 503, 708, 675, 0, 938, 122, 1108, 0, 887, 0] rerouted_tokens_per_ep_rank=[839, 1289, 1123, 1211, 675, 1060, 1108, 887] avg=1024.0000 max=1289 max_rank=1 min=675 min_rank=4 imbalance_ratio=1.258789 max_load_reduction=6903 max_load_reduction_pct=0.842651 moved_to_echo=3730 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[65, 906, 573, 401, 946, 0, 491, 731, 1002, 0, 737, 172, 1156, 0, 1012, 0] rerouted_tokens_per_ep_rank=[971, 974, 946, 1222, 1002, 909, 1156, 1012] avg=1024.0000 max=1222 max_rank=3 min=909 min_rank=5 imbalance_ratio=1.193359 max_load_reduction=6970 max_load_reduction_pct=0.850830 moved_to_echo=4079 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[54, 951, 700, 293, 1356, 0, 344, 722, 733, 0, 712, 126, 1140, 0, 1061, 0] rerouted_tokens_per_ep_rank=[1005, 993, 1356, 1066, 733, 838, 1140, 1061] avg=1024.0000 max=1356 max_rank=2 min=733 min_rank=4 imbalance_ratio=1.324219 max_load_reduction=6836 max_load_reduction_pct=0.834473 moved_to_echo=3772 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[57, 720, 319, 507, 995, 0, 268, 827, 1263, 0, 908, 218, 1307, 0, 803, 0] rerouted_tokens_per_ep_rank=[777, 826, 995, 1095, 1263, 1126, 1307, 803] avg=1024.0000 max=1307 max_rank=6 min=777 min_rank=0 imbalance_ratio=1.276367 max_load_reduction=6885 max_load_reduction_pct=0.840454 moved_to_echo=4499 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9002 expected_overflow=0 exact_recv_per_rank=[7997, 8192, 9002, 8177, 8207, 7382, 8387, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8207 expected_overflow=0 exact_recv_per_rank=[7997, 8192, 9002, 8177, 8207, 7382, 8387, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7997 expected_overflow=0 exact_recv_per_rank=[7997, 8192, 9002, 8177, 8207, 7382, 8387, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7997, 8192, 9002, 8177, 8207, 7382, 8387, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8177 expected_overflow=0 exact_recv_per_rank=[7997, 8192, 9002, 8177, 8207, 7382, 8387, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7382 expected_overflow=0 exact_recv_per_rank=[7997, 8192, 9002, 8177, 8207, 7382, 8387, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8387 expected_overflow=0 exact_recv_per_rank=[7997, 8192, 9002, 8177, 8207, 7382, 8387, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7997, 8192, 9002, 8177, 8207, 7382, 8387, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=7997 actual_valid_recv=8192 observed_loss=-195 tokens_per_expert=[768, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4992, 3328] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8177 actual_valid_recv=8320 observed_loss=-143 tokens_per_expert=[2944, 5376] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8387 actual_valid_recv=8448 observed_loss=-61 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7382 actual_valid_recv=7552 observed_loss=-170 tokens_per_expert=[6016, 1536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8207 actual_valid_recv=8320 observed_loss=-113 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=9002 actual_valid_recv=9088 observed_loss=-86 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1000, 2478, 210, 3790, 215, 327, 106, 66] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1744, 1681, 55, 3832, 311, 235, 201, 133] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1422, 2233, 144, 3897, 197, 119, 93, 87] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[803, 1660, 340, 3972, 372, 439, 562, 44] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1203, 2402, 429, 3594, 206, 121, 137, 100] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1907, 1257, 664, 3399, 279, 124, 289, 273] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1011, 2347, 559, 3396, 137, 128, 271, 343] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1508, 1610, 140, 3855, 210, 547, 199, 123] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1000, 0, 1296, 0, 210, 513, 1290, 0, 215, 798, 327, 669, 106, 807, 66, 895] rerouted_tokens_per_ep_rank=[1000, 1296, 723, 1290, 1013, 996, 913, 961] avg=1024.0000 max=1296 max_rank=1 min=723 min_rank=2 imbalance_ratio=1.265625 max_load_reduction=6896 max_load_reduction_pct=0.841797 moved_to_echo=3883 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1744, 0, 880, 0, 55, 348, 1304, 0, 311, 807, 235, 453, 201, 816, 133, 905] rerouted_tokens_per_ep_rank=[1744, 880, 403, 1304, 1118, 688, 1017, 1038] avg=1024.0000 max=1744 max_rank=0 min=403 min_rank=2 imbalance_ratio=1.703125 max_load_reduction=6448 max_load_reduction_pct=0.787109 moved_to_echo=3861 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[803, 0, 869, 0, 340, 343, 1352, 0, 372, 836, 439, 448, 562, 846, 44, 938] rerouted_tokens_per_ep_rank=[803, 869, 683, 1352, 1208, 887, 1408, 982] avg=1024.0000 max=1408 max_rank=6 min=683 min_rank=2 imbalance_ratio=1.375000 max_load_reduction=6784 max_load_reduction_pct=0.828125 moved_to_echo=4485 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1203, 0, 1257, 0, 429, 497, 1224, 0, 206, 757, 121, 648, 137, 765, 100, 848] rerouted_tokens_per_ep_rank=[1203, 1257, 926, 1224, 963, 769, 902, 948] avg=1024.0000 max=1257 max_rank=1 min=769 min_rank=5 imbalance_ratio=1.227539 max_load_reduction=6935 max_load_reduction_pct=0.846558 moved_to_echo=3582 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1907, 0, 651, 0, 664, 263, 1149, 0, 279, 719, 124, 343, 289, 726, 273, 805] rerouted_tokens_per_ep_rank=[1907, 651, 927, 1149, 998, 467, 1015, 1078] avg=1024.0000 max=1907 max_rank=0 min=467 min_rank=5 imbalance_ratio=1.862305 max_load_reduction=6285 max_load_reduction_pct=0.767212 moved_to_echo=3558 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1011, 0, 1228, 0, 559, 486, 1156, 0, 137, 715, 128, 633, 271, 723, 343, 802] rerouted_tokens_per_ep_rank=[1011, 1228, 1045, 1156, 852, 761, 994, 1145] avg=1024.0000 max=1228 max_rank=1 min=761 min_rank=5 imbalance_ratio=1.199219 max_load_reduction=6964 max_load_reduction_pct=0.850098 moved_to_echo=3752 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1422, 0, 1168, 0, 144, 462, 1326, 0, 197, 821, 119, 603, 93, 830, 87, 920] rerouted_tokens_per_ep_rank=[1422, 1168, 606, 1326, 1018, 722, 923, 1007] avg=1024.0000 max=1422 max_rank=0 min=606 min_rank=2 imbalance_ratio=1.388672 max_load_reduction=6770 max_load_reduction_pct=0.826416 moved_to_echo=3670 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1508, 0, 843, 0, 140, 333, 1312, 0, 210, 812, 547, 434, 199, 821, 123, 910] rerouted_tokens_per_ep_rank=[1508, 843, 473, 1312, 1022, 981, 1020, 1033] avg=1024.0000 max=1508 max_rank=0 min=473 min_rank=2 imbalance_ratio=1.472656 max_load_reduction=6684 max_load_reduction_pct=0.815918 moved_to_echo=4056 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10113 expected_overflow=0 exact_recv_per_rank=[10598, 8192, 5786, 10113, 8192, 6271, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5786 expected_overflow=0 exact_recv_per_rank=[10598, 8192, 5786, 10113, 8192, 6271, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10598, 8192, 5786, 10113, 8192, 6271, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10598 expected_overflow=0 exact_recv_per_rank=[10598, 8192, 5786, 10113, 8192, 6271, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10598, 8192, 5786, 10113, 8192, 6271, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10598, 8192, 5786, 10113, 8192, 6271, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10598, 8192, 5786, 10113, 8192, 6271, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6271 expected_overflow=0 exact_recv_per_rank=[10598, 8192, 5786, 10113, 8192, 6271, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=5786 actual_valid_recv=5888 observed_loss=-102 tokens_per_expert=[2560, 3328] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=6271 actual_valid_recv=6400 observed_loss=-129 tokens_per_expert=[2048, 4352] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1280, 7040] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1920, 6400] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2048, 6272] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=10598 actual_valid_recv=10624 observed_loss=-26 tokens_per_expert=[10624, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=10113 actual_valid_recv=10240 observed_loss=-127 tokens_per_expert=[10240, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=5888 slack_slots=10496 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=6400 slack_slots=9984 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=10624 slack_slots=5760 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=10240 slack_slots=6144 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[204, 355, 55, 476, 1533, 187, 2391, 2991] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[609, 229, 659, 102, 658, 135, 3306, 2494] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[331, 66, 31, 204, 1327, 134, 2357, 3742] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[18, 178, 29, 18, 537, 32, 3521, 3859] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[302, 63, 187, 17, 1284, 1477, 1270, 3592] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[107, 525, 79, 165, 1139, 104, 2256, 3817] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[111, 44, 30, 59, 879, 23, 3183, 3863] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[66, 104, 52, 77, 1994, 106, 1906, 3887] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[204, 763, 355, 625, 55, 748, 476, 749, 1533, 0, 187, 572, 1056, 0, 869, 0] rerouted_tokens_per_ep_rank=[967, 980, 803, 1225, 1533, 759, 1056, 869] avg=1024.0000 max=1533 max_rank=4 min=759 min_rank=5 imbalance_ratio=1.497070 max_load_reduction=6659 max_load_reduction_pct=0.812866 moved_to_echo=4217 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[609, 1058, 229, 525, 659, 629, 102, 628, 658, 0, 135, 794, 1454, 0, 712, 0] rerouted_tokens_per_ep_rank=[1667, 754, 1288, 730, 658, 929, 1454, 712] avg=1024.0000 max=1667 max_rank=0 min=658 min_rank=4 imbalance_ratio=1.627930 max_load_reduction=6525 max_load_reduction_pct=0.796509 moved_to_echo=3753 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[18, 1123, 178, 807, 29, 965, 18, 966, 537, 0, 32, 843, 1555, 0, 1121, 0] rerouted_tokens_per_ep_rank=[1141, 985, 994, 984, 537, 875, 1555, 1121] avg=1024.0000 max=1555 max_rank=6 min=537 min_rank=4 imbalance_ratio=1.518555 max_load_reduction=6637 max_load_reduction_pct=0.810181 moved_to_echo=4088 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[302, 405, 63, 751, 187, 899, 17, 899, 1284, 0, 1477, 304, 561, 0, 1043, 0] rerouted_tokens_per_ep_rank=[707, 814, 1086, 916, 1284, 1781, 561, 1043] avg=1024.0000 max=1781 max_rank=5 min=561 min_rank=6 imbalance_ratio=1.739258 max_load_reduction=6411 max_load_reduction_pct=0.782593 moved_to_echo=4669 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[331, 752, 66, 782, 31, 936, 204, 937, 1327, 0, 134, 564, 1041, 0, 1087, 0] rerouted_tokens_per_ep_rank=[1083, 848, 967, 1141, 1327, 698, 1041, 1087] avg=1024.0000 max=1327 max_rank=4 min=698 min_rank=5 imbalance_ratio=1.295898 max_load_reduction=6865 max_load_reduction_pct=0.838013 moved_to_echo=4153 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[111, 1015, 44, 808, 30, 966, 59, 967, 879, 0, 23, 762, 1406, 0, 1122, 0] rerouted_tokens_per_ep_rank=[1126, 852, 996, 1026, 879, 785, 1406, 1122] avg=1024.0000 max=1406 max_rank=6 min=785 min_rank=5 imbalance_ratio=1.373047 max_load_reduction=6786 max_load_reduction_pct=0.828369 moved_to_echo=4192 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[107, 720, 525, 798, 79, 955, 165, 955, 1139, 0, 104, 540, 996, 0, 1109, 0] rerouted_tokens_per_ep_rank=[827, 1323, 1034, 1120, 1139, 644, 996, 1109] avg=1024.0000 max=1323 max_rank=1 min=644 min_rank=5 imbalance_ratio=1.291992 max_load_reduction=6869 max_load_reduction_pct=0.838501 moved_to_echo=3888 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[66, 608, 104, 813, 52, 972, 77, 973, 1994, 0, 106, 456, 842, 0, 1129, 0] rerouted_tokens_per_ep_rank=[674, 917, 1024, 1050, 1994, 562, 842, 1129] avg=1024.0000 max=1994 max_rank=4 min=562 min_rank=5 imbalance_ratio=1.947266 max_load_reduction=6198 max_load_reduction_pct=0.756592 moved_to_echo=4527 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7473, 8192, 8192, 9351, 7033, 8911, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7473, 8192, 8192, 9351, 7033, 8911, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7473 expected_overflow=0 exact_recv_per_rank=[8192, 7473, 8192, 8192, 9351, 7033, 8911, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7033 expected_overflow=0 exact_recv_per_rank=[8192, 7473, 8192, 8192, 9351, 7033, 8911, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9351 expected_overflow=0 exact_recv_per_rank=[8192, 7473, 8192, 8192, 9351, 7033, 8911, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7473, 8192, 8192, 9351, 7033, 8911, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7473, 8192, 8192, 9351, 7033, 8911, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8911 expected_overflow=0 exact_recv_per_rank=[8192, 7473, 8192, 8192, 9351, 7033, 8911, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7033 actual_valid_recv=7168 observed_loss=-135 tokens_per_expert=[2304, 4864] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1792, 6528] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=7473 actual_valid_recv=7680 observed_loss=-207 tokens_per_expert=[1664, 6016] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8911 actual_valid_recv=8960 observed_loss=-49 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=9351 actual_valid_recv=9472 observed_loss=-121 tokens_per_expert=[9472, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7168 slack_slots=9216 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=9472 slack_slots=6912 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1694, 73, 1564, 2, 73, 1659, 2365, 762] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1162, 186, 1274, 0, 154, 3095, 2274, 47] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[255, 6, 894, 25, 1340, 1446, 3792, 434] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[673, 11, 1119, 2, 485, 2425, 3046, 431] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[435, 61, 634, 1036, 1804, 546, 3069, 607] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1255, 36, 788, 12, 185, 3056, 2741, 119] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[218, 7, 202, 3, 2423, 2070, 1860, 1409] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[871, 282, 1428, 23, 78, 1694, 3797, 19] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1694, 169, 73, 776, 1564, 29, 2, 730, 73, 171, 852, 0, 859, 0, 762, 438] rerouted_tokens_per_ep_rank=[1863, 849, 1593, 732, 244, 852, 859, 1200] avg=1024.0000 max=1863 max_rank=0 min=244 min_rank=4 imbalance_ratio=1.819336 max_load_reduction=6329 max_load_reduction_pct=0.772583 moved_to_echo=3155 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1162, 318, 186, 749, 1274, 60, 0, 706, 154, 322, 1572, 0, 819, 0, 47, 823] rerouted_tokens_per_ep_rank=[1480, 935, 1334, 706, 476, 1572, 819, 870] avg=1024.0000 max=1572 max_rank=5 min=476 min_rank=4 imbalance_ratio=1.535156 max_load_reduction=6620 max_load_reduction_pct=0.808105 moved_to_echo=3737 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[435, 55, 61, 1007, 634, 9, 1036, 948, 1804, 56, 282, 0, 1114, 0, 607, 144] rerouted_tokens_per_ep_rank=[490, 1068, 643, 1984, 1860, 282, 1114, 751] avg=1024.0000 max=1984 max_rank=3 min=282 min_rank=5 imbalance_ratio=1.937500 max_load_reduction=6208 max_load_reduction_pct=0.757812 moved_to_echo=4007 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[255, 147, 6, 1244, 894, 26, 25, 1171, 1340, 149, 742, 0, 1377, 0, 434, 382] rerouted_tokens_per_ep_rank=[402, 1250, 920, 1196, 1489, 742, 1377, 816] avg=1024.0000 max=1489 max_rank=4 min=402 min_rank=0 imbalance_ratio=1.454102 max_load_reduction=6703 max_load_reduction_pct=0.818237 moved_to_echo=4424 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[218, 210, 7, 610, 202, 37, 3, 574, 2423, 213, 1063, 0, 676, 0, 1409, 547] rerouted_tokens_per_ep_rank=[428, 617, 239, 577, 2636, 1063, 676, 1956] avg=1024.0000 max=2636 max_rank=4 min=239 min_rank=2 imbalance_ratio=2.574219 max_load_reduction=5556 max_load_reduction_pct=0.678223 moved_to_echo=6331 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1255, 311, 36, 899, 788, 55, 12, 846, 185, 315, 1567, 0, 996, 0, 119, 808] rerouted_tokens_per_ep_rank=[1566, 935, 843, 858, 500, 1567, 996, 927] avg=1024.0000 max=1567 max_rank=5 min=500 min_rank=4 imbalance_ratio=1.530273 max_load_reduction=6625 max_load_reduction_pct=0.808716 moved_to_echo=3990 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[673, 247, 11, 999, 1119, 43, 2, 941, 485, 250, 1244, 0, 1106, 0, 431, 641] rerouted_tokens_per_ep_rank=[920, 1010, 1162, 943, 735, 1244, 1106, 1072] avg=1024.0000 max=1244 max_rank=5 min=735 min_rank=4 imbalance_ratio=1.214844 max_load_reduction=6948 max_load_reduction_pct=0.848145 moved_to_echo=4157 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[871, 172, 282, 1246, 1428, 30, 23, 1173, 78, 174, 870, 0, 1378, 0, 19, 448] rerouted_tokens_per_ep_rank=[1043, 1528, 1458, 1196, 252, 870, 1378, 467] avg=1024.0000 max=1528 max_rank=1 min=252 min_rank=4 imbalance_ratio=1.492188 max_load_reduction=6664 max_load_reduction_pct=0.813477 moved_to_echo=2967 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8325, 8059] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8325, 8059] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8325 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8325, 8059] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8325, 8059] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8325, 8059] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8059 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8325, 8059] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8325, 8059] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8325, 8059] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6656, 1664] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6656, 1664] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8059 actual_valid_recv=8192 observed_loss=-133 tokens_per_expert=[3840, 4352] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7936, 384] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8325 actual_valid_recv=8448 observed_loss=-123 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[240, 512, 523, 3741, 0, 10, 3157, 9] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[565, 448, 2752, 617, 134, 128, 3445, 103] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[236, 364, 825, 3145, 0, 0, 3591, 31] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[130, 342, 1231, 2832, 65, 21, 3569, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[314, 525, 1207, 3654, 1, 48, 2417, 26] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[318, 896, 343, 2778, 3, 81, 3768, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[315, 393, 756, 3585, 0, 0, 3048, 95] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[34, 215, 182, 4012, 0, 2, 3733, 14] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[240, 927, 512, 690, 523, 57, 1260, 0, 0, 943, 10, 807, 1281, 0, 9, 933] rerouted_tokens_per_ep_rank=[1167, 1202, 580, 1260, 943, 817, 1281, 942] avg=1024.0000 max=1281 max_rank=6 min=580 min_rank=2 imbalance_ratio=1.250977 max_load_reduction=6911 max_load_reduction_pct=0.843628 moved_to_echo=3983 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[565, 157, 448, 118, 2752, 13, 192, 0, 134, 1033, 128, 137, 1389, 0, 103, 1023] rerouted_tokens_per_ep_rank=[722, 566, 2765, 192, 1167, 265, 1389, 1126] avg=1024.0000 max=2765 max_rank=2 min=192 min_rank=3 imbalance_ratio=2.700195 max_load_reduction=5427 max_load_reduction_pct=0.662476 moved_to_echo=3947 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[314, 905, 525, 674, 1207, 55, 1231, 0, 1, 722, 48, 789, 980, 0, 26, 715] rerouted_tokens_per_ep_rank=[1219, 1199, 1262, 1231, 723, 837, 980, 741] avg=1024.0000 max=1262 max_rank=2 min=723 min_rank=4 imbalance_ratio=1.232422 max_load_reduction=6930 max_load_reduction_pct=0.845947 moved_to_echo=3281 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[130, 702, 342, 522, 1231, 43, 954, 0, 65, 1066, 21, 611, 1448, 0, 2, 1055] rerouted_tokens_per_ep_rank=[832, 864, 1274, 954, 1131, 632, 1448, 1057] avg=1024.0000 max=1448 max_rank=6 min=632 min_rank=5 imbalance_ratio=1.414062 max_load_reduction=6744 max_load_reduction_pct=0.823242 moved_to_echo=4268 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[236, 779, 364, 580, 825, 48, 1059, 0, 0, 1073, 0, 679, 1456, 0, 31, 1062] rerouted_tokens_per_ep_rank=[1015, 944, 873, 1059, 1073, 679, 1456, 1093] avg=1024.0000 max=1456 max_rank=6 min=679 min_rank=5 imbalance_ratio=1.421875 max_load_reduction=6736 max_load_reduction_pct=0.822266 moved_to_echo=4301 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[315, 888, 393, 661, 756, 54, 1208, 0, 0, 911, 0, 774, 1236, 0, 95, 901] rerouted_tokens_per_ep_rank=[1203, 1054, 810, 1208, 911, 774, 1236, 996] avg=1024.0000 max=1236 max_rank=6 min=774 min_rank=5 imbalance_ratio=1.207031 max_load_reduction=6956 max_load_reduction_pct=0.849121 moved_to_echo=3917 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[318, 688, 896, 512, 343, 42, 937, 0, 3, 1126, 81, 599, 1528, 0, 5, 1114] rerouted_tokens_per_ep_rank=[1006, 1408, 385, 937, 1129, 680, 1528, 1119] avg=1024.0000 max=1528 max_rank=6 min=385 min_rank=2 imbalance_ratio=1.492188 max_load_reduction=6664 max_load_reduction_pct=0.813477 moved_to_echo=4456 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[34, 994, 215, 740, 182, 61, 1351, 0, 0, 1115, 2, 866, 1514, 0, 14, 1104] rerouted_tokens_per_ep_rank=[1028, 955, 243, 1351, 1115, 868, 1514, 1118] avg=1024.0000 max=1514 max_rank=6 min=243 min_rank=2 imbalance_ratio=1.478516 max_load_reduction=6678 max_load_reduction_pct=0.815186 moved_to_echo=4615 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5552, 10832, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5552 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5552, 10832, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5552, 10832, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5552, 10832, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5552, 10832, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5552, 10832, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10832 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5552, 10832, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5552, 10832, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=5552 actual_valid_recv=5760 observed_loss=-208 tokens_per_expert=[384, 5376] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2176, 6144] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3712, 4608] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7936, 384] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=10832 actual_valid_recv=10880 observed_loss=-48 tokens_per_expert=[10880, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=5760 slack_slots=10624 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=10880 slack_slots=5504 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2, 4070, 376, 24, 0, 3719, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[157, 3380, 20, 1018, 24, 3586, 7, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4, 4082, 0, 85, 5, 4016, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[76, 4074, 183, 32, 1, 3733, 93, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[117, 4051, 9, 389, 0, 3591, 35, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1, 4096, 5, 3, 0, 4087, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[76, 4044, 34, 0, 5, 4030, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 10, 204, 0, 3881, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 941, 1047, 0, 376, 916, 24, 781, 0, 1040, 1081, 0, 0, 938, 1, 1045] rerouted_tokens_per_ep_rank=[943, 1047, 1292, 805, 1040, 1081, 938, 1046] avg=1024.0000 max=1292 max_rank=2 min=805 min_rank=3 imbalance_ratio=1.261719 max_load_reduction=6900 max_load_reduction_pct=0.842285 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[117, 909, 1042, 0, 9, 885, 389, 754, 0, 1036, 1043, 0, 35, 933, 0, 1040] rerouted_tokens_per_ep_rank=[1026, 1042, 894, 1143, 1036, 1043, 968, 1040] avg=1024.0000 max=1143 max_rank=3 min=894 min_rank=2 imbalance_ratio=1.116211 max_load_reduction=7049 max_load_reduction_pct=0.860474 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 1034, 1054, 0, 5, 1007, 3, 858, 0, 1047, 1188, 0, 0, 944, 0, 1051] rerouted_tokens_per_ep_rank=[1035, 1054, 1012, 861, 1047, 1188, 944, 1051] avg=1024.0000 max=1188 max_rank=5 min=861 min_rank=3 imbalance_ratio=1.160156 max_load_reduction=7004 max_load_reduction_pct=0.854980 moved_to_echo=4230 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[157, 912, 858, 0, 20, 888, 1018, 756, 24, 868, 1030, 0, 7, 782, 0, 872] rerouted_tokens_per_ep_rank=[1069, 858, 908, 1774, 892, 1030, 789, 872] avg=1024.0000 max=1774 max_rank=3 min=789 min_rank=6 imbalance_ratio=1.732422 max_load_reduction=6418 max_load_reduction_pct=0.783447 moved_to_echo=3583 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4, 1016, 1049, 0, 0, 990, 85, 843, 5, 1044, 1167, 0, 0, 941, 0, 1048] rerouted_tokens_per_ep_rank=[1020, 1049, 990, 928, 1049, 1167, 941, 1048] avg=1024.0000 max=1167 max_rank=5 min=928 min_rank=3 imbalance_ratio=1.139648 max_load_reduction=7025 max_load_reduction_pct=0.857544 moved_to_echo=4205 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[76, 1020, 1040, 0, 34, 993, 0, 846, 5, 1034, 1171, 0, 3, 932, 0, 1038] rerouted_tokens_per_ep_rank=[1096, 1040, 1027, 846, 1039, 1171, 935, 1038] avg=1024.0000 max=1171 max_rank=5 min=846 min_rank=3 imbalance_ratio=1.143555 max_load_reduction=7021 max_load_reduction_pct=0.857056 moved_to_echo=4183 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[76, 945, 1048, 0, 183, 920, 32, 784, 1, 1041, 1084, 0, 93, 939, 0, 1046] rerouted_tokens_per_ep_rank=[1021, 1048, 1103, 816, 1042, 1084, 1032, 1046] avg=1024.0000 max=1103 max_rank=2 min=816 min_rank=3 imbalance_ratio=1.077148 max_load_reduction=7089 max_load_reduction_pct=0.865356 moved_to_echo=4204 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 982, 1054, 0, 10, 956, 204, 815, 0, 1047, 1128, 0, 1, 944, 0, 1051] rerouted_tokens_per_ep_rank=[982, 1054, 966, 1019, 1047, 1128, 945, 1051] avg=1024.0000 max=1128 max_rank=5 min=945 min_rank=6 imbalance_ratio=1.101562 max_load_reduction=7064 max_load_reduction_pct=0.862305 moved_to_echo=4171 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8892, 7492, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8892, 7492, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8892 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8892, 7492, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8892, 7492, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8892, 7492, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8892, 7492, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8892, 7492, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7492 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8892, 7492, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1792, 6528] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[640, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7492 actual_valid_recv=7680 observed_loss=-188 tokens_per_expert=[256, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8892 actual_valid_recv=8960 observed_loss=-68 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[9, 182, 3705, 130, 158, 0, 67, 3941] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 533, 3657, 49, 9, 33, 129, 3782] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 1434, 1655, 454, 27, 565, 506, 3551] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[11, 100, 4072, 0, 1, 0, 96, 3912] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[14, 2609, 3439, 100, 180, 0, 0, 1850] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[7, 87, 4048, 1, 429, 32, 7, 3581] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[5, 44, 4058, 19, 0, 10, 3, 4053] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[2, 2010, 3873, 232, 41, 2, 2, 2030] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[11, 1163, 100, 174, 1171, 0, 0, 1055, 1, 1076, 0, 1078, 96, 660, 1607, 0] rerouted_tokens_per_ep_rank=[1174, 274, 1171, 1055, 1077, 1078, 756, 1607] avg=1024.0000 max=1607 max_rank=7 min=274 min_rank=1 imbalance_ratio=1.569336 max_load_reduction=6585 max_load_reduction_pct=0.803833 moved_to_echo=4518 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[9, 1058, 182, 176, 1066, 0, 130, 1063, 158, 1084, 0, 981, 67, 600, 1618, 0] rerouted_tokens_per_ep_rank=[1067, 358, 1066, 1193, 1242, 981, 667, 1618] avg=1024.0000 max=1618 max_rank=7 min=358 min_rank=1 imbalance_ratio=1.580078 max_load_reduction=6574 max_load_reduction_pct=0.802490 moved_to_echo=4508 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[14, 982, 2609, 82, 990, 0, 100, 499, 180, 509, 0, 910, 0, 557, 760, 0] rerouted_tokens_per_ep_rank=[996, 2691, 990, 599, 689, 910, 557, 760] avg=1024.0000 max=2691 max_rank=1 min=557 min_rank=6 imbalance_ratio=2.627930 max_load_reduction=5501 max_load_reduction_pct=0.671509 moved_to_echo=2916 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1044, 533, 168, 1053, 0, 49, 1020, 9, 1040, 33, 968, 129, 592, 1554, 0] rerouted_tokens_per_ep_rank=[1044, 701, 1053, 1069, 1049, 1001, 721, 1554] avg=1024.0000 max=1554 max_rank=7 min=701 min_rank=1 imbalance_ratio=1.517578 max_load_reduction=6638 max_load_reduction_pct=0.810303 moved_to_echo=4325 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 1159, 44, 181, 1168, 0, 19, 1094, 0, 1115, 10, 1074, 3, 657, 1663, 0] rerouted_tokens_per_ep_rank=[1164, 225, 1168, 1113, 1115, 1084, 660, 1663] avg=1024.0000 max=1663 max_rank=7 min=225 min_rank=1 imbalance_ratio=1.624023 max_load_reduction=6529 max_load_reduction_pct=0.796997 moved_to_echo=4522 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 476, 1434, 162, 465, 0, 454, 963, 27, 980, 565, 442, 506, 272, 1446, 0] rerouted_tokens_per_ep_rank=[476, 1596, 465, 1417, 1007, 1007, 778, 1446] avg=1024.0000 max=1596 max_rank=1 min=465 min_rank=2 imbalance_ratio=1.558594 max_load_reduction=6596 max_load_reduction_pct=0.805176 moved_to_echo=4238 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[7, 1156, 87, 160, 1164, 0, 1, 966, 429, 985, 32, 1072, 7, 656, 1470, 0] rerouted_tokens_per_ep_rank=[1163, 247, 1164, 967, 1414, 1104, 663, 1470] avg=1024.0000 max=1470 max_rank=7 min=247 min_rank=1 imbalance_ratio=1.435547 max_load_reduction=6722 max_load_reduction_pct=0.820557 moved_to_echo=4651 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 1106, 2010, 90, 1115, 0, 232, 547, 41, 558, 2, 1025, 2, 627, 835, 0] rerouted_tokens_per_ep_rank=[1108, 2100, 1115, 779, 599, 1027, 629, 835] avg=1024.0000 max=2100 max_rank=1 min=599 min_rank=4 imbalance_ratio=2.050781 max_load_reduction=6092 max_load_reduction_pct=0.743652 moved_to_echo=3090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 5431, 10953] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 5431, 10953] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 5431, 10953] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 5431, 10953] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10953 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 5431, 10953] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 5431, 10953] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 5431, 10953] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5431 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 5431, 10953] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=5431 actual_valid_recv=5632 observed_loss=-201 tokens_per_expert=[896, 4736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7040, 1280] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1024, 7296] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[896, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=10953 actual_valid_recv=11008 observed_loss=-55 tokens_per_expert=[11008, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=5632 slack_slots=10752 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=11008 slack_slots=5376 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[7, 2, 384, 181, 3954, 743, 209, 2712] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[50, 57, 3502, 387, 3786, 334, 63, 13] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 1597, 2037, 4062, 493, 0, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2732, 724, 2996, 1601, 100, 39] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[78, 0, 1810, 971, 3313, 1253, 1, 766] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 3, 3758, 262, 4070, 1, 0, 98] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[7, 1, 2639, 1353, 3919, 211, 49, 13] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[6, 0, 3574, 3, 4066, 17, 1, 525] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1083, 0, 1094, 813, 0, 2037, 181, 1104, 0, 493, 282, 0, 781, 3, 321] rerouted_tokens_per_ep_rank=[1083, 1094, 813, 2218, 1104, 775, 781, 324] avg=1024.0000 max=2218 max_rank=3 min=324 min_rank=7 imbalance_ratio=2.166016 max_load_reduction=5974 max_load_reduction_pct=0.729248 moved_to_echo=2984 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[50, 1009, 57, 1020, 1781, 0, 387, 398, 1029, 0, 334, 619, 63, 728, 13, 704] rerouted_tokens_per_ep_rank=[1059, 1077, 1781, 785, 1029, 953, 791, 717] avg=1024.0000 max=1781 max_rank=2 min=717 min_rank=7 imbalance_ratio=1.739258 max_load_reduction=6411 max_load_reduction_pct=0.782593 moved_to_echo=3490 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 798, 0, 807, 1390, 0, 724, 310, 815, 0, 1601, 483, 100, 576, 39, 549] rerouted_tokens_per_ep_rank=[798, 807, 1390, 1034, 815, 2084, 676, 588] avg=1024.0000 max=2084 max_rank=5 min=588 min_rank=7 imbalance_ratio=2.035156 max_load_reduction=6108 max_load_reduction_pct=0.745605 moved_to_echo=4163 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[7, 1057, 2, 1069, 186, 0, 181, 47, 1065, 0, 743, 71, 209, 763, 2712, 80] rerouted_tokens_per_ep_rank=[1064, 1071, 186, 228, 1065, 814, 972, 2792] avg=1024.0000 max=2792 max_rank=7 min=186 min_rank=2 imbalance_ratio=2.726562 max_load_reduction=5400 max_load_reduction_pct=0.659180 moved_to_echo=5643 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1085, 3, 1096, 1910, 0, 262, 427, 1107, 0, 1, 665, 0, 782, 98, 756] rerouted_tokens_per_ep_rank=[1085, 1099, 1910, 689, 1107, 666, 782, 854] avg=1024.0000 max=1910 max_rank=2 min=666 min_rank=5 imbalance_ratio=1.865234 max_load_reduction=6282 max_load_reduction_pct=0.766846 moved_to_echo=3409 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[78, 883, 0, 892, 921, 0, 971, 205, 901, 0, 1253, 320, 1, 637, 766, 364] rerouted_tokens_per_ep_rank=[961, 892, 921, 1176, 901, 1573, 638, 1130] avg=1024.0000 max=1573 max_rank=5 min=638 min_rank=6 imbalance_ratio=1.536133 max_load_reduction=6619 max_load_reduction_pct=0.807983 moved_to_echo=4242 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[7, 1045, 1, 1056, 1342, 0, 1353, 300, 1065, 0, 211, 467, 49, 753, 13, 530] rerouted_tokens_per_ep_rank=[1052, 1057, 1342, 1653, 1065, 678, 802, 543] avg=1024.0000 max=1653 max_rank=3 min=543 min_rank=7 imbalance_ratio=1.614258 max_load_reduction=6539 max_load_reduction_pct=0.798218 moved_to_echo=3088 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[6, 1084, 0, 1095, 1817, 0, 3, 406, 1106, 0, 17, 632, 1, 781, 525, 719] rerouted_tokens_per_ep_rank=[1090, 1095, 1817, 409, 1106, 649, 782, 1244] avg=1024.0000 max=1817 max_rank=2 min=409 min_rank=3 imbalance_ratio=1.774414 max_load_reduction=6375 max_load_reduction_pct=0.778198 moved_to_echo=3781 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10160 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 10160, 8192, 8192, 8192, 6224, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 10160, 8192, 8192, 8192, 6224, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 10160, 8192, 8192, 8192, 6224, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 10160, 8192, 8192, 8192, 6224, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 10160, 8192, 8192, 8192, 6224, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 10160, 8192, 8192, 8192, 6224, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 10160, 8192, 8192, 8192, 6224, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6224 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 10160, 8192, 8192, 8192, 6224, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=6224 actual_valid_recv=6400 observed_loss=-176 tokens_per_expert=[512, 5888] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6016, 2304] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4224, 4096] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4736, 3584] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=10160 actual_valid_recv=10240 observed_loss=-80 tokens_per_expert=[10240, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=6400 slack_slots=9984 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=10240 slack_slots=6144 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[53, 898, 0, 5, 3753, 128, 0, 3355] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[40, 1809, 0, 422, 3873, 20, 0, 2028] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[352, 546, 0, 3, 4033, 49, 4, 3205] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[2118, 1551, 0, 103, 4019, 123, 68, 210] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[268, 1162, 0, 257, 4001, 0, 0, 2504] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[187, 2471, 0, 147, 2674, 10, 3, 2700] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[17, 2289, 0, 9, 3608, 72, 11, 2186] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[95, 3136, 0, 0, 3207, 120, 0, 1634] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[53, 327, 571, 0, 0, 1054, 5, 1249, 1056, 0, 128, 601, 0, 1042, 2106, 0] rerouted_tokens_per_ep_rank=[380, 571, 1054, 1254, 1056, 729, 1042, 2106] avg=1024.0000 max=2106 max_rank=7 min=380 min_rank=0 imbalance_ratio=2.056641 max_load_reduction=6086 max_load_reduction_pct=0.742920 moved_to_echo=4933 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[40, 660, 1149, 0, 0, 1087, 422, 755, 1089, 0, 20, 621, 0, 1076, 1273, 0] rerouted_tokens_per_ep_rank=[700, 1149, 1087, 1177, 1089, 641, 1076, 1273] avg=1024.0000 max=1273 max_rank=7 min=641 min_rank=5 imbalance_ratio=1.243164 max_load_reduction=6919 max_load_reduction_pct=0.844604 moved_to_echo=4079 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[187, 902, 1569, 0, 0, 751, 147, 1005, 752, 0, 10, 428, 3, 743, 1695, 0] rerouted_tokens_per_ep_rank=[1089, 1569, 751, 1152, 752, 438, 746, 1695] avg=1024.0000 max=1695 max_rank=7 min=438 min_rank=5 imbalance_ratio=1.655273 max_load_reduction=6497 max_load_reduction_pct=0.793091 moved_to_echo=3631 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[268, 424, 738, 0, 0, 1123, 257, 932, 1126, 0, 0, 641, 0, 1111, 1572, 0] rerouted_tokens_per_ep_rank=[692, 738, 1123, 1189, 1126, 641, 1111, 1572] avg=1024.0000 max=1572 max_rank=7 min=641 min_rank=5 imbalance_ratio=1.535156 max_load_reduction=6620 max_load_reduction_pct=0.808105 moved_to_echo=4450 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2118, 570, 981, 0, 0, 1132, 103, 82, 1117, 0, 123, 649, 68, 1121, 128, 0] rerouted_tokens_per_ep_rank=[2688, 981, 1132, 185, 1117, 772, 1189, 128] avg=1024.0000 max=2688 max_rank=0 min=128 min_rank=7 imbalance_ratio=2.625000 max_load_reduction=5504 max_load_reduction_pct=0.671875 moved_to_echo=3206 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[352, 199, 347, 0, 0, 1132, 3, 1193, 1135, 0, 49, 646, 4, 1120, 2012, 0] rerouted_tokens_per_ep_rank=[551, 347, 1132, 1196, 1135, 695, 1124, 2012] avg=1024.0000 max=2012 max_rank=7 min=347 min_rank=1 imbalance_ratio=1.964844 max_load_reduction=6180 max_load_reduction_pct=0.754395 moved_to_echo=4966 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[17, 835, 1454, 0, 0, 1013, 9, 814, 1015, 0, 72, 578, 11, 1002, 1372, 0] rerouted_tokens_per_ep_rank=[852, 1454, 1013, 823, 1015, 650, 1013, 1372] avg=1024.0000 max=1454 max_rank=1 min=650 min_rank=5 imbalance_ratio=1.419922 max_load_reduction=6738 max_load_reduction_pct=0.822510 moved_to_echo=4050 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[95, 1145, 1991, 0, 0, 900, 0, 608, 902, 0, 120, 514, 0, 891, 1026, 0] rerouted_tokens_per_ep_rank=[1240, 1991, 900, 608, 902, 634, 891, 1026] avg=1024.0000 max=1991 max_rank=1 min=608 min_rank=3 imbalance_ratio=1.944336 max_load_reduction=6201 max_load_reduction_pct=0.756958 moved_to_echo=3453 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8800, 8192, 7584, 8192, 5200, 8192, 11184] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8800, 8192, 7584, 8192, 5200, 8192, 11184] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7584 expected_overflow=0 exact_recv_per_rank=[8192, 8800, 8192, 7584, 8192, 5200, 8192, 11184] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5200 expected_overflow=0 exact_recv_per_rank=[8192, 8800, 8192, 7584, 8192, 5200, 8192, 11184] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8800, 8192, 7584, 8192, 5200, 8192, 11184] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8800, 8192, 7584, 8192, 5200, 8192, 11184] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8800 expected_overflow=0 exact_recv_per_rank=[8192, 8800, 8192, 7584, 8192, 5200, 8192, 11184] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11184 expected_overflow=0 exact_recv_per_rank=[8192, 8800, 8192, 7584, 8192, 5200, 8192, 11184] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=5200 actual_valid_recv=5376 observed_loss=-176 tokens_per_expert=[640, 4736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3200, 5120] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=7584 actual_valid_recv=7680 observed_loss=-96 tokens_per_expert=[1024, 6656] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8800 actual_valid_recv=8832 observed_loss=-32 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=11184 actual_valid_recv=11264 observed_loss=-80 tokens_per_expert=[11264, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=5376 slack_slots=11008 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=11264 slack_slots=5120 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 165, 35, 2111, 2793, 20, 0, 3068] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 250, 23, 316, 3896, 0, 3, 3704] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 45, 0, 3941, 358, 0, 0, 3848] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[206, 117, 1043, 2004, 2688, 25, 294, 1815] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[9, 252, 2, 2457, 2194, 1, 23, 3254] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[47, 797, 51, 1812, 1724, 19, 2, 3740] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[15, 5, 9, 1133, 3306, 15, 2, 3707] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 193, 17, 1981, 2321, 1, 2, 3677] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 905, 165, 853, 35, 842, 1258, 0, 1188, 0, 20, 928, 0, 763, 1235, 0] rerouted_tokens_per_ep_rank=[905, 1018, 877, 1258, 1188, 948, 763, 1235] avg=1024.0000 max=1258 max_rank=3 min=763 min_rank=6 imbalance_ratio=1.228516 max_load_reduction=6934 max_load_reduction_pct=0.846436 moved_to_echo=4134 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1093, 250, 127, 23, 1175, 189, 0, 1656, 0, 0, 1120, 3, 1065, 1491, 0] rerouted_tokens_per_ep_rank=[1093, 377, 1198, 189, 1656, 1120, 1068, 1491] avg=1024.0000 max=1656 max_rank=4 min=189 min_rank=3 imbalance_ratio=1.617188 max_load_reduction=6536 max_load_reduction_pct=0.797852 moved_to_echo=5335 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1135, 45, 1592, 0, 108, 2349, 0, 153, 0, 0, 1164, 0, 97, 1549, 0] rerouted_tokens_per_ep_rank=[1135, 1637, 108, 2349, 153, 1164, 97, 1549] avg=1024.0000 max=2349 max_rank=3 min=97 min_rank=6 imbalance_ratio=2.293945 max_load_reduction=5843 max_load_reduction_pct=0.713257 moved_to_echo=2963 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[206, 539, 117, 814, 1043, 814, 1190, 0, 1135, 0, 25, 551, 294, 739, 725, 0] rerouted_tokens_per_ep_rank=[745, 931, 1857, 1190, 1135, 576, 1033, 725] avg=1024.0000 max=1857 max_rank=2 min=576 min_rank=5 imbalance_ratio=1.813477 max_load_reduction=6335 max_load_reduction_pct=0.773315 moved_to_echo=3469 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[47, 1104, 797, 732, 51, 520, 1080, 0, 733, 0, 19, 1131, 2, 471, 1505, 0] rerouted_tokens_per_ep_rank=[1151, 1529, 571, 1080, 733, 1150, 473, 1505] avg=1024.0000 max=1529 max_rank=1 min=473 min_rank=6 imbalance_ratio=1.493164 max_load_reduction=6663 max_load_reduction_pct=0.813354 moved_to_echo=3861 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[9, 960, 252, 993, 2, 661, 1464, 0, 934, 0, 1, 984, 23, 599, 1310, 0] rerouted_tokens_per_ep_rank=[969, 1245, 663, 1464, 934, 985, 622, 1310] avg=1024.0000 max=1464 max_rank=3 min=622 min_rank=6 imbalance_ratio=1.429688 max_load_reduction=6728 max_load_reduction_pct=0.821289 moved_to_echo=3851 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[15, 1094, 5, 457, 9, 997, 676, 0, 1406, 0, 15, 1121, 2, 903, 1492, 0] rerouted_tokens_per_ep_rank=[1109, 462, 1006, 676, 1406, 1136, 905, 1492] avg=1024.0000 max=1492 max_rank=7 min=462 min_rank=1 imbalance_ratio=1.457031 max_load_reduction=6700 max_load_reduction_pct=0.817871 moved_to_echo=4939 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1085, 193, 800, 17, 700, 1181, 0, 987, 0, 1, 1112, 2, 634, 1480, 0] rerouted_tokens_per_ep_rank=[1085, 993, 717, 1181, 987, 1113, 636, 1480] avg=1024.0000 max=1480 max_rank=7 min=636 min_rank=6 imbalance_ratio=1.445312 max_load_reduction=6712 max_load_reduction_pct=0.819336 moved_to_echo=4216 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9387 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6997, 9387, 8192, 8192, 5597, 10787] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6997, 9387, 8192, 8192, 5597, 10787] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6997, 9387, 8192, 8192, 5597, 10787] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6997 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6997, 9387, 8192, 8192, 5597, 10787] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6997, 9387, 8192, 8192, 5597, 10787] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10787 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6997, 9387, 8192, 8192, 5597, 10787] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6997, 9387, 8192, 8192, 5597, 10787] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5597 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6997, 9387, 8192, 8192, 5597, 10787] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=5597 actual_valid_recv=5760 observed_loss=-163 tokens_per_expert=[384, 5376] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1920, 6400] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=6997 actual_valid_recv=7168 observed_loss=-171 tokens_per_expert=[1280, 5888] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=9387 actual_valid_recv=9472 observed_loss=-85 tokens_per_expert=[9472, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=10787 actual_valid_recv=10880 observed_loss=-93 tokens_per_expert=[10880, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=5760 slack_slots=10624 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=7168 slack_slots=9216 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=9472 slack_slots=6912 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=10880 slack_slots=5504 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 5, 164, 937, 3740, 3305, 36, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 17, 2533, 389, 3169, 0, 2082] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 3784, 207, 4025, 4, 172] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[2, 18, 1822, 3084, 2114, 1145, 6, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 30, 1582, 2449, 3568, 7, 556] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4, 77, 3431, 3598, 884, 194, 4] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1, 69, 308, 194, 3416, 4058, 111, 35] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[2, 8, 62, 1009, 2757, 4049, 0, 305] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1361, 0, 1301, 0, 761, 1873, 0, 121, 0, 1363, 0, 4, 86, 172, 1150] rerouted_tokens_per_ep_rank=[1361, 1301, 761, 1873, 121, 1363, 90, 1322] avg=1024.0000 max=1873 max_rank=3 min=90 min_rank=6 imbalance_ratio=1.829102 max_load_reduction=6319 max_load_reduction_pct=0.771362 moved_to_echo=2896 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1071, 2, 1024, 17, 509, 1255, 0, 226, 0, 1074, 0, 0, 163, 2082, 769] rerouted_tokens_per_ep_rank=[1071, 1026, 526, 1255, 226, 1074, 163, 2851] avg=1024.0000 max=2851 max_rank=7 min=163 min_rank=6 imbalance_ratio=2.784180 max_load_reduction=5341 max_load_reduction_pct=0.651978 moved_to_echo=4314 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1206, 0, 1153, 30, 318, 784, 0, 1422, 0, 1209, 0, 7, 1027, 556, 480] rerouted_tokens_per_ep_rank=[1206, 1153, 348, 784, 1422, 1209, 1034, 1036] avg=1024.0000 max=1422 max_rank=4 min=348 min_rank=2 imbalance_ratio=1.388672 max_load_reduction=6770 max_load_reduction_pct=0.826416 moved_to_echo=4701 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1117, 5, 1068, 164, 188, 465, 0, 2171, 0, 1120, 0, 36, 1569, 5, 284] rerouted_tokens_per_ep_rank=[1117, 1073, 352, 465, 2171, 1120, 1605, 289] avg=1024.0000 max=2171 max_rank=4 min=289 min_rank=7 imbalance_ratio=2.120117 max_load_reduction=6021 max_load_reduction_pct=0.734985 moved_to_echo=5185 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 392, 18, 374, 1822, 623, 1518, 0, 1223, 0, 379, 0, 6, 891, 1, 943] rerouted_tokens_per_ep_rank=[394, 392, 2445, 1518, 1223, 379, 897, 944] avg=1024.0000 max=2445 max_rank=2 min=379 min_rank=5 imbalance_ratio=2.387695 max_load_reduction=5747 max_load_reduction_pct=0.701538 moved_to_echo=3443 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 299, 4, 285, 77, 690, 1699, 0, 2089, 0, 300, 0, 194, 1509, 4, 1042] rerouted_tokens_per_ep_rank=[299, 289, 767, 1699, 2089, 300, 1703, 1046] avg=1024.0000 max=2089 max_rank=4 min=289 min_rank=1 imbalance_ratio=2.040039 max_load_reduction=6103 max_load_reduction_pct=0.744995 moved_to_echo=5138 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1372, 69, 1311, 308, 39, 97, 0, 1983, 0, 1375, 0, 111, 1433, 35, 58] rerouted_tokens_per_ep_rank=[1373, 1380, 347, 97, 1983, 1375, 1544, 93] avg=1024.0000 max=1983 max_rank=4 min=93 min_rank=7 imbalance_ratio=1.936523 max_load_reduction=6209 max_load_reduction_pct=0.757935 moved_to_echo=4995 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 1369, 8, 1308, 62, 202, 501, 0, 1601, 0, 1372, 0, 0, 1156, 305, 306] rerouted_tokens_per_ep_rank=[1371, 1316, 264, 501, 1601, 1372, 1156, 611] avg=1024.0000 max=1601 max_rank=4 min=264 min_rank=2 imbalance_ratio=1.563477 max_load_reduction=6591 max_load_reduction_pct=0.804565 moved_to_echo=4740 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7930 expected_overflow=0 exact_recv_per_rank=[8192, 7930, 5810, 8192, 10836, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7930, 5810, 8192, 10836, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7930, 5810, 8192, 10836, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7930, 5810, 8192, 10836, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5810 expected_overflow=0 exact_recv_per_rank=[8192, 7930, 5810, 8192, 10836, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7930, 5810, 8192, 10836, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10836 expected_overflow=0 exact_recv_per_rank=[8192, 7930, 5810, 8192, 10836, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7930, 5810, 8192, 10836, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=5810 actual_valid_recv=6016 observed_loss=-206 tokens_per_expert=[2560, 3456] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=7930 actual_valid_recv=8064 observed_loss=-134 tokens_per_expert=[128, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3200, 5120] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=10836 actual_valid_recv=10880 observed_loss=-44 tokens_per_expert=[10880, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=6016 slack_slots=10368 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8064 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=10880 slack_slots=5504 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[220, 186, 3970, 17, 1510, 36, 2187, 66] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[2024, 1705, 2790, 1, 93, 0, 1384, 195] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[2684, 447, 3965, 0, 333, 1, 762, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[462, 1107, 3702, 500, 2063, 0, 344, 14] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1907, 123, 4040, 1, 114, 6, 2001, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1711, 517, 4096, 0, 797, 942, 129, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3007, 1091, 4052, 0, 4, 1, 4, 33] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3755, 36, 4082, 0, 11, 0, 308, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1086, 0, 1705, 382, 746, 0, 1, 697, 93, 419, 0, 631, 1384, 137, 195, 716] rerouted_tokens_per_ep_rank=[1086, 2087, 746, 698, 512, 631, 1521, 911] avg=1024.0000 max=2087 max_rank=1 min=512 min_rank=4 imbalance_ratio=2.038086 max_load_reduction=6105 max_load_reduction_pct=0.745239 moved_to_echo=3575 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[120, 0, 186, 41, 1061, 0, 17, 992, 1510, 45, 36, 898, 2187, 14, 66, 1019] rerouted_tokens_per_ep_rank=[120, 227, 1061, 1009, 1555, 934, 2201, 1085] avg=1024.0000 max=2201 max_rank=6 min=120 min_rank=0 imbalance_ratio=2.149414 max_load_reduction=5991 max_load_reduction_pct=0.731323 moved_to_echo=5775 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1439, 0, 447, 507, 1059, 0, 0, 991, 333, 556, 1, 897, 762, 182, 0, 1018] rerouted_tokens_per_ep_rank=[1439, 954, 1059, 991, 889, 898, 944, 1018] avg=1024.0000 max=1439 max_rank=0 min=889 min_rank=4 imbalance_ratio=1.405273 max_load_reduction=6753 max_load_reduction_pct=0.824341 moved_to_echo=3749 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[918, 0, 517, 323, 1095, 0, 0, 1023, 797, 354, 942, 927, 129, 116, 0, 1051] rerouted_tokens_per_ep_rank=[918, 840, 1095, 1023, 1151, 1869, 245, 1051] avg=1024.0000 max=1869 max_rank=5 min=245 min_rank=6 imbalance_ratio=1.825195 max_load_reduction=6323 max_load_reduction_pct=0.771851 moved_to_echo=4316 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[237, 0, 1107, 90, 977, 0, 500, 929, 2063, 99, 0, 841, 344, 36, 14, 955] rerouted_tokens_per_ep_rank=[237, 1197, 977, 1429, 2162, 841, 380, 969] avg=1024.0000 max=2162 max_rank=4 min=237 min_rank=0 imbalance_ratio=2.111328 max_load_reduction=6030 max_load_reduction_pct=0.736084 moved_to_echo=4352 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1023, 0, 123, 360, 1080, 0, 1, 1009, 114, 395, 6, 914, 2001, 129, 0, 1037] rerouted_tokens_per_ep_rank=[1023, 483, 1080, 1010, 509, 920, 2130, 1037] avg=1024.0000 max=2130 max_rank=6 min=483 min_rank=1 imbalance_ratio=2.080078 max_load_reduction=6062 max_load_reduction_pct=0.739990 moved_to_echo=4596 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1613, 0, 1091, 568, 1083, 0, 0, 1012, 4, 622, 1, 917, 4, 204, 33, 1040] rerouted_tokens_per_ep_rank=[1613, 1659, 1083, 1012, 626, 918, 208, 1073] avg=1024.0000 max=1659 max_rank=1 min=208 min_rank=6 imbalance_ratio=1.620117 max_load_reduction=6533 max_load_reduction_pct=0.797485 moved_to_echo=2825 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2014, 0, 36, 709, 1091, 0, 0, 1020, 11, 777, 0, 923, 308, 255, 0, 1048] rerouted_tokens_per_ep_rank=[2014, 745, 1091, 1020, 788, 923, 563, 1048] avg=1024.0000 max=2014 max_rank=0 min=563 min_rank=6 imbalance_ratio=1.966797 max_load_reduction=6178 max_load_reduction_pct=0.754150 moved_to_echo=3322 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8450, 8192, 8192, 8192, 8192, 7934, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8450, 8192, 8192, 8192, 8192, 7934, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8450, 8192, 8192, 8192, 8192, 7934, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8450, 8192, 8192, 8192, 8192, 7934, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7934 expected_overflow=0 exact_recv_per_rank=[8450, 8192, 8192, 8192, 8192, 7934, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8450, 8192, 8192, 8192, 8192, 7934, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8450, 8192, 8192, 8192, 8192, 7934, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8450 expected_overflow=0 exact_recv_per_rank=[8450, 8192, 8192, 8192, 8192, 7934, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8450 actual_valid_recv=8576 observed_loss=-126 tokens_per_expert=[8576, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7934 actual_valid_recv=8064 observed_loss=-130 tokens_per_expert=[1024, 7040] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[640, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5248, 3072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7168, 1152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4992, 3328] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8576 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8064 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[865, 3130, 129, 10, 3, 134, 3808, 113] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 3929, 0, 1, 0, 6, 3541, 715] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4085, 1, 0, 0, 36, 4065, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[211, 3589, 4, 37, 10, 4, 2742, 1595] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[13, 2917, 3, 995, 0, 25, 3952, 287] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[12, 4069, 9, 514, 0, 820, 2381, 387] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4081, 0, 402, 5, 1, 3628, 75] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[137, 3480, 0, 996, 19, 1, 3233, 326] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1033, 1145, 0, 1, 1122, 0, 778, 0, 1137, 36, 681, 1558, 0, 5, 696] rerouted_tokens_per_ep_rank=[1033, 1145, 1123, 778, 1137, 717, 1558, 701] avg=1024.0000 max=1558 max_rank=6 min=701 min_rank=7 imbalance_ratio=1.521484 max_load_reduction=6634 max_load_reduction_pct=0.809814 moved_to_echo=4113 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 900, 1101, 0, 0, 1079, 1, 678, 0, 1094, 6, 655, 1356, 0, 715, 607] rerouted_tokens_per_ep_rank=[900, 1101, 1079, 679, 1094, 661, 1356, 1322] avg=1024.0000 max=1356 max_rank=6 min=661 min_rank=5 imbalance_ratio=1.324219 max_load_reduction=6836 max_load_reduction_pct=0.834473 moved_to_echo=4433 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[865, 971, 866, 0, 129, 863, 10, 732, 3, 875, 134, 526, 1449, 0, 113, 656] rerouted_tokens_per_ep_rank=[1836, 866, 992, 742, 878, 660, 1449, 769] avg=1024.0000 max=1836 max_rank=0 min=660 min_rank=5 imbalance_ratio=1.792969 max_load_reduction=6356 max_load_reduction_pct=0.775879 moved_to_echo=3756 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 922, 1143, 0, 0, 1121, 402, 694, 5, 1136, 1, 681, 1391, 0, 75, 621] rerouted_tokens_per_ep_rank=[922, 1143, 1121, 1096, 1141, 682, 1391, 696] avg=1024.0000 max=1391 max_rank=6 min=682 min_rank=5 imbalance_ratio=1.358398 max_load_reduction=6801 max_load_reduction_pct=0.830200 moved_to_echo=3910 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[13, 1004, 818, 0, 3, 801, 995, 756, 0, 812, 25, 486, 1515, 0, 287, 677] rerouted_tokens_per_ep_rank=[1017, 818, 804, 1751, 812, 511, 1515, 964] avg=1024.0000 max=1751 max_rank=3 min=511 min_rank=5 imbalance_ratio=1.709961 max_load_reduction=6441 max_load_reduction_pct=0.786255 moved_to_echo=3802 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[211, 697, 1005, 0, 4, 986, 37, 525, 10, 999, 4, 599, 1050, 0, 1595, 470] rerouted_tokens_per_ep_rank=[908, 1005, 990, 562, 1009, 603, 1050, 2065] avg=1024.0000 max=2065 max_rank=7 min=562 min_rank=3 imbalance_ratio=2.016602 max_load_reduction=6127 max_load_reduction_pct=0.747925 moved_to_echo=4727 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 605, 1139, 0, 9, 1118, 514, 455, 0, 1133, 820, 679, 913, 0, 387, 408] rerouted_tokens_per_ep_rank=[617, 1139, 1127, 969, 1133, 1499, 913, 795] avg=1024.0000 max=1499 max_rank=5 min=617 min_rank=0 imbalance_ratio=1.463867 max_load_reduction=6693 max_load_reduction_pct=0.817017 moved_to_echo=4340 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[137, 822, 975, 0, 0, 956, 996, 619, 19, 969, 1, 580, 1238, 0, 326, 554] rerouted_tokens_per_ep_rank=[959, 975, 956, 1615, 988, 581, 1238, 880] avg=1024.0000 max=1615 max_rank=3 min=581 min_rank=5 imbalance_ratio=1.577148 max_load_reduction=6577 max_load_reduction_pct=0.802856 moved_to_echo=3687 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5914, 10470, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5914, 10470, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5914, 10470, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5914 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5914, 10470, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5914, 10470, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5914, 10470, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5914, 10470, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10470 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 5914, 10470, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=5914 actual_valid_recv=6144 observed_loss=-230 tokens_per_expert=[1152, 4992] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3584, 4736] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1280, 7040] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3072, 5248] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=10470 actual_valid_recv=10496 observed_loss=-26 tokens_per_expert=[10496, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=6144 slack_slots=10240 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=10496 slack_slots=5888 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[2883, 1, 1828, 933, 130, 2413, 1, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 3888, 1237, 14, 3018, 1, 34] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1813, 252, 719, 4051, 0, 1207, 5, 145] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1683, 870, 702, 4060, 97, 530, 35, 215] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[943, 722, 15, 3994, 73, 2445, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[3652, 73, 1142, 3252, 2, 64, 7, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[308, 1962, 1459, 3004, 3, 1047, 3, 406] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[2056, 935, 38, 3496, 0, 1642, 12, 13] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1807, 0, 1, 346, 1828, 0, 319, 0, 130, 299, 2067, 0, 1, 315, 3, 1076] rerouted_tokens_per_ep_rank=[1807, 347, 1828, 319, 429, 2067, 316, 1079] avg=1024.0000 max=2067 max_rank=5 min=316 min_rank=6 imbalance_ratio=2.018555 max_load_reduction=6125 max_load_reduction_pct=0.747681 moved_to_echo=3891 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 437, 3888, 0, 417, 0, 14, 399, 2581, 0, 1, 421, 34, 0] rerouted_tokens_per_ep_rank=[0, 437, 3888, 417, 413, 2581, 422, 34] avg=1024.0000 max=3888 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.796875 max_load_reduction=4304 max_load_reduction_pct=0.525391 moved_to_echo=3450 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1137, 0, 252, 173, 719, 0, 1382, 0, 0, 1299, 1034, 0, 5, 1370, 145, 676] rerouted_tokens_per_ep_rank=[1137, 425, 719, 1382, 1299, 1034, 1375, 821] avg=1024.0000 max=1382 max_rank=3 min=425 min_rank=1 imbalance_ratio=1.349609 max_load_reduction=6810 max_load_reduction_pct=0.831299 moved_to_echo=4529 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1055, 0, 870, 76, 702, 0, 1385, 0, 97, 1302, 454, 0, 35, 1373, 215, 628] rerouted_tokens_per_ep_rank=[1055, 946, 702, 1385, 1399, 454, 1408, 843] avg=1024.0000 max=1408 max_rank=6 min=454 min_rank=5 imbalance_ratio=1.375000 max_load_reduction=6784 max_load_reduction_pct=0.828125 moved_to_echo=4104 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[194, 0, 1962, 150, 1459, 0, 1025, 0, 3, 963, 897, 0, 3, 1016, 406, 114] rerouted_tokens_per_ep_rank=[194, 2112, 1459, 1025, 966, 897, 1019, 520] avg=1024.0000 max=2112 max_rank=1 min=194 min_rank=0 imbalance_ratio=2.062500 max_load_reduction=6080 max_load_reduction_pct=0.742188 moved_to_echo=3402 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[587, 0, 722, 351, 15, 0, 1362, 0, 73, 1281, 2094, 0, 0, 1351, 0, 356] rerouted_tokens_per_ep_rank=[587, 1073, 15, 1362, 1354, 2094, 1351, 356] avg=1024.0000 max=2094 max_rank=5 min=15 min_rank=2 imbalance_ratio=2.044922 max_load_reduction=6098 max_load_reduction_pct=0.744385 moved_to_echo=5155 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2289, 0, 73, 9, 1142, 0, 1109, 0, 2, 1043, 55, 0, 7, 1100, 0, 1363] rerouted_tokens_per_ep_rank=[2289, 82, 1142, 1109, 1045, 55, 1107, 1363] avg=1024.0000 max=2289 max_rank=0 min=55 min_rank=5 imbalance_ratio=2.235352 max_load_reduction=5903 max_load_reduction_pct=0.720581 moved_to_echo=3570 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1289, 0, 935, 236, 38, 0, 1193, 0, 0, 1121, 1406, 0, 12, 1182, 13, 767] rerouted_tokens_per_ep_rank=[1289, 1171, 38, 1193, 1121, 1406, 1194, 780] avg=1024.0000 max=1406 max_rank=5 min=38 min_rank=2 imbalance_ratio=1.373047 max_load_reduction=6786 max_load_reduction_pct=0.828369 moved_to_echo=4501 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8358 expected_overflow=0 exact_recv_per_rank=[8358, 6593, 9791, 8192, 8026, 10588, 8192, 5796] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8026 expected_overflow=0 exact_recv_per_rank=[8358, 6593, 9791, 8192, 8026, 10588, 8192, 5796] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9791 expected_overflow=0 exact_recv_per_rank=[8358, 6593, 9791, 8192, 8026, 10588, 8192, 5796] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8358, 6593, 9791, 8192, 8026, 10588, 8192, 5796] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10588 expected_overflow=0 exact_recv_per_rank=[8358, 6593, 9791, 8192, 8026, 10588, 8192, 5796] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8358, 6593, 9791, 8192, 8026, 10588, 8192, 5796] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5796 expected_overflow=0 exact_recv_per_rank=[8358, 6593, 9791, 8192, 8026, 10588, 8192, 5796] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6593 expected_overflow=0 exact_recv_per_rank=[8358, 6593, 9791, 8192, 8026, 10588, 8192, 5796] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=5796 actual_valid_recv=5888 observed_loss=-92 tokens_per_expert=[896, 4992] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=6593 actual_valid_recv=6656 observed_loss=-63 tokens_per_expert=[4864, 1792] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8358 actual_valid_recv=8448 observed_loss=-90 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8026 actual_valid_recv=8192 observed_loss=-166 tokens_per_expert=[384, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=9791 actual_valid_recv=9856 observed_loss=-65 tokens_per_expert=[9856, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=10588 actual_valid_recv=10624 observed_loss=-36 tokens_per_expert=[10624, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=5888 slack_slots=10496 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=6656 slack_slots=9728 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=9856 slack_slots=6528 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=10624 slack_slots=5760 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 1597, 2108, 3840, 13, 79, 36, 519] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[126, 16, 2594, 3935, 1521, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4055, 12, 3339, 239, 535, 0, 0, 12] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[371, 334, 1642, 2309, 3363, 37, 58, 78] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1, 441, 7, 3411, 3955, 3, 0, 374] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[11, 41, 887, 1111, 2347, 0, 0, 3795] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[3975, 21, 32, 4052, 31, 20, 6, 55] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[3557, 144, 2512, 1240, 317, 0, 0, 422] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2747, 0, 12, 1308, 2271, 0, 143, 0, 405, 0, 0, 1068, 0, 96, 12, 130] rerouted_tokens_per_ep_rank=[2747, 1320, 2271, 143, 405, 1068, 96, 142] avg=1024.0000 max=2747 max_rank=0 min=96 min_rank=6 imbalance_ratio=2.682617 max_load_reduction=5445 max_load_reduction_pct=0.664673 moved_to_echo=1711 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[86, 0, 16, 40, 1764, 0, 2354, 0, 1152, 0, 0, 830, 0, 1581, 0, 369] rerouted_tokens_per_ep_rank=[86, 56, 1764, 2354, 1152, 830, 1581, 369] avg=1024.0000 max=2354 max_rank=3 min=56 min_rank=1 imbalance_ratio=2.298828 max_load_reduction=5838 max_load_reduction_pct=0.712646 moved_to_echo=3932 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1597, 0, 1430, 0, 2294, 0, 7, 0, 79, 678, 36, 1546, 519, 6] rerouted_tokens_per_ep_rank=[0, 1597, 1430, 2294, 7, 757, 1582, 525] avg=1024.0000 max=2294 max_rank=3 min=0 min_rank=0 imbalance_ratio=2.240234 max_load_reduction=5898 max_load_reduction_pct=0.719971 moved_to_echo=2871 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 441, 0, 5, 0, 2041, 0, 2994, 0, 3, 2, 0, 1370, 374, 961] rerouted_tokens_per_ep_rank=[1, 441, 5, 2041, 2994, 5, 1370, 1335] avg=1024.0000 max=2994 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.923828 max_load_reduction=5198 max_load_reduction_pct=0.634521 moved_to_echo=5704 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[252, 0, 334, 119, 1117, 0, 1382, 0, 2546, 0, 37, 525, 58, 927, 78, 817] rerouted_tokens_per_ep_rank=[252, 453, 1117, 1382, 2546, 562, 985, 895] avg=1024.0000 max=2546 max_rank=4 min=252 min_rank=0 imbalance_ratio=2.486328 max_load_reduction=5646 max_load_reduction_pct=0.689209 moved_to_echo=4988 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 41, 7, 604, 0, 665, 0, 1777, 0, 0, 283, 0, 446, 3795, 570] rerouted_tokens_per_ep_rank=[4, 48, 604, 665, 1777, 283, 446, 4365] avg=1024.0000 max=4365 max_rank=7 min=4 min_rank=0 imbalance_ratio=4.262695 max_load_reduction=3827 max_load_reduction_pct=0.467163 moved_to_echo=6871 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2693, 0, 21, 1282, 22, 0, 2424, 0, 24, 0, 20, 10, 6, 1628, 55, 7] rerouted_tokens_per_ep_rank=[2693, 1303, 22, 2424, 24, 30, 1634, 62] avg=1024.0000 max=2693 max_rank=0 min=22 min_rank=2 imbalance_ratio=2.629883 max_load_reduction=5499 max_load_reduction_pct=0.671265 moved_to_echo=1750 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2409, 0, 144, 1148, 1708, 0, 742, 0, 240, 0, 0, 804, 0, 498, 422, 77] rerouted_tokens_per_ep_rank=[2409, 1292, 1708, 742, 240, 804, 498, 499] avg=1024.0000 max=2409 max_rank=0 min=240 min_rank=4 imbalance_ratio=2.352539 max_load_reduction=5783 max_load_reduction_pct=0.705933 moved_to_echo=2041 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6510, 8921, 12045, 9145, 4339, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=12045 expected_overflow=0 exact_recv_per_rank=[8192, 6510, 8921, 12045, 9145, 4339, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9145 expected_overflow=0 exact_recv_per_rank=[8192, 6510, 8921, 12045, 9145, 4339, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6510, 8921, 12045, 9145, 4339, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6510 expected_overflow=0 exact_recv_per_rank=[8192, 6510, 8921, 12045, 9145, 4339, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6510, 8921, 12045, 9145, 4339, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8921 expected_overflow=0 exact_recv_per_rank=[8192, 6510, 8921, 12045, 9145, 4339, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4339 expected_overflow=0 exact_recv_per_rank=[8192, 6510, 8921, 12045, 9145, 4339, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=4339 actual_valid_recv=4480 observed_loss=-141 tokens_per_expert=[256, 4224] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=6510 actual_valid_recv=6656 observed_loss=-146 tokens_per_expert=[2688, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5376, 2944] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=9145 actual_valid_recv=9216 observed_loss=-71 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8921 actual_valid_recv=8960 observed_loss=-39 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=12045 actual_valid_recv=12160 observed_loss=-115 tokens_per_expert=[12160, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=4480 slack_slots=11904 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=6656 slack_slots=9728 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=12160 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[226, 385, 0, 1172, 6, 4096, 1476, 831] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1269, 5, 7, 2056, 27, 4003, 787, 38] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[293, 1, 1, 3741, 12, 4096, 5, 43] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[479, 0, 413, 915, 37, 3856, 2466, 26] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[5, 1, 0, 4042, 10, 4094, 21, 19] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[77, 16, 35, 3940, 12, 4052, 44, 16] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[203, 7, 1, 3906, 0, 3970, 105, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[3, 0, 1, 4065, 0, 4095, 14, 14] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[226, 277, 385, 987, 0, 981, 404, 0, 6, 1026, 1102, 0, 1476, 160, 831, 331] rerouted_tokens_per_ep_rank=[503, 1372, 981, 404, 1032, 1102, 1636, 1162] avg=1024.0000 max=1636 max_rank=6 min=404 min_rank=3 imbalance_ratio=1.597656 max_load_reduction=6556 max_load_reduction_pct=0.800293 moved_to_echo=4932 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[293, 884, 1, 987, 1, 981, 1288, 0, 12, 1026, 1102, 0, 5, 513, 43, 1056] rerouted_tokens_per_ep_rank=[1177, 988, 982, 1288, 1038, 1102, 518, 1099] avg=1024.0000 max=1288 max_rank=3 min=518 min_rank=6 imbalance_ratio=1.257812 max_load_reduction=6904 max_load_reduction_pct=0.842773 moved_to_echo=3757 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 955, 1, 986, 0, 981, 1391, 0, 10, 1026, 1101, 0, 21, 555, 19, 1141] rerouted_tokens_per_ep_rank=[960, 987, 981, 1391, 1036, 1101, 576, 1160] avg=1024.0000 max=1391 max_rank=3 min=576 min_rank=6 imbalance_ratio=1.358398 max_load_reduction=6801 max_load_reduction_pct=0.830200 moved_to_echo=3873 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1269, 486, 5, 964, 7, 959, 708, 0, 27, 1003, 1077, 0, 787, 282, 38, 580] rerouted_tokens_per_ep_rank=[1755, 969, 966, 708, 1030, 1077, 1069, 618] avg=1024.0000 max=1755 max_rank=0 min=618 min_rank=7 imbalance_ratio=1.713867 max_load_reduction=6437 max_load_reduction_pct=0.785767 moved_to_echo=3794 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[479, 220, 0, 934, 413, 929, 304, 0, 37, 971, 1022, 0, 2466, 129, 26, 262] rerouted_tokens_per_ep_rank=[699, 934, 1342, 304, 1008, 1022, 2595, 288] avg=1024.0000 max=2595 max_rank=6 min=288 min_rank=7 imbalance_ratio=2.534180 max_load_reduction=5597 max_load_reduction_pct=0.683228 moved_to_echo=4913 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[203, 923, 7, 956, 1, 951, 1344, 0, 0, 995, 1068, 0, 105, 536, 0, 1103] rerouted_tokens_per_ep_rank=[1126, 963, 952, 1344, 995, 1068, 641, 1103] avg=1024.0000 max=1344 max_rank=3 min=641 min_rank=6 imbalance_ratio=1.312500 max_load_reduction=6848 max_load_reduction_pct=0.835938 moved_to_echo=3807 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[77, 931, 16, 976, 35, 971, 1355, 0, 12, 1015, 1090, 0, 44, 541, 16, 1113] rerouted_tokens_per_ep_rank=[1008, 992, 1006, 1355, 1027, 1090, 585, 1129] avg=1024.0000 max=1355 max_rank=3 min=585 min_rank=6 imbalance_ratio=1.323242 max_load_reduction=6837 max_load_reduction_pct=0.834595 moved_to_echo=3831 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 961, 0, 987, 1, 981, 1398, 0, 0, 1026, 1101, 0, 14, 558, 14, 1148] rerouted_tokens_per_ep_rank=[964, 987, 982, 1398, 1026, 1101, 572, 1162] avg=1024.0000 max=1398 max_rank=3 min=572 min_rank=6 imbalance_ratio=1.365234 max_load_reduction=6794 max_load_reduction_pct=0.829346 moved_to_echo=3861 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8663 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8663, 8192, 7721] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8663, 8192, 7721] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8663, 8192, 7721] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8663, 8192, 7721] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8663, 8192, 7721] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8663, 8192, 7721] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8663, 8192, 7721] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7721 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8663, 8192, 7721] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=7721 actual_valid_recv=7808 observed_loss=-87 tokens_per_expert=[1024, 6784] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8663 actual_valid_recv=8704 observed_loss=-41 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4992, 3328] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2560, 5760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True + [2026-08-30 04:54:01] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 671.6 | throughput per GPU (TFLOP/s/GPU): 496.4 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 9.656821E+00 | loss scale: 1.0 | grad norm: 40.249 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[53, 129, 49, 296, 2085, 894, 796, 3890] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[34, 31, 91, 535, 2192, 697, 725, 3887] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[16, 11, 124, 273, 2620, 415, 906, 3827] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[75, 113, 147, 392, 1855, 664, 1068, 3878] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[78, 114, 89, 283, 2420, 294, 1136, 3778] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[35, 95, 155, 764, 1677, 667, 961, 3838] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[9, 109, 179, 405, 2503, 442, 721, 3824] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[32, 23, 47, 401, 2156, 599, 1014, 3920] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[53, 991, 129, 954, 49, 911, 296, 576, 987, 0, 894, 419, 796, 103, 1034, 0] rerouted_tokens_per_ep_rank=[1044, 1083, 960, 872, 987, 1313, 899, 1034] avg=1024.0000 max=1313 max_rank=5 min=872 min_rank=3 imbalance_ratio=1.282227 max_load_reduction=6879 max_load_reduction_pct=0.839722 moved_to_echo=4233 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[34, 990, 31, 953, 91, 910, 535, 606, 1038, 0, 697, 440, 725, 108, 1034, 0] rerouted_tokens_per_ep_rank=[1024, 984, 1001, 1141, 1038, 1137, 833, 1034] avg=1024.0000 max=1141 max_rank=3 min=833 min_rank=6 imbalance_ratio=1.114258 max_load_reduction=7051 max_load_reduction_pct=0.860718 moved_to_echo=4042 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[75, 988, 113, 951, 147, 908, 392, 513, 879, 0, 664, 372, 1068, 91, 1031, 0] rerouted_tokens_per_ep_rank=[1063, 1064, 1055, 905, 879, 1036, 1159, 1031] avg=1024.0000 max=1159 max_rank=6 min=879 min_rank=4 imbalance_ratio=1.131836 max_load_reduction=7033 max_load_reduction_pct=0.858521 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[78, 962, 114, 926, 89, 884, 283, 669, 1146, 0, 294, 486, 1136, 119, 1006, 0] rerouted_tokens_per_ep_rank=[1040, 1040, 973, 952, 1146, 780, 1255, 1006] avg=1024.0000 max=1255 max_rank=6 min=780 min_rank=5 imbalance_ratio=1.225586 max_load_reduction=6937 max_load_reduction_pct=0.846802 moved_to_echo=4187 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[35, 981, 95, 946, 155, 901, 764, 467, 783, 0, 667, 341, 961, 86, 1010, 0] rerouted_tokens_per_ep_rank=[1016, 1041, 1056, 1231, 783, 1008, 1047, 1010] avg=1024.0000 max=1231 max_rank=3 min=783 min_rank=4 imbalance_ratio=1.202148 max_load_reduction=6961 max_load_reduction_pct=0.849731 moved_to_echo=3848 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 975, 11, 938, 124, 896, 273, 724, 1241, 0, 415, 526, 906, 129, 1018, 0] rerouted_tokens_per_ep_rank=[991, 949, 1020, 997, 1241, 941, 1035, 1018] avg=1024.0000 max=1241 max_rank=4 min=941 min_rank=5 imbalance_ratio=1.211914 max_load_reduction=6951 max_load_reduction_pct=0.848511 moved_to_echo=4235 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[9, 974, 109, 938, 179, 895, 405, 692, 1185, 0, 442, 503, 721, 123, 1017, 0] rerouted_tokens_per_ep_rank=[983, 1047, 1074, 1097, 1185, 945, 844, 1017] avg=1024.0000 max=1185 max_rank=4 min=844 min_rank=6 imbalance_ratio=1.157227 max_load_reduction=7007 max_load_reduction_pct=0.855347 moved_to_echo=3991 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[32, 999, 23, 961, 47, 918, 401, 596, 1021, 0, 599, 433, 1014, 106, 1042, 0] rerouted_tokens_per_ep_rank=[1031, 984, 965, 997, 1021, 1032, 1120, 1042] avg=1024.0000 max=1120 max_rank=6 min=965 min_rank=2 imbalance_ratio=1.093750 max_load_reduction=7072 max_load_reduction_pct=0.863281 moved_to_echo=4215 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8280 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8104, 8192, 8280, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8104, 8192, 8280, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8104, 8192, 8280, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8104, 8192, 8280, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8104, 8192, 8280, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8104, 8192, 8280, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8104, 8192, 8280, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8104 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8104, 8192, 8280, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8104 actual_valid_recv=8192 observed_loss=-88 tokens_per_expert=[896, 7296] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4736, 3584] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8280 actual_valid_recv=8320 observed_loss=-40 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7424, 896] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3456, 4864] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[640, 7680] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[985, 0, 959, 0, 1584, 484, 356, 3824] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[983, 2, 797, 30, 1406, 961, 178, 3835] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1168, 4, 995, 16, 2014, 436, 362, 3197] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[835, 11, 1437, 20, 1292, 719, 290, 3588] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[785, 0, 1176, 2, 1095, 767, 375, 3992] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[691, 0, 1058, 50, 1391, 669, 453, 3880] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[702, 0, 1093, 2, 1325, 930, 298, 3842] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[815, 0, 1356, 0, 1705, 806, 146, 3364] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[985, 73, 0, 1058, 886, 0, 0, 1045, 1260, 0, 484, 324, 356, 658, 1063, 0] rerouted_tokens_per_ep_rank=[1058, 1058, 886, 1045, 1260, 808, 1014, 1063] avg=1024.0000 max=1260 max_rank=4 min=808 min_rank=5 imbalance_ratio=1.230469 max_load_reduction=6932 max_load_reduction_pct=0.846191 moved_to_echo=4145 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[983, 61, 2, 1061, 736, 0, 30, 1048, 1118, 0, 961, 288, 178, 660, 1066, 0] rerouted_tokens_per_ep_rank=[1044, 1063, 736, 1078, 1118, 1249, 838, 1066] avg=1024.0000 max=1249 max_rank=5 min=736 min_rank=2 imbalance_ratio=1.219727 max_load_reduction=6943 max_load_reduction_pct=0.847534 moved_to_echo=4271 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[835, 109, 11, 993, 1328, 0, 20, 981, 1028, 0, 719, 264, 290, 617, 997, 0] rerouted_tokens_per_ep_rank=[944, 1004, 1328, 1001, 1028, 983, 907, 997] avg=1024.0000 max=1328 max_rank=2 min=907 min_rank=6 imbalance_ratio=1.296875 max_load_reduction=6864 max_load_reduction_pct=0.837891 moved_to_echo=3915 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1168, 80, 4, 890, 915, 0, 16, 878, 1598, 0, 436, 416, 362, 553, 876, 0] rerouted_tokens_per_ep_rank=[1248, 894, 915, 894, 1598, 852, 915, 876] avg=1024.0000 max=1598 max_rank=4 min=852 min_rank=5 imbalance_ratio=1.560547 max_load_reduction=6594 max_load_reduction_pct=0.804932 moved_to_echo=4241 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[785, 90, 0, 1105, 1086, 0, 2, 1091, 871, 0, 767, 224, 375, 687, 1109, 0] rerouted_tokens_per_ep_rank=[875, 1105, 1086, 1093, 871, 991, 1062, 1109] avg=1024.0000 max=1109 max_rank=7 min=871 min_rank=4 imbalance_ratio=1.083008 max_load_reduction=7083 max_load_reduction_pct=0.864624 moved_to_echo=4033 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[691, 80, 0, 1074, 978, 0, 50, 1060, 1107, 0, 669, 284, 453, 668, 1078, 0] rerouted_tokens_per_ep_rank=[771, 1074, 978, 1110, 1107, 953, 1121, 1078] avg=1024.0000 max=1121 max_rank=6 min=771 min_rank=0 imbalance_ratio=1.094727 max_load_reduction=7071 max_load_reduction_pct=0.863159 moved_to_echo=4259 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[702, 83, 0, 1063, 1010, 0, 2, 1050, 1054, 0, 930, 271, 298, 661, 1068, 0] rerouted_tokens_per_ep_rank=[785, 1063, 1010, 1052, 1054, 1201, 959, 1068] avg=1024.0000 max=1201 max_rank=5 min=785 min_rank=0 imbalance_ratio=1.172852 max_load_reduction=6991 max_load_reduction_pct=0.853394 moved_to_echo=4282 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[815, 103, 0, 931, 1253, 0, 0, 919, 1356, 0, 806, 349, 146, 579, 935, 0] rerouted_tokens_per_ep_rank=[918, 931, 1253, 919, 1356, 1155, 725, 935] avg=1024.0000 max=1356 max_rank=4 min=725 min_rank=6 imbalance_ratio=1.324219 max_load_reduction=6836 max_load_reduction_pct=0.834473 moved_to_echo=4171 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7643, 8192, 8192, 8192, 9392, 8192, 7541, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9392 expected_overflow=0 exact_recv_per_rank=[7643, 8192, 8192, 8192, 9392, 8192, 7541, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7541 expected_overflow=0 exact_recv_per_rank=[7643, 8192, 8192, 8192, 9392, 8192, 7541, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7643, 8192, 8192, 8192, 9392, 8192, 7541, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7643 expected_overflow=0 exact_recv_per_rank=[7643, 8192, 8192, 8192, 9392, 8192, 7541, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7643, 8192, 8192, 8192, 9392, 8192, 7541, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7643, 8192, 8192, 8192, 9392, 8192, 7541, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7643, 8192, 8192, 8192, 9392, 8192, 7541, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=7643 actual_valid_recv=7808 observed_loss=-165 tokens_per_expert=[7040, 768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7541 actual_valid_recv=7680 observed_loss=-139 tokens_per_expert=[2560, 5120] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5888, 2432] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=9392 actual_valid_recv=9472 observed_loss=-80 tokens_per_expert=[9472, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=9472 slack_slots=6912 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3873, 19, 1, 9, 7, 3550, 118, 615] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4037, 517, 0, 230, 14, 2638, 133, 623] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3971, 720, 1, 1, 8, 2337, 122, 1032] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3905, 248, 0, 72, 0, 2194, 118, 1655] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3958, 489, 2, 6, 0, 2958, 127, 652] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4084, 84, 0, 1, 0, 3676, 112, 235] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4040, 293, 0, 27, 0, 2782, 112, 938] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4080, 72, 6, 0, 0, 3851, 2, 181] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[995, 0, 19, 851, 1, 991, 9, 898, 7, 989, 1278, 0, 118, 1087, 615, 334] rerouted_tokens_per_ep_rank=[995, 870, 992, 907, 996, 1278, 1205, 949] avg=1024.0000 max=1278 max_rank=5 min=870 min_rank=1 imbalance_ratio=1.248047 max_load_reduction=6914 max_load_reduction_pct=0.843994 moved_to_echo=4428 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1003, 0, 248, 525, 0, 1000, 72, 905, 0, 997, 791, 0, 118, 672, 1655, 206] rerouted_tokens_per_ep_rank=[1003, 773, 1000, 977, 997, 791, 790, 1861] avg=1024.0000 max=1861 max_rank=7 min=773 min_rank=1 imbalance_ratio=1.817383 max_load_reduction=6331 max_load_reduction_pct=0.772827 moved_to_echo=4439 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 489, 709, 2, 1013, 6, 918, 0, 1011, 1065, 0, 127, 906, 652, 278] rerouted_tokens_per_ep_rank=[1016, 1198, 1015, 924, 1011, 1065, 1033, 930] avg=1024.0000 max=1198 max_rank=1 min=924 min_rank=3 imbalance_ratio=1.169922 max_load_reduction=6994 max_load_reduction_pct=0.853760 moved_to_echo=4039 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1037, 0, 517, 632, 0, 1033, 230, 936, 14, 1031, 950, 0, 133, 808, 623, 248] rerouted_tokens_per_ep_rank=[1037, 1149, 1033, 1166, 1045, 950, 941, 871] avg=1024.0000 max=1166 max_rank=3 min=871 min_rank=7 imbalance_ratio=1.138672 max_load_reduction=7026 max_load_reduction_pct=0.857666 moved_to_echo=3807 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1007, 0, 720, 563, 1, 1022, 1, 924, 8, 1018, 832, 0, 122, 718, 1032, 224] rerouted_tokens_per_ep_rank=[1007, 1283, 1023, 925, 1026, 832, 840, 1256] avg=1024.0000 max=1283 max_rank=1 min=832 min_rank=5 imbalance_ratio=1.252930 max_load_reduction=6909 max_load_reduction_pct=0.843384 moved_to_echo=3954 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1049, 0, 84, 881, 0, 1045, 1, 947, 0, 1043, 1323, 0, 112, 1126, 235, 346] rerouted_tokens_per_ep_rank=[1049, 965, 1045, 948, 1043, 1323, 1238, 581] avg=1024.0000 max=1323 max_rank=5 min=581 min_rank=7 imbalance_ratio=1.291992 max_load_reduction=6869 max_load_reduction_pct=0.838501 moved_to_echo=4185 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1037, 0, 293, 666, 0, 1034, 27, 937, 0, 1032, 1002, 0, 112, 852, 938, 262] rerouted_tokens_per_ep_rank=[1037, 959, 1034, 964, 1032, 1002, 964, 1200] avg=1024.0000 max=1200 max_rank=7 min=959 min_rank=1 imbalance_ratio=1.171875 max_load_reduction=6992 max_load_reduction_pct=0.853516 moved_to_echo=4198 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1048, 0, 72, 923, 6, 1044, 0, 946, 0, 1042, 1386, 0, 2, 1179, 181, 363] rerouted_tokens_per_ep_rank=[1048, 995, 1050, 946, 1042, 1386, 1181, 544] avg=1024.0000 max=1386 max_rank=5 min=544 min_rank=7 imbalance_ratio=1.353516 max_load_reduction=6806 max_load_reduction_pct=0.830811 moved_to_echo=4153 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7757 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7757, 8192, 8627, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8627 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7757, 8192, 8627, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7757, 8192, 8627, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7757, 8192, 8627, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7757, 8192, 8627, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7757, 8192, 8627, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7757, 8192, 8627, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7757, 8192, 8627, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2560, 5760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[896, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6016, 2304] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8627 actual_valid_recv=8704 observed_loss=-77 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=7757 actual_valid_recv=7808 observed_loss=-51 tokens_per_expert=[384, 7424] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[32, 0, 19, 21, 0, 0, 4096, 4024] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 4, 4, 0, 0, 4094, 4088] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[57, 4, 15, 5, 8, 10, 4072, 4021] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[177, 2, 89, 0, 1, 2, 4096, 3825] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[458, 1, 48, 0, 3, 0, 4093, 3589] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[136, 0, 43, 6, 9, 3, 4096, 3899] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[9, 0, 20, 98, 2, 8, 3971, 4084] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[5, 4, 7, 2, 4, 0, 4087, 4083] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[177, 885, 2, 1027, 89, 961, 0, 974, 1, 1012, 2, 1026, 1031, 0, 1005, 0] rerouted_tokens_per_ep_rank=[1062, 1029, 1050, 974, 1013, 1028, 1031, 1005] avg=1024.0000 max=1062 max_rank=0 min=974 min_rank=3 imbalance_ratio=1.037109 max_load_reduction=7130 max_load_reduction_pct=0.870361 moved_to_echo=4077 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[458, 830, 1, 1026, 48, 902, 0, 914, 3, 1012, 0, 1025, 1030, 0, 943, 0] rerouted_tokens_per_ep_rank=[1288, 1027, 950, 914, 1015, 1025, 1030, 943] avg=1024.0000 max=1288 max_rank=0 min=914 min_rank=3 imbalance_ratio=1.257812 max_load_reduction=6904 max_load_reduction_pct=0.842773 moved_to_echo=4013 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[57, 930, 4, 1021, 15, 1010, 5, 1024, 8, 1006, 10, 1020, 1025, 0, 1057, 0] rerouted_tokens_per_ep_rank=[987, 1025, 1025, 1029, 1014, 1030, 1025, 1057] avg=1024.0000 max=1057 max_rank=7 min=987 min_rank=0 imbalance_ratio=1.032227 max_load_reduction=7135 max_load_reduction_pct=0.870972 moved_to_echo=4126 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 945, 0, 996, 20, 1026, 98, 1040, 2, 982, 8, 994, 999, 0, 1073, 0] rerouted_tokens_per_ep_rank=[954, 996, 1046, 1138, 984, 1002, 999, 1073] avg=1024.0000 max=1138 max_rank=3 min=954 min_rank=0 imbalance_ratio=1.111328 max_load_reduction=7054 max_load_reduction_pct=0.861084 moved_to_echo=4058 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 946, 0, 1027, 4, 1027, 4, 1041, 0, 1012, 0, 1025, 1030, 0, 1074, 0] rerouted_tokens_per_ep_rank=[948, 1027, 1031, 1045, 1012, 1025, 1030, 1074] avg=1024.0000 max=1074 max_rank=7 min=948 min_rank=0 imbalance_ratio=1.048828 max_load_reduction=7118 max_load_reduction_pct=0.868896 moved_to_echo=4141 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[136, 905, 0, 1032, 43, 984, 6, 998, 9, 1017, 3, 1030, 1017, 0, 1012, 0] rerouted_tokens_per_ep_rank=[1041, 1032, 1027, 1004, 1026, 1033, 1017, 1012] avg=1024.0000 max=1041 max_rank=0 min=1004 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4088 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[32, 931, 0, 1027, 19, 1011, 21, 1025, 0, 1012, 0, 1026, 1031, 0, 1057, 0] rerouted_tokens_per_ep_rank=[963, 1027, 1030, 1046, 1012, 1026, 1031, 1057] avg=1024.0000 max=1057 max_rank=7 min=963 min_rank=0 imbalance_ratio=1.032227 max_load_reduction=7135 max_load_reduction_pct=0.870972 moved_to_echo=4126 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[5, 944, 4, 1025, 7, 1026, 2, 1040, 4, 1010, 0, 1023, 1029, 0, 1073, 0] rerouted_tokens_per_ep_rank=[949, 1029, 1033, 1042, 1014, 1023, 1029, 1073] avg=1024.0000 max=1073 max_rank=7 min=949 min_rank=0 imbalance_ratio=1.047852 max_load_reduction=7119 max_load_reduction_pct=0.869019 moved_to_echo=4139 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8090, 8192, 8192, 8294] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8090 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8090, 8192, 8192, 8294] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8294 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8090, 8192, 8192, 8294] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8090, 8192, 8192, 8294] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8090, 8192, 8192, 8294] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8090, 8192, 8192, 8294] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8090, 8192, 8192, 8294] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8090, 8192, 8192, 8294] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[896, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8090 actual_valid_recv=8192 observed_loss=-102 tokens_per_expert=[128, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8294 actual_valid_recv=8320 observed_loss=-26 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 2668, 28, 1, 0, 1440, 0, 4055] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[32, 1948, 0, 0, 0, 2138, 3, 4071] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 803, 0, 0, 10, 3345, 1, 4033] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 2216, 0, 1, 0, 1915, 0, 4060] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1457, 0, 0, 9, 2636, 7, 4083] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1900, 0, 0, 0, 2292, 8, 3992] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 2587, 0, 0, 0, 1509, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 1934, 0, 0, 3, 2259, 0, 3996] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[32, 919, 1029, 0, 0, 995, 0, 1029, 0, 984, 1143, 0, 3, 1027, 1031, 0] rerouted_tokens_per_ep_rank=[951, 1029, 995, 1029, 984, 1143, 1030, 1031] avg=1024.0000 max=1143 max_rank=5 min=951 min_rank=0 imbalance_ratio=1.116211 max_load_reduction=7049 max_load_reduction_pct=0.860474 moved_to_echo=4188 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1259, 1409, 0, 28, 670, 1, 1025, 0, 980, 770, 0, 0, 1023, 1027, 0] rerouted_tokens_per_ep_rank=[1259, 1409, 698, 1026, 980, 770, 1023, 1027] avg=1024.0000 max=1409 max_rank=1 min=698 min_rank=2 imbalance_ratio=1.375977 max_load_reduction=6783 max_load_reduction_pct=0.828003 moved_to_echo=3800 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1045, 1171, 0, 0, 891, 1, 1026, 0, 981, 1024, 0, 0, 1024, 1029, 0] rerouted_tokens_per_ep_rank=[1045, 1171, 891, 1027, 981, 1024, 1024, 1029] avg=1024.0000 max=1171 max_rank=1 min=891 min_rank=2 imbalance_ratio=1.143555 max_load_reduction=7021 max_load_reduction_pct=0.857056 moved_to_echo=4058 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 687, 770, 0, 0, 1227, 0, 1032, 9, 987, 1409, 0, 7, 1030, 1034, 0] rerouted_tokens_per_ep_rank=[687, 770, 1227, 1032, 996, 1409, 1037, 1034] avg=1024.0000 max=1409 max_rank=5 min=687 min_rank=0 imbalance_ratio=1.375977 max_load_reduction=6783 max_load_reduction_pct=0.828003 moved_to_echo=4476 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 383, 420, 0, 0, 1561, 0, 1024, 10, 978, 1784, 0, 1, 1021, 1010, 0] rerouted_tokens_per_ep_rank=[383, 420, 1561, 1024, 988, 1784, 1022, 1010] avg=1024.0000 max=1784 max_rank=5 min=383 min_rank=0 imbalance_ratio=1.742188 max_load_reduction=6408 max_load_reduction_pct=0.782227 moved_to_echo=4804 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1220, 1367, 0, 0, 702, 0, 1035, 0, 990, 807, 0, 0, 1033, 1038, 0] rerouted_tokens_per_ep_rank=[1220, 1367, 702, 1035, 990, 807, 1033, 1038] avg=1024.0000 max=1367 max_rank=1 min=702 min_rank=2 imbalance_ratio=1.334961 max_load_reduction=6825 max_load_reduction_pct=0.833130 moved_to_echo=3868 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 896, 1004, 0, 0, 1067, 0, 1009, 0, 965, 1225, 0, 8, 1007, 1011, 0] rerouted_tokens_per_ep_rank=[896, 1004, 1067, 1009, 965, 1225, 1015, 1011] avg=1024.0000 max=1225 max_rank=5 min=896 min_rank=0 imbalance_ratio=1.196289 max_load_reduction=6967 max_load_reduction_pct=0.850464 moved_to_echo=4216 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 912, 1022, 0, 0, 1051, 0, 1010, 3, 966, 1208, 0, 0, 1008, 1012, 0] rerouted_tokens_per_ep_rank=[912, 1022, 1051, 1010, 969, 1208, 1008, 1012] avg=1024.0000 max=1208 max_rank=5 min=912 min_rank=0 imbalance_ratio=1.179688 max_load_reduction=6984 max_load_reduction_pct=0.852539 moved_to_echo=4197 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7353, 8192, 8192, 8192, 7853, 9370, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7353, 8192, 8192, 8192, 7853, 9370, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7353, 8192, 8192, 8192, 7853, 9370, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9370 expected_overflow=0 exact_recv_per_rank=[7353, 8192, 8192, 8192, 7853, 9370, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7853 expected_overflow=0 exact_recv_per_rank=[7353, 8192, 8192, 8192, 7853, 9370, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7353, 8192, 8192, 8192, 7853, 9370, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7353, 8192, 8192, 8192, 7853, 9370, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7353 expected_overflow=0 exact_recv_per_rank=[7353, 8192, 8192, 8192, 7853, 9370, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=7353 actual_valid_recv=7552 observed_loss=-199 tokens_per_expert=[128, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7853 actual_valid_recv=8064 observed_loss=-211 tokens_per_expert=[128, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=9370 actual_valid_recv=9472 observed_loss=-102 tokens_per_expert=[9472, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8064 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=9472 slack_slots=6912 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[57, 23, 1120, 3130, 3, 7, 3849, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[312, 0, 2712, 1547, 6, 0, 3615, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[70, 2, 3279, 1584, 0, 0, 3257, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[17, 0, 3258, 1970, 0, 0, 2947, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[45, 9, 2097, 2291, 0, 0, 3750, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[171, 0, 2541, 1709, 0, 2, 3767, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[35, 8, 1940, 2909, 0, 0, 3300, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[52, 3, 1781, 2562, 0, 0, 3794, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[45, 961, 9, 679, 918, 0, 1330, 0, 0, 500, 0, 1085, 1580, 0, 0, 1085] rerouted_tokens_per_ep_rank=[1006, 688, 918, 1330, 500, 1085, 1580, 1085] avg=1024.0000 max=1580 max_rank=6 min=500 min_rank=4 imbalance_ratio=1.542969 max_load_reduction=6612 max_load_reduction_pct=0.807129 moved_to_echo=4250 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[70, 665, 2, 1062, 1436, 0, 919, 0, 0, 781, 0, 942, 1373, 0, 0, 942] rerouted_tokens_per_ep_rank=[735, 1064, 1436, 919, 781, 942, 1373, 942] avg=1024.0000 max=1436 max_rank=2 min=735 min_rank=0 imbalance_ratio=1.402344 max_load_reduction=6756 max_load_reduction_pct=0.824707 moved_to_echo=4038 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[171, 717, 0, 823, 1113, 0, 992, 0, 0, 605, 2, 1090, 1587, 0, 2, 1090] rerouted_tokens_per_ep_rank=[888, 823, 1113, 992, 605, 1092, 1587, 1092] avg=1024.0000 max=1587 max_rank=6 min=605 min_rank=4 imbalance_ratio=1.549805 max_load_reduction=6605 max_load_reduction_pct=0.806274 moved_to_echo=4376 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[57, 1314, 23, 363, 490, 0, 1816, 0, 3, 267, 7, 1113, 1622, 0, 3, 1114] rerouted_tokens_per_ep_rank=[1371, 386, 490, 1816, 270, 1120, 1622, 1117] avg=1024.0000 max=1816 max_rank=3 min=270 min_rank=4 imbalance_ratio=1.773438 max_load_reduction=6376 max_load_reduction_pct=0.778320 moved_to_echo=4129 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[17, 827, 0, 1055, 1427, 0, 1143, 0, 0, 776, 0, 852, 1242, 0, 0, 853] rerouted_tokens_per_ep_rank=[844, 1055, 1427, 1143, 776, 852, 1242, 853] avg=1024.0000 max=1427 max_rank=2 min=776 min_rank=4 imbalance_ratio=1.393555 max_load_reduction=6765 max_load_reduction_pct=0.825806 moved_to_echo=3723 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[312, 653, 0, 883, 1178, 0, 894, 0, 6, 651, 0, 1050, 1515, 0, 0, 1050] rerouted_tokens_per_ep_rank=[965, 883, 1178, 894, 657, 1050, 1515, 1050] avg=1024.0000 max=1515 max_rank=6 min=657 min_rank=4 imbalance_ratio=1.479492 max_load_reduction=6677 max_load_reduction_pct=0.815063 moved_to_echo=4272 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[52, 1075, 3, 577, 780, 0, 1487, 0, 0, 424, 0, 1097, 1599, 0, 0, 1098] rerouted_tokens_per_ep_rank=[1127, 580, 780, 1487, 424, 1097, 1599, 1098] avg=1024.0000 max=1599 max_rank=6 min=424 min_rank=4 imbalance_ratio=1.561523 max_load_reduction=6593 max_load_reduction_pct=0.804810 moved_to_echo=4218 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[35, 1221, 8, 628, 850, 0, 1688, 0, 0, 462, 0, 954, 1391, 0, 0, 955] rerouted_tokens_per_ep_rank=[1256, 636, 850, 1688, 462, 954, 1391, 955] avg=1024.0000 max=1688 max_rank=3 min=462 min_rank=4 imbalance_ratio=1.648438 max_load_reduction=6504 max_load_reduction_pct=0.793945 moved_to_echo=3762 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10269 expected_overflow=0 exact_recv_per_rank=[8192, 6115, 8192, 10269, 4475, 8192, 11909, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6115 expected_overflow=0 exact_recv_per_rank=[8192, 6115, 8192, 10269, 4475, 8192, 11909, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4475 expected_overflow=0 exact_recv_per_rank=[8192, 6115, 8192, 10269, 4475, 8192, 11909, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6115, 8192, 10269, 4475, 8192, 11909, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6115, 8192, 10269, 4475, 8192, 11909, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6115, 8192, 10269, 4475, 8192, 11909, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 6115, 8192, 10269, 4475, 8192, 11909, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11909 expected_overflow=0 exact_recv_per_rank=[8192, 6115, 8192, 10269, 4475, 8192, 11909, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=4475 actual_valid_recv=4608 observed_loss=-133 tokens_per_expert=[128, 4480] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=6115 actual_valid_recv=6272 observed_loss=-157 tokens_per_expert=[128, 6144] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=10269 actual_valid_recv=10368 observed_loss=-99 tokens_per_expert=[10368, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=11909 actual_valid_recv=12032 observed_loss=-123 tokens_per_expert=[12032, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=4608 slack_slots=11776 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=6272 slack_slots=10112 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=10368 slack_slots=6016 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=12032 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[2, 341, 0, 3768, 0, 15, 4066, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 79, 0, 4023, 0, 0, 4090, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[2, 2, 0, 4096, 0, 1, 4091, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 161, 0, 4060, 7, 0, 3964, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[7, 40, 0, 4045, 0, 0, 4094, 6] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 37, 0, 4087, 2, 0, 4066, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 11, 0, 4095, 0, 0, 4086, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 18, 2, 4071, 0, 7, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1020, 79, 936, 0, 1029, 1048, 0, 0, 1003, 0, 1019, 1030, 0, 0, 1028] rerouted_tokens_per_ep_rank=[1020, 1015, 1029, 1048, 1003, 1019, 1030, 1028] avg=1024.0000 max=1048 max_rank=3 min=1003 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=7144 max_load_reduction_pct=0.872070 moved_to_echo=4080 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 161, 944, 0, 997, 1058, 0, 7, 972, 0, 1028, 999, 0, 0, 996] rerouted_tokens_per_ep_rank=[1030, 1105, 997, 1058, 979, 1028, 999, 996] avg=1024.0000 max=1105 max_rank=1 min=979 min_rank=4 imbalance_ratio=1.079102 max_load_reduction=7087 max_load_reduction_pct=0.865112 moved_to_echo=4002 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 955, 341, 876, 0, 1023, 983, 0, 0, 997, 15, 954, 1024, 0, 0, 1022] rerouted_tokens_per_ep_rank=[957, 1217, 1023, 983, 997, 969, 1024, 1022] avg=1024.0000 max=1217 max_rank=1 min=957 min_rank=0 imbalance_ratio=1.188477 max_load_reduction=6975 max_load_reduction_pct=0.851440 moved_to_echo=4012 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 1039, 2, 953, 0, 1029, 1067, 0, 0, 1003, 1, 1037, 1031, 0, 0, 1028] rerouted_tokens_per_ep_rank=[1041, 955, 1029, 1067, 1003, 1038, 1031, 1028] avg=1024.0000 max=1067 max_rank=3 min=955 min_rank=1 imbalance_ratio=1.041992 max_load_reduction=7125 max_load_reduction_pct=0.869751 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[7, 1031, 40, 945, 0, 1031, 1041, 0, 0, 1005, 0, 1028, 1024, 0, 6, 1034] rerouted_tokens_per_ep_rank=[1038, 985, 1031, 1041, 1005, 1028, 1024, 1040] avg=1024.0000 max=1041 max_rank=3 min=985 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1036, 37, 950, 0, 1023, 1066, 0, 2, 997, 0, 1035, 1024, 0, 0, 1022] rerouted_tokens_per_ep_rank=[1036, 987, 1023, 1066, 999, 1035, 1024, 1022] avg=1024.0000 max=1066 max_rank=3 min=987 min_rank=1 imbalance_ratio=1.041016 max_load_reduction=7126 max_load_reduction_pct=0.869873 moved_to_echo=4080 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1038, 11, 952, 0, 1028, 1068, 0, 0, 1002, 0, 1037, 1029, 0, 0, 1027] rerouted_tokens_per_ep_rank=[1038, 963, 1028, 1068, 1002, 1037, 1029, 1027] avg=1024.0000 max=1068 max_rank=3 min=963 min_rank=1 imbalance_ratio=1.042969 max_load_reduction=7124 max_load_reduction_pct=0.869629 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1032, 18, 947, 2, 1030, 1061, 0, 0, 1004, 7, 1031, 1031, 0, 0, 1029] rerouted_tokens_per_ep_rank=[1032, 965, 1032, 1061, 1004, 1038, 1031, 1029] avg=1024.0000 max=1061 max_rank=3 min=965 min_rank=1 imbalance_ratio=1.036133 max_load_reduction=7131 max_load_reduction_pct=0.870483 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8392 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8392, 7992, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8392, 7992, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8392, 7992, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8392, 7992, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7992 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8392, 7992, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8392, 7992, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8392, 7992, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8392, 7992, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7992 actual_valid_recv=8192 observed_loss=-200 tokens_per_expert=[128, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8392 actual_valid_recv=8448 observed_loss=-56 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 3732, 0, 835, 943, 2675, 0, 7] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3, 3961, 2, 1444, 1039, 1715, 11, 17] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 3615, 0, 1393, 547, 2631, 0, 6] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 3809, 0, 1004, 763, 2616, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 3744, 0, 379, 616, 3446, 0, 7] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 3705, 0, 1222, 505, 2760, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 3694, 1, 257, 1055, 3175, 10, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 4040, 0, 769, 205, 3171, 6, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1008, 1011, 0, 0, 1008, 835, 107, 943, 303, 1282, 0, 0, 705, 7, 983] rerouted_tokens_per_ep_rank=[1008, 1011, 1008, 942, 1246, 1282, 705, 990] avg=1024.0000 max=1282 max_rank=5 min=705 min_rank=6 imbalance_ratio=1.251953 max_load_reduction=6910 max_load_reduction_pct=0.843506 moved_to_echo=4223 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 1070, 1072, 0, 2, 1070, 1444, 68, 1039, 194, 823, 0, 11, 749, 17, 630] rerouted_tokens_per_ep_rank=[1073, 1072, 1072, 1512, 1233, 823, 760, 647] avg=1024.0000 max=1512 max_rank=3 min=647 min_rank=7 imbalance_ratio=1.476562 max_load_reduction=6680 max_load_reduction_pct=0.815430 moved_to_echo=3463 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1011, 1014, 0, 0, 1011, 379, 138, 616, 391, 1651, 0, 0, 708, 7, 1266] rerouted_tokens_per_ep_rank=[1011, 1014, 1011, 517, 1007, 1651, 708, 1273] avg=1024.0000 max=1651 max_rank=5 min=517 min_rank=3 imbalance_ratio=1.612305 max_load_reduction=6541 max_load_reduction_pct=0.798462 moved_to_echo=4639 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1001, 1003, 0, 0, 1001, 1222, 110, 505, 313, 1323, 0, 0, 700, 0, 1014] rerouted_tokens_per_ep_rank=[1001, 1003, 1001, 1332, 818, 1323, 700, 1014] avg=1024.0000 max=1332 max_rank=3 min=700 min_rank=6 imbalance_ratio=1.300781 max_load_reduction=6860 max_load_reduction_pct=0.837402 moved_to_echo=3855 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1002, 989, 0, 1, 1002, 257, 130, 1055, 365, 1511, 0, 10, 701, 0, 1169] rerouted_tokens_per_ep_rank=[1002, 989, 1003, 387, 1420, 1511, 711, 1169] avg=1024.0000 max=1511 max_rank=5 min=387 min_rank=3 imbalance_ratio=1.475586 max_load_reduction=6681 max_load_reduction_pct=0.815552 moved_to_echo=4811 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1029, 1031, 0, 0, 1029, 1004, 104, 763, 296, 1255, 0, 0, 720, 0, 961] rerouted_tokens_per_ep_rank=[1029, 1031, 1029, 1108, 1059, 1255, 720, 961] avg=1024.0000 max=1255 max_rank=5 min=720 min_rank=6 imbalance_ratio=1.225586 max_load_reduction=6937 max_load_reduction_pct=0.846802 moved_to_echo=3995 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 977, 978, 0, 0, 977, 1393, 105, 547, 298, 1262, 0, 0, 683, 6, 966] rerouted_tokens_per_ep_rank=[977, 978, 977, 1498, 845, 1262, 683, 972] avg=1024.0000 max=1498 max_rank=3 min=683 min_rank=6 imbalance_ratio=1.462891 max_load_reduction=6694 max_load_reduction_pct=0.817139 moved_to_echo=3762 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1091, 1094, 0, 0, 1091, 769, 127, 205, 359, 1520, 0, 6, 764, 1, 1165] rerouted_tokens_per_ep_rank=[1091, 1094, 1091, 896, 564, 1520, 770, 1166] avg=1024.0000 max=1520 max_rank=5 min=564 min_rank=4 imbalance_ratio=1.484375 max_load_reduction=6672 max_load_reduction_pct=0.814453 moved_to_echo=4020 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 10627, 5757, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 10627, 5757, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 10627, 5757, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 10627, 5757, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5757 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 10627, 5757, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 10627, 5757, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10627 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 10627, 5757, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 10627, 5757, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=5757 actual_valid_recv=5888 observed_loss=-131 tokens_per_expert=[128, 5760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7424, 896] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5760, 2560] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=10627 actual_valid_recv=10752 observed_loss=-125 tokens_per_expert=[10752, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=5888 slack_slots=10496 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=10752 slack_slots=5632 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[928, 0, 1, 1, 0, 12, 4086, 3164] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2296, 0, 0, 0, 0, 175, 4095, 1626] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[948, 0, 0, 0, 2, 835, 4094, 2313] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[3475, 0, 0, 0, 2, 81, 4085, 549] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1725, 0, 1, 0, 0, 23, 4095, 2348] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[2103, 8, 2, 0, 6, 53, 4087, 1933] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[2789, 0, 8, 9, 8, 197, 3961, 1220] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1465, 0, 0, 2, 8, 165, 4089, 2463] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[492, 0, 0, 1026, 1, 1007, 1, 1025, 0, 436, 12, 1347, 1028, 0, 1817, 0] rerouted_tokens_per_ep_rank=[492, 1026, 1008, 1026, 436, 1359, 1028, 1817] avg=1024.0000 max=1817 max_rank=7 min=436 min_rank=4 imbalance_ratio=1.774414 max_load_reduction=6375 max_load_reduction_pct=0.778198 moved_to_echo=4640 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1842, 0, 0, 1025, 0, 1007, 0, 1025, 2, 1633, 81, 233, 1028, 0, 316, 0] rerouted_tokens_per_ep_rank=[1842, 1025, 1007, 1025, 1635, 314, 1028, 316] avg=1024.0000 max=1842 max_rank=0 min=314 min_rank=5 imbalance_ratio=1.798828 max_load_reduction=6350 max_load_reduction_pct=0.775146 moved_to_echo=3293 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[503, 0, 0, 1028, 0, 1009, 0, 1027, 2, 445, 835, 985, 1030, 0, 1328, 0] rerouted_tokens_per_ep_rank=[503, 1028, 1009, 1027, 447, 1820, 1030, 1328] avg=1024.0000 max=1820 max_rank=5 min=447 min_rank=4 imbalance_ratio=1.777344 max_load_reduction=6372 max_load_reduction_pct=0.777832 moved_to_echo=4625 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1111, 0, 8, 1029, 2, 1011, 0, 1029, 6, 992, 53, 826, 1018, 0, 1107, 0] rerouted_tokens_per_ep_rank=[1111, 1037, 1013, 1029, 998, 879, 1018, 1107] avg=1024.0000 max=1111 max_rank=0 min=879 min_rank=5 imbalance_ratio=1.084961 max_load_reduction=7081 max_load_reduction_pct=0.864380 moved_to_echo=4002 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1479, 0, 0, 994, 8, 976, 9, 994, 8, 1310, 197, 519, 997, 0, 701, 0] rerouted_tokens_per_ep_rank=[1479, 994, 984, 1003, 1318, 716, 997, 701] avg=1024.0000 max=1479 max_rank=0 min=701 min_rank=7 imbalance_ratio=1.444336 max_load_reduction=6713 max_load_reduction_pct=0.819458 moved_to_echo=3732 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1217, 0, 0, 1028, 0, 1009, 0, 1027, 0, 1079, 175, 692, 1031, 0, 934, 0] rerouted_tokens_per_ep_rank=[1217, 1028, 1009, 1027, 1079, 867, 1031, 934] avg=1024.0000 max=1217 max_rank=0 min=867 min_rank=5 imbalance_ratio=1.188477 max_load_reduction=6975 max_load_reduction_pct=0.851440 moved_to_echo=3911 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[915, 0, 0, 1028, 1, 1009, 0, 1027, 0, 810, 23, 1000, 1031, 0, 1348, 0] rerouted_tokens_per_ep_rank=[915, 1028, 1010, 1027, 810, 1023, 1031, 1348] avg=1024.0000 max=1348 max_rank=7 min=810 min_rank=4 imbalance_ratio=1.316406 max_load_reduction=6844 max_load_reduction_pct=0.835449 moved_to_echo=4212 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[777, 0, 0, 1026, 0, 1008, 2, 1026, 8, 688, 165, 1049, 1029, 0, 1414, 0] rerouted_tokens_per_ep_rank=[777, 1026, 1008, 1028, 696, 1214, 1029, 1414] avg=1024.0000 max=1414 max_rank=7 min=696 min_rank=4 imbalance_ratio=1.380859 max_load_reduction=6778 max_load_reduction_pct=0.827393 moved_to_echo=4353 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8336, 8192, 8048, 8192, 7419, 8192, 8192, 8965] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8965 expected_overflow=0 exact_recv_per_rank=[8336, 8192, 8048, 8192, 7419, 8192, 8192, 8965] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8336 expected_overflow=0 exact_recv_per_rank=[8336, 8192, 8048, 8192, 7419, 8192, 8192, 8965] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8336, 8192, 8048, 8192, 7419, 8192, 8192, 8965] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8048 expected_overflow=0 exact_recv_per_rank=[8336, 8192, 8048, 8192, 7419, 8192, 8192, 8965] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8336, 8192, 8048, 8192, 7419, 8192, 8192, 8965] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7419 expected_overflow=0 exact_recv_per_rank=[8336, 8192, 8048, 8192, 7419, 8192, 8192, 8965] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8336, 8192, 8048, 8192, 7419, 8192, 8192, 8965] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7419 actual_valid_recv=7552 observed_loss=-133 tokens_per_expert=[128, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8048 actual_valid_recv=8192 observed_loss=-144 tokens_per_expert=[128, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8336 actual_valid_recv=8448 observed_loss=-112 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1664, 6656] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8965 actual_valid_recv=9088 observed_loss=-123 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 6, 4090, 0, 0, 57, 4039, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1, 4, 4092, 0, 1, 30, 4064, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 4086, 2, 8, 118, 3968, 9] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 17, 4096, 0, 0, 198, 3881, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[7, 23, 4064, 0, 1, 77, 4020, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 101, 3995, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1, 3, 4096, 0, 0, 127, 3965, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 6, 4096, 2, 3, 72, 4013, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1019, 6, 1028, 1025, 0, 0, 1023, 0, 1034, 57, 937, 1040, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1019, 1034, 1025, 1023, 1034, 994, 1040, 1023] avg=1024.0000 max=1040 max_rank=6 min=994 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1020, 17, 987, 1027, 0, 0, 1025, 0, 993, 198, 900, 1001, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1020, 1004, 1027, 1025, 993, 1098, 1001, 1024] avg=1024.0000 max=1098 max_rank=5 min=993 min_rank=4 imbalance_ratio=1.072266 max_load_reduction=7094 max_load_reduction_pct=0.865967 moved_to_echo=4116 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 1019, 4, 1034, 1026, 0, 0, 1024, 1, 1040, 30, 942, 1048, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1020, 1038, 1026, 1024, 1041, 972, 1048, 1023] avg=1024.0000 max=1048 max_rank=6 min=972 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=7144 max_load_reduction_pct=0.872070 moved_to_echo=4084 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7, 1012, 23, 1023, 1019, 0, 0, 1017, 1, 1029, 77, 932, 1036, 0, 0, 1016] rerouted_tokens_per_ep_rank=[1019, 1046, 1019, 1017, 1030, 1009, 1036, 1016] avg=1024.0000 max=1046 max_rank=1 min=1009 min_rank=5 imbalance_ratio=1.021484 max_load_reduction=7146 max_load_reduction_pct=0.872314 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1020, 0, 1016, 1027, 0, 0, 1025, 0, 1022, 101, 926, 1031, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1020, 1016, 1027, 1025, 1022, 1027, 1031, 1024] avg=1024.0000 max=1031 max_rank=6 min=1016 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1014, 1014, 0, 2, 1024, 8, 1019, 118, 925, 1010, 0, 9, 1025] rerouted_tokens_per_ep_rank=[1023, 1015, 1014, 1026, 1027, 1043, 1010, 1034] avg=1024.0000 max=1043 max_rank=5 min=1010 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=7149 max_load_reduction_pct=0.872681 moved_to_echo=4114 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 1020, 3, 1009, 1027, 0, 0, 1025, 0, 1015, 127, 919, 1022, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1021, 1012, 1027, 1025, 1015, 1046, 1022, 1024] avg=1024.0000 max=1046 max_rank=5 min=1012 min_rank=1 imbalance_ratio=1.021484 max_load_reduction=7146 max_load_reduction_pct=0.872314 moved_to_echo=4107 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1020, 6, 1021, 1027, 0, 2, 1025, 3, 1027, 72, 931, 1034, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1020, 1027, 1027, 1027, 1030, 1003, 1034, 1024] avg=1024.0000 max=1034 max_rank=6 min=1003 min_rank=5 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8162, 8192, 8192, 8192, 8192, 8192, 8222, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8162, 8192, 8192, 8192, 8192, 8192, 8222, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8162, 8192, 8192, 8192, 8192, 8192, 8222, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8162 expected_overflow=0 exact_recv_per_rank=[8162, 8192, 8192, 8192, 8192, 8192, 8222, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8162, 8192, 8192, 8192, 8192, 8192, 8222, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8162, 8192, 8192, 8192, 8192, 8192, 8222, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8222 expected_overflow=0 exact_recv_per_rank=[8162, 8192, 8192, 8192, 8192, 8192, 8222, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8162, 8192, 8192, 8192, 8192, 8192, 8222, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8162 actual_valid_recv=8320 observed_loss=-158 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[896, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8222 actual_valid_recv=8320 observed_loss=-98 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4071, 4070, 0, 0, 48, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1, 2, 3795, 4094, 0, 2, 298, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[3, 4, 3602, 4088, 0, 0, 494, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4093, 4096, 0, 0, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 3814, 4095, 0, 0, 282, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[24, 17, 3443, 4061, 0, 5, 636, 6] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4075, 4096, 0, 0, 21, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 3951, 4083, 0, 3, 133, 20] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1077, 0, 1010, 1089, 0, 1022, 0, 0, 1020, 0, 1018, 48, 828, 3, 1077] rerouted_tokens_per_ep_rank=[1077, 1010, 1089, 1022, 1020, 1018, 876, 1080] avg=1024.0000 max=1089 max_rank=2 min=876 min_rank=6 imbalance_ratio=1.063477 max_load_reduction=7103 max_load_reduction_pct=0.867065 moved_to_echo=3994 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1004, 2, 1016, 1015, 0, 1028, 0, 0, 1026, 2, 1024, 298, 772, 0, 1004] rerouted_tokens_per_ep_rank=[1005, 1018, 1015, 1028, 1026, 1026, 1070, 1004] avg=1024.0000 max=1070 max_rank=6 min=1004 min_rank=7 imbalance_ratio=1.044922 max_load_reduction=7122 max_load_reduction_pct=0.869385 moved_to_echo=4126 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1009, 0, 1017, 1020, 0, 1027, 0, 0, 1026, 0, 1025, 282, 776, 1, 1009] rerouted_tokens_per_ep_rank=[1009, 1017, 1020, 1027, 1026, 1025, 1058, 1010] avg=1024.0000 max=1058 max_rank=6 min=1009 min_rank=0 imbalance_ratio=1.033203 max_load_reduction=7134 max_load_reduction_pct=0.870850 moved_to_echo=4119 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 957, 4, 1018, 953, 0, 1015, 0, 0, 1028, 0, 1027, 494, 736, 1, 956] rerouted_tokens_per_ep_rank=[960, 1022, 953, 1015, 1028, 1027, 1230, 957] avg=1024.0000 max=1230 max_rank=6 min=953 min_rank=2 imbalance_ratio=1.201172 max_load_reduction=6962 max_load_reduction_pct=0.849854 moved_to_echo=4242 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1083, 0, 1017, 1096, 0, 1028, 0, 0, 1026, 0, 1025, 3, 832, 0, 1082] rerouted_tokens_per_ep_rank=[1083, 1017, 1096, 1028, 1026, 1025, 835, 1082] avg=1024.0000 max=1096 max_rank=2 min=835 min_rank=6 imbalance_ratio=1.070312 max_load_reduction=7096 max_load_reduction_pct=0.866211 moved_to_echo=3968 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[24, 911, 17, 1008, 922, 0, 1020, 0, 0, 1017, 5, 1016, 636, 700, 6, 910] rerouted_tokens_per_ep_rank=[935, 1025, 922, 1020, 1017, 1021, 1336, 916] avg=1024.0000 max=1336 max_rank=6 min=916 min_rank=7 imbalance_ratio=1.304688 max_load_reduction=6856 max_load_reduction_pct=0.836914 moved_to_echo=4290 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1078, 0, 1017, 1090, 0, 1028, 0, 0, 1026, 0, 1025, 21, 829, 0, 1078] rerouted_tokens_per_ep_rank=[1078, 1017, 1090, 1028, 1026, 1025, 850, 1078] avg=1024.0000 max=1090 max_rank=2 min=850 min_rank=6 imbalance_ratio=1.064453 max_load_reduction=7102 max_load_reduction_pct=0.866943 moved_to_echo=3979 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1045, 2, 1014, 1057, 0, 1024, 0, 0, 1023, 3, 1022, 133, 804, 20, 1045] rerouted_tokens_per_ep_rank=[1045, 1016, 1057, 1024, 1023, 1025, 937, 1065] avg=1024.0000 max=1065 max_rank=7 min=937 min_rank=6 imbalance_ratio=1.040039 max_load_reduction=7127 max_load_reduction_pct=0.869995 moved_to_echo=4050 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8142, 8242, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8142 expected_overflow=0 exact_recv_per_rank=[8192, 8142, 8242, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8242 expected_overflow=0 exact_recv_per_rank=[8192, 8142, 8242, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8142, 8242, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8142, 8242, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8142, 8242, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8142, 8242, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8142, 8242, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8242 actual_valid_recv=8320 observed_loss=-78 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8142 actual_valid_recv=8320 observed_loss=-178 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1920, 6400] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3, 4088, 3, 0, 4094, 2, 0, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[22, 4064, 10, 0, 4087, 1, 8, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[27, 4058, 0, 11, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[143, 3959, 16, 0, 4059, 0, 14, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[2, 4094, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[18, 4078, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1, 4074, 6, 0, 4090, 7, 14, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[5, 4090, 0, 1, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 1002, 1036, 0, 3, 1025, 0, 1020, 1027, 0, 2, 1023, 0, 1025, 2, 1024] rerouted_tokens_per_ep_rank=[1005, 1036, 1028, 1020, 1027, 1025, 1025, 1026] avg=1024.0000 max=1036 max_rank=1 min=1005 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[27, 995, 1027, 0, 0, 1018, 11, 1020, 1027, 0, 0, 1024, 0, 1018, 0, 1025] rerouted_tokens_per_ep_rank=[1022, 1027, 1018, 1031, 1027, 1024, 1018, 1025] avg=1024.0000 max=1031 max_rank=3 min=1018 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[143, 970, 1003, 0, 16, 993, 0, 1011, 1017, 0, 0, 1015, 14, 993, 1, 1016] rerouted_tokens_per_ep_rank=[1113, 1003, 1009, 1011, 1017, 1015, 1007, 1017] avg=1024.0000 max=1113 max_rank=0 min=1003 min_rank=1 imbalance_ratio=1.086914 max_load_reduction=7079 max_load_reduction_pct=0.864136 moved_to_echo=4056 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[22, 1000, 1019, 0, 10, 1023, 0, 1021, 1014, 0, 1, 1026, 8, 1022, 0, 1026] rerouted_tokens_per_ep_rank=[1022, 1019, 1033, 1021, 1014, 1027, 1030, 1026] avg=1024.0000 max=1033 max_rank=2 min=1014 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[18, 1000, 1032, 0, 0, 1023, 0, 1020, 1027, 0, 0, 1024, 0, 1023, 0, 1025] rerouted_tokens_per_ep_rank=[1018, 1032, 1023, 1020, 1027, 1024, 1023, 1025] avg=1024.0000 max=1032 max_rank=1 min=1018 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 1003, 1037, 0, 0, 1027, 0, 1020, 1027, 0, 0, 1024, 0, 1027, 0, 1025] rerouted_tokens_per_ep_rank=[1005, 1037, 1027, 1020, 1027, 1024, 1027, 1025] avg=1024.0000 max=1037 max_rank=1 min=1005 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=7155 max_load_reduction_pct=0.873413 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[5, 1002, 1036, 0, 0, 1026, 1, 1020, 1027, 0, 0, 1024, 0, 1026, 0, 1025] rerouted_tokens_per_ep_rank=[1007, 1036, 1026, 1021, 1027, 1024, 1026, 1025] avg=1024.0000 max=1036 max_rank=1 min=1007 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 999, 1031, 0, 6, 1022, 0, 1019, 1026, 0, 7, 1022, 14, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1000, 1031, 1028, 1019, 1026, 1029, 1036, 1023] avg=1024.0000 max=1036 max_rank=6 min=1000 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4114 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8221, 8192, 8163, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8221, 8192, 8163, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8221, 8192, 8163, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8221 expected_overflow=0 exact_recv_per_rank=[8192, 8221, 8192, 8163, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8221, 8192, 8163, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8221, 8192, 8163, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8221, 8192, 8163, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8163 expected_overflow=0 exact_recv_per_rank=[8192, 8221, 8192, 8163, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8221 actual_valid_recv=8320 observed_loss=-99 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8163 actual_valid_recv=8320 observed_loss=-157 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 2, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 1, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 2, 4082, 12, 5, 3, 1, 4087] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4091, 2, 1, 1, 3, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[12, 4, 4060, 24, 0, 22, 0, 4070] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4094, 2, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[3, 0, 4074, 14, 4, 0, 1, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1029, 0, 0, 1019, 0, 1021, 2, 1022, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1029, 1019, 1021, 1024, 1024, 1027] avg=1024.0000 max=1029 max_rank=2 min=1019 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1030, 0, 0, 1019, 1, 1021, 0, 1022, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1030, 1019, 1022, 1022, 1024, 1027] avg=1024.0000 max=1030 max_rank=2 min=1019 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1029, 0, 2, 1018, 1, 1021, 1, 1021, 3, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1029, 1020, 1022, 1022, 1027, 1026] avg=1024.0000 max=1029 max_rank=2 min=1020 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 2, 1026, 1015, 0, 12, 1020, 5, 1025, 3, 1023, 1, 1025, 1011, 0] rerouted_tokens_per_ep_rank=[1024, 1028, 1015, 1032, 1030, 1026, 1026, 1011] avg=1024.0000 max=1032 max_rank=3 min=1011 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 1015, 4, 1017, 1021, 0, 24, 1010, 0, 1015, 22, 1014, 0, 1018, 1020, 0] rerouted_tokens_per_ep_rank=[1027, 1021, 1021, 1034, 1015, 1036, 1018, 1020] avg=1024.0000 max=1036 max_rank=5 min=1015 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1030, 0, 0, 1019, 0, 1021, 0, 1023, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1030, 1019, 1021, 1023, 1024, 1027] avg=1024.0000 max=1030 max_rank=2 min=1019 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1029, 0, 2, 1019, 0, 1021, 0, 1022, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1029, 1021, 1021, 1022, 1024, 1027] avg=1024.0000 max=1029 max_rank=2 min=1021 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 1019, 0, 1024, 1024, 0, 14, 1014, 4, 1021, 0, 1017, 1, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1022, 1024, 1024, 1028, 1025, 1017, 1025, 1027] avg=1024.0000 max=1028 max_rank=3 min=1017 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8207, 8192, 8177, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8207, 8192, 8177, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8207, 8192, 8177, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8207, 8192, 8177, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8207, 8192, 8177, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8177 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8207, 8192, 8177, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8207, 8192, 8177, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8207 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8207, 8192, 8177, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8177 actual_valid_recv=8320 observed_loss=-143 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8207 actual_valid_recv=8320 observed_loss=-113 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 3870, 0, 0, 227, 0, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 3126, 3, 19, 967, 0, 8, 4069] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 3220, 0, 0, 876, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 3518, 0, 0, 577, 1, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 3668, 4, 10, 421, 0, 4, 4085] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 3647, 4, 2, 449, 0, 0, 4090] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1, 3675, 0, 0, 413, 7, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[5, 3691, 0, 3, 406, 0, 1, 4086] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1121, 0, 0, 1020, 0, 1111, 227, 525, 0, 1024, 0, 1113, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1121, 1020, 1111, 752, 1024, 1113, 1027] avg=1024.0000 max=1121 max_rank=1 min=752 min_rank=4 imbalance_ratio=1.094727 max_load_reduction=7071 max_load_reduction_pct=0.863159 moved_to_echo=3916 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1019, 0, 0, 1020, 0, 1010, 577, 477, 1, 1024, 0, 1012, 1028, 0] rerouted_tokens_per_ep_rank=[1024, 1019, 1020, 1010, 1054, 1025, 1012, 1028] avg=1024.0000 max=1054 max_rank=4 min=1010 min_rank=3 imbalance_ratio=1.029297 max_load_reduction=7138 max_load_reduction_pct=0.871338 moved_to_echo=4119 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1057, 0, 4, 1019, 2, 1047, 449, 494, 0, 1023, 0, 1049, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1057, 1023, 1049, 943, 1023, 1049, 1025] avg=1024.0000 max=1057 max_rank=1 min=943 min_rank=4 imbalance_ratio=1.032227 max_load_reduction=7135 max_load_reduction_pct=0.870972 moved_to_echo=4040 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1018, 906, 0, 3, 1013, 19, 897, 967, 424, 0, 1017, 8, 899, 1021, 0] rerouted_tokens_per_ep_rank=[1018, 906, 1016, 916, 1391, 1017, 907, 1021] avg=1024.0000 max=1391 max_rank=4 min=906 min_rank=1 imbalance_ratio=1.358398 max_load_reduction=6801 max_load_reduction_pct=0.830200 moved_to_echo=4336 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 934, 0, 0, 1020, 0, 924, 876, 436, 0, 1024, 0, 926, 1028, 0] rerouted_tokens_per_ep_rank=[1024, 934, 1020, 924, 1312, 1024, 926, 1028] avg=1024.0000 max=1312 max_rank=4 min=924 min_rank=3 imbalance_ratio=1.281250 max_load_reduction=6880 max_load_reduction_pct=0.839844 moved_to_echo=4290 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 1050, 0, 4, 1021, 10, 1055, 421, 502, 0, 1026, 4, 1061, 1011, 0] rerouted_tokens_per_ep_rank=[1027, 1050, 1025, 1065, 923, 1026, 1065, 1011] avg=1024.0000 max=1065 max_rank=3 min=923 min_rank=4 imbalance_ratio=1.040039 max_load_reduction=7127 max_load_reduction_pct=0.869995 moved_to_echo=4025 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 1024, 1065, 0, 0, 1020, 0, 1055, 413, 498, 7, 1024, 0, 1057, 1028, 0] rerouted_tokens_per_ep_rank=[1025, 1065, 1020, 1055, 911, 1031, 1057, 1028] avg=1024.0000 max=1065 max_rank=1 min=911 min_rank=4 imbalance_ratio=1.040039 max_load_reduction=7127 max_load_reduction_pct=0.869995 moved_to_echo=4027 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5, 1022, 1070, 0, 0, 1018, 3, 1059, 406, 500, 0, 1022, 1, 1062, 1024, 0] rerouted_tokens_per_ep_rank=[1027, 1070, 1018, 1062, 906, 1022, 1063, 1024] avg=1024.0000 max=1070 max_rank=1 min=906 min_rank=4 imbalance_ratio=1.044922 max_load_reduction=7122 max_load_reduction_pct=0.869385 moved_to_echo=4015 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8222, 8162, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8222, 8162, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8222 expected_overflow=0 exact_recv_per_rank=[8192, 8222, 8162, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8222, 8162, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8222, 8162, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8222, 8162, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8162 expected_overflow=0 exact_recv_per_rank=[8192, 8222, 8162, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8222, 8162, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4352, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8222 actual_valid_recv=8320 observed_loss=-98 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8162 actual_valid_recv=8320 observed_loss=-158 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[8, 4071, 31, 1, 15, 18, 5, 4043] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 106, 0, 0, 0, 0, 3990] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[3, 4083, 1, 2, 7, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4091, 9, 0, 3, 0, 2, 4087] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 232, 0, 0, 0, 0, 3864] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 17, 0, 0, 1, 0, 4078] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 39, 0, 0, 0, 0, 4057] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4094, 109, 0, 0, 0, 9, 3980] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1026, 0, 9, 970, 0, 1023, 3, 1036, 0, 1037, 2, 1020, 1044, 0] rerouted_tokens_per_ep_rank=[1022, 1026, 979, 1023, 1039, 1037, 1022, 1044] avg=1024.0000 max=1044 max_rank=7 min=979 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=7148 max_load_reduction_pct=0.872559 moved_to_echo=4142 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 232, 917, 0, 1025, 0, 980, 0, 980, 0, 1021, 987, 0] rerouted_tokens_per_ep_rank=[1024, 1026, 1149, 1025, 980, 980, 1021, 987] avg=1024.0000 max=1149 max_rank=2 min=980 min_rank=4 imbalance_ratio=1.122070 max_load_reduction=7043 max_load_reduction_pct=0.859741 moved_to_echo=3968 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 1017, 1021, 0, 31, 960, 1, 1018, 15, 1025, 18, 1026, 5, 1015, 1032, 0] rerouted_tokens_per_ep_rank=[1025, 1021, 991, 1019, 1040, 1044, 1020, 1032] avg=1024.0000 max=1044 max_rank=5 min=991 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=7148 max_load_reduction_pct=0.872559 moved_to_echo=4136 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 17, 968, 0, 1025, 0, 1034, 1, 1035, 0, 1021, 1041, 0] rerouted_tokens_per_ep_rank=[1024, 1026, 985, 1025, 1034, 1036, 1021, 1041] avg=1024.0000 max=1041 max_rank=7 min=985 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4132 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3, 1023, 1015, 0, 1, 978, 2, 1024, 7, 1042, 0, 1044, 0, 1021, 1032, 0] rerouted_tokens_per_ep_rank=[1026, 1015, 979, 1026, 1049, 1044, 1021, 1032] avg=1024.0000 max=1049 max_rank=4 min=979 min_rank=2 imbalance_ratio=1.024414 max_load_reduction=7143 max_load_reduction_pct=0.871948 moved_to_echo=4146 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 106, 947, 0, 1025, 0, 1012, 0, 1012, 0, 1021, 1019, 0] rerouted_tokens_per_ep_rank=[1024, 1026, 1053, 1025, 1012, 1012, 1021, 1019] avg=1024.0000 max=1053 max_rank=2 min=1012 min_rank=4 imbalance_ratio=1.028320 max_load_reduction=7139 max_load_reduction_pct=0.871460 moved_to_echo=4064 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 39, 963, 0, 1025, 0, 1029, 0, 1029, 0, 1021, 1036, 0] rerouted_tokens_per_ep_rank=[1024, 1026, 1002, 1025, 1029, 1029, 1021, 1036] avg=1024.0000 max=1036 max_rank=7 min=1002 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4115 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 109, 945, 0, 1024, 0, 1009, 0, 1010, 9, 1021, 1016, 0] rerouted_tokens_per_ep_rank=[1023, 1026, 1054, 1024, 1009, 1010, 1030, 1016] avg=1024.0000 max=1054 max_rank=2 min=1009 min_rank=4 imbalance_ratio=1.029297 max_load_reduction=7138 max_load_reduction_pct=0.871338 moved_to_echo=4065 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8177, 8207] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8177, 8207] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8207 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8177, 8207] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8177, 8207] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8177, 8207] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8177 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8177, 8207] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8177, 8207] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8177, 8207] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8177 actual_valid_recv=8320 observed_loss=-143 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[640, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8207 actual_valid_recv=8320 observed_loss=-113 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 782, 0, 0, 0, 3314, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1, 4081, 2079, 30, 0, 5, 1996, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 2466, 0, 0, 0, 1630, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 1287, 1, 0, 0, 2808, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 878, 0, 0, 0, 3218, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 2529, 0, 3, 1, 1566, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[27, 4063, 2360, 3, 0, 7, 1691, 41] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4089, 2493, 0, 0, 4, 1599, 7] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1027, 0, 431, 0, 0, 1516, 0, 1025, 0, 1023, 1798, 0, 0, 351] rerouted_tokens_per_ep_rank=[1021, 1027, 431, 1516, 1025, 1023, 1798, 351] avg=1024.0000 max=1798 max_rank=6 min=351 min_rank=7 imbalance_ratio=1.755859 max_load_reduction=6394 max_load_reduction_pct=0.780518 moved_to_echo=4197 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1027, 0, 484, 0, 0, 1473, 0, 1025, 0, 1023, 1745, 0, 0, 394] rerouted_tokens_per_ep_rank=[1021, 1027, 484, 1473, 1025, 1023, 1745, 394] avg=1024.0000 max=1745 max_rank=6 min=394 min_rank=7 imbalance_ratio=1.704102 max_load_reduction=6447 max_load_reduction_pct=0.786987 moved_to_echo=4187 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1020, 1027, 0, 1393, 0, 0, 716, 3, 1024, 1, 1022, 850, 0, 0, 1136] rerouted_tokens_per_ep_rank=[1020, 1027, 1393, 716, 1027, 1023, 850, 1136] avg=1024.0000 max=1393 max_rank=2 min=716 min_rank=3 imbalance_ratio=1.360352 max_load_reduction=6799 max_load_reduction_pct=0.829956 moved_to_echo=4036 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1019, 1012, 0, 1142, 0, 30, 917, 0, 1025, 5, 1025, 1079, 0, 0, 937] rerouted_tokens_per_ep_rank=[1020, 1012, 1142, 947, 1025, 1030, 1079, 937] avg=1024.0000 max=1142 max_rank=2 min=937 min_rank=7 imbalance_ratio=1.115234 max_load_reduction=7050 max_load_reduction_pct=0.860596 moved_to_echo=4071 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[27, 1012, 1019, 0, 1300, 0, 3, 774, 0, 1017, 7, 1015, 917, 0, 41, 1060] rerouted_tokens_per_ep_rank=[1039, 1019, 1300, 777, 1017, 1022, 917, 1101] avg=1024.0000 max=1300 max_rank=2 min=777 min_rank=3 imbalance_ratio=1.269531 max_load_reduction=6892 max_load_reduction_pct=0.841309 moved_to_echo=4057 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1027, 0, 1359, 0, 0, 746, 0, 1025, 0, 1023, 884, 0, 0, 1107] rerouted_tokens_per_ep_rank=[1021, 1027, 1359, 746, 1025, 1023, 884, 1107] avg=1024.0000 max=1359 max_rank=2 min=746 min_rank=3 imbalance_ratio=1.327148 max_load_reduction=6833 max_load_reduction_pct=0.834106 moved_to_echo=4039 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1027, 0, 709, 0, 1, 1285, 0, 1025, 0, 1023, 1523, 0, 0, 578] rerouted_tokens_per_ep_rank=[1021, 1027, 709, 1286, 1025, 1023, 1523, 578] avg=1024.0000 max=1523 max_rank=6 min=578 min_rank=7 imbalance_ratio=1.487305 max_load_reduction=6669 max_load_reduction_pct=0.814087 moved_to_echo=4149 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1019, 1026, 0, 1374, 0, 0, 731, 0, 1023, 4, 1021, 868, 0, 7, 1119] rerouted_tokens_per_ep_rank=[1019, 1026, 1374, 731, 1023, 1025, 868, 1126] avg=1024.0000 max=1374 max_rank=2 min=731 min_rank=3 imbalance_ratio=1.341797 max_load_reduction=6818 max_load_reduction_pct=0.832275 moved_to_echo=4042 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8182, 8192, 8192, 8192, 8192, 8192, 9664, 6730] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8182, 8192, 8192, 8192, 8192, 8192, 9664, 6730] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8182, 8192, 8192, 8192, 8192, 8192, 9664, 6730] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9664 expected_overflow=0 exact_recv_per_rank=[8182, 8192, 8192, 8192, 8192, 8192, 9664, 6730] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8182, 8192, 8192, 8192, 8192, 8192, 9664, 6730] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8182 expected_overflow=0 exact_recv_per_rank=[8182, 8192, 8192, 8192, 8192, 8192, 9664, 6730] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6730 expected_overflow=0 exact_recv_per_rank=[8182, 8192, 8192, 8192, 8192, 8192, 9664, 6730] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8182, 8192, 8192, 8192, 8192, 8192, 9664, 6730] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=6730 actual_valid_recv=6912 observed_loss=-182 tokens_per_expert=[128, 6784] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8182 actual_valid_recv=8320 observed_loss=-138 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9664 actual_valid_recv=9728 observed_loss=-64 tokens_per_expert=[9728, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=6912 slack_slots=9472 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9728 slack_slots=6656 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[22, 142, 157, 346, 2005, 657, 1054, 3809] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[47, 97, 129, 298, 2206, 617, 1147, 3651] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[11, 30, 84, 371, 2247, 508, 1155, 3786] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[28, 86, 76, 451, 2173, 455, 979, 3944] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[12, 63, 70, 554, 2387, 403, 1007, 3696] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[48, 87, 112, 339, 2150, 432, 1194, 3830] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[38, 59, 206, 367, 2244, 559, 928, 3791] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[28, 56, 67, 407, 2187, 387, 1262, 3798] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[22, 1000, 142, 951, 157, 827, 346, 576, 1015, 0, 657, 414, 1054, 0, 1031, 0] rerouted_tokens_per_ep_rank=[1022, 1093, 984, 922, 1015, 1071, 1054, 1031] avg=1024.0000 max=1093 max_rank=1 min=922 min_rank=3 imbalance_ratio=1.067383 max_load_reduction=7099 max_load_reduction_pct=0.866577 moved_to_echo=4171 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[47, 962, 97, 917, 129, 797, 298, 637, 1109, 0, 617, 460, 1147, 0, 975, 0] rerouted_tokens_per_ep_rank=[1009, 1014, 926, 935, 1109, 1077, 1147, 975] avg=1024.0000 max=1147 max_rank=6 min=926 min_rank=2 imbalance_ratio=1.120117 max_load_reduction=7045 max_load_reduction_pct=0.859985 moved_to_echo=4308 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[28, 1035, 86, 985, 76, 856, 451, 624, 1100, 0, 455, 449, 979, 0, 1068, 0] rerouted_tokens_per_ep_rank=[1063, 1071, 932, 1075, 1100, 904, 979, 1068] avg=1024.0000 max=1100 max_rank=4 min=904 min_rank=5 imbalance_ratio=1.074219 max_load_reduction=7092 max_load_reduction_pct=0.865723 moved_to_echo=4051 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[48, 1005, 87, 956, 112, 831, 339, 618, 1088, 0, 432, 444, 1194, 0, 1038, 0] rerouted_tokens_per_ep_rank=[1053, 1043, 943, 957, 1088, 876, 1194, 1038] avg=1024.0000 max=1194 max_rank=6 min=876 min_rank=5 imbalance_ratio=1.166016 max_load_reduction=6998 max_load_reduction_pct=0.854248 moved_to_echo=4196 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 994, 30, 945, 84, 822, 371, 645, 1138, 0, 508, 464, 1155, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1005, 975, 906, 1016, 1138, 972, 1155, 1025] avg=1024.0000 max=1155 max_rank=6 min=906 min_rank=2 imbalance_ratio=1.127930 max_load_reduction=7037 max_load_reduction_pct=0.859009 moved_to_echo=4290 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[38, 995, 59, 947, 206, 823, 367, 645, 1135, 0, 559, 464, 928, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1033, 1006, 1029, 1012, 1135, 1023, 928, 1026] avg=1024.0000 max=1135 max_rank=4 min=928 min_rank=6 imbalance_ratio=1.108398 max_load_reduction=7057 max_load_reduction_pct=0.861450 moved_to_echo=4112 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[28, 997, 56, 948, 67, 825, 407, 628, 1107, 0, 387, 452, 1262, 0, 1028, 0] rerouted_tokens_per_ep_rank=[1025, 1004, 892, 1035, 1107, 839, 1262, 1028] avg=1024.0000 max=1262 max_rank=6 min=839 min_rank=5 imbalance_ratio=1.232422 max_load_reduction=6930 max_load_reduction_pct=0.845947 moved_to_echo=4236 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[12, 970, 63, 923, 70, 802, 554, 686, 1208, 0, 403, 493, 1007, 0, 1001, 0] rerouted_tokens_per_ep_rank=[982, 986, 872, 1240, 1208, 896, 1007, 1001] avg=1024.0000 max=1240 max_rank=3 min=872 min_rank=2 imbalance_ratio=1.210938 max_load_reduction=6952 max_load_reduction_pct=0.848633 moved_to_echo=4112 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7484 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7484, 8192, 8900, 7658, 8726, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7484, 8192, 8900, 7658, 8726, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7484, 8192, 8900, 7658, 8726, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7658 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7484, 8192, 8900, 7658, 8726, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7484, 8192, 8900, 7658, 8726, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7484, 8192, 8900, 7658, 8726, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8900 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7484, 8192, 8900, 7658, 8726, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8726 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7484, 8192, 8900, 7658, 8726, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=7484 actual_valid_recv=7680 observed_loss=-196 tokens_per_expert=[1024, 6656] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7658 actual_valid_recv=7808 observed_loss=-150 tokens_per_expert=[4096, 3712] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3200, 5120] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8726 actual_valid_recv=8832 observed_loss=-106 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[640, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8900 actual_valid_recv=8960 observed_loss=-60 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=7680 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[576, 0, 606, 25, 2057, 718, 224, 3986] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1438, 2, 845, 3, 1325, 735, 247, 3597] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[803, 0, 1220, 2, 1671, 328, 290, 3878] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[707, 0, 796, 0, 2388, 284, 204, 3813] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1532, 0, 834, 32, 1496, 509, 316, 3473] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1350, 0, 967, 1, 1418, 403, 302, 3751] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1169, 0, 1003, 0, 1779, 308, 201, 3732] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[687, 0, 635, 1, 1732, 859, 257, 4021] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[803, 0, 0, 1049, 1220, 146, 2, 1041, 1038, 0, 328, 487, 290, 735, 1053, 0] rerouted_tokens_per_ep_rank=[803, 1049, 1366, 1043, 1038, 815, 1025, 1053] avg=1024.0000 max=1366 max_rank=2 min=803 min_rank=0 imbalance_ratio=1.333984 max_load_reduction=6826 max_load_reduction_pct=0.833252 moved_to_echo=3931 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[576, 0, 0, 1079, 606, 180, 25, 1070, 1277, 0, 718, 600, 224, 756, 1081, 0] rerouted_tokens_per_ep_rank=[576, 1079, 786, 1095, 1277, 1318, 980, 1081] avg=1024.0000 max=1318 max_rank=5 min=576 min_rank=0 imbalance_ratio=1.287109 max_load_reduction=6874 max_load_reduction_pct=0.839111 moved_to_echo=4656 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[707, 0, 0, 1032, 796, 209, 0, 1024, 1482, 0, 284, 697, 204, 723, 1034, 0] rerouted_tokens_per_ep_rank=[707, 1032, 1005, 1024, 1482, 981, 927, 1034] avg=1024.0000 max=1482 max_rank=4 min=707 min_rank=0 imbalance_ratio=1.447266 max_load_reduction=6710 max_load_reduction_pct=0.819092 moved_to_echo=4424 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1438, 0, 2, 973, 845, 116, 3, 966, 823, 0, 735, 386, 247, 682, 976, 0] rerouted_tokens_per_ep_rank=[1438, 975, 961, 969, 823, 1121, 929, 976] avg=1024.0000 max=1438 max_rank=0 min=823 min_rank=4 imbalance_ratio=1.404297 max_load_reduction=6754 max_load_reduction_pct=0.824463 moved_to_echo=3849 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1532, 0, 0, 944, 834, 134, 32, 938, 921, 0, 509, 441, 316, 663, 928, 0] rerouted_tokens_per_ep_rank=[1532, 944, 968, 970, 921, 950, 979, 928] avg=1024.0000 max=1532 max_rank=0 min=921 min_rank=4 imbalance_ratio=1.496094 max_load_reduction=6660 max_load_reduction_pct=0.812988 moved_to_echo=3778 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1350, 0, 0, 1015, 967, 124, 1, 1007, 881, 0, 403, 413, 302, 711, 1018, 0] rerouted_tokens_per_ep_rank=[1350, 1015, 1091, 1008, 881, 816, 1013, 1018] avg=1024.0000 max=1350 max_rank=0 min=816 min_rank=5 imbalance_ratio=1.318359 max_load_reduction=6842 max_load_reduction_pct=0.835205 moved_to_echo=3728 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1169, 0, 0, 1010, 1003, 156, 0, 1002, 1104, 0, 308, 519, 201, 708, 1012, 0] rerouted_tokens_per_ep_rank=[1169, 1010, 1159, 1002, 1104, 827, 909, 1012] avg=1024.0000 max=1169 max_rank=0 min=827 min_rank=5 imbalance_ratio=1.141602 max_load_reduction=7023 max_load_reduction_pct=0.857300 moved_to_echo=3852 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[687, 0, 0, 1088, 635, 151, 1, 1080, 1076, 0, 859, 505, 257, 763, 1090, 0] rerouted_tokens_per_ep_rank=[687, 1088, 786, 1081, 1076, 1364, 1020, 1090] avg=1024.0000 max=1364 max_rank=5 min=687 min_rank=0 imbalance_ratio=1.332031 max_load_reduction=6828 max_load_reduction_pct=0.833496 moved_to_echo=4550 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8262, 8192, 8122, 8192, 8602, 8192, 7782, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8262 expected_overflow=0 exact_recv_per_rank=[8262, 8192, 8122, 8192, 8602, 8192, 7782, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8262, 8192, 8122, 8192, 8602, 8192, 7782, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7782 expected_overflow=0 exact_recv_per_rank=[8262, 8192, 8122, 8192, 8602, 8192, 7782, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8262, 8192, 8122, 8192, 8602, 8192, 7782, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8262, 8192, 8122, 8192, 8602, 8192, 7782, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8602 expected_overflow=0 exact_recv_per_rank=[8262, 8192, 8122, 8192, 8602, 8192, 7782, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8122 expected_overflow=0 exact_recv_per_rank=[8262, 8192, 8122, 8192, 8602, 8192, 7782, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4224, 4096] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7782 actual_valid_recv=7808 observed_loss=-26 tokens_per_expert=[2048, 5760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8602 actual_valid_recv=8704 observed_loss=-102 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8262 actual_valid_recv=8320 observed_loss=-58 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8122 actual_valid_recv=8192 observed_loss=-70 tokens_per_expert=[6912, 1280] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3949, 279, 0, 17, 1, 3265, 109, 572] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4048, 96, 0, 2, 0, 3089, 105, 852] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4032, 230, 9, 11, 7, 3021, 122, 760] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4043, 213, 0, 3, 0, 3098, 4, 831] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4074, 201, 1, 10, 0, 2694, 136, 1076] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3890, 259, 2, 7, 13, 3467, 144, 410] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4021, 266, 0, 0, 0, 3538, 105, 262] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3876, 137, 0, 1, 0, 3740, 113, 325] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1041, 0, 96, 776, 0, 1036, 2, 936, 0, 1035, 1067, 0, 105, 876, 852, 370] rerouted_tokens_per_ep_rank=[1041, 872, 1036, 938, 1035, 1067, 981, 1222] avg=1024.0000 max=1222 max_rank=7 min=872 min_rank=1 imbalance_ratio=1.193359 max_load_reduction=6970 max_load_reduction_pct=0.850830 moved_to_echo=4305 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1021, 0, 230, 762, 9, 1037, 11, 938, 7, 1036, 1034, 0, 122, 861, 760, 364] rerouted_tokens_per_ep_rank=[1021, 992, 1046, 949, 1043, 1034, 983, 1124] avg=1024.0000 max=1124 max_rank=7 min=949 min_rank=3 imbalance_ratio=1.097656 max_load_reduction=7068 max_load_reduction_pct=0.862793 moved_to_echo=4184 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1015, 0, 279, 820, 0, 1011, 17, 913, 1, 1010, 1128, 0, 109, 926, 572, 391] rerouted_tokens_per_ep_rank=[1015, 1099, 1011, 930, 1011, 1128, 1035, 963] avg=1024.0000 max=1128 max_rank=5 min=930 min_rank=3 imbalance_ratio=1.101562 max_load_reduction=7064 max_load_reduction_pct=0.862305 moved_to_echo=4137 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1039, 0, 213, 778, 0, 1035, 3, 935, 0, 1034, 1070, 0, 4, 879, 831, 371] rerouted_tokens_per_ep_rank=[1039, 991, 1035, 938, 1034, 1070, 883, 1202] avg=1024.0000 max=1202 max_rank=7 min=883 min_rank=6 imbalance_ratio=1.173828 max_load_reduction=6990 max_load_reduction_pct=0.853271 moved_to_echo=4189 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1047, 0, 201, 676, 1, 1043, 10, 942, 0, 1042, 932, 0, 136, 764, 1076, 322] rerouted_tokens_per_ep_rank=[1047, 877, 1044, 952, 1042, 932, 900, 1398] avg=1024.0000 max=1398 max_rank=7 min=877 min_rank=1 imbalance_ratio=1.365234 max_load_reduction=6794 max_load_reduction_pct=0.829346 moved_to_echo=4272 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[999, 0, 259, 871, 2, 996, 7, 900, 13, 995, 1198, 0, 144, 983, 410, 415] rerouted_tokens_per_ep_rank=[999, 1130, 998, 907, 1008, 1198, 1127, 825] avg=1024.0000 max=1198 max_rank=5 min=825 min_rank=7 imbalance_ratio=1.169922 max_load_reduction=6994 max_load_reduction_pct=0.853760 moved_to_echo=4158 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1033, 0, 266, 889, 0, 1030, 0, 930, 0, 1028, 1222, 0, 105, 1004, 262, 423] rerouted_tokens_per_ep_rank=[1033, 1155, 1030, 930, 1028, 1222, 1109, 685] avg=1024.0000 max=1222 max_rank=5 min=685 min_rank=7 imbalance_ratio=1.193359 max_load_reduction=6970 max_load_reduction_pct=0.850830 moved_to_echo=4044 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[997, 0, 137, 939, 0, 992, 1, 896, 0, 991, 1292, 0, 113, 1061, 325, 448] rerouted_tokens_per_ep_rank=[997, 1076, 992, 897, 991, 1292, 1174, 773] avg=1024.0000 max=1292 max_rank=5 min=773 min_rank=7 imbalance_ratio=1.261719 max_load_reduction=6900 max_load_reduction_pct=0.842285 moved_to_echo=4230 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7441 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7441, 8192, 8943, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8943 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7441, 8192, 8943, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7441, 8192, 8943, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7441, 8192, 8943, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7441, 8192, 8943, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7441, 8192, 8943, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7441, 8192, 8943, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 7441, 8192, 8943, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[896, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=7441 actual_valid_recv=7552 observed_loss=-111 tokens_per_expert=[128, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5120, 3200] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1792, 6528] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8943 actual_valid_recv=8960 observed_loss=-17 tokens_per_expert=[8960, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8960 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[3, 2, 7, 7, 1, 2, 4094, 4076] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[69, 0, 516, 0, 0, 0, 4096, 3511] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[13, 0, 43, 30, 5, 1, 4075, 4025] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[48, 5, 61, 19, 0, 0, 4074, 3985] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[61, 0, 15, 12, 0, 3, 4096, 4005] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[11, 2, 56, 2, 0, 0, 4095, 4026] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[133, 1, 24, 5, 0, 0, 4082, 3947] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[71, 0, 28, 8, 0, 0, 4095, 3990] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[69, 865, 0, 1019, 516, 827, 0, 901, 0, 1025, 0, 1025, 1027, 0, 918, 0] rerouted_tokens_per_ep_rank=[934, 1019, 1343, 901, 1025, 1025, 1027, 918] avg=1024.0000 max=1343 max_rank=2 min=901 min_rank=3 imbalance_ratio=1.311523 max_load_reduction=6849 max_load_reduction_pct=0.836060 moved_to_echo=3995 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 1005, 2, 1019, 7, 960, 7, 1047, 1, 1024, 2, 1024, 1027, 0, 1064, 0] rerouted_tokens_per_ep_rank=[1008, 1021, 967, 1054, 1025, 1026, 1027, 1064] avg=1024.0000 max=1064 max_rank=7 min=967 min_rank=2 imbalance_ratio=1.039062 max_load_reduction=7128 max_load_reduction_pct=0.870117 moved_to_echo=4142 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[133, 973, 1, 1016, 24, 930, 5, 1013, 0, 1021, 0, 1021, 1024, 0, 1031, 0] rerouted_tokens_per_ep_rank=[1106, 1017, 954, 1018, 1021, 1021, 1024, 1031] avg=1024.0000 max=1106 max_rank=0 min=954 min_rank=2 imbalance_ratio=1.080078 max_load_reduction=7086 max_load_reduction_pct=0.864990 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[11, 992, 2, 1019, 56, 949, 2, 1034, 0, 1024, 0, 1024, 1028, 0, 1051, 0] rerouted_tokens_per_ep_rank=[1003, 1021, 1005, 1036, 1024, 1024, 1028, 1051] avg=1024.0000 max=1051 max_rank=7 min=1003 min_rank=0 imbalance_ratio=1.026367 max_load_reduction=7141 max_load_reduction_pct=0.871704 moved_to_echo=4127 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[48, 986, 5, 1018, 61, 944, 19, 1027, 0, 1024, 0, 1024, 1008, 0, 1028, 0] rerouted_tokens_per_ep_rank=[1034, 1023, 1005, 1046, 1024, 1024, 1008, 1028] avg=1024.0000 max=1046 max_rank=3 min=1005 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=7146 max_load_reduction_pct=0.872314 moved_to_echo=4084 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[13, 992, 0, 1014, 43, 948, 30, 1034, 5, 1019, 1, 1019, 1023, 0, 1051, 0] rerouted_tokens_per_ep_rank=[1005, 1014, 991, 1064, 1024, 1020, 1023, 1051] avg=1024.0000 max=1064 max_rank=3 min=991 min_rank=2 imbalance_ratio=1.039062 max_load_reduction=7128 max_load_reduction_pct=0.870117 moved_to_echo=4118 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[61, 987, 0, 1019, 15, 944, 12, 1028, 0, 1025, 3, 1025, 1027, 0, 1046, 0] rerouted_tokens_per_ep_rank=[1048, 1019, 959, 1040, 1025, 1028, 1027, 1046] avg=1024.0000 max=1048 max_rank=0 min=959 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=7144 max_load_reduction_pct=0.872070 moved_to_echo=4126 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[71, 983, 0, 1019, 28, 940, 8, 1025, 0, 1024, 0, 1024, 1028, 0, 1042, 0] rerouted_tokens_per_ep_rank=[1054, 1019, 968, 1033, 1024, 1024, 1028, 1042] avg=1024.0000 max=1054 max_rank=0 min=968 min_rank=2 imbalance_ratio=1.029297 max_load_reduction=7138 max_load_reduction_pct=0.871338 moved_to_echo=4118 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8153, 8192, 8192, 8192, 8192, 8192, 8231] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8153, 8192, 8192, 8192, 8192, 8192, 8231] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8153, 8192, 8192, 8192, 8192, 8192, 8231] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8153 expected_overflow=0 exact_recv_per_rank=[8192, 8153, 8192, 8192, 8192, 8192, 8192, 8231] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8153, 8192, 8192, 8192, 8192, 8192, 8231] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8153, 8192, 8192, 8192, 8192, 8192, 8231] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8231 expected_overflow=0 exact_recv_per_rank=[8192, 8153, 8192, 8192, 8192, 8192, 8192, 8231] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8153, 8192, 8192, 8192, 8192, 8192, 8231] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8153 actual_valid_recv=8320 observed_loss=-167 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8231 actual_valid_recv=8320 observed_loss=-89 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 2001, 0, 1, 0, 2098, 0, 4092] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 1503, 15, 6, 5, 2587, 7, 4069] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 1598, 0, 0, 0, 2506, 0, 4088] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1269, 0, 2, 0, 2846, 0, 4075] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 2317, 0, 0, 0, 1782, 0, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 1303, 0, 0, 1, 2925, 0, 3963] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1865, 1, 0, 0, 2268, 0, 4058] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 1839, 3, 0, 0, 2257, 0, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 760, 0, 0, 509, 2, 1208, 0, 1025, 1638, 0, 0, 997, 1027, 0] rerouted_tokens_per_ep_rank=[1026, 760, 509, 1210, 1025, 1638, 997, 1027] avg=1024.0000 max=1638 max_rank=5 min=509 min_rank=2 imbalance_ratio=1.599609 max_load_reduction=6554 max_load_reduction_pct=0.800049 moved_to_echo=4687 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 1197, 0, 0, 804, 1, 890, 0, 1029, 1208, 0, 0, 1001, 1032, 0] rerouted_tokens_per_ep_rank=[1030, 1197, 804, 891, 1029, 1208, 1001, 1032] avg=1024.0000 max=1208 max_rank=5 min=804 min_rank=2 imbalance_ratio=1.179688 max_load_reduction=6984 max_load_reduction_pct=0.852539 moved_to_echo=4270 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1029, 956, 0, 0, 642, 0, 1064, 0, 1028, 1442, 0, 0, 1000, 1031, 0] rerouted_tokens_per_ep_rank=[1029, 956, 642, 1064, 1028, 1442, 1000, 1031] avg=1024.0000 max=1442 max_rank=5 min=642 min_rank=2 imbalance_ratio=1.408203 max_load_reduction=6750 max_load_reduction_pct=0.823975 moved_to_echo=4501 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1029, 896, 0, 15, 607, 6, 1102, 5, 1028, 1485, 0, 7, 999, 1013, 0] rerouted_tokens_per_ep_rank=[1029, 896, 622, 1108, 1033, 1485, 1006, 1013] avg=1024.0000 max=1485 max_rank=5 min=622 min_rank=2 imbalance_ratio=1.450195 max_load_reduction=6707 max_load_reduction_pct=0.818726 moved_to_echo=4537 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 1386, 0, 0, 931, 0, 756, 0, 1029, 1026, 0, 0, 1001, 1033, 0] rerouted_tokens_per_ep_rank=[1030, 1386, 931, 756, 1029, 1026, 1001, 1033] avg=1024.0000 max=1386 max_rank=1 min=756 min_rank=3 imbalance_ratio=1.353516 max_load_reduction=6806 max_load_reduction_pct=0.830811 moved_to_echo=4089 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 997, 780, 0, 0, 523, 0, 1242, 1, 997, 1683, 0, 0, 969, 1000, 0] rerouted_tokens_per_ep_rank=[997, 780, 523, 1242, 998, 1683, 969, 1000] avg=1024.0000 max=1683 max_rank=5 min=523 min_rank=2 imbalance_ratio=1.643555 max_load_reduction=6509 max_load_reduction_pct=0.794556 moved_to_echo=4650 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1116, 0, 1, 749, 0, 963, 0, 1021, 1305, 0, 0, 993, 1023, 0] rerouted_tokens_per_ep_rank=[1021, 1116, 750, 963, 1021, 1305, 993, 1023] avg=1024.0000 max=1305 max_rank=5 min=750 min_rank=2 imbalance_ratio=1.274414 max_load_reduction=6887 max_load_reduction_pct=0.840698 moved_to_echo=4342 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 1101, 0, 3, 738, 0, 958, 0, 1029, 1299, 0, 0, 1001, 1033, 0] rerouted_tokens_per_ep_rank=[1030, 1101, 741, 958, 1029, 1299, 1001, 1033] avg=1024.0000 max=1299 max_rank=5 min=741 min_rank=2 imbalance_ratio=1.268555 max_load_reduction=6893 max_load_reduction_pct=0.841431 moved_to_echo=4362 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5522 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5522, 8192, 8192, 11086, 7968, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5522, 8192, 8192, 11086, 7968, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5522, 8192, 8192, 11086, 7968, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11086 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5522, 8192, 8192, 11086, 7968, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5522, 8192, 8192, 11086, 7968, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7968 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5522, 8192, 8192, 11086, 7968, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5522, 8192, 8192, 11086, 7968, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5522, 8192, 8192, 11086, 7968, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=5522 actual_valid_recv=5632 observed_loss=-110 tokens_per_expert=[128, 5504] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7968 actual_valid_recv=8192 observed_loss=-224 tokens_per_expert=[128, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=11086 actual_valid_recv=11136 observed_loss=-50 tokens_per_expert=[11136, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=5632 slack_slots=10752 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=11136 slack_slots=5248 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[261, 20, 2170, 1960, 0, 8, 3773, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[94, 0, 1934, 2694, 0, 0, 3470, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[88, 1, 3891, 1289, 0, 0, 2923, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[220, 0, 1378, 2718, 4, 0, 3872, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[15, 0, 2494, 2199, 0, 0, 3484, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[8, 2, 2748, 2161, 0, 0, 3273, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[45, 0, 2923, 1642, 0, 0, 3582, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[162, 1, 2951, 1734, 0, 0, 3344, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[261, 878, 20, 773, 861, 0, 1082, 0, 0, 1118, 8, 536, 1537, 0, 0, 1118] rerouted_tokens_per_ep_rank=[1139, 793, 861, 1082, 1118, 544, 1537, 1118] avg=1024.0000 max=1537 max_rank=6 min=544 min_rank=5 imbalance_ratio=1.500977 max_load_reduction=6655 max_load_reduction_pct=0.812378 moved_to_echo=4317 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[88, 573, 1, 1379, 1556, 0, 716, 0, 0, 863, 0, 956, 1197, 0, 0, 863] rerouted_tokens_per_ep_rank=[661, 1380, 1556, 716, 863, 956, 1197, 863] avg=1024.0000 max=1556 max_rank=2 min=661 min_rank=0 imbalance_ratio=1.519531 max_load_reduction=6636 max_load_reduction_pct=0.810059 moved_to_echo=3879 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[15, 978, 0, 883, 999, 0, 1221, 0, 0, 1029, 0, 612, 1426, 0, 0, 1029] rerouted_tokens_per_ep_rank=[993, 883, 999, 1221, 1029, 612, 1426, 1029] avg=1024.0000 max=1426 max_rank=6 min=612 min_rank=5 imbalance_ratio=1.392578 max_load_reduction=6766 max_load_reduction_pct=0.825928 moved_to_echo=4096 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[94, 1199, 0, 685, 774, 0, 1495, 0, 0, 1024, 0, 475, 1421, 0, 0, 1025] rerouted_tokens_per_ep_rank=[1293, 685, 774, 1495, 1024, 475, 1421, 1025] avg=1024.0000 max=1495 max_rank=3 min=475 min_rank=5 imbalance_ratio=1.459961 max_load_reduction=6697 max_load_reduction_pct=0.817505 moved_to_echo=3945 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[220, 1209, 0, 488, 552, 0, 1509, 0, 4, 1143, 0, 338, 1585, 0, 0, 1144] rerouted_tokens_per_ep_rank=[1429, 488, 552, 1509, 1147, 338, 1585, 1144] avg=1024.0000 max=1585 max_rank=6 min=338 min_rank=5 imbalance_ratio=1.547852 max_load_reduction=6607 max_load_reduction_pct=0.806519 moved_to_echo=4214 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[8, 961, 2, 973, 1100, 0, 1200, 0, 0, 966, 0, 675, 1340, 0, 0, 967] rerouted_tokens_per_ep_rank=[969, 975, 1100, 1200, 966, 675, 1340, 967] avg=1024.0000 max=1340 max_rank=6 min=675 min_rank=5 imbalance_ratio=1.308594 max_load_reduction=6852 max_load_reduction_pct=0.836426 moved_to_echo=3948 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[162, 771, 1, 1045, 1181, 0, 963, 0, 0, 987, 0, 725, 1369, 0, 0, 988] rerouted_tokens_per_ep_rank=[933, 1046, 1181, 963, 987, 725, 1369, 988] avg=1024.0000 max=1369 max_rank=6 min=725 min_rank=5 imbalance_ratio=1.336914 max_load_reduction=6823 max_load_reduction_pct=0.832886 moved_to_echo=4069 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[45, 730, 0, 1036, 1169, 0, 912, 0, 0, 1058, 0, 718, 1466, 0, 0, 1058] rerouted_tokens_per_ep_rank=[775, 1036, 1169, 912, 1058, 718, 1466, 1058] avg=1024.0000 max=1466 max_rank=6 min=718 min_rank=5 imbalance_ratio=1.431641 max_load_reduction=6726 max_load_reduction_pct=0.821045 moved_to_echo=4300 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9098 expected_overflow=0 exact_recv_per_rank=[8192, 7286, 8192, 9098, 8192, 5043, 11341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7286, 8192, 9098, 8192, 5043, 11341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5043 expected_overflow=0 exact_recv_per_rank=[8192, 7286, 8192, 9098, 8192, 5043, 11341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7286, 8192, 9098, 8192, 5043, 11341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11341 expected_overflow=0 exact_recv_per_rank=[8192, 7286, 8192, 9098, 8192, 5043, 11341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7286, 8192, 9098, 8192, 5043, 11341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 7286, 8192, 9098, 8192, 5043, 11341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7286 expected_overflow=0 exact_recv_per_rank=[8192, 7286, 8192, 9098, 8192, 5043, 11341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=5043 actual_valid_recv=5248 observed_loss=-205 tokens_per_expert=[128, 5120] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=7286 actual_valid_recv=7424 observed_loss=-138 tokens_per_expert=[128, 7296] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[896, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=9098 actual_valid_recv=9216 observed_loss=-118 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=11341 actual_valid_recv=11392 observed_loss=-51 tokens_per_expert=[11392, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=5248 slack_slots=11136 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=7424 slack_slots=8960 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=11392 slack_slots=4992 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2, 4094, 1, 3, 4092, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[2, 14, 0, 4083, 0, 0, 4093, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 2, 0, 4096, 0, 0, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[17, 15, 1, 4050, 29, 0, 4075, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 86, 0, 4031, 0, 0, 4075, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 26, 0, 4090, 0, 1, 4073, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 3, 0, 4095, 0, 0, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 126, 0, 4047, 0, 0, 4019, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 995, 2, 1027, 1048, 0, 1, 1025, 3, 1027, 1029, 0, 0, 1009] rerouted_tokens_per_ep_rank=[1026, 995, 1029, 1048, 1026, 1030, 1029, 1009] avg=1024.0000 max=1048 max_rank=3 min=995 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=7144 max_load_reduction_pct=0.872070 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 1023, 14, 992, 0, 1027, 1046, 0, 0, 1022, 0, 1027, 1030, 0, 0, 1009] rerouted_tokens_per_ep_rank=[1025, 1006, 1027, 1046, 1022, 1027, 1030, 1009] avg=1024.0000 max=1046 max_rank=3 min=1006 min_rank=1 imbalance_ratio=1.021484 max_load_reduction=7146 max_load_reduction_pct=0.872314 moved_to_echo=4088 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 2, 995, 0, 1027, 1049, 0, 0, 1025, 0, 1027, 1031, 0, 0, 1009] rerouted_tokens_per_ep_rank=[1027, 997, 1027, 1049, 1025, 1027, 1031, 1009] avg=1024.0000 max=1049 max_rank=3 min=997 min_rank=1 imbalance_ratio=1.024414 max_load_reduction=7143 max_load_reduction_pct=0.871948 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1010, 86, 979, 0, 1023, 1033, 0, 0, 1009, 0, 1023, 1024, 0, 0, 1005] rerouted_tokens_per_ep_rank=[1010, 1065, 1023, 1033, 1009, 1023, 1024, 1005] avg=1024.0000 max=1065 max_rank=1 min=1005 min_rank=7 imbalance_ratio=1.040039 max_load_reduction=7127 max_load_reduction_pct=0.869995 moved_to_echo=4061 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[17, 1021, 15, 987, 1, 1027, 1023, 0, 29, 1019, 0, 1027, 1011, 0, 5, 1010] rerouted_tokens_per_ep_rank=[1038, 1002, 1028, 1023, 1048, 1027, 1011, 1015] avg=1024.0000 max=1048 max_rank=4 min=1002 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=7144 max_load_reduction_pct=0.872070 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 1025, 26, 994, 0, 1022, 1047, 0, 0, 1024, 1, 1022, 1025, 0, 1, 1004] rerouted_tokens_per_ep_rank=[1026, 1020, 1022, 1047, 1024, 1023, 1025, 1005] avg=1024.0000 max=1047 max_rank=3 min=1005 min_rank=7 imbalance_ratio=1.022461 max_load_reduction=7145 max_load_reduction_pct=0.872192 moved_to_echo=4077 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1014, 126, 983, 0, 1009, 1037, 0, 0, 1013, 0, 1008, 1011, 0, 0, 991] rerouted_tokens_per_ep_rank=[1014, 1109, 1009, 1037, 1013, 1008, 1011, 991] avg=1024.0000 max=1109 max_rank=1 min=991 min_rank=7 imbalance_ratio=1.083008 max_load_reduction=7083 max_load_reduction_pct=0.864624 moved_to_echo=4023 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 3, 995, 0, 1027, 1049, 0, 0, 1025, 0, 1027, 1031, 0, 0, 1009] rerouted_tokens_per_ep_rank=[1026, 998, 1027, 1049, 1025, 1027, 1031, 1009] avg=1024.0000 max=1049 max_rank=3 min=998 min_rank=1 imbalance_ratio=1.024414 max_load_reduction=7143 max_load_reduction_pct=0.871948 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8332, 8192, 8192, 8192, 8052] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8332, 8192, 8192, 8192, 8052] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8332, 8192, 8192, 8192, 8052] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8332, 8192, 8192, 8192, 8052] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8052 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8332, 8192, 8192, 8192, 8052] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8332 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8332, 8192, 8192, 8192, 8052] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8332, 8192, 8192, 8192, 8052] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8332, 8192, 8192, 8192, 8052] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8332 actual_valid_recv=8448 observed_loss=-116 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8052 actual_valid_recv=8192 observed_loss=-140 tokens_per_expert=[128, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1, 3773, 0, 1696, 161, 2560, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 3984, 0, 435, 344, 3425, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 3846, 7, 909, 440, 2986, 4, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 3587, 0, 982, 62, 3558, 0, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3, 3662, 0, 1324, 67, 3136, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 3803, 2, 695, 731, 2944, 0, 17] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 4011, 0, 401, 191, 3589, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 4038, 0, 689, 283, 3182, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1006, 1008, 0, 0, 754, 1696, 107, 161, 596, 1033, 0, 0, 1005, 1, 824] rerouted_tokens_per_ep_rank=[1007, 1008, 754, 1803, 757, 1033, 1005, 825] avg=1024.0000 max=1803 max_rank=3 min=754 min_rank=2 imbalance_ratio=1.760742 max_load_reduction=6389 max_load_reduction_pct=0.779907 moved_to_echo=3620 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1062, 1064, 0, 0, 796, 435, 143, 344, 797, 1383, 0, 3, 1062, 0, 1102] rerouted_tokens_per_ep_rank=[1063, 1064, 796, 578, 1141, 1383, 1065, 1102] avg=1024.0000 max=1383 max_rank=5 min=578 min_rank=3 imbalance_ratio=1.350586 max_load_reduction=6809 max_load_reduction_pct=0.831177 moved_to_echo=4691 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 976, 978, 0, 0, 732, 1324, 131, 67, 730, 1266, 0, 0, 976, 0, 1009] rerouted_tokens_per_ep_rank=[979, 978, 732, 1455, 797, 1266, 976, 1009] avg=1024.0000 max=1455 max_rank=3 min=732 min_rank=2 imbalance_ratio=1.420898 max_load_reduction=6737 max_load_reduction_pct=0.822388 moved_to_echo=4048 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 956, 958, 0, 0, 717, 982, 148, 62, 828, 1437, 0, 0, 956, 3, 1145] rerouted_tokens_per_ep_rank=[956, 958, 717, 1130, 890, 1437, 956, 1148] avg=1024.0000 max=1437 max_rank=5 min=717 min_rank=2 imbalance_ratio=1.403320 max_load_reduction=6755 max_load_reduction_pct=0.824585 moved_to_echo=4431 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 1018, 0, 7, 772, 909, 126, 440, 700, 1195, 0, 4, 1028, 0, 965] rerouted_tokens_per_ep_rank=[1028, 1018, 779, 1035, 1140, 1195, 1032, 965] avg=1024.0000 max=1195 max_rank=5 min=779 min_rank=2 imbalance_ratio=1.166992 max_load_reduction=6997 max_load_reduction_pct=0.854126 moved_to_echo=4332 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1014, 1016, 0, 2, 760, 695, 123, 731, 685, 1189, 0, 0, 1013, 17, 947] rerouted_tokens_per_ep_rank=[1014, 1016, 762, 818, 1416, 1189, 1013, 964] avg=1024.0000 max=1416 max_rank=4 min=762 min_rank=2 imbalance_ratio=1.382812 max_load_reduction=6776 max_load_reduction_pct=0.827148 moved_to_echo=4582 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1069, 1071, 0, 0, 802, 401, 150, 191, 836, 1448, 0, 0, 1069, 0, 1155] rerouted_tokens_per_ep_rank=[1069, 1071, 802, 551, 1027, 1448, 1069, 1155] avg=1024.0000 max=1448 max_rank=5 min=551 min_rank=3 imbalance_ratio=1.414062 max_load_reduction=6744 max_load_reduction_pct=0.823242 moved_to_echo=4699 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1076, 1079, 0, 0, 807, 689, 133, 283, 741, 1284, 0, 0, 1076, 0, 1024] rerouted_tokens_per_ep_rank=[1076, 1079, 807, 822, 1024, 1284, 1076, 1024] avg=1024.0000 max=1284 max_rank=5 min=807 min_rank=2 imbalance_ratio=1.253906 max_load_reduction=6908 max_load_reduction_pct=0.843262 moved_to_echo=4408 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6149, 8192, 8192, 10235, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6149, 8192, 8192, 10235, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6149, 8192, 8192, 10235, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10235 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6149, 8192, 8192, 10235, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6149, 8192, 8192, 10235, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6149 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6149, 8192, 8192, 10235, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6149, 8192, 8192, 10235, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 6149, 8192, 8192, 10235, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=6149 actual_valid_recv=6272 observed_loss=-123 tokens_per_expert=[128, 6144] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2304, 6016] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7168, 1152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=10235 actual_valid_recv=10240 observed_loss=-5 tokens_per_expert=[10240, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=6272 slack_slots=10112 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=10240 slack_slots=6144 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[2849, 0, 0, 0, 0, 255, 4096, 992] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2531, 0, 0, 0, 0, 254, 4095, 1312] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[505, 0, 0, 0, 0, 224, 4096, 3367] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1892, 0, 0, 2, 0, 87, 4095, 2116] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[2715, 5, 3, 4, 17, 157, 4078, 1213] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1304, 0, 0, 4, 0, 1170, 4093, 1621] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2013, 0, 0, 0, 0, 736, 4091, 1352] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[663, 0, 0, 0, 0, 10, 4077, 3442] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1605, 0, 0, 1024, 0, 1024, 0, 1020, 0, 612, 254, 926, 1027, 0, 700, 0] rerouted_tokens_per_ep_rank=[1605, 1024, 1024, 1020, 612, 1180, 1027, 700] avg=1024.0000 max=1605 max_rank=0 min=612 min_rank=4 imbalance_ratio=1.567383 max_load_reduction=6587 max_load_reduction_pct=0.804077 moved_to_echo=3519 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1806, 0, 0, 1024, 0, 1025, 0, 1020, 0, 462, 255, 1043, 1027, 0, 530, 0] rerouted_tokens_per_ep_rank=[1806, 1024, 1025, 1020, 462, 1298, 1027, 530] avg=1024.0000 max=1806 max_rank=0 min=462 min_rank=4 imbalance_ratio=1.763672 max_load_reduction=6386 max_load_reduction_pct=0.779541 moved_to_echo=3317 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1200, 0, 0, 1024, 0, 1024, 2, 1020, 0, 987, 87, 692, 1027, 0, 1129, 0] rerouted_tokens_per_ep_rank=[1200, 1024, 1024, 1022, 987, 779, 1027, 1129] avg=1024.0000 max=1200 max_rank=0 min=779 min_rank=5 imbalance_ratio=1.171875 max_load_reduction=6992 max_load_reduction_pct=0.853516 moved_to_echo=3922 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1276, 0, 0, 1023, 0, 1023, 0, 1019, 0, 630, 736, 737, 1026, 0, 722, 0] rerouted_tokens_per_ep_rank=[1276, 1023, 1023, 1019, 630, 1473, 1026, 722] avg=1024.0000 max=1473 max_rank=5 min=630 min_rank=4 imbalance_ratio=1.438477 max_load_reduction=6719 max_load_reduction_pct=0.820190 moved_to_echo=3851 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1717, 0, 5, 1024, 3, 1024, 4, 1020, 17, 570, 157, 998, 1010, 0, 643, 0] rerouted_tokens_per_ep_rank=[1717, 1029, 1027, 1024, 587, 1155, 1010, 643] avg=1024.0000 max=1717 max_rank=0 min=587 min_rank=4 imbalance_ratio=1.676758 max_load_reduction=6475 max_load_reduction_pct=0.790405 moved_to_echo=3395 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[827, 0, 0, 1024, 0, 1024, 4, 1019, 0, 756, 1170, 477, 1026, 0, 865, 0] rerouted_tokens_per_ep_rank=[827, 1024, 1024, 1023, 756, 1647, 1026, 865] avg=1024.0000 max=1647 max_rank=5 min=756 min_rank=4 imbalance_ratio=1.608398 max_load_reduction=6545 max_load_reduction_pct=0.798950 moved_to_echo=4294 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[321, 0, 0, 1024, 0, 1025, 0, 1020, 0, 1571, 224, 184, 1027, 0, 1796, 0] rerouted_tokens_per_ep_rank=[321, 1024, 1025, 1020, 1571, 408, 1027, 1796] avg=1024.0000 max=1796 max_rank=7 min=321 min_rank=0 imbalance_ratio=1.753906 max_load_reduction=6396 max_load_reduction_pct=0.780762 moved_to_echo=4802 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[421, 0, 0, 1020, 0, 1020, 0, 1015, 0, 1606, 10, 242, 1022, 0, 1836, 0] rerouted_tokens_per_ep_rank=[421, 1020, 1020, 1015, 1606, 252, 1022, 1836] avg=1024.0000 max=1836 max_rank=7 min=252 min_rank=5 imbalance_ratio=1.792969 max_load_reduction=6356 max_load_reduction_pct=0.775879 moved_to_echo=4716 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8163 expected_overflow=0 exact_recv_per_rank=[9173, 8192, 8192, 8163, 7211, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9173, 8192, 8192, 8163, 7211, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9173 expected_overflow=0 exact_recv_per_rank=[9173, 8192, 8192, 8163, 7211, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9173, 8192, 8192, 8163, 7211, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9173, 8192, 8192, 8163, 7211, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7211 expected_overflow=0 exact_recv_per_rank=[9173, 8192, 8192, 8163, 7211, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[9173, 8192, 8192, 8163, 7211, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8221 expected_overflow=0 exact_recv_per_rank=[9173, 8192, 8192, 8163, 7211, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7211 actual_valid_recv=7424 observed_loss=-213 tokens_per_expert=[128, 7296] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8221 actual_valid_recv=8320 observed_loss=-99 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=9173 actual_valid_recv=9216 observed_loss=-43 tokens_per_expert=[9216, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8163 actual_valid_recv=8320 observed_loss=-157 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2944, 5376] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=7424 slack_slots=8960 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=9216 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 106, 4096, 0, 0, 55, 3935, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[18, 35, 4069, 0, 0, 32, 4017, 21] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 1, 6, 4089, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 4095, 0, 0, 1, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4, 4096, 0, 0, 21, 4071, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 60, 4093, 1, 1, 20, 4017, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 4092, 1, 4, 36, 4058, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 51, 4045, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 0, 1009, 1027, 0, 0, 1024, 1, 1024, 6, 1008, 1039, 0, 0, 1033] rerouted_tokens_per_ep_rank=[1021, 1009, 1027, 1024, 1025, 1014, 1039, 1033] avg=1024.0000 max=1039 max_rank=6 min=1009 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=7153 max_load_reduction_pct=0.873169 moved_to_echo=4111 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 106, 971, 1027, 0, 0, 1024, 0, 1024, 55, 970, 1000, 0, 0, 994] rerouted_tokens_per_ep_rank=[1021, 1077, 1027, 1024, 1024, 1025, 1000, 994] avg=1024.0000 max=1077 max_rank=1 min=994 min_rank=7 imbalance_ratio=1.051758 max_load_reduction=7115 max_load_reduction_pct=0.868530 moved_to_echo=4043 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 2, 1011, 1026, 0, 0, 1024, 0, 1024, 1, 1009, 1040, 0, 0, 1034] rerouted_tokens_per_ep_rank=[1021, 1013, 1026, 1024, 1024, 1010, 1040, 1034] avg=1024.0000 max=1040 max_rank=6 min=1010 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[18, 1020, 35, 995, 1006, 0, 0, 1023, 0, 1020, 32, 993, 1010, 0, 21, 1019] rerouted_tokens_per_ep_rank=[1038, 1030, 1006, 1023, 1020, 1025, 1010, 1040] avg=1024.0000 max=1040 max_rank=7 min=1006 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1020, 1, 1002, 1026, 0, 1, 1023, 4, 1023, 36, 1000, 1031, 0, 0, 1025] rerouted_tokens_per_ep_rank=[1020, 1003, 1026, 1024, 1027, 1036, 1031, 1025] avg=1024.0000 max=1036 max_rank=5 min=1003 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4119 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1020, 60, 992, 1026, 0, 1, 1024, 1, 1023, 20, 990, 1020, 0, 0, 1015] rerouted_tokens_per_ep_rank=[1020, 1052, 1026, 1025, 1024, 1010, 1020, 1015] avg=1024.0000 max=1052 max_rank=1 min=1010 min_rank=5 imbalance_ratio=1.027344 max_load_reduction=7140 max_load_reduction_pct=0.871582 moved_to_echo=4069 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 4, 1005, 1027, 0, 0, 1024, 0, 1024, 21, 1003, 1034, 0, 0, 1029] rerouted_tokens_per_ep_rank=[1021, 1009, 1027, 1024, 1024, 1024, 1034, 1029] avg=1024.0000 max=1034 max_rank=6 min=1009 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4111 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 0, 999, 1027, 0, 0, 1024, 0, 1024, 51, 997, 1027, 0, 0, 1022] rerouted_tokens_per_ep_rank=[1021, 999, 1027, 1024, 1024, 1048, 1027, 1022] avg=1024.0000 max=1048 max_rank=5 min=999 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=7144 max_load_reduction_pct=0.872070 moved_to_echo=4121 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8183, 8192, 8192, 8192, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8183, 8192, 8192, 8192, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8183, 8192, 8192, 8192, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8201 expected_overflow=0 exact_recv_per_rank=[8183, 8192, 8192, 8192, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8183, 8192, 8192, 8192, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8183 expected_overflow=0 exact_recv_per_rank=[8183, 8192, 8192, 8192, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8183, 8192, 8192, 8192, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8183, 8192, 8192, 8192, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8183 actual_valid_recv=8320 observed_loss=-137 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8201 actual_valid_recv=8320 observed_loss=-119 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 3927, 4096, 0, 0, 169, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 3842, 4096, 0, 0, 254, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4047, 4096, 0, 0, 46, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[5, 32, 3920, 4072, 4, 6, 120, 33] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4083, 4096, 0, 0, 13, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4058, 4094, 0, 0, 40, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 3991, 4096, 0, 0, 105, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 3755, 4096, 0, 0, 341, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1013, 1020, 0, 1026, 0, 0, 1024, 0, 1022, 169, 882, 0, 1012] rerouted_tokens_per_ep_rank=[1024, 1013, 1020, 1026, 1024, 1022, 1051, 1012] avg=1024.0000 max=1051 max_rank=6 min=1012 min_rank=7 imbalance_ratio=1.026367 max_load_reduction=7141 max_load_reduction_pct=0.871704 moved_to_echo=4109 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1044, 1051, 0, 1026, 0, 0, 1024, 0, 1022, 46, 909, 3, 1043] rerouted_tokens_per_ep_rank=[1024, 1044, 1051, 1026, 1024, 1022, 955, 1046] avg=1024.0000 max=1051 max_rank=2 min=955 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=7141 max_load_reduction_pct=0.871704 moved_to_echo=4047 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 991, 998, 0, 1026, 0, 0, 1024, 0, 1022, 254, 863, 0, 990] rerouted_tokens_per_ep_rank=[1024, 991, 998, 1026, 1024, 1022, 1117, 990] avg=1024.0000 max=1117 max_rank=6 min=990 min_rank=7 imbalance_ratio=1.090820 max_load_reduction=7075 max_load_reduction_pct=0.863647 moved_to_echo=4153 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1053, 1060, 0, 1026, 0, 0, 1024, 0, 1022, 13, 917, 0, 1053] rerouted_tokens_per_ep_rank=[1024, 1053, 1060, 1026, 1024, 1022, 930, 1053] avg=1024.0000 max=1060 max_rank=2 min=930 min_rank=6 imbalance_ratio=1.035156 max_load_reduction=7132 max_load_reduction_pct=0.870605 moved_to_echo=4029 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[5, 1020, 32, 1015, 1007, 0, 1010, 0, 4, 1021, 6, 1021, 120, 883, 33, 1015] rerouted_tokens_per_ep_rank=[1025, 1047, 1007, 1010, 1025, 1027, 1003, 1048] avg=1024.0000 max=1048 max_rank=7 min=1003 min_rank=6 imbalance_ratio=1.023438 max_load_reduction=7144 max_load_reduction_pct=0.872070 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1047, 1054, 0, 1026, 0, 0, 1023, 0, 1022, 40, 911, 0, 1046] rerouted_tokens_per_ep_rank=[1023, 1047, 1054, 1026, 1023, 1022, 951, 1046] avg=1024.0000 max=1054 max_rank=2 min=951 min_rank=6 imbalance_ratio=1.029297 max_load_reduction=7138 max_load_reduction_pct=0.871338 moved_to_echo=4042 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1029, 1037, 0, 1026, 0, 0, 1024, 0, 1022, 105, 896, 0, 1029] rerouted_tokens_per_ep_rank=[1024, 1029, 1037, 1026, 1024, 1022, 1001, 1029] avg=1024.0000 max=1037 max_rank=2 min=1001 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=7155 max_load_reduction_pct=0.873413 moved_to_echo=4076 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 968, 976, 0, 1026, 0, 0, 1024, 0, 1022, 341, 843, 0, 968] rerouted_tokens_per_ep_rank=[1024, 968, 976, 1026, 1024, 1022, 1184, 968] avg=1024.0000 max=1184 max_rank=6 min=968 min_rank=1 imbalance_ratio=1.156250 max_load_reduction=7008 max_load_reduction_pct=0.855469 moved_to_echo=4198 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8203, 8192, 8192, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8203, 8192, 8192, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8181 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8203, 8192, 8192, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8203, 8192, 8192, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8203 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8203, 8192, 8192, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8203, 8192, 8192, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8203, 8192, 8192, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8203, 8192, 8192, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8203 actual_valid_recv=8320 observed_loss=-117 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8181 actual_valid_recv=8320 observed_loss=-139 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1, 4081, 2, 0, 4096, 10, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4, 4079, 1, 13, 4093, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[6, 4059, 6, 8, 4056, 35, 22, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[19, 4077, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[33, 4059, 0, 0, 4096, 0, 3, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[10, 4086, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 1015, 1028, 0, 2, 1024, 0, 1021, 1026, 0, 10, 1018, 1, 1020, 1, 1025] rerouted_tokens_per_ep_rank=[1016, 1028, 1026, 1021, 1026, 1028, 1021, 1026] avg=1024.0000 max=1028 max_rank=1 min=1016 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[19, 1014, 1027, 0, 0, 1024, 0, 1021, 1026, 0, 0, 1017, 0, 1019, 0, 1025] rerouted_tokens_per_ep_rank=[1033, 1027, 1024, 1021, 1026, 1017, 1019, 1025] avg=1024.0000 max=1033 max_rank=0 min=1017 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4, 1014, 1027, 0, 1, 1023, 13, 1020, 1026, 0, 0, 1018, 2, 1020, 0, 1024] rerouted_tokens_per_ep_rank=[1018, 1027, 1024, 1033, 1026, 1018, 1022, 1024] avg=1024.0000 max=1033 max_rank=3 min=1018 min_rank=0 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[33, 1009, 1022, 0, 0, 1024, 0, 1021, 1026, 0, 0, 1013, 3, 1015, 1, 1025] rerouted_tokens_per_ep_rank=[1042, 1022, 1024, 1021, 1026, 1013, 1018, 1026] avg=1024.0000 max=1042 max_rank=0 min=1013 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=7150 max_load_reduction_pct=0.872803 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[10, 1016, 1028, 0, 0, 1024, 0, 1021, 1026, 0, 0, 1020, 0, 1022, 0, 1025] rerouted_tokens_per_ep_rank=[1026, 1028, 1024, 1021, 1026, 1020, 1022, 1025] avg=1024.0000 max=1028 max_rank=1 min=1020 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[6, 1013, 1009, 0, 6, 1016, 8, 1014, 1010, 0, 35, 1017, 22, 1020, 0, 1016] rerouted_tokens_per_ep_rank=[1019, 1009, 1022, 1022, 1010, 1052, 1042, 1016] avg=1024.0000 max=1052 max_rank=5 min=1009 min_rank=1 imbalance_ratio=1.027344 max_load_reduction=7140 max_load_reduction_pct=0.871582 moved_to_echo=4120 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1019, 1031, 0, 0, 1024, 0, 1021, 1026, 0, 0, 1022, 0, 1024, 0, 1025] rerouted_tokens_per_ep_rank=[1019, 1031, 1024, 1021, 1026, 1022, 1024, 1025] avg=1024.0000 max=1031 max_rank=1 min=1019 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1019, 1031, 0, 0, 1024, 0, 1021, 1026, 0, 0, 1022, 0, 1024, 0, 1025] rerouted_tokens_per_ep_rank=[1019, 1031, 1024, 1021, 1026, 1022, 1024, 1025] avg=1024.0000 max=1031 max_rank=1 min=1019 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8181 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8192, 8181, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8192, 8181, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8192, 8181, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8192, 8181, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8192, 8181, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8192, 8181, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8203 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8192, 8181, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8192, 8181, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8203 actual_valid_recv=8320 observed_loss=-117 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8181 actual_valid_recv=8320 observed_loss=-139 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[2, 23, 4076, 4, 0, 18, 1, 4068] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 3, 4090, 3, 0, 0, 2, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 2, 4088, 3, 4, 0, 1, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1021, 1030, 0, 0, 1023, 0, 1021, 0, 1022, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1021, 1030, 1023, 1021, 1022, 1024, 1027] avg=1024.0000 max=1030 max_rank=2 min=1021 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 3, 1020, 1027, 0, 3, 1022, 0, 1021, 0, 1021, 2, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1027, 1025, 1021, 1021, 1026, 1025] avg=1024.0000 max=1027 max_rank=2 min=1021 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 1022, 23, 1020, 1010, 0, 4, 1023, 0, 1019, 18, 1023, 1, 1020, 1007, 0] rerouted_tokens_per_ep_rank=[1024, 1043, 1010, 1027, 1019, 1041, 1021, 1007] avg=1024.0000 max=1043 max_rank=1 min=1007 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=7149 max_load_reduction_pct=0.872681 moved_to_echo=4088 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1021, 1029, 0, 1, 1023, 0, 1021, 0, 1022, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1021, 1029, 1024, 1021, 1022, 1024, 1027] avg=1024.0000 max=1029 max_rank=2 min=1021 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1021, 1030, 0, 0, 1023, 0, 1021, 0, 1022, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1021, 1030, 1023, 1021, 1022, 1024, 1027] avg=1024.0000 max=1030 max_rank=2 min=1021 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 2, 1019, 1028, 0, 3, 1021, 4, 1021, 0, 1020, 1, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1021, 1028, 1024, 1025, 1020, 1025, 1025] avg=1024.0000 max=1028 max_rank=2 min=1020 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1021, 1030, 0, 0, 1023, 0, 1021, 0, 1022, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1021, 1030, 1023, 1021, 1022, 1024, 1027] avg=1024.0000 max=1030 max_rank=2 min=1021 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1021, 1030, 0, 0, 1023, 0, 1021, 0, 1022, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1021, 1030, 1023, 1021, 1022, 1024, 1027] avg=1024.0000 max=1030 max_rank=2 min=1021 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8214, 8192, 8170, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8170 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8214, 8192, 8170, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8214, 8192, 8170, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8214 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8214, 8192, 8170, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8214, 8192, 8170, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8214, 8192, 8170, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8214, 8192, 8170, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8214, 8192, 8170, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8214 actual_valid_recv=8320 observed_loss=-106 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8170 actual_valid_recv=8320 observed_loss=-150 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1, 3196, 0, 20, 879, 2, 22, 4072] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4, 3750, 0, 3, 341, 0, 0, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 3186, 0, 1, 908, 3, 2, 4092] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 3907, 0, 0, 188, 1, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 3275, 0, 0, 821, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 3890, 0, 0, 206, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 3795, 0, 0, 301, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 3830, 0, 0, 266, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1115, 0, 0, 1024, 0, 1106, 188, 580, 1, 1021, 0, 1106, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1115, 1024, 1106, 768, 1022, 1106, 1027] avg=1024.0000 max=1115 max_rank=1 min=768 min_rank=4 imbalance_ratio=1.088867 max_load_reduction=7077 max_load_reduction_pct=0.863892 moved_to_echo=3923 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 909, 0, 0, 1023, 1, 902, 908, 473, 3, 1020, 2, 902, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 909, 1023, 903, 1381, 1023, 904, 1026] avg=1024.0000 max=1381 max_rank=4 min=903 min_rank=3 imbalance_ratio=1.348633 max_load_reduction=6811 max_load_reduction_pct=0.831421 moved_to_echo=4334 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 935, 0, 0, 1024, 0, 927, 821, 486, 0, 1021, 0, 927, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 935, 1024, 927, 1307, 1021, 927, 1027] avg=1024.0000 max=1307 max_rank=4 min=927 min_rank=3 imbalance_ratio=1.276367 max_load_reduction=6885 max_load_reduction_pct=0.840454 moved_to_echo=4282 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 1021, 899, 0, 0, 1025, 20, 909, 879, 479, 2, 1021, 22, 909, 1005, 0] rerouted_tokens_per_ep_rank=[1022, 899, 1025, 929, 1358, 1023, 931, 1005] avg=1024.0000 max=1358 max_rank=4 min=899 min_rank=1 imbalance_ratio=1.326172 max_load_reduction=6834 max_load_reduction_pct=0.834229 moved_to_echo=4317 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 1023, 1070, 0, 0, 1024, 3, 1062, 341, 556, 0, 1021, 0, 1062, 1026, 0] rerouted_tokens_per_ep_rank=[1027, 1070, 1024, 1065, 897, 1021, 1062, 1026] avg=1024.0000 max=1070 max_rank=1 min=897 min_rank=4 imbalance_ratio=1.044922 max_load_reduction=7122 max_load_reduction_pct=0.869385 moved_to_echo=4006 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1109, 0, 0, 1024, 0, 1102, 206, 577, 0, 1021, 0, 1102, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1109, 1024, 1102, 783, 1021, 1102, 1027] avg=1024.0000 max=1109 max_rank=1 min=783 min_rank=4 imbalance_ratio=1.083008 max_load_reduction=7083 max_load_reduction_pct=0.864624 moved_to_echo=3933 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1082, 0, 0, 1024, 0, 1075, 301, 563, 0, 1021, 0, 1075, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1082, 1024, 1075, 864, 1021, 1075, 1027] avg=1024.0000 max=1082 max_rank=1 min=864 min_rank=4 imbalance_ratio=1.056641 max_load_reduction=7110 max_load_reduction_pct=0.867920 moved_to_echo=3987 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1092, 0, 0, 1024, 0, 1085, 266, 568, 0, 1021, 0, 1085, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1092, 1024, 1085, 834, 1021, 1085, 1027] avg=1024.0000 max=1092 max_rank=1 min=834 min_rank=4 imbalance_ratio=1.066406 max_load_reduction=7100 max_load_reduction_pct=0.866699 moved_to_echo=3967 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8211, 8192, 8192, 8192, 8173, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8173 expected_overflow=0 exact_recv_per_rank=[8192, 8211, 8192, 8192, 8192, 8173, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8211, 8192, 8192, 8192, 8173, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8211, 8192, 8192, 8192, 8173, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8211 expected_overflow=0 exact_recv_per_rank=[8192, 8211, 8192, 8192, 8192, 8173, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8211, 8192, 8192, 8192, 8173, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8211, 8192, 8192, 8192, 8173, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8211, 8192, 8192, 8192, 8173, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3968, 4352] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8173 actual_valid_recv=8320 observed_loss=-147 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8211 actual_valid_recv=8320 observed_loss=-109 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[26, 4069, 51, 9, 0, 1, 4, 4032] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 220, 0, 0, 0, 0, 3876] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[5, 4092, 165, 0, 5, 0, 2, 3923] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 12, 0, 0, 0, 0, 4084] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 78, 0, 0, 0, 0, 4018] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4094, 225, 0, 0, 1, 3, 3869] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 95, 0, 0, 0, 0, 4001] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 66, 0, 0, 0, 0, 4030] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1047, 1027, 0, 12, 933, 0, 1049, 0, 1024, 0, 1024, 0, 1021, 1055, 0] rerouted_tokens_per_ep_rank=[1047, 1027, 945, 1049, 1024, 1024, 1021, 1055] avg=1024.0000 max=1055 max_rank=7 min=945 min_rank=2 imbalance_ratio=1.030273 max_load_reduction=7137 max_load_reduction_pct=0.871216 moved_to_echo=4124 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[26, 1037, 1005, 0, 51, 926, 9, 1041, 0, 1021, 1, 1023, 4, 1020, 1028, 0] rerouted_tokens_per_ep_rank=[1063, 1005, 977, 1050, 1021, 1024, 1024, 1028] avg=1024.0000 max=1063 max_rank=0 min=977 min_rank=2 imbalance_ratio=1.038086 max_load_reduction=7129 max_load_reduction_pct=0.870239 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[5, 1005, 1026, 0, 165, 897, 0, 1008, 5, 1023, 0, 1023, 2, 1020, 1013, 0] rerouted_tokens_per_ep_rank=[1010, 1026, 1062, 1008, 1028, 1023, 1022, 1013] avg=1024.0000 max=1062 max_rank=2 min=1008 min_rank=3 imbalance_ratio=1.037109 max_load_reduction=7130 max_load_reduction_pct=0.870361 moved_to_echo=4086 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 1027, 0, 78, 918, 0, 1032, 0, 1024, 0, 1024, 0, 1021, 1038, 0] rerouted_tokens_per_ep_rank=[1030, 1027, 996, 1032, 1024, 1024, 1021, 1038] avg=1024.0000 max=1038 max_rank=7 min=996 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4107 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 1027, 0, 95, 915, 0, 1028, 0, 1024, 0, 1024, 0, 1021, 1033, 0] rerouted_tokens_per_ep_rank=[1025, 1027, 1010, 1028, 1024, 1024, 1021, 1033] avg=1024.0000 max=1033 max_rank=7 min=1010 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 991, 1026, 0, 225, 884, 0, 994, 0, 1023, 1, 1024, 3, 1021, 1000, 0] rerouted_tokens_per_ep_rank=[991, 1026, 1109, 994, 1023, 1025, 1024, 1000] avg=1024.0000 max=1109 max_rank=2 min=991 min_rank=0 imbalance_ratio=1.083008 max_load_reduction=7083 max_load_reduction_pct=0.864624 moved_to_echo=4072 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 993, 1027, 0, 220, 886, 0, 996, 0, 1024, 0, 1024, 0, 1021, 1001, 0] rerouted_tokens_per_ep_rank=[993, 1027, 1106, 996, 1024, 1024, 1021, 1001] avg=1024.0000 max=1106 max_rank=2 min=993 min_rank=0 imbalance_ratio=1.080078 max_load_reduction=7086 max_load_reduction_pct=0.864990 moved_to_echo=4070 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1033, 1027, 0, 66, 921, 0, 1035, 0, 1024, 0, 1024, 0, 1021, 1041, 0] rerouted_tokens_per_ep_rank=[1033, 1027, 987, 1035, 1024, 1024, 1021, 1041] avg=1024.0000 max=1041 max_rank=7 min=987 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4110 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8175, 8209] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8175, 8209] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8175, 8209] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8175, 8209] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8175, 8209] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8209 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8175, 8209] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8175 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8175, 8209] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8175, 8209] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8209 actual_valid_recv=8320 observed_loss=-111 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1024, 7296] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8175 actual_valid_recv=8320 observed_loss=-145 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[15, 4068, 1368, 2, 19, 14, 2696, 10] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 1097, 0, 0, 0, 2999, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 2629, 0, 0, 0, 1467, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 2371, 0, 0, 0, 1725, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4094, 3529, 2, 0, 2, 565, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1, 4094, 776, 0, 1, 2, 3318, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 1162, 0, 0, 0, 2934, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 1958, 0, 0, 0, 2138, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[15, 1020, 1007, 0, 749, 0, 2, 1020, 19, 619, 14, 1238, 1458, 0, 10, 1021] rerouted_tokens_per_ep_rank=[1035, 1007, 749, 1022, 638, 1252, 1458, 1031] avg=1024.0000 max=1458 max_rank=6 min=638 min_rank=4 imbalance_ratio=1.423828 max_load_reduction=6734 max_load_reduction_pct=0.822021 moved_to_echo=4379 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1027, 0, 604, 0, 0, 1024, 0, 493, 0, 1373, 1626, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1022, 1027, 604, 1024, 493, 1373, 1626, 1023] avg=1024.0000 max=1626 max_rank=6 min=493 min_rank=4 imbalance_ratio=1.587891 max_load_reduction=6566 max_load_reduction_pct=0.801514 moved_to_echo=4515 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1027, 0, 1305, 0, 0, 1024, 0, 1066, 0, 790, 935, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1022, 1027, 1305, 1024, 1066, 790, 935, 1023] avg=1024.0000 max=1305 max_rank=2 min=790 min_rank=5 imbalance_ratio=1.274414 max_load_reduction=6887 max_load_reduction_pct=0.840698 moved_to_echo=3814 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1025, 0, 1942, 0, 2, 1024, 0, 1587, 2, 258, 307, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1022, 1025, 1942, 1026, 1587, 260, 307, 1023] avg=1024.0000 max=1942 max_rank=2 min=260 min_rank=5 imbalance_ratio=1.896484 max_load_reduction=6250 max_load_reduction_pct=0.762939 moved_to_echo=3177 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1027, 0, 640, 0, 0, 1024, 0, 522, 0, 1344, 1590, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1022, 1027, 640, 1024, 522, 1344, 1590, 1023] avg=1024.0000 max=1590 max_rank=6 min=522 min_rank=4 imbalance_ratio=1.552734 max_load_reduction=6602 max_load_reduction_pct=0.805908 moved_to_echo=4479 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1022, 1025, 0, 427, 0, 0, 1024, 1, 349, 2, 1520, 1798, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 427, 1024, 350, 1522, 1798, 1023] avg=1024.0000 max=1798 max_rank=6 min=350 min_rank=4 imbalance_ratio=1.755859 max_load_reduction=6394 max_load_reduction_pct=0.780518 moved_to_echo=4693 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1027, 0, 1447, 0, 0, 1024, 0, 1182, 0, 672, 795, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1022, 1027, 1447, 1024, 1182, 672, 795, 1023] avg=1024.0000 max=1447 max_rank=2 min=672 min_rank=5 imbalance_ratio=1.413086 max_load_reduction=6745 max_load_reduction_pct=0.823364 moved_to_echo=3672 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1027, 0, 1078, 0, 0, 1024, 0, 880, 0, 979, 1159, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1022, 1027, 1078, 1024, 880, 979, 1159, 1023] avg=1024.0000 max=1159 max_rank=6 min=880 min_rank=4 imbalance_ratio=1.131836 max_load_reduction=7033 max_load_reduction_pct=0.858521 moved_to_echo=4041 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 6718, 8192, 9668, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6718 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 6718, 8192, 9668, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 6718, 8192, 9668, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 6718, 8192, 9668, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9668 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 6718, 8192, 9668, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 6718, 8192, 9668, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 6718, 8192, 9668, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 6718, 8192, 9668, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8190 actual_valid_recv=8320 observed_loss=-130 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=6718 actual_valid_recv=6912 observed_loss=-194 tokens_per_expert=[128, 6784] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9668 actual_valid_recv=9728 observed_loss=-60 tokens_per_expert=[9728, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=6912 slack_slots=9472 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9728 slack_slots=6656 device_initiated_grouped_gemm=True + [2026-08-30 04:54:02] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 669.8 | throughput per GPU (TFLOP/s/GPU): 497.7 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.082835E+01 | loss scale: 1.0 | grad norm: 92.786 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[14, 0, 18, 558, 255, 597, 3229, 3521] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[13, 14, 3, 437, 326, 1099, 3138, 3162] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[13, 13, 3, 806, 397, 942, 2561, 3457] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[19, 20, 2, 586, 327, 870, 2938, 3430] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[43, 1, 2, 788, 323, 959, 2891, 3185] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[45, 4, 1, 585, 356, 490, 3018, 3693] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[7, 1, 5, 404, 177, 640, 3506, 3452] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[10, 2, 2, 622, 527, 655, 2896, 3478] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[14, 685, 0, 1046, 18, 1048, 558, 454, 255, 735, 597, 259, 1096, 0, 1427, 0] rerouted_tokens_per_ep_rank=[699, 1046, 1066, 1012, 990, 856, 1096, 1427] avg=1024.0000 max=1427 max_rank=7 min=699 min_rank=0 imbalance_ratio=1.393555 max_load_reduction=6765 max_load_reduction_pct=0.825806 moved_to_echo=4369 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[13, 666, 14, 939, 3, 941, 437, 442, 326, 714, 1099, 251, 1065, 0, 1282, 0] rerouted_tokens_per_ep_rank=[679, 953, 944, 879, 1040, 1350, 1065, 1282] avg=1024.0000 max=1350 max_rank=5 min=679 min_rank=0 imbalance_ratio=1.318359 max_load_reduction=6842 max_load_reduction_pct=0.835205 moved_to_echo=4737 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[19, 624, 20, 1019, 2, 1021, 586, 413, 327, 668, 870, 235, 998, 0, 1390, 0] rerouted_tokens_per_ep_rank=[643, 1039, 1023, 999, 995, 1105, 998, 1390] avg=1024.0000 max=1390 max_rank=7 min=643 min_rank=0 imbalance_ratio=1.357422 max_load_reduction=6802 max_load_reduction_pct=0.830322 moved_to_echo=4488 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[13, 543, 13, 1027, 3, 1029, 806, 360, 397, 583, 942, 205, 870, 0, 1401, 0] rerouted_tokens_per_ep_rank=[556, 1040, 1032, 1166, 980, 1147, 870, 1401] avg=1024.0000 max=1401 max_rank=7 min=556 min_rank=0 imbalance_ratio=1.368164 max_load_reduction=6791 max_load_reduction_pct=0.828979 moved_to_echo=4398 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[45, 640, 4, 1097, 1, 1100, 585, 425, 356, 687, 490, 242, 1024, 0, 1496, 0] rerouted_tokens_per_ep_rank=[685, 1101, 1101, 1010, 1043, 732, 1024, 1496] avg=1024.0000 max=1496 max_rank=7 min=685 min_rank=0 imbalance_ratio=1.460938 max_load_reduction=6696 max_load_reduction_pct=0.817383 moved_to_echo=4295 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[43, 618, 1, 951, 2, 953, 788, 412, 323, 660, 959, 235, 966, 0, 1281, 0] rerouted_tokens_per_ep_rank=[661, 952, 955, 1200, 983, 1194, 966, 1281] avg=1024.0000 max=1281 max_rank=7 min=661 min_rank=0 imbalance_ratio=1.250977 max_load_reduction=6911 max_load_reduction_pct=0.843628 moved_to_echo=4424 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 744, 1, 1025, 5, 1028, 404, 493, 177, 798, 640, 281, 1190, 0, 1399, 0] rerouted_tokens_per_ep_rank=[751, 1026, 1033, 897, 975, 921, 1190, 1399] avg=1024.0000 max=1399 max_rank=7 min=751 min_rank=0 imbalance_ratio=1.366211 max_load_reduction=6793 max_load_reduction_pct=0.829224 moved_to_echo=4485 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 615, 2, 1033, 2, 1036, 622, 407, 527, 659, 655, 232, 983, 0, 1409, 0] rerouted_tokens_per_ep_rank=[625, 1035, 1038, 1029, 1186, 887, 983, 1409] avg=1024.0000 max=1409 max_rank=7 min=625 min_rank=0 imbalance_ratio=1.375977 max_load_reduction=6783 max_load_reduction_pct=0.828003 moved_to_echo=4465 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[5299, 8192, 8192, 8192, 8192, 8192, 8192, 11085] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[5299, 8192, 8192, 8192, 8192, 8192, 8192, 11085] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[5299, 8192, 8192, 8192, 8192, 8192, 8192, 11085] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[5299, 8192, 8192, 8192, 8192, 8192, 8192, 11085] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11085 expected_overflow=0 exact_recv_per_rank=[5299, 8192, 8192, 8192, 8192, 8192, 8192, 11085] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[5299, 8192, 8192, 8192, 8192, 8192, 8192, 11085] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5299 expected_overflow=0 exact_recv_per_rank=[5299, 8192, 8192, 8192, 8192, 8192, 8192, 11085] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[5299, 8192, 8192, 8192, 8192, 8192, 8192, 11085] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=5299 actual_valid_recv=5504 observed_loss=-205 tokens_per_expert=[256, 5248] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[2688, 5504] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6272, 2048] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4864, 3456] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=11085 actual_valid_recv=11136 observed_loss=-51 tokens_per_expert=[11136, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=5504 slack_slots=10880 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=11136 slack_slots=5248 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[185, 0, 0, 0, 4096, 280, 3502, 129] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[232, 2, 6, 0, 4092, 501, 3338, 21] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[228, 1, 0, 0, 4091, 165, 3697, 10] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[199, 0, 15, 2, 4090, 222, 3617, 47] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[207, 0, 2, 0, 4094, 100, 3752, 37] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[175, 0, 0, 0, 4096, 75, 3842, 4] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[230, 0, 6, 0, 4095, 372, 3445, 44] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[159, 0, 0, 0, 4095, 141, 3771, 26] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[228, 839, 1, 1022, 0, 1019, 0, 1023, 1027, 0, 165, 808, 1047, 0, 10, 1003] rerouted_tokens_per_ep_rank=[1067, 1023, 1019, 1023, 1027, 973, 1047, 1013] avg=1024.0000 max=1067 max_rank=0 min=973 min_rank=5 imbalance_ratio=1.041992 max_load_reduction=7125 max_load_reduction_pct=0.869751 moved_to_echo=4060 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[232, 757, 2, 1023, 6, 1019, 0, 1023, 1027, 0, 501, 730, 946, 0, 21, 905] rerouted_tokens_per_ep_rank=[989, 1025, 1025, 1023, 1027, 1231, 946, 926] avg=1024.0000 max=1231 max_rank=5 min=926 min_rank=7 imbalance_ratio=1.202148 max_load_reduction=6961 max_load_reduction_pct=0.849731 moved_to_echo=4130 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[185, 795, 0, 1024, 0, 1020, 0, 1024, 1028, 0, 280, 766, 991, 0, 129, 950] rerouted_tokens_per_ep_rank=[980, 1024, 1020, 1024, 1028, 1046, 991, 1079] avg=1024.0000 max=1079 max_rank=7 min=980 min_rank=0 imbalance_ratio=1.053711 max_load_reduction=7113 max_load_reduction_pct=0.868286 moved_to_echo=4144 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[207, 851, 0, 1023, 2, 1020, 0, 1023, 1028, 0, 100, 820, 1063, 0, 37, 1018] rerouted_tokens_per_ep_rank=[1058, 1023, 1022, 1023, 1028, 920, 1063, 1055] avg=1024.0000 max=1063 max_rank=6 min=920 min_rank=5 imbalance_ratio=1.038086 max_load_reduction=7129 max_load_reduction_pct=0.870239 moved_to_echo=4066 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[199, 825, 0, 1027, 15, 1025, 2, 1025, 1013, 0, 222, 795, 1013, 0, 47, 984] rerouted_tokens_per_ep_rank=[1024, 1027, 1040, 1027, 1013, 1017, 1013, 1031] avg=1024.0000 max=1040 max_rank=2 min=1013 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4074 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[230, 782, 0, 1023, 6, 1020, 0, 1024, 1028, 0, 372, 753, 976, 0, 44, 934] rerouted_tokens_per_ep_rank=[1012, 1023, 1026, 1024, 1028, 1125, 976, 978] avg=1024.0000 max=1125 max_rank=5 min=976 min_rank=6 imbalance_ratio=1.098633 max_load_reduction=7067 max_load_reduction_pct=0.862671 moved_to_echo=4107 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[175, 872, 0, 1024, 0, 1020, 0, 1024, 1028, 0, 75, 840, 1088, 0, 4, 1042] rerouted_tokens_per_ep_rank=[1047, 1024, 1020, 1024, 1028, 915, 1088, 1046] avg=1024.0000 max=1088 max_rank=6 min=915 min_rank=5 imbalance_ratio=1.062500 max_load_reduction=7104 max_load_reduction_pct=0.867188 moved_to_echo=4077 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[159, 856, 0, 1023, 0, 1020, 0, 1024, 1028, 0, 141, 824, 1068, 0, 26, 1023] rerouted_tokens_per_ep_rank=[1015, 1023, 1020, 1024, 1028, 965, 1068, 1049] avg=1024.0000 max=1068 max_rank=6 min=965 min_rank=5 imbalance_ratio=1.042969 max_load_reduction=7124 max_load_reduction_pct=0.869629 moved_to_echo=4110 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8192, 8192, 8177] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8192, 8192, 8177] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8192, 8192, 8177] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8192, 8192, 8177] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8207 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8192, 8192, 8177] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8192, 8192, 8177] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8192, 8192, 8177] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8177 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8192, 8192, 8177] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1664, 6656] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8207 actual_valid_recv=8320 observed_loss=-113 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1920, 6400] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8177 actual_valid_recv=8320 observed_loss=-143 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 4055, 6, 93, 60, 24, 29, 3925] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4095, 1, 127, 0, 0, 0, 3969] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 4094, 2, 94, 3, 2, 0, 3997] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1, 4094, 0, 48, 56, 0, 8, 3985] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 0, 87, 2, 0, 23, 3984] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 0, 119, 2, 0, 0, 3975] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1, 4096, 0, 70, 40, 0, 0, 3985] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1, 4095, 0, 121, 0, 0, 5, 3970] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1014, 1017, 0, 6, 1014, 93, 917, 60, 991, 24, 1010, 29, 1003, 1014, 0] rerouted_tokens_per_ep_rank=[1014, 1017, 1020, 1010, 1051, 1034, 1032, 1014] avg=1024.0000 max=1051 max_rank=4 min=1010 min_rank=3 imbalance_ratio=1.026367 max_load_reduction=7141 max_load_reduction_pct=0.871704 moved_to_echo=4131 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1027, 0, 2, 1023, 94, 934, 3, 1009, 2, 1020, 0, 1021, 1033, 0] rerouted_tokens_per_ep_rank=[1024, 1027, 1025, 1028, 1012, 1022, 1021, 1033] avg=1024.0000 max=1033 max_rank=7 min=1012 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4088 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1027, 0, 1, 1024, 127, 928, 0, 1002, 0, 1020, 0, 1014, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1027, 1025, 1055, 1002, 1020, 1014, 1025] avg=1024.0000 max=1055 max_rank=3 min=1002 min_rank=4 imbalance_ratio=1.030273 max_load_reduction=7137 max_load_reduction_pct=0.871216 moved_to_echo=4061 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1028, 1016, 0, 0, 1026, 48, 935, 56, 1010, 0, 1024, 8, 1023, 1017, 0] rerouted_tokens_per_ep_rank=[1029, 1016, 1026, 983, 1066, 1024, 1031, 1017] avg=1024.0000 max=1066 max_rank=4 min=983 min_rank=3 imbalance_ratio=1.041016 max_load_reduction=7126 max_load_reduction_pct=0.869873 moved_to_echo=4138 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1025, 1026, 0, 0, 1024, 70, 931, 40, 1006, 0, 1021, 0, 1018, 1030, 0] rerouted_tokens_per_ep_rank=[1026, 1026, 1024, 1001, 1046, 1021, 1018, 1030] avg=1024.0000 max=1046 max_rank=4 min=1001 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=7146 max_load_reduction_pct=0.872314 moved_to_echo=4115 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 1026, 0, 0, 1024, 87, 931, 2, 1006, 0, 1021, 23, 1018, 1029, 0] rerouted_tokens_per_ep_rank=[1025, 1026, 1024, 1018, 1008, 1021, 1041, 1029] avg=1024.0000 max=1041 max_rank=6 min=1008 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 1026, 0, 0, 1024, 119, 929, 2, 1003, 0, 1021, 0, 1016, 1027, 0] rerouted_tokens_per_ep_rank=[1025, 1026, 1024, 1048, 1005, 1021, 1016, 1027] avg=1024.0000 max=1048 max_rank=3 min=1005 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=7144 max_load_reduction_pct=0.872070 moved_to_echo=4069 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1024, 1027, 0, 0, 1024, 121, 928, 0, 1002, 0, 1020, 5, 1014, 1026, 0] rerouted_tokens_per_ep_rank=[1025, 1027, 1024, 1049, 1002, 1020, 1019, 1026] avg=1024.0000 max=1049 max_rank=3 min=1002 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=7143 max_load_reduction_pct=0.871948 moved_to_echo=4067 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8183 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8183, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8201 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8183, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8183, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8183, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8183, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8183, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8183, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8183, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8201 actual_valid_recv=8320 observed_loss=-119 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8183 actual_valid_recv=8320 observed_loss=-137 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[3922, 0, 175, 0, 0, 0, 1, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[3785, 0, 311, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[3637, 1, 443, 15, 1, 9, 1, 4085] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[3459, 0, 637, 1, 0, 2, 0, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[3731, 3, 361, 2, 1, 1, 0, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3655, 0, 441, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[3773, 0, 321, 2, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[3932, 0, 163, 2, 0, 0, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1078, 0, 0, 1022, 175, 700, 0, 1071, 0, 1023, 0, 1073, 1, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1078, 1022, 875, 1071, 1023, 1073, 1024, 1026] avg=1024.0000 max=1078 max_rank=0 min=875 min_rank=2 imbalance_ratio=1.052734 max_load_reduction=7114 max_load_reduction_pct=0.868408 moved_to_echo=4146 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1000, 0, 1, 1019, 443, 649, 15, 993, 1, 1021, 9, 995, 1, 1021, 1024, 0] rerouted_tokens_per_ep_rank=[1000, 1020, 1092, 1008, 1022, 1004, 1022, 1024] avg=1024.0000 max=1092 max_rank=2 min=1000 min_rank=0 imbalance_ratio=1.066406 max_load_reduction=7100 max_load_reduction_pct=0.866699 moved_to_echo=4072 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[951, 0, 0, 1021, 637, 617, 1, 945, 0, 1023, 2, 946, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[951, 1021, 1254, 946, 1023, 948, 1023, 1026] avg=1024.0000 max=1254 max_rank=2 min=946 min_rank=3 imbalance_ratio=1.224609 max_load_reduction=6938 max_load_reduction_pct=0.846924 moved_to_echo=4020 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1040, 0, 0, 1022, 311, 676, 0, 1034, 0, 1024, 0, 1035, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1040, 1022, 987, 1034, 1024, 1035, 1024, 1026] avg=1024.0000 max=1040 max_rank=0 min=987 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4109 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1012, 0, 3, 1026, 361, 671, 2, 1024, 1, 1027, 1, 1024, 0, 1027, 1013, 0] rerouted_tokens_per_ep_rank=[1012, 1029, 1032, 1026, 1028, 1025, 1027, 1013] avg=1024.0000 max=1032 max_rank=2 min=1012 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1037, 0, 0, 1022, 321, 673, 2, 1031, 0, 1024, 0, 1032, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1037, 1022, 994, 1033, 1024, 1032, 1024, 1026] avg=1024.0000 max=1037 max_rank=0 min=994 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=7155 max_load_reduction_pct=0.873413 moved_to_echo=4106 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1005, 0, 0, 1022, 441, 652, 0, 998, 0, 1024, 0, 1000, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1005, 1022, 1093, 998, 1024, 1000, 1024, 1026] avg=1024.0000 max=1093 max_rank=2 min=998 min_rank=3 imbalance_ratio=1.067383 max_load_reduction=7099 max_load_reduction_pct=0.866577 moved_to_echo=4074 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1081, 0, 0, 1022, 163, 702, 2, 1074, 0, 1024, 0, 1075, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1081, 1022, 865, 1076, 1024, 1075, 1024, 1025] avg=1024.0000 max=1081 max_rank=0 min=865 min_rank=2 imbalance_ratio=1.055664 max_load_reduction=7111 max_load_reduction_pct=0.868042 moved_to_echo=4148 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8204, 8180, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8204, 8180, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8204, 8180, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8204 expected_overflow=0 exact_recv_per_rank=[8204, 8180, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8204, 8180, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8204, 8180, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8204, 8180, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8180 expected_overflow=0 exact_recv_per_rank=[8204, 8180, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8204 actual_valid_recv=8320 observed_loss=-116 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2944, 5376] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8180 actual_valid_recv=8320 observed_loss=-140 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1, 3846, 232, 9, 0, 18, 0, 4086] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 4078, 14, 4, 2, 0, 1, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 4070, 26, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 4078, 18, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1, 4060, 34, 0, 0, 2, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4077, 17, 0, 0, 2, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 4087, 9, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1020, 974, 0, 232, 931, 9, 971, 0, 1021, 18, 970, 0, 1021, 1024, 0] rerouted_tokens_per_ep_rank=[1021, 974, 1163, 980, 1021, 988, 1021, 1024] avg=1024.0000 max=1163 max_rank=2 min=974 min_rank=1 imbalance_ratio=1.135742 max_load_reduction=7029 max_load_reduction_pct=0.858032 moved_to_echo=4054 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1022, 1029, 0, 34, 982, 0, 1025, 0, 1023, 2, 1024, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1029, 1016, 1025, 1023, 1026, 1024, 1026] avg=1024.0000 max=1029 max_rank=1 min=1016 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 14, 990, 4, 1032, 2, 1026, 0, 1032, 1, 1026, 1017, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1004, 1036, 1028, 1032, 1027, 1017] avg=1024.0000 max=1036 max_rank=3 min=1004 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1034, 0, 18, 987, 0, 1029, 0, 1024, 0, 1028, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1034, 1005, 1029, 1024, 1028, 1024, 1025] avg=1024.0000 max=1034 max_rank=1 min=1005 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1034, 0, 17, 986, 0, 1029, 0, 1024, 2, 1028, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1034, 1003, 1029, 1024, 1030, 1024, 1025] avg=1024.0000 max=1034 max_rank=1 min=1003 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1036, 0, 9, 989, 0, 1032, 0, 1024, 0, 1030, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1036, 998, 1032, 1024, 1030, 1024, 1025] avg=1024.0000 max=1036 max_rank=1 min=998 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1032, 0, 26, 985, 0, 1027, 0, 1024, 0, 1026, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1032, 1011, 1027, 1024, 1026, 1024, 1025] avg=1024.0000 max=1032 max_rank=1 min=1011 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1038, 0, 1, 991, 0, 1034, 0, 1024, 0, 1032, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1038, 992, 1034, 1024, 1032, 1024, 1025] avg=1024.0000 max=1038 max_rank=1 min=992 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8183, 8201, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8183, 8201, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8183, 8201, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8201 expected_overflow=0 exact_recv_per_rank=[8183, 8201, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8183, 8201, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8183, 8201, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8183 expected_overflow=0 exact_recv_per_rank=[8183, 8201, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8183, 8201, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8201 actual_valid_recv=8320 observed_loss=-119 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8183 actual_valid_recv=8320 observed_loss=-137 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4077, 4, 11, 5, 0, 4, 4089, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 2, 0, 0, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 3, 4094, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4089, 0, 7, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1022, 0, 4, 1021, 11, 1017, 5, 1019, 0, 1022, 4, 1019, 1024, 0, 2, 1022] rerouted_tokens_per_ep_rank=[1022, 1025, 1028, 1024, 1022, 1023, 1024, 1024] avg=1024.0000 max=1028 max_rank=2 min=1022 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1022, 0, 1024, 0, 1024, 0, 1024, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1022, 1024, 1024, 1024, 1026, 1023] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1022, 0, 1022, 2, 1024, 0, 1023, 0, 1024, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1022, 1022, 1026, 1023, 1024, 1026, 1023] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1022, 0, 1024, 0, 1024, 0, 1024, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1022, 1024, 1024, 1024, 1026, 1023] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 7, 1020, 0, 1022, 0, 1024, 0, 1022, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1027, 1022, 1024, 1022, 1026, 1023] avg=1024.0000 max=1027 max_rank=2 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1018, 0, 0, 1027, 0, 1027, 0, 1024, 0, 1027, 3, 1024, 1012, 0, 2, 1028] rerouted_tokens_per_ep_rank=[1018, 1027, 1027, 1024, 1027, 1027, 1012, 1030] avg=1024.0000 max=1030 max_rank=7 min=1012 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1022, 0, 1024, 0, 1024, 0, 1024, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1022, 1024, 1024, 1024, 1026, 1023] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1022, 0, 1024, 0, 1024, 0, 1024, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1022, 1024, 1024, 1024, 1026, 1023] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8189 actual_valid_recv=8320 observed_loss=-131 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3606, 486, 6, 0, 2, 5, 4084, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[3973, 118, 5, 0, 1, 0, 4092, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3909, 187, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[3947, 150, 0, 0, 0, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3773, 323, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[3956, 140, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[3613, 481, 0, 0, 2, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3852, 244, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[967, 0, 486, 713, 6, 963, 0, 1021, 2, 1020, 5, 1020, 1023, 0, 3, 963] rerouted_tokens_per_ep_rank=[967, 1199, 969, 1021, 1022, 1025, 1023, 966] avg=1024.0000 max=1199 max_rank=1 min=966 min_rank=7 imbalance_ratio=1.170898 max_load_reduction=6993 max_load_reduction_pct=0.853638 moved_to_echo=4036 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1053, 0, 118, 790, 5, 1064, 0, 1027, 1, 1029, 0, 1023, 1013, 0, 3, 1066] rerouted_tokens_per_ep_rank=[1053, 908, 1069, 1027, 1030, 1023, 1013, 1069] avg=1024.0000 max=1069 max_rank=2 min=908 min_rank=1 imbalance_ratio=1.043945 max_load_reduction=7123 max_load_reduction_pct=0.869507 moved_to_echo=4135 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1058, 0, 150, 781, 0, 1054, 0, 1024, 0, 1023, 0, 1022, 1026, 0, 0, 1054] rerouted_tokens_per_ep_rank=[1058, 931, 1054, 1024, 1023, 1022, 1026, 1054] avg=1024.0000 max=1058 max_rank=0 min=931 min_rank=1 imbalance_ratio=1.033203 max_load_reduction=7134 max_load_reduction_pct=0.870850 moved_to_echo=4125 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[968, 0, 481, 715, 0, 965, 0, 1024, 2, 1023, 0, 1023, 1026, 0, 0, 965] rerouted_tokens_per_ep_rank=[968, 1196, 965, 1024, 1025, 1023, 1026, 965] avg=1024.0000 max=1196 max_rank=1 min=965 min_rank=2 imbalance_ratio=1.167969 max_load_reduction=6996 max_load_reduction_pct=0.854004 moved_to_echo=4039 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1060, 0, 140, 783, 0, 1056, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1057] rerouted_tokens_per_ep_rank=[1060, 923, 1056, 1024, 1023, 1023, 1026, 1057] avg=1024.0000 max=1060 max_rank=0 min=923 min_rank=1 imbalance_ratio=1.035156 max_load_reduction=7132 max_load_reduction_pct=0.870605 moved_to_echo=4129 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1048, 0, 187, 773, 0, 1044, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1044] rerouted_tokens_per_ep_rank=[1048, 960, 1044, 1024, 1023, 1023, 1026, 1044] avg=1024.0000 max=1048 max_rank=0 min=960 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=7144 max_load_reduction_pct=0.872070 moved_to_echo=4116 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1012, 0, 323, 746, 0, 1007, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1008] rerouted_tokens_per_ep_rank=[1012, 1069, 1007, 1024, 1023, 1023, 1026, 1008] avg=1024.0000 max=1069 max_rank=1 min=1007 min_rank=2 imbalance_ratio=1.043945 max_load_reduction=7123 max_load_reduction_pct=0.869507 moved_to_echo=4080 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1033, 0, 244, 762, 0, 1028, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1029] rerouted_tokens_per_ep_rank=[1033, 1006, 1028, 1024, 1023, 1023, 1026, 1029] avg=1024.0000 max=1033 max_rank=0 min=1006 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8185 actual_valid_recv=8320 observed_loss=-135 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2176, 6144] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[56, 0, 2, 0, 1, 0, 4093, 4040] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1773, 9, 0, 34, 3, 0, 4025, 2348] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[906, 0, 0, 0, 0, 0, 4096, 3190] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[629, 0, 0, 2, 0, 0, 4082, 3479] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[549, 0, 0, 1, 0, 1, 4095, 3546] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[796, 0, 0, 3, 1, 0, 4094, 3298] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[199, 0, 0, 0, 0, 2, 4096, 3895] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[300, 0, 0, 3, 0, 0, 4093, 3796] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1773, 253, 9, 696, 0, 1008, 34, 693, 3, 998, 0, 1008, 1011, 0, 706, 0] rerouted_tokens_per_ep_rank=[2026, 705, 1008, 727, 1001, 1008, 1011, 706] avg=1024.0000 max=2026 max_rank=0 min=705 min_rank=1 imbalance_ratio=1.978516 max_load_reduction=6166 max_load_reduction_pct=0.752686 moved_to_echo=3726 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[56, 441, 0, 1201, 2, 1030, 0, 1195, 1, 1018, 0, 1029, 1016, 0, 1203, 0] rerouted_tokens_per_ep_rank=[497, 1201, 1032, 1195, 1019, 1029, 1016, 1203] avg=1024.0000 max=1203 max_rank=7 min=497 min_rank=0 imbalance_ratio=1.174805 max_load_reduction=6989 max_load_reduction_pct=0.853149 moved_to_echo=4267 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[906, 344, 0, 946, 0, 1026, 0, 942, 0, 1015, 0, 1026, 1029, 0, 958, 0] rerouted_tokens_per_ep_rank=[1250, 946, 1026, 942, 1015, 1026, 1029, 958] avg=1024.0000 max=1250 max_rank=0 min=942 min_rank=3 imbalance_ratio=1.220703 max_load_reduction=6942 max_load_reduction_pct=0.847412 moved_to_echo=4028 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[796, 356, 0, 978, 0, 1026, 3, 974, 1, 1015, 0, 1026, 1027, 0, 990, 0] rerouted_tokens_per_ep_rank=[1152, 978, 1026, 977, 1016, 1026, 1027, 990] avg=1024.0000 max=1152 max_rank=0 min=977 min_rank=3 imbalance_ratio=1.125000 max_load_reduction=7040 max_load_reduction_pct=0.859375 moved_to_echo=4059 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[549, 383, 0, 1051, 0, 1026, 1, 1047, 0, 1015, 1, 1026, 1028, 0, 1065, 0] rerouted_tokens_per_ep_rank=[932, 1051, 1026, 1048, 1015, 1027, 1028, 1065] avg=1024.0000 max=1065 max_rank=7 min=932 min_rank=0 imbalance_ratio=1.040039 max_load_reduction=7127 max_load_reduction_pct=0.869995 moved_to_echo=4135 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[629, 376, 0, 1031, 0, 1023, 2, 1027, 0, 1012, 0, 1023, 1024, 0, 1045, 0] rerouted_tokens_per_ep_rank=[1005, 1031, 1023, 1029, 1012, 1023, 1024, 1045] avg=1024.0000 max=1045 max_rank=7 min=1005 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=7147 max_load_reduction_pct=0.872437 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[199, 421, 0, 1155, 0, 1026, 0, 1150, 0, 1015, 2, 1026, 1029, 0, 1169, 0] rerouted_tokens_per_ep_rank=[620, 1155, 1026, 1150, 1015, 1028, 1029, 1169] avg=1024.0000 max=1169 max_rank=7 min=620 min_rank=0 imbalance_ratio=1.141602 max_load_reduction=7023 max_load_reduction_pct=0.857300 moved_to_echo=4241 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[300, 410, 0, 1125, 0, 1025, 3, 1121, 0, 1015, 0, 1025, 1028, 0, 1140, 0] rerouted_tokens_per_ep_rank=[710, 1125, 1025, 1124, 1015, 1025, 1028, 1140] avg=1024.0000 max=1140 max_rank=7 min=710 min_rank=0 imbalance_ratio=1.113281 max_load_reduction=7052 max_load_reduction_pct=0.860840 moved_to_echo=4208 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8108, 8192, 8192, 8276] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8108, 8192, 8192, 8276] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8108, 8192, 8192, 8276] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8108, 8192, 8192, 8276] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8108 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8108, 8192, 8192, 8276] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8108, 8192, 8192, 8276] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8276 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8108, 8192, 8192, 8276] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8108, 8192, 8192, 8276] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5248, 3072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8276 actual_valid_recv=8320 observed_loss=-44 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8108 actual_valid_recv=8320 observed_loss=-212 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[2755, 0, 3, 2, 2, 1335, 3, 4092] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[2864, 1, 1, 0, 4, 1219, 13, 4090] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1541, 0, 0, 0, 0, 2555, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2738, 0, 0, 0, 0, 1358, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2731, 0, 0, 0, 0, 1365, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1857, 0, 0, 0, 0, 2239, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[3060, 0, 0, 0, 0, 1036, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[2215, 0, 0, 0, 0, 1881, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[903, 0, 0, 1024, 0, 1023, 0, 1024, 0, 638, 1612, 0, 0, 943, 1025, 0] rerouted_tokens_per_ep_rank=[903, 1024, 1023, 1024, 638, 1612, 943, 1025] avg=1024.0000 max=1612 max_rank=5 min=638 min_rank=4 imbalance_ratio=1.574219 max_load_reduction=6580 max_load_reduction_pct=0.803223 moved_to_echo=4218 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1678, 0, 1, 1022, 1, 1021, 0, 1022, 4, 1186, 769, 0, 13, 450, 1025, 0] rerouted_tokens_per_ep_rank=[1678, 1023, 1022, 1022, 1190, 769, 463, 1025] avg=1024.0000 max=1678 max_rank=0 min=463 min_rank=6 imbalance_ratio=1.638672 max_load_reduction=6514 max_load_reduction_pct=0.795166 moved_to_echo=3447 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1611, 0, 0, 1025, 3, 1026, 2, 1024, 2, 1144, 839, 0, 3, 496, 1017, 0] rerouted_tokens_per_ep_rank=[1611, 1025, 1029, 1026, 1146, 839, 499, 1017] avg=1024.0000 max=1611 max_rank=0 min=499 min_rank=6 imbalance_ratio=1.573242 max_load_reduction=6581 max_load_reduction_pct=0.803345 moved_to_echo=3501 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1604, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1134, 857, 0, 0, 501, 1025, 0] rerouted_tokens_per_ep_rank=[1604, 1024, 1023, 1024, 1134, 857, 501, 1025] avg=1024.0000 max=1604 max_rank=0 min=501 min_rank=6 imbalance_ratio=1.566406 max_load_reduction=6588 max_load_reduction_pct=0.804199 moved_to_echo=3517 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1088, 0, 0, 1024, 0, 1023, 0, 1024, 0, 769, 1413, 0, 0, 826, 1025, 0] rerouted_tokens_per_ep_rank=[1088, 1024, 1023, 1024, 769, 1413, 826, 1025] avg=1024.0000 max=1413 max_rank=5 min=769 min_rank=4 imbalance_ratio=1.379883 max_load_reduction=6779 max_load_reduction_pct=0.827515 moved_to_echo=4033 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1600, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1131, 861, 0, 0, 504, 1025, 0] rerouted_tokens_per_ep_rank=[1600, 1024, 1023, 1024, 1131, 861, 504, 1025] avg=1024.0000 max=1600 max_rank=0 min=504 min_rank=6 imbalance_ratio=1.562500 max_load_reduction=6592 max_load_reduction_pct=0.804688 moved_to_echo=3521 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1298, 0, 0, 1024, 0, 1023, 0, 1024, 0, 917, 1187, 0, 0, 694, 1025, 0] rerouted_tokens_per_ep_rank=[1298, 1024, 1023, 1024, 917, 1187, 694, 1025] avg=1024.0000 max=1298 max_rank=0 min=694 min_rank=6 imbalance_ratio=1.267578 max_load_reduction=6894 max_load_reduction_pct=0.841553 moved_to_echo=3823 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1793, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1267, 654, 0, 0, 382, 1025, 0] rerouted_tokens_per_ep_rank=[1793, 1024, 1023, 1024, 1267, 654, 382, 1025] avg=1024.0000 max=1793 max_rank=0 min=382 min_rank=6 imbalance_ratio=1.750977 max_load_reduction=6399 max_load_reduction_pct=0.781128 moved_to_echo=3328 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11575, 8192, 8189, 8192, 8192, 8192, 4812, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11575, 8192, 8189, 8192, 8192, 8192, 4812, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[11575, 8192, 8189, 8192, 8192, 8192, 4812, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11575 expected_overflow=0 exact_recv_per_rank=[11575, 8192, 8189, 8192, 8192, 8192, 4812, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11575, 8192, 8189, 8192, 8192, 8192, 4812, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11575, 8192, 8189, 8192, 8192, 8192, 4812, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[11575, 8192, 8189, 8192, 8192, 8192, 4812, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4812 expected_overflow=0 exact_recv_per_rank=[11575, 8192, 8189, 8192, 8192, 8192, 4812, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=4812 actual_valid_recv=4992 observed_loss=-180 tokens_per_expert=[128, 4864] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8189 actual_valid_recv=8320 observed_loss=-131 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=11575 actual_valid_recv=11648 observed_loss=-73 tokens_per_expert=[11648, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=4992 slack_slots=11392 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=11648 slack_slots=4736 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4, 4092, 4088, 2, 1, 5, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4089, 4088, 3, 5, 1, 4, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1027, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1027, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1024, 0, 1025, 0, 3, 1021, 5, 1021, 1, 1021, 4, 1022, 2, 1022] rerouted_tokens_per_ep_rank=[1021, 1024, 1025, 1024, 1026, 1022, 1026, 1024] avg=1024.0000 max=1026 max_rank=4 min=1021 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 1028, 1012, 0, 1006, 0, 2, 1028, 1, 1027, 5, 1027, 0, 1028, 0, 1024] rerouted_tokens_per_ep_rank=[1032, 1012, 1006, 1030, 1028, 1032, 1028, 1024] avg=1024.0000 max=1032 max_rank=0 min=1006 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4112 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1027, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1027, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1027, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1027, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8191 actual_valid_recv=8320 observed_loss=-129 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4, 1, 2, 4089, 3, 0, 4086, 7] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[3, 0, 5, 4090, 0, 3, 4091, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 1021, 1, 1022, 2, 1021, 1024, 0, 3, 1022, 0, 1021, 1023, 0, 7, 1021] rerouted_tokens_per_ep_rank=[1025, 1023, 1023, 1024, 1025, 1021, 1023, 1028] avg=1024.0000 max=1028 max_rank=7 min=1021 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1024, 1023, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 1026, 0, 1025, 5, 1026, 1018, 0, 0, 1023, 3, 1024, 1013, 0, 0, 1026] rerouted_tokens_per_ep_rank=[1029, 1025, 1031, 1018, 1023, 1027, 1013, 1026] avg=1024.0000 max=1031 max_rank=2 min=1013 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1024, 1023, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1024, 1023, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1024, 1023, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1024, 1023, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1024, 1023, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8186, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8186, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8186, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8186 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8186, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8186, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8186, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8186, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8198 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8186, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8198 actual_valid_recv=8320 observed_loss=-122 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8186 actual_valid_recv=8320 observed_loss=-134 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 49, 7, 4035, 4087, 6, 0, 8] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 4, 4084, 4091, 3, 9, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 11, 0, 4085, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 35, 0, 4061, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1, 7, 0, 4088, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 62, 0, 4034, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4, 0, 4092, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 11, 1006, 0, 1025, 1029, 0, 1026, 0, 0, 1023, 0, 1025, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1017, 1025, 1029, 1026, 1023, 1025, 1023] avg=1024.0000 max=1029 max_rank=3 min=1017 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 49, 993, 7, 1013, 1017, 0, 1024, 0, 6, 1021, 0, 1012, 8, 1021] rerouted_tokens_per_ep_rank=[1021, 1042, 1020, 1017, 1024, 1027, 1012, 1029] avg=1024.0000 max=1042 max_rank=1 min=1012 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=7150 max_load_reduction_pct=0.872803 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 35, 1000, 0, 1019, 1023, 0, 1026, 0, 0, 1023, 0, 1019, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1035, 1019, 1023, 1026, 1023, 1019, 1023] avg=1024.0000 max=1035 max_rank=1 min=1019 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 1024, 0, 1010, 4, 1030, 1015, 0, 1018, 0, 3, 1024, 9, 1029, 0, 1025] rerouted_tokens_per_ep_rank=[1025, 1010, 1034, 1015, 1018, 1027, 1038, 1025] avg=1024.0000 max=1038 max_rank=6 min=1010 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 4, 1007, 0, 1027, 1031, 0, 1026, 0, 0, 1023, 0, 1027, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1011, 1027, 1031, 1026, 1023, 1027, 1023] avg=1024.0000 max=1031 max_rank=3 min=1011 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 62, 993, 0, 1013, 1016, 0, 1026, 0, 0, 1023, 0, 1012, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1055, 1013, 1016, 1026, 1023, 1012, 1023] avg=1024.0000 max=1055 max_rank=1 min=1012 min_rank=6 imbalance_ratio=1.030273 max_load_reduction=7137 max_load_reduction_pct=0.871216 moved_to_echo=4084 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 1024, 7, 1007, 0, 1026, 1029, 0, 1026, 0, 0, 1023, 0, 1026, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1014, 1026, 1029, 1026, 1023, 1026, 1023] avg=1024.0000 max=1029 max_rank=3 min=1014 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 1024, 0, 1008, 0, 1028, 1032, 0, 1026, 0, 0, 1023, 0, 1027, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1008, 1028, 1032, 1026, 1023, 1027, 1023] avg=1024.0000 max=1032 max_rank=3 min=1008 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8198, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8198 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8198, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8198, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8198, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8198, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8186 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8198, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8198, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8198, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8186 actual_valid_recv=8320 observed_loss=-134 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8198 actual_valid_recv=8320 observed_loss=-122 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4, 3, 4088, 0, 0, 4, 2, 4091] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[7, 8, 4085, 1, 0, 3, 0, 4088] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1027, 0, 0, 1024, 0, 1024, 0, 1023, 0, 1022, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1027, 1024, 1024, 1023, 1022, 1026] avg=1024.0000 max=1027 max_rank=2 min=1022 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1027, 0, 0, 1024, 0, 1024, 0, 1023, 0, 1022, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1027, 1024, 1024, 1023, 1022, 1026] avg=1024.0000 max=1027 max_rank=2 min=1022 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 1020, 8, 1020, 1025, 0, 1, 1022, 0, 1022, 3, 1020, 0, 1020, 1024, 0] rerouted_tokens_per_ep_rank=[1027, 1028, 1025, 1023, 1022, 1023, 1020, 1024] avg=1024.0000 max=1028 max_rank=1 min=1020 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4, 1023, 3, 1023, 1015, 0, 0, 1025, 0, 1026, 4, 1027, 2, 1028, 1012, 0] rerouted_tokens_per_ep_rank=[1027, 1026, 1015, 1025, 1026, 1031, 1030, 1012] avg=1024.0000 max=1031 max_rank=5 min=1012 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1027, 0, 0, 1024, 0, 1024, 0, 1023, 0, 1022, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1027, 1024, 1024, 1023, 1022, 1026] avg=1024.0000 max=1027 max_rank=2 min=1022 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1027, 0, 0, 1024, 0, 1024, 0, 1023, 0, 1022, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1027, 1024, 1024, 1023, 1022, 1026] avg=1024.0000 max=1027 max_rank=2 min=1022 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1027, 0, 0, 1024, 0, 1024, 0, 1023, 0, 1022, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1027, 1024, 1024, 1023, 1022, 1026] avg=1024.0000 max=1027 max_rank=2 min=1022 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1027, 0, 0, 1024, 0, 1024, 0, 1023, 0, 1022, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1027, 1024, 1024, 1023, 1022, 1026] avg=1024.0000 max=1027 max_rank=2 min=1022 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8202, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8202, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8202, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8202 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8202, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8182 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8202, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8202, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8202, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8202, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8182 actual_valid_recv=8320 observed_loss=-138 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8202 actual_valid_recv=8320 observed_loss=-118 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[2, 3, 1, 5, 4090, 0, 0, 4091] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 7, 1, 9, 4088, 1, 0, 4086] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1023, 0, 1024, 0, 1022, 1026, 0, 0, 1024, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1022, 1023, 1024, 1022, 1026, 1024, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1020, 7, 1020, 1, 1021, 9, 1020, 1024, 0, 1, 1022, 0, 1022, 1025, 0] rerouted_tokens_per_ep_rank=[1020, 1027, 1022, 1029, 1024, 1023, 1022, 1025] avg=1024.0000 max=1029 max_rank=3 min=1020 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 1027, 3, 1024, 1, 1025, 5, 1026, 1012, 0, 0, 1025, 0, 1026, 1016, 0] rerouted_tokens_per_ep_rank=[1029, 1027, 1026, 1031, 1012, 1025, 1026, 1016] avg=1024.0000 max=1031 max_rank=3 min=1012 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4079 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1023, 0, 1024, 0, 1022, 1026, 0, 0, 1024, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1022, 1023, 1024, 1022, 1026, 1024, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1023, 0, 1024, 0, 1022, 1026, 0, 0, 1024, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1022, 1023, 1024, 1022, 1026, 1024, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1023, 0, 1024, 0, 1022, 1026, 0, 0, 1024, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1022, 1023, 1024, 1022, 1026, 1024, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1023, 0, 1024, 0, 1022, 1026, 0, 0, 1024, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1022, 1023, 1024, 1022, 1026, 1024, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1023, 0, 1024, 0, 1022, 1026, 0, 0, 1024, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1022, 1023, 1024, 1022, 1026, 1024, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8181, 8192, 8192, 8192, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8181, 8192, 8192, 8192, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8181 expected_overflow=0 exact_recv_per_rank=[8181, 8192, 8192, 8192, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8181, 8192, 8192, 8192, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8181, 8192, 8192, 8192, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8181, 8192, 8192, 8192, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8181, 8192, 8192, 8192, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8203 expected_overflow=0 exact_recv_per_rank=[8181, 8192, 8192, 8192, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8181 actual_valid_recv=8320 observed_loss=-139 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8203 actual_valid_recv=8320 observed_loss=-117 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4, 4088, 3308, 7, 54, 4, 4, 723] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3494, 0, 110, 0, 0, 492] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4087, 3398, 6, 84, 4, 6, 607] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3509, 0, 163, 0, 0, 424] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3166, 0, 174, 0, 0, 756] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3237, 0, 112, 0, 0, 747] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3373, 0, 45, 0, 0, 678] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3581, 0, 21, 0, 0, 494] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4, 1022, 1024, 0, 1003, 0, 7, 999, 54, 907, 4, 1021, 4, 1021, 723, 399] rerouted_tokens_per_ep_rank=[1026, 1024, 1003, 1006, 961, 1025, 1025, 1122] avg=1024.0000 max=1122 max_rank=7 min=961 min_rank=4 imbalance_ratio=1.095703 max_load_reduction=7070 max_load_reduction_pct=0.863037 moved_to_echo=4133 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 1058, 0, 0, 1055, 110, 959, 0, 1023, 0, 1023, 492, 422] rerouted_tokens_per_ep_rank=[1024, 1026, 1058, 1055, 1069, 1023, 1023, 914] avg=1024.0000 max=1069 max_rank=4 min=914 min_rank=7 imbalance_ratio=1.043945 max_load_reduction=7123 max_load_reduction_pct=0.869507 moved_to_echo=4029 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 981, 0, 0, 977, 112, 888, 0, 1023, 0, 1023, 747, 391] rerouted_tokens_per_ep_rank=[1024, 1026, 981, 977, 1000, 1023, 1023, 1138] avg=1024.0000 max=1138 max_rank=7 min=977 min_rank=3 imbalance_ratio=1.111328 max_load_reduction=7054 max_load_reduction_pct=0.861084 moved_to_echo=4184 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1017, 0, 1018, 0, 6, 1029, 84, 937, 4, 1025, 6, 1023, 607, 414] rerouted_tokens_per_ep_rank=[1022, 1017, 1018, 1035, 1021, 1029, 1029, 1021] avg=1024.0000 max=1035 max_rank=3 min=1017 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 1023, 0, 0, 1018, 45, 925, 0, 1023, 0, 1023, 678, 407] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1018, 970, 1023, 1023, 1085] avg=1024.0000 max=1085 max_rank=7 min=970 min_rank=4 imbalance_ratio=1.059570 max_load_reduction=7107 max_load_reduction_pct=0.867554 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 960, 0, 0, 956, 174, 868, 0, 1023, 0, 1023, 756, 382] rerouted_tokens_per_ep_rank=[1024, 1026, 960, 956, 1042, 1023, 1023, 1138] avg=1024.0000 max=1138 max_rank=7 min=956 min_rank=3 imbalance_ratio=1.111328 max_load_reduction=7054 max_load_reduction_pct=0.861084 moved_to_echo=4226 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 1063, 0, 0, 1059, 163, 963, 0, 1023, 0, 1023, 424, 424] rerouted_tokens_per_ep_rank=[1024, 1026, 1063, 1059, 1126, 1023, 1023, 848] avg=1024.0000 max=1126 max_rank=4 min=848 min_rank=7 imbalance_ratio=1.099609 max_load_reduction=7066 max_load_reduction_pct=0.862549 moved_to_echo=4020 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 1086, 0, 0, 1081, 21, 982, 0, 1023, 0, 1023, 494, 432] rerouted_tokens_per_ep_rank=[1024, 1026, 1086, 1081, 1003, 1023, 1023, 926] avg=1024.0000 max=1086 max_rank=2 min=926 min_rank=7 imbalance_ratio=1.060547 max_load_reduction=7106 max_load_reduction_pct=0.867432 moved_to_echo=3975 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8187 expected_overflow=0 exact_recv_per_rank=[8192, 8197, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8197, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8197, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8197, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8197, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8197 expected_overflow=0 exact_recv_per_rank=[8192, 8197, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8197, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8197, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8197 actual_valid_recv=8320 observed_loss=-123 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4992, 3328] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8187 actual_valid_recv=8320 observed_loss=-133 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3322, 774] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 2, 4096, 0, 0, 0, 3785, 309] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[5, 2, 4088, 6, 6, 5, 3130, 950] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3594, 502] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3224, 872] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3646, 450] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4, 0, 4087, 3, 4, 2, 3523, 569] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 4096, 0, 0, 0, 3020, 1075] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 997, 0, 1023, 1027, 0, 0, 1023, 0, 997, 0, 1023, 1000, 0, 774, 328] rerouted_tokens_per_ep_rank=[997, 1023, 1027, 1023, 997, 1023, 1000, 1102] avg=1024.0000 max=1102 max_rank=7 min=997 min_rank=0 imbalance_ratio=1.076172 max_load_reduction=7090 max_load_reduction_pct=0.865479 moved_to_echo=4122 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 939, 2, 1021, 1025, 0, 6, 1021, 6, 940, 5, 1021, 942, 0, 950, 309] rerouted_tokens_per_ep_rank=[944, 1023, 1025, 1027, 946, 1026, 942, 1259] avg=1024.0000 max=1259 max_rank=7 min=942 min_rank=6 imbalance_ratio=1.229492 max_load_reduction=6933 max_load_reduction_pct=0.846313 moved_to_echo=4173 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1136, 2, 1023, 1027, 0, 0, 1023, 0, 1136, 0, 1023, 1140, 0, 309, 373] rerouted_tokens_per_ep_rank=[1136, 1025, 1027, 1023, 1136, 1023, 1140, 682] avg=1024.0000 max=1140 max_rank=6 min=682 min_rank=7 imbalance_ratio=1.113281 max_load_reduction=7052 max_load_reduction_pct=0.860840 moved_to_echo=3981 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1078, 0, 1023, 1027, 0, 0, 1023, 0, 1079, 0, 1023, 1083, 0, 502, 354] rerouted_tokens_per_ep_rank=[1078, 1023, 1027, 1023, 1079, 1023, 1083, 856] avg=1024.0000 max=1083 max_rank=6 min=856 min_rank=7 imbalance_ratio=1.057617 max_load_reduction=7109 max_load_reduction_pct=0.867798 moved_to_echo=4041 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1094, 0, 1023, 1027, 0, 0, 1023, 0, 1094, 0, 1023, 1098, 0, 450, 360] rerouted_tokens_per_ep_rank=[1094, 1023, 1027, 1023, 1094, 1023, 1098, 810] avg=1024.0000 max=1098 max_rank=6 min=810 min_rank=7 imbalance_ratio=1.072266 max_load_reduction=7094 max_load_reduction_pct=0.865967 moved_to_echo=4025 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4, 1062, 0, 1028, 1009, 0, 3, 1024, 4, 1062, 2, 1026, 1048, 0, 569, 351] rerouted_tokens_per_ep_rank=[1066, 1028, 1009, 1027, 1066, 1028, 1048, 920] avg=1024.0000 max=1066 max_rank=0 min=920 min_rank=7 imbalance_ratio=1.041016 max_load_reduction=7126 max_load_reduction_pct=0.869873 moved_to_echo=4062 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 967, 0, 1023, 1027, 0, 0, 1023, 0, 968, 0, 1023, 971, 0, 872, 318] rerouted_tokens_per_ep_rank=[967, 1023, 1027, 1023, 968, 1023, 971, 1190] avg=1024.0000 max=1190 max_rank=7 min=967 min_rank=0 imbalance_ratio=1.162109 max_load_reduction=7002 max_load_reduction_pct=0.854736 moved_to_echo=4152 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 906, 1, 1023, 1027, 0, 0, 1023, 0, 906, 0, 1023, 910, 0, 1075, 298] rerouted_tokens_per_ep_rank=[906, 1024, 1027, 1023, 906, 1023, 910, 1373] avg=1024.0000 max=1373 max_rank=7 min=906 min_rank=0 imbalance_ratio=1.340820 max_load_reduction=6819 max_load_reduction_pct=0.832397 moved_to_echo=4212 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8188 actual_valid_recv=8320 observed_loss=-132 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5504, 2816] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[4, 2, 2, 637, 213, 1307, 2870, 3157] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[18, 56, 5, 377, 173, 1438, 2398, 3727] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1, 1, 8, 600, 329, 927, 2958, 3368] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[10, 12, 6, 602, 111, 517, 3283, 3651] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1, 6, 27, 500, 162, 780, 3106, 3610] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 4, 515, 435, 780, 3306, 3151] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[28, 1, 9, 1179, 327, 921, 2443, 3284] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 11, 1, 548, 178, 641, 3297, 3516] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 938, 2, 620, 2, 938, 637, 397, 213, 763, 1307, 110, 980, 0, 1281, 0] rerouted_tokens_per_ep_rank=[942, 622, 940, 1034, 976, 1417, 980, 1281] avg=1024.0000 max=1417 max_rank=5 min=622 min_rank=1 imbalance_ratio=1.383789 max_load_reduction=6775 max_load_reduction_pct=0.827026 moved_to_echo=4654 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 996, 1, 636, 8, 997, 600, 404, 329, 783, 927, 110, 1025, 0, 1375, 0] rerouted_tokens_per_ep_rank=[997, 637, 1005, 1004, 1112, 1037, 1025, 1375] avg=1024.0000 max=1375 max_rank=7 min=637 min_rank=1 imbalance_ratio=1.342773 max_load_reduction=6817 max_load_reduction_pct=0.832153 moved_to_echo=4549 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[18, 1103, 56, 515, 5, 1103, 377, 327, 173, 634, 1438, 89, 833, 0, 1521, 0] rerouted_tokens_per_ep_rank=[1121, 571, 1108, 704, 807, 1527, 833, 1521] avg=1024.0000 max=1527 max_rank=5 min=571 min_rank=1 imbalance_ratio=1.491211 max_load_reduction=6665 max_load_reduction_pct=0.813599 moved_to_echo=4688 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 1080, 12, 706, 6, 1080, 602, 448, 111, 869, 517, 122, 1138, 0, 1491, 0] rerouted_tokens_per_ep_rank=[1090, 718, 1086, 1050, 980, 639, 1138, 1491] avg=1024.0000 max=1491 max_rank=7 min=639 min_rank=5 imbalance_ratio=1.456055 max_load_reduction=6701 max_load_reduction_pct=0.817993 moved_to_echo=4248 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[28, 972, 1, 525, 9, 972, 1179, 333, 327, 646, 921, 90, 849, 0, 1340, 0] rerouted_tokens_per_ep_rank=[1000, 526, 981, 1512, 973, 1011, 849, 1340] avg=1024.0000 max=1512 max_rank=3 min=526 min_rank=1 imbalance_ratio=1.476562 max_load_reduction=6680 max_load_reduction_pct=0.815430 moved_to_echo=4173 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 1068, 6, 668, 27, 1068, 500, 424, 162, 822, 780, 115, 1077, 0, 1474, 0] rerouted_tokens_per_ep_rank=[1069, 674, 1095, 924, 984, 895, 1077, 1474] avg=1024.0000 max=1474 max_rank=7 min=674 min_rank=1 imbalance_ratio=1.439453 max_load_reduction=6718 max_load_reduction_pct=0.820068 moved_to_echo=4430 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 1040, 11, 709, 1, 1040, 548, 450, 178, 872, 641, 122, 1144, 0, 1436, 0] rerouted_tokens_per_ep_rank=[1040, 720, 1041, 998, 1050, 763, 1144, 1436] avg=1024.0000 max=1436 max_rank=7 min=720 min_rank=1 imbalance_ratio=1.402344 max_load_reduction=6756 max_load_reduction_pct=0.824707 moved_to_echo=4393 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 932, 0, 711, 4, 932, 515, 451, 435, 875, 780, 123, 1146, 0, 1287, 0] rerouted_tokens_per_ep_rank=[933, 711, 936, 966, 1310, 903, 1146, 1287] avg=1024.0000 max=1310 max_rank=4 min=711 min_rank=1 imbalance_ratio=1.279297 max_load_reduction=6882 max_load_reduction_pct=0.840088 moved_to_echo=4646 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5179, 8192, 8192, 8192, 8192, 8192, 11205] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5179, 8192, 8192, 8192, 8192, 8192, 11205] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5179, 8192, 8192, 8192, 8192, 8192, 11205] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5179, 8192, 8192, 8192, 8192, 8192, 11205] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5179, 8192, 8192, 8192, 8192, 8192, 11205] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 5179, 8192, 8192, 8192, 8192, 8192, 11205] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11205 expected_overflow=0 exact_recv_per_rank=[8192, 5179, 8192, 8192, 8192, 8192, 8192, 11205] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5179 expected_overflow=0 exact_recv_per_rank=[8192, 5179, 8192, 8192, 8192, 8192, 8192, 11205] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=5179 actual_valid_recv=5248 observed_loss=-69 tokens_per_expert=[128, 5120] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2048, 6272] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4992, 3328] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7424, 896] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=11205 actual_valid_recv=11264 observed_loss=-59 tokens_per_expert=[11264, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=5248 slack_slots=11136 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=11264 slack_slots=5120 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[153, 1, 3, 0, 4081, 128, 3820, 6] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[173, 0, 0, 0, 4096, 127, 3795, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[162, 0, 0, 0, 4096, 318, 3604, 12] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[181, 1, 0, 1, 4096, 262, 3621, 30] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[180, 0, 4, 1, 4090, 229, 3386, 302] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[151, 0, 0, 0, 4096, 268, 3668, 9] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[212, 0, 1, 0, 4094, 57, 3826, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[205, 0, 1, 0, 4093, 148, 3745, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[153, 878, 1, 1020, 3, 1018, 0, 1020, 1023, 0, 128, 862, 1065, 0, 6, 1015] rerouted_tokens_per_ep_rank=[1031, 1021, 1021, 1020, 1023, 990, 1065, 1021] avg=1024.0000 max=1065 max_rank=6 min=990 min_rank=5 imbalance_ratio=1.040039 max_load_reduction=7127 max_load_reduction_pct=0.869995 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[162, 828, 0, 1024, 0, 1022, 0, 1024, 1026, 0, 318, 814, 1005, 0, 12, 957] rerouted_tokens_per_ep_rank=[990, 1024, 1022, 1024, 1026, 1132, 1005, 969] avg=1024.0000 max=1132 max_rank=5 min=969 min_rank=7 imbalance_ratio=1.105469 max_load_reduction=7060 max_load_reduction_pct=0.861816 moved_to_echo=4132 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[173, 876, 0, 1028, 0, 1024, 0, 1028, 1016, 0, 127, 861, 1046, 0, 1, 1012] rerouted_tokens_per_ep_rank=[1049, 1028, 1024, 1028, 1016, 988, 1046, 1013] avg=1024.0000 max=1049 max_rank=0 min=988 min_rank=5 imbalance_ratio=1.024414 max_load_reduction=7143 max_load_reduction_pct=0.871948 moved_to_echo=4063 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[151, 843, 0, 1024, 0, 1022, 0, 1024, 1026, 0, 268, 828, 1023, 0, 9, 974] rerouted_tokens_per_ep_rank=[994, 1024, 1022, 1024, 1026, 1096, 1023, 983] avg=1024.0000 max=1096 max_rank=5 min=983 min_rank=7 imbalance_ratio=1.070312 max_load_reduction=7096 max_load_reduction_pct=0.866211 moved_to_echo=4128 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[181, 832, 1, 1024, 0, 1022, 1, 1024, 1026, 0, 262, 817, 1010, 0, 30, 962] rerouted_tokens_per_ep_rank=[1013, 1025, 1022, 1025, 1026, 1079, 1010, 992] avg=1024.0000 max=1079 max_rank=5 min=992 min_rank=7 imbalance_ratio=1.053711 max_load_reduction=7113 max_load_reduction_pct=0.868286 moved_to_echo=4107 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[180, 778, 0, 1023, 4, 1020, 1, 1023, 1024, 0, 229, 764, 945, 0, 302, 899] rerouted_tokens_per_ep_rank=[958, 1023, 1024, 1024, 1024, 993, 945, 1201] avg=1024.0000 max=1201 max_rank=7 min=945 min_rank=6 imbalance_ratio=1.172852 max_load_reduction=6991 max_load_reduction_pct=0.853394 moved_to_echo=4163 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[212, 879, 0, 1024, 1, 1021, 0, 1024, 1025, 0, 57, 864, 1067, 0, 2, 1016] rerouted_tokens_per_ep_rank=[1091, 1024, 1022, 1024, 1025, 921, 1067, 1018] avg=1024.0000 max=1091 max_rank=0 min=921 min_rank=5 imbalance_ratio=1.065430 max_load_reduction=7101 max_load_reduction_pct=0.866821 moved_to_echo=4031 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[205, 861, 0, 1023, 1, 1021, 0, 1023, 1026, 0, 148, 845, 1044, 0, 0, 995] rerouted_tokens_per_ep_rank=[1066, 1023, 1022, 1023, 1026, 993, 1044, 995] avg=1024.0000 max=1066 max_rank=0 min=993 min_rank=5 imbalance_ratio=1.041016 max_load_reduction=7126 max_load_reduction_pct=0.869873 moved_to_echo=4058 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 8192, 8192, 8192, 8205, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 8192, 8192, 8192, 8205, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 8192, 8192, 8192, 8205, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 8192, 8192, 8192, 8205, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8205 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 8192, 8192, 8192, 8205, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8179 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 8192, 8192, 8192, 8205, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 8192, 8192, 8192, 8205, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 8192, 8192, 8192, 8205, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1664, 6656] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8205 actual_valid_recv=8320 observed_loss=-115 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1536, 6784] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8179 actual_valid_recv=8320 observed_loss=-141 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 4092, 2, 161, 1, 0, 2, 3933] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 0, 5, 57, 0, 0, 4034] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 4092, 0, 118, 32, 0, 0, 3950] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 4051, 18, 137, 0, 0, 37, 3949] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[11, 4086, 2, 127, 13, 1, 5, 3947] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 0, 123, 0, 0, 0, 3973] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 4093, 1, 51, 2, 0, 0, 4045] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1, 4094, 2, 38, 0, 0, 1, 4056] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 2, 1018, 161, 916, 1, 997, 0, 1025, 2, 1004, 1016, 0] rerouted_tokens_per_ep_rank=[1024, 1026, 1020, 1077, 998, 1025, 1006, 1016] avg=1024.0000 max=1077 max_rank=3 min=998 min_rank=4 imbalance_ratio=1.051758 max_load_reduction=7115 max_load_reduction_pct=0.868530 moved_to_echo=4045 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1018, 118, 920, 32, 1001, 0, 1025, 0, 1009, 1020, 0] rerouted_tokens_per_ep_rank=[1023, 1026, 1018, 1038, 1033, 1025, 1009, 1020] avg=1024.0000 max=1038 max_rank=3 min=1009 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 1017, 0, 0, 1023, 5, 945, 57, 1027, 0, 1028, 0, 1034, 1028, 0] rerouted_tokens_per_ep_rank=[1028, 1017, 1023, 950, 1084, 1028, 1034, 1028] avg=1024.0000 max=1084 max_rank=4 min=950 min_rank=3 imbalance_ratio=1.058594 max_load_reduction=7108 max_load_reduction_pct=0.867676 moved_to_echo=4174 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[11, 1021, 1026, 0, 2, 1016, 127, 919, 13, 1001, 1, 1023, 5, 1008, 1019, 0] rerouted_tokens_per_ep_rank=[1032, 1026, 1018, 1046, 1014, 1024, 1013, 1019] avg=1024.0000 max=1046 max_rank=3 min=1013 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=7146 max_load_reduction_pct=0.872314 moved_to_echo=4070 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1013, 1016, 0, 18, 1008, 137, 920, 0, 1001, 0, 1014, 37, 1008, 1020, 0] rerouted_tokens_per_ep_rank=[1013, 1016, 1026, 1057, 1001, 1014, 1045, 1020] avg=1024.0000 max=1057 max_rank=3 min=1001 min_rank=4 imbalance_ratio=1.032227 max_load_reduction=7135 max_load_reduction_pct=0.870972 moved_to_echo=4080 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1024, 1027, 0, 2, 1018, 38, 945, 0, 1028, 0, 1025, 1, 1036, 1047, 0] rerouted_tokens_per_ep_rank=[1025, 1027, 1020, 983, 1028, 1025, 1037, 1047] avg=1024.0000 max=1047 max_rank=7 min=983 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=7145 max_load_reduction_pct=0.872192 moved_to_echo=4137 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1027, 0, 1, 1018, 51, 942, 2, 1025, 0, 1025, 0, 1033, 1045, 0] rerouted_tokens_per_ep_rank=[1023, 1027, 1019, 993, 1027, 1025, 1033, 1045] avg=1024.0000 max=1045 max_rank=7 min=993 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=7147 max_load_reduction_pct=0.872437 moved_to_echo=4130 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1027, 0, 0, 1019, 123, 925, 0, 1007, 0, 1026, 0, 1015, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1027, 1019, 1048, 1007, 1026, 1015, 1026] avg=1024.0000 max=1048 max_rank=3 min=1007 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=7144 max_load_reduction_pct=0.872070 moved_to_echo=4074 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8163, 8192, 8192, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8163, 8192, 8192, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8163, 8192, 8192, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8221 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8163, 8192, 8192, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8163 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8163, 8192, 8192, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8163, 8192, 8192, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8163, 8192, 8192, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8163, 8192, 8192, 8192, 8192, 8221] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8221 actual_valid_recv=8320 observed_loss=-99 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8163 actual_valid_recv=8320 observed_loss=-157 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3889, 0, 208, 1, 1, 0, 0, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[3945, 0, 151, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[3869, 0, 224, 3, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[3821, 0, 275, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[3916, 0, 180, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[3346, 2, 742, 0, 0, 0, 6, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[3950, 0, 143, 1, 0, 2, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[3639, 0, 457, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1065, 0, 0, 1023, 151, 754, 0, 1063, 0, 1023, 0, 1023, 0, 1063, 1027, 0] rerouted_tokens_per_ep_rank=[1065, 1023, 905, 1063, 1023, 1023, 1063, 1027] avg=1024.0000 max=1065 max_rank=0 min=905 min_rank=2 imbalance_ratio=1.040039 max_load_reduction=7127 max_load_reduction_pct=0.869995 moved_to_echo=4136 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1057, 0, 0, 1023, 180, 749, 0, 1055, 0, 1023, 0, 1023, 0, 1055, 1027, 0] rerouted_tokens_per_ep_rank=[1057, 1023, 929, 1055, 1023, 1023, 1055, 1027] avg=1024.0000 max=1057 max_rank=0 min=929 min_rank=2 imbalance_ratio=1.032227 max_load_reduction=7135 max_load_reduction_pct=0.870972 moved_to_echo=4128 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1067, 0, 0, 1023, 143, 755, 1, 1064, 0, 1023, 2, 1023, 0, 1064, 1027, 0] rerouted_tokens_per_ep_rank=[1067, 1023, 898, 1065, 1023, 1025, 1064, 1027] avg=1024.0000 max=1067 max_rank=0 min=898 min_rank=2 imbalance_ratio=1.041992 max_load_reduction=7125 max_load_reduction_pct=0.869751 moved_to_echo=4139 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1021, 0, 0, 1029, 275, 734, 0, 1033, 0, 1030, 0, 1029, 0, 1033, 1008, 0] rerouted_tokens_per_ep_rank=[1021, 1029, 1009, 1033, 1030, 1029, 1033, 1008] avg=1024.0000 max=1033 max_rank=3 min=1008 min_rank=7 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1045, 0, 0, 1023, 224, 740, 3, 1042, 0, 1023, 0, 1023, 0, 1042, 1027, 0] rerouted_tokens_per_ep_rank=[1045, 1023, 964, 1045, 1023, 1023, 1042, 1027] avg=1024.0000 max=1045 max_rank=0 min=964 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=7147 max_load_reduction_pct=0.872437 moved_to_echo=4115 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1050, 0, 0, 1023, 208, 744, 1, 1047, 1, 1023, 0, 1023, 0, 1048, 1024, 0] rerouted_tokens_per_ep_rank=[1050, 1023, 952, 1048, 1024, 1023, 1048, 1024] avg=1024.0000 max=1050 max_rank=0 min=952 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=7142 max_load_reduction_pct=0.871826 moved_to_echo=4119 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[904, 0, 2, 1023, 742, 640, 0, 901, 0, 1023, 0, 1023, 6, 901, 1027, 0] rerouted_tokens_per_ep_rank=[904, 1025, 1382, 901, 1023, 1023, 907, 1027] avg=1024.0000 max=1382 max_rank=2 min=901 min_rank=3 imbalance_ratio=1.349609 max_load_reduction=6810 max_load_reduction_pct=0.831299 moved_to_echo=3980 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[983, 0, 0, 1023, 457, 696, 0, 980, 0, 1023, 0, 1023, 0, 980, 1027, 0] rerouted_tokens_per_ep_rank=[983, 1023, 1153, 980, 1023, 1023, 980, 1027] avg=1024.0000 max=1153 max_rank=2 min=980 min_rank=3 imbalance_ratio=1.125977 max_load_reduction=7039 max_load_reduction_pct=0.859253 moved_to_echo=4053 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8190 actual_valid_recv=8320 observed_loss=-130 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2432, 5888] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 4093, 3, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1, 4010, 83, 0, 0, 2, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4094, 2, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3, 3978, 116, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1, 4060, 35, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 3972, 124, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1, 4087, 7, 0, 0, 0, 2, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4053, 43, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1036, 1038, 0, 2, 984, 0, 1024, 0, 1024, 0, 1023, 0, 1036, 1025, 0] rerouted_tokens_per_ep_rank=[1036, 1038, 986, 1024, 1024, 1023, 1036, 1025] avg=1024.0000 max=1038 max_rank=1 min=986 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1014, 1017, 0, 83, 964, 0, 1024, 0, 1024, 2, 1023, 0, 1015, 1025, 0] rerouted_tokens_per_ep_rank=[1015, 1017, 1047, 1024, 1024, 1025, 1015, 1025] avg=1024.0000 max=1047 max_rank=2 min=1015 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=7145 max_load_reduction_pct=0.872192 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1039, 1026, 0, 3, 988, 0, 1026, 0, 1026, 0, 1029, 0, 1040, 1015, 0] rerouted_tokens_per_ep_rank=[1039, 1026, 991, 1026, 1026, 1029, 1040, 1015] avg=1024.0000 max=1040 max_rank=6 min=991 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4110 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 1006, 1009, 0, 116, 956, 0, 1023, 0, 1023, 0, 1023, 0, 1007, 1026, 0] rerouted_tokens_per_ep_rank=[1009, 1009, 1072, 1023, 1023, 1023, 1007, 1026] avg=1024.0000 max=1072 max_rank=2 min=1007 min_rank=6 imbalance_ratio=1.046875 max_load_reduction=7120 max_load_reduction_pct=0.869141 moved_to_echo=4079 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1027, 1030, 0, 35, 976, 0, 1024, 0, 1024, 0, 1023, 0, 1027, 1025, 0] rerouted_tokens_per_ep_rank=[1028, 1030, 1011, 1024, 1024, 1023, 1027, 1025] avg=1024.0000 max=1030 max_rank=1 min=1011 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1005, 1007, 0, 124, 955, 0, 1024, 0, 1024, 0, 1023, 0, 1005, 1025, 0] rerouted_tokens_per_ep_rank=[1005, 1007, 1079, 1024, 1024, 1023, 1005, 1025] avg=1024.0000 max=1079 max_rank=2 min=1005 min_rank=0 imbalance_ratio=1.053711 max_load_reduction=7113 max_load_reduction_pct=0.868286 moved_to_echo=4077 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1034, 1037, 0, 7, 982, 0, 1023, 0, 1023, 0, 1023, 2, 1034, 1026, 0] rerouted_tokens_per_ep_rank=[1035, 1037, 989, 1023, 1023, 1023, 1036, 1026] avg=1024.0000 max=1037 max_rank=1 min=989 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=7155 max_load_reduction_pct=0.873413 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 1028, 0, 43, 974, 0, 1024, 0, 1024, 0, 1023, 0, 1026, 1025, 0] rerouted_tokens_per_ep_rank=[1025, 1028, 1017, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1028 max_rank=1 min=1017 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4094, 0, 4, 5, 0, 0, 4089, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4094, 2, 3, 0, 0, 0, 4093, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 8, 0, 0, 1, 4088, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 17, 12, 0, 0, 4067, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4067, 0, 19, 11, 0, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 4, 1017, 5, 1020, 0, 1024, 0, 1020, 1029, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1023, 1021, 1025, 1024, 1020, 1029, 1024] avg=1024.0000 max=1029 max_rank=6 min=1020 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1017, 17, 1011, 12, 1014, 0, 1025, 0, 1020, 1025, 0, 0, 1025] rerouted_tokens_per_ep_rank=[1026, 1017, 1028, 1026, 1025, 1020, 1025, 1025] avg=1024.0000 max=1028 max_rank=2 min=1017 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1014, 0, 0, 1028, 0, 1022, 0, 1026, 0, 1029, 0, 1024, 1020, 0, 0, 1029] rerouted_tokens_per_ep_rank=[1014, 1028, 1022, 1026, 1029, 1024, 1020, 1029] avg=1024.0000 max=1029 max_rank=4 min=1014 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 2, 1024, 3, 1018, 0, 1021, 0, 1024, 0, 1020, 1030, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1026, 1021, 1021, 1024, 1020, 1030, 1024] avg=1024.0000 max=1030 max_rank=6 min=1020 min_rank=5 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1025, 0, 1019, 1, 1021, 0, 1024, 0, 1020, 1031, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1025, 1019, 1022, 1024, 1020, 1031, 1024] avg=1024.0000 max=1031 max_rank=6 min=1019 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1020, 0, 0, 1025, 19, 1018, 11, 1021, 0, 1017, 0, 1013, 1031, 0, 0, 1017] rerouted_tokens_per_ep_rank=[1020, 1025, 1037, 1032, 1017, 1013, 1031, 1017] avg=1024.0000 max=1037 max_rank=2 min=1013 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=7155 max_load_reduction_pct=0.873413 moved_to_echo=4078 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1023, 8, 1017, 0, 1019, 0, 1024, 1, 1020, 1029, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1023, 1025, 1019, 1024, 1021, 1029, 1024] avg=1024.0000 max=1029 max_rank=6 min=1019 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1025, 0, 1019, 0, 1021, 0, 1025, 0, 1020, 1031, 0, 0, 1025] rerouted_tokens_per_ep_rank=[1026, 1025, 1019, 1021, 1025, 1020, 1031, 1025] avg=1024.0000 max=1031 max_rank=6 min=1019 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8158, 8226, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8158, 8226, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8158, 8226, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8158, 8226, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8158 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8158, 8226, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8158, 8226, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8158, 8226, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8226 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8158, 8226, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8158 actual_valid_recv=8320 observed_loss=-162 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8226 actual_valid_recv=8320 observed_loss=-94 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[3977, 119, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[3553, 543, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[3990, 106, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[3758, 338, 0, 1, 1, 0, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3792, 304, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3966, 129, 0, 0, 1, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3729, 367, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3876, 220, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1064, 0, 119, 787, 0, 1024, 0, 1063, 0, 1063, 0, 1023, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1064, 906, 1024, 1063, 1063, 1023, 1025, 1024] avg=1024.0000 max=1064 max_rank=0 min=906 min_rank=1 imbalance_ratio=1.039062 max_load_reduction=7128 max_load_reduction_pct=0.870117 moved_to_echo=4135 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[952, 0, 543, 703, 0, 1024, 0, 949, 0, 949, 0, 1023, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[952, 1246, 1024, 949, 949, 1023, 1025, 1024] avg=1024.0000 max=1246 max_rank=1 min=949 min_rank=3 imbalance_ratio=1.216797 max_load_reduction=6946 max_load_reduction_pct=0.847900 moved_to_echo=4021 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 304, 750, 0, 1024, 0, 1013, 0, 1013, 0, 1023, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1016, 1054, 1024, 1013, 1013, 1023, 1025, 1024] avg=1024.0000 max=1054 max_rank=1 min=1013 min_rank=3 imbalance_ratio=1.029297 max_load_reduction=7138 max_load_reduction_pct=0.871338 moved_to_echo=4085 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1007, 0, 338, 743, 0, 1023, 1, 1004, 1, 1004, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1007, 1081, 1023, 1005, 1005, 1023, 1025, 1023] avg=1024.0000 max=1081 max_rank=1 min=1005 min_rank=3 imbalance_ratio=1.055664 max_load_reduction=7111 max_load_reduction_pct=0.868042 moved_to_echo=4076 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1058, 0, 106, 793, 0, 1025, 0, 1070, 0, 1069, 0, 1029, 1017, 0, 0, 1025] rerouted_tokens_per_ep_rank=[1058, 899, 1025, 1070, 1069, 1029, 1017, 1025] avg=1024.0000 max=1070 max_rank=3 min=899 min_rank=1 imbalance_ratio=1.044922 max_load_reduction=7122 max_load_reduction_pct=0.869385 moved_to_echo=4140 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[999, 0, 367, 738, 0, 1024, 0, 996, 0, 996, 0, 1023, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[999, 1105, 1024, 996, 996, 1023, 1025, 1024] avg=1024.0000 max=1105 max_rank=1 min=996 min_rank=3 imbalance_ratio=1.079102 max_load_reduction=7087 max_load_reduction_pct=0.865112 moved_to_echo=4068 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1061, 0, 129, 785, 0, 1024, 0, 1060, 1, 1060, 0, 1023, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1061, 914, 1024, 1060, 1061, 1023, 1025, 1024] avg=1024.0000 max=1061 max_rank=0 min=914 min_rank=1 imbalance_ratio=1.036133 max_load_reduction=7131 max_load_reduction_pct=0.870483 moved_to_echo=4133 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1037, 0, 220, 767, 0, 1024, 0, 1036, 0, 1036, 0, 1023, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1037, 987, 1024, 1036, 1036, 1023, 1025, 1024] avg=1024.0000 max=1037 max_rank=0 min=987 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=7155 max_load_reduction_pct=0.873413 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8194, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8194, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8194, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8194, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8194, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8194, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8194, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8194, 8192, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8190 actual_valid_recv=8192 observed_loss=-2 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2176, 6144] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[185, 0, 0, 0, 0, 1, 4095, 3911] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[927, 0, 0, 0, 0, 0, 4096, 3169] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1792, 0, 0, 0, 0, 2, 4085, 2313] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[456, 0, 0, 28, 0, 1, 4053, 3654] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[189, 0, 0, 138, 0, 0, 3957, 3908] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[529, 0, 0, 1, 0, 0, 4095, 3567] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[892, 0, 0, 0, 1, 2, 4096, 3201] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1049, 0, 0, 2, 0, 0, 4093, 3048] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1792, 187, 0, 1027, 0, 1027, 0, 693, 0, 1002, 2, 707, 1029, 0, 726, 0] rerouted_tokens_per_ep_rank=[1979, 1027, 1027, 693, 1002, 709, 1029, 726] avg=1024.0000 max=1979 max_rank=0 min=693 min_rank=3 imbalance_ratio=1.932617 max_load_reduction=6213 max_load_reduction_pct=0.758423 moved_to_echo=3466 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[185, 317, 0, 1029, 0, 1029, 0, 1172, 0, 1005, 1, 1195, 1032, 0, 1227, 0] rerouted_tokens_per_ep_rank=[502, 1029, 1029, 1172, 1005, 1196, 1032, 1227] avg=1024.0000 max=1227 max_rank=7 min=502 min_rank=0 imbalance_ratio=1.198242 max_load_reduction=6965 max_load_reduction_pct=0.850220 moved_to_echo=4460 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[456, 296, 0, 1019, 0, 1019, 28, 1095, 0, 994, 1, 1117, 1021, 0, 1146, 0] rerouted_tokens_per_ep_rank=[752, 1019, 1019, 1123, 994, 1118, 1021, 1146] avg=1024.0000 max=1146 max_rank=7 min=752 min_rank=0 imbalance_ratio=1.119141 max_load_reduction=7046 max_load_reduction_pct=0.860107 moved_to_echo=4279 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[529, 289, 0, 1029, 0, 1029, 1, 1068, 0, 1005, 0, 1090, 1032, 0, 1120, 0] rerouted_tokens_per_ep_rank=[818, 1029, 1029, 1069, 1005, 1090, 1032, 1120] avg=1024.0000 max=1120 max_rank=7 min=818 min_rank=0 imbalance_ratio=1.093750 max_load_reduction=7072 max_load_reduction_pct=0.863281 moved_to_echo=4247 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[927, 261, 0, 1034, 0, 1034, 0, 952, 0, 1008, 0, 973, 1020, 0, 983, 0] rerouted_tokens_per_ep_rank=[1188, 1034, 1034, 952, 1008, 973, 1020, 983] avg=1024.0000 max=1188 max_rank=0 min=952 min_rank=3 imbalance_ratio=1.160156 max_load_reduction=7004 max_load_reduction_pct=0.854980 moved_to_echo=3984 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[189, 317, 0, 995, 0, 995, 138, 1171, 0, 971, 0, 1194, 996, 0, 1226, 0] rerouted_tokens_per_ep_rank=[506, 995, 995, 1309, 971, 1194, 996, 1226] avg=1024.0000 max=1309 max_rank=3 min=506 min_rank=0 imbalance_ratio=1.278320 max_load_reduction=6883 max_load_reduction_pct=0.840210 moved_to_echo=4387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[892, 259, 0, 1030, 0, 1030, 0, 959, 1, 1005, 2, 978, 1031, 0, 1005, 0] rerouted_tokens_per_ep_rank=[1151, 1030, 1030, 959, 1006, 980, 1031, 1005] avg=1024.0000 max=1151 max_rank=0 min=959 min_rank=3 imbalance_ratio=1.124023 max_load_reduction=7041 max_load_reduction_pct=0.859497 moved_to_echo=4022 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1049, 247, 0, 1029, 0, 1029, 2, 913, 0, 1004, 0, 932, 1031, 0, 956, 0] rerouted_tokens_per_ep_rank=[1296, 1029, 1029, 915, 1004, 932, 1031, 956] avg=1024.0000 max=1296 max_rank=0 min=915 min_rank=3 imbalance_ratio=1.265625 max_load_reduction=6896 max_load_reduction_pct=0.841797 moved_to_echo=3923 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 7995, 8192, 8192, 8389] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 7995, 8192, 8192, 8389] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 7995, 8192, 8192, 8389] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 7995, 8192, 8192, 8389] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 7995, 8192, 8192, 8389] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7995 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 7995, 8192, 8192, 8389] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 7995, 8192, 8192, 8389] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8389 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 7995, 8192, 8192, 8389] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7995 actual_valid_recv=8192 observed_loss=-197 tokens_per_expert=[128, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8389 actual_valid_recv=8448 observed_loss=-59 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6144, 2176] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[2563, 0, 0, 0, 0, 1533, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2767, 0, 0, 0, 0, 1329, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[3238, 0, 0, 0, 0, 858, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[2938, 0, 0, 0, 0, 1158, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1939, 0, 0, 0, 0, 2157, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[2393, 0, 0, 0, 0, 1703, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[3737, 0, 0, 0, 0, 359, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[2722, 0, 0, 0, 0, 1374, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[943, 0, 0, 1024, 0, 1024, 0, 679, 0, 1024, 1533, 0, 0, 941, 1024, 0] rerouted_tokens_per_ep_rank=[943, 1024, 1024, 679, 1024, 1533, 941, 1024] avg=1024.0000 max=1533 max_rank=5 min=679 min_rank=3 imbalance_ratio=1.497070 max_load_reduction=6659 max_load_reduction_pct=0.812866 moved_to_echo=4522 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1018, 0, 0, 1024, 0, 1024, 0, 733, 0, 1024, 1329, 0, 0, 1016, 1024, 0] rerouted_tokens_per_ep_rank=[1018, 1024, 1024, 733, 1024, 1329, 1016, 1024] avg=1024.0000 max=1329 max_rank=5 min=733 min_rank=3 imbalance_ratio=1.297852 max_load_reduction=6863 max_load_reduction_pct=0.837769 moved_to_echo=4393 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1191, 0, 0, 1024, 0, 1024, 0, 858, 0, 1024, 858, 0, 0, 1189, 1024, 0] rerouted_tokens_per_ep_rank=[1191, 1024, 1024, 858, 1024, 858, 1189, 1024] avg=1024.0000 max=1191 max_rank=0 min=858 min_rank=3 imbalance_ratio=1.163086 max_load_reduction=7001 max_load_reduction_pct=0.854614 moved_to_echo=4095 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1080, 0, 0, 1024, 0, 1024, 0, 779, 0, 1024, 1158, 0, 0, 1079, 1024, 0] rerouted_tokens_per_ep_rank=[1080, 1024, 1024, 779, 1024, 1158, 1079, 1024] avg=1024.0000 max=1158 max_rank=5 min=779 min_rank=3 imbalance_ratio=1.130859 max_load_reduction=7034 max_load_reduction_pct=0.858643 moved_to_echo=4285 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1366, 0, 0, 1024, 0, 1024, 0, 995, 0, 1024, 359, 0, 0, 1376, 1024, 0] rerouted_tokens_per_ep_rank=[1366, 1024, 1024, 995, 1024, 359, 1376, 1024] avg=1024.0000 max=1376 max_rank=6 min=359 min_rank=5 imbalance_ratio=1.343750 max_load_reduction=6816 max_load_reduction_pct=0.832031 moved_to_echo=3783 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[713, 0, 0, 1024, 0, 1024, 0, 514, 0, 1024, 2157, 0, 0, 712, 1024, 0] rerouted_tokens_per_ep_rank=[713, 1024, 1024, 514, 1024, 2157, 712, 1024] avg=1024.0000 max=2157 max_rank=5 min=514 min_rank=3 imbalance_ratio=2.106445 max_load_reduction=6035 max_load_reduction_pct=0.736694 moved_to_echo=4917 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[880, 0, 0, 1024, 0, 1024, 0, 634, 0, 1024, 1703, 0, 0, 879, 1024, 0] rerouted_tokens_per_ep_rank=[880, 1024, 1024, 634, 1024, 1703, 879, 1024] avg=1024.0000 max=1703 max_rank=5 min=634 min_rank=3 imbalance_ratio=1.663086 max_load_reduction=6489 max_load_reduction_pct=0.792114 moved_to_echo=4630 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1001, 0, 0, 1024, 0, 1024, 0, 721, 0, 1024, 1374, 0, 0, 1000, 1024, 0] rerouted_tokens_per_ep_rank=[1001, 1024, 1024, 721, 1024, 1374, 1000, 1024] avg=1024.0000 max=1374 max_rank=5 min=721 min_rank=3 imbalance_ratio=1.341797 max_load_reduction=6818 max_load_reduction_pct=0.832275 moved_to_echo=4422 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5913 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5913, 8192, 10471, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5913, 8192, 10471, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5913, 8192, 10471, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10471 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5913, 8192, 10471, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5913, 8192, 10471, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5913, 8192, 10471, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5913, 8192, 10471, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5913, 8192, 10471, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=5913 actual_valid_recv=6016 observed_loss=-103 tokens_per_expert=[0, 6016] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=10471 actual_valid_recv=10496 observed_loss=-25 tokens_per_expert=[10496, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=6016 slack_slots=10368 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=10496 slack_slots=5888 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4095, 4095, 0, 1, 1, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1026, 1024, 1023, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 1016, 0, 1011, 0, 0, 1025, 0, 1030, 0, 1030, 0, 1025, 0, 1025] rerouted_tokens_per_ep_rank=[1030, 1016, 1011, 1025, 1030, 1030, 1025, 1025] avg=1024.0000 max=1030 max_rank=0 min=1011 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4110 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1026, 1024, 1023, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1026, 1024, 1023, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1026, 1024, 1023, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1026, 1024, 1023, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1026, 0, 0, 1023, 1, 1023, 1, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1026, 1023, 1024, 1024, 1023, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1026, 1024, 1023, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1, 4, 0, 4091, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 19, 0, 4075, 4096, 0, 0, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 22, 0, 4074, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 286, 0, 3810, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[2, 11, 0, 4083, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1, 1, 0, 4094, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1037, 0, 995, 0, 1024, 1026, 0, 1024, 0, 0, 1024, 0, 1024, 0, 1038] rerouted_tokens_per_ep_rank=[1037, 995, 1024, 1026, 1024, 1024, 1024, 1038] avg=1024.0000 max=1038 max_rank=7 min=995 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4110 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 22, 986, 0, 1024, 1031, 0, 1024, 0, 0, 1024, 0, 1024, 0, 1029] rerouted_tokens_per_ep_rank=[1028, 1008, 1024, 1031, 1024, 1024, 1024, 1029] avg=1024.0000 max=1031 max_rank=3 min=1008 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1029, 19, 986, 0, 1024, 1031, 0, 1024, 0, 0, 1024, 0, 1024, 2, 1029] rerouted_tokens_per_ep_rank=[1029, 1005, 1024, 1031, 1024, 1024, 1024, 1031] avg=1024.0000 max=1031 max_rank=3 min=1005 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 1033, 4, 990, 0, 1024, 1035, 0, 1024, 0, 0, 1024, 0, 1024, 0, 1033] rerouted_tokens_per_ep_rank=[1034, 994, 1024, 1035, 1024, 1024, 1024, 1033] avg=1024.0000 max=1035 max_rank=3 min=994 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1034, 0, 991, 0, 1024, 1037, 0, 1024, 0, 0, 1024, 0, 1024, 0, 1034] rerouted_tokens_per_ep_rank=[1034, 991, 1024, 1037, 1024, 1024, 1024, 1034] avg=1024.0000 max=1037 max_rank=3 min=991 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=7155 max_load_reduction_pct=0.873413 moved_to_echo=4106 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 962, 286, 922, 0, 1024, 964, 0, 1024, 0, 0, 1024, 0, 1024, 0, 962] rerouted_tokens_per_ep_rank=[962, 1208, 1024, 964, 1024, 1024, 1024, 962] avg=1024.0000 max=1208 max_rank=1 min=962 min_rank=0 imbalance_ratio=1.179688 max_load_reduction=6984 max_load_reduction_pct=0.852539 moved_to_echo=4034 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 1034, 1, 991, 0, 1024, 1035, 0, 1024, 0, 0, 1024, 0, 1024, 0, 1034] rerouted_tokens_per_ep_rank=[1035, 992, 1024, 1035, 1024, 1024, 1024, 1034] avg=1024.0000 max=1035 max_rank=0 min=992 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4106 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 1031, 11, 988, 0, 1024, 1033, 0, 1024, 0, 0, 1024, 0, 1024, 0, 1031] rerouted_tokens_per_ep_rank=[1033, 999, 1024, 1033, 1024, 1024, 1024, 1031] avg=1024.0000 max=1033 max_rank=0 min=999 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1024, 1024, 1024] avg=1024.0000 max=1025 max_rank=4 min=1023 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1024, 1024, 1024] avg=1024.0000 max=1025 max_rank=4 min=1023 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1024, 1024, 1024] avg=1024.0000 max=1025 max_rank=4 min=1023 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1030, 0, 1025, 1016, 0, 0, 1024, 0, 1025, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1030, 1025, 1016, 1024, 1025, 1024] avg=1024.0000 max=1030 max_rank=2 min=1016 min_rank=4 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1024, 1024, 1024] avg=1024.0000 max=1025 max_rank=4 min=1023 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1024, 1024, 1024] avg=1024.0000 max=1025 max_rank=4 min=1023 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1, 1023, 0, 1023, 1026, 0, 0, 1024, 0, 1023, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1023, 1026, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1024, 1024, 1024] avg=1024.0000 max=1025 max_rank=4 min=1023 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3703, 0, 78, 0, 0, 315] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3530, 0, 62, 0, 0, 504] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 2886, 0, 0, 0, 0, 1210] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 2897, 0, 192, 0, 0, 1007] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3696, 0, 44, 0, 0, 356] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 2833, 0, 53, 0, 0, 1210] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3188, 0, 127, 0, 0, 781] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3648, 0, 119, 0, 0, 329] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 1098, 0, 0, 1096, 62, 1005, 0, 1024, 0, 1024, 504, 331] rerouted_tokens_per_ep_rank=[1024, 1024, 1098, 1096, 1067, 1024, 1024, 835] avg=1024.0000 max=1098 max_rank=2 min=835 min_rank=7 imbalance_ratio=1.072266 max_load_reduction=7094 max_load_reduction_pct=0.865967 moved_to_echo=3950 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 1151, 0, 0, 1149, 78, 1055, 0, 1024, 0, 1024, 315, 348] rerouted_tokens_per_ep_rank=[1024, 1024, 1151, 1149, 1133, 1024, 1024, 663] avg=1024.0000 max=1151 max_rank=2 min=663 min_rank=7 imbalance_ratio=1.124023 max_load_reduction=7041 max_load_reduction_pct=0.859497 moved_to_echo=3844 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 901, 0, 0, 899, 192, 825, 0, 1024, 0, 1024, 1007, 272] rerouted_tokens_per_ep_rank=[1024, 1024, 901, 899, 1017, 1024, 1024, 1279] avg=1024.0000 max=1279 max_rank=7 min=899 min_rank=3 imbalance_ratio=1.249023 max_load_reduction=6913 max_load_reduction_pct=0.843872 moved_to_echo=4344 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 897, 0, 0, 896, 0, 822, 0, 1024, 0, 1024, 1210, 271] rerouted_tokens_per_ep_rank=[1024, 1024, 897, 896, 822, 1024, 1024, 1481] avg=1024.0000 max=1481 max_rank=7 min=822 min_rank=4 imbalance_ratio=1.446289 max_load_reduction=6711 max_load_reduction_pct=0.819214 moved_to_echo=4351 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 1137, 0, 0, 1152, 44, 1056, 0, 1024, 0, 1024, 356, 351] rerouted_tokens_per_ep_rank=[1024, 1024, 1137, 1152, 1100, 1024, 1024, 707] avg=1024.0000 max=1152 max_rank=3 min=707 min_rank=7 imbalance_ratio=1.125000 max_load_reduction=7040 max_load_reduction_pct=0.859375 moved_to_echo=3855 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 881, 0, 0, 879, 53, 807, 0, 1024, 0, 1024, 1210, 266] rerouted_tokens_per_ep_rank=[1024, 1024, 881, 879, 860, 1024, 1024, 1476] avg=1024.0000 max=1476 max_rank=7 min=860 min_rank=4 imbalance_ratio=1.441406 max_load_reduction=6716 max_load_reduction_pct=0.819824 moved_to_echo=4384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 992, 0, 0, 989, 127, 908, 0, 1024, 0, 1024, 781, 299] rerouted_tokens_per_ep_rank=[1024, 1024, 992, 989, 1035, 1024, 1024, 1080] avg=1024.0000 max=1080 max_rank=7 min=989 min_rank=3 imbalance_ratio=1.054688 max_load_reduction=7112 max_load_reduction_pct=0.868164 moved_to_echo=4163 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 1135, 0, 0, 1132, 119, 1039, 0, 1024, 0, 1024, 329, 342] rerouted_tokens_per_ep_rank=[1024, 1024, 1135, 1132, 1158, 1024, 1024, 671] avg=1024.0000 max=1158 max_rank=4 min=671 min_rank=7 imbalance_ratio=1.130859 max_load_reduction=7034 max_load_reduction_pct=0.858643 moved_to_echo=3877 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5760, 2560] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2458, 1638] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 3735, 358] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3050, 1046] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 3048, 1047] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[10, 0, 4096, 0, 0, 0, 3910, 176] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 4096, 0, 0, 0, 3256, 839] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2800, 1296] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3125, 971] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 792, 0, 792, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 795, 0, 1638, 79] rerouted_tokens_per_ep_rank=[792, 792, 1025, 1023, 1024, 1024, 795, 1717] avg=1024.0000 max=1717 max_rank=7 min=792 min_rank=0 imbalance_ratio=1.676758 max_load_reduction=6475 max_load_reduction_pct=0.790405 moved_to_echo=4560 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 983, 0, 983, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 986, 0, 1046, 98] rerouted_tokens_per_ep_rank=[983, 983, 1025, 1023, 1024, 1024, 986, 1144] avg=1024.0000 max=1144 max_rank=7 min=983 min_rank=0 imbalance_ratio=1.117188 max_load_reduction=7048 max_load_reduction_pct=0.860352 moved_to_echo=4178 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1203, 3, 1204, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 1208, 0, 358, 120] rerouted_tokens_per_ep_rank=[1203, 1207, 1025, 1023, 1024, 1024, 1208, 478] avg=1024.0000 max=1208 max_rank=6 min=478 min_rank=7 imbalance_ratio=1.179688 max_load_reduction=6984 max_load_reduction_pct=0.852539 moved_to_echo=3734 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 902, 0, 903, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 905, 0, 1296, 90] rerouted_tokens_per_ep_rank=[902, 903, 1025, 1023, 1024, 1024, 905, 1386] avg=1024.0000 max=1386 max_rank=7 min=902 min_rank=0 imbalance_ratio=1.353516 max_load_reduction=6806 max_load_reduction_pct=0.830811 moved_to_echo=4339 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1052, 0, 1054, 1018, 0, 0, 1030, 0, 1024, 0, 1024, 1041, 0, 839, 109] rerouted_tokens_per_ep_rank=[1053, 1054, 1018, 1030, 1024, 1024, 1041, 948] avg=1024.0000 max=1054 max_rank=1 min=948 min_rank=7 imbalance_ratio=1.029297 max_load_reduction=7138 max_load_reduction_pct=0.871338 moved_to_echo=4037 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[10, 1260, 0, 1261, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 1263, 0, 176, 126] rerouted_tokens_per_ep_rank=[1270, 1261, 1025, 1023, 1024, 1024, 1263, 302] avg=1024.0000 max=1270 max_rank=0 min=302 min_rank=7 imbalance_ratio=1.240234 max_load_reduction=6922 max_load_reduction_pct=0.844971 moved_to_echo=3613 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 982, 0, 983, 1025, 0, 1, 1023, 0, 1024, 0, 1024, 985, 0, 1047, 98] rerouted_tokens_per_ep_rank=[982, 983, 1025, 1024, 1024, 1024, 985, 1145] avg=1024.0000 max=1145 max_rank=7 min=982 min_rank=0 imbalance_ratio=1.118164 max_load_reduction=7047 max_load_reduction_pct=0.860229 moved_to_echo=4178 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1007, 0, 1008, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 1009, 0, 971, 101] rerouted_tokens_per_ep_rank=[1007, 1008, 1025, 1023, 1024, 1024, 1009, 1072] avg=1024.0000 max=1072 max_rank=7 min=1007 min_rank=0 imbalance_ratio=1.046875 max_load_reduction=7120 max_load_reduction_pct=0.869141 moved_to_echo=4129 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8191 actual_valid_recv=8320 observed_loss=-129 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7424, 896] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True + [2026-08-30 04:54:03] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 679.0 | throughput per GPU (TFLOP/s/GPU): 491.0 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.067946E+01 | loss scale: 1.0 | grad norm: 16.932 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[6, 0, 0, 2953, 34, 4092, 1105, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[5, 6, 2, 3119, 53, 4073, 933, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[10, 0, 0, 3037, 34, 4091, 1017, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[21, 0, 2, 2976, 28, 4083, 1079, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[51, 1, 0, 2987, 41, 4086, 1026, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[115, 1, 1, 3020, 33, 4079, 943, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[20, 0, 3, 3079, 28, 4057, 1005, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[7, 4, 1, 3189, 26, 4091, 871, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 992, 0, 1014, 0, 1025, 1033, 0, 34, 986, 1028, 0, 1017, 26, 3, 1024] rerouted_tokens_per_ep_rank=[1002, 1014, 1025, 1033, 1020, 1028, 1043, 1027] avg=1024.0000 max=1043 max_rank=6 min=1002 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=7149 max_load_reduction_pct=0.872681 moved_to_echo=4118 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[21, 972, 0, 1012, 2, 1023, 1012, 0, 28, 966, 1026, 0, 1079, 26, 3, 1022] rerouted_tokens_per_ep_rank=[993, 1012, 1025, 1012, 994, 1026, 1105, 1025] avg=1024.0000 max=1105 max_rank=6 min=993 min_rank=0 imbalance_ratio=1.079102 max_load_reduction=7087 max_load_reduction_pct=0.865112 moved_to_echo=4150 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[6, 964, 0, 1014, 0, 1025, 1005, 0, 34, 959, 1028, 0, 1105, 25, 2, 1025] rerouted_tokens_per_ep_rank=[970, 1014, 1025, 1005, 993, 1028, 1130, 1027] avg=1024.0000 max=1130 max_rank=6 min=970 min_rank=0 imbalance_ratio=1.103516 max_load_reduction=7062 max_load_reduction_pct=0.862061 moved_to_echo=4178 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 1022, 6, 1013, 2, 1023, 1049, 0, 53, 1017, 1012, 0, 933, 31, 1, 1025] rerouted_tokens_per_ep_rank=[1027, 1019, 1025, 1049, 1070, 1012, 964, 1026] avg=1024.0000 max=1070 max_rank=4 min=964 min_rank=6 imbalance_ratio=1.044922 max_load_reduction=7122 max_load_reduction_pct=0.869385 moved_to_echo=4072 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[51, 975, 1, 1013, 0, 1024, 1016, 0, 41, 970, 1026, 0, 1026, 26, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1014, 1024, 1016, 1011, 1026, 1052, 1023] avg=1024.0000 max=1052 max_rank=6 min=1011 min_rank=4 imbalance_ratio=1.027344 max_load_reduction=7140 max_load_reduction_pct=0.871582 moved_to_echo=4112 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[115, 986, 1, 1011, 1, 1022, 1027, 0, 33, 981, 1025, 0, 943, 26, 0, 1021] rerouted_tokens_per_ep_rank=[1101, 1012, 1023, 1027, 1014, 1025, 969, 1021] avg=1024.0000 max=1101 max_rank=0 min=969 min_rank=6 imbalance_ratio=1.075195 max_load_reduction=7091 max_load_reduction_pct=0.865601 moved_to_echo=4029 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[20, 1005, 0, 1006, 3, 1016, 1048, 0, 28, 1000, 1019, 0, 1005, 26, 0, 1016] rerouted_tokens_per_ep_rank=[1025, 1006, 1019, 1048, 1028, 1019, 1031, 1016] avg=1024.0000 max=1048 max_rank=3 min=1006 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=7144 max_load_reduction_pct=0.872070 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 1041, 4, 1014, 1, 1025, 1085, 0, 26, 1036, 1028, 0, 871, 27, 3, 1024] rerouted_tokens_per_ep_rank=[1048, 1018, 1026, 1085, 1062, 1028, 898, 1027] avg=1024.0000 max=1085 max_rank=3 min=898 min_rank=6 imbalance_ratio=1.059570 max_load_reduction=7107 max_load_reduction_pct=0.867554 moved_to_echo=4015 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8275 expected_overflow=0 exact_recv_per_rank=[8192, 8109, 8192, 8275, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8109, 8192, 8275, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8109, 8192, 8275, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8109, 8192, 8275, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8109 expected_overflow=0 exact_recv_per_rank=[8192, 8109, 8192, 8275, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8109, 8192, 8275, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8109, 8192, 8275, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8109, 8192, 8275, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8275 actual_valid_recv=8320 observed_loss=-45 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[8064, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8109 actual_valid_recv=8320 observed_loss=-211 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 0, 4090, 4088, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 4093, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 4094, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 4095, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 4094, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[3, 0, 3, 0, 0, 4090, 4094, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[3, 0, 6, 1, 0, 4091, 4089, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 4095, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1020, 11, 1020, 0, 1022, 0, 1022, 1026, 0, 1024, 0, 3, 1022] rerouted_tokens_per_ep_rank=[1022, 1020, 1031, 1022, 1022, 1026, 1024, 1025] avg=1024.0000 max=1031 max_rank=2 min=1020 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 1023, 0, 1022, 0, 1021, 0, 1024, 0, 1024, 1027, 0, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1022, 1021, 1024, 1024, 1027, 1026, 1023] avg=1024.0000 max=1027 max_rank=5 min=1021 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1022, 0, 1021, 0, 1024, 0, 1024, 1028, 0, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1022, 1021, 1024, 1024, 1028, 1026, 1023] avg=1024.0000 max=1028 max_rank=5 min=1021 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 1022, 0, 1021, 3, 1020, 0, 1023, 0, 1024, 1026, 0, 1026, 0, 2, 1022] rerouted_tokens_per_ep_rank=[1025, 1021, 1023, 1023, 1024, 1026, 1026, 1024] avg=1024.0000 max=1026 max_rank=5 min=1021 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 1022, 0, 1020, 6, 1020, 1, 1022, 0, 1022, 1027, 0, 1025, 0, 2, 1022] rerouted_tokens_per_ep_rank=[1025, 1020, 1026, 1023, 1022, 1027, 1025, 1024] avg=1024.0000 max=1027 max_rank=5 min=1020 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1022, 0, 1021, 1, 1024, 0, 1024, 1027, 0, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1022, 1021, 1025, 1024, 1027, 1026, 1023] avg=1024.0000 max=1027 max_rank=5 min=1021 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1021, 3, 1021, 0, 1023, 0, 1024, 1028, 0, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1021, 1024, 1023, 1024, 1028, 1026, 1023] avg=1024.0000 max=1028 max_rank=5 min=1021 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 3, 1026, 0, 1025, 0, 1028, 0, 1028, 1018, 0, 1013, 0, 1, 1026] rerouted_tokens_per_ep_rank=[1024, 1029, 1025, 1028, 1028, 1018, 1013, 1027] avg=1024.0000 max=1029 max_rank=1 min=1013 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4086 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8177, 8192, 8192, 8192, 8207, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8177, 8192, 8192, 8192, 8207, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8207 expected_overflow=0 exact_recv_per_rank=[8192, 8177, 8192, 8192, 8192, 8207, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8177, 8192, 8192, 8192, 8207, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8177 expected_overflow=0 exact_recv_per_rank=[8192, 8177, 8192, 8192, 8192, 8207, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8177, 8192, 8192, 8192, 8207, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8177, 8192, 8192, 8192, 8207, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8177, 8192, 8192, 8192, 8207, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8177 actual_valid_recv=8320 observed_loss=-143 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8207 actual_valid_recv=8320 observed_loss=-113 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3544, 1, 35, 4096, 491, 0, 25, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3660, 6, 21, 4095, 262, 1, 146, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3707, 86, 18, 4096, 239, 0, 46, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3751, 1, 31, 4095, 256, 4, 53, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3466, 4, 44, 4094, 576, 5, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3634, 18, 70, 4096, 259, 0, 115, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3345, 1, 47, 4096, 696, 0, 7, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[2916, 0, 59, 4095, 1119, 0, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1037, 0, 1, 1009, 35, 979, 1042, 0, 491, 543, 0, 1022, 25, 985, 0, 1023] rerouted_tokens_per_ep_rank=[1037, 1010, 1014, 1042, 1034, 1022, 1010, 1023] avg=1024.0000 max=1042 max_rank=3 min=1010 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=7150 max_load_reduction_pct=0.872803 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1072, 0, 6, 1009, 21, 1011, 1041, 0, 262, 560, 1, 1022, 146, 1017, 1, 1023] rerouted_tokens_per_ep_rank=[1072, 1015, 1032, 1041, 822, 1023, 1163, 1024] avg=1024.0000 max=1163 max_rank=6 min=822 min_rank=4 imbalance_ratio=1.135742 max_load_reduction=7029 max_load_reduction_pct=0.858032 moved_to_echo=4032 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1064, 0, 18, 1009, 70, 1004, 1042, 0, 259, 556, 0, 1022, 115, 1010, 0, 1023] rerouted_tokens_per_ep_rank=[1064, 1027, 1074, 1042, 815, 1022, 1125, 1023] avg=1024.0000 max=1125 max_rank=6 min=815 min_rank=4 imbalance_ratio=1.098633 max_load_reduction=7067 max_load_reduction_pct=0.862671 moved_to_echo=3985 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1098, 0, 1, 1009, 31, 1036, 1041, 0, 256, 574, 4, 1022, 53, 1043, 1, 1023] rerouted_tokens_per_ep_rank=[1098, 1010, 1067, 1041, 830, 1026, 1096, 1024] avg=1024.0000 max=1098 max_rank=0 min=830 min_rank=4 imbalance_ratio=1.072266 max_load_reduction=7094 max_load_reduction_pct=0.865967 moved_to_echo=3976 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1004, 0, 4, 1012, 44, 960, 1025, 0, 576, 535, 5, 1028, 3, 967, 0, 1029] rerouted_tokens_per_ep_rank=[1004, 1016, 1004, 1025, 1111, 1033, 970, 1029] avg=1024.0000 max=1111 max_rank=4 min=970 min_rank=6 imbalance_ratio=1.084961 max_load_reduction=7081 max_load_reduction_pct=0.864380 moved_to_echo=4143 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1084, 0, 86, 1009, 18, 1024, 1042, 0, 239, 568, 0, 1022, 46, 1031, 0, 1023] rerouted_tokens_per_ep_rank=[1084, 1095, 1042, 1042, 807, 1022, 1077, 1023] avg=1024.0000 max=1095 max_rank=1 min=807 min_rank=4 imbalance_ratio=1.069336 max_load_reduction=7097 max_load_reduction_pct=0.866333 moved_to_echo=3929 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[979, 0, 1, 1009, 47, 924, 1042, 0, 696, 512, 0, 1022, 7, 930, 0, 1023] rerouted_tokens_per_ep_rank=[979, 1010, 971, 1042, 1208, 1022, 937, 1023] avg=1024.0000 max=1208 max_rank=4 min=937 min_rank=6 imbalance_ratio=1.179688 max_load_reduction=6984 max_load_reduction_pct=0.852539 moved_to_echo=4190 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[854, 0, 0, 1009, 59, 805, 1041, 0, 1119, 446, 0, 1022, 3, 811, 0, 1023] rerouted_tokens_per_ep_rank=[854, 1009, 864, 1041, 1565, 1022, 814, 1023] avg=1024.0000 max=1565 max_rank=4 min=814 min_rank=6 imbalance_ratio=1.528320 max_load_reduction=6627 max_load_reduction_pct=0.808960 moved_to_echo=4424 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8068, 8316, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8068, 8316, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8068, 8316, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8068, 8316, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8316 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8068, 8316, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8068, 8316, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8068 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8068, 8316, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8068, 8316, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3968, 4352] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8316 actual_valid_recv=8320 observed_loss=-4 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8068 actual_valid_recv=8192 observed_loss=-124 tokens_per_expert=[384, 7808] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[192, 3879, 4093, 7, 2, 15, 1, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[515, 3557, 4096, 0, 0, 24, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[312, 3783, 4096, 0, 0, 1, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[289, 3807, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[797, 3253, 4095, 2, 0, 44, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[254, 3825, 4096, 0, 0, 17, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[359, 3727, 4094, 0, 0, 12, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[505, 3559, 4095, 0, 0, 33, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[312, 639, 1056, 0, 1027, 0, 0, 1053, 0, 1023, 1, 1035, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[951, 1056, 1027, 1053, 1023, 1036, 1023, 1023] avg=1024.0000 max=1056 max_rank=1 min=951 min_rank=0 imbalance_ratio=1.031250 max_load_reduction=7136 max_load_reduction_pct=0.871094 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[192, 660, 1069, 0, 1007, 0, 7, 1085, 2, 1029, 15, 1065, 1, 1029, 3, 1028] rerouted_tokens_per_ep_rank=[852, 1069, 1007, 1092, 1031, 1080, 1030, 1031] avg=1024.0000 max=1092 max_rank=3 min=852 min_rank=0 imbalance_ratio=1.066406 max_load_reduction=7100 max_load_reduction_pct=0.866699 moved_to_echo=4172 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[254, 646, 1068, 0, 1027, 0, 0, 1064, 0, 1023, 17, 1047, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[900, 1068, 1027, 1064, 1023, 1064, 1023, 1023] avg=1024.0000 max=1068 max_rank=1 min=900 min_rank=0 imbalance_ratio=1.042969 max_load_reduction=7124 max_load_reduction_pct=0.869629 moved_to_echo=4133 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 601, 993, 0, 1027, 0, 0, 990, 0, 1023, 24, 973, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1116, 993, 1027, 990, 1023, 997, 1023, 1023] avg=1024.0000 max=1116 max_rank=0 min=990 min_rank=3 imbalance_ratio=1.089844 max_load_reduction=7076 max_load_reduction_pct=0.863770 moved_to_echo=4066 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[289, 643, 1063, 0, 1027, 0, 0, 1059, 0, 1023, 0, 1042, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[932, 1063, 1027, 1059, 1023, 1042, 1023, 1023] avg=1024.0000 max=1063 max_rank=1 min=932 min_rank=0 imbalance_ratio=1.038086 max_load_reduction=7129 max_load_reduction_pct=0.870239 moved_to_echo=4111 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[359, 630, 1040, 0, 1025, 0, 0, 1037, 0, 1023, 12, 1020, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[989, 1040, 1025, 1037, 1023, 1032, 1023, 1023] avg=1024.0000 max=1040 max_rank=1 min=989 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[797, 549, 909, 0, 1026, 0, 2, 905, 0, 1023, 44, 890, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1346, 909, 1026, 907, 1023, 934, 1024, 1023] avg=1024.0000 max=1346 max_rank=0 min=907 min_rank=3 imbalance_ratio=1.314453 max_load_reduction=6846 max_load_reduction_pct=0.835693 moved_to_echo=4004 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[505, 601, 994, 0, 1026, 0, 0, 990, 0, 1023, 33, 974, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1106, 994, 1026, 990, 1023, 1007, 1023, 1023] avg=1024.0000 max=1106 max_rank=0 min=990 min_rank=3 imbalance_ratio=1.080078 max_load_reduction=7086 max_load_reduction_pct=0.864990 moved_to_echo=4076 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3328, 4992] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4087, 1, 4092, 4, 0, 0, 8, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4096, 1, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4095, 0, 4096, 0, 0, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4095, 1, 4095, 0, 1, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1023, 1025, 1023, 1023, 1024, 1023, 1024] avg=1024.0000 max=1027 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1023, 1025, 1023, 1023, 1024, 1023, 1024] avg=1024.0000 max=1027 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1023, 1025, 1023, 1023, 1024, 1023, 1024] avg=1024.0000 max=1027 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 1, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1027, 1024, 1026, 1023, 1023, 1023, 1023, 1023] avg=1024.0000 max=1027 max_rank=0 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1009, 0, 1, 1028, 1015, 0, 4, 1027, 0, 1026, 0, 1026, 8, 1023, 0, 1025] rerouted_tokens_per_ep_rank=[1009, 1029, 1015, 1031, 1026, 1026, 1031, 1025] avg=1024.0000 max=1031 max_rank=3 min=1009 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1023, 1, 1024] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1023, 1023, 1024, 1023, 1025] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1023, 1025, 1023, 1023, 1024, 1023, 1024] avg=1024.0000 max=1027 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1, 1023, 1026, 0, 0, 1023, 1, 1023, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1026, 1023, 1024, 1023, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8188 actual_valid_recv=8320 observed_loss=-132 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 4095, 4095, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4092, 4092, 2, 0, 3, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[2, 0, 4095, 4094, 0, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1025, 0, 1027, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1025, 1027, 1023, 1024, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1025, 0, 1027, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1025, 1027, 1023, 1024, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 1026, 0, 1026, 0, 0, 1023, 0, 1023, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1026, 1026, 1023, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1025, 0, 1027, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1025, 1027, 1023, 1024, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 0, 1026, 1015, 0, 1010, 0, 2, 1025, 0, 1026, 3, 1027, 3, 1027] rerouted_tokens_per_ep_rank=[1028, 1026, 1015, 1010, 1027, 1026, 1030, 1030] avg=1024.0000 max=1030 max_rank=6 min=1010 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4113 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1025, 0, 1027, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1025, 1027, 1023, 1024, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1025, 0, 1027, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1025, 1027, 1023, 1024, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 1023, 0, 1023, 1026, 0, 1025, 0, 0, 1023, 0, 1023, 0, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1026, 1025, 1023, 1023, 1023, 1024] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8188, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8188 actual_valid_recv=8320 observed_loss=-132 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1, 4091, 0, 4092, 0, 0, 4, 4] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1, 4095, 0, 4090, 6, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[5, 4096, 0, 4091, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4094, 0, 4095, 1, 0, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[7, 4096, 0, 4088, 1, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 4095, 1, 4095, 0, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 1025, 1016, 0, 0, 1026, 1014, 0, 0, 1027, 0, 1025, 4, 1024, 4, 1026] rerouted_tokens_per_ep_rank=[1026, 1016, 1026, 1014, 1027, 1025, 1028, 1030] avg=1024.0000 max=1030 max_rank=7 min=1014 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4110 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 1021, 1025, 0, 0, 1023, 1025, 0, 6, 1022, 0, 1024, 0, 1023, 0, 1022] rerouted_tokens_per_ep_rank=[1022, 1025, 1023, 1025, 1028, 1024, 1023, 1022] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1026, 0, 0, 1023, 1027, 0, 1, 1023, 0, 1023, 1, 1022, 1, 1023] rerouted_tokens_per_ep_rank=[1022, 1026, 1023, 1027, 1024, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=3 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[7, 1021, 1025, 0, 0, 1024, 1024, 0, 1, 1021, 0, 1024, 0, 1023, 0, 1022] rerouted_tokens_per_ep_rank=[1028, 1025, 1024, 1024, 1022, 1024, 1023, 1022] avg=1024.0000 max=1028 max_rank=0 min=1022 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 1021, 1025, 0, 0, 1024, 1026, 0, 0, 1022, 0, 1024, 0, 1023, 0, 1022] rerouted_tokens_per_ep_rank=[1026, 1025, 1024, 1026, 1022, 1024, 1023, 1022] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1025, 0, 1, 1023, 1027, 0, 0, 1023, 0, 1024, 0, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1022, 1025, 1024, 1027, 1023, 1024, 1023, 1024] avg=1024.0000 max=1027 max_rank=3 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8189 actual_valid_recv=8320 observed_loss=-131 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 4096, 3671, 0, 0, 424] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3531, 1, 0, 564] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 3, 4091, 3308, 0, 5, 784] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 4096, 3192, 5, 0, 898] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 7, 0, 4096, 3280, 0, 0, 809] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3206, 3, 0, 887] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 4095, 3175, 7, 1, 913] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 1, 4095, 3437, 0, 1, 658] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1, 1120, 0, 1023, 1026, 0, 1124, 0, 0, 1119, 0, 1023, 424, 308] rerouted_tokens_per_ep_rank=[1024, 1121, 1023, 1026, 1124, 1119, 1023, 732] avg=1024.0000 max=1124 max_rank=4 min=732 min_rank=7 imbalance_ratio=1.097656 max_load_reduction=7068 max_load_reduction_pct=0.862793 moved_to_echo=3998 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 1, 1013, 3, 1027, 1014, 0, 1000, 0, 0, 1013, 5, 1024, 784, 282] rerouted_tokens_per_ep_rank=[1026, 1014, 1030, 1014, 1000, 1013, 1029, 1066] avg=1024.0000 max=1066 max_rank=7 min=1000 min_rank=4 imbalance_ratio=1.041016 max_load_reduction=7126 max_load_reduction_pct=0.869873 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 978, 0, 1023, 1026, 0, 981, 0, 3, 978, 0, 1023, 887, 269] rerouted_tokens_per_ep_rank=[1024, 978, 1023, 1026, 981, 981, 1023, 1156] avg=1024.0000 max=1156 max_rank=7 min=978 min_rank=1 imbalance_ratio=1.128906 max_load_reduction=7036 max_load_reduction_pct=0.858887 moved_to_echo=4141 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1077, 0, 1023, 1026, 0, 1080, 0, 1, 1077, 0, 1023, 564, 297] rerouted_tokens_per_ep_rank=[1024, 1077, 1023, 1026, 1080, 1078, 1023, 861] avg=1024.0000 max=1080 max_rank=4 min=861 min_rank=7 imbalance_ratio=1.054688 max_load_reduction=7112 max_load_reduction_pct=0.868164 moved_to_echo=4042 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1, 973, 0, 1023, 1026, 0, 978, 0, 5, 973, 0, 1023, 898, 268] rerouted_tokens_per_ep_rank=[1024, 974, 1023, 1026, 978, 978, 1023, 1166] avg=1024.0000 max=1166 max_rank=7 min=974 min_rank=1 imbalance_ratio=1.138672 max_load_reduction=7026 max_load_reduction_pct=0.857666 moved_to_echo=4145 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 968, 0, 1023, 1026, 0, 972, 0, 7, 968, 1, 1023, 913, 267] rerouted_tokens_per_ep_rank=[1023, 969, 1023, 1026, 972, 975, 1024, 1180] avg=1024.0000 max=1180 max_rank=7 min=969 min_rank=1 imbalance_ratio=1.152344 max_load_reduction=7012 max_load_reduction_pct=0.855957 moved_to_echo=4151 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 7, 1000, 0, 1023, 1026, 0, 1005, 0, 0, 1000, 0, 1023, 809, 275] rerouted_tokens_per_ep_rank=[1024, 1007, 1023, 1026, 1005, 1000, 1023, 1084] avg=1024.0000 max=1084 max_rank=7 min=1000 min_rank=5 imbalance_ratio=1.058594 max_load_reduction=7108 max_load_reduction_pct=0.867676 moved_to_echo=4112 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1048, 1, 1023, 1026, 0, 1052, 0, 0, 1048, 1, 1023, 658, 289] rerouted_tokens_per_ep_rank=[1023, 1048, 1024, 1026, 1052, 1048, 1024, 947] avg=1024.0000 max=1052 max_rank=4 min=947 min_rank=7 imbalance_ratio=1.027344 max_load_reduction=7140 max_load_reduction_pct=0.871582 moved_to_echo=4071 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8188 actual_valid_recv=8320 observed_loss=-132 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6016, 2304] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4091, 0, 1, 2, 5, 4032, 59, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4093, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 4076, 19, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4090, 6, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 4095, 5, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 1, 4075, 20, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1027, 0, 1026, 1030, 0, 0, 1013, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1027, 1026, 1030, 1013, 1024] avg=1024.0000 max=1030 max_rank=5 min=1013 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1026, 0, 1025, 1030, 0, 3, 1012, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1026, 1025, 1030, 1015, 1024] avg=1024.0000 max=1030 max_rank=5 min=1015 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 0, 1025, 1, 1025, 2, 1015, 5, 1015, 1001, 0, 59, 1001, 2, 1025] rerouted_tokens_per_ep_rank=[1016, 1025, 1026, 1017, 1020, 1001, 1060, 1027] avg=1024.0000 max=1060 max_rank=6 min=1001 min_rank=5 imbalance_ratio=1.035156 max_load_reduction=7132 max_load_reduction_pct=0.870605 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 1, 1022, 0, 1022, 1, 1021, 1025, 0, 19, 1008, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1022, 1022, 1025, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1027, 0, 1026, 1030, 0, 0, 1013, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1027, 1026, 1030, 1013, 1024] avg=1024.0000 max=1030 max_rank=5 min=1013 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1023, 0, 1022, 0, 1026, 0, 1026, 1030, 0, 5, 1013, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1022, 1026, 1026, 1030, 1018, 1023] avg=1024.0000 max=1030 max_rank=5 min=1018 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1025, 0, 1025, 1028, 0, 6, 1012, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1025, 1025, 1028, 1018, 1024] avg=1024.0000 max=1028 max_rank=5 min=1018 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1022, 1, 1021, 1, 1021, 1025, 0, 20, 1008, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1022, 1022, 1022, 1025, 1028, 1023] avg=1024.0000 max=1028 max_rank=6 min=1022 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8185 actual_valid_recv=8320 observed_loss=-135 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1, 4095, 0, 0, 0, 4095, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[3, 4093, 1, 0, 0, 4091, 3, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1, 4095, 0, 0, 1, 4095, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 1027, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1024, 1024, 1027, 1023, 1023] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 1027, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1024, 1024, 1027, 1023, 1023] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 1027, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1024, 1024, 1027, 1023, 1023] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 0, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1023, 1023, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 1026, 1015, 0, 1, 1027, 0, 1026, 0, 1025, 1008, 0, 3, 1028, 1, 1029] rerouted_tokens_per_ep_rank=[1029, 1015, 1028, 1026, 1025, 1008, 1031, 1030] avg=1024.0000 max=1031 max_rank=6 min=1008 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 1027, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1024, 1024, 1027, 1023, 1023] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 1027, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1024, 1024, 1027, 1023, 1023] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 0, 1023, 0, 1023, 1, 1023, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1023, 1024, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8189 actual_valid_recv=8320 observed_loss=-131 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[5, 0, 4, 4090, 0, 2, 0, 4091] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 4093, 1, 1, 1, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 0, 4, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 1, 4095, 0, 0, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1027, 1024, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1027, 1024, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1023, 1, 1022, 1026, 0, 1, 1023, 1, 1023, 1, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1022, 1023, 1023, 1026, 1024, 1024, 1024, 1026] avg=1024.0000 max=1026 max_rank=3 min=1022 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[5, 1027, 0, 1026, 4, 1027, 1009, 0, 0, 1025, 2, 1025, 0, 1027, 1015, 0] rerouted_tokens_per_ep_rank=[1032, 1026, 1031, 1009, 1025, 1027, 1027, 1015] avg=1024.0000 max=1032 max_rank=0 min=1009 min_rank=3 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1027, 1024, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1024, 0, 1022, 1026, 0, 0, 1024, 0, 1023, 4, 1022, 1025, 0] rerouted_tokens_per_ep_rank=[1022, 1024, 1022, 1026, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=3 min=1022 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1027, 1024, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 1, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1026, 1023, 1023, 1023, 1026] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8195, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8189 actual_valid_recv=8320 observed_loss=-131 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[5, 0, 5, 4085, 0, 1, 5, 4091] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 1, 4094, 0, 0, 1, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 0, 0, 0, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[5, 1024, 0, 1025, 5, 1024, 1012, 0, 0, 1025, 1, 1025, 5, 1025, 1016, 0] rerouted_tokens_per_ep_rank=[1029, 1025, 1029, 1012, 1025, 1026, 1030, 1016] avg=1024.0000 max=1030 max_rank=6 min=1012 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1027, 0, 0, 1024, 0, 1024, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1027, 1024, 1024, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 1, 1023, 1025, 0, 0, 1024, 0, 1023, 1, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1024, 1023, 1024, 1025] avg=1024.0000 max=1025 max_rank=3 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1027, 0, 0, 1024, 0, 1024, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1027, 1024, 1024, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1027, 0, 0, 1024, 0, 1024, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1027, 1024, 1024, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1027, 0, 0, 1024, 0, 1024, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1027, 1024, 1024, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1027, 0, 0, 1024, 0, 1024, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1027, 1024, 1024, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1, 1022, 1, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1026, 1023, 1023, 1023, 1026] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8198 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8186 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8186 actual_valid_recv=8320 observed_loss=-134 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8198 actual_valid_recv=8320 observed_loss=-122 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 175, 4077, 0, 3940] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[5, 6, 3, 1, 326, 4079, 0, 3772] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 179, 4035, 0, 3978] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 194, 4056, 0, 3942] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1, 1, 2, 0, 162, 4085, 0, 3941] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 145, 4089, 0, 3958] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 325, 4058, 0, 3809] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 430, 4052, 0, 3708] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1048, 0, 1048, 0, 986, 0, 1015, 179, 801, 1018, 0, 0, 1016, 1081, 0] rerouted_tokens_per_ep_rank=[1048, 1048, 986, 1015, 980, 1018, 1016, 1081] avg=1024.0000 max=1081 max_rank=7 min=980 min_rank=4 imbalance_ratio=1.055664 max_load_reduction=7111 max_load_reduction_pct=0.868042 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1038, 0, 1038, 0, 997, 0, 1026, 175, 793, 1028, 0, 0, 1026, 1071, 0] rerouted_tokens_per_ep_rank=[1038, 1038, 997, 1026, 968, 1028, 1026, 1071] avg=1024.0000 max=1071 max_rank=7 min=968 min_rank=4 imbalance_ratio=1.045898 max_load_reduction=7121 max_load_reduction_pct=0.869263 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[5, 998, 6, 997, 3, 1001, 1, 1031, 326, 763, 1016, 0, 0, 1031, 1014, 0] rerouted_tokens_per_ep_rank=[1003, 1003, 1004, 1032, 1089, 1016, 1031, 1014] avg=1024.0000 max=1089 max_rank=4 min=1003 min_rank=0 imbalance_ratio=1.063477 max_load_reduction=7103 max_load_reduction_pct=0.867065 moved_to_echo=4150 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1039, 0, 1039, 0, 991, 0, 1021, 194, 794, 1023, 0, 0, 1021, 1070, 0] rerouted_tokens_per_ep_rank=[1039, 1039, 991, 1021, 988, 1023, 1021, 1070] avg=1024.0000 max=1070 max_rank=7 min=988 min_rank=4 imbalance_ratio=1.044922 max_load_reduction=7122 max_load_reduction_pct=0.869385 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 1038, 1, 1038, 2, 999, 0, 1028, 162, 794, 1030, 0, 0, 1028, 1071, 0] rerouted_tokens_per_ep_rank=[1039, 1039, 1001, 1028, 956, 1030, 1028, 1071] avg=1024.0000 max=1071 max_rank=7 min=956 min_rank=4 imbalance_ratio=1.045898 max_load_reduction=7121 max_load_reduction_pct=0.869263 moved_to_echo=4085 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1043, 0, 1043, 0, 1000, 0, 1029, 145, 797, 1031, 0, 0, 1029, 1075, 0] rerouted_tokens_per_ep_rank=[1043, 1043, 1000, 1029, 942, 1031, 1029, 1075] avg=1024.0000 max=1075 max_rank=7 min=942 min_rank=4 imbalance_ratio=1.049805 max_load_reduction=7117 max_load_reduction_pct=0.868774 moved_to_echo=4077 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1004, 0, 1004, 0, 992, 0, 1021, 325, 767, 1024, 0, 0, 1021, 1034, 0] rerouted_tokens_per_ep_rank=[1004, 1004, 992, 1021, 1092, 1024, 1021, 1034] avg=1024.0000 max=1092 max_rank=4 min=992 min_rank=2 imbalance_ratio=1.066406 max_load_reduction=7100 max_load_reduction_pct=0.866699 moved_to_echo=4171 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 977, 1, 977, 0, 990, 0, 1020, 430, 747, 1022, 0, 0, 1020, 1007, 0] rerouted_tokens_per_ep_rank=[978, 978, 990, 1020, 1177, 1022, 1020, 1007] avg=1024.0000 max=1177 max_rank=4 min=978 min_rank=0 imbalance_ratio=1.149414 max_load_reduction=7015 max_load_reduction_pct=0.856323 moved_to_echo=4226 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7961, 8192, 8192, 8192, 8192, 8423] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7961 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7961, 8192, 8192, 8192, 8192, 8423] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7961, 8192, 8192, 8192, 8192, 8423] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7961, 8192, 8192, 8192, 8192, 8423] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7961, 8192, 8192, 8192, 8192, 8423] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8423 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7961, 8192, 8192, 8192, 8192, 8423] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7961, 8192, 8192, 8192, 8192, 8423] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7961, 8192, 8192, 8192, 8192, 8423] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2048, 6272] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8423 actual_valid_recv=8448 observed_loss=-25 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=7961 actual_valid_recv=8192 observed_loss=-231 tokens_per_expert=[128, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4095, 1, 0, 1, 4091, 4, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4091, 2, 2, 6, 4091, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 1, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4095, 1, 0, 1, 4084, 11, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4078, 18, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1, 1023, 0, 1023, 1, 1023, 1026, 0, 4, 1019, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1026, 1023, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1024, 1027, 0, 0, 1021, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1024, 1027, 1021, 1024, 1024] avg=1024.0000 max=1027 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1024, 1027, 0, 1, 1020, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1024, 1027, 1021, 1024, 1024] avg=1024.0000 max=1027 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1015, 0, 2, 1028, 2, 1024, 6, 1025, 1016, 0, 0, 1022, 0, 1026, 0, 1026] rerouted_tokens_per_ep_rank=[1015, 1030, 1026, 1031, 1016, 1022, 1026, 1026] avg=1024.0000 max=1031 max_rank=3 min=1015 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1024, 1027, 0, 0, 1021, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1024, 1027, 1021, 1024, 1024] avg=1024.0000 max=1027 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1020, 0, 1023, 0, 1019, 1023, 0, 18, 1016, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1020, 1023, 1019, 1023, 1034, 1024, 1024] avg=1024.0000 max=1034 max_rank=5 min=1019 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1024, 1027, 0, 0, 1021, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1024, 1027, 1021, 1024, 1024] avg=1024.0000 max=1027 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1, 1021, 0, 1023, 1, 1021, 1024, 0, 11, 1018, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1022, 1023, 1022, 1024, 1029, 1023, 1023] avg=1024.0000 max=1029 max_rank=5 min=1022 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8197, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8197, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8197, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8197, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8187 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8197, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8197 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8197, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8197, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8197, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8187 actual_valid_recv=8320 observed_loss=-133 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8197 actual_valid_recv=8320 observed_loss=-123 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4091, 1, 4095, 1, 0, 4] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4088, 5, 4091, 6, 0, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 4096, 0, 4, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4092, 1, 4095, 1, 0, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1025, 0, 1, 1022, 1026, 0, 1, 1022, 0, 1023, 4, 1022] rerouted_tokens_per_ep_rank=[1023, 1023, 1025, 1023, 1026, 1023, 1023, 1026] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1027, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1027, 1023, 1025, 1023, 1023, 1023] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1027, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1027, 1023, 1025, 1023, 1023, 1023] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1027, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1027, 1023, 1025, 1023, 1023, 1023] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 1026, 1010, 0, 5, 1027, 1015, 0, 6, 1026, 0, 1024, 2, 1025] rerouted_tokens_per_ep_rank=[1026, 1026, 1010, 1032, 1015, 1032, 1024, 1027] avg=1024.0000 max=1032 max_rank=3 min=1010 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1026, 0, 0, 1022, 1025, 0, 0, 1022, 4, 1023, 0, 1022] rerouted_tokens_per_ep_rank=[1024, 1024, 1026, 1022, 1025, 1022, 1027, 1022] avg=1024.0000 max=1027 max_rank=6 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1027, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1027, 1023, 1025, 1023, 1023, 1023] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1026, 0, 1, 1022, 1026, 0, 1, 1022, 0, 1023, 3, 1022] rerouted_tokens_per_ep_rank=[1023, 1023, 1026, 1023, 1026, 1023, 1023, 1025] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8195, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8195, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8195, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8195, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8195, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8195, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8195, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8195, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8189 actual_valid_recv=8320 observed_loss=-131 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4071, 25, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4090, 0, 0, 2, 4, 6, 0, 4090] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4073, 22, 0, 0, 0, 0, 1, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4068, 27, 0, 0, 1, 1, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 1, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4051, 45, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4067, 28, 0, 0, 1, 1, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1031, 0, 0, 1010, 0, 1024, 0, 1023, 0, 1028, 0, 1027, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1031, 1010, 1024, 1023, 1028, 1027, 1024, 1025] avg=1024.0000 max=1031 max_rank=0 min=1010 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 22, 1004, 0, 1024, 0, 1023, 0, 1022, 0, 1022, 1, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1025, 1026, 1024, 1023, 1022, 1022, 1025, 1025] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 27, 1003, 0, 1024, 0, 1023, 1, 1021, 1, 1020, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1030, 1024, 1023, 1022, 1021, 1023, 1025] avg=1024.0000 max=1030 max_rank=1 min=1021 min_rank=5 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 0, 1012, 0, 1024, 2, 1025, 4, 1031, 6, 1031, 0, 1024, 1017, 0] rerouted_tokens_per_ep_rank=[1016, 1012, 1024, 1027, 1035, 1037, 1024, 1017] avg=1024.0000 max=1037 max_rank=5 min=1012 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=7155 max_load_reduction_pct=0.873413 moved_to_echo=4113 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 25, 1004, 0, 1024, 0, 1023, 0, 1021, 0, 1021, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1025, 1029, 1024, 1023, 1021, 1021, 1024, 1025] avg=1024.0000 max=1029 max_rank=1 min=1021 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1020, 0, 45, 999, 0, 1024, 0, 1023, 0, 1016, 0, 1016, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1020, 1044, 1024, 1023, 1016, 1016, 1024, 1025] avg=1024.0000 max=1044 max_rank=1 min=1016 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=7148 max_load_reduction_pct=0.872559 moved_to_echo=4081 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1031, 0, 0, 1010, 0, 1024, 0, 1023, 0, 1027, 0, 1027, 1, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1031, 1010, 1024, 1023, 1027, 1027, 1025, 1025] avg=1024.0000 max=1031 max_rank=0 min=1010 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 28, 1003, 0, 1024, 0, 1023, 1, 1020, 1, 1020, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1031, 1024, 1023, 1021, 1021, 1023, 1025] avg=1024.0000 max=1031 max_rank=1 min=1021 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8188, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8188 actual_valid_recv=8320 observed_loss=-132 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[36, 0, 0, 2775, 22, 4078, 1281, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1, 1, 0, 2938, 32, 4089, 1131, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[27, 0, 0, 3113, 85, 4091, 876, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[12, 0, 0, 2849, 25, 4093, 1213, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[12, 0, 3, 3146, 32, 4090, 904, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[20, 1, 2, 3006, 49, 4079, 1035, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[8, 4, 4, 3138, 69, 4086, 880, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[10, 1, 1, 2984, 24, 4091, 1081, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 989, 1, 1023, 0, 1017, 1012, 0, 32, 937, 1026, 0, 1131, 0, 0, 1023] rerouted_tokens_per_ep_rank=[990, 1024, 1017, 1012, 969, 1026, 1131, 1023] avg=1024.0000 max=1131 max_rank=6 min=969 min_rank=4 imbalance_ratio=1.104492 max_load_reduction=7061 max_load_reduction_pct=0.861938 moved_to_echo=4149 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[36, 934, 0, 1020, 0, 1014, 956, 0, 22, 885, 1024, 0, 1281, 0, 0, 1020] rerouted_tokens_per_ep_rank=[970, 1020, 1014, 956, 907, 1024, 1281, 1020] avg=1024.0000 max=1281 max_rank=6 min=907 min_rank=4 imbalance_ratio=1.250977 max_load_reduction=6911 max_load_reduction_pct=0.843628 moved_to_echo=4232 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[27, 1048, 0, 1024, 0, 1017, 1072, 0, 85, 993, 1027, 0, 876, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1075, 1024, 1017, 1072, 1078, 1027, 876, 1023] avg=1024.0000 max=1078 max_rank=4 min=876 min_rank=6 imbalance_ratio=1.052734 max_load_reduction=7114 max_load_reduction_pct=0.868408 moved_to_echo=4004 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[12, 963, 0, 1028, 0, 1024, 972, 0, 25, 914, 1011, 0, 1213, 0, 0, 1030] rerouted_tokens_per_ep_rank=[975, 1028, 1024, 972, 939, 1011, 1213, 1030] avg=1024.0000 max=1213 max_rank=6 min=939 min_rank=4 imbalance_ratio=1.184570 max_load_reduction=6979 max_load_reduction_pct=0.851929 moved_to_echo=4193 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[12, 1059, 0, 1023, 3, 1017, 1083, 0, 32, 1004, 1027, 0, 904, 0, 5, 1023] rerouted_tokens_per_ep_rank=[1071, 1023, 1020, 1083, 1036, 1027, 904, 1028] avg=1024.0000 max=1083 max_rank=3 min=904 min_rank=6 imbalance_ratio=1.057617 max_load_reduction=7109 max_load_reduction_pct=0.867798 moved_to_echo=3995 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[20, 1012, 1, 1021, 2, 1014, 1035, 0, 49, 959, 1024, 0, 1035, 0, 0, 1020] rerouted_tokens_per_ep_rank=[1032, 1022, 1016, 1035, 1008, 1024, 1035, 1020] avg=1024.0000 max=1035 max_rank=3 min=1008 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4087 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[8, 1056, 4, 1022, 4, 1016, 1081, 0, 69, 1001, 1026, 0, 880, 0, 3, 1022] rerouted_tokens_per_ep_rank=[1064, 1026, 1020, 1081, 1070, 1026, 880, 1025] avg=1024.0000 max=1081 max_rank=3 min=880 min_rank=6 imbalance_ratio=1.055664 max_load_reduction=7111 max_load_reduction_pct=0.868042 moved_to_echo=4001 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 1005, 1, 1024, 1, 1017, 1027, 0, 24, 952, 1027, 0, 1081, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1015, 1025, 1018, 1027, 976, 1027, 1081, 1023] avg=1024.0000 max=1081 max_rank=6 min=976 min_rank=4 imbalance_ratio=1.055664 max_load_reduction=7111 max_load_reduction_pct=0.868042 moved_to_echo=4107 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8238 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8146, 8238, 7983, 8192, 8401, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8146, 8238, 7983, 8192, 8401, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8146, 8238, 7983, 8192, 8401, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7983 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8146, 8238, 7983, 8192, 8401, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8146, 8238, 7983, 8192, 8401, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8146, 8238, 7983, 8192, 8401, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8146 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8146, 8238, 7983, 8192, 8401, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8401 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8146, 8238, 7983, 8192, 8401, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8401 actual_valid_recv=8448 observed_loss=-47 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8146 actual_valid_recv=8320 observed_loss=-174 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=7983 actual_valid_recv=8064 observed_loss=-81 tokens_per_expert=[384, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8238 actual_valid_recv=8320 observed_loss=-82 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8064 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 4091, 4095, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[11, 2, 0, 0, 3, 4087, 4089, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 6, 0, 0, 4089, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 4092, 4088, 8] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 4092, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[12, 0, 1, 0, 2, 4086, 4091, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1020, 0, 1024, 0, 1024, 0, 1024, 0, 1022, 1029, 0, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1020, 1024, 1024, 1024, 1022, 1029, 1026, 1023] avg=1024.0000 max=1029 max_rank=5 min=1020 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 1019, 0, 1023, 0, 1023, 0, 1024, 0, 1022, 1027, 0, 1026, 0, 2, 1022] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1024, 1022, 1027, 1026, 1024] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 0, 1028, 0, 1027, 0, 1029, 0, 1025, 1015, 0, 1014, 0, 0, 1029] rerouted_tokens_per_ep_rank=[1025, 1028, 1027, 1029, 1025, 1015, 1014, 1029] avg=1024.0000 max=1029 max_rank=3 min=1014 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 1019, 0, 1022, 0, 1023, 0, 1022, 0, 1020, 1028, 0, 1024, 0, 8, 1022] rerouted_tokens_per_ep_rank=[1023, 1022, 1023, 1022, 1020, 1028, 1024, 1030] avg=1024.0000 max=1030 max_rank=7 min=1020 min_rank=4 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1019, 2, 1023, 6, 1022, 0, 1024, 0, 1022, 1026, 0, 1026, 0, 0, 1022] rerouted_tokens_per_ep_rank=[1019, 1025, 1028, 1024, 1022, 1026, 1026, 1022] avg=1024.0000 max=1028 max_rank=2 min=1019 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[11, 1018, 2, 1022, 0, 1022, 0, 1022, 3, 1020, 1026, 0, 1025, 0, 0, 1021] rerouted_tokens_per_ep_rank=[1029, 1024, 1022, 1022, 1023, 1026, 1025, 1021] avg=1024.0000 max=1029 max_rank=0 min=1021 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 1019, 0, 1024, 0, 1023, 0, 1024, 0, 1022, 1028, 0, 1026, 0, 0, 1022] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1022, 1028, 1026, 1022] avg=1024.0000 max=1028 max_rank=5 min=1022 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[12, 1018, 0, 1022, 1, 1021, 0, 1023, 2, 1021, 1026, 0, 1025, 0, 0, 1021] rerouted_tokens_per_ep_rank=[1030, 1022, 1022, 1023, 1023, 1026, 1025, 1021] avg=1024.0000 max=1030 max_rank=0 min=1021 min_rank=7 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8205 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8179, 8205, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8179 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8179, 8205, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8179, 8205, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8179, 8205, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8179, 8205, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8179, 8205, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8179, 8205, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8179, 8205, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8179 actual_valid_recv=8320 observed_loss=-141 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8205 actual_valid_recv=8320 observed_loss=-115 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3555, 1, 62, 4096, 412, 0, 66, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3770, 18, 44, 4096, 171, 0, 93, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3542, 12, 80, 4096, 331, 0, 131, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3798, 34, 26, 4094, 123, 0, 117, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3496, 1, 37, 4095, 528, 6, 29, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3679, 0, 48, 4093, 365, 4, 1, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3538, 9, 3, 4096, 489, 0, 57, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3600, 0, 48, 4095, 393, 0, 56, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1075, 0, 34, 1014, 26, 1017, 1035, 0, 123, 705, 0, 1022, 117, 1001, 0, 1023] rerouted_tokens_per_ep_rank=[1075, 1048, 1043, 1035, 828, 1022, 1118, 1023] avg=1024.0000 max=1118 max_rank=6 min=828 min_rank=4 imbalance_ratio=1.091797 max_load_reduction=7074 max_load_reduction_pct=0.863525 moved_to_echo=3991 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1003, 0, 12, 1014, 80, 948, 1037, 0, 331, 657, 0, 1022, 131, 934, 0, 1023] rerouted_tokens_per_ep_rank=[1003, 1026, 1028, 1037, 988, 1022, 1065, 1023] avg=1024.0000 max=1065 max_rank=6 min=988 min_rank=4 imbalance_ratio=1.040039 max_load_reduction=7127 max_load_reduction_pct=0.869995 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1067, 0, 18, 1014, 44, 1010, 1037, 0, 171, 699, 0, 1022, 93, 994, 0, 1023] rerouted_tokens_per_ep_rank=[1067, 1032, 1054, 1037, 870, 1022, 1087, 1023] avg=1024.0000 max=1087 max_rank=6 min=870 min_rank=4 imbalance_ratio=1.061523 max_load_reduction=7105 max_load_reduction_pct=0.867310 moved_to_echo=4002 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[995, 0, 1, 1019, 62, 957, 1020, 0, 412, 663, 0, 1028, 66, 940, 0, 1029] rerouted_tokens_per_ep_rank=[995, 1020, 1019, 1020, 1075, 1028, 1006, 1029] avg=1024.0000 max=1075 max_rank=4 min=995 min_rank=0 imbalance_ratio=1.049805 max_load_reduction=7117 max_load_reduction_pct=0.868774 moved_to_echo=4138 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[990, 0, 1, 1014, 37, 936, 1036, 0, 528, 649, 6, 1022, 29, 921, 0, 1023] rerouted_tokens_per_ep_rank=[990, 1015, 973, 1036, 1177, 1028, 950, 1023] avg=1024.0000 max=1177 max_rank=4 min=950 min_rank=6 imbalance_ratio=1.149414 max_load_reduction=7015 max_load_reduction_pct=0.856323 moved_to_echo=4178 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1041, 0, 0, 1014, 48, 985, 1034, 0, 365, 683, 4, 1022, 1, 970, 2, 1023] rerouted_tokens_per_ep_rank=[1041, 1014, 1033, 1034, 1048, 1026, 971, 1025] avg=1024.0000 max=1048 max_rank=4 min=971 min_rank=6 imbalance_ratio=1.023438 max_load_reduction=7144 max_load_reduction_pct=0.872070 moved_to_echo=4070 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1002, 0, 9, 1014, 3, 947, 1037, 0, 489, 656, 0, 1022, 57, 933, 0, 1023] rerouted_tokens_per_ep_rank=[1002, 1023, 950, 1037, 1145, 1022, 990, 1023] avg=1024.0000 max=1145 max_rank=4 min=950 min_rank=2 imbalance_ratio=1.118164 max_load_reduction=7047 max_load_reduction_pct=0.860229 moved_to_echo=4180 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1019, 0, 0, 1014, 48, 964, 1036, 0, 393, 668, 0, 1022, 56, 949, 0, 1023] rerouted_tokens_per_ep_rank=[1019, 1014, 1012, 1036, 1061, 1022, 1005, 1023] avg=1024.0000 max=1061 max_rank=4 min=1005 min_rank=6 imbalance_ratio=1.036133 max_load_reduction=7131 max_load_reduction_pct=0.870483 moved_to_echo=4111 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8272 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8112, 8272, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8112, 8272, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8112, 8272, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8112, 8272, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8112, 8272, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8112, 8272, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8112 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8112, 8272, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8112, 8272, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8112 actual_valid_recv=8192 observed_loss=-80 tokens_per_expert=[384, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8272 actual_valid_recv=8320 observed_loss=-48 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2816, 5504] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[640, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[167, 3924, 4096, 0, 0, 5, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[452, 3612, 4096, 0, 0, 32, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[240, 3780, 4095, 0, 2, 72, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[221, 3867, 4096, 0, 0, 8, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[266, 3805, 4096, 0, 0, 25, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[304, 3770, 4096, 0, 0, 22, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[463, 3603, 4096, 0, 0, 30, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[163, 3932, 4094, 0, 0, 2, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[167, 766, 1063, 0, 1025, 0, 0, 1024, 0, 1023, 5, 1035, 0, 1060, 0, 1024] rerouted_tokens_per_ep_rank=[933, 1063, 1025, 1024, 1023, 1040, 1060, 1024] avg=1024.0000 max=1063 max_rank=1 min=933 min_rank=0 imbalance_ratio=1.038086 max_load_reduction=7129 max_load_reduction_pct=0.870239 moved_to_echo=4147 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[304, 736, 1020, 0, 1025, 0, 0, 1024, 0, 1023, 22, 995, 0, 1019, 0, 1024] rerouted_tokens_per_ep_rank=[1040, 1020, 1025, 1024, 1023, 1017, 1019, 1024] avg=1024.0000 max=1040 max_rank=0 min=1017 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[221, 758, 1036, 0, 1016, 0, 0, 1026, 0, 1028, 8, 1024, 0, 1049, 0, 1026] rerouted_tokens_per_ep_rank=[979, 1036, 1016, 1026, 1028, 1032, 1049, 1026] avg=1024.0000 max=1049 max_rank=6 min=979 min_rank=0 imbalance_ratio=1.024414 max_load_reduction=7143 max_load_reduction_pct=0.871948 moved_to_echo=4135 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[463, 703, 976, 0, 1025, 0, 0, 1024, 0, 1023, 30, 951, 0, 973, 0, 1024] rerouted_tokens_per_ep_rank=[1166, 976, 1025, 1024, 1023, 981, 973, 1024] avg=1024.0000 max=1166 max_rank=0 min=973 min_rank=6 imbalance_ratio=1.138672 max_load_reduction=7026 max_load_reduction_pct=0.857666 moved_to_echo=4001 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[240, 738, 1024, 0, 1026, 0, 0, 1023, 2, 1023, 72, 997, 3, 1021, 0, 1023] rerouted_tokens_per_ep_rank=[978, 1024, 1026, 1023, 1025, 1069, 1024, 1023] avg=1024.0000 max=1069 max_rank=5 min=978 min_rank=0 imbalance_ratio=1.043945 max_load_reduction=7123 max_load_reduction_pct=0.869507 moved_to_echo=4141 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[266, 743, 1030, 0, 1025, 0, 0, 1024, 0, 1023, 25, 1004, 0, 1028, 0, 1024] rerouted_tokens_per_ep_rank=[1009, 1030, 1025, 1024, 1023, 1029, 1028, 1024] avg=1024.0000 max=1030 max_rank=1 min=1009 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[452, 705, 978, 0, 1025, 0, 0, 1024, 0, 1023, 32, 953, 0, 976, 0, 1024] rerouted_tokens_per_ep_rank=[1157, 978, 1025, 1024, 1023, 985, 976, 1024] avg=1024.0000 max=1157 max_rank=0 min=976 min_rank=6 imbalance_ratio=1.129883 max_load_reduction=7035 max_load_reduction_pct=0.858765 moved_to_echo=4008 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[163, 767, 1066, 0, 1025, 0, 0, 1023, 0, 1023, 2, 1037, 1, 1062, 0, 1023] rerouted_tokens_per_ep_rank=[930, 1066, 1025, 1023, 1023, 1039, 1063, 1023] avg=1024.0000 max=1066 max_rank=1 min=930 min_rank=0 imbalance_ratio=1.041016 max_load_reduction=7126 max_load_reduction_pct=0.869873 moved_to_echo=4148 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2304, 6016] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8191 actual_valid_recv=8320 observed_loss=-129 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4095, 0, 4096, 0, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4094, 0, 4096, 0, 0, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4090, 4, 4091, 0, 2, 5, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 0, 0, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1023, 1025, 1024, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1023, 1025, 1024, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1023, 1025, 1024, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1010, 0, 0, 1028, 1016, 0, 0, 1027, 0, 1027, 0, 1027, 2, 1029, 0, 1026] rerouted_tokens_per_ep_rank=[1010, 1028, 1016, 1027, 1027, 1027, 1031, 1026] avg=1024.0000 max=1031 max_rank=6 min=1010 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4111 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 4, 1022, 1025, 0, 0, 1022, 2, 1022, 5, 1022, 0, 1022, 0, 1022] rerouted_tokens_per_ep_rank=[1024, 1026, 1025, 1022, 1024, 1027, 1022, 1022] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1023, 1025, 1024, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1023, 1, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1024, 1023, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1027, 1023, 1026, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8189, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8189, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8189, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8189, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8189, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8189, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8189, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8189, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8189 actual_valid_recv=8320 observed_loss=-131 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4091, 4089, 0, 3, 3, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1027, 1024, 1023, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 1027, 1017, 0, 1010, 0, 0, 1026, 0, 1029, 0, 1029, 0, 1028] rerouted_tokens_per_ep_rank=[1026, 1027, 1017, 1010, 1026, 1029, 1029, 1028] avg=1024.0000 max=1029 max_rank=5 min=1010 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4112 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1027, 1024, 1023, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1027, 1024, 1023, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1, 1022, 1025, 0, 1024, 0, 0, 1022, 3, 1022, 3, 1022, 5, 1021] rerouted_tokens_per_ep_rank=[1022, 1023, 1025, 1024, 1022, 1025, 1025, 1026] avg=1024.0000 max=1026 max_rank=7 min=1022 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1027, 1024, 1023, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1027, 1024, 1023, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1027, 1024, 1023, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8196, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8188 actual_valid_recv=8320 observed_loss=-132 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[2, 4096, 0, 4092, 2, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4, 4096, 0, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4086, 10, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4089, 0, 4092, 0, 7, 0, 4] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[3, 4096, 0, 4093, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4091, 5, 1, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1028, 0, 0, 1022, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1028, 1022, 1023, 1024, 1023] avg=1024.0000 max=1028 max_rank=3 min=1022 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 1022, 1025, 0, 0, 1024, 1027, 0, 2, 1021, 0, 1022, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1025, 1024, 1027, 1023, 1022, 1024, 1023] avg=1024.0000 max=1027 max_rank=3 min=1022 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 1027, 1019, 0, 0, 1026, 1011, 0, 0, 1026, 0, 1028, 0, 1026, 0, 1025] rerouted_tokens_per_ep_rank=[1031, 1019, 1026, 1011, 1026, 1028, 1026, 1025] avg=1024.0000 max=1031 max_rank=0 min=1011 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 1023, 1025, 0, 0, 1024, 1025, 0, 0, 1022, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1025, 1024, 1025, 1022, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1025, 0, 0, 1024, 1024, 0, 10, 1020, 0, 1021, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1021, 1025, 1024, 1024, 1030, 1021, 1024, 1023] avg=1024.0000 max=1030 max_rank=4 min=1021 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1024, 0, 0, 1022, 1027, 0, 0, 1021, 7, 1022, 0, 1022, 4, 1021] rerouted_tokens_per_ep_rank=[1022, 1024, 1022, 1027, 1021, 1029, 1022, 1025] avg=1024.0000 max=1029 max_rank=5 min=1021 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1028, 0, 0, 1022, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1028, 1022, 1023, 1024, 1023] avg=1024.0000 max=1028 max_rank=3 min=1022 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1024, 0, 0, 1024, 1026, 0, 5, 1021, 1, 1022, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1022, 1024, 1024, 1026, 1026, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=3 min=1022 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8192, 8192, 8192, 8188] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8192, 8192, 8192, 8188] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8192, 8192, 8192, 8188] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8192, 8192, 8192, 8188] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8192, 8192, 8192, 8188] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8192, 8192, 8192, 8188] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8192, 8192, 8192, 8188] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8196, 8192, 8192, 8192, 8188] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8188 actual_valid_recv=8320 observed_loss=-132 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3450, 0, 0, 646] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 4, 4, 4089, 3333, 13, 4, 745] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3316, 0, 0, 780] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 14, 0, 4096, 3507, 0, 0, 575] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3086, 1, 0, 1009] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3138, 2, 0, 956] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3395, 0, 0, 701] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3381, 0, 0, 715] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1059, 0, 1023, 1026, 0, 1064, 0, 0, 1060, 0, 1023, 646, 267] rerouted_tokens_per_ep_rank=[1024, 1059, 1023, 1026, 1064, 1060, 1023, 913] avg=1024.0000 max=1064 max_rank=4 min=913 min_rank=7 imbalance_ratio=1.039062 max_load_reduction=7128 max_load_reduction_pct=0.870117 moved_to_echo=4060 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1018, 0, 1023, 1026, 0, 1023, 0, 0, 1018, 0, 1023, 780, 257] rerouted_tokens_per_ep_rank=[1024, 1018, 1023, 1026, 1023, 1018, 1023, 1037] avg=1024.0000 max=1037 max_rank=7 min=1018 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=7155 max_load_reduction_pct=0.873413 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 948, 0, 1023, 1026, 0, 951, 0, 1, 948, 0, 1023, 1009, 239] rerouted_tokens_per_ep_rank=[1024, 948, 1023, 1026, 951, 949, 1023, 1248] avg=1024.0000 max=1248 max_rank=7 min=948 min_rank=1 imbalance_ratio=1.218750 max_load_reduction=6944 max_load_reduction_pct=0.847656 moved_to_echo=4171 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 4, 1023, 4, 1021, 1025, 0, 1028, 0, 13, 1024, 4, 1021, 745, 258] rerouted_tokens_per_ep_rank=[1022, 1027, 1025, 1025, 1028, 1037, 1025, 1003] avg=1024.0000 max=1037 max_rank=5 min=1003 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=7155 max_load_reduction_pct=0.873413 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 1047, 0, 1029, 1012, 0, 1035, 0, 0, 1046, 0, 1029, 701, 267] rerouted_tokens_per_ep_rank=[1026, 1047, 1029, 1012, 1035, 1046, 1029, 968] avg=1024.0000 max=1047 max_rank=1 min=968 min_rank=7 imbalance_ratio=1.022461 max_load_reduction=7145 max_load_reduction_pct=0.872192 moved_to_echo=4078 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 964, 0, 1023, 1026, 0, 967, 0, 2, 964, 0, 1023, 956, 243] rerouted_tokens_per_ep_rank=[1024, 964, 1023, 1026, 967, 966, 1023, 1199] avg=1024.0000 max=1199 max_rank=7 min=964 min_rank=1 imbalance_ratio=1.170898 max_load_reduction=6993 max_load_reduction_pct=0.853638 moved_to_echo=4155 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 14, 1077, 0, 1023, 1026, 0, 1081, 0, 0, 1077, 0, 1023, 575, 272] rerouted_tokens_per_ep_rank=[1024, 1091, 1023, 1026, 1081, 1077, 1023, 847] avg=1024.0000 max=1091 max_rank=1 min=847 min_rank=7 imbalance_ratio=1.065430 max_load_reduction=7101 max_load_reduction_pct=0.866821 moved_to_echo=4028 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1038, 0, 1023, 1026, 0, 1043, 0, 0, 1038, 0, 1023, 715, 262] rerouted_tokens_per_ep_rank=[1024, 1038, 1023, 1026, 1043, 1038, 1023, 977] avg=1024.0000 max=1043 max_rank=4 min=977 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=7149 max_load_reduction_pct=0.872681 moved_to_echo=4081 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6144, 2176] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8191 actual_valid_recv=8320 observed_loss=-129 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4085, 11, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 4096, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4089, 0, 1, 4, 5, 4058, 33, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 4096, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4093, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1024, 0, 1022, 0, 1022, 1025, 0, 11, 1016, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1024, 1022, 1022, 1025, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 0, 1025, 0, 1025, 1027, 0, 2, 1019, 0, 1022] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1025, 1025, 1027, 1021, 1022] avg=1024.0000 max=1027 max_rank=5 min=1021 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 0, 1027, 0, 1027, 0, 1027, 0, 1026, 1020, 0, 0, 1023, 0, 1026] rerouted_tokens_per_ep_rank=[1016, 1027, 1027, 1027, 1026, 1020, 1023, 1026] avg=1024.0000 max=1027 max_rank=1 min=1016 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1025, 0, 1025, 1027, 0, 1, 1019, 0, 1022] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1025, 1025, 1027, 1020, 1022] avg=1024.0000 max=1027 max_rank=5 min=1020 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1022, 1, 1022, 4, 1015, 5, 1015, 1019, 0, 33, 1009, 2, 1021] rerouted_tokens_per_ep_rank=[1024, 1022, 1023, 1019, 1020, 1019, 1042, 1023] avg=1024.0000 max=1042 max_rank=6 min=1019 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=7150 max_load_reduction_pct=0.872803 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1024, 0, 1025, 0, 1025, 1027, 0, 0, 1019, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1024, 1025, 1025, 1027, 1019, 1023] avg=1024.0000 max=1027 max_rank=5 min=1019 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1027, 0, 3, 1018, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1024, 1024, 1024, 1027, 1021, 1023] avg=1024.0000 max=1027 max_rank=5 min=1021 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1024, 0, 1025, 0, 1025, 1027, 0, 0, 1019, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1024, 1025, 1025, 1027, 1019, 1023] avg=1024.0000 max=1027 max_rank=5 min=1019 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8199, 8192, 8185] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8199, 8192, 8185] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8199, 8192, 8185] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8199, 8192, 8185] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8199, 8192, 8185] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8199, 8192, 8185] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8199, 8192, 8185] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8199, 8192, 8185] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8185 actual_valid_recv=8320 observed_loss=-135 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[5, 4092, 1, 0, 1, 4091, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4095, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1024, 1024, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1024, 1024, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 1018, 0, 0, 1028, 0, 1025, 0, 1026, 1014, 0, 0, 1029, 0, 1025] rerouted_tokens_per_ep_rank=[1027, 1018, 1028, 1025, 1026, 1014, 1029, 1025] avg=1024.0000 max=1029 max_rank=6 min=1014 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1024, 1024, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[5, 1022, 1024, 0, 1, 1022, 0, 1023, 1, 1022, 1025, 0, 2, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1027, 1024, 1023, 1023, 1023, 1025, 1024, 1023] avg=1024.0000 max=1027 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1024, 1024, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1024, 1024, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1023, 0, 1024, 0, 1023, 1026, 0, 1, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1024, 1023, 1026, 1024, 1024] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8189 actual_valid_recv=8320 observed_loss=-131 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[3, 0, 5, 4088, 1, 5, 0, 4090] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 0, 4, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 1, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1027, 1024, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1027, 1024, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 0, 1026, 0, 1029, 1009, 0, 0, 1025, 0, 1029, 0, 1029, 1018, 0] rerouted_tokens_per_ep_rank=[1027, 1026, 1029, 1009, 1025, 1029, 1029, 1018] avg=1024.0000 max=1029 max_rank=2 min=1009 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 1022, 0, 1022, 5, 1021, 1025, 0, 1, 1022, 5, 1021, 0, 1021, 1024, 0] rerouted_tokens_per_ep_rank=[1025, 1022, 1026, 1025, 1023, 1026, 1021, 1024] avg=1024.0000 max=1026 max_rank=2 min=1021 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1027, 1024, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1022, 1026, 0, 0, 1024, 0, 1022, 4, 1022, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1022, 1026, 1024, 1022, 1026, 1025] avg=1024.0000 max=1026 max_rank=3 min=1022 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1027, 1024, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1026, 0, 0, 1024, 0, 1023, 1, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1026, 1024, 1023, 1024, 1025] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8190 actual_valid_recv=8320 observed_loss=-130 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4, 0, 6, 4088, 1, 2, 5, 4086] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1023, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1024, 1023, 1023, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1023, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1024, 1023, 1023, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4, 1021, 0, 1022, 6, 1021, 1023, 0, 1, 1022, 2, 1021, 5, 1021, 1023, 0] rerouted_tokens_per_ep_rank=[1025, 1022, 1027, 1023, 1023, 1023, 1026, 1023] avg=1024.0000 max=1027 max_rank=2 min=1022 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1023, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1024, 1023, 1023, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1023, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1024, 1023, 1023, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 0, 1026, 0, 1027, 1019, 0, 0, 1025, 0, 1026, 0, 1028, 1013, 0] rerouted_tokens_per_ep_rank=[1028, 1026, 1027, 1019, 1025, 1026, 1028, 1013] avg=1024.0000 max=1028 max_rank=0 min=1013 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1023, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1024, 1023, 1023, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1025, 1024, 1023, 1023, 1026] avg=1024.0000 max=1026 max_rank=7 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8187 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8197 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8197 actual_valid_recv=8320 observed_loss=-123 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8187 actual_valid_recv=8320 observed_loss=-133 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 176, 4089, 0, 3927] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 223, 4091, 0, 3878] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 251, 4082, 0, 3859] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[5, 5, 2, 0, 249, 4071, 2, 3858] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 144, 4093, 0, 3955] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 211, 4083, 0, 3898] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 271, 3994, 0, 3927] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 248, 4071, 0, 3873] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1031, 0, 1031, 0, 1004, 0, 1028, 176, 808, 1029, 0, 0, 1028, 1057, 0] rerouted_tokens_per_ep_rank=[1031, 1031, 1004, 1028, 984, 1029, 1028, 1057] avg=1024.0000 max=1057 max_rank=7 min=984 min_rank=4 imbalance_ratio=1.032227 max_load_reduction=7135 max_load_reduction_pct=0.870972 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1038, 0, 1038, 0, 1005, 0, 1029, 144, 814, 1030, 0, 0, 1029, 1065, 0] rerouted_tokens_per_ep_rank=[1038, 1038, 1005, 1029, 958, 1030, 1029, 1065] avg=1024.0000 max=1065 max_rank=7 min=958 min_rank=4 imbalance_ratio=1.040039 max_load_reduction=7127 max_load_reduction_pct=0.869995 moved_to_echo=4082 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[5, 1013, 5, 1013, 2, 999, 0, 1023, 249, 794, 1026, 0, 2, 1023, 1038, 0] rerouted_tokens_per_ep_rank=[1018, 1018, 1001, 1023, 1043, 1026, 1025, 1038] avg=1024.0000 max=1043 max_rank=4 min=1001 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=7149 max_load_reduction_pct=0.872681 moved_to_echo=4132 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1018, 0, 1018, 0, 1004, 0, 1028, 223, 798, 1031, 0, 0, 1028, 1044, 0] rerouted_tokens_per_ep_rank=[1018, 1018, 1004, 1028, 1021, 1031, 1028, 1044] avg=1024.0000 max=1044 max_rank=7 min=1004 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=7148 max_load_reduction_pct=0.872559 moved_to_echo=4124 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 1026, 0, 1007, 0, 1031, 211, 806, 1016, 0, 0, 1029, 1040, 0] rerouted_tokens_per_ep_rank=[1026, 1026, 1007, 1031, 1017, 1016, 1029, 1040] avg=1024.0000 max=1040 max_rank=7 min=1007 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1031, 0, 1031, 0, 980, 0, 1004, 271, 808, 1006, 0, 0, 1004, 1057, 0] rerouted_tokens_per_ep_rank=[1031, 1031, 980, 1004, 1079, 1006, 1004, 1057] avg=1024.0000 max=1079 max_rank=4 min=980 min_rank=2 imbalance_ratio=1.053711 max_load_reduction=7113 max_load_reduction_pct=0.868286 moved_to_echo=4146 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1013, 0, 1013, 0, 1002, 0, 1026, 251, 794, 1028, 0, 0, 1026, 1039, 0] rerouted_tokens_per_ep_rank=[1013, 1013, 1002, 1026, 1045, 1028, 1026, 1039] avg=1024.0000 max=1045 max_rank=4 min=1002 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=7147 max_load_reduction_pct=0.872437 moved_to_echo=4138 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1017, 0, 1017, 0, 999, 0, 1023, 248, 797, 1026, 0, 0, 1023, 1042, 0] rerouted_tokens_per_ep_rank=[1017, 1017, 999, 1023, 1045, 1026, 1023, 1042] avg=1024.0000 max=1045 max_rank=4 min=999 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=7147 max_load_reduction_pct=0.872437 moved_to_echo=4136 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8002, 8192, 8192, 8192, 8192, 8382] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8002, 8192, 8192, 8192, 8192, 8382] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8002 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8002, 8192, 8192, 8192, 8192, 8382] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8002, 8192, 8192, 8192, 8192, 8382] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8002, 8192, 8192, 8192, 8192, 8382] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8002, 8192, 8192, 8192, 8192, 8382] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8382 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8002, 8192, 8192, 8192, 8192, 8382] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8002, 8192, 8192, 8192, 8192, 8382] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8002 actual_valid_recv=8192 observed_loss=-190 tokens_per_expert=[128, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1792, 6528] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8382 actual_valid_recv=8448 observed_loss=-66 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4089, 2, 3, 8, 4089, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 1, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 1, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4094, 2, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4088, 8, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 0, 1024, 0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1023, 1024, 1023, 1026, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 0, 1024, 0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1023, 1024, 1023, 1026, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 2, 1021, 3, 1022, 8, 1021, 1025, 0, 0, 1021, 0, 1022, 1, 1022] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1029, 1025, 1021, 1022, 1023] avg=1024.0000 max=1029 max_rank=3 min=1021 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1018, 0, 0, 1027, 0, 1027, 0, 1027, 1016, 0, 1, 1025, 0, 1026, 0, 1025] rerouted_tokens_per_ep_rank=[1018, 1027, 1027, 1027, 1016, 1026, 1026, 1025] avg=1024.0000 max=1027 max_rank=1 min=1016 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 2, 1022, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1023, 1023, 1023, 1026, 1024, 1024, 1024] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 0, 1023, 0, 1023, 1027, 0, 1, 1022, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1023, 1023, 1023, 1027, 1023, 1024, 1024] avg=1024.0000 max=1027 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 0, 1024, 0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1023, 1024, 1023, 1026, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 0, 1022, 0, 1021, 1024, 0, 8, 1021, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1023, 1022, 1021, 1024, 1029, 1024, 1024] avg=1024.0000 max=1029 max_rank=5 min=1021 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8192, 8196, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8192, 8196, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8192, 8196, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8192, 8196, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8192, 8196, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8192, 8196, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8192, 8196, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8192, 8192, 8196, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8188 actual_valid_recv=8320 observed_loss=-132 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 4088, 7, 4090, 6, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 4096, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 4096, 0, 0, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 4096, 0, 0, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1026, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1024, 1, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1026, 1023, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1027, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1027, 1023, 1025, 1023, 1024, 1023] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1027, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1027, 1023, 1025, 1023, 1024, 1023] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1021, 0, 1022, 1025, 0, 7, 1021, 1025, 0, 6, 1021, 0, 1022, 0, 1021] rerouted_tokens_per_ep_rank=[1022, 1022, 1025, 1028, 1025, 1027, 1022, 1021] avg=1024.0000 max=1028 max_rank=3 min=1021 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1027, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1027, 1023, 1025, 1023, 1024, 1023] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 0, 1026, 1011, 0, 0, 1028, 1017, 0, 0, 1029, 0, 1026, 0, 1028] rerouted_tokens_per_ep_rank=[1027, 1026, 1011, 1028, 1017, 1029, 1026, 1028] avg=1024.0000 max=1029 max_rank=5 min=1011 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1027, 0, 0, 1022, 1025, 0, 0, 1022, 0, 1024, 3, 1022] rerouted_tokens_per_ep_rank=[1023, 1024, 1027, 1022, 1025, 1022, 1024, 1025] avg=1024.0000 max=1027 max_rank=2 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1027, 0, 0, 1022, 1025, 0, 0, 1022, 0, 1024, 3, 1022] rerouted_tokens_per_ep_rank=[1023, 1024, 1027, 1022, 1025, 1022, 1024, 1025] avg=1024.0000 max=1027 max_rank=2 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8197, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8197, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8197 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8197, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8187 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8197, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8197, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8197, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8197, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8197, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8187 actual_valid_recv=8320 observed_loss=-133 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8197 actual_valid_recv=8320 observed_loss=-123 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4092, 4, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4084, 12, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4081, 15, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4079, 17, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4072, 17, 0, 2, 5, 7, 0, 4089] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4009, 87, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4058, 38, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1032, 0, 0, 1006, 0, 1024, 0, 1023, 0, 1029, 0, 1029, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1032, 1006, 1024, 1023, 1029, 1029, 1024, 1025] avg=1024.0000 max=1032 max_rank=0 min=1006 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4107 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 17, 1002, 0, 1024, 0, 1023, 0, 1025, 0, 1025, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1027, 1019, 1024, 1023, 1025, 1025, 1024, 1025] avg=1024.0000 max=1027 max_rank=0 min=1019 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1029, 0, 12, 1003, 0, 1024, 0, 1023, 0, 1026, 0, 1026, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1029, 1015, 1024, 1023, 1026, 1026, 1024, 1025] avg=1024.0000 max=1029 max_rank=0 min=1015 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1018, 0, 15, 1006, 0, 1026, 0, 1026, 0, 1029, 0, 1028, 0, 1026, 1018, 0] rerouted_tokens_per_ep_rank=[1018, 1021, 1026, 1026, 1029, 1028, 1026, 1018] avg=1024.0000 max=1029 max_rank=4 min=1018 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 17, 1000, 0, 1022, 2, 1021, 5, 1023, 7, 1023, 0, 1022, 1024, 0] rerouted_tokens_per_ep_rank=[1026, 1017, 1022, 1023, 1028, 1030, 1022, 1024] avg=1024.0000 max=1030 max_rank=5 min=1017 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1011, 0, 87, 984, 0, 1024, 0, 1023, 0, 1007, 0, 1007, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1011, 1071, 1024, 1023, 1007, 1007, 1024, 1025] avg=1024.0000 max=1071 max_rank=1 min=1007 min_rank=4 imbalance_ratio=1.045898 max_load_reduction=7121 max_load_reduction_pct=0.869263 moved_to_echo=4063 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1031, 0, 4, 1005, 0, 1024, 0, 1023, 0, 1028, 0, 1028, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1031, 1009, 1024, 1023, 1028, 1028, 1024, 1025] avg=1024.0000 max=1031 max_rank=0 min=1009 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1023, 0, 38, 996, 0, 1024, 0, 1023, 0, 1020, 0, 1019, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1034, 1024, 1023, 1020, 1019, 1024, 1025] avg=1024.0000 max=1034 max_rank=1 min=1019 min_rank=5 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4088 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8187 expected_overflow=0 exact_recv_per_rank=[8197, 8192, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8197, 8192, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8197, 8192, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8197, 8192, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8197, 8192, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8197 expected_overflow=0 exact_recv_per_rank=[8197, 8192, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8197, 8192, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8197, 8192, 8192, 8187, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8197 actual_valid_recv=8320 observed_loss=-123 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8187 actual_valid_recv=8320 observed_loss=-133 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True + [2026-08-30 04:54:03] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 678.7 | throughput per GPU (TFLOP/s/GPU): 491.2 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 9.805943E+00 | loss scale: 1.0 | grad norm: 37.727 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[4, 3, 0, 4095, 3, 3943, 144, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[39, 0, 0, 4075, 5, 3895, 175, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[4, 2, 0, 4091, 3, 3907, 185, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2, 0, 2, 4092, 0, 3887, 209, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[7, 1, 0, 4094, 2, 3887, 198, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[31, 0, 2, 4090, 1, 3920, 146, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[5, 0, 0, 4093, 0, 3936, 156, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1, 1, 2, 4091, 2, 3964, 131, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 1009, 2, 1023, 0, 1023, 1026, 0, 3, 1019, 1026, 0, 185, 853, 0, 1019] rerouted_tokens_per_ep_rank=[1013, 1025, 1023, 1026, 1022, 1026, 1038, 1019] avg=1024.0000 max=1038 max_rank=6 min=1013 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[39, 1010, 0, 1022, 0, 1023, 1009, 0, 5, 1019, 1012, 0, 175, 854, 3, 1021] rerouted_tokens_per_ep_rank=[1049, 1022, 1023, 1009, 1024, 1012, 1029, 1024] avg=1024.0000 max=1049 max_rank=0 min=1009 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=7143 max_load_reduction_pct=0.871948 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[31, 1013, 0, 1023, 2, 1023, 1025, 0, 1, 1022, 1029, 0, 146, 856, 2, 1019] rerouted_tokens_per_ep_rank=[1044, 1023, 1025, 1025, 1023, 1029, 1002, 1021] avg=1024.0000 max=1044 max_rank=0 min=1002 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=7148 max_load_reduction_pct=0.872559 moved_to_echo=4075 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 1018, 3, 1024, 0, 1024, 1027, 0, 3, 1028, 1036, 0, 144, 861, 0, 1020] rerouted_tokens_per_ep_rank=[1022, 1027, 1024, 1027, 1031, 1036, 1005, 1020] avg=1024.0000 max=1036 max_rank=5 min=1005 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 1004, 1, 1024, 0, 1024, 1026, 0, 2, 1014, 1020, 0, 198, 849, 3, 1020] rerouted_tokens_per_ep_rank=[1011, 1025, 1024, 1026, 1016, 1020, 1047, 1023] avg=1024.0000 max=1047 max_rank=6 min=1011 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=7145 max_load_reduction_pct=0.872192 moved_to_echo=4106 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1004, 0, 1023, 2, 1023, 1026, 0, 0, 1014, 1020, 0, 209, 849, 0, 1020] rerouted_tokens_per_ep_rank=[1006, 1023, 1025, 1026, 1014, 1020, 1058, 1020] avg=1024.0000 max=1058 max_rank=6 min=1006 min_rank=0 imbalance_ratio=1.033203 max_load_reduction=7134 max_load_reduction_pct=0.870850 moved_to_echo=4112 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 1017, 0, 1023, 0, 1023, 1027, 0, 0, 1026, 1033, 0, 156, 860, 2, 1020] rerouted_tokens_per_ep_rank=[1022, 1023, 1023, 1027, 1026, 1033, 1016, 1022] avg=1024.0000 max=1033 max_rank=5 min=1016 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 1024, 1, 1023, 2, 1023, 1026, 0, 2, 1034, 1040, 0, 131, 866, 0, 1019] rerouted_tokens_per_ep_rank=[1025, 1024, 1025, 1026, 1036, 1040, 997, 1019] avg=1024.0000 max=1040 max_rank=5 min=997 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8216, 8192, 8168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8216, 8192, 8168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8216 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8216, 8192, 8168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8216, 8192, 8168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8168 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8216, 8192, 8168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8216, 8192, 8168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8216, 8192, 8168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8216, 8192, 8168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8216 actual_valid_recv=8320 observed_loss=-104 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1408, 6912] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8168 actual_valid_recv=8320 observed_loss=-152 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4, 0, 2, 0, 1, 4092, 3, 4090] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[6, 0, 4, 3, 0, 4090, 3, 4086] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[3, 0, 0, 1, 1, 4091, 7, 4089] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1, 1, 6, 2, 0, 4092, 2, 4088] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2, 1, 22, 0, 0, 4094, 2, 4071] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[21, 0, 1, 0, 0, 4096, 0, 4074] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[15, 0, 0, 0, 0, 4096, 0, 4081] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 4095, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 1018, 0, 1023, 2, 1021, 0, 1021, 1, 1023, 1025, 0, 3, 1023, 1028, 0] rerouted_tokens_per_ep_rank=[1022, 1023, 1023, 1021, 1024, 1025, 1026, 1028] avg=1024.0000 max=1028 max_rank=7 min=1021 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 1017, 0, 1022, 4, 1020, 3, 1021, 0, 1022, 1025, 0, 3, 1022, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1022, 1024, 1024, 1022, 1025, 1025, 1027] avg=1024.0000 max=1027 max_rank=7 min=1022 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1018, 1, 1023, 6, 1020, 2, 1021, 0, 1023, 1025, 0, 2, 1022, 1028, 0] rerouted_tokens_per_ep_rank=[1019, 1024, 1026, 1023, 1023, 1025, 1024, 1028] avg=1024.0000 max=1028 max_rank=7 min=1019 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 1014, 1, 1023, 22, 1016, 0, 1022, 0, 1023, 1026, 0, 2, 1018, 1023, 0] rerouted_tokens_per_ep_rank=[1016, 1024, 1038, 1022, 1023, 1026, 1020, 1023] avg=1024.0000 max=1038 max_rank=2 min=1016 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 1018, 0, 1023, 0, 1020, 1, 1021, 1, 1023, 1024, 0, 7, 1023, 1028, 0] rerouted_tokens_per_ep_rank=[1021, 1023, 1020, 1022, 1024, 1024, 1030, 1028] avg=1024.0000 max=1030 max_rank=6 min=1020 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4106 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[21, 1018, 0, 1027, 1, 1020, 0, 1025, 0, 1028, 1016, 0, 0, 1022, 1014, 0] rerouted_tokens_per_ep_rank=[1039, 1027, 1021, 1025, 1028, 1016, 1022, 1014] avg=1024.0000 max=1039 max_rank=0 min=1014 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=7153 max_load_reduction_pct=0.873169 moved_to_echo=4080 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[15, 1016, 0, 1024, 0, 1018, 0, 1022, 0, 1024, 1026, 0, 0, 1021, 1026, 0] rerouted_tokens_per_ep_rank=[1031, 1024, 1018, 1022, 1024, 1026, 1021, 1026] avg=1024.0000 max=1031 max_rank=0 min=1018 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1020, 1, 1024, 0, 1022, 0, 1022, 0, 1024, 1025, 0, 0, 1024, 1029, 0] rerouted_tokens_per_ep_rank=[1021, 1025, 1022, 1022, 1024, 1025, 1024, 1029] avg=1024.0000 max=1029 max_rank=7 min=1021 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8181 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8181, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8181, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8181, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8181, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8203 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8181, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8181, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8181, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8181, 8192, 8192, 8192, 8203] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8181 actual_valid_recv=8320 observed_loss=-139 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8203 actual_valid_recv=8320 observed_loss=-117 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3972, 1, 1, 0, 123, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3943, 0, 18, 0, 135, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3968, 3, 0, 4, 122, 0, 4093, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3951, 0, 19, 2, 125, 1, 4093, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3941, 3, 0, 0, 155, 0, 4093, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3956, 2, 13, 0, 134, 0, 4087, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3962, 0, 17, 0, 117, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3973, 0, 0, 0, 123, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1029, 0, 3, 1025, 0, 1018, 4, 1021, 122, 896, 0, 1023, 1026, 0, 2, 1023] rerouted_tokens_per_ep_rank=[1029, 1028, 1018, 1025, 1018, 1023, 1026, 1025] avg=1024.0000 max=1029 max_rank=0 min=1018 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1030, 0, 1, 1026, 1, 1019, 0, 1022, 123, 897, 0, 1024, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1030, 1027, 1020, 1022, 1020, 1024, 1026, 1023] avg=1024.0000 max=1030 max_rank=0 min=1020 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1021, 19, 1013, 2, 1021, 125, 893, 1, 1023, 1026, 0, 1, 1023] rerouted_tokens_per_ep_rank=[1024, 1021, 1032, 1023, 1018, 1024, 1026, 1024] avg=1024.0000 max=1032 max_rank=2 min=1018 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1012, 0, 0, 1022, 18, 1014, 0, 1028, 135, 895, 0, 1028, 1012, 0, 0, 1028] rerouted_tokens_per_ep_rank=[1012, 1022, 1032, 1028, 1030, 1028, 1012, 1028] avg=1024.0000 max=1032 max_rank=2 min=1012 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1022, 0, 3, 1018, 0, 1011, 0, 1021, 155, 890, 0, 1023, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1022, 1021, 1011, 1021, 1045, 1023, 1026, 1023] avg=1024.0000 max=1045 max_rank=4 min=1011 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=7147 max_load_reduction_pct=0.872437 moved_to_echo=4117 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 2, 1022, 13, 1014, 0, 1020, 134, 894, 0, 1022, 1024, 0, 0, 1021] rerouted_tokens_per_ep_rank=[1026, 1024, 1027, 1020, 1028, 1022, 1024, 1021] avg=1024.0000 max=1028 max_rank=4 min=1020 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1030, 0, 0, 1026, 0, 1019, 0, 1022, 123, 898, 0, 1024, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1030, 1026, 1019, 1022, 1021, 1024, 1026, 1024] avg=1024.0000 max=1030 max_rank=0 min=1019 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 0, 1023, 17, 1016, 0, 1022, 117, 895, 0, 1024, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1028, 1023, 1033, 1022, 1012, 1024, 1026, 1024] avg=1024.0000 max=1033 max_rank=2 min=1012 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4086 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8183, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8183 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8183, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8183, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8183, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8183, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8183, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8201 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8183, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8183, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8201 actual_valid_recv=8320 observed_loss=-119 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8183 actual_valid_recv=8320 observed_loss=-137 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[3966, 0, 4096, 0, 0, 130, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[3991, 0, 4095, 1, 0, 103, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[3943, 0, 4096, 0, 0, 153, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[3963, 2, 4092, 0, 0, 130, 3, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[3970, 0, 4095, 2, 0, 124, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[3955, 0, 4096, 0, 0, 141, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3956, 0, 4096, 4, 0, 136, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[3935, 0, 4096, 0, 0, 161, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1023, 1026, 0, 0, 1024, 0, 1024, 130, 890, 0, 1023, 0, 1025] rerouted_tokens_per_ep_rank=[1027, 1023, 1026, 1024, 1024, 1020, 1023, 1025] avg=1024.0000 max=1027 max_rank=0 min=1020 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1033, 0, 0, 1023, 1026, 0, 1, 1031, 0, 1023, 103, 896, 1, 1023, 1, 1031] rerouted_tokens_per_ep_rank=[1033, 1023, 1026, 1032, 1023, 999, 1024, 1032] avg=1024.0000 max=1033 max_rank=0 min=999 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4078 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 2, 1022, 1025, 0, 0, 1023, 0, 1023, 130, 889, 3, 1022, 2, 1024] rerouted_tokens_per_ep_rank=[1027, 1024, 1025, 1023, 1023, 1019, 1025, 1026] avg=1024.0000 max=1027 max_rank=0 min=1019 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1010, 0, 0, 1030, 1011, 0, 0, 1023, 0, 1027, 153, 889, 0, 1028, 0, 1021] rerouted_tokens_per_ep_rank=[1010, 1030, 1011, 1023, 1027, 1042, 1028, 1021] avg=1024.0000 max=1042 max_rank=5 min=1010 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=7150 max_load_reduction_pct=0.872803 moved_to_echo=4118 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1023, 1026, 0, 0, 1021, 0, 1024, 141, 888, 0, 1023, 0, 1022] rerouted_tokens_per_ep_rank=[1024, 1023, 1026, 1021, 1024, 1029, 1023, 1022] avg=1024.0000 max=1029 max_rank=5 min=1021 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 0, 1023, 1026, 0, 2, 1025, 0, 1023, 124, 891, 0, 1023, 1, 1026] rerouted_tokens_per_ep_rank=[1028, 1023, 1026, 1027, 1023, 1015, 1023, 1027] avg=1024.0000 max=1028 max_rank=0 min=1015 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4088 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1023, 1026, 0, 4, 1022, 0, 1024, 136, 888, 0, 1023, 0, 1022] rerouted_tokens_per_ep_rank=[1024, 1023, 1026, 1026, 1024, 1024, 1023, 1022] avg=1024.0000 max=1026 max_rank=2 min=1022 min_rank=7 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1019, 0, 0, 1023, 1026, 0, 0, 1016, 0, 1024, 161, 883, 0, 1023, 0, 1017] rerouted_tokens_per_ep_rank=[1019, 1023, 1026, 1016, 1024, 1044, 1023, 1017] avg=1024.0000 max=1044 max_rank=5 min=1016 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=7148 max_load_reduction_pct=0.872559 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1, 2, 4093, 0, 2, 4094, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4, 4091, 0, 5, 4092, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[6, 0, 4096, 0, 0, 4090, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 4094, 3, 1, 4093, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[14, 0, 4096, 0, 0, 4082, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 4096, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 1020, 0, 1022, 1025, 0, 0, 1023, 0, 1022, 1026, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1022, 1025, 1023, 1022, 1026, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1021, 2, 1023, 1025, 0, 0, 1022, 2, 1023, 1027, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1022, 1025, 1025, 1022, 1025, 1027, 1023, 1023] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 4, 1023, 1024, 0, 0, 1021, 5, 1022, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1021, 1027, 1024, 1021, 1027, 1026, 1023, 1023] avg=1024.0000 max=1027 max_rank=1 min=1021 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 0, 1028, 1017, 0, 0, 1025, 0, 1028, 1015, 0, 0, 1027, 0, 1027] rerouted_tokens_per_ep_rank=[1025, 1028, 1017, 1025, 1028, 1015, 1027, 1027] avg=1024.0000 max=1028 max_rank=1 min=1015 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1021, 0, 1023, 1026, 0, 3, 1022, 1, 1023, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1022, 1023, 1026, 1025, 1024, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=2 min=1022 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1027, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1022, 1024, 1025, 1023, 1023, 1027, 1024, 1024] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[14, 1018, 0, 1020, 1025, 0, 0, 1023, 0, 1020, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1032, 1020, 1025, 1023, 1020, 1024, 1024, 1024] avg=1024.0000 max=1032 max_rank=0 min=1020 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1021, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 1028, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1022, 1023, 1025, 1023, 1023, 1028, 1024, 1024] avg=1024.0000 max=1028 max_rank=5 min=1022 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8185 actual_valid_recv=8320 observed_loss=-135 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4090, 4096, 6, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1, 4092, 4094, 1, 0, 0, 1, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4091, 4092, 0, 0, 4, 4, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4084, 4096, 12, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1, 4094, 4094, 0, 0, 1, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4090, 4096, 6, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 4096, 1, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4078, 4096, 18, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 1025, 0, 6, 1018, 0, 1024, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1025, 1024, 1024, 1023, 1023, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 1025, 0, 1, 1019, 0, 1023, 0, 1024, 1, 1023, 3, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1025, 1020, 1023, 1024, 1024, 1026] avg=1024.0000 max=1026 max_rank=1 min=1020 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1027, 0, 1024, 0, 0, 1018, 0, 1023, 4, 1023, 4, 1023, 1, 1022] rerouted_tokens_per_ep_rank=[1023, 1027, 1024, 1018, 1023, 1027, 1027, 1023] avg=1024.0000 max=1027 max_rank=1 min=1018 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1027, 0, 1025, 0, 0, 1019, 0, 1023, 1, 1024, 2, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1027, 1025, 1019, 1023, 1025, 1026, 1023] avg=1024.0000 max=1027 max_rank=1 min=1019 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 1013, 0, 1018, 0, 6, 1023, 0, 1027, 0, 1027, 0, 1027, 0, 1026] rerouted_tokens_per_ep_rank=[1025, 1013, 1018, 1029, 1027, 1027, 1027, 1026] avg=1024.0000 max=1029 max_rank=3 min=1013 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4107 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1028, 0, 1025, 0, 1, 1019, 0, 1024, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1028, 1025, 1020, 1024, 1024, 1024, 1023] avg=1024.0000 max=1028 max_rank=1 min=1020 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 1025, 0, 12, 1017, 0, 1024, 0, 1022, 0, 1021, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1025, 1029, 1024, 1022, 1021, 1023] avg=1024.0000 max=1029 max_rank=3 min=1021 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1023, 0, 1025, 0, 18, 1015, 0, 1024, 0, 1020, 0, 1020, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1033, 1024, 1020, 1020, 1023] avg=1024.0000 max=1033 max_rank=3 min=1020 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8192, 8192, 8190] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8190 actual_valid_recv=8320 observed_loss=-130 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 4092, 0, 4094, 2, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 6, 4090, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 1, 4091, 0, 4092, 4, 4] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 4095, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 4094, 2, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 18, 4078, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 1, 1022, 1024, 0, 0, 1021, 1027, 0, 2, 1023, 2, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1024, 1021, 1027, 1025, 1025] avg=1024.0000 max=1027 max_rank=5 min=1021 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1023, 1, 1021, 1025, 0, 0, 1020, 1026, 0, 4, 1023, 4, 1023] rerouted_tokens_per_ep_rank=[1022, 1023, 1022, 1025, 1020, 1026, 1027, 1027] avg=1024.0000 max=1027 max_rank=6 min=1020 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1025, 0, 6, 1020, 1026, 0, 0, 1022, 0, 1022] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1025, 1026, 1026, 1022, 1022] avg=1024.0000 max=1026 max_rank=4 min=1022 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 0, 1027, 0, 1025, 1017, 0, 0, 1026, 1016, 0, 0, 1027, 0, 1027] rerouted_tokens_per_ep_rank=[1027, 1027, 1025, 1017, 1026, 1016, 1027, 1027] avg=1024.0000 max=1027 max_rank=0 min=1016 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1022, 1026, 0, 0, 1021, 1027, 0, 2, 1023, 2, 1023] rerouted_tokens_per_ep_rank=[1023, 1023, 1022, 1026, 1021, 1027, 1025, 1025] avg=1024.0000 max=1027 max_rank=5 min=1021 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1025, 0, 18, 1017, 1023, 0, 0, 1019, 0, 1019] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1025, 1035, 1023, 1019, 1019] avg=1024.0000 max=1035 max_rank=4 min=1019 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1025, 0, 1, 1021, 1028, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1025, 1022, 1028, 1023, 1023] avg=1024.0000 max=1028 max_rank=5 min=1022 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1021, 1027, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1025, 1021, 1027, 1024, 1024] avg=1024.0000 max=1027 max_rank=5 min=1021 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8184, 8192, 8192, 8200, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8200 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8184, 8192, 8192, 8200, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8184, 8192, 8192, 8200, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8184, 8192, 8192, 8200, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8184, 8192, 8192, 8200, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8184, 8192, 8192, 8200, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8184, 8192, 8192, 8200, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8184 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8184, 8192, 8192, 8200, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8184 actual_valid_recv=8320 observed_loss=-136 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8200 actual_valid_recv=8320 observed_loss=-120 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[419, 3677, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[523, 3571, 0, 4094, 2, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[263, 3831, 2, 4094, 0, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[937, 3159, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[855, 3237, 3, 4091, 1, 0, 5, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[2652, 1444, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[707, 3389, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1034, 3062, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[419, 116, 1189, 0, 0, 1186, 1025, 0, 0, 1023, 0, 1024, 0, 1186, 0, 1024] rerouted_tokens_per_ep_rank=[535, 1189, 1186, 1025, 1023, 1024, 1186, 1024] avg=1024.0000 max=1189 max_rank=1 min=535 min_rank=0 imbalance_ratio=1.161133 max_load_reduction=7003 max_load_reduction_pct=0.854858 moved_to_echo=4257 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[263, 121, 1239, 0, 2, 1236, 1026, 0, 0, 1022, 0, 1023, 2, 1235, 0, 1023] rerouted_tokens_per_ep_rank=[384, 1239, 1238, 1026, 1022, 1023, 1237, 1023] avg=1024.0000 max=1239 max_rank=1 min=384 min_rank=0 imbalance_ratio=1.209961 max_load_reduction=6953 max_load_reduction_pct=0.848755 moved_to_echo=4305 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 112, 1156, 0, 0, 1152, 1026, 0, 2, 1022, 0, 1023, 2, 1151, 0, 1023] rerouted_tokens_per_ep_rank=[635, 1156, 1152, 1026, 1024, 1023, 1153, 1023] avg=1024.0000 max=1156 max_rank=1 min=635 min_rank=0 imbalance_ratio=1.128906 max_load_reduction=7036 max_load_reduction_pct=0.858887 moved_to_echo=4223 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[855, 102, 1047, 0, 3, 1044, 1024, 0, 1, 1021, 0, 1023, 5, 1044, 0, 1023] rerouted_tokens_per_ep_rank=[957, 1047, 1047, 1024, 1022, 1023, 1049, 1023] avg=1024.0000 max=1049 max_rank=6 min=957 min_rank=0 imbalance_ratio=1.024414 max_load_reduction=7143 max_load_reduction_pct=0.871948 moved_to_echo=4117 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2652, 49, 457, 0, 0, 469, 1016, 0, 0, 1026, 0, 1027, 0, 469, 0, 1027] rerouted_tokens_per_ep_rank=[2701, 457, 469, 1016, 1026, 1027, 469, 1027] avg=1024.0000 max=2701 max_rank=0 min=457 min_rank=1 imbalance_ratio=2.637695 max_load_reduction=5491 max_load_reduction_pct=0.670288 moved_to_echo=3549 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[707, 107, 1096, 0, 0, 1093, 1025, 0, 0, 1023, 0, 1024, 0, 1093, 0, 1024] rerouted_tokens_per_ep_rank=[814, 1096, 1093, 1025, 1023, 1024, 1093, 1024] avg=1024.0000 max=1096 max_rank=1 min=814 min_rank=0 imbalance_ratio=1.070312 max_load_reduction=7096 max_load_reduction_pct=0.866211 moved_to_echo=4164 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[937, 99, 1023, 0, 0, 1019, 1025, 0, 0, 1023, 0, 1024, 0, 1018, 0, 1024] rerouted_tokens_per_ep_rank=[1036, 1023, 1019, 1025, 1023, 1024, 1018, 1024] avg=1024.0000 max=1036 max_rank=0 min=1018 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1034, 96, 991, 0, 0, 988, 1025, 0, 0, 1023, 0, 1024, 0, 987, 0, 1024] rerouted_tokens_per_ep_rank=[1130, 991, 988, 1025, 1023, 1024, 987, 1024] avg=1024.0000 max=1130 max_rank=0 min=987 min_rank=6 imbalance_ratio=1.103516 max_load_reduction=7062 max_load_reduction_pct=0.862061 moved_to_echo=4058 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8186, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8186, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8186, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8186, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8198 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8186, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8186, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8186, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8186 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8186, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[7424, 896] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8186 actual_valid_recv=8320 observed_loss=-134 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8198 actual_valid_recv=8320 observed_loss=-122 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4091, 0, 3, 0, 5, 0, 4090, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4094, 0, 1, 1, 2, 0, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4094, 0, 1, 1, 2, 0, 4093, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1023, 3, 1022, 0, 1021, 5, 1021, 0, 1023, 1025, 0, 3, 1022] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1021, 1026, 1023, 1025, 1025] avg=1024.0000 max=1026 max_rank=4 min=1021 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1024, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1023, 1024, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1024, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1023, 1024, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1, 1022, 1, 1022, 2, 1022, 0, 1023, 1026, 0, 1, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1023, 1024, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1, 1023, 1, 1022, 2, 1022, 0, 1023, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1024, 1023, 1024, 1023, 1026, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 0, 1027, 0, 1028, 0, 1026, 0, 1026, 0, 1027, 1014, 0, 0, 1028] rerouted_tokens_per_ep_rank=[1016, 1027, 1028, 1026, 1026, 1027, 1014, 1028] avg=1024.0000 max=1028 max_rank=2 min=1014 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1024, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1023, 1024, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1024, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1023, 1024, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8185 actual_valid_recv=8320 observed_loss=-135 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 0, 2, 4094, 4094, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 5, 1, 4, 4090, 4091, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 2, 0, 4093, 4094, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 2, 1022, 0, 1023, 2, 1023, 1026, 0, 1026, 0, 0, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1026, 1026, 1022, 1023] avg=1024.0000 max=1026 max_rank=4 min=1022 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1027, 0, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1027, 1025, 1023, 1024] avg=1024.0000 max=1027 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 5, 1021, 1, 1022, 4, 1022, 1025, 0, 1024, 0, 1, 1021, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1026, 1025, 1024, 1022, 1023] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 2, 1022, 2, 1023, 0, 1022, 1026, 0, 1026, 0, 1, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1025, 1022, 1026, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=4 min=1022 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 0, 1026, 0, 1029, 0, 1027, 1014, 0, 1016, 0, 0, 1026, 0, 1027] rerouted_tokens_per_ep_rank=[1027, 1026, 1029, 1027, 1014, 1016, 1026, 1027] avg=1024.0000 max=1029 max_rank=2 min=1014 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1027, 0, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1027, 1025, 1023, 1024] avg=1024.0000 max=1027 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1027, 0, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1027, 1025, 1023, 1024] avg=1024.0000 max=1027 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1027, 0, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1027, 1025, 1023, 1024] avg=1024.0000 max=1027 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8185 actual_valid_recv=8320 observed_loss=-135 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[2, 0, 3, 4092, 1, 1, 0, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[2, 0, 2, 4094, 0, 0, 0, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[5, 0, 6, 4090, 0, 2, 0, 4089] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1022, 1027, 0, 0, 1022, 0, 1024, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1022, 1027, 1022, 1024, 1024, 1026] avg=1024.0000 max=1027 max_rank=3 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 1022, 0, 1023, 3, 1022, 1025, 0, 1, 1022, 1, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1025, 1023, 1024, 1023, 1025] avg=1024.0000 max=1025 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 1028, 0, 1028, 1017, 0, 0, 1027, 0, 1025, 0, 1028, 1013, 0] rerouted_tokens_per_ep_rank=[1026, 1028, 1028, 1017, 1027, 1025, 1028, 1013] avg=1024.0000 max=1028 max_rank=1 min=1013 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[5, 1021, 0, 1022, 6, 1021, 1026, 0, 0, 1021, 2, 1022, 0, 1022, 1024, 0] rerouted_tokens_per_ep_rank=[1026, 1022, 1027, 1026, 1021, 1024, 1022, 1024] avg=1024.0000 max=1027 max_rank=2 min=1021 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 1022, 0, 1023, 2, 1022, 1027, 0, 0, 1022, 0, 1023, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1027, 1022, 1023, 1023, 1026] avg=1024.0000 max=1027 max_rank=3 min=1022 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1022, 1027, 0, 0, 1022, 0, 1024, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1022, 1027, 1022, 1024, 1024, 1026] avg=1024.0000 max=1027 max_rank=3 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1022, 1027, 0, 0, 1022, 0, 1024, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1022, 1027, 1022, 1024, 1024, 1026] avg=1024.0000 max=1027 max_rank=3 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1022, 1027, 0, 0, 1022, 0, 1024, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1022, 1027, 1022, 1024, 1024, 1026] avg=1024.0000 max=1027 max_rank=3 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8203, 8181, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8203 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8203, 8181, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8181 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8203, 8181, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8203, 8181, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8203, 8181, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8203, 8181, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8203, 8181, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8203, 8181, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8203 actual_valid_recv=8320 observed_loss=-117 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8181 actual_valid_recv=8320 observed_loss=-139 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4094, 0, 2, 0, 2, 0, 0, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4093, 0, 3, 0, 2, 1, 0, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4090, 0, 6, 0, 5, 0, 1, 4090] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1024, 0, 1022, 0, 1024, 0, 1023, 0, 1024, 0, 1022, 1026, 0] rerouted_tokens_per_ep_rank=[1027, 1024, 1022, 1024, 1023, 1024, 1022, 1026] avg=1024.0000 max=1027 max_rank=0 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 3, 1022, 0, 1023, 2, 1022, 1, 1023, 0, 1022, 1025, 0] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1023, 1024, 1024, 1022, 1025] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1023, 2, 1022, 0, 1023, 2, 1022, 0, 1023, 0, 1022, 1026, 0] rerouted_tokens_per_ep_rank=[1027, 1023, 1024, 1023, 1024, 1023, 1022, 1026] avg=1024.0000 max=1027 max_rank=0 min=1022 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1022, 6, 1021, 0, 1022, 5, 1021, 0, 1022, 1, 1021, 1025, 0] rerouted_tokens_per_ep_rank=[1026, 1022, 1027, 1022, 1026, 1022, 1022, 1025] avg=1024.0000 max=1027 max_rank=2 min=1022 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1015, 0, 0, 1028, 0, 1028, 0, 1028, 0, 1026, 0, 1027, 0, 1028, 1012, 0] rerouted_tokens_per_ep_rank=[1015, 1028, 1028, 1028, 1026, 1027, 1028, 1012] avg=1024.0000 max=1028 max_rank=1 min=1012 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1024, 0, 1022, 0, 1024, 0, 1023, 0, 1024, 0, 1022, 1026, 0] rerouted_tokens_per_ep_rank=[1027, 1024, 1022, 1024, 1023, 1024, 1022, 1026] avg=1024.0000 max=1027 max_rank=0 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1024, 0, 1022, 0, 1024, 0, 1023, 0, 1024, 0, 1022, 1026, 0] rerouted_tokens_per_ep_rank=[1027, 1024, 1022, 1024, 1023, 1024, 1022, 1026] avg=1024.0000 max=1027 max_rank=0 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1024, 0, 1022, 0, 1024, 0, 1023, 0, 1024, 0, 1022, 1026, 0] rerouted_tokens_per_ep_rank=[1027, 1024, 1022, 1024, 1023, 1024, 1022, 1026] avg=1024.0000 max=1027 max_rank=0 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8202, 8192, 8192, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8202, 8192, 8192, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8202 expected_overflow=0 exact_recv_per_rank=[8202, 8192, 8192, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8202, 8192, 8192, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8202, 8192, 8192, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8182 expected_overflow=0 exact_recv_per_rank=[8202, 8192, 8192, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8202, 8192, 8192, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8202, 8192, 8192, 8192, 8192, 8192, 8182, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8202 actual_valid_recv=8320 observed_loss=-118 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8182 actual_valid_recv=8320 observed_loss=-138 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1, 3, 4093, 0, 4093, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1, 2, 4094, 0, 4094, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[2, 7, 4090, 0, 4089, 0, 4, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1022, 1026, 0, 0, 1024, 1028, 0, 0, 1022, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1022, 1026, 1024, 1028, 1022, 1023, 1024] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 3, 1022, 1025, 0, 0, 1023, 1026, 0, 0, 1022, 2, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1025, 1025, 1023, 1026, 1022, 1024, 1023] avg=1024.0000 max=1026 max_rank=4 min=1022 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 0, 1027, 1012, 0, 0, 1028, 1014, 0, 0, 1028, 0, 1027, 0, 1028] rerouted_tokens_per_ep_rank=[1028, 1027, 1012, 1028, 1014, 1028, 1027, 1028] avg=1024.0000 max=1028 max_rank=0 min=1012 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 1022, 7, 1021, 1025, 0, 0, 1022, 1025, 0, 0, 1021, 4, 1021, 0, 1022] rerouted_tokens_per_ep_rank=[1024, 1028, 1025, 1022, 1025, 1021, 1025, 1022] avg=1024.0000 max=1028 max_rank=1 min=1021 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 2, 1022, 1026, 0, 0, 1023, 1026, 0, 0, 1022, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1026, 1023, 1026, 1022, 1024, 1023] avg=1024.0000 max=1026 max_rank=2 min=1022 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1022, 1026, 0, 0, 1024, 1028, 0, 0, 1022, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1022, 1026, 1024, 1028, 1022, 1023, 1024] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1022, 1026, 0, 0, 1024, 1028, 0, 0, 1022, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1022, 1026, 1024, 1028, 1022, 1023, 1024] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1022, 1026, 0, 0, 1024, 1028, 0, 0, 1022, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1022, 1026, 1024, 1028, 1022, 1023, 1024] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8203, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8203 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8203, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8203, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8181 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8203, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8203, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8203, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8203, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8203, 8181, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8203 actual_valid_recv=8320 observed_loss=-117 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8181 actual_valid_recv=8192 observed_loss=-11 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4090, 1, 0, 7, 5, 4089, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 0, 2, 2, 4094, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4093, 1, 0, 3, 2, 4093, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1024, 0, 1022, 0, 1023, 1027, 0, 0, 1022, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1024, 1022, 1023, 1027, 1022, 1024] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1, 1022, 0, 1022, 7, 1021, 5, 1021, 1025, 0, 0, 1021, 0, 1022] rerouted_tokens_per_ep_rank=[1025, 1023, 1022, 1028, 1026, 1025, 1021, 1022] avg=1024.0000 max=1028 max_rank=3 min=1021 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1, 1023, 0, 1023, 3, 1022, 2, 1022, 1026, 0, 0, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1025, 1024, 1026, 1022, 1023] avg=1024.0000 max=1026 max_rank=5 min=1022 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 2, 1022, 2, 1022, 1027, 0, 0, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1024, 1024, 1027, 1022, 1023] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1012, 0, 0, 1026, 0, 1028, 0, 1027, 0, 1026, 1017, 0, 0, 1028, 0, 1028] rerouted_tokens_per_ep_rank=[1012, 1026, 1028, 1027, 1026, 1017, 1028, 1028] avg=1024.0000 max=1028 max_rank=2 min=1012 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1024, 0, 1022, 0, 1023, 1027, 0, 0, 1022, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1024, 1022, 1023, 1027, 1022, 1024] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1024, 0, 1022, 0, 1023, 1027, 0, 0, 1022, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1024, 1022, 1023, 1027, 1022, 1024] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1024, 0, 1022, 0, 1023, 1027, 0, 0, 1022, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1024, 1022, 1023, 1027, 1022, 1024] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8203, 8181, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8181 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8203, 8181, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8203, 8181, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8203, 8181, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8203, 8181, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8203, 8181, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8203 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8203, 8181, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8203, 8181, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8203 actual_valid_recv=8320 observed_loss=-117 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8181 actual_valid_recv=8192 observed_loss=-11 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4094, 3, 0, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 4093, 3, 0, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[5, 0, 0, 1, 4090, 7, 0, 4089] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1022, 1026, 0, 0, 1022, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1022, 1026, 1022, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 1022, 0, 1023, 0, 1023, 0, 1022, 1026, 0, 3, 1022, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1022, 1026, 1025, 1023, 1026] avg=1024.0000 max=1026 max_rank=4 min=1022 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[5, 1021, 0, 1022, 0, 1022, 1, 1021, 1025, 0, 7, 1021, 0, 1022, 1025, 0] rerouted_tokens_per_ep_rank=[1026, 1022, 1022, 1022, 1025, 1028, 1022, 1025] avg=1024.0000 max=1028 max_rank=5 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 1022, 1, 1023, 0, 1023, 0, 1022, 1025, 0, 3, 1022, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1022, 1025, 1025, 1023, 1026] avg=1024.0000 max=1026 max_rank=7 min=1022 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 1027, 0, 1028, 0, 1028, 1012, 0, 0, 1026, 0, 1028, 1017, 0] rerouted_tokens_per_ep_rank=[1026, 1027, 1028, 1028, 1012, 1026, 1028, 1017] avg=1024.0000 max=1028 max_rank=2 min=1012 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1022, 1026, 0, 0, 1022, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1022, 1026, 1022, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1022, 1026, 0, 0, 1022, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1022, 1026, 1022, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1022, 1026, 0, 0, 1022, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1022, 1026, 1022, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8182, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8182 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8182, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8182, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8182, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8182, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8202 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8182, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8182, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8182, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8202 actual_valid_recv=8320 observed_loss=-118 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8182 actual_valid_recv=8320 observed_loss=-138 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4094, 0, 0, 1, 0, 3, 4092, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 3, 4093, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4090, 0, 0, 1, 0, 9, 4086, 6] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 8, 0, 0, 0, 0, 4088, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1024, 0, 1022, 0, 1024, 0, 1022, 1028, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1024, 1022, 1024, 1022, 1028, 1023] avg=1024.0000 max=1028 max_rank=6 min=1022 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 0, 1023, 0, 1022, 0, 1023, 3, 1022, 1026, 0, 3, 1022] rerouted_tokens_per_ep_rank=[1025, 1023, 1023, 1022, 1023, 1025, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1022 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1021, 0, 1022, 1, 1021, 0, 1022, 9, 1020, 1025, 0, 6, 1020] rerouted_tokens_per_ep_rank=[1025, 1021, 1022, 1022, 1022, 1029, 1025, 1026] avg=1024.0000 max=1029 max_rank=5 min=1021 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1012, 0, 0, 1028, 0, 1028, 0, 1028, 0, 1028, 0, 1028, 1013, 0, 0, 1027] rerouted_tokens_per_ep_rank=[1012, 1028, 1028, 1028, 1028, 1028, 1013, 1027] avg=1024.0000 max=1028 max_rank=1 min=1012 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1022, 0, 1023, 1, 1022, 0, 1023, 3, 1021, 1027, 0, 2, 1022] rerouted_tokens_per_ep_rank=[1026, 1022, 1023, 1023, 1023, 1024, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1022 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1024, 0, 1022, 0, 1024, 0, 1022, 1028, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1024, 1022, 1024, 1022, 1028, 1023] avg=1024.0000 max=1028 max_rank=6 min=1022 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 8, 1021, 0, 1024, 0, 1022, 0, 1024, 0, 1020, 1026, 0, 0, 1021] rerouted_tokens_per_ep_rank=[1026, 1029, 1024, 1022, 1024, 1020, 1026, 1021] avg=1024.0000 max=1029 max_rank=1 min=1020 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1024, 0, 1022, 0, 1024, 0, 1022, 1028, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1024, 1022, 1024, 1022, 1028, 1023] avg=1024.0000 max=1028 max_rank=6 min=1022 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8183, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8183 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8183, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8201 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8183, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8183, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8183, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8183, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8183, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8183, 8192, 8192, 8201, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8201 actual_valid_recv=8320 observed_loss=-119 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8183 actual_valid_recv=8320 observed_loss=-137 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[126, 2, 0, 4079, 1, 3695, 289, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[11, 10, 1, 4084, 4, 3915, 167, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[8, 0, 0, 4095, 13, 3932, 144, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 2, 0, 4090, 2, 3887, 210, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[21, 2, 0, 4091, 1, 3895, 182, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[10, 0, 0, 4090, 0, 3978, 114, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 1, 4095, 0, 3961, 132, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 0, 4093, 15, 3910, 172, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[126, 949, 2, 1015, 0, 1021, 1022, 0, 1, 966, 977, 0, 289, 803, 0, 1021] rerouted_tokens_per_ep_rank=[1075, 1017, 1021, 1022, 967, 977, 1092, 1021] avg=1024.0000 max=1092 max_rank=6 min=967 min_rank=4 imbalance_ratio=1.066406 max_load_reduction=7100 max_load_reduction_pct=0.866699 moved_to_echo=4057 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 1006, 10, 1016, 1, 1022, 1024, 0, 4, 1024, 1034, 0, 167, 851, 0, 1022] rerouted_tokens_per_ep_rank=[1017, 1026, 1023, 1024, 1028, 1034, 1018, 1022] avg=1024.0000 max=1034 max_rank=5 min=1017 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 999, 2, 1018, 0, 1023, 1026, 0, 2, 1016, 1027, 0, 210, 845, 1, 1023] rerouted_tokens_per_ep_rank=[999, 1020, 1023, 1026, 1018, 1027, 1055, 1024] avg=1024.0000 max=1055 max_rank=6 min=999 min_rank=0 imbalance_ratio=1.030273 max_load_reduction=7137 max_load_reduction_pct=0.871216 moved_to_echo=4124 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[8, 1013, 0, 1023, 0, 1028, 1016, 0, 13, 1033, 1026, 0, 144, 860, 0, 1028] rerouted_tokens_per_ep_rank=[1021, 1023, 1028, 1016, 1046, 1026, 1004, 1028] avg=1024.0000 max=1046 max_rank=4 min=1004 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=7146 max_load_reduction_pct=0.872314 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 1004, 0, 1018, 0, 1024, 1027, 0, 15, 1022, 1034, 0, 172, 850, 1, 1024] rerouted_tokens_per_ep_rank=[1005, 1018, 1024, 1027, 1037, 1034, 1022, 1025] avg=1024.0000 max=1037 max_rank=4 min=1005 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=7155 max_load_reduction_pct=0.873413 moved_to_echo=4118 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 1018, 0, 1019, 1, 1025, 1026, 0, 0, 1036, 1046, 0, 132, 861, 0, 1025] rerouted_tokens_per_ep_rank=[1021, 1019, 1026, 1026, 1036, 1046, 993, 1025] avg=1024.0000 max=1046 max_rank=5 min=993 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=7146 max_load_reduction_pct=0.872314 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 1022, 0, 1018, 0, 1023, 1026, 0, 0, 1040, 1051, 0, 114, 865, 0, 1023] rerouted_tokens_per_ep_rank=[1032, 1018, 1023, 1026, 1040, 1051, 979, 1023] avg=1024.0000 max=1051 max_rank=5 min=979 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=7141 max_load_reduction_pct=0.871704 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[21, 1001, 2, 1018, 0, 1024, 1025, 0, 1, 1019, 1028, 0, 182, 847, 0, 1024] rerouted_tokens_per_ep_rank=[1022, 1020, 1024, 1025, 1020, 1028, 1029, 1024] avg=1024.0000 max=1029 max_rank=6 min=1020 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8161, 8192, 8192, 8192, 8223, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8161, 8192, 8192, 8192, 8223, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8161, 8192, 8192, 8192, 8223, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8161, 8192, 8192, 8192, 8223, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8161 expected_overflow=0 exact_recv_per_rank=[8192, 8161, 8192, 8192, 8192, 8223, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8161, 8192, 8192, 8192, 8223, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8223 expected_overflow=0 exact_recv_per_rank=[8192, 8161, 8192, 8192, 8192, 8223, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8161, 8192, 8192, 8192, 8223, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8223 actual_valid_recv=8320 observed_loss=-97 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8161 actual_valid_recv=8320 observed_loss=-159 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1536, 6784] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3, 1, 17, 1, 0, 4095, 0, 4075] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 8, 0, 0, 4095, 0, 4088] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2, 2, 15, 4, 0, 4085, 2, 4082] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1, 1, 1, 2, 0, 4094, 1, 4092] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 19, 0, 0, 4095, 0, 4077] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 4096, 0, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 4096, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[12, 0, 4, 0, 0, 4096, 0, 4080] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 1018, 1, 1022, 17, 1013, 1, 1020, 0, 1024, 1026, 0, 0, 1023, 1024, 0] rerouted_tokens_per_ep_rank=[1021, 1023, 1030, 1021, 1024, 1026, 1023, 1024] avg=1024.0000 max=1030 max_rank=2 min=1021 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1021, 0, 1022, 8, 1016, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 1028, 0] rerouted_tokens_per_ep_rank=[1022, 1022, 1024, 1023, 1024, 1026, 1023, 1028] avg=1024.0000 max=1028 max_rank=7 min=1022 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 1020, 2, 1020, 15, 1015, 4, 1022, 0, 1021, 1023, 0, 2, 1021, 1025, 0] rerouted_tokens_per_ep_rank=[1022, 1022, 1030, 1026, 1021, 1023, 1023, 1025] avg=1024.0000 max=1030 max_rank=2 min=1021 min_rank=4 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1027, 19, 1017, 0, 1025, 0, 1027, 1014, 0, 0, 1027, 1012, 0] rerouted_tokens_per_ep_rank=[1024, 1027, 1036, 1025, 1027, 1014, 1027, 1012] avg=1024.0000 max=1036 max_rank=2 min=1012 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4080 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1022, 1, 1022, 1, 1017, 2, 1024, 0, 1024, 1025, 0, 1, 1023, 1029, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1018, 1026, 1024, 1025, 1024, 1029] avg=1024.0000 max=1029 max_rank=7 min=1018 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 1023, 0, 1022, 0, 1018, 0, 1025, 0, 1024, 1026, 0, 0, 1024, 1028, 0] rerouted_tokens_per_ep_rank=[1025, 1022, 1018, 1025, 1024, 1026, 1024, 1028] avg=1024.0000 max=1028 max_rank=7 min=1018 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1022, 0, 1018, 0, 1025, 0, 1024, 1026, 0, 0, 1024, 1029, 0] rerouted_tokens_per_ep_rank=[1024, 1022, 1018, 1025, 1024, 1026, 1024, 1029] avg=1024.0000 max=1029 max_rank=7 min=1018 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[12, 1019, 0, 1022, 4, 1014, 0, 1021, 0, 1024, 1026, 0, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1031, 1022, 1018, 1021, 1024, 1026, 1024, 1026] avg=1024.0000 max=1031 max_rank=0 min=1018 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8192, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8192, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8192, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8192, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8192, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8192, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8183 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8192, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8201 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8192, 8192, 8201] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8183 actual_valid_recv=8320 observed_loss=-137 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8201 actual_valid_recv=8320 observed_loss=-119 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3959, 0, 0, 0, 141, 0, 4090, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3972, 0, 1, 0, 123, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3955, 1, 5, 0, 144, 0, 4087, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3884, 0, 16, 0, 196, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3973, 0, 6, 0, 120, 3, 4089, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3969, 0, 4, 0, 123, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3980, 0, 0, 0, 116, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3951, 0, 7, 0, 138, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 1, 1022, 5, 1019, 0, 1022, 144, 886, 0, 1023, 1024, 0, 0, 1019] rerouted_tokens_per_ep_rank=[1027, 1023, 1024, 1022, 1030, 1023, 1024, 1019] avg=1024.0000 max=1030 max_rank=4 min=1019 min_rank=7 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 0, 1023, 0, 1020, 0, 1023, 141, 887, 0, 1024, 1024, 0, 2, 1020] rerouted_tokens_per_ep_rank=[1028, 1023, 1020, 1023, 1028, 1024, 1024, 1022] avg=1024.0000 max=1028 max_rank=0 min=1020 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1009, 0, 0, 1024, 16, 1000, 0, 1024, 196, 870, 0, 1005, 1026, 0, 0, 1022] rerouted_tokens_per_ep_rank=[1009, 1024, 1016, 1024, 1066, 1005, 1026, 1022] avg=1024.0000 max=1066 max_rank=4 min=1005 min_rank=5 imbalance_ratio=1.041016 max_load_reduction=7126 max_load_reduction_pct=0.869873 moved_to_echo=4119 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1021, 0, 0, 1028, 1, 1027, 0, 1029, 123, 893, 0, 1031, 1015, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1021, 1028, 1028, 1029, 1016, 1031, 1015, 1024] avg=1024.0000 max=1031 max_rank=5 min=1015 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4086 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1032, 0, 0, 1022, 6, 1023, 0, 1022, 120, 890, 3, 1028, 1025, 0, 1, 1020] rerouted_tokens_per_ep_rank=[1032, 1022, 1029, 1022, 1010, 1031, 1025, 1021] avg=1024.0000 max=1032 max_rank=0 min=1010 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1031, 0, 0, 1024, 4, 1022, 0, 1024, 123, 889, 0, 1027, 1026, 0, 0, 1022] rerouted_tokens_per_ep_rank=[1031, 1024, 1026, 1024, 1012, 1027, 1026, 1022] avg=1024.0000 max=1031 max_rank=0 min=1012 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1035, 0, 0, 1024, 0, 1025, 0, 1024, 116, 891, 0, 1029, 1026, 0, 0, 1022] rerouted_tokens_per_ep_rank=[1035, 1024, 1025, 1024, 1007, 1029, 1026, 1022] avg=1024.0000 max=1035 max_rank=0 min=1007 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4084 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1024, 7, 1017, 0, 1024, 138, 885, 0, 1022, 1026, 0, 0, 1022] rerouted_tokens_per_ep_rank=[1027, 1024, 1024, 1024, 1023, 1022, 1026, 1022] avg=1024.0000 max=1027 max_rank=0 min=1022 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8210, 8192, 8192, 8192, 8192, 8192, 8192, 8174] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8210, 8192, 8192, 8192, 8192, 8192, 8192, 8174] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8210, 8192, 8192, 8192, 8192, 8192, 8192, 8174] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8210 expected_overflow=0 exact_recv_per_rank=[8210, 8192, 8192, 8192, 8192, 8192, 8192, 8174] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8210, 8192, 8192, 8192, 8192, 8192, 8192, 8174] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8210, 8192, 8192, 8192, 8192, 8192, 8192, 8174] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8210, 8192, 8192, 8192, 8192, 8192, 8192, 8174] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8174 expected_overflow=0 exact_recv_per_rank=[8210, 8192, 8192, 8192, 8192, 8192, 8192, 8174] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8210 actual_valid_recv=8320 observed_loss=-110 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8174 actual_valid_recv=8320 observed_loss=-146 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1152, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[3974, 0, 4093, 0, 1, 120, 2, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[3969, 0, 4094, 1, 2, 124, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[3986, 0, 4096, 0, 0, 110, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[3957, 0, 4096, 0, 0, 139, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[3955, 0, 4096, 0, 0, 141, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[3964, 0, 4096, 0, 0, 132, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3997, 0, 4096, 0, 0, 99, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[3965, 0, 4096, 0, 0, 131, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 0, 1023, 1, 1024, 120, 900, 2, 1024, 2, 1022] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1023, 1025, 1020, 1026, 1024] avg=1024.0000 max=1026 max_rank=0 min=1020 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1023, 1025, 0, 1, 1023, 2, 1023, 124, 899, 1, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1025 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1022, 0, 0, 1024, 1025, 0, 0, 1024, 0, 1019, 141, 895, 0, 1019, 0, 1023] rerouted_tokens_per_ep_rank=[1022, 1024, 1025, 1024, 1019, 1036, 1019, 1023] avg=1024.0000 max=1036 max_rank=5 min=1019 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1021, 0, 0, 1024, 1025, 0, 0, 1024, 0, 1020, 139, 896, 0, 1020, 0, 1023] rerouted_tokens_per_ep_rank=[1021, 1024, 1025, 1024, 1020, 1035, 1020, 1023] avg=1024.0000 max=1035 max_rank=5 min=1020 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1020, 0, 0, 1026, 1017, 0, 0, 1025, 0, 1030, 110, 906, 0, 1030, 0, 1028] rerouted_tokens_per_ep_rank=[1020, 1026, 1017, 1025, 1030, 1016, 1030, 1028] avg=1024.0000 max=1030 max_rank=4 min=1016 min_rank=5 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 1025, 0, 0, 1024, 0, 1021, 132, 897, 0, 1021, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1025, 1024, 1021, 1029, 1021, 1023] avg=1024.0000 max=1029 max_rank=5 min=1021 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1032, 0, 0, 1024, 1025, 0, 0, 1024, 0, 1030, 99, 905, 0, 1030, 0, 1023] rerouted_tokens_per_ep_rank=[1032, 1024, 1025, 1024, 1030, 1004, 1030, 1023] avg=1024.0000 max=1032 max_rank=0 min=1004 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1023, 0, 0, 1024, 1025, 0, 0, 1024, 0, 1022, 131, 898, 0, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1025, 1024, 1022, 1029, 1022, 1023] avg=1024.0000 max=1029 max_rank=5 min=1022 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8192, 8192, 8192, 8191] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8192, 8192, 8192, 8191] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8192, 8192, 8192, 8191] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8192, 8192, 8192, 8191] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8192, 8192, 8192, 8191] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8192, 8192, 8192, 8191] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8192, 8192, 8192, 8191] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8192, 8192, 8192, 8191] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1024, 7296] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8191 actual_valid_recv=8320 observed_loss=-129 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 4093, 1, 2, 4093, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 4093, 4, 2, 4090, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[5, 0, 4096, 0, 0, 4091, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 4096, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 2, 1023, 1025, 0, 1, 1023, 2, 1022, 1025, 0, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1022, 1025, 1025, 1024, 1024, 1025, 1024, 1023] avg=1024.0000 max=1025 max_rank=1 min=1022 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1024, 1026, 1023, 1023, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 2, 1022, 1025, 0, 4, 1022, 2, 1022, 1024, 0, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1022, 1024, 1025, 1026, 1024, 1024, 1024, 1023] avg=1024.0000 max=1026 max_rank=3 min=1022 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 1022, 0, 1022, 1026, 0, 0, 1022, 0, 1023, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1022, 1026, 1022, 1023, 1025, 1023, 1024] avg=1024.0000 max=1027 max_rank=0 min=1022 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 0, 1026, 1012, 0, 0, 1028, 0, 1029, 1015, 0, 0, 1029, 0, 1026] rerouted_tokens_per_ep_rank=[1027, 1026, 1012, 1028, 1029, 1015, 1029, 1026] avg=1024.0000 max=1029 max_rank=4 min=1012 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 1023, 0, 1023, 1026, 0, 0, 1023, 0, 1023, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1023, 1026, 1023, 1023, 1025, 1023, 1024] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1024, 1026, 1023, 1023, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1024, 1026, 1023, 1023, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1, 4092, 4093, 0, 1, 1, 1, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4082, 4096, 14, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 4096, 1, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4091, 4094, 0, 2, 0, 2, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1025, 0, 1024, 0, 0, 1021, 1, 1023, 1, 1023, 1, 1023, 3, 1023] rerouted_tokens_per_ep_rank=[1024, 1025, 1024, 1021, 1024, 1024, 1024, 1026] avg=1024.0000 max=1026 max_rank=7 min=1021 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 1025, 0, 0, 1022, 0, 1023, 0, 1024, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1026, 1025, 1022, 1023, 1024, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1023, 0, 1025, 0, 14, 1019, 0, 1023, 0, 1024, 0, 1020, 0, 1020] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1033, 1023, 1024, 1020, 1020] avg=1024.0000 max=1033 max_rank=3 min=1020 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 1015, 0, 1018, 0, 1, 1028, 0, 1028, 0, 1025, 0, 1027, 0, 1025] rerouted_tokens_per_ep_rank=[1025, 1015, 1018, 1029, 1028, 1025, 1027, 1025] avg=1024.0000 max=1029 max_rank=3 min=1015 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 1025, 0, 0, 1022, 0, 1023, 0, 1024, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1026, 1025, 1022, 1023, 1024, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 1025, 0, 0, 1021, 2, 1023, 0, 1023, 2, 1023, 3, 1022] rerouted_tokens_per_ep_rank=[1023, 1025, 1025, 1021, 1025, 1023, 1025, 1025] avg=1024.0000 max=1025 max_rank=1 min=1021 min_rank=3 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 1025, 0, 0, 1022, 0, 1023, 0, 1024, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1026, 1025, 1022, 1023, 1024, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 1025, 0, 0, 1022, 0, 1023, 0, 1024, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1026, 1025, 1022, 1023, 1024, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 1, 4093, 0, 4093, 2, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 1, 4094, 0, 4093, 2, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 12, 4084, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1, 1022, 1025, 0, 0, 1022, 1025, 0, 2, 1023, 3, 1023] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1025, 1022, 1025, 1025, 1026] avg=1024.0000 max=1026 max_rank=7 min=1022 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 1026, 0, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1025, 1023, 1026, 1024, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1025, 0, 12, 1020, 1023, 0, 0, 1021, 0, 1020] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1025, 1032, 1023, 1021, 1020] avg=1024.0000 max=1032 max_rank=4 min=1020 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1, 1023, 1025, 0, 0, 1022, 1025, 0, 2, 1023, 2, 1023] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1025, 1022, 1025, 1025, 1025] avg=1024.0000 max=1025 max_rank=3 min=1022 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 1026, 0, 1027, 1017, 0, 0, 1024, 1018, 0, 0, 1025, 0, 1029] rerouted_tokens_per_ep_rank=[1026, 1026, 1027, 1017, 1024, 1018, 1025, 1029] avg=1024.0000 max=1029 max_rank=7 min=1017 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 1026, 0, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1025, 1023, 1026, 1024, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 1026, 0, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1025, 1023, 1026, 1024, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 1026, 0, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1025, 1023, 1026, 1024, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8195, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8189 actual_valid_recv=8320 observed_loss=-131 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[315, 3781, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[617, 3477, 2, 4093, 0, 0, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[354, 3742, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[518, 3575, 3, 4093, 0, 0, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1385, 2711, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[713, 3383, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[507, 3589, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[650, 3446, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[315, 427, 1120, 0, 0, 1117, 1025, 0, 0, 1024, 0, 1023, 0, 1117, 0, 1024] rerouted_tokens_per_ep_rank=[742, 1120, 1117, 1025, 1024, 1023, 1117, 1024] avg=1024.0000 max=1120 max_rank=1 min=742 min_rank=0 imbalance_ratio=1.093750 max_load_reduction=7072 max_load_reduction_pct=0.863281 moved_to_echo=4188 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[354, 427, 1098, 0, 0, 1109, 1017, 0, 0, 1026, 0, 1027, 0, 1108, 0, 1026] rerouted_tokens_per_ep_rank=[781, 1098, 1109, 1017, 1026, 1027, 1108, 1026] avg=1024.0000 max=1109 max_rank=2 min=781 min_rank=0 imbalance_ratio=1.083008 max_load_reduction=7083 max_load_reduction_pct=0.864624 moved_to_echo=4187 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[518, 404, 1059, 0, 3, 1056, 1025, 0, 0, 1023, 0, 1022, 3, 1056, 0, 1023] rerouted_tokens_per_ep_rank=[922, 1059, 1059, 1025, 1023, 1022, 1059, 1023] avg=1024.0000 max=1059 max_rank=1 min=922 min_rank=0 imbalance_ratio=1.034180 max_load_reduction=7133 max_load_reduction_pct=0.870728 moved_to_echo=4127 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[617, 393, 1030, 0, 2, 1027, 1025, 0, 0, 1023, 0, 1022, 3, 1027, 0, 1023] rerouted_tokens_per_ep_rank=[1010, 1030, 1029, 1025, 1023, 1022, 1030, 1023] avg=1024.0000 max=1030 max_rank=1 min=1010 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1385, 306, 803, 0, 0, 801, 1025, 0, 0, 1024, 0, 1023, 0, 801, 0, 1024] rerouted_tokens_per_ep_rank=[1691, 803, 801, 1025, 1024, 1023, 801, 1024] avg=1024.0000 max=1691 max_rank=0 min=801 min_rank=2 imbalance_ratio=1.651367 max_load_reduction=6501 max_load_reduction_pct=0.793579 moved_to_echo=3872 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[713, 382, 1003, 0, 0, 999, 1025, 0, 0, 1024, 0, 1023, 0, 999, 0, 1024] rerouted_tokens_per_ep_rank=[1095, 1003, 999, 1025, 1024, 1023, 999, 1024] avg=1024.0000 max=1095 max_rank=0 min=999 min_rank=2 imbalance_ratio=1.069336 max_load_reduction=7097 max_load_reduction_pct=0.866333 moved_to_echo=4070 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[507, 405, 1064, 0, 0, 1060, 1025, 0, 0, 1024, 0, 1023, 0, 1060, 0, 1024] rerouted_tokens_per_ep_rank=[912, 1064, 1060, 1025, 1024, 1023, 1060, 1024] avg=1024.0000 max=1064 max_rank=1 min=912 min_rank=0 imbalance_ratio=1.039062 max_load_reduction=7128 max_load_reduction_pct=0.870117 moved_to_echo=4131 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[650, 389, 1021, 0, 0, 1018, 1025, 0, 0, 1024, 0, 1023, 0, 1018, 0, 1024] rerouted_tokens_per_ep_rank=[1039, 1021, 1018, 1025, 1024, 1023, 1018, 1024] avg=1024.0000 max=1039 max_rank=0 min=1018 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=7153 max_load_reduction_pct=0.873169 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8186 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8198 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8186, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8198 actual_valid_recv=8320 observed_loss=-122 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5120, 3200] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8186 actual_valid_recv=8192 observed_loss=-6 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4093, 0, 2, 0, 3, 0, 4093, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4093, 0, 2, 0, 3, 0, 4092, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1024, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1023, 1024, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1024, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1023, 1024, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 2, 1022, 0, 1022, 3, 1022, 0, 1023, 1026, 0, 2, 1022] rerouted_tokens_per_ep_rank=[1025, 1023, 1024, 1022, 1025, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1022 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1017, 0, 0, 1026, 0, 1029, 0, 1027, 0, 1027, 0, 1026, 1011, 0, 0, 1029] rerouted_tokens_per_ep_rank=[1017, 1026, 1029, 1027, 1027, 1026, 1011, 1029] avg=1024.0000 max=1029 max_rank=2 min=1011 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 2, 1022, 0, 1022, 3, 1022, 0, 1023, 1026, 0, 1, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1024, 1022, 1025, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1022 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1024, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1023, 1024, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1024, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1023, 1024, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1024, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1023, 1024, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8186 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8186, 8192, 8192, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8186, 8192, 8192, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8186, 8192, 8192, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8186, 8192, 8192, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8186, 8192, 8192, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8186, 8192, 8192, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8186, 8192, 8192, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8198 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8186, 8192, 8192, 8198, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8198 actual_valid_recv=8320 observed_loss=-122 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8186 actual_valid_recv=8192 observed_loss=-6 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 3, 2, 1, 4092, 4093, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 3, 2, 1, 4093, 4093, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1027, 0, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1027, 1025, 1023, 1024] avg=1024.0000 max=1027 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1027, 0, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1027, 1025, 1023, 1024] avg=1024.0000 max=1027 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 3, 1022, 2, 1022, 1, 1022, 1026, 0, 1025, 0, 1, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1023, 1026, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 3, 1022, 2, 1022, 1, 1023, 1026, 0, 1025, 0, 0, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1024, 1026, 1025, 1022, 1023] avg=1024.0000 max=1026 max_rank=4 min=1022 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1027, 0, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1027, 1025, 1023, 1024] avg=1024.0000 max=1027 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 1027, 0, 1029, 0, 1030, 1010, 0, 1017, 0, 0, 1027, 0, 1026] rerouted_tokens_per_ep_rank=[1026, 1027, 1029, 1030, 1010, 1017, 1027, 1026] avg=1024.0000 max=1030 max_rank=3 min=1010 min_rank=4 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4080 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1027, 0, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1027, 1025, 1023, 1024] avg=1024.0000 max=1027 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1027, 0, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1027, 1025, 1023, 1024] avg=1024.0000 max=1027 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8197 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8197, 8192, 8187, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8197, 8192, 8187, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8197, 8192, 8187, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8197, 8192, 8187, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8197, 8192, 8187, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8197, 8192, 8187, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8197, 8192, 8187, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8187 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8197, 8192, 8187, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8197 actual_valid_recv=8320 observed_loss=-123 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8187 actual_valid_recv=8320 observed_loss=-133 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[3, 0, 3, 4093, 0, 2, 0, 4091] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[3, 0, 2, 4093, 0, 1, 0, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 1022, 0, 1023, 3, 1022, 1025, 0, 0, 1022, 2, 1022, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1025, 1023, 1025, 1025, 1022, 1024, 1023, 1025] avg=1024.0000 max=1025 max_rank=0 min=1022 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1023, 1023, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 1022, 0, 1023, 2, 1022, 1025, 0, 0, 1022, 1, 1023, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1025, 1023, 1024, 1025, 1022, 1024, 1023, 1026] avg=1024.0000 max=1026 max_rank=7 min=1022 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1023, 1023, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1023, 1023, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 0, 1026, 0, 1028, 1017, 0, 0, 1027, 0, 1029, 0, 1026, 1012, 0] rerouted_tokens_per_ep_rank=[1027, 1026, 1028, 1017, 1027, 1029, 1026, 1012] avg=1024.0000 max=1029 max_rank=5 min=1012 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1023, 1023, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1025, 1023, 1023, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 8198] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 8198] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 8198] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 8198] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8198 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 8198] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8186 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 8198] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 8198] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8186, 8192, 8192, 8198] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8198 actual_valid_recv=8320 observed_loss=-122 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8186 actual_valid_recv=8192 observed_loss=-6 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4093, 0, 3, 0, 3, 0, 0, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4092, 0, 3, 0, 4, 0, 0, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1017, 0, 0, 1027, 0, 1027, 0, 1026, 0, 1026, 0, 1026, 0, 1026, 1017, 0] rerouted_tokens_per_ep_rank=[1017, 1027, 1027, 1026, 1026, 1026, 1026, 1017] avg=1024.0000 max=1027 max_rank=1 min=1017 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1024, 1023, 1024, 1024, 1025] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1022, 3, 1022, 0, 1023, 4, 1022, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1025, 1022, 1025, 1023, 1026, 1023, 1023, 1025] avg=1024.0000 max=1026 max_rank=4 min=1022 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1022, 3, 1022, 0, 1023, 3, 1022, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1026, 1022, 1025, 1023, 1025, 1023, 1023, 1025] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1024, 1023, 1024, 1024, 1025] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1024, 1023, 1024, 1024, 1025] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1024, 1023, 1024, 1024, 1025] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1024, 1023, 1024, 1024, 1025] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8198, 8186, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8198, 8186, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8198, 8186, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8198, 8186, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8198, 8186, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8198 expected_overflow=0 exact_recv_per_rank=[8198, 8186, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8198, 8186, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8186 expected_overflow=0 exact_recv_per_rank=[8198, 8186, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8198 actual_valid_recv=8320 observed_loss=-122 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8186 actual_valid_recv=8192 observed_loss=-6 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1, 3, 4093, 0, 4093, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[2, 4, 4092, 0, 4092, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1027, 0, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1027, 1024, 1025, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1027, 0, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1027, 1024, 1025, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 1022, 4, 1022, 1026, 0, 0, 1023, 1024, 0, 0, 1022, 2, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1026, 1023, 1024, 1022, 1024, 1023] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1029, 0, 1026, 1012, 0, 0, 1026, 1018, 0, 0, 1026, 0, 1029, 0, 1026] rerouted_tokens_per_ep_rank=[1029, 1026, 1012, 1026, 1018, 1026, 1029, 1026] avg=1024.0000 max=1029 max_rank=0 min=1012 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 3, 1022, 1026, 0, 0, 1023, 1025, 0, 0, 1022, 2, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1025, 1026, 1023, 1025, 1022, 1024, 1023] avg=1024.0000 max=1026 max_rank=2 min=1022 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1027, 0, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1027, 1024, 1025, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1027, 0, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1027, 1024, 1025, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 1027, 0, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1027, 1024, 1025, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8199, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8199, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8199, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8199, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8199, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8199, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8199, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8199, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4092, 0, 0, 4, 3, 4092, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 0, 3, 3, 4093, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 0, 1022, 4, 1022, 3, 1022, 1024, 0, 0, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1022, 1026, 1025, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1022 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1018, 0, 0, 1026, 0, 1026, 0, 1026, 0, 1027, 1018, 0, 0, 1026, 0, 1025] rerouted_tokens_per_ep_rank=[1018, 1026, 1026, 1026, 1027, 1018, 1026, 1025] avg=1024.0000 max=1027 max_rank=4 min=1018 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1022, 3, 1022, 3, 1022, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1022, 1025, 1025, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[3, 0, 0, 1, 4092, 4, 0, 4092] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4093, 3, 0, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1026, 1023, 1024, 1025] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3, 1022, 0, 1023, 0, 1022, 1, 1023, 1025, 0, 4, 1022, 0, 1023, 1024, 0] rerouted_tokens_per_ep_rank=[1025, 1023, 1022, 1024, 1025, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=5 min=1022 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1026, 1023, 1024, 1025] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 0, 1026, 0, 1026, 0, 1025, 1018, 0, 0, 1026, 0, 1026, 1018, 0] rerouted_tokens_per_ep_rank=[1027, 1026, 1026, 1025, 1018, 1026, 1026, 1018] avg=1024.0000 max=1027 max_rank=0 min=1018 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4088 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3, 1022, 0, 1023, 0, 1022, 0, 1023, 1026, 0, 3, 1022, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1025, 1023, 1022, 1023, 1026, 1025, 1023, 1025] avg=1024.0000 max=1026 max_rank=4 min=1022 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1026, 1023, 1024, 1025] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1026, 1023, 1024, 1025] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1026, 1023, 1024, 1025] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4092, 0, 1, 1, 0, 6, 4089, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 5, 4091, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 7, 0, 0, 0, 0, 4089, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1021, 1, 1022, 1, 1023, 0, 1023, 6, 1021, 1025, 0, 3, 1022] rerouted_tokens_per_ep_rank=[1024, 1021, 1023, 1024, 1023, 1027, 1025, 1025] avg=1024.0000 max=1027 max_rank=5 min=1021 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 7, 1021, 0, 1023, 0, 1024, 0, 1024, 0, 1021, 1025, 0, 0, 1022] rerouted_tokens_per_ep_rank=[1025, 1028, 1023, 1024, 1024, 1021, 1025, 1022] avg=1024.0000 max=1028 max_rank=1 min=1021 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1022, 0, 1022, 0, 1023, 0, 1023, 5, 1021, 1026, 0, 3, 1022] rerouted_tokens_per_ep_rank=[1025, 1022, 1022, 1023, 1023, 1026, 1026, 1025] avg=1024.0000 max=1026 max_rank=5 min=1022 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1018, 0, 0, 1029, 0, 1027, 0, 1025, 0, 1026, 0, 1026, 1013, 0, 0, 1028] rerouted_tokens_per_ep_rank=[1018, 1029, 1027, 1025, 1026, 1026, 1013, 1028] avg=1024.0000 max=1029 max_rank=1 min=1013 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1023, 1024, 1024, 1023, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1023, 1024, 1024, 1023, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1023, 1024, 1024, 1023, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1023, 1024, 1024, 1023, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8187 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8197 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8187 actual_valid_recv=8320 observed_loss=-133 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8197 actual_valid_recv=8320 observed_loss=-123 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True + [2026-08-30 04:54:04] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 667.8 | throughput per GPU (TFLOP/s/GPU): 499.2 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.273275E+00 | loss scale: 1.0 | grad norm: 6.766 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3762, 0, 0, 0, 349, 12, 4069, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3519, 0, 0, 2, 594, 4, 4073, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3895, 1, 0, 1, 209, 0, 4086, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3304, 0, 0, 0, 807, 2, 4079, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3323, 3, 1, 0, 821, 2, 4042, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3604, 0, 0, 0, 540, 0, 4048, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[3623, 1, 0, 0, 489, 0, 4079, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3712, 0, 0, 0, 414, 0, 4066, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1103, 0, 0, 1071, 0, 1024, 0, 996, 349, 519, 12, 1069, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1103, 1071, 1024, 996, 868, 1081, 1025, 1024] avg=1024.0000 max=1103 max_rank=0 min=868 min_rank=4 imbalance_ratio=1.077148 max_load_reduction=7089 max_load_reduction_pct=0.865356 moved_to_echo=3998 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1142, 0, 1, 1109, 0, 1028, 1, 1000, 209, 537, 0, 1107, 1030, 0, 0, 1028] rerouted_tokens_per_ep_rank=[1142, 1110, 1028, 1001, 746, 1107, 1030, 1028] avg=1024.0000 max=1142 max_rank=0 min=746 min_rank=4 imbalance_ratio=1.115234 max_load_reduction=7050 max_load_reduction_pct=0.860596 moved_to_echo=3911 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1032, 0, 0, 1002, 0, 1025, 2, 997, 594, 485, 4, 1000, 1026, 0, 0, 1025] rerouted_tokens_per_ep_rank=[1032, 1002, 1025, 999, 1079, 1004, 1026, 1025] avg=1024.0000 max=1079 max_rank=4 min=999 min_rank=3 imbalance_ratio=1.053711 max_load_reduction=7113 max_load_reduction_pct=0.868286 moved_to_echo=4134 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[968, 0, 0, 941, 0, 1026, 0, 998, 807, 456, 2, 939, 1029, 0, 0, 1026] rerouted_tokens_per_ep_rank=[968, 941, 1026, 998, 1263, 941, 1029, 1026] avg=1024.0000 max=1263 max_rank=4 min=941 min_rank=1 imbalance_ratio=1.233398 max_load_reduction=6929 max_load_reduction_pct=0.845825 moved_to_echo=4259 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1049, 0, 1, 1035, 0, 1030, 0, 1001, 489, 505, 0, 1034, 1018, 0, 0, 1030] rerouted_tokens_per_ep_rank=[1049, 1036, 1030, 1001, 994, 1034, 1018, 1030] avg=1024.0000 max=1049 max_rank=0 min=994 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=7143 max_load_reduction_pct=0.871948 moved_to_echo=4076 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1057, 0, 0, 1026, 0, 1018, 0, 991, 540, 497, 0, 1024, 1020, 0, 0, 1019] rerouted_tokens_per_ep_rank=[1057, 1026, 1018, 991, 1037, 1024, 1020, 1019] avg=1024.0000 max=1057 max_rank=0 min=991 min_rank=3 imbalance_ratio=1.032227 max_load_reduction=7135 max_load_reduction_pct=0.870972 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[975, 0, 3, 946, 1, 1017, 0, 989, 821, 458, 2, 944, 1019, 0, 0, 1017] rerouted_tokens_per_ep_rank=[975, 949, 1018, 989, 1279, 946, 1019, 1017] avg=1024.0000 max=1279 max_rank=4 min=946 min_rank=5 imbalance_ratio=1.249023 max_load_reduction=6913 max_load_reduction_pct=0.843872 moved_to_echo=4261 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1088, 0, 0, 1057, 0, 1023, 0, 995, 414, 512, 0, 1055, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1088, 1057, 1023, 995, 926, 1055, 1025, 1023] avg=1024.0000 max=1088 max_rank=0 min=926 min_rank=4 imbalance_ratio=1.062500 max_load_reduction=7104 max_load_reduction_pct=0.867188 moved_to_echo=4029 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8414, 8192, 8192, 7970, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7970 expected_overflow=0 exact_recv_per_rank=[8414, 8192, 8192, 7970, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8414, 8192, 8192, 7970, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8414, 8192, 8192, 7970, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8414 expected_overflow=0 exact_recv_per_rank=[8414, 8192, 8192, 7970, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8414, 8192, 8192, 7970, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8414, 8192, 8192, 7970, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8414, 8192, 8192, 7970, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4224, 4096] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8414 actual_valid_recv=8448 observed_loss=-34 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=7970 actual_valid_recv=8192 observed_loss=-222 tokens_per_expert=[128, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 1, 1, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4093, 2, 4094, 0, 3, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1024, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1014, 0, 0, 1029, 1016, 0, 0, 1025, 0, 1028, 0, 1026, 0, 1028, 0, 1026] rerouted_tokens_per_ep_rank=[1014, 1029, 1016, 1025, 1028, 1026, 1028, 1026] avg=1024.0000 max=1029 max_rank=1 min=1014 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1024, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1024, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1026, 0, 1, 1023, 1, 1023, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1026, 1024, 1024, 1023, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1024, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 2, 1023, 1025, 0, 0, 1023, 3, 1022, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1025, 1025, 1023, 1025, 1023, 1023, 1023] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=3 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1024, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8195, 8192, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8189 actual_valid_recv=8192 observed_loss=-3 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4095, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 4093, 1, 4093, 0, 0, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1024, 1026, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 1025, 0, 0, 1023, 1026, 0, 0, 1023, 0, 1024, 1, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1023, 1026, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1024, 1026, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 0, 1029, 1017, 0, 0, 1025, 1014, 0, 0, 1029, 0, 1025, 0, 1028] rerouted_tokens_per_ep_rank=[1025, 1029, 1017, 1025, 1014, 1029, 1025, 1028] avg=1024.0000 max=1029 max_rank=1 min=1014 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1024, 1026, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 2, 1023, 1025, 0, 1, 1023, 1025, 0, 0, 1022, 0, 1023, 3, 1022] rerouted_tokens_per_ep_rank=[1023, 1025, 1025, 1024, 1025, 1022, 1023, 1025] avg=1024.0000 max=1025 max_rank=1 min=1022 min_rank=5 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1024, 1026, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1025, 1024, 1026, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8195, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8195, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8195, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8195, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8195, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8195, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8195, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8195, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8189 actual_valid_recv=8192 observed_loss=-3 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 2, 4093, 0, 4093, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1026, 0, 0, 1023, 1026, 0, 1, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1026, 1023, 1026, 1024, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1026, 0, 0, 1024, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1026, 1024, 1025, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 0, 1025, 0, 1029, 1014, 0, 0, 1026, 1017, 0, 0, 1028, 0, 1025] rerouted_tokens_per_ep_rank=[1028, 1025, 1029, 1014, 1026, 1017, 1028, 1025] avg=1024.0000 max=1029 max_rank=2 min=1014 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1026, 0, 0, 1024, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1026, 1024, 1025, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1026, 0, 0, 1024, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1026, 1024, 1025, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1026, 0, 0, 1024, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1026, 1024, 1025, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1, 1023, 2, 1023, 1024, 0, 0, 1023, 1024, 0, 2, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1025, 1024, 1023, 1024, 1025, 1023] avg=1024.0000 max=1025 max_rank=2 min=1023 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 1026, 0, 0, 1024, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1026, 1024, 1025, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8194, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8190 actual_valid_recv=8320 observed_loss=-130 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4091, 0, 0, 4096, 0, 0, 5, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4028, 0, 0, 4096, 0, 0, 68, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4069, 0, 0, 4096, 0, 0, 27, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4040, 1, 0, 4095, 1, 0, 55, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4035, 3, 0, 4093, 2, 0, 59, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4065, 0, 0, 4096, 0, 0, 31, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4077, 0, 0, 4096, 0, 0, 19, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1034, 0, 0, 1030, 0, 1024, 1025, 0, 0, 1030, 0, 1023, 5, 997, 0, 1024] rerouted_tokens_per_ep_rank=[1034, 1030, 1024, 1025, 1030, 1023, 1002, 1024] avg=1024.0000 max=1034 max_rank=0 min=1002 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4079 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1018, 0, 0, 1014, 0, 1024, 1025, 0, 0, 1014, 0, 1023, 68, 982, 0, 1024] rerouted_tokens_per_ep_rank=[1018, 1014, 1024, 1025, 1014, 1023, 1050, 1024] avg=1024.0000 max=1050 max_rank=6 min=1014 min_rank=1 imbalance_ratio=1.025391 max_load_reduction=7142 max_load_reduction_pct=0.871826 moved_to_echo=4111 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1021, 0, 1, 1017, 0, 1023, 1026, 0, 1, 1017, 0, 1023, 55, 985, 0, 1023] rerouted_tokens_per_ep_rank=[1021, 1018, 1023, 1026, 1018, 1023, 1040, 1023] avg=1024.0000 max=1040 max_rank=6 min=1018 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1025, 0, 1024, 1025, 0, 0, 1025, 0, 1023, 27, 992, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1025, 1024, 1025, 1025, 1023, 1019, 1024] avg=1024.0000 max=1027 max_rank=0 min=1019 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1022, 0, 0, 1035, 0, 1026, 1016, 0, 0, 1036, 0, 1028, 1, 1002, 0, 1026] rerouted_tokens_per_ep_rank=[1022, 1035, 1026, 1016, 1036, 1028, 1003, 1026] avg=1024.0000 max=1036 max_rank=4 min=1003 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1024, 1025, 0, 0, 1024, 0, 1023, 31, 991, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1024, 1025, 1024, 1023, 1022, 1024] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1019, 0, 3, 1016, 0, 1023, 1025, 0, 2, 1016, 0, 1022, 59, 984, 0, 1023] rerouted_tokens_per_ep_rank=[1019, 1019, 1023, 1025, 1018, 1022, 1043, 1023] avg=1024.0000 max=1043 max_rank=6 min=1018 min_rank=4 imbalance_ratio=1.018555 max_load_reduction=7149 max_load_reduction_pct=0.872681 moved_to_echo=4106 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1029, 0, 0, 1027, 0, 1024, 1025, 0, 0, 1027, 0, 1023, 19, 994, 0, 1024] rerouted_tokens_per_ep_rank=[1029, 1027, 1024, 1025, 1027, 1023, 1013, 1024] avg=1024.0000 max=1029 max_rank=0 min=1013 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8188 actual_valid_recv=8192 observed_loss=-4 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 4095, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[3, 4093, 1, 0, 4093, 2, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1027, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1027, 1023, 1024, 1025, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1027, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1027, 1023, 1024, 1025, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1, 1023, 0, 1023, 1026, 0, 0, 1023, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1024, 1023, 1026, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 1010, 0, 0, 1029, 0, 1026, 1016, 0, 0, 1029, 0, 1028, 0, 1026] rerouted_tokens_per_ep_rank=[1028, 1010, 1029, 1026, 1016, 1029, 1028, 1026] avg=1024.0000 max=1029 max_rank=2 min=1010 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1027, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1027, 1023, 1024, 1025, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1027, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1027, 1023, 1024, 1025, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 1023, 1024, 0, 1, 1023, 0, 1023, 1025, 0, 2, 1023, 0, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1024, 1023, 1025, 1025, 1022, 1023] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1027, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1027, 1023, 1024, 1025, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8192, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8189 actual_valid_recv=8320 observed_loss=-131 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4093, 4093, 1, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1026, 0, 1, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1023, 1026, 1026, 1024, 1024] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 1027, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1023, 1025, 1027, 1023, 1023] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 1027, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1023, 1025, 1027, 1023, 1023] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 0, 1026, 0, 1026, 0, 1029, 1016, 0, 1010, 0, 0, 1029, 0, 1028] rerouted_tokens_per_ep_rank=[1028, 1026, 1026, 1029, 1016, 1010, 1029, 1028] avg=1024.0000 max=1029 max_rank=3 min=1010 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 1027, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1023, 1025, 1027, 1023, 1023] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 1027, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1023, 1025, 1027, 1023, 1023] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1023, 0, 1023, 2, 1023, 1025, 0, 1025, 0, 1, 1023, 3, 1022] rerouted_tokens_per_ep_rank=[1022, 1023, 1023, 1025, 1025, 1025, 1024, 1025] avg=1024.0000 max=1025 max_rank=3 min=1022 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 1027, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1023, 1025, 1027, 1023, 1023] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8188 actual_valid_recv=8192 observed_loss=-4 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4095, 4095, 1, 0, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4093, 4093, 2, 0, 0, 0, 3, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1026, 0, 1, 1023, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1026, 1024, 1023, 1023, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1025, 1023, 1024, 1024, 1023, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1025, 1023, 1024, 1024, 1023, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1025, 1023, 1024, 1024, 1023, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1025, 1023, 1024, 1024, 1023, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1015, 0, 1016, 0, 0, 1028, 0, 1026, 0, 1026, 0, 1028, 0, 1028, 0, 1025] rerouted_tokens_per_ep_rank=[1015, 1016, 1028, 1026, 1026, 1028, 1028, 1025] avg=1024.0000 max=1028 max_rank=2 min=1015 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4107 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1025, 0, 2, 1023, 0, 1023, 0, 1023, 0, 1022, 3, 1022, 1, 1023] rerouted_tokens_per_ep_rank=[1025, 1025, 1025, 1023, 1023, 1022, 1025, 1024] avg=1024.0000 max=1025 max_rank=0 min=1022 min_rank=5 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1025, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1025, 1023, 1024, 1024, 1023, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8188 actual_valid_recv=8192 observed_loss=-4 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4095, 4095, 1, 0, 1, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4093, 4093, 2, 0, 3, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1026, 0, 1, 1023, 0, 1023, 1, 1023, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1026, 1024, 1023, 1024, 1023, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1025, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1025, 1023, 1024, 1023, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1025, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1025, 1023, 1024, 1023, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1025, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1025, 1023, 1024, 1023, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1015, 0, 1016, 0, 0, 1028, 0, 1026, 0, 1028, 0, 1028, 0, 1026, 0, 1025] rerouted_tokens_per_ep_rank=[1015, 1016, 1028, 1026, 1028, 1028, 1026, 1025] avg=1024.0000 max=1028 max_rank=2 min=1015 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4107 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1025, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1025, 1023, 1024, 1023, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1025, 0, 2, 1023, 0, 1023, 3, 1022, 0, 1022, 0, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1025, 1025, 1025, 1023, 1025, 1022, 1023, 1024] avg=1024.0000 max=1025 max_rank=0 min=1022 min_rank=5 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1025, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1025, 1023, 1024, 1023, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8188 actual_valid_recv=8192 observed_loss=-4 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 663, 3433, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 604, 3492, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4095, 351, 3744, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 590, 3506, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 397, 3699, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 561, 3535, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 3, 3, 0, 4093, 648, 3445, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 416, 3680, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1074, 1, 1074, 0, 1023, 1026, 0, 351, 519, 1077, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1075, 1075, 1023, 1026, 870, 1077, 1023] avg=1024.0000 max=1077 max_rank=6 min=870 min_rank=5 imbalance_ratio=1.051758 max_load_reduction=7115 max_load_reduction_pct=0.868530 moved_to_echo=3996 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1061, 0, 1061, 0, 1023, 1025, 0, 397, 513, 1064, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1061, 1061, 1023, 1025, 910, 1064, 1024] avg=1024.0000 max=1064 max_rank=6 min=910 min_rank=5 imbalance_ratio=1.039062 max_load_reduction=7128 max_load_reduction_pct=0.870117 moved_to_echo=4023 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1002, 0, 1002, 0, 1023, 1025, 0, 604, 484, 1004, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1002, 1002, 1023, 1025, 1088, 1004, 1024] avg=1024.0000 max=1088 max_rank=5 min=1002 min_rank=1 imbalance_ratio=1.062500 max_load_reduction=7104 max_load_reduction_pct=0.867188 moved_to_echo=4141 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 988, 0, 988, 0, 1028, 1016, 0, 663, 482, 975, 0, 0, 1026] rerouted_tokens_per_ep_rank=[1026, 988, 988, 1028, 1016, 1145, 975, 1026] avg=1024.0000 max=1145 max_rank=5 min=975 min_rank=6 imbalance_ratio=1.118164 max_load_reduction=7047 max_load_reduction_pct=0.860229 moved_to_echo=4162 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1014, 0, 1014, 0, 1023, 1025, 0, 561, 490, 1017, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1014, 1014, 1023, 1025, 1051, 1017, 1024] avg=1024.0000 max=1051 max_rank=5 min=1014 min_rank=1 imbalance_ratio=1.026367 max_load_reduction=7141 max_load_reduction_pct=0.871704 moved_to_echo=4117 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1006, 0, 1006, 0, 1023, 1025, 0, 590, 486, 1008, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1006, 1006, 1023, 1025, 1076, 1008, 1024] avg=1024.0000 max=1076 max_rank=5 min=1006 min_rank=1 imbalance_ratio=1.050781 max_load_reduction=7116 max_load_reduction_pct=0.868652 moved_to_echo=4133 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 3, 988, 3, 988, 0, 1022, 1025, 0, 648, 478, 991, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 991, 991, 1022, 1025, 1126, 991, 1023] avg=1024.0000 max=1126 max_rank=5 min=991 min_rank=1 imbalance_ratio=1.099609 max_load_reduction=7066 max_load_reduction_pct=0.862549 moved_to_echo=4165 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1055, 0, 1055, 0, 1023, 1025, 0, 416, 510, 1060, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1055, 1055, 1023, 1025, 926, 1060, 1024] avg=1024.0000 max=1060 max_rank=6 min=926 min_rank=5 imbalance_ratio=1.035156 max_load_reduction=7132 max_load_reduction_pct=0.870605 moved_to_echo=4035 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4352, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8188 actual_valid_recv=8192 observed_loss=-4 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 3, 0, 4096, 4093, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 1, 4095, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[3, 0, 3, 4093, 4093, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1025, 1027, 1024, 1023, 1024] avg=1024.0000 max=1027 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 3, 1023, 0, 1022, 1025, 0, 1026, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1022, 1026, 1022, 1025, 1026, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 1, 1023, 1026, 0, 1026, 0, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1026, 1026, 1023, 1023, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 1022, 0, 1023, 3, 1022, 1025, 0, 1026, 0, 0, 1023, 0, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1025, 1025, 1026, 1023, 1022, 1023] avg=1024.0000 max=1026 max_rank=4 min=1022 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1025, 1027, 1024, 1023, 1024] avg=1024.0000 max=1027 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1029, 0, 1028, 0, 1029, 1016, 0, 1010, 0, 0, 1026, 0, 1028, 0, 1026] rerouted_tokens_per_ep_rank=[1029, 1028, 1029, 1016, 1010, 1026, 1028, 1026] avg=1024.0000 max=1029 max_rank=0 min=1010 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1025, 1027, 1024, 1023, 1024] avg=1024.0000 max=1027 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1025, 1027, 1024, 1023, 1024] avg=1024.0000 max=1027 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8196, 8192, 8188, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8196, 8192, 8188, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8196, 8192, 8188, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8196, 8192, 8188, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8196, 8192, 8188, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8196, 8192, 8188, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8196, 8192, 8188, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8196, 8192, 8188, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8188 actual_valid_recv=8192 observed_loss=-4 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4095, 4093, 1, 1, 0, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4093, 4086, 3, 1, 2, 0, 7, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1025, 0, 1, 1023, 1, 1023, 0, 1023, 0, 1023, 2, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1025, 1024, 1024, 1023, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1026, 1023, 1024, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1026, 1023, 1024, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 1015, 0, 0, 1029, 0, 1026, 0, 1028, 0, 1026, 0, 1026, 0, 1026] rerouted_tokens_per_ep_rank=[1016, 1015, 1029, 1026, 1028, 1026, 1026, 1026] avg=1024.0000 max=1029 max_rank=2 min=1015 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4106 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1026, 1023, 1024, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1026, 1023, 1024, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1024, 0, 3, 1021, 1, 1021, 2, 1022, 0, 1023, 7, 1020, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1024, 1022, 1024, 1023, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1026, 1023, 1024, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8190, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8190 actual_valid_recv=8320 observed_loss=-130 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4047, 0, 49] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 3951, 0, 145] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 1, 4095, 0, 0, 4043, 1, 52] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4057, 0, 39] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4005, 0, 91] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4015, 0, 81] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4037, 0, 59] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 3, 4093, 0, 0, 3719, 3, 374] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1038, 1026, 0, 0, 1023, 0, 1023, 1041, 0, 1, 1038, 52, 926] rerouted_tokens_per_ep_rank=[1023, 1039, 1026, 1023, 1023, 1041, 1039, 978] avg=1024.0000 max=1041 max_rank=5 min=978 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4081 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1014, 1025, 0, 0, 1023, 0, 1024, 1018, 0, 0, 1014, 145, 905] rerouted_tokens_per_ep_rank=[1024, 1014, 1025, 1023, 1024, 1018, 1014, 1050] avg=1024.0000 max=1050 max_rank=7 min=1014 min_rank=1 imbalance_ratio=1.025391 max_load_reduction=7142 max_load_reduction_pct=0.871826 moved_to_echo=4106 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 1043, 1016, 0, 0, 1028, 0, 1026, 1030, 0, 0, 1043, 49, 931] rerouted_tokens_per_ep_rank=[1026, 1043, 1016, 1028, 1026, 1030, 1043, 980] avg=1024.0000 max=1043 max_rank=1 min=980 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=7149 max_load_reduction_pct=0.872681 moved_to_echo=4079 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1028, 1025, 0, 0, 1023, 0, 1024, 1032, 0, 0, 1028, 91, 917] rerouted_tokens_per_ep_rank=[1024, 1028, 1025, 1023, 1024, 1032, 1028, 1008] avg=1024.0000 max=1032 max_rank=5 min=1008 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1042, 1025, 0, 0, 1023, 0, 1024, 1044, 0, 0, 1042, 39, 929] rerouted_tokens_per_ep_rank=[1024, 1042, 1025, 1023, 1024, 1044, 1042, 968] avg=1024.0000 max=1044 max_rank=5 min=968 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=7148 max_load_reduction_pct=0.872559 moved_to_echo=4078 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1031, 1025, 0, 0, 1023, 0, 1024, 1034, 0, 0, 1031, 81, 919] rerouted_tokens_per_ep_rank=[1024, 1031, 1025, 1023, 1024, 1034, 1031, 1000] avg=1024.0000 max=1034 max_rank=5 min=1000 min_rank=7 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 3, 955, 1025, 0, 0, 1022, 0, 1023, 958, 0, 3, 955, 374, 851] rerouted_tokens_per_ep_rank=[1023, 958, 1025, 1022, 1023, 958, 958, 1225] avg=1024.0000 max=1225 max_rank=7 min=958 min_rank=1 imbalance_ratio=1.196289 max_load_reduction=6967 max_load_reduction_pct=0.850464 moved_to_echo=4164 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1037, 1025, 0, 0, 1023, 0, 1024, 1039, 0, 0, 1037, 59, 924] rerouted_tokens_per_ep_rank=[1024, 1037, 1025, 1023, 1024, 1039, 1037, 983] avg=1024.0000 max=1039 max_rank=5 min=983 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=7153 max_load_reduction_pct=0.873169 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8188, 8192, 8196, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[896, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8188 actual_valid_recv=8192 observed_loss=-4 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 4095, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 3, 4093, 4069, 24] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1, 1024, 1, 1024, 1026, 0, 1026, 0, 0, 1021] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1025, 1025, 1026, 1026, 1021] avg=1024.0000 max=1026 max_rank=5 min=1021 min_rank=7 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 1027, 0, 0, 1021] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1024, 1024, 1025, 1027, 1021] avg=1024.0000 max=1027 max_rank=6 min=1021 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 1027, 0, 0, 1021] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1024, 1024, 1025, 1027, 1021] avg=1024.0000 max=1027 max_rank=6 min=1021 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 1027, 0, 0, 1021] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1024, 1024, 1025, 1027, 1021] avg=1024.0000 max=1027 max_rank=6 min=1021 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 0, 1026, 0, 1026, 0, 1027, 0, 1027, 1016, 0, 1016, 0, 0, 1026] rerouted_tokens_per_ep_rank=[1028, 1026, 1026, 1027, 1027, 1016, 1016, 1026] avg=1024.0000 max=1028 max_rank=0 min=1016 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4085 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 1027, 0, 0, 1021] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1024, 1024, 1025, 1027, 1021] avg=1024.0000 max=1027 max_rank=6 min=1021 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1023, 0, 1023, 3, 1017, 3, 1017, 1025, 0, 1020, 0, 24, 1015] rerouted_tokens_per_ep_rank=[1022, 1023, 1023, 1020, 1020, 1025, 1020, 1039] avg=1024.0000 max=1039 max_rank=7 min=1020 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=7153 max_load_reduction_pct=0.873169 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 1026, 0, 1, 1021] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1024, 1024, 1025, 1026, 1022] avg=1024.0000 max=1026 max_rank=6 min=1022 min_rank=7 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8188, 8192, 8192, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8188 actual_valid_recv=8192 observed_loss=-4 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 4095, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 3, 4093, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 1026, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1023, 1024, 1026, 1026] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 0, 1026, 0, 1028, 0, 1026, 0, 1026, 0, 1028, 1014, 0, 1016, 0] rerouted_tokens_per_ep_rank=[1028, 1026, 1028, 1026, 1026, 1028, 1014, 1016] avg=1024.0000 max=1028 max_rank=0 min=1014 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4084 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3, 1022, 0, 1023, 0, 1022, 0, 1023, 0, 1023, 3, 1022, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1025, 1023, 1022, 1023, 1023, 1025, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1022 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8188, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8188, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8188, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8188, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8188, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8188, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8188, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8188, 8192, 8192, 8192, 8196, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8188 actual_valid_recv=8192 observed_loss=-4 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 1, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 3, 4093, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 1026, 0, 1, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1023, 1023, 1024, 1026, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1024, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 1014, 0, 0, 1028, 0, 1026, 0, 1026, 0, 1028, 1016, 0, 0, 1028] rerouted_tokens_per_ep_rank=[1026, 1014, 1028, 1026, 1026, 1028, 1016, 1028] avg=1024.0000 max=1028 max_rank=2 min=1014 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1024, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1024, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1024, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1022, 0, 1023, 0, 1023, 3, 1022, 1025, 0, 3, 1022] rerouted_tokens_per_ep_rank=[1023, 1026, 1022, 1023, 1023, 1025, 1025, 1025] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1024, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8196, 8188, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8196, 8188, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8196, 8188, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8192, 8196, 8188, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8196, 8188, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8196, 8188, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8192, 8196, 8188, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8196, 8188, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8188 actual_valid_recv=8192 observed_loss=-4 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[3879, 3, 1, 0, 266, 0, 4042, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3764, 0, 0, 0, 351, 5, 4072, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3540, 19, 2, 1, 556, 0, 4073, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3907, 0, 0, 0, 189, 1, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3807, 0, 0, 0, 318, 0, 4067, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3774, 5, 0, 0, 320, 6, 4087, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3969, 2, 0, 0, 149, 0, 4072, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1050, 0, 3, 1041, 1, 999, 0, 1018, 266, 746, 0, 1042, 1005, 0, 1, 1020] rerouted_tokens_per_ep_rank=[1050, 1044, 1000, 1018, 1012, 1042, 1005, 1021] avg=1024.0000 max=1050 max_rank=0 min=1000 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=7142 max_load_reduction_pct=0.871826 moved_to_echo=4080 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[971, 0, 19, 946, 2, 1002, 1, 1023, 556, 676, 0, 947, 1025, 0, 1, 1023] rerouted_tokens_per_ep_rank=[971, 965, 1004, 1024, 1232, 947, 1025, 1024] avg=1024.0000 max=1232 max_rank=4 min=947 min_rank=5 imbalance_ratio=1.203125 max_load_reduction=6960 max_load_reduction_pct=0.849609 moved_to_echo=4228 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1032, 0, 0, 1006, 0, 1001, 0, 1023, 351, 719, 5, 1007, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1032, 1006, 1001, 1023, 1070, 1012, 1025, 1023] avg=1024.0000 max=1070 max_rank=4 min=1001 min_rank=2 imbalance_ratio=1.044922 max_load_reduction=7122 max_load_reduction_pct=0.869385 moved_to_echo=4130 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1071, 0, 0, 1044, 0, 1007, 0, 1029, 189, 746, 1, 1046, 1031, 0, 0, 1028] rerouted_tokens_per_ep_rank=[1071, 1044, 1007, 1029, 935, 1047, 1031, 1028] avg=1024.0000 max=1071 max_rank=0 min=935 min_rank=4 imbalance_ratio=1.045898 max_load_reduction=7121 max_load_reduction_pct=0.869263 moved_to_echo=4041 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1044, 0, 0, 1017, 0, 1000, 0, 1022, 318, 727, 0, 1019, 1024, 0, 0, 1021] rerouted_tokens_per_ep_rank=[1044, 1017, 1000, 1022, 1045, 1019, 1024, 1021] avg=1024.0000 max=1045 max_rank=4 min=1000 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=7147 max_load_reduction_pct=0.872437 moved_to_echo=4109 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1035, 0, 5, 1008, 0, 1005, 0, 1027, 320, 721, 6, 1010, 1029, 0, 0, 1026] rerouted_tokens_per_ep_rank=[1035, 1013, 1005, 1027, 1041, 1016, 1029, 1026] avg=1024.0000 max=1041 max_rank=4 min=1005 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4112 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1089, 0, 2, 1060, 0, 1001, 0, 1023, 149, 758, 0, 1062, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1089, 1062, 1001, 1023, 907, 1062, 1025, 1023] avg=1024.0000 max=1089 max_rank=0 min=907 min_rank=4 imbalance_ratio=1.063477 max_load_reduction=7103 max_load_reduction_pct=0.867065 moved_to_echo=4017 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3883, 4, 4, 0, 208, 8, 4085, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1065, 0, 4, 1037, 4, 1005, 0, 1026, 208, 742, 8, 1039, 1028, 0, 0, 1026] rerouted_tokens_per_ep_rank=[1065, 1041, 1009, 1026, 950, 1047, 1028, 1026] avg=1024.0000 max=1065 max_rank=0 min=950 min_rank=4 imbalance_ratio=1.040039 max_load_reduction=7127 max_load_reduction_pct=0.869995 moved_to_echo=4051 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8357, 8192, 8027, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8357 expected_overflow=0 exact_recv_per_rank=[8357, 8192, 8027, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8357, 8192, 8027, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8027 expected_overflow=0 exact_recv_per_rank=[8357, 8192, 8027, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8357, 8192, 8027, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8357, 8192, 8027, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8357, 8192, 8027, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8357, 8192, 8027, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8357 actual_valid_recv=8448 observed_loss=-91 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8027 actual_valid_recv=8192 observed_loss=-165 tokens_per_expert=[128, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2432, 5888] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 1, 1, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4085, 0, 4085, 10, 7, 2, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4095, 1, 4095, 0, 1, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4094, 0, 4093, 0, 0, 0, 0, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4091, 0, 4093, 2, 0, 5, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1024, 1026, 0, 1, 1022, 1, 1023, 0, 1023, 0, 1023, 0, 1022] rerouted_tokens_per_ep_rank=[1027, 1024, 1026, 1023, 1024, 1023, 1023, 1022] avg=1024.0000 max=1027 max_rank=0 min=1022 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1021, 1023, 0, 10, 1020, 7, 1020, 2, 1021, 3, 1021, 0, 1020] rerouted_tokens_per_ep_rank=[1024, 1021, 1023, 1030, 1027, 1023, 1024, 1020] avg=1024.0000 max=1030 max_rank=3 min=1020 min_rank=7 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 0, 1024, 1026, 0, 0, 1022, 0, 1023, 0, 1023, 0, 1024, 0, 1022] rerouted_tokens_per_ep_rank=[1028, 1024, 1026, 1022, 1023, 1023, 1024, 1022] avg=1024.0000 max=1028 max_rank=0 min=1022 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 0, 1024, 1026, 0, 0, 1022, 0, 1023, 0, 1023, 0, 1024, 0, 1022] rerouted_tokens_per_ep_rank=[1028, 1024, 1026, 1022, 1023, 1023, 1024, 1022] avg=1024.0000 max=1028 max_rank=0 min=1022 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1013, 0, 0, 1027, 1014, 0, 0, 1028, 0, 1026, 0, 1027, 0, 1026, 5, 1026] rerouted_tokens_per_ep_rank=[1013, 1027, 1014, 1028, 1026, 1027, 1026, 1031] avg=1024.0000 max=1031 max_rank=7 min=1013 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4110 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 2, 1021, 0, 1022, 5, 1022, 1, 1023, 0, 1022] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1023, 1022, 1027, 1024, 1022] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 1, 1024, 1026, 0, 0, 1022, 1, 1023, 0, 1023, 0, 1023, 0, 1022] rerouted_tokens_per_ep_rank=[1027, 1025, 1026, 1022, 1024, 1023, 1023, 1022] avg=1024.0000 max=1027 max_rank=0 min=1022 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 0, 1024, 1026, 0, 0, 1022, 0, 1023, 0, 1023, 0, 1024, 0, 1022] rerouted_tokens_per_ep_rank=[1028, 1024, 1026, 1022, 1023, 1023, 1024, 1022] avg=1024.0000 max=1028 max_rank=0 min=1022 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8201 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8192, 8192, 8192, 8192, 8183] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8192, 8192, 8192, 8192, 8183] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8192, 8192, 8192, 8192, 8183] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8192, 8192, 8192, 8192, 8183] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8192, 8192, 8192, 8192, 8183] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8192, 8192, 8192, 8192, 8183] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8192, 8192, 8192, 8192, 8183] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8183 expected_overflow=0 exact_recv_per_rank=[8201, 8192, 8192, 8192, 8192, 8192, 8192, 8183] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8201 actual_valid_recv=8320 observed_loss=-119 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8183 actual_valid_recv=8320 observed_loss=-137 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 4095, 0, 4095, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[11, 1, 4083, 0, 4082, 3, 1, 11] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4092, 0, 4, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[5, 0, 4091, 0, 4091, 0, 0, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 4095, 0, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1022, 0, 1024, 1026, 0, 0, 1024, 1028, 0, 0, 1021, 0, 1023, 1, 1022] rerouted_tokens_per_ep_rank=[1023, 1024, 1026, 1024, 1028, 1021, 1023, 1023] avg=1024.0000 max=1028 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[11, 1019, 1, 1021, 1022, 0, 0, 1021, 1024, 0, 3, 1019, 1, 1020, 11, 1019] rerouted_tokens_per_ep_rank=[1030, 1022, 1022, 1021, 1024, 1022, 1021, 1030] avg=1024.0000 max=1030 max_rank=0 min=1021 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1024, 1026, 0, 0, 1024, 1028, 0, 0, 1022, 0, 1024, 0, 1022] rerouted_tokens_per_ep_rank=[1022, 1024, 1026, 1024, 1028, 1022, 1024, 1022] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 0, 1027, 1016, 0, 0, 1028, 1017, 0, 0, 1025, 4, 1026, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1027, 1016, 1028, 1017, 1025, 1030, 1024] avg=1024.0000 max=1030 max_rank=6 min=1016 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 1021, 0, 1023, 1024, 0, 0, 1023, 1027, 0, 0, 1021, 0, 1022, 5, 1021] rerouted_tokens_per_ep_rank=[1026, 1023, 1024, 1023, 1027, 1021, 1022, 1026] avg=1024.0000 max=1027 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1024, 1026, 0, 0, 1024, 1028, 0, 0, 1022, 0, 1024, 0, 1022] rerouted_tokens_per_ep_rank=[1022, 1024, 1026, 1024, 1028, 1022, 1024, 1022] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1024, 1026, 0, 0, 1024, 1028, 0, 0, 1021, 1, 1023, 1, 1022] rerouted_tokens_per_ep_rank=[1022, 1024, 1026, 1024, 1028, 1021, 1024, 1023] avg=1024.0000 max=1028 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1024, 1026, 0, 0, 1024, 1028, 0, 0, 1022, 0, 1024, 0, 1022] rerouted_tokens_per_ep_rank=[1022, 1024, 1026, 1024, 1028, 1022, 1024, 1022] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8208, 8176, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8176 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8208, 8176, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8208, 8176, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8208, 8176, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8208, 8176, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8208, 8176, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8208 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8208, 8176, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8208, 8176, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8208 actual_valid_recv=8320 observed_loss=-112 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8176 actual_valid_recv=8320 observed_loss=-144 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4, 4084, 0, 4082, 12, 10] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 4095, 0, 4095, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 4095, 0, 4095, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 1, 4091, 4, 4090, 5, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 0, 1021, 4, 1020, 1023, 0, 0, 1019, 1025, 0, 12, 1018, 10, 1019] rerouted_tokens_per_ep_rank=[1021, 1021, 1024, 1023, 1019, 1025, 1030, 1029] avg=1024.0000 max=1030 max_rank=6 min=1019 min_rank=4 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1, 1023, 1025, 0, 0, 1022, 1027, 0, 1, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1025, 1022, 1027, 1023, 1023] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1026, 0, 0, 1022, 1028, 0, 0, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1026, 1022, 1028, 1022, 1023] avg=1024.0000 max=1028 max_rank=5 min=1022 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 0, 1027, 0, 1028, 1017, 0, 0, 1024, 1016, 0, 0, 1025, 0, 1027] rerouted_tokens_per_ep_rank=[1028, 1027, 1028, 1017, 1024, 1016, 1025, 1027] avg=1024.0000 max=1028 max_rank=0 min=1016 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1023, 1, 1022, 1024, 0, 4, 1021, 1027, 0, 5, 1020, 0, 1021] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1025, 1027, 1025, 1021] avg=1024.0000 max=1027 max_rank=5 min=1021 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1026, 0, 0, 1022, 1028, 0, 0, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1026, 1022, 1028, 1022, 1023] avg=1024.0000 max=1028 max_rank=5 min=1022 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1, 1023, 1025, 0, 0, 1022, 1027, 0, 1, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1025, 1022, 1027, 1023, 1023] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1026, 0, 0, 1022, 1028, 0, 0, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1026, 1022, 1028, 1022, 1023] avg=1024.0000 max=1028 max_rank=5 min=1022 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8178, 8206, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8206 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8178, 8206, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8178, 8206, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8178, 8206, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8178, 8206, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8178, 8206, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8178 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8178, 8206, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8178, 8206, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8206 actual_valid_recv=8320 observed_loss=-114 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8178 actual_valid_recv=8320 observed_loss=-142 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3906, 13, 0, 4083, 13, 0, 177, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4033, 0, 0, 4096, 0, 0, 63, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4051, 1, 0, 4095, 1, 0, 44, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4034, 0, 0, 4096, 0, 0, 62, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4074, 0, 0, 4096, 0, 0, 22, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4033, 1, 0, 4095, 1, 0, 62, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4038, 5, 0, 4091, 5, 0, 53, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4038, 0, 0, 4096, 0, 0, 58, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[997, 0, 13, 991, 0, 1021, 1023, 0, 13, 991, 0, 1018, 177, 927, 0, 1021] rerouted_tokens_per_ep_rank=[997, 1004, 1021, 1023, 1004, 1018, 1104, 1021] avg=1024.0000 max=1104 max_rank=6 min=997 min_rank=0 imbalance_ratio=1.078125 max_load_reduction=7088 max_load_reduction_pct=0.865234 moved_to_echo=4147 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1041, 0, 0, 1033, 0, 1024, 1026, 0, 0, 1033, 0, 1022, 22, 967, 0, 1024] rerouted_tokens_per_ep_rank=[1041, 1033, 1024, 1026, 1033, 1022, 989, 1024] avg=1024.0000 max=1041 max_rank=0 min=989 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4068 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1035, 0, 1, 1027, 0, 1024, 1026, 0, 1, 1027, 0, 1021, 44, 962, 0, 1024] rerouted_tokens_per_ep_rank=[1035, 1028, 1024, 1026, 1028, 1021, 1006, 1024] avg=1024.0000 max=1035 max_rank=0 min=1006 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4079 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1030, 0, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1022, 62, 958, 0, 1024] rerouted_tokens_per_ep_rank=[1030, 1023, 1024, 1026, 1023, 1022, 1020, 1024] avg=1024.0000 max=1030 max_rank=0 min=1020 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1018, 0, 0, 1027, 0, 1028, 1014, 0, 0, 1027, 0, 1026, 63, 961, 0, 1028] rerouted_tokens_per_ep_rank=[1018, 1027, 1028, 1014, 1027, 1026, 1024, 1028] avg=1024.0000 max=1028 max_rank=2 min=1014 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1029, 0, 1, 1023, 0, 1024, 1026, 0, 1, 1023, 0, 1021, 62, 958, 0, 1024] rerouted_tokens_per_ep_rank=[1029, 1024, 1024, 1026, 1024, 1021, 1020, 1024] avg=1024.0000 max=1029 max_rank=0 min=1020 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1031, 0, 5, 1024, 0, 1023, 1025, 0, 5, 1024, 0, 1020, 53, 959, 0, 1023] rerouted_tokens_per_ep_rank=[1031, 1029, 1023, 1025, 1029, 1020, 1012, 1023] avg=1024.0000 max=1031 max_rank=0 min=1012 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4084 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1031, 0, 0, 1024, 0, 1024, 1026, 0, 0, 1024, 0, 1022, 58, 959, 0, 1024] rerouted_tokens_per_ep_rank=[1031, 1024, 1024, 1026, 1024, 1022, 1017, 1024] avg=1024.0000 max=1031 max_rank=0 min=1017 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8212, 8192, 8192, 8192, 8192, 8172, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8212 expected_overflow=0 exact_recv_per_rank=[8212, 8192, 8192, 8192, 8192, 8172, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8172 expected_overflow=0 exact_recv_per_rank=[8212, 8192, 8192, 8192, 8192, 8172, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8212, 8192, 8192, 8192, 8192, 8172, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8212, 8192, 8192, 8192, 8192, 8172, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8212, 8192, 8192, 8192, 8192, 8172, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8212, 8192, 8192, 8192, 8192, 8172, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8212, 8192, 8192, 8192, 8192, 8172, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8212 actual_valid_recv=8320 observed_loss=-108 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8172 actual_valid_recv=8192 observed_loss=-20 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[640, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[3, 4082, 14, 0, 4081, 0, 12, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 4095, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1, 4091, 5, 0, 4090, 0, 5, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 4095, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1, 1021, 0, 1024, 1029, 0, 0, 1021, 1, 1022, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1022, 1024, 1029, 1021, 1023, 1024] avg=1024.0000 max=1029 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 1015, 0, 0, 1025, 0, 1028, 1018, 0, 0, 1025, 0, 1025, 0, 1028] rerouted_tokens_per_ep_rank=[1028, 1015, 1025, 1028, 1018, 1025, 1025, 1028] avg=1024.0000 max=1028 max_rank=0 min=1015 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 1020, 1022, 0, 14, 1018, 0, 1021, 1025, 0, 0, 1018, 12, 1018, 0, 1021] rerouted_tokens_per_ep_rank=[1023, 1022, 1032, 1021, 1025, 1018, 1030, 1021] avg=1024.0000 max=1032 max_rank=2 min=1018 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 0, 1022, 0, 1024, 1028, 0, 0, 1022, 0, 1022, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1026, 1022, 1024, 1028, 1022, 1022, 1024] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 1022, 1025, 0, 5, 1020, 0, 1023, 1028, 0, 0, 1020, 5, 1020, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1025, 1023, 1028, 1020, 1025, 1023] avg=1024.0000 max=1028 max_rank=4 min=1020 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 0, 1022, 0, 1024, 1028, 0, 0, 1022, 0, 1022, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1026, 1022, 1024, 1028, 1022, 1022, 1024] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1, 1021, 0, 1024, 1029, 0, 0, 1021, 1, 1022, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1022, 1024, 1029, 1021, 1023, 1024] avg=1024.0000 max=1029 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 0, 1022, 0, 1024, 1028, 0, 0, 1022, 0, 1022, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1026, 1022, 1024, 1028, 1022, 1022, 1024] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8213 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8171 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8171 actual_valid_recv=8192 observed_loss=-21 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8213 actual_valid_recv=8320 observed_loss=-107 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 2, 4080, 4081, 13, 14] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 4091, 5, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 0, 1024, 0, 1024, 0, 1024, 1029, 0, 1026, 0, 1, 1021, 1, 1021] rerouted_tokens_per_ep_rank=[1021, 1024, 1024, 1024, 1029, 1026, 1022, 1022] avg=1024.0000 max=1029 max_rank=4 min=1021 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 1018, 0, 1020, 1, 1020, 2, 1020, 1024, 0, 1023, 0, 13, 1018, 14, 1018] rerouted_tokens_per_ep_rank=[1019, 1020, 1021, 1022, 1024, 1023, 1031, 1032] avg=1024.0000 max=1032 max_rank=7 min=1019 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4110 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1024, 0, 1024, 0, 1024, 1028, 0, 1026, 0, 0, 1022, 0, 1022] rerouted_tokens_per_ep_rank=[1022, 1024, 1024, 1024, 1028, 1026, 1022, 1022] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 0, 1029, 0, 1028, 0, 1027, 1018, 0, 1014, 0, 0, 1026, 0, 1025] rerouted_tokens_per_ep_rank=[1025, 1029, 1028, 1027, 1018, 1014, 1026, 1025] avg=1024.0000 max=1029 max_rank=1 min=1014 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1024, 0, 1024, 0, 1024, 1028, 0, 1026, 0, 0, 1022, 0, 1022] rerouted_tokens_per_ep_rank=[1022, 1024, 1024, 1024, 1028, 1026, 1022, 1022] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 0, 1024, 0, 1024, 0, 1024, 1029, 0, 1026, 0, 1, 1021, 1, 1021] rerouted_tokens_per_ep_rank=[1021, 1024, 1024, 1024, 1029, 1026, 1022, 1022] avg=1024.0000 max=1029 max_rank=4 min=1021 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1020, 0, 1023, 0, 1023, 0, 1023, 1028, 0, 1025, 0, 5, 1020, 5, 1020] rerouted_tokens_per_ep_rank=[1020, 1023, 1023, 1023, 1028, 1025, 1025, 1025] avg=1024.0000 max=1028 max_rank=4 min=1020 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1024, 0, 1024, 0, 1024, 1028, 0, 1026, 0, 0, 1022, 0, 1022] rerouted_tokens_per_ep_rank=[1022, 1024, 1024, 1024, 1028, 1026, 1022, 1022] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8172, 8192, 8192, 8192, 8212, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8212 expected_overflow=0 exact_recv_per_rank=[8172, 8192, 8192, 8192, 8212, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8172 expected_overflow=0 exact_recv_per_rank=[8172, 8192, 8192, 8192, 8212, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8172, 8192, 8192, 8192, 8212, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8172, 8192, 8192, 8192, 8212, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8172, 8192, 8192, 8192, 8212, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8172, 8192, 8192, 8192, 8212, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8172, 8192, 8192, 8192, 8212, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8172 actual_valid_recv=8320 observed_loss=-148 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8212 actual_valid_recv=8320 observed_loss=-108 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4095, 4095, 1, 0, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4095, 4095, 1, 0, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4081, 4082, 12, 0, 2, 0, 15, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4089, 4091, 5, 0, 0, 2, 5, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1029, 0, 1026, 0, 0, 1022, 0, 1024, 0, 1022, 0, 1024, 0, 1021, 0, 1024] rerouted_tokens_per_ep_rank=[1029, 1026, 1022, 1024, 1022, 1024, 1021, 1024] avg=1024.0000 max=1029 max_rank=0 min=1021 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 1026, 0, 1, 1022, 0, 1024, 0, 1021, 0, 1024, 1, 1021, 0, 1024] rerouted_tokens_per_ep_rank=[1028, 1026, 1023, 1024, 1021, 1024, 1022, 1024] avg=1024.0000 max=1028 max_rank=0 min=1021 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1022, 0, 12, 1018, 0, 1021, 2, 1018, 0, 1020, 15, 1018, 0, 1021] rerouted_tokens_per_ep_rank=[1025, 1022, 1030, 1021, 1020, 1020, 1033, 1021] avg=1024.0000 max=1033 max_rank=6 min=1020 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 1015, 0, 0, 1025, 0, 1028, 0, 1025, 0, 1028, 0, 1027, 0, 1028] rerouted_tokens_per_ep_rank=[1016, 1015, 1025, 1028, 1025, 1028, 1027, 1028] avg=1024.0000 max=1028 max_rank=3 min=1015 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1029, 0, 1026, 0, 0, 1022, 0, 1024, 0, 1022, 0, 1024, 0, 1021, 0, 1024] rerouted_tokens_per_ep_rank=[1029, 1026, 1022, 1024, 1022, 1024, 1021, 1024] avg=1024.0000 max=1029 max_rank=0 min=1021 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 1026, 0, 1, 1022, 0, 1024, 0, 1021, 0, 1024, 1, 1021, 0, 1024] rerouted_tokens_per_ep_rank=[1028, 1026, 1023, 1024, 1021, 1024, 1022, 1024] avg=1024.0000 max=1028 max_rank=0 min=1021 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 1025, 0, 5, 1020, 0, 1023, 0, 1020, 2, 1022, 5, 1020, 0, 1023] rerouted_tokens_per_ep_rank=[1027, 1025, 1025, 1023, 1020, 1024, 1025, 1023] avg=1024.0000 max=1027 max_rank=0 min=1020 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1029, 0, 1026, 0, 0, 1022, 0, 1024, 0, 1022, 0, 1024, 0, 1021, 0, 1024] rerouted_tokens_per_ep_rank=[1029, 1026, 1022, 1024, 1022, 1024, 1021, 1024] avg=1024.0000 max=1029 max_rank=0 min=1021 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8211, 8192, 8192, 8192, 8173, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8211, 8192, 8192, 8192, 8173, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8211, 8192, 8192, 8192, 8173, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8211 expected_overflow=0 exact_recv_per_rank=[8211, 8192, 8192, 8192, 8173, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8173 expected_overflow=0 exact_recv_per_rank=[8211, 8192, 8192, 8192, 8173, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8211, 8192, 8192, 8192, 8173, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8211, 8192, 8192, 8192, 8173, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8211, 8192, 8192, 8192, 8173, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8173 actual_valid_recv=8320 observed_loss=-147 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8211 actual_valid_recv=8320 observed_loss=-109 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4095, 4095, 1, 0, 1, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4080, 4079, 8, 2, 16, 0, 0, 7] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4095, 4095, 0, 0, 1, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4090, 4090, 0, 1, 5, 0, 0, 6] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 1029, 0, 0, 1023, 0, 1021, 0, 1021, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1027, 1029, 1023, 1021, 1021, 1024, 1024, 1023] avg=1024.0000 max=1029 max_rank=1 min=1021 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1029, 0, 1, 1023, 0, 1021, 1, 1021, 0, 1024, 0, 1024, 0, 1022] rerouted_tokens_per_ep_rank=[1026, 1029, 1024, 1021, 1022, 1024, 1024, 1022] avg=1024.0000 max=1029 max_rank=1 min=1021 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 1029, 0, 0, 1023, 0, 1021, 0, 1021, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1027, 1029, 1023, 1021, 1021, 1024, 1024, 1023] avg=1024.0000 max=1029 max_rank=1 min=1021 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1012, 0, 1016, 0, 0, 1027, 0, 1026, 0, 1027, 0, 1029, 0, 1029, 0, 1026] rerouted_tokens_per_ep_rank=[1012, 1016, 1027, 1026, 1027, 1029, 1029, 1026] avg=1024.0000 max=1029 max_rank=5 min=1012 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4111 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1023, 0, 1025, 0, 8, 1019, 2, 1017, 16, 1017, 0, 1020, 0, 1020, 7, 1018] rerouted_tokens_per_ep_rank=[1023, 1025, 1027, 1019, 1033, 1020, 1020, 1025] avg=1024.0000 max=1033 max_rank=4 min=1019 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1029, 0, 0, 1023, 0, 1021, 1, 1021, 0, 1024, 0, 1024, 1, 1022] rerouted_tokens_per_ep_rank=[1026, 1029, 1023, 1021, 1022, 1024, 1024, 1023] avg=1024.0000 max=1029 max_rank=1 min=1021 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 1027, 0, 0, 1022, 1, 1020, 5, 1020, 0, 1023, 0, 1023, 6, 1021] rerouted_tokens_per_ep_rank=[1024, 1027, 1022, 1021, 1025, 1023, 1023, 1027] avg=1024.0000 max=1027 max_rank=1 min=1021 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 1029, 0, 0, 1023, 0, 1021, 0, 1021, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1027, 1029, 1023, 1021, 1021, 1024, 1024, 1023] avg=1024.0000 max=1029 max_rank=1 min=1021 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8213, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8213, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8213, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8171 expected_overflow=0 exact_recv_per_rank=[8192, 8213, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8213, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8213, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8213, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8213 expected_overflow=0 exact_recv_per_rank=[8192, 8213, 8192, 8171, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8213 actual_valid_recv=8320 observed_loss=-107 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8171 actual_valid_recv=8320 observed_loss=-149 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 236, 3860, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 369, 3727, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 298, 3798, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4095, 227, 3868, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[2, 17, 14, 1, 4079, 506, 3573, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1, 6, 3, 2, 4090, 508, 3582, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 4095, 400, 3695, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 176, 3920, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1013, 0, 1014, 0, 1022, 1026, 0, 369, 679, 1021, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1013, 1014, 1022, 1026, 1048, 1021, 1024] avg=1024.0000 max=1048 max_rank=5 min=1013 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=7144 max_load_reduction_pct=0.872070 moved_to_echo=4119 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 0, 1053, 0, 1054, 0, 1025, 1015, 0, 236, 707, 1046, 0, 0, 1029] rerouted_tokens_per_ep_rank=[1027, 1053, 1054, 1025, 1015, 943, 1046, 1029] avg=1024.0000 max=1054 max_rank=2 min=943 min_rank=5 imbalance_ratio=1.029297 max_load_reduction=7138 max_load_reduction_pct=0.871338 moved_to_echo=4033 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1033, 0, 1034, 0, 1022, 1026, 0, 298, 692, 1039, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1033, 1034, 1022, 1026, 990, 1039, 1024] avg=1024.0000 max=1039 max_rank=6 min=990 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=7153 max_load_reduction_pct=0.873169 moved_to_echo=4079 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1, 1052, 1, 1053, 0, 1021, 1026, 0, 227, 704, 1059, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1053, 1054, 1021, 1026, 931, 1059, 1024] avg=1024.0000 max=1059 max_rank=6 min=931 min_rank=5 imbalance_ratio=1.034180 max_load_reduction=7133 max_load_reduction_pct=0.870728 moved_to_echo=4040 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 1020, 17, 971, 14, 972, 1, 1017, 1022, 0, 506, 651, 979, 0, 0, 1020] rerouted_tokens_per_ep_rank=[1022, 988, 986, 1018, 1022, 1157, 979, 1020] avg=1024.0000 max=1157 max_rank=5 min=979 min_rank=6 imbalance_ratio=1.129883 max_load_reduction=7035 max_load_reduction_pct=0.858765 moved_to_echo=4178 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 1022, 6, 974, 3, 975, 2, 1020, 1025, 0, 508, 652, 981, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 980, 978, 1022, 1025, 1160, 981, 1023] avg=1024.0000 max=1160 max_rank=5 min=978 min_rank=2 imbalance_ratio=1.132812 max_load_reduction=7032 max_load_reduction_pct=0.858398 moved_to_echo=4189 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1, 1005, 0, 1005, 1, 1021, 1026, 0, 400, 673, 1012, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1006, 1005, 1022, 1026, 1073, 1012, 1024] avg=1024.0000 max=1073 max_rank=5 min=1005 min_rank=2 imbalance_ratio=1.047852 max_load_reduction=7119 max_load_reduction_pct=0.869019 moved_to_echo=4135 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1066, 0, 1067, 0, 1022, 1026, 0, 176, 714, 1073, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1066, 1067, 1022, 1026, 890, 1073, 1024] avg=1024.0000 max=1073 max_rank=6 min=890 min_rank=5 imbalance_ratio=1.047852 max_load_reduction=7119 max_load_reduction_pct=0.869019 moved_to_echo=4013 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8174 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8174, 8192, 8192, 8210, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8174, 8192, 8192, 8210, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8174, 8192, 8192, 8210, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8210 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8174, 8192, 8192, 8210, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8174, 8192, 8192, 8210, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8174, 8192, 8192, 8210, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8174, 8192, 8192, 8210, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8174, 8192, 8192, 8210, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2816, 5504] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8174 actual_valid_recv=8320 observed_loss=-146 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8210 actual_valid_recv=8320 observed_loss=-110 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 1, 4095, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[16, 0, 17, 4079, 4077, 3, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 1, 4095, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 8, 0, 4096, 4088, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[6, 0, 5, 4089, 4090, 2, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[16, 1017, 0, 1019, 17, 1017, 1022, 0, 1024, 0, 3, 1017, 0, 1020, 0, 1020] rerouted_tokens_per_ep_rank=[1033, 1019, 1034, 1022, 1024, 1020, 1020, 1020] avg=1024.0000 max=1034 max_rank=2 min=1019 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4084 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1021, 0, 1023, 1, 1021, 1026, 0, 1030, 0, 0, 1021, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1022, 1023, 1022, 1026, 1030, 1021, 1024, 1024] avg=1024.0000 max=1030 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1024, 0, 1022, 1027, 0, 1028, 0, 0, 1021, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1022, 1024, 1022, 1027, 1028, 1021, 1024, 1024] avg=1024.0000 max=1028 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1024, 0, 1022, 1027, 0, 1028, 0, 0, 1021, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1022, 1024, 1022, 1027, 1028, 1021, 1024, 1024] avg=1024.0000 max=1028 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1021, 0, 1023, 1, 1021, 1026, 0, 1030, 0, 0, 1021, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1022, 1023, 1022, 1026, 1030, 1021, 1024, 1024] avg=1024.0000 max=1030 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 8, 1025, 0, 1023, 1013, 0, 1017, 0, 0, 1025, 0, 1029, 0, 1029] rerouted_tokens_per_ep_rank=[1023, 1033, 1023, 1013, 1017, 1025, 1029, 1029] avg=1024.0000 max=1033 max_rank=1 min=1013 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 1020, 0, 1022, 5, 1020, 1024, 0, 1028, 0, 2, 1019, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1022, 1025, 1024, 1028, 1021, 1023, 1023] avg=1024.0000 max=1028 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1024, 0, 1022, 1027, 0, 1028, 0, 0, 1021, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1022, 1024, 1022, 1027, 1028, 1021, 1024, 1024] avg=1024.0000 max=1028 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8171 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8213 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8213, 8171, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8213 actual_valid_recv=8320 observed_loss=-107 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8171 actual_valid_recv=8320 observed_loss=-149 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4079, 4016, 16, 10, 6, 1, 64, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4095, 4092, 1, 1, 0, 0, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4095, 4094, 1, 0, 1, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4090, 4073, 6, 5, 1, 0, 17, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1022, 0, 1011, 0, 16, 1004, 10, 1005, 6, 1017, 1, 1020, 64, 996, 0, 1020] rerouted_tokens_per_ep_rank=[1022, 1011, 1020, 1015, 1023, 1021, 1060, 1020] avg=1024.0000 max=1060 max_rank=6 min=1011 min_rank=1 imbalance_ratio=1.035156 max_load_reduction=7132 max_load_reduction_pct=0.870605 moved_to_echo=4124 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 1031, 0, 0, 1024, 0, 1025, 0, 1021, 0, 1024, 0, 1016, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1031, 1024, 1025, 1021, 1024, 1016, 1024] avg=1024.0000 max=1031 max_rank=1 min=1016 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4085 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1031, 0, 1, 1023, 0, 1024, 1, 1021, 0, 1024, 1, 1016, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1031, 1024, 1024, 1022, 1024, 1017, 1024] avg=1024.0000 max=1031 max_rank=1 min=1017 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 1031, 0, 0, 1024, 0, 1025, 0, 1021, 0, 1024, 0, 1016, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1031, 1024, 1025, 1021, 1024, 1016, 1024] avg=1024.0000 max=1031 max_rank=1 min=1016 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4085 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1013, 0, 1018, 0, 0, 1028, 0, 1029, 0, 1026, 0, 1028, 0, 1021, 0, 1029] rerouted_tokens_per_ep_rank=[1013, 1018, 1028, 1029, 1026, 1028, 1021, 1029] avg=1024.0000 max=1029 max_rank=3 min=1013 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1030, 0, 1, 1023, 1, 1024, 0, 1021, 0, 1024, 3, 1015, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1030, 1024, 1025, 1021, 1024, 1018, 1024] avg=1024.0000 max=1030 max_rank=1 min=1018 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 1025, 0, 6, 1018, 5, 1019, 1, 1020, 0, 1023, 17, 1011, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1025, 1024, 1024, 1021, 1023, 1028, 1023] avg=1024.0000 max=1028 max_rank=6 min=1021 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 1031, 0, 0, 1024, 0, 1025, 0, 1021, 0, 1024, 0, 1016, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1031, 1024, 1025, 1021, 1024, 1016, 1024] avg=1024.0000 max=1031 max_rank=1 min=1016 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4085 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8208 expected_overflow=0 exact_recv_per_rank=[8192, 8208, 8192, 8192, 8176, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8208, 8192, 8192, 8176, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8208, 8192, 8192, 8176, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8208, 8192, 8192, 8176, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8208, 8192, 8192, 8176, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8176 expected_overflow=0 exact_recv_per_rank=[8192, 8208, 8192, 8192, 8176, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8208, 8192, 8192, 8176, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8208, 8192, 8192, 8176, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8208 actual_valid_recv=8320 observed_loss=-112 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8176 actual_valid_recv=8320 observed_loss=-144 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 1, 4095, 0, 0, 4053, 1, 42] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 1, 4095, 0, 0, 4037, 1, 58] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1, 6, 4090, 0, 0, 4041, 5, 49] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4030, 0, 66] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4040, 0, 56] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1, 17, 4077, 1, 0, 3920, 15, 161] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4073, 0, 23] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4040, 0, 56] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1, 1026, 1026, 0, 0, 1024, 0, 1024, 1035, 0, 1, 1027, 42, 965] rerouted_tokens_per_ep_rank=[1021, 1027, 1026, 1024, 1024, 1035, 1028, 1007] avg=1024.0000 max=1035 max_rank=5 min=1007 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 0, 1026, 1013, 0, 0, 1029, 0, 1029, 1014, 0, 0, 1025, 66, 965] rerouted_tokens_per_ep_rank=[1025, 1026, 1013, 1029, 1029, 1014, 1025, 1031] avg=1024.0000 max=1031 max_rank=7 min=1013 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 1016, 17, 993, 1022, 0, 1, 1019, 0, 1020, 1000, 0, 15, 993, 161, 934] rerouted_tokens_per_ep_rank=[1017, 1010, 1022, 1020, 1020, 1000, 1008, 1095] avg=1024.0000 max=1095 max_rank=7 min=1000 min_rank=5 imbalance_ratio=1.069336 max_load_reduction=7097 max_load_reduction_pct=0.866333 moved_to_echo=4123 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 0, 1023, 1027, 0, 0, 1024, 0, 1024, 1032, 0, 0, 1023, 56, 962] rerouted_tokens_per_ep_rank=[1021, 1023, 1027, 1024, 1024, 1032, 1023, 1018] avg=1024.0000 max=1032 max_rank=5 min=1018 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 0, 1031, 1027, 0, 0, 1024, 0, 1024, 1040, 0, 0, 1032, 23, 970] rerouted_tokens_per_ep_rank=[1021, 1031, 1027, 1024, 1024, 1040, 1032, 993] avg=1024.0000 max=1040 max_rank=5 min=993 min_rank=7 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 1020, 6, 1023, 1024, 0, 0, 1023, 0, 1023, 1032, 0, 5, 1024, 49, 962] rerouted_tokens_per_ep_rank=[1021, 1029, 1024, 1023, 1023, 1032, 1029, 1011] avg=1024.0000 max=1032 max_rank=5 min=1011 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1, 1022, 1026, 0, 0, 1024, 0, 1024, 1031, 0, 1, 1023, 58, 961] rerouted_tokens_per_ep_rank=[1021, 1023, 1026, 1024, 1024, 1031, 1024, 1019] avg=1024.0000 max=1031 max_rank=5 min=1019 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 0, 1023, 1027, 0, 0, 1024, 0, 1024, 1032, 0, 0, 1023, 56, 962] rerouted_tokens_per_ep_rank=[1021, 1023, 1027, 1024, 1024, 1032, 1023, 1018] avg=1024.0000 max=1032 max_rank=5 min=1018 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8168, 8192, 8192, 8192, 8192, 8216, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8168 expected_overflow=0 exact_recv_per_rank=[8168, 8192, 8192, 8192, 8192, 8216, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8168, 8192, 8192, 8192, 8192, 8216, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8168, 8192, 8192, 8192, 8192, 8216, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8216 expected_overflow=0 exact_recv_per_rank=[8168, 8192, 8192, 8192, 8192, 8216, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8168, 8192, 8192, 8192, 8192, 8216, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8168, 8192, 8192, 8192, 8192, 8216, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8168, 8192, 8192, 8192, 8192, 8216, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8168 actual_valid_recv=8320 observed_loss=-152 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8216 actual_valid_recv=8320 observed_loss=-104 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 4086, 10] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 18, 4078, 3755, 323] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 4092, 4] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 4095, 4060, 35] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 6, 4090, 3962, 128] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 4095, 4020, 75] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 4080, 16] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1017, 0, 1020, 0, 1020, 18, 953, 18, 953, 1021, 0, 962, 0, 323, 887] rerouted_tokens_per_ep_rank=[1017, 1020, 1020, 971, 971, 1021, 962, 1210] avg=1024.0000 max=1210 max_rank=7 min=962 min_rank=6 imbalance_ratio=1.181641 max_load_reduction=6982 max_load_reduction_pct=0.852295 moved_to_echo=4164 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 0, 1024, 0, 1024, 0, 1040, 0, 1040, 1027, 0, 1048, 0, 0, 968] rerouted_tokens_per_ep_rank=[1021, 1024, 1024, 1040, 1040, 1027, 1048, 968] avg=1024.0000 max=1048 max_rank=6 min=968 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=7144 max_load_reduction_pct=0.872070 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1029, 0, 1029, 0, 1040, 0, 1040, 1014, 0, 1036, 0, 10, 970] rerouted_tokens_per_ep_rank=[1024, 1029, 1029, 1040, 1040, 1014, 1036, 980] avg=1024.0000 max=1040 max_rank=3 min=980 min_rank=7 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4070 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 0, 1024, 0, 1024, 0, 1039, 0, 1039, 1027, 0, 1047, 0, 4, 967] rerouted_tokens_per_ep_rank=[1021, 1024, 1024, 1039, 1039, 1027, 1047, 971] avg=1024.0000 max=1047 max_rank=6 min=971 min_rank=7 imbalance_ratio=1.022461 max_load_reduction=7145 max_load_reduction_pct=0.872192 moved_to_echo=4084 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 0, 1024, 0, 1024, 1, 1031, 1, 1031, 1026, 0, 1039, 0, 35, 959] rerouted_tokens_per_ep_rank=[1021, 1024, 1024, 1032, 1032, 1026, 1039, 994] avg=1024.0000 max=1039 max_rank=6 min=994 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=7153 max_load_reduction_pct=0.873169 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1020, 0, 1023, 0, 1023, 6, 1006, 6, 1006, 1024, 0, 1014, 0, 128, 936] rerouted_tokens_per_ep_rank=[1020, 1023, 1023, 1012, 1012, 1024, 1014, 1064] avg=1024.0000 max=1064 max_rank=7 min=1012 min_rank=3 imbalance_ratio=1.039062 max_load_reduction=7128 max_load_reduction_pct=0.870117 moved_to_echo=4114 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 0, 1024, 0, 1024, 1, 1021, 1, 1021, 1026, 0, 1028, 0, 75, 950] rerouted_tokens_per_ep_rank=[1021, 1024, 1024, 1022, 1022, 1026, 1028, 1025] avg=1024.0000 max=1028 max_rank=6 min=1021 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 0, 1024, 0, 1024, 0, 1036, 0, 1036, 1027, 0, 1044, 0, 16, 964] rerouted_tokens_per_ep_rank=[1021, 1024, 1024, 1036, 1036, 1027, 1044, 980] avg=1024.0000 max=1044 max_rank=6 min=980 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=7148 max_load_reduction_pct=0.872559 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8166, 8192, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8166, 8192, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8166, 8192, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8166, 8192, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8166 expected_overflow=0 exact_recv_per_rank=[8166, 8192, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8218 expected_overflow=0 exact_recv_per_rank=[8166, 8192, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8166, 8192, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8166, 8192, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8166 actual_valid_recv=8192 observed_loss=-26 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[640, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8218 actual_valid_recv=8320 observed_loss=-102 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 4095, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[18, 0, 0, 0, 12, 18, 4066, 4078] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 1, 4094, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 3, 6, 4087, 4090] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1021, 0, 1021, 0, 1024, 0, 1024, 0, 1023, 1, 1021, 1030, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1022, 1021, 1024, 1024, 1023, 1022, 1030, 1026] avg=1024.0000 max=1030 max_rank=6 min=1021 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1021, 0, 1024, 0, 1024, 0, 1023, 0, 1022, 1029, 0, 1027, 0] rerouted_tokens_per_ep_rank=[1022, 1021, 1024, 1024, 1023, 1022, 1029, 1027] avg=1024.0000 max=1029 max_rank=6 min=1021 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[18, 1014, 0, 1017, 0, 1020, 0, 1020, 12, 1015, 18, 1014, 1023, 0, 1021, 0] rerouted_tokens_per_ep_rank=[1032, 1017, 1020, 1020, 1027, 1032, 1023, 1021] avg=1024.0000 max=1032 max_rank=0 min=1017 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1021, 0, 1024, 0, 1024, 0, 1023, 0, 1022, 1029, 0, 1027, 0] rerouted_tokens_per_ep_rank=[1022, 1021, 1024, 1024, 1023, 1022, 1029, 1027] avg=1024.0000 max=1029 max_rank=6 min=1021 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 0, 1024, 0, 1029, 0, 1029, 0, 1026, 0, 1025, 1020, 0, 1014, 0] rerouted_tokens_per_ep_rank=[1025, 1024, 1029, 1029, 1026, 1025, 1020, 1014] avg=1024.0000 max=1029 max_rank=2 min=1014 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4085 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[6, 1019, 0, 1020, 0, 1023, 0, 1023, 3, 1021, 6, 1019, 1028, 0, 1024, 0] rerouted_tokens_per_ep_rank=[1025, 1020, 1023, 1023, 1024, 1025, 1028, 1024] avg=1024.0000 max=1028 max_rank=6 min=1020 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1021, 0, 1021, 0, 1024, 0, 1024, 1, 1022, 1, 1021, 1030, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1022, 1021, 1024, 1024, 1023, 1022, 1030, 1026] avg=1024.0000 max=1030 max_rank=6 min=1021 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1021, 0, 1024, 0, 1024, 0, 1023, 0, 1022, 1029, 0, 1027, 0] rerouted_tokens_per_ep_rank=[1022, 1021, 1024, 1024, 1023, 1022, 1029, 1027] avg=1024.0000 max=1029 max_rank=6 min=1021 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8166, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8166, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8166, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8166, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8166, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8218 expected_overflow=0 exact_recv_per_rank=[8192, 8166, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8166, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8166 expected_overflow=0 exact_recv_per_rank=[8192, 8166, 8192, 8192, 8192, 8192, 8218, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8166 actual_valid_recv=8192 observed_loss=-26 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8218 actual_valid_recv=8320 observed_loss=-102 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 1, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 1, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1, 4077, 1, 0, 0, 16, 4078, 19] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4090, 1, 0, 0, 5, 4090, 6] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1017, 1025, 0, 1, 1019, 0, 1020, 0, 1020, 16, 1017, 1021, 0, 19, 1016] rerouted_tokens_per_ep_rank=[1018, 1025, 1020, 1020, 1020, 1033, 1021, 1035] avg=1024.0000 max=1035 max_rank=7 min=1018 min_rank=0 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4109 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1030, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1021, 1027, 0, 0, 1021] rerouted_tokens_per_ep_rank=[1021, 1030, 1024, 1024, 1024, 1021, 1027, 1021] avg=1024.0000 max=1030 max_rank=1 min=1021 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1016, 0, 0, 1028, 0, 1029, 0, 1029, 0, 1027, 1014, 0, 0, 1025] rerouted_tokens_per_ep_rank=[1024, 1016, 1028, 1029, 1029, 1027, 1014, 1025] avg=1024.0000 max=1029 max_rank=3 min=1014 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1029, 0, 0, 1024, 0, 1024, 0, 1024, 1, 1021, 1026, 0, 1, 1021] rerouted_tokens_per_ep_rank=[1021, 1029, 1024, 1024, 1024, 1022, 1026, 1022] avg=1024.0000 max=1029 max_rank=1 min=1021 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1030, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1021, 1027, 0, 0, 1021] rerouted_tokens_per_ep_rank=[1021, 1030, 1024, 1024, 1024, 1021, 1027, 1021] avg=1024.0000 max=1030 max_rank=1 min=1021 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1020, 1028, 0, 1, 1023, 0, 1023, 0, 1023, 5, 1020, 1024, 0, 6, 1019] rerouted_tokens_per_ep_rank=[1020, 1028, 1024, 1023, 1023, 1025, 1024, 1025] avg=1024.0000 max=1028 max_rank=1 min=1020 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1029, 0, 0, 1024, 0, 1024, 0, 1024, 1, 1021, 1026, 0, 1, 1021] rerouted_tokens_per_ep_rank=[1021, 1029, 1024, 1024, 1024, 1022, 1026, 1022] avg=1024.0000 max=1029 max_rank=1 min=1021 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1030, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1021, 1027, 0, 0, 1021] rerouted_tokens_per_ep_rank=[1021, 1030, 1024, 1024, 1024, 1021, 1027, 1021] avg=1024.0000 max=1030 max_rank=1 min=1021 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8167, 8217, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8167, 8217, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8167, 8217, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8167, 8217, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8217 expected_overflow=0 exact_recv_per_rank=[8167, 8217, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8167, 8217, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8167, 8217, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8167 expected_overflow=0 exact_recv_per_rank=[8167, 8217, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8167 actual_valid_recv=8320 observed_loss=-153 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8217 actual_valid_recv=8320 observed_loss=-103 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True + [2026-08-30 04:54:05] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 671.8 | throughput per GPU (TFLOP/s/GPU): 496.3 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.962954E+00 | loss scale: 1.0 | grad norm: 21.474 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3587, 0, 0, 0, 1665, 0, 2940, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3895, 11, 0, 0, 1561, 0, 2725, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3602, 2, 0, 2, 1668, 3, 2914, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[3423, 5, 3, 0, 1729, 0, 3032, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3923, 3, 0, 0, 1106, 0, 3160, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3650, 0, 0, 0, 1486, 4, 3052, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3164, 0, 0, 0, 2206, 0, 2822, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3803, 0, 0, 0, 1358, 1, 3030, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1100, 0, 11, 560, 0, 599, 0, 1098, 1001, 0, 0, 941, 1784, 0, 0, 1098] rerouted_tokens_per_ep_rank=[1100, 571, 599, 1098, 1001, 941, 1784, 1098] avg=1024.0000 max=1784 max_rank=6 min=571 min_rank=1 imbalance_ratio=1.742188 max_load_reduction=6408 max_load_reduction_pct=0.782227 moved_to_echo=4824 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1030, 0, 0, 533, 0, 562, 0, 1029, 953, 0, 4, 1055, 1997, 0, 0, 1029] rerouted_tokens_per_ep_rank=[1030, 533, 562, 1029, 953, 1059, 1997, 1029] avg=1024.0000 max=1997 max_rank=6 min=533 min_rank=1 imbalance_ratio=1.950195 max_load_reduction=6195 max_load_reduction_pct=0.756226 moved_to_echo=5038 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[957, 0, 5, 625, 3, 531, 0, 967, 1104, 0, 0, 1051, 1981, 0, 0, 968] rerouted_tokens_per_ep_rank=[957, 630, 534, 967, 1104, 1051, 1981, 968] avg=1024.0000 max=1981 max_rank=6 min=534 min_rank=2 imbalance_ratio=1.934570 max_load_reduction=6211 max_load_reduction_pct=0.758179 moved_to_echo=5104 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1107, 0, 3, 396, 0, 604, 0, 1106, 710, 0, 0, 1092, 2068, 0, 0, 1106] rerouted_tokens_per_ep_rank=[1107, 399, 604, 1106, 710, 1092, 2068, 1106] avg=1024.0000 max=2068 max_rank=6 min=399 min_rank=1 imbalance_ratio=2.019531 max_load_reduction=6124 max_load_reduction_pct=0.747559 moved_to_echo=4976 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1074, 0, 0, 487, 0, 585, 0, 1072, 871, 0, 1, 1047, 1983, 0, 0, 1072] rerouted_tokens_per_ep_rank=[1074, 487, 585, 1072, 871, 1048, 1983, 1072] avg=1024.0000 max=1983 max_rank=6 min=487 min_rank=1 imbalance_ratio=1.936523 max_load_reduction=6209 max_load_reduction_pct=0.757935 moved_to_echo=4974 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1013, 0, 0, 597, 0, 552, 0, 1011, 1068, 0, 0, 1016, 1924, 0, 0, 1011] rerouted_tokens_per_ep_rank=[1013, 597, 552, 1011, 1068, 1016, 1924, 1011] avg=1024.0000 max=1924 max_rank=6 min=552 min_rank=2 imbalance_ratio=1.878906 max_load_reduction=6268 max_load_reduction_pct=0.765137 moved_to_echo=5019 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1018, 0, 2, 598, 0, 554, 2, 1015, 1070, 0, 3, 1007, 1907, 0, 1, 1015] rerouted_tokens_per_ep_rank=[1018, 600, 554, 1017, 1070, 1010, 1907, 1016] avg=1024.0000 max=1907 max_rank=6 min=554 min_rank=2 imbalance_ratio=1.862305 max_load_reduction=6285 max_load_reduction_pct=0.767212 moved_to_echo=5003 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[893, 0, 0, 791, 0, 487, 0, 892, 1415, 0, 0, 975, 1847, 0, 0, 892] rerouted_tokens_per_ep_rank=[893, 791, 487, 892, 1415, 975, 1847, 892] avg=1024.0000 max=1847 max_rank=6 min=487 min_rank=2 imbalance_ratio=1.803711 max_load_reduction=6345 max_load_reduction_pct=0.774536 moved_to_echo=5129 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4477 expected_overflow=0 exact_recv_per_rank=[8192, 4608, 4477, 8192, 8192, 8192, 15491, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 4608, 4477, 8192, 8192, 8192, 15491, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 4608, 4477, 8192, 8192, 8192, 15491, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 4608, 4477, 8192, 8192, 8192, 15491, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 4608, 4477, 8192, 8192, 8192, 15491, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 4608, 4477, 8192, 8192, 8192, 15491, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4608 expected_overflow=0 exact_recv_per_rank=[8192, 4608, 4477, 8192, 8192, 8192, 15491, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=15491 expected_overflow=0 exact_recv_per_rank=[8192, 4608, 4477, 8192, 8192, 8192, 15491, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=4477 actual_valid_recv=4608 observed_loss=-131 tokens_per_expert=[128, 4480] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=4608 actual_valid_recv=4736 observed_loss=-128 tokens_per_expert=[128, 4608] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=15491 actual_valid_recv=15616 observed_loss=-125 tokens_per_expert=[15616, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=4608 slack_slots=11776 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=4736 slack_slots=11648 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=15616 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4095, 1, 4095, 0, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4094, 0, 0, 0, 0, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4090, 1, 4095, 0, 0, 0, 0, 6] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 1, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 0, 1022] rerouted_tokens_per_ep_rank=[1027, 1024, 1026, 1023, 1023, 1023, 1024, 1022] avg=1024.0000 max=1027 max_rank=0 min=1022 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1024, 2, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1023, 1023, 1023, 1024, 1025] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1013, 0, 1, 1029, 1016, 0, 0, 1026, 0, 1026, 0, 1027, 0, 1024, 6, 1024] rerouted_tokens_per_ep_rank=[1013, 1030, 1016, 1026, 1026, 1027, 1024, 1030] avg=1024.0000 max=1030 max_rank=1 min=1013 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4107 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1024, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1024, 1024, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1024, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1024, 1024, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1024, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1024, 1024, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1024, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1024, 1024, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1024, 0, 1023, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1025, 1024, 1024, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8196, 8192, 8192, 8192, 8192, 8188, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8188 actual_valid_recv=8192 observed_loss=-4 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1157, 4096, 0, 2937, 2, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 1273, 4096, 0, 2823, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1369, 4096, 0, 2727, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1, 1064, 4095, 0, 3021, 1, 9, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1, 1064, 4095, 0, 3030, 1, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1033, 4096, 0, 3063, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 1176, 4096, 0, 2910, 7, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 932, 4096, 0, 3157, 0, 7, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1033, 0, 1026, 0, 0, 1024, 1061, 0, 0, 1058, 0, 944, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1033, 1026, 1024, 1061, 1058, 944, 1023] avg=1024.0000 max=1061 max_rank=4 min=944 min_rank=6 imbalance_ratio=1.036133 max_load_reduction=7131 max_load_reduction_pct=0.870483 moved_to_echo=4086 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1064, 0, 1026, 0, 0, 1023, 1049, 0, 1, 1047, 0, 934, 1, 1023] rerouted_tokens_per_ep_rank=[1024, 1064, 1026, 1023, 1049, 1048, 934, 1024] avg=1024.0000 max=1064 max_rank=1 min=934 min_rank=6 imbalance_ratio=1.039062 max_load_reduction=7128 max_load_reduction_pct=0.870117 moved_to_echo=4055 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1369, 0, 1026, 0, 0, 1024, 945, 0, 0, 942, 0, 840, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1369, 1026, 1024, 945, 942, 840, 1023] avg=1024.0000 max=1369 max_rank=1 min=840 min_rank=6 imbalance_ratio=1.336914 max_load_reduction=6823 max_load_reduction_pct=0.832886 moved_to_echo=3750 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 932, 0, 1026, 0, 0, 1024, 1094, 0, 0, 1090, 7, 973, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 932, 1026, 1024, 1094, 1090, 980, 1023] avg=1024.0000 max=1094 max_rank=4 min=932 min_rank=1 imbalance_ratio=1.068359 max_load_reduction=7098 max_load_reduction_pct=0.866455 moved_to_echo=4187 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1157, 0, 1026, 0, 0, 1024, 1018, 0, 2, 1014, 0, 905, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1157, 1026, 1024, 1018, 1016, 905, 1023] avg=1024.0000 max=1157 max_rank=1 min=905 min_rank=6 imbalance_ratio=1.129883 max_load_reduction=7035 max_load_reduction_pct=0.858765 moved_to_echo=3962 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1029, 1064, 0, 1012, 0, 0, 1025, 1039, 0, 1, 1048, 9, 934, 1, 1029] rerouted_tokens_per_ep_rank=[1030, 1064, 1012, 1025, 1039, 1049, 943, 1030] avg=1024.0000 max=1064 max_rank=1 min=943 min_rank=6 imbalance_ratio=1.039062 max_load_reduction=7128 max_load_reduction_pct=0.870117 moved_to_echo=4061 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1273, 0, 1026, 0, 0, 1024, 978, 0, 0, 975, 0, 870, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1273, 1026, 1024, 978, 975, 870, 1023] avg=1024.0000 max=1273 max_rank=1 min=870 min_rank=6 imbalance_ratio=1.243164 max_load_reduction=6919 max_load_reduction_pct=0.844604 moved_to_echo=3846 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1176, 0, 1026, 0, 0, 1024, 1008, 0, 7, 1005, 3, 897, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1176, 1026, 1024, 1008, 1012, 900, 1023] avg=1024.0000 max=1176 max_rank=1 min=900 min_rank=6 imbalance_ratio=1.148438 max_load_reduction=7016 max_load_reduction_pct=0.856445 moved_to_echo=3943 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 9068, 8194, 8192, 8192, 8190, 7316, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 9068, 8194, 8192, 8192, 8190, 7316, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9068, 8194, 8192, 8192, 8190, 7316, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9068, 8194, 8192, 8192, 8190, 7316, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9068, 8194, 8192, 8192, 8190, 7316, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9068, 8194, 8192, 8192, 8190, 7316, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9068 expected_overflow=0 exact_recv_per_rank=[8192, 9068, 8194, 8192, 8192, 8190, 7316, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7316 expected_overflow=0 exact_recv_per_rank=[8192, 9068, 8194, 8192, 8192, 8190, 7316, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8190 actual_valid_recv=8320 observed_loss=-130 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=7316 actual_valid_recv=7552 observed_loss=-236 tokens_per_expert=[128, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=9068 actual_valid_recv=9088 observed_loss=-20 tokens_per_expert=[9088, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=7552 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=9088 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[3979, 0, 0, 29, 88, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[3973, 0, 1, 47, 75, 4095, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[3974, 0, 0, 25, 96, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[3964, 0, 0, 57, 75, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4007, 0, 0, 15, 74, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4016, 0, 0, 16, 64, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[3888, 0, 0, 111, 97, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[3901, 0, 0, 98, 97, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 0, 1023, 1, 1023, 47, 976, 75, 943, 1026, 0, 1, 1026, 0, 1023] rerouted_tokens_per_ep_rank=[1028, 1023, 1024, 1023, 1018, 1026, 1027, 1023] avg=1024.0000 max=1028 max_rank=0 min=1018 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1030, 0, 0, 1024, 0, 1023, 29, 978, 88, 944, 1026, 0, 0, 1027, 0, 1023] rerouted_tokens_per_ep_rank=[1030, 1024, 1023, 1007, 1032, 1026, 1027, 1023] avg=1024.0000 max=1032 max_rank=4 min=1007 min_rank=3 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 57, 974, 75, 941, 1026, 0, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1031, 1016, 1026, 1024, 1023] avg=1024.0000 max=1031 max_rank=3 min=1016 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1019, 0, 0, 1025, 0, 1030, 25, 980, 96, 945, 1010, 0, 1, 1030, 1, 1030] rerouted_tokens_per_ep_rank=[1019, 1025, 1030, 1005, 1041, 1010, 1031, 1031] avg=1024.0000 max=1041 max_rank=4 min=1005 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4113 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1039, 0, 0, 1024, 0, 1023, 16, 987, 64, 953, 1026, 0, 0, 1037, 0, 1023] rerouted_tokens_per_ep_rank=[1039, 1024, 1023, 1003, 1017, 1026, 1037, 1023] avg=1024.0000 max=1039 max_rank=0 min=1003 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=7153 max_load_reduction_pct=0.873169 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1036, 0, 0, 1024, 0, 1023, 15, 985, 74, 951, 1026, 0, 0, 1035, 0, 1023] rerouted_tokens_per_ep_rank=[1036, 1024, 1023, 1000, 1025, 1026, 1035, 1023] avg=1024.0000 max=1036 max_rank=0 min=1000 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4109 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1006, 0, 0, 1024, 0, 1023, 111, 955, 97, 923, 1026, 0, 0, 1004, 0, 1023] rerouted_tokens_per_ep_rank=[1006, 1024, 1023, 1066, 1020, 1026, 1004, 1023] avg=1024.0000 max=1066 max_rank=3 min=1004 min_rank=6 imbalance_ratio=1.041016 max_load_reduction=7126 max_load_reduction_pct=0.869873 moved_to_echo=4073 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1009, 0, 0, 1024, 0, 1023, 98, 959, 97, 926, 1026, 0, 0, 1007, 0, 1023] rerouted_tokens_per_ep_rank=[1009, 1024, 1023, 1057, 1023, 1026, 1007, 1023] avg=1024.0000 max=1057 max_rank=3 min=1007 min_rank=6 imbalance_ratio=1.032227 max_load_reduction=7135 max_load_reduction_pct=0.870972 moved_to_echo=4079 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[3, 1, 1, 4095, 0, 0, 4092, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 4095, 1, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1025, 1023, 1024, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 1028, 1, 1029, 1, 1024, 1016, 0, 0, 1029, 0, 1025, 1011, 0, 0, 1025] rerouted_tokens_per_ep_rank=[1031, 1030, 1025, 1016, 1029, 1025, 1011, 1025] avg=1024.0000 max=1031 max_rank=0 min=1011 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1023, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1026, 1024, 1023, 1026, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1025, 1023, 1024, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1025, 1023, 1024, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1025, 1023, 1024, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1025, 1023, 1024, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1025, 1023, 1024, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8191, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8191, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8191, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8191, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8191, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8191, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8191, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8191, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8191 actual_valid_recv=8320 observed_loss=-129 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 5, 0, 0, 0, 4091] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 1, 23, 0, 0, 1, 4072] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 0, 1, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 7, 0, 0, 0, 4089] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 3, 0, 0, 0, 4093] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1025, 0, 1020, 0, 1024, 0, 1023, 0, 1025, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1025, 1025, 1020, 1024, 1023, 1025, 1026] avg=1024.0000 max=1026 max_rank=7 min=1020 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1, 1024, 0, 1020, 0, 1023, 0, 1023, 1, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1023, 1026, 1025, 1020, 1023, 1023, 1025, 1027] avg=1024.0000 max=1027 max_rank=7 min=1020 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 5, 1019, 0, 1024, 0, 1023, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1024, 1024, 1023, 1023, 1026] avg=1024.0000 max=1026 max_rank=7 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 1016, 0, 1, 1021, 23, 1018, 0, 1025, 0, 1029, 1, 1021, 1012, 0] rerouted_tokens_per_ep_rank=[1025, 1016, 1022, 1041, 1025, 1029, 1022, 1012] avg=1024.0000 max=1041 max_rank=3 min=1012 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4088 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 7, 1018, 0, 1024, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1025, 1024, 1023, 1023, 1025] avg=1024.0000 max=1025 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1025, 0, 1020, 0, 1024, 0, 1023, 0, 1025, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1025, 1025, 1020, 1024, 1023, 1025, 1026] avg=1024.0000 max=1026 max_rank=7 min=1020 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1025, 0, 1020, 0, 1024, 0, 1023, 0, 1025, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1025, 1025, 1020, 1024, 1023, 1025, 1026] avg=1024.0000 max=1026 max_rank=7 min=1020 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1024, 3, 1019, 0, 1024, 0, 1023, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1025, 1024, 1022, 1024, 1023, 1024, 1026] avg=1024.0000 max=1026 max_rank=7 min=1022 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8190 actual_valid_recv=8192 observed_loss=-2 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1026, 0, 1, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1023, 1026, 1026, 1024, 1024] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 0, 1025, 0, 1029, 0, 1025, 1016, 0, 1012, 0, 1, 1029, 1, 1029] rerouted_tokens_per_ep_rank=[1025, 1025, 1029, 1025, 1016, 1012, 1030, 1030] avg=1024.0000 max=1030 max_rank=6 min=1012 min_rank=5 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4088 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8190 actual_valid_recv=8192 observed_loss=-2 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4095, 4095, 0, 0, 1, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4095, 4094, 0, 0, 1, 1, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1027, 1024, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1027, 1024, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1026, 0, 0, 1023, 0, 1023, 1, 1023, 0, 1023, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1026, 1023, 1023, 1024, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 1006, 0, 0, 1025, 0, 1029, 1, 1029, 1, 1030, 1, 1029, 0, 1025] rerouted_tokens_per_ep_rank=[1016, 1006, 1025, 1029, 1030, 1031, 1030, 1025] avg=1024.0000 max=1031 max_rank=5 min=1006 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4116 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1027, 1024, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1027, 1024, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1027, 1024, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1027, 1024, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8190, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8190 actual_valid_recv=8192 observed_loss=-2 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 1, 4095, 1, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4094, 1, 1, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1, 1023, 1026, 0, 1, 1023, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1024, 1026, 1024, 1023, 1023, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 1016, 0, 0, 1030, 1005, 0, 1, 1029, 1, 1029, 1, 1030, 0, 1025] rerouted_tokens_per_ep_rank=[1025, 1016, 1030, 1005, 1030, 1030, 1031, 1025] avg=1024.0000 max=1031 max_rank=6 min=1005 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4116 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8193, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8191 actual_valid_recv=8320 observed_loss=-129 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4095, 1, 0, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4095, 0, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1023, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1025, 1023, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1023, 1, 1023, 0, 1023, 1026, 0, 0, 1023, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1023, 1026, 1023, 1023, 1026] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 1, 1029, 1, 1029, 0, 1029, 1016, 0, 1, 1030, 0, 1025, 1006, 0] rerouted_tokens_per_ep_rank=[1025, 1030, 1030, 1029, 1016, 1031, 1025, 1006] avg=1024.0000 max=1031 max_rank=5 min=1006 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4078 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1023, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1025, 1023, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1023, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1025, 1023, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1023, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1025, 1023, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1023, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1025, 1023, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1023, 0, 1024, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1025, 1023, 1024, 1027] avg=1024.0000 max=1027 max_rank=7 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8190, 8192, 8192, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8190 actual_valid_recv=8192 observed_loss=-2 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1, 4095, 1, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1, 4095, 1, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 1, 1023, 0, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1024, 1023, 1026, 1023, 1023, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1029, 1012, 0, 1, 1029, 0, 1025, 1016, 0, 0, 1029, 0, 1025, 0, 1025] rerouted_tokens_per_ep_rank=[1030, 1012, 1030, 1025, 1016, 1029, 1025, 1025] avg=1024.0000 max=1030 max_rank=0 min=1012 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8192, 8192, 8192, 8190, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8190 actual_valid_recv=8192 observed_loss=-2 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 0, 2, 4093, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 0, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 1, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 7, 4089, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 2, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 0, 1022, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1024, 1024, 1023, 1022, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1022 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1, 1023, 1, 1023, 0, 1023, 0, 1022, 1027, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1024, 1024, 1023, 1022, 1027, 1023] avg=1024.0000 max=1027 max_rank=6 min=1022 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 0, 1025, 1, 1027, 1, 1027, 0, 1029, 2, 1027, 1012, 0, 0, 1025] rerouted_tokens_per_ep_rank=[1016, 1025, 1028, 1028, 1029, 1029, 1012, 1025] avg=1024.0000 max=1029 max_rank=4 min=1012 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1, 1022, 1027, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1023, 1023, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 2, 1022, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1023, 1024, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 0, 1022, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1024, 1024, 1023, 1022, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1022 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1022, 0, 1022, 0, 1023, 7, 1021, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1022, 1022, 1023, 1028, 1024, 1024] avg=1024.0000 max=1028 max_rank=5 min=1022 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 0, 1022, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1024, 1024, 1023, 1022, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1022 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8190, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8190, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8190, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8190, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8190, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8190, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8190, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8190, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8190 actual_valid_recv=8192 observed_loss=-2 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 4095, 1, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4095, 1, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1024, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1029, 1, 1030, 0, 1025, 0, 1025, 0, 1030, 1006, 0, 1, 1029, 1016, 0] rerouted_tokens_per_ep_rank=[1029, 1031, 1025, 1025, 1030, 1006, 1030, 1016] avg=1024.0000 max=1031 max_rank=1 min=1006 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4082 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 1026, 0, 1, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1023, 1024, 1026, 1024, 1026] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1024, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1024, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1024, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1024, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1024, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8190, 8192, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8190 actual_valid_recv=8192 observed_loss=-2 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 4095, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 4095, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1, 1023, 1, 1023, 1026, 0, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1024, 1024, 1026, 1023, 1026] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 0, 1029, 0, 1025, 1, 1029, 1, 1029, 1012, 0, 0, 1025, 1016, 0] rerouted_tokens_per_ep_rank=[1025, 1029, 1025, 1030, 1030, 1012, 1025, 1016] avg=1024.0000 max=1030 max_rank=3 min=1012 min_rank=5 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8190 actual_valid_recv=8192 observed_loss=-2 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 4095, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 4095, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 1026, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1023, 1024, 1026, 1026] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1029, 0, 1025, 0, 1029, 0, 1025, 0, 1025, 1, 1029, 1012, 0, 1016, 0] rerouted_tokens_per_ep_rank=[1030, 1025, 1029, 1025, 1025, 1030, 1012, 1016] avg=1024.0000 max=1030 max_rank=0 min=1012 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8192, 8194, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8190 actual_valid_recv=8192 observed_loss=-2 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1, 4095, 0, 0, 0, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1, 4095, 0, 0, 0, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1024, 1024, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1023, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1029, 1012, 0, 0, 1029, 0, 1025, 0, 1025, 0, 1025, 1016, 0, 1, 1029] rerouted_tokens_per_ep_rank=[1030, 1012, 1029, 1025, 1025, 1025, 1016, 1030] avg=1024.0000 max=1030 max_rank=0 min=1012 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1024, 1024, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1024, 1024, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1024, 1024, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1024, 1024, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1024, 1024, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8194, 8190, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8190 actual_valid_recv=8192 observed_loss=-2 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3733, 2, 4, 0, 945, 0, 3508, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3179, 7, 2, 3, 2416, 1, 2584, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[3503, 0, 0, 0, 1869, 0, 2820, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3613, 0, 0, 0, 1617, 0, 2962, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3723, 0, 0, 0, 1587, 0, 2882, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3702, 0, 0, 0, 1480, 6, 3004, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3235, 32, 0, 0, 2171, 16, 2737, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3780, 0, 0, 0, 1448, 0, 2964, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1586, 0, 2, 372, 4, 641, 0, 1073, 573, 0, 0, 1221, 1646, 0, 0, 1074] rerouted_tokens_per_ep_rank=[1586, 374, 645, 1073, 573, 1221, 1646, 1074] avg=1024.0000 max=1646 max_rank=6 min=374 min_rank=1 imbalance_ratio=1.607422 max_load_reduction=6546 max_load_reduction_pct=0.799072 moved_to_echo=4514 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1351, 0, 7, 953, 2, 472, 3, 914, 1463, 0, 1, 899, 1213, 0, 0, 914] rerouted_tokens_per_ep_rank=[1351, 960, 474, 917, 1463, 900, 1213, 914] avg=1024.0000 max=1463 max_rank=4 min=474 min_rank=2 imbalance_ratio=1.428711 max_load_reduction=6729 max_load_reduction_pct=0.821411 moved_to_echo=4490 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1480, 0, 0, 741, 0, 520, 0, 1012, 1128, 0, 0, 985, 1315, 0, 0, 1011] rerouted_tokens_per_ep_rank=[1480, 741, 520, 1012, 1128, 985, 1315, 1011] avg=1024.0000 max=1480 max_rank=0 min=520 min_rank=2 imbalance_ratio=1.445312 max_load_reduction=6712 max_load_reduction_pct=0.819336 moved_to_echo=4439 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1535, 0, 0, 638, 0, 541, 0, 1039, 979, 0, 0, 1031, 1390, 0, 0, 1039] rerouted_tokens_per_ep_rank=[1535, 638, 541, 1039, 979, 1031, 1390, 1039] avg=1024.0000 max=1535 max_rank=0 min=541 min_rank=2 imbalance_ratio=1.499023 max_load_reduction=6657 max_load_reduction_pct=0.812622 moved_to_echo=4439 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1573, 0, 0, 584, 0, 549, 0, 1064, 896, 0, 6, 1045, 1410, 0, 0, 1065] rerouted_tokens_per_ep_rank=[1573, 584, 549, 1064, 896, 1051, 1410, 1065] avg=1024.0000 max=1573 max_rank=0 min=549 min_rank=2 imbalance_ratio=1.536133 max_load_reduction=6619 max_load_reduction_pct=0.807983 moved_to_echo=4422 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1582, 0, 0, 626, 0, 526, 0, 1070, 961, 0, 0, 1003, 1353, 0, 0, 1071] rerouted_tokens_per_ep_rank=[1582, 626, 526, 1070, 961, 1003, 1353, 1071] avg=1024.0000 max=1582 max_rank=0 min=526 min_rank=2 imbalance_ratio=1.544922 max_load_reduction=6610 max_load_reduction_pct=0.806885 moved_to_echo=4388 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1375, 0, 32, 856, 0, 500, 0, 930, 1315, 0, 16, 953, 1284, 0, 1, 930] rerouted_tokens_per_ep_rank=[1375, 888, 500, 930, 1315, 969, 1284, 931] avg=1024.0000 max=1375 max_rank=0 min=500 min_rank=2 imbalance_ratio=1.342773 max_load_reduction=6817 max_load_reduction_pct=0.832153 moved_to_echo=4499 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1606, 0, 0, 571, 0, 541, 0, 1087, 877, 0, 0, 1032, 1391, 0, 0, 1087] rerouted_tokens_per_ep_rank=[1606, 571, 541, 1087, 877, 1032, 1391, 1087] avg=1024.0000 max=1606 max_rank=0 min=541 min_rank=2 imbalance_ratio=1.568359 max_load_reduction=6586 max_load_reduction_pct=0.803955 moved_to_echo=4387 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4296 expected_overflow=0 exact_recv_per_rank=[12088, 5382, 4296, 8192, 8192, 8192, 11002, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12088, 5382, 4296, 8192, 8192, 8192, 11002, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12088, 5382, 4296, 8192, 8192, 8192, 11002, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12088, 5382, 4296, 8192, 8192, 8192, 11002, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5382 expected_overflow=0 exact_recv_per_rank=[12088, 5382, 4296, 8192, 8192, 8192, 11002, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11002 expected_overflow=0 exact_recv_per_rank=[12088, 5382, 4296, 8192, 8192, 8192, 11002, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=12088 expected_overflow=0 exact_recv_per_rank=[12088, 5382, 4296, 8192, 8192, 8192, 11002, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12088, 5382, 4296, 8192, 8192, 8192, 11002, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=4296 actual_valid_recv=4480 observed_loss=-184 tokens_per_expert=[128, 4352] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=5382 actual_valid_recv=5504 observed_loss=-122 tokens_per_expert=[128, 5376] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=12088 actual_valid_recv=12160 observed_loss=-72 tokens_per_expert=[12160, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=11002 actual_valid_recv=11008 observed_loss=-6 tokens_per_expert=[11008, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=4480 slack_slots=11904 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=5504 slack_slots=10880 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=12160 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=11008 slack_slots=5376 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 4096, 0, 0, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4095, 1, 4093, 0, 1, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4090, 1, 4083, 0, 1, 3, 8, 6] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1, 1023, 1026, 0, 0, 1024, 1, 1023, 0, 1023, 2, 1022, 0, 1022] rerouted_tokens_per_ep_rank=[1025, 1024, 1026, 1024, 1024, 1023, 1024, 1022] avg=1024.0000 max=1026 max_rank=2 min=1022 min_rank=7 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1026, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1026, 1024, 1024, 1024, 1023, 1023] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1026, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1026, 1024, 1024, 1024, 1023, 1023] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1026, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1026, 1024, 1024, 1024, 1023, 1023] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1018, 0, 0, 1026, 1017, 0, 0, 1026, 0, 1025, 0, 1026, 0, 1025, 1, 1028] rerouted_tokens_per_ep_rank=[1018, 1026, 1017, 1026, 1025, 1026, 1025, 1029] avg=1024.0000 max=1029 max_rank=7 min=1017 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1026, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1026, 1024, 1024, 1024, 1023, 1023] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 1, 1022, 1023, 0, 0, 1022, 1, 1022, 3, 1020, 8, 1020, 6, 1020] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1022, 1023, 1023, 1028, 1026] avg=1024.0000 max=1028 max_rank=6 min=1022 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1026, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1026, 1024, 1024, 1024, 1023, 1023] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8192, 8188, 8196, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8188 actual_valid_recv=8320 observed_loss=-132 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 1472, 4096, 0, 2619, 0, 5, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1, 1369, 4092, 6, 2719, 0, 1, 4] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 1276, 4095, 0, 2817, 3, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1135, 4096, 0, 2961, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1082, 4096, 0, 3014, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 586, 4095, 1, 3500, 8, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 1214, 4096, 0, 2880, 2, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 1127, 4096, 0, 2969, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1472, 0, 1026, 0, 0, 1023, 914, 0, 0, 793, 5, 912, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1472, 1026, 1023, 914, 793, 917, 1023] avg=1024.0000 max=1472 max_rank=1 min=793 min_rank=5 imbalance_ratio=1.437500 max_load_reduction=6720 max_load_reduction_pct=0.820312 moved_to_echo=3647 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 1276, 0, 1018, 0, 0, 1025, 977, 0, 3, 856, 1, 984, 0, 1027] rerouted_tokens_per_ep_rank=[1025, 1276, 1018, 1025, 977, 859, 985, 1027] avg=1024.0000 max=1276 max_rank=1 min=859 min_rank=5 imbalance_ratio=1.246094 max_load_reduction=6916 max_load_reduction_pct=0.844238 moved_to_echo=3848 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 586, 0, 1026, 0, 1, 1023, 1223, 0, 8, 1059, 1, 1218, 1, 1023] rerouted_tokens_per_ep_rank=[1023, 586, 1026, 1024, 1223, 1067, 1219, 1024] avg=1024.0000 max=1223 max_rank=4 min=586 min_rank=1 imbalance_ratio=1.194336 max_load_reduction=6969 max_load_reduction_pct=0.850708 moved_to_echo=4533 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1082, 0, 1026, 0, 0, 1023, 1053, 0, 0, 912, 0, 1049, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1082, 1026, 1023, 1053, 912, 1049, 1023] avg=1024.0000 max=1082 max_rank=1 min=912 min_rank=5 imbalance_ratio=1.056641 max_load_reduction=7110 max_load_reduction_pct=0.867920 moved_to_echo=4037 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1135, 0, 1026, 0, 0, 1023, 1034, 0, 0, 896, 0, 1031, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1135, 1026, 1023, 1034, 896, 1031, 1023] avg=1024.0000 max=1135 max_rank=1 min=896 min_rank=5 imbalance_ratio=1.108398 max_load_reduction=7057 max_load_reduction_pct=0.861450 moved_to_echo=3984 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1369, 0, 1025, 0, 6, 1022, 950, 0, 0, 823, 1, 946, 4, 1022] rerouted_tokens_per_ep_rank=[1024, 1369, 1025, 1028, 950, 823, 947, 1026] avg=1024.0000 max=1369 max_rank=1 min=823 min_rank=5 imbalance_ratio=1.336914 max_load_reduction=6823 max_load_reduction_pct=0.832886 moved_to_echo=3746 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1214, 0, 1026, 0, 0, 1023, 1005, 0, 2, 872, 0, 1003, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1214, 1026, 1023, 1005, 874, 1003, 1023] avg=1024.0000 max=1214 max_rank=1 min=874 min_rank=5 imbalance_ratio=1.185547 max_load_reduction=6978 max_load_reduction_pct=0.851807 moved_to_echo=3905 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1127, 0, 1026, 0, 0, 1023, 1036, 0, 0, 899, 0, 1034, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1127, 1026, 1023, 1036, 899, 1034, 1023] avg=1024.0000 max=1127 max_rank=1 min=899 min_rank=5 imbalance_ratio=1.100586 max_load_reduction=7065 max_load_reduction_pct=0.862427 moved_to_echo=3992 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9261, 8199, 8192, 8192, 7123, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9261, 8199, 8192, 8192, 7123, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 9261, 8199, 8192, 8192, 7123, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7123 expected_overflow=0 exact_recv_per_rank=[8192, 9261, 8199, 8192, 8192, 7123, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9261, 8199, 8192, 8192, 7123, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 9261, 8199, 8192, 8192, 7123, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9261 expected_overflow=0 exact_recv_per_rank=[8192, 9261, 8199, 8192, 8192, 7123, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 9261, 8199, 8192, 8192, 7123, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7123 actual_valid_recv=7296 observed_loss=-173 tokens_per_expert=[128, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8185 actual_valid_recv=8320 observed_loss=-135 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=9261 actual_valid_recv=9344 observed_loss=-83 tokens_per_expert=[9344, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7296 slack_slots=9088 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=9344 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[3933, 0, 0, 79, 84, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[3998, 0, 0, 35, 63, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[3994, 0, 0, 18, 84, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4008, 9, 1, 33, 45, 4092, 4, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[3979, 0, 0, 26, 91, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[3990, 0, 0, 33, 73, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[3922, 0, 1, 30, 143, 4095, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[3991, 0, 0, 23, 82, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1014, 0, 0, 1011, 0, 1023, 79, 978, 84, 930, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1014, 1011, 1023, 1057, 1014, 1026, 1023, 1024] avg=1024.0000 max=1057 max_rank=3 min=1011 min_rank=1 imbalance_ratio=1.032227 max_load_reduction=7135 max_load_reduction_pct=0.870972 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1031, 0, 0, 1028, 0, 1023, 35, 994, 63, 945, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1031, 1028, 1023, 1029, 1008, 1026, 1023, 1024] avg=1024.0000 max=1031 max_rank=0 min=1008 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4081 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1018, 0, 0, 1029, 0, 1030, 18, 998, 84, 949, 1013, 0, 0, 1027, 0, 1026] rerouted_tokens_per_ep_rank=[1018, 1029, 1030, 1016, 1033, 1013, 1027, 1026] avg=1024.0000 max=1033 max_rank=4 min=1013 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1012, 0, 0, 1008, 1, 1023, 30, 975, 143, 927, 1026, 0, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1012, 1008, 1024, 1005, 1070, 1026, 1024, 1023] avg=1024.0000 max=1070 max_rank=4 min=1005 min_rank=3 imbalance_ratio=1.044922 max_load_reduction=7122 max_load_reduction_pct=0.869385 moved_to_echo=4143 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1029, 0, 0, 1026, 0, 1023, 33, 992, 73, 943, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1029, 1026, 1023, 1025, 1016, 1026, 1023, 1024] avg=1024.0000 max=1029 max_rank=0 min=1016 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 26, 989, 91, 941, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1015, 1032, 1026, 1023, 1024] avg=1024.0000 max=1032 max_rank=4 min=1015 min_rank=3 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1033, 0, 9, 1030, 1, 1022, 33, 997, 45, 948, 1025, 0, 4, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1033, 1039, 1023, 1030, 993, 1025, 1026, 1023] avg=1024.0000 max=1039 max_rank=1 min=993 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=7153 max_load_reduction_pct=0.873169 moved_to_echo=4067 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1029, 0, 0, 1026, 0, 1023, 23, 992, 82, 944, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1029, 1026, 1023, 1015, 1026, 1026, 1023, 1024] avg=1024.0000 max=1029 max_rank=0 min=1015 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8192, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8190 actual_valid_recv=8320 observed_loss=-130 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 0, 4095, 1, 0, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 4, 0, 4092, 4, 0, 4092, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1, 1023, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1026, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 0, 1027, 0, 1026, 1017, 0, 0, 1027, 0, 1027, 1017, 0, 0, 1026] rerouted_tokens_per_ep_rank=[1025, 1027, 1026, 1017, 1027, 1027, 1017, 1026] avg=1024.0000 max=1027 max_rank=1 min=1017 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 4, 1022, 0, 1023, 1024, 0, 4, 1022, 0, 1022, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1026, 1022, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8187 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8197 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8187, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8197 actual_valid_recv=8320 observed_loss=-123 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8187 actual_valid_recv=8192 observed_loss=-5 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 7, 0, 0, 0, 4089] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 1, 5, 0, 0, 1, 4090] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 2, 0, 0, 0, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1, 4091, 4, 71, 0, 0, 4, 4021] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 6, 0, 0, 0, 4090] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1026, 0, 1015, 0, 1024, 0, 1024, 0, 1026, 1029, 0] rerouted_tokens_per_ep_rank=[1023, 1025, 1026, 1015, 1024, 1024, 1026, 1029] avg=1024.0000 max=1029 max_rank=7 min=1015 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 7, 1013, 0, 1024, 0, 1024, 0, 1024, 1028, 0] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1020, 1024, 1024, 1024, 1028] avg=1024.0000 max=1028 max_rank=7 min=1020 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1, 1024, 5, 1014, 0, 1023, 0, 1023, 1, 1024, 1028, 0] rerouted_tokens_per_ep_rank=[1023, 1026, 1025, 1019, 1023, 1023, 1025, 1028] avg=1024.0000 max=1028 max_rank=7 min=1019 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1026, 0, 1015, 0, 1024, 0, 1024, 0, 1026, 1029, 0] rerouted_tokens_per_ep_rank=[1023, 1025, 1026, 1015, 1024, 1024, 1026, 1029] avg=1024.0000 max=1029 max_rank=7 min=1015 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 1016, 0, 0, 1030, 2, 1018, 0, 1027, 0, 1027, 0, 1030, 1016, 0] rerouted_tokens_per_ep_rank=[1026, 1016, 1030, 1020, 1027, 1027, 1030, 1016] avg=1024.0000 max=1030 max_rank=2 min=1016 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1026, 0, 1015, 0, 1024, 0, 1024, 0, 1026, 1029, 0] rerouted_tokens_per_ep_rank=[1023, 1025, 1026, 1015, 1024, 1024, 1026, 1029] avg=1024.0000 max=1029 max_rank=7 min=1015 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 1022, 1025, 0, 4, 1007, 71, 997, 0, 1022, 0, 1022, 4, 1007, 1010, 0] rerouted_tokens_per_ep_rank=[1023, 1025, 1011, 1068, 1022, 1022, 1011, 1010] avg=1024.0000 max=1068 max_rank=3 min=1010 min_rank=7 imbalance_ratio=1.042969 max_load_reduction=7124 max_load_reduction_pct=0.869629 moved_to_echo=4065 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 6, 1014, 0, 1024, 0, 1024, 0, 1024, 1028, 0] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1020, 1024, 1024, 1024, 1028] avg=1024.0000 max=1028 max_rank=7 min=1020 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8187 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8197 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8187 actual_valid_recv=8320 observed_loss=-133 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8197 actual_valid_recv=8320 observed_loss=-123 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4091, 4091, 4, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1026, 0, 1, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1023, 1026, 1026, 1024, 1024] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 1027, 0, 1027, 0, 1026, 1016, 0, 1017, 0, 0, 1027, 0, 1026] rerouted_tokens_per_ep_rank=[1026, 1027, 1027, 1026, 1016, 1017, 1027, 1026] avg=1024.0000 max=1027 max_rank=1 min=1016 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4086 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1022, 0, 1022, 1, 1022, 1025, 0, 1025, 0, 4, 1022, 5, 1022] rerouted_tokens_per_ep_rank=[1022, 1022, 1022, 1023, 1025, 1025, 1026, 1027] avg=1024.0000 max=1027 max_rank=7 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8186, 8192, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8186, 8192, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8186, 8192, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8186, 8192, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8186, 8192, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8198 expected_overflow=0 exact_recv_per_rank=[8186, 8192, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8186 expected_overflow=0 exact_recv_per_rank=[8186, 8192, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8186, 8192, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8186 actual_valid_recv=8192 observed_loss=-6 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8198 actual_valid_recv=8320 observed_loss=-122 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4095, 4095, 0, 0, 1, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4090, 4082, 0, 0, 5, 10, 5, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1027, 0, 0, 1023, 0, 1023, 1, 1023, 0, 1022, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1027, 1023, 1023, 1024, 1022, 1024, 1023] avg=1024.0000 max=1027 max_rank=1 min=1022 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1027, 1024, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 1014, 0, 0, 1027, 0, 1026, 0, 1028, 0, 1026, 0, 1028, 0, 1027] rerouted_tokens_per_ep_rank=[1016, 1014, 1027, 1026, 1028, 1026, 1028, 1027] avg=1024.0000 max=1028 max_rank=4 min=1014 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4109 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1027, 1024, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1027, 1024, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1027, 1024, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1023, 0, 0, 1022, 0, 1021, 5, 1020, 10, 1019, 5, 1020, 0, 1022] rerouted_tokens_per_ep_rank=[1025, 1023, 1022, 1021, 1025, 1029, 1025, 1022] avg=1024.0000 max=1029 max_rank=5 min=1021 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1027, 1024, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4095, 1, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4091, 0, 4091, 5, 0, 2, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1024, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1024, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1026, 1024, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1024, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 1016, 0, 0, 1026, 1012, 0, 0, 1026, 0, 1027, 0, 1029, 0, 1029] rerouted_tokens_per_ep_rank=[1027, 1016, 1026, 1012, 1026, 1027, 1029, 1029] avg=1024.0000 max=1029 max_rank=6 min=1012 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4111 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1024, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1025, 0, 0, 1022, 1025, 0, 5, 1022, 0, 1022, 2, 1022, 3, 1022] rerouted_tokens_per_ep_rank=[1022, 1025, 1022, 1025, 1027, 1022, 1024, 1025] avg=1024.0000 max=1027 max_rank=4 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1024, 1023, 1023] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8198 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8186, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8186, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8186, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8186 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8186, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8186, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8186, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8186, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8186, 8198, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8198 actual_valid_recv=8320 observed_loss=-122 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8186 actual_valid_recv=8192 observed_loss=-6 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4095, 0, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1, 5, 5, 0, 4091, 2, 0, 4088] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1024, 1025, 1024, 1024, 1026] avg=1024.0000 max=1026 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1024, 1025, 1024, 1024, 1026] avg=1024.0000 max=1026 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1023, 1, 1023, 0, 1023, 1026, 0, 0, 1023, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1023, 1026, 1023, 1023, 1026] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 1027, 0, 1027, 0, 1027, 1016, 0, 0, 1025, 0, 1027, 1017, 0] rerouted_tokens_per_ep_rank=[1026, 1027, 1027, 1027, 1016, 1025, 1027, 1017] avg=1024.0000 max=1027 max_rank=1 min=1016 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4085 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1024, 1025, 1024, 1024, 1026] avg=1024.0000 max=1026 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1024, 1025, 1024, 1024, 1026] avg=1024.0000 max=1026 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 1022, 5, 1021, 5, 1021, 0, 1022, 1025, 0, 2, 1022, 0, 1022, 1024, 0] rerouted_tokens_per_ep_rank=[1023, 1026, 1026, 1022, 1025, 1024, 1022, 1024] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1024, 1025, 1024, 1024, 1026] avg=1024.0000 max=1026 max_rank=7 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8187 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8197 expected_overflow=0 exact_recv_per_rank=[8187, 8192, 8192, 8192, 8192, 8192, 8192, 8197] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8197 actual_valid_recv=8320 observed_loss=-123 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8187 actual_valid_recv=8320 observed_loss=-133 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1, 4095, 1, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[5, 4090, 5, 0, 4091, 1, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 1016, 0, 0, 1027, 0, 1027, 1016, 0, 0, 1026, 0, 1026, 0, 1027] rerouted_tokens_per_ep_rank=[1027, 1016, 1027, 1027, 1016, 1026, 1026, 1027] avg=1024.0000 max=1027 max_rank=0 min=1016 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 1, 1023, 0, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1024, 1023, 1026, 1023, 1023, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[5, 1021, 1026, 0, 5, 1021, 0, 1022, 1025, 0, 1, 1022, 0, 1022, 0, 1022] rerouted_tokens_per_ep_rank=[1026, 1026, 1026, 1022, 1025, 1023, 1022, 1022] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8186 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8198 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8192, 8192, 8192, 8192, 8186, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8198 actual_valid_recv=8320 observed_loss=-122 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8186 actual_valid_recv=8192 observed_loss=-6 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4, 4092, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 5, 4091, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4, 4092, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 0, 3, 4092, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4090, 0, 5, 5, 0, 13, 4079, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1, 1023, 1, 1023, 0, 1023, 3, 1020, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1024, 1024, 1023, 1023, 1026, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 0, 1027, 0, 1026, 0, 1026, 0, 1026, 0, 1024, 1020, 0, 0, 1027] rerouted_tokens_per_ep_rank=[1016, 1027, 1026, 1026, 1026, 1024, 1020, 1027] avg=1024.0000 max=1027 max_rank=1 min=1016 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 4, 1020, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1023, 1024, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 4, 1020, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1023, 1024, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 0, 1021, 1027, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1024, 1024, 1023, 1021, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1021 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 5, 1020, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1023, 1025, 1025, 1024] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1022, 5, 1020, 5, 1020, 0, 1021, 13, 1017, 1022, 0, 0, 1022] rerouted_tokens_per_ep_rank=[1025, 1022, 1025, 1025, 1021, 1030, 1022, 1022] avg=1024.0000 max=1030 max_rank=5 min=1021 min_rank=4 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 0, 1021, 1027, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1024, 1024, 1023, 1021, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1021 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8185, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8185, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8185, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8185, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8185, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8185, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8185, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8185, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 4095, 1, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 2, 0, 1, 4, 4090, 4, 4091] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1023, 1023, 1026, 1024, 1026] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 0, 1029, 0, 1027, 0, 1026, 0, 1028, 1012, 0, 0, 1027, 1016, 0] rerouted_tokens_per_ep_rank=[1027, 1029, 1027, 1026, 1028, 1012, 1027, 1016] avg=1024.0000 max=1029 max_rank=1 min=1012 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 2, 1022, 0, 1022, 1, 1022, 4, 1022, 1024, 0, 4, 1022, 1025, 0] rerouted_tokens_per_ep_rank=[1022, 1024, 1022, 1023, 1026, 1024, 1026, 1025] avg=1024.0000 max=1026 max_rank=4 min=1022 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8187, 8192, 8197, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8197 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8187, 8192, 8197, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8187 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8187, 8192, 8197, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8187, 8192, 8197, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8187, 8192, 8197, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8187, 8192, 8197, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8187, 8192, 8197, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8187, 8192, 8197, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8197 actual_valid_recv=8320 observed_loss=-123 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8187 actual_valid_recv=8320 observed_loss=-133 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 4095, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 6, 4090, 0, 4091] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1, 1023, 1, 1023, 1026, 0, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1024, 1024, 1026, 1023, 1026] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 0, 1026, 0, 1027, 0, 1027, 0, 1026, 1016, 0, 0, 1027, 1016, 0] rerouted_tokens_per_ep_rank=[1027, 1026, 1027, 1027, 1026, 1016, 1027, 1016] avg=1024.0000 max=1027 max_rank=0 min=1016 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4085 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1022, 0, 1022, 5, 1021, 6, 1021, 1026, 0, 0, 1022, 1025, 0] rerouted_tokens_per_ep_rank=[1022, 1022, 1022, 1026, 1027, 1026, 1022, 1025] avg=1024.0000 max=1027 max_rank=4 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8198 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8186 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8186, 8192, 8192, 8192, 8198, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8186 actual_valid_recv=8192 observed_loss=-6 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8198 actual_valid_recv=8320 observed_loss=-122 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 4095, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[5, 1, 0, 0, 0, 6, 4090, 4090] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1023, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 1026, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1023, 1024, 1026, 1026] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 0, 1026, 0, 1027, 0, 1026, 0, 1027, 0, 1026, 1017, 0, 1016, 0] rerouted_tokens_per_ep_rank=[1027, 1026, 1027, 1026, 1027, 1026, 1017, 1016] avg=1024.0000 max=1027 max_rank=0 min=1016 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4086 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1023, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1023, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1023, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[5, 1021, 1, 1022, 0, 1022, 0, 1021, 0, 1022, 6, 1021, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1026, 1023, 1022, 1021, 1022, 1027, 1026, 1025] avg=1024.0000 max=1027 max_rank=5 min=1021 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1023, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8185, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1, 4095, 0, 0, 0, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[5, 4090, 1, 0, 0, 0, 4090, 6] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1023, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1024, 1024, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 1017, 0, 0, 1026, 0, 1027, 0, 1027, 0, 1026, 1016, 0, 0, 1026] rerouted_tokens_per_ep_rank=[1027, 1017, 1026, 1027, 1027, 1026, 1016, 1026] avg=1024.0000 max=1027 max_rank=0 min=1016 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1024, 1024, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1024, 1024, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1024, 1024, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 1021, 1026, 0, 1, 1022, 0, 1022, 0, 1022, 0, 1021, 1025, 0, 6, 1021] rerouted_tokens_per_ep_rank=[1026, 1026, 1023, 1022, 1022, 1021, 1025, 1027] avg=1024.0000 max=1027 max_rank=7 min=1021 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1024, 1024, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True + [2026-08-30 04:54:05] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 668.4 | throughput per GPU (TFLOP/s/GPU): 498.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.671444E+00 | loss scale: 1.0 | grad norm: 5.972 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2032, 0, 1, 0, 3684, 0, 2475, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2015, 0, 0, 0, 3631, 0, 2546, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2719, 0, 3, 0, 3540, 0, 1930, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2055, 0, 0, 0, 3920, 1, 2216, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[3073, 0, 0, 2, 3105, 0, 2012, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2440, 0, 0, 0, 3352, 0, 2400, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2461, 0, 3, 0, 3613, 1, 2114, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2136, 0, 0, 0, 3769, 0, 2287, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[880, 0, 0, 1054, 1, 1126, 0, 445, 1576, 0, 0, 707, 1349, 0, 0, 1054] rerouted_tokens_per_ep_rank=[880, 1054, 1127, 445, 1576, 707, 1349, 1054] avg=1024.0000 max=1576 max_rank=4 min=445 min_rank=3 imbalance_ratio=1.539062 max_load_reduction=6616 max_load_reduction_pct=0.807617 moved_to_echo=4686 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1177, 0, 0, 1013, 3, 878, 0, 596, 1514, 0, 0, 946, 1052, 0, 0, 1013] rerouted_tokens_per_ep_rank=[1177, 1013, 881, 596, 1514, 946, 1052, 1013] avg=1024.0000 max=1514 max_rank=4 min=596 min_rank=3 imbalance_ratio=1.478516 max_load_reduction=6678 max_load_reduction_pct=0.815186 moved_to_echo=4525 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[925, 0, 0, 1079, 0, 1041, 0, 468, 1611, 0, 0, 743, 1246, 0, 0, 1079] rerouted_tokens_per_ep_rank=[925, 1079, 1041, 468, 1611, 743, 1246, 1079] avg=1024.0000 max=1611 max_rank=4 min=468 min_rank=3 imbalance_ratio=1.573242 max_load_reduction=6581 max_load_reduction_pct=0.803345 moved_to_echo=4679 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1059, 0, 0, 1038, 3, 966, 0, 544, 1537, 0, 1, 858, 1148, 0, 0, 1038] rerouted_tokens_per_ep_rank=[1059, 1038, 969, 544, 1537, 859, 1148, 1038] avg=1024.0000 max=1537 max_rank=4 min=544 min_rank=3 imbalance_ratio=1.500977 max_load_reduction=6655 max_load_reduction_pct=0.812378 moved_to_echo=4582 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[890, 0, 0, 1122, 0, 1008, 0, 450, 1676, 0, 1, 715, 1208, 0, 0, 1122] rerouted_tokens_per_ep_rank=[890, 1122, 1008, 450, 1676, 716, 1208, 1122] avg=1024.0000 max=1676 max_rank=4 min=450 min_rank=3 imbalance_ratio=1.636719 max_load_reduction=6516 max_load_reduction_pct=0.795410 moved_to_echo=4722 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1057, 0, 0, 959, 0, 1092, 0, 535, 1434, 0, 0, 848, 1308, 0, 0, 959] rerouted_tokens_per_ep_rank=[1057, 959, 1092, 535, 1434, 848, 1308, 959] avg=1024.0000 max=1434 max_rank=4 min=535 min_rank=3 imbalance_ratio=1.400391 max_load_reduction=6758 max_load_reduction_pct=0.824951 moved_to_echo=4549 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1331, 0, 0, 888, 0, 915, 2, 673, 1329, 0, 0, 1069, 1097, 0, 0, 888] rerouted_tokens_per_ep_rank=[1331, 888, 915, 675, 1329, 1069, 1097, 888] avg=1024.0000 max=1331 max_rank=0 min=675 min_rank=3 imbalance_ratio=1.299805 max_load_reduction=6861 max_load_reduction_pct=0.837524 moved_to_echo=4383 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[873, 0, 0, 1039, 0, 1159, 0, 441, 1553, 0, 0, 701, 1387, 0, 0, 1039] rerouted_tokens_per_ep_rank=[873, 1039, 1159, 441, 1553, 701, 1387, 1039] avg=1024.0000 max=1553 max_rank=4 min=441 min_rank=3 imbalance_ratio=1.516602 max_load_reduction=6639 max_load_reduction_pct=0.810425 moved_to_echo=4680 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6589 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 4154, 12230, 6589, 9795, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4154 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 4154, 12230, 6589, 9795, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=12230 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 4154, 12230, 6589, 9795, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 4154, 12230, 6589, 9795, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 4154, 12230, 6589, 9795, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9795 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 4154, 12230, 6589, 9795, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 4154, 12230, 6589, 9795, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 4154, 12230, 6589, 9795, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=4154 actual_valid_recv=4352 observed_loss=-198 tokens_per_expert=[128, 4224] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=6589 actual_valid_recv=6784 observed_loss=-195 tokens_per_expert=[128, 6656] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9795 actual_valid_recv=9856 observed_loss=-61 tokens_per_expert=[9856, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=12230 actual_valid_recv=12288 observed_loss=-58 tokens_per_expert=[12288, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=4352 slack_slots=12032 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=6784 slack_slots=9600 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9856 slack_slots=6528 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=12288 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 0, 0, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 0, 0, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4094, 0, 0, 0, 0, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4093, 1, 4091, 3, 2, 0, 0, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1026, 1023, 1024, 1024, 1024, 1023] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1024, 2, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1025, 1023, 1023, 1024, 1024, 1025] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1, 1022, 1025, 0, 3, 1022, 2, 1022, 0, 1023, 0, 1023, 2, 1022] rerouted_tokens_per_ep_rank=[1025, 1023, 1025, 1025, 1024, 1023, 1023, 1024] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1017, 0, 0, 1029, 1014, 0, 0, 1029, 0, 1027, 0, 1025, 0, 1025, 0, 1026] rerouted_tokens_per_ep_rank=[1017, 1029, 1014, 1029, 1027, 1025, 1025, 1026] avg=1024.0000 max=1029 max_rank=1 min=1014 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1024, 1, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1026, 1023, 1023, 1024, 1024, 1024] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1026, 1023, 1024, 1024, 1024, 1023] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1024, 0, 1024, 1, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1026, 1023, 1023, 1024, 1024, 1024] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1026, 1023, 1024, 1024, 1024, 1023] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8194, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8194, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8194, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8194, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8194, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8194, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8194, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8190, 8194, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8190 actual_valid_recv=8320 observed_loss=-130 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 2892, 4093, 0, 1, 1204, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 2998, 4096, 0, 0, 1092, 6, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 3289, 4087, 0, 0, 813, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 3410, 4095, 0, 1, 654, 32, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 3357, 4096, 0, 0, 739, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 2454, 4095, 0, 0, 1643, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 3070, 4096, 0, 0, 1026, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 2927, 4096, 0, 0, 1169, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 974, 0, 1026, 0, 0, 1021, 1, 970, 1204, 0, 2, 948, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 974, 1026, 1021, 971, 1204, 950, 1023] avg=1024.0000 max=1204 max_rank=5 min=950 min_rank=6 imbalance_ratio=1.175781 max_load_reduction=6988 max_load_reduction_pct=0.853027 moved_to_echo=4148 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 999, 0, 1015, 0, 0, 1027, 0, 1011, 1092, 0, 6, 988, 0, 1027] rerouted_tokens_per_ep_rank=[1027, 999, 1015, 1027, 1011, 1092, 994, 1027] avg=1024.0000 max=1092 max_rank=5 min=994 min_rank=6 imbalance_ratio=1.066406 max_load_reduction=7100 max_load_reduction_pct=0.866699 moved_to_echo=4124 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1107, 0, 1023, 0, 0, 1020, 0, 1104, 813, 0, 3, 1078, 0, 1022] rerouted_tokens_per_ep_rank=[1022, 1107, 1023, 1020, 1104, 813, 1081, 1022] avg=1024.0000 max=1107 max_rank=1 min=813 min_rank=5 imbalance_ratio=1.081055 max_load_reduction=7085 max_load_reduction_pct=0.864868 moved_to_echo=4020 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1131, 0, 1026, 0, 0, 1022, 0, 1126, 739, 0, 0, 1100, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1131, 1026, 1022, 1126, 739, 1100, 1024] avg=1024.0000 max=1131 max_rank=1 min=739 min_rank=5 imbalance_ratio=1.104492 max_load_reduction=7061 max_load_reduction_pct=0.861938 moved_to_echo=3989 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1034, 0, 1026, 0, 0, 1022, 0, 1030, 1026, 0, 0, 1006, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1034, 1026, 1022, 1030, 1026, 1006, 1024] avg=1024.0000 max=1034 max_rank=1 min=1006 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4086 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 827, 0, 1025, 0, 0, 1022, 0, 823, 1643, 0, 0, 804, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 827, 1025, 1022, 823, 1643, 804, 1024] avg=1024.0000 max=1643 max_rank=5 min=804 min_rank=6 imbalance_ratio=1.604492 max_load_reduction=6549 max_load_reduction_pct=0.799438 moved_to_echo=4294 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1148, 0, 1025, 0, 0, 1022, 1, 1144, 654, 0, 32, 1118, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1148, 1025, 1022, 1145, 654, 1150, 1024] avg=1024.0000 max=1150 max_rank=6 min=654 min_rank=5 imbalance_ratio=1.123047 max_load_reduction=7042 max_load_reduction_pct=0.859619 moved_to_echo=3973 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 986, 0, 1026, 0, 0, 1022, 0, 982, 1169, 0, 0, 959, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 986, 1026, 1022, 982, 1169, 959, 1024] avg=1024.0000 max=1169 max_rank=5 min=959 min_rank=6 imbalance_ratio=1.141602 max_load_reduction=7023 max_load_reduction_pct=0.857300 moved_to_echo=4134 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8340 expected_overflow=0 exact_recv_per_rank=[8192, 8206, 8192, 8178, 8192, 8340, 8044, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8178 expected_overflow=0 exact_recv_per_rank=[8192, 8206, 8192, 8178, 8192, 8340, 8044, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8206, 8192, 8178, 8192, 8340, 8044, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8206, 8192, 8178, 8192, 8340, 8044, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8206, 8192, 8178, 8192, 8340, 8044, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8044 expected_overflow=0 exact_recv_per_rank=[8192, 8206, 8192, 8178, 8192, 8340, 8044, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8206, 8192, 8178, 8192, 8340, 8044, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8206 expected_overflow=0 exact_recv_per_rank=[8192, 8206, 8192, 8178, 8192, 8340, 8044, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8340 actual_valid_recv=8448 observed_loss=-108 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8044 actual_valid_recv=8192 observed_loss=-148 tokens_per_expert=[128, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8178 actual_valid_recv=8192 observed_loss=-14 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8206 actual_valid_recv=8320 observed_loss=-114 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1177, 0, 0, 0, 2919, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1280, 0, 0, 0, 2816, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1251, 0, 0, 1, 2844, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1165, 0, 0, 4, 2927, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1275, 0, 0, 0, 2821, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1788, 0, 0, 2, 2306, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[981, 0, 0, 0, 3115, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1321, 0, 0, 0, 2775, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1280, 0, 0, 1024, 0, 1024, 0, 767, 1025, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1280, 1024, 1024, 767, 1025, 1024, 1024, 1024] avg=1024.0000 max=1280 max_rank=0 min=767 min_rank=3 imbalance_ratio=1.250000 max_load_reduction=6912 max_load_reduction_pct=0.843750 moved_to_echo=4097 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1177, 0, 0, 1024, 0, 1061, 0, 795, 1063, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1177, 1024, 1061, 795, 1063, 1024, 1024, 1024] avg=1024.0000 max=1177 max_rank=0 min=795 min_rank=3 imbalance_ratio=1.149414 max_load_reduction=7015 max_load_reduction_pct=0.856323 moved_to_echo=4135 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1165, 0, 0, 1024, 0, 1064, 4, 797, 1066, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1165, 1024, 1064, 801, 1066, 1024, 1024, 1024] avg=1024.0000 max=1165 max_rank=0 min=801 min_rank=3 imbalance_ratio=1.137695 max_load_reduction=7027 max_load_reduction_pct=0.857788 moved_to_echo=4138 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1251, 0, 0, 1024, 0, 1038, 1, 779, 1027, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1251, 1024, 1038, 780, 1027, 1024, 1024, 1024] avg=1024.0000 max=1251 max_rank=0 min=780 min_rank=3 imbalance_ratio=1.221680 max_load_reduction=6941 max_load_reduction_pct=0.847290 moved_to_echo=4099 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1275, 0, 0, 1024, 0, 1026, 0, 768, 1027, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1275, 1024, 1026, 768, 1027, 1024, 1024, 1024] avg=1024.0000 max=1275 max_rank=0 min=768 min_rank=3 imbalance_ratio=1.245117 max_load_reduction=6917 max_load_reduction_pct=0.844360 moved_to_echo=4099 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1788, 0, 0, 1024, 0, 838, 2, 628, 840, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1788, 1024, 838, 630, 840, 1024, 1024, 1024] avg=1024.0000 max=1788 max_rank=0 min=630 min_rank=3 imbalance_ratio=1.746094 max_load_reduction=6404 max_load_reduction_pct=0.781738 moved_to_echo=3912 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[981, 0, 0, 1024, 0, 1132, 0, 849, 1134, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[981, 1024, 1132, 849, 1134, 1024, 1024, 1024] avg=1024.0000 max=1134 max_rank=4 min=849 min_rank=3 imbalance_ratio=1.107422 max_load_reduction=7058 max_load_reduction_pct=0.861572 moved_to_echo=4206 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1321, 0, 0, 1024, 0, 1009, 0, 756, 1010, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1321, 1024, 1009, 756, 1010, 1024, 1024, 1024] avg=1024.0000 max=1321 max_rank=0 min=756 min_rank=3 imbalance_ratio=1.290039 max_load_reduction=6871 max_load_reduction_pct=0.838745 moved_to_echo=4082 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10238, 8192, 8192, 6146, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10238, 8192, 8192, 6146, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10238 expected_overflow=0 exact_recv_per_rank=[10238, 8192, 8192, 6146, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10238, 8192, 8192, 6146, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6146 expected_overflow=0 exact_recv_per_rank=[10238, 8192, 8192, 6146, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10238, 8192, 8192, 6146, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10238, 8192, 8192, 6146, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10238, 8192, 8192, 6146, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=6146 actual_valid_recv=6272 observed_loss=-126 tokens_per_expert=[128, 6144] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=10238 actual_valid_recv=10240 observed_loss=-2 tokens_per_expert=[10240, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=6272 slack_slots=10112 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=10240 slack_slots=6144 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 12, 0, 0, 0, 4084] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 5, 0, 0, 0, 4091] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 12, 0, 0, 0, 4084] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 6, 0, 0, 0, 4090] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 14, 0, 0, 0, 4082] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1022, 12, 1016, 0, 1024, 0, 1024, 0, 1022, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1022, 1028, 1024, 1024, 1022, 1024] avg=1024.0000 max=1028 max_rank=3 min=1022 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1025, 0, 1019, 0, 1024, 0, 1024, 0, 1025, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1025, 1019, 1024, 1024, 1025, 1027] avg=1024.0000 max=1027 max_rank=7 min=1019 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1025, 1, 1019, 0, 1024, 0, 1024, 0, 1025, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1025, 1020, 1024, 1024, 1025, 1026] avg=1024.0000 max=1026 max_rank=7 min=1020 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 6, 1017, 0, 1024, 0, 1024, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1023, 1024, 1024, 1024, 1025] avg=1024.0000 max=1025 max_rank=7 min=1023 min_rank=3 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1025, 12, 1019, 0, 1024, 0, 1024, 0, 1025, 1015, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1025, 1031, 1024, 1024, 1025, 1015] avg=1024.0000 max=1031 max_rank=3 min=1015 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4088 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1022, 14, 1015, 0, 1024, 0, 1024, 0, 1022, 1023, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1022, 1029, 1024, 1024, 1022, 1023] avg=1024.0000 max=1029 max_rank=3 min=1022 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 5, 1018, 0, 1024, 0, 1024, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1023, 1024, 1024, 1024, 1025] avg=1024.0000 max=1025 max_rank=7 min=1023 min_rank=3 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1025, 0, 1019, 0, 1024, 0, 1024, 0, 1025, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1025, 1019, 1024, 1024, 1025, 1027] avg=1024.0000 max=1027 max_rank=7 min=1019 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2429, 0, 1667] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2169, 0, 1927] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2518, 0, 1578] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2256, 0, 1840] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2068, 0, 2028] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 3084, 0, 1012] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2159, 0, 1937] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2067, 0, 2029] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 692, 1024, 0, 1368, 0, 0, 1061, 975, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 692, 1024, 1368, 1061, 975] avg=1024.0000 max=1368 max_rank=5 min=692 min_rank=3 imbalance_ratio=1.335938 max_load_reduction=6824 max_load_reduction_pct=0.833008 moved_to_echo=4428 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 764, 1024, 0, 1271, 0, 0, 985, 1076, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 764, 1024, 1271, 985, 1076] avg=1024.0000 max=1271 max_rank=5 min=764 min_rank=3 imbalance_ratio=1.241211 max_load_reduction=6921 max_load_reduction_pct=0.844849 moved_to_echo=4356 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 805, 1024, 0, 1219, 0, 0, 950, 1122, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 805, 1024, 1219, 950, 1122] avg=1024.0000 max=1219 max_rank=5 min=805 min_rank=3 imbalance_ratio=1.190430 max_load_reduction=6973 max_load_reduction_pct=0.851196 moved_to_echo=4315 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 655, 1024, 0, 1418, 0, 0, 1100, 923, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 655, 1024, 1418, 1100, 923] avg=1024.0000 max=1418 max_rank=5 min=655 min_rank=3 imbalance_ratio=1.384766 max_load_reduction=6774 max_load_reduction_pct=0.826904 moved_to_echo=4465 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 805, 1024, 0, 1216, 0, 0, 943, 1132, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 805, 1024, 1216, 943, 1132] avg=1024.0000 max=1216 max_rank=5 min=805 min_rank=3 imbalance_ratio=1.187500 max_load_reduction=6976 max_load_reduction_pct=0.851562 moved_to_echo=4315 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 842, 1024, 0, 1165, 0, 0, 903, 1186, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 842, 1024, 1165, 903, 1186] avg=1024.0000 max=1186 max_rank=7 min=842 min_rank=3 imbalance_ratio=1.158203 max_load_reduction=7006 max_load_reduction_pct=0.855225 moved_to_echo=4278 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 420, 1024, 0, 1737, 0, 0, 1347, 592, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 420, 1024, 1737, 1347, 592] avg=1024.0000 max=1737 max_rank=5 min=420 min_rank=3 imbalance_ratio=1.696289 max_load_reduction=6455 max_load_reduction_pct=0.787964 moved_to_echo=4700 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 843, 1024, 0, 1164, 0, 0, 903, 1186, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 843, 1024, 1164, 903, 1186] avg=1024.0000 max=1186 max_rank=7 min=843 min_rank=3 imbalance_ratio=1.158203 max_load_reduction=7006 max_load_reduction_pct=0.855225 moved_to_echo=4277 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5826 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5826, 8192, 10558, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5826, 8192, 10558, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5826, 8192, 10558, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5826, 8192, 10558, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5826, 8192, 10558, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5826, 8192, 10558, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10558 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5826, 8192, 10558, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 5826, 8192, 10558, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=5826 actual_valid_recv=5888 observed_loss=-62 tokens_per_expert=[0, 5888] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=10558 actual_valid_recv=10624 observed_loss=-66 tokens_per_expert=[10624, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=5888 slack_slots=10496 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=10624 slack_slots=5760 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 4096, 0, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1024, 0, 1023, 1024, 0, 0, 1024, 3, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1025, 1024, 1023, 1024, 1024, 1025, 1023] avg=1024.0000 max=1025 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1024, 1024, 1024, 1024, 1023, 1024] avg=1024.0000 max=1025 max_rank=1 min=1023 min_rank=6 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1024, 1024, 1024, 1024, 1023, 1024] avg=1024.0000 max=1025 max_rank=1 min=1023 min_rank=6 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1024, 1024, 1024, 1024, 1023, 1024] avg=1024.0000 max=1025 max_rank=1 min=1023 min_rank=6 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1017, 0, 0, 1024, 0, 1025, 1024, 0, 0, 1024, 0, 1029, 0, 1025] rerouted_tokens_per_ep_rank=[1024, 1017, 1024, 1025, 1024, 1024, 1029, 1025] avg=1024.0000 max=1029 max_rank=6 min=1017 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1024, 1024, 1024, 1024, 1023, 1024] avg=1024.0000 max=1025 max_rank=1 min=1023 min_rank=6 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1024, 1024, 1024, 1024, 1023, 1024] avg=1024.0000 max=1025 max_rank=1 min=1023 min_rank=6 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1024, 1024, 1024, 1024, 1023, 1024] avg=1024.0000 max=1025 max_rank=1 min=1023 min_rank=6 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1024, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 1024, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024] avg=1024.0000 max=1024 max_rank=0 min=1024 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=7168 max_load_reduction_pct=0.875000 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1823, 0, 1, 0, 3818, 0, 2550, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3217, 0, 2, 0, 3103, 0, 1870, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1698, 0, 0, 1, 3758, 0, 2735, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2074, 0, 0, 0, 3749, 0, 2369, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1920, 0, 0, 2, 3847, 3, 2420, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1753, 2, 3, 0, 3627, 0, 2806, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2348, 4, 0, 0, 3599, 0, 2241, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2150, 1, 1, 0, 3575, 0, 2465, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1667, 0, 0, 1550, 2, 728, 0, 480, 876, 0, 0, 873, 1142, 0, 0, 874] rerouted_tokens_per_ep_rank=[1667, 1550, 730, 480, 876, 873, 1142, 874] avg=1024.0000 max=1667 max_rank=0 min=480 min_rank=3 imbalance_ratio=1.627930 max_load_reduction=6525 max_load_reduction_pct=0.796509 moved_to_echo=3765 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[945, 0, 0, 878, 1, 993, 0, 591, 1077, 0, 0, 1075, 1557, 0, 0, 1075] rerouted_tokens_per_ep_rank=[945, 878, 994, 591, 1077, 1075, 1557, 1075] avg=1024.0000 max=1557 max_rank=6 min=591 min_rank=3 imbalance_ratio=1.520508 max_load_reduction=6635 max_load_reduction_pct=0.809937 moved_to_echo=4784 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1075, 0, 0, 999, 0, 922, 0, 580, 1058, 0, 0, 1055, 1447, 0, 0, 1056] rerouted_tokens_per_ep_rank=[1075, 999, 922, 580, 1058, 1055, 1447, 1056] avg=1024.0000 max=1447 max_rank=6 min=580 min_rank=3 imbalance_ratio=1.413086 max_load_reduction=6745 max_load_reduction_pct=0.823364 moved_to_echo=4616 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[991, 0, 0, 929, 0, 946, 2, 600, 1072, 0, 3, 1088, 1474, 0, 0, 1087] rerouted_tokens_per_ep_rank=[991, 929, 946, 602, 1072, 1091, 1474, 1087] avg=1024.0000 max=1474 max_rank=6 min=602 min_rank=3 imbalance_ratio=1.439453 max_load_reduction=6718 max_load_reduction_pct=0.820068 moved_to_echo=4724 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[880, 0, 0, 818, 0, 1065, 1, 582, 1060, 0, 0, 1058, 1670, 0, 0, 1058] rerouted_tokens_per_ep_rank=[880, 818, 1065, 583, 1060, 1058, 1670, 1058] avg=1024.0000 max=1670 max_rank=6 min=583 min_rank=3 imbalance_ratio=1.630859 max_load_reduction=6522 max_load_reduction_pct=0.796143 moved_to_echo=4846 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[909, 0, 2, 844, 3, 1093, 0, 561, 1024, 0, 0, 1021, 1713, 0, 1, 1021] rerouted_tokens_per_ep_rank=[909, 846, 1096, 561, 1024, 1021, 1713, 1022] avg=1024.0000 max=1713 max_rank=6 min=561 min_rank=3 imbalance_ratio=1.672852 max_load_reduction=6479 max_load_reduction_pct=0.790894 moved_to_echo=4780 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1217, 0, 4, 1131, 0, 872, 0, 557, 1016, 0, 0, 1013, 1369, 0, 0, 1013] rerouted_tokens_per_ep_rank=[1217, 1135, 872, 557, 1016, 1013, 1369, 1013] avg=1024.0000 max=1369 max_rank=6 min=557 min_rank=3 imbalance_ratio=1.336914 max_load_reduction=6823 max_load_reduction_pct=0.832886 moved_to_echo=4411 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1114, 0, 1, 1036, 1, 960, 0, 553, 1009, 0, 0, 1006, 1505, 0, 0, 1007] rerouted_tokens_per_ep_rank=[1114, 1037, 961, 553, 1009, 1006, 1505, 1007] avg=1024.0000 max=1505 max_rank=6 min=553 min_rank=3 imbalance_ratio=1.469727 max_load_reduction=6687 max_load_reduction_pct=0.816284 moved_to_echo=4527 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8798, 8192, 7586, 4507, 8192, 8192, 11877, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7586 expected_overflow=0 exact_recv_per_rank=[8798, 8192, 7586, 4507, 8192, 8192, 11877, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8798, 8192, 7586, 4507, 8192, 8192, 11877, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4507 expected_overflow=0 exact_recv_per_rank=[8798, 8192, 7586, 4507, 8192, 8192, 11877, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8798, 8192, 7586, 4507, 8192, 8192, 11877, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8798 expected_overflow=0 exact_recv_per_rank=[8798, 8192, 7586, 4507, 8192, 8192, 11877, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11877 expected_overflow=0 exact_recv_per_rank=[8798, 8192, 7586, 4507, 8192, 8192, 11877, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8798, 8192, 7586, 4507, 8192, 8192, 11877, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=4507 actual_valid_recv=4736 observed_loss=-229 tokens_per_expert=[128, 4608] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=7586 actual_valid_recv=7808 observed_loss=-222 tokens_per_expert=[128, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8798 actual_valid_recv=8832 observed_loss=-34 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=11877 actual_valid_recv=11904 observed_loss=-27 tokens_per_expert=[11904, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=4736 slack_slots=11648 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=11904 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4092, 0, 4095, 0, 0, 0, 0, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 0, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1025, 1024, 1024, 1024, 1024, 1023] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1025, 1024, 1024, 1024, 1024, 1023] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1026, 0, 0, 1024, 0, 1023, 0, 1023, 1, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1026, 1024, 1023, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=2 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1019, 0, 0, 1029, 1016, 0, 0, 1024, 0, 1025, 0, 1025, 0, 1023, 5, 1026] rerouted_tokens_per_ep_rank=[1019, 1029, 1016, 1024, 1025, 1025, 1023, 1031] avg=1024.0000 max=1031 max_rank=7 min=1016 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1025, 1024, 1024, 1024, 1024, 1023] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1025, 1024, 1024, 1024, 1024, 1023] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1025, 1024, 1024, 1024, 1024, 1023] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 1025, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1025, 1024, 1024, 1024, 1024, 1023] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8194, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8194, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8194, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8194, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8194, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8194, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8194, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8194, 8190, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8190 actual_valid_recv=8192 observed_loss=-2 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 2751, 4090, 0, 0, 1350, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 3010, 4095, 0, 1, 1086, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 2973, 4096, 0, 0, 1123, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 3525, 4096, 0, 0, 551, 20, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 2545, 4096, 0, 0, 1544, 7, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 2275, 4096, 0, 0, 1820, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 2312, 4096, 0, 0, 1784, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 2553, 4095, 0, 0, 1541, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1021, 1029, 0, 1025, 0, 0, 1022, 0, 1026, 1350, 0, 0, 696, 0, 1022] rerouted_tokens_per_ep_rank=[1022, 1029, 1025, 1022, 1026, 1350, 696, 1022] avg=1024.0000 max=1350 max_rank=5 min=696 min_rank=6 imbalance_ratio=1.318359 max_load_reduction=6842 max_load_reduction_pct=0.835205 moved_to_echo=4094 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1318, 0, 1025, 0, 0, 1024, 0, 1315, 551, 0, 20, 892, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1318, 1025, 1024, 1315, 551, 912, 1024] avg=1024.0000 max=1318 max_rank=1 min=551 min_rank=5 imbalance_ratio=1.287109 max_load_reduction=6874 max_load_reduction_pct=0.839111 moved_to_echo=3802 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1112, 0, 1025, 0, 0, 1024, 0, 1109, 1123, 0, 0, 752, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1112, 1025, 1024, 1109, 1123, 752, 1024] avg=1024.0000 max=1123 max_rank=5 min=752 min_rank=6 imbalance_ratio=1.096680 max_load_reduction=7069 max_load_reduction_pct=0.862915 moved_to_echo=4008 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1126, 0, 1024, 0, 0, 1024, 1, 1123, 1086, 0, 0, 761, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1126, 1024, 1024, 1124, 1086, 761, 1024] avg=1024.0000 max=1126 max_rank=1 min=761 min_rank=6 imbalance_ratio=1.099609 max_load_reduction=7066 max_load_reduction_pct=0.862549 moved_to_echo=3995 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 943, 0, 1019, 0, 0, 1026, 0, 955, 1544, 0, 7, 647, 0, 1026] rerouted_tokens_per_ep_rank=[1025, 943, 1019, 1026, 955, 1544, 654, 1026] avg=1024.0000 max=1544 max_rank=5 min=654 min_rank=6 imbalance_ratio=1.507812 max_load_reduction=6648 max_load_reduction_pct=0.811523 moved_to_echo=4179 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 865, 0, 1025, 0, 0, 1024, 0, 862, 1784, 0, 0, 585, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 865, 1025, 1024, 862, 1784, 585, 1024] avg=1024.0000 max=1784 max_rank=5 min=585 min_rank=6 imbalance_ratio=1.742188 max_load_reduction=6408 max_load_reduction_pct=0.782227 moved_to_echo=4255 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 851, 0, 1025, 0, 0, 1024, 0, 849, 1820, 0, 1, 575, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 851, 1025, 1024, 849, 1820, 576, 1024] avg=1024.0000 max=1820 max_rank=5 min=576 min_rank=6 imbalance_ratio=1.777344 max_load_reduction=6372 max_load_reduction_pct=0.777832 moved_to_echo=4269 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 955, 0, 1024, 0, 0, 1024, 0, 952, 1541, 0, 3, 646, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 955, 1024, 1024, 952, 1541, 649, 1024] avg=1024.0000 max=1541 max_rank=5 min=649 min_rank=6 imbalance_ratio=1.504883 max_load_reduction=6651 max_load_reduction_pct=0.811890 moved_to_echo=4166 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10799 expected_overflow=0 exact_recv_per_rank=[8185, 8199, 8192, 8192, 8192, 10799, 5585, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8199, 8192, 8192, 8192, 10799, 5585, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8185, 8199, 8192, 8192, 8192, 10799, 5585, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8199, 8192, 8192, 8192, 10799, 5585, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5585 expected_overflow=0 exact_recv_per_rank=[8185, 8199, 8192, 8192, 8192, 10799, 5585, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8199, 8192, 8192, 8192, 10799, 5585, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8185, 8199, 8192, 8192, 8192, 10799, 5585, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8199, 8192, 8192, 8192, 10799, 5585, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=5585 actual_valid_recv=5760 observed_loss=-175 tokens_per_expert=[128, 5632] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8185 actual_valid_recv=8320 observed_loss=-135 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=10799 actual_valid_recv=10880 observed_loss=-81 tokens_per_expert=[10880, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=5760 slack_slots=10624 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=10880 slack_slots=5504 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1360, 0, 0, 0, 2736, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1863, 0, 0, 0, 2233, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1269, 0, 0, 0, 2827, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1617, 0, 0, 1, 2478, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1466, 0, 1, 0, 2629, 4095, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1200, 0, 0, 0, 2896, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1820, 0, 0, 0, 2276, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1679, 0, 0, 0, 2417, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1360, 0, 0, 1024, 0, 548, 0, 1023, 1095, 0, 1025, 0, 0, 1093, 0, 1024] rerouted_tokens_per_ep_rank=[1360, 1024, 548, 1023, 1095, 1025, 1093, 1024] avg=1024.0000 max=1360 max_rank=0 min=548 min_rank=2 imbalance_ratio=1.328125 max_load_reduction=6832 max_load_reduction_pct=0.833984 moved_to_echo=4237 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1466, 0, 0, 1023, 1, 527, 0, 1023, 1052, 0, 1026, 0, 1, 1050, 0, 1023] rerouted_tokens_per_ep_rank=[1466, 1023, 528, 1023, 1052, 1026, 1051, 1023] avg=1024.0000 max=1466 max_rank=0 min=528 min_rank=2 imbalance_ratio=1.431641 max_load_reduction=6726 max_load_reduction_pct=0.821045 moved_to_echo=4152 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1269, 0, 0, 1024, 0, 566, 0, 1023, 1132, 0, 1025, 0, 0, 1129, 0, 1024] rerouted_tokens_per_ep_rank=[1269, 1024, 566, 1023, 1132, 1025, 1129, 1024] avg=1024.0000 max=1269 max_rank=0 min=566 min_rank=2 imbalance_ratio=1.239258 max_load_reduction=6923 max_load_reduction_pct=0.845093 moved_to_echo=4310 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1200, 0, 0, 1024, 0, 580, 0, 1023, 1159, 0, 1025, 0, 0, 1157, 0, 1024] rerouted_tokens_per_ep_rank=[1200, 1024, 580, 1023, 1159, 1025, 1157, 1024] avg=1024.0000 max=1200 max_rank=0 min=580 min_rank=2 imbalance_ratio=1.171875 max_load_reduction=6992 max_load_reduction_pct=0.853516 moved_to_echo=4365 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1617, 0, 0, 1025, 0, 501, 1, 1030, 982, 0, 1016, 0, 0, 995, 0, 1025] rerouted_tokens_per_ep_rank=[1617, 1025, 501, 1031, 982, 1016, 995, 1025] avg=1024.0000 max=1617 max_rank=0 min=501 min_rank=2 imbalance_ratio=1.579102 max_load_reduction=6575 max_load_reduction_pct=0.802612 moved_to_echo=4018 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1863, 0, 0, 1024, 0, 447, 0, 1023, 894, 0, 1025, 0, 0, 892, 0, 1024] rerouted_tokens_per_ep_rank=[1863, 1024, 447, 1023, 894, 1025, 892, 1024] avg=1024.0000 max=1863 max_rank=0 min=447 min_rank=2 imbalance_ratio=1.819336 max_load_reduction=6329 max_load_reduction_pct=0.772583 moved_to_echo=3835 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1820, 0, 0, 1024, 0, 456, 0, 1023, 911, 0, 1025, 0, 0, 909, 0, 1024] rerouted_tokens_per_ep_rank=[1820, 1024, 456, 1023, 911, 1025, 909, 1024] avg=1024.0000 max=1820 max_rank=0 min=456 min_rank=2 imbalance_ratio=1.777344 max_load_reduction=6372 max_load_reduction_pct=0.777832 moved_to_echo=3869 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1679, 0, 0, 1024, 0, 484, 0, 1023, 967, 0, 1025, 0, 0, 966, 0, 1024] rerouted_tokens_per_ep_rank=[1679, 1024, 484, 1023, 967, 1025, 966, 1024] avg=1024.0000 max=1679 max_rank=0 min=484 min_rank=2 imbalance_ratio=1.639648 max_load_reduction=6513 max_load_reduction_pct=0.795044 moved_to_echo=3982 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4110 expected_overflow=0 exact_recv_per_rank=[12274, 8192, 4110, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12274, 8192, 4110, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=12274 expected_overflow=0 exact_recv_per_rank=[12274, 8192, 4110, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12274, 8192, 4110, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12274, 8192, 4110, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12274, 8192, 4110, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12274, 8192, 4110, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[12274, 8192, 4110, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=4110 actual_valid_recv=4352 observed_loss=-242 tokens_per_expert=[128, 4224] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=12274 actual_valid_recv=12288 observed_loss=-14 tokens_per_expert=[12288, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=4352 slack_slots=12032 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=12288 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 4095, 1, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1026, 1023, 1024, 1025, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1026, 1023, 1024, 1025, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1023, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1026, 1024, 1023, 1026, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 0, 1030, 0, 1025, 1011, 0, 0, 1030, 0, 1025, 1016, 0, 0, 1025] rerouted_tokens_per_ep_rank=[1030, 1030, 1025, 1011, 1030, 1025, 1016, 1025] avg=1024.0000 max=1030 max_rank=0 min=1011 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1026, 1023, 1024, 1025, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1026, 1023, 1024, 1025, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1026, 1023, 1024, 1025, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1026, 1023, 1024, 1025, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 17, 0, 0, 0, 4079] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 10, 0, 0, 0, 4086] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 2, 0, 0, 0, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 11, 0, 0, 0, 4085] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 1, 3, 0, 0, 1, 4092] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 9, 0, 0, 0, 4087] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1025, 0, 1019, 0, 1024, 0, 1023, 0, 1025, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1025, 1025, 1019, 1024, 1023, 1025, 1027] avg=1024.0000 max=1027 max_rank=7 min=1019 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1025, 0, 1019, 0, 1024, 0, 1023, 0, 1025, 1027, 0] rerouted_tokens_per_ep_rank=[1024, 1025, 1025, 1019, 1024, 1023, 1025, 1027] avg=1024.0000 max=1027 max_rank=7 min=1019 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 1016, 0, 0, 1024, 17, 1018, 0, 1025, 0, 1030, 0, 1024, 1013, 0] rerouted_tokens_per_ep_rank=[1025, 1016, 1024, 1035, 1025, 1030, 1024, 1013] avg=1024.0000 max=1035 max_rank=3 min=1013 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1, 1024, 3, 1018, 0, 1023, 0, 1023, 1, 1024, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1026, 1025, 1021, 1023, 1023, 1025, 1026] avg=1024.0000 max=1026 max_rank=1 min=1021 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 10, 1016, 0, 1024, 0, 1023, 0, 1023, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1026, 1024, 1023, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1022, 11, 1016, 0, 1024, 0, 1023, 0, 1022, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1025, 1022, 1027, 1024, 1023, 1022, 1025] avg=1024.0000 max=1027 max_rank=3 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1025, 2, 1018, 0, 1024, 0, 1023, 0, 1025, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1025, 1025, 1020, 1024, 1023, 1025, 1026] avg=1024.0000 max=1026 max_rank=7 min=1020 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 9, 1016, 0, 1024, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1025, 1024, 1023, 1023, 1025] avg=1024.0000 max=1025 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1026, 0, 1, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1023, 1026, 1026, 1024, 1024] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 0, 1025, 0, 1030, 0, 1025, 1016, 0, 1011, 0, 0, 1030, 0, 1030] rerouted_tokens_per_ep_rank=[1025, 1025, 1030, 1025, 1016, 1011, 1030, 1030] avg=1024.0000 max=1030 max_rank=2 min=1011 min_rank=5 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 0, 1024, 1025, 0, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1024, 1025, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 1, 4095, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1023, 1025, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 1026, 0, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1023, 1024, 1026, 1024, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1011, 0, 0, 1025, 0, 1030, 0, 1025, 0, 1030, 1016, 0, 0, 1030, 0, 1025] rerouted_tokens_per_ep_rank=[1011, 1025, 1030, 1025, 1030, 1016, 1030, 1025] avg=1024.0000 max=1030 max_rank=2 min=1011 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1023, 1025, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1023, 1025, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1023, 1025, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1023, 1025, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1023, 1025, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4095, 1, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1024, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1024, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 0, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1026, 1024, 1023, 1023, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1024, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 1016, 0, 0, 1025, 1011, 0, 0, 1030, 0, 1025, 0, 1025, 0, 1030] rerouted_tokens_per_ep_rank=[1030, 1016, 1025, 1011, 1030, 1025, 1025, 1030] avg=1024.0000 max=1030 max_rank=0 min=1011 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4110 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1024, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1024, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1024, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2363, 0, 1733] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2996, 0, 1100] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2191, 0, 1905] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1961, 0, 2135] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2520, 0, 1576] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4095, 2512, 0, 1583] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2065, 0, 2031] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2043, 0, 2053] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1037, 0, 727, 0, 1023, 1025, 0, 1326, 0, 0, 1024, 1006, 0] rerouted_tokens_per_ep_rank=[1024, 1037, 727, 1023, 1025, 1326, 1024, 1006] avg=1024.0000 max=1326 max_rank=5 min=727 min_rank=2 imbalance_ratio=1.294922 max_load_reduction=6866 max_load_reduction_pct=0.838135 moved_to_echo=4381 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1315, 0, 461, 0, 1023, 1025, 0, 1681, 0, 0, 1024, 639, 0] rerouted_tokens_per_ep_rank=[1024, 1315, 461, 1023, 1025, 1681, 1024, 639] avg=1024.0000 max=1681 max_rank=5 min=461 min_rank=2 imbalance_ratio=1.641602 max_load_reduction=6511 max_load_reduction_pct=0.794800 moved_to_echo=4369 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1103, 1, 664, 0, 1023, 1026, 0, 1409, 0, 0, 1023, 919, 0] rerouted_tokens_per_ep_rank=[1023, 1104, 665, 1023, 1026, 1409, 1023, 919] avg=1024.0000 max=1409 max_rank=5 min=665 min_rank=2 imbalance_ratio=1.375977 max_load_reduction=6783 max_load_reduction_pct=0.828003 moved_to_echo=4377 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1106, 0, 661, 0, 1023, 1025, 0, 1414, 0, 0, 1024, 915, 0] rerouted_tokens_per_ep_rank=[1024, 1106, 661, 1023, 1025, 1414, 1024, 915] avg=1024.0000 max=1414 max_rank=5 min=661 min_rank=2 imbalance_ratio=1.380859 max_load_reduction=6778 max_load_reduction_pct=0.827393 moved_to_echo=4378 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 0, 966, 0, 803, 0, 1030, 1016, 0, 1225, 0, 0, 1025, 1102, 0] rerouted_tokens_per_ep_rank=[1025, 966, 803, 1030, 1016, 1225, 1025, 1102] avg=1024.0000 max=1225 max_rank=5 min=803 min_rank=2 imbalance_ratio=1.196289 max_load_reduction=6967 max_load_reduction_pct=0.850464 moved_to_echo=4368 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 861, 0, 895, 0, 1023, 1025, 0, 1100, 0, 0, 1024, 1240, 0] rerouted_tokens_per_ep_rank=[1024, 861, 895, 1023, 1025, 1100, 1024, 1240] avg=1024.0000 max=1240 max_rank=7 min=861 min_rank=1 imbalance_ratio=1.210938 max_load_reduction=6952 max_load_reduction_pct=0.848633 moved_to_echo=4389 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 906, 0, 852, 0, 1023, 1025, 0, 1159, 0, 0, 1024, 1179, 0] rerouted_tokens_per_ep_rank=[1024, 906, 852, 1023, 1025, 1159, 1024, 1179] avg=1024.0000 max=1179 max_rank=7 min=852 min_rank=2 imbalance_ratio=1.151367 max_load_reduction=7013 max_load_reduction_pct=0.856079 moved_to_echo=4387 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 897, 0, 861, 0, 1023, 1025, 0, 1146, 0, 0, 1024, 1192, 0] rerouted_tokens_per_ep_rank=[1024, 897, 861, 1023, 1025, 1146, 1024, 1192] avg=1024.0000 max=1192 max_rank=7 min=861 min_rank=2 imbalance_ratio=1.164062 max_load_reduction=7000 max_load_reduction_pct=0.854492 moved_to_echo=4387 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5925, 8191, 8192, 10460, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5925, 8191, 8192, 10460, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5925 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5925, 8191, 8192, 10460, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10460 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5925, 8191, 8192, 10460, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5925, 8191, 8192, 10460, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5925, 8191, 8192, 10460, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5925, 8191, 8192, 10460, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5925, 8191, 8192, 10460, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=5925 actual_valid_recv=6144 observed_loss=-219 tokens_per_expert=[128, 6016] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=10460 actual_valid_recv=10496 observed_loss=-36 tokens_per_expert=[10496, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=6144 slack_slots=10240 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=10496 slack_slots=5888 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1, 4095, 1, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 1, 1023, 0, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1024, 1023, 1026, 1023, 1023, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 1011, 0, 0, 1030, 0, 1025, 1016, 0, 0, 1030, 0, 1025, 0, 1025] rerouted_tokens_per_ep_rank=[1030, 1011, 1030, 1025, 1016, 1030, 1025, 1025] avg=1024.0000 max=1030 max_rank=0 min=1011 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 0, 4095, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1, 1023, 1, 1023, 0, 1023, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1024, 1024, 1023, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1011, 0, 0, 1025, 0, 1030, 0, 1030, 0, 1030, 1016, 0, 0, 1025, 0, 1025] rerouted_tokens_per_ep_rank=[1011, 1025, 1030, 1030, 1030, 1016, 1025, 1025] avg=1024.0000 max=1030 max_rank=2 min=1011 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4095, 1, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 4095, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 1026, 0, 1, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1023, 1024, 1026, 1024, 1026] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 0, 1025, 1, 1030, 0, 1025, 0, 1030, 1005, 0, 0, 1030, 1016, 0] rerouted_tokens_per_ep_rank=[1030, 1025, 1031, 1025, 1030, 1005, 1030, 1016] avg=1024.0000 max=1031 max_rank=2 min=1005 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4081 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1023, 1027, 1023, 1025] avg=1024.0000 max=1027 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 4095, 0, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1, 1023, 1, 1023, 1026, 0, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1024, 1024, 1026, 1023, 1026] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 0, 1030, 0, 1025, 0, 1030, 0, 1030, 1011, 0, 0, 1025, 1016, 0] rerouted_tokens_per_ep_rank=[1025, 1030, 1025, 1030, 1030, 1011, 1025, 1016] avg=1024.0000 max=1030 max_rank=1 min=1011 min_rank=5 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4082 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1023, 1023, 1026, 1024, 1025] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8191, 8192, 8192, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 4095, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 1026, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1023, 1024, 1026, 1026] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 0, 1025, 0, 1030, 0, 1025, 0, 1025, 0, 1030, 1011, 0, 1016, 0] rerouted_tokens_per_ep_rank=[1030, 1025, 1030, 1025, 1025, 1030, 1011, 1016] avg=1024.0000 max=1030 max_rank=0 min=1011 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4082 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1, 4095, 0, 0, 0, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1024, 1024, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1023, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1024, 1024, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1024, 1024, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 1011, 0, 0, 1030, 0, 1025, 0, 1025, 0, 1025, 1016, 0, 0, 1030] rerouted_tokens_per_ep_rank=[1030, 1011, 1030, 1025, 1025, 1025, 1016, 1030] avg=1024.0000 max=1030 max_rank=0 min=1011 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1024, 1024, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1024, 1024, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1024, 1024, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True + [2026-08-30 04:54:06] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 666.9 | throughput per GPU (TFLOP/s/GPU): 499.9 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.563999E+00 | loss scale: 1.0 | grad norm: 5.559 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1000, 3, 0, 0, 3864, 3, 3322, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[538, 0, 0, 0, 4017, 0, 3637, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[811, 0, 0, 1, 4001, 0, 3378, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[324, 0, 1, 0, 4068, 1, 3798, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[764, 7, 6, 2, 4010, 0, 3403, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1123, 0, 0, 0, 3679, 0, 3390, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[704, 0, 0, 1, 3999, 1, 3486, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[436, 0, 0, 0, 4049, 1, 3705, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[811, 299, 0, 982, 0, 983, 1, 1033, 1037, 0, 0, 898, 1114, 0, 1, 1033] rerouted_tokens_per_ep_rank=[1110, 982, 983, 1034, 1037, 898, 1114, 1034] avg=1024.0000 max=1114 max_rank=6 min=898 min_rank=5 imbalance_ratio=1.087891 max_load_reduction=7078 max_load_reduction_pct=0.864014 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1000, 294, 3, 966, 0, 966, 0, 998, 1001, 0, 3, 867, 1096, 0, 0, 998] rerouted_tokens_per_ep_rank=[1294, 969, 966, 998, 1001, 870, 1096, 998] avg=1024.0000 max=1294 max_rank=0 min=870 min_rank=5 imbalance_ratio=1.263672 max_load_reduction=6898 max_load_reduction_pct=0.842041 moved_to_echo=3965 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1123, 300, 0, 986, 0, 986, 0, 950, 953, 0, 0, 826, 1118, 0, 0, 950] rerouted_tokens_per_ep_rank=[1423, 986, 986, 950, 953, 826, 1118, 950] avg=1024.0000 max=1423 max_rank=0 min=826 min_rank=5 imbalance_ratio=1.389648 max_load_reduction=6769 max_load_reduction_pct=0.826294 moved_to_echo=3847 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[764, 305, 7, 993, 6, 995, 2, 1039, 1026, 0, 0, 905, 1110, 0, 0, 1040] rerouted_tokens_per_ep_rank=[1069, 1000, 1001, 1041, 1026, 905, 1110, 1040] avg=1024.0000 max=1110 max_rank=6 min=905 min_rank=5 imbalance_ratio=1.083984 max_load_reduction=7082 max_load_reduction_pct=0.864502 moved_to_echo=4081 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[704, 308, 0, 1014, 0, 1014, 1, 1033, 1035, 0, 1, 898, 1150, 0, 1, 1033] rerouted_tokens_per_ep_rank=[1012, 1014, 1014, 1034, 1035, 899, 1150, 1034] avg=1024.0000 max=1150 max_rank=6 min=899 min_rank=5 imbalance_ratio=1.123047 max_load_reduction=7042 max_load_reduction_pct=0.859619 moved_to_echo=4118 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[436, 328, 0, 1078, 0, 1078, 0, 1046, 1048, 0, 1, 909, 1221, 0, 1, 1046] rerouted_tokens_per_ep_rank=[764, 1078, 1078, 1046, 1048, 910, 1221, 1047] avg=1024.0000 max=1221 max_rank=6 min=764 min_rank=0 imbalance_ratio=1.192383 max_load_reduction=6971 max_load_reduction_pct=0.850952 moved_to_echo=4226 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[538, 322, 0, 1058, 0, 1058, 0, 1038, 1039, 0, 0, 902, 1199, 0, 0, 1038] rerouted_tokens_per_ep_rank=[860, 1058, 1058, 1038, 1039, 902, 1199, 1038] avg=1024.0000 max=1199 max_rank=6 min=860 min_rank=0 imbalance_ratio=1.170898 max_load_reduction=6993 max_load_reduction_pct=0.853638 moved_to_echo=4178 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[324, 336, 0, 1105, 1, 1105, 0, 1051, 1053, 0, 1, 913, 1252, 0, 0, 1051] rerouted_tokens_per_ep_rank=[660, 1105, 1106, 1051, 1053, 914, 1252, 1051] avg=1024.0000 max=1252 max_rank=6 min=660 min_rank=0 imbalance_ratio=1.222656 max_load_reduction=6940 max_load_reduction_pct=0.847168 moved_to_echo=4270 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 7124, 9260, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 7124, 9260, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 7124, 9260, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7124 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 7124, 9260, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 7124, 9260, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9260 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 7124, 9260, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 7124, 9260, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 7124, 9260, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[5760, 2560] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=7124 actual_valid_recv=7296 observed_loss=-172 tokens_per_expert=[128, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9260 actual_valid_recv=9344 observed_loss=-84 tokens_per_expert=[9344, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=7296 slack_slots=9088 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9344 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 750, 0, 0, 0, 0, 3346] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4096, 1, 371, 0, 0, 0, 0, 3724] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 324, 0, 0, 0, 0, 3772] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4095, 1, 610, 0, 1, 0, 0, 3485] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 404, 0, 0, 0, 0, 3692] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 313, 0, 0, 0, 0, 3783] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 500, 0, 0, 0, 0, 3596] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4094, 1, 380, 0, 2, 1, 0, 3714] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1, 1047, 371, 580, 0, 1024, 0, 1047, 0, 1023, 0, 1024, 1050, 0] rerouted_tokens_per_ep_rank=[1025, 1048, 951, 1024, 1047, 1023, 1024, 1050] avg=1024.0000 max=1050 max_rank=7 min=951 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=7142 max_load_reduction_pct=0.871826 moved_to_echo=4144 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 0, 1064, 324, 593, 0, 1026, 0, 1064, 0, 1028, 0, 1026, 1051, 0] rerouted_tokens_per_ep_rank=[1016, 1064, 917, 1026, 1064, 1028, 1026, 1051] avg=1024.0000 max=1064 max_rank=1 min=917 min_rank=2 imbalance_ratio=1.039062 max_load_reduction=7128 max_load_reduction_pct=0.870117 moved_to_echo=4169 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 941, 750, 521, 0, 1024, 0, 941, 0, 1023, 0, 1024, 943, 0] rerouted_tokens_per_ep_rank=[1025, 941, 1271, 1024, 941, 1023, 1024, 943] avg=1024.0000 max=1271 max_rank=2 min=941 min_rank=1 imbalance_ratio=1.241211 max_load_reduction=6921 max_load_reduction_pct=0.844849 moved_to_echo=3931 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1, 980, 610, 543, 0, 1023, 1, 980, 0, 1023, 0, 1023, 982, 0] rerouted_tokens_per_ep_rank=[1026, 981, 1153, 1023, 981, 1023, 1023, 982] avg=1024.0000 max=1153 max_rank=2 min=981 min_rank=1 imbalance_ratio=1.125977 max_load_reduction=7039 max_load_reduction_pct=0.859253 moved_to_echo=4009 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1011, 500, 560, 0, 1024, 0, 1011, 0, 1023, 0, 1024, 1014, 0] rerouted_tokens_per_ep_rank=[1025, 1011, 1060, 1024, 1011, 1023, 1024, 1014] avg=1024.0000 max=1060 max_rank=2 min=1011 min_rank=1 imbalance_ratio=1.035156 max_load_reduction=7132 max_load_reduction_pct=0.870605 moved_to_echo=4072 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1038, 404, 575, 0, 1024, 0, 1038, 0, 1023, 0, 1024, 1041, 0] rerouted_tokens_per_ep_rank=[1025, 1038, 979, 1024, 1038, 1023, 1024, 1041] avg=1024.0000 max=1041 max_rank=7 min=979 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4126 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1064, 313, 589, 0, 1024, 0, 1064, 0, 1023, 0, 1024, 1066, 0] rerouted_tokens_per_ep_rank=[1025, 1064, 902, 1024, 1064, 1023, 1024, 1066] avg=1024.0000 max=1066 max_rank=7 min=902 min_rank=2 imbalance_ratio=1.041016 max_load_reduction=7126 max_load_reduction_pct=0.869873 moved_to_echo=4177 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1, 1044, 380, 579, 0, 1023, 2, 1044, 1, 1023, 0, 1023, 1047, 0] rerouted_tokens_per_ep_rank=[1025, 1045, 959, 1023, 1046, 1024, 1023, 1047] avg=1024.0000 max=1047 max_rank=7 min=959 min_rank=2 imbalance_ratio=1.022461 max_load_reduction=7145 max_load_reduction_pct=0.872192 moved_to_echo=4140 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8190, 8192, 8194] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8190 actual_valid_recv=8320 observed_loss=-130 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3712, 4608] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1, 4043, 0, 0, 0, 4025, 122, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 3999, 12, 0, 0, 4042, 139, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 4044, 4, 0, 1, 4047, 96, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[2, 4067, 1, 0, 0, 4025, 95, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 4000, 0, 0, 0, 4093, 99, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 4071, 0, 0, 0, 3989, 132, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 3998, 0, 0, 0, 4065, 129, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 4059, 43, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 972, 1079, 0, 4, 1023, 0, 1024, 1, 1024, 1027, 0, 96, 918, 0, 1024] rerouted_tokens_per_ep_rank=[972, 1079, 1027, 1024, 1025, 1027, 1014, 1024] avg=1024.0000 max=1079 max_rank=1 min=972 min_rank=0 imbalance_ratio=1.053711 max_load_reduction=7113 max_load_reduction_pct=0.868286 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 966, 1079, 0, 0, 1022, 0, 1019, 0, 1019, 1021, 0, 122, 918, 1, 1024] rerouted_tokens_per_ep_rank=[967, 1079, 1022, 1019, 1019, 1021, 1040, 1025] avg=1024.0000 max=1079 max_rank=1 min=967 min_rank=0 imbalance_ratio=1.053711 max_load_reduction=7113 max_load_reduction_pct=0.868286 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 966, 1086, 0, 1, 1028, 0, 1019, 0, 1019, 1021, 0, 95, 923, 2, 1030] rerouted_tokens_per_ep_rank=[968, 1086, 1029, 1019, 1019, 1021, 1018, 1032] avg=1024.0000 max=1086 max_rank=1 min=968 min_rank=0 imbalance_ratio=1.060547 max_load_reduction=7106 max_load_reduction_pct=0.867432 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 974, 1054, 0, 12, 1016, 0, 1027, 0, 1027, 1014, 0, 139, 911, 0, 1018] rerouted_tokens_per_ep_rank=[974, 1054, 1028, 1027, 1027, 1014, 1050, 1018] avg=1024.0000 max=1054 max_rank=1 min=974 min_rank=0 imbalance_ratio=1.029297 max_load_reduction=7138 max_load_reduction_pct=0.871338 moved_to_echo=4109 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 976, 1067, 0, 0, 1011, 0, 1029, 0, 1029, 1031, 0, 129, 907, 0, 1013] rerouted_tokens_per_ep_rank=[976, 1067, 1011, 1029, 1029, 1031, 1036, 1013] avg=1024.0000 max=1067 max_rank=1 min=976 min_rank=0 imbalance_ratio=1.041992 max_load_reduction=7125 max_load_reduction_pct=0.869751 moved_to_echo=4109 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 958, 1087, 0, 0, 1029, 0, 1010, 0, 1010, 1011, 0, 132, 924, 0, 1031] rerouted_tokens_per_ep_rank=[958, 1087, 1029, 1010, 1010, 1011, 1056, 1031] avg=1024.0000 max=1087 max_rank=1 min=958 min_rank=0 imbalance_ratio=1.061523 max_load_reduction=7105 max_load_reduction_pct=0.867310 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 983, 1067, 0, 0, 1012, 0, 1036, 0, 1036, 1038, 0, 99, 908, 0, 1013] rerouted_tokens_per_ep_rank=[983, 1067, 1012, 1036, 1036, 1038, 1007, 1013] avg=1024.0000 max=1067 max_rank=1 min=983 min_rank=0 imbalance_ratio=1.041992 max_load_reduction=7125 max_load_reduction_pct=0.869751 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 975, 1092, 0, 0, 1034, 0, 1028, 0, 1027, 1029, 0, 43, 928, 0, 1036] rerouted_tokens_per_ep_rank=[975, 1092, 1034, 1028, 1027, 1029, 971, 1036] avg=1024.0000 max=1092 max_rank=1 min=971 min_rank=6 imbalance_ratio=1.066406 max_load_reduction=7100 max_load_reduction_pct=0.866699 moved_to_echo=4063 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7773, 8611, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7773 expected_overflow=0 exact_recv_per_rank=[7773, 8611, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7773, 8611, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7773, 8611, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7773, 8611, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7773, 8611, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8611 expected_overflow=0 exact_recv_per_rank=[7773, 8611, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7773, 8611, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=7773 actual_valid_recv=7936 observed_loss=-163 tokens_per_expert=[128, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[896, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8611 actual_valid_recv=8704 observed_loss=-93 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=7936 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[2419, 0, 2, 380, 1817, 3572, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1632, 0, 0, 387, 2844, 3329, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[2555, 0, 0, 729, 1442, 3465, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2539, 0, 0, 368, 2357, 2928, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[2547, 0, 0, 507, 1337, 3801, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[2044, 0, 0, 403, 2109, 3636, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2242, 0, 1, 269, 2101, 3578, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[2216, 0, 0, 383, 1705, 3888, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1090, 0, 0, 1037, 2, 607, 380, 551, 1266, 0, 1498, 0, 2, 722, 0, 1037] rerouted_tokens_per_ep_rank=[1090, 1037, 609, 931, 1266, 1498, 724, 1037] avg=1024.0000 max=1498 max_rank=5 min=609 min_rank=2 imbalance_ratio=1.462891 max_load_reduction=6694 max_load_reduction_pct=0.817139 moved_to_echo=4525 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1151, 0, 0, 1006, 0, 641, 729, 437, 1005, 0, 1453, 0, 0, 763, 1, 1006] rerouted_tokens_per_ep_rank=[1151, 1006, 641, 1166, 1005, 1453, 763, 1007] avg=1024.0000 max=1453 max_rank=5 min=641 min_rank=2 imbalance_ratio=1.418945 max_load_reduction=6739 max_load_reduction_pct=0.822632 moved_to_echo=4228 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[736, 0, 0, 967, 0, 409, 387, 862, 1982, 0, 1395, 0, 0, 487, 0, 967] rerouted_tokens_per_ep_rank=[736, 967, 409, 1249, 1982, 1395, 487, 967] avg=1024.0000 max=1982 max_rank=4 min=409 min_rank=2 imbalance_ratio=1.935547 max_load_reduction=6210 max_load_reduction_pct=0.758057 moved_to_echo=4831 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1144, 0, 0, 850, 0, 637, 368, 714, 1643, 0, 1228, 0, 0, 758, 0, 850] rerouted_tokens_per_ep_rank=[1144, 850, 637, 1082, 1643, 1228, 758, 850] avg=1024.0000 max=1643 max_rank=4 min=637 min_rank=2 imbalance_ratio=1.604492 max_load_reduction=6549 max_load_reduction_pct=0.799438 moved_to_echo=4479 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1140, 0, 0, 1108, 0, 642, 507, 409, 928, 0, 1586, 0, 0, 765, 0, 1107] rerouted_tokens_per_ep_rank=[1140, 1108, 642, 916, 928, 1586, 765, 1107] avg=1024.0000 max=1586 max_rank=5 min=642 min_rank=2 imbalance_ratio=1.548828 max_load_reduction=6606 max_load_reduction_pct=0.806396 moved_to_echo=4386 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1011, 0, 0, 1039, 1, 562, 269, 637, 1464, 0, 1500, 0, 1, 669, 0, 1039] rerouted_tokens_per_ep_rank=[1011, 1039, 563, 906, 1464, 1500, 670, 1039] avg=1024.0000 max=1500 max_rank=5 min=563 min_rank=2 imbalance_ratio=1.464844 max_load_reduction=6692 max_load_reduction_pct=0.816895 moved_to_echo=4673 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[921, 0, 0, 1056, 0, 513, 403, 639, 1470, 0, 1524, 0, 0, 610, 0, 1056] rerouted_tokens_per_ep_rank=[921, 1056, 513, 1042, 1470, 1524, 610, 1056] avg=1024.0000 max=1524 max_rank=5 min=513 min_rank=2 imbalance_ratio=1.488281 max_load_reduction=6668 max_load_reduction_pct=0.813965 moved_to_echo=4660 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[999, 0, 0, 1129, 0, 556, 383, 517, 1188, 0, 1630, 0, 0, 661, 0, 1129] rerouted_tokens_per_ep_rank=[999, 1129, 556, 900, 1188, 1630, 661, 1129] avg=1024.0000 max=1630 max_rank=5 min=556 min_rank=2 imbalance_ratio=1.591797 max_load_reduction=6562 max_load_reduction_pct=0.801025 moved_to_echo=4608 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4570 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 4570, 8192, 10946, 11814, 5438, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 4570, 8192, 10946, 11814, 5438, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11814 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 4570, 8192, 10946, 11814, 5438, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 4570, 8192, 10946, 11814, 5438, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10946 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 4570, 8192, 10946, 11814, 5438, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5438 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 4570, 8192, 10946, 11814, 5438, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 4570, 8192, 10946, 11814, 5438, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 4570, 8192, 10946, 11814, 5438, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=4570 actual_valid_recv=4736 observed_loss=-166 tokens_per_expert=[128, 4608] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=5438 actual_valid_recv=5632 observed_loss=-194 tokens_per_expert=[128, 5504] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3456, 4864] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=11814 actual_valid_recv=11904 observed_loss=-90 tokens_per_expert=[11904, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=10946 actual_valid_recv=11008 observed_loss=-62 tokens_per_expert=[11008, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=4736 slack_slots=11648 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=5632 slack_slots=10752 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=11904 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=11008 slack_slots=5376 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 4094, 2, 0, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 4095, 1, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 2, 1023, 0, 1023, 1025, 0, 2, 1023, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1025, 1025, 1023, 1025, 1023] avg=1024.0000 max=1025 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 1028, 0, 1026, 1016, 0, 0, 1028, 0, 1028, 1014, 0, 0, 1026] rerouted_tokens_per_ep_rank=[1026, 1028, 1026, 1016, 1028, 1028, 1014, 1026] avg=1024.0000 max=1028 max_rank=1 min=1014 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1023, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1026, 1024, 1023, 1026, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8189, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8189, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8189, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8189, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8189, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8189, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8189, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8189, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8189 actual_valid_recv=8192 observed_loss=-3 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[3851, 4094, 2, 243, 0, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[3543, 4096, 0, 548, 0, 0, 0, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[3860, 4096, 0, 235, 0, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[3938, 4095, 1, 140, 0, 0, 1, 17] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[3842, 4096, 0, 163, 0, 0, 0, 91] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[3543, 4096, 0, 523, 0, 0, 0, 30] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[3987, 4096, 0, 105, 0, 0, 0, 4] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[3859, 4096, 0, 237, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1038, 0, 1025, 0, 2, 1036, 243, 759, 0, 1023, 0, 1023, 2, 1023, 0, 1018] rerouted_tokens_per_ep_rank=[1038, 1025, 1038, 1002, 1023, 1023, 1025, 1018] avg=1024.0000 max=1038 max_rank=0 min=1002 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1036, 0, 1025, 0, 0, 1034, 163, 757, 0, 1024, 0, 1024, 0, 1023, 91, 1015] rerouted_tokens_per_ep_rank=[1036, 1025, 1034, 920, 1024, 1024, 1023, 1106] avg=1024.0000 max=1106 max_rank=7 min=920 min_rank=3 imbalance_ratio=1.080078 max_load_reduction=7086 max_load_reduction_pct=0.864990 moved_to_echo=4177 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1040, 0, 1025, 0, 0, 1039, 235, 761, 0, 1024, 0, 1024, 0, 1023, 1, 1020] rerouted_tokens_per_ep_rank=[1040, 1025, 1039, 996, 1024, 1024, 1023, 1021] avg=1024.0000 max=1040 max_rank=0 min=996 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[945, 0, 1016, 0, 0, 957, 548, 701, 0, 1026, 0, 1026, 0, 1028, 5, 940] rerouted_tokens_per_ep_rank=[945, 1016, 957, 1249, 1026, 1026, 1028, 945] avg=1024.0000 max=1249 max_rank=3 min=945 min_rank=0 imbalance_ratio=1.219727 max_load_reduction=6943 max_load_reduction_pct=0.847534 moved_to_echo=4025 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[956, 0, 1025, 0, 0, 953, 523, 698, 0, 1024, 0, 1024, 0, 1023, 30, 936] rerouted_tokens_per_ep_rank=[956, 1025, 953, 1221, 1024, 1024, 1023, 966] avg=1024.0000 max=1221 max_rank=3 min=953 min_rank=2 imbalance_ratio=1.192383 max_load_reduction=6971 max_load_reduction_pct=0.850952 moved_to_echo=4037 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1062, 0, 1026, 0, 1, 1059, 140, 776, 0, 1023, 0, 1023, 1, 1023, 17, 1041] rerouted_tokens_per_ep_rank=[1062, 1026, 1060, 916, 1023, 1023, 1024, 1058] avg=1024.0000 max=1062 max_rank=0 min=916 min_rank=3 imbalance_ratio=1.037109 max_load_reduction=7130 max_load_reduction_pct=0.870361 moved_to_echo=4128 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1074, 0, 1025, 0, 0, 1073, 105, 786, 0, 1024, 0, 1024, 0, 1023, 4, 1054] rerouted_tokens_per_ep_rank=[1074, 1025, 1073, 891, 1024, 1024, 1023, 1058] avg=1024.0000 max=1074 max_rank=0 min=891 min_rank=3 imbalance_ratio=1.048828 max_load_reduction=7118 max_load_reduction_pct=0.868896 moved_to_echo=4129 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1041, 0, 1025, 0, 0, 1038, 237, 760, 0, 1024, 0, 1024, 0, 1023, 0, 1020] rerouted_tokens_per_ep_rank=[1041, 1025, 1038, 997, 1024, 1024, 1023, 1020] avg=1024.0000 max=1041 max_rank=0 min=997 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2304, 6016] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 4094, 2, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 1, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1024, 1024, 1024, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 2, 1023, 2, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1023, 1023, 1023, 1025, 1025, 1025] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1024, 1024, 1024, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1024, 0, 1024, 1, 1023, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1024, 1024, 1024, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1014, 0, 0, 1028, 0, 1026, 0, 1026, 0, 1026, 1016, 0, 0, 1028, 0, 1028] rerouted_tokens_per_ep_rank=[1014, 1028, 1026, 1026, 1026, 1016, 1028, 1028] avg=1024.0000 max=1028 max_rank=1 min=1014 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1024, 1024, 1024, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1023, 1023, 1026, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1024, 0, 1024, 0, 1024, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1024, 1024, 1024, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8195, 8189, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8189 actual_valid_recv=8192 observed_loss=-3 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4096, 92, 0, 0, 0, 4004, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4094, 115, 0, 0, 2, 3979, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4096, 43, 0, 0, 0, 4053, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4096, 20, 0, 0, 0, 4076, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 99, 0, 0, 0, 3997, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4095, 95, 1, 0, 0, 4000, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4096, 89, 0, 0, 0, 4007, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 117, 0, 0, 0, 3979, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 92, 938, 0, 1023, 0, 1024, 0, 1021, 1024, 0, 0, 1021, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1030, 1023, 1024, 1021, 1024, 1021, 1024] avg=1024.0000 max=1030 max_rank=1 min=1021 min_rank=4 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 115, 932, 0, 1023, 0, 1023, 2, 1015, 1017, 0, 2, 1015, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1047, 1023, 1023, 1017, 1017, 1017, 1023] avg=1024.0000 max=1047 max_rank=1 min=1017 min_rank=4 imbalance_ratio=1.022461 max_load_reduction=7145 max_load_reduction_pct=0.872192 moved_to_echo=4074 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 43, 953, 0, 1028, 0, 1026, 0, 1038, 1024, 0, 0, 1038, 0, 1026] rerouted_tokens_per_ep_rank=[1016, 996, 1028, 1026, 1038, 1024, 1038, 1026] avg=1024.0000 max=1038 max_rank=4 min=996 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4126 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 20, 955, 0, 1023, 0, 1024, 0, 1040, 1042, 0, 0, 1039, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 975, 1023, 1024, 1040, 1042, 1039, 1024] avg=1024.0000 max=1042 max_rank=5 min=975 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=7150 max_load_reduction_pct=0.872803 moved_to_echo=4145 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 99, 936, 0, 1023, 0, 1024, 0, 1019, 1023, 0, 0, 1019, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1035, 1023, 1024, 1019, 1023, 1019, 1024] avg=1024.0000 max=1035 max_rank=1 min=1019 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4085 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 95, 937, 1, 1023, 0, 1023, 0, 1020, 1023, 0, 1, 1020, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1032, 1024, 1023, 1020, 1023, 1021, 1023] avg=1024.0000 max=1032 max_rank=1 min=1020 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 89, 939, 0, 1023, 0, 1024, 0, 1022, 1024, 0, 0, 1022, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1028, 1023, 1024, 1022, 1024, 1022, 1024] avg=1024.0000 max=1028 max_rank=1 min=1022 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 117, 932, 0, 1023, 0, 1024, 0, 1015, 1017, 0, 0, 1015, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1049, 1023, 1024, 1015, 1017, 1015, 1024] avg=1024.0000 max=1049 max_rank=1 min=1015 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=7143 max_load_reduction_pct=0.871948 moved_to_echo=4071 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8194 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8190 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8190, 8192, 8192, 8194, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8190 actual_valid_recv=8320 observed_loss=-130 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8194 actual_valid_recv=8320 observed_loss=-126 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4094, 0, 4094, 2, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4095, 0, 1, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4095, 1, 1, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1026, 0, 0, 1023, 1, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1026, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1023, 1025, 0, 2, 1023, 0, 1023, 2, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1025, 1025, 1023, 1025, 1023] avg=1024.0000 max=1025 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 1016, 0, 0, 1028, 1010, 0, 0, 1028, 0, 1029, 0, 1029, 0, 1026] rerouted_tokens_per_ep_rank=[1026, 1016, 1028, 1010, 1028, 1029, 1029, 1026] avg=1024.0000 max=1029 max_rank=5 min=1010 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4112 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 1026, 0, 1, 1023, 1, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1026, 1024, 1024, 1023, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1027, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8195, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8195, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8195, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8195, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8195, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8195, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8195, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8195, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8189 actual_valid_recv=8192 observed_loss=-3 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 2, 4090, 4094, 4, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1, 1, 1, 4094, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4096, 1, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4074, 4096, 21, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4096, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4085, 4096, 11, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 4096, 2, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 2, 1023, 2, 1023, 1025, 0, 1025, 0, 4, 1019, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1025, 1025, 1025, 1023, 1023, 1023] avg=1024.0000 max=1025 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 1027, 0, 1026, 0, 1, 1020, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1027, 1026, 1021, 1024, 1023] avg=1024.0000 max=1027 max_rank=3 min=1021 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 1027, 0, 1026, 0, 0, 1020, 0, 1024, 1, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1027, 1026, 1020, 1024, 1024] avg=1024.0000 max=1027 max_rank=3 min=1020 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 0, 1023, 0, 1023, 1011, 0, 1011, 0, 21, 1017, 0, 1026, 1, 1029] rerouted_tokens_per_ep_rank=[1030, 1023, 1023, 1011, 1011, 1038, 1026, 1030] avg=1024.0000 max=1038 max_rank=5 min=1011 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1022, 0, 1022, 1024, 0, 1026, 0, 11, 1017, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1022, 1022, 1024, 1026, 1028, 1024, 1023] avg=1024.0000 max=1028 max_rank=5 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1, 1024, 1, 1024, 1026, 0, 1026, 0, 0, 1020, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1025, 1025, 1026, 1026, 1020, 1023, 1023] avg=1024.0000 max=1026 max_rank=3 min=1020 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1025, 0, 1025, 1026, 0, 1026, 0, 0, 1020, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1025, 1026, 1026, 1020, 1024, 1023] avg=1024.0000 max=1026 max_rank=3 min=1020 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 1026, 0, 1026, 0, 2, 1020, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1026, 1026, 1022, 1024, 1023] avg=1024.0000 max=1026 max_rank=3 min=1022 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 4096, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[2, 4094, 2, 0, 4094, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1, 4095, 1, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 1023, 1025, 0, 2, 1023, 0, 1023, 1025, 0, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1025, 1025, 1023, 1025, 1023, 1023, 1023] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=3 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 1, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 1014, 0, 0, 1028, 0, 1026, 1016, 0, 0, 1028, 0, 1026, 0, 1026] rerouted_tokens_per_ep_rank=[1028, 1014, 1028, 1026, 1016, 1028, 1026, 1026] avg=1024.0000 max=1028 max_rank=0 min=1014 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 1, 1023, 0, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1024, 1023, 1026, 1023, 1023, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8189 actual_valid_recv=8192 observed_loss=-3 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4094, 0, 2, 2, 0, 4094, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 1, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 0, 4095, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 2, 1023, 2, 1023, 0, 1023, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1023, 1025, 1025, 1023, 1025, 1023, 1023] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1, 1023, 0, 1023, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1024, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 0, 1026, 0, 1028, 0, 1028, 0, 1026, 1014, 0, 0, 1028, 0, 1026] rerouted_tokens_per_ep_rank=[1016, 1026, 1028, 1028, 1026, 1014, 1028, 1026] avg=1024.0000 max=1028 max_rank=2 min=1014 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1024, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1024, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 1, 1023, 1, 1023, 0, 1023, 1026, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1024, 1024, 1023, 1026, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1024, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1023, 1024, 1026, 1023, 1024] avg=1024.0000 max=1026 max_rank=5 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8195, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8195, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8195, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8195, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8195, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8195, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8195, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8195, 8189, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8189 actual_valid_recv=8192 observed_loss=-3 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 2, 0, 2, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3, 4093, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 2, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 1, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 1, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1027, 1023, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1025, 0, 2, 1023, 0, 1023, 2, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1025, 1025, 1023, 1025, 1025] avg=1024.0000 max=1025 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023, 2, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1025, 1023, 1025, 1023, 1025] avg=1024.0000 max=1025 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 0, 1026, 3, 1028, 1009, 0, 0, 1028, 0, 1028, 0, 1028, 1016, 0] rerouted_tokens_per_ep_rank=[1026, 1026, 1031, 1009, 1028, 1028, 1028, 1016] avg=1024.0000 max=1031 max_rank=2 min=1009 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 1, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1026, 1024, 1023, 1024, 1026] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1027, 1023, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1026, 0, 0, 1023, 1, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1026, 1023, 1024, 1023, 1025] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1027, 1023, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4070, 0, 0, 4096, 0, 26] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4080, 2, 2, 4094, 0, 14] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4064, 1, 1, 4095, 0, 31] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4083, 0, 0, 4096, 0, 13] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4063, 0, 0, 4096, 0, 33] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 0, 4096, 0, 4] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 4, 4031, 0, 0, 4096, 0, 61] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 3992, 0, 0, 4096, 0, 104] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1028, 1031, 0, 2, 1023, 2, 1028, 1025, 0, 0, 1023, 14, 993] rerouted_tokens_per_ep_rank=[1023, 1028, 1031, 1025, 1030, 1025, 1023, 1007] avg=1024.0000 max=1031 max_rank=2 min=1007 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4085 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1026, 1028, 0, 0, 1023, 0, 1026, 1025, 0, 0, 1024, 26, 990] rerouted_tokens_per_ep_rank=[1024, 1026, 1028, 1023, 1026, 1025, 1024, 1016] avg=1024.0000 max=1028 max_rank=2 min=1016 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1029, 1031, 0, 0, 1023, 0, 1029, 1025, 0, 0, 1024, 13, 994] rerouted_tokens_per_ep_rank=[1024, 1029, 1031, 1023, 1029, 1025, 1024, 1007] avg=1024.0000 max=1031 max_rank=2 min=1007 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4085 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1027, 0, 1, 1023, 1, 1024, 1026, 0, 0, 1023, 31, 989] rerouted_tokens_per_ep_rank=[1023, 1024, 1027, 1024, 1025, 1026, 1023, 1020] avg=1024.0000 max=1027 max_rank=2 min=1020 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 4, 1020, 1006, 0, 0, 1028, 0, 1021, 1016, 0, 0, 1026, 61, 984] rerouted_tokens_per_ep_rank=[1026, 1024, 1006, 1028, 1021, 1016, 1026, 1045] avg=1024.0000 max=1045 max_rank=7 min=1006 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=7147 max_load_reduction_pct=0.872437 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 33, 989] rerouted_tokens_per_ep_rank=[1024, 1024, 1026, 1023, 1024, 1025, 1024, 1022] avg=1024.0000 max=1026 max_rank=2 min=1022 min_rank=7 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1006, 1009, 0, 0, 1023, 0, 1006, 1025, 0, 0, 1024, 104, 971] rerouted_tokens_per_ep_rank=[1024, 1006, 1009, 1023, 1006, 1025, 1024, 1075] avg=1024.0000 max=1075 max_rank=7 min=1006 min_rank=1 imbalance_ratio=1.049805 max_load_reduction=7117 max_load_reduction_pct=0.868774 moved_to_echo=4130 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1031, 1034, 0, 0, 1023, 0, 1031, 1025, 0, 0, 1024, 4, 996] rerouted_tokens_per_ep_rank=[1024, 1031, 1034, 1023, 1031, 1025, 1024, 1000] avg=1024.0000 max=1034 max_rank=2 min=1000 min_rank=7 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4080 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 2, 4094, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 4095, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 2, 1023, 1025, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1025, 1023, 1023, 1023, 1023, 1025, 1025, 1025] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 0, 1026, 0, 1028, 0, 1026, 0, 1026, 0, 1028, 1014, 0, 1016, 0] rerouted_tokens_per_ep_rank=[1028, 1026, 1028, 1026, 1026, 1028, 1014, 1016] avg=1024.0000 max=1028 max_rank=0 min=1014 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4084 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 1026, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1023, 1024, 1026, 1026] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8192, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8192, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8192, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8192, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8192, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8192, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8192, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8189, 8192, 8192, 8192, 8195, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8189 actual_valid_recv=8192 observed_loss=-3 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[2, 3938, 0, 156, 0, 0, 4094, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 3913, 0, 183, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 3958, 0, 138, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 3958, 0, 138, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1, 3853, 0, 242, 0, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 3728, 0, 368, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 3675, 0, 421, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 3767, 0, 329, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 1047, 1049, 0, 0, 1023, 156, 795, 0, 1023, 0, 1023, 1025, 0, 2, 1047] rerouted_tokens_per_ep_rank=[1049, 1049, 1023, 951, 1023, 1023, 1025, 1049] avg=1024.0000 max=1049 max_rank=0 min=951 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=7143 max_load_reduction_pct=0.871948 moved_to_echo=4120 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1040, 1043, 0, 0, 1024, 183, 790, 0, 1024, 0, 1023, 1025, 0, 0, 1040] rerouted_tokens_per_ep_rank=[1040, 1043, 1024, 973, 1024, 1023, 1025, 1040] avg=1024.0000 max=1043 max_rank=1 min=973 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=7149 max_load_reduction_pct=0.872681 moved_to_echo=4112 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1001, 1005, 0, 0, 1024, 329, 760, 0, 1024, 0, 1023, 1025, 0, 0, 1001] rerouted_tokens_per_ep_rank=[1001, 1005, 1024, 1089, 1024, 1023, 1025, 1001] avg=1024.0000 max=1089 max_rank=3 min=1001 min_rank=0 imbalance_ratio=1.063477 max_load_reduction=7103 max_load_reduction_pct=0.867065 moved_to_echo=4073 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 996, 981, 0, 0, 1026, 368, 755, 0, 1026, 0, 1028, 1016, 0, 0, 996] rerouted_tokens_per_ep_rank=[996, 981, 1026, 1123, 1026, 1028, 1016, 996] avg=1024.0000 max=1123 max_rank=3 min=981 min_rank=1 imbalance_ratio=1.096680 max_load_reduction=7069 max_load_reduction_pct=0.862915 moved_to_echo=4066 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1024, 1028, 0, 0, 1023, 242, 777, 0, 1023, 0, 1023, 1026, 0, 1, 1024] rerouted_tokens_per_ep_rank=[1025, 1028, 1023, 1019, 1023, 1023, 1026, 1025] avg=1024.0000 max=1028 max_rank=1 min=1019 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 977, 979, 0, 0, 1024, 421, 742, 0, 1024, 0, 1023, 1025, 0, 0, 977] rerouted_tokens_per_ep_rank=[977, 979, 1024, 1163, 1024, 1023, 1025, 977] avg=1024.0000 max=1163 max_rank=3 min=977 min_rank=0 imbalance_ratio=1.135742 max_load_reduction=7029 max_load_reduction_pct=0.858032 moved_to_echo=4049 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1052, 1055, 0, 0, 1024, 138, 799, 0, 1024, 0, 1023, 1025, 0, 0, 1052] rerouted_tokens_per_ep_rank=[1052, 1055, 1024, 937, 1024, 1023, 1025, 1052] avg=1024.0000 max=1055 max_rank=1 min=937 min_rank=3 imbalance_ratio=1.030273 max_load_reduction=7137 max_load_reduction_pct=0.871216 moved_to_echo=4124 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1052, 1055, 0, 0, 1024, 138, 799, 0, 1024, 0, 1023, 1025, 0, 0, 1052] rerouted_tokens_per_ep_rank=[1052, 1055, 1024, 937, 1024, 1023, 1025, 1052] avg=1024.0000 max=1055 max_rank=1 min=937 min_rank=3 imbalance_ratio=1.030273 max_load_reduction=7137 max_load_reduction_pct=0.871216 moved_to_echo=4124 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8189 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8195 expected_overflow=0 exact_recv_per_rank=[8192, 8195, 8192, 8192, 8192, 8189, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2048, 6272] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8189 actual_valid_recv=8192 observed_loss=-3 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8195 actual_valid_recv=8320 observed_loss=-125 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[745, 1, 0, 0, 3901, 0, 3545, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[220, 0, 2, 0, 4087, 0, 3883, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[439, 0, 0, 1, 4055, 1, 3696, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[910, 1, 0, 0, 3796, 0, 3485, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[991, 1, 0, 0, 4018, 12, 3169, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1429, 1, 0, 0, 3795, 0, 2967, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1160, 1, 0, 0, 3922, 0, 3109, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[687, 9, 0, 0, 4041, 0, 3455, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[745, 209, 1, 1061, 0, 868, 0, 1010, 1013, 0, 0, 1061, 1214, 0, 0, 1010] rerouted_tokens_per_ep_rank=[954, 1062, 868, 1010, 1013, 1061, 1214, 1010] avg=1024.0000 max=1214 max_rank=6 min=868 min_rank=2 imbalance_ratio=1.185547 max_load_reduction=6978 max_load_reduction_pct=0.851807 moved_to_echo=4298 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[439, 218, 0, 1106, 0, 903, 1, 1050, 1052, 0, 1, 1106, 1266, 0, 0, 1050] rerouted_tokens_per_ep_rank=[657, 1106, 903, 1051, 1052, 1107, 1266, 1050] avg=1024.0000 max=1266 max_rank=6 min=657 min_rank=0 imbalance_ratio=1.236328 max_load_reduction=6926 max_load_reduction_pct=0.845459 moved_to_echo=4475 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[910, 205, 1, 1043, 0, 845, 0, 983, 985, 0, 0, 1043, 1194, 0, 0, 983] rerouted_tokens_per_ep_rank=[1115, 1044, 845, 983, 985, 1043, 1194, 983] avg=1024.0000 max=1194 max_rank=6 min=845 min_rank=2 imbalance_ratio=1.166016 max_load_reduction=6998 max_load_reduction_pct=0.854248 moved_to_echo=4205 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[991, 189, 1, 953, 0, 898, 0, 1045, 1030, 0, 12, 954, 1073, 0, 1, 1045] rerouted_tokens_per_ep_rank=[1180, 954, 898, 1045, 1030, 966, 1073, 1046] avg=1024.0000 max=1180 max_rank=0 min=898 min_rank=2 imbalance_ratio=1.152344 max_load_reduction=7012 max_load_reduction_pct=0.855957 moved_to_echo=4115 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1160, 183, 1, 931, 0, 873, 0, 1016, 1017, 0, 0, 931, 1064, 0, 0, 1016] rerouted_tokens_per_ep_rank=[1343, 932, 873, 1016, 1017, 931, 1064, 1016] avg=1024.0000 max=1343 max_rank=0 min=873 min_rank=2 imbalance_ratio=1.311523 max_load_reduction=6849 max_load_reduction_pct=0.836060 moved_to_echo=4028 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[220, 229, 0, 1162, 2, 910, 0, 1058, 1061, 0, 0, 1162, 1330, 0, 0, 1058] rerouted_tokens_per_ep_rank=[449, 1162, 912, 1058, 1061, 1162, 1330, 1058] avg=1024.0000 max=1330 max_rank=6 min=449 min_rank=0 imbalance_ratio=1.298828 max_load_reduction=6862 max_load_reduction_pct=0.837646 moved_to_echo=4611 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[687, 203, 9, 1034, 0, 899, 0, 1046, 1050, 0, 0, 1034, 1184, 0, 0, 1046] rerouted_tokens_per_ep_rank=[890, 1043, 899, 1046, 1050, 1034, 1184, 1046] avg=1024.0000 max=1184 max_rank=6 min=890 min_rank=0 imbalance_ratio=1.156250 max_load_reduction=7008 max_load_reduction_pct=0.855469 moved_to_echo=4314 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1429, 175, 1, 888, 0, 845, 0, 983, 984, 0, 0, 888, 1016, 0, 0, 983] rerouted_tokens_per_ep_rank=[1604, 889, 845, 983, 984, 888, 1016, 983] avg=1024.0000 max=1604 max_rank=0 min=845 min_rank=2 imbalance_ratio=1.566406 max_load_reduction=6588 max_load_reduction_pct=0.804199 moved_to_echo=3871 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7043 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7043, 8192, 8192, 8192, 9341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7043, 8192, 8192, 8192, 9341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7043, 8192, 8192, 8192, 9341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7043, 8192, 8192, 8192, 9341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7043, 8192, 8192, 8192, 9341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9341 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7043, 8192, 8192, 8192, 9341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7043, 8192, 8192, 8192, 9341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 7043, 8192, 8192, 8192, 9341, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=7043 actual_valid_recv=7296 observed_loss=-253 tokens_per_expert=[128, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6656, 1664] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=9341 actual_valid_recv=9344 observed_loss=-3 tokens_per_expert=[9344, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=7296 slack_slots=9088 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=9344 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 426, 0, 0, 0, 0, 3670] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 647, 0, 0, 0, 0, 3449] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 551, 0, 0, 0, 0, 3545] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4095, 1, 331, 0, 1, 0, 0, 3764] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 567, 0, 0, 0, 0, 3529] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 445, 0, 0, 0, 0, 3651] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 407, 0, 0, 0, 0, 3689] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 599, 0, 0, 0, 0, 3497] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1044, 426, 537, 0, 1024, 0, 1044, 0, 1023, 0, 1024, 1045, 0] rerouted_tokens_per_ep_rank=[1025, 1044, 963, 1024, 1044, 1023, 1024, 1045] avg=1024.0000 max=1045 max_rank=7 min=963 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=7147 max_load_reduction_pct=0.872437 moved_to_echo=4136 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1008, 551, 519, 0, 1024, 0, 1008, 0, 1023, 0, 1024, 1010, 0] rerouted_tokens_per_ep_rank=[1025, 1008, 1070, 1024, 1008, 1023, 1024, 1010] avg=1024.0000 max=1070 max_rank=2 min=1008 min_rank=1 imbalance_ratio=1.044922 max_load_reduction=7122 max_load_reduction_pct=0.869385 moved_to_echo=4065 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 981, 647, 505, 0, 1024, 0, 981, 0, 1023, 0, 1024, 982, 0] rerouted_tokens_per_ep_rank=[1025, 981, 1152, 1024, 981, 1023, 1024, 982] avg=1024.0000 max=1152 max_rank=2 min=981 min_rank=1 imbalance_ratio=1.125000 max_load_reduction=7040 max_load_reduction_pct=0.859375 moved_to_echo=4010 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1017, 0, 1, 1074, 331, 555, 0, 1024, 1, 1074, 0, 1030, 0, 1024, 1061, 0] rerouted_tokens_per_ep_rank=[1017, 1075, 886, 1024, 1075, 1030, 1024, 1061] avg=1024.0000 max=1075 max_rank=1 min=886 min_rank=2 imbalance_ratio=1.049805 max_load_reduction=7117 max_load_reduction_pct=0.868774 moved_to_echo=4190 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1038, 445, 534, 0, 1024, 0, 1038, 0, 1023, 0, 1024, 1041, 0] rerouted_tokens_per_ep_rank=[1025, 1038, 979, 1024, 1038, 1023, 1024, 1041] avg=1024.0000 max=1041 max_rank=7 min=979 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4126 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1003, 567, 517, 0, 1024, 0, 1003, 0, 1023, 0, 1024, 1006, 0] rerouted_tokens_per_ep_rank=[1025, 1003, 1084, 1024, 1003, 1023, 1024, 1006] avg=1024.0000 max=1084 max_rank=2 min=1003 min_rank=1 imbalance_ratio=1.058594 max_load_reduction=7108 max_load_reduction_pct=0.867676 moved_to_echo=4056 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1049, 407, 540, 0, 1024, 0, 1049, 0, 1023, 0, 1024, 1051, 0] rerouted_tokens_per_ep_rank=[1025, 1049, 947, 1024, 1049, 1023, 1024, 1051] avg=1024.0000 max=1051 max_rank=7 min=947 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=7141 max_load_reduction_pct=0.871704 moved_to_echo=4147 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 994, 599, 512, 0, 1024, 0, 994, 0, 1023, 0, 1024, 997, 0] rerouted_tokens_per_ep_rank=[1025, 994, 1111, 1024, 994, 1023, 1024, 997] avg=1024.0000 max=1111 max_rank=2 min=994 min_rank=1 imbalance_ratio=1.084961 max_load_reduction=7081 max_load_reduction_pct=0.864380 moved_to_echo=4038 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8192, 8193] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4096, 4224] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 4013, 83, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 0, 4008, 104, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 4028, 74, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1, 4095, 1, 0, 0, 4026, 68, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 4013, 0, 0, 0, 4067, 112, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 4083, 0, 0, 0, 4027, 82, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 3989, 107, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 4017, 0, 0, 0, 4095, 80, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1001, 1027, 0, 0, 1017, 0, 1026, 0, 1026, 1045, 0, 104, 929, 0, 1017] rerouted_tokens_per_ep_rank=[1001, 1027, 1017, 1026, 1026, 1045, 1033, 1017] avg=1024.0000 max=1045 max_rank=5 min=1001 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=7147 max_load_reduction_pct=0.872437 moved_to_echo=4121 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1002, 1029, 0, 0, 1022, 0, 1026, 0, 1026, 1049, 0, 82, 934, 0, 1022] rerouted_tokens_per_ep_rank=[1002, 1029, 1022, 1026, 1026, 1049, 1016, 1022] avg=1024.0000 max=1049 max_rank=5 min=1002 min_rank=0 imbalance_ratio=1.024414 max_load_reduction=7143 max_load_reduction_pct=0.871948 moved_to_echo=4113 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1004, 1031, 0, 1, 1019, 0, 1030, 0, 1030, 1045, 0, 83, 930, 0, 1019] rerouted_tokens_per_ep_rank=[1004, 1031, 1020, 1030, 1030, 1045, 1013, 1019] avg=1024.0000 max=1045 max_rank=5 min=1004 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=7147 max_load_reduction_pct=0.872437 moved_to_echo=4107 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1009, 1020, 0, 1, 1027, 0, 1033, 0, 1033, 1034, 0, 68, 938, 1, 1027] rerouted_tokens_per_ep_rank=[1010, 1020, 1028, 1033, 1033, 1034, 1006, 1028] avg=1024.0000 max=1034 max_rank=5 min=1006 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 984, 1011, 0, 0, 1032, 0, 1009, 0, 1009, 1060, 0, 112, 943, 0, 1032] rerouted_tokens_per_ep_rank=[984, 1011, 1032, 1009, 1009, 1060, 1055, 1032] avg=1024.0000 max=1060 max_rank=5 min=984 min_rank=0 imbalance_ratio=1.035156 max_load_reduction=7132 max_load_reduction_pct=0.870605 moved_to_echo=4156 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1003, 1031, 0, 0, 1022, 0, 1028, 0, 1028, 1050, 0, 74, 934, 0, 1022] rerouted_tokens_per_ep_rank=[1003, 1031, 1022, 1028, 1028, 1050, 1008, 1022] avg=1024.0000 max=1050 max_rank=5 min=1003 min_rank=0 imbalance_ratio=1.025391 max_load_reduction=7142 max_load_reduction_pct=0.871826 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1005, 1031, 0, 0, 1012, 0, 1030, 0, 1030, 1039, 0, 107, 925, 0, 1013] rerouted_tokens_per_ep_rank=[1005, 1031, 1012, 1030, 1030, 1039, 1032, 1013] avg=1024.0000 max=1039 max_rank=5 min=1005 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=7153 max_load_reduction_pct=0.873169 moved_to_echo=4114 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 985, 1012, 0, 0, 1039, 0, 1010, 0, 1010, 1068, 0, 80, 949, 0, 1039] rerouted_tokens_per_ep_rank=[985, 1012, 1039, 1010, 1010, 1068, 1029, 1039] avg=1024.0000 max=1068 max_rank=5 min=985 min_rank=0 imbalance_ratio=1.042969 max_load_reduction=7124 max_load_reduction_pct=0.869629 moved_to_echo=4146 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8390 expected_overflow=0 exact_recv_per_rank=[7994, 8192, 8192, 8192, 8192, 8390, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7994, 8192, 8192, 8192, 8192, 8390, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7994 expected_overflow=0 exact_recv_per_rank=[7994, 8192, 8192, 8192, 8192, 8390, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7994, 8192, 8192, 8192, 8192, 8390, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7994, 8192, 8192, 8192, 8192, 8390, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7994, 8192, 8192, 8192, 8192, 8390, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7994, 8192, 8192, 8192, 8192, 8390, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7994, 8192, 8192, 8192, 8192, 8390, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8390 actual_valid_recv=8448 observed_loss=-58 tokens_per_expert=[8448, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[768, 7552] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=7994 actual_valid_recv=8192 observed_loss=-198 tokens_per_expert=[128, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8448 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1985, 0, 0, 636, 2296, 3275, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[2463, 0, 1, 687, 1413, 3627, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1967, 0, 0, 175, 2847, 3203, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1944, 0, 0, 471, 2132, 3645, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1923, 0, 0, 641, 1992, 3636, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[2465, 0, 0, 418, 2258, 3051, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2220, 0, 0, 358, 2364, 3250, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1830, 0, 0, 580, 2213, 3569, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[969, 0, 0, 984, 0, 722, 636, 499, 1574, 0, 1307, 0, 0, 517, 0, 984] rerouted_tokens_per_ep_rank=[969, 984, 722, 1135, 1574, 1307, 517, 984] avg=1024.0000 max=1574 max_rank=4 min=517 min_rank=6 imbalance_ratio=1.537109 max_load_reduction=6618 max_load_reduction_pct=0.807861 moved_to_echo=4382 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[961, 0, 0, 962, 0, 895, 175, 494, 1952, 0, 1279, 0, 0, 512, 0, 962] rerouted_tokens_per_ep_rank=[961, 962, 895, 669, 1952, 1279, 512, 962] avg=1024.0000 max=1952 max_rank=4 min=512 min_rank=6 imbalance_ratio=1.906250 max_load_reduction=6240 max_load_reduction_pct=0.761719 moved_to_echo=4705 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[949, 0, 0, 1095, 0, 670, 471, 489, 1462, 0, 1455, 0, 0, 506, 0, 1095] rerouted_tokens_per_ep_rank=[949, 1095, 670, 960, 1462, 1455, 506, 1095] avg=1024.0000 max=1462 max_rank=4 min=506 min_rank=6 imbalance_ratio=1.427734 max_load_reduction=6730 max_load_reduction_pct=0.821533 moved_to_echo=4518 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1194, 0, 0, 1094, 1, 449, 687, 623, 964, 0, 1439, 0, 1, 646, 0, 1094] rerouted_tokens_per_ep_rank=[1194, 1094, 450, 1310, 964, 1439, 647, 1094] avg=1024.0000 max=1439 max_rank=5 min=450 min_rank=2 imbalance_ratio=1.405273 max_load_reduction=6753 max_load_reduction_pct=0.824341 moved_to_echo=4144 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[939, 0, 0, 1092, 0, 626, 641, 483, 1366, 0, 1452, 0, 0, 501, 0, 1092] rerouted_tokens_per_ep_rank=[939, 1092, 626, 1124, 1366, 1452, 501, 1092] avg=1024.0000 max=1452 max_rank=5 min=501 min_rank=6 imbalance_ratio=1.417969 max_load_reduction=6740 max_load_reduction_pct=0.822754 moved_to_echo=4411 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1084, 0, 0, 976, 0, 743, 358, 558, 1621, 0, 1298, 0, 0, 578, 0, 976] rerouted_tokens_per_ep_rank=[1084, 976, 743, 916, 1621, 1298, 578, 976] avg=1024.0000 max=1621 max_rank=4 min=578 min_rank=6 imbalance_ratio=1.583008 max_load_reduction=6571 max_load_reduction_pct=0.802124 moved_to_echo=4473 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1203, 0, 0, 917, 0, 710, 418, 620, 1548, 0, 1217, 0, 0, 642, 0, 917] rerouted_tokens_per_ep_rank=[1203, 917, 710, 1038, 1548, 1217, 642, 917] avg=1024.0000 max=1548 max_rank=4 min=642 min_rank=6 imbalance_ratio=1.511719 max_load_reduction=6644 max_load_reduction_pct=0.811035 moved_to_echo=4324 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[893, 0, 0, 1072, 0, 696, 580, 460, 1517, 0, 1425, 0, 0, 477, 0, 1072] rerouted_tokens_per_ep_rank=[893, 1072, 696, 1040, 1517, 1425, 477, 1072] avg=1024.0000 max=1517 max_rank=4 min=477 min_rank=6 imbalance_ratio=1.481445 max_load_reduction=6675 max_load_reduction_pct=0.814819 moved_to_echo=4491 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=5512 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5512, 8192, 12004, 10872, 4380, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10872 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5512, 8192, 12004, 10872, 4380, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5512, 8192, 12004, 10872, 4380, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4380 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5512, 8192, 12004, 10872, 4380, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5512, 8192, 12004, 10872, 4380, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5512, 8192, 12004, 10872, 4380, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5512, 8192, 12004, 10872, 4380, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=12004 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 5512, 8192, 12004, 10872, 4380, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=4380 actual_valid_recv=4608 observed_loss=-228 tokens_per_expert=[128, 4480] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=5512 actual_valid_recv=5760 observed_loss=-248 tokens_per_expert=[128, 5632] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[3968, 4352] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=10872 actual_valid_recv=10880 observed_loss=-8 tokens_per_expert=[10880, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=12004 actual_valid_recv=12032 observed_loss=-28 tokens_per_expert=[12032, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=4608 slack_slots=11776 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=5760 slack_slots=10624 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=10880 slack_slots=5504 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=12032 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 4095, 1, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1027, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1025, 1023, 1023, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1027, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1025, 1023, 1023, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1027, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1025, 1023, 1023, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 1, 1030, 0, 1024, 1017, 0, 1, 1030, 0, 1030, 1005, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1030, 1031, 1024, 1017, 1031, 1030, 1005, 1024] avg=1024.0000 max=1031 max_rank=1 min=1005 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1027, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1025, 1023, 1023, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1027, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1025, 1023, 1023, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 1027, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1025, 1023, 1023, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8191, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[3984, 4095, 1, 111, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[3975, 4096, 0, 121, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[3883, 4096, 0, 212, 0, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[3904, 4096, 0, 176, 0, 0, 0, 16] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[3933, 4096, 0, 158, 0, 0, 0, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[3864, 4096, 0, 232, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[3986, 4096, 0, 110, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[3952, 4096, 0, 142, 0, 0, 0, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1012, 0, 1025, 0, 0, 1010, 212, 854, 0, 1024, 0, 1024, 0, 1023, 1, 1007] rerouted_tokens_per_ep_rank=[1012, 1025, 1010, 1066, 1024, 1024, 1023, 1008] avg=1024.0000 max=1066 max_rank=3 min=1008 min_rank=7 imbalance_ratio=1.041016 max_load_reduction=7126 max_load_reduction_pct=0.869873 moved_to_echo=4079 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1035, 0, 1025, 0, 0, 1034, 121, 875, 0, 1024, 0, 1024, 0, 1023, 0, 1031] rerouted_tokens_per_ep_rank=[1035, 1025, 1034, 996, 1024, 1024, 1023, 1031] avg=1024.0000 max=1035 max_rank=0 min=996 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1025, 0, 0, 1023, 158, 865, 0, 1024, 0, 1024, 0, 1023, 5, 1020] rerouted_tokens_per_ep_rank=[1025, 1025, 1023, 1023, 1024, 1024, 1023, 1025] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1018, 0, 1025, 0, 0, 1015, 176, 859, 0, 1024, 0, 1024, 0, 1023, 16, 1012] rerouted_tokens_per_ep_rank=[1018, 1025, 1015, 1035, 1024, 1024, 1023, 1028] avg=1024.0000 max=1035 max_rank=3 min=1015 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 1017, 0, 1, 1039, 111, 881, 0, 1024, 0, 1024, 1, 1030, 0, 1037] rerouted_tokens_per_ep_rank=[1027, 1017, 1040, 992, 1024, 1024, 1031, 1037] avg=1024.0000 max=1040 max_rank=2 min=992 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4116 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1007, 0, 1025, 0, 0, 1005, 232, 850, 0, 1024, 0, 1024, 0, 1023, 0, 1002] rerouted_tokens_per_ep_rank=[1007, 1025, 1005, 1082, 1024, 1024, 1023, 1002] avg=1024.0000 max=1082 max_rank=3 min=1002 min_rank=7 imbalance_ratio=1.056641 max_load_reduction=7110 max_load_reduction_pct=0.867920 moved_to_echo=4073 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1038, 0, 1025, 0, 0, 1037, 110, 877, 0, 1024, 0, 1024, 0, 1023, 0, 1034] rerouted_tokens_per_ep_rank=[1038, 1025, 1037, 987, 1024, 1024, 1023, 1034] avg=1024.0000 max=1038 max_rank=0 min=987 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1030, 0, 1025, 0, 0, 1028, 142, 869, 0, 1024, 0, 1024, 0, 1023, 2, 1025] rerouted_tokens_per_ep_rank=[1030, 1025, 1028, 1011, 1024, 1024, 1023, 1027] avg=1024.0000 max=1030 max_rank=0 min=1011 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1280, 7040] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1024, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1024, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1024, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1024, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1011, 0, 0, 1024, 0, 1030, 0, 1024, 0, 1024, 1017, 0, 1, 1030, 1, 1030] rerouted_tokens_per_ep_rank=[1011, 1024, 1030, 1024, 1024, 1017, 1031, 1031] avg=1024.0000 max=1031 max_rank=6 min=1011 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1024, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1024, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1024, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8191, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 117, 0, 0, 0, 3979, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4095, 108, 0, 0, 1, 3987, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 40, 0, 0, 0, 4056, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4096, 126, 0, 0, 0, 3970, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4096, 123, 0, 0, 0, 3973, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4096, 121, 0, 0, 0, 3975, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4096, 103, 0, 0, 0, 3993, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 61, 0, 0, 0, 4035, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 121, 919, 0, 1024, 0, 1023, 0, 1018, 1020, 0, 0, 1018, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1040, 1024, 1023, 1018, 1020, 1018, 1024] avg=1024.0000 max=1040 max_rank=1 min=1018 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4080 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 103, 923, 0, 1024, 0, 1023, 0, 1023, 1024, 0, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1026, 1024, 1023, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1017, 0, 108, 924, 0, 1024, 0, 1030, 1, 1025, 1013, 0, 1, 1025, 0, 1024] rerouted_tokens_per_ep_rank=[1017, 1032, 1024, 1030, 1026, 1013, 1026, 1024] avg=1024.0000 max=1032 max_rank=1 min=1013 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 40, 938, 0, 1024, 0, 1023, 0, 1039, 1040, 0, 0, 1039, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 978, 1024, 1023, 1039, 1040, 1039, 1024] avg=1024.0000 max=1040 max_rank=5 min=978 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=7152 max_load_reduction_pct=0.873047 moved_to_echo=4142 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 123, 918, 0, 1024, 0, 1023, 0, 1017, 1021, 0, 0, 1017, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1041, 1024, 1023, 1017, 1021, 1017, 1024] avg=1024.0000 max=1041 max_rank=1 min=1017 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4079 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 117, 920, 0, 1024, 0, 1023, 0, 1019, 1021, 0, 0, 1019, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1037, 1024, 1023, 1019, 1021, 1019, 1024] avg=1024.0000 max=1037 max_rank=1 min=1019 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=7155 max_load_reduction_pct=0.873413 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 126, 918, 0, 1024, 0, 1023, 0, 1017, 1018, 0, 0, 1017, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1044, 1024, 1023, 1017, 1018, 1017, 1024] avg=1024.0000 max=1044 max_rank=1 min=1017 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=7148 max_load_reduction_pct=0.872559 moved_to_echo=4076 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 61, 933, 0, 1024, 0, 1023, 0, 1033, 1036, 0, 0, 1033, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 994, 1024, 1023, 1033, 1036, 1033, 1024] avg=1024.0000 max=1036 max_rank=5 min=994 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4126 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8191, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8191, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8191, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8191, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8191, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8191, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8191, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8191, 8192, 8193, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[896, 7424] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4095, 1, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 1017, 0, 0, 1024, 1011, 0, 1, 1030, 0, 1024, 1, 1030, 0, 1024] rerouted_tokens_per_ep_rank=[1030, 1017, 1024, 1011, 1031, 1024, 1031, 1024] avg=1024.0000 max=1031 max_rank=4 min=1011 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4110 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8192, 8193, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4088, 4096, 8, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4096, 0, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 4096, 2, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4085, 4096, 9, 0, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 1, 4091, 4095, 4, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 4096, 4, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0, 0, 1021, 0, 1024, 1, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1026, 1025, 1021, 1024, 1025] avg=1024.0000 max=1026 max_rank=3 min=1021 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1027, 0, 1025, 0, 0, 1021, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1027, 1025, 1021, 1024, 1024] avg=1024.0000 max=1027 max_rank=3 min=1021 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1022, 0, 1023, 1025, 0, 1025, 0, 8, 1019, 0, 1024, 0, 1022] rerouted_tokens_per_ep_rank=[1024, 1022, 1023, 1025, 1025, 1027, 1024, 1022] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1025, 0, 1025, 0, 2, 1021, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1025, 1025, 1023, 1024, 1024] avg=1024.0000 max=1025 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1022, 0, 1023, 1024, 0, 1025, 0, 9, 1018, 0, 1024, 2, 1021] rerouted_tokens_per_ep_rank=[1024, 1022, 1023, 1024, 1025, 1027, 1024, 1023] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1, 1028, 1, 1030, 1012, 0, 1017, 0, 4, 1024, 0, 1024, 0, 1027] rerouted_tokens_per_ep_rank=[1024, 1029, 1031, 1012, 1017, 1028, 1024, 1027] avg=1024.0000 max=1031 max_rank=2 min=1012 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1023, 1026, 0, 1025, 0, 4, 1020, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1026, 1025, 1024, 1024, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1027, 0, 1025, 0, 0, 1021, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1027, 1025, 1021, 1024, 1024] avg=1024.0000 max=1027 max_rank=3 min=1021 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1, 4095, 1, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1030, 1011, 0, 1, 1030, 0, 1024, 1017, 0, 0, 1030, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1031, 1011, 1031, 1024, 1017, 1030, 1024, 1024] avg=1024.0000 max=1031 max_rank=0 min=1011 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1024, 1025, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 0, 4095, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1011, 0, 0, 1024, 1, 1030, 1, 1030, 0, 1030, 1017, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1011, 1024, 1031, 1031, 1030, 1017, 1024, 1024] avg=1024.0000 max=1031 max_rank=2 min=1011 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1023, 1023, 1025, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8193, 8192, 8192, 8192, 8191, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 1, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 1, 4094, 0, 1, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1027, 1023, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1030, 1005, 0, 1, 1030, 0, 1030, 1, 1030, 1017, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1030, 1005, 1031, 1030, 1031, 1017] avg=1024.0000 max=1031 max_rank=4 min=1005 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4109 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1027, 1023, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 1, 1023, 1025, 0, 0, 1023, 1, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1024, 1025, 1023, 1024, 1023, 1025] avg=1024.0000 max=1025 max_rank=3 min=1023 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1027, 1023, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1027, 1023, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1027, 1023, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1024, 0, 1023, 1027, 0, 0, 1023, 0, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1027, 1023, 1023, 1023, 1025] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4085, 0, 0, 4096, 0, 11] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4056, 0, 0, 4096, 0, 40] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 4096, 0, 7] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4091, 1, 1, 4095, 0, 4] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 4096, 0, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4061, 0, 0, 4096, 0, 35] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1025, 0, 0, 1024, 0, 1024, 1025, 0, 0, 1024, 11, 1012] rerouted_tokens_per_ep_rank=[1023, 1024, 1025, 1024, 1024, 1025, 1024, 1023] avg=1024.0000 max=1025 max_rank=2 min=1023 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1030, 0, 0, 1026, 0, 1026, 1025, 0, 0, 1024, 0, 1014] rerouted_tokens_per_ep_rank=[1023, 1024, 1030, 1026, 1026, 1025, 1024, 1014] avg=1024.0000 max=1030 max_rank=2 min=1014 min_rank=7 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1026, 0, 0, 1025, 0, 1025, 1025, 0, 0, 1024, 7, 1013] rerouted_tokens_per_ep_rank=[1023, 1024, 1026, 1025, 1025, 1025, 1024, 1020] avg=1024.0000 max=1026 max_rank=2 min=1020 min_rank=7 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1020, 0, 0, 1016, 0, 1016, 1025, 0, 0, 1024, 40, 1004] rerouted_tokens_per_ep_rank=[1023, 1024, 1020, 1016, 1016, 1025, 1024, 1044] avg=1024.0000 max=1044 max_rank=7 min=1016 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=7148 max_load_reduction_pct=0.872559 moved_to_echo=4109 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1030, 0, 1024, 1014, 0, 1, 1030, 1, 1030, 1017, 0, 0, 1024, 4, 1017] rerouted_tokens_per_ep_rank=[1030, 1024, 1014, 1031, 1031, 1017, 1024, 1021] avg=1024.0000 max=1031 max_rank=3 min=1014 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1029, 0, 0, 1026, 0, 1026, 1025, 0, 0, 1024, 1, 1014] rerouted_tokens_per_ep_rank=[1023, 1024, 1029, 1026, 1026, 1025, 1024, 1015] avg=1024.0000 max=1029 max_rank=2 min=1015 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1030, 0, 0, 1026, 0, 1026, 1025, 0, 0, 1024, 0, 1014] rerouted_tokens_per_ep_rank=[1023, 1024, 1030, 1026, 1026, 1025, 1024, 1014] avg=1024.0000 max=1030 max_rank=2 min=1014 min_rank=7 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 1019, 0, 0, 1018, 0, 1018, 1025, 0, 0, 1024, 35, 1006] rerouted_tokens_per_ep_rank=[1023, 1024, 1019, 1018, 1018, 1025, 1024, 1041] avg=1024.0000 max=1041 max_rank=7 min=1018 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8191, 8192, 8193, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 4095, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1030, 0, 1024, 0, 1030, 0, 1024, 0, 1024, 1, 1030, 1011, 0, 1017, 0] rerouted_tokens_per_ep_rank=[1031, 1024, 1030, 1024, 1024, 1031, 1011, 1017] avg=1024.0000 max=1031 max_rank=0 min=1011 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 1026, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1024, 1024, 1023, 1026, 1025] avg=1024.0000 max=1026 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8191, 8192, 8192, 8192, 8193, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 3949, 0, 147, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1, 4004, 0, 91, 0, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 3861, 0, 235, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 3861, 0, 235, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 3935, 0, 161, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 3880, 0, 216, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 3806, 0, 290, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 3885, 0, 211, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1037, 1039, 0, 0, 1024, 147, 836, 0, 1024, 0, 1023, 1025, 0, 0, 1037] rerouted_tokens_per_ep_rank=[1037, 1039, 1024, 983, 1024, 1023, 1025, 1037] avg=1024.0000 max=1039 max_rank=1 min=983 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=7153 max_load_reduction_pct=0.873169 moved_to_echo=4109 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1014, 1016, 0, 0, 1024, 235, 817, 0, 1024, 0, 1023, 1025, 0, 0, 1014] rerouted_tokens_per_ep_rank=[1014, 1016, 1024, 1052, 1024, 1023, 1025, 1014] avg=1024.0000 max=1052 max_rank=3 min=1014 min_rank=0 imbalance_ratio=1.027344 max_load_reduction=7140 max_load_reduction_pct=0.871582 moved_to_echo=4086 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1055, 1042, 0, 0, 1024, 91, 852, 0, 1024, 0, 1030, 1017, 0, 1, 1055] rerouted_tokens_per_ep_rank=[1056, 1042, 1024, 943, 1024, 1030, 1017, 1056] avg=1024.0000 max=1056 max_rank=0 min=943 min_rank=3 imbalance_ratio=1.031250 max_load_reduction=7136 max_load_reduction_pct=0.871094 moved_to_echo=4127 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1033, 1036, 0, 0, 1024, 161, 833, 0, 1024, 0, 1023, 1025, 0, 0, 1033] rerouted_tokens_per_ep_rank=[1033, 1036, 1024, 994, 1024, 1023, 1025, 1033] avg=1024.0000 max=1036 max_rank=1 min=994 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 999, 1002, 0, 0, 1024, 290, 806, 0, 1024, 0, 1023, 1025, 0, 0, 999] rerouted_tokens_per_ep_rank=[999, 1002, 1024, 1096, 1024, 1023, 1025, 999] avg=1024.0000 max=1096 max_rank=3 min=999 min_rank=0 imbalance_ratio=1.070312 max_load_reduction=7096 max_load_reduction_pct=0.866211 moved_to_echo=4071 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1019, 1020, 0, 0, 1024, 216, 822, 0, 1024, 0, 1023, 1025, 0, 0, 1019] rerouted_tokens_per_ep_rank=[1019, 1020, 1024, 1038, 1024, 1023, 1025, 1019] avg=1024.0000 max=1038 max_rank=3 min=1019 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1014, 1016, 0, 0, 1024, 235, 817, 0, 1024, 0, 1023, 1025, 0, 0, 1014] rerouted_tokens_per_ep_rank=[1014, 1016, 1024, 1052, 1024, 1023, 1025, 1014] avg=1024.0000 max=1052 max_rank=3 min=1014 min_rank=0 imbalance_ratio=1.027344 max_load_reduction=7140 max_load_reduction_pct=0.871582 moved_to_echo=4086 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1020, 1022, 0, 0, 1024, 211, 823, 0, 1024, 0, 1023, 1025, 0, 0, 1020] rerouted_tokens_per_ep_rank=[1020, 1022, 1024, 1034, 1024, 1023, 1025, 1020] avg=1024.0000 max=1034 max_rank=3 min=1020 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[1664, 6656] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8191 actual_valid_recv=8192 observed_loss=-1 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True + [2026-08-30 04:54:07] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 670.6 | throughput per GPU (TFLOP/s/GPU): 497.1 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.447369E+00 | loss scale: 1.0 | grad norm: 4.688 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1192, 3, 2, 0, 3457, 0, 3538, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1332, 10, 6, 0, 3916, 0, 2927, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1290, 0, 0, 0, 3733, 1, 3168, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1339, 0, 0, 0, 3672, 1, 3179, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1441, 13, 0, 0, 3444, 2, 3292, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1321, 1, 3, 2, 3583, 0, 3282, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1598, 0, 0, 0, 3395, 0, 3199, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[602, 3, 0, 0, 3957, 1, 3627, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1332, 0, 10, 696, 6, 913, 0, 1099, 1102, 0, 0, 616, 1318, 0, 1, 1099] rerouted_tokens_per_ep_rank=[1332, 706, 919, 1099, 1102, 616, 1318, 1100] avg=1024.0000 max=1332 max_rank=0 min=616 min_rank=5 imbalance_ratio=1.300781 max_load_reduction=6860 max_load_reduction_pct=0.837402 moved_to_echo=4136 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1192, 0, 3, 842, 2, 1104, 0, 971, 973, 0, 0, 543, 1592, 0, 0, 970] rerouted_tokens_per_ep_rank=[1192, 845, 1106, 971, 973, 543, 1592, 970] avg=1024.0000 max=1592 max_rank=6 min=543 min_rank=5 imbalance_ratio=1.554688 max_load_reduction=6600 max_load_reduction_pct=0.805664 moved_to_echo=4078 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1441, 0, 13, 783, 0, 1027, 0, 967, 969, 0, 2, 541, 1482, 0, 0, 967] rerouted_tokens_per_ep_rank=[1441, 796, 1027, 967, 969, 543, 1482, 967] avg=1024.0000 max=1482 max_rank=6 min=543 min_rank=5 imbalance_ratio=1.447266 max_load_reduction=6710 max_load_reduction_pct=0.819092 moved_to_echo=3961 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1339, 0, 0, 756, 0, 992, 0, 1031, 1033, 0, 1, 577, 1431, 0, 1, 1031] rerouted_tokens_per_ep_rank=[1339, 756, 992, 1031, 1033, 578, 1431, 1032] avg=1024.0000 max=1431 max_rank=6 min=578 min_rank=5 imbalance_ratio=1.397461 max_load_reduction=6761 max_load_reduction_pct=0.825317 moved_to_echo=4074 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1321, 0, 1, 784, 3, 1027, 2, 1010, 997, 0, 0, 567, 1471, 0, 0, 1009] rerouted_tokens_per_ep_rank=[1321, 785, 1030, 1012, 997, 567, 1471, 1009] avg=1024.0000 max=1471 max_rank=6 min=567 min_rank=5 imbalance_ratio=1.436523 max_load_reduction=6721 max_load_reduction_pct=0.820435 moved_to_echo=4044 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1290, 0, 0, 754, 0, 988, 0, 1048, 1050, 0, 1, 587, 1426, 0, 0, 1048] rerouted_tokens_per_ep_rank=[1290, 754, 988, 1048, 1050, 588, 1426, 1048] avg=1024.0000 max=1426 max_rank=6 min=588 min_rank=5 imbalance_ratio=1.392578 max_load_reduction=6766 max_load_reduction_pct=0.825928 moved_to_echo=4112 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1598, 0, 0, 761, 0, 998, 0, 953, 955, 0, 0, 534, 1440, 0, 0, 953] rerouted_tokens_per_ep_rank=[1598, 761, 998, 953, 955, 534, 1440, 953] avg=1024.0000 max=1598 max_rank=0 min=534 min_rank=5 imbalance_ratio=1.560547 max_load_reduction=6594 max_load_reduction_pct=0.804932 moved_to_echo=3882 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[602, 0, 3, 863, 0, 1132, 0, 1111, 1113, 0, 1, 622, 1632, 0, 2, 1111] rerouted_tokens_per_ep_rank=[602, 866, 1132, 1111, 1113, 623, 1632, 1113] avg=1024.0000 max=1632 max_rank=6 min=602 min_rank=0 imbalance_ratio=1.593750 max_load_reduction=6560 max_load_reduction_pct=0.800781 moved_to_echo=4481 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10115, 6269, 8192, 8192, 8192, 4592, 11792, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=11792 expected_overflow=0 exact_recv_per_rank=[10115, 6269, 8192, 8192, 8192, 4592, 11792, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10115, 6269, 8192, 8192, 8192, 4592, 11792, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10115, 6269, 8192, 8192, 8192, 4592, 11792, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10115 expected_overflow=0 exact_recv_per_rank=[10115, 6269, 8192, 8192, 8192, 4592, 11792, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4592 expected_overflow=0 exact_recv_per_rank=[10115, 6269, 8192, 8192, 8192, 4592, 11792, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10115, 6269, 8192, 8192, 8192, 4592, 11792, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6269 expected_overflow=0 exact_recv_per_rank=[10115, 6269, 8192, 8192, 8192, 4592, 11792, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=4592 actual_valid_recv=4736 observed_loss=-144 tokens_per_expert=[128, 4608] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=6269 actual_valid_recv=6400 observed_loss=-131 tokens_per_expert=[128, 6272] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=10115 actual_valid_recv=10240 observed_loss=-125 tokens_per_expert=[10240, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=11792 actual_valid_recv=11904 observed_loss=-112 tokens_per_expert=[11904, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=4736 slack_slots=11648 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=6400 slack_slots=9984 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=10240 slack_slots=6144 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=11904 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4088, 0, 45, 1, 5, 9, 3, 4041] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4096, 1, 46, 0, 0, 1, 0, 4048] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 34, 0, 0, 0, 0, 4062] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4089, 0, 42, 0, 0, 0, 0, 4061] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4095, 1, 48, 1, 0, 0, 0, 4047] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4094, 2, 21, 1, 1, 0, 0, 4073] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 58, 0, 0, 0, 0, 4038] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4093, 0, 48, 0, 2, 2, 1, 4046] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1020, 45, 978, 1, 1022, 5, 1020, 9, 1019, 3, 1022, 1024, 0] rerouted_tokens_per_ep_rank=[1024, 1020, 1023, 1023, 1025, 1028, 1025, 1024] avg=1024.0000 max=1028 max_rank=5 min=1020 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1015, 0, 1, 1027, 46, 984, 0, 1027, 0, 1025, 1, 1026, 0, 1027, 1013, 0] rerouted_tokens_per_ep_rank=[1015, 1028, 1030, 1027, 1025, 1027, 1027, 1013] avg=1024.0000 max=1030 max_rank=2 min=1013 min_rank=7 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1021, 42, 983, 0, 1022, 0, 1025, 0, 1024, 0, 1022, 1029, 0] rerouted_tokens_per_ep_rank=[1024, 1021, 1025, 1022, 1025, 1024, 1022, 1029] avg=1024.0000 max=1029 max_rank=7 min=1021 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1, 1022, 48, 980, 1, 1024, 0, 1021, 0, 1021, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1026, 1023, 1028, 1025, 1021, 1021, 1023, 1025] avg=1024.0000 max=1028 max_rank=2 min=1021 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 2, 1022, 21, 986, 1, 1023, 1, 1028, 0, 1027, 0, 1023, 1032, 0] rerouted_tokens_per_ep_rank=[1026, 1024, 1007, 1024, 1029, 1027, 1023, 1032] avg=1024.0000 max=1032 max_rank=7 min=1007 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4111 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1022, 34, 983, 0, 1024, 0, 1025, 0, 1025, 0, 1024, 1029, 0] rerouted_tokens_per_ep_rank=[1026, 1022, 1017, 1024, 1025, 1025, 1024, 1029] avg=1024.0000 max=1029 max_rank=7 min=1017 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1022, 48, 979, 0, 1023, 2, 1021, 2, 1020, 1, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1025, 1022, 1027, 1023, 1023, 1022, 1024, 1026] avg=1024.0000 max=1027 max_rank=2 min=1022 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1022, 58, 977, 0, 1024, 0, 1019, 0, 1018, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1026, 1022, 1035, 1024, 1019, 1018, 1024, 1024] avg=1024.0000 max=1035 max_rank=2 min=1018 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=7157 max_load_reduction_pct=0.873657 moved_to_echo=4085 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8182, 8192, 8192, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8182, 8192, 8192, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8182, 8192, 8192, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8182, 8192, 8192, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8182, 8192, 8192, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8182, 8192, 8192, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8202 expected_overflow=0 exact_recv_per_rank=[8192, 8182, 8192, 8192, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8182 expected_overflow=0 exact_recv_per_rank=[8192, 8182, 8192, 8192, 8192, 8192, 8192, 8202] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[384, 7936] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8182 actual_valid_recv=8320 observed_loss=-138 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8202 actual_valid_recv=8320 observed_loss=-118 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[9, 4088, 0, 0, 1, 0, 4086, 8] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 3, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 4095, 0, 0, 0, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 4094, 0, 1, 0, 2, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[2, 4093, 0, 0, 0, 1, 4094, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[9, 1021, 1024, 0, 0, 1021, 0, 1021, 1, 1022, 0, 1020, 1024, 0, 8, 1021] rerouted_tokens_per_ep_rank=[1030, 1024, 1021, 1021, 1023, 1020, 1024, 1029] avg=1024.0000 max=1030 max_rank=0 min=1020 min_rank=5 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1026, 0, 0, 1024, 0, 1024, 3, 1023, 0, 1022, 1026, 0, 0, 1022] rerouted_tokens_per_ep_rank=[1022, 1026, 1024, 1024, 1026, 1022, 1026, 1022] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1022, 1028, 0, 0, 1024, 0, 1024, 0, 1023, 0, 1022, 1025, 0, 1, 1022] rerouted_tokens_per_ep_rank=[1023, 1028, 1024, 1024, 1023, 1022, 1025, 1023] avg=1024.0000 max=1028 max_rank=1 min=1022 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1022, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1024, 1024, 1024, 1022, 1026, 1023] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1022, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1024, 1024, 1024, 1022, 1026, 1023] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1022, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1024, 1024, 1024, 1022, 1026, 1023] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1020, 0, 0, 1028, 1, 1027, 0, 1025, 2, 1027, 1013, 0, 0, 1025] rerouted_tokens_per_ep_rank=[1024, 1020, 1028, 1028, 1025, 1029, 1013, 1025] avg=1024.0000 max=1029 max_rank=5 min=1013 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 1022, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 1, 1022, 1026, 0, 2, 1022] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1023, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8182 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8202 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8182 actual_valid_recv=8320 observed_loss=-138 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8202 actual_valid_recv=8320 observed_loss=-118 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4023, 0, 0, 4036, 115, 0, 9, 9] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[3948, 0, 0, 4056, 188, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[3714, 0, 0, 4046, 432, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[3389, 0, 0, 3971, 829, 3, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4030, 0, 1, 4047, 113, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3832, 0, 0, 4039, 321, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4037, 0, 0, 4024, 131, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[3968, 0, 2, 4046, 174, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1132, 0, 0, 1024, 0, 962, 1026, 0, 115, 765, 0, 1024, 9, 1063, 9, 1063] rerouted_tokens_per_ep_rank=[1132, 1024, 962, 1026, 880, 1024, 1072, 1072] avg=1024.0000 max=1132 max_rank=0 min=880 min_rank=4 imbalance_ratio=1.105469 max_load_reduction=7060 max_load_reduction_pct=0.861816 moved_to_echo=4048 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[953, 0, 0, 1008, 0, 946, 1010, 0, 829, 645, 3, 1007, 0, 895, 0, 896] rerouted_tokens_per_ep_rank=[953, 1008, 946, 1010, 1474, 1010, 895, 896] avg=1024.0000 max=1474 max_rank=4 min=895 min_rank=6 imbalance_ratio=1.439453 max_load_reduction=6718 max_load_reduction_pct=0.820068 moved_to_echo=4275 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1099, 0, 0, 1033, 0, 971, 1019, 0, 188, 754, 0, 1033, 0, 1047, 0, 1048] rerouted_tokens_per_ep_rank=[1099, 1033, 971, 1019, 942, 1033, 1047, 1048] avg=1024.0000 max=1099 max_rank=0 min=942 min_rank=4 imbalance_ratio=1.073242 max_load_reduction=7093 max_load_reduction_pct=0.865845 moved_to_echo=4070 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1133, 0, 0, 1027, 1, 964, 1029, 0, 113, 767, 0, 1027, 1, 1065, 0, 1065] rerouted_tokens_per_ep_rank=[1133, 1027, 965, 1029, 880, 1027, 1066, 1065] avg=1024.0000 max=1133 max_rank=0 min=880 min_rank=4 imbalance_ratio=1.106445 max_load_reduction=7059 max_load_reduction_pct=0.861694 moved_to_echo=4038 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1045, 0, 0, 1027, 0, 964, 1029, 0, 432, 706, 0, 1026, 0, 981, 0, 982] rerouted_tokens_per_ep_rank=[1045, 1027, 964, 1029, 1138, 1026, 981, 982] avg=1024.0000 max=1138 max_rank=4 min=964 min_rank=2 imbalance_ratio=1.111328 max_load_reduction=7054 max_load_reduction_pct=0.861084 moved_to_echo=4127 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1077, 0, 0, 1025, 0, 962, 1027, 0, 321, 729, 0, 1025, 0, 1013, 0, 1013] rerouted_tokens_per_ep_rank=[1077, 1025, 962, 1027, 1050, 1025, 1013, 1013] avg=1024.0000 max=1077 max_rank=0 min=962 min_rank=2 imbalance_ratio=1.051758 max_load_reduction=7115 max_load_reduction_pct=0.868530 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1115, 0, 0, 1027, 2, 964, 1029, 0, 174, 755, 0, 1026, 2, 1049, 0, 1049] rerouted_tokens_per_ep_rank=[1115, 1027, 966, 1029, 929, 1026, 1051, 1049] avg=1024.0000 max=1115 max_rank=0 min=929 min_rank=4 imbalance_ratio=1.088867 max_load_reduction=7077 max_load_reduction_pct=0.863892 moved_to_echo=4055 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1135, 0, 0, 1021, 0, 959, 1023, 0, 131, 768, 0, 1021, 0, 1067, 0, 1067] rerouted_tokens_per_ep_rank=[1135, 1021, 959, 1023, 899, 1021, 1067, 1067] avg=1024.0000 max=1135 max_rank=0 min=899 min_rank=4 imbalance_ratio=1.108398 max_load_reduction=7057 max_load_reduction_pct=0.861450 moved_to_echo=4054 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7695 expected_overflow=0 exact_recv_per_rank=[8689, 8192, 7695, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8689, 8192, 7695, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8689, 8192, 7695, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8689, 8192, 7695, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8689, 8192, 7695, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8689 expected_overflow=0 exact_recv_per_rank=[8689, 8192, 7695, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8689, 8192, 7695, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8689, 8192, 7695, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8689 actual_valid_recv=8704 observed_loss=-15 tokens_per_expert=[8704, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2304, 6016] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=7695 actual_valid_recv=7936 observed_loss=-241 tokens_per_expert=[128, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8704 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=7936 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1, 9, 5, 4086, 4, 1, 4086, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 4095, 1, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 4095, 0, 1, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 4092, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 4094, 2, 0, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1021, 9, 1020, 5, 1020, 1024, 0, 4, 1021, 1, 1021, 1024, 0, 0, 1021] rerouted_tokens_per_ep_rank=[1022, 1029, 1025, 1024, 1025, 1022, 1024, 1021] avg=1024.0000 max=1029 max_rank=1 min=1021 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1022, 0, 1022, 1026, 0, 1, 1023, 0, 1023, 1027, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1022, 1026, 1024, 1023, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4, 1022, 0, 1022, 0, 1022, 1026, 0, 0, 1022, 0, 1024, 1026, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1022, 1022, 1026, 1022, 1024, 1026, 1024] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 1027, 0, 1025, 0, 1028, 1012, 0, 0, 1027, 1, 1027, 1015, 0, 0, 1028] rerouted_tokens_per_ep_rank=[1029, 1025, 1028, 1012, 1027, 1028, 1015, 1028] avg=1024.0000 max=1029 max_rank=0 min=1012 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1022, 1026, 0, 0, 1023, 0, 1024, 1027, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1022, 1026, 1023, 1024, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1022, 1026, 0, 0, 1023, 0, 1024, 1027, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1022, 1026, 1023, 1024, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 2, 1022, 0, 1022, 1026, 0, 2, 1023, 0, 1023, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1022, 1026, 1025, 1023, 1026, 1023] avg=1024.0000 max=1026 max_rank=3 min=1022 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1022, 1026, 0, 0, 1023, 0, 1024, 1027, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1023, 1022, 1026, 1023, 1024, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8185, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8185 actual_valid_recv=8320 observed_loss=-135 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[716, 4095, 1, 3379, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[832, 4096, 0, 3264, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[175, 4086, 9, 3912, 0, 0, 9, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[857, 4096, 0, 3239, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[716, 4096, 0, 3380, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[499, 4096, 0, 3597, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[410, 4094, 2, 3684, 0, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[217, 4096, 0, 3879, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[716, 449, 1025, 0, 1, 975, 980, 0, 0, 1024, 0, 1024, 1, 975, 0, 1022] rerouted_tokens_per_ep_rank=[1165, 1025, 976, 980, 1024, 1024, 976, 1022] avg=1024.0000 max=1165 max_rank=0 min=976 min_rank=2 imbalance_ratio=1.137695 max_load_reduction=7027 max_load_reduction_pct=0.857788 moved_to_echo=4046 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[857, 430, 1026, 0, 0, 935, 939, 0, 0, 1024, 0, 1024, 0, 935, 0, 1022] rerouted_tokens_per_ep_rank=[1287, 1026, 935, 939, 1024, 1024, 935, 1022] avg=1024.0000 max=1287 max_rank=0 min=935 min_rank=2 imbalance_ratio=1.256836 max_load_reduction=6905 max_load_reduction_pct=0.842896 moved_to_echo=4005 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[175, 520, 1024, 0, 9, 1129, 1134, 0, 0, 1021, 0, 1021, 9, 1129, 1, 1020] rerouted_tokens_per_ep_rank=[695, 1024, 1138, 1134, 1021, 1021, 1138, 1021] avg=1024.0000 max=1138 max_rank=2 min=695 min_rank=0 imbalance_ratio=1.111328 max_load_reduction=7054 max_load_reduction_pct=0.861084 moved_to_echo=4201 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[832, 434, 1026, 0, 0, 942, 946, 0, 0, 1024, 0, 1024, 0, 942, 0, 1022] rerouted_tokens_per_ep_rank=[1266, 1026, 942, 946, 1024, 1024, 942, 1022] avg=1024.0000 max=1266 max_rank=0 min=942 min_rank=2 imbalance_ratio=1.236328 max_load_reduction=6926 max_load_reduction_pct=0.845459 moved_to_echo=4012 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[716, 453, 1013, 0, 0, 979, 969, 0, 0, 1028, 0, 1028, 0, 979, 0, 1027] rerouted_tokens_per_ep_rank=[1169, 1013, 979, 969, 1028, 1028, 979, 1027] avg=1024.0000 max=1169 max_rank=0 min=969 min_rank=3 imbalance_ratio=1.141602 max_load_reduction=7023 max_load_reduction_pct=0.857300 moved_to_echo=4062 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[499, 478, 1026, 0, 0, 1038, 1043, 0, 0, 1024, 0, 1024, 0, 1038, 0, 1022] rerouted_tokens_per_ep_rank=[977, 1026, 1038, 1043, 1024, 1024, 1038, 1022] avg=1024.0000 max=1043 max_rank=3 min=977 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=7149 max_load_reduction_pct=0.872681 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[410, 490, 1026, 0, 2, 1063, 1068, 0, 0, 1023, 0, 1023, 2, 1063, 0, 1022] rerouted_tokens_per_ep_rank=[900, 1026, 1065, 1068, 1023, 1023, 1065, 1022] avg=1024.0000 max=1068 max_rank=3 min=900 min_rank=0 imbalance_ratio=1.042969 max_load_reduction=7124 max_load_reduction_pct=0.869629 moved_to_echo=4133 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[217, 516, 1026, 0, 0, 1119, 1125, 0, 0, 1024, 0, 1024, 0, 1119, 0, 1022] rerouted_tokens_per_ep_rank=[733, 1026, 1119, 1125, 1024, 1024, 1119, 1022] avg=1024.0000 max=1125 max_rank=3 min=733 min_rank=0 imbalance_ratio=1.098633 max_load_reduction=7067 max_load_reduction_pct=0.862671 moved_to_echo=4189 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8204 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8180 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4480, 3840] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8180 actual_valid_recv=8320 observed_loss=-140 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8204 actual_valid_recv=8320 observed_loss=-116 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 4095, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4086, 0, 1, 0, 0, 4087, 9, 9] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 2, 4094, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 1, 4093, 2, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 0, 1021, 1, 1020, 0, 1021, 0, 1021, 1026, 0, 9, 1020, 9, 1020] rerouted_tokens_per_ep_rank=[1024, 1021, 1021, 1021, 1021, 1026, 1029, 1029] avg=1024.0000 max=1029 max_rank=6 min=1021 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1013, 0, 0, 1028, 0, 1027, 0, 1028, 1, 1026, 1013, 0, 0, 1028, 0, 1028] rerouted_tokens_per_ep_rank=[1013, 1028, 1027, 1028, 1027, 1013, 1028, 1028] avg=1024.0000 max=1028 max_rank=1 min=1013 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1024, 0, 1022, 0, 1024, 0, 1023, 1028, 0, 1, 1022, 1, 1022] rerouted_tokens_per_ep_rank=[1025, 1024, 1022, 1024, 1023, 1028, 1023, 1023] avg=1024.0000 max=1028 max_rank=5 min=1022 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1022, 0, 1024, 2, 1023, 1027, 0, 0, 1022, 0, 1022] rerouted_tokens_per_ep_rank=[1026, 1024, 1022, 1024, 1025, 1027, 1022, 1022] avg=1024.0000 max=1027 max_rank=5 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1022, 0, 1024, 0, 1024, 1028, 0, 0, 1022, 0, 1022] rerouted_tokens_per_ep_rank=[1026, 1024, 1022, 1024, 1024, 1028, 1022, 1022] avg=1024.0000 max=1028 max_rank=5 min=1022 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1022, 0, 1024, 0, 1024, 1028, 0, 0, 1022, 0, 1022] rerouted_tokens_per_ep_rank=[1026, 1024, 1022, 1024, 1024, 1028, 1022, 1022] avg=1024.0000 max=1028 max_rank=5 min=1022 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1022, 0, 1023, 1, 1023, 1026, 0, 2, 1022, 2, 1022] rerouted_tokens_per_ep_rank=[1026, 1023, 1022, 1023, 1024, 1026, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1022, 0, 1024, 0, 1024, 1028, 0, 0, 1022, 0, 1022] rerouted_tokens_per_ep_rank=[1026, 1024, 1022, 1024, 1024, 1028, 1022, 1022] avg=1024.0000 max=1028 max_rank=5 min=1022 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8204 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8180, 8192, 8192, 8204, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8180, 8192, 8192, 8204, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8180, 8192, 8192, 8204, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8180, 8192, 8192, 8204, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8180, 8192, 8192, 8204, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8180, 8192, 8192, 8204, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8180, 8192, 8192, 8204, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8180 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8180, 8192, 8192, 8204, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8204 actual_valid_recv=8320 observed_loss=-116 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8180 actual_valid_recv=8320 observed_loss=-140 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 3214, 0, 0, 0, 0, 0, 882] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4096, 3253, 0, 0, 0, 2, 0, 841] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4095, 3235, 0, 0, 1, 0, 1, 860] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4096, 3517, 0, 0, 0, 2, 0, 577] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4087, 2872, 7, 0, 2, 1, 9, 1214] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4096, 3152, 0, 0, 0, 0, 0, 944] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4094, 3527, 0, 1, 2, 0, 2, 566] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 3187, 0, 0, 0, 0, 0, 909] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1016, 0, 0, 1013, 0, 1024, 0, 1023, 0, 1023, 0, 1012, 882, 173] rerouted_tokens_per_ep_rank=[1026, 1016, 1013, 1024, 1023, 1023, 1012, 1055] avg=1024.0000 max=1055 max_rank=7 min=1012 min_rank=6 imbalance_ratio=1.030273 max_load_reduction=7137 max_load_reduction_pct=0.871216 moved_to_echo=4113 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1022, 0, 0, 1020, 0, 1024, 1, 1023, 0, 1023, 1, 1019, 860, 174] rerouted_tokens_per_ep_rank=[1025, 1022, 1020, 1024, 1024, 1023, 1020, 1034] avg=1024.0000 max=1034 max_rank=7 min=1020 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1111, 0, 0, 1109, 0, 1024, 0, 1023, 2, 1023, 0, 1108, 577, 189] rerouted_tokens_per_ep_rank=[1026, 1111, 1109, 1024, 1023, 1025, 1108, 766] avg=1024.0000 max=1111 max_rank=1 min=766 min_rank=7 imbalance_ratio=1.084961 max_load_reduction=7081 max_load_reduction_pct=0.864380 moved_to_echo=3922 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 0, 908, 0, 7, 905, 0, 1021, 2, 1021, 1, 1021, 9, 905, 1214, 154] rerouted_tokens_per_ep_rank=[1024, 908, 912, 1021, 1023, 1022, 914, 1368] avg=1024.0000 max=1368 max_rank=7 min=908 min_rank=1 imbalance_ratio=1.335938 max_load_reduction=6824 max_load_reduction_pct=0.833008 moved_to_echo=4327 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1014, 0, 1017, 0, 0, 1029, 0, 1027, 0, 1028, 2, 1027, 0, 1028, 841, 179] rerouted_tokens_per_ep_rank=[1014, 1017, 1029, 1027, 1028, 1029, 1028, 1020] avg=1024.0000 max=1029 max_rank=2 min=1014 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1114, 0, 0, 1112, 1, 1023, 2, 1023, 0, 1023, 2, 1111, 566, 190] rerouted_tokens_per_ep_rank=[1025, 1114, 1112, 1024, 1025, 1023, 1113, 756] avg=1024.0000 max=1114 max_rank=1 min=756 min_rank=7 imbalance_ratio=1.087891 max_load_reduction=7078 max_load_reduction_pct=0.864014 moved_to_echo=3917 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 997, 0, 0, 993, 0, 1024, 0, 1023, 0, 1023, 0, 993, 944, 169] rerouted_tokens_per_ep_rank=[1026, 997, 993, 1024, 1023, 1023, 993, 1113] avg=1024.0000 max=1113 max_rank=7 min=993 min_rank=2 imbalance_ratio=1.086914 max_load_reduction=7079 max_load_reduction_pct=0.864136 moved_to_echo=4152 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1008, 0, 0, 1004, 0, 1024, 0, 1023, 0, 1023, 0, 1004, 909, 171] rerouted_tokens_per_ep_rank=[1026, 1008, 1004, 1024, 1023, 1023, 1004, 1080] avg=1024.0000 max=1080 max_rank=7 min=1004 min_rank=2 imbalance_ratio=1.054688 max_load_reduction=7112 max_load_reduction_pct=0.868164 moved_to_echo=4130 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8193 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8191 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8193, 8192, 8192, 8192, 8191, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6912, 1408] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8191 actual_valid_recv=8320 observed_loss=-129 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8193 actual_valid_recv=8320 observed_loss=-127 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4095, 1, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4086, 1, 4085, 10, 0, 9, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 1, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 2, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4094, 1, 4093, 2, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1023, 1028, 0, 1, 1022, 0, 1024, 1, 1022, 0, 1022] rerouted_tokens_per_ep_rank=[1024, 1025, 1023, 1028, 1023, 1024, 1023, 1022] avg=1024.0000 max=1028 max_rank=3 min=1022 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1024, 0, 1, 1021, 1024, 0, 10, 1020, 0, 1021, 9, 1020, 1, 1020] rerouted_tokens_per_ep_rank=[1021, 1024, 1022, 1024, 1030, 1021, 1029, 1021] avg=1024.0000 max=1030 max_rank=4 min=1021 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 2, 1023, 1027, 0, 0, 1022, 0, 1024, 0, 1022, 0, 1022] rerouted_tokens_per_ep_rank=[1024, 1026, 1025, 1027, 1022, 1024, 1022, 1022] avg=1024.0000 max=1027 max_rank=3 min=1022 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 0, 1023, 1028, 0, 0, 1022, 0, 1024, 0, 1023, 0, 1022] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1028, 1022, 1024, 1023, 1022] avg=1024.0000 max=1028 max_rank=3 min=1022 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 1013, 0, 1, 1028, 1015, 0, 0, 1027, 0, 1028, 0, 1025, 0, 1027] rerouted_tokens_per_ep_rank=[1028, 1013, 1029, 1015, 1027, 1028, 1025, 1027] avg=1024.0000 max=1029 max_rank=2 min=1013 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4107 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 0, 1023, 1028, 0, 0, 1022, 0, 1024, 0, 1023, 0, 1022] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1028, 1022, 1024, 1023, 1022] avg=1024.0000 max=1028 max_rank=3 min=1022 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1, 1023, 1026, 0, 2, 1022, 0, 1023, 2, 1022, 0, 1022] rerouted_tokens_per_ep_rank=[1023, 1026, 1024, 1026, 1024, 1023, 1024, 1022] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=7 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 0, 1023, 1028, 0, 0, 1022, 0, 1024, 0, 1023, 0, 1022] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1028, 1022, 1024, 1023, 1022] avg=1024.0000 max=1028 max_rank=3 min=1022 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8204 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8180 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8204, 8192, 8192, 8192, 8180] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8180 actual_valid_recv=8320 observed_loss=-140 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8204 actual_valid_recv=8320 observed_loss=-116 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4095, 1, 1, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4086, 10, 9, 0, 4085, 1, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4096, 0, 0, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4094, 2, 2, 0, 4094, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 1, 1022, 1, 1022, 0, 1024, 1025, 0, 0, 1022, 0, 1024, 0, 1023] rerouted_tokens_per_ep_rank=[1028, 1023, 1023, 1024, 1025, 1022, 1024, 1023] avg=1024.0000 max=1028 max_rank=0 min=1022 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 0, 1022, 0, 1022, 0, 1024, 1026, 0, 0, 1022, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1028, 1022, 1022, 1024, 1026, 1022, 1024, 1024] avg=1024.0000 max=1028 max_rank=0 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 10, 1020, 9, 1020, 0, 1021, 1023, 0, 1, 1020, 1, 1021, 0, 1021] rerouted_tokens_per_ep_rank=[1025, 1030, 1029, 1021, 1023, 1021, 1022, 1021] avg=1024.0000 max=1030 max_rank=1 min=1021 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 0, 1022, 0, 1022, 0, 1024, 1026, 0, 0, 1022, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1028, 1022, 1022, 1024, 1026, 1022, 1024, 1024] avg=1024.0000 max=1028 max_rank=0 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1012, 0, 0, 1027, 0, 1028, 0, 1028, 1014, 0, 0, 1027, 0, 1027, 2, 1027] rerouted_tokens_per_ep_rank=[1012, 1027, 1028, 1028, 1014, 1027, 1027, 1029] avg=1024.0000 max=1029 max_rank=7 min=1012 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 0, 1022, 0, 1022, 0, 1024, 1026, 0, 0, 1022, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1028, 1022, 1022, 1024, 1026, 1022, 1024, 1024] avg=1024.0000 max=1028 max_rank=0 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 2, 1022, 2, 1022, 0, 1023, 1026, 0, 0, 1022, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1027, 1024, 1024, 1023, 1026, 1022, 1023, 1023] avg=1024.0000 max=1027 max_rank=0 min=1022 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1028, 0, 0, 1022, 0, 1022, 0, 1024, 1026, 0, 0, 1022, 0, 1024, 0, 1024] rerouted_tokens_per_ep_rank=[1028, 1022, 1022, 1024, 1026, 1022, 1024, 1024] avg=1024.0000 max=1028 max_rank=0 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8204, 8192, 8192, 8192, 8192, 8180, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8204, 8192, 8192, 8192, 8192, 8180, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8180 expected_overflow=0 exact_recv_per_rank=[8204, 8192, 8192, 8192, 8192, 8180, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8204 expected_overflow=0 exact_recv_per_rank=[8204, 8192, 8192, 8192, 8192, 8180, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8204, 8192, 8192, 8192, 8192, 8180, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8204, 8192, 8192, 8192, 8192, 8180, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8204, 8192, 8192, 8192, 8192, 8180, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8204, 8192, 8192, 8192, 8192, 8180, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8204 actual_valid_recv=8320 observed_loss=-116 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8180 actual_valid_recv=8320 observed_loss=-140 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 4095, 1, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4088, 0, 8, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[11, 4085, 11, 0, 4085, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4084, 0, 12, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4070, 0, 26, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4088, 0, 8, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[2, 4093, 2, 0, 4090, 0, 5, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1024, 1025, 0, 1, 1024, 0, 1024, 1029, 0, 0, 1022, 0, 1018, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1025, 1025, 1024, 1029, 1022, 1018, 1024] avg=1024.0000 max=1029 max_rank=4 min=1018 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[11, 1021, 1024, 0, 11, 1021, 0, 1021, 1027, 0, 0, 1019, 0, 1016, 0, 1021] rerouted_tokens_per_ep_rank=[1032, 1024, 1032, 1021, 1027, 1019, 1016, 1021] avg=1024.0000 max=1032 max_rank=0 min=1016 min_rank=6 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1026, 0, 0, 1022, 0, 1024, 1028, 0, 0, 1022, 8, 1016, 0, 1024] rerouted_tokens_per_ep_rank=[1022, 1026, 1022, 1024, 1028, 1022, 1024, 1024] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 1013, 0, 0, 1025, 0, 1028, 1014, 0, 0, 1027, 12, 1020, 0, 1028] rerouted_tokens_per_ep_rank=[1025, 1013, 1025, 1028, 1014, 1027, 1032, 1028] avg=1024.0000 max=1032 max_rank=6 min=1013 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1017, 1026, 0, 0, 1017, 0, 1024, 1024, 0, 0, 1022, 26, 1012, 0, 1024] rerouted_tokens_per_ep_rank=[1017, 1026, 1017, 1024, 1024, 1022, 1038, 1024] avg=1024.0000 max=1038 max_rank=6 min=1017 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1026, 0, 0, 1024, 0, 1024, 1030, 0, 0, 1022, 0, 1018, 0, 1024] rerouted_tokens_per_ep_rank=[1024, 1026, 1024, 1024, 1030, 1022, 1018, 1024] avg=1024.0000 max=1030 max_rank=4 min=1018 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 1023, 1026, 0, 2, 1023, 0, 1023, 1027, 0, 0, 1021, 5, 1017, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1026, 1025, 1023, 1027, 1021, 1022, 1023] avg=1024.0000 max=1027 max_rank=4 min=1021 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1026, 0, 0, 1022, 0, 1024, 1028, 0, 0, 1022, 8, 1016, 0, 1024] rerouted_tokens_per_ep_rank=[1022, 1026, 1022, 1024, 1028, 1022, 1024, 1024] avg=1024.0000 max=1028 max_rank=4 min=1022 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8177, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8207 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8177, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8177, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8177 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8177, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8177, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8177, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8177, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8207, 8177, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8177 actual_valid_recv=8192 observed_loss=-15 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8207 actual_valid_recv=8320 observed_loss=-113 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4095, 4091, 1, 1, 0, 1, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4082, 4019, 10, 9, 1, 4, 67, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4093, 4088, 2, 2, 0, 2, 5, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1029, 0, 0, 1025, 0, 1025, 0, 1024, 0, 1022, 0, 1017, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1029, 1025, 1025, 1024, 1022, 1017, 1024] avg=1024.0000 max=1029 max_rank=1 min=1017 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1023, 0, 1010, 0, 10, 1005, 9, 1006, 1, 1020, 4, 1018, 67, 998, 0, 1021] rerouted_tokens_per_ep_rank=[1023, 1010, 1015, 1015, 1021, 1022, 1065, 1021] avg=1024.0000 max=1065 max_rank=6 min=1010 min_rank=1 imbalance_ratio=1.040039 max_load_reduction=7127 max_load_reduction_pct=0.869995 moved_to_echo=4129 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1028, 0, 1, 1023, 1, 1024, 0, 1024, 1, 1022, 3, 1016, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1028, 1024, 1025, 1024, 1023, 1019, 1024] avg=1024.0000 max=1028 max_rank=1 min=1019 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1029, 0, 0, 1025, 0, 1025, 0, 1024, 0, 1022, 0, 1017, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1029, 1025, 1025, 1024, 1022, 1017, 1024] avg=1024.0000 max=1029 max_rank=1 min=1017 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1014, 0, 1021, 0, 0, 1028, 0, 1027, 0, 1028, 0, 1026, 0, 1020, 0, 1028] rerouted_tokens_per_ep_rank=[1014, 1021, 1028, 1027, 1028, 1026, 1020, 1028] avg=1024.0000 max=1028 max_rank=2 min=1014 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1027, 0, 2, 1023, 2, 1023, 0, 1023, 2, 1021, 5, 1015, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1027, 1025, 1025, 1023, 1023, 1020, 1023] avg=1024.0000 max=1027 max_rank=1 min=1020 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1029, 0, 0, 1025, 0, 1025, 0, 1024, 0, 1022, 0, 1017, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1029, 1025, 1025, 1024, 1022, 1017, 1024] avg=1024.0000 max=1029 max_rank=1 min=1017 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1029, 0, 0, 1025, 0, 1025, 0, 1024, 0, 1022, 0, 1017, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1029, 1025, 1025, 1024, 1022, 1017, 1024] avg=1024.0000 max=1029 max_rank=1 min=1017 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4087 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8182 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8202 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8202, 8192, 8192, 8192, 8182, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8182 actual_valid_recv=8320 observed_loss=-138 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8202 actual_valid_recv=8320 observed_loss=-118 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 2, 0, 4086, 10, 3982, 9, 103] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 2, 4096, 0, 4094, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 4092, 1, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 2, 4090, 2, 4] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4087, 0, 9] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 2, 1020, 0, 1021, 1024, 0, 10, 998, 1001, 0, 9, 998, 103, 985] rerouted_tokens_per_ep_rank=[1021, 1022, 1021, 1024, 1008, 1001, 1007, 1088] avg=1024.0000 max=1088 max_rank=7 min=1001 min_rank=5 imbalance_ratio=1.062500 max_load_reduction=7104 max_load_reduction_pct=0.867188 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 2, 1024, 1025, 0, 0, 1026, 1030, 0, 0, 1026, 0, 1012] rerouted_tokens_per_ep_rank=[1024, 1023, 1026, 1025, 1026, 1030, 1026, 1012] avg=1024.0000 max=1030 max_rank=5 min=1012 min_rank=7 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1022, 0, 1023, 1026, 0, 1, 1025, 1030, 0, 1, 1025, 3, 1012] rerouted_tokens_per_ep_rank=[1024, 1022, 1023, 1026, 1026, 1030, 1026, 1015] avg=1024.0000 max=1030 max_rank=5 min=1015 min_rank=7 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 0, 1025, 0, 1027, 1016, 0, 0, 1029, 1021, 0, 0, 1030, 0, 1016] rerouted_tokens_per_ep_rank=[1028, 1025, 1027, 1016, 1029, 1021, 1030, 1016] avg=1024.0000 max=1030 max_rank=6 min=1016 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1026, 1031, 0, 0, 1026, 0, 1013] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1026, 1031, 1026, 1013] avg=1024.0000 max=1031 max_rank=5 min=1013 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1022, 0, 1023, 1026, 0, 2, 1025, 1029, 0, 2, 1025, 4, 1011] rerouted_tokens_per_ep_rank=[1023, 1022, 1023, 1026, 1027, 1029, 1027, 1015] avg=1024.0000 max=1029 max_rank=5 min=1015 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1026, 1031, 0, 0, 1026, 0, 1013] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1026, 1031, 1026, 1013] avg=1024.0000 max=1031 max_rank=5 min=1013 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 0, 1023, 0, 1024, 1025, 0, 0, 1024, 1028, 0, 0, 1024, 9, 1011] rerouted_tokens_per_ep_rank=[1024, 1023, 1024, 1025, 1024, 1028, 1024, 1020] avg=1024.0000 max=1028 max_rank=5 min=1020 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8201, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8201 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8201, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8201, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8201, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8201, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8201, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8201, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8183 expected_overflow=0 exact_recv_per_rank=[8192, 8183, 8192, 8192, 8192, 8201, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8183 actual_valid_recv=8320 observed_loss=-137 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8201 actual_valid_recv=8320 observed_loss=-119 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 4076, 1, 0, 0, 4096, 0, 19] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 4076, 0, 0, 0, 4096, 0, 20] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 4084, 0, 1, 1, 4095, 0, 11] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 4050, 1, 11, 10, 4085, 0, 35] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 4058, 0, 0, 0, 4096, 0, 38] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 0, 4096, 0, 16] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 4073, 0, 0, 0, 4096, 0, 23] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 4043, 1, 2, 2, 4094, 0, 50] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1029, 0, 1, 1022, 0, 1024, 0, 1024, 1026, 0, 0, 1024, 19, 999] rerouted_tokens_per_ep_rank=[1024, 1029, 1023, 1024, 1024, 1026, 1024, 1018] avg=1024.0000 max=1029 max_rank=1 min=1018 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1031, 0, 0, 1022, 1, 1026, 1, 1026, 1025, 0, 0, 1024, 11, 1001] rerouted_tokens_per_ep_rank=[1024, 1031, 1022, 1027, 1027, 1025, 1024, 1012] avg=1024.0000 max=1031 max_rank=1 min=1012 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4088 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1021, 1023, 0, 1, 1019, 11, 1017, 10, 1017, 1024, 0, 0, 1021, 35, 993] rerouted_tokens_per_ep_rank=[1021, 1023, 1020, 1028, 1027, 1024, 1021, 1028] avg=1024.0000 max=1028 max_rank=3 min=1020 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1029, 0, 0, 1022, 0, 1024, 0, 1024, 1026, 0, 0, 1024, 20, 999] rerouted_tokens_per_ep_rank=[1024, 1029, 1022, 1024, 1024, 1026, 1024, 1019] avg=1024.0000 max=1029 max_rank=1 min=1019 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1028, 1017, 0, 0, 1027, 0, 1029, 0, 1030, 1013, 0, 0, 1028, 16, 1004] rerouted_tokens_per_ep_rank=[1028, 1017, 1027, 1029, 1030, 1013, 1028, 1020] avg=1024.0000 max=1030 max_rank=4 min=1013 min_rank=5 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1025, 0, 0, 1022, 0, 1019, 0, 1019, 1026, 0, 0, 1024, 38, 995] rerouted_tokens_per_ep_rank=[1024, 1025, 1022, 1019, 1019, 1026, 1024, 1033] avg=1024.0000 max=1033 max_rank=7 min=1019 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=7159 max_load_reduction_pct=0.873901 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1020, 0, 1, 1022, 2, 1016, 2, 1016, 1026, 0, 0, 1023, 50, 991] rerouted_tokens_per_ep_rank=[1023, 1020, 1023, 1018, 1018, 1026, 1023, 1041] avg=1024.0000 max=1041 max_rank=7 min=1018 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=7151 max_load_reduction_pct=0.872925 moved_to_echo=4108 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1029, 0, 0, 1022, 0, 1023, 0, 1023, 1026, 0, 0, 1024, 23, 998] rerouted_tokens_per_ep_rank=[1024, 1029, 1022, 1023, 1023, 1026, 1024, 1021] avg=1024.0000 max=1029 max_rank=1 min=1021 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8181, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8181, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8181, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8181, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8181 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8181, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8203 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8181, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8181, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8203, 8181, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8203 actual_valid_recv=8320 observed_loss=-117 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8181 actual_valid_recv=8320 observed_loss=-139 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 2, 1, 4093, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 17, 11, 4069, 4085] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 1, 2, 4093, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1022, 0, 1022, 0, 1024, 0, 1024, 2, 1021, 1, 1022, 1028, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1022, 1024, 1024, 1023, 1023, 1028, 1025] avg=1024.0000 max=1028 max_rank=6 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1022, 0, 1024, 0, 1024, 0, 1022, 0, 1023, 1028, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1022, 1024, 1024, 1022, 1023, 1028, 1026] avg=1024.0000 max=1028 max_rank=6 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1027, 0, 1027, 0, 1028, 0, 1028, 0, 1027, 0, 1026, 1016, 0, 1013, 0] rerouted_tokens_per_ep_rank=[1027, 1027, 1028, 1028, 1027, 1026, 1016, 1013] avg=1024.0000 max=1028 max_rank=2 min=1013 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4082 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[10, 1016, 0, 1019, 0, 1021, 0, 1021, 17, 1015, 11, 1016, 1022, 0, 1024, 0] rerouted_tokens_per_ep_rank=[1026, 1019, 1021, 1021, 1032, 1027, 1022, 1024] avg=1024.0000 max=1032 max_rank=4 min=1019 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1022, 0, 1024, 0, 1024, 0, 1022, 0, 1023, 1028, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1022, 1024, 1024, 1022, 1023, 1028, 1026] avg=1024.0000 max=1028 max_rank=6 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1022, 0, 1024, 0, 1024, 0, 1022, 0, 1023, 1028, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1022, 1024, 1024, 1022, 1023, 1028, 1026] avg=1024.0000 max=1028 max_rank=6 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 1022, 0, 1022, 0, 1023, 0, 1023, 1, 1021, 2, 1022, 1028, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1022, 1023, 1023, 1022, 1024, 1028, 1026] avg=1024.0000 max=1028 max_rank=6 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1022, 0, 1024, 0, 1024, 0, 1022, 0, 1023, 1028, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1023, 1022, 1024, 1024, 1022, 1023, 1028, 1026] avg=1024.0000 max=1028 max_rank=6 min=1022 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=7164 max_load_reduction_pct=0.874512 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8178, 8192, 8192, 8192, 8192, 8206, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8178, 8192, 8192, 8192, 8192, 8206, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8178, 8192, 8192, 8192, 8192, 8206, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8178, 8192, 8192, 8192, 8192, 8206, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8178, 8192, 8192, 8192, 8192, 8206, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8178 expected_overflow=0 exact_recv_per_rank=[8192, 8178, 8192, 8192, 8192, 8192, 8206, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8178, 8192, 8192, 8192, 8192, 8206, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8206 expected_overflow=0 exact_recv_per_rank=[8192, 8178, 8192, 8192, 8192, 8192, 8206, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8178 actual_valid_recv=8192 observed_loss=-14 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8206 actual_valid_recv=8320 observed_loss=-114 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 1263, 2833, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[10, 0, 0, 872, 3213, 0, 4086, 11] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 1659, 2437, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 611, 3485, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 1195, 2900, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 1024, 3072, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[2, 0, 0, 1966, 2128, 0, 4094, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 643, 3453, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[10, 1117, 0, 1021, 0, 1020, 872, 0, 1122, 0, 0, 1021, 1024, 0, 11, 974] rerouted_tokens_per_ep_rank=[1127, 1021, 1020, 872, 1122, 1021, 1024, 985] avg=1024.0000 max=1127 max_rank=0 min=872 min_rank=3 imbalance_ratio=1.100586 max_load_reduction=7065 max_load_reduction_pct=0.862427 moved_to_echo=4152 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 847, 0, 1024, 0, 1022, 1659, 0, 851, 0, 0, 1024, 1026, 0, 0, 739] rerouted_tokens_per_ep_rank=[847, 1024, 1022, 1659, 851, 1024, 1026, 739] avg=1024.0000 max=1659 max_rank=3 min=739 min_rank=7 imbalance_ratio=1.620117 max_load_reduction=6533 max_load_reduction_pct=0.797485 moved_to_echo=3640 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 985, 0, 1024, 0, 1022, 1263, 0, 989, 0, 0, 1024, 1026, 0, 0, 859] rerouted_tokens_per_ep_rank=[985, 1024, 1022, 1263, 989, 1024, 1026, 859] avg=1024.0000 max=1263 max_rank=3 min=859 min_rank=7 imbalance_ratio=1.233398 max_load_reduction=6929 max_load_reduction_pct=0.845825 moved_to_echo=3898 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1008, 0, 1024, 0, 1022, 1195, 0, 1013, 0, 0, 1024, 1025, 0, 1, 879] rerouted_tokens_per_ep_rank=[1009, 1024, 1022, 1195, 1013, 1024, 1025, 880] avg=1024.0000 max=1195 max_rank=3 min=880 min_rank=7 imbalance_ratio=1.166992 max_load_reduction=6997 max_load_reduction_pct=0.854126 moved_to_echo=3942 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1215, 0, 1028, 0, 1027, 611, 0, 1208, 0, 0, 1028, 1013, 0, 0, 1062] rerouted_tokens_per_ep_rank=[1215, 1028, 1027, 611, 1208, 1028, 1013, 1062] avg=1024.0000 max=1215 max_rank=0 min=611 min_rank=3 imbalance_ratio=1.186523 max_load_reduction=6977 max_load_reduction_pct=0.851685 moved_to_echo=4311 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 739, 0, 1023, 0, 1022, 1966, 0, 744, 0, 0, 1023, 1026, 0, 2, 645] rerouted_tokens_per_ep_rank=[741, 1023, 1022, 1966, 744, 1023, 1026, 647] avg=1024.0000 max=1966 max_rank=3 min=647 min_rank=7 imbalance_ratio=1.919922 max_load_reduction=6226 max_load_reduction_pct=0.760010 moved_to_echo=3440 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1068, 0, 1024, 0, 1022, 1024, 0, 1072, 0, 0, 1024, 1026, 0, 0, 932] rerouted_tokens_per_ep_rank=[1068, 1024, 1022, 1024, 1072, 1024, 1026, 932] avg=1024.0000 max=1072 max_rank=4 min=932 min_rank=7 imbalance_ratio=1.046875 max_load_reduction=7120 max_load_reduction_pct=0.869141 moved_to_echo=4054 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1200, 0, 1024, 0, 1022, 643, 0, 1206, 0, 0, 1024, 1026, 0, 0, 1047] rerouted_tokens_per_ep_rank=[1200, 1024, 1022, 643, 1206, 1024, 1026, 1047] avg=1024.0000 max=1206 max_rank=4 min=643 min_rank=3 imbalance_ratio=1.177734 max_load_reduction=6986 max_load_reduction_pct=0.852783 moved_to_echo=4303 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 9233, 8205, 8192, 8192, 7151] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=9233 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 9233, 8205, 8192, 8192, 7151] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7151 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 9233, 8205, 8192, 8192, 7151] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 9233, 8205, 8192, 8192, 7151] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8179 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 9233, 8205, 8192, 8192, 7151] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8205 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 9233, 8205, 8192, 8192, 7151] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 9233, 8205, 8192, 8192, 7151] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8179, 9233, 8205, 8192, 8192, 7151] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=7151 actual_valid_recv=7296 observed_loss=-145 tokens_per_expert=[128, 7168] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8179 actual_valid_recv=8192 observed_loss=-13 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8205 actual_valid_recv=8320 observed_loss=-115 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=9233 actual_valid_recv=9344 observed_loss=-111 tokens_per_expert=[9344, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=7296 slack_slots=9088 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=9344 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2176, 3, 1, 0, 3211, 0, 2801, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[858, 0, 0, 0, 3794, 11, 3529, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1240, 8, 0, 0, 3384, 2, 3557, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1295, 0, 0, 1, 3562, 0, 3334, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[942, 0, 0, 1, 3727, 0, 3522, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1354, 5, 0, 0, 3568, 3, 3261, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1284, 2, 1, 0, 3403, 1, 3500, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1205, 1, 2, 0, 3779, 0, 3205, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2176, 0, 3, 630, 1, 454, 0, 925, 1362, 0, 0, 857, 860, 0, 0, 924] rerouted_tokens_per_ep_rank=[2176, 633, 455, 925, 1362, 857, 860, 924] avg=1024.0000 max=2176 max_rank=0 min=455 min_rank=2 imbalance_ratio=2.125000 max_load_reduction=6016 max_load_reduction_pct=0.734375 moved_to_echo=4003 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1240, 0, 8, 800, 0, 576, 0, 974, 1436, 0, 2, 1088, 1093, 0, 1, 974] rerouted_tokens_per_ep_rank=[1240, 808, 576, 974, 1436, 1090, 1093, 975] avg=1024.0000 max=1436 max_rank=4 min=576 min_rank=2 imbalance_ratio=1.402344 max_load_reduction=6756 max_load_reduction_pct=0.824707 moved_to_echo=4594 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[858, 0, 0, 798, 0, 576, 0, 1097, 1600, 0, 11, 1083, 1072, 0, 0, 1097] rerouted_tokens_per_ep_rank=[858, 798, 576, 1097, 1600, 1094, 1072, 1097] avg=1024.0000 max=1600 max_rank=4 min=576 min_rank=2 imbalance_ratio=1.562500 max_load_reduction=6592 max_load_reduction_pct=0.804688 moved_to_echo=4863 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1295, 0, 0, 750, 0, 540, 1, 1026, 1510, 0, 0, 1020, 1024, 0, 0, 1026] rerouted_tokens_per_ep_rank=[1295, 750, 540, 1027, 1510, 1020, 1024, 1026] avg=1024.0000 max=1510 max_rank=4 min=540 min_rank=2 imbalance_ratio=1.474609 max_load_reduction=6682 max_load_reduction_pct=0.815674 moved_to_echo=4580 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[942, 0, 0, 792, 0, 571, 1, 1073, 1581, 0, 0, 1078, 1081, 0, 0, 1073] rerouted_tokens_per_ep_rank=[942, 792, 571, 1074, 1581, 1078, 1081, 1073] avg=1024.0000 max=1581 max_rank=4 min=571 min_rank=2 imbalance_ratio=1.543945 max_load_reduction=6611 max_load_reduction_pct=0.807007 moved_to_echo=4813 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1284, 0, 2, 787, 1, 567, 0, 980, 1443, 0, 1, 1071, 1075, 0, 1, 980] rerouted_tokens_per_ep_rank=[1284, 789, 568, 980, 1443, 1072, 1075, 981] avg=1024.0000 max=1443 max_rank=4 min=568 min_rank=2 imbalance_ratio=1.409180 max_load_reduction=6749 max_load_reduction_pct=0.823853 moved_to_echo=4571 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1354, 0, 5, 733, 0, 528, 0, 1027, 1514, 0, 3, 998, 1002, 0, 1, 1027] rerouted_tokens_per_ep_rank=[1354, 738, 528, 1027, 1514, 1001, 1002, 1028] avg=1024.0000 max=1514 max_rank=4 min=528 min_rank=2 imbalance_ratio=1.478516 max_load_reduction=6678 max_load_reduction_pct=0.815186 moved_to_echo=4545 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1205, 0, 1, 721, 2, 519, 0, 1088, 1603, 0, 0, 980, 985, 0, 0, 1088] rerouted_tokens_per_ep_rank=[1205, 722, 521, 1088, 1603, 980, 985, 1088] avg=1024.0000 max=1603 max_rank=4 min=521 min_rank=2 imbalance_ratio=1.565430 max_load_reduction=6589 max_load_reduction_pct=0.804321 moved_to_echo=4656 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=4335 expected_overflow=0 exact_recv_per_rank=[10354, 6030, 4335, 8192, 12049, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=10354 expected_overflow=0 exact_recv_per_rank=[10354, 6030, 4335, 8192, 12049, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10354, 6030, 4335, 8192, 12049, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=6030 expected_overflow=0 exact_recv_per_rank=[10354, 6030, 4335, 8192, 12049, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10354, 6030, 4335, 8192, 12049, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=12049 expected_overflow=0 exact_recv_per_rank=[10354, 6030, 4335, 8192, 12049, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10354, 6030, 4335, 8192, 12049, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[10354, 6030, 4335, 8192, 12049, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=4335 actual_valid_recv=4480 observed_loss=-145 tokens_per_expert=[128, 4352] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=6030 actual_valid_recv=6144 observed_loss=-114 tokens_per_expert=[128, 6016] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=10354 actual_valid_recv=10368 observed_loss=-14 tokens_per_expert=[10368, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=12049 actual_valid_recv=12160 observed_loss=-111 tokens_per_expert=[12160, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=4480 slack_slots=11904 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=6144 slack_slots=10240 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=10368 slack_slots=6016 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=12160 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 60, 0, 0, 0, 0, 4036] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4095, 1, 53, 0, 1, 0, 0, 4042] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4094, 1, 34, 0, 0, 0, 0, 4063] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4094, 1, 44, 0, 0, 2, 1, 4050] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 52, 0, 1, 1, 0, 4043] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4095, 1, 51, 0, 0, 1, 0, 4044] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 66, 0, 1, 1, 0, 4029] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4095, 1, 53, 0, 1, 0, 0, 4042] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1021, 60, 970, 0, 1024, 0, 1023, 0, 1021, 0, 1024, 1024, 0] rerouted_tokens_per_ep_rank=[1025, 1021, 1030, 1024, 1023, 1021, 1024, 1024] avg=1024.0000 max=1030 max_rank=2 min=1021 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1015, 0, 1, 1027, 53, 976, 0, 1026, 1, 1025, 0, 1027, 0, 1029, 1012, 0] rerouted_tokens_per_ep_rank=[1015, 1028, 1029, 1026, 1026, 1027, 1029, 1012] avg=1024.0000 max=1029 max_rank=2 min=1012 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1, 1024, 44, 973, 0, 1023, 0, 1022, 2, 1024, 1, 1023, 1029, 0] rerouted_tokens_per_ep_rank=[1026, 1025, 1017, 1023, 1022, 1026, 1024, 1029] avg=1024.0000 max=1029 max_rank=7 min=1017 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4101 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1, 1028, 34, 977, 0, 1023, 0, 1022, 0, 1028, 0, 1023, 1030, 0] rerouted_tokens_per_ep_rank=[1026, 1029, 1011, 1023, 1022, 1028, 1023, 1030] avg=1024.0000 max=1030 max_rank=7 min=1011 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4103 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1023, 52, 972, 0, 1024, 1, 1023, 1, 1023, 0, 1023, 1025, 0] rerouted_tokens_per_ep_rank=[1025, 1023, 1024, 1024, 1024, 1024, 1023, 1025] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1, 1023, 51, 972, 0, 1024, 0, 1023, 1, 1023, 0, 1023, 1026, 0] rerouted_tokens_per_ep_rank=[1025, 1024, 1023, 1024, 1023, 1024, 1023, 1026] avg=1024.0000 max=1026 max_rank=7 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 0, 1019, 66, 968, 0, 1024, 1, 1023, 1, 1019, 0, 1023, 1023, 0] rerouted_tokens_per_ep_rank=[1025, 1019, 1034, 1024, 1024, 1020, 1023, 1023] avg=1024.0000 max=1034 max_rank=2 min=1019 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=7158 max_load_reduction_pct=0.873779 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1, 1022, 53, 971, 0, 1024, 1, 1023, 0, 1022, 0, 1023, 1027, 0] rerouted_tokens_per_ep_rank=[1025, 1023, 1024, 1024, 1024, 1022, 1023, 1027] avg=1024.0000 max=1027 max_rank=7 min=1022 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8196] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8196] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8196] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8188 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8196] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8196] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8196 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8196] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8196] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8188, 8192, 8192, 8196] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8196 actual_valid_recv=8320 observed_loss=-124 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[512, 7808] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8188 actual_valid_recv=8320 observed_loss=-132 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[2, 4095, 0, 0, 0, 0, 4094, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1, 4095, 0, 0, 0, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1, 4095, 0, 0, 0, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1, 4095, 0, 0, 0, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1, 4095, 0, 0, 0, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1, 4095, 0, 0, 0, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1022, 1026, 0, 1, 1023] rerouted_tokens_per_ep_rank=[1025, 1026, 1023, 1023, 1023, 1022, 1026, 1024] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1024, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1024, 1024, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 1, 1023, 0, 1024, 0, 1024, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1024, 1024, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1024, 1017, 0, 0, 1029, 0, 1029, 0, 1029, 0, 1025, 1012, 0, 1, 1025] rerouted_tokens_per_ep_rank=[1025, 1017, 1029, 1029, 1029, 1025, 1012, 1026] avg=1024.0000 max=1029 max_rank=2 min=1012 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1023, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1023, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1023, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023] rerouted_tokens_per_ep_rank=[1024, 1026, 1023, 1023, 1023, 1023, 1026, 1024] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8192, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[3888, 0, 0, 4049, 253, 0, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[3967, 0, 0, 4045, 176, 0, 2, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[3867, 0, 0, 4064, 261, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[3783, 0, 0, 3997, 412, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4031, 0, 1, 4057, 102, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[3571, 0, 1, 3846, 773, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4028, 0, 1, 4035, 127, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4034, 0, 1, 4042, 114, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1096, 0, 0, 1036, 0, 1015, 1037, 0, 261, 741, 0, 1036, 0, 1015, 0, 955] rerouted_tokens_per_ep_rank=[1096, 1036, 1015, 1037, 1002, 1036, 1015, 955] avg=1024.0000 max=1096 max_rank=0 min=955 min_rank=7 imbalance_ratio=1.070312 max_load_reduction=7096 max_load_reduction_pct=0.866211 moved_to_echo=4008 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1124, 0, 0, 1031, 0, 1042, 1032, 0, 176, 760, 0, 1031, 2, 1041, 2, 951] rerouted_tokens_per_ep_rank=[1124, 1031, 1042, 1032, 936, 1031, 1043, 953] avg=1024.0000 max=1124 max_rank=0 min=936 min_rank=4 imbalance_ratio=1.097656 max_load_reduction=7068 max_load_reduction_pct=0.862793 moved_to_echo=3963 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1102, 0, 0, 1032, 0, 1021, 1033, 0, 253, 745, 0, 1032, 1, 1020, 1, 952] rerouted_tokens_per_ep_rank=[1102, 1032, 1021, 1033, 998, 1032, 1021, 953] avg=1024.0000 max=1102 max_rank=0 min=953 min_rank=7 imbalance_ratio=1.076172 max_load_reduction=7090 max_load_reduction_pct=0.865479 moved_to_echo=4004 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1072, 0, 0, 1018, 0, 993, 1021, 0, 412, 725, 0, 1018, 0, 993, 0, 940] rerouted_tokens_per_ep_rank=[1072, 1018, 993, 1021, 1137, 1018, 993, 940] avg=1024.0000 max=1137 max_rank=4 min=940 min_rank=7 imbalance_ratio=1.110352 max_load_reduction=7055 max_load_reduction_pct=0.861206 moved_to_echo=4088 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1132, 0, 0, 1037, 1, 1062, 1025, 0, 102, 774, 0, 1037, 1, 1063, 0, 958] rerouted_tokens_per_ep_rank=[1132, 1037, 1063, 1025, 876, 1037, 1064, 958] avg=1024.0000 max=1132 max_rank=0 min=876 min_rank=4 imbalance_ratio=1.105469 max_load_reduction=7060 max_load_reduction_pct=0.861816 moved_to_echo=3935 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1012, 0, 0, 980, 1, 938, 982, 0, 773, 684, 0, 980, 1, 937, 0, 904] rerouted_tokens_per_ep_rank=[1012, 980, 939, 982, 1457, 980, 938, 904] avg=1024.0000 max=1457 max_rank=4 min=904 min_rank=7 imbalance_ratio=1.422852 max_load_reduction=6735 max_load_reduction_pct=0.822144 moved_to_echo=4279 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1143, 0, 0, 1030, 1, 1059, 1032, 0, 114, 773, 0, 1030, 1, 1059, 0, 950] rerouted_tokens_per_ep_rank=[1143, 1030, 1060, 1032, 887, 1030, 1060, 950] avg=1024.0000 max=1143 max_rank=0 min=887 min_rank=4 imbalance_ratio=1.116211 max_load_reduction=7049 max_load_reduction_pct=0.860474 moved_to_echo=3927 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1141, 0, 0, 1028, 1, 1058, 1030, 0, 127, 772, 0, 1028, 1, 1057, 0, 949] rerouted_tokens_per_ep_rank=[1141, 1028, 1059, 1030, 899, 1028, 1058, 949] avg=1024.0000 max=1141 max_rank=0 min=899 min_rank=4 imbalance_ratio=1.114258 max_load_reduction=7051 max_load_reduction_pct=0.860718 moved_to_echo=3934 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8822, 8192, 8192, 8192, 8192, 8192, 8192, 7562] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8822, 8192, 8192, 8192, 8192, 8192, 8192, 7562] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8822, 8192, 8192, 8192, 8192, 8192, 8192, 7562] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8822 expected_overflow=0 exact_recv_per_rank=[8822, 8192, 8192, 8192, 8192, 8192, 8192, 7562] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8822, 8192, 8192, 8192, 8192, 8192, 8192, 7562] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7562 expected_overflow=0 exact_recv_per_rank=[8822, 8192, 8192, 8192, 8192, 8192, 8192, 7562] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8822, 8192, 8192, 8192, 8192, 8192, 8192, 7562] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8822, 8192, 8192, 8192, 8192, 8192, 8192, 7562] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8822 actual_valid_recv=8832 observed_loss=-10 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=7562 actual_valid_recv=7808 observed_loss=-246 tokens_per_expert=[128, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[2304, 6016] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 1, 4094, 1, 0, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 4096, 0, 0, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 1, 4095, 0, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 4095, 1, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 4095, 1, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 4096, 0, 0, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 0, 4095, 1, 0, 4094, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 4095, 1, 0, 4095, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 2, 1023, 1, 1022, 1026, 0, 1, 1023, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1026, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1023, 1023, 1025, 1023, 1024, 1025, 1024] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1026, 1, 1024, 0, 1025, 1012, 0, 1, 1026, 0, 1029, 1019, 0, 0, 1029] rerouted_tokens_per_ep_rank=[1026, 1025, 1025, 1012, 1027, 1029, 1019, 1029] avg=1024.0000 max=1029 max_rank=5 min=1012 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 1023, 0, 1023, 0, 1023, 1025, 0, 0, 1023, 0, 1024, 1025, 0, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1023, 1023, 1025, 1023, 1024, 1025, 1024] avg=1024.0000 max=1025 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=7167 max_load_reduction_pct=0.874878 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1023, 1, 1023, 1026, 0, 0, 1023, 0, 1023, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1026, 1023, 1023, 1026, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1023, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1026, 1024, 1023, 1026, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 1, 1023, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 1025, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1024, 1024, 1023, 1026, 1024, 1023, 1025, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1, 1023, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 1026, 0, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1024, 1023, 1026, 1024, 1023, 1026, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8187 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8197 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8187, 8192, 8192, 8192, 8197, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8187 actual_valid_recv=8320 observed_loss=-133 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8197 actual_valid_recv=8320 observed_loss=-123 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[245, 4095, 1, 3850, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[291, 4095, 1, 3804, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[479, 4095, 1, 3616, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[395, 4096, 0, 3701, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[520, 4096, 0, 3576, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[818, 4094, 2, 3276, 0, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[942, 4095, 1, 3153, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[479, 4095, 1, 3616, 0, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[395, 520, 1025, 0, 0, 1059, 1063, 0, 0, 1024, 0, 1023, 0, 1059, 0, 1024] rerouted_tokens_per_ep_rank=[915, 1025, 1059, 1063, 1024, 1023, 1059, 1024] avg=1024.0000 max=1063 max_rank=3 min=915 min_rank=0 imbalance_ratio=1.038086 max_load_reduction=7129 max_load_reduction_pct=0.870239 moved_to_echo=4130 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[291, 540, 1012, 0, 1, 1092, 1080, 0, 0, 1029, 0, 1025, 1, 1092, 0, 1029] rerouted_tokens_per_ep_rank=[831, 1012, 1093, 1080, 1029, 1025, 1093, 1029] avg=1024.0000 max=1093 max_rank=2 min=831 min_rank=0 imbalance_ratio=1.067383 max_load_reduction=7099 max_load_reduction_pct=0.866577 moved_to_echo=4176 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[245, 541, 1026, 0, 1, 1102, 1105, 0, 0, 1023, 0, 1023, 1, 1102, 0, 1023] rerouted_tokens_per_ep_rank=[786, 1026, 1103, 1105, 1023, 1023, 1103, 1023] avg=1024.0000 max=1105 max_rank=3 min=786 min_rank=0 imbalance_ratio=1.079102 max_load_reduction=7087 max_load_reduction_pct=0.865112 moved_to_echo=4172 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[520, 503, 1025, 0, 0, 1023, 1027, 0, 0, 1024, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1027, 1024, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[818, 460, 1026, 0, 2, 937, 942, 0, 0, 1023, 0, 1022, 2, 937, 0, 1023] rerouted_tokens_per_ep_rank=[1278, 1026, 939, 942, 1023, 1022, 939, 1023] avg=1024.0000 max=1278 max_rank=0 min=939 min_rank=2 imbalance_ratio=1.248047 max_load_reduction=6914 max_load_reduction_pct=0.843994 moved_to_echo=4007 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[479, 508, 1026, 0, 1, 1035, 1038, 0, 0, 1023, 0, 1023, 1, 1035, 0, 1023] rerouted_tokens_per_ep_rank=[987, 1026, 1036, 1038, 1023, 1023, 1036, 1023] avg=1024.0000 max=1038 max_rank=3 min=987 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[479, 508, 1026, 0, 1, 1035, 1038, 0, 0, 1023, 0, 1023, 1, 1035, 0, 1023] rerouted_tokens_per_ep_rank=[987, 1026, 1036, 1038, 1023, 1023, 1036, 1023] avg=1024.0000 max=1038 max_rank=3 min=987 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=7154 max_load_reduction_pct=0.873291 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[942, 443, 1026, 0, 1, 902, 906, 0, 0, 1023, 0, 1023, 1, 902, 0, 1023] rerouted_tokens_per_ep_rank=[1385, 1026, 903, 906, 1023, 1023, 903, 1023] avg=1024.0000 max=1385 max_rank=0 min=903 min_rank=2 imbalance_ratio=1.352539 max_load_reduction=6807 max_load_reduction_pct=0.830933 moved_to_echo=3972 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8199, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8199, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8199, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8199, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8199, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8199, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8199, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8199, 8192, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[4224, 4096] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 4094, 2, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 4095, 1, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1023, 0, 1022, 0, 1023, 0, 1023, 1026, 0, 2, 1022, 2, 1022] rerouted_tokens_per_ep_rank=[1027, 1023, 1022, 1023, 1023, 1026, 1024, 1024] avg=1024.0000 max=1027 max_rank=0 min=1022 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1024, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1016, 0, 0, 1029, 0, 1025, 0, 1029, 0, 1029, 1012, 0, 1, 1025, 1, 1025] rerouted_tokens_per_ep_rank=[1016, 1029, 1025, 1029, 1029, 1012, 1026, 1026] avg=1024.0000 max=1029 max_rank=1 min=1012 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 1025, 0, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1024, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1023, 1023, 1026, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1023, 1023, 1026, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1023, 1023, 1026, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023, 1, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1023, 1023, 1026, 1024, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8185, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4096, 3533, 0, 0, 0, 0, 0, 563] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4095, 3169, 0, 0, 1, 0, 1, 926] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4094, 3493, 0, 0, 2, 0, 2, 601] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4096, 2757, 0, 0, 0, 0, 0, 1339] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4095, 3499, 0, 0, 1, 0, 1, 596] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4095, 3401, 0, 0, 1, 0, 1, 694] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4095, 3218, 0, 0, 1, 0, 1, 877] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4095, 3295, 0, 0, 1, 0, 1, 800] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 860, 0, 0, 1024, 0, 1023, 0, 855, 0, 1024, 0, 855, 1339, 187] rerouted_tokens_per_ep_rank=[1025, 860, 1024, 1023, 855, 1024, 855, 1526] avg=1024.0000 max=1526 max_rank=7 min=855 min_rank=4 imbalance_ratio=1.490234 max_load_reduction=6666 max_load_reduction_pct=0.813721 moved_to_echo=4260 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1088, 0, 0, 1023, 0, 1022, 2, 1084, 0, 1023, 2, 1084, 601, 237] rerouted_tokens_per_ep_rank=[1026, 1088, 1023, 1022, 1086, 1023, 1086, 838] avg=1024.0000 max=1088 max_rank=1 min=838 min_rank=7 imbalance_ratio=1.062500 max_load_reduction=7104 max_load_reduction_pct=0.867188 moved_to_echo=4033 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1101, 0, 0, 1024, 0, 1023, 0, 1096, 0, 1024, 0, 1096, 563, 240] rerouted_tokens_per_ep_rank=[1025, 1101, 1024, 1023, 1096, 1024, 1096, 803] avg=1024.0000 max=1101 max_rank=1 min=803 min_rank=7 imbalance_ratio=1.075195 max_load_reduction=7091 max_load_reduction_pct=0.865601 moved_to_echo=4019 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 988, 0, 0, 1023, 0, 1023, 1, 983, 0, 1023, 1, 983, 926, 215] rerouted_tokens_per_ep_rank=[1026, 988, 1023, 1023, 984, 1023, 984, 1141] avg=1024.0000 max=1141 max_rank=7 min=984 min_rank=4 imbalance_ratio=1.114258 max_load_reduction=7051 max_load_reduction_pct=0.860718 moved_to_echo=4132 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1012, 0, 1074, 0, 0, 1029, 0, 1025, 1, 1091, 0, 1029, 1, 1091, 596, 243] rerouted_tokens_per_ep_rank=[1012, 1074, 1029, 1025, 1092, 1029, 1092, 839] avg=1024.0000 max=1092 max_rank=4 min=839 min_rank=7 imbalance_ratio=1.066406 max_load_reduction=7100 max_load_reduction_pct=0.866699 moved_to_echo=4052 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1060, 0, 0, 1023, 0, 1023, 1, 1055, 0, 1023, 1, 1055, 694, 231] rerouted_tokens_per_ep_rank=[1026, 1060, 1023, 1023, 1056, 1023, 1056, 925] avg=1024.0000 max=1060 max_rank=1 min=925 min_rank=7 imbalance_ratio=1.035156 max_load_reduction=7132 max_load_reduction_pct=0.870605 moved_to_echo=4060 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1001, 0, 0, 1023, 0, 1023, 1, 999, 0, 1023, 1, 999, 877, 219] rerouted_tokens_per_ep_rank=[1026, 1001, 1023, 1023, 1000, 1023, 1000, 1096] avg=1024.0000 max=1096 max_rank=7 min=1000 min_rank=4 imbalance_ratio=1.070312 max_load_reduction=7096 max_load_reduction_pct=0.866211 moved_to_echo=4119 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1027, 0, 0, 1023, 0, 1023, 1, 1022, 0, 1023, 1, 1022, 800, 224] rerouted_tokens_per_ep_rank=[1026, 1027, 1023, 1023, 1023, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=1 min=1023 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8199, 8192, 8185, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[6400, 1920] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4094, 0, 4094, 2, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4095, 1, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4095, 1, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4095, 1, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4095, 1, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4095, 1, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1026, 0, 0, 1023, 1027, 0, 2, 1022, 0, 1023, 2, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1022, 1026, 1023, 1027, 1024, 1023, 1024, 1023] avg=1024.0000 max=1027 max_rank=3 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1026, 1024, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1024, 1026, 0, 0, 1023, 0, 1024, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1024, 1026, 1023, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 1012, 0, 0, 1029, 1016, 0, 1, 1025, 0, 1029, 1, 1025, 0, 1029] rerouted_tokens_per_ep_rank=[1025, 1012, 1029, 1016, 1026, 1029, 1026, 1029] avg=1024.0000 max=1029 max_rank=2 min=1012 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4110 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1026, 1024, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1026, 1024, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 1, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1026, 1024, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8199, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8199, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8199, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8199, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8199, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8199, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8199, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8199, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4093, 1, 1, 0, 4095, 2, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4095, 1, 1, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4096, 1, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4094, 2, 2, 0, 4094, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4094, 1, 1, 0, 4095, 1, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4095, 1, 1, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4095, 1, 1, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 0, 1023, 0, 1023, 0, 1024, 1025, 0, 0, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1027, 1023, 1023, 1024, 1025, 1023, 1023, 1024] avg=1024.0000 max=1027 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1, 1022, 1, 1022, 0, 1023, 1026, 0, 2, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1023, 1026, 1025, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 2, 1022, 2, 1022, 0, 1023, 1026, 0, 0, 1023, 0, 1022, 0, 1023] rerouted_tokens_per_ep_rank=[1027, 1024, 1024, 1023, 1026, 1023, 1022, 1023] avg=1024.0000 max=1027 max_rank=0 min=1022 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 0, 1023, 0, 1023, 0, 1024, 1025, 0, 1, 1023, 0, 1023, 0, 1024] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1024, 1025, 1024, 1023, 1024] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1014, 0, 1, 1027, 1, 1027, 0, 1029, 1012, 0, 0, 1027, 0, 1025, 0, 1029] rerouted_tokens_per_ep_rank=[1014, 1028, 1028, 1029, 1012, 1027, 1025, 1029] avg=1024.0000 max=1029 max_rank=3 min=1012 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1027, 0, 1, 1022, 1, 1022, 0, 1023, 1026, 0, 1, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1027, 1023, 1023, 1023, 1026, 1024, 1023, 1023] avg=1024.0000 max=1027 max_rank=0 min=1023 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1, 1023, 1, 1023, 0, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1024, 1023, 1026, 1023, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1, 1023, 1, 1023, 0, 1023, 1026, 0, 0, 1023, 0, 1023, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1024, 1023, 1026, 1023, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8199, 8192, 8192, 8192, 8192, 8192, 8185, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4087, 0, 9, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4091, 0, 5, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1, 4095, 1, 0, 4074, 0, 21, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1, 4095, 1, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[2, 4094, 2, 0, 4094, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1, 4095, 1, 0, 4094, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1, 4095, 1, 0, 4094, 0, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1, 4095, 1, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 1025, 0, 0, 1022, 0, 1024, 1025, 0, 0, 1023, 9, 1018, 0, 1024] rerouted_tokens_per_ep_rank=[1022, 1025, 1022, 1024, 1025, 1023, 1027, 1024] avg=1024.0000 max=1027 max_rank=6 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1025, 0, 0, 1023, 0, 1024, 1026, 0, 0, 1023, 5, 1019, 0, 1024] rerouted_tokens_per_ep_rank=[1023, 1025, 1023, 1024, 1026, 1023, 1024, 1024] avg=1024.0000 max=1026 max_rank=4 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 1024, 1026, 0, 2, 1024, 0, 1023, 1026, 0, 0, 1022, 0, 1020, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1026, 1026, 1023, 1026, 1022, 1020, 1023] avg=1024.0000 max=1026 max_rank=0 min=1020 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4091 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1018, 1026, 0, 1, 1018, 0, 1023, 1023, 0, 0, 1023, 21, 1015, 0, 1023] rerouted_tokens_per_ep_rank=[1019, 1026, 1019, 1023, 1023, 1023, 1036, 1023] avg=1024.0000 max=1036 max_rank=6 min=1019 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=7156 max_load_reduction_pct=0.873535 moved_to_echo=4105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1026, 1012, 0, 1, 1026, 0, 1029, 1020, 0, 0, 1025, 0, 1023, 0, 1029] rerouted_tokens_per_ep_rank=[1027, 1012, 1027, 1029, 1020, 1025, 1023, 1029] avg=1024.0000 max=1029 max_rank=3 min=1012 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1024, 1026, 0, 1, 1024, 0, 1023, 1026, 0, 0, 1023, 1, 1020, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1026, 1025, 1023, 1026, 1023, 1021, 1023] avg=1024.0000 max=1026 max_rank=1 min=1021 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1024, 1026, 0, 1, 1024, 0, 1023, 1026, 0, 0, 1023, 1, 1020, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1026, 1025, 1023, 1026, 1023, 1021, 1023] avg=1024.0000 max=1026 max_rank=1 min=1021 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1024, 1026, 0, 1, 1024, 0, 1023, 1027, 0, 0, 1023, 0, 1020, 0, 1023] rerouted_tokens_per_ep_rank=[1025, 1026, 1025, 1023, 1027, 1023, 1020, 1023] avg=1024.0000 max=1027 max_rank=4 min=1020 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8199, 8185, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4094, 4084, 2, 1, 1, 0, 10, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4095, 4092, 1, 1, 0, 0, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4095, 4088, 1, 1, 0, 7, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4095, 4093, 1, 1, 0, 1, 1, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4095, 4093, 1, 1, 0, 0, 2, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4095, 4092, 1, 1, 0, 0, 3, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1023, 0, 2, 1021, 1, 1022, 1, 1023, 0, 1021, 10, 1019, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1023, 1023, 1023, 1024, 1021, 1029, 1023] avg=1024.0000 max=1029 max_rank=6 min=1021 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1022, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1026, 1024, 1023, 1024, 1024, 1022, 1024] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 0, 1026, 0, 0, 1024, 0, 1023, 0, 1024, 0, 1024, 0, 1022, 0, 1024] rerouted_tokens_per_ep_rank=[1025, 1026, 1024, 1023, 1024, 1024, 1022, 1024] avg=1024.0000 max=1026 max_rank=1 min=1022 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4094 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1012, 0, 1016, 0, 1, 1025, 1, 1026, 0, 1028, 0, 1024, 3, 1027, 0, 1029] rerouted_tokens_per_ep_rank=[1012, 1016, 1026, 1027, 1028, 1024, 1030, 1029] avg=1024.0000 max=1030 max_rank=6 min=1012 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4111 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1024, 0, 1, 1022, 1, 1023, 0, 1023, 7, 1022, 0, 1020, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1024, 1023, 1024, 1023, 1029, 1020, 1023] avg=1024.0000 max=1029 max_rank=5 min=1020 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4095 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1026, 0, 1, 1023, 1, 1023, 0, 1023, 1, 1023, 1, 1021, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1026, 1024, 1024, 1023, 1024, 1022, 1023] avg=1024.0000 max=1026 max_rank=0 min=1022 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1026, 0, 1, 1023, 1, 1023, 0, 1023, 0, 1023, 2, 1021, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1026, 1024, 1024, 1023, 1023, 1023, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 0, 1025, 0, 1, 1023, 1, 1023, 0, 1023, 0, 1023, 3, 1021, 0, 1023] rerouted_tokens_per_ep_rank=[1026, 1025, 1024, 1024, 1023, 1023, 1024, 1023] avg=1024.0000 max=1026 max_rank=0 min=1023 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4093 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 2, 4083, 2, 11] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 4082, 1, 13] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 4088, 1, 7] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 4085, 1, 10] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 4093, 1, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 4090, 1, 5] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 1025, 0, 0, 1024, 1029, 0, 0, 1024, 0, 1019] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1025, 1024, 1029, 1024, 1019] avg=1024.0000 max=1029 max_rank=5 min=1019 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1021, 1024, 0, 1, 1021, 13, 1016] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1026, 1022, 1024, 1022, 1029] avg=1024.0000 max=1029 max_rank=7 min=1022 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1024, 0, 1024, 1025, 0, 0, 1024, 1029, 0, 0, 1024, 0, 1019] rerouted_tokens_per_ep_rank=[1023, 1024, 1024, 1025, 1024, 1029, 1024, 1019] avg=1024.0000 max=1029 max_rank=5 min=1019 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4096 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1022, 0, 1023, 0, 1023, 1026, 0, 2, 1021, 1025, 0, 2, 1021, 11, 1016] rerouted_tokens_per_ep_rank=[1022, 1023, 1023, 1026, 1023, 1025, 1023, 1027] avg=1024.0000 max=1027 max_rank=7 min=1022 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1024, 1027, 0, 1, 1024, 2, 1018] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1026, 1025, 1027, 1025, 1020] avg=1024.0000 max=1027 max_rank=5 min=1020 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1023, 1026, 0, 1, 1023, 5, 1018] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1026, 1024, 1026, 1024, 1023] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1025, 0, 1029, 0, 1029, 1012, 0, 1, 1026, 1014, 0, 1, 1026, 7, 1022] rerouted_tokens_per_ep_rank=[1025, 1029, 1029, 1012, 1027, 1014, 1027, 1029] avg=1024.0000 max=1029 max_rank=1 min=1012 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1023, 1026, 0, 1, 1022, 1025, 0, 1, 1022, 10, 1016] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1026, 1023, 1025, 1023, 1026] avg=1024.0000 max=1026 max_rank=3 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8185, 8192, 8192, 8192, 8192, 8199, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 4096, 0, 6] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 4096, 0, 3] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 4082, 0, 1, 1, 4095, 0, 13] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 4007, 1, 2, 2, 4094, 0, 86] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 4082, 0, 1, 1, 4095, 0, 13] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 4077, 0, 1, 1, 4095, 0, 18] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 4068, 0, 1, 1, 4095, 0, 27] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 4058, 0, 1, 1, 4095, 0, 37] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1031, 0, 0, 1023, 0, 1029, 0, 1029, 1025, 0, 0, 1024, 3, 1004] rerouted_tokens_per_ep_rank=[1024, 1031, 1023, 1029, 1029, 1025, 1024, 1007] avg=1024.0000 max=1031 max_rank=1 min=1007 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4085 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1024, 1031, 0, 0, 1023, 0, 1028, 0, 1028, 1025, 0, 0, 1024, 6, 1003] rerouted_tokens_per_ep_rank=[1024, 1031, 1023, 1028, 1028, 1025, 1024, 1009] avg=1024.0000 max=1031 max_rank=1 min=1009 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=7161 max_load_reduction_pct=0.874146 moved_to_echo=4086 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1010, 0, 1, 1022, 2, 1007, 2, 1007, 1026, 0, 0, 1023, 86, 983] rerouted_tokens_per_ep_rank=[1023, 1010, 1023, 1009, 1009, 1026, 1023, 1069] avg=1024.0000 max=1069 max_rank=7 min=1009 min_rank=3 imbalance_ratio=1.043945 max_load_reduction=7123 max_load_reduction_pct=0.869507 moved_to_echo=4127 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1029, 0, 0, 1023, 1, 1026, 1, 1026, 1026, 0, 0, 1023, 13, 1001] rerouted_tokens_per_ep_rank=[1023, 1029, 1023, 1027, 1027, 1026, 1023, 1014] avg=1024.0000 max=1029 max_rank=1 min=1014 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4090 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1029, 1019, 0, 0, 1025, 1, 1029, 1, 1029, 1012, 0, 0, 1029, 13, 1005] rerouted_tokens_per_ep_rank=[1029, 1019, 1025, 1030, 1030, 1012, 1029, 1018] avg=1024.0000 max=1030 max_rank=3 min=1012 min_rank=5 imbalance_ratio=1.005859 max_load_reduction=7162 max_load_reduction_pct=0.874268 moved_to_echo=4089 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1029, 0, 0, 1023, 1, 1024, 1, 1024, 1026, 0, 0, 1023, 18, 1000] rerouted_tokens_per_ep_rank=[1023, 1029, 1023, 1025, 1025, 1026, 1023, 1018] avg=1024.0000 max=1029 max_rank=1 min=1018 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4092 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1023, 0, 0, 1023, 1, 1020, 1, 1020, 1026, 0, 0, 1023, 37, 995] rerouted_tokens_per_ep_rank=[1023, 1023, 1023, 1021, 1021, 1026, 1023, 1032] avg=1024.0000 max=1032 max_rank=7 min=1021 min_rank=3 imbalance_ratio=1.007812 max_load_reduction=7160 max_load_reduction_pct=0.874023 moved_to_echo=4102 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 1026, 0, 0, 1023, 1, 1022, 1, 1022, 1026, 0, 0, 1023, 27, 998] rerouted_tokens_per_ep_rank=[1023, 1026, 1023, 1023, 1023, 1026, 1023, 1025] avg=1024.0000 max=1026 max_rank=1 min=1023 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8186, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8186, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8198 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8186, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8186, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8186, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8186, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8186 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8186, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8198, 8186, 8192, 8192, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8198 actual_valid_recv=8320 observed_loss=-122 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[256, 8064] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8186 actual_valid_recv=8320 observed_loss=-134 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 2, 2, 4092, 4094] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 1, 4094, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 1, 4094, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 1, 4094, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 1, 4094, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 1, 4094, 4095] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 1022, 0, 1022, 0, 1023, 0, 1023, 2, 1022, 2, 1022, 1026, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1022, 1023, 1023, 1024, 1024, 1026, 1026] avg=1024.0000 max=1026 max_rank=6 min=1022 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4100 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 1, 1023, 1025, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1024, 1024, 1025, 1026] avg=1024.0000 max=1026 max_rank=7 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 0, 1023, 1027, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1024, 1023, 1023, 1027, 1025] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1025, 0, 1025, 0, 1029, 0, 1029, 1, 1025, 1, 1025, 1019, 0, 1012, 0] rerouted_tokens_per_ep_rank=[1026, 1025, 1029, 1029, 1026, 1026, 1019, 1012] avg=1024.0000 max=1029 max_rank=2 min=1012 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=7163 max_load_reduction_pct=0.874390 moved_to_echo=4083 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1023, 0, 1023, 0, 1024, 0, 1024, 0, 1023, 0, 1023, 1027, 0, 1025, 0] rerouted_tokens_per_ep_rank=[1023, 1023, 1024, 1024, 1023, 1023, 1027, 1025] avg=1024.0000 max=1027 max_rank=6 min=1023 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=7165 max_load_reduction_pct=0.874634 moved_to_echo=4098 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 1, 1023, 1025, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1024, 1024, 1025, 1026] avg=1024.0000 max=1026 max_rank=7 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 1, 1023, 1025, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1024, 1024, 1025, 1026] avg=1024.0000 max=1026 max_rank=7 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1023, 0, 1023, 0, 1023, 0, 1023, 1, 1023, 1, 1023, 1025, 0, 1026, 0] rerouted_tokens_per_ep_rank=[1024, 1023, 1023, 1023, 1024, 1024, 1025, 1026] avg=1024.0000 max=1026 max_rank=7 min=1023 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=7166 max_load_reduction_pct=0.874756 moved_to_echo=4099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8185, 8192, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8185, 8192, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8185, 8192, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[8192, 8185, 8192, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8185, 8192, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8185, 8192, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[8192, 8185, 8192, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[8192, 8185, 8192, 8192, 8192, 8192, 8199, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 1373, 2723, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 1908, 2187, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 698, 3398, 0, 4096, 0] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 596, 3499, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 918, 3177, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 964, 3131, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[2, 0, 0, 1393, 2701, 0, 4094, 2] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 944, 3151, 0, 4095, 1] tokens_per_ep_rank=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] avg=8192.0000 max=8192 max_rank=0 min=8192 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 854, 0, 1023, 0, 1022, 1393, 0, 925, 0, 0, 1023, 1026, 0, 2, 922] rerouted_tokens_per_ep_rank=[856, 1023, 1022, 1393, 925, 1023, 1026, 924] avg=1024.0000 max=1393 max_rank=3 min=856 min_rank=0 imbalance_ratio=1.360352 max_load_reduction=6799 max_load_reduction_pct=0.829956 moved_to_echo=3898 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 861, 0, 1024, 0, 1023, 1373, 0, 933, 0, 0, 1024, 1025, 0, 0, 929] rerouted_tokens_per_ep_rank=[861, 1024, 1023, 1373, 933, 1024, 1025, 929] avg=1024.0000 max=1373 max_rank=3 min=861 min_rank=0 imbalance_ratio=1.340820 max_load_reduction=6819 max_load_reduction_pct=0.832397 moved_to_echo=3911 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1009, 0, 1029, 0, 1025, 918, 0, 1079, 0, 0, 1029, 1012, 0, 1, 1089] rerouted_tokens_per_ep_rank=[1010, 1029, 1025, 918, 1079, 1029, 1012, 1090] avg=1024.0000 max=1090 max_rank=7 min=918 min_rank=3 imbalance_ratio=1.064453 max_load_reduction=7102 max_load_reduction_pct=0.866943 moved_to_echo=4210 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1075, 0, 1024, 0, 1023, 698, 0, 1163, 0, 0, 1024, 1025, 0, 0, 1160] rerouted_tokens_per_ep_rank=[1075, 1024, 1023, 698, 1163, 1024, 1025, 1160] avg=1024.0000 max=1163 max_rank=4 min=698 min_rank=3 imbalance_ratio=1.135742 max_load_reduction=7029 max_load_reduction_pct=0.858032 moved_to_echo=4372 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 691, 0, 1023, 0, 1023, 1908, 0, 750, 0, 0, 1023, 1026, 0, 1, 746] rerouted_tokens_per_ep_rank=[692, 1023, 1023, 1908, 750, 1023, 1026, 747] avg=1024.0000 max=1908 max_rank=3 min=692 min_rank=0 imbalance_ratio=1.863281 max_load_reduction=6284 max_load_reduction_pct=0.767090 moved_to_echo=3546 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 990, 0, 1023, 0, 1023, 964, 0, 1072, 0, 0, 1023, 1026, 0, 1, 1069] rerouted_tokens_per_ep_rank=[991, 1023, 1023, 964, 1072, 1023, 1026, 1070] avg=1024.0000 max=1072 max_rank=4 min=964 min_rank=3 imbalance_ratio=1.046875 max_load_reduction=7120 max_load_reduction_pct=0.869141 moved_to_echo=4191 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1107, 0, 1023, 0, 1023, 596, 0, 1198, 0, 0, 1023, 1026, 0, 1, 1194] rerouted_tokens_per_ep_rank=[1108, 1023, 1023, 596, 1198, 1023, 1026, 1195] avg=1024.0000 max=1198 max_rank=4 min=596 min_rank=3 imbalance_ratio=1.169922 max_load_reduction=6994 max_load_reduction_pct=0.853760 moved_to_echo=4442 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 996, 0, 1023, 0, 1023, 944, 0, 1079, 0, 0, 1023, 1026, 0, 1, 1076] rerouted_tokens_per_ep_rank=[997, 1023, 1023, 944, 1079, 1023, 1026, 1077] avg=1024.0000 max=1079 max_rank=4 min=944 min_rank=3 imbalance_ratio=1.053711 max_load_reduction=7113 max_load_reduction_pct=0.868286 moved_to_echo=4205 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8794 expected_overflow=0 exact_recv_per_rank=[7590, 8192, 8185, 8794, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7590, 8192, 8185, 8794, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7590, 8192, 8185, 8794, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7590, 8192, 8185, 8794, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8199 expected_overflow=0 exact_recv_per_rank=[7590, 8192, 8185, 8794, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8192 expected_overflow=0 exact_recv_per_rank=[7590, 8192, 8185, 8794, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=2 budget=16384 expected_recv=8185 expected_overflow=0 exact_recv_per_rank=[7590, 8192, 8185, 8794, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=2 budget=16384 expected_recv=7590 expected_overflow=0 exact_recv_per_rank=[7590, 8192, 8185, 8794, 8199, 8192, 8192, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[8192, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=16384 expected_recv=8192 actual_valid_recv=8192 observed_loss=0 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=16384 expected_recv=8192 actual_valid_recv=8320 observed_loss=-128 tokens_per_expert=[128, 8192] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=16384 expected_recv=7590 actual_valid_recv=7808 observed_loss=-218 tokens_per_expert=[128, 7680] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=16384 expected_recv=8794 actual_valid_recv=8832 observed_loss=-38 tokens_per_expert=[8832, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=16384 expected_recv=8199 actual_valid_recv=8320 observed_loss=-121 tokens_per_expert=[8320, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=16384 expected_recv=8185 actual_valid_recv=8192 observed_loss=-7 tokens_per_expert=[0, 8192] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=16384 valid_num_tokens=7808 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=16384 valid_num_tokens=8832 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=16384 valid_num_tokens=8320 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=16384 valid_num_tokens=8192 slack_slots=8192 device_initiated_grouped_gemm=True + [2026-08-30 04:54:07] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 676.5 | throughput per GPU (TFLOP/s/GPU): 492.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.375618E+00 | loss scale: 1.0 | grad norm: 4.700 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 04:54:07 +[rank0]:[W830 04:54:09.371570568 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..2eb880ad4d89cdbc5afffdcda19c8840f44d8fc2 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1/mem_samples.csv @@ -0,0 +1,256 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 27 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2231 +1, 2231 +2, 2231 +3, 2231 +4, 2231 +5, 2231 +6, 2231 +7, 2231 +0, 39507 +1, 39503 +2, 39503 +3, 39503 +4, 39503 +5, 39503 +6, 39503 +7, 39503 +0, 39691 +1, 39685 +2, 39663 +3, 39691 +4, 39661 +5, 39687 +6, 39689 +7, 39687 +0, 42263 +1, 42255 +2, 42255 +3, 42255 +4, 42255 +5, 42255 +6, 42255 +7, 42255 +0, 42507 +1, 42499 +2, 42499 +3, 42499 +4, 42499 +5, 42499 +6, 42499 +7, 42499 +0, 42609 +1, 42609 +2, 42629 +3, 42615 +4, 42647 +5, 42639 +6, 42657 +7, 42667 +0, 43423 +1, 43415 +2, 43415 +3, 43415 +4, 43415 +5, 43415 +6, 43415 +7, 43415 +0, 43701 +1, 43693 +2, 43693 +3, 43693 +4, 43693 +5, 43693 +6, 43693 +7, 43693 +0, 44481 +1, 44473 +2, 44473 +3, 44473 +4, 44473 +5, 44473 +6, 44473 +7, 44473 +0, 45133 +1, 45125 +2, 45125 +3, 45125 +4, 45125 +5, 45125 +6, 45125 +7, 45125 +0, 75713 +1, 75705 +2, 75705 +3, 75705 +4, 75705 +5, 75705 +6, 75705 +7, 75705 +0, 75713 +1, 75705 +2, 75705 +3, 75705 +4, 75705 +5, 75705 +6, 75705 +7, 75705 +0, 76199 +1, 76191 +2, 76191 +3, 76191 +4, 76191 +5, 76191 +6, 76191 +7, 76191 +0, 76199 +1, 76191 +2, 76191 +3, 76191 +4, 76191 +5, 76191 +6, 76191 +7, 76191 +0, 76199 +1, 76191 +2, 76191 +3, 76191 +4, 76191 +5, 76191 +6, 76191 +7, 76191 +0, 93981 +1, 93983 +2, 94021 +3, 93983 +4, 94009 +5, 94027 +6, 94009 +7, 94047 +0, 94789 +1, 94793 +2, 94843 +3, 94855 +4, 94845 +5, 94829 +6, 94925 +7, 94859 +0, 95581 +1, 95573 +2, 95573 +3, 95573 +4, 95573 +5, 95573 +6, 95573 +7, 95573 +0, 96169 +1, 96179 +2, 96169 +3, 96157 +4, 96213 +5, 96183 +6, 96177 +7, 96175 +0, 119267 +1, 119213 +2, 119213 +3, 119213 +4, 119187 +5, 119213 +6, 119213 +7, 119213 +0, 119267 +1, 119213 +2, 119213 +3, 119213 +4, 119187 +5, 119213 +6, 119213 +7, 119213 +0, 119267 +1, 119213 +2, 119213 +3, 119213 +4, 119187 +5, 119213 +6, 119213 +7, 119213 +0, 119267 +1, 119213 +2, 119213 +3, 119213 +4, 119187 +5, 119213 +6, 119213 +7, 119213 +0, 1862 +1, 1862 +2, 1862 +3, 1862 +4, 1862 +5, 2118 +6, 118297 +7, 3142 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1/run.env b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..76ca12f295602ade3951a9ac693c98dba2c029fb --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1/run.env @@ -0,0 +1,70 @@ +backend=echo +config=G2 +family=tierA +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=3584 +num_experts=32 +topk=8 +experts_per_rank=4 +expert_params=44040192 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=32768 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=4 +capacity_mode=static +capacity=1.08203125 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1/train.log b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3aa02c1b9ba826f4f614a6cfaff385ff58376413 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1/train.log @@ -0,0 +1,16499 @@ +[04:55:36] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G2 / echo (slots/rank=4 policy=matched) + E=32 K=8 H'=3584 m=4 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.08203125 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0830 04:55:38.162000 448070 torch/distributed/run.py:766] +W0830 04:55:38.162000 448070 torch/distributed/run.py:766] ***************************************** +W0830 04:55:38.162000 448070 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 04:55:38.162000 448070 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 3584 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 32 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.08203125 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 32 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 215 is connected to 4 is connected to is connected to 7 is connected to 77 peer ranks. 7 peer ranks. peer ranks. Expected number of connected peer ranks is : [Gloo] Rank peer ranks. Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 7[Gloo] Rank Expected number of connected peer ranks is : 7 is connected to 7 +76 +7 + + is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 2 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 0 is connected to 47 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank7]:[W830 04:55:56.316327532 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 04:55:56.316421319 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 04:55:56.316474226 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 04:55:56.316475392 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 04:55:56.316496709 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 04:55:56.316577777 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 04:55:56.317709257 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 04:55:56.473945480 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.851 seconds +time to initialize megatron (seconds): 7.282 +[after megatron is initialized] datetime: 2026-08-30 04:56:02 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3903983616 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42086400 elements, 42086400 padded size): + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_proj.weight +Params for bucket 3 (42082304 elements, 42082304 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight +Params for bucket 4 (42082304 elements, 42082304 padded size): + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight +Params for bucket 5 (42082304 elements, 42082304 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight +Params for bucket 6 (42082304 elements, 42082304 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight +Params for bucket 7 (42082304 elements, 42082304 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.pre_mlp_layernorm.weight +Params for bucket 8 (42082304 elements, 42082304 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight +Params for bucket 9 (42082304 elements, 42082304 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.pre_mlp_layernorm.weight +Params for bucket 10 (42082304 elements, 42082304 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight +Params for bucket 11 (42082304 elements, 42082304 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight +Params for bucket 12 (42082304 elements, 42082304 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight +Params for bucket 13 (42082304 elements, 42082304 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.weight +Params for bucket 14 (42082304 elements, 42082304 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight +Params for bucket 15 (42082304 elements, 42082304 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 16 (42082304 elements, 42082304 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight +Params for bucket 17 (42082304 elements, 42082304 padded size): + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 56 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 3 (58720256 elements, 58720256 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 6 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 7 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 +Params for bucket 10 (58720256 elements, 58720256 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 +Params for bucket 13 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 +Params for bucket 14 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 20 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 24 (58720256 elements, 58720256 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 28 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 30 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 34 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 38 (58720256 elements, 58720256 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 +Params for bucket 41 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 42 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 43 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 45 (58720256 elements, 58720256 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 +Params for bucket 47 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 48 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 49 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 52 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 +Params for bucket 55 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 56 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 04:56:02 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 04:56:02 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (332.85, 336.34) + train/valid/test-data-iterators-setup ..........: (61.52, 74.03) +[before the start of training step] datetime: 2026-08-30 04:56:02 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[68, 3107, 586, 237, 1848, 671, 1188, 817, 1101, 230, 1708, 378, 943, 869, 235, 347, 96, 1022, 670, 673, 1135, 213, 3308, 2120, 686, 1233, 972, 1331, 989, 381, 1609, 1997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[218, 2853, 1229, 315, 1285, 586, 1176, 1212, 1297, 227, 1026, 209, 1055, 693, 255, 549, 336, 431, 1314, 1006, 1162, 140, 3062, 1749, 618, 833, 1355, 742, 1641, 724, 1676, 1794] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[161, 3125, 861, 327, 1307, 667, 761, 1183, 863, 309, 1057, 464, 1189, 1067, 167, 450, 107, 565, 1481, 807, 917, 160, 2701, 1817, 864, 937, 1123, 1509, 1533, 360, 1810, 2119] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[56, 2890, 926, 456, 1435, 545, 827, 921, 984, 196, 994, 572, 1216, 655, 382, 552, 165, 474, 1182, 927, 1110, 103, 2981, 2197, 537, 958, 853, 1381, 1331, 427, 2033, 2502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[155, 2518, 951, 567, 1540, 284, 1128, 857, 1362, 194, 1623, 771, 874, 802, 215, 522, 153, 875, 669, 1111, 1230, 163, 2921, 2084, 688, 917, 1021, 1023, 1291, 213, 1984, 2062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[103, 3109, 814, 315, 1699, 775, 1139, 1132, 1210, 235, 1121, 237, 703, 660, 298, 620, 203, 805, 769, 1140, 1035, 178, 3067, 1918, 633, 856, 1113, 901, 1441, 516, 1843, 2180] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[105, 3071, 689, 291, 1413, 531, 884, 1316, 1242, 381, 1171, 224, 1045, 1000, 383, 469, 166, 503, 413, 1356, 1180, 132, 2817, 1901, 362, 1161, 1225, 1243, 1564, 534, 1610, 2386] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[236, 3079, 692, 641, 1529, 646, 1133, 640, 1225, 164, 1989, 146, 1189, 1211, 243, 519, 146, 450, 1336, 749, 831, 379, 2957, 2064, 602, 876, 1021, 1195, 1384, 319, 1665, 1512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[218, 2608, 1229, 315, 0, 0, 0, 0, 1249, 586, 1176, 1212, 0, 0, 0, 0, 1297, 227, 1026, 209, 825, 38, 0, 0, 1055, 693, 255, 549, 1468, 0, 0, 0, 336, 431, 1314, 1006, 719, 555, 0, 0, 1162, 140, 875, 1749, 0, 0, 0, 0, 618, 833, 1355, 742, 207, 36, 0, 0, 1641, 724, 1676, 414, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4370, 4223, 3622, 4020, 4361, 3926, 3791, 4455] avg=4096.0000 max=4455 max_rank=7 min=3622 min_rank=2 imbalance_ratio=1.087646 max_load_reduction=28313 max_load_reduction_pct=0.864044 moved_to_echo=16533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[56, 2650, 926, 456, 0, 0, 0, 0, 1399, 545, 827, 921, 0, 0, 0, 0, 984, 196, 994, 572, 1143, 35, 0, 0, 1216, 655, 382, 552, 1425, 0, 0, 0, 165, 474, 1182, 927, 697, 770, 0, 0, 1110, 103, 859, 2197, 0, 0, 0, 0, 537, 958, 853, 1381, 205, 36, 0, 0, 1331, 427, 2033, 589, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 3692, 3924, 4230, 4215, 4269, 3970, 4380] avg=4096.0000 max=4380 max_rank=7 min=3692 min_rank=1 imbalance_ratio=1.069336 max_load_reduction=28388 max_load_reduction_pct=0.866333 moved_to_echo=16834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[161, 2865, 861, 327, 0, 0, 0, 0, 1274, 667, 761, 1183, 0, 0, 0, 0, 863, 309, 1057, 464, 968, 38, 0, 0, 1189, 1067, 167, 450, 1291, 0, 0, 0, 107, 565, 1481, 807, 632, 652, 0, 0, 917, 160, 778, 1817, 0, 0, 0, 0, 864, 937, 1123, 1509, 222, 33, 0, 0, 1533, 360, 1810, 499, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4214, 3885, 3699, 4164, 4244, 3672, 4688, 4202] avg=4096.0000 max=4688 max_rank=6 min=3672 min_rank=5 imbalance_ratio=1.144531 max_load_reduction=28080 max_load_reduction_pct=0.856934 moved_to_echo=16806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[105, 2816, 689, 291, 0, 0, 0, 0, 1378, 531, 884, 1316, 0, 0, 0, 0, 1242, 381, 1171, 224, 1090, 37, 0, 0, 1045, 1000, 383, 469, 1347, 0, 0, 0, 166, 503, 413, 1356, 659, 734, 0, 0, 1180, 132, 811, 1901, 0, 0, 0, 0, 362, 1161, 1225, 1243, 218, 35, 0, 0, 1564, 534, 1610, 562, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3901, 4109, 4145, 4244, 3831, 4024, 4244, 4270] avg=4096.0000 max=4270 max_rank=7 min=3831 min_rank=4 imbalance_ratio=1.042480 max_load_reduction=28498 max_load_reduction_pct=0.869690 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[155, 2309, 951, 567, 0, 0, 0, 0, 1501, 284, 1128, 857, 0, 0, 0, 0, 1362, 194, 1623, 771, 942, 30, 0, 0, 874, 802, 215, 522, 1397, 0, 0, 0, 153, 875, 669, 1111, 683, 634, 0, 0, 1230, 163, 841, 2084, 0, 0, 0, 0, 688, 917, 1021, 1023, 179, 39, 0, 0, 1291, 213, 1984, 486, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3982, 3770, 4922, 3810, 4125, 4318, 3867, 3974] avg=4096.0000 max=4922 max_rank=2 min=3770 min_rank=1 imbalance_ratio=1.201660 max_load_reduction=27846 max_load_reduction_pct=0.849792 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[236, 2823, 692, 641, 0, 0, 0, 0, 1491, 646, 1133, 640, 0, 0, 0, 0, 1225, 164, 1989, 146, 691, 37, 0, 0, 1189, 1211, 243, 519, 1414, 0, 0, 0, 146, 450, 1336, 749, 692, 465, 0, 0, 831, 379, 851, 2064, 0, 0, 0, 0, 602, 876, 1021, 1195, 219, 38, 0, 0, 1384, 319, 1665, 356, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4392, 3910, 4252, 4576, 3838, 4125, 3951, 3724] avg=4096.0000 max=4576 max_rank=3 min=3724 min_rank=7 imbalance_ratio=1.117188 max_load_reduction=28192 max_load_reduction_pct=0.860352 moved_to_echo=15638 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[103, 2850, 814, 315, 0, 0, 0, 0, 1656, 775, 1139, 1132, 0, 0, 0, 0, 1210, 235, 1121, 237, 996, 38, 0, 0, 703, 660, 298, 620, 1467, 0, 0, 0, 203, 805, 769, 1140, 717, 671, 0, 0, 1035, 178, 883, 1918, 0, 0, 0, 0, 633, 856, 1113, 901, 221, 43, 0, 0, 1441, 516, 1843, 513, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4702, 3837, 3748, 4305, 4014, 3767, 4313] avg=4096.0000 max=4702 max_rank=1 min=3748 min_rank=3 imbalance_ratio=1.147949 max_load_reduction=28066 max_load_reduction_pct=0.856506 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[68, 2848, 586, 237, 0, 0, 0, 0, 1801, 671, 1188, 817, 0, 0, 0, 0, 1101, 230, 1708, 378, 912, 38, 0, 0, 943, 869, 235, 347, 1582, 0, 0, 0, 96, 1022, 670, 673, 774, 614, 0, 0, 1135, 213, 952, 2120, 0, 0, 0, 0, 686, 1233, 972, 1331, 221, 47, 0, 0, 989, 381, 1609, 471, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3739, 4477, 4367, 3976, 3849, 4420, 4490, 3450] avg=4096.0000 max=4490 max_rank=6 min=3450 min_rank=7 imbalance_ratio=1.096191 max_load_reduction=28278 max_load_reduction_pct=0.862976 moved_to_echo=16209 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1408, 5248, 7936, 7808, 5632, 5120, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9344, 2048, 10752, 3072, 7680, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 21888, 6784, 3200, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11264, 3584, 14336, 3968, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 7040, 2304, 4096, 11392, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4992, 7808, 8704, 9344, 1792, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11776, 4736, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8704, 1536, 6912, 15872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[472, 0, 2299, 6, 2394, 814, 70, 660, 4, 1, 2854, 3389, 330, 3549, 6, 960, 307, 241, 0, 4026, 14, 97, 2, 591, 437, 24, 1168, 3840, 4085, 18, 2, 108] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[277, 1, 994, 138, 3207, 1886, 309, 1054, 960, 2, 58, 3294, 13, 3750, 16, 1566, 267, 1267, 0, 4019, 549, 202, 0, 0, 296, 3, 569, 3925, 4050, 28, 25, 43] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1864, 0, 3131, 128, 1206, 1076, 2916, 3834, 10, 0, 814, 805, 12, 660, 1, 20, 13, 355, 0, 4092, 23, 0, 1, 13, 3549, 48, 240, 3923, 4000, 0, 34, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[168, 0, 1813, 4, 1177, 3862, 879, 1933, 28, 0, 2062, 245, 78, 1919, 0, 3051, 1, 17, 73, 4083, 16, 52, 11, 2, 147, 5, 3018, 4085, 4028, 0, 0, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1026, 2, 1043, 1, 999, 555, 2560, 2357, 209, 0, 1718, 147, 22, 1855, 187, 1976, 455, 906, 0, 4096, 11, 0, 2, 280, 3602, 2, 639, 4096, 3952, 0, 59, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[228, 0, 1280, 234, 3217, 895, 367, 1568, 3, 0, 239, 2371, 5, 3033, 19, 571, 8, 2190, 0, 4062, 149, 0, 4, 0, 2712, 49, 1391, 4036, 4085, 2, 50, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[128, 10, 576, 1, 2016, 3821, 634, 1102, 469, 2, 1220, 1893, 239, 3281, 72, 1263, 23, 1003, 1, 4086, 69, 183, 189, 1, 657, 62, 1596, 4096, 4006, 0, 61, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2313, 26, 217, 262, 899, 705, 2897, 2573, 16, 20, 256, 1456, 12, 3299, 0, 1365, 88, 774, 11, 4057, 213, 17, 1, 187, 2646, 1, 298, 3947, 4070, 7, 135, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[472, 0, 2299, 6, 1198, 812, 0, 0, 0, 814, 70, 373, 0, 0, 0, 0, 4, 1, 2854, 3389, 287, 147, 60, 13, 330, 3489, 6, 960, 0, 0, 0, 0, 307, 241, 0, 3067, 0, 0, 0, 0, 14, 97, 2, 591, 2685, 1196, 0, 0, 437, 24, 1168, 1155, 0, 0, 0, 0, 4072, 18, 2, 108, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4787, 1257, 6755, 4785, 3615, 4585, 2784, 4200] avg=4096.0000 max=6755 max_rank=2 min=1257 min_rank=1 imbalance_ratio=1.649170 max_load_reduction=26013 max_load_reduction_pct=0.793854 moved_to_echo=15184 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[228, 0, 1280, 234, 1610, 819, 0, 0, 0, 895, 367, 887, 0, 0, 0, 0, 3, 0, 239, 2371, 681, 148, 51, 13, 5, 2982, 19, 571, 0, 0, 0, 0, 8, 2190, 0, 3095, 0, 0, 0, 0, 149, 0, 4, 0, 2822, 1607, 0, 0, 2712, 49, 1391, 1214, 0, 0, 0, 0, 4072, 2, 50, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 2149, 3506, 3577, 5293, 4582, 5366, 4124] avg=4096.0000 max=5366 max_rank=6 min=2149 min_rank=1 imbalance_ratio=1.310059 max_load_reduction=27402 max_load_reduction_pct=0.836243 moved_to_echo=19365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2313, 26, 217, 262, 449, 818, 0, 0, 0, 705, 2897, 1454, 0, 0, 0, 0, 16, 20, 256, 1456, 1119, 148, 55, 13, 12, 3244, 0, 1365, 0, 0, 0, 0, 88, 774, 11, 3091, 0, 0, 0, 0, 213, 17, 1, 187, 2759, 450, 0, 0, 2646, 1, 298, 1188, 0, 0, 0, 0, 4057, 7, 135, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 5056, 3083, 4621, 3964, 3627, 4133, 4199] avg=4096.0000 max=5056 max_rank=1 min=3083 min_rank=2 imbalance_ratio=1.234375 max_load_reduction=27712 max_load_reduction_pct=0.845703 moved_to_echo=15923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 2, 1043, 1, 500, 830, 0, 0, 0, 555, 2560, 1329, 0, 0, 0, 0, 209, 0, 1718, 147, 1028, 155, 35, 19, 22, 1820, 187, 1976, 0, 0, 0, 0, 455, 906, 0, 3111, 0, 0, 0, 0, 11, 0, 2, 280, 2866, 499, 0, 0, 3602, 2, 639, 1230, 0, 0, 0, 0, 3933, 0, 59, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3402, 4444, 3311, 4005, 4472, 3658, 5473, 4003] avg=4096.0000 max=5473 max_rank=6 min=3311 min_rank=2 imbalance_ratio=1.336182 max_load_reduction=27295 max_load_reduction_pct=0.832977 moved_to_echo=17606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[168, 0, 1813, 4, 590, 823, 0, 0, 0, 3862, 879, 1093, 0, 0, 0, 0, 28, 0, 2062, 245, 840, 149, 32, 13, 78, 1887, 0, 3051, 0, 0, 0, 0, 1, 17, 73, 3111, 0, 0, 0, 0, 16, 52, 11, 2, 2856, 587, 0, 0, 147, 5, 3018, 1229, 0, 0, 0, 0, 4015, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3398, 5834, 3369, 5016, 3202, 3524, 4399, 4026] avg=4096.0000 max=5834 max_rank=1 min=3202 min_rank=4 imbalance_ratio=1.424316 max_load_reduction=26934 max_load_reduction_pct=0.821960 moved_to_echo=15151 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1864, 0, 3131, 128, 604, 825, 0, 0, 0, 1076, 2916, 2167, 0, 0, 0, 0, 10, 0, 814, 805, 1667, 149, 11, 13, 12, 649, 1, 20, 0, 0, 0, 0, 13, 355, 0, 3118, 0, 0, 0, 0, 23, 0, 1, 13, 2743, 602, 0, 0, 3549, 48, 240, 1180, 0, 0, 0, 0, 3987, 0, 34, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6552, 6159, 3469, 682, 3486, 3382, 5017, 4021] avg=4096.0000 max=6552 max_rank=0 min=682 min_rank=3 imbalance_ratio=1.599609 max_load_reduction=26216 max_load_reduction_pct=0.800049 moved_to_echo=15906 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[277, 1, 994, 138, 1605, 810, 0, 0, 0, 1886, 309, 596, 0, 0, 0, 0, 960, 2, 58, 3294, 458, 147, 63, 13, 13, 3687, 16, 1566, 0, 0, 0, 0, 267, 1267, 0, 3062, 0, 0, 0, 0, 549, 202, 0, 0, 2744, 1602, 0, 0, 296, 3, 569, 1181, 0, 0, 0, 0, 4037, 28, 25, 43, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3825, 2791, 4995, 5282, 4596, 5097, 2049, 4133] avg=4096.0000 max=5282 max_rank=3 min=2049 min_rank=6 imbalance_ratio=1.289551 max_load_reduction=27486 max_load_reduction_pct=0.838806 moved_to_echo=15875 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[128, 10, 576, 1, 1009, 824, 0, 0, 0, 3821, 634, 623, 0, 0, 0, 0, 469, 2, 1220, 1893, 479, 149, 55, 13, 239, 3226, 72, 1263, 0, 0, 0, 0, 23, 1003, 1, 3113, 0, 0, 0, 0, 69, 183, 189, 1, 2864, 1007, 0, 0, 657, 62, 1596, 1232, 0, 0, 0, 0, 3993, 0, 61, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2548, 5078, 4280, 4800, 4140, 4313, 3547, 4062] avg=4096.0000 max=5078 max_rank=1 min=2548 min_rank=0 imbalance_ratio=1.239746 max_load_reduction=27690 max_load_reduction_pct=0.845032 moved_to_echo=16062 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 13696, 10752, 8576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14080, 256, 8960, 9728, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6528, 128, 11392, 896, 7680, 6656, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1792, 128, 9344, 13696, 6656, 1280, 384, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 20992, 384, 10880, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1152, 640, 256, 1152, 22400, 7552, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 6784, 128, 24832, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32256, 128, 384, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3535, 1539, 6, 4007, 10, 0, 4095, 156, 0, 1328, 40, 28, 583, 3285, 17, 98, 0, 3904, 3628, 331, 288, 2257, 255, 2756, 299, 3, 5, 0, 0, 0, 79, 236] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[10, 3652, 414, 889, 7, 807, 4096, 0, 0, 3786, 25, 112, 20, 1486, 0, 0, 4, 3932, 1286, 0, 69, 3204, 25, 986, 1194, 1, 0, 3, 3291, 611, 0, 2858] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1915, 2082, 12, 3192, 1, 0, 4078, 0, 0, 72, 0, 35, 2, 2615, 154, 108, 261, 3848, 3801, 389, 2, 3215, 274, 4058, 10, 0, 567, 0, 1415, 5, 356, 301] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[2474, 4, 0, 3798, 15, 8, 4089, 0, 0, 71, 44, 2, 0, 3799, 0, 885, 5, 4096, 2193, 11, 146, 0, 0, 3825, 66, 35, 6, 9, 3305, 80, 2, 3800] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1481, 1033, 47, 1894, 27, 2001, 2680, 0, 204, 3313, 8, 171, 3328, 3824, 0, 0, 543, 4069, 2999, 308, 0, 3902, 261, 319, 128, 0, 0, 1, 199, 28, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[765, 676, 3, 3690, 422, 1079, 3989, 24, 0, 4093, 3915, 0, 2795, 3968, 20, 1448, 5, 4095, 296, 101, 8, 0, 162, 516, 159, 0, 14, 119, 103, 135, 0, 168] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4083, 829, 0, 2714, 0, 0, 3593, 194, 1, 1119, 3, 0, 9, 537, 196, 504, 14, 2872, 3772, 0, 0, 1589, 0, 3878, 0, 2, 438, 11, 3894, 0, 31, 2485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[2995, 1336, 161, 1883, 0, 0, 4037, 1, 0, 3830, 91, 15, 2, 3997, 824, 0, 11, 3975, 716, 52, 814, 316, 0, 1532, 3494, 18, 35, 5, 2597, 10, 0, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4083, 829, 0, 459, 0, 0, 0, 0, 0, 0, 3284, 194, 0, 0, 0, 0, 1, 1119, 3, 0, 1286, 0, 0, 0, 9, 498, 196, 504, 0, 0, 0, 0, 14, 1124, 3772, 0, 0, 0, 0, 0, 0, 1589, 0, 3469, 0, 0, 0, 0, 0, 2, 438, 11, 1748, 909, 0, 0, 3894, 0, 31, 2485, 309, 409, 39, 60] rerouted_tokens_per_ep_rank=[5371, 3478, 2409, 1207, 4910, 5058, 3108, 7227] avg=4096.0000 max=7227 max_rank=7 min=1207 min_rank=3 imbalance_ratio=1.764404 max_load_reduction=25541 max_load_reduction_pct=0.779449 moved_to_echo=20303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3535, 1539, 6, 676, 0, 0, 0, 0, 10, 0, 3743, 156, 0, 0, 0, 0, 0, 1328, 40, 28, 1899, 0, 0, 0, 583, 3043, 17, 98, 0, 0, 0, 0, 0, 1527, 3628, 331, 0, 0, 0, 0, 288, 2257, 255, 2465, 0, 0, 0, 0, 299, 3, 5, 0, 2377, 1343, 0, 0, 0, 0, 79, 236, 352, 291, 242, 89] rerouted_tokens_per_ep_rank=[5756, 3909, 3295, 3741, 5486, 5265, 4027, 1289] avg=4096.0000 max=5756 max_rank=0 min=1289 min_rank=7 imbalance_ratio=1.405273 max_load_reduction=27012 max_load_reduction_pct=0.824341 moved_to_echo=16067 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2474, 4, 0, 641, 0, 0, 0, 0, 15, 8, 3737, 0, 0, 0, 0, 0, 0, 71, 44, 2, 1800, 0, 0, 0, 0, 3519, 0, 885, 0, 0, 0, 0, 5, 1602, 2193, 11, 0, 0, 0, 0, 146, 0, 0, 3421, 0, 0, 0, 0, 66, 35, 6, 9, 2494, 1273, 0, 0, 3305, 80, 2, 3800, 352, 404, 280, 84] rerouted_tokens_per_ep_rank=[3119, 3760, 1917, 4404, 3811, 3567, 3883, 8307] avg=4096.0000 max=8307 max_rank=7 min=1917 min_rank=2 imbalance_ratio=2.028076 max_load_reduction=24461 max_load_reduction_pct=0.746490 moved_to_echo=19568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1481, 1033, 47, 321, 0, 0, 0, 0, 27, 2001, 2450, 0, 0, 0, 0, 0, 204, 3313, 8, 171, 897, 0, 0, 0, 3328, 3542, 0, 0, 0, 0, 0, 0, 543, 1592, 2999, 308, 0, 0, 0, 0, 0, 3902, 261, 286, 0, 0, 0, 0, 128, 0, 0, 1, 2477, 634, 0, 0, 199, 28, 0, 0, 230, 33, 282, 42] rerouted_tokens_per_ep_rank=[2882, 4478, 4593, 6870, 5442, 4449, 3240, 814] avg=4096.0000 max=6870 max_rank=3 min=814 min_rank=7 imbalance_ratio=1.677246 max_load_reduction=25898 max_load_reduction_pct=0.790344 moved_to_echo=13945 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1915, 2082, 12, 539, 0, 0, 0, 0, 1, 0, 3727, 0, 0, 0, 0, 0, 0, 72, 0, 35, 1513, 0, 0, 0, 2, 2422, 154, 108, 0, 0, 0, 0, 261, 1505, 3801, 389, 0, 0, 0, 0, 2, 3215, 274, 3630, 0, 0, 0, 0, 10, 0, 567, 0, 2343, 1069, 0, 0, 1415, 5, 356, 301, 351, 428, 193, 71] rerouted_tokens_per_ep_rank=[4548, 3728, 1620, 2686, 5956, 7121, 3989, 3120] avg=4096.0000 max=7121 max_rank=5 min=1620 min_rank=2 imbalance_ratio=1.738525 max_load_reduction=25647 max_load_reduction_pct=0.782684 moved_to_echo=20186 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[765, 676, 3, 623, 0, 0, 0, 0, 422, 1079, 3646, 24, 0, 0, 0, 0, 0, 4093, 3915, 0, 1749, 0, 0, 0, 2795, 3676, 20, 1448, 0, 0, 0, 0, 5, 1602, 296, 101, 0, 0, 0, 0, 8, 0, 162, 462, 0, 0, 0, 0, 159, 0, 14, 119, 2493, 1236, 0, 0, 103, 135, 0, 168, 343, 54, 292, 82] rerouted_tokens_per_ep_rank=[2067, 5171, 9757, 7939, 2004, 632, 4021, 1177] avg=4096.0000 max=9757 max_rank=2 min=632 min_rank=5 imbalance_ratio=2.382080 max_load_reduction=23011 max_load_reduction_pct=0.702240 moved_to_echo=7834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[10, 3652, 414, 138, 0, 0, 0, 0, 7, 807, 3739, 0, 0, 0, 0, 0, 0, 3786, 25, 112, 426, 0, 0, 0, 20, 1373, 0, 0, 0, 0, 0, 0, 4, 1535, 1286, 0, 0, 0, 0, 0, 69, 3204, 25, 877, 0, 0, 0, 0, 1194, 1, 0, 3, 2397, 302, 0, 0, 3291, 611, 0, 2858, 357, 109, 113, 23] rerouted_tokens_per_ep_rank=[4214, 4553, 4349, 1393, 2825, 4175, 3897, 7362] avg=4096.0000 max=7362 max_rank=7 min=1393 min_rank=3 imbalance_ratio=1.797363 max_load_reduction=25406 max_load_reduction_pct=0.775330 moved_to_echo=18259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2995, 1336, 161, 319, 0, 0, 0, 0, 0, 0, 3690, 1, 0, 0, 0, 0, 0, 3830, 91, 15, 892, 0, 0, 0, 2, 3702, 824, 0, 0, 0, 0, 0, 11, 1555, 716, 52, 0, 0, 0, 0, 814, 316, 0, 1371, 0, 0, 0, 0, 3494, 18, 35, 5, 2420, 631, 0, 0, 2597, 10, 0, 21, 347, 161, 295, 41] rerouted_tokens_per_ep_rank=[4811, 3691, 4828, 4528, 2334, 2501, 6603, 3472] avg=4096.0000 max=6603 max_rank=6 min=2334 min_rank=4 imbalance_ratio=1.612061 max_load_reduction=26165 max_load_reduction_pct=0.798492 moved_to_echo=14910 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14848, 896, 512, 9984, 2688, 1920, 1792, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6784, 21888, 1280, 3072, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[17280, 11264, 768, 3840, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 17664, 4224, 384, 10496, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 12160, 18816, 1280, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1408, 14592, 1024, 16000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5376, 128, 1152, 256, 18816, 7424, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[512, 3968, 28032, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[679, 0, 0, 4094, 2017, 1458, 6, 0, 78, 3366, 0, 0, 4093, 4, 0, 1728, 0, 3835, 0, 1841, 7, 42, 206, 4093, 0, 44, 680, 4049, 397, 1, 0, 50] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3977, 270, 11, 4070, 21, 42, 0, 0, 821, 3165, 0, 26, 3970, 0, 2797, 450, 0, 2613, 192, 2, 42, 0, 0, 4047, 2, 2316, 0, 51, 11, 17, 1, 3854] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[3181, 0, 1, 3979, 7, 2867, 514, 4, 0, 4092, 70, 93, 4092, 1, 92, 4, 0, 17, 2686, 4079, 983, 0, 0, 578, 0, 17, 100, 898, 121, 24, 182, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4087, 0, 0, 4076, 144, 174, 85, 0, 0, 134, 3, 11, 4096, 0, 30, 151, 59, 3750, 4076, 1, 97, 0, 0, 3781, 0, 3673, 2, 4085, 18, 0, 54, 181] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2407, 42, 183, 2654, 445, 4071, 1, 0, 15, 2696, 14, 73, 432, 0, 759, 63, 2, 1811, 2942, 1598, 0, 0, 0, 3314, 1, 3, 0, 1346, 2166, 4, 1636, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[84, 4095, 108, 3844, 1700, 3243, 4049, 0, 1, 1091, 0, 7, 3288, 44, 907, 679, 166, 3806, 4, 725, 0, 0, 0, 2530, 0, 0, 0, 65, 1400, 8, 158, 766] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4072, 0, 0, 1, 113, 0, 2550, 0, 2222, 3478, 6, 18, 4088, 1553, 1286, 0, 0, 4086, 0, 3191, 0, 0, 0, 2614, 123, 718, 126, 411, 0, 0, 0, 2112] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[3991, 70, 1, 2224, 2712, 342, 0, 650, 58, 2589, 62, 0, 4080, 4066, 30, 2282, 129, 28, 4, 553, 0, 0, 6, 4013, 0, 153, 8, 4064, 0, 653, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 0, 901, 0, 0, 0, 0, 144, 174, 85, 0, 1044, 0, 0, 0, 0, 134, 3, 11, 1248, 0, 0, 0, 2308, 0, 30, 151, 0, 0, 0, 0, 59, 1978, 4076, 1, 0, 0, 0, 0, 97, 0, 0, 3781, 728, 369, 0, 0, 0, 3673, 2, 4085, 1414, 171, 0, 0, 18, 0, 54, 181, 1761, 0, 0, 0] rerouted_tokens_per_ep_rank=[4988, 1447, 1396, 2489, 6114, 4975, 9345, 2014] avg=4096.0000 max=9345 max_rank=6 min=1396 min_rank=2 imbalance_ratio=2.281494 max_load_reduction=23423 max_load_reduction_pct=0.714813 moved_to_echo=22448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3977, 270, 11, 900, 0, 0, 0, 0, 21, 42, 0, 0, 727, 0, 0, 0, 821, 3165, 0, 26, 1210, 0, 0, 0, 2236, 0, 2797, 450, 0, 0, 0, 0, 0, 1379, 192, 2, 0, 0, 0, 0, 42, 0, 0, 4047, 507, 358, 0, 0, 2, 2316, 0, 51, 1412, 166, 0, 0, 11, 17, 1, 3854, 1758, 0, 0, 0] rerouted_tokens_per_ep_rank=[5158, 790, 5222, 5483, 1573, 4954, 3947, 5641] avg=4096.0000 max=5641 max_rank=7 min=790 min_rank=1 imbalance_ratio=1.377197 max_load_reduction=27127 max_load_reduction_pct=0.827850 moved_to_echo=16115 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4072, 0, 0, 1, 0, 0, 0, 0, 113, 0, 2550, 0, 1137, 0, 0, 0, 2222, 3478, 6, 18, 1246, 0, 0, 0, 2302, 1553, 1286, 0, 0, 0, 0, 0, 0, 2156, 0, 3191, 0, 0, 0, 0, 0, 0, 0, 2614, 793, 369, 0, 0, 123, 718, 126, 411, 0, 171, 0, 0, 0, 0, 0, 2112, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 3800, 6970, 5141, 5347, 3776, 1549, 2112] avg=4096.0000 max=6970 max_rank=2 min=1549 min_rank=6 imbalance_ratio=1.701660 max_load_reduction=25798 max_load_reduction_pct=0.787292 moved_to_echo=12784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[84, 4095, 108, 850, 0, 0, 0, 0, 1700, 3243, 4049, 0, 1059, 0, 0, 0, 1, 1091, 0, 7, 1002, 0, 0, 0, 1853, 44, 907, 679, 0, 0, 0, 0, 166, 2008, 4, 725, 0, 0, 0, 0, 0, 0, 0, 2530, 739, 296, 0, 0, 0, 0, 0, 65, 1333, 137, 0, 0, 1400, 8, 158, 766, 1661, 0, 0, 0] rerouted_tokens_per_ep_rank=[5137, 10051, 2101, 3483, 2903, 3565, 1535, 3993] avg=4096.0000 max=10051 max_rank=1 min=1535 min_rank=6 imbalance_ratio=2.453857 max_load_reduction=22717 max_load_reduction_pct=0.693268 moved_to_echo=11996 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3181, 0, 1, 872, 0, 0, 0, 0, 7, 2867, 514, 4, 8, 0, 0, 0, 0, 4092, 70, 93, 1252, 0, 0, 0, 2293, 1, 92, 4, 0, 0, 0, 0, 0, 2, 2686, 4079, 0, 0, 0, 0, 983, 0, 0, 578, 7, 373, 0, 0, 0, 17, 100, 898, 1385, 174, 0, 0, 121, 24, 182, 4086, 1722, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 3400, 5507, 2390, 6767, 1941, 2574, 6135] avg=4096.0000 max=6767 max_rank=4 min=1941 min_rank=5 imbalance_ratio=1.652100 max_load_reduction=26001 max_load_reduction_pct=0.793488 moved_to_echo=17417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[679, 0, 0, 905, 0, 0, 0, 0, 2017, 1458, 6, 0, 1067, 0, 0, 0, 78, 3366, 0, 0, 1247, 0, 0, 0, 2306, 4, 0, 1728, 0, 0, 0, 0, 0, 2023, 0, 1841, 0, 0, 0, 0, 7, 42, 206, 4093, 745, 369, 0, 0, 0, 44, 680, 4049, 1420, 171, 0, 0, 397, 1, 0, 50, 1769, 0, 0, 0] rerouted_tokens_per_ep_rank=[1584, 4548, 4691, 4038, 3864, 5462, 6364, 2217] avg=4096.0000 max=6364 max_rank=6 min=1584 min_rank=0 imbalance_ratio=1.553711 max_load_reduction=26404 max_load_reduction_pct=0.805786 moved_to_echo=17907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2407, 42, 183, 588, 0, 0, 0, 0, 445, 4071, 1, 0, 504, 0, 0, 0, 15, 2696, 14, 73, 131, 0, 0, 0, 245, 0, 759, 63, 0, 0, 0, 0, 2, 956, 2942, 1598, 0, 0, 0, 0, 0, 0, 0, 3314, 351, 38, 0, 0, 1, 3, 0, 1346, 920, 18, 0, 0, 2166, 4, 1636, 4090, 1146, 0, 0, 0] rerouted_tokens_per_ep_rank=[3220, 5021, 2929, 1067, 5498, 3703, 2288, 9042] avg=4096.0000 max=9042 max_rank=7 min=1067 min_rank=3 imbalance_ratio=2.207520 max_load_reduction=23726 max_load_reduction_pct=0.724060 moved_to_echo=20531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3991, 70, 1, 492, 0, 0, 0, 0, 2712, 342, 0, 650, 7, 0, 0, 0, 58, 2589, 62, 0, 1243, 0, 0, 0, 2299, 4066, 30, 2282, 0, 0, 0, 0, 129, 16, 4, 553, 0, 0, 0, 0, 0, 0, 6, 4013, 5, 368, 0, 0, 0, 153, 8, 4064, 771, 170, 0, 0, 0, 653, 0, 0, 961, 0, 0, 0] rerouted_tokens_per_ep_rank=[4554, 3711, 3952, 8677, 702, 4392, 5166, 1614] avg=4096.0000 max=8677 max_rank=3 min=702 min_rank=4 imbalance_ratio=2.118408 max_load_reduction=24091 max_load_reduction_pct=0.735199 moved_to_echo=11874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 10624, 9984, 12032, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7168, 12288, 7296, 768, 5632, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 768, 2048, 15232, 10880, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15872, 5760, 6016, 5376, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[22528, 4480, 384, 5632, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7040, 1024, 14976, 8704, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3200, 20736, 256, 256, 8704, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1152, 128, 256, 25088, 3968, 2560, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[11, 490, 731, 19, 3036, 2, 13, 3628, 625, 0, 2843, 3139, 4073, 0, 4020, 261, 10, 0, 0, 916, 0, 878, 0, 2541, 407, 12, 1484, 176, 25, 1014, 2413, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1394, 2, 4043, 471, 1250, 57, 2576, 4096, 68, 319, 10, 3963, 0, 229, 3902, 0, 124, 0, 3999, 36, 663, 161, 54, 12, 11, 12, 7, 3, 0, 1314, 3992, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 774, 8, 28, 138, 1336, 3290, 2, 35, 21, 3917, 1872, 6, 83, 1, 0, 0, 3320, 8, 2, 1079, 876, 2833, 98, 4096, 4067, 0, 11, 234, 1131, 3502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[3909, 0, 3253, 7, 14, 0, 277, 3968, 4096, 140, 779, 3483, 0, 140, 165, 0, 0, 0, 140, 170, 4096, 0, 6, 20, 18, 450, 3714, 0, 1, 0, 3919, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 66, 5, 39, 78, 2116, 2717, 3, 3646, 4037, 599, 0, 0, 2190, 4093, 51, 0, 0, 140, 1, 0, 4022, 1, 5, 436, 0, 0, 176, 4088, 163] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[413, 1042, 794, 9, 1481, 0, 0, 1032, 0, 6, 3167, 3520, 4095, 377, 13, 3533, 11, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 3772, 0, 0, 167, 2928, 2548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3556, 0, 3081, 56, 2500, 0, 0, 540, 844, 136, 4082, 4011, 164, 4076, 9, 0, 2201, 4, 46, 124, 0, 1045, 0, 451, 0, 4, 0, 0, 0, 1748, 4090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4019, 162, 3956, 27, 823, 0, 0, 4096, 3923, 0, 1813, 3223, 136, 1548, 0, 0, 94, 0, 3985, 127, 0, 78, 1, 207, 21, 457, 68, 0, 25, 0, 3973, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1105, 2, 4043, 471, 0, 0, 0, 0, 1250, 57, 2576, 3440, 0, 0, 0, 0, 68, 319, 10, 474, 0, 0, 0, 0, 0, 229, 3902, 0, 102, 0, 0, 0, 124, 0, 3999, 36, 1670, 144, 0, 0, 663, 161, 54, 12, 563, 656, 187, 0, 11, 12, 7, 3, 1819, 0, 0, 0, 0, 1314, 3285, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5621, 7323, 871, 4233, 5973, 2296, 1852, 4599] avg=4096.0000 max=7323 max_rank=1 min=871 min_rank=2 imbalance_ratio=1.787842 max_load_reduction=25445 max_load_reduction_pct=0.776520 moved_to_echo=14720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2817, 0, 3081, 56, 0, 0, 0, 0, 2500, 0, 0, 454, 0, 0, 0, 0, 844, 136, 4082, 479, 0, 0, 0, 0, 164, 4076, 9, 0, 260, 0, 0, 0, 2201, 4, 46, 124, 1691, 147, 0, 0, 0, 1045, 0, 451, 577, 86, 479, 0, 0, 4, 0, 0, 1841, 0, 0, 0, 0, 1748, 3366, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5954, 2954, 5541, 4509, 4213, 2638, 1845, 5114] avg=4096.0000 max=5954 max_rank=0 min=1845 min_rank=6 imbalance_ratio=1.453613 max_load_reduction=26814 max_load_reduction_pct=0.818298 moved_to_echo=13810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 774, 8, 0, 0, 0, 0, 28, 138, 1336, 2763, 0, 0, 0, 0, 2, 35, 21, 468, 0, 0, 0, 0, 1872, 6, 83, 1, 0, 0, 0, 0, 0, 0, 3320, 8, 1651, 40, 0, 0, 2, 1079, 876, 2833, 159, 527, 0, 0, 98, 4096, 4067, 0, 1798, 0, 0, 0, 11, 234, 932, 3502, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[782, 4265, 526, 1962, 5019, 5476, 10059, 4679] avg=4096.0000 max=10059 max_rank=6 min=526 min_rank=2 imbalance_ratio=2.455811 max_load_reduction=22709 max_load_reduction_pct=0.693024 moved_to_echo=25233 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3097, 0, 3253, 7, 0, 0, 0, 0, 14, 0, 277, 3332, 0, 0, 0, 0, 4096, 140, 779, 416, 0, 0, 0, 0, 0, 140, 165, 0, 286, 0, 0, 0, 0, 0, 140, 170, 1468, 141, 0, 0, 4096, 0, 6, 20, 553, 636, 526, 0, 18, 450, 3714, 0, 1599, 0, 0, 0, 1, 0, 3225, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6357, 3623, 5431, 591, 1919, 5837, 5781, 3229] avg=4096.0000 max=6357 max_rank=0 min=591 min_rank=3 imbalance_ratio=1.552002 max_load_reduction=26411 max_load_reduction_pct=0.806000 moved_to_echo=16766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[10, 490, 731, 19, 0, 0, 0, 0, 3036, 2, 13, 3047, 0, 0, 0, 0, 625, 0, 2843, 375, 0, 0, 0, 0, 4073, 0, 4020, 261, 0, 0, 0, 0, 10, 0, 0, 916, 1323, 87, 0, 0, 0, 878, 0, 2541, 340, 581, 1, 0, 407, 12, 1484, 176, 1441, 0, 0, 0, 25, 1014, 1986, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1250, 6098, 3843, 8354, 2336, 4341, 3520, 3026] avg=4096.0000 max=8354 max_rank=3 min=1250 min_rank=0 imbalance_ratio=2.039551 max_load_reduction=24414 max_load_reduction_pct=0.745056 moved_to_echo=13223 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[320, 1042, 794, 9, 0, 0, 0, 0, 1481, 0, 0, 862, 0, 0, 0, 0, 0, 6, 3167, 413, 0, 0, 0, 0, 4095, 377, 13, 3533, 34, 0, 0, 0, 11, 0, 0, 0, 1488, 109, 0, 0, 0, 0, 3860, 0, 417, 170, 59, 0, 0, 0, 3772, 0, 1619, 0, 0, 0, 0, 167, 2402, 2548, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2165, 2343, 3586, 8052, 1608, 4506, 5391, 5117] avg=4096.0000 max=8052 max_rank=3 min=1608 min_rank=4 imbalance_ratio=1.965820 max_load_reduction=24716 max_load_reduction_pct=0.754272 moved_to_echo=16622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3184, 162, 3956, 27, 0, 0, 0, 0, 823, 0, 0, 3440, 0, 0, 0, 0, 3923, 0, 1813, 386, 0, 0, 0, 0, 136, 1548, 0, 0, 294, 0, 0, 0, 94, 0, 3985, 127, 1358, 143, 0, 0, 0, 78, 1, 207, 560, 656, 541, 0, 21, 457, 68, 0, 1479, 0, 0, 0, 25, 0, 3270, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7329, 4263, 6122, 1978, 5707, 2043, 2025, 3301] avg=4096.0000 max=7329 max_rank=0 min=1978 min_rank=3 imbalance_ratio=1.789307 max_load_reduction=25439 max_load_reduction_pct=0.776337 moved_to_echo=13076 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3244, 0, 0, 66, 0, 0, 0, 0, 5, 39, 78, 1777, 0, 0, 0, 0, 2717, 3, 3646, 482, 0, 0, 0, 0, 599, 0, 0, 2190, 300, 0, 0, 0, 4093, 51, 0, 0, 1702, 147, 0, 0, 140, 1, 0, 4022, 577, 339, 552, 0, 1, 5, 436, 0, 1853, 0, 0, 0, 0, 176, 3364, 163, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3310, 1899, 6848, 3089, 5993, 5631, 2295, 3703] avg=4096.0000 max=6848 max_rank=2 min=1899 min_rank=1 imbalance_ratio=1.671875 max_load_reduction=25920 max_load_reduction_pct=0.791016 moved_to_echo=17622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 5120, 13568, 256, 13568, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13824, 1792, 16640, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[11008, 6400, 8192, 6016, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6656, 128, 11520, 1408, 12416, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4992, 3328, 4864, 10112, 3840, 3712, 2432, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[12288, 640, 16384, 3584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9216, 256, 4352, 19200, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 4736, 21888, 6272, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[829, 0, 4076, 63, 104, 0, 0, 9, 1326, 0, 2653, 3482, 0, 3711, 1045, 732, 0, 0, 0, 3936, 475, 0, 0, 0, 892, 4027, 1, 2480, 2901, 0, 5, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[506, 11, 0, 1, 0, 0, 3604, 518, 3402, 220, 3979, 4096, 0, 2823, 4085, 2, 0, 16, 62, 0, 590, 441, 218, 1600, 1870, 67, 3616, 827, 1, 213, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 3, 10, 65, 0, 0, 3255, 4071, 0, 0, 3644, 4096, 0, 1639, 3, 0, 841, 0, 0, 0, 381, 0, 0, 4, 58, 2792, 1, 4096, 3731, 4071, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[5, 0, 4091, 0, 458, 0, 419, 4, 0, 1234, 0, 3847, 63, 4081, 3373, 3431, 6, 20, 1871, 0, 0, 96, 1, 0, 0, 4092, 12, 3920, 0, 1479, 53, 212] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[3647, 0, 0, 0, 2, 111, 2094, 0, 0, 0, 85, 4090, 26, 3989, 2813, 0, 5, 0, 0, 0, 276, 9, 1, 0, 0, 3528, 1327, 4003, 4056, 1, 0, 2705] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 209, 6, 0, 1626, 32, 3054, 1695, 61, 4091, 2963, 0, 3876, 2492, 4067, 0, 0, 0, 0, 0, 8, 0, 156, 0, 4094, 2, 782, 1460, 2094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[29, 0, 564, 4, 352, 25, 1747, 0, 0, 4079, 15, 3173, 0, 3220, 3215, 2426, 0, 0, 139, 1483, 3167, 30, 0, 909, 0, 4094, 11, 3896, 187, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[167, 0, 0, 0, 0, 892, 3894, 5, 14, 207, 10, 4006, 1, 4077, 4003, 46, 3662, 3813, 77, 0, 448, 89, 4, 0, 3, 3050, 59, 3718, 0, 160, 1, 362] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3647, 0, 0, 0, 2175, 379, 0, 0, 2, 111, 2094, 0, 892, 0, 0, 0, 0, 0, 85, 833, 0, 0, 0, 0, 26, 138, 2813, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1188, 1122, 0, 0, 276, 9, 1, 0, 3472, 0, 0, 0, 0, 165, 1327, 4003, 0, 0, 0, 0, 4056, 1, 0, 2705, 1243, 0, 0, 0] rerouted_tokens_per_ep_rank=[6201, 3099, 918, 2977, 2315, 3758, 5495, 8005] avg=4096.0000 max=8005 max_rank=7 min=918 min_rank=2 imbalance_ratio=1.954346 max_load_reduction=24763 max_load_reduction_pct=0.755707 moved_to_echo=19573 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[167, 0, 0, 0, 1880, 387, 0, 0, 0, 892, 3894, 5, 874, 0, 0, 0, 14, 207, 10, 816, 0, 0, 0, 0, 1, 142, 4003, 46, 0, 0, 0, 0, 3662, 3813, 77, 0, 1027, 1099, 0, 0, 448, 89, 4, 0, 3548, 0, 0, 0, 3, 143, 59, 3718, 0, 0, 0, 0, 0, 160, 1, 362, 1217, 0, 0, 0] rerouted_tokens_per_ep_rank=[2434, 5665, 1047, 4192, 9678, 4089, 3923, 1740] avg=4096.0000 max=9678 max_rank=4 min=1047 min_rank=2 imbalance_ratio=2.362793 max_load_reduction=23090 max_load_reduction_pct=0.704651 moved_to_echo=19430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[829, 0, 4076, 63, 2483, 352, 0, 0, 104, 0, 0, 9, 759, 0, 0, 0, 1326, 0, 2653, 710, 0, 0, 0, 0, 0, 129, 1045, 732, 0, 0, 0, 0, 0, 0, 0, 3936, 1356, 955, 0, 0, 475, 0, 0, 0, 3230, 0, 0, 0, 892, 188, 1, 2480, 0, 0, 0, 0, 2901, 0, 5, 21, 1058, 0, 0, 0] rerouted_tokens_per_ep_rank=[7803, 872, 4689, 1906, 6247, 3705, 3561, 3985] avg=4096.0000 max=7803 max_rank=0 min=872 min_rank=1 imbalance_ratio=1.905029 max_load_reduction=24965 max_load_reduction_pct=0.761871 moved_to_echo=17498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 4091, 0, 2523, 387, 0, 0, 458, 0, 419, 4, 839, 0, 0, 0, 0, 1234, 0, 784, 0, 0, 0, 0, 63, 142, 3373, 3431, 0, 0, 0, 0, 6, 20, 1871, 0, 1378, 1055, 0, 0, 0, 96, 1, 0, 3552, 0, 0, 0, 0, 191, 12, 3920, 0, 0, 0, 0, 0, 1479, 53, 212, 1169, 0, 0, 0] rerouted_tokens_per_ep_rank=[7006, 1720, 2018, 7009, 4330, 3649, 4123, 2913] avg=4096.0000 max=7009 max_rank=3 min=1720 min_rank=1 imbalance_ratio=1.711182 max_load_reduction=25759 max_load_reduction_pct=0.786102 moved_to_echo=15015 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 209, 6, 2524, 368, 0, 0, 0, 1626, 32, 3054, 646, 0, 0, 0, 1695, 61, 4091, 605, 0, 0, 0, 0, 0, 135, 2492, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 1379, 812, 0, 0, 0, 8, 0, 156, 3373, 0, 0, 0, 0, 191, 2, 782, 0, 0, 0, 0, 1460, 2094, 0, 0, 900, 0, 0, 0] rerouted_tokens_per_ep_rank=[3107, 5358, 6452, 6694, 2191, 3537, 975, 4454] avg=4096.0000 max=6694 max_rank=3 min=975 min_rank=6 imbalance_ratio=1.634277 max_load_reduction=26074 max_load_reduction_pct=0.795715 moved_to_echo=11157 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 10, 65, 1726, 159, 0, 0, 0, 0, 3255, 4071, 897, 0, 0, 0, 0, 0, 3644, 823, 0, 0, 0, 0, 0, 49, 3, 0, 0, 0, 0, 0, 841, 0, 0, 0, 945, 1127, 0, 0, 381, 0, 0, 4, 1431, 0, 0, 0, 58, 121, 1, 4096, 0, 0, 0, 0, 3731, 4071, 7, 0, 1249, 0, 0, 0] rerouted_tokens_per_ep_rank=[1963, 8223, 4467, 52, 2913, 1816, 4276, 9058] avg=4096.0000 max=9058 max_rank=7 min=52 min_rank=3 imbalance_ratio=2.211426 max_load_reduction=23710 max_load_reduction_pct=0.723572 moved_to_echo=18063 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[506, 11, 0, 1, 41, 268, 0, 0, 0, 0, 3604, 518, 893, 0, 0, 0, 3402, 220, 3979, 835, 0, 0, 0, 0, 0, 98, 4085, 2, 0, 0, 0, 0, 0, 16, 62, 0, 22, 1123, 0, 0, 590, 441, 218, 1600, 2457, 0, 0, 0, 1870, 4, 3616, 827, 0, 0, 0, 0, 1, 213, 0, 0, 1245, 0, 0, 0] rerouted_tokens_per_ep_rank=[827, 5015, 8436, 4185, 1223, 5306, 6317, 1459] avg=4096.0000 max=8436 max_rank=2 min=827 min_rank=0 imbalance_ratio=2.059570 max_load_reduction=24332 max_load_reduction_pct=0.742554 moved_to_echo=14305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 564, 4, 2524, 306, 0, 0, 352, 25, 1747, 0, 692, 0, 0, 0, 0, 4079, 15, 647, 0, 0, 0, 0, 0, 112, 3215, 2426, 0, 0, 0, 0, 0, 0, 139, 1483, 1379, 870, 0, 0, 3167, 30, 0, 909, 2802, 0, 0, 0, 0, 191, 11, 3896, 0, 0, 0, 0, 187, 0, 0, 3, 964, 0, 0, 0] rerouted_tokens_per_ep_rank=[3427, 2816, 4741, 5753, 3871, 6908, 4098, 1154] avg=4096.0000 max=6908 max_rank=5 min=1154 min_rank=7 imbalance_ratio=1.686523 max_load_reduction=25860 max_load_reduction_pct=0.789185 moved_to_echo=16031 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12416, 8064, 128, 3328, 9088, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5248, 128, 8960, 256, 16000, 2688, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4608, 3968, 2176, 5504, 8704, 8192, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6528, 5888, 14592, 6144, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1024, 2688, 15104, 7680, 6528, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 1024, 21120, 10752, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2944, 1280, 5120, 23808, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5376, 768, 256, 2688, 23936, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 2, 2523, 3327, 233, 0, 1207, 3356, 4060, 668, 65, 1509, 1408, 0, 15, 487, 3734, 4080, 0, 0, 0, 0, 43, 3713, 2090, 0, 0, 246, 0, 1, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 3, 72, 17, 0, 0, 0, 4096, 1000, 4088, 4095, 3404, 1883, 0, 3213, 0, 1858, 3558, 0, 72, 1, 0, 0, 2931, 391, 0, 2, 70, 0, 4, 23, 1987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 1101, 14, 667, 13, 1460, 1940, 3061, 0, 38, 3440, 29, 0, 503, 2091, 3852, 22, 307, 17, 156, 1, 0, 4045, 1, 0, 0, 3487, 3794, 2724, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 10, 354, 0, 3037, 111, 4096, 0, 3188, 0, 7, 3924, 4089, 0, 542, 0, 9, 1, 1310, 0, 21, 2, 2, 4067, 0, 0, 0, 4095, 29, 600, 0, 3274] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1023, 0, 0, 4096, 3, 0, 0, 4082, 3825, 5, 4084, 814, 3, 0, 3764, 0, 3664, 3014, 25, 0, 0, 0, 74, 63, 8, 0, 49, 3985, 176, 0, 11, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3850, 0, 0, 3995, 3263, 52, 125, 4, 3796, 1388, 4, 159, 0, 0, 894, 1, 4069, 1706, 0, 0, 309, 0, 48, 753, 340, 0, 0, 121, 13, 4081, 3797, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[42, 0, 622, 972, 0, 878, 4091, 4048, 827, 19, 3767, 754, 0, 286, 0, 3633, 3747, 2224, 2097, 433, 5, 0, 57, 119, 0, 0, 0, 3849, 0, 0, 298, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[16, 0, 0, 3866, 1958, 1, 1, 802, 23, 3917, 140, 3636, 1021, 1, 430, 3949, 4060, 231, 4095, 0, 1, 0, 283, 0, 98, 0, 2, 3484, 0, 719, 34, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1023, 0, 0, 4096, 213, 496, 0, 0, 3, 0, 0, 2578, 0, 0, 0, 0, 0, 5, 4084, 485, 0, 0, 0, 0, 3, 0, 3764, 0, 1008, 0, 0, 0, 1405, 3014, 25, 0, 0, 0, 0, 0, 0, 0, 74, 63, 3109, 0, 0, 0, 8, 0, 49, 3985, 1161, 116, 0, 0, 176, 0, 11, 0, 1098, 716, 0, 0] rerouted_tokens_per_ep_rank=[5828, 2581, 4574, 4775, 4444, 3246, 5319, 2001] avg=4096.0000 max=5828 max_rank=0 min=2001 min_rank=7 imbalance_ratio=1.422852 max_load_reduction=26940 max_load_reduction_pct=0.822144 moved_to_echo=15010 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 2523, 3327, 396, 408, 0, 0, 233, 0, 1207, 2120, 0, 0, 0, 0, 0, 668, 65, 898, 0, 0, 0, 0, 1408, 0, 15, 487, 828, 0, 0, 0, 1431, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 43, 3713, 3300, 0, 0, 0, 2090, 0, 0, 246, 1184, 215, 0, 0, 0, 1, 1, 0, 1119, 760, 0, 0] rerouted_tokens_per_ep_rank=[6656, 3560, 1631, 2738, 5511, 7056, 3735, 1881] avg=4096.0000 max=7056 max_rank=5 min=1631 min_rank=2 imbalance_ratio=1.722656 max_load_reduction=25712 max_load_reduction_pct=0.784668 moved_to_echo=18183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1101, 14, 903, 236, 0, 0, 667, 13, 1460, 1225, 0, 0, 0, 0, 0, 0, 38, 2045, 0, 0, 0, 0, 29, 0, 503, 2091, 479, 0, 0, 0, 1477, 22, 307, 17, 0, 0, 0, 0, 156, 1, 0, 4045, 2487, 0, 0, 0, 1, 0, 0, 3487, 1221, 492, 0, 0, 3794, 2724, 0, 5, 1154, 574, 0, 0] rerouted_tokens_per_ep_rank=[2254, 3365, 2083, 3102, 1823, 6689, 5201, 8251] avg=4096.0000 max=8251 max_rank=7 min=1823 min_rank=4 imbalance_ratio=2.014404 max_load_reduction=24517 max_load_reduction_pct=0.748199 moved_to_echo=21964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 354, 0, 1030, 0, 0, 0, 3037, 111, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 2333, 0, 0, 0, 0, 4089, 0, 542, 0, 0, 0, 0, 0, 5, 1, 1310, 0, 0, 0, 0, 0, 21, 2, 2, 4067, 2591, 0, 0, 0, 0, 0, 0, 4095, 2, 561, 0, 0, 29, 600, 0, 3274, 2, 597, 0, 0] rerouted_tokens_per_ep_rank=[1394, 7244, 2340, 4631, 1316, 6683, 4658, 4502] avg=4096.0000 max=7244 max_rank=1 min=1316 min_rank=4 imbalance_ratio=1.768555 max_load_reduction=25524 max_load_reduction_pct=0.778931 moved_to_echo=17159 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[42, 0, 622, 972, 198, 492, 0, 0, 0, 878, 4091, 2557, 0, 0, 0, 0, 0, 19, 3767, 449, 0, 0, 0, 0, 0, 286, 0, 3633, 999, 0, 0, 0, 1436, 2224, 2097, 433, 0, 0, 0, 0, 5, 0, 57, 119, 672, 0, 0, 0, 0, 0, 0, 3849, 1188, 107, 0, 0, 0, 0, 298, 0, 1123, 155, 0, 0] rerouted_tokens_per_ep_rank=[2326, 7526, 4235, 4918, 6190, 853, 5144, 1576] avg=4096.0000 max=7526 max_rank=1 min=853 min_rank=5 imbalance_ratio=1.837402 max_load_reduction=25242 max_load_reduction_pct=0.770325 moved_to_echo=13763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 72, 17, 898, 501, 0, 0, 0, 0, 0, 2581, 0, 0, 0, 0, 0, 4088, 4095, 2015, 0, 0, 0, 0, 1883, 0, 3213, 0, 1014, 0, 0, 0, 705, 3558, 0, 72, 0, 0, 0, 0, 1, 0, 0, 2931, 813, 0, 0, 0, 391, 0, 2, 70, 593, 491, 0, 0, 0, 4, 23, 1987, 560, 187, 0, 0] rerouted_tokens_per_ep_rank=[1491, 2581, 10198, 6110, 4335, 3745, 1547, 2761] avg=4096.0000 max=10198 max_rank=2 min=1491 min_rank=0 imbalance_ratio=2.489746 max_load_reduction=22570 max_load_reduction_pct=0.688782 moved_to_echo=12388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 3866, 954, 97, 0, 0, 1958, 1, 1, 507, 0, 0, 0, 0, 0, 3917, 140, 2162, 0, 0, 0, 0, 1021, 1, 430, 3949, 198, 0, 0, 0, 1557, 231, 4095, 0, 0, 0, 0, 0, 1, 0, 283, 0, 18, 0, 0, 0, 98, 0, 2, 3484, 1287, 520, 0, 0, 0, 719, 34, 0, 1216, 5, 0, 0] rerouted_tokens_per_ep_rank=[4933, 2467, 6219, 5599, 5883, 302, 5391, 1974] avg=4096.0000 max=6219 max_rank=2 min=302 min_rank=5 imbalance_ratio=1.518311 max_load_reduction=26549 max_load_reduction_pct=0.810211 moved_to_echo=13550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3850, 0, 0, 3995, 41, 0, 0, 0, 3263, 52, 125, 4, 0, 0, 0, 0, 0, 1388, 4, 96, 0, 0, 0, 0, 0, 0, 894, 1, 0, 0, 0, 0, 1560, 1706, 0, 0, 0, 0, 0, 0, 309, 0, 48, 753, 3084, 0, 0, 0, 340, 0, 0, 121, 1290, 22, 0, 0, 13, 4081, 3797, 0, 1219, 712, 0, 0] rerouted_tokens_per_ep_rank=[7886, 3444, 1488, 895, 3266, 4194, 1773, 9822] avg=4096.0000 max=9822 max_rank=7 min=895 min_rank=3 imbalance_ratio=2.397949 max_load_reduction=22946 max_load_reduction_pct=0.700256 moved_to_echo=19055 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 8192, 4224, 5376, 7552, 3712, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4992, 128, 4736, 16384, 4736, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9216, 1152, 11008, 11648, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 128, 512, 15744, 16128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 10112, 12288, 10496, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 384, 9472, 10240, 4608, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2944, 0, 128, 19456, 7936, 2560, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9600, 14848, 7936, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 148, 0, 1, 0, 5, 4096, 110, 0, 3024, 311, 826, 70, 0, 4096, 1, 104, 0, 4009, 0, 3684, 0, 4094, 0, 4072, 4083, 29, 0, 5, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1099, 0, 4, 59, 0, 3636, 2783, 56, 3851, 0, 0, 0, 0, 0, 0, 6, 683, 333, 1419, 3255, 456, 4096, 2210, 0, 0, 212, 1385, 4072, 0, 0, 3119, 34] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 4014, 0, 0, 0, 1, 3954, 0, 2235, 803, 1892, 891, 2, 0, 169, 19, 4088, 0, 1997, 0, 4067, 341, 4078, 0, 4092, 3, 0, 2, 120, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 159, 0, 4004, 4008, 702, 0, 0, 10, 3667, 0, 4051, 69, 3561, 405, 2, 4088, 3, 0, 3942, 521, 0, 0, 3557, 0, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[2, 707, 0, 0, 0, 861, 3585, 4081, 1, 1091, 16, 0, 4081, 1, 120, 0, 0, 74, 4045, 0, 1, 3187, 3958, 97, 3, 1, 4096, 0, 2760, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[43, 11, 342, 3668, 101, 3488, 3647, 9, 3977, 1102, 5, 3, 62, 0, 0, 0, 0, 8, 1619, 2, 1118, 5, 2859, 2234, 3536, 4092, 240, 63, 0, 0, 533, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[102, 0, 1, 4092, 0, 1931, 1, 0, 3540, 3831, 0, 0, 8, 0, 0, 0, 0, 1693, 0, 186, 2991, 3, 4096, 9, 1941, 3477, 834, 1, 1, 0, 0, 4030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[29, 0, 8, 80, 604, 3635, 3307, 170, 3951, 0, 0, 0, 3953, 1, 2571, 1, 252, 1, 18, 0, 654, 172, 36, 1, 4096, 0, 3599, 209, 426, 0, 4092, 902] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 2518, 346, 0, 0, 159, 0, 1565, 4008, 0, 0, 0, 0, 702, 0, 0, 10, 0, 0, 0, 0, 3667, 0, 4051, 69, 0, 0, 0, 0, 3561, 405, 2, 4088, 0, 0, 0, 0, 3, 0, 1424, 521, 0, 0, 0, 0, 0, 0, 3557, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2093, 0, 0, 0] rerouted_tokens_per_ep_rank=[2881, 5732, 712, 7787, 8056, 1948, 3557, 2095] avg=4096.0000 max=8056 max_rank=4 min=712 min_rank=2 imbalance_ratio=1.966797 max_load_reduction=24712 max_load_reduction_pct=0.754150 moved_to_echo=15656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 148, 0, 1, 2615, 354, 0, 0, 0, 5, 1601, 110, 0, 0, 0, 0, 0, 3024, 311, 826, 162, 0, 0, 0, 70, 0, 4096, 1, 2254, 0, 0, 0, 104, 0, 4009, 0, 212, 0, 0, 0, 3684, 0, 1479, 0, 0, 0, 0, 0, 647, 4083, 29, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2141, 797, 0, 0] rerouted_tokens_per_ep_rank=[3118, 1716, 4323, 6421, 4325, 5163, 4759, 2943] avg=4096.0000 max=6421 max_rank=3 min=1716 min_rank=1 imbalance_ratio=1.567627 max_load_reduction=26347 max_load_reduction_pct=0.804047 moved_to_echo=17190 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 707, 0, 0, 2528, 312, 0, 0, 0, 861, 1396, 4081, 0, 0, 0, 0, 1, 1091, 16, 0, 0, 0, 0, 0, 4081, 1, 120, 0, 1, 0, 0, 0, 0, 74, 4045, 0, 0, 0, 0, 0, 1, 3187, 1430, 97, 0, 0, 0, 0, 2, 1, 4096, 0, 0, 0, 0, 0, 2760, 0, 0, 0, 1877, 0, 0, 0] rerouted_tokens_per_ep_rank=[3549, 6338, 1108, 4203, 4119, 4715, 4099, 4637] avg=4096.0000 max=6338 max_rank=1 min=1108 min_rank=2 imbalance_ratio=1.547363 max_load_reduction=26430 max_load_reduction_pct=0.806580 moved_to_echo=17570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 4014, 0, 0, 2605, 342, 0, 0, 0, 1, 1545, 0, 0, 0, 0, 0, 2235, 803, 1892, 891, 162, 0, 0, 0, 2, 0, 169, 19, 2265, 0, 0, 0, 4088, 0, 1997, 0, 213, 0, 0, 0, 4067, 341, 1473, 0, 0, 0, 0, 0, 651, 3, 0, 2, 0, 0, 0, 0, 120, 0, 0, 0, 2067, 801, 0, 0] rerouted_tokens_per_ep_rank=[6961, 1546, 5983, 2455, 6298, 5881, 656, 2988] avg=4096.0000 max=6961 max_rank=0 min=656 min_rank=6 imbalance_ratio=1.699463 max_load_reduction=25807 max_load_reduction_pct=0.787567 moved_to_echo=15823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1099, 0, 4, 59, 1411, 240, 0, 0, 0, 3636, 1088, 56, 0, 0, 0, 0, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 683, 333, 1419, 3255, 0, 0, 0, 0, 456, 4096, 799, 0, 0, 0, 0, 0, 0, 212, 1385, 4072, 0, 0, 0, 0, 0, 0, 3119, 34, 1455, 0, 0, 0] rerouted_tokens_per_ep_rank=[2813, 4780, 3851, 6, 5690, 5351, 5669, 4608] avg=4096.0000 max=5690 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.389160 max_load_reduction=27078 max_load_reduction_pct=0.826355 moved_to_echo=21318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[43, 11, 342, 3668, 1826, 315, 0, 0, 101, 3488, 1426, 9, 0, 0, 0, 0, 3977, 1102, 5, 3, 140, 0, 0, 0, 62, 0, 0, 0, 1957, 0, 0, 0, 0, 8, 1619, 2, 184, 0, 0, 0, 1118, 5, 1033, 2234, 0, 0, 0, 0, 563, 4092, 240, 63, 0, 0, 0, 0, 0, 0, 533, 1, 1906, 692, 0, 0] rerouted_tokens_per_ep_rank=[6205, 5024, 5227, 2019, 1813, 4390, 4958, 3132] avg=4096.0000 max=6205 max_rank=0 min=1813 min_rank=4 imbalance_ratio=1.514893 max_load_reduction=26563 max_load_reduction_pct=0.810638 moved_to_echo=14293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[102, 0, 1, 4092, 2620, 1, 0, 0, 0, 1931, 0, 0, 0, 0, 0, 0, 3540, 3831, 0, 0, 79, 0, 0, 0, 8, 0, 0, 0, 1076, 0, 0, 0, 0, 1693, 0, 186, 104, 0, 0, 0, 2991, 3, 1476, 9, 0, 0, 0, 0, 299, 3477, 834, 1, 0, 0, 0, 0, 1, 0, 0, 4030, 0, 383, 0, 0] rerouted_tokens_per_ep_rank=[6816, 1931, 7450, 1084, 1983, 4479, 4611, 4414] avg=4096.0000 max=7450 max_rank=2 min=1084 min_rank=3 imbalance_ratio=1.818848 max_load_reduction=25318 max_load_reduction_pct=0.772644 moved_to_echo=15487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 8, 80, 22, 286, 0, 0, 604, 3635, 1292, 170, 0, 0, 0, 0, 3951, 0, 0, 0, 163, 0, 0, 0, 3953, 1, 2571, 1, 2267, 0, 0, 0, 252, 1, 18, 0, 213, 0, 0, 0, 654, 172, 14, 1, 0, 0, 0, 0, 651, 0, 3599, 209, 0, 0, 0, 0, 426, 0, 4092, 902, 1729, 802, 0, 0] rerouted_tokens_per_ep_rank=[425, 5701, 4114, 8793, 484, 841, 4459, 7951] avg=4096.0000 max=8793 max_rank=3 min=425 min_rank=0 imbalance_ratio=2.146729 max_load_reduction=23975 max_load_reduction_pct=0.731659 moved_to_echo=13735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11904, 128, 11008, 128, 9856, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1280, 4992, 384, 7936, 16256, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3328, 128, 7808, 4992, 13312, 3584, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8704, 2560, 13184, 7552, 1024, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[18304, 9856, 2304, 1792, 768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[896, 13568, 9984, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13056, 7808, 9216, 2944, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[2816, 11904, 13824, 4352, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[67, 0, 3669, 0, 4094, 0, 17, 6, 408, 183, 0, 1486, 3973, 0, 3, 154, 3, 0, 0, 3696, 0, 2818, 0, 395, 5, 2199, 0, 4093, 90, 3924, 0, 1485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4094, 0, 146, 35, 3705, 0, 0, 25, 106, 4091, 77, 2173, 3992, 53, 0, 3392, 0, 14, 2, 754, 173, 3821, 0, 0, 1989, 0, 49, 4058, 18, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[3201, 0, 2015, 40, 3686, 0, 0, 3860, 0, 4094, 1, 4075, 422, 0, 20, 0, 50, 654, 0, 345, 6, 4070, 0, 172, 0, 3846, 527, 0, 0, 156, 1444, 84] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3900, 194, 2981, 0, 1, 10, 0, 0, 0, 4090, 4094, 12, 0, 0, 1410, 0, 0, 4067, 185, 8, 0, 4025, 0, 3992, 0, 113, 0, 3686, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 17, 4070, 0, 4030, 0, 0, 0, 1573, 3586, 0, 3786, 4040, 0, 13, 457, 23, 3, 0, 1094, 57, 0, 69, 4063, 0, 3626, 0, 1, 0, 2259, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[49, 84, 4024, 0, 4075, 66, 1, 3, 0, 1832, 73, 3450, 3743, 0, 200, 836, 611, 0, 94, 0, 0, 0, 3242, 2321, 13, 4049, 0, 0, 9, 3885, 47, 61] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[3436, 0, 0, 0, 4084, 1, 0, 93, 0, 2, 0, 0, 709, 0, 18, 4066, 3968, 0, 1, 4096, 44, 0, 0, 8, 1, 4074, 0, 4048, 0, 3822, 7, 290] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 39, 3832, 0, 4071, 4, 0, 4095, 66, 3465, 90, 3887, 3982, 0, 8, 342, 0, 0, 0, 4074, 42, 0, 0, 0, 1, 623, 4, 65, 0, 3790, 0, 288] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 4043, 0, 0, 0, 0, 0, 3226, 0, 0, 0, 0, 0, 0, 0, 1573, 3586, 0, 2166, 0, 0, 0, 0, 3755, 0, 13, 457, 0, 0, 0, 0, 23, 3, 0, 1094, 1288, 0, 0, 0, 57, 0, 69, 4063, 745, 285, 96, 27, 0, 2881, 0, 1, 0, 0, 0, 0, 0, 2259, 1, 0, 708, 332, 0, 0] rerouted_tokens_per_ep_rank=[4060, 3226, 7325, 4225, 2408, 5342, 2882, 3300] avg=4096.0000 max=7325 max_rank=2 min=2408 min_rank=4 imbalance_ratio=1.788330 max_load_reduction=25443 max_load_reduction_pct=0.776459 moved_to_echo=13932 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[67, 0, 3645, 0, 0, 0, 0, 0, 3276, 0, 17, 6, 0, 0, 0, 0, 408, 183, 0, 851, 0, 0, 0, 0, 3693, 0, 3, 154, 0, 0, 0, 0, 3, 0, 0, 3696, 505, 0, 0, 0, 0, 2818, 0, 395, 452, 280, 98, 24, 5, 1747, 0, 4093, 0, 0, 0, 0, 90, 3924, 0, 1485, 720, 130, 0, 0] rerouted_tokens_per_ep_rank=[3712, 3299, 1442, 3850, 4204, 4067, 5845, 6349] avg=4096.0000 max=6349 max_rank=7 min=1442 min_rank=2 imbalance_ratio=1.550049 max_load_reduction=26419 max_load_reduction_pct=0.806244 moved_to_echo=20465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3201, 0, 2002, 40, 0, 0, 0, 0, 2950, 0, 0, 3860, 0, 0, 0, 0, 0, 4094, 1, 2331, 0, 0, 0, 0, 393, 0, 20, 0, 0, 0, 0, 0, 50, 654, 0, 345, 1386, 0, 0, 0, 6, 4070, 0, 172, 790, 29, 88, 13, 0, 3056, 527, 0, 0, 0, 0, 0, 0, 156, 1444, 84, 648, 358, 0, 0] rerouted_tokens_per_ep_rank=[5243, 6810, 6426, 413, 2435, 5168, 3583, 2690] avg=4096.0000 max=6810 max_rank=1 min=413 min_rank=3 imbalance_ratio=1.662598 max_load_reduction=25958 max_load_reduction_pct=0.792175 moved_to_echo=13876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3436, 0, 0, 0, 0, 0, 0, 0, 3269, 1, 0, 93, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 659, 0, 18, 4066, 0, 0, 0, 0, 3968, 0, 1, 4096, 0, 0, 0, 0, 44, 0, 0, 8, 837, 50, 97, 0, 1, 3237, 0, 4048, 0, 0, 0, 0, 0, 3822, 7, 290, 718, 0, 0, 0] rerouted_tokens_per_ep_rank=[3436, 3363, 2, 4743, 8065, 1036, 7286, 4837] avg=4096.0000 max=8065 max_rank=4 min=2 min_rank=2 imbalance_ratio=1.968994 max_load_reduction=24703 max_load_reduction_pct=0.753876 moved_to_echo=21224 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 39, 3807, 0, 0, 0, 0, 0, 3259, 4, 0, 4095, 0, 0, 0, 0, 66, 3465, 90, 2224, 0, 0, 0, 0, 3702, 0, 8, 342, 0, 0, 0, 0, 0, 0, 0, 4074, 1322, 0, 0, 0, 42, 0, 0, 0, 128, 280, 97, 25, 1, 495, 4, 65, 0, 0, 0, 0, 0, 3790, 0, 288, 715, 341, 0, 0] rerouted_tokens_per_ep_rank=[3846, 7358, 5845, 4052, 5396, 572, 565, 5134] avg=4096.0000 max=7358 max_rank=1 min=565 min_rank=6 imbalance_ratio=1.796387 max_load_reduction=25410 max_load_reduction_pct=0.775452 moved_to_echo=11667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3871, 194, 0, 0, 0, 0, 2378, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 2333, 0, 0, 0, 0, 3802, 12, 0, 0, 0, 0, 0, 0, 1410, 0, 0, 4067, 1395, 0, 0, 0, 185, 8, 0, 4025, 824, 292, 75, 29, 0, 3168, 0, 113, 0, 0, 0, 0, 0, 3686, 0, 0, 528, 362, 0, 0] rerouted_tokens_per_ep_rank=[4065, 2389, 2333, 3814, 6872, 5438, 3281, 4576] avg=4096.0000 max=6872 max_rank=4 min=2333 min_rank=2 imbalance_ratio=1.677734 max_load_reduction=25896 max_load_reduction_pct=0.790283 moved_to_echo=20167 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 146, 35, 0, 0, 0, 0, 2966, 0, 0, 25, 0, 0, 0, 0, 106, 4091, 77, 1243, 0, 0, 0, 0, 3711, 53, 0, 3392, 0, 0, 0, 0, 0, 14, 2, 754, 739, 0, 0, 0, 173, 3821, 0, 0, 0, 281, 88, 0, 1989, 0, 49, 4058, 0, 0, 0, 0, 18, 0, 1, 0, 651, 191, 0, 0] rerouted_tokens_per_ep_rank=[4275, 2991, 5517, 7156, 1509, 4363, 6096, 861] avg=4096.0000 max=7156 max_rank=3 min=861 min_rank=7 imbalance_ratio=1.747070 max_load_reduction=25612 max_load_reduction_pct=0.781616 moved_to_echo=12829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[49, 84, 3998, 0, 0, 0, 0, 0, 3262, 66, 1, 3, 0, 0, 0, 0, 0, 1832, 73, 1973, 0, 0, 0, 0, 3479, 0, 200, 836, 0, 0, 0, 0, 611, 0, 94, 0, 1174, 0, 0, 0, 0, 0, 3242, 2321, 832, 264, 97, 26, 13, 3217, 0, 0, 0, 0, 0, 0, 9, 3885, 47, 61, 716, 303, 0, 0] rerouted_tokens_per_ep_rank=[4131, 3332, 3878, 4515, 1879, 6782, 3230, 5021] avg=4096.0000 max=6782 max_rank=5 min=1879 min_rank=4 imbalance_ratio=1.655762 max_load_reduction=25986 max_load_reduction_pct=0.793030 moved_to_echo=16912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[2176, 17280, 256, 13184, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6144, 768, 128, 18176, 7936, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 21632, 1536, 2304, 5504, 2048, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2048, 17920, 640, 12416, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 10752, 3328, 11008, 4608, 1792, 768, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[23296, 128, 384, 9344, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[24704, 128, 128, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10880, 256, 21632, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 453, 979, 0, 4096, 0, 0, 4046, 3293, 143, 3407, 76, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3921, 0, 3725, 0, 110, 0, 6, 320, 4095, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 4018, 0, 1582, 0, 3849, 0, 0, 338, 398, 0, 2806, 2160, 349, 1058, 1, 18, 283, 0, 0, 4, 0, 2309, 3703, 5, 51, 0, 2, 0, 1478, 3746, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4037, 0, 1, 0, 0, 4093, 5, 3980, 16, 12, 0, 2086, 4078, 219, 0, 124, 6, 635, 0, 7, 3753, 570, 13, 0, 1105, 3464, 0, 3, 0, 545, 0, 4016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4013, 0, 0, 2902, 0, 259, 0, 0, 0, 112, 0, 4095, 18, 0, 98, 0, 3759, 0, 0, 0, 0, 846, 3995, 3926, 19, 494, 1, 53, 0, 4083, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[153, 1, 0, 0, 0, 3452, 244, 4033, 9, 4085, 3, 366, 3878, 1, 63, 14, 85, 2734, 0, 359, 0, 145, 3959, 3971, 38, 0, 420, 4071, 683, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[258, 0, 1884, 0, 175, 3, 0, 235, 4065, 4, 0, 4096, 3658, 3, 0, 18, 26, 4090, 0, 90, 1668, 0, 0, 5, 3866, 4090, 2028, 0, 0, 0, 2, 2504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 1228, 1912, 0, 44, 1529, 0, 0, 4068, 0, 0, 2255, 4095, 0, 0, 18, 0, 0, 39, 0, 369, 0, 4035, 0, 3727, 3103, 2144, 0, 0, 0, 110, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3585, 3313, 0, 54, 2224, 3720, 0, 0, 0, 582, 169, 0, 3916, 0, 38, 1011, 3, 3448, 110, 0, 0, 0, 2430, 0, 1937, 0, 0, 1, 16, 1, 2114, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4013, 0, 0, 0, 0, 0, 0, 2902, 0, 259, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 4095, 18, 0, 98, 140, 0, 0, 0, 0, 3759, 0, 0, 1381, 1191, 0, 1, 0, 0, 705, 3995, 0, 0, 0, 0, 2735, 19, 494, 1, 0, 0, 0, 0, 53, 0, 4083, 2714, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 3161, 112, 4351, 6332, 4700, 3249, 6850] avg=4096.0000 max=6850 max_rank=7 min=112 min_rank=2 imbalance_ratio=1.672363 max_load_reduction=25918 max_load_reduction_pct=0.790955 moved_to_echo=21131 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4037, 0, 1, 0, 236, 0, 0, 0, 0, 3685, 5, 3980, 0, 0, 0, 0, 16, 12, 0, 1588, 0, 0, 0, 0, 4078, 219, 0, 124, 2, 172, 0, 0, 6, 635, 0, 7, 1354, 335, 498, 0, 3753, 570, 11, 0, 0, 0, 0, 0, 770, 3464, 0, 3, 0, 0, 0, 0, 0, 545, 0, 2662, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4274, 7670, 1616, 4595, 2835, 4334, 4237, 3207] avg=4096.0000 max=7670 max_rank=1 min=1616 min_rank=2 imbalance_ratio=1.872559 max_load_reduction=25098 max_load_reduction_pct=0.765930 moved_to_echo=14613 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[153, 1, 0, 0, 199, 0, 0, 0, 0, 3108, 244, 4033, 0, 0, 0, 0, 9, 4085, 3, 279, 0, 0, 0, 0, 3878, 1, 63, 14, 656, 145, 0, 0, 85, 2734, 0, 359, 0, 11, 87, 5, 0, 145, 3298, 3971, 0, 0, 0, 0, 27, 0, 420, 4071, 0, 0, 0, 0, 683, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[353, 7385, 4376, 4757, 3281, 7414, 4518, 684] avg=4096.0000 max=7414 max_rank=5 min=353 min_rank=0 imbalance_ratio=1.810059 max_load_reduction=25354 max_load_reduction_pct=0.773743 moved_to_echo=15897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 4018, 0, 1582, 222, 0, 0, 0, 0, 3465, 0, 0, 0, 0, 0, 0, 338, 398, 0, 2135, 0, 0, 0, 0, 2160, 349, 1058, 1, 383, 162, 0, 0, 18, 283, 0, 0, 173, 1, 671, 3, 4, 0, 1923, 3703, 0, 0, 0, 0, 4, 51, 0, 2, 0, 0, 0, 0, 0, 1478, 3746, 341, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9918, 3465, 2871, 4113, 1149, 5630, 57, 5565] avg=4096.0000 max=9918 max_rank=0 min=57 min_rank=6 imbalance_ratio=2.421387 max_load_reduction=22850 max_load_reduction_pct=0.697327 moved_to_echo=12401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3585, 3313, 0, 54, 215, 0, 0, 0, 2224, 3349, 0, 0, 0, 0, 0, 0, 0, 582, 169, 0, 0, 0, 0, 0, 3916, 0, 38, 1011, 403, 156, 0, 0, 3, 3448, 110, 0, 1381, 587, 0, 3, 0, 0, 2024, 0, 0, 0, 0, 0, 1350, 0, 0, 1, 0, 0, 0, 0, 16, 1, 2114, 2715, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7167, 5573, 751, 5524, 5532, 2024, 1351, 4846] avg=4096.0000 max=7167 max_rank=0 min=751 min_rank=2 imbalance_ratio=1.749756 max_load_reduction=25601 max_load_reduction_pct=0.781281 moved_to_echo=13753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1228, 1912, 0, 88, 0, 0, 0, 44, 1377, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 1716, 0, 0, 0, 0, 4095, 0, 0, 18, 669, 64, 0, 0, 0, 0, 39, 0, 1380, 1130, 539, 5, 369, 0, 3361, 0, 0, 0, 0, 0, 2597, 3103, 2144, 0, 0, 0, 0, 0, 0, 0, 110, 2712, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3228, 1421, 5784, 4846, 3093, 3730, 7844, 2822] avg=4096.0000 max=7844 max_rank=6 min=1421 min_rank=1 imbalance_ratio=1.915039 max_load_reduction=24924 max_load_reduction_pct=0.760620 moved_to_echo=17489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 453, 979, 241, 0, 0, 0, 0, 3680, 0, 0, 0, 0, 0, 0, 4046, 3293, 143, 2590, 0, 0, 0, 0, 76, 0, 0, 0, 652, 175, 0, 0, 0, 0, 0, 2, 1385, 1134, 817, 6, 0, 0, 3263, 0, 0, 0, 0, 0, 2591, 0, 110, 0, 0, 0, 0, 0, 6, 320, 4095, 2711, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1673, 3680, 10072, 903, 3344, 3263, 2701, 7132] avg=4096.0000 max=10072 max_rank=2 min=903 min_rank=3 imbalance_ratio=2.458984 max_load_reduction=22696 max_load_reduction_pct=0.692627 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[258, 0, 1884, 0, 0, 0, 0, 0, 175, 3, 0, 235, 0, 0, 0, 0, 4065, 4, 0, 3117, 0, 0, 0, 0, 3658, 3, 0, 18, 0, 0, 0, 0, 26, 4090, 0, 90, 844, 1173, 979, 0, 1668, 0, 0, 5, 0, 0, 0, 0, 2693, 4090, 2028, 0, 0, 0, 0, 0, 0, 0, 2, 1660, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2142, 413, 7186, 3679, 7202, 1673, 8811, 1662] avg=4096.0000 max=8811 max_rank=6 min=413 min_rank=1 imbalance_ratio=2.151123 max_load_reduction=23957 max_load_reduction_pct=0.731110 moved_to_echo=19348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12544, 8448, 512, 11520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 14976, 256, 512, 7936, 5632, 3712, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12160, 12672, 4352, 2688, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[5376, 18688, 512, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[12800, 10752, 5248, 4096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 2432, 14208, 15616, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5888, 768, 14592, 11776, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[25984, 640, 1280, 1408, 2944, 896, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 142, 0, 0, 1, 7, 100, 7, 4092, 0, 0, 0, 16, 57, 4, 3979, 4074, 0, 3397, 4077, 4093, 746, 2914, 1, 0, 762, 240, 3071, 988, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1, 196, 0, 0, 0, 1128, 1, 1, 4012, 1726, 2361, 0, 2, 0, 0, 118, 3752, 4017, 0, 1900, 4095, 588, 1868, 2569, 63, 243, 17, 14, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[235, 886, 0, 10, 32, 0, 1, 4, 60, 2886, 4096, 0, 2843, 2, 4085, 472, 2, 311, 151, 38, 0, 4088, 6, 0, 3909, 383, 5, 3708, 479, 0, 4076, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1893, 1, 0, 6, 0, 0, 0, 0, 3887, 0, 0, 0, 1943, 2657, 0, 67, 17, 3118, 3023, 4069, 3651, 2020, 78, 233, 670, 180, 32, 2113, 611, 2499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[3652, 462, 0, 24, 22, 386, 0, 0, 84, 3092, 3500, 15, 109, 2, 12, 0, 3844, 3779, 30, 0, 3936, 4065, 4, 0, 751, 1358, 0, 61, 0, 7, 3572, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 43, 0, 0, 0, 2, 1, 0, 183, 42, 3574, 0, 0, 195, 84, 752, 3186, 262, 3394, 0, 3878, 0, 4096, 0, 3991, 298, 0, 0, 606, 0, 4085, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 45, 0, 0, 0, 3, 2615, 4009, 4074, 0, 0, 17, 2024, 1410, 0, 4096, 12, 0, 4081, 3522, 3809, 0, 375, 0, 0, 1437, 0, 0, 1239, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[882, 19, 0, 2, 0, 0, 0, 6, 22, 3, 4076, 0, 0, 2, 3921, 0, 3482, 3797, 43, 0, 0, 4013, 4086, 1657, 0, 0, 1, 0, 0, 0, 2720, 4036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[235, 886, 0, 10, 1763, 0, 0, 0, 32, 0, 1, 4, 4088, 0, 0, 0, 60, 2886, 1923, 0, 0, 0, 0, 0, 2843, 2, 4085, 472, 138, 410, 0, 0, 2, 89, 151, 38, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 3909, 383, 5, 3708, 1366, 84, 0, 0, 479, 0, 2710, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2894, 4125, 4869, 7950, 280, 6, 9455, 3189] avg=4096.0000 max=9455 max_rank=6 min=6 min_rank=5 imbalance_ratio=2.308350 max_load_reduction=23313 max_load_reduction_pct=0.711456 moved_to_echo=12930 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 142, 1762, 1743, 0, 0, 0, 0, 1, 7, 4077, 1022, 0, 0, 100, 7, 1920, 0, 0, 0, 0, 0, 0, 0, 16, 57, 1777, 410, 0, 0, 4, 1122, 4074, 0, 0, 0, 0, 0, 632, 0, 4093, 746, 0, 0, 0, 0, 2914, 1, 0, 762, 331, 1080, 0, 0, 240, 3071, 657, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3647, 5107, 2027, 2260, 5200, 5471, 5088, 3968] avg=4096.0000 max=5471 max_rank=5 min=2027 min_rank=2 imbalance_ratio=1.335693 max_load_reduction=27297 max_load_reduction_pct=0.833038 moved_to_echo=19727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 196, 0, 0, 1016, 2101, 0, 0, 0, 1128, 1, 1, 588, 1232, 0, 0, 4012, 1726, 1109, 0, 0, 0, 0, 0, 2, 0, 0, 118, 1794, 236, 0, 0, 3752, 1132, 0, 1900, 0, 0, 0, 0, 762, 0, 1868, 2569, 0, 0, 0, 0, 63, 243, 17, 14, 0, 1091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3314, 2950, 6847, 2150, 6784, 5199, 1428, 4096] avg=4096.0000 max=6847 max_rank=2 min=1428 min_rank=6 imbalance_ratio=1.671631 max_load_reduction=25921 max_load_reduction_pct=0.791046 moved_to_echo=17507 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1893, 1, 1673, 1551, 0, 0, 0, 6, 0, 0, 4069, 909, 0, 0, 0, 0, 1825, 0, 0, 0, 0, 0, 0, 0, 1943, 2657, 29, 389, 0, 0, 0, 20, 17, 3118, 0, 0, 0, 0, 563, 0, 3651, 2020, 0, 0, 0, 0, 78, 233, 670, 180, 204, 18, 0, 0, 32, 2113, 407, 2499, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5118, 4984, 1825, 5018, 3155, 6234, 1383, 5051] avg=4096.0000 max=6234 max_rank=5 min=1383 min_rank=6 imbalance_ratio=1.521973 max_load_reduction=26534 max_load_reduction_pct=0.809753 moved_to_echo=15823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3652, 462, 0, 24, 1507, 2020, 0, 0, 22, 386, 0, 0, 4065, 1184, 0, 0, 84, 3092, 1642, 15, 0, 0, 0, 0, 109, 2, 12, 0, 1687, 351, 0, 0, 3844, 1066, 30, 0, 0, 0, 0, 0, 732, 0, 4, 0, 0, 0, 0, 0, 751, 1358, 0, 61, 1197, 1026, 0, 0, 0, 7, 2375, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7665, 5657, 4833, 2161, 4940, 736, 4393, 2383] avg=4096.0000 max=7665 max_rank=0 min=736 min_rank=5 imbalance_ratio=1.871338 max_load_reduction=25103 max_load_reduction_pct=0.766083 moved_to_echo=12452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 0, 0, 1543, 1993, 0, 0, 0, 2, 1, 0, 0, 1170, 0, 0, 183, 42, 1668, 0, 0, 0, 0, 0, 0, 195, 84, 752, 121, 363, 0, 0, 3186, 66, 3394, 0, 0, 0, 0, 0, 715, 0, 4096, 0, 0, 0, 0, 0, 3991, 298, 0, 0, 1374, 75, 0, 0, 606, 0, 2711, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3579, 1173, 1893, 1515, 6646, 4811, 5738, 7413] avg=4096.0000 max=7413 max_rank=7 min=1173 min_rank=1 imbalance_ratio=1.809814 max_load_reduction=25355 max_load_reduction_pct=0.773773 moved_to_echo=24608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 45, 1754, 2094, 0, 0, 0, 0, 0, 3, 3522, 1228, 0, 0, 2615, 4009, 1912, 0, 0, 0, 0, 0, 0, 17, 2024, 1410, 1829, 408, 0, 0, 0, 1155, 12, 0, 0, 0, 0, 0, 759, 0, 3809, 0, 0, 0, 0, 0, 375, 0, 0, 1437, 415, 1112, 0, 0, 0, 0, 824, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3893, 4753, 8536, 5688, 1167, 4568, 3339, 824] avg=4096.0000 max=8536 max_rank=2 min=824 min_rank=7 imbalance_ratio=2.083984 max_load_reduction=24232 max_load_reduction_pct=0.739502 moved_to_echo=9898 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[882, 19, 0, 2, 1755, 0, 0, 0, 0, 0, 0, 6, 4013, 0, 0, 0, 22, 3, 1913, 0, 0, 0, 0, 0, 0, 2, 3921, 0, 1695, 408, 0, 0, 3482, 1071, 43, 0, 0, 0, 0, 0, 0, 0, 4086, 1657, 0, 0, 0, 0, 0, 0, 1, 0, 912, 1031, 0, 0, 0, 0, 1808, 4036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2658, 4019, 1938, 6026, 4596, 5743, 1944, 5844] avg=4096.0000 max=6026 max_rank=3 min=1938 min_rank=2 imbalance_ratio=1.471191 max_load_reduction=26742 max_load_reduction_pct=0.816101 moved_to_echo=18127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4864, 1664, 1920, 256, 12800, 11520, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14336, 5760, 7808, 5120, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[12160, 2560, 768, 6272, 5888, 5632, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3072, 256, 12160, 5504, 9088, 3072, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 1536, 128, 128, 24448, 6784, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7168, 11776, 13952, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1408, 9344, 11520, 10752, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[4224, 0, 21632, 7040, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3772, 177, 0, 46, 1, 4012, 50, 507, 2, 0, 4077, 3540, 0, 0, 3934, 0, 7, 0, 10, 4024, 4, 0, 49, 0, 169, 0, 4023, 154, 3910, 295, 1, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 37, 10, 7, 4095, 0, 2919, 0, 0, 2, 587, 28, 0, 570, 0, 4096, 0, 4065, 4017, 0, 0, 4084, 9, 40, 0, 274, 0, 3795, 0, 15, 22] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[114, 3478, 3997, 4091, 4088, 1, 0, 0, 0, 6, 184, 3818, 169, 224, 2200, 88, 1, 0, 1182, 15, 496, 51, 0, 0, 1669, 0, 0, 511, 4018, 0, 74, 2293] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[2, 3592, 12, 0, 4095, 0, 4094, 0, 0, 0, 259, 1554, 1620, 104, 2340, 3827, 0, 2731, 4096, 0, 42, 0, 1506, 1884, 0, 24, 0, 0, 0, 0, 5, 981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[54, 0, 0, 40, 0, 4095, 0, 3486, 4096, 1796, 2577, 2951, 0, 0, 1437, 369, 2798, 391, 532, 4077, 0, 494, 0, 5, 0, 0, 16, 1193, 0, 1, 2316, 44] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4068, 0, 1121, 324, 2372, 0, 0, 0, 0, 2, 1503, 4077, 280, 628, 3857, 1927, 0, 4089, 4096, 0, 0, 0, 982, 0, 1, 1110, 1964, 1, 0, 0, 0, 366] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1410, 0, 4088, 860, 4034, 3589, 1, 2017, 0, 4, 1727, 286, 0, 35, 4082, 7, 105, 2019, 4094, 89, 0, 0, 0, 0, 0, 782, 6, 0, 3509, 14, 10, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 4094, 0, 4074, 16, 4061, 461, 497, 0, 983, 2875, 1, 1416, 2838, 0, 3, 85, 3676, 0, 1393, 0, 1, 0, 10, 4033, 2211, 0, 33, 0, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1899, 177, 0, 46, 0, 0, 0, 0, 0, 3851, 50, 507, 0, 0, 0, 0, 2, 0, 4077, 2707, 0, 0, 0, 0, 0, 0, 3934, 0, 161, 27, 0, 0, 7, 0, 3, 4024, 0, 0, 0, 0, 4, 0, 49, 0, 1, 1, 0, 0, 169, 0, 4023, 154, 6, 40, 0, 0, 3910, 295, 1, 4, 1833, 806, 0, 0] rerouted_tokens_per_ep_rank=[2122, 4408, 6786, 4122, 4034, 55, 4392, 6849] avg=4096.0000 max=6849 max_rank=7 min=55 min_rank=5 imbalance_ratio=1.672119 max_load_reduction=25919 max_load_reduction_pct=0.790985 moved_to_echo=15330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4094, 0, 0, 0, 0, 0, 0, 13, 4061, 461, 0, 0, 0, 0, 497, 0, 983, 2199, 0, 0, 0, 0, 1, 1416, 2838, 0, 3, 22, 0, 0, 3, 85, 714, 0, 0, 0, 0, 0, 1393, 0, 1, 0, 4074, 523, 0, 0, 10, 4033, 2211, 0, 2439, 0, 0, 0, 33, 0, 6, 0, 0, 654, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4535, 3679, 4280, 802, 5991, 8693, 693] avg=4096.0000 max=8693 max_rank=6 min=693 min_rank=7 imbalance_ratio=2.122314 max_load_reduction=24075 max_load_reduction_pct=0.734711 moved_to_echo=16179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[710, 0, 4088, 860, 0, 0, 0, 0, 0, 3445, 1, 2017, 0, 0, 0, 0, 0, 4, 1727, 219, 0, 0, 0, 0, 0, 35, 4082, 7, 144, 2, 0, 0, 105, 2019, 796, 89, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 582, 0, 0, 0, 782, 6, 0, 2716, 15, 0, 0, 3509, 14, 10, 0, 685, 65, 0, 0] rerouted_tokens_per_ep_rank=[5658, 5463, 1950, 4270, 3009, 4616, 3519, 4283] avg=4096.0000 max=5658 max_rank=0 min=1950 min_rank=2 imbalance_ratio=1.381348 max_load_reduction=27110 max_load_reduction_pct=0.827332 moved_to_echo=15427 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 3592, 12, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 259, 1190, 0, 0, 0, 0, 1620, 104, 2340, 3827, 0, 11, 0, 0, 0, 2731, 796, 0, 0, 0, 0, 0, 42, 0, 1506, 1884, 4095, 583, 0, 0, 0, 24, 0, 0, 2717, 0, 0, 0, 0, 0, 5, 981, 0, 353, 0, 0] rerouted_tokens_per_ep_rank=[3606, 4094, 1449, 7902, 3527, 8110, 2741, 1339] avg=4096.0000 max=8110 max_rank=5 min=1339 min_rank=7 imbalance_ratio=1.979980 max_load_reduction=24658 max_load_reduction_pct=0.752502 moved_to_echo=15717 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[58, 3478, 3997, 4091, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6, 184, 2920, 0, 0, 0, 0, 169, 224, 2200, 88, 0, 29, 0, 0, 1, 0, 230, 15, 0, 0, 0, 0, 496, 51, 0, 0, 4088, 168, 0, 0, 1669, 0, 0, 511, 784, 1, 0, 0, 4018, 0, 74, 2293, 55, 869, 0, 0] rerouted_tokens_per_ep_rank=[11624, 1, 3110, 2710, 246, 4803, 2965, 7309] avg=4096.0000 max=11624 max_rank=0 min=1 min_rank=1 imbalance_ratio=2.837891 max_load_reduction=21144 max_load_reduction_pct=0.645264 moved_to_echo=15323 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2041, 0, 1121, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1503, 3111, 0, 0, 0, 0, 280, 628, 3857, 1927, 0, 34, 0, 0, 0, 4089, 788, 0, 0, 0, 0, 0, 0, 0, 982, 0, 2372, 587, 0, 0, 1, 1110, 1964, 1, 2721, 45, 0, 0, 0, 0, 0, 366, 1982, 932, 0, 0] rerouted_tokens_per_ep_rank=[3486, 0, 4616, 6726, 4877, 3941, 5842, 3280] avg=4096.0000 max=6726 max_rank=3 min=0 min_rank=1 imbalance_ratio=1.642090 max_load_reduction=26042 max_load_reduction_pct=0.794739 moved_to_echo=17940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 0, 40, 0, 0, 0, 0, 0, 3931, 0, 3486, 0, 0, 0, 0, 4096, 1796, 2577, 2257, 0, 0, 0, 0, 0, 0, 1437, 369, 164, 22, 0, 0, 2798, 391, 104, 4077, 0, 0, 0, 0, 0, 494, 0, 5, 0, 75, 0, 0, 0, 0, 16, 1193, 353, 0, 0, 0, 0, 1, 2316, 44, 26, 672, 0, 0] rerouted_tokens_per_ep_rank=[68, 7417, 10726, 1992, 7370, 574, 1562, 3059] avg=4096.0000 max=10726 max_rank=2 min=68 min_rank=0 imbalance_ratio=2.618652 max_load_reduction=22042 max_load_reduction_pct=0.672668 moved_to_echo=12565 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2062, 0, 37, 10, 0, 0, 0, 0, 0, 3931, 0, 2919, 0, 0, 0, 0, 0, 0, 2, 450, 0, 0, 0, 0, 28, 0, 570, 0, 164, 4, 0, 0, 4096, 0, 790, 4017, 0, 0, 0, 0, 0, 0, 4084, 9, 7, 578, 0, 0, 40, 0, 274, 0, 2697, 43, 0, 0, 3795, 0, 15, 22, 1991, 133, 0, 0] rerouted_tokens_per_ep_rank=[2109, 6850, 452, 766, 8903, 4678, 3054, 5956] avg=4096.0000 max=8903 max_rank=4 min=452 min_rank=2 imbalance_ratio=2.173584 max_load_reduction=23865 max_load_reduction_pct=0.728302 moved_to_echo=22591 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4608, 1920, 11392, 15104, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[7040, 9344, 4224, 12288, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6912, 7296, 13440, 5376, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 15232, 8320, 9472, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[15360, 384, 2432, 3712, 6656, 4608, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1920, 6016, 8576, 1920, 14464, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2048, 640, 6656, 1920, 18688, 3200, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2176, 2432, 21376, 6272, 640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3831, 0, 0, 3, 0, 0, 0, 4094, 4066, 3257, 4051, 40, 10, 0, 4090, 0, 243, 0, 0, 0, 0, 0, 0, 623, 0, 4096, 0, 0, 286, 4078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4, 0, 4096, 3, 0, 3, 216, 59, 0, 4045, 0, 4031, 0, 0, 3, 8, 3389, 4096, 423, 4043, 0, 0, 0, 0, 10, 0, 69, 2858, 1230, 0, 3271, 911] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 222, 52, 12, 4020, 90, 0, 0, 0, 4093, 1995, 132, 12, 0, 0, 237, 3881, 59, 124, 2584, 29, 2, 530, 0, 0, 3172, 0, 3794, 2337, 1253, 44, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 2796, 360, 634, 0, 0, 0, 3607, 3986, 3861, 3655, 525, 0, 53, 3798, 0, 0, 4095, 1, 0, 0, 16, 0, 213, 0, 0, 148, 943, 29, 4048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 21, 50, 1659, 15, 49, 3532, 4096, 2474, 0, 0, 264, 4092, 11, 0, 132, 4074, 1025, 3618, 32, 2131, 0, 0, 52, 0, 0, 383, 0, 37, 3381, 1640] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3578, 2211, 0, 0, 79, 0, 500, 3032, 3940, 3315, 0, 0, 257, 0, 4095, 0, 0, 3860, 0, 8, 0, 156, 0, 0, 0, 4093, 22, 0, 351, 3271] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3215, 140, 85, 0, 0, 1, 2165, 4073, 4096, 3678, 58, 94, 1570, 0, 4089, 0, 481, 4095, 1, 0, 37, 0, 10, 51, 0, 4092, 0, 0, 55, 682] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1941, 106, 4096, 27, 1796, 0, 182, 0, 1, 4096, 3944, 603, 1581, 0, 180, 0, 4095, 527, 11, 4078, 0, 0, 0, 28, 1, 22, 0, 1371, 4034, 0, 48, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 222, 52, 12, 748, 288, 0, 0, 4020, 90, 0, 0, 2813, 0, 0, 0, 0, 0, 647, 132, 0, 0, 0, 0, 12, 0, 0, 237, 793, 1029, 0, 0, 0, 59, 124, 2127, 0, 0, 0, 0, 29, 2, 530, 0, 4093, 39, 0, 0, 0, 3172, 0, 3794, 555, 169, 0, 0, 2337, 1253, 44, 3346, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1322, 6923, 779, 2071, 2310, 4693, 7690, 6980] avg=4096.0000 max=7690 max_rank=6 min=779 min_rank=2 imbalance_ratio=1.877441 max_load_reduction=25078 max_load_reduction_pct=0.765320 moved_to_echo=21673 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3215, 140, 124, 456, 0, 0, 85, 0, 0, 1, 2964, 0, 0, 0, 2165, 0, 1326, 3678, 0, 0, 0, 0, 58, 94, 1570, 0, 1630, 1084, 0, 0, 0, 0, 481, 3371, 0, 0, 0, 0, 1, 0, 37, 0, 4073, 41, 0, 0, 10, 51, 0, 4092, 1140, 268, 0, 0, 0, 0, 55, 558, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3935, 3050, 7169, 4436, 3852, 4152, 5561, 613] avg=4096.0000 max=7169 max_rank=2 min=613 min_rank=7 imbalance_ratio=1.750244 max_load_reduction=25599 max_load_reduction_pct=0.781219 moved_to_echo=14178 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3831, 0, 750, 0, 0, 0, 0, 3, 0, 0, 2966, 0, 0, 0, 0, 0, 1317, 3257, 0, 0, 0, 0, 4051, 40, 10, 0, 1618, 1083, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 41, 0, 0, 0, 623, 0, 4096, 1131, 0, 0, 0, 0, 0, 286, 3328, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4581, 2969, 4574, 6802, 243, 4135, 5850, 3614] avg=4096.0000 max=6802 max_rank=3 min=243 min_rank=4 imbalance_ratio=1.660645 max_load_reduction=25966 max_load_reduction_pct=0.792419 moved_to_echo=13842 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1941, 106, 4096, 27, 0, 458, 0, 0, 1796, 0, 182, 0, 2969, 0, 0, 0, 1, 0, 1271, 603, 0, 0, 0, 0, 1581, 0, 180, 0, 1572, 1084, 0, 0, 0, 527, 11, 3351, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 42, 0, 0, 1, 22, 0, 1371, 1101, 269, 0, 0, 4034, 0, 48, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6628, 4947, 1875, 4417, 3889, 4166, 2764, 4082] avg=4096.0000 max=6628 max_rank=0 min=1875 min_rank=2 imbalance_ratio=1.618164 max_load_reduction=26140 max_load_reduction_pct=0.797729 moved_to_echo=14901 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2796, 740, 456, 0, 0, 360, 634, 0, 0, 2753, 0, 0, 0, 0, 0, 1291, 3861, 0, 0, 0, 0, 3655, 525, 0, 53, 1586, 1005, 0, 0, 0, 0, 0, 3371, 0, 0, 0, 0, 1, 0, 0, 16, 3607, 40, 0, 0, 0, 213, 0, 0, 1109, 268, 0, 0, 148, 943, 29, 3308, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3992, 3747, 5152, 6824, 3371, 3664, 1590, 4428] avg=4096.0000 max=6824 max_rank=3 min=1590 min_rank=6 imbalance_ratio=1.666016 max_load_reduction=25944 max_load_reduction_pct=0.791748 moved_to_echo=13053 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 3, 166, 451, 0, 0, 0, 3, 216, 59, 2457, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 3, 8, 0, 898, 0, 0, 0, 4096, 423, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 34, 0, 0, 10, 0, 69, 2858, 0, 264, 0, 0, 1230, 0, 3271, 745, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4720, 2735, 4031, 909, 7847, 4079, 3201, 5246] avg=4096.0000 max=7847 max_rank=4 min=909 min_rank=3 imbalance_ratio=1.915771 max_load_reduction=24921 max_load_reduction_pct=0.760529 moved_to_echo=20373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 21, 50, 299, 403, 0, 0, 1659, 15, 49, 3532, 95, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 264, 4092, 11, 0, 0, 34, 0, 0, 0, 4074, 1025, 2979, 0, 0, 0, 0, 32, 2131, 0, 0, 2474, 3, 0, 0, 52, 0, 0, 383, 0, 236, 0, 0, 0, 37, 3381, 1341, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[773, 5350, 4096, 4401, 8078, 4640, 671, 4759] avg=4096.0000 max=8078 max_rank=4 min=671 min_rank=6 imbalance_ratio=1.972168 max_load_reduction=24690 max_load_reduction_pct=0.753479 moved_to_echo=18148 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3578, 2211, 598, 430, 0, 0, 0, 0, 79, 0, 2968, 0, 0, 0, 500, 0, 1277, 3315, 0, 0, 0, 0, 0, 0, 257, 0, 1567, 1084, 0, 0, 0, 0, 0, 3178, 0, 0, 0, 0, 0, 8, 0, 156, 3032, 43, 0, 0, 0, 0, 0, 4093, 1096, 252, 0, 0, 22, 0, 351, 2673, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6817, 3047, 5092, 2908, 3178, 3239, 5441, 3046] avg=4096.0000 max=6817 max_rank=0 min=2908 min_rank=3 imbalance_ratio=1.664307 max_load_reduction=25951 max_load_reduction_pct=0.791962 moved_to_echo=14904 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 4096, 128, 20736, 6144, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7936, 768, 640, 3712, 20096, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2048, 384, 18944, 5248, 3456, 2944, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[6784, 0, 7168, 18944, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 8832, 2432, 21760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7808, 2304, 7552, 15360, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9728, 4864, 2048, 384, 8832, 7424, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 2176, 640, 256, 29568, 384, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 3887, 7, 60, 127, 0, 0, 0, 3012, 4056, 0, 39, 825, 164, 4, 0, 4028, 0, 0, 1, 4083, 0, 6, 0, 48, 1332, 0, 0, 4042, 2899, 52, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[69, 179, 0, 405, 573, 3933, 0, 0, 244, 4003, 24, 3, 1459, 14, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 81, 3, 2248, 0, 3667, 3728, 0, 4036, 4007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4012, 4064, 0, 0, 0, 2, 67, 0, 0, 3900, 4095, 6, 2, 2301, 3946, 1301, 1, 2727, 21, 0, 3012, 7, 0, 180, 0, 1, 648, 1394, 2, 2, 0, 1077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[111, 0, 0, 0, 3, 0, 1703, 0, 1, 4000, 27, 0, 1415, 0, 4084, 0, 2112, 0, 0, 0, 207, 0, 0, 4094, 271, 4070, 808, 3823, 1911, 76, 5, 4047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[194, 4060, 39, 289, 219, 0, 2, 0, 0, 4096, 1075, 0, 0, 0, 0, 3641, 5, 29, 0, 4094, 3465, 0, 488, 1, 0, 2827, 956, 4019, 31, 0, 0, 3238] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 2684, 8, 83, 3673, 33, 0, 772, 1, 0, 1822, 3430, 0, 7, 0, 7, 0, 1646, 10, 1008, 34, 1813, 4061, 0, 0, 1103, 2488, 0, 0, 3989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[3847, 169, 0, 2, 8, 14, 1887, 4077, 122, 0, 0, 0, 2230, 3123, 19, 3770, 92, 0, 164, 4096, 4094, 5, 0, 0, 82, 0, 0, 4071, 0, 896, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[22, 3049, 0, 2514, 1, 11, 0, 61, 286, 3658, 2643, 71, 3737, 0, 0, 4035, 3969, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4073, 0, 19, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[69, 170, 0, 405, 0, 0, 0, 0, 573, 3933, 0, 0, 2381, 0, 0, 0, 244, 3453, 24, 3, 0, 0, 0, 0, 1459, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 338, 0, 4092, 0, 0, 81, 203, 9, 0, 0, 3, 2248, 0, 3126, 0, 0, 0, 0, 3728, 0, 4036, 1626, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[644, 6887, 3724, 1473, 888, 4385, 5377, 9390] avg=4096.0000 max=9390 max_rank=7 min=644 min_rank=0 imbalance_ratio=2.292480 max_load_reduction=23378 max_load_reduction_pct=0.713440 moved_to_echo=20040 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[194, 3854, 39, 289, 0, 0, 0, 0, 219, 0, 2, 0, 1924, 1089, 0, 0, 0, 3533, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1198, 0, 0, 0, 0, 5, 29, 0, 4094, 1354, 563, 370, 0, 3465, 0, 488, 1, 222, 206, 0, 0, 0, 2827, 956, 3427, 0, 0, 0, 0, 31, 0, 0, 1314, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4376, 3234, 4608, 1198, 6415, 4382, 7210, 1345] avg=4096.0000 max=7210 max_rank=6 min=1198 min_rank=3 imbalance_ratio=1.760254 max_load_reduction=25558 max_load_reduction_pct=0.779968 moved_to_echo=19352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 3690, 7, 60, 0, 0, 0, 0, 127, 0, 0, 0, 2434, 0, 0, 0, 3012, 3499, 0, 39, 0, 0, 0, 0, 825, 164, 4, 0, 0, 0, 0, 0, 4028, 0, 0, 1, 0, 557, 0, 0, 4083, 0, 6, 0, 0, 197, 0, 0, 48, 1332, 0, 0, 0, 0, 0, 0, 4042, 2899, 52, 1662, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3757, 2561, 6550, 993, 4586, 4286, 1380, 8655] avg=4096.0000 max=8655 max_rank=7 min=993 min_rank=3 imbalance_ratio=2.113037 max_load_reduction=24113 max_load_reduction_pct=0.735870 moved_to_echo=18907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3847, 161, 0, 2, 0, 0, 0, 0, 8, 14, 1887, 4077, 0, 1128, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 2230, 3123, 19, 1240, 0, 0, 0, 0, 92, 0, 164, 4096, 1402, 0, 375, 0, 4094, 5, 0, 0, 225, 8, 0, 0, 82, 0, 0, 3471, 0, 0, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 7114, 122, 6612, 6129, 4332, 3553, 896] avg=4096.0000 max=7114 max_rank=1 min=122 min_rank=2 imbalance_ratio=1.736816 max_load_reduction=25654 max_load_reduction_pct=0.782898 moved_to_echo=14910 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 3885, 0, 2684, 0, 0, 0, 0, 8, 83, 3673, 33, 2373, 5, 0, 0, 0, 663, 1, 0, 0, 0, 0, 0, 1822, 3430, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1646, 2, 109, 104, 0, 10, 1008, 34, 1813, 63, 211, 0, 0, 4061, 0, 0, 936, 0, 0, 0, 0, 2488, 0, 0, 1616, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6569, 6175, 664, 5252, 1868, 3139, 4997, 4104] avg=4096.0000 max=6569 max_rank=0 min=664 min_rank=2 imbalance_ratio=1.603760 max_load_reduction=26199 max_load_reduction_pct=0.799530 moved_to_echo=14108 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[22, 2895, 0, 2514, 0, 0, 0, 0, 1, 11, 0, 61, 308, 1207, 0, 0, 286, 3156, 2643, 71, 0, 0, 0, 0, 3737, 0, 0, 1326, 0, 0, 0, 0, 3969, 0, 0, 0, 1502, 502, 0, 0, 4096, 0, 0, 0, 0, 154, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4073, 0, 19, 211, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5431, 1588, 6156, 5063, 5973, 4250, 4, 4303] avg=4096.0000 max=6156 max_rank=2 min=4 min_rank=6 imbalance_ratio=1.502930 max_load_reduction=26612 max_load_reduction_pct=0.812134 moved_to_echo=14530 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4012, 3858, 0, 0, 0, 0, 0, 0, 0, 2, 67, 0, 640, 389, 0, 0, 0, 3364, 4095, 6, 0, 0, 0, 0, 2, 2301, 3946, 429, 0, 0, 0, 0, 1, 2727, 21, 0, 483, 536, 128, 0, 3012, 7, 0, 180, 77, 206, 0, 0, 0, 1, 648, 1189, 0, 0, 0, 0, 2, 2, 0, 437, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7870, 1098, 7465, 6678, 3896, 3482, 1838, 441] avg=4096.0000 max=7870 max_rank=0 min=441 min_rank=7 imbalance_ratio=1.921387 max_load_reduction=24898 max_load_reduction_pct=0.759827 moved_to_echo=9657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[111, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1703, 0, 2405, 0, 0, 0, 1, 3451, 27, 0, 0, 0, 0, 0, 1415, 0, 4084, 0, 0, 0, 0, 0, 2112, 0, 0, 0, 0, 549, 352, 0, 207, 0, 0, 4094, 211, 0, 0, 0, 271, 4070, 808, 3260, 0, 0, 0, 0, 1911, 76, 5, 1642, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[111, 4111, 3479, 5499, 3013, 4512, 8409, 3634] avg=4096.0000 max=8409 max_rank=6 min=111 min_rank=0 imbalance_ratio=2.052979 max_load_reduction=24359 max_load_reduction_pct=0.743378 moved_to_echo=19568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 4096, 7424, 4224, 12544, 3840, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16384, 3968, 4224, 8576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 18560, 128, 6016, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[11520, 9088, 8064, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[3712, 21120, 7936, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[4480, 10496, 2432, 15488, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10240, 2816, 256, 9856, 4864, 3456, 1792, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[23168, 1024, 640, 6272, 1024, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 102, 141, 0, 4035, 0, 97, 0, 0, 0, 1507, 11, 3, 3, 4054, 0, 0, 0, 3525, 3912, 440, 3734, 25, 30, 17, 9, 3974, 548, 2, 0, 2503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4043, 0, 3866, 0, 3510, 0, 3235, 811, 3674, 7, 0, 0, 3980, 0, 0, 4090, 0, 232, 0, 4095, 134, 0, 0, 5, 0, 2, 408, 0, 0, 139, 537, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 547, 4076, 0, 0, 4068, 0, 2762, 15, 253, 0, 3954, 0, 3542, 0, 1622, 3222, 3, 0, 4096, 10, 0, 608, 0, 0, 0, 54, 95, 0, 0, 1, 3840] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4077, 0, 1, 4096, 2, 1, 95, 522, 3650, 4080, 13, 0, 0, 1, 3886, 3484, 240, 0, 3908, 0, 28, 16, 10, 0, 390, 274, 3994, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 3059, 3207, 4096, 4040, 0, 0, 0, 0, 0, 4077, 103, 14, 0, 3708, 197, 0, 0, 1598, 2, 0, 0, 3785, 0, 0, 0, 788] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[470, 0, 131, 0, 3052, 624, 10, 3074, 788, 0, 0, 0, 403, 3860, 0, 4031, 0, 0, 5, 3904, 0, 1, 4096, 3726, 201, 9, 0, 28, 0, 0, 260, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[2027, 15, 36, 0, 151, 328, 0, 2, 4079, 126, 3082, 11, 3462, 4094, 1, 4087, 0, 3, 200, 1726, 18, 0, 1855, 0, 0, 0, 1, 4096, 0, 0, 0, 3368] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 0, 3061, 0, 91, 3, 793, 1720, 0, 3601, 0, 0, 0, 1521, 4068, 0, 0, 3850, 359, 0, 1542, 1618, 0, 2232, 24, 0, 14, 55, 23, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4043, 0, 3866, 0, 550, 319, 0, 0, 640, 0, 3235, 811, 0, 0, 0, 0, 3674, 7, 0, 0, 57, 0, 0, 0, 3980, 0, 0, 987, 0, 0, 0, 0, 0, 232, 0, 3035, 0, 0, 0, 0, 134, 0, 0, 5, 684, 0, 0, 0, 0, 2, 408, 0, 3091, 0, 0, 0, 0, 139, 537, 0, 2320, 12, 0, 0] rerouted_tokens_per_ep_rank=[8778, 4686, 3738, 4967, 3267, 823, 3501, 3008] avg=4096.0000 max=8778 max_rank=0 min=823 min_rank=5 imbalance_ratio=2.143066 max_load_reduction=23990 max_load_reduction_pct=0.732117 moved_to_echo=10599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 547, 4076, 0, 0, 319, 0, 0, 0, 4068, 0, 2762, 0, 0, 0, 0, 15, 253, 0, 3954, 58, 0, 0, 0, 0, 3542, 0, 393, 0, 0, 0, 0, 3222, 3, 0, 3035, 0, 0, 0, 0, 10, 0, 608, 0, 684, 0, 0, 0, 0, 0, 54, 95, 1225, 0, 0, 0, 0, 0, 1, 3840, 0, 4, 0, 0] rerouted_tokens_per_ep_rank=[4942, 6830, 4280, 3935, 6260, 1302, 1374, 3845] avg=4096.0000 max=6830 max_rank=1 min=1302 min_rank=5 imbalance_ratio=1.667480 max_load_reduction=25938 max_load_reduction_pct=0.791565 moved_to_echo=12781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 102, 141, 0, 274, 0, 0, 0, 4035, 0, 97, 0, 0, 0, 0, 0, 0, 0, 1507, 49, 0, 0, 0, 11, 3, 3, 980, 0, 0, 0, 0, 0, 0, 0, 2614, 0, 0, 0, 0, 3912, 440, 3734, 25, 588, 0, 0, 0, 30, 17, 9, 3974, 3063, 0, 0, 0, 548, 2, 0, 2503, 0, 11, 0, 0] rerouted_tokens_per_ep_rank=[4613, 4132, 1556, 997, 2614, 8699, 7093, 3064] avg=4096.0000 max=8699 max_rank=5 min=997 min_rank=3 imbalance_ratio=2.123779 max_load_reduction=24069 max_load_reduction_pct=0.734528 moved_to_echo=21470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 639, 271, 0, 0, 743, 0, 1, 4096, 0, 0, 0, 0, 2, 1, 95, 522, 49, 0, 0, 0, 3650, 4080, 13, 0, 0, 0, 0, 0, 0, 1, 3886, 2583, 0, 0, 0, 0, 240, 0, 3908, 0, 581, 0, 0, 0, 28, 16, 10, 0, 0, 0, 0, 0, 390, 274, 3994, 0, 2695, 0, 0, 0] rerouted_tokens_per_ep_rank=[910, 4840, 669, 7743, 6470, 4729, 54, 7353] avg=4096.0000 max=7743 max_rank=3 min=54 min_rank=6 imbalance_ratio=1.890381 max_load_reduction=25025 max_load_reduction_pct=0.763702 moved_to_echo=18606 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2027, 15, 36, 0, 23, 134, 0, 0, 29, 328, 0, 2, 0, 0, 0, 0, 4079, 126, 3082, 11, 24, 0, 0, 0, 3462, 4094, 1, 987, 0, 0, 0, 0, 0, 3, 200, 1280, 0, 0, 0, 0, 18, 0, 1855, 0, 288, 0, 0, 0, 0, 0, 1, 4096, 3088, 0, 0, 0, 0, 0, 0, 3368, 99, 12, 0, 0] rerouted_tokens_per_ep_rank=[2235, 359, 7322, 8544, 1483, 2161, 7185, 3479] avg=4096.0000 max=8544 max_rank=3 min=359 min_rank=1 imbalance_ratio=2.085938 max_load_reduction=24224 max_load_reduction_pct=0.739258 moved_to_echo=14308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[470, 0, 131, 0, 478, 304, 0, 0, 556, 624, 10, 3074, 0, 0, 0, 0, 788, 0, 0, 0, 55, 0, 0, 0, 403, 3860, 0, 974, 0, 0, 0, 0, 0, 0, 5, 2893, 0, 0, 0, 0, 0, 1, 4096, 3726, 652, 0, 0, 0, 201, 9, 0, 28, 3046, 0, 0, 0, 0, 0, 260, 4095, 2018, 11, 0, 0] rerouted_tokens_per_ep_rank=[1383, 4264, 843, 5237, 2898, 8475, 3284, 6384] avg=4096.0000 max=8475 max_rank=5 min=843 min_rank=2 imbalance_ratio=2.069092 max_load_reduction=24293 max_load_reduction_pct=0.741364 moved_to_echo=21041 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4095, 0, 483, 303, 0, 0, 551, 0, 91, 3, 0, 0, 0, 0, 793, 1720, 0, 3601, 58, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 4068, 0, 0, 2843, 0, 0, 0, 0, 359, 0, 1542, 1618, 646, 0, 0, 0, 0, 2232, 24, 0, 1153, 0, 0, 0, 14, 55, 23, 2, 2027, 8, 0, 0] rerouted_tokens_per_ep_rank=[8977, 645, 6172, 360, 6911, 4165, 3409, 2129] avg=4096.0000 max=8977 max_rank=0 min=360 min_rank=3 imbalance_ratio=2.191650 max_load_reduction=23791 max_load_reduction_pct=0.726044 moved_to_echo=16614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 641, 289, 0, 0, 746, 0, 3059, 3207, 0, 0, 0, 0, 4096, 4040, 0, 0, 52, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 0, 103, 14, 0, 2748, 0, 0, 0, 0, 197, 0, 0, 1598, 619, 0, 0, 0, 2, 0, 0, 3785, 3081, 0, 0, 0, 0, 0, 0, 788, 2707, 11, 0, 0] rerouted_tokens_per_ep_rank=[930, 7012, 8188, 985, 2865, 2414, 6868, 3506] avg=4096.0000 max=8188 max_rank=2 min=930 min_rank=0 imbalance_ratio=1.999023 max_load_reduction=24580 max_load_reduction_pct=0.750122 moved_to_echo=15653 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10752, 4736, 12416, 256, 2816, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[7424, 256, 4096, 21120, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4992, 512, 15744, 7040, 4864, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11520, 15616, 128, 5760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13568, 6272, 3200, 9600, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 2304, 512, 12032, 17792, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[3328, 9088, 6400, 14080, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 512, 4864, 14720, 11904, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4025, 0, 67, 280, 50, 47, 450, 0, 3278, 2, 259, 3832, 2684, 5, 0, 0, 1417, 0, 297, 3156, 4094, 0, 95, 0, 543, 4096, 4, 4087, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4033, 1649, 1808, 41, 1, 2718, 0, 0, 3312, 358, 1, 0, 17, 0, 0, 163, 16, 3661, 412, 1, 2425, 0, 0, 161, 3799, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1480, 3640, 2619, 0, 0, 4032, 0, 117, 99, 4096, 81, 299, 0, 0, 0, 672, 0, 4, 197, 0, 340, 3890, 0, 0, 0, 0, 3754, 0, 4096, 0, 3352, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[306, 451, 0, 0, 0, 4096, 4013, 2, 31, 0, 0, 3768, 0, 0, 11, 2987, 2516, 1969, 4096, 3, 61, 3993, 0, 0, 3, 266, 434, 3703, 0, 10, 38, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 3965, 4065, 0, 202, 289, 0, 379, 98, 1438, 1930, 1, 643, 0, 553, 4079, 2583, 3448, 15, 0, 30, 2890, 0, 0, 0, 0, 3141, 35, 2984, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 3209, 0, 1, 1260, 16, 0, 0, 2835, 0, 22, 1, 0, 233, 953, 1811, 1986, 4095, 0, 88, 3738, 4096, 28, 0, 0, 0, 4089, 0, 596, 6, 3705, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 3204, 4096, 0, 332, 235, 0, 0, 222, 0, 2893, 2, 429, 13, 259, 44, 0, 4043, 0, 4073, 4091, 541, 0, 0, 2139, 0, 3420, 143, 2588, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4066, 0, 0, 122, 4096, 654, 2, 7, 19, 0, 0, 104, 1825, 40, 0, 0, 4085, 0, 0, 4007, 3602, 66, 0, 0, 0, 4085, 0, 1910, 0, 4062, 16] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2525, 0, 67, 0, 0, 0, 0, 280, 50, 47, 450, 1077, 194, 0, 0, 0, 3278, 2, 259, 594, 280, 0, 0, 3832, 2684, 5, 0, 1500, 265, 0, 0, 0, 943, 0, 297, 0, 0, 0, 0, 3156, 2752, 0, 95, 0, 0, 0, 0, 0, 543, 3320, 4, 0, 0, 0, 0, 4087, 0, 0, 0, 182, 0, 0, 0] rerouted_tokens_per_ep_rank=[2592, 2098, 4413, 8286, 1240, 6003, 3867, 4269] avg=4096.0000 max=8286 max_rank=3 min=1240 min_rank=4 imbalance_ratio=2.022949 max_load_reduction=24482 max_load_reduction_pct=0.747131 moved_to_echo=15379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2526, 1649, 1808, 0, 0, 0, 0, 41, 1, 2718, 0, 642, 5, 0, 0, 0, 3312, 358, 1, 597, 7, 0, 0, 0, 17, 0, 0, 1507, 162, 0, 0, 163, 4, 3661, 412, 0, 0, 0, 0, 1, 1621, 0, 0, 0, 0, 0, 0, 161, 3799, 3313, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 186, 0, 0, 0] rerouted_tokens_per_ep_rank=[5983, 3407, 4275, 1686, 4240, 1622, 7273, 4282] avg=4096.0000 max=7273 max_rank=6 min=1622 min_rank=5 imbalance_ratio=1.775635 max_load_reduction=25495 max_load_reduction_pct=0.778046 moved_to_echo=17417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2550, 0, 0, 0, 0, 0, 0, 122, 4096, 654, 2, 947, 560, 0, 0, 7, 19, 0, 0, 592, 810, 0, 0, 104, 1825, 40, 0, 1516, 233, 0, 0, 0, 2715, 0, 0, 0, 0, 0, 0, 4007, 2422, 66, 0, 0, 0, 0, 0, 0, 0, 3311, 0, 0, 0, 0, 0, 1910, 0, 4062, 16, 182, 0, 0, 0] rerouted_tokens_per_ep_rank=[2550, 6381, 1428, 3718, 2715, 6495, 3311, 6170] avg=4096.0000 max=6495 max_rank=5 min=1428 min_rank=2 imbalance_ratio=1.585693 max_load_reduction=26273 max_load_reduction_pct=0.801788 moved_to_echo=18691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2013, 0, 1, 0, 0, 0, 0, 1260, 16, 0, 0, 1077, 561, 0, 0, 2835, 0, 22, 1, 593, 812, 0, 0, 0, 233, 953, 1811, 1196, 265, 0, 0, 1986, 2722, 0, 88, 0, 0, 0, 0, 3738, 2754, 28, 0, 0, 0, 0, 0, 0, 0, 3314, 0, 0, 0, 0, 0, 596, 6, 3705, 0, 182, 0, 0, 0] rerouted_tokens_per_ep_rank=[2014, 2914, 4263, 4458, 4796, 6520, 3314, 4489] avg=4096.0000 max=6520 max_rank=5 min=2014 min_rank=0 imbalance_ratio=1.591797 max_load_reduction=26248 max_load_reduction_pct=0.801025 moved_to_echo=19119 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1480, 2283, 2619, 0, 0, 0, 0, 0, 0, 4032, 0, 117, 1023, 0, 0, 0, 99, 4096, 81, 299, 544, 0, 0, 0, 0, 0, 0, 672, 1357, 252, 0, 0, 0, 4, 197, 0, 0, 0, 0, 0, 340, 2615, 0, 0, 0, 0, 0, 0, 0, 0, 3043, 0, 0, 0, 0, 0, 4096, 0, 3352, 0, 167, 0, 0, 0] rerouted_tokens_per_ep_rank=[6382, 5172, 5119, 2281, 201, 2955, 3043, 7615] avg=4096.0000 max=7615 max_rank=7 min=201 min_rank=4 imbalance_ratio=1.859131 max_load_reduction=25153 max_load_reduction_pct=0.767609 moved_to_echo=13814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[306, 283, 0, 0, 0, 0, 0, 0, 0, 4096, 4013, 2, 1050, 270, 0, 0, 31, 0, 0, 3768, 62, 390, 0, 0, 0, 0, 11, 2987, 168, 259, 0, 0, 2516, 1309, 4096, 3, 0, 0, 0, 0, 61, 2684, 0, 0, 0, 0, 0, 0, 3, 266, 353, 3703, 0, 0, 0, 0, 0, 10, 38, 11, 19, 0, 0, 0] rerouted_tokens_per_ep_rank=[589, 9431, 4251, 3425, 7924, 2745, 4325, 78] avg=4096.0000 max=9431 max_rank=1 min=78 min_rank=7 imbalance_ratio=2.302490 max_load_reduction=23337 max_load_reduction_pct=0.712189 moved_to_echo=15072 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2010, 4096, 0, 0, 0, 0, 0, 332, 235, 0, 0, 142, 554, 0, 0, 222, 0, 2893, 2, 496, 801, 0, 0, 429, 13, 259, 44, 1194, 35, 0, 0, 0, 2688, 0, 4073, 0, 0, 0, 0, 4091, 364, 0, 0, 0, 0, 0, 0, 2139, 0, 2772, 143, 0, 0, 0, 0, 2588, 0, 0, 1, 152, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 1263, 4414, 1974, 6761, 4455, 5054, 2741] avg=4096.0000 max=6761 max_rank=4 min=1263 min_rank=1 imbalance_ratio=1.650635 max_load_reduction=26007 max_load_reduction_pct=0.793671 moved_to_echo=19011 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2487, 4065, 0, 0, 0, 0, 0, 202, 289, 0, 379, 760, 472, 0, 0, 98, 1438, 1930, 1, 455, 683, 0, 0, 643, 0, 553, 4079, 1478, 187, 0, 0, 2583, 2293, 15, 0, 0, 0, 0, 0, 30, 1943, 0, 0, 0, 0, 0, 0, 0, 0, 2546, 35, 0, 0, 0, 0, 2984, 0, 0, 0, 140, 0, 0, 0] rerouted_tokens_per_ep_rank=[6552, 2102, 4605, 6940, 4891, 1973, 2581, 3124] avg=4096.0000 max=6940 max_rank=3 min=1973 min_rank=5 imbalance_ratio=1.694336 max_load_reduction=25828 max_load_reduction_pct=0.788208 moved_to_echo=12569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2304, 12928, 7552, 1024, 6784, 2688, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5120, 4864, 1920, 9600, 9984, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1792, 16768, 12544, 1920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7296, 12800, 8064, 4992, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15488, 17280, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[20480, 128, 11264, 128, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[2304, 4608, 22016, 3968, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3328, 12160, 5376, 4352, 3968, 3840, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[155, 3006, 960, 333, 1630, 442, 882, 1251, 923, 75, 1268, 317, 1452, 777, 254, 525, 304, 555, 1475, 1628, 787, 90, 2853, 1570, 587, 727, 971, 1239, 1746, 621, 1290, 2075] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[272, 3062, 785, 418, 1272, 492, 820, 1116, 1436, 198, 1601, 413, 1165, 759, 151, 400, 200, 793, 1612, 741, 996, 169, 2792, 1858, 364, 846, 1155, 1035, 1570, 612, 1822, 1843] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[63, 3172, 553, 326, 1528, 330, 913, 1050, 1524, 108, 1672, 165, 1135, 1284, 559, 438, 73, 576, 875, 530, 1202, 199, 2832, 2195, 895, 951, 909, 1028, 1230, 484, 2000, 1969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[444, 2843, 402, 566, 1702, 349, 808, 1091, 1451, 179, 1569, 222, 1174, 1008, 245, 480, 105, 659, 1193, 751, 829, 249, 3176, 2338, 745, 861, 878, 1173, 1063, 617, 1913, 1685] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[51, 2838, 577, 460, 1942, 421, 802, 690, 1115, 128, 2059, 299, 1073, 731, 200, 561, 329, 1008, 648, 895, 718, 221, 2797, 2359, 667, 1567, 1119, 1287, 1027, 367, 1425, 2387] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[69, 2868, 1123, 433, 1774, 609, 1202, 1014, 962, 170, 1873, 637, 1323, 583, 299, 490, 230, 429, 970, 825, 1074, 74, 3545, 2225, 734, 1076, 922, 987, 1171, 289, 1108, 1680] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[81, 2554, 819, 238, 1275, 316, 966, 708, 1425, 217, 1658, 326, 544, 1113, 367, 480, 137, 700, 1716, 839, 1049, 286, 2945, 2186, 655, 1288, 849, 1259, 1446, 417, 2142, 1767] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[74, 2616, 598, 297, 1685, 341, 1179, 1150, 1155, 175, 1307, 369, 1160, 937, 430, 364, 70, 488, 628, 997, 1371, 100, 3477, 2062, 795, 967, 1091, 1277, 1421, 282, 1555, 2350] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[155, 2969, 960, 333, 0, 0, 0, 0, 1630, 442, 882, 1251, 96, 37, 0, 0, 923, 75, 1268, 317, 760, 0, 0, 0, 1452, 777, 254, 525, 1204, 0, 0, 0, 304, 555, 1475, 1628, 882, 294, 0, 0, 787, 90, 767, 1570, 0, 0, 0, 0, 587, 727, 971, 1239, 245, 0, 0, 0, 1746, 621, 1290, 680, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4417, 4338, 3343, 4212, 5138, 3214, 3769, 4337] avg=4096.0000 max=5138 max_rank=4 min=3214 min_rank=5 imbalance_ratio=1.254395 max_load_reduction=27630 max_load_reduction_pct=0.843201 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[272, 3024, 785, 418, 0, 0, 0, 0, 1272, 492, 820, 1116, 85, 38, 0, 0, 1436, 198, 1601, 413, 675, 0, 0, 0, 1165, 759, 151, 400, 1178, 0, 0, 0, 200, 793, 1612, 741, 863, 262, 0, 0, 996, 169, 751, 1858, 0, 0, 0, 0, 364, 846, 1155, 1035, 218, 0, 0, 0, 1570, 612, 1822, 603, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4499, 3823, 4323, 3653, 4471, 3774, 3618, 4607] avg=4096.0000 max=4607 max_rank=7 min=3618 min_rank=6 imbalance_ratio=1.124756 max_load_reduction=28161 max_load_reduction_pct=0.859406 moved_to_echo=16470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[63, 3133, 553, 326, 0, 0, 0, 0, 1528, 330, 913, 1050, 91, 39, 0, 0, 1524, 108, 1672, 165, 721, 0, 0, 0, 1135, 1284, 559, 438, 1195, 0, 0, 0, 73, 576, 875, 530, 875, 279, 0, 0, 1202, 199, 762, 2195, 0, 0, 0, 0, 895, 951, 909, 1028, 232, 0, 0, 0, 1230, 484, 2000, 646, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 3951, 4190, 4611, 3208, 4358, 4015, 4360] avg=4096.0000 max=4611 max_rank=3 min=3208 min_rank=4 imbalance_ratio=1.125732 max_load_reduction=28157 max_load_reduction_pct=0.859283 moved_to_echo=15941 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[69, 2833, 1123, 433, 0, 0, 0, 0, 1774, 609, 1202, 1014, 77, 35, 0, 0, 962, 170, 1873, 637, 615, 0, 0, 0, 1323, 583, 299, 490, 1496, 0, 0, 0, 230, 429, 970, 825, 1096, 238, 0, 0, 1074, 74, 953, 2225, 0, 0, 0, 0, 734, 1076, 922, 987, 198, 0, 0, 0, 1171, 289, 1108, 552, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4458, 4711, 4257, 4191, 3788, 4326, 3917, 3120] avg=4096.0000 max=4711 max_rank=1 min=3120 min_rank=7 imbalance_ratio=1.150146 max_load_reduction=28057 max_load_reduction_pct=0.856232 moved_to_echo=15151 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[444, 2803, 402, 566, 0, 0, 0, 0, 1702, 349, 808, 1091, 82, 40, 0, 0, 1451, 179, 1569, 222, 620, 0, 0, 0, 1174, 1008, 245, 480, 1344, 0, 0, 0, 105, 659, 1193, 751, 985, 243, 0, 0, 829, 249, 847, 2338, 0, 0, 0, 0, 745, 861, 878, 1173, 202, 0, 0, 0, 1063, 617, 1913, 538, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4215, 4072, 4041, 4251, 3936, 4263, 3859, 4131] avg=4096.0000 max=4263 max_rank=5 min=3859 min_rank=6 imbalance_ratio=1.040771 max_load_reduction=28505 max_load_reduction_pct=0.869904 moved_to_echo=16189 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[74, 2584, 598, 297, 0, 0, 0, 0, 1685, 341, 1179, 1150, 108, 32, 0, 0, 1155, 175, 1307, 369, 860, 0, 0, 0, 1160, 937, 430, 364, 1467, 0, 0, 0, 70, 488, 628, 997, 1074, 334, 0, 0, 1371, 100, 936, 2062, 0, 0, 0, 0, 795, 967, 1091, 1277, 278, 0, 0, 0, 1421, 282, 1555, 770, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3553, 4495, 3866, 4358, 3591, 4469, 4408, 4028] avg=4096.0000 max=4495 max_rank=1 min=3553 min_rank=0 imbalance_ratio=1.097412 max_load_reduction=28273 max_load_reduction_pct=0.862823 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[51, 2803, 577, 460, 0, 0, 0, 0, 1942, 421, 802, 690, 110, 35, 0, 0, 1115, 128, 2059, 299, 874, 0, 0, 0, 1073, 731, 200, 561, 1180, 0, 0, 0, 329, 1008, 648, 895, 864, 339, 0, 0, 718, 221, 753, 2359, 0, 0, 0, 0, 667, 1567, 1119, 1287, 282, 0, 0, 0, 1027, 367, 1425, 782, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3891, 4000, 4475, 3745, 4083, 4051, 4922, 3601] avg=4096.0000 max=4922 max_rank=6 min=3601 min_rank=7 imbalance_ratio=1.201660 max_load_reduction=27846 max_load_reduction_pct=0.849792 moved_to_echo=16657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[81, 2522, 819, 238, 0, 0, 0, 0, 1275, 316, 966, 708, 81, 32, 0, 0, 1425, 217, 1658, 326, 647, 0, 0, 0, 544, 1113, 367, 480, 1243, 0, 0, 0, 137, 700, 1716, 839, 910, 251, 0, 0, 1049, 286, 792, 2186, 0, 0, 0, 0, 655, 1288, 849, 1259, 209, 0, 0, 0, 1446, 417, 2142, 579, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3660, 3378, 4273, 3747, 4553, 4313, 4260, 4584] avg=4096.0000 max=4584 max_rank=7 min=3378 min_rank=1 imbalance_ratio=1.119141 max_load_reduction=28184 max_load_reduction_pct=0.860107 moved_to_echo=17710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1536, 5248, 9216, 7296, 7552, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10752, 3712, 13312, 5248, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10112, 1280, 13056, 2816, 5888, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 1408, 6656, 16896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5504, 8320, 7936, 9344, 1920, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[12928, 3328, 7680, 8192, 768, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9088, 7296, 2560, 3840, 10368, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 22784, 5888, 3072, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[265, 5, 87, 455, 2707, 3667, 2319, 3774, 7, 0, 284, 236, 7, 3518, 17, 46, 0, 694, 0, 3652, 86, 21, 6, 0, 2592, 12, 56, 4065, 4087, 0, 103, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[523, 16, 1264, 3, 1761, 3936, 469, 3138, 107, 0, 2381, 1527, 9, 1514, 7, 1635, 0, 11, 0, 4093, 2, 3, 27, 14, 1204, 220, 41, 4096, 4054, 1, 126, 586] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[226, 7, 1681, 240, 1292, 745, 3185, 1317, 113, 0, 133, 880, 3, 2461, 1622, 457, 139, 2029, 4, 4063, 16, 0, 1, 10, 3789, 5, 356, 4060, 3908, 20, 2, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[123, 0, 888, 476, 3407, 3652, 542, 3331, 172, 0, 0, 910, 17, 3835, 5, 1736, 575, 72, 0, 4083, 1, 43, 0, 361, 237, 24, 257, 3921, 4090, 5, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[350, 670, 1076, 1, 2155, 3493, 268, 3809, 3, 4, 432, 816, 458, 3526, 2, 2202, 2, 1402, 9, 4085, 1, 5, 0, 5, 72, 15, 490, 3533, 3878, 4, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[69, 0, 1092, 10, 3932, 3693, 746, 158, 6, 59, 886, 807, 92, 3880, 15, 372, 257, 1753, 0, 3463, 1268, 156, 14, 6, 1995, 33, 54, 3768, 4032, 19, 133, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[362, 0, 2790, 2, 3301, 3825, 767, 2379, 251, 0, 104, 513, 14, 4049, 27, 1466, 0, 18, 0, 4046, 12, 277, 2, 29, 540, 41, 44, 4072, 3803, 1, 15, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2150, 0, 1766, 38, 3417, 1267, 519, 575, 129, 0, 145, 398, 25, 3573, 0, 2966, 114, 1648, 4, 3998, 57, 2, 90, 4, 850, 190, 1283, 3588, 3808, 0, 4, 160] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[265, 5, 87, 455, 863, 906, 235, 12, 675, 0, 2319, 3774, 0, 0, 0, 0, 7, 0, 284, 236, 1429, 1297, 0, 0, 7, 2612, 17, 46, 0, 0, 0, 0, 0, 694, 0, 2789, 0, 0, 0, 0, 86, 21, 6, 0, 3667, 735, 0, 0, 2592, 12, 56, 2401, 0, 0, 0, 0, 4075, 0, 103, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2828, 6768, 3253, 2682, 3483, 4515, 5061, 4178] avg=4096.0000 max=6768 max_rank=1 min=2682 min_rank=3 imbalance_ratio=1.652344 max_load_reduction=26000 max_load_reduction_pct=0.793457 moved_to_echo=17237 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[226, 7, 1681, 240, 960, 633, 234, 12, 322, 0, 3185, 1317, 0, 0, 0, 0, 113, 0, 133, 880, 1427, 619, 0, 0, 3, 1828, 1622, 457, 0, 0, 0, 0, 139, 2029, 4, 3103, 0, 0, 0, 0, 16, 0, 1, 10, 745, 351, 0, 0, 3789, 5, 356, 2399, 0, 0, 0, 0, 3896, 20, 2, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3993, 4824, 3172, 3910, 5275, 1123, 6549, 3922] avg=4096.0000 max=6549 max_rank=6 min=1123 min_rank=5 imbalance_ratio=1.598877 max_load_reduction=26219 max_load_reduction_pct=0.800140 moved_to_echo=16869 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[123, 0, 888, 476, 969, 991, 231, 16, 842, 0, 542, 3331, 0, 0, 0, 0, 172, 0, 0, 910, 1382, 1636, 0, 0, 17, 2844, 5, 1736, 0, 0, 0, 0, 575, 72, 0, 3114, 0, 0, 0, 0, 1, 43, 0, 361, 3652, 929, 0, 0, 237, 24, 257, 2308, 0, 0, 0, 0, 4074, 5, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3694, 4715, 4100, 4602, 3761, 4986, 2826, 4084] avg=4096.0000 max=4986 max_rank=5 min=2826 min_rank=6 imbalance_ratio=1.217285 max_load_reduction=27782 max_load_reduction_pct=0.847839 moved_to_echo=15657 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 16, 1264, 3, 968, 389, 236, 12, 440, 0, 469, 3138, 0, 0, 0, 0, 107, 0, 2381, 1527, 1440, 843, 0, 0, 9, 1125, 7, 1635, 0, 0, 0, 0, 0, 11, 0, 3125, 0, 0, 0, 0, 2, 3, 27, 14, 3936, 478, 0, 0, 1204, 220, 41, 2420, 0, 0, 0, 0, 4042, 1, 126, 586, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3411, 4047, 6298, 2776, 3136, 4460, 3885, 4755] avg=4096.0000 max=6298 max_rank=2 min=2776 min_rank=3 imbalance_ratio=1.537598 max_load_reduction=26470 max_load_reduction_pct=0.807800 moved_to_echo=16236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[350, 670, 1076, 1, 966, 908, 204, 12, 538, 0, 268, 3809, 0, 0, 0, 0, 3, 4, 432, 816, 1242, 1032, 0, 0, 458, 2618, 2, 2202, 0, 0, 0, 0, 2, 1402, 9, 3119, 0, 0, 0, 0, 1, 5, 0, 5, 3493, 585, 0, 0, 72, 15, 490, 2087, 0, 0, 0, 0, 3866, 4, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4187, 4615, 3529, 5280, 4532, 4089, 2664, 3872] avg=4096.0000 max=5280 max_rank=3 min=2664 min_rank=6 imbalance_ratio=1.289062 max_load_reduction=27488 max_load_reduction_pct=0.838867 moved_to_echo=15157 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[69, 0, 1092, 10, 819, 999, 217, 12, 980, 0, 746, 158, 0, 0, 0, 0, 6, 59, 886, 807, 1324, 1884, 0, 0, 92, 2881, 15, 372, 0, 0, 0, 0, 257, 1753, 0, 2644, 0, 0, 0, 0, 1268, 156, 14, 6, 3693, 1068, 0, 0, 1995, 33, 54, 2227, 0, 0, 0, 0, 4020, 19, 133, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3218, 1884, 4966, 3360, 4654, 6205, 4309, 4172] avg=4096.0000 max=6205 max_rank=5 min=1884 min_rank=1 imbalance_ratio=1.514893 max_load_reduction=26563 max_load_reduction_pct=0.810638 moved_to_echo=19340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2150, 0, 1766, 38, 945, 920, 207, 12, 852, 0, 519, 575, 0, 0, 0, 0, 129, 0, 145, 398, 1261, 1637, 0, 0, 25, 2653, 0, 2966, 0, 0, 0, 0, 114, 1648, 4, 3053, 0, 0, 0, 0, 57, 2, 90, 4, 1267, 928, 0, 0, 850, 190, 1283, 2120, 0, 0, 0, 0, 3796, 0, 4, 160, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6038, 1946, 3570, 5644, 4819, 2348, 4443, 3960] avg=4096.0000 max=6038 max_rank=0 min=1946 min_rank=1 imbalance_ratio=1.474121 max_load_reduction=26730 max_load_reduction_pct=0.815735 moved_to_echo=15570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[362, 0, 2790, 2, 956, 1042, 235, 12, 823, 0, 767, 2379, 0, 0, 0, 0, 251, 0, 104, 513, 1431, 1581, 0, 0, 14, 3007, 27, 1466, 0, 0, 0, 0, 0, 18, 0, 3090, 0, 0, 0, 0, 12, 277, 2, 29, 3825, 897, 0, 0, 540, 41, 44, 2406, 0, 0, 0, 0, 3791, 1, 15, 18, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5399, 3969, 3880, 4514, 3108, 5042, 3031, 3825] avg=4096.0000 max=5399 max_rank=0 min=3031 min_rank=6 imbalance_ratio=1.318115 max_load_reduction=27369 max_load_reduction_pct=0.835236 moved_to_echo=15006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 768, 10752, 1280, 7552, 6912, 1920, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[896, 128, 4480, 6144, 11008, 10624, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[5504, 0, 8832, 18560, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11392, 640, 2688, 18432, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 7680, 128, 24064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[640, 19584, 1792, 10880, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1536, 512, 256, 512, 24320, 6016, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31616, 128, 512, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1401, 3837, 180, 288, 0, 92, 4085, 911, 0, 2788, 1695, 3, 1230, 399, 2496, 0, 0, 4062, 10, 0, 0, 2243, 183, 3891, 1941, 0, 3, 0, 940, 87, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3008, 3451, 142, 4072, 76, 216, 4082, 0, 2, 4094, 105, 0, 28, 2504, 4, 1648, 7, 3494, 3812, 0, 3, 751, 0, 3, 0, 0, 4, 675, 587, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4024, 1533, 1, 4023, 4, 150, 4096, 1, 0, 465, 3446, 0, 193, 82, 0, 2, 360, 4096, 27, 1731, 7, 183, 10, 0, 30, 0, 161, 51, 4070, 0, 104, 3918] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1739, 3106, 0, 127, 0, 15, 1567, 0, 0, 4094, 3018, 149, 3386, 3087, 1, 0, 19, 4096, 3463, 0, 1064, 260, 35, 497, 1254, 0, 0, 16, 1744, 0, 0, 31] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1150, 3775, 0, 2340, 18, 34, 4091, 0, 0, 584, 2824, 2, 78, 3551, 1, 220, 0, 4093, 3041, 2, 0, 110, 0, 3078, 125, 0, 62, 32, 143, 17, 36, 3361] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3852, 215, 19, 63, 173, 1, 4096, 91, 0, 3697, 3352, 135, 29, 940, 7, 0, 0, 4078, 2680, 112, 0, 3002, 33, 1434, 36, 352, 17, 0, 3371, 0, 301, 682] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3869, 666, 469, 2174, 484, 0, 3731, 5, 3, 3408, 3566, 0, 3166, 1530, 17, 36, 33, 4089, 151, 2336, 0, 361, 4, 2, 1192, 0, 0, 0, 4, 1472, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[2049, 2092, 2233, 2854, 1974, 14, 2224, 1, 2, 114, 3564, 0, 4037, 94, 277, 0, 25, 3850, 4079, 0, 291, 1005, 0, 7, 1331, 2, 0, 542, 10, 36, 60, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2548, 142, 4072, 0, 0, 0, 0, 76, 216, 4082, 0, 99, 0, 0, 0, 2, 4094, 65, 0, 0, 0, 0, 0, 28, 2504, 4, 1648, 688, 0, 0, 0, 7, 1195, 3812, 0, 0, 0, 0, 0, 3, 751, 0, 3, 1569, 0, 0, 0, 0, 0, 4, 675, 3008, 422, 0, 0, 587, 0, 0, 0, 40, 308, 116, 0] rerouted_tokens_per_ep_rank=[6762, 4473, 4161, 4872, 5014, 2326, 4109, 1051] avg=4096.0000 max=6762 max_rank=0 min=1051 min_rank=7 imbalance_ratio=1.650879 max_load_reduction=26006 max_load_reduction_pct=0.793640 moved_to_echo=12500 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2833, 180, 288, 0, 0, 0, 0, 0, 92, 4085, 911, 110, 0, 0, 0, 0, 2788, 1040, 3, 0, 0, 0, 0, 1230, 399, 2496, 0, 765, 0, 0, 0, 0, 1389, 10, 0, 0, 0, 0, 0, 0, 2243, 183, 3891, 1824, 0, 0, 0, 1941, 0, 3, 0, 1401, 490, 0, 0, 940, 87, 0, 3, 655, 359, 129, 0] rerouted_tokens_per_ep_rank=[3301, 5198, 3831, 4890, 1399, 8141, 3835, 2173] avg=4096.0000 max=8141 max_rank=5 min=1399 min_rank=4 imbalance_ratio=1.987549 max_load_reduction=24627 max_load_reduction_pct=0.751556 moved_to_echo=15548 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2294, 0, 127, 0, 0, 0, 0, 0, 15, 1567, 0, 89, 0, 0, 0, 0, 4094, 1851, 149, 0, 0, 0, 0, 3386, 3087, 1, 0, 619, 0, 0, 0, 19, 1401, 3463, 0, 0, 0, 0, 0, 1064, 260, 35, 497, 1839, 0, 0, 0, 1254, 0, 0, 16, 1739, 494, 0, 0, 1744, 0, 0, 31, 1167, 362, 104, 0] rerouted_tokens_per_ep_rank=[2421, 1671, 6094, 7093, 4883, 3695, 3503, 3408] avg=4096.0000 max=7093 max_rank=3 min=1671 min_rank=1 imbalance_ratio=1.731689 max_load_reduction=25675 max_load_reduction_pct=0.783539 moved_to_echo=15489 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 493, 469, 2174, 0, 0, 0, 0, 484, 0, 3731, 5, 19, 0, 0, 0, 3, 3408, 2187, 0, 0, 0, 0, 0, 3166, 1530, 17, 36, 132, 0, 0, 0, 33, 1398, 151, 2336, 0, 0, 0, 0, 0, 361, 4, 2, 1836, 0, 0, 0, 1192, 0, 0, 0, 3869, 494, 0, 0, 4, 1472, 0, 0, 1379, 361, 22, 0] rerouted_tokens_per_ep_rank=[3136, 4239, 5598, 4881, 3918, 2203, 5555, 3238] avg=4096.0000 max=5598 max_rank=2 min=2203 min_rank=5 imbalance_ratio=1.366699 max_load_reduction=27170 max_load_reduction_pct=0.829163 moved_to_echo=14914 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1122, 1, 4023, 0, 0, 0, 0, 4, 150, 4096, 1, 45, 0, 0, 0, 0, 465, 2111, 0, 0, 0, 0, 0, 193, 82, 0, 2, 310, 0, 0, 0, 360, 1386, 27, 1731, 0, 0, 0, 0, 7, 183, 10, 0, 1845, 0, 0, 0, 30, 0, 161, 51, 4024, 499, 0, 0, 4070, 0, 104, 3918, 1335, 366, 56, 0] rerouted_tokens_per_ep_rank=[5146, 4296, 2576, 587, 3504, 2045, 4765, 9849] avg=4096.0000 max=9849 max_rank=7 min=587 min_rank=3 imbalance_ratio=2.404541 max_load_reduction=22919 max_load_reduction_pct=0.699432 moved_to_echo=20163 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 160, 19, 63, 0, 0, 0, 0, 173, 1, 4096, 91, 6, 0, 0, 0, 0, 3697, 2056, 135, 0, 0, 0, 0, 29, 940, 7, 0, 42, 0, 0, 0, 0, 1395, 2680, 112, 0, 0, 0, 0, 0, 3002, 33, 1434, 1831, 0, 0, 0, 36, 352, 17, 0, 3852, 492, 0, 0, 3371, 0, 301, 682, 1296, 360, 7, 0] rerouted_tokens_per_ep_rank=[242, 4367, 5888, 1018, 4187, 6300, 4749, 6017] avg=4096.0000 max=6300 max_rank=5 min=242 min_rank=0 imbalance_ratio=1.538086 max_load_reduction=26468 max_load_reduction_pct=0.807739 moved_to_echo=21253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2788, 0, 2340, 0, 0, 0, 0, 18, 34, 4091, 0, 108, 0, 0, 0, 0, 584, 1732, 2, 0, 0, 0, 0, 78, 3551, 1, 220, 752, 0, 0, 0, 0, 1400, 3041, 2, 0, 0, 0, 0, 0, 110, 0, 3078, 1838, 0, 0, 0, 125, 0, 62, 32, 1150, 494, 0, 0, 143, 17, 36, 3361, 1092, 361, 127, 0] rerouted_tokens_per_ep_rank=[5128, 4251, 2318, 4602, 4443, 5026, 1863, 5137] avg=4096.0000 max=5137 max_rank=7 min=1863 min_rank=6 imbalance_ratio=1.254150 max_load_reduction=27631 max_load_reduction_pct=0.843231 moved_to_echo=16469 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1545, 2233, 2854, 0, 0, 0, 0, 1974, 14, 2224, 1, 60, 0, 0, 0, 2, 114, 2186, 0, 0, 0, 0, 0, 4037, 94, 277, 0, 417, 0, 0, 0, 25, 1316, 4079, 0, 0, 0, 0, 0, 291, 1005, 0, 7, 1729, 0, 0, 0, 1331, 2, 0, 542, 2049, 465, 0, 0, 10, 36, 60, 1, 1378, 340, 70, 0] rerouted_tokens_per_ep_rank=[6632, 4273, 2302, 4825, 5420, 3032, 4389, 1895] avg=4096.0000 max=6632 max_rank=0 min=1895 min_rank=7 imbalance_ratio=1.619141 max_load_reduction=26136 max_load_reduction_pct=0.797607 moved_to_echo=14736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10880, 1664, 512, 8064, 8448, 2944, 640, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12160, 12288, 2816, 1920, 3840, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 13824, 3072, 16000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1408, 7936, 384, 8960, 14336, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[512, 10880, 17280, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 19328, 13312, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6016, 384, 256, 1408, 21120, 3968, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2816, 640, 28032, 1024, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4036, 7, 16, 1006, 16, 4089, 3, 16, 2, 2172, 3, 0, 4090, 1, 175, 814, 0, 125, 5, 3960, 0, 317, 393, 4035, 0, 116, 354, 3490, 0, 0, 27, 3500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[61, 47, 0, 4095, 4078, 452, 1425, 0, 0, 1231, 273, 22, 4041, 8, 11, 2149, 7, 301, 0, 0, 0, 2, 1, 2409, 2481, 1263, 184, 4067, 0, 15, 65, 4080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[3423, 47, 0, 3946, 427, 0, 154, 38, 0, 4096, 12, 754, 3493, 14, 3, 4, 0, 2626, 59, 503, 0, 0, 3, 3555, 1427, 37, 444, 4040, 1, 0, 0, 3662] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 488, 0, 3567, 7, 0, 22, 0, 3, 3669, 717, 140, 3344, 0, 1997, 9, 0, 1682, 224, 2177, 0, 0, 0, 4095, 1337, 1536, 10, 3491, 122, 36, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4073, 0, 0, 75, 4011, 0, 0, 40, 13, 3650, 958, 872, 4089, 1, 0, 124, 0, 3952, 4, 3188, 6, 0, 0, 4, 0, 2552, 0, 88, 414, 569, 8, 4077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[813, 7, 0, 2845, 3803, 0, 0, 876, 0, 3434, 2, 2929, 3401, 1, 4020, 2930, 0, 1, 642, 0, 0, 0, 0, 2, 0, 3950, 1, 32, 1, 362, 83, 2633] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3901, 0, 0, 4094, 234, 143, 671, 0, 1, 4081, 0, 870, 3890, 350, 226, 145, 0, 20, 3275, 20, 13, 0, 0, 2703, 0, 4043, 0, 4064, 0, 0, 0, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 108, 14, 2829, 4073, 0, 1, 0, 1, 3940, 879, 2886, 4089, 16, 3776, 74, 0, 4088, 0, 37, 0, 156, 0, 3495, 10, 1937, 0, 126, 0, 1, 0, 232] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[61, 47, 0, 2869, 0, 0, 0, 0, 4078, 452, 1425, 0, 1039, 521, 0, 0, 0, 1005, 273, 22, 0, 0, 0, 0, 2114, 8, 11, 2149, 0, 0, 0, 0, 7, 301, 0, 0, 226, 187, 0, 0, 0, 2, 1, 2409, 1537, 0, 0, 0, 2481, 1263, 184, 2326, 0, 0, 0, 0, 0, 15, 65, 4080, 1220, 390, 0, 0] rerouted_tokens_per_ep_rank=[2977, 7515, 1300, 4282, 721, 3949, 6254, 5770] avg=4096.0000 max=7515 max_rank=1 min=721 min_rank=4 imbalance_ratio=1.834717 max_load_reduction=25253 max_load_reduction_pct=0.770660 moved_to_echo=16694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 0, 53, 0, 0, 0, 0, 4011, 0, 0, 40, 19, 11, 0, 0, 13, 2978, 958, 872, 0, 0, 0, 0, 2140, 1, 0, 124, 0, 0, 0, 0, 0, 3952, 4, 3188, 672, 3, 0, 0, 6, 0, 0, 4, 1555, 0, 0, 0, 0, 2552, 0, 51, 0, 0, 0, 0, 414, 569, 8, 4077, 26, 394, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4081, 4821, 2265, 7819, 1565, 2603, 5488] avg=4096.0000 max=7819 max_rank=4 min=1565 min_rank=5 imbalance_ratio=1.908936 max_load_reduction=24949 max_load_reduction_pct=0.761383 moved_to_echo=17475 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 488, 0, 2499, 0, 0, 0, 0, 7, 0, 22, 0, 905, 447, 0, 0, 3, 2993, 717, 140, 0, 0, 0, 0, 1750, 0, 1997, 9, 0, 0, 0, 0, 0, 1682, 224, 2177, 676, 163, 0, 0, 0, 0, 0, 4095, 1272, 0, 0, 0, 1337, 1536, 10, 1997, 0, 0, 0, 0, 122, 36, 0, 4095, 1047, 322, 0, 0] rerouted_tokens_per_ep_rank=[2987, 1381, 3853, 3756, 4922, 5367, 4880, 5622] avg=4096.0000 max=5622 max_rank=7 min=1381 min_rank=1 imbalance_ratio=1.372559 max_load_reduction=27146 max_load_reduction_pct=0.828430 moved_to_echo=20791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4036, 7, 16, 705, 0, 0, 0, 0, 16, 4089, 3, 16, 255, 447, 0, 0, 2, 1772, 3, 0, 0, 0, 0, 0, 2141, 1, 175, 814, 0, 0, 0, 0, 0, 125, 5, 3960, 400, 46, 0, 0, 0, 317, 393, 4035, 1555, 0, 0, 0, 0, 116, 354, 1996, 0, 0, 0, 0, 0, 0, 27, 3500, 1047, 394, 0, 0] rerouted_tokens_per_ep_rank=[4764, 4826, 1777, 3131, 4536, 6300, 2466, 4968] avg=4096.0000 max=6300 max_rank=5 min=1777 min_rank=2 imbalance_ratio=1.538086 max_load_reduction=26468 max_load_reduction_pct=0.807739 moved_to_echo=18270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[813, 7, 0, 1986, 0, 0, 0, 0, 3803, 0, 0, 876, 724, 8, 0, 0, 0, 2798, 2, 2929, 0, 0, 0, 0, 1771, 1, 4020, 2930, 0, 0, 0, 0, 0, 1, 642, 0, 636, 135, 0, 0, 0, 0, 0, 2, 1298, 0, 0, 0, 0, 3950, 1, 9, 0, 0, 0, 0, 1, 362, 83, 2633, 15, 332, 0, 0] rerouted_tokens_per_ep_rank=[2806, 5411, 5729, 8722, 1414, 1300, 3960, 3426] avg=4096.0000 max=8722 max_rank=3 min=1300 min_rank=5 imbalance_ratio=2.129395 max_load_reduction=24046 max_load_reduction_pct=0.733826 moved_to_echo=10100 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3423, 47, 0, 2765, 0, 0, 0, 0, 427, 0, 154, 38, 1001, 518, 0, 0, 0, 3342, 12, 754, 0, 0, 0, 0, 1828, 14, 3, 4, 0, 0, 0, 0, 0, 2626, 59, 503, 754, 180, 0, 0, 0, 0, 3, 3555, 1328, 0, 0, 0, 1427, 37, 444, 2310, 0, 0, 0, 0, 1, 0, 0, 3662, 1212, 337, 0, 0] rerouted_tokens_per_ep_rank=[6235, 2138, 4108, 1849, 4122, 4886, 4218, 5212] avg=4096.0000 max=6235 max_rank=0 min=1849 min_rank=3 imbalance_ratio=1.522217 max_load_reduction=26533 max_load_reduction_pct=0.809723 moved_to_echo=18438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3901, 0, 0, 2868, 0, 0, 0, 0, 234, 143, 671, 0, 1039, 521, 0, 0, 1, 3329, 0, 870, 0, 0, 0, 0, 2036, 350, 226, 145, 0, 0, 0, 0, 0, 20, 3275, 20, 752, 187, 0, 0, 13, 0, 0, 2703, 1479, 0, 0, 0, 0, 4043, 0, 2324, 0, 0, 0, 0, 0, 0, 0, 24, 1219, 375, 0, 0] rerouted_tokens_per_ep_rank=[6769, 2608, 4200, 2757, 4254, 4195, 6367, 1618] avg=4096.0000 max=6769 max_rank=0 min=1618 min_rank=7 imbalance_ratio=1.652588 max_load_reduction=25999 max_load_reduction_pct=0.793427 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 108, 14, 1982, 0, 0, 0, 0, 4073, 0, 1, 0, 718, 16, 0, 0, 1, 3214, 879, 2886, 0, 0, 0, 0, 2140, 16, 3776, 74, 0, 0, 0, 0, 0, 4088, 0, 37, 726, 129, 0, 0, 0, 156, 0, 3495, 1555, 0, 0, 0, 10, 1937, 0, 73, 0, 0, 0, 0, 0, 1, 0, 232, 37, 394, 0, 0] rerouted_tokens_per_ep_rank=[2104, 4808, 6980, 6006, 4980, 5206, 2020, 664] avg=4096.0000 max=6980 max_rank=2 min=664 min_rank=7 imbalance_ratio=1.704102 max_load_reduction=25788 max_load_reduction_pct=0.786987 moved_to_echo=12870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16768, 4736, 2304, 1024, 5760, 2560, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 12800, 4224, 9984, 4864, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 1024, 256, 22400, 5888, 2944, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16384, 768, 128, 15744, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 21504, 2944, 8576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 512, 20352, 11648, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16000, 512, 10240, 6272, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5376, 15488, 1024, 11136, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3039, 3, 4094, 0, 52, 0, 1684, 14, 1, 486, 2029, 4096, 667, 0, 4096, 28, 632, 496, 205, 0, 2925, 2135, 133, 3, 89, 1643, 152, 0, 73, 1, 3990, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1535, 24, 602, 1022, 2711, 87, 0, 35, 4079, 0, 4096, 3169, 449, 2, 2597, 2750, 61, 1, 4095, 5, 1123, 1141, 550, 24, 0, 27, 595, 397, 2, 1160, 427, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4084, 1, 144, 1434, 1378, 0, 127, 3259, 0, 0, 860, 3363, 0, 22, 681, 417, 4052, 21, 316, 1, 0, 0, 496, 4087, 106, 2973, 923, 106, 25, 5, 3766, 121] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1679, 2, 4019, 133, 424, 2222, 1227, 3783, 1949, 6, 4088, 1320, 1420, 0, 0, 0, 4075, 6, 4, 0, 0, 70, 0, 25, 1172, 273, 709, 55, 11, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4064, 1, 4093, 0, 9, 1620, 0, 4091, 6, 3, 0, 1341, 1468, 0, 31, 128, 0, 11, 3846, 0, 2247, 499, 0, 1517, 2375, 72, 3, 0, 2, 4096, 983, 262] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[2651, 8, 4061, 2, 52, 1, 681, 3161, 0, 14, 1, 3554, 3578, 1249, 360, 359, 1189, 0, 355, 6, 26, 216, 0, 2544, 0, 2736, 0, 0, 46, 3204, 2520, 194] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 5, 2982, 0, 0, 0, 5, 2805, 6, 252, 0, 2376, 3870, 385, 1025, 0, 0, 0, 1321, 144, 4046, 25, 0, 3093, 12, 0, 4014, 0, 2, 2305, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[5, 0, 4083, 0, 0, 721, 0, 3897, 100, 0, 11, 4027, 1426, 173, 288, 2, 125, 8, 1512, 7, 8, 0, 1265, 3267, 1, 806, 4065, 0, 0, 4000, 2970, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3039, 3, 2225, 0, 0, 0, 0, 0, 52, 0, 1684, 14, 0, 0, 0, 0, 1, 486, 2029, 2605, 0, 0, 0, 0, 667, 0, 4096, 28, 925, 0, 0, 0, 632, 496, 205, 0, 1746, 0, 0, 0, 2925, 2135, 133, 3, 0, 4, 0, 0, 89, 1643, 152, 0, 1491, 123, 47, 0, 73, 1, 3014, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5267, 1750, 5121, 5716, 3079, 5200, 3545, 3090] avg=4096.0000 max=5716 max_rank=3 min=1750 min_rank=1 imbalance_ratio=1.395508 max_load_reduction=27052 max_load_reduction_pct=0.825562 moved_to_echo=14914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1679, 2, 2184, 133, 0, 0, 0, 0, 424, 2222, 1227, 3554, 0, 0, 0, 0, 1949, 6, 4088, 840, 0, 0, 0, 0, 1420, 0, 0, 0, 949, 0, 0, 0, 4075, 6, 4, 0, 1714, 0, 0, 0, 0, 70, 0, 25, 229, 4, 0, 0, 1172, 273, 709, 55, 480, 121, 48, 0, 11, 0, 3095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 7427, 6883, 2369, 5799, 328, 2858, 3106] avg=4096.0000 max=7427 max_rank=1 min=328 min_rank=5 imbalance_ratio=1.813232 max_load_reduction=25341 max_load_reduction_pct=0.773346 moved_to_echo=12091 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4064, 1, 2215, 0, 0, 0, 0, 0, 9, 1620, 0, 3838, 0, 0, 0, 0, 6, 3, 0, 849, 0, 0, 0, 0, 1468, 0, 31, 128, 231, 0, 0, 0, 0, 11, 3846, 0, 1751, 0, 0, 0, 2247, 499, 0, 1517, 253, 4, 0, 0, 2375, 72, 3, 0, 492, 127, 15, 0, 2, 4096, 733, 262, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6280, 5467, 858, 1858, 5608, 4520, 3084, 5093] avg=4096.0000 max=6280 max_rank=0 min=858 min_rank=2 imbalance_ratio=1.533203 max_load_reduction=26488 max_load_reduction_pct=0.808350 moved_to_echo=18305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 1620, 0, 0, 0, 0, 0, 0, 0, 5, 2635, 0, 0, 0, 0, 6, 252, 0, 1511, 0, 0, 0, 0, 3870, 385, 1025, 0, 949, 0, 0, 0, 0, 0, 1321, 144, 1272, 0, 0, 0, 4046, 25, 0, 3093, 170, 4, 0, 0, 12, 0, 4014, 0, 865, 90, 48, 0, 2, 2305, 3094, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1625, 2640, 1769, 6229, 2737, 7338, 5029, 5401] avg=4096.0000 max=7338 max_rank=5 min=1625 min_rank=0 imbalance_ratio=1.791504 max_load_reduction=25430 max_load_reduction_pct=0.776062 moved_to_echo=20505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1535, 24, 328, 1022, 0, 0, 0, 0, 2711, 87, 0, 33, 0, 0, 0, 0, 4079, 0, 4096, 2016, 0, 0, 0, 0, 449, 2, 2597, 2750, 98, 0, 0, 0, 61, 1, 4095, 5, 256, 0, 0, 0, 1123, 1141, 550, 24, 2, 0, 0, 0, 0, 27, 595, 397, 1153, 18, 5, 0, 2, 1160, 324, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2909, 2831, 10191, 5896, 4418, 2840, 2195, 1488] avg=4096.0000 max=10191 max_rank=2 min=1488 min_rank=7 imbalance_ratio=2.488037 max_load_reduction=22577 max_load_reduction_pct=0.688995 moved_to_echo=10941 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 2218, 0, 0, 0, 0, 0, 0, 721, 0, 3661, 0, 0, 0, 0, 100, 0, 11, 2561, 0, 0, 0, 0, 1426, 173, 288, 2, 688, 0, 0, 0, 125, 8, 1512, 7, 1742, 0, 0, 0, 8, 0, 1265, 3267, 236, 3, 0, 0, 1, 806, 4065, 0, 1466, 123, 35, 0, 0, 4000, 2244, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2223, 4382, 2672, 2577, 3394, 4779, 6496, 6245] avg=4096.0000 max=6496 max_rank=6 min=2223 min_rank=0 imbalance_ratio=1.585938 max_load_reduction=26272 max_load_reduction_pct=0.801758 moved_to_echo=20914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4084, 1, 79, 1434, 0, 0, 0, 0, 1378, 0, 127, 3062, 0, 0, 0, 0, 0, 0, 860, 2139, 0, 0, 0, 0, 0, 22, 681, 417, 873, 0, 0, 0, 4052, 21, 316, 1, 61, 0, 0, 0, 0, 0, 496, 4087, 197, 4, 0, 0, 106, 2973, 923, 106, 1224, 4, 44, 0, 25, 5, 2845, 121, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5598, 4567, 2999, 1993, 4451, 4784, 5380, 2996] avg=4096.0000 max=5598 max_rank=0 min=1993 min_rank=3 imbalance_ratio=1.366699 max_load_reduction=27170 max_load_reduction_pct=0.829163 moved_to_echo=17611 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2651, 8, 2207, 2, 0, 0, 0, 0, 52, 1, 681, 2970, 0, 0, 0, 0, 0, 14, 1, 2260, 0, 0, 0, 0, 3578, 1249, 360, 359, 584, 0, 0, 0, 1189, 0, 355, 6, 1732, 0, 0, 0, 26, 216, 0, 2544, 191, 2, 0, 0, 0, 2736, 0, 0, 1294, 122, 29, 0, 46, 3204, 1905, 194, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4868, 3704, 2275, 6130, 3282, 2979, 4181, 5349] avg=4096.0000 max=6130 max_rank=3 min=2275 min_rank=2 imbalance_ratio=1.496582 max_load_reduction=26638 max_load_reduction_pct=0.812927 moved_to_echo=15791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[17152, 128, 13184, 2688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12928, 1920, 9088, 3712, 5376, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[6144, 768, 11136, 14848, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10240, 640, 11776, 256, 10368, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10496, 4096, 2560, 14592, 1280, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4736, 4736, 3840, 19840, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3840, 8576, 10496, 640, 8576, 768, 384, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 14848, 17280, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[466, 0, 2765, 0, 0, 0, 3418, 138, 0, 126, 2805, 3725, 20, 4056, 4020, 18, 3293, 1, 27, 0, 1504, 54, 809, 35, 0, 22, 0, 3979, 0, 1487, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[13, 0, 773, 10, 1, 0, 1802, 3338, 745, 0, 3827, 3918, 97, 3315, 3679, 3756, 1, 18, 3, 6, 328, 2, 0, 537, 1, 2, 1, 2448, 894, 1479, 1735, 39] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 4092, 1242, 1541, 420, 2790, 4085, 37, 2589, 3635, 322, 0, 833, 0, 0, 34, 354, 427, 21, 0, 4092, 1921, 514, 37, 3781, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[44, 0, 4083, 1, 8, 0, 1627, 762, 0, 3905, 3, 3876, 6, 2667, 3994, 0, 0, 106, 0, 26, 0, 1698, 1386, 0, 212, 3670, 3986, 624, 68, 16, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[25, 1, 0, 0, 0, 554, 4090, 0, 5, 74, 1881, 4096, 3294, 2703, 1739, 101, 3891, 0, 0, 0, 0, 1426, 4, 10, 0, 456, 1, 4093, 534, 3752, 0, 38] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 14, 1031, 0, 3146, 678, 7, 1152, 9, 3317, 4074, 0, 4011, 3800, 0, 2, 0, 36, 0, 48, 403, 0, 0, 0, 2436, 1808, 3290, 0, 482, 0, 3024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 2844, 26, 349, 2919, 5, 0, 4087, 145, 4023, 4074, 2, 1629, 4081, 2, 4072, 14, 15, 0, 0, 575, 0, 91, 53, 0, 0, 3761, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[20, 5, 0, 115, 0, 2, 2922, 4059, 4096, 0, 3563, 4096, 0, 1299, 0, 21, 0, 10, 0, 0, 0, 0, 14, 423, 3675, 3971, 0, 4051, 2, 1, 0, 423] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 773, 10, 1647, 1148, 0, 0, 1, 0, 1571, 3338, 0, 0, 0, 0, 745, 0, 2837, 0, 0, 0, 0, 0, 97, 3315, 417, 3756, 0, 0, 0, 0, 1, 18, 3, 6, 1615, 1017, 0, 0, 328, 2, 0, 537, 2770, 0, 0, 0, 1, 2, 1, 696, 0, 0, 0, 0, 894, 1479, 1735, 39, 735, 990, 231, 0] rerouted_tokens_per_ep_rank=[3591, 4910, 3582, 7585, 2660, 3637, 700, 6103] avg=4096.0000 max=7585 max_rank=3 min=700 min_rank=6 imbalance_ratio=1.851807 max_load_reduction=25183 max_load_reduction_pct=0.768524 moved_to_echo=13100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1627, 1196, 0, 0, 1, 0, 3567, 1242, 0, 0, 0, 0, 1541, 420, 2069, 0, 0, 0, 0, 0, 37, 2589, 413, 322, 0, 0, 0, 0, 0, 833, 0, 0, 1595, 213, 0, 0, 34, 354, 427, 21, 2889, 0, 0, 0, 0, 4092, 1921, 147, 0, 0, 0, 0, 37, 3781, 0, 0, 154, 721, 525, 0] rerouted_tokens_per_ep_rank=[2823, 4810, 4030, 3361, 2641, 3725, 6160, 5218] avg=4096.0000 max=6160 max_rank=6 min=2641 min_rank=4 imbalance_ratio=1.503906 max_load_reduction=26608 max_load_reduction_pct=0.812012 moved_to_echo=17744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[25, 1, 0, 0, 783, 1201, 0, 0, 0, 554, 3565, 0, 0, 0, 0, 0, 5, 74, 1395, 0, 0, 0, 0, 0, 3294, 2703, 190, 101, 0, 0, 0, 0, 3891, 0, 0, 0, 766, 1701, 0, 0, 0, 1426, 4, 10, 2895, 0, 0, 0, 0, 456, 1, 1163, 0, 0, 0, 0, 534, 3752, 0, 38, 1229, 486, 525, 0] rerouted_tokens_per_ep_rank=[2010, 4119, 1474, 6288, 6358, 4335, 1620, 6564] avg=4096.0000 max=6564 max_rank=7 min=1474 min_rank=2 imbalance_ratio=1.602539 max_load_reduction=26204 max_load_reduction_pct=0.799683 moved_to_echo=18877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[466, 0, 2765, 0, 1800, 1091, 0, 0, 0, 0, 2980, 138, 0, 0, 0, 0, 0, 126, 2080, 0, 0, 0, 0, 0, 20, 4056, 456, 18, 0, 0, 0, 0, 3293, 1, 27, 0, 1764, 1654, 0, 0, 1504, 54, 809, 35, 2634, 0, 0, 0, 0, 22, 0, 1130, 0, 0, 0, 0, 0, 1487, 0, 0, 1195, 725, 438, 0] rerouted_tokens_per_ep_rank=[6122, 3118, 2206, 4550, 6739, 5036, 1152, 3845] avg=4096.0000 max=6739 max_rank=4 min=1152 min_rank=6 imbalance_ratio=1.645264 max_load_reduction=26029 max_load_reduction_pct=0.794342 moved_to_echo=16772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[44, 0, 4083, 1, 1788, 1135, 0, 0, 8, 0, 1419, 762, 0, 0, 0, 0, 0, 3905, 3, 0, 0, 0, 0, 0, 6, 2667, 453, 0, 0, 0, 0, 0, 0, 106, 0, 26, 1753, 259, 0, 0, 0, 1698, 1386, 0, 2741, 0, 0, 0, 212, 3670, 3986, 178, 0, 0, 0, 0, 68, 16, 0, 0, 187, 0, 208, 0] rerouted_tokens_per_ep_rank=[7051, 2189, 3908, 3126, 2144, 5825, 8046, 479] avg=4096.0000 max=8046 max_rank=6 min=479 min_rank=7 imbalance_ratio=1.964355 max_load_reduction=24722 max_load_reduction_pct=0.754456 moved_to_echo=16494 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2844, 26, 1827, 1193, 0, 0, 349, 2919, 5, 0, 0, 0, 0, 0, 4087, 145, 2983, 0, 0, 0, 0, 0, 2, 1629, 463, 2, 0, 0, 0, 0, 4072, 14, 15, 0, 1791, 1563, 0, 0, 0, 575, 0, 91, 2881, 0, 0, 0, 53, 0, 0, 1069, 0, 0, 0, 0, 1, 0, 0, 0, 1129, 1040, 0, 0] rerouted_tokens_per_ep_rank=[5890, 3273, 7215, 2096, 7455, 3547, 1122, 2170] avg=4096.0000 max=7455 max_rank=4 min=1122 min_rank=6 imbalance_ratio=1.820068 max_load_reduction=25313 max_load_reduction_pct=0.772491 moved_to_echo=14294 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 14, 1031, 1701, 1194, 0, 0, 0, 3146, 591, 7, 0, 0, 0, 0, 1152, 9, 2459, 0, 0, 0, 0, 0, 0, 4011, 431, 0, 0, 0, 0, 0, 2, 0, 36, 0, 1668, 1367, 0, 0, 48, 403, 0, 0, 2880, 0, 0, 0, 0, 2436, 1808, 935, 0, 0, 0, 0, 0, 482, 0, 3024, 988, 858, 87, 0] rerouted_tokens_per_ep_rank=[3940, 3744, 3620, 4442, 3073, 3331, 5179, 5439] avg=4096.0000 max=5439 max_rank=7 min=3073 min_rank=4 imbalance_ratio=1.327881 max_load_reduction=27329 max_load_reduction_pct=0.834015 moved_to_echo=17022 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[20, 5, 0, 115, 0, 1201, 0, 0, 0, 2, 2544, 4059, 0, 0, 0, 0, 4096, 0, 2637, 0, 0, 0, 0, 0, 0, 1299, 0, 21, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1688, 0, 0, 0, 0, 14, 423, 2895, 0, 0, 0, 3675, 3971, 0, 1143, 0, 0, 0, 0, 2, 1, 0, 423, 1220, 926, 378, 0] rerouted_tokens_per_ep_rank=[1341, 6605, 6733, 1320, 1698, 3332, 8789, 2950] avg=4096.0000 max=8789 max_rank=6 min=1320 min_rank=3 imbalance_ratio=2.145752 max_load_reduction=23979 max_load_reduction_pct=0.731781 moved_to_echo=16769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 128, 10496, 1280, 11264, 9472, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3968, 14720, 7808, 6528, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11264, 1024, 128, 128, 11008, 9472, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[3456, 22272, 2944, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1536, 11008, 1792, 3584, 6912, 5760, 2432, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[384, 6656, 16256, 9600, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[11648, 4736, 16512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1920, 4608, 2688, 1152, 22656, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3996, 4096, 4076, 1, 4026, 4071, 3790, 24, 0, 2546, 1368, 1, 511, 298, 45, 0, 59, 0, 225, 0, 3, 0, 381, 0, 1, 3140, 0, 52, 55, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 7, 3626, 4060, 0, 4005, 1134, 3952, 4040, 1, 115, 4096, 0, 0, 2766, 3062, 227, 0, 0, 60, 8, 26, 202, 751, 118, 0, 508, 0, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[2381, 0, 0, 2994, 4096, 0, 2157, 4096, 4005, 84, 3, 794, 1, 0, 3981, 1, 4027, 0, 7, 0, 16, 0, 0, 1245, 19, 112, 520, 0, 2063, 121, 0, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[147, 0, 3910, 1669, 0, 0, 2550, 3667, 3339, 1702, 0, 2312, 8, 0, 4068, 0, 70, 3916, 2, 749, 15, 0, 0, 2314, 200, 0, 339, 788, 0, 114, 719, 170] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2471, 3541, 1146, 3295, 4096, 0, 0, 0, 3938, 341, 126, 0, 133, 2870, 2, 75, 0, 0, 0, 9, 0, 4096, 3213, 230, 0, 2318, 8, 0, 0, 860] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4082, 2974, 0, 3611, 2757, 3880, 4054, 15, 0, 1268, 0, 3765, 0, 4, 3813, 0, 0, 0, 2, 0, 1818, 115, 20, 0, 0, 0, 0, 590, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[2, 133, 0, 3904, 373, 0, 1494, 2616, 4048, 27, 2294, 4071, 0, 2, 0, 0, 0, 2965, 20, 2, 0, 22, 0, 4071, 1038, 0, 0, 3023, 0, 2447, 0, 216] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 76, 4019, 3700, 1, 2632, 2967, 28, 4093, 279, 0, 0, 722, 2890, 17, 1844, 0, 99, 1000, 0, 0, 84, 0, 4081, 139, 0, 4096, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 133, 0, 2747, 0, 0, 0, 0, 373, 0, 0, 1112, 0, 0, 0, 0, 358, 27, 2294, 4071, 0, 0, 0, 0, 0, 2, 0, 0, 541, 0, 0, 0, 0, 2965, 20, 2, 1111, 303, 0, 0, 0, 22, 0, 4071, 3259, 0, 0, 0, 1038, 0, 0, 3023, 616, 393, 0, 0, 0, 2447, 0, 216, 1494, 128, 0, 0] rerouted_tokens_per_ep_rank=[2882, 1485, 6750, 543, 4401, 7352, 5070, 4285] avg=4096.0000 max=7352 max_rank=5 min=543 min_rank=3 imbalance_ratio=1.794922 max_load_reduction=25416 max_load_reduction_pct=0.775635 moved_to_echo=21108 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[147, 0, 3910, 1175, 0, 0, 0, 0, 0, 0, 0, 1557, 0, 0, 0, 0, 296, 1702, 0, 2312, 0, 0, 0, 0, 8, 0, 4068, 0, 231, 0, 0, 0, 70, 3916, 2, 749, 1558, 250, 0, 0, 15, 0, 0, 2314, 2688, 0, 0, 0, 200, 0, 339, 788, 263, 552, 0, 0, 0, 114, 719, 170, 2550, 105, 0, 0] rerouted_tokens_per_ep_rank=[5232, 1557, 4310, 4307, 6545, 5017, 2142, 3658] avg=4096.0000 max=6545 max_rank=4 min=1557 min_rank=1 imbalance_ratio=1.597900 max_load_reduction=26223 max_load_reduction_pct=0.800262 moved_to_echo=17362 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3996, 2882, 0, 0, 0, 0, 4076, 1, 0, 1730, 0, 0, 0, 0, 337, 24, 0, 2546, 0, 0, 0, 0, 1368, 1, 511, 298, 568, 0, 0, 0, 45, 0, 59, 0, 1729, 283, 0, 0, 225, 0, 3, 0, 3051, 0, 0, 0, 381, 0, 1, 3140, 646, 612, 0, 0, 0, 52, 55, 3, 4026, 119, 0, 0] rerouted_tokens_per_ep_rank=[6878, 5807, 2907, 2746, 2116, 3279, 4780, 4255] avg=4096.0000 max=6878 max_rank=0 min=2116 min_rank=4 imbalance_ratio=1.679199 max_load_reduction=25890 max_load_reduction_pct=0.790100 moved_to_echo=14430 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2872, 0, 0, 0, 0, 2974, 0, 0, 1163, 0, 0, 0, 0, 335, 4054, 15, 0, 0, 0, 0, 0, 1268, 0, 3765, 0, 566, 0, 0, 0, 4, 3813, 0, 0, 1175, 292, 0, 0, 0, 2, 0, 1818, 3127, 0, 0, 0, 115, 20, 0, 0, 644, 419, 0, 0, 0, 0, 590, 0, 3611, 126, 0, 0] rerouted_tokens_per_ep_rank=[2872, 4137, 4404, 5599, 5284, 4947, 1198, 4327] avg=4096.0000 max=5599 max_rank=3 min=1198 min_rank=6 imbalance_ratio=1.366943 max_load_reduction=27169 max_load_reduction_pct=0.829132 moved_to_echo=15756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2381, 0, 0, 2107, 0, 0, 0, 0, 4096, 0, 0, 1740, 0, 0, 0, 0, 355, 84, 3, 794, 0, 0, 0, 0, 1, 0, 3981, 1, 415, 0, 0, 0, 4027, 0, 7, 0, 1740, 300, 0, 0, 16, 0, 0, 1245, 3224, 0, 0, 0, 19, 112, 520, 0, 472, 616, 0, 0, 2063, 121, 0, 45, 2157, 126, 0, 0] rerouted_tokens_per_ep_rank=[4488, 5836, 1236, 4398, 6074, 4485, 1739, 4512] avg=4096.0000 max=6074 max_rank=4 min=1236 min_rank=2 imbalance_ratio=1.482910 max_load_reduction=26694 max_load_reduction_pct=0.814636 moved_to_echo=16810 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 2551, 0, 0, 0, 0, 4060, 0, 0, 483, 0, 0, 0, 0, 350, 4040, 1, 115, 0, 0, 0, 0, 4096, 0, 0, 2766, 503, 0, 0, 0, 3062, 227, 0, 0, 481, 296, 0, 0, 60, 8, 26, 202, 3181, 0, 0, 0, 751, 118, 0, 508, 572, 170, 0, 0, 0, 0, 0, 4, 4005, 125, 0, 0] rerouted_tokens_per_ep_rank=[2558, 4543, 4506, 7365, 4066, 3477, 2119, 4134] avg=4096.0000 max=7365 max_rank=3 min=2119 min_rank=6 imbalance_ratio=1.798096 max_load_reduction=25403 max_load_reduction_pct=0.775238 moved_to_echo=13796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 76, 2828, 0, 0, 0, 0, 3700, 1, 0, 1261, 0, 0, 0, 0, 4, 4093, 279, 0, 0, 0, 0, 0, 0, 722, 2890, 17, 557, 0, 0, 0, 1844, 0, 99, 1000, 1260, 2, 0, 0, 0, 0, 84, 0, 22, 0, 0, 0, 4081, 139, 0, 4096, 634, 446, 0, 0, 0, 0, 0, 1, 2632, 0, 0, 0] rerouted_tokens_per_ep_rank=[2904, 4962, 4376, 4186, 4205, 106, 9396, 2633] avg=4096.0000 max=9396 max_rank=6 min=106 min_rank=5 imbalance_ratio=2.293945 max_load_reduction=23372 max_load_reduction_pct=0.713257 moved_to_echo=16340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2471, 2483, 0, 0, 0, 0, 1146, 3295, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 341, 0, 0, 0, 0, 126, 0, 133, 2870, 495, 0, 0, 0, 2, 75, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 3213, 230, 0, 2318, 563, 0, 0, 0, 8, 0, 0, 860, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4954, 4441, 4279, 3624, 77, 4105, 6324, 4964] avg=4096.0000 max=6324 max_rank=6 min=77 min_rank=4 imbalance_ratio=1.543945 max_load_reduction=26444 max_load_reduction_pct=0.807007 moved_to_echo=15470 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9856, 640, 896, 13952, 4480, 3328, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 128, 128, 13824, 18560, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9088, 11008, 256, 1792, 9088, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6912, 768, 15360, 6016, 3968, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[20480, 3328, 0, 9088, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2560, 256, 10496, 19712, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2048, 14080, 6656, 10240, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2176, 2816, 1408, 1408, 24576, 768, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[16, 0, 55, 35, 0, 7, 3025, 67, 2340, 1521, 321, 0, 9, 2, 4096, 0, 3784, 1545, 39, 64, 3920, 0, 4094, 0, 3297, 404, 0, 0, 0, 4, 4067, 56] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[336, 0, 1, 0, 0, 76, 2212, 5, 56, 3346, 3, 0, 4019, 0, 3791, 0, 69, 0, 537, 16, 0, 3731, 4052, 0, 2692, 1, 3710, 0, 0, 0, 4090, 25] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[2337, 0, 1713, 7, 0, 63, 4094, 132, 4086, 166, 824, 0, 2015, 0, 0, 6, 630, 1982, 3797, 8, 245, 3, 4057, 0, 2358, 4019, 191, 0, 0, 0, 0, 35] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 1200, 4096, 17, 3708, 4091, 9, 722, 956, 0, 4083, 2238, 38, 0, 1, 0, 0, 54, 4093, 3709, 3613, 137, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 183, 11, 880, 0, 0, 3393, 431, 3969, 0, 0, 0, 155, 4, 4094, 75, 4050, 0, 3524, 4, 1504, 5, 4085, 0, 3772, 3, 0, 139, 2440, 0, 47, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1, 4, 0, 3699, 4096, 0, 1, 0, 3586, 9, 215, 0, 4096, 232, 0, 36, 9, 799, 0, 258, 3552, 0, 4093, 0, 3994, 0, 4084, 3, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1, 13, 0, 3973, 1733, 16, 1744, 93, 0, 4085, 0, 0, 3957, 42, 4093, 2, 4070, 89, 6, 3, 11, 9, 4093, 6, 2188, 201, 0, 0, 2308, 0, 0, 32] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[10, 0, 0, 4068, 12, 3598, 4026, 0, 3918, 4095, 165, 0, 3, 0, 7, 0, 0, 5, 0, 250, 1, 70, 4095, 96, 4083, 4, 625, 9, 0, 0, 3601, 27] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 13, 0, 3973, 0, 520, 97, 0, 1733, 16, 1639, 93, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 3957, 42, 3029, 2, 0, 0, 0, 0, 4070, 89, 6, 3, 1064, 105, 48, 0, 11, 9, 1987, 6, 0, 0, 0, 0, 1620, 201, 0, 0, 0, 0, 0, 0, 2308, 0, 0, 32, 2009, 0, 0, 0] rerouted_tokens_per_ep_rank=[4604, 3481, 4085, 7030, 5385, 2013, 1821, 4349] avg=4096.0000 max=7030 max_rank=3 min=1821 min_rank=6 imbalance_ratio=1.716309 max_load_reduction=25738 max_load_reduction_pct=0.785461 moved_to_echo=13568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[336, 0, 1, 0, 21, 639, 96, 0, 0, 76, 2078, 5, 0, 0, 0, 0, 35, 3346, 3, 0, 0, 0, 0, 0, 4019, 0, 2803, 0, 0, 0, 0, 0, 69, 0, 537, 16, 988, 134, 59, 0, 0, 3731, 1967, 0, 0, 0, 0, 0, 1994, 1, 3710, 0, 0, 0, 0, 0, 0, 0, 4090, 25, 1989, 0, 0, 0] rerouted_tokens_per_ep_rank=[1093, 2159, 3384, 6822, 1803, 5698, 5705, 6104] avg=4096.0000 max=6822 max_rank=3 min=1093 min_rank=0 imbalance_ratio=1.665527 max_load_reduction=25946 max_load_reduction_pct=0.791809 moved_to_echo=19310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2337, 0, 1713, 7, 1600, 565, 98, 0, 0, 63, 3843, 132, 0, 0, 0, 0, 2486, 166, 824, 0, 0, 0, 0, 0, 2015, 0, 0, 6, 0, 0, 0, 0, 630, 1982, 3797, 8, 0, 251, 57, 0, 245, 3, 1965, 0, 0, 0, 0, 0, 1736, 4019, 191, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1994, 0, 0, 0] rerouted_tokens_per_ep_rank=[6320, 4038, 3476, 2021, 6725, 2213, 5946, 2029] avg=4096.0000 max=6725 max_rank=4 min=2021 min_rank=3 imbalance_ratio=1.641846 max_load_reduction=26043 max_load_reduction_pct=0.794769 moved_to_echo=16913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 183, 11, 880, 1551, 896, 97, 0, 0, 0, 3188, 431, 0, 0, 0, 0, 2418, 0, 0, 0, 0, 0, 0, 0, 155, 4, 3030, 75, 0, 0, 0, 0, 4050, 0, 3524, 4, 1064, 205, 84, 0, 1504, 5, 1983, 0, 0, 0, 0, 0, 2792, 3, 0, 139, 0, 0, 0, 0, 2440, 0, 47, 0, 2005, 0, 0, 0] rerouted_tokens_per_ep_rank=[3618, 3619, 2418, 3264, 8931, 3492, 2934, 4492] avg=4096.0000 max=8931 max_rank=4 min=2418 min_rank=2 imbalance_ratio=2.180420 max_load_reduction=23837 max_load_reduction_pct=0.727448 moved_to_echo=19849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 1449, 858, 97, 0, 0, 1200, 3848, 17, 0, 0, 0, 0, 2259, 4091, 9, 722, 0, 0, 0, 0, 956, 0, 3022, 2238, 0, 0, 0, 0, 38, 0, 1, 0, 1061, 248, 80, 0, 0, 54, 1987, 3709, 0, 0, 0, 0, 2675, 137, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2009, 0, 0, 0] rerouted_tokens_per_ep_rank=[2406, 5065, 7081, 6216, 1428, 5750, 2812, 2010] avg=4096.0000 max=7081 max_rank=2 min=1428 min_rank=4 imbalance_ratio=1.728760 max_load_reduction=25687 max_load_reduction_pct=0.783905 moved_to_echo=12000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 55, 35, 914, 783, 97, 0, 0, 7, 2842, 67, 0, 0, 0, 0, 1426, 1521, 321, 0, 0, 0, 0, 0, 9, 2, 3031, 0, 0, 0, 0, 0, 3784, 1545, 39, 64, 1065, 183, 73, 0, 3920, 0, 1988, 0, 0, 0, 0, 0, 2441, 404, 0, 0, 0, 0, 0, 0, 0, 4, 4067, 56, 2009, 0, 0, 0] rerouted_tokens_per_ep_rank=[1900, 2916, 3268, 3042, 6753, 5908, 2845, 6136] avg=4096.0000 max=6753 max_rank=4 min=1900 min_rank=0 imbalance_ratio=1.648682 max_load_reduction=26015 max_load_reduction_pct=0.793915 moved_to_echo=21642 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 0, 3699, 1401, 949, 97, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 2185, 9, 215, 0, 0, 0, 0, 0, 4096, 232, 0, 36, 0, 0, 0, 0, 9, 799, 0, 258, 0, 0, 88, 0, 3552, 0, 1987, 0, 0, 0, 0, 0, 2957, 0, 4084, 3, 0, 0, 0, 0, 0, 1, 0, 0, 2009, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 4097, 2409, 4364, 1154, 5539, 7044, 2010] avg=4096.0000 max=7044 max_rank=6 min=1154 min_rank=4 imbalance_ratio=1.719727 max_load_reduction=25724 max_load_reduction_pct=0.785034 moved_to_echo=15747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 4068, 1531, 970, 97, 0, 12, 3598, 3783, 0, 0, 0, 0, 0, 2387, 4095, 165, 0, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 0, 5, 0, 250, 1, 243, 90, 0, 1, 70, 1988, 96, 0, 0, 0, 0, 3023, 4, 625, 9, 0, 0, 0, 0, 0, 0, 3601, 27, 2010, 0, 0, 0] rerouted_tokens_per_ep_rank=[6676, 7393, 6647, 9, 589, 2155, 3661, 5638] avg=4096.0000 max=7393 max_rank=1 min=9 min_rank=3 imbalance_ratio=1.804932 max_load_reduction=25375 max_load_reduction_pct=0.774384 moved_to_echo=12043 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2816, 256, 1792, 12672, 8576, 6272, 896, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12672, 4480, 7936, 640, 5248, 1408, 640, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4864, 128, 11904, 256, 16128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9344, 3968, 15872, 3840, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15232, 384, 14976, 2432, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19328, 4864, 8704, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13312, 17408, 1664, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[5888, 4992, 21248, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[6, 0, 22, 0, 4089, 140, 31, 89, 0, 8, 0, 3080, 4029, 19, 0, 506, 0, 3114, 24, 286, 4040, 0, 0, 1, 400, 2788, 587, 947, 1084, 3943, 13, 3522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[100, 0, 3968, 21, 4003, 1, 0, 0, 0, 1095, 101, 4036, 4072, 10, 0, 7, 2, 0, 190, 2979, 733, 0, 2504, 2, 1, 3663, 0, 1182, 6, 4092, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4095, 9, 1, 2, 3956, 54, 0, 0, 0, 68, 0, 4090, 2986, 0, 172, 0, 4094, 1068, 0, 3301, 26, 1, 0, 1, 0, 1889, 0, 4096, 2135, 0, 179, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4007, 59, 0, 0, 3803, 0, 0, 18, 2109, 1819, 0, 4084, 3750, 0, 19, 405, 0, 28, 0, 3550, 275, 0, 13, 0, 153, 405, 73, 4095, 0, 7, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 40, 2687, 4095, 201, 0, 346, 67, 4092, 0, 4039, 4093, 3508, 1, 2, 260, 33, 0, 0, 0, 0, 0, 1286, 4, 3628, 165, 4, 11, 4096, 109, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 419, 0, 3723, 0, 0, 0, 373, 3267, 0, 782, 4093, 0, 6, 4096, 0, 57, 0, 4, 0, 0, 1, 3711, 3, 1, 3, 3607, 4083, 0, 443, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[385, 1, 3, 0, 3650, 8, 0, 1360, 2694, 4095, 3, 3410, 4090, 90, 26, 0, 0, 383, 0, 365, 28, 2630, 0, 47, 0, 3999, 0, 0, 1427, 3699, 0, 375] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4, 15, 4055, 14, 0, 3409, 12, 3494, 9, 295, 3825, 0, 52, 2529, 1701, 6, 0, 4, 109, 388, 8, 57, 675, 3893, 0, 125, 420, 3566, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[100, 0, 3968, 21, 545, 653, 176, 0, 3458, 1, 0, 0, 0, 0, 0, 0, 0, 1095, 101, 1604, 0, 0, 0, 0, 2807, 10, 0, 7, 0, 0, 0, 0, 2, 0, 190, 2979, 929, 897, 0, 0, 733, 0, 2504, 2, 2432, 336, 0, 0, 1, 3010, 0, 1182, 0, 0, 0, 0, 6, 3019, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5463, 3459, 2800, 2824, 4997, 6007, 4193, 3025] avg=4096.0000 max=6007 max_rank=5 min=2800 min_rank=2 imbalance_ratio=1.466553 max_load_reduction=26761 max_load_reduction_pct=0.816681 moved_to_echo=18222 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 40, 2687, 558, 647, 176, 0, 3537, 201, 0, 346, 0, 0, 0, 0, 67, 4092, 0, 1605, 0, 0, 0, 0, 2822, 3508, 1, 2, 0, 0, 0, 0, 260, 33, 0, 0, 934, 898, 0, 0, 0, 0, 0, 1286, 2434, 337, 0, 0, 4, 2981, 165, 4, 0, 0, 0, 0, 11, 3022, 109, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4084, 5764, 6333, 2125, 4057, 3154, 3142] avg=4096.0000 max=6333 max_rank=3 min=2125 min_rank=4 imbalance_ratio=1.546143 max_load_reduction=26435 max_load_reduction_pct=0.806732 moved_to_echo=12478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 9, 1, 2, 539, 337, 0, 0, 3417, 54, 0, 0, 0, 0, 0, 0, 0, 68, 0, 1625, 0, 0, 0, 0, 2059, 0, 172, 0, 0, 0, 0, 0, 4094, 1068, 0, 3301, 681, 0, 0, 0, 26, 1, 0, 1, 2465, 246, 0, 0, 0, 1552, 0, 4096, 0, 0, 0, 0, 2135, 0, 179, 545, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4983, 3471, 1693, 2231, 9144, 2739, 5648, 2859] avg=4096.0000 max=9144 max_rank=4 min=1693 min_rank=2 imbalance_ratio=2.232422 max_load_reduction=23624 max_load_reduction_pct=0.720947 moved_to_echo=20390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[385, 1, 3, 0, 497, 713, 159, 0, 3153, 8, 0, 1360, 0, 0, 0, 0, 2694, 4095, 3, 1355, 0, 0, 0, 0, 2820, 90, 26, 0, 0, 0, 0, 0, 0, 383, 0, 365, 933, 810, 0, 0, 28, 2630, 0, 47, 2055, 337, 0, 0, 0, 3286, 0, 0, 0, 0, 0, 0, 1427, 2730, 0, 375, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1758, 4521, 8147, 2936, 2491, 5097, 3286, 4532] avg=4096.0000 max=8147 max_rank=2 min=1758 min_rank=0 imbalance_ratio=1.989014 max_load_reduction=24621 max_load_reduction_pct=0.751373 moved_to_echo=15406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4007, 59, 0, 0, 518, 72, 0, 0, 3285, 0, 0, 18, 0, 0, 0, 0, 2109, 1819, 0, 1623, 0, 0, 0, 0, 2585, 0, 19, 405, 0, 0, 0, 0, 0, 28, 0, 3550, 856, 1, 0, 0, 275, 0, 13, 0, 2461, 309, 0, 0, 153, 333, 73, 4095, 0, 0, 0, 0, 0, 6, 1, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4656, 3303, 5551, 3009, 4435, 3058, 4654, 4102] avg=4096.0000 max=5551 max_rank=2 min=3009 min_rank=3 imbalance_ratio=1.355225 max_load_reduction=27217 max_load_reduction_pct=0.830597 moved_to_echo=16249 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 22, 0, 557, 497, 169, 0, 3532, 140, 31, 89, 0, 0, 0, 0, 0, 8, 0, 1224, 0, 0, 0, 0, 2777, 19, 0, 506, 0, 0, 0, 0, 0, 3114, 24, 286, 920, 864, 0, 0, 4040, 0, 0, 1, 1856, 332, 0, 0, 400, 2291, 587, 947, 0, 0, 0, 0, 1084, 2910, 13, 3522, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1251, 3792, 1232, 3302, 5208, 6229, 4225, 7529] avg=4096.0000 max=7529 max_rank=7 min=1232 min_rank=2 imbalance_ratio=1.838135 max_load_reduction=25239 max_load_reduction_pct=0.770233 moved_to_echo=23191 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 419, 0, 511, 1, 0, 0, 3212, 0, 0, 0, 0, 0, 0, 0, 373, 3267, 0, 308, 0, 0, 0, 0, 2813, 0, 6, 4096, 0, 0, 0, 0, 0, 57, 0, 4, 939, 0, 0, 0, 0, 0, 1, 3711, 474, 341, 0, 0, 3, 0, 3, 3607, 0, 0, 0, 0, 4083, 0, 443, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5027, 3212, 3948, 6915, 1000, 4527, 3613, 4526] avg=4096.0000 max=6915 max_rank=3 min=1000 min_rank=4 imbalance_ratio=1.688232 max_load_reduction=25853 max_load_reduction_pct=0.788971 moved_to_echo=13666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4, 15, 552, 698, 156, 0, 3503, 14, 0, 3409, 0, 0, 0, 0, 12, 3494, 9, 118, 0, 0, 0, 0, 2637, 0, 52, 2529, 0, 0, 0, 0, 1701, 6, 0, 4, 873, 784, 0, 0, 109, 388, 8, 57, 177, 315, 0, 0, 675, 3195, 0, 125, 0, 0, 0, 0, 420, 2626, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5521, 6926, 3633, 5218, 3368, 1054, 3995, 3053] avg=4096.0000 max=6926 max_rank=1 min=1054 min_rank=5 imbalance_ratio=1.690918 max_load_reduction=25842 max_load_reduction_pct=0.788635 moved_to_echo=11470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6144, 4736, 256, 10496, 7168, 4352, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5376, 18048, 128, 9472, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[16896, 128, 4480, 2816, 4352, 3712, 896, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21376, 3712, 384, 7552, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[27136, 512, 128, 5248, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[9216, 14336, 768, 8576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5248, 3072, 2560, 5120, 14464, 2560, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 16768, 896, 14080, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 16, 14, 12, 1, 2972, 0, 0, 4066, 0, 261, 4094, 3916, 0, 118, 0, 1, 3931, 95, 0, 0, 0, 0, 0, 4095, 0, 92, 0, 1030, 0, 4057, 3997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 38, 3583, 0, 0, 855, 0, 8, 4095, 59, 0, 0, 0, 0, 36, 37, 0, 17, 165, 4095, 3967, 4095, 0, 4026, 7, 0, 3549, 40, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3798, 82, 332, 0, 1, 11, 0, 1, 4095, 127, 0, 4021, 36, 0, 1, 0, 0, 3004, 30, 0, 16, 0, 0, 4090, 3053, 1624, 738, 5, 61, 18, 3538, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 14, 0, 3449, 0, 0, 4096, 97, 9, 4091, 3830, 306, 0, 367, 2, 4030, 465, 0, 0, 0, 4085, 627, 4066, 0, 9, 1, 0, 0, 3223, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[271, 2443, 1, 229, 39, 1, 0, 0, 3620, 0, 0, 2720, 4096, 0, 0, 0, 3, 4063, 0, 0, 0, 1989, 3973, 9, 3697, 0, 514, 3, 44, 0, 963, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[12, 0, 33, 0, 0, 95, 8, 83, 3935, 253, 0, 3845, 4059, 0, 11, 1, 10, 3170, 0, 112, 231, 0, 4073, 442, 2915, 1370, 22, 4093, 0, 0, 0, 3995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[109, 856, 1, 0, 3, 4045, 0, 0, 287, 39, 55, 1244, 3750, 0, 0, 0, 0, 0, 16, 0, 976, 4066, 4089, 681, 3359, 0, 1496, 39, 0, 34, 3529, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[69, 0, 78, 0, 4096, 4072, 0, 6, 3960, 10, 0, 960, 1714, 21, 0, 0, 349, 1693, 178, 0, 12, 1, 3640, 54, 4096, 2079, 291, 4, 0, 0, 1296, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 14, 12, 2951, 310, 0, 0, 1, 2972, 0, 0, 0, 284, 0, 0, 4066, 0, 261, 1143, 0, 0, 0, 0, 3916, 0, 118, 0, 1720, 0, 0, 0, 1, 3931, 95, 0, 1504, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2281, 0, 92, 0, 0, 0, 0, 0, 1030, 0, 4057, 1871, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3303, 3257, 5470, 5754, 5653, 0, 2373, 6958] avg=4096.0000 max=6958 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.698730 max_load_reduction=25810 max_load_reduction_pct=0.787659 moved_to_echo=14984 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[271, 2443, 1, 229, 1961, 280, 0, 0, 39, 1, 0, 0, 1414, 291, 0, 0, 3620, 0, 0, 759, 0, 0, 0, 0, 4096, 0, 0, 0, 1760, 0, 0, 0, 3, 4063, 0, 0, 1358, 125, 0, 0, 0, 1989, 2559, 9, 0, 0, 0, 0, 2059, 0, 514, 3, 0, 0, 0, 0, 44, 0, 963, 1914, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5185, 1745, 4379, 5856, 5549, 4557, 2576, 2921] avg=4096.0000 max=5856 max_rank=3 min=1745 min_rank=1 imbalance_ratio=1.429688 max_load_reduction=26912 max_load_reduction_pct=0.821289 moved_to_echo=15603 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 14, 2949, 308, 0, 0, 0, 3449, 0, 0, 1454, 0, 0, 0, 4096, 97, 9, 1142, 0, 0, 0, 0, 3830, 306, 0, 367, 0, 0, 0, 0, 2, 4030, 465, 0, 1494, 0, 0, 0, 0, 0, 2631, 627, 0, 0, 0, 0, 2264, 0, 9, 1, 0, 0, 0, 0, 0, 0, 3223, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3272, 4903, 5344, 4503, 5991, 3258, 2274, 3223] avg=4096.0000 max=5991 max_rank=4 min=2274 min_rank=6 imbalance_ratio=1.462646 max_load_reduction=26777 max_load_reduction_pct=0.817169 moved_to_echo=14746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[109, 856, 1, 0, 896, 254, 0, 0, 3, 4045, 0, 0, 1456, 294, 0, 0, 287, 39, 55, 348, 0, 0, 0, 0, 3750, 0, 0, 0, 1765, 0, 0, 0, 0, 0, 16, 0, 1234, 129, 0, 0, 976, 4066, 2633, 681, 0, 0, 0, 0, 1871, 0, 1496, 39, 0, 0, 0, 0, 0, 34, 3529, 1906, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2116, 5798, 729, 5515, 1379, 8356, 3406, 5469] avg=4096.0000 max=8356 max_rank=5 min=729 min_rank=2 imbalance_ratio=2.040039 max_load_reduction=24412 max_load_reduction_pct=0.744995 moved_to_echo=18610 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 33, 0, 2772, 220, 0, 0, 0, 95, 8, 83, 1450, 284, 0, 0, 3935, 253, 0, 1073, 0, 0, 0, 0, 4059, 0, 11, 1, 1719, 0, 0, 0, 10, 3170, 0, 112, 1071, 122, 0, 0, 231, 0, 2623, 442, 0, 0, 0, 0, 1624, 1370, 22, 4093, 0, 0, 0, 0, 0, 0, 0, 1870, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3037, 1920, 5261, 5790, 4485, 3296, 7109, 1870] avg=4096.0000 max=7109 max_rank=6 min=1870 min_rank=7 imbalance_ratio=1.735596 max_load_reduction=25659 max_load_reduction_pct=0.783051 moved_to_echo=16760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 2955, 313, 0, 0, 38, 3583, 0, 0, 1460, 0, 0, 0, 855, 0, 8, 1140, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 36, 37, 0, 1508, 0, 0, 0, 17, 165, 2635, 3967, 0, 0, 0, 0, 2274, 0, 4026, 7, 0, 0, 0, 0, 0, 3549, 40, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7364, 5081, 2003, 59, 1581, 6784, 6307, 3589] avg=4096.0000 max=7364 max_rank=0 min=59 min_rank=3 imbalance_ratio=1.797852 max_load_reduction=25404 max_load_reduction_pct=0.775269 moved_to_echo=18261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3798, 82, 332, 0, 2899, 231, 0, 0, 1, 11, 0, 1, 0, 290, 0, 0, 4095, 127, 0, 1122, 0, 0, 0, 0, 36, 0, 1, 0, 1759, 0, 0, 0, 0, 3004, 30, 0, 1121, 125, 0, 0, 16, 0, 0, 4090, 0, 0, 0, 0, 1701, 1624, 738, 5, 0, 0, 0, 0, 61, 18, 3538, 1912, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7342, 303, 5344, 1796, 4280, 4106, 4068, 5529] avg=4096.0000 max=7342 max_rank=0 min=303 min_rank=1 imbalance_ratio=1.792480 max_load_reduction=25426 max_load_reduction_pct=0.775940 moved_to_echo=17983 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[69, 0, 78, 0, 692, 310, 0, 0, 4096, 4072, 0, 6, 1296, 291, 0, 0, 3960, 10, 0, 268, 0, 0, 0, 0, 1714, 21, 0, 0, 1760, 0, 0, 0, 349, 1693, 178, 0, 1505, 125, 0, 0, 12, 1, 2344, 54, 0, 0, 0, 0, 2281, 2079, 291, 4, 0, 0, 0, 0, 0, 0, 1296, 1913, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1149, 9761, 4238, 3495, 3850, 2411, 4655, 3209] avg=4096.0000 max=9761 max_rank=1 min=1149 min_rank=0 imbalance_ratio=2.383057 max_load_reduction=23007 max_load_reduction_pct=0.702118 moved_to_echo=14125 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 3456, 512, 256, 18176, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 18304, 128, 128, 8576, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 3712, 16768, 11392, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 19968, 896, 128, 10880, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[24960, 640, 384, 7040, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 6272, 15488, 9984, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21504, 384, 256, 384, 10496, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16384, 5120, 7296, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 0, 8, 0, 0, 0, 3913, 3858, 338, 592, 24, 0, 0, 66, 4096, 1967, 0, 0, 269, 4020, 0, 3686, 209, 46, 3514, 158, 10, 1890, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 751, 1, 0, 0, 124, 0, 2581, 3899, 3283, 2, 203, 3420, 2930, 58, 0, 7, 342, 142, 3739, 179, 3522, 0, 1285, 0, 2, 0, 18, 1460, 4096, 724] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[169, 2, 531, 18, 69, 0, 1937, 0, 4054, 3732, 4086, 0, 0, 43, 65, 0, 1993, 4, 3839, 0, 3562, 100, 4080, 0, 0, 0, 6, 362, 0, 0, 77, 4039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 55, 26, 0, 1193, 0, 4090, 4084, 2529, 0, 4, 1, 316, 0, 44, 0, 883, 23, 135, 3744, 4096, 0, 1261, 1841, 0, 327, 0, 9, 4095, 4011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 165, 4046, 1723, 0, 0, 2238, 0, 6, 4096, 470, 3527, 274, 2596, 19, 3811, 2568, 1, 0, 0, 24, 0, 1160, 4095, 1949] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[177, 553, 2, 0, 0, 45, 0, 6, 0, 4088, 3900, 0, 6, 0, 1, 961, 38, 3549, 10, 0, 0, 573, 4045, 1660, 3032, 2174, 31, 0, 13, 0, 4066, 3838] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 2699, 0, 423, 0, 0, 0, 1, 0, 0, 4096, 0, 88, 0, 4092, 3, 0, 0, 3971, 3484, 1072, 854, 3776, 2, 0, 15, 0, 0, 15, 0, 4084, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 3280, 87, 0, 0, 11, 0, 19, 4096, 4077, 0, 0, 0, 4, 0, 9, 1682, 773, 4087, 170, 45, 0, 2967, 740, 2864, 14, 0, 11, 1280, 1998, 4095, 459] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 1538, 323, 0, 0, 0, 8, 0, 0, 3913, 106, 0, 0, 0, 0, 2867, 338, 0, 0, 0, 0, 592, 24, 0, 0, 2249, 78, 0, 0, 66, 4096, 1253, 0, 0, 0, 0, 0, 0, 269, 1448, 0, 0, 0, 0, 0, 3686, 209, 46, 3514, 991, 636, 0, 0, 158, 10, 246, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1869, 4027, 3205, 2943, 5415, 1717, 9082, 4510] avg=4096.0000 max=9082 max_rank=6 min=1717 min_rank=5 imbalance_ratio=2.217285 max_load_reduction=23686 max_load_reduction_pct=0.722839 moved_to_echo=20724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[177, 553, 2, 0, 3309, 325, 0, 0, 0, 45, 0, 6, 4088, 228, 0, 0, 0, 0, 2898, 0, 0, 0, 0, 0, 6, 0, 1, 961, 2263, 0, 0, 0, 38, 3549, 7, 0, 0, 0, 0, 0, 0, 573, 1457, 1660, 0, 0, 0, 0, 3032, 2174, 31, 0, 1002, 3, 0, 0, 13, 0, 529, 3838, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4366, 4367, 2898, 3231, 3594, 3690, 6242, 4380] avg=4096.0000 max=6242 max_rank=6 min=2898 min_rank=2 imbalance_ratio=1.523926 max_load_reduction=26526 max_load_reduction_pct=0.809509 moved_to_echo=17906 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2699, 0, 423, 3324, 304, 0, 0, 0, 0, 0, 1, 0, 229, 0, 0, 0, 0, 3044, 0, 0, 0, 0, 0, 88, 0, 4092, 3, 2113, 157, 0, 0, 0, 0, 2529, 3484, 0, 0, 0, 0, 1072, 854, 1359, 2, 0, 0, 0, 0, 0, 15, 0, 0, 1052, 1285, 0, 0, 15, 0, 531, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6750, 230, 3044, 6453, 6013, 3287, 2352, 4639] avg=4096.0000 max=6750 max_rank=0 min=230 min_rank=1 imbalance_ratio=1.647949 max_load_reduction=26018 max_load_reduction_pct=0.794006 moved_to_echo=16291 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[169, 2, 531, 18, 62, 328, 0, 0, 69, 0, 1937, 0, 3732, 4, 0, 0, 4054, 0, 3036, 0, 0, 0, 0, 0, 0, 43, 65, 0, 2283, 152, 0, 0, 1993, 4, 2445, 0, 0, 0, 0, 0, 3562, 100, 1469, 0, 0, 0, 0, 0, 0, 0, 6, 362, 1050, 1242, 0, 0, 0, 0, 11, 4039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1110, 5742, 7090, 2543, 4442, 5131, 2660, 4050] avg=4096.0000 max=7090 max_rank=2 min=1110 min_rank=0 imbalance_ratio=1.730957 max_load_reduction=25678 max_load_reduction_pct=0.783630 moved_to_echo=16283 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 55, 3333, 329, 0, 0, 26, 0, 1193, 0, 4084, 230, 0, 0, 4090, 0, 1880, 0, 0, 0, 0, 0, 4, 1, 316, 0, 2292, 35, 0, 0, 44, 0, 563, 23, 0, 0, 0, 0, 135, 3744, 1475, 0, 0, 0, 0, 0, 1261, 1841, 0, 327, 649, 285, 0, 0, 0, 9, 532, 4011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3718, 5533, 5970, 2648, 630, 5354, 4363, 4552] avg=4096.0000 max=5970 max_rank=2 min=630 min_rank=4 imbalance_ratio=1.457520 max_load_reduction=26798 max_load_reduction_pct=0.817810 moved_to_echo=14899 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3337, 311, 0, 0, 0, 0, 0, 0, 4046, 233, 0, 0, 165, 0, 1277, 0, 0, 0, 0, 0, 0, 2238, 0, 6, 2135, 142, 0, 0, 4096, 470, 2239, 274, 0, 0, 0, 0, 2596, 19, 1365, 2568, 0, 0, 0, 0, 1, 0, 0, 24, 446, 1146, 0, 0, 0, 1160, 525, 1949, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3648, 4279, 1442, 4521, 7079, 6548, 1617, 3634] avg=4096.0000 max=7079 max_rank=4 min=1442 min_rank=2 imbalance_ratio=1.728271 max_load_reduction=25689 max_load_reduction_pct=0.783966 moved_to_echo=18878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 751, 1, 3334, 283, 0, 0, 0, 0, 124, 0, 3899, 230, 0, 0, 2581, 0, 2440, 2, 0, 0, 0, 0, 203, 3420, 2930, 58, 1970, 13, 0, 0, 0, 7, 219, 142, 0, 0, 0, 0, 3739, 179, 1269, 0, 0, 0, 0, 0, 1285, 0, 2, 0, 843, 110, 0, 0, 18, 1460, 532, 724, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4369, 4253, 5023, 8594, 368, 5187, 2240, 2734] avg=4096.0000 max=8594 max_rank=3 min=368 min_rank=4 imbalance_ratio=2.098145 max_load_reduction=24174 max_load_reduction_pct=0.737732 moved_to_echo=10529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3280, 87, 0, 3333, 238, 0, 0, 0, 11, 0, 19, 4077, 230, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 9, 1660, 162, 0, 0, 1682, 773, 2602, 170, 0, 0, 0, 0, 45, 0, 1069, 740, 0, 0, 0, 0, 2864, 14, 0, 11, 0, 1323, 0, 0, 1280, 1998, 532, 459, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6938, 4337, 4096, 1835, 5227, 1854, 4212, 4269] avg=4096.0000 max=6938 max_rank=0 min=1835 min_rank=3 imbalance_ratio=1.693848 max_load_reduction=25830 max_load_reduction_pct=0.788269 moved_to_echo=15562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 6656, 1408, 512, 21632, 2560, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[12160, 4352, 128, 4352, 6144, 6144, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11264, 5760, 11008, 4992, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[896, 5760, 7424, 1152, 17024, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[7936, 8960, 11904, 4096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15104, 0, 17536, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1536, 4736, 3456, 23296, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 3328, 128, 27904, 1536, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[530, 0, 38, 731, 3377, 82, 568, 171, 296, 0, 3639, 3668, 0, 0, 1197, 25, 3, 4096, 79, 3283, 0, 128, 2965, 0, 431, 0, 3416, 0, 4037, 0, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3095, 0, 816, 0, 3703, 0, 510, 0, 0, 407, 204, 3222, 0, 2788, 3590, 0, 556, 4073, 4096, 3653, 0, 0, 56, 1193, 330, 0, 0, 0, 0, 0, 1, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[109, 0, 4086, 3820, 0, 11, 4030, 45, 6, 0, 1, 47, 0, 0, 4055, 1684, 4096, 10, 2327, 21, 0, 0, 0, 0, 3701, 0, 253, 0, 4078, 0, 388, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 3424, 83, 4096, 0, 0, 2703, 1, 23, 4017, 915, 534, 183, 3967, 1, 4096, 6, 2343, 2, 0, 0, 0, 190, 1, 0, 0, 1, 5, 1, 28, 2052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4093, 0, 2583, 0, 0, 0, 0, 1065, 1145, 199, 3817, 0, 13, 3603, 3934, 0, 9, 0, 1, 24, 0, 3687, 4085, 0, 0, 130, 11, 275] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 77, 0, 60, 3765, 13, 0, 1, 1, 0, 144, 502, 0, 4096, 4083, 0, 0, 16, 2425, 4087, 2569, 29, 3236, 122, 1063, 13, 4012, 493, 3, 5, 1953, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1415, 0, 3247, 0, 89, 4067, 0, 153, 0, 0, 1911, 0, 4095, 5, 0, 4096, 4093, 3944, 4, 0, 2640, 0, 0, 48, 31, 0, 2901, 0, 0, 29] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3178, 4091, 2317, 0, 2679, 1304, 0, 3996, 0, 0, 6, 0, 7, 11, 648, 112, 3, 2811, 3722, 4069, 381, 0, 8, 0, 1, 16, 3078, 0, 0, 0, 0, 330] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3204, 0, 3424, 83, 0, 0, 0, 0, 2067, 0, 0, 2703, 0, 0, 0, 0, 1, 23, 4017, 915, 2029, 0, 88, 0, 534, 183, 3129, 1, 0, 0, 0, 0, 4096, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 190, 1956, 0, 0, 0, 1, 0, 0, 1, 750, 892, 0, 0, 5, 1, 28, 2052, 1, 387, 0, 0] rerouted_tokens_per_ep_rank=[6711, 4770, 7073, 3847, 4103, 2146, 1644, 2474] avg=4096.0000 max=7073 max_rank=2 min=1644 min_rank=6 imbalance_ratio=1.726807 max_load_reduction=25695 max_load_reduction_pct=0.784149 moved_to_echo=10367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[415, 0, 38, 731, 0, 0, 0, 0, 1702, 82, 568, 171, 0, 0, 0, 0, 296, 0, 3639, 3668, 1675, 261, 26, 0, 0, 0, 945, 25, 0, 0, 0, 0, 3, 4096, 0, 912, 0, 0, 0, 0, 0, 128, 2965, 0, 66, 0, 0, 0, 431, 0, 3416, 0, 226, 115, 0, 0, 4037, 0, 8, 0, 2110, 13, 0, 0] rerouted_tokens_per_ep_rank=[1184, 2523, 9565, 970, 5011, 3159, 4188, 6168] avg=4096.0000 max=9565 max_rank=2 min=970 min_rank=3 imbalance_ratio=2.335205 max_load_reduction=23203 max_load_reduction_pct=0.708099 moved_to_echo=18526 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 77, 0, 60, 0, 0, 0, 0, 1900, 13, 0, 1, 0, 0, 0, 0, 1, 0, 144, 502, 1865, 325, 91, 0, 0, 4096, 3220, 0, 0, 0, 0, 0, 0, 16, 0, 1135, 0, 0, 0, 0, 2569, 29, 3236, 122, 2024, 0, 0, 0, 1063, 13, 4012, 493, 772, 0, 0, 0, 3, 5, 1953, 0, 2627, 401, 0, 0] rerouted_tokens_per_ep_rank=[137, 1914, 2928, 7316, 1151, 7980, 6353, 4989] avg=4096.0000 max=7980 max_rank=5 min=137 min_rank=0 imbalance_ratio=1.948242 max_load_reduction=24788 max_load_reduction_pct=0.756470 moved_to_echo=20473 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3202, 0, 0, 0, 0, 0, 0, 0, 2066, 0, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 2027, 0, 85, 0, 1145, 199, 3010, 0, 0, 0, 0, 0, 13, 3603, 0, 0, 0, 0, 0, 0, 9, 0, 1, 24, 3283, 0, 0, 0, 0, 3687, 4085, 0, 722, 892, 0, 0, 0, 130, 11, 275, 0, 651, 0, 0] rerouted_tokens_per_ep_rank=[3202, 4649, 3177, 4354, 3616, 3317, 9386, 1067] avg=4096.0000 max=9386 max_rank=6 min=1067 min_rank=7 imbalance_ratio=2.291504 max_load_reduction=23382 max_load_reduction_pct=0.713562 moved_to_echo=17386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1415, 0, 0, 0, 0, 0, 1639, 0, 89, 4067, 0, 0, 0, 0, 0, 153, 0, 0, 1608, 313, 91, 0, 1911, 0, 3230, 5, 0, 0, 0, 0, 0, 4096, 0, 1096, 0, 0, 0, 0, 4, 0, 2640, 0, 3417, 0, 0, 0, 0, 48, 31, 0, 774, 0, 0, 0, 2901, 0, 0, 29, 2535, 676, 0, 0] rerouted_tokens_per_ep_rank=[1415, 5795, 2165, 5146, 5192, 6061, 853, 6141] avg=4096.0000 max=6141 max_rank=7 min=853 min_rank=6 imbalance_ratio=1.499268 max_load_reduction=26627 max_load_reduction_pct=0.812592 moved_to_echo=18247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[82, 0, 4086, 3820, 0, 0, 0, 0, 0, 11, 4030, 45, 0, 0, 0, 0, 6, 0, 1, 47, 0, 5, 94, 0, 0, 0, 3190, 1684, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1943, 0, 0, 0, 3701, 0, 253, 0, 771, 27, 0, 0, 4078, 0, 388, 0, 16, 384, 0, 0] rerouted_tokens_per_ep_rank=[7988, 4086, 153, 4874, 4106, 1943, 4752, 4866] avg=4096.0000 max=7988 max_rank=0 min=153 min_rank=2 imbalance_ratio=1.950195 max_load_reduction=24780 max_load_reduction_pct=0.756226 moved_to_echo=15667 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2421, 0, 816, 0, 0, 0, 0, 0, 1869, 0, 510, 0, 0, 0, 0, 0, 0, 407, 204, 3222, 1834, 290, 80, 0, 0, 2788, 2831, 0, 0, 0, 0, 0, 556, 4073, 0, 1015, 0, 0, 0, 0, 0, 0, 56, 1193, 3418, 0, 0, 0, 330, 0, 0, 0, 679, 674, 0, 0, 0, 0, 1, 475, 2348, 678, 0, 0] rerouted_tokens_per_ep_rank=[3237, 2379, 6037, 5619, 5644, 4667, 1683, 3502] avg=4096.0000 max=6037 max_rank=2 min=1683 min_rank=6 imbalance_ratio=1.473877 max_load_reduction=26731 max_load_reduction_pct=0.815765 moved_to_echo=15496 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2486, 4091, 2317, 0, 0, 0, 0, 0, 1352, 1304, 0, 3996, 0, 0, 0, 0, 0, 0, 6, 0, 1327, 323, 14, 0, 7, 11, 512, 112, 0, 0, 0, 0, 3, 2811, 0, 1131, 0, 0, 0, 0, 381, 0, 8, 0, 3106, 0, 0, 0, 1, 16, 3078, 0, 122, 692, 0, 0, 0, 0, 0, 330, 2615, 616, 0, 0] rerouted_tokens_per_ep_rank=[8894, 6652, 1670, 642, 3945, 3495, 3909, 3561] avg=4096.0000 max=8894 max_rank=0 min=642 min_rank=3 imbalance_ratio=2.171387 max_load_reduction=23874 max_load_reduction_pct=0.728577 moved_to_echo=14910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11904, 4224, 12160, 4736, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 640, 8064, 9472, 12416, 1536, 640, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5632, 3840, 14976, 512, 4864, 3328, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3072, 256, 8960, 1536, 19328, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12672, 1536, 7808, 11008, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3712, 7296, 20096, 1920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8832, 18816, 0, 5376, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11136, 256, 2432, 3200, 12288, 3840, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 1187, 0, 4, 723, 0, 3543, 12, 3940, 4096, 2678, 630, 150, 0, 0, 3774, 0, 0, 4096, 595, 0, 0, 5, 0, 16, 0, 615, 4095, 2319, 283, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4058, 0, 0, 0, 44, 3878, 34, 3115, 3708, 30, 953, 0, 0, 1, 4093, 207, 77, 4076, 4, 223, 0, 23, 0, 1, 0, 3776, 4096, 0, 0, 371] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 3, 4085, 108, 0, 0, 4081, 0, 179, 771, 244, 80, 5, 416, 3907, 0, 3717, 116, 0, 4087, 0, 7, 0, 0, 3153, 86, 15, 3934, 3125, 0, 649, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3830, 0, 0, 0, 0, 0, 0, 315, 4096, 3879, 4058, 4095, 0, 361, 32, 0, 0, 2, 1402, 4096, 0, 1549, 51, 0, 127, 528, 4015, 9, 0, 323] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 558, 2758, 0, 0, 14, 51, 0, 0, 1196, 4066, 96, 3696, 262, 944, 425, 1236, 26, 0, 3650, 65, 4096, 112, 927, 10, 3022, 5, 0, 4090, 1392, 71, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4070, 3087, 0, 3949, 0, 2, 0, 0, 3767, 3680, 0, 0, 365, 0, 0, 4095, 0, 331, 4091, 0, 0, 1218, 0, 27, 2, 0, 4057, 0, 0, 0, 27] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 3523, 4042, 0, 4031, 0, 199, 0, 0, 2249, 3448, 4037, 36, 95, 0, 0, 4089, 38, 4051, 0, 2, 1304, 0, 0, 0, 1, 0, 0, 0, 1622, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4064, 0, 4095, 0, 174, 0, 0, 4016, 4085, 1483, 3923, 54, 0, 0, 4076, 26, 407, 3708, 0, 3, 0, 0, 4, 0, 5, 0, 0, 0, 2645, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 558, 2493, 0, 0, 0, 0, 0, 0, 14, 51, 0, 392, 0, 0, 0, 0, 1196, 134, 96, 0, 0, 0, 0, 3696, 262, 944, 425, 413, 0, 0, 0, 188, 26, 0, 3650, 0, 0, 0, 0, 65, 4096, 112, 927, 2540, 0, 0, 0, 10, 3022, 5, 0, 1392, 193, 0, 0, 4090, 1392, 71, 0, 265, 50, 0, 0] rerouted_tokens_per_ep_rank=[3051, 457, 1426, 5740, 3864, 7740, 4622, 5868] avg=4096.0000 max=7740 max_rank=5 min=457 min_rank=1 imbalance_ratio=1.889648 max_load_reduction=25028 max_load_reduction_pct=0.763794 moved_to_echo=22094 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3674, 0, 0, 0, 0, 0, 4095, 0, 174, 0, 1295, 0, 0, 0, 0, 4016, 134, 1483, 0, 0, 0, 0, 3923, 54, 0, 0, 1362, 0, 0, 0, 614, 26, 407, 3708, 0, 0, 0, 0, 0, 3, 0, 0, 2552, 0, 0, 0, 4, 0, 5, 0, 1399, 639, 0, 0, 0, 0, 2645, 0, 390, 166, 0, 0] rerouted_tokens_per_ep_rank=[3674, 5564, 5633, 5339, 4755, 2555, 2047, 3201] avg=4096.0000 max=5633 max_rank=2 min=2047 min_rank=6 imbalance_ratio=1.375244 max_load_reduction=27135 max_load_reduction_pct=0.828094 moved_to_echo=12558 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 315, 134, 3879, 0, 0, 0, 0, 4058, 4095, 0, 361, 10, 0, 0, 0, 6, 0, 0, 2, 0, 0, 0, 0, 1402, 4096, 0, 1549, 2559, 0, 0, 0, 51, 0, 127, 528, 1403, 5, 0, 0, 4015, 9, 0, 323, 368, 1, 0, 0] rerouted_tokens_per_ep_rank=[3462, 10, 4328, 8524, 8, 9606, 2114, 4716] avg=4096.0000 max=9606 max_rank=5 min=8 min_rank=4 imbalance_ratio=2.345215 max_load_reduction=23162 max_load_reduction_pct=0.706848 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1073, 0, 0, 0, 0, 0, 4, 723, 0, 3543, 1199, 0, 0, 0, 12, 3940, 134, 2678, 0, 0, 0, 0, 630, 150, 0, 0, 1261, 0, 0, 0, 570, 0, 0, 4096, 0, 0, 0, 0, 595, 0, 0, 5, 2559, 0, 0, 0, 0, 16, 0, 615, 1403, 591, 0, 0, 4095, 2319, 283, 7, 114, 153, 0, 0] rerouted_tokens_per_ep_rank=[1073, 5469, 6764, 2041, 4666, 3159, 2625, 6971] avg=4096.0000 max=6971 max_rank=7 min=1073 min_rank=0 imbalance_ratio=1.701904 max_load_reduction=25797 max_load_reduction_pct=0.787262 moved_to_echo=17421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 44, 3878, 1300, 0, 0, 0, 34, 3115, 121, 30, 0, 0, 0, 0, 953, 0, 0, 1, 1367, 0, 0, 0, 619, 207, 77, 4076, 0, 0, 0, 0, 4, 223, 0, 23, 2317, 0, 0, 0, 0, 1, 0, 3776, 1270, 641, 0, 0, 4096, 0, 0, 371, 389, 166, 0, 0] rerouted_tokens_per_ep_rank=[3669, 5222, 3300, 2321, 4979, 2567, 5688, 5022] avg=4096.0000 max=5688 max_rank=6 min=2321 min_rank=3 imbalance_ratio=1.388672 max_load_reduction=27080 max_load_reduction_pct=0.826416 moved_to_echo=18256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3693, 108, 0, 0, 0, 0, 0, 0, 4081, 0, 1181, 0, 0, 0, 179, 771, 9, 80, 0, 0, 0, 0, 5, 416, 3907, 0, 1242, 0, 0, 0, 561, 116, 0, 4087, 0, 0, 0, 0, 0, 7, 0, 0, 152, 0, 0, 0, 3153, 86, 15, 3934, 83, 582, 0, 0, 3125, 0, 649, 0, 392, 151, 0, 0] rerouted_tokens_per_ep_rank=[3804, 5262, 1039, 5570, 4764, 159, 7853, 4317] avg=4096.0000 max=7853 max_rank=6 min=159 min_rank=5 imbalance_ratio=1.917236 max_load_reduction=24915 max_load_reduction_pct=0.760345 moved_to_echo=17093 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3523, 3651, 0, 0, 0, 0, 0, 4031, 0, 199, 0, 1302, 0, 0, 0, 0, 2249, 104, 4037, 0, 0, 0, 0, 36, 95, 0, 0, 1369, 0, 0, 0, 603, 38, 4051, 0, 0, 0, 0, 0, 2, 1304, 0, 0, 2159, 0, 0, 0, 0, 1, 0, 0, 1185, 645, 0, 0, 0, 1622, 0, 1, 391, 170, 0, 0] rerouted_tokens_per_ep_rank=[7174, 5532, 6390, 1500, 4692, 3465, 1831, 2184] avg=4096.0000 max=7174 max_rank=0 min=1500 min_rank=3 imbalance_ratio=1.751465 max_load_reduction=25594 max_load_reduction_pct=0.781067 moved_to_echo=12172 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4070, 2791, 0, 0, 0, 0, 0, 3949, 0, 2, 0, 1301, 0, 0, 0, 0, 3767, 121, 0, 0, 0, 0, 0, 0, 365, 0, 0, 1368, 0, 0, 0, 618, 0, 331, 4091, 0, 0, 0, 0, 0, 0, 1218, 0, 2299, 0, 0, 0, 27, 2, 0, 4057, 1260, 642, 0, 0, 0, 0, 0, 27, 296, 166, 0, 0] rerouted_tokens_per_ep_rank=[6861, 5252, 3888, 1733, 5040, 3517, 5988, 489] avg=4096.0000 max=6861 max_rank=0 min=489 min_rank=7 imbalance_ratio=1.675049 max_load_reduction=25907 max_load_reduction_pct=0.790619 moved_to_echo=15034 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2176, 9856, 1408, 2560, 17152, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[19456, 5376, 3712, 768, 2688, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3328, 3200, 256, 12928, 9472, 3968, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13312, 5504, 4864, 896, 8448, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 19456, 896, 12288, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12160, 768, 4608, 7424, 8064, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 8192, 24576, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3840, 512, 4992, 23808, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4094, 4096, 0, 0, 0, 0, 1595, 1, 0, 4096, 4093, 0, 0, 9, 0, 0, 0, 104, 0, 4096, 2496, 0, 52, 0, 0, 72, 2, 4091, 351, 0, 3520, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 3662, 0, 34, 0, 0, 89, 0, 0, 76, 3919, 0, 0, 0, 232, 3829, 4, 323, 0, 4094, 0, 0, 349, 1, 0, 0, 0, 4002, 4025, 0, 0, 4033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 4051, 0, 0, 0, 432, 3, 39, 0, 4090, 4096, 0, 55, 0, 4013, 4093, 3194, 0, 0, 1427, 4096, 43, 0, 571, 0, 0, 0, 2054, 511, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[2243, 3727, 0, 0, 75, 0, 3967, 0, 275, 4083, 1, 186, 0, 577, 0, 0, 0, 0, 0, 1999, 4009, 92, 30, 4060, 0, 0, 0, 3348, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[2276, 2674, 745, 0, 0, 0, 0, 101, 0, 4049, 3438, 0, 0, 3192, 954, 0, 473, 0, 0, 3, 13, 205, 0, 4096, 4083, 209, 0, 0, 112, 2074, 4071, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1, 4077, 289, 1019, 11, 2300, 4062, 0, 107, 0, 0, 3, 3055, 1, 0, 97, 3721, 0, 0, 0, 2795, 0, 322, 0, 0, 2, 15, 347, 3999, 3616, 1495, 1434] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[3843, 3080, 0, 0, 24, 0, 0, 56, 0, 0, 3707, 0, 28, 0, 0, 0, 1281, 4096, 0, 4077, 3, 0, 4028, 452, 12, 0, 0, 4067, 4006, 0, 4, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[3820, 760, 70, 0, 39, 2329, 0, 0, 0, 3425, 0, 2, 3935, 0, 1821, 0, 1, 41, 148, 1478, 4096, 152, 0, 21, 1541, 40, 0, 1003, 20, 0, 4089, 3937] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4094, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 1595, 1, 2490, 357, 0, 0, 0, 2676, 4093, 0, 0, 0, 0, 0, 0, 9, 0, 0, 802, 609, 0, 0, 0, 104, 0, 4096, 456, 0, 0, 0, 2496, 0, 52, 0, 162, 0, 0, 0, 0, 72, 2, 4091, 1605, 0, 0, 0, 351, 0, 949, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5700, 4443, 6769, 1420, 4656, 2710, 5770, 1300] avg=4096.0000 max=6769 max_rank=2 min=1300 min_rank=7 imbalance_ratio=1.652588 max_load_reduction=25999 max_load_reduction_pct=0.793427 moved_to_echo=14436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2243, 1461, 0, 0, 0, 0, 0, 0, 75, 0, 3967, 0, 2266, 416, 0, 0, 275, 2668, 1, 186, 0, 0, 0, 0, 0, 577, 0, 0, 800, 709, 0, 0, 0, 0, 0, 1999, 454, 0, 0, 0, 4009, 92, 30, 4060, 161, 0, 0, 0, 0, 0, 0, 3348, 1868, 0, 0, 0, 0, 0, 1103, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3704, 6724, 3130, 2086, 2453, 8352, 5216, 1103] avg=4096.0000 max=8352 max_rank=5 min=1103 min_rank=7 imbalance_ratio=2.039062 max_load_reduction=24416 max_load_reduction_pct=0.745117 moved_to_echo=17124 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 1598, 289, 1019, 0, 0, 0, 0, 11, 2300, 4062, 0, 2479, 151, 0, 0, 107, 0, 0, 3, 0, 0, 0, 0, 3055, 1, 0, 97, 0, 259, 0, 0, 3721, 0, 0, 0, 0, 0, 0, 0, 2795, 0, 322, 0, 0, 0, 0, 0, 0, 2, 15, 347, 681, 0, 0, 0, 3999, 3616, 404, 1434, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2907, 9003, 110, 3412, 3721, 3117, 1045, 9453] avg=4096.0000 max=9453 max_rank=7 min=110 min_rank=2 imbalance_ratio=2.307861 max_load_reduction=23315 max_load_reduction_pct=0.711517 moved_to_echo=17336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 1435, 0, 34, 0, 0, 0, 0, 0, 0, 89, 0, 2227, 0, 0, 0, 0, 51, 3919, 0, 0, 0, 0, 0, 0, 0, 232, 3829, 14, 0, 0, 0, 4, 323, 0, 4094, 8, 0, 0, 0, 0, 0, 349, 1, 3, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 4025, 0, 0, 4033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5565, 2316, 3970, 4075, 4429, 353, 4002, 8058] avg=4096.0000 max=8058 max_rank=7 min=353 min_rank=5 imbalance_ratio=1.967285 max_load_reduction=24710 max_load_reduction_pct=0.754089 moved_to_echo=16842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2276, 1045, 745, 0, 0, 0, 0, 0, 0, 0, 0, 101, 1629, 417, 0, 0, 0, 2636, 3438, 0, 0, 0, 0, 0, 0, 3192, 954, 0, 796, 708, 0, 0, 473, 0, 0, 3, 454, 0, 0, 0, 13, 205, 0, 4096, 163, 0, 0, 0, 4083, 209, 0, 0, 1860, 0, 0, 0, 112, 2074, 1086, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 2147, 6074, 5650, 930, 4477, 6152, 3272] avg=4096.0000 max=6152 max_rank=6 min=930 min_rank=4 imbalance_ratio=1.501953 max_load_reduction=26616 max_load_reduction_pct=0.812256 moved_to_echo=14831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3843, 1207, 0, 0, 0, 0, 0, 0, 24, 0, 0, 56, 1873, 0, 0, 0, 0, 0, 3707, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 1281, 4096, 0, 4077, 0, 0, 0, 0, 3, 0, 4028, 452, 0, 0, 0, 0, 12, 0, 0, 4067, 1, 0, 0, 0, 4006, 0, 3, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5050, 1953, 3707, 28, 9454, 4483, 4080, 4013] avg=4096.0000 max=9454 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.308105 max_load_reduction=23314 max_load_reduction_pct=0.711487 moved_to_echo=22030 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1588, 0, 0, 0, 0, 0, 0, 0, 432, 3, 39, 2463, 0, 0, 0, 0, 2672, 4096, 0, 0, 0, 0, 0, 55, 0, 4013, 4093, 801, 0, 0, 0, 3194, 0, 0, 1427, 455, 0, 0, 0, 4096, 43, 0, 571, 162, 0, 0, 0, 0, 0, 0, 2054, 0, 0, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1588, 2937, 6768, 8962, 5076, 4872, 2054, 511] avg=4096.0000 max=8962 max_rank=3 min=511 min_rank=7 imbalance_ratio=2.187988 max_load_reduction=23806 max_load_reduction_pct=0.726501 moved_to_echo=12513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3820, 298, 70, 0, 0, 0, 0, 0, 39, 2329, 0, 0, 462, 415, 0, 0, 0, 2238, 0, 2, 0, 0, 0, 0, 3935, 0, 1821, 0, 671, 708, 0, 0, 1, 41, 148, 1478, 381, 0, 0, 0, 4096, 152, 0, 21, 135, 0, 0, 0, 1541, 40, 0, 1003, 1865, 0, 0, 0, 20, 0, 1101, 3937, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4188, 3245, 2240, 7135, 2049, 4404, 4449, 5058] avg=4096.0000 max=7135 max_rank=3 min=2049 min_rank=4 imbalance_ratio=1.741943 max_load_reduction=25633 max_load_reduction_pct=0.782257 moved_to_echo=15960 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13056, 5760, 4736, 9472, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7168, 3840, 7040, 8064, 3968, 3072, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 5120, 9728, 256, 16000, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 13056, 19328, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8704, 4608, 256, 17280, 2304, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[20480, 10240, 1152, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17536, 512, 4864, 9216, 896, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5760, 384, 128, 18944, 7936, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 13, 2850, 0, 1, 4075, 491, 684, 4042, 0, 0, 3844, 0, 200, 0, 4039, 2, 0, 500, 3833, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 1, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4055, 0, 522, 2, 4050, 5, 2362, 30, 3639, 0, 384, 0, 4096, 0, 0, 3533, 3574, 0, 0, 4096, 0, 0, 0, 0, 0, 1808, 329, 0, 0, 0, 16, 267] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[18, 64, 4012, 5, 4092, 3013, 76, 1458, 0, 0, 0, 280, 103, 4096, 0, 3887, 1736, 0, 0, 3666, 0, 5, 4046, 1, 552, 0, 3, 0, 12, 1434, 209, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 2, 2713, 368, 4034, 0, 3962, 4095, 116, 0, 0, 796, 10, 0, 0, 3415, 76, 0, 1, 3696, 0, 1290, 4096, 0, 0, 0, 0, 0, 1, 1, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[25, 880, 4073, 0, 3815, 4081, 2992, 0, 4096, 471, 0, 251, 0, 27, 0, 3, 4096, 0, 0, 4096, 0, 0, 236, 0, 0, 0, 0, 6, 0, 4, 1, 3615] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4075, 0, 4045, 229, 3116, 1851, 533, 4093, 3976, 126, 613, 0, 1143, 33, 0, 4091, 0, 403, 0, 0, 0, 0, 4067, 0, 335, 15, 0, 12, 7, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[353, 4, 0, 473, 33, 0, 106, 1929, 4088, 4079, 0, 0, 258, 4040, 72, 2891, 2417, 71, 0, 4094, 1773, 0, 7, 2011, 10, 1, 1, 0, 0, 80, 0, 3977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4084, 0, 4096, 0, 3564, 195, 1357, 0, 3383, 431, 0, 0, 2141, 4080, 759, 1, 49, 15, 195, 4096, 0, 0, 7, 4077, 0, 0, 0, 195, 36, 0, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4055, 0, 424, 2, 0, 0, 0, 0, 0, 4, 2362, 30, 0, 0, 0, 0, 3352, 0, 384, 0, 0, 0, 0, 0, 4096, 0, 0, 1893, 0, 0, 0, 0, 3574, 0, 0, 2917, 0, 0, 0, 0, 0, 0, 0, 0, 98, 287, 0, 0, 0, 1808, 329, 0, 4050, 1098, 0, 0, 0, 0, 16, 267, 1179, 1, 542, 0] rerouted_tokens_per_ep_rank=[4481, 2396, 3736, 5989, 6491, 385, 7285, 2005] avg=4096.0000 max=7285 max_rank=6 min=385 min_rank=5 imbalance_ratio=1.778564 max_load_reduction=25483 max_load_reduction_pct=0.777679 moved_to_echo=16166 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[18, 64, 3259, 5, 0, 0, 0, 0, 0, 1961, 76, 1458, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 103, 4096, 0, 2083, 0, 0, 0, 0, 1736, 0, 0, 2610, 0, 0, 0, 0, 0, 5, 4046, 1, 753, 0, 231, 0, 552, 0, 3, 0, 4092, 1208, 0, 0, 12, 1434, 209, 0, 1056, 821, 596, 0] rerouted_tokens_per_ep_rank=[3346, 3495, 280, 6282, 4346, 5036, 5855, 4128] avg=4096.0000 max=6282 max_rank=3 min=280 min_rank=2 imbalance_ratio=1.533691 max_load_reduction=26486 max_load_reduction_pct=0.808289 moved_to_echo=19365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 2315, 0, 0, 0, 0, 0, 0, 2646, 491, 684, 0, 0, 0, 0, 3723, 0, 0, 3844, 0, 0, 0, 0, 0, 200, 0, 2165, 0, 0, 0, 0, 2, 0, 500, 2729, 0, 0, 0, 0, 0, 0, 4096, 0, 535, 319, 316, 0, 0, 0, 0, 0, 1, 1255, 0, 0, 1, 1, 0, 4096, 1104, 1113, 619, 0] rerouted_tokens_per_ep_rank=[2328, 3821, 7567, 2365, 3231, 5266, 1256, 6934] avg=4096.0000 max=7567 max_rank=2 min=1256 min_rank=6 imbalance_ratio=1.847412 max_load_reduction=25201 max_load_reduction_pct=0.769073 moved_to_echo=16687 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 2200, 368, 0, 0, 0, 0, 0, 0, 3962, 4095, 0, 0, 0, 0, 103, 0, 0, 796, 0, 0, 0, 0, 10, 0, 0, 1822, 0, 0, 0, 0, 76, 0, 1, 2628, 0, 0, 0, 0, 0, 1290, 4096, 0, 513, 13, 0, 0, 0, 0, 0, 0, 4034, 1066, 0, 0, 1, 1, 0, 4096, 1068, 0, 527, 0] rerouted_tokens_per_ep_rank=[2570, 8057, 899, 1832, 2705, 5912, 5100, 5693] avg=4096.0000 max=8057 max_rank=1 min=899 min_rank=2 imbalance_ratio=1.967041 max_load_reduction=24711 max_load_reduction_pct=0.754120 moved_to_echo=19410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[353, 4, 0, 473, 0, 0, 0, 0, 0, 0, 106, 1929, 0, 0, 0, 0, 3765, 4079, 0, 0, 0, 0, 0, 0, 258, 4040, 72, 1550, 0, 0, 0, 0, 2417, 71, 0, 2915, 0, 0, 0, 0, 1773, 0, 7, 2011, 0, 323, 0, 0, 10, 1, 1, 0, 33, 898, 0, 0, 0, 80, 0, 3977, 1179, 0, 443, 0] rerouted_tokens_per_ep_rank=[830, 2035, 7844, 5920, 5403, 4114, 943, 5679] avg=4096.0000 max=7844 max_rank=2 min=830 min_rank=0 imbalance_ratio=1.915039 max_load_reduction=24924 max_load_reduction_pct=0.760620 moved_to_echo=16139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 3327, 0, 0, 0, 0, 0, 0, 128, 1357, 0, 0, 0, 0, 0, 3116, 431, 0, 0, 0, 0, 0, 0, 2141, 4080, 759, 1, 0, 0, 0, 0, 49, 15, 195, 2917, 0, 0, 0, 0, 0, 0, 7, 4077, 769, 267, 14, 0, 0, 0, 0, 195, 3564, 0, 0, 0, 36, 0, 7, 0, 1179, 53, 0, 0] rerouted_tokens_per_ep_rank=[7411, 1485, 3547, 6981, 3176, 5134, 3759, 1275] avg=4096.0000 max=7411 max_rank=0 min=1275 min_rank=7 imbalance_ratio=1.809326 max_load_reduction=25357 max_load_reduction_pct=0.773834 moved_to_echo=13344 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[25, 880, 3308, 0, 0, 0, 0, 0, 0, 2656, 2992, 0, 0, 0, 0, 0, 3772, 471, 0, 251, 0, 0, 0, 0, 0, 27, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 2917, 0, 0, 0, 0, 0, 0, 236, 0, 765, 324, 313, 0, 0, 0, 0, 6, 3815, 0, 0, 0, 0, 4, 1, 3615, 1179, 1112, 0, 0] rerouted_tokens_per_ep_rank=[4213, 5648, 4494, 30, 7013, 1638, 3821, 5911] avg=4096.0000 max=7013 max_rank=4 min=30 min_rank=3 imbalance_ratio=1.712158 max_load_reduction=25755 max_load_reduction_pct=0.785980 moved_to_echo=18383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4075, 0, 3285, 229, 0, 0, 0, 0, 0, 1205, 533, 4093, 0, 0, 0, 0, 3662, 126, 613, 0, 0, 0, 0, 0, 1143, 33, 0, 2193, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 760, 314, 142, 0, 335, 15, 0, 12, 3116, 1271, 0, 0, 7, 5, 0, 0, 0, 504, 627, 0] rerouted_tokens_per_ep_rank=[7589, 5831, 4401, 3369, 403, 5283, 4749, 1143] avg=4096.0000 max=7589 max_rank=0 min=403 min_rank=4 imbalance_ratio=1.852783 max_load_reduction=25179 max_load_reduction_pct=0.768402 moved_to_echo=11578 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12032, 512, 768, 19712, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7808, 12544, 896, 11776, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12672, 1024, 18176, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1536, 256, 16128, 8064, 3712, 3456, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[21504, 5120, 1024, 5248, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1792, 1408, 16640, 6144, 4224, 1920, 1024, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 8704, 11904, 12416, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1024, 1920, 384, 256, 22784, 6912, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 3765, 0, 0, 73, 0, 412, 976, 4072, 0, 0, 0, 3585, 0, 1851, 0, 2165, 0, 2537, 0, 4096, 0, 318, 0, 0, 4069, 9, 710, 0, 31, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 2757, 4095, 0, 3203, 0, 16, 34, 3992, 0, 0, 0, 37, 0, 602, 0, 426, 0, 0, 325, 4096, 0, 2, 0, 611, 3836, 2, 3980, 0, 219, 439] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 3901, 1274, 0, 4095, 0, 0, 0, 2, 0, 4095, 1, 16, 0, 1, 0, 2676, 4094, 277, 148, 57, 3398, 0, 0, 543, 0, 4094, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 4007, 0, 0, 50, 0, 0, 0, 0, 0, 114, 0, 0, 0, 3259, 8, 4088, 1156, 0, 3480, 3888, 0, 0, 3917, 0, 194, 123, 4082, 0, 214, 95] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 3777, 3958, 0, 675, 0, 0, 263, 1320, 2978, 458, 85, 141, 0, 99, 238, 122, 3935, 16, 366, 206, 4077, 22, 0, 3745, 4, 3964, 1, 2311, 0, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 3870, 3, 4, 1543, 0, 0, 1, 0, 2225, 3, 0, 0, 0, 3883, 673, 219, 19, 2366, 0, 45, 4096, 0, 0, 0, 0, 4087, 0, 4096, 38, 1873, 3724] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[11, 4074, 2, 0, 4096, 0, 0, 92, 0, 4092, 1, 0, 0, 91, 0, 722, 0, 4092, 1, 7, 3953, 3398, 0, 0, 0, 0, 4096, 70, 3970, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 904, 361, 5, 4085, 3160, 0, 0, 0, 515, 158, 1, 0, 1362, 284, 3801, 619, 0, 0, 0, 4018, 50, 192, 0, 1797, 3313, 4094, 0, 4049, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1782, 3765, 0, 0, 0, 0, 0, 0, 73, 0, 412, 979, 591, 0, 0, 976, 4072, 0, 0, 708, 423, 0, 0, 0, 3585, 0, 1851, 1723, 0, 0, 0, 0, 2165, 0, 2537, 341, 30, 0, 0, 0, 3047, 0, 318, 0, 0, 0, 0, 0, 0, 2667, 9, 0, 0, 0, 0, 680, 0, 31, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5547, 2055, 6179, 7159, 5073, 3365, 2676, 714] avg=4096.0000 max=7159 max_rank=3 min=714 min_rank=7 imbalance_ratio=1.747803 max_load_reduction=25609 max_load_reduction_pct=0.781525 moved_to_echo=11828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1643, 3958, 0, 0, 0, 0, 0, 675, 0, 0, 263, 953, 545, 0, 0, 1320, 2978, 458, 85, 705, 412, 0, 0, 141, 0, 99, 238, 1589, 0, 0, 0, 122, 3935, 16, 366, 340, 98, 0, 0, 206, 3032, 22, 0, 0, 0, 0, 0, 3745, 4, 2599, 1, 0, 0, 0, 0, 2213, 0, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5601, 2436, 5958, 2067, 4877, 3260, 6349, 2220] avg=4096.0000 max=6349 max_rank=6 min=2067 min_rank=3 imbalance_ratio=1.550049 max_load_reduction=26419 max_load_reduction_pct=0.806244 moved_to_echo=16706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1697, 1274, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 985, 563, 0, 0, 2, 0, 4095, 1, 587, 426, 0, 0, 16, 0, 1, 0, 1641, 0, 0, 0, 2676, 4094, 277, 148, 283, 175, 0, 0, 57, 2528, 0, 0, 0, 0, 0, 0, 543, 0, 2683, 0, 0, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2971, 5643, 5111, 1658, 7653, 2585, 3226, 3921] avg=4096.0000 max=7653 max_rank=4 min=1658 min_rank=3 imbalance_ratio=1.868408 max_load_reduction=25115 max_load_reduction_pct=0.766449 moved_to_echo=17385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1775, 2757, 4095, 0, 0, 0, 0, 0, 3203, 0, 16, 926, 595, 0, 0, 34, 3992, 0, 0, 712, 401, 0, 0, 0, 37, 0, 602, 1726, 0, 0, 0, 0, 426, 0, 0, 346, 174, 0, 0, 325, 3038, 0, 2, 0, 0, 0, 0, 0, 611, 2509, 2, 0, 0, 0, 0, 3806, 0, 219, 439, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8627, 4740, 5139, 2365, 946, 3365, 3122, 4464] avg=4096.0000 max=8627 max_rank=0 min=946 min_rank=4 imbalance_ratio=2.106201 max_load_reduction=24141 max_load_reduction_pct=0.736725 moved_to_echo=11897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1780, 4007, 0, 0, 0, 0, 0, 0, 50, 0, 0, 46, 591, 0, 0, 0, 0, 0, 114, 672, 20, 0, 0, 0, 0, 0, 3259, 1722, 0, 0, 0, 8, 4088, 1156, 0, 324, 174, 0, 0, 3480, 2892, 0, 0, 0, 0, 0, 0, 3917, 0, 128, 123, 0, 0, 0, 0, 3908, 0, 214, 95, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5787, 687, 806, 4981, 5750, 6372, 4168, 4217] avg=4096.0000 max=6372 max_rank=5 min=687 min_rank=1 imbalance_ratio=1.555664 max_load_reduction=26396 max_load_reduction_pct=0.805542 moved_to_echo=20507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 394, 361, 5, 0, 0, 0, 0, 4085, 3160, 0, 0, 985, 130, 0, 0, 0, 515, 158, 1, 8, 426, 0, 0, 0, 1362, 284, 3801, 380, 0, 0, 0, 619, 0, 0, 0, 4, 173, 0, 0, 4018, 38, 192, 0, 0, 0, 0, 0, 1797, 3313, 2683, 0, 0, 0, 0, 0, 3876, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[760, 8360, 1108, 5827, 796, 4248, 7793, 3876] avg=4096.0000 max=8360 max_rank=1 min=760 min_rank=0 imbalance_ratio=2.041016 max_load_reduction=24408 max_load_reduction_pct=0.744873 moved_to_echo=16713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1683, 3, 4, 0, 0, 0, 0, 1543, 0, 0, 1, 983, 559, 0, 0, 0, 2225, 3, 0, 708, 425, 0, 0, 0, 0, 3883, 673, 1628, 0, 0, 0, 219, 19, 2366, 0, 341, 175, 0, 0, 45, 3047, 0, 0, 0, 0, 0, 0, 0, 0, 2679, 0, 0, 0, 0, 0, 3921, 38, 1873, 3724, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1690, 3086, 3361, 6184, 3120, 3092, 2679, 9556] avg=4096.0000 max=9556 max_rank=7 min=1690 min_rank=0 imbalance_ratio=2.333008 max_load_reduction=23212 max_load_reduction_pct=0.708374 moved_to_echo=18447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[11, 1772, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 92, 985, 588, 0, 0, 0, 4092, 1, 0, 587, 426, 0, 0, 0, 91, 0, 722, 1714, 0, 0, 0, 0, 4092, 1, 7, 283, 170, 0, 0, 3953, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 2685, 70, 0, 0, 0, 0, 3800, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1785, 5761, 5106, 2527, 4553, 6481, 2755, 3800] avg=4096.0000 max=6481 max_rank=5 min=1785 min_rank=0 imbalance_ratio=1.582275 max_load_reduction=26287 max_load_reduction_pct=0.802216 moved_to_echo=17589 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14592, 6528, 0, 896, 6912, 4224, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 12544, 16128, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 5120, 4352, 11264, 12160, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3712, 18944, 3840, 3072, 2304, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2432, 17920, 4736, 256, 4736, 3072, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10112, 3968, 18688, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[26240, 128, 2432, 4352, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12160, 20224, 256, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-08-30 04:56:43] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 40919.7 | throughput per GPU (TFLOP/s/GPU): 8.1 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102614E+01 | loss scale: 1.0 | grad norm: 44.312 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 63622.4423828125 | max allocated: 85569.9892578125 | reserved: 88146.0 | max reserved: 88146.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[104, 3005, 730, 337, 1474, 376, 947, 776, 1103, 274, 1291, 194, 1405, 700, 180, 418, 156, 616, 929, 790, 1466, 149, 3767, 2083, 278, 1419, 1139, 1107, 939, 299, 1767, 2550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[118, 2822, 825, 259, 2190, 718, 1007, 1493, 1156, 206, 1177, 479, 1045, 842, 197, 531, 187, 599, 1421, 1222, 991, 235, 2478, 1727, 472, 1324, 1210, 1299, 1003, 370, 1397, 1768] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[56, 2739, 361, 191, 1648, 296, 1067, 894, 1160, 189, 2175, 283, 542, 908, 267, 260, 102, 534, 1182, 518, 1268, 221, 2968, 2099, 595, 1065, 1076, 961, 1784, 457, 2566, 2336] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[114, 3077, 525, 306, 1709, 242, 1270, 1263, 1246, 254, 1709, 214, 779, 852, 151, 378, 147, 945, 996, 825, 949, 118, 2884, 1664, 567, 1607, 835, 1227, 1703, 546, 1543, 2123] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[142, 2705, 768, 417, 1709, 536, 964, 645, 989, 148, 1512, 265, 1007, 993, 382, 456, 252, 714, 873, 1145, 1003, 426, 2990, 1792, 457, 955, 1501, 1097, 1218, 294, 1865, 2548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[206, 3389, 800, 318, 1593, 379, 705, 1438, 1107, 316, 2246, 316, 690, 903, 265, 414, 184, 451, 828, 1014, 1094, 215, 2785, 2265, 731, 1407, 1041, 1155, 860, 290, 1667, 1696] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[192, 2881, 501, 433, 1086, 374, 962, 685, 1260, 289, 1692, 398, 1097, 633, 343, 387, 156, 627, 812, 932, 1140, 577, 2962, 2138, 588, 1563, 1474, 925, 1543, 356, 1699, 2063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[207, 2976, 646, 349, 1655, 686, 1035, 1622, 1320, 180, 1754, 199, 832, 1043, 355, 499, 222, 614, 668, 960, 1321, 242, 2954, 1661, 404, 1141, 1123, 1216, 1015, 309, 1589, 1971] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[56, 2739, 361, 191, 19, 9, 0, 0, 1564, 296, 1067, 894, 0, 0, 0, 0, 1160, 189, 2175, 283, 694, 75, 0, 0, 542, 908, 267, 260, 1623, 0, 0, 0, 102, 534, 1182, 518, 862, 605, 0, 0, 1268, 221, 740, 2099, 0, 0, 0, 0, 595, 1046, 1076, 961, 0, 0, 0, 0, 1784, 457, 2566, 780, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3375, 3821, 4576, 3600, 3803, 4328, 3678, 5587] avg=4096.0000 max=5587 max_rank=7 min=3375 min_rank=0 imbalance_ratio=1.364014 max_load_reduction=27181 max_load_reduction_pct=0.829498 moved_to_echo=17396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[118, 2822, 825, 259, 24, 13, 0, 0, 2077, 718, 1007, 1493, 0, 0, 0, 0, 1156, 206, 1177, 479, 525, 100, 0, 0, 1045, 842, 197, 531, 1355, 0, 0, 0, 187, 599, 1421, 1222, 652, 505, 0, 0, 991, 235, 618, 1727, 0, 0, 0, 0, 472, 1300, 1210, 1299, 0, 0, 0, 0, 1003, 370, 1397, 591, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 5295, 3643, 3970, 4586, 3571, 4281, 3361] avg=4096.0000 max=5295 max_rank=1 min=3361 min_rank=7 imbalance_ratio=1.292725 max_load_reduction=27473 max_load_reduction_pct=0.838409 moved_to_echo=15799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[104, 3005, 730, 337, 29, 12, 0, 0, 1391, 376, 947, 776, 0, 0, 0, 0, 1103, 274, 1291, 194, 761, 71, 0, 0, 1405, 700, 180, 418, 2065, 0, 0, 0, 156, 616, 929, 790, 946, 771, 0, 0, 1466, 149, 931, 2083, 0, 0, 0, 0, 278, 1390, 1139, 1107, 0, 0, 0, 0, 939, 299, 1767, 843, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4217, 3490, 3694, 4768, 4208, 4629, 3914, 3848] avg=4096.0000 max=4768 max_rank=3 min=3490 min_rank=1 imbalance_ratio=1.164062 max_load_reduction=28000 max_load_reduction_pct=0.854492 moved_to_echo=16599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[206, 3389, 800, 318, 25, 9, 0, 0, 1512, 379, 705, 1438, 0, 0, 0, 0, 1107, 316, 2246, 316, 504, 72, 0, 0, 690, 903, 265, 414, 1523, 0, 0, 0, 184, 451, 828, 1014, 626, 567, 0, 0, 1094, 215, 695, 2265, 0, 0, 0, 0, 731, 1382, 1041, 1155, 0, 0, 0, 0, 860, 290, 1667, 566, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4747, 4034, 4561, 3795, 3670, 4269, 4309, 3383] avg=4096.0000 max=4747 max_rank=0 min=3383 min_rank=7 imbalance_ratio=1.158936 max_load_reduction=28021 max_load_reduction_pct=0.855133 moved_to_echo=15631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[114, 3077, 525, 306, 29, 10, 0, 0, 1621, 242, 1270, 1263, 0, 0, 0, 0, 1246, 254, 1709, 214, 630, 78, 0, 0, 779, 852, 151, 378, 1577, 0, 0, 0, 147, 945, 996, 825, 783, 588, 0, 0, 949, 118, 719, 1664, 0, 0, 0, 0, 567, 1578, 835, 1227, 0, 0, 0, 0, 1703, 546, 1543, 710, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4396, 4131, 3737, 4284, 3450, 4207, 4502] avg=4096.0000 max=4502 max_rank=7 min=3450 min_rank=5 imbalance_ratio=1.099121 max_load_reduction=28266 max_load_reduction_pct=0.862610 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[142, 2705, 768, 417, 17, 10, 0, 0, 1621, 536, 964, 645, 0, 0, 0, 0, 989, 148, 1512, 265, 757, 78, 0, 0, 1007, 993, 382, 456, 1635, 0, 0, 0, 252, 714, 873, 1145, 940, 609, 0, 0, 1003, 426, 746, 1792, 0, 0, 0, 0, 457, 938, 1501, 1097, 0, 0, 0, 0, 1218, 294, 1865, 851, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 3766, 3749, 4473, 4533, 3967, 3993, 4228] avg=4096.0000 max=4533 max_rank=4 min=3749 min_rank=2 imbalance_ratio=1.106689 max_load_reduction=28235 max_load_reduction_pct=0.861664 moved_to_echo=16721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[192, 2881, 501, 433, 28, 6, 0, 0, 1031, 374, 962, 685, 0, 0, 0, 0, 1260, 289, 1692, 398, 613, 49, 0, 0, 1097, 633, 343, 387, 1620, 0, 0, 0, 156, 627, 812, 932, 761, 603, 0, 0, 1140, 577, 739, 2138, 0, 0, 0, 0, 588, 1535, 1474, 925, 0, 0, 0, 0, 1543, 356, 1699, 689, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 3052, 4301, 4080, 3891, 4594, 4522, 4287] avg=4096.0000 max=4594 max_rank=5 min=3052 min_rank=1 imbalance_ratio=1.121582 max_load_reduction=28174 max_load_reduction_pct=0.859802 moved_to_echo=17294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[207, 2976, 646, 349, 20, 9, 0, 0, 1571, 686, 1035, 1622, 0, 0, 0, 0, 1320, 180, 1754, 199, 585, 75, 0, 0, 832, 1043, 355, 499, 1616, 0, 0, 0, 222, 614, 668, 960, 727, 602, 0, 0, 1321, 242, 736, 1661, 0, 0, 0, 0, 404, 1121, 1123, 1216, 0, 0, 0, 0, 1015, 309, 1589, 659, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4207, 4914, 4113, 4345, 3793, 3960, 3864, 3572] avg=4096.0000 max=4914 max_rank=1 min=3572 min_rank=7 imbalance_ratio=1.199707 max_load_reduction=27854 max_load_reduction_pct=0.850037 moved_to_echo=15189 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12416, 3712, 8064, 8832, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1408, 5120, 7808, 7424, 6400, 4864, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7424, 6912, 2176, 3456, 13056, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9344, 2304, 6016, 15488, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9344, 1920, 13568, 2432, 5120, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10112, 2944, 14208, 5760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1152, 23680, 5248, 2688, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 10368, 9472, 9088, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[65, 0, 2507, 80, 141, 3471, 418, 3250, 0, 0, 842, 2116, 0, 2294, 4, 1951, 72, 159, 0, 4095, 100, 0, 1, 287, 314, 262, 2155, 4093, 4090, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1992, 4, 84, 75, 3272, 3902, 2272, 384, 29, 0, 150, 332, 38, 1252, 439, 760, 53, 85, 0, 4082, 0, 3, 0, 13, 3088, 1429, 1203, 3688, 4092, 8, 31, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[695, 538, 806, 177, 3609, 4031, 3, 3852, 0, 0, 26, 2562, 106, 3403, 0, 623, 1, 6, 184, 4010, 0, 0, 2, 36, 4, 26, 4, 3825, 4084, 0, 145, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[274, 0, 1734, 0, 2483, 3669, 2723, 70, 445, 0, 921, 662, 0, 2931, 99, 333, 88, 695, 0, 4041, 26, 0, 54, 44, 3430, 8, 958, 3485, 3538, 0, 39, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[49, 1, 2214, 725, 1680, 2606, 1091, 3700, 30, 8, 2, 414, 689, 3905, 43, 527, 65, 33, 0, 4053, 1, 395, 2, 30, 1305, 332, 1343, 4058, 3462, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[72, 14, 3514, 182, 116, 2166, 349, 3439, 85, 14, 260, 3569, 0, 2999, 150, 446, 159, 456, 0, 4095, 260, 398, 6, 300, 740, 12, 669, 4079, 4090, 4, 114, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[236, 0, 3281, 89, 336, 3224, 2391, 3078, 14, 0, 22, 119, 31, 3069, 4, 804, 33, 670, 230, 4089, 4, 11, 1, 37, 605, 40, 2215, 4072, 4038, 0, 22, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[723, 0, 1097, 17, 733, 861, 3355, 2229, 115, 28, 1021, 252, 8, 3102, 203, 947, 77, 2542, 125, 3751, 50, 1, 0, 48, 2826, 9, 412, 4094, 4094, 3, 19, 26] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1992, 4, 84, 75, 168, 349, 0, 0, 3272, 0, 2272, 150, 0, 0, 0, 0, 29, 0, 150, 332, 1795, 66, 0, 0, 38, 1252, 439, 760, 203, 0, 0, 0, 53, 85, 0, 3427, 0, 0, 0, 0, 0, 3, 0, 13, 3902, 790, 0, 0, 3088, 1429, 1203, 1103, 0, 0, 0, 0, 4092, 8, 31, 8, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[2672, 5694, 2372, 2692, 3565, 4708, 6823, 4242] avg=4096.0000 max=6823 max_rank=6 min=2372 min_rank=2 imbalance_ratio=1.665771 max_load_reduction=25945 max_load_reduction_pct=0.791779 moved_to_echo=19338 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[695, 538, 806, 177, 1687, 343, 0, 0, 3609, 0, 3, 1502, 0, 0, 0, 0, 0, 0, 26, 2562, 1862, 663, 0, 0, 106, 3403, 0, 623, 200, 0, 0, 0, 1, 6, 184, 3366, 0, 0, 0, 0, 0, 0, 2, 36, 4031, 819, 0, 0, 4, 26, 4, 1144, 0, 0, 0, 0, 4084, 0, 145, 10, 101, 0, 0, 0] rerouted_tokens_per_ep_rank=[4246, 5114, 5113, 4332, 3557, 4888, 1178, 4340] avg=4096.0000 max=5114 max_rank=1 min=1178 min_rank=6 imbalance_ratio=1.248535 max_load_reduction=27654 max_load_reduction_pct=0.843933 moved_to_echo=13963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[72, 14, 3514, 182, 1506, 350, 0, 0, 116, 0, 349, 1342, 0, 0, 0, 0, 85, 14, 260, 3569, 1986, 591, 0, 0, 0, 2999, 150, 446, 204, 0, 0, 0, 159, 456, 0, 3438, 0, 0, 0, 0, 260, 398, 6, 300, 2166, 874, 0, 0, 740, 12, 669, 1219, 0, 0, 0, 0, 4090, 4, 114, 11, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[5638, 1807, 6505, 3799, 4053, 4004, 2640, 4322] avg=4096.0000 max=6505 max_rank=2 min=1807 min_rank=1 imbalance_ratio=1.588135 max_load_reduction=26263 max_load_reduction_pct=0.801483 moved_to_echo=15019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[65, 0, 2507, 80, 1423, 350, 0, 0, 141, 0, 418, 1268, 0, 0, 0, 0, 0, 0, 842, 2116, 1993, 559, 0, 0, 0, 2294, 4, 1951, 204, 0, 0, 0, 72, 159, 0, 3438, 0, 0, 0, 0, 100, 0, 1, 287, 3471, 877, 0, 0, 314, 262, 2155, 1223, 0, 0, 0, 0, 4090, 0, 0, 1, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[4425, 1827, 5510, 4453, 3669, 4736, 3954, 4194] avg=4096.0000 max=5510 max_rank=2 min=1827 min_rank=1 imbalance_ratio=1.345215 max_load_reduction=27258 max_load_reduction_pct=0.831848 moved_to_echo=16553 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[236, 0, 3281, 89, 1348, 350, 0, 0, 336, 0, 2391, 1201, 0, 0, 0, 0, 14, 0, 22, 119, 1982, 529, 0, 0, 31, 3069, 4, 804, 204, 0, 0, 0, 33, 670, 230, 3432, 0, 0, 0, 0, 4, 11, 1, 37, 3224, 872, 0, 0, 605, 40, 2215, 1218, 0, 0, 0, 0, 4038, 0, 22, 3, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[5304, 3928, 2666, 4112, 4365, 4149, 4078, 4166] avg=4096.0000 max=5304 max_rank=0 min=2666 min_rank=2 imbalance_ratio=1.294922 max_load_reduction=27464 max_load_reduction_pct=0.838135 moved_to_echo=16758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[274, 0, 1734, 0, 30, 346, 0, 0, 2483, 0, 2723, 28, 0, 0, 0, 0, 445, 0, 921, 662, 1696, 12, 0, 0, 0, 2931, 99, 333, 201, 0, 0, 0, 88, 695, 0, 3393, 0, 0, 0, 0, 26, 0, 54, 44, 3669, 746, 0, 0, 3430, 8, 958, 1043, 0, 0, 0, 0, 3538, 0, 39, 18, 101, 0, 0, 0] rerouted_tokens_per_ep_rank=[2384, 5234, 3736, 3564, 4176, 4539, 5439, 3696] avg=4096.0000 max=5439 max_rank=6 min=2384 min_rank=0 imbalance_ratio=1.327881 max_load_reduction=27329 max_load_reduction_pct=0.834015 moved_to_echo=17850 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[49, 1, 2214, 725, 1625, 351, 0, 0, 1680, 0, 1091, 1435, 0, 0, 0, 0, 30, 8, 2, 414, 1980, 640, 0, 0, 689, 3905, 43, 527, 205, 0, 0, 0, 65, 33, 0, 3392, 0, 0, 0, 0, 1, 395, 2, 30, 2606, 873, 0, 0, 1305, 332, 1343, 1205, 0, 0, 0, 0, 3462, 0, 5, 0, 105, 0, 0, 0] rerouted_tokens_per_ep_rank=[4965, 4206, 3074, 5369, 3490, 3907, 4185, 3572] avg=4096.0000 max=5369 max_rank=3 min=3074 min_rank=2 imbalance_ratio=1.310791 max_load_reduction=27399 max_load_reduction_pct=0.836151 moved_to_echo=15154 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[723, 0, 1097, 17, 976, 321, 0, 0, 733, 0, 3355, 870, 0, 0, 0, 0, 115, 28, 1021, 252, 1993, 383, 0, 0, 8, 3102, 203, 947, 187, 0, 0, 0, 77, 2542, 125, 3149, 0, 0, 0, 0, 50, 1, 0, 48, 861, 877, 0, 0, 2826, 9, 412, 1224, 0, 0, 0, 0, 4094, 3, 19, 26, 94, 0, 0, 0] rerouted_tokens_per_ep_rank=[3134, 4958, 3792, 4447, 5893, 1837, 4471, 4236] avg=4096.0000 max=5893 max_rank=4 min=1837 min_rank=5 imbalance_ratio=1.438721 max_load_reduction=26875 max_load_reduction_pct=0.820160 moved_to_echo=16437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[12416, 0, 12672, 7808, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12416, 2176, 8960, 9472, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 128, 3328, 10112, 15360, 3456, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 640, 15360, 1408, 8832, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 896, 128, 896, 23936, 6784, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[896, 23040, 1024, 6400, 1664, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 4736, 640, 27136, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31488, 128, 384, 128, 896, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1817, 3661, 12, 4095, 9, 2317, 813, 0, 0, 4082, 323, 0, 13, 68, 4, 0, 0, 4092, 3878, 0, 0, 4090, 8, 23, 3317, 0, 0, 1, 142, 1, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[40, 2985, 264, 4005, 33, 0, 4096, 0, 0, 73, 319, 2, 5, 4065, 321, 6, 0, 4093, 135, 0, 96, 57, 159, 4088, 705, 26, 1776, 2, 4042, 0, 142, 1233] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[198, 2883, 1155, 3907, 7, 31, 4095, 0, 5, 3, 3974, 2, 117, 3519, 1, 0, 1, 4095, 821, 0, 1, 235, 11, 3048, 2683, 0, 0, 2, 1590, 0, 330, 54] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1315, 1029, 249, 3135, 419, 2, 4046, 0, 71, 0, 477, 0, 0, 3737, 154, 2658, 30, 4093, 949, 2, 1746, 54, 0, 1982, 2356, 3, 291, 670, 3168, 3, 119, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[2529, 1258, 164, 1373, 2, 2665, 3692, 0, 16, 0, 62, 0, 519, 216, 10, 0, 1, 4045, 1720, 0, 0, 2054, 1298, 61, 4029, 0, 19, 1702, 4095, 0, 2, 1236] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1864, 762, 51, 2741, 0, 0, 4057, 126, 0, 4040, 1014, 1, 1461, 3453, 200, 38, 3, 4079, 1169, 3, 0, 3313, 0, 504, 3603, 6, 2, 2, 126, 1, 148, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[103, 3035, 484, 2514, 3, 71, 4016, 0, 0, 3, 2538, 1, 59, 3753, 2323, 504, 1, 4073, 1051, 0, 473, 0, 0, 3795, 168, 1, 0, 0, 3793, 1, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4086, 235, 11, 81, 58, 0, 3384, 86, 0, 3532, 3076, 13, 10, 4014, 0, 539, 0, 4096, 449, 3977, 41, 5, 0, 28, 12, 0, 0, 2105, 2804, 125, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[40, 2985, 264, 474, 0, 0, 0, 0, 33, 0, 3914, 0, 0, 0, 0, 0, 0, 73, 319, 2, 1096, 71, 0, 0, 5, 4065, 321, 6, 145, 0, 0, 0, 0, 2329, 135, 0, 0, 0, 0, 0, 96, 57, 159, 4088, 668, 37, 0, 0, 705, 26, 1776, 2, 1702, 0, 0, 0, 4042, 0, 142, 1233, 1758, 0, 0, 0] rerouted_tokens_per_ep_rank=[3763, 3947, 1561, 4542, 2464, 5105, 4211, 7175] avg=4096.0000 max=7175 max_rank=7 min=1561 min_rank=2 imbalance_ratio=1.751709 max_load_reduction=25593 max_load_reduction_pct=0.781036 moved_to_echo=18955 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1864, 762, 51, 325, 0, 0, 0, 0, 0, 0, 3876, 126, 0, 0, 0, 0, 0, 4040, 1014, 1, 1092, 49, 0, 0, 1461, 3453, 200, 38, 144, 0, 0, 0, 3, 2321, 1169, 3, 0, 0, 0, 0, 0, 3313, 0, 504, 666, 37, 0, 0, 3603, 6, 2, 2, 1164, 0, 0, 0, 126, 1, 148, 1, 1203, 0, 0, 0] rerouted_tokens_per_ep_rank=[3002, 4002, 6196, 5296, 3496, 4520, 4777, 1479] avg=4096.0000 max=6196 max_rank=2 min=1479 min_rank=7 imbalance_ratio=1.512695 max_load_reduction=26572 max_load_reduction_pct=0.810913 moved_to_echo=14272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1817, 3661, 12, 485, 0, 0, 0, 0, 9, 2317, 778, 0, 0, 0, 0, 0, 0, 4082, 323, 0, 1095, 73, 0, 0, 13, 68, 4, 0, 28, 0, 0, 0, 0, 2329, 3878, 0, 0, 0, 0, 0, 0, 4090, 8, 23, 668, 7, 0, 0, 3317, 0, 0, 1, 1740, 0, 0, 0, 142, 1, 0, 2, 1797, 0, 0, 0] rerouted_tokens_per_ep_rank=[5975, 3104, 5573, 113, 6207, 4796, 5058, 1942] avg=4096.0000 max=6207 max_rank=4 min=113 min_rank=3 imbalance_ratio=1.515381 max_load_reduction=26561 max_load_reduction_pct=0.810577 moved_to_echo=18003 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[198, 2883, 1155, 462, 0, 0, 0, 0, 7, 31, 3913, 0, 0, 0, 0, 0, 5, 3, 3974, 2, 1096, 70, 0, 0, 117, 3519, 1, 0, 145, 0, 0, 0, 1, 2330, 821, 0, 0, 0, 0, 0, 1, 235, 11, 3048, 669, 37, 0, 0, 2683, 0, 0, 2, 1660, 0, 0, 0, 1590, 0, 330, 54, 1715, 0, 0, 0] rerouted_tokens_per_ep_rank=[4698, 3951, 5150, 3782, 3152, 4001, 4345, 3689] avg=4096.0000 max=5150 max_rank=2 min=3152 min_rank=4 imbalance_ratio=1.257324 max_load_reduction=27618 max_load_reduction_pct=0.842834 moved_to_echo=15187 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2529, 1258, 164, 152, 0, 0, 0, 0, 2, 2665, 3521, 0, 0, 0, 0, 0, 16, 0, 62, 0, 1087, 27, 0, 0, 519, 216, 10, 0, 134, 0, 0, 0, 1, 2292, 1720, 0, 0, 0, 0, 0, 0, 2054, 1298, 61, 666, 37, 0, 0, 4029, 0, 19, 1702, 587, 0, 0, 0, 4095, 0, 2, 1236, 607, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 6188, 1192, 879, 4013, 4116, 6337, 5940] avg=4096.0000 max=6337 max_rank=6 min=879 min_rank=3 imbalance_ratio=1.547119 max_load_reduction=26431 max_load_reduction_pct=0.806610 moved_to_echo=20406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1315, 1029, 249, 371, 0, 0, 0, 0, 419, 2, 3866, 0, 0, 0, 0, 0, 71, 0, 477, 0, 1096, 56, 0, 0, 0, 3737, 154, 2658, 143, 0, 0, 0, 30, 2329, 949, 2, 0, 0, 0, 0, 1746, 54, 0, 1982, 668, 37, 0, 0, 2356, 3, 291, 670, 1332, 0, 0, 0, 3168, 3, 119, 10, 1376, 0, 0, 0] rerouted_tokens_per_ep_rank=[2964, 4287, 1700, 6692, 3310, 4487, 4652, 4676] avg=4096.0000 max=6692 max_rank=3 min=1700 min_rank=2 imbalance_ratio=1.633789 max_load_reduction=26076 max_load_reduction_pct=0.795776 moved_to_echo=17125 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[103, 3035, 484, 298, 0, 0, 0, 0, 3, 71, 3838, 0, 0, 0, 0, 0, 0, 3, 2538, 1, 1090, 45, 0, 0, 59, 3753, 2323, 504, 142, 0, 0, 0, 1, 2318, 1051, 0, 0, 0, 0, 0, 473, 0, 0, 3795, 665, 36, 0, 0, 168, 1, 0, 0, 1068, 0, 0, 0, 3793, 1, 0, 5, 1103, 0, 0, 0] rerouted_tokens_per_ep_rank=[3920, 3912, 3677, 6781, 3370, 4969, 1237, 4902] avg=4096.0000 max=6781 max_rank=3 min=1237 min_rank=6 imbalance_ratio=1.655518 max_load_reduction=25987 max_load_reduction_pct=0.793060 moved_to_echo=14478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4086, 235, 11, 11, 0, 0, 0, 0, 58, 0, 3233, 86, 0, 0, 0, 0, 0, 3532, 3076, 13, 1097, 1, 0, 0, 10, 4014, 0, 539, 120, 0, 0, 0, 0, 2330, 449, 3977, 0, 0, 0, 0, 41, 5, 0, 28, 669, 31, 0, 0, 12, 0, 0, 2105, 34, 0, 0, 0, 2804, 125, 0, 1, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[4343, 3377, 7719, 4683, 6756, 774, 2151, 2965] avg=4096.0000 max=7719 max_rank=2 min=774 min_rank=5 imbalance_ratio=1.884521 max_load_reduction=25049 max_load_reduction_pct=0.764435 moved_to_echo=12646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16896, 128, 2176, 4608, 9344, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[19840, 256, 768, 2560, 9600, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 11776, 11904, 128, 8832, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12032, 15872, 2432, 2688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 18688, 10240, 4096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2304, 22912, 3072, 3840, 1024, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2432, 9856, 1536, 13568, 5376, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[640, 5120, 27008, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[3989, 49, 0, 3562, 3086, 0, 460, 0, 1, 4013, 0, 363, 3836, 1, 29, 258, 0, 129, 11, 1344, 977, 0, 0, 4027, 188, 14, 108, 3323, 10, 0, 374, 2616] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[31, 1660, 0, 4096, 3, 0, 1326, 0, 0, 1987, 5, 122, 3698, 3, 41, 19, 0, 4054, 1092, 3983, 2296, 0, 0, 3472, 190, 1, 29, 1271, 1, 0, 164, 3224] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[93, 135, 0, 3525, 2233, 0, 2905, 0, 0, 14, 106, 86, 3308, 0, 3399, 569, 430, 3, 1565, 0, 0, 0, 0, 4017, 7, 1998, 2, 4087, 0, 5, 235, 4046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[3302, 4, 3, 606, 76, 338, 3, 0, 3892, 4061, 1330, 16, 4036, 0, 924, 3664, 20, 2694, 4, 3966, 0, 0, 0, 826, 2, 1361, 1188, 64, 237, 23, 1, 127] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[773, 956, 3, 3298, 3973, 1496, 32, 16, 405, 347, 40, 2, 359, 317, 3229, 15, 1464, 0, 789, 24, 0, 48, 0, 4077, 4023, 63, 0, 2914, 0, 0, 234, 3871] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[648, 78, 0, 4081, 2170, 123, 0, 0, 0, 4091, 2, 49, 3959, 0, 1393, 986, 0, 2045, 112, 3483, 3, 0, 0, 4094, 19, 17, 9, 1341, 12, 1, 42, 4010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[3917, 53, 0, 3672, 4067, 26, 2747, 0, 0, 1199, 0, 18, 273, 232, 3587, 167, 177, 310, 593, 569, 0, 0, 252, 1898, 638, 706, 0, 3627, 0, 1, 4, 4035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[2408, 0, 1, 743, 4095, 1142, 0, 0, 11, 2302, 17, 38, 4096, 0, 123, 1, 0, 2830, 8, 3595, 207, 0, 0, 3942, 12, 505, 0, 695, 2375, 0, 1579, 2043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[31, 1660, 0, 2548, 0, 0, 0, 0, 3, 0, 1326, 0, 575, 0, 0, 0, 0, 1987, 5, 122, 1294, 0, 0, 0, 2169, 3, 41, 19, 0, 0, 0, 0, 0, 4054, 1092, 3391, 0, 0, 0, 0, 2296, 0, 0, 3472, 444, 17, 0, 0, 190, 1, 29, 1271, 497, 235, 0, 0, 1, 0, 164, 3224, 607, 0, 0, 0] rerouted_tokens_per_ep_rank=[4239, 1904, 3408, 2232, 8537, 6229, 2223, 3996] avg=4096.0000 max=8537 max_rank=4 min=1904 min_rank=1 imbalance_ratio=2.084229 max_load_reduction=24231 max_load_reduction_pct=0.739471 moved_to_echo=20985 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3989, 49, 0, 2216, 0, 0, 0, 0, 3086, 0, 460, 0, 194, 0, 0, 0, 1, 4013, 0, 363, 1343, 0, 0, 0, 2249, 1, 29, 258, 0, 0, 0, 0, 0, 129, 11, 1145, 0, 0, 0, 0, 977, 0, 0, 4027, 386, 5, 0, 0, 188, 14, 108, 3323, 432, 244, 0, 0, 10, 0, 374, 2616, 528, 0, 0, 0] rerouted_tokens_per_ep_rank=[6254, 3740, 5720, 2537, 1285, 5395, 4309, 3528] avg=4096.0000 max=6254 max_rank=0 min=1285 min_rank=4 imbalance_ratio=1.526855 max_load_reduction=26514 max_load_reduction_pct=0.809143 moved_to_echo=14517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3302, 4, 3, 379, 0, 0, 0, 0, 76, 338, 3, 0, 573, 0, 0, 0, 3892, 4061, 1330, 16, 1413, 0, 0, 0, 2366, 0, 924, 3664, 0, 0, 0, 0, 20, 2694, 4, 3376, 0, 0, 0, 0, 0, 0, 0, 826, 65, 17, 0, 0, 2, 1361, 1188, 64, 73, 257, 0, 0, 237, 23, 1, 127, 89, 0, 0, 0] rerouted_tokens_per_ep_rank=[3688, 990, 10712, 6954, 6094, 908, 2945, 477] avg=4096.0000 max=10712 max_rank=2 min=477 min_rank=7 imbalance_ratio=2.615234 max_load_reduction=22056 max_load_reduction_pct=0.673096 moved_to_echo=10424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[773, 956, 3, 2052, 0, 0, 0, 0, 3973, 1496, 32, 16, 3, 0, 0, 0, 405, 347, 40, 2, 125, 0, 0, 0, 212, 317, 3229, 15, 0, 0, 0, 0, 1464, 0, 789, 21, 0, 0, 0, 0, 0, 48, 0, 4077, 357, 0, 0, 0, 4023, 63, 0, 2914, 400, 22, 0, 0, 0, 0, 234, 3871, 489, 0, 0, 0] rerouted_tokens_per_ep_rank=[3784, 5520, 919, 3773, 2274, 4482, 7422, 4594] avg=4096.0000 max=7422 max_rank=6 min=919 min_rank=2 imbalance_ratio=1.812012 max_load_reduction=25346 max_load_reduction_pct=0.773499 moved_to_echo=18772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[648, 78, 0, 2539, 0, 0, 0, 0, 2170, 123, 0, 0, 503, 0, 0, 0, 0, 4091, 2, 49, 1386, 0, 0, 0, 2321, 0, 1393, 986, 0, 0, 0, 0, 0, 2045, 112, 2965, 0, 0, 0, 0, 3, 0, 0, 4094, 442, 15, 0, 0, 19, 17, 9, 1341, 495, 252, 0, 0, 12, 1, 42, 4010, 605, 0, 0, 0] rerouted_tokens_per_ep_rank=[3265, 2796, 5528, 4700, 5122, 4554, 2133, 4670] avg=4096.0000 max=5528 max_rank=2 min=2133 min_rank=6 imbalance_ratio=1.349609 max_load_reduction=27240 max_load_reduction_pct=0.831299 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3917, 53, 0, 2274, 0, 0, 0, 0, 4067, 26, 2747, 0, 84, 0, 0, 0, 0, 1199, 0, 18, 98, 0, 0, 0, 153, 232, 3587, 167, 0, 0, 0, 0, 177, 310, 593, 479, 0, 0, 0, 0, 0, 0, 252, 1898, 401, 6, 0, 0, 638, 706, 0, 3627, 449, 22, 0, 0, 0, 1, 4, 4035, 548, 0, 0, 0] rerouted_tokens_per_ep_rank=[6244, 6924, 1315, 4139, 1559, 2557, 5442, 4588] avg=4096.0000 max=6924 max_rank=1 min=1315 min_rank=2 imbalance_ratio=1.690430 max_load_reduction=25844 max_load_reduction_pct=0.788696 moved_to_echo=14146 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[93, 135, 0, 2194, 0, 0, 0, 0, 2233, 0, 2905, 0, 0, 0, 0, 0, 0, 14, 106, 86, 1158, 0, 0, 0, 1940, 0, 3399, 569, 0, 0, 0, 0, 430, 3, 1565, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 382, 0, 0, 0, 7, 1998, 2, 4087, 427, 210, 0, 0, 0, 5, 235, 4046, 522, 0, 0, 0] rerouted_tokens_per_ep_rank=[2422, 5138, 1364, 5908, 1998, 4399, 6731, 4808] avg=4096.0000 max=6731 max_rank=6 min=1364 min_rank=2 imbalance_ratio=1.643311 max_load_reduction=26037 max_load_reduction_pct=0.794586 moved_to_echo=17936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2408, 0, 1, 463, 0, 0, 0, 0, 4095, 1142, 0, 0, 519, 0, 0, 0, 11, 2302, 17, 38, 1434, 0, 0, 0, 2401, 0, 123, 1, 0, 0, 0, 0, 0, 2830, 8, 3061, 0, 0, 0, 0, 207, 0, 0, 3942, 80, 15, 0, 0, 12, 505, 0, 695, 90, 261, 0, 0, 2375, 0, 1579, 2043, 110, 0, 0, 0] rerouted_tokens_per_ep_rank=[2872, 5756, 3802, 2525, 5899, 4244, 1563, 6107] avg=4096.0000 max=6107 max_rank=7 min=1563 min_rank=6 imbalance_ratio=1.490967 max_load_reduction=26661 max_load_reduction_pct=0.813629 moved_to_echo=17813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2176, 12160, 4224, 14464, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4352, 18048, 1536, 768, 8320, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15232, 2944, 128, 14720, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5120, 4736, 1408, 17408, 2944, 1536, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19712, 3200, 7552, 128, 2560, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13824, 640, 12800, 5760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2688, 128, 2688, 24064, 3584, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3584, 128, 256, 26368, 2560, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1003, 0, 4096, 2, 1296, 0, 0, 3996, 765, 2, 336, 4020, 2855, 33, 3, 1, 631, 0, 0, 0, 0, 0, 16, 4020, 0, 613, 1469, 64, 0, 3447, 4096, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4073, 0, 4082, 0, 3, 246, 0, 231, 52, 0, 3604, 3942, 2930, 2, 0, 20, 227, 0, 167, 0, 26, 0, 0, 3043, 35, 931, 0, 2055, 0, 3266, 3833, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4082, 0, 2969, 158, 1499, 2599, 24, 0, 1472, 5, 423, 4057, 3694, 1161, 256, 0, 2166, 0, 18, 1, 0, 38, 0, 3, 1, 2046, 2014, 0, 0, 4, 4078, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1469, 0, 4095, 0, 158, 0, 206, 4095, 4096, 0, 68, 3823, 3687, 14, 0, 968, 0, 0, 603, 0, 199, 0, 2516, 2978, 22, 946, 1912, 11, 0, 40, 803, 59] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1108, 0, 2368, 0, 194, 0, 13, 103, 3847, 0, 4093, 53, 35, 608, 980, 125, 1, 0, 3853, 0, 4092, 0, 2263, 1158, 0, 2808, 32, 0, 0, 2598, 2436, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3508, 0, 2836, 0, 4096, 74, 27, 4083, 53, 0, 90, 3312, 12, 4039, 559, 0, 1248, 0, 0, 0, 346, 122, 0, 16, 66, 177, 155, 3, 0, 3884, 4062, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[2464, 1, 1449, 304, 0, 3, 1197, 2563, 3933, 1, 1313, 4083, 827, 550, 10, 106, 130, 0, 238, 2, 2092, 242, 0, 3353, 1, 25, 5, 0, 4, 3813, 4040, 19] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[2929, 0, 30, 8, 16, 25, 568, 2000, 3855, 3, 3117, 4082, 1610, 9, 3264, 5, 3, 0, 0, 0, 0, 998, 1516, 35, 0, 1774, 211, 39, 0, 2561, 4093, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1003, 0, 2179, 2, 0, 0, 0, 0, 1296, 0, 0, 3996, 645, 0, 0, 0, 765, 2, 336, 242, 0, 0, 0, 0, 2855, 33, 3, 1, 582, 192, 0, 0, 631, 0, 0, 0, 3448, 0, 0, 0, 0, 0, 16, 4020, 690, 0, 0, 0, 0, 613, 1469, 64, 1955, 330, 0, 0, 0, 3447, 1949, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3184, 5937, 1345, 3666, 4079, 4726, 4431, 5400] avg=4096.0000 max=5937 max_rank=1 min=1345 min_rank=2 imbalance_ratio=1.449463 max_load_reduction=26831 max_load_reduction_pct=0.818817 moved_to_echo=18636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4082, 0, 1580, 158, 0, 0, 0, 0, 1499, 2599, 24, 0, 467, 0, 0, 0, 1472, 5, 423, 244, 0, 0, 0, 0, 3694, 1161, 256, 0, 422, 191, 0, 0, 2166, 0, 18, 1, 3480, 0, 0, 0, 0, 38, 0, 3, 500, 0, 0, 0, 1, 2046, 2014, 0, 1946, 333, 0, 0, 0, 4, 1941, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5820, 4589, 2144, 5724, 5665, 541, 6340, 1945] avg=4096.0000 max=6340 max_rank=6 min=541 min_rank=5 imbalance_ratio=1.547852 max_load_reduction=26428 max_load_reduction_pct=0.806519 moved_to_echo=14491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1108, 0, 1261, 0, 0, 0, 0, 0, 194, 0, 13, 103, 372, 0, 0, 0, 3847, 0, 4093, 4, 0, 0, 0, 0, 35, 608, 980, 125, 336, 114, 0, 0, 1, 0, 3853, 0, 45, 0, 0, 0, 4092, 0, 2263, 1158, 399, 0, 0, 0, 0, 2808, 32, 0, 1163, 4, 0, 0, 0, 2598, 1159, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2369, 682, 7944, 2198, 3899, 7912, 4007, 3757] avg=4096.0000 max=7944 max_rank=2 min=682 min_rank=1 imbalance_ratio=1.939453 max_load_reduction=24824 max_load_reduction_pct=0.757568 moved_to_echo=19575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1469, 0, 2169, 0, 0, 0, 0, 0, 158, 0, 206, 4095, 649, 0, 0, 0, 4096, 0, 68, 222, 0, 0, 0, 0, 3687, 14, 0, 968, 585, 41, 0, 0, 0, 0, 603, 0, 3282, 0, 0, 0, 199, 0, 2516, 2978, 692, 0, 0, 0, 22, 946, 1912, 11, 387, 319, 0, 0, 0, 40, 375, 59, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3638, 5108, 4386, 5295, 3885, 6385, 3597, 474] avg=4096.0000 max=6385 max_rank=5 min=474 min_rank=7 imbalance_ratio=1.558838 max_load_reduction=26383 max_load_reduction_pct=0.805145 moved_to_echo=14341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2464, 1, 772, 304, 0, 0, 0, 0, 0, 3, 1197, 2563, 228, 0, 0, 0, 3933, 1, 1313, 246, 0, 0, 0, 0, 827, 550, 10, 106, 205, 189, 0, 0, 130, 0, 238, 2, 3502, 0, 0, 0, 2092, 242, 0, 3353, 244, 0, 0, 0, 1, 25, 5, 0, 1928, 335, 0, 0, 4, 3813, 1923, 19, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3541, 3991, 5493, 1887, 3872, 5931, 2294, 5759] avg=4096.0000 max=5931 max_rank=5 min=1887 min_rank=3 imbalance_ratio=1.447998 max_load_reduction=26837 max_load_reduction_pct=0.819000 moved_to_echo=17856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3508, 0, 1509, 0, 0, 0, 0, 0, 4096, 74, 27, 4083, 446, 0, 0, 0, 53, 0, 90, 199, 0, 0, 0, 0, 12, 4039, 559, 0, 403, 190, 0, 0, 1248, 0, 0, 0, 2841, 0, 0, 0, 346, 122, 0, 16, 478, 0, 0, 0, 66, 177, 155, 3, 1939, 272, 0, 0, 0, 3884, 1933, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5017, 8726, 342, 5203, 4089, 962, 2612, 5817] avg=4096.0000 max=8726 max_rank=1 min=342 min_rank=2 imbalance_ratio=2.130371 max_load_reduction=24042 max_load_reduction_pct=0.733704 moved_to_echo=13480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 2172, 0, 0, 0, 0, 0, 3, 246, 0, 231, 642, 0, 0, 0, 52, 0, 3604, 237, 0, 0, 0, 0, 2930, 2, 0, 20, 580, 179, 0, 0, 227, 0, 167, 0, 3381, 0, 0, 0, 26, 0, 0, 3043, 688, 0, 0, 0, 35, 931, 0, 2055, 1829, 324, 0, 0, 0, 3266, 1825, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6245, 1122, 3893, 3711, 3775, 3757, 5174, 5091] avg=4096.0000 max=6245 max_rank=0 min=1122 min_rank=1 imbalance_ratio=1.524658 max_load_reduction=26523 max_load_reduction_pct=0.809418 moved_to_echo=17797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2929, 0, 17, 8, 0, 0, 0, 0, 16, 25, 568, 2000, 4, 0, 0, 0, 3855, 3, 3117, 246, 0, 0, 0, 0, 1610, 9, 3264, 5, 4, 192, 0, 0, 3, 0, 0, 0, 3501, 0, 0, 0, 0, 998, 1516, 35, 5, 0, 0, 0, 0, 1774, 211, 39, 1954, 335, 0, 0, 0, 2561, 1947, 17, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2954, 2613, 7221, 5084, 3504, 2554, 4313, 4525] avg=4096.0000 max=7221 max_rank=2 min=2554 min_rank=5 imbalance_ratio=1.762939 max_load_reduction=25547 max_load_reduction_pct=0.779633 moved_to_echo=14896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6784, 1408, 6400, 14720, 3712, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[18176, 128, 13056, 1664, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[20736, 128, 11776, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7296, 3072, 2048, 17152, 3456, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 9344, 5888, 2176, 13184, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 19712, 13056, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15744, 6528, 5120, 1280, 3200, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4480, 0, 4992, 128, 23552, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[8, 2782, 1785, 1, 4, 0, 15, 281, 362, 113, 74, 3337, 4, 4090, 4096, 0, 2019, 11, 38, 7, 1260, 1, 0, 0, 562, 4078, 1, 3725, 0, 4029, 18, 67] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[452, 159, 1573, 94, 42, 0, 2681, 3565, 9, 0, 3504, 4012, 0, 122, 204, 0, 328, 527, 0, 1252, 0, 864, 183, 2135, 3737, 15, 304, 3793, 0, 3178, 0, 35] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 7, 1340, 2131, 0, 84, 7, 3815, 32, 0, 4081, 4096, 0, 4094, 5, 1227, 2902, 0, 0, 0, 139, 0, 126, 0, 929, 164, 9, 4092, 7, 3481, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3259, 163, 0, 0, 2061, 0, 1416, 83, 0, 2354, 0, 4096, 3886, 0, 10, 6, 0, 1050, 0, 3913, 3, 0, 32, 3990, 611, 2952, 0, 2690, 1, 192] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 491, 49, 0, 47, 2, 101, 57, 3789, 4048, 4089, 0, 3465, 3195, 0, 0, 68, 0, 0, 640, 0, 0, 41, 3935, 4096, 0, 4069, 227, 358, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[384, 90, 1, 1411, 0, 46, 3863, 1356, 3070, 416, 1643, 3502, 97, 4070, 361, 238, 0, 163, 0, 0, 3234, 6, 0, 17, 749, 621, 1410, 2922, 29, 1, 1722, 1346] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 1063, 1, 0, 0, 4095, 74, 0, 88, 12, 2859, 0, 4096, 4079, 0, 4094, 14, 0, 0, 0, 12, 4, 0, 0, 4096, 1, 4095, 246, 3838, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 54, 326, 26, 0, 0, 0, 9, 2, 0, 0, 4096, 0, 3761, 4096, 0, 4073, 0, 0, 0, 7, 362, 0, 322, 3004, 60, 398, 3912, 281, 4051, 0, 3928] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[8, 2782, 1785, 1, 1439, 424, 0, 0, 4, 0, 15, 281, 1562, 0, 0, 0, 362, 113, 74, 1175, 0, 0, 0, 0, 4, 1662, 4096, 0, 0, 0, 0, 0, 2019, 11, 38, 7, 1267, 723, 0, 0, 1260, 1, 0, 0, 2458, 0, 0, 0, 562, 4070, 1, 0, 0, 0, 0, 0, 0, 4029, 18, 67, 442, 8, 0, 0] rerouted_tokens_per_ep_rank=[6439, 1862, 1724, 5762, 4065, 3719, 4633, 4564] avg=4096.0000 max=6439 max_rank=0 min=1724 min_rank=2 imbalance_ratio=1.572021 max_load_reduction=26329 max_load_reduction_pct=0.803497 moved_to_echo=16981 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 491, 49, 1764, 359, 0, 0, 0, 47, 2, 101, 1323, 0, 0, 0, 57, 3789, 4048, 1439, 0, 0, 0, 0, 0, 1409, 3195, 0, 0, 0, 0, 0, 0, 68, 0, 0, 1385, 886, 0, 0, 640, 0, 0, 41, 2684, 0, 0, 0, 3935, 4088, 0, 0, 0, 0, 0, 0, 227, 358, 0, 1, 374, 8, 0, 0] rerouted_tokens_per_ep_rank=[2663, 1473, 9333, 4604, 2339, 3365, 8023, 968] avg=4096.0000 max=9333 max_rank=2 min=968 min_rank=7 imbalance_ratio=2.278564 max_load_reduction=23435 max_load_reduction_pct=0.715179 moved_to_echo=14695 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[384, 90, 1, 1411, 1511, 422, 0, 0, 0, 46, 3863, 1356, 1555, 0, 0, 0, 3070, 416, 1643, 1232, 0, 0, 0, 0, 97, 1653, 361, 238, 0, 0, 0, 0, 0, 163, 0, 0, 995, 759, 0, 0, 3234, 6, 0, 17, 1927, 0, 0, 0, 749, 620, 1410, 0, 0, 0, 0, 0, 29, 1, 1722, 1346, 440, 1, 0, 0] rerouted_tokens_per_ep_rank=[3819, 6820, 6361, 2349, 1917, 5184, 2779, 3539] avg=4096.0000 max=6820 max_rank=1 min=1917 min_rank=4 imbalance_ratio=1.665039 max_load_reduction=25948 max_load_reduction_pct=0.791870 moved_to_echo=13419 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 1340, 2131, 1767, 425, 0, 0, 0, 84, 7, 3815, 1564, 0, 0, 0, 32, 0, 4081, 1442, 0, 0, 0, 0, 0, 1663, 5, 1227, 0, 0, 0, 0, 2902, 0, 0, 0, 1393, 887, 0, 0, 139, 0, 126, 0, 2699, 0, 0, 0, 929, 164, 9, 0, 0, 0, 0, 0, 7, 3481, 0, 0, 442, 0, 0, 0] rerouted_tokens_per_ep_rank=[5670, 5470, 5555, 2895, 5182, 2964, 1102, 3930] avg=4096.0000 max=5670 max_rank=0 min=1102 min_rank=6 imbalance_ratio=1.384277 max_load_reduction=27098 max_load_reduction_pct=0.826965 moved_to_echo=13178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1063, 1, 1233, 425, 0, 0, 0, 0, 4095, 74, 1565, 0, 0, 0, 0, 88, 12, 1007, 0, 0, 0, 0, 0, 1663, 4079, 0, 0, 0, 0, 0, 4094, 14, 0, 0, 1394, 619, 0, 0, 0, 12, 4, 0, 2701, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 246, 3838, 0, 1, 443, 8, 0, 0] rerouted_tokens_per_ep_rank=[2722, 5734, 1107, 5742, 6121, 2717, 4089, 4536] avg=4096.0000 max=6121 max_rank=4 min=1107 min_rank=2 imbalance_ratio=1.494385 max_load_reduction=26647 max_load_reduction_pct=0.813202 moved_to_echo=17463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[452, 159, 1573, 94, 1731, 12, 0, 0, 42, 0, 2681, 3565, 46, 0, 0, 0, 9, 0, 3504, 1412, 0, 0, 0, 0, 0, 51, 204, 0, 0, 0, 0, 0, 328, 527, 0, 1252, 1291, 869, 0, 0, 0, 864, 183, 2135, 2502, 0, 0, 0, 3737, 15, 304, 0, 0, 0, 0, 0, 0, 3178, 0, 35, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 6334, 4925, 255, 4267, 5684, 4056, 3226] avg=4096.0000 max=6334 max_rank=1 min=255 min_rank=3 imbalance_ratio=1.546387 max_load_reduction=26434 max_load_reduction_pct=0.806702 moved_to_echo=17233 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3259, 163, 1019, 430, 0, 0, 0, 0, 2061, 0, 1568, 0, 0, 0, 1416, 83, 0, 820, 0, 0, 0, 0, 0, 1651, 3886, 0, 0, 0, 0, 0, 10, 6, 0, 1050, 1005, 515, 0, 0, 0, 3913, 3, 0, 1947, 0, 0, 0, 32, 3981, 611, 0, 0, 0, 0, 0, 0, 2690, 1, 192, 447, 9, 0, 0] rerouted_tokens_per_ep_rank=[4871, 3629, 2319, 5537, 2586, 5863, 4624, 3339] avg=4096.0000 max=5863 max_rank=5 min=2319 min_rank=2 imbalance_ratio=1.431396 max_load_reduction=26905 max_load_reduction_pct=0.821075 moved_to_echo=16412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 54, 326, 26, 1767, 390, 0, 0, 0, 0, 0, 9, 1437, 0, 0, 0, 2, 0, 0, 1442, 0, 0, 0, 0, 0, 1528, 4096, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 1331, 887, 0, 0, 7, 362, 0, 322, 2581, 0, 0, 0, 3004, 60, 398, 0, 0, 0, 0, 0, 281, 4051, 0, 3928, 406, 0, 0, 0] rerouted_tokens_per_ep_rank=[2563, 1446, 1444, 5624, 6291, 3272, 3462, 8666] avg=4096.0000 max=8666 max_rank=7 min=1444 min_rank=2 imbalance_ratio=2.115723 max_load_reduction=24102 max_load_reduction_pct=0.735535 moved_to_echo=21691 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 3200, 9856, 3968, 12288, 2944, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4992, 4608, 13440, 9984, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 11392, 19968, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 256, 12800, 9216, 10624, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 21632, 1792, 5632, 3072, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[13440, 896, 128, 2432, 10112, 6272, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5376, 5248, 384, 2560, 19584, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13056, 17152, 2816, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[960, 0, 2199, 4068, 4062, 67, 4054, 646, 3937, 2, 0, 2114, 4033, 0, 3930, 35, 57, 626, 890, 5, 292, 0, 0, 0, 0, 389, 0, 11, 2, 0, 0, 389] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[48, 0, 2416, 90, 3088, 264, 0, 115, 3653, 284, 3807, 4033, 840, 0, 3922, 0, 4096, 1511, 0, 199, 97, 0, 0, 223, 0, 0, 246, 76, 263, 449, 2112, 936] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 18, 3616, 3992, 0, 3982, 1146, 419, 0, 4077, 16, 0, 2, 11, 3636, 4079, 0, 2790, 540, 4, 0, 0, 0, 287, 0, 158, 3955, 0, 2, 38, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[53, 0, 3672, 229, 4, 86, 111, 2511, 0, 0, 3449, 0, 0, 2, 3743, 1531, 4024, 0, 316, 198, 1, 484, 691, 781, 366, 0, 182, 2710, 4020, 3561, 38, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[8, 0, 2857, 14, 4096, 105, 538, 5, 6, 1, 1333, 3990, 1, 0, 2785, 439, 3859, 592, 3876, 183, 102, 0, 32, 10, 0, 3784, 0, 3847, 9, 0, 296, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 160, 0, 4054, 1108, 0, 3923, 17, 3909, 107, 4096, 0, 930, 0, 20, 8, 2113, 67, 3645, 0, 171, 443, 0, 192, 19, 0, 2, 4093, 3, 3673, 0, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[915, 0, 643, 0, 2853, 613, 152, 0, 6, 0, 3972, 1995, 1, 76, 3266, 0, 3576, 0, 242, 427, 0, 0, 44, 495, 3642, 1810, 0, 4096, 3834, 0, 0, 110] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[343, 44, 1097, 2762, 3312, 56, 7, 4032, 757, 2655, 721, 551, 3473, 0, 274, 0, 3702, 217, 0, 0, 10, 0, 0, 2, 55, 0, 89, 3149, 271, 1176, 3943, 70] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[960, 0, 2199, 4068, 4, 0, 41, 0, 1866, 67, 4054, 646, 0, 0, 0, 0, 3937, 2, 0, 2114, 0, 0, 0, 0, 4033, 0, 3889, 35, 0, 0, 0, 0, 38, 626, 890, 5, 0, 0, 0, 0, 292, 0, 0, 0, 0, 2087, 0, 0, 0, 389, 0, 11, 0, 0, 0, 0, 2, 0, 0, 389, 15, 109, 0, 0] rerouted_tokens_per_ep_rank=[7272, 6633, 6053, 7957, 1559, 2379, 400, 515] avg=4096.0000 max=7957 max_rank=3 min=400 min_rank=6 imbalance_ratio=1.942627 max_load_reduction=24811 max_load_reduction_pct=0.757172 moved_to_echo=4853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 18, 3616, 338, 35, 0, 0, 1834, 0, 3982, 1146, 0, 0, 0, 0, 419, 0, 824, 16, 0, 0, 0, 0, 0, 2, 11, 3636, 0, 0, 0, 0, 2630, 0, 2790, 540, 0, 0, 0, 0, 4, 0, 0, 0, 3253, 2051, 0, 0, 287, 0, 158, 3920, 0, 0, 0, 0, 0, 2, 38, 0, 1111, 107, 0, 0] rerouted_tokens_per_ep_rank=[4007, 6962, 1259, 3649, 5960, 5308, 4365, 1258] avg=4096.0000 max=6962 max_rank=1 min=1258 min_rank=7 imbalance_ratio=1.699707 max_load_reduction=25806 max_load_reduction_pct=0.787537 moved_to_echo=16891 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[48, 0, 2416, 90, 339, 0, 41, 0, 1419, 264, 0, 115, 0, 0, 0, 0, 3653, 284, 769, 4033, 0, 0, 0, 0, 840, 0, 3881, 0, 0, 0, 0, 0, 2642, 1511, 0, 199, 0, 0, 0, 0, 97, 0, 0, 223, 3038, 1587, 0, 0, 0, 0, 246, 76, 0, 0, 0, 0, 263, 449, 2112, 936, 1115, 82, 0, 0] rerouted_tokens_per_ep_rank=[2934, 1798, 8739, 4721, 4352, 4945, 322, 4957] avg=4096.0000 max=8739 max_rank=2 min=322 min_rank=6 imbalance_ratio=2.133545 max_load_reduction=24029 max_load_reduction_pct=0.733307 moved_to_echo=14576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[53, 0, 3672, 229, 333, 24, 39, 0, 2, 86, 111, 2511, 0, 0, 0, 0, 0, 0, 697, 0, 0, 0, 0, 0, 0, 2, 3704, 1531, 0, 0, 0, 0, 2595, 0, 316, 198, 0, 0, 0, 0, 1, 484, 691, 781, 2752, 2, 0, 0, 366, 0, 182, 2686, 0, 0, 0, 0, 4020, 3561, 38, 5, 1096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4350, 2710, 697, 5237, 3109, 4711, 3234, 8720] avg=4096.0000 max=8720 max_rank=7 min=697 min_rank=2 imbalance_ratio=2.128906 max_load_reduction=24048 max_load_reduction_pct=0.733887 moved_to_echo=19774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 160, 0, 4054, 175, 36, 0, 0, 510, 0, 3923, 17, 0, 0, 0, 0, 3909, 107, 828, 0, 0, 0, 0, 0, 930, 0, 20, 8, 0, 0, 0, 0, 1363, 67, 3645, 0, 0, 0, 0, 0, 171, 443, 0, 192, 3268, 569, 0, 0, 19, 0, 2, 4057, 0, 0, 0, 0, 3, 3673, 0, 15, 575, 29, 0, 0] rerouted_tokens_per_ep_rank=[4425, 4450, 4844, 958, 5075, 4643, 4078, 4295] avg=4096.0000 max=5075 max_rank=4 min=958 min_rank=3 imbalance_ratio=1.239014 max_load_reduction=27693 max_load_reduction_pct=0.845123 moved_to_echo=18091 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 2857, 14, 323, 39, 33, 0, 1874, 105, 538, 5, 0, 0, 0, 0, 6, 1, 266, 3990, 0, 0, 0, 0, 1, 0, 2752, 439, 0, 0, 0, 0, 2482, 592, 3876, 183, 0, 0, 0, 0, 102, 0, 32, 10, 1067, 2108, 0, 0, 0, 3784, 0, 3808, 0, 0, 0, 0, 9, 0, 296, 0, 1054, 114, 0, 0] rerouted_tokens_per_ep_rank=[3274, 2522, 4263, 3192, 7133, 3319, 7592, 1473] avg=4096.0000 max=7592 max_rank=6 min=1473 min_rank=7 imbalance_ratio=1.853516 max_load_reduction=25176 max_load_reduction_pct=0.768311 moved_to_echo=19517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[915, 0, 643, 0, 296, 36, 34, 0, 1311, 613, 152, 0, 0, 0, 0, 0, 6, 0, 803, 1995, 0, 0, 0, 0, 1, 76, 3232, 0, 0, 0, 0, 0, 2306, 0, 242, 427, 0, 0, 0, 0, 0, 0, 44, 495, 3169, 1466, 0, 0, 3642, 1810, 0, 4060, 0, 0, 0, 0, 3834, 0, 0, 110, 974, 76, 0, 0] rerouted_tokens_per_ep_rank=[1924, 2076, 2804, 3309, 2975, 5174, 9512, 4994] avg=4096.0000 max=9512 max_rank=6 min=1924 min_rank=0 imbalance_ratio=2.322266 max_load_reduction=23256 max_load_reduction_pct=0.709717 moved_to_echo=22655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[343, 44, 1097, 2762, 306, 28, 2, 0, 1522, 56, 7, 4032, 0, 0, 0, 0, 757, 2655, 146, 551, 0, 0, 0, 0, 3473, 0, 272, 0, 0, 0, 0, 0, 2388, 217, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 575, 1702, 0, 0, 55, 0, 89, 3121, 0, 0, 0, 0, 271, 1176, 3943, 70, 1008, 88, 0, 0] rerouted_tokens_per_ep_rank=[4582, 5617, 4109, 3745, 2605, 2289, 3265, 6556] avg=4096.0000 max=6556 max_rank=7 min=2289 min_rank=5 imbalance_ratio=1.600586 max_load_reduction=26212 max_load_reduction_pct=0.799927 moved_to_echo=14715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8960, 6528, 1536, 7040, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10368, 1280, 12800, 8576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12800, 3072, 4352, 12800, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16512, 3072, 11776, 1664, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4480, 6016, 768, 21760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2432, 256, 12928, 14848, 2176, 256, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 1024, 768, 1792, 17152, 11648, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9344, 128, 17792, 5760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1960, 3467, 530, 3930, 0, 5, 4090, 78, 3437, 3281, 0, 0, 545, 0, 0, 0, 1361, 60, 0, 4, 429, 349, 4096, 49, 453, 790, 8, 0, 1, 0, 3845, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[726, 0, 2031, 4079, 0, 707, 1320, 1652, 2449, 3992, 0, 0, 0, 28, 0, 0, 15, 0, 2657, 121, 3912, 86, 4065, 3791, 17, 1008, 0, 0, 0, 0, 83, 29] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[21, 15, 202, 4086, 1542, 3440, 4034, 3967, 830, 97, 71, 599, 0, 0, 30, 55, 0, 0, 0, 0, 4092, 2871, 4070, 8, 2, 0, 0, 332, 408, 2, 1994, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 357, 0, 568, 2, 81, 4096, 12, 1605, 1590, 0, 34, 3796, 0, 4034, 0, 124, 0, 11, 3, 1454, 2906, 4054, 0, 0, 3457, 0, 1, 3809, 56, 718, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1, 92, 3532, 2, 877, 6, 3893, 3335, 4021, 132, 1437, 229, 114, 98, 2902, 0, 3234, 2208, 80, 0, 83, 0, 15, 0, 3980, 1, 0, 0, 623, 1861, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 1528, 0, 709, 12, 4039, 4096, 66, 3262, 9, 24, 0, 0, 0, 0, 0, 2066, 0, 86, 1, 4020, 3130, 2827, 3295, 1888, 129, 0, 0, 467, 26, 1088, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 746, 0, 4095, 4087, 92, 31, 0, 2778, 1693, 0, 4, 504, 0, 0, 0, 554, 16, 0, 76, 4079, 0, 4061, 1, 3136, 2, 0, 0, 2160, 630, 4023, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 1389, 248, 3964, 3414, 3550, 590, 1378, 3736, 2902, 15, 12, 3257, 0, 39, 0, 0, 493, 1407, 0, 2164, 311, 19, 1590, 135, 0, 589, 0, 1158, 20, 388, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[21, 15, 202, 3037, 0, 0, 0, 0, 1542, 3440, 78, 3967, 0, 0, 0, 0, 625, 97, 71, 599, 0, 0, 0, 0, 0, 0, 30, 55, 2249, 879, 0, 0, 0, 0, 0, 0, 3191, 0, 0, 0, 2909, 2871, 0, 8, 0, 0, 0, 0, 2, 0, 0, 332, 1707, 1183, 307, 0, 408, 2, 1994, 0, 205, 742, 0, 0] rerouted_tokens_per_ep_rank=[3275, 9027, 1392, 3213, 3191, 5788, 3531, 3351] avg=4096.0000 max=9027 max_rank=1 min=1392 min_rank=2 imbalance_ratio=2.203857 max_load_reduction=23741 max_load_reduction_pct=0.724518 moved_to_echo=15861 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[726, 0, 2031, 3031, 0, 0, 0, 0, 0, 707, 27, 1652, 0, 0, 0, 0, 1844, 3992, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 735, 878, 0, 0, 15, 0, 2657, 121, 3187, 0, 0, 0, 2781, 86, 0, 3791, 0, 0, 0, 0, 17, 1008, 0, 0, 558, 1131, 307, 0, 0, 0, 83, 29, 605, 741, 0, 0] rerouted_tokens_per_ep_rank=[5788, 2386, 5836, 1641, 5980, 6658, 3021, 1458] avg=4096.0000 max=6658 max_rank=5 min=1458 min_rank=7 imbalance_ratio=1.625488 max_load_reduction=26110 max_load_reduction_pct=0.796814 moved_to_echo=17117 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1960, 3467, 530, 2921, 0, 0, 0, 0, 0, 5, 80, 78, 0, 0, 0, 0, 2587, 3281, 0, 0, 0, 0, 0, 0, 545, 0, 0, 0, 2280, 885, 0, 0, 1361, 60, 0, 4, 3211, 0, 0, 0, 305, 349, 0, 49, 0, 0, 0, 0, 453, 790, 8, 0, 1730, 124, 295, 0, 1, 0, 3845, 0, 850, 714, 0, 0] rerouted_tokens_per_ep_rank=[8878, 163, 5868, 3710, 4636, 703, 3400, 5410] avg=4096.0000 max=8878 max_rank=0 min=163 min_rank=1 imbalance_ratio=2.167480 max_load_reduction=23890 max_load_reduction_pct=0.729065 moved_to_echo=14149 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 92, 3532, 2, 0, 0, 0, 0, 877, 6, 76, 3335, 0, 0, 0, 0, 3027, 132, 1437, 229, 0, 0, 0, 0, 114, 98, 2902, 0, 2170, 4, 0, 0, 3234, 2208, 80, 0, 11, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 3980, 1, 0, 0, 1647, 24, 0, 0, 623, 1861, 12, 0, 994, 0, 0, 0] rerouted_tokens_per_ep_rank=[3627, 4294, 4825, 5288, 5533, 59, 5652, 3490] avg=4096.0000 max=5652 max_rank=6 min=59 min_rank=5 imbalance_ratio=1.379883 max_load_reduction=27116 max_load_reduction_pct=0.827515 moved_to_echo=14734 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 357, 0, 423, 0, 0, 0, 0, 2, 81, 80, 12, 0, 0, 0, 0, 1208, 1590, 0, 34, 0, 0, 0, 0, 3796, 0, 4034, 0, 2283, 876, 0, 0, 124, 0, 11, 3, 3178, 0, 0, 0, 1034, 2906, 0, 0, 0, 0, 0, 0, 0, 3457, 0, 1, 1733, 420, 42, 0, 3809, 56, 718, 0, 397, 103, 0, 0] rerouted_tokens_per_ep_rank=[780, 175, 2832, 10989, 3316, 3940, 5653, 5083] avg=4096.0000 max=10989 max_rank=3 min=175 min_rank=1 imbalance_ratio=2.682861 max_load_reduction=21779 max_load_reduction_pct=0.664642 moved_to_echo=17992 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1528, 0, 520, 0, 0, 0, 0, 12, 4039, 71, 66, 0, 0, 0, 0, 2453, 9, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2288, 612, 0, 0, 2066, 0, 86, 1, 2215, 0, 0, 0, 2854, 3130, 0, 3295, 0, 0, 0, 0, 1888, 129, 0, 0, 1737, 1166, 57, 0, 467, 26, 1088, 0, 809, 132, 0, 0] rerouted_tokens_per_ep_rank=[2048, 4188, 2486, 2900, 4368, 9279, 4977, 2522] avg=4096.0000 max=9279 max_rank=5 min=2048 min_rank=0 imbalance_ratio=2.265381 max_load_reduction=23489 max_load_reduction_pct=0.716827 moved_to_echo=21146 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 746, 0, 3043, 0, 0, 0, 0, 4087, 92, 1, 0, 0, 0, 0, 0, 2091, 1693, 0, 4, 0, 0, 0, 0, 504, 0, 0, 0, 17, 878, 0, 0, 554, 16, 0, 76, 3183, 0, 0, 0, 2900, 0, 0, 1, 0, 0, 0, 0, 3136, 2, 0, 0, 13, 1179, 308, 0, 2160, 630, 4023, 0, 687, 744, 0, 0] rerouted_tokens_per_ep_rank=[3789, 4180, 3788, 1399, 3829, 2901, 4638, 8244] avg=4096.0000 max=8244 max_rank=7 min=1399 min_rank=3 imbalance_ratio=2.012695 max_load_reduction=24524 max_load_reduction_pct=0.748413 moved_to_echo=19612 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1389, 248, 2946, 0, 0, 0, 0, 3414, 3550, 13, 1378, 0, 0, 0, 0, 2812, 2902, 15, 12, 0, 0, 0, 0, 3257, 0, 39, 0, 328, 4, 0, 0, 0, 493, 1407, 0, 15, 0, 0, 0, 1539, 311, 0, 1590, 0, 0, 0, 0, 135, 0, 589, 0, 249, 625, 298, 0, 1158, 20, 388, 0, 924, 720, 0, 0] rerouted_tokens_per_ep_rank=[4583, 8355, 5741, 3628, 1915, 3440, 1896, 3210] avg=4096.0000 max=8355 max_rank=1 min=1896 min_rank=6 imbalance_ratio=2.039795 max_load_reduction=24413 max_load_reduction_pct=0.745026 moved_to_echo=10461 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 128, 7040, 128, 12416, 5120, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9984, 12032, 512, 10496, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8704, 2688, 12160, 128, 5504, 3968, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14464, 9728, 0, 8832, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2816, 7680, 6656, 16000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16768, 13696, 1664, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9728, 5504, 640, 384, 9472, 5888, 1664, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7424, 2816, 4352, 256, 18304, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2195, 0, 2813, 236, 4095, 0, 1, 7, 0, 203, 0, 4060, 4088, 84, 0, 30, 5, 4, 0, 0, 9, 2, 0, 3040, 0, 4094, 0, 3523, 0, 3942, 337, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[3275, 0, 245, 1615, 4096, 0, 43, 62, 835, 4096, 0, 0, 3743, 0, 2, 0, 695, 288, 81, 4022, 1, 0, 0, 0, 0, 2811, 0, 0, 0, 2273, 4096, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[247, 9, 252, 0, 4052, 0, 263, 0, 2884, 743, 4, 3396, 4092, 0, 0, 1037, 0, 1, 0, 3611, 0, 567, 0, 4093, 0, 4065, 0, 0, 1, 3451, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[186, 0, 2312, 0, 1914, 0, 0, 139, 0, 1, 0, 2885, 4095, 1, 5, 9, 3032, 0, 0, 317, 36, 3808, 0, 4095, 1142, 4041, 0, 1315, 10, 3425, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[2706, 0, 3884, 272, 3860, 0, 3899, 0, 0, 88, 0, 4001, 3846, 0, 0, 0, 0, 1928, 42, 0, 382, 1092, 0, 80, 13, 4007, 0, 2233, 0, 152, 280, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[72, 4, 3433, 0, 3486, 3600, 202, 2126, 121, 8, 2, 1038, 3980, 0, 0, 1135, 0, 140, 0, 3553, 814, 1493, 0, 4093, 0, 2717, 0, 0, 0, 1, 355, 395] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[3106, 1, 0, 0, 4085, 0, 81, 3780, 205, 4096, 2, 4052, 4095, 0, 0, 729, 0, 4, 0, 0, 0, 224, 0, 0, 0, 4083, 0, 503, 0, 3722, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3425, 170, 4088, 12, 1, 58, 0, 3281, 0, 4089, 564, 21, 0, 0, 925, 343, 0, 591, 5, 3, 0, 2856, 0, 3617, 677, 3908, 0, 4095, 7, 32] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3275, 0, 245, 1615, 0, 0, 0, 0, 2553, 0, 43, 62, 0, 0, 0, 0, 835, 4096, 0, 0, 0, 0, 0, 0, 3743, 0, 2, 0, 0, 0, 0, 0, 695, 288, 81, 4022, 1543, 191, 0, 0, 1, 0, 0, 0, 580, 0, 0, 0, 0, 1859, 0, 0, 0, 0, 0, 0, 0, 2273, 4096, 489, 0, 181, 0, 0] rerouted_tokens_per_ep_rank=[5135, 2658, 4931, 3745, 6820, 581, 1859, 7039] avg=4096.0000 max=7039 max_rank=7 min=581 min_rank=5 imbalance_ratio=1.718506 max_load_reduction=25729 max_load_reduction_pct=0.785187 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2195, 0, 2813, 236, 398, 0, 0, 0, 2552, 0, 1, 7, 0, 0, 0, 0, 0, 203, 0, 2798, 0, 0, 0, 0, 4088, 84, 0, 30, 209, 0, 0, 0, 5, 4, 0, 0, 1543, 278, 0, 0, 9, 2, 0, 3040, 844, 0, 0, 0, 0, 2708, 0, 3523, 0, 0, 0, 0, 0, 3942, 337, 0, 655, 264, 0, 0] rerouted_tokens_per_ep_rank=[5642, 2560, 3001, 4411, 1830, 3895, 6231, 5198] avg=4096.0000 max=6231 max_rank=6 min=1830 min_rank=4 imbalance_ratio=1.521240 max_load_reduction=26537 max_load_reduction_pct=0.809845 moved_to_echo=17154 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[186, 0, 2312, 0, 283, 0, 0, 0, 1193, 0, 0, 139, 0, 0, 0, 0, 0, 1, 0, 1988, 0, 0, 0, 0, 4095, 1, 5, 9, 148, 0, 0, 0, 3032, 0, 0, 317, 721, 275, 0, 0, 36, 3808, 0, 4095, 834, 0, 0, 0, 1142, 2671, 0, 1315, 0, 0, 0, 0, 10, 3425, 0, 0, 466, 261, 0, 0] rerouted_tokens_per_ep_rank=[2781, 1332, 1989, 4258, 4345, 8773, 5128, 4162] avg=4096.0000 max=8773 max_rank=5 min=1332 min_rank=1 imbalance_ratio=2.141846 max_load_reduction=23995 max_load_reduction_pct=0.732269 moved_to_echo=22408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2706, 0, 3884, 272, 392, 0, 0, 0, 2406, 0, 3899, 0, 0, 0, 0, 0, 0, 88, 0, 2757, 0, 0, 0, 0, 3846, 0, 0, 0, 206, 0, 0, 0, 0, 1928, 42, 0, 1454, 272, 0, 0, 382, 1092, 0, 80, 826, 0, 0, 0, 13, 2651, 0, 2233, 0, 0, 0, 0, 0, 152, 280, 3, 646, 258, 0, 0] rerouted_tokens_per_ep_rank=[7254, 6305, 2845, 4052, 3696, 2380, 4897, 1339] avg=4096.0000 max=7254 max_rank=0 min=1339 min_rank=7 imbalance_ratio=1.770996 max_load_reduction=25514 max_load_reduction_pct=0.778625 moved_to_echo=12312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3106, 1, 0, 0, 397, 0, 0, 0, 2546, 0, 81, 3780, 0, 0, 0, 0, 205, 4096, 2, 2793, 0, 0, 0, 0, 4095, 0, 0, 729, 208, 0, 0, 0, 0, 4, 0, 0, 1539, 277, 0, 0, 0, 224, 0, 0, 842, 0, 0, 0, 0, 2701, 0, 503, 0, 0, 0, 0, 0, 3722, 0, 0, 654, 263, 0, 0] rerouted_tokens_per_ep_rank=[3504, 6407, 7096, 5032, 1820, 1066, 3204, 4639] avg=4096.0000 max=7096 max_rank=2 min=1066 min_rank=5 imbalance_ratio=1.732422 max_load_reduction=25672 max_load_reduction_pct=0.783447 moved_to_echo=10729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[247, 9, 252, 0, 338, 0, 0, 0, 2523, 0, 263, 0, 0, 0, 0, 0, 2884, 743, 4, 2328, 0, 0, 0, 0, 4092, 0, 0, 1037, 178, 0, 0, 0, 0, 1, 0, 3611, 1529, 281, 0, 0, 0, 567, 0, 4093, 843, 0, 0, 0, 0, 2674, 0, 0, 0, 0, 0, 0, 1, 3451, 0, 0, 552, 267, 0, 0] rerouted_tokens_per_ep_rank=[846, 2786, 5959, 5307, 5422, 5503, 2674, 4271] avg=4096.0000 max=5959 max_rank=2 min=846 min_rank=0 imbalance_ratio=1.454834 max_load_reduction=26809 max_load_reduction_pct=0.818146 moved_to_echo=17870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[72, 4, 3433, 0, 101, 0, 0, 0, 2173, 3600, 202, 2126, 0, 0, 0, 0, 121, 8, 2, 717, 0, 0, 0, 0, 3980, 0, 0, 1135, 53, 0, 0, 0, 0, 140, 0, 3553, 1313, 184, 0, 0, 814, 1493, 0, 4093, 560, 0, 0, 0, 0, 1798, 0, 0, 0, 0, 0, 0, 0, 1, 355, 395, 167, 175, 0, 0] rerouted_tokens_per_ep_rank=[3610, 8101, 848, 5168, 5190, 6960, 1798, 1093] avg=4096.0000 max=8101 max_rank=1 min=848 min_rank=2 imbalance_ratio=1.977783 max_load_reduction=24667 max_load_reduction_pct=0.752777 moved_to_echo=15041 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3425, 170, 401, 0, 0, 0, 2548, 12, 1, 58, 0, 0, 0, 0, 0, 3281, 0, 2818, 0, 0, 0, 0, 564, 21, 0, 0, 210, 0, 0, 0, 925, 343, 0, 591, 1540, 246, 0, 0, 5, 3, 0, 2856, 746, 0, 0, 0, 0, 2392, 677, 3908, 0, 0, 0, 0, 0, 4095, 7, 32, 660, 233, 0, 0] rerouted_tokens_per_ep_rank=[3996, 2619, 6099, 795, 3645, 3610, 6977, 5027] avg=4096.0000 max=6977 max_rank=6 min=795 min_rank=3 imbalance_ratio=1.703369 max_load_reduction=25791 max_load_reduction_pct=0.787079 moved_to_echo=19259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 7296, 0, 18304, 6144, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4736, 2816, 128, 12160, 11264, 2048, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11904, 128, 16384, 2304, 2432, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[18560, 3712, 4608, 6272, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 12544, 128, 16256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 19456, 768, 11520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 21120, 5120, 1024, 3840, 1920, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[28544, 128, 128, 2944, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 527, 79, 404, 0, 0, 0, 900, 2961, 3924, 0, 2832, 2984, 203, 0, 176, 0, 404, 120, 10, 2603, 0, 2801, 0, 3298, 3367, 494, 470, 147, 0, 0, 4064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1094, 67, 0, 1, 3975, 3864, 2, 0, 1723, 2666, 0, 1, 0, 0, 18, 0, 548, 0, 1, 0, 4095, 11, 4066, 74, 3901, 27, 0, 378, 2240, 0, 0, 4016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4094, 2109, 0, 0, 3995, 3689, 0, 1926, 1046, 101, 0, 38, 970, 0, 0, 52, 0, 68, 1, 341, 0, 0, 3958, 978, 4088, 20, 3361, 0, 0, 0, 7, 1926] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[2, 4019, 736, 0, 351, 3982, 0, 57, 2, 0, 825, 124, 4091, 0, 0, 29, 0, 3961, 1, 3109, 0, 0, 3274, 0, 4096, 2, 0, 0, 13, 0, 6, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 141, 4, 0, 1986, 2288, 0, 0, 4096, 40, 0, 553, 4070, 0, 0, 0, 0, 2248, 0, 458, 145, 0, 3917, 0, 4092, 1, 223, 0, 318, 1, 4092, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3971, 1078, 748, 1, 4096, 0, 85, 3490, 0, 0, 64, 4048, 2298, 3564, 341, 4019, 0, 57, 0, 0, 1981, 51, 0, 6, 477, 1, 88, 0, 1207, 0, 0, 1097] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[151, 0, 3856, 0, 3685, 0, 0, 4093, 3080, 0, 0, 398, 122, 328, 0, 0, 0, 129, 97, 0, 3981, 5, 3963, 42, 4062, 11, 288, 0, 732, 3481, 261, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[2026, 3, 0, 0, 0, 3912, 0, 0, 269, 11, 0, 84, 3216, 0, 0, 0, 39, 3, 0, 1258, 4094, 0, 3883, 1197, 3870, 460, 160, 0, 4, 100, 4083, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 527, 79, 404, 655, 279, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 2961, 3924, 0, 2832, 459, 0, 0, 0, 2984, 203, 0, 176, 990, 69, 0, 0, 0, 404, 120, 10, 0, 683, 0, 0, 2603, 0, 1463, 0, 0, 0, 0, 0, 2770, 3367, 494, 470, 0, 0, 0, 0, 147, 0, 0, 2795, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1944, 900, 10176, 4422, 1217, 4066, 7101, 2942] avg=4096.0000 max=10176 max_rank=2 min=900 min_rank=1 imbalance_ratio=2.484375 max_load_reduction=22592 max_load_reduction_pct=0.689453 moved_to_echo=15326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1094, 67, 0, 1, 951, 275, 0, 0, 985, 3864, 2, 0, 0, 0, 0, 0, 1723, 2666, 0, 1, 543, 0, 0, 0, 0, 0, 18, 0, 979, 81, 0, 0, 548, 0, 1, 0, 2990, 991, 0, 0, 4095, 11, 2124, 74, 0, 0, 0, 0, 3277, 27, 0, 378, 0, 0, 0, 0, 2240, 0, 0, 2762, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2388, 4851, 4933, 1078, 4530, 6304, 3682, 5002] avg=4096.0000 max=6304 max_rank=5 min=1078 min_rank=3 imbalance_ratio=1.539062 max_load_reduction=26464 max_load_reduction_pct=0.807617 moved_to_echo=19518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 4019, 736, 0, 765, 280, 0, 0, 87, 3982, 0, 57, 0, 0, 0, 0, 2, 0, 825, 124, 570, 0, 0, 0, 4091, 0, 0, 29, 996, 85, 0, 0, 0, 3961, 1, 3109, 264, 798, 0, 0, 0, 0, 1711, 0, 0, 0, 0, 0, 3441, 2, 0, 0, 0, 0, 0, 0, 13, 0, 6, 2812, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5802, 4126, 1521, 5201, 8133, 1711, 3443, 2831] avg=4096.0000 max=8133 max_rank=4 min=1521 min_rank=2 imbalance_ratio=1.985596 max_load_reduction=24635 max_load_reduction_pct=0.751801 moved_to_echo=16118 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3971, 1078, 748, 1, 0, 75, 0, 0, 1015, 0, 85, 3490, 0, 0, 0, 0, 0, 0, 64, 4048, 66, 0, 0, 0, 2298, 3564, 341, 4019, 267, 10, 0, 0, 0, 57, 0, 0, 3081, 0, 0, 0, 1981, 51, 0, 6, 0, 0, 0, 0, 401, 1, 88, 0, 0, 0, 0, 0, 1207, 0, 0, 755, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5873, 4590, 4178, 10499, 3138, 2038, 490, 1962] avg=4096.0000 max=10499 max_rank=3 min=490 min_rank=6 imbalance_ratio=2.563232 max_load_reduction=22269 max_load_reduction_pct=0.679596 moved_to_echo=7628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 141, 4, 0, 916, 281, 0, 0, 492, 2288, 0, 0, 0, 0, 0, 0, 4096, 40, 0, 553, 569, 0, 0, 0, 4070, 0, 0, 0, 998, 85, 0, 0, 0, 2248, 0, 458, 1494, 955, 0, 0, 145, 0, 2046, 0, 0, 0, 0, 0, 3438, 1, 223, 0, 0, 0, 0, 0, 318, 1, 4092, 2816, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1342, 2780, 5258, 5153, 5155, 2191, 3662, 7227] avg=4096.0000 max=7227 max_rank=7 min=1342 min_rank=0 imbalance_ratio=1.764404 max_load_reduction=25541 max_load_reduction_pct=0.779449 moved_to_echo=18235 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4094, 2109, 0, 0, 928, 136, 0, 0, 988, 3689, 0, 1926, 0, 0, 0, 0, 1046, 101, 0, 38, 572, 0, 0, 0, 970, 0, 0, 52, 474, 89, 0, 0, 0, 68, 1, 341, 3007, 968, 0, 0, 0, 0, 2062, 978, 0, 0, 0, 0, 3427, 20, 3361, 0, 0, 0, 0, 0, 0, 0, 7, 1316, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7267, 6603, 1757, 1585, 4385, 3040, 6808, 1323] avg=4096.0000 max=7267 max_rank=0 min=1323 min_rank=7 imbalance_ratio=1.774170 max_load_reduction=25501 max_load_reduction_pct=0.778229 moved_to_echo=15556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[151, 0, 3856, 0, 927, 0, 0, 0, 913, 0, 0, 4093, 0, 0, 0, 0, 3080, 0, 0, 398, 565, 0, 0, 0, 122, 328, 0, 0, 0, 85, 0, 0, 0, 129, 97, 0, 2772, 966, 0, 0, 3981, 5, 2070, 42, 0, 0, 0, 0, 3412, 11, 288, 0, 0, 0, 0, 0, 732, 3481, 261, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4934, 5006, 4043, 535, 3964, 6098, 3711, 4477] avg=4096.0000 max=6098 max_rank=5 min=535 min_rank=3 imbalance_ratio=1.488770 max_load_reduction=26670 max_load_reduction_pct=0.813904 moved_to_echo=18250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2026, 3, 0, 0, 908, 281, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 269, 11, 0, 84, 538, 0, 0, 0, 3216, 0, 0, 0, 998, 81, 0, 0, 39, 3, 0, 1258, 0, 946, 0, 0, 4094, 0, 2029, 1197, 0, 0, 0, 0, 3251, 460, 160, 0, 0, 0, 0, 0, 4, 100, 4083, 2817, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3218, 3912, 902, 4295, 2246, 7320, 3871, 7004] avg=4096.0000 max=7320 max_rank=5 min=902 min_rank=2 imbalance_ratio=1.787109 max_load_reduction=25448 max_load_reduction_pct=0.776611 moved_to_echo=20441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17024, 128, 13568, 2304, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17792, 4096, 384, 4352, 5760, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4736, 3584, 8576, 16128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11392, 8064, 5504, 512, 6144, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 6912, 256, 5248, 13696, 6400, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13184, 6784, 896, 8192, 3968, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[23424, 3968, 4736, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[4480, 17792, 128, 10496, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[89, 0, 1, 0, 2, 0, 1, 0, 1870, 490, 4090, 0, 13, 0, 2415, 1543, 0, 335, 0, 3985, 3690, 4074, 2360, 87, 1, 16, 40, 60, 5, 1794, 1713, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[141, 0, 2217, 26, 0, 663, 172, 0, 15, 8, 4056, 0, 9, 0, 4096, 97, 3663, 865, 362, 3513, 3891, 657, 4069, 1, 1, 0, 0, 11, 0, 139, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[36, 3078, 0, 356, 0, 0, 0, 0, 1100, 4066, 0, 0, 0, 3910, 3824, 150, 91, 1278, 100, 2347, 4022, 3989, 1, 0, 0, 323, 1, 0, 19, 0, 4057, 20] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 1353, 0, 0, 0, 0, 0, 2, 1124, 3987, 2837, 0, 0, 173, 0, 43, 1816, 0, 2836, 869, 2838, 10, 4093, 0, 349, 0, 0, 2033, 0, 384, 3925, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 212, 0, 1694, 2964, 4069, 0, 0, 4013, 1887, 0, 151, 1, 43, 451, 3727, 1316, 4031, 0, 14, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 396, 0, 3484, 0, 0, 0, 69, 0, 0, 4096, 0, 2, 128, 0, 4047, 4096, 357, 18, 95, 4086, 0, 4040, 4081, 0, 78, 32, 0, 0, 6, 3592, 65] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[199, 0, 0, 0, 79, 0, 0, 2, 4091, 3691, 3087, 11, 0, 3689, 783, 0, 213, 12, 1, 1559, 4081, 0, 3221, 0, 0, 0, 0, 0, 0, 73, 4093, 3883] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[2247, 11, 0, 1163, 0, 1, 0, 0, 0, 11, 4079, 2, 1622, 1, 2975, 0, 2, 39, 4093, 0, 0, 3014, 3773, 4, 1543, 0, 0, 0, 0, 0, 4092, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[89, 0, 1, 0, 927, 752, 400, 131, 2, 0, 1, 0, 3690, 350, 0, 0, 1870, 490, 1189, 0, 0, 0, 0, 0, 13, 0, 2015, 1543, 0, 0, 0, 0, 0, 335, 0, 3854, 0, 0, 0, 0, 0, 4074, 1433, 87, 0, 0, 0, 0, 1, 16, 40, 60, 967, 2149, 0, 0, 5, 1794, 396, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2300, 4043, 3549, 3571, 4189, 5594, 3233, 6289] avg=4096.0000 max=6289 max_rank=7 min=2300 min_rank=0 imbalance_ratio=1.535400 max_load_reduction=26479 max_load_reduction_pct=0.808075 moved_to_echo=19305 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[36, 3078, 0, 356, 0, 0, 638, 77, 0, 0, 0, 0, 4022, 829, 0, 0, 1100, 4066, 0, 0, 0, 0, 0, 0, 0, 3910, 3186, 150, 0, 0, 0, 0, 91, 1278, 100, 2270, 0, 0, 0, 0, 0, 3989, 1, 0, 0, 0, 0, 0, 0, 323, 1, 0, 2291, 0, 0, 0, 19, 0, 937, 20, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 4851, 5166, 7246, 3739, 3990, 2615, 976] avg=4096.0000 max=7246 max_rank=3 min=976 min_rank=7 imbalance_ratio=1.769043 max_load_reduction=25522 max_load_reduction_pct=0.778870 moved_to_echo=11320 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1353, 0, 0, 1608, 522, 0, 28, 0, 0, 0, 2, 2838, 802, 0, 0, 1124, 3987, 825, 0, 0, 0, 0, 0, 0, 173, 0, 43, 0, 0, 0, 0, 1816, 0, 2836, 841, 0, 0, 0, 0, 0, 10, 2485, 0, 0, 0, 0, 0, 349, 0, 0, 2033, 2216, 1490, 0, 0, 0, 384, 907, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3511, 3642, 5936, 216, 5493, 2495, 6088, 5387] avg=4096.0000 max=6088 max_rank=6 min=216 min_rank=3 imbalance_ratio=1.486328 max_load_reduction=26680 max_load_reduction_pct=0.814209 moved_to_echo=19463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[141, 0, 2217, 26, 1598, 746, 679, 115, 0, 663, 172, 0, 3891, 0, 0, 0, 15, 8, 1179, 0, 0, 0, 0, 0, 9, 0, 3417, 97, 0, 0, 0, 0, 3663, 865, 362, 3398, 0, 0, 0, 0, 0, 657, 2471, 1, 0, 0, 0, 0, 1, 0, 0, 11, 0, 2131, 0, 0, 0, 139, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5522, 4726, 1202, 3523, 8288, 3129, 2143, 4235] avg=4096.0000 max=8288 max_rank=4 min=1202 min_rank=2 imbalance_ratio=2.023438 max_load_reduction=24480 max_load_reduction_pct=0.747070 moved_to_echo=17795 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1583, 748, 313, 14, 0, 3, 212, 0, 3727, 837, 0, 0, 1694, 2964, 1183, 0, 0, 0, 0, 0, 0, 4013, 1574, 0, 0, 0, 0, 0, 151, 1, 43, 437, 0, 0, 0, 0, 0, 1316, 2448, 0, 0, 0, 0, 0, 14, 0, 0, 0, 2313, 2138, 0, 0, 0, 0, 946, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2658, 4779, 5841, 5587, 632, 3764, 4465, 5042] avg=4096.0000 max=5841 max_rank=2 min=632 min_rank=4 imbalance_ratio=1.426025 max_load_reduction=26927 max_load_reduction_pct=0.821747 moved_to_echo=13903 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 396, 0, 3484, 1590, 757, 0, 7, 0, 0, 0, 69, 4086, 737, 0, 0, 0, 0, 1185, 0, 0, 0, 0, 0, 2, 128, 0, 4047, 0, 0, 0, 0, 4096, 357, 18, 88, 0, 0, 0, 0, 0, 0, 2450, 4081, 0, 0, 0, 0, 0, 78, 32, 0, 2031, 2154, 0, 0, 0, 6, 824, 65, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6234, 4892, 1185, 4177, 4559, 6531, 4295, 895] avg=4096.0000 max=6531 max_rank=5 min=895 min_rank=7 imbalance_ratio=1.594482 max_load_reduction=26237 max_load_reduction_pct=0.800690 moved_to_echo=16280 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[199, 0, 0, 0, 1265, 568, 129, 51, 79, 0, 0, 2, 4081, 836, 0, 0, 4091, 3691, 897, 11, 0, 0, 0, 0, 0, 3689, 654, 0, 0, 0, 0, 0, 213, 12, 1, 1508, 0, 0, 0, 0, 0, 0, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2311, 1622, 0, 0, 0, 73, 946, 3883, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2212, 4998, 8690, 4343, 1734, 1956, 3933, 4902] avg=4096.0000 max=8690 max_rank=2 min=1734 min_rank=4 imbalance_ratio=2.121582 max_load_reduction=24078 max_load_reduction_pct=0.734802 moved_to_echo=12525 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2247, 11, 0, 1163, 1482, 750, 493, 0, 0, 1, 0, 0, 0, 836, 0, 0, 0, 11, 1186, 2, 0, 0, 0, 0, 1622, 1, 2482, 0, 0, 0, 0, 0, 2, 39, 4093, 0, 0, 0, 0, 0, 0, 3014, 2291, 4, 0, 0, 0, 0, 1543, 0, 0, 0, 2310, 2143, 0, 0, 0, 0, 946, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 837, 1199, 4105, 4134, 5309, 5996, 5042] avg=4096.0000 max=6146 max_rank=0 min=837 min_rank=1 imbalance_ratio=1.500488 max_load_reduction=26622 max_load_reduction_pct=0.812439 moved_to_echo=20481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2816, 4864, 2304, 5120, 10112, 4864, 2688, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9984, 15232, 7680, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1920, 512, 128, 2176, 14464, 13952, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1664, 12032, 13440, 5888, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10112, 2944, 7552, 12416, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 13184, 15616, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 2432, 6016, 24448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 768, 512, 128, 26368, 5248, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4095, 1, 416, 0, 4078, 72, 1, 2920, 0, 1707, 156, 3754, 853, 1, 3391, 1474, 0, 22, 4096, 0, 0, 7, 2733, 12, 0, 16, 1, 0, 0, 29, 2933, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3581, 624, 12, 104, 455, 3110, 19, 1561, 0, 3279, 1579, 3643, 318, 0, 4095, 13, 125, 1510, 4096, 35, 0, 0, 13, 0, 1, 224, 442, 0, 3893, 0, 0, 36] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[52, 3026, 3816, 824, 3748, 0, 29, 4058, 0, 0, 4096, 3582, 0, 692, 19, 1, 0, 38, 4096, 1025, 0, 300, 1, 60, 87, 8, 0, 0, 2821, 381, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[441, 188, 671, 39, 163, 3453, 0, 20, 0, 3401, 1, 2990, 39, 249, 3988, 0, 3186, 0, 3863, 3901, 0, 666, 0, 0, 0, 0, 0, 1133, 4095, 0, 281, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[128, 10, 1182, 18, 4010, 4096, 0, 448, 0, 3779, 3556, 3853, 0, 0, 2441, 797, 1, 14, 4096, 30, 0, 2, 11, 0, 2, 92, 30, 6, 52, 0, 4096, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[12, 10, 484, 0, 497, 3135, 3814, 0, 0, 626, 3103, 3795, 0, 0, 2946, 0, 215, 3528, 4096, 2203, 330, 1, 163, 1901, 1658, 0, 76, 0, 149, 0, 26, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[46, 6, 0, 4, 1322, 3979, 0, 0, 6, 0, 0, 2555, 1181, 39, 4096, 61, 4, 4096, 3693, 0, 0, 1, 1391, 0, 151, 470, 3130, 2, 9, 3001, 2932, 593] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[109, 0, 3549, 0, 4090, 1409, 1, 392, 0, 0, 0, 511, 922, 0, 4063, 38, 1, 4083, 3211, 1, 0, 0, 4069, 0, 11, 10, 57, 0, 0, 0, 3674, 2567] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 1, 416, 0, 1417, 0, 0, 0, 4078, 6, 1, 2920, 0, 0, 0, 0, 0, 1707, 156, 1140, 0, 0, 0, 0, 853, 1, 3391, 1474, 159, 0, 0, 0, 0, 22, 1147, 0, 0, 0, 0, 0, 0, 7, 2733, 12, 57, 860, 0, 0, 0, 16, 1, 0, 2949, 9, 0, 0, 0, 29, 2933, 0, 178, 0, 0, 0] rerouted_tokens_per_ep_rank=[5929, 7005, 3003, 5878, 1169, 3669, 2975, 3140] avg=4096.0000 max=7005 max_rank=1 min=1169 min_rank=4 imbalance_ratio=1.710205 max_load_reduction=25763 max_load_reduction_pct=0.786224 moved_to_echo=10953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[52, 3026, 3816, 824, 1352, 0, 0, 0, 3748, 0, 29, 4058, 0, 0, 0, 0, 0, 0, 4096, 1087, 0, 0, 0, 0, 0, 692, 19, 1, 152, 0, 0, 0, 0, 38, 1147, 1025, 0, 0, 0, 0, 0, 300, 1, 60, 0, 821, 0, 0, 87, 8, 0, 0, 2949, 0, 0, 0, 2821, 381, 0, 8, 170, 0, 0, 0] rerouted_tokens_per_ep_rank=[9070, 7835, 5183, 864, 2210, 1182, 3044, 3380] avg=4096.0000 max=9070 max_rank=0 min=864 min_rank=3 imbalance_ratio=2.214355 max_load_reduction=23698 max_load_reduction_pct=0.723206 moved_to_echo=9816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3581, 624, 12, 104, 1375, 0, 0, 0, 455, 185, 19, 1561, 0, 0, 0, 0, 0, 3279, 1579, 1105, 0, 0, 0, 0, 318, 0, 4095, 13, 155, 0, 0, 0, 125, 1510, 1147, 35, 0, 0, 0, 0, 0, 0, 13, 0, 2495, 835, 0, 0, 1, 224, 442, 0, 2949, 430, 0, 0, 3893, 0, 0, 36, 173, 0, 0, 0] rerouted_tokens_per_ep_rank=[5696, 2220, 5963, 4581, 2817, 3343, 4046, 4102] avg=4096.0000 max=5963 max_rank=2 min=2220 min_rank=1 imbalance_ratio=1.455811 max_load_reduction=26805 max_load_reduction_pct=0.818024 moved_to_echo=14308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[12, 10, 484, 0, 1432, 0, 0, 0, 497, 187, 3814, 0, 0, 0, 0, 0, 0, 626, 3103, 1152, 0, 0, 0, 0, 0, 0, 2946, 0, 161, 0, 0, 0, 215, 3528, 1147, 2203, 0, 0, 0, 0, 330, 1, 163, 1901, 2515, 870, 0, 0, 1658, 0, 76, 0, 2949, 433, 0, 0, 149, 0, 26, 0, 180, 0, 0, 0] rerouted_tokens_per_ep_rank=[1938, 4498, 4881, 3107, 7093, 5780, 5116, 355] avg=4096.0000 max=7093 max_rank=4 min=355 min_rank=7 imbalance_ratio=1.731689 max_load_reduction=25675 max_load_reduction_pct=0.783539 moved_to_echo=18344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[46, 6, 0, 4, 964, 0, 0, 0, 1322, 237, 0, 0, 0, 0, 0, 0, 6, 0, 0, 777, 0, 0, 0, 0, 1181, 39, 4096, 61, 108, 0, 0, 0, 4, 4096, 1035, 0, 0, 0, 0, 0, 0, 1, 1391, 0, 3192, 585, 0, 0, 151, 470, 3130, 2, 2658, 550, 0, 0, 9, 3001, 2932, 593, 121, 0, 0, 0] rerouted_tokens_per_ep_rank=[1020, 1559, 783, 5485, 5135, 5169, 6961, 6656] avg=4096.0000 max=6961 max_rank=6 min=783 min_rank=2 imbalance_ratio=1.699463 max_load_reduction=25807 max_load_reduction_pct=0.787567 moved_to_echo=23921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[128, 10, 1182, 18, 1460, 0, 0, 0, 4010, 236, 0, 448, 0, 0, 0, 0, 0, 3779, 3556, 1153, 0, 0, 0, 0, 0, 0, 2441, 797, 168, 0, 0, 0, 1, 14, 1145, 30, 0, 0, 0, 0, 0, 2, 11, 0, 3289, 886, 0, 0, 2, 92, 30, 6, 2951, 571, 0, 0, 52, 0, 4096, 18, 186, 0, 0, 0] rerouted_tokens_per_ep_rank=[2798, 4694, 8488, 3406, 1190, 4188, 3652, 4352] avg=4096.0000 max=8488 max_rank=2 min=1190 min_rank=4 imbalance_ratio=2.072266 max_load_reduction=24280 max_load_reduction_pct=0.740967 moved_to_echo=13382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[441, 188, 671, 39, 1128, 0, 0, 0, 163, 206, 0, 20, 0, 0, 0, 0, 0, 3401, 1, 908, 0, 0, 0, 0, 39, 249, 3988, 0, 127, 0, 0, 0, 3186, 0, 1082, 3901, 0, 0, 0, 0, 0, 666, 0, 0, 2770, 685, 0, 0, 0, 0, 0, 1133, 2781, 477, 0, 0, 4095, 0, 281, 0, 142, 0, 0, 0] rerouted_tokens_per_ep_rank=[2467, 389, 4310, 4403, 8169, 4121, 4391, 4518] avg=4096.0000 max=8169 max_rank=4 min=389 min_rank=1 imbalance_ratio=1.994385 max_load_reduction=24599 max_load_reduction_pct=0.750702 moved_to_echo=21199 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[109, 0, 3549, 0, 192, 0, 0, 0, 4090, 85, 1, 392, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 922, 0, 4063, 38, 21, 0, 0, 0, 1, 4083, 900, 1, 0, 0, 0, 0, 0, 0, 4069, 0, 1130, 117, 0, 0, 11, 10, 57, 0, 2311, 194, 0, 0, 0, 0, 3674, 2567, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[3850, 4568, 157, 5044, 4985, 5316, 2583, 6265] avg=4096.0000 max=6265 max_rank=7 min=157 min_rank=2 imbalance_ratio=1.529541 max_load_reduction=26503 max_load_reduction_pct=0.808807 moved_to_echo=19149 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3584, 13312, 8832, 7296, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8576, 3968, 10240, 1024, 9344, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[18432, 1152, 3968, 9472, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 12800, 12544, 7552, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 1024, 8448, 2048, 15488, 5760, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1920, 896, 3840, 1152, 22528, 2688, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11136, 3456, 13952, 3328, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3328, 1024, 25088, 2432, 1152, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 2313, 3937, 6, 3, 0, 0, 145, 0, 4065, 1509, 272, 0, 23, 0, 786, 3914, 4031, 384, 4096, 0, 0, 2815, 0, 0, 8, 134, 3976, 286, 54, 11, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[13, 153, 3847, 0, 0, 0, 0, 0, 0, 3935, 3655, 486, 0, 0, 0, 0, 4077, 4096, 238, 4096, 3469, 2518, 0, 0, 0, 54, 0, 92, 1856, 9, 0, 174] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[14, 3, 0, 0, 4069, 0, 0, 3949, 2, 4096, 4039, 4051, 0, 0, 7, 2, 4084, 36, 0, 4093, 110, 72, 2, 0, 0, 0, 1, 41, 4096, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[37, 2882, 4080, 3, 1, 0, 0, 0, 1795, 4092, 0, 5, 150, 0, 3397, 0, 2624, 666, 443, 3927, 20, 8, 75, 10, 0, 3, 0, 4090, 368, 0, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[5, 3097, 4096, 1, 0, 0, 0, 0, 2, 4043, 3897, 0, 0, 0, 47, 0, 4055, 1679, 7, 4096, 0, 3888, 3, 0, 0, 0, 0, 3579, 270, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[7, 0, 16, 3795, 114, 56, 0, 148, 348, 1645, 0, 829, 1, 567, 2263, 1, 4038, 0, 0, 3750, 44, 0, 0, 16, 0, 0, 2154, 1516, 3010, 4096, 259, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[18, 13, 3941, 0, 2203, 0, 0, 105, 0, 4012, 3202, 2192, 0, 0, 53, 0, 4015, 4096, 50, 3938, 0, 402, 0, 0, 0, 0, 0, 381, 51, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1796, 1609, 2842, 24, 0, 0, 0, 265, 3, 4096, 2014, 77, 0, 0, 288, 0, 3925, 24, 6, 4096, 0, 0, 8, 0, 16, 0, 0, 3507, 4081, 0, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2313, 2938, 6, 0, 0, 0, 0, 3, 0, 0, 145, 2006, 881, 0, 0, 0, 596, 1509, 272, 0, 0, 0, 0, 0, 23, 0, 786, 3215, 0, 0, 0, 2168, 4031, 384, 0, 0, 0, 0, 0, 0, 0, 2815, 0, 1463, 1084, 0, 0, 0, 8, 134, 3976, 999, 662, 45, 0, 241, 54, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5257, 3035, 2377, 4024, 6583, 5362, 5824, 306] avg=4096.0000 max=6583 max_rank=4 min=306 min_rank=7 imbalance_ratio=1.607178 max_load_reduction=26185 max_load_reduction_pct=0.799103 moved_to_echo=18075 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[14, 3, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 3949, 2021, 882, 0, 0, 2, 601, 4039, 4051, 0, 0, 0, 0, 0, 0, 7, 2, 3211, 0, 0, 0, 2262, 36, 0, 0, 0, 0, 0, 0, 110, 72, 2, 0, 1474, 1131, 0, 0, 0, 0, 1, 41, 0, 691, 651, 0, 3445, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17, 10921, 8693, 3220, 2298, 2789, 1384, 3446] avg=4096.0000 max=10921 max_rank=1 min=17 min_rank=0 imbalance_ratio=2.666260 max_load_reduction=21847 max_load_reduction_pct=0.666718 moved_to_echo=9917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[37, 2882, 3044, 3, 0, 0, 0, 0, 1, 0, 0, 0, 2019, 845, 0, 0, 1795, 601, 0, 5, 0, 0, 0, 0, 150, 0, 3397, 0, 3082, 0, 0, 0, 1453, 666, 443, 0, 0, 0, 0, 0, 20, 8, 75, 10, 1472, 727, 0, 0, 0, 3, 0, 4090, 1036, 444, 58, 0, 310, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5966, 2865, 2401, 6629, 2562, 2312, 5631, 4402] avg=4096.0000 max=6629 max_rank=3 min=2312 min_rank=5 imbalance_ratio=1.618408 max_load_reduction=26139 max_load_reduction_pct=0.797699 moved_to_echo=14907 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 12, 3795, 0, 0, 0, 0, 114, 56, 0, 148, 812, 808, 0, 0, 348, 241, 0, 829, 0, 0, 0, 0, 1, 567, 2263, 1, 2942, 0, 0, 0, 2237, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 16, 592, 1118, 0, 0, 0, 0, 2154, 1516, 4, 683, 479, 0, 2531, 4096, 259, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3814, 1938, 1418, 5774, 2237, 1770, 4836, 10981] avg=4096.0000 max=10981 max_rank=7 min=1418 min_rank=2 imbalance_ratio=2.680908 max_load_reduction=21787 max_load_reduction_pct=0.664886 moved_to_echo=19824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[5, 3097, 3056, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1995, 882, 0, 0, 2, 593, 3897, 0, 0, 0, 0, 0, 0, 0, 47, 0, 3214, 0, 0, 0, 2246, 1679, 7, 0, 0, 0, 0, 0, 0, 3888, 3, 0, 1455, 1123, 0, 0, 0, 0, 0, 3579, 1040, 686, 42, 0, 228, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6159, 2877, 4492, 3261, 3932, 6469, 5347, 231] avg=4096.0000 max=6469 max_rank=5 min=231 min_rank=7 imbalance_ratio=1.579346 max_load_reduction=26299 max_load_reduction_pct=0.802582 moved_to_echo=15979 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[13, 153, 2870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1942, 882, 0, 0, 0, 577, 3655, 486, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 0, 0, 0, 2258, 4096, 238, 0, 0, 0, 0, 0, 3469, 2518, 0, 0, 1416, 1129, 0, 0, 0, 54, 0, 92, 977, 690, 295, 0, 1561, 9, 0, 174, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3036, 2824, 4718, 3214, 6592, 8532, 2108, 1744] avg=4096.0000 max=8532 max_rank=5 min=1744 min_rank=7 imbalance_ratio=2.083008 max_load_reduction=24236 max_load_reduction_pct=0.739624 moved_to_echo=18976 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[18, 13, 2938, 0, 0, 0, 0, 0, 2203, 0, 0, 105, 1985, 848, 0, 0, 0, 580, 3202, 2192, 0, 0, 0, 0, 0, 0, 53, 0, 3090, 0, 0, 0, 2214, 4096, 50, 0, 0, 0, 0, 0, 0, 402, 0, 0, 1447, 1116, 0, 0, 0, 0, 0, 381, 1003, 685, 12, 0, 39, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2969, 5141, 5974, 3143, 6360, 2965, 2081, 4135] avg=4096.0000 max=6360 max_rank=4 min=2081 min_rank=6 imbalance_ratio=1.552734 max_load_reduction=26408 max_load_reduction_pct=0.805908 moved_to_echo=15541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1796, 1609, 2121, 24, 0, 0, 0, 0, 0, 0, 0, 265, 2021, 881, 0, 0, 3, 601, 2014, 77, 0, 0, 0, 0, 0, 0, 288, 0, 3215, 0, 0, 0, 2174, 24, 6, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1474, 1087, 0, 0, 16, 0, 0, 3507, 721, 664, 649, 0, 3432, 0, 4091, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5550, 3167, 2695, 3503, 2204, 2569, 5557, 7523] avg=4096.0000 max=7523 max_rank=7 min=2204 min_rank=4 imbalance_ratio=1.836670 max_load_reduction=25245 max_load_reduction_pct=0.770416 moved_to_echo=17853 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6400, 128, 0, 4736, 14848, 6912, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11904, 8320, 4480, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[1920, 10112, 17024, 3840, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[17024, 14720, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 2304, 17280, 5888, 5248, 2304, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2176, 4480, 18432, 7936, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3712, 6912, 2944, 128, 10880, 8576, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 640, 6144, 896, 25216, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 369, 9, 2045, 0, 0, 51, 0, 0, 4086, 4, 0, 2, 6, 0, 4, 0, 0, 53, 4044, 245, 0, 51, 4093, 16, 1367, 0, 4029, 4096, 0, 9, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[367, 4022, 68, 0, 0, 248, 4096, 0, 0, 3448, 4084, 6, 1643, 0, 0, 8, 4096, 1, 1128, 791, 0, 1, 0, 0, 1, 0, 0, 3923, 4084, 700, 0, 53] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[307, 716, 276, 3752, 0, 5, 3415, 0, 698, 0, 554, 0, 147, 4080, 0, 0, 0, 0, 0, 4018, 4038, 0, 1124, 2268, 0, 3922, 0, 3347, 3, 20, 4, 74] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 499, 4, 0, 0, 2286, 0, 4039, 0, 4095, 800, 7, 0, 0, 3940, 0, 945, 648, 0, 0, 3967, 0, 0, 3198, 0, 9, 0, 3611, 647, 4069, 1, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[5, 914, 0, 1, 0, 0, 2026, 0, 0, 4096, 34, 427, 22, 1, 1479, 1, 2214, 946, 0, 1210, 3892, 6, 420, 4091, 0, 3568, 0, 0, 0, 0, 3978, 3437] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1121, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 125, 7, 37, 0, 2362, 84, 492, 4003, 3518, 39, 1787, 2577, 0, 0, 0, 0, 98, 3159, 1013, 59] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4, 4096, 0, 0, 8, 23, 3802, 0, 0, 840, 1359, 0, 16, 4056, 2183, 10, 0, 202, 0, 4063, 1856, 0, 0, 4018, 0, 1983, 0, 4080, 6, 150, 0, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[3515, 758, 0, 0, 138, 3, 3595, 0, 66, 167, 775, 2, 2, 0, 3995, 86, 3996, 45, 0, 65, 513, 0, 0, 3629, 0, 3993, 0, 3287, 0, 42, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 369, 9, 2045, 394, 130, 0, 0, 0, 0, 51, 0, 595, 282, 0, 0, 0, 4086, 4, 0, 455, 194, 0, 0, 2, 6, 0, 4, 1871, 0, 0, 0, 0, 0, 53, 3459, 0, 0, 0, 0, 245, 0, 51, 1940, 0, 0, 0, 0, 16, 1367, 0, 3240, 0, 0, 0, 0, 4096, 0, 9, 3699, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7043, 928, 4739, 1883, 3512, 2236, 4623, 7804] avg=4096.0000 max=7804 max_rank=7 min=928 min_rank=1 imbalance_ratio=1.905273 max_load_reduction=24964 max_load_reduction_pct=0.761841 moved_to_echo=18175 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 499, 4, 0, 0, 0, 0, 0, 0, 2286, 0, 4039, 533, 220, 0, 0, 0, 4095, 800, 7, 0, 174, 0, 0, 0, 0, 3940, 0, 1462, 0, 0, 0, 945, 648, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 1516, 0, 0, 0, 0, 0, 9, 0, 2904, 0, 0, 0, 0, 647, 4069, 1, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[503, 7078, 5076, 5402, 1593, 5483, 2913, 4720] avg=4096.0000 max=7078 max_rank=1 min=503 min_rank=0 imbalance_ratio=1.728027 max_load_reduction=25690 max_load_reduction_pct=0.783997 moved_to_echo=14709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[367, 4022, 68, 0, 5, 25, 0, 0, 0, 248, 4096, 0, 579, 0, 0, 0, 0, 3448, 4084, 6, 89, 189, 0, 0, 1643, 0, 0, 8, 0, 0, 0, 0, 4096, 1, 1128, 677, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3155, 0, 0, 0, 0, 4084, 700, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4487, 4923, 7816, 1651, 5902, 1, 3156, 4832] avg=4096.0000 max=7816 max_rank=2 min=1 min_rank=5 imbalance_ratio=1.908203 max_load_reduction=24952 max_load_reduction_pct=0.761475 moved_to_echo=13891 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5, 914, 0, 1, 330, 39, 0, 0, 0, 0, 2026, 0, 0, 281, 0, 0, 0, 4096, 34, 427, 136, 0, 0, 0, 22, 1, 1479, 1, 1870, 0, 0, 0, 2214, 946, 0, 1035, 0, 0, 0, 0, 3892, 6, 420, 1940, 0, 0, 0, 0, 0, 3568, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 3107, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1289, 2307, 4693, 3373, 4195, 6258, 3568, 7085] avg=4096.0000 max=7085 max_rank=7 min=1289 min_rank=0 imbalance_ratio=1.729736 max_load_reduction=25683 max_load_reduction_pct=0.783783 moved_to_echo=21106 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 4096, 0, 0, 1, 131, 0, 0, 8, 23, 3802, 0, 602, 276, 0, 0, 0, 840, 1359, 0, 457, 197, 0, 0, 16, 4056, 2183, 10, 1837, 0, 0, 0, 0, 202, 0, 3475, 0, 0, 0, 0, 1856, 0, 0, 1905, 0, 0, 0, 0, 0, 1983, 0, 3281, 0, 0, 0, 0, 6, 150, 0, 12, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4232, 4711, 2853, 8102, 3677, 3761, 5264, 168] avg=4096.0000 max=8102 max_rank=3 min=168 min_rank=7 imbalance_ratio=1.978027 max_load_reduction=24666 max_load_reduction_pct=0.752747 moved_to_echo=12870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[307, 716, 276, 3752, 10, 133, 0, 0, 0, 5, 3415, 0, 498, 159, 0, 0, 698, 0, 554, 0, 455, 164, 0, 0, 147, 4080, 0, 0, 1041, 0, 0, 0, 0, 0, 0, 3430, 0, 0, 0, 0, 4038, 0, 1124, 1068, 0, 0, 0, 0, 0, 3922, 0, 2685, 0, 0, 0, 0, 3, 20, 4, 64, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5194, 4077, 1871, 5268, 3430, 6230, 6607, 91] avg=4096.0000 max=6607 max_rank=6 min=91 min_rank=7 imbalance_ratio=1.613037 max_load_reduction=26161 max_load_reduction_pct=0.798370 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1121, 4096, 0, 0, 5, 129, 0, 0, 0, 0, 4096, 0, 0, 177, 0, 0, 0, 0, 4095, 0, 450, 0, 0, 0, 125, 7, 37, 0, 1178, 0, 0, 0, 2362, 84, 492, 3424, 0, 0, 0, 0, 3518, 39, 1787, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 3159, 1013, 54, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5351, 4273, 4545, 1347, 6362, 6566, 0, 4324] avg=4096.0000 max=6566 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.603027 max_load_reduction=26202 max_load_reduction_pct=0.799622 moved_to_echo=17252 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3515, 758, 0, 0, 394, 2, 0, 0, 138, 3, 3595, 0, 485, 250, 0, 0, 66, 167, 775, 2, 7, 158, 0, 0, 2, 0, 3995, 86, 1659, 0, 0, 0, 3996, 45, 0, 56, 0, 0, 0, 0, 513, 0, 0, 1720, 0, 0, 0, 0, 0, 3993, 0, 2644, 0, 0, 0, 0, 0, 42, 0, 3702, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4669, 4471, 1175, 5742, 4097, 2233, 6637, 3744] avg=4096.0000 max=6637 max_rank=6 min=1175 min_rank=2 imbalance_ratio=1.620361 max_load_reduction=26131 max_load_reduction_pct=0.797455 moved_to_echo=16711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[18048, 128, 3456, 11392, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8960, 8192, 5120, 10752, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2048, 8192, 11648, 128, 11008, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 14848, 0, 17920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 16768, 11776, 512, 2176, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9472, 15488, 384, 5888, 1152, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 2688, 21120, 4096, 3328, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13696, 2048, 1792, 15616, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1639, 63, 0, 58, 4066, 1, 2134, 3557, 3335, 5, 0, 3794, 35, 47, 0, 4046, 2665, 7, 0, 2646, 0, 0, 0, 0, 5, 397, 0, 218, 1, 0, 0, 4049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4075, 22, 3851, 3303, 0, 103, 1961, 318, 0, 512, 0, 4053, 0, 2344, 3826, 0, 0, 4095, 654, 0, 3605, 30, 0, 0, 0, 6, 0, 0, 10, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 3901, 2, 1609, 4053, 60, 0, 0, 3544, 3222, 3998, 48, 17, 1, 0, 0, 2011, 0, 0, 0, 0, 544, 985, 10, 0, 0, 67, 556, 4070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1758, 0, 4091, 0, 4085, 2049, 1786, 3980, 0, 154, 0, 3368, 233, 2759, 0, 340, 0, 0, 0, 3981, 3995, 0, 0, 0, 65, 4, 0, 0, 8, 1, 14, 97] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 479, 850, 4096, 3807, 3915, 3678, 3255, 3947, 0, 0, 6, 4096, 3, 340, 18, 2, 0, 4045, 78, 4, 0, 0, 0, 0, 0, 130, 0, 0, 19, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 3, 867, 4096, 3793, 0, 4096, 264, 80, 0, 3, 0, 4088, 0, 4093, 0, 0, 0, 4059, 4074, 54, 0, 0, 0, 0, 0, 245, 8, 0, 2945, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 354, 933, 2901, 0, 4069, 259, 1503, 168, 3, 0, 994, 367, 0, 2845, 105, 454, 0, 0, 4079, 0, 0, 6, 472, 0, 3938, 25, 4096, 917, 0, 4070, 210] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[2759, 0, 1849, 0, 3419, 3775, 4096, 1, 4088, 11, 302, 18, 0, 0, 0, 22, 0, 0, 0, 4091, 3726, 0, 17, 0, 56, 2, 0, 0, 47, 2, 391, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1639, 63, 0, 58, 8, 224, 0, 0, 0, 1, 2134, 0, 0, 0, 0, 0, 3335, 5, 0, 3794, 0, 68, 0, 0, 35, 38, 0, 4046, 0, 0, 0, 0, 2665, 7, 0, 2354, 0, 0, 0, 0, 0, 0, 0, 0, 1794, 808, 0, 0, 5, 397, 0, 218, 3258, 0, 0, 0, 1, 0, 0, 4049, 1763, 1, 0, 0] rerouted_tokens_per_ep_rank=[1992, 2135, 7202, 4119, 5026, 2602, 3878, 5814] avg=4096.0000 max=7202 max_rank=2 min=1992 min_rank=0 imbalance_ratio=1.758301 max_load_reduction=25566 max_load_reduction_pct=0.780212 moved_to_echo=17320 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4075, 22, 753, 347, 0, 0, 0, 3013, 0, 0, 0, 0, 0, 0, 1961, 318, 0, 512, 290, 106, 0, 0, 0, 3130, 0, 2344, 0, 0, 0, 0, 3826, 0, 0, 3642, 0, 0, 0, 0, 654, 0, 3605, 30, 52, 766, 0, 0, 0, 0, 0, 6, 3085, 0, 0, 0, 0, 0, 10, 0, 51, 170, 0, 0] rerouted_tokens_per_ep_rank=[5197, 3013, 3187, 5474, 7468, 5107, 3091, 231] avg=4096.0000 max=7468 max_rank=4 min=231 min_rank=7 imbalance_ratio=1.823242 max_load_reduction=25300 max_load_reduction_pct=0.772095 moved_to_echo=15897 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 743, 170, 0, 0, 0, 2, 1609, 0, 0, 0, 0, 0, 60, 0, 0, 3544, 0, 52, 0, 0, 3222, 3088, 48, 17, 0, 0, 0, 0, 1, 0, 0, 1789, 0, 0, 0, 0, 0, 0, 0, 0, 2044, 776, 0, 0, 544, 985, 10, 0, 3125, 0, 0, 0, 0, 67, 556, 4070, 2009, 167, 0, 0] rerouted_tokens_per_ep_rank=[4983, 1611, 3656, 6375, 1790, 2820, 4664, 6869] avg=4096.0000 max=6869 max_rank=7 min=1611 min_rank=1 imbalance_ratio=1.677002 max_load_reduction=25899 max_load_reduction_pct=0.790375 moved_to_echo=16143 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1758, 0, 4091, 0, 512, 337, 0, 0, 0, 1869, 1786, 0, 0, 0, 0, 0, 0, 154, 0, 3368, 180, 103, 0, 0, 233, 2132, 0, 340, 0, 0, 0, 0, 0, 0, 0, 3541, 0, 0, 0, 0, 3995, 0, 0, 0, 2009, 813, 0, 0, 65, 4, 0, 0, 3272, 0, 0, 0, 8, 1, 14, 97, 1971, 115, 0, 0] rerouted_tokens_per_ep_rank=[6698, 3655, 3805, 2705, 3541, 6817, 3341, 2206] avg=4096.0000 max=6817 max_rank=5 min=2206 min_rank=7 imbalance_ratio=1.664307 max_load_reduction=25951 max_load_reduction_pct=0.791962 moved_to_echo=15905 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 354, 933, 2901, 0, 345, 0, 0, 0, 3711, 259, 0, 0, 0, 0, 0, 168, 3, 0, 994, 358, 106, 0, 0, 367, 0, 2845, 105, 0, 0, 0, 0, 454, 0, 0, 3628, 0, 0, 0, 0, 0, 0, 6, 472, 759, 0, 0, 0, 0, 3938, 25, 4096, 0, 0, 0, 0, 917, 0, 4070, 210, 744, 0, 0, 0] rerouted_tokens_per_ep_rank=[4533, 3970, 1629, 3317, 4082, 1237, 8059, 5941] avg=4096.0000 max=8059 max_rank=6 min=1237 min_rank=5 imbalance_ratio=1.967529 max_load_reduction=24709 max_load_reduction_pct=0.754059 moved_to_echo=19319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 867, 762, 347, 0, 0, 0, 3457, 0, 0, 0, 0, 0, 0, 264, 80, 0, 3, 336, 108, 0, 0, 0, 3151, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3604, 0, 0, 0, 0, 4074, 54, 0, 0, 2067, 815, 0, 0, 0, 0, 0, 245, 3281, 0, 0, 0, 8, 0, 2945, 0, 2029, 175, 0, 0] rerouted_tokens_per_ep_rank=[1979, 3457, 791, 7244, 3604, 7010, 3526, 5157] avg=4096.0000 max=7244 max_rank=3 min=791 min_rank=2 imbalance_ratio=1.768555 max_load_reduction=25524 max_load_reduction_pct=0.778931 moved_to_echo=19297 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 479, 850, 761, 342, 0, 0, 0, 3473, 3915, 0, 0, 0, 0, 0, 3255, 3947, 0, 0, 334, 105, 0, 0, 6, 3163, 3, 340, 0, 0, 0, 0, 18, 2, 0, 3598, 0, 0, 0, 0, 78, 4, 0, 0, 1856, 815, 0, 0, 0, 0, 0, 130, 3281, 0, 0, 0, 0, 0, 19, 0, 1822, 172, 0, 0] rerouted_tokens_per_ep_rank=[2432, 7388, 7641, 3512, 3618, 2753, 3411, 2013] avg=4096.0000 max=7641 max_rank=2 min=2013 min_rank=7 imbalance_ratio=1.865479 max_load_reduction=25127 max_load_reduction_pct=0.766815 moved_to_echo=11795 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2759, 0, 1849, 0, 0, 346, 0, 0, 0, 3443, 4096, 0, 0, 0, 0, 0, 4088, 11, 302, 18, 332, 106, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 3639, 0, 0, 0, 0, 3726, 0, 17, 0, 0, 679, 0, 0, 56, 2, 0, 0, 2740, 0, 0, 0, 47, 2, 391, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4954, 7539, 4857, 22, 3639, 4422, 2798, 4537] avg=4096.0000 max=7539 max_rank=1 min=22 min_rank=3 imbalance_ratio=1.840576 max_load_reduction=25229 max_load_reduction_pct=0.769928 moved_to_echo=15396 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12544, 128, 3712, 512, 10624, 5504, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10240, 512, 11520, 4736, 3584, 2560, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13184, 4608, 384, 12288, 1920, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 128, 8064, 12544, 10496, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3968, 14720, 2944, 11392, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7040, 128, 0, 25856, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 5376, 128, 4736, 22144, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 19072, 13824, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 3563, 4038, 1, 306, 96, 0, 0, 0, 0, 25, 0, 0, 161, 195, 302, 0, 506, 4094, 0, 0, 4083, 35, 0, 9, 3860, 4096, 0, 3468, 0, 3929, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 1077, 0, 47, 0, 0, 409, 1337, 292, 4059, 0, 3597, 69, 0, 14, 3716, 25, 0, 15, 0, 3467, 2360, 0, 0, 190, 0, 4079, 0, 3974, 0, 3919, 122] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 1170, 45, 1, 23, 0, 0, 0, 0, 0, 45, 532, 0, 2, 0, 0, 3809, 8, 1, 4090, 4078, 4060, 3651, 0, 0, 2710, 147, 4062, 0, 224, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 3582, 1, 200, 4, 0, 0, 198, 0, 4092, 4, 3247, 2, 196, 0, 3796, 0, 18, 183, 4096, 872, 2, 0, 4023, 0, 30, 3946, 0, 120, 60] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4026, 497, 91, 40, 94, 3437, 0, 0, 0, 0, 1, 0, 1, 0, 966, 0, 358, 123, 0, 2, 3846, 0, 0, 0, 4095, 4096, 2959, 4096, 0, 4040, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 8, 163, 62, 1927, 1665, 0, 4043, 76, 38, 828, 0, 66, 1, 159, 0, 544, 4087, 0, 0, 3995, 4096, 57, 0, 151, 0, 3888, 0, 3222, 0, 0, 3692] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 2210, 957, 102, 2, 0, 0, 0, 0, 0, 3615, 81, 0, 5, 0, 630, 3268, 0, 3291, 3869, 3495, 40, 559, 3486, 0, 0, 0, 2528, 0, 429, 105] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 3829, 3321, 65, 6, 0, 776, 1166, 25, 0, 133, 1, 0, 1, 2839, 0, 1, 4095, 147, 0, 76, 3929, 0, 0, 19, 361, 4096, 0, 3796, 0, 4086, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 719, 0, 47, 0, 0, 0, 0, 0, 0, 409, 1337, 2072, 154, 0, 0, 292, 4059, 0, 3597, 1692, 187, 258, 0, 69, 0, 14, 3716, 1590, 0, 0, 0, 25, 0, 15, 0, 171, 0, 0, 0, 3467, 616, 0, 0, 0, 0, 0, 0, 190, 0, 2387, 0, 0, 0, 0, 0, 1644, 0, 3919, 122, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[766, 3972, 10085, 5389, 211, 4083, 2577, 5685] avg=4096.0000 max=10085 max_rank=2 min=211 min_rank=4 imbalance_ratio=2.462158 max_load_reduction=22683 max_load_reduction_pct=0.692230 moved_to_echo=12556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2376, 4038, 1, 0, 0, 0, 0, 306, 96, 0, 0, 1808, 267, 0, 0, 0, 0, 25, 0, 1699, 620, 226, 0, 0, 161, 195, 302, 2752, 0, 0, 0, 0, 506, 4094, 0, 567, 0, 0, 0, 0, 1064, 35, 0, 0, 0, 0, 0, 9, 3860, 2397, 0, 0, 0, 0, 0, 1434, 0, 3929, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6415, 2477, 2570, 3410, 5167, 1099, 6266, 5364] avg=4096.0000 max=6415 max_rank=0 min=1099 min_rank=5 imbalance_ratio=1.566162 max_load_reduction=26353 max_load_reduction_pct=0.804230 moved_to_echo=17896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 163, 62, 0, 0, 0, 0, 1927, 1665, 0, 4043, 1679, 268, 0, 0, 76, 38, 828, 0, 1614, 1, 209, 0, 66, 1, 159, 0, 2760, 0, 0, 0, 544, 4087, 0, 0, 1, 0, 0, 0, 3995, 1068, 57, 0, 0, 0, 0, 0, 151, 0, 2274, 0, 0, 0, 0, 0, 1334, 0, 0, 3692, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[231, 9582, 2766, 2986, 4632, 5120, 2425, 5026] avg=4096.0000 max=9582 max_rank=1 min=231 min_rank=0 imbalance_ratio=2.339355 max_load_reduction=23186 max_load_reduction_pct=0.707581 moved_to_echo=17203 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2684, 497, 91, 0, 0, 0, 0, 40, 94, 3437, 0, 2135, 251, 0, 0, 0, 0, 0, 1, 1699, 701, 266, 0, 0, 1, 0, 966, 2592, 0, 0, 0, 0, 358, 123, 0, 641, 0, 0, 0, 2, 1003, 0, 0, 0, 0, 0, 0, 0, 4095, 2397, 2959, 0, 0, 0, 0, 1695, 0, 4040, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3272, 5957, 2667, 3559, 1122, 1005, 9451, 5735] avg=4096.0000 max=9451 max_rank=6 min=1005 min_rank=5 imbalance_ratio=2.307373 max_load_reduction=23317 max_load_reduction_pct=0.711578 moved_to_echo=17313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2731, 2210, 957, 0, 0, 0, 0, 102, 2, 0, 0, 1318, 228, 0, 0, 0, 0, 0, 3615, 0, 713, 164, 0, 81, 0, 5, 0, 2355, 0, 0, 0, 630, 3268, 0, 3291, 652, 0, 0, 0, 3869, 912, 40, 559, 0, 0, 0, 0, 3486, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 429, 105, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5898, 1650, 4492, 2441, 7841, 5380, 3486, 1580] avg=4096.0000 max=7841 max_rank=4 min=1580 min_rank=7 imbalance_ratio=1.914307 max_load_reduction=24927 max_load_reduction_pct=0.760712 moved_to_echo=18287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2720, 0, 3582, 0, 0, 0, 0, 1, 200, 4, 0, 2061, 272, 0, 0, 0, 198, 0, 4092, 0, 718, 262, 0, 4, 3247, 2, 196, 2764, 0, 0, 0, 0, 3796, 0, 18, 658, 0, 0, 0, 183, 1060, 872, 2, 0, 0, 0, 0, 0, 4023, 0, 30, 0, 0, 0, 0, 1623, 0, 120, 60, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6302, 2538, 5270, 6213, 4472, 2117, 4053, 1803] avg=4096.0000 max=6302 max_rank=0 min=1803 min_rank=7 imbalance_ratio=1.538574 max_load_reduction=26466 max_load_reduction_pct=0.807678 moved_to_echo=12445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2731, 1170, 45, 0, 0, 0, 0, 1, 23, 0, 0, 2117, 267, 0, 0, 0, 0, 0, 45, 1124, 713, 264, 0, 532, 0, 2, 0, 2748, 0, 0, 0, 0, 3809, 8, 1, 652, 0, 0, 0, 4090, 1063, 4060, 3651, 0, 0, 0, 0, 0, 0, 1586, 147, 0, 0, 0, 0, 1681, 0, 224, 14, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3946, 2408, 2146, 3282, 4470, 12864, 1733, 1919] avg=4096.0000 max=12864 max_rank=5 min=1733 min_rank=6 imbalance_ratio=3.140625 max_load_reduction=19904 max_load_reduction_pct=0.607422 moved_to_echo=20986 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2552, 3321, 65, 0, 0, 0, 0, 6, 0, 776, 1166, 1979, 257, 0, 0, 25, 0, 133, 1, 1699, 667, 247, 0, 0, 1, 2839, 0, 2648, 0, 0, 0, 1, 4095, 147, 0, 610, 0, 0, 0, 76, 1024, 0, 0, 0, 0, 0, 0, 19, 361, 2397, 0, 0, 0, 0, 0, 1570, 0, 4086, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5938, 4184, 2772, 5488, 4853, 1100, 2777, 5656] avg=4096.0000 max=5938 max_rank=0 min=1100 min_rank=5 imbalance_ratio=1.449707 max_load_reduction=26830 max_load_reduction_pct=0.818787 moved_to_echo=14386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2432, 2176, 4736, 6656, 15232, 2048, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 19968, 4480, 3328, 3968, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15744, 7936, 5120, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 4352, 1024, 11392, 9600, 4352, 1920, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 16640, 11520, 4864, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[12032, 0, 16768, 4096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3968, 12416, 13440, 3200, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 3456, 3328, 5248, 20224, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[104, 2900, 558, 476, 1581, 417, 782, 1275, 1115, 179, 1756, 369, 837, 791, 221, 656, 325, 479, 994, 1279, 925, 185, 3147, 2371, 284, 1134, 874, 1270, 1331, 527, 1620, 2006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[107, 3092, 508, 522, 1118, 355, 1034, 792, 1023, 122, 2096, 140, 873, 815, 333, 459, 212, 519, 1232, 528, 1046, 218, 3060, 2486, 693, 1577, 822, 1320, 1408, 469, 1845, 1944] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[119, 2861, 421, 345, 1670, 609, 679, 1392, 1095, 233, 1318, 239, 1117, 1278, 205, 383, 96, 687, 820, 797, 1484, 128, 3460, 2221, 458, 1181, 759, 1637, 896, 624, 1557, 1999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[220, 2995, 564, 418, 1036, 260, 910, 990, 1305, 189, 1845, 525, 672, 947, 622, 425, 71, 642, 1032, 640, 954, 360, 2930, 2116, 664, 1042, 1004, 1303, 1434, 655, 1967, 2031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[170, 2844, 1079, 248, 1651, 371, 960, 1336, 1245, 206, 1160, 313, 1047, 567, 198, 394, 158, 627, 902, 1151, 1170, 69, 3223, 1953, 666, 838, 1187, 1270, 1858, 290, 1639, 1978] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[76, 2849, 439, 329, 1164, 502, 636, 1371, 1218, 259, 1302, 217, 1351, 1344, 230, 429, 147, 487, 1745, 675, 1132, 209, 2537, 2033, 380, 1149, 905, 1319, 1153, 418, 2245, 2518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[102, 2935, 806, 427, 1114, 338, 800, 1036, 1760, 157, 1336, 494, 1084, 662, 189, 432, 99, 345, 1436, 1143, 1045, 172, 2731, 2254, 588, 791, 779, 935, 2064, 482, 2300, 1932] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[153, 3374, 406, 372, 1462, 399, 756, 1441, 1192, 206, 1578, 229, 544, 895, 502, 494, 108, 761, 1242, 801, 1054, 228, 2333, 2381, 559, 1174, 694, 1631, 1400, 532, 1845, 2022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[107, 3086, 508, 522, 0, 0, 0, 0, 1118, 355, 1034, 792, 299, 0, 0, 0, 1023, 122, 2096, 140, 750, 0, 0, 0, 873, 815, 333, 459, 1538, 0, 0, 0, 212, 519, 1232, 528, 924, 415, 0, 0, 1046, 218, 598, 2486, 0, 0, 0, 0, 693, 1577, 822, 1320, 216, 6, 0, 0, 1408, 469, 1845, 264, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4223, 3598, 4131, 4018, 3830, 4348, 4634, 3986] avg=4096.0000 max=4634 max_rank=6 min=3598 min_rank=1 imbalance_ratio=1.131348 max_load_reduction=28134 max_load_reduction_pct=0.858582 moved_to_echo=16798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[119, 2855, 421, 345, 0, 0, 0, 0, 1670, 609, 679, 1392, 307, 0, 0, 0, 1095, 233, 1318, 239, 772, 0, 0, 0, 1117, 1278, 205, 383, 1739, 0, 0, 0, 96, 687, 820, 797, 1045, 427, 0, 0, 1484, 128, 676, 2221, 0, 0, 0, 0, 458, 1181, 759, 1637, 222, 6, 0, 0, 896, 624, 1557, 271, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3740, 4657, 3657, 4722, 3872, 4509, 4263, 3348] avg=4096.0000 max=4722 max_rank=3 min=3348 min_rank=7 imbalance_ratio=1.152832 max_load_reduction=28046 max_load_reduction_pct=0.855896 moved_to_echo=15992 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[104, 2894, 558, 476, 0, 0, 0, 0, 1581, 417, 782, 1275, 309, 0, 0, 0, 1115, 179, 1756, 369, 774, 0, 0, 0, 837, 791, 221, 656, 1581, 0, 0, 0, 325, 479, 994, 1279, 950, 428, 0, 0, 925, 185, 616, 2371, 0, 0, 0, 0, 284, 1134, 874, 1270, 223, 6, 0, 0, 1331, 527, 1620, 272, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4364, 4193, 4086, 4455, 4097, 3791, 3750] avg=4096.0000 max=4455 max_rank=4 min=3750 min_rank=7 imbalance_ratio=1.087646 max_load_reduction=28313 max_load_reduction_pct=0.864044 moved_to_echo=16093 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[220, 2989, 564, 418, 0, 0, 0, 0, 1036, 260, 910, 990, 312, 0, 0, 0, 1305, 189, 1845, 525, 784, 0, 0, 0, 672, 947, 622, 425, 1472, 0, 0, 0, 71, 642, 1032, 640, 885, 434, 0, 0, 954, 360, 573, 2116, 0, 0, 0, 0, 664, 1042, 1004, 1303, 226, 6, 0, 0, 1434, 655, 1967, 275, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 3508, 4648, 4138, 3704, 4003, 4245, 4331] avg=4096.0000 max=4648 max_rank=2 min=3508 min_rank=1 imbalance_ratio=1.134766 max_load_reduction=28120 max_load_reduction_pct=0.858154 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[170, 2836, 1079, 248, 0, 0, 0, 0, 1651, 371, 960, 1336, 309, 0, 0, 0, 1245, 206, 1160, 313, 768, 0, 0, 0, 1047, 567, 198, 394, 1623, 0, 0, 0, 158, 627, 902, 1151, 977, 426, 0, 0, 1170, 69, 623, 1953, 0, 0, 0, 0, 666, 838, 1187, 1270, 223, 8, 0, 0, 1858, 290, 1639, 252, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4333, 4627, 3692, 3829, 4241, 3815, 4192, 4039] avg=4096.0000 max=4627 max_rank=1 min=3692 min_rank=2 imbalance_ratio=1.129639 max_load_reduction=28141 max_load_reduction_pct=0.858795 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[102, 2929, 806, 427, 0, 0, 0, 0, 1114, 338, 800, 1036, 297, 0, 0, 0, 1760, 157, 1336, 494, 746, 0, 0, 0, 1084, 662, 189, 432, 1372, 0, 0, 0, 99, 345, 1436, 1143, 824, 413, 0, 0, 1045, 172, 535, 2254, 0, 0, 0, 0, 588, 791, 779, 935, 215, 6, 0, 0, 2064, 482, 2300, 261, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4264, 3585, 4493, 3739, 4260, 4006, 3314, 5107] avg=4096.0000 max=5107 max_rank=7 min=3314 min_rank=6 imbalance_ratio=1.246826 max_load_reduction=27661 max_load_reduction_pct=0.844147 moved_to_echo=16687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[76, 2843, 439, 329, 0, 0, 0, 0, 1164, 502, 636, 1371, 387, 0, 0, 0, 1218, 259, 1302, 217, 972, 0, 0, 0, 1351, 1344, 230, 429, 1275, 0, 0, 0, 147, 487, 1745, 675, 766, 538, 0, 0, 1132, 209, 496, 2033, 0, 0, 0, 0, 380, 1149, 905, 1319, 280, 6, 0, 0, 1153, 418, 2245, 341, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3687, 4060, 3968, 4629, 4358, 3870, 4039, 4157] avg=4096.0000 max=4629 max_rank=3 min=3687 min_rank=0 imbalance_ratio=1.130127 max_load_reduction=28139 max_load_reduction_pct=0.858734 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[153, 3367, 406, 372, 0, 0, 0, 0, 1462, 399, 756, 1441, 311, 0, 0, 0, 1192, 206, 1578, 229, 781, 0, 0, 0, 544, 895, 502, 494, 1172, 0, 0, 0, 108, 761, 1242, 801, 704, 432, 0, 0, 1054, 228, 457, 2381, 0, 0, 0, 0, 559, 1174, 694, 1631, 225, 7, 0, 0, 1400, 532, 1845, 273, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4298, 4369, 3986, 3607, 4048, 4120, 4290, 4050] avg=4096.0000 max=4369 max_rank=1 min=3607 min_rank=3 imbalance_ratio=1.066650 max_load_reduction=28399 max_load_reduction_pct=0.866669 moved_to_echo=16508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7552, 7424, 2560, 3712, 11776, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10880, 3328, 6656, 9728, 2560, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1280, 4608, 9472, 7040, 7168, 3584, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11648, 4096, 15104, 2304, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8832, 1664, 4608, 17920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9984, 1664, 12416, 2560, 6400, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 23808, 4864, 3200, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4352, 8960, 7040, 10752, 1920, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[409, 2, 2233, 143, 2328, 3949, 1445, 3227, 210, 2, 500, 1315, 13, 3810, 0, 511, 40, 83, 0, 4036, 19, 11, 0, 84, 135, 129, 178, 3902, 4053, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[899, 0, 2328, 61, 3383, 3234, 129, 2922, 294, 0, 1379, 985, 46, 2615, 6, 1677, 23, 147, 4, 3993, 4, 13, 0, 16, 36, 9, 360, 3971, 3959, 0, 130, 145] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[223, 3, 525, 2, 2270, 3648, 3156, 387, 188, 1, 1029, 1851, 24, 3919, 0, 441, 60, 115, 0, 4065, 3, 14, 642, 143, 446, 169, 1967, 3518, 3854, 1, 103, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[204, 13, 397, 10, 1357, 2682, 160, 3630, 90, 4, 522, 2552, 178, 1849, 263, 1556, 32, 1816, 16, 4069, 2, 1019, 5, 4, 1032, 158, 796, 3929, 4074, 0, 214, 135] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[190, 113, 1096, 22, 226, 2581, 1026, 3188, 1, 32, 184, 2693, 206, 2264, 0, 1838, 3, 447, 3, 3848, 34, 0, 51, 0, 100, 1322, 2977, 4089, 3925, 0, 309, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2573, 0, 1663, 78, 952, 2244, 2113, 2245, 21, 0, 369, 1443, 4, 1849, 7, 2572, 65, 57, 449, 4089, 5, 14, 15, 141, 654, 25, 915, 3918, 4088, 14, 88, 98] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[24, 0, 3316, 12, 560, 3708, 188, 2509, 4, 1, 137, 1409, 0, 3362, 4, 3447, 7, 1196, 0, 4010, 19, 191, 0, 1, 410, 73, 404, 4091, 3663, 8, 0, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[14, 0, 2885, 2, 3315, 2275, 54, 3397, 323, 33, 1130, 107, 228, 3864, 14, 1859, 0, 85, 27, 4054, 88, 5, 70, 19, 20, 720, 257, 3866, 4035, 2, 15, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[409, 2, 2233, 143, 916, 510, 515, 21, 2328, 0, 1445, 1517, 0, 0, 0, 0, 210, 2, 500, 1315, 1195, 747, 0, 0, 13, 2894, 0, 511, 0, 0, 0, 0, 40, 83, 0, 3526, 0, 0, 0, 0, 19, 11, 0, 84, 3949, 725, 0, 0, 135, 129, 178, 2430, 0, 0, 0, 0, 4032, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4749, 5290, 3969, 3418, 3649, 4788, 2872, 4033] avg=4096.0000 max=5290 max_rank=1 min=2872 min_rank=6 imbalance_ratio=1.291504 max_load_reduction=27478 max_load_reduction_pct=0.838562 moved_to_echo=15342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[223, 3, 525, 2, 942, 514, 61, 20, 2270, 0, 3156, 183, 0, 0, 0, 0, 188, 1, 1029, 1851, 143, 673, 0, 0, 24, 2977, 0, 441, 0, 0, 0, 0, 60, 115, 0, 3551, 0, 0, 0, 0, 3, 14, 642, 143, 3648, 653, 0, 0, 446, 169, 1967, 2192, 0, 0, 0, 0, 3834, 1, 103, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2290, 5609, 3885, 3442, 3726, 5103, 4774, 3939] avg=4096.0000 max=5609 max_rank=1 min=2290 min_rank=0 imbalance_ratio=1.369385 max_load_reduction=27159 max_load_reduction_pct=0.828827 moved_to_echo=17542 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[899, 0, 2328, 61, 628, 505, 466, 20, 3383, 0, 129, 1374, 0, 0, 0, 0, 294, 0, 1379, 985, 1082, 760, 0, 0, 46, 1987, 6, 1677, 0, 0, 0, 0, 23, 147, 4, 3488, 0, 0, 0, 0, 4, 13, 0, 16, 3234, 738, 0, 0, 36, 9, 360, 2473, 0, 0, 0, 0, 3939, 0, 130, 145, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4907, 4886, 4500, 3716, 3662, 4005, 2878, 4214] avg=4096.0000 max=4907 max_rank=0 min=2878 min_rank=6 imbalance_ratio=1.197998 max_load_reduction=27861 max_load_reduction_pct=0.850250 moved_to_echo=14759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2573, 0, 1663, 78, 444, 517, 358, 21, 952, 0, 2113, 1056, 0, 0, 0, 0, 21, 0, 369, 1443, 831, 750, 0, 0, 4, 1405, 7, 2572, 0, 0, 0, 0, 65, 57, 449, 3572, 0, 0, 0, 0, 5, 14, 15, 141, 2244, 728, 0, 0, 654, 25, 915, 2440, 0, 0, 0, 0, 4067, 14, 88, 98, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5654, 4121, 3414, 3988, 4143, 3147, 4034, 4267] avg=4096.0000 max=5654 max_rank=0 min=3147 min_rank=5 imbalance_ratio=1.380371 max_load_reduction=27114 max_load_reduction_pct=0.827454 moved_to_echo=15591 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[24, 0, 3316, 12, 808, 507, 400, 19, 560, 0, 188, 1180, 0, 0, 0, 0, 4, 1, 137, 1409, 929, 783, 0, 0, 0, 2554, 4, 3447, 0, 0, 0, 0, 7, 1196, 0, 3503, 0, 0, 0, 0, 19, 191, 0, 1, 3708, 760, 0, 0, 410, 73, 404, 2548, 0, 0, 0, 0, 3644, 8, 0, 14, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5086, 1928, 3263, 6005, 4706, 4679, 3435, 3666] avg=4096.0000 max=6005 max_rank=3 min=1928 min_rank=1 imbalance_ratio=1.466064 max_load_reduction=26763 max_load_reduction_pct=0.816742 moved_to_echo=16486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[204, 13, 397, 10, 447, 518, 583, 24, 1357, 0, 160, 1699, 0, 0, 0, 0, 90, 4, 522, 2552, 1348, 757, 0, 0, 178, 1402, 263, 1556, 0, 0, 0, 0, 32, 1816, 16, 3551, 0, 0, 0, 0, 2, 1019, 5, 4, 2682, 733, 0, 0, 1032, 158, 796, 2439, 0, 0, 0, 0, 4050, 0, 214, 135, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2196, 3216, 5273, 3399, 5415, 4445, 4425, 4399] avg=4096.0000 max=5415 max_rank=4 min=2196 min_rank=0 imbalance_ratio=1.322021 max_load_reduction=27353 max_load_reduction_pct=0.834747 moved_to_echo=18684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[190, 113, 1096, 22, 544, 487, 508, 20, 226, 0, 1026, 1500, 0, 0, 0, 0, 1, 32, 184, 2693, 1180, 783, 0, 0, 206, 1720, 0, 1838, 0, 0, 0, 0, 3, 447, 3, 3361, 0, 0, 0, 0, 34, 0, 51, 0, 2581, 760, 0, 0, 100, 1322, 2977, 2546, 0, 0, 0, 0, 3905, 0, 309, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2980, 2752, 4873, 3764, 3814, 3426, 6945, 4214] avg=4096.0000 max=6945 max_rank=6 min=2752 min_rank=1 imbalance_ratio=1.695557 max_load_reduction=25823 max_load_reduction_pct=0.788055 moved_to_echo=18399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 2885, 2, 929, 513, 542, 21, 3315, 0, 54, 1597, 0, 0, 0, 0, 323, 33, 1130, 107, 1258, 740, 0, 0, 228, 2935, 14, 1859, 0, 0, 0, 0, 0, 85, 27, 3541, 0, 0, 0, 0, 88, 5, 70, 19, 2275, 718, 0, 0, 20, 720, 257, 2408, 0, 0, 0, 0, 4014, 2, 15, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4906, 4966, 3591, 5036, 3653, 3175, 3405, 4036] avg=4096.0000 max=5036 max_rank=3 min=3175 min_rank=5 imbalance_ratio=1.229492 max_load_reduction=27732 max_load_reduction_pct=0.846313 moved_to_echo=14269 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4608, 256, 14464, 384, 5760, 4096, 3456, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[14464, 0, 8320, 10112, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 1280, 896, 512, 24448, 5888, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 17920, 384, 13952, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2944, 2688, 7936, 19584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1152, 128, 5376, 12416, 8064, 6016, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 3968, 512, 28160, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31488, 128, 896, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[2625, 57, 1991, 3769, 908, 0, 3983, 1, 18, 781, 3387, 0, 96, 972, 306, 197, 76, 4037, 3373, 0, 9, 96, 1091, 11, 2053, 1, 292, 11, 1213, 477, 0, 937] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1453, 1379, 0, 0, 261, 83, 3577, 0, 0, 4056, 4028, 0, 0, 3972, 0, 0, 1, 4095, 3661, 0, 0, 3, 0, 1549, 2821, 107, 17, 38, 711, 8, 0, 948] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4040, 9, 133, 2301, 24, 0, 4032, 0, 18, 3787, 1080, 14, 0, 754, 2, 0, 4, 2368, 3560, 0, 782, 4081, 10, 0, 1587, 8, 1, 51, 4086, 4, 1, 31] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4085, 1505, 131, 4041, 0, 5, 3839, 0, 0, 99, 2308, 77, 13, 4094, 1312, 295, 0, 3986, 1412, 0, 36, 29, 30, 857, 1069, 0, 0, 27, 2148, 89, 828, 453] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4067, 932, 980, 1068, 203, 1, 4020, 0, 4, 36, 2762, 25, 2314, 337, 37, 5, 0, 3767, 3924, 3542, 250, 3309, 57, 260, 4, 445, 0, 0, 307, 2, 109, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3847, 4094, 2267, 182, 45, 0, 3196, 0, 0, 3731, 3411, 1, 0, 2352, 188, 68, 0, 4032, 3824, 0, 383, 680, 0, 122, 77, 20, 0, 38, 175, 0, 0, 35] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3873, 3129, 0, 4089, 0, 0, 4095, 0, 33, 0, 2752, 2, 0, 3914, 2, 12, 16, 4089, 31, 1, 629, 507, 0, 648, 1, 173, 1, 17, 50, 4, 2547, 2153] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4019, 3982, 28, 3615, 0, 26, 4095, 7, 2, 3474, 2446, 1, 2, 2817, 59, 170, 0, 4062, 825, 41, 8, 238, 46, 11, 23, 18, 15, 112, 590, 416, 73, 1547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 57, 1991, 2396, 0, 0, 0, 0, 908, 0, 3983, 1, 56, 0, 0, 0, 18, 781, 2538, 0, 0, 0, 0, 0, 96, 972, 306, 197, 793, 652, 0, 0, 76, 1125, 3373, 0, 0, 0, 0, 0, 9, 96, 1091, 11, 1693, 400, 0, 0, 2053, 1, 292, 11, 2225, 0, 0, 0, 1213, 477, 0, 937, 1219, 721, 0, 0] rerouted_tokens_per_ep_rank=[4444, 4948, 3337, 3016, 4574, 3300, 4582, 4567] avg=4096.0000 max=4948 max_rank=1 min=3016 min_rank=3 imbalance_ratio=1.208008 max_load_reduction=27820 max_load_reduction_pct=0.848999 moved_to_echo=17023 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 133, 1468, 0, 0, 0, 0, 24, 0, 4032, 0, 17, 0, 0, 0, 18, 3787, 810, 14, 0, 0, 0, 0, 0, 754, 2, 0, 253, 395, 0, 0, 4, 660, 3560, 0, 0, 0, 0, 0, 782, 4081, 10, 0, 993, 615, 0, 0, 1587, 8, 1, 51, 3425, 0, 0, 0, 4086, 4, 1, 31, 715, 438, 0, 0] rerouted_tokens_per_ep_rank=[1610, 4073, 4629, 1404, 4224, 6481, 5072, 5275] avg=4096.0000 max=6481 max_rank=5 min=1404 min_rank=3 imbalance_ratio=1.582275 max_load_reduction=26287 max_load_reduction_pct=0.802216 moved_to_echo=21052 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 932, 980, 682, 0, 0, 0, 0, 203, 1, 4020, 0, 45, 0, 0, 0, 4, 36, 2070, 25, 0, 0, 0, 0, 2314, 337, 37, 5, 647, 183, 0, 0, 0, 1050, 3924, 3542, 0, 0, 0, 0, 250, 3309, 57, 260, 1580, 619, 0, 0, 4, 445, 0, 0, 3448, 0, 0, 0, 307, 2, 109, 1, 1137, 203, 0, 0] rerouted_tokens_per_ep_rank=[2594, 4269, 2135, 3523, 8516, 6075, 3897, 1759] avg=4096.0000 max=8516 max_rank=4 min=1759 min_rank=7 imbalance_ratio=2.079102 max_load_reduction=24252 max_load_reduction_pct=0.740112 moved_to_echo=20247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 2267, 117, 0, 0, 0, 0, 45, 0, 3196, 0, 56, 0, 0, 0, 0, 3731, 2556, 1, 0, 0, 0, 0, 0, 2352, 188, 68, 799, 31, 0, 0, 0, 1124, 3824, 0, 0, 0, 0, 0, 383, 680, 0, 122, 1691, 586, 0, 0, 77, 20, 0, 38, 3261, 0, 0, 0, 175, 0, 0, 35, 1217, 34, 0, 0] rerouted_tokens_per_ep_rank=[6478, 3297, 6288, 3438, 4948, 3462, 3396, 1461] avg=4096.0000 max=6478 max_rank=0 min=1461 min_rank=7 imbalance_ratio=1.581543 max_load_reduction=26290 max_load_reduction_pct=0.802307 moved_to_echo=13267 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3129, 0, 2607, 0, 0, 0, 0, 0, 0, 4095, 0, 45, 0, 0, 0, 33, 0, 2062, 2, 0, 0, 0, 0, 0, 3914, 2, 12, 645, 703, 0, 0, 16, 1140, 31, 1, 0, 0, 0, 0, 629, 507, 0, 648, 1715, 591, 0, 0, 1, 173, 1, 17, 3282, 0, 0, 0, 50, 4, 2547, 2153, 1234, 779, 0, 0] rerouted_tokens_per_ep_rank=[5736, 4140, 2097, 5276, 1188, 4090, 3474, 6767] avg=4096.0000 max=6767 max_rank=7 min=1188 min_rank=4 imbalance_ratio=1.652100 max_load_reduction=26001 max_load_reduction_pct=0.793488 moved_to_echo=15519 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1379, 0, 0, 0, 0, 0, 0, 261, 83, 3577, 0, 70, 0, 0, 0, 0, 4056, 3011, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 947, 0, 0, 0, 1, 1135, 3661, 0, 0, 0, 0, 0, 0, 3, 0, 1549, 1720, 222, 0, 0, 2821, 107, 17, 38, 1231, 0, 0, 0, 711, 8, 0, 948, 1240, 0, 0, 0] rerouted_tokens_per_ep_rank=[1379, 3991, 7067, 4919, 4797, 3494, 4214, 2907] avg=4096.0000 max=7067 max_rank=2 min=1379 min_rank=0 imbalance_ratio=1.725342 max_load_reduction=25701 max_load_reduction_pct=0.784332 moved_to_echo=15412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1505, 131, 2576, 0, 0, 0, 0, 0, 5, 3839, 0, 38, 0, 0, 0, 0, 99, 1729, 77, 0, 0, 0, 0, 13, 4094, 1312, 295, 541, 695, 0, 0, 0, 1111, 1412, 0, 0, 0, 0, 0, 36, 29, 30, 857, 1672, 623, 0, 0, 1069, 0, 0, 27, 3462, 0, 0, 0, 2148, 89, 828, 453, 1203, 770, 0, 0] rerouted_tokens_per_ep_rank=[4212, 3882, 1905, 6950, 2523, 3247, 4558, 5491] avg=4096.0000 max=6950 max_rank=3 min=1905 min_rank=2 imbalance_ratio=1.696777 max_load_reduction=25818 max_load_reduction_pct=0.787903 moved_to_echo=15819 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3982, 28, 2305, 0, 0, 0, 0, 0, 26, 4095, 7, 40, 0, 0, 0, 2, 3474, 1833, 1, 0, 0, 0, 0, 2, 2817, 59, 170, 573, 621, 0, 0, 0, 1132, 825, 41, 0, 0, 0, 0, 8, 238, 46, 11, 1704, 612, 0, 0, 23, 18, 15, 112, 3407, 0, 0, 0, 590, 416, 73, 1547, 1226, 689, 0, 0] rerouted_tokens_per_ep_rank=[6315, 4168, 5310, 4242, 1998, 2619, 3575, 4541] avg=4096.0000 max=6315 max_rank=0 min=1998 min_rank=4 imbalance_ratio=1.541748 max_load_reduction=26453 max_load_reduction_pct=0.807281 moved_to_echo=12733 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2176, 8960, 1280, 3584, 12800, 4352, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9344, 1024, 3584, 6144, 9216, 3712, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 15104, 5632, 12160, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 8576, 20736, 3584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2432, 19328, 1920, 768, 5248, 3328, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 16000, 16640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7680, 896, 384, 384, 23808, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1536, 128, 30848, 128, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[9, 81, 0, 3721, 139, 0, 2480, 0, 0, 213, 116, 846, 3944, 124, 2307, 3004, 0, 4090, 948, 2550, 158, 0, 0, 3747, 18, 239, 548, 314, 0, 1, 10, 3161] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[51, 0, 0, 4094, 1202, 3081, 7, 1, 0, 3967, 56, 8, 4096, 0, 2221, 0, 0, 390, 1, 19, 21, 14, 0, 4091, 854, 10, 0, 4017, 532, 0, 2, 4033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[881, 2, 0, 3536, 3983, 12, 475, 587, 153, 3872, 6, 7, 4062, 308, 213, 4000, 4, 1722, 48, 0, 0, 3, 453, 1045, 3, 1667, 8, 3662, 41, 0, 0, 2015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[35, 0, 64, 4096, 1, 0, 0, 10, 0, 4045, 7, 92, 24, 0, 1687, 923, 0, 4065, 302, 1466, 10, 35, 0, 1585, 0, 3507, 16, 3772, 5, 4077, 65, 2879] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2509, 3831, 2, 1751, 164, 970, 136, 107, 2214, 4, 373, 465, 4094, 8, 14, 33, 0, 3956, 0, 4093, 1, 1, 0, 1454, 0, 0, 73, 1098, 197, 39, 1254, 3927] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4095, 0, 0, 462, 0, 0, 4, 0, 0, 1263, 0, 0, 2073, 804, 3969, 0, 0, 4022, 51, 122, 1100, 0, 0, 3680, 0, 3385, 360, 4010, 2, 1, 35, 3330] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4012, 0, 1, 4088, 3991, 0, 0, 0, 7, 3934, 35, 332, 4096, 0, 0, 35, 0, 2272, 0, 2072, 0, 0, 0, 3733, 0, 2, 163, 3957, 2, 4, 0, 32] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1811, 1276, 272, 3065, 3494, 2589, 114, 5, 0, 2345, 1, 2351, 605, 5, 5, 0, 0, 4092, 222, 49, 0, 30, 1, 220, 0, 0, 3, 4085, 0, 2007, 48, 4073] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[35, 0, 64, 2284, 0, 0, 0, 0, 1, 0, 0, 10, 9, 0, 0, 0, 0, 4045, 7, 92, 625, 303, 0, 0, 15, 0, 1687, 923, 0, 0, 0, 0, 0, 3440, 302, 1466, 0, 0, 0, 0, 10, 35, 0, 1585, 1812, 0, 0, 0, 0, 3507, 16, 3319, 0, 0, 0, 0, 5, 4077, 65, 2879, 150, 0, 0, 0] rerouted_tokens_per_ep_rank=[2383, 20, 5072, 2625, 5208, 3442, 6842, 7176] avg=4096.0000 max=7176 max_rank=7 min=20 min_rank=1 imbalance_ratio=1.751953 max_load_reduction=25592 max_load_reduction_pct=0.781006 moved_to_echo=22668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 81, 0, 2075, 0, 0, 0, 0, 139, 0, 2480, 0, 1580, 0, 0, 0, 0, 213, 116, 846, 629, 25, 44, 0, 2250, 124, 2307, 3004, 0, 0, 0, 0, 0, 3461, 948, 2550, 0, 0, 0, 0, 158, 0, 0, 3747, 1646, 70, 0, 0, 18, 239, 548, 277, 0, 0, 0, 0, 0, 1, 10, 3161, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[2165, 4199, 1873, 7685, 6959, 5621, 1082, 3184] avg=4096.0000 max=7685 max_rank=3 min=1082 min_rank=6 imbalance_ratio=1.876221 max_load_reduction=25083 max_load_reduction_pct=0.765472 moved_to_echo=16846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[51, 0, 0, 2283, 0, 0, 0, 0, 1202, 3081, 7, 1, 1641, 0, 0, 0, 0, 3967, 56, 8, 60, 323, 46, 0, 2337, 0, 2221, 0, 0, 0, 0, 0, 0, 330, 1, 19, 0, 0, 0, 0, 21, 14, 0, 4091, 1811, 72, 0, 0, 854, 10, 0, 3534, 0, 0, 0, 0, 532, 0, 2, 4033, 160, 0, 0, 0] rerouted_tokens_per_ep_rank=[2334, 5932, 4460, 4558, 350, 6009, 4398, 4727] avg=4096.0000 max=6009 max_rank=5 min=350 min_rank=4 imbalance_ratio=1.467041 max_load_reduction=26759 max_load_reduction_pct=0.816620 moved_to_echo=15484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[881, 2, 0, 1972, 0, 0, 0, 0, 3983, 12, 475, 587, 1628, 0, 0, 0, 153, 3872, 6, 7, 265, 294, 46, 0, 2316, 308, 213, 4000, 0, 0, 0, 0, 4, 1457, 48, 0, 0, 0, 0, 0, 0, 3, 453, 1045, 1564, 72, 0, 0, 3, 1667, 8, 3222, 0, 0, 0, 0, 41, 0, 0, 2015, 146, 0, 0, 0] rerouted_tokens_per_ep_rank=[2855, 6685, 4643, 6837, 1509, 3137, 4900, 2202] avg=4096.0000 max=6837 max_rank=3 min=1509 min_rank=4 imbalance_ratio=1.669189 max_load_reduction=25931 max_load_reduction_pct=0.791351 moved_to_echo=11748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 258, 0, 0, 0, 0, 0, 0, 4, 0, 830, 0, 0, 0, 0, 1263, 0, 0, 619, 323, 23, 0, 1184, 804, 3969, 0, 0, 0, 0, 0, 0, 3403, 51, 122, 0, 0, 0, 0, 1100, 0, 0, 3680, 204, 36, 0, 0, 0, 3385, 360, 3527, 0, 0, 0, 0, 2, 1, 35, 3330, 160, 0, 0, 0] rerouted_tokens_per_ep_rank=[4353, 834, 2228, 5957, 3576, 5020, 7272, 3528] avg=4096.0000 max=7272 max_rank=6 min=834 min_rank=1 imbalance_ratio=1.775391 max_load_reduction=25496 max_load_reduction_pct=0.778076 moved_to_echo=19396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4012, 0, 1, 2277, 0, 0, 0, 0, 3991, 0, 0, 0, 1646, 0, 0, 0, 7, 3934, 35, 332, 353, 322, 50, 0, 2323, 0, 0, 35, 0, 0, 0, 0, 0, 1919, 0, 2072, 0, 0, 0, 0, 0, 0, 0, 3733, 1811, 77, 0, 0, 0, 2, 163, 3473, 0, 0, 0, 0, 2, 4, 0, 32, 162, 0, 0, 0] rerouted_tokens_per_ep_rank=[6290, 5637, 5033, 2358, 3991, 5621, 3638, 200] avg=4096.0000 max=6290 max_rank=0 min=200 min_rank=7 imbalance_ratio=1.535645 max_load_reduction=26478 max_load_reduction_pct=0.808044 moved_to_echo=13450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2509, 3831, 2, 977, 0, 0, 0, 0, 164, 970, 136, 107, 1640, 0, 0, 0, 2214, 4, 373, 465, 608, 88, 46, 0, 2336, 8, 14, 33, 0, 0, 0, 0, 0, 3348, 0, 4093, 0, 0, 0, 0, 1, 1, 0, 1454, 774, 72, 0, 0, 0, 0, 73, 967, 0, 0, 0, 0, 197, 39, 1254, 3927, 43, 0, 0, 0] rerouted_tokens_per_ep_rank=[7319, 3017, 3798, 2391, 7441, 2302, 1040, 5460] avg=4096.0000 max=7441 max_rank=4 min=1040 min_rank=6 imbalance_ratio=1.816650 max_load_reduction=25327 max_load_reduction_pct=0.772919 moved_to_echo=16243 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1811, 1276, 272, 1710, 0, 0, 0, 0, 3494, 2589, 114, 5, 242, 0, 0, 0, 0, 2345, 1, 2351, 629, 329, 6, 0, 347, 5, 5, 0, 0, 0, 0, 0, 0, 3463, 222, 49, 0, 0, 0, 0, 0, 30, 1, 220, 1355, 10, 0, 0, 0, 0, 3, 3593, 0, 0, 0, 0, 0, 2007, 48, 4073, 163, 0, 0, 0] rerouted_tokens_per_ep_rank=[5069, 6444, 5661, 357, 3734, 1616, 3596, 6291] avg=4096.0000 max=6444 max_rank=1 min=357 min_rank=3 imbalance_ratio=1.573242 max_load_reduction=26324 max_load_reduction_pct=0.803345 moved_to_echo=15237 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13440, 5248, 384, 13952, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13184, 1280, 10496, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13056, 6656, 3328, 768, 9216, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 20864, 1664, 10496, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 6144, 1536, 23552, 1024, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2432, 19712, 640, 4224, 3840, 2048, 384, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[896, 8832, 1280, 22016, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1408, 128, 512, 19584, 11008, 512, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2212, 0, 3990, 9, 1, 0, 133, 3153, 2940, 2, 0, 4040, 0, 4, 208, 538, 1, 0, 0, 0, 63, 306, 0, 2510, 37, 3530, 4, 1, 3153, 2070, 3863, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4007, 0, 264, 1440, 204, 3804, 3491, 3, 0, 2743, 2030, 0, 12, 209, 515, 0, 1833, 0, 1, 58, 58, 4095, 0, 1119, 5, 0, 1780, 1003, 4092, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[426, 0, 1855, 787, 102, 5, 0, 3724, 247, 14, 3825, 559, 3660, 3547, 4, 151, 1, 0, 1, 15, 0, 0, 510, 88, 0, 3002, 18, 189, 12, 3669, 4094, 2263] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3955, 416, 2897, 3, 1258, 4016, 0, 4092, 57, 77, 4096, 4001, 3410, 209, 0, 37, 129, 0, 18, 3, 0, 476, 14, 158, 0, 0, 0, 131, 0, 0, 3313, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 1, 4088, 2047, 4, 0, 0, 4020, 4026, 1, 2855, 2906, 1, 346, 0, 0, 2703, 122, 568, 0, 6, 0, 0, 0, 393, 0, 354, 83, 0, 52, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[947, 427, 3734, 40, 3563, 11, 191, 3952, 4046, 0, 2, 4040, 2189, 2, 4, 52, 2, 0, 0, 0, 37, 4, 64, 3976, 8, 32, 787, 622, 0, 0, 4033, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4077, 0, 4096, 0, 10, 1491, 994, 21, 0, 74, 549, 4089, 2947, 291, 397, 0, 20, 23, 1, 6, 0, 0, 0, 3743, 0, 1157, 568, 43, 60, 4052, 4059, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4063, 38, 3725, 4, 406, 1, 0, 4089, 2, 1, 2963, 3507, 25, 0, 6, 0, 2352, 0, 3063, 39, 23, 25, 52, 4085, 102, 1919, 37, 0, 0, 0, 2232, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2212, 0, 1296, 9, 0, 0, 0, 0, 1, 0, 133, 2193, 0, 0, 0, 0, 2940, 2, 0, 547, 0, 0, 0, 0, 0, 4, 208, 538, 1414, 222, 0, 0, 1, 0, 0, 0, 3332, 0, 0, 0, 63, 306, 0, 2510, 960, 549, 0, 0, 37, 3530, 4, 1, 2472, 161, 0, 0, 3153, 2070, 1900, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3517, 2327, 3489, 2386, 3333, 4388, 6205, 7123] avg=4096.0000 max=7123 max_rank=7 min=2327 min_rank=1 imbalance_ratio=1.739014 max_load_reduction=25645 max_load_reduction_pct=0.782623 moved_to_echo=21049 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1303, 0, 0, 0, 0, 0, 264, 1440, 204, 2646, 0, 0, 0, 0, 3491, 3, 0, 372, 0, 0, 0, 0, 2030, 0, 12, 209, 1498, 222, 0, 0, 515, 0, 1833, 0, 2262, 0, 0, 0, 1, 58, 58, 4095, 1158, 581, 0, 0, 0, 1119, 5, 0, 2482, 109, 0, 0, 1780, 1003, 2013, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1303, 4554, 3866, 3971, 4610, 5951, 3715, 4798] avg=4096.0000 max=5951 max_rank=5 min=1303 min_rank=0 imbalance_ratio=1.452881 max_load_reduction=26817 max_load_reduction_pct=0.818390 moved_to_echo=19074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[426, 0, 603, 787, 0, 0, 0, 0, 102, 5, 0, 2590, 0, 0, 0, 0, 247, 14, 3825, 76, 0, 0, 0, 0, 3660, 3547, 4, 151, 1499, 103, 0, 0, 1, 0, 1, 15, 461, 0, 0, 0, 0, 0, 510, 88, 1134, 582, 0, 0, 0, 3002, 18, 189, 1149, 22, 0, 0, 12, 3669, 2013, 2263, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1816, 2697, 4162, 8964, 478, 2314, 4380, 7957] avg=4096.0000 max=8964 max_rank=3 min=478 min_rank=4 imbalance_ratio=2.188477 max_load_reduction=23804 max_load_reduction_pct=0.726440 moved_to_echo=15129 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3955, 416, 941, 3, 0, 0, 0, 0, 1258, 4016, 0, 2846, 0, 0, 0, 0, 57, 77, 4096, 542, 0, 0, 0, 0, 3410, 209, 0, 37, 1213, 161, 0, 0, 129, 0, 18, 3, 3300, 0, 0, 0, 0, 476, 14, 158, 1246, 471, 0, 0, 0, 0, 0, 131, 1795, 159, 0, 0, 0, 0, 1629, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5315, 8120, 4772, 5030, 3450, 2365, 2085, 1631] avg=4096.0000 max=8120 max_rank=1 min=1631 min_rank=7 imbalance_ratio=1.982422 max_load_reduction=24648 max_load_reduction_pct=0.752197 moved_to_echo=9531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 1, 1329, 2047, 0, 0, 0, 0, 4, 0, 0, 2796, 0, 0, 0, 0, 4026, 1, 2855, 393, 0, 0, 0, 0, 1, 346, 0, 0, 1500, 227, 0, 0, 2703, 122, 568, 0, 2397, 0, 0, 0, 6, 0, 0, 0, 1224, 582, 0, 0, 393, 0, 354, 83, 2532, 116, 0, 0, 0, 52, 2014, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7473, 2800, 7275, 2074, 5790, 1812, 3478, 2066] avg=4096.0000 max=7473 max_rank=0 min=1812 min_rank=5 imbalance_ratio=1.824463 max_load_reduction=25295 max_load_reduction_pct=0.771942 moved_to_echo=13146 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4077, 0, 1324, 0, 0, 0, 0, 0, 10, 1491, 994, 10, 0, 0, 0, 0, 0, 74, 549, 545, 0, 0, 0, 0, 2947, 291, 397, 0, 1491, 231, 0, 0, 20, 23, 1, 6, 3377, 0, 0, 0, 0, 0, 0, 3743, 11, 580, 0, 0, 0, 1157, 568, 43, 2541, 167, 0, 0, 60, 4052, 1988, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5401, 2505, 1168, 5357, 3427, 4334, 4476, 6100] avg=4096.0000 max=6100 max_rank=7 min=1168 min_rank=2 imbalance_ratio=1.489258 max_load_reduction=26668 max_load_reduction_pct=0.813843 moved_to_echo=18337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[947, 427, 1214, 40, 0, 0, 0, 0, 3563, 11, 191, 2749, 0, 0, 0, 0, 4046, 0, 2, 547, 0, 0, 0, 0, 2189, 2, 4, 52, 1477, 207, 0, 0, 2, 0, 0, 0, 3332, 0, 0, 0, 37, 4, 64, 3976, 1203, 573, 0, 0, 8, 32, 787, 622, 2313, 161, 0, 0, 0, 0, 1983, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2628, 6514, 4595, 3931, 3334, 5857, 3923, 1986] avg=4096.0000 max=6514 max_rank=1 min=1986 min_rank=7 imbalance_ratio=1.590332 max_load_reduction=26254 max_load_reduction_pct=0.801208 moved_to_echo=15100 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4063, 38, 1210, 4, 0, 0, 0, 0, 406, 1, 0, 2844, 0, 0, 0, 0, 2, 1, 2963, 475, 0, 0, 0, 0, 25, 0, 6, 0, 817, 207, 0, 0, 2352, 0, 3063, 39, 2892, 0, 0, 0, 23, 25, 52, 4085, 1245, 317, 0, 0, 102, 1919, 37, 0, 2308, 140, 0, 0, 0, 0, 1098, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5315, 3251, 3441, 1055, 8346, 5747, 4506, 1107] avg=4096.0000 max=8346 max_rank=4 min=1055 min_rank=3 imbalance_ratio=2.037598 max_load_reduction=24422 max_load_reduction_pct=0.745300 moved_to_echo=19706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[19840, 896, 9344, 2944, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14336, 4480, 640, 1024, 11008, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5120, 10880, 14720, 2304, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5760, 256, 5504, 128, 21376, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[5632, 7040, 1536, 18688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 896, 768, 18688, 8192, 4352, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14848, 256, 14336, 3584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 10880, 1792, 1152, 17664, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4, 45, 1945, 133, 0, 0, 3229, 1017, 2, 3, 0, 4096, 0, 4094, 4091, 3644, 436, 36, 0, 0, 2, 2, 0, 0, 32, 3086, 447, 991, 2034, 3399, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 207, 0, 0, 0, 0, 4091, 3360, 11, 0, 101, 3843, 20, 3314, 1, 4, 4044, 1359, 67, 368, 0, 44, 0, 14, 1326, 115, 4088, 3564, 0, 2550, 43, 234] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[3926, 1, 3909, 126, 0, 1, 689, 902, 1, 0, 1627, 2636, 0, 4096, 3562, 0, 0, 0, 809, 913, 113, 3229, 725, 21, 41, 4059, 0, 1318, 19, 0, 0, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[18, 116, 0, 0, 0, 3861, 46, 41, 3884, 0, 0, 4096, 16, 4034, 4086, 4096, 0, 18, 52, 0, 50, 35, 1015, 85, 127, 3595, 45, 3452, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[15, 0, 0, 0, 10, 0, 2942, 2, 0, 0, 2716, 3870, 1, 3368, 786, 1660, 4093, 3, 0, 0, 9, 639, 1, 0, 634, 3951, 3, 4089, 3314, 662, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1, 14, 16, 0, 31, 0, 0, 3780, 3661, 254, 7, 3881, 38, 4096, 3126, 23, 2, 6, 7, 33, 933, 4, 162, 0, 0, 2165, 936, 1896, 3812, 0, 1, 3883] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[6, 4082, 7, 0, 0, 38, 3910, 1, 20, 2, 3199, 4096, 43, 3822, 0, 3277, 863, 0, 1, 64, 0, 687, 23, 0, 431, 4037, 4, 4057, 0, 98, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[188, 3420, 0, 0, 0, 99, 3975, 0, 3842, 168, 239, 4001, 3593, 3734, 2606, 63, 2, 67, 0, 3, 11, 0, 0, 30, 2115, 219, 517, 3698, 0, 156, 2, 20] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 207, 0, 0, 1293, 782, 0, 0, 0, 0, 4091, 3360, 0, 0, 0, 0, 11, 0, 101, 1642, 0, 0, 0, 0, 20, 0, 1, 4, 0, 0, 0, 0, 4044, 1359, 67, 368, 2147, 609, 0, 0, 0, 44, 0, 14, 2705, 0, 0, 0, 1326, 115, 4088, 124, 0, 0, 0, 0, 0, 2550, 43, 234, 1419, 0, 0, 0] rerouted_tokens_per_ep_rank=[2282, 7451, 1754, 25, 8594, 2763, 5653, 4246] avg=4096.0000 max=8594 max_rank=4 min=25 min_rank=3 imbalance_ratio=2.098145 max_load_reduction=24174 max_load_reduction_pct=0.737732 moved_to_echo=21256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3926, 1, 3909, 126, 478, 536, 0, 0, 0, 1, 689, 902, 144, 0, 0, 0, 1, 0, 1627, 1127, 0, 0, 0, 0, 0, 0, 3180, 0, 0, 0, 0, 0, 0, 0, 809, 913, 794, 754, 0, 0, 113, 3229, 725, 21, 3342, 0, 0, 0, 41, 4059, 0, 46, 0, 0, 0, 0, 19, 0, 0, 45, 973, 238, 0, 0] rerouted_tokens_per_ep_rank=[8976, 1736, 2755, 3180, 3270, 7430, 4146, 1275] avg=4096.0000 max=8976 max_rank=0 min=1275 min_rank=7 imbalance_ratio=2.191406 max_load_reduction=23792 max_load_reduction_pct=0.726074 moved_to_echo=16121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4, 45, 1945, 133, 359, 834, 0, 0, 0, 0, 3229, 1017, 166, 0, 0, 0, 2, 3, 0, 1750, 0, 0, 0, 0, 0, 0, 3651, 3644, 0, 0, 0, 0, 436, 36, 0, 0, 597, 754, 0, 0, 2, 2, 0, 0, 3340, 0, 0, 0, 32, 3086, 447, 35, 0, 0, 0, 0, 2034, 3399, 0, 0, 1512, 274, 0, 0] rerouted_tokens_per_ep_rank=[3320, 4412, 1755, 7295, 1823, 3344, 3600, 7219] avg=4096.0000 max=7295 max_rank=3 min=1755 min_rank=2 imbalance_ratio=1.781006 max_load_reduction=25473 max_load_reduction_pct=0.777374 moved_to_echo=15986 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[18, 116, 0, 0, 1252, 834, 0, 0, 0, 3861, 46, 41, 166, 0, 0, 0, 3884, 0, 0, 1750, 0, 0, 0, 0, 16, 0, 3646, 4096, 0, 0, 0, 0, 0, 18, 52, 0, 2080, 742, 0, 0, 50, 35, 1015, 85, 3292, 0, 0, 0, 127, 3595, 45, 120, 0, 0, 0, 0, 0, 0, 0, 0, 1512, 274, 0, 0] rerouted_tokens_per_ep_rank=[2220, 4114, 5634, 7758, 2892, 4477, 3887, 1786] avg=4096.0000 max=7758 max_rank=3 min=1786 min_rank=7 imbalance_ratio=1.894043 max_load_reduction=25010 max_load_reduction_pct=0.763245 moved_to_echo=13042 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 14, 16, 0, 688, 790, 0, 0, 31, 0, 0, 3780, 127, 0, 0, 0, 3661, 254, 7, 1658, 0, 0, 0, 0, 38, 0, 2790, 23, 0, 0, 0, 0, 2, 6, 7, 33, 1142, 754, 0, 0, 933, 4, 162, 0, 3342, 0, 0, 0, 0, 2165, 936, 66, 0, 0, 0, 0, 3812, 0, 1, 3883, 1433, 209, 0, 0] rerouted_tokens_per_ep_rank=[1509, 3938, 5580, 2851, 1944, 4441, 3167, 9338] avg=4096.0000 max=9338 max_rank=7 min=1509 min_rank=0 imbalance_ratio=2.279785 max_load_reduction=23430 max_load_reduction_pct=0.715027 moved_to_echo=18890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[15, 0, 0, 0, 1488, 792, 0, 0, 10, 0, 2942, 2, 34, 0, 0, 0, 0, 0, 2716, 1645, 0, 0, 0, 0, 1, 0, 696, 1660, 0, 0, 0, 0, 4093, 3, 0, 0, 2466, 620, 0, 0, 9, 639, 1, 0, 2748, 0, 0, 0, 634, 3951, 3, 135, 0, 0, 0, 0, 3314, 662, 0, 0, 1433, 56, 0, 0] rerouted_tokens_per_ep_rank=[2295, 2988, 4361, 2357, 7182, 3397, 4723, 5465] avg=4096.0000 max=7182 max_rank=4 min=2295 min_rank=0 imbalance_ratio=1.753418 max_load_reduction=25586 max_load_reduction_pct=0.780823 moved_to_echo=20767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[6, 4082, 7, 0, 1472, 834, 0, 0, 0, 38, 3910, 1, 0, 0, 0, 0, 20, 2, 3199, 1750, 0, 0, 0, 0, 43, 0, 0, 3277, 0, 0, 0, 0, 863, 0, 1, 64, 2444, 704, 0, 0, 0, 687, 23, 0, 3118, 0, 0, 0, 431, 4037, 4, 141, 0, 0, 0, 0, 0, 98, 0, 0, 1512, 0, 0, 0] rerouted_tokens_per_ep_rank=[6401, 3949, 4971, 3320, 4076, 3828, 4613, 1610] avg=4096.0000 max=6401 max_rank=0 min=1610 min_rank=7 imbalance_ratio=1.562744 max_load_reduction=26367 max_load_reduction_pct=0.804657 moved_to_echo=14127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[188, 3420, 0, 0, 1342, 815, 0, 0, 0, 99, 3975, 0, 106, 0, 0, 0, 3842, 168, 239, 1709, 0, 0, 0, 0, 3593, 0, 2326, 63, 0, 0, 0, 0, 2, 67, 0, 3, 2228, 687, 0, 0, 11, 0, 0, 30, 3047, 0, 0, 0, 2115, 219, 517, 128, 0, 0, 0, 0, 0, 156, 2, 20, 1477, 174, 0, 0] rerouted_tokens_per_ep_rank=[5765, 4180, 5958, 5982, 2987, 3088, 2979, 1829] avg=4096.0000 max=5982 max_rank=3 min=1829 min_rank=7 imbalance_ratio=1.460449 max_load_reduction=26786 max_load_reduction_pct=0.817444 moved_to_echo=10883 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 7936, 5888, 384, 8448, 6272, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9216, 6912, 128, 4224, 11392, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[3712, 0, 16384, 12800, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9472, 1536, 1024, 1408, 13952, 5632, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11520, 512, 7936, 13056, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1152, 4736, 2048, 256, 24960, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 4096, 18944, 9216, 768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4736, 21248, 6144, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4066, 1796, 3983, 1, 0, 0, 4030, 355, 0, 4063, 17, 0, 289, 5, 0, 4, 3502, 36, 21, 132, 0, 4087, 0, 0, 1, 3829, 977, 0, 0, 1574] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[47, 0, 0, 6, 582, 0, 413, 262, 3864, 3858, 3923, 92, 12, 0, 0, 4064, 3908, 3999, 63, 11, 8, 0, 23, 3720, 0, 0, 0, 3719, 190, 3, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3770, 0, 0, 64, 3944, 0, 74, 437, 7, 4096, 21, 3621, 1645, 0, 133, 5, 4089, 2442, 153, 0, 5, 0, 0, 72, 4087, 7, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 1, 0, 419, 2783, 698, 4024, 4083, 3701, 0, 4, 1294, 2008, 3070, 1, 1829, 0, 0, 3, 22, 234, 0, 5, 0, 4081, 38, 684, 3784, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3902, 0, 23, 1142, 4035, 0, 325, 3879, 3810, 4096, 6, 1891, 8, 0, 0, 9, 4088, 0, 0, 0, 1, 0, 0, 1, 1400, 0, 121, 3971, 0, 60, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4, 0, 3996, 4096, 2, 0, 2022, 0, 4087, 46, 2245, 44, 3508, 0, 2399, 0, 3847, 2628, 1121, 0, 48, 0, 10, 2377, 4, 0, 0, 0, 0, 0, 284, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[62, 13, 80, 0, 3149, 0, 129, 263, 18, 2108, 4006, 4090, 512, 0, 2292, 0, 4096, 1027, 75, 0, 0, 0, 0, 33, 0, 1, 1, 3937, 3816, 0, 2198, 862] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 5, 3897, 3336, 1628, 0, 1301, 151, 531, 1, 0, 4094, 1227, 17, 0, 0, 3101, 1653, 3993, 41, 141, 0, 0, 2, 0, 0, 0, 4070, 3561, 10, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4066, 1796, 0, 77, 0, 0, 3983, 1, 0, 0, 0, 0, 0, 0, 4030, 29, 0, 0, 0, 0, 0, 0, 17, 0, 289, 5, 0, 41, 0, 0, 0, 4, 3502, 36, 0, 0, 0, 0, 21, 132, 0, 4087, 4063, 4, 0, 0, 0, 0, 1, 3752, 0, 0, 0, 0, 977, 0, 0, 1574, 281, 0, 0, 0] rerouted_tokens_per_ep_rank=[5939, 3984, 4059, 352, 3542, 8307, 3753, 2832] avg=4096.0000 max=8307 max_rank=5 min=352 min_rank=3 imbalance_ratio=2.028076 max_load_reduction=24461 max_load_reduction_pct=0.746490 moved_to_echo=18434 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3770, 0, 0, 64, 296, 83, 0, 0, 3944, 0, 74, 437, 465, 0, 0, 0, 7, 319, 21, 0, 0, 0, 0, 0, 1645, 0, 133, 5, 1736, 483, 0, 0, 1592, 2442, 153, 0, 0, 0, 0, 0, 5, 0, 0, 72, 3621, 51, 0, 0, 4087, 7, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 3243, 0, 0, 0] rerouted_tokens_per_ep_rank=[4213, 4920, 347, 4002, 4187, 3749, 8107, 3243] avg=4096.0000 max=8107 max_rank=6 min=347 min_rank=2 imbalance_ratio=1.979248 max_load_reduction=24661 max_load_reduction_pct=0.752594 moved_to_echo=19286 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[47, 0, 0, 6, 283, 75, 0, 0, 582, 0, 413, 262, 445, 0, 0, 0, 3864, 300, 3923, 0, 0, 0, 0, 0, 12, 0, 0, 4064, 1659, 455, 0, 0, 1521, 3999, 63, 11, 0, 0, 0, 0, 8, 0, 23, 3720, 92, 48, 0, 0, 0, 0, 0, 3644, 0, 0, 0, 0, 190, 3, 1, 0, 3055, 0, 0, 0] rerouted_tokens_per_ep_rank=[411, 1702, 8087, 6190, 5594, 3891, 3644, 3249] avg=4096.0000 max=8087 max_rank=2 min=411 min_rank=0 imbalance_ratio=1.974365 max_load_reduction=24681 max_load_reduction_pct=0.753204 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 222, 86, 0, 0, 1, 0, 419, 2783, 349, 0, 0, 0, 698, 306, 4083, 0, 0, 0, 0, 0, 0, 4, 1294, 2008, 1303, 475, 0, 0, 1196, 1, 1829, 0, 0, 0, 0, 0, 0, 3, 22, 234, 3701, 52, 0, 0, 0, 5, 0, 3995, 0, 0, 0, 0, 38, 684, 3784, 0, 3191, 0, 0, 0] rerouted_tokens_per_ep_rank=[310, 3552, 5087, 5084, 3026, 4012, 4000, 7697] avg=4096.0000 max=7697 max_rank=7 min=310 min_rank=0 imbalance_ratio=1.879150 max_load_reduction=25071 max_load_reduction_pct=0.765106 moved_to_echo=18735 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3902, 0, 23, 1142, 296, 80, 0, 0, 4035, 0, 325, 3879, 465, 0, 0, 0, 3810, 319, 6, 0, 0, 0, 0, 0, 8, 0, 0, 9, 1735, 483, 0, 0, 1592, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1891, 51, 0, 0, 1400, 0, 121, 3891, 0, 0, 0, 0, 0, 60, 0, 0, 3243, 0, 0, 0] rerouted_tokens_per_ep_rank=[5443, 8704, 4135, 2235, 1592, 1944, 5412, 3303] avg=4096.0000 max=8704 max_rank=1 min=1592 min_rank=4 imbalance_ratio=2.125000 max_load_reduction=24064 max_load_reduction_pct=0.734375 moved_to_echo=12251 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 3996, 4096, 282, 0, 0, 0, 2, 0, 2022, 0, 442, 0, 0, 0, 4087, 0, 2245, 0, 0, 0, 0, 0, 3508, 0, 2399, 0, 1637, 10, 0, 0, 1486, 2628, 1121, 0, 0, 0, 0, 0, 48, 0, 10, 2377, 44, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 36, 0, 0, 0] rerouted_tokens_per_ep_rank=[8378, 2466, 6332, 7554, 5235, 2479, 4, 320] avg=4096.0000 max=8378 max_rank=0 min=4 min_rank=6 imbalance_ratio=2.045410 max_load_reduction=24390 max_load_reduction_pct=0.744324 moved_to_echo=8038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[62, 13, 80, 0, 296, 79, 0, 0, 3149, 0, 129, 263, 466, 0, 0, 0, 18, 165, 4006, 0, 0, 0, 0, 0, 512, 0, 2292, 0, 1739, 248, 0, 0, 1595, 1027, 75, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4090, 26, 0, 0, 0, 1, 1, 3858, 0, 0, 0, 0, 3816, 0, 2198, 862, 1669, 0, 0, 0] rerouted_tokens_per_ep_rank=[530, 4007, 4189, 4791, 2697, 4149, 3860, 8545] avg=4096.0000 max=8545 max_rank=7 min=530 min_rank=0 imbalance_ratio=2.086182 max_load_reduction=24223 max_load_reduction_pct=0.739227 moved_to_echo=19251 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 3897, 3336, 224, 82, 0, 0, 1628, 0, 1301, 151, 353, 0, 0, 0, 531, 1, 0, 0, 0, 0, 0, 0, 1227, 17, 0, 0, 1316, 0, 0, 0, 1208, 1653, 3993, 41, 0, 0, 0, 0, 141, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 3988, 0, 0, 0, 0, 3561, 10, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7544, 3433, 532, 2560, 6895, 4237, 3988, 3579] avg=4096.0000 max=7544 max_rank=0 min=532 min_rank=2 imbalance_ratio=1.841797 max_load_reduction=25224 max_load_reduction_pct=0.769775 moved_to_echo=18699 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7808, 128, 12160, 10496, 1920, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7040, 128, 6528, 6144, 11136, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8704, 768, 6400, 2560, 14720, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[10240, 11776, 10752, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[17408, 128, 4736, 7808, 3072, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17152, 1536, 14336, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 256, 128, 10624, 21632, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5504, 128, 128, 27264, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3776, 4093, 0, 15, 1573, 0, 3071, 2048, 969, 4053, 34, 8, 4080, 0, 1111, 0, 0, 84, 7, 4, 22, 2, 3240, 525, 27, 4024, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3, 379, 257, 800, 3880, 12, 4088, 3592, 0, 324, 3591, 9, 3712, 403, 1, 0, 106, 0, 0, 1, 318, 3687, 1, 4010, 1, 3, 0, 0, 3549, 41] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[116, 0, 0, 594, 0, 13, 2691, 123, 2072, 0, 0, 1706, 4086, 0, 4089, 0, 3, 4, 430, 0, 3932, 0, 4044, 12, 4096, 0, 0, 1, 0, 0, 4089, 667] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 8, 3325, 14, 4089, 3838, 15, 4071, 283, 0, 219, 4080, 15, 0, 50, 4006, 7, 0, 0, 326, 461, 3926, 0, 3559, 85, 8, 3, 362, 7, 0, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[158, 0, 0, 1089, 29, 11, 4058, 11, 0, 2422, 862, 139, 1732, 1, 4078, 0, 2039, 0, 0, 147, 3888, 1268, 3991, 1090, 3402, 549, 0, 83, 1552, 0, 167, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 16, 886, 4076, 3814, 51, 2, 0, 456, 0, 263, 358, 0, 0, 0, 3240, 3, 4092, 37, 0, 4081, 47, 4092, 0, 0, 0, 3552, 3697] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[10, 79, 3940, 1, 97, 1, 4095, 3, 4004, 220, 0, 3256, 7, 0, 15, 0, 6, 0, 123, 9, 0, 0, 3989, 0, 3846, 4096, 0, 4074, 0, 0, 897, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 49, 601, 4086, 3191, 3626, 20, 4068, 283, 0, 35, 4008, 109, 4019, 35, 0, 134, 0, 1, 591, 0, 3753, 53, 48, 0, 6, 0, 118, 0, 3934, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 379, 1382, 780, 26, 0, 257, 800, 1823, 12, 0, 0, 0, 0, 2706, 3592, 0, 324, 0, 0, 0, 0, 2364, 9, 3712, 403, 0, 0, 0, 0, 1, 0, 106, 0, 2057, 116, 0, 0, 0, 1, 176, 3687, 0, 0, 0, 0, 1, 4010, 1, 3, 0, 0, 0, 0, 0, 0, 3549, 41, 0, 447, 0, 0] rerouted_tokens_per_ep_rank=[2570, 2892, 6622, 6488, 2280, 3864, 4015, 4037] avg=4096.0000 max=6622 max_rank=2 min=2280 min_rank=4 imbalance_ratio=1.616699 max_load_reduction=26146 max_load_reduction_pct=0.797913 moved_to_echo=14196 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3776, 4093, 0, 15, 327, 887, 268, 0, 1573, 0, 1443, 2048, 0, 0, 0, 0, 642, 4053, 34, 8, 0, 0, 0, 0, 2685, 0, 1111, 0, 0, 0, 0, 0, 0, 84, 7, 4, 1628, 1189, 0, 0, 22, 2, 1783, 525, 0, 0, 0, 0, 17, 4024, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 508, 0, 0] rerouted_tokens_per_ep_rank=[9366, 5064, 4737, 3796, 2912, 2332, 4043, 518] avg=4096.0000 max=9366 max_rank=0 min=518 min_rank=7 imbalance_ratio=2.286621 max_load_reduction=23402 max_load_reduction_pct=0.714172 moved_to_echo=9805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 3325, 1376, 887, 324, 0, 14, 4089, 1804, 15, 0, 0, 0, 0, 2695, 283, 0, 219, 0, 0, 0, 0, 2685, 15, 0, 50, 0, 0, 0, 0, 4006, 7, 0, 0, 2034, 1441, 0, 0, 326, 461, 2161, 0, 0, 0, 0, 0, 2182, 85, 8, 3, 0, 0, 0, 0, 362, 7, 0, 11, 1377, 508, 0, 0] rerouted_tokens_per_ep_rank=[5920, 5922, 3197, 2750, 7488, 2948, 2278, 2265] avg=4096.0000 max=7488 max_rank=4 min=2265 min_rank=7 imbalance_ratio=1.828125 max_load_reduction=25280 max_load_reduction_pct=0.771484 moved_to_echo=14979 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[158, 0, 0, 1089, 0, 376, 330, 0, 29, 11, 1907, 11, 0, 0, 0, 0, 0, 2422, 862, 139, 0, 0, 0, 0, 1141, 1, 4078, 0, 0, 0, 0, 0, 2039, 0, 0, 147, 2151, 1465, 0, 0, 3888, 1268, 2196, 1090, 0, 0, 0, 0, 2086, 549, 0, 83, 0, 0, 0, 0, 1552, 0, 167, 2, 1316, 215, 0, 0] rerouted_tokens_per_ep_rank=[1953, 1958, 3423, 5220, 5802, 8442, 2718, 3252] avg=4096.0000 max=8442 max_rank=5 min=1953 min_rank=0 imbalance_ratio=2.061035 max_load_reduction=24326 max_load_reduction_pct=0.742371 moved_to_echo=20214 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[116, 0, 0, 594, 704, 891, 337, 0, 0, 13, 1261, 123, 0, 0, 0, 0, 1368, 0, 0, 1706, 0, 0, 0, 0, 2681, 0, 4089, 0, 0, 0, 0, 0, 3, 4, 430, 0, 1430, 1489, 0, 0, 3932, 0, 2218, 12, 0, 0, 0, 0, 2508, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4089, 667, 1588, 514, 0, 0] rerouted_tokens_per_ep_rank=[2642, 1397, 3074, 6770, 3356, 6162, 2509, 6858] avg=4096.0000 max=6858 max_rank=7 min=1397 min_rank=1 imbalance_ratio=1.674316 max_load_reduction=25910 max_load_reduction_pct=0.790710 moved_to_echo=18885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 49, 601, 1375, 871, 310, 0, 4086, 3191, 1704, 20, 0, 0, 0, 0, 2693, 283, 0, 35, 0, 0, 0, 0, 2638, 109, 4019, 35, 0, 0, 0, 0, 0, 134, 0, 1, 1922, 1378, 0, 0, 591, 0, 2065, 53, 0, 0, 0, 0, 30, 0, 6, 0, 0, 0, 0, 0, 118, 0, 3934, 0, 18, 499, 0, 0] rerouted_tokens_per_ep_rank=[3206, 9001, 3011, 6801, 3435, 2709, 36, 4569] avg=4096.0000 max=9001 max_rank=1 min=36 min_rank=6 imbalance_ratio=2.197510 max_load_reduction=23767 max_load_reduction_pct=0.725311 moved_to_echo=10749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 1289, 99, 3, 0, 0, 16, 417, 4076, 0, 0, 0, 0, 2525, 51, 2, 0, 0, 0, 0, 0, 301, 0, 263, 358, 0, 0, 0, 0, 0, 0, 0, 3240, 469, 13, 0, 0, 3, 4092, 21, 0, 0, 0, 0, 0, 2502, 47, 4092, 0, 0, 0, 0, 0, 0, 0, 3552, 3697, 1579, 56, 0, 0] rerouted_tokens_per_ep_rank=[1396, 4509, 2578, 922, 3722, 4116, 6641, 8884] avg=4096.0000 max=8884 max_rank=7 min=922 min_rank=3 imbalance_ratio=2.168945 max_load_reduction=23884 max_load_reduction_pct=0.728882 moved_to_echo=23363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[10, 79, 3940, 1, 1354, 1, 330, 0, 97, 1, 1924, 3, 0, 0, 0, 0, 2650, 220, 0, 3256, 0, 0, 0, 0, 6, 0, 15, 0, 0, 0, 0, 0, 6, 0, 123, 9, 2171, 1464, 0, 0, 0, 0, 2195, 0, 0, 0, 0, 0, 2358, 4096, 0, 4074, 0, 0, 0, 0, 0, 0, 897, 0, 1488, 0, 0, 0] rerouted_tokens_per_ep_rank=[5715, 2025, 6126, 21, 3773, 2195, 10528, 2385] avg=4096.0000 max=10528 max_rank=6 min=21 min_rank=3 imbalance_ratio=2.570312 max_load_reduction=22240 max_load_reduction_pct=0.678711 moved_to_echo=18881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 6016, 7808, 4864, 2048, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6144, 8192, 12288, 6400, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6144, 256, 768, 3456, 13952, 8576, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2048, 128, 16256, 4480, 7424, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14592, 256, 17408, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11776, 12928, 4224, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8832, 5888, 12928, 5376, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[15360, 11008, 1024, 5760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 177, 0, 4081, 12, 0, 0, 535, 4031, 1, 3140, 4093, 2, 349, 4096, 23, 0, 2, 55, 4090, 0, 945, 0, 134, 2457, 156, 0, 237, 3803, 3, 346] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[906, 0, 0, 4, 1333, 0, 0, 0, 3753, 3841, 5, 3603, 4053, 289, 0, 85, 0, 39, 0, 3799, 0, 1753, 0, 1202, 0, 3964, 0, 4072, 0, 65, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 4039, 0, 1405, 0, 193, 224, 2, 4096, 3318, 0, 2, 92, 3882, 0, 0, 0, 0, 19, 0, 0, 0, 3933, 0, 4096, 3324, 4087, 0, 50] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[345, 0, 0, 3, 4096, 0, 1, 4096, 0, 4008, 0, 1498, 4056, 77, 0, 4069, 506, 4, 0, 3429, 77, 1, 16, 4, 0, 191, 0, 2382, 0, 3909, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[3957, 0, 3347, 0, 3100, 0, 0, 244, 5, 0, 24, 4096, 168, 0, 61, 577, 42, 0, 0, 1206, 1365, 4090, 0, 142, 5, 4096, 0, 2236, 69, 3938, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[505, 17, 0, 187, 3799, 0, 1, 1914, 0, 2830, 0, 4069, 4079, 15, 0, 4066, 0, 0, 0, 5, 0, 26, 0, 4080, 0, 7, 7, 4096, 1216, 739, 36, 1074] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2537, 0, 1503, 1, 4093, 0, 817, 21, 0, 122, 0, 2, 3443, 2, 94, 4089, 3976, 0, 0, 2861, 8, 1971, 2, 22, 14, 33, 5, 290, 1, 3992, 0, 2869] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 3823, 3, 0, 1, 4094, 4087, 0, 4012, 2, 259, 9, 3660, 0, 618, 52, 0, 13, 191, 76, 226, 0, 4096, 11, 0, 0, 3442, 23, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[906, 0, 0, 4, 2234, 0, 0, 0, 1140, 0, 0, 0, 0, 0, 0, 0, 3753, 3841, 5, 738, 0, 0, 0, 0, 1900, 289, 0, 85, 0, 0, 0, 0, 0, 39, 0, 3799, 731, 193, 741, 1, 0, 1753, 0, 1202, 1422, 631, 0, 0, 0, 3223, 0, 4072, 0, 0, 0, 0, 0, 64, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3144, 1140, 8337, 2274, 5504, 5008, 7295, 66] avg=4096.0000 max=8337 max_rank=2 min=66 min_rank=7 imbalance_ratio=2.035400 max_load_reduction=24431 max_load_reduction_pct=0.745575 moved_to_echo=17873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 177, 0, 1947, 0, 0, 0, 3490, 12, 0, 0, 0, 0, 0, 0, 535, 4031, 1, 643, 0, 0, 0, 0, 1919, 2, 349, 4096, 0, 0, 0, 0, 23, 0, 2, 55, 738, 591, 459, 72, 4090, 0, 945, 0, 1436, 550, 0, 0, 134, 1998, 156, 0, 0, 0, 0, 0, 237, 3731, 3, 346, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2124, 3502, 5210, 6366, 1940, 7021, 2288, 4317] avg=4096.0000 max=7021 max_rank=5 min=1940 min_rank=4 imbalance_ratio=1.714111 max_load_reduction=25747 max_load_reduction_pct=0.785736 moved_to_echo=15566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[345, 0, 0, 3, 928, 0, 0, 0, 3503, 0, 1, 4096, 0, 0, 0, 0, 0, 4008, 0, 308, 0, 0, 0, 0, 1902, 77, 0, 4069, 0, 0, 0, 0, 506, 4, 0, 3429, 731, 593, 35, 74, 77, 1, 16, 4, 1423, 262, 0, 0, 0, 156, 0, 2382, 0, 0, 0, 0, 0, 3835, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1276, 7600, 4316, 6048, 5372, 1783, 2538, 3835] avg=4096.0000 max=7600 max_rank=1 min=1276 min_rank=0 imbalance_ratio=1.855469 max_load_reduction=25168 max_load_reduction_pct=0.768066 moved_to_echo=13528 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[505, 17, 0, 187, 2523, 0, 0, 0, 3249, 0, 1, 1914, 0, 0, 0, 0, 0, 2830, 0, 833, 0, 0, 0, 0, 1912, 15, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 5, 735, 550, 1, 14, 0, 26, 0, 4080, 1432, 713, 0, 0, 0, 6, 7, 4096, 0, 0, 0, 0, 1216, 725, 36, 1074, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3232, 5164, 3663, 5993, 1305, 6251, 4109, 3051] avg=4096.0000 max=6251 max_rank=5 min=1305 min_rank=4 imbalance_ratio=1.526123 max_load_reduction=26517 max_load_reduction_pct=0.809235 moved_to_echo=14716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2537, 0, 1503, 1, 1, 0, 0, 0, 3500, 0, 817, 21, 0, 0, 0, 0, 0, 122, 0, 1, 0, 0, 0, 0, 1614, 2, 94, 4089, 0, 0, 0, 0, 3976, 0, 0, 2861, 621, 593, 6, 76, 8, 1971, 2, 22, 1208, 0, 0, 0, 14, 27, 5, 290, 0, 0, 0, 0, 1, 3916, 0, 2869, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4338, 123, 5799, 8133, 3211, 336, 6786] avg=4096.0000 max=8133 max_rank=4 min=123 min_rank=2 imbalance_ratio=1.985596 max_load_reduction=24635 max_load_reduction_pct=0.751801 moved_to_echo=18466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 2543, 0, 0, 0, 3451, 0, 1405, 0, 0, 0, 0, 0, 193, 224, 2, 830, 0, 0, 0, 0, 1546, 0, 2, 92, 0, 0, 0, 0, 3882, 0, 0, 0, 602, 588, 739, 80, 0, 19, 0, 0, 1170, 723, 0, 0, 0, 3194, 0, 4096, 0, 0, 0, 0, 3324, 4007, 0, 50, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2549, 4856, 1249, 1640, 5891, 1912, 7290, 7381] avg=4096.0000 max=7381 max_rank=7 min=1249 min_rank=2 imbalance_ratio=1.802002 max_load_reduction=25387 max_load_reduction_pct=0.774750 moved_to_echo=22474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3957, 0, 3347, 0, 2540, 0, 0, 0, 2651, 0, 0, 244, 0, 0, 0, 0, 5, 0, 24, 838, 0, 0, 0, 0, 80, 0, 61, 577, 0, 0, 0, 0, 42, 0, 0, 1206, 30, 449, 766, 75, 1365, 4090, 0, 142, 58, 718, 0, 0, 5, 3330, 0, 2236, 0, 0, 0, 0, 69, 3863, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9844, 2895, 867, 718, 2568, 6373, 5571, 3932] avg=4096.0000 max=9844 max_rank=0 min=718 min_rank=3 imbalance_ratio=2.403320 max_load_reduction=22924 max_load_reduction_pct=0.699585 moved_to_echo=18444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 2487, 0, 0, 0, 3269, 3, 0, 1, 0, 0, 0, 0, 4094, 4087, 0, 822, 0, 0, 0, 0, 2, 259, 9, 3660, 0, 0, 0, 0, 0, 618, 52, 0, 0, 554, 766, 65, 13, 191, 76, 226, 0, 703, 0, 0, 0, 3330, 11, 0, 0, 0, 0, 0, 0, 3377, 23, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6557, 3273, 9003, 3930, 2055, 1209, 3341, 3400] avg=4096.0000 max=9003 max_rank=2 min=1209 min_rank=5 imbalance_ratio=2.197998 max_load_reduction=23765 max_load_reduction_pct=0.725250 moved_to_echo=10005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8704, 19200, 128, 5120, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 15360, 256, 17280, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5632, 8064, 1152, 5760, 8192, 4352, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8448, 768, 128, 11392, 4224, 4224, 3584, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10880, 768, 640, 20736, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12416, 128, 5120, 256, 15232, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[4864, 23552, 128, 4352, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[24320, 128, 2304, 6400, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[139, 0, 0, 0, 3897, 0, 0, 0, 1835, 2, 0, 4, 4096, 3, 0, 2446, 0, 3021, 189, 17, 0, 3, 3988, 7, 4096, 738, 0, 9, 1, 104, 4079, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[3826, 237, 0, 0, 0, 4034, 1, 3175, 60, 4096, 0, 284, 208, 0, 0, 60, 156, 3592, 0, 88, 1683, 487, 0, 0, 4096, 0, 1281, 1, 0, 108, 1199, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4029, 6, 0, 14, 363, 0, 0, 1735, 1366, 0, 34, 4089, 0, 0, 0, 0, 4067, 0, 0, 909, 0, 4029, 0, 3909, 0, 39, 0, 19, 2, 4062, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 104, 0, 3567, 581, 0, 9, 0, 1, 0, 4090, 3794, 361, 21, 4096, 8, 1419, 1, 0, 0, 16, 2954, 0, 3879, 0, 0, 0, 0, 0, 3897, 3969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4072, 116, 0, 180, 4091, 3826, 4, 0, 4037, 120, 0, 1112, 55, 1541, 0, 0, 20, 1573, 0, 0, 0, 0, 3829, 4069, 0, 52, 0, 1, 0, 1, 4069, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[72, 0, 79, 0, 3282, 2134, 191, 4026, 4090, 0, 0, 659, 1345, 0, 10, 8, 0, 4091, 0, 10, 1343, 0, 4059, 2, 72, 4, 459, 1, 0, 6, 4091, 2734] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 1141, 3, 3977, 417, 0, 3858, 3472, 4095, 29, 15, 0, 2867, 0, 229, 3932, 1, 0, 1, 3631, 0, 0, 0, 0, 0, 0, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[91, 0, 0, 60, 34, 96, 883, 3530, 943, 0, 0, 3990, 0, 6, 1790, 2053, 88, 1508, 78, 1813, 3807, 1, 3867, 0, 4079, 0, 0, 9, 0, 0, 1, 4041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 104, 0, 2131, 358, 0, 0, 2103, 581, 0, 9, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 3611, 361, 21, 4096, 0, 0, 0, 0, 8, 1419, 1, 0, 1070, 450, 0, 0, 0, 16, 2146, 0, 0, 0, 0, 0, 3879, 0, 0, 0, 0, 394, 183, 0, 0, 0, 3897, 1838, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2594, 2693, 4091, 8089, 2948, 2162, 4456, 5735] avg=4096.0000 max=8089 max_rank=3 min=2162 min_rank=5 imbalance_ratio=1.974854 max_load_reduction=24679 max_load_reduction_pct=0.753143 moved_to_echo=15301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[139, 0, 0, 0, 2198, 483, 0, 0, 2297, 0, 0, 0, 0, 0, 0, 0, 1402, 2, 0, 4, 0, 0, 0, 0, 3898, 3, 0, 2446, 0, 0, 0, 0, 0, 3021, 189, 17, 1170, 607, 0, 0, 0, 3, 2898, 7, 0, 0, 0, 0, 4096, 738, 0, 9, 433, 430, 198, 0, 1, 104, 4079, 1896, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2820, 2297, 1408, 6347, 5004, 2908, 5904, 6080] avg=4096.0000 max=6347 max_rank=3 min=1408 min_rank=2 imbalance_ratio=1.549561 max_load_reduction=26421 max_load_reduction_pct=0.806305 moved_to_echo=19896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4072, 116, 0, 180, 0, 466, 0, 0, 2406, 3826, 4, 0, 0, 0, 0, 0, 3083, 120, 0, 1112, 0, 0, 0, 0, 53, 1541, 0, 0, 0, 0, 0, 0, 20, 1573, 0, 0, 1230, 586, 0, 0, 0, 0, 2777, 4069, 0, 0, 0, 0, 0, 52, 0, 1, 954, 455, 2, 0, 0, 1, 4069, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4834, 6236, 4315, 1594, 3409, 6846, 1464, 4070] avg=4096.0000 max=6846 max_rank=5 min=1464 min_rank=6 imbalance_ratio=1.671387 max_load_reduction=25922 max_load_reduction_pct=0.791077 moved_to_echo=15789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3826, 237, 0, 0, 2202, 0, 0, 0, 0, 4034, 1, 3175, 0, 0, 0, 0, 43, 4096, 0, 284, 0, 0, 0, 0, 195, 0, 0, 60, 0, 0, 0, 0, 156, 3592, 0, 88, 0, 0, 0, 0, 1683, 487, 0, 0, 0, 0, 0, 0, 4096, 0, 1281, 1, 17, 0, 13, 0, 0, 108, 1199, 1894, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6265, 7210, 4423, 255, 3836, 2170, 5408, 3201] avg=4096.0000 max=7210 max_rank=1 min=255 min_rank=3 imbalance_ratio=1.760254 max_load_reduction=25558 max_load_reduction_pct=0.779968 moved_to_echo=14615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4029, 6, 0, 2199, 488, 0, 0, 9, 363, 0, 0, 0, 0, 0, 0, 1325, 1366, 0, 34, 0, 0, 0, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 4, 614, 0, 0, 909, 0, 2927, 0, 0, 0, 0, 0, 3909, 0, 39, 0, 410, 1, 197, 0, 19, 2, 4062, 1897, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6722, 372, 2725, 3892, 4685, 3836, 4556, 5980] avg=4096.0000 max=6722 max_rank=0 min=372 min_rank=1 imbalance_ratio=1.641113 max_load_reduction=26046 max_load_reduction_pct=0.794861 moved_to_echo=19057 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 539, 0, 0, 0, 0, 0, 1141, 3, 0, 0, 0, 0, 3037, 417, 0, 3858, 0, 0, 0, 0, 3305, 4095, 29, 15, 0, 0, 0, 0, 0, 2867, 0, 229, 0, 0, 0, 0, 3932, 1, 0, 1, 0, 0, 0, 0, 3631, 0, 0, 0, 940, 0, 167, 0, 0, 0, 0, 466, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4634, 1144, 7312, 7444, 3096, 3934, 4738, 466] avg=4096.0000 max=7444 max_rank=3 min=466 min_rank=7 imbalance_ratio=1.817383 max_load_reduction=25324 max_load_reduction_pct=0.772827 moved_to_echo=12234 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[72, 0, 79, 0, 1468, 492, 0, 0, 1935, 2134, 191, 4026, 0, 0, 0, 0, 3124, 0, 0, 659, 0, 0, 0, 0, 1280, 0, 10, 8, 0, 0, 0, 0, 0, 4091, 0, 10, 985, 618, 0, 0, 1343, 0, 2949, 2, 0, 0, 0, 0, 72, 4, 459, 1, 966, 362, 65, 0, 0, 6, 4091, 1266, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2111, 8286, 3783, 1298, 5704, 4294, 1929, 5363] avg=4096.0000 max=8286 max_rank=1 min=1298 min_rank=3 imbalance_ratio=2.022949 max_load_reduction=24482 max_load_reduction_pct=0.747131 moved_to_echo=17290 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[91, 0, 0, 60, 2169, 468, 0, 0, 21, 96, 883, 3530, 0, 0, 0, 0, 721, 0, 0, 3990, 0, 0, 0, 0, 0, 6, 1790, 2053, 0, 0, 0, 0, 88, 1508, 78, 1813, 10, 589, 0, 0, 3807, 1, 2810, 0, 0, 0, 0, 0, 4079, 0, 0, 9, 222, 3, 0, 0, 0, 0, 1, 1872, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2788, 4530, 4711, 3849, 4086, 6618, 4313, 1873] avg=4096.0000 max=6618 max_rank=5 min=1873 min_rank=7 imbalance_ratio=1.615723 max_load_reduction=26150 max_load_reduction_pct=0.798035 moved_to_echo=16890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[16256, 6016, 1920, 8704, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8832, 11136, 2304, 10752, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 256, 21504, 11136, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12416, 4480, 256, 256, 12928, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 22144, 384, 2176, 4480, 3584, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[11776, 512, 16512, 4096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[23808, 896, 1792, 128, 3968, 1664, 896, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[12800, 6016, 0, 14080, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1, 57, 0, 85, 0, 210, 0, 500, 4096, 483, 3581, 0, 9, 0, 1, 0, 3979, 4091, 0, 130, 3154, 3130, 3991, 3581, 0, 1460, 0, 0, 12, 217, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 26, 0, 13, 4, 0, 927, 3294, 3971, 11, 4, 1121, 1233, 4053, 3931, 3239, 4094, 1, 3599, 976, 1745, 0, 0, 0, 0, 0, 102, 200, 23, 200] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 3644, 0, 26, 0, 54, 26, 0, 2, 2954, 4079, 389, 2572, 158, 4064, 2207, 0, 1167, 1169, 0, 248, 3779, 280, 106, 42, 0, 0, 4, 1362, 0, 4047, 389] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 107, 0, 4096, 4096, 4011, 0, 584, 0, 4094, 0, 72, 0, 2506, 0, 3410, 4041, 129, 1169, 0, 0, 0, 0, 360, 0, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[214, 4090, 1127, 0, 1, 0, 0, 0, 10, 366, 4091, 56, 14, 632, 665, 0, 3, 4094, 8, 517, 0, 3240, 2646, 3383, 0, 0, 5, 0, 0, 0, 4068, 3538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 753, 0, 30, 130, 32, 0, 1, 10, 3284, 43, 3990, 4096, 0, 241, 3955, 0, 3940, 3950, 0, 0, 122, 11, 0, 1, 7, 0, 4, 118, 4041, 4009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4071, 45, 11, 63, 10, 183, 1, 4093, 0, 0, 1, 4087, 3861, 10, 4096, 2985, 0, 95, 21, 2592, 0, 2, 1568, 0, 0, 0, 900, 4073, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1403, 28, 0, 6, 1, 3207, 1882, 0, 4033, 4096, 174, 0, 0, 1, 106, 4096, 0, 4085, 2825, 4, 3872, 0, 0, 170, 0, 0, 38, 0, 0, 1168, 1573] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3644, 0, 26, 516, 1444, 614, 325, 0, 54, 26, 0, 2668, 807, 0, 0, 2, 2954, 986, 389, 0, 0, 0, 0, 2572, 158, 2620, 2207, 0, 0, 0, 0, 0, 651, 0, 0, 0, 0, 0, 0, 248, 497, 280, 106, 0, 0, 0, 0, 42, 0, 0, 4, 3093, 362, 0, 0, 1362, 0, 3722, 389, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6569, 3555, 4331, 7557, 651, 1131, 3501, 5473] avg=4096.0000 max=7557 max_rank=3 min=651 min_rank=4 imbalance_ratio=1.844971 max_load_reduction=25211 max_load_reduction_pct=0.769379 moved_to_echo=10756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 57, 0, 85, 1813, 0, 508, 0, 0, 210, 0, 500, 2210, 0, 0, 0, 4096, 483, 866, 0, 0, 0, 0, 0, 9, 0, 1, 0, 0, 0, 0, 0, 3979, 2278, 0, 130, 0, 0, 0, 0, 3154, 412, 3991, 3581, 0, 0, 0, 0, 0, 1460, 0, 0, 2715, 0, 0, 0, 12, 217, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2464, 2920, 5445, 10, 6387, 11138, 4175, 229] avg=4096.0000 max=11138 max_rank=5 min=10 min_rank=3 imbalance_ratio=2.719238 max_load_reduction=21630 max_load_reduction_pct=0.660095 moved_to_echo=21929 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 26, 1434, 438, 158, 1, 0, 13, 4, 0, 689, 2828, 0, 0, 927, 3294, 960, 11, 0, 0, 0, 0, 4, 1121, 795, 4053, 0, 0, 0, 0, 3931, 1805, 0, 1, 0, 0, 0, 0, 3599, 129, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3011, 1266, 0, 0, 102, 200, 22, 200, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2058, 3534, 5192, 5973, 5737, 5473, 4277, 524] avg=4096.0000 max=5973 max_rank=3 min=524 min_rank=7 imbalance_ratio=1.458252 max_load_reduction=26795 max_load_reduction_pct=0.817719 moved_to_echo=16011 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[214, 4090, 1127, 0, 1812, 236, 526, 326, 1, 0, 0, 0, 2287, 5, 0, 0, 10, 366, 989, 56, 0, 0, 0, 0, 14, 632, 429, 0, 0, 0, 0, 0, 3, 2282, 0, 517, 0, 0, 0, 0, 0, 427, 2646, 3383, 0, 0, 0, 0, 0, 0, 5, 0, 3102, 3, 0, 0, 0, 0, 3742, 3538, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8331, 2293, 1421, 1075, 2802, 6456, 3110, 7280] avg=4096.0000 max=8331 max_rank=0 min=1075 min_rank=3 imbalance_ratio=2.033936 max_load_reduction=24437 max_load_reduction_pct=0.745758 moved_to_echo=19648 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4071, 1813, 1452, 3, 327, 45, 11, 63, 10, 14, 2063, 0, 0, 183, 1, 990, 0, 0, 0, 0, 0, 0, 1, 2635, 3861, 0, 0, 0, 0, 10, 2283, 0, 0, 0, 0, 0, 0, 95, 4, 2592, 0, 0, 0, 0, 0, 2, 1568, 0, 0, 3103, 922, 0, 0, 0, 900, 3746, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7666, 2206, 1174, 6497, 2293, 2691, 5595, 4646] avg=4096.0000 max=7666 max_rank=0 min=1174 min_rank=2 imbalance_ratio=1.871582 max_load_reduction=25102 max_load_reduction_pct=0.766052 moved_to_echo=15225 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1456, 661, 332, 0, 0, 107, 0, 2856, 1732, 0, 0, 4096, 4096, 967, 0, 0, 0, 0, 0, 584, 0, 2638, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 3410, 524, 129, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 3044, 774, 0, 0, 360, 0, 3760, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2450, 4695, 9159, 3222, 72, 5232, 3818, 4120] avg=4096.0000 max=9159 max_rank=2 min=72 min_rank=4 imbalance_ratio=2.236084 max_load_reduction=23609 max_load_reduction_pct=0.720490 moved_to_echo=13242 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 753, 0, 0, 0, 0, 324, 30, 130, 32, 0, 0, 2722, 0, 0, 1, 10, 794, 43, 0, 0, 0, 0, 3990, 4096, 0, 241, 0, 0, 0, 0, 3955, 0, 0, 3950, 0, 0, 0, 0, 0, 0, 122, 11, 0, 0, 0, 0, 0, 1, 7, 0, 2490, 1218, 0, 0, 4, 118, 3717, 4009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1077, 2914, 848, 8327, 7905, 133, 3716, 7848] avg=4096.0000 max=8327 max_rank=3 min=133 min_rank=5 imbalance_ratio=2.032959 max_load_reduction=24441 max_load_reduction_pct=0.745880 moved_to_echo=19602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1403, 28, 0, 0, 0, 629, 93, 6, 1, 3207, 1882, 2733, 2822, 0, 0, 0, 4033, 991, 174, 0, 0, 0, 0, 0, 0, 1, 106, 0, 0, 0, 0, 4096, 0, 0, 2825, 0, 0, 0, 0, 4, 510, 0, 0, 0, 0, 0, 0, 170, 0, 0, 38, 3105, 1263, 0, 0, 0, 0, 1075, 1573, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2153, 10651, 5198, 107, 6921, 514, 4576, 2648] avg=4096.0000 max=10651 max_rank=1 min=107 min_rank=3 imbalance_ratio=2.600342 max_load_reduction=22117 max_load_reduction_pct=0.674957 moved_to_echo=14659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 9216, 1920, 4224, 7424, 5120, 3200, 1792] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[16128, 9344, 0, 7424, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10624, 2560, 11520, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1920, 1536, 19840, 9728, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9344, 15360, 7552, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7296, 6016, 9216, 10496, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 3456, 2432, 13568, 13056, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 3072, 128, 128, 23680, 5888, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[2550, 4056, 880, 0, 0, 3466, 0, 24, 0, 2958, 0, 4, 1291, 314, 0, 0, 0, 154, 3907, 4092, 6, 33, 1051, 23, 4040, 344, 3486, 0, 15, 0, 74, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[138, 2, 6, 4096, 4069, 0, 0, 0, 0, 0, 3879, 15, 268, 124, 3172, 1, 226, 3932, 113, 1423, 0, 0, 60, 3602, 3, 3738, 13, 0, 3626, 0, 0, 262] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4092, 4051, 1, 1852, 28, 2594, 15, 4, 2, 0, 1, 4083, 0, 0, 4095, 225, 0, 1, 3954, 250, 3162, 0, 244, 0, 3340, 0, 11, 97, 620, 0, 2, 44] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 3999, 2599, 0, 3983, 1, 1288, 0, 0, 204, 0, 3402, 4, 2771, 337, 64, 0, 33, 3991, 858, 0, 0, 95, 0, 1342, 23, 4084, 320, 3349, 18, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3809, 0, 1663, 0, 4093, 13, 11, 35, 0, 8, 3510, 45, 293, 3361, 0, 55, 0, 0, 3519, 0, 9, 4, 5, 0, 17, 0, 145, 0, 4095, 0, 4096, 3982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3843, 0, 2991, 22, 4095, 3833, 82, 4037, 0, 0, 3998, 92, 0, 616, 4065, 0, 67, 0, 4083, 0, 203, 0, 0, 0, 1, 6, 590, 0, 4, 83, 0, 57] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 3475, 3139, 84, 0, 452, 3994, 4096, 5, 0, 178, 4096, 0, 0, 4049, 0, 16, 175, 4053, 4028, 44, 0, 0, 0, 874, 0, 1, 0, 0, 8, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 3590, 8, 0, 0, 3913, 0, 0, 68, 0, 34, 4062, 5, 0, 4092, 0, 1, 0, 4096, 4091, 1299, 1208, 193, 25, 0, 0, 109, 0, 2975, 0, 2936, 63] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2550, 211, 880, 0, 0, 0, 0, 0, 0, 3466, 0, 24, 0, 0, 0, 0, 0, 2958, 0, 4, 0, 0, 0, 0, 1291, 314, 0, 0, 0, 0, 0, 0, 0, 154, 1894, 4092, 0, 0, 0, 0, 6, 33, 1051, 23, 3845, 468, 0, 0, 4040, 344, 3486, 0, 1435, 0, 0, 0, 15, 0, 74, 0, 0, 110, 0, 0] rerouted_tokens_per_ep_rank=[3641, 3490, 2962, 1605, 6140, 5426, 9305, 199] avg=4096.0000 max=9305 max_rank=6 min=199 min_rank=7 imbalance_ratio=2.271729 max_load_reduction=23463 max_load_reduction_pct=0.716034 moved_to_echo=21070 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[138, 1, 6, 4096, 0, 0, 0, 0, 1230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 15, 531, 0, 0, 0, 268, 124, 3172, 1, 891, 0, 0, 0, 226, 3932, 56, 1423, 0, 0, 0, 0, 0, 0, 60, 3602, 1, 13, 0, 0, 3, 3738, 13, 0, 41, 0, 0, 0, 3626, 0, 0, 262, 1417, 3, 0, 0] rerouted_tokens_per_ep_rank=[4241, 1230, 4425, 4456, 5637, 3676, 3795, 5308] avg=4096.0000 max=5637 max_rank=4 min=1230 min_rank=1 imbalance_ratio=1.376221 max_load_reduction=27131 max_load_reduction_pct=0.827972 moved_to_echo=18416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 208, 2599, 0, 0, 0, 0, 0, 1203, 1, 1288, 0, 0, 0, 0, 0, 0, 204, 0, 3402, 520, 0, 0, 0, 4, 2771, 337, 64, 873, 0, 0, 0, 0, 33, 1934, 858, 0, 0, 0, 0, 0, 0, 95, 0, 3791, 478, 0, 0, 1342, 23, 4084, 320, 1466, 0, 0, 0, 3349, 18, 0, 3, 1387, 113, 0, 0] rerouted_tokens_per_ep_rank=[2807, 2492, 4126, 4049, 2825, 4364, 7235, 4870] avg=4096.0000 max=7235 max_rank=6 min=2492 min_rank=1 imbalance_ratio=1.766357 max_load_reduction=25533 max_load_reduction_pct=0.779205 moved_to_echo=19294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3809, 0, 1663, 0, 0, 0, 0, 0, 1236, 13, 11, 35, 0, 0, 0, 0, 0, 8, 3510, 45, 534, 0, 0, 0, 293, 3361, 0, 55, 897, 0, 0, 0, 0, 0, 1705, 0, 0, 0, 0, 0, 9, 4, 5, 0, 0, 422, 0, 0, 17, 0, 145, 0, 1293, 0, 0, 0, 4095, 0, 4096, 3982, 1426, 99, 0, 0] rerouted_tokens_per_ep_rank=[5472, 1295, 4097, 4606, 1705, 440, 1455, 13698] avg=4096.0000 max=13698 max_rank=7 min=440 min_rank=5 imbalance_ratio=3.344238 max_load_reduction=19070 max_load_reduction_pct=0.581970 moved_to_echo=17298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3843, 0, 2991, 22, 0, 0, 0, 0, 1238, 3833, 82, 4037, 0, 0, 0, 0, 0, 0, 3998, 92, 534, 0, 0, 0, 0, 616, 4065, 0, 897, 0, 0, 0, 67, 0, 1979, 0, 0, 0, 0, 0, 203, 0, 0, 0, 0, 489, 0, 0, 1, 6, 590, 0, 1500, 0, 0, 0, 4, 83, 0, 57, 1426, 115, 0, 0] rerouted_tokens_per_ep_rank=[6856, 9190, 4624, 5578, 2046, 692, 2097, 1685] avg=4096.0000 max=9190 max_rank=1 min=692 min_rank=5 imbalance_ratio=2.243652 max_load_reduction=23578 max_load_reduction_pct=0.719543 moved_to_echo=6520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4092, 207, 1, 1852, 0, 0, 0, 0, 3, 2594, 15, 4, 0, 0, 0, 0, 2, 0, 1, 4083, 5, 0, 0, 0, 0, 0, 4095, 225, 8, 0, 0, 0, 0, 1, 1906, 250, 0, 0, 0, 0, 3162, 0, 244, 0, 3844, 477, 0, 0, 3340, 0, 11, 97, 1455, 0, 0, 0, 620, 0, 2, 44, 12, 116, 0, 0] rerouted_tokens_per_ep_rank=[6152, 2616, 4091, 4328, 2157, 7727, 4903, 794] avg=4096.0000 max=7727 max_rank=5 min=794 min_rank=7 imbalance_ratio=1.886475 max_load_reduction=25041 max_load_reduction_pct=0.764191 moved_to_echo=15581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 187, 8, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 68, 0, 34, 4062, 0, 0, 0, 0, 5, 0, 4092, 0, 0, 0, 0, 0, 1, 0, 1984, 4091, 0, 0, 0, 0, 1299, 1208, 193, 25, 3403, 491, 0, 0, 0, 0, 109, 0, 1505, 0, 0, 0, 2975, 0, 2936, 63, 0, 116, 0, 0] rerouted_tokens_per_ep_rank=[195, 3913, 4164, 4097, 6076, 6619, 1614, 6090] avg=4096.0000 max=6619 max_rank=5 min=195 min_rank=0 imbalance_ratio=1.615967 max_load_reduction=26149 max_load_reduction_pct=0.798004 moved_to_echo=20399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 181, 3139, 84, 0, 0, 0, 0, 0, 452, 3994, 4096, 0, 0, 0, 0, 5, 0, 178, 4096, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 16, 175, 1963, 4028, 0, 0, 0, 0, 44, 0, 0, 0, 3294, 486, 0, 0, 874, 0, 1, 0, 1489, 0, 0, 0, 0, 8, 1, 0, 0, 115, 0, 0] rerouted_tokens_per_ep_rank=[3404, 8542, 4279, 4049, 6182, 3824, 2364, 124] avg=4096.0000 max=8542 max_rank=1 min=124 min_rank=7 imbalance_ratio=2.085449 max_load_reduction=24226 max_load_reduction_pct=0.739319 moved_to_echo=12494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14720, 128, 7168, 4480, 5760, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4992, 14336, 5504, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9728, 4224, 8448, 512, 10240, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4736, 1280, 1664, 3712, 18304, 3328, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 4352, 13440, 14848, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14464, 1024, 11392, 6144, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1920, 7296, 19840, 384, 3584, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 3200, 11648, 15872, 2176, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 82, 0, 0, 0, 0, 0, 2, 4096, 4095, 2911, 0, 0, 2612, 0, 4074, 13, 1, 4095, 0, 0, 0, 0, 0, 22, 0, 3391, 4086, 0, 2965, 323] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 3475, 4086, 2, 87, 27, 0, 0, 0, 4091, 4096, 0, 1, 0, 0, 1, 3905, 2, 8, 4092, 224, 3, 0, 0, 0, 252, 0, 3120, 4096, 0, 0, 1200] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 21, 3584, 0, 0, 0, 0, 4063, 0, 4095, 3318, 3862, 0, 0, 42, 0, 4096, 0, 855, 2115, 1213, 10, 807, 0, 0, 556, 7, 62, 4040, 22, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[228, 2941, 3621, 4090, 1, 1, 5, 0, 0, 4096, 3875, 0, 275, 19, 0, 0, 4096, 4094, 164, 4083, 0, 0, 1, 0, 0, 0, 0, 0, 601, 1, 0, 576] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3960, 11, 94, 0, 0, 1010, 7, 4088, 4096, 0, 0, 0, 3507, 0, 4096, 2900, 0, 4096, 0, 0, 133, 338, 0, 835, 0, 3597, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 3175, 0, 3, 0, 0, 2341, 4095, 3585, 216, 3359, 19, 0, 3991, 1429, 0, 0, 0, 3, 0, 0, 43, 1910, 71, 32, 329, 15, 4, 4053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 217, 3918, 0, 805, 0, 1, 1795, 1, 4041, 389, 92, 4002, 1708, 3, 0, 3170, 928, 15, 4086, 0, 0, 3, 4087, 86, 0, 0, 3358, 0, 62, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 1105, 49, 0, 0, 0, 464, 16, 4093, 116, 3912, 0, 32, 2445, 0, 3708, 1514, 3327, 4082, 0, 9, 0, 11, 0, 3, 0, 3802, 2642, 0, 1436, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 681, 0, 0, 2, 0, 3127, 2911, 0, 0, 0, 0, 0, 0, 2612, 0, 1924, 0, 0, 0, 0, 13, 1, 2692, 0, 0, 0, 0, 0, 0, 0, 0, 3393, 0, 0, 0, 0, 22, 0, 3391, 1403, 357, 51, 0, 4086, 0, 2965, 323, 611, 9, 0, 0] rerouted_tokens_per_ep_rank=[73, 2802, 6040, 4536, 2706, 3393, 5224, 7994] avg=4096.0000 max=7994 max_rank=7 min=73 min_rank=0 imbalance_ratio=1.951660 max_load_reduction=24774 max_load_reduction_pct=0.756042 moved_to_echo=19317 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3475, 3632, 2, 0, 0, 0, 0, 87, 27, 0, 0, 2119, 653, 0, 0, 0, 0, 3128, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1921, 0, 0, 0, 0, 2, 8, 2690, 0, 0, 0, 0, 224, 3, 0, 0, 3252, 0, 0, 0, 0, 252, 0, 3120, 1402, 357, 51, 0, 4096, 0, 0, 1200, 611, 454, 0, 0] rerouted_tokens_per_ep_rank=[7109, 2886, 3128, 1923, 2700, 3479, 5182, 6361] avg=4096.0000 max=7109 max_rank=0 min=1923 min_rank=3 imbalance_ratio=1.735596 max_load_reduction=25659 max_load_reduction_pct=0.783051 moved_to_echo=17722 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 21, 3186, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 2121, 685, 0, 0, 0, 0, 2534, 3862, 0, 0, 0, 0, 0, 0, 42, 0, 1922, 0, 0, 0, 0, 0, 855, 1391, 0, 0, 0, 0, 1213, 10, 807, 0, 3411, 0, 0, 0, 0, 556, 7, 62, 724, 289, 52, 0, 4040, 22, 0, 0, 495, 398, 0, 0] rerouted_tokens_per_ep_rank=[3207, 6869, 6396, 1964, 2246, 5441, 1690, 4955] avg=4096.0000 max=6869 max_rank=1 min=1690 min_rank=6 imbalance_ratio=1.677002 max_load_reduction=25899 max_load_reduction_pct=0.790375 moved_to_echo=14332 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3640, 0, 0, 0, 0, 0, 3175, 0, 3, 0, 1213, 668, 0, 0, 0, 0, 3127, 3585, 0, 0, 0, 0, 216, 3359, 19, 0, 1099, 0, 0, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3323, 0, 0, 0, 43, 1910, 71, 32, 0, 357, 29, 0, 329, 15, 4, 4053, 611, 455, 0, 0] rerouted_tokens_per_ep_rank=[3640, 5059, 6712, 4693, 1429, 3326, 2442, 5467] avg=4096.0000 max=6712 max_rank=2 min=1429 min_rank=4 imbalance_ratio=1.638672 max_load_reduction=26056 max_load_reduction_pct=0.795166 moved_to_echo=12664 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3516, 11, 0, 0, 0, 0, 94, 0, 0, 1010, 2118, 686, 0, 0, 7, 0, 3118, 0, 0, 0, 0, 0, 0, 0, 3507, 0, 1919, 0, 0, 0, 0, 2900, 0, 2690, 0, 0, 0, 0, 0, 0, 133, 338, 3410, 0, 0, 0, 0, 835, 0, 3597, 1406, 362, 51, 0, 0, 0, 0, 0, 616, 444, 0, 0] rerouted_tokens_per_ep_rank=[3527, 3908, 3125, 5426, 5590, 3881, 6251, 1060] avg=4096.0000 max=6251 max_rank=6 min=1060 min_rank=7 imbalance_ratio=1.526123 max_load_reduction=26517 max_load_reduction_pct=0.809235 moved_to_echo=16782 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[228, 2941, 3219, 4090, 0, 0, 0, 0, 1, 1, 5, 0, 2122, 686, 0, 0, 0, 0, 2959, 0, 0, 0, 0, 0, 275, 19, 0, 0, 1923, 0, 0, 0, 0, 4094, 164, 2685, 0, 0, 0, 0, 0, 0, 1, 0, 3410, 0, 0, 0, 0, 0, 0, 0, 1398, 338, 51, 0, 601, 1, 0, 576, 578, 402, 0, 0] rerouted_tokens_per_ep_rank=[10478, 2815, 2959, 2217, 6943, 3411, 1787, 2158] avg=4096.0000 max=10478 max_rank=0 min=1787 min_rank=6 imbalance_ratio=2.558105 max_load_reduction=22290 max_load_reduction_pct=0.680237 moved_to_echo=14299 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 983, 49, 0, 0, 0, 0, 0, 0, 0, 464, 2120, 621, 0, 0, 16, 0, 89, 3912, 0, 0, 0, 0, 0, 32, 2445, 0, 1922, 0, 0, 0, 0, 1514, 3327, 2684, 0, 0, 0, 0, 0, 9, 0, 11, 3087, 0, 0, 0, 0, 3, 0, 3802, 1398, 10, 51, 0, 2642, 0, 1436, 0, 17, 122, 0, 0] rerouted_tokens_per_ep_rank=[1034, 3205, 4017, 4399, 7525, 3107, 5264, 4217] avg=4096.0000 max=7525 max_rank=4 min=1034 min_rank=0 imbalance_ratio=1.837158 max_load_reduction=25243 max_load_reduction_pct=0.770355 moved_to_echo=20113 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 217, 3483, 0, 0, 0, 0, 0, 805, 0, 1, 1795, 2093, 530, 0, 0, 1, 0, 298, 92, 0, 0, 0, 0, 4002, 1708, 3, 0, 1897, 0, 0, 0, 0, 928, 15, 2686, 0, 0, 0, 0, 0, 0, 3, 4087, 2640, 0, 0, 0, 86, 0, 0, 3358, 1400, 33, 51, 0, 0, 62, 0, 1, 58, 435, 0, 0] rerouted_tokens_per_ep_rank=[3700, 5224, 391, 7610, 3629, 6730, 4928, 556] avg=4096.0000 max=7610 max_rank=3 min=391 min_rank=2 imbalance_ratio=1.857910 max_load_reduction=25158 max_load_reduction_pct=0.767761 moved_to_echo=15843 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 6656, 21760, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15872, 128, 4480, 6272, 3712, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 128, 128, 7424, 16128, 5248, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4608, 5120, 8704, 128, 14592, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 3584, 128, 17408, 9216, 2176, 512, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 18432, 14464, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 10880, 4480, 17536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1536, 128, 1024, 4480, 25984, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[16, 4039, 23, 56, 2, 0, 2472, 8, 0, 4092, 0, 114, 4096, 0, 139, 1531, 1244, 0, 0, 2390, 4019, 0, 5, 4, 11, 8, 0, 3643, 469, 0, 291, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[13, 4096, 0, 245, 1, 3064, 0, 0, 0, 4095, 1105, 0, 380, 0, 0, 82, 0, 2761, 0, 105, 3669, 0, 0, 326, 0, 0, 13, 2699, 2046, 0, 3972, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4095, 1229, 1, 0, 0, 0, 4067, 1, 0, 4094, 3724, 0, 1261, 2358, 18, 0, 0, 0, 63, 1484, 641, 0, 261, 27, 1, 0, 32, 1350, 4082, 0, 0, 3979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 3645, 0, 0, 131, 0, 3837, 1, 159, 5, 0, 2523, 0, 0, 4095, 1, 0, 1079, 2257, 0, 3512, 570, 13, 1994, 10, 4096, 0, 1929, 0, 2898, 13, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[261, 3797, 2, 0, 247, 2676, 116, 0, 414, 3751, 3, 0, 2, 0, 2, 0, 0, 1, 0, 3888, 4026, 0, 2872, 3177, 26, 0, 67, 3749, 3669, 22, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[89, 3862, 107, 130, 0, 0, 2554, 3598, 54, 3338, 71, 4, 4096, 36, 0, 0, 3886, 0, 0, 4, 0, 220, 9, 4046, 0, 0, 758, 2, 9, 1803, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[75, 4090, 690, 0, 17, 0, 0, 517, 70, 110, 2913, 0, 3977, 11, 8, 0, 3377, 0, 0, 3942, 0, 0, 0, 1756, 0, 4096, 0, 1, 3069, 4048, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4095, 3792, 0, 16, 0, 0, 4033, 0, 0, 3923, 3974, 0, 2, 0, 25, 3, 3, 1, 200, 89, 3977, 50, 475, 0, 0, 9, 0, 131, 4095, 6, 3604, 265] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4095, 985, 1, 0, 0, 0, 0, 0, 0, 0, 4067, 1, 93, 91, 133, 0, 0, 3086, 3724, 0, 0, 0, 0, 0, 1261, 2358, 18, 0, 2491, 0, 0, 0, 0, 0, 63, 1484, 153, 424, 0, 0, 548, 0, 261, 27, 0, 0, 0, 0, 1, 0, 32, 1350, 1591, 584, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5081, 4385, 6810, 6128, 2124, 836, 3558, 3846] avg=4096.0000 max=6810 max_rank=2 min=836 min_rank=5 imbalance_ratio=1.662598 max_load_reduction=25958 max_load_reduction_pct=0.792175 moved_to_echo=10364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[13, 3279, 0, 245, 0, 0, 0, 0, 1, 3064, 0, 0, 532, 305, 137, 0, 0, 3087, 1105, 0, 0, 0, 0, 0, 380, 0, 0, 82, 1249, 0, 0, 0, 0, 2761, 0, 105, 512, 424, 0, 0, 3137, 0, 0, 326, 0, 0, 0, 0, 0, 0, 13, 2699, 797, 584, 0, 0, 0, 0, 3972, 3959, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3537, 4039, 4192, 1711, 3802, 3463, 4093, 7931] avg=4096.0000 max=7931 max_rank=7 min=1711 min_rank=3 imbalance_ratio=1.936279 max_load_reduction=24837 max_load_reduction_pct=0.757965 moved_to_echo=19289 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[16, 3226, 23, 56, 0, 0, 0, 0, 2, 0, 2472, 8, 586, 305, 140, 0, 0, 3076, 0, 114, 0, 0, 0, 0, 4096, 0, 139, 1531, 287, 0, 0, 0, 1244, 0, 0, 2390, 508, 427, 0, 0, 3433, 0, 5, 4, 0, 0, 0, 0, 11, 8, 0, 3643, 182, 589, 0, 0, 0, 0, 291, 3956, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3321, 3513, 3190, 6053, 4569, 3442, 4433, 4247] avg=4096.0000 max=6053 max_rank=3 min=3190 min_rank=2 imbalance_ratio=1.477783 max_load_reduction=26715 max_load_reduction_pct=0.815277 moved_to_echo=16691 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[261, 3041, 2, 0, 0, 0, 0, 0, 247, 2676, 116, 0, 584, 282, 0, 0, 414, 2828, 3, 0, 0, 0, 0, 0, 2, 0, 2, 0, 2240, 0, 0, 0, 0, 1, 0, 3888, 474, 388, 0, 0, 3442, 0, 2872, 3177, 0, 0, 0, 0, 26, 0, 67, 3749, 1429, 535, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3304, 3905, 3245, 2244, 4751, 9491, 5806, 22] avg=4096.0000 max=9491 max_rank=5 min=22 min_rank=7 imbalance_ratio=2.317139 max_load_reduction=23277 max_load_reduction_pct=0.710358 moved_to_echo=20070 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[75, 3275, 690, 0, 0, 0, 0, 0, 17, 0, 0, 517, 0, 304, 0, 0, 70, 84, 2913, 0, 0, 0, 0, 0, 3977, 11, 8, 0, 1874, 0, 0, 0, 3377, 0, 0, 3942, 511, 11, 0, 0, 0, 0, 0, 1756, 0, 0, 0, 0, 0, 4096, 0, 1, 1195, 15, 0, 0, 0, 4048, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 838, 3067, 5870, 7841, 1756, 5307, 4049] avg=4096.0000 max=7841 max_rank=4 min=838 min_rank=1 imbalance_ratio=1.914307 max_load_reduction=24927 max_load_reduction_pct=0.760712 moved_to_echo=18953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4095, 3036, 0, 16, 0, 0, 0, 0, 0, 0, 4033, 0, 577, 282, 8, 0, 0, 2957, 3974, 0, 0, 0, 0, 0, 2, 0, 25, 3, 2499, 0, 0, 0, 3, 1, 200, 89, 474, 406, 0, 0, 3400, 50, 475, 0, 0, 0, 0, 0, 0, 9, 0, 131, 1596, 560, 0, 0, 0, 6, 3604, 257, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7147, 4900, 6931, 2529, 1173, 3925, 2296, 3867] avg=4096.0000 max=7147 max_rank=0 min=1173 min_rank=4 imbalance_ratio=1.744873 max_load_reduction=25621 max_load_reduction_pct=0.781891 moved_to_echo=11261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 2919, 0, 0, 0, 0, 0, 0, 131, 0, 3837, 1, 509, 271, 0, 0, 159, 5, 0, 2523, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1079, 2257, 0, 455, 0, 0, 0, 3003, 570, 13, 1994, 0, 0, 0, 0, 10, 4096, 0, 1929, 0, 0, 0, 0, 0, 2898, 13, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2919, 4749, 2687, 4096, 3791, 5580, 6035, 2911] avg=4096.0000 max=6035 max_rank=6 min=2687 min_rank=2 imbalance_ratio=1.473389 max_load_reduction=26733 max_load_reduction_pct=0.815826 moved_to_echo=18317 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[89, 3093, 107, 130, 0, 0, 0, 0, 0, 0, 2554, 3598, 0, 287, 0, 0, 54, 2517, 71, 4, 0, 0, 0, 0, 4096, 36, 0, 0, 5, 0, 0, 0, 3886, 0, 0, 4, 482, 345, 0, 0, 0, 220, 9, 4046, 0, 0, 0, 0, 0, 0, 758, 2, 4, 476, 0, 0, 0, 1803, 4092, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3419, 6439, 2646, 4137, 4717, 4275, 1240, 5895] avg=4096.0000 max=6439 max_rank=1 min=1240 min_rank=6 imbalance_ratio=1.572021 max_load_reduction=26329 max_load_reduction_pct=0.803497 moved_to_echo=16127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8576, 3968, 2560, 11904, 3584, 2432, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17024, 896, 3712, 11392, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13824, 2432, 4352, 1664, 10752, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 5760, 17152, 4224, 2944, 2176, 512, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 8832, 12032, 12032, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 8320, 896, 13568, 6912, 3456, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8704, 22912, 896, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 17664, 11904, 2688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1822, 0, 0, 0, 2754, 3440, 588, 4082, 4096, 2494, 0, 1017, 0, 3651, 0, 3330, 0, 0, 37, 3765, 0, 302, 127, 106, 14, 0, 36, 0, 0, 0, 0, 1107] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 30, 13, 0, 3995, 4081, 0, 6, 2850, 0, 0, 70, 0, 3820, 27, 0, 3966, 2, 86, 4095, 111, 0, 4096, 0, 0, 0, 328, 2235, 2, 15, 0, 2940] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[51, 1, 6, 69, 3846, 2, 4095, 3257, 75, 0, 18, 617, 3836, 1, 1, 4047, 170, 0, 845, 2358, 0, 101, 0, 315, 49, 0, 0, 4071, 1039, 3876, 0, 22] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[26, 0, 0, 4079, 3766, 4096, 330, 2, 4096, 116, 0, 1839, 829, 1736, 0, 3897, 84, 0, 0, 4021, 0, 0, 0, 412, 0, 4, 63, 0, 0, 0, 17, 3355] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[922, 0, 2171, 0, 2870, 3971, 3555, 2153, 4050, 22, 28, 0, 0, 4079, 0, 0, 4060, 0, 0, 3996, 457, 0, 0, 0, 0, 12, 0, 0, 238, 31, 1, 152] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[6, 0, 2356, 7, 3219, 3, 2, 0, 3197, 0, 295, 0, 3986, 3795, 0, 0, 0, 0, 6, 4089, 4087, 0, 1517, 208, 1109, 641, 0, 0, 5, 143, 1, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 181, 1, 136, 3951, 3853, 108, 412, 4049, 0, 0, 2, 1, 247, 20, 35, 1581, 0, 1, 4076, 0, 3, 116, 4096, 0, 36, 15, 3986, 0, 0, 3984, 1878] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[2, 1, 588, 4013, 3444, 0, 0, 4087, 3954, 0, 22, 0, 253, 1906, 9, 4093, 3927, 100, 8, 3900, 1729, 0, 0, 11, 133, 0, 0, 503, 80, 0, 4, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[922, 0, 2171, 0, 856, 1051, 0, 0, 0, 2061, 3555, 2153, 0, 0, 0, 0, 4029, 22, 28, 0, 0, 0, 0, 0, 0, 1784, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 2361, 0, 0, 0, 0, 457, 0, 0, 0, 2235, 584, 0, 0, 0, 12, 0, 0, 2014, 0, 0, 0, 238, 31, 1, 152, 1910, 60, 21, 0] rerouted_tokens_per_ep_rank=[5000, 7769, 4079, 1784, 6421, 3276, 2026, 2413] avg=4096.0000 max=7769 max_rank=1 min=1784 min_rank=3 imbalance_ratio=1.896729 max_load_reduction=24999 max_load_reduction_pct=0.762909 moved_to_echo=14136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[51, 1, 6, 69, 1149, 620, 0, 0, 0, 2, 4095, 3257, 0, 0, 0, 0, 75, 0, 18, 617, 0, 0, 0, 0, 3836, 1, 1, 4047, 0, 0, 0, 0, 170, 0, 845, 1394, 0, 0, 0, 0, 0, 101, 0, 315, 0, 344, 0, 0, 49, 0, 0, 4071, 2697, 0, 0, 0, 1039, 3876, 0, 22, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1896, 7354, 710, 7885, 2409, 760, 6817, 4937] avg=4096.0000 max=7885 max_rank=3 min=710 min_rank=2 imbalance_ratio=1.925049 max_load_reduction=24883 max_load_reduction_pct=0.759369 moved_to_echo=14923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[26, 0, 0, 4079, 1124, 1058, 0, 0, 0, 2126, 330, 2, 0, 0, 0, 0, 4075, 116, 0, 1839, 0, 0, 0, 0, 829, 760, 0, 3897, 0, 0, 0, 0, 84, 0, 0, 2376, 0, 0, 0, 0, 0, 0, 0, 412, 951, 587, 0, 0, 0, 4, 63, 0, 2642, 0, 0, 0, 0, 0, 17, 3355, 1970, 25, 21, 0] rerouted_tokens_per_ep_rank=[6287, 2458, 6030, 5486, 2460, 1950, 2709, 5388] avg=4096.0000 max=6287 max_rank=0 min=1950 min_rank=5 imbalance_ratio=1.534912 max_load_reduction=26481 max_load_reduction_pct=0.808136 moved_to_echo=12507 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 2356, 7, 960, 1076, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 3181, 0, 295, 0, 0, 0, 0, 0, 3986, 1659, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2416, 0, 0, 0, 0, 4087, 0, 1517, 208, 2080, 597, 0, 0, 1109, 641, 0, 0, 2259, 0, 0, 0, 5, 143, 1, 4096, 1, 56, 16, 0] rerouted_tokens_per_ep_rank=[4405, 4, 3476, 5645, 2422, 8489, 4009, 4318] avg=4096.0000 max=8489 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.072510 max_load_reduction=24279 max_load_reduction_pct=0.740936 moved_to_echo=19238 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 588, 4013, 1028, 1026, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 3934, 0, 22, 0, 0, 0, 0, 0, 253, 834, 9, 4093, 0, 0, 0, 0, 3927, 100, 8, 2304, 0, 0, 0, 0, 1729, 0, 0, 11, 1044, 570, 0, 0, 133, 0, 0, 503, 2416, 0, 0, 0, 80, 0, 4, 1, 0, 28, 20, 0] rerouted_tokens_per_ep_rank=[6658, 4087, 3956, 5189, 6339, 3354, 3052, 133] avg=4096.0000 max=6658 max_rank=0 min=133 min_rank=7 imbalance_ratio=1.625488 max_load_reduction=26110 max_load_reduction_pct=0.796814 moved_to_echo=12878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1822, 0, 0, 0, 823, 990, 0, 0, 0, 1786, 588, 4082, 0, 0, 0, 0, 4075, 2494, 0, 1017, 0, 0, 0, 0, 0, 1596, 0, 3330, 0, 0, 0, 0, 0, 0, 37, 2225, 0, 0, 0, 0, 0, 302, 127, 106, 2001, 550, 0, 0, 14, 0, 36, 0, 1931, 0, 0, 0, 0, 0, 0, 1107, 1654, 54, 21, 0] rerouted_tokens_per_ep_rank=[3635, 6456, 7586, 4926, 2262, 3086, 1981, 2836] avg=4096.0000 max=7586 max_rank=2 min=1981 min_rank=6 imbalance_ratio=1.852051 max_load_reduction=25182 max_load_reduction_pct=0.768494 moved_to_echo=10165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 13, 0, 1193, 1082, 0, 0, 0, 2114, 0, 6, 0, 0, 0, 0, 2831, 0, 0, 70, 0, 0, 0, 0, 0, 1661, 27, 0, 0, 0, 0, 0, 3966, 2, 86, 2410, 0, 0, 0, 0, 111, 0, 4096, 0, 2098, 603, 0, 0, 0, 0, 328, 2235, 2802, 0, 0, 0, 2, 15, 0, 2940, 1967, 61, 19, 0] rerouted_tokens_per_ep_rank=[2318, 2120, 2901, 1688, 6464, 6908, 5365, 5004] avg=4096.0000 max=6908 max_rank=5 min=1688 min_rank=3 imbalance_ratio=1.686523 max_load_reduction=25860 max_load_reduction_pct=0.789185 moved_to_echo=23741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 181, 1, 136, 1179, 1072, 0, 0, 0, 2000, 108, 412, 0, 0, 0, 0, 4028, 0, 0, 2, 0, 0, 0, 0, 1, 109, 20, 35, 0, 0, 0, 0, 1581, 0, 1, 2409, 0, 0, 0, 0, 0, 3, 116, 4096, 135, 595, 0, 0, 0, 36, 15, 3986, 2772, 0, 0, 0, 0, 0, 3984, 1878, 1853, 3, 21, 0] rerouted_tokens_per_ep_rank=[2569, 2520, 4030, 165, 3991, 4945, 6809, 7739] avg=4096.0000 max=7739 max_rank=7 min=165 min_rank=3 imbalance_ratio=1.889404 max_load_reduction=25029 max_load_reduction_pct=0.763824 moved_to_echo=23484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2944, 256, 5248, 8320, 8320, 8064, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6400, 512, 5888, 5248, 10624, 4480, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 10112, 8704, 14080, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8960, 8448, 128, 15488, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[13824, 128, 1024, 17920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1408, 768, 512, 10880, 19584, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1408, 4096, 4096, 13568, 9472, 384, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[26240, 2688, 384, 3584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4064, 4074, 1, 234, 0, 0, 0, 0, 1, 539, 1, 24, 1, 3494, 1652, 0, 4086, 24, 0, 28, 3733, 295, 0, 0, 0, 4096, 0, 4096, 0, 2325, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[156, 3952, 90, 0, 4096, 0, 0, 0, 4, 0, 2, 217, 1, 1498, 4, 2021, 187, 2979, 0, 2394, 1640, 4096, 0, 0, 1331, 0, 3758, 0, 4087, 0, 255, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 3821, 4002, 115, 0, 11, 10, 2549, 0, 42, 0, 0, 4082, 484, 188, 531, 77, 329, 0, 133, 3264, 3806, 0, 0, 576, 0, 3996, 7, 3460, 289, 5, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[2219, 4086, 217, 0, 0, 2034, 21, 14, 0, 4096, 3, 0, 0, 0, 0, 2084, 0, 3992, 1692, 0, 4093, 0, 2, 0, 51, 0, 4065, 3, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 12, 40, 0, 0, 4062, 27, 0, 1907, 0, 0, 0, 4096, 0, 0, 111, 0, 97, 0, 521, 4004, 5, 0, 12, 1878, 4091, 16, 3896, 1, 3899, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 2473, 3699, 78, 0, 0, 0, 0, 175, 29, 0, 36, 0, 153, 1875, 4020, 43, 4071, 0, 28, 4096, 4096, 0, 0, 0, 0, 3590, 0, 4086, 15, 202, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 4096, 3, 0, 10, 1, 0, 0, 2, 0, 0, 4091, 14, 10, 473, 0, 0, 4081, 0, 1860, 363, 1127, 3750, 0, 4096, 0, 4096, 604] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 3964, 3693, 8, 378, 2, 0, 4094, 0, 354, 1026, 0, 4086, 1115, 0, 1980, 434, 2967, 0, 709, 0, 4065, 2, 0, 232, 24, 1309, 4, 2322, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[156, 1859, 90, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1134, 410, 0, 0, 4, 0, 2, 217, 2093, 1200, 0, 0, 1, 1498, 4, 1440, 0, 0, 0, 0, 187, 2979, 0, 2394, 581, 229, 198, 0, 1640, 2486, 0, 0, 0, 0, 0, 0, 1331, 0, 3560, 0, 0, 0, 0, 0, 2724, 0, 255, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2105, 5640, 3516, 2943, 6568, 4126, 4891, 2979] avg=4096.0000 max=6568 max_rank=4 min=2105 min_rank=0 imbalance_ratio=1.603516 max_load_reduction=26200 max_load_reduction_pct=0.799561 moved_to_echo=18564 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1797, 4002, 115, 0, 0, 0, 0, 0, 11, 10, 2549, 960, 378, 0, 0, 0, 42, 0, 0, 2024, 1111, 0, 0, 4082, 484, 188, 379, 0, 0, 0, 0, 77, 329, 0, 133, 152, 194, 210, 0, 3264, 2317, 0, 0, 0, 0, 0, 0, 576, 0, 3786, 7, 0, 0, 0, 0, 2306, 289, 5, 991, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5914, 3908, 3177, 5133, 1095, 5581, 4369, 3591] avg=4096.0000 max=5914 max_rank=0 min=1095 min_rank=4 imbalance_ratio=1.443848 max_load_reduction=26854 max_load_reduction_pct=0.819519 moved_to_echo=14636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1912, 4074, 1, 0, 0, 0, 0, 234, 0, 0, 0, 1136, 371, 0, 0, 0, 1, 539, 1, 2152, 1090, 0, 0, 24, 1, 3494, 1177, 0, 0, 0, 0, 0, 4086, 24, 0, 475, 230, 215, 0, 28, 2272, 295, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 0, 0, 0, 2730, 0, 2325, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5987, 1741, 3783, 4696, 5030, 2595, 3881, 5055] avg=4096.0000 max=5987 max_rank=0 min=1741 min_rank=1 imbalance_ratio=1.461670 max_load_reduction=26781 max_load_reduction_pct=0.817291 moved_to_echo=16561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1924, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 1136, 406, 0, 0, 10, 1, 0, 0, 2167, 1192, 0, 0, 2, 0, 0, 2914, 0, 0, 0, 0, 14, 10, 473, 0, 1177, 230, 197, 0, 0, 2483, 0, 1860, 0, 0, 0, 0, 363, 1127, 3553, 0, 0, 0, 0, 0, 2730, 0, 4096, 604, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1924, 5641, 3370, 2916, 2101, 4343, 5043, 7430] avg=4096.0000 max=7430 max_rank=7 min=1924 min_rank=0 imbalance_ratio=1.813965 max_load_reduction=25338 max_load_reduction_pct=0.773254 moved_to_echo=18917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2219, 1919, 217, 0, 0, 0, 0, 0, 0, 2034, 21, 14, 1139, 0, 0, 0, 0, 4096, 3, 0, 2167, 0, 0, 0, 0, 0, 0, 1480, 0, 0, 0, 0, 0, 3992, 1692, 0, 604, 233, 218, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 51, 0, 3847, 3, 0, 0, 0, 0, 2724, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4355, 3208, 6266, 1480, 6739, 4095, 3901, 2724] avg=4096.0000 max=6739 max_rank=4 min=1480 min_rank=3 imbalance_ratio=1.645264 max_load_reduction=26029 max_load_reduction_pct=0.794342 moved_to_echo=17459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1865, 3693, 8, 0, 0, 0, 0, 378, 2, 0, 4094, 644, 404, 0, 0, 0, 354, 1026, 0, 2099, 1187, 0, 0, 4086, 1115, 0, 1411, 0, 0, 0, 0, 434, 2967, 0, 709, 569, 130, 69, 0, 0, 2474, 2, 0, 0, 0, 0, 0, 232, 24, 1240, 4, 0, 0, 0, 0, 1548, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5566, 5522, 4666, 6612, 4878, 2476, 1500, 1548] avg=4096.0000 max=6612 max_rank=3 min=1500 min_rank=6 imbalance_ratio=1.614258 max_load_reduction=26156 max_load_reduction_pct=0.798218 moved_to_echo=10402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1925, 12, 40, 0, 0, 0, 0, 0, 0, 4062, 27, 1081, 398, 0, 0, 0, 1907, 0, 0, 2168, 1169, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 111, 0, 97, 0, 0, 218, 215, 0, 521, 2437, 5, 0, 0, 0, 0, 0, 12, 1878, 3876, 16, 0, 0, 0, 0, 2597, 1, 3899, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1977, 5568, 5244, 4096, 641, 2963, 5782, 6497] avg=4096.0000 max=6497 max_rank=7 min=641 min_rank=4 imbalance_ratio=1.586182 max_load_reduction=26271 max_load_reduction_pct=0.801727 moved_to_echo=15883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1163, 3699, 78, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 407, 0, 0, 175, 29, 0, 36, 1310, 1196, 0, 0, 0, 153, 1875, 2864, 0, 0, 0, 0, 43, 4071, 0, 28, 1156, 229, 189, 0, 4096, 2493, 0, 0, 0, 0, 0, 0, 0, 0, 3401, 0, 0, 0, 0, 0, 2724, 15, 202, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4940, 1540, 2746, 4892, 5716, 6589, 3401, 2944] avg=4096.0000 max=6589 max_rank=5 min=1540 min_rank=1 imbalance_ratio=1.608643 max_load_reduction=26179 max_load_reduction_pct=0.798920 moved_to_echo=18650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 6528, 1664, 256, 16256, 8192, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4736, 6144, 4096, 6784, 8448, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 18560, 2304, 3328, 4736, 1792, 1536, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2432, 14464, 15872, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13696, 17024, 384, 1920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[20096, 384, 10880, 1664, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 7424, 5632, 11776, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2688, 3072, 27264, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True + [2026-08-30 04:56:44] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 878.8 | throughput per GPU (TFLOP/s/GPU): 379.4 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101619E+01 | loss scale: 1.0 | grad norm: 45.748 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[102, 3158, 848, 170, 1783, 48, 977, 541, 1374, 234, 1885, 559, 1224, 1050, 167, 524, 269, 548, 1686, 1481, 2214, 351, 2166, 1341, 112, 1246, 635, 1454, 1113, 469, 1805, 1234] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[60, 2976, 680, 180, 2548, 93, 641, 455, 1475, 81, 1743, 870, 727, 906, 231, 463, 493, 776, 2084, 2322, 1735, 192, 3162, 1086, 88, 1107, 411, 1113, 869, 328, 1635, 1238] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[60, 3130, 1195, 182, 2024, 175, 848, 663, 1134, 133, 1383, 432, 1209, 753, 124, 466, 317, 642, 1571, 1954, 2212, 192, 2779, 1119, 78, 1168, 992, 1411, 1272, 462, 1335, 1353] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[37, 2963, 964, 137, 2318, 116, 1113, 427, 1063, 88, 1740, 624, 941, 1109, 156, 571, 223, 1071, 997, 1736, 1841, 471, 2911, 1138, 86, 1869, 1095, 1317, 827, 394, 967, 1458] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[166, 2995, 585, 114, 2648, 184, 984, 454, 1302, 73, 1945, 577, 820, 850, 208, 502, 225, 840, 1263, 2157, 2019, 376, 2985, 1343, 32, 1519, 612, 1056, 650, 271, 1656, 1357] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[188, 2854, 1185, 189, 1829, 60, 916, 495, 1281, 81, 2001, 864, 922, 765, 144, 455, 261, 634, 1502, 1900, 2228, 242, 2679, 1109, 96, 1338, 703, 1479, 976, 431, 1714, 1247] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[69, 3073, 1230, 208, 2306, 75, 888, 500, 1453, 134, 1804, 501, 574, 801, 124, 424, 229, 601, 1073, 2462, 1747, 455, 2549, 1350, 119, 1404, 598, 1152, 1234, 291, 1972, 1368] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[60, 2843, 680, 180, 0, 0, 0, 0, 2548, 93, 641, 455, 256, 152, 0, 0, 1475, 81, 1743, 870, 133, 46, 0, 0, 727, 906, 231, 463, 1906, 0, 0, 0, 493, 776, 2084, 1847, 0, 0, 0, 0, 1735, 192, 702, 1086, 0, 0, 0, 0, 88, 1107, 411, 1113, 554, 219, 0, 0, 869, 328, 1437, 1238, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3763, 4145, 4348, 4233, 5200, 3715, 3492, 3872] avg=4096.0000 max=5200 max_rank=4 min=3492 min_rank=6 imbalance_ratio=1.269531 max_load_reduction=27568 max_load_reduction_pct=0.841309 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[102, 3017, 848, 170, 0, 0, 0, 0, 1783, 48, 977, 541, 163, 168, 0, 0, 1374, 234, 1885, 559, 141, 50, 0, 0, 1224, 1050, 167, 524, 1305, 0, 0, 0, 269, 548, 1686, 1178, 0, 0, 0, 0, 2214, 351, 482, 1341, 0, 0, 0, 0, 112, 1246, 635, 1454, 379, 140, 0, 0, 1113, 469, 1587, 1234, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 3680, 4243, 4270, 3681, 4388, 3966, 4403] avg=4096.0000 max=4403 max_rank=7 min=3680 min_rank=1 imbalance_ratio=1.074951 max_load_reduction=28365 max_load_reduction_pct=0.865631 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[37, 2831, 964, 137, 0, 0, 0, 0, 2318, 116, 1113, 427, 192, 90, 0, 0, 1063, 88, 1740, 624, 132, 27, 0, 0, 941, 1109, 156, 571, 1754, 0, 0, 0, 223, 1071, 997, 1380, 0, 0, 0, 0, 1841, 471, 647, 1138, 0, 0, 0, 0, 86, 1869, 1095, 1317, 510, 164, 0, 0, 827, 394, 850, 1458, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3969, 4256, 3674, 4531, 3671, 4097, 5041, 3529] avg=4096.0000 max=5041 max_rank=6 min=3529 min_rank=7 imbalance_ratio=1.230713 max_load_reduction=27727 max_load_reduction_pct=0.846161 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[166, 2861, 585, 114, 0, 0, 0, 0, 2648, 184, 984, 454, 238, 154, 0, 0, 1302, 73, 1945, 577, 134, 46, 0, 0, 820, 850, 208, 502, 1799, 0, 0, 0, 225, 840, 1263, 1715, 0, 0, 0, 0, 2019, 376, 663, 1343, 0, 0, 0, 0, 32, 1519, 612, 1056, 523, 204, 0, 0, 650, 271, 1456, 1357, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3726, 4662, 4077, 4179, 4043, 4401, 3946, 3734] avg=4096.0000 max=4662 max_rank=1 min=3726 min_rank=0 imbalance_ratio=1.138184 max_load_reduction=28106 max_load_reduction_pct=0.857727 moved_to_echo=16124 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[60, 2990, 1195, 182, 0, 0, 0, 0, 2024, 175, 848, 663, 216, 124, 0, 0, 1134, 133, 1383, 432, 140, 37, 0, 0, 1209, 753, 124, 466, 1675, 0, 0, 0, 317, 642, 1571, 1553, 0, 0, 0, 0, 2212, 192, 617, 1119, 0, 0, 0, 0, 78, 1168, 992, 1411, 487, 185, 0, 0, 1272, 462, 1174, 1353, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4427, 4050, 3259, 4227, 4083, 4140, 4321, 4261] avg=4096.0000 max=4427 max_rank=0 min=3259 min_rank=2 imbalance_ratio=1.080811 max_load_reduction=28341 max_load_reduction_pct=0.864899 moved_to_echo=16805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[188, 2721, 1185, 189, 0, 0, 0, 0, 1829, 60, 916, 495, 214, 164, 0, 0, 1281, 81, 2001, 864, 133, 52, 0, 0, 922, 765, 144, 455, 1618, 0, 0, 0, 261, 634, 1502, 1504, 0, 0, 0, 0, 2228, 242, 587, 1109, 0, 0, 0, 0, 96, 1338, 703, 1479, 474, 182, 0, 0, 976, 431, 1498, 1247, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4283, 3678, 4412, 3904, 3901, 4166, 4272, 4152] avg=4096.0000 max=4412 max_rank=2 min=3678 min_rank=1 imbalance_ratio=1.077148 max_load_reduction=28356 max_load_reduction_pct=0.865356 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[69, 2936, 1230, 208, 0, 0, 0, 0, 2306, 75, 888, 500, 272, 184, 0, 0, 1453, 134, 1804, 501, 137, 55, 0, 0, 574, 801, 124, 424, 1536, 0, 0, 0, 229, 601, 1073, 1957, 0, 0, 0, 0, 1747, 455, 566, 1350, 0, 0, 0, 0, 119, 1404, 598, 1152, 447, 233, 0, 0, 1234, 291, 1733, 1368, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4443, 4225, 4084, 3459, 3860, 4118, 3953, 4626] avg=4096.0000 max=4626 max_rank=7 min=3459 min_rank=3 imbalance_ratio=1.129395 max_load_reduction=28142 max_load_reduction_pct=0.858826 moved_to_echo=16557 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[51, 2804, 1107, 183, 2035, 112, 878, 633, 1324, 114, 2357, 700, 528, 1025, 174, 627, 273, 928, 1365, 2218, 1909, 247, 2673, 993, 23, 1269, 379, 1428, 985, 195, 1813, 1418] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[51, 2679, 1107, 183, 0, 0, 0, 0, 2035, 112, 878, 633, 245, 169, 0, 0, 1324, 114, 2357, 700, 125, 51, 0, 0, 528, 1025, 174, 627, 1611, 0, 0, 0, 273, 928, 1365, 1763, 0, 0, 0, 0, 1909, 247, 594, 993, 0, 0, 0, 0, 23, 1269, 379, 1428, 468, 210, 0, 0, 985, 195, 1593, 1418, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4072, 4671, 3965, 4329, 3743, 3777, 4191] avg=4096.0000 max=4671 max_rank=2 min=3743 min_rank=5 imbalance_ratio=1.140381 max_load_reduction=28097 max_load_reduction_pct=0.857452 moved_to_echo=16040 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[17536, 896, 7296, 4224, 1920, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10496, 1024, 14976, 5248, 1152, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 11008, 5504, 10496, 3968, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7040, 7296, 1408, 4096, 13312, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[768, 22912, 7808, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2304, 6144, 11648, 12928, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7936, 2944, 11392, 10752, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16000, 2560, 4864, 9600, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[10, 434, 4044, 1, 1, 0, 2753, 90, 150, 39, 1, 391, 1, 25, 3, 0, 954, 4, 0, 6, 3961, 0, 3151, 9, 4095, 380, 4069, 3, 3687, 1, 1433, 3072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[38, 10, 3892, 0, 18, 0, 2195, 0, 7, 175, 12, 0, 27, 37, 6, 0, 61, 0, 10, 27, 1847, 4, 1449, 4, 3614, 2552, 4078, 3476, 3883, 0, 1283, 4063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[75, 74, 3906, 73, 0, 0, 3140, 1, 7, 0, 0, 0, 0, 17, 0, 0, 626, 0, 0, 0, 2534, 4, 3725, 12, 4089, 482, 3899, 354, 3994, 0, 3372, 2384] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4, 417, 3386, 67, 0, 0, 2250, 268, 1, 0, 0, 1, 7, 0, 0, 0, 294, 0, 1, 1, 2384, 0, 1808, 50, 4030, 3675, 3766, 753, 4070, 0, 2467, 3068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3, 359, 3795, 85, 1, 11, 2943, 71, 2, 10, 40, 1, 0, 319, 0, 15, 2611, 31, 2, 0, 3030, 11, 188, 9, 3927, 3896, 3750, 1262, 4087, 0, 306, 2003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[44, 0, 3924, 1, 0, 0, 3830, 151, 53, 1, 0, 0, 0, 2, 101, 11, 130, 13, 7, 0, 3703, 0, 1965, 0, 3914, 2451, 4089, 493, 4090, 0, 176, 3619] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[17, 1, 4094, 2, 1, 0, 2420, 41, 6, 0, 0, 38, 0, 0, 0, 0, 1335, 3, 0, 0, 2481, 3, 3594, 71, 3997, 1460, 4084, 2295, 4079, 0, 526, 2220] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[425, 211, 3901, 4, 0, 0, 1201, 98, 0, 0, 0, 1, 0, 0, 2, 2, 36, 0, 0, 0, 4081, 0, 596, 143, 4088, 3576, 3949, 960, 4087, 0, 2557, 2850] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 359, 3731, 85, 0, 0, 0, 0, 1, 11, 2943, 71, 1015, 226, 64, 8, 2, 10, 40, 1, 3900, 34, 0, 0, 0, 319, 0, 15, 4087, 27, 0, 0, 2611, 31, 2, 0, 3150, 0, 0, 0, 2015, 11, 188, 9, 0, 0, 0, 0, 0, 3896, 558, 1262, 0, 0, 0, 0, 0, 0, 306, 1777, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4339, 3987, 4448, 5794, 2223, 5716, 2083] avg=4096.0000 max=5794 max_rank=4 min=2083 min_rank=7 imbalance_ratio=1.414551 max_load_reduction=26974 max_load_reduction_pct=0.823181 moved_to_echo=15816 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[10, 434, 3975, 1, 0, 0, 0, 0, 1, 0, 2753, 90, 1327, 347, 69, 9, 150, 39, 1, 391, 4067, 37, 0, 0, 1, 25, 3, 0, 3687, 28, 0, 0, 954, 4, 0, 6, 3418, 0, 0, 0, 2634, 0, 3151, 9, 0, 0, 0, 0, 0, 380, 605, 3, 0, 0, 0, 0, 0, 1, 1433, 2725, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4420, 4596, 4685, 3744, 4382, 5794, 988, 4159] avg=4096.0000 max=5794 max_rank=5 min=988 min_rank=6 imbalance_ratio=1.414551 max_load_reduction=26974 max_load_reduction_pct=0.823181 moved_to_echo=15323 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[38, 10, 3826, 0, 0, 0, 0, 0, 18, 0, 2195, 0, 618, 459, 66, 9, 7, 175, 12, 0, 3590, 37, 0, 0, 27, 37, 6, 0, 3883, 24, 0, 0, 61, 0, 10, 27, 3425, 0, 0, 0, 1229, 4, 1449, 4, 0, 0, 0, 0, 0, 2552, 607, 3476, 0, 0, 0, 0, 0, 0, 1283, 3604, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3874, 3365, 3821, 3977, 3523, 2686, 6635, 4887] avg=4096.0000 max=6635 max_rank=6 min=2686 min_rank=5 imbalance_ratio=1.619873 max_load_reduction=26133 max_load_reduction_pct=0.797516 moved_to_echo=17731 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[44, 0, 3857, 1, 0, 0, 0, 0, 0, 0, 3830, 151, 1240, 409, 67, 9, 53, 1, 0, 0, 3887, 37, 0, 0, 0, 2, 101, 11, 4090, 27, 0, 0, 130, 13, 7, 0, 3434, 0, 0, 0, 2463, 0, 1965, 0, 0, 0, 0, 0, 0, 2451, 609, 493, 0, 0, 0, 0, 0, 0, 176, 3210, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3902, 5706, 3978, 4231, 3584, 4428, 3553, 3386] avg=4096.0000 max=5706 max_rank=1 min=3386 min_rank=7 imbalance_ratio=1.393066 max_load_reduction=27062 max_load_reduction_pct=0.825867 moved_to_echo=14951 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[17, 1, 4020, 2, 0, 0, 0, 0, 1, 0, 2420, 41, 835, 254, 74, 13, 6, 0, 0, 38, 3970, 43, 0, 0, 0, 0, 0, 0, 4079, 27, 0, 0, 1335, 3, 0, 0, 3434, 0, 0, 0, 1646, 3, 3594, 71, 0, 0, 0, 0, 0, 1460, 594, 2295, 0, 0, 0, 0, 0, 0, 526, 1966, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 3638, 4057, 4106, 4772, 5314, 4349, 2492] avg=4096.0000 max=5314 max_rank=5 min=2492 min_rank=7 imbalance_ratio=1.297363 max_load_reduction=27454 max_load_reduction_pct=0.837830 moved_to_echo=16927 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[75, 74, 3840, 73, 0, 0, 0, 0, 0, 0, 3140, 1, 849, 269, 66, 9, 7, 0, 0, 0, 4062, 36, 0, 0, 0, 17, 0, 0, 3994, 27, 0, 0, 626, 0, 0, 0, 3275, 0, 0, 0, 1685, 4, 3725, 12, 0, 0, 0, 0, 0, 482, 579, 354, 0, 0, 0, 0, 0, 0, 3372, 2115, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4334, 4105, 4038, 3901, 5426, 1415, 5487] avg=4096.0000 max=5487 max_rank=7 min=1415 min_rank=6 imbalance_ratio=1.339600 max_load_reduction=27281 max_load_reduction_pct=0.832550 moved_to_echo=16229 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 417, 3329, 67, 0, 0, 0, 0, 0, 0, 2250, 268, 798, 346, 57, 8, 1, 0, 0, 1, 4002, 34, 0, 0, 7, 0, 0, 0, 4070, 28, 0, 0, 294, 0, 1, 1, 3163, 0, 0, 0, 1586, 0, 1808, 50, 0, 0, 0, 0, 0, 3675, 561, 753, 0, 0, 0, 0, 0, 0, 2467, 2722, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3817, 3727, 4038, 4105, 3459, 3444, 4989, 5189] avg=4096.0000 max=5189 max_rank=7 min=3444 min_rank=5 imbalance_ratio=1.266846 max_load_reduction=27579 max_load_reduction_pct=0.841644 moved_to_echo=17081 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[425, 211, 3835, 4, 0, 0, 0, 0, 0, 0, 1201, 98, 1367, 322, 66, 9, 0, 0, 0, 1, 4060, 36, 0, 0, 0, 0, 2, 2, 4087, 28, 0, 0, 36, 0, 0, 0, 3317, 0, 0, 0, 2714, 0, 596, 143, 0, 0, 0, 0, 0, 3576, 587, 960, 0, 0, 0, 0, 0, 0, 2557, 2528, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4475, 3063, 4097, 4119, 3353, 3453, 5123, 5085] avg=4096.0000 max=5123 max_rank=6 min=3063 min_rank=1 imbalance_ratio=1.250732 max_load_reduction=27645 max_load_reduction_pct=0.843658 moved_to_echo=17014 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 20736, 768, 8064, 2688, 640, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 12160, 20736, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16000, 128, 16512, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 18560, 4736, 9600, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6144, 128, 128, 128, 26624, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 256, 128, 512, 31616, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[640, 1536, 30464, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 128, 128, 32000, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[675, 3622, 0, 248, 2, 1868, 0, 3908, 0, 4046, 0, 226, 771, 335, 4094, 0, 0, 0, 3, 4018, 59, 4092, 9, 3, 3935, 0, 664, 13, 51, 0, 0, 126] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[13, 4072, 0, 0, 1, 118, 0, 4094, 0, 4086, 0, 1, 27, 3997, 3880, 6, 0, 2, 0, 4096, 0, 4096, 0, 2, 4069, 1, 80, 0, 0, 0, 0, 127] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[22, 3918, 0, 3, 0, 28, 0, 4092, 0, 4022, 0, 6, 348, 11, 4093, 3, 1, 1, 18, 3584, 0, 4093, 0, 24, 4096, 110, 3970, 22, 51, 1, 53, 198] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[128, 4045, 1, 0, 0, 3, 2, 3157, 0, 3716, 0, 1, 160, 106, 3545, 62, 0, 1, 53, 2336, 601, 3983, 3, 2796, 4090, 0, 3004, 0, 537, 417, 0, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1914, 4062, 0, 0, 21, 1923, 8, 4063, 0, 4087, 0, 0, 64, 122, 3966, 58, 0, 1, 92, 3155, 263, 4069, 0, 302, 3960, 0, 635, 1, 1, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[313, 3769, 1, 2, 0, 19, 0, 4091, 0, 4080, 0, 9, 146, 1839, 4096, 0, 0, 0, 1, 3693, 113, 4096, 0, 1231, 4006, 0, 1186, 0, 9, 0, 44, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[542, 4096, 0, 0, 0, 2718, 0, 3848, 0, 4096, 0, 0, 3, 510, 3378, 0, 0, 0, 0, 3750, 272, 4095, 0, 8, 4096, 0, 1302, 0, 0, 0, 1, 53] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 3660, 0, 0, 0, 482, 0, 4055, 0, 4096, 5, 2, 0, 3596, 3985, 0, 107, 67, 0, 4005, 14, 4092, 0, 33, 4069, 11, 323, 11, 0, 7, 148, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[13, 3768, 0, 0, 0, 0, 0, 0, 1, 118, 0, 3345, 0, 0, 0, 0, 0, 4086, 0, 1, 37, 0, 0, 0, 27, 3997, 2576, 6, 0, 0, 0, 0, 0, 2, 0, 4096, 267, 217, 0, 0, 0, 3397, 0, 2, 0, 0, 0, 0, 2699, 1, 80, 0, 0, 0, 0, 0, 0, 0, 0, 127, 1370, 1304, 749, 482] rerouted_tokens_per_ep_rank=[3781, 3464, 4124, 6606, 4582, 3399, 2780, 4032] avg=4096.0000 max=6606 max_rank=3 min=2780 min_rank=6 imbalance_ratio=1.612793 max_load_reduction=26162 max_load_reduction_pct=0.798401 moved_to_echo=14793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[675, 3352, 0, 248, 0, 0, 0, 0, 2, 1868, 0, 3193, 0, 0, 0, 0, 0, 4046, 0, 226, 33, 0, 0, 0, 771, 335, 2718, 0, 0, 0, 0, 0, 0, 0, 3, 4018, 237, 217, 0, 0, 59, 3393, 9, 3, 0, 0, 0, 0, 2610, 0, 664, 13, 0, 0, 0, 0, 51, 0, 0, 126, 1325, 1376, 715, 482] rerouted_tokens_per_ep_rank=[4275, 5063, 4305, 3824, 4475, 3464, 3287, 4075] avg=4096.0000 max=5063 max_rank=1 min=3287 min_rank=6 imbalance_ratio=1.236084 max_load_reduction=27705 max_load_reduction_pct=0.845490 moved_to_echo=15301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[128, 3743, 1, 0, 0, 0, 0, 0, 0, 3, 2, 2579, 0, 0, 0, 0, 0, 3716, 0, 1, 37, 0, 0, 0, 160, 106, 2354, 62, 0, 0, 0, 0, 0, 1, 53, 2336, 265, 211, 0, 0, 601, 3303, 3, 2796, 0, 0, 0, 0, 2713, 0, 3004, 0, 0, 0, 0, 0, 537, 417, 0, 21, 1377, 1191, 578, 469] rerouted_tokens_per_ep_rank=[3872, 2584, 3754, 2682, 2866, 6703, 5717, 4590] avg=4096.0000 max=6703 max_rank=5 min=2584 min_rank=1 imbalance_ratio=1.636475 max_load_reduction=26065 max_load_reduction_pct=0.795441 moved_to_echo=19876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1914, 3759, 0, 0, 0, 0, 0, 0, 21, 1923, 8, 3319, 0, 0, 0, 0, 0, 4087, 0, 0, 37, 0, 0, 0, 64, 122, 2633, 58, 0, 0, 0, 0, 0, 1, 92, 3155, 266, 216, 0, 0, 263, 3374, 0, 302, 0, 0, 0, 0, 2627, 0, 635, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1333, 1333, 744, 479] rerouted_tokens_per_ep_rank=[5673, 5271, 4124, 2877, 3730, 3939, 3263, 3891] avg=4096.0000 max=5673 max_rank=0 min=2877 min_rank=3 imbalance_ratio=1.385010 max_load_reduction=27095 max_load_reduction_pct=0.826874 moved_to_echo=14823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[22, 3618, 0, 3, 0, 0, 0, 0, 0, 28, 0, 3340, 0, 0, 0, 0, 0, 4022, 0, 6, 41, 0, 0, 0, 348, 11, 2714, 3, 0, 0, 0, 0, 1, 1, 18, 3584, 259, 223, 0, 0, 0, 3382, 0, 24, 0, 0, 0, 0, 2713, 110, 3970, 22, 0, 0, 0, 0, 51, 1, 53, 198, 1383, 1379, 752, 488] rerouted_tokens_per_ep_rank=[3643, 3368, 4069, 3076, 4086, 3406, 6815, 4305] avg=4096.0000 max=6815 max_rank=6 min=3076 min_rank=3 imbalance_ratio=1.663818 max_load_reduction=25953 max_load_reduction_pct=0.792023 moved_to_echo=18612 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[313, 3488, 1, 2, 0, 0, 0, 0, 0, 19, 0, 3342, 0, 0, 0, 0, 0, 4080, 0, 9, 34, 0, 0, 0, 146, 1839, 2720, 0, 0, 0, 0, 0, 0, 0, 1, 3693, 247, 217, 0, 0, 113, 3397, 0, 1231, 0, 0, 0, 0, 2657, 0, 1186, 0, 0, 0, 0, 0, 9, 0, 44, 24, 1349, 1376, 749, 482] rerouted_tokens_per_ep_rank=[3804, 3361, 4123, 4705, 4158, 4741, 3843, 4033] avg=4096.0000 max=4741 max_rank=5 min=3361 min_rank=1 imbalance_ratio=1.157471 max_load_reduction=28027 max_load_reduction_pct=0.855316 moved_to_echo=16775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[542, 3791, 0, 0, 0, 0, 0, 0, 0, 2718, 0, 3144, 0, 0, 0, 0, 0, 4096, 0, 0, 37, 0, 0, 0, 3, 510, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 268, 217, 0, 0, 272, 3396, 0, 8, 0, 0, 0, 0, 2717, 0, 1302, 0, 0, 0, 0, 0, 0, 0, 1, 53, 1379, 1135, 704, 482] rerouted_tokens_per_ep_rank=[4333, 5862, 4133, 2756, 4235, 3676, 4019, 3754] avg=4096.0000 max=5862 max_rank=1 min=2756 min_rank=3 imbalance_ratio=1.431152 max_load_reduction=26906 max_load_reduction_pct=0.821106 moved_to_echo=15684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3387, 0, 0, 0, 0, 0, 0, 0, 482, 0, 3313, 0, 0, 0, 0, 0, 4096, 5, 2, 33, 0, 0, 0, 0, 3596, 2646, 0, 0, 0, 0, 0, 107, 67, 0, 4005, 240, 217, 0, 0, 14, 3393, 0, 33, 0, 0, 0, 0, 2699, 11, 323, 11, 0, 0, 0, 0, 0, 7, 148, 0, 1370, 1339, 742, 482] rerouted_tokens_per_ep_rank=[3387, 3795, 4136, 6242, 4636, 3440, 3044, 4088] avg=4096.0000 max=6242 max_rank=3 min=3044 min_rank=6 imbalance_ratio=1.523926 max_load_reduction=26526 max_load_reduction_pct=0.809509 moved_to_echo=15208 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[768, 512, 256, 640, 11008, 10496, 5760, 3968] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21504, 128, 11264, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1536, 10624, 20608, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3712, 28928, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1408, 27136, 128, 4480, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 7168, 128, 25600, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 256, 28672, 2176, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32256, 128, 256, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[33, 3725, 0, 4096, 0, 2, 218, 2780, 13, 63, 0, 4088, 2636, 29, 3132, 0, 0, 0, 0, 0, 1, 1, 0, 2, 3594, 11, 0, 2, 4068, 4008, 160, 106] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[7, 745, 0, 3963, 0, 0, 548, 2243, 1272, 47, 0, 4079, 1320, 1, 1479, 316, 1, 0, 339, 28, 1393, 43, 2, 0, 3342, 0, 0, 2, 4096, 3790, 0, 3712] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[105, 364, 0, 4096, 135, 0, 3578, 1500, 75, 909, 0, 3500, 3662, 174, 3137, 2, 0, 39, 0, 1, 0, 0, 0, 0, 3473, 28, 0, 2, 4067, 3915, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[442, 3, 0, 4083, 1, 0, 0, 2364, 30, 109, 0, 4095, 3750, 0, 3391, 1, 0, 0, 1, 21, 0, 0, 0, 110, 4069, 0, 8, 0, 4054, 3617, 46, 2573] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[64, 4019, 7, 4071, 1, 17, 3, 3904, 1094, 639, 77, 4000, 4076, 3, 64, 0, 6, 2, 0, 17, 0, 23, 0, 12, 2215, 5, 0, 42, 4091, 3922, 0, 394] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[108, 31, 0, 4096, 8, 0, 1126, 3924, 34, 879, 0, 4096, 3975, 0, 2078, 0, 14, 0, 47, 0, 0, 0, 0, 0, 3973, 0, 0, 0, 4071, 4051, 192, 65] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 1002, 0, 4095, 10, 8, 2114, 666, 1232, 2181, 0, 4085, 3775, 23, 180, 0, 6, 0, 221, 0, 0, 24, 1, 1, 3682, 0, 0, 3, 4005, 4069, 72, 1313] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 3576, 0, 4096, 0, 0, 471, 632, 227, 0, 0, 4096, 2489, 0, 1018, 0, 379, 0, 28, 7, 0, 1, 0, 6, 4096, 0, 0, 0, 4029, 3683, 115, 3819] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[33, 3725, 0, 2330, 0, 0, 0, 0, 0, 2, 218, 2780, 382, 356, 0, 0, 13, 63, 0, 3049, 0, 0, 0, 0, 1821, 29, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 0, 1, 1, 0, 2, 1766, 1401, 683, 109, 3594, 11, 0, 2, 433, 0, 0, 0, 0, 2607, 160, 106, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6088, 3738, 3125, 4982, 3959, 3963, 4040, 2873] avg=4096.0000 max=6088 max_rank=0 min=2873 min_rank=7 imbalance_ratio=1.486328 max_load_reduction=26680 max_load_reduction_pct=0.814209 moved_to_echo=14835 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[7, 745, 0, 2254, 0, 0, 0, 0, 0, 0, 548, 2243, 191, 355, 0, 0, 1272, 47, 0, 3042, 0, 0, 0, 0, 913, 1, 1479, 316, 0, 0, 0, 0, 1, 0, 339, 28, 3986, 0, 0, 0, 1393, 43, 2, 0, 1709, 1325, 682, 110, 3342, 0, 0, 2, 216, 0, 0, 0, 0, 2465, 0, 3712, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3006, 3337, 4361, 2709, 4354, 5264, 3560, 6177] avg=4096.0000 max=6177 max_rank=7 min=2709 min_rank=3 imbalance_ratio=1.508057 max_load_reduction=26591 max_load_reduction_pct=0.811493 moved_to_echo=19355 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[105, 364, 0, 2330, 0, 0, 0, 0, 135, 0, 3578, 1500, 530, 305, 0, 0, 75, 909, 0, 2610, 0, 0, 0, 0, 2531, 174, 3137, 2, 0, 0, 0, 0, 0, 39, 0, 1, 3958, 0, 0, 0, 0, 0, 0, 0, 1766, 1369, 585, 109, 3473, 28, 0, 2, 601, 0, 0, 0, 0, 2546, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2799, 6048, 3594, 5844, 3998, 3829, 4104, 2552] avg=4096.0000 max=6048 max_rank=1 min=2552 min_rank=7 imbalance_ratio=1.476562 max_load_reduction=26720 max_load_reduction_pct=0.815430 moved_to_echo=14483 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[64, 4019, 7, 2316, 0, 0, 0, 0, 1, 17, 3, 3904, 590, 348, 0, 0, 1094, 639, 77, 2983, 0, 0, 0, 0, 2817, 3, 64, 0, 0, 0, 0, 0, 6, 2, 0, 17, 3982, 0, 0, 0, 0, 23, 0, 12, 1755, 1371, 669, 109, 2215, 5, 0, 42, 669, 0, 0, 0, 0, 2551, 0, 394, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6406, 4863, 4793, 2884, 4007, 3939, 2931, 2945] avg=4096.0000 max=6406 max_rank=0 min=2884 min_rank=3 imbalance_ratio=1.563965 max_load_reduction=26362 max_load_reduction_pct=0.804504 moved_to_echo=13822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[442, 3, 0, 2322, 0, 0, 0, 0, 1, 0, 0, 2364, 543, 356, 0, 0, 30, 109, 0, 3055, 0, 0, 0, 0, 2591, 0, 3391, 1, 0, 0, 0, 0, 0, 0, 1, 21, 3945, 0, 0, 0, 0, 0, 0, 110, 1761, 1264, 684, 109, 4069, 0, 8, 0, 616, 0, 0, 0, 0, 2353, 46, 2573, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2767, 3264, 3194, 5983, 3967, 3928, 4693, 4972] avg=4096.0000 max=5983 max_rank=3 min=2767 min_rank=0 imbalance_ratio=1.460693 max_load_reduction=26785 max_load_reduction_pct=0.817413 moved_to_echo=17560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1002, 0, 2325, 0, 0, 0, 0, 10, 8, 2114, 666, 550, 359, 0, 0, 1232, 2181, 0, 3040, 0, 0, 0, 0, 2601, 23, 180, 0, 0, 0, 0, 0, 6, 0, 221, 0, 3898, 0, 0, 0, 0, 24, 1, 1, 1770, 1427, 686, 107, 3682, 0, 0, 3, 624, 0, 0, 0, 0, 2642, 72, 1313, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3327, 3707, 6453, 2804, 4125, 4016, 4309, 4027] avg=4096.0000 max=6453 max_rank=2 min=2804 min_rank=3 imbalance_ratio=1.575439 max_load_reduction=26315 max_load_reduction_pct=0.803070 moved_to_echo=16477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[108, 31, 0, 2330, 0, 0, 0, 0, 8, 0, 1126, 3924, 576, 357, 0, 0, 34, 879, 0, 3054, 0, 0, 0, 0, 2746, 0, 2078, 0, 0, 0, 0, 0, 14, 0, 47, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 1766, 1416, 685, 109, 3973, 0, 0, 0, 653, 0, 0, 0, 0, 2635, 192, 65, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2469, 5991, 3967, 4824, 4023, 3976, 4626, 2892] avg=4096.0000 max=5991 max_rank=1 min=2469 min_rank=0 imbalance_ratio=1.462646 max_load_reduction=26777 max_load_reduction_pct=0.817169 moved_to_echo=15517 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3576, 0, 2330, 0, 0, 0, 0, 0, 0, 471, 632, 360, 357, 0, 0, 227, 0, 0, 3054, 0, 0, 0, 0, 1720, 0, 1018, 0, 0, 0, 0, 0, 379, 0, 28, 7, 3921, 0, 0, 0, 0, 1, 0, 6, 1766, 1287, 685, 108, 4096, 0, 0, 0, 409, 0, 0, 0, 0, 2396, 115, 3819, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5906, 1820, 3281, 2738, 4335, 3853, 4505, 6330] avg=4096.0000 max=6330 max_rank=7 min=1820 min_rank=1 imbalance_ratio=1.545410 max_load_reduction=26438 max_load_reduction_pct=0.806824 moved_to_echo=19023 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1408, 128, 128, 256, 14080, 10880, 5376, 896] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 13568, 128, 18560, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 8064, 18048, 3840, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17792, 256, 14592, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 20224, 640, 12032, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4864, 128, 23936, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[28544, 128, 128, 128, 4224, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 128, 640, 128, 31616, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 388, 0, 34, 0, 0, 2, 1, 0, 705, 4072, 4095, 4069, 0, 2, 195, 0, 1, 3658, 29, 3018, 81, 2729, 4096, 0, 0, 0, 4094, 1499, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 38, 0, 242, 8, 355, 127, 2, 0, 0, 0, 6, 3974, 4019, 4096, 4, 85, 2184, 53, 0, 1603, 0, 4082, 22, 3693, 4044, 34, 0, 1, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 6, 214, 0, 12, 23, 1, 31, 105, 0, 4050, 4094, 4053, 4096, 0, 0, 3662, 12, 0, 4043, 0, 522, 0, 7, 3751, 0, 0, 0, 4086, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 100, 3, 0, 0, 0, 81, 3, 27, 3981, 3992, 4063, 3998, 88, 1, 315, 1, 0, 3862, 5, 3714, 29, 412, 4014, 1, 2, 1, 4075, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1650, 2, 87, 0, 0, 7, 34, 0, 3846, 4003, 4096, 4040, 0, 10, 5, 735, 0, 2896, 0, 3181, 0, 74, 4082, 0, 2, 0, 4016, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 0, 2564, 0, 78, 0, 49, 0, 9, 0, 3121, 4087, 4016, 4096, 3, 0, 3, 0, 0, 2238, 0, 3445, 0, 865, 4096, 0, 0, 0, 4096, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 3816, 1, 11, 0, 185, 0, 0, 0, 28, 4073, 4095, 4096, 11, 17, 2737, 17, 1, 2407, 0, 3125, 0, 9, 4035, 2, 0, 5, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 132, 0, 0, 6, 0, 0, 2306, 0, 1799, 4040, 4096, 4096, 0, 0, 3095, 0, 0, 823, 0, 514, 0, 3669, 4094, 0, 0, 0, 4096, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 388, 2871, 100, 0, 0, 0, 34, 0, 0, 3969, 0, 0, 0, 2, 1, 0, 705, 1471, 175, 89, 0, 4072, 42, 0, 0, 0, 0, 0, 0, 2, 195, 0, 1, 1302, 1182, 0, 0, 3658, 29, 1547, 81, 0, 0, 0, 0, 2729, 2705, 0, 0, 0, 0, 0, 0, 0, 3919, 1499, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3359, 4003, 2443, 4114, 2682, 5315, 5434, 5418] avg=4096.0000 max=5434 max_rank=6 min=2443 min_rank=2 imbalance_ratio=1.326660 max_load_reduction=27334 max_load_reduction_pct=0.834167 moved_to_echo=18849 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 38, 0, 242, 2817, 101, 0, 0, 8, 355, 127, 2, 3995, 0, 0, 0, 0, 0, 0, 6, 1989, 175, 88, 0, 3974, 42, 0, 4, 0, 0, 0, 0, 85, 2184, 53, 0, 1285, 1160, 0, 0, 1603, 0, 2093, 22, 0, 0, 0, 0, 3693, 2671, 34, 0, 0, 0, 0, 0, 1, 3921, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3198, 4487, 2258, 4020, 4767, 3718, 6398, 3922] avg=4096.0000 max=6398 max_rank=6 min=2258 min_rank=2 imbalance_ratio=1.562012 max_load_reduction=26370 max_load_reduction_pct=0.804749 moved_to_echo=18805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 100, 2848, 99, 0, 0, 3, 0, 0, 0, 3899, 0, 0, 0, 81, 3, 27, 3981, 1810, 174, 87, 0, 3992, 42, 0, 88, 0, 0, 0, 0, 1, 315, 1, 0, 1276, 1173, 0, 0, 3862, 5, 1904, 29, 0, 0, 0, 0, 412, 2651, 1, 2, 0, 0, 0, 0, 1, 3901, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3047, 3902, 6163, 4122, 2766, 5800, 3066, 3902] avg=4096.0000 max=6163 max_rank=2 min=2766 min_rank=4 imbalance_ratio=1.504639 max_load_reduction=26605 max_load_reduction_pct=0.811920 moved_to_echo=15534 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 214, 2841, 101, 0, 0, 0, 12, 23, 1, 3995, 0, 0, 0, 31, 105, 0, 4050, 254, 174, 81, 0, 4094, 42, 0, 0, 0, 0, 0, 0, 0, 3662, 12, 0, 1192, 1170, 0, 0, 4043, 0, 268, 0, 0, 0, 0, 0, 7, 2478, 0, 0, 0, 0, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3162, 4031, 4695, 4136, 6036, 4311, 2485, 3912] avg=4096.0000 max=6036 max_rank=4 min=2485 min_rank=6 imbalance_ratio=1.473633 max_load_reduction=26732 max_load_reduction_pct=0.815796 moved_to_echo=16744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1650, 2871, 100, 0, 0, 2, 87, 0, 0, 3940, 0, 0, 0, 7, 34, 0, 3846, 1550, 171, 88, 0, 4003, 43, 0, 0, 0, 0, 0, 0, 10, 5, 735, 0, 1297, 1182, 0, 0, 2896, 0, 1631, 0, 0, 0, 0, 0, 74, 2697, 0, 2, 0, 0, 0, 0, 0, 3845, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4621, 4029, 5696, 4046, 3229, 4527, 2773, 3847] avg=4096.0000 max=5696 max_rank=2 min=2773 min_rank=6 imbalance_ratio=1.390625 max_load_reduction=27072 max_load_reduction_pct=0.826172 moved_to_echo=14376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 3816, 2877, 101, 0, 0, 1, 11, 0, 185, 3995, 0, 0, 0, 0, 0, 0, 28, 1526, 177, 90, 0, 4073, 31, 0, 11, 0, 0, 0, 0, 17, 2737, 17, 1, 1287, 1187, 0, 0, 2407, 0, 1599, 0, 0, 0, 0, 0, 9, 2658, 2, 0, 0, 0, 0, 0, 5, 3918, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6796, 4192, 1821, 4115, 5246, 4006, 2669, 3923] avg=4096.0000 max=6796 max_rank=0 min=1821 min_rank=2 imbalance_ratio=1.659180 max_load_reduction=25972 max_load_reduction_pct=0.792603 moved_to_echo=15844 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 2564, 2815, 101, 0, 0, 0, 78, 0, 49, 3995, 0, 0, 0, 0, 9, 0, 3121, 1679, 175, 89, 0, 4087, 42, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 1302, 1159, 0, 0, 2238, 0, 1766, 0, 0, 0, 0, 0, 865, 2705, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5481, 4122, 5073, 4132, 2464, 4004, 3570, 3922] avg=4096.0000 max=5481 max_rank=0 min=2464 min_rank=4 imbalance_ratio=1.338135 max_load_reduction=27287 max_load_reduction_pct=0.832733 moved_to_echo=13960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 132, 2871, 100, 0, 0, 0, 0, 6, 0, 3996, 0, 0, 0, 0, 2306, 0, 1799, 250, 175, 89, 0, 4040, 43, 0, 0, 0, 0, 0, 0, 0, 3095, 0, 0, 1301, 1182, 0, 0, 823, 0, 264, 0, 0, 0, 0, 0, 3669, 2704, 0, 0, 0, 0, 0, 0, 0, 3921, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3104, 4002, 4619, 4083, 5578, 1087, 6373, 3922] avg=4096.0000 max=6373 max_rank=6 min=1087 min_rank=5 imbalance_ratio=1.555908 max_load_reduction=26395 max_load_reduction_pct=0.805511 moved_to_echo=16960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 9216, 22912, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[21632, 128, 11136, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 12288, 896, 128, 10368, 9472, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11520, 21376, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 2560, 128, 17536, 10624, 1408, 768, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31360, 1536, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 640, 256, 256, 31872, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32384, 384, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[280, 0, 1257, 0, 0, 3, 0, 405, 4023, 0, 3, 908, 1, 0, 2, 0, 27, 4031, 3121, 0, 29, 2, 0, 4066, 4051, 1904, 0, 612, 4091, 4, 3930, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[2917, 0, 323, 1388, 0, 0, 627, 0, 4092, 6, 4, 1, 0, 391, 928, 9, 310, 4090, 4092, 0, 2, 0, 1, 1044, 4085, 313, 6, 0, 4095, 0, 4044, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[2906, 0, 3, 0, 0, 0, 2, 60, 4086, 6, 2, 48, 1, 0, 0, 152, 4, 3888, 3998, 0, 3, 0, 0, 4039, 4096, 3958, 0, 112, 4065, 3, 1336, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 193, 0, 512, 0, 619, 0, 3849, 5, 0, 141, 1, 0, 78, 1, 1505, 2546, 3058, 0, 0, 99, 16, 0, 4080, 3768, 24, 6, 4096, 0, 4075, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[2264, 0, 937, 3, 0, 0, 20, 17, 4096, 0, 0, 0, 0, 0, 436, 0, 118, 1910, 4095, 0, 0, 0, 0, 3830, 4032, 2853, 0, 0, 4096, 0, 4061, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2066, 0, 3039, 0, 9, 0, 0, 163, 4053, 0, 0, 0, 12, 50, 582, 15, 0, 157, 4038, 3, 0, 354, 13, 2150, 3880, 4043, 0, 0, 4055, 0, 4086, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 8, 78, 54, 0, 1, 306, 1, 4095, 0, 12, 0, 0, 0, 0, 5, 3370, 962, 4084, 1, 0, 0, 39, 3796, 4095, 3720, 0, 0, 4090, 0, 4044, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[3665, 0, 2627, 277, 0, 0, 0, 1, 4096, 0, 0, 1, 0, 0, 1, 0, 8, 1553, 4094, 0, 0, 0, 0, 4012, 4096, 154, 0, 0, 4096, 0, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[280, 0, 1257, 0, 453, 94, 0, 0, 0, 3, 0, 405, 2225, 1027, 0, 0, 3929, 0, 3, 908, 0, 0, 0, 0, 1, 0, 2, 0, 3706, 50, 0, 0, 27, 4031, 846, 0, 0, 0, 0, 0, 29, 2, 0, 4066, 1158, 0, 0, 0, 1413, 1904, 0, 612, 0, 0, 0, 0, 385, 4, 3930, 18, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2084, 3660, 4840, 3759, 4904, 5255, 3929, 4337] avg=4096.0000 max=5255 max_rank=5 min=2084 min_rank=0 imbalance_ratio=1.282959 max_load_reduction=27513 max_load_reduction_pct=0.839630 moved_to_echo=18425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2906, 0, 3, 0, 458, 95, 0, 0, 0, 0, 2, 60, 2850, 1038, 0, 0, 3991, 6, 2, 48, 0, 0, 0, 0, 1, 0, 0, 152, 3682, 64, 0, 0, 4, 3888, 1084, 0, 0, 0, 0, 0, 3, 0, 0, 4039, 1171, 0, 0, 0, 1429, 3958, 0, 112, 0, 0, 0, 0, 383, 3, 1336, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3462, 3950, 4047, 3899, 4976, 5213, 5499, 1722] avg=4096.0000 max=5499 max_rank=6 min=1722 min_rank=7 imbalance_ratio=1.342529 max_load_reduction=27269 max_load_reduction_pct=0.832184 moved_to_echo=17410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2917, 0, 323, 1388, 457, 95, 0, 0, 0, 0, 627, 0, 2917, 1035, 0, 0, 3997, 6, 4, 1, 0, 0, 0, 0, 0, 391, 928, 9, 3709, 65, 0, 0, 310, 4090, 1110, 0, 0, 0, 0, 0, 2, 0, 1, 1044, 1168, 0, 0, 0, 1425, 313, 6, 0, 0, 0, 0, 0, 386, 0, 4044, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5180, 4579, 4008, 5102, 5510, 2215, 1744, 4430] avg=4096.0000 max=5510 max_rank=4 min=1744 min_rank=6 imbalance_ratio=1.345215 max_load_reduction=27258 max_load_reduction_pct=0.831848 moved_to_echo=13899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 193, 0, 456, 90, 0, 0, 512, 0, 619, 0, 2180, 1034, 0, 0, 3759, 5, 0, 141, 0, 0, 0, 0, 1, 0, 78, 1, 3710, 49, 0, 0, 1505, 2546, 829, 0, 0, 0, 0, 0, 0, 99, 16, 0, 1167, 0, 0, 0, 1423, 3768, 24, 6, 0, 0, 0, 0, 386, 0, 4075, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4835, 4345, 3905, 3839, 4880, 1282, 5221, 4461] avg=4096.0000 max=5221 max_rank=6 min=1282 min_rank=5 imbalance_ratio=1.274658 max_load_reduction=27547 max_load_reduction_pct=0.840668 moved_to_echo=15844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2066, 0, 3039, 0, 434, 94, 0, 0, 9, 0, 0, 163, 2879, 983, 0, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 12, 50, 582, 15, 3673, 64, 0, 0, 0, 157, 1095, 3, 0, 0, 0, 0, 0, 354, 13, 2150, 1109, 0, 0, 0, 1354, 4043, 0, 0, 0, 0, 0, 0, 382, 0, 4086, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5633, 4034, 3959, 4396, 1255, 3626, 5397, 4468] avg=4096.0000 max=5633 max_rank=0 min=1255 min_rank=4 imbalance_ratio=1.375244 max_load_reduction=27135 max_load_reduction_pct=0.828094 moved_to_echo=14746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 78, 54, 461, 101, 0, 0, 0, 1, 306, 1, 2915, 1042, 0, 0, 3994, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3711, 70, 0, 0, 3370, 962, 1099, 1, 0, 0, 0, 0, 0, 0, 39, 3796, 1176, 0, 0, 0, 1416, 3720, 0, 0, 0, 0, 0, 0, 379, 0, 4044, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[702, 4265, 4006, 3786, 5432, 5011, 5136, 4430] avg=4096.0000 max=5432 max_rank=4 min=702 min_rank=0 imbalance_ratio=1.326172 max_load_reduction=27336 max_load_reduction_pct=0.834229 moved_to_echo=20009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2264, 0, 937, 3, 451, 95, 0, 0, 0, 0, 20, 17, 2919, 1022, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 0, 3710, 65, 0, 0, 118, 1910, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 1153, 0, 0, 0, 1406, 2853, 0, 0, 0, 0, 0, 0, 386, 0, 4061, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3750, 3978, 4001, 4211, 3139, 4983, 4259, 4447] avg=4096.0000 max=4983 max_rank=5 min=3139 min_rank=4 imbalance_ratio=1.216553 max_load_reduction=27785 max_load_reduction_pct=0.847931 moved_to_echo=16828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3665, 0, 2627, 277, 458, 95, 0, 0, 0, 0, 0, 1, 2918, 1038, 0, 0, 4001, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 3710, 65, 0, 0, 8, 1553, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 1171, 0, 0, 0, 1429, 154, 0, 0, 0, 0, 0, 0, 386, 0, 4087, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7122, 3957, 4002, 3776, 2672, 5183, 1583, 4473] avg=4096.0000 max=7122 max_rank=0 min=1583 min_rank=6 imbalance_ratio=1.738770 max_load_reduction=25646 max_load_reduction_pct=0.782654 moved_to_echo=13911 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[18304, 128, 8576, 1792, 3712, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[640, 128, 1664, 768, 21888, 8320, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5376, 19200, 8320, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11392, 20736, 128, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 128, 23040, 9344, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3200, 128, 29696, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 2048, 256, 29696, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[31744, 128, 128, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 14, 1891, 91, 5, 4082, 60, 0, 4, 0, 0, 1304, 0, 4094, 4076, 7, 0, 681, 3, 4081, 0, 4073, 0, 0, 4004, 0, 0, 0, 0, 115, 4079, 104] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[139, 0, 1943, 980, 0, 4090, 1210, 0, 0, 0, 77, 4, 0, 3984, 4064, 0, 29, 3761, 0, 4096, 25, 226, 0, 67, 4027, 1, 0, 68, 0, 2, 3962, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[398, 0, 576, 594, 1, 4096, 10, 0, 3, 0, 0, 35, 3, 4091, 3309, 0, 0, 3606, 0, 4096, 612, 3115, 0, 0, 4092, 14, 0, 0, 0, 27, 4090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[307, 0, 1844, 1927, 0, 4037, 0, 0, 0, 0, 0, 396, 0, 4090, 3736, 3, 69, 4084, 0, 4094, 50, 158, 0, 0, 4095, 0, 2, 6, 0, 0, 3868, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4064, 728, 21, 3883, 0, 0, 1, 219, 0, 2, 9, 4071, 4096, 0, 3, 3376, 0, 3947, 0, 131, 0, 0, 4096, 0, 0, 5, 92, 0, 4024, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[566, 31, 1, 1683, 14, 4049, 0, 19, 77, 0, 0, 32, 52, 3984, 2967, 0, 0, 3996, 0, 3907, 0, 2657, 6, 2, 4019, 180, 0, 425, 0, 2, 4086, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1962, 1, 3770, 0, 0, 4009, 0, 13, 0, 90, 11, 792, 11, 1666, 12, 0, 0, 4091, 0, 4079, 7, 4090, 0, 0, 4077, 0, 0, 1, 11, 0, 4075, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1289, 0, 4096, 0, 1, 0, 1, 0, 2334, 1, 4078, 4043, 0, 0, 388, 0, 4096, 7, 119, 0, 0, 4096, 0, 0, 0, 0, 33, 4090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[139, 0, 1943, 980, 342, 117, 44, 0, 0, 3973, 1210, 0, 0, 0, 0, 0, 0, 0, 77, 4, 2998, 486, 0, 0, 0, 847, 4064, 0, 0, 0, 0, 0, 29, 3761, 0, 1098, 0, 0, 0, 0, 25, 226, 0, 67, 2309, 0, 0, 0, 3974, 1, 0, 68, 0, 0, 0, 0, 0, 2, 3962, 13, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[3565, 5183, 3565, 4911, 4888, 2627, 4043, 3986] avg=4096.0000 max=5183 max_rank=1 min=2627 min_rank=5 imbalance_ratio=1.265381 max_load_reduction=27585 max_load_reduction_pct=0.841827 moved_to_echo=15544 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[307, 0, 1844, 1927, 351, 115, 45, 0, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 2996, 499, 0, 0, 0, 870, 3736, 3, 0, 0, 0, 0, 69, 4084, 0, 1098, 0, 0, 0, 0, 50, 158, 0, 0, 2370, 0, 0, 0, 4040, 0, 2, 6, 0, 0, 0, 0, 0, 0, 3868, 2, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4589, 3922, 3891, 4609, 5251, 2578, 4048, 3880] avg=4096.0000 max=5251 max_rank=4 min=2578 min_rank=5 imbalance_ratio=1.281982 max_load_reduction=27517 max_load_reduction_pct=0.839752 moved_to_echo=15757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[566, 31, 1, 1683, 342, 116, 44, 0, 14, 3933, 0, 19, 0, 0, 0, 0, 77, 0, 0, 32, 2860, 486, 0, 0, 52, 847, 2967, 0, 0, 0, 0, 0, 0, 3996, 0, 1047, 0, 0, 0, 0, 0, 2657, 6, 2, 2309, 0, 0, 0, 3966, 180, 0, 425, 0, 0, 0, 0, 0, 2, 4086, 13, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[2783, 3966, 3455, 3866, 5043, 4974, 4571, 4110] avg=4096.0000 max=5043 max_rank=4 min=2783 min_rank=0 imbalance_ratio=1.231201 max_load_reduction=27725 max_load_reduction_pct=0.846100 moved_to_echo=18698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4064, 728, 350, 111, 45, 0, 21, 3772, 0, 0, 0, 0, 0, 0, 1, 219, 0, 2, 2889, 497, 0, 0, 9, 865, 4096, 0, 0, 0, 0, 0, 3, 3376, 0, 1058, 0, 0, 0, 0, 0, 131, 0, 0, 2359, 0, 0, 0, 4041, 0, 0, 5, 0, 0, 0, 0, 92, 0, 4024, 0, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[5298, 3793, 3608, 4970, 4437, 2490, 4046, 4126] avg=4096.0000 max=5298 max_rank=0 min=2490 min_rank=5 imbalance_ratio=1.293457 max_load_reduction=27470 max_load_reduction_pct=0.838318 moved_to_echo=15099 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1962, 1, 3770, 0, 143, 115, 45, 0, 0, 3894, 0, 13, 0, 0, 0, 0, 0, 90, 11, 792, 2985, 203, 0, 0, 11, 355, 12, 0, 0, 0, 0, 0, 0, 4091, 0, 1094, 0, 0, 0, 0, 7, 4090, 0, 0, 965, 0, 0, 0, 4022, 0, 0, 1, 0, 0, 0, 0, 11, 0, 4075, 0, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[6036, 3907, 4081, 378, 5185, 5062, 4023, 4096] avg=4096.0000 max=6036 max_rank=0 min=378 min_rank=3 imbalance_ratio=1.473633 max_load_reduction=26732 max_load_reduction_pct=0.815796 moved_to_echo=18366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 14, 1891, 91, 356, 120, 47, 0, 5, 3962, 60, 0, 0, 0, 0, 0, 4, 0, 0, 1304, 2991, 503, 0, 0, 0, 858, 4076, 7, 0, 0, 0, 0, 0, 681, 3, 1090, 0, 0, 0, 0, 0, 4073, 0, 0, 2377, 0, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 115, 4079, 104, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[2519, 4027, 4802, 4941, 1774, 6450, 3945, 4310] avg=4096.0000 max=6450 max_rank=5 min=1774 min_rank=4 imbalance_ratio=1.574707 max_load_reduction=26318 max_load_reduction_pct=0.803162 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[398, 0, 576, 594, 351, 117, 45, 0, 1, 3979, 10, 0, 0, 0, 0, 0, 3, 0, 0, 35, 2998, 499, 0, 0, 3, 870, 3309, 0, 0, 0, 0, 0, 0, 3606, 0, 1098, 0, 0, 0, 0, 612, 3115, 0, 0, 2371, 0, 0, 0, 4037, 14, 0, 0, 0, 0, 0, 0, 0, 27, 4090, 0, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[2081, 3990, 3535, 4182, 4704, 6098, 4051, 4127] avg=4096.0000 max=6098 max_rank=5 min=2081 min_rank=0 imbalance_ratio=1.488770 max_load_reduction=26670 max_load_reduction_pct=0.813904 moved_to_echo=18980 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 1289, 350, 117, 45, 0, 0, 3979, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2334, 2998, 498, 0, 0, 1, 867, 4043, 0, 0, 0, 0, 0, 0, 388, 0, 1098, 0, 0, 0, 0, 7, 119, 0, 0, 2363, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4090, 0, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[5897, 3980, 5831, 4911, 1486, 2489, 4041, 4133] avg=4096.0000 max=5897 max_rank=0 min=1486 min_rank=4 imbalance_ratio=1.439697 max_load_reduction=26871 max_load_reduction_pct=0.820038 moved_to_echo=12149 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3456, 128, 18304, 7296, 2688, 1024, 384, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 14592, 128, 128, 17536, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 24064, 128, 8704, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 384, 128, 4992, 23808, 3712, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 31488, 1280, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 6400, 26368, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32128, 256, 128, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 32384, 256, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 12, 0, 0, 4082, 4080, 427, 5, 14, 0, 83, 3, 2784, 0, 4086, 35, 0, 0, 3, 2, 60, 137, 3956, 3993, 11, 72, 2, 0, 0, 88, 641, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3, 4095, 4090, 0, 45, 0, 0, 0, 0, 0, 0, 3268, 0, 20, 2, 0, 0, 497, 0, 3670, 4096, 0, 0, 3, 5, 0, 3948, 1017, 3913] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1155, 0, 0, 0, 3879, 3959, 4, 0, 0, 0, 0, 0, 0, 48, 4087, 0, 2037, 8, 0, 13, 61, 47, 4096, 4089, 0, 0, 0, 72, 2, 2915, 2214, 4082] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3486, 19, 7, 0, 4082, 4084, 623, 0, 2, 1, 0, 0, 0, 1, 3740, 0, 19, 0, 1, 0, 48, 0, 4075, 4092, 0, 17, 0, 0, 5, 4095, 279, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4096, 4094, 11, 15, 0, 0, 0, 0, 1765, 0, 3345, 0, 8, 0, 0, 0, 104, 0, 3869, 4096, 0, 567, 0, 0, 0, 62, 2550, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4075, 0, 0, 0, 4088, 3864, 69, 0, 29, 0, 7, 4, 0, 0, 2151, 0, 23, 23, 0, 8, 0, 658, 4091, 4038, 0, 22, 0, 238, 0, 4060, 1253, 4067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 3695, 4076, 0, 0, 0, 0, 0, 0, 292, 0, 4087, 12, 13, 0, 0, 0, 45, 0, 4096, 4093, 0, 0, 0, 4, 0, 3354, 823, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4095, 0, 0, 2, 4093, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 910, 1126, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 2, 0, 0, 4072, 2079, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 3, 315, 96, 35, 0, 4027, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 3268, 0, 68, 84, 0, 0, 20, 2, 0, 0, 3775, 88, 0, 0, 497, 0, 3574, 0, 0, 0, 0, 0, 0, 0, 3, 5, 3822, 0, 0, 0, 0, 3829, 1017, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4545, 4072, 4099, 3420, 3885, 4071, 3830, 4846] avg=4096.0000 max=4846 max_rank=7 min=3420 min_rank=3 imbalance_ratio=1.183105 max_load_reduction=27922 max_load_reduction_pct=0.852112 moved_to_echo=16632 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 12, 0, 0, 314, 103, 0, 0, 4014, 0, 427, 5, 0, 0, 0, 0, 14, 0, 83, 3, 3993, 4, 0, 0, 2784, 0, 4086, 35, 68, 1, 0, 0, 0, 0, 3, 2, 3766, 93, 0, 0, 60, 137, 3853, 0, 0, 0, 0, 0, 11, 72, 2, 0, 3999, 0, 0, 0, 0, 87, 641, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4525, 4446, 4097, 6974, 3864, 4050, 4084, 728] avg=4096.0000 max=6974 max_rank=3 min=728 min_rank=7 imbalance_ratio=1.702637 max_load_reduction=25794 max_load_reduction_pct=0.787170 moved_to_echo=12726 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1155, 0, 0, 0, 304, 107, 26, 0, 3814, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 0, 48, 4087, 0, 65, 62, 0, 0, 2037, 8, 0, 13, 3655, 92, 0, 0, 61, 47, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 72, 3986, 0, 0, 0, 2, 2827, 2214, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1592, 3818, 4093, 4262, 5805, 4097, 4058, 5043] avg=4096.0000 max=5805 max_rank=4 min=1592 min_rank=0 imbalance_ratio=1.417236 max_load_reduction=26963 max_load_reduction_pct=0.822845 moved_to_echo=19003 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4075, 0, 0, 0, 298, 107, 36, 0, 4020, 0, 69, 0, 0, 0, 0, 0, 29, 0, 7, 4, 4038, 4, 0, 0, 0, 0, 2151, 0, 68, 87, 0, 0, 23, 23, 0, 8, 3566, 93, 0, 0, 0, 658, 3984, 0, 0, 0, 0, 0, 0, 22, 0, 238, 3970, 0, 0, 0, 0, 3937, 1253, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4516, 4089, 4082, 2306, 3713, 4642, 4230, 5190] avg=4096.0000 max=5190 max_rank=7 min=2306 min_rank=3 imbalance_ratio=1.267090 max_load_reduction=27578 max_load_reduction_pct=0.841614 moved_to_echo=17775 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 314, 107, 30, 0, 3633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 292, 0, 4087, 12, 62, 71, 0, 0, 13, 0, 0, 0, 3762, 93, 0, 0, 45, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3987, 0, 0, 0, 0, 3253, 823, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4545, 3633, 4097, 4524, 3868, 4034, 3991, 4076] avg=4096.0000 max=4545 max_rank=0 min=3633 min_rank=1 imbalance_ratio=1.109619 max_load_reduction=28223 max_load_reduction_pct=0.861298 moved_to_echo=15969 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3486, 19, 7, 0, 315, 110, 41, 0, 4009, 0, 623, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4092, 5, 0, 0, 0, 1, 3740, 0, 73, 92, 0, 0, 19, 0, 1, 0, 3769, 92, 0, 0, 48, 0, 3965, 0, 0, 0, 0, 0, 0, 17, 0, 0, 3995, 0, 0, 0, 5, 3962, 279, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3978, 4632, 4100, 3906, 3881, 4013, 4012, 4246] avg=4096.0000 max=4632 max_rank=1 min=3881 min_rank=4 imbalance_ratio=1.130859 max_load_reduction=28136 max_load_reduction_pct=0.858643 moved_to_echo=16152 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 316, 101, 0, 0, 4027, 0, 11, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 1765, 0, 3345, 0, 69, 1, 0, 0, 8, 0, 0, 0, 3778, 93, 0, 0, 104, 0, 3768, 0, 0, 0, 0, 0, 0, 567, 0, 0, 3994, 0, 0, 0, 0, 61, 2550, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4512, 4053, 4100, 5180, 3879, 3872, 4561, 2611] avg=4096.0000 max=5180 max_rank=3 min=2611 min_rank=7 imbalance_ratio=1.264648 max_load_reduction=27588 max_load_reduction_pct=0.841919 moved_to_echo=14923 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 2, 315, 107, 36, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 5, 0, 910, 1126, 68, 87, 0, 0, 0, 0, 0, 0, 3781, 92, 0, 0, 0, 0, 3989, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3999, 0, 0, 0, 0, 3949, 2079, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4555, 4025, 4100, 2196, 3873, 3989, 4002, 6028] avg=4096.0000 max=6028 max_rank=7 min=2196 min_rank=3 imbalance_ratio=1.471680 max_load_reduction=26740 max_load_reduction_pct=0.816040 moved_to_echo=17892 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 22016, 10880, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4864, 128, 25728, 1280, 640, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[29312, 128, 128, 128, 2560, 896, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2176, 128, 128, 128, 29952, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[31616, 0, 1152, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 768, 128, 384, 31872, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[896, 896, 31232, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[508, 0, 0, 0, 0, 0, 0, 4, 1, 423, 333, 0, 12, 0, 114, 4093, 2469, 4095, 1, 0, 2814, 4090, 4010, 0, 0, 0, 1469, 0, 0, 4096, 140, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 3, 0, 52, 27, 5, 0, 28, 91, 0, 2, 1, 0, 20, 771, 4012, 2671, 4096, 0, 0, 2465, 3996, 4094, 0, 0, 307, 0, 37, 77, 4077, 1888, 4048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 0, 8, 1, 1358, 15, 0, 0, 0, 0, 0, 21, 4096, 4096, 4093, 0, 0, 2638, 4067, 4079, 23, 0, 4, 0, 0, 0, 4095, 74, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4046, 0, 2, 0, 0, 225, 0, 5, 13, 1, 0, 526, 24, 0, 419, 4096, 324, 4096, 0, 46, 0, 3169, 3753, 0, 0, 586, 0, 0, 0, 3833, 3528, 4076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[49, 0, 0, 0, 0, 0, 0, 0, 0, 1, 113, 0, 0, 0, 0, 4096, 3934, 4086, 0, 0, 4082, 4096, 4096, 0, 0, 0, 30, 0, 0, 4088, 1, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[958, 26, 0, 0, 1, 0, 5, 0, 3, 638, 10, 10, 0, 0, 7, 4095, 2344, 4092, 0, 0, 3928, 3977, 4086, 0, 0, 0, 0, 1, 12, 4096, 395, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2190, 0, 0, 0, 0, 0, 2, 13, 0, 0, 29, 646, 0, 0, 53, 4089, 3996, 1971, 0, 0, 3395, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[490, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 1, 135, 0, 0, 4087, 4092, 4096, 0, 0, 3519, 4043, 4095, 0, 0, 0, 0, 0, 0, 4096, 1, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[508, 0, 0, 0, 2222, 734, 184, 0, 0, 0, 0, 4, 3894, 0, 0, 0, 1, 423, 333, 0, 2918, 583, 445, 0, 12, 0, 114, 3909, 0, 0, 0, 0, 2469, 1177, 1, 0, 0, 0, 0, 0, 2814, 1285, 0, 0, 0, 0, 0, 0, 0, 0, 1469, 0, 3565, 202, 0, 0, 0, 3362, 140, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3648, 3898, 4703, 4035, 3647, 4099, 5236, 3502] avg=4096.0000 max=5236 max_rank=6 min=3502 min_rank=7 imbalance_ratio=1.278320 max_load_reduction=27532 max_load_reduction_pct=0.840210 moved_to_echo=16484 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 52, 2171, 731, 180, 0, 27, 5, 0, 28, 3849, 0, 0, 0, 91, 0, 2, 1, 2919, 569, 455, 0, 0, 20, 771, 3832, 0, 0, 0, 0, 2671, 1177, 0, 0, 0, 0, 0, 0, 2465, 1256, 0, 0, 0, 0, 0, 0, 0, 307, 0, 37, 3639, 199, 0, 0, 77, 3346, 1888, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3137, 3909, 4037, 4623, 3848, 3721, 4182, 5311] avg=4096.0000 max=5311 max_rank=7 min=3137 min_rank=0 imbalance_ratio=1.296631 max_load_reduction=27457 max_load_reduction_pct=0.837921 moved_to_echo=17062 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 2209, 734, 184, 0, 0, 8, 1, 1358, 3886, 0, 0, 0, 15, 0, 0, 0, 2917, 579, 452, 0, 0, 0, 21, 3912, 0, 0, 0, 0, 4096, 1176, 0, 0, 0, 0, 0, 0, 2638, 1279, 0, 23, 0, 0, 0, 0, 0, 4, 0, 0, 3627, 202, 0, 0, 0, 3361, 74, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3139, 5253, 3963, 3933, 5272, 3940, 3833, 3435] avg=4096.0000 max=5272 max_rank=4 min=3139 min_rank=0 imbalance_ratio=1.287109 max_load_reduction=27496 max_load_reduction_pct=0.839111 moved_to_echo=16480 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4046, 0, 2, 0, 1721, 687, 184, 0, 0, 225, 0, 5, 3875, 0, 0, 0, 13, 1, 0, 526, 2919, 451, 416, 0, 24, 0, 419, 3912, 0, 0, 0, 0, 324, 1177, 0, 46, 0, 0, 0, 0, 0, 997, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 3337, 201, 0, 0, 0, 3146, 3528, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6640, 4105, 4326, 4355, 1547, 997, 4124, 6674] avg=4096.0000 max=6674 max_rank=7 min=997 min_rank=5 imbalance_ratio=1.629395 max_load_reduction=26094 max_load_reduction_pct=0.796326 moved_to_echo=13342 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2190, 0, 0, 0, 2225, 734, 184, 0, 0, 0, 2, 13, 3894, 0, 0, 0, 0, 0, 29, 646, 1404, 583, 455, 0, 0, 0, 53, 3905, 0, 0, 0, 0, 3996, 567, 0, 0, 0, 0, 0, 0, 3395, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3641, 202, 0, 0, 0, 3362, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5333, 3909, 3117, 3958, 4563, 4683, 3843, 3362] avg=4096.0000 max=5333 max_rank=0 min=3117 min_rank=2 imbalance_ratio=1.302002 max_load_reduction=27435 max_load_reduction_pct=0.837250 moved_to_echo=16451 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[958, 26, 0, 0, 2166, 738, 188, 0, 1, 0, 5, 0, 3883, 0, 0, 0, 3, 638, 10, 10, 2922, 571, 454, 0, 0, 0, 7, 3907, 0, 0, 0, 0, 2344, 1170, 0, 0, 0, 0, 0, 0, 3928, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3632, 201, 0, 0, 12, 3358, 395, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 3889, 4608, 3914, 3514, 5168, 3834, 3765] avg=4096.0000 max=5168 max_rank=5 min=3514 min_rank=4 imbalance_ratio=1.261719 max_load_reduction=27600 max_load_reduction_pct=0.842285 moved_to_echo=16281 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[49, 0, 0, 0, 2225, 733, 184, 0, 0, 0, 0, 0, 3894, 0, 0, 0, 0, 1, 113, 0, 2912, 583, 455, 0, 0, 0, 0, 3912, 0, 0, 0, 0, 3934, 1174, 0, 0, 0, 0, 0, 0, 4082, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 3641, 202, 0, 0, 0, 3355, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3191, 3894, 4064, 3912, 5108, 5370, 3873, 3356] avg=4096.0000 max=5370 max_rank=5 min=3191 min_rank=0 imbalance_ratio=1.311035 max_load_reduction=27398 max_load_reduction_pct=0.836121 moved_to_echo=17707 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[490, 0, 0, 0, 2196, 734, 184, 0, 0, 0, 0, 17, 3894, 0, 0, 0, 0, 0, 0, 1, 2919, 576, 454, 0, 135, 0, 0, 3903, 0, 0, 0, 0, 4092, 1177, 0, 0, 0, 0, 0, 0, 3519, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3641, 202, 0, 0, 0, 3362, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3604, 3911, 3950, 4038, 5269, 4790, 3843, 3363] avg=4096.0000 max=5269 max_rank=4 min=3363 min_rank=7 imbalance_ratio=1.286377 max_load_reduction=27499 max_load_reduction_pct=0.839203 moved_to_echo=17265 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 128, 128, 128, 17152, 5888, 1536, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[22912, 9984, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1152, 512, 1280, 21888, 4608, 3712, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[23936, 8832, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 26752, 6144, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 1024, 1536, 128, 28800, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 128, 1536, 31104, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 1408, 31232, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[2, 4096, 4096, 0, 0, 0, 4091, 4096, 8, 0, 3713, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4008, 3921, 0, 0, 22, 399, 0, 187, 4096, 0, 21, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[756, 3638, 4065, 435, 11, 1, 4087, 4013, 0, 0, 3259, 25, 274, 0, 0, 0, 40, 8, 4, 0, 4096, 3061, 0, 0, 10, 21, 365, 91, 4039, 0, 469, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4091, 3057, 1, 0, 89, 4006, 4056, 0, 0, 4085, 0, 382, 0, 0, 0, 69, 0, 13, 3, 4096, 8, 0, 0, 1071, 3122, 27, 509, 4083, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4095, 3882, 0, 0, 29, 4095, 4096, 1159, 1, 110, 0, 38, 0, 0, 5, 10, 0, 0, 0, 3553, 4096, 0, 0, 5, 1953, 736, 1, 4094, 0, 810, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4091, 4096, 0, 1, 4, 118, 4096, 0, 0, 3582, 0, 78, 0, 1, 0, 2, 0, 0, 0, 4096, 3751, 0, 0, 9, 4083, 0, 664, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 3954, 4073, 0, 0, 0, 0, 4085, 373, 1, 2440, 0, 2, 0, 0, 1, 7, 18, 0, 26, 4089, 4089, 0, 0, 111, 3023, 0, 2388, 4087, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4080, 4096, 0, 0, 0, 4086, 4096, 0, 7, 1186, 6, 0, 0, 2, 0, 0, 0, 0, 0, 4043, 2854, 9, 0, 0, 81, 6, 4079, 4085, 0, 52, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 3169, 4096, 926, 0, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 6, 1, 6, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 3057, 1, 0, 0, 0, 0, 0, 89, 4006, 1117, 0, 0, 0, 0, 0, 0, 4085, 0, 812, 157, 0, 0, 382, 0, 0, 0, 3147, 910, 0, 0, 69, 0, 13, 3, 4076, 68, 0, 0, 881, 8, 0, 0, 0, 0, 0, 0, 1071, 3122, 27, 509, 1217, 0, 0, 0, 3926, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3073, 5212, 5054, 4439, 4229, 889, 5946, 3926] avg=4096.0000 max=5946 max_rank=6 min=889 min_rank=5 imbalance_ratio=1.451660 max_load_reduction=26822 max_load_reduction_pct=0.818542 moved_to_echo=14990 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 3882, 0, 0, 0, 0, 0, 0, 29, 4095, 1128, 0, 0, 0, 0, 1159, 1, 110, 0, 820, 157, 0, 0, 38, 0, 0, 5, 2730, 919, 0, 0, 10, 0, 0, 0, 4080, 59, 0, 0, 764, 4096, 0, 0, 0, 0, 0, 0, 5, 1953, 736, 1, 1229, 0, 0, 0, 3937, 0, 810, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3897, 5252, 2247, 3692, 4149, 4860, 3924, 4747] avg=4096.0000 max=5252 max_rank=1 min=2247 min_rank=2 imbalance_ratio=1.282227 max_load_reduction=27516 max_load_reduction_pct=0.839722 moved_to_echo=17680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 4096, 0, 0, 0, 0, 0, 1, 4, 118, 1128, 0, 0, 0, 0, 0, 0, 3582, 0, 820, 158, 0, 0, 78, 0, 1, 0, 3147, 919, 0, 0, 2, 0, 0, 0, 4076, 68, 0, 0, 881, 3751, 0, 0, 0, 0, 0, 0, 9, 4083, 0, 664, 1229, 0, 0, 0, 3938, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 1251, 4560, 4145, 4146, 4632, 5985, 3938] avg=4096.0000 max=5985 max_rank=6 min=1251 min_rank=1 imbalance_ratio=1.461182 max_load_reduction=26783 max_load_reduction_pct=0.817352 moved_to_echo=18701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[756, 13, 4065, 435, 0, 0, 0, 0, 11, 1, 4087, 1106, 0, 0, 0, 0, 0, 0, 3259, 25, 803, 155, 0, 0, 274, 0, 0, 0, 3147, 900, 0, 0, 40, 8, 4, 0, 3625, 68, 0, 0, 881, 3061, 0, 0, 0, 0, 0, 0, 10, 21, 365, 91, 1204, 0, 0, 0, 3884, 0, 469, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5269, 5205, 4242, 4321, 3745, 3942, 1691, 4353] avg=4096.0000 max=5269 max_rank=0 min=1691 min_rank=6 imbalance_ratio=1.286377 max_load_reduction=27499 max_load_reduction_pct=0.839203 moved_to_echo=13731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 4086, 1128, 0, 0, 0, 0, 0, 7, 1186, 6, 820, 157, 0, 0, 0, 0, 2, 0, 3106, 919, 0, 0, 0, 0, 0, 0, 4065, 68, 0, 0, 869, 2854, 9, 0, 0, 0, 0, 0, 0, 81, 6, 4079, 1229, 0, 0, 0, 3928, 0, 52, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 5214, 2176, 4027, 4133, 3732, 5395, 3980] avg=4096.0000 max=5395 max_rank=6 min=2176 min_rank=2 imbalance_ratio=1.317139 max_load_reduction=27373 max_load_reduction_pct=0.835358 moved_to_echo=17240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 1118, 0, 0, 0, 0, 373, 1, 2440, 0, 821, 161, 0, 0, 2, 0, 0, 1, 3144, 920, 0, 0, 7, 18, 0, 26, 3944, 74, 0, 0, 871, 4089, 0, 0, 0, 0, 0, 0, 111, 3023, 0, 2388, 1226, 0, 0, 0, 3926, 0, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 1118, 3796, 4067, 4069, 4960, 6748, 3927] avg=4096.0000 max=6748 max_rank=6 min=1118 min_rank=1 imbalance_ratio=1.647461 max_load_reduction=26020 max_load_reduction_pct=0.794067 moved_to_echo=19704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 1128, 0, 0, 0, 0, 8, 0, 3713, 0, 820, 158, 0, 0, 12, 0, 0, 0, 3079, 919, 0, 0, 0, 0, 0, 0, 4081, 67, 0, 0, 862, 3921, 0, 0, 0, 0, 0, 0, 22, 399, 0, 187, 1229, 0, 0, 0, 3938, 0, 21, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 5219, 4699, 4010, 4148, 4783, 1837, 3959] avg=4096.0000 max=5219 max_rank=1 min=1837 min_rank=6 imbalance_ratio=1.274170 max_load_reduction=27549 max_load_reduction_pct=0.840729 moved_to_echo=14727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 3169, 1128, 0, 0, 0, 0, 926, 0, 4090, 0, 820, 158, 0, 0, 1, 0, 0, 0, 3147, 919, 0, 0, 0, 0, 0, 0, 4081, 68, 0, 0, 881, 4089, 0, 0, 0, 0, 0, 0, 0, 6, 1, 6, 1229, 0, 0, 0, 3938, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4297, 5994, 4067, 4149, 4970, 1242, 3938] avg=4096.0000 max=5994 max_rank=2 min=1242 min_rank=6 imbalance_ratio=1.463379 max_load_reduction=26774 max_load_reduction_pct=0.817078 moved_to_echo=14299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[896, 0, 128, 128, 24704, 7424, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 12800, 1152, 7936, 9856, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 23680, 9088, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2560, 128, 22528, 128, 6656, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6912, 25984, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[768, 128, 31488, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31488, 0, 1408, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32128, 640, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4090, 343, 4, 0, 0, 1, 730, 2, 0, 0, 0, 4091, 4093, 0, 3882, 3, 159, 0, 4073, 0, 4094, 0, 0, 2, 7, 0, 2, 3103, 0, 4089, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 3475, 0, 0, 2, 0, 0, 522, 34, 4, 1, 1, 4092, 4094, 5, 4096, 0, 21, 0, 4096, 11, 4092, 0, 0, 0, 0, 0, 6, 4003, 0, 4083, 130] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4096, 4096, 0, 4040, 0, 0, 414, 4096, 0, 4096, 0, 0, 0, 99, 0, 0, 3643, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[10, 4093, 0, 2, 0, 1, 0, 0, 0, 0, 2, 0, 4086, 4051, 28, 3784, 128, 1, 0, 4096, 0, 4095, 46, 0, 0, 182, 1, 3, 4093, 0, 4066, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[19, 3791, 565, 0, 56, 0, 0, 3991, 208, 0, 0, 12, 3719, 4085, 36, 6, 1, 209, 3, 4046, 0, 3736, 25, 0, 0, 4069, 69, 15, 59, 8, 4038, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 53, 0, 0, 57, 44, 0, 0, 4068, 4094, 4, 3655, 165, 0, 22, 3593, 0, 4091, 0, 0, 19, 674, 0, 0, 4043, 16, 4074, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3910, 0, 0, 0, 16, 0, 215, 11, 0, 0, 0, 4088, 4096, 0, 696, 7, 2, 111, 3885, 45, 4095, 0, 0, 0, 3635, 0, 0, 3860, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 91, 0, 4, 0, 0, 4093, 4096, 0, 25, 2, 3, 0, 4095, 0, 4092, 0, 0, 0, 4086, 0, 0, 3988, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 343, 4, 24, 0, 0, 0, 0, 0, 1, 730, 3352, 39, 0, 0, 2, 0, 0, 0, 4054, 0, 0, 0, 1583, 0, 0, 3882, 0, 0, 0, 0, 3, 159, 0, 4016, 0, 0, 0, 0, 0, 4094, 0, 0, 31, 0, 0, 0, 2, 7, 0, 2, 2508, 7, 2, 0, 3103, 0, 730, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4461, 4122, 4056, 5465, 4178, 4125, 2528, 3833] avg=4096.0000 max=5465 max_rank=3 min=2528 min_rank=6 imbalance_ratio=1.334229 max_load_reduction=27303 max_load_reduction_pct=0.833221 moved_to_echo=14664 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[10, 4093, 0, 2, 24, 0, 0, 0, 0, 1, 0, 0, 3333, 39, 0, 0, 0, 0, 2, 0, 4012, 0, 0, 0, 1582, 0, 28, 3784, 0, 0, 0, 0, 128, 1, 0, 4038, 0, 0, 0, 0, 0, 4095, 46, 0, 32, 0, 0, 0, 0, 182, 1, 3, 2504, 7, 2, 0, 4093, 0, 726, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 3373, 4014, 5394, 4167, 4173, 2699, 4819] avg=4096.0000 max=5394 max_rank=3 min=2699 min_rank=6 imbalance_ratio=1.316895 max_load_reduction=27374 max_load_reduction_pct=0.835388 moved_to_echo=15858 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 21, 0, 0, 0, 0, 53, 0, 0, 3340, 40, 0, 0, 57, 44, 0, 0, 4054, 0, 0, 0, 1575, 0, 4, 3655, 0, 0, 0, 0, 165, 0, 22, 3542, 0, 0, 0, 0, 0, 4091, 0, 0, 28, 0, 0, 0, 19, 674, 0, 0, 2493, 7, 2, 0, 4043, 16, 727, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 3433, 4155, 5234, 3729, 4119, 3195, 4786] avg=4096.0000 max=5234 max_rank=3 min=3195 min_rank=6 imbalance_ratio=1.277832 max_load_reduction=27534 max_load_reduction_pct=0.840271 moved_to_echo=15829 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[19, 3791, 565, 0, 24, 0, 0, 0, 56, 0, 0, 3991, 3310, 40, 0, 0, 208, 0, 0, 12, 4045, 0, 0, 0, 1440, 0, 36, 6, 0, 0, 0, 0, 1, 209, 3, 3989, 0, 0, 0, 0, 0, 3736, 25, 0, 31, 0, 0, 0, 0, 4069, 69, 15, 2279, 7, 2, 0, 59, 8, 721, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4399, 7397, 4265, 1482, 4202, 3792, 6441, 790] avg=4096.0000 max=7397 max_rank=1 min=790 min_rank=7 imbalance_ratio=1.805908 max_load_reduction=25371 max_load_reduction_pct=0.774261 moved_to_echo=15225 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3910, 0, 0, 23, 0, 0, 0, 0, 16, 0, 215, 3358, 40, 0, 0, 11, 0, 0, 0, 4056, 0, 0, 0, 1582, 0, 0, 696, 0, 0, 0, 0, 7, 2, 111, 3830, 0, 0, 0, 0, 45, 4095, 0, 0, 30, 0, 0, 0, 0, 3635, 0, 0, 2506, 7, 2, 0, 3860, 0, 731, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3933, 3629, 4067, 2278, 3950, 4170, 6150, 4591] avg=4096.0000 max=6150 max_rank=6 min=2278 min_rank=3 imbalance_ratio=1.501465 max_load_reduction=26618 max_load_reduction_pct=0.812317 moved_to_echo=18861 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3475, 0, 0, 30, 0, 0, 0, 2, 0, 0, 522, 3351, 40, 0, 0, 34, 4, 1, 1, 4054, 0, 0, 0, 1580, 0, 5, 4096, 0, 0, 0, 0, 0, 21, 0, 4027, 0, 0, 0, 0, 11, 4092, 0, 0, 34, 0, 0, 0, 0, 0, 0, 6, 2512, 9, 5, 0, 4003, 0, 723, 130, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3505, 3915, 4094, 5681, 4048, 4137, 2532, 4856] avg=4096.0000 max=5681 max_rank=3 min=2532 min_rank=6 imbalance_ratio=1.386963 max_load_reduction=27087 max_load_reduction_pct=0.826630 moved_to_echo=15573 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 24, 0, 0, 0, 0, 0, 0, 12, 3358, 40, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 1585, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 414, 4038, 0, 0, 0, 0, 0, 4096, 0, 0, 32, 0, 0, 0, 0, 99, 0, 0, 2511, 7, 2, 0, 3643, 0, 731, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 3410, 4056, 5625, 4452, 4128, 2619, 4374] avg=4096.0000 max=5625 max_rank=3 min=2619 min_rank=6 imbalance_ratio=1.373291 max_load_reduction=27143 max_load_reduction_pct=0.828339 moved_to_echo=15573 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 24, 0, 0, 0, 0, 1, 0, 91, 3358, 39, 0, 0, 0, 4, 0, 0, 4057, 0, 0, 0, 1584, 0, 0, 25, 0, 0, 0, 0, 2, 3, 0, 4037, 0, 0, 0, 0, 0, 4092, 0, 0, 32, 0, 0, 0, 0, 4086, 0, 0, 2509, 7, 2, 0, 3988, 0, 731, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 3489, 4061, 1609, 4042, 4124, 6604, 4719] avg=4096.0000 max=6604 max_rank=6 min=1609 min_rank=3 imbalance_ratio=1.612305 max_load_reduction=26164 max_load_reduction_pct=0.798462 moved_to_echo=19489 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 12800, 128, 128, 19840, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 5632, 26880, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[26880, 128, 5888, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[12544, 0, 128, 20224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 512, 640, 31616, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32512, 128, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 31744, 1024, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 128, 128, 128, 32512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[3907, 1, 3441, 0, 0, 476, 3660, 4059, 0, 19, 2, 0, 0, 4092, 0, 0, 4092, 0, 4096, 0, 0, 14, 0, 0, 796, 0, 0, 0, 17, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4021, 0, 0, 6, 4085, 3942, 71, 0, 0, 1365, 0, 2782, 0, 0, 4091, 0, 4075, 0, 7, 0, 93, 0, 4092, 0, 0, 0, 42, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3698, 26, 457, 0, 0, 0, 3807, 3978, 3, 0, 0, 403, 0, 4094, 4, 0, 4096, 0, 3559, 0, 0, 124, 0, 0, 4093, 0, 0, 95, 223, 12, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4067, 29, 3773, 0, 0, 3, 4096, 4080, 0, 0, 51, 1018, 0, 202, 0, 41, 4096, 0, 3075, 149, 34, 10, 0, 0, 4000, 0, 1, 0, 0, 0, 14, 4029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[79, 0, 4096, 0, 0, 91, 3850, 4093, 0, 1, 0, 60, 0, 4066, 0, 0, 4091, 0, 4095, 0, 0, 148, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4068, 9, 0, 7, 4015, 4086, 0, 86, 0, 28, 0, 4089, 0, 6, 4062, 0, 4074, 0, 0, 0, 1, 0, 4060, 1, 25, 34, 0, 0, 26, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4087, 57, 3934, 162, 0, 809, 4096, 196, 4, 0, 0, 3973, 0, 4035, 0, 0, 2463, 0, 421, 0, 0, 0, 5, 307, 4085, 0, 0, 38, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[208, 0, 3822, 0, 0, 7, 4096, 4085, 0, 0, 0, 1, 2, 4033, 0, 1, 4095, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 130, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3907, 1, 2050, 0, 0, 0, 0, 0, 0, 476, 330, 4059, 0, 0, 0, 0, 0, 19, 2, 0, 2999, 360, 0, 0, 0, 4092, 0, 0, 663, 0, 0, 0, 676, 0, 4096, 0, 0, 0, 0, 0, 0, 14, 0, 0, 3330, 417, 0, 0, 796, 0, 0, 0, 368, 49, 0, 0, 17, 0, 0, 4047, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5958, 4865, 3380, 4755, 4772, 3761, 1213, 4064] avg=4096.0000 max=5958 max_rank=0 min=1213 min_rank=6 imbalance_ratio=1.454590 max_load_reduction=26810 max_load_reduction_pct=0.818176 moved_to_echo=13810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2396, 0, 0, 0, 0, 0, 0, 6, 368, 3942, 0, 0, 0, 0, 71, 0, 0, 1365, 2998, 421, 0, 0, 0, 2782, 0, 0, 774, 0, 0, 0, 676, 0, 4075, 0, 0, 0, 0, 0, 7, 0, 93, 0, 3717, 417, 0, 0, 4092, 0, 0, 0, 430, 49, 0, 0, 42, 0, 1, 4046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2396, 4316, 4855, 3556, 4751, 4234, 4571, 4089] avg=4096.0000 max=4855 max_rank=2 min=2396 min_rank=0 imbalance_ratio=1.185303 max_load_reduction=27913 max_load_reduction_pct=0.851837 moved_to_echo=17645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3698, 26, 274, 0, 0, 0, 0, 0, 0, 0, 343, 3978, 0, 0, 0, 0, 3, 0, 0, 403, 3002, 47, 0, 0, 0, 4094, 4, 0, 88, 0, 0, 0, 677, 0, 3559, 0, 0, 0, 0, 0, 0, 124, 0, 0, 3464, 417, 0, 0, 4093, 0, 0, 95, 48, 49, 0, 0, 223, 12, 0, 4047, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 4321, 3455, 4186, 4236, 4005, 4285, 4282] avg=4096.0000 max=4321 max_rank=1 min=3455 min_rank=2 imbalance_ratio=1.054932 max_load_reduction=28447 max_load_reduction_pct=0.868134 moved_to_echo=16808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4067, 29, 2247, 0, 0, 0, 0, 0, 0, 3, 369, 4080, 0, 0, 0, 0, 0, 0, 51, 1018, 3002, 395, 0, 0, 0, 202, 0, 41, 727, 0, 0, 0, 677, 0, 3075, 149, 0, 0, 0, 0, 34, 10, 0, 0, 3727, 417, 0, 0, 4000, 0, 1, 0, 404, 48, 0, 0, 0, 0, 14, 3981, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6343, 4452, 4466, 970, 3901, 4188, 4453, 3995] avg=4096.0000 max=6343 max_rank=0 min=970 min_rank=3 imbalance_ratio=1.548584 max_load_reduction=26425 max_load_reduction_pct=0.806427 moved_to_echo=16537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4087, 57, 2343, 162, 0, 0, 0, 0, 0, 809, 369, 196, 0, 0, 0, 0, 4, 0, 0, 3973, 1805, 412, 0, 0, 0, 4035, 0, 0, 758, 0, 0, 0, 407, 0, 421, 0, 0, 0, 0, 0, 0, 0, 5, 307, 3727, 251, 0, 0, 4085, 0, 0, 38, 421, 49, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6649, 1374, 6194, 4793, 828, 4290, 4593, 4047] avg=4096.0000 max=6649 max_rank=0 min=828 min_rank=4 imbalance_ratio=1.623291 max_load_reduction=26119 max_load_reduction_pct=0.797089 moved_to_echo=13758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2411, 9, 0, 0, 0, 0, 0, 7, 355, 4086, 0, 0, 0, 0, 0, 86, 0, 28, 2983, 431, 0, 0, 0, 4089, 0, 6, 786, 0, 0, 0, 662, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3660, 417, 0, 0, 4060, 1, 25, 34, 440, 50, 0, 0, 0, 0, 26, 4041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2420, 4448, 3528, 4881, 4736, 4078, 4610, 4067] avg=4096.0000 max=4881 max_rank=3 min=2420 min_rank=0 imbalance_ratio=1.191650 max_load_reduction=27887 max_load_reduction_pct=0.851044 moved_to_echo=17491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[79, 0, 2440, 0, 0, 0, 0, 0, 0, 91, 347, 4093, 0, 0, 0, 0, 0, 1, 0, 60, 2998, 429, 0, 0, 0, 4066, 0, 0, 789, 0, 0, 0, 676, 0, 4095, 0, 0, 0, 0, 0, 0, 148, 0, 0, 3503, 417, 0, 0, 4002, 0, 0, 0, 438, 49, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2519, 4531, 3488, 4855, 4771, 4068, 4489, 4047] avg=4096.0000 max=4855 max_rank=3 min=2519 min_rank=0 imbalance_ratio=1.185303 max_load_reduction=27913 max_load_reduction_pct=0.851837 moved_to_echo=17375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[208, 0, 2277, 0, 0, 0, 0, 0, 0, 7, 369, 4085, 0, 0, 0, 0, 0, 0, 0, 1, 3001, 400, 0, 0, 2, 4033, 0, 1, 736, 0, 0, 0, 677, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3727, 417, 0, 0, 4096, 0, 0, 0, 409, 49, 0, 0, 0, 130, 0, 4047, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2485, 4461, 3402, 4772, 4773, 4144, 4554, 4177] avg=4096.0000 max=4773 max_rank=4 min=2485 min_rank=0 imbalance_ratio=1.165283 max_load_reduction=27995 max_load_reduction_pct=0.854340 moved_to_echo=17648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16128, 128, 16512, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 6912, 22912, 2944, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5248, 0, 27520, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 1408, 2944, 28544, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 27520, 128, 128, 5376, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 256, 128, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 384, 128, 384, 28928, 3200, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[29312, 128, 128, 256, 3072, 512, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4095, 33, 0, 1, 2298, 1, 4, 4094, 4088, 0, 226, 8, 0, 3, 758, 0, 0, 0, 1, 3148, 0, 1367, 0, 0, 4, 4065, 4010, 0, 467, 0, 1, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3936, 0, 0, 4096, 4096, 0, 541, 0, 77, 7, 953, 0, 0, 0, 0, 3723, 22, 0, 0, 0, 0, 37, 4066, 0, 3059, 0, 0, 4059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4090, 10, 0, 0, 3905, 39, 151, 3435, 269, 0, 4077, 52, 0, 3, 6, 0, 0, 16, 0, 427, 3361, 3527, 0, 404, 0, 4050, 847, 0, 43, 11, 0, 4045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 1353, 0, 0, 4049, 3951, 0, 4028, 950, 0, 0, 1, 0, 3, 37, 0, 50, 111, 4033, 0, 1, 0, 4077, 1341, 4, 632, 1, 0, 4051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4091, 2, 0, 0, 0, 117, 252, 4071, 3926, 0, 1450, 402, 0, 0, 65, 3, 0, 14, 15, 3693, 874, 441, 0, 22, 0, 4095, 3930, 0, 1162, 56, 2, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[3913, 1, 152, 0, 3, 1, 51, 4096, 4088, 79, 2162, 0, 0, 0, 889, 0, 0, 452, 1, 3911, 0, 0, 0, 18, 12, 954, 4096, 235, 3998, 0, 0, 3656] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1050, 0, 0, 4096, 3813, 0, 1418, 688, 1, 0, 1775, 0, 0, 1, 0, 72, 2379, 992, 0, 0, 0, 4095, 4096, 0, 100, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3327, 0, 0, 4094, 4096, 0, 470, 936, 0, 0, 4, 1, 0, 3, 0, 6, 57, 4, 0, 0, 0, 4083, 4095, 0, 3402, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 33, 0, 1, 0, 0, 0, 0, 2298, 1, 4, 2082, 0, 0, 0, 0, 2206, 0, 226, 8, 0, 0, 0, 0, 0, 3, 758, 0, 2941, 1124, 0, 0, 0, 0, 1, 3148, 1478, 888, 0, 0, 0, 1367, 0, 0, 1571, 404, 0, 0, 4, 4065, 1069, 0, 0, 0, 0, 0, 467, 0, 1, 2525, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4385, 2440, 4826, 5515, 3342, 5138, 2993] avg=4096.0000 max=5515 max_rank=4 min=2440 min_rank=2 imbalance_ratio=1.346436 max_load_reduction=27253 max_load_reduction_pct=0.831696 moved_to_echo=16988 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4090, 10, 0, 0, 0, 0, 0, 0, 3905, 39, 151, 1747, 0, 0, 0, 0, 146, 0, 4077, 52, 0, 0, 0, 0, 0, 3, 6, 0, 621, 943, 0, 0, 0, 16, 0, 427, 97, 745, 0, 0, 3361, 3527, 0, 404, 1552, 26, 0, 0, 0, 4050, 226, 0, 0, 0, 0, 0, 43, 11, 0, 2493, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 5842, 4275, 1573, 1285, 8870, 4276, 2547] avg=4096.0000 max=8870 max_rank=5 min=1285 min_rank=4 imbalance_ratio=2.165527 max_load_reduction=23898 max_load_reduction_pct=0.729309 moved_to_echo=16978 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 0, 2082, 0, 0, 0, 0, 2211, 0, 541, 0, 0, 0, 0, 0, 77, 7, 953, 0, 2982, 1125, 0, 0, 0, 0, 0, 3723, 1480, 889, 0, 0, 22, 0, 0, 0, 1557, 405, 0, 0, 0, 37, 1084, 0, 0, 0, 0, 0, 3059, 0, 0, 2502, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 6018, 2752, 5144, 6092, 1984, 1121, 5561] avg=4096.0000 max=6092 max_rank=4 min=1121 min_rank=6 imbalance_ratio=1.487305 max_load_reduction=26676 max_load_reduction_pct=0.814087 moved_to_echo=14758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3913, 1, 152, 0, 0, 0, 0, 0, 3, 1, 51, 2082, 0, 0, 0, 0, 2206, 79, 2162, 0, 0, 0, 0, 0, 0, 0, 889, 0, 3004, 1125, 0, 0, 0, 452, 1, 3911, 1478, 889, 0, 0, 0, 0, 0, 18, 1402, 404, 0, 0, 12, 954, 1092, 235, 0, 0, 0, 0, 3998, 0, 0, 2254, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 2137, 4447, 5018, 6731, 1824, 2293, 6252] avg=4096.0000 max=6731 max_rank=4 min=1824 min_rank=5 imbalance_ratio=1.643311 max_load_reduction=26037 max_load_reduction_pct=0.794586 moved_to_echo=17100 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4091, 2, 0, 0, 0, 0, 0, 0, 0, 117, 252, 2070, 0, 0, 0, 0, 2119, 0, 1450, 402, 0, 0, 0, 0, 0, 0, 65, 3, 2882, 1118, 0, 0, 0, 14, 15, 3693, 1419, 883, 0, 0, 874, 441, 0, 22, 1567, 388, 0, 0, 0, 4095, 1048, 0, 0, 0, 0, 0, 1162, 56, 2, 2518, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 2439, 3971, 4068, 6024, 3292, 5143, 3738] avg=4096.0000 max=6024 max_rank=4 min=2439 min_rank=1 imbalance_ratio=1.470703 max_load_reduction=26744 max_load_reduction_pct=0.816162 moved_to_echo=18197 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 0, 0, 1353, 0, 0, 2053, 0, 0, 0, 0, 2124, 0, 4028, 950, 0, 0, 0, 0, 0, 0, 1, 0, 987, 1114, 0, 0, 3, 37, 0, 50, 1432, 882, 0, 0, 111, 4033, 0, 1, 1558, 395, 3, 0, 0, 4077, 354, 4, 0, 0, 0, 0, 632, 1, 0, 2493, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 3406, 7102, 2102, 2404, 6101, 4435, 3126] avg=4096.0000 max=7102 max_rank=2 min=2102 min_rank=3 imbalance_ratio=1.733887 max_load_reduction=25666 max_load_reduction_pct=0.783264 moved_to_echo=16066 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 2082, 0, 0, 0, 0, 2058, 0, 1418, 688, 0, 0, 0, 0, 1, 0, 1775, 0, 3004, 1125, 0, 0, 0, 1, 0, 72, 1378, 889, 0, 0, 2379, 992, 0, 0, 1571, 377, 0, 0, 0, 4095, 1092, 0, 0, 0, 0, 0, 100, 0, 0, 2525, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3132, 4164, 5905, 2340, 5319, 5187, 2625] avg=4096.0000 max=5905 max_rank=3 min=2340 min_rank=4 imbalance_ratio=1.441650 max_load_reduction=26863 max_load_reduction_pct=0.819794 moved_to_echo=15471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 3327, 0, 0, 2082, 0, 0, 0, 0, 2211, 0, 470, 936, 0, 0, 0, 0, 0, 0, 4, 1, 3003, 1124, 0, 0, 0, 3, 0, 6, 1480, 888, 0, 0, 57, 4, 0, 0, 1570, 405, 0, 0, 0, 4083, 1092, 0, 0, 0, 0, 0, 3402, 0, 0, 2524, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 5409, 3617, 4132, 2377, 2036, 5175, 5926] avg=4096.0000 max=5926 max_rank=7 min=2036 min_rank=5 imbalance_ratio=1.446777 max_load_reduction=26842 max_load_reduction_pct=0.819153 moved_to_echo=15514 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6912, 10368, 0, 512, 12416, 2816, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 4480, 128, 19456, 8832, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15872, 256, 512, 16384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 640, 128, 15104, 10368, 7040, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15360, 128, 14464, 3072, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12928, 128, 128, 19840, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 25472, 7168, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[14, 0, 3, 0, 0, 0, 80, 14, 0, 3899, 113, 4083, 0, 0, 23, 0, 4048, 3630, 4086, 0, 3941, 23, 10, 4082, 0, 0, 4, 658, 4007, 1, 47, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[418, 48, 31, 14, 73, 0, 0, 302, 0, 3677, 0, 3952, 242, 27, 0, 0, 4090, 4022, 4048, 0, 3671, 0, 0, 3986, 82, 5, 2, 0, 4072, 0, 1, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 481, 0, 8, 0, 0, 4079, 3454, 0, 3, 0, 4095, 4096, 4084, 0, 4096, 0, 6, 4096, 251, 0, 0, 0, 3058, 1, 0, 958] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 13, 0, 0, 0, 2, 0, 0, 1, 807, 226, 3871, 2, 0, 0, 0, 4095, 4094, 4076, 0, 4043, 0, 84, 4096, 1, 0, 0, 0, 4080, 3277, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 73, 0, 0, 0, 0, 5, 1248, 0, 153, 2, 4096, 19, 4, 3, 0, 4096, 4093, 4094, 0, 4029, 0, 0, 4090, 0, 0, 0, 0, 3905, 2858, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[43, 4, 33, 2, 0, 0, 4, 64, 0, 1921, 2, 4089, 0, 12, 0, 0, 4094, 4096, 4079, 0, 3379, 1, 0, 4088, 2501, 0, 4, 9, 3994, 349, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3, 0, 3550, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 4096, 532, 0, 0, 1, 4096, 10, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 4096, 27, 0, 0, 1, 4096, 4096, 4096, 0, 4096, 0, 0, 4096, 336, 0, 0, 0, 4037, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 3, 0, 3969, 0, 0, 0, 0, 0, 80, 14, 3734, 79, 0, 0, 0, 3899, 113, 1845, 0, 0, 0, 0, 0, 0, 23, 0, 3338, 283, 0, 0, 0, 3630, 69, 0, 0, 0, 0, 0, 3941, 23, 10, 163, 0, 0, 0, 0, 0, 0, 4, 658, 2238, 768, 581, 0, 3239, 1, 47, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3986, 3907, 5857, 3644, 3699, 4137, 4249, 3289] avg=4096.0000 max=5857 max_rank=2 min=3289 min_rank=7 imbalance_ratio=1.429932 max_load_reduction=26911 max_load_reduction_pct=0.821259 moved_to_echo=15374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[418, 48, 31, 14, 4010, 0, 0, 0, 73, 0, 0, 302, 3700, 80, 0, 0, 0, 3677, 0, 1786, 0, 0, 0, 0, 242, 27, 0, 0, 3260, 280, 0, 0, 0, 4022, 68, 0, 0, 0, 0, 0, 3671, 0, 0, 159, 0, 0, 0, 0, 82, 5, 2, 0, 2166, 780, 567, 0, 3292, 0, 1, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4521, 4155, 5463, 3809, 4090, 3830, 3602, 3298] avg=4096.0000 max=5463 max_rank=2 min=3298 min_rank=7 imbalance_ratio=1.333740 max_load_reduction=27305 max_load_reduction_pct=0.833282 moved_to_echo=14820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 481, 0, 3732, 80, 0, 0, 8, 0, 0, 1843, 0, 0, 0, 0, 3454, 0, 3, 0, 3350, 282, 0, 0, 0, 4096, 70, 0, 0, 0, 0, 0, 4096, 0, 6, 163, 0, 0, 0, 0, 251, 0, 0, 0, 2236, 586, 583, 0, 2472, 1, 0, 958, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4293, 1851, 7089, 4166, 4265, 3656, 3431] avg=4096.0000 max=7089 max_rank=3 min=1851 min_rank=2 imbalance_ratio=1.730713 max_load_reduction=25679 max_load_reduction_pct=0.783661 moved_to_echo=15518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 73, 0, 0, 4015, 0, 0, 0, 0, 0, 5, 1248, 3742, 81, 0, 0, 0, 153, 2, 1851, 0, 0, 0, 0, 19, 4, 3, 0, 3345, 283, 0, 0, 0, 4093, 69, 0, 0, 0, 0, 0, 4029, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 2245, 748, 582, 0, 3157, 2858, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 5076, 2006, 3654, 4162, 4192, 3575, 6015] avg=4096.0000 max=6015 max_rank=7 min=2006 min_rank=2 imbalance_ratio=1.468506 max_load_reduction=26753 max_load_reduction_pct=0.816437 moved_to_echo=17944 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 0, 0, 4015, 0, 0, 0, 0, 2, 0, 0, 3725, 80, 0, 0, 1, 807, 226, 1749, 0, 0, 0, 0, 2, 0, 0, 0, 3350, 282, 0, 0, 0, 4094, 69, 0, 0, 0, 0, 0, 4043, 0, 84, 163, 0, 0, 0, 0, 1, 0, 0, 0, 2122, 782, 583, 0, 3298, 3277, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 3807, 2783, 3634, 4163, 4290, 3488, 6575] avg=4096.0000 max=6575 max_rank=7 min=2783 min_rank=2 imbalance_ratio=1.605225 max_load_reduction=26193 max_load_reduction_pct=0.799347 moved_to_echo=18516 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[43, 4, 33, 2, 4014, 0, 0, 0, 0, 0, 4, 64, 3733, 80, 0, 0, 0, 1921, 2, 1842, 0, 0, 0, 0, 0, 12, 0, 0, 3346, 286, 0, 0, 0, 4096, 60, 0, 0, 0, 0, 0, 3379, 1, 0, 156, 0, 0, 0, 0, 2501, 0, 4, 9, 2247, 768, 586, 0, 3226, 349, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3881, 3765, 3644, 4156, 3536, 6115, 3575] avg=4096.0000 max=6115 max_rank=6 min=3536 min_rank=5 imbalance_ratio=1.492920 max_load_reduction=26653 max_load_reduction_pct=0.813385 moved_to_echo=17382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 3, 3743, 80, 0, 0, 0, 3550, 0, 1851, 0, 0, 0, 0, 0, 0, 0, 0, 3350, 283, 0, 0, 0, 4096, 70, 0, 0, 0, 0, 0, 4096, 0, 0, 163, 0, 0, 0, 0, 532, 0, 0, 1, 2245, 785, 583, 0, 3311, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 3826, 5401, 3633, 4166, 4259, 4146, 3321] avg=4096.0000 max=5401 max_rank=2 min=3321 min_rank=7 imbalance_ratio=1.318604 max_load_reduction=27367 max_load_reduction_pct=0.835175 moved_to_echo=15892 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 3743, 80, 0, 0, 0, 3791, 0, 1851, 0, 0, 0, 0, 27, 0, 0, 1, 3350, 283, 0, 0, 0, 4096, 70, 0, 0, 0, 0, 0, 4096, 0, 0, 163, 0, 0, 0, 0, 336, 0, 0, 0, 2245, 773, 583, 0, 3264, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 3823, 5642, 3661, 4166, 4259, 3937, 3264] avg=4096.0000 max=5642 max_rank=2 min=3264 min_rank=7 imbalance_ratio=1.377441 max_load_reduction=27126 max_load_reduction_pct=0.827820 moved_to_echo=15626 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 17920, 384, 14720, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3712, 128, 128, 768, 17792, 6016, 4736, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[25344, 6528, 128, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3840, 128, 128, 128, 26752, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 256, 128, 128, 32128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 640, 1664, 29952, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32256, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31360, 128, 128, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[11, 4091, 3104, 4003, 597, 0, 4083, 459, 0, 0, 0, 0, 1, 0, 0, 0, 0, 28, 0, 30, 4018, 4096, 0, 0, 2, 4094, 0, 0, 78, 4069, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4087, 3447, 4092, 0, 0, 3779, 0, 69, 0, 10, 1, 0, 0, 5, 0, 0, 0, 322, 3873, 224, 4096, 101, 94, 1, 4070, 398, 0, 4, 4090, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[2, 4096, 3827, 4096, 4, 1, 4043, 3, 0, 0, 10, 0, 0, 1, 33, 0, 428, 1481, 0, 296, 0, 4085, 0, 2, 0, 3496, 2742, 0, 0, 4035, 87, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3816, 3349, 3, 0, 4093, 338, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4039, 4096, 0, 0, 0, 4096, 974, 0, 6, 3850, 11, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 2483, 3976, 2240, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 197, 0, 0, 123, 0, 4, 799, 4084, 0, 0, 0, 4085, 1833, 0, 3513, 1214, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 4031, 3733, 2, 0, 4096, 31, 0, 12, 0, 0, 0, 18, 57, 0, 6, 43, 0, 3388, 4095, 4038, 0, 0, 17, 4059, 297, 2, 0, 747, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[6, 4060, 999, 4076, 1058, 0, 4049, 3908, 0, 7, 12, 5, 0, 0, 26, 0, 22, 0, 0, 178, 2172, 4091, 6, 0, 0, 4083, 39, 0, 1, 3949, 21, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 265, 4066, 0, 0, 4096, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1695, 0, 219, 1594, 4094, 0, 0, 0, 4096, 19, 0, 4088, 4041, 252, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 3104, 1375, 0, 0, 0, 0, 597, 0, 3025, 459, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 1, 0, 0, 0, 2628, 1471, 10, 0, 0, 28, 0, 30, 1058, 719, 666, 203, 4018, 1959, 0, 0, 0, 0, 0, 0, 2, 3375, 0, 0, 0, 0, 0, 0, 78, 3866, 0, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4490, 4081, 4081, 4110, 2704, 5977, 3377, 3948] avg=4096.0000 max=5977 max_rank=5 min=2704 min_rank=4 imbalance_ratio=1.459229 max_load_reduction=26791 max_load_reduction_pct=0.817596 moved_to_echo=16006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3447, 1405, 0, 0, 0, 0, 0, 0, 2800, 0, 0, 0, 0, 0, 69, 0, 10, 1, 4078, 0, 0, 0, 0, 0, 5, 0, 2687, 1471, 9, 0, 0, 0, 322, 3873, 979, 714, 666, 204, 224, 1959, 101, 94, 0, 0, 0, 0, 1, 3356, 398, 0, 0, 0, 0, 0, 4, 3886, 0, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4852, 2800, 4158, 4172, 6758, 2378, 3755, 3895] avg=4096.0000 max=6758 max_rank=4 min=2378 min_rank=5 imbalance_ratio=1.649902 max_load_reduction=26010 max_load_reduction_pct=0.793762 moved_to_echo=16786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3827, 1407, 0, 0, 0, 0, 4, 1, 2995, 3, 0, 0, 0, 0, 0, 0, 10, 0, 4086, 0, 0, 0, 0, 1, 33, 0, 2689, 1467, 10, 0, 428, 1481, 0, 296, 1048, 614, 664, 201, 0, 1954, 0, 2, 0, 0, 0, 0, 0, 2882, 2742, 0, 0, 0, 0, 0, 0, 3834, 87, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5236, 3003, 4096, 4200, 4732, 1956, 5624, 3921] avg=4096.0000 max=5624 max_rank=6 min=1956 min_rank=5 imbalance_ratio=1.373047 max_load_reduction=27144 max_load_reduction_pct=0.828369 moved_to_echo=16233 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4031, 1282, 0, 0, 0, 0, 2, 0, 3035, 31, 0, 0, 0, 0, 0, 12, 0, 0, 4086, 0, 0, 0, 0, 18, 57, 0, 2451, 1450, 10, 0, 6, 43, 0, 3388, 1061, 713, 656, 37, 4095, 1932, 0, 0, 0, 0, 0, 0, 17, 3346, 297, 2, 0, 0, 0, 0, 0, 710, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5313, 3068, 4098, 3986, 5904, 6027, 3662, 710] avg=4096.0000 max=6027 max_rank=5 min=710 min_rank=7 imbalance_ratio=1.471436 max_load_reduction=26741 max_load_reduction_pct=0.816071 moved_to_echo=16303 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2483, 1366, 0, 0, 0, 0, 2240, 0, 3035, 25, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 197, 0, 2610, 1467, 9, 0, 0, 123, 0, 4, 1061, 717, 664, 60, 799, 1953, 0, 0, 0, 0, 0, 0, 0, 3368, 1833, 0, 0, 0, 0, 0, 3513, 1154, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3849, 5300, 4087, 4283, 2629, 2752, 5201, 4667] avg=4096.0000 max=5300 max_rank=1 min=2629 min_rank=4 imbalance_ratio=1.293945 max_load_reduction=27468 max_load_reduction_pct=0.838257 moved_to_echo=15249 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 999, 1396, 0, 0, 0, 0, 1058, 0, 2995, 3908, 0, 0, 0, 0, 0, 7, 12, 5, 4051, 0, 0, 0, 0, 0, 26, 0, 2680, 1472, 9, 0, 22, 0, 0, 178, 1054, 720, 669, 199, 2172, 1950, 6, 0, 0, 0, 0, 0, 0, 3363, 39, 0, 0, 0, 0, 0, 1, 3750, 21, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2401, 7961, 4075, 4187, 2842, 4128, 3402, 3772] avg=4096.0000 max=7961 max_rank=1 min=2401 min_rank=0 imbalance_ratio=1.943604 max_load_reduction=24807 max_load_reduction_pct=0.757050 moved_to_echo=14144 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3816, 1150, 0, 0, 0, 0, 3, 0, 3032, 338, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 1, 0, 2199, 1471, 9, 0, 0, 0, 0, 0, 1061, 719, 666, 192, 4039, 1959, 0, 0, 0, 0, 0, 0, 0, 3377, 974, 0, 0, 0, 0, 0, 6, 3658, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4966, 3373, 4087, 3680, 2638, 5998, 4351, 3675] avg=4096.0000 max=5998 max_rank=5 min=2638 min_rank=4 imbalance_ratio=1.464355 max_load_reduction=26770 max_load_reduction_pct=0.816956 moved_to_echo=16662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 265, 1396, 0, 0, 0, 0, 0, 0, 3035, 147, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 2670, 1470, 10, 0, 0, 1695, 0, 219, 1061, 719, 666, 201, 1594, 1958, 0, 0, 0, 0, 0, 0, 0, 3377, 19, 0, 0, 0, 0, 0, 4088, 3840, 252, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1661, 3182, 4086, 4150, 4561, 3552, 3396, 8180] avg=4096.0000 max=8180 max_rank=7 min=1661 min_rank=0 imbalance_ratio=1.997070 max_load_reduction=24588 max_load_reduction_pct=0.750366 moved_to_echo=19689 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[512, 3456, 384, 8064, 8448, 5760, 5376, 1408] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 384, 0, 20736, 11776, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17024, 15744, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 22016, 10880, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 128, 24064, 4992, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7808, 24704, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 26496, 6400, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[33, 6, 4094, 0, 1662, 0, 4, 3998, 2698, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 3032, 0, 0, 0, 4080, 0, 0, 1, 13, 874, 4096, 4078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[39, 0, 3179, 3, 4074, 0, 0, 4092, 4061, 4, 0, 0, 0, 0, 2, 0, 4090, 0, 0, 58, 0, 256, 0, 85, 4010, 0, 0, 23, 0, 1163, 4034, 3595] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 46, 4063, 0, 4065, 0, 0, 3989, 4096, 0, 0, 19, 0, 7, 0, 91, 4091, 34, 0, 0, 5, 0, 68, 0, 4096, 0, 43, 0, 0, 0, 4090, 3965] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[352, 0, 4096, 0, 3787, 0, 0, 3421, 4095, 0, 0, 0, 0, 0, 0, 30, 4068, 0, 0, 8, 885, 0, 0, 0, 4091, 0, 0, 0, 7, 3832, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 29, 3996, 30, 1226, 0, 0, 3782, 4096, 69, 0, 2, 10, 0, 11, 0, 4091, 0, 0, 0, 1410, 25, 0, 5, 4065, 0, 0, 0, 196, 1596, 4096, 4033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[3, 0, 4090, 55, 997, 374, 0, 4051, 3624, 0, 0, 0, 10, 1, 0, 0, 4036, 29, 6, 0, 51, 28, 10, 0, 4079, 5, 0, 0, 11, 3160, 4069, 4079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 1467, 4095, 0, 2383, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 255, 4096, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3099, 3, 0, 4030, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 850, 0, 0, 0, 4096, 0, 3, 0, 3, 217, 4096, 4083] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[33, 6, 3966, 0, 0, 0, 0, 0, 1662, 0, 4, 1410, 0, 0, 0, 0, 2698, 0, 0, 3, 122, 125, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3032, 0, 0, 0, 2588, 821, 9, 0, 4080, 0, 0, 1, 6, 3, 0, 0, 13, 874, 0, 3132, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 3076, 2948, 4087, 4093, 6450, 4090, 4019] avg=4096.0000 max=6450 max_rank=5 min=2948 min_rank=2 imbalance_ratio=1.574707 max_load_reduction=26318 max_load_reduction_pct=0.803162 moved_to_echo=18652 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 46, 3936, 0, 0, 0, 0, 0, 4065, 0, 0, 1407, 0, 0, 0, 0, 4096, 0, 0, 19, 121, 121, 0, 0, 0, 7, 0, 91, 4081, 0, 0, 0, 4088, 34, 0, 0, 0, 0, 0, 0, 5, 0, 68, 0, 2582, 798, 9, 0, 4096, 0, 43, 0, 6, 3, 0, 0, 0, 0, 0, 3046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3982, 5472, 4357, 4179, 4122, 3462, 4148, 3046] avg=4096.0000 max=5472 max_rank=1 min=3046 min_rank=7 imbalance_ratio=1.335938 max_load_reduction=27296 max_load_reduction_pct=0.833008 moved_to_echo=14778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[39, 0, 3079, 3, 0, 0, 0, 0, 4074, 0, 0, 1443, 0, 0, 0, 0, 4061, 4, 0, 0, 95, 110, 0, 0, 0, 0, 2, 0, 4026, 0, 0, 0, 4087, 0, 0, 58, 0, 0, 0, 0, 0, 256, 0, 85, 2649, 724, 8, 0, 4010, 0, 0, 23, 5, 3, 0, 0, 0, 1163, 0, 2761, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3121, 5517, 4270, 4028, 4145, 3722, 4041, 3924] avg=4096.0000 max=5517 max_rank=1 min=3121 min_rank=0 imbalance_ratio=1.346924 max_load_reduction=27251 max_load_reduction_pct=0.831635 moved_to_echo=15832 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[352, 0, 3968, 0, 0, 0, 0, 0, 3787, 0, 0, 1207, 0, 0, 0, 0, 4095, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 30, 4088, 0, 0, 0, 4065, 0, 0, 8, 0, 0, 0, 0, 885, 0, 0, 0, 2214, 0, 8, 0, 4091, 0, 0, 0, 6, 3, 0, 0, 7, 3832, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4320, 4994, 4217, 4118, 4073, 3107, 4100, 3839] avg=4096.0000 max=4994 max_rank=1 min=3107 min_rank=5 imbalance_ratio=1.219238 max_load_reduction=27774 max_load_reduction_pct=0.847595 moved_to_echo=15119 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 29, 3871, 30, 0, 0, 0, 0, 1226, 0, 0, 1334, 0, 0, 0, 0, 4096, 69, 0, 2, 119, 123, 0, 0, 10, 0, 11, 0, 4088, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 1410, 25, 0, 5, 2448, 812, 8, 0, 4065, 0, 0, 0, 6, 3, 0, 0, 196, 1596, 0, 3098, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3930, 2560, 4409, 4109, 4088, 4708, 4074, 4890] avg=4096.0000 max=4890 max_rank=7 min=2560 min_rank=1 imbalance_ratio=1.193848 max_load_reduction=27878 max_load_reduction_pct=0.850769 moved_to_echo=17760 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3950, 55, 0, 0, 0, 0, 997, 374, 0, 1425, 0, 0, 0, 0, 3624, 0, 0, 0, 129, 128, 0, 0, 10, 1, 0, 0, 4061, 0, 0, 0, 4026, 29, 6, 0, 0, 0, 0, 0, 51, 28, 10, 0, 2626, 825, 8, 0, 4079, 5, 0, 0, 11, 10, 0, 0, 11, 3160, 0, 3126, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4008, 2796, 3881, 4072, 4061, 3548, 4105, 6297] avg=4096.0000 max=6297 max_rank=7 min=2796 min_rank=1 imbalance_ratio=1.537354 max_load_reduction=26471 max_load_reduction_pct=0.807831 moved_to_echo=18011 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1467, 3967, 0, 0, 0, 0, 0, 2383, 1, 0, 1445, 0, 0, 0, 0, 4096, 0, 0, 0, 122, 125, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 823, 8, 0, 4093, 0, 0, 0, 6, 3, 0, 0, 0, 255, 0, 3142, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5434, 3829, 4343, 4088, 4093, 3482, 4102, 3397] avg=4096.0000 max=5434 max_rank=0 min=3397 min_rank=7 imbalance_ratio=1.326660 max_load_reduction=27334 max_load_reduction_pct=0.834167 moved_to_echo=15074 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3968, 0, 0, 0, 0, 0, 3099, 3, 0, 1422, 0, 0, 0, 0, 4096, 0, 0, 0, 122, 125, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 850, 0, 0, 0, 2608, 822, 9, 0, 4096, 0, 3, 0, 6, 3, 0, 0, 3, 217, 0, 3136, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3968, 4524, 4343, 4087, 4093, 4289, 4108, 3356] avg=4096.0000 max=4524 max_rank=1 min=3356 min_rank=7 imbalance_ratio=1.104492 max_load_reduction=28244 max_load_reduction_pct=0.861938 moved_to_echo=15846 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21376, 384, 128, 11136, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6272, 384, 128, 128, 20480, 5632, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 11136, 0, 21504, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[30976, 128, 0, 128, 1024, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 1664, 30720, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[50, 2495, 628, 46, 2268, 114, 1077, 297, 1340, 104, 1887, 832, 808, 1207, 47, 428, 200, 1042, 1120, 2219, 1943, 267, 2625, 1533, 33, 1375, 415, 1369, 1021, 355, 1831, 1792] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[100, 2865, 785, 115, 2676, 92, 841, 646, 1086, 113, 2256, 512, 917, 907, 220, 324, 254, 580, 1483, 2002, 1713, 395, 2589, 1344, 133, 1136, 499, 1316, 1024, 480, 2113, 1252] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[147, 2923, 617, 105, 2523, 163, 892, 853, 1275, 114, 2107, 719, 615, 692, 118, 491, 386, 1057, 1378, 2500, 1969, 312, 3007, 1020, 53, 1229, 631, 894, 847, 268, 1501, 1362] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[192, 3411, 1042, 146, 2057, 67, 814, 487, 1460, 101, 2254, 337, 798, 983, 108, 484, 430, 1172, 1094, 2169, 1738, 211, 2509, 1098, 65, 1163, 607, 1572, 1114, 175, 1305, 1605] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[95, 2993, 796, 288, 2222, 214, 1232, 645, 1309, 96, 2183, 313, 727, 760, 263, 625, 221, 738, 1638, 2030, 1754, 533, 2046, 1199, 196, 1374, 830, 637, 1134, 315, 1830, 1532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[46, 3313, 653, 180, 1419, 234, 901, 356, 903, 304, 1604, 831, 754, 927, 136, 664, 510, 741, 1887, 1902, 2480, 127, 2501, 1373, 20, 1899, 517, 1326, 1090, 384, 1192, 1594] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[67, 3128, 1474, 42, 2256, 179, 1045, 704, 1014, 125, 1536, 679, 690, 494, 268, 376, 270, 626, 1613, 2587, 1888, 239, 2655, 1333, 114, 1613, 564, 1310, 997, 305, 1308, 1269] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[49, 3157, 962, 79, 1768, 90, 941, 574, 1356, 139, 1926, 368, 618, 865, 106, 355, 234, 557, 2109, 1768, 2057, 416, 2291, 1295, 46, 1664, 558, 1178, 1431, 469, 1945, 1397] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[50, 2473, 628, 46, 0, 0, 0, 0, 2268, 114, 1077, 297, 295, 2, 0, 0, 1340, 104, 1887, 832, 191, 22, 0, 0, 808, 1207, 47, 428, 1946, 0, 0, 0, 200, 1042, 1120, 1477, 0, 0, 0, 0, 1943, 267, 589, 1533, 0, 0, 0, 0, 33, 1375, 415, 1369, 740, 90, 0, 0, 1021, 355, 1345, 1792, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3197, 4053, 4376, 4436, 3839, 4332, 4022, 4513] avg=4096.0000 max=4513 max_rank=7 min=3197 min_rank=0 imbalance_ratio=1.101807 max_load_reduction=28255 max_load_reduction_pct=0.862274 moved_to_echo=16706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[147, 2897, 617, 105, 0, 0, 0, 0, 2523, 163, 892, 853, 242, 2, 0, 0, 1275, 114, 2107, 719, 157, 26, 0, 0, 615, 692, 118, 491, 2229, 0, 0, 0, 386, 1057, 1378, 1664, 0, 0, 0, 0, 1969, 312, 675, 1020, 0, 0, 0, 0, 53, 1229, 631, 894, 834, 103, 0, 0, 847, 268, 1102, 1362, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3766, 4675, 4398, 4145, 4485, 3976, 3744, 3579] avg=4096.0000 max=4675 max_rank=1 min=3579 min_rank=7 imbalance_ratio=1.141357 max_load_reduction=28093 max_load_reduction_pct=0.857330 moved_to_echo=15784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[95, 2966, 796, 288, 0, 0, 0, 0, 2222, 214, 1232, 645, 295, 1, 0, 0, 1309, 96, 2183, 313, 191, 27, 0, 0, 727, 760, 263, 625, 1516, 0, 0, 0, 221, 738, 1638, 1352, 0, 0, 0, 0, 1754, 533, 460, 1199, 0, 0, 0, 0, 196, 1374, 830, 637, 677, 70, 0, 0, 1134, 315, 1344, 1532, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4609, 4119, 3891, 3949, 3946, 3784, 4325] avg=4096.0000 max=4609 max_rank=1 min=3784 min_rank=6 imbalance_ratio=1.125244 max_load_reduction=28159 max_load_reduction_pct=0.859344 moved_to_echo=16004 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[100, 2839, 785, 115, 0, 0, 0, 0, 2676, 92, 841, 646, 341, 1, 0, 0, 1086, 113, 2256, 512, 221, 26, 0, 0, 917, 907, 220, 324, 1919, 0, 0, 0, 254, 580, 1483, 1333, 0, 0, 0, 0, 1713, 395, 581, 1344, 0, 0, 0, 0, 133, 1136, 499, 1316, 668, 89, 0, 0, 1024, 480, 1551, 1252, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3839, 4597, 4214, 4287, 3650, 4033, 3841, 4307] avg=4096.0000 max=4597 max_rank=1 min=3650 min_rank=4 imbalance_ratio=1.122314 max_load_reduction=28171 max_load_reduction_pct=0.859711 moved_to_echo=15831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[46, 3280, 653, 180, 0, 0, 0, 0, 1419, 234, 901, 356, 197, 5, 0, 0, 903, 304, 1604, 831, 129, 33, 0, 0, 754, 927, 136, 664, 1857, 0, 0, 0, 510, 741, 1887, 1259, 0, 0, 0, 0, 2480, 127, 553, 1373, 0, 0, 0, 0, 20, 1899, 517, 1326, 638, 91, 0, 0, 1090, 384, 866, 1594, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 3112, 3804, 4338, 4397, 4533, 4491, 3934] avg=4096.0000 max=4533 max_rank=5 min=3112 min_rank=1 imbalance_ratio=1.106689 max_load_reduction=28235 max_load_reduction_pct=0.861664 moved_to_echo=17355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[192, 3380, 1042, 146, 0, 0, 0, 0, 2057, 67, 814, 487, 210, 2, 0, 0, 1460, 101, 2254, 337, 136, 31, 0, 0, 798, 983, 108, 484, 1860, 0, 0, 0, 430, 1172, 1094, 1444, 0, 0, 0, 0, 1738, 211, 563, 1098, 0, 0, 0, 0, 65, 1163, 607, 1572, 723, 86, 0, 0, 1114, 175, 959, 1605, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4760, 3637, 4319, 4233, 4140, 3610, 4216, 3853] avg=4096.0000 max=4760 max_rank=0 min=3610 min_rank=5 imbalance_ratio=1.162109 max_load_reduction=28008 max_load_reduction_pct=0.854736 moved_to_echo=15819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[67, 3100, 1474, 42, 0, 0, 0, 0, 2256, 179, 1045, 704, 211, 2, 0, 0, 1014, 125, 1536, 679, 136, 28, 0, 0, 690, 494, 268, 376, 1968, 0, 0, 0, 270, 626, 1613, 1722, 0, 0, 0, 0, 1888, 239, 596, 1333, 0, 0, 0, 0, 114, 1613, 564, 1310, 863, 91, 0, 0, 997, 305, 961, 1269, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4683, 4397, 3518, 3796, 4231, 4056, 4555, 3532] avg=4096.0000 max=4683 max_rank=0 min=3518 min_rank=2 imbalance_ratio=1.143311 max_load_reduction=28085 max_load_reduction_pct=0.857086 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[49, 3129, 962, 79, 0, 0, 0, 0, 1768, 90, 941, 574, 314, 1, 0, 0, 1356, 139, 1926, 368, 203, 28, 0, 0, 618, 865, 106, 355, 1698, 0, 0, 0, 234, 557, 2109, 1177, 0, 0, 0, 0, 2057, 416, 514, 1295, 0, 0, 0, 0, 46, 1664, 558, 1178, 590, 79, 0, 0, 1431, 469, 1428, 1397, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4219, 3688, 4020, 3642, 4077, 4282, 4115, 4725] avg=4096.0000 max=4725 max_rank=7 min=3642 min_rank=3 imbalance_ratio=1.153564 max_load_reduction=28043 max_load_reduction_pct=0.855804 moved_to_echo=17199 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15616, 2560, 4608, 10240, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6016, 6912, 1280, 3840, 15104, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[17280, 1280, 7808, 4608, 2176, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8704, 2816, 9600, 11904, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[768, 24064, 7040, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 11520, 4736, 9728, 5760, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9856, 1152, 15872, 4608, 1408, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2560, 6528, 12416, 11520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[81, 1623, 4081, 575, 0, 0, 3175, 1, 2, 0, 1, 10, 0, 0, 1, 0, 533, 2, 0, 26, 3250, 0, 1032, 50, 4075, 1725, 4055, 216, 4077, 3, 2433, 1741] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[97, 230, 3713, 0, 2, 0, 3576, 1, 0, 0, 8, 0, 7, 0, 2, 8, 167, 4, 26, 0, 3763, 6, 348, 0, 4093, 3228, 4058, 313, 4080, 0, 1763, 3275] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 8, 4065, 53, 0, 4, 895, 0, 0, 0, 30, 0, 0, 1, 0, 1, 1108, 0, 2, 1, 3867, 0, 2600, 56, 3660, 4057, 3943, 160, 4087, 0, 535, 3635] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[373, 97, 4069, 22, 286, 0, 2948, 61, 3, 0, 0, 22, 95, 0, 2, 47, 28, 0, 5, 27, 3499, 0, 2306, 21, 3889, 3448, 2694, 426, 3918, 24, 1172, 3286] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 1379, 4055, 5, 0, 0, 562, 583, 0, 0, 7, 0, 0, 22, 13, 6, 466, 3, 3, 5, 485, 0, 3756, 173, 4028, 3356, 4093, 1371, 4032, 1, 2127, 2237] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[487, 1590, 3923, 136, 5, 47, 636, 140, 1, 1, 0, 23, 0, 0, 0, 21, 24, 0, 0, 0, 2567, 21, 1629, 2, 3594, 1817, 3947, 459, 4085, 104, 3851, 3658] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[14, 1001, 3886, 278, 1, 0, 338, 2, 43, 7, 27, 43, 0, 8, 0, 0, 417, 0, 6, 0, 1382, 0, 1216, 84, 4068, 3071, 3689, 2186, 4093, 0, 3024, 3884] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4, 2512, 4052, 3, 12, 1, 1773, 73, 2, 1, 12, 26, 3, 20, 1, 0, 783, 0, 11, 3, 1173, 30, 2459, 25, 3505, 665, 4082, 387, 4088, 5, 3230, 3827] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[81, 1623, 2846, 575, 0, 0, 0, 0, 0, 0, 3175, 1, 1235, 338, 493, 0, 2, 0, 1, 10, 4069, 217, 0, 0, 0, 0, 1, 0, 4077, 6, 0, 0, 533, 2, 0, 26, 3057, 414, 0, 0, 2757, 0, 1032, 50, 0, 0, 0, 0, 0, 1725, 781, 216, 0, 0, 0, 0, 0, 3, 2433, 989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5125, 5242, 4299, 4084, 4032, 3839, 2722, 3425] avg=4096.0000 max=5242 max_rank=1 min=2722 min_rank=6 imbalance_ratio=1.279785 max_load_reduction=27526 max_load_reduction_pct=0.840027 moved_to_echo=14018 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[97, 230, 2589, 0, 0, 0, 0, 0, 2, 0, 3576, 1, 1124, 637, 570, 0, 0, 0, 8, 0, 4086, 217, 0, 0, 7, 0, 2, 8, 4080, 7, 0, 0, 167, 4, 26, 0, 3059, 779, 0, 0, 3193, 6, 348, 0, 0, 0, 0, 0, 0, 3228, 782, 313, 0, 0, 0, 0, 0, 0, 1763, 1859, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2916, 5910, 4311, 4104, 4035, 3547, 4323, 3622] avg=4096.0000 max=5910 max_rank=1 min=2916 min_rank=0 imbalance_ratio=1.442871 max_load_reduction=26858 max_load_reduction_pct=0.819641 moved_to_echo=15527 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 2834, 53, 0, 0, 0, 0, 0, 4, 895, 0, 1231, 707, 586, 0, 0, 0, 30, 0, 3654, 211, 0, 0, 0, 1, 0, 1, 4087, 6, 0, 0, 1108, 0, 2, 1, 2972, 864, 0, 0, 3281, 0, 2600, 56, 0, 0, 0, 0, 0, 4057, 760, 160, 0, 0, 0, 0, 0, 0, 535, 2064, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2895, 3423, 3895, 4095, 4947, 5937, 4977, 2599] avg=4096.0000 max=5937 max_rank=5 min=2599 min_rank=7 imbalance_ratio=1.449463 max_load_reduction=26831 max_load_reduction_pct=0.818817 moved_to_echo=18460 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[487, 1590, 2735, 136, 0, 0, 0, 0, 5, 47, 636, 140, 1188, 711, 389, 0, 1, 1, 0, 23, 3588, 211, 0, 0, 0, 0, 0, 21, 4085, 6, 0, 0, 24, 0, 0, 0, 2975, 870, 0, 0, 2178, 21, 1629, 2, 0, 0, 0, 0, 0, 1817, 761, 459, 0, 0, 0, 0, 0, 104, 3851, 2077, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4948, 3116, 3824, 4112, 3869, 3830, 3037, 6032] avg=4096.0000 max=6032 max_rank=7 min=3037 min_rank=6 imbalance_ratio=1.472656 max_load_reduction=26736 max_load_reduction_pct=0.815918 moved_to_echo=16768 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[14, 1001, 2710, 278, 0, 0, 0, 0, 1, 0, 338, 2, 1176, 755, 209, 0, 43, 7, 27, 43, 4061, 197, 0, 0, 0, 8, 0, 0, 4093, 7, 0, 0, 417, 0, 6, 0, 2781, 924, 0, 0, 1173, 0, 1216, 84, 0, 0, 0, 0, 0, 3071, 711, 2186, 0, 0, 0, 0, 0, 0, 3024, 2205, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4003, 2481, 4378, 4108, 4128, 2473, 5968, 5229] avg=4096.0000 max=5968 max_rank=6 min=2473 min_rank=5 imbalance_ratio=1.457031 max_load_reduction=26800 max_load_reduction_pct=0.817871 moved_to_echo=17798 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[373, 97, 2834, 22, 0, 0, 0, 0, 286, 0, 2948, 61, 1235, 643, 535, 0, 3, 0, 0, 22, 3883, 147, 0, 0, 95, 0, 2, 47, 3918, 6, 0, 0, 28, 0, 5, 27, 2035, 784, 0, 0, 2964, 0, 2306, 21, 0, 0, 0, 0, 0, 3448, 512, 426, 0, 0, 0, 0, 0, 24, 1172, 1859, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3326, 5708, 4055, 4068, 2879, 5291, 4386, 3055] avg=4096.0000 max=5708 max_rank=1 min=2879 min_rank=4 imbalance_ratio=1.393555 max_load_reduction=27060 max_load_reduction_pct=0.825806 moved_to_echo=15611 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 2512, 2825, 3, 0, 0, 0, 0, 12, 1, 1773, 73, 1227, 744, 177, 0, 2, 1, 12, 26, 3499, 218, 0, 0, 3, 20, 1, 0, 4088, 6, 0, 0, 783, 0, 11, 3, 3077, 910, 0, 0, 996, 30, 2459, 25, 0, 0, 0, 0, 0, 665, 787, 387, 0, 0, 0, 0, 0, 5, 3230, 2173, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5344, 4007, 3758, 4118, 4784, 3510, 1839, 5408] avg=4096.0000 max=5408 max_rank=7 min=1839 min_rank=6 imbalance_ratio=1.320312 max_load_reduction=27360 max_load_reduction_pct=0.834961 moved_to_echo=15541 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1379, 2827, 5, 0, 0, 0, 0, 0, 0, 562, 583, 1228, 435, 73, 0, 0, 0, 7, 0, 4022, 219, 0, 0, 0, 22, 13, 6, 4032, 6, 0, 0, 466, 3, 3, 5, 3085, 532, 0, 0, 412, 0, 3756, 173, 0, 0, 0, 0, 0, 3356, 789, 1371, 0, 0, 0, 0, 0, 1, 2127, 1270, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4211, 2881, 4248, 4079, 4094, 4341, 5516, 3398] avg=4096.0000 max=5516 max_rank=6 min=2881 min_rank=1 imbalance_ratio=1.346680 max_load_reduction=27252 max_load_reduction_pct=0.831665 moved_to_echo=17349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 13952, 896, 9728, 4992, 3072, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 256, 18176, 14592, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3584, 128, 128, 128, 23168, 6144, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 8448, 22272, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 21376, 5888, 5632, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17024, 128, 15360, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 30976, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[621, 4062, 0, 0, 0, 1, 0, 4028, 0, 4060, 0, 3, 43, 1891, 3803, 0, 0, 18, 2, 3915, 9, 4096, 0, 198, 4094, 0, 1472, 0, 74, 373, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[264, 4091, 0, 0, 0, 27, 211, 3680, 12, 4055, 0, 23, 14, 55, 3969, 4, 0, 1, 3, 3910, 233, 4090, 0, 3603, 4093, 6, 225, 5, 0, 142, 46, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4, 4089, 0, 1, 0, 95, 9, 3700, 0, 3759, 0, 4, 960, 106, 4091, 0, 44, 48, 0, 3770, 535, 4096, 0, 829, 4093, 6, 2473, 0, 54, 1, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[8, 4003, 0, 226, 0, 500, 47, 3960, 2, 4076, 0, 5, 35, 794, 3282, 1, 506, 0, 0, 3599, 4, 4023, 27, 54, 4095, 0, 3356, 0, 1, 7, 0, 157] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[50, 3566, 17, 49, 228, 2, 2, 3269, 0, 3976, 0, 24, 79, 0, 4028, 24, 0, 25, 3, 1871, 2515, 4095, 0, 3357, 4094, 2, 67, 608, 5, 812, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[21, 3618, 1, 31, 5, 23, 0, 3983, 5, 3999, 0, 0, 5, 211, 4077, 0, 0, 0, 0, 4087, 28, 4081, 2, 26, 4078, 1, 4096, 143, 60, 65, 98, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 1787, 0, 3, 0, 0, 0, 3756, 6, 4096, 4, 1, 802, 1343, 3868, 0, 0, 35, 36, 4092, 0, 3876, 2, 3877, 4073, 0, 1009, 0, 0, 12, 0, 90] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[151, 4075, 2, 3, 1, 1, 0, 4092, 2, 4047, 0, 3, 88, 3583, 3983, 0, 0, 31, 90, 4096, 50, 4090, 0, 13, 4094, 1, 229, 0, 3, 38, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8, 4003, 0, 226, 213, 0, 0, 0, 0, 500, 47, 3960, 121, 0, 0, 0, 2, 4076, 0, 5, 63, 0, 0, 0, 35, 794, 2402, 1, 0, 0, 0, 0, 506, 0, 0, 3599, 272, 0, 0, 0, 4, 2159, 27, 54, 0, 0, 0, 0, 2387, 0, 3356, 0, 0, 0, 0, 0, 1, 7, 0, 157, 1864, 1708, 211, 0] rerouted_tokens_per_ep_rank=[4450, 4628, 4146, 3232, 4377, 2244, 5743, 3948] avg=4096.0000 max=5743 max_rank=6 min=2244 min_rank=5 imbalance_ratio=1.402100 max_load_reduction=27025 max_load_reduction_pct=0.824738 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[264, 4091, 0, 0, 258, 0, 0, 0, 0, 27, 211, 3680, 146, 0, 0, 0, 12, 4055, 0, 23, 77, 0, 0, 0, 14, 55, 2903, 4, 0, 0, 0, 0, 0, 1, 3, 3910, 330, 0, 0, 0, 233, 2195, 0, 3603, 0, 0, 0, 0, 2386, 6, 225, 5, 0, 0, 0, 0, 0, 142, 46, 6, 1895, 1707, 255, 0] rerouted_tokens_per_ep_rank=[4613, 4064, 4167, 2976, 4244, 6031, 2622, 4051] avg=4096.0000 max=6031 max_rank=5 min=2622 min_rank=6 imbalance_ratio=1.472412 max_load_reduction=26737 max_load_reduction_pct=0.815948 moved_to_echo=16948 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[621, 4062, 0, 0, 247, 0, 0, 0, 0, 1, 0, 4028, 140, 0, 0, 0, 0, 4060, 0, 3, 74, 0, 0, 0, 43, 1891, 2781, 0, 0, 0, 0, 0, 0, 18, 2, 3915, 316, 0, 0, 0, 9, 2198, 0, 198, 0, 0, 0, 0, 2387, 0, 1472, 0, 0, 0, 0, 0, 74, 373, 0, 5, 1898, 1707, 245, 0] rerouted_tokens_per_ep_rank=[4930, 4169, 4137, 4715, 4251, 2405, 3859, 4302] avg=4096.0000 max=4930 max_rank=0 min=2405 min_rank=5 imbalance_ratio=1.203613 max_load_reduction=27838 max_load_reduction_pct=0.849548 moved_to_echo=14817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 4089, 0, 1, 266, 0, 0, 0, 0, 95, 9, 3700, 151, 0, 0, 0, 0, 3759, 0, 4, 79, 0, 0, 0, 960, 106, 2992, 0, 0, 0, 0, 0, 44, 48, 0, 3770, 340, 0, 0, 0, 535, 2198, 0, 829, 0, 0, 0, 0, 2386, 6, 2473, 0, 0, 0, 0, 0, 54, 1, 1, 0, 1898, 1707, 263, 0] rerouted_tokens_per_ep_rank=[4360, 3955, 3842, 4058, 4202, 3562, 4865, 3924] avg=4096.0000 max=4865 max_rank=6 min=3562 min_rank=5 imbalance_ratio=1.187744 max_load_reduction=27903 max_load_reduction_pct=0.851532 moved_to_echo=16553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[50, 3566, 17, 49, 266, 0, 0, 0, 228, 2, 2, 3269, 151, 0, 0, 0, 0, 3976, 0, 24, 82, 0, 0, 0, 79, 0, 2926, 24, 0, 0, 0, 0, 0, 25, 3, 1871, 338, 0, 0, 0, 2515, 2191, 0, 3357, 0, 0, 0, 0, 2383, 2, 67, 608, 0, 0, 0, 0, 5, 812, 0, 0, 1904, 1711, 265, 0] rerouted_tokens_per_ep_rank=[3948, 3652, 4082, 3029, 2237, 8063, 3060, 4697] avg=4096.0000 max=8063 max_rank=5 min=2237 min_rank=4 imbalance_ratio=1.968506 max_load_reduction=24705 max_load_reduction_pct=0.753937 moved_to_echo=18057 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1787, 0, 3, 251, 0, 0, 0, 0, 0, 0, 3756, 142, 0, 0, 0, 6, 4096, 4, 1, 75, 0, 0, 0, 802, 1343, 2830, 0, 0, 0, 0, 0, 0, 35, 36, 4092, 321, 0, 0, 0, 0, 2080, 2, 3877, 0, 0, 0, 0, 2375, 0, 1009, 0, 0, 0, 0, 0, 0, 12, 0, 90, 1796, 1698, 249, 0] rerouted_tokens_per_ep_rank=[2041, 3898, 4182, 4975, 4484, 5959, 3384, 3845] avg=4096.0000 max=5959 max_rank=5 min=2041 min_rank=0 imbalance_ratio=1.454834 max_load_reduction=26809 max_load_reduction_pct=0.818146 moved_to_echo=17672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[21, 3618, 1, 31, 265, 0, 0, 0, 5, 23, 0, 3983, 150, 0, 0, 0, 5, 3999, 0, 0, 79, 0, 0, 0, 5, 211, 2983, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 338, 0, 0, 0, 28, 2190, 2, 26, 0, 0, 0, 0, 2378, 1, 4096, 143, 0, 0, 0, 0, 60, 65, 98, 24, 1891, 1700, 262, 0] rerouted_tokens_per_ep_rank=[3936, 4161, 4083, 3199, 4425, 2246, 6618, 4100] avg=4096.0000 max=6618 max_rank=6 min=2246 min_rank=5 imbalance_ratio=1.615723 max_load_reduction=26150 max_load_reduction_pct=0.798035 moved_to_echo=17389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[151, 4075, 2, 3, 259, 0, 0, 0, 1, 1, 0, 4092, 147, 0, 0, 0, 2, 4047, 0, 3, 77, 0, 0, 0, 88, 3583, 2913, 0, 0, 0, 0, 0, 0, 31, 90, 4096, 331, 0, 0, 0, 50, 2195, 0, 13, 0, 0, 0, 0, 2387, 1, 229, 0, 0, 0, 0, 0, 3, 38, 0, 2, 1895, 1707, 256, 0] rerouted_tokens_per_ep_rank=[4490, 4241, 4129, 6584, 4548, 2258, 2617, 3901] avg=4096.0000 max=6584 max_rank=3 min=2258 min_rank=5 imbalance_ratio=1.607422 max_load_reduction=26184 max_load_reduction_pct=0.799072 moved_to_echo=13324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 1536, 256, 384, 15104, 13696, 2048, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3456, 17408, 128, 12032, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[19072, 128, 12928, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2048, 8064, 22784, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 29312, 128, 384, 2048, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 256, 256, 29440, 2688, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32128, 128, 128, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 768, 384, 30592, 1152, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4, 35, 8, 4029, 6, 0, 304, 2028, 141, 456, 188, 4087, 3949, 0, 3471, 5, 0, 0, 0, 11, 205, 3, 0, 17, 3948, 0, 3, 0, 4077, 4037, 124, 1632] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 1801, 0, 4096, 1690, 0, 933, 61, 0, 1542, 0, 3987, 751, 0, 4067, 0, 0, 4, 1, 0, 3, 0, 0, 13, 4087, 0, 0, 0, 4082, 3870, 254, 1526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 3963, 43, 4051, 0, 17, 2992, 2633, 42, 0, 30, 4014, 1910, 0, 750, 0, 109, 0, 1, 0, 0, 0, 0, 4, 4090, 0, 0, 0, 4004, 3140, 868, 107] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 2984, 48, 3596, 2440, 484, 634, 113, 1, 4, 5, 4087, 3756, 0, 14, 23, 0, 0, 0, 278, 0, 183, 95, 0, 2301, 0, 6, 2, 3811, 4041, 18, 3844] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2, 1, 0, 4075, 29, 13, 4029, 1602, 235, 2092, 47, 4087, 2888, 0, 941, 1, 3, 0, 263, 0, 17, 0, 0, 24, 4051, 1, 1, 0, 4087, 3990, 0, 289] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 1294, 0, 4096, 76, 0, 2247, 2716, 2265, 9, 0, 4074, 3869, 0, 23, 2, 0, 0, 2, 136, 0, 0, 0, 0, 4095, 0, 0, 0, 4089, 2821, 1, 953] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 2386, 9, 4096, 2099, 0, 0, 2131, 3, 1, 867, 4077, 4060, 0, 665, 0, 11, 0, 3, 33, 2, 2, 0, 0, 4092, 29, 0, 0, 4091, 4096, 15, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 741, 0, 4042, 0, 0, 3423, 17, 2335, 44, 249, 4087, 3171, 0, 2750, 0, 2, 0, 3, 7, 0, 2, 1, 0, 3294, 257, 0, 1, 4096, 4089, 0, 157] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 35, 8, 2443, 0, 0, 0, 0, 6, 0, 304, 2028, 8, 0, 0, 0, 141, 456, 188, 2794, 0, 0, 0, 0, 3254, 0, 3471, 5, 0, 0, 0, 0, 0, 0, 0, 11, 1586, 1293, 953, 281, 205, 3, 0, 17, 4059, 0, 0, 0, 3948, 0, 3, 0, 406, 0, 0, 0, 18, 3084, 124, 1632, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2490, 2346, 3579, 6730, 4124, 4284, 4357, 4858] avg=4096.0000 max=6730 max_rank=3 min=2346 min_rank=1 imbalance_ratio=1.643066 max_load_reduction=26038 max_load_reduction_pct=0.794617 moved_to_echo=17623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1801, 0, 2484, 0, 0, 0, 0, 1690, 0, 933, 61, 1, 0, 0, 0, 0, 1542, 0, 2725, 0, 0, 0, 0, 620, 0, 4067, 0, 0, 0, 0, 0, 0, 4, 1, 0, 1612, 1262, 913, 53, 3, 0, 0, 13, 4064, 0, 0, 0, 4087, 0, 0, 0, 77, 0, 0, 0, 18, 2957, 254, 1526, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4285, 2685, 4267, 4687, 3845, 4080, 4164, 4755] avg=4096.0000 max=4755 max_rank=7 min=2685 min_rank=1 imbalance_ratio=1.160889 max_load_reduction=28013 max_load_reduction_pct=0.854889 moved_to_echo=16844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2386, 9, 2484, 0, 0, 0, 0, 2099, 0, 0, 2131, 8, 0, 0, 0, 3, 1, 867, 2787, 0, 0, 0, 0, 3345, 0, 665, 0, 0, 0, 0, 0, 11, 0, 3, 33, 1612, 1290, 967, 289, 2, 2, 0, 0, 4073, 0, 0, 0, 4092, 29, 0, 0, 418, 0, 0, 0, 18, 3129, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4879, 4238, 3658, 4010, 4205, 4077, 4539, 3162] avg=4096.0000 max=4879 max_rank=0 min=3162 min_rank=7 imbalance_ratio=1.191162 max_load_reduction=27889 max_load_reduction_pct=0.851105 moved_to_echo=15983 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1294, 0, 2484, 0, 0, 0, 0, 76, 0, 2247, 2716, 8, 0, 0, 0, 2265, 9, 0, 2785, 0, 0, 0, 0, 3188, 0, 23, 2, 0, 0, 0, 0, 0, 0, 2, 136, 1612, 1289, 666, 275, 0, 0, 0, 0, 4071, 0, 0, 0, 4095, 0, 0, 0, 398, 0, 0, 0, 18, 2155, 1, 953, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3778, 5047, 5059, 3213, 3980, 4071, 4493, 3127] avg=4096.0000 max=5059 max_rank=2 min=3127 min_rank=7 imbalance_ratio=1.235107 max_load_reduction=27709 max_load_reduction_pct=0.845612 moved_to_echo=15671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2984, 48, 2176, 0, 0, 0, 0, 2440, 484, 634, 113, 11, 0, 0, 0, 1, 4, 5, 2789, 0, 0, 0, 0, 3081, 0, 14, 23, 0, 0, 0, 0, 0, 0, 0, 278, 1420, 1298, 957, 272, 0, 183, 95, 0, 3797, 0, 0, 0, 2301, 0, 6, 2, 392, 0, 0, 0, 14, 3084, 18, 3844, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5208, 3682, 2799, 3118, 4225, 4075, 2701, 6960] avg=4096.0000 max=6960 max_rank=7 min=2701 min_rank=6 imbalance_ratio=1.699219 max_load_reduction=25808 max_load_reduction_pct=0.787598 moved_to_echo=17961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3963, 43, 2456, 0, 0, 0, 0, 0, 17, 2992, 2633, 3, 0, 0, 0, 42, 0, 30, 2744, 0, 0, 0, 0, 1575, 0, 750, 0, 0, 0, 0, 0, 109, 0, 1, 0, 1595, 1270, 741, 136, 0, 0, 0, 4, 3986, 0, 0, 0, 4090, 0, 0, 0, 196, 0, 0, 0, 18, 2399, 868, 107, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6462, 5645, 2816, 2325, 3852, 3990, 4286, 3392] avg=4096.0000 max=6462 max_rank=0 min=2325 min_rank=3 imbalance_ratio=1.577637 max_load_reduction=26306 max_load_reduction_pct=0.802795 moved_to_echo=15520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 2471, 0, 0, 0, 0, 29, 13, 4029, 1602, 6, 0, 0, 0, 235, 2092, 47, 2794, 0, 0, 0, 0, 2380, 0, 941, 1, 0, 0, 0, 0, 3, 0, 263, 0, 1604, 1293, 942, 205, 17, 0, 0, 24, 4069, 0, 0, 0, 4051, 1, 1, 0, 297, 0, 0, 0, 18, 3048, 0, 289, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2474, 5679, 5168, 3322, 4310, 4110, 4350, 3355] avg=4096.0000 max=5679 max_rank=1 min=2474 min_rank=0 imbalance_ratio=1.386475 max_load_reduction=27089 max_load_reduction_pct=0.826691 moved_to_echo=16125 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 741, 0, 2451, 0, 0, 0, 0, 0, 0, 3423, 17, 6, 0, 0, 0, 2335, 44, 249, 2794, 0, 0, 0, 0, 2613, 0, 2750, 0, 0, 0, 0, 0, 2, 0, 3, 7, 1591, 1293, 965, 226, 0, 2, 1, 0, 4078, 0, 0, 0, 3294, 257, 0, 1, 326, 0, 0, 0, 18, 3124, 0, 157, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3192, 3446, 5422, 5363, 4087, 4081, 3878, 3299] avg=4096.0000 max=5422 max_rank=2 min=3192 min_rank=0 imbalance_ratio=1.323730 max_load_reduction=27346 max_load_reduction_pct=0.834534 moved_to_echo=15345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32628, 32768, 32768, 32908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32628, 32768, 32768, 32908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32628, 32768, 32768, 32908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32628, 32768, 32768, 32908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32628, 32768, 32768, 32908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32628, 32768, 32768, 32908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32628, 32768, 32768, 32908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32628, 32768, 32768, 32908] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32628 actual_valid_recv=33152 observed_loss=-524 tokens_per_expert=[128, 128, 384, 512, 12672, 10368, 7168, 1792] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 13312, 128, 19456, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32908 actual_valid_recv=33152 observed_loss=-244 tokens_per_expert=[256, 23040, 1280, 8576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6400, 640, 14592, 11392, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5120, 4224, 1408, 22272, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[20096, 0, 12800, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 256, 128, 128, 32256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[30080, 384, 128, 128, 2560, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 82, 4, 0, 0, 585, 1, 13, 3074, 0, 14, 4096, 4095, 4005, 0, 0, 2905, 0, 0, 217, 0, 3448, 0, 2361, 3766, 0, 1, 0, 4089, 9, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 380, 94, 0, 1, 677, 1, 2861, 0, 1260, 4025, 4096, 4095, 0, 0, 0, 0, 0, 233, 0, 965, 0, 3763, 3819, 53, 0, 0, 4076, 2344, 25] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 3, 0, 3376, 0, 1264, 0, 0, 18, 0, 0, 64, 4094, 4054, 4096, 0, 0, 474, 10, 0, 1535, 0, 3675, 7, 1377, 4086, 0, 0, 0, 4093, 542, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 45, 1, 0, 12, 0, 76, 3, 3663, 3519, 4014, 4096, 0, 0, 954, 2, 0, 6, 57, 4063, 494, 3559, 4092, 3, 0, 0, 4074, 31, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 0, 3342, 3, 1468, 0, 799, 4093, 4010, 4096, 4, 0, 129, 0, 0, 4096, 0, 395, 0, 2136, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 26, 41, 879, 11, 387, 0, 5, 8, 19, 0, 1685, 4044, 4084, 4096, 74, 0, 649, 0, 7, 3422, 0, 967, 242, 4045, 4071, 0, 0, 0, 4005, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[30, 1, 0, 12, 0, 2253, 263, 9, 163, 1, 1, 1666, 4095, 3515, 3707, 0, 2, 167, 638, 691, 37, 399, 4094, 2, 3826, 2963, 477, 28, 0, 3170, 558, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 342, 0, 0, 134, 415, 11, 408, 0, 944, 3629, 4059, 4094, 23, 0, 14, 57, 6, 2952, 83, 1517, 157, 2674, 4049, 11, 0, 14, 4083, 3089, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 82, 4, 3422, 0, 0, 0, 0, 0, 585, 1, 1676, 1217, 0, 0, 13, 3074, 0, 14, 1060, 714, 53, 0, 4096, 139, 0, 0, 0, 0, 0, 0, 0, 2905, 0, 0, 2739, 583, 0, 0, 217, 0, 3395, 0, 0, 0, 0, 0, 2361, 1030, 0, 1, 0, 0, 0, 0, 0, 3375, 9, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3508, 3479, 4928, 4235, 6227, 3612, 3392, 3387] avg=4096.0000 max=6227 max_rank=4 min=3387 min_rank=7 imbalance_ratio=1.520264 max_load_reduction=26541 max_load_reduction_pct=0.809967 moved_to_echo=16618 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 3376, 3499, 0, 0, 0, 0, 1264, 0, 0, 1819, 1205, 0, 0, 18, 0, 0, 64, 1150, 715, 56, 0, 4094, 137, 0, 0, 0, 0, 0, 0, 0, 474, 10, 0, 2712, 597, 0, 0, 1535, 0, 3619, 7, 0, 0, 0, 0, 1377, 1117, 0, 0, 0, 0, 0, 0, 0, 3378, 542, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6878, 4288, 2003, 4231, 3793, 5161, 2494, 3920] avg=4096.0000 max=6878 max_rank=0 min=2003 min_rank=2 imbalance_ratio=1.679199 max_load_reduction=25890 max_load_reduction_pct=0.790100 moved_to_echo=15368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 380, 3498, 0, 0, 0, 94, 0, 1, 677, 1700, 1218, 0, 0, 1, 2861, 0, 1260, 1075, 712, 14, 0, 4025, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2740, 597, 0, 0, 233, 0, 951, 0, 0, 0, 0, 0, 3763, 1044, 53, 0, 0, 0, 0, 0, 0, 3364, 2344, 25, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3878, 3690, 5923, 4163, 3337, 1184, 4860, 5733] avg=4096.0000 max=5923 max_rank=2 min=1184 min_rank=5 imbalance_ratio=1.446045 max_load_reduction=26845 max_load_reduction_pct=0.819244 moved_to_echo=15114 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 3499, 0, 0, 0, 45, 1, 0, 12, 1821, 1193, 0, 0, 0, 76, 3, 3663, 1152, 711, 62, 0, 3519, 136, 0, 0, 0, 0, 0, 0, 0, 954, 2, 0, 2685, 597, 0, 0, 6, 57, 4001, 494, 0, 0, 0, 0, 3559, 1119, 3, 0, 0, 0, 0, 0, 0, 3363, 31, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3503, 3072, 5667, 3655, 4238, 4558, 4681, 3394] avg=4096.0000 max=5667 max_rank=2 min=3072 min_rank=1 imbalance_ratio=1.383545 max_load_reduction=27101 max_load_reduction_pct=0.827057 moved_to_echo=16871 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 3500, 0, 0, 0, 0, 0, 0, 3342, 1823, 1192, 0, 0, 3, 1468, 0, 799, 1153, 715, 6, 0, 4093, 136, 0, 4, 0, 0, 0, 0, 0, 129, 0, 0, 2682, 596, 0, 0, 4096, 0, 389, 0, 0, 0, 0, 0, 2136, 1120, 0, 0, 0, 0, 0, 0, 0, 3381, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3505, 6357, 4144, 4233, 3407, 4485, 3256, 3381] avg=4096.0000 max=6357 max_rank=1 min=3256 min_rank=6 imbalance_ratio=1.552002 max_load_reduction=26411 max_load_reduction_pct=0.806000 moved_to_echo=14529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[30, 1, 0, 12, 3168, 0, 0, 0, 0, 2253, 263, 9, 1322, 1048, 0, 0, 163, 1, 1, 1666, 837, 558, 67, 0, 4095, 111, 0, 0, 0, 0, 0, 0, 2, 167, 638, 691, 2356, 539, 0, 0, 37, 399, 4027, 2, 0, 0, 0, 0, 3826, 804, 477, 28, 0, 0, 0, 0, 0, 2612, 558, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3211, 4895, 3293, 4206, 4393, 4465, 5135, 3170] avg=4096.0000 max=5135 max_rank=6 min=3170 min_rank=7 imbalance_ratio=1.253662 max_load_reduction=27633 max_load_reduction_pct=0.843292 moved_to_echo=17163 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 26, 41, 879, 3500, 0, 0, 0, 11, 387, 0, 5, 1812, 1214, 0, 0, 8, 19, 0, 1685, 1146, 699, 14, 0, 4044, 138, 0, 74, 0, 0, 0, 0, 0, 649, 0, 7, 2732, 596, 0, 0, 3422, 0, 953, 242, 0, 0, 0, 0, 4045, 1113, 0, 0, 0, 0, 0, 0, 0, 3306, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4446, 3429, 3571, 4256, 3984, 4617, 5158, 3307] avg=4096.0000 max=5158 max_rank=6 min=3307 min_rank=7 imbalance_ratio=1.259277 max_load_reduction=27610 max_load_reduction_pct=0.842590 moved_to_echo=17066 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 342, 3497, 0, 0, 0, 0, 0, 134, 415, 1802, 1207, 0, 0, 11, 408, 0, 944, 1140, 713, 23, 0, 3629, 137, 0, 23, 0, 0, 0, 0, 0, 14, 57, 6, 2715, 597, 0, 0, 2952, 83, 1494, 157, 0, 0, 0, 0, 2674, 1107, 11, 0, 0, 0, 0, 0, 14, 3370, 3089, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3839, 3558, 3239, 3789, 3389, 4686, 3792, 6476] avg=4096.0000 max=6476 max_rank=7 min=3239 min_rank=2 imbalance_ratio=1.581055 max_load_reduction=26292 max_load_reduction_pct=0.802368 moved_to_echo=18343 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 3968, 1024, 4480, 13824, 9600, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 5376, 768, 768, 21376, 4736, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 7936, 128, 10112, 8832, 5632, 384, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 26240, 6656, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[23808, 8576, 640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 5120, 27648, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12544, 640, 18944, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[31616, 1152, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[3691, 12, 1319, 1, 0, 10, 212, 0, 4014, 44, 0, 0, 0, 67, 191, 98, 0, 4005, 4060, 28, 0, 155, 0, 3268, 4031, 102, 2, 0, 4078, 22, 3337, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[3600, 0, 43, 504, 0, 0, 23, 0, 4087, 7, 0, 0, 1, 0, 0, 0, 4058, 2817, 4095, 0, 0, 1, 4, 1245, 4095, 2, 0, 0, 4085, 1, 4090, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4095, 120, 1282, 1, 0, 1, 9, 0, 4095, 0, 2, 520, 0, 0, 0, 1, 3312, 2266, 4093, 0, 0, 49, 0, 10, 3224, 1910, 8, 0, 4088, 0, 3682, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[3651, 0, 5, 45, 4, 0, 1441, 0, 4089, 0, 0, 1, 13, 0, 46, 0, 856, 4067, 4093, 8, 1, 0, 41, 115, 4087, 2087, 3, 0, 4041, 0, 4074, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 123, 12, 0, 0, 388, 0, 4096, 0, 0, 76, 0, 139, 0, 178, 26, 4095, 4075, 0, 0, 0, 0, 4094, 3827, 3517, 0, 0, 4096, 0, 4026, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[3871, 11, 0, 289, 8, 0, 0, 1, 4067, 0, 44, 1220, 0, 137, 310, 0, 4023, 3828, 1085, 0, 0, 27, 3, 3863, 2911, 3001, 10, 0, 3797, 1, 260, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1051, 241, 670, 172, 0, 226, 1345, 0, 3904, 1, 0, 0, 0, 7, 53, 161, 0, 3884, 4054, 1, 0, 0, 0, 3540, 3953, 1551, 21, 0, 4020, 0, 3913, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[22, 3, 2849, 0, 0, 0, 445, 0, 4093, 5, 21, 0, 17, 4, 2, 315, 0, 3786, 4017, 0, 0, 44, 0, 1160, 4094, 3901, 119, 0, 4055, 3, 3772, 41] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3691, 12, 1319, 1, 483, 200, 0, 0, 0, 10, 212, 0, 3196, 448, 0, 0, 3814, 44, 0, 0, 0, 0, 0, 0, 0, 67, 191, 98, 4060, 184, 0, 0, 0, 2850, 0, 28, 0, 0, 0, 0, 0, 155, 0, 3268, 1376, 672, 0, 0, 2207, 102, 2, 0, 0, 0, 0, 0, 698, 22, 3337, 21, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5706, 3866, 3858, 4600, 2878, 5471, 2311, 4078] avg=4096.0000 max=5706 max_rank=0 min=2311 min_rank=6 imbalance_ratio=1.393066 max_load_reduction=27062 max_load_reduction_pct=0.825867 moved_to_echo=14738 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 120, 1282, 1, 273, 204, 0, 0, 0, 1, 9, 0, 3204, 358, 0, 0, 3891, 0, 2, 520, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 184, 0, 0, 3312, 1613, 0, 0, 0, 0, 0, 0, 0, 49, 0, 10, 1101, 380, 0, 0, 1765, 1910, 8, 0, 0, 0, 0, 0, 700, 0, 3682, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5975, 3572, 4413, 4278, 4925, 1540, 3683, 4382] avg=4096.0000 max=5975 max_rank=0 min=1540 min_rank=5 imbalance_ratio=1.458740 max_load_reduction=26793 max_load_reduction_pct=0.817657 moved_to_echo=14530 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3600, 0, 43, 504, 339, 203, 0, 0, 0, 0, 23, 0, 3202, 455, 0, 0, 3884, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 184, 0, 0, 4058, 2006, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1245, 1398, 472, 0, 0, 2242, 2, 0, 0, 0, 0, 0, 0, 699, 1, 4090, 10, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4689, 3680, 3891, 4280, 6064, 3120, 2244, 4800] avg=4096.0000 max=6064 max_rank=4 min=2244 min_rank=6 imbalance_ratio=1.480469 max_load_reduction=26704 max_load_reduction_pct=0.814941 moved_to_echo=16228 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1051, 241, 670, 172, 468, 194, 0, 0, 0, 226, 1345, 0, 3151, 440, 0, 0, 3710, 1, 0, 0, 0, 0, 0, 0, 0, 7, 53, 161, 4054, 181, 0, 0, 0, 2765, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3540, 1350, 651, 0, 0, 2163, 1551, 21, 0, 0, 0, 0, 0, 688, 0, 3913, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2796, 5162, 3711, 4456, 2766, 5541, 3735, 4601] avg=4096.0000 max=5541 max_rank=5 min=2766 min_rank=4 imbalance_ratio=1.352783 max_load_reduction=27227 max_load_reduction_pct=0.830902 moved_to_echo=16643 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 123, 12, 493, 204, 0, 0, 0, 0, 388, 0, 3211, 426, 0, 0, 3892, 0, 0, 76, 0, 0, 0, 0, 0, 139, 0, 178, 4075, 184, 0, 0, 26, 2915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1307, 687, 0, 0, 2094, 3517, 0, 0, 0, 0, 0, 0, 701, 0, 4026, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[832, 4025, 3968, 4576, 2941, 6088, 5611, 4727] avg=4096.0000 max=6088 max_rank=5 min=832 min_rank=0 imbalance_ratio=1.486328 max_load_reduction=26680 max_load_reduction_pct=0.814209 moved_to_echo=19367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3871, 11, 0, 289, 465, 206, 0, 0, 8, 0, 0, 1, 2981, 328, 0, 0, 3861, 0, 44, 1220, 0, 0, 0, 0, 0, 137, 310, 0, 1085, 174, 0, 0, 4023, 2717, 0, 0, 0, 0, 0, 0, 0, 27, 3, 3863, 997, 646, 0, 0, 1586, 3001, 10, 0, 0, 0, 0, 0, 642, 1, 260, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4842, 3318, 5125, 1706, 6740, 5536, 4597, 904] avg=4096.0000 max=6740 max_rank=4 min=904 min_rank=7 imbalance_ratio=1.645508 max_load_reduction=26028 max_load_reduction_pct=0.794312 moved_to_echo=17777 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3651, 0, 5, 45, 490, 203, 0, 0, 4, 0, 1441, 0, 3167, 455, 0, 0, 3886, 0, 0, 1, 0, 0, 0, 0, 13, 0, 46, 0, 4093, 182, 0, 0, 856, 2895, 0, 8, 0, 0, 0, 0, 1, 0, 41, 115, 1396, 682, 0, 0, 2236, 2087, 3, 0, 0, 0, 0, 0, 692, 0, 4074, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4394, 5067, 3887, 4334, 3759, 2235, 4326, 4766] avg=4096.0000 max=5067 max_rank=1 min=2235 min_rank=5 imbalance_ratio=1.237061 max_load_reduction=27701 max_load_reduction_pct=0.845367 moved_to_echo=15086 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[22, 3, 2849, 0, 456, 204, 0, 0, 0, 0, 445, 0, 3178, 455, 0, 0, 3889, 5, 21, 0, 0, 0, 0, 0, 17, 4, 2, 315, 4017, 183, 0, 0, 0, 2695, 0, 0, 0, 0, 0, 0, 0, 44, 0, 1160, 1398, 635, 0, 0, 2241, 3901, 119, 0, 0, 0, 0, 0, 694, 3, 3772, 41, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3534, 4078, 3915, 4538, 2695, 3237, 6261, 4510] avg=4096.0000 max=6261 max_rank=6 min=2695 min_rank=4 imbalance_ratio=1.528564 max_load_reduction=26507 max_load_reduction_pct=0.808929 moved_to_echo=16703 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 384, 128, 17408, 10368, 4864, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16640, 16128, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[20096, 512, 6400, 1024, 3584, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 3968, 128, 25344, 3456, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[12288, 20480, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5632, 128, 27264, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[30848, 128, 128, 1920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 384, 640, 768, 29696, 1536, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 13, 2280, 0, 15, 4078, 0, 0, 68, 0, 0, 0, 0, 3128, 3067, 0, 0, 3733, 75, 4033, 1, 3976, 1, 0, 3990, 27, 0, 128, 0, 20, 4053, 82] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 4093, 18, 0, 3947, 169, 0, 0, 1, 0, 6, 0, 3799, 3769, 0, 0, 4061, 0, 4095, 397, 176, 2, 97, 4006, 3, 31, 0, 6, 0, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[27, 0, 4042, 3, 0, 3997, 2, 0, 0, 0, 0, 382, 26, 4089, 3569, 0, 99, 164, 0, 4080, 1, 4085, 2, 0, 4057, 0, 0, 44, 0, 9, 4090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4072, 27, 0, 4096, 7, 2, 519, 502, 214, 3848, 1029, 3942, 0, 17, 22, 33, 0, 3486, 0, 3383, 1, 0, 3650, 0, 0, 2, 5, 0, 3910, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2513, 1, 70, 4096, 251, 0, 1, 0, 0, 137, 0, 4094, 3956, 0, 0, 4091, 0, 4096, 0, 1914, 0, 0, 4088, 7, 0, 0, 0, 0, 3453, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 7, 84, 3877, 0, 3858, 234, 0, 0, 1, 0, 3149, 55, 3870, 3400, 14, 1, 1381, 0, 4075, 121, 387, 0, 0, 4096, 0, 0, 0, 17, 98, 4043, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 1666, 1829, 1, 3865, 0, 0, 0, 2, 0, 5, 0, 4084, 4043, 0, 257, 4055, 0, 4092, 0, 697, 0, 7, 3905, 0, 52, 0, 35, 110, 4061, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 76, 98, 66, 4084, 856, 0, 0, 0, 0, 0, 0, 4047, 3984, 9, 14, 4094, 53, 4077, 0, 3057, 0, 55, 4047, 5, 7, 0, 0, 6, 4090, 42] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 2280, 0, 1011, 0, 0, 0, 15, 3961, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 2410, 0, 0, 0, 0, 589, 3067, 0, 0, 0, 0, 0, 0, 3733, 75, 1341, 0, 0, 0, 0, 1, 3976, 1, 0, 1651, 129, 0, 0, 3990, 27, 0, 128, 48, 30, 0, 0, 0, 20, 4053, 82, 69, 0, 0, 0] rerouted_tokens_per_ep_rank=[3304, 3976, 2478, 3656, 5149, 5758, 4223, 4224] avg=4096.0000 max=5758 max_rank=5 min=2478 min_rank=2 imbalance_ratio=1.405762 max_load_reduction=27010 max_load_reduction_pct=0.824280 moved_to_echo=19354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4093, 18, 1027, 0, 0, 0, 0, 3834, 169, 0, 0, 0, 0, 0, 0, 1, 0, 6, 2927, 0, 0, 0, 0, 715, 3769, 0, 0, 0, 0, 0, 0, 4061, 0, 1361, 0, 0, 0, 0, 397, 176, 2, 97, 1676, 157, 0, 0, 4006, 3, 31, 0, 46, 31, 0, 0, 6, 0, 4091, 0, 67, 0, 0, 0] rerouted_tokens_per_ep_rank=[5139, 4003, 2934, 4484, 5422, 2505, 4117, 4164] avg=4096.0000 max=5422 max_rank=4 min=2505 min_rank=5 imbalance_ratio=1.323730 max_load_reduction=27346 max_load_reduction_pct=0.834534 moved_to_echo=16208 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[27, 0, 4042, 3, 1023, 0, 0, 0, 0, 3882, 2, 0, 0, 0, 0, 0, 0, 0, 0, 382, 3151, 0, 0, 0, 26, 769, 3569, 0, 0, 0, 0, 0, 99, 164, 0, 1357, 0, 0, 0, 0, 1, 4085, 2, 0, 1670, 169, 0, 0, 4057, 0, 0, 44, 47, 30, 0, 0, 0, 9, 4090, 0, 68, 0, 0, 0] rerouted_tokens_per_ep_rank=[5095, 3884, 3533, 4364, 1620, 5927, 4178, 4167] avg=4096.0000 max=5927 max_rank=5 min=1620 min_rank=4 imbalance_ratio=1.447021 max_load_reduction=26841 max_load_reduction_pct=0.819122 moved_to_echo=15892 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2513, 1, 1027, 0, 0, 0, 70, 3979, 251, 0, 0, 0, 0, 0, 1, 0, 0, 137, 3155, 0, 0, 0, 0, 769, 3956, 0, 0, 0, 0, 0, 0, 4091, 0, 1361, 0, 0, 0, 0, 0, 1914, 0, 0, 1677, 170, 0, 0, 4088, 7, 0, 0, 48, 31, 0, 0, 0, 0, 3453, 0, 69, 0, 0, 0] rerouted_tokens_per_ep_rank=[3541, 4300, 3293, 4725, 5452, 3761, 4174, 3522] avg=4096.0000 max=5452 max_rank=4 min=3293 min_rank=2 imbalance_ratio=1.331055 max_load_reduction=27316 max_load_reduction_pct=0.833618 moved_to_echo=16909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1666, 1829, 1026, 0, 0, 0, 1, 3755, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 3147, 0, 0, 0, 0, 768, 4043, 0, 0, 0, 0, 0, 257, 4055, 0, 1360, 0, 0, 0, 0, 0, 697, 0, 7, 1675, 169, 0, 0, 3905, 0, 52, 0, 45, 31, 0, 0, 35, 110, 4061, 0, 65, 0, 0, 0] rerouted_tokens_per_ep_rank=[4523, 3756, 3154, 4811, 5672, 2548, 4033, 4271] avg=4096.0000 max=5672 max_rank=4 min=2548 min_rank=5 imbalance_ratio=1.384766 max_load_reduction=27096 max_load_reduction_pct=0.826904 moved_to_echo=16524 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4072, 27, 879, 0, 0, 0, 0, 3969, 7, 2, 0, 0, 0, 0, 519, 502, 214, 3848, 3042, 0, 0, 0, 1029, 732, 0, 17, 0, 0, 0, 0, 22, 33, 0, 1145, 0, 0, 0, 0, 0, 3383, 1, 0, 1432, 168, 0, 0, 3650, 0, 0, 2, 53, 30, 0, 0, 5, 0, 3910, 1, 74, 0, 0, 0] rerouted_tokens_per_ep_rank=[4978, 3978, 8125, 1778, 1200, 4984, 3735, 3990] avg=4096.0000 max=8125 max_rank=2 min=1200 min_rank=4 imbalance_ratio=1.983643 max_load_reduction=24643 max_load_reduction_pct=0.752045 moved_to_echo=13909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 84, 3877, 1022, 0, 0, 0, 0, 3748, 234, 0, 0, 0, 0, 0, 0, 1, 0, 3149, 2982, 0, 0, 0, 55, 728, 3400, 14, 0, 0, 0, 0, 1, 1381, 0, 1355, 0, 0, 0, 0, 121, 387, 0, 0, 1668, 160, 0, 0, 4096, 0, 0, 0, 45, 30, 0, 0, 17, 98, 4043, 0, 65, 0, 0, 0] rerouted_tokens_per_ep_rank=[4990, 3982, 6132, 4197, 2737, 2336, 4171, 4223] avg=4096.0000 max=6132 max_rank=2 min=2336 min_rank=5 imbalance_ratio=1.497070 max_load_reduction=26636 max_load_reduction_pct=0.812866 moved_to_echo=13467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 76, 98, 1023, 0, 0, 0, 66, 3967, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3119, 0, 0, 0, 0, 760, 3984, 9, 0, 0, 0, 0, 14, 4094, 53, 1355, 0, 0, 0, 0, 0, 3057, 0, 55, 1669, 168, 0, 0, 4047, 5, 7, 0, 48, 30, 0, 0, 0, 6, 4090, 42, 69, 0, 0, 0] rerouted_tokens_per_ep_rank=[1198, 4889, 3119, 4753, 5516, 4949, 4137, 4207] avg=4096.0000 max=5516 max_rank=4 min=1198 min_rank=0 imbalance_ratio=1.346680 max_load_reduction=27252 max_load_reduction_pct=0.831665 moved_to_echo=18809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 18944, 5888, 8064, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[640, 17792, 128, 256, 13184, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 21632, 128, 10752, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[640, 512, 256, 7552, 23936, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 5888, 25856, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 256, 31872, 128, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 31104, 1536, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31872, 128, 128, 256, 384, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3854, 34, 0, 102, 4050, 4048, 542, 288, 10, 0, 0, 28, 3316, 3, 991, 96, 5, 0, 18, 12, 0, 0, 4068, 4069, 51, 1, 2, 0, 0, 5, 3248, 3927] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4081, 33, 0, 0, 4083, 4066, 0, 0, 0, 1, 8, 0, 189, 0, 3157, 0, 2251, 0, 0, 88, 27, 0, 4096, 4032, 18, 0, 0, 0, 0, 736, 1834, 4068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[3345, 0, 15, 0, 4096, 3376, 0, 0, 0, 0, 1, 5, 2, 0, 4083, 375, 968, 23, 0, 0, 911, 0, 4087, 4091, 0, 86, 0, 0, 0, 895, 2341, 4068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3374, 9, 2, 0, 3619, 3914, 0, 2, 0, 0, 419, 3, 0, 30, 868, 1, 8, 346, 0, 0, 0, 136, 4090, 4095, 0, 2700, 0, 0, 296, 4096, 3146, 1614] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 4095, 0, 0, 41, 0, 0, 2, 0, 0, 3745, 2331, 0, 0, 0, 0, 0, 27, 3061, 4051, 0, 0, 0, 8, 0, 3105, 15, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4094, 0, 1, 0, 4003, 3820, 84, 13, 3, 0, 811, 0, 101, 0, 3933, 3214, 0, 8, 0, 20, 0, 18, 4093, 4088, 0, 80, 12, 0, 0, 1255, 222, 2895] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3507, 0, 0, 0, 4096, 4095, 894, 2, 0, 0, 0, 0, 1747, 0, 1333, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 823, 0, 0, 0, 3975, 6, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4005, 0, 0, 3, 4070, 4085, 315, 8, 0, 3, 0, 0, 1, 0, 151, 0, 4, 0, 0, 17, 1, 52, 4076, 4059, 0, 193, 1, 0, 2, 3935, 4038, 3749] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3854, 34, 0, 102, 121, 112, 0, 9, 0, 3936, 542, 288, 0, 0, 0, 0, 10, 0, 0, 28, 3925, 0, 0, 0, 3316, 3, 991, 96, 426, 0, 0, 0, 5, 0, 18, 12, 3512, 144, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 51, 1, 2, 0, 3380, 538, 0, 0, 0, 5, 3248, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4232, 4766, 3963, 4832, 3691, 4059, 3972, 3253] avg=4096.0000 max=4832 max_rank=3 min=3253 min_rank=7 imbalance_ratio=1.179688 max_load_reduction=27936 max_load_reduction_pct=0.852539 moved_to_echo=14975 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3345, 0, 15, 0, 125, 94, 18, 9, 0, 3282, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 3947, 0, 0, 0, 2, 0, 4083, 375, 443, 0, 0, 0, 968, 23, 0, 0, 3553, 144, 0, 0, 911, 0, 4078, 0, 0, 0, 0, 0, 0, 86, 0, 0, 3500, 543, 0, 0, 0, 877, 2341, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3606, 3282, 3953, 4903, 4688, 4989, 4129, 3218] avg=4096.0000 max=4989 max_rank=5 min=3218 min_rank=7 imbalance_ratio=1.218018 max_load_reduction=27779 max_load_reduction_pct=0.847748 moved_to_echo=17024 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 1, 0, 91, 106, 26, 9, 0, 3714, 84, 13, 0, 0, 0, 0, 3, 0, 811, 0, 3944, 0, 0, 0, 101, 0, 3933, 3214, 314, 0, 0, 0, 0, 8, 0, 20, 3472, 144, 0, 0, 0, 18, 4084, 0, 0, 0, 0, 0, 0, 80, 12, 0, 2490, 531, 0, 0, 0, 1229, 222, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4327, 3811, 4758, 7562, 3644, 4102, 3113, 1451] avg=4096.0000 max=7562 max_rank=3 min=1451 min_rank=7 imbalance_ratio=1.846191 max_load_reduction=25206 max_load_reduction_pct=0.769226 moved_to_echo=12310 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3507, 0, 0, 0, 126, 114, 84, 9, 0, 3981, 894, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 0, 0, 0, 1747, 0, 1333, 2, 446, 0, 0, 0, 0, 0, 0, 0, 3553, 144, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 823, 0, 0, 3524, 543, 0, 0, 0, 3891, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3840, 4877, 3952, 3528, 3697, 4087, 4890, 3897] avg=4096.0000 max=4890 max_rank=6 min=3528 min_rank=3 imbalance_ratio=1.193848 max_load_reduction=27878 max_load_reduction_pct=0.850769 moved_to_echo=16571 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 126, 114, 65, 6, 0, 3981, 0, 0, 0, 0, 0, 0, 41, 0, 0, 2, 3909, 0, 0, 0, 0, 0, 3745, 2331, 446, 0, 0, 0, 0, 0, 0, 0, 3551, 142, 0, 0, 0, 27, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3524, 544, 0, 0, 0, 3040, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4407, 3981, 3952, 6522, 3693, 3082, 4076, 3055] avg=4096.0000 max=6522 max_rank=3 min=3055 min_rank=7 imbalance_ratio=1.592285 max_load_reduction=26246 max_load_reduction_pct=0.800964 moved_to_echo=13906 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3374, 9, 2, 0, 51, 113, 91, 11, 0, 3801, 0, 2, 0, 0, 0, 0, 0, 0, 419, 3, 3951, 0, 0, 0, 0, 30, 868, 1, 175, 0, 0, 0, 8, 346, 0, 0, 3139, 144, 0, 0, 0, 136, 4079, 0, 0, 0, 0, 0, 0, 2700, 0, 0, 1388, 480, 0, 0, 296, 4005, 3146, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3651, 3803, 4373, 1074, 3637, 4215, 4568, 7447] avg=4096.0000 max=7447 max_rank=7 min=1074 min_rank=3 imbalance_ratio=1.818115 max_load_reduction=25321 max_load_reduction_pct=0.772736 moved_to_echo=19867 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4081, 33, 0, 0, 126, 113, 15, 9, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 3890, 0, 0, 0, 189, 0, 3157, 0, 442, 0, 0, 0, 2251, 0, 0, 88, 3542, 142, 0, 0, 27, 0, 4087, 0, 0, 0, 0, 0, 18, 0, 0, 0, 3500, 541, 0, 0, 0, 721, 1834, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4377, 3953, 3899, 3788, 6023, 4114, 4059, 2555] avg=4096.0000 max=6023 max_rank=4 min=2555 min_rank=7 imbalance_ratio=1.470459 max_load_reduction=26745 max_load_reduction_pct=0.816193 moved_to_echo=16751 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4005, 0, 0, 3, 115, 113, 83, 9, 0, 3972, 315, 8, 0, 0, 0, 0, 0, 3, 0, 0, 3915, 0, 0, 0, 1, 0, 151, 0, 407, 0, 0, 0, 4, 0, 0, 17, 3530, 144, 0, 0, 1, 52, 4067, 0, 0, 0, 0, 0, 0, 193, 1, 0, 3227, 540, 0, 0, 2, 3852, 4038, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4328, 4295, 3918, 559, 3695, 4120, 3961, 7892] avg=4096.0000 max=7892 max_rank=7 min=559 min_rank=3 imbalance_ratio=1.926758 max_load_reduction=24876 max_load_reduction_pct=0.759155 moved_to_echo=19668 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 17664, 14976, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5376, 128, 18304, 6144, 3200, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 3968, 128, 128, 24576, 4352, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3328, 384, 128, 256, 27904, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 30720, 1920, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 1280, 128, 31488, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[1024, 256, 31616, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[30464, 128, 128, 128, 896, 896, 384, 128] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[378, 4, 4, 0, 0, 0, 0, 0, 1, 0, 118, 9, 43, 0, 5, 4091, 4005, 3844, 0, 0, 3947, 4090, 4078, 0, 17, 0, 0, 0, 0, 4035, 30, 4069] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 149, 2757, 0, 0, 0, 0, 0, 8, 0, 3988, 1217, 3883, 0, 26, 3851, 4078, 4008, 0, 17, 177, 44, 0, 232, 4048, 183, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4001, 0, 0, 4, 0, 52, 0, 78, 0, 0, 0, 0, 48, 3, 4, 4086, 4010, 4074, 79, 0, 1926, 290, 4096, 0, 0, 1432, 34, 0, 1, 4044, 579, 3927] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[18, 80, 0, 0, 0, 0, 0, 2595, 5, 0, 25, 0, 181, 0, 0, 3878, 1278, 4096, 12, 112, 4096, 4027, 4068, 0, 0, 0, 0, 0, 136, 4062, 3, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3187, 0, 0, 0, 0, 0, 0, 0, 0, 5, 997, 0, 3, 0, 1, 4096, 3113, 4096, 0, 0, 3760, 715, 4096, 0, 0, 10, 0, 0, 0, 3756, 837, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[3176, 0, 362, 220, 20, 9, 50, 3253, 118, 0, 0, 0, 0, 1, 0, 109, 4096, 3991, 0, 0, 4090, 1797, 3912, 70, 4, 697, 0, 28, 0, 474, 3202, 3089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[39, 2, 0, 0, 4, 0, 1, 2, 0, 25, 0, 0, 0, 0, 0, 4046, 4096, 4057, 1, 12, 4095, 4094, 4078, 0, 2, 37, 0, 0, 0, 4085, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4044, 32, 6, 0, 5, 0, 0, 0, 3, 100, 2188, 0, 0, 7, 1, 3806, 4092, 4087, 0, 4, 39, 1902, 4089, 14, 0, 38, 7, 0, 1, 4048, 168, 4087] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[378, 4, 4, 0, 1668, 753, 0, 0, 0, 0, 0, 0, 2298, 590, 0, 0, 1, 0, 118, 9, 3479, 276, 0, 0, 43, 0, 5, 4091, 490, 163, 0, 0, 4005, 793, 0, 0, 0, 0, 0, 0, 1789, 4090, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 3802, 0, 0, 0, 0, 3872, 30, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2807, 2888, 3883, 4792, 4798, 5879, 3819, 3902] avg=4096.0000 max=5879 max_rank=5 min=2807 min_rank=0 imbalance_ratio=1.435303 max_load_reduction=26889 max_load_reduction_pct=0.820587 moved_to_echo=18398 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 1627, 761, 0, 0, 0, 0, 149, 2757, 2322, 594, 0, 0, 0, 0, 0, 0, 3502, 272, 0, 0, 0, 8, 0, 3988, 478, 163, 0, 0, 1217, 800, 0, 26, 0, 0, 0, 0, 1746, 4078, 0, 0, 0, 0, 0, 0, 17, 177, 44, 0, 3736, 0, 0, 0, 232, 3885, 183, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2394, 5822, 3774, 4637, 2043, 5824, 3974, 4300] avg=4096.0000 max=5824 max_rank=5 min=2043 min_rank=4 imbalance_ratio=1.421875 max_load_reduction=26944 max_load_reduction_pct=0.822266 moved_to_echo=16141 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[39, 2, 0, 0, 1730, 795, 0, 0, 4, 0, 1, 2, 2426, 594, 0, 0, 0, 25, 0, 0, 3498, 277, 0, 0, 0, 0, 0, 4046, 508, 165, 0, 0, 4096, 836, 1, 12, 0, 0, 0, 0, 1857, 4094, 0, 0, 0, 0, 0, 0, 2, 37, 0, 0, 3801, 0, 0, 0, 0, 3920, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2566, 3027, 3800, 4719, 4945, 5951, 3840, 3920] avg=4096.0000 max=5951 max_rank=5 min=2566 min_rank=0 imbalance_ratio=1.452881 max_load_reduction=26817 max_load_reduction_pct=0.818390 moved_to_echo=18656 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3187, 0, 0, 0, 1589, 803, 0, 0, 0, 0, 0, 0, 2449, 595, 0, 0, 0, 5, 997, 0, 3501, 278, 0, 0, 3, 0, 1, 4096, 467, 152, 0, 0, 3113, 844, 0, 0, 0, 0, 0, 0, 1704, 715, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 3818, 0, 0, 0, 0, 3604, 837, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5579, 3044, 4781, 4719, 3957, 2419, 3828, 4441] avg=4096.0000 max=5579 max_rank=0 min=2419 min_rank=5 imbalance_ratio=1.362061 max_load_reduction=27189 max_load_reduction_pct=0.829742 moved_to_echo=14645 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3176, 0, 362, 220, 1731, 785, 0, 0, 20, 9, 50, 3253, 2390, 449, 0, 0, 118, 0, 0, 0, 2640, 266, 0, 0, 0, 1, 0, 109, 512, 23, 0, 0, 4096, 816, 0, 0, 0, 0, 0, 0, 1847, 1797, 0, 70, 0, 0, 0, 0, 4, 697, 0, 28, 3646, 0, 0, 0, 0, 451, 3202, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6274, 6171, 3024, 645, 4912, 3714, 4375, 3653] avg=4096.0000 max=6274 max_rank=0 min=645 min_rank=3 imbalance_ratio=1.531738 max_load_reduction=26494 max_load_reduction_pct=0.808533 moved_to_echo=16654 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4001, 0, 0, 4, 814, 798, 0, 0, 0, 52, 0, 78, 2436, 570, 0, 0, 0, 0, 0, 0, 3357, 279, 0, 0, 48, 3, 4, 4086, 239, 163, 0, 0, 4010, 840, 79, 0, 0, 0, 0, 0, 873, 290, 0, 0, 0, 0, 0, 0, 0, 1432, 34, 0, 3817, 0, 0, 0, 1, 3881, 579, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5617, 3136, 3636, 4543, 4929, 1163, 5283, 4461] avg=4096.0000 max=5617 max_rank=0 min=1163 min_rank=5 imbalance_ratio=1.371338 max_load_reduction=27151 max_load_reduction_pct=0.828583 moved_to_echo=15836 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4044, 32, 6, 0, 16, 801, 0, 0, 5, 0, 0, 0, 2444, 593, 0, 0, 3, 100, 2188, 0, 3494, 277, 0, 0, 0, 7, 1, 3806, 4, 163, 0, 0, 4092, 842, 0, 4, 0, 0, 0, 0, 19, 1902, 0, 14, 0, 0, 0, 0, 0, 38, 7, 0, 3812, 0, 0, 0, 1, 3885, 168, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4899, 3042, 6062, 3981, 4938, 1935, 3857, 4054] avg=4096.0000 max=6062 max_rank=2 min=1935 min_rank=5 imbalance_ratio=1.479980 max_load_reduction=26706 max_load_reduction_pct=0.815002 moved_to_echo=14784 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[18, 80, 0, 0, 1731, 803, 0, 0, 0, 0, 0, 2595, 2449, 594, 0, 0, 5, 0, 25, 0, 3502, 276, 0, 0, 181, 0, 0, 3878, 509, 164, 0, 0, 1278, 844, 12, 112, 0, 0, 0, 0, 1856, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3792, 0, 0, 0, 136, 3898, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2632, 5638, 3808, 4732, 2246, 5883, 3792, 4037] avg=4096.0000 max=5883 max_rank=5 min=2246 min_rank=4 imbalance_ratio=1.436279 max_load_reduction=26885 max_load_reduction_pct=0.820465 moved_to_echo=15958 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14976, 128, 384, 256, 11008, 6400, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11776, 21120, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 256, 8704, 19328, 4608, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[25984, 6656, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 128, 28160, 3328, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 3328, 128, 27008, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 2432, 128, 128, 30336, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 27520, 5120, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[2, 4082, 4090, 0, 0, 13, 4096, 4089, 319, 1, 1424, 1, 1071, 0, 0, 1, 1, 0, 0, 0, 4094, 1584, 0, 1, 0, 1039, 865, 1886, 4095, 14, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[185, 4096, 4096, 0, 0, 0, 476, 4096, 3, 0, 4091, 0, 3440, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 0, 0, 13, 2, 0, 0, 4096, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[29, 4017, 4076, 0, 2, 0, 4088, 4096, 0, 2, 3838, 86, 2, 0, 5, 0, 1, 6, 1, 0, 4094, 4080, 0, 44, 0, 217, 0, 1, 4033, 0, 50, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[2, 4096, 4046, 0, 2, 0, 1580, 4096, 4041, 0, 2068, 0, 35, 0, 0, 0, 0, 6, 0, 0, 4096, 36, 0, 0, 0, 478, 1, 4091, 4088, 5, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[146, 3924, 4078, 0, 0, 0, 4096, 4094, 3823, 0, 267, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4096, 3678, 0, 3, 363, 5, 0, 0, 4089, 11, 74, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[379, 4096, 4006, 0, 95, 0, 3805, 4096, 22, 0, 4091, 4, 1843, 8, 0, 5, 0, 2, 0, 16, 4084, 526, 0, 0, 0, 0, 0, 1571, 4096, 0, 23, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 3648, 4086, 0, 58, 310, 2461, 4085, 0, 0, 4058, 84, 3490, 0, 17, 0, 0, 0, 0, 30, 1870, 37, 29, 0, 3796, 281, 2, 331, 4095, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[124, 4096, 2898, 25, 0, 0, 1924, 4028, 0, 0, 3954, 0, 3179, 0, 4, 0, 4, 0, 0, 0, 4091, 4053, 0, 97, 180, 15, 0, 0, 4095, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 64, 4090, 0, 0, 0, 0, 0, 0, 13, 4096, 1223, 0, 0, 0, 0, 319, 1, 1424, 1, 65, 12, 0, 0, 1071, 0, 0, 1, 2458, 0, 0, 0, 1, 0, 0, 0, 4018, 143, 0, 0, 1948, 1584, 0, 1, 0, 0, 0, 0, 0, 1039, 865, 1886, 2081, 265, 0, 0, 4083, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 5332, 1822, 3530, 4162, 3533, 6136, 4097] avg=4096.0000 max=6136 max_rank=6 min=1822 min_rank=2 imbalance_ratio=1.498047 max_load_reduction=26632 max_load_reduction_pct=0.812744 moved_to_echo=17928 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[29, 63, 4076, 0, 0, 0, 0, 0, 2, 0, 4088, 1225, 0, 0, 0, 0, 0, 2, 3838, 86, 65, 12, 0, 0, 2, 0, 5, 0, 2462, 0, 0, 0, 1, 6, 1, 0, 3954, 143, 0, 0, 1948, 4080, 0, 44, 0, 0, 0, 0, 0, 217, 0, 1, 2081, 266, 0, 0, 4021, 0, 50, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4168, 5315, 4003, 2469, 4105, 6072, 2565, 4071] avg=4096.0000 max=6072 max_rank=5 min=2469 min_rank=3 imbalance_ratio=1.482422 max_load_reduction=26696 max_load_reduction_pct=0.814697 moved_to_echo=16813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[185, 64, 4096, 0, 0, 0, 0, 0, 0, 0, 476, 1225, 0, 0, 0, 0, 3, 0, 4091, 0, 65, 12, 0, 0, 3440, 0, 0, 0, 2462, 0, 0, 0, 0, 0, 0, 0, 4032, 143, 0, 0, 1939, 4096, 0, 0, 0, 0, 0, 0, 13, 2, 0, 0, 2071, 266, 0, 0, 4084, 0, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4345, 1701, 4171, 5902, 4175, 6035, 2352, 4087] avg=4096.0000 max=6035 max_rank=5 min=1701 min_rank=1 imbalance_ratio=1.473389 max_load_reduction=26733 max_load_reduction_pct=0.815826 moved_to_echo=16649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 64, 4046, 0, 0, 0, 0, 0, 2, 0, 1580, 1225, 0, 0, 0, 0, 4041, 0, 2068, 0, 65, 12, 0, 0, 35, 0, 0, 0, 2462, 0, 0, 0, 0, 6, 0, 0, 4032, 143, 0, 0, 1949, 36, 0, 0, 0, 0, 0, 0, 0, 478, 1, 4091, 2082, 266, 0, 0, 4076, 5, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 2807, 6186, 2497, 4181, 1985, 6918, 4082] avg=4096.0000 max=6918 max_rank=6 min=1985 min_rank=5 imbalance_ratio=1.688965 max_load_reduction=25850 max_load_reduction_pct=0.788879 moved_to_echo=17166 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[379, 64, 4006, 0, 0, 0, 0, 0, 95, 0, 3805, 1225, 0, 0, 0, 0, 22, 0, 4091, 4, 65, 12, 0, 0, 1843, 8, 0, 5, 2462, 0, 0, 0, 0, 2, 0, 16, 4032, 143, 0, 0, 1943, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1571, 2076, 266, 0, 0, 4084, 0, 23, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4449, 5125, 4194, 4318, 4193, 2469, 3913, 4107] avg=4096.0000 max=5125 max_rank=1 min=2469 min_rank=5 imbalance_ratio=1.251221 max_load_reduction=27643 max_load_reduction_pct=0.843597 moved_to_echo=14682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 55, 4086, 0, 0, 0, 0, 0, 58, 310, 2461, 1210, 0, 0, 0, 0, 0, 0, 4058, 84, 35, 17, 0, 0, 3490, 0, 17, 0, 2459, 0, 0, 0, 0, 0, 0, 30, 3593, 147, 0, 0, 881, 37, 29, 0, 0, 0, 0, 0, 3796, 281, 2, 331, 954, 269, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4039, 4194, 5966, 3770, 947, 5633, 4078] avg=4096.0000 max=5966 max_rank=3 min=947 min_rank=5 imbalance_ratio=1.456543 max_load_reduction=26802 max_load_reduction_pct=0.817932 moved_to_echo=14428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[146, 62, 4078, 0, 0, 0, 0, 0, 0, 0, 4096, 1224, 0, 0, 0, 0, 3823, 0, 267, 0, 65, 12, 0, 0, 0, 0, 21, 0, 2461, 0, 0, 0, 0, 0, 0, 0, 3862, 143, 0, 0, 1949, 3678, 0, 3, 0, 0, 0, 0, 363, 5, 0, 0, 2082, 266, 0, 0, 4077, 11, 74, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4286, 5320, 4167, 2482, 4005, 5630, 2716, 4162] avg=4096.0000 max=5630 max_rank=5 min=2482 min_rank=3 imbalance_ratio=1.374512 max_load_reduction=27138 max_load_reduction_pct=0.828186 moved_to_echo=16513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[124, 64, 2898, 25, 0, 0, 0, 0, 0, 0, 1924, 1205, 0, 0, 0, 0, 0, 0, 3954, 0, 65, 12, 0, 0, 3179, 0, 4, 0, 2421, 0, 0, 0, 4, 0, 0, 0, 4032, 141, 0, 0, 1947, 4053, 0, 97, 0, 0, 0, 0, 180, 15, 0, 0, 2079, 261, 0, 0, 4083, 0, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3111, 3129, 4031, 5604, 4177, 6097, 2535, 4084] avg=4096.0000 max=6097 max_rank=5 min=2535 min_rank=6 imbalance_ratio=1.488525 max_load_reduction=26671 max_load_reduction_pct=0.813934 moved_to_echo=16893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14592, 18176, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[13184, 128, 128, 128, 19712, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 384, 22528, 9856, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4352, 2048, 896, 7936, 15616, 2176, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[896, 512, 31488, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 128, 23808, 256, 512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 31616, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 3531, 314, 0, 2, 0, 0, 673, 17, 0, 0, 0, 4080, 4092, 0, 3986, 0, 0, 0, 4091, 50, 4044, 2, 0, 0, 50, 8, 0, 3732, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[27, 3164, 0, 0, 59, 0, 0, 432, 288, 0, 0, 26, 3614, 4065, 0, 3962, 0, 0, 834, 4041, 0, 4091, 0, 0, 0, 2, 1, 5, 4064, 27, 4066, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4091, 49, 0, 0, 0, 0, 0, 123, 0, 3, 151, 3923, 4086, 0, 1244, 0, 0, 352, 4094, 0, 3867, 0, 3, 118, 2930, 0, 0, 3639, 0, 4094, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 212, 0, 0, 35, 0, 3862, 0, 0, 1, 51, 4081, 4095, 0, 0, 21, 0, 0, 4096, 8, 4039, 36, 0, 0, 3998, 0, 0, 49, 0, 4088, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[22, 3218, 0, 0, 18, 2, 0, 0, 20, 37, 0, 0, 3681, 4060, 0, 4096, 9, 834, 423, 4096, 0, 4078, 0, 9, 0, 32, 0, 0, 4060, 0, 4073, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 3892, 4096, 0, 4096, 0, 0, 151, 4096, 0, 4094, 0, 0, 0, 3598, 0, 0, 479, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[17, 4095, 3576, 0, 1, 3, 12, 85, 1, 0, 0, 0, 1526, 4092, 211, 1341, 0, 1739, 1, 3938, 0, 3913, 1, 3, 997, 0, 111, 0, 2974, 0, 4084, 47] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4084, 12, 0, 0, 5, 12, 5, 230, 81, 0, 0, 3666, 4052, 0, 3914, 0, 194, 0, 4081, 0, 3904, 0, 15, 8, 3987, 43, 0, 274, 0, 4096, 105] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3319, 314, 0, 0, 0, 0, 0, 2, 0, 0, 673, 2427, 1033, 113, 0, 17, 0, 0, 0, 3979, 0, 0, 0, 1423, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 3548, 0, 0, 0, 0, 50, 4044, 2, 0, 71, 0, 0, 0, 0, 50, 8, 0, 1624, 543, 141, 0, 3732, 0, 1669, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3633, 4248, 3996, 5409, 3548, 4167, 2366, 5401] avg=4096.0000 max=5409 max_rank=3 min=2366 min_rank=6 imbalance_ratio=1.320557 max_load_reduction=27359 max_load_reduction_pct=0.834930 moved_to_echo=15482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[27, 2974, 0, 0, 0, 0, 0, 0, 59, 0, 0, 432, 2409, 915, 111, 0, 288, 0, 0, 26, 3954, 0, 0, 0, 1260, 0, 0, 3962, 0, 0, 0, 0, 0, 0, 834, 3505, 0, 0, 0, 0, 0, 4091, 0, 0, 63, 0, 0, 0, 0, 2, 1, 5, 1439, 536, 127, 0, 4064, 27, 1657, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3001, 3926, 4268, 5222, 4339, 4154, 2110, 5748] avg=4096.0000 max=5748 max_rank=7 min=2110 min_rank=6 imbalance_ratio=1.403320 max_load_reduction=27020 max_load_reduction_pct=0.824585 moved_to_echo=16351 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3850, 212, 0, 0, 0, 0, 0, 0, 35, 0, 3862, 2422, 1033, 113, 0, 0, 0, 1, 51, 3982, 0, 0, 0, 1423, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 3552, 0, 0, 0, 0, 8, 4039, 36, 0, 82, 0, 0, 0, 0, 3998, 0, 0, 1625, 544, 164, 0, 49, 0, 1666, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 7465, 4034, 1423, 3573, 4165, 6331, 1715] avg=4096.0000 max=7465 max_rank=1 min=1423 min_rank=3 imbalance_ratio=1.822510 max_load_reduction=25303 max_load_reduction_pct=0.772186 moved_to_echo=15784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3845, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2426, 993, 113, 0, 123, 0, 3, 151, 3973, 0, 0, 0, 1368, 0, 0, 1244, 0, 0, 0, 0, 0, 0, 352, 3550, 0, 0, 0, 0, 0, 3867, 0, 3, 82, 0, 0, 0, 118, 2930, 0, 0, 1562, 544, 164, 0, 3639, 0, 1668, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3894, 3532, 4250, 2612, 3902, 3952, 5318, 5308] avg=4096.0000 max=5318 max_rank=6 min=2612 min_rank=3 imbalance_ratio=1.298340 max_load_reduction=27450 max_load_reduction_pct=0.837708 moved_to_echo=18480 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 2427, 985, 113, 0, 0, 0, 0, 0, 3983, 0, 0, 0, 1357, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 151, 3552, 0, 0, 0, 0, 0, 4094, 0, 0, 82, 0, 0, 0, 0, 3598, 0, 0, 1550, 544, 164, 0, 479, 0, 1669, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3846, 3603, 3983, 5453, 3703, 4176, 5856, 2148] avg=4096.0000 max=5856 max_rank=6 min=2148 min_rank=7 imbalance_ratio=1.429688 max_load_reduction=26912 max_load_reduction_pct=0.821289 moved_to_echo=15883 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[17, 3842, 3576, 0, 0, 0, 0, 0, 1, 3, 12, 85, 2425, 390, 113, 0, 1, 0, 0, 0, 3979, 0, 0, 0, 526, 0, 211, 1341, 0, 0, 0, 0, 0, 1739, 1, 3412, 0, 0, 0, 0, 0, 3913, 1, 3, 85, 0, 0, 0, 997, 0, 111, 0, 610, 526, 168, 0, 2974, 0, 1659, 47, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7435, 3029, 3980, 2078, 5152, 4002, 2412, 4680] avg=4096.0000 max=7435 max_rank=0 min=2078 min_rank=3 imbalance_ratio=1.815186 max_load_reduction=25333 max_load_reduction_pct=0.773102 moved_to_echo=16246 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[22, 3025, 0, 0, 0, 0, 0, 0, 18, 2, 0, 0, 2413, 932, 112, 0, 20, 37, 0, 0, 3948, 0, 0, 0, 1283, 0, 0, 4096, 0, 0, 0, 0, 9, 834, 423, 3552, 0, 0, 0, 0, 0, 4078, 0, 9, 64, 0, 0, 0, 0, 32, 0, 0, 1466, 544, 129, 0, 4060, 0, 1660, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3047, 3477, 4005, 5379, 4818, 4151, 2171, 5720] avg=4096.0000 max=5720 max_rank=7 min=2171 min_rank=6 imbalance_ratio=1.396484 max_load_reduction=27048 max_load_reduction_pct=0.825439 moved_to_echo=16860 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3838, 12, 0, 0, 0, 0, 0, 0, 5, 12, 5, 2427, 928, 111, 0, 230, 81, 0, 0, 3941, 0, 0, 0, 1278, 0, 0, 3914, 0, 0, 0, 0, 0, 194, 0, 3539, 0, 0, 0, 0, 0, 3904, 0, 15, 82, 0, 0, 0, 8, 3987, 43, 0, 1460, 542, 164, 0, 274, 0, 1669, 105, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3850, 3488, 4252, 5192, 3733, 4001, 6204, 2048] avg=4096.0000 max=6204 max_rank=6 min=2048 min_rank=7 imbalance_ratio=1.514648 max_load_reduction=26564 max_load_reduction_pct=0.810669 moved_to_echo=15986 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 5248, 19456, 7296, 1024, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19328, 128, 13440, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1152, 14720, 256, 128, 11392, 4352, 1280, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[9984, 0, 256, 22656, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 28544, 4224, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 2816, 1792, 28288, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32128, 128, 128, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 128, 128, 256, 31744, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[53, 0, 3757, 0, 0, 1407, 4096, 3567, 0, 5, 0, 3980, 14, 3482, 30, 0, 3822, 0, 4081, 0, 37, 5, 148, 0, 30, 3, 0, 161, 0, 0, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[3442, 0, 245, 0, 0, 273, 4007, 114, 0, 35, 0, 3841, 0, 4028, 1, 0, 4059, 0, 4080, 94, 0, 674, 119, 0, 3672, 0, 62, 0, 0, 0, 17, 4005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4044, 216, 2321, 73, 0, 0, 1453, 4046, 0, 0, 30, 4040, 0, 3956, 0, 3, 2974, 4, 2925, 0, 0, 7, 0, 1, 2554, 0, 0, 26, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1, 65, 4031, 12, 0, 0, 4051, 4096, 454, 0, 1, 461, 230, 3917, 0, 0, 3652, 0, 4083, 94, 0, 1, 0, 10, 3216, 0, 0, 25, 0, 330, 19, 4019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[2288, 0, 837, 0, 0, 0, 3934, 3998, 1, 0, 9, 769, 145, 4076, 2, 3, 2837, 6, 4096, 0, 6, 6, 0, 0, 2856, 0, 0, 0, 0, 2809, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[75, 0, 3710, 0, 0, 42, 4096, 4087, 0, 7, 0, 196, 0, 4094, 0, 0, 4082, 0, 4062, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 138, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[479, 0, 113, 100, 0, 1, 4067, 3909, 0, 70, 0, 3665, 0, 667, 532, 0, 4089, 0, 3552, 0, 0, 32, 0, 7, 3939, 4, 0, 395, 0, 3676, 0, 3471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3662, 0, 860, 0, 0, 3568, 4083, 4082, 0, 1, 0, 102, 0, 4095, 0, 24, 4062, 0, 3970, 0, 171, 140, 12, 0, 5, 0, 22, 0, 0, 0, 0, 3909] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[53, 0, 3757, 0, 252, 53, 0, 0, 0, 1407, 0, 3514, 0, 0, 0, 0, 0, 5, 0, 3980, 1997, 0, 0, 0, 14, 3482, 30, 0, 446, 0, 0, 0, 3822, 0, 397, 0, 0, 0, 0, 0, 37, 5, 148, 0, 4096, 212, 0, 0, 30, 3, 0, 161, 1475, 72, 0, 0, 0, 0, 0, 3320, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4921, 5982, 3972, 4219, 4498, 1741, 3320] avg=4096.0000 max=5982 max_rank=2 min=1741 min_rank=6 imbalance_ratio=1.460449 max_load_reduction=26786 max_load_reduction_pct=0.817444 moved_to_echo=13778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3442, 0, 245, 0, 247, 1, 0, 0, 0, 273, 0, 113, 0, 0, 0, 0, 0, 35, 0, 3841, 1997, 0, 0, 0, 0, 4028, 1, 0, 437, 0, 0, 0, 4059, 0, 397, 94, 0, 0, 0, 0, 0, 674, 119, 0, 4007, 212, 0, 0, 3672, 0, 62, 0, 1474, 71, 0, 0, 0, 0, 17, 3250, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3935, 386, 5873, 4466, 4550, 5012, 5279, 3267] avg=4096.0000 max=5873 max_rank=2 min=386 min_rank=1 imbalance_ratio=1.433838 max_load_reduction=26895 max_load_reduction_pct=0.820770 moved_to_echo=18108 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2288, 0, 837, 0, 252, 60, 0, 0, 0, 0, 0, 3938, 0, 0, 0, 0, 1, 0, 9, 769, 2005, 0, 0, 0, 145, 4076, 2, 3, 446, 0, 0, 0, 2837, 6, 398, 0, 0, 0, 0, 0, 6, 6, 0, 0, 3934, 213, 0, 0, 2856, 0, 0, 0, 1480, 72, 0, 0, 0, 2809, 0, 3320, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3437, 3938, 2784, 4672, 3241, 4159, 4408, 6129] avg=4096.0000 max=6129 max_rank=7 min=2784 min_rank=2 imbalance_ratio=1.496338 max_load_reduction=26639 max_load_reduction_pct=0.812958 moved_to_echo=17937 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[75, 0, 3710, 0, 252, 61, 0, 0, 0, 42, 0, 4026, 0, 0, 0, 0, 0, 7, 0, 196, 1988, 0, 0, 0, 0, 4094, 0, 0, 447, 0, 0, 0, 4082, 0, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 211, 0, 0, 4083, 0, 0, 0, 1468, 72, 0, 0, 0, 138, 0, 3325, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4068, 2191, 4541, 4477, 4307, 5623, 3463] avg=4096.0000 max=5623 max_rank=6 min=2191 min_rank=2 imbalance_ratio=1.372803 max_load_reduction=27145 max_load_reduction_pct=0.828400 moved_to_echo=17870 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 65, 4031, 12, 248, 61, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 454, 0, 1, 461, 1998, 0, 0, 0, 230, 3917, 0, 0, 438, 0, 0, 0, 3652, 0, 398, 94, 0, 0, 0, 0, 0, 1, 0, 10, 4051, 212, 0, 0, 3216, 0, 0, 25, 1475, 71, 0, 0, 0, 330, 19, 3262, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4418, 4035, 2914, 4585, 4144, 4274, 4787, 3611] avg=4096.0000 max=4787 max_rank=6 min=2914 min_rank=2 imbalance_ratio=1.168701 max_load_reduction=27981 max_load_reduction_pct=0.853912 moved_to_echo=16816 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[479, 0, 113, 100, 218, 64, 0, 0, 0, 1, 0, 3845, 0, 0, 0, 0, 0, 70, 0, 3665, 1742, 0, 0, 0, 0, 667, 532, 0, 382, 0, 0, 0, 4089, 0, 336, 0, 0, 0, 0, 0, 0, 32, 0, 7, 4067, 187, 0, 0, 3939, 4, 0, 395, 1287, 66, 0, 0, 0, 3676, 0, 2805, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[974, 3846, 5477, 1581, 4425, 4293, 5691, 6481] avg=4096.0000 max=6481 max_rank=7 min=974 min_rank=0 imbalance_ratio=1.582275 max_load_reduction=26287 max_load_reduction_pct=0.802216 moved_to_echo=20890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4044, 216, 2321, 73, 252, 61, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 30, 4040, 1431, 0, 0, 0, 0, 3956, 0, 3, 447, 0, 0, 0, 2974, 4, 285, 0, 0, 0, 0, 0, 0, 7, 0, 1, 1453, 152, 0, 0, 2554, 0, 0, 26, 1057, 72, 0, 0, 0, 0, 0, 3324, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6967, 3985, 5501, 4406, 3263, 1613, 3709, 3324] avg=4096.0000 max=6967 max_rank=0 min=1613 min_rank=5 imbalance_ratio=1.700928 max_load_reduction=25801 max_load_reduction_pct=0.787384 moved_to_echo=11909 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3662, 0, 860, 0, 241, 61, 0, 0, 0, 3568, 0, 4021, 0, 0, 0, 0, 0, 1, 0, 102, 1943, 0, 0, 0, 0, 4095, 0, 24, 426, 0, 0, 0, 4062, 0, 387, 0, 0, 0, 0, 0, 171, 140, 12, 0, 4083, 206, 0, 0, 5, 0, 22, 0, 1434, 69, 0, 0, 0, 0, 0, 3173, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4824, 7589, 2046, 4545, 4449, 4612, 1530, 3173] avg=4096.0000 max=7589 max_rank=1 min=1530 min_rank=6 imbalance_ratio=1.852783 max_load_reduction=25179 max_load_reduction_pct=0.768402 moved_to_echo=13764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[14080, 384, 16000, 256, 2048, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 128, 128, 17152, 15104, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[20480, 128, 128, 640, 11264, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 5376, 0, 27520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 7040, 128, 25856, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 28416, 640, 128, 3584, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[29696, 128, 3072, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 896, 384, 128, 29824, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 70, 0, 1, 3802, 1, 0, 4083, 3945, 0, 500, 1004, 0, 0, 0, 0, 0, 249, 0, 2369, 0, 1006, 0, 1412, 3, 1919, 4076, 0, 183, 20, 0, 4029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4079, 15, 0, 0, 4, 0, 0, 4089, 2612, 0, 0, 0, 3, 57, 0, 124, 5, 43, 0, 4045, 221, 1670, 2, 0, 0, 4090, 3815, 1, 3906, 0, 0, 3987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4092, 0, 1937, 54, 97, 0, 13, 4096, 3565, 30, 4, 2171, 0, 18, 4023, 0, 0, 1717, 0, 22, 825, 162, 0, 0, 5, 1498, 1298, 69, 2976, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3335, 0, 912, 4088, 2224, 0, 3950, 127, 0, 308, 0, 0, 0, 63, 0, 0, 2224, 2309, 0, 0, 0, 27, 4096, 0, 913, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4068, 0, 0, 17, 3833, 0, 1, 4088, 3474, 3, 0, 10, 0, 0, 8, 0, 0, 0, 0, 3809, 0, 423, 2, 0, 0, 3979, 4096, 24, 734, 150, 0, 4049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[3713, 261, 0, 1, 35, 2, 7, 4091, 3979, 90, 232, 3, 753, 0, 3501, 1480, 9, 0, 1158, 3985, 2885, 20, 0, 0, 0, 2212, 0, 0, 295, 0, 0, 4056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4085, 51, 0, 0, 266, 0, 2586, 2288, 4091, 0, 3940, 0, 12, 0, 0, 0, 0, 92, 0, 2263, 923, 0, 0, 0, 0, 4096, 24, 0, 4049, 5, 1, 3996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 4096, 84, 0, 4093, 27, 0, 805, 3251, 4, 0, 0, 50, 0, 0, 0, 2998, 2383, 856, 2, 0, 0, 1942, 3997, 0, 7, 250, 25, 3828] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3835, 15, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1812, 0, 0, 0, 0, 1819, 0, 0, 0, 0, 0, 0, 0, 3, 57, 0, 124, 2277, 645, 0, 0, 5, 43, 0, 4045, 793, 244, 130, 0, 221, 1670, 2, 0, 1384, 952, 0, 0, 0, 4090, 2301, 1, 0, 0, 0, 0, 3906, 0, 0, 2390, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3850, 1816, 1819, 3106, 5260, 4229, 6392, 6296] avg=4096.0000 max=6392 max_rank=6 min=1816 min_rank=1 imbalance_ratio=1.560547 max_load_reduction=26376 max_load_reduction_pct=0.804932 moved_to_echo=22177 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3851, 70, 0, 1, 0, 0, 0, 0, 3802, 1, 0, 1809, 0, 0, 0, 0, 2747, 0, 500, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 2274, 652, 0, 0, 0, 249, 0, 2369, 1198, 245, 143, 0, 0, 1006, 0, 1412, 1481, 962, 0, 0, 3, 1919, 2452, 0, 0, 0, 0, 0, 183, 20, 0, 2415, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3922, 5612, 4251, 2926, 4204, 4861, 4374, 2618] avg=4096.0000 max=5612 max_rank=1 min=2618 min_rank=7 imbalance_ratio=1.370117 max_load_reduction=27156 max_load_reduction_pct=0.828735 moved_to_echo=16057 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3847, 0, 1937, 54, 0, 0, 0, 0, 97, 0, 13, 1815, 0, 0, 0, 0, 2482, 30, 4, 2171, 0, 0, 0, 0, 0, 18, 4023, 0, 2281, 663, 0, 0, 0, 1717, 0, 22, 1083, 245, 44, 0, 825, 162, 0, 0, 470, 978, 0, 0, 5, 1498, 784, 69, 0, 0, 0, 0, 2976, 0, 0, 2455, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5838, 1925, 4687, 6985, 3111, 2435, 2356, 5431] avg=4096.0000 max=6985 max_rank=3 min=1925 min_rank=1 imbalance_ratio=1.705322 max_load_reduction=25783 max_load_reduction_pct=0.786835 moved_to_echo=13333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3851, 0, 0, 0, 0, 0, 0, 0, 3335, 0, 912, 1811, 0, 0, 0, 0, 1549, 0, 3950, 127, 0, 0, 0, 0, 0, 308, 0, 0, 2277, 663, 0, 0, 0, 63, 0, 0, 675, 245, 140, 0, 2224, 2309, 0, 0, 1486, 978, 0, 0, 0, 27, 2470, 0, 0, 0, 0, 0, 913, 0, 0, 2455, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3851, 6058, 5626, 3248, 1123, 6997, 2497, 3368] avg=4096.0000 max=6997 max_rank=5 min=1123 min_rank=4 imbalance_ratio=1.708252 max_load_reduction=25771 max_load_reduction_pct=0.786469 moved_to_echo=13985 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3824, 0, 0, 17, 0, 0, 0, 0, 3833, 0, 1, 1811, 0, 0, 0, 0, 2419, 3, 0, 10, 0, 0, 0, 0, 0, 0, 8, 0, 2277, 655, 0, 0, 0, 0, 0, 3809, 1055, 244, 140, 0, 0, 423, 2, 0, 1486, 967, 0, 0, 0, 3979, 2470, 24, 0, 0, 0, 0, 734, 150, 0, 2427, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3841, 5645, 2432, 2940, 5248, 2878, 6473, 3311] avg=4096.0000 max=6473 max_rank=6 min=2432 min_rank=2 imbalance_ratio=1.580322 max_load_reduction=26295 max_load_reduction_pct=0.802460 moved_to_echo=17910 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3487, 261, 0, 1, 0, 0, 0, 0, 35, 2, 7, 1809, 0, 0, 0, 0, 2765, 90, 232, 3, 0, 0, 0, 0, 753, 0, 3501, 1480, 2282, 661, 0, 0, 9, 0, 1158, 3985, 1214, 226, 0, 0, 2885, 20, 0, 0, 0, 973, 0, 0, 0, 2212, 0, 0, 0, 0, 0, 0, 295, 0, 0, 2422, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3749, 1853, 3090, 8677, 6592, 3878, 2212, 2717] avg=4096.0000 max=8677 max_rank=3 min=1853 min_rank=1 imbalance_ratio=2.118408 max_load_reduction=24091 max_load_reduction_pct=0.735199 moved_to_echo=15399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3840, 51, 0, 0, 0, 0, 0, 0, 266, 0, 2586, 1014, 0, 0, 0, 0, 2848, 0, 3940, 0, 0, 0, 0, 0, 12, 0, 0, 0, 1274, 647, 0, 0, 0, 92, 0, 2263, 1243, 245, 0, 0, 923, 0, 0, 0, 8, 954, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 4049, 5, 1, 2395, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3891, 3866, 6788, 1933, 3843, 1885, 4112, 6450] avg=4096.0000 max=6788 max_rank=2 min=1885 min_rank=5 imbalance_ratio=1.657227 max_load_reduction=25980 max_load_reduction_pct=0.792847 moved_to_echo=16290 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3826, 0, 0, 0, 0, 0, 0, 0, 4096, 84, 0, 1813, 0, 0, 0, 0, 19, 0, 805, 3251, 0, 0, 0, 0, 4, 0, 0, 50, 2280, 619, 0, 0, 0, 0, 0, 2998, 8, 244, 137, 0, 2383, 856, 2, 0, 1450, 914, 0, 0, 0, 1942, 2410, 0, 0, 0, 0, 0, 7, 250, 25, 2295, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3826, 5993, 4075, 2953, 3387, 5605, 4352, 2577] avg=4096.0000 max=5993 max_rank=1 min=2577 min_rank=7 imbalance_ratio=1.463135 max_load_reduction=26775 max_load_reduction_pct=0.817108 moved_to_echo=15921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9472, 6528, 128, 1536, 7808, 7680, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[896, 384, 7552, 1664, 17280, 5248, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 19840, 12928, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13184, 512, 128, 19328, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[15488, 128, 3584, 13696, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16768, 128, 9472, 6656, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 2176, 1280, 19584, 7296, 2048, 768, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[30464, 512, 2048, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 57, 0, 0, 2, 1320, 0, 0, 2381, 0, 4066, 0, 7, 10, 247, 4091, 4050, 4096, 0, 3993, 35, 0, 3924, 391, 0, 0, 0, 3913, 8, 27, 150] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[21, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 4061, 8, 0, 0, 300, 4074, 4068, 4092, 0, 4096, 0, 0, 4058, 3788, 0, 3, 69, 4096, 0, 0, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 1, 121, 14, 1, 60, 4085, 391, 0, 17, 3800, 4096, 3893, 4095, 0, 4096, 0, 279, 3471, 4096, 0, 0, 15, 236, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 75, 19, 0, 7, 0, 0, 69, 12, 0, 0, 3967, 4094, 1, 0, 0, 4079, 4095, 4082, 2, 4033, 0, 36, 4091, 6, 0, 0, 0, 4094, 0, 3, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[93, 45, 22, 0, 24, 0, 1, 282, 0, 2, 55, 3992, 52, 0, 37, 5, 4072, 4053, 4068, 0, 4096, 0, 59, 4018, 0, 1, 0, 556, 4032, 3197, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 0, 1, 0, 0, 4079, 0, 4088, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 4096, 19, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 8, 0, 152, 0, 4091, 0, 141, 7, 0, 3935, 4083, 3957, 10, 4096, 0, 0, 4078, 12, 0, 0, 1, 4096, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[13, 0, 1, 0, 0, 0, 12, 0, 0, 3953, 6, 4084, 119, 0, 0, 0, 4095, 4096, 4091, 0, 4025, 0, 0, 4083, 26, 0, 0, 55, 3975, 0, 0, 134] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 0, 4072, 0, 0, 0, 2, 0, 0, 4, 3849, 20, 0, 0, 0, 0, 0, 2762, 0, 0, 0, 0, 8, 0, 0, 300, 1299, 1225, 491, 0, 42, 4068, 0, 0, 0, 0, 0, 0, 69, 0, 0, 4058, 0, 0, 0, 0, 3788, 0, 3, 69, 2802, 183, 0, 0, 3605, 0, 0, 28, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 3875, 2762, 3323, 4110, 4127, 6845, 3633] avg=4096.0000 max=6845 max_rank=6 min=2762 min_rank=2 imbalance_ratio=1.671143 max_load_reduction=25923 max_load_reduction_pct=0.791107 moved_to_echo=18715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 57, 0, 4075, 0, 0, 0, 0, 2, 1320, 0, 3865, 21, 0, 0, 0, 2381, 0, 2765, 0, 0, 0, 0, 0, 7, 10, 247, 1301, 1194, 469, 0, 42, 4050, 0, 0, 0, 0, 0, 0, 67, 35, 0, 3924, 0, 0, 0, 0, 391, 0, 0, 0, 2732, 184, 0, 0, 3444, 8, 27, 150, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 5208, 5146, 3228, 4092, 4026, 3307, 3629] avg=4096.0000 max=5208 max_rank=1 min=3228 min_rank=3 imbalance_ratio=1.271484 max_load_reduction=27560 max_load_reduction_pct=0.841064 moved_to_echo=15054 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 75, 19, 0, 4062, 0, 0, 0, 7, 0, 0, 69, 3854, 20, 0, 0, 12, 0, 0, 2698, 0, 0, 0, 0, 4094, 1, 0, 0, 1269, 1206, 490, 0, 41, 4095, 0, 2, 0, 0, 0, 0, 68, 0, 36, 4091, 0, 0, 0, 0, 6, 0, 0, 0, 2759, 184, 0, 0, 3604, 0, 3, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 3950, 2710, 7060, 4138, 4195, 2949, 3610] avg=4096.0000 max=7060 max_rank=3 min=2710 min_rank=2 imbalance_ratio=1.723633 max_load_reduction=25708 max_load_reduction_pct=0.784546 moved_to_echo=14892 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 1, 0, 3870, 20, 0, 0, 0, 4079, 0, 2780, 0, 0, 0, 0, 0, 0, 0, 0, 1308, 1225, 491, 0, 42, 4096, 0, 0, 0, 0, 0, 0, 69, 0, 0, 4096, 0, 0, 0, 0, 19, 0, 0, 0, 2802, 184, 0, 0, 3605, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 3891, 6859, 3024, 4138, 4165, 3005, 3605] avg=4096.0000 max=6859 max_rank=2 min=3005 min_rank=6 imbalance_ratio=1.674561 max_load_reduction=25909 max_load_reduction_pct=0.790680 moved_to_echo=14913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3938, 0, 0, 0, 0, 6, 0, 8, 3718, 19, 0, 0, 0, 152, 0, 2782, 0, 0, 0, 0, 0, 141, 7, 0, 1309, 1225, 491, 0, 40, 4083, 0, 10, 0, 0, 0, 0, 69, 0, 0, 4078, 0, 0, 0, 0, 12, 0, 0, 1, 2802, 177, 0, 0, 3605, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3938, 3751, 2934, 3173, 4133, 4147, 2992, 7700] avg=4096.0000 max=7700 max_rank=7 min=2934 min_rank=2 imbalance_ratio=1.879883 max_load_reduction=25068 max_load_reduction_pct=0.765015 moved_to_echo=18972 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4075, 0, 0, 0, 0, 1, 1, 121, 3875, 20, 0, 0, 14, 1, 60, 2774, 0, 0, 0, 0, 391, 0, 17, 3800, 1311, 1227, 30, 0, 31, 3893, 0, 0, 0, 0, 0, 0, 61, 0, 279, 3471, 0, 0, 0, 0, 4096, 0, 0, 15, 2808, 190, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4018, 2849, 6776, 3924, 3811, 7109, 206] avg=4096.0000 max=7109 max_rank=6 min=206 min_rank=7 imbalance_ratio=1.735596 max_load_reduction=25659 max_load_reduction_pct=0.783051 moved_to_echo=15050 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[93, 45, 22, 0, 4049, 0, 0, 0, 24, 0, 1, 282, 3847, 19, 0, 0, 0, 2, 55, 2715, 0, 0, 0, 0, 52, 0, 37, 5, 1277, 1225, 483, 0, 42, 4053, 0, 0, 0, 0, 0, 0, 69, 0, 59, 4018, 0, 0, 0, 0, 0, 1, 0, 556, 2802, 183, 0, 0, 3549, 3197, 0, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4209, 4173, 2772, 3079, 4095, 4146, 3542, 6752] avg=4096.0000 max=6752 max_rank=7 min=2772 min_rank=2 imbalance_ratio=1.648438 max_load_reduction=26016 max_load_reduction_pct=0.793945 moved_to_echo=18535 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 1, 0, 4070, 0, 0, 0, 0, 0, 12, 0, 3869, 21, 0, 0, 0, 3953, 6, 2777, 0, 0, 0, 0, 119, 0, 0, 0, 1307, 1203, 476, 0, 42, 4096, 0, 0, 0, 0, 0, 0, 68, 0, 0, 4083, 0, 0, 0, 0, 26, 0, 0, 55, 2754, 184, 0, 0, 3499, 0, 0, 134, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 3902, 6736, 3105, 4138, 4151, 3019, 3633] avg=4096.0000 max=6736 max_rank=2 min=3019 min_rank=6 imbalance_ratio=1.644531 max_load_reduction=26032 max_load_reduction_pct=0.794434 moved_to_echo=14941 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4736, 256, 128, 4352, 10496, 9856, 3456, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[25216, 7424, 128, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8448, 128, 128, 768, 22272, 1536, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 10624, 128, 22144, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 128, 0, 32512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 1408, 512, 30848, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 32512, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[640, 128, 384, 31872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4085, 4041, 4086, 15, 0, 4032, 18, 0, 0, 0, 0, 0, 24, 0, 4, 0, 0, 6, 1, 720, 4094, 0, 0, 0, 4090, 5, 0, 4094, 3390, 63, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[27, 4061, 226, 4056, 0, 8, 4084, 55, 150, 0, 8, 0, 0, 0, 0, 14, 2, 4016, 0, 17, 13, 4082, 0, 0, 0, 4096, 1, 0, 3915, 3937, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 3955, 1, 4094, 0, 3, 3928, 830, 0, 1, 5, 154, 0, 0, 513, 0, 0, 4068, 100, 0, 1, 3274, 94, 0, 0, 3577, 0, 3, 4093, 3989, 85, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3659, 4096, 9, 0, 4094, 2693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 2319, 4069, 0, 0, 0, 4096, 207, 0, 0, 3364, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1, 4084, 1688, 4094, 3994, 0, 4067, 1255, 6, 0, 0, 0, 25, 0, 0, 7, 0, 0, 0, 0, 2, 4081, 0, 0, 0, 4090, 0, 17, 1572, 3763, 18, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4071, 3958, 4087, 3553, 0, 4002, 2, 0, 30, 0, 1, 0, 0, 126, 0, 0, 0, 0, 0, 1589, 4096, 0, 0, 90, 4094, 0, 0, 8, 2998, 0, 63] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[2, 4072, 4094, 0, 3686, 67, 4096, 3884, 0, 2381, 0, 0, 0, 0, 12, 0, 0, 359, 0, 1121, 333, 4095, 0, 0, 0, 4062, 197, 5, 1, 301, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4076, 4096, 4096, 18, 0, 4096, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3979, 4035, 0, 143, 6, 4096, 0, 0, 0, 3812, 40, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4041, 1562, 0, 0, 0, 0, 15, 0, 1578, 18, 0, 0, 0, 0, 0, 0, 0, 0, 2454, 1374, 58, 0, 0, 24, 0, 4, 4027, 0, 0, 0, 0, 0, 6, 1, 1061, 1150, 890, 0, 720, 3033, 0, 0, 0, 0, 0, 0, 0, 4090, 5, 0, 4, 0, 0, 0, 4094, 2496, 63, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5603, 1611, 3886, 4055, 3108, 3753, 4099, 6653] avg=4096.0000 max=6653 max_rank=7 min=1611 min_rank=1 imbalance_ratio=1.624268 max_load_reduction=26115 max_load_reduction_pct=0.796967 moved_to_echo=17613 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[27, 0, 226, 1557, 0, 0, 0, 0, 0, 8, 1599, 55, 0, 0, 0, 0, 150, 0, 8, 0, 2485, 1361, 57, 0, 0, 0, 0, 14, 4004, 0, 0, 0, 2, 4016, 0, 17, 1058, 1138, 1033, 0, 13, 3024, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 5, 0, 0, 0, 3915, 2899, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1810, 1662, 4061, 4018, 7264, 3037, 4102, 6814] avg=4096.0000 max=7264 max_rank=4 min=1662 min_rank=1 imbalance_ratio=1.773438 max_load_reduction=25504 max_load_reduction_pct=0.778320 moved_to_echo=21217 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1688, 1572, 0, 0, 0, 0, 3994, 0, 1592, 1255, 0, 0, 0, 0, 6, 0, 0, 0, 2475, 1374, 57, 0, 25, 0, 0, 7, 4027, 0, 0, 0, 0, 0, 0, 0, 1058, 1148, 988, 0, 2, 3023, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 17, 5, 0, 0, 0, 1572, 2770, 18, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3261, 6841, 3912, 4059, 3194, 3025, 4112, 4364] avg=4096.0000 max=6841 max_rank=1 min=3025 min_rank=5 imbalance_ratio=1.670166 max_load_reduction=25927 max_load_reduction_pct=0.791229 moved_to_echo=14695 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3659, 1572, 0, 0, 0, 0, 9, 0, 1603, 2693, 0, 0, 0, 0, 0, 0, 0, 0, 2491, 1375, 57, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 61, 1054, 1149, 883, 0, 2319, 3015, 0, 0, 0, 0, 0, 0, 0, 4096, 207, 0, 4, 0, 0, 0, 0, 2477, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5231, 4305, 3923, 4039, 3147, 5334, 4307, 2482] avg=4096.0000 max=5334 max_rank=5 min=2482 min_rank=7 imbalance_ratio=1.302246 max_load_reduction=27434 max_load_reduction_pct=0.837219 moved_to_echo=15270 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 1572, 0, 0, 0, 0, 0, 3, 1538, 830, 0, 0, 0, 0, 0, 1, 5, 154, 2390, 1374, 55, 0, 0, 0, 513, 0, 3900, 0, 0, 0, 0, 4068, 100, 0, 848, 1148, 1047, 0, 1, 2426, 94, 0, 0, 0, 0, 0, 0, 3577, 0, 3, 5, 0, 0, 0, 4093, 2937, 85, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1573, 2371, 3979, 4413, 7211, 2521, 3585, 7115] avg=4096.0000 max=7211 max_rank=4 min=1573 min_rank=0 imbalance_ratio=1.760498 max_load_reduction=25557 max_load_reduction_pct=0.779938 moved_to_echo=20432 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4094, 0, 0, 0, 0, 0, 3686, 67, 1598, 3884, 0, 0, 0, 0, 0, 2381, 0, 0, 2498, 0, 58, 0, 0, 0, 12, 0, 4014, 0, 0, 0, 0, 359, 0, 1121, 1065, 0, 82, 0, 333, 3030, 0, 0, 0, 0, 0, 0, 0, 4062, 197, 5, 4, 0, 0, 0, 1, 215, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 9235, 4937, 4026, 2627, 3363, 4268, 216] avg=4096.0000 max=9235 max_rank=1 min=216 min_rank=7 imbalance_ratio=2.254639 max_load_reduction=23533 max_load_reduction_pct=0.718170 moved_to_echo=10474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3958, 1568, 0, 0, 0, 0, 3553, 0, 1567, 2, 0, 0, 0, 0, 0, 30, 0, 1, 2435, 1372, 58, 0, 0, 0, 126, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 1061, 1147, 787, 0, 1589, 3035, 0, 0, 0, 0, 0, 0, 90, 4094, 0, 0, 4, 0, 0, 0, 8, 2207, 0, 63, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5526, 5122, 3896, 4139, 2995, 4624, 4188, 2278] avg=4096.0000 max=5526 max_rank=0 min=2278 min_rank=7 imbalance_ratio=1.349121 max_load_reduction=27242 max_load_reduction_pct=0.831360 moved_to_echo=14085 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 1572, 0, 0, 0, 0, 18, 0, 1603, 0, 0, 0, 0, 0, 0, 249, 0, 0, 2493, 1375, 57, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 26, 1046, 1149, 1001, 0, 3979, 2989, 0, 143, 0, 0, 0, 0, 6, 4096, 0, 0, 5, 0, 0, 0, 0, 2806, 40, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5668, 1621, 4174, 4019, 3222, 7111, 4107, 2846] avg=4096.0000 max=7111 max_rank=5 min=1621 min_rank=1 imbalance_ratio=1.736084 max_load_reduction=25657 max_load_reduction_pct=0.782990 moved_to_echo=17286 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 2688, 128, 256, 19840, 9728, 512, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 8448, 128, 1280, 8320, 8064, 6784, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11392, 128, 12800, 8832, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[13696, 18816, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 21888, 11008, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8960, 23680, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 768, 128, 32128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32256, 512, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 3481, 4084, 0, 3750, 1, 0, 2474, 4039, 26, 0, 0, 770, 0, 115, 0, 4095, 0, 0, 54, 30, 111, 0, 6, 3476, 0, 333, 0, 0, 1, 3972, 1950] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4095, 26, 3471, 0, 1, 4066, 4092, 0, 4, 1, 0, 0, 0, 157, 4096, 0, 0, 28, 4094, 0, 0, 2, 4070, 16, 0, 0, 0, 453, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[65, 1401, 4076, 0, 1207, 113, 113, 4096, 4061, 0, 0, 0, 0, 0, 1, 0, 3952, 0, 0, 0, 4045, 24, 8, 26, 3192, 0, 0, 21, 0, 127, 4076, 2164] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3993, 141, 1, 3952, 4091, 0, 0, 0, 0, 0, 29, 1, 4064, 8, 5, 0, 10, 69, 0, 0, 4085, 0, 0, 3, 0, 4028, 4096, 96] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 141, 4096, 0, 4096, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3925, 0, 0, 0, 0, 28, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[14, 1248, 3986, 0, 3510, 184, 0, 4092, 173, 0, 1024, 0, 0, 11, 0, 80, 4084, 2, 0, 51, 114, 0, 30, 0, 3683, 0, 463, 0, 0, 2012, 4051, 3956] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 3819, 10, 4079, 0, 0, 4083, 4092, 0, 0, 2, 0, 0, 0, 0, 4082, 0, 0, 12, 659, 0, 0, 22, 4015, 0, 81, 0, 223, 15, 4095, 3479] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 297, 4014, 0, 680, 1479, 2, 4092, 4087, 0, 0, 1, 29, 0, 32, 11, 4096, 0, 0, 0, 1618, 0, 0, 85, 4009, 0, 20, 0, 62, 0, 4058, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3312, 26, 0, 0, 0, 0, 3471, 0, 1, 782, 0, 0, 0, 0, 4092, 0, 4, 1, 378, 0, 0, 0, 0, 0, 0, 157, 3343, 653, 0, 0, 4096, 0, 0, 28, 5, 0, 0, 0, 4094, 0, 0, 2, 2631, 226, 0, 0, 4070, 16, 0, 0, 174, 0, 0, 0, 0, 453, 753, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3338, 4254, 4475, 4153, 4129, 6953, 4260, 1206] avg=4096.0000 max=6953 max_rank=5 min=1206 min_rank=7 imbalance_ratio=1.697510 max_load_reduction=25815 max_load_reduction_pct=0.787811 moved_to_echo=16548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3313, 0, 0, 0, 0, 0, 3993, 141, 1, 760, 0, 0, 0, 0, 4091, 0, 0, 0, 378, 0, 0, 0, 0, 0, 29, 1, 3343, 635, 0, 0, 4064, 8, 5, 0, 5, 0, 0, 0, 10, 69, 0, 0, 2557, 226, 0, 0, 4085, 0, 0, 3, 174, 0, 0, 0, 0, 4028, 753, 96, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3313, 4895, 4469, 4008, 4082, 2862, 4262, 4877] avg=4096.0000 max=4895 max_rank=1 min=2862 min_rank=5 imbalance_ratio=1.195068 max_load_reduction=27873 max_load_reduction_pct=0.850616 moved_to_echo=16083 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3481, 3303, 0, 0, 0, 0, 0, 3750, 1, 0, 476, 0, 0, 0, 0, 4039, 26, 0, 0, 377, 0, 0, 0, 770, 0, 115, 0, 3241, 397, 0, 0, 4095, 0, 0, 54, 5, 0, 0, 0, 30, 111, 0, 6, 1601, 225, 0, 0, 3476, 0, 333, 0, 174, 0, 0, 0, 0, 1, 731, 1950, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6784, 4227, 4442, 4523, 4154, 1973, 3983, 2682] avg=4096.0000 max=6784 max_rank=0 min=1973 min_rank=5 imbalance_ratio=1.656250 max_load_reduction=25984 max_load_reduction_pct=0.792969 moved_to_echo=12792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[65, 1401, 3297, 0, 0, 0, 0, 0, 1207, 113, 113, 788, 0, 0, 0, 0, 4061, 0, 0, 0, 376, 0, 0, 0, 0, 0, 1, 0, 3326, 658, 0, 0, 3952, 0, 0, 0, 5, 0, 0, 0, 4045, 24, 8, 26, 2650, 225, 0, 0, 3192, 0, 0, 21, 173, 0, 0, 0, 0, 127, 750, 2164, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4763, 2221, 4437, 3985, 3957, 6978, 3386, 3041] avg=4096.0000 max=6978 max_rank=5 min=2221 min_rank=1 imbalance_ratio=1.703613 max_load_reduction=25790 max_load_reduction_pct=0.787048 moved_to_echo=17362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 141, 3313, 0, 0, 0, 0, 0, 4096, 2, 0, 788, 0, 0, 0, 0, 4096, 0, 0, 0, 378, 0, 0, 0, 0, 0, 0, 0, 3343, 658, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2650, 226, 0, 0, 3925, 0, 0, 0, 174, 0, 0, 0, 0, 28, 753, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3454, 4886, 4474, 4001, 4101, 2876, 4099, 4877] avg=4096.0000 max=4886 max_rank=1 min=2876 min_rank=5 imbalance_ratio=1.192871 max_load_reduction=27882 max_load_reduction_pct=0.850891 moved_to_echo=15953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[14, 1248, 3211, 0, 0, 0, 0, 0, 3510, 184, 0, 779, 0, 0, 0, 0, 173, 0, 1024, 0, 370, 0, 0, 0, 0, 11, 0, 80, 3309, 660, 0, 0, 4084, 2, 0, 51, 8, 0, 0, 0, 114, 0, 30, 0, 2653, 223, 0, 0, 3683, 0, 463, 0, 174, 0, 0, 0, 0, 2012, 742, 3956, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4473, 4473, 1567, 4060, 4145, 3020, 4320, 6710] avg=4096.0000 max=6710 max_rank=7 min=1567 min_rank=2 imbalance_ratio=1.638184 max_load_reduction=26058 max_load_reduction_pct=0.795227 moved_to_echo=18195 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3089, 10, 0, 0, 0, 0, 4079, 0, 0, 785, 0, 0, 0, 0, 4092, 0, 0, 2, 352, 0, 0, 0, 0, 0, 0, 0, 3342, 656, 0, 0, 4082, 0, 0, 12, 5, 0, 0, 0, 659, 0, 0, 22, 2642, 211, 0, 0, 4015, 0, 81, 0, 162, 0, 0, 0, 223, 15, 753, 3479, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3099, 4864, 4446, 3998, 4099, 3534, 4258, 4470] avg=4096.0000 max=4864 max_rank=1 min=3099 min_rank=0 imbalance_ratio=1.187500 max_load_reduction=27904 max_load_reduction_pct=0.851562 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 297, 3247, 0, 0, 0, 0, 0, 680, 1479, 2, 787, 0, 0, 0, 0, 4087, 0, 0, 1, 370, 0, 0, 0, 29, 0, 32, 11, 3311, 657, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 1618, 0, 0, 85, 2648, 221, 0, 0, 4009, 0, 20, 0, 171, 0, 0, 0, 62, 0, 747, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3544, 2948, 4458, 4040, 4101, 4572, 4200, 4905] avg=4096.0000 max=4905 max_rank=7 min=2948 min_rank=1 imbalance_ratio=1.197510 max_load_reduction=27863 max_load_reduction_pct=0.850311 moved_to_echo=17778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10624, 256, 128, 256, 20096, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 6656, 26112, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 6784, 6016, 19840, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 128, 256, 256, 26624, 4992, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[24832, 1920, 128, 6016, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[28800, 128, 1152, 128, 3072, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[30464, 128, 1024, 128, 1408, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 256, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True + [2026-08-30 04:56:45] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 662.5 | throughput per GPU (TFLOP/s/GPU): 503.2 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.032271E+01 | loss scale: 1.0 | grad norm: 204.376 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[53, 3406, 1604, 30, 2943, 14, 1277, 57, 1735, 39, 3478, 966, 401, 521, 193, 274, 594, 944, 1261, 3586, 3251, 844, 525, 255, 2, 1669, 243, 586, 547, 130, 1125, 215] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[43, 3347, 1827, 23, 2527, 26, 952, 30, 1176, 95, 3360, 730, 305, 700, 44, 283, 418, 1452, 1736, 3841, 3158, 532, 765, 199, 2, 1861, 173, 1031, 566, 252, 1210, 104] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[11, 3642, 1930, 56, 3018, 4, 875, 45, 1242, 69, 3126, 591, 376, 468, 93, 203, 392, 1353, 1777, 3608, 3430, 746, 555, 172, 1, 1587, 208, 839, 730, 241, 1246, 134] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[28, 3045, 1902, 93, 2850, 31, 868, 60, 1496, 74, 3317, 843, 405, 490, 114, 287, 378, 1493, 2301, 3528, 3011, 504, 764, 216, 2, 1469, 183, 837, 628, 111, 1321, 119] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[11, 3113, 2114, 18, 3219, 11, 989, 33, 1162, 95, 2888, 486, 617, 368, 18, 401, 321, 2098, 1991, 3941, 3037, 154, 910, 142, 0, 1298, 171, 732, 1031, 51, 1189, 159] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[42, 3542, 1570, 22, 2695, 33, 841, 110, 1660, 60, 3166, 687, 519, 686, 25, 358, 625, 1101, 2170, 3660, 2947, 190, 587, 216, 0, 1935, 240, 731, 629, 128, 1443, 150] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[14, 3456, 2250, 68, 2190, 15, 830, 59, 1586, 122, 3350, 560, 455, 519, 97, 255, 524, 1295, 1988, 3620, 3135, 299, 546, 147, 2, 1434, 189, 941, 790, 143, 1690, 199] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[24, 2977, 1378, 39, 2836, 12, 960, 81, 1638, 59, 3343, 979, 458, 587, 70, 294, 472, 1487, 2207, 3857, 3266, 297, 603, 150, 0, 1603, 207, 629, 566, 98, 1431, 160] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[43, 2223, 1827, 23, 0, 0, 0, 0, 2527, 26, 952, 30, 284, 0, 0, 0, 1176, 95, 1888, 730, 0, 0, 0, 0, 305, 700, 44, 283, 2835, 0, 0, 0, 418, 1452, 1736, 311, 0, 0, 0, 0, 2811, 532, 765, 199, 0, 0, 0, 0, 2, 1861, 173, 1031, 1124, 328, 63, 0, 566, 252, 1210, 104, 1144, 695, 0, 0] rerouted_tokens_per_ep_rank=[4116, 3819, 3889, 4167, 3917, 4307, 4582, 3971] avg=4096.0000 max=4582 max_rank=6 min=3819 min_rank=1 imbalance_ratio=1.118652 max_load_reduction=28186 max_load_reduction_pct=0.860168 moved_to_echo=16777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 2419, 1930, 56, 0, 0, 0, 0, 3018, 4, 875, 45, 309, 0, 0, 0, 1242, 69, 1757, 591, 0, 0, 0, 0, 376, 468, 93, 203, 2663, 0, 0, 0, 392, 1353, 1777, 292, 0, 0, 0, 0, 3052, 746, 555, 172, 0, 0, 0, 0, 1, 1587, 208, 839, 1223, 305, 69, 0, 730, 241, 1246, 134, 1064, 653, 0, 0] rerouted_tokens_per_ep_rank=[4416, 4251, 3659, 3803, 3814, 4525, 4232, 4068] avg=4096.0000 max=4525 max_rank=5 min=3659 min_rank=2 imbalance_ratio=1.104736 max_load_reduction=28243 max_load_reduction_pct=0.861908 moved_to_echo=16639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 2068, 2114, 18, 0, 0, 0, 0, 3219, 11, 989, 33, 274, 0, 0, 0, 1162, 95, 1623, 486, 0, 0, 0, 0, 617, 368, 18, 401, 2909, 0, 0, 0, 321, 2098, 1991, 319, 0, 0, 0, 0, 2702, 154, 910, 142, 0, 0, 0, 0, 0, 1298, 171, 732, 1045, 282, 61, 0, 1031, 51, 1189, 159, 983, 713, 0, 0] rerouted_tokens_per_ep_rank=[4211, 4526, 3366, 4313, 4729, 3908, 3589, 4126] avg=4096.0000 max=4729 max_rank=4 min=3366 min_rank=2 imbalance_ratio=1.154541 max_load_reduction=28039 max_load_reduction_pct=0.855682 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[53, 2263, 1604, 30, 0, 0, 0, 0, 2943, 14, 1277, 57, 293, 0, 0, 0, 1735, 39, 1954, 966, 0, 0, 0, 0, 401, 521, 193, 274, 2647, 0, 0, 0, 594, 944, 1261, 290, 0, 0, 0, 0, 2893, 844, 525, 255, 0, 0, 0, 0, 2, 1669, 243, 586, 1143, 340, 65, 0, 547, 130, 1125, 215, 1184, 649, 0, 0] rerouted_tokens_per_ep_rank=[3950, 4584, 4694, 4036, 3089, 4517, 4048, 3850] avg=4096.0000 max=4694 max_rank=2 min=3089 min_rank=4 imbalance_ratio=1.145996 max_load_reduction=28074 max_load_reduction_pct=0.856750 moved_to_echo=15504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[28, 2023, 1902, 93, 0, 0, 0, 0, 2850, 31, 868, 60, 271, 0, 0, 0, 1496, 74, 1864, 843, 0, 0, 0, 0, 405, 490, 114, 287, 2604, 0, 0, 0, 378, 1493, 2301, 286, 0, 0, 0, 0, 2680, 504, 764, 216, 0, 0, 0, 0, 2, 1469, 183, 837, 1022, 324, 60, 0, 628, 111, 1321, 119, 1129, 638, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4080, 4277, 3900, 4458, 4164, 3897, 3946] avg=4096.0000 max=4458 max_rank=4 min=3897 min_rank=6 imbalance_ratio=1.088379 max_load_reduction=28310 max_load_reduction_pct=0.863953 moved_to_echo=16465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[14, 2291, 2250, 68, 0, 0, 0, 0, 2190, 15, 830, 59, 287, 0, 0, 0, 1586, 122, 1874, 560, 0, 0, 0, 0, 455, 519, 97, 255, 2677, 0, 0, 0, 524, 1295, 1988, 284, 0, 0, 0, 0, 2782, 299, 546, 147, 0, 0, 0, 0, 2, 1434, 189, 941, 1165, 331, 66, 0, 790, 143, 1690, 199, 1145, 659, 0, 0] rerouted_tokens_per_ep_rank=[4623, 3381, 4142, 4003, 4091, 3774, 4128, 4626] avg=4096.0000 max=4626 max_rank=7 min=3381 min_rank=1 imbalance_ratio=1.129395 max_load_reduction=28142 max_load_reduction_pct=0.858826 moved_to_echo=16619 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[42, 2353, 1570, 22, 0, 0, 0, 0, 2695, 33, 841, 110, 265, 0, 0, 0, 1660, 60, 1779, 687, 0, 0, 0, 0, 519, 686, 25, 358, 2702, 0, 0, 0, 625, 1101, 2170, 296, 0, 0, 0, 0, 2623, 190, 587, 216, 0, 0, 0, 0, 0, 1935, 240, 731, 1189, 309, 59, 0, 629, 128, 1443, 150, 1078, 662, 0, 0] rerouted_tokens_per_ep_rank=[3987, 3944, 4186, 4290, 4192, 3616, 4463, 4090] avg=4096.0000 max=4463 max_rank=6 min=3616 min_rank=5 imbalance_ratio=1.089600 max_load_reduction=28305 max_load_reduction_pct=0.863800 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[24, 1978, 1378, 39, 0, 0, 0, 0, 2836, 12, 960, 81, 294, 0, 0, 0, 1638, 59, 1879, 979, 0, 0, 0, 0, 458, 587, 70, 294, 2847, 0, 0, 0, 472, 1487, 2207, 312, 0, 0, 0, 0, 2907, 297, 603, 150, 0, 0, 0, 0, 0, 1603, 207, 629, 999, 326, 65, 0, 566, 98, 1431, 160, 1138, 698, 0, 0] rerouted_tokens_per_ep_rank=[3419, 4183, 4555, 4256, 4478, 3957, 3829, 4091] avg=4096.0000 max=4555 max_rank=2 min=3419 min_rank=0 imbalance_ratio=1.112061 max_load_reduction=28213 max_load_reduction_pct=0.860992 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 17664, 14592, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 12928, 1664, 6400, 8960, 2560, 512, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3840, 11264, 15488, 2432, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[22528, 3584, 5376, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5504, 1280, 10752, 1280, 8960, 5376, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11776, 640, 14720, 5888, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[22400, 256, 7680, 512, 2304, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3584, 4352, 768, 2432, 21888, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[47, 4090, 641, 0, 62, 0, 1, 36, 9, 4087, 8, 4, 0, 0, 3779, 9, 1, 0, 0, 0, 4088, 9, 16, 0, 4078, 3735, 4079, 0, 269, 0, 4, 3716] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 4074, 1321, 404, 0, 2, 98, 0, 0, 4039, 2, 0, 0, 85, 2647, 12, 1, 19, 1, 7, 4021, 8, 4, 47, 3797, 4084, 4068, 1, 129, 0, 161, 3736] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[48, 4096, 296, 1172, 0, 0, 37, 64, 0, 4040, 3, 0, 0, 0, 3814, 1, 0, 0, 0, 0, 2873, 20, 210, 4, 4044, 3779, 4091, 10, 135, 0, 230, 3801] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8, 4096, 2057, 3, 21, 0, 15, 34, 0, 4048, 0, 3, 0, 0, 4058, 4, 0, 26, 0, 0, 3976, 1, 1, 45, 4023, 2302, 4075, 6, 320, 0, 463, 3183] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 4096, 425, 405, 14, 1, 3, 38, 0, 3581, 1, 0, 1, 15, 4000, 1, 2, 0, 1, 1, 3917, 35, 0, 190, 3993, 3879, 4088, 1, 41, 0, 6, 4033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 4094, 1616, 291, 0, 0, 4, 0, 0, 3797, 0, 2, 0, 0, 4027, 1, 0, 0, 6, 0, 3811, 0, 20, 3, 4059, 3885, 4089, 0, 155, 0, 47, 2861] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[5, 4095, 771, 18, 0, 0, 67, 0, 0, 3738, 1, 0, 0, 0, 3939, 1, 10, 8, 4, 0, 4056, 1, 2, 0, 4027, 3867, 4094, 1, 7, 0, 57, 3999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 4082, 1699, 28, 5, 0, 1, 1, 6, 4085, 14, 1, 5, 2, 3259, 0, 12, 0, 3, 0, 4040, 0, 16, 114, 4083, 2830, 4093, 0, 351, 0, 26, 4012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[47, 2691, 641, 0, 0, 0, 0, 0, 62, 0, 1, 36, 3520, 567, 1, 0, 9, 4087, 8, 4, 162, 0, 0, 0, 0, 0, 3779, 9, 388, 0, 0, 0, 1, 0, 0, 0, 4078, 0, 0, 0, 4088, 9, 16, 0, 154, 0, 0, 0, 558, 3735, 0, 0, 0, 0, 0, 0, 269, 0, 4, 3716, 128, 0, 0, 0] rerouted_tokens_per_ep_rank=[3379, 4187, 4270, 4176, 4079, 4267, 4293, 4117] avg=4096.0000 max=4293 max_rank=6 min=3379 min_rank=0 imbalance_ratio=1.048096 max_load_reduction=28475 max_load_reduction_pct=0.868988 moved_to_echo=16756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2682, 1321, 404, 0, 0, 0, 0, 0, 2, 98, 0, 3278, 564, 1, 0, 0, 4039, 2, 0, 161, 0, 0, 0, 0, 85, 2647, 12, 386, 0, 0, 0, 1, 19, 1, 7, 4067, 0, 0, 0, 4021, 8, 4, 47, 154, 0, 0, 0, 519, 4084, 0, 1, 0, 0, 0, 0, 129, 0, 161, 3736, 127, 0, 0, 0] rerouted_tokens_per_ep_rank=[4407, 3943, 4202, 3130, 4095, 4234, 4604, 4153] avg=4096.0000 max=4604 max_rank=6 min=3130 min_rank=3 imbalance_ratio=1.124023 max_load_reduction=28164 max_load_reduction_pct=0.859497 moved_to_echo=17086 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[48, 2695, 296, 1172, 0, 0, 0, 0, 0, 0, 37, 64, 3491, 567, 2, 0, 0, 4040, 3, 0, 162, 0, 0, 0, 0, 0, 3814, 1, 389, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 2873, 20, 210, 4, 155, 0, 0, 0, 553, 3779, 0, 10, 0, 0, 0, 0, 135, 0, 230, 3801, 128, 0, 0, 0] rerouted_tokens_per_ep_rank=[4211, 4161, 4205, 4204, 4089, 3262, 4342, 4294] avg=4096.0000 max=4342 max_rank=6 min=3262 min_rank=5 imbalance_ratio=1.060059 max_load_reduction=28426 max_load_reduction_pct=0.867493 moved_to_echo=15987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8, 2695, 2057, 3, 0, 0, 0, 0, 21, 0, 15, 34, 3473, 567, 2, 0, 0, 4048, 0, 3, 162, 0, 0, 0, 0, 0, 4058, 4, 389, 0, 0, 0, 0, 26, 0, 0, 4073, 0, 0, 0, 3976, 1, 1, 45, 155, 0, 0, 0, 550, 2302, 0, 6, 0, 0, 0, 0, 320, 0, 463, 3183, 128, 0, 0, 0] rerouted_tokens_per_ep_rank=[4763, 4112, 4213, 4451, 4099, 4178, 2858, 4094] avg=4096.0000 max=4763 max_rank=0 min=2858 min_rank=6 imbalance_ratio=1.162842 max_load_reduction=28005 max_load_reduction_pct=0.854645 moved_to_echo=15229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2694, 1616, 291, 0, 0, 0, 0, 0, 0, 4, 0, 3504, 567, 1, 0, 0, 3797, 0, 2, 162, 0, 0, 0, 0, 0, 4027, 1, 388, 0, 0, 0, 0, 0, 6, 0, 4088, 0, 0, 0, 3811, 0, 20, 3, 155, 0, 0, 0, 555, 3885, 0, 0, 0, 0, 0, 0, 155, 0, 47, 2861, 128, 0, 0, 0] rerouted_tokens_per_ep_rank=[4601, 4076, 3961, 4416, 4094, 3989, 4440, 3191] avg=4096.0000 max=4601 max_rank=0 min=3191 min_rank=7 imbalance_ratio=1.123291 max_load_reduction=28167 max_load_reduction_pct=0.859589 moved_to_echo=15714 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2695, 425, 405, 0, 0, 0, 0, 14, 1, 3, 38, 3447, 567, 1, 0, 0, 3581, 1, 0, 162, 0, 0, 0, 1, 15, 4000, 1, 389, 0, 0, 0, 2, 0, 1, 1, 4087, 0, 0, 0, 3917, 35, 0, 190, 155, 0, 0, 0, 546, 3879, 0, 1, 0, 0, 0, 0, 41, 0, 6, 4033, 128, 0, 0, 0] rerouted_tokens_per_ep_rank=[3525, 4071, 3744, 4406, 4091, 4297, 4426, 4208] avg=4096.0000 max=4426 max_rank=6 min=3525 min_rank=0 imbalance_ratio=1.080566 max_load_reduction=28342 max_load_reduction_pct=0.864929 moved_to_echo=17022 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[5, 2695, 771, 18, 0, 0, 0, 0, 0, 0, 67, 0, 3476, 567, 1, 0, 0, 3738, 1, 0, 162, 0, 0, 0, 0, 0, 3939, 1, 388, 0, 0, 0, 10, 8, 4, 0, 4093, 0, 0, 0, 4056, 1, 2, 0, 155, 0, 0, 0, 551, 3867, 0, 1, 0, 0, 0, 0, 7, 0, 57, 3999, 128, 0, 0, 0] rerouted_tokens_per_ep_rank=[3489, 4111, 3901, 4328, 4115, 4214, 4419, 4191] avg=4096.0000 max=4419 max_rank=6 min=3489 min_rank=0 imbalance_ratio=1.078857 max_load_reduction=28349 max_load_reduction_pct=0.865143 moved_to_echo=16939 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2666, 1699, 28, 0, 0, 0, 0, 5, 0, 1, 1, 3527, 571, 2, 0, 6, 4085, 14, 1, 166, 0, 0, 0, 5, 2, 3259, 0, 391, 0, 0, 0, 12, 0, 3, 0, 4091, 0, 0, 0, 4040, 0, 16, 114, 157, 0, 0, 0, 556, 2830, 0, 0, 0, 0, 0, 0, 351, 0, 26, 4012, 131, 0, 0, 0] rerouted_tokens_per_ep_rank=[4393, 4107, 4272, 3657, 4106, 4327, 3386, 4520] avg=4096.0000 max=4520 max_rank=7 min=3386 min_rank=6 imbalance_ratio=1.103516 max_load_reduction=28248 max_load_reduction_pct=0.862061 moved_to_echo=16339 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 21632, 8832, 2432, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 256, 256, 27776, 4608, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 29568, 128, 3200, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1408, 0, 1024, 29440, 1152, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4480, 28416, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[30848, 128, 384, 512, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 31488, 128, 128, 1408, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[8, 0, 21, 4, 0, 2207, 1, 4077, 78, 3716, 0, 2210, 4085, 4057, 4092, 0, 4052, 1, 0, 0, 0, 2, 50, 36, 26, 3605, 239, 6, 158, 0, 37, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4, 4074, 0, 4095, 0, 3908, 6, 15, 4089, 4087, 4090, 66, 4084, 0, 0, 22, 0, 125, 0, 6, 2, 4065, 0, 1, 4, 0, 21, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 17, 0, 0, 4076, 1, 4094, 15, 4052, 0, 1968, 4094, 4093, 4065, 7, 4082, 0, 0, 119, 0, 45, 0, 0, 0, 1578, 27, 0, 0, 0, 433, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 34, 0, 0, 0, 4023, 0, 4092, 0, 4014, 0, 1458, 4093, 4096, 4082, 55, 4059, 0, 0, 436, 0, 1, 0, 1, 1, 2227, 0, 0, 81, 0, 15, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4, 7, 0, 5, 5, 4069, 25, 4087, 0, 4076, 0, 2, 4094, 4081, 4055, 0, 4095, 0, 0, 75, 24, 164, 8, 5, 25, 3652, 204, 1, 0, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[5, 0, 15, 0, 7, 3619, 0, 4056, 0, 3890, 0, 1553, 4096, 4094, 4092, 35, 4095, 1, 0, 1070, 0, 50, 0, 0, 66, 1341, 658, 0, 20, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 18, 4071, 14, 4093, 0, 3910, 0, 2, 4093, 4096, 4090, 4, 4090, 3, 0, 1, 0, 187, 1, 2, 38, 3854, 198, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[7, 0, 7, 0, 2, 3930, 0, 4096, 3, 2894, 7, 933, 4091, 4095, 4095, 88, 4088, 0, 0, 43, 0, 23, 1, 1, 0, 3971, 0, 2, 0, 0, 391, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 21, 4, 4071, 0, 0, 0, 0, 2207, 1, 328, 0, 0, 0, 0, 78, 2992, 0, 2210, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 3848, 1, 0, 0, 0, 0, 0, 0, 0, 2, 50, 36, 3961, 14, 0, 0, 26, 3605, 239, 6, 632, 204, 18, 0, 158, 0, 37, 0, 3749, 96, 92, 0] rerouted_tokens_per_ep_rank=[4104, 2536, 5280, 4074, 3849, 4063, 4730, 4132] avg=4096.0000 max=5280 max_rank=2 min=2536 min_rank=1 imbalance_ratio=1.289062 max_load_reduction=27488 max_load_reduction_pct=0.838867 moved_to_echo=16774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4075, 0, 0, 0, 4, 4074, 0, 329, 0, 0, 0, 0, 0, 3146, 6, 15, 0, 0, 0, 0, 0, 0, 4072, 66, 0, 0, 0, 0, 3878, 0, 0, 22, 0, 0, 0, 0, 0, 125, 0, 6, 3990, 14, 0, 0, 2, 4065, 0, 1, 665, 206, 18, 0, 4, 0, 21, 0, 3766, 97, 97, 0] rerouted_tokens_per_ep_rank=[4079, 4407, 3167, 4138, 3900, 4135, 4957, 3985] avg=4096.0000 max=4957 max_rank=6 min=3167 min_rank=2 imbalance_ratio=1.210205 max_load_reduction=27811 max_load_reduction_pct=0.848724 moved_to_echo=16977 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 34, 0, 0, 4080, 0, 0, 0, 0, 4023, 0, 329, 0, 0, 0, 0, 0, 3232, 0, 1458, 0, 0, 0, 0, 0, 0, 4064, 55, 0, 0, 0, 0, 3855, 0, 0, 436, 0, 0, 0, 0, 0, 1, 0, 1, 4000, 13, 0, 0, 1, 2227, 0, 0, 683, 204, 18, 0, 81, 0, 15, 0, 3763, 96, 99, 0] rerouted_tokens_per_ep_rank=[4114, 4352, 4690, 4119, 4291, 4015, 3133, 4054] avg=4096.0000 max=4690 max_rank=2 min=3133 min_rank=6 imbalance_ratio=1.145020 max_load_reduction=28078 max_load_reduction_pct=0.856873 moved_to_echo=15493 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 17, 0, 4081, 0, 0, 0, 0, 4076, 1, 329, 0, 0, 0, 0, 15, 3263, 0, 1968, 0, 0, 0, 0, 0, 0, 4047, 7, 0, 0, 0, 0, 3876, 0, 0, 119, 0, 0, 0, 0, 0, 45, 0, 0, 3997, 13, 0, 0, 0, 1578, 27, 0, 689, 206, 18, 0, 0, 0, 433, 0, 3765, 96, 100, 0] rerouted_tokens_per_ep_rank=[4100, 4406, 5246, 4054, 3995, 4055, 2518, 4394] avg=4096.0000 max=5246 max_rank=2 min=2518 min_rank=6 imbalance_ratio=1.280762 max_load_reduction=27522 max_load_reduction_pct=0.839905 moved_to_echo=14962 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4079, 0, 0, 0, 18, 4071, 14, 329, 0, 0, 0, 0, 0, 3148, 0, 2, 0, 0, 0, 0, 0, 0, 4072, 4, 0, 0, 0, 0, 3884, 3, 0, 1, 0, 0, 0, 0, 0, 187, 1, 2, 3999, 14, 0, 0, 38, 3854, 198, 0, 665, 206, 18, 0, 0, 0, 2, 0, 3764, 97, 97, 0] rerouted_tokens_per_ep_rank=[4080, 4432, 3150, 4076, 3888, 4203, 4979, 3960] avg=4096.0000 max=4979 max_rank=6 min=3150 min_rank=2 imbalance_ratio=1.215576 max_load_reduction=27789 max_load_reduction_pct=0.848053 moved_to_echo=17030 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 7, 0, 5, 4081, 0, 0, 0, 5, 4069, 25, 323, 0, 0, 0, 0, 0, 3274, 0, 2, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 3885, 0, 0, 75, 0, 0, 0, 0, 24, 164, 8, 5, 3985, 13, 0, 0, 25, 3652, 204, 1, 697, 210, 22, 0, 0, 5, 0, 0, 3764, 96, 105, 0] rerouted_tokens_per_ep_rank=[4097, 4422, 3276, 4033, 3960, 4199, 4811, 3970] avg=4096.0000 max=4811 max_rank=6 min=3276 min_rank=2 imbalance_ratio=1.174561 max_load_reduction=27957 max_load_reduction_pct=0.853180 moved_to_echo=16940 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 15, 0, 4083, 0, 0, 0, 7, 3619, 0, 326, 0, 0, 0, 0, 0, 3132, 0, 1553, 0, 0, 0, 0, 0, 0, 4074, 35, 0, 0, 0, 0, 3889, 1, 0, 1070, 0, 0, 0, 0, 0, 50, 0, 0, 3998, 13, 0, 0, 66, 1341, 658, 0, 662, 206, 18, 0, 20, 0, 0, 5, 3730, 96, 96, 0] rerouted_tokens_per_ep_rank=[4103, 3952, 4685, 4109, 4960, 4061, 2951, 3947] avg=4096.0000 max=4960 max_rank=4 min=2951 min_rank=6 imbalance_ratio=1.210938 max_load_reduction=27808 max_load_reduction_pct=0.848633 moved_to_echo=15919 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 7, 0, 4077, 0, 0, 0, 2, 3930, 0, 329, 0, 0, 0, 0, 3, 2331, 7, 933, 0, 0, 0, 0, 0, 0, 4077, 88, 0, 0, 0, 0, 3882, 0, 0, 43, 0, 0, 0, 0, 0, 23, 1, 1, 3998, 14, 0, 0, 0, 3971, 0, 2, 492, 206, 18, 0, 0, 0, 391, 0, 3767, 97, 71, 0] rerouted_tokens_per_ep_rank=[4091, 4261, 3274, 4165, 3925, 4037, 4689, 4326] avg=4096.0000 max=4689 max_rank=6 min=3274 min_rank=2 imbalance_ratio=1.144775 max_load_reduction=28079 max_load_reduction_pct=0.856903 moved_to_echo=16977 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 24320, 1408, 128, 5248, 1664, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 24576, 128, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 30080, 128, 2688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31104, 128, 0, 1792, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32640, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 1024, 128, 30080, 896, 768, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 640, 128, 128, 32000, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[19, 4, 0, 4087, 1, 0, 0, 5, 4057, 5, 7, 0, 4085, 16, 4092, 4054, 0, 0, 0, 16, 1, 4085, 0, 3829, 27, 0, 0, 8, 0, 4088, 279, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4071, 4, 1, 0, 2, 4095, 0, 0, 0, 4095, 0, 4096, 3838, 0, 2, 22, 0, 0, 4087, 3, 4092, 0, 0, 0, 4, 0, 4091, 264, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[294, 0, 35, 4072, 0, 0, 5, 0, 4096, 0, 48, 0, 4096, 0, 4096, 3776, 1, 0, 98, 0, 0, 4068, 7, 3868, 0, 0, 0, 0, 10, 4096, 102, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4082, 2, 1, 29, 0, 4043, 7, 19, 0, 4093, 0, 4095, 4060, 0, 0, 0, 0, 0, 4094, 13, 4083, 0, 0, 2, 47, 1, 4096, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[40, 0, 97, 4047, 0, 0, 8, 0, 4086, 0, 4, 0, 4096, 1, 4096, 2981, 1, 17, 38, 0, 5, 4057, 5, 3971, 128, 0, 0, 0, 0, 4083, 1000, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 9, 0, 111, 0, 4054, 12, 154, 0, 4079, 0, 4095, 3860, 1, 0, 4, 3, 0, 4083, 0, 4058, 0, 0, 0, 55, 0, 4088, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 35, 4081, 0, 0, 0, 0, 4056, 0, 8, 0, 4096, 0, 4088, 3726, 0, 0, 10, 1, 0, 4065, 0, 4091, 37, 68, 5, 7, 7, 4088, 298, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 2, 0, 4096, 0, 0, 0, 13, 4096, 0, 3, 0, 4096, 0, 4092, 4077, 0, 0, 2, 0, 0, 4096, 0, 4015, 0, 1, 0, 0, 0, 4082, 97, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[19, 4, 0, 4039, 0, 0, 0, 0, 1, 0, 0, 5, 4070, 0, 0, 0, 4047, 5, 7, 0, 0, 0, 0, 0, 298, 16, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 16, 3993, 58, 22, 0, 1, 92, 0, 3829, 0, 0, 0, 0, 27, 0, 0, 8, 3729, 252, 48, 10, 0, 3836, 279, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4076, 4059, 4368, 4089, 3922, 4074, 4118] avg=4096.0000 max=4368 max_rank=3 min=3922 min_rank=5 imbalance_ratio=1.066406 max_load_reduction=28400 max_load_reduction_pct=0.866699 moved_to_echo=16203 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4023, 0, 0, 0, 0, 4, 1, 0, 2, 4074, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 3838, 0, 0, 0, 0, 0, 2, 22, 0, 3995, 58, 22, 0, 0, 92, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 4, 3738, 252, 48, 10, 0, 3839, 264, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4081, 4085, 4137, 4099, 4187, 4052, 4104] avg=4096.0000 max=4187 max_rank=5 min=4023 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16442 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[294, 0, 35, 4024, 0, 0, 0, 0, 0, 0, 5, 0, 4075, 0, 0, 0, 4086, 0, 48, 0, 0, 0, 0, 0, 299, 0, 0, 3776, 0, 0, 0, 0, 1, 0, 98, 0, 3977, 58, 21, 0, 0, 91, 7, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 253, 48, 10, 10, 3843, 102, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4353, 4080, 4134, 4075, 4155, 3966, 4050, 3955] avg=4096.0000 max=4353 max_rank=0 min=3955 min_rank=7 imbalance_ratio=1.062744 max_load_reduction=28415 max_load_reduction_pct=0.867157 moved_to_echo=16126 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4034, 0, 0, 0, 0, 2, 1, 29, 0, 4073, 0, 0, 0, 4033, 7, 19, 0, 0, 0, 0, 0, 299, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 58, 22, 0, 0, 92, 13, 4083, 0, 0, 0, 0, 0, 0, 2, 47, 3736, 253, 48, 10, 1, 3843, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4105, 4059, 4359, 4082, 4188, 4096, 3845] avg=4096.0000 max=4359 max_rank=3 min=3845 min_rank=7 imbalance_ratio=1.064209 max_load_reduction=28409 max_load_reduction_pct=0.866974 moved_to_echo=16211 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 35, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 4046, 0, 8, 0, 0, 0, 0, 0, 299, 0, 0, 3726, 0, 0, 0, 0, 0, 0, 10, 1, 3974, 58, 22, 0, 0, 91, 0, 4091, 0, 0, 0, 0, 37, 68, 5, 7, 3739, 252, 48, 10, 7, 3836, 298, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4066, 4054, 4025, 4065, 4182, 4166, 4142] avg=4096.0000 max=4182 max_rank=5 min=4025 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16555 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4042, 0, 0, 0, 0, 9, 0, 111, 0, 4074, 0, 0, 0, 4042, 12, 154, 0, 0, 0, 0, 0, 287, 0, 0, 3860, 0, 0, 0, 0, 1, 0, 4, 3, 3997, 65, 21, 0, 0, 86, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 55, 3727, 255, 54, 12, 0, 3833, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4194, 4208, 4147, 4091, 4144, 4103, 3839] avg=4096.0000 max=4208 max_rank=2 min=3839 min_rank=7 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[40, 0, 97, 3999, 0, 0, 0, 0, 0, 0, 8, 0, 4075, 0, 0, 0, 4076, 0, 4, 0, 0, 0, 0, 0, 299, 1, 0, 2981, 0, 0, 0, 0, 1, 17, 38, 0, 3966, 58, 21, 0, 5, 91, 5, 3971, 0, 0, 0, 0, 128, 0, 0, 0, 3739, 252, 48, 10, 0, 3831, 1000, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4083, 4080, 3281, 4101, 4072, 4177, 4838] avg=4096.0000 max=4838 max_rank=7 min=3281 min_rank=3 imbalance_ratio=1.181152 max_load_reduction=27930 max_load_reduction_pct=0.852356 moved_to_echo=17188 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 13, 4070, 0, 0, 0, 4086, 0, 3, 0, 0, 0, 0, 0, 299, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 2, 0, 4004, 58, 22, 0, 0, 92, 0, 4015, 0, 0, 0, 0, 0, 1, 0, 0, 3739, 252, 48, 10, 0, 3830, 97, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4083, 4089, 4376, 4086, 4107, 4050, 3927] avg=4096.0000 max=4376 max_rank=3 min=3927 min_rank=7 imbalance_ratio=1.068359 max_load_reduction=28392 max_load_reduction_pct=0.866455 moved_to_echo=16170 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 256, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 128, 256, 32256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2432, 128, 0, 30464, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 256, 128, 32000, 512, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 128, 128, 29952, 2048, 512, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 30720, 2048, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 768, 128, 32128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32512, 128, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 1, 3908, 3455, 0, 4096, 3, 299, 5, 36, 4096, 0, 471, 3871, 1, 3, 3929, 0, 0, 0, 4050, 0, 0, 0, 1, 4083, 214, 234] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 188, 0, 0, 4081, 7, 17, 4089, 0, 22, 0, 87, 4095, 3, 12, 4083, 7, 0, 3983, 3, 0, 0, 4096, 0, 0, 0, 0, 4084, 5, 3906] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 3, 0, 11, 0, 4091, 269, 0, 4096, 0, 63, 0, 20, 4096, 0, 1047, 3021, 0, 5, 4096, 0, 5, 0, 4073, 0, 0, 0, 1, 4094, 0, 3775] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 127, 0, 0, 4094, 15, 0, 3385, 0, 136, 0, 0, 4096, 0, 1391, 3239, 0, 0, 4093, 0, 0, 0, 4096, 0, 5, 0, 0, 4084, 4, 4003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 11, 0, 0, 4093, 32, 1, 3374, 34, 716, 3, 8, 4027, 4, 2068, 2348, 2, 0, 4093, 0, 0, 3, 4094, 0, 2, 0, 1, 4001, 0, 3852] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 4008, 0, 0, 4068, 6, 23, 0, 445, 4079, 13, 33, 3678, 2, 0, 4085, 13, 0, 13, 4083, 0, 0, 0, 0, 4068, 62, 4087] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 32, 2, 0, 0, 4092, 390, 0, 4096, 4, 15, 0, 418, 4094, 2, 3458, 86, 0, 0, 4007, 0, 1, 0, 4096, 11, 0, 0, 0, 4091, 3, 3870] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 28, 22, 0, 0, 4096, 1373, 0, 4040, 0, 3, 0, 24, 4092, 0, 673, 3879, 0, 0, 4096, 0, 0, 0, 4091, 1, 0, 0, 0, 4096, 0, 2253] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3269, 633, 122, 1, 12, 1, 3275, 3455, 0, 0, 0, 0, 0, 4096, 3, 299, 29, 0, 0, 0, 5, 36, 3974, 0, 0, 0, 0, 0, 471, 3775, 1, 3, 0, 0, 0, 0, 3929, 0, 0, 0, 55, 0, 0, 0, 4050, 0, 0, 0, 11, 0, 0, 0, 1, 814, 214, 234, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 6743, 4427, 4015, 4250, 3984, 4061, 1263] avg=4096.0000 max=6743 max_rank=1 min=1263 min_rank=7 imbalance_ratio=1.646240 max_load_reduction=26025 max_load_reduction_pct=0.794220 moved_to_echo=13558 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 127, 3270, 663, 122, 1, 0, 0, 3431, 15, 0, 0, 0, 0, 0, 3385, 0, 136, 24, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 1391, 3159, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 46, 0, 0, 0, 4096, 0, 5, 0, 9, 0, 0, 0, 0, 814, 4, 4003, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4183, 3446, 3545, 3974, 4550, 4139, 4110, 4821] avg=4096.0000 max=4821 max_rank=7 min=3446 min_rank=1 imbalance_ratio=1.177002 max_load_reduction=27947 max_load_reduction_pct=0.852875 moved_to_echo=17620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 3257, 649, 121, 1, 0, 0, 3359, 0, 0, 0, 0, 0, 0, 4068, 6, 23, 27, 0, 0, 0, 0, 445, 3958, 13, 0, 0, 0, 0, 33, 3588, 2, 0, 0, 0, 0, 0, 4085, 13, 0, 13, 52, 0, 0, 0, 4083, 0, 0, 0, 10, 0, 0, 0, 0, 811, 62, 4087, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 3359, 4124, 4416, 3623, 4163, 4093, 4960] avg=4096.0000 max=4960 max_rank=7 min=3359 min_rank=1 imbalance_ratio=1.210938 max_load_reduction=27808 max_load_reduction_pct=0.848633 moved_to_echo=16839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 11, 3203, 663, 120, 0, 0, 0, 3430, 32, 0, 0, 0, 0, 1, 3374, 34, 716, 17, 0, 0, 0, 3, 8, 3907, 4, 0, 0, 0, 0, 2068, 2292, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 33, 0, 0, 0, 4094, 0, 2, 0, 6, 0, 0, 0, 1, 798, 0, 3852, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 3462, 4142, 3922, 4362, 4129, 4102, 4651] avg=4096.0000 max=4651 max_rank=7 min=3462 min_rank=1 imbalance_ratio=1.135498 max_load_reduction=28117 max_load_reduction_pct=0.858063 moved_to_echo=17244 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3, 0, 3281, 665, 124, 5, 11, 0, 3426, 269, 0, 0, 0, 0, 0, 4096, 0, 63, 26, 0, 0, 0, 0, 20, 3972, 0, 0, 0, 0, 0, 1047, 2930, 0, 5, 0, 0, 0, 0, 4096, 0, 5, 0, 48, 0, 0, 0, 4073, 0, 0, 0, 12, 0, 0, 0, 1, 813, 0, 3775, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 3706, 4185, 3992, 3982, 4149, 4085, 4589] avg=4096.0000 max=4589 max_rank=7 min=3706 min_rank=1 imbalance_ratio=1.120361 max_load_reduction=28179 max_load_reduction_pct=0.859955 moved_to_echo=16805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 32, 2, 3275, 663, 122, 0, 0, 0, 3429, 390, 0, 0, 0, 0, 0, 4096, 4, 15, 0, 0, 0, 0, 0, 418, 3972, 2, 0, 0, 0, 0, 3458, 85, 0, 0, 0, 0, 0, 0, 4007, 0, 1, 0, 1, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 816, 3, 3870, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 3819, 4115, 4392, 3543, 4009, 4107, 4689] avg=4096.0000 max=4689 max_rank=7 min=3543 min_rank=4 imbalance_ratio=1.144775 max_load_reduction=28079 max_load_reduction_pct=0.856903 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 188, 3270, 661, 122, 1, 0, 0, 3420, 7, 0, 0, 0, 0, 17, 4089, 0, 22, 30, 0, 0, 0, 0, 87, 3973, 3, 0, 0, 0, 0, 12, 3983, 7, 0, 0, 0, 0, 0, 3983, 3, 0, 0, 58, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 814, 5, 3906, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4242, 3427, 4158, 4063, 4002, 4044, 4107, 4725] avg=4096.0000 max=4725 max_rank=7 min=3427 min_rank=1 imbalance_ratio=1.153564 max_load_reduction=28043 max_load_reduction_pct=0.855804 moved_to_echo=16878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 28, 22, 3279, 663, 122, 1, 0, 0, 3433, 1373, 0, 0, 0, 0, 0, 4040, 0, 3, 29, 0, 0, 0, 0, 24, 3970, 0, 0, 0, 0, 0, 673, 3783, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 55, 0, 0, 0, 4091, 1, 0, 0, 11, 0, 0, 0, 0, 817, 0, 2253, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4806, 4072, 3994, 4456, 4151, 4103, 3070] avg=4096.0000 max=4806 max_rank=1 min=3070 min_rank=7 imbalance_ratio=1.173340 max_load_reduction=27962 max_load_reduction_pct=0.853333 moved_to_echo=15780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 384, 26112, 5376, 1024, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 6528, 384, 25984, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 27264, 5632, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9216, 23680, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31360, 128, 1280, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 1152, 31744, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32384, 128, 128, 128, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4022, 0, 4064, 5, 0, 10, 0, 2, 0, 0, 55, 9, 0, 0, 61, 0, 4090, 0, 0, 4087, 2, 5, 4093, 27, 4084, 0, 4057, 0, 1097, 2998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4031, 0, 3977, 0, 3, 1, 26, 0, 0, 110, 0, 0, 22, 0, 109, 1, 3948, 0, 0, 4058, 7, 0, 4090, 10, 4077, 18, 4075, 0, 110, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4090, 15, 4095, 0, 0, 0, 0, 198, 0, 12, 0, 0, 0, 0, 0, 0, 4095, 0, 9, 4093, 0, 3, 4023, 7, 4088, 0, 4085, 0, 0, 3955] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[2, 7, 4093, 0, 4023, 12, 0, 0, 0, 141, 0, 1, 41, 0, 0, 0, 0, 0, 3889, 0, 0, 4096, 0, 598, 4096, 0, 4089, 0, 4085, 0, 1144, 2451] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[63, 0, 4096, 1, 4030, 0, 0, 1, 0, 102, 0, 0, 13, 0, 0, 0, 0, 3, 4012, 0, 0, 4094, 0, 39, 4096, 14, 4096, 0, 4093, 0, 0, 4015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[12, 77, 4075, 0, 4087, 0, 0, 63, 0, 28, 0, 135, 0, 0, 0, 0, 9, 3, 4012, 0, 0, 4093, 0, 44, 4064, 0, 4065, 0, 4091, 0, 342, 3568] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 4076, 0, 4, 41, 2, 0, 0, 1, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 4089, 0, 4074, 0, 4095, 2, 1, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4084, 0, 0, 14, 0, 11, 0, 25, 0, 0, 0, 0, 2, 0, 4072, 0, 1, 4096, 0, 1038, 4096, 6, 4096, 0, 4084, 0, 91, 2956] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 7, 4093, 0, 0, 0, 0, 0, 4023, 12, 0, 0, 21, 0, 0, 0, 0, 141, 0, 1, 3854, 143, 1, 0, 41, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 3889, 0, 45, 0, 0, 0, 0, 3887, 0, 598, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 231, 0, 1144, 2451, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4056, 4140, 4126, 3934, 4485, 4099, 3826] avg=4096.0000 max=4485 max_rank=5 min=3826 min_rank=7 imbalance_ratio=1.094971 max_load_reduction=28283 max_load_reduction_pct=0.863129 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4031, 0, 0, 0, 0, 0, 3977, 0, 3, 1, 21, 0, 0, 0, 26, 0, 0, 110, 3844, 142, 0, 0, 0, 0, 22, 0, 4071, 0, 0, 0, 109, 1, 3948, 0, 45, 0, 0, 0, 0, 3850, 7, 0, 0, 0, 0, 0, 4090, 10, 6, 18, 0, 0, 0, 0, 231, 0, 110, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4002, 4122, 4093, 4103, 3857, 4124, 4436] avg=4096.0000 max=4436 max_rank=7 min=3857 min_rank=5 imbalance_ratio=1.083008 max_load_reduction=28332 max_load_reduction_pct=0.864624 moved_to_echo=16520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4022, 0, 0, 0, 0, 0, 4064, 5, 0, 10, 21, 0, 0, 0, 0, 2, 0, 0, 3827, 143, 0, 0, 55, 9, 0, 0, 4078, 0, 0, 0, 61, 0, 4090, 0, 45, 0, 0, 0, 0, 3878, 2, 5, 0, 0, 0, 0, 4093, 27, 6, 0, 0, 0, 0, 0, 230, 0, 1097, 2998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4100, 3972, 4142, 4196, 3885, 4126, 4325] avg=4096.0000 max=4325 max_rank=7 min=3885 min_rank=5 imbalance_ratio=1.055908 max_load_reduction=28443 max_load_reduction_pct=0.868011 moved_to_echo=16532 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[63, 0, 4096, 1, 0, 0, 0, 0, 4030, 0, 0, 1, 21, 0, 0, 0, 0, 102, 0, 0, 3861, 143, 0, 0, 13, 0, 0, 0, 4090, 0, 0, 0, 0, 3, 4012, 0, 45, 0, 0, 0, 0, 3885, 0, 39, 0, 0, 0, 0, 4096, 14, 6, 0, 0, 0, 0, 0, 232, 0, 0, 4015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4052, 4106, 4103, 4060, 3924, 4116, 4247] avg=4096.0000 max=4247 max_rank=7 min=3924 min_rank=5 imbalance_ratio=1.036865 max_load_reduction=28521 max_load_reduction_pct=0.870392 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 0, 0, 0, 0, 4076, 0, 4, 41, 21, 0, 0, 0, 2, 0, 0, 1, 3863, 143, 0, 0, 5, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 4096, 0, 45, 0, 0, 0, 0, 3885, 0, 0, 0, 0, 0, 0, 4089, 0, 6, 0, 0, 0, 0, 0, 232, 2, 1, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4142, 4009, 4073, 4141, 3885, 4095, 4331] avg=4096.0000 max=4331 max_rank=7 min=3885 min_rank=5 imbalance_ratio=1.057373 max_load_reduction=28437 max_load_reduction_pct=0.867828 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4086, 15, 0, 0, 0, 0, 4095, 0, 0, 0, 27, 4, 0, 0, 0, 198, 0, 12, 3857, 149, 6, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 4095, 0, 51, 0, 0, 0, 9, 3866, 0, 3, 0, 0, 0, 0, 4023, 7, 0, 0, 0, 0, 0, 0, 228, 0, 0, 3955, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4126, 4222, 4082, 4146, 3878, 4030, 4183] avg=4096.0000 max=4222 max_rank=2 min=3878 min_rank=5 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16237 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[12, 77, 4075, 0, 0, 0, 0, 0, 4087, 0, 0, 63, 21, 0, 0, 0, 0, 28, 0, 135, 3859, 143, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 9, 3, 4012, 0, 45, 0, 0, 0, 0, 3884, 0, 44, 0, 0, 0, 0, 4064, 0, 6, 0, 0, 0, 0, 0, 232, 0, 342, 3568, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4171, 4165, 4059, 4069, 3928, 4070, 4142] avg=4096.0000 max=4171 max_rank=1 min=3928 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16209 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 14, 21, 0, 0, 0, 0, 11, 0, 25, 3853, 143, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 2, 0, 4072, 0, 45, 0, 0, 0, 1, 3887, 0, 1038, 0, 0, 0, 0, 4096, 6, 6, 0, 0, 0, 0, 0, 231, 0, 91, 2956, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4119, 4032, 4090, 4119, 4926, 4108, 3278] avg=4096.0000 max=4926 max_rank=5 min=3278 min_rank=7 imbalance_ratio=1.202637 max_load_reduction=27842 max_load_reduction_pct=0.849670 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1920, 128, 2816, 28160, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 0, 384, 30848, 1152, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 32256, 0, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31104, 128, 1792, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 128, 0, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32512, 128, 128, 256, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1, 1, 0, 6, 0, 4080, 12, 428, 50, 3998, 0, 492, 9, 0, 4087, 8, 0, 0, 1, 2776, 4075, 4083, 2434, 0, 45, 2044, 19, 15, 0, 0, 4081, 23] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[19, 0, 0, 0, 0, 4096, 0, 118, 25, 4096, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 3773, 4080, 4080, 481, 0, 0, 3766, 0, 25, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[24, 1, 0, 0, 1, 4095, 2, 447, 18, 4083, 3, 0, 14, 0, 4077, 0, 0, 0, 4, 2769, 4091, 4096, 4054, 0, 693, 205, 0, 0, 0, 0, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4, 11, 0, 1, 0, 4043, 3, 0, 30, 4096, 0, 0, 0, 0, 4085, 0, 0, 3, 0, 3760, 4096, 4073, 100, 1, 238, 4089, 0, 0, 0, 9, 4072, 54] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[6, 3, 0, 0, 0, 4096, 0, 28, 0, 4093, 3, 169, 22, 0, 4069, 43, 0, 0, 0, 2437, 3904, 4069, 2008, 3, 135, 3402, 1, 187, 2, 0, 4088, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[9, 0, 2, 0, 0, 4094, 1, 0, 0, 4091, 4, 4, 0, 0, 4094, 0, 0, 3, 0, 1260, 4095, 4096, 4096, 2, 6, 2815, 3, 0, 1, 0, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[44, 0, 0, 0, 0, 4096, 4, 1, 0, 4059, 0, 0, 2, 0, 4090, 1, 0, 77, 14, 3414, 4053, 4064, 1999, 0, 3, 2711, 0, 41, 0, 4, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1317, 0, 2, 0, 1, 4088, 0, 55, 5, 4091, 0, 0, 5, 4, 4096, 0, 0, 0, 0, 680, 4091, 4093, 2969, 0, 0, 3170, 0, 5, 0, 3, 4093, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 6, 3915, 0, 0, 0, 0, 3953, 12, 428, 0, 0, 0, 0, 50, 3920, 0, 492, 0, 0, 0, 0, 9, 0, 4083, 8, 0, 0, 0, 0, 0, 0, 1, 2776, 1309, 168, 0, 0, 1836, 0, 2434, 0, 0, 0, 0, 0, 45, 2044, 19, 15, 930, 127, 78, 4, 0, 0, 4081, 23, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3923, 4393, 4462, 4100, 4254, 4270, 3262, 4104] avg=4096.0000 max=4462 max_rank=2 min=3262 min_rank=6 imbalance_ratio=1.089355 max_load_reduction=28306 max_load_reduction_pct=0.863831 moved_to_echo=15890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[24, 1, 0, 0, 3928, 0, 0, 0, 1, 3968, 2, 447, 0, 0, 0, 0, 18, 4003, 3, 0, 0, 0, 0, 0, 14, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 4, 2769, 1315, 168, 0, 0, 1842, 0, 4054, 0, 0, 0, 0, 0, 693, 205, 0, 0, 934, 127, 80, 4, 0, 0, 4091, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3953, 4418, 4024, 4087, 4256, 5896, 2043, 4091] avg=4096.0000 max=5896 max_rank=5 min=2043 min_rank=6 imbalance_ratio=1.439453 max_load_reduction=26872 max_load_reduction_pct=0.820068 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 0, 0, 3913, 0, 0, 0, 0, 3969, 0, 118, 0, 0, 0, 0, 25, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 17, 0, 3773, 1311, 167, 0, 0, 1837, 0, 481, 0, 0, 0, 0, 0, 0, 3766, 0, 25, 932, 127, 80, 4, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3932, 4087, 4041, 4092, 5268, 2318, 4934, 4096] avg=4096.0000 max=5268 max_rank=4 min=2318 min_rank=5 imbalance_ratio=1.286133 max_load_reduction=27500 max_load_reduction_pct=0.839233 moved_to_echo=16616 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 2, 0, 3929, 0, 0, 0, 0, 3967, 1, 0, 0, 0, 0, 0, 0, 4011, 4, 4, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 3, 0, 1260, 1316, 167, 0, 0, 1844, 0, 4096, 2, 0, 0, 0, 0, 6, 2815, 3, 0, 935, 127, 80, 4, 1, 0, 4092, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3940, 3968, 4019, 4090, 2746, 5942, 3970, 4093] avg=4096.0000 max=5942 max_rank=5 min=2746 min_rank=4 imbalance_ratio=1.450684 max_load_reduction=26826 max_load_reduction_pct=0.818665 moved_to_echo=16751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[44, 0, 0, 0, 3897, 0, 0, 0, 0, 3969, 4, 1, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 2, 0, 4086, 1, 0, 0, 0, 0, 0, 77, 14, 3414, 1302, 167, 0, 0, 1826, 0, 1999, 0, 0, 0, 0, 0, 3, 2711, 0, 41, 925, 127, 79, 4, 0, 4, 4091, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3941, 3974, 3980, 4089, 4974, 3825, 3890, 4095] avg=4096.0000 max=4974 max_rank=4 min=3825 min_rank=5 imbalance_ratio=1.214355 max_load_reduction=27794 max_load_reduction_pct=0.848206 moved_to_echo=16784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 11, 0, 1, 3907, 0, 0, 0, 0, 3911, 3, 0, 0, 0, 0, 0, 30, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 3, 0, 3760, 1320, 166, 0, 0, 1836, 0, 100, 1, 0, 0, 0, 0, 238, 4089, 0, 0, 940, 132, 85, 6, 0, 9, 4072, 54, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3923, 3914, 4041, 4079, 5249, 1937, 5490, 4135] avg=4096.0000 max=5490 max_rank=6 min=1937 min_rank=5 imbalance_ratio=1.340332 max_load_reduction=27278 max_load_reduction_pct=0.832458 moved_to_echo=16811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 3, 0, 0, 3903, 0, 0, 0, 0, 3969, 0, 28, 0, 0, 0, 0, 0, 4013, 3, 169, 0, 0, 0, 0, 22, 0, 4065, 43, 0, 0, 0, 0, 0, 0, 0, 2437, 1255, 166, 0, 0, 1758, 0, 2008, 3, 0, 0, 0, 0, 135, 3402, 1, 187, 891, 127, 80, 4, 2, 0, 4088, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3912, 3997, 4185, 4130, 3858, 3769, 4827, 4090] avg=4096.0000 max=4827 max_rank=6 min=3769 min_rank=5 imbalance_ratio=1.178467 max_load_reduction=27941 max_load_reduction_pct=0.852692 moved_to_echo=16544 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1317, 0, 2, 0, 3925, 0, 0, 0, 1, 3961, 0, 55, 0, 0, 0, 0, 5, 4011, 0, 0, 0, 0, 0, 0, 5, 4, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 680, 1315, 168, 0, 0, 1842, 0, 2969, 0, 0, 0, 0, 0, 0, 3170, 0, 5, 934, 127, 80, 4, 0, 3, 4093, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5244, 4017, 4016, 4101, 2163, 4811, 4320, 4096] avg=4096.0000 max=5244 max_rank=0 min=2163 min_rank=4 imbalance_ratio=1.280273 max_load_reduction=27524 max_load_reduction_pct=0.839966 moved_to_echo=15390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32736, 32800] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32736, 32800] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32736, 32800] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32736, 32800] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32736, 32800] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32736, 32800] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32736, 32800] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32736, 32800] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14720, 0, 18176, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 20992, 10496, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32736 actual_valid_recv=33280 observed_loss=-544 tokens_per_expert=[1152, 22272, 128, 384, 7424, 1024, 768, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1536, 128, 128, 128, 31360, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31744, 128, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32800 actual_valid_recv=33152 observed_loss=-352 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32000, 128, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[106, 4021, 2708, 0, 2483, 1, 0, 0, 78, 0, 4079, 0, 4065, 0, 0, 0, 4052, 30, 13, 1, 0, 53, 16, 12, 11, 1, 4068, 4095, 8, 5, 133, 2729] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 4095, 3653, 0, 1537, 1, 7, 0, 2186, 0, 4092, 0, 4021, 106, 0, 0, 4092, 2, 3, 358, 0, 0, 0, 2, 2, 0, 4096, 3736, 1, 0, 0, 778] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[2, 4078, 3758, 25, 30, 0, 0, 0, 2241, 41, 4082, 0, 4096, 1, 0, 2, 4096, 2, 0, 2259, 0, 1502, 1, 0, 0, 0, 4096, 2406, 0, 27, 0, 23] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 2078, 0, 4, 0, 0, 0, 109, 73, 4096, 0, 4084, 0, 0, 0, 4096, 0, 0, 1801, 0, 602, 0, 0, 0, 0, 4096, 4059, 0, 0, 0, 3574] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 4094, 4096, 1, 103, 0, 192, 0, 2, 21, 4078, 0, 4072, 92, 16, 1, 4096, 0, 12, 558, 0, 0, 0, 0, 0, 0, 4085, 3857, 6, 0, 0, 3386] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[2, 4076, 3948, 2, 21, 9, 0, 0, 0, 18, 4096, 0, 4056, 2, 1, 0, 4049, 0, 0, 544, 0, 136, 0, 0, 0, 0, 4087, 4096, 0, 50, 25, 3550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 4061, 3249, 0, 3170, 0, 29, 0, 31, 10, 4092, 0, 4052, 0, 0, 0, 4085, 0, 5, 1532, 0, 4, 0, 0, 1, 2, 4084, 4092, 4, 46, 18, 201] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 4085, 4, 0, 0, 0, 0, 0, 0, 4095, 23, 4063, 37, 0, 2, 4092, 0, 0, 2879, 1, 20, 0, 0, 0, 10, 4091, 3885, 0, 1, 0, 1384] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[106, 624, 2708, 0, 0, 0, 0, 0, 2483, 1, 0, 0, 3104, 0, 0, 0, 78, 0, 3484, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 2826, 30, 13, 1, 0, 0, 0, 0, 0, 53, 16, 12, 3755, 28, 0, 0, 11, 1, 313, 4095, 0, 0, 0, 0, 8, 5, 133, 2729, 1226, 595, 265, 0] rerouted_tokens_per_ep_rank=[3438, 5588, 3562, 4065, 2870, 3864, 4420, 4961] avg=4096.0000 max=5588 max_rank=1 min=2870 min_rank=4 imbalance_ratio=1.364258 max_load_reduction=27180 max_load_reduction_pct=0.829468 moved_to_echo=16115 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 636, 4096, 1, 0, 0, 0, 0, 103, 0, 192, 0, 3160, 0, 0, 0, 2, 21, 3483, 0, 0, 0, 0, 0, 4072, 92, 16, 1, 0, 0, 0, 0, 2856, 0, 12, 558, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 28, 0, 0, 0, 0, 315, 3857, 0, 0, 0, 0, 6, 0, 0, 3386, 1240, 595, 270, 0] rerouted_tokens_per_ep_rank=[4733, 3455, 3506, 4181, 3426, 3798, 4172, 5497] avg=4096.0000 max=5497 max_rank=7 min=3426 min_rank=4 imbalance_ratio=1.342041 max_load_reduction=27271 max_load_reduction_pct=0.832245 moved_to_echo=16893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 635, 2078, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3162, 0, 0, 0, 109, 73, 3499, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 2856, 0, 0, 1801, 0, 0, 0, 0, 0, 602, 0, 0, 3781, 29, 0, 0, 0, 0, 315, 4059, 0, 0, 0, 0, 0, 0, 0, 3574, 1240, 597, 270, 0] rerouted_tokens_per_ep_rank=[2713, 3166, 3681, 4084, 4657, 4412, 4374, 5681] avg=4096.0000 max=5681 max_rank=7 min=2713 min_rank=0 imbalance_ratio=1.386963 max_load_reduction=27087 max_load_reduction_pct=0.826630 moved_to_echo=19124 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 635, 3653, 0, 0, 0, 0, 0, 1537, 1, 7, 0, 3161, 0, 0, 0, 2186, 0, 3495, 0, 0, 0, 0, 0, 4021, 106, 0, 0, 0, 0, 0, 0, 2853, 2, 3, 358, 0, 0, 0, 0, 0, 0, 0, 2, 3781, 29, 0, 0, 2, 0, 315, 3736, 0, 0, 0, 0, 1, 0, 0, 778, 1239, 597, 270, 0] rerouted_tokens_per_ep_rank=[4288, 4706, 5681, 4127, 3216, 3812, 4053, 2885] avg=4096.0000 max=5681 max_rank=2 min=2885 min_rank=7 imbalance_ratio=1.386963 max_load_reduction=27087 max_load_reduction_pct=0.826630 moved_to_echo=13966 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 633, 3948, 2, 0, 0, 0, 0, 21, 9, 0, 0, 3146, 0, 0, 0, 0, 18, 3499, 0, 0, 0, 0, 0, 4056, 2, 1, 0, 0, 0, 0, 0, 2823, 0, 0, 544, 0, 0, 0, 0, 0, 136, 0, 0, 3772, 28, 0, 0, 0, 0, 315, 4096, 0, 0, 0, 0, 0, 50, 25, 3550, 1226, 597, 269, 0] rerouted_tokens_per_ep_rank=[4585, 3176, 3517, 4059, 3367, 3936, 4411, 5717] avg=4096.0000 max=5717 max_rank=7 min=3176 min_rank=1 imbalance_ratio=1.395752 max_load_reduction=27051 max_load_reduction_pct=0.825531 moved_to_echo=17431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 633, 3758, 25, 0, 0, 0, 0, 30, 0, 0, 0, 3148, 0, 0, 0, 2241, 41, 3487, 0, 0, 0, 0, 0, 4096, 1, 0, 2, 0, 0, 0, 0, 2856, 2, 0, 2259, 0, 0, 0, 0, 0, 1502, 1, 0, 3781, 28, 0, 0, 0, 0, 315, 2406, 0, 0, 0, 0, 0, 27, 0, 23, 1240, 595, 269, 0] rerouted_tokens_per_ep_rank=[4418, 3178, 5769, 4099, 5117, 5312, 2721, 2154] avg=4096.0000 max=5769 max_rank=2 min=2154 min_rank=7 imbalance_ratio=1.408447 max_load_reduction=26999 max_load_reduction_pct=0.823944 moved_to_echo=15304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 635, 4085, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3162, 0, 0, 0, 0, 0, 3498, 23, 0, 0, 0, 0, 4063, 37, 0, 2, 0, 0, 0, 0, 2853, 0, 0, 2879, 0, 0, 0, 0, 1, 20, 0, 0, 3776, 29, 0, 0, 0, 10, 315, 3885, 0, 0, 0, 0, 0, 1, 0, 1384, 1239, 597, 270, 0] rerouted_tokens_per_ep_rank=[4724, 3162, 3521, 4102, 5732, 3826, 4210, 3491] avg=4096.0000 max=5732 max_rank=4 min=3162 min_rank=1 imbalance_ratio=1.399414 max_load_reduction=27036 max_load_reduction_pct=0.825073 moved_to_echo=17259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 620, 3249, 0, 0, 0, 0, 0, 3170, 0, 29, 0, 3138, 0, 0, 0, 31, 10, 3490, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 2846, 0, 5, 1532, 0, 0, 0, 0, 0, 4, 0, 0, 3772, 32, 0, 0, 1, 2, 312, 4092, 0, 0, 0, 0, 4, 46, 18, 201, 1239, 602, 271, 1] rerouted_tokens_per_ep_rank=[3869, 6337, 3531, 4051, 4383, 3808, 4407, 2382] avg=4096.0000 max=6337 max_rank=1 min=2382 min_rank=7 imbalance_ratio=1.547119 max_load_reduction=26431 max_load_reduction_pct=0.806610 moved_to_echo=14980 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 256, 256, 15744, 9984, 4864, 2176, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[22784, 128, 128, 9984, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7424, 128, 256, 0, 25216, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 5120, 27648, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4736, 256, 28032, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 2432, 128, 128, 30208, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 2560, 30336, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[18, 4042, 4056, 1, 0, 0, 0, 1, 0, 0, 4095, 0, 4074, 0, 15, 0, 4083, 4079, 0, 0, 60, 2460, 1604, 40, 2, 0, 0, 9, 33, 0, 4086, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[7, 4089, 4095, 245, 0, 0, 0, 0, 0, 0, 4058, 0, 4081, 0, 0, 0, 4083, 4094, 0, 0, 14, 2586, 0, 7, 0, 1322, 27, 0, 0, 10, 4050, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4083, 4095, 200, 0, 14, 3, 2, 6, 3, 4096, 0, 4086, 0, 0, 11, 4014, 4086, 4, 0, 0, 3851, 0, 0, 3, 108, 0, 6, 0, 1, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1, 4094, 4091, 0, 7, 0, 0, 2, 10, 2, 4092, 14, 4058, 2, 0, 8, 4093, 4094, 0, 3, 0, 3275, 6, 129, 690, 0, 0, 1, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[5, 4080, 4070, 221, 0, 0, 0, 0, 0, 1, 3962, 48, 4090, 0, 0, 0, 4096, 4089, 3, 0, 16, 4050, 20, 0, 0, 0, 0, 0, 9, 0, 4008, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4094, 4096, 0, 0, 0, 4037, 0, 59, 5, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[7, 4090, 4070, 0, 0, 0, 25, 0, 12, 9, 4083, 0, 4083, 0, 0, 2, 4072, 4086, 4, 4, 0, 3065, 52, 0, 60, 948, 8, 0, 0, 0, 4088, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4066, 4096, 102, 0, 0, 0, 0, 0, 0, 4031, 35, 4096, 0, 0, 0, 4093, 4096, 0, 0, 0, 4026, 27, 0, 0, 0, 24, 0, 6, 1, 4069, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[18, 3960, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4056, 5, 0, 0, 0, 0, 4095, 0, 14, 0, 0, 0, 4074, 0, 15, 0, 8, 0, 0, 0, 4083, 17, 0, 0, 0, 0, 0, 0, 60, 2460, 1604, 40, 373, 47, 0, 0, 2, 0, 0, 9, 3684, 0, 0, 0, 33, 0, 4086, 10, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[3979, 4062, 4109, 4097, 4100, 4584, 3695, 4142] avg=4096.0000 max=4584 max_rank=5 min=3695 min_rank=6 imbalance_ratio=1.119141 max_load_reduction=28184 max_load_reduction_pct=0.860107 moved_to_echo=16521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4000, 0, 200, 0, 0, 0, 0, 0, 14, 3, 2, 4095, 5, 0, 0, 6, 3, 4096, 0, 14, 0, 0, 0, 4086, 0, 0, 11, 8, 0, 0, 0, 4014, 17, 4, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 374, 48, 0, 0, 3, 108, 0, 6, 3690, 0, 0, 0, 0, 1, 4096, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4200, 4119, 4119, 4105, 4035, 4273, 3807, 4110] avg=4096.0000 max=4273 max_rank=5 min=3807 min_rank=6 imbalance_ratio=1.043213 max_load_reduction=28495 max_load_reduction_pct=0.869598 moved_to_echo=16225 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 0, 4093, 0, 0, 0, 8, 0, 0, 0, 4094, 17, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 59, 375, 48, 0, 0, 5, 0, 0, 0, 3699, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4101, 4110, 4101, 4111, 4519, 3704, 4109] avg=4096.0000 max=4519 max_rank=5 min=3704 min_rank=6 imbalance_ratio=1.103271 max_load_reduction=28249 max_load_reduction_pct=0.862091 moved_to_echo=16443 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 4011, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 4091, 5, 0, 0, 10, 2, 4092, 14, 14, 0, 0, 0, 4058, 2, 0, 8, 8, 0, 0, 0, 4093, 17, 0, 3, 0, 0, 0, 0, 0, 3275, 6, 129, 375, 48, 0, 0, 690, 0, 0, 1, 3697, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4105, 4132, 4076, 4113, 3833, 4388, 4109] avg=4096.0000 max=4388 max_rank=6 min=3833 min_rank=5 imbalance_ratio=1.071289 max_load_reduction=28380 max_load_reduction_pct=0.866089 moved_to_echo=16443 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5, 3997, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 5, 0, 0, 0, 1, 3962, 48, 14, 0, 0, 0, 4090, 0, 0, 0, 8, 0, 0, 0, 4096, 17, 3, 0, 0, 0, 0, 0, 16, 4050, 20, 0, 374, 48, 0, 0, 0, 0, 0, 0, 3693, 0, 0, 0, 9, 0, 4008, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4223, 4075, 4025, 4098, 4116, 4508, 3693, 4030] avg=4096.0000 max=4508 max_rank=5 min=3693 min_rank=6 imbalance_ratio=1.100586 max_load_reduction=28260 max_load_reduction_pct=0.862427 moved_to_echo=16347 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[7, 4006, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 4058, 0, 14, 0, 0, 0, 4081, 0, 0, 0, 8, 0, 0, 0, 4083, 17, 0, 0, 0, 0, 0, 0, 14, 2586, 0, 7, 375, 48, 0, 0, 0, 1322, 27, 0, 3697, 0, 0, 0, 0, 10, 4050, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4258, 4100, 4072, 4089, 4100, 3030, 5046, 4073] avg=4096.0000 max=5046 max_rank=6 min=3030 min_rank=5 imbalance_ratio=1.231934 max_load_reduction=27722 max_load_reduction_pct=0.846008 moved_to_echo=16249 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[7, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4070, 10, 0, 0, 12, 9, 4083, 0, 17, 0, 0, 0, 4083, 0, 0, 2, 13, 0, 0, 0, 4072, 3, 4, 4, 0, 0, 0, 0, 0, 3065, 52, 0, 377, 51, 0, 0, 60, 948, 8, 0, 3696, 0, 0, 0, 0, 0, 4088, 0, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 4105, 4121, 4098, 4083, 3545, 4712, 4106] avg=4096.0000 max=4712 max_rank=6 min=3545 min_rank=5 imbalance_ratio=1.150391 max_load_reduction=28056 max_load_reduction_pct=0.856201 moved_to_echo=16446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3983, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 4031, 35, 14, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 4093, 17, 0, 0, 0, 0, 0, 0, 0, 4026, 27, 0, 375, 48, 0, 0, 0, 0, 24, 0, 3699, 0, 0, 0, 6, 1, 4069, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4101, 4080, 4104, 4110, 4476, 3723, 4089] avg=4096.0000 max=4476 max_rank=5 min=3723 min_rank=6 imbalance_ratio=1.092773 max_load_reduction=28292 max_load_reduction_pct=0.863403 moved_to_echo=16398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32000, 0, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 27392, 1792, 256, 3072, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 2432, 128, 128, 29568, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 3980, 4056, 3834, 31, 60, 16, 3877, 10, 0, 4079, 0, 4035, 0, 4, 0, 0, 118, 3711, 0, 0, 0, 0, 16, 412, 47, 0, 4046, 0, 1, 434, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4034, 4091, 4091, 4, 5, 4, 3819, 0, 1, 4085, 1, 4091, 0, 1, 0, 0, 0, 4063, 3, 97, 0, 0, 0, 0, 0, 0, 4090, 2, 254, 32, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4095, 4095, 3910, 0, 0, 0, 1591, 0, 0, 4096, 0, 4096, 0, 0, 1, 6, 14, 4056, 39, 0, 0, 0, 0, 0, 1, 1, 4089, 3, 0, 2675, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4075, 4087, 4061, 0, 0, 49, 4055, 1, 0, 4096, 4, 4077, 0, 1, 0, 20, 0, 4049, 40, 29, 1, 3, 0, 0, 17, 0, 4095, 0, 8, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[10, 3927, 4073, 4065, 0, 0, 0, 3869, 3, 16, 4096, 0, 4085, 8, 0, 0, 1, 86, 4061, 0, 8, 0, 6, 0, 0, 17, 2, 4089, 36, 0, 304, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 3981, 4060, 4094, 0, 0, 0, 3801, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 21, 4096, 31, 0, 0, 0, 0, 39, 0, 0, 4094, 0, 238, 121, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[6, 3737, 4085, 4077, 3, 0, 3, 0, 0, 2, 4093, 17, 4048, 1, 283, 0, 0, 0, 4089, 43, 0, 0, 18, 7, 0, 0, 17, 4090, 0, 46, 4085, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 4095, 0, 1, 4, 4020, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 20, 2, 56, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3980, 0, 100, 0, 0, 0, 0, 31, 60, 16, 3877, 836, 58, 6, 0, 10, 0, 4077, 0, 0, 0, 0, 0, 4017, 0, 4, 0, 0, 0, 0, 0, 0, 118, 3711, 0, 12, 2, 0, 0, 0, 0, 0, 16, 4048, 0, 0, 0, 412, 47, 0, 3988, 0, 0, 0, 0, 0, 1, 434, 1, 2898, 8, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4884, 4087, 4021, 3843, 4064, 4447, 3342] avg=4096.0000 max=4884 max_rank=1 min=3342 min_rank=7 imbalance_ratio=1.192383 max_load_reduction=27884 max_load_reduction_pct=0.850952 moved_to_echo=15696 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[10, 3927, 0, 105, 0, 0, 0, 0, 0, 0, 0, 3869, 887, 59, 6, 0, 3, 16, 4093, 0, 0, 0, 0, 0, 4067, 8, 0, 0, 0, 0, 0, 0, 1, 86, 4061, 0, 12, 3, 0, 0, 8, 0, 6, 0, 4066, 0, 0, 0, 0, 17, 2, 4030, 0, 0, 0, 0, 36, 0, 304, 6, 3073, 7, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4821, 4112, 4075, 4163, 4080, 4049, 3426] avg=4096.0000 max=4821 max_rank=1 min=3426 min_rank=7 imbalance_ratio=1.177002 max_load_reduction=27947 max_load_reduction_pct=0.852875 moved_to_echo=15718 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3981, 0, 106, 0, 0, 0, 0, 0, 0, 0, 3801, 893, 59, 6, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 31, 12, 3, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 39, 0, 0, 4035, 0, 0, 0, 0, 0, 238, 121, 0, 3095, 7, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4759, 4093, 4078, 4163, 4053, 4074, 3461] avg=4096.0000 max=4759 max_rank=1 min=3461 min_rank=7 imbalance_ratio=1.161865 max_load_reduction=28009 max_load_reduction_pct=0.854767 moved_to_echo=15751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[6, 3737, 0, 106, 0, 0, 0, 0, 3, 0, 3, 0, 889, 59, 6, 0, 0, 2, 4090, 17, 0, 0, 0, 0, 4030, 1, 283, 0, 0, 0, 0, 0, 0, 0, 4089, 43, 12, 3, 0, 0, 0, 0, 18, 7, 4078, 0, 0, 0, 0, 0, 17, 4031, 0, 0, 0, 0, 0, 46, 4085, 18, 3082, 7, 0, 0] rerouted_tokens_per_ep_rank=[3849, 960, 4109, 4314, 4147, 4103, 4048, 7238] avg=4096.0000 max=7238 max_rank=7 min=960 min_rank=1 imbalance_ratio=1.767090 max_load_reduction=25530 max_load_reduction_pct=0.779114 moved_to_echo=19536 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4075, 0, 97, 0, 0, 0, 0, 0, 0, 49, 4055, 890, 61, 8, 0, 1, 0, 4091, 4, 0, 0, 0, 0, 4051, 0, 1, 0, 0, 0, 0, 0, 20, 0, 4049, 40, 18, 5, 0, 0, 29, 1, 3, 0, 4080, 0, 0, 0, 0, 17, 0, 4034, 0, 0, 0, 0, 0, 8, 0, 0, 3074, 7, 0, 0] rerouted_tokens_per_ep_rank=[4172, 5063, 4096, 4052, 4132, 4113, 4051, 3089] avg=4096.0000 max=5063 max_rank=1 min=3089 min_rank=7 imbalance_ratio=1.236084 max_load_reduction=27705 max_load_reduction_pct=0.845490 moved_to_echo=15385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4034, 0, 106, 0, 0, 0, 0, 4, 5, 4, 3819, 892, 59, 6, 0, 0, 1, 4083, 1, 0, 0, 0, 0, 4073, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4063, 3, 12, 2, 0, 0, 97, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 2, 254, 32, 0, 3093, 7, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4789, 4085, 4074, 4080, 4181, 4031, 3388] avg=4096.0000 max=4789 max_rank=1 min=3388 min_rank=7 imbalance_ratio=1.169189 max_load_reduction=27979 max_load_reduction_pct=0.853851 moved_to_echo=15680 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 106, 0, 0, 0, 0, 0, 1, 4, 4020, 893, 59, 6, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 12, 3, 0, 0, 0, 0, 0, 2, 4085, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 20, 2, 56, 0, 3096, 8, 0, 0] rerouted_tokens_per_ep_rank=[4202, 4983, 4093, 4075, 4110, 4087, 4036, 3182] avg=4096.0000 max=4983 max_rank=1 min=3182 min_rank=7 imbalance_ratio=1.216553 max_load_reduction=27785 max_load_reduction_pct=0.847931 moved_to_echo=15415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 101, 0, 0, 0, 0, 0, 0, 0, 1591, 853, 59, 6, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4078, 0, 0, 1, 0, 0, 0, 0, 6, 14, 4056, 39, 12, 3, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 1, 1, 4030, 0, 0, 0, 0, 3, 0, 2675, 0, 2956, 8, 0, 0] rerouted_tokens_per_ep_rank=[4196, 2509, 4093, 4079, 4130, 4087, 4032, 5642] avg=4096.0000 max=5642 max_rank=7 min=2509 min_rank=1 imbalance_ratio=1.377441 max_load_reduction=27126 max_load_reduction_pct=0.827820 moved_to_echo=17891 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 25088, 7040, 512, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 640, 7808, 128, 24448, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 128, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32000, 0, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 128, 128, 32256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 32256, 256, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[51, 77, 0, 0, 4054, 4, 0, 0, 0, 15, 4036, 4020, 0, 1559, 0, 1, 3007, 5, 2, 0, 4048, 24, 0, 0, 89, 35, 37, 4020, 4093, 0, 7, 3584] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 4095, 0, 47, 0, 83, 0, 4069, 4090, 0, 2272, 0, 9, 3847, 1, 0, 0, 4096, 1, 2, 25, 4053, 0, 1, 0, 1978, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 4096, 0, 6, 2, 0, 101, 4055, 3447, 0, 2327, 45, 0, 726, 0, 9, 0, 4086, 0, 19, 0, 195, 0, 0, 4094, 4078, 0, 1395, 4077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4, 0, 2, 0, 4076, 6, 0, 0, 39, 47, 3704, 3997, 3, 0, 0, 0, 3293, 0, 0, 3, 4067, 0, 0, 0, 3128, 0, 0, 1004, 4096, 0, 1204, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[19, 0, 0, 10, 4096, 0, 23, 6, 80, 11, 4037, 4060, 0, 2762, 0, 3, 89, 0, 0, 0, 4078, 0, 25, 0, 1163, 0, 1, 4010, 4055, 2, 143, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4066, 10, 558, 1, 0, 285, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 3191, 4096, 0, 84, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[20, 1, 0, 0, 4083, 1, 0, 8, 75, 0, 4092, 4054, 0, 2029, 0, 0, 2785, 0, 0, 11, 4096, 0, 0, 0, 3118, 0, 12, 150, 4096, 0, 156, 3981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 39, 4096, 4096, 0, 1357, 3, 0, 3530, 0, 2, 2, 4093, 0, 0, 1, 391, 0, 0, 2673, 4050, 0, 273, 4067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[51, 77, 0, 0, 3584, 60, 0, 0, 4052, 4, 0, 0, 0, 0, 0, 0, 0, 15, 58, 4020, 0, 0, 0, 0, 0, 1559, 0, 1, 2486, 0, 0, 0, 3007, 5, 2, 0, 1432, 565, 139, 1, 4048, 24, 0, 0, 1, 0, 0, 0, 89, 35, 37, 3455, 0, 0, 0, 0, 3954, 0, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3772, 4056, 4093, 4046, 5151, 4073, 3616, 3961] avg=4096.0000 max=5151 max_rank=4 min=3616 min_rank=6 imbalance_ratio=1.257568 max_load_reduction=27617 max_load_reduction_pct=0.842804 moved_to_echo=16801 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 4093, 60, 0, 0, 4093, 0, 47, 0, 0, 0, 0, 0, 83, 0, 59, 4090, 0, 0, 0, 0, 0, 2272, 0, 9, 2506, 0, 0, 0, 3847, 1, 0, 0, 1444, 0, 67, 1, 4096, 1, 2, 25, 1, 0, 0, 0, 4053, 0, 1, 0, 0, 0, 0, 0, 1911, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4140, 4232, 4787, 5360, 4125, 4054, 1911] avg=4096.0000 max=5360 max_rank=4 min=1911 min_rank=7 imbalance_ratio=1.308594 max_load_reduction=27408 max_load_reduction_pct=0.836426 moved_to_echo=15450 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 0, 10, 4095, 60, 0, 0, 4094, 0, 23, 6, 0, 0, 0, 0, 80, 11, 57, 4060, 0, 0, 0, 0, 0, 2762, 0, 3, 2487, 0, 0, 0, 89, 0, 0, 0, 1433, 563, 137, 1, 4078, 0, 25, 0, 1, 0, 0, 0, 1163, 0, 1, 3447, 0, 0, 0, 0, 3918, 2, 143, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4184, 4123, 4208, 5252, 2223, 4104, 4611, 4063] avg=4096.0000 max=5252 max_rank=3 min=2223 min_rank=4 imbalance_ratio=1.282227 max_load_reduction=27516 max_load_reduction_pct=0.839722 moved_to_echo=15001 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 61, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 4066, 0, 0, 0, 0, 10, 558, 1, 0, 2523, 0, 0, 0, 285, 0, 0, 0, 1454, 448, 139, 1, 4096, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 2743, 0, 0, 0, 0, 3957, 0, 84, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4094, 4124, 3092, 2327, 4097, 6838, 4041] avg=4096.0000 max=6838 max_rank=6 min=2327 min_rank=4 imbalance_ratio=1.669434 max_load_reduction=25930 max_load_reduction_pct=0.791321 moved_to_echo=17303 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 2, 0, 4095, 55, 0, 0, 4074, 6, 0, 0, 0, 0, 0, 0, 39, 47, 53, 3997, 0, 0, 0, 0, 3, 0, 0, 0, 2282, 0, 0, 0, 3293, 0, 0, 3, 1314, 141, 139, 1, 4067, 0, 0, 0, 1, 0, 0, 0, 3128, 0, 0, 863, 0, 0, 0, 0, 3957, 0, 1204, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4080, 4136, 2285, 4891, 4068, 3991, 5161] avg=4096.0000 max=5161 max_rank=7 min=2285 min_rank=3 imbalance_ratio=1.260010 max_load_reduction=27607 max_load_reduction_pct=0.842499 moved_to_echo=18111 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 4077, 64, 0, 0, 4084, 0, 6, 2, 0, 0, 0, 0, 0, 101, 47, 3447, 0, 0, 0, 0, 0, 2327, 45, 0, 2501, 0, 0, 0, 726, 0, 9, 0, 1443, 579, 141, 8, 4086, 0, 19, 0, 4, 0, 0, 0, 195, 0, 0, 3515, 0, 0, 0, 0, 3937, 0, 1395, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4092, 3595, 4873, 2906, 4109, 3710, 5332] avg=4096.0000 max=5332 max_rank=7 min=2906 min_rank=4 imbalance_ratio=1.301758 max_load_reduction=27436 max_load_reduction_pct=0.837280 moved_to_echo=16057 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[20, 1, 0, 0, 3981, 61, 0, 0, 4081, 1, 0, 8, 0, 0, 0, 0, 75, 0, 58, 4054, 0, 0, 0, 0, 0, 2029, 0, 0, 2521, 0, 0, 0, 2785, 0, 0, 11, 1452, 21, 139, 1, 4096, 0, 0, 0, 1, 0, 0, 0, 3118, 0, 12, 129, 0, 0, 0, 0, 3957, 0, 156, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4090, 4187, 4550, 4409, 4097, 3259, 4113] avg=4096.0000 max=4550 max_rank=3 min=3259 min_rank=6 imbalance_ratio=1.110840 max_load_reduction=28218 max_load_reduction_pct=0.861145 moved_to_echo=15878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4067, 61, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 39, 58, 4096, 0, 0, 0, 0, 0, 1357, 3, 0, 2523, 0, 0, 0, 3530, 0, 2, 2, 1454, 375, 137, 1, 4093, 0, 0, 1, 1, 0, 0, 0, 391, 0, 0, 2298, 0, 0, 0, 0, 3913, 0, 273, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4093, 4193, 3883, 5501, 4095, 2689, 4186] avg=4096.0000 max=5501 max_rank=4 min=2689 min_rank=6 imbalance_ratio=1.343018 max_load_reduction=27267 max_load_reduction_pct=0.832123 moved_to_echo=16471 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[17664, 128, 128, 128, 11520, 2816, 1152, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 12928, 128, 128, 19840, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[29568, 128, 3328, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16256, 128, 128, 16512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 256, 512, 31872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32128, 512, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33664 slack_slots=1792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4, 3, 0, 2, 586, 0, 24, 0, 4091, 4073, 3994, 6, 0, 2263, 0, 28, 232, 15, 4068, 28, 4081, 0, 41, 0, 9, 1131, 0, 3884, 4079, 81, 14, 31] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 0, 0, 0, 4096, 4022, 4084, 0, 0, 0, 0, 0, 0, 0, 4095, 1450, 4057, 2, 24, 0, 0, 3422, 18, 3390, 4096, 10, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 5, 0, 2, 1, 4096, 4096, 4048, 1, 0, 3917, 0, 0, 103, 0, 4096, 0, 4071, 0, 0, 0, 1, 114, 0, 4096, 4089, 28, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 19, 0, 0, 0, 4092, 4096, 4095, 0, 0, 3764, 1, 0, 768, 0, 4094, 11, 4096, 3, 0, 0, 0, 169, 3, 3461, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 7, 0, 14, 4, 0, 0, 0, 4096, 4044, 4083, 0, 8, 4096, 6, 1, 32, 3, 4096, 0, 4082, 0, 0, 0, 0, 0, 0, 4091, 4083, 0, 0, 22] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[5, 4, 18, 2, 103, 6, 19, 0, 4093, 4051, 4096, 0, 0, 3570, 0, 0, 0, 7, 4070, 0, 3982, 0, 0, 0, 0, 201, 591, 3856, 4077, 0, 17, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[2, 15, 0, 0, 0, 0, 5, 6, 3785, 2130, 4092, 0, 8, 2393, 2, 3, 0, 0, 4087, 33, 4090, 1, 0, 1, 2, 3546, 0, 4034, 4061, 0, 469, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 154, 10, 0, 7, 0, 2, 0, 4096, 4078, 4036, 0, 0, 3515, 0, 0, 0, 0, 4095, 151, 4096, 0, 0, 0, 0, 0, 341, 4091, 4095, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4084, 0, 0, 0, 1, 0, 0, 0, 3821, 186, 0, 0, 275, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 346, 55, 0, 0, 0, 3749, 1450, 0, 0, 0, 0, 4057, 2, 24, 0, 17, 0, 0, 0, 0, 3422, 18, 2548, 0, 0, 0, 0, 4024, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4008, 4297, 1057, 5199, 4100, 5988, 4034] avg=4096.0000 max=5988 max_rank=6 min=1057 min_rank=3 imbalance_ratio=1.461914 max_load_reduction=26780 max_load_reduction_pct=0.817261 moved_to_echo=19321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4, 3, 0, 2, 3993, 0, 0, 0, 586, 0, 24, 0, 3817, 213, 1, 0, 274, 4073, 0, 6, 0, 0, 0, 0, 0, 2263, 0, 28, 751, 344, 55, 0, 232, 15, 3724, 28, 0, 0, 0, 0, 4081, 0, 41, 0, 17, 0, 0, 0, 9, 1131, 0, 2920, 0, 0, 0, 0, 4007, 81, 14, 31, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4002, 4641, 4353, 3441, 3999, 4139, 4060, 4133] avg=4096.0000 max=4641 max_rank=1 min=3441 min_rank=3 imbalance_ratio=1.133057 max_load_reduction=28127 max_load_reduction_pct=0.858368 moved_to_echo=16331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[5, 4, 18, 2, 4096, 0, 0, 0, 103, 6, 19, 0, 3819, 212, 0, 0, 274, 4051, 0, 0, 0, 0, 0, 0, 0, 3570, 0, 0, 746, 344, 55, 0, 0, 7, 3726, 0, 0, 0, 0, 0, 3982, 0, 0, 0, 17, 0, 0, 0, 0, 201, 591, 2898, 0, 0, 0, 0, 4005, 0, 17, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4159, 4325, 4715, 3733, 3999, 3690, 4022] avg=4096.0000 max=4715 max_rank=3 min=3690 min_rank=6 imbalance_ratio=1.151123 max_load_reduction=28053 max_load_reduction_pct=0.856110 moved_to_echo=15444 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4047, 0, 0, 0, 5, 0, 2, 1, 3821, 225, 1, 0, 275, 4096, 0, 1, 0, 0, 0, 0, 0, 3917, 0, 0, 792, 346, 55, 0, 103, 0, 3750, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 17, 0, 0, 0, 1, 114, 0, 3079, 0, 0, 0, 0, 4017, 28, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4055, 4372, 5110, 3853, 4088, 3194, 4047] avg=4096.0000 max=5110 max_rank=3 min=3194 min_rank=6 imbalance_ratio=1.247559 max_load_reduction=27658 max_load_reduction_pct=0.844055 moved_to_echo=15182 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 15, 0, 0, 4092, 0, 0, 0, 0, 0, 5, 6, 3531, 222, 0, 0, 254, 2130, 0, 0, 0, 0, 0, 0, 8, 2393, 2, 3, 780, 345, 55, 0, 0, 0, 3742, 33, 0, 0, 0, 0, 4090, 1, 0, 1, 17, 0, 0, 0, 2, 3546, 0, 3032, 0, 0, 0, 0, 3989, 0, 469, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 3764, 2384, 3586, 3775, 4109, 6580, 4461] avg=4096.0000 max=6580 max_rank=6 min=2384 min_rank=2 imbalance_ratio=1.606445 max_load_reduction=26188 max_load_reduction_pct=0.799194 moved_to_echo=18925 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 14, 4083, 0, 0, 0, 4, 0, 0, 0, 3826, 228, 0, 0, 270, 4044, 0, 0, 0, 0, 0, 0, 8, 4096, 6, 1, 797, 349, 60, 0, 32, 3, 3747, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 3066, 0, 0, 0, 0, 4001, 0, 0, 22, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4058, 4314, 5317, 3782, 4104, 3066, 4023] avg=4096.0000 max=5317 max_rank=3 min=3066 min_rank=6 imbalance_ratio=1.298096 max_load_reduction=27451 max_load_reduction_pct=0.837738 moved_to_echo=14975 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 19, 0, 0, 0, 3818, 190, 0, 0, 274, 4096, 0, 0, 0, 0, 0, 0, 0, 3764, 1, 0, 669, 346, 55, 0, 768, 0, 3748, 11, 0, 0, 0, 0, 4096, 3, 0, 0, 17, 0, 0, 0, 0, 169, 3, 2602, 0, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4027, 4370, 4835, 4527, 4116, 2774, 4024] avg=4096.0000 max=4835 max_rank=3 min=2774 min_rank=6 imbalance_ratio=1.180420 max_load_reduction=27933 max_load_reduction_pct=0.852448 moved_to_echo=15441 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 154, 10, 0, 4035, 0, 0, 0, 7, 0, 2, 0, 3821, 225, 1, 0, 275, 4078, 0, 0, 0, 0, 0, 0, 0, 3515, 0, 0, 791, 346, 55, 0, 0, 0, 3749, 151, 0, 0, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 0, 341, 3075, 0, 0, 0, 0, 4023, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4199, 4056, 4353, 4707, 3900, 4113, 3416, 4024] avg=4096.0000 max=4707 max_rank=3 min=3416 min_rank=6 imbalance_ratio=1.149170 max_load_reduction=28061 max_load_reduction_pct=0.856354 moved_to_echo=15453 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 8704, 1024, 23296, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 23552, 128, 128, 6016, 2816, 512, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[2176, 30592, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 128, 29952, 1792, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32128, 128, 512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[768, 128, 128, 128, 30336, 1792, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 4088, 0, 0, 2, 0, 4094, 2, 4087, 9, 0, 32, 1, 0, 2, 4094, 3927, 155, 2, 0, 4094, 1, 3, 0, 4063, 2, 0, 15, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4086, 0, 4065, 0, 67, 1, 1, 4081, 10, 4029, 15, 0, 0, 0, 0, 9, 4096, 4083, 36, 0, 0, 178, 0, 0, 10, 4096, 0, 3904, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4047, 0, 3987, 0, 0, 0, 0, 4048, 0, 3964, 19, 32, 0, 25, 0, 0, 4065, 4082, 150, 0, 0, 4069, 9, 0, 4, 4076, 20, 0, 48, 0, 0, 123] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4012, 7, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 467, 0, 0, 0, 4096, 4045, 135, 0, 57, 3655, 0, 0, 0, 3990, 0, 13, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4032, 0, 4096, 0, 0, 0, 0, 4091, 0, 4062, 0, 0, 182, 0, 0, 0, 4094, 4096, 9, 26, 0, 3886, 15, 0, 0, 4096, 0, 77, 0, 0, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4052, 0, 4091, 0, 0, 0, 0, 4095, 0, 4059, 0, 1, 137, 0, 41, 6, 4094, 4080, 0, 29, 0, 3973, 8, 0, 0, 4094, 0, 4, 3, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[3910, 0, 4096, 0, 0, 0, 0, 4093, 0, 4095, 0, 0, 1, 0, 0, 0, 4082, 4096, 1195, 3, 14, 2489, 0, 0, 0, 4096, 0, 375, 0, 0, 223, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4092, 3, 4094, 0, 2, 0, 32, 4085, 0, 4096, 0, 0, 166, 14, 0, 1, 4090, 4096, 0, 0, 23, 3258, 2, 3, 0, 4034, 0, 663, 1, 0, 13, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 56, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 0, 0, 0, 0, 2, 4087, 9, 0, 20, 2, 0, 0, 32, 1, 0, 2, 3923, 48, 0, 0, 0, 3750, 155, 2, 0, 0, 0, 0, 0, 4094, 1, 3, 605, 157, 109, 0, 0, 3458, 2, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4046, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4094, 4120, 4006, 3907, 4969, 3460, 4061] avg=4096.0000 max=4969 max_rank=5 min=3460 min_rank=6 imbalance_ratio=1.213135 max_load_reduction=27799 max_load_reduction_pct=0.848358 moved_to_echo=16397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4012, 7, 55, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 21, 2, 0, 0, 467, 0, 0, 0, 3931, 48, 0, 0, 0, 3862, 135, 0, 0, 0, 0, 0, 57, 3655, 0, 0, 594, 162, 110, 0, 0, 3396, 0, 13, 0, 0, 0, 0, 0, 0, 3, 0, 4048, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4094, 4119, 4446, 3997, 4578, 3409, 4051] avg=4096.0000 max=4578 max_rank=5 min=3409 min_rank=6 imbalance_ratio=1.117676 max_load_reduction=28190 max_load_reduction_pct=0.860291 moved_to_echo=16035 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 55, 0, 0, 0, 0, 0, 67, 1, 1, 4079, 0, 0, 0, 0, 10, 4029, 15, 0, 21, 2, 0, 0, 0, 0, 0, 9, 3901, 47, 0, 0, 0, 3899, 36, 0, 0, 0, 0, 0, 0, 178, 0, 0, 610, 163, 109, 0, 10, 3486, 0, 3904, 0, 0, 0, 0, 0, 1, 0, 0, 4049, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4148, 4077, 3957, 3935, 1060, 7400, 4050] avg=4096.0000 max=7400 max_rank=6 min=1060 min_rank=5 imbalance_ratio=1.806641 max_load_reduction=25368 max_load_reduction_pct=0.774170 moved_to_echo=16445 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4047, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 3964, 19, 32, 21, 2, 0, 0, 0, 25, 0, 0, 3826, 47, 0, 0, 0, 3898, 150, 0, 0, 0, 0, 0, 0, 4069, 9, 0, 607, 163, 107, 0, 4, 3469, 20, 0, 0, 0, 0, 0, 48, 0, 0, 123, 4018, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4046, 4038, 3898, 4048, 4955, 3493, 4189] avg=4096.0000 max=4955 max_rank=5 min=3493 min_rank=6 imbalance_ratio=1.209717 max_load_reduction=27813 max_load_reduction_pct=0.848785 moved_to_echo=16685 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4032, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4062, 0, 0, 21, 2, 0, 0, 182, 0, 0, 0, 3931, 47, 0, 0, 0, 3911, 9, 26, 0, 0, 0, 0, 0, 3886, 15, 0, 610, 164, 110, 0, 0, 3486, 0, 77, 0, 0, 0, 0, 0, 0, 6, 0, 4047, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4089, 4085, 4160, 3946, 4785, 3563, 4053] avg=4096.0000 max=4785 max_rank=5 min=3563 min_rank=6 imbalance_ratio=1.168213 max_load_reduction=27983 max_load_reduction_pct=0.853973 moved_to_echo=16347 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4092, 3, 47, 0, 0, 0, 0, 0, 2, 0, 32, 4079, 0, 0, 0, 0, 0, 4096, 0, 0, 26, 6, 0, 0, 166, 14, 0, 1, 3935, 48, 0, 0, 0, 3901, 0, 0, 0, 0, 0, 0, 23, 3258, 2, 3, 604, 169, 112, 0, 0, 3430, 0, 663, 0, 0, 0, 0, 1, 0, 13, 0, 4042, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4113, 4128, 4164, 3901, 4171, 4093, 4056] avg=4096.0000 max=4171 max_rank=5 min=3901 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16221 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3910, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4095, 0, 0, 21, 2, 0, 0, 1, 0, 0, 0, 3931, 48, 0, 0, 0, 3911, 1195, 3, 0, 0, 0, 0, 14, 2489, 0, 0, 610, 164, 110, 0, 0, 3486, 0, 375, 0, 0, 0, 0, 0, 0, 223, 0, 4034, 0, 0, 0] rerouted_tokens_per_ep_rank=[3965, 4091, 4118, 3980, 5109, 3387, 3861, 4257] avg=4096.0000 max=5109 max_rank=4 min=3387 min_rank=5 imbalance_ratio=1.247314 max_load_reduction=27659 max_load_reduction_pct=0.844086 moved_to_echo=16614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4052, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4059, 0, 1, 21, 2, 0, 0, 137, 0, 41, 6, 3926, 47, 0, 0, 0, 3896, 0, 29, 0, 0, 0, 0, 0, 3973, 8, 0, 609, 163, 110, 0, 0, 3485, 0, 4, 0, 0, 0, 0, 3, 0, 0, 1, 4047, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4093, 4083, 4157, 3925, 4863, 3489, 4051] avg=4096.0000 max=4863 max_rank=5 min=3489 min_rank=6 imbalance_ratio=1.187256 max_load_reduction=27905 max_load_reduction_pct=0.851593 moved_to_echo=16328 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 25728, 128, 128, 4864, 1408, 896, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 27776, 128, 5120, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32384, 128, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 128, 128, 128, 31360, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 256, 128, 32384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 31104, 1792, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32512, 128, 128, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 3952, 2, 4081, 0, 3991, 17, 83, 14, 4027, 0, 4081, 0, 4093, 70, 0, 7, 0, 115, 0, 19, 0, 25, 14, 24, 0, 0, 4081, 0, 4072, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 37, 0, 4096, 0, 3965, 0, 0, 0, 3991, 0, 4095, 0, 3952, 0, 0, 0, 0, 4096, 0, 354, 0, 0, 0, 0, 0, 0, 4050, 36, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 1, 2055, 0, 4094, 45, 4092, 0, 2, 1, 3746, 24, 4007, 0, 1892, 0, 3, 0, 4, 4096, 0, 429, 0, 0, 0, 1, 4, 46, 4055, 79, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 8, 3554, 18, 4025, 133, 4060, 1, 0, 0, 4088, 0, 4085, 11, 197, 0, 9, 3, 0, 3756, 0, 0, 0, 106, 0, 4, 0, 25, 4067, 540, 4078, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 3284, 0, 4051, 1, 4087, 0, 1, 0, 4095, 0, 4096, 2, 1054, 0, 0, 2, 1, 3805, 0, 0, 1, 83, 0, 0, 2, 1, 4094, 0, 4091, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[22, 8, 3921, 0, 4083, 0, 4095, 0, 0, 0, 4096, 23, 4052, 0, 1202, 0, 14, 0, 0, 3068, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4085, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 131, 0, 4096, 9, 3997, 0, 0, 0, 4090, 0, 4089, 0, 4096, 0, 0, 0, 0, 3939, 14, 80, 36, 0, 0, 0, 0, 0, 4093, 0, 4096, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4005, 0, 4096, 0, 4088, 0, 6, 0, 4066, 5, 4090, 0, 1168, 6, 0, 0, 0, 3032, 3, 0, 7, 0, 0, 0, 6, 0, 4096, 0, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3952, 2, 1361, 111, 0, 0, 24, 0, 3991, 17, 0, 0, 0, 0, 83, 14, 4027, 0, 51, 0, 0, 0, 1883, 0, 4093, 70, 0, 0, 0, 0, 0, 7, 0, 115, 837, 15, 0, 0, 0, 19, 0, 25, 3946, 8, 0, 0, 14, 24, 0, 0, 4064, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5426, 4032, 4175, 6046, 974, 3998, 4102, 4015] avg=4096.0000 max=6046 max_rank=3 min=974 min_rank=4 imbalance_ratio=1.476074 max_load_reduction=26722 max_load_reduction_pct=0.815491 moved_to_echo=13089 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 37, 0, 1366, 111, 0, 0, 25, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 50, 0, 0, 0, 1889, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 840, 15, 0, 0, 0, 354, 0, 0, 3960, 9, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 3985, 36, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1514, 3990, 4041, 5841, 4951, 4323, 4087, 4021] avg=4096.0000 max=5841 max_rank=3 min=1514 min_rank=0 imbalance_ratio=1.426025 max_load_reduction=26927 max_load_reduction_pct=0.821747 moved_to_echo=17382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2055, 0, 1337, 111, 0, 0, 25, 45, 4092, 0, 0, 0, 0, 0, 2, 1, 3746, 24, 50, 0, 0, 0, 1848, 0, 1892, 0, 0, 0, 0, 0, 3, 0, 4, 4096, 822, 15, 0, 0, 0, 429, 0, 0, 3958, 8, 0, 0, 0, 1, 4, 46, 4084, 0, 0, 0, 3990, 79, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3504, 4162, 3823, 3740, 4940, 4395, 4135, 4069] avg=4096.0000 max=4940 max_rank=4 min=3504 min_rank=0 imbalance_ratio=1.206055 max_load_reduction=27828 max_load_reduction_pct=0.849243 moved_to_echo=17539 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 3554, 18, 1363, 109, 0, 0, 24, 133, 4060, 1, 0, 0, 0, 0, 0, 0, 4088, 0, 51, 0, 0, 0, 1884, 11, 197, 0, 0, 0, 0, 0, 9, 3, 0, 3756, 838, 15, 0, 0, 0, 0, 0, 106, 3892, 8, 0, 0, 0, 4, 0, 25, 4070, 0, 0, 0, 4001, 540, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5052, 4218, 4139, 2092, 4621, 4006, 4099, 4541] avg=4096.0000 max=5052 max_rank=0 min=2092 min_rank=3 imbalance_ratio=1.233398 max_load_reduction=27716 max_load_reduction_pct=0.845825 moved_to_echo=17267 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3284, 0, 1366, 110, 0, 0, 24, 1, 4087, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 51, 0, 0, 0, 1890, 2, 1054, 0, 0, 0, 0, 0, 0, 2, 1, 3805, 840, 15, 0, 0, 0, 0, 1, 83, 3917, 8, 0, 0, 0, 0, 2, 1, 4083, 0, 0, 0, 4028, 0, 0, 15, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4762, 4112, 4147, 2946, 4663, 4009, 4086, 4043] avg=4096.0000 max=4762 max_rank=0 min=2946 min_rank=3 imbalance_ratio=1.162598 max_load_reduction=28006 max_load_reduction_pct=0.854675 moved_to_echo=16801 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[22, 8, 3921, 0, 1357, 116, 0, 0, 15, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 23, 55, 0, 0, 0, 1858, 0, 1202, 0, 0, 0, 0, 0, 14, 0, 0, 3068, 837, 21, 0, 0, 0, 0, 0, 0, 3952, 9, 0, 0, 0, 0, 0, 3, 4076, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5424, 4110, 4174, 3060, 3940, 3961, 4079, 4020] avg=4096.0000 max=5424 max_rank=0 min=3060 min_rank=3 imbalance_ratio=1.324219 max_load_reduction=27344 max_load_reduction_pct=0.834473 moved_to_echo=16000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 131, 0, 1364, 111, 0, 0, 25, 9, 3997, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 51, 0, 0, 0, 1886, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 839, 15, 0, 0, 14, 80, 36, 0, 3960, 8, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4027, 0, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1606, 4031, 4141, 5982, 4793, 4098, 4088, 4029] avg=4096.0000 max=5982 max_rank=3 min=1606 min_rank=0 imbalance_ratio=1.460449 max_load_reduction=26786 max_load_reduction_pct=0.817444 moved_to_echo=17008 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4005, 0, 1364, 111, 0, 0, 25, 0, 4088, 0, 0, 0, 0, 0, 6, 0, 4066, 5, 51, 0, 0, 0, 1887, 0, 1168, 6, 0, 0, 0, 0, 0, 0, 0, 3032, 839, 15, 0, 0, 3, 0, 7, 0, 3960, 8, 0, 0, 0, 0, 6, 0, 4086, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5480, 4113, 4128, 3061, 3886, 3978, 4092, 4030] avg=4096.0000 max=5480 max_rank=0 min=3061 min_rank=3 imbalance_ratio=1.337891 max_load_reduction=27288 max_load_reduction_pct=0.832764 moved_to_echo=15986 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 20992, 128, 10880, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15104, 128, 17664, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 25984, 6784, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32128, 768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32256, 128, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 256, 32384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 896, 128, 256, 31616, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4057, 0, 0, 123, 0, 4085, 4084, 33, 6, 4075, 3973, 0, 12, 4063, 0, 0, 8, 20, 4082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 33, 38, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 76, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4012, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4089, 0, 0, 7, 2, 4091, 3704, 2, 2, 4092, 4093, 0, 0, 2763, 691, 2, 2, 0, 4094, 4, 0, 0, 1583, 0, 98, 0, 0, 11, 3427, 4, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 0, 3996, 4087, 0, 9, 4096, 4011, 17, 0, 3995, 0, 6, 0, 199, 4075, 0, 96, 0, 0, 0, 0, 1, 0, 15, 4095, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[3999, 0, 2, 0, 0, 4075, 4082, 14, 1, 4083, 4094, 0, 11, 3861, 86, 6, 0, 236, 4086, 0, 1, 0, 16, 0, 13, 0, 0, 19, 4046, 0, 27, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4096, 5, 0, 0, 0, 4083, 4096, 0, 0, 4064, 4091, 0, 0, 4085, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 32, 2, 0, 0, 4092, 7, 19, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 7, 0, 4096, 4096, 0, 0, 4088, 4095, 0, 0, 252, 0, 0, 0, 0, 4096, 0, 0, 0, 3786, 0, 55, 0, 0, 0, 4096, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4095, 1, 0, 0, 3, 4096, 4096, 0, 0, 4094, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 76, 39, 4096, 0, 0, 0, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 596, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3424, 0, 0, 0, 0, 0, 0, 0, 4072, 2, 0, 0, 4012, 0, 0, 0, 50, 35, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4211, 4120, 4688, 4043, 3436, 4074, 4097] avg=4096.0000 max=4688 max_rank=3 min=3436 min_rank=5 imbalance_ratio=1.144531 max_load_reduction=28080 max_load_reduction_pct=0.856934 moved_to_echo=15650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4057, 0, 0, 123, 3, 0, 0, 0, 0, 39, 4084, 33, 0, 0, 0, 0, 6, 23, 3973, 0, 0, 0, 0, 0, 12, 4063, 0, 0, 594, 0, 0, 0, 8, 20, 4029, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3415, 0, 0, 0, 0, 0, 0, 0, 4052, 2, 0, 0, 4064, 33, 38, 0, 50, 35, 0, 0] rerouted_tokens_per_ep_rank=[4183, 4156, 4002, 4669, 4069, 3415, 4054, 4220] avg=4096.0000 max=4669 max_rank=3 min=3415 min_rank=5 imbalance_ratio=1.139893 max_load_reduction=28099 max_load_reduction_pct=0.857513 moved_to_echo=15758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 0, 7, 3, 0, 0, 0, 2, 39, 3704, 2, 0, 0, 0, 0, 2, 24, 4093, 0, 0, 0, 0, 0, 0, 2763, 691, 2, 595, 0, 0, 0, 2, 0, 4041, 4, 0, 0, 0, 0, 0, 0, 1583, 0, 3420, 0, 0, 0, 98, 0, 0, 11, 4068, 2, 0, 0, 3427, 4, 7, 0, 50, 35, 0, 0] rerouted_tokens_per_ep_rank=[4099, 3747, 4119, 4051, 4047, 5003, 4179, 3523] avg=4096.0000 max=5003 max_rank=5 min=3523 min_rank=7 imbalance_ratio=1.221436 max_load_reduction=27765 max_load_reduction_pct=0.847321 moved_to_echo=16752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4096, 5, 0, 0, 3, 0, 0, 0, 0, 39, 4096, 0, 0, 0, 0, 0, 0, 23, 4091, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 594, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3413, 0, 0, 0, 32, 2, 0, 0, 4041, 2, 0, 0, 4092, 7, 19, 0, 50, 35, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4135, 4114, 4679, 4041, 3415, 4077, 4203] avg=4096.0000 max=4679 max_rank=3 min=3415 min_rank=5 imbalance_ratio=1.142334 max_load_reduction=28089 max_load_reduction_pct=0.857208 moved_to_echo=15736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3999, 0, 2, 0, 3, 0, 0, 0, 0, 39, 4082, 14, 0, 0, 0, 0, 1, 23, 4094, 0, 0, 0, 0, 0, 11, 3861, 86, 6, 593, 0, 0, 0, 0, 236, 4033, 0, 0, 0, 0, 0, 1, 0, 16, 0, 3406, 0, 0, 0, 13, 0, 0, 19, 4060, 2, 0, 0, 4046, 0, 27, 10, 50, 35, 0, 0] rerouted_tokens_per_ep_rank=[4004, 4135, 4118, 4557, 4269, 3423, 4094, 4168] avg=4096.0000 max=4557 max_rank=3 min=3423 min_rank=5 imbalance_ratio=1.112549 max_load_reduction=28211 max_load_reduction_pct=0.860931 moved_to_echo=15954 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 4, 0, 0, 0, 0, 24, 4087, 0, 0, 0, 0, 0, 9, 19, 4011, 17, 0, 0, 0, 0, 0, 3995, 0, 6, 583, 0, 0, 0, 0, 199, 4014, 0, 0, 0, 0, 0, 96, 0, 0, 0, 3345, 0, 0, 0, 0, 1, 0, 15, 4077, 4, 0, 0, 4095, 0, 0, 0, 57, 40, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4111, 4056, 4584, 4213, 3441, 4097, 4192] avg=4096.0000 max=4584 max_rank=3 min=3441 min_rank=5 imbalance_ratio=1.119141 max_load_reduction=28184 max_load_reduction_pct=0.860107 moved_to_echo=15943 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 7, 3, 0, 0, 0, 0, 39, 4096, 0, 0, 0, 0, 0, 0, 24, 4095, 0, 0, 0, 0, 0, 0, 252, 0, 0, 596, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 3786, 0, 3424, 0, 0, 0, 55, 0, 0, 0, 4064, 2, 0, 0, 4096, 5, 0, 0, 50, 35, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4135, 4119, 848, 4043, 7210, 4121, 4186] avg=4096.0000 max=7210 max_rank=5 min=848 min_rank=3 imbalance_ratio=1.760254 max_load_reduction=25558 max_load_reduction_pct=0.779968 moved_to_echo=19560 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4095, 1, 0, 0, 3, 0, 0, 0, 3, 39, 4096, 0, 0, 0, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 596, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3424, 0, 0, 0, 0, 0, 0, 0, 4070, 2, 0, 0, 4094, 0, 0, 0, 50, 35, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4138, 4120, 4692, 4043, 3425, 4072, 4179] avg=4096.0000 max=4692 max_rank=3 min=3425 min_rank=5 imbalance_ratio=1.145508 max_load_reduction=28076 max_load_reduction_pct=0.856812 moved_to_echo=15719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 27264, 896, 128, 4864, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 5504, 128, 27392, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 256, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32000, 128, 128, 128, 512, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 32384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 0, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 384, 32384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[5, 104, 0, 0, 3937, 8, 0, 0, 4071, 4008, 0, 0, 4095, 99, 4096, 0, 0, 0, 12, 3, 0, 168, 4089, 0, 0, 199, 201, 4082, 3578, 1, 3, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 3880, 0, 0, 4096, 0, 0, 0, 4096, 4090, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 0, 205, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[44, 3208, 29, 0, 4079, 8, 0, 0, 4086, 4094, 204, 0, 4068, 0, 3811, 0, 9, 25, 12, 24, 14, 1, 4076, 0, 1, 1, 0, 4072, 892, 2, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[5, 4009, 8, 0, 3972, 0, 0, 0, 1100, 4088, 0, 0, 4092, 2995, 4080, 2, 3, 2, 11, 0, 0, 0, 4089, 0, 1, 189, 29, 4083, 0, 0, 10, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 1363, 3, 20, 4075, 0, 0, 22, 4092, 4073, 1, 0, 4096, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 19, 0, 0, 4078, 2733, 0, 0, 66] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 1741, 0, 0, 4093, 0, 0, 0, 4095, 4093, 144, 0, 4084, 29, 3918, 0, 22, 0, 0, 0, 5, 0, 4091, 3, 0, 3, 0, 4094, 2350, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 3712, 0, 0, 4094, 0, 0, 0, 4086, 4096, 0, 0, 4094, 148, 4017, 0, 2, 0, 6, 0, 0, 219, 4089, 107, 0, 0, 0, 4096, 2, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 2711, 0, 0, 4093, 0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 4026, 1, 0, 0, 8, 0, 0, 3, 4093, 0, 3, 0, 1, 4095, 1448, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 104, 0, 0, 1027, 329, 97, 24, 3937, 8, 0, 0, 36, 0, 0, 0, 4071, 332, 0, 0, 0, 0, 0, 0, 0, 99, 3767, 0, 0, 0, 0, 0, 0, 0, 12, 3, 4083, 0, 0, 0, 0, 168, 4029, 0, 0, 0, 0, 0, 0, 199, 201, 3985, 0, 0, 0, 0, 3578, 1, 3, 9, 2649, 12, 0, 0] rerouted_tokens_per_ep_rank=[1586, 3981, 4403, 3866, 4098, 4197, 4385, 6252] avg=4096.0000 max=6252 max_rank=7 min=1586 min_rank=0 imbalance_ratio=1.526367 max_load_reduction=26516 max_load_reduction_pct=0.809204 moved_to_echo=18932 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3880, 0, 0, 1048, 329, 97, 24, 4096, 0, 0, 0, 36, 0, 0, 0, 4096, 339, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 17, 4036, 0, 0, 0, 0, 0, 0, 0, 205, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 2703, 11, 0, 0] rerouted_tokens_per_ep_rank=[5378, 4132, 4435, 3767, 4085, 4053, 4204, 2714] avg=4096.0000 max=5378 max_rank=0 min=2714 min_rank=7 imbalance_ratio=1.312988 max_load_reduction=27390 max_load_reduction_pct=0.835876 moved_to_echo=15056 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[44, 3208, 29, 0, 1049, 306, 97, 24, 4079, 8, 0, 0, 36, 0, 0, 0, 4086, 340, 204, 0, 0, 0, 0, 0, 0, 0, 3505, 0, 0, 0, 0, 0, 9, 25, 12, 24, 4056, 0, 0, 0, 14, 1, 4016, 0, 0, 0, 0, 0, 1, 1, 0, 3975, 0, 0, 0, 0, 892, 2, 0, 8, 2705, 12, 0, 0] rerouted_tokens_per_ep_rank=[4757, 4123, 4630, 3505, 4126, 4031, 3977, 3619] avg=4096.0000 max=4757 max_rank=0 min=3505 min_rank=3 imbalance_ratio=1.161377 max_load_reduction=28011 max_load_reduction_pct=0.854828 moved_to_echo=15753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 4009, 8, 0, 1048, 328, 97, 24, 3972, 0, 0, 0, 36, 0, 0, 0, 1100, 339, 0, 0, 0, 0, 0, 0, 0, 2995, 3752, 2, 0, 0, 0, 0, 3, 2, 11, 0, 4081, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 1, 189, 29, 3986, 0, 0, 0, 0, 0, 0, 10, 0, 2701, 11, 0, 0] rerouted_tokens_per_ep_rank=[5519, 4008, 1439, 6749, 4097, 4029, 4205, 2722] avg=4096.0000 max=6749 max_rank=3 min=1439 min_rank=2 imbalance_ratio=1.647705 max_load_reduction=26019 max_load_reduction_pct=0.794037 moved_to_echo=15053 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1741, 0, 0, 1049, 315, 97, 24, 4093, 0, 0, 0, 36, 0, 0, 0, 4095, 339, 144, 0, 0, 0, 0, 0, 0, 29, 3603, 0, 0, 0, 0, 0, 22, 0, 0, 0, 4072, 0, 0, 0, 5, 0, 4031, 3, 0, 0, 0, 0, 0, 3, 0, 3997, 0, 0, 0, 0, 2350, 0, 3, 0, 2705, 12, 0, 0] rerouted_tokens_per_ep_rank=[3226, 4129, 4578, 3632, 4094, 4039, 4000, 5070] avg=4096.0000 max=5070 max_rank=7 min=3226 min_rank=0 imbalance_ratio=1.237793 max_load_reduction=27698 max_load_reduction_pct=0.845276 moved_to_echo=17203 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1363, 3, 20, 1048, 328, 101, 28, 4075, 0, 0, 22, 39, 0, 0, 0, 4092, 329, 1, 0, 0, 0, 0, 0, 0, 0, 3704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 19, 0, 0, 3977, 0, 0, 0, 0, 2733, 0, 0, 66, 2696, 11, 0, 0] rerouted_tokens_per_ep_rank=[2891, 4136, 4422, 3704, 4085, 4028, 3996, 5506] avg=4096.0000 max=5506 max_rank=7 min=2891 min_rank=0 imbalance_ratio=1.344238 max_load_reduction=27262 max_load_reduction_pct=0.831970 moved_to_echo=17615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3712, 0, 0, 1050, 323, 97, 24, 4094, 0, 0, 0, 36, 0, 0, 0, 4086, 339, 0, 0, 0, 0, 0, 0, 0, 148, 3694, 0, 0, 0, 0, 0, 2, 0, 6, 0, 4083, 0, 0, 0, 0, 219, 4029, 107, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 2, 0, 0, 0, 2707, 11, 0, 0] rerouted_tokens_per_ep_rank=[5206, 4130, 4425, 3842, 4091, 4355, 3999, 2720] avg=4096.0000 max=5206 max_rank=0 min=2720 min_rank=7 imbalance_ratio=1.270996 max_load_reduction=27562 max_load_reduction_pct=0.841125 moved_to_echo=15165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2711, 0, 0, 1049, 324, 97, 24, 4093, 0, 0, 0, 36, 0, 0, 0, 4096, 340, 0, 0, 0, 0, 0, 0, 0, 0, 3702, 1, 0, 0, 0, 0, 0, 0, 8, 0, 4084, 0, 0, 0, 0, 3, 4033, 0, 0, 0, 0, 0, 3, 0, 1, 3998, 0, 0, 0, 0, 1448, 0, 0, 0, 2705, 12, 0, 0] rerouted_tokens_per_ep_rank=[4205, 4129, 4436, 3703, 4092, 4036, 4002, 4165] avg=4096.0000 max=4436 max_rank=2 min=3703 min_rank=3 imbalance_ratio=1.083008 max_load_reduction=28332 max_load_reduction_pct=0.864624 moved_to_echo=16295 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11008, 128, 128, 128, 21632, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 20736, 128, 128, 8448, 2688, 896, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[29824, 2816, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 3328, 29568, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 512, 32000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 512, 32256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 128, 0, 128, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[10, 3379, 1692, 107, 2984, 6, 1187, 31, 1730, 34, 3142, 642, 448, 466, 163, 242, 580, 1312, 1481, 3600, 3365, 679, 633, 376, 2, 1740, 257, 618, 186, 74, 1400, 202] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[9, 3169, 1683, 11, 2740, 7, 1034, 37, 1077, 130, 3226, 1137, 565, 435, 140, 244, 589, 1040, 1623, 3851, 3298, 776, 566, 182, 0, 1724, 239, 655, 779, 114, 1534, 154] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[15, 3545, 1584, 67, 2912, 1, 947, 43, 1721, 53, 3373, 920, 900, 397, 46, 292, 597, 890, 1676, 3692, 3147, 842, 589, 206, 2, 1762, 199, 507, 439, 75, 1288, 41] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[19, 3260, 2068, 57, 2919, 16, 906, 65, 1133, 126, 3142, 1057, 536, 486, 111, 267, 504, 1356, 1635, 3731, 3141, 463, 622, 140, 14, 1658, 357, 771, 534, 224, 1031, 419] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[13, 3125, 1699, 25, 2564, 3, 1024, 27, 2076, 133, 3532, 673, 476, 774, 52, 218, 389, 748, 2077, 3423, 3285, 885, 513, 298, 0, 1467, 210, 719, 563, 99, 1639, 39] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[26, 3256, 1815, 28, 2646, 32, 1076, 40, 1107, 167, 3278, 927, 407, 508, 157, 223, 411, 1223, 1873, 3650, 3160, 555, 584, 288, 3, 1370, 208, 770, 896, 160, 1343, 581] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[16, 3575, 2028, 16, 2503, 6, 894, 14, 1467, 57, 3340, 671, 406, 651, 51, 207, 275, 1253, 1608, 3747, 3366, 383, 784, 171, 1, 2207, 196, 915, 527, 157, 1127, 149] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[16, 3444, 1804, 45, 2990, 1, 1323, 19, 1750, 114, 3378, 684, 312, 557, 67, 183, 353, 1549, 1206, 3548, 3374, 625, 681, 162, 8, 1923, 173, 708, 240, 64, 1339, 128] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 2264, 1692, 107, 0, 0, 0, 0, 2984, 6, 1187, 31, 228, 0, 0, 0, 1730, 34, 1569, 642, 0, 0, 0, 0, 448, 466, 163, 242, 2681, 0, 0, 0, 580, 1312, 1481, 804, 0, 0, 0, 0, 2674, 679, 633, 376, 0, 0, 0, 0, 2, 1740, 257, 618, 983, 463, 0, 0, 186, 74, 1400, 202, 1573, 132, 115, 0] rerouted_tokens_per_ep_rank=[4073, 4436, 3975, 4000, 4177, 4362, 4063, 3682] avg=4096.0000 max=4436 max_rank=1 min=3682 min_rank=7 imbalance_ratio=1.083008 max_load_reduction=28332 max_load_reduction_pct=0.864624 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[9, 2123, 1683, 11, 0, 0, 0, 0, 2740, 7, 1034, 37, 223, 0, 0, 0, 1077, 130, 1611, 1137, 0, 0, 0, 0, 565, 435, 140, 244, 2868, 0, 0, 0, 589, 1040, 1623, 860, 0, 0, 0, 0, 2621, 776, 566, 182, 0, 0, 0, 0, 0, 1724, 239, 655, 922, 454, 0, 0, 779, 114, 1534, 154, 1615, 124, 123, 0] rerouted_tokens_per_ep_rank=[3826, 4041, 3955, 4252, 4112, 4145, 3994, 4443] avg=4096.0000 max=4443 max_rank=7 min=3826 min_rank=0 imbalance_ratio=1.084717 max_load_reduction=28325 max_load_reduction_pct=0.864410 moved_to_echo=16694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[15, 2375, 1584, 67, 0, 0, 0, 0, 2912, 1, 947, 43, 213, 0, 0, 0, 1721, 53, 1684, 920, 0, 0, 0, 0, 900, 397, 46, 292, 2749, 0, 0, 0, 597, 890, 1676, 825, 0, 0, 0, 0, 2501, 842, 589, 206, 0, 0, 0, 0, 2, 1762, 199, 507, 1031, 433, 0, 0, 439, 75, 1288, 41, 1689, 139, 118, 0] rerouted_tokens_per_ep_rank=[4041, 4116, 4378, 4384, 3988, 4138, 3934, 3789] avg=4096.0000 max=4384 max_rank=3 min=3789 min_rank=7 imbalance_ratio=1.070312 max_load_reduction=28384 max_load_reduction_pct=0.866211 moved_to_echo=15849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[19, 2184, 2068, 57, 0, 0, 0, 0, 2919, 16, 906, 65, 212, 0, 0, 0, 1133, 126, 1569, 1057, 0, 0, 0, 0, 536, 486, 111, 267, 2779, 0, 0, 0, 504, 1356, 1635, 832, 0, 0, 0, 0, 2497, 463, 622, 140, 0, 0, 0, 0, 14, 1658, 357, 771, 948, 432, 0, 0, 534, 224, 1031, 419, 1573, 128, 120, 0] rerouted_tokens_per_ep_rank=[4328, 4118, 3885, 4179, 4327, 3722, 4180, 4029] avg=4096.0000 max=4328 max_rank=0 min=3722 min_rank=5 imbalance_ratio=1.056641 max_load_reduction=28440 max_load_reduction_pct=0.867920 moved_to_echo=16258 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[13, 2093, 1699, 25, 0, 0, 0, 0, 2564, 3, 1024, 27, 222, 0, 0, 0, 2076, 133, 1763, 673, 0, 0, 0, 0, 476, 774, 52, 218, 2549, 0, 0, 0, 389, 748, 2077, 764, 0, 0, 0, 0, 2611, 885, 513, 298, 0, 0, 0, 0, 0, 1467, 210, 719, 909, 452, 0, 0, 563, 99, 1639, 39, 1769, 123, 110, 0] rerouted_tokens_per_ep_rank=[3830, 3840, 4645, 4069, 3978, 4307, 3757, 4342] avg=4096.0000 max=4645 max_rank=2 min=3757 min_rank=6 imbalance_ratio=1.134033 max_load_reduction=28123 max_load_reduction_pct=0.858246 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[26, 2174, 1815, 28, 0, 0, 0, 0, 2646, 32, 1076, 40, 217, 0, 0, 0, 1107, 167, 1632, 927, 0, 0, 0, 0, 407, 508, 157, 223, 2723, 0, 0, 0, 411, 1223, 1873, 806, 0, 0, 0, 0, 2504, 555, 584, 288, 0, 0, 0, 0, 3, 1370, 208, 770, 950, 439, 0, 0, 896, 160, 1343, 581, 1646, 132, 121, 0] rerouted_tokens_per_ep_rank=[4043, 4011, 3833, 4018, 4313, 3931, 3740, 4879] avg=4096.0000 max=4879 max_rank=7 min=3740 min_rank=6 imbalance_ratio=1.191162 max_load_reduction=27889 max_load_reduction_pct=0.851105 moved_to_echo=16863 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 2395, 2028, 16, 0, 0, 0, 0, 2503, 6, 894, 14, 228, 0, 0, 0, 1467, 57, 1668, 671, 0, 0, 0, 0, 406, 651, 51, 207, 2790, 0, 0, 0, 275, 1253, 1608, 837, 0, 0, 0, 0, 2675, 383, 784, 171, 0, 0, 0, 0, 1, 2207, 196, 915, 1040, 463, 0, 0, 527, 157, 1127, 149, 1672, 140, 120, 0] rerouted_tokens_per_ep_rank=[4455, 3645, 3863, 4105, 3973, 4013, 4822, 3892] avg=4096.0000 max=4822 max_rank=6 min=3645 min_rank=1 imbalance_ratio=1.177246 max_load_reduction=27946 max_load_reduction_pct=0.852844 moved_to_echo=16700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 2307, 1804, 45, 0, 0, 0, 0, 2990, 1, 1323, 19, 228, 0, 0, 0, 1750, 114, 1686, 684, 0, 0, 0, 0, 312, 557, 67, 183, 2642, 0, 0, 0, 353, 1549, 1206, 792, 0, 0, 0, 0, 2682, 625, 681, 162, 0, 0, 0, 0, 8, 1923, 173, 708, 1002, 464, 0, 0, 240, 64, 1339, 128, 1692, 135, 114, 0] rerouted_tokens_per_ep_rank=[4172, 4561, 4234, 3761, 3900, 4150, 4278, 3712] avg=4096.0000 max=4561 max_rank=1 min=3712 min_rank=7 imbalance_ratio=1.113525 max_load_reduction=28207 max_load_reduction_pct=0.860809 moved_to_echo=16040 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[3712, 9472, 13184, 6528, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 13952, 1920, 5760, 7808, 3712, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 17920, 14464, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 1024, 10752, 1792, 13312, 1152, 1024, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12160, 896, 13184, 6784, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[22272, 128, 8448, 384, 1792, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[20864, 5248, 4992, 1920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4352, 896, 1920, 21888, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 4061, 1675, 39, 0, 4, 33, 2, 1, 3450, 0, 0, 6, 8, 4045, 0, 0, 7, 1, 1, 3899, 0, 62, 4, 4022, 2975, 4091, 22, 529, 2, 1, 3828] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 4096, 709, 18, 3, 0, 0, 211, 2, 3759, 0, 3, 0, 0, 3826, 1, 0, 0, 0, 0, 4092, 1, 32, 6, 4084, 3895, 4096, 0, 46, 1, 0, 3887] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2, 4054, 2547, 42, 0, 1, 15, 10, 0, 3838, 86, 1, 1, 0, 3563, 0, 0, 0, 0, 0, 4087, 24, 9, 0, 4054, 3625, 4089, 0, 926, 19, 43, 1732] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[7, 4091, 556, 81, 0, 0, 82, 10, 0, 3918, 3, 2, 1, 0, 4007, 0, 0, 0, 2, 0, 4033, 2, 2, 33, 4077, 4066, 4093, 3, 123, 1, 34, 3541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 4067, 1733, 6, 0, 0, 0, 177, 11, 4076, 1, 2, 0, 0, 4023, 0, 17, 0, 0, 0, 4006, 2, 32, 20, 4096, 2367, 4092, 0, 85, 0, 0, 3955] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[6, 4078, 677, 53, 0, 0, 35, 0, 2, 3881, 0, 0, 0, 0, 3759, 0, 0, 0, 0, 0, 4053, 0, 52, 28, 4096, 3992, 4096, 0, 5, 0, 2, 3953] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 4094, 1277, 60, 0, 0, 3, 17, 0, 4090, 65, 0, 0, 0, 3729, 0, 0, 0, 0, 0, 3959, 0, 0, 2, 4089, 3189, 4096, 2, 4, 0, 26, 4066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[7, 4093, 1450, 364, 0, 0, 176, 29, 0, 3778, 20, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 4063, 0, 1, 72, 4030, 3920, 4096, 0, 3, 0, 40, 2603] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 2669, 2547, 42, 0, 0, 0, 0, 0, 1, 15, 10, 3467, 511, 1, 0, 0, 3838, 86, 1, 220, 0, 0, 0, 1, 0, 3563, 0, 220, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4087, 24, 9, 0, 23, 0, 0, 0, 587, 3625, 0, 0, 0, 0, 0, 0, 926, 19, 43, 1732, 411, 0, 0, 0] rerouted_tokens_per_ep_rank=[5260, 4005, 4145, 3784, 4088, 4143, 4212, 3131] avg=4096.0000 max=5260 max_rank=0 min=3131 min_rank=7 imbalance_ratio=1.284180 max_load_reduction=27508 max_load_reduction_pct=0.839478 moved_to_echo=15574 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[7, 2692, 556, 81, 0, 0, 0, 0, 0, 0, 82, 10, 3486, 515, 1, 0, 0, 3918, 3, 2, 223, 0, 0, 0, 1, 0, 4007, 0, 222, 0, 0, 0, 0, 0, 2, 0, 4092, 0, 0, 0, 4033, 2, 2, 33, 24, 0, 0, 0, 591, 4066, 0, 3, 0, 0, 0, 0, 123, 1, 34, 3541, 415, 0, 0, 0] rerouted_tokens_per_ep_rank=[3336, 4094, 4146, 4230, 4094, 4094, 4660, 4114] avg=4096.0000 max=4660 max_rank=6 min=3336 min_rank=0 imbalance_ratio=1.137695 max_load_reduction=28108 max_load_reduction_pct=0.857788 moved_to_echo=16962 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2696, 709, 18, 0, 0, 0, 0, 3, 0, 0, 211, 3492, 516, 1, 0, 2, 3759, 0, 3, 223, 0, 0, 0, 0, 0, 3826, 1, 222, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4092, 1, 32, 6, 24, 0, 0, 0, 592, 3895, 0, 0, 0, 0, 0, 0, 46, 1, 0, 3887, 415, 0, 0, 0] rerouted_tokens_per_ep_rank=[3423, 4223, 3987, 4049, 4095, 4155, 4487, 4349] avg=4096.0000 max=4487 max_rank=6 min=3423 min_rank=0 imbalance_ratio=1.095459 max_load_reduction=28281 max_load_reduction_pct=0.863068 moved_to_echo=17086 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 2685, 677, 53, 0, 0, 0, 0, 0, 0, 35, 0, 3503, 514, 1, 0, 2, 3881, 0, 0, 222, 0, 0, 0, 0, 0, 3759, 0, 221, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4053, 0, 52, 28, 23, 0, 0, 0, 593, 3992, 0, 0, 0, 0, 0, 0, 5, 0, 2, 3953, 413, 0, 0, 0] rerouted_tokens_per_ep_rank=[3421, 4053, 4105, 3980, 4095, 4156, 4585, 4373] avg=4096.0000 max=4585 max_rank=6 min=3421 min_rank=0 imbalance_ratio=1.119385 max_load_reduction=28183 max_load_reduction_pct=0.860077 moved_to_echo=17209 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2652, 1675, 39, 0, 0, 0, 0, 0, 4, 33, 2, 3442, 515, 2, 0, 1, 3450, 0, 0, 224, 0, 0, 0, 6, 8, 4045, 0, 226, 0, 0, 0, 0, 7, 1, 1, 4089, 0, 0, 0, 3899, 0, 62, 4, 27, 0, 0, 0, 580, 2975, 0, 22, 0, 0, 0, 0, 529, 2, 1, 3828, 417, 0, 0, 0] rerouted_tokens_per_ep_rank=[4366, 3998, 3675, 4285, 4098, 3992, 3577, 4777] avg=4096.0000 max=4777 max_rank=7 min=3577 min_rank=6 imbalance_ratio=1.166260 max_load_reduction=27991 max_load_reduction_pct=0.854218 moved_to_echo=16444 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2678, 1733, 6, 0, 0, 0, 0, 0, 0, 0, 177, 3503, 512, 1, 0, 11, 4076, 1, 2, 221, 0, 0, 0, 0, 0, 4023, 0, 221, 0, 0, 0, 17, 0, 0, 0, 4091, 0, 0, 0, 4006, 2, 32, 20, 23, 0, 0, 0, 593, 2367, 0, 0, 0, 0, 0, 0, 85, 0, 0, 3955, 412, 0, 0, 0] rerouted_tokens_per_ep_rank=[4417, 4193, 4311, 4244, 4108, 4083, 2960, 4452] avg=4096.0000 max=4452 max_rank=7 min=2960 min_rank=6 imbalance_ratio=1.086914 max_load_reduction=28316 max_load_reduction_pct=0.864136 moved_to_echo=15603 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2694, 1277, 60, 0, 0, 0, 0, 0, 0, 3, 17, 3497, 516, 1, 0, 0, 4090, 65, 0, 223, 0, 0, 0, 0, 0, 3729, 0, 222, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3959, 0, 0, 2, 24, 0, 0, 0, 592, 3189, 0, 2, 0, 0, 0, 0, 4, 0, 26, 4066, 415, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4034, 4378, 3951, 4095, 3985, 3783, 4511] avg=4096.0000 max=4511 max_rank=7 min=3783 min_rank=6 imbalance_ratio=1.101318 max_load_reduction=28257 max_load_reduction_pct=0.862335 moved_to_echo=16374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[7, 2693, 1450, 364, 0, 0, 0, 0, 0, 0, 176, 29, 3446, 516, 1, 0, 0, 3778, 20, 0, 223, 0, 0, 0, 0, 0, 4023, 0, 222, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4063, 0, 1, 72, 24, 0, 0, 0, 584, 3920, 0, 0, 0, 0, 0, 0, 3, 0, 40, 2603, 415, 0, 0, 0] rerouted_tokens_per_ep_rank=[4514, 4168, 4021, 4245, 4095, 4160, 4504, 3061] avg=4096.0000 max=4514 max_rank=0 min=3061 min_rank=7 imbalance_ratio=1.102051 max_load_reduction=28254 max_load_reduction_pct=0.862244 moved_to_echo=15820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 21504, 10624, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 384, 512, 27904, 4224, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1792, 128, 256, 27648, 3328, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[4736, 28032, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 30848, 256, 128, 1792, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 30976, 128, 1792, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32256, 128, 256, 256, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 4070, 0, 4096, 0, 4019, 0, 57, 4094, 4093, 4093, 0, 4083, 0, 9, 35, 0, 1, 0, 1, 0, 4025, 40, 0, 48, 1, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 2, 0, 0, 4095, 2, 4090, 2, 3614, 1, 31, 4076, 4093, 4096, 0, 4096, 0, 0, 0, 0, 504, 0, 1, 37, 4005, 21, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 13, 4083, 0, 4092, 0, 3096, 0, 856, 4037, 4096, 4072, 29, 4083, 0, 18, 505, 0, 130, 0, 1, 0, 3565, 88, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 3945, 0, 4096, 0, 3230, 0, 1052, 4096, 4096, 4096, 41, 4096, 0, 0, 39, 0, 2, 0, 0, 0, 3774, 184, 0, 9, 0, 10, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 45, 0, 0, 0, 4067, 0, 4095, 2, 4033, 0, 10, 4084, 4094, 4079, 17, 4095, 0, 0, 397, 0, 325, 0, 1, 1, 3399, 23, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 3985, 0, 4096, 0, 2430, 0, 9, 4095, 4090, 4094, 0, 4090, 0, 0, 1689, 0, 126, 0, 5, 48, 3982, 22, 0, 6, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[60, 0, 2, 0, 71, 3920, 0, 4074, 0, 3799, 82, 6, 4058, 4059, 4022, 88, 4075, 23, 17, 196, 62, 0, 3, 1, 89, 3637, 369, 2, 9, 0, 44, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4017, 0, 4096, 0, 3783, 0, 14, 4096, 4096, 4096, 0, 4096, 0, 0, 441, 0, 249, 0, 0, 0, 3721, 34, 0, 27, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4084, 0, 0, 0, 0, 4070, 0, 63, 0, 0, 0, 0, 0, 4019, 0, 57, 206, 123, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 3670, 0, 9, 35, 0, 0, 0, 0, 0, 1, 0, 1, 3910, 13, 0, 0, 0, 4025, 40, 0, 207, 5, 0, 0, 48, 1, 0, 1, 4080, 9, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4133, 4405, 4089, 3714, 3925, 4277, 4139] avg=4096.0000 max=4405 max_rank=2 min=3714 min_rank=4 imbalance_ratio=1.075439 max_load_reduction=28363 max_load_reduction_pct=0.865570 moved_to_echo=16055 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 0, 4085, 0, 0, 0, 0, 4095, 2, 63, 0, 0, 0, 0, 2, 3614, 1, 31, 207, 123, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 3682, 0, 0, 0, 0, 0, 0, 0, 0, 504, 0, 1, 3904, 13, 0, 0, 37, 4005, 21, 0, 207, 5, 0, 0, 1, 0, 0, 0, 4083, 8, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4160, 3978, 4071, 3682, 4422, 4275, 4092] avg=4096.0000 max=4422 max_rank=5 min=3682 min_rank=4 imbalance_ratio=1.079590 max_load_reduction=28346 max_load_reduction_pct=0.865051 moved_to_echo=16471 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4088, 0, 0, 0, 13, 4083, 0, 63, 0, 0, 0, 0, 0, 3096, 0, 856, 206, 123, 0, 0, 4032, 0, 0, 29, 0, 0, 0, 0, 3670, 0, 18, 505, 0, 0, 0, 0, 0, 130, 0, 1, 3906, 14, 0, 0, 0, 3565, 88, 0, 207, 5, 0, 0, 1, 0, 0, 0, 4058, 8, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4159, 4281, 4061, 4193, 4051, 3865, 4067] avg=4096.0000 max=4281 max_rank=2 min=3865 min_rank=6 imbalance_ratio=1.045166 max_load_reduction=28487 max_load_reduction_pct=0.869354 moved_to_echo=16176 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4087, 0, 0, 0, 0, 3945, 0, 63, 0, 0, 0, 0, 0, 3230, 0, 1052, 207, 123, 0, 0, 4091, 0, 0, 41, 0, 0, 0, 0, 3682, 0, 0, 39, 0, 0, 0, 0, 0, 2, 0, 0, 3910, 13, 0, 0, 0, 3774, 184, 0, 207, 5, 0, 0, 9, 0, 10, 0, 4083, 9, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4008, 4612, 4132, 3721, 3925, 4170, 4111] avg=4096.0000 max=4612 max_rank=2 min=3721 min_rank=4 imbalance_ratio=1.125977 max_load_reduction=28156 max_load_reduction_pct=0.859253 moved_to_echo=15927 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 45, 0, 0, 4085, 0, 0, 0, 0, 4067, 0, 63, 0, 0, 0, 0, 2, 4033, 0, 10, 207, 123, 0, 0, 4079, 0, 0, 17, 0, 0, 0, 0, 3681, 0, 0, 397, 0, 0, 0, 0, 0, 325, 0, 1, 3909, 13, 0, 0, 1, 3399, 23, 0, 207, 5, 0, 0, 0, 0, 0, 0, 4066, 9, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4130, 4375, 4096, 4078, 4248, 3635, 4075] avg=4096.0000 max=4375 max_rank=2 min=3635 min_rank=6 imbalance_ratio=1.068115 max_load_reduction=28393 max_load_reduction_pct=0.866486 moved_to_echo=16036 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[60, 0, 2, 0, 4051, 0, 0, 0, 71, 3920, 0, 59, 0, 0, 0, 0, 0, 3799, 82, 6, 209, 125, 0, 0, 4050, 0, 0, 88, 0, 0, 0, 0, 3656, 23, 17, 196, 0, 0, 0, 0, 62, 0, 3, 1, 3890, 14, 0, 0, 89, 3637, 369, 2, 210, 8, 0, 0, 9, 0, 44, 0, 4008, 8, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4050, 4221, 4138, 3892, 3970, 4315, 4069] avg=4096.0000 max=4315 max_rank=6 min=3892 min_rank=4 imbalance_ratio=1.053467 max_load_reduction=28453 max_load_reduction_pct=0.868317 moved_to_echo=16246 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4082, 0, 0, 0, 0, 3985, 0, 63, 0, 0, 0, 0, 0, 2430, 0, 9, 207, 123, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 3676, 0, 0, 1689, 0, 0, 0, 0, 0, 126, 0, 5, 3910, 14, 0, 0, 48, 3982, 22, 0, 207, 5, 0, 0, 6, 0, 0, 0, 4080, 8, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4048, 2769, 4090, 5365, 4055, 4264, 4094] avg=4096.0000 max=5365 max_rank=4 min=2769 min_rank=2 imbalance_ratio=1.309814 max_load_reduction=27403 max_load_reduction_pct=0.836273 moved_to_echo=17778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 0, 0, 0, 0, 4017, 0, 63, 0, 0, 0, 0, 0, 3783, 0, 14, 207, 123, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 3682, 0, 0, 441, 0, 0, 0, 0, 0, 249, 0, 0, 3910, 13, 0, 0, 0, 3721, 34, 0, 207, 5, 0, 0, 27, 0, 2, 0, 4083, 9, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4080, 4127, 4091, 4123, 4172, 3967, 4121] avg=4096.0000 max=4172 max_rank=5 min=3967 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16383 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[29440, 128, 128, 3328, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 30208, 896, 128, 1664, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32256, 128, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 28032, 128, 2048, 1664, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1408, 128, 128, 31360, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32640, 0, 0, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 3, 20, 4090, 0, 0, 1, 0, 4090, 0, 0, 1, 4080, 0, 4093, 3067, 0, 0, 1024, 0, 0, 4094, 8, 4090, 0, 4, 0, 0, 5, 4092, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[21, 0, 0, 4089, 0, 0, 4, 0, 4084, 0, 0, 4, 4093, 8, 4096, 4094, 0, 1, 0, 0, 0, 4095, 0, 4095, 0, 2, 0, 1, 4, 4072, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[32, 0, 32, 4024, 2, 2, 500, 1, 4079, 7, 32, 0, 4050, 7, 4063, 4082, 1, 0, 199, 7, 0, 4052, 6, 3449, 46, 0, 4, 6, 0, 4083, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3050, 0, 0, 1024, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 17, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[49, 0, 495, 4094, 0, 0, 640, 0, 4096, 0, 93, 1, 4096, 0, 4096, 3209, 0, 0, 10, 0, 19, 4091, 9, 3642, 0, 0, 0, 0, 0, 4094, 34, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[29, 0, 27, 4093, 0, 0, 135, 0, 4095, 16, 6, 1, 4095, 0, 4094, 3808, 0, 1, 5, 0, 0, 4096, 0, 4096, 0, 1, 0, 70, 0, 4096, 3, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 101, 4083, 0, 0, 328, 0, 4096, 0, 0, 0, 4091, 6, 4069, 3512, 15, 4, 190, 0, 1, 4092, 0, 4081, 0, 0, 0, 3, 4, 4090, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 137, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 3488, 6, 0, 472, 0, 0, 4096, 0, 4065, 0, 0, 0, 0, 0, 4096, 23, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 20, 3984, 0, 0, 0, 0, 0, 0, 1, 0, 3890, 9, 0, 0, 4075, 0, 0, 1, 0, 0, 0, 0, 554, 0, 0, 3067, 0, 0, 0, 0, 0, 0, 1024, 0, 3526, 106, 64, 15, 0, 140, 8, 4090, 0, 0, 0, 0, 0, 4, 0, 0, 4084, 0, 0, 0, 5, 4092, 0, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 3900, 4076, 3621, 4735, 4238, 4088, 4103] avg=4096.0000 max=4735 max_rank=4 min=3621 min_rank=3 imbalance_ratio=1.156006 max_load_reduction=28033 max_load_reduction_pct=0.855499 moved_to_echo=17164 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 4, 0, 3891, 10, 0, 0, 4069, 0, 0, 4, 0, 0, 0, 0, 556, 8, 0, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 3537, 106, 64, 15, 0, 140, 0, 4095, 0, 0, 0, 0, 0, 2, 0, 1, 4086, 0, 0, 0, 4, 4072, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4004, 3905, 4073, 4658, 3723, 4235, 4089, 4081] avg=4096.0000 max=4658 max_rank=3 min=3723 min_rank=4 imbalance_ratio=1.137207 max_load_reduction=28110 max_load_reduction_pct=0.857849 moved_to_echo=16128 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 9, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 3050, 0, 0, 0, 0, 0, 0, 1024, 0, 3540, 106, 64, 15, 0, 140, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3995, 3901, 4081, 3606, 4749, 4236, 4087, 4113] avg=4096.0000 max=4749 max_rank=4 min=3606 min_rank=3 imbalance_ratio=1.159424 max_load_reduction=28019 max_load_reduction_pct=0.855072 moved_to_echo=17185 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 27, 3987, 0, 0, 0, 0, 0, 0, 135, 0, 3892, 9, 0, 0, 4080, 16, 6, 1, 0, 0, 0, 0, 556, 0, 0, 3808, 0, 0, 0, 0, 0, 1, 5, 0, 3539, 106, 64, 15, 0, 140, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 70, 4085, 0, 0, 0, 0, 4096, 3, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4036, 4103, 4364, 3730, 4236, 4156, 4100] avg=4096.0000 max=4364 max_rank=3 min=3730 min_rank=4 imbalance_ratio=1.065430 max_load_reduction=28404 max_load_reduction_pct=0.866821 moved_to_echo=16222 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 101, 3978, 0, 0, 0, 0, 0, 0, 328, 0, 3888, 8, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 556, 6, 0, 3512, 0, 0, 0, 0, 15, 4, 190, 0, 3535, 105, 64, 15, 1, 140, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 3, 4061, 0, 0, 0, 4, 4090, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4224, 4081, 4074, 3928, 4222, 4064, 4096] avg=4096.0000 max=4224 max_rank=1 min=3928 min_rank=4 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 32, 3917, 0, 0, 0, 0, 2, 2, 500, 1, 3851, 9, 0, 0, 4059, 7, 32, 0, 0, 0, 0, 0, 547, 7, 0, 4082, 0, 0, 0, 0, 1, 0, 199, 7, 3503, 107, 70, 20, 0, 131, 6, 3449, 0, 0, 0, 0, 46, 0, 4, 6, 4054, 0, 0, 0, 0, 4083, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3981, 4365, 4098, 4636, 3907, 3586, 4110, 4085] avg=4096.0000 max=4636 max_rank=3 min=3586 min_rank=5 imbalance_ratio=1.131836 max_load_reduction=28132 max_load_reduction_pct=0.858521 moved_to_echo=15688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[49, 0, 495, 3988, 0, 0, 0, 0, 0, 0, 640, 0, 3887, 9, 0, 0, 4081, 0, 93, 1, 0, 0, 0, 0, 556, 0, 0, 3209, 0, 0, 0, 0, 0, 0, 10, 0, 3540, 106, 64, 15, 19, 140, 9, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4094, 34, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4532, 4536, 4175, 3765, 3735, 3810, 4087, 4128] avg=4096.0000 max=4536 max_rank=1 min=3735 min_rank=4 imbalance_ratio=1.107422 max_load_reduction=28232 max_load_reduction_pct=0.861572 moved_to_echo=15760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 137, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 9, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 556, 0, 0, 3488, 0, 0, 0, 0, 6, 0, 472, 0, 3538, 106, 64, 15, 0, 140, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 23, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 3901, 4081, 4044, 4201, 4205, 4087, 4122] avg=4096.0000 max=4205 max_rank=5 min=3901 min_rank=1 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32708, 32768, 32768, 32828] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32708, 32768, 32768, 32828] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32708, 32768, 32768, 32828] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32708, 32768, 32768, 32828] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32708, 32768, 32768, 32828] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32708, 32768, 32768, 32828] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32708, 32768, 32768, 32828] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32708, 32768, 32768, 32828] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4480, 128, 0, 28416, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32828 actual_valid_recv=33152 observed_loss=-324 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32708 actual_valid_recv=33280 observed_loss=-572 tokens_per_expert=[128, 128, 2944, 128, 28288, 896, 640, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 1664, 128, 31104, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 896, 31872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 1152, 128, 31616, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 15, 0, 0, 4, 3970, 2237, 0, 4072, 0, 10, 0, 15, 4085, 1, 671, 2670, 0, 0, 4090, 0, 0, 5, 4094, 0, 2, 0, 0, 4094, 3, 2728] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 4095, 1282, 0, 4091, 0, 189, 0, 0, 4096, 0, 769, 2436, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 3, 0, 4093, 0, 3515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 0, 4093, 23, 22, 4090, 0, 25, 0, 2, 4026, 0, 780, 3821, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 4, 0, 4090, 52, 3534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 27, 0, 4069, 3215, 0, 4005, 0, 17, 0, 66, 4086, 15, 244, 3947, 49, 10, 4057, 8, 12, 26, 4053, 0, 1, 5, 34, 4063, 181, 575] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4067, 316, 0, 4041, 0, 1488, 0, 0, 4096, 0, 791, 2050, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3631] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[7, 0, 8, 81, 1, 2, 4095, 1522, 0, 4088, 13, 0, 26, 0, 4096, 0, 891, 4011, 1, 2, 4081, 0, 0, 0, 4092, 0, 9, 3, 0, 4026, 0, 1713] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4060, 24, 0, 4095, 0, 669, 0, 27, 4096, 0, 1070, 2858, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 24, 3556] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4088, 561, 0, 4091, 0, 50, 0, 0, 4077, 0, 270, 3301, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 15, 0, 2943, 1096, 22, 0, 0, 4, 2874, 2237, 0, 0, 0, 0, 0, 3787, 0, 10, 0, 0, 0, 0, 0, 15, 4081, 1, 0, 0, 0, 0, 671, 2670, 0, 0, 263, 1, 0, 0, 4090, 0, 0, 5, 1, 0, 0, 0, 4094, 0, 2, 0, 2, 0, 0, 0, 0, 1151, 3, 2728, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 5115, 3797, 4097, 3605, 4096, 4098, 3882] avg=4096.0000 max=5115 max_rank=1 min=3605 min_rank=4 imbalance_ratio=1.248779 max_load_reduction=27653 max_load_reduction_pct=0.843903 moved_to_echo=15681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2944, 1122, 22, 0, 0, 0, 2945, 316, 0, 0, 0, 0, 0, 3758, 0, 1488, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 791, 2050, 0, 0, 261, 1, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 1152, 0, 3631, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 3261, 5246, 4092, 3103, 4097, 4098, 4783] avg=4096.0000 max=5246 max_rank=2 min=3103 min_rank=4 imbalance_ratio=1.280762 max_load_reduction=27522 max_load_reduction_pct=0.839905 moved_to_echo=16081 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 8, 81, 2894, 1130, 22, 0, 1, 2, 2965, 1522, 0, 0, 0, 0, 0, 3802, 13, 0, 0, 0, 0, 0, 26, 0, 4092, 0, 0, 0, 0, 0, 891, 4011, 1, 2, 264, 1, 0, 0, 4081, 0, 0, 0, 1, 0, 0, 0, 4092, 0, 9, 3, 2, 0, 0, 0, 0, 1132, 0, 1713, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4490, 3815, 4118, 5170, 4082, 4106, 2845] avg=4096.0000 max=5170 max_rank=4 min=2845 min_rank=7 imbalance_ratio=1.262207 max_load_reduction=27598 max_load_reduction_pct=0.842224 moved_to_echo=16203 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 2940, 1130, 22, 0, 0, 0, 2963, 23, 0, 0, 0, 0, 22, 3804, 0, 25, 0, 0, 0, 0, 0, 2, 4022, 0, 0, 0, 0, 0, 780, 3821, 0, 0, 264, 1, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 3, 0, 4, 2, 0, 0, 0, 0, 1150, 52, 3534, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 2986, 3851, 4024, 4866, 4097, 4105, 4736] avg=4096.0000 max=4866 max_rank=4 min=2986 min_rank=1 imbalance_ratio=1.187988 max_load_reduction=27902 max_load_reduction_pct=0.851501 moved_to_echo=17804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2943, 1120, 22, 0, 0, 0, 2940, 24, 0, 0, 0, 0, 0, 3808, 0, 669, 0, 0, 0, 0, 0, 27, 4092, 0, 0, 0, 0, 0, 1070, 2858, 2, 0, 265, 1, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 1152, 24, 3556, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 2964, 4477, 4119, 4196, 4097, 4098, 4732] avg=4096.0000 max=4732 max_rank=7 min=2964 min_rank=1 imbalance_ratio=1.155273 max_load_reduction=28036 max_load_reduction_pct=0.855591 moved_to_echo=17123 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 2923, 1127, 25, 0, 27, 0, 2942, 3215, 0, 0, 0, 0, 0, 3716, 0, 17, 0, 0, 0, 0, 0, 66, 4072, 15, 0, 0, 0, 0, 244, 3947, 49, 10, 264, 8, 0, 0, 4057, 8, 12, 26, 2, 0, 0, 0, 4053, 0, 1, 5, 4, 0, 0, 0, 34, 1140, 181, 575, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 6184, 3733, 4153, 4522, 4105, 4063, 1930] avg=4096.0000 max=6184 max_rank=1 min=1930 min_rank=7 imbalance_ratio=1.509766 max_load_reduction=26584 max_load_reduction_pct=0.811279 moved_to_echo=14620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 2942, 1130, 22, 0, 0, 0, 2965, 1282, 0, 0, 0, 0, 0, 3805, 0, 189, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 769, 2436, 0, 0, 264, 1, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 1, 0, 3, 2, 0, 0, 0, 0, 1151, 0, 3515, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4247, 3994, 4092, 3470, 4097, 4102, 4666] avg=4096.0000 max=4666 max_rank=7 min=3470 min_rank=4 imbalance_ratio=1.139160 max_load_reduction=28102 max_load_reduction_pct=0.857605 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2944, 1128, 22, 0, 0, 0, 2960, 561, 0, 0, 0, 0, 0, 3805, 0, 50, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 270, 3301, 0, 0, 264, 1, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 1152, 0, 4042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 3521, 3855, 4073, 3836, 4097, 4098, 5194] avg=4096.0000 max=5194 max_rank=7 min=3521 min_rank=1 imbalance_ratio=1.268066 max_load_reduction=27574 max_load_reduction_pct=0.841492 moved_to_echo=17225 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 23552, 9088, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 9216, 384, 23296, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 23680, 9216, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 30336, 128, 2560, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5504, 25216, 128, 128, 2176, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[6, 67, 4087, 8, 4017, 0, 0, 0, 1, 213, 0, 2, 14, 56, 0, 0, 0, 2, 4093, 0, 1, 4096, 1, 1814, 4028, 1, 4094, 0, 3968, 0, 120, 2079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 16, 4065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[3, 0, 4076, 8, 4095, 0, 0, 1, 0, 11, 1, 3, 0, 0, 0, 0, 0, 3, 4092, 1, 11, 4094, 0, 198, 4093, 0, 4093, 0, 4096, 0, 427, 3462] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 4096, 0, 4095, 0, 0, 0, 0, 502, 0, 0, 0, 4, 0, 5, 92, 1, 4095, 0, 2, 4096, 0, 16, 4080, 0, 4088, 1, 4096, 0, 0, 3498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4020, 1, 4086, 3, 0, 30, 73, 5, 1, 23, 0, 7, 4, 4, 0, 0, 4061, 0, 19, 4058, 35, 1, 4094, 19, 4070, 10, 4075, 0, 0, 4068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[7, 2, 4096, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 54, 8, 4096, 0, 0, 4096, 0, 28, 4085, 5, 4096, 0, 4094, 0, 94, 3913] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[13, 0, 4090, 0, 4096, 0, 0, 1, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4026, 0, 4096, 0, 4093, 0, 50, 4048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 4096, 0, 4089, 0, 4096, 0, 103, 3981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[6, 67, 4086, 8, 0, 0, 0, 0, 4017, 0, 0, 0, 6, 1, 0, 0, 1, 213, 0, 2, 3612, 261, 9, 0, 14, 56, 0, 0, 4077, 9, 0, 0, 0, 2, 4078, 0, 0, 0, 0, 0, 1, 3835, 1, 1814, 0, 0, 0, 0, 4028, 1, 17, 0, 0, 0, 0, 0, 347, 0, 120, 2079, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4024, 4098, 4156, 4080, 5651, 4046, 2546] avg=4096.0000 max=5651 max_rank=5 min=2546 min_rank=7 imbalance_ratio=1.379639 max_load_reduction=27117 max_load_reduction_pct=0.827545 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4075, 8, 0, 0, 0, 0, 4095, 0, 0, 1, 6, 1, 0, 0, 0, 11, 1, 3, 3729, 261, 9, 0, 0, 0, 0, 0, 4076, 9, 0, 0, 0, 3, 4077, 1, 0, 0, 0, 0, 11, 3833, 0, 198, 0, 0, 0, 0, 4093, 0, 17, 0, 0, 0, 0, 0, 358, 0, 427, 3462, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4103, 4014, 4085, 4081, 4042, 4110, 4247] avg=4096.0000 max=4247 max_rank=7 min=4014 min_rank=2 imbalance_ratio=1.036865 max_load_reduction=28521 max_load_reduction_pct=0.870392 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 1, 0, 0, 0, 13, 0, 0, 3729, 261, 9, 0, 0, 0, 0, 0, 4079, 9, 0, 0, 0, 2, 4081, 0, 0, 0, 0, 0, 0, 3835, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 0, 0, 0, 358, 0, 16, 4065, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4103, 4012, 4088, 4083, 3835, 4113, 4439] avg=4096.0000 max=4439 max_rank=7 min=3835 min_rank=5 imbalance_ratio=1.083740 max_load_reduction=28329 max_load_reduction_pct=0.864532 moved_to_echo=16470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 6, 1, 0, 0, 0, 502, 0, 0, 3729, 261, 9, 0, 0, 4, 0, 5, 4071, 9, 0, 0, 92, 1, 4080, 0, 0, 0, 0, 0, 2, 3835, 0, 16, 0, 0, 0, 0, 4080, 0, 17, 1, 0, 0, 0, 0, 358, 0, 0, 3498, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4102, 4501, 4089, 4173, 3853, 4098, 3856] avg=4096.0000 max=4501 max_rank=2 min=3853 min_rank=5 imbalance_ratio=1.098877 max_load_reduction=28267 max_load_reduction_pct=0.862640 moved_to_echo=15980 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4017, 1, 0, 0, 0, 0, 4086, 3, 0, 30, 6, 3, 0, 0, 73, 5, 1, 23, 3712, 265, 9, 0, 0, 7, 4, 4, 4055, 14, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 19, 3793, 35, 1, 0, 0, 0, 0, 4094, 19, 15, 10, 0, 0, 0, 0, 349, 0, 0, 4068, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4128, 4088, 4084, 4046, 3848, 4138, 4417] avg=4096.0000 max=4417 max_rank=7 min=3848 min_rank=5 imbalance_ratio=1.078369 max_load_reduction=28351 max_load_reduction_pct=0.865204 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 6, 1, 0, 0, 0, 63, 0, 0, 3726, 261, 9, 0, 0, 0, 0, 0, 4079, 9, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 3835, 0, 0, 0, 0, 0, 0, 4026, 0, 17, 0, 0, 0, 0, 0, 358, 0, 50, 4048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4104, 4059, 4088, 4081, 3835, 4043, 4456] avg=4096.0000 max=4456 max_rank=7 min=3835 min_rank=5 imbalance_ratio=1.087891 max_load_reduction=28312 max_load_reduction_pct=0.864014 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[7, 2, 4095, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 6, 1, 0, 0, 0, 0, 0, 0, 3727, 261, 9, 0, 0, 0, 0, 0, 4079, 9, 0, 0, 54, 8, 4081, 0, 0, 0, 0, 0, 0, 3835, 0, 28, 0, 0, 0, 0, 4085, 5, 17, 0, 0, 0, 0, 0, 358, 0, 94, 3913, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4101, 3997, 4088, 4143, 3863, 4107, 4365] avg=4096.0000 max=4365 max_rank=7 min=3863 min_rank=5 imbalance_ratio=1.065674 max_load_reduction=28403 max_load_reduction_pct=0.866791 moved_to_echo=16478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 3729, 261, 9, 0, 9, 0, 0, 0, 4072, 9, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 3835, 0, 6, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 0, 0, 0, 358, 0, 103, 3981, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4103, 3999, 4090, 4081, 3841, 4113, 4442] avg=4096.0000 max=4442 max_rank=7 min=3841 min_rank=5 imbalance_ratio=1.084473 max_load_reduction=28326 max_load_reduction_pct=0.864441 moved_to_echo=16477 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 896, 128, 128, 29696, 2176, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2944, 0, 896, 29184, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 30720, 128, 2176, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1223, 0, 0, 0, 0, 4095, 0, 73, 0, 4095, 0, 7, 0, 0, 4096, 0, 2, 1, 0, 3364, 4095, 4095, 149, 1, 9, 3366, 0, 6, 0, 0, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1740, 0, 0, 7, 0, 4091, 1, 3, 0, 4092, 0, 41, 0, 0, 4094, 8, 0, 0, 7, 2330, 4093, 4021, 155, 0, 36, 3937, 0, 18, 0, 0, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[3877, 25, 0, 9, 18, 4079, 25, 40, 0, 4034, 2, 2, 2, 35, 4047, 46, 0, 51, 0, 209, 4002, 4058, 1125, 0, 1066, 1887, 13, 0, 33, 0, 4081, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[542, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 82, 1, 865, 4092, 4091, 4092, 0, 0, 2617, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[186, 0, 0, 0, 0, 4096, 0, 21, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2945, 4096, 4096, 1228, 0, 1, 3811, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[813, 0, 0, 0, 0, 4096, 0, 0, 2, 4095, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 2143, 4095, 4096, 3814, 0, 0, 1422, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 2, 0, 4075, 0, 23, 0, 0, 4096, 0, 0, 0, 0, 2776, 4096, 4096, 3939, 0, 2, 1455, 0, 16, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[179, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3149, 4096, 4096, 4006, 0, 1, 855, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1223, 0, 0, 0, 3029, 0, 0, 0, 0, 4075, 0, 73, 0, 0, 0, 0, 0, 4095, 0, 7, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 2, 1, 0, 3364, 1066, 795, 0, 0, 0, 1789, 149, 1, 0, 0, 0, 0, 9, 3366, 0, 6, 1511, 20, 4, 0, 0, 0, 4090, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4252, 4148, 4103, 4092, 5228, 1939, 4916, 4090] avg=4096.0000 max=5228 max_rank=4 min=1939 min_rank=5 imbalance_ratio=1.276367 max_load_reduction=27540 max_load_reduction_pct=0.840454 moved_to_echo=16173 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1740, 0, 0, 7, 3029, 0, 0, 0, 0, 4071, 1, 3, 0, 0, 0, 0, 0, 4092, 0, 41, 1, 0, 0, 0, 0, 0, 4090, 8, 0, 0, 0, 0, 0, 0, 7, 2330, 1064, 781, 0, 0, 0, 1757, 155, 0, 0, 0, 0, 0, 36, 3937, 0, 18, 1483, 20, 4, 0, 0, 0, 4093, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4776, 4075, 4134, 4098, 4182, 1912, 5498, 4093] avg=4096.0000 max=5498 max_rank=6 min=1912 min_rank=5 imbalance_ratio=1.342285 max_load_reduction=27270 max_load_reduction_pct=0.832214 moved_to_echo=15685 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[186, 0, 0, 0, 3030, 0, 0, 0, 0, 4076, 0, 21, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2945, 1066, 795, 0, 0, 0, 1790, 1228, 0, 0, 0, 0, 0, 1, 3811, 0, 0, 1511, 20, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3216, 4097, 4097, 4092, 4806, 3018, 5347, 4095] avg=4096.0000 max=5347 max_rank=6 min=3018 min_rank=5 imbalance_ratio=1.305420 max_load_reduction=27421 max_load_reduction_pct=0.836823 moved_to_echo=17266 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[813, 0, 0, 0, 3030, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 3, 0, 2143, 1065, 795, 0, 0, 0, 1790, 3814, 0, 0, 0, 0, 0, 0, 1422, 0, 0, 1511, 20, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3843, 4076, 4098, 4089, 4006, 5604, 2957, 4095] avg=4096.0000 max=5604 max_rank=5 min=2957 min_rank=6 imbalance_ratio=1.368164 max_load_reduction=27164 max_load_reduction_pct=0.828979 moved_to_echo=16662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3030, 0, 0, 0, 0, 4076, 0, 2, 0, 0, 0, 0, 0, 4075, 0, 23, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2776, 1066, 795, 0, 0, 0, 1790, 3939, 0, 0, 0, 0, 0, 2, 1455, 0, 16, 1511, 20, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3030, 4078, 4099, 4092, 4637, 5729, 3008, 4095] avg=4096.0000 max=5729 max_rank=5 min=3008 min_rank=6 imbalance_ratio=1.398682 max_load_reduction=27039 max_load_reduction_pct=0.825165 moved_to_echo=17469 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3877, 25, 0, 9, 2961, 0, 0, 0, 18, 4057, 25, 40, 0, 0, 0, 0, 0, 4034, 2, 2, 5, 0, 0, 0, 2, 35, 4036, 46, 0, 0, 0, 0, 0, 51, 0, 209, 1041, 792, 0, 0, 0, 1765, 1125, 0, 0, 0, 0, 0, 1066, 1887, 13, 0, 1501, 22, 11, 1, 33, 0, 4075, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6872, 4140, 4043, 4119, 2093, 2890, 4501, 4110] avg=4096.0000 max=6872 max_rank=0 min=2093 min_rank=4 imbalance_ratio=1.677734 max_load_reduction=25896 max_load_reduction_pct=0.790283 moved_to_echo=13594 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[542, 0, 0, 0, 3028, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 82, 1, 865, 1064, 794, 0, 0, 0, 1788, 4092, 0, 0, 0, 0, 0, 0, 2617, 0, 0, 1509, 20, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3570, 4076, 4097, 4094, 2806, 5880, 4150, 4095] avg=4096.0000 max=5880 max_rank=5 min=2806 min_rank=4 imbalance_ratio=1.435547 max_load_reduction=26888 max_load_reduction_pct=0.820557 moved_to_echo=16931 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[179, 0, 0, 0, 3030, 0, 0, 0, 0, 4076, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3149, 1066, 795, 0, 0, 0, 1790, 4006, 0, 0, 0, 0, 0, 1, 855, 0, 0, 1511, 20, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3209, 4078, 4097, 4092, 5010, 5796, 2391, 4095] avg=4096.0000 max=5796 max_rank=5 min=2391 min_rank=6 imbalance_ratio=1.415039 max_load_reduction=26972 max_load_reduction_pct=0.823120 moved_to_echo=17292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 128, 17792, 8576, 6400, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 14336, 18560, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1152, 19456, 128, 128, 12160, 256, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8576, 128, 0, 128, 24192, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 4067, 2184, 0, 2288, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 4095, 0, 0, 301, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 30, 0, 3418] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 3989, 4096, 9, 1010, 42, 5, 0, 445, 28, 4051, 0, 4088, 0, 0, 28, 4070, 0, 1, 381, 22, 377, 2, 1, 7, 0, 4039, 4029, 27, 13, 53, 1955] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 4057, 3819, 8, 0, 0, 19, 0, 180, 4, 4063, 13, 4093, 274, 0, 0, 4095, 3, 0, 1375, 0, 144, 0, 5, 0, 0, 4096, 4079, 1, 54, 0, 2386] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 4095, 3162, 0, 1144, 0, 0, 0, 5, 0, 4096, 0, 4094, 0, 0, 1, 4096, 0, 0, 3514, 0, 219, 0, 0, 0, 0, 4096, 4096, 0, 31, 0, 119] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 4093, 3897, 0, 196, 0, 1, 0, 0, 0, 4096, 9, 4090, 34, 7, 5, 4095, 0, 0, 3922, 0, 18, 0, 1, 0, 0, 4096, 4078, 0, 14, 0, 116] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 4075, 0, 347, 0, 0, 0, 137, 0, 4096, 0, 4096, 1, 0, 0, 4096, 0, 0, 1611, 0, 0, 0, 0, 0, 0, 4096, 3559, 0, 0, 0, 2558] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 3257, 0, 1237, 0, 0, 0, 70, 0, 4096, 4, 4096, 6, 1, 11, 4096, 0, 0, 3667, 0, 4, 0, 0, 0, 3, 4096, 4000, 0, 0, 0, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 3653, 0, 109, 0, 0, 0, 169, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 2579, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 61, 0, 1624] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 575, 2184, 0, 0, 0, 0, 0, 2288, 0, 0, 0, 3290, 0, 0, 0, 0, 0, 3973, 1, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 1928, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 93, 4096, 0, 0, 0, 0, 0, 30, 0, 3418, 2167, 202, 123, 43] rerouted_tokens_per_ep_rank=[2760, 5578, 3974, 4053, 2229, 4002, 4189, 5983] avg=4096.0000 max=5983 max_rank=7 min=2229 min_rank=4 imbalance_ratio=1.460693 max_load_reduction=26785 max_load_reduction_pct=0.817413 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 575, 3819, 8, 0, 0, 0, 0, 0, 0, 19, 0, 3281, 0, 0, 0, 180, 4, 3941, 13, 0, 0, 0, 0, 4050, 274, 0, 0, 0, 0, 0, 0, 1928, 3, 0, 1375, 0, 0, 0, 0, 0, 144, 0, 5, 4003, 0, 0, 0, 0, 0, 93, 4079, 0, 0, 0, 0, 1, 54, 0, 2386, 2167, 201, 122, 43] rerouted_tokens_per_ep_rank=[4402, 3300, 4138, 4324, 3306, 4152, 4172, 4974] avg=4096.0000 max=4974 max_rank=7 min=3300 min_rank=1 imbalance_ratio=1.214355 max_load_reduction=27794 max_load_reduction_pct=0.848206 moved_to_echo=16604 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 580, 4075, 0, 0, 0, 0, 0, 347, 0, 0, 0, 3313, 0, 0, 0, 137, 0, 3973, 0, 0, 0, 0, 0, 4053, 1, 0, 0, 0, 0, 0, 0, 1929, 0, 0, 1611, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 93, 3559, 0, 0, 0, 0, 0, 0, 0, 2558, 2167, 203, 123, 43] rerouted_tokens_per_ep_rank=[4655, 3660, 4110, 4054, 3540, 4003, 3652, 5094] avg=4096.0000 max=5094 max_rank=7 min=3540 min_rank=4 imbalance_ratio=1.243652 max_load_reduction=27674 max_load_reduction_pct=0.844543 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 558, 4096, 9, 0, 0, 0, 0, 1010, 42, 5, 0, 3230, 7, 0, 0, 445, 28, 3924, 0, 0, 0, 0, 0, 4040, 0, 0, 28, 0, 0, 0, 0, 1914, 0, 1, 381, 0, 0, 0, 0, 22, 377, 2, 1, 3955, 0, 0, 0, 7, 0, 77, 4029, 0, 0, 0, 0, 27, 13, 53, 1955, 2156, 201, 127, 48] rerouted_tokens_per_ep_rank=[4663, 4294, 4397, 4068, 2296, 4357, 4113, 4580] avg=4096.0000 max=4663 max_rank=0 min=2296 min_rank=4 imbalance_ratio=1.138428 max_load_reduction=28105 max_load_reduction_pct=0.857697 moved_to_echo=15346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 579, 3897, 0, 0, 0, 0, 0, 196, 0, 1, 0, 3311, 0, 0, 0, 0, 0, 3973, 9, 0, 0, 0, 0, 4047, 34, 7, 5, 0, 0, 0, 0, 1928, 0, 0, 3922, 0, 0, 0, 0, 0, 18, 0, 1, 4003, 0, 0, 0, 0, 0, 93, 4078, 0, 0, 0, 0, 0, 14, 0, 116, 2167, 203, 123, 43] rerouted_tokens_per_ep_rank=[4476, 3508, 3982, 4093, 5850, 4022, 4171, 2666] avg=4096.0000 max=5850 max_rank=4 min=2666 min_rank=7 imbalance_ratio=1.428223 max_load_reduction=26918 max_load_reduction_pct=0.821472 moved_to_echo=16709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 580, 3257, 0, 0, 0, 0, 0, 1237, 0, 0, 0, 3313, 0, 0, 0, 70, 0, 3973, 4, 0, 0, 0, 0, 4053, 6, 1, 11, 0, 0, 0, 0, 1929, 0, 0, 3667, 0, 0, 0, 0, 0, 4, 0, 0, 4003, 0, 0, 0, 0, 3, 93, 4000, 0, 0, 0, 0, 0, 0, 0, 28, 2167, 203, 123, 43] rerouted_tokens_per_ep_rank=[3837, 4550, 4047, 4071, 5596, 4007, 4096, 2564] avg=4096.0000 max=5596 max_rank=4 min=2564 min_rank=7 imbalance_ratio=1.366211 max_load_reduction=27172 max_load_reduction_pct=0.829224 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 580, 3162, 0, 0, 0, 0, 0, 1144, 0, 0, 0, 3312, 0, 0, 0, 5, 0, 3973, 0, 0, 0, 0, 0, 4051, 0, 0, 1, 0, 0, 0, 0, 1929, 0, 0, 3514, 0, 0, 0, 0, 0, 219, 0, 0, 4003, 0, 0, 0, 0, 0, 93, 4096, 0, 0, 0, 0, 0, 31, 0, 119, 2167, 203, 123, 43] rerouted_tokens_per_ep_rank=[3742, 4456, 3978, 4052, 5443, 4222, 4189, 2686] avg=4096.0000 max=5443 max_rank=4 min=2686 min_rank=7 imbalance_ratio=1.328857 max_load_reduction=27325 max_load_reduction_pct=0.833893 moved_to_echo=16540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 580, 3653, 0, 0, 0, 0, 0, 109, 0, 0, 0, 3313, 0, 0, 0, 169, 0, 3973, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 1929, 0, 0, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 93, 4093, 0, 0, 0, 0, 0, 61, 0, 1624, 2167, 203, 123, 43] rerouted_tokens_per_ep_rank=[4233, 3422, 4142, 4053, 4508, 4003, 4186, 4221] avg=4096.0000 max=4508 max_rank=4 min=3422 min_rank=1 imbalance_ratio=1.100586 max_load_reduction=28260 max_load_reduction_pct=0.862427 moved_to_echo=16918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[15488, 128, 128, 17408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 128, 12288, 17408, 1664, 1024, 384] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 4608, 28160, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6400, 128, 128, 0, 26368, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1024, 128, 31744, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 768, 128, 128, 32000, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 384, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 768, 32128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 4091, 59, 19, 0, 0, 7, 0, 5, 4095, 276, 4044, 0, 0, 0, 4096, 4091, 0, 5, 2, 3969, 0, 179, 71, 0, 0, 1, 0, 0, 3661, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4093, 2, 0, 1, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4088, 4094, 222, 1, 1, 0, 0, 1, 0, 4094, 198, 4096, 0, 0, 0, 4092, 4094, 0, 2, 0, 3678, 7, 0, 0, 0, 4, 0, 0, 0, 4093, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 4090, 0, 4081, 0, 0, 0, 4095, 4096, 0, 0, 0, 544, 5, 0, 3552, 13, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4, 4014, 4060, 870, 14, 0, 0, 0, 6, 54, 3956, 0, 4072, 33, 21, 85, 4050, 4061, 0, 2, 36, 1471, 495, 1207, 279, 0, 0, 30, 0, 34, 3896, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 4087, 0, 4044, 0, 0, 0, 4096, 4094, 0, 0, 0, 3539, 512, 2, 109, 0, 3, 0, 0, 0, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4085, 4096, 0, 0, 3, 0, 0, 0, 0, 4075, 0, 4096, 0, 0, 0, 4093, 4096, 0, 0, 0, 4063, 42, 19, 5, 4, 12, 0, 0, 0, 4079, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 40, 0, 0, 0, 0, 0, 0, 4096, 435, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 3335, 0, 0, 285, 0, 1, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3958, 0, 59, 0, 0, 0, 0, 19, 0, 0, 7, 4091, 0, 0, 0, 0, 5, 3996, 276, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 5, 0, 0, 0, 0, 2, 3969, 0, 179, 535, 137, 23, 0, 71, 0, 0, 1, 3549, 0, 0, 0, 0, 0, 3661, 2, 76, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4117, 4277, 4044, 4108, 4845, 3621, 3739] avg=4096.0000 max=4845 max_rank=5 min=3621 min_rank=6 imbalance_ratio=1.182861 max_load_reduction=27923 max_load_reduction_pct=0.852142 moved_to_echo=16313 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3951, 0, 222, 0, 0, 0, 0, 1, 1, 0, 0, 4093, 0, 0, 0, 1, 0, 3995, 198, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 7, 0, 2, 0, 0, 0, 0, 0, 3678, 7, 0, 535, 137, 23, 0, 0, 0, 4, 0, 3552, 1, 0, 0, 0, 0, 4093, 3, 76, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4095, 4194, 4096, 4101, 4380, 3557, 4172] avg=4096.0000 max=4380 max_rank=5 min=3557 min_rank=6 imbalance_ratio=1.069336 max_load_reduction=28388 max_load_reduction_pct=0.866333 moved_to_echo=16210 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 536, 137, 23, 0, 1, 0, 0, 0, 3554, 0, 0, 0, 0, 0, 4096, 0, 76, 0, 0, 0] rerouted_tokens_per_ep_rank=[3959, 4096, 3997, 4096, 4102, 4791, 3555, 4172] avg=4096.0000 max=4791 max_rank=5 min=3555 min_rank=6 imbalance_ratio=1.169678 max_load_reduction=27977 max_load_reduction_pct=0.853790 moved_to_echo=16620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3959, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 544, 5, 0, 536, 137, 23, 0, 3552, 13, 0, 0, 3554, 0, 0, 0, 0, 0, 4096, 0, 76, 0, 0, 0] rerouted_tokens_per_ep_rank=[3963, 4096, 3991, 4081, 4101, 1245, 7119, 4172] avg=4096.0000 max=7119 max_rank=6 min=1245 min_rank=5 imbalance_ratio=1.738037 max_load_reduction=25649 max_load_reduction_pct=0.782745 moved_to_echo=16637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3948, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 3977, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 6, 0, 0, 0, 0, 0, 0, 0, 4063, 42, 19, 536, 137, 23, 0, 5, 4, 12, 0, 3554, 0, 0, 0, 0, 0, 4079, 0, 75, 0, 0, 0] rerouted_tokens_per_ep_rank=[3948, 4099, 3977, 4096, 4099, 4820, 3575, 4154] avg=4096.0000 max=4820 max_rank=5 min=3575 min_rank=6 imbalance_ratio=1.176758 max_load_reduction=27948 max_load_reduction_pct=0.852905 moved_to_echo=16648 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 3873, 0, 870, 0, 0, 0, 0, 14, 0, 0, 0, 4060, 0, 0, 0, 6, 54, 3852, 0, 0, 0, 0, 0, 4072, 33, 21, 85, 4, 0, 0, 0, 4050, 0, 0, 2, 0, 0, 0, 0, 36, 1471, 495, 1207, 534, 141, 24, 0, 279, 0, 0, 30, 3527, 0, 0, 0, 0, 34, 3896, 18, 76, 0, 0, 0] rerouted_tokens_per_ep_rank=[4747, 4074, 3912, 4215, 4052, 3908, 3836, 4024] avg=4096.0000 max=4747 max_rank=0 min=3836 min_rank=6 imbalance_ratio=1.158936 max_load_reduction=28021 max_load_reduction_pct=0.855133 moved_to_echo=15820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3959, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 3539, 512, 2, 535, 137, 23, 0, 109, 0, 3, 0, 3552, 0, 0, 0, 0, 0, 4087, 0, 76, 0, 0, 0] rerouted_tokens_per_ep_rank=[3962, 4096, 3988, 4044, 4103, 4748, 3664, 4163] avg=4096.0000 max=4748 max_rank=5 min=3664 min_rank=6 imbalance_ratio=1.159180 max_load_reduction=28020 max_load_reduction_pct=0.855103 moved_to_echo=16678 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3959, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3997, 435, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 3335, 0, 0, 536, 137, 23, 0, 285, 0, 1, 0, 3554, 1, 0, 0, 0, 0, 4096, 0, 76, 0, 0, 0] rerouted_tokens_per_ep_rank=[3999, 4095, 4432, 4096, 4102, 4031, 3841, 4172] avg=4096.0000 max=4432 max_rank=2 min=3841 min_rank=6 imbalance_ratio=1.082031 max_load_reduction=28336 max_load_reduction_pct=0.864746 moved_to_echo=16146 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 31616, 0, 1280, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 24704, 1152, 1408, 4352, 1152, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4352, 128, 128, 128, 28416, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32128, 128, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 31872, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 4036, 0, 0, 6, 4068, 0, 0, 4085, 0, 4096, 3, 2, 0, 0, 7, 4093, 50, 0, 1, 0, 1, 36, 0, 0, 4094, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4090, 4096, 4095, 0, 0, 0, 1974, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 2134, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 0, 2795, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 62, 1239, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[46, 4064, 4051, 3790, 38, 0, 128, 4085, 13, 0, 4060, 1, 3996, 0, 0, 2, 0, 0, 3820, 298, 0, 0, 0, 30, 198, 139, 9, 3931, 44, 0, 25, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4093, 4090, 4091, 0, 0, 9, 3325, 1, 7, 3942, 4, 4077, 0, 0, 0, 0, 15, 4091, 0, 11, 0, 0, 0, 0, 10, 1, 4085, 0, 112, 780, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4039, 4096, 4096, 0, 22, 0, 547, 0, 1, 4086, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 3575, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 0, 269, 1, 0, 4078, 0, 4096, 0, 0, 0, 0, 0, 4096, 80, 0, 0, 0, 0, 34, 0, 0, 4096, 0, 907, 2815, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4093, 0, 0, 0, 2023, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 2076, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 0, 4036, 0, 0, 0, 0, 0, 0, 6, 4068, 1677, 12, 0, 0, 0, 0, 4085, 0, 19, 6, 0, 0, 4096, 3, 2, 0, 13, 0, 0, 0, 0, 7, 4074, 50, 0, 0, 0, 0, 0, 1, 0, 1, 4093, 1, 0, 0, 36, 0, 0, 4063, 0, 0, 0, 0, 0, 1, 0, 0, 2375, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 5763, 4110, 4114, 4131, 4096, 4099, 2376] avg=4096.0000 max=5763 max_rank=1 min=2376 min_rank=7 imbalance_ratio=1.406982 max_load_reduction=27005 max_load_reduction_pct=0.824127 moved_to_echo=14702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1974, 1675, 12, 0, 0, 0, 0, 4096, 0, 19, 6, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 2134, 0, 2371, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 3661, 4121, 4109, 4071, 4097, 4066, 4505] avg=4096.0000 max=4505 max_rank=7 min=3661 min_rank=1 imbalance_ratio=1.099854 max_load_reduction=28263 max_load_reduction_pct=0.862518 moved_to_echo=16739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2795, 1677, 12, 0, 0, 0, 0, 4094, 0, 19, 6, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 2, 62, 1239, 0, 2375, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4484, 4119, 4109, 4077, 4097, 4065, 3678] avg=4096.0000 max=4484 max_rank=1 min=3678 min_rank=7 imbalance_ratio=1.094727 max_load_reduction=28284 max_load_reduction_pct=0.863159 moved_to_echo=15917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 0, 4091, 0, 0, 0, 0, 0, 0, 9, 3325, 1676, 12, 0, 0, 1, 7, 3942, 4, 19, 6, 0, 0, 4077, 0, 0, 0, 13, 0, 0, 0, 0, 15, 4072, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4090, 1, 0, 0, 0, 10, 1, 4054, 0, 0, 0, 0, 0, 112, 780, 24, 2373, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 5022, 3979, 4090, 4087, 4102, 4065, 3289] avg=4096.0000 max=5022 max_rank=1 min=3289 min_rank=7 imbalance_ratio=1.226074 max_load_reduction=27746 max_load_reduction_pct=0.846741 moved_to_echo=15543 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 42, 0, 4096, 0, 0, 0, 0, 0, 22, 0, 547, 1654, 12, 0, 0, 0, 1, 4086, 0, 19, 6, 0, 0, 4096, 0, 1, 0, 13, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 17, 3575, 0, 2342, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 2235, 4112, 4110, 4077, 4097, 4065, 5934] avg=4096.0000 max=5934 max_rank=7 min=2235 min_rank=1 imbalance_ratio=1.448730 max_load_reduction=26834 max_load_reduction_pct=0.818909 moved_to_echo=18173 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[46, 29, 0, 3790, 0, 0, 0, 0, 38, 0, 128, 4085, 1670, 12, 0, 0, 13, 0, 4060, 1, 22, 6, 0, 0, 3996, 0, 0, 2, 20, 0, 0, 0, 0, 0, 3794, 298, 0, 0, 0, 0, 0, 0, 0, 30, 4051, 4, 0, 0, 198, 139, 9, 3897, 0, 0, 0, 0, 44, 0, 25, 0, 2361, 0, 0, 0] rerouted_tokens_per_ep_rank=[3865, 5933, 4102, 4018, 4092, 4085, 4243, 2430] avg=4096.0000 max=5933 max_rank=1 min=2430 min_rank=7 imbalance_ratio=1.448486 max_load_reduction=26835 max_load_reduction_pct=0.818939 moved_to_echo=14850 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 269, 1677, 12, 0, 0, 1, 0, 4078, 0, 19, 6, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4077, 80, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 34, 0, 0, 4065, 0, 0, 0, 0, 0, 907, 2815, 8, 2375, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 1958, 4104, 4109, 4157, 4097, 4099, 6105] avg=4096.0000 max=6105 max_rank=7 min=1958 min_rank=1 imbalance_ratio=1.490479 max_load_reduction=26663 max_load_reduction_pct=0.813690 moved_to_echo=18458 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2023, 1677, 12, 0, 0, 0, 0, 4096, 0, 19, 6, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 2076, 0, 2375, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 3712, 4121, 4109, 4076, 4097, 4066, 4451] avg=4096.0000 max=4451 max_rank=7 min=3712 min_rank=1 imbalance_ratio=1.086670 max_load_reduction=28317 max_load_reduction_pct=0.864166 moved_to_echo=16690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 1152, 12672, 128, 19072, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 256, 19200, 13440, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 384, 0, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32384, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32640, 128, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 256, 128, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 4095, 0, 414, 0, 0, 1304, 0, 0, 0, 4096, 0, 0, 0, 2364, 0, 0, 4092, 4094, 0, 17, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[3, 0, 0, 13, 4087, 0, 0, 0, 0, 16, 4094, 3993, 0, 102, 0, 2, 862, 1, 0, 0, 4095, 2, 8, 1, 3212, 0, 0, 4080, 4072, 0, 214, 3911] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4029, 0, 1990, 0, 3, 742, 0, 0, 5, 4096, 0, 0, 38, 3370, 0, 0, 2052, 4096, 0, 59, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[53, 68, 0, 0, 4019, 31, 1, 21, 55, 0, 3917, 4027, 81, 10, 0, 9, 3371, 29, 1, 0, 4038, 37, 0, 0, 165, 34, 2, 3884, 4055, 0, 907, 3953] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4044, 0, 3, 1, 1, 1, 4067, 4080, 42, 622, 0, 0, 1554, 0, 2, 3, 4096, 2, 0, 0, 1837, 0, 0, 3246, 4024, 0, 1368, 3775] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 2, 553, 3, 0, 1595, 0, 0, 0, 4093, 0, 0, 0, 4094, 0, 0, 2139, 4093, 18, 32, 3857] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 4096, 42, 3, 0, 0, 3415, 0, 0, 0, 4095, 0, 0, 0, 3918, 0, 0, 712, 4094, 0, 157, 4052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 4090, 5, 2826, 0, 0, 3342, 0, 0, 0, 4096, 0, 0, 0, 766, 0, 0, 1254, 4094, 0, 10, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 13, 4072, 0, 0, 0, 4087, 0, 0, 0, 6, 3, 0, 0, 0, 16, 24, 3993, 0, 0, 0, 0, 0, 102, 0, 2, 3278, 0, 0, 0, 862, 1, 0, 0, 1607, 792, 220, 0, 4092, 2, 8, 1, 0, 0, 0, 0, 3212, 0, 0, 2473, 0, 0, 0, 0, 0, 0, 214, 3685, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4096, 4033, 3382, 3482, 4103, 5685, 3899] avg=4096.0000 max=5685 max_rank=6 min=3382 min_rank=3 imbalance_ratio=1.387939 max_load_reduction=27083 max_load_reduction_pct=0.826508 moved_to_echo=17169 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 7, 3, 0, 0, 0, 4, 25, 4095, 0, 0, 0, 0, 0, 414, 0, 0, 3279, 0, 0, 0, 1304, 0, 0, 0, 1611, 792, 230, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2364, 0, 0, 2481, 0, 0, 0, 0, 0, 0, 17, 3859, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4106, 4124, 3693, 3937, 4093, 4845, 3876] avg=4096.0000 max=4845 max_rank=6 min=3693 min_rank=3 imbalance_ratio=1.182861 max_load_reduction=27923 max_load_reduction_pct=0.852142 moved_to_echo=16751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4024, 0, 0, 0, 4044, 0, 3, 1, 6, 3, 0, 0, 1, 1, 24, 4080, 0, 0, 0, 0, 42, 622, 0, 0, 3256, 0, 0, 0, 1554, 0, 2, 3, 1278, 787, 212, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 1837, 0, 0, 1968, 0, 0, 0, 0, 0, 0, 1368, 3557, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4057, 4106, 3920, 3836, 4095, 3805, 4925] avg=4096.0000 max=4925 max_rank=7 min=3805 min_rank=6 imbalance_ratio=1.202393 max_load_reduction=27843 max_load_reduction_pct=0.849701 moved_to_echo=16661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 1, 6, 3, 0, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 2, 553, 3, 0, 3279, 0, 0, 0, 1595, 0, 0, 0, 842, 792, 217, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1297, 0, 0, 0, 0, 0, 18, 32, 3634, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4106, 4121, 3837, 3446, 4090, 5391, 3684] avg=4096.0000 max=5391 max_rank=6 min=3446 min_rank=4 imbalance_ratio=1.316162 max_load_reduction=27377 max_load_reduction_pct=0.835480 moved_to_echo=16611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 7, 3, 0, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 42, 3, 0, 0, 3267, 0, 0, 0, 3415, 0, 0, 0, 280, 789, 227, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 0, 432, 0, 0, 0, 0, 0, 0, 157, 3818, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4106, 4120, 3312, 4711, 4092, 4350, 3975] avg=4096.0000 max=4711 max_rank=4 min=3312 min_rank=3 imbalance_ratio=1.150146 max_load_reduction=28057 max_load_reduction_pct=0.856232 moved_to_echo=17128 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[53, 68, 0, 0, 4055, 1, 0, 0, 4019, 31, 1, 21, 9, 4, 0, 0, 55, 0, 12, 4027, 0, 0, 0, 0, 81, 10, 0, 9, 3142, 0, 0, 0, 3371, 29, 1, 0, 1534, 762, 225, 0, 4034, 37, 0, 0, 0, 0, 0, 0, 165, 34, 2, 2350, 0, 0, 0, 0, 0, 0, 907, 3719, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4177, 4085, 4094, 3242, 5922, 4071, 2551, 4626] avg=4096.0000 max=5922 max_rank=4 min=2551 min_rank=6 imbalance_ratio=1.445801 max_load_reduction=26846 max_load_reduction_pct=0.819275 moved_to_echo=17170 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 7, 3, 0, 0, 0, 0, 25, 4029, 0, 0, 0, 0, 0, 1990, 0, 3, 3279, 0, 0, 0, 742, 0, 0, 5, 808, 792, 230, 0, 4093, 0, 0, 38, 0, 0, 0, 0, 3370, 0, 0, 1244, 0, 0, 0, 0, 0, 0, 59, 3859, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4106, 4054, 5272, 2577, 4131, 4614, 3918] avg=4096.0000 max=5272 max_rank=3 min=2577 min_rank=4 imbalance_ratio=1.287109 max_load_reduction=27496 max_load_reduction_pct=0.839111 moved_to_echo=15240 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 7, 3, 0, 0, 0, 1, 25, 4090, 0, 0, 0, 0, 5, 2826, 0, 0, 3279, 0, 0, 0, 3342, 0, 0, 0, 493, 792, 230, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 766, 0, 0, 761, 0, 0, 0, 0, 0, 0, 10, 3855, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4106, 4116, 6110, 4857, 4093, 1527, 3865] avg=4096.0000 max=6110 max_rank=3 min=1527 min_rank=6 imbalance_ratio=1.491699 max_load_reduction=26658 max_load_reduction_pct=0.813538 moved_to_echo=14342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[16256, 128, 128, 128, 8576, 6400, 1792, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 6528, 128, 128, 26112, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19840, 128, 128, 13056, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 2816, 30080, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 256, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 22, 0, 0, 0, 1, 2, 4096, 4090, 4059, 0, 0, 3674, 1, 0, 3, 0, 4092, 84, 4096, 0, 0, 0, 0, 297, 11, 4094, 4091, 5, 47, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 10, 1, 0, 9, 4092, 4091, 4094, 0, 0, 3705, 0, 2, 9, 0, 4088, 62, 4095, 0, 0, 0, 0, 417, 0, 3947, 4091, 50, 1, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4095, 4096, 4093, 0, 0, 2958, 0, 0, 0, 0, 4096, 305, 4096, 0, 0, 0, 0, 4049, 68, 737, 4096, 35, 43, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 15, 0, 0, 0, 4090, 4065, 4089, 0, 0, 215, 0, 0, 0, 0, 4089, 0, 4093, 0, 0, 0, 0, 3912, 4, 4093, 4096, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[50, 29, 0, 16, 0, 12, 44, 32, 4043, 3728, 3940, 9, 58, 4074, 0, 2, 3, 0, 4067, 140, 4046, 0, 0, 15, 43, 1, 0, 4060, 4026, 315, 0, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 2, 0, 0, 0, 4096, 4080, 4094, 0, 0, 113, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4043, 0, 4049, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 4000, 0, 0, 3352, 0, 0, 3, 0, 4096, 15, 4096, 0, 0, 0, 0, 430, 92, 4096, 4096, 0, 297, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 1365, 0, 0, 0, 0, 4096, 5, 4096, 0, 0, 0, 0, 2721, 8, 4093, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 22, 0, 4006, 73, 8, 0, 0, 0, 1, 2, 4088, 0, 0, 0, 0, 4090, 53, 0, 0, 0, 0, 0, 0, 3674, 1, 0, 1680, 91, 67, 0, 3, 0, 4021, 84, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 297, 11, 2341, 0, 0, 0, 0, 4000, 5, 47, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4091, 4143, 5513, 4108, 4100, 2649, 4054] avg=4096.0000 max=5513 max_rank=3 min=2649 min_rank=6 imbalance_ratio=1.345947 max_load_reduction=27255 max_load_reduction_pct=0.831757 moved_to_echo=14911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4041, 70, 8, 0, 10, 1, 0, 9, 4084, 0, 0, 0, 0, 4091, 53, 0, 0, 0, 0, 0, 0, 3705, 0, 2, 1620, 91, 67, 0, 9, 0, 4017, 62, 0, 0, 0, 0, 4095, 0, 0, 0, 4, 0, 0, 0, 0, 417, 0, 2257, 0, 0, 0, 0, 4000, 50, 1, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4104, 4144, 5485, 4088, 4099, 2674, 4053] avg=4096.0000 max=5485 max_rank=3 min=2674 min_rank=6 imbalance_ratio=1.339111 max_load_reduction=27283 max_load_reduction_pct=0.832611 moved_to_echo=14914 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 4036, 73, 8, 0, 15, 0, 0, 0, 4082, 0, 0, 0, 0, 4065, 53, 0, 0, 0, 0, 0, 0, 215, 0, 0, 1680, 91, 67, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4, 0, 0, 0, 0, 3912, 4, 2340, 0, 0, 0, 0, 4005, 0, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4097, 4118, 2053, 4018, 4097, 6256, 4007] avg=4096.0000 max=6256 max_rank=6 min=2053 min_rank=3 imbalance_ratio=1.527344 max_load_reduction=26512 max_load_reduction_pct=0.809082 moved_to_echo=18378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 3948, 73, 8, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 52, 0, 0, 0, 0, 0, 0, 3352, 0, 0, 1681, 91, 67, 0, 3, 0, 4025, 15, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 430, 92, 2342, 0, 0, 0, 0, 4005, 0, 297, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4088, 4148, 5191, 4043, 4100, 2864, 4302] avg=4096.0000 max=5191 max_rank=3 min=2864 min_rank=6 imbalance_ratio=1.267334 max_load_reduction=27577 max_load_reduction_pct=0.841583 moved_to_echo=15309 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4041, 72, 8, 0, 2, 0, 0, 0, 4088, 0, 0, 0, 0, 4080, 53, 0, 0, 0, 0, 0, 0, 113, 0, 0, 1661, 91, 67, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4043, 0, 2316, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4090, 4133, 1932, 4025, 4100, 6359, 4005] avg=4096.0000 max=6359 max_rank=6 min=1932 min_rank=3 imbalance_ratio=1.552490 max_load_reduction=26409 max_load_reduction_pct=0.805939 moved_to_echo=18489 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[50, 29, 0, 16, 3893, 77, 9, 0, 0, 12, 44, 32, 4034, 0, 0, 0, 0, 3728, 47, 9, 0, 0, 0, 0, 58, 4074, 0, 2, 1669, 97, 73, 0, 3, 0, 3983, 140, 0, 0, 0, 0, 4046, 0, 0, 15, 11, 0, 0, 0, 43, 1, 0, 2314, 0, 0, 0, 0, 3929, 315, 0, 15, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4122, 3784, 5973, 4126, 4072, 2358, 4259] avg=4096.0000 max=5973 max_rank=3 min=2358 min_rank=6 imbalance_ratio=1.458252 max_load_reduction=26795 max_load_reduction_pct=0.817719 moved_to_echo=14815 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4040, 13, 8, 0, 1, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 53, 0, 0, 0, 0, 0, 0, 2958, 0, 0, 302, 91, 67, 0, 0, 0, 4025, 305, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4049, 68, 422, 0, 0, 0, 0, 4005, 35, 43, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4088, 4149, 3418, 4330, 4100, 4539, 4083] avg=4096.0000 max=4539 max_rank=6 min=3418 min_rank=3 imbalance_ratio=1.108154 max_load_reduction=28229 max_load_reduction_pct=0.861481 moved_to_echo=17052 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4041, 73, 8, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 53, 0, 0, 0, 0, 0, 0, 1365, 0, 0, 1680, 91, 67, 0, 0, 0, 4025, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 2721, 8, 2340, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4088, 4149, 3203, 4030, 4100, 5069, 4005] avg=4096.0000 max=5069 max_rank=6 min=3203 min_rank=3 imbalance_ratio=1.237549 max_load_reduction=27699 max_load_reduction_pct=0.845306 moved_to_echo=17204 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 19456, 128, 128, 12032, 768, 640, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 15872, 256, 16768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32128, 640, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32256, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32000, 512, 512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32384, 512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4093, 0, 4096, 0, 0, 0, 5, 4091, 0, 4053, 0, 0, 29, 0, 8, 0, 4088, 4096, 0, 1, 0, 4061, 0, 0, 3, 4066, 0, 77, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4092, 0, 4072, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 190, 0, 0, 1, 4096, 4095, 0, 19, 1, 3906, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4077, 0, 4090, 3, 4, 0, 1, 4096, 0, 4096, 0, 0, 9, 0, 3, 0, 4096, 4096, 0, 14, 0, 4084, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4094, 6, 4088, 4, 25, 2, 0, 4095, 0, 4088, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 145, 0, 3703, 0, 0, 0, 4076, 0, 239, 10, 3, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 4028, 2, 99, 3, 63, 4052, 5, 3989, 1, 7, 122, 36, 0, 0, 4063, 4089, 149, 0, 0, 3987, 0, 0, 2, 3801, 38, 3, 83, 2, 0, 48] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4016, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 291, 0, 0, 0, 4096, 4068, 2, 29, 0, 4027, 40, 0, 0, 3911, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 18, 0, 0, 0, 4096, 4094, 0, 0, 0, 4074, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 4073, 0, 0, 0, 0, 0, 4053, 0, 0, 18, 0, 0, 0, 29, 0, 8, 0, 4004, 13, 0, 0, 4050, 0, 0, 1, 0, 0, 0, 0, 0, 4061, 0, 0, 77, 21, 0, 0, 3, 4066, 0, 77, 17, 0, 0, 0, 0, 1, 0, 0, 4083, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4078, 4071, 4054, 4051, 4159, 4163, 4084] avg=4096.0000 max=4163 max_rank=6 min=4051 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4007, 14, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 77, 21, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4078, 4114, 4021, 4058, 4194, 4113, 4082] avg=4096.0000 max=4194 max_rank=5 min=4021 min_rank=3 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16447 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 6, 4088, 4, 0, 0, 0, 0, 25, 2, 0, 4077, 0, 0, 0, 0, 0, 4088, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4005, 14, 0, 0, 4055, 0, 0, 145, 0, 0, 0, 0, 0, 3703, 0, 0, 77, 21, 0, 0, 0, 4076, 0, 239, 17, 0, 0, 0, 10, 3, 1, 0, 4082, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4104, 4106, 4019, 4200, 3801, 4332, 4096] avg=4096.0000 max=4332 max_rank=6 min=3801 min_rank=5 imbalance_ratio=1.057617 max_load_reduction=28436 max_load_reduction_pct=0.867798 moved_to_echo=16429 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 4090, 3, 0, 0, 0, 0, 4, 0, 1, 4078, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 9, 0, 3, 0, 3988, 14, 0, 0, 4058, 0, 0, 14, 0, 0, 0, 0, 0, 4084, 0, 0, 76, 21, 0, 0, 0, 4096, 0, 3, 17, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4083, 4114, 4014, 4072, 4181, 4116, 4082] avg=4096.0000 max=4181 max_rank=5 min=4014 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16451 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 190, 0, 0, 1, 4003, 14, 0, 0, 4058, 0, 0, 19, 0, 0, 0, 0, 1, 3906, 0, 0, 77, 21, 0, 0, 0, 4096, 0, 9, 17, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4077, 4114, 4208, 4077, 4005, 4122, 4081] avg=4096.0000 max=4208 max_rank=3 min=4005 min_rank=5 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16285 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 4028, 2, 0, 0, 0, 0, 99, 3, 63, 4027, 0, 0, 0, 0, 5, 3989, 1, 7, 19, 0, 0, 0, 122, 36, 0, 0, 4009, 14, 0, 0, 4014, 0, 149, 0, 0, 0, 0, 0, 0, 3987, 0, 0, 80, 26, 0, 0, 2, 3801, 38, 3, 23, 6, 0, 0, 83, 2, 0, 48, 4075, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4192, 4021, 4181, 4163, 4093, 3873, 4208] avg=4096.0000 max=4208 max_rank=7 min=3873 min_rank=6 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16337 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 291, 0, 0, 0, 3928, 14, 0, 0, 4058, 0, 2, 29, 0, 0, 0, 0, 0, 4027, 40, 0, 75, 21, 0, 0, 0, 3911, 0, 1, 17, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4078, 4114, 4233, 4089, 4163, 3929, 4054] avg=4096.0000 max=4233 max_rank=3 min=3929 min_rank=6 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16235 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 18, 0, 0, 0, 4007, 13, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 77, 21, 0, 0, 0, 4096, 0, 7, 17, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4078, 4114, 4038, 4058, 4172, 4120, 4081] avg=4096.0000 max=4172 max_rank=5 min=4038 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16431 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 0, 256, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32256, 128, 384, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32000, 128, 0, 640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 128, 128, 128, 32000, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32640, 128, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 2754, 0, 4095, 0, 4095, 2, 0, 3, 4096, 158, 4096, 0, 1442, 0, 0, 0, 0, 3764, 0, 0, 0, 74, 0, 0, 0, 0, 4096, 0, 4093, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 313, 0, 4096, 0, 4095, 0, 0, 0, 4096, 1, 4096, 0, 3787, 0, 0, 0, 0, 4086, 0, 1, 0, 5, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[40, 16, 1384, 1, 4032, 0, 3962, 0, 76, 25, 4078, 140, 4056, 0, 3903, 5, 50, 0, 30, 2756, 52, 0, 0, 116, 1, 0, 0, 0, 4061, 0, 3979, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 3463, 6, 4096, 0, 4093, 0, 0, 0, 4094, 0, 4092, 0, 742, 0, 0, 0, 0, 3970, 0, 19, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 3683, 0, 4094, 8, 4047, 0, 0, 0, 4096, 4, 4092, 0, 559, 0, 0, 0, 0, 3992, 0, 0, 2, 0, 0, 0, 0, 0, 4093, 2, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 385, 0, 4096, 0, 4092, 0, 0, 0, 4092, 0, 4096, 0, 4061, 0, 0, 0, 0, 3739, 0, 3, 0, 12, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 5, 4015, 0, 4096, 0, 4095, 3, 0, 1, 4093, 0, 4091, 0, 1093, 0, 0, 9, 0, 2978, 0, 7, 1, 0, 0, 8, 0, 0, 4087, 96, 4090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 603, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 4096, 0, 3167, 0, 0, 0, 0, 4091, 0, 324, 0, 4, 0, 0, 0, 0, 4096, 2, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2754, 0, 1976, 39, 0, 0, 24, 0, 4095, 2, 0, 0, 0, 0, 0, 3, 4049, 158, 0, 0, 0, 0, 1755, 0, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 3764, 365, 47, 0, 0, 0, 0, 0, 74, 4025, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4769, 4121, 4210, 3197, 4176, 4099, 4100, 4096] avg=4096.0000 max=4769 max_rank=0 min=3197 min_rank=3 imbalance_ratio=1.164307 max_load_reduction=27999 max_load_reduction_pct=0.854462 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 313, 0, 1976, 39, 0, 0, 24, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4049, 1, 0, 0, 0, 0, 1755, 0, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 365, 47, 0, 0, 0, 1, 0, 5, 4026, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2328, 4119, 4050, 5542, 4498, 4032, 4100, 4099] avg=4096.0000 max=5542 max_rank=3 min=2328 min_rank=0 imbalance_ratio=1.353027 max_load_reduction=27226 max_load_reduction_pct=0.830872 moved_to_echo=16729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4015, 0, 1973, 39, 0, 0, 24, 0, 4095, 3, 0, 0, 0, 0, 0, 1, 4046, 0, 0, 0, 0, 0, 1753, 0, 1093, 0, 0, 0, 0, 0, 0, 9, 0, 2978, 365, 47, 0, 0, 0, 7, 1, 0, 4026, 0, 0, 0, 0, 8, 0, 0, 4085, 7, 0, 0, 2, 96, 4090, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6032, 4122, 4047, 2846, 3399, 4034, 4100, 4188] avg=4096.0000 max=6032 max_rank=0 min=2846 min_rank=3 imbalance_ratio=1.472656 max_load_reduction=26736 max_load_reduction_pct=0.815918 moved_to_echo=15721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3463, 6, 1974, 39, 0, 0, 24, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 1753, 0, 742, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 365, 47, 0, 0, 0, 19, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 0, 3, 1, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5482, 4117, 4047, 2495, 4382, 4045, 4100, 4100] avg=4096.0000 max=5482 max_rank=0 min=2495 min_rank=3 imbalance_ratio=1.338379 max_load_reduction=27286 max_load_reduction_pct=0.832703 moved_to_echo=16627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 385, 0, 1976, 39, 0, 0, 24, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 1755, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 365, 47, 0, 0, 0, 3, 0, 12, 4026, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2400, 4116, 4045, 5816, 4151, 4041, 4100, 4099] avg=4096.0000 max=5816 max_rank=3 min=2400 min_rank=0 imbalance_ratio=1.419922 max_load_reduction=26952 max_load_reduction_pct=0.822510 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[40, 16, 1384, 1, 1960, 42, 0, 0, 12, 0, 3962, 0, 0, 0, 0, 0, 76, 25, 4025, 140, 0, 0, 0, 0, 1730, 0, 3903, 5, 0, 0, 0, 0, 50, 0, 30, 2756, 366, 53, 0, 0, 52, 0, 0, 116, 3969, 0, 0, 0, 1, 0, 0, 0, 4061, 9, 0, 0, 0, 0, 3979, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3443, 3974, 4266, 5638, 3255, 4137, 4071, 3984] avg=4096.0000 max=5638 max_rank=3 min=3255 min_rank=4 imbalance_ratio=1.376465 max_load_reduction=27130 max_load_reduction_pct=0.827942 moved_to_echo=15447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3683, 0, 1974, 39, 0, 0, 24, 8, 4047, 0, 0, 0, 0, 0, 0, 0, 4049, 4, 0, 0, 0, 0, 1753, 0, 559, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 365, 47, 0, 0, 0, 0, 2, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 4090, 7, 0, 0, 3, 2, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5696, 4079, 4053, 2312, 4404, 4026, 4097, 4101] avg=4096.0000 max=5696 max_rank=0 min=2312 min_rank=3 imbalance_ratio=1.390625 max_load_reduction=27072 max_load_reduction_pct=0.826172 moved_to_echo=16628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 603, 0, 1976, 39, 0, 0, 24, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4049, 1, 0, 0, 0, 0, 1755, 0, 3167, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 365, 47, 0, 0, 0, 324, 0, 4, 4026, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 0, 3, 2, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2618, 4120, 4050, 4922, 4503, 4354, 4100, 4101] avg=4096.0000 max=4922 max_rank=3 min=2618 min_rank=0 imbalance_ratio=1.201660 max_load_reduction=27846 max_load_reduction_pct=0.849792 moved_to_echo=17058 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 16640, 128, 15872, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14080, 0, 18816, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 29440, 2944, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32384, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 384, 128, 256, 32256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4092, 0, 3, 15, 0, 4095, 4082, 0, 0, 4090, 4096, 6, 2, 4051, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 44, 0, 0, 2, 4094, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 5, 4096, 4076, 0, 0, 4096, 4096, 4, 0, 4092, 0, 0, 1, 0, 4086, 0, 2, 0, 0, 6, 17, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 4096, 4077, 6, 2, 4095, 4092, 0, 2, 3968, 0, 0, 0, 32, 4095, 3, 19, 0, 0, 0, 0, 0, 92, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[3986, 26, 0, 103, 0, 4022, 4082, 0, 19, 4034, 3932, 0, 58, 3124, 0, 10, 0, 972, 4073, 20, 148, 1, 0, 0, 0, 14, 0, 0, 4070, 74, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 26, 0, 0, 3, 4096, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4065, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4091, 0, 0, 4004, 0, 0, 0, 1, 4096, 0, 0, 0, 97, 0, 0, 0, 0, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 3877, 263, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 3, 15, 0, 0, 0, 0, 0, 4095, 9, 0, 0, 0, 0, 0, 0, 19, 4096, 6, 0, 0, 0, 0, 2, 4051, 0, 0, 124, 12, 0, 0, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4, 0, 0, 44, 0, 0, 2, 4069, 0, 0, 0, 4094, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4104, 4121, 4189, 3972, 4063, 4115, 4097] avg=4096.0000 max=4189 max_rank=3 min=3972 min_rank=4 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16247 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 6, 0, 0, 0, 0, 2, 19, 4092, 0, 0, 0, 0, 0, 2, 3968, 0, 0, 124, 12, 0, 0, 0, 32, 3971, 3, 0, 0, 0, 0, 19, 0, 0, 0, 4064, 4, 0, 0, 0, 0, 92, 0, 4064, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4111, 4113, 4106, 4006, 4087, 4156, 4099] avg=4096.0000 max=4156 max_rank=6 min=4006 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 9, 0, 0, 0, 0, 0, 0, 19, 4096, 4, 0, 0, 0, 0, 0, 4092, 0, 0, 124, 12, 0, 0, 1, 0, 3962, 0, 0, 0, 0, 0, 2, 0, 0, 6, 4065, 4, 0, 0, 17, 0, 0, 0, 4063, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4110, 4119, 4228, 3963, 4077, 4080, 4099] avg=4096.0000 max=4228 max_rank=3 min=3963 min_rank=4 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16219 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 4096, 26, 0, 124, 12, 0, 0, 0, 3, 3972, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4065, 5, 0, 0, 1, 0, 0, 0, 4083, 0, 0, 0, 4065, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4104, 4115, 4258, 3975, 4071, 4084, 4068] avg=4096.0000 max=4258 max_rank=3 min=3975 min_rank=4 imbalance_ratio=1.039551 max_load_reduction=28510 max_load_reduction_pct=0.870056 moved_to_echo=16198 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 19, 4091, 0, 0, 0, 0, 0, 0, 4004, 0, 0, 124, 12, 0, 0, 0, 1, 3972, 0, 0, 0, 0, 0, 0, 0, 97, 0, 4065, 5, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4095, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4104, 4110, 4140, 3973, 4167, 4083, 4098] avg=4096.0000 max=4167 max_rank=5 min=3973 min_rank=4 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16321 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3978, 26, 0, 103, 0, 0, 0, 0, 0, 4022, 4, 0, 0, 0, 0, 0, 19, 9, 3932, 0, 0, 0, 0, 0, 58, 3124, 0, 10, 129, 19, 0, 0, 0, 972, 3937, 20, 0, 0, 0, 0, 148, 1, 0, 0, 4006, 8, 0, 0, 0, 14, 0, 0, 4070, 0, 0, 0, 4070, 74, 0, 0, 8, 7, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4026, 3960, 3340, 4929, 4163, 4084, 4159] avg=4096.0000 max=4929 max_rank=4 min=3340 min_rank=3 imbalance_ratio=1.203369 max_load_reduction=27839 max_load_reduction_pct=0.849579 moved_to_echo=17335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 8, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 124, 12, 0, 0, 0, 6, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 5, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4090, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4105, 4115, 4231, 3978, 4070, 4083, 4093] avg=4096.0000 max=4231 max_rank=3 min=3978 min_rank=4 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16224 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 3877, 263, 0, 124, 12, 0, 0, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 5, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4052, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4104, 4115, 4276, 3972, 4070, 4083, 4055] avg=4096.0000 max=4276 max_rank=3 min=3972 min_rank=4 imbalance_ratio=1.043945 max_load_reduction=28492 max_load_reduction_pct=0.869507 moved_to_echo=16180 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 128, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31360, 384, 128, 1024, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 1024, 31744, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4021, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 33, 0, 4096, 42, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[7, 3632, 0, 0, 4096, 0, 0, 3, 4088, 4096, 3, 0, 4096, 0, 4095, 0, 0, 2, 0, 0, 5, 253, 4090, 0, 1, 0, 0, 4092, 209, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4084, 0, 0, 0, 4000, 4096, 65, 0, 4094, 0, 4089, 0, 0, 2, 4, 1, 0, 17, 4094, 0, 0, 18, 12, 4094, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4085, 2, 0, 0, 4095, 4096, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 6, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 1634, 15, 0, 4035, 12, 8, 77, 4005, 3927, 1, 0, 4039, 0, 3723, 0, 29, 0, 0, 15, 71, 159, 4067, 0, 0, 150, 0, 3980, 2657, 11, 126, 27] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 3579, 0, 0, 4094, 0, 0, 0, 4096, 4096, 37, 0, 4096, 50, 4086, 0, 0, 0, 0, 0, 0, 6, 4094, 1, 0, 0, 0, 4096, 437, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 3955, 0, 19, 4081, 2, 0, 0, 4075, 4087, 43, 10, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 11, 0, 0, 0, 4096, 103, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4096, 8, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4021, 0, 0, 381, 60, 12, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 44, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 22, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 33, 0, 4084, 0, 0, 0, 0, 42, 0, 0, 0, 3649, 0, 0, 0] rerouted_tokens_per_ep_rank=[4474, 4096, 4104, 4140, 4110, 4036, 4117, 3691] avg=4096.0000 max=4474 max_rank=0 min=3691 min_rank=7 imbalance_ratio=1.092285 max_load_reduction=28294 max_load_reduction_pct=0.863464 moved_to_echo=15954 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[7, 3632, 0, 0, 381, 60, 12, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 4088, 8, 3, 0, 0, 0, 0, 0, 44, 0, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4088, 22, 0, 0, 5, 253, 4030, 0, 0, 0, 0, 0, 1, 0, 0, 4080, 0, 0, 0, 0, 209, 0, 0, 0, 3649, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4099, 4099, 4139, 4112, 4288, 4081, 3858] avg=4096.0000 max=4288 max_rank=5 min=3858 min_rank=7 imbalance_ratio=1.046875 max_load_reduction=28480 max_load_reduction_pct=0.869141 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 381, 60, 12, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4000, 8, 65, 0, 0, 0, 0, 0, 43, 0, 4089, 0, 0, 0, 0, 0, 0, 2, 4, 1, 4088, 22, 0, 0, 0, 17, 4034, 0, 0, 0, 0, 0, 0, 18, 12, 4082, 0, 0, 0, 0, 0, 2, 0, 0, 3648, 0, 0, 0] rerouted_tokens_per_ep_rank=[4549, 4084, 4073, 4132, 4117, 4051, 4112, 3650] avg=4096.0000 max=4549 max_rank=0 min=3650 min_rank=7 imbalance_ratio=1.110596 max_load_reduction=28219 max_load_reduction_pct=0.861176 moved_to_echo=15930 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3579, 0, 0, 381, 60, 12, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 37, 0, 0, 0, 0, 0, 44, 50, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 22, 0, 0, 0, 6, 4034, 1, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 437, 0, 0, 0, 3649, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4094, 4141, 4180, 4110, 4041, 4084, 4086] avg=4096.0000 max=4180 max_rank=3 min=4032 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16321 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3955, 0, 19, 381, 60, 12, 0, 4081, 2, 0, 0, 0, 0, 0, 0, 4075, 8, 43, 10, 0, 0, 0, 0, 44, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 22, 0, 0, 0, 0, 4034, 11, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 103, 0, 0, 0, 3649, 0, 0, 0] rerouted_tokens_per_ep_rank=[4427, 4083, 4136, 4140, 4101, 4045, 4084, 3752] avg=4096.0000 max=4427 max_rank=0 min=3752 min_rank=7 imbalance_ratio=1.080811 max_load_reduction=28341 max_load_reduction_pct=0.864899 moved_to_echo=15982 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1634, 15, 0, 378, 64, 18, 3, 4032, 12, 8, 77, 0, 0, 0, 0, 4005, 0, 1, 0, 0, 0, 0, 0, 30, 0, 3723, 0, 0, 0, 0, 0, 29, 0, 0, 15, 3927, 27, 0, 0, 71, 159, 4003, 0, 0, 0, 0, 0, 0, 150, 0, 3962, 0, 0, 0, 0, 2657, 11, 126, 27, 3604, 0, 0, 0] rerouted_tokens_per_ep_rank=[2112, 4129, 4006, 3753, 3998, 4233, 4112, 6425] avg=4096.0000 max=6425 max_rank=7 min=2112 min_rank=0 imbalance_ratio=1.568604 max_load_reduction=26343 max_load_reduction_pct=0.803925 moved_to_echo=18768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 381, 60, 12, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 4095, 8, 0, 2, 0, 0, 0, 0, 44, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 22, 0, 0, 0, 2, 4036, 0, 0, 0, 0, 0, 0, 0, 6, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 3649, 0, 0, 0] rerouted_tokens_per_ep_rank=[4549, 4087, 4105, 4140, 4110, 4038, 4090, 3649] avg=4096.0000 max=4549 max_rank=0 min=3649 min_rank=7 imbalance_ratio=1.110596 max_load_reduction=28219 max_load_reduction_pct=0.861176 moved_to_echo=15887 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 381, 60, 12, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 44, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 22, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 4, 0, 4084, 0, 0, 0, 0, 8, 0, 0, 0, 3649, 0, 0, 0] rerouted_tokens_per_ep_rank=[4533, 4096, 4104, 4144, 4110, 4036, 4088, 3657] avg=4096.0000 max=4533 max_rank=0 min=3657 min_rank=7 imbalance_ratio=1.106689 max_load_reduction=28235 max_load_reduction_pct=0.861664 moved_to_echo=15891 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 29184, 128, 128, 3072, 512, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3456, 128, 128, 128, 29184, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 512, 32256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[384, 128, 32384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True + [2026-08-30 04:56:45] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 656.9 | throughput per GPU (TFLOP/s/GPU): 507.5 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.318620E+01 | loss scale: 1.0 | grad norm: 34.245 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[7, 3994, 1878, 7, 1566, 1, 786, 12, 2848, 113, 592, 2270, 1680, 159, 42, 428, 1016, 782, 955, 4085, 1676, 1478, 786, 83, 0, 627, 319, 716, 1039, 77, 2726, 20] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[10, 4032, 1483, 13, 1422, 2, 758, 20, 2816, 235, 1136, 1577, 955, 506, 36, 549, 941, 665, 948, 4094, 1695, 1428, 528, 53, 0, 640, 205, 506, 2102, 107, 3304, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[9, 4034, 1553, 20, 1876, 2, 722, 13, 2868, 161, 1098, 2153, 1187, 406, 26, 516, 844, 572, 868, 4093, 1613, 1363, 716, 51, 0, 659, 205, 1064, 1066, 30, 2978, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[15, 4041, 1608, 4, 1515, 4, 807, 12, 3130, 170, 976, 1930, 1019, 498, 6, 431, 1149, 969, 864, 4093, 1603, 1339, 565, 44, 0, 781, 212, 478, 1289, 36, 3179, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[5, 4042, 2073, 9, 2145, 0, 748, 7, 2410, 187, 816, 2026, 1284, 511, 11, 434, 1012, 1090, 830, 4096, 1103, 1368, 755, 49, 1, 728, 214, 845, 967, 35, 2967, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[10, 4025, 1696, 12, 1302, 0, 688, 9, 2796, 308, 1150, 1809, 1217, 733, 13, 500, 683, 1007, 1060, 4089, 1469, 1527, 621, 56, 1, 432, 208, 744, 1533, 35, 3030, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[8, 4079, 1862, 0, 1504, 0, 1075, 3, 3042, 113, 1094, 1954, 630, 570, 11, 541, 1101, 1062, 694, 4096, 1670, 1173, 835, 11, 0, 720, 233, 516, 1141, 35, 2992, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[16, 4067, 2132, 8, 1882, 10, 1126, 24, 2904, 105, 628, 2017, 1209, 513, 6, 553, 551, 1401, 550, 4085, 1487, 1270, 732, 21, 2, 682, 184, 633, 1099, 31, 2837, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 2267, 1878, 7, 0, 0, 0, 0, 1566, 1, 786, 12, 1518, 54, 0, 0, 1019, 113, 592, 2270, 0, 0, 0, 0, 1680, 159, 42, 428, 1775, 170, 0, 0, 1016, 782, 955, 1393, 0, 0, 0, 0, 1676, 1478, 786, 83, 216, 209, 0, 0, 0, 627, 319, 716, 2522, 0, 0, 0, 1039, 77, 2510, 20, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 3937, 3994, 4254, 4146, 4448, 4184, 3646] avg=4096.0000 max=4448 max_rank=5 min=3646 min_rank=7 imbalance_ratio=1.085938 max_load_reduction=28320 max_load_reduction_pct=0.864258 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[9, 2290, 1553, 20, 0, 0, 0, 0, 1876, 2, 722, 13, 1533, 55, 0, 0, 1025, 161, 1098, 2153, 0, 0, 0, 0, 1187, 406, 26, 516, 1788, 170, 0, 0, 844, 572, 868, 1396, 0, 0, 0, 0, 1613, 1363, 716, 51, 236, 211, 0, 0, 0, 659, 205, 1064, 2527, 0, 0, 0, 1066, 30, 2742, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3872, 4201, 4437, 4093, 3680, 4190, 4455, 3840] avg=4096.0000 max=4455 max_rank=6 min=3680 min_rank=4 imbalance_ratio=1.087646 max_load_reduction=28313 max_load_reduction_pct=0.864044 moved_to_echo=16165 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[15, 2293, 1608, 4, 0, 0, 0, 0, 1515, 4, 807, 12, 1536, 60, 0, 0, 1119, 170, 976, 1930, 0, 0, 0, 0, 1019, 498, 6, 431, 1951, 170, 0, 0, 1149, 969, 864, 1396, 0, 0, 0, 0, 1603, 1339, 565, 44, 251, 212, 0, 0, 0, 781, 212, 478, 2527, 0, 0, 0, 1289, 36, 2928, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3920, 3934, 4195, 4075, 4378, 4014, 3998, 4254] avg=4096.0000 max=4378 max_rank=4 min=3920 min_rank=0 imbalance_ratio=1.068848 max_load_reduction=28390 max_load_reduction_pct=0.866394 moved_to_echo=16644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 2294, 2073, 9, 0, 0, 0, 0, 2145, 0, 748, 7, 1536, 46, 0, 0, 862, 187, 816, 2026, 0, 0, 0, 0, 1284, 511, 11, 434, 1502, 170, 0, 0, 1012, 1090, 830, 1397, 0, 0, 0, 0, 1103, 1368, 755, 49, 235, 212, 0, 0, 1, 728, 214, 845, 2529, 0, 0, 0, 967, 35, 2732, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4381, 4482, 3891, 3912, 4329, 3722, 4317, 3734] avg=4096.0000 max=4482 max_rank=1 min=3722 min_rank=5 imbalance_ratio=1.094238 max_load_reduction=28286 max_load_reduction_pct=0.863220 moved_to_echo=16102 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 2275, 1696, 12, 0, 0, 0, 0, 1302, 0, 688, 9, 1535, 57, 0, 0, 992, 308, 1150, 1809, 0, 0, 0, 0, 1217, 733, 13, 500, 1747, 174, 0, 0, 683, 1007, 1060, 1386, 0, 0, 0, 0, 1469, 1527, 621, 56, 243, 215, 0, 0, 1, 432, 208, 744, 2529, 0, 0, 0, 1533, 35, 2787, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3993, 3591, 4259, 4384, 4136, 4131, 3914, 4360] avg=4096.0000 max=4384 max_rank=3 min=3591 min_rank=1 imbalance_ratio=1.070312 max_load_reduction=28384 max_load_reduction_pct=0.866211 moved_to_echo=16541 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 2288, 1483, 13, 0, 0, 0, 0, 1422, 2, 758, 20, 1533, 54, 0, 0, 1007, 235, 1136, 1577, 0, 0, 0, 0, 955, 506, 36, 549, 1755, 170, 0, 0, 941, 665, 948, 1396, 0, 0, 0, 0, 1695, 1428, 528, 53, 261, 211, 0, 0, 0, 640, 205, 506, 2528, 0, 0, 0, 2102, 107, 3043, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3794, 3789, 3955, 3971, 3950, 4176, 3879, 5254] avg=4096.0000 max=5254 max_rank=7 min=3789 min_rank=1 imbalance_ratio=1.282715 max_load_reduction=27514 max_load_reduction_pct=0.839661 moved_to_echo=17259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 2308, 2132, 8, 0, 0, 0, 0, 1882, 10, 1126, 24, 1546, 56, 0, 0, 1038, 105, 628, 2017, 0, 0, 0, 0, 1209, 513, 6, 553, 1810, 170, 0, 0, 551, 1401, 550, 1393, 0, 0, 0, 0, 1487, 1270, 732, 21, 224, 213, 0, 0, 2, 682, 184, 633, 2522, 0, 0, 0, 1099, 31, 2613, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4464, 4644, 3788, 4261, 3895, 3947, 4023, 3746] avg=4096.0000 max=4644 max_rank=1 min=3746 min_rank=7 imbalance_ratio=1.133789 max_load_reduction=28124 max_load_reduction_pct=0.858276 moved_to_echo=15611 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[8, 2315, 1862, 0, 0, 0, 0, 0, 1504, 0, 1075, 3, 1550, 58, 0, 0, 1088, 113, 1094, 1954, 0, 0, 0, 0, 630, 570, 11, 541, 1896, 170, 0, 0, 1101, 1062, 694, 1397, 0, 0, 0, 0, 1670, 1173, 835, 11, 237, 214, 0, 0, 0, 720, 233, 516, 2529, 0, 0, 0, 1141, 35, 2755, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 4190, 4249, 3818, 4254, 4140, 3998, 3934] avg=4096.0000 max=4254 max_rank=4 min=3818 min_rank=3 imbalance_ratio=1.038574 max_load_reduction=28514 max_load_reduction_pct=0.870178 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12416, 11008, 5632, 384, 1920, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13312, 128, 6784, 128, 12288, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 18432, 14336, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7424, 7552, 6784, 11264, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 1408, 7552, 15744, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9216, 3968, 256, 3968, 14336, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 5376, 1792, 5504, 20224, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10240, 512, 22144, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[6, 0, 594, 4083, 0, 0, 4095, 4081, 5, 3, 0, 0, 0, 0, 4096, 1, 0, 0, 4087, 0, 11, 0, 4048, 694, 0, 0, 4096, 1, 2867, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 178, 4075, 0, 0, 4090, 4096, 0, 2, 0, 0, 0, 0, 4032, 0, 0, 0, 4096, 0, 0, 0, 4093, 400, 0, 5, 4094, 0, 3607, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 766, 4006, 0, 3, 4042, 4024, 3, 81, 0, 2, 5, 4, 4095, 0, 29, 0, 3997, 1, 6, 0, 4082, 1111, 1, 10, 4086, 22, 2380, 0, 9, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 160, 4067, 0, 0, 4093, 4078, 0, 25, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 3923, 222, 0, 0, 4096, 0, 3915, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[9, 0, 136, 4091, 4, 0, 4086, 4093, 1, 0, 1, 0, 3, 0, 4092, 0, 0, 1, 4085, 0, 2, 0, 3988, 63, 1, 5, 4094, 0, 4010, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 183, 4086, 0, 0, 4095, 4094, 1, 2, 2, 0, 0, 0, 4092, 2, 2, 3, 4096, 0, 0, 0, 4090, 1547, 0, 0, 4093, 0, 2377, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2, 1, 883, 4073, 0, 2, 4090, 3974, 3, 0, 0, 0, 0, 9, 4096, 0, 1, 0, 4076, 0, 10, 0, 4088, 128, 0, 2, 4093, 0, 3226, 0, 11, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 178, 4083, 4, 0, 4045, 4096, 9, 58, 0, 0, 0, 0, 4094, 0, 0, 0, 4075, 0, 0, 0, 4088, 721, 0, 0, 4096, 3, 3216, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 160, 3707, 0, 0, 0, 0, 0, 0, 28, 4078, 0, 0, 0, 0, 0, 25, 0, 0, 4065, 18, 0, 0, 0, 0, 4096, 0, 3, 2, 0, 0, 0, 0, 4092, 0, 15, 0, 0, 0, 0, 0, 3373, 222, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3915, 0, 0, 0, 532, 343, 0, 0] rerouted_tokens_per_ep_rank=[3868, 4106, 4108, 4101, 4107, 3595, 4093, 4790] avg=4096.0000 max=4790 max_rank=7 min=3595 min_rank=5 imbalance_ratio=1.169434 max_load_reduction=27978 max_load_reduction_pct=0.853821 moved_to_echo=16585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 178, 3715, 0, 0, 0, 0, 0, 0, 28, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 4062, 19, 0, 0, 0, 0, 4032, 0, 3, 2, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 3519, 400, 0, 0, 0, 0, 0, 5, 4091, 0, 0, 0, 0, 0, 3607, 0, 0, 0, 555, 343, 0, 0] rerouted_tokens_per_ep_rank=[3893, 4124, 4083, 4037, 4111, 3919, 4096, 4505] avg=4096.0000 max=4505 max_rank=7 min=3893 min_rank=0 imbalance_ratio=1.099854 max_load_reduction=28263 max_load_reduction_pct=0.862518 moved_to_echo=16631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 594, 3722, 0, 0, 0, 0, 0, 0, 28, 4081, 0, 0, 0, 0, 5, 3, 0, 0, 4067, 19, 0, 0, 0, 0, 4096, 1, 3, 2, 0, 0, 0, 0, 4087, 0, 15, 0, 0, 0, 11, 0, 3481, 694, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 2867, 0, 0, 0, 548, 344, 0, 0] rerouted_tokens_per_ep_rank=[4322, 4109, 4094, 4102, 4102, 4186, 4094, 3759] avg=4096.0000 max=4322 max_rank=0 min=3759 min_rank=7 imbalance_ratio=1.055176 max_load_reduction=28446 max_load_reduction_pct=0.868103 moved_to_echo=16141 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 766, 3652, 0, 0, 0, 0, 0, 3, 28, 4024, 0, 0, 0, 0, 3, 81, 0, 2, 4014, 19, 0, 0, 5, 4, 4095, 0, 3, 2, 0, 0, 29, 0, 3997, 1, 15, 0, 0, 0, 6, 0, 3510, 1111, 0, 0, 0, 0, 1, 10, 4083, 22, 0, 0, 0, 0, 2380, 0, 9, 2, 553, 337, 0, 0] rerouted_tokens_per_ep_rank=[4419, 4055, 4119, 4109, 4042, 4627, 4116, 3281] avg=4096.0000 max=4627 max_rank=5 min=3281 min_rank=7 imbalance_ratio=1.129639 max_load_reduction=28141 max_load_reduction_pct=0.858795 moved_to_echo=16066 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 883, 3713, 0, 0, 0, 0, 0, 2, 28, 3974, 0, 0, 0, 0, 3, 0, 0, 0, 4062, 19, 0, 0, 0, 9, 4096, 0, 3, 2, 0, 0, 1, 0, 4076, 0, 15, 0, 0, 0, 10, 0, 3515, 128, 0, 0, 0, 0, 0, 2, 4090, 0, 0, 0, 0, 0, 3226, 0, 11, 0, 554, 343, 0, 0] rerouted_tokens_per_ep_rank=[4599, 4004, 4084, 4110, 4092, 3653, 4092, 4134] avg=4096.0000 max=4599 max_rank=0 min=3653 min_rank=5 imbalance_ratio=1.122803 max_load_reduction=28169 max_load_reduction_pct=0.859650 moved_to_echo=15971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 183, 3709, 0, 0, 0, 0, 0, 0, 23, 4094, 0, 0, 0, 0, 1, 2, 2, 0, 4072, 22, 0, 0, 0, 0, 4092, 2, 9, 7, 0, 0, 2, 3, 4096, 0, 22, 0, 0, 0, 0, 0, 3510, 1547, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 2377, 0, 0, 2, 558, 348, 0, 0] rerouted_tokens_per_ep_rank=[3893, 4117, 4099, 4110, 4123, 5057, 4084, 3285] avg=4096.0000 max=5057 max_rank=5 min=3285 min_rank=7 imbalance_ratio=1.234619 max_load_reduction=27711 max_load_reduction_pct=0.845673 moved_to_echo=16549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 136, 3729, 0, 0, 0, 0, 4, 0, 28, 4093, 0, 0, 0, 0, 1, 0, 1, 0, 4058, 18, 0, 0, 3, 0, 4092, 0, 3, 2, 0, 0, 0, 1, 4085, 0, 15, 0, 0, 0, 2, 0, 3430, 63, 0, 0, 0, 0, 1, 5, 4091, 0, 0, 0, 0, 0, 4010, 0, 3, 0, 540, 345, 0, 0] rerouted_tokens_per_ep_rank=[3874, 4125, 4078, 4100, 4101, 3495, 4097, 4898] avg=4096.0000 max=4898 max_rank=7 min=3495 min_rank=5 imbalance_ratio=1.195801 max_load_reduction=27870 max_load_reduction_pct=0.850525 moved_to_echo=16591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 178, 3722, 0, 0, 0, 0, 4, 0, 28, 4096, 0, 0, 0, 0, 9, 58, 0, 0, 4017, 19, 0, 0, 0, 0, 4094, 0, 3, 2, 0, 0, 0, 0, 4075, 0, 15, 0, 0, 0, 0, 0, 3515, 721, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 3216, 0, 2, 0, 554, 344, 0, 0] rerouted_tokens_per_ep_rank=[3900, 4128, 4103, 4099, 4090, 4236, 4096, 4116] avg=4096.0000 max=4236 max_rank=5 min=3900 min_rank=0 imbalance_ratio=1.034180 max_load_reduction=28532 max_load_reduction_pct=0.870728 moved_to_echo=16538 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[25600, 0, 128, 128, 4480, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 3200, 29696, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 128, 128, 32512, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 256, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 27904, 4992, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 1939, 4096, 0, 0, 0, 0, 4050, 21, 2005, 0, 0, 3573, 0, 4032, 0, 4096, 0, 770, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 4022, 0, 2004, 4096, 0, 0, 0, 0, 4095, 5, 1737, 0, 0, 3948, 0, 4079, 0, 4096, 1, 582, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3348, 0, 3399, 4096, 11, 0, 0, 0, 4088, 2, 1981, 0, 0, 2359, 0, 3862, 0, 4095, 0, 1431, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 0, 3, 4095, 1, 2, 0, 4087, 5, 4093, 4094, 4, 5, 0, 0, 4095, 0, 198, 0, 0, 3880, 1, 3927, 0, 4095, 0, 180, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 6, 0, 4, 4090, 9, 0, 0, 4031, 0, 3461, 4089, 0, 0, 2, 0, 4053, 6, 2171, 2, 5, 2687, 1, 3685, 0, 4089, 0, 373, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[45, 0, 1, 4, 0, 16, 14, 4083, 0, 0, 13, 3971, 1, 4069, 4088, 0, 14, 3, 0, 3881, 147, 211, 0, 15, 4081, 2, 3697, 3, 4094, 1, 314, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 4, 4096, 2, 5, 0, 4062, 0, 3113, 4094, 13, 0, 0, 0, 4093, 0, 1685, 2, 0, 3625, 0, 3477, 2, 4079, 0, 414, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4000, 0, 4030, 4088, 0, 0, 0, 0, 4092, 52, 271, 0, 0, 3726, 0, 4046, 0, 4096, 0, 266, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3267, 862, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4090, 139, 0, 0, 0, 0, 1939, 829, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 33, 3, 0, 0, 21, 2005, 0, 0, 2530, 365, 0, 0, 3573, 0, 640, 0, 0, 0, 0, 0, 3559, 0, 770, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4093, 4229, 2768, 4086, 4921, 4213, 4329] avg=4096.0000 max=4921 max_rank=5 min=2768 min_rank=3 imbalance_ratio=1.201416 max_load_reduction=27847 max_load_reduction_pct=0.849823 moved_to_echo=17549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3267, 826, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3348, 139, 0, 0, 0, 0, 3399, 829, 11, 0, 0, 0, 0, 0, 0, 0, 4088, 33, 3, 0, 0, 2, 1981, 0, 0, 2423, 365, 0, 0, 2359, 0, 613, 0, 0, 0, 0, 0, 3558, 0, 1431, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4093, 3487, 4239, 4124, 4771, 2972, 4989] avg=4096.0000 max=4989 max_rank=7 min=2972 min_rank=6 imbalance_ratio=1.218018 max_load_reduction=27779 max_load_reduction_pct=0.847748 moved_to_echo=16856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3267, 872, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 6, 0, 0, 4022, 139, 0, 0, 0, 0, 2004, 829, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 33, 3, 0, 0, 5, 1737, 0, 0, 2559, 365, 0, 0, 3948, 0, 648, 0, 0, 0, 0, 0, 3559, 1, 582, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4093, 4167, 2833, 4131, 4666, 4596, 4143] avg=4096.0000 max=4666 max_rank=5 min=2833 min_rank=3 imbalance_ratio=1.139160 max_load_reduction=28102 max_load_reduction_pct=0.857605 moved_to_echo=17536 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[45, 0, 1, 4, 3260, 791, 0, 0, 0, 16, 14, 4080, 0, 0, 0, 0, 0, 0, 13, 3971, 139, 0, 0, 0, 1, 4069, 828, 0, 0, 0, 0, 0, 14, 3, 0, 3881, 33, 3, 0, 0, 147, 211, 0, 15, 2320, 365, 0, 0, 4081, 2, 586, 3, 0, 0, 0, 0, 3557, 1, 314, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4110, 4123, 4898, 3934, 3058, 4672, 3872] avg=4096.0000 max=4898 max_rank=3 min=3058 min_rank=5 imbalance_ratio=1.195801 max_load_reduction=27870 max_load_reduction_pct=0.850525 moved_to_echo=15536 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3265, 744, 0, 0, 0, 0, 4, 4093, 0, 0, 0, 0, 2, 5, 0, 4062, 139, 0, 0, 0, 0, 3113, 829, 13, 0, 0, 0, 0, 0, 0, 0, 4093, 33, 3, 0, 0, 0, 1685, 2, 0, 2182, 363, 0, 0, 3625, 0, 551, 2, 0, 0, 0, 0, 3544, 0, 414, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4097, 4208, 3955, 4129, 4232, 4178, 3958] avg=4096.0000 max=4232 max_rank=5 min=3955 min_rank=3 imbalance_ratio=1.033203 max_load_reduction=28536 max_load_reduction_pct=0.870850 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 3264, 792, 0, 0, 6, 0, 4, 4084, 0, 0, 0, 0, 9, 0, 0, 4031, 144, 0, 0, 0, 0, 3461, 825, 0, 0, 0, 0, 0, 0, 2, 0, 4053, 39, 6, 0, 0, 6, 2171, 2, 5, 2316, 366, 0, 0, 2687, 1, 577, 0, 0, 0, 0, 0, 3540, 0, 373, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4094, 4184, 4286, 4100, 4866, 3265, 3913] avg=4096.0000 max=4866 max_rank=5 min=3265 min_rank=6 imbalance_ratio=1.187988 max_load_reduction=27902 max_load_reduction_pct=0.851501 moved_to_echo=16144 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 3265, 840, 0, 0, 0, 0, 3, 4092, 0, 0, 0, 0, 1, 2, 0, 4087, 139, 0, 0, 0, 5, 4093, 829, 4, 0, 0, 0, 0, 5, 0, 0, 4095, 33, 3, 0, 0, 0, 198, 0, 0, 2464, 365, 0, 0, 3880, 1, 623, 0, 0, 0, 0, 0, 3558, 0, 180, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4095, 4229, 4931, 4136, 3027, 4504, 3739] avg=4096.0000 max=4931 max_rank=3 min=3027 min_rank=5 imbalance_ratio=1.203857 max_load_reduction=27837 max_load_reduction_pct=0.849518 moved_to_echo=15406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 3260, 865, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2, 0, 4000, 139, 0, 0, 0, 0, 4030, 828, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 33, 3, 0, 0, 52, 271, 0, 0, 2539, 365, 0, 0, 3726, 0, 642, 0, 0, 0, 0, 0, 3559, 0, 266, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4093, 4141, 4858, 4128, 3227, 4368, 3825] avg=4096.0000 max=4858 max_rank=3 min=3227 min_rank=5 imbalance_ratio=1.186035 max_load_reduction=27910 max_load_reduction_pct=0.851746 moved_to_echo=15548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 26240, 6656, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 26112, 6656, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 10368, 128, 128, 19456, 2944, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[28544, 128, 4352, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[27904, 128, 4992, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32512, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 31616, 1152, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 1667, 0, 0, 6, 4096, 0, 0, 3620, 0, 4096, 4096, 0, 2899, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4042, 0, 0, 0, 0, 0, 3, 4096, 795, 0, 0, 1, 4096, 0, 0, 4088, 0, 4096, 4096, 0, 3359, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4090, 0, 14, 0, 0, 0, 0, 4094, 1768, 0, 0, 557, 4092, 0, 0, 4003, 2, 4084, 4096, 0, 1863, 2, 0, 0, 0, 6, 1, 0, 0, 4094, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 1524, 0, 0, 362, 4096, 0, 0, 4039, 1, 4096, 4095, 0, 2262, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4087, 10, 0, 0, 13, 4, 0, 4089, 2458, 1, 7, 16, 4076, 0, 18, 4048, 1, 4090, 4092, 19, 1641, 0, 10, 0, 9, 0, 4, 0, 0, 4075, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 0, 0, 4, 4094, 3039, 0, 0, 0, 4096, 0, 1, 4007, 3, 4093, 4096, 2, 1059, 0, 0, 0, 0, 76, 0, 0, 0, 4096, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4083, 0, 0, 5, 2, 2, 0, 4089, 2565, 0, 2, 23, 4089, 0, 0, 3857, 0, 4093, 4093, 2, 1739, 0, 6, 4, 6, 12, 3, 4, 0, 4089, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 2938, 0, 0, 6, 4096, 0, 0, 4083, 0, 4094, 4096, 0, 1160, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1667, 0, 0, 6, 1880, 0, 0, 0, 128, 0, 0, 3620, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2899, 0, 0, 0, 2081, 15, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4095, 3553, 3748, 4096, 4995, 4081, 4097] avg=4096.0000 max=4995 max_rank=5 min=3553 min_rank=2 imbalance_ratio=1.219482 max_load_reduction=27773 max_load_reduction_pct=0.847565 moved_to_echo=17269 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4042, 0, 0, 0, 7, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 795, 0, 0, 1, 1880, 0, 0, 0, 128, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3359, 0, 0, 0, 2081, 16, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4098, 2676, 4216, 4096, 5456, 4080, 4097] avg=4096.0000 max=5456 max_rank=5 min=2676 min_rank=2 imbalance_ratio=1.332031 max_load_reduction=27312 max_load_reduction_pct=0.833496 moved_to_echo=17729 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1524, 0, 0, 362, 1880, 0, 0, 0, 128, 0, 0, 4039, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 2262, 0, 0, 0, 2081, 16, 0, 0, 0, 5, 0, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4095, 3766, 4167, 4097, 4359, 4084, 4097] avg=4096.0000 max=4359 max_rank=5 min=3766 min_rank=2 imbalance_ratio=1.064209 max_load_reduction=28409 max_load_reduction_pct=0.866974 moved_to_echo=16637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 14, 0, 7, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 1768, 0, 0, 557, 1879, 0, 0, 0, 127, 0, 0, 4003, 0, 0, 0, 0, 2, 4084, 0, 0, 0, 0, 0, 0, 1863, 2, 0, 0, 2079, 16, 0, 0, 0, 6, 1, 0, 4080, 0, 0, 0, 0, 4094, 0, 2, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4093, 4204, 4130, 4086, 3960, 4087, 4097] avg=4096.0000 max=4204 max_rank=2 min=3960 min_rank=5 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16230 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4087, 10, 0, 0, 7, 0, 0, 0, 13, 4, 0, 4088, 0, 0, 0, 0, 2458, 1, 7, 16, 1871, 0, 0, 0, 127, 0, 18, 4048, 0, 0, 0, 0, 1, 4090, 0, 19, 0, 0, 0, 0, 1641, 0, 10, 0, 2071, 15, 0, 0, 9, 0, 4, 0, 4077, 0, 0, 0, 0, 4075, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4105, 4353, 4193, 4110, 3737, 4090, 4076] avg=4096.0000 max=4353 max_rank=2 min=3737 min_rank=5 imbalance_ratio=1.062744 max_load_reduction=28415 max_load_reduction_pct=0.867157 moved_to_echo=16013 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 0, 5, 10, 0, 0, 0, 2, 2, 0, 4086, 0, 0, 0, 0, 2565, 0, 2, 23, 1883, 0, 0, 0, 110, 0, 0, 3857, 0, 0, 0, 0, 0, 4089, 0, 2, 0, 0, 0, 0, 1739, 0, 6, 4, 2084, 16, 0, 0, 6, 12, 3, 4, 4077, 0, 0, 0, 0, 4089, 0, 0, 3, 4, 2, 0] rerouted_tokens_per_ep_rank=[4098, 4090, 4473, 3967, 4091, 3849, 4102, 4098] avg=4096.0000 max=4473 max_rank=2 min=3849 min_rank=5 imbalance_ratio=1.092041 max_load_reduction=28295 max_load_reduction_pct=0.863495 moved_to_echo=16140 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4, 4093, 0, 0, 0, 0, 3039, 0, 0, 0, 1880, 0, 0, 0, 128, 0, 1, 4007, 0, 0, 0, 0, 3, 4093, 0, 2, 0, 0, 0, 0, 1059, 0, 0, 0, 2081, 16, 0, 0, 0, 76, 0, 0, 4080, 0, 0, 0, 0, 4096, 12, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4097, 4919, 4136, 4098, 3156, 4156, 4109] avg=4096.0000 max=4919 max_rank=2 min=3156 min_rank=5 imbalance_ratio=1.200928 max_load_reduction=27849 max_load_reduction_pct=0.849884 moved_to_echo=15519 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2938, 0, 0, 6, 1880, 0, 0, 0, 128, 0, 0, 4083, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1160, 0, 0, 0, 2081, 15, 0, 0, 0, 7, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4095, 4824, 4211, 4094, 3256, 4088, 4097] avg=4096.0000 max=4824 max_rank=2 min=3256 min_rank=5 imbalance_ratio=1.177734 max_load_reduction=27944 max_load_reduction_pct=0.852783 moved_to_echo=15535 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16768, 128, 128, 1024, 15104, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16000, 128, 128, 128, 16640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1024, 0, 128, 31872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4087, 4096, 1, 0, 0, 4096, 0, 3655, 0, 0, 0, 0, 4096, 0, 0, 0, 127, 0, 4096, 0, 3778, 2, 6, 362, 0, 0, 0, 4096, 270, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4089, 4095, 0, 0, 0, 4096, 0, 4091, 1, 0, 0, 0, 4096, 0, 0, 0, 42, 0, 4095, 0, 3854, 8, 1, 0, 2, 2, 1, 4094, 201, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 4091, 4089, 4, 0, 0, 3902, 0, 3779, 1, 2, 3, 5, 4090, 0, 5, 0, 12, 3, 4092, 11, 4073, 0, 92, 108, 0, 0, 2, 4093, 309, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[6, 26, 4039, 4056, 50, 0, 0, 4029, 0, 4059, 0, 26, 12, 2, 4044, 0, 6, 6, 206, 12, 4075, 11, 3959, 0, 0, 8, 5, 0, 20, 4085, 26, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4091, 4096, 0, 0, 0, 4096, 1, 3147, 0, 0, 0, 0, 4094, 0, 0, 0, 222, 0, 4096, 0, 3817, 9, 130, 87, 0, 0, 0, 4096, 786, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4087, 4096, 0, 0, 0, 4096, 0, 3796, 0, 0, 0, 0, 4095, 0, 0, 0, 302, 0, 4096, 0, 4027, 1, 21, 1, 0, 0, 0, 4096, 54, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4044, 4096, 0, 0, 0, 4096, 2, 3831, 0, 0, 0, 0, 4096, 0, 0, 3, 66, 0, 4096, 0, 4093, 9, 0, 216, 0, 0, 0, 4096, 24, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4012, 4096, 0, 0, 0, 4095, 0, 4012, 0, 0, 0, 0, 4096, 0, 0, 0, 148, 0, 4096, 0, 4093, 4, 0, 2, 0, 0, 0, 4096, 18, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4087, 25, 0, 0, 0, 0, 1, 0, 0, 4096, 25, 0, 0, 0, 0, 3655, 0, 0, 181, 114, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 127, 0, 3853, 100, 0, 0, 129, 0, 3778, 2, 0, 0, 0, 0, 6, 362, 0, 0, 3971, 0, 0, 0, 0, 3886, 270, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4122, 3950, 4100, 4080, 3909, 4339, 4156] avg=4096.0000 max=4339 max_rank=6 min=3909 min_rank=5 imbalance_ratio=1.059326 max_load_reduction=28429 max_load_reduction_pct=0.867584 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 25, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 4091, 1, 0, 181, 114, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 42, 0, 3852, 100, 0, 0, 129, 0, 3854, 8, 0, 0, 0, 0, 1, 0, 2, 2, 3970, 0, 0, 0, 1, 3884, 201, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4121, 4387, 4100, 3994, 3991, 3975, 4086] avg=4096.0000 max=4387 max_rank=2 min=3975 min_rank=6 imbalance_ratio=1.071045 max_load_reduction=28381 max_load_reduction_pct=0.866119 moved_to_echo=16046 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4044, 25, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 0, 0, 0, 2, 3831, 0, 0, 181, 114, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 3, 66, 0, 3853, 100, 0, 0, 129, 0, 4093, 9, 0, 0, 0, 0, 0, 216, 0, 0, 3971, 0, 0, 0, 0, 3886, 24, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4121, 4128, 4100, 4022, 4231, 4187, 3910] avg=4096.0000 max=4231 max_rank=5 min=3910 min_rank=7 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4087, 25, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 3796, 0, 0, 181, 114, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 0, 0, 0, 302, 0, 3853, 100, 0, 0, 129, 0, 4027, 1, 0, 0, 0, 0, 21, 1, 0, 0, 3971, 0, 0, 0, 0, 3886, 54, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4121, 4091, 4099, 4255, 4157, 3993, 3940] avg=4096.0000 max=4255 max_rank=4 min=3940 min_rank=7 imbalance_ratio=1.038818 max_load_reduction=28513 max_load_reduction_pct=0.870148 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 26, 4039, 25, 0, 0, 0, 0, 50, 0, 0, 4029, 25, 0, 0, 0, 0, 4059, 0, 26, 180, 114, 0, 0, 12, 2, 4044, 0, 4, 0, 0, 0, 6, 6, 206, 12, 3834, 99, 0, 0, 127, 11, 3959, 0, 0, 0, 0, 0, 0, 8, 5, 0, 3932, 0, 0, 0, 20, 3876, 26, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4104, 4379, 4062, 4163, 4097, 3945, 3922] avg=4096.0000 max=4379 max_rank=2 min=3922 min_rank=7 imbalance_ratio=1.069092 max_load_reduction=28389 max_load_reduction_pct=0.866364 moved_to_echo=16127 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4091, 19, 0, 0, 0, 0, 4, 0, 0, 3902, 32, 0, 0, 0, 0, 3779, 1, 2, 183, 118, 0, 0, 3, 5, 4090, 0, 11, 0, 0, 0, 5, 0, 12, 3, 3856, 102, 0, 0, 118, 11, 4073, 0, 0, 0, 0, 0, 92, 108, 0, 0, 3968, 0, 0, 0, 2, 3867, 309, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 3938, 4083, 4109, 3978, 4202, 4168, 4178] avg=4096.0000 max=4202 max_rank=5 min=3938 min_rank=1 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16526 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 25, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 0, 0, 0, 1, 3147, 0, 0, 181, 114, 0, 0, 0, 0, 4094, 0, 4, 0, 0, 0, 0, 0, 222, 0, 3853, 100, 0, 0, 129, 0, 3817, 9, 0, 0, 0, 0, 130, 87, 0, 0, 3971, 0, 0, 0, 0, 3886, 786, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4121, 3443, 4098, 4175, 3955, 4188, 4672] avg=4096.0000 max=4672 max_rank=7 min=3443 min_rank=2 imbalance_ratio=1.140625 max_load_reduction=28096 max_load_reduction_pct=0.857422 moved_to_echo=16990 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4012, 25, 0, 0, 0, 0, 0, 0, 0, 4095, 25, 0, 0, 0, 0, 4012, 0, 0, 181, 114, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 148, 0, 3853, 100, 0, 0, 129, 0, 4093, 4, 0, 0, 0, 0, 0, 2, 0, 0, 3971, 0, 0, 0, 0, 3886, 18, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4120, 4307, 4100, 4101, 4226, 3973, 3904] avg=4096.0000 max=4307 max_rank=2 min=3904 min_rank=7 imbalance_ratio=1.051514 max_load_reduction=28461 max_load_reduction_pct=0.868561 moved_to_echo=16204 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 31104, 1792, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 30464, 128, 128, 1536, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 896, 128, 128, 31744, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1024, 128, 31744, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32512, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 1152, 128, 30848, 896, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4027, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4094, 5, 0, 5, 0, 0, 1, 2, 0, 0, 0, 4093, 2, 0, 29, 0, 4067, 4096, 4083, 2, 1, 0, 0, 4, 4095, 0, 0, 0, 4094, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4087, 4088, 4093, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4091, 0, 8, 80, 3, 8, 3, 3, 2, 2, 0, 4079, 0, 0, 12, 0, 4022, 4015, 4093, 1, 2, 17, 2, 38, 4091, 0, 3, 5, 4092, 3, 4088, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 79, 0, 4056, 4004, 4096, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4073, 0, 2, 232, 7, 20, 0, 0, 23, 56, 0, 4075, 0, 2, 4, 0, 3944, 3753, 4070, 5, 28, 1, 6, 225, 4043, 17, 0, 14, 4075, 17, 4066, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 4049, 4089, 0, 0, 0, 0, 47, 4096, 0, 8, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 4047, 4028, 4096, 0, 0, 0, 0, 73, 4096, 0, 32, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4044, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 79, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 52, 14, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4093, 4091, 4084, 4174, 4019, 4096, 4097] avg=4096.0000 max=4174 max_rank=4 min=4019 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4044, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 2, 0, 4079, 2, 0, 0, 80, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 52, 14, 5, 4096, 0, 15, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4093, 4091, 4083, 4168, 4078, 4111, 4097] avg=4096.0000 max=4168 max_rank=4 min=4047 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4042, 5, 0, 5, 0, 0, 0, 0, 0, 0, 1, 2, 4091, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 2, 0, 29, 0, 4070, 3, 0, 0, 80, 4096, 0, 2, 0, 0, 0, 0, 1, 0, 0, 4, 3987, 52, 13, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4094, 4088, 4104, 4178, 4062, 4095, 4095] avg=4096.0000 max=4178 max_rank=4 min=4052 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4044, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 79, 0, 4083, 2, 0, 0, 80, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3976, 52, 13, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4094, 4091, 4164, 4084, 4059, 4096, 4096] avg=4096.0000 max=4164 max_rank=3 min=4059 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4022, 0, 2, 232, 0, 0, 0, 0, 7, 20, 0, 0, 4072, 0, 0, 0, 23, 56, 0, 4070, 0, 0, 0, 0, 0, 2, 4, 0, 4056, 2, 0, 0, 78, 3753, 0, 5, 0, 0, 0, 0, 28, 1, 6, 225, 3866, 51, 14, 5, 4043, 17, 0, 14, 0, 0, 0, 0, 1, 17, 4066, 10, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4256, 4099, 4149, 4064, 3836, 4196, 4074, 4094] avg=4096.0000 max=4256 max_rank=0 min=3836 min_rank=4 imbalance_ratio=1.039062 max_load_reduction=28512 max_load_reduction_pct=0.870117 moved_to_echo=16200 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4038, 0, 8, 80, 0, 0, 0, 0, 3, 8, 3, 3, 4090, 0, 0, 0, 2, 2, 0, 4072, 0, 0, 0, 0, 0, 0, 12, 0, 4080, 2, 0, 0, 75, 4015, 0, 1, 0, 0, 0, 0, 2, 17, 2, 38, 3947, 53, 13, 7, 4091, 0, 3, 5, 0, 0, 0, 0, 0, 3, 4088, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4107, 4076, 4094, 4091, 4079, 4099, 4096] avg=4096.0000 max=4126 max_rank=0 min=4076 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4044, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 3, 0, 0, 80, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4012, 52, 13, 5, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4095, 4091, 4079, 4129, 4129, 4104, 4096] avg=4096.0000 max=4129 max_rank=4 min=4045 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 12, 0, 4082, 2, 0, 0, 80, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 3967, 52, 14, 5, 4096, 0, 32, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4093, 4091, 4096, 4108, 4111, 4128, 4097] avg=4096.0000 max=4128 max_rank=6 min=4044 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32733, 32803, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32733, 32803, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32733, 32803, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32733, 32803, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32733, 32803, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32733, 32803, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32733, 32803, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32733, 32803, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[640, 32128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32384, 128, 128, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32803 actual_valid_recv=33152 observed_loss=-349 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 256, 0, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32733 actual_valid_recv=33408 observed_loss=-675 tokens_per_expert=[128, 128, 128, 512, 31744, 512, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 2, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 1, 5, 0, 4094, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4091, 1, 0, 1, 0, 0, 2, 4092, 19, 0, 0, 4091, 0, 0, 4087, 0, 5, 0, 0, 4082, 5, 14, 5, 0, 4093, 1, 6, 4091, 0, 4, 4076, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 1, 4093, 0, 0, 5, 2, 4096, 0, 0, 0, 0, 4095, 2, 0, 4094, 0, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 4096, 520, 27, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 3554, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4063, 0, 11, 0, 0, 0, 10, 4074, 3, 121, 15, 3974, 0, 0, 4072, 0, 12, 0, 0, 4071, 23, 20, 16, 0, 4084, 3, 18, 4069, 0, 21, 4062, 26] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 26, 0, 0, 4096, 0, 0, 4095, 0, 0, 19, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4052, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 67, 7, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 4026, 4099, 4096, 4087, 4098, 4167] avg=4096.0000 max=4167 max_rank=7 min=4026 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 67, 7, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4098, 4024, 4100, 4097, 4089, 4096, 4165] avg=4096.0000 max=4165 max_rank=7 min=4024 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 5, 0, 4020, 0, 0, 0, 0, 1, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 67, 9, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 4026, 4101, 4096, 4087, 4096, 4167] avg=4096.0000 max=4167 max_rank=7 min=4026 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 1, 4093, 0, 4, 0, 0, 0, 0, 5, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 67, 9, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4094, 4022, 4098, 4103, 4086, 4096, 4171] avg=4096.0000 max=4171 max_rank=7 min=4022 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4091, 1, 0, 1, 5, 0, 0, 0, 0, 0, 2, 4092, 6, 5, 0, 0, 19, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 4087, 0, 8, 0, 0, 0, 5, 0, 0, 4077, 0, 0, 0, 0, 5, 14, 5, 0, 4086, 0, 0, 0, 0, 1, 6, 4091, 0, 0, 0, 0, 0, 4, 4076, 2, 72, 7, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4105, 4019, 4095, 4082, 4110, 4098, 4161] avg=4096.0000 max=4161 max_rank=7 min=4019 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4063, 0, 11, 0, 3, 0, 0, 0, 0, 0, 10, 4074, 0, 0, 0, 0, 3, 121, 15, 3902, 0, 0, 0, 0, 0, 0, 4072, 0, 4, 0, 0, 0, 12, 0, 0, 4071, 0, 0, 0, 0, 23, 20, 16, 0, 4075, 0, 0, 0, 0, 3, 18, 4069, 0, 0, 0, 0, 0, 21, 4062, 26, 65, 9, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4084, 4041, 4076, 4083, 4134, 4090, 4183] avg=4096.0000 max=4183 max_rank=7 min=4041 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 520, 27, 0, 4015, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3554, 0, 67, 7, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4099, 4562, 4100, 4096, 4088, 4096, 3628] avg=4096.0000 max=4562 max_rank=2 min=3628 min_rank=7 imbalance_ratio=1.113770 max_load_reduction=28206 max_load_reduction_pct=0.860779 moved_to_echo=15908 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 26, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 0, 0, 19, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4052, 0, 67, 7, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 4048, 4099, 4115, 4087, 4098, 4126] avg=4096.0000 max=4126 max_rank=7 min=4048 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 256, 128, 32128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 32128, 128, 640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 4096, 3847, 0, 0, 410, 0, 0, 4096, 3, 3926] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 5, 4096, 0, 0, 6, 4096, 4096, 4071, 0, 0, 1774, 0, 0, 4096, 0, 2314] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 2, 4094, 0, 2, 2, 4093, 4094, 4083, 0, 1, 823, 0, 1, 4094, 0, 3286] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4073, 4, 26, 3, 60, 0, 0, 25, 0, 5, 27, 32, 4070, 2, 5, 0, 0, 4058, 20, 0, 0, 4056, 4068, 4029, 16, 0, 1189, 18, 0, 4069, 0, 2913] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 4096, 3828, 0, 0, 1218, 0, 0, 4096, 0, 3140] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 9, 4096, 0, 0, 0, 4096, 4096, 3947, 0, 0, 242, 0, 0, 4096, 0, 3993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0, 1998, 0, 0, 4096, 0, 2098] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4086, 4, 9, 0, 8, 0, 0, 3, 0, 0, 5, 0, 4093, 2, 7, 0, 0, 4087, 4, 0, 1, 4091, 4090, 4060, 2, 0, 909, 5, 0, 4092, 0, 3210] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 88, 6, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 22, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 6, 101, 0, 4071, 0, 0, 0, 0, 0, 0, 1774, 0, 3021, 1, 0, 0, 0, 987, 0, 2314, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4089, 4090, 4118, 4101, 4178, 4796, 3301] avg=4096.0000 max=4796 max_rank=6 min=3301 min_rank=7 imbalance_ratio=1.170898 max_load_reduction=27972 max_load_reduction_pct=0.853638 moved_to_echo=16376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4072, 4, 26, 3, 0, 0, 0, 0, 60, 0, 0, 25, 3956, 87, 4, 0, 0, 5, 27, 32, 4064, 0, 0, 0, 4070, 2, 5, 0, 0, 0, 0, 0, 0, 4058, 20, 0, 0, 0, 0, 0, 0, 100, 0, 4029, 0, 0, 0, 0, 16, 0, 1189, 18, 3001, 1, 0, 0, 0, 981, 0, 2913, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4132, 4128, 4077, 4078, 4129, 4225, 3894] avg=4096.0000 max=4225 max_rank=6 min=3894 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16326 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3992, 88, 5, 0, 2, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 2, 0, 0, 0, 0, 2, 101, 0, 4083, 0, 0, 0, 0, 0, 1, 823, 0, 3019, 1, 0, 0, 1, 987, 0, 3286, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4088, 4091, 4096, 4098, 4186, 3844, 4274] avg=4096.0000 max=4274 max_rank=7 min=3844 min_rank=6 imbalance_ratio=1.043457 max_load_reduction=28494 max_load_reduction_pct=0.869568 moved_to_echo=16402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3995, 88, 5, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 101, 0, 3828, 0, 0, 0, 0, 0, 0, 1218, 0, 3021, 1, 0, 0, 0, 987, 0, 3140, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4093, 4091, 4096, 4097, 3929, 4240, 4127] avg=4096.0000 max=4240 max_rank=6 min=3929 min_rank=5 imbalance_ratio=1.035156 max_load_reduction=28528 max_load_reduction_pct=0.870605 moved_to_echo=16393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 88, 5, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 101, 0, 3947, 0, 0, 0, 0, 0, 0, 242, 0, 3021, 1, 0, 0, 0, 987, 0, 3993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4088, 4092, 4096, 4105, 4048, 3264, 4980] avg=4096.0000 max=4980 max_rank=7 min=3264 min_rank=6 imbalance_ratio=1.215820 max_load_reduction=27788 max_load_reduction_pct=0.848022 moved_to_echo=16397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 88, 5, 0, 0, 4, 0, 0, 4091, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 101, 0, 3847, 0, 0, 0, 0, 0, 0, 410, 0, 3021, 1, 0, 0, 0, 987, 3, 3926, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4088, 4095, 4093, 4101, 3948, 3432, 4916] avg=4096.0000 max=4916 max_rank=7 min=3432 min_rank=6 imbalance_ratio=1.200195 max_load_reduction=27852 max_load_reduction_pct=0.849976 moved_to_echo=16397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4084, 4, 9, 0, 0, 0, 0, 0, 8, 0, 0, 3, 3997, 89, 5, 0, 0, 0, 5, 0, 4085, 0, 0, 0, 4085, 2, 7, 0, 0, 0, 0, 0, 0, 4087, 4, 0, 1, 0, 0, 0, 1, 94, 0, 4060, 0, 0, 0, 0, 2, 0, 909, 5, 3022, 2, 7, 0, 0, 981, 0, 3210, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4102, 4090, 4094, 4092, 4155, 3947, 4191] avg=4096.0000 max=4191 max_rank=7 min=3947 min_rank=6 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 88, 5, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 101, 0, 4094, 0, 0, 0, 0, 0, 0, 1998, 0, 3021, 1, 0, 0, 0, 987, 0, 2098, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4088, 4091, 4098, 4096, 4195, 5020, 3085] avg=4096.0000 max=5020 max_rank=6 min=3085 min_rank=7 imbalance_ratio=1.225586 max_load_reduction=27748 max_load_reduction_pct=0.846802 moved_to_echo=16396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7936, 128, 24960, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 8576, 128, 24192, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32000, 768, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 896, 0, 32000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 95, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4001, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 207, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3901, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 4081, 5, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 768, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3328, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 102, 4090, 3, 4, 5, 6, 0, 3, 0, 4089, 162, 5, 0, 3849, 0, 4089, 0, 10, 0, 4090, 4066, 2, 0, 4091, 0, 3, 4091, 1, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4071, 46, 26, 21, 18, 0, 11, 0, 4062, 129, 23, 0, 3994, 8, 4075, 0, 34, 39, 4011, 4027, 17, 0, 4064, 0, 24, 4067, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 4093, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 210, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3887, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 268, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3828, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 95, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4001, 196, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[3990, 4090, 4099, 4197, 4094, 4102, 4098, 4098] avg=4096.0000 max=4197 max_rank=3 min=3990 min_rank=0 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 768, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3328, 196, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4663, 4091, 4099, 3524, 4094, 4101, 4098, 4098] avg=4096.0000 max=4663 max_rank=0 min=3524 min_rank=3 imbalance_ratio=1.138428 max_load_reduction=28105 max_load_reduction_pct=0.857697 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 207, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3901, 196, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 4081, 5, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4090, 4099, 4097, 4092, 4102, 4088, 4098] avg=4096.0000 max=4102 max_rank=0 min=4088 min_rank=6 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3871, 0, 0, 0, 0, 46, 26, 21, 18, 4005, 0, 0, 0, 0, 11, 0, 4062, 3, 0, 0, 0, 129, 23, 0, 3994, 195, 0, 0, 0, 8, 4073, 0, 34, 0, 0, 0, 0, 39, 6, 4027, 17, 0, 0, 0, 0, 0, 4064, 0, 24, 2, 0, 0, 0, 4067, 1, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[3871, 4116, 4076, 4341, 4115, 4089, 4090, 4070] avg=4096.0000 max=4341 max_rank=3 min=3871 min_rank=0 imbalance_ratio=1.059814 max_load_reduction=28427 max_load_reduction_pct=0.867523 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 210, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3887, 196, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4089, 4099, 4083, 4094, 4102, 4098, 4098] avg=4096.0000 max=4105 max_rank=0 min=4083 min_rank=3 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 102, 3870, 0, 0, 0, 0, 3, 4, 5, 6, 4090, 7, 0, 0, 0, 3, 0, 4089, 6, 0, 0, 0, 162, 5, 0, 3849, 198, 0, 0, 0, 0, 4081, 0, 10, 0, 0, 0, 0, 0, 0, 4066, 2, 0, 0, 0, 0, 0, 4091, 0, 3, 6, 0, 0, 0, 4091, 1, 7, 0, 9, 2, 0, 0] rerouted_tokens_per_ep_rank=[3972, 4115, 4098, 4214, 4091, 4068, 4100, 4110] avg=4096.0000 max=4214 max_rank=3 min=3972 min_rank=0 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4092, 196, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[3902, 4087, 4099, 4288, 4094, 4102, 4098, 4098] avg=4096.0000 max=4288 max_rank=3 min=3902 min_rank=0 imbalance_ratio=1.046875 max_load_reduction=28480 max_load_reduction_pct=0.869141 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 268, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3828, 196, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4090, 4099, 4024, 4094, 4102, 4098, 4098] avg=4096.0000 max=4163 max_rank=0 min=4024 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 0, 30976, 1664, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 1664, 31232, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 6, 0, 0, 4086, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 6, 0, 2, 4091, 5, 4094, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4091, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[3, 4091, 7, 5, 0, 4053, 41, 4089, 0, 8, 0, 5, 4088, 3, 3, 0, 21, 4062, 1, 0, 8, 0, 0, 3, 4087, 0, 4, 0, 4090, 0, 4092, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[12, 4055, 0, 21, 0, 3986, 104, 4070, 0, 42, 0, 24, 4056, 7, 24, 11, 42, 4056, 6, 0, 40, 0, 0, 11, 4037, 8, 8, 0, 4069, 0, 4079, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 6, 0, 0, 0, 0, 0, 0, 4086, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 1, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4090, 4089, 4096, 4099, 4096, 4101, 4096] avg=4096.0000 max=4101 max_rank=0 min=4089 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16392 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 3, 1, 1, 0, 4096, 0, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4100, 4089, 4096, 4095, 4091, 4101, 4101] avg=4096.0000 max=4101 max_rank=6 min=4089 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 6, 0, 0, 0, 0, 0, 2, 4091, 5, 4, 0, 0, 0, 0, 2, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 3, 1, 1, 0, 4094, 0, 5, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4102, 4089, 4096, 4093, 4086, 4101, 4100] avg=4096.0000 max=4102 max_rank=1 min=4086 min_rank=5 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16380 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 3, 1, 1, 0, 4096, 0, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4100, 4089, 4096, 4096, 4089, 4101, 4101] avg=4096.0000 max=4101 max_rank=6 min=4089 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[12, 4054, 0, 21, 0, 0, 0, 0, 0, 3986, 104, 5, 0, 0, 0, 0, 0, 42, 0, 24, 4063, 0, 0, 0, 4056, 7, 24, 11, 0, 0, 0, 0, 42, 4055, 6, 0, 0, 0, 0, 0, 40, 0, 0, 11, 4074, 0, 0, 0, 4037, 8, 8, 0, 2, 1, 1, 0, 4069, 0, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4095, 4129, 4098, 4103, 4125, 4057, 4074] avg=4096.0000 max=4129 max_rank=2 min=4057 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16359 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 4083, 7, 5, 0, 0, 0, 0, 0, 4053, 41, 0, 0, 0, 0, 0, 0, 8, 0, 5, 4085, 4, 0, 0, 4088, 3, 3, 0, 0, 0, 0, 0, 21, 4060, 1, 0, 0, 0, 0, 0, 8, 0, 0, 3, 4088, 0, 0, 0, 4087, 0, 4, 0, 4, 8, 2, 0, 4090, 0, 0, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4094, 4102, 4094, 4082, 4099, 4105, 4094] avg=4096.0000 max=4105 max_rank=6 min=4082 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16380 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 3, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 3, 1, 1, 0, 4096, 0, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4100, 4092, 4096, 4092, 4091, 4101, 4101] avg=4096.0000 max=4101 max_rank=6 min=4091 min_rank=5 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 3, 1, 1, 0, 4096, 0, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4087, 4089, 4096, 4108, 4091, 4101, 4101] avg=4096.0000 max=4108 max_rank=4 min=4087 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16401 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4077, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4075, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[11, 4086, 0, 5, 5, 0, 3, 0, 4091, 4090, 4069, 6, 4089, 4090, 1, 2, 0, 8, 0, 0, 1, 0, 1, 0, 19, 10, 0, 8, 4074, 2, 4088, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 4090, 4096, 4090, 0, 4096, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 3712, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 4035, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 4086, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 3950, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[34, 4050, 0, 17, 26, 0, 0, 1, 4068, 4064, 4022, 22, 4054, 4070, 0, 0, 0, 37, 7, 0, 0, 18, 39, 0, 78, 30, 0, 9, 4020, 20, 4054, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4074, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4080, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 4062, 17, 0, 0, 0, 0, 0, 21, 3995, 1, 0, 0, 4096, 0, 84, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4107, 4102, 4092, 4079, 4017, 4180] avg=4096.0000 max=4180 max_rank=7 min=4017 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 4083, 17, 0, 0, 0, 0, 0, 19, 3995, 1, 0, 0, 4077, 0, 84, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4107, 4102, 4092, 4100, 4015, 4161] avg=4096.0000 max=4161 max_rank=7 min=4015 min_rank=6 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 11, 0, 4090, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 4077, 17, 0, 0, 0, 0, 0, 390, 3995, 1, 0, 0, 3712, 0, 84, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4097, 4101, 4102, 4097, 4094, 4386, 3796] avg=4096.0000 max=4386 max_rank=6 min=3796 min_rank=7 imbalance_ratio=1.070801 max_load_reduction=28382 max_load_reduction_pct=0.866150 moved_to_echo=16373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 4081, 17, 0, 0, 0, 0, 0, 63, 3995, 1, 0, 0, 4035, 0, 84, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4107, 4102, 4092, 4098, 4059, 4119] avg=4096.0000 max=4119 max_rank=7 min=4059 min_rank=6 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[34, 4049, 0, 17, 0, 0, 0, 0, 26, 0, 0, 1, 4064, 0, 0, 0, 11, 0, 4022, 22, 0, 0, 0, 0, 4054, 6, 0, 0, 0, 0, 0, 0, 0, 37, 7, 0, 4064, 2, 0, 0, 0, 18, 39, 0, 4055, 16, 0, 0, 78, 30, 0, 9, 3954, 1, 0, 0, 4020, 20, 84, 28, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4091, 4055, 4060, 4110, 4128, 4072, 4152] avg=4096.0000 max=4152 max_rank=7 min=4055 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16462 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[11, 4082, 0, 5, 0, 0, 0, 0, 5, 0, 3, 0, 4090, 1, 0, 0, 2, 0, 4069, 6, 0, 0, 0, 0, 4089, 4, 1, 2, 0, 0, 0, 0, 0, 8, 0, 0, 4085, 8, 0, 0, 1, 0, 1, 0, 4081, 18, 0, 0, 19, 10, 0, 8, 3993, 4, 0, 0, 4074, 2, 77, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4099, 4077, 4096, 4101, 4101, 4034, 4162] avg=4096.0000 max=4162 max_rank=7 min=4034 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 4073, 17, 0, 0, 0, 0, 0, 155, 3995, 1, 0, 0, 3950, 0, 84, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4097, 4107, 4102, 4092, 4090, 4151, 4034] avg=4096.0000 max=4151 max_rank=6 min=4034 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 4061, 17, 0, 0, 0, 0, 0, 38, 3995, 1, 0, 0, 4080, 0, 84, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4107, 4102, 4092, 4078, 4034, 4164] avg=4096.0000 max=4164 max_rank=7 min=4034 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 768, 32000, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32128, 128, 768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 10, 0, 2, 4091, 0, 0, 4096, 0, 4096, 12, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[53, 0, 4058, 9, 0, 0, 70, 1, 0, 0, 4067, 41, 0, 4061, 4, 4067, 195, 25, 4044, 4081, 0, 2, 21, 24, 0, 31, 26, 9, 4037, 3800, 0, 42] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 30, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[9, 25, 4088, 0, 0, 0, 6, 0, 3, 0, 4083, 0, 0, 4089, 1, 4090, 0, 4, 4086, 4091, 0, 9, 0, 5, 0, 3, 6, 7, 4085, 4063, 0, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 4096, 2, 0, 4096, 0, 4096, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 20, 6, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 1, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 36, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4087, 4096, 4101, 4077, 4096, 4090, 4131] avg=4096.0000 max=4131 max_rank=7 min=4077 min_rank=4 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4060, 20, 6, 1, 0, 2, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 12, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 36, 4087, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4097, 4093, 4101, 4078, 4096, 4090, 4123] avg=4096.0000 max=4123 max_rank=7 min=4078 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4060, 20, 6, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 36, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4092, 4096, 4101, 4071, 4096, 4090, 4132] avg=4096.0000 max=4132 max_rank=7 min=4071 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 20, 6, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 36, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4087, 4096, 4101, 4076, 4096, 4090, 4132] avg=4096.0000 max=4132 max_rank=7 min=4076 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[53, 0, 4052, 9, 0, 0, 0, 0, 0, 0, 70, 1, 4002, 20, 6, 1, 0, 0, 4067, 41, 0, 0, 0, 0, 0, 4061, 4, 5, 0, 0, 0, 0, 195, 25, 0, 4061, 0, 0, 0, 0, 0, 2, 21, 24, 4044, 0, 0, 0, 0, 31, 26, 9, 4061, 0, 0, 0, 35, 3800, 0, 42, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4100, 4108, 4070, 4281, 4091, 4127, 3877] avg=4096.0000 max=4281 max_rank=4 min=3877 min_rank=7 imbalance_ratio=1.045166 max_load_reduction=28487 max_load_reduction_pct=0.869354 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[9, 25, 4080, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4054, 21, 8, 2, 3, 0, 4083, 0, 0, 0, 0, 0, 0, 4089, 1, 2, 0, 0, 0, 0, 0, 4, 0, 4070, 0, 0, 0, 0, 0, 9, 0, 5, 4086, 1, 0, 0, 0, 3, 6, 7, 4085, 0, 0, 0, 31, 4063, 0, 15, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4091, 4086, 4092, 4074, 4101, 4101, 4109] avg=4096.0000 max=4114 max_rank=0 min=4074 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4060, 20, 6, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 36, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4117, 4096, 4101, 4046, 4096, 4090, 4132] avg=4096.0000 max=4132 max_rank=7 min=4046 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4060, 20, 6, 1, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 36, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4096, 4098, 4101, 4065, 4096, 4090, 4132] avg=4096.0000 max=4132 max_rank=7 min=4065 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32769, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32769, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32769, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32769, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32769, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32769, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32769, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32769, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 32512, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32767 actual_valid_recv=33280 observed_loss=-513 tokens_per_expert=[0, 0, 128, 128, 32512, 256, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 0, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32769 actual_valid_recv=33152 observed_loss=-383 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[41, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4050, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[60, 11, 0, 5, 9, 1, 4054, 24, 7, 0, 0, 7, 4067, 4, 4079, 5, 0, 0, 0, 4085, 1, 5, 4, 0, 3, 4089, 0, 4089, 4071, 0, 0, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[306, 40, 1, 23, 53, 0, 4057, 62, 31, 0, 0, 34, 3981, 24, 4008, 18, 0, 0, 14, 4014, 0, 28, 5, 0, 8, 4060, 1, 4065, 3879, 8, 4, 4044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4070, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 1, 4095, 4094, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[41, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 4095, 4053, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 19, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4068, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4031, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 17, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4089, 4098, 4108, 4102, 4095, 4100, 4144] avg=4096.0000 max=4144 max_rank=7 min=4032 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 4031, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 17, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4070, 0, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4089, 4098, 4108, 4103, 4095, 4100, 4118] avg=4096.0000 max=4118 max_rank=7 min=4057 min_rank=0 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[41, 0, 0, 0, 4031, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 17, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4050, 0, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4089, 4098, 4108, 4107, 4095, 4100, 4098] avg=4096.0000 max=4108 max_rank=3 min=4073 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4031, 1, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4081, 17, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 2, 4096, 1, 4, 0, 0, 0, 0, 4094, 0, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4087, 4100, 4108, 4102, 4094, 4103, 4142] avg=4096.0000 max=4142 max_rank=7 min=4032 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[60, 11, 0, 5, 4028, 5, 0, 0, 9, 1, 4037, 24, 0, 0, 0, 0, 7, 0, 0, 7, 4067, 20, 0, 0, 4067, 4, 8, 5, 0, 0, 0, 0, 0, 0, 0, 4085, 12, 0, 0, 0, 1, 5, 4, 0, 4089, 4, 0, 0, 3, 4089, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 40, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4071, 4101, 4084, 4097, 4103, 4092, 4111] avg=4096.0000 max=4111 max_rank=7 min=4071 min_rank=1 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[306, 40, 1, 23, 3980, 1, 0, 0, 53, 0, 4050, 62, 0, 0, 0, 0, 31, 0, 0, 34, 3993, 17, 0, 0, 3981, 24, 13, 18, 0, 0, 0, 0, 0, 0, 14, 4014, 6, 0, 0, 0, 0, 28, 5, 0, 4061, 2, 0, 0, 8, 4060, 1, 4, 0, 0, 0, 0, 3879, 8, 4, 47, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4351, 4165, 4075, 4036, 4034, 4096, 4073, 3938] avg=4096.0000 max=4351 max_rank=0 min=3938 min_rank=7 imbalance_ratio=1.062256 max_load_reduction=28417 max_load_reduction_pct=0.867218 moved_to_echo=16141 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[41, 0, 0, 0, 4031, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4081, 17, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 1, 4091, 3, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4053, 0, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4089, 4100, 4108, 4102, 4095, 4100, 4101] avg=4096.0000 max=4108 max_rank=3 min=4073 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 4030, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 17, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4068, 0, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4089, 4098, 4108, 4121, 4095, 4100, 4116] avg=4096.0000 max=4121 max_rank=4 min=4041 min_rank=0 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 128, 128, 128, 32256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32384, 128, 128, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 4064, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 32] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[19, 0, 0, 7, 7, 0, 4089, 8, 3, 10, 0, 12, 4091, 6, 0, 4091, 0, 0, 0, 7, 4067, 0, 1, 4089, 0, 4, 4, 4081, 4084, 4088, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 2, 0, 0, 0, 4092, 0, 0, 0, 0, 4, 4095, 0, 7, 4089, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 3, 4096, 0, 7, 4026, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4095, 4096, 4096, 0, 59] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[38, 0, 0, 31, 36, 0, 4070, 46, 11, 38, 0, 99, 4054, 30, 0, 4072, 0, 0, 14, 43, 3966, 0, 0, 4066, 0, 10, 25, 4010, 4048, 4060, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 34, 4062, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4084, 2, 2, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4073, 4, 0, 0, 12, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 2, 0, 0, 4092, 0, 0, 32, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4090, 4078, 4076, 4094, 4116, 4102, 4124] avg=4096.0000 max=4124 max_rank=7 min=4076 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16436 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4084, 2, 3, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 2, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4090, 4077, 4108, 4093, 4117, 4102, 4092] avg=4096.0000 max=4117 max_rank=5 min=4077 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4084, 2, 2, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4, 0, 0, 12, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 2, 0, 0, 4092, 0, 0, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4090, 4077, 4104, 4094, 4117, 4102, 4096] avg=4096.0000 max=4117 max_rank=5 min=4077 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16409 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 0, 4083, 2, 2, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4073, 4, 0, 0, 12, 0, 7, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 2, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4086, 4081, 4108, 4092, 4117, 4102, 4092] avg=4096.0000 max=4117 max_rank=5 min=4081 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[38, 0, 0, 31, 4042, 2, 2, 0, 36, 0, 4064, 46, 0, 0, 0, 0, 11, 38, 0, 99, 4043, 4, 0, 0, 12, 30, 0, 4072, 0, 0, 0, 0, 0, 0, 14, 43, 4058, 0, 0, 0, 3966, 0, 0, 21, 0, 0, 0, 0, 0, 10, 25, 4010, 4, 2, 0, 0, 4044, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4146, 4195, 4114, 4115, 3987, 4051, 4045] avg=4096.0000 max=4195 max_rank=2 min=3987 min_rank=5 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16198 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 0, 7, 4083, 9, 3, 0, 7, 0, 4071, 8, 0, 0, 0, 0, 3, 10, 0, 12, 4068, 9, 0, 0, 8, 6, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 7, 4085, 0, 0, 0, 4067, 0, 1, 12, 0, 0, 0, 0, 0, 4, 4, 4081, 8, 9, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4086, 4102, 4105, 4092, 4080, 4106, 4076] avg=4096.0000 max=4121 max_rank=0 min=4076 min_rank=7 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4084, 2, 2, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4073, 4, 0, 0, 12, 0, 7, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 2, 0, 0, 4092, 0, 0, 59, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4090, 4081, 4045, 4094, 4117, 4101, 4151] avg=4096.0000 max=4151 max_rank=7 min=4045 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4084, 2, 2, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4, 0, 0, 12, 0, 34, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 2, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4090, 4077, 4108, 4094, 4117, 4102, 4092] avg=4096.0000 max=4117 max_rank=5 min=4077 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16405 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 0, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32640, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 105, 1309, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4095, 4096, 4095, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 2682, 4, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 957, 499, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2640, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 330, 226, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4050, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3584, 2, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[17, 0, 339, 949, 0, 2, 4086, 5, 4, 11, 9, 10, 2, 4090, 0, 6, 4051, 4088, 4090, 7, 4077, 0, 0, 1, 2, 0, 3, 11, 2821, 0, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 194, 2435, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3903, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1659, 1, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 174, 403, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4071, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3546, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[3, 0, 117, 3384, 0, 6, 4058, 29, 16, 85, 38, 36, 0, 4057, 6, 22, 3942, 4061, 4074, 35, 3991, 0, 0, 0, 0, 0, 33, 19, 680, 0, 4067, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 274, 2797, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1025, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 105, 1309, 2284, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 27, 5, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 4095, 53, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 2682, 4, 1769, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3699, 4096, 4075, 4093, 4148, 4112, 4090, 4455] avg=4096.0000 max=4455 max_rank=7 min=3699 min_rank=0 imbalance_ratio=1.087646 max_load_reduction=28313 max_load_reduction_pct=0.864044 moved_to_echo=16805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 957, 499, 2284, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 27, 4, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 53, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 2640, 0, 1769, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3740, 4096, 4074, 4097, 4149, 4111, 4092, 4409] avg=4096.0000 max=4409 max_rank=7 min=3740 min_rank=0 imbalance_ratio=1.076416 max_load_reduction=28359 max_load_reduction_pct=0.865448 moved_to_echo=16761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 330, 226, 2284, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 27, 5, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4050, 53, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 3584, 2, 1769, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2840, 4096, 4075, 4097, 4103, 4111, 4091, 5355] avg=4096.0000 max=5355 max_rank=7 min=2840 min_rank=0 imbalance_ratio=1.307373 max_load_reduction=27413 max_load_reduction_pct=0.836578 moved_to_echo=17660 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 174, 403, 2284, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 27, 5, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4071, 53, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 3546, 0, 1768, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2861, 4096, 4075, 4096, 4124, 4111, 4091, 5314] avg=4096.0000 max=5314 max_rank=7 min=2861 min_rank=0 imbalance_ratio=1.297363 max_load_reduction=27454 max_load_reduction_pct=0.837830 moved_to_echo=17640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 117, 3384, 2268, 0, 0, 0, 0, 6, 4058, 29, 0, 0, 0, 0, 16, 85, 38, 36, 4008, 26, 5, 0, 0, 4057, 6, 22, 1, 0, 0, 0, 3942, 53, 0, 35, 0, 0, 0, 0, 3991, 0, 0, 0, 15, 0, 0, 0, 0, 0, 33, 19, 4069, 0, 0, 0, 680, 0, 1757, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5772, 4093, 4214, 4086, 4030, 4006, 4121, 2446] avg=4096.0000 max=5772 max_rank=0 min=2446 min_rank=7 imbalance_ratio=1.409180 max_load_reduction=26996 max_load_reduction_pct=0.823853 moved_to_echo=14603 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 339, 949, 2283, 0, 0, 0, 0, 2, 4086, 5, 6, 0, 0, 0, 4, 11, 9, 10, 4037, 29, 5, 0, 2, 4090, 0, 6, 7, 0, 0, 0, 4051, 51, 0, 7, 0, 0, 0, 0, 4077, 0, 0, 1, 17, 0, 0, 0, 2, 0, 3, 11, 4085, 0, 0, 0, 2821, 0, 1745, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3588, 4099, 4105, 4105, 4109, 4095, 4101, 4566] avg=4096.0000 max=4566 max_rank=7 min=3588 min_rank=0 imbalance_ratio=1.114746 max_load_reduction=28202 max_load_reduction_pct=0.860657 moved_to_echo=16871 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 194, 2435, 2284, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 27, 5, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3903, 53, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 1659, 1, 1769, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4913, 4096, 4075, 4097, 3956, 4111, 4091, 3429] avg=4096.0000 max=4913 max_rank=0 min=3429 min_rank=7 imbalance_ratio=1.199463 max_load_reduction=27855 max_load_reduction_pct=0.850067 moved_to_echo=15587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 274, 2797, 2284, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 27, 5, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 53, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 1025, 0, 1769, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5355, 4096, 4075, 4097, 4149, 4111, 4091, 2794] avg=4096.0000 max=5355 max_rank=0 min=2794 min_rank=7 imbalance_ratio=1.307373 max_load_reduction=27413 max_load_reduction_pct=0.836578 moved_to_echo=15145 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 2560, 12032, 18304, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[18688, 128, 14208, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32384, 512, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32384, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 20, 0, 0, 4076, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 4089, 0, 4089, 10, 9, 4088, 0, 4083, 0, 0, 8, 0, 7, 9, 4086, 5, 0, 0, 4064, 6, 16, 0, 0, 5, 28, 6, 4065, 7, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 4094, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[24, 64, 4059, 14, 4064, 47, 30, 4056, 0, 4042, 0, 0, 31, 7, 37, 42, 4047, 29, 0, 0, 3780, 2, 0, 0, 0, 69, 230, 1, 3933, 36, 4060, 64] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4095, 0, 0, 0, 38, 1, 0, 0, 0, 0, 0, 0, 4043, 13, 0, 0, 4096, 0, 15, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4091, 4103, 4083, 4099, 4134, 4056, 4112] avg=4096.0000 max=4134 max_rank=5 min=4056 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 38, 1, 0, 0, 0, 0, 0, 0, 4043, 13, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4091, 4103, 4083, 4099, 4135, 4056, 4111] avg=4096.0000 max=4135 max_rank=5 min=4056 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 38, 1, 0, 0, 0, 20, 0, 0, 4043, 12, 0, 0, 4076, 0, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4091, 4103, 4084, 4099, 4135, 4075, 4091] avg=4096.0000 max=4135 max_rank=5 min=4075 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 1, 0, 5, 2, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 2, 0, 38, 1, 0, 0, 0, 0, 0, 0, 4043, 12, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4091, 4104, 4082, 4099, 4137, 4055, 4111] avg=4096.0000 max=4137 max_rank=5 min=4055 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[24, 64, 4053, 14, 0, 0, 0, 0, 0, 47, 30, 4051, 0, 0, 0, 0, 0, 4042, 0, 0, 5, 2, 0, 0, 31, 7, 37, 42, 4051, 0, 0, 0, 4047, 29, 0, 0, 3, 0, 0, 0, 3780, 2, 0, 0, 38, 1, 0, 0, 0, 69, 230, 1, 4008, 13, 0, 0, 3933, 36, 14, 64, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4128, 4049, 4168, 4079, 3821, 4321, 4047] avg=4096.0000 max=4321 max_rank=6 min=3821 min_rank=5 imbalance_ratio=1.054932 max_load_reduction=28447 max_load_reduction_pct=0.868134 moved_to_echo=16268 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4073, 0, 0, 0, 0, 0, 0, 10, 9, 4075, 0, 0, 0, 0, 0, 4083, 0, 0, 10, 9, 0, 0, 8, 0, 7, 9, 4077, 0, 0, 0, 4086, 5, 0, 0, 4, 0, 0, 0, 4064, 6, 16, 0, 44, 6, 0, 0, 0, 5, 28, 6, 4041, 12, 0, 0, 4065, 7, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4094, 4102, 4101, 4095, 4136, 4092, 4074] avg=4096.0000 max=4136 max_rank=5 min=4074 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4094, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 38, 1, 0, 0, 0, 2, 0, 0, 4043, 12, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4091, 4101, 4084, 4099, 4135, 4057, 4111] avg=4096.0000 max=4135 max_rank=5 min=4057 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 38, 1, 0, 0, 0, 0, 0, 0, 4043, 13, 0, 0, 4095, 0, 15, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4091, 4103, 4083, 4099, 4135, 4056, 4111] avg=4096.0000 max=4135 max_rank=5 min=4056 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32512, 128, 128, 0, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 384, 128, 32384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1, 4064, 1619, 3, 1733, 1, 1032, 0, 3055, 180, 866, 2190, 874, 435, 13, 429, 713, 1516, 639, 4096, 1280, 1537, 470, 46, 0, 528, 203, 496, 1347, 25, 3371, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3, 4078, 2183, 3, 1143, 2, 560, 33, 3042, 249, 894, 2018, 1088, 309, 7, 531, 917, 740, 1455, 4089, 1151, 1109, 705, 38, 0, 613, 221, 925, 1480, 46, 3122, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[9, 3826, 1314, 133, 1521, 0, 735, 31, 2830, 611, 1021, 1277, 877, 577, 37, 737, 939, 903, 710, 4045, 1965, 2036, 716, 85, 4, 754, 219, 1144, 849, 26, 2824, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[7, 4082, 2290, 17, 1010, 0, 955, 5, 2821, 328, 1105, 1925, 1415, 491, 12, 547, 700, 1095, 902, 4096, 1772, 1382, 466, 18, 0, 767, 177, 667, 789, 43, 2882, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3, 4066, 1995, 19, 1758, 1, 893, 3, 3118, 116, 912, 1925, 1328, 331, 1, 780, 1017, 1172, 764, 4091, 1156, 1050, 643, 82, 0, 727, 213, 1081, 680, 24, 2817, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1, 4037, 2195, 13, 1251, 0, 989, 8, 3329, 117, 1198, 2507, 1249, 428, 5, 516, 713, 1169, 482, 4089, 1672, 1016, 712, 30, 0, 619, 184, 520, 906, 29, 2778, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[4, 4077, 1966, 13, 1190, 1, 769, 0, 3144, 130, 835, 2027, 1273, 485, 4, 710, 929, 937, 968, 4096, 1120, 1207, 814, 21, 0, 758, 308, 734, 1247, 54, 2941, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 2206, 1619, 3, 0, 0, 0, 0, 1733, 1, 1032, 0, 1124, 707, 0, 0, 782, 180, 866, 2190, 0, 0, 0, 0, 874, 435, 13, 429, 1566, 248, 0, 0, 713, 1516, 639, 1404, 0, 0, 0, 0, 1280, 1537, 470, 46, 617, 0, 0, 0, 0, 528, 203, 496, 2444, 0, 0, 0, 1347, 25, 3371, 6, 117, 0, 0, 0] rerouted_tokens_per_ep_rank=[3829, 4597, 4018, 3565, 4272, 3950, 3671, 4866] avg=4096.0000 max=4866 max_rank=7 min=3565 min_rank=3 imbalance_ratio=1.187988 max_load_reduction=27902 max_load_reduction_pct=0.851501 moved_to_echo=16759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 2214, 2183, 3, 0, 0, 0, 0, 1143, 2, 560, 33, 1128, 704, 0, 0, 779, 249, 894, 2018, 0, 0, 0, 0, 1088, 309, 7, 531, 1559, 247, 0, 0, 917, 740, 1455, 1402, 0, 0, 0, 0, 1151, 1109, 705, 38, 619, 0, 0, 0, 0, 613, 221, 925, 2440, 0, 0, 0, 1480, 46, 3122, 14, 117, 0, 0, 0] rerouted_tokens_per_ep_rank=[4403, 3570, 3940, 3741, 4514, 3622, 4199, 4779] avg=4096.0000 max=4779 max_rank=7 min=3570 min_rank=1 imbalance_ratio=1.166748 max_load_reduction=27989 max_load_reduction_pct=0.854156 moved_to_echo=17114 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[9, 2077, 1314, 133, 0, 0, 0, 0, 1521, 0, 735, 31, 1058, 655, 0, 0, 725, 611, 1021, 1277, 0, 0, 0, 0, 877, 577, 37, 737, 1450, 244, 0, 0, 939, 903, 710, 1387, 0, 0, 0, 0, 1965, 2036, 716, 85, 581, 0, 0, 0, 4, 754, 219, 1144, 2414, 0, 0, 0, 849, 26, 2824, 13, 110, 0, 0, 0] rerouted_tokens_per_ep_rank=[3533, 4000, 3634, 3922, 3939, 5383, 4535, 3822] avg=4096.0000 max=5383 max_rank=5 min=3533 min_rank=0 imbalance_ratio=1.314209 max_load_reduction=27385 max_load_reduction_pct=0.835724 moved_to_echo=17679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 2216, 2290, 17, 0, 0, 0, 0, 1010, 0, 955, 5, 1129, 653, 0, 0, 722, 328, 1105, 1925, 0, 0, 0, 0, 1415, 491, 12, 547, 1446, 248, 0, 0, 700, 1095, 902, 1404, 0, 0, 0, 0, 1772, 1382, 466, 18, 620, 0, 0, 0, 0, 767, 177, 667, 2444, 0, 0, 0, 789, 43, 2882, 2, 117, 0, 0, 0] rerouted_tokens_per_ep_rank=[4530, 3752, 4080, 4159, 4101, 4258, 4055, 3833] avg=4096.0000 max=4530 max_rank=0 min=3752 min_rank=1 imbalance_ratio=1.105957 max_load_reduction=28238 max_load_reduction_pct=0.861755 moved_to_echo=16247 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 2207, 1995, 19, 0, 0, 0, 0, 1758, 1, 893, 3, 1125, 722, 0, 0, 798, 116, 912, 1925, 0, 0, 0, 0, 1328, 331, 1, 780, 1598, 247, 0, 0, 1017, 1172, 764, 1403, 0, 0, 0, 0, 1156, 1050, 643, 82, 617, 0, 0, 0, 0, 727, 213, 1081, 2441, 0, 0, 0, 680, 24, 2817, 2, 117, 0, 0, 0] rerouted_tokens_per_ep_rank=[4224, 4502, 3751, 4285, 4356, 3548, 4462, 3640] avg=4096.0000 max=4502 max_rank=1 min=3548 min_rank=5 imbalance_ratio=1.099121 max_load_reduction=28266 max_load_reduction_pct=0.862610 moved_to_echo=16006 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 2199, 1966, 13, 0, 0, 0, 0, 1190, 1, 769, 0, 1133, 731, 0, 0, 797, 130, 835, 2027, 0, 0, 0, 0, 1273, 485, 4, 710, 1616, 252, 0, 0, 929, 937, 968, 1394, 0, 0, 0, 0, 1120, 1207, 814, 21, 623, 0, 0, 0, 0, 758, 308, 734, 2450, 0, 0, 0, 1247, 54, 2941, 6, 122, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 3824, 3789, 4340, 4228, 3785, 4250, 4370] avg=4096.0000 max=4370 max_rank=7 min=3785 min_rank=5 imbalance_ratio=1.066895 max_load_reduction=28398 max_load_reduction_pct=0.866638 moved_to_echo=16633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 2191, 2195, 13, 0, 0, 0, 0, 1251, 0, 989, 8, 1117, 770, 0, 0, 853, 117, 1198, 2507, 0, 0, 0, 0, 1249, 428, 5, 516, 1706, 247, 0, 0, 713, 1169, 482, 1402, 0, 0, 0, 0, 1672, 1016, 712, 30, 613, 0, 0, 0, 0, 619, 184, 520, 2440, 0, 0, 0, 906, 29, 2778, 6, 116, 0, 0, 0] rerouted_tokens_per_ep_rank=[4400, 4135, 4675, 4151, 3766, 4043, 3763, 3835] avg=4096.0000 max=4675 max_rank=2 min=3763 min_rank=6 imbalance_ratio=1.141357 max_load_reduction=28093 max_load_reduction_pct=0.857330 moved_to_echo=15407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1, 4081, 1441, 10, 1434, 0, 1028, 3, 3430, 87, 1331, 2585, 1445, 565, 6, 584, 708, 1001, 479, 4095, 1552, 1552, 440, 15, 1, 770, 152, 466, 508, 15, 2981, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 2215, 1441, 10, 0, 0, 0, 0, 1434, 0, 1028, 3, 1129, 794, 0, 0, 878, 87, 1331, 2585, 0, 0, 0, 0, 1445, 565, 6, 584, 1758, 247, 0, 0, 708, 1001, 479, 1404, 0, 0, 0, 0, 1552, 1552, 440, 15, 620, 0, 0, 0, 1, 770, 152, 466, 2444, 0, 0, 0, 508, 15, 2981, 2, 117, 0, 0, 0] rerouted_tokens_per_ep_rank=[3667, 4388, 4881, 4605, 3592, 4179, 3833, 3623] avg=4096.0000 max=4881 max_rank=2 min=3592 min_rank=4 imbalance_ratio=1.191650 max_load_reduction=27887 max_load_reduction_pct=0.851044 moved_to_echo=15227 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11136, 128, 7040, 128, 8960, 5760, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[6656, 8576, 6400, 11264, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9600, 3712, 128, 4864, 12800, 2048, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6400, 1920, 8192, 16512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11776, 11008, 4992, 384, 4992, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 17536, 15104, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 5632, 1792, 6144, 19584, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7808, 384, 23808, 128, 1024, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[16, 0, 234, 4089, 7, 0, 4054, 4091, 0, 131, 0, 3, 0, 0, 4094, 0, 38, 6, 4082, 0, 2, 0, 4072, 190, 0, 0, 4094, 3, 3548, 0, 14, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[6, 1, 431, 4088, 0, 0, 4084, 4066, 0, 7, 0, 0, 0, 0, 4093, 1, 1, 2, 4095, 1, 0, 1, 4032, 544, 0, 3, 4095, 1, 3216, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 73, 4084, 0, 0, 4080, 4096, 9, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4081, 0, 0, 0, 4070, 475, 5, 0, 4095, 2, 3600, 0, 4, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1058, 4096, 6, 3, 4090, 4096, 3, 0, 1, 0, 0, 0, 4095, 1, 1, 1, 4094, 0, 0, 0, 4087, 1048, 0, 15, 4096, 0, 1977, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[21, 1, 927, 4044, 3, 0, 4082, 4065, 0, 6, 1, 0, 0, 36, 4088, 1, 0, 0, 4085, 11, 0, 0, 4072, 650, 0, 20, 4095, 0, 2558, 1, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[7, 1, 170, 4096, 2, 0, 4082, 4091, 0, 8, 0, 0, 1, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 4096, 772, 0, 0, 4089, 0, 3158, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 94, 4096, 0, 0, 4095, 4094, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 4088, 187, 0, 1, 4096, 0, 3825, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[5, 0, 121, 4092, 0, 0, 4091, 4095, 0, 23, 4, 1, 0, 8, 4087, 0, 0, 0, 4048, 0, 0, 1, 4089, 1517, 0, 0, 4096, 1, 2486, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 234, 3704, 0, 0, 0, 0, 7, 0, 4054, 12, 0, 0, 0, 0, 0, 131, 0, 3, 4075, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 38, 6, 4082, 0, 2, 1, 0, 0, 2, 0, 3420, 190, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 3548, 0, 14, 0, 652, 385, 4, 2] rerouted_tokens_per_ep_rank=[3954, 4073, 4209, 4092, 4129, 3612, 4094, 4605] avg=4096.0000 max=4605 max_rank=7 min=3612 min_rank=5 imbalance_ratio=1.124268 max_load_reduction=28163 max_load_reduction_pct=0.859467 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 1, 431, 3703, 0, 0, 0, 0, 0, 0, 4084, 12, 0, 0, 0, 0, 0, 7, 0, 0, 4050, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 1, 2, 4095, 1, 2, 1, 0, 0, 0, 1, 3386, 544, 0, 0, 0, 0, 0, 3, 4091, 1, 0, 0, 0, 0, 3216, 0, 0, 0, 646, 385, 4, 3] rerouted_tokens_per_ep_rank=[4141, 4096, 4057, 4092, 4102, 3931, 4095, 4254] avg=4096.0000 max=4254 max_rank=7 min=3931 min_rank=5 imbalance_ratio=1.038574 max_load_reduction=28514 max_load_reduction_pct=0.870178 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 73, 3699, 0, 0, 0, 0, 0, 0, 4080, 12, 0, 0, 0, 0, 9, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 2, 1, 0, 0, 0, 0, 3418, 475, 0, 0, 0, 0, 5, 0, 4091, 2, 0, 0, 0, 0, 3600, 0, 4, 4, 652, 385, 4, 3] rerouted_tokens_per_ep_rank=[3772, 4092, 4089, 4088, 4084, 3893, 4098, 4652] avg=4096.0000 max=4652 max_rank=7 min=3772 min_rank=0 imbalance_ratio=1.135742 max_load_reduction=28116 max_load_reduction_pct=0.858032 moved_to_echo=16727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1058, 3710, 0, 0, 0, 0, 6, 3, 4090, 12, 0, 0, 0, 0, 3, 0, 1, 0, 4080, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 1, 1, 4094, 0, 2, 1, 0, 0, 0, 0, 3433, 1048, 0, 0, 0, 0, 0, 15, 4092, 0, 0, 0, 0, 0, 1977, 0, 0, 0, 654, 386, 4, 3] rerouted_tokens_per_ep_rank=[4768, 4111, 4084, 4094, 4099, 4481, 4107, 3024] avg=4096.0000 max=4768 max_rank=0 min=3024 min_rank=7 imbalance_ratio=1.164062 max_load_reduction=28000 max_load_reduction_pct=0.854492 moved_to_echo=15711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[7, 1, 170, 3710, 0, 0, 0, 0, 2, 0, 4082, 12, 0, 0, 0, 0, 0, 8, 0, 0, 4075, 0, 0, 0, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 3440, 772, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 3158, 0, 5, 0, 656, 386, 4, 2] rerouted_tokens_per_ep_rank=[3888, 4096, 4083, 4093, 4099, 4212, 4086, 4211] avg=4096.0000 max=4212 max_rank=5 min=3888 min_rank=0 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[21, 1, 927, 3663, 0, 0, 0, 0, 3, 0, 4082, 12, 0, 0, 0, 0, 0, 6, 1, 0, 4049, 0, 0, 0, 0, 36, 4086, 1, 0, 0, 0, 0, 0, 0, 4085, 11, 2, 1, 0, 0, 0, 0, 3420, 650, 0, 0, 0, 0, 0, 20, 4091, 0, 0, 0, 0, 0, 2558, 1, 1, 0, 652, 381, 4, 3] rerouted_tokens_per_ep_rank=[4612, 4097, 4056, 4123, 4099, 4070, 4111, 3600] avg=4096.0000 max=4612 max_rank=0 min=3600 min_rank=7 imbalance_ratio=1.125977 max_load_reduction=28156 max_load_reduction_pct=0.859253 moved_to_echo=15880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 94, 3706, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 1, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 8, 0, 0, 0, 0, 3430, 187, 0, 0, 0, 0, 0, 1, 4083, 0, 0, 0, 0, 0, 3825, 0, 0, 0, 658, 390, 7, 5] rerouted_tokens_per_ep_rank=[3801, 4100, 4083, 4093, 4105, 3617, 4084, 4885] avg=4096.0000 max=4885 max_rank=7 min=3617 min_rank=5 imbalance_ratio=1.192627 max_load_reduction=27883 max_load_reduction_pct=0.850922 moved_to_echo=16691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 121, 3706, 0, 0, 0, 0, 0, 0, 4091, 12, 0, 0, 0, 0, 0, 23, 4, 1, 4079, 0, 0, 0, 0, 8, 4085, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 2, 1, 0, 0, 0, 1, 3434, 1517, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 2486, 0, 3, 0, 655, 386, 4, 3] rerouted_tokens_per_ep_rank=[3832, 4103, 4107, 4093, 4051, 4952, 4093, 3537] avg=4096.0000 max=4952 max_rank=5 min=3537 min_rank=7 imbalance_ratio=1.208984 max_load_reduction=27816 max_load_reduction_pct=0.848877 moved_to_echo=16633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[24448, 128, 128, 128, 5248, 3200, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 27392, 5504, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 3200, 29696, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[52, 0, 1, 0, 68, 0, 0, 4095, 0, 1, 0, 4049, 0, 3344, 4092, 254, 50, 0, 13, 3698, 54, 538, 1, 1, 2351, 0, 4087, 1, 4080, 0, 1938, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4087, 2, 4076, 4096, 0, 0, 0, 1, 4095, 0, 398, 0, 0, 3938, 0, 3224, 0, 4094, 2, 656, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4, 4032, 0, 3360, 4091, 1, 0, 0, 0, 4075, 0, 374, 0, 0, 3752, 0, 3954, 0, 4096, 0, 926, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 8, 0, 1, 1, 1, 12, 4094, 0, 0, 17, 4050, 9, 2162, 4096, 0, 0, 3, 0, 4048, 67, 2380, 0, 0, 3957, 0, 3341, 0, 4070, 0, 445, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4058, 0, 2286, 4096, 0, 0, 0, 0, 4095, 18, 2073, 0, 0, 3715, 0, 3144, 0, 4096, 0, 1091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 2, 0, 1, 4096, 0, 3, 3, 4078, 0, 3783, 4094, 12, 0, 0, 0, 4083, 0, 1986, 0, 0, 1805, 0, 3416, 0, 4094, 0, 1310, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 1, 0, 0, 2, 0, 4096, 0, 0, 0, 4058, 0, 2944, 4092, 5, 1, 0, 0, 4043, 0, 685, 0, 0, 3986, 0, 4028, 0, 4095, 0, 726, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4045, 0, 2653, 4094, 0, 0, 0, 1, 4095, 21, 1607, 0, 0, 3755, 1, 3470, 0, 4093, 6, 823, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3110, 879, 0, 0, 0, 0, 1, 4086, 0, 0, 0, 0, 0, 0, 0, 4087, 23, 10, 0, 0, 2, 4076, 986, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 58, 0, 0, 0, 0, 398, 0, 0, 1725, 904, 0, 0, 3938, 0, 620, 0, 0, 0, 0, 0, 3109, 2, 656, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3989, 4087, 4120, 5064, 4154, 3027, 4558, 3769] avg=4096.0000 max=5064 max_rank=3 min=3027 min_rank=5 imbalance_ratio=1.236328 max_load_reduction=27704 max_load_reduction_pct=0.845459 moved_to_echo=15508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[52, 0, 1, 0, 3107, 1114, 0, 0, 68, 0, 0, 4085, 0, 0, 0, 0, 0, 1, 0, 4049, 23, 10, 0, 0, 0, 3344, 985, 254, 0, 0, 0, 0, 50, 0, 13, 3698, 58, 0, 0, 0, 54, 538, 1, 1, 2186, 901, 0, 0, 2351, 0, 787, 1, 0, 0, 0, 0, 3098, 0, 1938, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4274, 4153, 4083, 4583, 3819, 3681, 3139, 5036] avg=4096.0000 max=5036 max_rank=7 min=3139 min_rank=6 imbalance_ratio=1.229492 max_load_reduction=27732 max_load_reduction_pct=0.846313 moved_to_echo=15675 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 3107, 1078, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 6, 4, 4032, 23, 10, 0, 0, 0, 3360, 984, 1, 0, 0, 0, 0, 0, 0, 0, 4075, 58, 0, 0, 0, 0, 374, 0, 0, 2115, 904, 0, 0, 3752, 0, 761, 0, 0, 0, 0, 0, 3111, 0, 926, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4086, 4075, 4345, 4133, 3393, 4513, 4037] avg=4096.0000 max=4513 max_rank=6 min=3393 min_rank=5 imbalance_ratio=1.101807 max_load_reduction=28255 max_load_reduction_pct=0.862274 moved_to_echo=16076 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 3109, 931, 0, 0, 2, 0, 1, 4086, 0, 0, 0, 0, 0, 3, 3, 4078, 23, 10, 0, 0, 0, 3783, 985, 12, 0, 0, 0, 0, 0, 0, 0, 4083, 58, 0, 0, 0, 0, 1986, 0, 0, 1827, 904, 0, 0, 1805, 0, 658, 0, 0, 0, 0, 0, 3109, 0, 1310, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4089, 4117, 4780, 4141, 4717, 2463, 4419] avg=4096.0000 max=4780 max_rank=3 min=2463 min_rank=6 imbalance_ratio=1.166992 max_load_reduction=27988 max_load_reduction_pct=0.854126 moved_to_echo=15740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 1, 0, 3107, 1098, 0, 0, 0, 2, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 4058, 23, 10, 0, 0, 0, 2944, 985, 5, 0, 0, 0, 0, 1, 0, 0, 4043, 58, 0, 0, 0, 0, 685, 0, 0, 2155, 904, 0, 0, 3986, 0, 775, 0, 0, 0, 0, 0, 3110, 0, 726, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4212, 4088, 4091, 3934, 4102, 3744, 4761, 3836] avg=4096.0000 max=4761 max_rank=6 min=3744 min_rank=5 imbalance_ratio=1.162354 max_load_reduction=28007 max_load_reduction_pct=0.854706 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3115, 862, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4058, 27, 15, 0, 0, 0, 2286, 981, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 61, 0, 0, 0, 18, 2073, 0, 0, 1687, 907, 0, 0, 3715, 0, 595, 0, 0, 0, 0, 0, 3101, 0, 1091, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3977, 4081, 4100, 3267, 4156, 4685, 4310, 4192] avg=4096.0000 max=4685 max_rank=5 min=3267 min_rank=3 imbalance_ratio=1.143799 max_load_reduction=28083 max_load_reduction_pct=0.857025 moved_to_echo=17343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 1, 3110, 911, 0, 0, 1, 1, 12, 4084, 0, 0, 0, 0, 0, 0, 17, 4050, 23, 10, 0, 0, 9, 2162, 986, 0, 0, 0, 0, 0, 0, 3, 0, 4048, 58, 0, 0, 0, 67, 2380, 0, 0, 1787, 899, 0, 0, 3957, 0, 643, 0, 0, 0, 0, 0, 3090, 0, 445, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4098, 4100, 3157, 4109, 5133, 4600, 3541] avg=4096.0000 max=5133 max_rank=5 min=3157 min_rank=3 imbalance_ratio=1.253174 max_load_reduction=27635 max_load_reduction_pct=0.843353 moved_to_echo=17383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 0, 0, 3109, 946, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4, 4045, 23, 10, 0, 0, 0, 2653, 985, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 58, 0, 0, 0, 21, 1607, 0, 0, 1856, 904, 0, 0, 3755, 1, 668, 0, 0, 0, 0, 0, 3108, 6, 823, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4086, 4082, 3638, 4154, 4388, 4424, 3938] avg=4096.0000 max=4424 max_rank=6 min=3638 min_rank=3 imbalance_ratio=1.080078 max_load_reduction=28344 max_load_reduction_pct=0.864990 moved_to_echo=16904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 24960, 7936, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 10112, 128, 128, 15360, 7296, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 24704, 7936, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[24960, 128, 7936, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[27264, 128, 5632, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32256, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 32512, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 1, 0, 1, 1, 4, 4094, 2331, 0, 1, 2, 4096, 0, 1, 3641, 0, 4095, 4091, 0, 2214, 0, 2, 0, 0, 0, 0, 0, 1, 4096, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4053, 0, 1, 0, 1, 0, 39, 4094, 1506, 0, 0, 84, 4029, 1, 0, 4013, 0, 4096, 4096, 0, 2598, 0, 0, 62, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 1373, 0, 0, 120, 4096, 0, 0, 4091, 0, 4096, 4096, 0, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 4096, 787, 0, 0, 691, 4096, 0, 0, 4095, 0, 4094, 4095, 0, 2623, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 15, 4096, 3668, 0, 0, 0, 4096, 0, 0, 3963, 0, 4092, 4096, 0, 524, 0, 0, 7, 0, 21, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4047, 0, 48, 6, 0, 0, 1, 4090, 223, 2, 0, 6, 4092, 0, 0, 4048, 3, 4084, 4071, 3, 3902, 4, 0, 1, 25, 0, 0, 0, 0, 4096, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4068, 0, 0, 0, 0, 0, 1, 4095, 407, 0, 0, 209, 4096, 0, 0, 3963, 1, 4087, 4096, 0, 3634, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4082, 3, 3, 0, 1, 0, 9, 4092, 3400, 1, 0, 25, 4093, 0, 0, 4011, 0, 4094, 4095, 0, 755, 0, 0, 0, 1, 6, 1, 0, 0, 4095, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 1, 0, 5, 4, 0, 0, 1, 1, 4, 4088, 0, 0, 0, 0, 2331, 0, 1, 2, 2246, 0, 0, 0, 119, 0, 1, 3641, 0, 0, 0, 0, 0, 4, 4091, 0, 0, 0, 0, 0, 2214, 0, 2, 0, 1731, 1, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 1, 4092, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4094, 4580, 3761, 4095, 3948, 4091, 4095] avg=4096.0000 max=4580 max_rank=2 min=3761 min_rank=3 imbalance_ratio=1.118164 max_load_reduction=28188 max_load_reduction_pct=0.860229 moved_to_echo=16229 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4053, 0, 1, 0, 5, 4, 0, 0, 1, 0, 39, 4088, 0, 0, 0, 0, 1506, 0, 0, 84, 2209, 0, 0, 0, 117, 1, 0, 4013, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 2598, 0, 0, 62, 1703, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4128, 3799, 4131, 4097, 4364, 4095, 4091] avg=4096.0000 max=4364 max_rank=5 min=3799 min_rank=2 imbalance_ratio=1.065430 max_load_reduction=28404 max_load_reduction_pct=0.866821 moved_to_echo=16647 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 5, 4, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 787, 0, 0, 691, 2246, 0, 0, 0, 119, 0, 0, 4095, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 2623, 0, 0, 0, 1731, 1, 0, 0, 0, 1, 0, 0, 4090, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4090, 3724, 4214, 4099, 4355, 4091, 4092] avg=4096.0000 max=4355 max_rank=5 min=3724 min_rank=2 imbalance_ratio=1.063232 max_load_reduction=28413 max_load_reduction_pct=0.867096 moved_to_echo=16637 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 5, 4, 0, 0, 0, 0, 15, 4090, 0, 0, 0, 0, 3668, 0, 0, 0, 2246, 0, 0, 0, 119, 0, 0, 3963, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 524, 0, 0, 7, 1731, 1, 0, 0, 0, 21, 0, 0, 4088, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4105, 5914, 4082, 4100, 2263, 4109, 4092] avg=4096.0000 max=5914 max_rank=2 min=2263 min_rank=5 imbalance_ratio=1.443848 max_load_reduction=26854 max_load_reduction_pct=0.819519 moved_to_echo=14564 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 9, 6, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 1373, 0, 0, 120, 2251, 0, 0, 0, 110, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2607, 0, 0, 0, 1735, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4080, 3744, 4201, 4096, 4349, 4096, 4091] avg=4096.0000 max=4349 max_rank=5 min=3744 min_rank=2 imbalance_ratio=1.061768 max_load_reduction=28419 max_load_reduction_pct=0.867279 moved_to_echo=16632 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4068, 0, 0, 0, 5, 4, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 407, 0, 0, 209, 2246, 0, 0, 0, 119, 0, 0, 3963, 0, 0, 0, 0, 1, 4, 4096, 0, 0, 0, 0, 0, 3634, 0, 0, 0, 1731, 1, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4092, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4090, 2862, 4082, 4101, 5366, 4083, 4107] avg=4096.0000 max=5366 max_rank=5 min=2862 min_rank=2 imbalance_ratio=1.310059 max_load_reduction=27402 max_load_reduction_pct=0.836243 moved_to_echo=17657 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4047, 0, 48, 6, 5, 4, 0, 0, 0, 0, 1, 4084, 0, 0, 0, 0, 223, 2, 0, 6, 2244, 0, 0, 0, 119, 0, 0, 4048, 0, 0, 0, 0, 3, 4, 4071, 3, 0, 0, 0, 0, 3902, 4, 0, 1, 1729, 1, 0, 0, 25, 0, 0, 0, 4080, 0, 0, 0, 0, 4092, 16, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4085, 2475, 4167, 4081, 5637, 4105, 4108] avg=4096.0000 max=5637 max_rank=5 min=2475 min_rank=2 imbalance_ratio=1.376221 max_load_reduction=27131 max_load_reduction_pct=0.827972 moved_to_echo=17931 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4082, 3, 3, 0, 5, 4, 0, 0, 1, 0, 9, 4086, 0, 0, 0, 0, 3400, 1, 0, 25, 2244, 0, 0, 0, 119, 0, 0, 4011, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 755, 0, 0, 0, 1730, 1, 0, 0, 1, 6, 1, 0, 4090, 0, 0, 0, 0, 4091, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 5670, 4130, 4099, 2486, 4098, 4092] avg=4096.0000 max=5670 max_rank=2 min=2486 min_rank=5 imbalance_ratio=1.384277 max_load_reduction=27098 max_load_reduction_pct=0.826965 moved_to_echo=14775 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[18944, 128, 128, 128, 13824, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13696, 128, 128, 1152, 18048, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 128, 128, 31872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4087, 4096, 2, 0, 0, 4096, 0, 3976, 1, 0, 0, 0, 4095, 0, 0, 1, 262, 0, 4096, 0, 3699, 1, 3, 1, 0, 0, 0, 4096, 254, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4096, 0, 0, 0, 4095, 0, 2605, 0, 0, 0, 0, 4096, 0, 0, 0, 351, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 1146, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3996, 4095, 1, 0, 1, 4093, 3, 2180, 0, 0, 0, 0, 4047, 0, 0, 0, 1721, 0, 4096, 0, 4087, 4, 0, 91, 0, 0, 0, 4096, 257, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4039, 4096, 0, 0, 0, 4096, 0, 3967, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 4057, 7, 2, 45, 0, 0, 0, 4096, 166, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4068, 4096, 0, 1, 0, 4096, 1, 3913, 0, 0, 0, 0, 4085, 0, 0, 0, 249, 0, 4095, 0, 4048, 10, 0, 10, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3988, 4096, 0, 0, 0, 4056, 5, 2531, 2, 0, 0, 14, 4096, 0, 0, 23, 708, 0, 4077, 0, 4096, 47, 13, 45, 0, 1, 0, 4096, 874, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 4089, 4094, 5, 0, 2, 4095, 0, 3598, 0, 0, 0, 1, 4096, 0, 0, 0, 124, 0, 4093, 2, 4035, 0, 37, 265, 1, 0, 0, 4095, 135, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4087, 63, 0, 0, 0, 0, 2, 0, 0, 4096, 6, 0, 0, 0, 0, 3976, 1, 0, 430, 342, 0, 0, 0, 0, 4095, 0, 5, 0, 0, 0, 0, 1, 262, 0, 3603, 60, 0, 0, 53, 0, 3699, 1, 0, 0, 0, 0, 3, 1, 0, 0, 3983, 0, 0, 0, 0, 3743, 254, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4104, 4749, 4100, 3926, 3753, 3987, 3999] avg=4096.0000 max=4749 max_rank=2 min=3753 min_rank=5 imbalance_ratio=1.159424 max_load_reduction=28019 max_load_reduction_pct=0.855072 moved_to_echo=15665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4039, 63, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 3967, 0, 0, 430, 342, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 5, 0, 3603, 60, 0, 0, 53, 0, 4057, 7, 0, 0, 0, 0, 2, 45, 0, 0, 3983, 0, 0, 0, 0, 3743, 166, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4102, 4739, 4101, 3668, 4117, 4030, 3909] avg=4096.0000 max=4739 max_rank=2 min=3668 min_rank=4 imbalance_ratio=1.156982 max_load_reduction=28029 max_load_reduction_pct=0.855377 moved_to_echo=15724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3996, 63, 0, 0, 0, 0, 1, 0, 1, 4093, 6, 0, 0, 0, 3, 2180, 0, 0, 430, 342, 0, 0, 0, 0, 4047, 0, 5, 0, 0, 0, 0, 0, 1721, 0, 3602, 60, 0, 0, 53, 0, 4087, 4, 0, 0, 0, 0, 0, 91, 0, 0, 3983, 0, 0, 0, 0, 3743, 257, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4101, 2955, 4052, 5383, 4144, 4074, 4000] avg=4096.0000 max=5383 max_rank=4 min=2955 min_rank=2 imbalance_ratio=1.314209 max_load_reduction=27385 max_load_reduction_pct=0.835724 moved_to_echo=17601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4068, 63, 0, 0, 0, 0, 0, 1, 0, 4096, 6, 0, 0, 0, 1, 3913, 0, 0, 430, 342, 0, 0, 0, 0, 4085, 0, 5, 0, 0, 0, 0, 0, 249, 0, 3603, 60, 0, 0, 53, 0, 4048, 10, 0, 0, 0, 0, 0, 10, 0, 0, 3982, 0, 0, 0, 0, 3743, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4103, 4686, 4090, 3912, 4111, 3992, 3743] avg=4096.0000 max=4686 max_rank=2 min=3743 min_rank=7 imbalance_ratio=1.144043 max_load_reduction=28082 max_load_reduction_pct=0.856995 moved_to_echo=15758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 53, 0, 0, 0, 0, 0, 0, 0, 4095, 13, 0, 0, 0, 0, 2605, 0, 0, 434, 347, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 351, 0, 3609, 65, 0, 0, 46, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 3725, 1146, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4108, 3386, 4107, 4025, 4139, 3985, 4871] avg=4096.0000 max=4871 max_rank=7 min=3386 min_rank=2 imbalance_ratio=1.189209 max_load_reduction=27897 max_load_reduction_pct=0.851349 moved_to_echo=17020 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3988, 63, 0, 0, 0, 0, 0, 0, 0, 4056, 6, 0, 0, 0, 5, 2531, 2, 0, 430, 342, 0, 0, 0, 14, 4096, 0, 5, 0, 0, 0, 0, 23, 708, 0, 3603, 60, 0, 0, 53, 0, 4096, 47, 0, 0, 0, 0, 13, 45, 0, 1, 3964, 0, 0, 0, 0, 3743, 874, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4062, 3310, 4115, 4394, 4196, 4023, 4617] avg=4096.0000 max=4617 max_rank=7 min=3310 min_rank=2 imbalance_ratio=1.127197 max_load_reduction=28151 max_load_reduction_pct=0.859100 moved_to_echo=17230 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4089, 62, 0, 0, 0, 0, 5, 0, 2, 4095, 6, 0, 0, 0, 0, 3598, 0, 0, 430, 342, 0, 0, 0, 1, 4096, 0, 5, 0, 0, 0, 0, 0, 124, 0, 3602, 60, 0, 0, 53, 2, 4035, 0, 0, 0, 0, 0, 37, 265, 1, 0, 3980, 0, 0, 0, 0, 3742, 135, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4108, 4370, 4102, 3786, 4090, 4283, 3877] avg=4096.0000 max=4370 max_rank=2 min=3786 min_rank=4 imbalance_ratio=1.066895 max_load_reduction=28398 max_load_reduction_pct=0.866638 moved_to_echo=16036 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3913, 4096, 0, 0, 25, 4049, 157, 3643, 1, 0, 0, 0, 4096, 0, 0, 2, 9, 0, 4059, 0, 4094, 72, 0, 447, 0, 0, 0, 4095, 10, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3913, 63, 0, 0, 0, 0, 0, 0, 25, 4049, 6, 0, 0, 0, 157, 3643, 1, 0, 430, 342, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 2, 9, 0, 3603, 60, 0, 0, 52, 0, 4094, 72, 0, 0, 0, 0, 0, 447, 0, 0, 3947, 0, 0, 0, 0, 3742, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3976, 4080, 4573, 4101, 3674, 4218, 4394, 3752] avg=4096.0000 max=4573 max_rank=2 min=3674 min_rank=4 imbalance_ratio=1.116455 max_load_reduction=28195 max_load_reduction_pct=0.860443 moved_to_echo=16038 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 26496, 128, 0, 3456, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32384, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 29952, 2944, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 3456, 0, 28928, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 128, 32256, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1024, 128, 128, 31872, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 4, 0, 18, 0, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1, 4094, 4067, 4095, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 4096, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 52, 1, 517, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 367, 0, 4043, 3200, 4025, 0, 0, 5, 0, 35, 4085, 0, 45, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 29, 0, 4082, 4054, 4096, 0, 0, 2, 0, 26, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 28, 0, 4088, 4063, 4085, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4, 0, 4081, 4018, 4079, 0, 0, 8, 0, 91, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3, 101, 0, 4096, 4088, 4002, 0, 0, 0, 0, 1, 4092, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 4080, 4096, 0, 0, 0, 0, 2, 4096, 0, 19, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4, 4095, 4096, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 1, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4022, 52, 1, 517, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 367, 0, 3888, 74, 9, 9, 0, 3200, 137, 0, 0, 0, 0, 0, 0, 5, 0, 35, 4034, 0, 0, 0, 4076, 0, 45, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4592, 4096, 4096, 4356, 3337, 4074, 4121, 4096] avg=4096.0000 max=4592 max_rank=0 min=3337 min_rank=4 imbalance_ratio=1.121094 max_load_reduction=28176 max_load_reduction_pct=0.859863 moved_to_echo=15628 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4022, 4, 0, 18, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 1, 3956, 74, 9, 8, 0, 4067, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4087, 0, 8, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4096, 4096, 4049, 4206, 4086, 4095, 4096] avg=4096.0000 max=4206 max_rank=4 min=4044 min_rank=0 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16483 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 29, 0, 3957, 74, 9, 9, 0, 4054, 139, 0, 0, 0, 0, 0, 0, 2, 0, 26, 4073, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4096, 4095, 4078, 4193, 4101, 4087, 4096] avg=4096.0000 max=4193 max_rank=4 min=4022 min_rank=0 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4022, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 28, 0, 3946, 74, 9, 8, 0, 4063, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 4087, 0, 22, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4096, 4096, 4065, 4202, 4080, 4109, 4096] avg=4096.0000 max=4202 max_rank=4 min=4024 min_rank=0 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4022, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 4, 0, 3940, 74, 9, 8, 0, 4018, 139, 0, 0, 0, 0, 0, 0, 8, 0, 91, 4073, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4096, 4096, 4041, 4157, 4172, 4087, 4096] avg=4096.0000 max=4172 max_rank=5 min=4023 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16512 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4015, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3962, 81, 15, 9, 0, 4080, 134, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4077, 0, 0, 0, 4080, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4096, 4097, 4067, 4214, 4079, 4099, 4096] avg=4096.0000 max=4214 max_rank=4 min=4020 min_rank=0 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16488 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 3, 101, 0, 3866, 74, 9, 9, 0, 4088, 136, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4096, 4096, 4063, 4224, 4088, 4083, 4096] avg=4096.0000 max=4224 max_rank=4 min=4022 min_rank=0 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16491 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4021, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4, 3953, 74, 9, 8, 0, 4096, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 4096, 4096, 4049, 4235, 4088, 4087, 4096] avg=4096.0000 max=4235 max_rank=4 min=4021 min_rank=0 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16506 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32256, 128, 128, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 640, 128, 31488, 640, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 31744, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4095, 5, 7, 0, 4089, 2, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 1, 0, 4094, 0, 0, 4096, 0, 0, 4089, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 11, 2, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4083, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 68, 0, 0, 4096, 0, 0, 0, 4061, 0, 0, 4065, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 237, 0, 0, 4096, 0, 0, 4094, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3857, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 3, 0, 0, 4096, 34, 98, 0, 4085, 11, 0, 4096, 0, 0, 4, 0, 4096, 0, 0, 0, 3, 4092, 0, 0, 4096, 0, 0, 3962, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 15, 1, 0, 4096, 0, 0, 0, 4096, 2, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4085, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 2, 0, 0, 4096, 10, 75, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4011, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4, 0, 0, 4095, 4, 1, 0, 4088, 0, 1, 4096, 0, 0, 0, 1, 4095, 1, 0, 0, 0, 4095, 0, 1, 4095, 0, 1, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 4065, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 52, 8, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4154, 4009, 4067, 4096, 4096, 4096, 4154] avg=4096.0000 max=4154 max_rank=1 min=4009 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 5, 7, 0, 4037, 0, 0, 0, 0, 2, 0, 4096, 2, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4089, 0, 52, 8, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4085, 4049, 4102, 4096, 4096, 4095, 4149] avg=4096.0000 max=4149 max_rank=7 min=4049 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 237, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3857, 0, 52, 8, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4086, 4281, 4096, 4100, 4095, 4097, 3917] avg=4096.0000 max=4281 max_rank=2 min=3917 min_rank=7 imbalance_ratio=1.045166 max_load_reduction=28487 max_load_reduction_pct=0.869354 moved_to_echo=16209 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 15, 1, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 2, 0, 4089, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4085, 0, 52, 8, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4102, 4044, 4093, 4096, 4095, 4097, 4145] avg=4096.0000 max=4145 max_rank=7 min=4044 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4086, 0, 0, 0, 0, 10, 75, 0, 4042, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4011, 0, 52, 8, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4088, 4127, 4098, 4096, 4095, 4097, 4071] avg=4096.0000 max=4127 max_rank=2 min=4071 min_rank=7 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 11, 2, 0, 4039, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 2, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 55, 15, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4075, 4052, 4104, 4088, 4098, 4096, 4153] avg=4096.0000 max=4153 max_rank=7 min=4052 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4086, 0, 0, 0, 0, 34, 98, 0, 4033, 0, 0, 0, 0, 11, 0, 4096, 0, 2, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3962, 0, 52, 8, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4089, 4165, 4109, 4100, 4098, 4093, 4022] avg=4096.0000 max=4165 max_rank=2 min=4022 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4085, 0, 0, 0, 0, 4, 1, 0, 4036, 0, 0, 0, 0, 0, 1, 4096, 0, 2, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 1, 4096, 0, 52, 8, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4089, 4041, 4099, 4096, 4095, 4097, 4157] avg=4096.0000 max=4157 max_rank=7 min=4041 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 256, 0, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 32384, 0, 512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4, 4, 4096, 4096, 4089, 0, 0, 174, 0, 0, 4096, 0, 3921] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 0, 0, 4096, 4096, 4091, 0, 0, 129, 0, 0, 4096, 0, 3969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 10, 4096, 4096, 4095, 0, 0, 262, 0, 0, 4096, 0, 3824] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 0, 10, 179, 4096, 0, 45, 0, 4089, 4096, 3984, 0, 0, 822, 0, 0, 4096, 0, 3159] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 27, 4096, 0, 24, 2, 4096, 4096, 3313, 0, 0, 1139, 0, 0, 4096, 0, 3687] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 187, 4096, 0, 0, 11, 4094, 4096, 4073, 0, 0, 483, 0, 0, 4096, 0, 3440] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4096, 0, 9, 15, 4092, 4096, 4087, 0, 0, 1371, 0, 0, 4096, 0, 2711] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 0, 2, 0, 1, 4, 1, 0, 4094, 0, 0, 6, 10, 4092, 1, 1, 0, 4094, 4095, 4077, 0, 0, 1405, 0, 0, 4095, 0, 2694] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3981, 115, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 4, 0, 0, 0, 0, 4, 115, 0, 4089, 0, 0, 0, 0, 0, 0, 174, 0, 3310, 60, 2, 0, 0, 671, 0, 3921, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4097, 4094, 4040, 4208, 3546, 4592] avg=4096.0000 max=4592 max_rank=7 min=3546 min_rank=6 imbalance_ratio=1.121094 max_load_reduction=28176 max_load_reduction_pct=0.859863 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3981, 115, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 10, 115, 0, 4095, 0, 0, 0, 0, 0, 0, 262, 0, 3310, 60, 2, 0, 0, 671, 0, 3824, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4095, 4036, 4220, 3634, 4495] avg=4096.0000 max=4495 max_rank=7 min=3634 min_rank=6 imbalance_ratio=1.097412 max_load_reduction=28273 max_load_reduction_pct=0.862823 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3981, 115, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 27, 4036, 0, 24, 0, 0, 0, 0, 2, 115, 0, 3313, 0, 0, 0, 0, 0, 0, 1139, 0, 3310, 60, 2, 0, 0, 671, 0, 3687, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4094, 4087, 3430, 4511, 4358] avg=4096.0000 max=4511 max_rank=6 min=3430 min_rank=5 imbalance_ratio=1.101318 max_load_reduction=28257 max_load_reduction_pct=0.862335 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3977, 115, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 4036, 0, 9, 0, 0, 0, 0, 15, 115, 0, 4087, 0, 0, 0, 0, 0, 0, 1371, 0, 3310, 60, 2, 0, 0, 671, 0, 2711, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4092, 4097, 4045, 4217, 4743, 3382] avg=4096.0000 max=4743 max_rank=6 min=3382 min_rank=7 imbalance_ratio=1.157959 max_load_reduction=28025 max_load_reduction_pct=0.855255 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3984, 119, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 3, 4030, 0, 0, 0, 0, 0, 0, 0, 112, 0, 4091, 0, 0, 0, 0, 0, 0, 129, 0, 3311, 66, 3, 0, 0, 666, 0, 3969, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4103, 4092, 4033, 4203, 3509, 4635] avg=4096.0000 max=4635 max_rank=7 min=3509 min_rank=6 imbalance_ratio=1.131592 max_load_reduction=28133 max_load_reduction_pct=0.858551 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3979, 115, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 187, 4036, 0, 0, 0, 0, 0, 0, 11, 115, 0, 4073, 0, 0, 0, 0, 0, 0, 483, 0, 3310, 60, 2, 0, 0, 671, 0, 3440, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4094, 4094, 4223, 4199, 3855, 4111] avg=4096.0000 max=4223 max_rank=4 min=3855 min_rank=6 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 3974, 115, 0, 0, 4093, 1, 0, 10, 0, 0, 0, 0, 179, 4036, 0, 45, 0, 0, 0, 0, 0, 115, 0, 3984, 0, 0, 0, 0, 0, 0, 822, 0, 3310, 60, 2, 0, 0, 671, 0, 3159, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4090, 4104, 4260, 4099, 4194, 3830] avg=4096.0000 max=4260 max_rank=4 min=3830 min_rank=7 imbalance_ratio=1.040039 max_load_reduction=28508 max_load_reduction_pct=0.869995 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 1, 4, 1, 0, 3979, 115, 0, 0, 4092, 0, 0, 6, 0, 0, 0, 0, 10, 4032, 1, 1, 0, 0, 0, 0, 0, 115, 0, 4077, 0, 0, 0, 0, 0, 0, 1405, 0, 3309, 60, 2, 0, 0, 671, 0, 2694, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4100, 4098, 4044, 4192, 4776, 3365] avg=4096.0000 max=4776 max_rank=6 min=3365 min_rank=7 imbalance_ratio=1.166016 max_load_reduction=27992 max_load_reduction_pct=0.854248 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 5888, 0, 26496, 512, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 5376, 0, 27520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 1024, 0, 31872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 31872, 1024, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 32384, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 415, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 3681, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 216, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 3880, 0, 4094, 0, 0, 0, 4096, 4095, 0, 0, 4096, 2, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2984, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1124, 6, 4096, 0, 0, 0, 4080, 4081, 0, 13, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 99, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 3997, 0, 4096, 0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1, 1, 744, 4096, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 3367, 8, 4096, 1, 0, 0, 4072, 4095, 0, 0, 4096, 1, 0, 4096, 1, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 906, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 3171, 6, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4088, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 578, 4095, 0, 1, 1, 1, 0, 0, 0, 4095, 4, 1, 0, 3510, 7, 4095, 1, 1, 0, 4089, 4093, 0, 0, 4095, 3, 1, 4095, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 216, 3352, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 8, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3880, 736, 3, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3568, 4097, 4097, 4619, 4091, 4102, 4098, 4096] avg=4096.0000 max=4619 max_rank=3 min=3568 min_rank=0 imbalance_ratio=1.127686 max_load_reduction=28149 max_load_reduction_pct=0.859039 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 415, 3352, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 3681, 736, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3767, 4098, 4096, 4421, 4093, 4102, 4095, 4096] avg=4096.0000 max=4421 max_rank=3 min=3767 min_rank=0 imbalance_ratio=1.079346 max_load_reduction=28347 max_load_reduction_pct=0.865082 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2984, 3352, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1124, 736, 3, 0, 0, 6, 4093, 0, 0, 0, 0, 0, 0, 0, 4080, 6, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6336, 4083, 4096, 1863, 4099, 4086, 4109, 4096] avg=4096.0000 max=6336 max_rank=0 min=1863 min_rank=3 imbalance_ratio=1.546875 max_load_reduction=26432 max_load_reduction_pct=0.806641 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 99, 3352, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 3997, 736, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3451, 4096, 4096, 4738, 4093, 4102, 4096, 4096] avg=4096.0000 max=4738 max_rank=3 min=3451 min_rank=0 imbalance_ratio=1.156738 max_load_reduction=28030 max_load_reduction_pct=0.855408 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 906, 3352, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 0, 0, 3171, 736, 3, 0, 0, 6, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4258, 4098, 4096, 3931, 4099, 4102, 4088, 4096] avg=4096.0000 max=4258 max_rank=0 min=3931 min_rank=3 imbalance_ratio=1.039551 max_load_reduction=28510 max_load_reduction_pct=0.870056 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 10, 3348, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 10, 0, 0, 0, 0, 0, 4096, 3, 1, 0, 0, 0, 0, 0, 4086, 738, 6, 7, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3358, 4101, 4100, 4837, 4090, 4101, 4086, 4095] avg=4096.0000 max=4837 max_rank=3 min=3358 min_rank=0 imbalance_ratio=1.180908 max_load_reduction=27931 max_load_reduction_pct=0.852386 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 744, 3352, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 8, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 3367, 736, 3, 0, 0, 8, 4093, 1, 0, 0, 0, 0, 0, 0, 4072, 6, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4097, 4092, 4106, 4102, 4078, 4097, 4098] avg=4096.0000 max=4106 max_rank=3 min=4078 min_rank=5 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 578, 3352, 0, 0, 0, 0, 0, 1, 1, 1, 4087, 8, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 1, 0, 3510, 735, 3, 0, 0, 7, 4092, 1, 1, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 0, 0, 0, 4095, 3, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3932, 4098, 4095, 4253, 4101, 4095, 4099, 4095] avg=4096.0000 max=4253 max_rank=3 min=3932 min_rank=0 imbalance_ratio=1.038330 max_load_reduction=28515 max_load_reduction_pct=0.870209 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 6016, 26880, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 26880, 6016, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 15, 0, 0, 4094, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4083, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 2, 4093, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 687, 0, 0, 3423, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4062, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4094, 44, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4052, 0, 15, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 4, 4081, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 10, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 57, 0, 0, 4039, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 5, 0, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1, 4095, 20, 0, 0, 4075, 0, 4095, 0, 0, 0, 1, 4095, 1, 1, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 4096, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3994, 15, 0, 0, 0, 0, 0, 0, 4094, 0, 96, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 3993, 101, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 8, 2, 4093, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4009, 4190, 4096, 4096, 4085, 4094, 4095, 4103] avg=4096.0000 max=4190 max_rank=1 min=4009 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3994, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 3993, 101, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 4185, 4095, 4096, 4104, 4094, 4096, 4104] avg=4096.0000 max=4185 max_rank=1 min=3994 min_rank=0 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3994, 687, 0, 0, 0, 0, 0, 0, 3423, 1, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 1, 0, 0, 0, 0, 0, 0, 0, 3993, 101, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4062, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4681, 3520, 4095, 4096, 4116, 4094, 4096, 4070] avg=4096.0000 max=4681 max_rank=0 min=3520 min_rank=1 imbalance_ratio=1.142822 max_load_reduction=28087 max_load_reduction_pct=0.857147 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3979, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 9, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4000, 107, 0, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3979, 4183, 4102, 4095, 4103, 4107, 4100, 4099] avg=4096.0000 max=4183 max_rank=1 min=3979 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3994, 57, 0, 0, 0, 0, 0, 0, 4039, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 5, 1, 0, 0, 0, 0, 2, 0, 0, 3993, 101, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4091, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4135, 4095, 4096, 4100, 4096, 4096, 4099] avg=4096.0000 max=4135 max_rank=1 min=4051 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3994, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3993, 101, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 8, 0, 4087, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4004, 4192, 4095, 4096, 4095, 4094, 4097, 4095] avg=4096.0000 max=4192 max_rank=1 min=4004 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3992, 44, 0, 0, 0, 0, 0, 0, 4096, 1, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 15, 1, 0, 0, 0, 0, 0, 0, 0, 3993, 101, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 8, 4, 4081, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4193, 4095, 4096, 4068, 4094, 4093, 4093] avg=4096.0000 max=4193 max_rank=1 min=4036 min_rank=0 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 3993, 20, 0, 0, 0, 0, 0, 0, 4075, 0, 95, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 7, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 3993, 101, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4014, 4170, 4095, 4097, 4097, 4095, 4095, 4105] avg=4096.0000 max=4170 max_rank=1 min=4014 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32000, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32000, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32128, 128, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 4083, 4096, 4096, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4081, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4092, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 3734, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 4095, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 4042, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 3920, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4032, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 217, 3942, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4068, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4094, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4057, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 4094, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 1, 0, 0, 0, 4095, 4095, 4094, 1, 4095, 4095, 0, 2, 0, 1, 0, 0, 0, 0, 1, 2, 0, 0, 0, 28, 4067, 1, 4095, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 4083, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4085, 17, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 25, 3978, 0, 0, 0, 4081, 0, 101, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4094, 4096, 4105, 4096, 4003, 4182] avg=4096.0000 max=4182 max_rank=7 min=4003 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16386 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 17, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 366, 3978, 0, 0, 0, 3734, 0, 101, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4103, 4096, 4102, 4096, 4344, 3835] avg=4096.0000 max=4344 max_rank=6 min=3835 min_rank=7 imbalance_ratio=1.060547 max_load_reduction=28424 max_load_reduction_pct=0.867432 moved_to_echo=16377 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4068, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 17, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 30, 3978, 0, 0, 0, 4094, 0, 101, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4079, 4096, 4103, 4096, 4008, 4195] avg=4096.0000 max=4195 max_rank=7 min=4008 min_rank=6 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16402 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 17, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 54, 3978, 0, 0, 0, 4042, 0, 101, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4107, 4096, 4102, 4096, 4032, 4143] avg=4096.0000 max=4143 max_rank=7 min=4032 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4021, 17, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 217, 3978, 0, 0, 0, 3942, 0, 101, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4107, 4096, 4039, 4095, 4195, 4043] avg=4096.0000 max=4195 max_rank=6 min=4039 min_rank=4 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 24, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 176, 3985, 0, 0, 0, 3920, 0, 86, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4097, 4103, 4095, 4113, 4095, 4161, 4006] avg=4096.0000 max=4161 max_rank=6 min=4006 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4057, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 17, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 41, 3978, 0, 0, 0, 4094, 0, 101, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4068, 4096, 4102, 4096, 4019, 4195] avg=4096.0000 max=4195 max_rank=7 min=4019 min_rank=6 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16412 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 11, 1, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 4083, 17, 1, 0, 0, 0, 1, 2, 4095, 0, 0, 0, 0, 0, 0, 28, 3978, 0, 0, 0, 4067, 1, 100, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4095, 4107, 4097, 4102, 4098, 4006, 4169] avg=4096.0000 max=4169 max_rank=7 min=4006 min_rank=6 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 32640, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 1024, 31872, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32000, 128, 896, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 1, 4095, 1, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 5, 4096, 0, 0, 0, 0, 31, 0, 0, 4096, 17, 0, 4096, 0, 4094, 0, 0, 4096, 4045, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 33, 0, 0, 4096, 1, 0, 4096, 0, 4092, 0, 0, 4096, 4064, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 54, 0, 4096, 0, 4096, 0, 0, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 4096, 0, 4096, 21, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 4095, 1, 1, 4095, 0, 4095, 18, 1, 4095, 4090, 0, 0, 1, 1, 4, 1, 1, 0, 4095, 4077, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4078, 9, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 1, 0, 16, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4088, 4087, 4097, 4111, 4095, 4095, 4099] avg=4096.0000 max=4111 max_rank=4 min=4087 min_rank=2 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4078, 9, 0, 0, 0, 0, 4087, 17, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 16, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 1, 0, 0, 0, 4093, 2, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4118, 4104, 4094, 4061, 4098, 4096, 4096] avg=4096.0000 max=4118 max_rank=1 min=4061 min_rank=4 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4078, 9, 0, 0, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 16, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 4093, 2, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4120, 4088, 4093, 4080, 4095, 4097, 4099] avg=4096.0000 max=4120 max_rank=1 min=4080 min_rank=4 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 9, 0, 0, 0, 0, 4087, 2, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 16, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4087, 4089, 4097, 4110, 4095, 4095, 4099] avg=4096.0000 max=4110 max_rank=4 min=4087 min_rank=1 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 10, 4, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4093, 4086, 4096, 4110, 4099, 4096, 4096] avg=4096.0000 max=4110 max_rank=4 min=4086 min_rank=2 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4078, 9, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 21, 0, 16, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4088, 4086, 4097, 4112, 4095, 4095, 4099] avg=4096.0000 max=4112 max_rank=4 min=4086 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 9, 0, 0, 0, 0, 4087, 54, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 16, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4087, 4141, 4097, 4058, 4095, 4095, 4099] avg=4096.0000 max=4141 max_rank=2 min=4058 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4077, 9, 0, 0, 0, 0, 4086, 1, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 18, 1, 17, 4090, 0, 0, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 4, 1, 1, 0, 4092, 1, 0, 0, 3, 4077, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4087, 4087, 4097, 4126, 4096, 4099, 4081] avg=4096.0000 max=4126 max_rank=4 min=4081 min_rank=7 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32640, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 4094, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[306, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3789, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[534, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 4096, 0, 4096, 0, 0, 158, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3398, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[924, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 46, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3127, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 4096, 0, 4096, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4072, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[904, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3178, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 23, 0, 4096, 0, 4096, 0, 0, 137, 0, 4096, 0, 0, 0, 0, 0, 4079, 0, 4096, 3934, 0, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[32, 2, 0, 0, 0, 0, 4095, 4, 1, 0, 0, 1, 4094, 0, 4093, 1, 0, 0, 0, 4094, 2, 0, 0, 0, 3, 4092, 0, 4095, 4064, 1, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 3707, 45, 2, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4094, 0, 0, 389, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3756, 4097, 4090, 4097, 4051, 4095, 4099, 4483] avg=4096.0000 max=4483 max_rank=7 min=3756 min_rank=0 imbalance_ratio=1.094482 max_load_reduction=28285 max_load_reduction_pct=0.863190 moved_to_echo=16728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[534, 0, 0, 0, 3707, 45, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 158, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3398, 0, 0, 389, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4288, 4096, 4098, 4096, 4209, 4098, 4096, 3787] avg=4096.0000 max=4288 max_rank=0 min=3787 min_rank=7 imbalance_ratio=1.046875 max_load_reduction=28480 max_load_reduction_pct=0.869141 moved_to_echo=16190 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[924, 0, 0, 0, 3706, 45, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 46, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 3127, 0, 0, 389, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4677, 4096, 4091, 4096, 4097, 4096, 4099, 3516] avg=4096.0000 max=4677 max_rank=0 min=3516 min_rank=7 imbalance_ratio=1.141846 max_load_reduction=28091 max_load_reduction_pct=0.857269 moved_to_echo=15808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 3707, 45, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4072, 0, 0, 389, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3761, 4096, 4099, 4096, 4060, 4096, 4099, 4461] avg=4096.0000 max=4461 max_rank=7 min=3761 min_rank=0 imbalance_ratio=1.089111 max_load_reduction=28307 max_load_reduction_pct=0.863861 moved_to_echo=16716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[306, 0, 0, 0, 3707, 49, 3, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3789, 0, 0, 388, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4092, 4093, 4096, 4049, 4096, 4096, 4177] avg=4096.0000 max=4177 max_rank=7 min=4049 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[904, 0, 0, 0, 3707, 45, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 14, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 3178, 0, 0, 389, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4658, 4096, 4091, 4096, 4065, 4096, 4099, 3567] avg=4096.0000 max=4658 max_rank=0 min=3567 min_rank=7 imbalance_ratio=1.137207 max_load_reduction=28110 max_load_reduction_pct=0.857849 moved_to_echo=15827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 3701, 45, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 137, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4079, 0, 3, 0, 0, 0, 0, 3934, 0, 0, 389, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3773, 4096, 4114, 4096, 4188, 4096, 4082, 4323] avg=4096.0000 max=4323 max_rank=7 min=3773 min_rank=0 imbalance_ratio=1.055420 max_load_reduction=28445 max_load_reduction_pct=0.868073 moved_to_echo=16689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[32, 2, 0, 0, 3705, 45, 2, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 1, 0, 0, 1, 4090, 0, 0, 0, 1, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 2, 0, 0, 0, 4093, 0, 0, 0, 3, 4092, 0, 3, 0, 0, 0, 0, 4064, 1, 0, 389, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3786, 4099, 4092, 4095, 4049, 4095, 4098, 4454] avg=4096.0000 max=4454 max_rank=7 min=3786 min_rank=0 imbalance_ratio=1.087402 max_load_reduction=28314 max_load_reduction_pct=0.864075 moved_to_echo=16696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2816, 128, 0, 0, 29696, 384, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[29696, 128, 0, 3200, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 384, 0, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 4074, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 1, 4023, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 73] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 8, 4085, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 80, 4015, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4075, 0, 78, 4018, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 20] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4079, 0, 17, 3987, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 109] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 0, 4095, 1, 0, 1, 0, 4, 4091, 0, 0, 4095, 1, 1, 1, 1, 4094, 0, 0, 4095, 0, 1, 1, 4095, 4094, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 29, 0, 0, 24, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4067, 0, 0, 24, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4099, 4098, 4095, 4097, 4096, 4091] avg=4096.0000 max=4099 max_rank=2 min=4091 min_rank=7 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16379 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4072, 29, 0, 0, 24, 0, 8, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4067, 0, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4102, 4117, 4096, 4096, 4096, 4069] avg=4096.0000 max=4117 max_rank=3 min=4069 min_rank=7 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16357 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4051, 29, 0, 0, 24, 0, 78, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4067, 0, 0, 20, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4081, 4120, 4096, 4096, 4096, 4087] avg=4096.0000 max=4120 max_rank=3 min=4081 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16375 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 29, 0, 0, 24, 0, 1, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4067, 0, 0, 73, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4100, 4048, 4096, 4096, 4096, 4140] avg=4096.0000 max=4140 max_rank=7 min=4048 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16428 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 29, 0, 0, 24, 0, 80, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4067, 0, 0, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4119, 4096, 4096, 4096, 4073] avg=4096.0000 max=4119 max_rank=3 min=4073 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16361 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 30, 1, 1, 23, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4105, 4119, 4095, 4096, 4096, 4066] avg=4096.0000 max=4119 max_rank=3 min=4066 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16353 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 29, 0, 0, 24, 0, 17, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4067, 0, 0, 109, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4084, 4028, 4096, 4096, 4096, 4176] avg=4096.0000 max=4176 max_rank=7 min=4028 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16464 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 4, 4067, 28, 0, 0, 24, 0, 0, 4095, 0, 0, 0, 0, 1, 1, 1, 1, 4094, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4100, 4119, 4098, 4095, 4097, 4066] avg=4096.0000 max=4119 max_rank=3 min=4066 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16356 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32768, 32769, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32768, 32769, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32768, 32769, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32768, 32769, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32768, 32769, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32768, 32769, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32768, 32769, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32768, 32769, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32640, 0, 0, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32769 actual_valid_recv=33024 observed_loss=-255 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 0, 256, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32767 actual_valid_recv=33536 observed_loss=-769 tokens_per_expert=[0, 128, 128, 128, 32640, 256, 128, 128] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 152, 659, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3972, 4096, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 3409, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 127, 2066, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1904, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 179, 2398, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3948, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1667, 25, 4075, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 334, 202, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 1863, 1911, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 324, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 133, 1127, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4011, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2921, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 138, 3017, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3991, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 182, 2168, 1, 1, 4092, 0, 0, 4, 1, 1, 0, 4094, 0, 0, 4091, 4095, 4095, 2, 4094, 0, 0, 0, 1, 0, 2, 1, 1749, 0, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 152, 659, 2014, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 57, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3972, 0, 59, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3409, 0, 2024, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2825, 4096, 4094, 4096, 4031, 4097, 4096, 5433] avg=4096.0000 max=5433 max_rank=7 min=2825 min_rank=0 imbalance_ratio=1.326416 max_load_reduction=27335 max_load_reduction_pct=0.834198 moved_to_echo=17657 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1863, 1911, 2015, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 57, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 59, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 324, 0, 2024, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5789, 4096, 4095, 4095, 4154, 4096, 4095, 2348] avg=4096.0000 max=5789 max_rank=0 min=2348 min_rank=7 imbalance_ratio=1.413330 max_load_reduction=26979 max_load_reduction_pct=0.823334 moved_to_echo=14693 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 334, 202, 2015, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 57, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 59, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3560, 0, 2024, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2551, 4096, 4094, 4096, 4155, 4096, 4096, 5584] avg=4096.0000 max=5584 max_rank=7 min=2551 min_rank=0 imbalance_ratio=1.363281 max_load_reduction=27184 max_load_reduction_pct=0.829590 moved_to_echo=17931 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 133, 1127, 2015, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 57, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4011, 0, 59, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2921, 0, 2024, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3275, 4096, 4094, 4096, 4070, 4096, 4096, 4945] avg=4096.0000 max=4945 max_rank=7 min=3275 min_rank=0 imbalance_ratio=1.207275 max_load_reduction=27823 max_load_reduction_pct=0.849091 moved_to_echo=17207 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 138, 3017, 2015, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 57, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3991, 0, 59, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1046, 0, 2024, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5170, 4096, 4094, 4096, 4050, 4096, 4096, 3070] avg=4096.0000 max=5170 max_rank=0 min=3070 min_rank=7 imbalance_ratio=1.262207 max_load_reduction=27598 max_load_reduction_pct=0.842224 moved_to_echo=15312 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 127, 2066, 2019, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4042, 60, 1, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 4095, 0, 54, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1904, 0, 2007, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4212, 4098, 4103, 4103, 4149, 4097, 4095, 3911] avg=4096.0000 max=4212 max_rank=0 min=3911 min_rank=7 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 179, 2398, 2005, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 57, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3948, 0, 59, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1667, 25, 2013, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4582, 4096, 4095, 4092, 4007, 4096, 4095, 3705] avg=4096.0000 max=4582 max_rank=0 min=3705 min_rank=7 imbalance_ratio=1.118652 max_load_reduction=28186 max_load_reduction_pct=0.860168 moved_to_echo=15903 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 182, 2168, 2014, 0, 0, 0, 1, 1, 4092, 0, 0, 0, 0, 0, 0, 4, 1, 1, 4036, 57, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4091, 0, 59, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 4095, 0, 0, 0, 1749, 0, 2023, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4364, 4094, 4099, 4094, 4152, 4094, 4099, 3772] avg=4096.0000 max=4364 max_rank=0 min=3772 min_rank=7 imbalance_ratio=1.065430 max_load_reduction=28404 max_load_reduction_pct=0.866821 moved_to_echo=16117 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 3200, 13568, 16128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16640, 128, 16256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32384, 0, 512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32384, 512, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4093, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4092, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4092, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1, 4095, 1, 1, 4095, 0, 4091, 0, 0, 1, 0, 1, 1, 4095, 0, 0, 1, 4095, 0, 0, 0, 0, 10, 1, 0, 4080, 1, 4094, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4100, 4096, 4095, 4098, 4095, 4092] avg=4096.0000 max=4100 max_rank=2 min=4092 min_rank=7 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 1, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4096, 4095, 4096, 4096, 4096, 4099] avg=4096.0000 max=4099 max_rank=7 min=4094 min_rank=0 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4095, 4096, 4096, 4096, 4095, 4098] avg=4096.0000 max=4098 max_rank=7 min=4095 min_rank=2 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4095, 4097, 4096, 4097, 4095, 4094, 4096] avg=4096.0000 max=4098 max_rank=0 min=4094 min_rank=6 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4093, 4099] avg=4096.0000 max=4099 max_rank=7 min=4093 min_rank=6 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4097, 4096, 4096, 4096, 4095, 4096] avg=4096.0000 max=4097 max_rank=2 min=4095 min_rank=6 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4094, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 4092, 1, 0, 0, 4080, 1, 2, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4091, 4097, 4096, 4095, 4104, 4092] avg=4096.0000 max=4104 max_rank=6 min=4091 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True + [2026-08-30 04:56:46] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 669.3 | throughput per GPU (TFLOP/s/GPU): 498.1 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.039608E+01 | loss scale: 1.0 | grad norm: 16.819 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3, 3958, 1865, 0, 960, 2, 568, 0, 2147, 602, 11, 2311, 1973, 770, 5, 2544, 2234, 1104, 447, 228, 40, 2094, 852, 2, 0, 335, 191, 1513, 2412, 32, 3565, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[6, 4016, 1971, 0, 783, 0, 815, 3, 1473, 891, 13, 2629, 2179, 948, 2, 2072, 1500, 1974, 317, 252, 39, 2082, 858, 2, 5, 248, 195, 1695, 2148, 23, 3629, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 3915, 1588, 3, 672, 0, 489, 0, 1857, 950, 7, 2555, 1712, 928, 0, 2924, 2074, 1769, 192, 304, 67, 2996, 804, 0, 0, 212, 207, 1191, 1813, 19, 3512, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 3875, 1652, 1, 993, 4, 545, 7, 2085, 335, 3, 2263, 2044, 962, 1, 2666, 2001, 1757, 404, 507, 19, 2568, 929, 9, 10, 572, 300, 1167, 1500, 27, 3562, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[9, 4009, 1313, 1, 778, 0, 703, 0, 2050, 812, 16, 2677, 1742, 662, 5, 2140, 2018, 1351, 172, 244, 55, 2644, 708, 4, 1, 355, 298, 1632, 2520, 24, 3824, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[5, 4044, 1564, 0, 889, 0, 532, 2, 2398, 653, 6, 2739, 2604, 461, 8, 2509, 1850, 844, 211, 354, 66, 3209, 847, 0, 1, 379, 243, 1024, 1408, 18, 3888, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 4054, 1524, 1, 1005, 0, 641, 0, 1412, 802, 5, 2454, 1493, 1289, 0, 2891, 2366, 1165, 84, 225, 15, 3160, 896, 3, 0, 248, 179, 1399, 1601, 15, 3841, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 4043, 1383, 0, 850, 0, 724, 0, 2106, 900, 6, 2216, 2484, 1006, 0, 2824, 1560, 1084, 207, 354, 46, 3001, 728, 5, 0, 194, 217, 1135, 1819, 19, 3857, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 2467, 1865, 0, 0, 0, 0, 0, 960, 2, 568, 0, 1480, 1050, 0, 0, 2147, 602, 11, 1308, 0, 0, 0, 0, 1973, 770, 5, 1174, 0, 0, 0, 0, 2234, 1104, 447, 228, 188, 0, 0, 0, 40, 2094, 852, 2, 468, 0, 0, 0, 0, 335, 191, 1513, 1370, 441, 347, 0, 2412, 32, 2085, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4335, 4060, 4068, 3922, 4201, 3456, 4197, 4529] avg=4096.0000 max=4529 max_rank=7 min=3456 min_rank=5 imbalance_ratio=1.105713 max_load_reduction=28239 max_load_reduction_pct=0.861786 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[6, 2502, 1971, 0, 0, 0, 0, 0, 783, 0, 815, 3, 1507, 1066, 0, 0, 1473, 891, 13, 1490, 0, 0, 0, 0, 2179, 948, 2, 956, 0, 0, 0, 0, 1500, 1974, 317, 252, 213, 0, 0, 0, 39, 2082, 858, 2, 532, 0, 0, 0, 5, 248, 195, 1695, 1116, 448, 394, 0, 2148, 23, 2122, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4479, 4174, 3867, 4085, 4256, 3513, 4101, 4293] avg=4096.0000 max=4479 max_rank=0 min=3513 min_rank=5 imbalance_ratio=1.093506 max_load_reduction=28289 max_load_reduction_pct=0.863312 moved_to_echo=16163 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[9, 2498, 1313, 1, 0, 0, 0, 0, 778, 0, 703, 0, 1588, 1064, 0, 0, 2050, 812, 16, 1516, 0, 0, 0, 0, 1742, 662, 5, 988, 0, 0, 0, 0, 2018, 1351, 172, 244, 217, 0, 0, 0, 55, 2644, 708, 4, 542, 0, 0, 0, 1, 355, 298, 1632, 1152, 447, 402, 0, 2520, 24, 2236, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3821, 4133, 4394, 3397, 4002, 3953, 4287, 4781] avg=4096.0000 max=4781 max_rank=7 min=3397 min_rank=3 imbalance_ratio=1.167236 max_load_reduction=27987 max_load_reduction_pct=0.854095 moved_to_echo=17023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 2520, 1564, 0, 0, 0, 0, 0, 889, 0, 532, 2, 1615, 1073, 0, 0, 2398, 653, 6, 1552, 0, 0, 0, 0, 2604, 461, 8, 1158, 0, 0, 0, 0, 1850, 844, 211, 354, 222, 0, 0, 0, 66, 3209, 847, 0, 554, 0, 0, 0, 1, 379, 243, 1024, 1351, 451, 411, 0, 1408, 18, 2273, 12, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4111, 4609, 4231, 3481, 4676, 3860, 3711] avg=4096.0000 max=4676 max_rank=5 min=3481 min_rank=4 imbalance_ratio=1.141602 max_load_reduction=28092 max_load_reduction_pct=0.857300 moved_to_echo=15728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2432, 1588, 3, 0, 0, 0, 0, 672, 0, 489, 0, 1463, 1044, 0, 0, 1857, 950, 7, 1434, 0, 0, 0, 0, 1712, 928, 0, 1347, 0, 0, 0, 0, 2074, 1769, 192, 304, 212, 0, 0, 0, 67, 2996, 804, 0, 521, 0, 0, 0, 0, 212, 207, 1191, 1577, 439, 388, 0, 1813, 19, 2049, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 3668, 4248, 3987, 4551, 4388, 4014, 3889] avg=4096.0000 max=4551 max_rank=4 min=3668 min_rank=1 imbalance_ratio=1.111084 max_load_reduction=28217 max_load_reduction_pct=0.861115 moved_to_echo=16842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2415, 1652, 1, 0, 0, 0, 0, 993, 4, 545, 7, 1479, 1028, 0, 0, 2085, 335, 3, 1282, 0, 0, 0, 0, 2044, 962, 1, 1230, 0, 0, 0, 0, 2001, 1757, 404, 507, 184, 0, 0, 0, 19, 2568, 929, 9, 458, 0, 0, 0, 10, 572, 300, 1167, 1436, 432, 339, 0, 1500, 27, 2083, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4056, 3705, 4237, 4853, 3983, 4256, 3610] avg=4096.0000 max=4853 max_rank=4 min=3610 min_rank=7 imbalance_ratio=1.184814 max_load_reduction=27915 max_load_reduction_pct=0.851898 moved_to_echo=16702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2526, 1524, 1, 0, 0, 0, 0, 1005, 0, 641, 0, 1595, 1076, 0, 0, 1412, 802, 5, 1390, 0, 0, 0, 0, 1493, 1289, 0, 1334, 0, 0, 0, 0, 2366, 1165, 84, 225, 199, 0, 0, 0, 15, 3160, 896, 3, 497, 0, 0, 0, 0, 248, 179, 1399, 1557, 452, 368, 0, 1601, 15, 2246, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4317, 3609, 4116, 4039, 4571, 4203, 3862] avg=4096.0000 max=4571 max_rank=5 min=3609 min_rank=2 imbalance_ratio=1.115967 max_load_reduction=28197 max_load_reduction_pct=0.860504 moved_to_echo=16675 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2519, 1383, 0, 0, 0, 0, 0, 850, 0, 724, 0, 1602, 1073, 0, 0, 2106, 900, 6, 1256, 0, 0, 0, 0, 2484, 1006, 0, 1303, 0, 0, 0, 0, 1560, 1084, 207, 354, 180, 0, 0, 0, 46, 3001, 728, 5, 448, 0, 0, 0, 0, 194, 217, 1135, 1521, 451, 332, 0, 1819, 19, 2255, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3902, 4249, 4268, 4793, 3385, 4228, 3850, 4093] avg=4096.0000 max=4793 max_rank=3 min=3385 min_rank=4 imbalance_ratio=1.170166 max_load_reduction=27975 max_load_reduction_pct=0.853729 moved_to_echo=15556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[7040, 128, 5120, 128, 12416, 8576, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 2560, 1920, 10880, 11136, 3584, 3072, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15616, 6016, 128, 11264, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15616, 11136, 2048, 2560, 1664, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16256, 7040, 128, 9600, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 19968, 12928, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15232, 256, 17408, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 21760, 6656, 128, 4096, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[52, 0, 4084, 3894, 1, 0, 8, 4086, 131, 3396, 0, 0, 0, 7, 1298, 0, 14, 0, 4073, 0, 25, 0, 0, 4095, 0, 0, 0, 17, 16, 0, 3486, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[25, 0, 4085, 4076, 0, 0, 0, 4096, 1087, 2780, 0, 0, 0, 0, 1975, 1, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 20, 24, 0, 2311, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[10, 0, 4095, 3672, 0, 0, 0, 4088, 471, 3534, 0, 0, 0, 0, 985, 5, 5, 0, 4096, 0, 1, 0, 0, 4096, 0, 1, 0, 4, 42, 0, 3582, 4081] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[12, 0, 4089, 4053, 1, 0, 0, 4094, 367, 2991, 0, 0, 0, 0, 2736, 0, 0, 0, 4091, 0, 0, 0, 1, 4096, 0, 0, 0, 37, 2, 0, 2103, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[19, 1, 4057, 4063, 0, 0, 1, 4095, 266, 2362, 0, 0, 0, 0, 2317, 1, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4, 46, 77, 0, 3174, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[28, 0, 4095, 3755, 0, 0, 2, 4083, 159, 2506, 0, 0, 0, 5, 1675, 0, 28, 0, 4076, 2, 10, 1, 2, 4091, 0, 3, 0, 121, 224, 7, 3804, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[30, 2, 4089, 3800, 0, 3, 0, 4074, 61, 2361, 0, 0, 6, 0, 2549, 0, 0, 1, 4093, 3, 6, 0, 1, 4090, 0, 22, 3, 30, 6, 0, 3442, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[7, 4, 4089, 3918, 12, 0, 0, 4096, 396, 3553, 0, 0, 3, 1, 1783, 2, 0, 0, 4085, 2, 0, 0, 8, 4093, 0, 11, 4, 16, 1, 0, 2601, 4083] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 169, 3672, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 471, 3534, 0, 0, 791, 0, 0, 0, 0, 0, 985, 5, 2172, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 1, 0, 4, 3926, 136, 0, 0, 42, 0, 3582, 982, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3851, 4091, 4796, 3162, 4101, 4094, 4067, 4606] avg=4096.0000 max=4796 max_rank=2 min=3162 min_rank=3 imbalance_ratio=1.170898 max_load_reduction=27972 max_load_reduction_pct=0.853638 moved_to_echo=16868 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 169, 3755, 0, 0, 0, 0, 0, 0, 2, 4083, 3, 0, 0, 0, 159, 2506, 0, 0, 792, 0, 0, 0, 0, 5, 1675, 0, 2177, 0, 0, 0, 28, 0, 4076, 2, 0, 0, 0, 0, 10, 1, 2, 4088, 0, 0, 0, 0, 0, 3, 0, 121, 3926, 136, 0, 0, 224, 7, 3804, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3952, 4088, 3457, 3857, 4106, 4101, 4186, 5021] avg=4096.0000 max=5021 max_rank=7 min=3457 min_rank=2 imbalance_ratio=1.225830 max_load_reduction=27747 max_load_reduction_pct=0.846771 moved_to_echo=17414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[30, 2, 165, 3800, 0, 0, 0, 0, 0, 3, 0, 4074, 7, 0, 0, 0, 61, 2361, 0, 0, 798, 5, 0, 0, 6, 0, 2549, 0, 2184, 0, 0, 0, 0, 1, 4093, 3, 6, 0, 0, 0, 6, 0, 1, 4072, 0, 0, 0, 0, 0, 22, 3, 30, 3924, 141, 0, 0, 6, 0, 3442, 973, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3997, 4084, 3225, 4739, 4103, 4079, 4120, 4421] avg=4096.0000 max=4739 max_rank=3 min=3225 min_rank=2 imbalance_ratio=1.156982 max_load_reduction=28029 max_load_reduction_pct=0.855377 moved_to_echo=16723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[7, 4, 169, 3918, 0, 0, 0, 0, 12, 0, 0, 4096, 3, 0, 0, 0, 396, 3553, 0, 0, 791, 0, 0, 0, 3, 1, 1783, 2, 2173, 0, 0, 0, 0, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 8, 4090, 0, 0, 0, 0, 0, 11, 4, 16, 3920, 136, 0, 0, 1, 0, 2601, 983, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4111, 4740, 3962, 4087, 4098, 4087, 3585] avg=4096.0000 max=4740 max_rank=2 min=3585 min_rank=7 imbalance_ratio=1.157227 max_load_reduction=28028 max_load_reduction_pct=0.855347 moved_to_echo=15857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[52, 0, 169, 3894, 0, 0, 0, 0, 1, 0, 8, 4086, 3, 0, 0, 0, 131, 3396, 0, 0, 791, 0, 0, 0, 0, 7, 1298, 0, 2174, 0, 0, 0, 14, 0, 4073, 0, 0, 0, 0, 0, 25, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 17, 3915, 136, 0, 0, 16, 0, 3486, 984, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4098, 4318, 3479, 4087, 4117, 4068, 4486] avg=4096.0000 max=4486 max_rank=7 min=3479 min_rank=3 imbalance_ratio=1.095215 max_load_reduction=28282 max_load_reduction_pct=0.863098 moved_to_echo=16758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[19, 1, 168, 4063, 0, 0, 0, 0, 0, 0, 1, 4095, 3, 0, 0, 0, 266, 2362, 0, 0, 793, 0, 0, 0, 0, 0, 2317, 1, 2179, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4, 46, 3889, 136, 0, 0, 77, 0, 3174, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4251, 4099, 3421, 4497, 4096, 4092, 4075, 4237] avg=4096.0000 max=4497 max_rank=3 min=3421 min_rank=2 imbalance_ratio=1.097900 max_load_reduction=28271 max_load_reduction_pct=0.862762 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 169, 4076, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 1087, 2780, 0, 0, 793, 0, 0, 0, 0, 0, 1975, 1, 2180, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 20, 3916, 136, 0, 0, 24, 0, 2311, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4270, 4099, 4660, 4156, 4097, 4093, 4072, 3321] avg=4096.0000 max=4660 max_rank=2 min=3321 min_rank=7 imbalance_ratio=1.137695 max_load_reduction=28108 max_load_reduction_pct=0.857788 moved_to_echo=15583 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 169, 4053, 0, 0, 0, 0, 1, 0, 0, 4094, 3, 0, 0, 0, 367, 2991, 0, 0, 793, 0, 0, 0, 0, 0, 2736, 0, 2180, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 37, 3920, 136, 0, 0, 2, 0, 2103, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4234, 4098, 4151, 4916, 4091, 4094, 4093, 3091] avg=4096.0000 max=4916 max_rank=3 min=3091 min_rank=7 imbalance_ratio=1.200195 max_load_reduction=27852 max_load_reduction_pct=0.849976 moved_to_echo=15369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 15360, 128, 17536, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 128, 24576, 7936, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 384, 31360, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 1408, 31232, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2944, 23552, 0, 0, 6400, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4082, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 44, 4096, 0, 0, 0, 549, 0, 0, 0, 0, 4091, 3609, 0, 0, 9, 0, 3995, 0, 4096, 9, 4088, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4066, 2, 0, 1, 12, 0, 0, 4095, 1, 0, 2, 4093, 0, 0, 0, 192, 0, 0, 0, 0, 4084, 4058, 0, 0, 2, 2, 3969, 2, 4095, 0, 4091, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 17, 1, 2, 4095, 0, 0, 64, 4085, 3, 0, 0, 24, 3, 0, 2, 4, 4078, 4045, 0, 0, 2, 0, 4055, 5, 4093, 1, 4095, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4082, 9, 0, 0, 0, 0, 1, 4096, 0, 4, 9, 4096, 0, 0, 0, 600, 0, 0, 0, 0, 4047, 3669, 0, 0, 0, 0, 3963, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4078, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 4096, 0, 0, 0, 864, 0, 0, 0, 0, 4094, 3382, 0, 0, 1, 0, 3959, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4094, 8, 3, 1, 0, 1, 15, 4095, 0, 0, 26, 4095, 0, 0, 0, 247, 0, 1, 1, 0, 4078, 3980, 2, 0, 9, 0, 3921, 0, 4095, 0, 4092, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4091, 1, 0, 0, 19, 0, 0, 4091, 0, 0, 32, 4083, 0, 0, 2, 320, 1, 4, 3, 2, 3992, 3910, 4, 0, 6, 1, 4014, 0, 4092, 0, 4096, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4064, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 45, 4096, 0, 0, 1, 944, 0, 0, 0, 0, 4094, 3360, 0, 0, 4, 0, 3872, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4082, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4, 44, 4071, 0, 0, 0, 0, 0, 0, 0, 549, 3645, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 346, 3609, 0, 0, 0, 0, 0, 0, 9, 0, 3995, 0, 100, 22, 0, 0, 0, 9, 4088, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4089, 4119, 4196, 4094, 3955, 4126, 4097] avg=4096.0000 max=4196 max_rank=3 min=3955 min_rank=5 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4066, 2, 0, 1, 7, 3, 0, 0, 12, 0, 0, 4088, 0, 0, 0, 0, 1, 0, 2, 4068, 0, 0, 0, 0, 0, 0, 0, 192, 3638, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 346, 4058, 0, 0, 0, 0, 0, 0, 2, 2, 3969, 2, 100, 22, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4100, 4071, 3831, 4094, 4404, 4097, 4092] avg=4096.0000 max=4404 max_rank=5 min=3831 min_rank=3 imbalance_ratio=1.075195 max_load_reduction=28364 max_load_reduction_pct=0.865601 moved_to_echo=16687 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 7, 3, 0, 0, 17, 1, 2, 4088, 0, 0, 0, 0, 0, 0, 64, 4060, 0, 0, 0, 0, 3, 0, 0, 24, 3633, 1, 0, 0, 3, 0, 2, 4, 4092, 0, 0, 0, 345, 4045, 0, 0, 0, 0, 0, 0, 2, 0, 4055, 5, 100, 22, 0, 0, 0, 1, 4095, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4108, 4124, 3661, 4101, 4390, 4184, 4097] avg=4096.0000 max=4390 max_rank=5 min=3661 min_rank=3 imbalance_ratio=1.071777 max_load_reduction=28378 max_load_reduction_pct=0.866028 moved_to_echo=16772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4082, 9, 0, 0, 7, 3, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 4, 9, 4071, 0, 0, 0, 0, 0, 0, 0, 600, 3605, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 343, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 99, 22, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4090, 4084, 4207, 4094, 4012, 4084, 4096] avg=4096.0000 max=4207 max_rank=3 min=4012 min_rank=5 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16286 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4078, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 6, 4071, 0, 0, 0, 0, 0, 0, 0, 864, 3647, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 347, 3382, 0, 0, 0, 0, 0, 0, 1, 0, 3959, 0, 100, 22, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4089, 4077, 4512, 4095, 3729, 4082, 4096] avg=4096.0000 max=4512 max_rank=3 min=3729 min_rank=5 imbalance_ratio=1.101562 max_load_reduction=28256 max_load_reduction_pct=0.862305 moved_to_echo=16002 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4091, 1, 0, 0, 11, 10, 2, 0, 19, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 32, 4049, 0, 0, 0, 0, 0, 0, 2, 320, 3562, 2, 0, 0, 1, 4, 3, 2, 4090, 0, 0, 0, 330, 3910, 4, 0, 0, 0, 0, 0, 6, 1, 4014, 0, 100, 24, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4099, 4081, 3886, 4100, 4244, 4145, 4098] avg=4096.0000 max=4244 max_rank=5 min=3886 min_rank=3 imbalance_ratio=1.036133 max_load_reduction=28524 max_load_reduction_pct=0.870483 moved_to_echo=16587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4094, 8, 3, 1, 7, 3, 0, 0, 0, 1, 15, 4088, 0, 0, 0, 0, 0, 0, 26, 4070, 0, 0, 0, 0, 0, 0, 0, 247, 3633, 2, 0, 0, 0, 1, 1, 0, 4093, 0, 0, 0, 345, 3980, 2, 0, 0, 0, 0, 0, 9, 0, 3921, 0, 100, 22, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4104, 4096, 3882, 4095, 4327, 4052, 4096] avg=4096.0000 max=4327 max_rank=5 min=3882 min_rank=3 imbalance_ratio=1.056396 max_load_reduction=28441 max_load_reduction_pct=0.867950 moved_to_echo=16570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4064, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 45, 4071, 0, 0, 0, 0, 0, 0, 1, 944, 3647, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 347, 3360, 0, 0, 0, 0, 0, 0, 4, 0, 3872, 0, 100, 22, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4089, 4116, 4593, 4095, 3707, 3998, 4096] avg=4096.0000 max=4593 max_rank=3 min=3707 min_rank=5 imbalance_ratio=1.121338 max_load_reduction=28175 max_load_reduction_pct=0.859833 moved_to_echo=15896 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 3840, 29056, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 256, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 31872, 128, 896, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2816, 30080, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 1, 4094, 4095, 0, 0, 0, 4095, 0, 1, 0, 2, 0, 4095, 1771, 0, 4095, 0, 1, 0, 0, 1959, 1, 1, 1, 4093, 1, 367] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 2, 4090, 0, 0, 0, 4096, 4096, 5, 0, 0, 4092, 0, 0, 1, 0, 0, 4095, 1362, 0, 4096, 0, 0, 1, 6, 2269, 0, 0, 0, 4091, 0, 466] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4085, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1592, 0, 4096, 0, 0, 0, 0, 2462, 0, 0, 0, 4096, 0, 53] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 699, 0, 4096, 0, 0, 0, 0, 3269, 0, 0, 0, 4096, 0, 128] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2, 2, 4093, 1, 1, 1, 4067, 4095, 0, 1, 1, 4094, 0, 0, 1, 0, 1, 4092, 2119, 4, 4093, 0, 0, 1, 1, 1916, 0, 0, 3, 4095, 0, 84] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 7, 4092, 0, 2, 4, 4086, 4090, 0, 0, 4, 4092, 3, 3, 6, 0, 0, 4092, 2867, 14, 4090, 4, 0, 0, 0, 1103, 0, 5, 0, 4089, 0, 115] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2617, 0, 4096, 0, 0, 0, 0, 1191, 0, 0, 0, 4096, 0, 288] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4090, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2255, 0, 4096, 0, 0, 0, 0, 1667, 0, 0, 0, 4096, 0, 180] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 2, 4088, 5, 0, 0, 0, 2184, 1771, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 1959, 1, 1, 1906, 208, 0, 0, 1, 3885, 1, 367, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4102, 4095, 4096, 3955, 4096, 4075, 4254] avg=4096.0000 max=4254 max_rank=7 min=3955 min_rank=4 imbalance_ratio=1.038574 max_load_reduction=28514 max_load_reduction_pct=0.870178 moved_to_echo=16380 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 2185, 699, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3269, 0, 0, 1906, 208, 0, 0, 0, 3888, 0, 128, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4103, 4096, 4094, 2884, 4096, 5383, 4016] avg=4096.0000 max=5383 max_rank=6 min=2884 min_rank=4 imbalance_ratio=1.314209 max_load_reduction=27385 max_load_reduction_pct=0.835724 moved_to_echo=16379 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 5, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 5, 0, 0, 0, 2184, 1362, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 6, 2269, 0, 0, 1906, 208, 0, 0, 0, 3883, 0, 466, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4103, 4097, 4095, 3546, 4097, 4389, 4349] avg=4096.0000 max=4389 max_rank=6 min=3546 min_rank=4 imbalance_ratio=1.071533 max_load_reduction=28379 max_load_reduction_pct=0.866058 moved_to_echo=16381 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 2185, 1592, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2462, 0, 0, 1906, 208, 0, 0, 0, 3888, 0, 53, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4092, 4096, 4094, 3777, 4096, 4576, 3941] avg=4096.0000 max=4576 max_rank=6 min=3777 min_rank=4 imbalance_ratio=1.117188 max_load_reduction=28192 max_load_reduction_pct=0.860352 moved_to_echo=16390 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 4092, 0, 0, 0, 0, 0, 2, 4, 4086, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 0, 0, 0, 0, 3, 3, 6, 0, 4090, 6, 0, 0, 0, 2176, 2867, 14, 0, 0, 0, 0, 4090, 4, 0, 0, 3, 0, 0, 0, 0, 1103, 0, 5, 1910, 211, 0, 0, 0, 3875, 0, 115, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4092, 4096, 4108, 5057, 4097, 3229, 3990] avg=4096.0000 max=5057 max_rank=4 min=3229 min_rank=6 imbalance_ratio=1.234619 max_load_reduction=27711 max_load_reduction_pct=0.845673 moved_to_echo=16373 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 4093, 1, 0, 0, 0, 0, 1, 1, 4067, 7, 0, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 4, 0, 0, 1, 2183, 2119, 4, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 1, 1916, 0, 0, 1905, 208, 0, 0, 3, 3887, 0, 84, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4076, 4096, 4093, 4307, 4094, 4030, 3974] avg=4096.0000 max=4307 max_rank=4 min=3974 min_rank=7 imbalance_ratio=1.051514 max_load_reduction=28461 max_load_reduction_pct=0.868561 moved_to_echo=16405 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 2185, 2255, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1667, 0, 0, 1906, 208, 0, 0, 0, 3888, 0, 180, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4096, 4094, 4440, 4096, 3781, 4068] avg=4096.0000 max=4440 max_rank=4 min=3781 min_rank=6 imbalance_ratio=1.083984 max_load_reduction=28328 max_load_reduction_pct=0.864502 moved_to_echo=16385 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 2185, 2617, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1191, 0, 0, 1906, 208, 0, 0, 0, 3888, 0, 288, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4103, 4096, 4094, 4802, 4096, 3305, 4176] avg=4096.0000 max=4802 max_rank=4 min=3305 min_rank=6 imbalance_ratio=1.172363 max_load_reduction=27966 max_load_reduction_pct=0.853455 moved_to_echo=16379 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 17536, 15360, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 15872, 128, 128, 15360, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31104, 128, 1792, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 4029, 4095, 6, 0, 0, 3488, 0, 0, 0, 0, 1, 1, 0, 0, 2, 0, 4095, 1, 0, 4091, 0, 2220, 4095, 4094, 0, 1, 0, 0, 2547, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1, 2, 4003, 4092, 0, 4, 4, 2492, 11, 0, 0, 2, 1, 2, 5, 0, 0, 0, 4092, 1, 4, 4089, 0, 1971, 4094, 4091, 0, 2, 0, 8, 3797, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4073, 4096, 0, 0, 0, 2183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4, 3971, 1, 2332, 4096, 4096, 0, 0, 0, 0, 3817, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4092, 4096, 0, 0, 2, 3489, 27, 0, 2, 0, 0, 0, 0, 9, 0, 12, 4094, 0, 0, 4076, 4, 2991, 4094, 4095, 1, 0, 1, 0, 1683, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3963, 4096, 0, 0, 0, 2420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 2380, 4096, 4096, 0, 0, 0, 15, 3522, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[5, 3, 4095, 4085, 18, 0, 19, 3156, 4, 0, 3, 6, 4, 8, 0, 0, 3, 0, 4091, 3, 0, 4093, 0, 1239, 4093, 4090, 1, 3, 0, 0, 3746, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4088, 4096, 3, 0, 0, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 0, 2684, 4096, 4096, 0, 0, 0, 0, 2927, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4096, 0, 0, 0, 2126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 2142, 4096, 4096, 0, 0, 0, 2, 3928, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4029, 40, 0, 0, 0, 0, 6, 0, 0, 3488, 1351, 0, 0, 0, 0, 0, 0, 0, 4055, 32, 2, 0, 1, 1, 0, 0, 4092, 0, 0, 0, 2, 0, 4094, 1, 0, 0, 0, 0, 0, 1860, 0, 2220, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 2547, 0, 848, 1, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4845, 4089, 4094, 4097, 4080, 4096, 3396] avg=4096.0000 max=4845 max_rank=1 min=3396 min_rank=7 imbalance_ratio=1.182861 max_load_reduction=27923 max_load_reduction_pct=0.852142 moved_to_echo=15669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4073, 40, 0, 0, 0, 0, 0, 0, 0, 2183, 1311, 0, 0, 0, 0, 0, 0, 0, 4056, 31, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 0, 4, 1805, 1, 2332, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 0, 824, 1, 0, 0] rerouted_tokens_per_ep_rank=[4113, 3494, 4089, 4094, 4098, 4142, 4096, 4642] avg=4096.0000 max=4642 max_rank=7 min=3494 min_rank=1 imbalance_ratio=1.133301 max_load_reduction=28126 max_load_reduction_pct=0.858337 moved_to_echo=16978 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4092, 40, 0, 0, 0, 0, 0, 0, 2, 3489, 1346, 0, 0, 0, 27, 0, 2, 0, 4056, 32, 2, 0, 0, 0, 0, 9, 4091, 0, 0, 0, 0, 12, 4093, 0, 0, 0, 0, 0, 0, 1853, 4, 2991, 0, 0, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 1683, 0, 845, 1, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4837, 4119, 4100, 4105, 4848, 4097, 2530] avg=4096.0000 max=4848 max_rank=5 min=2530 min_rank=7 imbalance_ratio=1.183594 max_load_reduction=27920 max_load_reduction_pct=0.852051 moved_to_echo=15580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3963, 40, 0, 0, 0, 0, 0, 0, 0, 2420, 1349, 0, 0, 0, 0, 0, 0, 0, 4056, 32, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1856, 0, 2380, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 15, 3522, 0, 847, 1, 0, 0] rerouted_tokens_per_ep_rank=[4003, 3769, 4090, 4093, 4095, 4236, 4097, 4385] avg=4096.0000 max=4385 max_rank=7 min=3769 min_rank=1 imbalance_ratio=1.070557 max_load_reduction=28383 max_load_reduction_pct=0.866180 moved_to_echo=16813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4088, 40, 0, 0, 0, 0, 3, 0, 0, 2593, 1350, 0, 0, 0, 0, 0, 0, 0, 4056, 32, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1859, 0, 2684, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2927, 0, 848, 1, 0, 0] rerouted_tokens_per_ep_rank=[4128, 3946, 4090, 4093, 4095, 4543, 4097, 3776] avg=4096.0000 max=4543 max_rank=5 min=3776 min_rank=7 imbalance_ratio=1.109131 max_load_reduction=28225 max_load_reduction_pct=0.861359 moved_to_echo=16511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 4003, 40, 0, 0, 0, 0, 0, 4, 4, 2492, 1350, 0, 0, 0, 11, 0, 0, 2, 4052, 32, 2, 0, 1, 2, 5, 0, 4091, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 4, 1859, 0, 1971, 0, 0, 0, 0, 1, 4091, 0, 2, 0, 0, 0, 0, 0, 8, 3797, 0, 848, 1, 0, 0] rerouted_tokens_per_ep_rank=[4046, 3850, 4099, 4099, 4092, 3834, 4094, 4654] avg=4096.0000 max=4654 max_rank=7 min=3834 min_rank=5 imbalance_ratio=1.136230 max_load_reduction=28114 max_load_reduction_pct=0.857971 moved_to_echo=16674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 3, 4095, 38, 0, 0, 0, 0, 18, 0, 19, 3156, 1357, 0, 0, 0, 4, 0, 3, 6, 4047, 39, 3, 0, 4, 8, 0, 0, 4090, 0, 0, 0, 3, 0, 4085, 3, 0, 0, 0, 0, 0, 1844, 0, 1239, 0, 0, 0, 0, 0, 4090, 1, 3, 0, 0, 0, 0, 0, 0, 3746, 0, 853, 6, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4550, 4102, 4102, 4091, 3083, 4094, 4605] avg=4096.0000 max=4605 max_rank=7 min=3083 min_rank=5 imbalance_ratio=1.124268 max_load_reduction=28163 max_load_reduction_pct=0.859467 moved_to_echo=15873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 40, 0, 0, 0, 0, 0, 0, 0, 2126, 1351, 0, 0, 0, 0, 0, 0, 0, 4056, 32, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1860, 0, 2142, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 3928, 0, 849, 1, 0, 0] rerouted_tokens_per_ep_rank=[4134, 3477, 4090, 4093, 4095, 4002, 4097, 4780] avg=4096.0000 max=4780 max_rank=7 min=3477 min_rank=1 imbalance_ratio=1.166992 max_load_reduction=27988 max_load_reduction_pct=0.854126 moved_to_echo=16974 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 14848, 128, 18048, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 22016, 10880, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 25984, 0, 6784, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32512, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 32512, 384, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4091, 112, 3148, 4092, 0, 0, 0, 4, 5, 9, 4079, 0, 0, 885, 0, 0, 0, 0, 5, 3, 0, 4027, 4, 31, 4085, 0, 1, 3, 4090, 6, 4083, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4095, 398, 1781, 4095, 0, 0, 0, 1, 1, 1, 4095, 0, 0, 1987, 0, 0, 0, 0, 3, 1, 0, 3991, 0, 42, 4083, 1, 2, 0, 4095, 1, 4094, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 327, 2632, 4089, 0, 0, 1, 0, 0, 0, 4096, 3, 0, 993, 39, 0, 0, 0, 0, 0, 0, 4065, 0, 144, 4091, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4092, 161, 2193, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1810, 4, 0, 0, 0, 0, 0, 0, 4043, 0, 7, 4074, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4089, 223, 2388, 4094, 0, 1, 0, 1, 2, 7, 4085, 0, 0, 1744, 6, 1, 0, 2, 0, 0, 0, 3773, 1, 94, 4072, 3, 0, 2, 4093, 2, 4082, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 424, 2414, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1253, 3, 0, 0, 0, 0, 0, 0, 4084, 0, 14, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4094, 116, 1452, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2556, 2, 0, 0, 0, 0, 0, 0, 4077, 0, 6, 4081, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 30, 660, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3407, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1792, 327, 2632, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 993, 39, 0, 2261, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 4065, 0, 144, 33, 0, 0, 0, 4091, 0, 0, 0, 9, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4751, 4090, 4099, 3293, 4094, 4242, 4100, 4099] avg=4096.0000 max=4751 max_rank=0 min=3293 min_rank=3 imbalance_ratio=1.159912 max_load_reduction=28017 max_load_reduction_pct=0.855011 moved_to_echo=16535 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1791, 161, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1810, 4, 0, 2258, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 4043, 0, 7, 33, 0, 0, 0, 4074, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4099, 4096, 4072, 4094, 4083, 4083, 4096] avg=4096.0000 max=4145 max_rank=0 min=4072 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16356 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1792, 398, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 1987, 0, 0, 2260, 0, 0, 0, 0, 0, 3, 1, 4093, 1, 0, 0, 0, 3991, 0, 42, 33, 0, 0, 0, 4083, 1, 2, 0, 9, 0, 0, 0, 2, 1, 4094, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3971, 4096, 4097, 4247, 4098, 4066, 4095, 4098] avg=4096.0000 max=4247 max_rank=3 min=3971 min_rank=0 imbalance_ratio=1.036865 max_load_reduction=28521 max_load_reduction_pct=0.870392 moved_to_echo=16357 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1769, 112, 3148, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 3, 0, 0, 5, 9, 4079, 0, 1, 0, 0, 0, 0, 885, 0, 0, 2261, 0, 0, 0, 0, 0, 5, 3, 4087, 6, 0, 0, 0, 4027, 4, 31, 39, 0, 0, 0, 4085, 0, 1, 3, 15, 0, 0, 0, 0, 6, 4083, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5029, 4099, 4094, 3146, 4101, 4101, 4104, 4094] avg=4096.0000 max=5029 max_rank=0 min=3146 min_rank=3 imbalance_ratio=1.227783 max_load_reduction=27739 max_load_reduction_pct=0.846527 moved_to_echo=16400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1792, 116, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2556, 2, 0, 2259, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4077, 0, 6, 33, 0, 0, 0, 4081, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3360, 4096, 4096, 4817, 4097, 4116, 4090, 4096] avg=4096.0000 max=4817 max_rank=3 min=3360 min_rank=0 imbalance_ratio=1.176025 max_load_reduction=27951 max_load_reduction_pct=0.852997 moved_to_echo=16399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1792, 424, 2414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1253, 3, 0, 2261, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 4084, 0, 14, 33, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4630, 4096, 4096, 3517, 4094, 4131, 4105, 4099] avg=4096.0000 max=4630 max_rank=0 min=3517 min_rank=3 imbalance_ratio=1.130371 max_load_reduction=28138 max_load_reduction_pct=0.858704 moved_to_echo=16429 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1789, 223, 2388, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4094, 0, 0, 0, 2, 7, 4085, 0, 0, 0, 0, 0, 0, 1744, 6, 1, 2257, 0, 0, 0, 0, 2, 0, 0, 4093, 1, 0, 0, 0, 3773, 1, 94, 33, 0, 0, 0, 4072, 3, 0, 2, 9, 0, 0, 0, 0, 2, 4082, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4400, 4096, 4094, 4008, 4096, 3901, 4086, 4087] avg=4096.0000 max=4400 max_rank=0 min=3901 min_rank=5 imbalance_ratio=1.074219 max_load_reduction=28368 max_load_reduction_pct=0.865723 moved_to_echo=16170 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1792, 30, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3407, 0, 0, 2261, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 4094, 0, 1, 33, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2482, 4096, 4096, 5668, 4094, 4128, 4105, 4099] avg=4096.0000 max=5668 max_rank=3 min=2482 min_rank=0 imbalance_ratio=1.383789 max_load_reduction=27100 max_load_reduction_pct=0.827026 moved_to_echo=16426 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 14720, 128, 128, 18176, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[14336, 1792, 16768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32256, 128, 384, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 4095, 1, 1, 4095, 4095, 4095, 0, 0, 0, 0, 0, 8, 1, 4005, 0, 4095, 1, 4088, 0, 0, 0, 0, 1, 91, 0, 1, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 4096, 0, 0, 4095, 4096, 4092, 0, 0, 4, 0, 6, 19, 0, 4051, 77, 4095, 0, 4026, 0, 1, 0, 0, 0, 20, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 11, 0, 4093, 19, 4096, 0, 4060, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4091, 4096, 4096, 0, 0, 0, 0, 0, 9, 0, 4083, 11, 4096, 0, 4086, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4088, 4, 0, 4, 4087, 2, 3, 4088, 4089, 4085, 0, 3, 0, 0, 0, 14, 5, 4085, 0, 4072, 6, 4035, 6, 0, 8, 1, 7, 65, 0, 7, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4091, 2, 0, 0, 4087, 0, 1, 4092, 4092, 4093, 0, 0, 0, 0, 6, 12, 1, 4092, 0, 4086, 2, 4078, 3, 5, 2, 4, 3, 13, 0, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4009, 19, 4096, 0, 4079, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4029, 4, 4096, 0, 4093, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 0, 0, 4, 4092, 0, 0, 0, 0, 0, 0, 4, 0, 6, 19, 4086, 0, 0, 0, 0, 4051, 77, 25, 0, 0, 0, 0, 0, 4026, 0, 1, 24, 0, 0, 0, 0, 0, 0, 20, 4045, 5, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4097, 4096, 4115, 4153, 4051, 4070, 4095] avg=4096.0000 max=4153 max_rank=4 min=4051 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 1, 0, 0, 0, 2, 1, 1, 4095, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4085, 0, 0, 0, 1, 4005, 0, 25, 0, 0, 0, 0, 1, 4088, 0, 0, 24, 0, 0, 0, 0, 0, 1, 91, 4045, 5, 0, 0, 0, 1, 0, 2, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4099, 4099, 4093, 4031, 4113, 4142, 4097] avg=4096.0000 max=4142 max_rank=6 min=4031 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4086, 0, 0, 0, 0, 4083, 11, 25, 0, 0, 0, 0, 0, 4086, 0, 0, 24, 0, 0, 0, 0, 0, 0, 8, 4046, 8, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4091, 4096, 4095, 4119, 4110, 4062, 4098] avg=4096.0000 max=4119 max_rank=4 min=4062 min_rank=6 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4086, 0, 0, 0, 0, 4093, 19, 25, 0, 0, 0, 0, 0, 4060, 0, 0, 24, 0, 0, 0, 0, 0, 0, 13, 4046, 5, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4098, 4096, 4097, 4137, 4084, 4064, 4095] avg=4096.0000 max=4137 max_rank=4 min=4064 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4009, 19, 25, 0, 0, 0, 0, 0, 4079, 0, 0, 24, 0, 0, 0, 0, 0, 0, 85, 4046, 5, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4098, 4100, 4086, 4053, 4103, 4136, 4095] avg=4096.0000 max=4136 max_rank=6 min=4053 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4088, 4, 0, 4, 5, 0, 0, 0, 0, 2, 3, 4088, 0, 0, 0, 0, 0, 4085, 0, 3, 0, 0, 0, 0, 0, 0, 0, 14, 4082, 0, 0, 0, 5, 4085, 0, 9, 0, 0, 0, 0, 6, 4035, 6, 0, 31, 0, 0, 0, 8, 1, 7, 65, 4027, 6, 0, 0, 0, 7, 0, 4, 4087, 1, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4093, 4088, 4096, 4099, 4078, 4114, 4099] avg=4096.0000 max=4114 max_rank=6 min=4078 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4091, 2, 0, 0, 1, 0, 0, 0, 1, 0, 1, 4092, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 6, 12, 4082, 0, 0, 0, 1, 4092, 0, 25, 0, 0, 0, 0, 2, 4078, 3, 5, 24, 0, 0, 0, 2, 4, 3, 13, 4036, 5, 0, 0, 0, 3, 0, 0, 4086, 5, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4094, 4093, 4100, 4118, 4112, 4063, 4094] avg=4096.0000 max=4118 max_rank=4 min=4063 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4029, 4, 25, 0, 0, 0, 0, 0, 4093, 0, 0, 24, 0, 0, 0, 0, 0, 0, 66, 4046, 5, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4098, 4100, 4086, 4058, 4117, 4117, 4095] avg=4096.0000 max=4117 max_rank=5 min=4058 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 384, 32384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32640, 128, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32512, 256, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 23, 0, 0, 0, 10, 0, 4079, 0, 0, 0, 4096, 4096, 0, 0, 1, 2, 4095, 4096, 0, 0, 4080, 4092, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 4095, 0, 0, 0, 7, 4096, 4096, 0, 0, 4086, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4081, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 4095, 1, 1, 1, 0, 0, 1, 4092, 0, 0, 1, 4094, 4095, 1, 1, 0, 0, 4095, 4095, 0, 0, 4095, 4095, 1, 0, 1, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[3, 2, 5, 0, 4094, 32, 2, 3, 0, 2, 3, 4063, 0, 0, 1, 4091, 4092, 0, 3, 0, 0, 4093, 4093, 0, 0, 4087, 4094, 3, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[2, 7, 10, 0, 4090, 24, 8, 6, 0, 14, 8, 4070, 0, 0, 1, 4063, 4086, 0, 5, 0, 0, 4088, 4089, 0, 0, 4076, 4088, 8, 0, 17, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 3, 2, 0, 4084, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 4086, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4092, 4097, 4100, 4095, 4103, 4094, 4094] avg=4096.0000 max=4103 max_rank=5 min=4092 min_rank=1 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4084, 3, 2, 0, 4084, 23, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4079, 5, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 2, 4095, 1, 0, 0, 0, 0, 0, 0, 4080, 8, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4107, 4094, 4100, 4097, 4098, 4088, 4095] avg=4096.0000 max=4107 max_rank=1 min=4088 min_rank=6 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 4087, 3, 2, 0, 4083, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 4092, 5, 0, 0, 0, 0, 0, 1, 4094, 4, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4095, 8, 1, 0, 0, 0, 0, 0, 1, 0, 1, 4092, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4086, 4098, 4099, 4097, 4096, 4104, 4094] avg=4096.0000 max=4104 max_rank=6 min=4086 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 3, 2, 0, 4084, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4081, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4099, 4101, 4100, 4096, 4096, 4089, 4094] avg=4096.0000 max=4101 max_rank=2 min=4089 min_rank=6 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 3, 2, 0, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4094, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4086, 4101, 4100, 4096, 4097, 4102, 4093] avg=4096.0000 max=4102 max_rank=6 min=4086 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 7, 10, 0, 4083, 8, 3, 0, 4057, 24, 8, 6, 0, 0, 0, 0, 0, 14, 8, 4070, 11, 0, 0, 0, 0, 0, 1, 4063, 9, 0, 0, 0, 4086, 0, 5, 0, 5, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4076, 5, 8, 0, 0, 0, 0, 0, 17, 0, 8, 4086, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4095, 4103, 4073, 4096, 4088, 4089, 4111] avg=4096.0000 max=4113 max_rank=0 min=4073 min_rank=3 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 2, 5, 0, 4086, 3, 1, 0, 4082, 32, 2, 3, 0, 0, 0, 0, 0, 2, 3, 4063, 5, 0, 0, 0, 0, 0, 1, 4091, 4, 0, 0, 0, 4092, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4087, 8, 3, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4119, 4073, 4096, 4095, 4093, 4098, 4094] avg=4096.0000 max=4119 max_rank=1 min=4073 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 3, 2, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4084, 4101, 4100, 4096, 4097, 4104, 4093] avg=4096.0000 max=4104 max_rank=6 min=4084 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4, 0, 697, 4096, 0, 0, 0, 0, 0, 1, 0, 4067, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4053, 4092, 0, 0, 4096, 3470, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 1139, 4091, 1, 0, 1, 1, 0, 0, 3, 4094, 0, 1, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 1, 4041, 4095, 0, 1, 4095, 3010, 2, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 415, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 4094, 0, 0, 4096, 3692, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 39, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4057, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 371, 4089, 0, 0, 2, 3, 0, 0, 1, 4091, 0, 3, 2, 0, 0, 4093, 0, 5, 4, 0, 5, 2, 4070, 4095, 0, 3, 4093, 3740, 4, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 856, 4089, 1, 4, 7, 7, 2, 38, 0, 4088, 10, 8, 0, 0, 0, 4089, 0, 13, 1, 0, 0, 5, 4000, 4091, 0, 5, 4089, 3289, 0, 4076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 323, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4024, 4096, 0, 0, 4096, 3845, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 194, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4027, 4096, 0, 0, 4096, 3971, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 697, 3594, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 502, 1, 0, 0, 1, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 42, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4053, 46, 0, 0, 0, 0, 0, 0, 0, 3470, 0, 463, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4295, 4094, 4068, 4089, 4095, 4095, 4099, 3933] avg=4096.0000 max=4295 max_rank=0 min=3933 min_rank=7 imbalance_ratio=1.048584 max_load_reduction=28473 max_load_reduction_pct=0.868927 moved_to_echo=16222 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 415, 3594, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 502, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 42, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4087, 46, 0, 0, 0, 0, 0, 0, 0, 3692, 0, 463, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4009, 4094, 4096, 4092, 4095, 4094, 4133, 4155] avg=4096.0000 max=4155 max_rank=7 min=4009 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1139, 3589, 0, 0, 0, 0, 1, 0, 1, 1, 3590, 502, 1, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 4049, 42, 1, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 4094, 0, 0, 0, 4041, 46, 0, 1, 0, 0, 0, 0, 0, 3010, 2, 462, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4729, 4096, 4097, 4093, 4095, 4096, 4088, 3474] avg=4096.0000 max=4729 max_rank=0 min=3474 min_rank=7 imbalance_ratio=1.154541 max_load_reduction=28039 max_load_reduction_pct=0.855682 moved_to_echo=15753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 39, 3594, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 502, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 42, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 46, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 463, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3633, 4094, 4096, 4093, 4095, 4095, 4142, 4520] avg=4096.0000 max=4520 max_rank=7 min=3633 min_rank=0 imbalance_ratio=1.103516 max_load_reduction=28248 max_load_reduction_pct=0.862061 moved_to_echo=16852 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 323, 3594, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 502, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 42, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4024, 46, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 463, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3917, 4094, 4096, 4093, 4095, 4095, 4070, 4308] avg=4096.0000 max=4308 max_rank=7 min=3917 min_rank=0 imbalance_ratio=1.051758 max_load_reduction=28460 max_load_reduction_pct=0.868530 moved_to_echo=16568 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 856, 3582, 0, 0, 0, 0, 1, 4, 7, 7, 3579, 507, 2, 1, 2, 38, 0, 4087, 0, 0, 0, 0, 10, 8, 0, 0, 4052, 48, 1, 0, 0, 4087, 0, 13, 0, 0, 0, 0, 1, 0, 0, 5, 4088, 0, 0, 0, 4000, 39, 0, 5, 0, 0, 0, 0, 0, 3289, 0, 449, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4438, 4108, 4127, 4119, 4100, 4094, 4044, 3738] avg=4096.0000 max=4438 max_rank=0 min=3738 min_rank=7 imbalance_ratio=1.083496 max_load_reduction=28330 max_load_reduction_pct=0.864563 moved_to_echo=15976 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 371, 3587, 0, 0, 0, 0, 0, 0, 2, 3, 3587, 502, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 3, 2, 0, 4049, 42, 0, 0, 0, 4093, 0, 5, 0, 0, 0, 0, 4, 0, 5, 2, 4093, 0, 0, 0, 4070, 46, 0, 3, 0, 0, 0, 0, 0, 3740, 4, 462, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3959, 4094, 4092, 4096, 4098, 4104, 4119, 4206] avg=4096.0000 max=4206 max_rank=7 min=3959 min_rank=0 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16527 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 194, 3594, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 502, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 42, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4027, 46, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 463, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3788, 4094, 4096, 4093, 4095, 4095, 4073, 4434] avg=4096.0000 max=4434 max_rank=7 min=3788 min_rank=0 imbalance_ratio=1.082520 max_load_reduction=28334 max_load_reduction_pct=0.864685 moved_to_echo=16697 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 29184, 128, 3712, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 28800, 4096, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 32512, 384, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 4096, 28800, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 384, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33664 slack_slots=1792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3659, 0, 41, 0, 0, 4, 0, 0, 0, 4096, 396, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4062, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 25, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[3, 4094, 29, 1, 4094, 0, 4095, 4061, 0, 2, 0, 1, 4095, 1, 0, 1, 0, 4082, 1, 2, 0, 0, 0, 1, 1, 0, 4092, 4, 4095, 0, 13, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[2, 4092, 5, 3, 4091, 0, 4092, 4077, 4, 4, 0, 3, 4092, 3, 2, 4, 3, 4015, 2, 0, 8, 0, 0, 1, 0, 0, 4089, 54, 4088, 0, 34, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 8, 0, 4096, 0, 4096, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4064, 0, 19, 0, 0, 0, 0, 0, 0, 4094, 11, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 249, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[6, 4087, 0, 9, 4088, 0, 4088, 4052, 0, 13, 0, 6, 4085, 4, 7, 0, 8, 4022, 7, 0, 1, 0, 0, 7, 0, 0, 4084, 65, 4086, 0, 42, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4071, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 20, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 10, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3659, 0, 41, 90, 8, 6, 0, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 0, 3996, 396, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4106, 4093, 4096, 3804, 4099, 4392, 4088] avg=4096.0000 max=4392 max_rank=6 min=3804 min_rank=4 imbalance_ratio=1.072266 max_load_reduction=28376 max_load_reduction_pct=0.865967 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 4088, 29, 1, 0, 0, 0, 0, 14, 0, 0, 4061, 0, 0, 0, 0, 0, 2, 0, 1, 4080, 9, 1, 0, 4095, 1, 0, 1, 0, 0, 0, 0, 0, 4082, 1, 2, 89, 8, 6, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 1, 0, 3994, 4, 0, 0, 0, 0, 4087, 0, 13, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4075, 4093, 4097, 4188, 4095, 3999, 4100] avg=4096.0000 max=4188 max_rank=4 min=3999 min_rank=6 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 8, 0, 0, 0, 0, 0, 14, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 10, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 19, 90, 8, 6, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3994, 11, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4106, 4093, 4096, 4187, 4095, 4005, 4088] avg=4096.0000 max=4187 max_rank=4 min=4005 min_rank=6 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 10, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 10, 90, 8, 6, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3996, 25, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4109, 4093, 4096, 4176, 4095, 4021, 4088] avg=4096.0000 max=4176 max_rank=4 min=4021 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 10, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 0, 0, 90, 8, 6, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3996, 249, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4110, 4093, 4096, 3951, 4095, 4245, 4088] avg=4096.0000 max=4245 max_rank=6 min=3951 min_rank=4 imbalance_ratio=1.036377 max_load_reduction=28523 max_load_reduction_pct=0.870453 moved_to_echo=16379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[6, 4077, 0, 9, 0, 0, 0, 0, 9, 0, 0, 4052, 0, 0, 0, 0, 0, 13, 0, 6, 4079, 12, 2, 0, 4079, 4, 7, 0, 0, 0, 0, 0, 8, 4022, 7, 0, 92, 15, 10, 6, 1, 0, 0, 7, 4086, 0, 0, 0, 0, 0, 3980, 65, 0, 0, 0, 0, 4071, 0, 42, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4061, 4112, 4090, 4160, 4094, 4045, 4114] avg=4096.0000 max=4160 max_rank=4 min=4045 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 4086, 5, 3, 0, 0, 0, 0, 14, 0, 0, 4077, 0, 0, 0, 0, 4, 4, 0, 3, 4077, 9, 1, 0, 4092, 3, 2, 4, 0, 0, 0, 0, 3, 4015, 2, 0, 89, 8, 6, 0, 8, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 3991, 54, 0, 0, 0, 0, 4080, 0, 34, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4091, 4098, 4101, 4123, 4100, 4045, 4114] avg=4096.0000 max=4123 max_rank=4 min=4045 min_rank=6 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 1, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 10, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4, 90, 8, 6, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3996, 20, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4110, 4093, 4096, 4179, 4095, 4016, 4088] avg=4096.0000 max=4179 max_rank=4 min=4016 min_rank=6 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32000, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 31872, 128, 128, 768, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 20, 173, 0, 4096, 4096, 0, 0, 0, 1335, 0, 0, 4092, 4096, 0, 0, 4096, 3499, 0, 3164, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 3, 0, 0, 0, 4092, 2, 0, 3, 11, 225, 4, 4092, 4088, 4, 1, 1, 810, 0, 2, 4091, 4093, 5, 4, 4081, 3487, 0, 3661, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 0, 31, 1, 4095, 0, 2, 1, 0, 314, 1, 4095, 4093, 1, 0, 1, 490, 0, 1, 4095, 4095, 2, 0, 4094, 3500, 0, 3853, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 17, 207, 0, 4096, 4096, 0, 0, 0, 1162, 0, 0, 4093, 4096, 0, 0, 4096, 3086, 0, 3716, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 5, 0, 0, 0, 4088, 0, 1, 9, 15, 85, 8, 4087, 4070, 10, 4, 0, 370, 0, 8, 4086, 4089, 10, 4, 4033, 3688, 1, 4074, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 191, 0, 4096, 4096, 0, 0, 0, 1251, 0, 0, 4096, 4096, 0, 0, 4096, 3549, 0, 3199, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 682, 0, 4096, 4096, 0, 0, 0, 567, 0, 0, 4096, 4096, 0, 0, 4096, 3154, 0, 3789, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 5, 0, 0, 0, 0, 5, 0, 2609, 803, 256, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 20, 173, 0, 3839, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 1335, 0, 0, 3289, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 3499, 0, 550, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 3673, 4096, 4032, 4097, 4624, 4103, 4049] avg=4096.0000 max=4624 max_rank=5 min=3673 min_rank=1 imbalance_ratio=1.128906 max_load_reduction=28144 max_load_reduction_pct=0.858887 moved_to_echo=16873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4088, 6, 0, 0, 1, 0, 31, 1, 3177, 804, 256, 0, 4095, 0, 2, 1, 0, 0, 0, 0, 0, 314, 1, 3838, 0, 0, 0, 0, 4093, 1, 0, 1, 1, 0, 0, 0, 490, 0, 1, 3291, 0, 0, 0, 0, 7, 2, 0, 4094, 0, 0, 0, 0, 3500, 0, 670, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4270, 4098, 4153, 4096, 3782, 4103, 4171] avg=4096.0000 max=4270 max_rank=1 min=3782 min_rank=5 imbalance_ratio=1.042480 max_load_reduction=28498 max_load_reduction_pct=0.869690 moved_to_echo=16152 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 6, 0, 0, 0, 0, 7, 0, 3064, 804, 256, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 207, 0, 3839, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 1162, 0, 0, 3289, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 3086, 0, 646, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4131, 4096, 4063, 4097, 4451, 4103, 3732] avg=4096.0000 max=4451 max_rank=5 min=3732 min_rank=7 imbalance_ratio=1.086670 max_load_reduction=28317 max_load_reduction_pct=0.864166 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 5, 0, 0, 0, 0, 0, 0, 2638, 804, 256, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 191, 0, 3839, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 1251, 0, 0, 3292, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 3549, 0, 556, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 3698, 4096, 4032, 4097, 4543, 4103, 4105] avg=4096.0000 max=4543 max_rank=5 min=3698 min_rank=1 imbalance_ratio=1.109131 max_load_reduction=28225 max_load_reduction_pct=0.861359 moved_to_echo=16848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 4086, 6, 0, 0, 3, 0, 0, 0, 3019, 803, 256, 0, 4092, 2, 0, 3, 0, 0, 0, 0, 11, 225, 4, 3835, 0, 0, 0, 0, 4088, 4, 1, 1, 1, 0, 0, 0, 810, 0, 2, 3288, 0, 0, 0, 0, 7, 5, 4, 4081, 0, 0, 0, 0, 3487, 0, 636, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4081, 4097, 4075, 4095, 4100, 4097, 4127] avg=4096.0000 max=4127 max_rank=7 min=4075 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 4083, 11, 0, 0, 5, 0, 0, 0, 3364, 807, 261, 0, 4083, 0, 1, 9, 0, 0, 0, 0, 15, 85, 8, 3823, 0, 0, 0, 0, 4070, 10, 4, 0, 3, 5, 0, 0, 370, 0, 8, 3279, 0, 0, 0, 0, 6, 10, 4, 4033, 0, 0, 0, 0, 3688, 1, 699, 13, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4437, 4093, 3931, 4092, 3657, 4053, 4401] avg=4096.0000 max=4437 max_rank=1 min=3657 min_rank=5 imbalance_ratio=1.083252 max_load_reduction=28331 max_load_reduction_pct=0.864594 moved_to_echo=16203 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 6, 0, 0, 0, 0, 0, 0, 3125, 804, 256, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 682, 0, 3839, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 567, 0, 0, 3292, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 3154, 0, 658, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4185, 4096, 4521, 4097, 3859, 4103, 3812] avg=4096.0000 max=4521 max_rank=3 min=3812 min_rank=7 imbalance_ratio=1.103760 max_load_reduction=28247 max_load_reduction_pct=0.862030 moved_to_echo=15871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 24, 98, 0, 4096, 4096, 0, 0, 0, 467, 0, 0, 4087, 4096, 0, 0, 4096, 3691, 0, 3913, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 6, 0, 0, 0, 0, 8, 0, 3227, 802, 256, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 24, 98, 0, 3839, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 467, 0, 0, 3285, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 3691, 0, 680, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4293, 4096, 3961, 4097, 3752, 4103, 4371] avg=4096.0000 max=4371 max_rank=7 min=3752 min_rank=5 imbalance_ratio=1.067139 max_load_reduction=28397 max_load_reduction_pct=0.866608 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 24320, 6528, 2176, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6528, 0, 128, 26368, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[27776, 128, 5120, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 2048, 128, 30720, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4095, 4091, 1, 0, 0, 2, 4073, 0, 0, 1, 4095, 0, 0, 0, 23, 1, 0, 0, 3, 0, 1, 4095, 1, 4094, 1, 0, 2, 4095, 0, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4090, 4082, 3, 0, 0, 7, 4068, 3, 1, 4, 4092, 0, 1, 0, 25, 2, 1, 0, 5, 0, 4, 4092, 4, 4089, 3, 0, 4, 4091, 0, 4088, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[3, 4083, 4087, 6, 0, 0, 16, 3948, 8, 0, 0, 4086, 0, 9, 0, 141, 5, 3, 0, 40, 0, 7, 4087, 7, 4080, 8, 1, 7, 4080, 0, 4052, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4091, 1, 0, 0, 0, 0, 0, 0, 2, 4073, 20, 1, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 23, 4069, 1, 0, 0, 1, 0, 0, 3, 4089, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 0, 0, 4094, 1, 0, 2, 0, 0, 0, 0, 6, 0, 4094, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4096, 4093, 4093, 4096, 4097, 4100] avg=4096.0000 max=4100 max_rank=7 min=4093 min_rank=3 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4117, 4096, 4071, 4090, 4095, 4096, 4102] avg=4096.0000 max=4117 max_rank=1 min=4071 min_rank=3 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 4070, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4117, 4096, 4078, 4090, 4095, 4096, 4102] avg=4096.0000 max=4117 max_rank=1 min=4078 min_rank=3 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 4070, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4117, 4096, 4074, 4090, 4095, 4096, 4102] avg=4096.0000 max=4117 max_rank=1 min=4074 min_rank=3 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 4070, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4117, 4096, 4078, 4093, 4095, 4096, 4102] avg=4096.0000 max=4117 max_rank=1 min=4078 min_rank=3 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4087, 6, 0, 0, 0, 0, 0, 0, 16, 3948, 21, 3, 2, 1, 8, 0, 0, 4084, 0, 0, 0, 0, 0, 9, 0, 141, 4056, 2, 0, 0, 5, 3, 0, 40, 4081, 0, 0, 0, 0, 7, 4084, 7, 0, 0, 0, 0, 4079, 8, 1, 7, 0, 0, 0, 0, 3, 0, 4052, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3991, 4092, 4208, 4129, 4098, 4095, 4059] avg=4096.0000 max=4208 max_rank=3 min=3991 min_rank=1 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4082, 3, 0, 0, 0, 0, 0, 0, 7, 4068, 20, 1, 0, 0, 3, 1, 4, 4092, 0, 0, 0, 0, 0, 1, 0, 25, 4065, 0, 0, 0, 2, 1, 0, 5, 4085, 0, 0, 0, 0, 4, 4091, 4, 0, 0, 0, 0, 4089, 3, 0, 4, 0, 0, 0, 0, 6, 0, 4088, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4096, 4100, 4091, 4093, 4099, 4096, 4103] avg=4096.0000 max=4103 max_rank=7 min=4090 min_rank=0 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 4070, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4092, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4117, 4096, 4075, 4090, 4095, 4096, 4098] avg=4096.0000 max=4117 max_rank=1 min=4075 min_rank=3 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 128, 32640, 256, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3981, 4096, 4092, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 108, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[68, 1, 16, 8, 7, 4019, 4083, 4089, 7, 8, 13, 0, 8, 6, 4069, 0, 1, 4082, 1, 4078, 0, 0, 0, 4, 0, 0, 4, 4088, 12, 0, 12, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4001, 4096, 4096, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 86, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[67, 3, 2, 3, 7, 4010, 4090, 4092, 4, 2, 14, 6, 2, 3, 4086, 0, 2, 4091, 0, 4090, 0, 0, 0, 4, 0, 0, 1, 4092, 4, 0, 3, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4068, 4096, 4096, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 4061, 4096, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 30, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[29, 0, 1, 1, 0, 4066, 4095, 4095, 1, 1, 0, 1, 1, 1, 4093, 0, 1, 4095, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 4095, 1, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4084, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4064, 8, 0, 0, 0, 3981, 59, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4084, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 108, 4036, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4040, 4095, 4097, 4100, 4172, 4096, 4096] avg=4096.0000 max=4172 max_rank=5 min=4040 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4066, 9, 0, 0, 0, 4068, 59, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 23, 4036, 30, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4127, 4092, 4097, 4096, 4089, 4096, 4096] avg=4096.0000 max=4127 max_rank=1 min=4075 min_rank=0 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4064, 9, 0, 0, 0, 4001, 59, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 86, 4036, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4060, 4096, 4097, 4100, 4150, 4096, 4096] avg=4096.0000 max=4150 max_rank=5 min=4060 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16442 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[68, 1, 16, 8, 4054, 9, 0, 0, 7, 4019, 51, 0, 0, 0, 0, 0, 7, 8, 13, 0, 4080, 0, 0, 0, 8, 6, 4069, 0, 5, 7, 0, 0, 1, 0, 1, 4078, 0, 0, 0, 0, 0, 0, 0, 4, 4027, 28, 0, 0, 0, 0, 4, 4088, 6, 0, 0, 0, 12, 0, 12, 4071, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4077, 4108, 4095, 4080, 4059, 4098, 4095] avg=4096.0000 max=4156 max_rank=0 min=4059 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16332 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4064, 8, 0, 0, 0, 4061, 59, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 30, 4036, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4120, 4090, 4097, 4100, 4094, 4096, 4096] avg=4096.0000 max=4120 max_rank=1 min=4075 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[67, 3, 2, 3, 4059, 9, 0, 0, 7, 4010, 59, 0, 0, 0, 0, 0, 4, 2, 14, 6, 4083, 0, 0, 0, 2, 3, 4086, 0, 1, 0, 0, 0, 2, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4, 4030, 31, 0, 0, 0, 0, 1, 4092, 0, 0, 0, 0, 4, 0, 3, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4076, 4109, 4092, 4093, 4065, 4093, 4097] avg=4096.0000 max=4143 max_rank=0 min=4065 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 1, 1, 4063, 8, 0, 0, 0, 4066, 59, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4087, 0, 0, 0, 1, 1, 4093, 0, 1, 0, 0, 0, 1, 4, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 28, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4125, 4090, 4096, 4099, 4063, 4097, 4096] avg=4096.0000 max=4125 max_rank=1 min=4063 min_rank=5 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16355 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4064, 8, 0, 0, 0, 4084, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4036, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4143, 4088, 4097, 4100, 4076, 4096, 4096] avg=4096.0000 max=4143 max_rank=1 min=4072 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 128, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32384, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 384, 32384, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 2912, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 29, 0, 4096, 4089, 0, 0, 0, 1158, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3811, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 284, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[3, 0, 0, 9, 9, 0, 4088, 10, 13, 13, 3691, 4085, 6, 4, 4083, 1, 0, 1, 0, 11, 0, 0, 0, 4088, 4080, 4, 7, 0, 394, 4087, 2, 4079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4, 0, 0, 4, 4, 0, 4092, 4, 0, 5, 3183, 4092, 4, 3, 4090, 0, 0, 1, 0, 5, 0, 7, 0, 4092, 4088, 1, 3, 0, 908, 4092, 1, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 3294, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 4095, 4096, 0, 0, 0, 799, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3804, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 291, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2856, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 0, 4096, 4093, 0, 0, 0, 1233, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 12, 1, 1, 0, 4095, 1, 1, 1, 3568, 4095, 0, 1, 4093, 1, 0, 0, 0, 1, 0, 0, 0, 4095, 4084, 0, 1, 0, 527, 4095, 1, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 3389, 696, 3, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3294, 702, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 1, 0, 0, 799, 0, 0, 3400, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4095, 3996, 4096, 4096, 4098, 4098, 4199] avg=4096.0000 max=4199 max_rank=7 min=3996 min_rank=2 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16491 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 3393, 696, 3, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2912, 703, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 29, 0, 4092, 0, 0, 0, 0, 4089, 0, 0, 0, 1, 1, 0, 0, 1158, 0, 0, 3400, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4095, 3615, 4096, 4095, 4121, 4091, 4558] avg=4096.0000 max=4558 max_rank=7 min=3615 min_rank=2 imbalance_ratio=1.112793 max_load_reduction=28210 max_load_reduction_pct=0.860901 moved_to_echo=16865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3393, 696, 3, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3811, 703, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 1, 0, 0, 284, 0, 0, 3400, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4095, 4514, 4096, 4096, 4093, 4098, 3684] avg=4096.0000 max=4514 max_rank=2 min=3684 min_rank=7 imbalance_ratio=1.102051 max_load_reduction=28254 max_load_reduction_pct=0.862244 moved_to_echo=15971 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 9, 3387, 698, 9, 1, 9, 0, 4079, 10, 0, 0, 0, 0, 13, 13, 3691, 698, 0, 0, 0, 0, 6, 4, 4083, 1, 2, 0, 0, 0, 0, 1, 0, 11, 4086, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 4080, 4, 7, 0, 8, 7, 0, 0, 394, 0, 2, 3381, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4098, 4415, 4096, 4098, 4071, 4106, 3777] avg=4096.0000 max=4415 max_rank=2 min=3777 min_rank=7 imbalance_ratio=1.077881 max_load_reduction=28353 max_load_reduction_pct=0.865265 moved_to_echo=16052 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3389, 696, 3, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2856, 702, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 15, 0, 4092, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 1, 0, 0, 1233, 0, 0, 3400, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4095, 3558, 4096, 4095, 4107, 4095, 4633] avg=4096.0000 max=4633 max_rank=7 min=3558 min_rank=2 imbalance_ratio=1.131104 max_load_reduction=28135 max_load_reduction_pct=0.858612 moved_to_echo=16930 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 1, 3392, 696, 3, 1, 1, 0, 4094, 1, 0, 0, 0, 0, 1, 1, 3568, 703, 0, 0, 0, 0, 0, 1, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4084, 0, 1, 0, 1, 1, 0, 0, 527, 0, 1, 3398, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4096, 4273, 4095, 4095, 4091, 4087, 3926] avg=4096.0000 max=4273 max_rank=2 min=3926 min_rank=7 imbalance_ratio=1.043213 max_load_reduction=28495 max_load_reduction_pct=0.869598 moved_to_echo=16199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 4, 3390, 694, 3, 1, 4, 0, 4091, 4, 0, 0, 0, 0, 0, 5, 3183, 702, 0, 0, 0, 0, 4, 3, 4090, 0, 0, 0, 0, 0, 0, 1, 0, 5, 4091, 0, 0, 0, 0, 7, 0, 4088, 0, 0, 0, 0, 4088, 1, 3, 0, 1, 1, 0, 0, 908, 0, 1, 3391, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4099, 3890, 4097, 4097, 4095, 4094, 4300] avg=4096.0000 max=4300 max_rank=7 min=3890 min_rank=2 imbalance_ratio=1.049805 max_load_reduction=28468 max_load_reduction_pct=0.868774 moved_to_echo=16586 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3393, 696, 3, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3804, 703, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 1, 0, 0, 291, 0, 0, 3400, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4095, 4507, 4096, 4096, 4092, 4099, 3691] avg=4096.0000 max=4507 max_rank=2 min=3691 min_rank=7 imbalance_ratio=1.100342 max_load_reduction=28261 max_load_reduction_pct=0.862457 moved_to_echo=15978 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 27136, 5632, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 27136, 5632, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5632, 0, 128, 27264, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4, 0, 4096, 0, 0, 4063, 0, 0, 0, 0, 4096, 4092, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 33, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4058, 0, 0, 0, 0, 0, 4096, 0, 0, 4069, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 4096, 0, 66, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4075, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 26, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 0, 4092, 0, 4, 0, 0, 4, 4092, 5, 4, 4087, 8, 4, 0, 8, 4090, 4092, 5, 2, 4091, 0, 0, 0, 4083, 4, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[6, 0, 0, 4086, 0, 8, 0, 0, 9, 4086, 11, 10, 4085, 18, 6, 0, 0, 4085, 4087, 11, 1, 4084, 0, 0, 0, 4072, 10, 4085, 0, 0, 4, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 0, 12, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 3, 0, 4095, 0, 1, 0, 0, 1, 4095, 1, 1, 4092, 0, 0, 0, 0, 4095, 4095, 1, 1, 4095, 0, 2, 0, 4093, 1, 4095, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 7, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 3, 3, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 33, 0, 0, 4078, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4100, 4093, 4069, 4092, 4096, 4101, 4111] avg=4096.0000 max=4111 max_rank=7 min=4069 min_rank=3 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4058, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 4, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 4082, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4096, 4093, 4076, 4096, 4096, 4095, 4148] avg=4096.0000 max=4148 max_rank=7 min=4068 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16435 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4075, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 3, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 26, 0, 0, 4078, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4096, 4093, 4097, 4096, 4096, 4101, 4104] avg=4096.0000 max=4104 max_rank=7 min=4085 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 4095, 10, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 1, 4092, 1, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 3, 3, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 1, 4095, 0, 2, 0, 0, 0, 0, 0, 4093, 1, 5, 0, 0, 0, 0, 0, 0, 0, 1, 4077, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4096, 4095, 4098, 4096, 4098, 4099, 4078] avg=4096.0000 max=4108 max_rank=0 min=4078 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 5, 0, 0, 0, 0, 0, 12, 0, 0, 4078, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4093, 4101, 4096, 4096, 4099, 4090] avg=4096.0000 max=4101 max_rank=3 min=4090 min_rank=7 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 4086, 11, 0, 0, 0, 0, 8, 0, 0, 4085, 1, 0, 0, 9, 4076, 11, 10, 0, 0, 0, 0, 4085, 18, 6, 0, 6, 5, 0, 0, 0, 0, 4080, 11, 0, 0, 0, 0, 1, 4084, 0, 0, 7, 5, 0, 0, 0, 4072, 10, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4067, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4094, 4106, 4120, 4091, 4097, 4082, 4075] avg=4096.0000 max=4120 max_rank=3 min=4075 min_rank=7 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16345 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4092, 9, 0, 0, 0, 0, 4, 0, 0, 4090, 0, 0, 0, 4, 4089, 5, 4, 0, 0, 0, 0, 4087, 8, 4, 0, 3, 3, 0, 0, 8, 0, 4092, 5, 0, 0, 0, 0, 2, 4091, 0, 0, 0, 0, 0, 0, 0, 4083, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4094, 4102, 4105, 4105, 4093, 4090, 4077] avg=4096.0000 max=4105 max_rank=3 min=4077 min_rank=7 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4089, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 7, 0, 0, 4078, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 4093, 4102, 4096, 4096, 4101, 4085] avg=4096.0000 max=4102 max_rank=3 min=4085 min_rank=7 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 256, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4091, 4092, 0, 11, 7, 0, 4092, 0, 0, 0, 0, 3, 0, 4, 6, 4083, 4, 4, 0, 0, 5, 0, 1, 0, 4087, 4092, 4, 0, 0, 4092, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1, 4094, 4095, 0, 3, 1, 0, 4095, 0, 0, 0, 0, 1, 0, 1, 2, 4091, 1, 1, 0, 1, 2, 0, 0, 0, 4094, 4095, 1, 0, 0, 4095, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4085, 4083, 0, 30, 14, 0, 4087, 0, 0, 0, 0, 7, 0, 10, 16, 4078, 10, 11, 0, 0, 13, 0, 0, 0, 4059, 4086, 9, 0, 2, 4086, 4082] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 4086, 5, 0, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 4089, 5, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4089, 4098, 4091, 4093, 4098, 4101, 4099] avg=4096.0000 max=4101 max_rank=6 min=4089 min_rank=1 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 0, 0, 0, 4086, 5, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4092, 6, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4090, 4098, 4091, 4097, 4098, 4100, 4098] avg=4096.0000 max=4100 max_rank=6 min=4090 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 4086, 5, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4090, 4097, 4091, 4097, 4094, 4101, 4099] avg=4096.0000 max=4101 max_rank=6 min=4090 min_rank=1 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 3, 4095, 0, 0, 0, 0, 0, 3, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 1, 0, 1, 2, 4085, 5, 0, 0, 4091, 1, 1, 0, 1, 0, 0, 0, 1, 2, 0, 0, 4087, 5, 0, 0, 0, 4094, 5, 1, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4093, 4096, 4094, 4094, 4095, 4100, 4097] avg=4096.0000 max=4100 max_rank=6 min=4093 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 4086, 5, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4090, 4097, 4091, 4097, 4094, 4101, 4099] avg=4096.0000 max=4101 max_rank=6 min=4090 min_rank=1 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4092, 0, 0, 0, 0, 0, 11, 7, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 3, 0, 4, 6, 4082, 5, 0, 0, 4083, 4, 4, 0, 1, 0, 0, 0, 0, 5, 0, 1, 4085, 5, 0, 0, 0, 4087, 5, 4, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4104, 4093, 4100, 4092, 4096, 4096, 4093] avg=4096.0000 max=4104 max_rank=1 min=4092 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4083, 0, 0, 0, 0, 0, 30, 14, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 0, 0, 7, 0, 10, 16, 4079, 7, 0, 0, 4078, 10, 11, 0, 2, 0, 0, 0, 0, 13, 0, 0, 4079, 7, 0, 0, 0, 4059, 0, 9, 0, 0, 0, 0, 0, 2, 0, 4082, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4122, 4092, 4119, 4101, 4099, 4068, 4084] avg=4096.0000 max=4122 max_rank=1 min=4068 min_rank=6 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 4086, 5, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4090, 4097, 4091, 4097, 4094, 4101, 4099] avg=4096.0000 max=4101 max_rank=6 min=4090 min_rank=1 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1, 4012, 1876, 3, 847, 1, 528, 20, 1992, 761, 5, 2644, 1811, 1010, 4, 2418, 2359, 1125, 223, 396, 36, 2560, 642, 1, 3, 151, 268, 1207, 2127, 23, 3712, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 4065, 2100, 2, 917, 0, 673, 1, 1776, 542, 5, 2354, 1610, 885, 0, 2268, 2092, 1972, 129, 168, 24, 2403, 978, 0, 0, 365, 169, 1129, 2374, 19, 3747, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1, 3943, 1839, 0, 892, 0, 485, 0, 1377, 962, 7, 2764, 1889, 704, 0, 2368, 2562, 1344, 376, 369, 41, 2849, 909, 10, 0, 358, 243, 1294, 1770, 24, 3388, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3, 3595, 2157, 7, 813, 0, 738, 2, 2056, 567, 15, 2800, 2055, 659, 2, 2517, 2008, 1128, 260, 361, 76, 2434, 761, 6, 1, 438, 184, 1118, 2206, 34, 3761, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2, 3904, 1810, 12, 964, 4, 461, 11, 1697, 553, 0, 1982, 1681, 777, 4, 2640, 2459, 1704, 393, 198, 58, 2807, 920, 7, 0, 109, 211, 1236, 2348, 34, 3782, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 3986, 1786, 1, 776, 0, 762, 0, 1777, 971, 4, 2356, 2270, 664, 0, 2487, 1957, 1689, 173, 286, 23, 2318, 622, 1, 0, 407, 182, 1026, 2300, 13, 3928, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1, 3959, 1770, 0, 956, 0, 581, 1, 2175, 920, 18, 2219, 2197, 787, 1, 2617, 1935, 1328, 102, 407, 48, 2407, 1039, 9, 0, 376, 311, 1416, 1456, 30, 3701, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 3941, 1792, 4, 919, 1, 648, 0, 1839, 553, 3, 2287, 1627, 741, 0, 2896, 2335, 1042, 537, 339, 6, 2397, 1227, 5, 0, 238, 189, 1472, 1935, 15, 3780, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 2249, 1876, 3, 0, 0, 0, 0, 847, 1, 528, 20, 1763, 867, 0, 0, 1992, 761, 5, 1662, 0, 0, 0, 0, 1811, 1010, 4, 1363, 0, 0, 0, 0, 2228, 1125, 223, 396, 0, 0, 0, 0, 36, 2560, 642, 1, 566, 131, 0, 0, 3, 151, 268, 1207, 1055, 845, 416, 0, 2127, 23, 2000, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4026, 4420, 4188, 3972, 3936, 3945, 4152] avg=4096.0000 max=4420 max_rank=2 min=3936 min_rank=5 imbalance_ratio=1.079102 max_load_reduction=28348 max_load_reduction_pct=0.865112 moved_to_echo=16005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2279, 2100, 2, 0, 0, 0, 0, 917, 0, 673, 1, 1786, 875, 0, 0, 1776, 542, 5, 1480, 0, 0, 0, 0, 1610, 885, 0, 1279, 0, 0, 0, 0, 1976, 1972, 129, 168, 0, 0, 0, 0, 24, 2403, 978, 0, 504, 116, 0, 0, 0, 365, 169, 1129, 989, 853, 370, 0, 2374, 19, 2019, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4381, 4252, 3803, 3774, 4245, 4025, 3875, 4413] avg=4096.0000 max=4413 max_rank=7 min=3774 min_rank=3 imbalance_ratio=1.077393 max_load_reduction=28355 max_load_reduction_pct=0.865326 moved_to_echo=16558 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 2189, 1810, 12, 0, 0, 0, 0, 964, 4, 461, 11, 1715, 883, 0, 0, 1697, 553, 0, 1246, 0, 0, 0, 0, 1681, 777, 4, 1488, 0, 0, 0, 0, 2322, 1704, 393, 198, 0, 0, 0, 0, 58, 2807, 920, 7, 424, 137, 0, 0, 0, 109, 211, 1236, 1152, 861, 312, 0, 2348, 34, 2038, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4038, 3496, 3950, 4617, 4353, 3881, 4420] avg=4096.0000 max=4617 max_rank=4 min=3496 min_rank=2 imbalance_ratio=1.127197 max_load_reduction=28151 max_load_reduction_pct=0.859100 moved_to_echo=17271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 2210, 1839, 0, 0, 0, 0, 0, 892, 0, 485, 0, 1733, 791, 0, 0, 1377, 962, 7, 1738, 0, 0, 0, 0, 1889, 704, 0, 1335, 0, 0, 0, 0, 2420, 1344, 376, 369, 0, 0, 0, 0, 41, 2849, 909, 10, 591, 142, 0, 0, 0, 358, 243, 1294, 1033, 771, 435, 0, 1770, 24, 1826, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 3901, 4084, 3928, 4509, 4542, 4134, 3620] avg=4096.0000 max=4542 max_rank=5 min=3620 min_rank=7 imbalance_ratio=1.108887 max_load_reduction=28226 max_load_reduction_pct=0.861389 moved_to_echo=16805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 2015, 2157, 7, 0, 0, 0, 0, 813, 0, 738, 2, 1580, 878, 0, 0, 2056, 567, 15, 1760, 0, 0, 0, 0, 2055, 659, 2, 1419, 0, 0, 0, 0, 1896, 1128, 260, 361, 0, 0, 0, 0, 76, 2434, 761, 6, 599, 112, 0, 0, 1, 438, 184, 1118, 1098, 856, 441, 0, 2206, 34, 2027, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 4011, 4398, 4135, 3645, 3988, 4136, 4273] avg=4096.0000 max=4398 max_rank=2 min=3645 min_rank=4 imbalance_ratio=1.073730 max_load_reduction=28370 max_load_reduction_pct=0.865784 moved_to_echo=16042 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2231, 1786, 1, 0, 0, 0, 0, 776, 0, 762, 0, 1755, 922, 0, 0, 1777, 971, 4, 1474, 0, 0, 0, 0, 2270, 664, 0, 1398, 0, 0, 0, 0, 1844, 1689, 173, 286, 0, 0, 0, 0, 23, 2318, 622, 1, 508, 113, 0, 0, 0, 407, 182, 1026, 1089, 899, 374, 0, 2300, 13, 2107, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4215, 4226, 4332, 3992, 3585, 3977, 4423] avg=4096.0000 max=4423 max_rank=7 min=3585 min_rank=5 imbalance_ratio=1.079834 max_load_reduction=28345 max_load_reduction_pct=0.865021 moved_to_echo=15977 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2209, 1792, 4, 0, 0, 0, 0, 919, 1, 648, 0, 1732, 883, 0, 0, 1839, 553, 3, 1438, 0, 0, 0, 0, 1627, 741, 0, 1633, 0, 0, 0, 0, 2205, 1042, 537, 339, 0, 0, 0, 0, 6, 2397, 1227, 5, 489, 130, 0, 0, 0, 238, 189, 1472, 1263, 861, 360, 0, 1935, 15, 2036, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 4183, 3833, 4001, 4123, 4254, 4383, 3986] avg=4096.0000 max=4383 max_rank=6 min=3833 min_rank=2 imbalance_ratio=1.070068 max_load_reduction=28385 max_load_reduction_pct=0.866241 moved_to_echo=16746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 2219, 1770, 0, 0, 0, 0, 0, 956, 0, 581, 1, 1740, 864, 0, 0, 2175, 920, 18, 1395, 0, 0, 0, 0, 2197, 787, 1, 1475, 0, 0, 0, 0, 1828, 1328, 102, 407, 0, 0, 0, 0, 48, 2407, 1039, 9, 475, 107, 0, 0, 0, 376, 311, 1416, 1142, 843, 349, 0, 1456, 30, 1994, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3990, 4142, 4508, 4460, 3665, 4085, 4437, 3481] avg=4096.0000 max=4508 max_rank=2 min=3481 min_rank=7 imbalance_ratio=1.100586 max_load_reduction=28260 max_load_reduction_pct=0.862427 moved_to_echo=15668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7168, 128, 4992, 128, 13824, 7040, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 2560, 1792, 9984, 8832, 6912, 3072, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16640, 256, 16128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 20224, 7168, 128, 4224, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 17664, 15232, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14720, 5888, 128, 12288, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15232, 6272, 128, 11392, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16768, 11392, 2304, 2560, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[18, 0, 4079, 3453, 0, 1, 0, 4081, 241, 2689, 0, 0, 0, 0, 2765, 2, 7, 0, 4095, 0, 6, 3, 0, 4093, 1, 0, 0, 1, 0, 0, 3138, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4055, 4050, 0, 0, 0, 4096, 818, 1971, 0, 0, 0, 1, 3171, 0, 5, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 32, 3, 0, 2283, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[9, 1, 4041, 3796, 0, 0, 1, 4093, 485, 1518, 0, 0, 3, 11, 2632, 3, 0, 0, 4088, 2, 0, 4, 14, 4083, 0, 2, 0, 80, 84, 5, 3723, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[22, 0, 4089, 3986, 1, 0, 0, 4096, 189, 2609, 0, 0, 0, 0, 1563, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 168, 4, 0, 3753, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[83, 0, 4095, 4074, 0, 0, 0, 4096, 140, 2406, 0, 0, 0, 1, 1490, 4, 1, 0, 4095, 0, 0, 0, 0, 4088, 1, 1, 12, 207, 125, 0, 3753, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[3, 0, 4089, 3914, 0, 0, 0, 4096, 188, 2824, 0, 0, 0, 0, 1598, 0, 1, 0, 4094, 0, 0, 0, 2, 4096, 0, 0, 2, 98, 99, 0, 3570, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 4, 4078, 3945, 0, 5, 0, 4087, 380, 1563, 0, 2, 0, 7, 3164, 0, 1, 0, 4075, 0, 22, 0, 11, 4090, 0, 3, 0, 43, 27, 0, 3169, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[44, 0, 4074, 3995, 1, 0, 0, 4096, 418, 1741, 0, 0, 0, 0, 2690, 0, 9, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 32, 6, 0, 3377, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[9, 1, 170, 3796, 0, 0, 0, 0, 0, 0, 1, 4093, 2, 0, 0, 0, 485, 1518, 0, 0, 1571, 0, 0, 0, 3, 11, 2632, 3, 1706, 0, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 4, 14, 4081, 0, 0, 0, 0, 0, 2, 0, 80, 3871, 106, 0, 0, 84, 5, 3723, 707, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3976, 4096, 3574, 4355, 4090, 4099, 4059, 4519] avg=4096.0000 max=4519 max_rank=7 min=3574 min_rank=2 imbalance_ratio=1.103271 max_load_reduction=28249 max_load_reduction_pct=0.862091 moved_to_echo=16767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 172, 3453, 0, 0, 0, 0, 0, 1, 0, 4081, 2, 0, 0, 0, 241, 2689, 0, 0, 1573, 0, 0, 0, 0, 0, 2765, 2, 1708, 0, 0, 0, 7, 0, 4095, 0, 1, 0, 0, 0, 6, 3, 0, 4090, 0, 0, 0, 0, 1, 0, 0, 1, 3907, 107, 0, 0, 0, 0, 3138, 707, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3643, 4084, 4503, 4475, 4103, 4099, 4016, 3845] avg=4096.0000 max=4503 max_rank=2 min=3643 min_rank=0 imbalance_ratio=1.099365 max_load_reduction=28265 max_load_reduction_pct=0.862579 moved_to_echo=16063 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[83, 0, 173, 4074, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 140, 2406, 0, 0, 1573, 0, 0, 0, 0, 1, 1490, 4, 1708, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 1, 1, 12, 207, 3922, 107, 0, 0, 125, 0, 3753, 708, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4330, 4098, 4119, 3203, 4096, 4086, 4250, 4586] avg=4096.0000 max=4586 max_rank=7 min=3203 min_rank=3 imbalance_ratio=1.119629 max_load_reduction=28182 max_load_reduction_pct=0.860046 moved_to_echo=17018 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 172, 3986, 0, 0, 0, 0, 1, 0, 0, 4096, 2, 0, 0, 0, 189, 2609, 0, 0, 1573, 0, 0, 0, 0, 0, 1563, 0, 1708, 0, 0, 0, 1, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 168, 3917, 107, 0, 0, 4, 0, 3753, 707, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4099, 4371, 3271, 4098, 4093, 4192, 4464] avg=4096.0000 max=4464 max_rank=7 min=3271 min_rank=3 imbalance_ratio=1.089844 max_load_reduction=28304 max_load_reduction_pct=0.863770 moved_to_echo=16847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 172, 3945, 0, 0, 0, 0, 0, 5, 0, 4087, 2, 0, 0, 0, 380, 1563, 0, 2, 1571, 0, 0, 0, 0, 7, 3164, 0, 1707, 0, 0, 0, 1, 0, 4075, 0, 0, 0, 0, 0, 22, 0, 11, 4088, 0, 0, 0, 0, 0, 3, 0, 43, 3906, 106, 0, 0, 27, 0, 3169, 708, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4094, 3516, 4878, 4076, 4121, 4058, 3904] avg=4096.0000 max=4878 max_rank=3 min=3516 min_rank=2 imbalance_ratio=1.190918 max_load_reduction=27890 max_load_reduction_pct=0.851135 moved_to_echo=16159 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 168, 4050, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 818, 1971, 0, 0, 1575, 6, 0, 0, 0, 1, 3171, 0, 1711, 0, 0, 0, 5, 0, 4094, 0, 4, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 32, 3887, 110, 0, 0, 3, 0, 2283, 697, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4218, 4100, 4370, 4883, 4103, 4082, 4029, 2983] avg=4096.0000 max=4883 max_rank=3 min=2983 min_rank=7 imbalance_ratio=1.192139 max_load_reduction=27885 max_load_reduction_pct=0.850983 moved_to_echo=15197 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 172, 3914, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 188, 2824, 0, 0, 1572, 0, 0, 0, 0, 0, 1598, 0, 1708, 0, 0, 0, 1, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 2, 98, 3917, 107, 0, 0, 99, 0, 3570, 707, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4098, 4584, 3306, 4096, 4095, 4124, 4376] avg=4096.0000 max=4584 max_rank=2 min=3306 min_rank=3 imbalance_ratio=1.119141 max_load_reduction=28184 max_load_reduction_pct=0.860107 moved_to_echo=16691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[44, 0, 172, 3995, 0, 0, 0, 0, 1, 0, 0, 4096, 2, 0, 0, 0, 418, 1741, 0, 0, 1572, 0, 0, 0, 0, 0, 2690, 0, 1707, 0, 0, 0, 9, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 32, 3902, 106, 0, 0, 6, 0, 3377, 708, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4211, 4099, 3731, 4397, 4106, 4093, 4040, 4091] avg=4096.0000 max=4397 max_rank=3 min=3731 min_rank=2 imbalance_ratio=1.073486 max_load_reduction=28371 max_load_reduction_pct=0.865814 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2944, 17408, 0, 128, 12672, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 19200, 128, 13696, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 26880, 5760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 1408, 31232, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 768, 31232, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4086, 6, 0, 0, 0, 0, 0, 4095, 0, 1, 2, 4094, 0, 0, 0, 400, 0, 0, 0, 0, 4095, 3795, 0, 0, 0, 0, 4002, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 0, 0, 6, 4096, 0, 0, 8, 4095, 0, 0, 0, 144, 0, 0, 0, 0, 4093, 3960, 0, 2, 0, 1, 4078, 1, 4092, 4, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4096, 3, 1, 0, 0, 0, 0, 4095, 0, 18, 0, 4093, 1, 0, 3, 191, 0, 0, 0, 0, 4046, 3946, 0, 0, 0, 0, 4084, 0, 4094, 0, 4096, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4088, 0, 0, 1, 2, 3, 0, 4089, 7, 3, 9, 4092, 6, 1, 0, 37, 0, 2, 0, 0, 4088, 4044, 0, 0, 11, 5, 4079, 22, 4087, 0, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4089, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4095, 0, 0, 2, 90, 0, 0, 0, 0, 4089, 4021, 0, 0, 0, 0, 4093, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4096, 1, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4094, 0, 0, 2, 17, 0, 0, 0, 0, 4095, 4083, 0, 0, 0, 0, 4090, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4096, 0, 0, 1, 296, 0, 0, 0, 0, 4085, 3913, 0, 0, 0, 0, 3998, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4095, 0, 0, 0, 122, 0, 0, 0, 0, 4000, 4074, 0, 0, 0, 0, 4091, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4086, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 400, 3952, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 118, 3795, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 23, 2, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4095, 4093, 4352, 4096, 3913, 4027, 4098] avg=4096.0000 max=4352 max_rank=3 min=3913 min_rank=5 imbalance_ratio=1.062500 max_load_reduction=28416 max_load_reduction_pct=0.867188 moved_to_echo=16134 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 2, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 8, 4090, 0, 0, 0, 0, 0, 0, 0, 144, 3950, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 118, 3960, 0, 2, 0, 0, 0, 0, 0, 1, 4078, 1, 23, 3, 0, 0, 4092, 4, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4102, 4098, 4094, 4091, 4080, 4106, 4098] avg=4096.0000 max=4106 max_rank=6 min=4080 min_rank=5 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 0, 1, 2, 0, 0, 0, 2, 3, 0, 4089, 0, 0, 0, 0, 7, 3, 9, 4088, 0, 0, 0, 0, 6, 1, 0, 37, 3945, 0, 0, 0, 0, 2, 0, 0, 4090, 2, 0, 0, 118, 4044, 0, 0, 0, 0, 0, 0, 11, 5, 4079, 22, 23, 2, 0, 0, 4087, 0, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4094, 4107, 3989, 4094, 4162, 4142, 4089] avg=4096.0000 max=4162 max_rank=5 min=3989 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 3, 1, 0, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 18, 0, 4089, 0, 0, 0, 0, 1, 0, 3, 191, 3904, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 117, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 23, 2, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4095, 4107, 4099, 4098, 4063, 4109, 4095] avg=4096.0000 max=4109 max_rank=6 min=4063 min_rank=5 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 2, 90, 3946, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 118, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 23, 3, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4096, 4091, 4038, 4096, 4139, 4119, 4098] avg=4096.0000 max=4139 max_rank=5 min=4038 min_rank=3 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4090, 0, 0, 0, 0, 0, 0, 2, 17, 3952, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 118, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 23, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 4092, 3971, 4098, 4201, 4115, 4096] avg=4096.0000 max=4201 max_rank=5 min=3971 min_rank=3 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4, 4084, 0, 0, 0, 0, 0, 0, 1, 296, 3946, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 108, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 28, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4094, 4088, 4243, 4099, 4021, 4033, 4096] avg=4096.0000 max=4243 max_rank=3 min=4021 min_rank=5 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16249 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 122, 3860, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 115, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 23, 3, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4092, 3982, 4096, 4189, 4117, 4098] avg=4096.0000 max=4189 max_rank=5 min=3982 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 1408, 31488, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32640, 128, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1024, 31872, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4061, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1385, 1, 4096, 0, 0, 8, 0, 2683, 0, 0, 0, 4088, 0, 62] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 4090, 0, 4, 5, 4091, 4090, 0, 0, 5, 4092, 0, 2, 5, 8, 0, 4094, 2983, 8, 4086, 3, 0, 0, 0, 583, 0, 2, 3, 4091, 0, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2374, 0, 4096, 0, 0, 0, 0, 1200, 0, 0, 0, 4096, 0, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4087, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2251, 0, 4096, 0, 0, 2, 0, 1176, 0, 0, 0, 4092, 0, 680] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1184, 0, 4096, 0, 0, 0, 0, 2905, 0, 0, 0, 4096, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 4047, 4093, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 4094, 1629, 2, 4096, 0, 0, 6, 2, 2458, 0, 0, 5, 4092, 0, 52] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4093, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1152, 0, 4096, 0, 0, 0, 0, 2861, 0, 0, 0, 4095, 0, 87] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4086, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1277, 0, 4096, 0, 0, 0, 0, 2720, 0, 0, 0, 4096, 0, 109] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4061, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 12, 0, 0, 0, 2316, 1385, 1, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 2683, 0, 0, 1768, 253, 0, 0, 0, 3835, 0, 62, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4074, 4096, 4095, 3702, 4104, 4704, 3897] avg=4096.0000 max=4704 max_rank=6 min=3702 min_rank=4 imbalance_ratio=1.148438 max_load_reduction=28064 max_load_reduction_pct=0.856445 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4090, 0, 0, 0, 0, 0, 4, 5, 4091, 13, 0, 0, 0, 0, 0, 0, 5, 4092, 0, 0, 0, 0, 0, 2, 5, 8, 4077, 12, 0, 0, 0, 2315, 2983, 8, 0, 0, 0, 0, 4086, 3, 0, 0, 0, 0, 0, 0, 0, 583, 0, 2, 1767, 253, 0, 0, 3, 3838, 0, 522, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4113, 4097, 4104, 5306, 4089, 2605, 4363] avg=4096.0000 max=5306 max_rank=4 min=2605 min_rank=6 imbalance_ratio=1.295410 max_load_reduction=27462 max_load_reduction_pct=0.838074 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 12, 0, 0, 0, 2316, 2251, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 1176, 0, 0, 1768, 253, 0, 0, 0, 3839, 0, 680, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4100, 4096, 4095, 4567, 4098, 3197, 4519] avg=4096.0000 max=4567 max_rank=4 min=3197 min_rank=6 imbalance_ratio=1.114990 max_load_reduction=28201 max_load_reduction_pct=0.860626 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4047, 13, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 12, 0, 0, 0, 2315, 1629, 2, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 2, 2458, 0, 0, 1767, 253, 0, 0, 5, 3839, 0, 52, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4060, 4099, 4092, 3946, 4102, 4480, 3896] avg=4096.0000 max=4480 max_rank=6 min=3896 min_rank=7 imbalance_ratio=1.093750 max_load_reduction=28288 max_load_reduction_pct=0.863281 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 12, 0, 0, 0, 2316, 1184, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2905, 0, 0, 1768, 253, 0, 0, 0, 3843, 0, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4108, 4096, 4095, 3500, 4096, 4926, 3851] avg=4096.0000 max=4926 max_rank=6 min=3500 min_rank=4 imbalance_ratio=1.202637 max_load_reduction=27842 max_load_reduction_pct=0.849670 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4, 4, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 13, 0, 0, 0, 2312, 2374, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 0, 0, 1771, 257, 5, 0, 0, 3839, 0, 522, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4108, 4092, 4097, 4686, 4087, 3233, 4361] avg=4096.0000 max=4686 max_rank=4 min=3233 min_rank=6 imbalance_ratio=1.144043 max_load_reduction=28082 max_load_reduction_pct=0.856995 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 12, 0, 0, 0, 2316, 1152, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2861, 0, 0, 1768, 253, 0, 0, 0, 3842, 0, 87, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4106, 4096, 4095, 3468, 4096, 4882, 3929] avg=4096.0000 max=4882 max_rank=6 min=3468 min_rank=4 imbalance_ratio=1.191895 max_load_reduction=27886 max_load_reduction_pct=0.851013 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4086, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 12, 0, 0, 0, 2316, 1277, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2720, 0, 0, 1768, 253, 0, 0, 0, 3843, 0, 109, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4099, 4096, 4095, 3593, 4096, 4741, 3952] avg=4096.0000 max=4741 max_rank=6 min=3593 min_rank=4 imbalance_ratio=1.157471 max_load_reduction=28027 max_load_reduction_pct=0.855316 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 18560, 14336, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 16640, 0, 128, 14208, 2048, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 30720, 0, 2048, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[5, 7, 4080, 4088, 0, 2, 15, 1410, 38, 0, 0, 8, 6, 8, 0, 0, 5, 0, 4089, 5, 0, 4092, 0, 2944, 4090, 4081, 6, 1, 1, 0, 3787, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4077, 4096, 8, 0, 0, 2450, 29, 6, 1, 2, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 4093, 3, 2024, 4096, 4096, 0, 0, 0, 0, 3691, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4017, 4096, 0, 0, 0, 3497, 2, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 4078, 6, 824, 4096, 4096, 0, 0, 0, 0, 3941, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4093, 4096, 1, 0, 0, 3406, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 1774, 4096, 4096, 0, 0, 0, 6, 3016, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4096, 1, 0, 0, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2818, 4096, 4095, 0, 0, 0, 2, 3646, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4089, 4096, 2, 0, 0, 1297, 11, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 3161, 4096, 4096, 0, 0, 0, 0, 3735, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4055, 4096, 0, 0, 0, 2936, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 6, 2692, 4096, 4096, 0, 0, 0, 2, 2608, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4083, 4096, 2, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 0, 705, 4096, 4096, 0, 0, 0, 0, 3564, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4017, 22, 0, 0, 0, 0, 0, 0, 0, 3497, 1504, 0, 0, 0, 2, 0, 19, 0, 4074, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 1973, 6, 824, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 594, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 5001, 4102, 4096, 4095, 2804, 4096, 4535] avg=4096.0000 max=5001 max_rank=1 min=2804 min_rank=5 imbalance_ratio=1.220947 max_load_reduction=27767 max_load_reduction_pct=0.847382 moved_to_echo=15530 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 7, 4080, 21, 0, 0, 0, 0, 0, 2, 15, 1410, 1509, 0, 0, 0, 38, 0, 0, 8, 4067, 7, 0, 0, 6, 8, 0, 0, 4088, 0, 0, 0, 5, 0, 4089, 5, 0, 0, 0, 0, 0, 1980, 0, 2944, 0, 0, 0, 0, 2, 4081, 6, 1, 0, 0, 0, 0, 1, 0, 3787, 0, 596, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 2936, 4120, 4102, 4099, 4924, 4090, 4384] avg=4096.0000 max=4924 max_rank=5 min=2936 min_rank=1 imbalance_ratio=1.202148 max_load_reduction=27844 max_load_reduction_pct=0.849731 moved_to_echo=17497 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4077, 22, 0, 0, 0, 0, 8, 0, 0, 2450, 1510, 0, 0, 0, 29, 6, 1, 2, 4074, 7, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1980, 3, 2024, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3691, 0, 596, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 3968, 4119, 4095, 4095, 4007, 4098, 4287] avg=4096.0000 max=4287 max_rank=7 min=3968 min_rank=1 imbalance_ratio=1.046631 max_load_reduction=28481 max_load_reduction_pct=0.869171 moved_to_echo=16487 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 22, 0, 0, 0, 0, 1, 0, 0, 1728, 1511, 0, 0, 0, 0, 0, 0, 0, 4074, 7, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1981, 0, 2818, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 3646, 0, 597, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 3240, 4081, 4094, 4096, 4799, 4097, 4245] avg=4096.0000 max=4799 max_rank=5 min=3240 min_rank=1 imbalance_ratio=1.171631 max_load_reduction=27969 max_load_reduction_pct=0.853546 moved_to_echo=17237 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 15, 0, 0, 0, 0, 1, 0, 0, 3406, 1511, 0, 0, 0, 1, 0, 0, 0, 4081, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1969, 0, 1774, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 3016, 0, 599, 1, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4918, 4090, 4096, 4095, 3743, 4096, 3622] avg=4096.0000 max=4918 max_rank=1 min=3622 min_rank=7 imbalance_ratio=1.200684 max_load_reduction=27850 max_load_reduction_pct=0.849915 moved_to_echo=15556 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4055, 22, 0, 0, 0, 0, 0, 0, 0, 2936, 1506, 0, 0, 0, 1, 0, 0, 0, 4074, 7, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1976, 6, 2692, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 2608, 0, 595, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4442, 4082, 4095, 4096, 4674, 4097, 3205] avg=4096.0000 max=4674 max_rank=5 min=3205 min_rank=7 imbalance_ratio=1.141113 max_load_reduction=28094 max_load_reduction_pct=0.857361 moved_to_echo=16072 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 22, 0, 0, 0, 0, 2, 0, 0, 1297, 1508, 0, 0, 0, 11, 0, 1, 0, 4074, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1977, 0, 3161, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3735, 0, 596, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 2807, 4093, 4096, 4096, 5138, 4096, 4331] avg=4096.0000 max=5138 max_rank=5 min=2807 min_rank=1 imbalance_ratio=1.254395 max_load_reduction=27630 max_load_reduction_pct=0.843201 moved_to_echo=17661 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4083, 22, 0, 0, 0, 0, 2, 0, 0, 3948, 1506, 0, 0, 0, 0, 0, 0, 0, 4074, 7, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1974, 0, 705, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3564, 0, 595, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 5456, 4081, 4094, 4096, 2679, 4098, 4159] avg=4096.0000 max=5456 max_rank=1 min=2679 min_rank=5 imbalance_ratio=1.332031 max_load_reduction=27312 max_load_reduction_pct=0.833496 moved_to_echo=15032 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 15872, 128, 17024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 20736, 12160, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 28032, 0, 4864, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4074, 115, 3049, 4091, 0, 10, 0, 4, 6, 9, 4083, 0, 0, 975, 53, 0, 0, 0, 8, 4, 0, 4079, 7, 0, 4048, 1, 0, 7, 4088, 5, 4048, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4074, 569, 2165, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1440, 14, 0, 0, 3, 0, 0, 0, 4026, 0, 26, 4067, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4090, 370, 3124, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 493, 7, 0, 0, 3, 0, 0, 0, 4081, 0, 134, 4082, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4088, 108, 880, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3154, 9, 0, 0, 0, 0, 0, 0, 4052, 0, 11, 4082, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 116, 2747, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1263, 1, 0, 0, 0, 0, 0, 0, 4087, 0, 7, 4067, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 42, 3008, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 16, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 144, 2373, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1591, 10, 0, 0, 0, 0, 0, 0, 4072, 0, 9, 4089, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4062, 461, 1889, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1822, 3, 0, 0, 0, 0, 0, 0, 4039, 0, 23, 4085, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1450, 569, 2165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1440, 14, 0, 2598, 0, 0, 0, 0, 3, 0, 0, 4091, 2, 1, 0, 0, 4026, 0, 26, 7, 0, 0, 0, 4067, 0, 0, 0, 17, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4184, 4095, 4096, 4052, 4097, 4059, 4084, 4101] avg=4096.0000 max=4184 max_rank=0 min=4052 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16341 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1450, 115, 3049, 0, 0, 0, 0, 0, 0, 10, 0, 4, 4089, 0, 0, 0, 6, 9, 4083, 0, 0, 0, 0, 0, 0, 975, 53, 0, 2598, 0, 0, 0, 0, 0, 8, 4, 4083, 2, 2, 0, 0, 4079, 7, 0, 7, 0, 0, 0, 4048, 1, 0, 7, 17, 0, 0, 0, 5, 5, 4048, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4614, 4103, 4098, 3626, 4099, 4093, 4073, 4062] avg=4096.0000 max=4614 max_rank=0 min=3626 min_rank=3 imbalance_ratio=1.126465 max_load_reduction=28154 max_load_reduction_pct=0.859192 moved_to_echo=16327 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1455, 370, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 493, 7, 0, 2608, 0, 0, 0, 0, 3, 0, 0, 4091, 2, 1, 0, 0, 4081, 0, 134, 7, 0, 0, 0, 4082, 0, 0, 0, 18, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4949, 4095, 4096, 3108, 4097, 4222, 4100, 4101] avg=4096.0000 max=4949 max_rank=0 min=3108 min_rank=3 imbalance_ratio=1.208252 max_load_reduction=27819 max_load_reduction_pct=0.848969 moved_to_echo=16520 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1454, 108, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3154, 9, 0, 2607, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 1, 0, 0, 4052, 0, 11, 7, 0, 0, 0, 4082, 0, 0, 0, 18, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2442, 4095, 4096, 5770, 4094, 4070, 4100, 4101] avg=4096.0000 max=5770 max_rank=3 min=2442 min_rank=0 imbalance_ratio=1.408691 max_load_reduction=26998 max_load_reduction_pct=0.823914 moved_to_echo=16365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1457, 42, 3008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1092, 0, 0, 2612, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 1, 0, 0, 4034, 0, 16, 7, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4507, 4095, 4096, 3704, 4094, 4057, 4114, 4101] avg=4096.0000 max=4507 max_rank=0 min=3704 min_rank=3 imbalance_ratio=1.100342 max_load_reduction=28261 max_load_reduction_pct=0.862457 moved_to_echo=16366 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1439, 144, 2373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1591, 10, 0, 2616, 0, 0, 0, 0, 0, 0, 0, 4092, 6, 1, 0, 0, 4072, 0, 9, 14, 2, 0, 0, 4089, 0, 0, 0, 21, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3956, 4095, 4094, 4217, 4099, 4097, 4110, 4100] avg=4096.0000 max=4217 max_rank=3 min=3956 min_rank=0 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1457, 116, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1263, 1, 0, 2612, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 1, 0, 0, 4087, 0, 7, 7, 0, 0, 0, 4067, 0, 0, 0, 18, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4320, 4095, 4096, 3876, 4094, 4101, 4085, 4101] avg=4096.0000 max=4320 max_rank=0 min=3876 min_rank=3 imbalance_ratio=1.054688 max_load_reduction=28448 max_load_reduction_pct=0.868164 moved_to_echo=16381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1446, 461, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1822, 3, 0, 2590, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 1, 0, 0, 4039, 0, 23, 7, 0, 0, 0, 4085, 0, 0, 0, 17, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3796, 4095, 4096, 4415, 4094, 4069, 4102, 4101] avg=4096.0000 max=4415 max_rank=3 min=3796 min_rank=0 imbalance_ratio=1.077881 max_load_reduction=28353 max_load_reduction_pct=0.865265 moved_to_echo=16366 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11648, 2048, 19328, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 11904, 128, 0, 20864, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32512, 128, 256, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 0, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 13, 0, 4069, 30, 4096, 0, 4075, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 9, 0, 4096, 4, 4096, 0, 4079, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4087, 6, 0, 8, 4065, 1, 4, 4090, 4088, 4089, 0, 0, 0, 0, 26, 0, 1, 4087, 0, 3966, 8, 4094, 7, 5, 8, 0, 5, 116, 2, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 6, 0, 4096, 14, 4096, 0, 4075, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 4096, 0, 0, 4088, 4096, 4094, 0, 0, 1, 0, 0, 8, 0, 4042, 88, 4096, 0, 4061, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4053, 5, 4088, 0, 4096, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4067, 2, 4093, 0, 4089, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 12, 0, 4096, 0, 4096, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4088, 0, 0, 0, 0, 4069, 30, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 11, 0, 0, 0, 0, 0, 0, 8, 4085, 5, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4094, 4101, 4099, 4086, 4098, 4098] avg=4096.0000 max=4101 max_rank=3 min=4086 min_rank=5 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4087, 6, 0, 8, 0, 0, 0, 0, 4065, 1, 4, 5, 0, 0, 0, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4082, 0, 0, 0, 1, 4087, 0, 4, 0, 0, 0, 0, 8, 4094, 7, 5, 10, 0, 0, 0, 8, 0, 5, 116, 3952, 2, 0, 0, 2, 5, 0, 0, 4086, 1, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4075, 4091, 4108, 4092, 4124, 4083, 4094] avg=4096.0000 max=4124 max_rank=5 min=4075 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 4080, 0, 0, 0, 0, 4042, 88, 3, 0, 0, 0, 0, 0, 4061, 0, 0, 11, 0, 0, 0, 0, 0, 0, 1, 4082, 2, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4101, 4096, 4089, 4133, 4072, 4085, 4095] avg=4096.0000 max=4133 max_rank=4 min=4072 min_rank=5 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4088, 0, 0, 0, 0, 4096, 4, 3, 0, 0, 0, 0, 0, 4079, 0, 0, 11, 0, 0, 0, 0, 0, 0, 5, 4082, 2, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4097, 4097, 4103, 4090, 4089, 4095] avg=4096.0000 max=4103 max_rank=4 min=4089 min_rank=6 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4053, 5, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 0, 46, 4074, 2, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4098, 4088, 4061, 4107, 4122, 4095] avg=4096.0000 max=4122 max_rank=6 min=4061 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4091, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 13, 6, 0, 0, 0, 0, 0, 1, 4083, 4, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4096, 4096, 4097, 4110, 4094, 4088, 4097] avg=4096.0000 max=4110 max_rank=4 min=4088 min_rank=6 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4067, 2, 2, 0, 0, 0, 0, 0, 4089, 0, 0, 11, 0, 0, 0, 0, 0, 0, 37, 4080, 2, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4098, 4088, 4071, 4100, 4119, 4099] avg=4096.0000 max=4119 max_rank=6 min=4071 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4088, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 4084, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4098, 4100, 4099, 4095, 4084, 4095] avg=4096.0000 max=4101 max_rank=1 min=4084 min_rank=6 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8, 6, 5, 4089, 3, 4, 7, 0, 0, 9, 4087, 0, 0, 5, 4079, 4085, 0, 6, 0, 0, 4087, 4088, 7, 0, 4078, 4089, 11, 0, 8, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 4096, 4096, 0, 0, 4086, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 4096, 0, 0, 0, 16, 4096, 4096, 0, 0, 4080, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4082, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 19, 0, 0, 0, 1, 0, 4077, 0, 0, 0, 4096, 4093, 5, 0, 0, 0, 4096, 4096, 0, 0, 4095, 4095, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4090, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 4094, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 6, 5, 4082, 4, 1, 0, 4083, 3, 4, 7, 0, 0, 0, 0, 0, 0, 9, 4087, 2, 0, 0, 0, 0, 0, 5, 4079, 1, 0, 0, 0, 4085, 0, 6, 0, 0, 0, 0, 0, 0, 4086, 0, 7, 0, 0, 0, 0, 0, 4078, 7, 11, 0, 0, 0, 0, 0, 8, 0, 7, 4087, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4097, 4098, 4085, 4091, 4093, 4096, 4102] avg=4096.0000 max=4106 max_rank=0 min=4085 min_rank=3 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 4, 2, 0, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 16, 4095, 0, 0, 0, 0, 0, 0, 0, 4080, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4095, 4093, 4097, 4096, 4111, 4088, 4094] avg=4096.0000 max=4111 max_rank=5 min=4088 min_rank=6 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 4, 1, 0, 4089, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 4086, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4098, 4099, 4097, 4096, 4096, 4094, 4095] avg=4096.0000 max=4099 max_rank=2 min=4093 min_rank=0 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 4, 1, 0, 4086, 19, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4077, 2, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4105, 4080, 4097, 4098, 4095, 4102, 4098] avg=4096.0000 max=4105 max_rank=1 min=4080 min_rank=2 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 4, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4089, 4099, 4097, 4096, 4095, 4104, 4095] avg=4096.0000 max=4104 max_rank=6 min=4089 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16390 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 7, 2, 0, 4084, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 1, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4098, 4102, 4101, 4098, 4088, 4084, 4094] avg=4096.0000 max=4103 max_rank=0 min=4084 min_rank=6 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 4, 1, 0, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4090, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4095, 4099, 4097, 4096, 4095, 4098, 4095] avg=4096.0000 max=4099 max_rank=2 min=4093 min_rank=0 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 4, 1, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4091, 4098, 4097, 4097, 4095, 4102, 4095] avg=4096.0000 max=4102 max_rank=6 min=4091 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 613, 4096, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 4091, 0, 0, 4096, 3486, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4, 1590, 4088, 0, 4, 7, 8, 3, 16, 0, 4087, 8, 9, 0, 0, 0, 4089, 0, 16, 6, 0, 3, 6, 4066, 4090, 0, 6, 4088, 2494, 0, 4080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 130, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 4096, 3975, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 483, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4036, 4096, 0, 0, 4096, 3673, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2098, 4077, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 0, 4096, 2028, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 484, 4090, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3918, 4091, 0, 0, 4096, 3802, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 954, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 4096, 3148, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 907, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 4096, 3196, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 613, 3192, 0, 0, 0, 0, 0, 0, 0, 0, 3188, 904, 2, 0, 0, 0, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 36, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4092, 38, 0, 0, 0, 0, 0, 0, 0, 3486, 0, 872, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3806, 4094, 4099, 4090, 4096, 4095, 4130, 4358] avg=4096.0000 max=4358 max_rank=7 min=3806 min_rank=0 imbalance_ratio=1.063965 max_load_reduction=28410 max_load_reduction_pct=0.867004 moved_to_echo=16679 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 1590, 3185, 0, 0, 0, 0, 0, 4, 7, 8, 3175, 903, 2, 0, 3, 16, 0, 4085, 0, 0, 0, 0, 8, 9, 0, 0, 4052, 36, 0, 0, 0, 4089, 0, 16, 0, 0, 0, 0, 6, 0, 3, 6, 4088, 0, 0, 0, 4066, 38, 0, 6, 0, 0, 0, 0, 0, 2494, 0, 869, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4779, 4099, 4104, 4105, 4105, 4103, 4110, 3363] avg=4096.0000 max=4779 max_rank=0 min=3363 min_rank=7 imbalance_ratio=1.166748 max_load_reduction=27989 max_load_reduction_pct=0.854156 moved_to_echo=15681 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 484, 3187, 0, 0, 0, 0, 0, 0, 0, 0, 3188, 903, 2, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 36, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3918, 38, 0, 0, 0, 0, 0, 0, 0, 3802, 0, 872, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3671, 4093, 4096, 4090, 4093, 4095, 3956, 4674] avg=4096.0000 max=4674 max_rank=7 min=3671 min_rank=0 imbalance_ratio=1.141113 max_load_reduction=28094 max_load_reduction_pct=0.857361 moved_to_echo=16818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 130, 3192, 0, 0, 0, 0, 0, 0, 0, 0, 3188, 904, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 36, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4086, 38, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 872, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3323, 4094, 4094, 4095, 4096, 4095, 4124, 4847] avg=4096.0000 max=4847 max_rank=7 min=3323 min_rank=0 imbalance_ratio=1.183350 max_load_reduction=27921 max_load_reduction_pct=0.852081 moved_to_echo=17162 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 483, 3187, 0, 0, 0, 0, 0, 0, 0, 0, 3192, 909, 3, 6, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 39, 1, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4036, 33, 0, 0, 0, 0, 0, 0, 0, 3673, 0, 865, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3670, 4110, 4093, 4103, 4090, 4095, 4069, 4538] avg=4096.0000 max=4538 max_rank=7 min=3670 min_rank=0 imbalance_ratio=1.107910 max_load_reduction=28230 max_load_reduction_pct=0.861511 moved_to_echo=16792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 907, 3191, 0, 0, 0, 0, 0, 0, 0, 0, 3188, 904, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 36, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4090, 38, 0, 0, 0, 0, 0, 0, 0, 3196, 0, 872, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4094, 4094, 4095, 4096, 4095, 4128, 4068] avg=4096.0000 max=4128 max_rank=6 min=4068 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 954, 3192, 0, 0, 0, 0, 0, 0, 0, 0, 3188, 904, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 36, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4090, 38, 0, 0, 0, 0, 0, 0, 0, 3148, 0, 872, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4094, 4094, 4095, 4096, 4095, 4128, 4020] avg=4096.0000 max=4146 max_rank=0 min=4020 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16339 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2098, 3177, 0, 0, 0, 0, 0, 0, 0, 0, 3188, 900, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 36, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4085, 38, 0, 0, 0, 0, 0, 0, 0, 2028, 0, 872, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5275, 4090, 4094, 4095, 4096, 4095, 4123, 2900] avg=4096.0000 max=5275 max_rank=0 min=2900 min_rank=7 imbalance_ratio=1.287842 max_load_reduction=27493 max_load_reduction_pct=0.839020 moved_to_echo=15214 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 25600, 7296, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 7296, 25600, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 25856, 0, 7040, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 384, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 32512, 384, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4085, 0, 11, 4084, 0, 4087, 4079, 0, 16, 0, 9, 4086, 5, 8, 0, 7, 3897, 8, 0, 8, 0, 0, 7, 1, 2, 4077, 119, 4087, 0, 85, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 8, 0, 4096, 0, 4096, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4003, 0, 22, 0, 0, 0, 0, 0, 0, 4096, 70, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 4096, 0, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3972, 0, 39, 0, 1, 0, 0, 0, 0, 4096, 85, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 22, 0, 4096, 0, 4096, 4074, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3532, 0, 119, 0, 1, 0, 0, 0, 0, 4095, 445, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4019, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 59, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 21, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3990, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 83, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 117, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3814, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 267, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4079, 0, 11, 0, 0, 0, 0, 7, 0, 0, 4079, 0, 0, 0, 0, 0, 16, 0, 9, 4077, 3, 1, 0, 4086, 5, 8, 0, 0, 0, 0, 0, 7, 3897, 8, 0, 149, 9, 6, 0, 8, 0, 0, 7, 4086, 0, 0, 0, 1, 2, 3925, 119, 0, 0, 0, 0, 4078, 0, 85, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4086, 4106, 4099, 4076, 4101, 4047, 4163] avg=4096.0000 max=4163 max_rank=7 min=4047 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 8, 0, 0, 0, 0, 0, 7, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 22, 149, 9, 6, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3944, 70, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4093, 4096, 4189, 4095, 4014, 4087] avg=4096.0000 max=4189 max_rank=4 min=4014 min_rank=6 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 22, 0, 0, 0, 0, 0, 7, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 0, 119, 149, 9, 6, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 3943, 445, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4081, 4093, 4096, 3815, 4096, 4388, 4087] avg=4096.0000 max=4388 max_rank=6 min=3815 min_rank=4 imbalance_ratio=1.071289 max_load_reduction=28380 max_load_reduction_pct=0.866089 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 2, 0, 0, 0, 0, 0, 7, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 39, 149, 9, 6, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 3944, 85, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4100, 4093, 4096, 4175, 4096, 4029, 4087] avg=4096.0000 max=4175 max_rank=4 min=4029 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 1, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 149, 9, 6, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3944, 117, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4103, 4093, 4096, 4142, 4095, 4061, 4087] avg=4096.0000 max=4142 max_rank=4 min=4061 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 8, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 18, 155, 13, 12, 3, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3933, 59, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4097, 4104, 4093, 4220, 4095, 3992, 4083] avg=4096.0000 max=4220 max_rank=4 min=3992 min_rank=6 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16390 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 21, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 3, 149, 9, 6, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3943, 83, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4103, 4093, 4096, 4157, 4095, 4026, 4087] avg=4096.0000 max=4157 max_rank=4 min=4026 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3814, 0, 16, 149, 9, 6, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3944, 267, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4102, 4093, 4096, 3994, 4095, 4211, 4087] avg=4096.0000 max=4211 max_rank=6 min=3994 min_rank=4 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 31232, 128, 256, 1280, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 31616, 1280, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 26, 0, 4096, 0, 0, 0, 16, 116, 0, 4096, 4096, 0, 0, 0, 1317, 0, 0, 4080, 4096, 0, 0, 4096, 3234, 0, 3499, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 18, 15, 0, 4095, 4096, 0, 0, 0, 555, 0, 0, 4095, 4095, 0, 1, 4096, 3754, 0, 3851, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 11, 0, 4096, 4096, 0, 0, 0, 1158, 0, 0, 4086, 4096, 0, 0, 4096, 3921, 0, 3102, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[11, 0, 0, 0, 8, 0, 4, 0, 4086, 0, 0, 10, 0, 144, 9, 4085, 4010, 12, 4, 9, 142, 0, 15, 4084, 4088, 9, 0, 4075, 3879, 0, 4070, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 217, 0, 4096, 4096, 0, 0, 0, 549, 0, 0, 4088, 4096, 0, 0, 4096, 3857, 0, 3569, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 348, 0, 4096, 4096, 0, 0, 0, 437, 0, 0, 4090, 4096, 0, 0, 4096, 3555, 0, 3850, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 41, 0, 4096, 4033, 0, 0, 0, 1150, 0, 0, 4096, 4096, 0, 0, 4096, 3793, 0, 3271, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 41, 0, 4096, 0, 0, 0, 2, 327, 0, 4096, 4096, 0, 0, 0, 1437, 0, 0, 4088, 4096, 0, 0, 4096, 3599, 0, 2794, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 26, 0, 2714, 835, 142, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 16, 116, 0, 3939, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 1317, 0, 0, 3245, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 520, 0, 3499, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3717, 4096, 4071, 4111, 4562, 4096, 4019] avg=4096.0000 max=4562 max_rank=5 min=3717 min_rank=1 imbalance_ratio=1.113770 max_load_reduction=28206 max_load_reduction_pct=0.860779 moved_to_echo=16788 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 0, 0, 4086, 1, 0, 0, 8, 0, 4, 0, 3255, 836, 141, 0, 4086, 0, 0, 10, 0, 0, 0, 0, 0, 144, 9, 3929, 0, 0, 0, 0, 4010, 12, 4, 9, 15, 0, 0, 0, 142, 0, 15, 3248, 0, 0, 0, 0, 2, 9, 0, 4075, 0, 0, 0, 0, 623, 0, 4070, 14, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4244, 4096, 4082, 4050, 3405, 4086, 4707] avg=4096.0000 max=4707 max_rank=7 min=3405 min_rank=5 imbalance_ratio=1.149170 max_load_reduction=28061 max_load_reduction_pct=0.856354 moved_to_echo=16248 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 1, 0, 3150, 838, 142, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 18, 15, 0, 3938, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 555, 0, 0, 3257, 0, 0, 0, 0, 2, 0, 1, 4096, 0, 0, 0, 0, 603, 0, 3851, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4131, 4096, 3971, 4111, 3812, 4099, 4454] avg=4096.0000 max=4454 max_rank=7 min=3812 min_rank=5 imbalance_ratio=1.087402 max_load_reduction=28314 max_load_reduction_pct=0.864075 moved_to_echo=16476 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 8, 0, 2983, 837, 142, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 3939, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 437, 0, 0, 3253, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 571, 0, 3850, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 3970, 4096, 4287, 4111, 3690, 4098, 4421] avg=4096.0000 max=4421 max_rank=7 min=3690 min_rank=5 imbalance_ratio=1.079346 max_load_reduction=28347 max_load_reduction_pct=0.865082 moved_to_echo=16320 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 3183, 838, 142, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 3939, 0, 0, 0, 0, 4033, 0, 0, 0, 15, 0, 0, 0, 1150, 0, 0, 3258, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 609, 0, 3271, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4163, 4096, 3980, 4048, 4408, 4098, 3880] avg=4096.0000 max=4408 max_rank=5 min=3880 min_rank=7 imbalance_ratio=1.076172 max_load_reduction=28360 max_load_reduction_pct=0.865479 moved_to_echo=16434 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 8, 0, 3240, 842, 145, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 3932, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 549, 0, 0, 3246, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 614, 0, 3569, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4235, 4096, 4149, 4115, 3795, 4096, 4183] avg=4096.0000 max=4235 max_rank=1 min=3795 min_rank=5 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16189 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 3290, 836, 142, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 11, 0, 3939, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 1158, 0, 0, 3250, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 630, 0, 3102, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4268, 4096, 3960, 4111, 4408, 4097, 3732] avg=4096.0000 max=4408 max_rank=5 min=3732 min_rank=7 imbalance_ratio=1.076172 max_load_reduction=28360 max_load_reduction_pct=0.865479 moved_to_echo=16348 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 41, 0, 3020, 837, 142, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 327, 0, 3939, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 1437, 0, 0, 3251, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 578, 0, 2794, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4040, 4096, 4268, 4111, 4688, 4098, 3372] avg=4096.0000 max=4688 max_rank=5 min=3372 min_rank=7 imbalance_ratio=1.144531 max_load_reduction=28080 max_load_reduction_pct=0.856934 moved_to_echo=16269 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 24960, 6784, 1152, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6784, 0, 128, 26112, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4864, 0, 28032, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 1280, 128, 31616, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[3, 4083, 4069, 5, 0, 0, 14, 3985, 10, 2, 0, 4084, 0, 11, 0, 99, 7, 4, 0, 41, 0, 10, 4084, 9, 4078, 7, 3, 12, 4075, 0, 4061, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4094, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4081, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4107, 4096, 4078, 4092, 4096, 4096, 4105] avg=4096.0000 max=4107 max_rank=1 min=4078 min_rank=3 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16389 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 7, 4069, 5, 0, 0, 0, 0, 0, 0, 14, 3985, 11, 0, 0, 0, 10, 2, 0, 4084, 0, 0, 0, 0, 0, 11, 0, 99, 4065, 1, 0, 0, 7, 4, 0, 41, 4071, 0, 0, 0, 0, 10, 4084, 9, 0, 0, 0, 0, 4078, 7, 3, 12, 0, 0, 0, 0, 3, 0, 4061, 12, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4010, 4096, 4176, 4123, 4103, 4100, 4076] avg=4096.0000 max=4176 max_rank=3 min=4010 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16402 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 15, 4083, 4, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4107, 4096, 4102, 4092, 4096, 4096, 4096] avg=4096.0000 max=4107 max_rank=1 min=4083 min_rank=0 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16380 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 4077, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4107, 4096, 4081, 4092, 4096, 4096, 4099] avg=4096.0000 max=4107 max_rank=1 min=4081 min_rank=3 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16383 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 4077, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4099, 4096, 4086, 4092, 4096, 4096, 4099] avg=4096.0000 max=4104 max_rank=0 min=4086 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16383 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 7, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4125, 4096, 4082, 4092, 4089, 4092, 4096] avg=4096.0000 max=4125 max_rank=1 min=4082 min_rank=3 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16369 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 4077, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4106, 4096, 4081, 4093, 4096, 4096, 4098] avg=4096.0000 max=4106 max_rank=1 min=4081 min_rank=3 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16383 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 4077, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4107, 4096, 4082, 4092, 4096, 4096, 4099] avg=4096.0000 max=4107 max_rank=1 min=4082 min_rank=3 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16383 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[214, 0, 9, 9, 21, 3864, 4074, 4088, 12, 10, 0, 14, 7, 9, 4068, 0, 0, 4082, 0, 4078, 0, 2, 0, 11, 0, 11, 3, 4087, 14, 0, 1, 4080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4017, 4096, 4091, 0, 0, 15, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 69, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4031, 4096, 4091, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 2, 65, 0, 0, 0, 4094, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4090, 4096, 4096, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4084, 4096, 4096, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4017, 4096, 4093, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 79, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[14, 0, 0, 0, 0, 4081, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 4087, 4096, 4096, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[214, 0, 9, 9, 4013, 32, 0, 0, 21, 3864, 60, 0, 0, 0, 0, 0, 12, 10, 0, 14, 4081, 0, 0, 0, 7, 9, 4068, 0, 1, 0, 0, 0, 0, 3, 0, 4078, 0, 0, 0, 0, 0, 2, 0, 11, 4047, 7, 0, 0, 0, 11, 3, 4087, 0, 0, 0, 0, 14, 0, 1, 4080, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4277, 3945, 4117, 4085, 4081, 4067, 4101, 4095] avg=4096.0000 max=4277 max_rank=0 min=3945 min_rank=1 imbalance_ratio=1.044189 max_load_reduction=28491 max_load_reduction_pct=0.869476 moved_to_echo=16344 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4035, 34, 0, 0, 0, 4017, 60, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4085, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 69, 4062, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4077, 4100, 4097, 4096, 4137, 4096, 4096] avg=4096.0000 max=4137 max_rank=5 min=4069 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16425 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4035, 32, 0, 0, 0, 4031, 60, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4084, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 65, 4061, 7, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4091, 4089, 4097, 4099, 4135, 4094, 4096] avg=4096.0000 max=4135 max_rank=5 min=4067 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4035, 32, 0, 0, 0, 4017, 60, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 79, 4061, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4077, 4090, 4097, 4099, 4146, 4096, 4096] avg=4096.0000 max=4146 max_rank=5 min=4067 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 0, 0, 4035, 32, 0, 0, 0, 4081, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4061, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4141, 4088, 4097, 4099, 4070, 4096, 4096] avg=4096.0000 max=4141 max_rank=1 min=4070 min_rank=5 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16361 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4037, 35, 0, 0, 0, 4090, 56, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 3, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4061, 7, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4146, 4094, 4101, 4096, 4069, 4093, 4097] avg=4096.0000 max=4146 max_rank=1 min=4069 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16355 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4035, 32, 0, 0, 0, 4084, 60, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4061, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4144, 4095, 4097, 4099, 4074, 4096, 4096] avg=4096.0000 max=4144 max_rank=1 min=4067 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4035, 32, 0, 0, 0, 4087, 60, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4061, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4147, 4095, 4097, 4099, 4070, 4096, 4096] avg=4096.0000 max=4147 max_rank=1 min=4068 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16361 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32384, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 0, 128, 128, 32384, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 256, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 3495, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4095, 0, 0, 0, 597, 4095, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3822, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 274, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[3, 0, 0, 11, 11, 0, 4087, 10, 10, 14, 3169, 4085, 2, 6, 4080, 3, 0, 0, 0, 13, 0, 7, 0, 4086, 4072, 5, 9, 11, 906, 4086, 15, 4067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3281, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 4096, 0, 0, 0, 816, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3049, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 4094, 0, 0, 0, 1045, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2844, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1252, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4055, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 4096, 0, 0, 0, 38, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3997, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 99, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 3466, 627, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3495, 630, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 597, 0, 0, 3469, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4125, 4096, 4095, 4097, 4095, 4066] avg=4096.0000 max=4125 max_rank=2 min=4066 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 11, 3457, 622, 1, 0, 11, 0, 4087, 10, 0, 0, 0, 0, 10, 14, 3169, 628, 0, 0, 0, 0, 2, 6, 4080, 3, 0, 0, 0, 0, 0, 0, 0, 13, 4085, 0, 0, 0, 0, 7, 0, 4085, 0, 0, 0, 0, 4072, 5, 9, 11, 0, 0, 0, 0, 906, 1, 15, 3445, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4108, 3821, 4091, 4098, 4092, 4097, 4367] avg=4096.0000 max=4367 max_rank=7 min=3821 min_rank=2 imbalance_ratio=1.066162 max_load_reduction=28401 max_load_reduction_pct=0.866730 moved_to_echo=16654 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3461, 627, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3281, 629, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 816, 0, 0, 3469, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4096, 3910, 4096, 4096, 4100, 4096, 4285] avg=4096.0000 max=4285 max_rank=7 min=3910 min_rank=2 imbalance_ratio=1.046143 max_load_reduction=28483 max_load_reduction_pct=0.869232 moved_to_echo=16577 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3466, 627, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2844, 630, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1252, 0, 0, 3469, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 3474, 4096, 4096, 4095, 4096, 4721] avg=4096.0000 max=4721 max_rank=7 min=3474 min_rank=2 imbalance_ratio=1.152588 max_load_reduction=28047 max_load_reduction_pct=0.855927 moved_to_echo=17008 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3463, 627, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4055, 629, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 3469, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4096, 4684, 4096, 4096, 4102, 4096, 3507] avg=4096.0000 max=4684 max_rank=2 min=3507 min_rank=7 imbalance_ratio=1.143555 max_load_reduction=28084 max_load_reduction_pct=0.857056 moved_to_echo=15801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3470, 629, 8, 5, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 3822, 626, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 274, 0, 0, 3467, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4084, 4448, 4101, 4096, 4088, 4098, 3741] avg=4096.0000 max=4448 max_rank=2 min=3741 min_rank=7 imbalance_ratio=1.085938 max_load_reduction=28320 max_load_reduction_pct=0.864258 moved_to_echo=16023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3466, 627, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3049, 630, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1045, 0, 0, 3469, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 3679, 4096, 4096, 4099, 4094, 4514] avg=4096.0000 max=4514 max_rank=7 min=3679 min_rank=2 imbalance_ratio=1.102051 max_load_reduction=28254 max_load_reduction_pct=0.862244 moved_to_echo=16803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3466, 627, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3997, 630, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 99, 1, 0, 3469, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4627, 4096, 4095, 4095, 4096, 3569] avg=4096.0000 max=4627 max_rank=2 min=3569 min_rank=7 imbalance_ratio=1.129639 max_load_reduction=28141 max_load_reduction_pct=0.858795 moved_to_echo=15855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32766 actual_valid_recv=33536 observed_loss=-770 tokens_per_expert=[128, 0, 128, 128, 27776, 5120, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32770 actual_valid_recv=33152 observed_loss=-382 tokens_per_expert=[5120, 128, 128, 27776, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 27776, 5120, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4070, 0, 0, 0, 1, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 26, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4086, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 10, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4073, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 4096, 0, 38, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 0, 0, 0, 0, 4096, 4092, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 23, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[10, 0, 0, 4084, 0, 9, 0, 0, 10, 4086, 13, 12, 4084, 12, 8, 0, 6, 4077, 4088, 12, 0, 4080, 0, 8, 0, 4067, 10, 4086, 0, 0, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4078, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 29, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4064, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 32, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 6, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 11, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 5, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 23, 0, 0, 4075, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4092, 4093, 4093, 4096, 4097, 4101, 4098] avg=4096.0000 max=4101 max_rank=6 min=4092 min_rank=1 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4073, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 5, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 38, 0, 0, 4079, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4096, 4093, 4089, 4096, 4097, 4096, 4117] avg=4096.0000 max=4117 max_rank=7 min=4084 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 4084, 11, 0, 0, 0, 0, 9, 0, 0, 4088, 0, 0, 0, 10, 4084, 13, 12, 0, 0, 0, 0, 4084, 12, 8, 0, 5, 2, 0, 0, 6, 4077, 0, 12, 0, 0, 0, 0, 0, 4080, 0, 8, 0, 0, 0, 0, 0, 4067, 10, 4, 0, 0, 0, 0, 0, 0, 0, 6, 4066, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4097, 4119, 4111, 4095, 4088, 4081, 4072] avg=4096.0000 max=4119 max_rank=2 min=4072 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4086, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 10, 0, 0, 4075, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4093, 4103, 4096, 4097, 4101, 4085] avg=4096.0000 max=4103 max_rank=3 min=4085 min_rank=7 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4078, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 5, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 4075, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4091, 4093, 4096, 4098, 4096, 4104] avg=4096.0000 max=4104 max_rank=7 min=4091 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4064, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 32, 0, 0, 4075, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4096, 4093, 4103, 4096, 4097, 4101, 4107] avg=4096.0000 max=4107 max_rank=7 min=4075 min_rank=0 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 7, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4078, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4096, 4093, 4099, 4096, 4097, 4096, 4084] avg=4096.0000 max=4107 max_rank=0 min=4084 min_rank=7 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 5, 2, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 4075, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4099, 4093, 4077, 4097, 4097, 4096, 4101] avg=4096.0000 max=4108 max_rank=0 min=4077 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 256, 0, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4085, 4085, 0, 44, 17, 1, 4082, 0, 0, 0, 0, 11, 0, 9, 14, 4074, 9, 12, 0, 0, 16, 0, 0, 0, 4044, 4085, 11, 0, 6, 4084, 4079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4091, 4096, 4092, 4096, 4093, 4103, 4098] avg=4096.0000 max=4103 max_rank=6 min=4091 min_rank=1 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4085, 0, 0, 0, 0, 0, 44, 17, 1, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 11, 0, 9, 14, 4077, 5, 0, 0, 4074, 9, 12, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4080, 2, 0, 0, 0, 4044, 6, 11, 0, 0, 0, 0, 0, 6, 2, 4079, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4139, 4084, 4116, 4095, 4098, 4061, 4087] avg=4096.0000 max=4139 max_rank=1 min=4061 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4091, 2, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4091, 4096, 4092, 4096, 4096, 4103, 4098] avg=4096.0000 max=4103 max_rank=6 min=4091 min_rank=1 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4091, 4096, 4092, 4096, 4093, 4103, 4098] avg=4096.0000 max=4103 max_rank=6 min=4091 min_rank=1 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4091, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4091, 4101, 4092, 4090, 4107, 4096, 4096] avg=4096.0000 max=4107 max_rank=5 min=4090 min_rank=4 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4083, 4101, 4100, 4103, 4093, 4096, 4096] avg=4096.0000 max=4103 max_rank=4 min=4083 min_rank=1 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4091, 4098, 4092, 4096, 4095, 4103, 4097] avg=4096.0000 max=4103 max_rank=6 min=4091 min_rank=1 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4091, 4096, 4092, 4096, 4093, 4103, 4098] avg=4096.0000 max=4103 max_rank=6 min=4091 min_rank=1 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-08-30 04:56:47] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 682.6 | throughput per GPU (TFLOP/s/GPU): 488.4 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.039475E+01 | loss scale: 1.0 | grad norm: 32.041 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2013, 0, 597, 0, 572, 0, 702, 3684, 0, 3036, 587, 1370, 1, 3614, 3301, 3097, 130, 0, 4, 1853, 2383, 1, 0, 41, 521, 1546, 757, 4, 2953, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 2, 1835, 0, 647, 1, 548, 2, 917, 3619, 2, 3290, 762, 1739, 4, 3691, 3156, 2648, 103, 0, 0, 1702, 2229, 4, 3, 71, 493, 1944, 664, 3, 2687, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 1668, 10, 644, 6, 612, 0, 908, 3276, 2, 3115, 1082, 1763, 1, 3624, 2410, 3126, 153, 0, 7, 1829, 2200, 50, 17, 26, 419, 1890, 1286, 19, 2588, 37] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3, 1, 1908, 2, 744, 0, 606, 2, 918, 3537, 0, 3308, 798, 1787, 1, 3607, 2759, 2998, 108, 0, 2, 2228, 1674, 0, 2, 18, 639, 1632, 753, 13, 2720, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2101, 0, 682, 0, 393, 1, 443, 3503, 0, 3070, 994, 1963, 0, 3612, 3326, 2587, 121, 0, 0, 1714, 2145, 1, 1, 24, 555, 1703, 1111, 18, 2700, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[12, 0, 2259, 0, 1219, 0, 576, 5, 904, 3051, 0, 2360, 1025, 1593, 0, 3540, 2988, 2999, 482, 0, 1, 1411, 2207, 0, 0, 127, 560, 1493, 956, 16, 2979, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2, 1, 2549, 0, 367, 0, 624, 3, 857, 3717, 0, 3460, 875, 2188, 0, 3281, 1965, 2654, 119, 3, 3, 1834, 1888, 5, 1, 31, 531, 1567, 1186, 7, 3050, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[5, 0, 2129, 8, 583, 4, 569, 1, 628, 3264, 7, 3096, 1274, 1965, 3, 3556, 2779, 2473, 165, 1, 1, 1470, 2283, 0, 1, 50, 615, 2161, 950, 13, 2704, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2013, 0, 1661, 419, 0, 0, 597, 0, 572, 0, 3032, 0, 0, 0, 702, 233, 0, 3036, 0, 0, 0, 0, 587, 1370, 1, 1394, 0, 0, 0, 0, 1283, 3097, 130, 0, 0, 0, 0, 0, 4, 1853, 2383, 1, 238, 0, 0, 0, 0, 41, 521, 1546, 1406, 559, 0, 0, 757, 4, 2953, 1, 374, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4201, 3971, 3352, 4510, 4479, 4073, 4089] avg=4096.0000 max=4510 max_rank=4 min=3352 min_rank=3 imbalance_ratio=1.101074 max_load_reduction=28258 max_load_reduction_pct=0.862366 moved_to_echo=17151 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1668, 10, 1666, 372, 0, 0, 644, 6, 612, 0, 2696, 0, 0, 0, 908, 208, 2, 3115, 0, 0, 0, 0, 1082, 1763, 1, 1397, 0, 0, 0, 0, 936, 3126, 153, 0, 0, 0, 0, 0, 7, 1829, 2200, 50, 174, 0, 0, 0, 17, 26, 419, 1890, 1027, 561, 0, 0, 1286, 19, 2588, 37, 273, 0, 0, 0] rerouted_tokens_per_ep_rank=[3716, 3958, 4233, 4243, 4215, 4260, 3940, 4203] avg=4096.0000 max=4260 max_rank=5 min=3716 min_rank=0 imbalance_ratio=1.040039 max_load_reduction=28508 max_load_reduction_pct=0.869995 moved_to_echo=16618 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2101, 0, 1660, 398, 0, 0, 682, 0, 393, 1, 2883, 0, 0, 0, 443, 222, 0, 3070, 0, 0, 0, 0, 994, 1963, 0, 1393, 0, 0, 0, 0, 1292, 2587, 121, 0, 0, 0, 0, 0, 0, 1714, 2145, 1, 240, 0, 0, 0, 1, 24, 555, 1703, 1417, 559, 0, 0, 1111, 18, 2700, 0, 377, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 3959, 3735, 4350, 4000, 4100, 4259, 4206] avg=4096.0000 max=4350 max_rank=3 min=3735 min_rank=2 imbalance_ratio=1.062012 max_load_reduction=28418 max_load_reduction_pct=0.867249 moved_to_echo=16565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1835, 0, 1696, 411, 0, 0, 647, 1, 548, 2, 2978, 0, 0, 0, 917, 230, 2, 3290, 0, 0, 0, 0, 762, 1739, 4, 1424, 0, 0, 0, 0, 1225, 2648, 103, 0, 0, 0, 0, 0, 0, 1702, 2229, 4, 228, 0, 0, 0, 3, 71, 493, 1944, 1345, 571, 0, 0, 664, 3, 2687, 2, 358, 0, 0, 0] rerouted_tokens_per_ep_rank=[3944, 4176, 4439, 3929, 3976, 4163, 4427, 3714] avg=4096.0000 max=4439 max_rank=2 min=3714 min_rank=7 imbalance_ratio=1.083740 max_load_reduction=28329 max_load_reduction_pct=0.864532 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 1908, 2, 1658, 402, 0, 0, 744, 0, 606, 2, 2911, 0, 0, 0, 918, 224, 0, 3308, 0, 0, 0, 0, 798, 1787, 1, 1391, 0, 0, 0, 0, 1072, 2998, 108, 0, 0, 0, 0, 0, 2, 2228, 1674, 0, 199, 0, 0, 0, 2, 18, 639, 1632, 1175, 558, 0, 0, 753, 13, 2720, 0, 313, 0, 0, 0] rerouted_tokens_per_ep_rank=[3974, 4263, 4450, 3977, 4178, 4103, 4024, 3799] avg=4096.0000 max=4450 max_rank=2 min=3799 min_rank=7 imbalance_ratio=1.086426 max_load_reduction=28318 max_load_reduction_pct=0.864197 moved_to_echo=16104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 2259, 0, 1631, 351, 0, 0, 1219, 0, 576, 5, 2515, 0, 0, 0, 904, 185, 0, 2360, 0, 0, 0, 0, 1025, 1593, 0, 1357, 0, 0, 0, 0, 1149, 2999, 482, 0, 0, 0, 0, 0, 1, 1411, 2207, 0, 219, 0, 0, 0, 0, 127, 560, 1493, 1277, 552, 0, 0, 956, 16, 2979, 5, 343, 0, 0, 0] rerouted_tokens_per_ep_rank=[4253, 4315, 3449, 3975, 4630, 3838, 4009, 4299] avg=4096.0000 max=4630 max_rank=4 min=3449 min_rank=2 imbalance_ratio=1.130371 max_load_reduction=28138 max_load_reduction_pct=0.858704 moved_to_echo=16776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 2549, 0, 1508, 422, 0, 0, 367, 0, 624, 3, 3059, 0, 0, 0, 857, 236, 0, 3460, 0, 0, 0, 0, 875, 2188, 0, 1265, 0, 0, 0, 0, 764, 2654, 119, 3, 0, 0, 0, 0, 3, 1834, 1888, 5, 141, 0, 0, 0, 1, 31, 531, 1567, 837, 508, 0, 0, 1186, 7, 3050, 0, 223, 0, 0, 0] rerouted_tokens_per_ep_rank=[4482, 4053, 4553, 4328, 3540, 3871, 3475, 4466] avg=4096.0000 max=4553 max_rank=2 min=3475 min_rank=6 imbalance_ratio=1.111572 max_load_reduction=28215 max_load_reduction_pct=0.861053 moved_to_echo=15352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 2129, 8, 1634, 371, 0, 0, 583, 4, 569, 1, 2686, 0, 0, 0, 628, 207, 7, 3096, 0, 0, 0, 0, 1274, 1965, 3, 1372, 0, 0, 0, 0, 1080, 2473, 165, 1, 0, 0, 0, 0, 1, 1470, 2283, 0, 200, 0, 0, 0, 1, 50, 615, 2161, 1184, 550, 0, 0, 950, 13, 2704, 10, 315, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 3843, 3938, 4614, 3719, 3954, 4561, 3992] avg=4096.0000 max=4614 max_rank=3 min=3719 min_rank=4 imbalance_ratio=1.126465 max_load_reduction=28154 max_load_reduction_pct=0.859192 moved_to_echo=16226 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 16512, 128, 13184, 3200, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7424, 14464, 128, 11008, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8832, 22656, 1408, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5504, 128, 4608, 128, 22784, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 14080, 17024, 128, 1664, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7680, 128, 22400, 128, 2688, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 4352, 13952, 9728, 4480, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6400, 1792, 128, 24832, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 3435, 0, 0, 0, 0, 0, 4096, 33, 4, 0, 0, 4087, 0, 0, 658, 0, 0, 0, 94, 0, 0, 4096, 0, 4, 22, 3995, 0, 0, 4053, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 3065, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 4096, 0, 2, 1098, 0, 0, 1, 2, 0, 0, 4096, 0, 7, 0, 4094, 0, 0, 4018, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4092, 1, 2332, 2, 9, 0, 0, 0, 4092, 0, 0, 4, 1, 4090, 1, 0, 1850, 0, 3, 0, 7, 1, 0, 4095, 0, 3, 3, 4014, 1, 2, 4072, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4094, 1, 3661, 0, 10, 0, 0, 0, 4093, 3, 0, 0, 3, 4096, 4, 0, 481, 1, 1, 2, 3, 0, 0, 4092, 0, 0, 0, 4087, 1, 2, 4040, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4094, 0, 2697, 2, 18, 0, 0, 0, 4092, 0, 6, 3, 4, 4095, 5, 0, 1433, 3, 0, 0, 0, 1, 0, 4087, 0, 2, 0, 4083, 0, 0, 4051, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 3462, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4094, 4, 0, 722, 0, 0, 0, 41, 0, 0, 4094, 0, 0, 0, 4085, 3, 6, 3972, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4094, 0, 2972, 2, 9, 0, 0, 0, 4094, 1, 0, 1, 0, 4094, 0, 0, 1146, 2, 1, 1, 0, 1, 0, 4095, 0, 0, 1, 4095, 0, 0, 4065, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[973, 0, 3065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 67, 0, 0, 4088, 1, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 1098, 0, 0, 1, 3055, 0, 0, 0, 2, 0, 0, 4073, 0, 0, 0, 0, 0, 7, 0, 4094, 23, 7, 1, 0, 0, 0, 4018, 72, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4091, 4090, 4098, 4154, 4075, 4132, 4090] avg=4096.0000 max=4154 max_rank=4 min=4038 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[973, 0, 3435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 67, 0, 0, 4089, 33, 4, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 658, 0, 0, 0, 3054, 0, 0, 0, 94, 0, 0, 4073, 0, 0, 0, 0, 0, 4, 22, 3995, 23, 7, 1, 0, 0, 0, 4053, 72, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4408, 4091, 4126, 4087, 3712, 4167, 4052, 4125] avg=4096.0000 max=4408 max_rank=0 min=3712 min_rank=4 imbalance_ratio=1.076172 max_load_reduction=28360 max_load_reduction_pct=0.865479 moved_to_echo=16056 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[972, 1, 3661, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4021, 67, 0, 0, 4086, 3, 0, 0, 0, 0, 0, 0, 3, 4096, 4, 0, 0, 0, 0, 0, 481, 1, 1, 2, 3054, 0, 0, 0, 3, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 4087, 23, 7, 1, 0, 1, 2, 4040, 72, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4634, 4098, 4089, 4103, 3539, 4072, 4118, 4115] avg=4096.0000 max=4634 max_rank=0 min=3539 min_rank=4 imbalance_ratio=1.131348 max_load_reduction=28134 max_load_reduction_pct=0.858582 moved_to_echo=15844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[972, 0, 2697, 2, 0, 0, 0, 0, 18, 0, 0, 0, 4020, 67, 0, 0, 4085, 0, 6, 3, 0, 0, 0, 0, 4, 4095, 5, 0, 0, 0, 0, 0, 1433, 3, 0, 0, 3054, 0, 0, 0, 0, 1, 0, 4064, 0, 0, 0, 0, 0, 2, 0, 4083, 23, 7, 1, 0, 0, 0, 4051, 72, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3671, 4105, 4094, 4104, 4490, 4065, 4116, 4123] avg=4096.0000 max=4490 max_rank=4 min=3671 min_rank=0 imbalance_ratio=1.096191 max_load_reduction=28278 max_load_reduction_pct=0.862976 moved_to_echo=16794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[972, 1, 2332, 2, 0, 0, 0, 0, 9, 0, 0, 0, 4021, 67, 0, 0, 4085, 0, 0, 4, 0, 0, 0, 0, 1, 4090, 1, 0, 0, 0, 0, 0, 1850, 0, 3, 0, 3052, 0, 0, 0, 7, 1, 0, 4072, 0, 0, 0, 0, 0, 3, 3, 4014, 23, 7, 1, 0, 1, 2, 4072, 72, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3307, 4097, 4089, 4092, 4905, 4080, 4051, 4147] avg=4096.0000 max=4905 max_rank=4 min=3307 min_rank=0 imbalance_ratio=1.197510 max_load_reduction=27863 max_load_reduction_pct=0.850311 moved_to_echo=17183 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[959, 0, 3462, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 73, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4, 0, 0, 0, 0, 0, 722, 0, 0, 0, 3059, 0, 0, 0, 41, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 4085, 26, 8, 5, 7, 3, 6, 3972, 67, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4422, 4102, 4084, 4091, 3781, 4109, 4131, 4048] avg=4096.0000 max=4422 max_rank=0 min=3781 min_rank=4 imbalance_ratio=1.079590 max_load_reduction=28346 max_load_reduction_pct=0.865051 moved_to_echo=16069 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[972, 0, 2972, 2, 0, 0, 0, 0, 9, 0, 0, 0, 4022, 67, 0, 0, 4087, 1, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1146, 2, 1, 1, 3054, 0, 0, 0, 0, 1, 0, 4072, 0, 0, 0, 0, 0, 0, 1, 4095, 23, 7, 1, 0, 0, 0, 4065, 72, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3946, 4098, 4089, 4094, 4204, 4073, 4127, 4137] avg=4096.0000 max=4204 max_rank=4 min=3946 min_rank=0 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16541 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4091, 0, 3361, 9, 1, 0, 0, 0, 4095, 18, 1, 0, 2, 4096, 1, 0, 923, 6, 0, 3, 59, 0, 0, 4091, 0, 0, 0, 4010, 0, 4, 3907, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[972, 0, 3361, 9, 0, 0, 0, 0, 1, 0, 0, 0, 4018, 67, 0, 0, 4088, 18, 1, 0, 0, 0, 0, 0, 2, 4096, 1, 0, 0, 0, 0, 0, 923, 6, 0, 3, 3051, 0, 0, 0, 59, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 4010, 23, 7, 1, 0, 0, 4, 3907, 72, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4342, 4086, 4107, 4099, 3983, 4127, 4041, 3983] avg=4096.0000 max=4342 max_rank=0 min=3983 min_rank=4 imbalance_ratio=1.060059 max_load_reduction=28426 max_load_reduction_pct=0.867493 moved_to_echo=16134 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 128, 128, 128, 24448, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7808, 128, 25088, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32256, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 0, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32256, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32512, 256, 128, 128, 128] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4081, 0, 975, 83, 0, 3082, 4067, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4096, 4095, 1, 0, 4096, 0, 1504, 144, 0, 2463, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4080, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4095, 4095, 0, 1, 4095, 0, 370, 190, 2, 3579, 4094, 0, 0, 0, 0, 4094, 1, 1, 1, 0, 4095, 0, 0, 0, 3, 0, 0, 4044, 7, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4092, 4092, 3, 2, 4090, 0, 972, 319, 0, 2801, 4095, 4, 0, 4, 0, 4092, 2, 4, 2, 0, 4086, 0, 1, 10, 0, 5, 0, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4093, 4092, 3, 1, 4096, 0, 1604, 168, 0, 2326, 4093, 1, 0, 0, 0, 4095, 1, 0, 1, 0, 4091, 0, 0, 3, 0, 0, 0, 4093, 0, 6, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4094, 0, 1032, 557, 0, 2530, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4, 4072, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4094, 4093, 0, 2, 4094, 0, 1757, 135, 0, 2218, 4096, 2, 4, 1, 0, 4095, 1, 1, 0, 2, 4094, 0, 2, 0, 0, 0, 1, 4076, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4093, 4095, 1, 0, 4096, 0, 1324, 112, 0, 2659, 4091, 10, 0, 0, 0, 4095, 0, 0, 3, 0, 4091, 0, 8, 5, 0, 0, 0, 4084, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2684, 0, 975, 83, 0, 0, 0, 0, 0, 3082, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 14, 0, 0, 0, 6, 0, 0, 0, 2688, 1383, 1, 0] rerouted_tokens_per_ep_rank=[4096, 3742, 4461, 4096, 4095, 4096, 4104, 4078] avg=4096.0000 max=4461 max_rank=2 min=3742 min_rank=1 imbalance_ratio=1.089111 max_load_reduction=28307 max_load_reduction_pct=0.863861 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 0, 0, 0, 2694, 0, 1504, 144, 0, 0, 0, 0, 0, 2463, 1388, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4080, 14, 0, 0, 0, 0, 0, 1, 0, 2707, 1388, 1, 0] rerouted_tokens_per_ep_rank=[4096, 4342, 3851, 4097, 4095, 4096, 4094, 4097] avg=4096.0000 max=4342 max_rank=1 min=3851 min_rank=2 imbalance_ratio=1.060059 max_load_reduction=28426 max_load_reduction_pct=0.867493 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 3, 1, 0, 0, 0, 0, 2694, 0, 1604, 168, 0, 0, 0, 0, 0, 2326, 1388, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 1, 0, 4092, 0, 0, 0, 4091, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 14, 0, 0, 0, 0, 6, 1, 0, 2705, 1388, 1, 0] rerouted_tokens_per_ep_rank=[4096, 4466, 3715, 4095, 4094, 4094, 4107, 4101] avg=4096.0000 max=4466 max_rank=1 min=3715 min_rank=2 imbalance_ratio=1.090332 max_load_reduction=28302 max_load_reduction_pct=0.863708 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 3, 2, 0, 0, 0, 0, 2690, 0, 972, 319, 0, 0, 0, 0, 0, 2801, 1388, 4, 0, 0, 0, 0, 0, 4, 0, 4092, 0, 0, 0, 0, 2, 4, 2, 0, 4091, 0, 0, 0, 4086, 0, 1, 10, 0, 0, 0, 0, 0, 5, 0, 4092, 14, 0, 0, 0, 0, 0, 0, 0, 2707, 1386, 1, 0] rerouted_tokens_per_ep_rank=[4097, 3981, 4193, 4096, 4099, 4097, 4111, 4094] avg=4096.0000 max=4193 max_rank=2 min=3981 min_rank=1 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2681, 0, 1032, 557, 0, 0, 0, 0, 0, 2530, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4072, 20, 5, 1, 0, 0, 0, 0, 0, 2712, 1393, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4270, 3914, 4095, 4096, 4090, 4102, 4105] avg=4096.0000 max=4270 max_rank=1 min=3914 min_rank=2 imbalance_ratio=1.042480 max_load_reduction=28498 max_load_reduction_pct=0.869690 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 1, 0, 0, 0, 0, 2693, 0, 370, 190, 0, 0, 0, 0, 2, 3579, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 1, 1, 0, 4094, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4044, 14, 0, 0, 0, 7, 1, 0, 0, 2706, 1388, 1, 0] rerouted_tokens_per_ep_rank=[4096, 3253, 4969, 4094, 4097, 4095, 4061, 4103] avg=4096.0000 max=4969 max_rank=2 min=3253 min_rank=1 imbalance_ratio=1.213135 max_load_reduction=27799 max_load_reduction_pct=0.848358 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 2, 0, 0, 0, 0, 2692, 0, 1757, 135, 0, 0, 0, 0, 0, 2218, 1389, 2, 0, 0, 0, 0, 4, 1, 0, 4095, 0, 0, 0, 0, 1, 1, 0, 2, 4093, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 4076, 14, 0, 0, 0, 0, 0, 0, 0, 2707, 1388, 1, 0] rerouted_tokens_per_ep_rank=[4095, 4584, 3609, 4100, 4097, 4096, 4091, 4096] avg=4096.0000 max=4584 max_rank=1 min=3609 min_rank=2 imbalance_ratio=1.119141 max_load_reduction=28184 max_load_reduction_pct=0.860107 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 0, 0, 0, 2694, 0, 1324, 112, 0, 0, 0, 0, 0, 2659, 1387, 10, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 0, 0, 0, 4091, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 4084, 14, 0, 0, 0, 0, 0, 0, 1, 2704, 1388, 1, 0] rerouted_tokens_per_ep_rank=[4096, 4130, 4056, 4095, 4095, 4104, 4098, 4094] avg=4096.0000 max=4130 max_rank=1 min=4056 min_rank=2 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 21760, 11136, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 21760, 11136, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21632, 0, 9600, 1792, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32640, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 4087, 4096, 0, 0, 0, 4094, 605, 0, 0, 0, 4096, 0, 0, 0, 0, 323, 0, 0, 4092, 1585, 0, 0, 7, 7, 4096, 0, 0, 0, 1584, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 4095, 4096, 0, 0, 0, 4095, 161, 0, 0, 0, 4096, 0, 0, 0, 0, 225, 1, 0, 4095, 2684, 0, 0, 1, 1, 4096, 0, 0, 0, 1027, 2, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 4095, 4095, 1, 1, 0, 4095, 388, 0, 0, 1, 4095, 0, 2, 1, 0, 584, 0, 0, 4094, 1797, 0, 4, 1, 2, 4094, 0, 1, 2, 1319, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 4092, 4096, 0, 0, 0, 4096, 1255, 0, 0, 0, 4096, 0, 0, 0, 0, 243, 0, 0, 4096, 2197, 0, 0, 4, 0, 4096, 0, 0, 0, 401, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4, 4092, 4091, 4, 1, 0, 4089, 540, 0, 1, 4, 4091, 0, 2, 5, 2, 117, 0, 4, 4091, 2521, 0, 0, 0, 78, 4089, 4, 7, 0, 855, 0, 4076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 4091, 4096, 0, 0, 0, 4092, 276, 0, 0, 0, 4096, 0, 0, 0, 0, 201, 0, 0, 4093, 2895, 0, 0, 4, 18, 4096, 0, 0, 3, 712, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1, 4095, 4095, 1, 1, 0, 4094, 338, 0, 0, 1, 4095, 0, 0, 2, 0, 418, 0, 2, 4095, 1975, 0, 5, 0, 7, 4094, 0, 1, 0, 1359, 0, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 4095, 4096, 0, 0, 0, 4084, 312, 0, 0, 0, 4096, 0, 0, 0, 0, 179, 9, 0, 4091, 2992, 2, 0, 10, 4, 4090, 0, 0, 5, 621, 0, 4082] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 3613, 605, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 323, 0, 0, 3807, 0, 0, 0, 0, 1585, 0, 0, 7, 981, 481, 285, 14, 7, 4082, 0, 0, 0, 0, 0, 0, 0, 1584, 0, 3115, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4218, 4096, 4094, 4130, 3353, 4089, 4699] avg=4096.0000 max=4699 max_rank=7 min=3353 min_rank=5 imbalance_ratio=1.147217 max_load_reduction=28069 max_load_reduction_pct=0.856598 moved_to_echo=16271 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 3614, 161, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 225, 1, 0, 3810, 0, 0, 0, 0, 2684, 0, 0, 1, 980, 481, 285, 14, 1, 4082, 0, 0, 0, 0, 0, 0, 0, 1027, 2, 3113, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3775, 4096, 4094, 4036, 4445, 4083, 4142] avg=4096.0000 max=4445 max_rank=5 min=3775 min_rank=1 imbalance_ratio=1.085205 max_load_reduction=28323 max_load_reduction_pct=0.864349 moved_to_echo=16706 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 4092, 2, 4, 0, 0, 0, 0, 1, 0, 3609, 540, 0, 0, 0, 0, 0, 1, 4, 4091, 0, 0, 0, 0, 0, 2, 5, 2, 4089, 0, 0, 0, 117, 0, 4, 3806, 0, 0, 0, 0, 2521, 0, 0, 0, 976, 480, 285, 14, 78, 4075, 4, 7, 0, 0, 0, 0, 0, 855, 0, 3100, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4150, 4096, 4098, 3927, 4276, 4164, 3955] avg=4096.0000 max=4276 max_rank=5 min=3927 min_rank=4 imbalance_ratio=1.043945 max_load_reduction=28492 max_load_reduction_pct=0.869507 moved_to_echo=16322 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 3611, 276, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 201, 0, 0, 3808, 0, 0, 0, 0, 2895, 0, 0, 4, 980, 481, 285, 14, 18, 4082, 0, 0, 0, 0, 0, 0, 3, 712, 0, 3115, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 3887, 4096, 4094, 4009, 4659, 4100, 3830] avg=4096.0000 max=4659 max_rank=5 min=3830 min_rank=7 imbalance_ratio=1.137451 max_load_reduction=28109 max_load_reduction_pct=0.857819 moved_to_echo=16598 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 4095, 2, 1, 0, 0, 0, 0, 1, 0, 3614, 388, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 2, 1, 0, 4093, 0, 0, 0, 584, 0, 0, 3809, 0, 0, 0, 0, 1797, 0, 4, 1, 980, 481, 285, 14, 2, 4080, 0, 1, 0, 0, 0, 0, 2, 1319, 0, 3115, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4003, 4096, 4096, 4393, 3562, 4083, 4436] avg=4096.0000 max=4436 max_rank=7 min=3562 min_rank=5 imbalance_ratio=1.083008 max_load_reduction=28332 max_load_reduction_pct=0.864624 moved_to_echo=16474 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 1255, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 243, 0, 0, 3806, 0, 0, 0, 0, 2197, 0, 0, 4, 984, 484, 290, 15, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 401, 0, 3105, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4867, 4096, 4103, 4049, 3974, 4081, 3506] avg=4096.0000 max=4867 max_rank=1 min=3506 min_rank=7 imbalance_ratio=1.188232 max_load_reduction=27901 max_load_reduction_pct=0.851471 moved_to_echo=15610 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 4095, 2, 1, 0, 0, 0, 0, 1, 0, 3613, 338, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 418, 0, 2, 3810, 0, 0, 0, 0, 1975, 0, 5, 0, 979, 481, 285, 14, 7, 4080, 0, 1, 0, 0, 0, 0, 0, 1359, 0, 3110, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 3952, 4096, 4095, 4230, 3739, 4088, 4469] avg=4096.0000 max=4469 max_rank=7 min=3739 min_rank=5 imbalance_ratio=1.091064 max_load_reduction=28299 max_load_reduction_pct=0.863617 moved_to_echo=16526 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 3604, 312, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 179, 9, 0, 3806, 0, 0, 0, 0, 2992, 2, 0, 10, 977, 480, 285, 14, 4, 4076, 0, 0, 0, 0, 0, 0, 5, 621, 0, 3105, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3916, 4096, 4094, 3994, 4760, 4080, 3731] avg=4096.0000 max=4760 max_rank=5 min=3731 min_rank=7 imbalance_ratio=1.162109 max_load_reduction=28008 max_load_reduction_pct=0.854736 moved_to_echo=16565 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[18688, 128, 128, 128, 7936, 3968, 2304, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7936, 128, 24960, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 28928, 3968, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2304, 128, 128, 30464, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32640, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 4096, 0, 4096, 0, 3907, 0, 4076, 0, 0, 0, 0, 21, 3, 4093, 4096, 0, 0, 4077, 0, 2, 4093, 190, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[5, 4, 6, 0, 4089, 0, 4091, 0, 4091, 0, 4086, 10, 5, 7, 0, 0, 8, 4090, 4087, 0, 3, 4093, 0, 0, 4085, 0, 0, 1, 0, 7, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4095, 0, 4096, 0, 4075, 1, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4095, 0, 0, 4095, 20, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 2, 4093, 1, 4095, 0, 4091, 0, 4087, 1, 1, 1, 0, 2, 5, 4089, 4093, 1, 1, 4095, 0, 0, 4092, 8, 0, 2, 0, 0, 3, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 4096, 0, 4088, 0, 4044, 0, 4081, 0, 0, 0, 0, 1, 4, 4084, 4093, 0, 0, 4092, 0, 3, 4096, 75, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2, 1, 2, 0, 4091, 0, 4095, 1, 4083, 0, 4094, 2, 2, 1, 0, 0, 0, 4092, 4094, 0, 1, 4095, 0, 0, 4094, 1, 0, 6, 0, 0, 11, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 21, 4096, 0, 4090, 0, 4083, 3, 4092, 0, 0, 0, 0, 8, 0, 4092, 4086, 0, 0, 4088, 3, 7, 4086, 8, 0, 0, 0, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 4065, 0, 4095, 0, 4000, 8, 4093, 0, 0, 0, 0, 1, 0, 4092, 4079, 0, 0, 4082, 7, 16, 4088, 120, 0, 0, 0, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 4024, 48, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 3907, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4086, 5, 0, 0, 3, 3, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 2, 1, 0, 0, 0, 4044, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4103, 3954, 4112, 4102, 4080, 4234, 4093] avg=4096.0000 max=4234 max_rank=6 min=3954 min_rank=2 imbalance_ratio=1.033691 max_load_reduction=28534 max_load_reduction_pct=0.870789 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4043, 48, 0, 0, 4095, 0, 3, 0, 0, 0, 0, 0, 4075, 1, 48, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 5, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4046, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4098, 4124, 4092, 4099, 4096, 4066, 4100] avg=4096.0000 max=4124 max_rank=2 min=4066 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 4, 6, 0, 4034, 47, 0, 0, 4089, 0, 7, 0, 0, 0, 0, 0, 4091, 0, 47, 10, 0, 0, 0, 0, 5, 7, 0, 0, 4081, 5, 0, 0, 8, 3, 4087, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 4087, 3, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4148, 4098, 4098, 4096, 4039, 4097] avg=4096.0000 max=4148 max_rank=2 min=4039 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 4033, 51, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4082, 12, 0, 0, 4, 2, 4093, 0, 0, 0, 0, 0, 0, 4092, 0, 3, 4, 0, 0, 0, 4041, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 6, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4080, 4095, 4099, 4099, 4116, 4088] avg=4096.0000 max=4116 max_rank=6 min=4080 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 21, 4039, 47, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 4083, 3, 48, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4080, 5, 0, 0, 0, 3, 4086, 0, 0, 0, 0, 0, 0, 4088, 3, 7, 0, 0, 0, 0, 4039, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4089, 3, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4103, 4134, 4093, 4089, 4098, 4047, 4097] avg=4096.0000 max=4134 max_rank=2 min=4047 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 2, 4035, 48, 0, 0, 4093, 1, 7, 0, 0, 0, 0, 0, 4091, 0, 47, 1, 0, 0, 0, 0, 1, 1, 0, 2, 4085, 5, 0, 0, 5, 3, 4093, 1, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 0, 0, 4043, 8, 0, 2, 0, 0, 0, 0, 0, 0, 3, 2, 4086, 3, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4101, 4139, 4094, 4102, 4097, 4053, 4094] avg=4096.0000 max=4139 max_rank=2 min=4053 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 4040, 47, 0, 0, 4065, 0, 7, 0, 0, 0, 0, 0, 4000, 8, 48, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 5, 0, 0, 0, 3, 4079, 0, 0, 0, 0, 0, 0, 4082, 7, 16, 0, 0, 0, 0, 4041, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4089, 3, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4072, 4056, 4091, 4082, 4105, 4161, 4096] avg=4096.0000 max=4161 max_rank=6 min=4056 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 2, 0, 4041, 48, 0, 0, 4091, 0, 7, 1, 0, 0, 0, 0, 4083, 0, 48, 2, 0, 0, 0, 0, 2, 1, 0, 0, 4085, 5, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 0, 0, 4045, 1, 0, 6, 0, 0, 0, 0, 0, 0, 11, 0, 4089, 3, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4099, 4133, 4093, 4097, 4097, 4052, 4103] avg=4096.0000 max=4133 max_rank=2 min=4052 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32384, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32384, 128, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32384, 512, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 327, 4096, 0, 0, 4096, 3769, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[8, 4087, 10, 4092, 0, 7, 0, 4, 6, 1651, 4086, 0, 0, 4080, 2440, 0, 0, 0, 5, 4, 0, 4088, 3, 4090, 4, 5, 1, 4, 4086, 5, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4091, 0, 0, 5, 0, 0, 1639, 4096, 0, 0, 4096, 2457, 0, 0, 1, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 4092, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 722, 4096, 0, 0, 4096, 3374, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4094, 1, 4095, 0, 1, 0, 1, 1, 830, 4092, 0, 0, 4094, 3267, 1, 0, 0, 2, 1, 0, 4095, 0, 4095, 0, 1, 0, 0, 4095, 1, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 422, 4096, 0, 0, 4096, 3674, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[2, 4092, 3, 4095, 0, 3, 0, 1, 2, 1702, 4094, 1, 0, 4093, 2392, 0, 0, 0, 1, 1, 0, 4094, 1, 4094, 0, 2, 0, 0, 4093, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4090, 0, 0, 6, 0, 0, 461, 4096, 0, 0, 4096, 3635, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3125, 968, 0, 0, 0, 327, 3128, 0, 0, 0, 0, 0, 0, 971, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4093, 3455, 4740, 4096, 4097, 4095, 4096] avg=4096.0000 max=4740 max_rank=3 min=3455 min_rank=2 imbalance_ratio=1.157227 max_load_reduction=28028 max_load_reduction_pct=0.855347 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3125, 968, 0, 0, 0, 722, 3128, 0, 0, 0, 0, 0, 0, 971, 3374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4093, 3850, 4345, 4096, 4097, 4095, 4096] avg=4096.0000 max=4345 max_rank=3 min=3850 min_rank=2 imbalance_ratio=1.060791 max_load_reduction=28423 max_load_reduction_pct=0.867401 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 10, 4092, 0, 0, 0, 0, 0, 7, 0, 4, 3113, 966, 0, 0, 6, 1651, 3120, 0, 0, 0, 0, 0, 0, 967, 2440, 0, 0, 0, 0, 0, 0, 0, 5, 4, 4087, 0, 0, 0, 0, 4088, 3, 1, 0, 0, 0, 0, 4, 5, 1, 4, 4089, 0, 0, 0, 4086, 5, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4090, 4777, 3407, 4096, 4092, 4103, 4093] avg=4096.0000 max=4777 max_rank=2 min=3407 min_rank=3 imbalance_ratio=1.166260 max_load_reduction=27991 max_load_reduction_pct=0.854218 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 5, 0, 3125, 968, 0, 0, 0, 1639, 3128, 0, 0, 0, 0, 0, 0, 971, 2457, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4092, 0, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4098, 4767, 3428, 4097, 4096, 4095, 4096] avg=4096.0000 max=4767 max_rank=2 min=3428 min_rank=3 imbalance_ratio=1.163818 max_load_reduction=28001 max_load_reduction_pct=0.854523 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 3129, 974, 4, 0, 0, 422, 3122, 0, 0, 0, 0, 0, 0, 967, 3674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4107, 3544, 4641, 4096, 4097, 4095, 4099] avg=4096.0000 max=4641 max_rank=3 min=3544 min_rank=2 imbalance_ratio=1.133057 max_load_reduction=28127 max_load_reduction_pct=0.858368 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 3123, 968, 0, 0, 1, 830, 3124, 0, 0, 0, 0, 0, 0, 971, 3267, 1, 0, 0, 0, 0, 0, 0, 2, 1, 4094, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4093, 3955, 4239, 4097, 4096, 4095, 4097] avg=4096.0000 max=4239 max_rank=3 min=3955 min_rank=2 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 6, 0, 3125, 968, 0, 0, 0, 461, 3128, 0, 0, 0, 0, 0, 0, 971, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4099, 3589, 4606, 4096, 4097, 4095, 4096] avg=4096.0000 max=4606 max_rank=3 min=3589 min_rank=2 imbalance_ratio=1.124512 max_load_reduction=28162 max_load_reduction_pct=0.859436 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3, 4095, 0, 0, 0, 0, 0, 3, 0, 1, 3123, 968, 0, 0, 2, 1702, 3126, 1, 0, 0, 0, 0, 0, 970, 2392, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4092, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4095, 4831, 3362, 4094, 4096, 4095, 4095] avg=4096.0000 max=4831 max_rank=2 min=3362 min_rank=3 imbalance_ratio=1.179443 max_load_reduction=27937 max_load_reduction_pct=0.852570 moved_to_echo=16380 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 25088, 7808, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7808, 25088, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 7808, 25088, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 3, 0, 2, 4, 11, 3988, 4089, 0, 2, 0, 0, 5, 4080, 0, 4088, 4083, 0, 7, 4091, 107, 0, 8, 0, 8, 4089, 0, 6, 6, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 0, 4085, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 16, 0, 0, 0, 0, 4086, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 0, 4092, 4094, 0, 0, 4, 4, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 30, 0, 0, 2, 0, 4060, 0, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 22, 0, 0, 0, 0, 4093, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 2, 2, 0, 4084, 4095, 0, 1, 0, 0, 1, 4095, 0, 4094, 4094, 0, 1, 4095, 10, 1, 2, 0, 1, 4095, 0, 1, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[2, 2, 0, 0, 0, 1, 0, 3, 4085, 4094, 0, 0, 0, 0, 1, 4091, 1, 4093, 4093, 0, 2, 4094, 10, 1, 2, 0, 0, 4095, 1, 1, 2, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 0, 0, 0, 4070, 4094, 0, 0, 1, 4, 0, 4096, 0, 4094, 4096, 0, 0, 4086, 25, 0, 0, 8, 0, 4086, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 3, 4085, 2, 0, 0, 0, 2, 4, 11, 4061, 22, 0, 0, 3988, 26, 0, 2, 0, 0, 0, 0, 0, 0, 5, 4080, 0, 0, 0, 0, 0, 3, 4083, 0, 0, 0, 0, 0, 7, 4065, 107, 0, 0, 0, 0, 0, 8, 0, 8, 4089, 4, 0, 0, 0, 0, 6, 6, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4100, 4016, 4085, 4086, 4179, 4109, 4102] avg=4096.0000 max=4179 max_rank=5 min=4016 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 3, 0, 0, 1, 0, 0, 0, 4068, 22, 0, 0, 4091, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4070, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4091, 4116, 4096, 4096, 4074, 4100, 4096] avg=4096.0000 max=4116 max_rank=2 min=4074 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 3, 0, 0, 4, 0, 0, 0, 4068, 22, 0, 0, 4073, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4070, 22, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 4098, 4096, 4099, 4092, 4097, 4096] avg=4096.0000 max=4099 max_rank=4 min=4092 min_rank=5 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 2, 0, 0, 6, 0, 0, 0, 4066, 22, 0, 0, 4092, 26, 0, 0, 0, 0, 0, 0, 4, 4, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4069, 30, 0, 0, 0, 0, 0, 0, 2, 0, 4060, 4, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4094, 4118, 4104, 4099, 4099, 4066, 4093] avg=4096.0000 max=4118 max_rank=2 min=4066 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 7, 0, 0, 5, 0, 0, 0, 4071, 27, 0, 0, 4085, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4061, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 8, 1, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4103, 4103, 4098, 4096, 4077, 4095, 4093] avg=4096.0000 max=4103 max_rank=0 min=4077 min_rank=5 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 3, 0, 0, 1, 2, 2, 0, 4067, 22, 0, 0, 4084, 25, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 1, 4069, 10, 1, 0, 0, 0, 0, 2, 0, 1, 4095, 4, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4094, 4110, 4096, 4096, 4081, 4102, 4094] avg=4096.0000 max=4110 max_rank=2 min=4081 min_rank=5 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 2, 0, 0, 12, 0, 0, 0, 4066, 22, 0, 0, 4070, 26, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4060, 25, 0, 0, 0, 0, 0, 0, 8, 0, 4086, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4100, 4096, 4101, 4099, 4085, 4098, 4096] avg=4096.0000 max=4101 max_rank=3 min=4085 min_rank=5 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 0, 0, 4090, 2, 0, 0, 0, 1, 0, 3, 4066, 22, 0, 0, 4085, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 1, 3, 4093, 0, 0, 0, 0, 0, 2, 4068, 10, 1, 0, 0, 0, 0, 2, 0, 0, 4095, 4, 0, 0, 0, 1, 1, 2, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4092, 4111, 4092, 4097, 4081, 4101, 4098] avg=4096.0000 max=4111 max_rank=2 min=4081 min_rank=5 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 256, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2076, 0, 0, 0, 0, 4092, 4096, 4096, 0, 0, 1, 5, 4096, 4095, 0, 0, 2019, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 2, 3466, 0, 0, 0, 0, 4088, 4096, 4096, 0, 0, 0, 14, 4096, 4096, 0, 0, 622, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 5, 3332, 0, 0, 0, 0, 4048, 4096, 4096, 0, 0, 14, 33, 4096, 4092, 0, 0, 764, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 8, 9, 0, 4090, 4086, 4, 6, 0, 2077, 6, 0, 0, 0, 4090, 4083, 4086, 0, 6, 0, 0, 4089, 4090, 7, 0, 2011, 3, 10, 0, 0, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3395, 0, 0, 0, 0, 4086, 4096, 4096, 0, 0, 9, 7, 4096, 4090, 0, 0, 697, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 4095, 4095, 0, 1, 0, 1613, 1, 0, 1, 0, 4090, 4094, 4095, 0, 1, 1, 0, 4095, 4095, 1, 0, 2485, 0, 1, 1, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[2, 2, 1, 0, 4094, 4093, 0, 2, 0, 2636, 2, 0, 0, 0, 4094, 4090, 4093, 0, 2, 0, 0, 4094, 4094, 2, 0, 1462, 0, 3, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2659, 0, 0, 0, 0, 4074, 4096, 4096, 0, 0, 15, 19, 4092, 4089, 0, 0, 1432, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2076, 0, 0, 1426, 7, 4, 0, 0, 0, 4092, 14, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 5, 0, 4088, 0, 0, 0, 0, 0, 0, 2019, 0, 0, 2656, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 3513, 4106, 4093, 4093, 4677, 4095] avg=4096.0000 max=4677 max_rank=6 min=3513 min_rank=2 imbalance_ratio=1.141846 max_load_reduction=28091 max_load_reduction_pct=0.857269 moved_to_echo=16958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 3332, 0, 0, 1426, 7, 4, 0, 0, 0, 4048, 14, 0, 0, 0, 0, 4092, 0, 0, 14, 0, 0, 0, 0, 33, 0, 4085, 0, 0, 0, 0, 0, 0, 764, 0, 0, 2656, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4774, 4062, 4106, 4118, 3422, 4095] avg=4096.0000 max=4774 max_rank=2 min=3422 min_rank=6 imbalance_ratio=1.165527 max_load_reduction=27994 max_load_reduction_pct=0.854309 moved_to_echo=15741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 9, 0, 4087, 1, 0, 0, 0, 4086, 4, 6, 0, 0, 0, 0, 0, 2077, 6, 0, 1421, 7, 4, 0, 0, 0, 4090, 14, 0, 0, 0, 0, 4082, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4083, 7, 0, 0, 0, 0, 0, 2011, 3, 10, 2648, 2, 0, 0, 0, 0, 0, 7, 4089, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4096, 3515, 4104, 4088, 4090, 4674, 4096] avg=4096.0000 max=4674 max_rank=6 min=3515 min_rank=2 imbalance_ratio=1.141113 max_load_reduction=28094 max_load_reduction_pct=0.857361 moved_to_echo=16948 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 3466, 0, 0, 1428, 12, 7, 0, 0, 0, 4088, 7, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4084, 0, 0, 0, 0, 0, 0, 622, 0, 0, 2661, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4915, 4095, 4089, 4098, 3285, 4094] avg=4096.0000 max=4915 max_rank=2 min=3285 min_rank=6 imbalance_ratio=1.199951 max_load_reduction=27853 max_load_reduction_pct=0.850006 moved_to_echo=15566 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3395, 0, 0, 1426, 7, 4, 0, 0, 0, 4086, 14, 0, 0, 0, 0, 4092, 0, 0, 9, 0, 0, 0, 0, 7, 0, 4083, 0, 0, 0, 0, 0, 0, 697, 0, 0, 2656, 3, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4832, 4100, 4101, 4090, 3356, 4099] avg=4096.0000 max=4832 max_rank=2 min=3356 min_rank=6 imbalance_ratio=1.179688 max_load_reduction=27936 max_load_reduction_pct=0.852539 moved_to_echo=15646 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 4093, 1, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 1613, 1, 0, 1425, 7, 4, 0, 1, 0, 4090, 14, 0, 0, 0, 0, 4091, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 2485, 0, 1, 2655, 2, 0, 0, 1, 0, 0, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 3050, 4105, 4093, 4089, 5143, 4096] avg=4096.0000 max=5143 max_rank=6 min=3050 min_rank=2 imbalance_ratio=1.255615 max_load_reduction=27625 max_load_reduction_pct=0.843048 moved_to_echo=17421 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 1, 0, 4091, 1, 0, 0, 1, 4093, 0, 2, 0, 0, 0, 0, 0, 2636, 2, 0, 1424, 7, 4, 0, 0, 0, 4094, 14, 0, 0, 0, 0, 4089, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 0, 0, 0, 0, 0, 1462, 0, 3, 2652, 3, 0, 0, 0, 0, 0, 2, 4092, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4073, 4108, 4091, 4089, 4120, 4094] avg=4096.0000 max=4120 max_rank=6 min=4073 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2659, 0, 0, 1426, 7, 4, 0, 0, 0, 4074, 14, 0, 0, 0, 0, 4092, 0, 0, 15, 0, 0, 0, 0, 19, 0, 4082, 0, 0, 0, 0, 0, 0, 1432, 0, 0, 2656, 3, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4096, 4096, 4088, 4107, 4101, 4091, 4099] avg=4096.0000 max=4107 max_rank=4 min=4088 min_rank=3 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 21376, 128, 0, 11520, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 11520, 128, 128, 21248, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4094, 0, 4094, 0, 6, 0, 1, 2, 4093, 4088, 4094, 2, 1, 1, 0, 2, 1, 0, 0, 3, 3, 0, 2, 0, 1, 4094, 0, 0, 4095, 4091, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 83, 0, 0, 0, 4069, 4054, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 4015, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4087, 0, 4084, 0, 0, 0, 6, 7, 4090, 4057, 4065, 8, 6, 9, 0, 6, 0, 15, 0, 15, 50, 0, 9, 0, 0, 4090, 0, 0, 4088, 4076, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 22, 0, 0, 0, 4077, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4087, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 6, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4092, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 25, 0, 0, 0, 4071, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4091, 0, 0, 4096, 4096, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 79, 0, 0, 0, 4019, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 4072, 0, 0, 4096, 4096, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4094, 0, 4093, 0, 0, 0, 2, 2, 4093, 4077, 4091, 2, 2, 2, 0, 2, 0, 2, 0, 3, 22, 0, 3, 0, 0, 4094, 0, 0, 4093, 4091, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4096, 0, 0, 0, 0, 0, 83, 0, 0, 0, 4018, 19, 0, 0, 4069, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 67, 0, 0, 0, 4077, 0, 0, 0, 0, 4015, 0, 0, 16, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4120, 4089, 4093, 4092, 4144, 4031, 4100] avg=4096.0000 max=4144 max_rank=5 min=4031 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4060, 19, 0, 0, 4092, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 2, 0, 0, 0, 4077, 4, 0, 0, 0, 4092, 0, 0, 16, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4085, 4112, 4096, 4092, 4083, 4108, 4096] avg=4096.0000 max=4112 max_rank=2 min=4083 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 6, 7, 4021, 19, 0, 0, 4090, 20, 0, 8, 0, 0, 0, 0, 6, 9, 0, 6, 4084, 0, 0, 0, 0, 15, 0, 15, 4084, 0, 0, 0, 50, 0, 9, 0, 4046, 0, 0, 0, 0, 4090, 0, 0, 16, 0, 0, 0, 4, 4076, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4053, 4118, 4105, 4114, 4105, 4106, 4080] avg=4096.0000 max=4118 max_rank=2 min=4053 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16405 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4096, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4060, 19, 0, 0, 4071, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 2, 0, 0, 0, 4077, 0, 0, 0, 0, 4091, 0, 0, 16, 0, 0, 0, 4, 4096, 0, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4104, 4091, 4093, 4092, 4079, 4107, 4103] avg=4096.0000 max=4107 max_rank=6 min=4079 min_rank=5 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 22, 0, 0, 0, 4064, 19, 0, 0, 4077, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 4077, 0, 0, 0, 0, 4087, 0, 0, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4105, 4089, 4096, 4096, 4083, 4107, 4096] avg=4096.0000 max=4107 max_rank=6 min=4083 min_rank=5 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4094, 0, 0, 0, 0, 0, 6, 0, 1, 2, 4052, 19, 0, 0, 4093, 20, 0, 2, 0, 0, 0, 0, 1, 1, 0, 2, 4091, 0, 0, 0, 1, 0, 0, 3, 4091, 1, 0, 0, 3, 0, 2, 0, 4075, 1, 0, 0, 1, 4094, 0, 0, 16, 0, 0, 0, 3, 4091, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4080, 4115, 4095, 4096, 4081, 4111, 4094] avg=4096.0000 max=4115 max_rank=2 min=4080 min_rank=1 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4096, 0, 0, 0, 0, 0, 79, 0, 0, 0, 4060, 19, 0, 0, 4019, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 18, 0, 0, 0, 4077, 0, 0, 0, 0, 4072, 0, 0, 16, 0, 0, 0, 4, 4096, 0, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4158, 4039, 4093, 4092, 4095, 4088, 4104] avg=4096.0000 max=4158 max_rank=1 min=4039 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4041, 18, 0, 0, 4093, 20, 0, 2, 0, 0, 0, 0, 2, 2, 0, 2, 4091, 0, 0, 0, 0, 2, 0, 3, 4089, 0, 0, 0, 22, 0, 3, 0, 4073, 0, 0, 0, 0, 4094, 0, 0, 16, 0, 0, 0, 4, 4091, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4063, 4115, 4097, 4094, 4098, 4110, 4095] avg=4096.0000 max=4115 max_rank=2 min=4063 min_rank=1 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 0, 128, 128, 32384, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32640, 0, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 256, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 0, 128, 0, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4095, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4094, 0, 0, 4096, 0, 0, 525, 0, 0, 0, 1, 0, 2, 3571, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 2, 0, 4096, 0, 4096, 0, 0, 4054, 0, 0, 4094, 0, 0, 17, 0, 0, 0, 0, 0, 69, 4053, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4095, 4095, 8, 1, 4095, 1, 4095, 2, 2, 4094, 0, 1, 4043, 1, 1, 973, 0, 0, 1, 0, 0, 1, 3163, 2, 0, 0, 4092, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4090, 4096, 0, 0, 4096, 0, 4096, 0, 5, 4089, 0, 0, 4096, 0, 0, 112, 0, 0, 0, 4, 0, 7, 3984, 0, 0, 0, 4092, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4092, 0, 0, 4093, 0, 0, 473, 0, 0, 0, 3, 0, 15, 3612, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4087, 4086, 0, 9, 4083, 3, 4087, 0, 0, 4090, 0, 8, 4040, 5, 7, 899, 5, 0, 6, 0, 39, 0, 3199, 6, 9, 0, 4080, 0, 5, 0, 15, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4094, 4096, 0, 0, 4096, 0, 4096, 0, 9, 4076, 0, 0, 4096, 0, 0, 411, 0, 0, 0, 2, 0, 19, 3685, 0, 0, 0, 4088, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4093, 4092, 1, 3, 4093, 0, 4094, 2, 0, 4094, 0, 2, 4077, 2, 2, 708, 0, 1, 2, 0, 15, 0, 3386, 1, 3, 0, 4093, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 7, 0, 0, 0, 3595, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 2, 3571, 0, 492, 4, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4101, 4120, 4096, 4069, 4098, 4093] avg=4096.0000 max=4120 max_rank=3 min=4069 min_rank=5 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16356 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 7, 0, 0, 0, 3593, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 69, 4053, 0, 492, 1, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4097, 4061, 3610, 4095, 4615, 4097, 4095] avg=4096.0000 max=4615 max_rank=5 min=3610 min_rank=3 imbalance_ratio=1.126709 max_load_reduction=28153 max_load_reduction_pct=0.859161 moved_to_echo=16902 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 5, 4089, 0, 0, 7, 0, 0, 0, 3595, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 7, 3984, 0, 492, 1, 0, 0, 0, 0, 4092, 1, 2, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4099, 4101, 3707, 4099, 4484, 4095, 4093] avg=4096.0000 max=4484 max_rank=5 min=3707 min_rank=3 imbalance_ratio=1.094727 max_load_reduction=28284 max_load_reduction_pct=0.863159 moved_to_echo=16771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 0, 9, 0, 0, 0, 0, 4083, 3, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 8, 7, 0, 0, 0, 3546, 5, 7, 899, 0, 0, 0, 0, 5, 0, 6, 0, 4085, 0, 0, 0, 39, 0, 3199, 6, 485, 1, 0, 0, 9, 0, 4080, 0, 2, 0, 0, 0, 5, 0, 15, 0, 4086, 1, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4086, 4105, 4457, 4096, 3730, 4091, 4107] avg=4096.0000 max=4457 max_rank=3 min=3730 min_rank=5 imbalance_ratio=1.088135 max_load_reduction=28311 max_load_reduction_pct=0.863983 moved_to_echo=16024 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 9, 0, 0, 0, 3582, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 15, 3612, 0, 495, 4, 0, 0, 0, 0, 4096, 0, 7, 2, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4101, 4055, 4098, 4126, 4105, 4093] avg=4096.0000 max=4126 max_rank=5 min=4055 min_rank=3 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 8, 1, 0, 0, 0, 0, 4095, 1, 0, 2, 0, 0, 0, 0, 2, 4094, 0, 1, 7, 0, 0, 0, 3548, 1, 1, 973, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 1, 3163, 2, 486, 2, 0, 0, 0, 0, 4092, 0, 2, 0, 0, 0, 0, 0, 2, 0, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4098, 4104, 4523, 4095, 3654, 4094, 4096] avg=4096.0000 max=4523 max_rank=3 min=3654 min_rank=5 imbalance_ratio=1.104248 max_load_reduction=28245 max_load_reduction_pct=0.861969 moved_to_echo=15939 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 9, 4076, 0, 0, 7, 0, 0, 0, 3595, 0, 0, 411, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 19, 3685, 0, 492, 1, 0, 0, 0, 0, 4088, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4099, 4092, 4006, 4096, 4197, 4090, 4094] avg=4096.0000 max=4197 max_rank=5 min=4006 min_rank=3 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 1, 3, 0, 0, 0, 0, 4093, 0, 2, 2, 0, 0, 0, 0, 0, 4094, 0, 2, 7, 0, 0, 0, 3578, 2, 2, 708, 0, 0, 0, 0, 0, 1, 2, 0, 4090, 0, 0, 0, 15, 0, 3386, 1, 490, 1, 0, 0, 3, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 4, 0, 4091, 2, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4097, 4103, 4290, 4093, 3893, 4098, 4097] avg=4096.0000 max=4290 max_rank=3 min=3893 min_rank=5 imbalance_ratio=1.047363 max_load_reduction=28478 max_load_reduction_pct=0.869080 moved_to_echo=16181 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[28672, 128, 128, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 28672, 128, 3968, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 26, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4081, 0, 4011, 4096, 73, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4096, 4096, 4066, 0, 4087, 4096, 30, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 4096, 8, 0, 0, 4096, 4096, 4095, 0, 4087, 4096, 0, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[2, 1, 4094, 0, 1, 0, 0, 0, 4094, 0, 0, 2, 0, 1, 1, 4094, 0, 2, 1, 4094, 4091, 4090, 2, 4072, 4095, 23, 5, 0, 0, 0, 1, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 4096, 4095, 0, 4094, 4096, 1, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[11, 6, 4085, 0, 5, 0, 0, 0, 4085, 0, 1, 8, 0, 17, 9, 4086, 0, 10, 5, 4087, 4081, 3739, 6, 4069, 4089, 354, 0, 0, 0, 0, 0, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 4096, 4096, 4090, 0, 4069, 4096, 0, 6, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[3, 1, 4093, 0, 1, 0, 0, 0, 4093, 1, 0, 2, 0, 5, 2, 4094, 0, 3, 2, 4094, 4090, 4029, 1, 4078, 4094, 76, 2, 0, 0, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 4095, 0, 60, 0, 0, 0, 0, 4026, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 70, 6, 4] rerouted_tokens_per_ep_rank=[4091, 4098, 4096, 4096, 4093, 4155, 4027, 4112] avg=4096.0000 max=4155 max_rank=5 min=4027 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 26, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4081, 0, 59, 0, 0, 0, 0, 4026, 73, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 70, 6, 4] rerouted_tokens_per_ep_rank=[4116, 4098, 4096, 4096, 4092, 4140, 4100, 4030] avg=4096.0000 max=4140 max_rank=5 min=4030 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 4092, 0, 0, 0, 0, 0, 4095, 0, 60, 0, 0, 0, 0, 4026, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 70, 6, 4] rerouted_tokens_per_ep_rank=[4090, 4098, 4095, 4097, 4100, 4155, 4028, 4105] avg=4096.0000 max=4155 max_rank=5 min=4028 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[11, 6, 4079, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4081, 2, 0, 0, 4085, 0, 1, 8, 0, 0, 0, 0, 0, 17, 9, 4086, 0, 0, 0, 0, 0, 10, 5, 4083, 0, 0, 0, 0, 0, 3739, 6, 60, 0, 0, 0, 0, 4019, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4007, 70, 6, 4] rerouted_tokens_per_ep_rank=[4096, 4088, 4094, 4112, 4098, 3805, 4373, 4102] avg=4096.0000 max=4373 max_rank=6 min=3805 min_rank=5 imbalance_ratio=1.067627 max_load_reduction=28395 max_load_reduction_pct=0.866547 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4087, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 5, 0, 0, 4085, 0, 0, 0, 0, 0, 4066, 0, 55, 0, 0, 0, 0, 4023, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 73, 9, 11] rerouted_tokens_per_ep_rank=[4091, 4101, 4099, 4093, 4090, 4121, 4053, 4120] avg=4096.0000 max=4121 max_rank=5 min=4053 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 4088, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4091, 2, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 2, 1, 4090, 0, 0, 0, 0, 0, 4090, 2, 60, 0, 0, 0, 0, 4025, 23, 5, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4010, 70, 6, 4] rerouted_tokens_per_ep_rank=[4091, 4094, 4096, 4096, 4093, 4152, 4053, 4093] avg=4096.0000 max=4152 max_rank=5 min=4053 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 15, 0, 0, 4092, 0, 0, 0, 0, 0, 4090, 0, 60, 0, 0, 0, 0, 4026, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 70, 6, 4] rerouted_tokens_per_ep_rank=[4102, 4098, 4096, 4096, 4107, 4150, 4032, 4087] avg=4096.0000 max=4150 max_rank=5 min=4032 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 4087, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4090, 2, 0, 0, 4093, 1, 0, 2, 0, 0, 0, 0, 0, 5, 2, 4094, 0, 0, 0, 0, 0, 3, 2, 4090, 0, 0, 0, 0, 0, 4029, 1, 60, 0, 0, 0, 0, 4024, 76, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4016, 70, 6, 4] rerouted_tokens_per_ep_rank=[4091, 4093, 4096, 4101, 4095, 4090, 4102, 4100] avg=4096.0000 max=4102 max_rank=6 min=4090 min_rank=5 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32787, 32768, 32768, 32768, 32749] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32787, 32768, 32768, 32768, 32749] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32787, 32768, 32768, 32768, 32749] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32787, 32768, 32768, 32768, 32749] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32787, 32768, 32768, 32768, 32749] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32787, 32768, 32768, 32768, 32749] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32787, 32768, 32768, 32768, 32749] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32787, 32768, 32768, 32768, 32749] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32787 actual_valid_recv=33152 observed_loss=-365 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32749 actual_valid_recv=33280 observed_loss=-531 tokens_per_expert=[0, 0, 128, 128, 32128, 640, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32384, 128, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32256, 640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4087, 4084, 3, 4, 0, 0, 4089, 0, 6, 5, 0, 4084, 0, 12, 0, 14, 3, 0, 6, 50, 0, 8, 4085, 8, 4075, 7, 5, 1, 4044, 0, 0, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 80, 4016, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 18, 4076, 3, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4093, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 6, 4088, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4095, 4093, 0, 1, 0, 3, 4095, 0, 0, 1, 0, 4094, 10, 2, 0, 2, 0, 0, 1, 2, 0, 1, 4093, 1, 4094, 1, 0, 0, 4082, 2, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4093, 4092, 2, 1, 0, 0, 4094, 0, 2, 2, 0, 4093, 0, 4, 0, 4, 1, 0, 2, 16, 0, 2, 4092, 2, 4090, 2, 0, 0, 4080, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 47, 4047, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 72, 4013, 37, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 0, 0, 0, 80, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4082, 4, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4093, 4096, 4093, 4088, 4096, 4097, 4107] avg=4096.0000 max=4107 max_rank=7 min=4088 min_rank=4 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4085, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4083, 4, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4096, 4092, 4089, 4096, 4097, 4107] avg=4096.0000 max=4107 max_rank=7 min=4089 min_rank=4 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 3, 4, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 6, 5, 0, 4084, 0, 0, 0, 0, 0, 12, 0, 14, 4082, 0, 0, 0, 3, 0, 6, 50, 4077, 5, 2, 0, 0, 8, 4085, 8, 0, 0, 0, 0, 4075, 7, 5, 1, 1, 0, 0, 0, 4044, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4086, 4095, 4108, 4143, 4101, 4089, 4055] avg=4096.0000 max=4143 max_rank=4 min=4055 min_rank=7 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 0, 0, 0, 47, 4044, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4079, 4, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4099, 4096, 4090, 4085, 4096, 4097, 4107] avg=4096.0000 max=4107 max_rank=7 min=4085 min_rank=4 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 18, 4069, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 4, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 3, 0, 0, 4093, 0, 0, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4090, 4097, 4095, 4097, 4092, 4102, 4099] avg=4096.0000 max=4102 max_rank=6 min=4090 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 0, 0, 0, 72, 4010, 37, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4059, 4, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4119, 4096, 4090, 4065, 4096, 4097, 4107] avg=4096.0000 max=4119 max_rank=1 min=4065 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 4092, 2, 1, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 2, 2, 0, 4093, 0, 0, 0, 0, 0, 4, 0, 4, 4087, 0, 0, 0, 1, 0, 2, 16, 4083, 4, 2, 0, 0, 2, 4092, 2, 0, 0, 0, 0, 4090, 2, 0, 0, 1, 0, 0, 0, 4080, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4091, 4097, 4095, 4108, 4096, 4093, 4091] avg=4096.0000 max=4108 max_rank=4 min=4091 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 1, 0, 0, 0, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 10, 2, 0, 2, 4091, 0, 0, 0, 0, 0, 1, 2, 4084, 4, 2, 0, 0, 1, 4093, 1, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 0, 0, 0, 4082, 2, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4095, 4095, 4105, 4093, 4095, 4096, 4095] avg=4096.0000 max=4105 max_rank=3 min=4093 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32640, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 256, 32512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4071, 0, 0, 0, 4096, 4091, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 24, 4096, 6, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4089, 0, 18, 8, 4086, 4080, 4073, 4087, 2, 12, 0, 0, 11, 9, 0, 0, 10, 4084, 0, 3264, 0, 0, 0, 4078, 14, 7, 4, 819, 13, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4031, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 67, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 4096, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 11, 4096, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4076, 0, 0, 0, 4092, 4092, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4096, 0, 4096, 0, 0, 20, 4096, 3, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4094, 1, 2, 1, 4094, 4090, 4092, 4094, 1, 1, 2, 1, 2, 1, 0, 0, 1, 4094, 0, 3856, 0, 0, 0, 4093, 146, 2, 1, 96, 1, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4094, 0, 6, 2, 4094, 4090, 4089, 4093, 1, 3, 0, 0, 3, 2, 0, 0, 4, 4092, 0, 3865, 0, 0, 0, 4091, 17, 2, 0, 215, 4, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4050, 0, 0, 0, 4094, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0, 0, 46, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4031, 0, 0, 0, 17, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4091, 3, 1, 0, 0, 162, 0, 4094, 0, 0, 0, 0, 0, 0, 67, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4101, 4096, 4095, 4256, 4146, 3931, 4095] avg=4096.0000 max=4256 max_rank=4 min=3931 min_rank=6 imbalance_ratio=1.039062 max_load_reduction=28512 max_load_reduction_pct=0.870117 moved_to_echo=16428 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 17, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4091, 3, 1, 0, 0, 162, 0, 4096, 0, 0, 0, 0, 0, 0, 11, 4079, 0, 0, 0, 0, 1, 0, 0, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4100, 4096, 4095, 4258, 4090, 3932, 4095] avg=4096.0000 max=4258 max_rank=4 min=3932 min_rank=6 imbalance_ratio=1.039551 max_load_reduction=28510 max_load_reduction_pct=0.870056 moved_to_echo=16375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 18, 8, 17, 0, 0, 0, 0, 4080, 5, 0, 0, 0, 0, 0, 2, 12, 0, 0, 4085, 1, 0, 0, 11, 9, 0, 0, 4068, 3, 1, 0, 10, 162, 0, 3264, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 14, 7, 4, 819, 3919, 0, 0, 0, 13, 0, 0, 0, 4086, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4085, 4100, 4092, 3436, 4061, 4763, 4099] avg=4096.0000 max=4763 max_rank=6 min=3436 min_rank=4 imbalance_ratio=1.162842 max_load_reduction=28005 max_load_reduction_pct=0.854645 moved_to_echo=16359 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4071, 0, 0, 0, 21, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4093, 6, 1, 0, 0, 156, 0, 4096, 0, 0, 0, 0, 0, 0, 24, 4073, 0, 0, 0, 0, 6, 0, 0, 0, 3934, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4094, 4096, 4100, 4252, 4097, 3942, 4095] avg=4096.0000 max=4252 max_rank=4 min=3942 min_rank=6 imbalance_ratio=1.038086 max_load_reduction=28516 max_load_reduction_pct=0.870239 moved_to_echo=16386 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4076, 0, 0, 0, 17, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 1, 4, 4091, 3, 1, 0, 0, 162, 0, 4096, 0, 0, 0, 0, 0, 0, 20, 4079, 0, 0, 0, 0, 3, 0, 0, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4097, 4092, 4100, 4258, 4099, 3934, 4095] avg=4096.0000 max=4258 max_rank=4 min=3934 min_rank=6 imbalance_ratio=1.039551 max_load_reduction=28510 max_load_reduction_pct=0.870056 moved_to_echo=16386 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4094, 1, 2, 1, 17, 0, 0, 0, 0, 4090, 5, 0, 0, 0, 0, 0, 1, 1, 2, 1, 4093, 1, 0, 0, 2, 1, 0, 0, 4087, 3, 1, 0, 1, 162, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 146, 2, 1, 96, 3929, 0, 0, 0, 1, 0, 2, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4095, 4099, 4094, 4019, 4076, 4174, 4096] avg=4096.0000 max=4174 max_rank=6 min=4019 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16365 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4050, 0, 0, 0, 17, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 2, 0, 4091, 3, 2, 0, 0, 162, 0, 4096, 0, 0, 0, 0, 0, 0, 46, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4101, 4093, 4098, 4258, 4125, 3931, 4095] avg=4096.0000 max=4258 max_rank=4 min=3931 min_rank=6 imbalance_ratio=1.039551 max_load_reduction=28510 max_load_reduction_pct=0.870056 moved_to_echo=16409 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 6, 2, 17, 0, 0, 0, 0, 4090, 5, 0, 0, 0, 0, 0, 1, 3, 0, 0, 4092, 0, 0, 0, 3, 2, 0, 0, 4084, 3, 2, 0, 4, 162, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 17, 2, 0, 215, 3927, 0, 0, 0, 4, 0, 1, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4095, 4096, 4094, 4031, 4074, 4161, 4098] avg=4096.0000 max=4161 max_rank=6 min=4031 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16364 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 1408, 0, 31488, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 256, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 128, 1152, 31488, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4015, 81, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 0, 2, 2, 0, 4095, 2, 1, 1, 4093, 4094, 1, 0, 4093, 1, 0, 0, 0, 2, 1, 4093, 0, 4094, 0, 1, 1, 1, 0, 4094, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4074, 21, 0, 0, 0, 0, 4096, 4096, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4073, 23, 0, 0, 0, 0, 4096, 4096, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[3, 0, 0, 12, 9, 0, 4089, 10, 32, 0, 4082, 4085, 4, 0, 4058, 11, 0, 0, 0, 10, 6, 4083, 0, 4090, 0, 4, 7, 0, 0, 4086, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 4088, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4044, 49, 0, 0, 0, 0, 4096, 4095, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4096, 0, 3989, 107, 0, 0, 0, 0, 4096, 4091, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 0, 3, 3, 0, 4094, 3, 7, 0, 4093, 4093, 2, 0, 4087, 3, 0, 0, 0, 2, 2, 4092, 0, 4094, 0, 0, 2, 0, 0, 4093, 4093, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 37, 0, 4015, 0, 0, 0, 0, 81, 0, 0, 0, 4059, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4094, 4098, 4096, 4052, 4142, 4096] avg=4096.0000 max=4142 max_rank=6 min=4052 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 37, 0, 4074, 0, 0, 0, 0, 21, 0, 0, 0, 4059, 2, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4094, 4098, 4095, 4111, 4082, 4098] avg=4096.0000 max=4111 max_rank=5 min=4082 min_rank=6 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 12, 4084, 1, 0, 0, 9, 0, 4089, 10, 0, 0, 0, 0, 32, 0, 4080, 0, 0, 0, 0, 0, 4, 0, 4058, 11, 2, 0, 0, 0, 0, 0, 0, 10, 4085, 0, 0, 0, 6, 37, 0, 4090, 0, 0, 0, 0, 0, 4, 7, 0, 4046, 1, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4108, 4112, 4075, 4095, 4133, 4058, 4087] avg=4096.0000 max=4133 max_rank=5 min=4058 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 4094, 0, 6, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 7, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 27, 0, 4073, 0, 0, 0, 0, 23, 0, 0, 0, 4062, 2, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4100, 4089, 4110, 4095, 4100, 4087, 4092] avg=4096.0000 max=4110 max_rank=3 min=4087 min_rank=6 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4094, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 37, 0, 4044, 0, 0, 0, 0, 49, 0, 0, 0, 4059, 2, 0, 0, 0, 0, 4095, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4088, 4094, 4098, 4096, 4081, 4110, 4103] avg=4096.0000 max=4110 max_rank=6 min=4081 min_rank=5 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16390 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 2, 4092, 1, 0, 0, 2, 0, 4095, 2, 0, 0, 0, 0, 1, 1, 4091, 0, 0, 0, 0, 0, 1, 0, 4093, 1, 2, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 1, 37, 0, 4094, 0, 0, 0, 0, 0, 1, 1, 1, 4056, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4099, 4093, 4097, 4095, 4132, 4061, 4094] avg=4096.0000 max=4132 max_rank=5 min=4061 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 37, 0, 3989, 0, 0, 0, 0, 107, 0, 0, 0, 4059, 3, 0, 0, 0, 0, 4091, 14, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4082, 4094, 4098, 4101, 4026, 4169, 4105] avg=4096.0000 max=4169 max_rank=6 min=4026 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 3, 4091, 0, 0, 0, 3, 0, 4094, 3, 0, 0, 0, 0, 7, 0, 4091, 0, 0, 0, 0, 0, 2, 0, 4087, 3, 2, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 2, 37, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 4055, 2, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4100, 4098, 4094, 4095, 4133, 4059, 4093] avg=4096.0000 max=4133 max_rank=5 min=4059 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 384, 0, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 128, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 4096, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[3, 0, 2, 50, 0, 2, 0, 4092, 2, 4049, 2, 2, 0, 2, 1, 0, 1, 12, 0, 1, 2, 4094, 0, 4093, 4093, 4094, 0, 4094, 0, 4077, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[12, 0, 0, 119, 0, 8, 0, 4070, 9, 4087, 10, 10, 4, 12, 8, 0, 0, 6, 55, 11, 9, 4084, 0, 4085, 4038, 4082, 6, 4073, 0, 3967, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 0, 0, 0, 4096, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 0, 0, 4096, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 30, 0, 0, 0, 4096, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[3, 0, 1, 39, 0, 2, 0, 4089, 2, 4093, 3, 3, 1, 5, 2, 1, 0, 0, 16, 3, 3, 4093, 0, 4093, 4079, 4092, 0, 4083, 0, 4062, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4060, 3, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4065, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 31, 0, 0, 4012, 36, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 25, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4099, 4113, 4093, 4079, 4096, 4104, 4121] avg=4096.0000 max=4121 max_rank=7 min=4063 min_rank=0 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4059, 2, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 3960, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 134, 0, 0, 4012, 37, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4099, 4008, 4094, 4183, 4096, 4107, 4118] avg=4096.0000 max=4183 max_rank=4 min=4008 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16504 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 119, 4047, 1, 0, 0, 0, 8, 0, 4070, 3, 0, 0, 0, 9, 4087, 10, 10, 48, 0, 0, 0, 4, 12, 8, 0, 4081, 0, 0, 0, 0, 6, 55, 11, 3990, 37, 0, 0, 9, 0, 0, 4085, 0, 0, 0, 0, 4038, 0, 6, 10, 0, 0, 0, 0, 0, 3967, 3, 0, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4081, 4164, 4105, 4099, 4094, 4054, 3992] avg=4096.0000 max=4179 max_rank=0 min=3992 min_rank=7 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16239 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 4059, 2, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4010, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 76, 0, 0, 4012, 36, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4099, 4058, 4094, 4124, 4097, 4107, 4118] avg=4096.0000 max=4124 max_rank=4 min=4058 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 4060, 3, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4064, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 8, 0, 0, 4018, 36, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4100, 4116, 4093, 4062, 4096, 4096, 4118] avg=4096.0000 max=4118 max_rank=7 min=4062 min_rank=4 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 2, 50, 4057, 2, 0, 0, 0, 2, 0, 4092, 3, 0, 0, 0, 2, 4049, 2, 2, 48, 0, 0, 0, 0, 2, 1, 0, 4092, 0, 0, 0, 1, 12, 0, 1, 4010, 37, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 4093, 0, 0, 11, 0, 0, 0, 0, 0, 4077, 0, 0, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4097, 4103, 4095, 4061, 4095, 4104, 4099] avg=4096.0000 max=4114 max_rank=0 min=4061 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16359 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 30, 4059, 2, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4009, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 57, 0, 0, 4012, 36, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4099, 4057, 4094, 4105, 4097, 4107, 4118] avg=4096.0000 max=4118 max_rank=7 min=4057 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16427 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 1, 39, 4056, 1, 0, 0, 0, 2, 0, 4089, 3, 0, 0, 0, 2, 4093, 3, 3, 48, 0, 0, 0, 1, 5, 2, 1, 4091, 0, 0, 0, 0, 0, 16, 3, 4000, 36, 0, 0, 3, 1, 0, 4093, 0, 0, 0, 0, 4079, 0, 0, 10, 0, 0, 0, 0, 0, 4062, 0, 0, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4094, 4149, 4100, 4055, 4097, 4089, 4084] avg=4096.0000 max=4149 max_rank=2 min=4055 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16325 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 384, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 384, 128, 128, 32128, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32640, 128, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32384, 128, 128, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1424, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2672, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1162, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2934, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4094, 8, 1164, 3, 3, 0, 29, 1, 0, 4091, 0, 2, 0, 0, 1, 2913, 2, 2, 0, 4075, 3, 0, 0, 4093, 0, 4094, 1, 0, 2, 4093, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1604, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2492, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[16, 4086, 33, 1807, 19, 14, 0, 117, 4, 0, 4051, 0, 10, 0, 0, 6, 2279, 7, 10, 0, 3879, 15, 0, 0, 4074, 0, 4086, 9, 0, 83, 4081, 4082] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4094, 0, 2882, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1216, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 19, 2892, 6, 4, 1, 45, 0, 0, 4087, 0, 3, 0, 2, 3, 1274, 1, 3, 0, 3958, 4, 0, 0, 4090, 0, 4093, 2, 0, 4, 4092, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 0, 829, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3270, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2369, 0, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 1, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2672, 0, 0, 0, 1685, 27, 0, 0, 4096, 0, 0, 0, 42, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3793, 4070, 4102, 4095, 4384, 4139, 4096, 4089] avg=4096.0000 max=4384 max_rank=4 min=3793 min_rank=0 imbalance_ratio=1.070312 max_load_reduction=28384 max_load_reduction_pct=0.866211 moved_to_echo=16708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2369, 0, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 1, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2934, 0, 0, 0, 1685, 26, 0, 0, 4096, 0, 0, 0, 42, 1, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3531, 4067, 4102, 4095, 4645, 4139, 4100, 4089] avg=4096.0000 max=4645 max_rank=4 min=3531 min_rank=0 imbalance_ratio=1.134033 max_load_reduction=28123 max_load_reduction_pct=0.858246 moved_to_echo=16973 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2367, 0, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 1, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1216, 0, 0, 0, 1685, 26, 0, 0, 4096, 0, 0, 0, 42, 1, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5249, 4067, 4102, 4095, 2927, 4139, 4100, 4089] avg=4096.0000 max=5249 max_rank=0 min=2927 min_rank=4 imbalance_ratio=1.281494 max_load_reduction=27519 max_load_reduction_pct=0.839813 moved_to_echo=15255 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[16, 2363, 33, 1807, 0, 0, 0, 0, 19, 14, 0, 117, 4056, 1, 0, 0, 4, 0, 4051, 0, 6, 0, 0, 0, 10, 0, 0, 6, 4081, 0, 0, 0, 2279, 7, 10, 0, 1681, 27, 0, 0, 3879, 15, 0, 0, 42, 1, 0, 0, 4074, 0, 3, 9, 0, 0, 0, 0, 0, 83, 4074, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4219, 4207, 4061, 4097, 4004, 3937, 4086, 4157] avg=4096.0000 max=4219 max_rank=0 min=3937 min_rank=5 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16184 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2359, 0, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 1, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2492, 0, 0, 0, 1690, 26, 0, 0, 4096, 0, 0, 0, 47, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3963, 4071, 4108, 4095, 4208, 4149, 4096, 4078] avg=4096.0000 max=4208 max_rank=4 min=3963 min_rank=0 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2367, 8, 1164, 0, 0, 0, 0, 3, 3, 0, 29, 4064, 1, 0, 0, 1, 0, 4091, 0, 6, 0, 0, 0, 2, 0, 0, 1, 4093, 0, 0, 0, 2913, 2, 2, 0, 1685, 30, 0, 0, 4075, 3, 0, 0, 42, 1, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 2, 4086, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3539, 4100, 4098, 4096, 4632, 4121, 4094, 4088] avg=4096.0000 max=4632 max_rank=4 min=3539 min_rank=0 imbalance_ratio=1.130859 max_load_reduction=28136 max_load_reduction_pct=0.858643 moved_to_echo=16935 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2367, 0, 829, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 1, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3270, 0, 0, 0, 1684, 26, 0, 0, 4096, 0, 0, 0, 42, 1, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3196, 4067, 4102, 4095, 4980, 4139, 4100, 4089] avg=4096.0000 max=4980 max_rank=4 min=3196 min_rank=0 imbalance_ratio=1.215820 max_load_reduction=27788 max_load_reduction_pct=0.848022 moved_to_echo=17308 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2367, 19, 2892, 0, 0, 0, 0, 6, 4, 1, 45, 4063, 0, 0, 0, 0, 0, 4087, 0, 6, 0, 0, 0, 3, 0, 2, 3, 4092, 0, 0, 0, 1274, 1, 3, 0, 1684, 26, 0, 0, 3958, 4, 0, 0, 42, 1, 0, 0, 4090, 0, 4, 2, 0, 0, 0, 0, 0, 4, 4085, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5278, 4119, 4093, 4100, 2988, 4005, 4096, 4089] avg=4096.0000 max=5278 max_rank=0 min=2988 min_rank=4 imbalance_ratio=1.288574 max_load_reduction=27490 max_load_reduction_pct=0.838928 moved_to_echo=15178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 18944, 128, 13824, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19072, 128, 128, 0, 13568, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 128, 0, 0, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 2073, 0, 630, 0, 576, 1, 559, 3558, 0, 3080, 959, 2483, 0, 3258, 2403, 2515, 77, 0, 0, 1649, 1997, 3, 0, 90, 466, 2078, 1283, 4, 3024, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 1922, 0, 534, 0, 635, 2, 543, 3460, 0, 3037, 619, 1447, 0, 3555, 3337, 2993, 26, 0, 0, 2292, 2352, 0, 4, 66, 730, 1639, 844, 8, 2722, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2353, 0, 517, 0, 379, 0, 877, 3454, 0, 3155, 844, 2052, 0, 3459, 2899, 2662, 372, 0, 0, 1597, 1863, 0, 0, 24, 634, 2051, 661, 20, 2895, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2296, 4, 918, 0, 392, 0, 742, 3421, 0, 3021, 1104, 1277, 0, 3681, 3400, 2650, 164, 0, 0, 1563, 2159, 6, 0, 47, 673, 1944, 815, 4, 2487, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2, 3, 2227, 2, 371, 0, 495, 0, 773, 3324, 1, 3011, 684, 1803, 0, 3229, 2578, 2962, 349, 3, 6, 2187, 1597, 1, 2, 59, 319, 1669, 1840, 8, 3263, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[13, 1, 1977, 2, 688, 1, 651, 0, 1311, 3482, 3, 3154, 991, 2221, 0, 3706, 2896, 2849, 111, 0, 8, 1726, 1561, 2, 1, 54, 360, 1492, 559, 12, 2932, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 2107, 1, 772, 0, 586, 2, 765, 3552, 0, 3011, 452, 1665, 0, 3739, 3524, 2995, 72, 0, 15, 1721, 2481, 2, 5, 76, 420, 1591, 639, 5, 2567, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2, 1, 1812, 4, 454, 2, 532, 0, 741, 3415, 0, 2993, 632, 1997, 0, 3847, 3281, 2950, 91, 0, 0, 1797, 2589, 0, 0, 48, 567, 1648, 537, 24, 2804, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2353, 0, 1690, 255, 0, 0, 517, 0, 379, 0, 2950, 0, 0, 0, 877, 249, 0, 3155, 0, 0, 0, 0, 844, 2052, 0, 1403, 0, 0, 0, 0, 1065, 2662, 372, 0, 0, 0, 0, 0, 0, 1597, 1863, 0, 190, 0, 0, 0, 0, 24, 634, 2051, 1310, 366, 0, 0, 661, 20, 2895, 0, 334, 0, 0, 0] rerouted_tokens_per_ep_rank=[4298, 3846, 4281, 4299, 4099, 3650, 4385, 3910] avg=4096.0000 max=4385 max_rank=6 min=3650 min_rank=5 imbalance_ratio=1.070557 max_load_reduction=28383 max_load_reduction_pct=0.866180 moved_to_echo=16044 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2073, 0, 1592, 262, 0, 0, 630, 0, 576, 1, 3038, 0, 0, 0, 559, 258, 0, 3080, 0, 0, 0, 0, 959, 2483, 0, 1321, 0, 0, 0, 0, 883, 2515, 77, 0, 0, 0, 0, 0, 0, 1649, 1997, 3, 157, 0, 0, 0, 0, 90, 466, 2078, 1086, 345, 0, 0, 1283, 4, 3024, 1, 277, 0, 0, 0] rerouted_tokens_per_ep_rank=[3928, 4245, 3897, 4763, 3475, 3806, 4065, 4589] avg=4096.0000 max=4763 max_rank=3 min=3475 min_rank=4 imbalance_ratio=1.162842 max_load_reduction=28005 max_load_reduction_pct=0.854645 moved_to_echo=15935 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1922, 0, 1737, 255, 0, 0, 534, 0, 635, 2, 2955, 0, 0, 0, 543, 250, 0, 3037, 0, 0, 0, 0, 619, 1447, 0, 1442, 0, 0, 0, 0, 1227, 2993, 26, 0, 0, 0, 0, 0, 0, 2292, 2352, 0, 218, 0, 0, 0, 4, 66, 730, 1639, 1508, 376, 0, 0, 844, 8, 2722, 1, 384, 0, 0, 0] rerouted_tokens_per_ep_rank=[3914, 4126, 3830, 3508, 4246, 4862, 4323, 3959] avg=4096.0000 max=4862 max_rank=5 min=3508 min_rank=3 imbalance_ratio=1.187012 max_load_reduction=27906 max_load_reduction_pct=0.851624 moved_to_echo=17390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 2227, 2, 1581, 250, 0, 0, 371, 0, 495, 0, 2843, 0, 0, 0, 773, 231, 1, 3011, 0, 0, 0, 0, 684, 1803, 0, 1301, 0, 0, 0, 0, 937, 2962, 349, 3, 0, 0, 0, 0, 6, 2187, 1597, 1, 173, 0, 0, 0, 2, 59, 319, 1669, 1168, 347, 0, 0, 1840, 8, 3263, 0, 300, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 3709, 4016, 3788, 4251, 3964, 3564, 5411] avg=4096.0000 max=5411 max_rank=7 min=3564 min_rank=6 imbalance_ratio=1.321045 max_load_reduction=27357 max_load_reduction_pct=0.834869 moved_to_echo=17190 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2296, 4, 1799, 252, 0, 0, 918, 0, 392, 0, 2921, 0, 0, 0, 742, 248, 0, 3021, 0, 0, 0, 0, 1104, 1277, 0, 1492, 0, 0, 0, 0, 1250, 2650, 164, 0, 0, 0, 0, 0, 0, 1563, 2159, 6, 222, 0, 0, 0, 0, 47, 673, 1944, 1536, 390, 0, 0, 815, 4, 2487, 0, 392, 0, 0, 0] rerouted_tokens_per_ep_rank=[4351, 4231, 4011, 3873, 4064, 3950, 4590, 3698] avg=4096.0000 max=4590 max_rank=6 min=3698 min_rank=7 imbalance_ratio=1.120605 max_load_reduction=28178 max_load_reduction_pct=0.859924 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 2107, 1, 1827, 262, 0, 0, 772, 0, 586, 2, 3033, 0, 0, 0, 765, 257, 0, 3011, 0, 0, 0, 0, 452, 1665, 0, 1516, 0, 0, 0, 0, 1296, 2995, 72, 0, 0, 0, 0, 0, 15, 1721, 2481, 2, 230, 0, 0, 0, 5, 76, 420, 1591, 1592, 396, 0, 0, 639, 5, 2567, 0, 406, 0, 0, 0] rerouted_tokens_per_ep_rank=[4200, 4393, 4033, 3633, 4363, 4449, 4080, 3617] avg=4096.0000 max=4449 max_rank=5 min=3617 min_rank=7 imbalance_ratio=1.086182 max_load_reduction=28319 max_load_reduction_pct=0.864227 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[13, 1, 1977, 2, 1811, 257, 0, 0, 688, 1, 651, 0, 2974, 0, 0, 0, 1311, 251, 3, 3154, 0, 0, 0, 0, 991, 2221, 0, 1503, 0, 0, 0, 0, 1064, 2849, 111, 0, 0, 0, 0, 0, 8, 1726, 1561, 2, 189, 0, 0, 0, 1, 54, 360, 1492, 1309, 392, 0, 0, 559, 12, 2932, 4, 334, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4314, 4719, 4715, 4024, 3486, 3608, 3841] avg=4096.0000 max=4719 max_rank=2 min=3486 min_rank=5 imbalance_ratio=1.152100 max_load_reduction=28049 max_load_reduction_pct=0.855988 moved_to_echo=14959 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 1812, 4, 1880, 252, 0, 0, 454, 2, 532, 0, 2916, 0, 0, 0, 741, 247, 0, 2993, 0, 0, 0, 0, 632, 1997, 0, 1560, 0, 0, 0, 0, 1205, 2950, 91, 0, 0, 0, 0, 0, 0, 1797, 2589, 0, 215, 0, 0, 0, 0, 48, 567, 1648, 1483, 407, 0, 0, 537, 24, 2804, 0, 378, 0, 0, 0] rerouted_tokens_per_ep_rank=[3951, 3904, 3981, 4189, 4246, 4601, 4153, 3743] avg=4096.0000 max=4601 max_rank=5 min=3743 min_rank=7 imbalance_ratio=1.123291 max_load_reduction=28167 max_load_reduction_pct=0.859589 moved_to_echo=16743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6400, 14976, 0, 11648, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 16768, 128, 13952, 2048, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8960, 22656, 1280, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 14592, 16640, 128, 1664, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 4224, 14208, 11008, 3072, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6400, 2048, 128, 24576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7296, 128, 22784, 128, 2816, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4992, 128, 4352, 128, 23680, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 3123, 0, 14, 0, 0, 0, 4095, 1, 0, 0, 0, 4096, 0, 0, 1151, 0, 0, 0, 2, 0, 1, 4096, 0, 1, 0, 4074, 0, 0, 3923, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4091, 0, 2766, 0, 0, 3, 0, 3, 4093, 0, 4, 0, 9, 4092, 1, 12, 1366, 0, 1, 0, 7, 0, 0, 4093, 2, 0, 3, 4081, 0, 0, 4047, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 3578, 1, 1, 0, 0, 0, 4094, 0, 1, 0, 0, 4095, 1, 2, 562, 0, 0, 0, 3, 0, 0, 4094, 0, 0, 0, 4093, 0, 0, 4052, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 3092, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 2, 1098, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 4054, 0, 0, 4027, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4095, 1, 3124, 1, 11, 1, 0, 0, 4095, 0, 1, 0, 0, 4094, 0, 0, 986, 1, 0, 1, 0, 1, 0, 4095, 0, 2, 1, 4090, 2, 0, 4071, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 3595, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 534, 0, 0, 0, 3, 0, 0, 4096, 3, 0, 0, 4073, 0, 0, 4079, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 3139, 0, 1, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 1, 1136, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 3917, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4094, 2, 3830, 0, 0, 1, 0, 0, 4093, 1, 1, 0, 0, 4092, 4, 1, 376, 0, 0, 2, 7, 0, 3, 4093, 0, 8, 3, 4088, 0, 0, 3975, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[816, 0, 3123, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4010, 80, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1151, 0, 0, 0, 3195, 0, 0, 0, 2, 0, 1, 4092, 0, 0, 0, 0, 0, 1, 0, 4074, 5, 4, 2, 0, 0, 0, 3923, 85, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3939, 4104, 4096, 4094, 4346, 4095, 4086, 4008] avg=4096.0000 max=4346 max_rank=4 min=3939 min_rank=0 imbalance_ratio=1.061035 max_load_reduction=28422 max_load_reduction_pct=0.867371 moved_to_echo=16535 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[815, 0, 2766, 0, 0, 0, 0, 0, 0, 3, 0, 3, 4009, 80, 0, 0, 4093, 0, 4, 0, 0, 0, 0, 0, 9, 4090, 1, 12, 0, 0, 0, 0, 1366, 0, 1, 0, 3191, 0, 0, 0, 7, 0, 0, 4090, 0, 0, 0, 0, 2, 0, 3, 4081, 5, 3, 2, 0, 0, 0, 4047, 85, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3581, 4095, 4097, 4112, 4558, 4097, 4096, 4132] avg=4096.0000 max=4558 max_rank=4 min=3581 min_rank=0 imbalance_ratio=1.112793 max_load_reduction=28210 max_load_reduction_pct=0.860901 moved_to_echo=16883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[816, 0, 3595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 80, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 534, 0, 0, 0, 3195, 0, 0, 0, 3, 0, 0, 4092, 0, 0, 0, 0, 3, 0, 0, 4073, 5, 4, 2, 0, 0, 0, 4079, 85, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4411, 4091, 4097, 4094, 3729, 4095, 4087, 4164] avg=4096.0000 max=4411 max_rank=0 min=3729 min_rank=4 imbalance_ratio=1.076904 max_load_reduction=28357 max_load_reduction_pct=0.865387 moved_to_echo=16075 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[816, 1, 3124, 1, 0, 0, 0, 0, 11, 1, 0, 0, 4010, 80, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 986, 1, 0, 1, 3194, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 2, 1, 4090, 5, 4, 2, 0, 2, 0, 4071, 85, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3942, 4102, 4096, 4092, 4182, 4092, 4104, 4158] avg=4096.0000 max=4182 max_rank=4 min=3942 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16536 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[805, 0, 3578, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4016, 85, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 4086, 1, 2, 0, 0, 0, 0, 562, 0, 0, 0, 3196, 0, 0, 0, 3, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4093, 9, 6, 9, 1, 0, 0, 4052, 80, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4384, 4102, 4094, 4089, 3758, 4091, 4118, 4132] avg=4096.0000 max=4384 max_rank=0 min=3758 min_rank=4 imbalance_ratio=1.070312 max_load_reduction=28384 max_load_reduction_pct=0.866211 moved_to_echo=16099 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[816, 0, 3092, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4011, 80, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 2, 0, 0, 0, 0, 1098, 0, 0, 0, 3195, 0, 0, 0, 13, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4054, 5, 4, 2, 0, 0, 0, 4027, 85, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3908, 4092, 4096, 4097, 4293, 4105, 4065, 4112] avg=4096.0000 max=4293 max_rank=4 min=3908 min_rank=0 imbalance_ratio=1.048096 max_load_reduction=28475 max_load_reduction_pct=0.868988 moved_to_echo=16575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[816, 0, 3139, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4011, 80, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 1136, 0, 0, 0, 3195, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 4, 2, 0, 0, 0, 3917, 85, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3955, 4092, 4097, 4095, 4331, 4093, 4103, 4002] avg=4096.0000 max=4331 max_rank=4 min=3955 min_rank=0 imbalance_ratio=1.057373 max_load_reduction=28437 max_load_reduction_pct=0.867828 moved_to_echo=16529 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[816, 2, 3830, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4009, 80, 0, 0, 4093, 1, 1, 0, 0, 0, 0, 0, 0, 4090, 4, 1, 0, 0, 0, 0, 376, 0, 0, 2, 3193, 0, 0, 0, 7, 0, 3, 4090, 0, 0, 0, 0, 0, 8, 3, 4088, 5, 3, 2, 0, 0, 0, 3975, 85, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4648, 4090, 4095, 4095, 3571, 4100, 4109, 4060] avg=4096.0000 max=4648 max_rank=0 min=3571 min_rank=4 imbalance_ratio=1.134766 max_load_reduction=28120 max_load_reduction_pct=0.858154 moved_to_echo=15840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7296, 128, 128, 128, 25600, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6528, 128, 26368, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32128, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32128, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33664 slack_slots=1792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 1842, 81, 0, 2196, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 496, 273, 0, 3346, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4094, 4092, 0, 1, 4094, 1, 2228, 199, 1, 1670, 4095, 1, 1, 1, 0, 4094, 1, 1, 0, 2, 4095, 0, 0, 0, 4, 1, 0, 4088, 4, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 682, 98, 0, 3340, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2, 4074, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 1397, 285, 0, 2425, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 558, 544, 0, 3020, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4089, 0, 677, 442, 0, 3013, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4093, 4092, 0, 2, 4094, 1, 1302, 474, 0, 2319, 4094, 0, 2, 2, 0, 4094, 2, 2, 0, 2, 4095, 0, 0, 0, 3, 0, 0, 4094, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2651, 0, 1842, 81, 0, 0, 0, 0, 0, 2196, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 1431, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 14, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4574, 3627, 4096, 4096, 4096, 4087, 4096] avg=4096.0000 max=4574 max_rank=1 min=3627 min_rank=2 imbalance_ratio=1.116699 max_load_reduction=28194 max_load_reduction_pct=0.860413 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 0, 0, 2651, 0, 496, 273, 0, 0, 0, 0, 0, 3346, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 1431, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 14, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3420, 4777, 4096, 4096, 4096, 4091, 4095] avg=4096.0000 max=4777 max_rank=2 min=3420 min_rank=1 imbalance_ratio=1.166260 max_load_reduction=27991 max_load_reduction_pct=0.854218 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4092, 0, 1, 0, 0, 0, 0, 2650, 1, 2228, 199, 0, 0, 0, 0, 1, 1670, 1431, 1, 0, 0, 0, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 1, 1, 0, 2, 2664, 1430, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 4088, 14, 0, 0, 0, 4, 0, 0, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 5078, 3103, 4096, 4098, 4095, 4107, 4097] avg=4096.0000 max=5078 max_rank=1 min=3103 min_rank=2 imbalance_ratio=1.239746 max_load_reduction=27690 max_load_reduction_pct=0.845032 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 0, 0, 2651, 0, 558, 544, 0, 0, 0, 0, 0, 3020, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2657, 1431, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 14, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3753, 4447, 4096, 4088, 4095, 4096, 4096] avg=4096.0000 max=4447 max_rank=2 min=3753 min_rank=1 imbalance_ratio=1.085693 max_load_reduction=28321 max_load_reduction_pct=0.864288 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2637, 0, 682, 98, 0, 0, 0, 0, 0, 3340, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2667, 1435, 0, 0, 4093, 0, 0, 0, 4, 2, 0, 0, 0, 0, 2, 4074, 20, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3417, 4764, 4094, 4102, 4099, 4096, 4100] avg=4096.0000 max=4764 max_rank=2 min=3417 min_rank=1 imbalance_ratio=1.163086 max_load_reduction=28004 max_load_reduction_pct=0.854614 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 0, 0, 2651, 0, 1397, 285, 0, 0, 0, 0, 0, 2425, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 1431, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 14, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4333, 3856, 4096, 4096, 4096, 4099, 4095] avg=4096.0000 max=4333 max_rank=1 min=3856 min_rank=2 imbalance_ratio=1.057861 max_load_reduction=28435 max_load_reduction_pct=0.867767 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2647, 0, 677, 442, 0, 0, 0, 0, 0, 3013, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 1428, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 14, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3766, 4444, 4096, 4093, 4096, 4081, 4096] avg=4096.0000 max=4444 max_rank=2 min=3766 min_rank=1 imbalance_ratio=1.084961 max_load_reduction=28324 max_load_reduction_pct=0.864380 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4092, 0, 2, 0, 0, 0, 0, 2650, 1, 1302, 474, 0, 0, 0, 0, 0, 2319, 1431, 0, 0, 0, 0, 0, 2, 2, 0, 4094, 0, 0, 0, 0, 2, 2, 0, 2, 2663, 1430, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4094, 14, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4427, 3750, 4098, 4099, 4095, 4111, 4093] avg=4096.0000 max=4427 max_rank=1 min=3750 min_rank=2 imbalance_ratio=1.080811 max_load_reduction=28341 max_load_reduction_pct=0.864899 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 21376, 11520, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 21376, 11520, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21248, 128, 9216, 2432, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32640, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 4096, 245, 0, 0, 0, 4096, 0, 0, 0, 0, 99, 0, 0, 4096, 3310, 0, 0, 0, 0, 4096, 0, 0, 0, 442, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 4091, 4096, 0, 0, 0, 4094, 356, 0, 0, 0, 4096, 0, 0, 0, 0, 665, 0, 0, 4088, 2367, 0, 0, 6, 6, 4096, 0, 0, 0, 708, 10, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 4096, 373, 0, 0, 0, 4096, 0, 0, 0, 0, 281, 0, 0, 4095, 2738, 0, 0, 1, 22, 4096, 0, 0, 0, 682, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 4094, 4096, 0, 0, 0, 4096, 884, 0, 0, 0, 4096, 0, 0, 0, 0, 941, 0, 0, 4094, 1705, 0, 1, 2, 3, 4096, 0, 0, 0, 564, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4, 4094, 4094, 1, 2, 1, 4094, 543, 0, 0, 2, 4094, 0, 0, 2, 2, 635, 0, 3, 4094, 896, 1, 31, 0, 36, 4093, 0, 0, 0, 1955, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 4095, 4096, 0, 0, 0, 4096, 336, 0, 0, 0, 4096, 0, 0, 0, 0, 774, 0, 0, 4096, 1736, 0, 0, 0, 0, 4096, 0, 0, 0, 1251, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 4096, 129, 0, 0, 0, 4096, 0, 0, 0, 0, 218, 0, 0, 4096, 3572, 0, 0, 0, 4, 4096, 0, 0, 0, 173, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[3, 4094, 4093, 2, 3, 1, 4093, 151, 0, 0, 2, 4093, 0, 0, 2, 2, 311, 8, 2, 4094, 1678, 0, 0, 0, 22, 4092, 0, 3, 0, 1933, 0, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3719, 373, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 281, 0, 0, 3605, 0, 0, 0, 0, 2738, 0, 0, 1, 962, 490, 377, 11, 22, 4085, 0, 0, 0, 0, 0, 0, 0, 682, 0, 3134, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4092, 4096, 4096, 3886, 4579, 4107, 3816] avg=4096.0000 max=4579 max_rank=5 min=3816 min_rank=7 imbalance_ratio=1.117920 max_load_reduction=28189 max_load_reduction_pct=0.860260 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 3717, 356, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 665, 0, 0, 3599, 0, 0, 0, 0, 2367, 0, 0, 6, 961, 489, 377, 11, 6, 4085, 0, 0, 0, 0, 0, 0, 0, 708, 10, 3128, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4073, 4096, 4096, 4264, 4211, 4091, 3846] avg=4096.0000 max=4264 max_rank=4 min=3846 min_rank=7 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16412 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3719, 245, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 99, 0, 0, 3606, 0, 0, 0, 0, 3310, 0, 0, 0, 962, 490, 377, 11, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 442, 0, 3134, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3964, 4096, 4096, 3705, 5150, 4085, 3576] avg=4096.0000 max=5150 max_rank=5 min=3576 min_rank=7 imbalance_ratio=1.257324 max_load_reduction=27618 max_load_reduction_pct=0.842834 moved_to_echo=16516 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 4094, 1, 1, 0, 0, 0, 0, 2, 1, 3717, 543, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 2, 2, 4093, 0, 0, 0, 635, 0, 3, 3604, 0, 0, 0, 0, 896, 1, 31, 0, 961, 490, 377, 10, 36, 4083, 0, 0, 0, 0, 0, 0, 0, 1955, 0, 3130, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4263, 4096, 4097, 4242, 2766, 4119, 5085] avg=4096.0000 max=5085 max_rank=7 min=2766 min_rank=5 imbalance_ratio=1.241455 max_load_reduction=27683 max_load_reduction_pct=0.844818 moved_to_echo=16212 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3717, 884, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 941, 0, 0, 3601, 0, 0, 0, 0, 1705, 0, 1, 2, 966, 493, 379, 13, 3, 4082, 0, 0, 0, 0, 0, 0, 0, 564, 0, 3130, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4601, 4097, 4096, 4542, 3559, 4085, 3694] avg=4096.0000 max=4601 max_rank=1 min=3559 min_rank=5 imbalance_ratio=1.123291 max_load_reduction=28167 max_load_reduction_pct=0.859589 moved_to_echo=15880 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3719, 336, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 774, 0, 0, 3606, 0, 0, 0, 0, 1736, 0, 0, 0, 962, 490, 377, 11, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 1251, 0, 3134, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4055, 4096, 4096, 4380, 3576, 4085, 4385] avg=4096.0000 max=4385 max_rank=7 min=3576 min_rank=5 imbalance_ratio=1.070557 max_load_reduction=28383 max_load_reduction_pct=0.866180 moved_to_echo=16426 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3719, 129, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 218, 0, 0, 3606, 0, 0, 0, 0, 3572, 0, 0, 0, 962, 490, 377, 11, 4, 4085, 0, 0, 0, 0, 0, 0, 0, 173, 0, 3134, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3848, 4096, 4095, 3824, 5412, 4089, 3307] avg=4096.0000 max=5412 max_rank=5 min=3307 min_rank=7 imbalance_ratio=1.321289 max_load_reduction=27356 max_load_reduction_pct=0.834839 moved_to_echo=16632 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 4094, 1, 2, 0, 0, 0, 0, 3, 1, 3717, 151, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 2, 2, 4092, 0, 0, 0, 311, 8, 2, 3604, 0, 0, 0, 0, 1678, 0, 0, 0, 960, 490, 376, 10, 22, 4082, 0, 3, 0, 0, 0, 0, 0, 1933, 0, 3126, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 3872, 4095, 4096, 3925, 3514, 4107, 5059] avg=4096.0000 max=5059 max_rank=7 min=3514 min_rank=5 imbalance_ratio=1.235107 max_load_reduction=27709 max_load_reduction_pct=0.845612 moved_to_echo=16605 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32767 actual_valid_recv=33408 observed_loss=-641 tokens_per_expert=[18048, 128, 128, 128, 7808, 3968, 3072, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 7808, 128, 25088, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 128, 128, 28928, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 29824, 3072, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32769 actual_valid_recv=33152 observed_loss=-383 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4096, 0, 4096, 0, 4087, 0, 4093, 0, 0, 0, 0, 1, 0, 4094, 4096, 0, 0, 4096, 0, 7, 4096, 4, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4096, 0, 4089, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4096, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 26, 4096, 0, 4095, 0, 3978, 0, 4075, 0, 0, 0, 0, 0, 8, 4090, 4096, 0, 0, 4087, 0, 0, 4096, 121, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4096, 0, 4096, 0, 4046, 0, 4093, 0, 0, 0, 0, 1, 2, 4094, 4096, 0, 0, 4094, 0, 0, 4096, 50, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4096, 0, 4091, 0, 4080, 0, 4095, 0, 0, 0, 0, 0, 0, 4043, 4096, 0, 0, 4095, 0, 0, 4096, 75, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 3, 5, 4094, 1, 4094, 0, 4075, 0, 4093, 2, 2, 2, 0, 0, 2, 4083, 4092, 1, 2, 4095, 0, 0, 4093, 0, 0, 0, 0, 2, 25, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2, 2, 3, 0, 4094, 0, 4093, 0, 4092, 0, 4092, 2, 3, 3, 0, 0, 2, 4088, 4093, 2, 3, 4094, 0, 0, 4093, 1, 0, 0, 0, 2, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4091, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4057, 33, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 4087, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 1, 0, 0, 0, 4094, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 4063, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 7, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4099, 4116, 4095, 4104, 4103, 4067, 4092] avg=4096.0000 max=4116 max_rank=2 min=4067 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4060, 33, 0, 0, 3, 0, 4089, 0, 0, 0, 0, 0, 4090, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 4089, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4063, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 7, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4092, 4119, 4094, 4100, 4096, 4082, 4091] avg=4096.0000 max=4119 max_rank=2 min=4082 min_rank=6 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 26, 4039, 33, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 3978, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 8, 4090, 11, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4063, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 7, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4098, 4007, 4094, 4109, 4087, 4184, 4091] avg=4096.0000 max=4184 max_rank=6 min=4007 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 3, 5, 4057, 33, 0, 0, 3, 1, 4094, 0, 0, 0, 0, 0, 4075, 0, 29, 2, 0, 0, 0, 0, 2, 2, 0, 0, 4091, 1, 0, 0, 2, 4083, 10, 1, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 2, 25, 0, 4081, 7, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4098, 4106, 4096, 4096, 4097, 4060, 4115] avg=4096.0000 max=4115 max_rank=7 min=4060 min_rank=6 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4058, 40, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4046, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 7, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 4053, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 13, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4097, 4068, 4103, 4096, 4097, 4103, 4102] avg=4096.0000 max=4103 max_rank=3 min=4068 min_rank=2 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4059, 33, 0, 0, 3, 0, 4091, 0, 0, 0, 0, 0, 4080, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 4043, 11, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4063, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 7, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4094, 4109, 4094, 4054, 4095, 4138, 4091] avg=4096.0000 max=4138 max_rank=6 min=4054 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4060, 33, 0, 0, 3, 0, 4091, 0, 0, 0, 0, 0, 4091, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4063, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 7, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4094, 4120, 4094, 4107, 4096, 4073, 4091] avg=4096.0000 max=4120 max_rank=2 min=4073 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 3, 0, 4056, 33, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 4092, 0, 29, 2, 0, 0, 0, 0, 3, 3, 0, 0, 4091, 1, 0, 0, 2, 4088, 10, 2, 0, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 4060, 1, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4082, 7, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4123, 4098, 4102, 4097, 4061, 4095] avg=4096.0000 max=4123 max_rank=2 min=4061 min_rank=6 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32512, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 0, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 524, 4096, 0, 0, 4096, 3572, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1613, 4096, 0, 0, 4096, 2483, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1174, 4096, 0, 0, 4096, 2922, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 959, 4096, 0, 0, 4096, 3137, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 792, 4096, 0, 0, 4096, 3304, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4092, 4, 4093, 0, 3, 0, 2, 3, 1677, 4093, 0, 0, 4092, 2417, 0, 3, 0, 2, 2, 0, 4093, 1, 4094, 0, 2, 1, 1, 4090, 2, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1477, 4096, 0, 0, 4096, 2619, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4091, 8, 4092, 0, 3, 0, 2, 4, 787, 4092, 0, 0, 4086, 3307, 0, 0, 0, 6, 2, 0, 4092, 4, 4093, 0, 3, 0, 1, 4090, 3, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1174, 2971, 0, 0, 0, 0, 0, 0, 1127, 2922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2969, 1125, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4145, 4049, 4094, 4096, 4096, 4096] avg=4096.0000 max=4145 max_rank=2 min=4049 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 524, 2971, 0, 0, 0, 0, 0, 0, 1127, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2969, 1125, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 3495, 4699, 4095, 4096, 4096, 4096] avg=4096.0000 max=4699 max_rank=3 min=3495 min_rank=2 imbalance_ratio=1.147217 max_load_reduction=28069 max_load_reduction_pct=0.856598 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 792, 2971, 0, 0, 0, 0, 0, 0, 1127, 3304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2969, 1125, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 3763, 4431, 4094, 4096, 4096, 4096] avg=4096.0000 max=4431 max_rank=3 min=3763 min_rank=2 imbalance_ratio=1.081787 max_load_reduction=28337 max_load_reduction_pct=0.864777 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 4, 0, 0, 0, 0, 0, 0, 3, 0, 2, 4092, 0, 0, 0, 3, 1677, 2969, 0, 0, 0, 0, 0, 0, 1126, 2417, 0, 0, 0, 0, 0, 3, 0, 2, 2, 2966, 1124, 1, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 2, 1, 1, 4094, 0, 0, 0, 4090, 2, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4649, 3543, 4098, 4094, 4098, 4093] avg=4096.0000 max=4649 max_rank=2 min=3543 min_rank=3 imbalance_ratio=1.135010 max_load_reduction=28119 max_load_reduction_pct=0.858124 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 1613, 2966, 0, 0, 0, 0, 0, 0, 1122, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2974, 1129, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4097, 4579, 3605, 4103, 4096, 4095, 4100] avg=4096.0000 max=4579 max_rank=2 min=3605 min_rank=3 imbalance_ratio=1.117920 max_load_reduction=28189 max_load_reduction_pct=0.860260 moved_to_echo=16394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 959, 2971, 0, 0, 0, 0, 0, 0, 1127, 3137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2969, 1125, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 3930, 4264, 4094, 4096, 4096, 4096] avg=4096.0000 max=4264 max_rank=3 min=3930 min_rank=2 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1477, 2971, 0, 0, 0, 0, 0, 0, 1127, 2619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2969, 1125, 1, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4448, 3746, 4095, 4097, 4095, 4096] avg=4096.0000 max=4448 max_rank=2 min=3746 min_rank=3 imbalance_ratio=1.085938 max_load_reduction=28320 max_load_reduction_pct=0.864258 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 8, 0, 0, 0, 0, 0, 0, 3, 0, 2, 4091, 0, 0, 0, 4, 787, 2968, 0, 0, 0, 0, 0, 0, 1124, 3307, 0, 0, 0, 0, 0, 0, 0, 6, 2, 2962, 1124, 1, 0, 0, 4092, 4, 1, 0, 0, 0, 0, 0, 3, 0, 1, 4092, 0, 0, 0, 4090, 3, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 3759, 4431, 4095, 4097, 4096, 4095] avg=4096.0000 max=4431 max_rank=3 min=3759 min_rank=2 imbalance_ratio=1.081787 max_load_reduction=28337 max_load_reduction_pct=0.864777 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 23808, 9088, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 9088, 23808, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 9088, 23808, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 3976, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 120, 0, 0, 0, 0, 4095, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4094, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 2, 1, 5, 4071, 4094, 0, 0, 0, 0, 3, 4088, 1, 4093, 4092, 0, 3, 4094, 26, 0, 4, 0, 0, 4092, 1, 1, 2, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3946, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 150, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 3, 2, 8, 4089, 4092, 0, 0, 0, 0, 2, 4087, 0, 4092, 4091, 0, 3, 4093, 1, 3, 5, 0, 1, 4093, 0, 3, 3, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 0, 0, 1, 0, 0, 0, 4055, 37, 0, 0, 3976, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4059, 120, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4093, 4016, 4096, 4096, 4179, 4095, 4096] avg=4096.0000 max=4179 max_rank=5 min=4016 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 4, 0, 0, 0, 4054, 37, 0, 0, 4095, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4057, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4134, 4096, 4098, 4058, 4096, 4096] avg=4096.0000 max=4134 max_rank=2 min=4058 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 4055, 37, 0, 0, 4093, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4059, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4092, 4133, 4096, 4098, 4062, 4096, 4096] avg=4096.0000 max=4133 max_rank=2 min=4062 min_rank=5 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4092, 1, 0, 0, 0, 2, 1, 5, 4054, 37, 0, 0, 4071, 39, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4088, 0, 0, 0, 0, 1, 1, 4092, 0, 0, 0, 0, 0, 3, 4057, 26, 0, 0, 0, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 1, 1, 2, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4099, 4110, 4091, 4094, 4086, 4096, 4097] avg=4096.0000 max=4110 max_rank=2 min=4086 min_rank=5 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4060, 43, 0, 0, 4093, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 4, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4047, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4103, 4127, 4108, 4096, 4050, 4094, 4092] avg=4096.0000 max=4127 max_rank=2 min=4050 min_rank=5 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4055, 37, 0, 0, 4094, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4059, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4092, 4134, 4096, 4096, 4061, 4096, 4096] avg=4096.0000 max=4134 max_rank=2 min=4061 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 4055, 37, 0, 0, 3946, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4059, 150, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4092, 3986, 4096, 4098, 4209, 4096, 4096] avg=4096.0000 max=4209 max_rank=5 min=3986 min_rank=2 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 4091, 1, 0, 0, 0, 3, 2, 8, 4052, 37, 0, 0, 4089, 39, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4087, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 3, 4056, 1, 3, 0, 0, 0, 0, 5, 0, 1, 4093, 0, 0, 0, 0, 0, 3, 3, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4102, 4128, 4089, 4092, 4063, 4099, 4099] avg=4096.0000 max=4128 max_rank=2 min=4063 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32512, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32512, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2590, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 2, 4096, 4096, 0, 0, 1505, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 1, 3128, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 968, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 11, 2263, 0, 0, 0, 0, 4085, 4096, 4096, 0, 0, 4, 21, 4096, 4095, 0, 0, 1808, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2398, 0, 0, 0, 0, 4093, 4096, 4096, 0, 0, 0, 7, 4096, 4096, 0, 0, 1694, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 3, 4, 1, 4093, 4092, 1, 2, 0, 2861, 3, 0, 0, 0, 4093, 4086, 4093, 0, 3, 0, 0, 4092, 4093, 3, 0, 1237, 1, 4, 0, 1, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3384, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 8, 4096, 4096, 0, 0, 705, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3136, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 960, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3, 4, 5, 0, 4093, 4090, 0, 3, 0, 2320, 4, 0, 0, 0, 4093, 4086, 4091, 0, 3, 0, 0, 4092, 4092, 4, 0, 1777, 0, 5, 0, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2590, 0, 0, 1329, 4, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 1505, 0, 0, 2767, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 3923, 4095, 4096, 4094, 4273, 4096] avg=4096.0000 max=4273 max_rank=6 min=3923 min_rank=2 imbalance_ratio=1.043213 max_load_reduction=28495 max_load_reduction_pct=0.869598 moved_to_echo=16559 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 1, 3128, 0, 0, 1329, 4, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 968, 0, 0, 2763, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4097, 4462, 4099, 4096, 4092, 3732, 4096] avg=4096.0000 max=4462 max_rank=2 min=3732 min_rank=6 imbalance_ratio=1.089355 max_load_reduction=28306 max_load_reduction_pct=0.863831 moved_to_echo=16016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 11, 2263, 0, 0, 1329, 4, 0, 0, 0, 0, 4085, 4, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 21, 0, 4091, 0, 0, 0, 0, 0, 0, 1808, 0, 0, 2763, 1, 0, 0, 0, 0, 1, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 3607, 4089, 4100, 4112, 4572, 4098] avg=4096.0000 max=4572 max_rank=6 min=3607 min_rank=2 imbalance_ratio=1.116211 max_load_reduction=28196 max_load_reduction_pct=0.860474 moved_to_echo=16882 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4, 1, 4091, 0, 0, 0, 1, 4092, 1, 2, 0, 0, 0, 0, 0, 2861, 3, 0, 1326, 4, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 1237, 1, 4, 2756, 1, 0, 0, 0, 1, 0, 2, 4092, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 4194, 4097, 4096, 4092, 3999, 4095] avg=4096.0000 max=4194 max_rank=2 min=3999 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16282 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2398, 0, 0, 1333, 9, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4087, 0, 0, 0, 0, 0, 0, 1694, 0, 0, 2763, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 3742, 4093, 4094, 4094, 4458, 4096] avg=4096.0000 max=4458 max_rank=6 min=3742 min_rank=2 imbalance_ratio=1.088379 max_load_reduction=28310 max_load_reduction_pct=0.863953 moved_to_echo=16742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3384, 0, 0, 1329, 4, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4092, 0, 0, 0, 0, 0, 0, 705, 0, 0, 2764, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4717, 4098, 4096, 4100, 3471, 4096] avg=4096.0000 max=4717 max_rank=2 min=3471 min_rank=6 imbalance_ratio=1.151611 max_load_reduction=28051 max_load_reduction_pct=0.856049 moved_to_echo=15763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 3136, 0, 0, 1329, 4, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 960, 0, 0, 2763, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4097, 4469, 4100, 4096, 4092, 3724, 4096] avg=4096.0000 max=4469 max_rank=2 min=3724 min_rank=6 imbalance_ratio=1.091064 max_load_reduction=28299 max_load_reduction_pct=0.863617 moved_to_echo=16008 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 4, 5, 0, 4091, 0, 0, 0, 1, 4090, 0, 3, 0, 0, 0, 0, 0, 2320, 4, 0, 1326, 4, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 4091, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 0, 0, 1777, 0, 5, 2756, 1, 0, 0, 0, 0, 0, 3, 4092, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4094, 3654, 4097, 4094, 4092, 4539, 4095] avg=4096.0000 max=4539 max_rank=6 min=3654 min_rank=2 imbalance_ratio=1.108154 max_load_reduction=28229 max_load_reduction_pct=0.861481 moved_to_echo=16820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 22144, 128, 0, 10752, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 10752, 128, 128, 22144, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 6, 0, 0, 0, 4091, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 30, 0, 0, 0, 4078, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4075, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 6, 0, 0, 0, 4089, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4094, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 39, 0, 0, 0, 4089, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4062, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 20, 0, 0, 0, 4083, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4084, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4093, 0, 4092, 1, 0, 0, 3, 3, 4089, 4090, 4087, 2, 3, 3, 0, 2, 0, 3, 0, 4, 16, 0, 3, 1, 0, 4094, 0, 1, 4093, 4085, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 1, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4092, 0, 4089, 0, 0, 0, 3, 4, 4092, 4091, 4086, 3, 4, 4, 0, 2, 0, 7, 0, 6, 18, 0, 4, 1, 0, 4094, 0, 1, 4092, 4075, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4078, 4, 0, 0, 4091, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4088, 4099, 4095, 4096, 4091, 4105, 4098] avg=4096.0000 max=4105 max_rank=6 min=4088 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16390 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4096, 0, 0, 0, 0, 0, 30, 0, 0, 0, 4079, 4, 0, 0, 4078, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 9, 0, 0, 0, 4087, 1, 0, 0, 0, 4075, 0, 0, 9, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4113, 4086, 4095, 4094, 4097, 4084, 4101] avg=4096.0000 max=4113 max_rank=1 min=4084 min_rank=6 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4079, 4, 0, 0, 4089, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 3, 0, 0, 0, 4087, 1, 0, 0, 0, 4094, 0, 0, 9, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4089, 4097, 4095, 4094, 4091, 4103, 4101] avg=4096.0000 max=4103 max_rank=6 min=4089 min_rank=1 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 3, 3, 4071, 4, 0, 0, 4089, 0, 8, 2, 0, 0, 0, 0, 3, 3, 0, 2, 4089, 0, 0, 0, 0, 3, 0, 4, 4090, 1, 0, 0, 16, 0, 3, 1, 4084, 1, 0, 0, 0, 4094, 0, 1, 8, 0, 0, 0, 5, 4085, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4081, 4099, 4097, 4098, 4105, 4103, 4090] avg=4096.0000 max=4105 max_rank=5 min=4081 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4085, 5, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 5, 0, 0, 0, 4091, 1, 0, 0, 0, 4084, 0, 0, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4110, 4083, 4096, 4095, 4097, 4095, 4096] avg=4096.0000 max=4110 max_rank=1 min=4083 min_rank=2 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 39, 0, 0, 0, 4079, 4, 0, 0, 4089, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 2, 0, 0, 0, 4087, 3, 0, 0, 0, 4062, 0, 0, 9, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4122, 4097, 4095, 4094, 4092, 4071, 4101] avg=4096.0000 max=4122 max_rank=1 min=4071 min_rank=6 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16358 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4079, 4, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4087, 1, 0, 0, 0, 4095, 0, 0, 9, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4084, 4104, 4095, 4094, 4088, 4104, 4101] avg=4096.0000 max=4104 max_rank=2 min=4084 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4070, 4, 0, 0, 4092, 0, 8, 3, 0, 0, 0, 0, 4, 4, 0, 2, 4090, 0, 0, 0, 0, 7, 0, 6, 4089, 1, 0, 0, 18, 0, 4, 1, 4083, 1, 0, 0, 0, 4094, 0, 1, 8, 0, 0, 0, 5, 4075, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4081, 4103, 4100, 4103, 4107, 4103, 4080] avg=4096.0000 max=4107 max_rank=5 min=4080 min_rank=7 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4094, 0, 0, 4091, 0, 0, 30, 0, 0, 0, 0, 0, 9, 4064, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 2, 0, 4096, 0, 4096, 0, 0, 4085, 0, 0, 4084, 0, 0, 164, 0, 0, 0, 1, 0, 22, 3932, 0, 0, 0, 4094, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0, 0, 759, 0, 0, 0, 0, 0, 3, 3337, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 2, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 4092, 0, 0, 339, 0, 0, 0, 0, 0, 2, 3759, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 4090, 0, 0, 897, 0, 0, 0, 0, 0, 2, 3204, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 0, 878, 0, 0, 0, 0, 0, 1, 3219, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4093, 4092, 0, 3, 4093, 1, 4093, 2, 0, 4094, 0, 3, 4070, 2, 3, 821, 2, 0, 2, 0, 16, 0, 3278, 2, 4, 0, 4089, 0, 0, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4089, 4090, 0, 4, 4090, 2, 4092, 2, 0, 4093, 0, 4, 4057, 2, 3, 849, 4, 0, 3, 0, 23, 0, 3253, 3, 5, 0, 4091, 0, 4, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 3510, 0, 0, 897, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 3204, 0, 579, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4407, 4096, 3786, 4096, 4095] avg=4096.0000 max=4407 max_rank=3 min=3786 min_rank=5 imbalance_ratio=1.075928 max_load_reduction=28361 max_load_reduction_pct=0.865509 moved_to_echo=16073 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 1, 0, 0, 0, 3504, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 22, 3932, 0, 579, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4086, 3668, 4096, 4534, 4094, 4096] avg=4096.0000 max=4534 max_rank=5 min=3668 min_rank=3 imbalance_ratio=1.106934 max_load_reduction=28234 max_load_reduction_pct=0.861633 moved_to_echo=16820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3513, 0, 0, 878, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 3219, 0, 580, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4097, 4391, 4095, 3801, 4096, 4096] avg=4096.0000 max=4391 max_rank=3 min=3801 min_rank=5 imbalance_ratio=1.072021 max_load_reduction=28377 max_load_reduction_pct=0.865997 moved_to_echo=16088 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 6, 0, 0, 0, 3498, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 4064, 0, 582, 1, 0, 0, 0, 0, 4096, 0, 5, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4100, 3528, 4096, 4656, 4102, 4095] avg=4096.0000 max=4656 max_rank=5 min=3528 min_rank=3 imbalance_ratio=1.136719 max_load_reduction=28112 max_load_reduction_pct=0.857910 moved_to_echo=16949 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 3, 0, 0, 0, 0, 4093, 1, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 3, 1, 0, 0, 0, 3492, 2, 3, 821, 0, 0, 0, 0, 2, 0, 2, 0, 4092, 0, 0, 0, 16, 0, 3278, 2, 577, 1, 0, 0, 4, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4092, 1, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4098, 4318, 4096, 3874, 4093, 4098] avg=4096.0000 max=4318 max_rank=3 min=3874 min_rank=5 imbalance_ratio=1.054199 max_load_reduction=28450 max_load_reduction_pct=0.868225 moved_to_echo=16161 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3512, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3, 3337, 0, 580, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4097, 4271, 4095, 3921, 4096, 4095] avg=4096.0000 max=4271 max_rank=3 min=3921 min_rank=5 imbalance_ratio=1.042725 max_load_reduction=28497 max_load_reduction_pct=0.869659 moved_to_echo=16207 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 3511, 0, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 3759, 0, 580, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4096, 3850, 4096, 4342, 4095, 4095] avg=4096.0000 max=4342 max_rank=5 min=3850 min_rank=3 imbalance_ratio=1.060059 max_load_reduction=28426 max_load_reduction_pct=0.867493 moved_to_echo=16628 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 4, 0, 0, 0, 0, 4090, 2, 1, 2, 0, 0, 0, 0, 0, 4093, 0, 4, 1, 0, 0, 0, 3481, 2, 3, 849, 0, 0, 0, 0, 4, 0, 3, 0, 4091, 0, 0, 0, 23, 0, 3253, 3, 575, 0, 0, 0, 5, 0, 4091, 0, 0, 0, 0, 0, 4, 0, 5, 0, 4089, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4095, 4098, 4335, 4098, 3854, 4096, 4098] avg=4096.0000 max=4335 max_rank=3 min=3854 min_rank=5 imbalance_ratio=1.058350 max_load_reduction=28433 max_load_reduction_pct=0.867706 moved_to_echo=16146 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[28032, 128, 128, 4864, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 28160, 128, 4736, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 32768, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 4096, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4, 2, 4092, 0, 2, 1, 0, 0, 4092, 1, 2, 3, 0, 5, 3, 4093, 0, 4, 2, 4092, 4089, 4076, 1, 4080, 4093, 27, 0, 0, 0, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4070, 0, 4095, 4096, 26, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 3, 0, 0, 4096, 4096, 4095, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 14, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4078, 0, 4079, 4096, 20, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 1, 0, 0, 4096, 4096, 4093, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[6, 2, 4090, 0, 2, 1, 0, 0, 4092, 0, 3, 4, 0, 0, 4, 4091, 0, 6, 3, 4091, 4087, 4040, 2, 4074, 4092, 68, 3, 0, 0, 0, 1, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 4074, 4096, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 2, 4090, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4089, 1, 0, 0, 4092, 1, 2, 3, 0, 0, 0, 0, 0, 5, 3, 4089, 0, 0, 0, 0, 0, 4, 2, 4092, 0, 0, 0, 0, 0, 4076, 1, 16, 0, 0, 0, 0, 4076, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4063, 17, 4, 2] rerouted_tokens_per_ep_rank=[4096, 4093, 4098, 4097, 4098, 4093, 4103, 4090] avg=4096.0000 max=4103 max_rank=6 min=4090 min_rank=7 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4078, 0, 16, 0, 0, 0, 0, 4079, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 17, 4, 2] rerouted_tokens_per_ep_rank=[4108, 4097, 4096, 4092, 4096, 4094, 4100, 4085] avg=4096.0000 max=4108 max_rank=0 min=4085 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16375 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4070, 0, 16, 0, 0, 0, 0, 4079, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 17, 4, 2] rerouted_tokens_per_ep_rank=[4095, 4097, 4096, 4092, 4096, 4086, 4105, 4101] avg=4096.0000 max=4105 max_rank=6 min=4086 min_rank=5 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 17, 4, 2] rerouted_tokens_per_ep_rank=[4094, 4097, 4096, 4092, 4097, 4112, 4079, 4101] avg=4096.0000 max=4112 max_rank=5 min=4079 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16389 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4090, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 11, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 21, 6, 7] rerouted_tokens_per_ep_rank=[4090, 4099, 4096, 4097, 4099, 4106, 4075, 4106] avg=4096.0000 max=4106 max_rank=5 min=4075 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16386 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 16, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 17, 4, 2] rerouted_tokens_per_ep_rank=[4094, 4097, 4096, 4094, 4097, 4109, 4079, 4102] avg=4096.0000 max=4109 max_rank=5 min=4079 min_rank=6 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 2, 4088, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4087, 1, 0, 0, 4092, 0, 3, 4, 0, 0, 0, 0, 0, 0, 4, 4087, 0, 0, 0, 0, 0, 6, 3, 4091, 0, 0, 0, 0, 0, 4040, 2, 16, 0, 0, 0, 0, 4075, 68, 3, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4057, 17, 4, 2] rerouted_tokens_per_ep_rank=[4096, 4091, 4099, 4091, 4100, 4058, 4146, 4087] avg=4096.0000 max=4146 max_rank=6 min=4058 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16391 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 16, 0, 0, 0, 0, 4079, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 17, 4, 2] rerouted_tokens_per_ep_rank=[4095, 4097, 4096, 4113, 4096, 4110, 4081, 4080] avg=4096.0000 max=4113 max_rank=3 min=4080 min_rank=7 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16367 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32768, 32779, 32768, 32768, 32752] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32768, 32779, 32768, 32768, 32752] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32768, 32779, 32768, 32768, 32752] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32768, 32779, 32768, 32768, 32752] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32768, 32779, 32768, 32768, 32752] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32768, 32779, 32768, 32768, 32752] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32768, 32779, 32768, 32768, 32752] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32768, 32779, 32768, 32768, 32752] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32773 actual_valid_recv=33152 observed_loss=-379 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32779 actual_valid_recv=33152 observed_loss=-373 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32752 actual_valid_recv=33408 observed_loss=-656 tokens_per_expert=[0, 0, 128, 128, 32640, 256, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 42, 4054, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 5, 4094, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4092, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 8, 4078, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4092, 4090, 0, 2, 0, 0, 4093, 0, 3, 2, 0, 4091, 0, 5, 0, 4, 4, 1, 3, 19, 0, 3, 4091, 3, 4090, 3, 1, 0, 4075, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4092, 4089, 2, 2, 0, 0, 4090, 0, 5, 2, 0, 4090, 0, 6, 0, 7, 4, 0, 3, 23, 1, 3, 4090, 4, 4085, 4, 2, 0, 4072, 0, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 7, 4071, 18, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 4092, 0, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4099, 4096, 4096, 4092, 4096, 4099, 4094] avg=4096.0000 max=4099 max_rank=1 min=4092 min_rank=4 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 42, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4094, 4090, 4096, 4096, 4104] avg=4096.0000 max=4104 max_rank=7 min=4090 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4094, 4096, 4095, 4090, 4096, 4096, 4104] avg=4096.0000 max=4104 max_rank=7 min=4090 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 4090, 0, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3, 2, 0, 4091, 0, 0, 0, 0, 0, 5, 0, 4, 4089, 0, 0, 0, 4, 1, 3, 19, 4085, 1, 0, 0, 0, 3, 4091, 3, 0, 0, 0, 0, 4090, 3, 1, 0, 0, 0, 0, 0, 4075, 0, 0, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4093, 4096, 4098, 4113, 4097, 4094, 4083] avg=4096.0000 max=4113 max_rank=4 min=4083 min_rank=7 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 4078, 10, 7, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4103, 4095, 4095, 4090, 4093, 4100, 4096] avg=4096.0000 max=4103 max_rank=1 min=4090 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4097, 4096, 4093, 4089, 4096, 4096, 4103] avg=4096.0000 max=4103 max_rank=7 min=4089 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 4071, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4095, 4089, 4096, 4096, 4104] avg=4096.0000 max=4104 max_rank=7 min=4089 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 4089, 2, 2, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 5, 2, 0, 4090, 0, 0, 0, 0, 0, 6, 0, 7, 4089, 0, 0, 0, 4, 0, 3, 23, 4084, 1, 0, 0, 1, 3, 4090, 4, 0, 0, 0, 0, 4085, 4, 2, 0, 0, 0, 0, 0, 4072, 0, 0, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4090, 4097, 4102, 4115, 4098, 4091, 4080] avg=4096.0000 max=4115 max_rank=4 min=4080 min_rank=7 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4, 4096, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 2, 4096, 3, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4065, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 31, 4096, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4069, 0, 0, 22, 4096, 9, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4093, 0, 8, 3, 4093, 4090, 4086, 4092, 0, 4, 0, 0, 5, 3, 0, 0, 1, 4091, 0, 3692, 0, 0, 0, 4088, 137, 2, 2, 270, 5, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4061, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 0, 17, 4096, 37, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4093, 0, 9, 5, 4091, 4087, 4084, 4091, 1, 6, 0, 2, 5, 3, 0, 0, 0, 4089, 0, 3748, 0, 0, 0, 4088, 24, 7, 2, 323, 6, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 7, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 103, 0, 4095, 0, 0, 0, 0, 0, 0, 4, 4089, 0, 0, 0, 0, 2, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4099, 4096, 4095, 4198, 4093, 3995, 4094] avg=4096.0000 max=4198 max_rank=4 min=3995 min_rank=6 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16380 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4065, 0, 0, 0, 7, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 103, 0, 4094, 0, 0, 0, 0, 0, 0, 31, 4089, 0, 0, 0, 0, 2, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4099, 4095, 4096, 4197, 4120, 3995, 4094] avg=4096.0000 max=4197 max_rank=4 min=3995 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16406 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 7, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 103, 0, 4092, 0, 0, 0, 0, 0, 0, 2, 4089, 0, 0, 0, 0, 3, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4099, 4095, 4095, 4195, 4091, 3996, 4095] avg=4096.0000 max=4195 max_rank=4 min=3996 min_rank=6 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16377 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 8, 3, 7, 0, 0, 0, 0, 3, 4086, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 5, 3, 0, 0, 4090, 1, 0, 0, 1, 102, 0, 3692, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 137, 2, 2, 270, 3989, 0, 0, 0, 5, 0, 3, 0, 4087, 2, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4089, 4096, 4099, 3795, 4081, 4400, 4097] avg=4096.0000 max=4400 max_rank=6 min=3795 min_rank=4 imbalance_ratio=1.074219 max_load_reduction=28368 max_load_reduction_pct=0.865723 moved_to_echo=16373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 10, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 90, 0, 4069, 0, 0, 0, 0, 0, 0, 22, 4085, 0, 0, 0, 0, 9, 0, 0, 2, 3999, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 7, 0] rerouted_tokens_per_ep_rank=[4100, 4097, 4096, 4095, 4159, 4107, 4011, 4103] avg=4096.0000 max=4159 max_rank=4 min=4011 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16380 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 7, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 103, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 5, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4099, 4095, 4095, 4194, 4089, 3998, 4095] avg=4096.0000 max=4194 max_rank=4 min=3998 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16376 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4061, 0, 0, 0, 7, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 103, 0, 4077, 0, 0, 0, 0, 0, 0, 17, 4089, 0, 0, 0, 0, 37, 0, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4099, 4096, 4095, 4180, 4106, 4030, 4094] avg=4096.0000 max=4180 max_rank=4 min=4030 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16410 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 9, 5, 7, 0, 0, 0, 0, 3, 4084, 0, 0, 0, 0, 0, 1, 6, 0, 2, 4090, 0, 0, 0, 5, 3, 0, 0, 4089, 1, 0, 0, 0, 102, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 24, 7, 2, 323, 3987, 0, 0, 0, 6, 0, 4, 0, 4084, 2, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4087, 4099, 4098, 3850, 4081, 4343, 4096] avg=4096.0000 max=4343 max_rank=6 min=3850 min_rank=4 imbalance_ratio=1.060303 max_load_reduction=28425 max_load_reduction_pct=0.867462 moved_to_echo=16370 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 896, 0, 32000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 128, 640, 32000, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 4, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4079, 17, 0, 0, 0, 0, 4096, 4094, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 1, 0, 0, 0, 0, 4096, 4096, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[3, 0, 0, 4, 4, 0, 4093, 4, 11, 2, 4091, 4092, 0, 0, 4086, 3, 0, 0, 0, 4, 2, 4087, 0, 4093, 0, 2, 3, 0, 0, 4092, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 1, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 2, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4, 0, 0, 5, 5, 0, 4093, 4, 14, 1, 4090, 4091, 0, 0, 4088, 4, 0, 0, 0, 5, 3, 4080, 0, 4092, 0, 2, 4, 0, 0, 4091, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 4092, 0, 0, 0, 0, 4, 0, 0, 0, 4093, 2, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4094, 4097, 4096, 4095, 4100, 4096] avg=4096.0000 max=4100 max_rank=6 min=4094 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4094, 4097, 4096, 4101, 4094, 4096] avg=4096.0000 max=4101 max_rank=5 min=4094 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 4089, 0, 0, 0, 0, 1, 0, 0, 0, 4091, 2, 1, 0, 0, 0, 4096, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4094, 4097, 4096, 4094, 4095, 4102] avg=4096.0000 max=4102 max_rank=7 min=4094 min_rank=2 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4, 4091, 0, 0, 0, 4, 0, 4092, 4, 0, 0, 0, 0, 11, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 4086, 3, 1, 0, 0, 0, 0, 0, 0, 4, 4092, 0, 0, 0, 2, 5, 0, 4093, 0, 0, 0, 0, 0, 2, 3, 0, 4082, 2, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4100, 4102, 4090, 4096, 4100, 4090, 4092] avg=4096.0000 max=4102 max_rank=2 min=4090 min_rank=3 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16378 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4092, 4093, 4100, 4096, 4079, 4116, 4096] avg=4096.0000 max=4116 max_rank=6 min=4079 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 4091, 2, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4094, 4097, 4096, 4100, 4095, 4096] avg=4096.0000 max=4100 max_rank=5 min=4094 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 4091, 2, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4094, 4097, 4096, 4099, 4095, 4097] avg=4096.0000 max=4099 max_rank=5 min=4094 min_rank=2 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 5, 4090, 0, 0, 0, 5, 0, 4092, 4, 0, 0, 0, 0, 14, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 1, 0, 0, 0, 0, 0, 0, 5, 4091, 0, 0, 0, 3, 5, 0, 4092, 0, 0, 0, 0, 0, 2, 4, 0, 4075, 2, 0, 0, 0, 1, 4092, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4101, 4103, 4093, 4096, 4100, 4083, 4093] avg=4096.0000 max=4103 max_rank=2 min=4083 min_rank=6 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16372 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 29, 0, 0, 0, 4096, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[6, 0, 0, 43, 0, 3, 0, 4085, 3, 4093, 4, 4, 1, 8, 3, 2, 0, 0, 24, 5, 4, 4090, 0, 4091, 4069, 4091, 0, 4087, 0, 4051, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[8, 0, 1, 71, 0, 4, 0, 4082, 5, 4092, 5, 6, 2, 10, 3, 2, 0, 0, 29, 5, 4, 4089, 0, 4090, 4064, 4089, 0, 4078, 0, 4027, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4076, 3, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4012, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 84, 0, 0, 4022, 20, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4098, 4065, 4093, 4126, 4096, 4096, 4115] avg=4096.0000 max=4126 max_rank=4 min=4065 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16433 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 29, 4075, 2, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4011, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 56, 0, 0, 4018, 20, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4098, 4064, 4094, 4094, 4097, 4106, 4109] avg=4096.0000 max=4109 max_rank=7 min=4064 min_rank=2 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4075, 2, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4040, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 56, 0, 0, 4018, 20, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4098, 4093, 4094, 4094, 4097, 4106, 4109] avg=4096.0000 max=4109 max_rank=7 min=4077 min_rank=0 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 43, 4070, 2, 0, 0, 0, 3, 0, 4085, 2, 0, 0, 0, 3, 4093, 4, 4, 53, 0, 0, 0, 1, 8, 3, 2, 4089, 0, 0, 0, 0, 0, 24, 5, 4009, 20, 0, 0, 4, 4090, 0, 1, 0, 0, 0, 0, 4069, 0, 0, 10, 0, 0, 0, 0, 0, 4051, 1, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4090, 4157, 4103, 4058, 4095, 4079, 4065] avg=4096.0000 max=4157 max_rank=2 min=4058 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16297 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4076, 3, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4055, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 41, 0, 0, 4019, 20, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4100, 4115, 4093, 4080, 4096, 4096, 4109] avg=4096.0000 max=4115 max_rank=2 min=4079 min_rank=0 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4075, 3, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 3940, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 156, 0, 0, 4018, 21, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4098, 3993, 4093, 4195, 4096, 4105, 4110] avg=4096.0000 max=4195 max_rank=4 min=3993 min_rank=2 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16506 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4075, 2, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4067, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 29, 0, 0, 4018, 20, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4098, 4120, 4094, 4067, 4097, 4106, 4109] avg=4096.0000 max=4120 max_rank=2 min=4067 min_rank=4 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 1, 71, 4069, 2, 0, 0, 0, 4, 0, 4082, 2, 0, 0, 0, 5, 4092, 5, 6, 53, 0, 0, 0, 2, 10, 3, 2, 4087, 0, 0, 0, 0, 0, 29, 5, 4000, 20, 0, 0, 4, 4089, 0, 1, 0, 0, 0, 0, 4064, 0, 0, 10, 0, 0, 0, 0, 0, 4027, 2, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4088, 4161, 4104, 4054, 4094, 4074, 4042] avg=4096.0000 max=4161 max_rank=2 min=4042 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32384, 128, 128, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 512, 128, 128, 32128, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 525, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3571, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1369, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2728, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3, 4092, 25, 2680, 8, 6, 0, 55, 1, 0, 4083, 0, 4, 0, 0, 4, 1490, 3, 4, 0, 3933, 6, 0, 0, 4088, 0, 4092, 3, 0, 6, 4091, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 996, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3100, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1802, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2294, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1014, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3105, 0, 0, 0, 4073, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1715, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2383, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[5, 4091, 20, 2493, 9, 8, 1, 56, 2, 0, 4076, 0, 5, 0, 0, 5, 1562, 5, 6, 0, 4043, 8, 0, 0, 4086, 0, 4091, 4, 0, 14, 4089, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2518, 0, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2728, 0, 0, 0, 1548, 16, 0, 0, 4095, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3887, 4079, 4099, 4095, 4292, 4123, 4098, 4095] avg=4096.0000 max=4292 max_rank=4 min=3887 min_rank=0 imbalance_ratio=1.047852 max_load_reduction=28476 max_load_reduction_pct=0.869019 moved_to_echo=16608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2518, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3571, 0, 0, 0, 1548, 16, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3043, 4079, 4099, 4095, 5135, 4124, 4098, 4095] avg=4096.0000 max=5135 max_rank=4 min=3043 min_rank=0 imbalance_ratio=1.253662 max_load_reduction=27633 max_load_reduction_pct=0.843292 moved_to_echo=17452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2518, 0, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 1, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3105, 0, 0, 0, 1548, 16, 0, 0, 4073, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3532, 4081, 4099, 4095, 4669, 4101, 4096, 4095] avg=4096.0000 max=4669 max_rank=4 min=3532 min_rank=0 imbalance_ratio=1.139893 max_load_reduction=28099 max_load_reduction_pct=0.857513 moved_to_echo=16961 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 2515, 25, 2680, 0, 0, 0, 0, 8, 6, 0, 55, 4074, 1, 0, 0, 1, 0, 4083, 0, 2, 0, 0, 0, 4, 0, 0, 4, 4090, 0, 0, 0, 1490, 3, 4, 0, 1547, 16, 0, 0, 3933, 6, 0, 0, 28, 0, 0, 0, 4088, 0, 2, 3, 0, 0, 0, 0, 0, 6, 4091, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5223, 4144, 4086, 4098, 3060, 3967, 4093, 4097] avg=4096.0000 max=5223 max_rank=0 min=3060 min_rank=4 imbalance_ratio=1.275146 max_load_reduction=27545 max_load_reduction_pct=0.840607 moved_to_echo=15217 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2502, 0, 996, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 1, 0, 0, 0, 0, 4096, 0, 8, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3100, 0, 0, 0, 1554, 16, 0, 0, 4096, 0, 0, 0, 32, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3498, 4081, 4107, 4095, 4670, 4128, 4096, 4093] avg=4096.0000 max=4670 max_rank=4 min=3498 min_rank=0 imbalance_ratio=1.140137 max_load_reduction=28098 max_load_reduction_pct=0.857483 moved_to_echo=16987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2518, 0, 1802, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2294, 0, 0, 0, 1548, 17, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4320, 4079, 4099, 4095, 3859, 4124, 4097, 4095] avg=4096.0000 max=4320 max_rank=0 min=3859 min_rank=4 imbalance_ratio=1.054688 max_load_reduction=28448 max_load_reduction_pct=0.868164 moved_to_echo=16175 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2518, 0, 1715, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2383, 0, 0, 0, 1548, 16, 0, 0, 4094, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4233, 4078, 4099, 4096, 3947, 4122, 4098, 4095] avg=4096.0000 max=4233 max_rank=0 min=3947 min_rank=4 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16262 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 2514, 20, 2493, 0, 0, 0, 0, 9, 8, 1, 56, 4073, 0, 0, 0, 2, 0, 4076, 0, 2, 0, 0, 0, 5, 0, 0, 5, 4089, 0, 0, 0, 1562, 5, 6, 0, 1547, 16, 0, 0, 4043, 8, 0, 0, 28, 0, 0, 0, 4086, 0, 2, 4, 0, 0, 0, 0, 0, 14, 4089, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5032, 4147, 4080, 4099, 3136, 4079, 4092, 4103] avg=4096.0000 max=5032 max_rank=0 min=3136 min_rank=4 imbalance_ratio=1.228516 max_load_reduction=27736 max_load_reduction_pct=0.846436 moved_to_echo=15410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 20224, 128, 12672, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[20352, 128, 128, 0, 12416, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 32768, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 128, 0, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-08-30 04:56:47] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 672.5 | throughput per GPU (TFLOP/s/GPU): 495.7 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 1.009814E+01 | loss scale: 1.0 | grad norm: 7.218 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[11, 0, 841, 18, 3169, 0, 2268, 5, 2883, 645, 0, 46, 2382, 3954, 0, 530, 60, 1034, 3114, 0, 0, 437, 568, 0, 3, 578, 3770, 3151, 3065, 117, 119, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 449, 1, 3618, 0, 2955, 2, 2997, 457, 0, 52, 2735, 3448, 1, 678, 34, 796, 2882, 0, 0, 175, 899, 5, 0, 951, 3870, 2769, 2812, 97, 81, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[20, 0, 806, 11, 2556, 1, 3220, 0, 2057, 1151, 0, 34, 2576, 3919, 0, 454, 18, 1083, 2866, 0, 0, 231, 825, 5, 0, 803, 3732, 2936, 3056, 231, 177, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[4, 0, 616, 20, 3058, 0, 2880, 2, 2427, 850, 0, 35, 2020, 3867, 0, 604, 9, 1474, 2914, 0, 0, 114, 799, 7, 0, 1018, 3782, 3082, 2971, 103, 112, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[6, 0, 600, 0, 2992, 7, 3141, 4, 3063, 693, 0, 64, 2534, 3722, 0, 640, 38, 1388, 2790, 1, 0, 208, 663, 8, 1, 885, 3853, 2505, 2778, 95, 89, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[19, 1, 899, 2, 3311, 0, 2590, 0, 2718, 613, 2, 62, 2555, 3784, 1, 427, 59, 849, 3154, 0, 1, 202, 968, 20, 3, 477, 3776, 2975, 3120, 93, 87, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[32, 0, 702, 3, 3016, 0, 2963, 0, 1915, 846, 0, 60, 2409, 3930, 0, 667, 52, 1090, 3060, 0, 0, 340, 650, 5, 0, 713, 3957, 3013, 3006, 286, 53, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2, 0, 929, 0, 2969, 0, 2926, 0, 2242, 669, 0, 34, 2615, 3867, 0, 333, 15, 744, 3219, 0, 0, 176, 689, 3, 0, 741, 3947, 2899, 3473, 137, 139, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 841, 18, 3289, 0, 0, 0, 1260, 0, 2268, 5, 0, 0, 0, 0, 2883, 645, 0, 46, 789, 0, 0, 0, 2382, 1117, 0, 530, 0, 0, 0, 0, 60, 1034, 3114, 0, 3, 0, 0, 0, 0, 437, 568, 0, 2837, 287, 79, 0, 3, 578, 402, 3151, 0, 0, 0, 0, 3065, 117, 119, 0, 830, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 3533, 4363, 4029, 4211, 4208, 4134, 4131] avg=4096.0000 max=4363 max_rank=2 min=3533 min_rank=1 imbalance_ratio=1.065186 max_load_reduction=28405 max_load_reduction_pct=0.866852 moved_to_echo=16684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 806, 11, 3255, 0, 0, 0, 1017, 1, 3220, 0, 0, 0, 0, 0, 2057, 1151, 0, 34, 637, 0, 0, 0, 2576, 1107, 0, 454, 0, 0, 0, 0, 18, 1083, 2866, 0, 2, 0, 0, 0, 0, 231, 825, 5, 2812, 231, 78, 0, 0, 803, 399, 2936, 0, 0, 0, 0, 3056, 231, 177, 0, 669, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4238, 3879, 4137, 3969, 4182, 4138, 4133] avg=4096.0000 max=4238 max_rank=1 min=3879 min_rank=2 imbalance_ratio=1.034668 max_load_reduction=28530 max_load_reduction_pct=0.870667 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 616, 20, 3299, 0, 0, 0, 1215, 0, 2880, 2, 0, 0, 0, 0, 2427, 850, 0, 35, 762, 0, 0, 0, 2020, 1092, 0, 604, 0, 0, 0, 0, 9, 1474, 2914, 0, 3, 0, 0, 0, 0, 114, 799, 7, 2775, 277, 79, 0, 0, 1018, 404, 3082, 0, 0, 0, 0, 2971, 103, 112, 0, 801, 0, 0, 0] rerouted_tokens_per_ep_rank=[3939, 4097, 4074, 3716, 4400, 4051, 4504, 3987] avg=4096.0000 max=4504 max_rank=6 min=3716 min_rank=3 imbalance_ratio=1.099609 max_load_reduction=28264 max_load_reduction_pct=0.862549 moved_to_echo=16942 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[19, 1, 899, 2, 3294, 0, 0, 0, 1316, 0, 2590, 0, 0, 0, 0, 0, 2718, 613, 2, 62, 825, 0, 0, 0, 2555, 1069, 1, 427, 0, 0, 0, 0, 59, 849, 3154, 0, 3, 0, 0, 0, 1, 202, 968, 20, 2715, 300, 79, 0, 3, 477, 403, 2975, 0, 0, 0, 0, 3120, 93, 87, 0, 867, 0, 0, 0] rerouted_tokens_per_ep_rank=[4215, 3906, 4220, 4052, 4065, 4285, 3858, 4167] avg=4096.0000 max=4285 max_rank=5 min=3858 min_rank=6 imbalance_ratio=1.046143 max_load_reduction=28483 max_load_reduction_pct=0.869232 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 449, 1, 3376, 0, 0, 0, 1439, 0, 2955, 2, 0, 0, 0, 0, 2997, 457, 0, 52, 901, 0, 0, 0, 2735, 974, 1, 678, 0, 0, 0, 0, 34, 796, 2882, 0, 3, 0, 0, 0, 0, 175, 899, 5, 2474, 327, 81, 0, 0, 951, 413, 2769, 0, 0, 0, 0, 2812, 97, 81, 1, 948, 0, 0, 0] rerouted_tokens_per_ep_rank=[3829, 4396, 4407, 4388, 3715, 3961, 4133, 3939] avg=4096.0000 max=4407 max_rank=2 min=3715 min_rank=4 imbalance_ratio=1.075928 max_load_reduction=28361 max_load_reduction_pct=0.865509 moved_to_echo=15748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 702, 3, 3456, 0, 0, 0, 1188, 0, 2963, 0, 0, 0, 0, 0, 1915, 846, 0, 60, 755, 0, 0, 0, 2409, 1105, 0, 667, 0, 0, 0, 0, 52, 1090, 3060, 0, 5, 0, 0, 0, 0, 340, 650, 5, 2825, 275, 89, 0, 0, 713, 412, 3013, 0, 0, 0, 0, 3006, 286, 53, 0, 793, 0, 0, 0] rerouted_tokens_per_ep_rank=[4193, 4151, 3576, 4181, 4207, 4184, 4138, 4138] avg=4096.0000 max=4207 max_rank=4 min=3576 min_rank=2 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 600, 0, 3361, 0, 0, 0, 1189, 7, 3141, 4, 0, 0, 0, 0, 3063, 693, 0, 64, 745, 0, 0, 0, 2534, 1051, 0, 640, 0, 0, 0, 0, 38, 1388, 2790, 1, 3, 0, 0, 0, 0, 208, 663, 8, 2671, 271, 81, 0, 1, 885, 411, 2505, 0, 0, 0, 0, 2778, 95, 89, 0, 784, 0, 0, 0] rerouted_tokens_per_ep_rank=[3967, 4341, 4565, 4225, 4220, 3902, 3802, 3746] avg=4096.0000 max=4565 max_rank=2 min=3746 min_rank=7 imbalance_ratio=1.114502 max_load_reduction=28203 max_load_reduction_pct=0.860687 moved_to_echo=15670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 929, 0, 3443, 0, 0, 0, 1180, 0, 2926, 0, 0, 0, 0, 0, 2242, 669, 0, 34, 739, 0, 0, 0, 2615, 1092, 0, 333, 0, 0, 0, 0, 15, 744, 3219, 0, 3, 0, 0, 0, 0, 176, 689, 3, 2775, 269, 83, 0, 0, 741, 421, 2899, 0, 0, 0, 0, 3473, 137, 139, 0, 778, 0, 0, 0] rerouted_tokens_per_ep_rank=[4374, 4106, 3684, 4040, 3981, 3995, 4061, 4527] avg=4096.0000 max=4527 max_rank=7 min=3684 min_rank=2 imbalance_ratio=1.105225 max_load_reduction=28241 max_load_reduction_pct=0.861847 moved_to_echo=16564 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[19840, 8704, 128, 4352, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 5888, 128, 26880, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 8576, 24064, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1920, 6144, 128, 21888, 2304, 768, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[24320, 1280, 896, 128, 6528, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[20352, 6016, 128, 512, 6272, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9856, 128, 23040, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 6272, 3328, 23424, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 3810, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 294, 4089, 0, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 12, 4094, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 4083, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4093, 1, 0, 3, 4039, 0, 0, 0, 1, 4096, 0, 1, 3, 0, 10, 4093, 4094, 2, 3, 1, 4083, 0, 0, 0, 62, 4085, 0, 0, 3, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 4, 0, 1, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 15, 4076, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 4095, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 4095, 0, 0, 0, 1, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 12, 4096, 4096, 0, 0, 0, 4093, 0, 0, 0, 1, 4085, 0, 0, 0, 1, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 4093, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 6, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 45, 4049, 0, 0, 0, 4, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 2, 4091, 0, 0, 0, 5, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 38, 5, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 1, 4047, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4139, 4097, 4103, 4096, 4094, 4048, 4095] avg=4096.0000 max=4139 max_rank=1 min=4048 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 38, 5, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 1, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 15, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4138, 4097, 4096, 4096, 4097, 4053, 4095] avg=4096.0000 max=4138 max_rank=1 min=4053 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 38, 5, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4138, 4096, 4091, 4096, 4098, 4057, 4095] avg=4096.0000 max=4138 max_rank=1 min=4057 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 0, 0, 0, 3810, 0, 0, 0, 38, 5, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 294, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 3853, 4096, 4095, 4096, 4096, 4345, 4094] avg=4096.0000 max=4345 max_rank=6 min=3853 min_rank=1 imbalance_ratio=1.060791 max_load_reduction=28423 max_load_reduction_pct=0.867401 moved_to_echo=16631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4093, 1, 0, 3, 0, 0, 0, 0, 4039, 0, 0, 0, 38, 5, 0, 0, 1, 4096, 0, 1, 0, 0, 0, 0, 3, 0, 10, 4088, 0, 0, 0, 0, 4094, 2, 3, 1, 0, 0, 0, 0, 4083, 0, 0, 0, 1, 0, 0, 0, 62, 4047, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4082, 4098, 4101, 4100, 4084, 4109, 4097] avg=4096.0000 max=4109 max_rank=6 min=4082 min_rank=1 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 4, 3, 0, 0, 4093, 0, 0, 0, 41, 6, 0, 0, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4084, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3, 6, 1, 0, 45, 4008, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4140, 4093, 4090, 4092, 4105, 4053, 4097] avg=4096.0000 max=4140 max_rank=1 min=4053 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 38, 5, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 2, 4053, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4139, 4094, 4091, 4096, 4097, 4055, 4100] avg=4096.0000 max=4139 max_rank=1 min=4055 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 38, 5, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 3, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4139, 4097, 4101, 4096, 4097, 4048, 4095] avg=4096.0000 max=4139 max_rank=1 min=4048 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 0, 0, 0, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[512, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 128, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 4091, 0, 0, 0, 4096, 5, 4090, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 4092, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 4095, 4083, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 14, 0, 0, 0, 0, 0, 4089, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3, 4074, 4095, 0, 0, 0, 4093, 1, 4092, 4096, 0, 0, 4093, 1, 0, 4094, 1, 0, 0, 0, 0, 0, 25, 2, 2, 0, 2, 0, 2, 4092, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 4090, 0, 0, 0, 4096, 0, 4080, 4096, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 20, 4, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 4090, 4094, 0, 0, 0, 4096, 0, 4092, 4094, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 10, 7, 0, 0, 0, 0, 0, 4091, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 3910, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 9, 181, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 4090, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 2, 4063, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4101, 4095, 4096, 4096, 4069, 4095, 4097] avg=4096.0000 max=4119 max_rank=0 min=4069 min_rank=5 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4, 4063, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4096, 4099, 4096, 4096, 4068, 4096, 4097] avg=4096.0000 max=4120 max_rank=0 min=4068 min_rank=5 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 4083, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 7, 14, 4062, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4096, 4101, 4096, 4096, 4083, 4095, 4090] avg=4096.0000 max=4111 max_rank=0 min=4083 min_rank=5 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 1, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 4091, 9, 0, 0, 0, 4096, 0, 0, 3, 3, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4093, 4096, 4096, 4101, 4066, 4100, 4102] avg=4096.0000 max=4114 max_rank=0 min=4066 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 10, 7, 4057, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 0, 4091, 0, 1, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4096, 4097, 4098, 4095, 4074, 4093, 4093] avg=4096.0000 max=4122 max_rank=0 min=4074 min_rank=5 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 28, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 25, 2, 4041, 0, 0, 0, 2, 0, 2, 0, 4091, 4, 0, 0, 2, 4092, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4094, 4097, 4095, 4094, 4068, 4099, 4095] avg=4096.0000 max=4126 max_rank=0 min=4068 min_rank=5 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 5, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 20, 4, 4063, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4096, 4085, 4094, 4096, 4087, 4095, 4097] avg=4096.0000 max=4118 max_rank=0 min=4085 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 3910, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 9, 181, 4063, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3938, 4096, 4098, 4097, 4094, 4253, 4095, 4097] avg=4096.0000 max=4253 max_rank=5 min=3938 min_rank=0 imbalance_ratio=1.038330 max_load_reduction=28515 max_load_reduction_pct=0.870209 moved_to_echo=16539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 256, 32640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 256, 32512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 2728, 0, 0, 0, 0, 4096, 1372] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 4084, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 2, 4096, 2492, 0, 0, 0, 0, 4094, 1606] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1, 4096, 2687, 0, 0, 0, 0, 4096, 1410] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 4082, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 4096, 3, 4095, 1311, 0, 0, 0, 0, 4096, 2796] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 4084, 6, 0, 0, 0, 0, 1, 0, 2, 0, 4093, 0, 0, 0, 0, 4092, 0, 0, 4094, 2, 0, 4096, 8, 4095, 1755, 0, 0, 1, 0, 4096, 2343] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 1792, 0, 0, 0, 0, 4096, 2304] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 4074, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 15, 4096, 3184, 0, 0, 0, 0, 4096, 918] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 6, 4096, 1220, 0, 0, 0, 0, 4096, 2877] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 1, 0, 4, 2, 0, 0, 0, 0, 0, 0, 2145, 1950, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 1951, 2728, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 1372, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4095, 4095, 4096, 4096, 4092, 4679, 3516] avg=4096.0000 max=4679 max_rank=6 min=3516 min_rank=7 imbalance_ratio=1.142334 max_load_reduction=28089 max_load_reduction_pct=0.857208 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 10, 0, 4, 2, 0, 0, 0, 0, 0, 0, 2145, 1949, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 1951, 2492, 0, 0, 0, 0, 0, 0, 0, 0, 2143, 1606, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4094, 4096, 4095, 4097, 4094, 4443, 3749] avg=4096.0000 max=4443 max_rank=6 min=3749 min_rank=7 imbalance_ratio=1.084717 max_load_reduction=28325 max_load_reduction_pct=0.864410 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 3, 0, 4, 2, 0, 0, 0, 0, 0, 0, 2144, 1950, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 1951, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 2796, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4094, 4095, 4094, 4097, 4095, 3262, 4940] avg=4096.0000 max=4940 max_rank=7 min=3262 min_rank=6 imbalance_ratio=1.206055 max_load_reduction=27828 max_load_reduction_pct=0.849243 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 1, 0, 4, 2, 0, 0, 0, 0, 0, 0, 2145, 1950, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 1951, 2687, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 1410, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4095, 4096, 4096, 4096, 4093, 4638, 3554] avg=4096.0000 max=4638 max_rank=6 min=3554 min_rank=7 imbalance_ratio=1.132324 max_load_reduction=28130 max_load_reduction_pct=0.858459 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 6, 0, 4, 2, 0, 0, 0, 0, 0, 1, 2144, 1950, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 2, 0, 4092, 8, 0, 0, 0, 0, 1951, 1755, 0, 0, 0, 0, 0, 0, 1, 0, 2144, 2343, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4095, 4093, 4093, 4102, 3706, 4488] avg=4096.0000 max=4488 max_rank=7 min=3706 min_rank=6 imbalance_ratio=1.095703 max_load_reduction=28280 max_load_reduction_pct=0.863037 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 2147, 1957, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 1942, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 2134, 2304, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4104, 4099, 4103, 4096, 4088, 3734, 4438] avg=4096.0000 max=4438 max_rank=7 min=3734 min_rank=6 imbalance_ratio=1.083496 max_load_reduction=28330 max_load_reduction_pct=0.864563 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4074, 0, 0, 4, 2, 0, 0, 0, 0, 0, 1, 2145, 1950, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 15, 0, 0, 0, 0, 1951, 3184, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 918, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4096, 4096, 4096, 4096, 4107, 5135, 3062] avg=4096.0000 max=5135 max_rank=6 min=3062 min_rank=7 imbalance_ratio=1.253662 max_load_reduction=27633 max_load_reduction_pct=0.843292 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 2145, 1950, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4091, 6, 0, 0, 0, 0, 1951, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 2877, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4096, 4095, 4097, 4097, 3171, 5021] avg=4096.0000 max=5021 max_rank=7 min=3171 min_rank=6 imbalance_ratio=1.225830 max_load_reduction=27747 max_load_reduction_pct=0.846771 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 17280, 15616, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 17152, 15744, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[15616, 17280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 156, 0, 0, 0, 0, 3940, 0, 4096, 0, 4095, 0, 4095, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 889, 0, 0, 0, 0, 3207, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 100, 0, 0, 0, 0, 3996, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 82, 0, 0, 0, 0, 4012, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 4095, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 106, 0, 0, 0, 0, 3990, 0, 4096, 0, 4096, 0, 4092, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 288, 0, 0, 0, 0, 3808, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 2, 4094, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 61, 0, 0, 0, 0, 4034, 0, 4096, 0, 4096, 0, 4091, 4092, 0, 4095, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4095, 0, 4096, 0, 4095, 0, 4096, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 156, 0, 0, 0, 3885, 1, 0, 0, 0, 3940, 0, 211, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4095, 4096, 4042, 4151, 4095, 4096] avg=4096.0000 max=4151 max_rank=5 min=4042 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 100, 0, 0, 0, 3885, 0, 0, 0, 0, 3996, 0, 211, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4095, 4096, 3985, 4207, 4098, 4096] avg=4096.0000 max=4207 max_rank=5 min=3985 min_rank=4 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16386 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 82, 0, 0, 0, 3885, 0, 0, 0, 0, 4012, 0, 211, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4095, 3967, 4223, 4097, 4098] avg=4096.0000 max=4223 max_rank=5 min=3967 min_rank=4 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 3885, 0, 0, 0, 0, 4095, 0, 211, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4094, 4096, 3887, 4306, 4098, 4096] avg=4096.0000 max=4306 max_rank=5 min=3887 min_rank=4 imbalance_ratio=1.051270 max_load_reduction=28462 max_load_reduction_pct=0.868591 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 288, 0, 0, 0, 3887, 0, 0, 0, 0, 3808, 0, 209, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4096, 4097, 4175, 4017, 4096, 4096] avg=4096.0000 max=4175 max_rank=4 min=4017 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4094, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 61, 0, 0, 0, 3885, 0, 0, 0, 0, 4034, 0, 211, 0, 0, 0, 0, 0, 2, 0, 4091, 0, 0, 0, 0, 4092, 0, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 4099, 4096, 3946, 4245, 4093, 4094] avg=4096.0000 max=4245 max_rank=5 min=3946 min_rank=4 imbalance_ratio=1.036377 max_load_reduction=28523 max_load_reduction_pct=0.870453 moved_to_echo=16378 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 106, 0, 0, 0, 3885, 1, 0, 0, 0, 3990, 0, 211, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4098, 4096, 3992, 4201, 4093, 4096] avg=4096.0000 max=4201 max_rank=5 min=3992 min_rank=4 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 889, 0, 0, 0, 3885, 0, 0, 0, 0, 3207, 0, 211, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4095, 4096, 4774, 3418, 4098, 4096] avg=4096.0000 max=4774 max_rank=4 min=3418 min_rank=5 imbalance_ratio=1.165527 max_load_reduction=27994 max_load_reduction_pct=0.854309 moved_to_echo=16386 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1792, 0, 0, 0, 31104, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 31104, 0, 1792, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4084, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 0, 953, 0, 0, 0, 0, 4096, 101, 0, 4096, 0, 0, 3049, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4077, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 165, 0, 0, 0, 0, 4096, 242, 0, 4096, 0, 0, 3705, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4075, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 21, 0, 351, 0, 0, 0, 0, 4096, 65, 0, 4096, 0, 0, 3682, 4093, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4074, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 21, 0, 584, 0, 0, 0, 0, 4096, 31, 0, 4096, 0, 0, 3487, 4090, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 192, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 3905, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4080, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 20, 1, 125, 0, 3, 0, 0, 4096, 150, 0, 4096, 0, 0, 3818, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4063, 0, 4, 0, 4096, 0, 0, 4092, 0, 0, 0, 31, 3, 453, 0, 3, 0, 0, 4095, 109, 0, 4096, 0, 0, 3537, 4090, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4046, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 75, 0, 0, 0, 0, 4096, 19, 0, 4096, 0, 0, 4051, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 953, 3632, 89, 0, 0, 0, 0, 0, 0, 4074, 20, 0, 0, 4007, 101, 0, 0, 0, 0, 0, 0, 0, 0, 3049, 443, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4096, 4096, 4680, 4094, 4108, 3492, 4096] avg=4096.0000 max=4680 max_rank=3 min=3492 min_rank=6 imbalance_ratio=1.142578 max_load_reduction=28088 max_load_reduction_pct=0.857178 moved_to_echo=15790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 4075, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 21, 0, 351, 3630, 89, 0, 0, 0, 0, 0, 0, 4074, 20, 0, 0, 4007, 65, 0, 0, 0, 0, 0, 0, 0, 0, 3682, 443, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4096, 4091, 4094, 4072, 4125, 4096] avg=4096.0000 max=4125 max_rank=6 min=4072 min_rank=5 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 3632, 89, 0, 0, 0, 0, 0, 0, 4074, 20, 0, 0, 4007, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 443, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4096, 4096, 3913, 4094, 4009, 4348, 4096] avg=4096.0000 max=4348 max_rank=6 min=3913 min_rank=3 imbalance_ratio=1.061523 max_load_reduction=28420 max_load_reduction_pct=0.867310 moved_to_echo=16547 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 4077, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 1, 0, 0, 0, 4, 0, 165, 3635, 94, 0, 0, 0, 0, 0, 0, 4077, 27, 0, 0, 4001, 242, 0, 0, 0, 0, 0, 0, 0, 0, 3705, 433, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4092, 4101, 3898, 4104, 4243, 4138, 4096] avg=4096.0000 max=4243 max_rank=5 min=3898 min_rank=3 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 4080, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 20, 1, 125, 3629, 89, 0, 0, 0, 3, 0, 0, 4074, 20, 0, 0, 4007, 150, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 443, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4096, 4095, 3864, 4097, 4157, 4261, 4096] avg=4096.0000 max=4261 max_rank=6 min=3864 min_rank=3 imbalance_ratio=1.040283 max_load_reduction=28507 max_load_reduction_pct=0.869965 moved_to_echo=16611 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 4074, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 21, 0, 584, 3627, 89, 0, 0, 0, 0, 0, 0, 4074, 20, 0, 0, 4007, 31, 0, 0, 0, 0, 0, 0, 0, 0, 3487, 443, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4096, 4321, 4094, 4038, 3930, 4096] avg=4096.0000 max=4321 max_rank=3 min=3930 min_rank=6 imbalance_ratio=1.054932 max_load_reduction=28447 max_load_reduction_pct=0.868134 moved_to_echo=16158 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 4063, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 31, 3, 453, 3627, 89, 0, 0, 0, 3, 0, 0, 4074, 20, 0, 0, 4006, 109, 0, 0, 0, 0, 0, 0, 0, 0, 3537, 443, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4100, 4092, 4203, 4097, 4115, 3980, 4096] avg=4096.0000 max=4203 max_rank=3 min=3980 min_rank=6 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16288 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 4046, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 75, 3631, 89, 0, 0, 0, 0, 0, 0, 4074, 20, 0, 0, 4007, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 443, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4096, 4096, 3798, 4094, 4026, 4494, 4096] avg=4096.0000 max=4494 max_rank=6 min=3798 min_rank=3 imbalance_ratio=1.097168 max_load_reduction=28274 max_load_reduction_pct=0.862854 moved_to_echo=16710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 29312, 3584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 2944, 29056, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 256, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32128, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4063, 0, 3, 2969, 0, 0, 4090, 0, 4096, 0, 1160, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4070, 0, 0, 3757, 0, 0, 4083, 0, 4096, 0, 378, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 3962, 1, 2, 3066, 0, 0, 4031, 0, 4096, 0, 1226, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 3946, 0, 3, 3921, 0, 0, 4079, 0, 4096, 0, 339, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4077, 0, 1, 3379, 0, 0, 4085, 0, 4096, 0, 743, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4051, 0, 0, 3847, 0, 2, 4078, 0, 4096, 0, 310, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 4093, 0, 4096, 3, 0, 0, 3874, 0, 19, 3189, 0, 4, 4033, 0, 4096, 2, 1165, 4096, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4084, 0, 3, 3878, 0, 0, 3786, 0, 4096, 0, 535, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 79, 0, 0, 0, 0, 0, 3757, 0, 591, 0, 0, 0, 0, 4083, 0, 63, 0, 0, 0, 0, 0, 378, 3365, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4033, 61, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4149, 4348, 4146, 3743, 4096, 4096, 4094] avg=4096.0000 max=4348 max_rank=2 min=3743 min_rank=4 imbalance_ratio=1.061523 max_load_reduction=28420 max_load_reduction_pct=0.867310 moved_to_echo=16029 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 79, 0, 0, 0, 1, 2, 3066, 0, 591, 0, 0, 0, 0, 4031, 0, 63, 0, 0, 0, 0, 0, 1226, 3365, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4033, 61, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4041, 3660, 4094, 4591, 4096, 4096, 4094] avg=4096.0000 max=4591 max_rank=4 min=3660 min_rank=2 imbalance_ratio=1.120850 max_load_reduction=28177 max_load_reduction_pct=0.859894 moved_to_echo=16877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 79, 0, 0, 0, 0, 3, 2969, 0, 591, 0, 0, 0, 0, 4090, 0, 63, 0, 0, 0, 0, 0, 1160, 3365, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4033, 61, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4142, 3563, 4153, 4525, 4096, 4096, 4097] avg=4096.0000 max=4525 max_rank=4 min=3563 min_rank=2 imbalance_ratio=1.104736 max_load_reduction=28243 max_load_reduction_pct=0.861908 moved_to_echo=16814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4051, 79, 0, 0, 0, 0, 0, 3847, 0, 591, 0, 0, 0, 2, 4078, 0, 63, 0, 0, 0, 0, 0, 310, 3365, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4033, 61, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4130, 4438, 4143, 3675, 4096, 4095, 4094] avg=4096.0000 max=4438 max_rank=2 min=3675 min_rank=4 imbalance_ratio=1.083496 max_load_reduction=28330 max_load_reduction_pct=0.864563 moved_to_echo=15960 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 85, 0, 0, 0, 0, 3, 3921, 0, 593, 0, 0, 0, 0, 4079, 0, 56, 0, 0, 0, 0, 0, 339, 3355, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4038, 62, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4031, 4517, 4135, 3694, 4097, 4098, 4100] avg=4096.0000 max=4517 max_rank=2 min=3694 min_rank=4 imbalance_ratio=1.102783 max_load_reduction=28251 max_load_reduction_pct=0.862152 moved_to_echo=15989 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 1, 0, 0, 0, 0, 3, 0, 0, 3874, 79, 0, 0, 0, 0, 19, 3189, 0, 591, 0, 0, 0, 4, 4033, 0, 63, 0, 0, 0, 0, 2, 1165, 3365, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 2, 4033, 61, 0, 0] rerouted_tokens_per_ep_rank=[4094, 3956, 3799, 4100, 4532, 4095, 4096, 4096] avg=4096.0000 max=4532 max_rank=4 min=3799 min_rank=2 imbalance_ratio=1.106445 max_load_reduction=28236 max_load_reduction_pct=0.861694 moved_to_echo=16819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 79, 0, 0, 0, 0, 1, 3379, 0, 591, 0, 0, 0, 0, 4085, 0, 63, 0, 0, 0, 0, 0, 743, 3365, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4033, 61, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4156, 3971, 4148, 4108, 4096, 4096, 4097] avg=4096.0000 max=4156 max_rank=1 min=3971 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4084, 79, 0, 0, 0, 0, 3, 3878, 0, 591, 0, 0, 0, 0, 3786, 0, 63, 0, 0, 0, 0, 0, 535, 3365, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 2, 4033, 61, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4163, 4472, 3849, 3900, 4096, 4095, 4096] avg=4096.0000 max=4472 max_rank=2 min=3849 min_rank=3 imbalance_ratio=1.091797 max_load_reduction=28296 max_load_reduction_pct=0.863525 moved_to_echo=16187 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 5888, 27008, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 28032, 0, 4736, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32384, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32128, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32384, 0, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2899, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 2832, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 4096, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3039, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2789, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2329, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 4096, 1766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2760, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 4096, 1334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 1637, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 4096, 2456, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3467, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2329, 1377, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2719, 1375, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2721, 1766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3706, 4096, 4096, 4096, 4487, 4096, 4095] avg=4096.0000 max=4487 max_rank=5 min=3706 min_rank=1 imbalance_ratio=1.095459 max_load_reduction=28281 max_load_reduction_pct=0.863068 moved_to_echo=16774 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3039, 1377, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 1375, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2721, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4416, 4096, 4094, 4096, 3778, 4096, 4096] avg=4096.0000 max=4416 max_rank=1 min=3778 min_rank=5 imbalance_ratio=1.078125 max_load_reduction=28352 max_load_reduction_pct=0.865234 moved_to_echo=16066 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 2832, 1377, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2719, 1375, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2721, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4209, 4096, 4096, 4096, 3983, 4096, 4097] avg=4096.0000 max=4209 max_rank=1 min=3983 min_rank=5 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16272 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4, 0, 0, 0, 2899, 1377, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 1379, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2713, 1197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4276, 4096, 4099, 4096, 3910, 4096, 4095] avg=4096.0000 max=4276 max_rank=1 min=3910 min_rank=5 imbalance_ratio=1.043945 max_load_reduction=28492 max_load_reduction_pct=0.869507 moved_to_echo=16197 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2789, 1377, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 1375, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2721, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4166, 4096, 4095, 4096, 4028, 4096, 4095] avg=4096.0000 max=4166 max_rank=1 min=4028 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16315 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 0, 0, 1637, 1377, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2719, 1375, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2721, 2456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 3014, 4096, 4097, 4096, 5177, 4096, 4099] avg=4096.0000 max=5177 max_rank=5 min=3014 min_rank=1 imbalance_ratio=1.263916 max_load_reduction=27591 max_load_reduction_pct=0.842010 moved_to_echo=17468 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2760, 1377, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2719, 1375, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2721, 1334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4137, 4096, 4097, 4096, 4055, 4096, 4095] avg=4096.0000 max=4137 max_rank=1 min=4055 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16342 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3467, 1377, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 1375, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2721, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4844, 4096, 4094, 4096, 3350, 4096, 4096] avg=4096.0000 max=4844 max_rank=1 min=3350 min_rank=5 imbalance_ratio=1.182617 max_load_reduction=27924 max_load_reduction_pct=0.852173 moved_to_echo=15638 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 21760, 11008, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[21760, 11136, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[21760, 11008, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 4, 4096, 0, 4096, 0, 0, 0, 4092, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4095, 3, 4096, 0, 4096, 0, 0, 0, 4089, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4097, 4096, 4095, 4096] avg=4096.0000 max=4097 max_rank=4 min=4095 min_rank=6 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16384 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4098, 4096, 4096, 4096, 4095, 4095] avg=4096.0000 max=4098 max_rank=2 min=4095 min_rank=6 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16382 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4098, 4096, 4096, 4096, 4096, 4094] avg=4096.0000 max=4098 max_rank=2 min=4094 min_rank=7 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16382 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4096, 4094, 4096, 4096, 4096, 4096, 4094] avg=4096.0000 max=4100 max_rank=0 min=4094 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16382 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4096, 4096, 4096, 4096, 4096, 4096, 4102] avg=4096.0000 max=4102 max_rank=7 min=4090 min_rank=0 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16390 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 5, 4092, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4091, 4096, 4096, 4096, 4098, 4097] avg=4096.0000 max=4098 max_rank=0 min=4091 min_rank=2 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16387 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4097, 4096, 4096, 4096, 4096, 4095] avg=4096.0000 max=4097 max_rank=2 min=4095 min_rank=7 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16383 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4098, 4096, 4095, 4096, 4096, 4095] avg=4096.0000 max=4098 max_rank=2 min=4095 min_rank=4 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16382 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4060, 0, 4096, 0, 4096, 4069, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 44, 4090, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3966, 0, 4096, 0, 4096, 4067, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4096, 38, 0, 0, 0, 0, 124, 4091, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4062, 0, 4096, 0, 4096, 4077, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 4095, 31, 1, 0, 0, 0, 22, 4095, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3964, 0, 4096, 0, 4096, 4075, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 37, 0, 0, 0, 0, 116, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3868, 0, 4096, 0, 4096, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 38, 0, 0, 0, 0, 201, 4089, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4076, 0, 4096, 0, 4096, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 22, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3953, 0, 4096, 0, 4096, 4049, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 6, 4094, 84, 6, 0, 0, 0, 105, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4086, 0, 4096, 0, 4096, 4081, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4060, 0, 92, 0, 0, 0, 0, 0, 21, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 3984, 77, 32, 0, 4063, 26, 0, 0, 0, 0, 0, 0, 0, 0, 44, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 20, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4152, 4090, 4096, 4093, 4089, 4057, 4095] avg=4096.0000 max=4152 max_rank=1 min=4057 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16334 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4062, 0, 92, 0, 0, 0, 0, 0, 21, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 3, 3984, 77, 32, 0, 4062, 31, 1, 0, 0, 0, 0, 0, 0, 0, 22, 4018, 0, 0, 0, 0, 0, 1, 0, 0, 4075, 20, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4154, 4098, 4094, 4096, 4094, 4040, 4096] avg=4096.0000 max=4154 max_rank=1 min=4040 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16326 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4076, 0, 92, 0, 0, 0, 0, 0, 21, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3984, 77, 32, 0, 4063, 3, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 20, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4168, 4114, 4097, 4093, 4066, 4039, 4095] avg=4096.0000 max=4168 max_rank=1 min=4039 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16293 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3868, 0, 92, 0, 0, 0, 0, 0, 21, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3984, 77, 32, 0, 4063, 38, 0, 0, 0, 0, 0, 0, 0, 0, 201, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 20, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3960, 4113, 4097, 4093, 4101, 4213, 4095] avg=4096.0000 max=4213 max_rank=6 min=3960 min_rank=1 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16502 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3966, 0, 92, 0, 0, 0, 0, 0, 21, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 1, 3984, 77, 32, 0, 4063, 38, 0, 0, 0, 0, 0, 0, 0, 0, 124, 4014, 0, 0, 0, 0, 0, 0, 2, 0, 4075, 20, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4058, 4088, 4096, 4094, 4101, 4138, 4097] avg=4096.0000 max=4138 max_rank=6 min=4058 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16430 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3964, 0, 89, 0, 0, 0, 0, 0, 18, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 3985, 84, 38, 0, 4051, 37, 0, 0, 0, 0, 0, 0, 0, 0, 116, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 22, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4053, 4093, 4103, 4107, 4088, 4128, 4100] avg=4096.0000 max=4128 max_rank=6 min=4053 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16423 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3953, 0, 92, 0, 0, 0, 0, 0, 21, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 6, 3984, 77, 32, 0, 4061, 84, 6, 0, 0, 0, 0, 0, 0, 0, 105, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 20, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4045, 4070, 4088, 4099, 4151, 4124, 4095] avg=4096.0000 max=4151 max_rank=5 min=4045 min_rank=1 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16469 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4086, 0, 92, 0, 0, 0, 0, 0, 21, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3984, 77, 32, 0, 4063, 15, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 20, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4178, 4102, 4097, 4093, 4078, 4029, 4095] avg=4096.0000 max=4178 max_rank=1 min=4029 min_rank=6 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16295 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 32000, 640, 384, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32128, 0, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 768, 32128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4096, 4096, 4096, 4096, 4097, 4096] avg=4096.0000 max=4097 max_rank=6 min=4095 min_rank=0 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4095, 4096, 4096, 4097, 4096] avg=4096.0000 max=4097 max_rank=6 min=4095 min_rank=3 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4097, 4096, 4096, 4097, 4094, 4096] avg=4096.0000 max=4097 max_rank=2 min=4094 min_rank=6 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16383 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4095, 4097, 4096, 4095, 4096, 4096] avg=4096.0000 max=4097 max_rank=0 min=4095 min_rank=2 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16383 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 4095, 1, 4096, 0, 0, 2, 0, 3601, 4096, 4096, 0, 0, 495, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4086, 4096, 4096, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4089, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4084, 4096, 4096, 0, 0, 12, 0, 7, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4080, 4096, 4096, 0, 0, 16, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 3437, 4096, 4096, 0, 0, 659, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4080, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 3756, 4096, 4096, 0, 0, 340, 0, 15, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 3601, 192, 0, 0, 0, 0, 3905, 0, 0, 495, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 191, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4096, 4095, 4097, 3795, 4400, 4092] avg=4096.0000 max=4400 max_rank=6 min=3795 min_rank=5 imbalance_ratio=1.074219 max_load_reduction=28368 max_load_reduction_pct=0.865723 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 192, 0, 0, 0, 0, 3905, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 191, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4100, 4096, 4096, 4278, 3915, 4092] avg=4096.0000 max=4278 max_rank=5 min=3915 min_rank=6 imbalance_ratio=1.044434 max_load_reduction=28490 max_load_reduction_pct=0.869446 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 3437, 192, 0, 0, 0, 0, 3905, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 191, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4099, 4096, 4096, 3630, 4564, 4092] avg=4096.0000 max=4564 max_rank=6 min=3630 min_rank=5 imbalance_ratio=1.114258 max_load_reduction=28204 max_load_reduction_pct=0.860718 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 192, 0, 0, 0, 0, 3905, 0, 0, 12, 0, 0, 0, 0, 0, 7, 0, 0, 3901, 191, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4093, 4096, 4096, 4276, 3917, 4099] avg=4096.0000 max=4276 max_rank=5 min=3917 min_rank=6 imbalance_ratio=1.043945 max_load_reduction=28492 max_load_reduction_pct=0.869507 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 185, 0, 0, 0, 0, 3900, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 195, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4096, 4096, 4098, 4095, 4265, 3916, 4101] avg=4096.0000 max=4265 max_rank=5 min=3916 min_rank=6 imbalance_ratio=1.041260 max_load_reduction=28503 max_load_reduction_pct=0.869843 moved_to_echo=16377 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4080, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 3756, 192, 0, 0, 0, 0, 3905, 0, 0, 340, 0, 0, 0, 0, 0, 15, 0, 0, 3901, 191, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4084, 4096, 4096, 3949, 4245, 4107] avg=4096.0000 max=4245 max_rank=6 min=3949 min_rank=5 imbalance_ratio=1.036377 max_load_reduction=28523 max_load_reduction_pct=0.870453 moved_to_echo=16397 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 192, 0, 0, 0, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3901, 191, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4100, 4095, 4096, 4288, 3905, 4093] avg=4096.0000 max=4288 max_rank=5 min=3905 min_rank=6 imbalance_ratio=1.046875 max_load_reduction=28480 max_load_reduction_pct=0.869141 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 192, 0, 0, 0, 0, 3905, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 191, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4100, 4096, 4096, 4287, 3906, 4092] avg=4096.0000 max=4287 max_rank=5 min=3906 min_rank=6 imbalance_ratio=1.046631 max_load_reduction=28481 max_load_reduction_pct=0.869171 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[31360, 0, 0, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 31360, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 31232, 1536, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 2092, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 2004, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 2097, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 1999, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 490, 0, 4091, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3605, 0, 7, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 294, 0, 4094, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3804, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 513, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3583, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 278, 0, 4086, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3828, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 2108, 0, 4087, 0, 0, 0, 0, 4096, 4096, 4096, 0, 1991, 0, 9, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 176, 0, 4096, 0, 0, 0, 0, 4096, 4092, 4096, 0, 3924, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2092, 0, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1004, 0, 0, 2004, 0, 0, 0, 0, 0, 0, 0, 0, 3092, 1003, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 5185, 4096, 3008, 4095] avg=4096.0000 max=5185 max_rank=4 min=3008 min_rank=6 imbalance_ratio=1.265869 max_load_reduction=27583 max_load_reduction_pct=0.841766 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 294, 0, 3092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1004, 0, 0, 3804, 0, 0, 0, 0, 0, 0, 0, 0, 3092, 1002, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 3386, 4096, 4808, 4094] avg=4096.0000 max=4808 max_rank=6 min=3386 min_rank=4 imbalance_ratio=1.173828 max_load_reduction=27960 max_load_reduction_pct=0.853271 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2097, 0, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1004, 0, 0, 1999, 0, 0, 0, 0, 0, 0, 0, 0, 3092, 1003, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 5190, 4096, 3003, 4095] avg=4096.0000 max=5190 max_rank=4 min=3003 min_rank=6 imbalance_ratio=1.267090 max_load_reduction=27578 max_load_reduction_pct=0.841614 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 490, 0, 3090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1004, 0, 0, 3605, 0, 0, 0, 0, 0, 7, 0, 0, 3092, 1001, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4095, 3580, 4096, 4609, 4100] avg=4096.0000 max=4609 max_rank=6 min=3580 min_rank=4 imbalance_ratio=1.125244 max_load_reduction=28159 max_load_reduction_pct=0.859344 moved_to_echo=16385 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 513, 0, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1003, 0, 0, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 3093, 1006, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4100, 3599, 4096, 4586, 4099] avg=4096.0000 max=4586 max_rank=6 min=3599 min_rank=4 imbalance_ratio=1.119629 max_load_reduction=28182 max_load_reduction_pct=0.860046 moved_to_echo=16380 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2108, 0, 3087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1004, 0, 0, 1991, 0, 0, 0, 0, 0, 9, 0, 0, 3092, 1000, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4093, 5195, 4096, 2995, 4101] avg=4096.0000 max=5195 max_rank=4 min=2995 min_rank=6 imbalance_ratio=1.268311 max_load_reduction=27573 max_load_reduction_pct=0.841461 moved_to_echo=16387 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 278, 0, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1004, 0, 0, 3828, 0, 0, 0, 0, 0, 0, 0, 0, 3092, 1000, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 3364, 4096, 4832, 4092] avg=4096.0000 max=4832 max_rank=6 min=3364 min_rank=4 imbalance_ratio=1.179688 max_load_reduction=27936 max_load_reduction_pct=0.852539 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 176, 0, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1003, 0, 0, 3924, 0, 0, 0, 0, 0, 0, 0, 0, 3089, 1003, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 3269, 4096, 4927, 4092] avg=4096.0000 max=4927 max_rank=6 min=3269 min_rank=4 imbalance_ratio=1.202881 max_load_reduction=27841 max_load_reduction_pct=0.849640 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 0, 0, 24832, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 0, 24832, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 24832, 8064, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4087, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4071, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4060, 4095, 4096, 0, 0, 18, 1, 0, 0, 0, 23, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 4090, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4073, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4070, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4094, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4090, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 4079, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4096, 4081, 4095, 4096, 4095, 4096, 4096] avg=4096.0000 max=4113 max_rank=0 min=4081 min_rank=2 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 3, 4079, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4094, 4085, 4096, 4096, 4095, 4096, 4096] avg=4096.0000 max=4110 max_rank=0 min=4085 min_rank=2 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4073, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 23, 4079, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4093, 4104, 4096, 4096, 4095, 4096, 4098] avg=4096.0000 max=4104 max_rank=2 min=4090 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4087, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 10, 4083, 2, 0, 0, 0, 4095, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4095, 4099, 4096, 4095, 4096, 4096] avg=4096.0000 max=4099 max_rank=3 min=4095 min_rank=1 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4070, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 26, 4079, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4092, 4107, 4096, 4096, 4098, 4096, 4096] avg=4096.0000 max=4107 max_rank=2 min=4087 min_rank=0 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4060, 17, 0, 0, 0, 0, 0, 0, 0, 18, 1, 4092, 0, 0, 0, 0, 0, 0, 23, 4078, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4111, 4103, 4094, 4096, 4099, 4096, 4092] avg=4096.0000 max=4111 max_rank=1 min=4077 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4071, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 25, 4079, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4095, 4106, 4096, 4096, 4095, 4096, 4096] avg=4096.0000 max=4106 max_rank=2 min=4088 min_rank=0 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 6, 4079, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4092, 4087, 4096, 4096, 4096, 4096, 4098] avg=4096.0000 max=4107 max_rank=0 min=4087 min_rank=2 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 32640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[8, 1, 513, 3, 2904, 0, 3446, 0, 2745, 704, 3, 81, 2609, 3795, 1, 472, 13, 1214, 2696, 0, 0, 180, 563, 2, 0, 853, 3767, 2610, 3298, 96, 191, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1, 5, 965, 2, 3238, 6, 3190, 2, 2896, 633, 3, 36, 2125, 3800, 17, 832, 11, 1249, 3228, 0, 0, 125, 748, 13, 3, 619, 3662, 2588, 2602, 115, 52, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[7, 0, 365, 0, 2575, 1, 3267, 0, 2647, 1437, 0, 24, 2738, 3883, 0, 499, 9, 718, 2774, 0, 0, 90, 574, 1, 0, 527, 3849, 2682, 3557, 432, 112, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[12, 3, 539, 3, 3095, 3, 3221, 3, 2666, 575, 0, 43, 2506, 3863, 6, 543, 46, 1229, 2685, 1, 0, 236, 961, 14, 2, 947, 3824, 2683, 2810, 100, 144, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[5, 3, 721, 0, 2359, 1, 3454, 0, 2120, 1093, 0, 24, 2438, 3885, 0, 464, 36, 1232, 2511, 1, 0, 104, 756, 2, 0, 979, 3901, 2834, 3401, 242, 201, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[37, 0, 1070, 1, 1964, 0, 3250, 9, 2414, 755, 0, 48, 2938, 3755, 0, 667, 9, 1064, 2713, 0, 0, 385, 901, 10, 0, 636, 3895, 2734, 3227, 153, 133, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[11, 0, 742, 0, 3027, 0, 2807, 1, 2396, 752, 0, 60, 2853, 3832, 0, 472, 53, 1335, 2888, 0, 0, 107, 420, 0, 0, 1089, 3933, 2839, 2987, 111, 53, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 667, 0, 2625, 0, 3064, 0, 2656, 1105, 0, 28, 2988, 3934, 0, 402, 5, 1310, 2189, 0, 0, 117, 610, 0, 0, 471, 3854, 3025, 3431, 173, 113, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 5, 965, 2, 3107, 129, 0, 0, 3238, 6, 1362, 2, 0, 0, 0, 0, 2896, 633, 3, 36, 598, 0, 0, 0, 2125, 890, 17, 832, 0, 0, 0, 0, 11, 1249, 3228, 0, 192, 0, 0, 0, 0, 125, 748, 13, 2781, 415, 0, 0, 3, 619, 555, 2588, 0, 0, 0, 0, 2602, 115, 52, 2, 623, 0, 0, 0] rerouted_tokens_per_ep_rank=[4209, 4608, 4166, 3864, 4680, 4082, 3765, 3394] avg=4096.0000 max=4680 max_rank=4 min=3394 min_rank=7 imbalance_ratio=1.142578 max_load_reduction=28088 max_load_reduction_pct=0.857178 moved_to_echo=15921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[8, 1, 513, 3, 3196, 128, 0, 0, 2904, 0, 1472, 0, 0, 0, 0, 0, 2745, 704, 3, 81, 646, 0, 0, 0, 2609, 890, 1, 472, 0, 0, 0, 0, 13, 1214, 2696, 0, 207, 0, 0, 0, 0, 180, 563, 2, 2777, 448, 0, 0, 0, 853, 571, 2610, 0, 0, 0, 0, 3298, 96, 191, 0, 673, 0, 0, 0] rerouted_tokens_per_ep_rank=[3849, 4376, 4179, 3972, 4130, 3970, 4034, 4258] avg=4096.0000 max=4376 max_rank=1 min=3849 min_rank=0 imbalance_ratio=1.068359 max_load_reduction=28392 max_load_reduction_pct=0.866455 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[12, 3, 539, 3, 3244, 131, 0, 0, 3095, 3, 1375, 3, 0, 0, 0, 0, 2666, 575, 0, 43, 604, 0, 0, 0, 2506, 905, 6, 543, 0, 0, 0, 0, 46, 1229, 2685, 1, 194, 0, 0, 0, 0, 236, 961, 14, 2827, 419, 0, 0, 2, 947, 580, 2683, 0, 0, 0, 0, 2810, 100, 144, 5, 629, 0, 0, 0] rerouted_tokens_per_ep_rank=[3932, 4476, 3888, 3960, 4155, 4457, 4212, 3688] avg=4096.0000 max=4476 max_rank=1 min=3688 min_rank=7 imbalance_ratio=1.092773 max_load_reduction=28292 max_load_reduction_pct=0.863403 moved_to_echo=16512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 365, 0, 3266, 131, 0, 0, 2575, 1, 1395, 0, 0, 0, 0, 0, 2647, 1437, 0, 24, 613, 0, 0, 0, 2738, 910, 0, 499, 0, 0, 0, 0, 9, 718, 2774, 0, 196, 0, 0, 0, 0, 90, 574, 1, 2842, 425, 0, 0, 0, 527, 583, 2682, 0, 0, 0, 0, 3557, 432, 112, 0, 638, 0, 0, 0] rerouted_tokens_per_ep_rank=[3769, 3971, 4721, 4147, 3697, 3932, 3792, 4739] avg=4096.0000 max=4739 max_rank=7 min=3697 min_rank=4 imbalance_ratio=1.156982 max_load_reduction=28029 max_load_reduction_pct=0.855377 moved_to_echo=16160 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[37, 0, 1070, 1, 3308, 131, 0, 0, 1964, 0, 1370, 9, 0, 0, 0, 0, 2414, 755, 0, 48, 614, 0, 0, 0, 2938, 873, 0, 667, 0, 0, 0, 0, 9, 1064, 2713, 0, 199, 0, 0, 0, 0, 385, 901, 10, 2751, 426, 0, 0, 0, 636, 587, 2734, 0, 0, 0, 0, 3227, 153, 133, 0, 641, 0, 0, 0] rerouted_tokens_per_ep_rank=[4547, 3343, 3831, 4478, 3985, 4473, 3957, 4154] avg=4096.0000 max=4547 max_rank=0 min=3343 min_rank=1 imbalance_ratio=1.110107 max_load_reduction=28221 max_load_reduction_pct=0.861237 moved_to_echo=16569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 3, 721, 0, 3310, 132, 0, 0, 2359, 1, 1474, 0, 0, 0, 0, 0, 2120, 1093, 0, 24, 648, 0, 0, 0, 2438, 910, 0, 464, 0, 0, 0, 0, 36, 1232, 2511, 1, 208, 0, 0, 0, 0, 104, 756, 2, 2843, 449, 0, 0, 0, 979, 591, 2834, 0, 0, 0, 0, 3401, 242, 201, 1, 675, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 3834, 3885, 3812, 3988, 4154, 4404, 4520] avg=4096.0000 max=4520 max_rank=7 min=3812 min_rank=3 imbalance_ratio=1.103516 max_load_reduction=28248 max_load_reduction_pct=0.862061 moved_to_echo=17066 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 742, 0, 3337, 130, 0, 0, 3027, 0, 1199, 1, 0, 0, 0, 0, 2396, 752, 0, 60, 526, 0, 0, 0, 2853, 898, 0, 472, 0, 0, 0, 0, 53, 1335, 2888, 0, 169, 0, 0, 0, 0, 107, 420, 0, 2804, 365, 0, 0, 0, 1089, 596, 2839, 0, 0, 0, 0, 2987, 111, 53, 0, 548, 0, 0, 0] rerouted_tokens_per_ep_rank=[4220, 4227, 3734, 4223, 4445, 3696, 4524, 3699] avg=4096.0000 max=4524 max_rank=6 min=3696 min_rank=5 imbalance_ratio=1.104492 max_load_reduction=28244 max_load_reduction_pct=0.861938 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 667, 0, 3270, 133, 0, 0, 2625, 0, 1308, 0, 0, 0, 0, 0, 2656, 1105, 0, 28, 575, 0, 0, 0, 2988, 922, 0, 402, 0, 0, 0, 0, 5, 1310, 2189, 0, 184, 0, 0, 0, 0, 117, 610, 0, 2879, 398, 0, 0, 0, 471, 584, 3025, 0, 0, 0, 0, 3431, 173, 113, 0, 599, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 3933, 4364, 4312, 3688, 4004, 4080, 4316] avg=4096.0000 max=4364 max_rank=2 min=3688 min_rank=4 imbalance_ratio=1.065430 max_load_reduction=28404 max_load_reduction_pct=0.866821 moved_to_echo=16088 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21248, 7296, 128, 4352, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 6144, 4736, 22016, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 9472, 21760, 128, 1664, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 1408, 5632, 128, 22528, 3456, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[21888, 128, 11008, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 5632, 128, 26112, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[20608, 7168, 128, 384, 4864, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[25344, 1536, 1024, 128, 5120, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 4096, 0, 0, 0, 0, 4094, 0, 0, 3, 0, 22, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 1, 4073, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4071, 2, 1, 0, 4080, 3, 1, 0, 0, 4092, 0, 0, 1, 8, 10, 4089, 4084, 2, 0, 2, 4093, 1, 5, 0, 24, 4095, 10, 2, 1, 7, 0, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4094, 0, 1, 0, 0, 4096, 1, 0, 2, 0, 1, 4096, 4095, 0, 0, 0, 4094, 1, 0, 0, 3, 4093, 0, 0, 0, 0, 1, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4095, 1, 1, 0, 4095, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 4095, 0, 1, 0, 4092, 0, 0, 0, 7, 4095, 0, 0, 1, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4087, 0, 2, 0, 4093, 1, 0, 0, 0, 4096, 0, 0, 0, 4, 6, 4086, 4096, 0, 0, 0, 4095, 0, 0, 4, 9, 4095, 0, 0, 0, 2, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4093, 1, 0, 2, 4095, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 10, 4087, 4091, 1, 3, 0, 4096, 0, 0, 5, 0, 4095, 1, 0, 0, 1, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 4096, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 12, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 6, 4082, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 3, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 3, 0, 22, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4097, 4095, 4116, 4097, 4094, 4073, 4096] avg=4096.0000 max=4116 max_rank=3 min=4073 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16360 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4071, 2, 1, 0, 3, 0, 0, 0, 4080, 3, 1, 0, 1, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 1, 8, 10, 4084, 0, 0, 0, 0, 4084, 2, 0, 2, 1, 0, 0, 0, 4093, 1, 5, 0, 0, 0, 0, 0, 24, 4094, 10, 2, 0, 0, 0, 0, 1, 7, 0, 4084, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4085, 4093, 4103, 4089, 4099, 4130, 4092] avg=4096.0000 max=4130 max_rank=6 min=4077 min_rank=0 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16410 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 2, 0, 3, 0, 0, 0, 4093, 1, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4, 6, 4081, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 4, 0, 0, 0, 0, 9, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4095, 4097, 4091, 4097, 4099, 4103, 4094] avg=4096.0000 max=4103 max_rank=6 min=4091 min_rank=3 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16393 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4095, 1, 1, 0, 3, 0, 0, 0, 4095, 0, 1, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 7, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4097, 4095, 4091, 4097, 4092, 4101, 4095] avg=4096.0000 max=4101 max_rank=6 min=4091 min_rank=3 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 3, 0, 0, 0, 4094, 0, 1, 0, 1, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 0, 0, 2, 0, 1, 4091, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4098, 4094, 4096, 4095, 4095, 4097] avg=4096.0000 max=4098 max_rank=2 min=4094 min_rank=3 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16383 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4093, 1, 0, 2, 7, 0, 0, 0, 4095, 0, 0, 2, 7, 0, 0, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 0, 0, 10, 4069, 0, 0, 0, 0, 4091, 1, 3, 0, 3, 0, 0, 0, 4096, 0, 0, 5, 4, 0, 0, 0, 0, 4083, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4104, 4098, 4079, 4098, 4105, 4084, 4097] avg=4096.0000 max=4105 max_rank=5 min=4079 min_rank=3 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 3, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4092, 1, 0, 1, 0, 0, 0, 0, 0, 12, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 6, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4097, 4094, 4103, 4097, 4094, 4087, 4096] avg=4096.0000 max=4103 max_rank=3 min=4087 min_rank=6 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16374 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 1, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4097, 4098, 4091, 4097, 4090, 4095, 4101] avg=4096.0000 max=4101 max_rank=7 min=4090 min_rank=5 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16383 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[6, 4090, 4094, 0, 2, 0, 4096, 5, 4086, 4096, 0, 1, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 16, 8, 0, 0, 2, 0, 1, 4080, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 4093, 4004, 3, 0, 1, 4091, 2, 4082, 4090, 0, 1, 4094, 1, 4, 4085, 4, 4, 0, 2, 0, 1, 84, 2, 6, 0, 5, 0, 1, 4091, 1, 16] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 4095, 4086, 0, 1, 1, 4096, 3, 4090, 4095, 0, 0, 4095, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 11, 2, 2, 0, 0, 1, 0, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 4093, 4095, 0, 0, 0, 4096, 3, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1, 4096, 4062, 0, 0, 0, 4096, 8, 4079, 4096, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 31, 20, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 4087, 4071, 1, 1, 0, 4095, 1, 4093, 4093, 7, 1, 4095, 1, 0, 4094, 1, 1, 0, 1, 0, 0, 24, 3, 0, 0, 0, 0, 0, 4095, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4088, 4082, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 13, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 4085, 4084, 0, 0, 0, 4096, 1, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 23, 4062, 0, 0, 0, 0, 0, 0, 0, 4094, 8, 0, 0, 0, 0, 4079, 4, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 31, 20, 4067, 2, 0, 0, 0, 0, 0, 0, 4091, 6, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4102, 4083, 4089, 4096, 4120, 4097, 4095] avg=4096.0000 max=4120 max_rank=5 min=4083 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 23, 4004, 3, 0, 0, 0, 0, 0, 1, 4089, 2, 0, 0, 0, 0, 4082, 6, 0, 1, 0, 0, 0, 0, 4094, 1, 4, 2, 0, 0, 0, 0, 4, 4, 0, 2, 4083, 1, 0, 0, 0, 1, 84, 2, 4064, 2, 0, 0, 6, 0, 5, 0, 4083, 6, 0, 0, 1, 4091, 1, 16, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4092, 4089, 4101, 4094, 4153, 4100, 4109] avg=4096.0000 max=4153 max_rank=5 min=4030 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 23, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 1, 4064, 2, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4097, 4102, 4098, 4095, 4067, 4095, 4096] avg=4096.0000 max=4118 max_rank=0 min=4067 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 24, 4071, 1, 0, 0, 0, 0, 1, 0, 4093, 1, 0, 0, 0, 0, 4093, 6, 7, 1, 0, 0, 0, 0, 4095, 1, 0, 2, 0, 0, 0, 0, 1, 1, 0, 1, 4092, 1, 0, 0, 0, 0, 24, 3, 4058, 2, 0, 0, 0, 0, 0, 0, 4086, 5, 0, 0, 0, 4095, 0, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4107, 4098, 4096, 4087, 4091, 4098] avg=4096.0000 max=4107 max_rank=2 min=4087 min_rank=5 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[6, 15, 4094, 0, 0, 0, 0, 0, 2, 0, 4087, 5, 0, 0, 0, 0, 4086, 0, 0, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 16, 8, 4066, 6, 0, 0, 0, 0, 2, 0, 4090, 9, 0, 0, 1, 4080, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4094, 4087, 4089, 4102, 4096, 4101, 4084] avg=4096.0000 max=4115 max_rank=0 min=4084 min_rank=7 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 23, 4086, 0, 0, 0, 0, 0, 1, 1, 4094, 3, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 1, 0, 0, 0, 0, 11, 2, 4066, 2, 0, 0, 2, 0, 0, 1, 4088, 6, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4099, 4096, 4097, 4095, 4081, 4097, 4094] avg=4096.0000 max=4109 max_rank=0 min=4081 min_rank=5 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 24, 4082, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 10, 13, 4059, 2, 0, 0, 0, 0, 0, 0, 4088, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4094, 4102, 4098, 4095, 4084, 4093, 4096] avg=4096.0000 max=4106 max_rank=0 min=4084 min_rank=5 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 24, 4084, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 22, 4056, 2, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4095, 4102, 4098, 4095, 4080, 4094, 4096] avg=4096.0000 max=4108 max_rank=0 min=4080 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 256, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 4059, 3, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 16, 4096, 2582, 0, 0, 0, 0, 4096, 1541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 4065, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 3, 4096, 1286, 0, 0, 0, 0, 4096, 2837] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 1248, 0, 0, 0, 0, 4096, 2850] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[2, 4095, 5, 1, 1, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 1, 1, 4095, 0, 1, 4095, 0, 1, 4092, 0, 4095, 2516, 1, 0, 0, 0, 4093, 1576] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 4018, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 4088, 38, 4096, 2594, 0, 0, 0, 0, 4096, 1548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[6, 4072, 24, 5, 2, 1, 7, 0, 0, 0, 4, 4088, 5, 0, 4, 0, 4091, 0, 6, 4091, 1, 2, 4085, 3, 4090, 1841, 0, 5, 0, 0, 4087, 2248] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 4079, 0, 0, 0, 0, 0, 1, 0, 2, 0, 4094, 0, 0, 0, 0, 4095, 0, 0, 4095, 1, 0, 4095, 3, 4096, 611, 0, 0, 0, 0, 4096, 3500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 4081, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1, 4096, 852, 0, 0, 0, 0, 4096, 3257] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4059, 3, 0, 14, 3, 0, 0, 0, 0, 0, 0, 2418, 1676, 0, 0, 0, 1, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 16, 0, 0, 0, 0, 2406, 2582, 0, 0, 0, 0, 0, 0, 0, 0, 1678, 1541, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4094, 4090, 4096, 4096, 4106, 4988, 3219] avg=4096.0000 max=4988 max_rank=6 min=3219 min_rank=7 imbalance_ratio=1.217773 max_load_reduction=27780 max_load_reduction_pct=0.847778 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4065, 1, 0, 14, 3, 0, 0, 0, 0, 0, 0, 2418, 1676, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 2406, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 1678, 2837, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4094, 4097, 4095, 4097, 4095, 3692, 4515] avg=4096.0000 max=4515 max_rank=7 min=3692 min_rank=6 imbalance_ratio=1.102295 max_load_reduction=28253 max_load_reduction_pct=0.862213 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 4095, 5, 1, 14, 3, 0, 0, 1, 0, 0, 0, 2417, 1676, 0, 0, 0, 1, 1, 4095, 1, 0, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 2405, 2516, 1, 0, 0, 0, 0, 0, 0, 0, 1676, 1576, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4094, 4098, 4096, 4097, 4089, 4922, 3252] avg=4096.0000 max=4922 max_rank=6 min=3252 min_rank=7 imbalance_ratio=1.201660 max_load_reduction=27846 max_load_reduction_pct=0.849792 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[6, 4072, 24, 5, 14, 3, 0, 0, 2, 1, 7, 0, 2413, 1674, 0, 0, 0, 0, 4, 4088, 1, 0, 0, 0, 5, 0, 4, 0, 4090, 0, 0, 0, 1, 0, 6, 4091, 0, 0, 0, 0, 1, 2, 4081, 3, 0, 0, 0, 0, 2402, 1841, 0, 5, 0, 0, 0, 0, 0, 0, 1674, 2248, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4097, 4093, 4099, 4098, 4087, 4248, 3922] avg=4096.0000 max=4248 max_rank=6 min=3922 min_rank=7 imbalance_ratio=1.037109 max_load_reduction=28520 max_load_reduction_pct=0.870361 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 1, 0, 14, 3, 0, 0, 0, 0, 0, 0, 2418, 1676, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 2406, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 1678, 2850, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4094, 4096, 4096, 4096, 4092, 3654, 4528] avg=4096.0000 max=4528 max_rank=7 min=3654 min_rank=6 imbalance_ratio=1.105469 max_load_reduction=28240 max_load_reduction_pct=0.861816 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4018, 8, 0, 19, 10, 0, 0, 0, 0, 0, 0, 2424, 1682, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4073, 38, 0, 0, 0, 0, 2395, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 1671, 1548, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4106, 4100, 4097, 4091, 4111, 4989, 3219] avg=4096.0000 max=4989 max_rank=6 min=3219 min_rank=7 imbalance_ratio=1.218018 max_load_reduction=27779 max_load_reduction_pct=0.847748 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4079, 0, 0, 14, 3, 0, 0, 0, 0, 0, 1, 2418, 1676, 0, 0, 0, 2, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 4091, 3, 0, 0, 0, 0, 2406, 611, 0, 0, 0, 0, 0, 0, 0, 0, 1678, 3500, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4097, 4094, 4096, 4095, 3017, 5178] avg=4096.0000 max=5178 max_rank=7 min=3017 min_rank=6 imbalance_ratio=1.264160 max_load_reduction=27590 max_load_reduction_pct=0.841980 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4081, 1, 0, 14, 3, 0, 0, 0, 0, 0, 0, 2418, 1676, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 2406, 852, 0, 0, 0, 0, 0, 0, 0, 0, 1678, 3257, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4094, 4097, 4095, 4097, 4093, 3258, 4935] avg=4096.0000 max=4935 max_rank=7 min=3258 min_rank=6 imbalance_ratio=1.204834 max_load_reduction=27833 max_load_reduction_pct=0.849396 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 19456, 13440, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19328, 13568, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 13440, 19456, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32640, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 417, 0, 0, 0, 0, 3679, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 0, 0, 0, 4095, 0, 1, 0, 0, 6, 1, 1, 0, 4095, 447, 0, 0, 0, 1, 3660, 0, 4095, 0, 4079, 0, 4095, 4093, 1, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 138, 0, 0, 0, 0, 3958, 0, 4096, 0, 4096, 1, 4090, 4095, 0, 4094, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 712, 0, 0, 0, 0, 3384, 0, 4096, 0, 4095, 0, 4095, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 8, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 209, 0, 0, 0, 0, 3884, 0, 4096, 0, 4096, 0, 4087, 4095, 0, 4090, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[6, 6, 7, 0, 0, 4, 4091, 0, 4, 0, 2, 6, 6, 7, 0, 4089, 17, 2, 0, 0, 4, 4085, 0, 4078, 0, 4086, 1, 4090, 4084, 4, 4089, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 1029, 0, 0, 0, 0, 3067, 0, 4096, 0, 4096, 0, 4095, 4095, 0, 4096, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 397, 0, 0, 0, 0, 3699, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 417, 0, 0, 0, 3675, 0, 0, 0, 0, 3679, 0, 420, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4095, 4096, 4092, 4099, 4100, 4096] avg=4096.0000 max=4100 max_rank=6 min=4092 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 138, 0, 0, 0, 3675, 0, 0, 0, 0, 3958, 0, 420, 0, 0, 0, 0, 0, 4096, 1, 2, 0, 0, 0, 0, 4095, 0, 0, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4092, 4096, 3813, 4378, 4099, 4100] avg=4096.0000 max=4378 max_rank=5 min=3813 min_rank=4 imbalance_ratio=1.068848 max_load_reduction=28390 max_load_reduction_pct=0.866394 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 0, 4093, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 6, 4091, 1, 2, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 447, 0, 0, 0, 3674, 0, 0, 0, 1, 3660, 0, 420, 0, 0, 0, 0, 0, 4079, 0, 4, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4101, 4097, 4121, 4081, 4083, 4094] avg=4096.0000 max=4121 max_rank=4 min=4081 min_rank=5 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 6, 7, 0, 4089, 0, 0, 0, 0, 4, 4091, 0, 0, 0, 0, 0, 4, 0, 2, 6, 4086, 1, 0, 0, 6, 7, 0, 4089, 0, 0, 0, 0, 17, 2, 0, 0, 3659, 0, 0, 0, 4, 4085, 0, 418, 0, 0, 0, 0, 0, 4086, 1, 4, 0, 0, 0, 0, 4084, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4095, 4099, 4102, 3678, 4507, 4091, 4088] avg=4096.0000 max=4507 max_rank=5 min=3678 min_rank=4 imbalance_ratio=1.100342 max_load_reduction=28261 max_load_reduction_pct=0.862457 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4090, 0, 0, 0, 0, 8, 4090, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4087, 8, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 209, 0, 0, 0, 3679, 7, 6, 0, 0, 3884, 0, 409, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4098, 4101, 4089, 3901, 4293, 4096, 4099] avg=4096.0000 max=4293 max_rank=5 min=3901 min_rank=4 imbalance_ratio=1.048096 max_load_reduction=28475 max_load_reduction_pct=0.868988 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 712, 0, 0, 0, 3675, 0, 0, 0, 0, 3384, 0, 420, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4093, 4096, 4387, 3804, 4099, 4096] avg=4096.0000 max=4387 max_rank=4 min=3804 min_rank=5 imbalance_ratio=1.071045 max_load_reduction=28381 max_load_reduction_pct=0.866119 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 1, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1029, 0, 0, 0, 3675, 0, 0, 0, 0, 3067, 0, 420, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4094, 4096, 4704, 3487, 4100, 4097] avg=4096.0000 max=4704 max_rank=4 min=3487 min_rank=5 imbalance_ratio=1.148438 max_load_reduction=28064 max_load_reduction_pct=0.856445 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 397, 0, 0, 0, 3675, 0, 0, 0, 0, 3699, 0, 420, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4093, 4096, 4072, 4119, 4100, 4098] avg=4096.0000 max=4119 max_rank=5 min=4072 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 29440, 0, 3456, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3456, 128, 0, 0, 29440, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4056, 6, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 15, 1, 946, 0, 0, 0, 0, 4096, 43, 0, 4096, 0, 0, 3136, 4085, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4089, 4092, 6, 0, 7, 4087, 5, 7, 4090, 0, 0, 0, 1, 11, 26, 2, 0, 6, 6, 4089, 25, 4, 4090, 0, 7, 4035, 4076, 0, 6, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 133, 0, 0, 0, 0, 4096, 71, 0, 4096, 0, 0, 3892, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4046, 0, 11, 0, 4096, 0, 0, 4093, 0, 0, 0, 49, 3, 24, 0, 0, 0, 0, 4096, 583, 0, 4096, 0, 0, 3490, 4085, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 4094, 1, 0, 2, 4095, 1, 1, 4095, 0, 1, 0, 0, 0, 30, 0, 0, 2, 1, 4095, 61, 1, 4095, 0, 1, 4003, 4093, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4064, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 27, 0, 84, 0, 0, 0, 0, 4096, 208, 0, 4096, 0, 0, 3810, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 2, 0, 4096, 0, 0, 4095, 0, 0, 0, 3, 1, 65, 0, 1, 0, 0, 4096, 187, 0, 4096, 0, 0, 3852, 4085, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4071, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 13, 0, 40, 0, 0, 0, 0, 4096, 69, 0, 4096, 0, 0, 3999, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 4056, 6, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 15, 1, 946, 3750, 155, 2, 0, 0, 0, 0, 0, 4077, 16, 0, 0, 3941, 43, 0, 0, 0, 0, 0, 0, 0, 0, 3136, 319, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4094, 4096, 4869, 4093, 3984, 3455, 4096] avg=4096.0000 max=4869 max_rank=3 min=3455 min_rank=6 imbalance_ratio=1.188721 max_load_reduction=27899 max_load_reduction_pct=0.851410 moved_to_echo=15628 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 4092, 6, 0, 0, 0, 0, 0, 7, 4086, 5, 0, 0, 0, 0, 7, 4090, 0, 0, 0, 0, 0, 0, 0, 1, 11, 26, 3742, 155, 1, 0, 2, 0, 6, 6, 4071, 16, 0, 0, 3934, 25, 4, 0, 0, 0, 0, 0, 0, 7, 4035, 318, 0, 0, 0, 0, 0, 6, 1, 0, 4090, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4098, 4097, 3936, 4101, 3963, 4360, 4097] avg=4096.0000 max=4360 max_rank=6 min=3936 min_rank=3 imbalance_ratio=1.064453 max_load_reduction=28408 max_load_reduction_pct=0.866943 moved_to_echo=16521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 4064, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 27, 0, 84, 3759, 155, 2, 0, 0, 0, 0, 0, 4077, 16, 0, 0, 3941, 208, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 320, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4094, 4096, 4027, 4093, 4149, 4130, 4096] avg=4096.0000 max=4149 max_rank=5 min=4027 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16468 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 4094, 1, 0, 0, 0, 0, 0, 2, 4093, 1, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 30, 3758, 155, 2, 0, 0, 0, 2, 1, 4077, 16, 0, 0, 3940, 61, 1, 0, 0, 0, 0, 0, 0, 1, 4003, 319, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4096, 4097, 3945, 4096, 4002, 4323, 4096] avg=4096.0000 max=4323 max_rank=6 min=3945 min_rank=3 imbalance_ratio=1.055420 max_load_reduction=28445 max_load_reduction_pct=0.868073 moved_to_echo=16517 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 4046, 0, 0, 0, 0, 0, 11, 0, 4091, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 49, 3, 24, 3756, 159, 3, 0, 0, 0, 0, 0, 4083, 21, 1, 0, 3937, 583, 0, 0, 0, 0, 0, 0, 0, 0, 3490, 308, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4102, 4095, 3994, 4105, 4520, 3798, 4095] avg=4096.0000 max=4520 max_rank=5 min=3798 min_rank=6 imbalance_ratio=1.103516 max_load_reduction=28248 max_load_reduction_pct=0.862061 moved_to_echo=16518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 4093, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3, 1, 65, 3750, 155, 2, 0, 0, 1, 0, 0, 4077, 16, 0, 0, 3941, 187, 0, 0, 0, 0, 0, 0, 0, 0, 3852, 319, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4096, 4095, 3976, 4094, 4128, 4171, 4096] avg=4096.0000 max=4171 max_rank=6 min=3976 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16489 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 133, 3759, 155, 2, 0, 0, 0, 0, 0, 4077, 16, 0, 0, 3941, 71, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 320, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4094, 4096, 4051, 4093, 4012, 4212, 4096] avg=4096.0000 max=4212 max_rank=6 min=4012 min_rank=5 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 4071, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 13, 0, 40, 3760, 155, 2, 0, 0, 0, 0, 0, 4077, 16, 0, 0, 3941, 69, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 320, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4094, 4096, 3970, 4093, 4010, 4319, 4096] avg=4096.0000 max=4319 max_rank=6 min=3970 min_rank=3 imbalance_ratio=1.054443 max_load_reduction=28449 max_load_reduction_pct=0.868195 moved_to_echo=16518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31616, 1280, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 256, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 30336, 2560, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 1408, 30080, 1280, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32640, 256, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[6, 4090, 0, 4088, 0, 3, 0, 3940, 1, 0, 2418, 5, 0, 3589, 7, 4088, 3, 2297, 4084, 0, 7, 7, 4088, 10, 9, 0, 1, 19, 0, 2, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1, 4095, 0, 4095, 0, 1, 0, 3951, 1, 14, 3950, 1, 0, 3773, 1, 4094, 0, 596, 4095, 0, 1, 1, 4095, 0, 1, 0, 0, 1, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 3991, 1, 3, 3682, 0, 0, 4086, 0, 4096, 0, 525, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4096, 6, 0, 0, 3997, 0, 1, 3457, 0, 5, 4063, 0, 4096, 0, 760, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 3996, 0, 1, 3773, 0, 0, 4075, 0, 4096, 0, 443, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 8, 0, 0, 4051, 0, 0, 3694, 0, 7, 4062, 0, 4096, 0, 473, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 2, 0, 0, 4038, 0, 0, 4084, 0, 1, 4063, 0, 4096, 0, 100, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4036, 0, 1, 3809, 0, 0, 4035, 0, 4096, 0, 407, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 1, 0, 0, 0, 0, 6, 0, 0, 3997, 91, 2, 0, 0, 0, 1, 3457, 0, 484, 0, 0, 0, 5, 4063, 0, 126, 0, 0, 0, 0, 0, 760, 3398, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 123, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 3942, 4194, 4158, 4094, 4092, 4096] avg=4096.0000 max=4194 max_rank=3 min=3942 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 4090, 0, 1, 0, 0, 0, 0, 0, 3, 0, 3940, 91, 2, 0, 0, 1, 0, 2418, 5, 482, 0, 0, 0, 0, 3589, 7, 126, 0, 0, 0, 0, 3, 2297, 3388, 0, 0, 0, 0, 0, 7, 7, 4086, 10, 0, 0, 0, 0, 9, 0, 1, 19, 3962, 123, 0, 0, 0, 2, 6, 0, 4087, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4036, 2906, 3722, 5688, 4110, 4114, 4095] avg=4096.0000 max=5688 max_rank=4 min=2906 min_rank=2 imbalance_ratio=1.388672 max_load_reduction=27080 max_load_reduction_pct=0.826416 moved_to_echo=18007 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 4095, 0, 1, 0, 0, 0, 0, 0, 1, 0, 3951, 91, 2, 0, 0, 1, 14, 3950, 1, 484, 0, 0, 0, 0, 3773, 1, 127, 0, 0, 0, 0, 0, 596, 3397, 0, 0, 0, 0, 0, 1, 1, 4093, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3967, 123, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4045, 4450, 3901, 3993, 4095, 4092, 4095] avg=4096.0000 max=4450 max_rank=2 min=3901 min_rank=3 imbalance_ratio=1.086426 max_load_reduction=28318 max_load_reduction_pct=0.864197 moved_to_echo=16275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3996, 91, 2, 0, 0, 0, 1, 3773, 0, 484, 0, 0, 0, 0, 4075, 0, 126, 0, 0, 0, 0, 0, 443, 3398, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 123, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4089, 4258, 4201, 3841, 4094, 4092, 4096] avg=4096.0000 max=4258 max_rank=2 min=3841 min_rank=4 imbalance_ratio=1.039551 max_load_reduction=28510 max_load_reduction_pct=0.870056 moved_to_echo=16123 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 1, 0, 0, 0, 0, 8, 0, 0, 4051, 94, 3, 0, 0, 0, 0, 3694, 0, 486, 0, 0, 0, 7, 4062, 0, 118, 0, 0, 0, 0, 0, 473, 3389, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 127, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4156, 4180, 4187, 3862, 4093, 4102, 4098] avg=4096.0000 max=4187 max_rank=3 min=3862 min_rank=4 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16155 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3991, 91, 2, 0, 0, 1, 3, 3682, 0, 484, 0, 0, 0, 0, 4086, 0, 126, 0, 0, 0, 0, 0, 525, 3398, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 123, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4084, 4170, 4212, 3923, 4094, 4092, 4096] avg=4096.0000 max=4212 max_rank=3 min=3923 min_rank=4 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16205 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 2, 0, 0, 4038, 91, 2, 0, 0, 0, 0, 4084, 0, 484, 0, 0, 0, 1, 4063, 0, 126, 0, 0, 0, 0, 0, 100, 3398, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 123, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4133, 4568, 4190, 3498, 4094, 4092, 4096] avg=4096.0000 max=4568 max_rank=2 min=3498 min_rank=4 imbalance_ratio=1.115234 max_load_reduction=28200 max_load_reduction_pct=0.860596 moved_to_echo=15780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4036, 91, 2, 0, 0, 0, 1, 3809, 0, 484, 0, 0, 0, 0, 4035, 0, 126, 0, 0, 0, 0, 0, 407, 3398, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 123, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4129, 4294, 4161, 3805, 4094, 4092, 4096] avg=4096.0000 max=4294 max_rank=2 min=3805 min_rank=4 imbalance_ratio=1.048340 max_load_reduction=28474 max_load_reduction_pct=0.868958 moved_to_echo=16087 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 5632, 27264, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 28928, 128, 3968, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32000, 768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 31872, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31872, 128, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 2788, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 4096, 4096, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2594, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 4096, 1501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 3610, 4095, 0, 1, 4094, 4095, 1, 0, 0, 0, 1, 0, 4095, 0, 1, 4095, 4094, 485, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 3446, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 649, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2137, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 4096, 4096, 1953, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2903, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[10, 8, 3, 4088, 1934, 4083, 0, 7, 4088, 4088, 8, 3, 0, 0, 0, 0, 4086, 0, 7, 4087, 4085, 2158, 1, 7, 0, 0, 1, 8, 0, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3509, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 2137, 1231, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2865, 1228, 1, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 2868, 1953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 3368, 4096, 4100, 4096, 4821, 4096, 4096] avg=4096.0000 max=4821 max_rank=5 min=3368 min_rank=1 imbalance_ratio=1.177002 max_load_reduction=27947 max_load_reduction_pct=0.852875 moved_to_echo=17109 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 2903, 1231, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2865, 1228, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2868, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4134, 4096, 4094, 4096, 4061, 4096, 4096] avg=4096.0000 max=4134 max_rank=1 min=4061 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[10, 8, 3, 4087, 0, 0, 0, 0, 1934, 1227, 0, 7, 0, 0, 0, 0, 4088, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2856, 1224, 1, 0, 4086, 0, 7, 0, 0, 0, 0, 0, 2861, 2158, 1, 7, 0, 0, 0, 0, 0, 0, 1, 8, 4088, 0, 0, 0, 0, 0, 0, 8, 4087, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 3168, 4099, 4081, 4093, 5027, 4097, 4095] avg=4096.0000 max=5027 max_rank=5 min=3168 min_rank=1 imbalance_ratio=1.227295 max_load_reduction=27741 max_load_reduction_pct=0.846588 moved_to_echo=17312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4094, 0, 0, 0, 0, 3610, 1231, 0, 1, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2864, 1227, 1, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 2867, 485, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4842, 4095, 4093, 4097, 3353, 4096, 4095] avg=4096.0000 max=4842 max_rank=1 min=3353 min_rank=5 imbalance_ratio=1.182129 max_load_reduction=27926 max_load_reduction_pct=0.852234 moved_to_echo=15641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4089, 0, 0, 0, 0, 2788, 1226, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2870, 1231, 3, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 2865, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4014, 4096, 4113, 4096, 4165, 4096, 4099] avg=4096.0000 max=4165 max_rank=5 min=4014 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 3446, 1231, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2865, 1228, 1, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 2868, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4677, 4096, 4095, 4097, 3517, 4096, 4096] avg=4096.0000 max=4677 max_rank=1 min=3517 min_rank=5 imbalance_ratio=1.141846 max_load_reduction=28091 max_load_reduction_pct=0.857269 moved_to_echo=15806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 2594, 1231, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2865, 1228, 1, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 2868, 1501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 3825, 4096, 4096, 4096, 4369, 4096, 4095] avg=4096.0000 max=4369 max_rank=5 min=3825 min_rank=1 imbalance_ratio=1.066650 max_load_reduction=28399 max_load_reduction_pct=0.866669 moved_to_echo=16656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 3509, 1231, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2865, 1228, 1, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 2868, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4740, 4096, 4094, 4097, 3455, 4095, 4096] avg=4096.0000 max=4740 max_rank=1 min=3455 min_rank=5 imbalance_ratio=1.157227 max_load_reduction=28028 max_load_reduction_pct=0.855347 moved_to_echo=15743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32770, 32766, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32770, 32766, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32770, 32766, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32770, 32766, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32770, 32766, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32770, 32766, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32770, 32766, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32770, 32766, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32766 actual_valid_recv=33408 observed_loss=-642 tokens_per_expert=[0, 0, 128, 128, 23040, 9856, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[23040, 9856, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[23040, 9856, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32770 actual_valid_recv=33024 observed_loss=-254 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 7, 4096, 0, 4096, 0, 0, 0, 4087, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 8, 4096, 0, 4096, 0, 0, 0, 4087, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 5, 4096, 0, 4096, 0, 0, 0, 4091, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4080, 0, 4061, 7, 4086, 0, 7, 7, 4089, 4068, 4086, 8, 7, 9, 0, 0, 0, 0, 5, 24, 4088, 5, 9, 3, 4067, 48, 0, 0, 4, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 4078, 0, 4095, 0, 1, 1, 4095, 4095, 4094, 1, 1, 1, 0, 0, 0, 1, 0, 5, 4095, 1, 1, 1, 4089, 18, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 7, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 4092, 3, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 2, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4096, 4087, 4096, 4096, 4095, 4093, 4100] avg=4096.0000 max=4105 max_rank=0 min=4087 min_rank=2 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4061, 7, 0, 0, 0, 0, 4086, 0, 7, 7, 0, 0, 0, 0, 4089, 4, 0, 8, 0, 0, 0, 0, 7, 9, 0, 0, 4076, 3, 0, 0, 0, 0, 5, 24, 4061, 3, 1, 0, 4087, 5, 9, 3, 0, 0, 0, 0, 4064, 48, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4086, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4100, 4101, 4095, 4094, 4104, 4112, 4090] avg=4096.0000 max=4112 max_rank=6 min=4072 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4096, 4100, 4094, 4093, 4095, 4093, 4097] avg=4096.0000 max=4100 max_rank=0 min=4093 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 4089, 9, 4, 5, 4092, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4091, 4087, 4103, 4107, 4092, 4087, 4097] avg=4096.0000 max=4107 max_rank=4 min=4087 min_rank=2 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 5, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 4088, 3, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4096, 4095, 4095, 4092, 4095, 4093, 4097] avg=4096.0000 max=4105 max_rank=0 min=4092 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4078, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 4095, 4, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 4091, 3, 0, 0, 0, 1, 0, 5, 4088, 3, 1, 0, 4094, 1, 1, 1, 0, 0, 0, 0, 4086, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4097, 4100, 4096, 4098, 4097, 4104, 4094] avg=4096.0000 max=4104 max_rank=6 min=4082 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 1, 0, 0, 4090, 3, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4096, 4098, 4095, 4095, 4095, 4093, 4096] avg=4096.0000 max=4100 max_rank=0 min=4093 min_rank=6 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4096, 4100, 4094, 4093, 4095, 4093, 4097] avg=4096.0000 max=4100 max_rank=0 min=4093 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3930, 0, 4096, 0, 4096, 4073, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 7, 4096, 111, 4, 0, 0, 0, 77, 4090, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3958, 0, 4096, 0, 4096, 3959, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 7, 4090, 136, 6, 0, 0, 0, 171, 4082, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4050, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 31, 0, 0, 0, 0, 16, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[5, 1, 4088, 11, 3807, 3, 4087, 8, 3976, 4088, 0, 8, 0, 5, 8, 4085, 9, 0, 6, 0, 4088, 1, 0, 4, 6, 0, 287, 3914, 2, 0, 271, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4068, 0, 4096, 0, 4096, 4050, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 27, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1, 3994, 1, 4095, 1, 4064, 4095, 0, 1, 0, 1, 1, 4095, 0, 0, 1, 0, 4095, 4, 0, 1, 0, 0, 199, 3932, 0, 0, 92, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3965, 0, 4096, 0, 4096, 4069, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 4096, 42, 2, 0, 0, 0, 133, 4077, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4047, 0, 4096, 0, 4096, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 33, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 3930, 0, 118, 0, 0, 0, 0, 0, 31, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 1, 0, 0, 0, 0, 0, 7, 4065, 45, 0, 0, 4047, 111, 4, 0, 0, 0, 0, 0, 0, 0, 77, 4020, 0, 0, 0, 0, 0, 0, 3, 0, 3933, 70, 47, 0] rerouted_tokens_per_ep_rank=[4095, 4048, 4104, 4092, 4117, 4162, 4097, 4053] avg=4096.0000 max=4162 max_rank=5 min=4048 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 4068, 0, 118, 0, 0, 0, 0, 0, 31, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 0, 0, 0, 0, 0, 0, 4065, 45, 0, 0, 4047, 47, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 71, 47, 0] rerouted_tokens_per_ep_rank=[4095, 4186, 4081, 4099, 4110, 4094, 4052, 4051] avg=4096.0000 max=4186 max_rank=1 min=4051 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[5, 1, 4087, 11, 0, 0, 0, 0, 3807, 3, 118, 8, 0, 0, 0, 0, 30, 4088, 0, 8, 0, 0, 0, 0, 0, 5, 8, 4085, 2, 1, 0, 0, 9, 0, 6, 0, 3946, 45, 0, 0, 4039, 1, 0, 4, 0, 0, 0, 0, 6, 0, 287, 3847, 0, 0, 0, 0, 2, 0, 271, 0, 3924, 67, 47, 0] rerouted_tokens_per_ep_rank=[4104, 3936, 4126, 4101, 4006, 4044, 4140, 4311] avg=4096.0000 max=4311 max_rank=7 min=3936 min_rank=1 imbalance_ratio=1.052490 max_load_reduction=28457 max_load_reduction_pct=0.868439 moved_to_echo=16501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 0, 0, 0, 3958, 0, 110, 0, 0, 0, 0, 0, 26, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 8, 2, 0, 0, 0, 0, 0, 7, 4070, 48, 0, 0, 4033, 136, 6, 0, 0, 0, 0, 0, 0, 0, 171, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 74, 49, 0] rerouted_tokens_per_ep_rank=[4094, 4068, 3985, 4081, 4125, 4175, 4179, 4061] avg=4096.0000 max=4179 max_rank=6 min=3985 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 1, 0, 0, 0, 0, 3994, 1, 118, 1, 0, 0, 0, 0, 31, 4095, 0, 1, 0, 0, 0, 0, 0, 1, 1, 4095, 2, 1, 0, 0, 0, 0, 1, 0, 4033, 45, 0, 0, 4046, 4, 0, 1, 0, 0, 0, 0, 0, 0, 199, 3864, 0, 0, 0, 0, 0, 0, 92, 0, 3932, 68, 47, 0] rerouted_tokens_per_ep_rank=[4095, 4114, 4127, 4100, 4079, 4051, 4063, 4139] avg=4096.0000 max=4139 max_rank=7 min=4051 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 4050, 0, 118, 0, 0, 0, 0, 0, 31, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 0, 0, 0, 0, 0, 0, 4065, 45, 0, 0, 4047, 31, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 71, 47, 0] rerouted_tokens_per_ep_rank=[4095, 4168, 4127, 4099, 4110, 4078, 4040, 4051] avg=4096.0000 max=4168 max_rank=1 min=4040 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 4047, 0, 118, 0, 0, 0, 0, 0, 31, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 0, 0, 0, 0, 0, 0, 4065, 45, 0, 0, 4047, 26, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 71, 47, 0] rerouted_tokens_per_ep_rank=[4095, 4165, 4118, 4099, 4110, 4073, 4057, 4051] avg=4096.0000 max=4165 max_rank=1 min=4051 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 3965, 0, 118, 0, 0, 0, 0, 0, 31, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 1, 0, 0, 0, 0, 0, 1, 4065, 45, 0, 0, 4047, 42, 2, 0, 0, 0, 0, 0, 0, 0, 133, 4007, 0, 0, 0, 0, 0, 1, 0, 0, 3933, 70, 47, 0] rerouted_tokens_per_ep_rank=[4095, 4083, 4100, 4097, 4111, 4091, 4140, 4051] avg=4096.0000 max=4140 max_rank=6 min=4051 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 384, 0, 31488, 640, 384, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 1024, 31872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[31872, 128, 1024, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 32640, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32384, 512, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32384, 384, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[2, 1, 4094, 4078, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 4095, 4094, 1, 1, 4095, 0, 0, 1, 0, 4095, 17, 4094, 0, 0, 0, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 4096, 0, 0, 4096, 2, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[9, 7, 4077, 3976, 5, 0, 0, 0, 0, 5, 1, 9, 0, 4086, 11, 4085, 4074, 11, 2, 4087, 0, 0, 17, 3, 4088, 109, 4084, 0, 0, 0, 10, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 4080, 14, 2, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4097, 4095, 4093, 4096, 4099, 4082, 4094] avg=4096.0000 max=4112 max_rank=0 min=4082 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 16, 4078, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 4094, 1, 1, 2, 0, 0, 0, 0, 0, 0, 1, 0, 4078, 14, 0, 0, 0, 17, 4080, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4095, 4097, 4098, 4093, 4097, 4095] avg=4096.0000 max=4098 max_rank=4 min=4093 min_rank=5 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 15, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4097, 4097, 4096, 4096, 4102, 4081, 4094] avg=4096.0000 max=4105 max_rank=0 min=4081 min_rank=6 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[9, 7, 15, 3976, 0, 0, 0, 0, 5, 0, 0, 0, 4088, 0, 0, 0, 0, 5, 1, 9, 4085, 0, 0, 0, 0, 4086, 11, 0, 0, 0, 0, 0, 4074, 11, 2, 1, 0, 0, 0, 0, 0, 0, 17, 3, 4062, 14, 0, 0, 0, 109, 4070, 0, 0, 0, 0, 0, 0, 0, 10, 12, 4086, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4093, 4100, 4097, 4088, 4096, 4179, 4108] avg=4096.0000 max=4179 max_rank=6 min=4007 min_rank=0 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16471 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 14, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4096, 4096, 4096, 4098, 4094, 4082, 4094] avg=4096.0000 max=4112 max_rank=0 min=4082 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 16, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 14, 1, 0, 0, 1, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4097, 4095, 4096, 4096, 4095, 4083, 4095] avg=4096.0000 max=4111 max_rank=0 min=4083 min_rank=6 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 14, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4096, 4095, 4097, 4098, 4094, 4082, 4094] avg=4096.0000 max=4112 max_rank=0 min=4082 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 4080, 14, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4096, 4095, 4096, 4098, 4095, 4082, 4094] avg=4096.0000 max=4112 max_rank=0 min=4082 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32640, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4074, 10, 0, 0, 4055, 31, 4036, 0, 5, 4085, 4080, 4085, 13, 0, 4087, 10, 1, 9, 54, 4, 0, 4082, 8, 0, 10, 11, 0, 0, 1, 3, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4085, 0, 0, 0, 4093, 6, 4089, 0, 1, 4095, 4094, 4095, 2, 0, 4095, 1, 0, 1, 10, 1, 0, 4094, 1, 0, 1, 1, 0, 0, 1, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4096, 4096, 4094, 4097, 4096, 4095] avg=4096.0000 max=4098 max_rank=0 min=4094 min_rank=4 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4101, 4098, 4096, 4090, 4096, 4094, 4095] avg=4096.0000 max=4101 max_rank=1 min=4090 min_rank=4 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4074, 10, 0, 2, 0, 0, 0, 0, 5, 31, 4036, 0, 0, 0, 0, 0, 5, 3, 4080, 0, 0, 0, 0, 0, 13, 0, 4087, 0, 0, 0, 0, 10, 1, 9, 54, 4048, 0, 0, 0, 4, 0, 4082, 8, 0, 0, 0, 0, 0, 10, 11, 0, 4082, 1, 0, 0, 0, 1, 3, 14, 4084, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4072, 4088, 4100, 4122, 4094, 4104, 4102] avg=4096.0000 max=4122 max_rank=4 min=4072 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4085, 0, 0, 2, 0, 0, 0, 0, 5, 6, 4089, 0, 0, 0, 0, 0, 1, 2, 4094, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 1, 0, 1, 10, 4086, 1, 0, 0, 1, 0, 4094, 1, 0, 0, 0, 0, 0, 1, 1, 0, 4092, 1, 0, 0, 0, 1, 1, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4100, 4097, 4097, 4099, 4096, 4095, 4097] avg=4096.0000 max=4100 max_rank=1 min=4087 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4096, 4096, 4091, 4093, 4097, 4096, 4094] avg=4096.0000 max=4105 max_rank=0 min=4091 min_rank=3 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4101, 4097, 4096, 4090, 4096, 4095, 4095] avg=4096.0000 max=4101 max_rank=1 min=4090 min_rank=4 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4101, 4098, 4096, 4090, 4096, 4094, 4095] avg=4096.0000 max=4101 max_rank=1 min=4090 min_rank=4 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4101, 4098, 4096, 4090, 4096, 4094, 4095] avg=4096.0000 max=4101 max_rank=1 min=4090 min_rank=4 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[49, 5, 16, 9, 20, 0, 0, 4085, 2, 11, 4087, 4079, 11, 0, 0, 3591, 469, 4066, 0, 0, 0, 0, 1872, 4045, 4083, 7, 2, 2240, 9, 0, 10, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4080, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4088, 4096, 4096, 0, 0, 8, 0, 16, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 4081, 4096, 4096, 0, 0, 15, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[8, 1, 3, 1, 1, 0, 0, 4095, 0, 2, 4095, 4092, 1, 0, 0, 3968, 112, 4093, 0, 0, 0, 0, 3865, 4088, 4094, 1, 1, 245, 1, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4088, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 0, 3901, 4096, 4096, 0, 0, 195, 0, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 4094, 1, 4096, 0, 0, 2, 0, 4092, 4096, 4096, 0, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4096, 4086, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 4096, 4096, 0, 0, 4, 0, 10, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3761, 330, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 68, 8, 1, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 1, 0, 4081, 335, 0, 0, 0, 0, 3758, 0, 0, 15, 0, 0, 0, 0, 0, 1, 0, 0, 4090, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4095, 4100, 4173, 4028, 4417, 3773, 4091] avg=4096.0000 max=4417 max_rank=5 min=3773 min_rank=6 imbalance_ratio=1.078369 max_load_reduction=28351 max_load_reduction_pct=0.865204 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[49, 5, 16, 9, 3714, 329, 0, 0, 20, 0, 0, 4084, 0, 0, 0, 0, 2, 11, 6, 4079, 0, 0, 0, 0, 11, 0, 0, 3591, 68, 8, 1, 0, 469, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 1872, 331, 0, 0, 0, 0, 3746, 7, 2, 2240, 0, 0, 0, 0, 9, 0, 10, 0, 4081, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4104, 4098, 3679, 4467, 2203, 5995, 4100] avg=4096.0000 max=5995 max_rank=6 min=2203 min_rank=5 imbalance_ratio=1.463623 max_load_reduction=26773 max_load_reduction_pct=0.817047 moved_to_echo=16765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3761, 330, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 6, 4088, 0, 0, 0, 0, 0, 0, 0, 4096, 68, 8, 1, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 3, 0, 3901, 335, 0, 0, 0, 0, 3758, 0, 0, 195, 0, 0, 0, 0, 0, 5, 0, 0, 4090, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4095, 4094, 4173, 4028, 4239, 3953, 4095] avg=4096.0000 max=4239 max_rank=5 min=3953 min_rank=6 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[8, 1, 3, 1, 3753, 330, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 6, 4092, 0, 0, 0, 0, 1, 0, 0, 3968, 68, 8, 1, 0, 112, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 335, 0, 0, 0, 0, 3756, 1, 1, 245, 0, 0, 0, 0, 1, 0, 1, 0, 4089, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4100, 4046, 4137, 4200, 4003, 4091] avg=4096.0000 max=4200 max_rank=5 min=4003 min_rank=6 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3762, 333, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 4080, 0, 0, 0, 0, 0, 0, 0, 4096, 73, 9, 2, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 333, 0, 0, 0, 0, 3754, 0, 0, 8, 0, 0, 0, 0, 0, 16, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4094, 4082, 4180, 4023, 4421, 3762, 4111] avg=4096.0000 max=4421 max_rank=5 min=3762 min_rank=6 imbalance_ratio=1.079346 max_load_reduction=28347 max_load_reduction_pct=0.865082 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3761, 330, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 6, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 68, 8, 1, 0, 1, 4028, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 335, 0, 0, 0, 0, 3758, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4095, 4101, 4171, 4029, 4429, 3762, 4090] avg=4096.0000 max=4429 max_rank=5 min=3762 min_rank=6 imbalance_ratio=1.081299 max_load_reduction=28339 max_load_reduction_pct=0.864838 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3761, 330, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 6, 4086, 0, 0, 0, 0, 0, 0, 0, 4096, 68, 8, 1, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 335, 0, 0, 0, 0, 3758, 0, 0, 4, 0, 0, 0, 0, 0, 10, 0, 0, 4090, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4095, 4092, 4173, 4028, 4427, 3762, 4100] avg=4096.0000 max=4427 max_rank=5 min=3762 min_rank=6 imbalance_ratio=1.080811 max_load_reduction=28341 max_load_reduction_pct=0.864899 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3761, 330, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 6, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 68, 8, 1, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 335, 0, 0, 0, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4095, 4101, 4173, 4028, 4432, 3758, 4090] avg=4096.0000 max=4432 max_rank=5 min=3758 min_rank=6 imbalance_ratio=1.082031 max_load_reduction=28336 max_load_reduction_pct=0.864746 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 30080, 2688, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 30208, 2688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 32256, 640, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[30080, 128, 128, 2816, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[640, 32256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[9, 0, 0, 9, 10, 0, 66, 11, 1636, 8, 4020, 4084, 0, 0, 0, 11, 4068, 0, 4077, 12, 7, 2461, 0, 20, 4067, 2, 8, 4077, 0, 19, 134, 3952] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4084, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 1, 0, 7, 1, 433, 1, 4077, 4095, 0, 0, 0, 1, 4091, 0, 4094, 1, 1, 3662, 0, 1, 4093, 1, 1, 4094, 0, 5, 20, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 250, 4, 2, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4089, 3846, 4097, 4096, 4352, 4098, 4094] avg=4096.0000 max=4352 max_rank=5 min=3846 min_rank=2 imbalance_ratio=1.062500 max_load_reduction=28416 max_load_reduction_pct=0.867188 moved_to_echo=16640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 12, 0, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 250, 4, 2, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4086, 3858, 4096, 4100, 4340, 4099, 4094] avg=4096.0000 max=4340 max_rank=5 min=3858 min_rank=2 imbalance_ratio=1.059570 max_load_reduction=28428 max_load_reduction_pct=0.867554 moved_to_echo=16633 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 9, 4074, 2, 0, 0, 10, 0, 66, 11, 4067, 0, 0, 0, 1636, 8, 3775, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4084, 0, 0, 0, 4068, 0, 4, 12, 0, 0, 0, 0, 7, 2461, 0, 20, 245, 4, 2, 0, 4067, 2, 8, 3, 0, 0, 0, 0, 0, 19, 134, 3950, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4154, 5419, 4095, 4084, 2739, 4080, 4103] avg=4096.0000 max=5419 max_rank=2 min=2739 min_rank=5 imbalance_ratio=1.322998 max_load_reduction=27349 max_load_reduction_pct=0.834625 moved_to_echo=15006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 4091, 2, 0, 0, 1, 0, 7, 1, 4084, 0, 0, 0, 433, 1, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 4091, 0, 4, 1, 0, 0, 0, 0, 1, 3662, 0, 1, 248, 4, 2, 0, 4093, 1, 1, 3, 0, 0, 0, 0, 0, 5, 20, 4083, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4093, 4263, 4096, 4096, 3918, 4098, 4108] avg=4096.0000 max=4263 max_rank=2 min=3918 min_rank=5 imbalance_ratio=1.040771 max_load_reduction=28505 max_load_reduction_pct=0.869904 moved_to_echo=16220 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 250, 4, 2, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4086, 3846, 4096, 4100, 4352, 4099, 4094] avg=4096.0000 max=4352 max_rank=5 min=3846 min_rank=2 imbalance_ratio=1.062500 max_load_reduction=28416 max_load_reduction_pct=0.867188 moved_to_echo=16645 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 252, 4, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4086, 3844, 4096, 4096, 4361, 4096, 4087] avg=4096.0000 max=4361 max_rank=5 min=3844 min_rank=2 imbalance_ratio=1.064697 max_load_reduction=28407 max_load_reduction_pct=0.866913 moved_to_echo=16640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 250, 6, 2, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4088, 3846, 4096, 4096, 4354, 4099, 4094] avg=4096.0000 max=4354 max_rank=5 min=3846 min_rank=2 imbalance_ratio=1.062988 max_load_reduction=28414 max_load_reduction_pct=0.867126 moved_to_echo=16643 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 250, 4, 2, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4086, 3846, 4096, 4100, 4352, 4099, 4094] avg=4096.0000 max=4352 max_rank=5 min=3846 min_rank=2 imbalance_ratio=1.062500 max_load_reduction=28416 max_load_reduction_pct=0.867188 moved_to_echo=16645 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2176, 128, 30720, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 256, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 30720, 0, 128, 2048, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 52, 0, 4087, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4052, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 362, 0, 4081, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3749, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[17, 99, 11, 4086, 15, 10, 0, 4071, 9, 13, 11, 11, 4085, 25, 4, 6, 1443, 0, 4087, 0, 11, 1, 0, 4083, 3832, 4047, 0, 2785, 4, 0, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 69, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4027, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 84, 0, 4090, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4014, 0, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 353, 0, 4078, 0, 0, 0, 0, 4096, 4095, 4095, 0, 3762, 0, 6, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 179, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3917, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[2, 22, 1, 4095, 0, 1, 0, 4093, 1, 2, 1, 1, 4095, 3, 1, 1, 391, 0, 4095, 1, 1, 4, 1, 4094, 4064, 4089, 0, 3709, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3743, 359, 17, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 52, 0, 3728, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 349, 0, 0, 4052, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4096, 4121, 4094, 3780, 4096, 4401, 4101] avg=4096.0000 max=4401 max_rank=6 min=3780 min_rank=4 imbalance_ratio=1.074463 max_load_reduction=28367 max_load_reduction_pct=0.865692 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3743, 358, 17, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 362, 0, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 349, 0, 0, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4096, 4120, 4094, 4085, 4096, 4098, 4100] avg=4096.0000 max=4120 max_rank=2 min=4079 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[17, 99, 11, 4069, 0, 0, 0, 0, 15, 10, 0, 4071, 0, 0, 0, 0, 9, 13, 11, 11, 3502, 359, 17, 2, 4083, 25, 4, 6, 0, 0, 0, 0, 1443, 0, 3728, 0, 0, 0, 0, 0, 11, 1, 0, 4083, 0, 0, 0, 0, 326, 0, 0, 2785, 0, 0, 0, 0, 4, 0, 1, 1, 4047, 4, 0, 0] rerouted_tokens_per_ep_rank=[4196, 4096, 3924, 4118, 5171, 4095, 3111, 4057] avg=4096.0000 max=5171 max_rank=4 min=3111 min_rank=6 imbalance_ratio=1.262451 max_load_reduction=27597 max_load_reduction_pct=0.842194 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 22, 1, 4078, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 1, 2, 1, 1, 3714, 360, 17, 2, 4093, 3, 1, 1, 0, 0, 0, 0, 391, 0, 3735, 1, 0, 0, 0, 0, 1, 4, 1, 4094, 0, 0, 0, 0, 346, 0, 0, 3709, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4094, 4098, 4098, 4127, 4100, 4055, 4093] avg=4096.0000 max=4127 max_rank=4 min=4055 min_rank=6 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3743, 359, 17, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 84, 0, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 349, 0, 0, 4014, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4096, 4121, 4093, 3815, 4096, 4363, 4105] avg=4096.0000 max=4363 max_rank=6 min=3815 min_rank=4 imbalance_ratio=1.065186 max_load_reduction=28405 max_load_reduction_pct=0.866852 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 3748, 361, 22, 8, 4083, 0, 0, 0, 0, 0, 0, 0, 353, 0, 3717, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 336, 0, 0, 3762, 0, 0, 0, 0, 0, 6, 0, 0, 4095, 11, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4098, 4139, 4083, 4070, 4094, 4098, 4112] avg=4096.0000 max=4139 max_rank=2 min=4070 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3743, 360, 17, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 179, 0, 3736, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 349, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4096, 4122, 4094, 3915, 4096, 4266, 4100] avg=4096.0000 max=4266 max_rank=6 min=3915 min_rank=4 imbalance_ratio=1.041504 max_load_reduction=28502 max_load_reduction_pct=0.869812 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3743, 360, 17, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 69, 0, 3736, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 349, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4096, 4122, 4094, 3805, 4096, 4376, 4100] avg=4096.0000 max=4376 max_rank=6 min=3805 min_rank=4 imbalance_ratio=1.068359 max_load_reduction=28392 max_load_reduction_pct=0.866455 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32767, 32768, 32768, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[2816, 0, 0, 30080, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2944, 0, 29952, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32767 actual_valid_recv=33536 observed_loss=-769 tokens_per_expert=[128, 128, 128, 128, 29696, 2944, 256, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32769 actual_valid_recv=33152 observed_loss=-383 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4059, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 37, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4066, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4090, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4036, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 60, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 4088, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[47, 4086, 4083, 4035, 16, 17, 506, 116, 7, 0, 0, 0, 11, 3574, 0, 7, 0, 8, 13, 0, 0, 15, 0, 0, 0, 0, 4085, 10, 0, 4081, 4079, 3972] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[10, 4095, 4095, 4085, 2, 2, 117, 19, 1, 0, 0, 0, 1, 3977, 0, 1, 0, 1, 1, 0, 0, 2, 0, 0, 0, 0, 4095, 1, 0, 4094, 4094, 4075] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4082, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4095, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4088, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4066, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 30, 4082, 3, 0, 0, 0, 4096, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 6, 0, 4093, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 3994, 4115, 4173, 4095, 4100, 4096, 4115] avg=4096.0000 max=4173 max_rank=3 min=3994 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16406 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4059, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 37, 4082, 3, 0, 0, 0, 4096, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 3999, 4122, 4173, 4095, 4094, 4096, 4116] avg=4096.0000 max=4173 max_rank=3 min=3999 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16401 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[47, 4086, 0, 14, 0, 0, 0, 0, 16, 17, 506, 116, 3985, 0, 0, 0, 7, 0, 0, 0, 4021, 3, 0, 0, 11, 3574, 0, 7, 77, 0, 0, 0, 0, 8, 13, 0, 4082, 0, 0, 0, 0, 15, 0, 0, 4076, 1, 0, 0, 0, 0, 4085, 10, 0, 0, 0, 0, 0, 19, 0, 3972, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4640, 4031, 3669, 4103, 4092, 4095, 3991] avg=4096.0000 max=4640 max_rank=1 min=3669 min_rank=3 imbalance_ratio=1.132812 max_load_reduction=28128 max_load_reduction_pct=0.858398 moved_to_echo=16281 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[10, 4095, 0, 14, 0, 0, 0, 0, 2, 2, 117, 19, 3997, 0, 0, 0, 1, 0, 0, 0, 4071, 3, 0, 0, 1, 3977, 0, 1, 77, 0, 0, 0, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 4091, 1, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 20, 0, 4075, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4137, 4075, 4056, 4096, 4094, 4096, 4095] avg=4096.0000 max=4137 max_rank=1 min=4056 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16381 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4036, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 4, 0, 0, 0, 0, 0, 60, 4087, 3, 0, 0, 0, 4096, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 8, 0, 4093, 1, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4001, 4150, 4179, 4095, 4102, 4097, 4103] avg=4096.0000 max=4179 max_rank=3 min=4001 min_rank=1 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16397 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 14, 4082, 2, 0, 0, 0, 4096, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 4094, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3998, 4098, 4173, 4095, 4096, 4096, 4116] avg=4096.0000 max=4173 max_rank=3 min=3998 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16403 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 0, 1, 3999, 0, 0, 0, 0, 0, 0, 0, 4082, 3, 0, 0, 0, 4095, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4000, 4085, 4172, 4095, 4094, 4096, 4116] avg=4096.0000 max=4172 max_rank=3 min=4000 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16401 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4088, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 8, 4082, 2, 0, 0, 0, 4096, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 3999, 4092, 4173, 4094, 4096, 4096, 4116] avg=4096.0000 max=4173 max_rank=3 min=3999 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16402 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 32640, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 640, 256, 32000, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32128, 0, 128, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 256, 0, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True + [2026-08-30 04:56:48] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 666.5 | throughput per GPU (TFLOP/s/GPU): 500.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.538370E+00 | loss scale: 1.0 | grad norm: 5.810 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[229, 0, 1389, 20, 1482, 1, 2641, 4, 1274, 1433, 0, 25, 1135, 3459, 5, 800, 2, 1604, 2046, 0, 0, 782, 1287, 31, 0, 3085, 3947, 2223, 2345, 1496, 20, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[301, 0, 1110, 16, 1586, 0, 2899, 18, 1621, 1793, 2, 61, 2050, 3298, 12, 551, 21, 967, 2064, 0, 0, 482, 897, 25, 2, 3228, 3447, 1558, 3092, 1517, 149, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[229, 0, 1986, 5, 735, 0, 2601, 8, 1212, 1626, 0, 20, 1864, 3688, 2, 558, 4, 1282, 2026, 0, 0, 630, 1233, 16, 2, 2647, 3549, 1740, 3219, 1819, 67, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[178, 0, 1478, 8, 1404, 0, 2733, 5, 1486, 903, 0, 35, 1798, 3199, 8, 588, 10, 1143, 2582, 0, 0, 908, 1085, 42, 0, 3041, 3764, 1785, 2811, 1715, 59, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[220, 1, 1759, 12, 1345, 1, 2509, 11, 1375, 1320, 0, 19, 1210, 3630, 5, 622, 22, 1591, 2283, 0, 0, 610, 1039, 14, 0, 2945, 3764, 1897, 3064, 1455, 45, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[334, 0, 1570, 11, 1515, 1, 2093, 48, 1627, 1082, 0, 21, 1352, 3511, 13, 723, 10, 1581, 2552, 0, 0, 586, 1545, 83, 0, 2865, 3824, 1687, 2648, 1374, 112, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[190, 0, 1740, 15, 1294, 0, 2332, 31, 1056, 1065, 0, 24, 1848, 3146, 8, 765, 29, 1201, 2400, 0, 1, 540, 1786, 70, 8, 2970, 3793, 1886, 2865, 1649, 55, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[371, 11, 1998, 26, 1072, 0, 2461, 51, 1667, 1471, 0, 20, 2013, 3131, 14, 384, 6, 1354, 1841, 0, 4, 720, 786, 121, 3, 2779, 3673, 1743, 3107, 1785, 154, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[229, 0, 1389, 20, 2329, 0, 0, 0, 1482, 1, 2641, 4, 191, 0, 0, 0, 1274, 1433, 0, 25, 942, 412, 0, 0, 1135, 1847, 5, 800, 0, 0, 0, 0, 2, 1604, 2046, 0, 284, 199, 0, 0, 0, 782, 1287, 31, 1618, 670, 0, 0, 0, 2389, 0, 2223, 0, 0, 0, 0, 1955, 1496, 20, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3967, 4319, 4086, 3787, 4135, 4388, 4612, 3474] avg=4096.0000 max=4612 max_rank=6 min=3474 min_rank=7 imbalance_ratio=1.125977 max_load_reduction=28156 max_load_reduction_pct=0.859253 moved_to_echo=16609 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[301, 0, 1110, 16, 2033, 0, 0, 0, 1586, 0, 2899, 18, 252, 0, 0, 0, 1621, 1793, 2, 61, 899, 431, 0, 0, 2050, 1760, 12, 551, 0, 0, 0, 0, 21, 967, 2064, 0, 297, 263, 0, 0, 0, 482, 897, 25, 1414, 639, 0, 0, 2, 2500, 0, 1558, 0, 0, 0, 0, 2577, 1517, 149, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3460, 4755, 4807, 4373, 3612, 3457, 4060, 4244] avg=4096.0000 max=4807 max_rank=2 min=3457 min_rank=5 imbalance_ratio=1.173584 max_load_reduction=27961 max_load_reduction_pct=0.853302 moved_to_echo=15373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[178, 0, 1478, 8, 2221, 0, 0, 0, 1404, 0, 2733, 5, 229, 0, 0, 0, 1486, 903, 0, 35, 872, 406, 0, 0, 1798, 1707, 8, 588, 0, 0, 0, 0, 10, 1143, 2582, 0, 280, 239, 0, 0, 0, 908, 1085, 42, 1543, 620, 0, 0, 0, 2355, 0, 1785, 0, 0, 0, 0, 2343, 1715, 59, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3885, 4371, 3702, 4101, 4254, 4198, 4140, 4117] avg=4096.0000 max=4371 max_rank=1 min=3702 min_rank=2 imbalance_ratio=1.067139 max_load_reduction=28397 max_load_reduction_pct=0.866608 moved_to_echo=16709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[220, 1, 1759, 12, 2221, 0, 0, 0, 1345, 1, 2509, 11, 249, 0, 0, 0, 1375, 1320, 0, 19, 989, 394, 0, 0, 1210, 1938, 5, 622, 0, 0, 0, 0, 22, 1591, 2283, 0, 271, 261, 0, 0, 0, 610, 1039, 14, 1543, 703, 0, 0, 0, 2280, 0, 1897, 0, 0, 0, 0, 2554, 1455, 45, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4213, 4115, 4097, 3775, 4428, 3909, 4177, 4054] avg=4096.0000 max=4428 max_rank=4 min=3775 min_rank=3 imbalance_ratio=1.081055 max_load_reduction=28340 max_load_reduction_pct=0.864868 moved_to_echo=16568 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[229, 0, 1986, 5, 2094, 0, 0, 0, 735, 0, 2601, 8, 262, 0, 0, 0, 1212, 1626, 0, 20, 1005, 354, 0, 0, 1864, 1969, 2, 558, 0, 0, 0, 0, 4, 1282, 2026, 0, 244, 274, 0, 0, 0, 630, 1233, 16, 1455, 714, 0, 0, 2, 2049, 0, 1740, 0, 0, 0, 0, 2683, 1819, 67, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4314, 3606, 4217, 4393, 3830, 4048, 3791, 4569] avg=4096.0000 max=4569 max_rank=7 min=3606 min_rank=1 imbalance_ratio=1.115479 max_load_reduction=28199 max_load_reduction_pct=0.860565 moved_to_echo=16238 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[371, 11, 1998, 26, 2167, 0, 0, 0, 1072, 0, 2461, 51, 253, 0, 0, 0, 1667, 1471, 0, 20, 853, 371, 0, 0, 2013, 1672, 14, 384, 0, 0, 0, 0, 6, 1354, 1841, 0, 256, 264, 0, 0, 4, 720, 786, 121, 1506, 606, 0, 0, 3, 2152, 0, 1743, 0, 0, 0, 0, 2590, 1785, 154, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4573, 3837, 4382, 4083, 3721, 3743, 3898, 4531] avg=4096.0000 max=4573 max_rank=0 min=3721 min_rank=4 imbalance_ratio=1.116455 max_load_reduction=28195 max_load_reduction_pct=0.860443 moved_to_echo=15893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[334, 0, 1570, 11, 2257, 0, 0, 0, 1515, 1, 2093, 48, 215, 0, 0, 0, 1627, 1082, 0, 21, 957, 383, 0, 0, 1352, 1874, 13, 723, 0, 0, 0, 0, 10, 1581, 2552, 0, 264, 225, 0, 0, 0, 586, 1545, 83, 1567, 680, 0, 0, 0, 2218, 0, 1687, 0, 0, 0, 0, 2208, 1374, 112, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 3872, 4070, 3962, 4632, 4461, 3905, 3694] avg=4096.0000 max=4632 max_rank=4 min=3694 min_rank=7 imbalance_ratio=1.130859 max_load_reduction=28136 max_load_reduction_pct=0.858643 moved_to_echo=16692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[190, 0, 1740, 15, 2239, 0, 0, 0, 1294, 0, 2332, 31, 236, 0, 0, 0, 1056, 1065, 0, 24, 860, 402, 0, 0, 1848, 1673, 8, 765, 0, 0, 0, 0, 29, 1201, 2400, 0, 277, 249, 0, 0, 1, 540, 1786, 70, 1554, 613, 0, 0, 8, 2291, 0, 1886, 0, 0, 0, 0, 2380, 1649, 55, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4184, 3893, 3407, 4294, 4156, 4564, 4185, 4085] avg=4096.0000 max=4564 max_rank=5 min=3407 min_rank=2 imbalance_ratio=1.114258 max_load_reduction=28204 max_load_reduction_pct=0.860718 moved_to_echo=16990 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11392, 10752, 128, 256, 7424, 3200, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2176, 128, 13056, 128, 17664, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 5376, 9728, 512, 12288, 5248, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 10752, 17920, 0, 2176, 2048, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10496, 128, 20352, 256, 1920, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 18304, 0, 14592, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[13312, 14464, 128, 4992, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19328, 12928, 768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[581, 0, 0, 0, 2431, 0, 0, 0, 0, 799, 1002, 0, 3, 0, 4096, 4096, 4096, 0, 0, 0, 3488, 0, 0, 0, 4096, 3984, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[15, 1, 0, 0, 473, 0, 0, 1, 0, 1866, 1947, 0, 2, 0, 4093, 4096, 4095, 0, 1, 0, 3936, 0, 1, 0, 4089, 4056, 0, 0, 0, 4095, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[263, 0, 0, 2, 920, 0, 0, 3, 0, 663, 2381, 4, 10, 0, 4094, 4096, 4092, 0, 0, 0, 3993, 0, 0, 0, 4089, 4061, 0, 0, 1, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[78, 0, 0, 0, 1876, 0, 0, 0, 0, 921, 1449, 0, 46, 0, 4096, 4096, 4096, 0, 0, 0, 3966, 0, 0, 0, 4094, 3954, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[213, 0, 0, 0, 977, 0, 0, 0, 0, 645, 2622, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 3792, 0, 0, 0, 4093, 4046, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[305, 0, 0, 0, 1435, 0, 0, 0, 0, 1600, 1724, 4, 0, 1, 4094, 4096, 4093, 0, 0, 0, 3228, 0, 0, 0, 4095, 4012, 0, 0, 0, 4081, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[974, 0, 0, 0, 2265, 0, 0, 0, 0, 961, 2917, 0, 0, 2, 4092, 4096, 4094, 0, 0, 0, 2480, 0, 0, 0, 4096, 2693, 0, 0, 0, 4091, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[498, 0, 0, 1, 1758, 0, 0, 0, 2, 1370, 1227, 0, 14, 4, 4096, 4087, 4096, 1, 0, 0, 3571, 0, 0, 0, 4095, 3839, 0, 0, 0, 4096, 1, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[581, 0, 0, 0, 3731, 0, 0, 0, 2431, 0, 0, 0, 2214, 365, 0, 0, 0, 799, 1002, 0, 1083, 0, 0, 0, 3, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3488, 0, 0, 0, 531, 7, 0, 0, 268, 3984, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4312, 5010, 2884, 4092, 4096, 4026, 4252, 4096] avg=4096.0000 max=5010 max_rank=1 min=2884 min_rank=2 imbalance_ratio=1.223145 max_load_reduction=27758 max_load_reduction_pct=0.847107 moved_to_echo=16470 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[263, 0, 0, 2, 3731, 0, 0, 0, 920, 0, 0, 3, 2211, 365, 0, 0, 0, 663, 2381, 4, 1081, 0, 0, 0, 10, 0, 4087, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 531, 7, 0, 0, 266, 4061, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3996, 3499, 4129, 4097, 4092, 4531, 4327, 4097] avg=4096.0000 max=4531 max_rank=5 min=3499 min_rank=1 imbalance_ratio=1.106201 max_load_reduction=28237 max_load_reduction_pct=0.861725 moved_to_echo=17047 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[213, 0, 0, 0, 3730, 0, 0, 0, 977, 0, 0, 0, 2213, 366, 0, 0, 0, 645, 2622, 0, 1082, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3792, 0, 0, 0, 531, 7, 0, 0, 267, 4046, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3943, 3556, 4349, 4085, 4096, 4330, 4313, 4096] avg=4096.0000 max=4349 max_rank=2 min=3556 min_rank=1 imbalance_ratio=1.061768 max_load_reduction=28419 max_load_reduction_pct=0.867279 moved_to_echo=16835 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[78, 0, 0, 0, 3730, 0, 0, 0, 1876, 0, 0, 0, 2213, 366, 0, 0, 0, 921, 1449, 0, 1083, 0, 0, 0, 46, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 0, 531, 7, 0, 0, 267, 3954, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3808, 4455, 3453, 4135, 4096, 4504, 4221, 4096] avg=4096.0000 max=4504 max_rank=5 min=3453 min_rank=2 imbalance_ratio=1.099609 max_load_reduction=28264 max_load_reduction_pct=0.862549 moved_to_echo=16917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[974, 0, 0, 0, 3730, 0, 0, 0, 2265, 0, 0, 0, 2214, 366, 0, 0, 0, 961, 2917, 0, 1083, 0, 0, 0, 0, 2, 4085, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2480, 0, 0, 0, 531, 7, 0, 0, 268, 2693, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4704, 4845, 4961, 4087, 4094, 3018, 2961, 4098] avg=4096.0000 max=4961 max_rank=2 min=2961 min_rank=6 imbalance_ratio=1.211182 max_load_reduction=27807 max_load_reduction_pct=0.848602 moved_to_echo=14171 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[305, 0, 0, 0, 3731, 0, 0, 0, 1435, 0, 0, 0, 2214, 365, 0, 0, 0, 1600, 1724, 4, 1083, 0, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3228, 0, 0, 0, 531, 7, 0, 0, 267, 4012, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4014, 4411, 4088, 4093, 3766, 4279, 4081] avg=4096.0000 max=4411 max_rank=2 min=3766 min_rank=5 imbalance_ratio=1.076904 max_load_reduction=28357 max_load_reduction_pct=0.865387 moved_to_echo=16219 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[15, 1, 0, 0, 3731, 0, 0, 0, 473, 0, 0, 1, 2211, 365, 0, 0, 0, 1866, 1947, 0, 1081, 0, 0, 0, 2, 0, 4086, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 3936, 0, 1, 0, 531, 7, 0, 0, 266, 4056, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3747, 3050, 4894, 4088, 4096, 4475, 4322, 4096] avg=4096.0000 max=4894 max_rank=2 min=3050 min_rank=1 imbalance_ratio=1.194824 max_load_reduction=27874 max_load_reduction_pct=0.850647 moved_to_echo=16989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[498, 0, 0, 1, 3723, 0, 0, 0, 1758, 0, 0, 0, 2217, 364, 0, 0, 2, 1370, 1227, 0, 1088, 0, 0, 0, 14, 4, 4078, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 7, 1, 0, 0, 3571, 0, 0, 0, 536, 11, 0, 0, 254, 3839, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 12, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4222, 4339, 3687, 4096, 4105, 4118, 4093, 4108] avg=4096.0000 max=4339 max_rank=1 min=3687 min_rank=2 imbalance_ratio=1.059326 max_load_reduction=28429 max_load_reduction_pct=0.867584 moved_to_echo=16424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12160, 0, 0, 128, 17792, 2944, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 8832, 15360, 128, 8704, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2944, 128, 0, 128, 29952, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[28544, 0, 128, 0, 4352, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[2176, 30720, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 3341, 0, 0, 0, 4096, 1, 4006, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 835, 4, 0, 0, 0, 351, 0, 3745] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3214, 0, 0, 0, 4096, 6, 4096, 4095, 3, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 2, 4096, 4096, 879, 4, 0, 0, 0, 180, 0, 3903] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[2, 5, 3549, 0, 1, 0, 4096, 0, 3953, 4096, 0, 0, 4092, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4096, 760, 0, 2, 0, 0, 34, 0, 3977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3029, 1, 0, 0, 4095, 2, 4079, 4092, 4, 1, 4095, 0, 0, 0, 1, 1, 0, 1, 0, 1, 4095, 4095, 1068, 12, 0, 1, 0, 43, 0, 4052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 10, 3776, 0, 0, 0, 4096, 0, 4057, 4096, 0, 0, 4092, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4095, 295, 3, 0, 0, 0, 180, 0, 3963] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 3380, 0, 0, 0, 4096, 1, 3902, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1068, 8, 0, 0, 0, 97, 0, 3829] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 4, 3632, 0, 0, 0, 4096, 1, 4053, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 690, 1, 2, 0, 0, 145, 0, 3764] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3, 16, 3981, 0, 0, 0, 4096, 7, 4084, 4096, 2, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 918, 30, 0, 0, 0, 515, 0, 2733] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 3341, 0, 601, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 4006, 68, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 835, 4, 0, 0, 3273, 0, 0, 0, 0, 351, 0, 3745, 154, 2, 0, 0] rerouted_tokens_per_ep_rank=[3948, 4095, 4074, 4095, 4096, 4096, 4112, 4252] avg=4096.0000 max=4252 max_rank=7 min=3948 min_rank=0 imbalance_ratio=1.038086 max_load_reduction=28516 max_load_reduction_pct=0.870239 moved_to_echo=16556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 3776, 0, 601, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4057, 68, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 295, 3, 0, 0, 3273, 0, 0, 0, 0, 180, 0, 3963, 154, 2, 0, 0] rerouted_tokens_per_ep_rank=[4387, 4094, 4125, 4101, 4096, 4095, 3571, 4299] avg=4096.0000 max=4387 max_rank=0 min=3571 min_rank=6 imbalance_ratio=1.071045 max_load_reduction=28381 max_load_reduction_pct=0.866119 moved_to_echo=16061 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 5, 3549, 0, 601, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 3953, 68, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 760, 0, 2, 0, 3273, 0, 0, 0, 0, 34, 0, 3977, 154, 2, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4095, 4021, 4101, 4096, 4096, 4035, 4167] avg=4096.0000 max=4167 max_rank=7 min=4021 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3380, 0, 601, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 3902, 68, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1068, 8, 0, 0, 3273, 1, 0, 0, 0, 97, 0, 3829, 154, 2, 0, 0] rerouted_tokens_per_ep_rank=[3984, 4095, 3970, 4096, 4095, 4096, 4350, 4082] avg=4096.0000 max=4350 max_rank=6 min=3970 min_rank=2 imbalance_ratio=1.062012 max_load_reduction=28418 max_load_reduction_pct=0.867249 moved_to_echo=16623 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3214, 0, 601, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 0, 0, 4096, 68, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 879, 4, 0, 0, 3272, 1, 0, 0, 0, 180, 0, 3903, 154, 2, 0, 0] rerouted_tokens_per_ep_rank=[3816, 4100, 4167, 4097, 4095, 4098, 4156, 4239] avg=4096.0000 max=4239 max_rank=7 min=3816 min_rank=0 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 16, 3981, 0, 601, 0, 0, 0, 0, 0, 4094, 7, 0, 0, 0, 0, 4084, 68, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 918, 30, 0, 0, 3273, 0, 0, 0, 0, 515, 0, 2733, 154, 2, 0, 0] rerouted_tokens_per_ep_rank=[4601, 4101, 4154, 4095, 4096, 4096, 4221, 3404] avg=4096.0000 max=4601 max_rank=0 min=3404 min_rank=7 imbalance_ratio=1.123291 max_load_reduction=28167 max_load_reduction_pct=0.859589 moved_to_echo=15817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 3632, 0, 608, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 4053, 53, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 690, 1, 2, 0, 3279, 0, 0, 0, 0, 145, 0, 3764, 156, 4, 4, 1] rerouted_tokens_per_ep_rank=[4244, 4093, 4106, 4088, 4096, 4095, 3972, 4074] avg=4096.0000 max=4244 max_rank=0 min=3972 min_rank=6 imbalance_ratio=1.036133 max_load_reduction=28524 max_load_reduction_pct=0.870483 moved_to_echo=16237 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3029, 1, 601, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 4079, 67, 4, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 1068, 12, 0, 1, 3270, 0, 0, 0, 0, 43, 0, 4052, 154, 2, 0, 0] rerouted_tokens_per_ep_rank=[3631, 4095, 4151, 4095, 4098, 4096, 4351, 4251] avg=4096.0000 max=4351 max_rank=6 min=3631 min_rank=0 imbalance_ratio=1.062256 max_load_reduction=28417 max_load_reduction_pct=0.867218 moved_to_echo=16796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6528, 128, 128, 128, 26240, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 27904, 128, 4864, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 1664, 0, 30080, 1280, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32256, 640, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1276, 0, 4094, 0, 0, 4095, 4096, 2044, 113, 0, 0, 0, 0, 4087, 678] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 5, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1370, 0, 4094, 0, 0, 4089, 4096, 1334, 122, 0, 0, 0, 0, 4085, 1292] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1011, 0, 4092, 0, 0, 4080, 4096, 1473, 61, 0, 0, 0, 0, 4087, 1576] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1302, 0, 4094, 0, 0, 4089, 4096, 1219, 32, 0, 0, 0, 0, 4091, 1563] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 7, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1045, 0, 4096, 0, 0, 4096, 4096, 1073, 43, 0, 0, 0, 0, 4094, 1931] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 75, 0, 4095, 0, 4095, 1, 0, 1, 0, 1, 0, 4095, 1189, 1, 4095, 0, 0, 4092, 4091, 1913, 92, 2, 4, 0, 1, 4070, 853] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1097, 0, 4094, 0, 0, 4081, 4096, 1894, 123, 0, 0, 0, 0, 4096, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 36, 0, 4085, 0, 4080, 0, 0, 0, 0, 0, 0, 4096, 1258, 0, 4096, 0, 0, 4081, 4096, 712, 66, 0, 0, 0, 0, 4078, 2084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 8, 0, 0, 0, 5, 0, 4077, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1276, 0, 2903, 0, 0, 0, 0, 0, 0, 4095, 9, 0, 0, 0, 0, 2044, 113, 0, 0, 1354, 1191, 10, 0, 0, 0, 2725, 678, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4082, 4097, 4096, 4179, 4104, 4712, 3403] avg=4096.0000 max=4712 max_rank=6 min=3403 min_rank=7 imbalance_ratio=1.150391 max_load_reduction=28056 max_load_reduction_pct=0.856201 moved_to_echo=16398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 8, 0, 0, 0, 5, 0, 4078, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1370, 0, 2903, 0, 0, 0, 0, 0, 0, 4089, 9, 0, 0, 0, 0, 1334, 122, 0, 0, 1353, 1191, 10, 0, 0, 0, 2724, 1292, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4083, 4097, 4096, 4273, 4098, 4010, 4016] avg=4096.0000 max=4273 max_rank=4 min=4010 min_rank=6 imbalance_ratio=1.043213 max_load_reduction=28495 max_load_reduction_pct=0.869598 moved_to_echo=16397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4087, 8, 0, 0, 0, 2, 0, 4084, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1011, 0, 2901, 0, 0, 0, 0, 0, 0, 4080, 9, 0, 0, 0, 0, 1473, 61, 0, 0, 1354, 1191, 10, 0, 0, 0, 2725, 1576, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4086, 4098, 4096, 3912, 4089, 4089, 4301] avg=4096.0000 max=4301 max_rank=7 min=3912 min_rank=4 imbalance_ratio=1.050049 max_load_reduction=28467 max_load_reduction_pct=0.868744 moved_to_echo=16391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 8, 0, 0, 0, 7, 0, 4083, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1045, 0, 2904, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 1073, 43, 0, 0, 1356, 1192, 10, 0, 0, 0, 2730, 1931, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4090, 4098, 4096, 3949, 4105, 3674, 4661] avg=4096.0000 max=4661 max_rank=7 min=3674 min_rank=6 imbalance_ratio=1.137939 max_load_reduction=28107 max_load_reduction_pct=0.857758 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 8, 0, 0, 0, 36, 0, 4074, 0, 0, 0, 0, 0, 4080, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1258, 0, 2904, 0, 0, 0, 0, 0, 0, 4081, 9, 0, 0, 0, 0, 712, 66, 0, 0, 1351, 1192, 10, 0, 0, 0, 2719, 2084, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4110, 4081, 4096, 4162, 4090, 3331, 4803] avg=4096.0000 max=4803 max_rank=7 min=3331 min_rank=6 imbalance_ratio=1.172607 max_load_reduction=27965 max_load_reduction_pct=0.853424 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 8, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1302, 0, 2903, 0, 0, 0, 0, 0, 0, 4089, 9, 0, 0, 0, 0, 1219, 32, 0, 0, 1355, 1191, 10, 0, 0, 0, 2728, 1563, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4079, 4097, 4096, 4205, 4098, 3807, 4291] avg=4096.0000 max=4291 max_rank=7 min=3807 min_rank=6 imbalance_ratio=1.047607 max_load_reduction=28477 max_load_reduction_pct=0.869049 moved_to_echo=16401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 10, 1, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1097, 0, 2898, 0, 0, 0, 0, 0, 0, 4081, 2, 0, 0, 0, 0, 1894, 123, 0, 0, 1360, 1196, 11, 0, 0, 0, 2726, 999, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4079, 4102, 4095, 3995, 4083, 4584, 3725] avg=4096.0000 max=4584 max_rank=6 min=3725 min_rank=7 imbalance_ratio=1.119141 max_load_reduction=28184 max_load_reduction_pct=0.860107 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4082, 8, 0, 0, 1, 75, 0, 4083, 0, 0, 0, 0, 0, 4095, 1, 0, 2, 0, 0, 0, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 1189, 1, 2903, 0, 0, 0, 0, 0, 0, 4092, 9, 0, 0, 0, 0, 1913, 92, 2, 4, 1348, 1192, 10, 0, 0, 1, 2714, 853, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4159, 4098, 4097, 4093, 4101, 4561, 3568] avg=4096.0000 max=4561 max_rank=6 min=3568 min_rank=7 imbalance_ratio=1.113525 max_load_reduction=28207 max_load_reduction_pct=0.860809 moved_to_echo=16323 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[11776, 768, 128, 128, 10880, 9600, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 21888, 11008, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 9600, 128, 23296, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 256, 0, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 405, 0, 4092, 4096, 0, 0, 5, 0, 4090, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 128, 0, 10, 4088, 0, 4086, 0, 3573, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 873, 0, 4087, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 162, 0, 13, 4096, 0, 4091, 0, 3069, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 979, 0, 4075, 4096, 0, 0, 0, 0, 4071, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 18, 0, 26, 4092, 0, 4085, 0, 3129, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1108, 1, 4058, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 77, 0, 0, 4086, 0, 4094, 0, 2964, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 1030, 0, 4091, 4095, 1, 1, 0, 0, 4095, 1, 0, 0, 4095, 4075, 0, 0, 0, 1, 0, 0, 44, 0, 0, 4088, 0, 4094, 1, 3053, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1324, 0, 4070, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 21, 0, 9, 4085, 0, 4087, 0, 2792, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1015, 0, 4050, 4096, 0, 0, 2, 0, 4092, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 8, 0, 8, 4092, 0, 4087, 0, 3141, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1499, 0, 4089, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 6, 0, 25, 4095, 0, 4094, 0, 2592, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 873, 3028, 38, 0, 0, 0, 4087, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 162, 4037, 0, 0, 0, 0, 13, 4091, 0, 0, 0, 0, 0, 1055, 0, 3069, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3939, 4108, 4093, 4096, 4102, 4199, 4104, 4127] avg=4096.0000 max=4199 max_rank=5 min=3939 min_rank=0 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16532 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 405, 3024, 38, 0, 0, 0, 4092, 21, 0, 0, 0, 0, 0, 0, 5, 0, 4090, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 128, 4037, 0, 0, 0, 0, 10, 4083, 0, 0, 0, 0, 0, 1054, 0, 3573, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3467, 4113, 4102, 4096, 4097, 4165, 4093, 4635] avg=4096.0000 max=4635 max_rank=7 min=3467 min_rank=0 imbalance_ratio=1.131592 max_load_reduction=28133 max_load_reduction_pct=0.858551 moved_to_echo=16990 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1015, 3025, 38, 0, 0, 0, 4050, 21, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4075, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 8, 4037, 0, 0, 0, 0, 8, 4087, 0, 0, 0, 0, 0, 1054, 0, 3141, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4071, 4101, 4096, 4081, 4045, 4095, 4201] avg=4096.0000 max=4201 max_rank=7 min=4045 min_rank=5 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16422 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 979, 3030, 44, 0, 0, 0, 4075, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 10, 6, 0, 0, 0, 0, 0, 18, 4044, 0, 0, 0, 0, 26, 4082, 0, 0, 0, 0, 0, 1039, 0, 3129, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4083, 4081, 4096, 4108, 4062, 4108, 4177] avg=4096.0000 max=4177 max_rank=7 min=4053 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16455 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1324, 3025, 38, 0, 0, 0, 4070, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 21, 4037, 0, 0, 0, 0, 9, 4080, 0, 0, 0, 0, 0, 1054, 0, 2792, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4387, 4091, 4095, 4096, 4102, 4058, 4089, 3850] avg=4096.0000 max=4387 max_rank=0 min=3850 min_rank=7 imbalance_ratio=1.071045 max_load_reduction=28381 max_load_reduction_pct=0.866119 moved_to_echo=16099 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1499, 3030, 38, 0, 0, 0, 4089, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 6, 4037, 0, 0, 0, 0, 25, 4090, 0, 0, 0, 0, 0, 1056, 0, 2592, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4567, 4110, 4090, 4096, 4099, 4043, 4115, 3648] avg=4096.0000 max=4567 max_rank=0 min=3648 min_rank=7 imbalance_ratio=1.114990 max_load_reduction=28201 max_load_reduction_pct=0.860626 moved_to_echo=15905 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1108, 3030, 38, 0, 0, 1, 4058, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 77, 4037, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 1056, 0, 2964, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4080, 4103, 4096, 4098, 4114, 4081, 4020] avg=4096.0000 max=4176 max_rank=0 min=4020 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16313 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 1030, 3030, 38, 0, 0, 0, 4091, 20, 1, 0, 0, 0, 0, 1, 0, 0, 4095, 7, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 4075, 0, 0, 0, 5, 1, 0, 0, 1, 0, 0, 44, 4037, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 1056, 1, 3053, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4112, 4103, 4096, 4081, 4082, 4083, 4110] avg=4096.0000 max=4112 max_rank=1 min=4081 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16356 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 8320, 24320, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 128, 24320, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 512, 32384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3077, 5, 4096, 0, 0, 4083, 0, 0, 0, 13, 2, 4096, 0, 4090, 0, 0, 4094, 6, 0, 4096, 0, 0, 0, 1014, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3359, 13, 4096, 0, 0, 4082, 0, 0, 0, 32, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 706, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2983, 14, 4096, 0, 0, 4080, 0, 0, 0, 38, 0, 4096, 0, 4096, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 1075, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3902, 0, 4096, 0, 0, 4087, 0, 0, 0, 11, 0, 4096, 0, 4096, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 189, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3621, 2, 4095, 0, 0, 4088, 0, 0, 0, 8, 1, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 473, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 2, 0, 818, 19, 4093, 1, 1, 4095, 0, 0, 0, 0, 0, 4093, 1, 4092, 1, 1, 4095, 1, 0, 4095, 1, 1, 2, 3260, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3346, 13, 4096, 0, 0, 4096, 0, 0, 0, 87, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 650, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2713, 7, 4093, 0, 0, 4096, 0, 0, 0, 14, 0, 4096, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 1365, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3359, 13, 1110, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 7, 0, 0, 0, 0, 32, 0, 4071, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 706, 2986, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4482, 4089, 4103, 4096, 4096, 3710, 4096] avg=4096.0000 max=4482 max_rank=1 min=3710 min_rank=6 imbalance_ratio=1.094238 max_load_reduction=28286 max_load_reduction_pct=0.863220 moved_to_echo=15998 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3077, 5, 1110, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 7, 0, 0, 0, 0, 13, 2, 4071, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 2986, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4192, 4090, 4086, 4090, 4100, 4018, 4096] avg=4096.0000 max=4192 max_rank=1 min=4018 min_rank=6 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3621, 2, 1110, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 7, 0, 0, 0, 0, 8, 1, 4071, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 2985, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4733, 4095, 4080, 4096, 4096, 3476, 4096] avg=4096.0000 max=4733 max_rank=1 min=3476 min_rank=6 imbalance_ratio=1.155518 max_load_reduction=28035 max_load_reduction_pct=0.855560 moved_to_echo=15764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3346, 13, 1110, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 87, 0, 4071, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 2986, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4469, 4103, 4158, 4096, 4096, 3654, 4096] avg=4096.0000 max=4469 max_rank=1 min=3654 min_rank=6 imbalance_ratio=1.091064 max_load_reduction=28299 max_load_reduction_pct=0.863617 moved_to_echo=15942 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3902, 0, 1110, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 7, 0, 0, 0, 0, 11, 0, 4071, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 2986, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 5012, 4094, 4082, 4096, 4099, 3193, 4096] avg=4096.0000 max=5012 max_rank=1 min=3193 min_rank=6 imbalance_ratio=1.223633 max_load_reduction=27756 max_load_reduction_pct=0.847046 moved_to_echo=15484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 2983, 14, 1107, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 9, 2, 0, 0, 0, 38, 0, 4068, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 1, 0, 0, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 2989, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4104, 4091, 4106, 4104, 4089, 4083, 4096] avg=4096.0000 max=4106 max_rank=3 min=4083 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 2, 0, 0, 0, 0, 0, 818, 19, 1109, 1, 0, 0, 0, 0, 1, 4095, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 1, 4092, 1, 1, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3260, 2984, 18, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 1947, 4103, 4068, 4095, 4096, 6266, 4096] avg=4096.0000 max=6266 max_rank=6 min=1947 min_rank=1 imbalance_ratio=1.529785 max_load_reduction=26502 max_load_reduction_pct=0.808777 moved_to_echo=18553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2713, 7, 1109, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 14, 0, 4071, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1365, 2984, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3829, 4103, 4085, 4095, 4096, 4368, 4096] avg=4096.0000 max=4368 max_rank=6 min=3829 min_rank=1 imbalance_ratio=1.066406 max_load_reduction=28400 max_load_reduction_pct=0.866699 moved_to_echo=16655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[23936, 128, 8960, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 8832, 23936, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 256, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 4085, 0, 4096, 3919, 0, 0, 0, 0, 176, 5, 0, 6, 0, 0, 4096, 4096, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 4083, 0, 4096, 3999, 0, 0, 0, 0, 97, 0, 0, 13, 1, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 4074, 0, 4096, 3996, 0, 0, 0, 0, 107, 0, 0, 15, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 4085, 0, 4096, 4069, 0, 0, 0, 0, 27, 6, 0, 10, 5, 0, 4096, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4096, 4029, 0, 0, 0, 1, 67, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 4091, 0, 4095, 4045, 1, 0, 0, 1, 37, 0, 1, 0, 0, 1, 4095, 4095, 16, 4094, 0, 1, 1, 4095, 0, 2, 0, 0, 1, 0, 4094, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 4086, 0, 4096, 3739, 0, 0, 0, 0, 357, 9, 0, 5, 1, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4092, 0, 4096, 4086, 0, 0, 0, 0, 10, 4, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4085, 0, 10, 0, 0, 0, 0, 3919, 0, 0, 0, 103, 7, 0, 0, 0, 176, 5, 0, 3983, 0, 0, 0, 6, 0, 0, 4089, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4029, 4164, 4095, 4096, 4096, 4096, 4097] avg=4096.0000 max=4164 max_rank=2 min=4029 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4085, 0, 11, 0, 0, 0, 0, 4069, 0, 0, 0, 103, 7, 0, 0, 0, 27, 6, 0, 3982, 0, 0, 0, 10, 5, 0, 4089, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4179, 4015, 4104, 4090, 4096, 4096, 4092] avg=4096.0000 max=4179 max_rank=1 min=4015 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 0, 11, 0, 0, 0, 0, 3739, 0, 0, 0, 103, 7, 0, 0, 0, 357, 9, 0, 3982, 0, 0, 0, 5, 1, 0, 4089, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3849, 4348, 4095, 4094, 4096, 4096, 4093] avg=4096.0000 max=4348 max_rank=2 min=3849 min_rank=1 imbalance_ratio=1.061523 max_load_reduction=28420 max_load_reduction_pct=0.867310 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4074, 0, 11, 0, 0, 0, 0, 3996, 0, 0, 0, 103, 7, 0, 0, 0, 107, 0, 0, 3982, 0, 0, 0, 15, 0, 0, 4089, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4106, 4089, 4104, 4095, 4096, 4096, 4097] avg=4096.0000 max=4106 max_rank=1 min=4085 min_rank=0 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4083, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 108, 8, 0, 0, 0, 97, 0, 0, 3988, 1, 0, 0, 13, 1, 0, 4088, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 2, 1, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4115, 4086, 4102, 4093, 4095, 4095, 4099] avg=4096.0000 max=4115 max_rank=1 min=4083 min_rank=0 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 11, 0, 0, 0, 0, 4029, 0, 0, 0, 103, 7, 0, 0, 1, 67, 0, 0, 3982, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4139, 4050, 4089, 4095, 4096, 4096, 4097] avg=4096.0000 max=4139 max_rank=1 min=4050 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 11, 0, 0, 0, 0, 4086, 0, 0, 0, 103, 7, 0, 0, 0, 10, 4, 0, 3982, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4196, 3996, 4089, 4095, 4096, 4096, 4097] avg=4096.0000 max=4196 max_rank=1 min=3996 min_rank=2 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 4091, 0, 11, 0, 0, 0, 0, 4045, 1, 0, 0, 103, 6, 0, 0, 1, 37, 0, 1, 3981, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 4094, 16, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 2, 0, 0, 1, 4094, 0, 0, 0, 0, 4094, 1, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4155, 4020, 4090, 4110, 4097, 4097, 4096] avg=4096.0000 max=4155 max_rank=1 min=4020 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 896, 128, 128, 31872, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32000, 128, 0, 0, 896, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 12, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3381, 703, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 4096, 0, 0, 4096, 4096, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3805, 282, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 4096, 0, 0, 4096, 4096, 0, 15, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3571, 514, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 27, 4095, 1, 4068, 0, 1, 4092, 4095, 1, 2, 0, 0, 3, 0, 4095, 0, 1, 4095, 4094, 0, 0, 1, 3763, 330, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 4096, 0, 0, 4096, 4096, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3821, 270, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3634, 461, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0, 4095, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3731, 365, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4073, 23, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 1, 0, 0, 4091, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3381, 703, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4099, 4103, 4096, 4096, 4096, 4089, 4095] avg=4096.0000 max=4103 max_rank=2 min=4089 min_rank=6 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 1, 0, 0, 4091, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3805, 282, 0, 0, 4, 1, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4099, 4099, 4096, 4096, 4096, 4092, 4099] avg=4096.0000 max=4099 max_rank=1 min=4091 min_rank=0 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 1, 0, 0, 4091, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3821, 270, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4099, 4098, 4096, 4096, 4096, 4096, 4095] avg=4096.0000 max=4099 max_rank=1 min=4092 min_rank=0 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 1, 0, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3634, 461, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4099, 4092, 4095, 4096, 4096, 4100, 4096] avg=4096.0000 max=4100 max_rank=6 min=4092 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 3, 0, 0, 4088, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 3571, 514, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4103, 4103, 4096, 4096, 4097, 4091, 4095] avg=4096.0000 max=4103 max_rank=1 min=4087 min_rank=0 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3731, 365, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4098, 4091, 4096, 4096, 4096, 4101, 4096] avg=4096.0000 max=4101 max_rank=6 min=4091 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4073, 23, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4099, 4091, 4095, 4096, 4096, 4100, 4097] avg=4096.0000 max=4100 max_rank=6 min=4091 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 27, 4093, 0, 0, 0, 0, 1, 4068, 0, 1, 2, 0, 0, 0, 4088, 0, 1, 2, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 3763, 330, 0, 1, 4, 1, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4072, 4091, 4098, 4096, 4095, 4099, 4095] avg=4096.0000 max=4122 max_rank=0 min=4072 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[29824, 3072, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 27, 0, 4096, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 4096, 0, 0, 0, 28, 0, 0, 0, 0, 7, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 43, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 11, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 4095, 0, 1, 1, 4095, 4092, 4095, 1, 1, 1, 0, 0, 0, 1, 1, 4095, 4095, 1, 1, 0, 0, 15, 0, 0, 0, 0, 0, 4081] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 24, 0, 4096, 0, 0, 0, 4091, 4093, 4096, 0, 0, 0, 0, 1, 0, 0, 1, 4034, 4096, 0, 0, 0, 29, 0, 0, 0, 0, 15, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4094, 93, 0, 4096, 0, 0, 0, 4088, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 38, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 76, 0, 4096, 0, 0, 0, 4087, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 4096, 0, 0, 0, 20, 0, 0, 0, 0, 67, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 60, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 32, 0, 4096, 0, 0, 0, 4079, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 4096, 0, 0, 0, 75, 0, 0, 0, 0, 21, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4060, 43, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4035, 36, 25, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4073, 0, 0, 0, 0, 11, 0, 4071, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4096, 4102, 4096, 4113, 4096, 4080, 4082] avg=4096.0000 max=4113 max_rank=4 min=4080 min_rank=6 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4060, 27, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4038, 36, 25, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 4073, 0, 0, 0, 0, 7, 0, 4071, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4096, 4098, 4096, 4116, 4096, 4101, 4078] avg=4096.0000 max=4116 max_rank=4 min=4078 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4060, 24, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 1, 4034, 36, 25, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 4070, 0, 0, 0, 0, 15, 0, 4071, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4096, 4097, 4097, 4113, 4096, 4099, 4086] avg=4096.0000 max=4113 max_rank=4 min=4084 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4060, 76, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3942, 36, 25, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4073, 0, 0, 0, 0, 67, 0, 4071, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4096, 4093, 4096, 4020, 4096, 4093, 4138] avg=4096.0000 max=4138 max_rank=7 min=4020 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4052, 93, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3963, 42, 29, 21, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4074, 0, 0, 0, 0, 38, 0, 4067, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4095, 4089, 4096, 4056, 4096, 4086, 4105] avg=4096.0000 max=4145 max_rank=0 min=4056 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4059, 1, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 4095, 6, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 1, 4095, 36, 25, 17, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4097, 4102, 4097, 4175, 4097, 4084, 4056] avg=4096.0000 max=4175 max_rank=4 min=4056 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4060, 32, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4079, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3985, 36, 25, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 4073, 1, 0, 0, 0, 21, 0, 4071, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4096, 4085, 4095, 4063, 4096, 4149, 4092] avg=4096.0000 max=4149 max_rank=6 min=4063 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4060, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4033, 36, 25, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4073, 1, 0, 0, 0, 60, 0, 4071, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4096, 4102, 4095, 4111, 4096, 4076, 4131] avg=4096.0000 max=4131 max_rank=7 min=4061 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32767, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32767, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32767, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32767, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32767, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32767, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32767, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32767, 32769, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32767 actual_valid_recv=33408 observed_loss=-641 tokens_per_expert=[0, 128, 128, 32128, 384, 256, 256, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32512, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 0, 0, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 256, 0, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32769 actual_valid_recv=33024 observed_loss=-255 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4085, 0, 553, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 3469, 0, 0, 86, 0, 4096, 8, 0, 0, 0, 4094, 0, 0, 1, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 563, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 3485, 0, 0, 54, 0, 4096, 4, 0, 0, 0, 4092, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4059, 0, 375, 0, 4088, 0, 2, 0, 0, 0, 0, 4089, 3690, 0, 0, 73, 0, 4096, 9, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 1604, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 2390, 0, 0, 109, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4080, 0, 804, 0, 4096, 0, 0, 0, 0, 0, 0, 4086, 3190, 0, 0, 117, 0, 4096, 10, 0, 0, 0, 4096, 0, 0, 1, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 4087, 0, 616, 1, 3891, 34, 0, 1, 1, 1, 1, 4095, 3652, 0, 1, 0, 6, 4092, 0, 0, 1, 0, 4095, 1, 0, 0, 4095, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 937, 0, 4092, 1, 0, 0, 0, 0, 0, 4096, 3135, 0, 0, 28, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4047, 0, 341, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 3734, 0, 0, 58, 0, 4096, 15, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3384, 0, 1604, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 1, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 2390, 0, 0, 109, 689, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4988, 4118, 4092, 3188, 4094, 4096, 4096] avg=4096.0000 max=4988 max_rank=1 min=3188 min_rank=4 imbalance_ratio=1.217773 max_load_reduction=27780 max_load_reduction_pct=0.847778 moved_to_echo=15474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3373, 0, 804, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 1, 0, 0, 0, 0, 0, 4086, 4, 0, 0, 0, 3190, 0, 0, 117, 686, 0, 0, 0, 0, 4091, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4177, 4118, 4090, 3993, 4101, 4096, 4097] avg=4096.0000 max=4177 max_rank=1 min=3993 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 3345, 0, 341, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 1, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 3734, 0, 0, 58, 681, 1, 0, 0, 0, 4091, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 3686, 4118, 4097, 4474, 4106, 4096, 4096] avg=4096.0000 max=4474 max_rank=4 min=3686 min_rank=1 imbalance_ratio=1.092285 max_load_reduction=28294 max_load_reduction_pct=0.863464 moved_to_echo=16772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 4095, 0, 0, 0, 3378, 0, 616, 1, 0, 0, 0, 0, 3891, 34, 0, 1, 21, 0, 0, 0, 1, 1, 1, 4095, 4, 0, 0, 0, 3652, 0, 1, 0, 688, 0, 0, 0, 6, 4088, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3995, 3947, 4102, 4341, 4094, 4097, 4095] avg=4096.0000 max=4341 max_rank=4 min=3947 min_rank=2 imbalance_ratio=1.059814 max_load_reduction=28427 max_load_reduction_pct=0.867523 moved_to_echo=16627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3355, 0, 375, 0, 0, 0, 0, 0, 4088, 0, 2, 0, 21, 1, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 0, 3690, 0, 0, 73, 683, 0, 0, 0, 0, 4091, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3730, 4112, 4093, 4446, 4100, 4096, 4095] avg=4096.0000 max=4446 max_rank=4 min=3730 min_rank=1 imbalance_ratio=1.085449 max_load_reduction=28322 max_load_reduction_pct=0.864319 moved_to_echo=16737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3385, 0, 563, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 1, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 3485, 0, 0, 54, 689, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3948, 4118, 4095, 4228, 4095, 4092, 4096] avg=4096.0000 max=4228 max_rank=4 min=3948 min_rank=1 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3370, 0, 553, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 24, 2, 0, 0, 0, 0, 0, 4088, 11, 0, 0, 0, 3469, 0, 0, 86, 691, 0, 0, 0, 0, 4079, 8, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3923, 4122, 4099, 4246, 4087, 4098, 4097] avg=4096.0000 max=4246 max_rank=4 min=3923 min_rank=1 imbalance_ratio=1.036621 max_load_reduction=28522 max_load_reduction_pct=0.870422 moved_to_echo=16528 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3384, 0, 937, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 21, 1, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 3135, 0, 0, 28, 689, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4321, 4115, 4100, 3852, 4091, 4097, 4096] avg=4096.0000 max=4321 max_rank=1 min=3852 min_rank=4 imbalance_ratio=1.054932 max_load_reduction=28447 max_load_reduction_pct=0.868134 moved_to_echo=16136 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[26752, 0, 128, 640, 5504, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[27008, 0, 5888, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 128, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4066, 0, 5, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4034, 0, 5, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3957, 0, 11, 0, 4096, 4093, 4096, 0, 0, 0, 0, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4060, 0, 3, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1, 1, 4092, 168, 1, 0, 0, 1, 0, 0, 0, 1, 0, 19, 1, 1, 4094, 1, 0, 4095, 3876, 0, 39, 1, 4095, 4095, 4090, 0, 0, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3960, 0, 8, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 4096, 4063, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 3999, 0, 4, 0, 4096, 4092, 4094, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4010, 62, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 5, 0, 83, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4096, 4096, 4095, 4096, 4123, 4095, 4095] avg=4096.0000 max=4123 max_rank=5 min=4072 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16409 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4010, 30, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 5, 0, 83, 2, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4096, 4096, 4094, 4096, 4156, 4097, 4093] avg=4096.0000 max=4156 max_rank=5 min=4040 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16442 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4010, 36, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 3, 0, 83, 1, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4096, 4096, 4093, 4096, 4147, 4098, 4096] avg=4096.0000 max=4147 max_rank=5 min=4046 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16437 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4003, 139, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3957, 0, 11, 0, 85, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4096, 4096, 4095, 4096, 4054, 4093, 4096] avg=4096.0000 max=4142 max_rank=0 min=4054 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16339 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4010, 136, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 8, 0, 83, 1, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4096, 4096, 4093, 4096, 4052, 4098, 4091] avg=4096.0000 max=4146 max_rank=0 min=4052 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16337 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4010, 98, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 3999, 0, 4, 0, 83, 1, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4095, 4096, 4092, 4097, 4087, 4094, 4099] avg=4096.0000 max=4108 max_rank=0 min=4087 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16377 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4006, 168, 0, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 19, 1, 1, 4087, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 3876, 0, 39, 1, 83, 3, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4097, 4096, 4108, 4095, 4002, 4095, 4099] avg=4096.0000 max=4176 max_rank=0 min=4002 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16291 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4010, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 83, 1, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4096, 4096, 4098, 4096, 4147, 4098, 4099] avg=4096.0000 max=4147 max_rank=5 min=4038 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16440 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32128, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32128, 0, 128, 128, 768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4093, 4096, 0, 0, 4096, 2, 4096, 0, 6, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 9, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4093, 4096, 0, 0, 0, 4096, 4088, 4096, 0, 0, 4096, 0, 4096, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4060, 4093, 1, 0, 0, 4092, 4095, 4051, 0, 1, 4095, 0, 4095, 2, 0, 4095, 1, 0, 1, 76, 1, 0, 0, 1, 0, 1, 1, 4, 0, 0, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4095, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4099, 4096, 4095, 4086, 4096, 4100, 4096] avg=4096.0000 max=4100 max_rank=0 min=4086 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16378 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4103, 4098, 4096, 4083, 4096, 4094, 4096] avg=4096.0000 max=4103 max_rank=1 min=4083 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16369 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4103, 4096, 4096, 4085, 4096, 4094, 4096] avg=4096.0000 max=4103 max_rank=1 min=4085 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16371 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4103, 4096, 4096, 4085, 4096, 4094, 4096] avg=4096.0000 max=4103 max_rank=1 min=4085 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16371 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 11, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 8, 2, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 4094, 4099, 4093, 4098, 4100, 4095] avg=4096.0000 max=4100 max_rank=6 min=4093 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16386 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4103, 4096, 4095, 4086, 4096, 4094, 4096] avg=4096.0000 max=4103 max_rank=1 min=4086 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4060, 5, 1, 0, 0, 0, 0, 0, 0, 0, 7, 4051, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 1, 0, 1, 76, 4084, 1, 0, 0, 1, 0, 0, 1, 4092, 0, 0, 0, 0, 1, 1, 4, 4088, 4, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4058, 4096, 4096, 4163, 4094, 4098, 4097] avg=4096.0000 max=4163 max_rank=4 min=4058 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16452 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4103, 4096, 4095, 4087, 4096, 4094, 4096] avg=4096.0000 max=4103 max_rank=1 min=4087 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 238, 0, 0, 3989, 0, 12, 4094, 1, 0, 0, 4096, 0, 12, 4096, 4096, 0, 0, 3942, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 66, 0, 0, 4089, 0, 1, 4095, 0, 0, 0, 4096, 0, 7, 4096, 4096, 0, 0, 4030, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 100, 0, 0, 4031, 0, 0, 4096, 1, 0, 0, 4096, 0, 6, 4096, 4096, 0, 0, 4054, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 178, 0, 0, 4081, 0, 0, 4096, 1, 0, 0, 4096, 0, 9, 4096, 4096, 0, 0, 3923, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 105, 0, 0, 4043, 0, 0, 4096, 0, 0, 0, 4096, 0, 10, 4096, 4096, 0, 0, 4034, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[12, 1, 2, 1, 1, 0, 0, 4095, 0, 5, 89, 0, 1, 4035, 0, 0, 4095, 0, 0, 0, 4094, 0, 0, 4062, 4094, 0, 0, 4095, 1, 4084, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 4093, 0, 0, 4096, 4, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4078, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 197, 0, 0, 4000, 0, 0, 4096, 0, 0, 0, 4096, 0, 11, 4096, 4096, 0, 0, 3984, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 100, 0, 3970, 2, 0, 0, 0, 4031, 0, 0, 47, 1, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4095, 0, 0, 0, 0, 79, 0, 0, 4054, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4072, 4079, 4097, 4101, 4133, 4096] avg=4096.0000 max=4133 max_rank=6 min=4072 min_rank=2 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 66, 0, 3970, 2, 0, 0, 0, 4089, 0, 1, 47, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4095, 0, 0, 0, 0, 79, 0, 0, 4030, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4038, 4138, 4095, 4102, 4109, 4096] avg=4096.0000 max=4138 max_rank=3 min=4038 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 105, 0, 3970, 1, 0, 0, 0, 4043, 0, 0, 47, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4095, 0, 0, 0, 0, 79, 0, 0, 4034, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4076, 4091, 4096, 4105, 4113, 4096] avg=4096.0000 max=4113 max_rank=6 min=4076 min_rank=2 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 178, 0, 3970, 2, 0, 0, 0, 4081, 0, 0, 47, 1, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4095, 0, 0, 0, 0, 79, 0, 0, 3923, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4150, 4129, 4097, 4104, 4002, 4096] avg=4096.0000 max=4150 max_rank=2 min=4002 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 238, 0, 3971, 1, 0, 0, 0, 3989, 0, 12, 50, 7, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4082, 0, 0, 0, 0, 75, 0, 0, 3942, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 3, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4210, 4058, 4092, 4094, 4017, 4106] avg=4096.0000 max=4210 max_rank=2 min=4017 min_rank=6 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 197, 0, 3970, 2, 0, 0, 0, 4000, 0, 0, 47, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4095, 0, 0, 0, 0, 79, 0, 0, 3984, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4169, 4048, 4096, 4106, 4063, 4096] avg=4096.0000 max=4169 max_rank=2 min=4048 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 17, 0, 3970, 2, 0, 0, 0, 4093, 0, 0, 47, 1, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 79, 0, 0, 4078, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 3989, 4141, 4100, 4095, 4157, 4096] avg=4096.0000 max=4157 max_rank=6 min=3989 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 1, 2, 1, 4092, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 5, 89, 0, 3968, 2, 0, 0, 1, 4035, 0, 0, 47, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 79, 0, 0, 4095, 0, 0, 0, 0, 1, 4084, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4096, 4064, 4084, 4095, 4061, 4174, 4086] avg=4096.0000 max=4174 max_rank=6 min=4061 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 1024, 0, 31872, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[640, 0, 0, 32256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32384, 0, 128, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4073, 1, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 30, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4086, 9, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 6, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4083, 6, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 7, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[149, 0, 0, 1, 1, 0, 10, 1, 4095, 1, 3891, 26, 0, 0, 0, 1, 4094, 4094, 4092, 1, 1, 0, 0, 2, 4078, 0, 1, 4095, 0, 18, 64, 4052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4093, 2, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 10, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4088, 3, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 14, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 2, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 4, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 5, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4063, 7, 4, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 28, 0, 4086, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4085, 2, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4094, 4123, 4096, 4096, 4093, 4100, 4092] avg=4096.0000 max=4123 max_rank=2 min=4074 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4060, 7, 4, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 28, 0, 4073, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 30, 0, 0, 4088, 1, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4094, 4102, 4097, 4096, 4119, 4097, 4092] avg=4096.0000 max=4119 max_rank=5 min=4071 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4067, 10, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 28, 0, 4083, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4096, 4117, 4096, 4096, 4093, 4096, 4092] avg=4096.0000 max=4117 max_rank=2 min=4082 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4059, 7, 4, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 28, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4085, 2, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4094, 4119, 4096, 4096, 4101, 4100, 4092] avg=4096.0000 max=4119 max_rank=2 min=4070 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4064, 11, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 23, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4096, 4118, 4096, 4096, 4095, 4096, 4086] avg=4096.0000 max=4118 max_rank=2 min=4085 min_rank=0 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4068, 7, 4, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 28, 0, 4087, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4094, 4119, 4096, 4098, 4090, 4100, 4092] avg=4096.0000 max=4119 max_rank=2 min=4079 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4068, 7, 4, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 28, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4094, 4124, 4096, 4097, 4086, 4100, 4092] avg=4096.0000 max=4124 max_rank=2 min=4079 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[149, 0, 0, 1, 4067, 7, 4, 0, 1, 0, 10, 1, 4094, 0, 0, 0, 28, 1, 3891, 26, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 1, 0, 0, 2, 4088, 0, 0, 0, 4078, 0, 1, 0, 0, 0, 0, 0, 0, 18, 64, 4048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4228, 4106, 3946, 4095, 4093, 4091, 4079, 4130] avg=4096.0000 max=4228 max_rank=0 min=3946 min_rank=2 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 0, 0, 128, 32640, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 32512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 0, 771, 3890, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3518, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 745, 3938, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 28, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 3480, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 339, 4008, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 3825, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1, 2, 1, 1647, 4019, 1, 0, 4094, 1, 1, 1, 1, 4095, 1, 1, 1, 4087, 0, 12, 148, 1, 0, 0, 4095, 4029, 0, 0, 0, 0, 2435, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 104, 4051, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 4018, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 549, 4009, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 112, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3522, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4, 0, 948, 3957, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3275, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 2, 0, 203, 3969, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4015, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 771, 3395, 36, 0, 0, 3890, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 115, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 3518, 0, 586, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4203, 4006, 4095, 4096, 4065, 4096, 4103, 4104] avg=4096.0000 max=4203 max_rank=0 min=4006 min_rank=1 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16368 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 745, 3395, 36, 0, 0, 3938, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 115, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 7, 0, 0, 0, 0, 3480, 0, 586, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4054, 4095, 4096, 4081, 4096, 4104, 4066] avg=4096.0000 max=4176 max_rank=0 min=4054 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16347 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 339, 3399, 37, 0, 0, 4008, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 119, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2, 9, 2, 0, 0, 0, 3825, 0, 578, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3775, 4122, 4101, 4094, 4068, 4096, 4109, 4403] avg=4096.0000 max=4403 max_rank=7 min=3775 min_rank=0 imbalance_ratio=1.074951 max_load_reduction=28365 max_load_reduction_pct=0.865631 moved_to_echo=16676 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 549, 3395, 36, 0, 0, 4009, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 115, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 3522, 0, 586, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3980, 4125, 4095, 4096, 4165, 4096, 4103, 4108] avg=4096.0000 max=4165 max_rank=4 min=3980 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16472 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 104, 3395, 36, 0, 0, 4051, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 115, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 7, 0, 0, 0, 0, 4018, 0, 586, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3535, 4167, 4095, 4096, 4068, 4096, 4107, 4604] avg=4096.0000 max=4604 max_rank=7 min=3535 min_rank=0 imbalance_ratio=1.124023 max_load_reduction=28164 max_load_reduction_pct=0.859497 moved_to_echo=16875 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 1, 1647, 3393, 36, 0, 0, 4019, 1, 0, 116, 0, 0, 0, 0, 1, 1, 1, 1, 3978, 115, 0, 0, 4095, 1, 1, 1, 0, 0, 0, 0, 4044, 0, 12, 148, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 4029, 0, 0, 0, 7, 0, 0, 0, 0, 2435, 0, 586, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5080, 4136, 4097, 4098, 4204, 4096, 4036, 3021] avg=4096.0000 max=5080 max_rank=0 min=3021 min_rank=7 imbalance_ratio=1.240234 max_load_reduction=27688 max_load_reduction_pct=0.844971 moved_to_echo=15357 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 203, 3395, 36, 0, 0, 3969, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 115, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 4015, 0, 586, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3636, 4085, 4095, 4096, 4056, 4096, 4103, 4601] avg=4096.0000 max=4601 max_rank=7 min=3636 min_rank=0 imbalance_ratio=1.123291 max_load_reduction=28167 max_load_reduction_pct=0.859589 moved_to_echo=16856 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 948, 3395, 36, 0, 0, 3957, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 115, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 3275, 0, 586, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4383, 4073, 4095, 4096, 4061, 4096, 4103, 3861] avg=4096.0000 max=4383 max_rank=0 min=3861 min_rank=7 imbalance_ratio=1.070068 max_load_reduction=28385 max_load_reduction_pct=0.866241 moved_to_echo=16121 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 5376, 27264, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 28160, 0, 4736, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31872, 128, 0, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 31872, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 0, 128, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4095, 4095, 9, 4086, 2, 2, 4095, 63, 1, 0, 0, 4093, 1, 4032, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 4095, 1, 0, 1, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4088, 4089, 4, 4096, 0, 0, 4096, 10, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 0, 0, 4082, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4087, 4084, 0, 4096, 0, 0, 4095, 12, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 0, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4068, 4074, 4, 4096, 0, 0, 4095, 33, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 4096, 0, 0, 0, 4083, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4083, 4088, 11, 4096, 0, 0, 4096, 16, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 0, 0, 0, 4073, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4095, 4096, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 4096, 0, 0, 0, 4039, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4087, 4086, 0, 4096, 0, 0, 4096, 14, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 4081, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 1, 4096, 0, 0, 4096, 3, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4088, 6, 4, 0, 0, 0, 0, 0, 0, 0, 4078, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 18, 0, 4077, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 10, 5, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4088, 4096, 4101, 4097, 4095, 4096, 4097] avg=4096.0000 max=4101 max_rank=3 min=4088 min_rank=1 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4087, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 12, 0, 4072, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 10, 10, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4089, 4096, 4104, 4096, 4084, 4096, 4114] avg=4096.0000 max=4114 max_rank=7 min=4084 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4083, 7, 11, 0, 0, 0, 0, 0, 0, 0, 4078, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 17, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 10, 5, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4094, 4096, 4104, 4096, 4093, 4096, 4088] avg=4096.0000 max=4104 max_rank=3 min=4088 min_rank=7 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4087, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 14, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 23, 0, 4074, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 10, 5, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4092, 4096, 4101, 4096, 4097, 4096, 4096] avg=4096.0000 max=4101 max_rank=3 min=4092 min_rank=1 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 55, 0, 4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 10, 5, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4081, 4096, 4104, 4097, 4139, 4096, 4054] avg=4096.0000 max=4139 max_rank=5 min=4054 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4095, 6, 9, 0, 0, 0, 0, 0, 2, 2, 4077, 63, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 1, 4032, 0, 0, 8, 0, 0, 0, 0, 1, 1, 0, 4086, 1, 0, 0, 0, 1, 0, 0, 4083, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 4094, 0, 10, 5, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4144, 4094, 4041, 4089, 4084, 4096, 4110] avg=4096.0000 max=4144 max_rank=1 min=4041 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4068, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4066, 33, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4094, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 29, 0, 4063, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 11, 7, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4099, 4099, 4109, 4100, 4092, 4096, 4101] avg=4096.0000 max=4109 max_rank=3 min=4072 min_rank=0 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4078, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 10, 5, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4081, 4095, 4104, 4097, 4084, 4096, 4108] avg=4096.0000 max=4108 max_rank=7 min=4081 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32640, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 256, 0, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[288, 0, 2187, 4, 1102, 1, 2213, 31, 1137, 1334, 0, 30, 2024, 3376, 2, 688, 2, 775, 2970, 0, 2, 334, 1257, 39, 0, 3126, 3651, 1887, 3197, 1065, 45, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[189, 0, 1289, 8, 1586, 0, 2797, 6, 1532, 1285, 0, 25, 1358, 3363, 5, 688, 3, 2054, 2646, 0, 0, 856, 889, 52, 0, 2759, 3473, 1446, 3063, 1340, 56, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[108, 3, 1657, 64, 1652, 3, 2153, 3, 1617, 1091, 0, 26, 2086, 3478, 9, 604, 5, 1322, 2133, 0, 0, 584, 1502, 154, 0, 2506, 3788, 2048, 2893, 1178, 101, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[120, 4, 1696, 14, 1387, 26, 2183, 33, 1557, 1571, 0, 25, 1439, 2817, 6, 679, 14, 2079, 1392, 1, 5, 520, 1666, 75, 2, 3042, 3661, 1849, 3394, 1421, 84, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[300, 0, 2004, 24, 1606, 0, 2375, 9, 1504, 1284, 0, 29, 1015, 3659, 14, 865, 31, 1603, 2267, 0, 0, 651, 1471, 57, 0, 2879, 3873, 1855, 2100, 1265, 28, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[325, 1, 1561, 19, 1588, 1, 2817, 8, 1398, 1191, 3, 25, 1703, 3547, 0, 536, 12, 1231, 2024, 0, 5, 569, 1308, 40, 0, 2572, 3729, 2020, 2924, 1535, 76, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[212, 3, 1326, 3, 1188, 0, 2883, 23, 1791, 1292, 0, 13, 1682, 3538, 10, 711, 2, 1182, 2213, 2, 4, 724, 928, 37, 1, 3119, 3909, 1658, 3021, 1246, 47, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[174, 1, 1105, 5, 1097, 0, 2808, 33, 1326, 1475, 1, 31, 2114, 3476, 3, 676, 4, 1390, 1995, 0, 1, 528, 1283, 32, 0, 3363, 3758, 1679, 2749, 1601, 60, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[189, 0, 1289, 8, 2104, 0, 0, 0, 1586, 0, 2797, 6, 151, 0, 0, 0, 1532, 1285, 0, 25, 984, 259, 0, 0, 1358, 1710, 5, 688, 0, 0, 0, 0, 3, 2054, 2646, 0, 335, 75, 0, 0, 0, 856, 889, 52, 1369, 669, 0, 0, 0, 2165, 0, 1446, 0, 0, 0, 0, 2837, 1340, 56, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3590, 4540, 4085, 3761, 5113, 3835, 3611, 4233] avg=4096.0000 max=5113 max_rank=4 min=3590 min_rank=0 imbalance_ratio=1.248291 max_load_reduction=27655 max_load_reduction_pct=0.843964 moved_to_echo=16792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[288, 0, 2187, 4, 2211, 0, 0, 0, 1102, 1, 2213, 31, 158, 0, 0, 0, 1137, 1334, 0, 30, 988, 294, 0, 0, 2024, 1716, 2, 688, 0, 0, 0, 0, 2, 775, 2970, 0, 380, 78, 0, 0, 2, 334, 1257, 39, 1440, 672, 0, 0, 0, 2452, 0, 1887, 0, 0, 0, 0, 2961, 1065, 45, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4690, 3505, 3783, 4430, 4205, 3744, 4339, 4072] avg=4096.0000 max=4690 max_rank=0 min=3505 min_rank=1 imbalance_ratio=1.145020 max_load_reduction=28078 max_load_reduction_pct=0.856873 moved_to_echo=16360 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[325, 1, 1561, 19, 2258, 0, 0, 0, 1588, 1, 2817, 8, 144, 0, 0, 0, 1398, 1191, 3, 25, 1038, 242, 0, 0, 1703, 1803, 0, 536, 0, 0, 0, 0, 12, 1231, 2024, 0, 313, 71, 0, 0, 5, 569, 1308, 40, 1471, 706, 0, 0, 0, 2017, 0, 2020, 0, 0, 0, 0, 2709, 1535, 76, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4558, 3897, 4042, 3651, 4099, 4037, 4320] avg=4096.0000 max=4558 max_rank=1 min=3651 min_rank=4 imbalance_ratio=1.112793 max_load_reduction=28210 max_load_reduction_pct=0.860901 moved_to_echo=16107 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[108, 3, 1657, 64, 2294, 0, 0, 0, 1652, 3, 2153, 3, 143, 0, 0, 0, 1617, 1091, 0, 26, 1017, 235, 0, 0, 2086, 1769, 9, 604, 0, 0, 0, 0, 5, 1322, 2133, 0, 305, 70, 0, 0, 0, 584, 1502, 154, 1494, 692, 0, 0, 0, 1966, 0, 2048, 0, 0, 0, 0, 2680, 1178, 101, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 3954, 3986, 4468, 3835, 4426, 4014, 3959] avg=4096.0000 max=4468 max_rank=3 min=3835 min_rank=4 imbalance_ratio=1.090820 max_load_reduction=28300 max_load_reduction_pct=0.863647 moved_to_echo=16234 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[300, 0, 2004, 24, 2345, 0, 0, 0, 1606, 0, 2375, 9, 104, 0, 0, 0, 1504, 1284, 0, 29, 1070, 270, 0, 0, 1015, 1861, 14, 865, 0, 0, 0, 0, 31, 1603, 2267, 0, 350, 51, 0, 0, 0, 651, 1471, 57, 1528, 728, 0, 0, 0, 2259, 0, 1855, 0, 0, 0, 0, 1945, 1265, 28, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4673, 4094, 4157, 3755, 4302, 4435, 4114, 3238] avg=4096.0000 max=4673 max_rank=0 min=3238 min_rank=7 imbalance_ratio=1.140869 max_load_reduction=28095 max_load_reduction_pct=0.857391 moved_to_echo=16089 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[120, 4, 1696, 14, 2217, 0, 0, 0, 1387, 26, 2183, 33, 168, 0, 0, 0, 1557, 1571, 0, 25, 824, 286, 0, 0, 1439, 1433, 6, 679, 0, 0, 0, 0, 14, 2079, 1392, 1, 370, 83, 0, 0, 5, 520, 1666, 75, 1444, 560, 0, 0, 2, 2386, 0, 1849, 0, 0, 0, 0, 3143, 1421, 84, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 3797, 4263, 3557, 3939, 4270, 4237, 4654] avg=4096.0000 max=4654 max_rank=7 min=3557 min_rank=3 imbalance_ratio=1.136230 max_load_reduction=28114 max_load_reduction_pct=0.857971 moved_to_echo=17100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[174, 1, 1105, 5, 2277, 0, 0, 0, 1097, 0, 2808, 33, 139, 0, 0, 0, 1326, 1475, 1, 31, 1020, 320, 0, 0, 2114, 1760, 3, 676, 0, 0, 0, 0, 4, 1390, 1995, 0, 412, 70, 0, 0, 1, 528, 1283, 32, 1481, 696, 0, 0, 0, 2631, 0, 1679, 0, 0, 0, 0, 2540, 1601, 60, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3562, 4077, 4173, 4553, 3871, 4021, 4310, 4201] avg=4096.0000 max=4553 max_rank=3 min=3562 min_rank=0 imbalance_ratio=1.111572 max_load_reduction=28215 max_load_reduction_pct=0.861053 moved_to_echo=16403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[212, 3, 1326, 3, 2368, 0, 0, 0, 1188, 0, 2883, 23, 149, 0, 0, 0, 1791, 1292, 0, 13, 1035, 293, 0, 0, 1682, 1799, 10, 711, 0, 0, 0, 0, 2, 1182, 2213, 2, 379, 74, 0, 0, 4, 724, 928, 37, 1541, 704, 0, 0, 1, 2447, 0, 1658, 0, 0, 0, 0, 2798, 1246, 47, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3912, 4243, 4424, 4202, 3852, 3938, 4106, 4091] avg=4096.0000 max=4424 max_rank=2 min=3852 min_rank=4 imbalance_ratio=1.080078 max_load_reduction=28344 max_load_reduction_pct=0.864990 moved_to_echo=15987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11904, 10624, 128, 256, 8064, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 18432, 0, 14464, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 4864, 10368, 512, 11776, 5504, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1792, 128, 12928, 256, 18176, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13440, 13952, 128, 5504, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21632, 10752, 512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 11648, 17664, 128, 2944, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11264, 128, 20352, 256, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[239, 0, 0, 0, 1547, 0, 0, 0, 0, 1348, 2176, 0, 4, 0, 4095, 4093, 4096, 0, 0, 0, 3649, 0, 0, 0, 4095, 3331, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[976, 0, 0, 0, 1582, 0, 0, 0, 0, 783, 2995, 0, 1, 0, 4096, 4092, 4096, 0, 0, 0, 2790, 0, 1, 1, 4094, 3165, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[245, 0, 0, 0, 1494, 0, 0, 0, 0, 637, 2570, 0, 14, 0, 4096, 4095, 4096, 0, 0, 0, 3830, 0, 0, 0, 4096, 3513, 0, 0, 0, 4082, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[392, 0, 0, 0, 1270, 0, 0, 0, 0, 556, 2551, 0, 9, 0, 4096, 4096, 4096, 0, 0, 0, 3603, 0, 0, 0, 4093, 3910, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[86, 4, 0, 0, 421, 0, 14, 0, 18, 1621, 2168, 0, 0, 0, 4082, 4073, 4080, 5, 1, 0, 3883, 0, 3, 0, 4083, 4092, 16, 14, 7, 4074, 8, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[48, 0, 0, 0, 1383, 0, 0, 0, 0, 1013, 1796, 0, 15, 0, 4096, 4095, 4096, 0, 0, 0, 4051, 0, 0, 0, 4094, 3978, 0, 0, 0, 4096, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1195, 1, 0, 3, 1881, 0, 0, 0, 0, 408, 2184, 0, 4, 0, 4096, 4086, 4096, 1, 0, 0, 3023, 0, 0, 0, 4091, 3605, 0, 0, 0, 4093, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[152, 0, 0, 0, 443, 0, 0, 0, 0, 1587, 2180, 0, 5, 0, 4096, 4096, 4096, 0, 0, 0, 3933, 0, 0, 0, 4093, 3991, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[239, 0, 0, 0, 3679, 0, 0, 0, 1547, 0, 0, 0, 2427, 416, 0, 0, 0, 1348, 2176, 0, 772, 0, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 3649, 0, 0, 0, 500, 0, 0, 0, 396, 3331, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3918, 4390, 4296, 4096, 4097, 4149, 3727, 4095] avg=4096.0000 max=4390 max_rank=1 min=3727 min_rank=6 imbalance_ratio=1.071777 max_load_reduction=28378 max_load_reduction_pct=0.866028 moved_to_echo=16068 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[392, 0, 0, 0, 3680, 0, 0, 0, 1270, 0, 0, 0, 2426, 416, 0, 0, 0, 556, 2551, 0, 772, 0, 0, 0, 9, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 3603, 0, 0, 0, 500, 0, 0, 0, 395, 3910, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4112, 3879, 4104, 4097, 4103, 4305, 4096] avg=4096.0000 max=4305 max_rank=6 min=3879 min_rank=2 imbalance_ratio=1.051025 max_load_reduction=28463 max_load_reduction_pct=0.868622 moved_to_echo=16601 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[245, 0, 0, 0, 3680, 0, 0, 0, 1494, 0, 0, 0, 2428, 416, 0, 0, 0, 637, 2570, 0, 772, 0, 0, 0, 14, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 3830, 0, 0, 0, 500, 0, 0, 0, 396, 3513, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3925, 4338, 3979, 4108, 4097, 4330, 3909, 4082] avg=4096.0000 max=4338 max_rank=1 min=3909 min_rank=6 imbalance_ratio=1.059082 max_load_reduction=28430 max_load_reduction_pct=0.867615 moved_to_echo=16418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[48, 0, 0, 0, 3680, 0, 0, 0, 1383, 0, 0, 0, 2426, 416, 0, 0, 0, 1013, 1796, 0, 772, 0, 0, 0, 15, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4051, 0, 0, 0, 500, 0, 0, 0, 396, 3978, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3728, 4225, 3581, 4109, 4097, 4551, 4374, 4103] avg=4096.0000 max=4551 max_rank=5 min=3581 min_rank=2 imbalance_ratio=1.111084 max_load_reduction=28217 max_load_reduction_pct=0.861115 moved_to_echo=17125 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[976, 0, 0, 0, 3681, 0, 0, 0, 1582, 0, 0, 0, 2429, 415, 0, 0, 0, 783, 2995, 0, 776, 0, 0, 0, 1, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 2790, 0, 1, 1, 504, 0, 0, 0, 384, 3165, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4657, 4426, 4554, 4088, 4100, 3296, 3549, 4098] avg=4096.0000 max=4657 max_rank=0 min=3296 min_rank=5 imbalance_ratio=1.136963 max_load_reduction=28111 max_load_reduction_pct=0.857880 moved_to_echo=15043 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[86, 4, 0, 0, 3667, 0, 0, 0, 421, 0, 14, 0, 2420, 415, 0, 0, 18, 1621, 2168, 0, 770, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 4080, 5, 1, 0, 0, 0, 0, 0, 3883, 0, 3, 0, 498, 0, 0, 0, 395, 4092, 16, 14, 0, 0, 0, 0, 7, 4074, 8, 15, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3757, 3270, 4577, 4073, 4086, 4384, 4517, 4104] avg=4096.0000 max=4577 max_rank=2 min=3270 min_rank=1 imbalance_ratio=1.117432 max_load_reduction=28191 max_load_reduction_pct=0.860321 moved_to_echo=17091 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[152, 0, 0, 0, 3680, 0, 0, 0, 443, 0, 0, 0, 2426, 416, 0, 0, 0, 1587, 2180, 0, 772, 0, 0, 0, 5, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 3933, 0, 0, 0, 500, 0, 0, 0, 395, 3991, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3832, 3285, 4539, 4100, 4097, 4433, 4386, 4096] avg=4096.0000 max=4539 max_rank=2 min=3285 min_rank=1 imbalance_ratio=1.108154 max_load_reduction=28229 max_load_reduction_pct=0.861481 moved_to_echo=17012 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1195, 1, 0, 3, 3680, 0, 0, 0, 1881, 0, 0, 0, 2425, 416, 0, 0, 0, 408, 2184, 0, 771, 0, 0, 0, 4, 0, 0, 4086, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 3023, 0, 0, 0, 499, 0, 0, 0, 396, 3605, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4879, 4722, 3363, 4090, 4097, 3522, 4001, 4094] avg=4096.0000 max=4879 max_rank=0 min=3363 min_rank=2 imbalance_ratio=1.191162 max_load_reduction=27889 max_load_reduction_pct=0.851105 moved_to_echo=15714 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10112, 0, 128, 0, 19456, 3328, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 8064, 18688, 0, 6272, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[28800, 0, 128, 128, 4096, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3456, 128, 0, 128, 29440, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3200, 29696, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3662, 0, 0, 0, 4096, 12, 4077, 4096, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 650, 2, 0, 0, 0, 7, 0, 3877] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3643, 0, 0, 0, 4096, 0, 4090, 4095, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 400, 10, 0, 0, 0, 79, 0, 4062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3776, 0, 0, 0, 4096, 1, 4054, 4096, 3, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 431, 0, 0, 0, 0, 147, 0, 3872] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 7, 2573, 0, 0, 0, 4096, 8, 3942, 4096, 0, 0, 4089, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 4096, 1657, 5, 0, 0, 0, 102, 0, 3988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 3785, 0, 0, 0, 4096, 5, 4093, 4096, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 703, 12, 0, 0, 1, 218, 0, 3469] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3, 4, 3566, 0, 2, 0, 4095, 0, 4088, 4096, 2, 1, 4092, 0, 0, 3, 1, 0, 0, 0, 0, 0, 4096, 4092, 707, 3, 0, 0, 0, 349, 0, 3568] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[5, 2, 2953, 4, 0, 0, 4082, 0, 4023, 4092, 0, 10, 4093, 2, 2, 0, 10, 4, 0, 3, 0, 0, 4086, 4074, 1876, 6, 11, 1, 26, 1, 0, 3402] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3452, 0, 0, 0, 4096, 2, 4094, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 686, 6, 0, 0, 0, 165, 0, 3882] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3643, 0, 666, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4090, 38, 3, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 1, 1, 4096, 0, 0, 0, 0, 400, 10, 0, 0, 3199, 0, 0, 0, 0, 79, 0, 4062, 190, 1, 1, 0] rerouted_tokens_per_ep_rank=[4310, 4095, 4131, 4095, 4097, 4098, 3609, 4333] avg=4096.0000 max=4333 max_rank=7 min=3609 min_rank=6 imbalance_ratio=1.057861 max_load_reduction=28435 max_load_reduction_pct=0.867767 moved_to_echo=16137 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3662, 0, 666, 0, 0, 0, 0, 0, 4095, 12, 0, 0, 0, 0, 4077, 37, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 650, 2, 0, 0, 3200, 0, 0, 0, 0, 7, 0, 3877, 191, 1, 1, 0] rerouted_tokens_per_ep_rank=[4328, 4107, 4114, 4096, 4094, 4100, 3852, 4077] avg=4096.0000 max=4328 max_rank=0 min=3852 min_rank=6 imbalance_ratio=1.056641 max_load_reduction=28440 max_load_reduction_pct=0.867920 moved_to_echo=16123 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3776, 0, 666, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4054, 37, 3, 0, 0, 0, 0, 0, 4095, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 431, 0, 0, 0, 3200, 0, 0, 0, 0, 147, 0, 3872, 191, 1, 1, 0] rerouted_tokens_per_ep_rank=[4442, 4096, 4094, 4099, 4094, 4100, 3631, 4212] avg=4096.0000 max=4442 max_rank=0 min=3631 min_rank=6 imbalance_ratio=1.084473 max_load_reduction=28326 max_load_reduction_pct=0.864441 moved_to_echo=16037 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 2573, 0, 666, 0, 0, 0, 0, 0, 4095, 8, 0, 0, 0, 0, 3942, 37, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 1657, 5, 0, 0, 3200, 0, 0, 0, 0, 102, 0, 3988, 191, 1, 1, 0] rerouted_tokens_per_ep_rank=[3246, 4103, 3979, 4101, 4094, 4100, 4862, 4283] avg=4096.0000 max=4862 max_rank=6 min=3246 min_rank=0 imbalance_ratio=1.187012 max_load_reduction=27906 max_load_reduction_pct=0.851624 moved_to_echo=17339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 3785, 0, 668, 0, 0, 0, 0, 0, 4088, 5, 0, 0, 0, 0, 4093, 26, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 703, 12, 0, 0, 3206, 0, 0, 0, 1, 218, 0, 3469, 193, 8, 3, 0] rerouted_tokens_per_ep_rank=[4455, 4093, 4120, 4092, 4099, 4096, 3921, 3892] avg=4096.0000 max=4455 max_rank=0 min=3892 min_rank=7 imbalance_ratio=1.087646 max_load_reduction=28313 max_load_reduction_pct=0.864044 moved_to_echo=16008 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3452, 0, 666, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4094, 37, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 686, 6, 0, 0, 3200, 0, 0, 0, 0, 165, 0, 3882, 191, 1, 1, 0] rerouted_tokens_per_ep_rank=[4119, 4097, 4131, 4095, 4094, 4100, 3892, 4240] avg=4096.0000 max=4240 max_rank=7 min=3892 min_rank=6 imbalance_ratio=1.035156 max_load_reduction=28528 max_load_reduction_pct=0.870605 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 4, 3566, 0, 666, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 4088, 37, 2, 1, 0, 0, 0, 0, 4091, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 4, 4092, 0, 0, 0, 0, 707, 3, 0, 0, 3200, 0, 0, 0, 0, 349, 0, 3568, 191, 1, 1, 0] rerouted_tokens_per_ep_rank=[4239, 4096, 4128, 4094, 4095, 4096, 3910, 4110] avg=4096.0000 max=4239 max_rank=0 min=3910 min_rank=6 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16211 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 2, 2953, 4, 665, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 4023, 38, 0, 10, 0, 0, 0, 0, 4092, 2, 2, 0, 0, 0, 0, 0, 10, 4, 0, 3, 4082, 2, 0, 0, 0, 0, 4, 4074, 0, 0, 0, 0, 1876, 6, 11, 1, 3197, 0, 0, 0, 26, 1, 0, 3402, 190, 1, 1, 0] rerouted_tokens_per_ep_rank=[3629, 4081, 4071, 4096, 4101, 4078, 5091, 3621] avg=4096.0000 max=5091 max_rank=6 min=3621 min_rank=7 imbalance_ratio=1.242920 max_load_reduction=27677 max_load_reduction_pct=0.844635 moved_to_echo=16891 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 27520, 128, 5376, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7168, 128, 128, 128, 25728, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1152, 0, 30208, 1536, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 384, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1145, 0, 4096, 0, 0, 4096, 4096, 1322, 22, 0, 0, 0, 0, 4084, 1620] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1431, 0, 4095, 0, 0, 4096, 4096, 1164, 26, 0, 0, 0, 0, 4093, 1480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1085, 0, 4096, 0, 0, 4090, 4096, 884, 84, 0, 0, 0, 0, 4095, 2051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 520, 0, 4082, 0, 0, 4096, 4096, 1414, 47, 1, 0, 0, 0, 4071, 2154] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 121, 0, 4076, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 595, 0, 4093, 1, 0, 4087, 4096, 1291, 151, 0, 0, 1, 0, 4075, 1989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4086, 0, 4093, 0, 0, 0, 0, 0, 0, 4095, 1454, 0, 4087, 1, 0, 4092, 4096, 1465, 20, 3, 0, 0, 0, 4090, 1186] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 56, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 1141, 0, 4096, 0, 0, 4096, 4096, 2022, 314, 0, 0, 0, 0, 4094, 566] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[5, 0, 0, 4, 4, 17, 6, 4080, 0, 4089, 4, 0, 4, 0, 4, 3, 4092, 786, 5, 4092, 0, 1, 4031, 4090, 1684, 56, 0, 0, 0, 5, 4082, 1624] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 8, 0, 0, 0, 1145, 0, 3079, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 1322, 22, 0, 0, 1564, 1017, 18, 0, 0, 0, 2512, 1620, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4077, 4096, 4093, 4224, 4107, 3943, 4132] avg=4096.0000 max=4224 max_rank=4 min=3943 min_rank=6 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 9, 1, 0, 0, 1085, 0, 3079, 0, 0, 0, 0, 0, 0, 4090, 11, 0, 0, 0, 0, 884, 84, 0, 0, 1568, 1017, 18, 0, 0, 0, 2518, 2051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4077, 4096, 4095, 4164, 4101, 3571, 4569] avg=4096.0000 max=4569 max_rank=7 min=3571 min_rank=6 imbalance_ratio=1.115479 max_load_reduction=28199 max_load_reduction_pct=0.860565 moved_to_echo=16405 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4073, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 8, 0, 0, 0, 520, 0, 3069, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 1414, 47, 1, 0, 1559, 1013, 18, 0, 0, 0, 2504, 2154, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4077, 4096, 4093, 3589, 4107, 4052, 4658] avg=4096.0000 max=4658 max_rank=7 min=3589 min_rank=4 imbalance_ratio=1.137207 max_load_reduction=28110 max_load_reduction_pct=0.857849 moved_to_echo=16406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 121, 0, 4058, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 8, 1, 0, 0, 595, 0, 3077, 0, 0, 0, 0, 1, 0, 4087, 11, 0, 0, 0, 0, 1291, 151, 0, 0, 1560, 1016, 18, 0, 1, 0, 2507, 1989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4179, 4096, 4094, 3672, 4099, 4036, 4497] avg=4096.0000 max=4497 max_rank=7 min=3672 min_rank=4 imbalance_ratio=1.097900 max_load_reduction=28271 max_load_reduction_pct=0.862762 moved_to_echo=16304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4092, 13, 0, 0, 0, 1431, 0, 3075, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 1164, 26, 0, 0, 1572, 1020, 21, 0, 0, 0, 2508, 1480, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4067, 4103, 4105, 4506, 4100, 3803, 3988] avg=4096.0000 max=4506 max_rank=4 min=3803 min_rank=6 imbalance_ratio=1.100098 max_load_reduction=28262 max_load_reduction_pct=0.862488 moved_to_echo=16397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 56, 0, 4078, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 9, 1, 0, 0, 1141, 0, 3079, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 2022, 314, 0, 0, 1568, 1017, 18, 0, 0, 0, 2517, 566, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4134, 4095, 4095, 4220, 4107, 4939, 3083] avg=4096.0000 max=4939 max_rank=6 min=3083 min_rank=7 imbalance_ratio=1.205811 max_load_reduction=27829 max_load_reduction_pct=0.849274 moved_to_echo=16349 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 9, 0, 0, 0, 1454, 0, 3072, 0, 0, 0, 0, 1, 0, 4092, 11, 0, 0, 0, 0, 1465, 20, 3, 0, 1566, 1015, 18, 0, 0, 0, 2515, 1186, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4068, 4093, 4094, 4526, 4104, 4087, 3701] avg=4096.0000 max=4526 max_rank=4 min=3701 min_rank=7 imbalance_ratio=1.104980 max_load_reduction=28242 max_load_reduction_pct=0.861877 moved_to_echo=16418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 4, 4091, 0, 0, 0, 4, 17, 6, 4062, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 0, 0, 0, 4, 0, 4, 3, 4079, 8, 1, 0, 0, 786, 5, 3076, 0, 0, 0, 0, 0, 1, 4031, 11, 0, 0, 0, 0, 1684, 56, 0, 0, 1563, 1016, 18, 0, 0, 5, 2511, 1624, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4089, 4093, 4099, 3867, 4043, 4337, 4140] avg=4096.0000 max=4337 max_rank=6 min=3867 min_rank=4 imbalance_ratio=1.058838 max_load_reduction=28431 max_load_reduction_pct=0.867645 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11264, 768, 128, 0, 12544, 8192, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 20096, 12672, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 8192, 128, 24704, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 708, 0, 4087, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 33, 0, 14, 4095, 0, 4086, 0, 3362, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 408, 1, 4092, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 31, 0, 0, 4092, 0, 4095, 0, 3670, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 676, 0, 4081, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 5, 0, 8, 4095, 0, 4087, 0, 3435, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 520, 0, 4083, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 27, 0, 1, 4095, 0, 4094, 0, 3563, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 811, 0, 4090, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 47, 0, 6, 4096, 0, 4090, 0, 3255, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 821, 0, 4079, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 214, 0, 0, 4095, 0, 4094, 0, 3083, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4, 4, 5, 402, 0, 4089, 4092, 0, 3, 0, 0, 4088, 5, 4, 0, 4091, 4081, 0, 0, 0, 5, 6, 0, 2, 0, 1, 4090, 0, 4092, 2, 3702, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 999, 0, 4052, 4096, 0, 0, 2, 0, 4087, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 85, 0, 34, 4094, 0, 4074, 0, 3055, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 408, 3388, 42, 0, 0, 1, 4092, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 31, 4039, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 705, 0, 3670, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3838, 4108, 4099, 4096, 4095, 4070, 4087, 4375] avg=4096.0000 max=4375 max_rank=7 min=3838 min_rank=0 imbalance_ratio=1.068115 max_load_reduction=28393 max_load_reduction_pct=0.866486 moved_to_echo=16627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 676, 3382, 42, 0, 0, 0, 4081, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 5, 4039, 0, 0, 0, 0, 8, 4090, 0, 0, 0, 0, 0, 703, 0, 3435, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4096, 4099, 4096, 4097, 4044, 4098, 4138] avg=4096.0000 max=4138 max_rank=7 min=4044 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 811, 3384, 42, 0, 0, 0, 4090, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 47, 4039, 0, 0, 0, 0, 6, 4091, 0, 0, 0, 0, 0, 704, 0, 3255, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4237, 4105, 4093, 4096, 4095, 4086, 4097, 3959] avg=4096.0000 max=4237 max_rank=0 min=3959 min_rank=7 imbalance_ratio=1.034424 max_load_reduction=28531 max_load_reduction_pct=0.870697 moved_to_echo=16237 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 821, 3388, 42, 0, 0, 0, 4079, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 214, 4039, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 704, 0, 3083, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4251, 4094, 4099, 4096, 4098, 4253, 4090, 3787] avg=4096.0000 max=4253 max_rank=5 min=3787 min_rank=7 imbalance_ratio=1.038330 max_load_reduction=28515 max_load_reduction_pct=0.870209 moved_to_echo=16228 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 520, 3392, 44, 0, 0, 0, 4083, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 4, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 4, 0, 0, 0, 0, 0, 27, 4043, 0, 0, 0, 0, 1, 4082, 0, 0, 0, 0, 0, 698, 0, 3563, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3956, 4092, 4103, 4092, 4109, 4070, 4083, 4263] avg=4096.0000 max=4263 max_rank=7 min=3956 min_rank=0 imbalance_ratio=1.040771 max_load_reduction=28505 max_load_reduction_pct=0.869904 moved_to_echo=16525 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 708, 3381, 42, 0, 0, 0, 4087, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 2, 0, 0, 0, 0, 1, 33, 4039, 0, 0, 0, 0, 14, 4090, 0, 0, 0, 0, 0, 703, 0, 3362, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4102, 4089, 4096, 4100, 4073, 4104, 4073] avg=4096.0000 max=4131 max_rank=0 min=4073 min_rank=5 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4, 4, 5, 402, 3386, 42, 0, 0, 0, 4089, 15, 0, 0, 0, 0, 0, 3, 0, 0, 4088, 3, 0, 0, 0, 5, 4, 0, 4091, 0, 0, 0, 0, 4081, 0, 0, 0, 2, 2, 0, 0, 5, 6, 0, 2, 4035, 0, 0, 0, 0, 1, 4085, 0, 0, 0, 0, 0, 704, 2, 3702, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3843, 4104, 4094, 4100, 4085, 4048, 4086, 4408] avg=4096.0000 max=4408 max_rank=7 min=3843 min_rank=0 imbalance_ratio=1.076172 max_load_reduction=28360 max_load_reduction_pct=0.865479 moved_to_echo=16627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 999, 3371, 42, 0, 0, 0, 4052, 15, 0, 0, 0, 0, 0, 0, 2, 0, 4087, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 85, 4039, 0, 0, 0, 0, 34, 4089, 0, 0, 0, 0, 0, 701, 0, 3055, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4412, 4067, 4092, 4096, 4089, 4124, 4123, 3765] avg=4096.0000 max=4412 max_rank=0 min=3765 min_rank=7 imbalance_ratio=1.077148 max_load_reduction=28356 max_load_reduction_pct=0.865356 moved_to_echo=16101 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 5376, 27136, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5632, 128, 27136, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 512, 32384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2776, 15, 4095, 0, 0, 4096, 0, 0, 0, 17, 0, 4096, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 1289, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2459, 1, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1632, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2970, 3, 4096, 0, 0, 4079, 0, 0, 0, 79, 0, 4096, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 1060, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3784, 1, 4096, 0, 0, 4094, 0, 0, 0, 5, 0, 4096, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 310, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2039, 3, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2053, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3790, 0, 4096, 0, 0, 4085, 0, 0, 0, 14, 1, 4095, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 303, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4090, 5, 2, 1755, 54, 4089, 4, 6, 4091, 0, 5, 0, 5, 8, 4080, 1, 4083, 7, 6, 4091, 76, 0, 4091, 0, 7, 0, 2207, 0, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 5, 1414, 10, 4095, 0, 0, 4060, 0, 0, 0, 66, 2, 4096, 0, 4095, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 2642, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2776, 15, 1462, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 17, 0, 4073, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1289, 2633, 23, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4253, 4103, 4090, 4095, 4088, 3947, 4096] avg=4096.0000 max=4253 max_rank=1 min=3947 min_rank=6 imbalance_ratio=1.038330 max_load_reduction=28515 max_load_reduction_pct=0.870209 moved_to_echo=16226 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2970, 3, 1462, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 7, 0, 0, 0, 0, 79, 0, 4073, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 2634, 23, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4435, 4086, 4152, 4096, 4089, 3718, 4096] avg=4096.0000 max=4435 max_rank=1 min=3718 min_rank=6 imbalance_ratio=1.082764 max_load_reduction=28333 max_load_reduction_pct=0.864655 moved_to_echo=15999 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2459, 1, 1462, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 4, 0, 4073, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1632, 2634, 23, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3922, 4103, 4077, 4096, 4088, 4290, 4096] avg=4096.0000 max=4290 max_rank=6 min=3922 min_rank=1 imbalance_ratio=1.047363 max_load_reduction=28478 max_load_reduction_pct=0.869080 moved_to_echo=16570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2039, 3, 1462, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 3, 0, 4073, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2053, 2634, 23, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3504, 4103, 4076, 4094, 4088, 4711, 4096] avg=4096.0000 max=4711 max_rank=6 min=3504 min_rank=1 imbalance_ratio=1.150146 max_load_reduction=28057 max_load_reduction_pct=0.856232 moved_to_echo=16989 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 3784, 1, 1458, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 10, 1, 0, 0, 0, 5, 0, 4070, 0, 0, 0, 0, 0, 4094, 0, 0, 5, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 2638, 26, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 5243, 4105, 4075, 4099, 4083, 2977, 4096] avg=4096.0000 max=5243 max_rank=1 min=2977 min_rank=6 imbalance_ratio=1.280029 max_load_reduction=27525 max_load_reduction_pct=0.839996 moved_to_echo=15255 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 5, 0, 0, 0, 0, 1414, 10, 1462, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 7, 0, 0, 0, 0, 66, 2, 4073, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2642, 2633, 23, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 2886, 4067, 4141, 4095, 4084, 5298, 4096] avg=4096.0000 max=5298 max_rank=6 min=2886 min_rank=1 imbalance_ratio=1.293457 max_load_reduction=27470 max_load_reduction_pct=0.838318 moved_to_echo=17573 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 5, 2, 0, 0, 0, 0, 1755, 54, 1460, 4, 0, 0, 0, 0, 6, 4091, 0, 5, 7, 0, 0, 0, 0, 5, 8, 4057, 0, 0, 0, 0, 1, 4083, 7, 6, 0, 0, 0, 0, 4084, 76, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2207, 2629, 23, 0, 0, 0, 5, 0, 0, 4091, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3273, 4109, 4070, 4097, 4160, 4866, 4096] avg=4096.0000 max=4866 max_rank=6 min=3273 min_rank=1 imbalance_ratio=1.187988 max_load_reduction=27902 max_load_reduction_pct=0.851501 moved_to_echo=17219 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3790, 0, 1462, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 7, 0, 0, 0, 0, 14, 1, 4072, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 2634, 23, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 5252, 4092, 4087, 4096, 4088, 2961, 4096] avg=4096.0000 max=5252 max_rank=1 min=2961 min_rank=6 imbalance_ratio=1.282227 max_load_reduction=27516 max_load_reduction_pct=0.839722 moved_to_echo=15241 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[20992, 128, 11776, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 11520, 21120, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 256, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 4093, 0, 4096, 4058, 0, 0, 0, 0, 38, 0, 0, 1, 0, 0, 4096, 4096, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 4091, 0, 4096, 3680, 0, 0, 0, 0, 418, 0, 0, 3, 1, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4096, 3970, 0, 0, 0, 1, 126, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4084, 0, 4096, 3750, 0, 0, 0, 0, 348, 1, 0, 9, 5, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 4070, 0, 4096, 4039, 0, 0, 0, 0, 58, 26, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 4069, 0, 4096, 4054, 0, 0, 0, 0, 42, 20, 0, 14, 8, 0, 4096, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4086, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4, 4065, 0, 4090, 3841, 7, 6, 2, 0, 239, 0, 6, 6, 0, 7, 4089, 4090, 19, 4082, 0, 6, 7, 4089, 4, 8, 0, 0, 11, 0, 4086, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 4064, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 14, 0, 0, 0, 0, 4058, 0, 0, 0, 156, 5, 0, 0, 0, 38, 0, 0, 3926, 1, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4219, 3965, 4092, 4098, 4095, 4095, 4097] avg=4096.0000 max=4219 max_rank=1 min=3965 min_rank=2 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 14, 0, 0, 0, 0, 3970, 0, 0, 0, 156, 5, 0, 0, 1, 126, 0, 0, 3926, 1, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4131, 4054, 4091, 4096, 4095, 4095, 4097] avg=4096.0000 max=4131 max_rank=1 min=4054 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4070, 0, 14, 0, 0, 0, 0, 4039, 0, 0, 0, 156, 5, 0, 0, 0, 58, 26, 0, 3926, 1, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4200, 4011, 4091, 4096, 4095, 4094, 4097] avg=4096.0000 max=4200 max_rank=1 min=4011 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4069, 0, 14, 0, 0, 0, 0, 4054, 0, 0, 0, 156, 5, 0, 0, 0, 42, 20, 0, 3926, 1, 0, 0, 14, 8, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4086, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4215, 3989, 4113, 4096, 4095, 4090, 4087] avg=4096.0000 max=4215 max_rank=1 min=3989 min_rank=2 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 0, 3, 0, 0, 0, 0, 3680, 0, 0, 0, 163, 7, 0, 0, 0, 418, 0, 0, 3930, 1, 0, 0, 3, 1, 0, 4084, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 3, 7, 5, 0] rerouted_tokens_per_ep_rank=[4094, 3850, 4349, 4088, 4089, 4093, 4095, 4110] avg=4096.0000 max=4349 max_rank=2 min=3850 min_rank=1 imbalance_ratio=1.061768 max_load_reduction=28419 max_load_reduction_pct=0.867279 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 0, 14, 0, 0, 0, 0, 3750, 0, 0, 0, 156, 5, 0, 0, 0, 348, 1, 0, 3926, 1, 0, 0, 9, 5, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 3911, 4276, 4105, 4096, 4095, 4095, 4092] avg=4096.0000 max=4276 max_rank=2 min=3911 min_rank=1 imbalance_ratio=1.043945 max_load_reduction=28492 max_load_reduction_pct=0.869507 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 14, 0, 0, 0, 0, 4064, 0, 0, 0, 156, 5, 0, 0, 0, 32, 0, 0, 3926, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4225, 3958, 4091, 4096, 4095, 4096, 4097] avg=4096.0000 max=4225 max_rank=1 min=3958 min_rank=2 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 4065, 0, 14, 0, 0, 0, 0, 3841, 7, 6, 2, 156, 5, 0, 0, 0, 239, 0, 6, 3920, 1, 0, 0, 6, 0, 7, 4084, 0, 0, 0, 0, 0, 19, 4082, 0, 0, 0, 0, 0, 6, 7, 4088, 4, 0, 0, 0, 0, 8, 0, 0, 11, 4089, 0, 0, 0, 0, 4086, 4, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4017, 4166, 4097, 4101, 4105, 4108, 4091] avg=4096.0000 max=4166 max_rank=2 min=4017 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16405 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[31488, 128, 128, 128, 1280, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 1408, 128, 128, 31488, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4007, 81, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3804, 291, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 4096, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 2776, 1318, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0, 4095, 4096, 0, 4, 0, 0, 1, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3600, 492, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 4096, 0, 0, 4096, 4096, 0, 18, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3281, 797, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 4096, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4009, 82, 0, 0, 0, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[8, 7, 31, 4086, 2, 4088, 0, 5, 4090, 4090, 6, 9, 0, 0, 0, 0, 4089, 0, 6, 4088, 4084, 0, 0, 6, 3757, 303, 0, 7, 0, 0, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4094, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3879, 217, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3600, 492, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4094, 4099, 4096, 4096, 4099, 4094] avg=4096.0000 max=4099 max_rank=3 min=4094 min_rank=0 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4007, 81, 0, 0, 5, 2, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4099, 4096, 4096, 4096, 4096, 4095] avg=4096.0000 max=4099 max_rank=2 min=4094 min_rank=0 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3804, 291, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4092, 4096, 4096, 4096, 4102, 4096] avg=4096.0000 max=4102 max_rank=6 min=4092 min_rank=2 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16390 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4009, 82, 0, 0, 5, 2, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 4092, 4096, 4096, 4096, 4098, 4101] avg=4096.0000 max=4101 max_rank=7 min=4092 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4079, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 4088, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 2776, 1318, 0, 0, 8, 7, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4099, 4089, 4099, 4096, 4102, 4109, 4095] avg=4096.0000 max=4109 max_rank=6 min=4079 min_rank=0 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3281, 797, 0, 0, 5, 2, 2, 0, 0, 0, 4, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4096, 4109, 4096, 4096, 4096, 4087, 4098] avg=4096.0000 max=4109 max_rank=2 min=4087 min_rank=6 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3879, 217, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4094, 4093, 4096, 4098, 4096, 4103, 4094] avg=4096.0000 max=4103 max_rank=6 min=4093 min_rank=2 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[8, 7, 31, 4084, 0, 0, 0, 0, 2, 4088, 0, 5, 0, 0, 0, 0, 4085, 0, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 6, 4088, 0, 0, 0, 0, 4084, 0, 0, 6, 0, 0, 0, 0, 3757, 303, 0, 7, 5, 2, 0, 0, 0, 0, 0, 6, 4089, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4095, 4100, 4090, 4094, 4090, 4074, 4095] avg=4096.0000 max=4130 max_rank=0 min=4074 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[29184, 3584, 0, 128, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 14, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 42, 0, 4096, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 4096, 0, 0, 0, 16, 0, 0, 0, 0, 10, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 23, 0, 4096, 0, 0, 0, 4084, 4095, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 3943, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 141, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4092, 7, 0, 4096, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 4096, 0, 0, 0, 90, 0, 0, 0, 0, 50, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 40, 0, 4096, 0, 0, 0, 4079, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 4096, 0, 0, 0, 109, 0, 0, 0, 0, 2, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 11, 0, 4096, 0, 0, 0, 4093, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 85, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4058, 0, 4, 4088, 0, 6, 6, 4091, 4079, 4088, 8, 7, 5, 0, 0, 0, 0, 8, 4091, 4090, 4, 5, 0, 19, 27, 0, 0, 4, 0, 0, 4080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4085, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4065, 40, 30, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4066, 0, 0, 0, 0, 14, 0, 4056, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4096, 4096, 4096, 4146, 4096, 4081, 4070] avg=4096.0000 max=4146 max_rank=4 min=4070 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4085, 42, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4030, 40, 29, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4061, 1, 0, 0, 0, 10, 0, 4056, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4096, 4100, 4095, 4110, 4096, 4078, 4066] avg=4096.0000 max=4127 max_rank=0 min=4066 min_rank=7 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4081, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3955, 40, 29, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 4061, 1, 0, 0, 0, 50, 0, 4056, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4096, 4100, 4095, 4035, 4096, 4152, 4106] avg=4096.0000 max=4152 max_rank=6 min=4035 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4085, 40, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4079, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3962, 40, 29, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 4061, 1, 0, 0, 0, 2, 0, 4056, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4096, 4085, 4095, 4042, 4096, 4171, 4058] avg=4096.0000 max=4171 max_rank=6 min=4042 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 11, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3995, 42, 35, 12, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4061, 0, 0, 0, 0, 85, 0, 4054, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4092, 4093, 4096, 4088, 4096, 4069, 4139] avg=4096.0000 max=4139 max_rank=7 min=4069 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4085, 23, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4084, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 3943, 40, 29, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4062, 0, 0, 0, 0, 141, 0, 4056, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4096, 4088, 4097, 4023, 4096, 4063, 4197] avg=4096.0000 max=4197 max_rank=7 min=4023 min_rank=4 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4048, 0, 4, 0, 0, 0, 0, 4088, 0, 6, 6, 0, 0, 0, 0, 4091, 5, 0, 8, 0, 0, 0, 0, 7, 5, 0, 0, 4087, 0, 0, 0, 0, 0, 8, 4091, 40, 29, 10, 0, 4090, 4, 5, 0, 0, 0, 0, 0, 19, 27, 0, 0, 4045, 1, 0, 0, 4, 0, 0, 4040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4100, 4104, 4099, 4178, 4099, 4092, 4044] avg=4096.0000 max=4178 max_rank=4 min=4044 min_rank=7 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4085, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4063, 40, 29, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 1, 0, 0, 0, 34, 0, 4054, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4096, 4102, 4095, 4143, 4096, 4062, 4088] avg=4096.0000 max=4143 max_rank=4 min=4062 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32765, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32765, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32765, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32765, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32765, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32765, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32765, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32765, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 384, 0, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 0, 0, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32765 actual_valid_recv=33152 observed_loss=-387 tokens_per_expert=[0, 0, 128, 32128, 384, 256, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32640, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32771 actual_valid_recv=33024 observed_loss=-253 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 806, 0, 4096, 0, 1, 0, 0, 0, 0, 4094, 3273, 0, 0, 20, 0, 4096, 3, 0, 0, 0, 4095, 1, 0, 0, 4095, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 722, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3343, 0, 0, 32, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4085, 0, 1379, 0, 4095, 1, 0, 0, 0, 0, 0, 4096, 2701, 0, 0, 25, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4070, 0, 507, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3562, 0, 0, 43, 0, 4096, 10, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4066, 0, 648, 0, 4095, 0, 1, 0, 0, 0, 0, 4087, 3300, 0, 0, 169, 0, 4096, 10, 0, 0, 0, 4096, 2, 0, 7, 4094, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 0, 558, 0, 4001, 1, 0, 0, 0, 0, 0, 4096, 3624, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4, 0, 48, 10, 4043, 5, 704, 7, 4058, 4, 0, 6, 4, 5, 7, 4080, 3412, 0, 6, 0, 22, 4083, 0, 0, 0, 0, 4081, 0, 0, 0, 4091, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4089, 0, 1105, 0, 4094, 0, 0, 0, 0, 0, 0, 4086, 2757, 0, 0, 236, 0, 4096, 12, 0, 0, 0, 4096, 0, 0, 4, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 0, 3297, 0, 1379, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 13, 2, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 2701, 0, 0, 25, 775, 7, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4676, 4111, 4098, 3508, 4092, 4098, 4096] avg=4096.0000 max=4676 max_rank=1 min=3508 min_rank=4 imbalance_ratio=1.141602 max_load_reduction=28092 max_load_reduction_pct=0.857300 moved_to_echo=15794 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 0, 3305, 0, 722, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 2, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 3343, 0, 0, 32, 777, 7, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4027, 4111, 4098, 4159, 4092, 4096, 4096] avg=4096.0000 max=4159 max_rank=4 min=4027 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16443 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 0, 3285, 0, 507, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 2, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 3562, 0, 0, 43, 772, 7, 0, 0, 0, 4092, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 3792, 4111, 4098, 4384, 4102, 4096, 4096] avg=4096.0000 max=4384 max_rank=4 min=3792 min_rank=1 imbalance_ratio=1.070312 max_load_reduction=28384 max_load_reduction_pct=0.866211 moved_to_echo=16678 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 0, 3297, 0, 806, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 15, 3, 0, 0, 0, 0, 0, 4094, 7, 0, 0, 0, 3273, 0, 0, 20, 780, 7, 0, 0, 0, 4080, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 6, 3, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4103, 4115, 4101, 4080, 4083, 4105, 4092] avg=4096.0000 max=4115 max_rank=2 min=4080 min_rank=4 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16360 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 0, 0, 0, 3298, 0, 558, 0, 0, 0, 0, 0, 4001, 1, 0, 0, 13, 2, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 3624, 0, 0, 16, 775, 8, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 3856, 4017, 4098, 4423, 4092, 4098, 4096] avg=4096.0000 max=4423 max_rank=4 min=3856 min_rank=1 imbalance_ratio=1.079834 max_load_reduction=28345 max_load_reduction_pct=0.865021 moved_to_echo=16709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4089, 0, 0, 0, 3300, 0, 1105, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 13, 2, 0, 0, 0, 0, 0, 4086, 2, 0, 0, 0, 2757, 0, 0, 236, 776, 7, 0, 0, 0, 4092, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4405, 4109, 4088, 3776, 4104, 4096, 4100] avg=4096.0000 max=4405 max_rank=1 min=3776 min_rank=4 imbalance_ratio=1.075439 max_load_reduction=28363 max_load_reduction_pct=0.865570 moved_to_echo=16076 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4089, 0, 0, 0, 3282, 0, 648, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 13, 2, 0, 0, 0, 0, 0, 4087, 2, 0, 0, 0, 3300, 0, 0, 169, 771, 7, 0, 0, 0, 4092, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 7, 4094, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 3930, 4111, 4089, 4247, 4102, 4098, 4101] avg=4096.0000 max=4247 max_rank=4 min=3930 min_rank=1 imbalance_ratio=1.036865 max_load_reduction=28521 max_load_reduction_pct=0.870392 moved_to_echo=16548 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 48, 10, 4082, 0, 0, 0, 3263, 5, 704, 7, 0, 0, 0, 0, 4058, 4, 0, 6, 13, 2, 0, 0, 4, 5, 7, 4080, 2, 0, 0, 0, 3412, 0, 6, 0, 767, 6, 0, 0, 22, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 3979, 4083, 4098, 4191, 4101, 4081, 4091] avg=4096.0000 max=4191 max_rank=4 min=3979 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[25984, 0, 128, 640, 6272, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[26368, 128, 6528, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 4072, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 103, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4096, 0, 0, 4096, 3993, 0, 1, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4025, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3930, 0, 13, 0, 4096, 4091, 4096, 0, 0, 0, 0, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 67, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 4096, 0, 0, 4096, 4032, 0, 5, 0, 4096, 4092, 4096, 0, 0, 0, 0, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4058, 0, 1, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 4096, 4083, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[11, 6, 4063, 230, 5, 0, 0, 0, 0, 0, 0, 7, 0, 136, 10, 11, 4081, 8, 0, 4088, 3674, 0, 161, 4, 4090, 4088, 3994, 0, 2, 0, 8, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4011, 22, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 2, 0, 0, 4077, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4072, 0, 0, 0, 84, 5, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4096, 4096, 4079, 4097, 4161, 4109, 4097] avg=4096.0000 max=4161 max_rank=5 min=4033 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4011, 38, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4058, 0, 1, 0, 84, 5, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4096, 4096, 4076, 4097, 4148, 4109, 4097] avg=4096.0000 max=4148 max_rank=5 min=4049 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16451 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4011, 71, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 1, 0, 0, 4077, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4025, 0, 0, 0, 84, 5, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4096, 4096, 4078, 4097, 4114, 4109, 4096] avg=4096.0000 max=4114 max_rank=5 min=4078 min_rank=3 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16416 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4011, 166, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 3930, 0, 13, 0, 84, 4, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4177, 4096, 4096, 4072, 4097, 4031, 4110, 4089] avg=4096.0000 max=4177 max_rank=0 min=4031 min_rank=5 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16327 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4001, 103, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 1, 0, 0, 1, 4077, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 1, 0, 89, 8, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4097, 4097, 4079, 4096, 4091, 4104, 4100] avg=4096.0000 max=4104 max_rank=0 min=4079 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4011, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 5, 0, 0, 4077, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4083, 0, 0, 0, 84, 5, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4096, 4096, 4082, 4097, 4172, 4109, 4097] avg=4096.0000 max=4172 max_rank=5 min=4019 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16475 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4011, 67, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 1, 2, 0, 0, 4073, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4032, 0, 5, 0, 84, 4, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4096, 4096, 4076, 4097, 4125, 4110, 4090] avg=4096.0000 max=4125 max_rank=5 min=4076 min_rank=3 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[11, 6, 3979, 230, 0, 0, 0, 0, 5, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 7, 4087, 1, 0, 0, 0, 136, 10, 11, 4069, 0, 0, 0, 4081, 8, 0, 1, 0, 0, 0, 0, 3674, 0, 161, 4, 83, 4, 0, 0, 0, 14, 3994, 0, 0, 0, 0, 0, 2, 0, 8, 4091, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4226, 4095, 4095, 4226, 4090, 3926, 4008, 4102] avg=4096.0000 max=4226 max_rank=0 min=3926 min_rank=5 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16126 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32128, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[31872, 0, 256, 128, 768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4095, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4094, 4096, 0, 0, 4096, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4091, 4096, 0, 0, 4096, 4, 4096, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4088, 4096, 0, 0, 4096, 8, 4096, 0, 10, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4040, 4079, 8, 0, 0, 4072, 4087, 3856, 2, 4, 4088, 0, 4088, 12, 0, 4089, 9, 0, 7, 264, 4, 0, 4, 7, 0, 8, 6, 13, 0, 2, 2, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4095, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4061, 1, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4127, 4095, 4095, 4064, 4095, 4094, 4096] avg=4096.0000 max=4127 max_rank=1 min=4064 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 1, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4127, 4095, 4095, 4063, 4095, 4094, 4096] avg=4096.0000 max=4127 max_rank=1 min=4063 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16348 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 1, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4127, 4095, 4097, 4061, 4095, 4094, 4096] avg=4096.0000 max=4127 max_rank=1 min=4061 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16346 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 0, 0, 0, 0, 0, 0, 4095, 8, 0, 0, 0, 0, 0, 0, 10, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4127, 4103, 4096, 4054, 4095, 4093, 4097] avg=4096.0000 max=4127 max_rank=1 min=4054 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16339 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 3, 4, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4119, 4093, 4093, 4072, 4095, 4103, 4096] avg=4096.0000 max=4119 max_rank=1 min=4072 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16366 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 1, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4127, 4099, 4096, 4057, 4096, 4094, 4096] avg=4096.0000 max=4127 max_rank=1 min=4057 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16343 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4061, 1, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4127, 4095, 4095, 4064, 4095, 4093, 4097] avg=4096.0000 max=4127 max_rank=1 min=4064 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4040, 7, 8, 0, 0, 0, 0, 0, 0, 0, 31, 3856, 0, 0, 0, 0, 2, 4, 4087, 0, 0, 0, 0, 0, 0, 12, 0, 4089, 0, 0, 0, 0, 9, 0, 7, 264, 4052, 1, 0, 0, 4, 0, 4, 7, 4087, 0, 0, 0, 0, 8, 6, 13, 4072, 4, 0, 0, 0, 2, 2, 17, 4072, 1, 0, 0] rerouted_tokens_per_ep_rank=[4055, 3887, 4093, 4101, 4333, 4102, 4103, 4094] avg=4096.0000 max=4333 max_rank=4 min=3887 min_rank=1 imbalance_ratio=1.057861 max_load_reduction=28435 max_load_reduction_pct=0.867767 moved_to_echo=16632 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 384, 32512, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 256, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 117, 0, 0, 4067, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 4096, 4096, 0, 0, 4004, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 131, 0, 0, 4089, 0, 0, 4096, 1, 0, 0, 4096, 0, 6, 4096, 4096, 0, 0, 3965, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 46, 0, 0, 4089, 0, 1, 4096, 5, 0, 0, 4096, 0, 1, 4096, 4096, 0, 0, 4051, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 0, 4089, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4081, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 147, 0, 0, 4076, 0, 1, 4095, 0, 0, 0, 4096, 0, 18, 4096, 4096, 0, 0, 3951, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 183, 0, 0, 4077, 0, 5, 4092, 5, 0, 0, 4096, 0, 7, 4096, 4096, 0, 0, 3920, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 4089, 0, 0, 4096, 3, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4081, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[66, 9, 11, 8, 27, 0, 0, 4087, 0, 8, 643, 4, 9, 3480, 0, 0, 4088, 0, 0, 0, 4077, 0, 0, 3994, 4083, 5, 0, 4091, 7, 4065, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 117, 0, 3931, 1, 0, 0, 0, 4067, 0, 0, 86, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4, 4096, 0, 0, 0, 0, 79, 0, 0, 4004, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4095, 4049, 4153, 4096, 4109, 4083, 4097] avg=4096.0000 max=4153 max_rank=3 min=4049 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 131, 0, 3931, 1, 0, 0, 0, 4089, 0, 0, 86, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 9, 0, 6, 4096, 0, 0, 0, 0, 79, 0, 0, 3965, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4095, 4063, 4175, 4097, 4111, 4044, 4097] avg=4096.0000 max=4175 max_rank=3 min=4044 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 20, 0, 3931, 1, 0, 0, 0, 4089, 0, 0, 86, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 79, 0, 0, 4081, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4095, 3952, 4175, 4098, 4105, 4160, 4097] avg=4096.0000 max=4175 max_rank=3 min=3952 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 147, 0, 3931, 1, 0, 0, 0, 4076, 0, 1, 86, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 9, 0, 18, 4096, 0, 0, 0, 0, 79, 0, 0, 3951, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4095, 4079, 4163, 4095, 4123, 4030, 4097] avg=4096.0000 max=4163 max_rank=3 min=4030 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 46, 0, 3936, 2, 0, 0, 0, 4089, 0, 1, 93, 1, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 3, 0, 1, 4096, 0, 0, 0, 0, 67, 0, 0, 4051, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 3, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4086, 3984, 4184, 4098, 4100, 4118, 4107] avg=4096.0000 max=4184 max_rank=3 min=3984 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 19, 0, 3931, 1, 0, 0, 0, 4089, 0, 0, 86, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 79, 0, 0, 4081, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4095, 3951, 4175, 4099, 4105, 4160, 4097] avg=4096.0000 max=4175 max_rank=3 min=3951 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[66, 9, 11, 8, 4067, 0, 0, 0, 27, 0, 0, 4086, 0, 0, 0, 0, 0, 8, 643, 4, 3919, 1, 0, 0, 9, 3480, 0, 0, 86, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3994, 0, 0, 0, 0, 78, 5, 0, 4091, 0, 0, 0, 0, 7, 4065, 6, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4113, 4575, 3575, 4088, 4003, 4174, 4079] avg=4096.0000 max=4575 max_rank=2 min=3575 min_rank=3 imbalance_ratio=1.116943 max_load_reduction=28193 max_load_reduction_pct=0.860382 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 183, 0, 3931, 1, 0, 0, 0, 4077, 0, 5, 86, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 9, 0, 7, 4096, 0, 0, 0, 0, 79, 0, 0, 3920, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4094, 4115, 4168, 4097, 4112, 3999, 4097] avg=4096.0000 max=4168 max_rank=3 min=3999 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 1408, 128, 31488, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32128, 0, 128, 768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[640, 128, 0, 32256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 8, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 2, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[317, 0, 0, 7, 8, 0, 55, 9, 4089, 16, 3636, 17, 0, 0, 0, 8, 4073, 4080, 4072, 9, 7, 9, 0, 13, 4036, 0, 5, 4087, 0, 108, 189, 3918] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 10, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 5, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4042, 14, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 54, 0, 4088, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4088, 4150, 4097, 4096, 4096, 4103, 4082] avg=4096.0000 max=4150 max_rank=2 min=4056 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4042, 14, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 54, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4088, 4149, 4097, 4099, 4092, 4104, 4082] avg=4096.0000 max=4149 max_rank=2 min=4057 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4042, 14, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 54, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4088, 4150, 4097, 4099, 4092, 4104, 4082] avg=4096.0000 max=4150 max_rank=2 min=4056 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4031, 14, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 54, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4088, 4150, 4096, 4099, 4102, 4104, 4082] avg=4096.0000 max=4150 max_rank=2 min=4047 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4044, 21, 3, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 51, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4089, 4147, 4098, 4096, 4099, 4096, 4075] avg=4096.0000 max=4147 max_rank=2 min=4068 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16366 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4042, 14, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 54, 0, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4088, 4149, 4097, 4099, 4093, 4104, 4082] avg=4096.0000 max=4149 max_rank=2 min=4056 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[317, 0, 0, 7, 4035, 14, 0, 0, 8, 0, 55, 9, 4079, 0, 0, 0, 54, 16, 3636, 17, 0, 0, 0, 0, 0, 0, 0, 8, 4080, 1, 0, 0, 2, 0, 4072, 9, 0, 0, 0, 0, 7, 9, 0, 13, 4070, 0, 0, 0, 4036, 0, 5, 8, 0, 0, 0, 0, 0, 108, 189, 3904, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4373, 4151, 3723, 4089, 4083, 4099, 4049, 4201] avg=4096.0000 max=4373 max_rank=0 min=3723 min_rank=2 imbalance_ratio=1.067627 max_load_reduction=28395 max_load_reduction_pct=0.866547 moved_to_echo=16432 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4037, 14, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 54, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4088, 4150, 4097, 4097, 4095, 4104, 4082] avg=4096.0000 max=4150 max_rank=2 min=4055 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 0, 128, 128, 32384, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 256, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 128, 32384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 3, 0, 344, 4044, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3797, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 478, 4009, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 22, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3683, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 0, 815, 3767, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 30, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 3578, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 2, 0, 279, 3951, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3959, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4, 0, 59, 4033, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 76, 0, 0, 0, 4096, 4096, 0, 0, 3, 0, 4017, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[9, 67, 8, 3495, 4062, 8, 0, 4078, 7, 3, 9, 8, 4087, 12, 0, 4, 4084, 0, 21, 561, 9, 0, 0, 4082, 3800, 0, 0, 13, 6, 249, 0, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 621, 4013, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 7, 0, 3550, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 0, 618, 4026, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 31, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3516, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 344, 3188, 58, 0, 0, 4044, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 105, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 33, 0, 0, 0, 0, 3797, 0, 803, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3593, 4151, 4094, 4096, 4009, 4096, 4129, 4600] avg=4096.0000 max=4600 max_rank=7 min=3593 min_rank=0 imbalance_ratio=1.123047 max_load_reduction=28168 max_load_reduction_pct=0.859619 moved_to_echo=16834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 478, 3188, 58, 0, 0, 4009, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 105, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 33, 0, 0, 0, 0, 3683, 0, 803, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3724, 4116, 4094, 4096, 4027, 4096, 4129, 4486] avg=4096.0000 max=4486 max_rank=7 min=3724 min_rank=0 imbalance_ratio=1.095215 max_load_reduction=28282 max_load_reduction_pct=0.863098 moved_to_echo=16738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 279, 3188, 58, 0, 0, 3951, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 105, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 33, 0, 0, 0, 0, 3959, 0, 803, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3527, 4058, 4094, 4096, 4006, 4096, 4129, 4762] avg=4096.0000 max=4762 max_rank=7 min=3527 min_rank=0 imbalance_ratio=1.162598 max_load_reduction=28006 max_load_reduction_pct=0.854675 moved_to_echo=16993 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 815, 3191, 60, 0, 0, 3767, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 111, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 4096, 0, 0, 1, 39, 2, 0, 0, 0, 3578, 0, 794, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 3873, 4101, 4089, 4027, 4101, 4138, 4372] avg=4096.0000 max=4372 max_rank=7 min=3873 min_rank=1 imbalance_ratio=1.067383 max_load_reduction=28396 max_load_reduction_pct=0.866577 moved_to_echo=16638 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 621, 3188, 58, 0, 0, 4013, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 105, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 7, 33, 0, 0, 0, 0, 3550, 0, 803, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3867, 4120, 4094, 4096, 4006, 4096, 4136, 4353] avg=4096.0000 max=4353 max_rank=7 min=3867 min_rank=0 imbalance_ratio=1.062744 max_load_reduction=28415 max_load_reduction_pct=0.867157 moved_to_echo=16591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 618, 3188, 58, 0, 0, 4026, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 105, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 33, 0, 0, 0, 0, 3516, 0, 803, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3865, 4133, 4094, 4096, 4036, 4096, 4129, 4319] avg=4096.0000 max=4319 max_rank=7 min=3865 min_rank=0 imbalance_ratio=1.054443 max_load_reduction=28449 max_load_reduction_pct=0.868195 moved_to_echo=16580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 59, 3188, 58, 0, 0, 4033, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 105, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 3, 33, 0, 0, 0, 0, 4017, 0, 803, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3309, 4140, 4094, 4096, 4081, 4096, 4132, 4820] avg=4096.0000 max=4820 max_rank=7 min=3309 min_rank=0 imbalance_ratio=1.176758 max_load_reduction=27948 max_load_reduction_pct=0.852905 moved_to_echo=17129 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[9, 67, 8, 3495, 3180, 57, 0, 0, 4062, 8, 0, 107, 0, 0, 0, 0, 7, 3, 9, 8, 3971, 105, 0, 0, 4087, 12, 0, 4, 0, 0, 0, 0, 3994, 0, 21, 561, 0, 0, 0, 0, 9, 0, 0, 4082, 0, 0, 0, 0, 3800, 0, 0, 13, 33, 0, 0, 0, 6, 249, 0, 801, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6816, 4177, 4103, 4103, 4576, 4091, 3846, 1056] avg=4096.0000 max=6816 max_rank=0 min=1056 min_rank=7 imbalance_ratio=1.664062 max_load_reduction=25952 max_load_reduction_pct=0.791992 moved_to_echo=13569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 6784, 25600, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 26368, 0, 6528, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32000, 128, 0, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32128, 0, 128, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 0, 0, 128, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32000, 896, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4093, 4095, 1, 4096, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 4089, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 1, 4096, 0, 0, 4096, 2, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 4090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4095, 4094, 1, 4096, 0, 0, 4096, 11, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4085, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4083, 4089, 0, 4096, 0, 0, 4094, 6, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4071, 4075, 0, 4096, 0, 0, 4094, 23, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4087, 4090, 7, 4096, 0, 0, 4096, 9, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 0, 0, 0, 4078, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 2, 4096, 0, 0, 4096, 2, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4080, 4089, 65, 4039, 10, 14, 4083, 262, 6, 0, 0, 4085, 9, 3889, 0, 11, 0, 7, 12, 0, 0, 11, 0, 0, 0, 0, 4087, 8, 0, 13, 3988, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4056, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 1, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 7, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4059, 4096, 4120, 4098, 4099, 4096, 4106] avg=4096.0000 max=4120 max_rank=3 min=4059 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16399 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4056, 2, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 1, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4058, 4095, 4120, 4099, 4096, 4096, 4107] avg=4096.0000 max=4120 max_rank=3 min=4058 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16398 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 1, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 16, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4060, 4096, 4120, 4092, 4108, 4096, 4113] avg=4096.0000 max=4120 max_rank=3 min=4060 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16409 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 23, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 23, 1, 0, 0, 0, 0, 0, 0, 4075, 3, 0, 0, 0, 0, 31, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4077, 4096, 4114, 4078, 4123, 4096, 4113] avg=4096.0000 max=4123 max_rank=5 min=4071 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16410 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4051, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 4096, 0, 0, 26, 1, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 2, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4085, 0, 19, 2, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4062, 4103, 4123, 4097, 4094, 4097, 4106] avg=4096.0000 max=4123 max_rank=3 min=4062 min_rank=1 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16394 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4056, 2, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4058, 4095, 4119, 4099, 4093, 4096, 4110] avg=4096.0000 max=4119 max_rank=3 min=4058 min_rank=1 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16398 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4056, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 1, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 0, 0, 17, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4065, 4096, 4120, 4093, 4109, 4096, 4095] avg=4096.0000 max=4120 max_rank=3 min=4065 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16393 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4080, 0, 65, 0, 0, 0, 0, 0, 10, 14, 4043, 262, 0, 0, 0, 0, 6, 0, 0, 4085, 0, 0, 0, 0, 9, 3889, 0, 11, 23, 0, 0, 0, 0, 7, 12, 0, 4089, 4, 0, 0, 0, 11, 0, 0, 4035, 0, 0, 0, 0, 0, 4087, 8, 0, 0, 0, 0, 0, 13, 3988, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4329, 4091, 3932, 4112, 4046, 4095, 4018] avg=4096.0000 max=4329 max_rank=1 min=3932 min_rank=3 imbalance_ratio=1.056885 max_load_reduction=28439 max_load_reduction_pct=0.867889 moved_to_echo=16271 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32640, 0, 128, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 32640, 0, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32512, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-08-30 04:56:48] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 644.6 | throughput per GPU (TFLOP/s/GPU): 517.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.254466E+00 | loss scale: 1.0 | grad norm: 5.709 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[449, 0, 1511, 61, 1202, 0, 2442, 29, 1687, 1627, 0, 30, 1092, 3316, 8, 850, 35, 1642, 1639, 0, 0, 784, 1425, 163, 0, 3221, 3009, 1646, 2166, 2614, 120, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[806, 0, 2000, 42, 884, 0, 2520, 77, 1204, 1318, 0, 19, 1153, 2538, 25, 624, 5, 1766, 2564, 0, 2, 819, 1463, 119, 1, 3122, 3051, 1528, 2900, 2016, 201, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[718, 0, 1467, 30, 1577, 0, 2678, 23, 1164, 1643, 0, 39, 1076, 2781, 16, 1116, 10, 1420, 2089, 0, 0, 1004, 1659, 131, 0, 2860, 2913, 1512, 2670, 2058, 114, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[970, 0, 2042, 12, 1081, 0, 2774, 53, 585, 2255, 0, 22, 1517, 2764, 7, 730, 18, 1323, 1894, 0, 0, 944, 1858, 208, 0, 2786, 3364, 1345, 2438, 1690, 88, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1151, 0, 1403, 103, 1717, 1, 2224, 47, 1688, 1281, 0, 31, 944, 2877, 7, 764, 9, 2140, 1783, 1, 0, 789, 1680, 253, 1, 2785, 3093, 1716, 1977, 2256, 47, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1093, 0, 1421, 63, 1013, 0, 2322, 100, 1392, 1980, 1, 49, 1404, 3131, 13, 899, 21, 1487, 1670, 0, 2, 1585, 1342, 124, 0, 2792, 3311, 1484, 1805, 2200, 64, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[554, 0, 1900, 96, 1398, 0, 1731, 71, 1176, 1392, 0, 69, 1633, 2930, 12, 702, 17, 1227, 2184, 0, 0, 1023, 1888, 441, 0, 2920, 3069, 1485, 2892, 1817, 141, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1058, 0, 1704, 103, 1520, 0, 2240, 65, 1533, 1754, 1, 63, 1656, 2333, 21, 894, 74, 1641, 1909, 1, 1, 844, 1712, 358, 0, 2894, 2979, 1471, 2161, 1734, 44, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[449, 0, 1511, 61, 1457, 0, 0, 0, 1202, 0, 2442, 29, 385, 20, 0, 0, 1687, 1627, 0, 30, 897, 321, 0, 0, 1092, 2283, 8, 850, 0, 0, 0, 0, 35, 1642, 1639, 0, 372, 136, 0, 0, 0, 784, 1425, 163, 1231, 0, 0, 0, 0, 2836, 0, 1646, 0, 0, 0, 0, 1774, 2614, 120, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3478, 4078, 4562, 4233, 3824, 3603, 4482, 4508] avg=4096.0000 max=4562 max_rank=2 min=3478 min_rank=0 imbalance_ratio=1.113770 max_load_reduction=28206 max_load_reduction_pct=0.860779 moved_to_echo=16417 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[718, 0, 1467, 30, 1411, 0, 0, 0, 1577, 0, 2678, 23, 342, 25, 0, 0, 1164, 1643, 0, 39, 752, 310, 0, 0, 1076, 1915, 16, 1116, 0, 0, 0, 0, 10, 1420, 2089, 0, 459, 114, 0, 0, 0, 1004, 1659, 131, 1192, 0, 0, 0, 0, 2518, 0, 1512, 0, 0, 0, 0, 2186, 2058, 114, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3626, 4645, 3908, 4123, 4092, 3986, 4030, 4358] avg=4096.0000 max=4645 max_rank=1 min=3626 min_rank=0 imbalance_ratio=1.134033 max_load_reduction=28123 max_load_reduction_pct=0.858246 moved_to_echo=16466 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[970, 0, 2042, 12, 1630, 0, 0, 0, 1081, 0, 2774, 53, 333, 22, 0, 0, 585, 2255, 0, 22, 747, 357, 0, 0, 1517, 1904, 7, 730, 0, 0, 0, 0, 18, 1323, 1894, 0, 419, 113, 0, 0, 0, 944, 1858, 208, 1377, 0, 0, 0, 0, 2453, 0, 1345, 0, 0, 0, 0, 1997, 1690, 88, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4654, 4263, 3966, 4158, 3767, 4387, 3798, 3775] avg=4096.0000 max=4654 max_rank=0 min=3767 min_rank=4 imbalance_ratio=1.136230 max_load_reduction=28114 max_load_reduction_pct=0.857971 moved_to_echo=15727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[554, 0, 1900, 96, 1487, 0, 0, 0, 1398, 0, 1731, 71, 349, 27, 0, 0, 1176, 1392, 0, 69, 792, 325, 0, 0, 1633, 2018, 12, 702, 0, 0, 0, 0, 17, 1227, 2184, 0, 497, 120, 0, 0, 0, 1023, 1888, 441, 1257, 0, 0, 0, 0, 2571, 0, 1485, 0, 0, 0, 0, 2368, 1817, 141, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 3576, 3754, 4365, 4045, 4609, 4056, 4326] avg=4096.0000 max=4609 max_rank=5 min=3576 min_rank=1 imbalance_ratio=1.125244 max_load_reduction=28159 max_load_reduction_pct=0.859344 moved_to_echo=17036 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1093, 0, 1421, 63, 1605, 0, 0, 0, 1013, 0, 2322, 100, 334, 16, 0, 0, 1392, 1980, 1, 49, 847, 351, 0, 0, 1404, 2156, 13, 899, 0, 0, 0, 0, 21, 1487, 1670, 0, 310, 128, 0, 0, 2, 1585, 1342, 124, 1355, 0, 0, 0, 0, 2458, 0, 1484, 0, 0, 0, 0, 1479, 2200, 64, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 3785, 4620, 4472, 3616, 4408, 3942, 3743] avg=4096.0000 max=4620 max_rank=2 min=3616 min_rank=4 imbalance_ratio=1.127930 max_load_reduction=28148 max_load_reduction_pct=0.859009 moved_to_echo=15709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1058, 0, 1704, 103, 1444, 0, 0, 0, 1520, 0, 2240, 65, 351, 23, 0, 0, 1533, 1754, 1, 63, 635, 316, 0, 0, 1656, 1600, 21, 894, 0, 0, 0, 0, 74, 1641, 1909, 1, 374, 98, 0, 0, 1, 844, 1712, 358, 1219, 0, 0, 0, 0, 2543, 0, 1471, 0, 0, 0, 0, 1764, 1734, 44, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4309, 4199, 4302, 4171, 4097, 4134, 4014, 3542] avg=4096.0000 max=4309 max_rank=0 min=3542 min_rank=7 imbalance_ratio=1.052002 max_load_reduction=28459 max_load_reduction_pct=0.868500 moved_to_echo=15787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[806, 0, 2000, 42, 1478, 0, 0, 0, 884, 0, 2520, 77, 374, 27, 0, 0, 1204, 1318, 0, 19, 686, 323, 0, 0, 1153, 1748, 25, 624, 0, 0, 0, 0, 5, 1766, 2564, 0, 498, 104, 0, 0, 2, 819, 1463, 119, 1250, 0, 0, 0, 1, 2748, 0, 1528, 0, 0, 0, 0, 2375, 2016, 201, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4326, 3882, 3550, 3550, 4937, 3653, 4277, 4593] avg=4096.0000 max=4937 max_rank=4 min=3550 min_rank=2 imbalance_ratio=1.205322 max_load_reduction=27831 max_load_reduction_pct=0.849335 moved_to_echo=17460 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1151, 0, 1403, 103, 1499, 0, 0, 0, 1717, 1, 2224, 47, 333, 18, 0, 0, 1688, 1281, 0, 31, 778, 328, 0, 0, 944, 1981, 7, 764, 0, 0, 0, 0, 9, 2140, 1783, 1, 339, 118, 0, 0, 0, 789, 1680, 253, 1266, 0, 0, 0, 1, 2452, 0, 1716, 0, 0, 0, 0, 1620, 2256, 47, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4340, 4106, 3696, 4390, 3988, 4169, 3923] avg=4096.0000 max=4390 max_rank=4 min=3696 min_rank=3 imbalance_ratio=1.071777 max_load_reduction=28378 max_load_reduction_pct=0.866028 moved_to_echo=16470 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6912, 0, 13568, 512, 12032, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10496, 13312, 128, 384, 6144, 2688, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 12672, 15744, 128, 3328, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7808, 13056, 1920, 10240, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[10496, 15616, 128, 6656, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10496, 128, 18944, 512, 2816, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 20608, 0, 12288, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[15616, 16512, 896, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[210, 0, 0, 4, 1208, 0, 0, 0, 0, 2243, 4094, 10, 2022, 0, 4094, 4086, 4093, 0, 0, 0, 1053, 0, 0, 0, 4095, 1461, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[158, 0, 0, 0, 1736, 0, 0, 0, 0, 2365, 4079, 0, 2241, 0, 4095, 4096, 4096, 0, 0, 0, 406, 0, 0, 0, 4091, 1309, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[18, 0, 0, 0, 291, 0, 0, 0, 0, 1268, 4075, 0, 2759, 1, 4096, 4083, 4095, 0, 0, 0, 2193, 0, 0, 0, 4095, 1698, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[102, 2, 0, 2, 848, 3, 3, 0, 1, 2532, 4058, 0, 2961, 3, 4090, 4092, 4093, 0, 0, 0, 903, 0, 0, 0, 4092, 885, 0, 0, 0, 4094, 3, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[5, 0, 1, 0, 260, 0, 0, 0, 0, 2289, 4064, 0, 2604, 3, 4093, 4091, 4094, 0, 0, 0, 1067, 0, 0, 0, 4095, 2006, 0, 0, 0, 4095, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[179, 0, 4, 0, 653, 0, 0, 0, 0, 2205, 4084, 0, 2917, 2, 4094, 4094, 4095, 1, 0, 0, 853, 0, 0, 0, 4090, 1399, 1, 0, 1, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[91, 0, 0, 0, 988, 0, 0, 0, 0, 2373, 4090, 0, 1838, 0, 4081, 4094, 4096, 0, 0, 0, 340, 0, 0, 0, 4095, 2586, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 84, 0, 0, 0, 0, 1911, 4092, 8, 2028, 0, 4096, 4092, 4095, 0, 0, 0, 1446, 0, 0, 0, 4096, 2714, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[210, 0, 0, 4, 4000, 0, 0, 0, 1208, 0, 0, 0, 2414, 827, 94, 0, 0, 2243, 1953, 10, 0, 0, 0, 0, 2022, 0, 0, 1672, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 1053, 0, 0, 0, 1754, 1314, 0, 0, 2340, 1461, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4214, 4543, 4206, 3694, 4094, 4121, 3801, 4095] avg=4096.0000 max=4543 max_rank=1 min=3694 min_rank=3 imbalance_ratio=1.109131 max_load_reduction=28225 max_load_reduction_pct=0.861359 moved_to_echo=16111 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[102, 2, 0, 2, 3995, 0, 0, 0, 848, 3, 3, 0, 2417, 820, 95, 0, 1, 2532, 1936, 0, 0, 0, 0, 0, 2961, 3, 0, 1675, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 903, 0, 0, 0, 1753, 1302, 0, 0, 2339, 885, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4186, 4469, 4639, 4093, 3958, 3224, 4098] avg=4096.0000 max=4639 max_rank=3 min=3224 min_rank=6 imbalance_ratio=1.132568 max_load_reduction=28129 max_load_reduction_pct=0.858429 moved_to_echo=15373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[179, 0, 4, 0, 3999, 0, 0, 0, 653, 0, 0, 0, 2419, 825, 95, 0, 0, 2205, 1949, 0, 0, 0, 0, 0, 2917, 2, 0, 1675, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 1752, 1310, 0, 0, 2338, 1399, 1, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 3992, 4154, 4594, 4096, 3915, 3738, 4097] avg=4096.0000 max=4594 max_rank=3 min=3738 min_rank=6 imbalance_ratio=1.121582 max_load_reduction=28174 max_load_reduction_pct=0.859802 moved_to_echo=15846 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 0, 0, 4001, 0, 0, 0, 291, 0, 0, 0, 2412, 823, 95, 0, 0, 1268, 1945, 0, 0, 0, 0, 0, 2759, 1, 0, 1671, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 2193, 0, 0, 0, 1754, 1307, 0, 0, 2340, 1698, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 3621, 3213, 4431, 4096, 5254, 4038, 4096] avg=4096.0000 max=5254 max_rank=5 min=3213 min_rank=2 imbalance_ratio=1.282715 max_load_reduction=27514 max_load_reduction_pct=0.839661 moved_to_echo=17484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[158, 0, 0, 0, 4000, 0, 0, 0, 1736, 0, 0, 0, 2420, 824, 95, 0, 0, 2365, 1946, 0, 0, 0, 0, 0, 2241, 0, 0, 1676, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 1752, 1309, 0, 0, 2339, 1309, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 5075, 4311, 3917, 4096, 3467, 3648, 4096] avg=4096.0000 max=5075 max_rank=1 min=3467 min_rank=5 imbalance_ratio=1.239014 max_load_reduction=27693 max_load_reduction_pct=0.845123 moved_to_echo=15307 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[91, 0, 0, 0, 3986, 0, 0, 0, 988, 0, 0, 0, 2419, 826, 95, 0, 0, 2373, 1952, 0, 0, 0, 0, 0, 1838, 0, 0, 1675, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 340, 0, 0, 0, 1754, 1312, 0, 0, 2340, 2586, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4328, 4325, 3513, 4097, 3406, 4926, 4096] avg=4096.0000 max=4926 max_rank=6 min=3406 min_rank=5 imbalance_ratio=1.202637 max_load_reduction=27842 max_load_reduction_pct=0.849670 moved_to_echo=16525 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 1, 0, 3999, 0, 0, 0, 260, 0, 0, 0, 2417, 821, 94, 0, 0, 2289, 1939, 0, 0, 0, 0, 0, 2604, 3, 0, 1674, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 1067, 0, 0, 0, 1754, 1304, 0, 0, 2340, 2006, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 3592, 4228, 4281, 4095, 4125, 4346, 4096] avg=4096.0000 max=4346 max_rank=6 min=3592 min_rank=1 imbalance_ratio=1.061035 max_load_reduction=28422 max_load_reduction_pct=0.867371 moved_to_echo=16662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 4002, 0, 0, 0, 84, 0, 0, 0, 2421, 832, 94, 0, 0, 1911, 1943, 8, 0, 0, 0, 0, 2028, 0, 0, 1671, 0, 0, 0, 0, 4095, 0, 0, 0, 4, 2, 0, 0, 1446, 0, 0, 0, 1759, 1317, 0, 0, 2333, 2714, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 3431, 3862, 3699, 4101, 4522, 5047, 4094] avg=4096.0000 max=5047 max_rank=6 min=3431 min_rank=1 imbalance_ratio=1.232178 max_load_reduction=27721 max_load_reduction_pct=0.845978 moved_to_echo=17764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 0, 0, 0, 14080, 10496, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6144, 128, 128, 0, 19456, 6656, 768, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 17280, 15616, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[18816, 14080, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[19456, 128, 0, 13440, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[896, 128, 128, 128, 32000, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 300, 0, 0, 0, 4096, 1, 3337, 4092, 805, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2913, 4027, 0, 0, 0, 240, 0, 669] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 183, 0, 0, 0, 4096, 14, 3102, 4095, 684, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3617, 4070, 0, 0, 0, 144, 0, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 1464, 0, 0, 1, 4096, 24, 3020, 4096, 705, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 2109, 4055, 4, 0, 0, 455, 0, 448] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 1542, 2, 0, 0, 4096, 21, 3518, 4094, 1139, 2, 4093, 1, 0, 1, 0, 1, 1, 0, 0, 0, 4096, 4090, 1929, 3985, 1, 1, 2, 35, 0, 117] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 472, 0, 0, 0, 4096, 1, 2680, 4096, 534, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 3140, 4090, 0, 0, 0, 147, 0, 1222] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 643, 0, 0, 0, 4096, 7, 2564, 4096, 466, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3614, 4079, 0, 0, 0, 216, 0, 696] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 601, 0, 0, 1, 4096, 10, 3097, 4096, 1083, 0, 4095, 0, 1, 0, 0, 0, 1, 0, 0, 0, 4096, 4095, 2965, 3975, 0, 0, 0, 121, 0, 435] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 484, 0, 0, 0, 4096, 5, 2869, 4096, 354, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 3520, 4079, 0, 0, 0, 63, 0, 912] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 300, 0, 3380, 0, 0, 0, 0, 0, 4086, 1, 0, 0, 0, 0, 3337, 353, 805, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 2913, 1115, 0, 0, 0, 0, 0, 0, 0, 240, 0, 669, 2912, 359, 10, 0] rerouted_tokens_per_ep_rank=[3680, 4087, 4495, 4096, 4095, 4097, 4028, 4190] avg=4096.0000 max=4495 max_rank=2 min=3680 min_rank=0 imbalance_ratio=1.097412 max_load_reduction=28273 max_load_reduction_pct=0.862823 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1464, 0, 3384, 0, 0, 0, 0, 1, 4086, 24, 0, 0, 0, 0, 3020, 352, 705, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 2109, 1123, 4, 0, 0, 0, 0, 0, 0, 455, 0, 448, 2932, 360, 10, 0] rerouted_tokens_per_ep_rank=[4850, 4111, 4077, 4097, 4095, 4097, 3236, 4205] avg=4096.0000 max=4850 max_rank=0 min=3236 min_rank=6 imbalance_ratio=1.184082 max_load_reduction=27918 max_load_reduction_pct=0.851990 moved_to_echo=15633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1542, 2, 3382, 0, 0, 0, 0, 0, 4086, 21, 0, 0, 0, 0, 3518, 353, 1139, 2, 0, 0, 0, 0, 4093, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 1929, 1103, 1, 1, 0, 0, 0, 0, 2, 35, 0, 117, 2882, 359, 10, 0] rerouted_tokens_per_ep_rank=[4927, 4107, 5012, 4095, 4097, 4091, 3034, 3405] avg=4096.0000 max=5012 max_rank=2 min=3034 min_rank=6 imbalance_ratio=1.223633 max_load_reduction=27756 max_load_reduction_pct=0.847046 moved_to_echo=14627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 183, 0, 3383, 0, 0, 0, 0, 0, 4086, 14, 0, 0, 0, 0, 3102, 353, 684, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3617, 1127, 0, 0, 0, 0, 0, 0, 0, 144, 0, 475, 2943, 359, 10, 0] rerouted_tokens_per_ep_rank=[3566, 4100, 4139, 4096, 4095, 4097, 4744, 3931] avg=4096.0000 max=4744 max_rank=6 min=3566 min_rank=0 imbalance_ratio=1.158203 max_load_reduction=28024 max_load_reduction_pct=0.855225 moved_to_echo=16867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 484, 0, 3384, 0, 0, 0, 0, 0, 4086, 5, 0, 0, 0, 0, 2869, 352, 354, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3520, 1129, 0, 0, 0, 0, 0, 0, 0, 63, 0, 912, 2950, 360, 10, 0] rerouted_tokens_per_ep_rank=[3870, 4091, 3575, 4096, 4095, 4097, 4649, 4295] avg=4096.0000 max=4649 max_rank=6 min=3575 min_rank=2 imbalance_ratio=1.135010 max_load_reduction=28119 max_load_reduction_pct=0.858124 moved_to_echo=17136 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 643, 0, 3387, 0, 0, 0, 0, 0, 4081, 7, 0, 0, 0, 0, 2564, 342, 466, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3614, 1125, 0, 0, 0, 0, 0, 0, 0, 216, 0, 696, 2954, 363, 15, 1] rerouted_tokens_per_ep_rank=[4032, 4088, 3372, 4096, 4100, 4096, 4739, 4245] avg=4096.0000 max=4739 max_rank=6 min=3372 min_rank=2 imbalance_ratio=1.156982 max_load_reduction=28029 max_load_reduction_pct=0.855377 moved_to_echo=17180 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 472, 0, 3384, 0, 0, 0, 0, 0, 4086, 1, 0, 0, 0, 0, 2680, 352, 534, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3140, 1132, 0, 0, 0, 0, 0, 0, 0, 147, 0, 1222, 2958, 360, 10, 0] rerouted_tokens_per_ep_rank=[3858, 4087, 3566, 4096, 4095, 4097, 4272, 4697] avg=4096.0000 max=4697 max_rank=7 min=3566 min_rank=2 imbalance_ratio=1.146729 max_load_reduction=28071 max_load_reduction_pct=0.856659 moved_to_echo=17161 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 601, 0, 3384, 0, 0, 0, 0, 1, 4086, 10, 0, 0, 0, 0, 3097, 352, 1083, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 2965, 1101, 0, 0, 0, 0, 0, 0, 0, 121, 0, 435, 2874, 360, 10, 0] rerouted_tokens_per_ep_rank=[3985, 4097, 4532, 4096, 4096, 4096, 4066, 3800] avg=4096.0000 max=4532 max_rank=2 min=3800 min_rank=7 imbalance_ratio=1.106445 max_load_reduction=28236 max_load_reduction_pct=0.861694 moved_to_echo=16058 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 5760, 128, 27136, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[23808, 8960, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[24192, 2816, 5888, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1536, 0, 4992, 23424, 2944, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3998, 0, 4084, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 563, 0, 4095, 0, 0, 2296, 4096, 13, 2757, 214, 0, 0, 0, 212, 2248] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4061, 0, 4087, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 185, 0, 4094, 0, 0, 2462, 4096, 15, 3594, 225, 1, 0, 0, 168, 1588] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4008, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 631, 0, 4096, 0, 0, 2616, 4096, 2, 3159, 253, 1, 0, 0, 105, 1514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 4029, 2, 4092, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 440, 0, 4094, 3, 0, 1064, 4096, 2, 2867, 375, 35, 0, 0, 257, 3219] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4087, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 223, 0, 4096, 0, 0, 3547, 4096, 0, 2068, 117, 0, 0, 0, 117, 2129] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4089, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 394, 0, 4094, 0, 0, 2618, 4096, 6, 3805, 211, 1, 0, 0, 25, 1142] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4013, 0, 4095, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 375, 0, 4095, 0, 0, 1750, 4096, 8, 2609, 170, 0, 1, 0, 142, 3227] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4029, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 188, 0, 4095, 0, 0, 1352, 4096, 4, 3676, 210, 0, 0, 0, 46, 2784] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4027, 59, 0, 0, 0, 3998, 0, 57, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 563, 0, 3722, 0, 0, 0, 0, 0, 0, 2296, 1884, 0, 0, 0, 0, 13, 2757, 214, 0, 423, 372, 0, 0, 0, 0, 212, 2248, 1730, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4055, 4097, 4096, 4285, 4180, 3779, 4190] avg=4096.0000 max=4285 max_rank=4 min=3779 min_rank=6 imbalance_ratio=1.046143 max_load_reduction=28483 max_load_reduction_pct=0.869232 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4039, 59, 0, 0, 0, 4008, 0, 57, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 631, 0, 3723, 0, 0, 0, 0, 0, 0, 2616, 1884, 0, 0, 0, 0, 2, 3159, 253, 1, 423, 372, 0, 0, 0, 0, 105, 1514, 1730, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4065, 4096, 4096, 4354, 4500, 4210, 3349] avg=4096.0000 max=4500 max_rank=5 min=3349 min_rank=7 imbalance_ratio=1.098633 max_load_reduction=28268 max_load_reduction_pct=0.862671 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4035, 59, 0, 0, 0, 4029, 2, 57, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 440, 0, 3722, 0, 0, 0, 0, 3, 0, 1064, 1884, 0, 0, 0, 0, 2, 2867, 375, 35, 423, 372, 0, 0, 0, 0, 257, 3219, 1730, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4088, 4094, 4096, 4162, 2951, 4074, 5206] avg=4096.0000 max=5206 max_rank=7 min=2951 min_rank=5 imbalance_ratio=1.270996 max_load_reduction=27562 max_load_reduction_pct=0.841125 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4039, 59, 0, 0, 0, 4087, 0, 57, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 223, 0, 3723, 0, 0, 0, 0, 0, 0, 3547, 1884, 0, 0, 0, 0, 0, 2068, 117, 0, 423, 372, 0, 0, 0, 0, 117, 2129, 1730, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4144, 4097, 4096, 3946, 5431, 2980, 3976] avg=4096.0000 max=5431 max_rank=5 min=2980 min_rank=6 imbalance_ratio=1.325928 max_load_reduction=27337 max_load_reduction_pct=0.834259 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4034, 63, 0, 0, 0, 4061, 0, 53, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 185, 0, 3713, 0, 0, 0, 0, 0, 0, 2462, 1872, 0, 0, 0, 0, 15, 3594, 225, 1, 427, 378, 0, 0, 0, 0, 168, 1588, 1734, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4114, 4099, 4096, 3898, 4334, 4640, 3490] avg=4096.0000 max=4640 max_rank=6 min=3490 min_rank=7 imbalance_ratio=1.132812 max_load_reduction=28128 max_load_reduction_pct=0.858398 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4039, 59, 0, 0, 0, 4029, 0, 57, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 188, 0, 3722, 0, 0, 0, 0, 0, 0, 1352, 1884, 0, 0, 0, 0, 4, 3676, 210, 0, 423, 372, 0, 0, 0, 0, 46, 2784, 1730, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4086, 4097, 4096, 3910, 3236, 4685, 4560] avg=4096.0000 max=4685 max_rank=6 min=3236 min_rank=5 imbalance_ratio=1.143799 max_load_reduction=28083 max_load_reduction_pct=0.857025 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4038, 59, 0, 0, 0, 4089, 0, 57, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 394, 0, 3722, 0, 0, 0, 0, 0, 0, 2618, 1884, 0, 0, 0, 0, 6, 3805, 211, 1, 423, 372, 0, 0, 0, 0, 25, 1142, 1730, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4146, 4096, 4096, 4116, 4502, 4818, 2897] avg=4096.0000 max=4818 max_rank=6 min=2897 min_rank=7 imbalance_ratio=1.176270 max_load_reduction=27950 max_load_reduction_pct=0.852966 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4038, 59, 0, 0, 0, 4013, 0, 57, 0, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 375, 0, 3722, 0, 0, 0, 0, 0, 0, 1750, 1884, 0, 0, 0, 0, 8, 2609, 170, 0, 423, 372, 0, 0, 1, 0, 142, 3227, 1730, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4070, 4092, 4096, 4097, 3634, 3582, 5100] avg=4096.0000 max=5100 max_rank=7 min=3582 min_rank=6 imbalance_ratio=1.245117 max_load_reduction=27668 max_load_reduction_pct=0.844360 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 1152, 17920, 13952, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 17792, 15104, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 24576, 1792, 128, 3456, 3072, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 32384, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 3072, 0, 29824, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32384, 128, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4076, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3170, 0, 0, 0, 0, 0, 0, 945, 0, 0, 4094, 0, 4093, 0, 1, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 2885, 0, 0, 0, 0, 0, 0, 1211, 0, 8, 4093, 0, 4087, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1458, 0, 0, 0, 0, 0, 0, 2639, 0, 0, 4095, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 2, 0, 4095, 0, 0, 0, 4096, 2590, 0, 0, 0, 0, 0, 0, 1501, 0, 2, 4096, 0, 4093, 0, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 4085, 4095, 0, 0, 3, 0, 4091, 0, 0, 0, 4096, 1855, 0, 0, 0, 0, 0, 0, 2251, 0, 3, 4095, 0, 4094, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2, 4095, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 1087, 0, 0, 0, 0, 0, 0, 3009, 0, 9, 4096, 0, 4081, 0, 0, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 2593, 0, 0, 0, 0, 0, 0, 1504, 0, 4, 4096, 0, 4085, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 2, 0, 4092, 0, 0, 0, 4096, 1310, 0, 0, 0, 0, 0, 0, 2787, 0, 4, 4094, 0, 4089, 0, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4076, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3170, 0, 0, 0, 1976, 0, 0, 0, 0, 0, 0, 945, 2115, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 1, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4081, 4097, 4096, 5146, 3060, 4093, 4099] avg=4096.0000 max=5146 max_rank=4 min=3060 min_rank=5 imbalance_ratio=1.256348 max_load_reduction=27622 max_load_reduction_pct=0.842957 moved_to_echo=16398 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2885, 0, 0, 0, 1978, 0, 0, 0, 0, 0, 0, 1211, 2117, 0, 0, 0, 0, 8, 4092, 0, 0, 0, 0, 0, 4087, 0, 0, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4093, 4096, 4863, 3328, 4100, 4095] avg=4096.0000 max=4863 max_rank=4 min=3328 min_rank=5 imbalance_ratio=1.187256 max_load_reduction=27905 max_load_reduction_pct=0.851593 moved_to_echo=16386 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1458, 0, 0, 0, 1976, 0, 0, 0, 0, 0, 0, 2639, 2115, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4098, 4096, 3434, 4754, 4093, 4096] avg=4096.0000 max=4754 max_rank=5 min=3434 min_rank=4 imbalance_ratio=1.160645 max_load_reduction=28014 max_load_reduction_pct=0.854919 moved_to_echo=16377 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 4085, 5, 0, 0, 0, 0, 0, 0, 3, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1855, 0, 0, 0, 1976, 0, 0, 0, 0, 0, 0, 2251, 2114, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 4094, 0, 0, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4090, 4096, 4096, 3831, 4365, 4096, 4099] avg=4096.0000 max=4365 max_rank=5 min=3831 min_rank=4 imbalance_ratio=1.065674 max_load_reduction=28403 max_load_reduction_pct=0.866791 moved_to_echo=16391 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1087, 0, 0, 0, 1981, 5, 0, 0, 0, 0, 0, 3009, 2115, 0, 0, 0, 0, 9, 4086, 0, 0, 0, 0, 0, 4079, 0, 0, 12, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4097, 4094, 4096, 3073, 5124, 4095, 4091] avg=4096.0000 max=5124 max_rank=5 min=3073 min_rank=4 imbalance_ratio=1.250977 max_load_reduction=27644 max_load_reduction_pct=0.843628 moved_to_echo=16383 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1310, 0, 0, 0, 1976, 0, 0, 0, 0, 0, 0, 2787, 2115, 0, 0, 0, 0, 4, 4093, 0, 0, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4100, 4095, 4096, 3286, 4902, 4097, 4096] avg=4096.0000 max=4902 max_rank=5 min=3286 min_rank=4 imbalance_ratio=1.196777 max_load_reduction=27866 max_load_reduction_pct=0.850403 moved_to_echo=16381 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2590, 0, 0, 0, 1976, 0, 0, 0, 0, 0, 0, 1501, 2115, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 4093, 0, 0, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4101, 4099, 4096, 4566, 3616, 4096, 4099] avg=4096.0000 max=4566 max_rank=4 min=3616 min_rank=5 imbalance_ratio=1.114746 max_load_reduction=28202 max_load_reduction_pct=0.860657 moved_to_echo=16377 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2593, 0, 0, 0, 1976, 0, 0, 0, 0, 0, 0, 1504, 2115, 0, 0, 0, 0, 4, 4094, 0, 0, 0, 0, 0, 4085, 0, 0, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4096, 4096, 4569, 3619, 4098, 4093] avg=4096.0000 max=4569 max_rank=4 min=3619 min_rank=5 imbalance_ratio=1.115479 max_load_reduction=28199 max_load_reduction_pct=0.860565 moved_to_echo=16379 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 15872, 17024, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17024, 0, 0, 0, 15872, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2688, 4087, 1206, 0, 0, 4013, 0, 0, 4, 288, 4, 4091, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 5, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 2283, 4064, 1754, 0, 0, 4043, 0, 0, 0, 119, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 24, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2321, 4096, 1063, 0, 0, 4090, 0, 0, 0, 673, 0, 4096, 0, 4092, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 46, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 2381, 4085, 919, 0, 0, 4088, 0, 0, 0, 818, 0, 4096, 0, 4089, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2426, 3972, 1492, 0, 0, 4078, 0, 0, 0, 295, 0, 4096, 0, 4096, 0, 0, 4096, 8, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2256, 4088, 1598, 0, 0, 4065, 0, 0, 0, 210, 1, 4096, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 71, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2759, 4095, 1067, 0, 0, 4093, 0, 0, 0, 274, 0, 4096, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1975, 4092, 1509, 0, 0, 4087, 0, 0, 0, 581, 1, 4096, 0, 4096, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 40, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2321, 387, 1063, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 26, 0, 0, 0, 0, 673, 0, 3689, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 46, 3709, 381, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3771, 4116, 4362, 4093, 4098, 4136, 4096] avg=4096.0000 max=4362 max_rank=3 min=3771 min_rank=1 imbalance_ratio=1.064941 max_load_reduction=28406 max_load_reduction_pct=0.866882 moved_to_echo=16423 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2688, 386, 1206, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 26, 0, 0, 0, 4, 288, 4, 3684, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 5, 3701, 381, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4280, 4039, 3980, 4097, 4092, 4087, 4097] avg=4096.0000 max=4280 max_rank=1 min=3980 min_rank=3 imbalance_ratio=1.044922 max_load_reduction=28488 max_load_reduction_pct=0.869385 moved_to_echo=16373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 2381, 386, 919, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 26, 0, 0, 0, 0, 818, 0, 3689, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 3699, 381, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3686, 4114, 4507, 4090, 4096, 4082, 4096] avg=4096.0000 max=4507 max_rank=3 min=3686 min_rank=1 imbalance_ratio=1.100342 max_load_reduction=28261 max_load_reduction_pct=0.862457 moved_to_echo=16364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 2283, 384, 1754, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 26, 0, 0, 0, 0, 119, 0, 3689, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 24, 3680, 381, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4421, 4069, 3808, 4097, 4095, 4085, 4096] avg=4096.0000 max=4421 max_rank=1 min=3808 min_rank=3 imbalance_ratio=1.079346 max_load_reduction=28347 max_load_reduction_pct=0.865082 moved_to_echo=16373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2759, 387, 1067, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 26, 0, 0, 0, 0, 274, 0, 3689, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 3708, 381, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4213, 4119, 3963, 4096, 4095, 4090, 4096] avg=4096.0000 max=4213 max_rank=1 min=3963 min_rank=3 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 2426, 369, 1492, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 28, 1, 0, 0, 0, 295, 0, 3682, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 2, 0, 0, 0, 8, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 17, 3603, 386, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4287, 4107, 3977, 4102, 4099, 4007, 4095] avg=4096.0000 max=4287 max_rank=1 min=3977 min_rank=3 imbalance_ratio=1.046631 max_load_reduction=28481 max_load_reduction_pct=0.869171 moved_to_echo=16303 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2256, 386, 1598, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 26, 0, 0, 0, 0, 210, 1, 3689, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 71, 3702, 381, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4240, 4091, 3900, 4096, 4095, 4154, 4096] avg=4096.0000 max=4240 max_rank=1 min=3900 min_rank=3 imbalance_ratio=1.035156 max_load_reduction=28528 max_load_reduction_pct=0.870605 moved_to_echo=16441 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 1975, 386, 1509, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 26, 0, 0, 0, 0, 581, 1, 3689, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 40, 3706, 381, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3870, 4113, 4271, 4097, 4098, 4127, 4096] avg=4096.0000 max=4271 max_rank=3 min=3870 min_rank=1 imbalance_ratio=1.042725 max_load_reduction=28497 max_load_reduction_pct=0.869659 moved_to_echo=16418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[19200, 3072, 10624, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 3328, 128, 29568, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 256, 29568, 3072, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32640, 0, 0, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 224, 0, 4096, 2682, 0, 0, 0, 0, 4072, 10, 0, 15, 0, 0, 4096, 4095, 1190, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 269, 0, 4096, 2335, 0, 0, 0, 0, 4096, 109, 0, 2, 0, 0, 4096, 4096, 1381, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 255, 0, 4096, 3328, 0, 0, 0, 0, 4063, 12, 0, 12, 1, 0, 4096, 4096, 522, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 121, 0, 4096, 3565, 0, 0, 0, 0, 4053, 8, 0, 13, 0, 0, 4096, 4096, 432, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 122, 0, 4096, 2448, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 4096, 4096, 1562, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 369, 0, 4094, 2492, 0, 0, 0, 0, 4079, 24, 0, 9, 3, 0, 4096, 4096, 1219, 4096, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4093, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 333, 0, 4096, 2928, 0, 1, 0, 0, 4084, 7, 0, 15, 0, 0, 4096, 4096, 825, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 101, 0, 4096, 3525, 0, 0, 0, 0, 4007, 10, 0, 9, 0, 0, 4096, 4094, 543, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 224, 0, 3873, 0, 0, 0, 0, 2682, 0, 0, 0, 958, 223, 1, 0, 0, 4072, 10, 0, 9, 0, 0, 0, 15, 0, 0, 4087, 0, 0, 0, 0, 3137, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3864, 4091, 4102, 4327, 4096, 4095, 4096] avg=4096.0000 max=4327 max_rank=4 min=3864 min_rank=1 imbalance_ratio=1.056396 max_load_reduction=28441 max_load_reduction_pct=0.867950 moved_to_echo=16614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 269, 0, 3873, 0, 0, 0, 0, 2335, 0, 0, 0, 958, 223, 0, 0, 0, 4096, 109, 0, 9, 0, 0, 0, 2, 0, 0, 4087, 0, 0, 0, 0, 3138, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 3516, 4214, 4089, 4519, 4096, 4096, 4096] avg=4096.0000 max=4519 max_rank=4 min=3516 min_rank=1 imbalance_ratio=1.103271 max_load_reduction=28249 max_load_reduction_pct=0.862091 moved_to_echo=16807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 369, 0, 3871, 0, 0, 0, 0, 2492, 0, 0, 0, 958, 223, 0, 0, 0, 4079, 24, 0, 9, 0, 0, 0, 9, 3, 0, 4087, 0, 0, 0, 0, 3138, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4240, 3673, 4112, 4099, 4357, 4096, 4098, 4093] avg=4096.0000 max=4357 max_rank=4 min=3673 min_rank=1 imbalance_ratio=1.063721 max_load_reduction=28411 max_load_reduction_pct=0.867035 moved_to_echo=16644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 122, 0, 3873, 0, 0, 0, 0, 2448, 0, 0, 0, 958, 223, 0, 0, 0, 4060, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 3138, 1562, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3995, 3629, 4069, 4087, 4700, 4096, 4096, 4096] avg=4096.0000 max=4700 max_rank=4 min=3629 min_rank=1 imbalance_ratio=1.147461 max_load_reduction=28068 max_load_reduction_pct=0.856567 moved_to_echo=16988 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 121, 0, 3865, 0, 0, 0, 0, 3565, 0, 0, 0, 965, 229, 1, 0, 0, 4053, 8, 0, 9, 2, 0, 0, 13, 0, 0, 4080, 0, 0, 0, 0, 3131, 432, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[3986, 4760, 4072, 4093, 3563, 4096, 4095, 4103] avg=4096.0000 max=4760 max_rank=1 min=3563 min_rank=4 imbalance_ratio=1.162109 max_load_reduction=28008 max_load_reduction_pct=0.854736 moved_to_echo=15857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 255, 0, 3873, 0, 0, 0, 0, 3328, 0, 0, 0, 958, 223, 1, 0, 0, 4063, 12, 0, 9, 0, 0, 0, 12, 1, 0, 4087, 0, 0, 0, 0, 3138, 522, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4510, 4084, 4100, 3660, 4096, 4095, 4095] avg=4096.0000 max=4510 max_rank=1 min=3660 min_rank=4 imbalance_ratio=1.101074 max_load_reduction=28258 max_load_reduction_pct=0.862366 moved_to_echo=15946 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 333, 0, 3873, 0, 0, 0, 0, 2928, 0, 1, 0, 958, 223, 0, 0, 0, 4084, 7, 0, 9, 0, 0, 0, 15, 0, 0, 4087, 0, 0, 0, 0, 3138, 825, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4206, 4110, 4100, 4102, 3963, 4096, 4096, 4095] avg=4096.0000 max=4206 max_rank=0 min=3963 min_rank=4 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 101, 0, 3873, 0, 0, 0, 0, 3525, 0, 0, 0, 958, 223, 0, 0, 0, 4007, 10, 0, 9, 0, 0, 0, 9, 0, 0, 4087, 0, 0, 0, 0, 3136, 543, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3974, 4706, 4026, 4096, 3679, 4096, 4097, 4094] avg=4096.0000 max=4706 max_rank=1 min=3679 min_rank=4 imbalance_ratio=1.148926 max_load_reduction=28062 max_load_reduction_pct=0.856384 moved_to_echo=15966 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[23424, 0, 128, 0, 7680, 1792, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[25216, 7680, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 1920, 0, 30976, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32640, 256, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3282, 4094, 0, 1321, 0, 0, 4096, 4096, 0, 2121, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 1, 1469, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3439, 4095, 0, 1485, 0, 0, 4096, 4096, 0, 1521, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1748, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2737, 4096, 0, 1179, 0, 0, 4096, 4096, 0, 1957, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 2319, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3318, 4095, 0, 589, 0, 0, 4096, 4096, 0, 3198, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1088, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3099, 4086, 0, 1130, 0, 0, 4096, 4096, 0, 2771, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1199, 0, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3105, 4095, 0, 392, 0, 0, 4096, 4096, 0, 2760, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 2, 1933, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2631, 4094, 0, 1102, 0, 0, 4096, 4096, 0, 2864, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1595, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2429, 4096, 0, 1769, 0, 0, 4096, 4096, 0, 2373, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1620, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3282, 1092, 0, 0, 0, 0, 0, 1321, 0, 0, 2974, 1, 0, 0, 0, 1651, 0, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 1469, 0, 0, 2445, 28, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4374, 4296, 3772, 4096, 4096, 4096, 3943, 4095] avg=4096.0000 max=4374 max_rank=0 min=3772 min_rank=2 imbalance_ratio=1.067871 max_load_reduction=28394 max_load_reduction_pct=0.866516 moved_to_echo=16230 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3318, 1092, 0, 0, 0, 0, 0, 589, 0, 0, 2975, 1, 0, 0, 0, 1651, 0, 3198, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 0, 0, 2445, 28, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4410, 3565, 4849, 4096, 4096, 4096, 3561, 4095] avg=4096.0000 max=4849 max_rank=2 min=3561 min_rank=6 imbalance_ratio=1.183838 max_load_reduction=27919 max_load_reduction_pct=0.852020 moved_to_echo=15848 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2737, 1093, 0, 0, 0, 0, 0, 1179, 0, 0, 2975, 1, 0, 0, 0, 1651, 0, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2319, 0, 0, 2445, 28, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[3830, 4155, 3608, 4096, 4096, 4096, 4792, 4095] avg=4096.0000 max=4792 max_rank=6 min=3608 min_rank=2 imbalance_ratio=1.169922 max_load_reduction=27976 max_load_reduction_pct=0.853760 moved_to_echo=17079 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3099, 1090, 0, 0, 0, 0, 0, 1130, 0, 0, 2968, 1, 0, 0, 0, 1651, 0, 2771, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 0, 0, 2445, 28, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4189, 4099, 4422, 4096, 4096, 4096, 3672, 4098] avg=4096.0000 max=4422 max_rank=2 min=3672 min_rank=6 imbalance_ratio=1.079590 max_load_reduction=28346 max_load_reduction_pct=0.865051 moved_to_echo=15962 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3439, 1084, 0, 0, 0, 0, 0, 1485, 0, 0, 2978, 1, 0, 0, 0, 1646, 0, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1748, 0, 0, 2450, 33, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4523, 4464, 3167, 4096, 4096, 4096, 4231, 4095] avg=4096.0000 max=4523 max_rank=0 min=3167 min_rank=2 imbalance_ratio=1.104248 max_load_reduction=28245 max_load_reduction_pct=0.861969 moved_to_echo=16518 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2429, 1093, 0, 0, 0, 0, 0, 1769, 0, 0, 2975, 0, 0, 0, 0, 1651, 0, 2373, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 0, 0, 2445, 28, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3522, 4744, 4024, 4096, 4096, 4096, 4093, 4097] avg=4096.0000 max=4744 max_rank=1 min=3522 min_rank=0 imbalance_ratio=1.158203 max_load_reduction=28024 max_load_reduction_pct=0.855225 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3105, 1092, 0, 0, 0, 0, 0, 392, 0, 0, 2975, 1, 0, 0, 0, 1651, 0, 2760, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 1933, 0, 0, 2445, 28, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4197, 3368, 4411, 4096, 4096, 4096, 4408, 4096] avg=4096.0000 max=4411 max_rank=2 min=3368 min_rank=1 imbalance_ratio=1.076904 max_load_reduction=28357 max_load_reduction_pct=0.865387 moved_to_echo=16696 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2631, 1092, 0, 0, 0, 0, 0, 1102, 0, 0, 2974, 1, 0, 0, 0, 1651, 0, 2864, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1595, 0, 0, 2445, 28, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[3723, 4077, 4515, 4096, 4096, 4096, 4068, 4097] avg=4096.0000 max=4515 max_rank=2 min=3723 min_rank=0 imbalance_ratio=1.102295 max_load_reduction=28253 max_load_reduction_pct=0.862213 moved_to_echo=16357 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 13312, 0, 19584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 24064, 8832, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 9088, 0, 0, 23808, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 13056, 0, 0, 19584, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 1549, 6, 0, 4096, 0, 0, 0, 4090, 4095, 4096, 0, 0, 0, 0, 4084, 0, 0, 0, 1, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 2550, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 2897, 4, 0, 4096, 0, 0, 0, 4088, 4096, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 69, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 1137, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 1751, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2340, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 2656, 5, 0, 4096, 0, 0, 0, 4094, 4095, 4096, 0, 0, 0, 0, 4093, 0, 0, 1, 97, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 1341, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 2523, 11, 0, 4096, 0, 0, 0, 4085, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 11, 4094, 0, 0, 0, 0, 0, 2, 2, 0, 1561, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 2775, 14, 0, 4096, 0, 0, 0, 4090, 4096, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 1319, 0, 4087] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 2335, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 33, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1731, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 2999, 2, 0, 4096, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 2, 39, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 1070, 0, 4082] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1549, 6, 0, 1624, 32, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 3, 0, 0, 0, 0, 0, 0, 1, 4063, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4092, 0, 0, 0, 0, 2550, 0, 2468, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3211, 4096, 4090, 4087, 4068, 4096, 4102, 5018] avg=4096.0000 max=5018 max_rank=7 min=3211 min_rank=0 imbalance_ratio=1.225098 max_load_reduction=27750 max_load_reduction_pct=0.846863 moved_to_echo=17284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2523, 11, 0, 1624, 31, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 1, 11, 4059, 4, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4092, 0, 0, 0, 0, 1561, 0, 2467, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4189, 4096, 4091, 4099, 4075, 4094, 4096, 4028] avg=4096.0000 max=4189 max_rank=0 min=4028 min_rank=7 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1751, 0, 0, 1624, 31, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 7, 4059, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 2340, 0, 2467, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3406, 4096, 4102, 4099, 4069, 4096, 4093, 4807] avg=4096.0000 max=4807 max_rank=7 min=3406 min_rank=0 imbalance_ratio=1.173584 max_load_reduction=27961 max_load_reduction_pct=0.853302 moved_to_echo=17065 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2897, 4, 0, 1627, 37, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 69, 4059, 4, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 4092, 0, 0, 0, 0, 1137, 0, 2460, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4565, 4096, 4088, 4096, 4132, 4098, 4096, 3597] avg=4096.0000 max=4565 max_rank=0 min=3597 min_rank=7 imbalance_ratio=1.114502 max_load_reduction=28203 max_load_reduction_pct=0.860687 moved_to_echo=15923 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2335, 0, 0, 1624, 31, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 33, 4059, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 1731, 0, 2467, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3990, 4096, 4102, 4098, 4095, 4096, 4093, 4198] avg=4096.0000 max=4198 max_rank=7 min=3990 min_rank=0 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2775, 14, 0, 1621, 31, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 3, 4059, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 1319, 0, 2463, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4441, 4096, 4096, 4096, 4065, 4096, 4096, 3782] avg=4096.0000 max=4441 max_rank=0 min=3782 min_rank=7 imbalance_ratio=1.084229 max_load_reduction=28327 max_load_reduction_pct=0.864471 moved_to_echo=16039 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2999, 2, 0, 1619, 31, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 2, 39, 4059, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 1070, 0, 2460, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4651, 4096, 4100, 4097, 4103, 4096, 4095, 3530] avg=4096.0000 max=4651 max_rank=0 min=3530 min_rank=7 imbalance_ratio=1.135498 max_load_reduction=28117 max_load_reduction_pct=0.858063 moved_to_echo=15824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2656, 5, 0, 1623, 31, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 1, 97, 4059, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4092, 0, 0, 0, 0, 1341, 0, 2467, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4315, 4096, 4099, 4096, 4161, 4096, 4097, 3808] avg=4096.0000 max=4315 max_rank=0 min=3808 min_rank=7 imbalance_ratio=1.053467 max_load_reduction=28453 max_load_reduction_pct=0.868317 moved_to_echo=16162 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 19584, 128, 0, 13056, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 13056, 0, 19840, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 384, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 73, 0, 4, 0, 0, 0, 0, 4084, 2350, 0, 0, 4059, 0, 4096, 1703, 0, 0, 0, 4093, 1, 0, 21, 4093, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 26, 0, 3, 0, 0, 0, 0, 4087, 2113, 0, 0, 4065, 0, 4096, 1979, 0, 0, 0, 4090, 0, 0, 23, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 4096, 2966, 0, 0, 4092, 0, 4096, 1047, 0, 0, 0, 4096, 7, 0, 3, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 1734, 0, 0, 4040, 0, 4096, 2394, 0, 0, 0, 4096, 7, 0, 4, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 4089, 2397, 0, 0, 4075, 0, 4096, 1691, 0, 0, 0, 4096, 0, 0, 4, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4094, 2871, 0, 0, 4046, 0, 4096, 1270, 0, 0, 0, 4095, 0, 0, 1, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 2, 0, 0, 0, 0, 4089, 2358, 0, 0, 4023, 0, 4096, 1799, 0, 0, 0, 4093, 0, 0, 9, 4094, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4084, 2310, 0, 0, 4000, 0, 4096, 1872, 0, 0, 0, 4095, 2, 0, 19, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 26, 0, 3, 0, 2350, 1719, 3, 0, 0, 0, 0, 4087, 6, 0, 0, 0, 2113, 0, 0, 1715, 0, 0, 0, 0, 0, 2377, 1979, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 23, 4087, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 4101, 4093, 3828, 4356, 4090, 4110] avg=4096.0000 max=4356 max_rank=5 min=3828 min_rank=4 imbalance_ratio=1.063477 max_load_reduction=28412 max_load_reduction_pct=0.867065 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 73, 0, 4, 0, 2346, 1719, 3, 0, 0, 0, 0, 4084, 6, 0, 0, 0, 2350, 0, 0, 1713, 0, 0, 0, 0, 0, 2377, 1703, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 21, 4084, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4145, 4090, 4063, 4080, 4094, 4105] avg=4096.0000 max=4145 max_rank=2 min=4063 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 2365, 1719, 3, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 2966, 0, 0, 1727, 0, 0, 0, 0, 0, 2377, 1047, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 3, 4087, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4164, 4102, 4693, 3424, 4103, 4090] avg=4096.0000 max=4693 max_rank=4 min=3424 min_rank=5 imbalance_ratio=1.145752 max_load_reduction=28075 max_load_reduction_pct=0.856781 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 2335, 1719, 3, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 1734, 0, 0, 1705, 0, 0, 0, 0, 0, 2377, 2394, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 4, 4087, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4071, 4102, 3439, 4771, 4103, 4091] avg=4096.0000 max=4771 max_rank=5 min=3439 min_rank=4 imbalance_ratio=1.164795 max_load_reduction=27997 max_load_reduction_pct=0.854401 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 5, 3, 0, 0, 34, 0, 0, 0, 2359, 1722, 4, 0, 0, 0, 0, 4089, 7, 0, 0, 0, 2397, 0, 0, 1716, 0, 0, 0, 0, 0, 2374, 1691, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4, 4080, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4102, 4119, 4096, 4113, 4065, 4093, 4084] avg=4096.0000 max=4119 max_rank=2 min=4065 min_rank=5 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2339, 1719, 3, 0, 0, 0, 0, 4094, 6, 0, 0, 0, 2871, 0, 0, 1707, 0, 0, 0, 0, 0, 2377, 1270, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4068, 4100, 4578, 3647, 4095, 4088] avg=4096.0000 max=4578 max_rank=4 min=3647 min_rank=5 imbalance_ratio=1.117676 max_load_reduction=28190 max_load_reduction_pct=0.860291 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 13, 0, 2, 0, 2325, 1719, 3, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 2358, 0, 0, 1698, 0, 0, 0, 0, 0, 2377, 1799, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 9, 4085, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4062, 4095, 4056, 4176, 4093, 4094] avg=4096.0000 max=4176 max_rank=5 min=4056 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2312, 1719, 3, 0, 0, 0, 0, 4084, 6, 0, 0, 0, 2310, 0, 0, 1688, 0, 0, 0, 0, 0, 2377, 1872, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 19, 4087, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 4038, 4090, 3998, 4249, 4097, 4106] avg=4096.0000 max=4249 max_rank=5 min=3998 min_rank=4 imbalance_ratio=1.037354 max_load_reduction=28519 max_load_reduction_pct=0.870331 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 0, 128, 0, 18816, 13824, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[19200, 0, 0, 13696, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 19072, 13824, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3645, 0, 0, 4096, 0, 0, 4096, 0, 0, 53, 0, 4096, 2043, 3688, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3000, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3573, 0, 0, 4096, 0, 0, 4096, 0, 0, 27, 0, 4096, 2931, 2756, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3309, 0, 0, 0, 4, 0, 0, 0, 0, 4, 3591, 0, 2, 4096, 0, 0, 4096, 1, 0, 32, 0, 4096, 1423, 3926, 0, 0, 0, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2681, 0, 0, 0, 6, 0, 0, 0, 0, 1, 3739, 0, 0, 4096, 0, 0, 4096, 0, 0, 44, 0, 4096, 2676, 3142, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3092, 0, 0, 4096, 0, 0, 4096, 1, 0, 180, 0, 4096, 2289, 3986, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2662, 0, 0, 0, 1, 0, 0, 0, 0, 1, 3507, 0, 0, 4096, 0, 0, 4096, 0, 0, 42, 0, 4096, 2706, 3370, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3456, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2682, 0, 0, 4096, 0, 0, 4096, 15, 0, 72, 0, 4096, 2204, 3856, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3339, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2999, 0, 0, 4096, 0, 0, 4096, 0, 0, 56, 0, 4096, 2268, 3627, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1091, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3645, 0, 0, 771, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 53, 0, 3005, 1064, 2, 0, 0, 2043, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3951, 4094, 4096, 4416, 4096, 4124, 3895, 4096] avg=4096.0000 max=4416 max_rank=3 min=3895 min_rank=6 imbalance_ratio=1.078125 max_load_reduction=28352 max_load_reduction_pct=0.865234 moved_to_echo=16211 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1091, 3000, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3573, 0, 0, 576, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 27, 0, 3005, 795, 2, 0, 0, 2931, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4095, 4096, 4149, 4096, 3829, 4316, 4096] avg=4096.0000 max=4316 max_rank=6 min=3829 min_rank=5 imbalance_ratio=1.053711 max_load_reduction=28452 max_load_reduction_pct=0.868286 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1091, 3309, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 3591, 0, 2, 821, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 32, 0, 3005, 1133, 2, 0, 0, 1423, 1971, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4400, 4098, 4096, 4418, 4096, 4173, 3394, 4093] avg=4096.0000 max=4418 max_rank=3 min=3394 min_rank=6 imbalance_ratio=1.078613 max_load_reduction=28350 max_load_reduction_pct=0.865173 moved_to_echo=15756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1091, 3456, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2682, 0, 0, 806, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 15, 0, 72, 0, 3005, 1112, 2, 0, 0, 2204, 1937, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4547, 4097, 4096, 3488, 4096, 4206, 4141, 4097] avg=4096.0000 max=4547 max_rank=0 min=3488 min_rank=3 imbalance_ratio=1.110107 max_load_reduction=28221 max_load_reduction_pct=0.861237 moved_to_echo=16540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1084, 2740, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3092, 0, 0, 837, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 180, 0, 3012, 1155, 1, 0, 0, 2289, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[3824, 4095, 4096, 3929, 4096, 4349, 4279, 4100] avg=4096.0000 max=4349 max_rank=5 min=3824 min_rank=0 imbalance_ratio=1.061768 max_load_reduction=28419 max_load_reduction_pct=0.867279 moved_to_echo=16824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1091, 2681, 0, 0, 0, 0, 0, 0, 0, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 3739, 0, 0, 657, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 44, 0, 3005, 906, 2, 0, 0, 2676, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3772, 4100, 4096, 4397, 4096, 3957, 4255, 4095] avg=4096.0000 max=4397 max_rank=3 min=3772 min_rank=0 imbalance_ratio=1.073486 max_load_reduction=28371 max_load_reduction_pct=0.865814 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1091, 3339, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 2999, 0, 0, 758, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 56, 0, 3005, 1046, 2, 0, 0, 2268, 1822, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4430, 4094, 4096, 3758, 4096, 4109, 4090, 4095] avg=4096.0000 max=4430 max_rank=0 min=3758 min_rank=3 imbalance_ratio=1.081543 max_load_reduction=28338 max_load_reduction_pct=0.864807 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1091, 2662, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 3507, 0, 0, 705, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 42, 0, 3005, 972, 2, 0, 0, 2706, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3753, 4095, 4096, 4213, 4096, 4021, 4398, 4096] avg=4096.0000 max=4398 max_rank=6 min=3753 min_rank=0 imbalance_ratio=1.073730 max_load_reduction=28370 max_load_reduction_pct=0.865784 moved_to_echo=16611 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 512, 0, 24064, 8192, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 8832, 24064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 18560, 14336, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 26880, 0, 128, 6016, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[347, 4096, 0, 0, 0, 4096, 4096, 1835, 0, 0, 4096, 0, 4096, 0, 2504, 4069, 0, 0, 0, 3533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[83, 4096, 0, 0, 0, 4096, 4096, 2993, 0, 0, 4096, 0, 4096, 0, 1304, 4013, 0, 0, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[321, 4096, 0, 0, 0, 4096, 4096, 3037, 0, 0, 4096, 0, 4096, 0, 1710, 3642, 0, 0, 0, 3578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[9, 4096, 0, 0, 0, 4096, 4096, 3662, 0, 0, 4096, 0, 4096, 0, 788, 3887, 0, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[113, 4096, 0, 0, 0, 4096, 4096, 2949, 0, 0, 4096, 0, 4096, 0, 2259, 3909, 0, 0, 0, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[72, 4096, 0, 0, 0, 4096, 4096, 3346, 0, 0, 4096, 0, 4096, 0, 1014, 3836, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[43, 4096, 0, 0, 0, 4096, 4096, 2672, 0, 0, 4096, 0, 4096, 0, 2458, 3738, 0, 0, 0, 3377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[92, 4096, 0, 0, 0, 4096, 4095, 2777, 0, 0, 4096, 0, 4096, 0, 1927, 3698, 0, 0, 0, 3795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[83, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 2993, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1304, 2452, 0, 0, 0, 0, 0, 0, 0, 3895, 324, 135, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2908, 1237, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4181, 4096, 3756, 4354, 4096, 4145, 4096] avg=4096.0000 max=4354 max_rank=4 min=3756 min_rank=3 imbalance_ratio=1.062988 max_load_reduction=28414 max_load_reduction_pct=0.867126 moved_to_echo=16691 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[347, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 1835, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2504, 2485, 0, 0, 0, 0, 0, 0, 0, 3533, 329, 135, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2908, 1255, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4308, 3023, 4096, 4989, 3997, 4096, 4163, 4096] avg=4096.0000 max=4989 max_rank=3 min=3023 min_rank=1 imbalance_ratio=1.218018 max_load_reduction=27779 max_load_reduction_pct=0.847748 moved_to_echo=16352 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[72, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 3346, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1014, 2343, 0, 0, 0, 0, 0, 0, 0, 4020, 310, 135, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2908, 1183, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4534, 4096, 3357, 4465, 4096, 4091, 4096] avg=4096.0000 max=4534 max_rank=1 min=3357 min_rank=3 imbalance_ratio=1.106934 max_load_reduction=28234 max_load_reduction_pct=0.861633 moved_to_echo=16748 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[113, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 2949, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2259, 2388, 0, 0, 0, 0, 0, 0, 0, 3058, 316, 135, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2908, 1205, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4137, 4096, 4647, 3509, 4096, 4113, 4096] avg=4096.0000 max=4647 max_rank=3 min=3509 min_rank=4 imbalance_ratio=1.134521 max_load_reduction=28121 max_load_reduction_pct=0.858185 moved_to_echo=15814 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[321, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 1182, 3037, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1710, 2219, 0, 0, 0, 0, 0, 0, 0, 3578, 296, 135, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2914, 1127, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4282, 4219, 4096, 3929, 4009, 4096, 4041, 4096] avg=4096.0000 max=4282 max_rank=0 min=3929 min_rank=3 imbalance_ratio=1.045410 max_load_reduction=28486 max_load_reduction_pct=0.869324 moved_to_echo=16242 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[43, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 2672, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2458, 2283, 0, 0, 0, 0, 0, 0, 0, 3377, 302, 135, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2908, 1153, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4004, 3860, 4096, 4741, 3814, 4096, 4061, 4096] avg=4096.0000 max=4741 max_rank=3 min=3814 min_rank=4 imbalance_ratio=1.157471 max_load_reduction=28027 max_load_reduction_pct=0.855316 moved_to_echo=16067 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[9, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 3662, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 788, 2375, 0, 0, 0, 0, 0, 0, 0, 3942, 314, 135, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2908, 1198, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3970, 4850, 4096, 3163, 4391, 4096, 4106, 4096] avg=4096.0000 max=4850 max_rank=1 min=3163 min_rank=3 imbalance_ratio=1.184082 max_load_reduction=27918 max_load_reduction_pct=0.851990 moved_to_echo=16689 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[92, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 2777, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1927, 2259, 0, 0, 0, 0, 0, 0, 0, 3795, 299, 135, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2908, 1140, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 3964, 4096, 4186, 4229, 4096, 4048, 4096] avg=4096.0000 max=4229 max_rank=4 min=3964 min_rank=1 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16469 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 23296, 9600, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 14080, 18816, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 9600, 23296, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 29312, 2560, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[1152, 31744, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 4096, 0, 6, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 4091, 0, 9, 4094, 0, 0, 0, 4096, 0, 4, 4096, 4096, 0, 2, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 4096, 0, 5, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4096, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4094, 0, 1, 4096, 0, 0, 0, 4096, 0, 2, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4096, 4092, 4096, 4099, 4096, 4096] avg=4096.0000 max=4099 max_rank=5 min=4092 min_rank=3 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16387 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4096, 4098, 4096, 4093, 4096, 4096] avg=4096.0000 max=4098 max_rank=3 min=4093 min_rank=5 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16381 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 9, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4093, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4089, 4096, 4102, 4094, 4097, 4098, 4096] avg=4096.0000 max=4102 max_rank=3 min=4089 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16385 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4096, 4098, 4096, 4093, 4096, 4096] avg=4096.0000 max=4098 max_rank=3 min=4093 min_rank=5 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16381 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 5, 4092, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4098, 4096, 4093, 4098, 4097, 4094, 4096] avg=4096.0000 max=4098 max_rank=1 min=4093 min_rank=3 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16385 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4096, 4094, 4096, 4097, 4096, 4096] avg=4096.0000 max=4097 max_rank=1 min=4094 min_rank=3 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16385 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4096, 4094, 4096, 4097, 4096, 4096] avg=4096.0000 max=4097 max_rank=1 min=4094 min_rank=3 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16385 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4097, 4096, 4095, 4096, 4096] avg=4096.0000 max=4097 max_rank=3 min=4095 min_rank=5 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16383 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4091, 0, 5, 0, 0, 0, 0, 0, 4091, 2939, 18, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 15, 0, 0, 1129, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 21, 0, 0, 0, 0, 0, 4090, 3295, 3, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 0, 0, 782, 0, 0, 4096, 2, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 0, 0, 0, 4096, 3439, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 15, 0, 0, 642, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 5, 0, 0, 0, 0, 0, 4094, 3111, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 8, 0, 0, 975, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 0, 0, 0, 0, 4092, 2559, 1, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 16, 0, 0, 1523, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3797, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 298, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 4, 0, 0, 0, 0, 0, 4092, 3415, 9, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 32, 0, 0, 642, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 2, 0, 0, 0, 0, 0, 4092, 2673, 17, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 15, 0, 0, 1394, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 938, 3439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3158, 923, 3, 0, 642, 0, 0, 3173, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 4377, 4096, 4096, 4099, 3815, 4096] avg=4096.0000 max=4377 max_rank=2 min=3815 min_rank=6 imbalance_ratio=1.068604 max_load_reduction=28391 max_load_reduction_pct=0.866425 moved_to_echo=16106 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 937, 2939, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3154, 923, 3, 0, 1129, 0, 0, 3173, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 3894, 4096, 4096, 4095, 4302, 4096] avg=4096.0000 max=4302 max_rank=6 min=3894 min_rank=2 imbalance_ratio=1.050293 max_load_reduction=28466 max_load_reduction_pct=0.868713 moved_to_echo=16589 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 936, 3295, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3154, 923, 3, 0, 782, 0, 0, 3173, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4096, 4234, 4096, 4096, 4082, 3955, 4098] avg=4096.0000 max=4234 max_rank=2 min=3955 min_rank=6 imbalance_ratio=1.033691 max_load_reduction=28534 max_load_reduction_pct=0.870789 moved_to_echo=16231 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 938, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3158, 923, 3, 0, 298, 0, 0, 3173, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 4735, 4096, 4096, 4085, 3471, 4096] avg=4096.0000 max=4735 max_rank=2 min=3471 min_rank=6 imbalance_ratio=1.156006 max_load_reduction=28033 max_load_reduction_pct=0.855499 moved_to_echo=15748 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 932, 2559, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 16, 0, 0, 3160, 924, 5, 2, 1523, 0, 0, 3172, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4096, 3492, 4096, 4096, 4107, 4695, 4094] avg=4096.0000 max=4695 max_rank=6 min=3492 min_rank=2 imbalance_ratio=1.146240 max_load_reduction=28073 max_load_reduction_pct=0.856720 moved_to_echo=16992 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 937, 2673, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3155, 923, 3, 0, 1394, 0, 0, 3173, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 3627, 4096, 4096, 4096, 4567, 4096] avg=4096.0000 max=4567 max_rank=6 min=3627 min_rank=2 imbalance_ratio=1.114990 max_load_reduction=28201 max_load_reduction_pct=0.860626 moved_to_echo=16855 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 937, 3111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3157, 923, 3, 0, 975, 0, 0, 3173, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4048, 4096, 4096, 4091, 4148, 4096] avg=4096.0000 max=4148 max_rank=6 min=4048 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16431 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 937, 3415, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 32, 0, 0, 3155, 923, 3, 0, 642, 0, 0, 3173, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4361, 4096, 4096, 4113, 3815, 4096] avg=4096.0000 max=4361 max_rank=2 min=3815 min_rank=6 imbalance_ratio=1.064697 max_load_reduction=28407 max_load_reduction_pct=0.866913 moved_to_echo=16120 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 25344, 7424, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7552, 25344, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[7424, 0, 0, 25472, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 263, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 4093, 0, 0, 1, 0, 3838, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 216, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 3889, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 160, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 3939, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4087, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 4088, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 30, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4066, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 106, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 3990, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 27, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 4095, 0, 0, 1, 0, 4070, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 216, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 1, 0, 0, 0, 0, 3889, 0, 102, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4209, 4096, 4096, 4096, 4096, 4096, 4088, 3991] avg=4096.0000 max=4209 max_rank=0 min=3991 min_rank=7 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16271 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 160, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 0, 3939, 0, 102, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4096, 4096, 4096, 4096, 4096, 4094, 4041] avg=4096.0000 max=4153 max_rank=0 min=4041 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16327 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 263, 0, 0, 3993, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 1, 1, 0, 0, 0, 0, 3838, 0, 102, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4256, 4099, 4096, 4091, 4095, 4096, 4095, 3940] avg=4096.0000 max=4256 max_rank=0 min=3940 min_rank=7 imbalance_ratio=1.039062 max_load_reduction=28512 max_load_reduction_pct=0.870117 moved_to_echo=16226 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4066, 0, 102, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4096, 4095, 4096, 4097, 4096, 4097, 4168] avg=4096.0000 max=4168 max_rank=7 min=4023 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16458 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4087, 0, 87, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4092, 4099, 4103, 4096, 4096, 4101, 4174] avg=4096.0000 max=4174 max_rank=7 min=4007 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16467 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 0, 0, 0, 0, 4070, 0, 102, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4096, 4095, 4096, 4096, 4096, 4097, 4172] avg=4096.0000 max=4172 max_rank=7 min=4020 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16461 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 106, 0, 0, 3993, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0, 0, 0, 3990, 0, 102, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4097, 4095, 4095, 4096, 4096, 4098, 4092] avg=4096.0000 max=4099 max_rank=0 min=4092 min_rank=7 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0, 0, 0, 4088, 0, 102, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4001, 4096, 4096, 4095, 4096, 4096, 4098, 4190] avg=4096.0000 max=4190 max_rank=7 min=4001 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16480 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32000, 0, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 896, 0, 0, 32000, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 1205, 4096, 0, 0, 0, 3854, 0, 0, 0, 3172, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4082, 4096, 1424, 4096, 0, 0, 0, 3727, 0, 0, 0, 3074, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4083, 4096, 287, 4096, 0, 0, 2, 3885, 0, 0, 0, 4038, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4095, 4096, 935, 4096, 0, 0, 0, 3943, 0, 0, 0, 3352, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4086, 4092, 769, 4096, 0, 0, 8, 3890, 0, 0, 3, 3571, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4087, 4094, 426, 4096, 0, 0, 5, 3920, 0, 0, 1, 3858, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4091, 4095, 798, 4096, 0, 0, 3, 3851, 0, 0, 0, 3589, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4080, 4096, 107, 4096, 0, 0, 0, 4042, 0, 0, 0, 4061, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3350, 0, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 3727, 204, 0, 0, 0, 0, 0, 0, 3074, 505, 0, 0, 0, 0, 4077, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4774, 3931, 3579, 4100, 4096, 4096, 4096, 4096] avg=4096.0000 max=4774 max_rank=0 min=3579 min_rank=2 imbalance_ratio=1.165527 max_load_reduction=27994 max_load_reduction_pct=0.854309 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3361, 0, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 3854, 205, 0, 0, 0, 0, 0, 0, 3172, 507, 0, 0, 0, 0, 4057, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4566, 4059, 3679, 4080, 4096, 4096, 4096, 4096] avg=4096.0000 max=4566 max_rank=0 min=3679 min_rank=2 imbalance_ratio=1.114746 max_load_reduction=28202 max_load_reduction_pct=0.860657 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3354, 0, 769, 0, 0, 0, 0, 0, 0, 0, 8, 3890, 204, 0, 0, 0, 0, 0, 3, 3571, 505, 0, 0, 0, 0, 4061, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4102, 4079, 4084, 4092, 4096, 4096, 4096] avg=4096.0000 max=4123 max_rank=0 min=4079 min_rank=2 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16380 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3360, 0, 935, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 205, 0, 0, 0, 0, 0, 0, 3352, 507, 0, 0, 0, 0, 4059, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4295, 4148, 3859, 4082, 4096, 4096, 4096, 4096] avg=4096.0000 max=4295 max_rank=0 min=3859 min_rank=2 imbalance_ratio=1.048584 max_load_reduction=28473 max_load_reduction_pct=0.868927 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3332, 0, 287, 0, 0, 0, 0, 0, 0, 0, 2, 3885, 208, 0, 0, 0, 0, 0, 0, 4038, 508, 0, 0, 0, 0, 4089, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3619, 4095, 4546, 4117, 4103, 4096, 4096, 4096] avg=4096.0000 max=4546 max_rank=2 min=3619 min_rank=0 imbalance_ratio=1.109863 max_load_reduction=28222 max_load_reduction_pct=0.861267 moved_to_echo=16391 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3348, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 204, 0, 0, 0, 0, 0, 0, 4061, 505, 0, 0, 0, 0, 4094, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3455, 4246, 4566, 4117, 4096, 4096, 4096, 4096] avg=4096.0000 max=4566 max_rank=2 min=3455 min_rank=0 imbalance_ratio=1.114746 max_load_reduction=28202 max_load_reduction_pct=0.860657 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3354, 0, 426, 0, 0, 0, 0, 0, 0, 0, 5, 3920, 204, 0, 0, 0, 0, 0, 1, 3858, 506, 0, 0, 0, 0, 4089, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3780, 4129, 4365, 4112, 4094, 4096, 4096, 4096] avg=4096.0000 max=4365 max_rank=2 min=3780 min_rank=0 imbalance_ratio=1.065674 max_load_reduction=28403 max_load_reduction_pct=0.866791 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3358, 0, 798, 0, 0, 0, 0, 0, 0, 0, 3, 3851, 204, 0, 0, 0, 0, 0, 0, 3589, 506, 0, 0, 0, 0, 4053, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4058, 4095, 4076, 4095, 4096, 4096, 4096] avg=4096.0000 max=4156 max_rank=0 min=4058 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16383 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[26880, 0, 6016, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 28800, 4096, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 31232, 1664, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32640, 0, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32768 slack_slots=2688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[546, 1, 1524, 99, 1037, 0, 2665, 23, 1100, 2114, 1, 33, 1196, 3270, 36, 845, 16, 1462, 1784, 0, 0, 997, 1279, 187, 0, 2829, 3284, 1565, 2816, 1979, 80, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[648, 0, 1370, 61, 1793, 5, 2081, 118, 1286, 1447, 0, 32, 1184, 3005, 25, 881, 29, 1333, 1863, 0, 0, 1920, 1603, 254, 0, 3394, 3339, 1321, 1971, 1744, 61, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[506, 0, 1717, 96, 901, 0, 1912, 19, 1110, 2061, 0, 25, 1145, 3521, 13, 1154, 37, 2226, 1953, 0, 0, 1169, 969, 177, 0, 2759, 2603, 1351, 2726, 2556, 62, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[415, 0, 1961, 98, 1507, 2, 2070, 166, 1462, 909, 0, 39, 1611, 2716, 40, 790, 33, 1600, 2010, 0, 0, 1234, 2086, 203, 0, 2772, 2835, 1346, 2530, 2214, 119, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[466, 0, 1969, 68, 764, 0, 2280, 17, 1440, 1248, 1, 32, 2061, 3261, 8, 770, 7, 1670, 1713, 0, 0, 936, 1772, 280, 1, 2898, 3155, 1754, 1909, 2176, 112, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[822, 2, 1931, 81, 1193, 1, 2359, 57, 1487, 2034, 0, 53, 1343, 2877, 7, 1066, 51, 1316, 1535, 0, 2, 1012, 1341, 132, 0, 2997, 3186, 1656, 2284, 1882, 61, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[931, 0, 2078, 53, 990, 0, 2312, 80, 1363, 1543, 0, 22, 1745, 2167, 31, 562, 9, 1570, 1784, 0, 0, 734, 1308, 164, 0, 3314, 3431, 1691, 3039, 1695, 152, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[656, 0, 2071, 58, 1569, 2, 1701, 23, 1436, 955, 5, 35, 1727, 3033, 23, 894, 15, 1294, 2590, 2, 0, 1307, 1508, 326, 2, 2886, 3369, 1268, 2287, 1585, 137, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[546, 1, 1524, 99, 1635, 0, 0, 0, 1037, 0, 2665, 23, 485, 227, 0, 0, 1100, 2114, 1, 33, 918, 364, 0, 0, 1196, 1867, 36, 845, 0, 0, 0, 0, 16, 1462, 1784, 0, 360, 263, 0, 0, 0, 997, 1279, 187, 1285, 0, 0, 0, 0, 2469, 0, 1565, 0, 0, 0, 0, 2326, 1979, 80, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3805, 4437, 4530, 3944, 3885, 3748, 4034, 4385] avg=4096.0000 max=4530 max_rank=2 min=3748 min_rank=5 imbalance_ratio=1.105957 max_load_reduction=28238 max_load_reduction_pct=0.861755 moved_to_echo=16052 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[648, 0, 1370, 61, 1661, 0, 0, 0, 1793, 5, 2081, 118, 445, 159, 0, 0, 1286, 1447, 0, 32, 844, 371, 0, 0, 1184, 1716, 25, 881, 0, 0, 0, 0, 29, 1333, 1863, 0, 432, 184, 0, 0, 0, 1920, 1603, 254, 1307, 0, 0, 0, 0, 2962, 0, 1321, 0, 0, 0, 0, 1628, 1744, 61, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3740, 4601, 3980, 3806, 3841, 5084, 4283, 3433] avg=4096.0000 max=5084 max_rank=5 min=3433 min_rank=7 imbalance_ratio=1.241211 max_load_reduction=27684 max_load_reduction_pct=0.844849 moved_to_echo=16641 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[415, 0, 1961, 98, 1410, 0, 0, 0, 1507, 2, 2070, 166, 402, 204, 0, 0, 1462, 909, 0, 39, 763, 315, 0, 0, 1611, 1551, 40, 790, 0, 0, 0, 0, 33, 1600, 2010, 0, 352, 236, 0, 0, 0, 1234, 2086, 203, 1110, 0, 0, 0, 0, 2420, 0, 1346, 0, 0, 0, 0, 2090, 2214, 119, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3884, 4351, 3488, 3992, 4231, 4633, 3766, 4423] avg=4096.0000 max=4633 max_rank=5 min=3488 min_rank=2 imbalance_ratio=1.131104 max_load_reduction=28135 max_load_reduction_pct=0.858612 moved_to_echo=17053 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[506, 0, 1717, 96, 1295, 0, 0, 0, 901, 0, 1912, 19, 522, 220, 0, 0, 1110, 2061, 0, 25, 989, 289, 0, 0, 1145, 2010, 13, 1154, 0, 0, 0, 0, 37, 2226, 1953, 0, 351, 255, 0, 0, 0, 1169, 969, 177, 1019, 0, 0, 0, 0, 2408, 0, 1351, 0, 0, 0, 0, 2251, 2556, 62, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3614, 3574, 4474, 4322, 4822, 3334, 3759, 4869] avg=4096.0000 max=4869 max_rank=7 min=3334 min_rank=5 imbalance_ratio=1.188721 max_load_reduction=27899 max_load_reduction_pct=0.851410 moved_to_echo=16784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[822, 2, 1931, 81, 1585, 0, 0, 0, 1193, 1, 2359, 57, 426, 184, 0, 0, 1487, 2034, 0, 53, 808, 353, 0, 0, 1343, 1643, 7, 1066, 0, 0, 0, 0, 51, 1316, 1535, 0, 381, 213, 0, 0, 2, 1012, 1341, 132, 1248, 0, 0, 0, 0, 2616, 0, 1656, 0, 0, 0, 0, 1887, 1882, 61, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4421, 4220, 4735, 4059, 3496, 3735, 4272, 3830] avg=4096.0000 max=4735 max_rank=2 min=3496 min_rank=4 imbalance_ratio=1.156006 max_load_reduction=28033 max_load_reduction_pct=0.855499 moved_to_echo=15333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[656, 0, 2071, 58, 1677, 0, 0, 0, 1569, 2, 1701, 23, 454, 190, 0, 0, 1436, 955, 5, 35, 855, 373, 0, 0, 1727, 1724, 23, 894, 0, 0, 0, 0, 15, 1294, 2590, 2, 371, 217, 0, 0, 0, 1307, 1508, 326, 1319, 0, 0, 0, 2, 2515, 0, 1268, 0, 0, 0, 0, 1880, 1585, 137, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4462, 3939, 3659, 4368, 4489, 4460, 3785, 3606] avg=4096.0000 max=4489 max_rank=4 min=3606 min_rank=7 imbalance_ratio=1.095947 max_load_reduction=28279 max_load_reduction_pct=0.863007 moved_to_echo=16340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[466, 0, 1969, 68, 1569, 0, 0, 0, 764, 0, 2280, 17, 483, 154, 0, 0, 1440, 1248, 1, 32, 916, 349, 0, 0, 2061, 1862, 8, 770, 0, 0, 0, 0, 7, 1670, 1713, 0, 368, 178, 0, 0, 0, 936, 1772, 280, 1237, 0, 0, 0, 1, 2530, 0, 1754, 0, 0, 0, 0, 1577, 2176, 112, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 3698, 3986, 4701, 3936, 4225, 4285, 3865] avg=4096.0000 max=4701 max_rank=3 min=3698 min_rank=1 imbalance_ratio=1.147705 max_load_reduction=28067 max_load_reduction_pct=0.856537 moved_to_echo=16311 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[931, 0, 2078, 53, 1708, 0, 0, 0, 990, 0, 2312, 80, 321, 245, 0, 0, 1363, 1543, 0, 22, 608, 380, 0, 0, 1745, 1238, 31, 562, 0, 0, 0, 0, 9, 1570, 1784, 0, 421, 284, 0, 0, 0, 734, 1308, 164, 1343, 0, 0, 0, 0, 2893, 0, 1691, 0, 0, 0, 0, 2510, 1695, 152, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4770, 3948, 3916, 3576, 4068, 3549, 4584, 4357] avg=4096.0000 max=4770 max_rank=0 min=3549 min_rank=5 imbalance_ratio=1.164551 max_load_reduction=27998 max_load_reduction_pct=0.854431 moved_to_echo=16558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10752, 12416, 128, 384, 6784, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4992, 128, 14720, 640, 12544, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 9344, 11904, 1792, 9984, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9856, 128, 17408, 512, 3584, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 12544, 15232, 128, 3072, 1920, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12032, 13696, 256, 7040, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16256, 15872, 896, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 20864, 0, 12032, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[754, 0, 2, 0, 2267, 0, 0, 0, 0, 1570, 4085, 2, 2489, 6, 4088, 4096, 4093, 0, 0, 0, 517, 0, 0, 0, 4095, 603, 0, 3, 0, 4096, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[19, 2, 0, 0, 542, 0, 5, 0, 4, 1973, 4057, 1, 2035, 5, 4084, 4083, 4091, 5, 4, 2, 1003, 0, 5, 0, 4091, 2660, 4, 3, 0, 4088, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[28, 0, 0, 5, 50, 0, 0, 0, 0, 1858, 4080, 3, 1562, 2, 4096, 4085, 4096, 0, 0, 0, 1560, 0, 0, 0, 4090, 3159, 0, 0, 0, 4093, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[33, 0, 0, 0, 764, 0, 1, 0, 0, 1883, 4096, 0, 2034, 0, 4083, 4095, 4095, 0, 1, 0, 781, 1, 0, 0, 4095, 2720, 0, 0, 0, 4086, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[455, 0, 0, 0, 2088, 0, 0, 0, 0, 2417, 4096, 0, 2780, 0, 4091, 4075, 4096, 0, 0, 0, 283, 0, 0, 0, 4096, 196, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[252, 0, 0, 0, 1314, 0, 0, 0, 0, 1608, 4072, 0, 2307, 0, 4095, 4090, 4096, 0, 0, 0, 1048, 0, 0, 0, 4095, 1695, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[231, 0, 0, 0, 2717, 0, 0, 0, 0, 1846, 4096, 0, 2356, 0, 4078, 4096, 4096, 0, 0, 0, 384, 0, 0, 0, 4095, 677, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[480, 0, 1, 0, 1567, 0, 0, 0, 0, 1641, 4096, 0, 2327, 0, 4096, 4092, 4096, 0, 0, 0, 838, 0, 0, 0, 4096, 1345, 0, 0, 0, 4093, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 0, 5, 3809, 0, 0, 0, 50, 0, 0, 0, 1627, 1051, 0, 0, 0, 1858, 2243, 3, 0, 0, 0, 0, 1562, 2, 1862, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1560, 0, 0, 0, 2234, 786, 276, 0, 2461, 3159, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[3842, 2728, 4104, 3426, 4096, 4856, 5620, 4096] avg=4096.0000 max=5620 max_rank=6 min=2728 min_rank=1 imbalance_ratio=1.372070 max_load_reduction=27148 max_load_reduction_pct=0.828491 moved_to_echo=18668 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[754, 0, 2, 0, 3820, 0, 0, 0, 2267, 0, 0, 0, 1629, 1052, 0, 0, 0, 1570, 2246, 2, 0, 0, 0, 0, 2489, 6, 1858, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 2230, 787, 276, 0, 2464, 603, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 2, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4576, 4948, 3818, 4353, 4093, 3810, 3070, 4100] avg=4096.0000 max=4948 max_rank=1 min=3070 min_rank=6 imbalance_ratio=1.208008 max_load_reduction=27820 max_load_reduction_pct=0.848999 moved_to_echo=15073 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[33, 0, 0, 0, 3819, 0, 0, 0, 764, 0, 1, 0, 1629, 1055, 0, 0, 0, 1883, 2252, 0, 0, 0, 0, 0, 2034, 0, 1856, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 781, 1, 0, 0, 2227, 789, 276, 0, 2464, 2720, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[3852, 3449, 4135, 3890, 4096, 4074, 5184, 4088] avg=4096.0000 max=5184 max_rank=6 min=3449 min_rank=1 imbalance_ratio=1.265625 max_load_reduction=27584 max_load_reduction_pct=0.841797 moved_to_echo=17442 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[455, 0, 0, 0, 3800, 0, 0, 0, 2088, 0, 0, 0, 1629, 1055, 0, 0, 0, 2417, 2252, 0, 0, 0, 0, 0, 2780, 0, 1860, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 283, 0, 0, 0, 2231, 789, 275, 0, 2465, 196, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4255, 4772, 4669, 4640, 4096, 3578, 2661, 4097] avg=4096.0000 max=4772 max_rank=1 min=2661 min_rank=6 imbalance_ratio=1.165039 max_load_reduction=27996 max_load_reduction_pct=0.854370 moved_to_echo=14432 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[19, 2, 0, 0, 3808, 0, 0, 0, 542, 0, 5, 0, 1632, 1048, 0, 0, 4, 1973, 2226, 1, 0, 0, 0, 0, 2035, 5, 1852, 0, 0, 0, 0, 0, 4088, 5, 4, 2, 0, 0, 0, 0, 1003, 0, 5, 0, 2232, 783, 275, 0, 2456, 2660, 4, 3, 0, 0, 0, 0, 0, 4088, 2, 0, 3, 3, 0, 0] rerouted_tokens_per_ep_rank=[3829, 3227, 4204, 3892, 4099, 4298, 5123, 4096] avg=4096.0000 max=5123 max_rank=6 min=3227 min_rank=1 imbalance_ratio=1.250732 max_load_reduction=27645 max_load_reduction_pct=0.843658 moved_to_echo=17616 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[480, 0, 1, 0, 3816, 0, 0, 0, 1567, 0, 0, 0, 1629, 1055, 0, 0, 0, 1641, 2252, 0, 0, 0, 0, 0, 2327, 0, 1862, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 838, 0, 0, 0, 2234, 789, 276, 0, 2465, 1345, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4297, 4251, 3893, 4189, 4096, 4137, 3810, 4095] avg=4096.0000 max=4297 max_rank=0 min=3810 min_rank=6 imbalance_ratio=1.049072 max_load_reduction=28471 max_load_reduction_pct=0.868866 moved_to_echo=16138 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[252, 0, 0, 0, 3815, 0, 0, 0, 1314, 0, 0, 0, 1629, 1049, 0, 0, 0, 1608, 2239, 0, 0, 0, 0, 0, 2307, 0, 1861, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 0, 0, 2234, 784, 275, 0, 2464, 1695, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 3992, 3847, 4168, 4096, 4341, 4159, 4098] avg=4096.0000 max=4341 max_rank=5 min=3847 min_rank=2 imbalance_ratio=1.059814 max_load_reduction=28427 max_load_reduction_pct=0.867523 moved_to_echo=16694 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[231, 0, 0, 0, 3819, 0, 0, 0, 2717, 0, 0, 0, 1629, 1055, 0, 0, 0, 1846, 2252, 0, 0, 0, 0, 0, 2356, 0, 1854, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 2224, 789, 277, 0, 2464, 677, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 5401, 4098, 4210, 4096, 3674, 3141, 4098] avg=4096.0000 max=5401 max_rank=1 min=3141 min_rank=6 imbalance_ratio=1.318604 max_load_reduction=27367 max_load_reduction_pct=0.835175 moved_to_echo=15009 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11392, 0, 128, 0, 13056, 8448, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[19712, 13056, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[6528, 128, 128, 0, 17920, 6400, 2304, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 14848, 18048, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17920, 128, 14976, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2304, 128, 128, 128, 30592, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 1126, 0, 1, 0, 4095, 6, 3084, 4096, 1790, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4095, 2147, 3801, 0, 0, 0, 192, 0, 137] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1331, 0, 0, 0, 4096, 3, 3561, 4096, 1172, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1626, 3969, 0, 0, 0, 358, 0, 268] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 349, 0, 0, 0, 4096, 10, 1866, 4096, 347, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3683, 4024, 0, 0, 0, 40, 0, 1969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 176, 5, 0, 0, 4090, 1, 3434, 4060, 399, 1, 4092, 5, 15, 0, 5, 5, 0, 6, 3, 0, 4091, 4090, 3270, 4066, 0, 1, 5, 82, 1, 865] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 331, 0, 0, 0, 4096, 0, 3193, 4095, 953, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3220, 3963, 0, 0, 0, 95, 0, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 418, 0, 0, 1, 4095, 0, 3441, 4096, 489, 1, 4095, 0, 2, 0, 1, 0, 0, 0, 0, 0, 4095, 4094, 2937, 4085, 3, 0, 0, 176, 0, 738] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 270, 0, 0, 0, 4096, 5, 3826, 4096, 691, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 3048, 4075, 0, 0, 0, 77, 0, 294] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1, 1, 574, 0, 0, 0, 4096, 20, 3299, 4095, 1168, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 2921, 4049, 0, 0, 0, 132, 0, 123] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 1126, 0, 3525, 2, 0, 0, 1, 0, 4091, 6, 0, 0, 0, 0, 3084, 5, 1790, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2147, 1177, 0, 0, 0, 0, 0, 0, 0, 192, 0, 137, 2624, 566, 4, 2] rerouted_tokens_per_ep_rank=[4659, 4098, 4879, 4094, 4094, 4095, 3324, 3525] avg=4096.0000 max=4879 max_rank=2 min=3324 min_rank=6 imbalance_ratio=1.191162 max_load_reduction=27889 max_load_reduction_pct=0.851105 moved_to_echo=15038 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 349, 0, 3525, 0, 0, 0, 0, 0, 4092, 10, 0, 0, 0, 0, 1866, 8, 347, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3683, 1246, 0, 0, 0, 0, 0, 0, 0, 40, 0, 1969, 2778, 563, 4, 2] rerouted_tokens_per_ep_rank=[3874, 4102, 2221, 4094, 4096, 4096, 4929, 5356] avg=4096.0000 max=5356 max_rank=7 min=2221 min_rank=2 imbalance_ratio=1.307617 max_load_reduction=27412 max_load_reduction_pct=0.836548 moved_to_echo=18477 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1331, 0, 3525, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 3561, 8, 1172, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 1626, 1229, 0, 0, 0, 0, 0, 0, 0, 358, 0, 268, 2740, 563, 4, 2] rerouted_tokens_per_ep_rank=[4856, 4095, 4741, 4094, 4095, 4097, 2855, 3935] avg=4096.0000 max=4856 max_rank=0 min=2855 min_rank=6 imbalance_ratio=1.185547 max_load_reduction=27912 max_load_reduction_pct=0.851807 moved_to_echo=14982 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 418, 0, 3525, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 3441, 8, 489, 1, 0, 0, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2937, 1265, 3, 0, 0, 0, 0, 0, 0, 176, 0, 738, 2820, 563, 4, 2] rerouted_tokens_per_ep_rank=[3944, 4092, 3939, 4095, 4096, 4094, 4205, 4303] avg=4096.0000 max=4303 max_rank=7 min=3939 min_rank=2 imbalance_ratio=1.050537 max_load_reduction=28465 max_load_reduction_pct=0.868683 moved_to_echo=16698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 574, 0, 3524, 0, 0, 0, 0, 0, 4092, 20, 0, 0, 0, 0, 3299, 8, 1168, 0, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 2921, 1253, 0, 0, 0, 0, 0, 0, 0, 132, 0, 123, 2796, 563, 4, 2] rerouted_tokens_per_ep_rank=[4100, 4112, 4475, 4095, 4094, 4098, 4174, 3620] avg=4096.0000 max=4475 max_rank=2 min=3620 min_rank=7 imbalance_ratio=1.092529 max_load_reduction=28293 max_load_reduction_pct=0.863434 moved_to_echo=15986 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 176, 5, 3499, 2, 0, 0, 0, 0, 4079, 1, 0, 0, 0, 0, 3434, 0, 399, 1, 0, 0, 0, 0, 4088, 5, 15, 0, 0, 0, 0, 0, 5, 5, 0, 6, 4089, 0, 0, 0, 3, 0, 0, 4090, 0, 0, 0, 0, 3270, 1254, 0, 1, 0, 0, 0, 0, 5, 82, 1, 865, 2812, 561, 11, 4] rerouted_tokens_per_ep_rank=[3682, 4080, 3834, 4108, 4105, 4093, 4525, 4341] avg=4096.0000 max=4525 max_rank=6 min=3682 min_rank=0 imbalance_ratio=1.104736 max_load_reduction=28243 max_load_reduction_pct=0.861908 moved_to_echo=17064 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 331, 0, 3524, 2, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 3193, 8, 953, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3220, 1227, 0, 0, 0, 0, 0, 0, 0, 95, 0, 534, 2736, 563, 4, 2] rerouted_tokens_per_ep_rank=[3857, 4092, 4154, 4094, 4094, 4096, 4447, 3934] avg=4096.0000 max=4447 max_rank=6 min=3857 min_rank=0 imbalance_ratio=1.085693 max_load_reduction=28321 max_load_reduction_pct=0.864288 moved_to_echo=16571 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 270, 0, 3525, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 3826, 8, 691, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 2, 4096, 0, 0, 0, 0, 3048, 1261, 0, 0, 0, 0, 0, 0, 0, 77, 0, 294, 2814, 563, 4, 2] rerouted_tokens_per_ep_rank=[3796, 4097, 4525, 4094, 4094, 4099, 4309, 3754] avg=4096.0000 max=4525 max_rank=2 min=3754 min_rank=7 imbalance_ratio=1.104736 max_load_reduction=28243 max_load_reduction_pct=0.861908 moved_to_echo=16256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[22912, 9984, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 1152, 128, 4992, 22144, 4608, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 4608, 128, 28288, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[25728, 128, 7040, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4012, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 456, 0, 4096, 0, 0, 2312, 4096, 0, 2893, 342, 10, 0, 0, 188, 2075] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 4009, 1, 4094, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 201, 0, 4096, 1, 0, 2925, 4096, 1, 3365, 207, 10, 0, 0, 213, 1357] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4069, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 840, 0, 4095, 0, 0, 2783, 4096, 20, 1342, 177, 2, 0, 0, 115, 2945] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[11, 0, 0, 5, 3, 4088, 0, 4091, 0, 4091, 5, 0, 5, 0, 6, 0, 4090, 182, 6, 4091, 0, 4, 3011, 4082, 3, 2745, 302, 47, 0, 2, 43, 1855] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4049, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 239, 0, 4096, 0, 0, 3440, 4096, 10, 2542, 116, 0, 0, 0, 71, 1826] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4049, 0, 4085, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 611, 0, 4096, 0, 0, 2300, 4096, 6, 2998, 320, 12, 1, 0, 229, 1774] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4066, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 233, 0, 4086, 0, 0, 2676, 4096, 5, 1858, 324, 0, 0, 0, 195, 2941] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4071, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 608, 0, 4093, 0, 0, 3344, 4096, 1, 1781, 197, 1, 2, 0, 139, 2147] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4052, 44, 1, 0, 0, 4012, 0, 44, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 456, 0, 3677, 0, 0, 0, 0, 0, 0, 2312, 1248, 0, 0, 0, 0, 0, 2893, 342, 10, 972, 419, 0, 0, 0, 0, 188, 2075, 1832, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4056, 4096, 4095, 4133, 3560, 4636, 4095] avg=4096.0000 max=4636 max_rank=6 min=3560 min_rank=5 imbalance_ratio=1.131836 max_load_reduction=28132 max_load_reduction_pct=0.858521 moved_to_echo=16424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4050, 44, 1, 0, 0, 4009, 1, 44, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 201, 0, 3677, 0, 0, 0, 0, 1, 0, 2925, 1248, 0, 0, 0, 0, 1, 3365, 207, 10, 972, 419, 0, 0, 0, 0, 213, 1357, 1832, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4054, 4095, 4095, 3878, 4174, 4974, 3402] avg=4096.0000 max=4974 max_rank=6 min=3402 min_rank=7 imbalance_ratio=1.214355 max_load_reduction=27794 max_load_reduction_pct=0.848206 moved_to_echo=16428 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4047, 44, 1, 0, 0, 4049, 0, 44, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 239, 0, 3677, 0, 0, 0, 0, 0, 0, 3440, 1248, 0, 0, 0, 0, 10, 2542, 116, 0, 972, 419, 0, 0, 0, 0, 71, 1826, 1832, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4093, 4096, 4095, 3916, 4688, 4059, 3729] avg=4096.0000 max=4688 max_rank=5 min=3729 min_rank=7 imbalance_ratio=1.144531 max_load_reduction=28080 max_load_reduction_pct=0.856934 moved_to_echo=16392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4052, 44, 0, 0, 0, 4066, 0, 44, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 233, 0, 3668, 0, 0, 0, 0, 0, 0, 2676, 1248, 0, 0, 0, 0, 5, 1858, 324, 0, 972, 418, 0, 0, 0, 0, 195, 2941, 1832, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4110, 4096, 4096, 3901, 3924, 3577, 4968] avg=4096.0000 max=4968 max_rank=7 min=3577 min_rank=6 imbalance_ratio=1.212891 max_load_reduction=27800 max_load_reduction_pct=0.848389 moved_to_echo=16370 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 0, 5, 4048, 46, 1, 0, 3, 4088, 0, 43, 0, 0, 0, 0, 0, 4091, 5, 0, 4, 0, 0, 0, 5, 0, 6, 0, 4089, 0, 0, 0, 0, 182, 6, 3666, 0, 0, 0, 0, 0, 4, 3011, 1236, 0, 0, 0, 0, 3, 2745, 302, 47, 974, 421, 0, 0, 0, 2, 43, 1855, 1826, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4134, 4100, 4100, 3854, 4251, 4492, 3726] avg=4096.0000 max=4492 max_rank=6 min=3726 min_rank=7 imbalance_ratio=1.096680 max_load_reduction=28276 max_load_reduction_pct=0.862915 moved_to_echo=16323 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4050, 44, 1, 0, 0, 4069, 0, 44, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 840, 0, 3676, 0, 0, 0, 0, 0, 0, 2783, 1248, 0, 0, 0, 0, 20, 1342, 177, 2, 972, 419, 0, 0, 0, 0, 115, 2945, 1832, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4113, 4094, 4095, 4516, 4031, 2932, 4892] avg=4096.0000 max=4892 max_rank=7 min=2932 min_rank=6 imbalance_ratio=1.194336 max_load_reduction=27876 max_load_reduction_pct=0.850708 moved_to_echo=16371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4041, 44, 0, 0, 0, 4049, 0, 44, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 611, 0, 3677, 0, 0, 0, 0, 0, 0, 2300, 1248, 0, 0, 0, 0, 6, 2998, 320, 12, 972, 419, 0, 0, 1, 0, 229, 1774, 1832, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4093, 4095, 4096, 4288, 3548, 4727, 3836] avg=4096.0000 max=4727 max_rank=6 min=3548 min_rank=5 imbalance_ratio=1.154053 max_load_reduction=28041 max_load_reduction_pct=0.855743 moved_to_echo=16399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4052, 44, 0, 0, 0, 4071, 0, 44, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 608, 0, 3674, 0, 0, 0, 0, 0, 0, 3344, 1248, 0, 0, 0, 0, 1, 1781, 197, 1, 972, 419, 0, 0, 2, 0, 139, 2147, 1832, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4115, 4096, 4096, 4282, 4592, 3371, 4120] avg=4096.0000 max=4592 max_rank=5 min=3371 min_rank=6 imbalance_ratio=1.121094 max_load_reduction=28176 max_load_reduction_pct=0.859863 moved_to_echo=16365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 1280, 17024, 14720, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 19584, 2048, 128, 7808, 3456, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 22912, 9984, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 32512, 384, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 3456, 128, 29440, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32512, 128, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 2133, 0, 0, 0, 0, 0, 0, 1964, 0, 6, 4095, 0, 4082, 0, 0, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4093, 4095, 0, 0, 1, 0, 4093, 0, 0, 0, 4096, 1106, 0, 0, 0, 0, 0, 0, 2998, 0, 1, 4091, 0, 4096, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3065, 0, 0, 0, 0, 0, 0, 1035, 0, 0, 4095, 0, 4091, 0, 1, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[5, 4, 7, 4089, 0, 4083, 4090, 0, 5, 0, 0, 4091, 5, 2, 3, 4090, 1382, 1, 0, 0, 5, 20, 0, 2695, 1, 0, 4090, 5, 4082, 6, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3799, 0, 0, 0, 0, 0, 0, 300, 0, 0, 4096, 0, 4089, 0, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 4092, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 2239, 0, 0, 0, 0, 0, 0, 1867, 0, 2, 4096, 0, 4088, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 2574, 0, 0, 0, 0, 0, 0, 1523, 0, 5, 4096, 0, 4091, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 3014, 0, 0, 0, 0, 0, 0, 1080, 0, 6, 4096, 0, 4091, 0, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2133, 0, 0, 0, 1682, 0, 0, 0, 0, 0, 0, 1964, 2410, 0, 0, 0, 0, 6, 4094, 0, 0, 0, 0, 0, 4082, 0, 0, 13, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4100, 4092, 4096, 3815, 4374, 4100, 4095] avg=4096.0000 max=4374 max_rank=5 min=3815 min_rank=4 imbalance_ratio=1.067871 max_load_reduction=28394 max_load_reduction_pct=0.866516 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1106, 0, 0, 0, 1681, 0, 0, 0, 0, 0, 0, 2998, 2410, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4095, 4096, 2787, 5408, 4091, 4098] avg=4096.0000 max=5408 max_rank=5 min=2787 min_rank=4 imbalance_ratio=1.320312 max_load_reduction=27360 max_load_reduction_pct=0.834961 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3799, 0, 0, 0, 1682, 0, 0, 0, 0, 0, 0, 300, 2410, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4089, 0, 0, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4098, 4097, 4096, 5481, 2710, 4095, 4095] avg=4096.0000 max=5481 max_rank=4 min=2710 min_rank=5 imbalance_ratio=1.338135 max_load_reduction=27287 max_load_reduction_pct=0.832733 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2574, 0, 0, 0, 1682, 0, 0, 0, 0, 0, 0, 1523, 2410, 0, 0, 0, 0, 5, 4095, 0, 0, 0, 0, 0, 4091, 0, 0, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4099, 4093, 4096, 4256, 3933, 4100, 4095] avg=4096.0000 max=4256 max_rank=4 min=3933 min_rank=5 imbalance_ratio=1.039062 max_load_reduction=28512 max_load_reduction_pct=0.870117 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 4, 7, 4082, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4091, 6, 6, 0, 0, 5, 2, 3, 4086, 0, 0, 0, 0, 1382, 1, 0, 0, 1682, 0, 0, 0, 5, 20, 0, 2695, 2408, 0, 0, 0, 1, 0, 4084, 5, 0, 0, 0, 0, 4082, 6, 7, 0, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4083, 4108, 4096, 3065, 5128, 4090, 4100] avg=4096.0000 max=5128 max_rank=5 min=3065 min_rank=4 imbalance_ratio=1.251953 max_load_reduction=27640 max_load_reduction_pct=0.843506 moved_to_echo=16383 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3065, 0, 0, 0, 1682, 0, 0, 0, 0, 0, 0, 1035, 2410, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4091, 0, 1, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4098, 4097, 4096, 4747, 3445, 4094, 4095] avg=4096.0000 max=4747 max_rank=4 min=3445 min_rank=5 imbalance_ratio=1.158936 max_load_reduction=28021 max_load_reduction_pct=0.855133 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2239, 0, 0, 0, 1682, 0, 0, 0, 0, 0, 0, 1867, 2410, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 4088, 0, 0, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4095, 4096, 3921, 4277, 4097, 4092] avg=4096.0000 max=4277 max_rank=5 min=3921 min_rank=4 imbalance_ratio=1.044189 max_load_reduction=28491 max_load_reduction_pct=0.869476 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3014, 0, 0, 0, 1682, 0, 0, 0, 0, 0, 0, 1080, 2413, 0, 0, 0, 0, 6, 4095, 0, 0, 0, 0, 0, 4091, 0, 0, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4091, 4096, 4696, 3493, 4101, 4098] avg=4096.0000 max=4696 max_rank=4 min=3493 min_rank=5 imbalance_ratio=1.146484 max_load_reduction=28072 max_load_reduction_pct=0.856689 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 13568, 19328, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[19328, 128, 0, 0, 13568, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 1586, 4077, 2126, 0, 0, 4068, 0, 0, 0, 377, 0, 4096, 0, 4091, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 57, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2495, 4076, 717, 0, 0, 3989, 0, 0, 0, 1008, 0, 4096, 0, 4094, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[2, 4085, 3, 0, 2177, 4091, 1751, 5, 6, 4091, 2, 5, 0, 106, 0, 4087, 4, 4069, 33, 5, 4089, 1, 2, 4090, 3, 5, 5, 45, 0, 6, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2134, 4091, 1829, 0, 0, 4076, 0, 0, 0, 130, 1, 4095, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 29, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2104, 4092, 1209, 0, 0, 4094, 0, 0, 0, 779, 0, 4096, 0, 4092, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 2547, 4094, 1128, 0, 0, 4082, 0, 0, 0, 415, 0, 4096, 0, 4092, 0, 0, 4095, 1, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3360, 4083, 604, 0, 0, 4086, 0, 0, 0, 145, 0, 4096, 0, 4095, 0, 0, 4094, 7, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2947, 4063, 846, 0, 0, 4024, 0, 0, 0, 395, 2, 4096, 0, 4096, 0, 0, 4096, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 1586, 400, 2126, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 29, 1, 0, 0, 0, 377, 0, 3678, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 3677, 389, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4112, 4098, 4055, 4091, 4096, 4123, 4096] avg=4096.0000 max=4123 max_rank=6 min=4055 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2495, 400, 717, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 29, 1, 0, 0, 0, 1008, 0, 3678, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3676, 389, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3612, 4019, 4686, 4094, 4096, 4069, 4096] avg=4096.0000 max=4686 max_rank=3 min=3612 min_rank=1 imbalance_ratio=1.144043 max_load_reduction=28082 max_load_reduction_pct=0.856995 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2134, 402, 1829, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 29, 1, 0, 0, 0, 130, 1, 3677, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 3689, 389, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4365, 4106, 3808, 4095, 4095, 4107, 4096] avg=4096.0000 max=4365 max_rank=1 min=3808 min_rank=3 imbalance_ratio=1.065674 max_load_reduction=28403 max_load_reduction_pct=0.866791 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 4085, 3, 0, 4, 0, 0, 0, 2177, 397, 1751, 5, 0, 0, 0, 0, 6, 4091, 2, 5, 33, 2, 0, 0, 0, 106, 0, 3665, 0, 0, 0, 0, 4, 4069, 33, 5, 2, 0, 0, 0, 4081, 1, 2, 0, 0, 0, 0, 0, 3, 5, 5, 45, 3694, 389, 0, 0, 0, 6, 0, 0, 4090, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4330, 4139, 3771, 4113, 4084, 4141, 4096] avg=4096.0000 max=4330 max_rank=1 min=3771 min_rank=3 imbalance_ratio=1.057129 max_load_reduction=28438 max_load_reduction_pct=0.867859 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3360, 401, 604, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 29, 1, 0, 0, 0, 145, 0, 3678, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3682, 389, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4365, 4116, 3823, 4095, 4100, 4077, 4096] avg=4096.0000 max=4365 max_rank=1 min=3823 min_rank=3 imbalance_ratio=1.065674 max_load_reduction=28403 max_load_reduction_pct=0.866791 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2947, 399, 846, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 29, 1, 0, 0, 0, 395, 2, 3678, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3664, 389, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4192, 4054, 4075, 4096, 4106, 4053, 4096] avg=4096.0000 max=4192 max_rank=1 min=4053 min_rank=6 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 2547, 402, 1128, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 29, 1, 0, 0, 0, 415, 0, 3678, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 3692, 389, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4077, 4112, 4093, 4092, 4095, 4106, 4096] avg=4096.0000 max=4112 max_rank=2 min=4077 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2104, 402, 1209, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 29, 1, 0, 0, 0, 779, 0, 3678, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3690, 389, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3715, 4124, 4457, 4092, 4096, 4092, 4096] avg=4096.0000 max=4457 max_rank=3 min=3715 min_rank=1 imbalance_ratio=1.088135 max_load_reduction=28311 max_load_reduction_pct=0.863983 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19456, 3328, 10240, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 3456, 128, 29440, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 256, 29568, 3200, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32512, 128, 128, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 195, 0, 4096, 2447, 0, 0, 0, 0, 4084, 62, 0, 11, 3, 0, 4096, 4093, 1394, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 457, 0, 4096, 2914, 0, 0, 0, 0, 3982, 28, 0, 12, 0, 0, 4096, 4096, 804, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 237, 0, 4096, 2352, 0, 0, 0, 0, 4083, 19, 0, 3, 0, 0, 4096, 4096, 1498, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[5, 113, 0, 4090, 2216, 6, 0, 8, 10, 4089, 10, 5, 0, 0, 6, 4084, 4087, 1752, 4084, 0, 6, 7, 4090, 2, 9, 0, 0, 5, 0, 4083, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 254, 0, 4096, 2998, 0, 0, 0, 0, 4090, 90, 0, 2, 0, 0, 4096, 4091, 763, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 104, 0, 4096, 3466, 0, 0, 0, 0, 4068, 23, 0, 6, 1, 0, 4096, 4096, 525, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 47, 0, 4096, 2560, 0, 0, 0, 0, 4093, 26, 0, 5, 0, 0, 4096, 4091, 1466, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 102, 0, 4096, 2509, 0, 0, 0, 0, 4086, 2, 0, 9, 1, 0, 4096, 4095, 1484, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 195, 0, 3908, 0, 0, 0, 0, 2447, 0, 0, 0, 1207, 188, 10, 4, 0, 4074, 62, 0, 0, 0, 0, 0, 11, 3, 0, 4091, 0, 0, 0, 0, 2886, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4103, 3856, 4136, 4105, 4280, 4095, 4096, 4097] avg=4096.0000 max=4280 max_rank=4 min=3856 min_rank=1 imbalance_ratio=1.044922 max_load_reduction=28488 max_load_reduction_pct=0.869385 moved_to_echo=16568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 457, 0, 3908, 0, 0, 0, 0, 2914, 0, 0, 0, 1208, 188, 10, 4, 0, 3972, 28, 0, 0, 0, 0, 0, 12, 0, 0, 4091, 0, 0, 0, 0, 2888, 804, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4365, 4324, 4000, 4103, 3692, 4095, 4096, 4093] avg=4096.0000 max=4365 max_rank=0 min=3692 min_rank=4 imbalance_ratio=1.065674 max_load_reduction=28403 max_load_reduction_pct=0.866791 moved_to_echo=15976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 104, 0, 3908, 0, 0, 0, 0, 3466, 0, 0, 0, 1208, 188, 10, 4, 0, 4058, 23, 0, 0, 0, 0, 0, 6, 1, 0, 4091, 0, 0, 0, 0, 2888, 525, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4876, 4081, 4098, 3413, 4095, 4096, 4097] avg=4096.0000 max=4876 max_rank=1 min=3413 min_rank=4 imbalance_ratio=1.190430 max_load_reduction=27892 max_load_reduction_pct=0.851196 moved_to_echo=15701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 254, 0, 3908, 0, 0, 0, 0, 2998, 0, 0, 0, 1207, 188, 10, 4, 0, 4080, 90, 0, 0, 0, 0, 0, 2, 0, 0, 4091, 0, 0, 0, 0, 2884, 763, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4407, 4170, 4093, 3647, 4095, 4096, 4098] avg=4096.0000 max=4407 max_rank=1 min=3647 min_rank=4 imbalance_ratio=1.075928 max_load_reduction=28361 max_load_reduction_pct=0.865509 moved_to_echo=15936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 113, 0, 3898, 0, 0, 0, 0, 2216, 6, 0, 8, 1210, 192, 12, 9, 10, 4077, 10, 5, 0, 0, 0, 0, 0, 0, 6, 4072, 0, 0, 0, 0, 2877, 1752, 0, 0, 0, 0, 0, 0, 6, 7, 4088, 2, 0, 0, 0, 0, 9, 0, 0, 5, 4084, 0, 0, 0, 0, 4083, 1, 0, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4016, 3653, 4102, 4078, 4629, 4103, 4098, 4089] avg=4096.0000 max=4629 max_rank=4 min=3653 min_rank=1 imbalance_ratio=1.130127 max_load_reduction=28139 max_load_reduction_pct=0.858734 moved_to_echo=16919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 102, 0, 3908, 0, 0, 0, 0, 2509, 0, 0, 0, 1208, 188, 10, 4, 0, 4076, 2, 0, 0, 0, 0, 0, 9, 1, 0, 4091, 0, 0, 0, 0, 2887, 1484, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4010, 3919, 4078, 4101, 4372, 4095, 4095, 4098] avg=4096.0000 max=4372 max_rank=4 min=3919 min_rank=1 imbalance_ratio=1.067383 max_load_reduction=28396 max_load_reduction_pct=0.866577 moved_to_echo=16660 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 237, 0, 3908, 0, 0, 0, 0, 2352, 0, 0, 0, 1208, 188, 10, 4, 0, 4073, 19, 0, 0, 0, 0, 0, 3, 0, 0, 4091, 0, 0, 0, 0, 2888, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4145, 3762, 4092, 4094, 4386, 4095, 4096, 4098] avg=4096.0000 max=4386 max_rank=4 min=3762 min_rank=1 imbalance_ratio=1.070801 max_load_reduction=28382 max_load_reduction_pct=0.866150 moved_to_echo=16675 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 47, 0, 3908, 0, 0, 0, 0, 2560, 0, 0, 0, 1207, 188, 10, 4, 0, 4083, 26, 0, 0, 0, 0, 0, 5, 0, 0, 4091, 0, 0, 0, 0, 2884, 1466, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[3955, 3969, 4109, 4096, 4351, 4095, 4095, 4098] avg=4096.0000 max=4351 max_rank=4 min=3955 min_rank=0 imbalance_ratio=1.062256 max_load_reduction=28417 max_load_reduction_pct=0.867218 moved_to_echo=16639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32770 actual_valid_recv=33024 observed_loss=-254 tokens_per_expert=[23168, 9728, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32766 actual_valid_recv=33280 observed_loss=-514 tokens_per_expert=[21504, 128, 0, 128, 9728, 1536, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 1536, 0, 31360, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32512, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3312, 4095, 0, 998, 0, 0, 4096, 4096, 0, 3273, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2456, 4093, 0, 1034, 0, 0, 4096, 4096, 0, 1565, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 3137, 0, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[5, 7, 3805, 4087, 3, 690, 0, 6, 4076, 4090, 6, 2187, 4, 0, 19, 0, 4088, 0, 6, 4089, 4064, 1, 1, 6, 0, 1518, 0, 4, 0, 0, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3437, 4095, 0, 1207, 0, 0, 4096, 4096, 0, 2625, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 923, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2590, 4096, 0, 1339, 0, 0, 4096, 4096, 0, 2651, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1612, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2413, 4096, 0, 867, 0, 0, 4096, 4096, 0, 3373, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1539, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3026, 4096, 0, 441, 0, 0, 4096, 4096, 0, 1997, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 2728, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2971, 4095, 0, 384, 0, 0, 4096, 4096, 0, 2447, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 2390, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3312, 1094, 0, 0, 0, 0, 0, 998, 0, 0, 3001, 221, 2, 0, 1583, 0, 0, 3273, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 610, 0, 0, 2289, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4406, 4222, 4856, 4095, 4096, 4099, 2899, 4095] avg=4096.0000 max=4856 max_rank=2 min=2899 min_rank=6 imbalance_ratio=1.185547 max_load_reduction=27912 max_load_reduction_pct=0.851807 moved_to_echo=15189 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2456, 1093, 0, 0, 0, 0, 0, 1034, 0, 0, 3000, 221, 3, 0, 1583, 0, 0, 1565, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 3137, 0, 0, 2289, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[3549, 4258, 3148, 4094, 4096, 4099, 5426, 4098] avg=4096.0000 max=5426 max_rank=6 min=3148 min_rank=2 imbalance_ratio=1.324707 max_load_reduction=27342 max_load_reduction_pct=0.834412 moved_to_echo=17719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3437, 1094, 0, 0, 0, 0, 0, 1207, 0, 0, 3001, 221, 2, 0, 1583, 0, 0, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 923, 0, 0, 2289, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4531, 4431, 4208, 4095, 4096, 4099, 3212, 4096] avg=4096.0000 max=4531 max_rank=0 min=3212 min_rank=6 imbalance_ratio=1.106201 max_load_reduction=28237 max_load_reduction_pct=0.861725 moved_to_echo=15503 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2590, 1094, 0, 0, 0, 0, 0, 1339, 0, 0, 3002, 221, 2, 0, 1583, 0, 0, 2651, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 1612, 0, 0, 2289, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3684, 4564, 4234, 4094, 4096, 4099, 3901, 4096] avg=4096.0000 max=4564 max_rank=1 min=3684 min_rank=0 imbalance_ratio=1.114258 max_load_reduction=28204 max_load_reduction_pct=0.860718 moved_to_echo=16192 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[5, 7, 3805, 1089, 0, 0, 0, 0, 3, 690, 0, 6, 2998, 226, 3, 0, 1563, 0, 6, 2187, 0, 0, 0, 0, 4, 0, 19, 0, 4086, 1, 0, 0, 4088, 0, 6, 0, 0, 0, 0, 0, 4064, 1, 1, 6, 3, 0, 0, 0, 0, 1518, 0, 4, 2284, 0, 0, 0, 0, 0, 0, 6, 4089, 0, 0, 0] rerouted_tokens_per_ep_rank=[4906, 3926, 3756, 4110, 4094, 4075, 3806, 4095] avg=4096.0000 max=4906 max_rank=0 min=3756 min_rank=2 imbalance_ratio=1.197754 max_load_reduction=27862 max_load_reduction_pct=0.850281 moved_to_echo=16070 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2971, 1094, 0, 0, 0, 0, 0, 384, 0, 0, 3001, 221, 2, 0, 1583, 0, 0, 2447, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 2390, 0, 0, 2289, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 3608, 4030, 4093, 4097, 4099, 4679, 4097] avg=4096.0000 max=4679 max_rank=6 min=3608 min_rank=1 imbalance_ratio=1.142334 max_load_reduction=28089 max_load_reduction_pct=0.857208 moved_to_echo=16972 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3026, 1094, 0, 0, 0, 0, 0, 441, 0, 0, 3002, 221, 2, 0, 1583, 0, 0, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 2728, 0, 0, 2289, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 3666, 3580, 4093, 4097, 4099, 5017, 4096] avg=4096.0000 max=5017 max_rank=6 min=3580 min_rank=2 imbalance_ratio=1.224854 max_load_reduction=27751 max_load_reduction_pct=0.846893 moved_to_echo=17309 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2413, 1094, 0, 0, 0, 0, 0, 867, 0, 0, 3002, 221, 3, 0, 1583, 0, 0, 3373, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 1539, 0, 0, 2289, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[3507, 4093, 4956, 4094, 4096, 4099, 3828, 4095] avg=4096.0000 max=4956 max_rank=2 min=3507 min_rank=0 imbalance_ratio=1.209961 max_load_reduction=27812 max_load_reduction_pct=0.848755 moved_to_echo=16118 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12672, 0, 128, 20224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 14464, 0, 128, 18432, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 7040, 0, 128, 24064, 1792, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 24064, 8832, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 1386, 0, 2, 4089, 0, 6, 6, 4090, 4081, 4080, 7, 7, 5, 0, 4069, 0, 11, 0, 1358, 4089, 5, 10, 4, 0, 72, 0, 0, 0, 1390, 0, 4001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 1005, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 2, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 3091, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 1546, 5, 0, 4096, 0, 0, 0, 4091, 4096, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2556, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 3268, 0, 0, 4096, 0, 0, 0, 4091, 4096, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 167, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 667, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 3108, 5, 0, 4096, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 4089, 0, 0, 1, 9, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 982, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 2619, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1480, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 1480, 4, 0, 4096, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 7, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 2621, 0, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 1317, 5, 0, 4096, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 3, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 2781, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1005, 0, 0, 1933, 200, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 1, 0, 0, 0, 0, 0, 2, 3894, 9, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4087, 0, 0, 0, 0, 3091, 0, 2158, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3138, 4096, 4098, 4096, 3905, 4095, 4091, 5249] avg=4096.0000 max=5249 max_rank=7 min=3138 min_rank=0 imbalance_ratio=1.281494 max_load_reduction=27519 max_load_reduction_pct=0.839813 moved_to_echo=17340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3268, 0, 0, 1933, 200, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 1, 0, 0, 0, 0, 0, 167, 3894, 9, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4087, 0, 0, 0, 0, 667, 0, 2158, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5401, 4096, 4093, 4096, 4070, 4095, 4092, 2825] avg=4096.0000 max=5401 max_rank=0 min=2825 min_rank=7 imbalance_ratio=1.318604 max_load_reduction=27367 max_load_reduction_pct=0.835175 moved_to_echo=15082 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3108, 5, 0, 1934, 200, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 1, 0, 0, 0, 0, 1, 9, 3894, 9, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4087, 0, 0, 0, 0, 982, 0, 2158, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5247, 4096, 4096, 4094, 3913, 4094, 4088, 3140] avg=4096.0000 max=5247 max_rank=0 min=3140 min_rank=7 imbalance_ratio=1.281006 max_load_reduction=27521 max_load_reduction_pct=0.839874 moved_to_echo=15235 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1546, 5, 0, 1934, 200, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 1, 0, 0, 0, 0, 0, 0, 3894, 9, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 2556, 0, 2158, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3685, 4096, 4093, 4095, 3903, 4095, 4087, 4714] avg=4096.0000 max=4714 max_rank=7 min=3685 min_rank=0 imbalance_ratio=1.150879 max_load_reduction=28054 max_load_reduction_pct=0.856140 moved_to_echo=16799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1386, 0, 2, 1893, 201, 0, 0, 4084, 0, 6, 6, 0, 0, 0, 0, 4090, 0, 0, 7, 0, 0, 0, 0, 7, 5, 0, 4069, 4, 4, 5, 0, 0, 11, 0, 1358, 3879, 9, 0, 0, 4085, 5, 10, 4, 0, 0, 0, 0, 0, 72, 0, 0, 4072, 0, 0, 0, 0, 1390, 0, 2104, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3482, 4096, 4097, 4094, 5257, 4104, 4144, 3494] avg=4096.0000 max=5257 max_rank=4 min=3482 min_rank=0 imbalance_ratio=1.283447 max_load_reduction=27511 max_load_reduction_pct=0.839569 moved_to_echo=16999 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1317, 5, 0, 1931, 200, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 1, 0, 0, 0, 0, 0, 3, 3894, 8, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4088, 0, 0, 0, 0, 2781, 0, 2156, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3453, 4096, 4096, 4096, 3905, 4095, 4090, 4937] avg=4096.0000 max=4937 max_rank=7 min=3453 min_rank=0 imbalance_ratio=1.205322 max_load_reduction=27831 max_load_reduction_pct=0.849335 moved_to_echo=17027 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2619, 0, 0, 1931, 200, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 1, 0, 0, 0, 0, 0, 2, 3894, 9, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 1480, 0, 2156, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4750, 4096, 4098, 4101, 3905, 4095, 4087, 3636] avg=4096.0000 max=4750 max_rank=0 min=3636 min_rank=7 imbalance_ratio=1.159668 max_load_reduction=28018 max_load_reduction_pct=0.855042 moved_to_echo=15723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1480, 4, 0, 1928, 200, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 1, 0, 0, 0, 0, 0, 7, 3894, 9, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4087, 0, 0, 0, 0, 2621, 0, 2152, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3612, 4096, 4097, 4096, 3910, 4095, 4089, 4773] avg=4096.0000 max=4773 max_rank=7 min=3612 min_rank=0 imbalance_ratio=1.165283 max_load_reduction=27995 max_load_reduction_pct=0.854340 moved_to_echo=16867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 15744, 128, 128, 15488, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 15616, 0, 17280, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 1664, 31232, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 4092, 2160, 0, 0, 4034, 0, 4096, 1985, 0, 0, 0, 4095, 0, 0, 15, 4094, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 29, 0, 2, 0, 0, 0, 0, 4089, 1211, 0, 0, 4038, 0, 4096, 2914, 0, 0, 0, 4094, 0, 0, 7, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[2, 6, 0, 7, 4072, 0, 19, 6, 52, 115, 0, 7, 5, 5, 6, 4089, 3014, 4, 6, 3337, 37, 4042, 1572, 0, 7, 0, 4088, 93, 0, 0, 4090, 4087] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 0, 215, 0, 0, 0, 0, 0, 0, 4096, 1925, 0, 0, 4087, 0, 4096, 1956, 0, 0, 0, 4096, 5, 0, 9, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 4096, 1600, 0, 0, 4087, 0, 4096, 2459, 0, 0, 0, 4096, 0, 0, 1, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 30, 0, 2, 0, 0, 0, 0, 4088, 2788, 0, 0, 4094, 0, 4096, 1275, 0, 0, 0, 4092, 0, 0, 17, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 0, 216, 0, 1, 0, 0, 0, 0, 4089, 2164, 0, 0, 4076, 0, 4096, 1727, 0, 0, 0, 4095, 11, 0, 9, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 4093, 2078, 0, 0, 3918, 0, 4096, 2168, 0, 0, 0, 4096, 0, 0, 14, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 3, 0, 2, 0, 2008, 2019, 12, 3, 0, 0, 0, 4092, 2, 0, 0, 0, 2160, 0, 0, 2015, 0, 0, 0, 0, 0, 2088, 1985, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 15, 1, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4097, 4047, 4094, 4175, 4073, 4083, 4106] avg=4096.0000 max=4175 max_rank=4 min=4047 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 29, 0, 2, 0, 2008, 2021, 12, 3, 0, 0, 0, 4089, 2, 0, 0, 0, 1211, 0, 0, 2017, 0, 0, 0, 0, 0, 2088, 2914, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 7, 1, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4097, 4075, 4091, 3228, 5002, 4082, 4098] avg=4096.0000 max=5002 max_rank=5 min=3228 min_rank=4 imbalance_ratio=1.221191 max_load_reduction=27766 max_load_reduction_pct=0.847351 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 30, 0, 2, 0, 2008, 2049, 12, 3, 0, 0, 0, 4088, 2, 0, 0, 0, 2788, 0, 0, 2045, 0, 0, 0, 0, 0, 2088, 1275, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 17, 1, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4104, 4090, 4833, 3363, 4080, 4108] avg=4096.0000 max=4833 max_rank=4 min=3363 min_rank=5 imbalance_ratio=1.179932 max_load_reduction=27935 max_load_reduction_pct=0.852509 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 46, 0, 0, 0, 2008, 2045, 12, 3, 0, 0, 0, 4096, 2, 0, 0, 0, 1600, 0, 0, 2042, 0, 0, 0, 0, 0, 2088, 2459, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 1, 1, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4114, 4098, 3642, 4547, 4084, 4092] avg=4096.0000 max=4547 max_rank=5 min=3642 min_rank=4 imbalance_ratio=1.110107 max_load_reduction=28221 max_load_reduction_pct=0.861237 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 6, 0, 7, 4090, 0, 0, 0, 4072, 0, 19, 6, 6, 0, 0, 0, 52, 115, 0, 7, 1983, 1673, 16, 9, 5, 5, 6, 4089, 6, 0, 0, 0, 3014, 4, 6, 1664, 0, 0, 0, 0, 37, 2059, 1572, 0, 0, 0, 0, 0, 7, 0, 4072, 93, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4103, 3855, 4111, 4688, 3668, 4172, 4066] avg=4096.0000 max=4688 max_rank=4 min=3668 min_rank=5 imbalance_ratio=1.144531 max_load_reduction=28080 max_load_reduction_pct=0.856934 moved_to_echo=16594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 216, 0, 1, 0, 2008, 2040, 12, 3, 0, 0, 0, 4089, 2, 0, 0, 0, 2164, 0, 0, 2036, 0, 0, 0, 0, 0, 2088, 1727, 0, 0, 0, 0, 0, 0, 0, 4083, 11, 0, 0, 0, 0, 0, 9, 1, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4093, 4280, 4091, 4200, 3815, 4094, 4100] avg=4096.0000 max=4280 max_rank=2 min=3815 min_rank=5 imbalance_ratio=1.044922 max_load_reduction=28488 max_load_reduction_pct=0.869385 moved_to_echo=16209 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4091, 0, 0, 0, 1, 0, 0, 0, 215, 0, 0, 0, 2008, 2045, 12, 3, 0, 0, 0, 4096, 2, 0, 0, 0, 1925, 0, 0, 2042, 0, 0, 0, 0, 0, 2088, 1956, 0, 0, 0, 0, 0, 0, 0, 4084, 5, 0, 0, 0, 0, 0, 9, 1, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4092, 4283, 4098, 3967, 4044, 4089, 4100] avg=4096.0000 max=4283 max_rank=2 min=3967 min_rank=4 imbalance_ratio=1.045654 max_load_reduction=28485 max_load_reduction_pct=0.869293 moved_to_echo=16200 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 19, 0, 0, 0, 2008, 1961, 12, 3, 0, 0, 0, 4093, 2, 0, 0, 0, 2078, 0, 0, 1957, 0, 0, 0, 0, 0, 2088, 2168, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 14, 1, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4003, 4095, 4035, 4256, 4084, 4105] avg=4096.0000 max=4256 max_rank=5 min=4003 min_rank=2 imbalance_ratio=1.039062 max_load_reduction=28512 max_load_reduction_pct=0.870117 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32761, 32768, 32768, 32768, 32768, 32775] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32761, 32768, 32768, 32768, 32768, 32775] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32761, 32768, 32768, 32768, 32768, 32775] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32761, 32768, 32768, 32768, 32768, 32775] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32761, 32768, 32768, 32768, 32768, 32775] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32761, 32768, 32768, 32768, 32768, 32775] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32761, 32768, 32768, 32768, 32768, 32775] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32761, 32768, 32768, 32768, 32768, 32775] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32761 actual_valid_recv=33280 observed_loss=-519 tokens_per_expert=[640, 128, 128, 128, 16128, 15872, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[17024, 128, 128, 15872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 16768, 16128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32775 actual_valid_recv=33024 observed_loss=-249 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3035, 0, 0, 0, 40, 0, 0, 0, 0, 0, 3421, 0, 0, 4096, 0, 0, 4096, 5, 0, 122, 0, 4096, 1931, 3734, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[5, 6, 4074, 3681, 5, 0, 0, 59, 0, 1, 16, 7, 0, 2354, 7, 5, 4081, 8, 4, 4088, 14, 0, 705, 7, 4089, 2097, 3357, 0, 2, 0, 6, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3508, 0, 0, 4096, 0, 0, 4096, 0, 0, 36, 0, 4096, 2510, 3782, 0, 0, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2995, 0, 0, 4096, 0, 0, 4096, 0, 0, 10, 0, 4096, 2660, 3949, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1946, 0, 0, 0, 14, 0, 0, 0, 0, 1, 3569, 0, 0, 4096, 0, 0, 4096, 3, 0, 21, 0, 4096, 2931, 3804, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2532, 0, 0, 4096, 0, 0, 4096, 0, 0, 21, 0, 4096, 2695, 3612, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3309, 0, 0, 4096, 0, 0, 4096, 0, 0, 29, 0, 4096, 2355, 3924, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 0, 0, 4096, 0, 0, 4096, 0, 0, 2, 0, 4096, 2221, 3778, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1254, 3035, 0, 0, 0, 0, 0, 0, 0, 40, 4080, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3421, 0, 0, 950, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 122, 0, 2842, 1115, 16, 0, 0, 1931, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4289, 4122, 4094, 4371, 4096, 4100, 3600, 4096] avg=4096.0000 max=4371 max_rank=3 min=3600 min_rank=6 imbalance_ratio=1.067139 max_load_reduction=28397 max_load_reduction_pct=0.866608 moved_to_echo=15892 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1254, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3, 3508, 0, 0, 963, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 2842, 1129, 16, 0, 0, 2510, 1690, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3706, 4082, 4094, 4474, 4096, 4023, 4200, 4093] avg=4096.0000 max=4474 max_rank=3 min=3706 min_rank=0 imbalance_ratio=1.092285 max_load_reduction=28294 max_load_reduction_pct=0.863464 moved_to_echo=16412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1254, 2674, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2995, 0, 0, 1005, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 10, 0, 2842, 1179, 15, 0, 0, 2660, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3928, 4082, 4094, 4000, 4097, 4046, 4425, 4096] avg=4096.0000 max=4425 max_rank=6 min=3928 min_rank=0 imbalance_ratio=1.080322 max_load_reduction=28343 max_load_reduction_pct=0.864960 moved_to_echo=16664 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1254, 1946, 0, 0, 0, 0, 0, 0, 0, 14, 4080, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 3569, 0, 0, 968, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 21, 0, 2842, 1136, 16, 0, 0, 2931, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3200, 4096, 4094, 4538, 4096, 4018, 4631, 4095] avg=4096.0000 max=4631 max_rank=6 min=3200 min_rank=0 imbalance_ratio=1.130615 max_load_reduction=28137 max_load_reduction_pct=0.858673 moved_to_echo=16840 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[5, 6, 1243, 3681, 0, 0, 0, 0, 5, 0, 0, 59, 4073, 3, 0, 0, 0, 1, 16, 7, 4086, 0, 0, 0, 0, 2354, 7, 5, 858, 0, 0, 0, 4081, 8, 4, 0, 0, 0, 0, 0, 14, 0, 705, 7, 2831, 1006, 15, 0, 0, 2097, 1489, 0, 0, 0, 0, 0, 2, 0, 6, 4090, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4935, 4140, 4110, 3224, 4093, 4578, 3586, 4102] avg=4096.0000 max=4935 max_rank=0 min=3224 min_rank=3 imbalance_ratio=1.204834 max_load_reduction=27833 max_load_reduction_pct=0.849396 moved_to_echo=16359 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1254, 2671, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3309, 0, 0, 999, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 29, 0, 2842, 1172, 15, 0, 0, 2355, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3925, 4082, 4094, 4308, 4097, 4058, 4108, 4096] avg=4096.0000 max=4308 max_rank=3 min=3925 min_rank=0 imbalance_ratio=1.051758 max_load_reduction=28460 max_load_reduction_pct=0.868530 moved_to_echo=16359 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1254, 3428, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2532, 0, 0, 919, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 2842, 1079, 16, 0, 0, 2695, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4682, 4082, 4094, 3451, 4096, 3958, 4309, 4096] avg=4096.0000 max=4682 max_rank=0 min=3451 min_rank=3 imbalance_ratio=1.143066 max_load_reduction=28086 max_load_reduction_pct=0.857117 moved_to_echo=16459 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1254, 2849, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3440, 0, 0, 962, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 2842, 1128, 15, 0, 0, 2221, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4082, 4094, 4402, 4097, 3987, 3909, 4094] avg=4096.0000 max=4402 max_rank=3 min=3909 min_rank=6 imbalance_ratio=1.074707 max_load_reduction=28366 max_load_reduction_pct=0.865662 moved_to_echo=16087 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 1024, 128, 22784, 8960, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 10112, 22784, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 19456, 13440, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 25216, 128, 128, 7680, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[173, 4096, 0, 0, 0, 4096, 4096, 2417, 0, 0, 4096, 0, 4096, 0, 2233, 4076, 0, 0, 0, 3389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[59, 4096, 0, 0, 0, 4096, 4096, 3535, 0, 0, 4096, 0, 4096, 0, 1290, 3698, 0, 0, 0, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[49, 4096, 0, 0, 1, 4096, 4096, 2858, 0, 0, 4096, 0, 4096, 0, 1656, 3926, 0, 0, 0, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1086, 4080, 10, 0, 0, 4063, 4088, 1436, 3, 5, 4080, 0, 4087, 9, 1601, 4068, 28, 4, 6, 4042, 6, 0, 0, 6, 0, 7, 12, 22, 0, 1, 5, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[107, 4096, 0, 0, 0, 4096, 4096, 2879, 0, 0, 4096, 0, 4096, 0, 1578, 4016, 0, 0, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[49, 4096, 0, 0, 0, 4096, 4096, 2251, 0, 0, 4096, 0, 4096, 0, 2363, 3948, 0, 0, 0, 3677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[72, 4096, 0, 0, 0, 4096, 4096, 3076, 0, 0, 4096, 0, 4096, 0, 1613, 3657, 0, 0, 0, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[140, 4096, 0, 0, 0, 4096, 4096, 1558, 0, 0, 4096, 0, 4096, 0, 2856, 4007, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[49, 3880, 0, 0, 0, 0, 0, 0, 1, 1597, 0, 2858, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 1656, 2198, 0, 0, 0, 0, 0, 0, 0, 3798, 215, 136, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2499, 1592, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[3929, 4456, 4097, 3854, 4149, 4096, 4093, 4094] avg=4096.0000 max=4456 max_rank=1 min=3854 min_rank=3 imbalance_ratio=1.087891 max_load_reduction=28312 max_load_reduction_pct=0.864014 moved_to_echo=16432 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[59, 3880, 0, 0, 0, 0, 0, 0, 0, 1597, 0, 3535, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 1290, 2070, 0, 0, 0, 0, 0, 0, 0, 3706, 215, 128, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2499, 1500, 2, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[3939, 5132, 4097, 3360, 4049, 4095, 4001, 4095] avg=4096.0000 max=5132 max_rank=1 min=3360 min_rank=3 imbalance_ratio=1.252930 max_load_reduction=27636 max_load_reduction_pct=0.843384 moved_to_echo=16240 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[173, 3880, 0, 0, 0, 0, 0, 0, 0, 1597, 0, 2417, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 2233, 2282, 0, 0, 0, 0, 0, 0, 0, 3389, 215, 141, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2499, 1653, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4014, 4097, 4515, 3745, 4096, 4154, 4094] avg=4096.0000 max=4515 max_rank=3 min=3745 min_rank=4 imbalance_ratio=1.102295 max_load_reduction=28253 max_load_reduction_pct=0.862213 moved_to_echo=16089 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[107, 3880, 0, 0, 0, 0, 0, 0, 0, 1597, 0, 2879, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 1578, 2249, 0, 0, 0, 0, 0, 0, 0, 3708, 215, 139, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2499, 1628, 2, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[3987, 4476, 4097, 3827, 4062, 4095, 4129, 4095] avg=4096.0000 max=4476 max_rank=1 min=3827 min_rank=3 imbalance_ratio=1.092773 max_load_reduction=28292 max_load_reduction_pct=0.863403 moved_to_echo=16381 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1086, 3863, 10, 0, 0, 0, 0, 0, 0, 1578, 0, 1436, 0, 0, 0, 0, 3, 5, 4080, 0, 1, 0, 0, 0, 0, 9, 1601, 2270, 0, 0, 0, 0, 28, 4, 6, 4042, 216, 145, 0, 0, 6, 0, 0, 6, 4088, 0, 0, 0, 0, 7, 12, 22, 2485, 1653, 2, 0, 0, 1, 5, 13, 4085, 0, 0, 0] rerouted_tokens_per_ep_rank=[4959, 3014, 4089, 3880, 4441, 4100, 4181, 4104] avg=4096.0000 max=4959 max_rank=0 min=3014 min_rank=1 imbalance_ratio=1.210693 max_load_reduction=27809 max_load_reduction_pct=0.848663 moved_to_echo=16826 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[140, 3880, 0, 0, 0, 0, 0, 0, 0, 1597, 0, 1558, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 2856, 2243, 0, 0, 0, 0, 0, 0, 0, 3727, 215, 139, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2499, 1625, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4020, 3155, 4097, 5099, 4081, 4095, 4125, 4096] avg=4096.0000 max=5099 max_rank=3 min=3155 min_rank=1 imbalance_ratio=1.244873 max_load_reduction=27669 max_load_reduction_pct=0.844391 moved_to_echo=16397 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[49, 3880, 0, 0, 0, 0, 0, 0, 0, 1597, 0, 2251, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 2363, 2210, 0, 0, 0, 0, 0, 0, 0, 3677, 215, 137, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2499, 1601, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[3929, 3848, 4097, 4573, 4029, 4095, 4101, 4096] avg=4096.0000 max=4573 max_rank=3 min=3848 min_rank=1 imbalance_ratio=1.116455 max_load_reduction=28195 max_load_reduction_pct=0.860443 moved_to_echo=16321 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[72, 3880, 0, 0, 0, 0, 0, 0, 0, 1597, 0, 3076, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 1613, 2047, 0, 0, 0, 0, 0, 0, 0, 3870, 215, 127, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2499, 1483, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[3952, 4673, 4097, 3660, 4212, 4096, 3984, 4094] avg=4096.0000 max=4673 max_rank=1 min=3660 min_rank=3 imbalance_ratio=1.140869 max_load_reduction=28095 max_load_reduction_pct=0.857391 moved_to_echo=16386 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 20096, 12800, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 15232, 17664, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 12800, 0, 20096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1792, 31104, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 29952, 1792, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 4096, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 4096, 0, 5, 4096, 4096, 2, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 4096, 0, 7, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4096, 0, 0, 4092, 0, 4, 4095, 0, 0, 0, 4096, 0, 1, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[117, 6, 21, 7, 4, 0, 0, 4089, 0, 24, 4076, 0, 11, 3961, 0, 0, 4089, 0, 0, 5, 4076, 0, 0, 3822, 4080, 1, 0, 318, 8, 4046, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4063, 15, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 7, 4096, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4096, 4096, 4106, 4096, 4136, 4060, 4100] avg=4096.0000 max=4136 max_rank=5 min=4060 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4063, 15, 0, 0, 0, 0, 1, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 5, 4096, 0, 0, 0, 0, 4060, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4093, 4096, 4109, 4096, 4134, 4062, 4100] avg=4096.0000 max=4134 max_rank=5 min=4062 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4063, 15, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 4, 17, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 33, 0, 1, 4096, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4096, 4096, 4113, 4095, 4130, 4060, 4100] avg=4096.0000 max=4130 max_rank=5 min=4060 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4063, 15, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 4096, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4096, 4097, 4112, 4096, 4129, 4060, 4100] avg=4096.0000 max=4129 max_rank=5 min=4060 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4063, 15, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 2, 4096, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4096, 4096, 4111, 4096, 4131, 4060, 4100] avg=4096.0000 max=4131 max_rank=5 min=4060 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[117, 6, 21, 7, 4049, 22, 0, 0, 4, 0, 0, 4089, 4, 2, 0, 0, 0, 24, 4071, 0, 0, 0, 0, 0, 11, 3961, 0, 0, 20, 0, 0, 0, 4089, 0, 0, 5, 3, 0, 0, 0, 27, 0, 0, 3822, 0, 0, 0, 0, 4027, 1, 0, 318, 0, 0, 0, 0, 8, 4046, 7, 0, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4222, 4099, 4095, 3992, 4097, 3849, 4346, 4068] avg=4096.0000 max=4346 max_rank=6 min=3849 min_rank=5 imbalance_ratio=1.061035 max_load_reduction=28422 max_load_reduction_pct=0.867371 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4063, 15, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 4096, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4096, 4096, 4113, 4096, 4129, 4060, 4100] avg=4096.0000 max=4129 max_rank=5 min=4060 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4063, 15, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 1, 4096, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4096, 4096, 4112, 4096, 4130, 4060, 4100] avg=4096.0000 max=4130 max_rank=5 min=4060 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 0, 128, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 0, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 128, 0, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 4, 0, 0, 0, 0, 0, 4090, 2850, 26, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 15, 0, 0, 1208, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3684, 10, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 401, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 0, 0, 0, 4094, 3283, 18, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 796, 0, 0, 4096, 1, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4088, 0, 0, 9, 8, 0, 52, 7, 4090, 3641, 0, 0, 0, 0, 0, 8, 4081, 4071, 4059, 8, 6, 1, 0, 15, 352, 0, 6, 4087, 0, 276, 97, 3806] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 5, 0, 0, 0, 0, 0, 4090, 3651, 3, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 23, 0, 0, 420, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 0, 0, 0, 4096, 3569, 3, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 11, 0, 0, 513, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4095, 3410, 1, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 13, 0, 0, 673, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 10, 0, 0, 0, 0, 0, 4093, 3611, 4, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 6, 0, 0, 471, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 626, 2850, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3464, 604, 10, 2, 1208, 0, 0, 3492, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4090, 3502, 4102, 4096, 4095, 4700, 4086] avg=4096.0000 max=4700 max_rank=6 min=3502 min_rank=2 imbalance_ratio=1.147461 max_load_reduction=28068 max_load_reduction_pct=0.856567 moved_to_echo=16977 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 626, 3283, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 604, 10, 2, 796, 0, 0, 3492, 0, 0, 0, 0, 1, 0, 0, 4086, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4090, 3927, 4096, 4102, 4084, 4288, 4087] avg=4096.0000 max=4288 max_rank=6 min=3927 min_rank=2 imbalance_ratio=1.046875 max_load_reduction=28480 max_load_reduction_pct=0.869141 moved_to_echo=16561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 626, 3684, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3470, 604, 10, 2, 401, 0, 0, 3492, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4090, 4320, 4096, 4102, 4087, 3893, 4086] avg=4096.0000 max=4320 max_rank=2 min=3893 min_rank=6 imbalance_ratio=1.054688 max_load_reduction=28448 max_load_reduction_pct=0.868164 moved_to_echo=16168 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 626, 3651, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 23, 0, 0, 3464, 604, 10, 2, 420, 0, 0, 3492, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4090, 4280, 4096, 4102, 4103, 3912, 4086] avg=4096.0000 max=4280 max_rank=2 min=3912 min_rank=6 imbalance_ratio=1.044922 max_load_reduction=28488 max_load_reduction_pct=0.869385 moved_to_echo=16203 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 626, 3611, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3467, 604, 10, 1, 471, 0, 0, 3492, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4090, 4241, 4096, 4102, 4088, 3963, 4086] avg=4096.0000 max=4241 max_rank=2 min=3963 min_rank=6 imbalance_ratio=1.035400 max_load_reduction=28527 max_load_reduction_pct=0.870575 moved_to_echo=16239 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 0, 9, 0, 0, 0, 0, 8, 0, 52, 7, 4071, 0, 0, 0, 622, 3641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4081, 0, 0, 0, 0, 0, 4059, 8, 0, 0, 0, 0, 6, 1, 0, 15, 3468, 603, 14, 3, 352, 0, 6, 3484, 0, 0, 0, 0, 0, 276, 97, 3792, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4138, 4263, 4089, 4067, 4110, 3842, 4165] avg=4096.0000 max=4263 max_rank=2 min=3842 min_rank=6 imbalance_ratio=1.040771 max_load_reduction=28505 max_load_reduction_pct=0.869904 moved_to_echo=16184 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 626, 3410, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 13, 0, 0, 3469, 604, 10, 2, 673, 0, 0, 3492, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4090, 4037, 4096, 4102, 4098, 4165, 4086] avg=4096.0000 max=4165 max_rank=6 min=4037 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 626, 3569, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3470, 604, 10, 2, 513, 0, 0, 3492, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4090, 4198, 4097, 4101, 4097, 4005, 4086] avg=4096.0000 max=4198 max_rank=2 min=4005 min_rank=6 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32774, 32762, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32774, 32762, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32774, 32762, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32774, 32762, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32774, 32762, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32774, 32762, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32774, 32762, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32774, 32762, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32762 actual_valid_recv=33280 observed_loss=-518 tokens_per_expert=[128, 128, 0, 128, 27776, 4864, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5120, 27776, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 384, 128, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4864, 0, 128, 28032, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32774 actual_valid_recv=33024 observed_loss=-250 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 85, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 4021, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 34, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 4064, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[8, 2559, 7, 30, 235, 8, 0, 4078, 6, 2, 8, 8, 4088, 12, 0, 5, 4070, 0, 145, 4088, 8, 0, 4, 4086, 3830, 0, 0, 0, 6, 1392, 0, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 91, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 4096, 4094, 0, 0, 3, 0, 4008, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 85, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4011, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 84, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 4013, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 175, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 4093, 0, 0, 1, 0, 3924, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 34, 0, 0, 3691, 9, 1, 0, 1, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 18, 14, 1, 0, 0, 4064, 0, 405, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3735, 4070, 4095, 4094, 4082, 4096, 4127, 4469] avg=4096.0000 max=4469 max_rank=7 min=3735 min_rank=0 imbalance_ratio=1.091064 max_load_reduction=28299 max_load_reduction_pct=0.863617 moved_to_echo=16774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 91, 0, 0, 3691, 9, 2, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 3, 18, 14, 1, 0, 0, 4008, 0, 405, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3793, 4069, 4090, 4095, 4082, 4096, 4130, 4413] avg=4096.0000 max=4413 max_rank=7 min=3793 min_rank=0 imbalance_ratio=1.077393 max_load_reduction=28355 max_load_reduction_pct=0.865326 moved_to_echo=16721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 3691, 9, 1, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 14, 1, 0, 0, 4094, 0, 405, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3703, 4069, 4095, 4095, 4082, 4096, 4129, 4499] avg=4096.0000 max=4499 max_rank=7 min=3703 min_rank=0 imbalance_ratio=1.098389 max_load_reduction=28269 max_load_reduction_pct=0.862701 moved_to_echo=16806 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 85, 0, 0, 3691, 9, 1, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 14, 1, 0, 0, 4011, 0, 405, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3786, 4069, 4095, 4095, 4082, 4096, 4129, 4416] avg=4096.0000 max=4416 max_rank=7 min=3786 min_rank=0 imbalance_ratio=1.078125 max_load_reduction=28352 max_load_reduction_pct=0.865234 moved_to_echo=16723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 2559, 7, 30, 3685, 12, 2, 0, 235, 8, 0, 4041, 0, 0, 0, 0, 6, 2, 8, 8, 4086, 0, 0, 0, 4087, 12, 0, 5, 0, 0, 0, 0, 4049, 0, 145, 0, 0, 0, 0, 0, 8, 0, 4, 4084, 0, 0, 0, 0, 3830, 0, 0, 0, 25, 21, 1, 2, 6, 1392, 0, 400, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6303, 4284, 4110, 4104, 4194, 4096, 3879, 1798] avg=4096.0000 max=6303 max_rank=0 min=1798 min_rank=7 imbalance_ratio=1.538818 max_load_reduction=26465 max_load_reduction_pct=0.807648 moved_to_echo=13967 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 85, 0, 0, 3691, 9, 2, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 18, 14, 1, 0, 0, 4021, 0, 405, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3787, 4069, 4094, 4095, 4082, 4096, 4119, 4426] avg=4096.0000 max=4426 max_rank=7 min=3787 min_rank=0 imbalance_ratio=1.080566 max_load_reduction=28342 max_load_reduction_pct=0.864929 moved_to_echo=16723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 175, 0, 0, 3691, 9, 1, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 1, 18, 14, 1, 0, 0, 3924, 0, 405, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3876, 4069, 4094, 4095, 4082, 4096, 4127, 4329] avg=4096.0000 max=4329 max_rank=7 min=3876 min_rank=0 imbalance_ratio=1.056885 max_load_reduction=28439 max_load_reduction_pct=0.867889 moved_to_echo=16634 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 84, 0, 0, 3691, 9, 1, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 18, 14, 1, 0, 0, 4013, 0, 405, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3785, 4069, 4095, 4095, 4082, 4096, 4128, 4418] avg=4096.0000 max=4418 max_rank=7 min=3785 min_rank=0 imbalance_ratio=1.078613 max_load_reduction=28350 max_load_reduction_pct=0.865173 moved_to_echo=16724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 3200, 128, 128, 29568, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 29568, 0, 3328, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 0, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32640, 0, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32512, 0, 0, 128, 256, 128, 128, 128] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4075, 4093, 1778, 4096, 0, 0, 4, 3820, 0, 0, 0, 2698, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4088, 4096, 1770, 4096, 0, 0, 0, 3850, 0, 0, 0, 2604, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4089, 4096, 1676, 4096, 0, 0, 0, 4021, 0, 0, 0, 2527, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4081, 4096, 67, 4096, 0, 0, 2, 4071, 0, 0, 0, 4070, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 1110, 4096, 0, 0, 0, 3794, 0, 0, 0, 3338, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4090, 4090, 174, 4033, 12, 15, 286, 4078, 6, 0, 0, 4054, 8, 3854, 0, 3, 0, 6, 10, 0, 0, 11, 3903, 0, 0, 0, 4088, 7, 0, 1, 39, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4088, 4095, 423, 4096, 0, 0, 6, 4059, 0, 0, 1, 3741, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4094, 4096, 1039, 4096, 0, 0, 0, 3957, 0, 0, 0, 3256, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3085, 0, 1778, 0, 0, 0, 0, 0, 0, 0, 4, 3820, 98, 0, 0, 0, 0, 0, 0, 2698, 806, 0, 0, 0, 0, 4012, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0] rerouted_tokens_per_ep_rank=[4863, 3922, 3504, 4074, 4093, 4118, 4096, 4098] avg=4096.0000 max=4863 max_rank=0 min=3504 min_rank=2 imbalance_ratio=1.187256 max_load_reduction=27905 max_load_reduction_pct=0.851593 moved_to_echo=16405 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3094, 0, 1676, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 99, 0, 0, 0, 0, 0, 0, 2527, 809, 0, 0, 0, 0, 4071, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 0] rerouted_tokens_per_ep_rank=[4770, 4120, 3336, 4134, 4095, 4118, 4096, 4099] avg=4096.0000 max=4770 max_rank=0 min=3336 min_rank=2 imbalance_ratio=1.164551 max_load_reduction=27998 max_load_reduction_pct=0.854431 moved_to_echo=16408 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3090, 0, 67, 0, 0, 0, 0, 0, 0, 0, 2, 4071, 98, 0, 0, 0, 0, 0, 0, 4070, 807, 0, 0, 0, 0, 4093, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 0] rerouted_tokens_per_ep_rank=[3157, 4171, 4877, 4155, 4095, 4118, 4096, 4099] avg=4096.0000 max=4877 max_rank=2 min=3157 min_rank=0 imbalance_ratio=1.190674 max_load_reduction=27891 max_load_reduction_pct=0.851166 moved_to_echo=16408 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3093, 0, 423, 0, 0, 0, 0, 0, 0, 0, 6, 4059, 99, 0, 0, 0, 0, 0, 1, 3741, 809, 0, 0, 0, 0, 4067, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 0] rerouted_tokens_per_ep_rank=[3516, 4164, 4551, 4130, 4094, 4118, 4096, 4099] avg=4096.0000 max=4551 max_rank=2 min=3516 min_rank=0 imbalance_ratio=1.111084 max_load_reduction=28217 max_load_reduction_pct=0.861115 moved_to_echo=16407 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3093, 0, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 99, 0, 0, 0, 0, 0, 0, 2604, 809, 0, 0, 0, 0, 4072, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 0] rerouted_tokens_per_ep_rank=[4863, 3949, 3413, 4135, 4095, 4118, 4096, 4099] avg=4096.0000 max=4863 max_rank=0 min=3413 min_rank=2 imbalance_ratio=1.187256 max_load_reduction=27905 max_load_reduction_pct=0.851593 moved_to_echo=16408 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3078, 0, 174, 0, 0, 0, 0, 0, 12, 15, 286, 4078, 102, 0, 0, 0, 6, 0, 0, 4054, 813, 0, 0, 0, 8, 3854, 0, 3, 65, 0, 0, 0, 0, 6, 10, 0, 4090, 0, 0, 0, 0, 11, 3903, 0, 28, 0, 0, 0, 0, 0, 4088, 7, 1, 0, 0, 0, 0, 1, 39, 0, 4033, 3, 0, 0] rerouted_tokens_per_ep_rank=[3252, 4493, 4873, 3930, 4106, 3942, 4096, 4076] avg=4096.0000 max=4873 max_rank=2 min=3252 min_rank=0 imbalance_ratio=1.189697 max_load_reduction=27895 max_load_reduction_pct=0.851288 moved_to_echo=16220 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3100, 0, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 99, 0, 0, 0, 0, 0, 0, 3338, 810, 0, 0, 0, 0, 4046, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 0] rerouted_tokens_per_ep_rank=[4210, 3893, 4148, 4109, 4095, 4118, 4096, 4099] avg=4096.0000 max=4210 max_rank=0 min=3893 min_rank=1 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16408 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3098, 0, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 99, 0, 0, 0, 0, 0, 0, 3256, 810, 0, 0, 0, 0, 4038, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 0] rerouted_tokens_per_ep_rank=[4137, 4056, 4066, 4101, 4095, 4118, 4096, 4099] avg=4096.0000 max=4137 max_rank=0 min=4056 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16408 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[24832, 0, 8064, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 26368, 6528, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 384, 31744, 896, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32640, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32256, 0, 128, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True + [2026-08-30 04:56:49] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 642.0 | throughput per GPU (TFLOP/s/GPU): 519.3 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.090033E+00 | loss scale: 1.0 | grad norm: 5.503 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1735, 0, 1529, 154, 1332, 1, 2294, 250, 1115, 1577, 0, 92, 1346, 1472, 233, 1076, 37, 1723, 1703, 1, 1, 1240, 2397, 838, 1, 2125, 2762, 1656, 1546, 2235, 296, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1936, 1, 1319, 412, 1000, 0, 1498, 319, 1098, 1728, 1, 49, 1103, 2372, 186, 1666, 64, 1651, 1504, 0, 0, 1605, 2225, 705, 0, 1968, 2633, 1641, 1136, 2712, 234, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1466, 0, 1333, 166, 810, 0, 1790, 208, 1829, 2616, 0, 34, 919, 2399, 66, 1607, 95, 2172, 584, 0, 0, 2016, 2380, 1173, 0, 1795, 2274, 1136, 1163, 2584, 153, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1401, 0, 1753, 407, 942, 7, 1609, 203, 1395, 1718, 1, 22, 1543, 1612, 153, 2002, 55, 1531, 1577, 0, 1, 1496, 2195, 1077, 0, 1917, 2438, 1334, 1499, 2614, 266, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1720, 0, 2243, 233, 959, 0, 2015, 234, 1219, 2493, 0, 23, 1331, 1907, 180, 1584, 40, 1817, 959, 1, 0, 1451, 1796, 377, 2, 1458, 2650, 1231, 2251, 2262, 332, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1408, 0, 2043, 246, 845, 0, 1883, 129, 1091, 2607, 0, 51, 1626, 2056, 240, 1336, 35, 1368, 1025, 0, 0, 1103, 2404, 1035, 2, 1408, 2822, 1553, 1731, 2556, 165, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1500, 0, 1552, 229, 748, 0, 1465, 274, 638, 1996, 0, 103, 1786, 1913, 176, 1716, 58, 1562, 1949, 0, 0, 1006, 2262, 795, 0, 2161, 2280, 1373, 2550, 2443, 231, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1986, 0, 1889, 280, 993, 0, 2164, 122, 1813, 1859, 0, 41, 1121, 2099, 201, 1793, 64, 1559, 1345, 0, 0, 1992, 1381, 896, 0, 1922, 2234, 1236, 848, 2668, 262, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1735, 0, 1529, 154, 219, 260, 0, 0, 1332, 1, 2294, 250, 1192, 0, 0, 0, 1115, 1577, 0, 92, 402, 172, 0, 0, 1346, 725, 233, 1076, 0, 0, 0, 0, 37, 1723, 1703, 1, 628, 345, 0, 0, 1, 1240, 1965, 838, 0, 0, 0, 0, 1, 2125, 942, 1656, 0, 0, 0, 0, 1546, 2016, 296, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3897, 5069, 3358, 3380, 4437, 4044, 4724, 3859] avg=4096.0000 max=5069 max_rank=1 min=3358 min_rank=2 imbalance_ratio=1.237549 max_load_reduction=27699 max_load_reduction_pct=0.845306 moved_to_echo=17064 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1936, 1, 1319, 412, 266, 241, 0, 0, 1000, 0, 1498, 319, 1136, 0, 0, 0, 1098, 1728, 1, 49, 649, 160, 0, 0, 1103, 1166, 186, 1666, 0, 0, 0, 0, 64, 1651, 1504, 0, 598, 557, 0, 0, 0, 1605, 1824, 705, 0, 0, 0, 0, 0, 1968, 899, 1641, 0, 0, 0, 0, 1136, 2446, 234, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 3953, 3685, 4121, 4374, 4134, 4508, 3818] avg=4096.0000 max=4508 max_rank=6 min=3685 min_rank=2 imbalance_ratio=1.100586 max_load_reduction=28260 max_load_reduction_pct=0.862427 moved_to_echo=16834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1720, 0, 2243, 233, 222, 195, 0, 0, 959, 0, 2015, 234, 1143, 0, 0, 0, 1219, 2493, 0, 23, 521, 129, 0, 0, 1331, 938, 180, 1584, 0, 0, 0, 0, 40, 1817, 959, 1, 602, 448, 0, 0, 0, 1451, 1472, 377, 0, 0, 0, 0, 2, 1458, 905, 1231, 0, 0, 0, 0, 2251, 2040, 332, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4613, 4351, 4385, 4033, 3867, 3300, 3596, 4623] avg=4096.0000 max=4623 max_rank=7 min=3300 min_rank=5 imbalance_ratio=1.128662 max_load_reduction=28145 max_load_reduction_pct=0.858917 moved_to_echo=15386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1408, 0, 2043, 246, 251, 261, 0, 0, 845, 0, 1883, 129, 1218, 0, 0, 0, 1091, 2607, 0, 51, 562, 172, 0, 0, 1626, 1011, 240, 1336, 0, 0, 0, 0, 35, 1368, 1025, 0, 641, 483, 0, 0, 0, 1103, 1971, 1035, 0, 0, 0, 0, 2, 1408, 963, 1553, 0, 0, 0, 0, 1731, 2305, 165, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4209, 4075, 4483, 4213, 3552, 4109, 3926, 4201] avg=4096.0000 max=4483 max_rank=2 min=3552 min_rank=4 imbalance_ratio=1.094482 max_load_reduction=28285 max_load_reduction_pct=0.863190 moved_to_echo=15788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1466, 0, 1333, 166, 254, 258, 0, 0, 810, 0, 1790, 208, 981, 0, 0, 0, 1829, 2616, 0, 34, 656, 171, 0, 0, 919, 1180, 66, 1607, 0, 0, 0, 0, 95, 2172, 584, 0, 517, 563, 0, 0, 0, 2016, 1951, 1173, 0, 0, 0, 0, 0, 1795, 776, 1136, 0, 0, 0, 0, 1163, 2330, 153, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3477, 3789, 5306, 3772, 3931, 5140, 3707, 3646] avg=4096.0000 max=5306 max_rank=2 min=3477 min_rank=0 imbalance_ratio=1.295410 max_load_reduction=27462 max_load_reduction_pct=0.838074 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1401, 0, 1753, 407, 260, 243, 0, 0, 942, 7, 1609, 203, 1056, 0, 0, 0, 1395, 1718, 1, 22, 446, 161, 0, 0, 1543, 785, 153, 2002, 0, 0, 0, 0, 55, 1531, 1577, 0, 558, 381, 0, 0, 1, 1496, 1791, 1077, 0, 0, 0, 0, 0, 1917, 824, 1334, 0, 0, 0, 0, 1499, 2354, 266, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 3817, 3743, 4483, 4102, 4365, 4075, 4119] avg=4096.0000 max=4483 max_rank=3 min=3743 min_rank=2 imbalance_ratio=1.094482 max_load_reduction=28285 max_load_reduction_pct=0.863190 moved_to_echo=16661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1500, 0, 1552, 229, 240, 245, 0, 0, 748, 0, 1465, 274, 984, 0, 0, 0, 638, 1996, 0, 103, 523, 162, 0, 0, 1786, 941, 176, 1716, 0, 0, 0, 0, 58, 1562, 1949, 0, 518, 449, 0, 0, 0, 1006, 1855, 795, 0, 0, 0, 0, 0, 2161, 778, 1373, 0, 0, 0, 0, 2550, 2203, 231, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3766, 3471, 3422, 4619, 4536, 3656, 4312, 4986] avg=4096.0000 max=4986 max_rank=7 min=3422 min_rank=2 imbalance_ratio=1.217285 max_load_reduction=27782 max_load_reduction_pct=0.847839 moved_to_echo=17490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1986, 0, 1889, 280, 262, 150, 0, 0, 993, 0, 2164, 122, 964, 0, 0, 0, 1813, 1859, 0, 41, 574, 99, 0, 0, 1121, 1032, 201, 1793, 0, 0, 0, 0, 64, 1559, 1345, 0, 508, 493, 0, 0, 0, 1992, 1132, 896, 0, 0, 0, 0, 0, 1922, 762, 1236, 0, 0, 0, 0, 848, 2406, 262, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4567, 4243, 4386, 4147, 3969, 4020, 3920, 3516] avg=4096.0000 max=4567 max_rank=0 min=3516 min_rank=7 imbalance_ratio=1.114990 max_load_reduction=28201 max_load_reduction_pct=0.860626 moved_to_echo=15425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13184, 128, 13696, 2176, 2048, 1920, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10240, 16640, 128, 512, 4352, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 13440, 10752, 128, 4608, 3840, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7680, 128, 14720, 1792, 8704, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 14848, 6912, 11264, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10880, 7808, 1536, 12800, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12800, 18176, 2048, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 12032, 14080, 6912, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[366, 0, 0, 0, 3151, 0, 0, 0, 0, 2541, 4095, 4058, 4012, 0, 18, 1958, 4094, 0, 0, 0, 320, 0, 0, 0, 4095, 2, 0, 1, 0, 4057, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[41, 1, 0, 0, 902, 1, 0, 0, 2, 2866, 4079, 3857, 4016, 8, 36, 3188, 4095, 0, 1, 1, 1474, 0, 0, 0, 4091, 102, 0, 1, 0, 4004, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 336, 0, 0, 0, 0, 3276, 4087, 4048, 3990, 1, 8, 3702, 4096, 0, 0, 0, 1001, 0, 0, 0, 4095, 170, 0, 0, 0, 3948, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[18, 1, 1, 0, 1531, 1, 0, 0, 0, 3562, 4093, 3758, 3941, 0, 2, 3069, 4095, 0, 0, 1, 654, 1, 1, 0, 4094, 4, 0, 0, 0, 3940, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[105, 0, 0, 0, 1801, 0, 0, 0, 0, 2579, 4095, 4064, 4093, 0, 37, 3046, 4096, 0, 0, 0, 728, 0, 0, 0, 4095, 4, 0, 0, 0, 4025, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[217, 0, 0, 0, 2188, 0, 0, 0, 0, 2563, 4096, 4011, 3670, 0, 3, 3483, 4096, 0, 0, 0, 272, 0, 0, 0, 4095, 8, 0, 0, 0, 4066, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[26, 0, 0, 0, 884, 0, 0, 0, 0, 2844, 4094, 4074, 3999, 4, 9, 3803, 4095, 0, 0, 0, 801, 0, 0, 7, 4095, 22, 0, 0, 0, 4011, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[83, 1, 0, 0, 1582, 2, 0, 0, 1, 2953, 4089, 4045, 3973, 0, 57, 3219, 4094, 2, 0, 1, 592, 0, 2, 0, 4093, 35, 1, 1, 0, 3940, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[41, 1, 0, 0, 3976, 0, 0, 0, 902, 1, 0, 0, 2464, 0, 0, 0, 2, 2866, 0, 1159, 0, 0, 0, 0, 902, 8, 36, 3188, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 1474, 0, 0, 0, 3114, 182, 103, 0, 4050, 102, 0, 1, 0, 0, 0, 0, 0, 4004, 1, 1, 52, 41, 0, 0] rerouted_tokens_per_ep_rank=[4018, 3367, 4027, 4134, 4097, 4873, 4153, 4099] avg=4096.0000 max=4873 max_rank=5 min=3367 min_rank=1 imbalance_ratio=1.189697 max_load_reduction=27895 max_load_reduction_pct=0.851288 moved_to_echo=17222 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[18, 1, 1, 0, 3989, 0, 0, 0, 1531, 1, 0, 0, 2400, 0, 0, 0, 0, 3562, 0, 1130, 0, 0, 0, 0, 885, 0, 2, 3069, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 654, 1, 1, 0, 3056, 177, 104, 0, 4053, 4, 0, 0, 0, 0, 0, 0, 0, 3940, 1, 0, 51, 41, 0, 0] rerouted_tokens_per_ep_rank=[4009, 3932, 4692, 3956, 4096, 3993, 4057, 4033] avg=4096.0000 max=4692 max_rank=2 min=3932 min_rank=1 imbalance_ratio=1.145508 max_load_reduction=28076 max_load_reduction_pct=0.856812 moved_to_echo=16179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[217, 0, 0, 0, 3992, 0, 0, 0, 2188, 0, 0, 0, 2562, 0, 0, 0, 0, 2563, 0, 1206, 0, 0, 0, 0, 825, 0, 3, 3483, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 2845, 189, 104, 0, 4054, 8, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 54, 41, 0, 0] rerouted_tokens_per_ep_rank=[4209, 4750, 3769, 4311, 4096, 3410, 4062, 4161] avg=4096.0000 max=4750 max_rank=1 min=3410 min_rank=5 imbalance_ratio=1.159668 max_load_reduction=28018 max_load_reduction_pct=0.855042 moved_to_echo=15729 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[105, 0, 0, 0, 3991, 0, 0, 0, 1801, 0, 0, 0, 2596, 0, 0, 0, 0, 2579, 0, 1221, 0, 0, 0, 0, 920, 0, 37, 3046, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 728, 0, 0, 0, 3173, 192, 104, 0, 4054, 4, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 55, 41, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4397, 3800, 4003, 4096, 4197, 4058, 4121] avg=4096.0000 max=4397 max_rank=1 min=3800 min_rank=2 imbalance_ratio=1.073486 max_load_reduction=28371 max_load_reduction_pct=0.865814 moved_to_echo=16472 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[83, 1, 0, 0, 3985, 0, 0, 0, 1582, 2, 0, 0, 2584, 0, 0, 0, 1, 2953, 0, 1215, 0, 0, 0, 0, 893, 0, 57, 3219, 0, 0, 0, 0, 4094, 2, 0, 1, 0, 0, 0, 0, 592, 0, 2, 0, 3080, 191, 104, 0, 4052, 35, 1, 1, 0, 0, 0, 0, 0, 3940, 2, 0, 55, 41, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4168, 4169, 4169, 4097, 3969, 4089, 4038] avg=4096.0000 max=4169 max_rank=2 min=3969 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16193 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 3983, 0, 0, 0, 336, 0, 0, 0, 2586, 0, 0, 0, 0, 3276, 0, 1216, 0, 0, 0, 0, 896, 1, 8, 3702, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1001, 0, 0, 0, 3094, 191, 104, 0, 4054, 170, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 0, 55, 41, 0, 0] rerouted_tokens_per_ep_rank=[3993, 2922, 4492, 4607, 4096, 4390, 4224, 4044] avg=4096.0000 max=4607 max_rank=3 min=2922 min_rank=1 imbalance_ratio=1.124756 max_load_reduction=28161 max_load_reduction_pct=0.859406 moved_to_echo=16754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[26, 0, 0, 0, 3991, 0, 0, 0, 884, 0, 0, 0, 2605, 0, 0, 0, 0, 2844, 0, 1214, 0, 0, 0, 0, 895, 4, 9, 3803, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 801, 0, 0, 7, 3104, 195, 103, 0, 4046, 22, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 60, 48, 0, 0] rerouted_tokens_per_ep_rank=[4017, 3489, 4058, 4711, 4096, 4210, 4068, 4119] avg=4096.0000 max=4711 max_rank=3 min=3489 min_rank=1 imbalance_ratio=1.150146 max_load_reduction=28057 max_load_reduction_pct=0.856232 moved_to_echo=16493 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[366, 0, 0, 0, 3991, 0, 0, 0, 3151, 0, 0, 0, 2592, 0, 0, 0, 0, 2541, 0, 1220, 0, 0, 0, 0, 901, 0, 18, 1958, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 0, 3111, 191, 104, 0, 4054, 2, 0, 1, 0, 0, 0, 0, 0, 4057, 0, 0, 55, 41, 0, 0] rerouted_tokens_per_ep_rank=[4357, 5743, 3761, 2877, 4094, 3726, 4057, 4153] avg=4096.0000 max=5743 max_rank=1 min=2877 min_rank=3 imbalance_ratio=1.402100 max_load_reduction=27025 max_load_reduction_pct=0.824738 moved_to_echo=16030 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12416, 128, 0, 0, 20480, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[5888, 128, 128, 128, 24704, 1536, 896, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 23296, 0, 9600, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7168, 128, 256, 25472, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 128, 128, 0, 32000, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 384, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32000, 128, 128, 512, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 941, 1, 21, 0, 4094, 453, 1342, 584, 4085, 0, 4095, 1, 1103, 0, 1, 1, 0, 1, 1, 0, 4095, 4095, 19, 4060, 0, 0, 0, 2523, 1, 1251] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1451, 1, 15, 0, 4095, 262, 1146, 314, 4087, 0, 4094, 1, 1348, 0, 1, 1, 0, 2, 1, 0, 4095, 4095, 31, 4046, 0, 0, 0, 3205, 1, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2115, 0, 1, 0, 4096, 422, 1115, 1029, 4080, 0, 4096, 0, 1211, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 8, 4083, 0, 0, 0, 2217, 0, 103] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1452, 0, 6, 0, 4096, 805, 2210, 294, 4093, 0, 4096, 0, 1217, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4067, 0, 0, 0, 1973, 0, 263] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[7, 8, 1207, 0, 31, 1, 4096, 324, 553, 1143, 4054, 0, 4087, 0, 2068, 1, 0, 0, 0, 0, 0, 0, 4096, 4090, 6, 4036, 0, 0, 0, 2474, 0, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2002, 2, 1, 0, 4094, 141, 1623, 709, 4094, 0, 4094, 1, 1313, 0, 1, 2, 0, 2, 1, 0, 4094, 4089, 25, 4070, 0, 3, 3, 2112, 0, 292] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1478, 0, 0, 0, 4096, 188, 691, 476, 3885, 0, 4096, 0, 1406, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 2, 4077, 0, 0, 0, 3184, 0, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 1222, 0, 1, 0, 4095, 824, 1842, 664, 4051, 0, 4095, 0, 986, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 4079, 0, 0, 0, 2665, 0, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 941, 1, 1939, 686, 0, 0, 21, 0, 3659, 453, 0, 0, 0, 0, 1342, 584, 2146, 0, 0, 0, 0, 0, 2765, 1, 1103, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4095, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 19, 4060, 0, 0, 18, 0, 0, 0, 0, 2523, 1, 1251, 644, 417, 0, 0] rerouted_tokens_per_ep_rank=[3567, 4133, 4072, 3869, 4098, 4096, 4097, 4836] avg=4096.0000 max=4836 max_rank=7 min=3567 min_rank=0 imbalance_ratio=1.180664 max_load_reduction=27932 max_load_reduction_pct=0.852417 moved_to_echo=17127 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1451, 1, 1939, 685, 0, 0, 15, 0, 3660, 262, 0, 0, 0, 0, 1146, 314, 2148, 0, 0, 0, 0, 0, 2765, 1, 1348, 0, 0, 0, 0, 0, 1, 1, 0, 2, 4093, 0, 0, 0, 1, 0, 2, 4095, 0, 0, 0, 0, 31, 4046, 0, 0, 18, 0, 0, 0, 0, 3205, 1, 476, 644, 417, 0, 0] rerouted_tokens_per_ep_rank=[4076, 3937, 3608, 4114, 4097, 4098, 4095, 4743] avg=4096.0000 max=4743 max_rank=7 min=3608 min_rank=2 imbalance_ratio=1.157959 max_load_reduction=28025 max_load_reduction_pct=0.855255 moved_to_echo=17033 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2115, 0, 1936, 686, 0, 0, 1, 0, 3661, 422, 0, 0, 0, 0, 1115, 1029, 2144, 0, 0, 0, 0, 0, 2766, 0, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 8, 4083, 0, 0, 18, 0, 0, 0, 0, 2217, 0, 103, 644, 417, 0, 0] rerouted_tokens_per_ep_rank=[4737, 4084, 4288, 3977, 4094, 4098, 4109, 3381] avg=4096.0000 max=4737 max_rank=0 min=3381 min_rank=7 imbalance_ratio=1.156494 max_load_reduction=28031 max_load_reduction_pct=0.855438 moved_to_echo=15682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2002, 2, 1943, 685, 0, 0, 1, 0, 3659, 141, 0, 0, 0, 0, 1623, 709, 2151, 0, 0, 0, 0, 0, 2765, 1, 1313, 0, 0, 0, 0, 0, 1, 2, 0, 2, 4092, 0, 0, 0, 1, 0, 2, 4089, 0, 0, 0, 0, 25, 4070, 0, 3, 18, 0, 0, 0, 3, 2112, 0, 292, 644, 417, 0, 0] rerouted_tokens_per_ep_rank=[4632, 3801, 4483, 4079, 4097, 4092, 4116, 3468] avg=4096.0000 max=4632 max_rank=0 min=3468 min_rank=7 imbalance_ratio=1.130859 max_load_reduction=28136 max_load_reduction_pct=0.858643 moved_to_echo=15773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[7, 8, 1207, 0, 1928, 687, 0, 0, 31, 1, 3651, 324, 0, 0, 0, 0, 553, 1143, 2123, 0, 0, 0, 0, 0, 2753, 0, 2068, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 6, 4036, 0, 0, 21, 0, 0, 0, 0, 2474, 0, 486, 647, 424, 0, 0] rerouted_tokens_per_ep_rank=[3837, 4007, 3819, 4822, 4099, 4090, 4063, 4031] avg=4096.0000 max=4822 max_rank=3 min=3819 min_rank=2 imbalance_ratio=1.177246 max_load_reduction=27946 max_load_reduction_pct=0.852844 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1452, 0, 1942, 686, 0, 0, 6, 0, 3661, 805, 0, 0, 0, 0, 2210, 294, 2151, 0, 0, 0, 0, 0, 2766, 0, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4, 4067, 0, 0, 18, 0, 0, 0, 0, 1973, 0, 263, 644, 417, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4472, 4655, 3983, 4094, 4098, 4089, 3297] avg=4096.0000 max=4655 max_rank=2 min=3297 min_rank=7 imbalance_ratio=1.136475 max_load_reduction=28113 max_load_reduction_pct=0.857941 moved_to_echo=15578 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1222, 0, 1922, 686, 0, 0, 1, 0, 3660, 824, 0, 0, 0, 0, 1842, 664, 2129, 0, 0, 0, 0, 0, 2765, 0, 986, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 5, 4079, 0, 0, 18, 0, 0, 0, 0, 2665, 0, 45, 644, 417, 0, 0] rerouted_tokens_per_ep_rank=[3831, 4485, 4635, 3752, 4094, 4098, 4102, 3771] avg=4096.0000 max=4635 max_rank=2 min=3752 min_rank=3 imbalance_ratio=1.131592 max_load_reduction=28133 max_load_reduction_pct=0.858551 moved_to_echo=16065 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1478, 0, 1844, 686, 0, 0, 0, 0, 3661, 188, 0, 0, 0, 0, 691, 476, 2041, 0, 0, 0, 0, 0, 2766, 0, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 1, 4096, 0, 0, 0, 0, 2, 4077, 0, 0, 18, 0, 0, 0, 0, 3184, 0, 996, 644, 417, 0, 0] rerouted_tokens_per_ep_rank=[4008, 3849, 3208, 4172, 4095, 4098, 4097, 5241] avg=4096.0000 max=5241 max_rank=7 min=3208 min_rank=2 imbalance_ratio=1.279541 max_load_reduction=27527 max_load_reduction_pct=0.840057 moved_to_echo=17531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 11904, 128, 15488, 5504, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10624, 5248, 17152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 20480, 128, 3968, 5248, 3456, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[22144, 128, 10752, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 29312, 3456, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 0, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3615, 0, 3910, 0, 4057, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 2136, 0, 2807, 4052, 3822, 10, 0, 0, 165] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 2833, 0, 4014, 0, 4083, 1, 0, 1, 0, 1, 0, 4094, 153, 1, 4088, 0, 1, 0, 2272, 13, 2961, 4095, 3796, 4, 1, 1, 352] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[3, 0, 0, 1, 1, 3303, 2, 4023, 0, 3940, 1, 0, 1, 0, 1, 0, 4094, 42, 1, 4071, 0, 1, 0, 1689, 30, 3276, 4095, 3977, 3, 0, 5, 208] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3053, 0, 3960, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 78, 0, 4077, 3, 0, 1, 2302, 12, 2831, 4092, 3845, 0, 0, 16, 307] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[2, 5, 0, 2, 2, 3529, 5, 3974, 0, 4092, 1, 0, 2, 0, 2, 0, 4091, 235, 2, 4094, 0, 0, 0, 1894, 4, 2135, 4094, 4065, 50, 2, 0, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3173, 0, 3789, 0, 4052, 0, 0, 0, 0, 0, 0, 4096, 30, 0, 4086, 0, 0, 0, 2686, 1, 2602, 4055, 3884, 1, 0, 0, 313] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3454, 0, 3666, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 121, 0, 4096, 0, 0, 0, 2906, 2, 2144, 4096, 4003, 9, 0, 0, 84] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2910, 0, 3202, 0, 4079, 0, 0, 0, 0, 0, 0, 4096, 85, 0, 4095, 0, 0, 0, 3058, 1, 2790, 4075, 3899, 12, 0, 2, 464] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 13, 0, 0, 0, 2833, 0, 906, 0, 0, 0, 0, 0, 4083, 1, 0, 34, 0, 0, 0, 1, 0, 1, 0, 4094, 0, 0, 0, 0, 153, 1, 4003, 0, 0, 0, 0, 0, 1, 0, 2272, 1626, 51, 0, 0, 13, 2961, 0, 1355, 0, 0, 0, 0, 4, 1, 1, 352, 3095, 815, 0, 0] rerouted_tokens_per_ep_rank=[4111, 3739, 4118, 4096, 4157, 3950, 4329, 4268] avg=4096.0000 max=4329 max_rank=6 min=3739 min_rank=1 imbalance_ratio=1.056885 max_load_reduction=28439 max_load_reduction_pct=0.867889 moved_to_echo=16704 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 1, 4095, 13, 0, 0, 1, 3303, 2, 908, 0, 0, 0, 0, 0, 3940, 1, 0, 34, 0, 0, 0, 1, 0, 1, 0, 4094, 0, 0, 0, 0, 42, 1, 3987, 0, 0, 0, 0, 0, 1, 0, 1689, 1704, 50, 0, 0, 30, 3276, 0, 1419, 0, 0, 0, 0, 3, 0, 5, 208, 3102, 854, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4214, 3975, 4096, 4030, 3444, 4725, 4172] avg=4096.0000 max=4725 max_rank=6 min=3444 min_rank=5 imbalance_ratio=1.153564 max_load_reduction=28043 max_load_reduction_pct=0.855804 moved_to_echo=16371 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 5, 0, 2, 4094, 12, 0, 0, 2, 3529, 5, 898, 0, 0, 0, 0, 0, 4092, 1, 0, 34, 0, 0, 0, 2, 0, 2, 0, 4091, 0, 0, 0, 0, 235, 2, 4009, 0, 0, 0, 0, 0, 0, 0, 1894, 1742, 51, 0, 0, 4, 2135, 0, 1450, 0, 0, 0, 0, 50, 2, 0, 486, 3064, 873, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4434, 4127, 4095, 4246, 3687, 3589, 4475] avg=4096.0000 max=4475 max_rank=7 min=3589 min_rank=6 imbalance_ratio=1.092529 max_load_reduction=28293 max_load_reduction_pct=0.863434 moved_to_echo=15997 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4075, 10, 0, 0, 0, 2910, 0, 723, 0, 0, 0, 0, 0, 4079, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 85, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 3058, 1670, 51, 0, 0, 1, 2790, 0, 1392, 0, 0, 0, 0, 12, 0, 2, 464, 2469, 837, 0, 0] rerouted_tokens_per_ep_rank=[4085, 3633, 4113, 4096, 4095, 4779, 4183, 3784] avg=4096.0000 max=4779 max_rank=5 min=3633 min_rank=1 imbalance_ratio=1.166748 max_load_reduction=27989 max_load_reduction_pct=0.854156 moved_to_echo=16841 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4054, 12, 0, 0, 0, 3173, 0, 855, 0, 0, 0, 0, 0, 4052, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 30, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 2686, 1664, 51, 0, 0, 1, 2602, 0, 1386, 0, 0, 0, 0, 1, 0, 0, 313, 2922, 834, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4028, 4086, 4097, 4031, 4401, 3989, 4070] avg=4096.0000 max=4401 max_rank=5 min=3989 min_rank=6 imbalance_ratio=1.074463 max_load_reduction=28367 max_load_reduction_pct=0.865692 moved_to_echo=16491 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 16, 0, 0, 0, 3053, 0, 887, 0, 0, 0, 0, 0, 4095, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 78, 0, 3986, 0, 0, 0, 0, 3, 0, 1, 2302, 1652, 53, 0, 0, 12, 2831, 0, 1364, 0, 0, 0, 0, 0, 0, 16, 307, 3057, 829, 0, 0] rerouted_tokens_per_ep_rank=[4108, 3940, 4133, 4096, 4064, 4011, 4207, 4209] avg=4096.0000 max=4209 max_rank=7 min=3940 min_rank=1 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16491 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4052, 12, 0, 0, 0, 3615, 0, 883, 0, 0, 0, 0, 0, 4057, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 2136, 1637, 51, 0, 0, 0, 2807, 0, 1364, 0, 0, 0, 0, 10, 0, 0, 165, 3015, 821, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4498, 4091, 4096, 4013, 3824, 4171, 4011] avg=4096.0000 max=4498 max_rank=1 min=3824 min_rank=5 imbalance_ratio=1.098145 max_load_reduction=28270 max_load_reduction_pct=0.862732 moved_to_echo=16019 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 11, 0, 0, 0, 3454, 0, 828, 0, 0, 0, 0, 0, 4091, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 121, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 2906, 1715, 51, 0, 0, 2, 2144, 0, 1429, 0, 0, 0, 0, 9, 0, 0, 84, 2827, 859, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4282, 4125, 4096, 4132, 4672, 3575, 3779] avg=4096.0000 max=4672 max_rank=5 min=3575 min_rank=6 imbalance_ratio=1.140625 max_load_reduction=28096 max_load_reduction_pct=0.857422 moved_to_echo=16158 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 18944, 13440, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 2432, 23552, 6784, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 25984, 128, 6912, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 21632, 0, 11264, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 768, 128, 32128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32512, 128, 0, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[2, 2, 2, 4094, 2756, 4021, 4094, 0, 2, 0, 0, 4094, 2, 1, 0, 4094, 137, 0, 0, 0, 2, 14, 0, 927, 0, 0, 4093, 2, 4094, 2, 0, 333] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 2934, 3307, 4095, 0, 1, 0, 0, 4095, 1, 0, 0, 4095, 195, 0, 0, 0, 1, 3, 0, 1325, 0, 0, 4094, 1, 4047, 1, 0, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 1926, 3893, 4095, 0, 1, 3, 0, 4095, 1, 0, 1, 4095, 43, 0, 0, 0, 1, 2, 0, 2219, 0, 0, 4095, 0, 4043, 1, 0, 156] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1969, 3874, 4096, 0, 0, 16, 0, 4076, 0, 0, 0, 4094, 43, 0, 0, 0, 0, 0, 0, 1820, 0, 22, 4090, 0, 4030, 0, 0, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2850, 3691, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 1575, 0, 0, 4096, 0, 4090, 0, 0, 70] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2874, 3170, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 86, 0, 0, 0, 0, 0, 0, 1770, 0, 0, 4094, 0, 4086, 0, 0, 304] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2792, 3959, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 82, 0, 0, 0, 0, 0, 0, 735, 0, 0, 4095, 0, 4086, 0, 0, 635] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 3179, 2962, 4096, 0, 0, 2, 0, 4095, 0, 0, 0, 4096, 67, 0, 0, 0, 0, 0, 0, 1046, 0, 1, 4095, 0, 4079, 0, 0, 955] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 0, 0, 0, 0, 1926, 1549, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4095, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 43, 0, 0, 0, 4013, 0, 0, 0, 1, 2, 0, 2219, 2344, 404, 82, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 3639, 1, 0, 156, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 3475, 4099, 4097, 4056, 5053, 4094, 3796] avg=4096.0000 max=5053 max_rank=5 min=3475 min_rank=1 imbalance_ratio=1.233643 max_load_reduction=27715 max_load_reduction_pct=0.845795 moved_to_echo=16999 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 0, 0, 0, 0, 2934, 1316, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 195, 0, 0, 0, 4012, 0, 0, 0, 1, 3, 0, 1325, 1991, 405, 83, 1, 0, 0, 4093, 1, 0, 0, 0, 0, 3642, 1, 0, 475, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4250, 4096, 4096, 4207, 3809, 4094, 4118] avg=4096.0000 max=4250 max_rank=1 min=3809 min_rank=5 imbalance_ratio=1.037598 max_load_reduction=28518 max_load_reduction_pct=0.870300 moved_to_echo=16228 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 2, 4094, 0, 0, 0, 0, 2756, 1600, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 2, 1, 0, 4094, 0, 0, 0, 0, 137, 0, 0, 0, 4011, 0, 0, 0, 2, 14, 0, 927, 2421, 410, 83, 1, 0, 0, 4092, 2, 0, 0, 0, 0, 3684, 2, 0, 333, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4356, 4096, 4097, 4148, 3858, 4094, 4019] avg=4096.0000 max=4356 max_rank=1 min=3858 min_rank=5 imbalance_ratio=1.063477 max_load_reduction=28412 max_load_reduction_pct=0.867065 moved_to_echo=16119 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 3179, 1179, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 67, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 1046, 1783, 408, 83, 1, 0, 1, 4094, 0, 0, 0, 0, 0, 3671, 0, 0, 955, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4358, 4097, 4096, 4080, 3321, 4095, 4626] avg=4096.0000 max=4626 max_rank=7 min=3321 min_rank=5 imbalance_ratio=1.129395 max_load_reduction=28142 max_load_reduction_pct=0.858826 moved_to_echo=16122 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2792, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 82, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 735, 2384, 409, 83, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 3677, 0, 0, 635, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4367, 4096, 4096, 4095, 3612, 4094, 4312] avg=4096.0000 max=4367 max_rank=1 min=3612 min_rank=5 imbalance_ratio=1.066162 max_load_reduction=28401 max_load_reduction_pct=0.866730 moved_to_echo=16113 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 1969, 1538, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 43, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 1820, 2336, 407, 82, 3, 0, 22, 4087, 0, 0, 0, 0, 0, 3623, 0, 0, 542, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3507, 4092, 4094, 4057, 4648, 4109, 4165] avg=4096.0000 max=4648 max_rank=5 min=3507 min_rank=1 imbalance_ratio=1.134766 max_load_reduction=28120 max_load_reduction_pct=0.858154 moved_to_echo=16979 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2874, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 86, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 1770, 1908, 409, 83, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 3677, 0, 0, 304, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4136, 4096, 4096, 4099, 4171, 4093, 3981] avg=4096.0000 max=4171 max_rank=5 min=3981 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16344 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2850, 1469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 1575, 2222, 409, 82, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 3681, 0, 0, 70, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4319, 4096, 4096, 4026, 4289, 4095, 3751] avg=4096.0000 max=4319 max_rank=1 min=3751 min_rank=7 imbalance_ratio=1.054443 max_load_reduction=28449 max_load_reduction_pct=0.868195 moved_to_echo=16161 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32775, 32768, 32768, 32768, 32768, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32775, 32768, 32768, 32768, 32768, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32775, 32768, 32768, 32768, 32768, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32775, 32768, 32768, 32768, 32768, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32775, 32768, 32768, 32768, 32768, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32775, 32768, 32768, 32768, 32768, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32775, 32768, 32768, 32768, 32768, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32775, 32768, 32768, 32768, 32768, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[21376, 11520, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32761 actual_valid_recv=33408 observed_loss=-647 tokens_per_expert=[128, 128, 0, 11520, 17408, 3328, 768, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[29312, 128, 0, 3584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32775 actual_valid_recv=33152 observed_loss=-377 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[768, 0, 0, 0, 32128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 3244, 0, 0, 911, 4095, 0, 1, 1, 2081, 0, 1, 0, 4079, 0, 4093, 1, 653, 4092, 1, 4095, 855, 1, 4095, 0, 0, 2, 465, 0, 1, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 3166, 1, 0, 1125, 4094, 0, 2, 2, 1481, 0, 2, 0, 4048, 0, 4093, 0, 621, 4096, 2, 4094, 1529, 2, 4094, 2, 0, 2, 310, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 3056, 0, 0, 1521, 4095, 0, 1, 1, 1230, 1, 1, 0, 4058, 0, 4079, 0, 1365, 4050, 1, 4095, 728, 0, 4095, 1, 0, 1, 387, 0, 1, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 3356, 0, 0, 939, 4096, 0, 0, 0, 1224, 0, 0, 0, 4096, 0, 4096, 0, 1270, 4085, 0, 4096, 767, 0, 4095, 0, 0, 0, 648, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 3043, 0, 4, 1525, 4068, 15, 0, 0, 1468, 0, 0, 0, 4094, 8, 4096, 0, 1401, 4008, 0, 4096, 727, 0, 4089, 0, 0, 0, 126, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 2920, 0, 0, 916, 4082, 0, 0, 0, 2399, 0, 0, 0, 4096, 0, 4096, 0, 692, 4055, 0, 4096, 1108, 0, 4096, 0, 0, 0, 212, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 3101, 0, 1, 1743, 4078, 3, 0, 0, 1350, 0, 0, 0, 4094, 1, 4095, 0, 958, 4023, 0, 4095, 1095, 0, 4096, 0, 0, 0, 35, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 3034, 0, 0, 862, 4096, 0, 0, 0, 1987, 0, 0, 0, 4022, 0, 4096, 0, 985, 4076, 0, 4096, 1260, 0, 4096, 0, 0, 0, 158, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3056, 0, 0, 955, 21, 0, 0, 1521, 2906, 0, 1, 0, 0, 0, 0, 1, 1230, 1, 1, 1189, 985, 268, 0, 0, 4058, 0, 22, 0, 0, 0, 0, 0, 1365, 3095, 1, 0, 0, 0, 0, 0, 728, 0, 3089, 0, 0, 0, 0, 1, 0, 1, 387, 3789, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4428, 3675, 4080, 4461, 3817, 4178, 4097] avg=4096.0000 max=4461 max_rank=4 min=3675 min_rank=2 imbalance_ratio=1.089111 max_load_reduction=28307 max_load_reduction_pct=0.863861 moved_to_echo=16553 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3244, 0, 0, 965, 21, 0, 0, 911, 2906, 0, 1, 0, 0, 0, 0, 1, 2081, 0, 1, 1189, 985, 269, 0, 0, 4079, 0, 22, 0, 0, 0, 0, 1, 653, 3127, 1, 0, 0, 0, 0, 0, 855, 1, 3089, 0, 0, 0, 0, 0, 0, 2, 465, 3802, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4230, 3818, 4526, 4101, 3782, 3945, 4269, 4097] avg=4096.0000 max=4526 max_rank=2 min=3782 min_rank=4 imbalance_ratio=1.104980 max_load_reduction=28242 max_load_reduction_pct=0.861877 moved_to_echo=16093 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3166, 1, 0, 966, 21, 0, 0, 1125, 2905, 0, 2, 0, 0, 0, 0, 2, 1481, 0, 2, 1189, 985, 269, 0, 0, 4048, 0, 22, 0, 0, 0, 0, 0, 621, 3130, 2, 0, 0, 0, 0, 0, 1529, 2, 3088, 0, 0, 0, 0, 2, 0, 2, 310, 3802, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4032, 3928, 4070, 3753, 4619, 4116, 4096] avg=4096.0000 max=4619 max_rank=5 min=3753 min_rank=4 imbalance_ratio=1.127686 max_load_reduction=28149 max_load_reduction_pct=0.859039 moved_to_echo=16584 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3101, 0, 1, 949, 21, 0, 0, 1743, 2894, 3, 0, 0, 0, 0, 0, 0, 1350, 0, 0, 1184, 985, 269, 0, 0, 4094, 1, 22, 0, 0, 0, 0, 0, 958, 3074, 0, 0, 0, 0, 0, 0, 1095, 0, 3090, 0, 0, 0, 0, 0, 0, 0, 35, 3804, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4640, 3788, 4117, 4032, 4185, 3839, 4095] avg=4096.0000 max=4640 max_rank=1 min=3788 min_rank=2 imbalance_ratio=1.132812 max_load_reduction=28128 max_load_reduction_pct=0.858398 moved_to_echo=16151 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2920, 0, 0, 957, 21, 0, 0, 916, 2897, 0, 0, 0, 0, 0, 0, 0, 2399, 0, 0, 1185, 985, 269, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 692, 3098, 0, 0, 0, 0, 0, 0, 1108, 0, 3090, 0, 0, 0, 0, 0, 0, 0, 212, 3805, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3898, 3813, 4838, 4118, 3790, 4198, 4017, 4096] avg=4096.0000 max=4838 max_rank=2 min=3790 min_rank=4 imbalance_ratio=1.181152 max_load_reduction=27930 max_load_reduction_pct=0.852356 moved_to_echo=16101 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3043, 0, 4, 949, 28, 0, 0, 1525, 2882, 15, 0, 0, 0, 0, 0, 0, 1468, 0, 0, 1186, 990, 272, 0, 0, 4094, 8, 18, 0, 0, 0, 0, 0, 1401, 3059, 0, 0, 0, 0, 0, 0, 727, 0, 3071, 0, 0, 0, 0, 0, 0, 0, 126, 3806, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4422, 3916, 4120, 4460, 3798, 3933, 4095] avg=4096.0000 max=4460 max_rank=4 min=3798 min_rank=5 imbalance_ratio=1.088867 max_load_reduction=28308 max_load_reduction_pct=0.863892 moved_to_echo=16286 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3356, 0, 0, 964, 21, 0, 0, 939, 2907, 0, 0, 0, 0, 0, 0, 0, 1224, 0, 0, 1189, 985, 269, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 1270, 3121, 0, 0, 0, 0, 0, 0, 767, 0, 3089, 0, 0, 0, 0, 0, 0, 0, 648, 3805, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4341, 3846, 3667, 4118, 4391, 3856, 4453, 4096] avg=4096.0000 max=4453 max_rank=6 min=3667 min_rank=2 imbalance_ratio=1.087158 max_load_reduction=28315 max_load_reduction_pct=0.864105 moved_to_echo=16796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3034, 0, 0, 962, 21, 0, 0, 862, 2907, 0, 0, 0, 0, 0, 0, 0, 1987, 0, 0, 1189, 985, 269, 0, 0, 4022, 0, 22, 0, 0, 0, 0, 0, 985, 3114, 0, 0, 0, 0, 0, 0, 1260, 0, 3090, 0, 0, 0, 0, 0, 0, 0, 158, 3805, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 3769, 4430, 4044, 4099, 4350, 3963, 4096] avg=4096.0000 max=4430 max_rank=2 min=3769 min_rank=1 imbalance_ratio=1.081543 max_load_reduction=28338 max_load_reduction_pct=0.864807 moved_to_echo=16508 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 13312, 128, 128, 9600, 7936, 2176, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 24960, 128, 128, 7680, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 8064, 24832, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 8192, 128, 24704, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9600, 23296, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 2432, 30464, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32640, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[2, 1, 0, 4094, 0, 2, 0, 3, 1, 382, 2331, 2, 2010, 0, 2, 4093, 4094, 164, 4092, 0, 2, 2, 4094, 3237, 0, 0, 67, 1, 0, 4091, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 19, 0, 2, 0, 0, 147, 1795, 0, 2839, 0, 0, 4096, 4096, 417, 4096, 0, 0, 0, 4096, 2786, 0, 0, 187, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 25, 0, 0, 0, 0, 147, 2856, 0, 2237, 0, 0, 4096, 4096, 100, 4096, 0, 0, 0, 4096, 2837, 0, 0, 1, 0, 0, 4085, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 1, 0, 0, 2, 317, 1903, 1, 2435, 0, 1, 4094, 4095, 114, 4094, 0, 1, 1, 4095, 3358, 2, 0, 63, 0, 1, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 1, 0, 1, 1, 242, 2617, 1, 1177, 0, 1, 4095, 4095, 618, 4094, 0, 1, 1, 4095, 3456, 1, 0, 79, 0, 1, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 38, 0, 0, 0, 0, 220, 2847, 0, 2037, 0, 0, 4096, 4095, 12, 4096, 0, 0, 0, 4096, 3022, 0, 0, 17, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 141, 0, 0, 0, 0, 299, 2928, 0, 1713, 0, 0, 4096, 4096, 81, 4096, 0, 0, 0, 4096, 3024, 0, 0, 6, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 18, 0, 0, 0, 0, 500, 3047, 0, 1269, 0, 0, 4096, 4096, 67, 4096, 0, 0, 0, 4096, 3242, 0, 0, 49, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 3, 4063, 0, 0, 0, 1, 382, 2331, 2, 1078, 193, 0, 0, 2010, 0, 2, 2131, 0, 0, 0, 0, 0, 164, 3898, 0, 0, 0, 0, 0, 2, 2, 975, 3237, 0, 0, 0, 0, 0, 0, 67, 1, 3119, 884, 31, 0, 0, 4091, 1, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4068, 3987, 4143, 4062, 4216, 4102, 4093] avg=4096.0000 max=4216 max_rank=5 min=3987 min_rank=2 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16473 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 4065, 0, 0, 0, 1, 242, 2617, 1, 1078, 193, 0, 0, 1177, 0, 1, 2132, 0, 0, 0, 0, 0, 618, 3900, 0, 0, 0, 0, 0, 1, 1, 975, 3456, 0, 0, 0, 0, 1, 0, 79, 0, 3120, 885, 30, 0, 1, 4095, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4067, 4132, 3310, 4518, 4433, 4115, 4097] avg=4096.0000 max=4518 max_rank=4 min=3310 min_rank=3 imbalance_ratio=1.103027 max_load_reduction=28250 max_load_reduction_pct=0.862122 moved_to_echo=17163 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 4064, 0, 0, 0, 2, 317, 1903, 1, 1078, 193, 0, 0, 2435, 0, 1, 2131, 0, 0, 0, 0, 0, 114, 3900, 0, 0, 0, 0, 0, 1, 1, 975, 3358, 0, 0, 0, 0, 2, 0, 63, 0, 3120, 885, 31, 0, 1, 4094, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4065, 3494, 4567, 4014, 4335, 4101, 4096] avg=4096.0000 max=4567 max_rank=3 min=3494 min_rank=2 imbalance_ratio=1.114990 max_load_reduction=28201 max_load_reduction_pct=0.860626 moved_to_echo=16546 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 38, 0, 0, 0, 4064, 0, 0, 0, 0, 220, 2847, 0, 1079, 193, 0, 0, 2037, 0, 0, 2132, 0, 0, 0, 0, 0, 12, 3902, 0, 0, 0, 0, 0, 0, 0, 975, 3022, 0, 0, 0, 0, 0, 0, 17, 0, 3121, 885, 31, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4102, 4339, 4169, 3914, 3997, 4054, 4097] avg=4096.0000 max=4339 max_rank=2 min=3914 min_rank=4 imbalance_ratio=1.059326 max_load_reduction=28429 max_load_reduction_pct=0.867584 moved_to_echo=16062 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 25, 0, 0, 0, 4066, 0, 0, 0, 0, 147, 2856, 0, 1082, 199, 0, 0, 2237, 0, 0, 2125, 0, 0, 0, 0, 0, 100, 3889, 0, 0, 0, 0, 0, 0, 0, 973, 2837, 0, 0, 0, 0, 0, 0, 1, 0, 3123, 889, 30, 0, 0, 4085, 0, 0, 8, 1, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4091, 4284, 4362, 3989, 3810, 4043, 4094] avg=4096.0000 max=4362 max_rank=3 min=3810 min_rank=5 imbalance_ratio=1.064941 max_load_reduction=28406 max_load_reduction_pct=0.866882 moved_to_echo=15936 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 141, 0, 0, 0, 4065, 0, 0, 0, 0, 299, 2928, 0, 1079, 193, 0, 0, 1713, 0, 0, 2132, 0, 0, 0, 0, 0, 81, 3902, 0, 0, 0, 0, 0, 0, 0, 975, 3024, 0, 0, 0, 0, 0, 0, 6, 0, 3121, 885, 31, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4206, 4499, 3845, 3983, 3999, 4043, 4097] avg=4096.0000 max=4499 max_rank=2 min=3845 min_rank=3 imbalance_ratio=1.098389 max_load_reduction=28269 max_load_reduction_pct=0.862701 moved_to_echo=16122 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 19, 0, 2, 0, 4065, 0, 0, 0, 0, 147, 1795, 0, 1079, 193, 0, 0, 2839, 0, 0, 2132, 0, 0, 0, 0, 0, 417, 3902, 0, 0, 0, 0, 0, 0, 0, 975, 2786, 0, 0, 0, 0, 0, 0, 187, 0, 3121, 885, 31, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4086, 3214, 4971, 4319, 3761, 4224, 4097] avg=4096.0000 max=4971 max_rank=3 min=3214 min_rank=2 imbalance_ratio=1.213623 max_load_reduction=27797 max_load_reduction_pct=0.848297 moved_to_echo=16401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 18, 0, 0, 0, 4065, 0, 0, 0, 0, 500, 3047, 0, 1079, 193, 0, 0, 1269, 0, 0, 2132, 0, 0, 0, 0, 0, 67, 3902, 0, 0, 0, 0, 0, 0, 0, 975, 3242, 0, 0, 0, 0, 0, 0, 49, 0, 3121, 885, 31, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4083, 4819, 3401, 3969, 4217, 4086, 4097] avg=4096.0000 max=4819 max_rank=2 min=3401 min_rank=3 imbalance_ratio=1.176514 max_load_reduction=27949 max_load_reduction_pct=0.852936 moved_to_echo=16369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 2304, 20352, 128, 8704, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 7808, 25088, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15744, 0, 128, 17152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 512, 128, 25088, 7168, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 1664, 31232, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[2, 2, 4094, 3640, 0, 3316, 0, 2, 4091, 4094, 2, 555, 1, 0, 8, 0, 4093, 0, 2, 4094, 4071, 2, 0, 2, 0, 694, 0, 1, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 4095, 3175, 0, 3460, 0, 1, 4093, 4095, 1, 227, 1, 0, 3, 0, 4095, 0, 1, 4095, 4090, 0, 0, 1, 4, 1328, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4048, 3470, 0, 3380, 0, 0, 4096, 4095, 0, 650, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 6, 723, 0, 0, 0, 0, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3566, 0, 4051, 0, 0, 4096, 4094, 0, 426, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 12, 139, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 4095, 3425, 0, 3735, 0, 1, 4094, 4071, 1, 191, 1, 0, 6, 0, 4095, 0, 1, 4095, 4090, 0, 0, 1, 3, 860, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4095, 3930, 0, 3273, 0, 0, 4096, 4096, 0, 266, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 724, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4091, 3052, 0, 3661, 0, 0, 4096, 4095, 0, 671, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 1, 813, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4081, 3595, 0, 3093, 0, 0, 4096, 4096, 0, 381, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1137, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 615, 3175, 0, 0, 0, 0, 0, 3460, 0, 1, 413, 186, 0, 0, 0, 3679, 1, 227, 0, 0, 0, 0, 1, 0, 3, 0, 4090, 2, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 4090, 0, 0, 1, 3, 0, 0, 0, 4, 1328, 0, 0, 3294, 3, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[3792, 4060, 3907, 4096, 4096, 4094, 4629, 4094] avg=4096.0000 max=4629 max_rank=6 min=3792 min_rank=0 imbalance_ratio=1.130127 max_load_reduction=28139 max_load_reduction_pct=0.858734 moved_to_echo=16913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 615, 3425, 0, 0, 0, 0, 0, 3735, 0, 1, 411, 186, 0, 0, 0, 3657, 1, 191, 0, 0, 0, 0, 1, 0, 6, 0, 4092, 2, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 4090, 0, 0, 1, 3, 0, 0, 0, 3, 860, 0, 0, 3294, 2, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4333, 3849, 4101, 4096, 4094, 4159, 4094] avg=4096.0000 max=4333 max_rank=1 min=3849 min_rank=2 imbalance_ratio=1.057861 max_load_reduction=28435 max_load_reduction_pct=0.867767 moved_to_echo=16443 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 615, 3640, 0, 0, 0, 0, 0, 3316, 0, 2, 413, 186, 0, 0, 0, 3678, 2, 555, 0, 0, 0, 0, 1, 0, 8, 0, 4088, 1, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 4071, 2, 0, 2, 3, 0, 0, 0, 0, 694, 0, 1, 3293, 3, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4259, 3917, 4235, 4098, 4095, 4078, 3991, 4095] avg=4096.0000 max=4259 max_rank=0 min=3917 min_rank=1 imbalance_ratio=1.039795 max_load_reduction=28509 max_load_reduction_pct=0.870026 moved_to_echo=16259 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 614, 3595, 0, 0, 0, 0, 0, 3093, 0, 0, 413, 185, 0, 0, 0, 3680, 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 1137, 0, 0, 3282, 3, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4209, 3691, 4061, 4094, 4096, 4099, 4422, 4096] avg=4096.0000 max=4422 max_rank=6 min=3691 min_rank=1 imbalance_ratio=1.079590 max_load_reduction=28346 max_load_reduction_pct=0.865051 moved_to_echo=16713 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 602, 3470, 0, 0, 0, 0, 0, 3380, 0, 0, 419, 186, 0, 0, 0, 3666, 0, 650, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 6, 723, 0, 0, 3260, 2, 0, 0, 0, 0, 12, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 3985, 4316, 4096, 4096, 4106, 3991, 4106] avg=4096.0000 max=4316 max_rank=2 min=3985 min_rank=1 imbalance_ratio=1.053711 max_load_reduction=28452 max_load_reduction_pct=0.868286 moved_to_echo=16299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 615, 3930, 0, 0, 0, 0, 0, 3273, 0, 0, 413, 186, 0, 0, 0, 3680, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 724, 0, 0, 3294, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4545, 3872, 3946, 4094, 4096, 4099, 4021, 4095] avg=4096.0000 max=4545 max_rank=0 min=3872 min_rank=1 imbalance_ratio=1.109619 max_load_reduction=28223 max_load_reduction_pct=0.861298 moved_to_echo=16311 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 615, 3052, 0, 0, 0, 0, 0, 3661, 0, 0, 413, 186, 0, 0, 0, 3679, 0, 671, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 1, 813, 0, 0, 3290, 3, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[3667, 4260, 4350, 4094, 4097, 4099, 4107, 4094] avg=4096.0000 max=4350 max_rank=2 min=3667 min_rank=0 imbalance_ratio=1.062012 max_load_reduction=28418 max_load_reduction_pct=0.867249 moved_to_echo=16397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 616, 3566, 0, 0, 0, 0, 0, 4051, 0, 0, 413, 186, 0, 0, 0, 3678, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 12, 139, 0, 0, 3294, 3, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 4650, 4104, 4095, 4096, 4099, 3448, 4094] avg=4096.0000 max=4650 max_rank=1 min=3448 min_rank=6 imbalance_ratio=1.135254 max_load_reduction=28118 max_load_reduction_pct=0.858093 moved_to_echo=15737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 4992, 27904, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 6528, 0, 128, 26368, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 28032, 0, 128, 3328, 1536, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 29440, 128, 3456, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 1, 1, 4095, 4088, 4051, 1, 1, 1, 0, 4095, 0, 1, 7, 191, 4095, 1, 1, 1, 0, 14, 0, 0, 0, 4058, 0, 3969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 1, 1, 4095, 4086, 4094, 1, 1, 1, 0, 4095, 0, 0, 9, 186, 4095, 1, 1, 1, 0, 14, 0, 0, 0, 3918, 0, 4072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 4094, 0, 2, 2, 4094, 4091, 4076, 2, 2, 2, 0, 4088, 0, 2, 9, 331, 4094, 2, 3, 1, 0, 34, 0, 0, 0, 3881, 0, 3955] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 4090, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 16, 0, 4096, 0, 0, 0, 4082, 4089, 4096, 0, 0, 0, 0, 4080, 0, 0, 12, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 9, 4051, 1, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 1, 7, 191, 3913, 0, 0, 0, 4095, 1, 1, 1, 0, 0, 0, 0, 0, 14, 0, 0, 4079, 1, 0, 0, 0, 4058, 0, 55, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4061, 4097, 4112, 4098, 4094, 4113] avg=4096.0000 max=4113 max_rank=7 min=4061 min_rank=2 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16417 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 9, 4094, 1, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 9, 186, 4014, 0, 0, 0, 4095, 1, 1, 1, 0, 0, 0, 0, 0, 14, 0, 0, 4077, 1, 0, 0, 0, 3918, 0, 57, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4104, 4097, 4209, 4098, 4092, 3975] avg=4096.0000 max=4209 max_rank=4 min=3975 min_rank=7 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 4093, 0, 0, 0, 4094, 0, 2, 2, 0, 0, 0, 0, 0, 9, 4076, 2, 0, 0, 0, 0, 2, 2, 0, 4088, 0, 0, 0, 0, 0, 2, 9, 331, 3899, 0, 0, 0, 4094, 2, 3, 1, 0, 0, 0, 0, 0, 34, 0, 0, 4082, 1, 1, 0, 0, 3881, 0, 55, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4098, 4087, 4092, 4241, 4100, 4118, 3936] avg=4096.0000 max=4241 max_rank=4 min=3936 min_rank=7 imbalance_ratio=1.035400 max_load_reduction=28527 max_load_reduction_pct=0.870575 moved_to_echo=16395 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 4038, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 1, 0, 0, 4096, 0, 57, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4105, 4096, 4039, 4095, 4088, 4153] avg=4096.0000 max=4153 max_rank=7 min=4039 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 1, 0, 0, 4096, 0, 57, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4105, 4097, 4038, 4095, 4089, 4153] avg=4096.0000 max=4153 max_rank=7 min=4038 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 16, 0, 4082, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4, 7, 4, 0, 0, 0, 12, 0, 4041, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4087, 6, 0, 0, 0, 4096, 0, 42, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4092, 4098, 4095, 4053, 4092, 4102, 4138] avg=4096.0000 max=4138 max_rank=7 min=4053 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4087, 1, 0, 0, 0, 4090, 0, 57, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4103, 4097, 4038, 4095, 4096, 4147] avg=4096.0000 max=4147 max_rank=7 min=4038 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 1, 0, 0, 4096, 0, 57, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4105, 4097, 4038, 4095, 4089, 4153] avg=4096.0000 max=4153 max_rank=7 min=4038 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32384, 0, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 768, 32128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4087, 0, 0, 0, 0, 4066, 234, 0, 0, 61, 0, 343, 4095, 0, 0, 0, 3448, 58, 0, 4088, 4096, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 0, 2, 4090, 0, 3, 2, 25, 13, 3414, 2, 2, 2, 2, 4094, 460, 4, 2, 3, 3, 901, 4088, 0, 2, 0, 3303, 312, 0, 3849, 4094, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 4093, 0, 0, 1, 7, 12, 3501, 1, 1, 1, 1, 4095, 949, 4, 1, 3, 1, 362, 4047, 0, 0, 0, 3463, 149, 0, 3883, 4095, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 1, 1, 4093, 0, 0, 1, 4, 5, 3512, 1, 1, 1, 1, 4095, 466, 2, 1, 36, 4, 151, 4089, 0, 1, 0, 3918, 201, 0, 3993, 4095, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4090, 577, 0, 0, 2, 0, 91, 4090, 0, 0, 0, 3428, 13, 0, 4095, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 201, 0, 0, 1, 0, 398, 4090, 0, 0, 0, 3380, 123, 0, 4095, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 474, 0, 0, 1, 0, 612, 4092, 0, 0, 0, 2978, 57, 0, 4073, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4078, 149, 0, 0, 6, 0, 307, 4092, 0, 0, 0, 3517, 145, 0, 4090, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 4095, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 7, 12, 3501, 1, 222, 0, 0, 0, 1, 1, 1, 4095, 5, 0, 0, 0, 949, 4, 1, 3, 3640, 0, 0, 0, 1, 362, 3667, 0, 0, 0, 0, 0, 0, 0, 3463, 149, 379, 153, 0, 0, 0, 3883, 0, 76, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4094, 3743, 4103, 4597, 4030, 4144, 3959] avg=4096.0000 max=4597 max_rank=4 min=3743 min_rank=2 imbalance_ratio=1.122314 max_load_reduction=28171 max_load_reduction_pct=0.859711 moved_to_echo=16730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 1, 4094, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 4, 5, 3512, 1, 225, 0, 0, 0, 1, 1, 1, 4095, 5, 0, 0, 0, 466, 2, 1, 36, 3639, 1, 0, 0, 4, 151, 3705, 0, 0, 0, 0, 0, 1, 0, 3918, 201, 379, 154, 0, 0, 0, 3993, 0, 76, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4094, 3747, 4103, 4145, 3860, 4653, 4069] avg=4096.0000 max=4653 max_rank=6 min=3747 min_rank=2 imbalance_ratio=1.135986 max_load_reduction=28115 max_load_reduction_pct=0.858002 moved_to_echo=16727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 2, 4093, 0, 0, 0, 4090, 0, 3, 2, 0, 0, 0, 0, 25, 13, 3414, 2, 225, 0, 0, 0, 2, 2, 2, 4094, 5, 0, 0, 0, 460, 4, 2, 3, 3639, 1, 0, 0, 3, 901, 3704, 0, 0, 0, 0, 0, 2, 0, 3303, 312, 379, 154, 0, 0, 0, 3849, 0, 76, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4095, 3679, 4105, 4109, 4608, 4150, 3925] avg=4096.0000 max=4608 max_rank=5 min=3679 min_rank=2 imbalance_ratio=1.125000 max_load_reduction=28160 max_load_reduction_pct=0.859375 moved_to_echo=16792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 225, 0, 0, 0, 0, 0, 0, 4078, 5, 0, 0, 0, 149, 0, 0, 6, 3641, 1, 0, 0, 0, 307, 3708, 0, 0, 0, 0, 0, 0, 0, 3517, 145, 379, 154, 0, 0, 0, 4090, 0, 76, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4321, 4083, 3797, 4015, 4195, 4166] avg=4096.0000 max=4321 max_rank=2 min=3797 min_rank=4 imbalance_ratio=1.054932 max_load_reduction=28447 max_load_reduction_pct=0.868134 moved_to_echo=16173 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 1, 0, 4087, 0, 228, 0, 0, 0, 0, 0, 0, 4066, 11, 0, 0, 0, 234, 0, 0, 61, 3643, 0, 0, 0, 0, 343, 3691, 0, 0, 0, 0, 0, 0, 0, 3448, 58, 382, 160, 0, 0, 0, 4088, 0, 70, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4316, 4077, 3938, 4034, 4048, 4158] avg=4096.0000 max=4316 max_rank=2 min=3938 min_rank=4 imbalance_ratio=1.053711 max_load_reduction=28452 max_load_reduction_pct=0.868286 moved_to_echo=16178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 225, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 474, 0, 0, 1, 3641, 1, 0, 0, 0, 612, 3708, 0, 0, 0, 0, 0, 0, 0, 2978, 57, 379, 154, 0, 0, 0, 4073, 0, 76, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4322, 4101, 4117, 4320, 3568, 4149] avg=4096.0000 max=4322 max_rank=2 min=3568 min_rank=6 imbalance_ratio=1.055176 max_load_reduction=28446 max_load_reduction_pct=0.868103 moved_to_echo=16154 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 225, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 201, 0, 0, 1, 3641, 1, 0, 0, 0, 398, 3706, 0, 0, 0, 0, 0, 0, 0, 3380, 123, 379, 154, 0, 0, 0, 4095, 0, 76, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4321, 4101, 3844, 4104, 4036, 4171] avg=4096.0000 max=4321 max_rank=2 min=3844 min_rank=4 imbalance_ratio=1.054932 max_load_reduction=28447 max_load_reduction_pct=0.868134 moved_to_echo=16155 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 225, 0, 0, 0, 0, 0, 0, 4090, 5, 0, 0, 0, 577, 0, 0, 2, 3641, 1, 0, 0, 0, 91, 3706, 0, 0, 0, 0, 0, 0, 0, 3428, 13, 379, 154, 0, 0, 0, 4095, 0, 76, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4319, 4095, 4221, 3797, 3974, 4171] avg=4096.0000 max=4319 max_rank=2 min=3797 min_rank=5 imbalance_ratio=1.054443 max_load_reduction=28449 max_load_reduction_pct=0.868195 moved_to_echo=16163 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 27520, 1152, 3072, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 3200, 29696, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3584, 128, 128, 128, 29184, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32256, 0, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 30976, 128, 1920, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[2, 2, 4079, 903, 2, 0, 0, 4088, 0, 0, 6, 2, 0, 3196, 2, 0, 4091, 3, 1, 4094, 0, 0, 4093, 3, 4094, 11, 0, 0, 0, 0, 2, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 241, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3855, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 13, 0, 0, 0, 0, 4083] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 4080, 832, 1, 0, 0, 4091, 0, 0, 4, 1, 0, 3263, 1, 1, 4094, 1, 1, 4095, 0, 0, 4095, 1, 4095, 15, 0, 0, 0, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 4088, 390, 1, 0, 0, 4073, 0, 0, 5, 1, 0, 3716, 1, 0, 4095, 1, 1, 4095, 0, 0, 4095, 1, 4095, 11, 2, 0, 0, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 98, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3998, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 90, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4017, 0, 0, 4096, 0, 0, 4096, 0, 0, 4090, 0, 4096, 0, 3, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 304, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3794, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 109, 0, 0, 0, 4081, 0, 0, 0, 0, 1, 4000, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 4096, 0, 3, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3723, 390, 0, 0, 0, 0, 1, 0, 0, 4073, 3, 2, 0, 0, 0, 0, 5, 1, 4093, 0, 0, 0, 0, 3716, 1, 0, 363, 2, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4091, 11, 2, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4079, 4099, 4082, 4097, 4096, 4104, 4097] avg=4096.0000 max=4114 max_rank=0 min=4079 min_rank=1 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 3715, 903, 0, 0, 0, 0, 2, 0, 0, 4088, 3, 2, 0, 0, 0, 0, 6, 2, 4092, 0, 0, 0, 0, 3196, 2, 0, 362, 2, 0, 0, 4091, 3, 1, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 4090, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4622, 4095, 4100, 3562, 4095, 4096, 4101, 4097] avg=4096.0000 max=4622 max_rank=0 min=3562 min_rank=3 imbalance_ratio=1.128418 max_load_reduction=28146 max_load_reduction_pct=0.858948 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3716, 832, 0, 0, 0, 0, 1, 0, 0, 4091, 3, 2, 0, 0, 0, 0, 4, 1, 4093, 0, 0, 0, 0, 3263, 1, 1, 362, 2, 0, 0, 4094, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4091, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4549, 4097, 4098, 3629, 4096, 4096, 4106, 4097] avg=4096.0000 max=4549 max_rank=0 min=3629 min_rank=3 imbalance_ratio=1.110596 max_load_reduction=28219 max_load_reduction_pct=0.861176 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3721, 241, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 6, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3855, 0, 0, 369, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 4078, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[3962, 4109, 4095, 4225, 4096, 4100, 4091, 4090] avg=4096.0000 max=4225 max_rank=3 min=3962 min_rank=0 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3730, 109, 0, 0, 0, 0, 0, 0, 0, 4081, 4, 3, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 4000, 0, 0, 363, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 4090, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3839, 4088, 4094, 4366, 4096, 4096, 4093, 4096] avg=4096.0000 max=4366 max_rank=3 min=3839 min_rank=0 imbalance_ratio=1.065918 max_load_reduction=28402 max_load_reduction_pct=0.866760 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3730, 304, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 3, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3794, 0, 0, 363, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4102, 4094, 4159, 4096, 4096, 4090, 4097] avg=4096.0000 max=4159 max_rank=3 min=4034 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3730, 98, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 3, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3998, 0, 0, 363, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3828, 4103, 4094, 4363, 4096, 4097, 4090, 4097] avg=4096.0000 max=4363 max_rank=3 min=3828 min_rank=0 imbalance_ratio=1.065186 max_load_reduction=28405 max_load_reduction_pct=0.866852 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3730, 90, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 3, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4017, 0, 0, 363, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 0, 0, 0, 4090, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3820, 4095, 4094, 4382, 4096, 4091, 4093, 4097] avg=4096.0000 max=4382 max_rank=3 min=3820 min_rank=0 imbalance_ratio=1.069824 max_load_reduction=28386 max_load_reduction_pct=0.866272 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 29824, 3072, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 29952, 128, 128, 2944, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3805, 4094, 0, 0, 0, 4090, 4095, 0, 0, 1, 4086, 0, 4095, 1, 4025, 432, 5, 1, 1, 3995, 1, 0, 0, 1, 0, 1, 1, 34, 0, 1, 1, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4043, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 161, 0, 0, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4026, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 81, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3981, 4093, 0, 0, 0, 4090, 4095, 0, 0, 1, 4074, 0, 4095, 1, 3974, 246, 4, 1, 1, 4074, 1, 0, 0, 1, 0, 1, 1, 29, 0, 1, 1, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3798, 4090, 2, 0, 0, 4079, 4094, 0, 1, 2, 4091, 0, 4093, 4, 3977, 536, 15, 2, 2, 3940, 2, 0, 0, 2, 0, 2, 2, 29, 0, 0, 2, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4078, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 232, 0, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4021, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 173, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3955, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 274, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3805, 134, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 1, 4086, 0, 3, 0, 0, 0, 0, 1, 3800, 432, 0, 0, 0, 0, 5, 1, 1, 3995, 104, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 1, 34, 3959, 118, 0, 0, 0, 1, 1, 2, 4091, 1, 0, 0] rerouted_tokens_per_ep_rank=[3939, 4094, 4090, 4233, 4106, 4097, 4113, 4096] avg=4096.0000 max=4233 max_rank=3 min=3939 min_rank=0 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3981, 134, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 1, 4074, 0, 3, 0, 0, 0, 0, 1, 3752, 246, 0, 0, 0, 0, 4, 1, 1, 4074, 103, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 1, 29, 3958, 116, 0, 0, 0, 1, 1, 3, 4091, 1, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4094, 4078, 3999, 4183, 4097, 4105, 4097] avg=4096.0000 max=4183 max_rank=4 min=3999 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4078, 133, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 3866, 232, 0, 0, 0, 0, 0, 0, 0, 3880, 106, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 3961, 120, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0] rerouted_tokens_per_ep_rank=[4211, 4100, 4100, 4098, 3986, 4096, 4083, 4094] avg=4096.0000 max=4211 max_rank=0 min=3986 min_rank=4 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16259 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3798, 134, 2, 0, 0, 0, 0, 0, 0, 4079, 4, 0, 0, 0, 0, 0, 1, 2, 4091, 0, 3, 0, 0, 0, 0, 4, 3754, 536, 0, 0, 0, 0, 15, 2, 2, 3940, 103, 0, 0, 0, 2, 0, 0, 2, 4092, 0, 0, 0, 0, 2, 2, 29, 3955, 117, 0, 0, 0, 0, 2, 3, 4090, 1, 1, 0] rerouted_tokens_per_ep_rank=[3934, 4083, 4097, 4294, 4062, 4096, 4105, 4097] avg=4096.0000 max=4294 max_rank=3 min=3934 min_rank=0 imbalance_ratio=1.048340 max_load_reduction=28474 max_load_reduction_pct=0.868958 moved_to_echo=16360 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4043, 125, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 3857, 161, 0, 0, 0, 0, 0, 0, 0, 3960, 108, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 28, 3966, 124, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0] rerouted_tokens_per_ep_rank=[4168, 4097, 4103, 4018, 4068, 4096, 4118, 4100] avg=4096.0000 max=4168 max_rank=0 min=4018 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4021, 133, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 3866, 173, 0, 0, 0, 0, 0, 0, 0, 3998, 106, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3961, 120, 0, 0, 0, 0, 0, 0, 4092, 2, 1, 0] rerouted_tokens_per_ep_rank=[4154, 4100, 4100, 4039, 4104, 4095, 4081, 4095] avg=4096.0000 max=4154 max_rank=0 min=4039 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4026, 133, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 3866, 81, 0, 0, 0, 0, 0, 0, 0, 4084, 106, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 3961, 120, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4100, 4100, 3947, 4190, 4096, 4082, 4094] avg=4096.0000 max=4190 max_rank=4 min=3947 min_rank=3 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16462 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3955, 133, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 3866, 274, 0, 0, 0, 0, 0, 0, 0, 3963, 106, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3961, 120, 0, 0, 0, 0, 0, 0, 4092, 2, 1, 0] rerouted_tokens_per_ep_rank=[4088, 4100, 4100, 4140, 4069, 4095, 4081, 4095] avg=4096.0000 max=4140 max_rank=3 min=4069 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 30720, 2176, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 31744, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31744, 1152, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32000, 896, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 968, 4096, 0, 3082, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 46, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[20, 1, 3, 1, 0, 0, 0, 1357, 4085, 5, 2785, 0, 2, 4068, 0, 0, 4095, 0, 0, 1, 4088, 0, 0, 4006, 4093, 22, 1, 51, 1, 4082, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 859, 4075, 0, 3218, 0, 0, 4082, 0, 0, 4094, 0, 0, 0, 4096, 0, 11, 4096, 4096, 43, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 947, 4088, 0, 3083, 0, 0, 4078, 0, 1, 4095, 0, 0, 0, 4096, 0, 1, 4096, 4096, 91, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[40, 1, 2, 1, 1, 1, 0, 678, 4079, 3, 3429, 0, 1, 4047, 0, 0, 4095, 0, 0, 1, 4083, 0, 0, 4005, 4094, 81, 0, 41, 1, 4083, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[73, 2, 8, 2, 0, 0, 0, 436, 4066, 12, 3788, 0, 3, 4009, 0, 0, 4094, 0, 0, 2, 4082, 0, 0, 3926, 4090, 22, 0, 85, 2, 4064, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 871, 4096, 0, 3035, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 190, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1049, 4091, 0, 3014, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 40, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[20, 1, 3, 1, 4045, 27, 0, 0, 0, 0, 0, 1357, 3143, 57, 0, 0, 915, 5, 2785, 0, 0, 0, 0, 0, 2, 4068, 0, 0, 23, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 43, 0, 0, 4006, 0, 0, 0, 0, 4007, 22, 1, 51, 0, 0, 0, 0, 1, 4082, 1, 0, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4557, 3705, 4093, 4096, 4049, 4081, 4090] avg=4096.0000 max=4557 max_rank=1 min=3705 min_rank=2 imbalance_ratio=1.112549 max_load_reduction=28211 max_load_reduction_pct=0.860931 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4053, 27, 0, 0, 0, 0, 0, 947, 3145, 57, 0, 0, 916, 0, 3083, 0, 0, 0, 0, 0, 0, 4078, 0, 1, 23, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 43, 0, 1, 4096, 0, 0, 0, 0, 4010, 91, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4149, 3999, 4102, 4095, 4140, 4101, 4102] avg=4096.0000 max=4149 max_rank=1 min=3999 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[40, 1, 2, 1, 4040, 27, 0, 0, 1, 1, 0, 678, 3138, 57, 0, 0, 914, 3, 3429, 0, 0, 0, 0, 0, 1, 4047, 0, 0, 23, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 43, 0, 0, 4005, 0, 0, 0, 0, 4008, 81, 0, 41, 0, 0, 0, 0, 1, 4083, 1, 0, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 3875, 4346, 4071, 4096, 4048, 4130, 4091] avg=4096.0000 max=4346 max_rank=2 min=3875 min_rank=1 imbalance_ratio=1.061035 max_load_reduction=28422 max_load_reduction_pct=0.867371 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4053, 27, 0, 0, 0, 0, 0, 968, 3151, 57, 0, 0, 918, 0, 3082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 4096, 0, 0, 0, 0, 4010, 46, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4176, 4000, 4119, 4096, 4139, 4056, 4102] avg=4096.0000 max=4176 max_rank=1 min=4000 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[73, 2, 8, 2, 4039, 27, 0, 0, 0, 0, 0, 436, 3128, 57, 0, 0, 911, 12, 3788, 0, 0, 0, 0, 0, 3, 4009, 0, 0, 23, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 43, 0, 0, 3926, 0, 0, 0, 0, 4004, 22, 0, 85, 0, 0, 0, 0, 2, 4064, 2, 0, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 3621, 4711, 4035, 4096, 3969, 4111, 4074] avg=4096.0000 max=4711 max_rank=2 min=3621 min_rank=1 imbalance_ratio=1.150146 max_load_reduction=28057 max_load_reduction_pct=0.856232 moved_to_echo=16250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4058, 31, 0, 0, 0, 0, 2, 859, 3139, 58, 0, 0, 905, 0, 3218, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 30, 0, 0, 0, 4094, 0, 0, 0, 3, 0, 0, 0, 38, 0, 11, 4096, 0, 0, 0, 0, 3996, 43, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4058, 4123, 4112, 4097, 4145, 4039, 4105] avg=4096.0000 max=4145 max_rank=5 min=4039 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4053, 27, 0, 0, 0, 0, 0, 871, 3151, 57, 0, 0, 918, 0, 3035, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 4096, 0, 0, 0, 0, 4010, 190, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4079, 3953, 4119, 4096, 4139, 4200, 4102] avg=4096.0000 max=4200 max_rank=6 min=3953 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4053, 27, 0, 0, 0, 0, 0, 1049, 3147, 57, 0, 0, 917, 0, 3014, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 4096, 0, 0, 0, 0, 4010, 40, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4253, 3931, 4117, 4096, 4139, 4050, 4102] avg=4096.0000 max=4253 max_rank=1 min=3931 min_rank=2 imbalance_ratio=1.038330 max_load_reduction=28515 max_load_reduction_pct=0.870209 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 7168, 25216, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7424, 128, 25472, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 128, 128, 32512, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 0, 128, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32128, 640, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 0, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4092, 0, 2938, 1, 1, 0, 11, 1, 4095, 4076, 0, 0, 0, 0, 0, 1, 4094, 4094, 4088, 1, 1, 24, 0, 2, 0, 0, 1, 4095, 0, 1133, 19, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 2747, 1, 1, 0, 9, 1, 4095, 4080, 0, 0, 0, 0, 0, 1, 4094, 4094, 4087, 1, 1, 2, 0, 2, 0, 0, 1, 4095, 0, 1346, 15, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 3735, 0, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 1, 359, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 4022, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 4096, 0, 64, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 3739, 0, 0, 0, 0, 0, 4078, 4075, 3, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 62, 0, 0, 0, 0, 0, 4096, 0, 319, 0, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 2297, 3, 2, 0, 25, 2, 4094, 4074, 0, 0, 0, 0, 0, 2, 4092, 4088, 4080, 2, 2, 6, 2, 4, 0, 0, 2, 4093, 0, 1789, 15, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 3067, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 1022, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 3897, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 196, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[790, 0, 2938, 1, 0, 0, 0, 0, 1, 0, 11, 1, 4090, 0, 0, 0, 11, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 1, 24, 0, 2, 4080, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 1133, 19, 0, 3302, 4, 0, 0] rerouted_tokens_per_ep_rank=[3729, 4103, 4087, 4099, 4089, 4107, 4096, 4458] avg=4096.0000 max=4458 max_rank=7 min=3729 min_rank=0 imbalance_ratio=1.088379 max_load_reduction=28310 max_load_reduction_pct=0.863953 moved_to_echo=16750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[791, 0, 2747, 1, 0, 0, 0, 0, 1, 0, 9, 1, 4088, 0, 0, 0, 11, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 6, 4087, 1, 0, 0, 0, 0, 1, 2, 0, 2, 4080, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 1346, 15, 0, 3304, 4, 0, 0] rerouted_tokens_per_ep_rank=[3539, 4099, 4091, 4095, 4094, 4085, 4096, 4669] avg=4096.0000 max=4669 max_rank=7 min=3539 min_rank=0 imbalance_ratio=1.139893 max_load_reduction=28099 max_load_reduction_pct=0.857513 moved_to_echo=16944 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[791, 0, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 11, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4079, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 359, 0, 1, 3304, 4, 0, 0] rerouted_tokens_per_ep_rank=[4526, 4090, 4106, 4096, 4101, 4084, 4096, 3669] avg=4096.0000 max=4526 max_rank=0 min=3669 min_rank=7 imbalance_ratio=1.104980 max_load_reduction=28242 max_load_reduction_pct=0.861877 moved_to_echo=15950 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[791, 0, 2297, 3, 0, 0, 0, 0, 2, 0, 25, 2, 4082, 0, 0, 0, 11, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 0, 0, 0, 0, 6, 4080, 2, 0, 0, 0, 0, 2, 6, 2, 4, 4079, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 1789, 15, 0, 3303, 4, 0, 0] rerouted_tokens_per_ep_rank=[3091, 4111, 4085, 4094, 4088, 4093, 4095, 5111] avg=4096.0000 max=5111 max_rank=7 min=3091 min_rank=0 imbalance_ratio=1.247803 max_load_reduction=27657 max_load_reduction_pct=0.844025 moved_to_echo=17387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[785, 0, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3, 4075, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 62, 0, 0, 4065, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 319, 0, 13, 3309, 10, 0, 0] rerouted_tokens_per_ep_rank=[4524, 4095, 4081, 4096, 4096, 4128, 4097, 3651] avg=4096.0000 max=4524 max_rank=0 min=3651 min_rank=7 imbalance_ratio=1.104492 max_load_reduction=28244 max_load_reduction_pct=0.861938 moved_to_echo=15972 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[791, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 11, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4081, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 64, 0, 0, 3305, 4, 0, 0] rerouted_tokens_per_ep_rank=[4813, 4090, 4104, 4096, 4098, 4098, 4096, 3373] avg=4096.0000 max=4813 max_rank=0 min=3373 min_rank=7 imbalance_ratio=1.175049 max_load_reduction=27955 max_load_reduction_pct=0.853119 moved_to_echo=15665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[791, 0, 3067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4081, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1022, 0, 1, 3305, 4, 0, 0] rerouted_tokens_per_ep_rank=[3858, 4090, 4107, 4096, 4101, 4088, 4096, 4332] avg=4096.0000 max=4332 max_rank=7 min=3858 min_rank=0 imbalance_ratio=1.057617 max_load_reduction=28436 max_load_reduction_pct=0.867798 moved_to_echo=16617 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[791, 0, 3897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4081, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 196, 0, 0, 3305, 4, 0, 0] rerouted_tokens_per_ep_rank=[4688, 4090, 4107, 4096, 4101, 4085, 4096, 3505] avg=4096.0000 max=4688 max_rank=0 min=3505 min_rank=7 imbalance_ratio=1.144531 max_load_reduction=28080 max_load_reduction_pct=0.856934 moved_to_echo=15787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6400, 0, 26496, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 6272, 128, 128, 26496, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4087, 1, 3, 27, 1, 0, 4094, 1, 0, 1, 1, 4019, 2, 1, 0, 4093, 0, 54, 4095, 1, 0, 0, 4095, 433, 0, 0, 18, 1, 3646, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4, 4091, 2, 0, 165, 2, 0, 4092, 2, 1, 2, 2, 4090, 4, 0, 1, 4090, 0, 109, 4094, 2, 0, 2, 4093, 195, 0, 0, 0, 2, 3629, 1, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4067, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3830, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 285, 0, 0, 10, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3952, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 146, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3981, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 116, 0, 0, 0, 0, 4095, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4024, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 72, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3899, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 198, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 118, 1, 1, 4094, 1, 0, 1, 1, 4053, 2, 0, 0, 4092, 0, 20, 4095, 1, 0, 1, 4095, 314, 0, 0, 1, 1, 3686, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4087, 1, 3, 4, 0, 0, 0, 27, 1, 0, 4056, 0, 0, 0, 0, 1, 0, 1, 1, 4094, 0, 0, 0, 4019, 2, 1, 0, 58, 38, 16, 0, 4073, 0, 54, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 433, 0, 0, 18, 3870, 1, 0, 0, 1, 3646, 0, 166, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4084, 4097, 4134, 4127, 4096, 4322, 3813] avg=4096.0000 max=4322 max_rank=6 min=3813 min_rank=7 imbalance_ratio=1.055176 max_load_reduction=28446 max_load_reduction_pct=0.868103 moved_to_echo=16358 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4, 4091, 2, 0, 4, 0, 0, 0, 165, 2, 0, 4054, 0, 0, 0, 0, 2, 1, 2, 2, 4093, 0, 0, 0, 4090, 4, 0, 1, 58, 38, 16, 0, 4070, 0, 109, 0, 0, 0, 0, 0, 2, 0, 2, 4093, 0, 0, 0, 0, 195, 0, 0, 0, 3869, 1, 0, 0, 2, 3629, 1, 166, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4221, 4100, 4207, 4179, 4097, 4065, 3798] avg=4096.0000 max=4221 max_rank=1 min=3798 min_rank=7 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3952, 0, 0, 0, 58, 38, 16, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 146, 0, 0, 1, 3872, 1, 0, 0, 0, 4096, 0, 166, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4058, 4095, 4064, 4076, 4096, 4020, 4262] avg=4096.0000 max=4262 max_rank=7 min=4020 min_rank=6 imbalance_ratio=1.040527 max_load_reduction=28506 max_load_reduction_pct=0.869934 moved_to_echo=16454 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 4, 0, 0, 0, 118, 1, 1, 4056, 0, 0, 0, 0, 1, 0, 1, 1, 4094, 0, 0, 0, 4053, 2, 0, 0, 58, 38, 16, 0, 4072, 0, 20, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 314, 0, 0, 1, 3870, 1, 0, 0, 1, 3686, 0, 166, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4176, 4097, 4167, 4092, 4097, 4186, 3853] avg=4096.0000 max=4186 max_rank=6 min=3853 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16228 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4024, 0, 0, 0, 58, 38, 16, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 72, 0, 0, 0, 3872, 1, 0, 0, 0, 4096, 0, 166, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4058, 4095, 4136, 4076, 4096, 3945, 4262] avg=4096.0000 max=4262 max_rank=7 min=3945 min_rank=6 imbalance_ratio=1.040527 max_load_reduction=28506 max_load_reduction_pct=0.869934 moved_to_echo=16379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4067, 0, 0, 7, 2, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3830, 0, 0, 0, 62, 41, 23, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 285, 0, 0, 10, 3873, 1, 0, 0, 0, 4096, 0, 161, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4055, 4095, 3956, 4066, 4094, 4169, 4257] avg=4096.0000 max=4257 max_rank=7 min=3956 min_rank=3 imbalance_ratio=1.039307 max_load_reduction=28511 max_load_reduction_pct=0.870087 moved_to_echo=16586 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3899, 0, 0, 0, 58, 38, 16, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 198, 0, 0, 0, 3872, 1, 0, 0, 0, 4096, 0, 166, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4058, 4095, 4011, 4076, 4096, 4071, 4262] avg=4096.0000 max=4262 max_rank=7 min=4011 min_rank=3 imbalance_ratio=1.040527 max_load_reduction=28506 max_load_reduction_pct=0.869934 moved_to_echo=16505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3981, 0, 0, 0, 58, 38, 16, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 116, 0, 0, 0, 3872, 2, 0, 0, 0, 4095, 0, 166, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4058, 4094, 4093, 4076, 4096, 3990, 4261] avg=4096.0000 max=4261 max_rank=7 min=3990 min_rank=6 imbalance_ratio=1.040283 max_load_reduction=28507 max_load_reduction_pct=0.869965 moved_to_echo=16423 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 128, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32640, 0, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[31872, 128, 128, 128, 512, 384, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31488, 128, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1792, 0, 0, 128, 30976, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4095, 4095, 4095, 4077, 2, 2, 4, 4095, 1, 0, 0, 57, 1, 4051, 0, 0, 0, 1, 0, 0, 0, 1, 4093, 0, 0, 0, 4095, 1, 0, 0, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4082, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4095, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4095, 4095, 4095, 4085, 2, 2, 7, 4095, 1, 0, 0, 76, 1, 4019, 0, 0, 0, 1, 0, 0, 0, 1, 4094, 0, 0, 0, 4095, 1, 0, 0, 2, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4058, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 38, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4094, 4094, 4093, 4072, 3, 4, 13, 4094, 2, 0, 0, 83, 2, 4027, 0, 2, 0, 2, 2, 0, 0, 3, 4080, 0, 0, 0, 4094, 2, 1, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 8, 0, 0, 0, 0, 2, 2, 4, 4092, 0, 0, 0, 0, 1, 0, 0, 57, 4049, 1, 0, 0, 1, 4051, 0, 0, 20, 2, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 4093, 0, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4100, 4108, 4074, 4096, 4095, 4096, 4096] avg=4096.0000 max=4108 max_rank=2 min=4074 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 8, 0, 0, 0, 0, 2, 2, 7, 4092, 0, 0, 0, 0, 1, 0, 0, 76, 4057, 1, 0, 0, 1, 4019, 0, 0, 20, 2, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 2, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4103, 4135, 4042, 4096, 4096, 4096, 4097] avg=4096.0000 max=4135 max_rank=2 min=4042 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4082, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 14, 4068, 0, 0, 0, 0, 4096, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4092, 4082, 4118, 4096, 4098, 4096, 4096] avg=4096.0000 max=4118 max_rank=3 min=4082 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 8, 0, 0, 0, 0, 3, 4, 13, 4091, 0, 0, 0, 0, 2, 0, 0, 83, 4044, 0, 0, 0, 2, 4027, 0, 2, 20, 2, 0, 0, 0, 2, 2, 0, 4094, 0, 0, 0, 0, 3, 4080, 0, 1, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 1, 0, 1, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4111, 4129, 4053, 4098, 4084, 4096, 4095] avg=4096.0000 max=4129 max_rank=2 min=4053 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 1, 4068, 1, 0, 0, 0, 4096, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4092, 4070, 4118, 4096, 4098, 4096, 4095] avg=4096.0000 max=4118 max_rank=3 min=4070 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4058, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 38, 4069, 1, 0, 0, 0, 4096, 0, 0, 26, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4086, 4108, 4127, 4096, 4101, 4096, 4095] avg=4096.0000 max=4127 max_rank=3 min=4059 min_rank=0 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 4096, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4092, 4068, 4118, 4095, 4098, 4096, 4097] avg=4096.0000 max=4118 max_rank=3 min=4068 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 4096, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4092, 4068, 4118, 4095, 4098, 4096, 4097] avg=4096.0000 max=4118 max_rank=3 min=4068 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32640, 0, 128, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 384, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1719, 0, 2058, 181, 1142, 1, 1600, 328, 1284, 2012, 0, 42, 1218, 1699, 51, 1108, 80, 1588, 1972, 0, 0, 1330, 1812, 678, 0, 2349, 2547, 1540, 1905, 2279, 245, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1952, 2, 1892, 256, 1091, 5, 2086, 183, 1390, 2292, 0, 99, 1022, 1733, 183, 1309, 42, 1694, 933, 0, 6, 1739, 2067, 421, 0, 1358, 2485, 1239, 2338, 2780, 171, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2034, 1, 2243, 202, 964, 2, 1359, 251, 1131, 1607, 0, 56, 1470, 1158, 108, 813, 32, 1790, 1884, 1, 5, 1468, 2362, 1214, 0, 2123, 2248, 1757, 2097, 2094, 287, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1603, 0, 1945, 207, 942, 0, 2595, 84, 1067, 2800, 0, 47, 776, 2203, 81, 907, 34, 2139, 859, 0, 0, 2306, 1924, 458, 0, 1924, 2649, 1192, 1000, 2730, 295, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1982, 1, 2159, 291, 825, 0, 1797, 337, 1611, 2301, 0, 62, 1178, 1553, 82, 985, 64, 1631, 1260, 0, 0, 1539, 2016, 476, 0, 2083, 2785, 1224, 2260, 2040, 226, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2206, 0, 1928, 170, 970, 0, 1702, 149, 1185, 1929, 0, 33, 962, 1789, 93, 1307, 36, 1622, 1472, 0, 1, 1604, 1794, 489, 0, 2201, 2159, 1866, 2148, 2838, 114, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1693, 0, 1760, 118, 942, 0, 1582, 305, 1123, 2207, 0, 54, 1140, 2295, 58, 1407, 27, 1831, 1997, 0, 2, 1186, 1489, 362, 0, 2329, 1759, 1263, 2407, 3282, 150, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2099, 0, 1412, 284, 838, 0, 1999, 122, 1656, 2476, 1, 84, 1016, 2116, 202, 1936, 40, 1417, 1666, 0, 1, 1567, 1788, 427, 1, 1842, 2279, 1183, 1425, 2808, 83, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1719, 0, 2058, 181, 43, 0, 0, 0, 1142, 1, 1600, 328, 1153, 0, 0, 0, 1284, 2012, 0, 42, 397, 68, 0, 0, 1218, 1561, 51, 1108, 0, 0, 0, 0, 80, 1588, 1972, 0, 680, 175, 0, 0, 0, 1330, 1812, 678, 27, 0, 0, 0, 0, 2349, 714, 1540, 0, 0, 0, 0, 1905, 1707, 245, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4001, 4224, 3803, 3938, 4495, 3847, 4603, 3857] avg=4096.0000 max=4603 max_rank=6 min=3803 min_rank=2 imbalance_ratio=1.123779 max_load_reduction=28165 max_load_reduction_pct=0.859528 moved_to_echo=16802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1952, 2, 1892, 256, 44, 0, 0, 0, 1091, 5, 2086, 183, 1125, 0, 0, 0, 1390, 2292, 0, 99, 484, 69, 0, 0, 1022, 1592, 183, 1309, 0, 0, 0, 0, 42, 1694, 933, 0, 663, 214, 0, 0, 6, 1739, 2067, 421, 28, 0, 0, 0, 0, 1358, 697, 1239, 0, 0, 0, 0, 2338, 2082, 171, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4490, 4334, 4106, 3546, 4261, 3294, 4591] avg=4096.0000 max=4591 max_rank=7 min=3294 min_rank=6 imbalance_ratio=1.120850 max_load_reduction=28177 max_load_reduction_pct=0.859894 moved_to_echo=15692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1603, 0, 1945, 207, 56, 0, 0, 0, 942, 0, 2595, 84, 1200, 0, 0, 0, 1067, 2800, 0, 47, 476, 88, 0, 0, 776, 2024, 81, 907, 0, 0, 0, 0, 34, 2139, 859, 0, 707, 210, 0, 0, 0, 2306, 1924, 458, 35, 0, 0, 0, 0, 1924, 742, 1192, 0, 0, 0, 0, 1000, 2044, 295, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3811, 4821, 4478, 3788, 3949, 4723, 3858, 3340] avg=4096.0000 max=4821 max_rank=1 min=3340 min_rank=7 imbalance_ratio=1.177002 max_load_reduction=27947 max_load_reduction_pct=0.852875 moved_to_echo=15870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2206, 0, 1928, 170, 45, 0, 0, 0, 970, 0, 1702, 149, 978, 0, 0, 0, 1185, 1929, 0, 33, 494, 71, 0, 0, 962, 1644, 93, 1307, 0, 0, 0, 0, 36, 1622, 1472, 0, 576, 218, 0, 0, 1, 1604, 1794, 489, 29, 0, 0, 0, 0, 2201, 605, 1866, 0, 0, 0, 0, 2148, 2126, 114, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4349, 3799, 3712, 4006, 3924, 3917, 4672, 4389] avg=4096.0000 max=4672 max_rank=6 min=3712 min_rank=2 imbalance_ratio=1.140625 max_load_reduction=28096 max_load_reduction_pct=0.857422 moved_to_echo=16902 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2034, 1, 2243, 202, 32, 0, 0, 0, 964, 2, 1359, 251, 1022, 0, 0, 0, 1131, 1607, 0, 56, 369, 50, 0, 0, 1470, 1054, 108, 813, 0, 0, 0, 0, 32, 1790, 1884, 1, 604, 165, 0, 0, 5, 1468, 2362, 1214, 22, 0, 0, 0, 0, 2123, 622, 1757, 0, 0, 0, 0, 2097, 1560, 287, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4512, 3598, 3213, 3445, 4476, 5071, 4502, 3951] avg=4096.0000 max=5071 max_rank=5 min=3213 min_rank=2 imbalance_ratio=1.238037 max_load_reduction=27697 max_load_reduction_pct=0.845245 moved_to_echo=18000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1982, 1, 2159, 291, 39, 0, 0, 0, 825, 0, 1797, 337, 1261, 0, 0, 0, 1611, 2301, 0, 62, 355, 62, 0, 0, 1178, 1427, 82, 985, 0, 0, 0, 0, 64, 1631, 1260, 0, 743, 157, 0, 0, 0, 1539, 2016, 476, 25, 0, 0, 0, 0, 2083, 781, 1224, 0, 0, 0, 0, 2260, 1528, 226, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4472, 4220, 4391, 3672, 3855, 4056, 4088, 4014] avg=4096.0000 max=4472 max_rank=0 min=3672 min_rank=3 imbalance_ratio=1.091797 max_load_reduction=28296 max_load_reduction_pct=0.863525 moved_to_echo=16013 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1693, 0, 1760, 118, 58, 0, 0, 0, 942, 0, 1582, 305, 796, 0, 0, 0, 1123, 2207, 0, 54, 572, 91, 0, 0, 1140, 2109, 58, 1407, 0, 0, 0, 0, 27, 1831, 1997, 0, 469, 252, 0, 0, 2, 1186, 1489, 362, 37, 0, 0, 0, 0, 2329, 494, 1263, 0, 0, 0, 0, 2407, 2458, 150, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3629, 3625, 4047, 4714, 4576, 3076, 4086, 5015] avg=4096.0000 max=5015 max_rank=7 min=3076 min_rank=5 imbalance_ratio=1.224365 max_load_reduction=27753 max_load_reduction_pct=0.846954 moved_to_echo=16753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2099, 0, 1412, 284, 53, 0, 0, 0, 838, 0, 1999, 122, 1032, 0, 0, 0, 1656, 2476, 1, 84, 489, 84, 0, 0, 1016, 1945, 202, 1936, 0, 0, 0, 0, 40, 1417, 1666, 0, 608, 216, 0, 0, 1, 1567, 1788, 427, 34, 0, 0, 0, 1, 1842, 639, 1183, 0, 0, 0, 0, 1425, 2103, 83, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3848, 3991, 4790, 5099, 3947, 3817, 3665, 3611] avg=4096.0000 max=5099 max_rank=3 min=3611 min_rank=7 imbalance_ratio=1.244873 max_load_reduction=27669 max_load_reduction_pct=0.844391 moved_to_echo=15040 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7808, 128, 14720, 1792, 8576, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[15360, 128, 15488, 1792, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8832, 13440, 896, 9856, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10496, 17664, 128, 512, 3712, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 13824, 12160, 128, 5120, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 16256, 5376, 11264, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15616, 15616, 1664, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 12800, 15360, 4608, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[69, 0, 0, 2, 1294, 0, 0, 0, 0, 3117, 4096, 4077, 4021, 0, 1, 3487, 4096, 0, 0, 0, 327, 0, 0, 0, 4096, 26, 0, 0, 0, 4059, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[497, 0, 0, 0, 2531, 0, 0, 0, 0, 2603, 4096, 3906, 3824, 0, 12, 2567, 4096, 0, 0, 0, 625, 0, 0, 0, 4087, 3, 0, 0, 0, 3921, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[223, 0, 0, 0, 2333, 0, 0, 0, 1, 3072, 4088, 4076, 3941, 0, 9, 2072, 4096, 0, 0, 0, 660, 0, 0, 0, 4095, 11, 0, 0, 0, 4091, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[177, 0, 0, 0, 1329, 0, 0, 0, 0, 3127, 4095, 4062, 3959, 0, 15, 2934, 4096, 0, 0, 0, 776, 0, 0, 0, 4096, 18, 0, 0, 0, 4084, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[102, 1, 0, 0, 2836, 0, 1, 0, 3, 2270, 3959, 4058, 4031, 2, 18, 2211, 4093, 0, 0, 1, 1007, 0, 1, 0, 4091, 17, 2, 0, 0, 4063, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[191, 0, 1, 1, 2925, 0, 0, 0, 0, 2956, 4096, 4025, 4039, 3, 15, 1957, 4096, 0, 0, 0, 315, 0, 0, 0, 4093, 18, 0, 0, 0, 4036, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[70, 0, 0, 0, 2820, 0, 0, 0, 0, 3029, 4095, 3934, 4031, 0, 0, 1929, 4096, 0, 0, 0, 612, 0, 0, 0, 4095, 19, 0, 0, 0, 4038, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[50, 0, 0, 0, 2645, 0, 0, 0, 2, 3584, 4061, 3953, 4001, 0, 25, 1634, 4096, 0, 0, 0, 657, 0, 0, 0, 4095, 16, 0, 0, 0, 3949, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[69, 0, 0, 2, 3945, 0, 0, 0, 1294, 0, 0, 0, 1774, 0, 0, 0, 0, 3117, 0, 1144, 0, 0, 0, 0, 1754, 0, 1, 3487, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 0, 2933, 441, 151, 0, 4083, 26, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 52, 13, 0, 0] rerouted_tokens_per_ep_rank=[4016, 3068, 4261, 5242, 4096, 3852, 4109, 4124] avg=4096.0000 max=5242 max_rank=3 min=3068 min_rank=1 imbalance_ratio=1.279785 max_load_reduction=27526 max_load_reduction_pct=0.840027 moved_to_echo=16181 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[223, 0, 0, 0, 3938, 0, 0, 0, 2333, 0, 0, 0, 1739, 0, 0, 0, 1, 3072, 0, 1144, 0, 0, 0, 0, 1719, 0, 9, 2072, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 660, 0, 0, 0, 2932, 432, 150, 0, 4082, 11, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 51, 13, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4072, 4217, 3800, 4096, 4174, 4093, 4155] avg=4096.0000 max=4217 max_rank=2 min=3800 min_rank=3 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[497, 0, 0, 0, 3945, 0, 0, 0, 2531, 0, 0, 0, 1687, 0, 0, 0, 0, 2603, 0, 1096, 0, 0, 0, 0, 1667, 0, 12, 2567, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 625, 0, 0, 0, 2810, 420, 151, 0, 4074, 3, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 0, 50, 13, 0, 0] rerouted_tokens_per_ep_rank=[4442, 4218, 3699, 4246, 4096, 4006, 4077, 3984] avg=4096.0000 max=4442 max_rank=0 min=3699 min_rank=2 imbalance_ratio=1.084473 max_load_reduction=28326 max_load_reduction_pct=0.864441 moved_to_echo=16163 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[191, 0, 1, 1, 3945, 0, 0, 0, 2925, 0, 0, 0, 1782, 0, 0, 0, 0, 2956, 0, 1130, 0, 0, 0, 0, 1762, 3, 15, 1957, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 315, 0, 0, 0, 2895, 443, 151, 0, 4080, 18, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 1, 52, 13, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4707, 4086, 3737, 4096, 3804, 4098, 4102] avg=4096.0000 max=4707 max_rank=1 min=3737 min_rank=3 imbalance_ratio=1.149170 max_load_reduction=28061 max_load_reduction_pct=0.856354 moved_to_echo=16100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[102, 1, 0, 0, 3813, 0, 0, 0, 2836, 0, 1, 0, 1782, 0, 0, 0, 3, 2270, 0, 1136, 0, 0, 0, 0, 1744, 2, 18, 2211, 0, 0, 0, 0, 4093, 0, 0, 1, 2, 0, 0, 0, 1007, 0, 1, 0, 2922, 448, 146, 0, 4072, 17, 2, 0, 0, 0, 0, 0, 0, 4063, 1, 0, 57, 17, 0, 0] rerouted_tokens_per_ep_rank=[3916, 4619, 3409, 3975, 4096, 4524, 4091, 4138] avg=4096.0000 max=4619 max_rank=1 min=3409 min_rank=2 imbalance_ratio=1.127686 max_load_reduction=28149 max_load_reduction_pct=0.859039 moved_to_echo=16849 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[70, 0, 0, 0, 3943, 0, 0, 0, 2820, 0, 0, 0, 1778, 0, 0, 0, 0, 3029, 0, 1104, 0, 0, 0, 0, 1759, 0, 0, 1929, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 612, 0, 0, 0, 2830, 442, 152, 0, 4082, 19, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 52, 13, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4598, 4133, 3688, 4096, 4036, 4101, 4103] avg=4096.0000 max=4598 max_rank=1 min=3688 min_rank=3 imbalance_ratio=1.122559 max_load_reduction=28170 max_load_reduction_pct=0.859680 moved_to_echo=16336 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[177, 0, 0, 0, 3944, 0, 0, 0, 1329, 0, 0, 0, 1747, 0, 0, 0, 0, 3127, 0, 1140, 0, 0, 0, 0, 1727, 0, 15, 2934, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 776, 0, 0, 0, 2922, 434, 151, 0, 4083, 18, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 51, 13, 0, 0] rerouted_tokens_per_ep_rank=[4121, 3076, 4267, 4676, 4096, 4283, 4101, 4148] avg=4096.0000 max=4676 max_rank=3 min=3076 min_rank=1 imbalance_ratio=1.141602 max_load_reduction=28092 max_load_reduction_pct=0.857300 moved_to_echo=16628 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[50, 0, 0, 0, 3911, 0, 0, 0, 2645, 0, 0, 0, 1765, 0, 0, 0, 2, 3584, 0, 1110, 0, 0, 0, 0, 1745, 0, 25, 1634, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 2843, 439, 150, 0, 4082, 16, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 52, 13, 0, 0] rerouted_tokens_per_ep_rank=[3961, 4410, 4696, 3404, 4096, 4089, 4098, 4014] avg=4096.0000 max=4696 max_rank=2 min=3404 min_rank=3 imbalance_ratio=1.146484 max_load_reduction=28072 max_load_reduction_pct=0.856689 moved_to_echo=16297 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[18816, 0, 128, 0, 14080, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13952, 128, 128, 18816, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 23808, 0, 9088, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4992, 0, 128, 0, 23168, 3584, 1280, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1408, 128, 128, 128, 31488, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32256, 128, 128, 512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1262, 0, 31, 0, 4096, 215, 2167, 349, 4096, 0, 4096, 0, 968, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4025, 0, 0, 0, 3080, 0, 187] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1, 2, 1431, 0, 19, 2, 4095, 561, 1597, 634, 4086, 0, 4095, 0, 1400, 3, 0, 0, 0, 0, 0, 0, 4096, 4094, 12, 4091, 0, 0, 0, 2438, 0, 111] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 1690, 0, 1, 0, 4096, 354, 1934, 185, 4095, 0, 4095, 0, 1175, 1, 0, 0, 0, 0, 0, 10, 4096, 4096, 4, 4059, 0, 0, 0, 2698, 0, 178] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1177, 0, 3, 0, 4096, 379, 1576, 632, 4094, 0, 4096, 0, 1334, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 191, 4071, 0, 0, 0, 2605, 0, 322] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 614, 1, 0, 0, 4095, 287, 1788, 771, 4085, 1, 4095, 1, 1149, 0, 1, 1, 0, 1, 0, 0, 4095, 4095, 45, 4048, 0, 0, 0, 3042, 0, 552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 1416, 0, 0, 1, 4096, 280, 1586, 548, 4077, 0, 4096, 0, 609, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 4085, 0, 0, 0, 3467, 0, 311] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 1, 1208, 0, 0, 0, 4096, 329, 2226, 208, 4094, 0, 4095, 0, 1012, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 48, 4066, 1, 0, 1, 3079, 0, 112] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 978, 0, 0, 0, 4096, 313, 1347, 483, 4033, 0, 4096, 0, 2147, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 4088, 0, 0, 0, 2864, 0, 128] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 1431, 0, 2242, 632, 0, 0, 19, 2, 3749, 561, 0, 0, 0, 0, 1597, 634, 1844, 0, 0, 0, 0, 0, 2871, 0, 1400, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 12, 4082, 0, 0, 0, 0, 0, 0, 0, 2438, 0, 111, 592, 346, 9, 0] rerouted_tokens_per_ep_rank=[4308, 4331, 4075, 4274, 4096, 4094, 4094, 3496] avg=4096.0000 max=4331 max_rank=1 min=3496 min_rank=7 imbalance_ratio=1.057373 max_load_reduction=28437 max_load_reduction_pct=0.867828 moved_to_echo=15780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1262, 0, 2247, 632, 0, 0, 31, 0, 3750, 215, 0, 0, 0, 0, 2167, 349, 1849, 0, 0, 0, 0, 0, 2872, 0, 968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4016, 0, 0, 0, 0, 0, 0, 0, 3080, 0, 187, 592, 346, 9, 0] rerouted_tokens_per_ep_rank=[4141, 3996, 4365, 3840, 4096, 4096, 4020, 4214] avg=4096.0000 max=4365 max_rank=2 min=3840 min_rank=3 imbalance_ratio=1.065674 max_load_reduction=28403 max_load_reduction_pct=0.866791 moved_to_echo=16426 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1690, 0, 2247, 632, 0, 0, 1, 0, 3750, 354, 0, 0, 0, 0, 1934, 185, 1848, 0, 0, 0, 0, 0, 2871, 0, 1175, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 4, 4050, 0, 0, 0, 0, 0, 0, 0, 2698, 0, 178, 592, 346, 9, 0] rerouted_tokens_per_ep_rank=[4570, 4105, 3967, 4047, 4096, 4106, 4054, 3823] avg=4096.0000 max=4570 max_rank=0 min=3823 min_rank=7 imbalance_ratio=1.115723 max_load_reduction=28198 max_load_reduction_pct=0.860535 moved_to_echo=16079 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1208, 0, 2246, 632, 0, 0, 0, 0, 3750, 329, 0, 0, 0, 0, 2226, 208, 1848, 0, 0, 0, 0, 0, 2871, 0, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 48, 4057, 1, 0, 0, 0, 0, 0, 1, 3079, 0, 112, 592, 346, 9, 0] rerouted_tokens_per_ep_rank=[4088, 4079, 4282, 3883, 4095, 4096, 4106, 4139] avg=4096.0000 max=4282 max_rank=2 min=3883 min_rank=3 imbalance_ratio=1.045410 max_load_reduction=28486 max_load_reduction_pct=0.869324 moved_to_echo=16436 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 614, 1, 2246, 634, 1, 0, 0, 0, 3744, 287, 0, 0, 0, 0, 1788, 771, 1839, 1, 0, 0, 0, 0, 2868, 1, 1149, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 45, 4035, 0, 0, 0, 0, 0, 0, 0, 3042, 0, 552, 593, 351, 13, 7] rerouted_tokens_per_ep_rank=[3497, 4031, 4399, 4018, 4097, 4088, 4080, 4558] avg=4096.0000 max=4558 max_rank=7 min=3497 min_rank=0 imbalance_ratio=1.112793 max_load_reduction=28210 max_load_reduction_pct=0.860901 moved_to_echo=16823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1416, 0, 2237, 632, 0, 0, 0, 1, 3750, 280, 0, 0, 0, 0, 1586, 548, 1840, 0, 0, 0, 0, 0, 2872, 0, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4076, 0, 0, 0, 0, 0, 0, 0, 3467, 0, 311, 592, 346, 9, 0] rerouted_tokens_per_ep_rank=[4286, 4031, 3974, 3481, 4096, 4096, 4079, 4725] avg=4096.0000 max=4725 max_rank=7 min=3481 min_rank=3 imbalance_ratio=1.153564 max_load_reduction=28043 max_load_reduction_pct=0.855804 moved_to_echo=16996 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1177, 0, 2246, 632, 0, 0, 3, 0, 3750, 379, 0, 0, 0, 0, 1576, 632, 1848, 0, 0, 0, 0, 0, 2872, 0, 1334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 191, 4062, 0, 0, 0, 0, 0, 0, 0, 2605, 0, 322, 592, 346, 9, 0] rerouted_tokens_per_ep_rank=[4055, 4132, 4056, 4206, 4096, 4096, 4253, 3874] avg=4096.0000 max=4253 max_rank=6 min=3874 min_rank=7 imbalance_ratio=1.038330 max_load_reduction=28515 max_load_reduction_pct=0.870209 moved_to_echo=16319 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 978, 0, 2213, 632, 0, 0, 0, 0, 3750, 313, 0, 0, 0, 0, 1347, 483, 1820, 0, 0, 0, 0, 0, 2872, 0, 2147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4079, 0, 0, 0, 0, 0, 0, 0, 2864, 0, 128, 592, 346, 9, 0] rerouted_tokens_per_ep_rank=[3823, 4063, 3650, 5019, 4096, 4096, 4082, 3939] avg=4096.0000 max=5019 max_rank=3 min=3650 min_rank=2 imbalance_ratio=1.225342 max_load_reduction=27749 max_load_reduction_pct=0.846832 moved_to_echo=16213 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14336, 3840, 14848, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 9856, 128, 18048, 5120, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 23296, 0, 1920, 4864, 2816, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[23040, 128, 9856, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 30080, 2816, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 32512, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3017, 2, 3982, 0, 3998, 0, 0, 0, 0, 0, 0, 4096, 176, 0, 4096, 0, 0, 0, 2338, 0, 2925, 4093, 3916, 8, 0, 0, 121] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3350, 0, 4008, 0, 4080, 0, 0, 0, 0, 0, 0, 4096, 129, 0, 4092, 0, 0, 0, 1276, 6, 3494, 4096, 4043, 0, 0, 3, 95] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2918, 0, 3947, 0, 4086, 0, 0, 0, 0, 0, 0, 4096, 79, 0, 4091, 0, 0, 0, 2536, 5, 2883, 4092, 3773, 2, 0, 1, 259] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 1, 3048, 0, 3801, 0, 4082, 1, 0, 1, 0, 1, 0, 4095, 111, 1, 4094, 0, 0, 0, 2516, 11, 2699, 4087, 3944, 15, 1, 0, 257] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3261, 0, 4027, 0, 4037, 0, 0, 0, 0, 0, 0, 4096, 139, 0, 4093, 0, 0, 0, 1628, 1, 3198, 4095, 3997, 1, 0, 5, 190] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3144, 0, 4009, 0, 4062, 0, 0, 0, 0, 0, 0, 4096, 29, 0, 4069, 0, 0, 0, 2795, 4, 2162, 4093, 4025, 15, 0, 1, 264] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3071, 0, 4026, 0, 3982, 0, 0, 0, 0, 0, 0, 4096, 116, 0, 4081, 1, 0, 0, 1988, 1, 2991, 4095, 3945, 2, 0, 8, 365] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3196, 0, 3977, 0, 4070, 0, 0, 0, 0, 0, 0, 4096, 60, 0, 4096, 0, 0, 0, 2290, 1, 3126, 4023, 3764, 2, 0, 1, 66] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 3350, 0, 980, 0, 0, 0, 0, 0, 4080, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 129, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 1276, 1929, 51, 0, 0, 6, 3494, 0, 1193, 0, 0, 0, 0, 0, 0, 3, 95, 3017, 921, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4330, 4126, 4107, 4124, 3256, 4693, 4036] avg=4096.0000 max=4693 max_rank=6 min=3256 min_rank=5 imbalance_ratio=1.145752 max_load_reduction=28075 max_load_reduction_pct=0.856781 moved_to_echo=16109 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 2918, 0, 965, 0, 0, 0, 0, 0, 4086, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 4092, 11, 0, 0, 0, 79, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 2536, 1800, 51, 0, 0, 5, 2883, 0, 1113, 0, 0, 0, 0, 2, 0, 1, 259, 2971, 860, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3883, 4132, 4103, 4073, 4387, 4001, 4093] avg=4096.0000 max=4387 max_rank=5 min=3883 min_rank=1 imbalance_ratio=1.071045 max_load_reduction=28381 max_load_reduction_pct=0.866119 moved_to_echo=16554 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 3261, 0, 984, 0, 0, 0, 0, 0, 4037, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 4095, 11, 0, 0, 0, 139, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 1628, 1907, 51, 0, 0, 1, 3198, 0, 1179, 0, 0, 0, 0, 1, 0, 5, 190, 3032, 911, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4245, 4083, 4106, 4135, 3586, 4378, 4139] avg=4096.0000 max=4378 max_rank=6 min=3586 min_rank=5 imbalance_ratio=1.068848 max_load_reduction=28390 max_load_reduction_pct=0.866394 moved_to_echo=16238 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 0, 0, 0, 1, 3048, 0, 922, 0, 0, 0, 0, 0, 4082, 1, 0, 48, 0, 0, 0, 1, 0, 1, 0, 4087, 14, 0, 0, 0, 111, 1, 3989, 0, 0, 0, 0, 0, 0, 0, 2516, 1884, 57, 0, 0, 11, 2699, 0, 1158, 0, 0, 0, 0, 15, 1, 0, 257, 2865, 902, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3971, 4131, 4103, 4101, 4457, 3868, 4040] avg=4096.0000 max=4457 max_rank=5 min=3868 min_rank=6 imbalance_ratio=1.088135 max_load_reduction=28311 max_load_reduction_pct=0.863983 moved_to_echo=16466 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 3071, 0, 984, 0, 0, 0, 0, 0, 3982, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 4095, 11, 0, 0, 0, 116, 0, 3984, 0, 0, 0, 0, 1, 0, 0, 1988, 1882, 51, 0, 0, 1, 2991, 0, 1164, 0, 0, 0, 0, 2, 0, 8, 365, 3031, 899, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4055, 4028, 4106, 4100, 3922, 4156, 4305] avg=4096.0000 max=4305 max_rank=7 min=3922 min_rank=5 imbalance_ratio=1.051025 max_load_reduction=28463 max_load_reduction_pct=0.868622 moved_to_echo=16483 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 3017, 2, 973, 0, 0, 0, 0, 0, 3998, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 4093, 11, 0, 0, 0, 176, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 2338, 1868, 51, 0, 0, 0, 2925, 0, 1156, 0, 0, 0, 0, 8, 0, 0, 121, 2998, 892, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3992, 4044, 4104, 4175, 4257, 4081, 4019] avg=4096.0000 max=4257 max_rank=5 min=3992 min_rank=1 imbalance_ratio=1.039307 max_load_reduction=28511 max_load_reduction_pct=0.870087 moved_to_echo=16532 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 3144, 0, 980, 0, 0, 0, 0, 0, 4062, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 4093, 11, 0, 0, 0, 29, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 2795, 1920, 51, 0, 0, 4, 2162, 0, 1188, 0, 0, 0, 0, 15, 0, 1, 264, 3018, 917, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4124, 4108, 4104, 4001, 4766, 3354, 4215] avg=4096.0000 max=4766 max_rank=5 min=3354 min_rank=6 imbalance_ratio=1.163574 max_load_reduction=28002 max_load_reduction_pct=0.854553 moved_to_echo=16336 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 3196, 0, 972, 0, 0, 0, 0, 0, 4070, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 4023, 11, 1, 0, 0, 60, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 2290, 1796, 51, 0, 0, 1, 3126, 0, 1110, 0, 0, 0, 0, 2, 0, 1, 66, 2994, 858, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4168, 4116, 4035, 4059, 4137, 4237, 3921] avg=4096.0000 max=4237 max_rank=6 min=3921 min_rank=7 imbalance_ratio=1.034424 max_load_reduction=28531 max_load_reduction_pct=0.870697 moved_to_echo=16354 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 17408, 15104, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 1664, 23936, 7168, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 23552, 0, 9344, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 25088, 128, 7808, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 896, 128, 32000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32512, 128, 0, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2576, 3641, 4096, 0, 0, 2, 0, 4095, 0, 0, 0, 4096, 38, 0, 0, 0, 0, 0, 0, 580, 0, 1, 4088, 0, 4062, 0, 0, 1397] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2810, 3891, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 1241, 1, 0, 4095, 0, 4092, 0, 0, 225] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2912, 2923, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 71, 0, 0, 0, 0, 0, 0, 1501, 0, 0, 4094, 0, 4073, 0, 1, 809] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2805, 3869, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 35, 0, 0, 0, 0, 0, 0, 1182, 0, 5, 4096, 0, 4071, 0, 0, 329] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 2362, 3617, 4095, 0, 1, 0, 0, 4095, 1, 0, 1, 4095, 32, 0, 0, 0, 1, 0, 0, 1163, 0, 0, 4095, 0, 4020, 1, 0, 1091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3014, 3814, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 69, 0, 0, 0, 0, 0, 0, 700, 0, 1, 4094, 0, 4081, 0, 0, 612] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1930, 3775, 4096, 0, 0, 9, 0, 4095, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 0, 1067, 0, 1, 4096, 0, 4079, 0, 0, 1381] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3404, 2853, 4096, 0, 0, 1, 0, 4094, 0, 0, 0, 4096, 36, 0, 0, 0, 0, 0, 0, 540, 0, 0, 4096, 0, 4095, 0, 0, 1361] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2805, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 35, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 1182, 2375, 875, 44, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 3196, 0, 0, 329, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4299, 4088, 4096, 4087, 4476, 4101, 3525] avg=4096.0000 max=4476 max_rank=5 min=3525 min_rank=7 imbalance_ratio=1.092773 max_load_reduction=28292 max_load_reduction_pct=0.863403 moved_to_echo=16189 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2912, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 71, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 1501, 1794, 876, 45, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3197, 0, 1, 809, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4041, 4096, 4096, 4122, 4216, 4094, 4007] avg=4096.0000 max=4216 max_rank=5 min=4007 min_rank=7 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2576, 1406, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 38, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 580, 2235, 874, 44, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 3188, 0, 0, 1397, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3982, 4097, 4096, 4090, 3733, 4089, 4585] avg=4096.0000 max=4585 max_rank=7 min=3733 min_rank=5 imbalance_ratio=1.119385 max_load_reduction=28183 max_load_reduction_pct=0.860077 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 1930, 1458, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 47, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 1067, 2317, 877, 45, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 3202, 0, 0, 1381, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3388, 4104, 4096, 4098, 4306, 4097, 4583] avg=4096.0000 max=4583 max_rank=7 min=3388 min_rank=1 imbalance_ratio=1.118896 max_load_reduction=28185 max_load_reduction_pct=0.860138 moved_to_echo=17084 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2810, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 29, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 1241, 2389, 880, 44, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 3212, 0, 0, 225, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4312, 4096, 4096, 4081, 4554, 4096, 3437] avg=4096.0000 max=4554 max_rank=5 min=3437 min_rank=7 imbalance_ratio=1.111816 max_load_reduction=28214 max_load_reduction_pct=0.861023 moved_to_echo=16168 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4093, 0, 0, 0, 0, 2362, 1391, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 0, 0, 0, 1, 0, 1, 4094, 0, 0, 0, 0, 32, 0, 0, 0, 4051, 0, 0, 0, 1, 0, 0, 1163, 2226, 868, 44, 0, 0, 0, 4095, 0, 3, 2, 0, 0, 3149, 1, 0, 1091, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3753, 4097, 4096, 4083, 4302, 4100, 4241] avg=4096.0000 max=4302 max_rank=5 min=3753 min_rank=1 imbalance_ratio=1.050293 max_load_reduction=28466 max_load_reduction_pct=0.868713 moved_to_echo=16726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3014, 1473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 69, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 700, 2341, 878, 45, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 3203, 0, 0, 612, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4487, 4095, 4096, 4120, 3964, 4095, 3815] avg=4096.0000 max=4487 max_rank=1 min=3815 min_rank=7 imbalance_ratio=1.095459 max_load_reduction=28281 max_load_reduction_pct=0.863068 moved_to_echo=15994 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3404, 1102, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 36, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 540, 1751, 881, 45, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3214, 0, 0, 1361, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4506, 4095, 4096, 4087, 3217, 4096, 4575] avg=4096.0000 max=4575 max_rank=7 min=3217 min_rank=5 imbalance_ratio=1.116943 max_load_reduction=28193 max_load_reduction_pct=0.860382 moved_to_echo=15975 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 8064, 17536, 7040, 384, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[21888, 11008, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[25600, 128, 128, 7296, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[384, 0, 0, 0, 32512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 3401, 0, 2, 947, 4095, 0, 1, 1, 1451, 0, 1, 0, 4065, 2, 4095, 1, 1381, 4067, 1, 4093, 1032, 1, 4095, 1, 0, 0, 34, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 2759, 0, 0, 2344, 4094, 3, 0, 0, 1118, 0, 0, 0, 4096, 1, 4096, 0, 1183, 3952, 0, 4095, 822, 0, 4096, 0, 0, 0, 109, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 2396, 0, 0, 1135, 4095, 0, 0, 0, 2268, 0, 0, 0, 4070, 0, 4096, 0, 733, 4080, 0, 4096, 1572, 0, 4095, 0, 0, 0, 132, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 3573, 0, 0, 1255, 4093, 0, 0, 0, 1091, 0, 0, 0, 4085, 0, 4096, 0, 765, 4020, 0, 4096, 1551, 0, 4096, 0, 0, 0, 47, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 2870, 0, 0, 1459, 4091, 3, 0, 0, 959, 0, 0, 1, 4095, 1, 4096, 0, 1114, 4030, 0, 4096, 1821, 0, 4095, 0, 0, 0, 37, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 2567, 0, 2, 1436, 4084, 6, 0, 0, 2121, 0, 0, 0, 4080, 0, 4096, 0, 576, 4035, 0, 4096, 1390, 0, 4096, 0, 0, 0, 183, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 2665, 0, 0, 1250, 4096, 0, 0, 0, 1604, 0, 0, 0, 4091, 0, 4096, 0, 1355, 4064, 0, 4096, 1245, 0, 4096, 0, 0, 0, 110, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 2207, 0, 0, 2243, 4095, 1, 0, 0, 1332, 0, 0, 0, 4090, 0, 4096, 0, 1551, 4075, 0, 4096, 723, 0, 4096, 0, 0, 0, 163, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2759, 0, 0, 1004, 263, 0, 0, 2344, 2587, 3, 0, 0, 0, 0, 0, 0, 1118, 0, 0, 1507, 1005, 90, 0, 0, 4096, 1, 12, 0, 0, 0, 0, 0, 1183, 2948, 0, 0, 0, 0, 0, 2827, 822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 3994, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4934, 3720, 4109, 4131, 3649, 4103, 4096] avg=4096.0000 max=4934 max_rank=1 min=3649 min_rank=5 imbalance_ratio=1.204590 max_load_reduction=27834 max_load_reduction_pct=0.849426 moved_to_echo=15979 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3573, 0, 0, 1021, 263, 0, 0, 1255, 2586, 0, 0, 0, 0, 0, 0, 0, 1091, 0, 0, 1507, 1005, 90, 0, 0, 4085, 0, 12, 0, 0, 0, 0, 0, 765, 2999, 0, 0, 0, 0, 0, 2828, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 3994, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4857, 3841, 3693, 4097, 3764, 4379, 4041, 4096] avg=4096.0000 max=4857 max_rank=0 min=3693 min_rank=2 imbalance_ratio=1.185791 max_load_reduction=27911 max_load_reduction_pct=0.851776 moved_to_echo=16280 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2567, 0, 2, 1025, 263, 0, 0, 1436, 2581, 6, 0, 0, 0, 0, 0, 0, 2121, 0, 0, 1503, 1005, 90, 0, 0, 4080, 0, 12, 0, 0, 0, 0, 0, 576, 3010, 0, 0, 0, 0, 0, 2828, 1390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 3994, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3857, 4023, 4719, 4092, 3586, 4218, 4177, 4096] avg=4096.0000 max=4719 max_rank=2 min=3586 min_rank=4 imbalance_ratio=1.152100 max_load_reduction=28049 max_load_reduction_pct=0.855988 moved_to_echo=16077 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2870, 0, 0, 1024, 263, 0, 0, 1459, 2585, 3, 0, 0, 0, 0, 0, 0, 959, 0, 0, 1506, 1005, 90, 0, 1, 4095, 1, 12, 0, 0, 0, 0, 0, 1114, 3006, 0, 0, 0, 0, 0, 2828, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 3994, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4047, 3560, 4109, 4120, 4649, 4031, 4095] avg=4096.0000 max=4649 max_rank=5 min=3560 min_rank=2 imbalance_ratio=1.135010 max_load_reduction=28119 max_load_reduction_pct=0.858124 moved_to_echo=16895 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3401, 0, 2, 1038, 270, 0, 0, 947, 2584, 0, 1, 0, 0, 0, 0, 1, 1451, 0, 1, 1511, 1007, 92, 0, 0, 4065, 2, 7, 0, 0, 0, 0, 1, 1381, 3029, 1, 0, 0, 0, 0, 2816, 1032, 1, 0, 0, 0, 0, 0, 1, 0, 0, 34, 3994, 0, 0, 0, 0, 1, 0, 0, 4095, 2, 0, 0] rerouted_tokens_per_ep_rank=[4711, 3532, 4063, 4074, 4412, 3849, 4029, 4098] avg=4096.0000 max=4711 max_rank=0 min=3532 min_rank=1 imbalance_ratio=1.150146 max_load_reduction=28057 max_load_reduction_pct=0.856232 moved_to_echo=16388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2396, 0, 0, 1036, 263, 0, 0, 1135, 2587, 0, 0, 0, 0, 0, 0, 0, 2268, 0, 0, 1508, 1005, 90, 0, 0, 4070, 0, 12, 0, 0, 0, 0, 0, 733, 3044, 0, 0, 0, 0, 0, 2828, 1572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 3994, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[3695, 3722, 4871, 4082, 3777, 4400, 4126, 4095] avg=4096.0000 max=4871 max_rank=2 min=3695 min_rank=0 imbalance_ratio=1.189209 max_load_reduction=27897 max_load_reduction_pct=0.851349 moved_to_echo=16398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2207, 0, 0, 1035, 263, 0, 0, 2243, 2587, 1, 0, 0, 0, 0, 0, 0, 1332, 0, 0, 1508, 1005, 90, 0, 0, 4090, 0, 12, 0, 0, 0, 0, 0, 1551, 3040, 0, 0, 0, 0, 0, 2828, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 3994, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3505, 4831, 3935, 4102, 4591, 3551, 4157, 4096] avg=4096.0000 max=4831 max_rank=1 min=3505 min_rank=0 imbalance_ratio=1.179443 max_load_reduction=27937 max_load_reduction_pct=0.852570 moved_to_echo=16395 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2665, 0, 0, 1032, 263, 0, 0, 1250, 2588, 0, 0, 0, 0, 0, 0, 0, 1604, 0, 0, 1508, 1005, 90, 0, 0, 4091, 0, 12, 0, 0, 0, 0, 0, 1355, 3032, 0, 0, 0, 0, 0, 2828, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 3994, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3960, 3838, 4207, 4103, 4387, 4073, 4104, 4096] avg=4096.0000 max=4387 max_rank=4 min=3838 min_rank=1 imbalance_ratio=1.071045 max_load_reduction=28381 max_load_reduction_pct=0.866119 moved_to_echo=16660 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 12032, 0, 128, 12160, 8064, 768, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 22528, 0, 128, 8320, 2176, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 8704, 24192, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12160, 20736, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[22656, 10240, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 896, 32000, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 18, 0, 1, 0, 0, 505, 2255, 0, 2006, 0, 0, 4096, 4092, 70, 4096, 0, 0, 0, 4096, 3334, 0, 0, 7, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4, 0, 1, 0, 0, 659, 2962, 0, 1788, 0, 0, 4096, 4094, 62, 4096, 0, 0, 0, 4096, 2685, 0, 0, 33, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 19, 0, 0, 1, 0, 467, 2472, 1, 1875, 0, 1, 4095, 4095, 422, 4094, 0, 1, 1, 4095, 2924, 1, 0, 11, 0, 1, 4095, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 0, 0, 322, 2886, 0, 1784, 0, 0, 4096, 4096, 7, 4096, 0, 0, 0, 4096, 3188, 0, 0, 4, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 7, 0, 2, 0, 0, 430, 2056, 0, 2412, 0, 0, 4096, 4096, 112, 4096, 0, 0, 0, 4096, 3171, 0, 0, 2, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 16, 0, 1, 0, 0, 603, 2707, 0, 1527, 1, 0, 4096, 4095, 75, 4096, 0, 0, 0, 4096, 3151, 0, 0, 113, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 25, 0, 0, 0, 0, 353, 3191, 0, 1305, 0, 0, 4096, 4096, 418, 4096, 0, 0, 0, 4096, 2834, 0, 0, 66, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3, 0, 0, 0, 0, 198, 2461, 0, 2655, 0, 0, 4096, 4096, 120, 4096, 0, 0, 0, 4096, 2733, 0, 0, 23, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 18, 0, 1, 0, 4084, 0, 0, 0, 0, 505, 2255, 0, 870, 159, 0, 0, 2006, 0, 0, 2178, 0, 0, 0, 0, 3933, 70, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 3334, 0, 0, 0, 0, 0, 0, 7, 0, 3002, 1048, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4103, 3789, 4184, 4003, 4428, 4069, 4096] avg=4096.0000 max=4428 max_rank=5 min=3789 min_rank=2 imbalance_ratio=1.081055 max_load_reduction=28340 max_load_reduction_pct=0.864868 moved_to_echo=16596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 1, 0, 4084, 0, 0, 0, 0, 659, 2962, 0, 870, 159, 0, 0, 1788, 0, 0, 2178, 0, 0, 0, 0, 3935, 62, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 2685, 0, 0, 0, 0, 0, 0, 33, 0, 3002, 1048, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4089, 4650, 3966, 3997, 3779, 4095, 4096] avg=4096.0000 max=4650 max_rank=2 min=3779 min_rank=5 imbalance_ratio=1.135254 max_load_reduction=28118 max_load_reduction_pct=0.858093 moved_to_echo=15967 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4084, 0, 0, 0, 0, 322, 2886, 0, 870, 159, 0, 0, 1784, 0, 0, 2178, 0, 0, 0, 0, 3937, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 3188, 0, 0, 0, 0, 0, 0, 4, 0, 3002, 1048, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4085, 4237, 3962, 3944, 4282, 4066, 4096] avg=4096.0000 max=4282 max_rank=5 min=3944 min_rank=4 imbalance_ratio=1.045410 max_load_reduction=28486 max_load_reduction_pct=0.869324 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 16, 0, 1, 0, 4084, 0, 0, 0, 0, 603, 2707, 0, 870, 159, 0, 0, 1527, 1, 0, 2178, 0, 0, 0, 0, 3936, 75, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 3151, 0, 0, 0, 0, 0, 0, 113, 0, 3002, 1048, 12, 0, 0, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4339, 3706, 4011, 4245, 4175, 4095] avg=4096.0000 max=4339 max_rank=2 min=3706 min_rank=3 imbalance_ratio=1.059326 max_load_reduction=28429 max_load_reduction_pct=0.867584 moved_to_echo=16526 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 0, 0, 0, 19, 0, 0, 1, 4082, 0, 0, 0, 0, 467, 2472, 1, 873, 164, 0, 0, 1875, 0, 1, 2168, 0, 0, 0, 0, 3930, 422, 0, 0, 0, 0, 0, 0, 1, 1, 1088, 2924, 0, 0, 0, 0, 1, 0, 11, 0, 3007, 1054, 12, 0, 1, 4095, 1, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4102, 3977, 4044, 4352, 4014, 4085, 4098] avg=4096.0000 max=4352 max_rank=4 min=3977 min_rank=2 imbalance_ratio=1.062500 max_load_reduction=28416 max_load_reduction_pct=0.867188 moved_to_echo=16549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 4083, 0, 0, 0, 0, 198, 2461, 0, 870, 159, 0, 0, 2655, 0, 0, 2178, 0, 0, 0, 0, 3937, 120, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 2733, 0, 0, 0, 0, 0, 0, 23, 0, 3002, 1048, 13, 0, 0, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4086, 3688, 4833, 4057, 3827, 4086, 4095] avg=4096.0000 max=4833 max_rank=3 min=3688 min_rank=2 imbalance_ratio=1.179932 max_load_reduction=27935 max_load_reduction_pct=0.852509 moved_to_echo=16065 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 25, 0, 0, 0, 4084, 0, 0, 0, 0, 353, 3191, 0, 870, 159, 0, 0, 1305, 0, 0, 2178, 0, 0, 0, 0, 3937, 418, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 2834, 0, 0, 0, 0, 0, 0, 66, 0, 3002, 1048, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4109, 4573, 3483, 4355, 3928, 4128, 4096] avg=4096.0000 max=4573 max_rank=2 min=3483 min_rank=3 imbalance_ratio=1.116455 max_load_reduction=28195 max_load_reduction_pct=0.860443 moved_to_echo=16507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 2, 0, 4084, 0, 0, 0, 0, 430, 2056, 0, 870, 159, 0, 0, 2412, 0, 0, 2178, 0, 0, 0, 0, 3937, 112, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 3171, 0, 0, 0, 0, 0, 0, 2, 0, 3002, 1048, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4093, 3515, 4590, 4049, 4265, 4064, 4096] avg=4096.0000 max=4590 max_rank=3 min=3515 min_rank=2 imbalance_ratio=1.120605 max_load_reduction=28178 max_load_reduction_pct=0.859924 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 384, 0, 24064, 8448, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[15360, 128, 128, 17536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 3584, 20992, 128, 7040, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 8832, 24064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[31488, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4088, 3719, 0, 3255, 0, 0, 4096, 4096, 0, 181, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 2, 1042, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4077, 3785, 0, 2616, 0, 0, 4096, 4096, 0, 1186, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 1, 620, 0, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 4095, 3324, 0, 3234, 0, 1, 4094, 4095, 1, 519, 0, 0, 1, 0, 4095, 0, 1, 4095, 4093, 0, 0, 1, 1, 1114, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4091, 3776, 0, 3420, 0, 0, 4096, 4096, 0, 450, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 549, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3679, 0, 3426, 0, 0, 4096, 4096, 0, 399, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 688, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4092, 4033, 0, 2998, 0, 0, 4095, 4096, 0, 607, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 559, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4094, 3537, 0, 3313, 0, 0, 4096, 4096, 0, 1046, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 2, 296, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4077, 3408, 0, 3054, 0, 0, 4096, 4096, 0, 502, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 5, 1242, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 438, 3785, 0, 0, 0, 0, 0, 2616, 0, 0, 610, 319, 0, 0, 3486, 0, 0, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 620, 0, 0, 3320, 1, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4223, 3545, 4672, 4096, 4096, 4096, 3942, 4098] avg=4096.0000 max=4672 max_rank=2 min=3545 min_rank=1 imbalance_ratio=1.140625 max_load_reduction=28096 max_load_reduction_pct=0.857422 moved_to_echo=16232 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 439, 3719, 0, 0, 0, 0, 0, 3255, 0, 0, 610, 320, 0, 0, 3486, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 1042, 0, 0, 3329, 1, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4185, 3667, 4096, 4096, 4096, 4374, 4096] avg=4096.0000 max=4374 max_rank=6 min=3667 min_rank=2 imbalance_ratio=1.067871 max_load_reduction=28394 max_load_reduction_pct=0.866516 moved_to_echo=16662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 440, 3776, 0, 0, 0, 0, 0, 3420, 0, 0, 610, 320, 0, 0, 3486, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 3331, 1, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4216, 4350, 3936, 4097, 4096, 4096, 3881, 4096] avg=4096.0000 max=4350 max_rank=1 min=3881 min_rank=6 imbalance_ratio=1.062012 max_load_reduction=28418 max_load_reduction_pct=0.867249 moved_to_echo=16169 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 440, 4033, 0, 0, 0, 0, 0, 2998, 0, 0, 610, 320, 0, 0, 3485, 0, 0, 607, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 3332, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4473, 3928, 4092, 4096, 4096, 4096, 3892, 4095] avg=4096.0000 max=4473 max_rank=0 min=3892 min_rank=6 imbalance_ratio=1.092041 max_load_reduction=28295 max_load_reduction_pct=0.863495 moved_to_echo=16179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 431, 3324, 0, 0, 0, 0, 0, 3234, 0, 1, 616, 325, 0, 0, 3476, 0, 1, 519, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4093, 0, 0, 1, 2, 0, 0, 0, 1, 1114, 0, 1, 3339, 1, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[3757, 4176, 3996, 4096, 4096, 4096, 4456, 4095] avg=4096.0000 max=4456 max_rank=6 min=3757 min_rank=0 imbalance_ratio=1.087891 max_load_reduction=28312 max_load_reduction_pct=0.864014 moved_to_echo=16743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 440, 3679, 0, 0, 0, 0, 0, 3426, 0, 0, 610, 321, 0, 0, 3486, 0, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 688, 0, 0, 3335, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4357, 3885, 4096, 4096, 4096, 4024, 4095] avg=4096.0000 max=4357 max_rank=1 min=3885 min_rank=2 imbalance_ratio=1.063721 max_load_reduction=28411 max_load_reduction_pct=0.867035 moved_to_echo=16311 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 439, 3537, 0, 0, 0, 0, 0, 3313, 0, 0, 610, 321, 0, 0, 3486, 0, 0, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 296, 0, 0, 3334, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3976, 4244, 4532, 4096, 4096, 4096, 3632, 4096] avg=4096.0000 max=4532 max_rank=2 min=3632 min_rank=6 imbalance_ratio=1.106445 max_load_reduction=28236 max_load_reduction_pct=0.861694 moved_to_echo=15920 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 438, 3408, 0, 0, 0, 0, 0, 3054, 0, 0, 610, 319, 0, 0, 3486, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 1242, 0, 0, 3320, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[3846, 3983, 3988, 4095, 4096, 4096, 4567, 4097] avg=4096.0000 max=4567 max_rank=6 min=3846 min_rank=0 imbalance_ratio=1.114990 max_load_reduction=28201 max_load_reduction_pct=0.860626 moved_to_echo=16856 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 25344, 0, 128, 4992, 2688, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 3584, 29312, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 6144, 0, 128, 26752, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[27904, 0, 128, 4992, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 0, 0, 0, 4095, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 2, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 2, 0, 4095, 0, 1, 1, 4095, 4090, 4011, 1, 1, 1, 0, 4084, 0, 0, 13, 201, 4095, 1, 1, 1, 5, 8, 0, 0, 0, 4071, 0, 3989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 0, 0, 0, 4094, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4096, 0, 0, 0, 4094, 4095, 4096, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 2, 4071, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4084, 9, 0, 0, 0, 15, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4107, 4096, 4074, 4096, 4094, 4109] avg=4096.0000 max=4109 max_rank=7 min=4074 min_rank=4 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 2, 4071, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 9, 0, 0, 0, 15, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4105, 4097, 4073, 4096, 4094, 4111] avg=4096.0000 max=4111 max_rank=7 min=4073 min_rank=4 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 3, 0, 4071, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 9, 0, 0, 0, 15, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4107, 4096, 4074, 4096, 4093, 4111] avg=4096.0000 max=4111 max_rank=7 min=4074 min_rank=4 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 4071, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 9, 0, 0, 0, 15, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4107, 4097, 4072, 4096, 4093, 4111] avg=4096.0000 max=4111 max_rank=7 min=4072 min_rank=4 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2, 0, 4095, 0, 0, 0, 4094, 0, 1, 1, 0, 0, 0, 0, 0, 10, 4011, 1, 0, 0, 0, 0, 1, 1, 0, 4084, 2, 2, 1, 0, 0, 0, 13, 201, 4050, 0, 0, 0, 4093, 1, 1, 1, 0, 0, 0, 0, 5, 8, 0, 0, 4080, 15, 0, 0, 0, 4, 0, 3989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4022, 4091, 4264, 4096, 4108, 3993] avg=4096.0000 max=4264 max_rank=4 min=3993 min_rank=7 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16461 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 9, 0, 0, 0, 15, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4107, 4097, 4071, 4096, 4094, 4111] avg=4096.0000 max=4111 max_rank=7 min=4071 min_rank=4 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 9, 1, 0, 0, 15, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4107, 4097, 4071, 4096, 4095, 4111] avg=4096.0000 max=4111 max_rank=7 min=4071 min_rank=4 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4085, 9, 1, 0, 0, 15, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4106, 4097, 4069, 4096, 4097, 4111] avg=4096.0000 max=4111 max_rank=7 min=4069 min_rank=4 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 256, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 407, 0, 0, 0, 0, 159, 4091, 0, 0, 0, 3424, 119, 0, 4094, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 2, 1, 4094, 0, 0, 0, 6, 0, 3592, 1, 1, 1, 1, 4095, 659, 3, 1, 7, 2, 399, 4082, 0, 1, 0, 3430, 189, 1, 4009, 4095, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 4089, 176, 0, 0, 15, 0, 382, 4095, 0, 0, 0, 3212, 322, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4091, 163, 0, 0, 5, 0, 59, 4094, 0, 0, 0, 3831, 47, 0, 4095, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 359, 0, 0, 0, 0, 195, 4092, 0, 0, 0, 3447, 111, 0, 4087, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4093, 64, 0, 0, 2, 0, 145, 4095, 0, 0, 0, 3815, 75, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 389, 0, 0, 6, 0, 260, 4095, 0, 0, 0, 3421, 27, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4087, 65, 0, 0, 11, 0, 98, 4090, 0, 0, 0, 3921, 17, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 62, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 407, 0, 0, 0, 3804, 0, 0, 0, 0, 159, 3886, 0, 0, 0, 0, 0, 0, 0, 3424, 119, 279, 140, 0, 0, 0, 4094, 13, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4092, 4156, 4099, 4211, 4045, 3962, 4107] avg=4096.0000 max=4211 max_rank=4 min=3962 min_rank=6 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16325 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 2, 0, 4093, 0, 62, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 176, 0, 0, 15, 3804, 0, 0, 0, 0, 382, 3889, 0, 0, 0, 0, 0, 0, 0, 3212, 322, 279, 140, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4157, 4092, 3995, 4271, 3953, 4109] avg=4096.0000 max=4271 max_rank=5 min=3953 min_rank=6 imbalance_ratio=1.042725 max_load_reduction=28497 max_load_reduction_pct=0.869659 moved_to_echo=16328 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 62, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 163, 0, 0, 5, 3804, 1, 0, 0, 0, 59, 3888, 0, 0, 0, 0, 0, 0, 0, 3831, 47, 279, 140, 0, 0, 0, 4095, 13, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4097, 4157, 4094, 3973, 3947, 4297, 4108] avg=4096.0000 max=4297 max_rank=6 min=3947 min_rank=5 imbalance_ratio=1.049072 max_load_reduction=28471 max_load_reduction_pct=0.868866 moved_to_echo=16325 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2, 1, 4095, 0, 0, 0, 4094, 0, 0, 0, 3, 0, 0, 0, 6, 0, 3592, 1, 69, 0, 0, 0, 1, 1, 1, 4095, 10, 0, 0, 0, 659, 3, 1, 7, 3805, 0, 0, 0, 2, 399, 3857, 0, 0, 0, 0, 0, 1, 0, 3430, 189, 283, 143, 0, 0, 1, 4009, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4097, 3668, 4108, 4475, 4258, 4046, 4017] avg=4096.0000 max=4475 max_rank=4 min=3668 min_rank=2 imbalance_ratio=1.092529 max_load_reduction=28293 max_load_reduction_pct=0.863434 moved_to_echo=16796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 62, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 0, 389, 0, 0, 6, 3804, 0, 0, 0, 0, 260, 3889, 0, 0, 0, 0, 0, 0, 0, 3421, 27, 279, 140, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4158, 4093, 4199, 4149, 3867, 4109] avg=4096.0000 max=4199 max_rank=4 min=3867 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16324 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 62, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 64, 0, 0, 2, 3804, 1, 0, 0, 0, 145, 3889, 0, 0, 0, 0, 0, 0, 0, 3815, 75, 279, 140, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4097, 4157, 4096, 3871, 4034, 4309, 4109] avg=4096.0000 max=4309 max_rank=6 min=3871 min_rank=4 imbalance_ratio=1.052002 max_load_reduction=28459 max_load_reduction_pct=0.868500 moved_to_echo=16323 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 62, 0, 0, 0, 0, 0, 0, 4087, 3, 0, 0, 0, 65, 0, 0, 11, 3804, 1, 0, 0, 0, 98, 3885, 0, 0, 0, 0, 0, 0, 0, 3921, 17, 279, 140, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4157, 4090, 3881, 3983, 4357, 4109] avg=4096.0000 max=4357 max_rank=6 min=3881 min_rank=4 imbalance_ratio=1.063721 max_load_reduction=28411 max_load_reduction_pct=0.867035 moved_to_echo=16330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 62, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 359, 0, 0, 0, 3804, 0, 0, 0, 0, 195, 3886, 0, 0, 0, 0, 0, 0, 0, 3447, 111, 279, 140, 0, 0, 0, 4087, 13, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4158, 4096, 4163, 4081, 3977, 4100] avg=4096.0000 max=4163 max_rank=4 min=3977 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 28544, 1024, 2304, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2304, 128, 128, 128, 30464, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 1792, 31104, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32256, 128, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4095, 169, 0, 0, 0, 4091, 0, 0, 0, 0, 2, 3931, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4, 0, 0, 0, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 4082, 317, 1, 0, 0, 4092, 0, 0, 4, 1, 0, 3776, 1, 0, 4094, 1, 0, 4095, 0, 0, 4095, 1, 4095, 2, 13, 0, 1, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 59, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4048, 0, 0, 4096, 0, 0, 4096, 0, 0, 4091, 0, 4096, 0, 2, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4095, 87, 0, 0, 0, 4090, 0, 0, 0, 0, 1, 4014, 0, 0, 4096, 0, 0, 4096, 0, 0, 4092, 0, 4096, 0, 7, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4095, 84, 0, 0, 0, 4063, 0, 0, 0, 0, 1, 4071, 0, 0, 4096, 0, 0, 4096, 0, 0, 4047, 0, 4096, 0, 24, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 40, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4060, 0, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 88, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4012, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 4096, 0, 2, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 112, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 3987, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 3, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3979, 169, 0, 0, 0, 0, 0, 0, 0, 4091, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 3931, 0, 0, 107, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 1, 0, 0, 4090, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4099, 4096, 4040, 4096, 4103, 4094, 4092] avg=4096.0000 max=4148 max_rank=0 min=4040 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3979, 84, 0, 0, 0, 0, 0, 0, 0, 4063, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4071, 0, 0, 107, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 6, 1, 0, 0, 4090, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4071, 4096, 4179, 4096, 4054, 4114, 4095] avg=4096.0000 max=4179 max_rank=3 min=4054 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3980, 59, 0, 0, 0, 0, 0, 0, 0, 4088, 8, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4048, 0, 0, 107, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 6, 1, 0, 0, 4090, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4096, 4095, 4156, 4096, 4098, 4092, 4096] avg=4096.0000 max=4156 max_rank=3 min=4039 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3979, 87, 0, 0, 0, 0, 0, 0, 0, 4090, 8, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4014, 0, 0, 107, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 6, 1, 0, 0, 4090, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4098, 4095, 4123, 4096, 4099, 4097, 4094] avg=4096.0000 max=4123 max_rank=3 min=4066 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3954, 317, 0, 0, 0, 0, 1, 0, 0, 4092, 11, 0, 0, 0, 0, 0, 4, 1, 4095, 0, 0, 0, 0, 3776, 1, 0, 112, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 8, 5, 0, 0, 4081, 2, 13, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4272, 4104, 4100, 3889, 4095, 4109, 4096, 4103] avg=4096.0000 max=4272 max_rank=0 min=3889 min_rank=3 imbalance_ratio=1.042969 max_load_reduction=28496 max_load_reduction_pct=0.869629 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3980, 112, 0, 0, 0, 0, 0, 0, 0, 4090, 8, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3987, 0, 0, 107, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 1, 0, 0, 4090, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4098, 4095, 4094, 4097, 4103, 4093, 4096] avg=4096.0000 max=4103 max_rank=5 min=4092 min_rank=0 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3980, 40, 0, 0, 0, 0, 0, 0, 0, 4095, 8, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4060, 0, 0, 107, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 6, 1, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4103, 4095, 4168, 4096, 4100, 4090, 4096] avg=4096.0000 max=4168 max_rank=3 min=4020 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3980, 88, 0, 0, 0, 0, 0, 0, 0, 4091, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4012, 0, 0, 107, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 1, 0, 0, 4090, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4099, 4096, 4119, 4096, 4102, 4092, 4096] avg=4096.0000 max=4119 max_rank=3 min=4068 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 31872, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32000, 128, 0, 896, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4077, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 91, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[3762, 4091, 0, 0, 0, 4091, 4095, 0, 0, 1, 4092, 0, 4095, 1, 4057, 661, 4, 1, 1, 3778, 1, 0, 0, 1, 0, 1, 0, 29, 0, 1, 1, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4072, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4095, 0, 4096, 0, 4096, 169, 0, 0, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4064, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 98, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4084, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 136, 0, 0, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3986, 4096, 0, 0, 0, 4096, 4096, 1, 0, 0, 4095, 0, 4096, 0, 4096, 284, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4083, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 64, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4087, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 101, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4064, 70, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3901, 98, 0, 0, 0, 0, 0, 0, 0, 4020, 133, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 4026, 62, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4096, 4096, 3999, 4153, 4096, 4099, 4095] avg=4096.0000 max=4153 max_rank=4 min=3999 min_rank=3 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16443 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4072, 70, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3901, 169, 0, 0, 0, 0, 0, 0, 0, 3951, 133, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4026, 62, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4097, 4095, 4070, 4084, 4096, 4089, 4095] avg=4096.0000 max=4142 max_rank=0 min=4070 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3986, 70, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3901, 284, 0, 0, 0, 0, 0, 0, 0, 3919, 133, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4026, 62, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4097, 4095, 4185, 4052, 4096, 4091, 4096] avg=4096.0000 max=4185 max_rank=3 min=4052 min_rank=4 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16335 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4084, 70, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3901, 136, 0, 0, 0, 0, 0, 0, 0, 3962, 133, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 4026, 62, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4097, 4096, 4037, 4095, 4096, 4098, 4095] avg=4096.0000 max=4154 max_rank=0 min=4037 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3762, 63, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 5, 0, 0, 0, 0, 1, 3856, 661, 0, 0, 0, 0, 4, 1, 1, 3778, 132, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 29, 4028, 64, 1, 0, 0, 1, 1, 4, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[3825, 4091, 4098, 4518, 3916, 4097, 4123, 4100] avg=4096.0000 max=4518 max_rank=3 min=3825 min_rank=0 imbalance_ratio=1.103027 max_load_reduction=28250 max_load_reduction_pct=0.862122 moved_to_echo=16236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4077, 70, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3901, 91, 0, 0, 0, 0, 0, 0, 0, 4024, 133, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4026, 62, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4096, 4096, 3992, 4157, 4096, 4088, 4096] avg=4096.0000 max=4157 max_rank=4 min=3992 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4083, 70, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3901, 64, 0, 0, 0, 0, 0, 0, 0, 4043, 133, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4026, 62, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4097, 4096, 3965, 4176, 4096, 4090, 4095] avg=4096.0000 max=4176 max_rank=4 min=3965 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4087, 70, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3901, 101, 0, 0, 0, 0, 0, 0, 0, 4002, 133, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 2, 4026, 62, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4097, 4096, 4002, 4135, 4095, 4090, 4096] avg=4096.0000 max=4157 max_rank=0 min=4002 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16416 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32256, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 31232, 1664, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 31744, 1152, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 32256, 512, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1561, 4092, 0, 2431, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 4096, 4096, 113, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1378, 4090, 0, 2682, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 42, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 757, 4096, 0, 3225, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 114, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1337, 4095, 0, 2741, 0, 0, 4094, 0, 0, 4095, 0, 0, 0, 4096, 1, 1, 4096, 4096, 20, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[9, 1, 2, 1, 1, 0, 0, 848, 4085, 2, 3158, 0, 1, 4077, 0, 0, 4095, 0, 0, 1, 4094, 0, 0, 4024, 4093, 132, 0, 54, 1, 4087, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1062, 4095, 0, 3018, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 4096, 4096, 21, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1674, 4095, 0, 2396, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 30, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 555, 4093, 0, 3538, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 7, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 6, 0, 0, 0, 0, 0, 1561, 2888, 60, 0, 0, 1198, 0, 2431, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 4096, 0, 0, 0, 0, 4032, 113, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4509, 3629, 4093, 4096, 4107, 4145, 4096] avg=4096.0000 max=4509 max_rank=1 min=3629 min_rank=2 imbalance_ratio=1.100830 max_load_reduction=28259 max_load_reduction_pct=0.862396 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 7, 0, 0, 0, 0, 0, 757, 2890, 60, 0, 0, 1199, 0, 3225, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 4032, 114, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 3707, 4424, 4100, 4096, 4105, 4146, 4096] avg=4096.0000 max=4424 max_rank=2 min=3707 min_rank=1 imbalance_ratio=1.080078 max_load_reduction=28344 max_load_reduction_pct=0.864990 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 7, 0, 0, 0, 0, 0, 1337, 2890, 60, 0, 0, 1198, 0, 2741, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 9, 1, 1, 4096, 0, 0, 0, 0, 4032, 20, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4287, 3939, 4098, 4095, 4107, 4052, 4096] avg=4096.0000 max=4287 max_rank=1 min=3939 min_rank=2 imbalance_ratio=1.046631 max_load_reduction=28481 max_load_reduction_pct=0.869171 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 7, 0, 0, 0, 0, 0, 1062, 2890, 60, 0, 0, 1198, 0, 3018, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1, 4096, 0, 0, 0, 0, 4032, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4012, 4216, 4095, 4096, 4106, 4053, 4096] avg=4096.0000 max=4216 max_rank=2 min=4012 min_rank=1 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[9, 1, 2, 1, 4090, 9, 0, 0, 1, 0, 0, 848, 2886, 67, 0, 0, 1190, 2, 3158, 0, 0, 0, 0, 0, 1, 4077, 0, 0, 8, 0, 0, 0, 4095, 0, 0, 1, 1, 0, 0, 0, 4, 0, 0, 4024, 0, 0, 0, 0, 4011, 132, 0, 54, 0, 0, 0, 0, 1, 4087, 1, 1, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 3802, 4350, 4086, 4097, 4028, 4197, 4096] avg=4096.0000 max=4350 max_rank=2 min=3802 min_rank=1 imbalance_ratio=1.062012 max_load_reduction=28418 max_load_reduction_pct=0.867249 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 7, 0, 0, 0, 0, 0, 555, 2888, 60, 0, 0, 1198, 0, 3538, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 4032, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 3503, 4736, 4099, 4096, 4105, 4039, 4096] avg=4096.0000 max=4736 max_rank=2 min=3503 min_rank=1 imbalance_ratio=1.156250 max_load_reduction=28032 max_load_reduction_pct=0.855469 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 6, 0, 0, 0, 0, 0, 1378, 2886, 60, 0, 0, 1198, 0, 2682, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 4032, 42, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4324, 3880, 4100, 4096, 4105, 4074, 4096] avg=4096.0000 max=4324 max_rank=1 min=3880 min_rank=2 imbalance_ratio=1.055664 max_load_reduction=28444 max_load_reduction_pct=0.868042 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 7, 0, 0, 0, 0, 0, 1674, 2890, 60, 0, 0, 1198, 0, 2396, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 4032, 30, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4624, 3594, 4097, 4096, 4105, 4062, 4096] avg=4096.0000 max=4624 max_rank=1 min=3594 min_rank=2 imbalance_ratio=1.128906 max_load_reduction=28144 max_load_reduction_pct=0.858887 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 9216, 23168, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9600, 128, 23296, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32256, 512, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 2948, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 1141, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 3709, 0, 0, 0, 0, 0, 4094, 4087, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 1, 394, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 3549, 0, 0, 0, 0, 0, 4088, 4093, 1, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 543, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 3260, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 4096, 0, 824, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 3850, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 245, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 2676, 1, 1, 0, 6, 1, 4095, 4078, 0, 0, 0, 0, 0, 1, 4093, 4095, 4088, 1, 1, 40, 2, 2, 0, 0, 1, 4092, 0, 1391, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 3494, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 601, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 3703, 0, 0, 0, 0, 0, 4093, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 396, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[698, 0, 3549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 543, 0, 5, 3396, 6, 0, 0] rerouted_tokens_per_ep_rank=[4247, 4095, 4097, 4096, 4097, 4090, 4096, 3950] avg=4096.0000 max=4247 max_rank=0 min=3950 min_rank=7 imbalance_ratio=1.036865 max_load_reduction=28521 max_load_reduction_pct=0.870392 moved_to_echo=16233 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[698, 0, 2948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1141, 0, 0, 3397, 4, 0, 0] rerouted_tokens_per_ep_rank=[3646, 4095, 4100, 4096, 4097, 4096, 4096, 4542] avg=4096.0000 max=4542 max_rank=7 min=3646 min_rank=0 imbalance_ratio=1.108887 max_load_reduction=28226 max_load_reduction_pct=0.861389 moved_to_echo=16831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[698, 0, 3709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 6, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 394, 0, 2, 3396, 4, 0, 0] rerouted_tokens_per_ep_rank=[4407, 4095, 4093, 4096, 4097, 4087, 4096, 3797] avg=4096.0000 max=4407 max_rank=0 min=3797 min_rank=7 imbalance_ratio=1.075928 max_load_reduction=28361 max_load_reduction_pct=0.865509 moved_to_echo=16077 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[698, 0, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 6, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 245, 0, 0, 3398, 4, 0, 0] rerouted_tokens_per_ep_rank=[4548, 4095, 4099, 4096, 4097, 4090, 4096, 3647] avg=4096.0000 max=4548 max_rank=0 min=3647 min_rank=7 imbalance_ratio=1.110352 max_load_reduction=28220 max_load_reduction_pct=0.861206 moved_to_echo=15930 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[698, 0, 3260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 6, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 824, 0, 0, 3398, 4, 0, 0] rerouted_tokens_per_ep_rank=[3958, 4095, 4101, 4096, 4097, 4099, 4096, 4226] avg=4096.0000 max=4226 max_rank=7 min=3958 min_rank=0 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[692, 0, 2676, 1, 0, 0, 0, 0, 1, 0, 6, 1, 4093, 2, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 1, 40, 2, 2, 4089, 0, 0, 0, 0, 0, 1, 4092, 3, 0, 0, 0, 0, 1391, 8, 0, 3400, 4, 0, 0] rerouted_tokens_per_ep_rank=[3369, 4103, 4078, 4096, 4089, 4134, 4096, 4803] avg=4096.0000 max=4803 max_rank=7 min=3369 min_rank=0 imbalance_ratio=1.172607 max_load_reduction=27965 max_load_reduction_pct=0.853424 moved_to_echo=17122 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[698, 0, 3494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 601, 0, 1, 3398, 4, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4095, 4102, 4096, 4097, 4086, 4096, 4004] avg=4096.0000 max=4192 max_rank=0 min=4004 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[698, 0, 3703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 6, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 396, 0, 2, 3397, 4, 0, 0] rerouted_tokens_per_ep_rank=[4401, 4095, 4098, 4096, 4097, 4086, 4096, 3799] avg=4096.0000 max=4401 max_rank=0 min=3799 min_rank=7 imbalance_ratio=1.074463 max_load_reduction=28367 max_load_reduction_pct=0.865692 moved_to_echo=16078 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5632, 0, 27264, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 5632, 128, 128, 27264, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3963, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 133, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3965, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 131, 0, 0, 3, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3886, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 210, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3975, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 124, 0, 0, 0, 0, 4093, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3922, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 174, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4093, 1, 3, 55, 1, 0, 4094, 1, 0, 1, 1, 4005, 2, 1, 0, 4094, 0, 20, 4095, 1, 0, 1, 4095, 384, 0, 0, 20, 1, 3705, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3967, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 129, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4023, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 73, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3965, 0, 0, 0, 123, 6, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 131, 0, 0, 3, 3923, 0, 0, 0, 0, 4096, 0, 50, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4090, 4096, 4096, 4094, 4096, 4057, 4146] avg=4096.0000 max=4146 max_rank=7 min=4057 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3963, 0, 0, 0, 123, 6, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 133, 0, 0, 0, 3923, 0, 0, 0, 0, 4096, 0, 50, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4090, 4096, 4094, 4094, 4096, 4056, 4146] avg=4096.0000 max=4146 max_rank=7 min=4056 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3886, 0, 0, 0, 123, 6, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 210, 0, 0, 1, 3923, 0, 0, 0, 0, 4096, 0, 50, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4090, 4096, 4016, 4094, 4096, 4134, 4146] avg=4096.0000 max=4146 max_rank=7 min=4016 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16470 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3922, 0, 0, 0, 123, 6, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 174, 0, 0, 0, 3923, 1, 0, 0, 0, 4096, 0, 50, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4090, 4095, 4053, 4094, 4096, 4098, 4146] avg=4096.0000 max=4146 max_rank=7 min=4053 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16434 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3967, 0, 0, 0, 123, 6, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 129, 0, 0, 0, 3923, 0, 0, 0, 0, 4096, 0, 50, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4090, 4096, 4098, 4094, 4096, 4052, 4146] avg=4096.0000 max=4146 max_rank=7 min=4052 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 1, 3, 2, 1, 0, 0, 55, 1, 0, 4082, 0, 0, 0, 0, 1, 0, 1, 1, 4094, 0, 0, 0, 4005, 2, 1, 0, 128, 12, 3, 0, 0, 0, 20, 4090, 0, 0, 0, 0, 1, 0, 1, 4094, 0, 0, 0, 0, 384, 0, 0, 20, 3923, 0, 0, 0, 1, 3705, 0, 43, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4138, 4097, 4151, 4110, 4096, 4327, 3749] avg=4096.0000 max=4327 max_rank=6 min=3749 min_rank=7 imbalance_ratio=1.056396 max_load_reduction=28441 max_load_reduction_pct=0.867950 moved_to_echo=16282 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3975, 0, 0, 0, 123, 6, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 124, 0, 0, 0, 3923, 0, 0, 0, 0, 4093, 0, 50, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4090, 4096, 4106, 4094, 4096, 4047, 4143] avg=4096.0000 max=4143 max_rank=7 min=4047 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4023, 0, 0, 0, 123, 6, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 73, 0, 0, 1, 3923, 0, 0, 0, 0, 4096, 0, 50, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4090, 4096, 4154, 4094, 4096, 3997, 4146] avg=4096.0000 max=4154 max_rank=3 min=3997 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1408, 0, 0, 128, 31488, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[31744, 128, 128, 0, 1024, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32384, 0, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4091, 4095, 4095, 4079, 1, 2, 12, 4095, 1, 0, 0, 50, 1, 4051, 0, 0, 0, 1, 0, 0, 0, 1, 4094, 0, 0, 0, 4095, 1, 0, 2, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4093, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4091, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4086, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4094, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4087, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4095, 4087, 4100, 4096, 4096, 4096, 4096] avg=4096.0000 max=4102 max_rank=0 min=4087 min_rank=2 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 10, 4088, 1, 0, 0, 0, 4096, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4095, 4099, 4105, 4096, 4096, 4096, 4095] avg=4096.0000 max=4105 max_rank=3 min=4086 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 0, 0, 0, 0, 1, 2, 12, 4088, 0, 0, 0, 0, 1, 0, 0, 50, 4076, 1, 0, 0, 1, 4051, 0, 0, 3, 6, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 2, 0, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4103, 4128, 4061, 4096, 4096, 4096, 4097] avg=4096.0000 max=4128 max_rank=2 min=4061 min_rank=3 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 5, 4087, 0, 0, 0, 0, 4096, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4095, 4092, 4100, 4096, 4096, 4096, 4096] avg=4096.0000 max=4100 max_rank=3 min=4092 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4095, 4087, 4102, 4096, 4096, 4096, 4096] avg=4096.0000 max=4102 max_rank=3 min=4087 min_rank=2 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3, 4087, 0, 0, 0, 0, 4096, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4095, 4090, 4100, 4096, 4096, 4096, 4096] avg=4096.0000 max=4100 max_rank=3 min=4090 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 4087, 0, 0, 0, 0, 4096, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4095, 4089, 4100, 4096, 4096, 4096, 4096] avg=4096.0000 max=4100 max_rank=0 min=4089 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 9, 4087, 0, 0, 0, 0, 4096, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4095, 4096, 4100, 4096, 4096, 4096, 4096] avg=4096.0000 max=4100 max_rank=3 min=4093 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True + [2026-08-30 04:56:50] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 654.3 | throughput per GPU (TFLOP/s/GPU): 509.5 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.956200E+00 | loss scale: 1.0 | grad norm: 5.277 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2721, 3, 923, 736, 1364, 0, 1288, 930, 1479, 1558, 2, 95, 856, 437, 493, 2780, 80, 1954, 621, 2, 0, 1921, 3179, 2289, 1, 864, 1464, 1372, 1118, 1870, 368, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3240, 1, 1361, 801, 895, 0, 1396, 973, 1298, 2567, 0, 93, 779, 1038, 611, 2054, 110, 1861, 701, 0, 2, 2278, 2257, 1872, 0, 941, 1216, 1039, 828, 2256, 300, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2471, 0, 1550, 764, 959, 0, 1763, 795, 1433, 2001, 0, 99, 1291, 774, 455, 2676, 72, 1938, 953, 0, 0, 2045, 2011, 1922, 2, 928, 1047, 1109, 1041, 1890, 779, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2547, 0, 1446, 659, 797, 4, 1459, 718, 1065, 2857, 0, 109, 1138, 1072, 788, 2148, 60, 1426, 618, 0, 1, 2066, 1855, 1997, 0, 940, 1058, 1322, 1470, 2624, 523, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2685, 0, 1446, 814, 1418, 1, 1023, 647, 1260, 2162, 0, 84, 1153, 522, 685, 2467, 72, 1614, 1020, 0, 0, 1996, 2195, 2514, 0, 1140, 1261, 887, 1037, 2131, 534, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2993, 0, 1711, 728, 594, 0, 1148, 667, 1002, 2490, 1, 51, 693, 1292, 458, 2436, 168, 1894, 683, 1, 2, 2276, 2637, 1627, 10, 907, 1356, 1124, 734, 2766, 319, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2839, 5, 1302, 731, 1218, 0, 982, 789, 1020, 2514, 0, 230, 1213, 1104, 1338, 2191, 66, 1901, 478, 0, 0, 1950, 2227, 2204, 0, 644, 1246, 869, 670, 2531, 499, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2756, 0, 1231, 773, 939, 9, 896, 1443, 1113, 2026, 0, 182, 1280, 632, 879, 2583, 104, 1480, 1128, 0, 4, 1640, 2644, 2508, 0, 807, 1097, 935, 1251, 1974, 450, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1931, 3, 923, 736, 0, 0, 0, 0, 1364, 0, 1288, 930, 1212, 59, 0, 0, 1479, 1558, 2, 95, 486, 0, 0, 0, 856, 437, 493, 1695, 0, 0, 0, 0, 80, 1954, 621, 2, 1967, 0, 0, 0, 0, 1921, 0, 2243, 0, 0, 0, 0, 1, 864, 1464, 1372, 1026, 5, 0, 0, 1118, 1870, 368, 0, 299, 46, 0, 0] rerouted_tokens_per_ep_rank=[3593, 4853, 3620, 3481, 4624, 4164, 4732, 3701] avg=4096.0000 max=4853 max_rank=1 min=3481 min_rank=3 imbalance_ratio=1.184814 max_load_reduction=27915 max_load_reduction_pct=0.851898 moved_to_echo=17221 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2299, 1, 1361, 801, 0, 0, 0, 0, 895, 0, 1396, 973, 861, 43, 0, 0, 1298, 2567, 0, 93, 579, 0, 0, 0, 779, 1038, 611, 1253, 0, 0, 0, 0, 110, 1861, 701, 0, 1396, 0, 0, 0, 2, 2278, 0, 1834, 0, 0, 0, 0, 0, 941, 1216, 1039, 758, 6, 0, 0, 828, 2256, 300, 0, 356, 38, 0, 0] rerouted_tokens_per_ep_rank=[4462, 4168, 4537, 3681, 4068, 4114, 3960, 3778] avg=4096.0000 max=4537 max_rank=2 min=3681 min_rank=3 imbalance_ratio=1.107666 max_load_reduction=28231 max_load_reduction_pct=0.861542 moved_to_echo=15920 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1906, 0, 1446, 814, 0, 0, 0, 0, 1418, 1, 1023, 647, 836, 52, 0, 0, 1260, 2162, 0, 84, 479, 0, 0, 0, 1153, 522, 685, 1505, 0, 0, 0, 0, 72, 1614, 1020, 0, 1359, 0, 0, 0, 0, 1996, 0, 2463, 0, 0, 0, 0, 0, 1140, 1261, 887, 910, 5, 0, 0, 1037, 2131, 534, 0, 295, 51, 0, 0] rerouted_tokens_per_ep_rank=[4166, 3977, 3985, 3865, 4065, 4459, 4203, 4048] avg=4096.0000 max=4459 max_rank=5 min=3865 min_rank=3 imbalance_ratio=1.088623 max_load_reduction=28309 max_load_reduction_pct=0.863922 moved_to_echo=16775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1956, 0, 1231, 773, 0, 0, 0, 0, 939, 9, 896, 1443, 1007, 54, 0, 0, 1113, 2026, 0, 182, 492, 0, 0, 0, 1280, 632, 879, 1576, 0, 0, 0, 0, 104, 1480, 1128, 0, 1637, 0, 0, 0, 4, 1640, 0, 2457, 0, 0, 0, 0, 0, 807, 1097, 935, 953, 5, 0, 0, 1251, 1974, 450, 4, 303, 51, 0, 0] rerouted_tokens_per_ep_rank=[3960, 4348, 3813, 4367, 4349, 4101, 3797, 4033] avg=4096.0000 max=4367 max_rank=3 min=3797 min_rank=6 imbalance_ratio=1.066162 max_load_reduction=28401 max_load_reduction_pct=0.866730 moved_to_echo=16280 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1807, 0, 1446, 659, 0, 0, 0, 0, 797, 4, 1459, 718, 706, 45, 0, 0, 1065, 2857, 0, 109, 455, 0, 0, 0, 1138, 1072, 788, 1311, 0, 0, 0, 0, 60, 1426, 618, 0, 1149, 0, 0, 0, 1, 2066, 0, 1957, 0, 0, 0, 0, 0, 940, 1058, 1322, 792, 5, 0, 0, 1470, 2624, 523, 1, 280, 40, 0, 0] rerouted_tokens_per_ep_rank=[3912, 3729, 4486, 4309, 3253, 4024, 4117, 4938] avg=4096.0000 max=4938 max_rank=7 min=3253 min_rank=4 imbalance_ratio=1.205566 max_load_reduction=27830 max_load_reduction_pct=0.849304 moved_to_echo=16332 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2114, 0, 1711, 728, 0, 0, 0, 0, 594, 0, 1148, 667, 1005, 56, 0, 0, 1002, 2490, 1, 51, 538, 0, 0, 0, 693, 1292, 458, 1477, 0, 0, 0, 0, 168, 1894, 683, 1, 1632, 0, 0, 0, 2, 2276, 0, 1591, 0, 0, 0, 0, 10, 907, 1356, 1124, 903, 10, 0, 0, 734, 2766, 319, 0, 331, 36, 0, 0] rerouted_tokens_per_ep_rank=[4553, 3470, 4082, 3920, 4378, 3869, 4310, 4186] avg=4096.0000 max=4553 max_rank=0 min=3470 min_rank=1 imbalance_ratio=1.111572 max_load_reduction=28215 max_load_reduction_pct=0.861053 moved_to_echo=16743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1755, 0, 1550, 764, 0, 0, 0, 0, 959, 0, 1763, 795, 766, 56, 0, 0, 1433, 2001, 0, 99, 441, 0, 0, 0, 1291, 774, 455, 1633, 0, 0, 0, 0, 72, 1938, 953, 0, 1245, 0, 0, 0, 0, 2045, 0, 1883, 0, 0, 0, 0, 2, 928, 1047, 1109, 987, 4, 0, 0, 1041, 1890, 779, 0, 271, 39, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4339, 3974, 4153, 4208, 3928, 4077, 4020] avg=4096.0000 max=4339 max_rank=1 min=3928 min_rank=5 imbalance_ratio=1.059326 max_load_reduction=28429 max_load_reduction_pct=0.867584 moved_to_echo=16233 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2015, 5, 1302, 731, 0, 0, 0, 0, 1218, 0, 982, 789, 849, 46, 0, 0, 1020, 2514, 0, 230, 507, 0, 0, 0, 1213, 1104, 1338, 1337, 0, 0, 0, 0, 66, 1901, 478, 0, 1378, 0, 0, 0, 0, 1950, 0, 2159, 0, 0, 0, 0, 0, 644, 1246, 869, 808, 5, 0, 0, 670, 2531, 499, 7, 312, 45, 0, 0] rerouted_tokens_per_ep_rank=[4053, 3884, 4271, 4992, 3823, 4109, 3572, 4064] avg=4096.0000 max=4992 max_rank=3 min=3572 min_rank=6 imbalance_ratio=1.218750 max_load_reduction=27776 max_load_reduction_pct=0.847656 moved_to_echo=15568 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 128, 9984, 7040, 7296, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 7296, 9856, 8704, 7168, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 6912, 5760, 11904, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9728, 18176, 128, 1024, 4096, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 14080, 6272, 128, 11776, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 16256, 0, 16640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15872, 128, 11008, 6016, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 18048, 3840, 128, 2560, 384, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3089, 0, 0, 4, 4064, 3, 3, 1, 0, 2747, 4088, 4092, 3984, 0, 4, 60, 4093, 0, 0, 3, 2330, 0, 0, 1, 4081, 26, 2, 0, 0, 90, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1289, 0, 0, 1, 4021, 0, 0, 0, 1, 2915, 4095, 4096, 3957, 1, 0, 29, 4096, 0, 0, 0, 3769, 0, 0, 0, 4069, 172, 0, 0, 0, 257, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[962, 0, 0, 0, 4042, 0, 0, 1, 0, 3309, 4094, 4093, 4005, 12, 1, 94, 4096, 0, 0, 0, 3761, 1, 0, 0, 4081, 120, 0, 0, 0, 95, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2619, 0, 0, 2, 3965, 0, 0, 0, 3, 2979, 4096, 4093, 3804, 18, 1, 163, 4095, 0, 0, 0, 2675, 0, 0, 0, 4093, 67, 0, 0, 0, 95, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[3394, 0, 0, 18, 4096, 0, 0, 0, 0, 2723, 4096, 4096, 4090, 1, 3, 1, 4092, 0, 0, 0, 2070, 1, 0, 0, 4006, 6, 0, 0, 0, 75, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[804, 0, 3, 1, 4066, 0, 0, 0, 1, 3355, 4090, 4096, 4074, 8, 1, 51, 4096, 0, 0, 0, 3902, 0, 0, 0, 4094, 11, 0, 0, 0, 115, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[800, 6, 12, 0, 3492, 0, 5, 0, 1, 3337, 4065, 4088, 4034, 19, 0, 78, 4090, 2, 3, 1, 4070, 0, 0, 0, 4096, 61, 10, 7, 0, 481, 2, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[658, 2, 3, 0, 3912, 2, 2, 0, 0, 2885, 4010, 4093, 3860, 46, 0, 660, 4094, 0, 1, 2, 3835, 3, 0, 1, 4077, 466, 0, 3, 0, 150, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3089, 0, 0, 4, 2223, 169, 0, 0, 4064, 3, 3, 1, 97, 35, 3, 0, 0, 2747, 1067, 0, 0, 0, 0, 0, 3949, 0, 4, 60, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 2330, 0, 0, 1, 795, 0, 0, 0, 3984, 26, 2, 0, 0, 0, 0, 0, 0, 90, 3, 0, 3923, 0, 0, 0] rerouted_tokens_per_ep_rank=[5485, 4206, 3814, 4013, 4096, 3126, 4012, 4016] avg=4096.0000 max=5485 max_rank=0 min=3126 min_rank=5 imbalance_ratio=1.339111 max_load_reduction=27283 max_load_reduction_pct=0.832611 moved_to_echo=15250 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1289, 0, 0, 1, 2226, 169, 0, 0, 4021, 0, 0, 0, 97, 35, 3, 0, 1, 2915, 1070, 0, 0, 0, 0, 0, 3922, 1, 0, 29, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3769, 0, 0, 0, 796, 0, 0, 0, 3972, 172, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 3927, 0, 0, 0] rerouted_tokens_per_ep_rank=[3685, 4156, 3986, 3952, 4096, 4565, 4144, 4184] avg=4096.0000 max=4565 max_rank=5 min=3685 min_rank=0 imbalance_ratio=1.114502 max_load_reduction=28203 max_load_reduction_pct=0.860687 moved_to_echo=16989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[962, 0, 0, 0, 2226, 169, 0, 0, 4042, 0, 0, 1, 97, 36, 3, 0, 0, 3309, 1069, 0, 0, 0, 0, 0, 3969, 12, 1, 94, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3761, 1, 0, 0, 796, 0, 0, 0, 3984, 120, 0, 0, 0, 0, 0, 0, 0, 95, 0, 1, 3924, 0, 0, 0] rerouted_tokens_per_ep_rank=[3357, 4179, 4378, 4076, 4096, 4558, 4104, 4020] avg=4096.0000 max=4558 max_rank=5 min=3357 min_rank=0 imbalance_ratio=1.112793 max_load_reduction=28210 max_load_reduction_pct=0.860901 moved_to_echo=16778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[658, 2, 3, 0, 2180, 170, 0, 0, 3912, 2, 2, 0, 97, 34, 3, 0, 0, 2885, 1047, 0, 0, 0, 0, 0, 3826, 46, 0, 660, 0, 0, 0, 0, 4094, 0, 1, 2, 0, 0, 0, 0, 3835, 3, 0, 1, 780, 0, 0, 0, 3980, 466, 0, 3, 0, 0, 0, 0, 0, 150, 3, 0, 3923, 0, 0, 0] rerouted_tokens_per_ep_rank=[3013, 4050, 3932, 4532, 4097, 4619, 4449, 4076] avg=4096.0000 max=4619 max_rank=5 min=3013 min_rank=0 imbalance_ratio=1.127686 max_load_reduction=28149 max_load_reduction_pct=0.859039 moved_to_echo=17241 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[804, 0, 3, 1, 2224, 169, 0, 0, 4066, 0, 0, 0, 97, 36, 3, 0, 1, 3355, 1068, 0, 0, 0, 0, 0, 4038, 8, 1, 51, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 0, 0, 795, 0, 0, 0, 3997, 11, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 3927, 0, 0, 0] rerouted_tokens_per_ep_rank=[3201, 4202, 4424, 4098, 4096, 4697, 4008, 4042] avg=4096.0000 max=4697 max_rank=5 min=3201 min_rank=0 imbalance_ratio=1.146729 max_load_reduction=28071 max_load_reduction_pct=0.856659 moved_to_echo=16843 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[800, 6, 12, 0, 2210, 169, 0, 0, 3492, 0, 5, 0, 98, 36, 3, 0, 1, 3337, 1062, 0, 0, 0, 0, 0, 3998, 19, 0, 78, 0, 0, 0, 0, 4090, 2, 3, 1, 0, 0, 0, 0, 4070, 0, 0, 0, 790, 0, 0, 0, 3998, 61, 10, 7, 0, 0, 0, 0, 0, 481, 2, 8, 3919, 0, 0, 0] rerouted_tokens_per_ep_rank=[3197, 3634, 4400, 4095, 4096, 4860, 4076, 4410] avg=4096.0000 max=4860 max_rank=5 min=3197 min_rank=0 imbalance_ratio=1.186523 max_load_reduction=27908 max_load_reduction_pct=0.851685 moved_to_echo=17442 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3394, 0, 0, 18, 2227, 169, 0, 0, 4096, 0, 0, 0, 95, 36, 3, 0, 0, 2723, 1070, 0, 0, 0, 0, 0, 4054, 1, 3, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 2070, 1, 0, 0, 796, 0, 0, 0, 3911, 6, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 3927, 0, 0, 0] rerouted_tokens_per_ep_rank=[5808, 4230, 3793, 4059, 4092, 2867, 3917, 4002] avg=4096.0000 max=5808 max_rank=0 min=2867 min_rank=5 imbalance_ratio=1.417969 max_load_reduction=26960 max_load_reduction_pct=0.822754 moved_to_echo=14878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2619, 0, 0, 2, 2231, 170, 0, 0, 3965, 0, 0, 0, 102, 39, 5, 0, 3, 2979, 1059, 0, 0, 0, 0, 0, 3761, 18, 1, 163, 0, 0, 0, 0, 4095, 0, 0, 0, 4, 0, 0, 0, 2675, 0, 0, 0, 801, 0, 0, 0, 3991, 67, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 3923, 0, 0, 0] rerouted_tokens_per_ep_rank=[5022, 4111, 4041, 3943, 4099, 3476, 4058, 4018] avg=4096.0000 max=5022 max_rank=0 min=3476 min_rank=5 imbalance_ratio=1.226074 max_load_reduction=27746 max_load_reduction_pct=0.846741 moved_to_echo=15651 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[13696, 128, 128, 128, 17792, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 24320, 8576, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[26496, 128, 0, 128, 6400, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31616, 128, 128, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[31872, 1024, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[31744, 128, 128, 128, 896, 384, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 1408, 128, 128, 31488, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1546, 1, 4095, 1, 1699, 535, 80, 0, 4092, 1, 4091, 0, 4096, 5, 0, 0, 4, 0, 0, 188, 4092, 3681, 0, 295, 599, 0, 8, 3506, 0, 153] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1085, 0, 4052, 0, 2104, 278, 53, 0, 4080, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 5, 4096, 3507, 0, 415, 1318, 0, 0, 3199, 0, 386] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 1, 1125, 0, 3998, 1, 1864, 324, 17, 0, 4093, 0, 4095, 0, 4077, 1, 0, 0, 0, 0, 0, 27, 4095, 2870, 0, 368, 2062, 0, 0, 2595, 0, 1154] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 232, 2, 4056, 4, 1503, 243, 64, 3, 4060, 2, 4094, 2, 4094, 0, 2, 2, 0, 3, 2, 10, 4094, 3689, 9, 726, 2687, 10, 0, 2422, 0, 753] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 1063, 0, 4042, 1, 2776, 147, 8, 6, 4088, 0, 4094, 0, 4086, 0, 0, 0, 0, 0, 0, 14, 4094, 3714, 3, 298, 709, 0, 0, 3294, 0, 329] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 1403, 2, 4047, 0, 1764, 25, 16, 0, 4085, 0, 4088, 2, 4096, 0, 9, 7, 0, 8, 0, 1, 4085, 1798, 23, 341, 1600, 0, 29, 3284, 0, 2054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 1743, 0, 4093, 0, 947, 399, 119, 4, 4087, 0, 4091, 0, 4085, 0, 0, 0, 1, 0, 0, 22, 4094, 3773, 0, 391, 1152, 0, 0, 3534, 0, 231] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 2669, 0, 4090, 0, 758, 305, 100, 9, 4090, 0, 4085, 0, 4065, 2, 0, 0, 0, 0, 0, 7, 4096, 3798, 0, 260, 935, 0, 0, 3488, 0, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 232, 2, 2736, 0, 0, 0, 2136, 4, 1503, 243, 0, 0, 0, 0, 64, 3, 4012, 2, 0, 0, 0, 0, 8, 2, 4094, 0, 0, 0, 0, 0, 2, 2, 0, 3, 4086, 7, 0, 0, 2, 10, 710, 3689, 0, 0, 0, 0, 9, 726, 2687, 10, 1677, 641, 0, 0, 0, 2422, 0, 753, 243, 48, 0, 0] rerouted_tokens_per_ep_rank=[2970, 3886, 4081, 4104, 4100, 4411, 5750, 3466] avg=4096.0000 max=5750 max_rank=6 min=2970 min_rank=0 imbalance_ratio=1.403809 max_load_reduction=27018 max_load_reduction_pct=0.824524 moved_to_echo=17727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1063, 0, 2736, 0, 0, 0, 2128, 1, 2776, 147, 0, 0, 0, 0, 8, 6, 4040, 0, 0, 0, 0, 0, 8, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 7, 0, 0, 0, 14, 710, 3714, 0, 0, 0, 0, 3, 298, 709, 0, 1672, 641, 0, 0, 0, 3294, 0, 329, 242, 48, 0, 0] rerouted_tokens_per_ep_rank=[3801, 5052, 4054, 4094, 4093, 4438, 3323, 3913] avg=4096.0000 max=5052 max_rank=1 min=3323 min_rank=6 imbalance_ratio=1.233398 max_load_reduction=27716 max_load_reduction_pct=0.845825 moved_to_echo=15767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1546, 1, 2735, 0, 0, 0, 2157, 1, 1699, 535, 0, 0, 0, 0, 80, 0, 4044, 1, 0, 0, 0, 0, 8, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4, 0, 4083, 7, 0, 0, 0, 188, 709, 3681, 0, 0, 0, 0, 0, 295, 599, 0, 1693, 641, 0, 0, 8, 3506, 0, 153, 245, 48, 0, 0] rerouted_tokens_per_ep_rank=[4282, 4392, 4125, 4109, 4094, 4578, 3228, 3960] avg=4096.0000 max=4578 max_rank=5 min=3228 min_rank=6 imbalance_ratio=1.117676 max_load_reduction=28190 max_load_reduction_pct=0.860291 moved_to_echo=15860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1125, 0, 2737, 0, 0, 0, 2105, 1, 1864, 324, 0, 0, 0, 0, 17, 0, 4045, 0, 0, 0, 0, 0, 8, 0, 4077, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 7, 0, 0, 0, 27, 709, 2870, 0, 0, 0, 0, 0, 368, 2062, 0, 1653, 642, 0, 0, 0, 2595, 0, 1154, 240, 48, 0, 0] rerouted_tokens_per_ep_rank=[3864, 4294, 4062, 4086, 4094, 3606, 4725, 4037] avg=4096.0000 max=4725 max_rank=6 min=3606 min_rank=5 imbalance_ratio=1.153564 max_load_reduction=28043 max_load_reduction_pct=0.855804 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1085, 0, 2737, 0, 0, 0, 2133, 0, 2104, 278, 0, 0, 0, 0, 53, 0, 4032, 0, 0, 0, 0, 0, 8, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 0, 5, 710, 3507, 0, 0, 0, 0, 0, 415, 1318, 0, 1676, 642, 0, 0, 0, 3199, 0, 386, 243, 48, 0, 0] rerouted_tokens_per_ep_rank=[3822, 4515, 4085, 4102, 4095, 4222, 4051, 3876] avg=4096.0000 max=4515 max_rank=1 min=3822 min_rank=0 imbalance_ratio=1.102295 max_load_reduction=28253 max_load_reduction_pct=0.862213 moved_to_echo=16244 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1743, 0, 2741, 0, 0, 0, 2147, 0, 947, 399, 0, 0, 0, 0, 119, 4, 4034, 0, 0, 0, 0, 0, 7, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4084, 12, 0, 0, 0, 22, 696, 3773, 0, 0, 0, 0, 0, 391, 1152, 0, 1697, 645, 0, 0, 0, 3534, 0, 231, 249, 53, 0, 0] rerouted_tokens_per_ep_rank=[4486, 3493, 4157, 4092, 4097, 4491, 3885, 4067] avg=4096.0000 max=4491 max_rank=5 min=3493 min_rank=1 imbalance_ratio=1.096436 max_load_reduction=28277 max_load_reduction_pct=0.862946 moved_to_echo=16540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1403, 2, 2730, 0, 0, 0, 2130, 0, 1764, 25, 0, 0, 0, 0, 16, 0, 4037, 0, 0, 0, 0, 0, 8, 2, 4096, 0, 0, 0, 0, 0, 9, 7, 0, 8, 4080, 7, 0, 0, 0, 1, 708, 1798, 0, 0, 0, 0, 23, 341, 1600, 0, 1674, 640, 0, 0, 29, 3284, 0, 2054, 243, 48, 0, 0] rerouted_tokens_per_ep_rank=[4136, 3919, 4053, 4106, 4111, 2507, 4278, 5658] avg=4096.0000 max=5658 max_rank=7 min=2507 min_rank=5 imbalance_ratio=1.381348 max_load_reduction=27110 max_load_reduction_pct=0.827332 moved_to_echo=16554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2669, 0, 2737, 0, 0, 0, 2154, 0, 758, 305, 0, 0, 0, 0, 100, 9, 4042, 0, 0, 0, 0, 0, 8, 0, 4065, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 7, 0, 0, 0, 7, 710, 3798, 0, 0, 0, 0, 0, 260, 935, 0, 1691, 642, 0, 0, 0, 3488, 0, 10, 245, 48, 0, 0] rerouted_tokens_per_ep_rank=[5407, 3217, 4151, 4075, 4084, 4515, 3528, 3791] avg=4096.0000 max=5407 max_rank=0 min=3217 min_rank=1 imbalance_ratio=1.320068 max_load_reduction=27361 max_load_reduction_pct=0.834991 moved_to_echo=15918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 10880, 128, 22016, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 384, 5760, 26880, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 3200, 11136, 128, 13440, 5248, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 25344, 0, 5120, 2048, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17152, 128, 13440, 2304, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 128, 32384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[26, 0, 1, 2, 0, 13, 2200, 1, 0, 0, 2, 0, 2, 0, 2, 0, 4094, 3365, 2, 4079, 0, 2, 0, 9, 2, 2810, 4020, 4079, 4060, 2, 0, 3995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 61, 2289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3339, 0, 4089, 0, 0, 0, 89, 0, 2629, 3916, 4067, 4096, 0, 13, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[95, 10, 0, 7, 0, 24, 3071, 0, 0, 3, 0, 5, 7, 0, 6, 0, 4089, 3230, 8, 4084, 0, 4, 0, 2, 0, 1970, 4003, 4074, 4015, 0, 0, 4061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 34, 1315, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4087, 3770, 0, 4046, 3, 0, 0, 163, 0, 3023, 4085, 4082, 4084, 0, 10, 4059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 79, 1736, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 3307, 0, 4074, 1, 0, 0, 40, 2, 3245, 3940, 4082, 4095, 0, 5, 4066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 32, 835, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 3654, 0, 4093, 0, 0, 0, 98, 12, 3635, 4073, 4063, 4096, 0, 3, 4075] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 25, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3447, 0, 4095, 0, 0, 0, 15, 0, 2449, 3995, 4089, 4092, 0, 1, 4067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 72, 1419, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 3896, 0, 4093, 0, 0, 0, 168, 1, 2802, 4007, 4076, 4091, 0, 2, 4037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[26, 0, 1, 2, 4007, 51, 0, 0, 0, 13, 2200, 1, 2153, 0, 0, 0, 0, 0, 2, 0, 4079, 16, 0, 0, 2, 0, 2, 0, 4094, 0, 0, 0, 0, 3365, 2, 595, 0, 0, 0, 0, 0, 2, 0, 9, 3433, 587, 0, 0, 2, 2810, 1280, 0, 0, 0, 0, 0, 37, 2, 0, 3995, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4367, 4097, 4098, 3962, 4031, 4092, 4034] avg=4096.0000 max=4367 max_rank=1 min=3962 min_rank=4 imbalance_ratio=1.066162 max_load_reduction=28401 max_load_reduction_pct=0.866730 moved_to_echo=16119 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4043, 51, 0, 0, 0, 61, 2289, 0, 2098, 0, 0, 0, 0, 0, 0, 0, 4066, 16, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3339, 0, 596, 0, 0, 0, 0, 0, 0, 0, 89, 3442, 572, 0, 0, 0, 2629, 1246, 0, 0, 0, 0, 0, 37, 0, 13, 4084, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4448, 4082, 4097, 3935, 4103, 3875, 4134] avg=4096.0000 max=4448 max_rank=1 min=3875 min_rank=6 imbalance_ratio=1.085938 max_load_reduction=28320 max_load_reduction_pct=0.864258 moved_to_echo=16047 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4031, 51, 0, 0, 0, 34, 1315, 0, 2188, 0, 0, 0, 0, 7, 0, 0, 4082, 16, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 3770, 0, 589, 0, 0, 0, 0, 3, 0, 0, 163, 3406, 597, 0, 0, 0, 3023, 1300, 0, 0, 0, 0, 0, 37, 0, 10, 4059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 3537, 4105, 4087, 4359, 4169, 4323, 4106] avg=4096.0000 max=4359 max_rank=4 min=3537 min_rank=1 imbalance_ratio=1.064209 max_load_reduction=28409 max_load_reduction_pct=0.866974 moved_to_echo=16957 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4042, 51, 0, 0, 0, 79, 1736, 0, 2110, 0, 0, 0, 0, 3, 0, 0, 4081, 16, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 3307, 0, 594, 0, 0, 0, 0, 1, 0, 0, 40, 3429, 576, 0, 0, 2, 3245, 1254, 0, 0, 0, 0, 0, 37, 0, 5, 4066, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 3925, 4100, 4094, 3901, 4046, 4501, 4108] avg=4096.0000 max=4501 max_rank=6 min=3901 min_rank=4 imbalance_ratio=1.098877 max_load_reduction=28267 max_load_reduction_pct=0.862640 moved_to_echo=16556 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4048, 56, 0, 0, 0, 32, 835, 0, 2186, 0, 0, 0, 0, 2, 0, 0, 4063, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3654, 0, 588, 0, 0, 0, 0, 0, 0, 0, 98, 3449, 599, 0, 0, 12, 3635, 1288, 0, 0, 0, 0, 0, 29, 0, 3, 4075, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 3053, 4084, 4096, 4242, 4146, 4935, 4107] avg=4096.0000 max=4935 max_rank=6 min=3053 min_rank=1 imbalance_ratio=1.204834 max_load_reduction=27833 max_load_reduction_pct=0.849396 moved_to_echo=17430 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[95, 10, 0, 7, 3963, 51, 0, 0, 0, 24, 3071, 0, 2144, 0, 0, 0, 0, 3, 0, 5, 4074, 15, 0, 0, 7, 0, 6, 0, 4089, 0, 0, 0, 0, 3230, 8, 596, 0, 0, 0, 0, 0, 4, 0, 2, 3437, 585, 0, 0, 0, 1970, 1274, 0, 0, 0, 0, 0, 37, 0, 0, 4061, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 5239, 4097, 4102, 3834, 4028, 3244, 4098] avg=4096.0000 max=5239 max_rank=1 min=3244 min_rank=6 imbalance_ratio=1.279053 max_load_reduction=27529 max_load_reduction_pct=0.840118 moved_to_echo=15204 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4038, 52, 0, 0, 0, 72, 1419, 0, 2146, 0, 0, 0, 0, 8, 0, 0, 4075, 16, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3896, 0, 596, 0, 0, 0, 0, 0, 0, 0, 168, 3445, 586, 0, 0, 1, 2802, 1275, 0, 0, 0, 0, 0, 37, 0, 2, 4037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 3637, 4099, 4097, 4492, 4199, 4078, 4076] avg=4096.0000 max=4492 max_rank=4 min=3637 min_rank=1 imbalance_ratio=1.096680 max_load_reduction=28276 max_load_reduction_pct=0.862915 moved_to_echo=16845 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4039, 52, 0, 0, 0, 25, 2397, 0, 2140, 0, 0, 0, 0, 0, 0, 0, 4088, 16, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3447, 0, 596, 0, 0, 0, 0, 0, 0, 0, 15, 3447, 584, 0, 0, 0, 2449, 1271, 0, 0, 0, 0, 0, 37, 0, 1, 4067, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4562, 4104, 4097, 4043, 4046, 3720, 4105] avg=4096.0000 max=4562 max_rank=1 min=3720 min_rank=6 imbalance_ratio=1.113770 max_load_reduction=28206 max_load_reduction_pct=0.860779 moved_to_echo=15914 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 384, 15360, 128, 17280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 22656, 10240, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 640, 27520, 4736, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 28032, 128, 4864, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 128, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32256, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4088, 0, 4096, 0, 0, 582, 0, 4092, 0, 0, 0, 4095, 2043, 0, 0, 2, 0, 9, 0, 16, 1144, 4, 3968, 0, 437, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[2, 2, 2, 4094, 4083, 8, 4094, 2, 2, 2432, 1, 4094, 2, 1, 0, 4094, 138, 1, 0, 0, 2, 27, 0, 4, 738, 0, 3929, 30, 891, 2, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 1, 4096, 0, 0, 2461, 0, 4095, 0, 0, 0, 4095, 384, 0, 0, 0, 0, 4, 1, 6, 926, 1, 3972, 0, 438, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 1, 4096, 0, 0, 1558, 0, 4096, 0, 0, 0, 4096, 591, 0, 0, 0, 0, 0, 0, 5, 1306, 0, 4036, 0, 695, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4089, 9, 4096, 0, 0, 1065, 0, 4086, 0, 0, 0, 4096, 1030, 0, 0, 0, 0, 42, 0, 8, 1487, 10, 3965, 0, 593, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[7, 6, 8, 4087, 4072, 5, 4088, 2, 6, 1000, 0, 4089, 8, 14, 0, 4087, 931, 3, 0, 0, 7, 535, 0, 11, 865, 0, 3986, 9, 860, 6, 2, 4074] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4096, 4, 4096, 0, 0, 1225, 0, 4079, 0, 0, 0, 4096, 541, 0, 0, 0, 0, 2, 0, 10, 548, 5, 4075, 0, 1800, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 4091, 6, 4096, 0, 0, 1099, 0, 4086, 0, 0, 0, 4096, 2055, 0, 0, 0, 0, 2, 0, 1, 902, 5, 3901, 0, 238, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 2, 4094, 0, 0, 0, 0, 4083, 8, 0, 2, 0, 0, 0, 0, 2, 2432, 1, 2670, 0, 0, 0, 0, 2, 1, 0, 4094, 0, 0, 0, 0, 138, 1, 0, 0, 1424, 869, 742, 85, 2, 27, 0, 4, 4009, 0, 0, 0, 738, 0, 3060, 30, 0, 0, 0, 0, 891, 2, 0, 3351, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4093, 5105, 4097, 3259, 4042, 3828, 4244] avg=4096.0000 max=5105 max_rank=2 min=3259 min_rank=4 imbalance_ratio=1.246338 max_load_reduction=27663 max_load_reduction_pct=0.844208 moved_to_echo=15373 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 3, 0, 0, 0, 0, 0, 0, 582, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2043, 0, 0, 2, 1423, 878, 742, 82, 0, 9, 0, 16, 4011, 0, 0, 0, 1144, 4, 3090, 0, 0, 0, 0, 0, 437, 0, 0, 3354, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4091, 3251, 4095, 5170, 4036, 4238, 3791] avg=4096.0000 max=5170 max_rank=4 min=3251 min_rank=2 imbalance_ratio=1.262207 max_load_reduction=27598 max_load_reduction_pct=0.842224 moved_to_echo=17235 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4, 0, 0, 0, 0, 0, 0, 2461, 0, 2671, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 384, 0, 0, 0, 1424, 878, 742, 82, 0, 4, 1, 6, 4010, 0, 0, 0, 926, 1, 3094, 0, 0, 0, 0, 0, 438, 0, 0, 3354, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 5132, 4095, 3510, 4021, 4021, 3792] avg=4096.0000 max=5132 max_rank=2 min=3510 min_rank=4 imbalance_ratio=1.252930 max_load_reduction=27636 max_load_reduction_pct=0.843384 moved_to_echo=15344 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4, 4, 0, 0, 0, 0, 0, 0, 1225, 0, 2661, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 541, 0, 0, 0, 1418, 901, 742, 82, 0, 2, 0, 10, 4010, 0, 0, 0, 548, 5, 3174, 0, 0, 0, 0, 0, 1800, 0, 0, 3354, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4104, 3886, 4096, 3684, 4022, 3727, 5154] avg=4096.0000 max=5154 max_rank=7 min=3684 min_rank=4 imbalance_ratio=1.258301 max_load_reduction=27614 max_load_reduction_pct=0.842712 moved_to_echo=16587 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4089, 9, 0, 0, 0, 0, 0, 0, 0, 1065, 0, 2662, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1030, 0, 0, 0, 1424, 880, 747, 86, 0, 42, 0, 8, 4010, 0, 0, 0, 1487, 10, 3085, 0, 0, 0, 0, 0, 593, 0, 0, 3349, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4098, 3727, 4096, 4167, 4060, 4582, 3942] avg=4096.0000 max=4582 max_rank=6 min=3727 min_rank=2 imbalance_ratio=1.118652 max_load_reduction=28186 max_load_reduction_pct=0.860168 moved_to_echo=16751 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[7, 6, 8, 4087, 0, 0, 0, 0, 4072, 5, 0, 2, 0, 0, 0, 0, 6, 1000, 0, 2667, 0, 0, 0, 0, 8, 14, 0, 4087, 0, 0, 0, 0, 931, 3, 0, 0, 1422, 882, 738, 82, 7, 535, 0, 11, 4006, 0, 0, 0, 865, 0, 3104, 9, 0, 0, 0, 0, 860, 6, 2, 3336, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4079, 3673, 4109, 4058, 4559, 3978, 4204] avg=4096.0000 max=4559 max_rank=5 min=3673 min_rank=2 imbalance_ratio=1.113037 max_load_reduction=28209 max_load_reduction_pct=0.860870 moved_to_echo=16799 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 4091, 6, 4, 0, 0, 0, 0, 0, 0, 1099, 0, 2665, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2055, 0, 0, 0, 1421, 863, 742, 82, 0, 2, 0, 1, 4010, 0, 0, 0, 902, 5, 3038, 0, 0, 0, 0, 0, 238, 0, 0, 3354, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4101, 3764, 4096, 5163, 4013, 3945, 3592] avg=4096.0000 max=5163 max_rank=4 min=3592 min_rank=7 imbalance_ratio=1.260498 max_load_reduction=27605 max_load_reduction_pct=0.842438 moved_to_echo=16713 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4, 0, 0, 0, 0, 0, 0, 1558, 0, 2672, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 591, 0, 0, 0, 1424, 893, 742, 82, 0, 0, 0, 5, 4010, 0, 0, 0, 1306, 0, 3143, 0, 0, 0, 0, 0, 695, 0, 0, 3354, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4230, 4096, 3732, 4015, 4449, 4049] avg=4096.0000 max=4449 max_rank=6 min=3732 min_rank=4 imbalance_ratio=1.086182 max_load_reduction=28319 max_load_reduction_pct=0.864227 moved_to_echo=16245 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32781, 32768, 32768, 32780, 32743, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32781, 32768, 32768, 32780, 32743, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32781, 32768, 32768, 32780, 32743, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32781, 32768, 32768, 32780, 32743, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32781, 32768, 32768, 32780, 32743, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32781, 32768, 32768, 32780, 32743, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32781, 32768, 32768, 32780, 32743, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32781, 32768, 32768, 32780, 32743, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32743 actual_valid_recv=33408 observed_loss=-665 tokens_per_expert=[7808, 128, 0, 128, 11392, 7168, 6016, 768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 11520, 128, 21376, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6016, 128, 128, 26880, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7936, 128, 24832, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 640, 128, 128, 32128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32780 actual_valid_recv=33024 observed_loss=-244 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32781 actual_valid_recv=33152 observed_loss=-371 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 51, 0, 4, 3806, 4086, 10, 0, 0, 325, 0, 0, 0, 4096, 3, 4087, 0, 0, 4057, 0, 4095, 4065, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 75, 0, 0, 3528, 4094, 0, 2, 2, 639, 2, 2, 0, 4016, 5, 4030, 2, 1, 4090, 2, 4093, 4085, 0, 7, 2, 1, 1, 4087, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 13, 0, 1, 3762, 4094, 1, 0, 0, 401, 0, 0, 0, 4096, 2, 4070, 0, 0, 4083, 0, 4096, 4075, 0, 0, 0, 0, 0, 4073, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 41, 0, 0, 3925, 4092, 11, 0, 0, 198, 0, 0, 0, 4096, 3, 4096, 0, 1, 4076, 0, 4096, 4077, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 107, 3, 0, 3617, 4089, 1, 7, 8, 672, 6, 7, 0, 4044, 0, 3791, 0, 0, 4096, 7, 4088, 4081, 7, 37, 0, 3, 0, 4089, 0, 8, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 51, 0, 1, 3908, 4090, 3, 0, 0, 342, 0, 0, 3, 4096, 3, 4071, 0, 4, 4055, 0, 4096, 3983, 0, 19, 0, 0, 0, 4043, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 3779, 4096, 0, 0, 0, 344, 0, 0, 0, 4095, 0, 4093, 0, 0, 4077, 0, 4096, 4092, 0, 1, 0, 0, 1, 4089, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 60, 0, 2, 4069, 4092, 21, 0, 0, 40, 0, 0, 1, 4096, 3, 4094, 0, 0, 4034, 0, 4094, 4093, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 75, 0, 0, 3946, 35, 0, 0, 3528, 292, 0, 2, 0, 0, 0, 0, 2, 639, 2, 2, 3724, 0, 0, 0, 0, 52, 5, 4030, 0, 0, 0, 0, 2, 1, 4090, 2, 22, 0, 0, 0, 19, 4085, 0, 7, 0, 0, 0, 0, 2, 1, 1, 4087, 21, 0, 0, 0, 0, 2, 0, 0, 4074, 18, 0, 0] rerouted_tokens_per_ep_rank=[4056, 3822, 4369, 4087, 4117, 4111, 4112, 4094] avg=4096.0000 max=4369 max_rank=2 min=3822 min_rank=1 imbalance_ratio=1.066650 max_load_reduction=28399 max_load_reduction_pct=0.866669 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 41, 0, 0, 4025, 35, 0, 0, 3925, 292, 11, 0, 0, 0, 0, 0, 0, 198, 0, 0, 3722, 0, 0, 0, 0, 53, 3, 4096, 0, 0, 0, 0, 0, 1, 4076, 0, 22, 0, 0, 0, 19, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 21, 0, 0, 0, 0, 0, 0, 0, 4077, 18, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4228, 3920, 4152, 4099, 4096, 4077, 4095] avg=4096.0000 max=4228 max_rank=1 min=3920 min_rank=2 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 0, 1, 4025, 35, 0, 0, 3762, 292, 1, 0, 0, 0, 0, 0, 0, 401, 0, 0, 3724, 0, 0, 0, 0, 53, 2, 4070, 0, 0, 0, 0, 0, 0, 4083, 0, 22, 0, 0, 0, 19, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 21, 0, 0, 0, 1, 0, 0, 0, 4077, 18, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4055, 4125, 4125, 4105, 4094, 4094, 4096] avg=4096.0000 max=4125 max_rank=2 min=4055 min_rank=1 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 0, 1, 4025, 35, 0, 0, 3908, 291, 3, 0, 0, 0, 0, 0, 0, 342, 0, 0, 3721, 0, 0, 0, 3, 53, 3, 4071, 0, 0, 0, 0, 0, 4, 4055, 0, 22, 0, 0, 0, 19, 3983, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4043, 21, 0, 0, 0, 0, 0, 0, 0, 4077, 18, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4202, 4063, 4130, 4081, 4021, 4064, 4095] avg=4096.0000 max=4202 max_rank=1 min=4021 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 0, 4, 4029, 36, 0, 0, 3806, 276, 10, 0, 0, 0, 0, 0, 0, 325, 0, 0, 3721, 0, 0, 0, 0, 44, 3, 4087, 0, 0, 0, 0, 0, 0, 4057, 0, 29, 0, 0, 0, 13, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 24, 0, 0, 0, 0, 0, 0, 0, 4082, 23, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4092, 4046, 4134, 4086, 4078, 4107, 4105] avg=4096.0000 max=4134 max_rank=3 min=4046 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 4024, 35, 0, 0, 3779, 292, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 3726, 0, 0, 0, 0, 53, 0, 4093, 0, 0, 0, 0, 0, 0, 4077, 0, 22, 0, 0, 0, 19, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4089, 21, 0, 0, 0, 0, 0, 0, 0, 4077, 18, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4071, 4070, 4146, 4099, 4112, 4111, 4095] avg=4096.0000 max=4146 max_rank=3 min=4064 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 107, 3, 0, 3974, 35, 0, 0, 3617, 291, 1, 7, 0, 0, 0, 0, 8, 672, 6, 7, 3720, 0, 0, 0, 0, 52, 0, 3791, 0, 0, 0, 0, 0, 0, 4096, 7, 22, 0, 0, 0, 19, 4081, 7, 37, 0, 0, 0, 0, 0, 3, 0, 4089, 21, 0, 0, 0, 0, 8, 0, 0, 4069, 18, 0, 0] rerouted_tokens_per_ep_rank=[4119, 3916, 4413, 3843, 4125, 4144, 4113, 4095] avg=4096.0000 max=4413 max_rank=2 min=3843 min_rank=3 imbalance_ratio=1.077393 max_load_reduction=28355 max_load_reduction_pct=0.865326 moved_to_echo=16477 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 60, 0, 2, 4025, 35, 0, 0, 4069, 292, 21, 0, 0, 0, 0, 0, 0, 40, 0, 0, 3722, 0, 0, 0, 1, 53, 3, 4094, 0, 0, 0, 0, 0, 0, 4034, 0, 22, 0, 0, 0, 19, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 21, 0, 0, 0, 0, 0, 0, 0, 4075, 18, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4382, 3762, 4151, 4056, 4112, 4090, 4093] avg=4096.0000 max=4382 max_rank=1 min=3762 min_rank=2 imbalance_ratio=1.069824 max_load_reduction=28386 max_load_reduction_pct=0.866272 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[30464, 2432, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 3072, 128, 128, 29824, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 512, 128, 128, 32128, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 32640, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 128, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32640, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32640, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 4, 0, 4091, 0, 0, 4096, 4095, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 0, 4093, 0, 2, 4046, 2, 19, 0, 1, 2, 3981, 0, 2, 4094, 4094, 0, 4089, 0, 2, 2, 4093, 60, 2, 0, 87, 1, 2, 4089, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[7, 7, 0, 4085, 0, 9, 3989, 10, 59, 14, 29, 7, 3764, 0, 8, 4067, 4086, 8, 4071, 0, 8, 9, 4087, 97, 4, 0, 223, 30, 0, 4086, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 10, 0, 4075, 0, 0, 2, 7, 0, 4096, 4, 0, 4096, 4086, 0, 4096, 0, 0, 0, 4093, 0, 0, 3, 14, 0, 0, 4093, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 7, 0, 4034, 0, 0, 0, 3, 0, 4091, 2, 0, 4096, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 59, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 3, 0, 4083, 0, 0, 0, 2, 0, 4075, 1, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 30, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4058, 0, 1, 3, 12, 0, 4092, 0, 0, 4096, 4092, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 30, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4, 0, 4022, 0, 0, 0, 35, 0, 4095, 0, 0, 4096, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 41, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4093, 0, 0, 0, 0, 0, 2, 4046, 2, 22, 18, 0, 0, 19, 0, 1, 2, 4069, 0, 0, 0, 3981, 0, 2, 59, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4074, 60, 0, 0, 0, 0, 2, 0, 87, 1, 4013, 20, 0, 0, 2, 4089, 3, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4090, 4091, 4042, 4094, 4138, 4123, 4095] avg=4096.0000 max=4138 max_rank=5 min=4042 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 22, 18, 0, 0, 0, 0, 4, 0, 4074, 0, 0, 0, 4091, 0, 0, 59, 0, 0, 0, 0, 4095, 1, 5, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4015, 17, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4129, 4078, 4150, 4101, 4077, 4040, 4097] avg=4096.0000 max=4150 max_rank=3 min=4040 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 0, 0, 10, 0, 4075, 0, 22, 18, 0, 0, 0, 2, 7, 0, 4074, 0, 0, 0, 4096, 4, 0, 59, 0, 0, 0, 0, 4086, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 3, 14, 0, 4015, 17, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4125, 4083, 4159, 4091, 4074, 4049, 4094] avg=4096.0000 max=4159 max_rank=3 min=4049 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 3, 0, 4083, 0, 22, 18, 0, 0, 0, 0, 2, 0, 4074, 0, 0, 0, 4075, 1, 0, 59, 0, 0, 0, 0, 4095, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 1, 30, 0, 4015, 17, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4126, 4076, 4135, 4100, 4077, 4063, 4096] avg=4096.0000 max=4135 max_rank=3 min=4063 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2, 0, 0, 0, 7, 0, 4034, 0, 24, 21, 0, 0, 0, 0, 3, 0, 4079, 0, 0, 0, 4091, 2, 0, 53, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 59, 0, 4019, 17, 0, 0, 0, 4094, 0, 0, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4086, 4082, 4146, 4094, 4066, 4095, 4101] avg=4096.0000 max=4146 max_rank=3 min=4066 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4058, 0, 22, 18, 0, 0, 1, 3, 12, 0, 4074, 0, 0, 0, 4092, 0, 0, 59, 0, 0, 0, 0, 4092, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4015, 17, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4098, 4090, 4151, 4097, 4077, 4062, 4097] avg=4096.0000 max=4151 max_rank=3 min=4062 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[7, 7, 0, 4085, 0, 0, 0, 0, 0, 9, 3989, 10, 22, 18, 0, 0, 59, 14, 29, 7, 4050, 0, 0, 0, 3764, 0, 8, 59, 0, 0, 0, 0, 4086, 8, 1, 0, 0, 0, 0, 0, 8, 9, 4068, 97, 0, 0, 0, 0, 4, 0, 223, 30, 3986, 20, 0, 0, 0, 4086, 4, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4048, 4159, 3831, 4095, 4182, 4263, 4091] avg=4096.0000 max=4263 max_rank=6 min=3831 min_rank=3 imbalance_ratio=1.040771 max_load_reduction=28505 max_load_reduction_pct=0.869904 moved_to_echo=16631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4022, 0, 22, 18, 0, 0, 0, 0, 35, 0, 4074, 0, 0, 0, 4095, 0, 0, 59, 0, 0, 0, 0, 4091, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 41, 0, 4015, 17, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4066, 4109, 4154, 4096, 4077, 4073, 4097] avg=4096.0000 max=4154 max_rank=3 min=4066 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32384, 128, 128, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 512, 128, 32128, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32512, 128, 256, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[2, 2, 4094, 585, 2, 3655, 0, 2, 4083, 4017, 2, 101, 3, 0, 1829, 0, 4088, 0, 2, 4093, 1, 0, 0, 2, 2248, 3955, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4075, 681, 0, 3685, 0, 0, 4096, 4022, 0, 82, 0, 0, 2542, 0, 4096, 0, 0, 4096, 101, 0, 0, 0, 1770, 3515, 0, 0, 0, 0, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4096, 733, 0, 3960, 0, 0, 4094, 4047, 0, 36, 0, 0, 2525, 0, 4096, 0, 0, 4096, 39, 0, 0, 0, 1959, 3087, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4081, 589, 0, 3707, 0, 0, 4091, 4081, 0, 48, 0, 0, 3223, 2, 4096, 0, 0, 4096, 26, 0, 0, 0, 805, 3918, 0, 0, 0, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4090, 416, 0, 3966, 0, 0, 4096, 3991, 0, 16, 0, 0, 2903, 1, 4096, 0, 0, 4096, 19, 0, 0, 0, 1368, 3708, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4079, 571, 0, 4054, 0, 0, 4096, 4083, 0, 47, 0, 0, 1344, 0, 4096, 0, 0, 4096, 57, 0, 0, 0, 2592, 3650, 0, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[9, 10, 4088, 717, 5, 3679, 0, 9, 4045, 3974, 9, 79, 6, 1, 1224, 0, 4083, 0, 9, 4086, 0, 2, 2, 8, 2877, 3836, 0, 2, 0, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4075, 504, 0, 3839, 0, 0, 4096, 4067, 0, 145, 0, 0, 3224, 0, 4096, 0, 0, 4096, 125, 0, 0, 0, 664, 3831, 0, 0, 0, 0, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 3502, 585, 0, 0, 0, 0, 2, 3655, 0, 2, 266, 9, 0, 0, 0, 4008, 2, 101, 0, 0, 0, 0, 3, 0, 1829, 0, 1477, 326, 41, 0, 4088, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 2, 4042, 1, 0, 0, 2248, 2478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 3934, 4111, 3676, 4090, 4046, 4726, 4094] avg=4096.0000 max=4726 max_rank=6 min=3676 min_rank=3 imbalance_ratio=1.153809 max_load_reduction=28042 max_load_reduction_pct=0.855774 moved_to_echo=16956 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3491, 589, 0, 0, 0, 0, 0, 3707, 0, 0, 265, 9, 0, 0, 0, 4072, 0, 48, 0, 0, 0, 0, 0, 0, 3223, 2, 1463, 325, 40, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 26, 0, 0, 0, 4051, 1, 0, 0, 805, 2455, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 3981, 4120, 5053, 4098, 4078, 3260, 4098] avg=4096.0000 max=5053 max_rank=3 min=3260 min_rank=6 imbalance_ratio=1.233643 max_load_reduction=27715 max_load_reduction_pct=0.845795 moved_to_echo=15534 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3490, 571, 0, 0, 0, 0, 0, 4054, 0, 0, 265, 9, 0, 0, 0, 4074, 0, 47, 0, 0, 0, 0, 0, 0, 1344, 0, 1363, 324, 40, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 57, 0, 0, 0, 4056, 1, 0, 0, 2592, 2287, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4328, 4121, 3071, 4098, 4114, 4879, 4096] avg=4096.0000 max=4879 max_rank=6 min=3071 min_rank=3 imbalance_ratio=1.191162 max_load_reduction=27889 max_load_reduction_pct=0.851105 moved_to_echo=17187 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3499, 416, 0, 0, 0, 0, 0, 3966, 0, 0, 266, 9, 0, 0, 0, 3982, 0, 16, 0, 0, 0, 0, 0, 0, 2903, 1, 1384, 325, 40, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 19, 0, 0, 0, 4056, 1, 0, 0, 1368, 2324, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[3915, 4241, 3998, 4653, 4098, 4076, 3692, 4095] avg=4096.0000 max=4653 max_rank=3 min=3692 min_rank=6 imbalance_ratio=1.135986 max_load_reduction=28115 max_load_reduction_pct=0.858002 moved_to_echo=15961 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3479, 681, 0, 0, 0, 0, 0, 3685, 0, 0, 268, 16, 0, 0, 0, 4006, 0, 82, 0, 0, 0, 0, 0, 0, 2542, 0, 1316, 328, 41, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 4055, 3, 0, 0, 1770, 2199, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 3969, 4088, 4227, 4096, 4159, 3969, 4100] avg=4096.0000 max=4227 max_rank=3 min=3969 min_rank=1 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16324 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3504, 733, 0, 0, 0, 0, 0, 3960, 0, 0, 266, 9, 0, 0, 0, 4038, 0, 36, 0, 0, 0, 0, 0, 0, 2525, 0, 1152, 326, 40, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 39, 0, 0, 0, 4054, 1, 0, 0, 1959, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4237, 4235, 4074, 4043, 4098, 4094, 3894, 4093] avg=4096.0000 max=4237 max_rank=0 min=3894 min_rank=6 imbalance_ratio=1.034424 max_load_reduction=28531 max_load_reduction_pct=0.870697 moved_to_echo=16179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[9, 10, 3498, 717, 0, 0, 0, 0, 5, 3679, 0, 9, 265, 9, 0, 0, 0, 3965, 9, 79, 0, 0, 0, 0, 6, 1, 1224, 0, 1432, 325, 39, 0, 4083, 0, 9, 0, 0, 0, 0, 0, 0, 2, 2, 8, 4006, 1, 0, 0, 2877, 2404, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8, 4085, 0, 0, 0] rerouted_tokens_per_ep_rank=[4234, 3967, 4053, 3027, 4092, 4019, 5283, 4093] avg=4096.0000 max=5283 max_rank=6 min=3027 min_rank=3 imbalance_ratio=1.289795 max_load_reduction=27485 max_load_reduction_pct=0.838776 moved_to_echo=17487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3486, 504, 0, 0, 0, 0, 0, 3839, 0, 0, 265, 9, 0, 0, 0, 4058, 0, 145, 0, 0, 0, 0, 0, 0, 3224, 0, 1430, 324, 40, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 125, 0, 0, 0, 4056, 1, 0, 0, 664, 2401, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[3990, 4113, 4203, 5018, 4098, 4182, 3065, 4099] avg=4096.0000 max=5018 max_rank=3 min=3065 min_rank=6 imbalance_ratio=1.225098 max_load_reduction=27750 max_load_reduction_pct=0.846863 moved_to_echo=15444 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 18816, 128, 11136, 2688, 384, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14336, 18560, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 28032, 4864, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 128, 128, 128, 32384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32256, 128, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 30592, 0, 128, 2176, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 4, 1, 4093, 0, 2, 2, 4093, 4083, 6, 2, 3, 2, 0, 4086, 0, 0, 4093, 3588, 4093, 2, 4, 2, 253, 58, 0, 0, 1, 3964, 0, 333] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4096, 0, 0, 0, 4094, 4096, 10, 0, 0, 0, 2, 4086, 0, 0, 4096, 1068, 4094, 0, 0, 0, 1236, 0, 0, 0, 0, 4092, 0, 1794] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4085, 0, 8, 8, 4087, 4070, 0, 10, 9, 7, 0, 4084, 0, 9, 4083, 3503, 4088, 5, 11, 0, 131, 149, 0, 0, 0, 3929, 0, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 1581, 4096, 0, 0, 0, 1274, 0, 0, 0, 0, 4096, 0, 1241] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 4096, 0, 0, 0, 4092, 4095, 4, 0, 0, 0, 3, 4093, 0, 0, 4086, 1225, 4089, 0, 0, 0, 1122, 0, 3, 0, 0, 4096, 0, 1749] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4096, 0, 0, 0, 4095, 4095, 4, 0, 0, 0, 1, 4094, 0, 0, 4096, 1995, 4093, 0, 0, 0, 1231, 0, 1, 0, 0, 4096, 0, 869] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 870, 4096, 0, 0, 0, 1929, 0, 0, 0, 0, 4096, 0, 1298] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4096, 0, 0, 0, 4096, 4091, 5, 0, 0, 0, 0, 4094, 0, 0, 4096, 535, 4092, 0, 0, 0, 1353, 0, 0, 0, 0, 4096, 0, 2211] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 1, 4092, 0, 0, 0, 4093, 0, 2, 2, 0, 0, 0, 0, 0, 4083, 6, 2, 0, 0, 0, 0, 3, 2, 0, 4086, 1, 0, 0, 0, 0, 0, 2300, 3588, 0, 0, 0, 0, 4093, 2, 4, 2, 0, 0, 0, 0, 253, 58, 0, 0, 1793, 1180, 1, 0, 1, 2783, 0, 333, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4097, 4091, 4092, 5888, 4101, 3285, 3117] avg=4096.0000 max=5888 max_rank=4 min=3117 min_rank=7 imbalance_ratio=1.437500 max_load_reduction=26880 max_load_reduction_pct=0.820312 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 1, 0, 0, 0, 0, 0, 2296, 1225, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 3, 0, 1790, 1220, 1, 0, 0, 2875, 0, 1749, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4096, 4099, 4097, 3521, 4089, 4136, 4624] avg=4096.0000 max=4624 max_rank=7 min=3521 min_rank=4 imbalance_ratio=1.128906 max_load_reduction=28144 max_load_reduction_pct=0.858887 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 2302, 1581, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1274, 0, 0, 0, 1794, 1220, 3, 0, 0, 2875, 0, 1241, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 4096, 4097, 3883, 4096, 4291, 4116] avg=4096.0000 max=4291 max_rank=6 min=3883 min_rank=4 imbalance_ratio=1.047607 max_load_reduction=28477 max_load_reduction_pct=0.869049 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 0, 0, 0, 2302, 1995, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1231, 0, 1, 0, 1794, 1220, 2, 0, 0, 2875, 0, 869, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4100, 4096, 4297, 4093, 4248, 3744] avg=4096.0000 max=4297 max_rank=4 min=3744 min_rank=7 imbalance_ratio=1.049072 max_load_reduction=28471 max_load_reduction_pct=0.868866 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4093, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 2, 4086, 2, 3, 0, 0, 0, 0, 2297, 1068, 0, 0, 0, 0, 4094, 0, 0, 0, 3, 0, 0, 0, 1236, 0, 0, 0, 1799, 1221, 1, 0, 0, 2869, 0, 1794, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4090, 4106, 4093, 3365, 4097, 4257, 4663] avg=4096.0000 max=4663 max_rank=7 min=3365 min_rank=4 imbalance_ratio=1.138428 max_load_reduction=28105 max_load_reduction_pct=0.857697 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4086, 0, 0, 0, 4085, 0, 8, 8, 0, 0, 0, 0, 0, 4070, 0, 10, 0, 0, 0, 0, 9, 7, 0, 4084, 1, 0, 0, 0, 0, 9, 2294, 3503, 0, 0, 0, 0, 4088, 5, 11, 0, 0, 0, 0, 0, 131, 149, 0, 0, 1789, 1170, 1, 0, 0, 2758, 0, 487, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4101, 4080, 4101, 5806, 4104, 3240, 3245] avg=4096.0000 max=5806 max_rank=4 min=3240 min_rank=6 imbalance_ratio=1.417480 max_load_reduction=26962 max_load_reduction_pct=0.822815 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 2301, 870, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1929, 0, 0, 0, 1793, 1220, 1, 0, 0, 2875, 0, 1298, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4098, 4097, 3171, 4096, 4943, 4173] avg=4096.0000 max=4943 max_rank=6 min=3171 min_rank=4 imbalance_ratio=1.206787 max_load_reduction=27825 max_load_reduction_pct=0.849152 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 2302, 535, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 1353, 0, 0, 0, 1794, 1220, 1, 0, 0, 2875, 0, 2211, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4098, 4095, 2837, 4092, 4368, 5086] avg=4096.0000 max=5086 max_rank=7 min=2837 min_rank=4 imbalance_ratio=1.241699 max_load_reduction=27682 max_load_reduction_pct=0.844788 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8576, 256, 128, 0, 14464, 9728, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 18432, 14464, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 22912, 0, 9984, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[2, 3, 1, 3, 4092, 0, 0, 2, 3027, 5, 3345, 2, 2, 2, 3, 4092, 1622, 139, 2, 0, 0, 0, 4090, 0, 2, 0, 2, 4092, 0, 90, 4055, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3455, 0, 4094, 0, 0, 0, 0, 4075, 521, 0, 0, 23, 0, 12, 4096, 0, 0, 0, 0, 4085, 0, 207, 4008, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 2896, 0, 4096, 0, 0, 0, 0, 4069, 1077, 1, 0, 6, 0, 5, 4096, 0, 0, 0, 0, 4086, 0, 213, 4031, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 0, 0, 0, 3386, 0, 4094, 0, 0, 0, 0, 4075, 518, 0, 0, 4, 0, 11, 4096, 0, 0, 0, 0, 4093, 0, 304, 3996, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 2749, 0, 4095, 0, 0, 0, 0, 4085, 1085, 1, 0, 3, 0, 5, 4096, 0, 0, 0, 0, 4095, 0, 368, 3995, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1, 8, 5, 10, 4073, 0, 4, 8, 2756, 39, 2864, 9, 7, 7, 9, 4084, 2272, 220, 8, 0, 1, 0, 4075, 0, 10, 0, 0, 4086, 0, 40, 4088, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 2906, 0, 4091, 0, 0, 0, 0, 4092, 854, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 710, 3727, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 2995, 0, 4096, 0, 0, 0, 0, 4083, 727, 1, 0, 7, 0, 9, 4096, 0, 0, 0, 0, 4057, 0, 464, 4041, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 1, 3, 4092, 0, 0, 0, 4092, 0, 0, 2, 1, 0, 0, 0, 3027, 5, 929, 2, 0, 0, 0, 0, 2, 2, 3, 4092, 10, 0, 0, 0, 1622, 139, 2, 0, 2416, 180, 1, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 2, 0, 2, 4092, 7, 0, 0, 0, 0, 90, 3858, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4095, 3963, 4109, 4360, 4089, 4103, 3948] avg=4096.0000 max=4360 max_rank=4 min=3948 min_rank=7 imbalance_ratio=1.064453 max_load_reduction=28408 max_load_reduction_pct=0.866943 moved_to_echo=16500 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 2896, 0, 1138, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 10, 0, 0, 0, 1077, 1, 0, 6, 2958, 179, 2, 0, 0, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 7, 0, 0, 0, 0, 213, 3835, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4097, 4034, 4079, 4223, 4100, 4093, 4048] avg=4096.0000 max=4223 max_rank=4 min=4034 min_rank=2 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 3386, 0, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 10, 0, 0, 0, 518, 0, 0, 4, 2957, 178, 2, 0, 0, 11, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 0, 0, 0, 304, 3801, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4097, 4523, 4085, 3659, 4106, 4100, 4105] avg=4096.0000 max=4523 max_rank=2 min=3659 min_rank=4 imbalance_ratio=1.104248 max_load_reduction=28245 max_load_reduction_pct=0.861969 moved_to_echo=15970 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 2749, 0, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 10, 0, 0, 0, 1085, 1, 0, 3, 2958, 178, 2, 0, 0, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 368, 3800, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4097, 3886, 4095, 4227, 4100, 4102, 4168] avg=4096.0000 max=4227 max_rank=4 min=3886 min_rank=2 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16597 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 3455, 0, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 14, 0, 0, 0, 521, 0, 0, 23, 2962, 182, 2, 0, 0, 12, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 12, 3, 0, 0, 0, 207, 3800, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4102, 4587, 4089, 3690, 4099, 4100, 4007] avg=4096.0000 max=4587 max_rank=2 min=3690 min_rank=4 imbalance_ratio=1.119873 max_load_reduction=28181 max_load_reduction_pct=0.860016 moved_to_echo=15896 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 2906, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 9, 0, 0, 0, 854, 4, 0, 0, 2955, 166, 3, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 710, 3545, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4097, 4042, 4101, 3982, 4095, 4103, 4255] avg=4096.0000 max=4255 max_rank=7 min=3982 min_rank=4 imbalance_ratio=1.038818 max_load_reduction=28513 max_load_reduction_pct=0.870148 moved_to_echo=16435 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 8, 5, 10, 4083, 0, 0, 0, 4073, 0, 4, 8, 1, 0, 0, 0, 2756, 39, 796, 9, 0, 0, 0, 0, 7, 7, 9, 4084, 10, 0, 0, 0, 2272, 220, 8, 0, 2068, 182, 1, 0, 1, 0, 4074, 0, 0, 0, 0, 0, 10, 0, 0, 4086, 7, 0, 0, 0, 0, 40, 3889, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4086, 3600, 4117, 4751, 4075, 4103, 3929] avg=4096.0000 max=4751 max_rank=4 min=3600 min_rank=2 imbalance_ratio=1.159912 max_load_reduction=28017 max_load_reduction_pct=0.855011 moved_to_echo=16858 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 2995, 0, 1138, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 10, 0, 0, 0, 727, 1, 0, 7, 2958, 180, 3, 0, 0, 9, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 7, 0, 0, 0, 0, 464, 3844, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4097, 4133, 4093, 3876, 4104, 4064, 4308] avg=4096.0000 max=4308 max_rank=7 min=3876 min_rank=4 imbalance_ratio=1.051758 max_load_reduction=28460 max_load_reduction_pct=0.868530 moved_to_echo=16352 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8704, 384, 128, 128, 22272, 1536, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[24192, 128, 8576, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 2432, 30464, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4094, 0, 0, 0, 1471, 0, 0, 4061, 0, 70, 2226, 0, 0, 4096, 0, 0, 4096, 0, 1, 354, 0, 4096, 0, 20, 0, 0, 0, 0, 4087] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[2, 3, 4020, 3897, 2, 0, 4, 3347, 0, 0, 3477, 3, 0, 400, 3, 8, 4088, 3, 2, 4093, 0, 0, 1226, 2, 4093, 0, 0, 0, 0, 0, 3, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4093, 4028, 0, 0, 0, 2206, 0, 0, 3732, 0, 3, 430, 0, 0, 4096, 0, 0, 4096, 0, 0, 1878, 0, 4096, 0, 14, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[12, 9, 3721, 3296, 7, 0, 0, 2955, 0, 0, 3325, 10, 0, 355, 11, 6, 4071, 13, 0, 4086, 0, 0, 2671, 6, 4087, 22, 0, 0, 8, 0, 9, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4089, 4087, 0, 0, 0, 1636, 0, 0, 3942, 0, 6, 1745, 0, 0, 4096, 0, 0, 4096, 0, 0, 866, 0, 4096, 0, 17, 0, 0, 0, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4077, 0, 0, 0, 1353, 0, 0, 3748, 0, 28, 1557, 0, 0, 4096, 0, 0, 4096, 0, 0, 1519, 0, 4096, 0, 8, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4095, 3995, 0, 0, 0, 2008, 0, 0, 3876, 0, 4, 744, 0, 1, 4096, 0, 0, 4096, 0, 0, 1660, 0, 4096, 0, 1, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 1673, 0, 0, 3962, 0, 20, 1867, 0, 0, 4096, 0, 0, 4096, 0, 0, 670, 0, 4096, 0, 5, 0, 0, 0, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 150, 4094, 0, 0, 0, 0, 0, 0, 0, 1471, 2042, 0, 0, 0, 0, 0, 4061, 0, 325, 7, 0, 0, 70, 2226, 0, 0, 2911, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 1, 354, 0, 1579, 1182, 0, 0, 4088, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4244, 3513, 4393, 5207, 4099, 3116, 4108, 4088] avg=4096.0000 max=5207 max_rank=3 min=3116 min_rank=5 imbalance_ratio=1.271240 max_load_reduction=27561 max_load_reduction_pct=0.841095 moved_to_echo=15411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 147, 3897, 0, 0, 0, 0, 2, 0, 4, 3347, 2004, 0, 0, 0, 0, 0, 3477, 3, 319, 7, 0, 0, 0, 400, 3, 8, 2911, 0, 0, 0, 4088, 3, 2, 0, 0, 0, 0, 0, 0, 0, 1226, 2, 1550, 1182, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 5357, 3806, 3322, 4093, 3960, 4085, 4096] avg=4096.0000 max=5357 max_rank=1 min=3322 min_rank=3 imbalance_ratio=1.307861 max_load_reduction=27411 max_load_reduction_pct=0.836517 moved_to_echo=16234 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 150, 4028, 0, 0, 0, 0, 0, 0, 0, 2206, 2040, 0, 0, 0, 0, 0, 3732, 0, 325, 7, 0, 0, 3, 430, 0, 0, 2911, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1878, 0, 1578, 1182, 0, 0, 4088, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4246, 4064, 3344, 4099, 4638, 4102, 4097] avg=4096.0000 max=4638 max_rank=5 min=3344 min_rank=3 imbalance_ratio=1.132324 max_load_reduction=28130 max_load_reduction_pct=0.858459 moved_to_echo=16936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 150, 4077, 0, 0, 0, 0, 0, 0, 0, 1353, 2042, 0, 0, 0, 0, 0, 3748, 0, 325, 7, 0, 0, 28, 1557, 0, 0, 2911, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1519, 0, 1579, 1182, 0, 0, 4088, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4227, 3395, 4080, 4496, 4099, 4280, 4096, 4095] avg=4096.0000 max=4496 max_rank=3 min=3395 min_rank=1 imbalance_ratio=1.097656 max_load_reduction=28272 max_load_reduction_pct=0.862793 moved_to_echo=16570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[12, 9, 136, 3296, 0, 0, 0, 0, 7, 0, 0, 2955, 1855, 0, 0, 0, 0, 0, 3325, 10, 295, 7, 0, 0, 0, 355, 11, 6, 2904, 0, 0, 0, 4071, 13, 0, 0, 0, 0, 0, 0, 0, 0, 2671, 6, 1435, 1182, 0, 0, 4079, 22, 0, 0, 0, 0, 0, 0, 8, 0, 9, 4088, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3453, 4817, 3637, 3276, 4084, 5294, 4101, 4106] avg=4096.0000 max=5294 max_rank=5 min=3276 min_rank=3 imbalance_ratio=1.292480 max_load_reduction=27474 max_load_reduction_pct=0.838440 moved_to_echo=17585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 139, 4087, 0, 0, 0, 0, 0, 0, 0, 1636, 2040, 0, 0, 0, 0, 0, 3942, 0, 330, 14, 0, 0, 6, 1745, 0, 0, 2914, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 866, 0, 1580, 1182, 0, 0, 4077, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4226, 3676, 4286, 4665, 4096, 3628, 4094, 4097] avg=4096.0000 max=4665 max_rank=3 min=3628 min_rank=5 imbalance_ratio=1.138916 max_load_reduction=28103 max_load_reduction_pct=0.857635 moved_to_echo=15915 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 150, 3995, 0, 0, 0, 0, 0, 0, 0, 2008, 2041, 0, 0, 0, 0, 0, 3876, 0, 325, 7, 0, 0, 4, 744, 0, 1, 2911, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1660, 0, 1579, 1182, 0, 0, 4088, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4049, 4208, 3660, 4099, 4421, 4089, 4097] avg=4096.0000 max=4421 max_rank=5 min=3660 min_rank=3 imbalance_ratio=1.079346 max_load_reduction=28347 max_load_reduction_pct=0.865082 moved_to_echo=16706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 150, 4096, 0, 0, 0, 0, 0, 0, 0, 1673, 2042, 0, 0, 0, 0, 0, 3962, 0, 325, 7, 0, 0, 20, 1867, 0, 0, 2911, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 670, 0, 1579, 1182, 0, 0, 4088, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4246, 3715, 4294, 4798, 4099, 3431, 4093, 4092] avg=4096.0000 max=4798 max_rank=3 min=3431 min_rank=5 imbalance_ratio=1.171387 max_load_reduction=27970 max_load_reduction_pct=0.853577 moved_to_echo=15715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 10880, 128, 12544, 9472, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 16768, 16128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 9344, 128, 128, 23296, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 1280, 31616, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 30208, 128, 2688, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4010, 4096, 0, 2, 1, 4096, 4096, 3, 0, 0, 4039, 0, 4096, 0, 4096, 6, 0, 0, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 2787, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[3923, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 3804, 0, 4096, 0, 4096, 18, 0, 0, 0, 1204, 0, 0, 0, 0, 0, 0, 0, 3339, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3942, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 3873, 0, 4096, 0, 4096, 1, 0, 0, 0, 1217, 0, 0, 0, 0, 0, 0, 0, 3255, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3774, 4089, 0, 0, 0, 4077, 4093, 0, 0, 2, 3862, 0, 4093, 6, 3989, 235, 46, 2, 3, 2167, 2, 0, 0, 3, 0, 3, 5, 2302, 0, 2, 2, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[3751, 4074, 8, 0, 0, 3996, 4085, 0, 6, 7, 4003, 0, 4083, 26, 3826, 428, 111, 6, 9, 3585, 8, 0, 0, 9, 0, 11, 19, 671, 0, 0, 5, 41] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4053, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4082, 0, 4096, 0, 4096, 15, 0, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 2474, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4052, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4077, 0, 4096, 0, 4096, 0, 0, 0, 0, 2440, 0, 0, 0, 0, 0, 0, 0, 1719, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3964, 4096, 0, 0, 1, 4096, 4096, 0, 0, 0, 4075, 0, 4096, 0, 4095, 38, 0, 0, 0, 1210, 0, 0, 0, 0, 0, 0, 0, 3001, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4010, 164, 0, 2, 0, 0, 0, 0, 1, 4096, 15, 3, 0, 0, 0, 0, 0, 0, 4039, 0, 67, 49, 0, 0, 0, 0, 4047, 6, 0, 0, 0, 0, 0, 0, 0, 1440, 2209, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 2787, 1649, 0, 0, 0, 0, 0, 0, 0, 4081, 7, 1, 0] rerouted_tokens_per_ep_rank=[4176, 4115, 4155, 4053, 3649, 4095, 4436, 4089] avg=4096.0000 max=4436 max_rank=6 min=3649 min_rank=4 imbalance_ratio=1.083008 max_load_reduction=28332 max_load_reduction_pct=0.864624 moved_to_echo=16269 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3774, 163, 0, 0, 0, 0, 0, 0, 0, 4077, 15, 0, 0, 0, 0, 0, 0, 2, 3862, 0, 67, 48, 0, 0, 0, 6, 3941, 235, 0, 0, 0, 0, 46, 2, 3, 2167, 2206, 0, 0, 0, 2, 0, 0, 3, 4093, 0, 0, 0, 0, 3, 5, 2302, 1646, 0, 0, 0, 0, 2, 2, 11, 4078, 7, 0, 0] rerouted_tokens_per_ep_rank=[3937, 4092, 3979, 4182, 4424, 4098, 3956, 4100] avg=4096.0000 max=4424 max_rank=4 min=3937 min_rank=0 imbalance_ratio=1.080078 max_load_reduction=28344 max_load_reduction_pct=0.864990 moved_to_echo=16578 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3942, 164, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 3873, 0, 67, 49, 0, 0, 0, 0, 4047, 1, 0, 0, 0, 0, 0, 0, 0, 1217, 2209, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3255, 1649, 0, 0, 0, 0, 0, 0, 0, 4081, 7, 1, 0] rerouted_tokens_per_ep_rank=[4106, 4111, 3989, 4048, 3426, 4095, 4904, 4089] avg=4096.0000 max=4904 max_rank=6 min=3426 min_rank=4 imbalance_ratio=1.197266 max_load_reduction=27864 max_load_reduction_pct=0.850342 moved_to_echo=16514 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4053, 164, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 67, 49, 0, 0, 0, 0, 4047, 15, 0, 0, 0, 0, 0, 0, 0, 1664, 2209, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 2474, 1649, 0, 0, 0, 0, 0, 0, 0, 4081, 7, 1, 0] rerouted_tokens_per_ep_rank=[4217, 4111, 4198, 4062, 3873, 4095, 4123, 4089] avg=4096.0000 max=4217 max_rank=0 min=3873 min_rank=4 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16180 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3923, 144, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 3804, 0, 74, 56, 0, 0, 0, 0, 4040, 18, 0, 0, 0, 0, 0, 0, 0, 1204, 2215, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3339, 1651, 0, 0, 0, 0, 0, 0, 0, 4087, 12, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4105, 3934, 4058, 3419, 4096, 4990, 4099] avg=4096.0000 max=4990 max_rank=6 min=3419 min_rank=4 imbalance_ratio=1.218262 max_load_reduction=27778 max_load_reduction_pct=0.847717 moved_to_echo=16604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4052, 164, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 67, 49, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 2440, 2209, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1719, 1649, 0, 0, 0, 0, 0, 0, 0, 4081, 7, 1, 0] rerouted_tokens_per_ep_rank=[4216, 4111, 4193, 4047, 4649, 4095, 3368, 4089] avg=4096.0000 max=4649 max_rank=4 min=3368 min_rank=6 imbalance_ratio=1.135010 max_load_reduction=28119 max_load_reduction_pct=0.858124 moved_to_echo=16201 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3751, 162, 8, 0, 0, 0, 0, 0, 0, 3996, 15, 0, 0, 0, 0, 0, 6, 7, 4003, 0, 67, 46, 0, 0, 0, 26, 3780, 428, 0, 0, 0, 0, 111, 6, 9, 3585, 2198, 0, 0, 0, 8, 0, 0, 9, 4082, 0, 0, 0, 0, 11, 19, 671, 1640, 0, 0, 0, 0, 0, 5, 41, 4070, 7, 1, 0] rerouted_tokens_per_ep_rank=[3921, 4011, 4129, 4234, 5909, 4099, 2341, 4124] avg=4096.0000 max=5909 max_rank=4 min=2341 min_rank=6 imbalance_ratio=1.442627 max_load_reduction=26859 max_load_reduction_pct=0.819672 moved_to_echo=16473 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3964, 164, 0, 0, 0, 0, 0, 0, 1, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 67, 49, 0, 0, 0, 0, 4046, 38, 0, 0, 0, 0, 0, 0, 0, 1210, 2209, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3001, 1649, 0, 0, 0, 0, 0, 0, 0, 4081, 7, 1, 0] rerouted_tokens_per_ep_rank=[4128, 4112, 4191, 4084, 3419, 4095, 4650, 4089] avg=4096.0000 max=4650 max_rank=6 min=3419 min_rank=4 imbalance_ratio=1.135254 max_load_reduction=28118 max_load_reduction_pct=0.858093 moved_to_echo=16253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 128, 14976, 17664, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 19584, 13184, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[31488, 1408, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32640, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 31872, 0, 640, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32000, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[674, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 40, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[105, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 54, 0, 0, 4056, 0, 2, 4094, 0, 0, 0, 4087, 0, 2, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[21, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 23, 0, 0, 4069, 0, 4, 4091, 0, 0, 0, 4095, 1, 4, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1264, 2, 8, 3, 1, 0, 0, 18, 3339, 41, 221, 0, 4, 3956, 0, 0, 4093, 0, 0, 2, 3709, 0, 0, 3791, 4088, 4091, 0, 77, 3, 4054, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[2488, 8, 35, 10, 9, 0, 0, 79, 2709, 79, 436, 0, 14, 3921, 0, 0, 4086, 0, 0, 6, 3162, 0, 0, 3410, 4061, 4077, 0, 187, 10, 3970, 11, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[205, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 21, 0, 0, 4089, 0, 1, 4095, 0, 0, 0, 4076, 0, 1, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[296, 0, 0, 0, 0, 0, 0, 0, 3802, 0, 17, 0, 0, 4082, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 4096, 4096, 4095, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[87, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 51, 0, 0, 4052, 0, 2, 4094, 0, 0, 0, 4088, 0, 5, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[674, 0, 0, 0, 3542, 10, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 3397, 0, 40, 0, 332, 0, 0, 0, 0, 4090, 0, 0, 46, 8, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4087, 0, 0, 176, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4226, 4086, 3769, 4144, 4097, 4263, 4070, 4113] avg=4096.0000 max=4263 max_rank=5 min=3769 min_rank=2 imbalance_ratio=1.040771 max_load_reduction=28505 max_load_reduction_pct=0.869904 moved_to_echo=16543 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1264, 2, 8, 3, 3278, 11, 0, 0, 1, 0, 0, 18, 4080, 0, 0, 0, 3339, 41, 221, 0, 307, 0, 0, 0, 4, 3956, 0, 0, 43, 8, 0, 0, 4093, 0, 0, 2, 1, 0, 0, 0, 3709, 0, 0, 163, 0, 0, 0, 0, 4062, 0, 0, 77, 0, 0, 0, 0, 3, 4054, 3, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4566, 4099, 3908, 4011, 4096, 3872, 4139, 4077] avg=4096.0000 max=4566 max_rank=0 min=3872 min_rank=5 imbalance_ratio=1.114746 max_load_reduction=28202 max_load_reduction_pct=0.860657 moved_to_echo=16184 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 0, 3542, 10, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4076, 0, 23, 0, 332, 0, 0, 0, 0, 4069, 0, 4, 46, 8, 0, 0, 4091, 0, 0, 0, 1, 0, 0, 0, 4095, 1, 4, 176, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[3573, 4086, 4431, 4127, 4092, 4276, 4070, 4113] avg=4096.0000 max=4431 max_rank=2 min=3573 min_rank=0 imbalance_ratio=1.081787 max_load_reduction=28337 max_load_reduction_pct=0.864777 moved_to_echo=16551 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[205, 0, 0, 0, 3542, 10, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 3896, 0, 21, 0, 332, 0, 0, 0, 0, 4089, 0, 1, 46, 8, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 4076, 0, 1, 176, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[3757, 4086, 4249, 4144, 4096, 4253, 4070, 4113] avg=4096.0000 max=4253 max_rank=5 min=3757 min_rank=0 imbalance_ratio=1.038330 max_load_reduction=28515 max_load_reduction_pct=0.870209 moved_to_echo=16532 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[105, 0, 0, 0, 3544, 11, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 3984, 0, 54, 0, 334, 0, 0, 0, 0, 4056, 0, 2, 50, 9, 0, 0, 4094, 0, 0, 0, 8, 0, 0, 0, 4087, 0, 2, 168, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[3660, 4085, 4372, 4117, 4102, 4257, 4057, 4118] avg=4096.0000 max=4372 max_rank=2 min=3660 min_rank=0 imbalance_ratio=1.067383 max_load_reduction=28396 max_load_reduction_pct=0.866577 moved_to_echo=16534 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[87, 0, 0, 0, 3542, 10, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4005, 0, 51, 0, 332, 0, 0, 0, 0, 4052, 0, 2, 46, 8, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 4088, 0, 5, 176, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[3639, 4086, 4388, 4108, 4095, 4269, 4070, 4113] avg=4096.0000 max=4388 max_rank=2 min=3639 min_rank=0 imbalance_ratio=1.071289 max_load_reduction=28380 max_load_reduction_pct=0.866089 moved_to_echo=16547 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[296, 0, 0, 0, 3542, 10, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 3802, 0, 17, 0, 332, 0, 0, 0, 0, 4082, 0, 0, 46, 8, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4092, 0, 0, 176, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[3848, 4085, 4151, 4136, 4097, 4268, 4070, 4113] avg=4096.0000 max=4268 max_rank=5 min=3848 min_rank=0 imbalance_ratio=1.041992 max_load_reduction=28500 max_load_reduction_pct=0.869751 moved_to_echo=16548 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2488, 8, 35, 10, 2948, 10, 0, 0, 9, 0, 0, 79, 4067, 0, 0, 0, 2709, 79, 436, 0, 276, 0, 0, 0, 14, 3921, 0, 0, 38, 8, 0, 0, 4086, 0, 0, 6, 1, 0, 0, 0, 3162, 0, 0, 148, 0, 0, 0, 0, 4035, 0, 0, 187, 0, 0, 0, 0, 10, 3970, 11, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[5499, 4155, 3500, 3981, 4093, 3310, 4222, 4008] avg=4096.0000 max=5499 max_rank=0 min=3310 min_rank=5 imbalance_ratio=1.342529 max_load_reduction=27269 max_load_reduction_pct=0.832184 moved_to_echo=15633 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5248, 128, 128, 128, 27520, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[29312, 128, 896, 0, 2688, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[31488, 128, 128, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 128, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32512, 0, 0, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32384, 0, 128, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4089, 0, 0, 3, 3, 6, 27, 3, 4043, 4091, 0, 0, 0, 0, 0, 3, 4091, 4086, 4067, 3, 3, 0, 45, 5, 0, 0, 2, 4093, 0, 4093, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4091, 0, 88, 0, 0, 16, 0, 0, 3853, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 9, 135, 0, 0, 0, 0, 4096, 5, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4081, 0, 0, 11, 10, 0, 113, 9, 4085, 4086, 0, 0, 0, 0, 0, 11, 4077, 4071, 3972, 12, 9, 0, 7, 19, 0, 0, 7, 4084, 0, 4085, 19, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4089, 0, 94, 0, 0, 4, 0, 0, 3896, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 15, 94, 0, 0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4093, 2, 39, 0, 0, 1, 0, 0, 3637, 4091, 2, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 418, 0, 0, 0, 0, 4096, 4, 4092, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 22, 0, 0, 2, 0, 0, 3855, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 216, 0, 0, 0, 0, 4096, 1, 4095, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 10, 0, 0, 35, 0, 0, 3861, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 5, 186, 0, 0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4090, 0, 113, 0, 0, 32, 0, 0, 3681, 4093, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 5, 270, 0, 0, 0, 0, 4096, 4, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4047, 0, 0, 3, 0, 0, 0, 0, 3, 6, 27, 3, 4057, 0, 0, 0, 4043, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 1, 0, 0, 0, 19, 4067, 3, 0, 0, 0, 0, 3, 0, 45, 5, 3859, 42, 9, 2, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 4091, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4096, 4275, 4095, 4089, 3965, 4095, 4103] avg=4096.0000 max=4275 max_rank=2 min=3965 min_rank=5 imbalance_ratio=1.043701 max_load_reduction=28493 max_load_reduction_pct=0.869537 moved_to_echo=16252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4047, 0, 94, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4067, 0, 0, 0, 3896, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 15, 94, 0, 3863, 42, 9, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4071, 4129, 4097, 4115, 4025, 4096, 4094] avg=4096.0000 max=4141 max_rank=0 min=4025 min_rank=5 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4051, 2, 39, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4067, 0, 0, 0, 3637, 232, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 2, 418, 0, 3859, 42, 9, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4090, 0, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4068, 3871, 4097, 4115, 4332, 4096, 4097] avg=4096.0000 max=4332 max_rank=5 min=3871 min_rank=2 imbalance_ratio=1.057617 max_load_reduction=28436 max_load_reduction_pct=0.867798 moved_to_echo=16640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4048, 0, 88, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4073, 0, 0, 0, 3853, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 9, 135, 0, 3864, 43, 15, 9, 0, 0, 0, 4096, 6, 0, 0, 0, 5, 4080, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4089, 4081, 4099, 4101, 4075, 4102, 4085] avg=4096.0000 max=4136 max_rank=0 min=4075 min_rank=5 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16363 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4053, 0, 22, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4067, 0, 0, 0, 3855, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 2, 216, 0, 3862, 42, 9, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4093, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4069, 4088, 4097, 4115, 4133, 4096, 4095] avg=4096.0000 max=4133 max_rank=5 min=4069 min_rank=1 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4039, 0, 0, 11, 0, 0, 0, 0, 10, 0, 113, 9, 4042, 0, 0, 0, 4085, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4077, 1, 0, 0, 0, 19, 3972, 12, 0, 0, 0, 0, 9, 0, 7, 19, 3854, 42, 9, 2, 0, 0, 7, 4084, 0, 0, 0, 0, 0, 4083, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4174, 4317, 4089, 4003, 3942, 4091, 4102] avg=4096.0000 max=4317 max_rank=2 min=3942 min_rank=5 imbalance_ratio=1.053955 max_load_reduction=28451 max_load_reduction_pct=0.868256 moved_to_echo=16138 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4048, 0, 113, 0, 0, 0, 0, 0, 0, 32, 0, 0, 4067, 0, 0, 0, 3681, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 5, 270, 0, 3861, 42, 9, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4094, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4099, 3913, 4097, 4115, 4189, 4096, 4098] avg=4096.0000 max=4189 max_rank=5 min=3913 min_rank=2 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16498 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4053, 0, 10, 0, 0, 0, 0, 0, 0, 35, 0, 0, 4067, 0, 0, 0, 3861, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 5, 186, 0, 3863, 42, 9, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4102, 4094, 4097, 4115, 4107, 4096, 4094] avg=4096.0000 max=4115 max_rank=4 min=4063 min_rank=0 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 128, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[30976, 1920, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 1408, 128, 30976, 384, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 256, 128, 32512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 256, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 2691, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 72, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1613, 0, 0, 3816, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 2325, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2125, 0, 0, 3734, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 2054, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2353, 0, 0, 3779, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[14, 2554, 9, 2, 148, 10, 0, 4075, 7, 5, 11, 11, 2492, 12, 1, 0, 4059, 0, 142, 4085, 11, 0, 7, 4083, 1001, 0, 0, 2720, 8, 3217, 0, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[2, 2924, 3, 0, 149, 3, 2, 4089, 2, 0, 3, 3, 927, 4, 0, 0, 4090, 0, 58, 4093, 3, 0, 0, 4092, 1806, 0, 0, 2781, 2, 3639, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 2488, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 66, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1692, 0, 0, 3946, 0, 4094, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 3214, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 51, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1025, 0, 0, 3900, 0, 4095, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 2528, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1864, 0, 0, 3799, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2325, 0, 0, 1204, 290, 36, 19, 5, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 3639, 3, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2125, 0, 0, 2530, 0, 0, 0, 0, 0, 4096, 0, 167, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3874, 4065, 4093, 3645, 4077, 4096, 4655, 4263] avg=4096.0000 max=4655 max_rank=6 min=3645 min_rank=3 imbalance_ratio=1.136475 max_load_reduction=28113 max_load_reduction_pct=0.857941 moved_to_echo=17091 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 2924, 3, 0, 897, 289, 36, 19, 149, 3, 2, 4053, 0, 0, 0, 0, 2, 0, 3, 3, 4090, 0, 0, 0, 927, 4, 0, 0, 3637, 3, 0, 0, 4071, 0, 58, 0, 0, 0, 0, 0, 3, 0, 0, 4092, 0, 0, 0, 0, 1806, 0, 0, 1884, 0, 0, 0, 0, 2, 3639, 0, 167, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4207, 4098, 4571, 4129, 4095, 3690, 3808] avg=4096.0000 max=4571 max_rank=3 min=3690 min_rank=6 imbalance_ratio=1.115967 max_load_reduction=28197 max_load_reduction_pct=0.860504 moved_to_echo=15722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2691, 0, 0, 1231, 290, 36, 19, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 72, 0, 0, 0, 3639, 4, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1613, 0, 0, 2585, 0, 0, 0, 0, 0, 4096, 0, 167, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4267, 4060, 4092, 3715, 4077, 4096, 4198, 4263] avg=4096.0000 max=4267 max_rank=0 min=3715 min_rank=3 imbalance_ratio=1.041748 max_load_reduction=28501 max_load_reduction_pct=0.869781 moved_to_echo=16634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2488, 0, 0, 1272, 290, 36, 19, 2, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 66, 0, 0, 0, 3639, 4, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1692, 0, 0, 2674, 0, 0, 0, 0, 0, 4094, 0, 167, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4062, 4092, 3709, 4077, 4096, 4366, 4261] avg=4096.0000 max=4366 max_rank=6 min=3709 min_rank=3 imbalance_ratio=1.065918 max_load_reduction=28402 max_load_reduction_pct=0.866760 moved_to_echo=16800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2054, 0, 0, 1222, 291, 42, 24, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 6, 0, 0, 0, 3644, 3, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2353, 0, 0, 2557, 0, 0, 0, 0, 0, 4096, 0, 161, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3633, 4054, 4093, 3653, 4072, 4096, 4910, 4257] avg=4096.0000 max=4910 max_rank=6 min=3633 min_rank=0 imbalance_ratio=1.198730 max_load_reduction=27858 max_load_reduction_pct=0.850159 moved_to_echo=17335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[14, 2554, 9, 2, 877, 289, 36, 19, 148, 10, 0, 4039, 0, 0, 0, 0, 7, 5, 11, 11, 4082, 0, 0, 0, 2492, 12, 1, 0, 3629, 3, 0, 0, 4040, 0, 142, 0, 0, 0, 0, 0, 11, 0, 7, 4083, 0, 0, 0, 0, 1001, 0, 0, 1843, 0, 0, 0, 0, 8, 3217, 0, 166, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3800, 4197, 4116, 6137, 4182, 4101, 2844, 3391] avg=4096.0000 max=6137 max_rank=3 min=2844 min_rank=6 imbalance_ratio=1.498291 max_load_reduction=26631 max_load_reduction_pct=0.812714 moved_to_echo=14518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3214, 0, 0, 1258, 290, 36, 19, 3, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 51, 0, 0, 0, 3639, 4, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1025, 0, 0, 2642, 0, 0, 0, 0, 0, 4095, 0, 167, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4817, 4063, 4092, 3694, 4077, 4096, 3667, 4262] avg=4096.0000 max=4817 max_rank=0 min=3667 min_rank=6 imbalance_ratio=1.176025 max_load_reduction=27951 max_load_reduction_pct=0.852997 moved_to_echo=16102 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2528, 0, 0, 1225, 290, 36, 19, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 3639, 4, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1864, 0, 0, 2574, 0, 0, 0, 0, 0, 4096, 0, 167, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4060, 4092, 3644, 4077, 4096, 4438, 4263] avg=4096.0000 max=4438 max_rank=6 min=3644 min_rank=3 imbalance_ratio=1.083496 max_load_reduction=28330 max_load_reduction_pct=0.864563 moved_to_echo=16874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32764, 32768, 32768, 32768, 32768, 32772, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32764, 32768, 32768, 32768, 32768, 32772, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32764, 32768, 32768, 32768, 32768, 32772, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32764, 32768, 32768, 32768, 32768, 32772, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32764, 32768, 32768, 32768, 32768, 32772, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32764, 32768, 32768, 32768, 32768, 32772, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32764, 32768, 32768, 32768, 32768, 32772, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32764, 32768, 32768, 32768, 32768, 32772, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32764 actual_valid_recv=33536 observed_loss=-772 tokens_per_expert=[128, 20864, 128, 128, 9216, 2432, 384, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[13568, 0, 0, 19328, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3712, 128, 128, 0, 29184, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 128, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32640, 0, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 31488, 0, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32772 actual_valid_recv=33024 observed_loss=-252 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[465, 4093, 4092, 4023, 4, 6, 47, 4089, 2, 0, 0, 24, 3, 4073, 0, 6, 0, 3, 2, 0, 0, 4, 4071, 0, 0, 0, 4093, 3, 0, 3662, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1472, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 37, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2583, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1077, 4096, 4096, 4096, 0, 0, 0, 4082, 0, 0, 3, 15, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3018, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1117, 4096, 4096, 4096, 0, 0, 0, 4086, 0, 0, 8, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2981, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[653, 4096, 4096, 4096, 0, 0, 0, 4083, 0, 0, 0, 12, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3444, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1069, 4087, 4083, 3884, 15, 19, 71, 4049, 7, 0, 0, 1, 12, 4078, 0, 2, 0, 9, 10, 0, 0, 14, 4011, 0, 0, 0, 4086, 9, 0, 3242, 0, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[632, 4096, 4096, 4096, 0, 0, 0, 4085, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3474, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1368, 4096, 4096, 4096, 0, 0, 0, 4095, 0, 0, 6, 10, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2715, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[465, 0, 0, 3086, 0, 0, 0, 0, 4, 6, 47, 4082, 0, 0, 0, 0, 2, 0, 0, 24, 4077, 2, 0, 0, 3, 4073, 0, 6, 3, 0, 0, 0, 0, 3, 2, 0, 4091, 0, 0, 0, 0, 4, 4071, 0, 7, 4, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 3662, 0, 3, 930, 15, 0, 0] rerouted_tokens_per_ep_rank=[3551, 4139, 4105, 4085, 4096, 4086, 4096, 4610] avg=4096.0000 max=4610 max_rank=7 min=3551 min_rank=0 imbalance_ratio=1.125488 max_load_reduction=28158 max_load_reduction_pct=0.859314 moved_to_echo=16888 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1472, 0, 0, 3143, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 37, 4, 4080, 2, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 7, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2583, 0, 0, 946, 16, 0, 0] rerouted_tokens_per_ep_rank=[4615, 4089, 4123, 4099, 4094, 4107, 4096, 3545] avg=4096.0000 max=4615 max_rank=0 min=3545 min_rank=7 imbalance_ratio=1.126709 max_load_reduction=28153 max_load_reduction_pct=0.859161 moved_to_echo=15842 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1117, 0, 0, 3143, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 8, 1, 4080, 1, 0, 0, 0, 4095, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 7, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2981, 0, 0, 946, 16, 0, 0] rerouted_tokens_per_ep_rank=[4260, 4079, 4090, 4098, 4095, 4107, 4096, 3943] avg=4096.0000 max=4260 max_rank=0 min=3943 min_rank=7 imbalance_ratio=1.040039 max_load_reduction=28508 max_load_reduction_pct=0.869995 moved_to_echo=16241 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[653, 0, 0, 3143, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 12, 4080, 1, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 7, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3444, 0, 0, 946, 16, 0, 0] rerouted_tokens_per_ep_rank=[3796, 4077, 4093, 4098, 4095, 4107, 4096, 4406] avg=4096.0000 max=4406 max_rank=7 min=3796 min_rank=0 imbalance_ratio=1.075684 max_load_reduction=28362 max_load_reduction_pct=0.865540 moved_to_echo=16704 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1077, 0, 0, 3133, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 3, 15, 4081, 2, 0, 0, 0, 4093, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 10, 6, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 3018, 0, 0, 953, 15, 0, 0] rerouted_tokens_per_ep_rank=[4210, 4070, 4101, 4098, 4094, 4112, 4097, 3986] avg=4096.0000 max=4210 max_rank=0 min=3986 min_rank=7 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16289 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1368, 0, 0, 3143, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 6, 10, 4080, 1, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 7, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2715, 0, 0, 946, 16, 0, 0] rerouted_tokens_per_ep_rank=[4511, 4088, 4097, 4097, 4095, 4107, 4096, 3677] avg=4096.0000 max=4511 max_rank=0 min=3677 min_rank=7 imbalance_ratio=1.101318 max_load_reduction=28257 max_load_reduction_pct=0.862335 moved_to_echo=15975 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1069, 0, 0, 2981, 0, 0, 0, 0, 15, 19, 71, 4043, 0, 0, 0, 0, 7, 0, 0, 1, 4067, 2, 0, 0, 12, 4078, 0, 2, 2, 0, 0, 0, 0, 9, 10, 0, 4085, 0, 0, 0, 0, 14, 4011, 0, 6, 4, 0, 0, 0, 0, 4086, 9, 0, 0, 0, 0, 0, 3242, 0, 10, 897, 16, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4148, 4077, 4094, 4104, 4035, 4095, 4165] avg=4096.0000 max=4165 max_rank=7 min=4035 min_rank=5 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16399 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[632, 0, 0, 3143, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 1, 0, 4080, 1, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 7, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3474, 0, 0, 946, 16, 0, 0] rerouted_tokens_per_ep_rank=[3775, 4078, 4082, 4099, 4095, 4107, 4096, 4436] avg=4096.0000 max=4436 max_rank=7 min=3775 min_rank=0 imbalance_ratio=1.083008 max_load_reduction=28332 max_load_reduction_pct=0.864624 moved_to_echo=16734 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[7936, 0, 0, 24960, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 25216, 0, 128, 7552, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2743, 0, 1248, 562, 969, 2, 1151, 1110, 1505, 2579, 0, 176, 1197, 981, 689, 2120, 116, 1641, 903, 0, 2, 1620, 2590, 1510, 0, 868, 1598, 971, 960, 2437, 518, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2249, 0, 1804, 746, 797, 3, 1521, 773, 1769, 2358, 0, 61, 916, 1088, 546, 2880, 49, 1949, 948, 0, 0, 2529, 1979, 1977, 0, 654, 1245, 714, 592, 2024, 597, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2983, 0, 1005, 549, 769, 0, 1029, 1198, 803, 2549, 1, 50, 623, 1507, 820, 2509, 88, 1522, 1329, 0, 1, 2323, 2397, 1406, 0, 989, 878, 695, 1499, 2245, 1000, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2532, 0, 1634, 1220, 731, 0, 1535, 785, 1067, 2559, 0, 104, 1249, 824, 689, 2270, 83, 1627, 674, 1, 0, 1863, 2299, 1992, 0, 1056, 1082, 1144, 1084, 2177, 487, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2836, 1, 1365, 873, 870, 0, 1268, 939, 982, 2170, 1, 139, 882, 895, 627, 2160, 91, 1784, 901, 1, 4, 2083, 3065, 1915, 2, 836, 1478, 872, 886, 2363, 479, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2886, 1, 1128, 1007, 994, 0, 1747, 583, 988, 2212, 0, 135, 1009, 1078, 786, 2514, 96, 1890, 605, 0, 0, 1608, 2806, 1892, 11, 1165, 1286, 997, 776, 1991, 577, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2389, 0, 1518, 746, 745, 0, 1538, 699, 981, 2677, 1, 98, 1131, 918, 619, 2021, 84, 1654, 911, 0, 0, 1705, 2888, 1833, 0, 725, 1211, 1403, 1139, 2849, 282, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2146, 0, 1964, 1279, 1453, 6, 572, 944, 1119, 1131, 0, 178, 994, 885, 446, 2197, 193, 1328, 1889, 0, 0, 1882, 3052, 2880, 1, 481, 1622, 1898, 216, 1688, 324, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1723, 0, 1634, 1220, 0, 0, 0, 0, 731, 0, 1535, 785, 876, 0, 0, 0, 1067, 2559, 0, 104, 374, 158, 0, 0, 1249, 824, 689, 1383, 0, 0, 0, 0, 83, 1627, 674, 1, 1135, 0, 0, 0, 0, 1863, 192, 1992, 0, 0, 0, 0, 0, 1056, 1082, 1144, 729, 96, 0, 0, 1084, 2177, 487, 0, 435, 0, 0, 0] rerouted_tokens_per_ep_rank=[4577, 3927, 4262, 4145, 3520, 4047, 4107, 4183] avg=4096.0000 max=4577 max_rank=0 min=3520 min_rank=4 imbalance_ratio=1.117432 max_load_reduction=28191 max_load_reduction_pct=0.860321 moved_to_echo=15857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1531, 0, 1804, 746, 0, 0, 0, 0, 797, 3, 1521, 773, 754, 0, 0, 0, 1769, 2358, 0, 61, 332, 200, 0, 0, 916, 1088, 546, 1755, 0, 0, 0, 0, 49, 1949, 948, 0, 977, 0, 0, 0, 0, 2529, 165, 1977, 0, 0, 0, 0, 0, 654, 1245, 714, 925, 83, 0, 0, 592, 2024, 597, 0, 386, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 3848, 4720, 4305, 3923, 4671, 3621, 3599] avg=4096.0000 max=4720 max_rank=2 min=3599 min_rank=7 imbalance_ratio=1.152344 max_load_reduction=28048 max_load_reduction_pct=0.855957 moved_to_echo=15814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1866, 0, 1248, 562, 0, 0, 0, 0, 969, 2, 1151, 1110, 987, 0, 0, 0, 1505, 2579, 0, 176, 406, 147, 0, 0, 1197, 981, 689, 1292, 0, 0, 0, 0, 116, 1641, 903, 0, 1279, 0, 0, 0, 2, 1620, 216, 1510, 0, 0, 0, 0, 0, 868, 1598, 971, 681, 108, 0, 0, 960, 2437, 518, 2, 471, 0, 0, 0] rerouted_tokens_per_ep_rank=[3676, 4219, 4813, 4159, 3939, 3348, 4226, 4388] avg=4096.0000 max=4813 max_rank=2 min=3348 min_rank=5 imbalance_ratio=1.175049 max_load_reduction=27955 max_load_reduction_pct=0.853119 moved_to_echo=15901 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1626, 0, 1518, 746, 0, 0, 0, 0, 745, 0, 1538, 699, 1101, 0, 0, 0, 981, 2677, 1, 98, 353, 140, 0, 0, 1131, 918, 619, 1232, 0, 0, 0, 0, 84, 1654, 911, 0, 1426, 0, 0, 0, 0, 1705, 240, 1833, 0, 0, 0, 0, 0, 725, 1211, 1403, 649, 121, 0, 0, 1139, 2849, 282, 3, 410, 0, 0, 0] rerouted_tokens_per_ep_rank=[3890, 4083, 4250, 3900, 4075, 3778, 4109, 4683] avg=4096.0000 max=4683 max_rank=7 min=3778 min_rank=5 imbalance_ratio=1.143311 max_load_reduction=28085 max_load_reduction_pct=0.857086 moved_to_echo=16645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1930, 1, 1365, 873, 0, 0, 0, 0, 870, 0, 1268, 939, 1168, 0, 0, 0, 982, 2170, 1, 139, 419, 150, 0, 0, 882, 895, 627, 1316, 0, 0, 0, 0, 91, 1784, 901, 1, 1514, 0, 0, 0, 4, 2083, 255, 1915, 0, 0, 0, 0, 2, 836, 1478, 872, 694, 128, 0, 0, 886, 2363, 479, 0, 487, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4245, 3861, 3720, 4291, 4257, 4010, 4215] avg=4096.0000 max=4291 max_rank=4 min=3720 min_rank=3 imbalance_ratio=1.047607 max_load_reduction=28477 max_load_reduction_pct=0.869049 moved_to_echo=16773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1963, 1, 1128, 1007, 0, 0, 0, 0, 994, 0, 1747, 583, 1070, 0, 0, 0, 988, 2212, 0, 135, 427, 175, 0, 0, 1009, 1078, 786, 1531, 0, 0, 0, 0, 96, 1890, 605, 0, 1386, 0, 0, 0, 0, 1608, 233, 1892, 0, 0, 0, 0, 11, 1165, 1286, 997, 808, 117, 0, 0, 776, 1991, 577, 0, 496, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4394, 3937, 4404, 3977, 3733, 4384, 3840] avg=4096.0000 max=4404 max_rank=3 min=3733 min_rank=5 imbalance_ratio=1.075195 max_load_reduction=28364 max_load_reduction_pct=0.865601 moved_to_echo=15934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2029, 0, 1005, 549, 0, 0, 0, 0, 769, 0, 1029, 1198, 914, 0, 0, 0, 803, 2549, 1, 50, 441, 174, 0, 0, 623, 1507, 820, 1529, 0, 0, 0, 0, 88, 1522, 1329, 0, 1184, 0, 0, 0, 1, 2323, 199, 1406, 0, 0, 0, 0, 0, 989, 878, 695, 806, 100, 0, 0, 1499, 2245, 1000, 1, 513, 0, 0, 0] rerouted_tokens_per_ep_rank=[3583, 3910, 4018, 4479, 4123, 3929, 3468, 5258] avg=4096.0000 max=5258 max_rank=7 min=3468 min_rank=6 imbalance_ratio=1.283691 max_load_reduction=27510 max_load_reduction_pct=0.839539 moved_to_echo=16778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1450, 0, 1964, 1279, 0, 0, 0, 0, 1453, 6, 572, 944, 1167, 0, 0, 0, 1119, 1131, 0, 178, 322, 157, 0, 0, 994, 885, 446, 1331, 0, 0, 0, 0, 193, 1328, 1889, 0, 1510, 0, 0, 0, 0, 1882, 243, 2880, 0, 0, 0, 0, 1, 481, 1622, 1898, 709, 132, 0, 0, 216, 1688, 324, 0, 374, 0, 0, 0] rerouted_tokens_per_ep_rank=[4693, 4142, 2907, 3656, 4920, 5005, 4843, 2602] avg=4096.0000 max=5005 max_rank=5 min=2602 min_rank=7 imbalance_ratio=1.221924 max_load_reduction=27763 max_load_reduction_pct=0.847260 moved_to_echo=17370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7424, 128, 10368, 7040, 8064, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14208, 128, 11776, 7040, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 6784, 10496, 8704, 6016, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9216, 18304, 128, 1024, 3200, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 13440, 8192, 128, 10496, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 15616, 1792, 15488, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8192, 5248, 11392, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7168, 17792, 4352, 128, 3584, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2228, 1, 0, 12, 4087, 0, 1, 0, 1, 2614, 4094, 4095, 4059, 9, 6, 8, 4095, 0, 0, 1, 3320, 1, 0, 0, 4094, 1, 0, 1, 0, 39, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2790, 0, 0, 0, 4073, 0, 1, 0, 0, 2612, 4095, 4073, 3965, 2, 5, 60, 4095, 0, 0, 0, 2562, 0, 0, 0, 4092, 202, 1, 0, 0, 139, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[244, 0, 0, 0, 3907, 0, 0, 1, 0, 3977, 4050, 4094, 4079, 1, 2, 208, 4096, 0, 0, 0, 3970, 0, 0, 0, 4092, 36, 0, 0, 1, 8, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1108, 1, 0, 1, 3762, 0, 0, 0, 0, 3759, 4086, 4095, 4083, 8, 0, 15, 4095, 1, 0, 1, 3484, 0, 0, 0, 4083, 173, 0, 0, 0, 13, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1830, 0, 0, 0, 4052, 2, 1, 1, 0, 2632, 4092, 4093, 4004, 22, 4, 13, 4096, 0, 0, 0, 3680, 0, 2, 0, 4081, 45, 0, 1, 0, 117, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1102, 0, 0, 1, 4010, 0, 0, 1, 1, 3504, 4094, 4096, 3957, 29, 1, 20, 4094, 0, 0, 0, 3662, 0, 0, 0, 4090, 56, 0, 0, 0, 49, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1479, 1, 0, 1, 3749, 0, 0, 0, 0, 3323, 4092, 4094, 3902, 27, 2, 14, 4095, 0, 1, 1, 3545, 1, 0, 0, 4071, 225, 0, 0, 1, 143, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[971, 1, 3, 0, 3589, 1, 1, 1, 0, 3510, 4093, 4094, 3966, 144, 1, 139, 4095, 0, 1, 0, 3865, 0, 0, 0, 4094, 131, 0, 0, 1, 67, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2228, 1, 0, 12, 2559, 69, 0, 0, 4087, 0, 1, 0, 98, 92, 0, 0, 1, 2614, 858, 0, 0, 0, 0, 0, 4059, 9, 6, 8, 1, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 3320, 1, 0, 0, 585, 0, 0, 0, 3995, 1, 0, 1, 0, 0, 0, 0, 0, 39, 0, 1, 4026, 0, 0, 0] rerouted_tokens_per_ep_rank=[4869, 4278, 3473, 4083, 4096, 3906, 3997, 4066] avg=4096.0000 max=4869 max_rank=0 min=3473 min_rank=2 imbalance_ratio=1.188721 max_load_reduction=27899 max_load_reduction_pct=0.851410 moved_to_echo=16065 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2790, 0, 0, 0, 2560, 69, 0, 0, 4073, 0, 1, 0, 98, 92, 0, 0, 0, 2612, 858, 0, 0, 0, 0, 0, 3965, 2, 5, 60, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2562, 0, 0, 0, 585, 0, 0, 0, 3993, 202, 1, 0, 0, 0, 0, 0, 0, 139, 0, 1, 4004, 0, 0, 0] rerouted_tokens_per_ep_rank=[5419, 4264, 3470, 4033, 4095, 3147, 4196, 4144] avg=4096.0000 max=5419 max_rank=0 min=3147 min_rank=5 imbalance_ratio=1.322998 max_load_reduction=27349 max_load_reduction_pct=0.834625 moved_to_echo=15582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1102, 0, 0, 1, 2559, 69, 0, 0, 4010, 0, 0, 1, 98, 92, 0, 0, 1, 3504, 858, 0, 0, 0, 0, 0, 3957, 29, 1, 20, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 3662, 0, 0, 0, 585, 0, 0, 0, 3991, 56, 0, 0, 0, 0, 0, 0, 0, 49, 1, 0, 4027, 0, 0, 0] rerouted_tokens_per_ep_rank=[3731, 4201, 4363, 4008, 4094, 4247, 4047, 4077] avg=4096.0000 max=4363 max_rank=2 min=3731 min_rank=0 imbalance_ratio=1.065186 max_load_reduction=28405 max_load_reduction_pct=0.866852 moved_to_echo=16465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1830, 0, 0, 0, 2558, 69, 0, 0, 4052, 2, 1, 1, 98, 92, 0, 0, 0, 2632, 857, 0, 0, 0, 0, 0, 4004, 22, 4, 13, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3680, 0, 2, 0, 585, 0, 0, 0, 3982, 45, 0, 1, 0, 0, 0, 0, 0, 117, 0, 0, 4024, 0, 0, 0] rerouted_tokens_per_ep_rank=[4457, 4246, 3489, 4044, 4096, 4267, 4028, 4141] avg=4096.0000 max=4457 max_rank=0 min=3489 min_rank=2 imbalance_ratio=1.088135 max_load_reduction=28311 max_load_reduction_pct=0.863983 moved_to_echo=16532 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[244, 0, 0, 0, 2537, 69, 0, 0, 3907, 0, 0, 1, 102, 97, 0, 0, 0, 3977, 834, 0, 0, 0, 0, 0, 4079, 1, 2, 208, 6, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 3970, 0, 0, 0, 582, 0, 0, 0, 3983, 36, 0, 0, 0, 0, 0, 0, 1, 8, 1, 1, 4025, 0, 0, 0] rerouted_tokens_per_ep_rank=[2850, 4107, 4811, 4296, 4097, 4552, 4019, 4036] avg=4096.0000 max=4811 max_rank=2 min=2850 min_rank=0 imbalance_ratio=1.174561 max_load_reduction=27957 max_load_reduction_pct=0.853180 moved_to_echo=16704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1479, 1, 0, 1, 2558, 68, 0, 0, 3749, 0, 0, 0, 97, 92, 0, 0, 0, 3323, 857, 0, 0, 0, 0, 0, 3902, 27, 2, 14, 1, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 3545, 1, 0, 0, 585, 0, 0, 0, 3973, 225, 0, 0, 0, 0, 0, 0, 1, 143, 1, 0, 4026, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 3938, 4180, 3946, 4097, 4131, 4198, 4171] avg=4096.0000 max=4198 max_rank=6 min=3938 min_rank=1 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16597 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1108, 1, 0, 1, 2554, 69, 0, 0, 3762, 0, 0, 0, 98, 92, 0, 0, 0, 3759, 856, 0, 0, 0, 0, 0, 4083, 8, 0, 15, 1, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 0, 3484, 0, 0, 0, 584, 0, 0, 0, 3984, 173, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4026, 0, 0, 0] rerouted_tokens_per_ep_rank=[3733, 3952, 4615, 4107, 4097, 4068, 4157, 4039] avg=4096.0000 max=4615 max_rank=2 min=3733 min_rank=0 imbalance_ratio=1.126709 max_load_reduction=28153 max_load_reduction_pct=0.859161 moved_to_echo=16361 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[971, 1, 3, 0, 2559, 68, 0, 0, 3589, 1, 1, 1, 98, 92, 0, 0, 0, 3510, 857, 0, 0, 0, 0, 0, 3966, 144, 1, 139, 1, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 3865, 0, 0, 0, 585, 0, 0, 0, 3995, 131, 0, 0, 0, 0, 0, 0, 1, 67, 0, 0, 4026, 0, 0, 0] rerouted_tokens_per_ep_rank=[3602, 3782, 4367, 4251, 4096, 4450, 4126, 4094] avg=4096.0000 max=4450 max_rank=5 min=3602 min_rank=0 imbalance_ratio=1.086426 max_load_reduction=28318 max_load_reduction_pct=0.864197 moved_to_echo=16766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11776, 128, 128, 128, 20480, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 25984, 6912, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[28160, 128, 128, 0, 4736, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[31232, 128, 128, 128, 896, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32000, 896, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 640, 128, 128, 32256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32128, 256, 128, 512, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2513, 1, 4072, 0, 2100, 427, 12, 0, 4092, 1, 4096, 1, 4085, 0, 1, 1, 0, 1, 0, 21, 4093, 3741, 0, 371, 376, 0, 0, 2628, 0, 135] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[2, 3, 810, 0, 4065, 0, 1553, 901, 40, 2, 4092, 0, 4092, 0, 4078, 2, 0, 0, 0, 0, 0, 43, 4096, 2967, 0, 358, 2012, 13, 0, 3090, 0, 549] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 2316, 0, 4086, 0, 1305, 240, 101, 2, 4062, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 86, 4095, 3030, 0, 648, 1945, 3, 7, 2488, 0, 167] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1, 1, 1655, 0, 4083, 7, 1670, 879, 82, 0, 4091, 0, 4093, 0, 4096, 0, 1, 1, 0, 0, 0, 44, 4096, 3893, 0, 269, 726, 0, 1, 2979, 0, 100] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 209, 0, 3747, 0, 2400, 342, 16, 1, 4090, 0, 4095, 0, 4086, 0, 1, 0, 1, 0, 0, 191, 4095, 3364, 0, 481, 1392, 6, 0, 3305, 0, 946] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1455, 1, 4093, 0, 1548, 213, 61, 0, 4083, 1, 4095, 1, 4094, 0, 1, 1, 0, 1, 0, 19, 4095, 3176, 3, 531, 1833, 1, 0, 3215, 1, 246] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 2345, 1, 4033, 2, 812, 200, 10, 0, 4063, 0, 4095, 0, 4089, 0, 0, 0, 1, 0, 0, 28, 4095, 3115, 0, 141, 2296, 41, 1, 2925, 0, 474] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 883, 1, 4021, 0, 836, 146, 17, 0, 4082, 0, 4078, 0, 4035, 4, 0, 1, 0, 1, 1, 31, 4095, 2535, 0, 180, 3564, 3, 0, 3029, 1, 1223] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2513, 1, 2569, 0, 0, 0, 2174, 0, 2100, 427, 0, 0, 0, 0, 12, 0, 4063, 1, 0, 0, 0, 0, 14, 1, 4085, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4082, 15, 0, 0, 0, 21, 811, 3741, 0, 0, 0, 0, 0, 371, 376, 0, 1263, 698, 0, 0, 0, 2628, 0, 135, 635, 29, 0, 0] rerouted_tokens_per_ep_rank=[5083, 4701, 4076, 4100, 4100, 4573, 2708, 3427] avg=4096.0000 max=5083 max_rank=0 min=2708 min_rank=6 imbalance_ratio=1.240967 max_load_reduction=27685 max_load_reduction_pct=0.844879 moved_to_echo=14808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 810, 0, 2571, 0, 0, 0, 2171, 0, 1553, 901, 0, 0, 0, 0, 40, 2, 4063, 0, 0, 0, 0, 0, 13, 0, 4078, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 15, 0, 0, 0, 43, 812, 2967, 0, 0, 0, 0, 0, 358, 2012, 13, 1260, 698, 0, 0, 0, 3090, 0, 549, 634, 29, 0, 0] rerouted_tokens_per_ep_rank=[3386, 4625, 4105, 4093, 4094, 3822, 4341, 4302] avg=4096.0000 max=4625 max_rank=1 min=3386 min_rank=0 imbalance_ratio=1.129150 max_load_reduction=28143 max_load_reduction_pct=0.858856 moved_to_echo=16559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1655, 0, 2571, 0, 0, 0, 2180, 7, 1670, 879, 0, 0, 0, 0, 82, 0, 4062, 0, 0, 0, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4079, 15, 0, 0, 0, 44, 812, 3893, 0, 0, 0, 0, 0, 269, 726, 0, 1266, 698, 0, 0, 1, 2979, 0, 100, 637, 29, 0, 0] rerouted_tokens_per_ep_rank=[4228, 4736, 4144, 4110, 4096, 4749, 2959, 3746] avg=4096.0000 max=4749 max_rank=5 min=2959 min_rank=6 imbalance_ratio=1.159424 max_load_reduction=28019 max_load_reduction_pct=0.855072 moved_to_echo=15550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2316, 0, 2571, 0, 0, 0, 2182, 0, 1305, 240, 0, 0, 0, 0, 101, 2, 4034, 0, 0, 0, 0, 0, 14, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 15, 0, 0, 0, 86, 811, 3030, 0, 0, 0, 0, 0, 648, 1945, 3, 1267, 698, 0, 0, 7, 2488, 0, 167, 637, 28, 0, 0] rerouted_tokens_per_ep_rank=[4888, 3727, 4137, 4107, 4094, 3927, 4561, 3327] avg=4096.0000 max=4888 max_rank=0 min=3327 min_rank=7 imbalance_ratio=1.193359 max_load_reduction=27880 max_load_reduction_pct=0.850830 moved_to_echo=15909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 209, 0, 2573, 0, 0, 0, 1993, 0, 2400, 342, 0, 0, 0, 0, 16, 1, 4058, 0, 0, 0, 0, 0, 8, 0, 4086, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4087, 17, 0, 0, 0, 191, 804, 3364, 0, 0, 0, 0, 0, 481, 1392, 6, 1166, 701, 0, 0, 0, 3305, 0, 946, 588, 32, 0, 0] rerouted_tokens_per_ep_rank=[2782, 4735, 4075, 4094, 4106, 4359, 3746, 4871] avg=4096.0000 max=4871 max_rank=7 min=2782 min_rank=0 imbalance_ratio=1.189209 max_load_reduction=27897 max_load_reduction_pct=0.851349 moved_to_echo=17082 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2345, 1, 2571, 0, 0, 0, 2154, 2, 812, 200, 0, 0, 0, 0, 10, 0, 4035, 0, 0, 0, 0, 0, 14, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4081, 15, 0, 0, 0, 28, 811, 3115, 0, 0, 0, 0, 0, 141, 2296, 41, 1250, 698, 0, 0, 1, 2925, 0, 474, 629, 28, 0, 0] rerouted_tokens_per_ep_rank=[4918, 3168, 4045, 4103, 4097, 3954, 4426, 4057] avg=4096.0000 max=4918 max_rank=0 min=3168 min_rank=1 imbalance_ratio=1.200684 max_load_reduction=27850 max_load_reduction_pct=0.849915 moved_to_echo=16534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 883, 1, 2571, 0, 0, 0, 2147, 0, 836, 146, 0, 0, 0, 0, 17, 0, 4053, 0, 0, 0, 0, 0, 13, 0, 4035, 4, 0, 0, 0, 0, 0, 1, 0, 1, 4065, 15, 0, 0, 1, 31, 811, 2535, 0, 0, 0, 0, 0, 180, 3564, 3, 1247, 698, 0, 0, 0, 3029, 1, 1223, 627, 29, 0, 0] rerouted_tokens_per_ep_rank=[3456, 3129, 4070, 4052, 4082, 3378, 5692, 4909] avg=4096.0000 max=5692 max_rank=6 min=3129 min_rank=1 imbalance_ratio=1.389648 max_load_reduction=27076 max_load_reduction_pct=0.826294 moved_to_echo=18061 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1455, 1, 2571, 0, 0, 0, 2186, 0, 1548, 213, 0, 0, 0, 0, 61, 0, 4054, 1, 0, 0, 0, 0, 14, 1, 4094, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4081, 15, 0, 0, 0, 19, 811, 3176, 0, 0, 0, 0, 3, 531, 1833, 1, 1269, 698, 0, 0, 0, 3215, 1, 246, 638, 29, 0, 0] rerouted_tokens_per_ep_rank=[4027, 3947, 4116, 4109, 4099, 4006, 4335, 4129] avg=4096.0000 max=4335 max_rank=6 min=3947 min_rank=1 imbalance_ratio=1.058350 max_load_reduction=28433 max_load_reduction_pct=0.867706 moved_to_echo=16569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[17280, 128, 12288, 3456, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 23680, 128, 3840, 5120, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 3072, 14208, 128, 10112, 5632, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 512, 6528, 25856, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 12288, 128, 20608, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 32512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 30, 2306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3496, 0, 4069, 2, 0, 0, 56, 4, 2596, 3988, 3995, 4093, 0, 8, 4029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 0, 1, 0, 8, 493, 0, 0, 3, 1, 1, 1, 0, 1, 0, 4088, 3866, 1, 4074, 0, 1, 0, 68, 7, 3849, 4085, 4082, 4090, 0, 0, 4044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 53, 1882, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 3485, 0, 4087, 1, 0, 0, 48, 0, 2933, 4020, 4068, 4090, 0, 0, 4002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 73, 2251, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 3027, 0, 4087, 1, 0, 1, 34, 2, 2977, 4045, 4062, 4096, 0, 1, 4014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 39, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 3493, 0, 4092, 1, 0, 0, 88, 3, 3112, 4048, 4023, 4092, 0, 5, 4075] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 62, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3604, 0, 4076, 1, 0, 0, 87, 11, 3469, 4068, 4083, 4095, 0, 11, 4049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 45, 2436, 0, 0, 7, 1, 1, 1, 0, 1, 0, 4095, 2772, 1, 4091, 0, 0, 0, 224, 0, 2953, 4004, 4009, 4090, 1, 14, 4020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[9, 0, 1, 1, 1, 9, 1429, 0, 0, 0, 1, 0, 1, 0, 1, 0, 4094, 3412, 1, 4086, 0, 1, 0, 10, 4, 3416, 4057, 4072, 4090, 0, 0, 4072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 1, 4082, 44, 0, 0, 0, 8, 493, 0, 2401, 0, 0, 0, 0, 3, 1, 1, 3992, 99, 0, 0, 1, 0, 1, 0, 4088, 0, 0, 0, 0, 3866, 1, 701, 0, 0, 0, 0, 0, 1, 0, 68, 3274, 744, 0, 0, 7, 3849, 940, 0, 0, 0, 0, 0, 54, 0, 0, 4044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 2902, 4096, 4090, 4568, 4087, 4796, 4098] avg=4096.0000 max=4796 max_rank=6 min=2902 min_rank=1 imbalance_ratio=1.170898 max_load_reduction=27972 max_load_reduction_pct=0.853638 moved_to_echo=17549 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3995, 45, 0, 0, 0, 30, 2306, 0, 2344, 0, 0, 0, 0, 0, 0, 0, 3995, 99, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3496, 0, 700, 0, 0, 0, 0, 2, 0, 0, 56, 3270, 726, 0, 0, 4, 2596, 918, 0, 0, 0, 0, 0, 53, 0, 8, 4029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4680, 4094, 4096, 4196, 4054, 3518, 4090] avg=4096.0000 max=4680 max_rank=1 min=3518 min_rank=6 imbalance_ratio=1.142578 max_load_reduction=28088 max_load_reduction_pct=0.857178 moved_to_echo=15858 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4067, 44, 0, 0, 0, 53, 1882, 0, 2362, 0, 0, 0, 0, 3, 0, 0, 3992, 99, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3485, 0, 703, 0, 0, 0, 0, 1, 0, 0, 48, 3285, 732, 0, 0, 0, 2933, 926, 0, 0, 0, 0, 0, 54, 0, 0, 4002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4297, 4094, 4097, 4188, 4066, 3859, 4056] avg=4096.0000 max=4297 max_rank=1 min=3859 min_rank=6 imbalance_ratio=1.049072 max_load_reduction=28471 max_load_reduction_pct=0.868866 moved_to_echo=16169 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4082, 45, 0, 0, 0, 62, 1055, 0, 2391, 0, 0, 0, 0, 0, 0, 0, 3997, 99, 0, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 3604, 0, 701, 0, 0, 0, 0, 1, 0, 0, 87, 3276, 741, 0, 0, 11, 3469, 936, 0, 0, 0, 0, 0, 53, 0, 11, 4049, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 3508, 4096, 4098, 4305, 4105, 4416, 4113] avg=4096.0000 max=4416 max_rank=6 min=3508 min_rank=1 imbalance_ratio=1.078125 max_load_reduction=28352 max_load_reduction_pct=0.865234 moved_to_echo=16939 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4062, 49, 0, 0, 0, 73, 2251, 0, 2381, 0, 0, 0, 0, 1, 0, 0, 3999, 103, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3027, 0, 694, 0, 0, 0, 0, 1, 0, 1, 34, 3290, 740, 0, 0, 2, 2977, 924, 0, 0, 0, 0, 0, 48, 0, 1, 4014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4705, 4103, 4096, 3721, 4066, 3903, 4063] avg=4096.0000 max=4705 max_rank=1 min=3721 min_rank=4 imbalance_ratio=1.148682 max_load_reduction=28063 max_load_reduction_pct=0.856415 moved_to_echo=15753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4008, 44, 0, 0, 0, 45, 2436, 0, 2353, 0, 0, 0, 0, 7, 1, 1, 3992, 99, 0, 0, 1, 0, 1, 0, 4095, 1, 0, 0, 0, 2772, 1, 704, 0, 0, 0, 0, 0, 0, 0, 224, 3288, 729, 0, 0, 0, 2953, 922, 0, 0, 0, 0, 0, 54, 1, 14, 4020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4834, 4100, 4098, 3477, 4241, 3875, 4089] avg=4096.0000 max=4834 max_rank=1 min=3477 min_rank=4 imbalance_ratio=1.180176 max_load_reduction=27934 max_load_reduction_pct=0.852478 moved_to_echo=15682 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4023, 45, 0, 0, 0, 39, 1601, 0, 2379, 0, 0, 0, 0, 0, 0, 0, 3994, 99, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 3493, 0, 704, 0, 0, 0, 0, 1, 0, 0, 88, 3289, 737, 0, 0, 3, 3112, 932, 0, 0, 0, 0, 0, 53, 0, 5, 4075, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4019, 4093, 4096, 4197, 4115, 4047, 4133] avg=4096.0000 max=4197 max_rank=4 min=4019 min_rank=1 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16492 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 1, 1, 4071, 44, 0, 0, 1, 9, 1429, 0, 2384, 0, 0, 0, 0, 0, 1, 0, 3992, 99, 0, 0, 1, 0, 1, 0, 4094, 1, 0, 0, 0, 3412, 1, 703, 0, 0, 0, 0, 0, 1, 0, 10, 3284, 739, 0, 0, 4, 3416, 934, 0, 0, 0, 0, 0, 54, 0, 0, 4072, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 3823, 4092, 4097, 4116, 4034, 4354, 4126] avg=4096.0000 max=4354 max_rank=6 min=3823 min_rank=1 imbalance_ratio=1.062988 max_load_reduction=28414 max_load_reduction_pct=0.867126 moved_to_echo=16630 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 25344, 7552, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 384, 13568, 0, 19072, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 640, 26368, 5888, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32000, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 128, 128, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 27264, 128, 5632, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32512, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 4091, 4, 4095, 0, 1, 718, 0, 4095, 1, 0, 0, 4095, 703, 0, 0, 0, 1, 37, 0, 1, 1342, 0, 3907, 0, 1483, 1, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4096, 6, 4096, 0, 0, 1726, 0, 4093, 0, 0, 0, 4096, 704, 0, 0, 0, 0, 9, 0, 5, 1386, 3, 3989, 0, 368, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4094, 7, 4095, 0, 0, 563, 1, 4095, 0, 0, 0, 4096, 663, 0, 0, 0, 0, 2, 0, 20, 2461, 1, 4037, 0, 442, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 6, 4096, 0, 0, 2236, 0, 4094, 0, 0, 0, 4096, 715, 0, 0, 0, 0, 0, 0, 0, 988, 2, 4001, 0, 246, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 6, 4096, 0, 0, 1332, 0, 4094, 0, 0, 0, 4096, 82, 0, 0, 0, 0, 104, 0, 356, 1122, 2, 4026, 1, 1163, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4087, 2, 4096, 0, 0, 2163, 0, 4091, 0, 0, 0, 4096, 512, 0, 0, 0, 0, 21, 0, 30, 714, 6, 3953, 0, 805, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 4091, 2, 4095, 0, 1, 3257, 0, 4095, 1, 0, 1, 4095, 337, 0, 0, 0, 1, 19, 0, 4, 302, 0, 3929, 0, 344, 1, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 4088, 32, 4095, 0, 1, 2255, 0, 4095, 1, 0, 0, 4095, 538, 0, 0, 0, 1, 5, 0, 9, 741, 1, 4012, 2, 603, 1, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 1, 718, 0, 2315, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 703, 0, 0, 0, 1780, 1000, 681, 78, 1, 37, 0, 1, 4017, 0, 0, 0, 1342, 0, 2907, 0, 0, 0, 0, 0, 1483, 1, 0, 3414, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4095, 3034, 4096, 4242, 4056, 4249, 4898] avg=4096.0000 max=4898 max_rank=7 min=3034 min_rank=2 imbalance_ratio=1.195801 max_load_reduction=27870 max_load_reduction_pct=0.850525 moved_to_echo=17445 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 563, 1, 2315, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 663, 0, 0, 0, 1780, 1034, 681, 78, 0, 2, 0, 20, 4017, 0, 0, 0, 2461, 1, 3003, 0, 0, 0, 0, 0, 442, 0, 0, 3414, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 2879, 4096, 4236, 4039, 5465, 3856] avg=4096.0000 max=5465 max_rank=6 min=2879 min_rank=2 imbalance_ratio=1.334229 max_load_reduction=27303 max_load_reduction_pct=0.833221 moved_to_echo=17596 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 1726, 0, 2314, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 704, 0, 0, 0, 1779, 1021, 681, 78, 0, 9, 0, 5, 4018, 0, 0, 0, 1386, 3, 2968, 0, 0, 0, 0, 0, 368, 0, 0, 3415, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4102, 4040, 4096, 4263, 4032, 4357, 3783] avg=4096.0000 max=4357 max_rank=6 min=3783 min_rank=7 imbalance_ratio=1.063721 max_load_reduction=28411 max_load_reduction_pct=0.867035 moved_to_echo=16435 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 2163, 0, 2313, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 512, 0, 0, 0, 1778, 1012, 681, 77, 0, 21, 0, 30, 4019, 0, 0, 0, 714, 6, 2941, 0, 0, 0, 0, 0, 805, 0, 0, 3415, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4089, 4476, 4096, 4060, 4070, 3661, 4220] avg=4096.0000 max=4476 max_rank=2 min=3661 min_rank=6 imbalance_ratio=1.092773 max_load_reduction=28292 max_load_reduction_pct=0.863403 moved_to_echo=16011 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 1332, 0, 2312, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 82, 0, 0, 0, 1782, 1036, 686, 78, 0, 104, 0, 356, 4018, 0, 0, 0, 1122, 2, 2990, 1, 0, 0, 0, 0, 1163, 0, 0, 3410, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4102, 3644, 4096, 3664, 4478, 4115, 4573] avg=4096.0000 max=4573 max_rank=7 min=3644 min_rank=2 imbalance_ratio=1.116455 max_load_reduction=28195 max_load_reduction_pct=0.860443 moved_to_echo=16830 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 2236, 0, 2315, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 715, 0, 0, 0, 1779, 1024, 681, 78, 0, 0, 0, 0, 4018, 0, 0, 0, 988, 2, 2977, 0, 0, 0, 0, 0, 246, 0, 0, 3415, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4102, 4551, 4096, 4277, 4018, 3967, 3661] avg=4096.0000 max=4551 max_rank=2 min=3661 min_rank=7 imbalance_ratio=1.111084 max_load_reduction=28217 max_load_reduction_pct=0.861115 moved_to_echo=15923 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 1, 3257, 0, 2315, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 337, 0, 0, 0, 1780, 1006, 681, 77, 1, 19, 0, 4, 4018, 0, 0, 0, 302, 0, 2923, 0, 0, 0, 0, 0, 344, 1, 0, 3414, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4093, 5573, 4097, 3881, 4042, 3225, 3759] avg=4096.0000 max=5573 max_rank=2 min=3225 min_rank=6 imbalance_ratio=1.360596 max_load_reduction=27195 max_load_reduction_pct=0.829926 moved_to_echo=14907 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 0, 0, 0, 0, 4088, 32, 0, 0, 0, 0, 0, 0, 1, 2255, 0, 2315, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 538, 0, 0, 0, 1780, 1027, 681, 77, 1, 5, 0, 9, 4018, 0, 0, 0, 741, 1, 2985, 2, 0, 0, 0, 0, 603, 1, 0, 3414, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4120, 4571, 4096, 4103, 4033, 3729, 4018] avg=4096.0000 max=4571 max_rank=2 min=3729 min_rank=6 imbalance_ratio=1.115967 max_load_reduction=28197 max_load_reduction_pct=0.860504 moved_to_echo=15883 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32773, 32804, 32768, 32769, 32726, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32773, 32804, 32768, 32769, 32726, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32773, 32804, 32768, 32769, 32726, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32773, 32804, 32768, 32769, 32726, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32773, 32804, 32768, 32769, 32726, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32773, 32804, 32768, 32769, 32726, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32773, 32804, 32768, 32769, 32726, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32773, 32804, 32768, 32769, 32726, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32726 actual_valid_recv=33024 observed_loss=-298 tokens_per_expert=[4352, 0, 0, 0, 14336, 8192, 5504, 640] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 14336, 128, 18560, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5504, 128, 0, 27392, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9088, 128, 23808, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32773 actual_valid_recv=33152 observed_loss=-379 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32769 actual_valid_recv=33024 observed_loss=-255 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32804 actual_valid_recv=32896 observed_loss=-92 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 0, 512, 32256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 21, 0, 0, 3991, 4082, 2, 0, 0, 112, 0, 0, 0, 4096, 1, 4086, 0, 1, 4093, 0, 4096, 4094, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 22, 0, 4, 3892, 4090, 0, 0, 0, 215, 0, 0, 0, 4096, 2, 4086, 0, 0, 4079, 0, 4096, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 23, 0, 1, 2794, 4091, 9, 0, 0, 1342, 0, 0, 0, 4096, 1, 4094, 0, 0, 4071, 0, 4096, 4085, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 30, 0, 1, 3982, 4089, 1, 0, 0, 108, 0, 0, 3, 4096, 2, 4085, 0, 15, 4082, 0, 4096, 4088, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 29, 0, 0, 4049, 4094, 0, 1, 1, 139, 1, 0, 0, 4094, 0, 4021, 1, 0, 4067, 1, 4091, 4075, 1, 7, 0, 1, 1, 4093, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 43, 0, 2, 3968, 4092, 12, 0, 0, 143, 0, 0, 2, 4096, 3, 4089, 0, 0, 4063, 0, 4096, 4089, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 3685, 4095, 0, 1, 1, 508, 1, 1, 0, 4094, 0, 3997, 1, 0, 4093, 1, 4095, 4093, 0, 2, 0, 0, 1, 4081, 0, 1, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 114, 1, 2, 3610, 4079, 1, 1, 1, 439, 0, 1, 0, 4095, 1, 4086, 0, 0, 4077, 1, 4091, 4078, 0, 4, 1, 0, 0, 4084, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 21, 0, 0, 4057, 0, 0, 0, 3991, 347, 2, 0, 0, 0, 0, 0, 0, 112, 0, 0, 3708, 4, 0, 0, 0, 28, 1, 4086, 0, 0, 0, 0, 0, 1, 4093, 0, 15, 0, 0, 0, 8, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 12, 0, 0, 0, 1, 0, 0, 0, 4088, 7, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4340, 3824, 4115, 4109, 4102, 4104, 4096] avg=4096.0000 max=4340 max_rank=1 min=3824 min_rank=2 imbalance_ratio=1.059570 max_load_reduction=28428 max_load_reduction_pct=0.867554 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 0, 4, 4057, 0, 0, 0, 3892, 347, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 3716, 4, 0, 0, 0, 28, 2, 4086, 0, 0, 0, 0, 0, 0, 4079, 0, 15, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 12, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4239, 3935, 4116, 4094, 4104, 4102, 4095] avg=4096.0000 max=4239 max_rank=1 min=3935 min_rank=2 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 29, 0, 0, 4056, 0, 0, 0, 4049, 348, 0, 1, 0, 0, 0, 0, 1, 139, 1, 0, 3719, 4, 0, 0, 0, 27, 0, 4021, 0, 0, 0, 0, 1, 0, 4067, 1, 15, 0, 0, 0, 7, 4075, 1, 7, 0, 0, 0, 0, 0, 1, 1, 4093, 12, 0, 0, 0, 0, 1, 0, 0, 4084, 7, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4398, 3864, 4048, 4084, 4090, 4107, 4092] avg=4096.0000 max=4398 max_rank=1 min=3864 min_rank=2 imbalance_ratio=1.073730 max_load_reduction=28370 max_load_reduction_pct=0.865784 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 0, 2, 4057, 0, 0, 0, 3968, 348, 12, 0, 0, 0, 0, 0, 0, 143, 0, 0, 3717, 4, 0, 0, 2, 28, 3, 4089, 0, 0, 0, 0, 0, 0, 4063, 0, 15, 0, 0, 0, 8, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 12, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4328, 3864, 4122, 4078, 4097, 4082, 4095] avg=4096.0000 max=4328 max_rank=1 min=3864 min_rank=2 imbalance_ratio=1.056641 max_load_reduction=28440 max_load_reduction_pct=0.867920 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 23, 0, 1, 4062, 0, 0, 0, 2794, 338, 9, 0, 0, 0, 0, 0, 0, 1342, 0, 0, 3721, 5, 0, 0, 0, 16, 1, 4094, 0, 0, 0, 0, 0, 0, 4071, 0, 17, 0, 0, 0, 2, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 15, 0, 0, 0, 0, 0, 0, 0, 4094, 13, 0, 0] rerouted_tokens_per_ep_rank=[4086, 3141, 5068, 4111, 4088, 4087, 4080, 4107] avg=4096.0000 max=5068 max_rank=2 min=3141 min_rank=1 imbalance_ratio=1.237305 max_load_reduction=27700 max_load_reduction_pct=0.845337 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 0, 1, 4057, 0, 0, 0, 3982, 347, 1, 0, 0, 0, 0, 0, 0, 108, 0, 0, 3715, 4, 0, 0, 3, 28, 2, 4085, 0, 0, 0, 0, 0, 15, 4082, 0, 15, 0, 0, 0, 8, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 12, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4330, 3827, 4118, 4112, 4096, 4102, 4095] avg=4096.0000 max=4330 max_rank=1 min=3827 min_rank=2 imbalance_ratio=1.057129 max_load_reduction=28438 max_load_reduction_pct=0.867859 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 114, 1, 2, 4057, 0, 0, 0, 3610, 346, 1, 1, 0, 0, 0, 0, 1, 439, 0, 1, 3706, 4, 0, 0, 0, 27, 1, 4086, 0, 0, 0, 0, 0, 0, 4077, 1, 15, 0, 0, 0, 7, 4078, 0, 4, 0, 0, 0, 0, 1, 0, 0, 4084, 12, 0, 0, 0, 0, 1, 0, 0, 4084, 7, 0, 0] rerouted_tokens_per_ep_rank=[4174, 3958, 4151, 4114, 4093, 4089, 4097, 4092] avg=4096.0000 max=4174 max_rank=0 min=3958 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 4056, 0, 0, 0, 3685, 348, 0, 1, 0, 0, 0, 0, 1, 508, 1, 1, 3720, 4, 0, 0, 0, 27, 0, 3997, 0, 0, 0, 0, 1, 0, 4093, 1, 15, 0, 0, 0, 8, 4093, 0, 2, 0, 0, 0, 0, 0, 0, 1, 4081, 12, 0, 0, 0, 0, 1, 1, 0, 4087, 7, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4034, 4235, 4024, 4110, 4103, 4094, 4096] avg=4096.0000 max=4235 max_rank=2 min=4024 min_rank=3 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[30080, 2816, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 3072, 128, 128, 29824, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 384, 128, 128, 32512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4094, 0, 1, 3844, 0, 4, 7, 46, 1, 4088, 0, 1, 4095, 4094, 3, 4093, 0, 1, 1, 4095, 2, 1, 0, 205, 0, 0, 4090, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4, 0, 4093, 0, 0, 0, 10, 0, 4080, 2, 0, 4096, 4085, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 17, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 2, 0, 4090, 0, 0, 0, 4, 0, 4090, 1, 0, 4096, 4092, 0, 4096, 0, 0, 0, 4095, 0, 0, 1, 11, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 4082, 0, 0, 0, 12, 0, 4095, 0, 0, 4095, 4096, 0, 4096, 1, 0, 0, 4096, 0, 0, 1, 3, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 1, 4050, 1, 1, 13, 4, 1, 4081, 0, 1, 4095, 4094, 0, 4094, 0, 1, 1, 4095, 9, 1, 0, 33, 0, 1, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 3, 0, 4092, 0, 0, 0, 0, 0, 4084, 1, 0, 4096, 4092, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 17, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 0, 4065, 1, 1, 1, 2, 1, 4081, 0, 1, 4095, 4095, 4, 4093, 0, 1, 1, 4095, 0, 3, 0, 37, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 7, 0, 4047, 0, 0, 1, 5, 0, 4059, 4, 0, 4096, 4089, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 77, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 3844, 0, 47, 0, 0, 0, 4, 7, 46, 1, 4081, 0, 0, 0, 4088, 0, 1, 14, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 2, 0, 0, 0, 0, 1, 0, 205, 0, 4034, 12, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 3892, 4139, 4103, 4097, 4099, 4252, 4091] avg=4096.0000 max=4252 max_rank=6 min=3892 min_rank=1 imbalance_ratio=1.038086 max_load_reduction=28516 max_load_reduction_pct=0.870239 moved_to_echo=16539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4093, 0, 48, 0, 0, 0, 0, 0, 10, 0, 4082, 0, 0, 0, 4080, 2, 0, 13, 0, 0, 0, 0, 4085, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4035, 10, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4145, 4092, 4095, 4089, 4095, 4062, 4094] avg=4096.0000 max=4145 max_rank=1 min=4062 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 4082, 0, 47, 0, 0, 0, 0, 0, 12, 0, 4082, 0, 0, 0, 4095, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 4, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4034, 10, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4130, 4094, 4109, 4101, 4095, 4048, 4096] avg=4096.0000 max=4130 max_rank=1 min=4048 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4047, 0, 48, 0, 0, 0, 0, 1, 5, 0, 4082, 0, 0, 0, 4059, 4, 0, 13, 0, 0, 0, 0, 4089, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 77, 0, 4035, 10, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4102, 4088, 4076, 4093, 4095, 4122, 4096] avg=4096.0000 max=4122 max_rank=6 min=4076 min_rank=3 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4, 0, 0, 0, 2, 0, 4090, 0, 52, 0, 0, 0, 0, 0, 4, 0, 4086, 0, 0, 0, 4090, 1, 0, 4, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 1, 11, 0, 4036, 10, 0, 0, 0, 4095, 0, 0, 5, 4, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4144, 4090, 4095, 4092, 4086, 4058, 4104] avg=4096.0000 max=4144 max_rank=1 min=4058 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4065, 1, 47, 0, 0, 0, 1, 1, 2, 1, 4079, 0, 0, 0, 4081, 0, 1, 14, 0, 0, 0, 0, 4095, 4, 4, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 3, 0, 37, 0, 4034, 10, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4113, 4084, 4096, 4103, 4097, 4084, 4095] avg=4096.0000 max=4113 max_rank=1 min=4084 min_rank=2 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4050, 1, 47, 0, 0, 0, 1, 13, 4, 1, 4080, 0, 0, 0, 4081, 0, 1, 14, 0, 0, 0, 0, 4094, 0, 4, 0, 0, 0, 0, 0, 1, 1, 4095, 9, 0, 0, 0, 0, 1, 0, 33, 0, 4034, 10, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4099, 4099, 4096, 4098, 4106, 4078, 4096] avg=4096.0000 max=4106 max_rank=5 min=4078 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 3, 0, 4092, 0, 48, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4084, 1, 0, 13, 0, 0, 0, 0, 4092, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 17, 0, 4035, 11, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4143, 4082, 4098, 4095, 4095, 4064, 4096] avg=4096.0000 max=4143 max_rank=1 min=4064 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32384, 128, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 512, 0, 32384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4076, 776, 0, 3370, 0, 0, 4096, 4050, 0, 22, 0, 0, 2879, 0, 4096, 0, 0, 4096, 57, 0, 0, 0, 1365, 3881, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 4095, 514, 2, 3815, 0, 1, 4083, 4086, 1, 44, 1, 0, 2156, 0, 4095, 0, 1, 4095, 70, 0, 0, 1, 1848, 3857, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4088, 584, 0, 4012, 0, 0, 4095, 4031, 0, 265, 0, 0, 2493, 0, 4096, 0, 0, 4096, 43, 0, 0, 0, 1611, 3351, 0, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4090, 123, 0, 3883, 0, 0, 4096, 3894, 0, 80, 0, 0, 2116, 0, 4096, 0, 0, 4096, 39, 0, 0, 0, 2549, 3704, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4095, 832, 0, 3461, 0, 0, 4094, 4074, 0, 68, 0, 0, 2435, 1, 4096, 0, 0, 4096, 131, 0, 0, 0, 1884, 3500, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 4095, 1043, 0, 3750, 0, 1, 4093, 3943, 1, 15, 1, 0, 1570, 0, 4095, 0, 1, 4095, 6, 0, 0, 1, 2526, 3529, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4084, 643, 0, 4027, 0, 0, 4096, 3968, 0, 283, 0, 0, 2780, 0, 4096, 0, 0, 4096, 29, 0, 0, 0, 1483, 3181, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 4095, 645, 1, 3814, 0, 1, 4095, 4013, 1, 122, 0, 0, 2930, 0, 4095, 0, 1, 4095, 15, 0, 0, 1, 1399, 3441, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 3455, 514, 0, 0, 0, 0, 2, 3815, 0, 1, 305, 24, 0, 0, 0, 4062, 1, 44, 0, 0, 0, 0, 1, 0, 2156, 0, 1402, 335, 48, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 70, 0, 0, 1, 4035, 2, 0, 0, 1848, 2455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[3971, 4147, 4107, 3942, 4096, 4108, 4303, 4094] avg=4096.0000 max=4303 max_rank=6 min=3942 min_rank=3 imbalance_ratio=1.050537 max_load_reduction=28465 max_load_reduction_pct=0.868683 moved_to_echo=16601 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3450, 584, 0, 0, 0, 0, 0, 4012, 0, 0, 304, 24, 0, 0, 0, 4007, 0, 265, 0, 0, 0, 0, 0, 0, 2493, 0, 1218, 334, 48, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 43, 0, 0, 0, 4047, 2, 0, 0, 1611, 2133, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4340, 4272, 4093, 4096, 4092, 3744, 4097] avg=4096.0000 max=4340 max_rank=1 min=3744 min_rank=6 imbalance_ratio=1.059570 max_load_reduction=28428 max_load_reduction_pct=0.867554 moved_to_echo=16029 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3440, 776, 0, 0, 0, 0, 0, 3370, 0, 0, 303, 24, 0, 0, 0, 4026, 0, 22, 0, 0, 0, 0, 0, 0, 2879, 0, 1411, 333, 48, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 57, 0, 0, 0, 4048, 1, 0, 0, 1365, 2470, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4216, 3697, 4048, 4671, 4096, 4106, 3835, 4099] avg=4096.0000 max=4671 max_rank=3 min=3697 min_rank=1 imbalance_ratio=1.140381 max_load_reduction=28097 max_load_reduction_pct=0.857452 moved_to_echo=16136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3455, 832, 0, 0, 0, 0, 0, 3461, 0, 0, 305, 24, 0, 0, 0, 4050, 0, 68, 0, 0, 0, 0, 0, 0, 2435, 1, 1272, 335, 48, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 131, 0, 0, 0, 4046, 1, 0, 0, 1884, 2228, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4287, 3790, 4118, 4091, 4096, 4178, 4112, 4096] avg=4096.0000 max=4287 max_rank=0 min=3790 min_rank=1 imbalance_ratio=1.046631 max_load_reduction=28481 max_load_reduction_pct=0.869171 moved_to_echo=16482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3446, 123, 0, 0, 0, 0, 0, 3883, 0, 0, 306, 27, 0, 0, 0, 3867, 0, 80, 0, 0, 0, 0, 0, 0, 2116, 0, 1349, 338, 49, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 39, 0, 0, 0, 4047, 2, 0, 0, 2549, 2355, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[3569, 4216, 3947, 3852, 4096, 4088, 4904, 4096] avg=4096.0000 max=4904 max_rank=6 min=3569 min_rank=0 imbalance_ratio=1.197266 max_load_reduction=27864 max_load_reduction_pct=0.850342 moved_to_echo=17184 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 3455, 645, 0, 0, 0, 0, 1, 3814, 0, 1, 305, 24, 0, 0, 0, 3989, 1, 122, 0, 0, 0, 0, 0, 0, 2930, 0, 1251, 335, 48, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 15, 0, 0, 1, 4047, 1, 0, 0, 1399, 2190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4145, 4112, 4564, 4096, 4064, 3590, 4095] avg=4096.0000 max=4564 max_rank=3 min=3590 min_rank=6 imbalance_ratio=1.114258 max_load_reduction=28204 max_load_reduction_pct=0.860718 moved_to_echo=15845 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 3455, 1043, 0, 0, 0, 0, 0, 3750, 0, 1, 305, 23, 0, 0, 0, 3920, 1, 15, 0, 0, 0, 0, 1, 0, 1570, 0, 1283, 335, 48, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 6, 0, 0, 1, 4045, 1, 0, 0, 2526, 2246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4500, 4079, 3936, 3237, 4096, 4053, 4772, 4095] avg=4096.0000 max=4772 max_rank=6 min=3237 min_rank=3 imbalance_ratio=1.165039 max_load_reduction=27996 max_load_reduction_pct=0.854370 moved_to_echo=17016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3446, 643, 0, 0, 0, 0, 0, 4027, 0, 0, 304, 23, 0, 0, 0, 3945, 0, 283, 0, 0, 0, 0, 0, 0, 2780, 0, 1156, 334, 48, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 29, 0, 0, 0, 4048, 2, 0, 0, 1483, 2025, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4354, 4228, 4318, 4096, 4079, 3508, 4096] avg=4096.0000 max=4354 max_rank=1 min=3508 min_rank=6 imbalance_ratio=1.062988 max_load_reduction=28414 max_load_reduction_pct=0.867126 moved_to_echo=15779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 19456, 128, 10368, 2688, 512, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14720, 18176, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 27648, 5248, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 30208, 0, 128, 2560, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 31872, 128, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 0, 0, 128, 32384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4096, 0, 0, 0, 4095, 4094, 2, 0, 0, 0, 0, 4095, 0, 0, 4096, 2819, 4095, 0, 0, 0, 944, 0, 1, 0, 0, 4096, 0, 333] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4095, 0, 1, 1, 4095, 4083, 2, 1, 1, 1, 0, 4095, 0, 0, 4090, 2267, 4095, 1, 1, 1, 109, 24, 0, 0, 0, 4062, 0, 1741] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4096, 0, 0, 0, 4094, 4095, 3, 0, 0, 0, 2, 4093, 0, 0, 4096, 1744, 4093, 0, 0, 0, 1388, 0, 0, 0, 0, 4096, 0, 964] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4096, 0, 0, 0, 4095, 4096, 1, 0, 0, 0, 0, 4095, 0, 0, 4095, 511, 4093, 0, 0, 0, 1929, 0, 1, 1, 0, 4096, 0, 1656] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 4096, 1, 0, 0, 4094, 4092, 8, 0, 0, 0, 2, 4094, 0, 0, 4095, 674, 4092, 0, 0, 0, 1874, 0, 0, 0, 0, 4084, 0, 1557] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4095, 0, 1, 1, 4095, 4081, 11, 1, 1, 1, 0, 4084, 0, 0, 4095, 1970, 4095, 1, 1, 1, 314, 13, 0, 0, 0, 4083, 0, 1822] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 4096, 0, 0, 0, 4094, 4095, 2, 0, 0, 0, 2, 4094, 0, 0, 4096, 1660, 4091, 0, 0, 0, 1800, 0, 0, 0, 0, 4096, 0, 637] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 2, 1, 4095, 0, 1, 1, 4095, 4078, 29, 1, 1, 1, 0, 4090, 0, 0, 4095, 2995, 4095, 0, 1, 0, 782, 5, 0, 0, 0, 3942, 0, 456] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4093, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 4081, 11, 1, 0, 0, 0, 0, 1, 1, 0, 4084, 2, 0, 0, 0, 0, 0, 2267, 1970, 0, 0, 0, 0, 4095, 1, 1, 1, 0, 0, 0, 0, 314, 13, 0, 0, 1828, 1120, 2, 0, 0, 2961, 0, 1822, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4097, 4093, 4088, 4237, 4098, 3277, 4783] avg=4096.0000 max=4783 max_rank=7 min=3277 min_rank=6 imbalance_ratio=1.167725 max_load_reduction=27985 max_load_reduction_pct=0.854034 moved_to_echo=16395 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 2, 0, 0, 0, 0, 0, 2267, 1744, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1388, 0, 0, 0, 1829, 1124, 2, 0, 0, 2970, 0, 964, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4098, 4097, 4011, 4093, 4343, 3934] avg=4096.0000 max=4343 max_rank=6 min=3934 min_rank=7 imbalance_ratio=1.060303 max_load_reduction=28425 max_load_reduction_pct=0.867462 moved_to_echo=16381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 2267, 511, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1929, 0, 1, 1, 1828, 1124, 2, 0, 0, 2970, 0, 1656, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4097, 4097, 2778, 4093, 4885, 4626] avg=4096.0000 max=4885 max_rank=6 min=2778 min_rank=4 imbalance_ratio=1.192627 max_load_reduction=27883 max_load_reduction_pct=0.850922 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4093, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 2262, 2819, 0, 0, 0, 0, 4095, 0, 0, 0, 7, 4, 1, 0, 944, 0, 1, 0, 1834, 1127, 2, 0, 0, 2964, 0, 333, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4092, 4089, 4099, 5081, 4107, 3908, 3297] avg=4096.0000 max=5081 max_rank=4 min=3297 min_rank=7 imbalance_ratio=1.240479 max_load_reduction=27687 max_load_reduction_pct=0.844940 moved_to_echo=16393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 2, 0, 0, 0, 0, 0, 2267, 1660, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 1800, 0, 0, 0, 1829, 1124, 2, 0, 0, 2970, 0, 637, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4097, 4098, 3927, 4091, 4755, 3607] avg=4096.0000 max=4755 max_rank=6 min=3607 min_rank=7 imbalance_ratio=1.160889 max_load_reduction=28013 max_load_reduction_pct=0.854889 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4093, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 4083, 2, 1, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 2264, 2267, 0, 0, 0, 0, 4095, 1, 1, 1, 0, 0, 0, 0, 109, 24, 0, 0, 1826, 1114, 2, 0, 0, 2947, 0, 1741, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4097, 4086, 4098, 4531, 4098, 3075, 4688] avg=4096.0000 max=4688 max_rank=7 min=3075 min_rank=6 imbalance_ratio=1.144531 max_load_reduction=28080 max_load_reduction_pct=0.856934 moved_to_echo=16392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 2, 1, 4092, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 4078, 29, 1, 0, 0, 0, 0, 1, 1, 0, 4090, 1, 0, 0, 0, 0, 0, 2267, 2995, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 782, 5, 0, 0, 1828, 1082, 3, 0, 0, 2859, 0, 456, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4097, 4108, 4093, 5262, 4096, 3700, 3315] avg=4096.0000 max=5262 max_rank=4 min=3315 min_rank=7 imbalance_ratio=1.284668 max_load_reduction=27506 max_load_reduction_pct=0.839417 moved_to_echo=16373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 4092, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4092, 8, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 2, 0, 0, 0, 0, 0, 2267, 674, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 1874, 0, 0, 0, 1828, 1121, 2, 0, 0, 2961, 0, 1557, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4097, 4100, 4098, 2941, 4092, 4825, 4518] avg=4096.0000 max=4825 max_rank=6 min=2941 min_rank=4 imbalance_ratio=1.177979 max_load_reduction=27943 max_load_reduction_pct=0.852753 moved_to_echo=16376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[9216, 128, 128, 128, 14720, 8960, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 18176, 14720, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 23680, 0, 9216, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 1, 1, 4092, 0, 0, 1, 1816, 1, 3968, 1, 1, 1, 1, 4095, 2423, 58, 1, 0, 0, 0, 4095, 0, 1, 0, 2, 4095, 0, 14, 4005, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3262, 0, 4095, 0, 0, 0, 0, 4084, 679, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 1, 4090, 0, 329, 3936, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 2948, 0, 4096, 0, 0, 0, 0, 4081, 961, 2, 0, 10, 0, 8, 4096, 0, 0, 0, 0, 4092, 0, 276, 4006, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3526, 0, 4088, 0, 0, 0, 0, 4078, 522, 0, 0, 16, 0, 6, 4096, 0, 0, 0, 0, 4094, 0, 161, 3989, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 1683, 0, 4091, 0, 0, 0, 0, 4083, 2652, 0, 0, 6, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 266, 3599, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 1, 1, 4093, 0, 0, 0, 2453, 2, 3731, 1, 1, 1, 1, 4095, 1840, 74, 1, 14, 0, 0, 4095, 0, 2, 0, 9, 4095, 1, 94, 4068, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 2899, 0, 4096, 0, 0, 0, 0, 4071, 565, 0, 0, 6, 0, 4, 4096, 0, 0, 0, 0, 4091, 0, 688, 4061, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 2, 1, 4094, 0, 0, 1, 2656, 0, 3785, 1, 1, 1, 1, 4095, 1446, 49, 1, 3, 0, 0, 4095, 0, 0, 0, 0, 4094, 0, 318, 4028, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 1, 4094, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 1816, 1, 1431, 1, 0, 0, 0, 0, 1, 1, 1, 4095, 8, 1, 0, 0, 2423, 58, 1, 0, 2537, 127, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 0, 2, 4095, 0, 0, 0, 0, 0, 14, 3870, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4093, 3249, 4107, 5146, 4094, 4098, 3884] avg=4096.0000 max=5146 max_rank=4 min=3249 min_rank=2 imbalance_ratio=1.256348 max_load_reduction=27622 max_load_reduction_pct=0.842957 moved_to_echo=17222 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3262, 0, 1477, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 8, 1, 0, 0, 679, 0, 0, 0, 2618, 124, 0, 0, 0, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 329, 3804, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4739, 4093, 3421, 4100, 4091, 4133] avg=4096.0000 max=4739 max_rank=2 min=3421 min_rank=4 imbalance_ratio=1.156982 max_load_reduction=28029 max_load_reduction_pct=0.855377 moved_to_echo=15745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2948, 0, 1477, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 8, 1, 0, 0, 961, 2, 0, 10, 2619, 127, 0, 0, 0, 8, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 276, 3871, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4425, 4090, 3719, 4103, 4092, 4147] avg=4096.0000 max=4425 max_rank=2 min=3719 min_rank=4 imbalance_ratio=1.080322 max_load_reduction=28343 max_load_reduction_pct=0.864960 moved_to_echo=16061 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3526, 0, 1474, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 8, 1, 0, 0, 522, 0, 0, 16, 2614, 126, 0, 0, 0, 6, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 161, 3855, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 5000, 4087, 3278, 4101, 4094, 4016] avg=4096.0000 max=5000 max_rank=2 min=3278 min_rank=4 imbalance_ratio=1.220703 max_load_reduction=27768 max_load_reduction_pct=0.847412 moved_to_echo=15489 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 1683, 0, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 9, 5, 0, 0, 2652, 0, 0, 6, 2619, 118, 0, 0, 0, 5, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 266, 3472, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4103, 3155, 4097, 5395, 4083, 4102, 3738] avg=4096.0000 max=5395 max_rank=4 min=3155 min_rank=2 imbalance_ratio=1.317139 max_load_reduction=27373 max_load_reduction_pct=0.835358 moved_to_echo=17318 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 1, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2453, 2, 1346, 1, 0, 0, 0, 0, 1, 1, 1, 4095, 8, 1, 0, 0, 1840, 74, 1, 14, 2385, 129, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 9, 4095, 0, 0, 0, 0, 1, 94, 3931, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4093, 3802, 4107, 4444, 4094, 4106, 4026] avg=4096.0000 max=4444 max_rank=4 min=3802 min_rank=2 imbalance_ratio=1.084961 max_load_reduction=28324 max_load_reduction_pct=0.864380 moved_to_echo=16670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2899, 0, 1477, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 8, 1, 0, 0, 565, 0, 0, 6, 2619, 128, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 688, 3925, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4376, 4080, 3318, 4099, 4091, 4613] avg=4096.0000 max=4613 max_rank=7 min=3318 min_rank=4 imbalance_ratio=1.126221 max_load_reduction=28155 max_load_reduction_pct=0.859222 moved_to_echo=16121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2, 1, 4094, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 2656, 0, 1365, 1, 0, 0, 0, 0, 1, 1, 1, 4095, 8, 1, 0, 0, 1446, 49, 1, 3, 2420, 127, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 318, 3893, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4095, 4022, 4107, 4047, 4094, 4094, 4211] avg=4096.0000 max=4211 max_rank=7 min=4022 min_rank=2 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11136, 256, 128, 128, 20480, 1024, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21248, 128, 11520, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 2176, 30720, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3825, 0, 0, 0, 3225, 0, 0, 3609, 0, 16, 521, 0, 0, 4096, 0, 0, 4096, 0, 0, 1092, 0, 4096, 0, 2, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 3998, 4055, 1, 0, 2, 2723, 0, 0, 3941, 1, 0, 689, 1, 7, 4093, 1, 0, 4095, 0, 0, 967, 1, 4095, 0, 0, 0, 1, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4093, 0, 1, 0, 2700, 0, 0, 3825, 0, 39, 1084, 0, 0, 4096, 0, 0, 4095, 0, 1, 546, 0, 4096, 0, 3, 0, 0, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4094, 0, 0, 0, 1203, 0, 0, 3982, 0, 17, 1992, 0, 0, 4096, 0, 0, 4096, 0, 0, 996, 0, 4096, 0, 12, 0, 0, 0, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4085, 0, 0, 0, 1986, 0, 0, 3165, 0, 13, 405, 0, 0, 4096, 0, 0, 4096, 0, 0, 2634, 0, 4096, 0, 2, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4081, 0, 0, 0, 1235, 0, 0, 3355, 0, 13, 1106, 0, 0, 4096, 0, 0, 4096, 0, 0, 2494, 0, 4096, 0, 7, 0, 0, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 4042, 4038, 1, 0, 2, 2572, 0, 0, 3388, 1, 0, 547, 1, 6, 4093, 1, 0, 4095, 0, 0, 1787, 1, 4095, 0, 0, 0, 1, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 4011, 4059, 1, 0, 0, 2456, 0, 0, 3460, 1, 0, 354, 1, 9, 4095, 1, 1, 4095, 0, 0, 2031, 0, 4095, 0, 0, 0, 1, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 55, 4055, 0, 0, 0, 0, 1, 0, 2, 2723, 1801, 0, 0, 0, 0, 0, 3941, 1, 495, 1, 0, 0, 0, 689, 1, 7, 3244, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1, 1647, 851, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4527, 4438, 3941, 4094, 3466, 4093, 4098] avg=4096.0000 max=4527 max_rank=1 min=3466 min_rank=5 imbalance_ratio=1.105225 max_load_reduction=28241 max_load_reduction_pct=0.861847 moved_to_echo=15751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 57, 4093, 0, 0, 0, 0, 0, 1, 0, 2700, 1845, 0, 0, 0, 0, 0, 3825, 0, 507, 1, 0, 0, 39, 1084, 0, 0, 3242, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 546, 0, 1687, 853, 0, 0, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4546, 4333, 4365, 4096, 3087, 4097, 4094] avg=4096.0000 max=4546 max_rank=1 min=3087 min_rank=5 imbalance_ratio=1.109863 max_load_reduction=28222 max_load_reduction_pct=0.861267 moved_to_echo=15374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 57, 4081, 0, 0, 0, 0, 0, 0, 0, 1235, 1845, 0, 0, 0, 0, 0, 3355, 0, 507, 1, 0, 0, 13, 1106, 0, 0, 3244, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2494, 0, 1687, 852, 0, 0, 4094, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 3080, 3863, 4363, 4096, 5033, 4101, 4094] avg=4096.0000 max=5033 max_rank=5 min=3080 min_rank=1 imbalance_ratio=1.228760 max_load_reduction=27735 max_load_reduction_pct=0.846405 moved_to_echo=17324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 56, 4094, 0, 0, 0, 0, 0, 0, 0, 1203, 1845, 0, 0, 0, 0, 0, 3982, 0, 507, 1, 0, 0, 17, 1992, 0, 0, 3243, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 996, 0, 1686, 853, 0, 0, 4094, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 3048, 4490, 5252, 4096, 3535, 4106, 4091] avg=4096.0000 max=5252 max_rank=3 min=3048 min_rank=1 imbalance_ratio=1.282227 max_load_reduction=27516 max_load_reduction_pct=0.839722 moved_to_echo=15828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 41, 3825, 0, 0, 0, 0, 0, 0, 0, 3225, 1851, 0, 0, 0, 0, 0, 3609, 0, 511, 3, 0, 0, 16, 521, 0, 0, 3245, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 0, 1693, 851, 0, 0, 4087, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[3866, 5076, 4123, 3782, 4096, 3636, 4089, 4100] avg=4096.0000 max=5076 max_rank=1 min=3636 min_rank=5 imbalance_ratio=1.239258 max_load_reduction=27692 max_load_reduction_pct=0.845093 moved_to_echo=15921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 57, 4085, 0, 0, 0, 0, 0, 0, 0, 1986, 1845, 0, 0, 0, 0, 0, 3165, 0, 507, 1, 0, 0, 13, 405, 0, 0, 3245, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2634, 0, 1687, 851, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 3831, 3673, 3663, 4096, 5172, 4096, 4095] avg=4096.0000 max=5172 max_rank=5 min=3663 min_rank=3 imbalance_ratio=1.262695 max_load_reduction=27596 max_load_reduction_pct=0.842163 moved_to_echo=17459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 56, 4059, 0, 0, 0, 0, 1, 0, 0, 2456, 1807, 0, 0, 0, 0, 0, 3460, 1, 496, 1, 0, 0, 0, 354, 1, 9, 3242, 0, 0, 0, 4095, 1, 1, 1, 0, 0, 0, 0, 0, 0, 2031, 0, 1652, 852, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4264, 3958, 3606, 4098, 4535, 4093, 4098] avg=4096.0000 max=4535 max_rank=5 min=3606 min_rank=3 imbalance_ratio=1.107178 max_load_reduction=28233 max_load_reduction_pct=0.861603 moved_to_echo=16824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 56, 4038, 0, 0, 0, 0, 1, 0, 2, 2572, 1821, 0, 0, 0, 0, 0, 3388, 1, 500, 1, 0, 0, 0, 547, 1, 6, 3242, 0, 0, 0, 4093, 1, 0, 2, 0, 0, 0, 0, 0, 0, 1787, 1, 1665, 851, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4396, 3890, 3796, 4096, 4304, 4093, 4098] avg=4096.0000 max=4396 max_rank=1 min=3796 min_rank=3 imbalance_ratio=1.073242 max_load_reduction=28372 max_load_reduction_pct=0.865845 moved_to_echo=16591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 12672, 128, 13440, 6912, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 18176, 14720, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 6784, 128, 128, 25984, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 28800, 128, 4096, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 512, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3991, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4020, 0, 4096, 0, 4096, 2, 0, 0, 0, 1822, 0, 0, 0, 0, 0, 0, 0, 2453, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4003, 4096, 0, 1, 0, 4096, 4096, 1, 0, 0, 4035, 0, 4096, 0, 4096, 15, 0, 0, 0, 1696, 0, 0, 0, 0, 0, 0, 0, 2537, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[2756, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 3936, 0, 4096, 0, 4096, 0, 0, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 3445, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4002, 4096, 0, 0, 1, 4096, 4096, 0, 0, 0, 4002, 2, 4096, 0, 4096, 11, 0, 0, 0, 990, 0, 0, 0, 0, 0, 0, 0, 3280, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3987, 4093, 0, 0, 0, 4085, 4095, 0, 0, 1, 3993, 0, 4095, 0, 4016, 95, 27, 1, 1, 1415, 1, 0, 0, 1, 0, 1, 1, 2854, 0, 1, 1, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4046, 4096, 0, 1, 0, 4096, 4096, 0, 0, 0, 4023, 0, 4096, 0, 4096, 10, 0, 0, 0, 2378, 0, 0, 0, 0, 0, 0, 0, 1830, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3983, 4091, 0, 0, 0, 4086, 4095, 0, 0, 1, 4019, 0, 4095, 0, 4036, 80, 8, 1, 1, 2678, 1, 0, 0, 1, 0, 1, 1, 1583, 0, 1, 1, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[3834, 4092, 0, 0, 0, 4085, 4095, 0, 1, 1, 4032, 0, 4095, 2, 4034, 60, 10, 1, 1, 1840, 1, 0, 0, 1, 0, 1, 2, 2579, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4003, 272, 0, 1, 0, 0, 0, 0, 0, 4096, 4, 1, 0, 0, 0, 0, 0, 0, 4035, 0, 78, 9, 0, 0, 0, 0, 4087, 15, 0, 0, 0, 0, 0, 0, 0, 1696, 2219, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2537, 1525, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0] rerouted_tokens_per_ep_rank=[4276, 4101, 4122, 4102, 3915, 4096, 4062, 4094] avg=4096.0000 max=4276 max_rank=0 min=3915 min_rank=4 imbalance_ratio=1.043945 max_load_reduction=28492 max_load_reduction_pct=0.869507 moved_to_echo=16167 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3991, 272, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 78, 9, 0, 0, 0, 0, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 1822, 2219, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2453, 1525, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0] rerouted_tokens_per_ep_rank=[4263, 4100, 4107, 4089, 4041, 4096, 3978, 4094] avg=4096.0000 max=4263 max_rank=0 min=3978 min_rank=6 imbalance_ratio=1.040771 max_load_reduction=28505 max_load_reduction_pct=0.869904 moved_to_echo=16209 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3983, 273, 0, 0, 0, 0, 0, 0, 0, 4086, 4, 0, 0, 0, 0, 0, 0, 1, 4019, 0, 78, 9, 0, 0, 0, 0, 4027, 80, 0, 0, 0, 0, 8, 1, 1, 2678, 2216, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 1, 1583, 1523, 0, 0, 0, 0, 1, 1, 5, 4091, 1, 0, 0] rerouted_tokens_per_ep_rank=[4256, 4090, 4107, 4107, 4904, 4097, 3108, 4099] avg=4096.0000 max=4904 max_rank=4 min=3108 min_rank=6 imbalance_ratio=1.197266 max_load_reduction=27864 max_load_reduction_pct=0.850342 moved_to_echo=16208 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4002, 272, 0, 0, 0, 0, 0, 0, 1, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4002, 2, 78, 9, 0, 0, 0, 0, 4087, 11, 0, 0, 0, 0, 0, 0, 0, 990, 2219, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3280, 1525, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 1, 0] rerouted_tokens_per_ep_rank=[4274, 4101, 4091, 4098, 3209, 4095, 4805, 4095] avg=4096.0000 max=4805 max_rank=6 min=3209 min_rank=4 imbalance_ratio=1.173096 max_load_reduction=27963 max_load_reduction_pct=0.853363 moved_to_echo=16204 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2756, 258, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 83, 10, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 2221, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3445, 1529, 0, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 0] rerouted_tokens_per_ep_rank=[3014, 4098, 4029, 4086, 4372, 4096, 4974, 4099] avg=4096.0000 max=4974 max_rank=6 min=3014 min_rank=0 imbalance_ratio=1.214355 max_load_reduction=27794 max_load_reduction_pct=0.848206 moved_to_echo=17541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3987, 273, 0, 0, 0, 0, 0, 0, 0, 4085, 4, 0, 0, 0, 0, 0, 0, 1, 3993, 0, 78, 9, 0, 0, 0, 0, 4007, 95, 0, 0, 0, 0, 27, 1, 1, 1415, 2217, 0, 0, 0, 1, 0, 0, 1, 4094, 0, 0, 0, 0, 1, 1, 2854, 1524, 0, 0, 0, 0, 1, 1, 4, 4091, 1, 1, 0] rerouted_tokens_per_ep_rank=[4260, 4089, 4081, 4102, 3661, 4096, 4380, 4099] avg=4096.0000 max=4380 max_rank=6 min=3661 min_rank=4 imbalance_ratio=1.069336 max_load_reduction=28388 max_load_reduction_pct=0.866333 moved_to_echo=16236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4046, 272, 0, 1, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 78, 9, 0, 0, 0, 0, 4087, 10, 0, 0, 0, 0, 0, 0, 0, 2378, 2219, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1830, 1525, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0] rerouted_tokens_per_ep_rank=[4319, 4100, 4110, 4097, 4597, 4096, 3355, 4094] avg=4096.0000 max=4597 max_rank=4 min=3355 min_rank=6 imbalance_ratio=1.122314 max_load_reduction=28171 max_load_reduction_pct=0.859711 moved_to_echo=16142 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3834, 272, 0, 0, 0, 0, 0, 0, 0, 4085, 4, 0, 0, 0, 0, 0, 1, 1, 4032, 0, 78, 9, 0, 0, 0, 2, 4025, 60, 0, 0, 0, 0, 10, 1, 1, 1840, 2217, 0, 0, 0, 1, 0, 0, 1, 4094, 0, 0, 0, 0, 1, 2, 2579, 1524, 0, 0, 0, 0, 0, 0, 1, 4091, 1, 1, 0] rerouted_tokens_per_ep_rank=[4106, 4089, 4121, 4087, 4069, 4096, 4106, 4094] avg=4096.0000 max=4121 max_rank=2 min=4069 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 14976, 17792, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 20608, 12288, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[30720, 2176, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32512, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32128, 128, 640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[73, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 29, 0, 0, 4066, 0, 2, 4094, 0, 0, 0, 4095, 0, 0, 4096, 4096, 4096, 1, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[841, 0, 0, 0, 0, 0, 0, 0, 3271, 0, 84, 0, 0, 4032, 0, 1, 4095, 0, 0, 0, 4059, 0, 1, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[661, 0, 0, 0, 0, 0, 2, 0, 3635, 0, 31, 0, 0, 4066, 0, 0, 4093, 0, 0, 0, 3893, 1, 2, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[254, 0, 0, 0, 0, 0, 0, 0, 3868, 0, 31, 0, 0, 4060, 0, 1, 4095, 0, 0, 0, 4074, 0, 1, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[840, 1, 2, 1, 1, 0, 0, 14, 3630, 7, 97, 0, 1, 3982, 0, 0, 4094, 0, 0, 1, 3895, 0, 0, 3906, 4094, 4094, 0, 26, 1, 4080, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[867, 1, 3, 1, 0, 0, 0, 11, 3623, 16, 100, 0, 2, 4043, 0, 0, 4095, 1, 0, 1, 3801, 0, 0, 3906, 4093, 4094, 0, 26, 1, 4082, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[89, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 29, 0, 0, 4066, 0, 1, 4090, 0, 0, 0, 4081, 1, 2, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[954, 1, 2, 1, 0, 0, 0, 8, 3550, 10, 192, 0, 1, 3884, 0, 0, 4095, 0, 0, 1, 3792, 0, 0, 3972, 4094, 4095, 0, 26, 1, 4088, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[73, 0, 0, 0, 3573, 4, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4024, 0, 29, 0, 319, 0, 0, 0, 0, 4066, 0, 2, 67, 3, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 137, 0, 0, 0, 0, 4088, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[3650, 4092, 4372, 4138, 4095, 4232, 4089, 4100] avg=4096.0000 max=4372 max_rank=2 min=3650 min_rank=0 imbalance_ratio=1.067383 max_load_reduction=28396 max_load_reduction_pct=0.866577 moved_to_echo=16516 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[661, 0, 0, 0, 3573, 4, 0, 0, 0, 0, 2, 0, 4092, 0, 0, 0, 3635, 0, 31, 0, 319, 0, 0, 0, 0, 4066, 0, 0, 67, 3, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 3893, 1, 2, 137, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4238, 4094, 3985, 4136, 4094, 4033, 4088, 4100] avg=4096.0000 max=4238 max_rank=0 min=3985 min_rank=2 imbalance_ratio=1.034668 max_load_reduction=28530 max_load_reduction_pct=0.870667 moved_to_echo=16315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[840, 1, 2, 1, 3407, 4, 0, 0, 1, 0, 0, 14, 4090, 0, 0, 0, 3630, 7, 97, 0, 304, 0, 0, 0, 1, 3982, 0, 0, 64, 3, 0, 0, 4094, 0, 0, 1, 1, 0, 0, 0, 3895, 0, 0, 131, 0, 0, 0, 0, 4087, 0, 0, 26, 0, 0, 0, 0, 1, 4080, 1, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4255, 4105, 4038, 4050, 4096, 4026, 4113, 4085] avg=4096.0000 max=4255 max_rank=0 min=4026 min_rank=5 imbalance_ratio=1.038818 max_load_reduction=28513 max_load_reduction_pct=0.870148 moved_to_echo=16320 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[841, 0, 0, 0, 3575, 4, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3271, 0, 84, 0, 323, 0, 0, 0, 0, 4032, 0, 1, 72, 6, 0, 0, 4095, 0, 0, 0, 6, 0, 0, 0, 4059, 0, 1, 126, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4420, 4092, 3678, 4111, 4101, 4186, 4077, 4103] avg=4096.0000 max=4420 max_rank=0 min=3678 min_rank=2 imbalance_ratio=1.079102 max_load_reduction=28348 max_load_reduction_pct=0.865112 moved_to_echo=16467 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[89, 0, 0, 0, 3573, 4, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4025, 0, 29, 0, 319, 0, 0, 0, 0, 4066, 0, 1, 67, 3, 0, 0, 4090, 0, 0, 0, 1, 0, 0, 0, 4081, 1, 2, 137, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[3666, 4092, 4373, 4137, 4091, 4221, 4088, 4100] avg=4096.0000 max=4373 max_rank=2 min=3666 min_rank=0 imbalance_ratio=1.067627 max_load_reduction=28395 max_load_reduction_pct=0.866547 moved_to_echo=16500 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[867, 1, 3, 1, 3407, 4, 0, 0, 0, 0, 0, 11, 4090, 0, 0, 0, 3623, 16, 100, 0, 304, 0, 0, 0, 2, 4043, 0, 0, 64, 3, 0, 0, 4095, 1, 0, 1, 1, 0, 0, 0, 3801, 0, 0, 131, 0, 0, 0, 0, 4086, 0, 0, 26, 0, 0, 0, 0, 1, 4082, 1, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4283, 4101, 4043, 4112, 4098, 3932, 4112, 4087] avg=4096.0000 max=4283 max_rank=0 min=3932 min_rank=5 imbalance_ratio=1.045654 max_load_reduction=28485 max_load_reduction_pct=0.869293 moved_to_echo=16229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[254, 0, 0, 0, 3573, 4, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3868, 0, 31, 0, 319, 0, 0, 0, 0, 4060, 0, 1, 67, 3, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 4074, 0, 1, 137, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[3831, 4092, 4218, 4131, 4096, 4212, 4088, 4100] avg=4096.0000 max=4218 max_rank=2 min=3831 min_rank=0 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16496 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[954, 1, 2, 1, 3464, 3, 0, 0, 0, 0, 0, 8, 4092, 0, 0, 0, 3550, 10, 192, 0, 309, 0, 0, 0, 1, 3884, 0, 0, 65, 3, 0, 0, 4095, 0, 0, 1, 1, 0, 0, 0, 3792, 0, 0, 134, 0, 0, 0, 0, 4087, 0, 0, 26, 0, 0, 0, 0, 1, 4088, 1, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4425, 4100, 4061, 3953, 4097, 3926, 4113, 4093] avg=4096.0000 max=4425 max_rank=0 min=3926 min_rank=5 imbalance_ratio=1.080322 max_load_reduction=28343 max_load_reduction_pct=0.864960 moved_to_echo=16229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4608, 128, 128, 128, 28160, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[29696, 128, 640, 0, 2560, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[31744, 128, 128, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32256, 0, 128, 640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4093, 1, 42, 0, 0, 25, 0, 0, 3741, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 3, 282, 0, 0, 0, 0, 4096, 7, 4095, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 41, 0, 0, 3, 0, 0, 4035, 4093, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 16, 0, 0, 0, 0, 4096, 3, 4095, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4092, 0, 52, 0, 0, 8, 0, 0, 3893, 4090, 2, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 8, 143, 0, 0, 0, 0, 4096, 2, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4095, 1, 63, 0, 0, 11, 0, 0, 3979, 4087, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4, 43, 0, 0, 0, 0, 4096, 6, 4093, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4085, 0, 30, 0, 0, 5, 0, 0, 3932, 4088, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 26, 115, 0, 0, 0, 0, 4096, 7, 4094, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4089, 0, 4, 1, 1, 9, 11, 1, 3986, 4051, 0, 0, 0, 0, 0, 1, 4094, 4095, 4083, 1, 1, 1, 141, 1, 0, 0, 1, 4095, 0, 4095, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4089, 0, 0, 1, 1, 0, 16, 1, 4000, 4088, 0, 0, 0, 0, 0, 1, 4094, 4095, 4079, 1, 1, 0, 100, 2, 0, 0, 1, 4095, 0, 4095, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 12, 1, 1, 2, 5, 1, 4034, 4090, 0, 0, 0, 0, 0, 1, 4094, 4092, 4076, 1, 1, 14, 54, 1, 0, 0, 1, 4095, 0, 4095, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4067, 1, 42, 0, 0, 0, 0, 0, 0, 25, 0, 0, 4084, 0, 0, 0, 3741, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 3, 282, 0, 3947, 26, 5, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 4091, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4109, 3888, 4096, 4103, 4267, 4096, 4099] avg=4096.0000 max=4267 max_rank=5 min=3888 min_rank=2 imbalance_ratio=1.041748 max_load_reduction=28501 max_load_reduction_pct=0.869781 moved_to_echo=16565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4066, 0, 52, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4084, 0, 0, 0, 3893, 146, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 8, 143, 0, 3944, 26, 5, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4090, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4092, 4041, 4096, 4103, 4130, 4096, 4092] avg=4096.0000 max=4130 max_rank=5 min=4041 min_rank=2 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4063, 0, 4, 1, 0, 0, 0, 0, 1, 9, 11, 1, 4083, 0, 0, 0, 3986, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 5, 4083, 1, 0, 0, 0, 0, 1, 1, 141, 1, 3906, 26, 7, 4, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 4091, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4105, 4131, 4095, 4089, 4087, 4096, 4097] avg=4096.0000 max=4131 max_rank=2 min=4068 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4059, 0, 30, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4084, 0, 0, 0, 3932, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 26, 115, 0, 3942, 26, 5, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 4090, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4089, 4078, 4096, 4103, 4118, 4096, 4099] avg=4096.0000 max=4118 max_rank=5 min=4078 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4063, 0, 41, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4085, 0, 0, 0, 4035, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 16, 0, 3949, 32, 8, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4088, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4088, 4179, 4099, 4096, 4014, 4096, 4092] avg=4096.0000 max=4179 max_rank=2 min=4014 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4063, 0, 0, 1, 0, 0, 0, 0, 1, 0, 16, 1, 4083, 0, 0, 0, 4000, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 7, 4079, 1, 0, 0, 0, 0, 1, 0, 100, 2, 3942, 26, 5, 4, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 4091, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4101, 4146, 4095, 4087, 4080, 4096, 4099] avg=4096.0000 max=4146 max_rank=2 min=4064 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4069, 1, 63, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4084, 0, 0, 0, 3979, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 4, 43, 0, 3941, 26, 5, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4089, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4095, 4125, 4096, 4103, 4023, 4096, 4097] avg=4096.0000 max=4133 max_rank=0 min=4023 min_rank=5 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4069, 0, 12, 1, 0, 0, 0, 0, 1, 2, 5, 1, 4080, 0, 0, 0, 4034, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 7, 4076, 1, 0, 0, 0, 0, 1, 14, 54, 1, 3944, 26, 5, 4, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4089, 4180, 4095, 4084, 4049, 4096, 4093] avg=4096.0000 max=4180 max_rank=2 min=4049 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31616, 1280, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 896, 128, 31616, 256, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 2315, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 41, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1832, 0, 0, 4004, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 2168, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2185, 0, 0, 3822, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 2827, 1, 0, 20, 1, 0, 4094, 1, 0, 1, 1, 366, 3, 0, 1, 4093, 0, 3, 4095, 1, 0, 1, 4095, 2456, 0, 0, 2713, 1, 3899, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 2720, 0, 0, 23, 0, 0, 4096, 0, 0, 0, 0, 138, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1350, 0, 0, 3961, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 2606, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1858, 0, 0, 3720, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 3094, 1, 0, 21, 1, 1, 4094, 1, 1, 1, 1, 497, 2, 0, 0, 4092, 0, 11, 4095, 1, 0, 1, 4095, 1778, 0, 0, 2998, 1, 3887, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 3255, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 39, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1048, 0, 0, 3849, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1, 2209, 1, 0, 77, 1, 0, 4094, 1, 0, 1, 1, 748, 2, 0, 0, 4095, 0, 3, 4095, 1, 0, 0, 4095, 1817, 0, 0, 3540, 1, 3890, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2827, 1, 0, 963, 156, 17, 1, 20, 1, 0, 4077, 0, 0, 0, 0, 1, 0, 1, 1, 4094, 0, 0, 0, 366, 3, 0, 1, 3862, 1, 0, 0, 4092, 0, 3, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 2456, 0, 0, 1750, 0, 0, 0, 0, 1, 3899, 0, 77, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3965, 4098, 4097, 4233, 4095, 4097, 4206, 3977] avg=4096.0000 max=4233 max_rank=3 min=3965 min_rank=0 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2315, 0, 0, 1422, 156, 17, 1, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 41, 0, 0, 0, 3862, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1832, 0, 0, 2582, 0, 0, 0, 0, 0, 4096, 0, 78, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3911, 4079, 4095, 3904, 4095, 4096, 4414, 4174] avg=4096.0000 max=4414 max_rank=6 min=3904 min_rank=3 imbalance_ratio=1.077637 max_load_reduction=28354 max_load_reduction_pct=0.865295 moved_to_echo=16779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2606, 0, 0, 1321, 156, 17, 1, 1, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 7, 0, 0, 0, 3862, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1858, 0, 0, 2399, 0, 0, 0, 0, 0, 4096, 0, 78, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4080, 4095, 3870, 4095, 4096, 4257, 4174] avg=4096.0000 max=4257 max_rank=6 min=3870 min_rank=3 imbalance_ratio=1.039307 max_load_reduction=28511 max_load_reduction_pct=0.870087 moved_to_echo=16622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3255, 0, 0, 1367, 156, 17, 1, 1, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 39, 0, 0, 0, 3862, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1048, 0, 0, 2482, 0, 0, 0, 0, 0, 4096, 0, 78, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4796, 4080, 4095, 3902, 4095, 4096, 3530, 4174] avg=4096.0000 max=4796 max_rank=0 min=3530 min_rank=6 imbalance_ratio=1.170898 max_load_reduction=27972 max_load_reduction_pct=0.853638 moved_to_echo=15895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2168, 0, 0, 1357, 156, 17, 1, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 17, 0, 0, 0, 3862, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2185, 0, 0, 2465, 0, 0, 0, 0, 0, 4096, 0, 78, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3699, 4079, 4095, 3880, 4095, 4096, 4650, 4174] avg=4096.0000 max=4650 max_rank=6 min=3699 min_rank=0 imbalance_ratio=1.135254 max_load_reduction=28118 max_load_reduction_pct=0.858093 moved_to_echo=17015 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2720, 0, 0, 1411, 163, 22, 2, 23, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 138, 0, 0, 0, 3867, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1350, 0, 0, 2550, 0, 0, 0, 0, 0, 4096, 0, 66, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4318, 4097, 4096, 4005, 4094, 4096, 3900, 4162] avg=4096.0000 max=4318 max_rank=0 min=3900 min_rank=6 imbalance_ratio=1.054199 max_load_reduction=28450 max_load_reduction_pct=0.868225 moved_to_echo=16252 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3094, 1, 0, 1065, 156, 17, 1, 21, 1, 1, 4077, 0, 0, 0, 0, 1, 1, 1, 1, 4094, 0, 0, 0, 497, 2, 0, 0, 3861, 1, 0, 0, 4091, 0, 11, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 1778, 0, 0, 1933, 0, 0, 0, 0, 1, 3887, 0, 77, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4334, 4100, 4098, 4361, 4102, 4097, 3711, 3965] avg=4096.0000 max=4361 max_rank=3 min=3711 min_rank=6 imbalance_ratio=1.064697 max_load_reduction=28407 max_load_reduction_pct=0.866913 moved_to_echo=15875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 2209, 1, 0, 1257, 156, 17, 1, 77, 1, 0, 4077, 0, 0, 0, 0, 1, 0, 1, 1, 4094, 0, 0, 0, 748, 2, 0, 0, 3862, 1, 0, 0, 4094, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 1817, 0, 0, 2283, 0, 0, 0, 0, 1, 3890, 0, 77, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3642, 4155, 4097, 4613, 4097, 4096, 4100, 3968] avg=4096.0000 max=4613 max_rank=3 min=3642 min_rank=0 imbalance_ratio=1.126221 max_load_reduction=28155 max_load_reduction_pct=0.859222 moved_to_echo=16261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32770, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32770, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32770, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32770, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32770, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32770, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32770, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32770, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32766 actual_valid_recv=33408 observed_loss=-642 tokens_per_expert=[128, 21248, 128, 0, 10240, 1280, 256, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[14336, 0, 0, 18560, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1920, 128, 0, 128, 30976, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32256, 0, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32770 actual_valid_recv=33024 observed_loss=-254 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1046, 4095, 4095, 4063, 1, 2, 0, 4083, 1, 0, 0, 32, 1, 4093, 0, 3, 0, 1, 0, 0, 0, 1, 4095, 0, 0, 0, 4095, 1, 0, 3059, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1542, 4096, 4096, 4096, 0, 0, 0, 4036, 0, 0, 4, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2608, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1047, 4096, 4096, 4096, 0, 0, 0, 4094, 0, 0, 11, 7, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3036, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[930, 4095, 4096, 4096, 0, 0, 2, 4094, 0, 0, 12, 2, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3155, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1859, 4096, 4096, 4096, 0, 0, 0, 4092, 0, 0, 1, 12, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2229, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[847, 4095, 4095, 4063, 2, 3, 2, 4091, 1, 0, 0, 31, 1, 4091, 0, 1, 0, 1, 0, 0, 0, 1, 4093, 0, 0, 0, 4095, 1, 0, 3253, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[780, 4096, 4096, 4096, 0, 0, 0, 4089, 0, 0, 11, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3310, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[486, 4095, 4095, 4077, 1, 2, 0, 4092, 1, 0, 0, 54, 1, 4093, 0, 3, 0, 1, 0, 0, 0, 1, 4094, 0, 0, 0, 4095, 1, 0, 3575, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1046, 0, 0, 3014, 0, 0, 0, 0, 1, 2, 0, 4083, 10, 0, 0, 0, 1, 0, 0, 32, 4073, 0, 0, 0, 1, 4093, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 3059, 0, 1, 1039, 22, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4096, 4106, 4097, 4096, 4096, 4096, 4121] avg=4096.0000 max=4121 max_rank=7 min=4060 min_rank=0 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16409 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[930, 0, 0, 3038, 0, 0, 0, 0, 0, 0, 2, 4094, 10, 0, 0, 0, 0, 0, 12, 2, 4072, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3155, 0, 0, 1047, 23, 0, 0] rerouted_tokens_per_ep_rank=[3968, 4106, 4086, 4094, 4097, 4096, 4096, 4225] avg=4096.0000 max=4225 max_rank=7 min=3968 min_rank=0 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16514 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1047, 0, 0, 3038, 0, 0, 0, 0, 0, 0, 0, 4094, 10, 0, 0, 0, 0, 0, 11, 7, 4073, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3036, 0, 0, 1047, 23, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4104, 4091, 4094, 4096, 4096, 4096, 4106] avg=4096.0000 max=4106 max_rank=7 min=4085 min_rank=0 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[780, 0, 0, 3038, 0, 0, 0, 0, 0, 0, 0, 4089, 10, 0, 0, 0, 0, 0, 11, 2, 4073, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3310, 0, 0, 1047, 23, 0, 0] rerouted_tokens_per_ep_rank=[3818, 4099, 4086, 4097, 4096, 4096, 4096, 4380] avg=4096.0000 max=4380 max_rank=7 min=3818 min_rank=0 imbalance_ratio=1.069336 max_load_reduction=28388 max_load_reduction_pct=0.866333 moved_to_echo=16668 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1542, 0, 0, 3026, 0, 0, 0, 0, 0, 0, 0, 4036, 12, 0, 0, 0, 0, 0, 4, 2, 4074, 1, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2608, 0, 0, 1051, 22, 0, 0] rerouted_tokens_per_ep_rank=[4568, 4048, 4081, 4100, 4095, 4099, 4096, 3681] avg=4096.0000 max=4568 max_rank=0 min=3681 min_rank=7 imbalance_ratio=1.115234 max_load_reduction=28200 max_load_reduction_pct=0.860596 moved_to_echo=15971 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[847, 0, 0, 3014, 0, 0, 0, 0, 2, 3, 2, 4091, 10, 0, 0, 0, 1, 0, 0, 31, 4072, 0, 0, 0, 1, 4091, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 3253, 0, 1, 1039, 23, 0, 0] rerouted_tokens_per_ep_rank=[3861, 4108, 4104, 4093, 4096, 4094, 4096, 4316] avg=4096.0000 max=4316 max_rank=7 min=3861 min_rank=0 imbalance_ratio=1.053711 max_load_reduction=28452 max_load_reduction_pct=0.868286 moved_to_echo=16602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1859, 0, 0, 3038, 0, 0, 0, 0, 0, 0, 0, 4092, 10, 0, 0, 0, 0, 0, 1, 12, 4074, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2229, 0, 0, 1047, 22, 0, 0] rerouted_tokens_per_ep_rank=[4897, 4102, 4087, 4096, 4096, 4096, 4096, 3298] avg=4096.0000 max=4897 max_rank=0 min=3298 min_rank=7 imbalance_ratio=1.195557 max_load_reduction=27871 max_load_reduction_pct=0.850555 moved_to_echo=15586 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[486, 0, 0, 3025, 0, 0, 0, 0, 1, 2, 0, 4092, 10, 0, 0, 0, 1, 0, 0, 54, 4072, 0, 0, 0, 1, 4093, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 3575, 0, 1, 1042, 23, 0, 0] rerouted_tokens_per_ep_rank=[3511, 4105, 4127, 4097, 4096, 4095, 4096, 4641] avg=4096.0000 max=4641 max_rank=7 min=3511 min_rank=0 imbalance_ratio=1.133057 max_load_reduction=28127 max_load_reduction_pct=0.858368 moved_to_echo=16928 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8576, 0, 0, 24320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 24320, 0, 128, 8448, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True + [2026-08-30 04:56:50] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 660.5 | throughput per GPU (TFLOP/s/GPU): 504.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.820198E+00 | loss scale: 1.0 | grad norm: 5.230 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 04:56:50 +[rank0]:[W830 04:56:52.360070681 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 04:56:56.870000 448070 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 04:56:56.870000 448070 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 448169 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 448070 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..7b6670a7d561506ecd7271cf0786ecb56dd11981 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1/mem_samples.csv @@ -0,0 +1,248 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 629 +3, 629 +4, 629 +5, 629 +6, 629 +7, 629 +0, 917 +1, 921 +2, 915 +3, 911 +4, 925 +5, 909 +6, 915 +7, 911 +0, 1895 +1, 1895 +2, 1895 +3, 1895 +4, 1895 +5, 1895 +6, 1895 +7, 1895 +0, 39339 +1, 39347 +2, 39375 +3, 39385 +4, 39357 +5, 39389 +6, 39401 +7, 39417 +0, 39701 +1, 39669 +2, 39675 +3, 39667 +4, 39697 +5, 39721 +6, 39679 +7, 39669 +0, 42263 +1, 42255 +2, 42255 +3, 42255 +4, 42255 +5, 42255 +6, 42255 +7, 42255 +0, 42507 +1, 42499 +2, 42499 +3, 42499 +4, 42499 +5, 42499 +6, 42499 +7, 42499 +0, 42597 +1, 42587 +2, 42589 +3, 42579 +4, 42589 +5, 42593 +6, 42589 +7, 42613 +0, 43423 +1, 43415 +2, 43415 +3, 43415 +4, 43415 +5, 43415 +6, 43415 +7, 43415 +0, 43701 +1, 43693 +2, 43693 +3, 43693 +4, 43693 +5, 43693 +6, 43693 +7, 43693 +0, 45133 +1, 45125 +2, 45125 +3, 45125 +4, 45125 +5, 45125 +6, 45125 +7, 45125 +0, 45133 +1, 45125 +2, 45125 +3, 45125 +4, 45125 +5, 45125 +6, 45125 +7, 45125 +0, 75713 +1, 75705 +2, 75705 +3, 75705 +4, 75705 +5, 75705 +6, 75705 +7, 75705 +0, 76199 +1, 76191 +2, 76191 +3, 76191 +4, 76191 +5, 76191 +6, 76191 +7, 76191 +0, 76199 +1, 76191 +2, 76191 +3, 76191 +4, 76191 +5, 76191 +6, 76191 +7, 76191 +0, 76199 +1, 76191 +2, 76191 +3, 76191 +4, 76191 +5, 76191 +6, 76191 +7, 76191 +0, 94035 +1, 94007 +2, 94007 +3, 94029 +4, 94019 +5, 94051 +6, 94053 +7, 94021 +0, 94895 +1, 94843 +2, 94911 +3, 94879 +4, 94917 +5, 94925 +6, 94925 +7, 94925 +0, 95581 +1, 95573 +2, 95573 +3, 95573 +4, 95573 +5, 95573 +6, 95573 +7, 95573 +0, 96159 +1, 96167 +2, 96169 +3, 96161 +4, 96153 +5, 96193 +6, 96165 +7, 96209 +0, 119267 +1, 119213 +2, 119213 +3, 119213 +4, 119187 +5, 119213 +6, 119213 +7, 119213 +0, 119267 +1, 119213 +2, 119213 +3, 119213 +4, 119187 +5, 119213 +6, 119213 +7, 119213 +0, 119267 +1, 119213 +2, 119213 +3, 119213 +4, 119187 +5, 119213 +6, 119213 +7, 119213 +0, 119267 +1, 119213 +2, 119213 +3, 119213 +4, 119187 +5, 119213 +6, 119213 +7, 119213 +0, 2118 +1, 118297 +2, 3398 +3, 118297 +4, 118271 +5, 118297 +6, 3142 +7, 1862 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1/run.env b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..4dffff4af09ede1c16232cb6cf2eb622dae9ab91 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1/run.env @@ -0,0 +1,70 @@ +backend=echo +config=G2 +family=tierA +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=3584 +num_experts=32 +topk=8 +experts_per_rank=4 +expert_params=44040192 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=32768 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=4 +capacity_mode=static +capacity=1.08203125 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1/train.log b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..35fc47025d43c101dfb3239345498e520bd088e2 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1/train.log @@ -0,0 +1,16469 @@ +[04:56:58] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G2 / echo (slots/rank=4 policy=matched) + E=32 K=8 H'=3584 m=4 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.08203125 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0830 04:56:59.794000 451943 torch/distributed/run.py:766] +W0830 04:56:59.794000 451943 torch/distributed/run.py:766] ***************************************** +W0830 04:56:59.794000 451943 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 04:56:59.794000 451943 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 3584 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 32 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.08203125 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 32 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank [Gloo] Rank [Gloo] Rank 0[Gloo] Rank 52 is connected to 1 is connected to is connected to 7 is connected to 77 peer ranks. 7 peer ranks. peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 77 + +7 + +[Gloo] Rank [Gloo] Rank [Gloo] Rank 36 is connected to is connected to 477 is connected to peer ranks. peer ranks. 7Expected number of connected peer ranks is : Expected number of connected peer ranks is : peer ranks. 7[Gloo] Rank 7Expected number of connected peer ranks is : + +77 is connected to +7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 20 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 0 is connected to 00 is connected to peer ranks. Expected number of connected peer ranks is : 0 peer ranks. 0Expected number of connected peer ranks is : 0[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank1]:[W830 04:57:17.879886431 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 04:57:17.879987831 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 04:57:17.880016157 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 04:57:17.880021597 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 04:57:17.880052114 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 04:57:17.880124350 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 04:57:17.881548041 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 04:57:17.075910859 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.807 seconds +time to initialize megatron (seconds): 7.137 +[after megatron is initialized] datetime: 2026-08-30 04:57:23 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3903983616 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42086400 elements, 42086400 padded size): + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight +Params for bucket 3 (42082304 elements, 42082304 padded size): + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.weight +Params for bucket 4 (42082304 elements, 42082304 padded size): + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_proj.weight +Params for bucket 5 (42082304 elements, 42082304 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.pre_mlp_layernorm.weight +Params for bucket 6 (42082304 elements, 42082304 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_proj.weight +Params for bucket 7 (42082304 elements, 42082304 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.pre_mlp_layernorm.weight +Params for bucket 8 (42082304 elements, 42082304 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight +Params for bucket 9 (42082304 elements, 42082304 padded size): + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight +Params for bucket 10 (42082304 elements, 42082304 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 11 (42082304 elements, 42082304 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.pre_mlp_layernorm.weight +Params for bucket 12 (42082304 elements, 42082304 padded size): + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 13 (42082304 elements, 42082304 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.pre_mlp_layernorm.weight +Params for bucket 14 (42082304 elements, 42082304 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight +Params for bucket 15 (42082304 elements, 42082304 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 16 (42082304 elements, 42082304 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight +Params for bucket 17 (42082304 elements, 42082304 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 56 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 3 (58720256 elements, 58720256 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 6 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 7 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 +Params for bucket 10 (58720256 elements, 58720256 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 +Params for bucket 13 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 +Params for bucket 14 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 +Params for bucket 20 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 24 (58720256 elements, 58720256 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 28 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 30 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 34 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 38 (58720256 elements, 58720256 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 +Params for bucket 41 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 42 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 43 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 45 (58720256 elements, 58720256 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 +Params for bucket 47 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 48 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 49 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 52 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 +Params for bucket 55 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 56 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 04:57:23 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 04:57:24 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (335.21, 337.28) + train/valid/test-data-iterators-setup ..........: (64.25, 69.87) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 04:57:24 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1041, 1003, 1054, 1035, 995, 1018, 1057, 1074, 1012, 1012, 1063, 1009, 1030, 1007, 999, 1021, 1055, 998, 998, 1015, 1045, 1006, 1022, 1027, 1006, 1050, 990, 1003, 1074, 1066, 1002, 981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1050, 986, 1003, 1063, 993, 1039, 1063, 1002, 1056, 1040, 1030, 1007, 1041, 1035, 1012, 1070, 1032, 985, 1024, 1018, 1011, 1036, 1053, 985, 990, 986, 1043, 1037, 994, 1020, 1059, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1026, 1054, 1024, 1043, 1042, 1013, 986, 1092, 984, 1042, 1008, 1060, 1032, 1041, 1022, 1052, 1012, 945, 1060, 1031, 1013, 1016, 975, 1007, 1029, 1019, 987, 1053, 1034, 991, 1040, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1067, 1026, 1041, 1024, 1023, 1024, 987, 1003, 1024, 1072, 1051, 1013, 1001, 1032, 990, 1019, 1008, 1023, 1002, 1040, 1049, 1029, 1049, 1032, 1025, 1017, 1008, 1055, 1002, 997, 1018, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1016, 1046, 1010, 1067, 1032, 1044, 997, 986, 1051, 1079, 1029, 967, 993, 1035, 1012, 1036, 1046, 1022, 1051, 1048, 1048, 999, 1019, 1024, 1010, 1041, 1036, 1000, 1005, 993, 1006, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1003, 1018, 1025, 1052, 1036, 1050, 1056, 1031, 1040, 1040, 1041, 940, 1012, 977, 988, 1020, 1038, 1058, 1038, 996, 1000, 1027, 1042, 1026, 1058, 1011, 1010, 1031, 1032, 1054, 983, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1080, 1032, 1019, 1034, 1016, 1069, 1027, 1024, 1001, 1044, 1014, 1043, 1028, 1026, 1039, 994, 1007, 1005, 1056, 968, 1033, 1009, 1008, 1029, 1008, 952, 986, 1055, 1052, 1042, 1008, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1041, 1048, 1043, 994, 1026, 1023, 1078, 1051, 1002, 1061, 1037, 1003, 1000, 972, 1019, 1018, 1034, 1016, 994, 1016, 1006, 995, 1036, 999, 1036, 1011, 1018, 1021, 1085, 1013, 1028, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[981, 1046, 1010, 1067, 0, 0, 0, 0, 1032, 1021, 997, 986, 0, 0, 0, 0, 1051, 1066, 1029, 967, 0, 0, 0, 0, 993, 1035, 1012, 1036, 23, 0, 0, 0, 1046, 1022, 1051, 1048, 13, 2, 0, 0, 1048, 999, 1019, 1024, 11, 3, 0, 0, 1010, 1041, 1036, 1000, 12, 10, 0, 0, 1002, 993, 1006, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4036, 4113, 4099, 4182, 4104, 4109, 4021] avg=4096.0000 max=4182 max_rank=4 min=4021 min_rank=7 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[968, 1018, 1025, 1052, 0, 0, 0, 0, 1036, 1027, 1056, 1031, 0, 0, 0, 0, 1040, 1028, 1041, 940, 0, 0, 0, 0, 1012, 977, 988, 1020, 23, 0, 0, 0, 1038, 1058, 1038, 996, 13, 2, 0, 0, 1000, 1027, 1042, 1026, 10, 3, 0, 0, 1058, 1011, 1010, 1031, 12, 10, 0, 0, 1029, 1054, 983, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4150, 4049, 4020, 4145, 4108, 4132, 4101] avg=4096.0000 max=4150 max_rank=1 min=4020 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1004, 1003, 1054, 1035, 0, 0, 0, 0, 995, 996, 1057, 1074, 0, 0, 0, 0, 1012, 1000, 1063, 1009, 0, 0, 0, 0, 1030, 1007, 999, 1021, 24, 0, 0, 0, 1055, 998, 998, 1015, 13, 2, 0, 0, 1045, 1006, 1022, 1027, 10, 3, 0, 0, 1006, 1050, 990, 1003, 13, 9, 0, 0, 1071, 1066, 1002, 981, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4122, 4084, 4081, 4081, 4113, 4071, 4120] avg=4096.0000 max=4122 max_rank=1 min=4071 min_rank=6 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1022, 1026, 1041, 1024, 0, 0, 0, 0, 1023, 994, 987, 1003, 0, 0, 0, 0, 1024, 1050, 1051, 1013, 0, 0, 0, 0, 1001, 1032, 990, 1019, 28, 0, 0, 0, 1008, 1023, 1002, 1040, 17, 7, 0, 0, 1049, 1029, 1049, 1032, 15, 6, 0, 0, 1025, 1017, 1008, 1055, 17, 13, 0, 0, 996, 997, 1018, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4007, 4138, 4070, 4097, 4180, 4135, 4028] avg=4096.0000 max=4180 max_rank=5 min=4007 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1013, 986, 1003, 1063, 0, 0, 0, 0, 993, 1016, 1063, 1002, 0, 0, 0, 0, 1056, 1028, 1030, 1007, 0, 0, 0, 0, 1041, 1035, 1012, 1070, 24, 0, 0, 0, 1032, 985, 1024, 1018, 13, 2, 0, 0, 1011, 1036, 1053, 985, 10, 3, 0, 0, 990, 986, 1043, 1037, 13, 10, 0, 0, 991, 1020, 1059, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4074, 4121, 4182, 4074, 4098, 4079, 4075] avg=4096.0000 max=4182 max_rank=3 min=4065 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1004, 1048, 1043, 994, 0, 0, 0, 0, 1026, 1000, 1078, 1051, 0, 0, 0, 0, 1002, 1049, 1037, 1003, 0, 0, 0, 0, 1000, 972, 1019, 1018, 24, 0, 0, 0, 1034, 1016, 994, 1016, 13, 2, 0, 0, 1006, 995, 1036, 999, 10, 3, 0, 0, 1036, 1011, 1018, 1021, 13, 10, 0, 0, 1082, 1013, 1028, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4155, 4091, 4033, 4075, 4049, 4109, 4167] avg=4096.0000 max=4167 max_rank=7 min=4033 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[990, 1054, 1024, 1043, 0, 0, 0, 0, 1042, 991, 986, 1092, 0, 0, 0, 0, 984, 1030, 1008, 1060, 0, 0, 0, 0, 1032, 1041, 1022, 1052, 24, 0, 0, 0, 1012, 945, 1060, 1031, 13, 2, 0, 0, 1013, 1016, 975, 1007, 10, 3, 0, 0, 1029, 1019, 987, 1053, 12, 9, 0, 0, 1031, 991, 1040, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4111, 4082, 4171, 4063, 4024, 4109, 4097] avg=4096.0000 max=4171 max_rank=3 min=4024 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1042, 1032, 1019, 1034, 0, 0, 0, 0, 1016, 1046, 1027, 1024, 0, 0, 0, 0, 1001, 1032, 1014, 1043, 0, 0, 0, 0, 1028, 1026, 1039, 994, 25, 0, 0, 0, 1007, 1005, 1056, 968, 13, 2, 0, 0, 1033, 1009, 1008, 1029, 10, 3, 0, 0, 1008, 952, 986, 1055, 13, 10, 0, 0, 1049, 1042, 1008, 1060, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4113, 4090, 4112, 4051, 4092, 4024, 4159] avg=4096.0000 max=4159 max_rank=7 min=4024 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1000, 1041, 1050, 1045, 1018, 995, 978, 1011, 1010, 1032, 1036, 993, 1065, 1009, 1039, 1035, 960, 998, 1039, 1078, 1025, 997, 1028, 1032, 1014, 1028, 1024, 1039, 1070, 1039, 1030, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[981, 1066, 1043, 1024, 1033, 1007, 1015, 989, 1039, 1018, 1038, 1034, 1004, 984, 1016, 1022, 1022, 1034, 1041, 1053, 1016, 1076, 1027, 1024, 1007, 1035, 1021, 1026, 1021, 972, 1046, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1047, 1034, 1036, 1040, 1026, 984, 1044, 1040, 998, 1012, 1034, 1040, 1018, 1034, 1003, 1057, 1075, 935, 996, 1011, 1004, 1003, 1064, 1017, 1053, 1022, 952, 1005, 1101, 1095, 961, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[992, 995, 1014, 999, 1057, 1050, 1006, 1052, 991, 1054, 1068, 1024, 1028, 1028, 1029, 970, 1041, 992, 1052, 1025, 984, 1040, 1000, 1032, 1024, 1072, 1014, 1067, 1032, 1000, 1060, 976] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1020, 1000, 1085, 1010, 1007, 992, 1012, 999, 1072, 1022, 1022, 1081, 1064, 990, 940, 1044, 1038, 991, 999, 1022, 1014, 1083, 1035, 1038, 1029, 961, 1029, 1041, 1071, 1004, 1008, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1026, 1009, 1048, 986, 1069, 1014, 1012, 1080, 986, 971, 1044, 1045, 1048, 1059, 1065, 977, 1050, 994, 994, 1011, 1036, 1021, 1042, 997, 1000, 1035, 979, 1028, 1029, 1005, 1069, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1022, 1029, 1057, 1007, 1045, 1011, 1033, 1037, 993, 1011, 1001, 1003, 994, 981, 1009, 1026, 1073, 1060, 1009, 1041, 1012, 1014, 1021, 1042, 1061, 1056, 991, 1042, 1039, 1002, 968, 1078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1061, 996, 1055, 991, 968, 1010, 1025, 999, 1042, 1017, 994, 1058, 988, 1030, 1045, 1046, 1034, 1065, 1065, 997, 1034, 1009, 1017, 1039, 1008, 1061, 1012, 1013, 1000, 1006, 1044, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1000, 1041, 1045, 1045, 0, 0, 0, 0, 1018, 995, 978, 1011, 19, 0, 0, 0, 1010, 1032, 1036, 992, 0, 0, 0, 0, 1065, 1009, 1039, 1035, 6, 5, 2, 0, 958, 998, 1039, 1078, 0, 0, 0, 0, 1025, 991, 1028, 1032, 0, 0, 0, 0, 1014, 1028, 1024, 1039, 1, 0, 0, 0, 1051, 1039, 1030, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4021, 4070, 4161, 4073, 4076, 4106, 4130] avg=4096.0000 max=4161 max_rank=3 min=4021 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[981, 1066, 1038, 1024, 0, 0, 0, 0, 1033, 1007, 1015, 989, 18, 0, 0, 0, 1039, 1018, 1038, 1033, 0, 0, 0, 0, 1004, 984, 1016, 1022, 7, 5, 2, 0, 1020, 1034, 1041, 1053, 0, 0, 0, 0, 1016, 1069, 1027, 1024, 0, 0, 0, 0, 1007, 1035, 1021, 1026, 1, 0, 0, 0, 1003, 972, 1046, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4062, 4128, 4040, 4148, 4136, 4090, 4055] avg=4096.0000 max=4148 max_rank=4 min=4040 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 1009, 1043, 986, 0, 0, 0, 0, 1069, 1014, 1012, 1080, 18, 0, 0, 0, 986, 971, 1044, 1044, 0, 0, 0, 0, 1048, 1059, 1065, 977, 6, 5, 2, 0, 1048, 994, 994, 1011, 0, 0, 0, 0, 1036, 1015, 1042, 997, 0, 0, 0, 0, 1000, 1035, 979, 1028, 1, 0, 0, 0, 1011, 1005, 1069, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4193, 4045, 4162, 4047, 4090, 4043, 4124] avg=4096.0000 max=4193 max_rank=1 min=4043 min_rank=6 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1022, 1029, 1052, 1007, 0, 0, 0, 0, 1045, 1011, 1033, 1037, 18, 0, 0, 0, 993, 1011, 1001, 1002, 0, 0, 0, 0, 994, 981, 1009, 1026, 6, 5, 2, 0, 1071, 1060, 1009, 1041, 0, 0, 0, 0, 1012, 1008, 1021, 1042, 0, 0, 0, 0, 1061, 1056, 991, 1042, 1, 0, 0, 0, 1021, 1002, 968, 1078, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4144, 4007, 4023, 4181, 4083, 4151, 4069] avg=4096.0000 max=4181 max_rank=4 min=4007 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[992, 995, 1010, 999, 0, 0, 0, 0, 1057, 1050, 1006, 1052, 18, 0, 0, 0, 991, 1054, 1068, 1023, 0, 0, 0, 0, 1028, 1028, 1029, 970, 6, 4, 2, 0, 1039, 992, 1052, 1025, 0, 0, 0, 0, 984, 1034, 1000, 1032, 0, 0, 0, 0, 1024, 1072, 1014, 1067, 1, 0, 0, 0, 1014, 1000, 1060, 976, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4183, 4136, 4067, 4108, 4050, 4178, 4050] avg=4096.0000 max=4183 max_rank=1 min=3996 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1047, 1034, 1029, 1040, 0, 0, 0, 0, 1026, 984, 1044, 1040, 23, 0, 0, 0, 998, 1012, 1034, 1032, 0, 0, 0, 0, 1018, 1034, 1003, 1057, 11, 7, 9, 2, 1062, 935, 996, 1011, 0, 0, 0, 0, 1004, 992, 1064, 1017, 0, 0, 0, 0, 1053, 1022, 952, 1005, 8, 4, 0, 0, 1076, 1095, 961, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4117, 4076, 4141, 4004, 4077, 4044, 4159] avg=4096.0000 max=4159 max_rank=7 min=4004 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1020, 1000, 1080, 1010, 0, 0, 0, 0, 1007, 992, 1012, 999, 19, 0, 0, 0, 1072, 1022, 1022, 1080, 0, 0, 0, 0, 1064, 990, 940, 1044, 7, 5, 2, 0, 1036, 991, 999, 1022, 0, 0, 0, 0, 1014, 1076, 1035, 1038, 0, 0, 0, 0, 1029, 961, 1029, 1041, 1, 0, 0, 0, 1052, 1004, 1008, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4029, 4196, 4052, 4048, 4163, 4061, 4109] avg=4096.0000 max=4196 max_rank=2 min=4029 min_rank=1 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1061, 996, 1050, 991, 0, 0, 0, 0, 968, 1010, 1025, 999, 17, 0, 0, 0, 1042, 1017, 994, 1057, 0, 0, 0, 0, 988, 1030, 1045, 1046, 6, 5, 2, 0, 1032, 1065, 1065, 997, 0, 0, 0, 0, 1034, 1003, 1017, 1039, 0, 0, 0, 0, 1008, 1061, 1012, 1013, 1, 0, 0, 0, 983, 1006, 1044, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4019, 4110, 4122, 4159, 4093, 4095, 4072] avg=4096.0000 max=4159 max_rank=4 min=4019 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8448, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1010, 970, 1054, 1072, 1003, 1013, 1023, 995, 1013, 1042, 1052, 1002, 993, 990, 999, 1066, 1023, 1031, 963, 1054, 1015, 1064, 1056, 1032, 1037, 1020, 1046, 1033, 1041, 1025, 1012, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1046, 1046, 1113, 969, 1001, 1011, 1040, 1025, 1014, 972, 1019, 1022, 1079, 1042, 1012, 1008, 1018, 1051, 1011, 1004, 1000, 1041, 1026, 1016, 1030, 1073, 979, 984, 1035, 1015, 1019, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1023, 1042, 1077, 1015, 1041, 989, 1025, 1063, 1028, 1008, 1019, 1028, 1011, 1007, 1051, 1025, 1005, 984, 1033, 1015, 1013, 1018, 1056, 990, 1039, 1022, 1061, 1032, 1070, 1016, 952, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1041, 1021, 1069, 1074, 1028, 1027, 1022, 1057, 1011, 1035, 1047, 1019, 1010, 998, 1020, 1005, 953, 1035, 982, 1047, 984, 983, 1043, 969, 1033, 1035, 1004, 1098, 1052, 1017, 1026, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1038, 1056, 1037, 995, 1031, 1069, 1045, 1065, 1046, 1074, 957, 1038, 993, 1033, 1021, 1023, 995, 1044, 993, 1007, 1061, 1039, 1030, 1043, 985, 1004, 1017, 987, 1072, 963, 989, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1032, 1001, 1040, 1006, 1082, 1031, 1009, 1010, 1043, 991, 1032, 994, 1057, 1003, 1006, 1019, 1057, 999, 985, 1001, 1025, 1035, 1050, 991, 1074, 1049, 1072, 1004, 990, 1044, 1023, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1064, 982, 1003, 1035, 1024, 1004, 1040, 1059, 1037, 1035, 976, 973, 1018, 1053, 1037, 1019, 1040, 1053, 1015, 1020, 1030, 1052, 1012, 1009, 1078, 1017, 1020, 1026, 1045, 1022, 978, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1048, 960, 1019, 1020, 1027, 995, 1022, 1056, 1060, 1074, 969, 989, 1068, 1062, 1069, 1061, 997, 1028, 1041, 1046, 1044, 1030, 1014, 1012, 1017, 1016, 1004, 977, 1034, 1030, 985, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1010, 970, 1028, 1072, 0, 0, 0, 0, 1003, 1013, 1023, 976, 0, 0, 0, 0, 1013, 1042, 1052, 1002, 10, 5, 1, 0, 983, 990, 999, 1066, 0, 0, 0, 0, 1023, 1031, 963, 1054, 26, 3, 0, 0, 1015, 1064, 1055, 1032, 0, 0, 0, 0, 1025, 1020, 1046, 1033, 0, 0, 0, 0, 1041, 1025, 1012, 1019, 16, 7, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4015, 4125, 4038, 4100, 4166, 4124, 4120] avg=4096.0000 max=4166 max_rank=5 min=4015 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1064, 982, 973, 1035, 0, 0, 0, 0, 1024, 1004, 1040, 1032, 0, 0, 0, 0, 1037, 1035, 976, 973, 15, 9, 8, 0, 1003, 1053, 1037, 1019, 0, 0, 0, 0, 1040, 1053, 1015, 1020, 30, 7, 0, 0, 1030, 1052, 1004, 1009, 0, 0, 0, 0, 1057, 1017, 1020, 1026, 0, 0, 0, 0, 1045, 1022, 978, 992, 20, 12, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4100, 4053, 4112, 4165, 4095, 4120, 4069] avg=4096.0000 max=4165 max_rank=4 min=4053 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1038, 1056, 1012, 995, 0, 0, 0, 0, 1031, 1069, 1045, 1045, 0, 0, 0, 0, 1046, 1074, 957, 1038, 10, 5, 1, 0, 983, 1033, 1021, 1023, 0, 0, 0, 0, 995, 1044, 993, 1007, 25, 3, 0, 0, 1061, 1039, 1029, 1043, 0, 0, 0, 0, 973, 1004, 1017, 987, 0, 0, 0, 0, 1072, 963, 989, 1018, 17, 7, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4190, 4131, 4060, 4067, 4172, 3981, 4066] avg=4096.0000 max=4190 max_rank=1 min=3981 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1048, 960, 994, 1020, 0, 0, 0, 0, 1027, 995, 1022, 1036, 0, 0, 0, 0, 1060, 1074, 969, 989, 11, 5, 1, 0, 1057, 1062, 1069, 1061, 0, 0, 0, 0, 997, 1028, 1041, 1046, 25, 3, 0, 0, 1044, 1030, 1013, 1012, 0, 0, 0, 0, 1005, 1016, 1004, 977, 0, 0, 0, 0, 1034, 1030, 985, 994, 17, 7, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4080, 4109, 4249, 4140, 4099, 4002, 4067] avg=4096.0000 max=4249 max_rank=3 min=4002 min_rank=6 imbalance_ratio=1.037354 max_load_reduction=28519 max_load_reduction_pct=0.870331 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1032, 1001, 1015, 1006, 0, 0, 0, 0, 1082, 1031, 1009, 991, 0, 0, 0, 0, 1043, 991, 1032, 994, 11, 5, 1, 0, 1046, 1003, 1006, 1019, 0, 0, 0, 0, 1057, 999, 985, 1001, 25, 3, 0, 0, 1025, 1035, 1049, 991, 0, 0, 0, 0, 1062, 1049, 1072, 1004, 0, 0, 0, 0, 990, 1044, 1023, 1013, 16, 7, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4113, 4077, 4074, 4070, 4100, 4187, 4093] avg=4096.0000 max=4187 max_rank=6 min=4054 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1041, 1021, 1043, 1074, 0, 0, 0, 0, 1028, 1027, 1022, 1037, 0, 0, 0, 0, 1011, 1035, 1047, 1019, 11, 5, 1, 0, 999, 998, 1020, 1005, 0, 0, 0, 0, 953, 1035, 982, 1047, 26, 3, 0, 0, 984, 983, 1042, 969, 0, 0, 0, 0, 1021, 1035, 1004, 1098, 0, 0, 0, 0, 1052, 1017, 1026, 1023, 17, 7, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4114, 4129, 4022, 4046, 3978, 4158, 4142] avg=4096.0000 max=4179 max_rank=0 min=3978 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1023, 1042, 1051, 1015, 0, 0, 0, 0, 1041, 989, 1025, 1043, 0, 0, 0, 0, 1028, 1008, 1019, 1028, 11, 5, 1, 0, 1000, 1007, 1051, 1025, 0, 0, 0, 0, 1005, 984, 1033, 1015, 26, 3, 0, 0, 1013, 1018, 1055, 990, 0, 0, 0, 0, 1027, 1022, 1061, 1032, 0, 0, 0, 0, 1070, 1016, 952, 1010, 17, 7, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4098, 4100, 4083, 4066, 4076, 4142, 4072] avg=4096.0000 max=4142 max_rank=6 min=4066 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1046, 1046, 1086, 969, 0, 0, 0, 0, 1001, 1011, 1040, 1006, 0, 0, 0, 0, 1014, 972, 1019, 1022, 11, 5, 1, 0, 1068, 1042, 1012, 1008, 0, 0, 0, 0, 1018, 1051, 1011, 1004, 27, 3, 0, 0, 1000, 1041, 1025, 1016, 0, 0, 0, 0, 1018, 1073, 979, 984, 0, 0, 0, 0, 1035, 1015, 1019, 1047, 16, 7, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4058, 4044, 4130, 4114, 4082, 4054, 4139] avg=4096.0000 max=4147 max_rank=0 min=4044 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8448, 8192, 8064, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1026, 997, 968, 1004, 1024, 1002, 1021, 1009, 993, 1018, 998, 1048, 1085, 1047, 1036, 1031, 1033, 985, 1034, 1029, 1001, 1014, 1040, 1017, 1005, 1028, 1060, 1047, 1047, 1031, 1013, 1077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1049, 1061, 998, 1060, 990, 985, 1071, 1064, 1001, 984, 977, 1044, 1027, 1034, 1038, 993, 1045, 1047, 1058, 1010, 995, 1001, 1033, 983, 992, 1046, 1024, 982, 1030, 1072, 1033, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1025, 999, 1043, 1025, 1041, 1036, 1050, 1042, 1022, 1020, 1021, 1015, 993, 1099, 1041, 1013, 1006, 1022, 1034, 1029, 1053, 994, 1027, 994, 1029, 1019, 1013, 1025, 1040, 988, 1018, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[991, 1018, 1005, 1023, 1006, 1052, 1044, 1017, 1016, 994, 1101, 1043, 1022, 987, 1016, 1014, 992, 1053, 1000, 1050, 1074, 1081, 1054, 1017, 1019, 970, 1076, 1023, 986, 950, 1035, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1074, 1022, 1044, 1038, 1022, 1004, 1059, 991, 1046, 1028, 1013, 1036, 1069, 1022, 987, 1032, 1008, 1021, 988, 1045, 983, 991, 1065, 1009, 977, 1005, 1020, 992, 999, 1073, 1022, 1083] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1033, 1005, 1042, 1042, 1027, 1042, 1016, 1023, 1019, 1003, 975, 1012, 1007, 984, 1011, 1040, 1046, 1014, 1033, 1052, 1036, 1017, 1002, 1027, 1036, 1066, 1052, 1036, 1043, 1018, 956, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1032, 984, 1005, 1002, 1014, 1040, 1028, 1005, 1060, 1022, 998, 1015, 1002, 1029, 1022, 1018, 993, 1057, 1039, 1015, 987, 1039, 1040, 1023, 1051, 1053, 1031, 1024, 1010, 972, 1068, 1090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1060, 1055, 1051, 1064, 1038, 1022, 1034, 993, 1066, 1032, 987, 1014, 1006, 1008, 1002, 1018, 1033, 1039, 982, 1009, 1053, 1004, 1004, 964, 1028, 1028, 1045, 1032, 1036, 1000, 1025, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1018, 997, 968, 1004, 0, 0, 0, 0, 1024, 1002, 1016, 1009, 0, 0, 0, 0, 993, 1018, 998, 1048, 13, 4, 0, 0, 1085, 1047, 1036, 1031, 4, 0, 0, 0, 1033, 985, 1034, 1025, 0, 0, 0, 0, 1001, 1014, 1040, 1017, 8, 5, 4, 0, 1005, 1028, 1052, 1047, 0, 0, 0, 0, 1047, 1031, 1013, 1064, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3987, 4051, 4074, 4203, 4077, 4089, 4132, 4155] avg=4096.0000 max=4203 max_rank=3 min=3987 min_rank=0 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1017, 999, 1043, 1025, 0, 0, 0, 0, 1041, 1036, 1045, 1042, 0, 0, 0, 0, 1022, 1020, 1021, 1015, 12, 4, 0, 0, 993, 1099, 1041, 1013, 4, 0, 0, 0, 1006, 1022, 1034, 1025, 0, 0, 0, 0, 1053, 994, 1027, 994, 8, 5, 4, 0, 1029, 1019, 1005, 1025, 0, 0, 0, 0, 1040, 988, 1018, 980, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4164, 4094, 4150, 4087, 4085, 4078, 4026] avg=4096.0000 max=4164 max_rank=1 min=4026 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16276 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1025, 1005, 1042, 1042, 0, 0, 0, 0, 1027, 1042, 1011, 1023, 0, 0, 0, 0, 1019, 1003, 975, 1012, 13, 4, 0, 0, 1007, 984, 1011, 1040, 4, 0, 0, 0, 1046, 1014, 1033, 1048, 0, 0, 0, 0, 1036, 1017, 1002, 1027, 8, 5, 4, 0, 1036, 1066, 1044, 1036, 0, 0, 0, 0, 1043, 1018, 956, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4103, 4026, 4046, 4141, 4099, 4182, 4057] avg=4096.0000 max=4182 max_rank=6 min=4026 min_rank=2 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1052, 1055, 1051, 1064, 0, 0, 0, 0, 1038, 1022, 1029, 993, 0, 0, 0, 0, 1066, 1032, 987, 1014, 13, 4, 0, 0, 1006, 1008, 1002, 1018, 4, 0, 0, 0, 1033, 1039, 982, 1005, 0, 0, 0, 0, 1053, 1004, 1004, 964, 8, 5, 4, 0, 1028, 1028, 1037, 1032, 0, 0, 0, 0, 1036, 1000, 1025, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4222, 4082, 4116, 4038, 4059, 4042, 4125, 4084] avg=4096.0000 max=4222 max_rank=0 min=4038 min_rank=3 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[983, 1018, 1005, 1023, 0, 0, 0, 0, 1006, 1052, 1039, 1017, 0, 0, 0, 0, 1016, 994, 1101, 1043, 13, 4, 0, 0, 1022, 987, 1016, 1014, 4, 0, 0, 0, 992, 1053, 1000, 1046, 0, 0, 0, 0, 1074, 1081, 1054, 1017, 8, 5, 4, 0, 1019, 970, 1068, 1023, 0, 0, 0, 0, 986, 950, 1035, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4114, 4171, 4043, 4091, 4243, 4080, 3997] avg=4096.0000 max=4243 max_rank=5 min=3997 min_rank=7 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1041, 1061, 998, 1060, 0, 0, 0, 0, 990, 985, 1066, 1064, 0, 0, 0, 0, 1001, 984, 977, 1044, 13, 4, 0, 0, 1027, 1034, 1038, 993, 4, 0, 0, 0, 1045, 1047, 1058, 1006, 0, 0, 0, 0, 995, 1001, 1033, 983, 8, 5, 4, 0, 992, 1046, 1016, 982, 0, 0, 0, 0, 1030, 1072, 1033, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4105, 4023, 4096, 4156, 4029, 4036, 4163] avg=4096.0000 max=4163 max_rank=7 min=4023 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1053, 1022, 1044, 1038, 0, 0, 0, 0, 1022, 1004, 1050, 991, 0, 0, 0, 0, 1046, 1028, 1013, 1036, 18, 11, 0, 0, 1069, 1022, 987, 1032, 5, 2, 0, 0, 1008, 1021, 988, 1040, 0, 0, 0, 0, 983, 991, 1065, 1009, 10, 7, 10, 0, 977, 1005, 1010, 992, 0, 0, 0, 0, 999, 1073, 1022, 1065, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4067, 4152, 4117, 4057, 4075, 3984, 4159] avg=4096.0000 max=4159 max_rank=7 min=3984 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1024, 984, 1005, 1002, 0, 0, 0, 0, 1014, 1040, 1023, 1005, 0, 0, 0, 0, 1060, 1022, 998, 1015, 13, 4, 0, 0, 1002, 1029, 1022, 1018, 4, 0, 0, 0, 993, 1057, 1039, 1011, 0, 0, 0, 0, 987, 1039, 1040, 1023, 8, 5, 4, 0, 1051, 1053, 1023, 1024, 0, 0, 0, 0, 1010, 972, 1068, 1077, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4082, 4112, 4075, 4100, 4106, 4151, 4127] avg=4096.0000 max=4151 max_rank=6 min=4015 min_rank=0 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8064, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[985, 1051, 1046, 1080, 990, 1038, 1046, 1025, 1059, 963, 969, 1018, 1028, 1049, 994, 1025, 1034, 1029, 1009, 1040, 965, 1034, 1057, 1002, 1053, 1069, 1029, 983, 1007, 1033, 1026, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1029, 1043, 1006, 1011, 1067, 1024, 1017, 1033, 1064, 973, 1018, 988, 1010, 1036, 1020, 1017, 1011, 1033, 1006, 1039, 1041, 1007, 1030, 979, 1043, 1014, 1020, 1045, 1031, 1036, 1016, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1026, 1054, 983, 1015, 1030, 955, 1004, 996, 1046, 994, 989, 992, 1063, 991, 1032, 1006, 1011, 1036, 1043, 1056, 1049, 1011, 1012, 1027, 1063, 1058, 1079, 1021, 1045, 1028, 1032, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1033, 999, 1009, 1067, 1025, 1030, 1043, 971, 975, 1109, 994, 997, 1029, 1044, 1079, 978, 1023, 1065, 1008, 1034, 988, 1006, 988, 1038, 1083, 1022, 1067, 1030, 967, 1060, 970, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1009, 1020, 1008, 1016, 1033, 1064, 1013, 1029, 1026, 1043, 1009, 1023, 1023, 1016, 1044, 1060, 1062, 958, 961, 1013, 1037, 1034, 1042, 1032, 1023, 1004, 1029, 1040, 1048, 1027, 1033, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1038, 1010, 997, 1025, 1047, 1038, 1013, 1027, 1007, 974, 1036, 1022, 1080, 1010, 1035, 1012, 1044, 1007, 1030, 1057, 1032, 1026, 1025, 1036, 1018, 1006, 1045, 1048, 1004, 1002, 1011, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[977, 1060, 1060, 1037, 1039, 1004, 1000, 970, 1053, 1076, 1001, 1014, 1019, 1013, 1000, 1058, 1040, 999, 1022, 1017, 1018, 1022, 988, 991, 1051, 1014, 1013, 1038, 1034, 1100, 1033, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[984, 1029, 1043, 1001, 1056, 1010, 1051, 1031, 989, 987, 1063, 1059, 1007, 983, 1019, 1014, 1024, 1012, 1034, 1054, 1029, 1023, 1022, 1034, 978, 1011, 1001, 1007, 1133, 1018, 1057, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1029, 1043, 1006, 1011, 2, 0, 0, 0, 1067, 1024, 1017, 1033, 5, 1, 0, 0, 1064, 973, 1018, 988, 29, 0, 0, 0, 1007, 1036, 1020, 1017, 0, 0, 0, 0, 1011, 1033, 1006, 1034, 0, 0, 0, 0, 1041, 1007, 1030, 979, 17, 0, 0, 0, 1014, 1014, 1020, 1045, 0, 0, 0, 0, 1031, 1019, 1016, 1061, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4147, 4072, 4080, 4084, 4074, 4093, 4127] avg=4096.0000 max=4147 max_rank=1 min=4072 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[985, 1051, 1046, 1080, 2, 0, 0, 0, 990, 1038, 1046, 1025, 5, 1, 0, 0, 1059, 963, 969, 1018, 30, 0, 0, 0, 1025, 1049, 994, 1025, 0, 0, 0, 0, 1034, 1029, 1009, 1035, 0, 0, 0, 0, 965, 1034, 1057, 1002, 17, 0, 0, 0, 1023, 1069, 1029, 983, 0, 0, 0, 0, 1007, 1016, 1026, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4105, 4039, 4093, 4107, 4075, 4104, 4081] avg=4096.0000 max=4164 max_rank=0 min=4039 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1009, 1020, 1008, 1016, 2, 0, 0, 0, 1033, 1064, 1013, 1029, 4, 1, 0, 0, 1026, 1043, 1009, 1023, 29, 0, 0, 0, 1020, 1016, 1044, 1060, 0, 0, 0, 0, 1062, 958, 961, 1009, 0, 0, 0, 0, 1037, 1034, 1042, 1032, 17, 0, 0, 0, 994, 1004, 1029, 1040, 0, 0, 0, 0, 1048, 1010, 1033, 989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4144, 4130, 4140, 3990, 4162, 4067, 4080] avg=4096.0000 max=4162 max_rank=5 min=3990 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1038, 1010, 997, 1025, 3, 0, 0, 0, 1047, 1038, 1013, 1027, 8, 2, 0, 0, 1007, 974, 1036, 1022, 33, 1, 0, 0, 1075, 1010, 1035, 1012, 0, 0, 0, 0, 1044, 1007, 1030, 1046, 0, 0, 0, 0, 1032, 1026, 1025, 1036, 20, 3, 0, 0, 985, 1006, 1045, 1048, 0, 0, 0, 0, 1004, 981, 1011, 1006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4135, 4073, 4132, 4127, 4142, 4084, 4002] avg=4096.0000 max=4142 max_rank=5 min=4002 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1033, 999, 1009, 1067, 2, 0, 0, 0, 1025, 1030, 1043, 971, 4, 1, 0, 0, 975, 1109, 994, 997, 30, 0, 0, 0, 1026, 1044, 1079, 978, 0, 0, 0, 0, 1023, 1065, 1008, 1030, 0, 0, 0, 0, 988, 1006, 988, 1038, 17, 0, 0, 0, 1053, 1022, 1067, 1030, 0, 0, 0, 0, 967, 1043, 970, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4074, 4105, 4127, 4126, 4037, 4172, 4017] avg=4096.0000 max=4172 max_rank=6 min=4017 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[984, 1029, 1043, 1001, 2, 0, 0, 0, 1056, 1010, 1051, 1031, 5, 1, 0, 0, 989, 987, 1063, 1059, 27, 0, 0, 0, 1004, 983, 1019, 1014, 0, 0, 0, 0, 1024, 1012, 1034, 1049, 0, 0, 0, 0, 1029, 1023, 1022, 1034, 17, 0, 0, 0, 951, 1011, 1001, 1007, 0, 0, 0, 0, 1133, 1001, 1057, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4154, 4125, 4020, 4119, 4125, 3970, 4196] avg=4096.0000 max=4196 max_rank=7 min=3970 min_rank=6 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1026, 1054, 983, 1015, 2, 0, 0, 0, 1030, 955, 1004, 996, 5, 1, 0, 0, 1046, 994, 989, 992, 30, 0, 0, 0, 1060, 991, 1032, 1006, 0, 0, 0, 0, 1011, 1036, 1043, 1051, 0, 0, 0, 0, 1049, 1011, 1012, 1027, 17, 0, 0, 0, 1033, 1058, 1079, 1021, 0, 0, 0, 0, 1045, 1011, 1032, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 3991, 4051, 4089, 4141, 4116, 4191, 4109] avg=4096.0000 max=4191 max_rank=6 min=3991 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16557 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[977, 1060, 1060, 1037, 2, 0, 0, 0, 1039, 1004, 1000, 970, 4, 1, 0, 0, 1053, 1076, 1001, 1014, 29, 0, 0, 0, 1016, 1013, 1000, 1058, 0, 0, 0, 0, 1040, 999, 1022, 1013, 0, 0, 0, 0, 1018, 1022, 988, 991, 18, 0, 0, 0, 1022, 1014, 1013, 1038, 0, 0, 0, 0, 1034, 1082, 1033, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4018, 4173, 4087, 4074, 4037, 4087, 4156] avg=4096.0000 max=4173 max_rank=2 min=4018 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1037, 1044, 981, 1054, 1018, 1003, 1045, 1030, 1045, 996, 1002, 1038, 1013, 1030, 1005, 1032, 1038, 1055, 1025, 959, 1065, 1045, 984, 1007, 1008, 1036, 1040, 988, 1048, 1073, 1019, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[990, 1027, 1045, 1058, 1015, 1006, 1049, 1023, 1006, 1034, 1000, 1062, 998, 1001, 1000, 998, 1060, 994, 1008, 1024, 1005, 1046, 1028, 1025, 1070, 1033, 1000, 962, 1056, 1039, 1054, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1032, 1027, 1067, 1027, 979, 1060, 1020, 995, 991, 1019, 1040, 1021, 1037, 1027, 1044, 1100, 1000, 1064, 1009, 1041, 1011, 1006, 1024, 980, 1059, 1016, 1056, 1010, 1047, 959, 1018, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1046, 1015, 1001, 1018, 1053, 1034, 1064, 1023, 1013, 977, 999, 1000, 1030, 1029, 999, 1011, 1000, 1049, 1052, 1050, 1037, 1026, 983, 1053, 1026, 1039, 1024, 1006, 1014, 1001, 1036, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1007, 1032, 1014, 1011, 985, 1005, 996, 1028, 1059, 1031, 1012, 1040, 1040, 1036, 1025, 1005, 993, 1046, 984, 1023, 1053, 1006, 1021, 1060, 1078, 980, 1044, 1023, 1032, 1045, 1026, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1052, 989, 1024, 1006, 1023, 1022, 1008, 1105, 999, 1067, 1038, 1015, 1057, 995, 1019, 988, 970, 1039, 995, 998, 1042, 1030, 1048, 1015, 986, 1034, 1018, 1039, 1062, 1003, 1022, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1017, 1024, 1009, 1046, 1018, 1026, 1087, 1013, 1001, 1002, 1048, 1010, 1025, 1042, 1023, 999, 1076, 1017, 983, 994, 1045, 1036, 1055, 1045, 1000, 1045, 1003, 1020, 1021, 1020, 1021, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1031, 1044, 1000, 986, 999, 1026, 997, 1054, 1075, 1023, 1015, 1040, 1046, 1072, 1056, 1034, 1040, 992, 981, 1001, 977, 1052, 1017, 1025, 1066, 1046, 1009, 1022, 994, 1053, 1012, 983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1037, 1044, 981, 1054, 0, 0, 0, 0, 1018, 1003, 1045, 1026, 0, 0, 0, 0, 1045, 996, 1002, 1038, 4, 1, 0, 0, 1008, 1030, 1005, 1032, 0, 0, 0, 0, 1038, 1055, 1025, 959, 10, 9, 5, 0, 1065, 1035, 984, 1007, 0, 0, 0, 0, 1007, 1036, 1040, 988, 0, 0, 0, 0, 1039, 1073, 1019, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4092, 4086, 4075, 4101, 4091, 4071, 4136] avg=4096.0000 max=4136 max_rank=7 min=4071 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1052, 989, 1024, 1006, 0, 0, 0, 0, 1023, 1022, 1008, 1100, 0, 0, 0, 0, 999, 1067, 1038, 1015, 5, 1, 0, 0, 1051, 995, 1019, 988, 0, 0, 0, 0, 970, 1039, 995, 998, 10, 9, 6, 0, 1042, 1020, 1048, 1015, 0, 0, 0, 0, 985, 1034, 1018, 1039, 0, 0, 0, 0, 1053, 1003, 1022, 1060, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4153, 4125, 4053, 4027, 4125, 4076, 4138] avg=4096.0000 max=4153 max_rank=1 min=4027 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1017, 1024, 1009, 1046, 0, 0, 0, 0, 1018, 1026, 1087, 1009, 0, 0, 0, 0, 1001, 1002, 1048, 1010, 4, 1, 0, 0, 1020, 1042, 1023, 999, 0, 0, 0, 0, 1076, 1017, 983, 994, 10, 9, 5, 0, 1045, 1026, 1055, 1045, 0, 0, 0, 0, 999, 1045, 1003, 1020, 0, 0, 0, 0, 1012, 1020, 1021, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4140, 4066, 4084, 4094, 4171, 4067, 4050] avg=4096.0000 max=4171 max_rank=5 min=4050 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[990, 1027, 1045, 1058, 0, 0, 0, 0, 1015, 1006, 1049, 1019, 0, 0, 0, 0, 1006, 1034, 1000, 1062, 4, 1, 0, 0, 993, 1001, 1000, 998, 0, 0, 0, 0, 1060, 994, 1008, 1024, 10, 9, 5, 0, 1005, 1036, 1028, 1025, 0, 0, 0, 0, 1069, 1033, 1000, 962, 0, 0, 0, 0, 1047, 1039, 1054, 1052, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4089, 4107, 3992, 4110, 4094, 4064, 4192] avg=4096.0000 max=4192 max_rank=7 min=3992 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1046, 1015, 1001, 1018, 0, 0, 0, 0, 1053, 1034, 1064, 1019, 0, 0, 0, 0, 1013, 977, 999, 1000, 4, 1, 0, 0, 1025, 1029, 999, 1011, 0, 0, 0, 0, 1000, 1049, 1052, 1050, 10, 9, 5, 0, 1037, 1016, 983, 1053, 0, 0, 0, 0, 1025, 1039, 1024, 1006, 0, 0, 0, 0, 1005, 1001, 1036, 1060, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4170, 3994, 4064, 4175, 4089, 4094, 4102] avg=4096.0000 max=4175 max_rank=4 min=3994 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1007, 1032, 1014, 1011, 0, 0, 0, 0, 985, 1005, 996, 1024, 0, 0, 0, 0, 1059, 1031, 1012, 1040, 4, 1, 0, 0, 1034, 1036, 1025, 1005, 0, 0, 0, 0, 993, 1046, 984, 1023, 10, 9, 6, 0, 1053, 996, 1021, 1060, 0, 0, 0, 0, 1077, 980, 1044, 1023, 0, 0, 0, 0, 1023, 1045, 1026, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4010, 4147, 4100, 4071, 4130, 4124, 4122] avg=4096.0000 max=4147 max_rank=2 min=4010 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1031, 1044, 1000, 986, 7, 0, 0, 0, 999, 1026, 997, 1042, 0, 0, 0, 0, 1075, 1023, 1015, 1040, 10, 4, 0, 0, 1036, 1072, 1056, 1034, 0, 0, 0, 0, 1040, 992, 981, 1001, 14, 11, 10, 2, 977, 1038, 1017, 1025, 0, 0, 0, 0, 1055, 1046, 1009, 1022, 0, 0, 0, 0, 983, 1053, 1012, 983, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4064, 4167, 4198, 4051, 4057, 4132, 4031] avg=4096.0000 max=4198 max_rank=3 min=4031 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1032, 1027, 1067, 1027, 0, 0, 0, 0, 979, 1060, 1020, 991, 0, 0, 0, 0, 991, 1019, 1040, 1021, 4, 1, 0, 0, 1031, 1027, 1044, 1100, 0, 0, 0, 0, 1000, 1064, 1009, 1041, 10, 9, 6, 0, 1011, 996, 1024, 980, 0, 0, 0, 0, 1058, 1016, 1056, 1010, 0, 0, 0, 0, 1038, 959, 1018, 982, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4050, 4076, 4202, 4139, 4011, 4140, 3997] avg=4096.0000 max=4202 max_rank=3 min=3997 min_rank=7 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[997, 1011, 1031, 1017, 1017, 1025, 1028, 980, 1002, 982, 979, 968, 1003, 1024, 1057, 1077, 1069, 1058, 1000, 1058, 1070, 1049, 1029, 1041, 1008, 1025, 1024, 1031, 1074, 993, 1016, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1033, 1003, 1059, 1033, 1038, 1068, 1026, 967, 996, 993, 1050, 1059, 1013, 1061, 1090, 1048, 965, 974, 1022, 999, 1015, 1087, 1058, 1032, 1018, 1011, 1032, 1029, 965, 1018, 974, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1062, 1021, 1022, 991, 1050, 1014, 986, 1011, 984, 996, 1039, 1061, 1034, 1012, 1033, 1037, 1027, 1052, 1013, 1038, 1019, 1103, 1018, 1016, 987, 1045, 1048, 967, 974, 1023, 1035, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[990, 1023, 994, 973, 1012, 1032, 1083, 1025, 1080, 1047, 1039, 1012, 990, 961, 1059, 1038, 1017, 1006, 1051, 1052, 994, 1013, 977, 1080, 1057, 1070, 978, 1044, 1003, 1030, 1039, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[969, 962, 1062, 1000, 1066, 991, 1028, 1007, 993, 1019, 1023, 1048, 1028, 1081, 1045, 1014, 1013, 991, 1046, 1036, 1048, 1004, 1057, 1030, 1043, 1018, 1046, 1042, 1007, 1042, 994, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[993, 1002, 1033, 1044, 984, 1022, 1024, 989, 1078, 1019, 1038, 1073, 1035, 1017, 972, 993, 1014, 1063, 1019, 1018, 1022, 1018, 993, 1021, 1008, 1054, 1044, 1039, 1081, 1016, 993, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1003, 1041, 985, 1065, 998, 1031, 984, 1019, 1083, 1073, 1005, 1053, 992, 1020, 1027, 1067, 976, 1016, 1004, 1026, 1057, 979, 996, 1066, 1022, 1067, 1000, 1029, 1063, 1023, 970, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1008, 1051, 976, 1059, 1014, 966, 1042, 1027, 1016, 1056, 1050, 1055, 990, 1023, 1053, 994, 982, 981, 1030, 989, 1012, 1029, 1039, 1059, 1061, 1027, 1028, 1024, 989, 1021, 1062, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1033, 1003, 1059, 1033, 31, 0, 0, 0, 1038, 1068, 1026, 967, 25, 0, 0, 0, 996, 993, 1050, 1034, 0, 0, 0, 0, 1013, 1061, 1075, 1048, 0, 0, 0, 0, 965, 974, 1022, 999, 18, 1, 0, 0, 1015, 1087, 1058, 1001, 0, 0, 0, 0, 1018, 993, 1032, 1029, 0, 0, 0, 0, 965, 1018, 974, 1032, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4124, 4073, 4197, 3979, 4161, 4072, 4003] avg=4096.0000 max=4197 max_rank=3 min=3979 min_rank=4 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16215 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[969, 962, 1062, 1000, 31, 0, 0, 0, 1066, 991, 1028, 1007, 25, 0, 0, 0, 993, 1019, 1023, 1023, 0, 0, 0, 0, 1028, 1081, 1031, 1014, 0, 0, 0, 0, 1013, 991, 1046, 1036, 18, 1, 0, 0, 1048, 1004, 1057, 999, 0, 0, 0, 0, 1043, 1000, 1046, 1042, 0, 0, 0, 0, 1007, 1042, 994, 1015, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4117, 4058, 4154, 4105, 4108, 4131, 4071] avg=4096.0000 max=4154 max_rank=3 min=4024 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1003, 1041, 985, 1065, 32, 0, 0, 0, 998, 1031, 984, 1019, 25, 1, 0, 0, 1083, 1073, 1005, 1028, 0, 0, 0, 0, 992, 1020, 1013, 1067, 0, 0, 0, 0, 976, 1016, 1004, 1026, 19, 1, 0, 0, 1057, 979, 996, 1033, 0, 0, 0, 0, 1022, 1048, 1000, 1029, 0, 0, 0, 0, 1063, 1023, 970, 1028, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4058, 4189, 4092, 4042, 4065, 4099, 4097] avg=4096.0000 max=4189 max_rank=2 min=4042 min_rank=4 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[997, 1011, 1031, 1017, 31, 0, 0, 0, 1017, 1025, 1028, 980, 23, 0, 0, 0, 1002, 982, 979, 945, 0, 0, 0, 0, 1003, 1024, 1043, 1077, 0, 0, 0, 0, 1069, 1058, 1000, 1058, 18, 1, 0, 0, 1070, 1049, 1029, 1010, 0, 0, 0, 0, 1008, 1007, 1024, 1031, 0, 0, 0, 0, 1074, 993, 1016, 1025, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4073, 3908, 4147, 4204, 4158, 4070, 4121] avg=4096.0000 max=4204 max_rank=4 min=3908 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[993, 1002, 1033, 1044, 31, 0, 0, 0, 984, 1022, 1024, 989, 25, 0, 0, 0, 1078, 1019, 1038, 1048, 0, 0, 0, 0, 1035, 1017, 959, 993, 0, 0, 0, 0, 1014, 1063, 1019, 1018, 19, 1, 0, 0, 1022, 1018, 993, 990, 0, 0, 0, 0, 1008, 1035, 1044, 1039, 0, 0, 0, 0, 1081, 1016, 993, 1049, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4044, 4183, 4004, 4134, 4023, 4126, 4151] avg=4096.0000 max=4183 max_rank=2 min=4004 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[990, 1023, 994, 973, 33, 0, 0, 0, 1012, 1032, 1083, 1025, 24, 1, 0, 0, 1080, 1047, 1039, 988, 0, 0, 0, 0, 990, 961, 1045, 1038, 0, 0, 0, 0, 1017, 1006, 1051, 1052, 19, 1, 0, 0, 994, 1013, 977, 1046, 0, 0, 0, 0, 1057, 1051, 978, 1044, 0, 0, 0, 0, 1003, 1030, 1039, 999, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4177, 4154, 4034, 4146, 4030, 4130, 4084] avg=4096.0000 max=4177 max_rank=1 min=4013 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1062, 1021, 1022, 991, 34, 0, 0, 0, 1050, 1014, 986, 1011, 29, 5, 5, 0, 984, 996, 1039, 1032, 0, 0, 0, 0, 1034, 1012, 1011, 1037, 0, 0, 0, 0, 1027, 1052, 1013, 1038, 24, 3, 0, 0, 1019, 1103, 1018, 977, 0, 0, 0, 0, 987, 1016, 1048, 967, 0, 0, 0, 0, 974, 1023, 1035, 1050, 19, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4100, 4051, 4094, 4157, 4117, 4018, 4101] avg=4096.0000 max=4157 max_rank=4 min=4018 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1008, 1051, 976, 1059, 32, 0, 0, 0, 1014, 966, 1042, 1027, 25, 1, 0, 0, 1016, 1056, 1050, 1030, 0, 0, 0, 0, 990, 1023, 1039, 994, 0, 0, 0, 0, 982, 981, 1030, 989, 18, 1, 0, 0, 1012, 1029, 1039, 1026, 0, 0, 0, 0, 1061, 1009, 1028, 1024, 0, 0, 0, 0, 989, 1021, 1062, 1055, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4075, 4152, 4046, 4001, 4106, 4122, 4140] avg=4096.0000 max=4152 max_rank=2 min=4001 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8064, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1046, 1035, 1021, 1024, 1010, 1003, 1060, 1066, 989, 1021, 1046, 1016, 1006, 960, 1023, 1030, 1028, 1040, 1035, 1021, 1020, 1038, 1027, 1050, 1023, 985, 1003, 1007, 1062, 1020, 1066, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1024, 1050, 1074, 1000, 1021, 1027, 1029, 1089, 1066, 1014, 983, 1001, 1018, 1029, 1005, 1041, 1041, 980, 1031, 1054, 1045, 956, 1046, 1042, 1005, 1055, 967, 1006, 1029, 1006, 1002, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1008, 1053, 1031, 1032, 1041, 1019, 1022, 1034, 984, 999, 992, 1040, 1079, 1026, 1034, 1009, 1032, 1049, 1045, 1004, 972, 976, 1023, 998, 1062, 1042, 1042, 1014, 980, 1043, 1068, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[995, 984, 1005, 1020, 1011, 1011, 965, 1056, 998, 1067, 1020, 1053, 1048, 1016, 1031, 1038, 1046, 1020, 1017, 1004, 1011, 1007, 1021, 1042, 1033, 995, 1068, 1021, 1046, 1037, 1028, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1006, 1024, 1050, 1008, 992, 1002, 1007, 983, 980, 1018, 1034, 1061, 1033, 1032, 1019, 1036, 1048, 1023, 1049, 1004, 1053, 982, 1042, 1055, 1036, 1025, 1006, 1023, 989, 1065, 1051, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1028, 1037, 1022, 1043, 969, 1060, 1017, 1043, 1027, 1038, 1038, 1028, 1019, 1057, 1022, 1034, 1013, 1002, 1010, 1005, 1015, 1051, 1047, 1028, 981, 1009, 1028, 1040, 999, 1032, 1010, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[962, 1045, 988, 1022, 1031, 1018, 1051, 1044, 1024, 1010, 1045, 1053, 1044, 1077, 997, 1061, 1043, 1023, 1004, 1011, 1026, 1041, 1024, 984, 1017, 953, 994, 1001, 1048, 1003, 1083, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1029, 1023, 1005, 991, 1031, 1025, 968, 995, 1006, 1022, 1040, 1003, 1067, 1088, 1034, 1083, 985, 1025, 1018, 1037, 1042, 1025, 1037, 1062, 1005, 1084, 1005, 1015, 1027, 1014, 968, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1046, 1035, 1021, 1024, 10, 0, 0, 0, 1010, 1003, 1060, 1066, 5, 3, 0, 0, 989, 1021, 1046, 1016, 6, 0, 0, 0, 1006, 960, 1023, 991, 0, 0, 0, 0, 1028, 1040, 1035, 1021, 2, 0, 0, 0, 1020, 1038, 1025, 1050, 0, 0, 0, 0, 1023, 985, 1003, 1007, 26, 0, 0, 0, 1062, 1020, 1055, 987, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4147, 4078, 3980, 4126, 4133, 4044, 4124] avg=4096.0000 max=4147 max_rank=1 min=3980 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1028, 1037, 1022, 1043, 10, 0, 0, 0, 969, 1060, 1017, 1043, 5, 3, 0, 0, 1027, 1038, 1038, 1028, 6, 0, 0, 0, 1019, 1057, 1022, 994, 0, 0, 0, 0, 1013, 1002, 1010, 1005, 2, 0, 0, 0, 1015, 1051, 1045, 1028, 0, 0, 0, 0, 981, 1009, 1028, 1040, 27, 0, 0, 0, 999, 1032, 999, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4097, 4137, 4092, 4032, 4139, 4085, 4046] avg=4096.0000 max=4140 max_rank=0 min=4032 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[995, 984, 1005, 1020, 10, 0, 0, 0, 1011, 1011, 965, 1056, 5, 3, 0, 0, 998, 1067, 1020, 1053, 6, 0, 0, 0, 1048, 1016, 1031, 998, 0, 0, 0, 0, 1046, 1020, 1017, 1004, 2, 0, 0, 0, 1011, 1007, 1019, 1042, 0, 0, 0, 0, 1033, 995, 1068, 1021, 27, 0, 0, 0, 1046, 1037, 1017, 1054, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4014, 4051, 4144, 4093, 4089, 4079, 4144, 4154] avg=4096.0000 max=4154 max_rank=7 min=4014 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1029, 1023, 1005, 991, 10, 0, 0, 0, 1031, 1025, 968, 995, 4, 3, 0, 0, 1006, 1022, 1040, 1003, 6, 0, 0, 0, 1067, 1088, 1034, 1042, 0, 0, 0, 0, 985, 1025, 1018, 1037, 2, 0, 0, 0, 1042, 1025, 1035, 1062, 0, 0, 0, 0, 1005, 1084, 1005, 1015, 28, 0, 0, 0, 1027, 1014, 958, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4026, 4077, 4231, 4067, 4164, 4137, 4008] avg=4096.0000 max=4231 max_rank=3 min=4008 min_rank=7 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 1050, 1074, 1000, 13, 0, 0, 0, 1021, 1027, 1029, 1089, 7, 6, 0, 0, 1066, 1014, 983, 1001, 10, 0, 0, 0, 1018, 1029, 1005, 992, 0, 0, 0, 0, 1041, 980, 1031, 1054, 6, 1, 0, 0, 1045, 956, 1040, 1042, 0, 0, 0, 0, 1005, 1055, 967, 1006, 30, 0, 0, 0, 1029, 1006, 984, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4179, 4074, 4044, 4113, 4083, 4063, 4051] avg=4096.0000 max=4179 max_rank=1 min=4044 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[962, 1045, 988, 1022, 10, 0, 0, 0, 1031, 1018, 1051, 1044, 5, 3, 0, 0, 1024, 1010, 1045, 1053, 6, 0, 0, 0, 1044, 1077, 997, 1021, 0, 0, 0, 0, 1043, 1023, 1004, 1011, 2, 0, 0, 0, 1026, 1041, 1022, 984, 0, 0, 0, 0, 1017, 953, 994, 1001, 27, 0, 0, 0, 1048, 1003, 1072, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4152, 4138, 4139, 4083, 4073, 3992, 4164] avg=4096.0000 max=4164 max_rank=7 min=3992 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1008, 1053, 1031, 1032, 10, 0, 0, 0, 1041, 1019, 1022, 1034, 5, 3, 0, 0, 984, 999, 992, 1040, 6, 0, 0, 0, 1079, 1026, 1034, 970, 0, 0, 0, 0, 1032, 1049, 1045, 1004, 2, 0, 0, 0, 972, 976, 1021, 998, 0, 0, 0, 0, 1062, 1042, 1042, 1014, 26, 0, 0, 0, 980, 1043, 1057, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4124, 4021, 4109, 4132, 3967, 4186, 4095] avg=4096.0000 max=4186 max_rank=6 min=3967 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1006, 1024, 1050, 1008, 10, 0, 0, 0, 992, 1002, 1007, 983, 5, 3, 0, 0, 980, 1018, 1034, 1061, 6, 0, 0, 0, 1033, 1032, 1019, 996, 0, 0, 0, 0, 1048, 1023, 1049, 1004, 2, 0, 0, 0, 1053, 982, 1040, 1055, 0, 0, 0, 0, 1036, 1025, 1006, 1023, 27, 0, 0, 0, 989, 1065, 1040, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 3992, 4099, 4080, 4126, 4130, 4117, 4126] avg=4096.0000 max=4130 max_rank=5 min=3992 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8320, 8192, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1012, 981, 1009, 1025, 1093, 1055, 999, 1046, 1019, 1071, 1042, 1043, 1054, 1028, 956, 992, 1023, 1031, 1014, 1057, 978, 1042, 1043, 1008, 1040, 1010, 1003, 1039, 977, 1010, 1011, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1036, 1046, 986, 998, 949, 1042, 982, 1026, 1037, 1048, 1068, 1025, 1020, 1017, 1019, 981, 994, 1052, 1004, 1019, 1015, 1033, 1045, 1034, 1074, 1028, 980, 1053, 1000, 1033, 1058, 1066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1031, 1028, 975, 1007, 1046, 947, 1035, 1062, 1052, 1042, 1091, 991, 1020, 985, 1016, 997, 989, 993, 1023, 1042, 1050, 1037, 1052, 1042, 1011, 1061, 997, 1034, 1042, 962, 1038, 1070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1026, 982, 970, 1039, 1037, 1026, 1010, 1006, 1062, 1015, 1081, 1022, 1056, 1037, 991, 1003, 1024, 1014, 1039, 1031, 1048, 994, 1017, 1040, 1020, 1027, 1067, 1020, 1018, 1057, 973, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1007, 1029, 1019, 996, 1052, 1009, 993, 1033, 1023, 1062, 1044, 974, 1018, 1050, 1012, 1041, 1022, 1001, 1088, 984, 1003, 1074, 1039, 988, 997, 1062, 1004, 1004, 1057, 1020, 1027, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1050, 1022, 1004, 1006, 1015, 959, 1077, 1055, 1005, 1032, 1014, 1057, 1008, 1013, 1040, 1026, 1011, 1031, 1022, 1039, 1027, 985, 1003, 1086, 1082, 1012, 1010, 1002, 1008, 1031, 1021, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1025, 990, 1000, 1037, 1074, 1040, 1030, 1014, 1009, 1025, 995, 1029, 1011, 966, 1040, 1047, 1029, 1025, 1017, 1043, 1074, 1059, 1019, 1031, 1065, 1015, 1013, 1029, 997, 982, 1024, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1065, 1026, 1017, 1004, 1022, 1014, 1005, 1016, 1042, 1001, 1052, 1035, 1016, 1003, 1039, 1014, 1033, 1032, 1039, 1023, 1047, 1021, 1009, 975, 967, 1059, 1077, 1053, 1018, 1008, 1014, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1012, 981, 1009, 1025, 39, 0, 0, 0, 1093, 1055, 999, 1046, 0, 0, 0, 0, 1019, 1071, 1001, 1043, 0, 0, 0, 0, 1054, 1028, 956, 992, 18, 10, 2, 0, 1023, 1031, 1012, 1057, 0, 0, 0, 0, 978, 1024, 1043, 1008, 0, 0, 0, 0, 1040, 993, 1003, 1039, 0, 0, 0, 0, 977, 1010, 1011, 1057, 7, 2, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4193, 4134, 4060, 4123, 4053, 4075, 4064] avg=4096.0000 max=4193 max_rank=1 min=4053 min_rank=5 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1050, 1022, 1004, 1006, 38, 0, 0, 0, 1015, 959, 1077, 1055, 0, 0, 0, 0, 1005, 1032, 974, 1057, 0, 0, 0, 0, 1008, 1013, 1040, 1026, 17, 10, 2, 0, 1011, 1031, 1020, 1039, 0, 0, 0, 0, 1027, 968, 1003, 1086, 0, 0, 0, 0, 1082, 995, 1010, 1002, 0, 0, 0, 0, 1008, 1031, 1021, 1015, 7, 2, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4106, 4068, 4116, 4101, 4084, 4089, 4084] avg=4096.0000 max=4120 max_rank=0 min=4068 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 982, 970, 1039, 41, 0, 0, 0, 1037, 1026, 1010, 1006, 0, 0, 0, 0, 1062, 1015, 1038, 1022, 0, 0, 0, 0, 1056, 1037, 991, 1003, 18, 10, 2, 0, 1024, 1014, 1037, 1031, 0, 0, 0, 0, 1048, 976, 1017, 1040, 0, 0, 0, 0, 1020, 1009, 1067, 1020, 0, 0, 0, 0, 1018, 1057, 973, 1016, 8, 2, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4079, 4137, 4117, 4106, 4081, 4116, 4074] avg=4096.0000 max=4137 max_rank=2 min=4058 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 990, 1000, 1037, 42, 0, 0, 0, 1073, 1040, 1030, 1014, 0, 0, 0, 0, 1009, 1025, 947, 1029, 0, 0, 0, 0, 1011, 966, 1040, 1047, 24, 12, 6, 0, 1029, 1025, 1011, 1043, 0, 0, 0, 0, 1074, 1035, 1019, 1031, 0, 0, 0, 0, 1065, 992, 1013, 1029, 0, 0, 0, 0, 997, 982, 1024, 1014, 11, 6, 1, 0] rerouted_tokens_per_ep_rank=[4094, 4157, 4010, 4106, 4108, 4159, 4099, 4035] avg=4096.0000 max=4159 max_rank=5 min=4010 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1036, 1046, 986, 998, 40, 0, 0, 0, 949, 1042, 982, 1026, 0, 0, 0, 0, 1037, 1048, 1026, 1025, 0, 0, 0, 0, 1020, 1017, 1019, 981, 18, 10, 2, 0, 994, 1052, 1002, 1019, 0, 0, 0, 0, 1015, 1015, 1045, 1034, 0, 0, 0, 0, 1074, 1010, 980, 1053, 0, 0, 0, 0, 1000, 1033, 1058, 1066, 8, 2, 0, 0] rerouted_tokens_per_ep_rank=[4106, 3999, 4136, 4067, 4067, 4109, 4117, 4167] avg=4096.0000 max=4167 max_rank=7 min=3999 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1065, 1026, 1017, 1004, 40, 0, 0, 0, 1022, 1014, 1005, 1016, 0, 0, 0, 0, 1042, 1001, 1010, 1035, 0, 0, 0, 0, 1016, 1003, 1039, 1014, 18, 10, 2, 0, 1033, 1032, 1037, 1023, 0, 0, 0, 0, 1047, 1003, 1009, 975, 0, 0, 0, 0, 967, 1041, 1077, 1053, 0, 0, 0, 0, 1018, 1008, 1014, 1022, 8, 2, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4057, 4088, 4102, 4125, 4034, 4138, 4072] avg=4096.0000 max=4152 max_rank=0 min=4034 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1031, 1028, 975, 1007, 41, 0, 0, 0, 1046, 947, 1035, 1062, 0, 0, 0, 0, 1052, 1042, 1048, 991, 0, 0, 0, 0, 1020, 985, 1016, 997, 18, 10, 2, 0, 989, 993, 1021, 1042, 0, 0, 0, 0, 1050, 1019, 1052, 1042, 0, 0, 0, 0, 1011, 1043, 997, 1034, 0, 0, 0, 0, 1042, 962, 1038, 1070, 8, 2, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4090, 4133, 4048, 4045, 4163, 4085, 4122] avg=4096.0000 max=4163 max_rank=5 min=4045 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1007, 1029, 1019, 996, 39, 0, 0, 0, 1052, 1009, 993, 1033, 0, 0, 0, 0, 1023, 1062, 1003, 974, 0, 0, 0, 0, 1018, 1050, 1012, 1041, 19, 10, 2, 0, 1022, 1001, 1086, 984, 0, 0, 0, 0, 1003, 1055, 1039, 988, 0, 0, 0, 0, 997, 1044, 1004, 1004, 0, 0, 0, 0, 1057, 1020, 1027, 1036, 8, 2, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4087, 4062, 4152, 4093, 4085, 4049, 4150] avg=4096.0000 max=4152 max_rank=3 min=4049 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8320, 8064, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1023, 984, 1031, 1039, 1020, 1047, 1006, 1004, 1015, 993, 1038, 1041, 1017, 1006, 1000, 1015, 1040, 1047, 994, 1041, 1027, 1035, 1049, 1007, 1045, 1006, 1036, 1010, 1036, 1018, 1000, 1098] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1066, 1040, 1043, 1046, 1009, 1028, 1042, 1043, 1016, 1051, 1036, 1042, 989, 1027, 1000, 1033, 959, 1017, 1029, 1015, 1015, 1013, 1028, 993, 998, 1051, 1023, 990, 1049, 1016, 985, 1076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1047, 984, 1014, 979, 1036, 1074, 1025, 1004, 1021, 979, 1034, 1017, 1014, 1014, 1027, 1009, 1010, 1007, 1041, 1012, 1007, 1065, 1044, 1021, 1028, 1041, 1005, 1077, 1042, 1027, 1013, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1030, 1055, 1026, 1018, 1023, 1025, 1005, 994, 1024, 1028, 1052, 1002, 1067, 1074, 968, 983, 986, 1043, 968, 1038, 1026, 1044, 1029, 1006, 1011, 1011, 1080, 1071, 981, 1019, 1061, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1050, 1029, 1023, 974, 1052, 1038, 1041, 1034, 1035, 1028, 992, 1004, 987, 997, 1047, 1042, 956, 1000, 1019, 1048, 1012, 1073, 1038, 1046, 1024, 1026, 1009, 996, 1046, 1041, 1048, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[994, 1074, 1022, 1022, 1055, 985, 1055, 1031, 1035, 992, 1035, 1022, 1018, 1022, 1043, 972, 1029, 999, 988, 1022, 1035, 1022, 1044, 1025, 1027, 1022, 1047, 1107, 1047, 980, 977, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1033, 1057, 1074, 1050, 1045, 1009, 1020, 974, 1036, 1037, 1044, 1008, 999, 1014, 991, 1010, 988, 1025, 1014, 1038, 982, 1050, 1006, 1015, 999, 1027, 1034, 1052, 1002, 1023, 1065, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1045, 993, 1055, 998, 1030, 1050, 1030, 1017, 1066, 1056, 991, 1002, 998, 961, 1012, 1072, 1033, 1019, 1016, 1050, 1024, 1027, 1035, 1026, 992, 1019, 1051, 1020, 1027, 1006, 1002, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1005, 984, 1031, 1039, 0, 0, 0, 0, 1010, 1047, 1006, 1004, 0, 0, 0, 0, 1015, 993, 1038, 1041, 0, 0, 0, 0, 1017, 1006, 1000, 1015, 20, 18, 3, 0, 1040, 1047, 994, 1041, 12, 11, 10, 0, 1027, 1023, 1049, 1007, 0, 0, 0, 0, 1045, 1006, 1036, 990, 0, 0, 0, 0, 1036, 1018, 1000, 1084, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4067, 4087, 4079, 4155, 4106, 4077, 4138] avg=4096.0000 max=4155 max_rank=4 min=4059 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1027, 993, 1055, 998, 0, 0, 0, 0, 1020, 1050, 1030, 1017, 0, 0, 0, 0, 1066, 1056, 991, 1002, 0, 0, 0, 0, 998, 961, 1012, 1072, 20, 18, 2, 0, 1033, 1019, 1016, 1050, 12, 11, 10, 0, 1024, 1015, 1035, 1026, 0, 0, 0, 0, 992, 1019, 1051, 1000, 0, 0, 0, 0, 1027, 1006, 1002, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4117, 4115, 4083, 4151, 4100, 4062, 4067] avg=4096.0000 max=4151 max_rank=4 min=4062 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1012, 1055, 1026, 1018, 0, 0, 0, 0, 1013, 1025, 1005, 994, 0, 0, 0, 0, 1024, 1028, 1052, 1002, 0, 0, 0, 0, 1067, 1074, 968, 983, 21, 18, 2, 0, 986, 1043, 968, 1038, 12, 10, 10, 0, 1026, 1032, 1029, 1006, 0, 0, 0, 0, 1011, 1011, 1080, 1050, 0, 0, 0, 0, 981, 1019, 1061, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4037, 4106, 4133, 4067, 4093, 4152, 4069] avg=4096.0000 max=4152 max_rank=6 min=4037 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1031, 1029, 1023, 974, 0, 0, 0, 0, 1042, 1038, 1041, 1034, 0, 0, 0, 0, 1035, 1028, 992, 1004, 0, 0, 0, 0, 987, 997, 1047, 1042, 19, 19, 2, 0, 956, 1000, 1019, 1048, 13, 10, 10, 0, 1012, 1060, 1038, 1046, 0, 0, 0, 0, 1024, 1026, 1009, 977, 0, 0, 0, 0, 1046, 1041, 1048, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4155, 4059, 4113, 4056, 4156, 4036, 4136] avg=4096.0000 max=4156 max_rank=5 min=4036 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[977, 1074, 1022, 1022, 0, 0, 0, 0, 1045, 985, 1055, 1031, 0, 0, 0, 0, 1035, 992, 1035, 1022, 0, 0, 0, 0, 1018, 1022, 1043, 972, 22, 17, 2, 0, 1029, 999, 988, 1022, 12, 10, 10, 0, 1035, 1010, 1044, 1025, 0, 0, 0, 0, 1027, 1022, 1047, 1085, 0, 0, 0, 0, 1047, 980, 977, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4116, 4084, 4096, 4070, 4114, 4181, 4012] avg=4096.0000 max=4181 max_rank=6 min=4012 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1029, 984, 1014, 979, 0, 0, 0, 0, 1026, 1074, 1025, 1004, 0, 0, 0, 0, 1021, 979, 1034, 1017, 0, 0, 0, 0, 1014, 1014, 1027, 1009, 21, 18, 2, 0, 1010, 1007, 1041, 1012, 12, 11, 10, 0, 1007, 1053, 1044, 1021, 0, 0, 0, 0, 1028, 1041, 1005, 1056, 0, 0, 0, 0, 1042, 1027, 1013, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4129, 4051, 4105, 4103, 4125, 4130, 4119] avg=4096.0000 max=4130 max_rank=6 min=4006 min_rank=0 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16477 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1010, 1057, 1074, 1050, 0, 0, 0, 0, 1032, 1009, 1020, 974, 0, 0, 0, 0, 1032, 1037, 1044, 1008, 0, 0, 0, 0, 999, 1014, 991, 1010, 25, 23, 8, 0, 988, 1025, 1014, 1038, 16, 15, 13, 4, 982, 1034, 1006, 1015, 0, 0, 0, 0, 999, 1027, 1034, 1027, 0, 0, 0, 0, 1002, 1023, 1065, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 4035, 4121, 4070, 4113, 4037, 4087, 4114] avg=4096.0000 max=4191 max_rank=0 min=4035 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1047, 1040, 1043, 1046, 0, 0, 0, 0, 999, 1028, 1042, 1043, 0, 0, 0, 0, 1016, 1051, 1036, 1042, 0, 0, 0, 0, 989, 1027, 1000, 1033, 19, 19, 2, 0, 959, 1017, 1029, 1015, 12, 11, 10, 0, 1015, 1001, 1028, 993, 0, 0, 0, 0, 998, 1051, 1023, 971, 0, 0, 0, 0, 1049, 1016, 985, 1063, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4112, 4145, 4089, 4053, 4037, 4043, 4113] avg=4096.0000 max=4176 max_rank=0 min=4037 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8320, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1010, 1055, 1004, 1042, 1008, 1071, 1022, 1014, 1038, 1060, 1033, 1018, 1001, 1039, 1030, 1051, 1027, 996, 968, 1040, 1023, 1039, 1051, 980, 991, 1053, 1053, 996, 963, 1069, 1034, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[998, 1038, 1035, 1040, 1010, 1018, 992, 961, 1071, 988, 1024, 1040, 1025, 1046, 1001, 1055, 1031, 1008, 1080, 1045, 1000, 1014, 1087, 1007, 1011, 1003, 1065, 998, 1019, 1025, 1022, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1046, 1007, 1050, 1030, 1090, 993, 1072, 1063, 984, 1000, 1010, 1008, 1019, 1028, 1039, 1017, 1032, 982, 1049, 997, 1038, 1019, 1008, 972, 1042, 1017, 998, 1037, 1020, 1078, 990, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1002, 1032, 1021, 1046, 985, 1074, 983, 1018, 1008, 1024, 1030, 1021, 1035, 1053, 972, 999, 1043, 988, 1037, 1043, 1031, 1053, 1066, 1010, 1033, 1046, 987, 1010, 1052, 1078, 991, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1012, 1011, 1052, 997, 1030, 1039, 1051, 1025, 1035, 983, 1035, 994, 1043, 1061, 1033, 1031, 1007, 1092, 986, 1074, 1024, 1060, 1000, 980, 1082, 947, 1020, 1006, 983, 1046, 1013, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1026, 1027, 989, 1053, 1042, 1032, 1017, 1038, 1025, 1078, 1007, 1030, 1006, 1009, 1029, 985, 979, 1030, 1018, 986, 1033, 1020, 1031, 1029, 995, 1054, 1073, 1028, 1054, 1058, 987, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1037, 987, 1011, 1006, 982, 1019, 983, 1007, 1037, 1025, 1031, 1010, 1055, 1018, 1069, 1058, 1068, 1068, 1003, 1058, 1031, 1014, 1038, 1013, 971, 1042, 1062, 1005, 1028, 1062, 990, 980] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1023, 997, 1013, 1065, 1028, 1041, 1024, 1007, 1075, 1054, 1030, 1044, 1012, 1030, 994, 1021, 1040, 1031, 1031, 1082, 1056, 1015, 1030, 1000, 982, 972, 1024, 995, 1016, 1070, 1003, 963] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[998, 1038, 1035, 1040, 0, 0, 0, 0, 1010, 1018, 992, 961, 3, 0, 0, 0, 1062, 988, 1024, 1040, 0, 0, 0, 0, 1025, 1035, 1001, 1055, 0, 0, 0, 0, 1031, 1008, 1080, 1027, 0, 0, 0, 0, 1000, 1014, 1087, 1007, 0, 0, 0, 0, 1011, 1003, 1065, 998, 18, 2, 0, 0, 1019, 1025, 1022, 1011, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4111, 3984, 4114, 4116, 4146, 4108, 4097, 4092] avg=4096.0000 max=4146 max_rank=4 min=3984 min_rank=1 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1010, 1055, 1004, 1042, 0, 0, 0, 0, 1008, 1071, 1022, 1014, 3, 0, 0, 0, 1029, 1060, 1033, 1018, 0, 0, 0, 0, 1001, 1028, 1030, 1051, 0, 0, 0, 0, 1027, 996, 968, 1022, 0, 0, 0, 0, 1023, 1039, 1051, 980, 0, 0, 0, 0, 991, 1053, 1053, 996, 18, 2, 0, 0, 963, 1069, 1034, 989, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4118, 4140, 4110, 4013, 4093, 4113, 4070] avg=4096.0000 max=4140 max_rank=2 min=4013 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1037, 987, 1011, 1006, 4, 2, 0, 0, 982, 1019, 983, 1007, 8, 0, 0, 0, 1018, 1025, 1031, 1010, 0, 0, 0, 0, 1055, 999, 1069, 1058, 0, 0, 0, 0, 1068, 1068, 1003, 1034, 0, 0, 0, 0, 1031, 1014, 1034, 1013, 0, 0, 0, 0, 971, 1042, 1062, 1005, 24, 5, 0, 0, 1028, 1062, 990, 980, 14, 9, 0, 0] rerouted_tokens_per_ep_rank=[4047, 3999, 4084, 4181, 4173, 4092, 4109, 4083] avg=4096.0000 max=4181 max_rank=3 min=3999 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1002, 1032, 1021, 1046, 0, 0, 0, 0, 985, 1074, 983, 1018, 3, 0, 0, 0, 999, 1024, 1030, 1021, 0, 0, 0, 0, 1035, 1042, 972, 999, 0, 0, 0, 0, 1043, 988, 1037, 1025, 0, 0, 0, 0, 1031, 1053, 1066, 1010, 0, 0, 0, 0, 1033, 1046, 987, 1010, 18, 2, 0, 0, 1052, 1078, 991, 997, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4063, 4074, 4048, 4093, 4160, 4096, 4133] avg=4096.0000 max=4160 max_rank=5 min=4048 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1023, 997, 1013, 1065, 0, 0, 0, 0, 1028, 1041, 1024, 1007, 3, 0, 0, 0, 1066, 1054, 1030, 1044, 0, 0, 0, 0, 1012, 1019, 994, 1021, 0, 0, 0, 0, 1040, 1031, 1031, 1063, 0, 0, 0, 0, 1056, 1015, 1030, 1000, 0, 0, 0, 0, 982, 972, 1024, 995, 19, 2, 0, 0, 1016, 1070, 1003, 963, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4103, 4194, 4046, 4165, 4101, 3994, 4067] avg=4096.0000 max=4194 max_rank=2 min=3994 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1012, 1011, 1052, 997, 0, 0, 0, 0, 1030, 1039, 1051, 1025, 3, 0, 0, 0, 1026, 983, 1035, 994, 0, 0, 0, 0, 1043, 1050, 1033, 1031, 0, 0, 0, 0, 1007, 1092, 986, 1055, 0, 0, 0, 0, 1024, 1060, 1000, 980, 0, 0, 0, 0, 1082, 947, 1020, 1006, 19, 2, 0, 0, 983, 1046, 1013, 1016, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4148, 4038, 4157, 4140, 4064, 4076, 4073] avg=4096.0000 max=4157 max_rank=3 min=4038 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1026, 1027, 989, 1053, 0, 0, 0, 0, 1042, 1032, 1017, 1038, 3, 0, 0, 0, 1016, 1078, 1007, 1030, 0, 0, 0, 0, 1006, 998, 1029, 985, 0, 0, 0, 0, 979, 1030, 1018, 969, 0, 0, 0, 0, 1033, 1020, 1031, 1029, 0, 0, 0, 0, 995, 1054, 1073, 1028, 17, 2, 0, 0, 1054, 1058, 987, 1000, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4132, 4131, 4018, 3996, 4113, 4169, 4114] avg=4096.0000 max=4169 max_rank=6 min=3996 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1046, 1007, 1050, 1030, 0, 0, 0, 0, 1090, 993, 1072, 1063, 3, 0, 0, 0, 975, 1000, 1010, 1008, 0, 0, 0, 0, 1019, 1017, 1039, 1017, 0, 0, 0, 0, 1032, 982, 1049, 979, 0, 0, 0, 0, 1038, 1019, 1008, 972, 0, 0, 0, 0, 1042, 1017, 998, 1037, 18, 2, 0, 0, 1020, 1078, 990, 1033, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4221, 3993, 4092, 4042, 4037, 4114, 4136] avg=4096.0000 max=4221 max_rank=1 min=3993 min_rank=2 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8576, 8064, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[994, 1047, 1034, 1079, 975, 997, 1050, 1015, 1013, 941, 1019, 1027, 1025, 1088, 1036, 1038, 1029, 995, 1069, 1039, 1013, 1048, 1022, 1021, 1025, 1050, 996, 1003, 1016, 1021, 1012, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1027, 1047, 1040, 1024, 1005, 1040, 974, 950, 998, 1033, 1028, 1041, 1021, 1001, 1037, 1114, 1025, 1025, 1030, 1005, 999, 997, 1026, 1056, 982, 1058, 996, 1022, 1059, 997, 1049, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1005, 1003, 1089, 1032, 1050, 1010, 1033, 991, 1015, 1026, 1040, 1004, 1025, 1021, 1004, 1014, 1030, 1013, 995, 1021, 1014, 1039, 1001, 967, 991, 1073, 1059, 1029, 1039, 1056, 1071, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1051, 1003, 1018, 1026, 1074, 970, 1015, 1046, 1035, 1031, 1003, 1036, 1018, 1040, 1004, 1060, 1026, 1032, 983, 967, 1044, 1031, 1055, 990, 1042, 1030, 1060, 994, 1019, 1037, 1002, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[991, 1050, 1039, 1024, 1029, 1025, 1019, 1017, 1033, 992, 1028, 1036, 1012, 996, 1009, 1041, 1047, 1024, 1052, 1055, 1028, 1024, 1013, 1050, 1029, 983, 1004, 992, 993, 1025, 1074, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1034, 1052, 1011, 1085, 1039, 1035, 1045, 1011, 1000, 1027, 991, 985, 1036, 1057, 1017, 1007, 1062, 995, 1005, 1026, 1023, 1029, 1084, 994, 1045, 1008, 1029, 1021, 1021, 979, 1013, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1001, 1017, 1038, 1005, 1043, 1051, 1001, 1085, 1009, 976, 1046, 1007, 1031, 998, 1033, 991, 1047, 1045, 1018, 1070, 999, 1062, 1004, 1014, 1024, 1070, 1000, 1028, 1037, 1016, 1014, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1022, 1022, 994, 1042, 1001, 988, 964, 1026, 1008, 1039, 1022, 1083, 961, 996, 1041, 996, 1018, 1013, 999, 1052, 1082, 1053, 1049, 1051, 1047, 1059, 1015, 1057, 1049, 1019, 982, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1005, 1003, 1089, 1010, 0, 0, 0, 0, 1050, 1010, 1033, 991, 12, 6, 5, 0, 1015, 1026, 1040, 1004, 22, 2, 0, 0, 1025, 1021, 1004, 1014, 0, 0, 0, 0, 1025, 1013, 995, 1021, 0, 0, 0, 0, 1014, 1025, 1001, 967, 0, 0, 0, 0, 991, 1067, 1059, 1029, 0, 0, 0, 0, 1039, 1056, 1071, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4107, 4109, 4064, 4054, 4007, 4146, 4174] avg=4096.0000 max=4174 max_rank=7 min=4007 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1022, 1022, 994, 1020, 0, 0, 0, 0, 1001, 988, 964, 1026, 12, 6, 5, 0, 1008, 1039, 1022, 1083, 22, 2, 0, 0, 961, 996, 1041, 996, 0, 0, 0, 0, 1013, 1013, 999, 1052, 0, 0, 0, 0, 1082, 1039, 1049, 1051, 0, 0, 0, 0, 1047, 1053, 1015, 1057, 0, 0, 0, 0, 1049, 1019, 982, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4002, 4176, 3994, 4077, 4221, 4172, 4068] avg=4096.0000 max=4221 max_rank=5 min=3994 min_rank=3 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16538 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[991, 1050, 1039, 1003, 0, 0, 0, 0, 1029, 1025, 1019, 1017, 11, 6, 5, 0, 1033, 992, 1028, 1036, 21, 2, 0, 0, 1012, 996, 1009, 1041, 0, 0, 0, 0, 1042, 1024, 1052, 1055, 0, 0, 0, 0, 1028, 1011, 1013, 1050, 0, 0, 0, 0, 1029, 977, 1004, 992, 0, 0, 0, 0, 993, 1025, 1074, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4112, 4112, 4058, 4173, 4102, 4002, 4126] avg=4096.0000 max=4173 max_rank=4 min=4002 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16403 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[994, 1047, 1034, 1056, 0, 0, 0, 0, 975, 997, 1050, 1015, 12, 6, 5, 0, 1013, 941, 1019, 1027, 23, 2, 0, 0, 1025, 1088, 1036, 1038, 0, 0, 0, 0, 1024, 995, 1069, 1039, 0, 0, 0, 0, 1013, 1034, 1022, 1021, 0, 0, 0, 0, 1025, 1044, 996, 1003, 0, 0, 0, 0, 1016, 1021, 1012, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4060, 4025, 4187, 4127, 4090, 4068, 4080] avg=4096.0000 max=4187 max_rank=3 min=4025 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16365 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1051, 1003, 1018, 1005, 0, 0, 0, 0, 1074, 970, 1015, 1046, 11, 6, 5, 0, 1035, 1031, 1003, 1036, 21, 2, 0, 0, 1018, 1040, 1004, 1060, 0, 0, 0, 0, 1021, 1032, 983, 967, 0, 0, 0, 0, 1044, 1018, 1055, 990, 0, 0, 0, 0, 1042, 1024, 1060, 994, 0, 0, 0, 0, 1019, 1037, 1002, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4127, 4128, 4122, 4003, 4107, 4120, 4084] avg=4096.0000 max=4128 max_rank=2 min=4003 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16314 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1034, 1052, 1011, 1058, 0, 0, 0, 0, 1039, 1035, 1045, 1011, 15, 11, 9, 1, 1000, 1027, 991, 985, 27, 4, 0, 0, 1036, 1057, 1017, 1007, 0, 0, 0, 0, 1053, 995, 1005, 1026, 0, 0, 0, 0, 1023, 1010, 1084, 994, 0, 0, 0, 0, 1045, 997, 1029, 1021, 0, 0, 0, 0, 1020, 979, 1013, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4166, 4034, 4117, 4079, 4111, 4092, 4014] avg=4096.0000 max=4166 max_rank=1 min=4014 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16296 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1027, 1047, 1040, 1003, 0, 0, 0, 0, 1005, 1040, 974, 950, 11, 6, 5, 0, 998, 1033, 1028, 1041, 21, 2, 0, 0, 1021, 1001, 1037, 1114, 0, 0, 0, 0, 1020, 1025, 1030, 1005, 0, 0, 0, 0, 999, 984, 1026, 1056, 0, 0, 0, 0, 982, 1052, 996, 1022, 0, 0, 0, 0, 1059, 997, 1049, 1062, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 3991, 4123, 4173, 4080, 4065, 4052, 4167] avg=4096.0000 max=4173 max_rank=3 min=3991 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16364 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1001, 1017, 1038, 984, 0, 0, 0, 0, 1043, 1051, 1001, 1085, 12, 6, 5, 0, 1009, 976, 1046, 1007, 21, 2, 0, 0, 1031, 998, 1033, 991, 0, 0, 0, 0, 1042, 1045, 1018, 1070, 0, 0, 0, 0, 999, 1048, 1004, 1014, 0, 0, 0, 0, 1024, 1064, 1000, 1028, 0, 0, 0, 0, 1037, 1016, 1014, 988, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4203, 4061, 4053, 4175, 4065, 4116, 4055] avg=4096.0000 max=4203 max_rank=1 min=4040 min_rank=0 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16411 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1015, 1039, 1027, 987, 1060, 1041, 969, 1048, 988, 1009, 1082, 1020, 1029, 995, 1017, 1037, 1008, 1034, 976, 1042, 1071, 994, 1030, 1008, 1072, 1030, 1012, 1057, 1034, 981, 1054, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1029, 1007, 971, 1029, 980, 1008, 1032, 995, 1032, 1029, 1035, 1039, 1013, 1029, 1046, 1051, 1010, 952, 1037, 1053, 995, 1036, 1002, 1031, 1034, 1091, 1074, 1031, 1052, 1004, 1056, 985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[984, 1049, 1037, 1043, 1049, 1044, 1034, 1048, 981, 994, 1030, 1061, 1005, 1061, 996, 1061, 1004, 1059, 998, 1030, 1012, 995, 1013, 990, 994, 1025, 1013, 1013, 1022, 1043, 1054, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1021, 995, 1017, 1012, 1028, 1026, 1042, 1005, 1032, 1046, 1050, 1011, 1028, 1019, 1035, 1007, 1040, 1007, 1028, 1063, 1016, 968, 1036, 1059, 1041, 1061, 1059, 1006, 972, 995, 1035, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1019, 1045, 1041, 1018, 1069, 1000, 1050, 1032, 989, 993, 1063, 1040, 1008, 1057, 984, 1038, 1061, 1009, 969, 1040, 1006, 1057, 1029, 1010, 1003, 979, 1025, 1049, 1033, 999, 1031, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1043, 1060, 997, 989, 1034, 1018, 1021, 1014, 1057, 1049, 1008, 1023, 1013, 1111, 1040, 987, 1007, 990, 999, 1061, 1050, 980, 1064, 1055, 978, 995, 1028, 1022, 1044, 1019, 1014, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1057, 996, 1038, 1039, 1044, 1036, 1012, 967, 1039, 1013, 1031, 1006, 1038, 1024, 1019, 1050, 1011, 980, 1037, 1001, 985, 1023, 1062, 1100, 1021, 1024, 1033, 1031, 1026, 1019, 1026, 980] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1028, 1033, 973, 1000, 1013, 1007, 1003, 1023, 1024, 1055, 1021, 1070, 1008, 1067, 1020, 1076, 996, 1008, 995, 1034, 1008, 1032, 1009, 1029, 1043, 937, 1029, 1048, 1057, 996, 1095, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[984, 1049, 1037, 1043, 15, 0, 0, 0, 1049, 1044, 1034, 1048, 1, 0, 0, 0, 981, 994, 1012, 1061, 0, 0, 0, 0, 1005, 1036, 996, 1061, 0, 0, 0, 0, 1004, 1059, 998, 1030, 25, 3, 0, 0, 1012, 995, 1013, 990, 1, 0, 0, 0, 994, 1025, 1005, 1013, 0, 0, 0, 0, 1022, 1043, 1054, 1026, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4176, 4048, 4098, 4119, 4011, 4037, 4151] avg=4096.0000 max=4176 max_rank=1 min=4011 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1015, 1039, 1027, 987, 16, 0, 0, 0, 1060, 1041, 969, 1048, 1, 0, 0, 0, 988, 1009, 1063, 1020, 0, 0, 0, 0, 1029, 972, 1017, 1037, 0, 0, 0, 0, 1008, 1034, 976, 1042, 23, 3, 0, 0, 1071, 994, 1030, 1008, 1, 0, 0, 0, 1072, 1030, 1004, 1057, 0, 0, 0, 0, 1034, 981, 1054, 1002, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4119, 4080, 4055, 4086, 4104, 4163, 4077] avg=4096.0000 max=4163 max_rank=6 min=4055 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1019, 1045, 1041, 1018, 15, 0, 0, 0, 1069, 1000, 1050, 1032, 1, 0, 0, 0, 989, 993, 1045, 1040, 0, 0, 0, 0, 1008, 1032, 984, 1038, 0, 0, 0, 0, 1061, 1009, 969, 1040, 25, 3, 0, 0, 1006, 1057, 1029, 1010, 1, 0, 0, 0, 1003, 979, 1017, 1049, 0, 0, 0, 0, 1033, 999, 1031, 1022, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4152, 4067, 4062, 4107, 4103, 4048, 4091] avg=4096.0000 max=4152 max_rank=1 min=4048 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1029, 1007, 971, 1029, 15, 0, 0, 0, 980, 1008, 1032, 995, 2, 0, 0, 0, 1032, 1029, 1017, 1039, 0, 0, 0, 0, 1013, 1005, 1046, 1051, 0, 0, 0, 0, 1010, 952, 1037, 1053, 24, 3, 0, 0, 995, 1036, 1002, 1031, 1, 0, 0, 0, 1034, 1091, 1064, 1031, 0, 0, 0, 0, 1052, 1004, 1056, 985, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4017, 4117, 4115, 4079, 4065, 4220, 4104] avg=4096.0000 max=4220 max_rank=6 min=4017 min_rank=1 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1021, 995, 1017, 1012, 15, 0, 0, 0, 1028, 1026, 1042, 1005, 2, 0, 0, 0, 1032, 1046, 1032, 1011, 0, 0, 0, 0, 1028, 995, 1035, 1007, 0, 0, 0, 0, 1040, 1007, 1028, 1063, 24, 3, 0, 0, 1016, 968, 1036, 1059, 1, 0, 0, 0, 1041, 1061, 1049, 1006, 0, 0, 0, 0, 972, 995, 1035, 1008, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4103, 4121, 4065, 4165, 4080, 4157, 4017] avg=4096.0000 max=4165 max_rank=4 min=4017 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1057, 996, 1038, 1039, 15, 0, 0, 0, 1044, 1036, 1012, 967, 1, 0, 0, 0, 1039, 1013, 1013, 1006, 0, 0, 0, 0, 1038, 1000, 1019, 1050, 0, 0, 0, 0, 1011, 980, 1037, 1001, 24, 3, 0, 0, 985, 1023, 1062, 1100, 1, 0, 0, 0, 1021, 1024, 1025, 1031, 0, 0, 0, 0, 1026, 1019, 1026, 980, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4060, 4071, 4107, 4056, 4171, 4101, 4057] avg=4096.0000 max=4171 max_rank=5 min=4056 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1028, 1033, 973, 1000, 15, 0, 0, 0, 1013, 1007, 1003, 1023, 1, 0, 0, 0, 1024, 1055, 1003, 1070, 0, 0, 0, 0, 1008, 1042, 1020, 1076, 0, 0, 0, 0, 996, 1008, 995, 1034, 25, 3, 0, 0, 1008, 1032, 1009, 1029, 1, 0, 0, 0, 1043, 937, 1021, 1048, 0, 0, 0, 0, 1057, 996, 1095, 1031, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4047, 4152, 4146, 4061, 4079, 4049, 4185] avg=4096.0000 max=4185 max_rank=7 min=4047 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1043, 1060, 997, 989, 18, 6, 0, 0, 1034, 1018, 1021, 1014, 7, 0, 0, 0, 1057, 1049, 983, 1023, 0, 0, 0, 0, 1013, 1080, 1040, 987, 0, 0, 0, 0, 1007, 990, 999, 1061, 31, 7, 0, 0, 1050, 980, 1064, 1055, 6, 0, 0, 0, 978, 995, 998, 1022, 0, 0, 0, 0, 1044, 1019, 1014, 998, 11, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4094, 4112, 4120, 4095, 4155, 3993, 4086] avg=4096.0000 max=4155 max_rank=5 min=3993 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8064, 8448, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8448, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1058, 1035, 1070, 1070, 998, 1046, 1078, 1017, 992, 1019, 985, 1007, 1005, 1066, 1012, 1029, 996, 1010, 1029, 1050, 1012, 1002, 1032, 1051, 1019, 998, 1054, 1003, 1052, 980, 1000, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1041, 1048, 1040, 1033, 1008, 995, 1059, 1065, 996, 1030, 1037, 1015, 1023, 965, 1034, 1006, 1007, 1053, 1032, 1021, 1019, 964, 1025, 1057, 976, 1042, 1001, 1007, 1078, 987, 1065, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1025, 1068, 1053, 982, 995, 1037, 1027, 974, 1022, 1028, 1018, 995, 972, 1017, 1056, 1047, 1013, 1033, 999, 1033, 977, 1019, 1078, 1029, 1027, 1078, 1014, 1032, 1068, 1066, 989, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1051, 1042, 971, 1076, 985, 1033, 1037, 993, 1060, 1040, 1043, 1037, 1030, 1003, 1026, 1036, 1053, 1012, 1016, 1022, 1025, 1057, 999, 1048, 993, 1016, 1012, 1001, 1000, 1042, 1015, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1069, 1022, 1049, 991, 1006, 1010, 1070, 1065, 1004, 1060, 1005, 1023, 1011, 1044, 1050, 1042, 1011, 1021, 997, 1060, 1009, 986, 1012, 1031, 968, 1019, 982, 1014, 1049, 1083, 988, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1041, 1027, 984, 1044, 1034, 1055, 1012, 949, 1053, 1044, 1013, 1026, 1024, 1028, 1033, 1071, 990, 1006, 1033, 991, 1051, 1011, 1028, 1030, 1012, 1003, 1035, 1023, 1042, 1067, 989, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1030, 978, 1022, 1052, 961, 1035, 990, 1042, 1073, 1019, 995, 1036, 1013, 1033, 1052, 1039, 1003, 1015, 1018, 1033, 1032, 1025, 978, 1017, 1038, 1006, 1030, 1033, 1032, 1023, 1070, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1031, 983, 981, 1012, 1020, 1006, 1025, 980, 1042, 1056, 993, 1045, 1080, 1001, 1041, 1020, 1030, 1028, 968, 998, 1043, 1037, 1033, 1023, 1079, 994, 1023, 1032, 1030, 1018, 1004, 1112] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1016, 1048, 1040, 1033, 0, 0, 0, 0, 1008, 995, 1059, 1065, 17, 1, 0, 0, 996, 1026, 1037, 1015, 0, 0, 0, 0, 1023, 965, 1017, 1006, 0, 0, 0, 0, 1007, 1053, 1032, 1021, 23, 0, 0, 0, 1019, 964, 1025, 1057, 3, 0, 0, 0, 976, 1042, 1001, 1007, 25, 0, 0, 0, 1055, 987, 1065, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4145, 4074, 4011, 4136, 4068, 4051, 4146] avg=4096.0000 max=4146 max_rank=7 min=4011 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1000, 1068, 1053, 982, 0, 0, 0, 0, 995, 1037, 1027, 974, 17, 1, 0, 0, 1022, 1024, 1018, 995, 0, 0, 0, 0, 972, 1017, 1039, 1047, 0, 0, 0, 0, 1013, 1033, 999, 1033, 23, 0, 0, 0, 977, 1019, 1078, 1029, 3, 0, 0, 0, 1027, 1078, 1014, 1032, 25, 0, 0, 0, 1045, 1066, 989, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4051, 4059, 4075, 4101, 4106, 4176, 4097] avg=4096.0000 max=4176 max_rank=6 min=4051 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1016, 1027, 984, 1044, 0, 0, 0, 0, 1034, 1055, 1012, 949, 17, 1, 0, 0, 1053, 1040, 1013, 1026, 0, 0, 0, 0, 1024, 1028, 1016, 1071, 0, 0, 0, 0, 990, 1006, 1033, 991, 22, 0, 0, 0, 1051, 1011, 1028, 1030, 3, 0, 0, 0, 1012, 1003, 1035, 1023, 25, 0, 0, 0, 1020, 1067, 989, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4068, 4132, 4139, 4042, 4123, 4098, 4095] avg=4096.0000 max=4139 max_rank=3 min=4042 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1033, 1035, 1070, 1070, 0, 0, 0, 0, 998, 1046, 1078, 1017, 17, 1, 0, 0, 992, 1015, 985, 1007, 0, 0, 0, 0, 1005, 1066, 995, 1029, 0, 0, 0, 0, 996, 1010, 1029, 1050, 22, 0, 0, 0, 1012, 1002, 1032, 1051, 3, 0, 0, 0, 1019, 998, 1054, 1003, 25, 0, 0, 0, 1030, 980, 1000, 993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4208, 4157, 3999, 4095, 4107, 4100, 4099, 4003] avg=4096.0000 max=4208 max_rank=0 min=3999 min_rank=2 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1033, 1022, 1049, 991, 0, 0, 0, 0, 1006, 1010, 1070, 1065, 22, 8, 5, 0, 1004, 1045, 1005, 1023, 0, 0, 0, 0, 1011, 1044, 1028, 1042, 0, 0, 0, 0, 1011, 1021, 997, 1060, 25, 7, 0, 0, 1009, 986, 1012, 1031, 7, 0, 0, 0, 968, 1019, 982, 1014, 29, 0, 0, 0, 1019, 1083, 988, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4186, 4077, 4125, 4121, 4045, 4012, 4107] avg=4096.0000 max=4186 max_rank=1 min=4012 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1006, 983, 981, 1012, 0, 0, 0, 0, 1020, 1006, 1025, 980, 17, 1, 0, 0, 1042, 1052, 993, 1045, 0, 0, 0, 0, 1080, 1001, 1024, 1020, 0, 0, 0, 0, 1030, 1028, 968, 998, 22, 0, 0, 0, 1043, 1037, 1033, 1023, 3, 0, 0, 0, 1079, 994, 1023, 1032, 25, 0, 0, 0, 1008, 1018, 1004, 1112, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3982, 4049, 4132, 4125, 4046, 4139, 4153, 4142] avg=4096.0000 max=4153 max_rank=6 min=3982 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 1042, 971, 1076, 0, 0, 0, 0, 985, 1033, 1037, 993, 17, 1, 0, 0, 1060, 1036, 1043, 1037, 0, 0, 0, 0, 1030, 1003, 1009, 1036, 0, 0, 0, 0, 1053, 1012, 1016, 1022, 21, 0, 0, 0, 1025, 1057, 999, 1048, 3, 0, 0, 0, 993, 1016, 1012, 1001, 25, 0, 0, 0, 979, 1042, 1015, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4066, 4176, 4078, 4124, 4132, 4047, 4030] avg=4096.0000 max=4176 max_rank=2 min=4030 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1005, 978, 1022, 1052, 0, 0, 0, 0, 961, 1035, 990, 1042, 17, 1, 0, 0, 1073, 1015, 995, 1036, 0, 0, 0, 0, 1013, 1033, 1035, 1039, 0, 0, 0, 0, 1003, 1015, 1018, 1033, 22, 0, 0, 0, 1032, 1025, 978, 1017, 3, 0, 0, 0, 1038, 1006, 1030, 1033, 25, 0, 0, 0, 1010, 1023, 1070, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4046, 4119, 4120, 4091, 4055, 4132, 4148] avg=4096.0000 max=4148 max_rank=7 min=4046 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8064, 8320, 8320, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1017, 1070, 1023, 999, 1028, 1039, 986, 1042, 1057, 1048, 999, 1039, 1018, 1060, 1059, 1015, 992, 971, 997, 1003, 987, 1037, 1016, 1009, 1065, 1024, 1013, 1052, 1024, 1044, 1033, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[992, 1065, 1039, 1034, 1026, 1024, 1029, 1053, 1019, 1048, 1002, 1076, 1006, 1001, 985, 1030, 1074, 996, 991, 1015, 1010, 1018, 1049, 1016, 1005, 1085, 990, 994, 1035, 1039, 1002, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1012, 994, 1091, 1049, 1056, 994, 1067, 990, 1018, 1040, 1010, 1027, 967, 1047, 986, 1021, 1031, 1048, 999, 1044, 989, 1051, 960, 1025, 1029, 1058, 1024, 1072, 1042, 1034, 1000, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[991, 1093, 1048, 1105, 1016, 1030, 1031, 933, 977, 1025, 1038, 998, 1003, 984, 1040, 1054, 1032, 1062, 1058, 1065, 999, 1040, 964, 1018, 996, 1021, 1012, 1048, 999, 1058, 1046, 984] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1008, 1004, 1040, 1047, 1079, 1016, 965, 1017, 1004, 1030, 1047, 1028, 1021, 1019, 1044, 991, 1031, 1084, 1027, 1071, 1019, 1035, 991, 994, 1045, 988, 1024, 1028, 1106, 993, 993, 979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1036, 1047, 1023, 1067, 1005, 1046, 1020, 1005, 1006, 993, 1057, 994, 990, 1042, 1048, 972, 984, 1002, 1038, 1089, 1077, 1036, 1014, 983, 1031, 1022, 993, 1041, 1029, 1039, 1029, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[983, 1056, 960, 1001, 1017, 983, 990, 984, 1034, 1027, 1012, 1042, 1004, 1048, 1038, 1060, 1059, 1009, 1013, 1009, 1043, 1022, 1025, 1026, 1035, 1038, 1100, 1029, 1037, 999, 1050, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1064, 993, 1067, 1042, 966, 1007, 1010, 1027, 1028, 1045, 1015, 1060, 982, 1060, 1042, 1005, 1075, 1042, 1000, 1019, 986, 1006, 1030, 1010, 1047, 998, 1033, 1003, 1025, 1007, 1042, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1064, 993, 1067, 1007, 0, 0, 0, 0, 966, 1007, 1010, 1027, 35, 0, 0, 0, 1028, 1045, 1015, 1051, 0, 0, 0, 0, 982, 1060, 1042, 1005, 8, 7, 0, 0, 1075, 1042, 1000, 1000, 0, 0, 0, 0, 986, 1006, 1030, 1010, 21, 12, 0, 0, 1047, 998, 1033, 982, 0, 0, 0, 0, 1025, 1007, 1042, 1032, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4045, 4139, 4104, 4117, 4065, 4060, 4107] avg=4096.0000 max=4139 max_rank=2 min=4045 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[992, 1065, 1039, 999, 0, 0, 0, 0, 1026, 1024, 1029, 1053, 35, 0, 0, 0, 1019, 1048, 1002, 1067, 0, 0, 0, 0, 1006, 1001, 985, 1030, 8, 7, 0, 0, 1074, 996, 991, 996, 0, 0, 0, 0, 1010, 1018, 1049, 1016, 21, 12, 0, 0, 1005, 1085, 990, 973, 0, 0, 0, 0, 1035, 1039, 1002, 1020, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4167, 4136, 4037, 4057, 4126, 4053, 4097] avg=4096.0000 max=4167 max_rank=1 min=4037 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1036, 1047, 1023, 1031, 0, 0, 0, 0, 1005, 1046, 1020, 1005, 36, 0, 0, 0, 1006, 993, 1057, 986, 0, 0, 0, 0, 990, 1042, 1048, 972, 8, 7, 0, 0, 984, 1002, 1038, 1069, 0, 0, 0, 0, 1077, 1036, 1014, 983, 22, 13, 0, 0, 1031, 1022, 993, 1019, 0, 0, 0, 0, 1029, 1039, 1029, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4112, 4042, 4067, 4093, 4145, 4065, 4107] avg=4096.0000 max=4145 max_rank=5 min=4042 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1017, 1070, 1023, 957, 0, 0, 0, 0, 1028, 1039, 986, 1042, 37, 0, 0, 0, 1057, 1048, 999, 1023, 0, 0, 0, 0, 1018, 1060, 1059, 1015, 11, 10, 0, 0, 992, 971, 997, 977, 0, 0, 0, 0, 987, 1037, 1016, 1009, 25, 16, 5, 0, 1065, 1024, 1013, 1027, 0, 0, 0, 0, 1024, 1044, 1033, 1002, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4132, 4127, 4173, 3937, 4095, 4129, 4108] avg=4096.0000 max=4173 max_rank=3 min=3937 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16269 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[983, 1056, 960, 967, 0, 0, 0, 0, 1017, 983, 990, 984, 34, 0, 0, 0, 1034, 1027, 1012, 1033, 0, 0, 0, 0, 1004, 1048, 1038, 1060, 8, 7, 0, 0, 1059, 1009, 1013, 990, 0, 0, 0, 0, 1043, 1022, 1025, 1026, 21, 12, 0, 0, 1035, 1038, 1100, 1008, 0, 0, 0, 0, 1037, 999, 1050, 1035, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3966, 4008, 4106, 4165, 4071, 4149, 4181, 4122] avg=4096.0000 max=4181 max_rank=6 min=3966 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16523 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[991, 1093, 1048, 1067, 0, 0, 0, 0, 1016, 1030, 1031, 933, 38, 0, 0, 0, 977, 1025, 1038, 990, 0, 0, 0, 0, 1003, 984, 1040, 1054, 8, 7, 0, 0, 1032, 1062, 1058, 1045, 0, 0, 0, 0, 999, 1040, 964, 1018, 22, 13, 0, 0, 996, 1021, 1012, 1026, 0, 0, 0, 0, 999, 1058, 1046, 984, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4199, 4048, 4030, 4096, 4197, 4056, 4055, 4087] avg=4096.0000 max=4199 max_rank=0 min=4030 min_rank=2 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1008, 1004, 1040, 1011, 0, 0, 0, 0, 1079, 1016, 965, 1017, 36, 0, 0, 0, 1004, 1030, 1047, 1020, 0, 0, 0, 0, 1021, 1019, 1044, 991, 8, 7, 0, 0, 1031, 1084, 1027, 1051, 0, 0, 0, 0, 1019, 1035, 991, 994, 21, 13, 0, 0, 1045, 988, 1024, 1007, 0, 0, 0, 0, 1106, 993, 993, 979, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4113, 4101, 4090, 4193, 4073, 4064, 4071] avg=4096.0000 max=4193 max_rank=4 min=4063 min_rank=0 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1012, 994, 1091, 1013, 0, 0, 0, 0, 1056, 994, 1067, 990, 36, 0, 0, 0, 1018, 1040, 1010, 1019, 0, 0, 0, 0, 967, 1047, 986, 1021, 8, 7, 0, 0, 1031, 1048, 999, 1025, 0, 0, 0, 0, 989, 1051, 960, 1025, 22, 12, 0, 0, 1029, 1058, 1024, 1050, 0, 0, 0, 0, 1042, 1034, 1000, 993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4143, 4087, 4036, 4103, 4059, 4161, 4069] avg=4096.0000 max=4161 max_rank=6 min=4036 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8448, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8064, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1027, 1013, 1055, 987, 1074, 1030, 1002, 1026, 961, 1009, 1051, 1074, 985, 1034, 1037, 1013, 1003, 1013, 990, 1055, 1026, 1047, 1064, 1011, 1061, 1056, 991, 1040, 1036, 1025, 974, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1008, 991, 1012, 1023, 1007, 1004, 1038, 1037, 1033, 1022, 998, 996, 1074, 977, 1068, 1065, 988, 1032, 1047, 1075, 1026, 1060, 1009, 1098, 1015, 1016, 971, 1012, 971, 1086, 979, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[961, 1026, 1023, 1017, 1056, 978, 1061, 979, 1069, 1010, 1019, 1055, 1028, 1035, 1005, 1033, 979, 1029, 1061, 1054, 1034, 1043, 1019, 1008, 1060, 1037, 1001, 1011, 1020, 1028, 1016, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[996, 1088, 1012, 1006, 1009, 990, 1028, 1020, 1050, 1025, 1027, 993, 1083, 1039, 982, 1000, 1105, 992, 960, 1016, 1017, 1007, 1074, 1058, 1011, 1057, 1014, 1033, 1049, 1038, 990, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[992, 1005, 1018, 1055, 1036, 1049, 956, 994, 986, 1057, 1024, 988, 1057, 1046, 1050, 976, 1045, 1025, 1078, 1016, 1014, 1003, 971, 1067, 1008, 1070, 1026, 1062, 1067, 1031, 1028, 968] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1094, 1036, 1023, 1009, 1004, 1059, 1022, 1016, 985, 986, 1054, 1030, 1066, 1018, 980, 1002, 967, 995, 1034, 1031, 1047, 1032, 1030, 1035, 997, 1068, 998, 1062, 1039, 994, 1032, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[996, 1026, 966, 1011, 1053, 1084, 1074, 1060, 1027, 1048, 1032, 1016, 1033, 1010, 1027, 1027, 998, 1056, 1000, 966, 1019, 1054, 1016, 993, 1003, 1021, 1024, 987, 1081, 1070, 979, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1052, 1022, 1030, 1033, 984, 972, 1022, 1036, 1069, 1068, 1006, 1032, 1028, 1003, 994, 1020, 1018, 1030, 945, 1029, 995, 1031, 1070, 1051, 1084, 1026, 1041, 1009, 1008, 994, 1024, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1027, 1013, 1055, 987, 18, 0, 0, 0, 1074, 1030, 1002, 1026, 0, 0, 0, 0, 961, 1006, 1051, 1074, 0, 0, 0, 0, 983, 1034, 1037, 1013, 0, 0, 0, 0, 1003, 1013, 990, 1055, 12, 3, 0, 0, 1026, 1047, 1064, 981, 0, 0, 0, 0, 1061, 1044, 991, 1040, 0, 0, 0, 0, 1036, 1025, 974, 998, 9, 3, 2, 0] rerouted_tokens_per_ep_rank=[4100, 4132, 4092, 4067, 4076, 4118, 4136, 4047] avg=4096.0000 max=4136 max_rank=6 min=4047 min_rank=7 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[992, 1005, 1018, 1055, 19, 0, 0, 0, 1036, 1049, 956, 994, 1, 0, 0, 0, 986, 1053, 1024, 988, 0, 0, 0, 0, 1054, 1046, 1050, 976, 0, 0, 0, 0, 1045, 1025, 1078, 1016, 13, 3, 0, 0, 1014, 1003, 971, 1035, 0, 0, 0, 0, 1008, 1058, 1026, 1062, 0, 0, 0, 0, 1067, 1031, 1028, 968, 9, 4, 2, 0] rerouted_tokens_per_ep_rank=[4089, 4036, 4051, 4126, 4180, 4023, 4154, 4109] avg=4096.0000 max=4180 max_rank=4 min=4023 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1094, 1036, 1023, 1009, 19, 0, 0, 0, 1004, 1059, 1022, 1016, 1, 0, 0, 0, 985, 983, 1054, 1030, 0, 0, 0, 0, 1063, 1018, 980, 1002, 0, 0, 0, 0, 967, 995, 1034, 1031, 13, 3, 0, 0, 1047, 1032, 1030, 1003, 0, 0, 0, 0, 997, 1056, 998, 1062, 0, 0, 0, 0, 1039, 994, 1032, 1023, 9, 3, 2, 0] rerouted_tokens_per_ep_rank=[4181, 4102, 4052, 4063, 4043, 4112, 4113, 4102] avg=4096.0000 max=4181 max_rank=0 min=4043 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1008, 991, 1012, 1023, 20, 0, 0, 0, 1007, 1004, 1038, 1037, 1, 0, 0, 0, 1033, 1019, 998, 996, 0, 0, 0, 0, 1071, 977, 1068, 1065, 0, 0, 0, 0, 988, 1032, 1047, 1075, 13, 3, 0, 0, 1026, 1060, 1009, 1065, 0, 0, 0, 0, 1015, 1004, 971, 1012, 0, 0, 0, 0, 971, 1086, 979, 1030, 9, 3, 2, 0] rerouted_tokens_per_ep_rank=[4054, 4087, 4046, 4181, 4158, 4160, 4002, 4080] avg=4096.0000 max=4181 max_rank=3 min=4002 min_rank=6 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[996, 1026, 966, 1011, 18, 0, 0, 0, 1053, 1084, 1074, 1060, 0, 0, 0, 0, 1027, 1044, 1032, 1016, 0, 0, 0, 0, 1031, 1010, 1027, 1027, 0, 0, 0, 0, 998, 1056, 1000, 966, 12, 3, 0, 0, 1019, 1054, 1016, 963, 0, 0, 0, 0, 1003, 1009, 1024, 987, 0, 0, 0, 0, 1081, 1070, 979, 1011, 9, 4, 2, 0] rerouted_tokens_per_ep_rank=[4017, 4271, 4119, 4095, 4035, 4052, 4023, 4156] avg=4096.0000 max=4271 max_rank=1 min=4017 min_rank=0 imbalance_ratio=1.042725 max_load_reduction=28497 max_load_reduction_pct=0.869659 moved_to_echo=16266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[961, 1026, 1023, 1017, 18, 0, 0, 0, 1056, 978, 1061, 979, 0, 0, 0, 0, 1069, 1007, 1019, 1055, 0, 0, 0, 0, 1026, 1035, 1005, 1033, 0, 0, 0, 0, 979, 1029, 1061, 1054, 12, 3, 0, 0, 1034, 1043, 1019, 978, 0, 0, 0, 0, 1060, 1025, 1001, 1011, 0, 0, 0, 0, 1020, 1028, 1016, 1013, 9, 3, 2, 0] rerouted_tokens_per_ep_rank=[4045, 4074, 4150, 4099, 4138, 4074, 4097, 4091] avg=4096.0000 max=4150 max_rank=2 min=4045 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[996, 1088, 1012, 1006, 19, 0, 0, 0, 1009, 990, 1028, 1020, 1, 0, 0, 0, 1050, 1022, 1027, 993, 0, 0, 0, 0, 1080, 1039, 982, 1000, 0, 0, 0, 0, 1105, 992, 960, 1016, 13, 3, 0, 0, 1017, 1007, 1074, 1026, 0, 0, 0, 0, 1011, 1045, 1014, 1033, 0, 0, 0, 0, 1049, 1038, 990, 999, 9, 3, 2, 0] rerouted_tokens_per_ep_rank=[4121, 4048, 4092, 4101, 4089, 4124, 4103, 4090] avg=4096.0000 max=4124 max_rank=5 min=4048 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1052, 1022, 1030, 1033, 24, 0, 0, 0, 984, 972, 1022, 1036, 4, 0, 0, 0, 1069, 1059, 1006, 1032, 0, 0, 0, 0, 1019, 1003, 994, 1020, 0, 0, 0, 0, 1018, 1030, 945, 1029, 18, 9, 0, 0, 995, 1031, 1070, 1009, 0, 0, 0, 0, 1084, 1006, 1041, 1009, 0, 0, 0, 0, 1008, 994, 1024, 1042, 11, 9, 5, 0] rerouted_tokens_per_ep_rank=[4161, 4018, 4166, 4036, 4049, 4105, 4140, 4093] avg=4096.0000 max=4166 max_rank=2 min=4018 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1013, 1015, 1024, 1034, 1009, 1041, 1057, 1046, 1020, 1027, 1028, 1055, 996, 973, 1027, 1034, 1057, 1032, 1038, 950, 1026, 957, 1013, 1034, 1017, 1054, 1035, 1050, 1019, 1031, 1016, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1055, 1042, 1045, 1066, 1047, 1038, 1037, 1049, 1021, 1083, 1015, 1005, 1027, 1012, 1020, 990, 1008, 1047, 1024, 1048, 1067, 968, 999, 1012, 1023, 1058, 948, 970, 1010, 1023, 981, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1024, 1095, 1092, 1079, 987, 1037, 1027, 986, 1030, 1014, 1043, 965, 1023, 990, 1018, 989, 995, 1023, 1039, 980, 1077, 1053, 1030, 1021, 1059, 1002, 1010, 957, 999, 1060, 1032, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1043, 1039, 1077, 993, 1021, 998, 975, 1009, 1063, 992, 954, 1021, 1038, 1021, 1037, 1102, 1030, 1073, 1043, 1080, 1057, 1028, 995, 1048, 1008, 992, 1004, 988, 1030, 986, 1026, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1061, 1099, 1019, 1034, 994, 1009, 1084, 988, 1027, 990, 1031, 1039, 1061, 1018, 1003, 1020, 1034, 1048, 972, 1028, 1036, 1023, 1045, 976, 1039, 994, 1016, 1012, 1018, 1008, 1022, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1009, 1043, 1017, 1024, 1028, 1042, 1020, 999, 1018, 1049, 1007, 974, 1033, 1033, 1027, 1027, 1028, 1039, 999, 1115, 950, 1016, 1034, 1050, 1005, 986, 1018, 1020, 1032, 1038, 1041, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[977, 992, 1036, 1057, 999, 1019, 994, 1012, 1013, 1078, 1075, 1024, 999, 1056, 1050, 995, 1023, 1023, 1025, 1024, 1049, 1007, 1073, 1005, 1032, 1052, 1003, 1011, 978, 1000, 1029, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1042, 1022, 1038, 1068, 1085, 958, 995, 1051, 1020, 1042, 1027, 996, 1012, 1001, 1036, 1020, 1034, 954, 1047, 986, 1040, 1007, 1017, 1019, 1006, 1029, 1059, 1028, 1023, 1014, 1057, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1013, 1015, 1024, 974, 0, 0, 0, 0, 1009, 1041, 1057, 1046, 15, 0, 0, 0, 1020, 1027, 1028, 1055, 2, 0, 0, 0, 996, 973, 1027, 1034, 9, 0, 0, 0, 1057, 1024, 1038, 950, 0, 0, 0, 0, 1026, 957, 1013, 1034, 4, 0, 0, 0, 1017, 1054, 1035, 1050, 35, 0, 0, 0, 1019, 1031, 1016, 1040, 2, 1, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4168, 4132, 4039, 4069, 4034, 4191, 4109] avg=4096.0000 max=4191 max_rank=6 min=4026 min_rank=0 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1024, 1095, 1092, 1001, 0, 0, 0, 0, 987, 1037, 1027, 986, 19, 0, 0, 0, 1030, 1014, 1043, 965, 8, 0, 0, 0, 1023, 990, 1018, 989, 14, 0, 0, 0, 995, 1001, 1039, 980, 0, 0, 0, 0, 1077, 1053, 1030, 1021, 8, 0, 0, 0, 1059, 1002, 1010, 957, 39, 0, 0, 0, 999, 1060, 1032, 1032, 6, 6, 0, 0] rerouted_tokens_per_ep_rank=[4212, 4056, 4060, 4034, 4015, 4189, 4067, 4135] avg=4096.0000 max=4212 max_rank=0 min=4015 min_rank=4 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1009, 1043, 1017, 965, 0, 0, 0, 0, 1028, 1042, 1020, 999, 15, 0, 0, 0, 1018, 1049, 1007, 974, 2, 0, 0, 0, 1033, 1033, 1027, 1027, 9, 0, 0, 0, 1028, 1031, 999, 1115, 0, 0, 0, 0, 950, 1016, 1034, 1050, 4, 0, 0, 0, 1005, 986, 1018, 1020, 34, 0, 0, 0, 1032, 1038, 1041, 1047, 2, 1, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4104, 4050, 4129, 4173, 4054, 4063, 4161] avg=4096.0000 max=4173 max_rank=4 min=4034 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1043, 1039, 1077, 935, 0, 0, 0, 0, 1021, 998, 975, 1009, 15, 0, 0, 0, 1063, 992, 954, 1021, 2, 0, 0, 0, 1038, 1021, 1037, 1102, 9, 0, 0, 0, 1030, 1065, 1043, 1080, 0, 0, 0, 0, 1057, 1028, 995, 1048, 4, 0, 0, 0, 1008, 992, 1004, 988, 33, 0, 0, 0, 1030, 986, 1026, 997, 2, 1, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4018, 4032, 4207, 4218, 4132, 4025, 4042] avg=4096.0000 max=4218 max_rank=4 min=4018 min_rank=1 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1055, 1042, 1045, 1002, 0, 0, 0, 0, 1047, 1038, 1037, 1049, 16, 0, 0, 0, 1021, 1083, 1015, 1005, 2, 0, 0, 0, 1027, 1012, 1020, 990, 10, 0, 0, 0, 1008, 1039, 1024, 1048, 0, 0, 0, 0, 1067, 968, 999, 1012, 4, 0, 0, 0, 1023, 1058, 948, 970, 36, 0, 0, 0, 1010, 1023, 981, 1030, 2, 2, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4187, 4126, 4059, 4119, 4050, 4035, 4048] avg=4096.0000 max=4187 max_rank=1 min=4035 min_rank=6 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16252 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[977, 992, 1036, 994, 0, 0, 0, 0, 999, 1019, 994, 1012, 16, 0, 0, 0, 1013, 1078, 1075, 1024, 2, 0, 0, 0, 999, 1056, 1050, 995, 10, 0, 0, 0, 1023, 1015, 1025, 1024, 0, 0, 0, 0, 1049, 1007, 1073, 1005, 4, 0, 0, 0, 1032, 1052, 1003, 1011, 35, 0, 0, 0, 978, 1000, 1029, 1058, 2, 2, 0, 0] rerouted_tokens_per_ep_rank=[3999, 4040, 4192, 4110, 4087, 4138, 4133, 4069] avg=4096.0000 max=4192 max_rank=2 min=3999 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1061, 1099, 1019, 974, 0, 0, 0, 0, 994, 1009, 1084, 988, 15, 0, 0, 0, 1027, 990, 1031, 1039, 2, 0, 0, 0, 1061, 1018, 1003, 1020, 9, 0, 0, 0, 1034, 1040, 972, 1028, 0, 0, 0, 0, 1036, 1023, 1045, 976, 4, 0, 0, 0, 1039, 994, 1016, 1012, 35, 0, 0, 0, 1018, 1008, 1022, 1020, 2, 1, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4090, 4089, 4111, 4074, 4084, 4096, 4071] avg=4096.0000 max=4153 max_rank=0 min=4071 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1042, 1022, 1038, 1004, 0, 0, 0, 0, 1085, 958, 995, 1051, 16, 0, 0, 0, 1020, 1042, 1027, 996, 2, 0, 0, 0, 1012, 1001, 1036, 1020, 10, 0, 0, 0, 1034, 946, 1047, 986, 0, 0, 0, 0, 1040, 1007, 1017, 1019, 4, 0, 0, 0, 1006, 1029, 1059, 1028, 36, 0, 0, 0, 1023, 1014, 1057, 1035, 2, 2, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4105, 4087, 4079, 4013, 4087, 4158, 4133] avg=4096.0000 max=4158 max_rank=6 min=4013 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8448, 8448, 7936, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8064, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[986, 1035, 984, 1058, 1069, 1029, 1042, 1061, 1031, 968, 1012, 1057, 1066, 1082, 1060, 1013, 1010, 1008, 1022, 1058, 1033, 1042, 999, 1053, 1015, 961, 1008, 1001, 1020, 998, 1009, 978] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1078, 1001, 1028, 1036, 1037, 989, 1058, 1023, 1002, 1059, 1014, 1023, 1019, 1040, 1028, 991, 1065, 987, 1007, 1031, 1035, 1050, 1028, 1027, 1006, 977, 1002, 1032, 1037, 1024, 1036, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1040, 1001, 1011, 1000, 1039, 1049, 1006, 1015, 1016, 1042, 1054, 1043, 1017, 1038, 1021, 1036, 991, 1026, 1009, 1011, 1052, 1017, 1011, 1002, 1066, 1046, 1001, 1054, 1066, 1025, 978, 985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1015, 990, 1061, 1025, 1049, 960, 972, 1013, 1029, 1043, 1025, 1018, 982, 1089, 1041, 979, 972, 1053, 1039, 1008, 1014, 1022, 1062, 1040, 1038, 1059, 1026, 1028, 1039, 1036, 992, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1045, 1027, 976, 1030, 1012, 1024, 1040, 1054, 980, 1053, 1040, 1056, 984, 1052, 1019, 1022, 982, 1071, 1025, 1030, 1008, 1049, 1031, 986, 1010, 1052, 1015, 1014, 1042, 1012, 996, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1031, 1046, 1043, 1004, 1074, 1062, 1067, 1008, 1055, 1041, 1023, 1009, 1032, 974, 1035, 1004, 1043, 1060, 996, 1012, 1039, 948, 1016, 1041, 991, 1036, 1038, 980, 1014, 1017, 1022, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[986, 1047, 1025, 1036, 981, 1017, 969, 1043, 1067, 1015, 1074, 1028, 1043, 1035, 1040, 1029, 1069, 1031, 995, 1050, 997, 1056, 1017, 990, 1025, 1027, 987, 1001, 1013, 984, 1010, 1081] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1000, 1031, 999, 1003, 1019, 1016, 1056, 1022, 1027, 992, 979, 1050, 987, 1056, 1010, 1004, 1046, 1029, 1011, 1028, 1049, 1010, 1051, 1074, 974, 1049, 1029, 1012, 1023, 1015, 1041, 1076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[986, 1035, 984, 1058, 7, 2, 0, 0, 1056, 1029, 1042, 1061, 0, 0, 0, 0, 1031, 968, 1012, 1037, 0, 0, 0, 0, 1066, 1075, 1060, 1013, 0, 0, 0, 0, 1010, 1008, 1022, 1058, 0, 0, 0, 0, 1024, 1042, 999, 1053, 0, 0, 0, 0, 1015, 961, 1008, 1001, 20, 6, 0, 0, 1020, 998, 1009, 978, 7, 7, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4188, 4048, 4214, 4098, 4118, 4011, 4019] avg=4096.0000 max=4214 max_rank=3 min=4011 min_rank=6 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1040, 1001, 1011, 1000, 7, 2, 0, 0, 1026, 1049, 1006, 1015, 0, 0, 0, 0, 1016, 1042, 1054, 1024, 0, 0, 0, 0, 1017, 1031, 1021, 1036, 0, 0, 0, 0, 991, 1026, 1009, 1011, 0, 0, 0, 0, 1043, 1017, 1011, 1002, 0, 0, 0, 0, 1066, 1046, 1001, 1054, 19, 6, 0, 0, 1066, 1025, 978, 985, 7, 7, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4096, 4136, 4105, 4037, 4073, 4192, 4068] avg=4096.0000 max=4192 max_rank=6 min=4037 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1078, 1001, 1028, 1036, 12, 9, 7, 0, 1018, 989, 1058, 1023, 0, 0, 0, 0, 1002, 1059, 1014, 1001, 0, 0, 0, 0, 1019, 1028, 1028, 991, 0, 0, 0, 0, 1065, 980, 1007, 1031, 0, 0, 0, 0, 1017, 1050, 1028, 1027, 0, 0, 0, 0, 1006, 977, 1002, 1032, 22, 9, 0, 0, 1037, 1024, 1036, 998, 9, 10, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4088, 4076, 4066, 4083, 4122, 4048, 4114] avg=4096.0000 max=4171 max_rank=0 min=4048 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1015, 990, 1061, 1025, 7, 2, 0, 0, 1036, 960, 972, 1013, 0, 0, 0, 0, 1029, 1043, 1025, 999, 0, 0, 0, 0, 982, 1082, 1041, 979, 0, 0, 0, 0, 972, 1053, 1039, 1008, 0, 0, 0, 0, 1005, 1022, 1062, 1040, 0, 0, 0, 0, 1038, 1059, 1026, 1028, 19, 6, 0, 0, 1039, 1036, 992, 1049, 7, 7, 0, 0] rerouted_tokens_per_ep_rank=[4100, 3981, 4096, 4084, 4072, 4129, 4176, 4130] avg=4096.0000 max=4176 max_rank=6 min=3981 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1045, 1027, 976, 1030, 7, 2, 0, 0, 1000, 1024, 1040, 1054, 0, 0, 0, 0, 980, 1053, 1040, 1036, 0, 0, 0, 0, 984, 1045, 1019, 1022, 0, 0, 0, 0, 982, 1071, 1025, 1030, 0, 0, 0, 0, 999, 1049, 1031, 986, 0, 0, 0, 0, 1010, 1052, 1015, 1014, 20, 6, 0, 0, 1042, 1012, 996, 1031, 7, 6, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4118, 4109, 4070, 4108, 4065, 4117, 4094] avg=4096.0000 max=4118 max_rank=1 min=4065 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1031, 1046, 1043, 1004, 6, 2, 0, 0, 1061, 1062, 1067, 1008, 0, 0, 0, 0, 1055, 1041, 1023, 990, 0, 0, 0, 0, 1032, 968, 1035, 1004, 0, 0, 0, 0, 1043, 1060, 996, 1012, 0, 0, 0, 0, 1030, 948, 1016, 1041, 0, 0, 0, 0, 991, 1036, 1038, 980, 19, 6, 0, 0, 1014, 1017, 1022, 1007, 7, 7, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4198, 4109, 4039, 4111, 4035, 4070, 4074] avg=4096.0000 max=4198 max_rank=1 min=4035 min_rank=5 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16290 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[986, 1047, 1025, 1036, 7, 2, 0, 0, 969, 1017, 969, 1043, 0, 0, 0, 0, 1067, 1015, 1074, 1009, 0, 0, 0, 0, 1043, 1028, 1040, 1029, 0, 0, 0, 0, 1069, 1031, 995, 1050, 0, 0, 0, 0, 988, 1056, 1017, 990, 0, 0, 0, 0, 1025, 1027, 987, 1001, 19, 6, 0, 0, 1013, 984, 1010, 1081, 7, 6, 0, 0] rerouted_tokens_per_ep_rank=[4103, 3998, 4165, 4140, 4145, 4051, 4065, 4101] avg=4096.0000 max=4165 max_rank=2 min=3998 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1000, 1031, 999, 1003, 7, 2, 0, 0, 1007, 1016, 1056, 1022, 0, 0, 0, 0, 1027, 992, 979, 1031, 0, 0, 0, 0, 987, 1049, 1010, 1004, 0, 0, 0, 0, 1046, 1029, 1011, 1028, 0, 0, 0, 0, 1040, 1010, 1051, 1074, 0, 0, 0, 0, 974, 1049, 1029, 1012, 19, 6, 0, 0, 1023, 1015, 1041, 1076, 7, 6, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4101, 4029, 4050, 4114, 4175, 4089, 4168] avg=4096.0000 max=4175 max_rank=5 min=4029 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16546 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1047, 989, 995, 1009, 1055, 1043, 1019, 1029, 1022, 1008, 1005, 1023, 1042, 1053, 1013, 1001, 1061, 974, 1014, 1040, 1010, 1026, 1063, 1035, 971, 1103, 1005, 1031, 978, 1012, 1004, 1088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1046, 1043, 996, 1092, 1012, 1059, 948, 1028, 996, 1033, 972, 1056, 1018, 1021, 1032, 1009, 1033, 975, 1010, 1106, 1035, 1009, 993, 996, 976, 1042, 1054, 1012, 1046, 1058, 1013, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1065, 1080, 1009, 983, 1035, 1026, 969, 1064, 1015, 1074, 1038, 967, 1053, 1115, 1001, 1004, 1029, 1033, 1049, 1014, 1018, 1018, 932, 1063, 1008, 1021, 1032, 990, 1014, 1009, 1045, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1055, 1005, 1033, 1046, 946, 1019, 966, 1027, 1016, 1074, 1053, 1002, 1058, 1001, 1000, 1022, 999, 1039, 1024, 1069, 1013, 1060, 1031, 965, 1065, 1042, 1023, 996, 1001, 1032, 1048, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1000, 1058, 1081, 1034, 1022, 990, 1040, 1060, 1033, 1025, 992, 1031, 1020, 1038, 1060, 1011, 1021, 1030, 1027, 1040, 1028, 1009, 1009, 1009, 1044, 992, 1047, 1012, 989, 1035, 1012, 969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1078, 1044, 1010, 1031, 1018, 976, 1066, 1046, 988, 1001, 1033, 981, 1037, 1029, 1049, 982, 1018, 999, 1048, 1063, 998, 1023, 1024, 1022, 992, 1016, 1033, 1068, 1065, 1006, 972, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[998, 1021, 1000, 994, 994, 1065, 1040, 996, 1014, 1045, 1035, 1020, 1086, 973, 1038, 965, 1046, 1077, 1058, 965, 1015, 1043, 1059, 1005, 1029, 1020, 998, 1070, 1057, 1026, 976, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1012, 1057, 1005, 1067, 969, 1023, 1029, 1000, 1037, 1002, 1061, 1037, 1044, 999, 1064, 989, 1082, 1013, 1014, 1004, 983, 983, 1047, 1008, 1046, 1036, 1032, 1006, 1036, 1019, 1056, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1020, 989, 995, 1009, 0, 0, 0, 0, 1055, 1043, 1019, 1029, 23, 0, 0, 0, 1022, 1008, 1005, 1023, 6, 3, 0, 0, 1036, 1053, 1013, 1001, 0, 0, 0, 0, 1061, 974, 1014, 1015, 0, 0, 0, 0, 1010, 1026, 1063, 1035, 27, 2, 0, 0, 971, 1098, 1005, 1031, 0, 0, 0, 0, 978, 1012, 1004, 1088, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4169, 4067, 4103, 4064, 4163, 4105, 4084] avg=4096.0000 max=4169 max_rank=1 min=4013 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1038, 1080, 1009, 983, 0, 0, 0, 0, 1035, 1026, 969, 1064, 22, 0, 0, 0, 1015, 1074, 1038, 967, 6, 2, 0, 0, 1047, 1115, 1001, 1004, 0, 0, 0, 0, 1029, 1033, 1049, 990, 0, 0, 0, 0, 1018, 1018, 932, 1063, 27, 2, 0, 0, 1008, 1017, 1032, 990, 0, 0, 0, 0, 1014, 1009, 1045, 995, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4116, 4102, 4167, 4101, 4060, 4047, 4065] avg=4096.0000 max=4167 max_rank=3 min=4047 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[986, 1057, 1005, 1067, 0, 0, 0, 0, 969, 1023, 1029, 1000, 22, 0, 0, 0, 1037, 1002, 1061, 1037, 6, 3, 0, 0, 1038, 999, 1064, 989, 0, 0, 0, 0, 1082, 1013, 1014, 980, 0, 0, 0, 0, 983, 983, 1047, 1008, 26, 2, 0, 0, 1046, 1031, 1032, 1006, 0, 0, 0, 0, 1036, 1019, 1056, 1008, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4043, 4146, 4090, 4089, 4049, 4115, 4121] avg=4096.0000 max=4146 max_rank=2 min=4043 min_rank=1 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1051, 1044, 1010, 1031, 0, 0, 0, 0, 1018, 976, 1066, 1046, 23, 0, 0, 0, 988, 1001, 1033, 981, 6, 2, 0, 0, 1031, 1029, 1049, 982, 0, 0, 0, 0, 1018, 999, 1048, 1038, 0, 0, 0, 0, 998, 1023, 1024, 1022, 27, 2, 0, 0, 992, 1012, 1033, 1068, 0, 0, 0, 0, 1065, 1006, 972, 1052, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4129, 4011, 4091, 4103, 4096, 4105, 4097] avg=4096.0000 max=4136 max_rank=0 min=4011 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1028, 1005, 1033, 1046, 0, 0, 0, 0, 946, 1019, 966, 1027, 23, 0, 0, 0, 1016, 1074, 1053, 1002, 6, 3, 0, 0, 1052, 1001, 1000, 1022, 0, 0, 0, 0, 999, 1039, 1024, 1044, 0, 0, 0, 0, 1013, 1060, 1031, 965, 27, 2, 0, 0, 1065, 1037, 1023, 996, 0, 0, 0, 0, 1001, 1032, 1048, 1038, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 3981, 4154, 4075, 4106, 4098, 4121, 4121] avg=4096.0000 max=4154 max_rank=2 min=3981 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1019, 1043, 996, 1092, 0, 0, 0, 0, 1012, 1059, 948, 1028, 24, 0, 0, 0, 996, 1033, 972, 1056, 6, 3, 0, 0, 1012, 1021, 1032, 1009, 0, 0, 0, 0, 1033, 975, 1010, 1080, 0, 0, 0, 0, 1035, 1009, 993, 996, 27, 2, 0, 0, 976, 1037, 1054, 1012, 0, 0, 0, 0, 1046, 1058, 1013, 1049, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4071, 4066, 4074, 4098, 4062, 4079, 4168] avg=4096.0000 max=4168 max_rank=7 min=4062 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[969, 1021, 1000, 994, 0, 0, 0, 0, 994, 1065, 1040, 996, 25, 4, 0, 0, 1014, 1045, 1035, 1020, 13, 6, 0, 0, 1069, 973, 1038, 965, 0, 0, 0, 0, 1046, 1077, 1058, 933, 0, 0, 0, 0, 1015, 1043, 1059, 1005, 29, 7, 0, 0, 1029, 1008, 998, 1070, 0, 0, 0, 0, 1057, 1026, 976, 1040, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[3984, 4124, 4133, 4045, 4114, 4158, 4105, 4105] avg=4096.0000 max=4158 max_rank=5 min=3984 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[975, 1058, 1081, 1034, 0, 0, 0, 0, 1022, 990, 1040, 1060, 23, 0, 0, 0, 1033, 1025, 992, 1031, 6, 2, 0, 0, 1014, 1038, 1060, 1011, 0, 0, 0, 0, 1021, 1030, 1027, 1015, 0, 0, 0, 0, 1028, 1009, 1009, 1009, 25, 2, 0, 0, 1044, 988, 1047, 1012, 0, 0, 0, 0, 989, 1035, 1012, 969, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4135, 4089, 4123, 4093, 4082, 4091, 4007] avg=4096.0000 max=4148 max_rank=0 min=4007 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1051, 997, 981, 1093, 1091, 1013, 992, 1019, 1034, 984, 1038, 1025, 1044, 1014, 1057, 1040, 1000, 1008, 990, 1030, 1035, 995, 1008, 1039, 1026, 1066, 998, 1012, 998, 1040, 1018, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1016, 1058, 1003, 996, 1036, 1017, 1046, 1014, 1072, 1036, 989, 1038, 1025, 1081, 998, 1025, 1028, 1002, 1016, 998, 1059, 1024, 1026, 1013, 1038, 1039, 1021, 1060, 1015, 980, 967, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1033, 1037, 1021, 1002, 1055, 1014, 989, 1036, 1017, 1053, 1035, 1036, 1010, 964, 1023, 1021, 1020, 974, 1085, 1007, 1027, 1003, 1037, 1051, 1011, 992, 1070, 1013, 1055, 1023, 1040, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1024, 1079, 1089, 1032, 1021, 1053, 1011, 1026, 1022, 1018, 994, 1010, 1039, 1005, 1053, 1013, 1027, 991, 987, 1014, 1031, 993, 1030, 1060, 1038, 977, 1046, 1004, 1061, 1000, 1027, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1062, 1021, 1001, 1001, 1018, 1008, 1033, 1003, 992, 1004, 1065, 1041, 1011, 1026, 1013, 1077, 973, 1036, 1062, 1003, 1026, 1028, 1071, 1043, 1035, 957, 1022, 1027, 1009, 1066, 1043, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1001, 1025, 1034, 1024, 991, 1022, 1043, 1064, 1059, 1021, 1039, 1037, 965, 980, 1006, 1028, 1014, 1029, 1059, 1056, 1034, 1077, 1024, 991, 994, 957, 1071, 1041, 983, 1043, 1016, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1039, 1043, 1022, 1043, 1010, 1056, 1000, 1044, 970, 978, 992, 1009, 1025, 1012, 1012, 1007, 1030, 982, 995, 1072, 1048, 1007, 1030, 1054, 1059, 1024, 1027, 1020, 1076, 1022, 1050, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[979, 1046, 1054, 985, 998, 998, 1046, 1009, 971, 992, 1054, 1023, 1064, 1001, 1010, 1046, 1043, 1016, 1022, 1004, 1093, 1052, 1084, 1022, 1028, 1023, 980, 1006, 1031, 1088, 996, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1024, 1063, 1089, 1032, 0, 0, 0, 0, 1021, 1053, 1011, 1026, 0, 0, 0, 0, 1022, 1018, 994, 1010, 14, 0, 0, 0, 1039, 1005, 1053, 1013, 9, 0, 0, 0, 1027, 991, 987, 1014, 24, 0, 0, 0, 990, 993, 1030, 1060, 0, 0, 0, 0, 1038, 977, 1046, 1004, 7, 3, 0, 0, 1061, 1000, 1027, 993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4208, 4111, 4058, 4119, 4043, 4073, 4075, 4081] avg=4096.0000 max=4208 max_rank=0 min=4043 min_rank=4 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1051, 983, 981, 1093, 0, 0, 0, 0, 1091, 1013, 992, 1019, 0, 0, 0, 0, 1034, 984, 1038, 1025, 14, 0, 0, 0, 1044, 1014, 1057, 1040, 8, 0, 0, 0, 1000, 1008, 990, 1030, 24, 0, 0, 0, 994, 995, 1008, 1039, 0, 0, 0, 0, 1026, 1066, 998, 1012, 6, 3, 0, 0, 998, 1040, 1018, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4115, 4095, 4163, 4052, 4036, 4111, 4088] avg=4096.0000 max=4163 max_rank=3 min=4036 min_rank=5 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1016, 1044, 1003, 996, 0, 0, 0, 0, 1036, 1017, 1046, 1014, 0, 0, 0, 0, 1072, 1036, 989, 1038, 15, 0, 0, 0, 1025, 1081, 998, 1025, 8, 0, 0, 0, 1028, 1002, 1016, 998, 24, 0, 0, 0, 1016, 1024, 1026, 1013, 0, 0, 0, 0, 1038, 1039, 1021, 1060, 6, 4, 0, 0, 1015, 980, 967, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4113, 4150, 4137, 4068, 4079, 4168, 3994] avg=4096.0000 max=4168 max_rank=6 min=3994 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1062, 1007, 1001, 1001, 0, 0, 0, 0, 1018, 1008, 1033, 1003, 0, 0, 0, 0, 992, 1004, 1065, 1041, 14, 0, 0, 0, 1011, 1026, 1013, 1077, 8, 0, 0, 0, 973, 1036, 1062, 1003, 23, 0, 0, 0, 986, 1028, 1071, 1043, 0, 0, 0, 0, 1035, 957, 1022, 1027, 6, 3, 0, 0, 1009, 1066, 1043, 991, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4062, 4116, 4135, 4097, 4128, 4050, 4109] avg=4096.0000 max=4135 max_rank=3 min=4050 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1033, 1013, 1021, 1002, 0, 0, 0, 0, 1047, 1014, 989, 1036, 0, 0, 0, 0, 1017, 1053, 1035, 1036, 19, 0, 0, 0, 1010, 964, 1023, 1021, 13, 3, 0, 0, 1020, 974, 1085, 1007, 27, 0, 0, 0, 973, 1003, 1037, 1051, 0, 0, 0, 0, 1011, 992, 1070, 1013, 11, 8, 0, 0, 1055, 1023, 1040, 1014, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4086, 4160, 4034, 4113, 4064, 4105, 4137] avg=4096.0000 max=4160 max_rank=2 min=4034 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1039, 1029, 1022, 1043, 0, 0, 0, 0, 1010, 1056, 1000, 1044, 0, 0, 0, 0, 970, 978, 992, 1009, 15, 0, 0, 0, 1025, 1012, 1012, 1007, 8, 0, 0, 0, 1030, 982, 995, 1072, 24, 0, 0, 0, 1005, 1007, 1030, 1054, 0, 0, 0, 0, 1059, 1024, 1027, 1020, 6, 4, 0, 0, 1076, 1022, 1050, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4110, 3964, 4064, 4103, 4096, 4140, 4158] avg=4096.0000 max=4158 max_rank=7 min=3964 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1001, 1011, 1034, 1024, 0, 0, 0, 0, 991, 1022, 1043, 1064, 0, 0, 0, 0, 1059, 1021, 1039, 1037, 14, 0, 0, 0, 965, 980, 1006, 1028, 8, 0, 0, 0, 1014, 1029, 1059, 1056, 24, 0, 0, 0, 993, 1077, 1024, 991, 0, 0, 0, 0, 994, 957, 1071, 1041, 6, 3, 0, 0, 983, 1043, 1016, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4120, 4170, 3987, 4182, 4085, 4072, 4082] avg=4096.0000 max=4182 max_rank=4 min=3987 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[979, 1032, 1054, 985, 0, 0, 0, 0, 998, 998, 1046, 1009, 0, 0, 0, 0, 971, 992, 1054, 1023, 15, 0, 0, 0, 1064, 1001, 1010, 1046, 8, 0, 0, 0, 1043, 1016, 1022, 1004, 25, 0, 0, 0, 1049, 1052, 1084, 1022, 0, 0, 0, 0, 1028, 1023, 980, 1006, 6, 4, 0, 0, 1031, 1088, 996, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4051, 4055, 4129, 4110, 4207, 4047, 4119] avg=4096.0000 max=4207 max_rank=5 min=4047 min_rank=6 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16483 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1037, 999, 1061, 1017, 995, 1028, 1032, 1043, 1027, 1055, 1014, 1025, 983, 1019, 1034, 1014, 1044, 1011, 1013, 1037, 1027, 1018, 1005, 1024, 1017, 1011, 1061, 1029, 1031, 1073, 1001, 983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1068, 1019, 1003, 1050, 1017, 1015, 1047, 1013, 1037, 987, 996, 1053, 1051, 1015, 1000, 1068, 1005, 1023, 1031, 1000, 1041, 1003, 1025, 1001, 1000, 1041, 1061, 1038, 1016, 1022, 1000, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[984, 1066, 1001, 1068, 1059, 1000, 1031, 1000, 1024, 1031, 1001, 1043, 1058, 1064, 960, 1039, 997, 1010, 978, 1039, 1001, 1003, 1076, 1041, 1058, 1014, 993, 1023, 1017, 1029, 1025, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[998, 1020, 1009, 1032, 1016, 1028, 1038, 1087, 998, 1044, 974, 1020, 1039, 1034, 1022, 1042, 1028, 1009, 1015, 1037, 1004, 1055, 1008, 1012, 1040, 1043, 1011, 1041, 995, 1048, 1009, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1005, 989, 994, 1035, 1044, 1021, 934, 1068, 1015, 996, 996, 985, 1037, 1039, 1058, 1080, 1022, 1048, 1043, 1015, 1025, 1025, 1001, 1034, 1047, 1037, 1036, 1048, 981, 1068, 1033, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1017, 1063, 1025, 968, 1018, 1003, 1031, 996, 1050, 1012, 1044, 1038, 1007, 1014, 1036, 1025, 1025, 1058, 1039, 1032, 989, 1061, 1026, 996, 1016, 1027, 985, 990, 1058, 1021, 1084, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[994, 1062, 1042, 1030, 1046, 1055, 1001, 1046, 1003, 1016, 999, 1067, 1051, 1030, 1094, 984, 1004, 974, 1063, 1037, 1002, 1033, 1027, 1034, 1071, 1025, 1033, 977, 967, 1019, 1003, 979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1001, 1005, 1069, 1000, 1018, 1011, 1018, 1007, 987, 1011, 1001, 1000, 996, 1046, 1043, 1063, 1056, 973, 1009, 1033, 1019, 1051, 1088, 1024, 952, 1053, 1008, 1046, 1045, 1076, 1031, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1037, 999, 1061, 1017, 4, 0, 0, 0, 995, 1028, 1032, 1043, 0, 0, 0, 0, 1027, 1055, 1014, 1025, 26, 0, 0, 0, 983, 1019, 1034, 981, 0, 0, 0, 0, 1044, 1011, 1013, 1037, 7, 0, 0, 0, 1027, 1018, 1004, 1024, 0, 0, 0, 0, 1017, 1004, 1061, 1029, 0, 0, 0, 0, 1031, 1073, 1001, 983, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4098, 4147, 4017, 4112, 4073, 4111, 4092] avg=4096.0000 max=4147 max_rank=2 min=4017 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1068, 1019, 1003, 1050, 4, 0, 0, 0, 1017, 1015, 1047, 1013, 0, 0, 0, 0, 1037, 987, 996, 1053, 28, 0, 0, 0, 1051, 1015, 1000, 1033, 0, 0, 0, 0, 1005, 1023, 1031, 1000, 7, 0, 0, 0, 1041, 1003, 1024, 1001, 0, 0, 0, 0, 1000, 1034, 1061, 1038, 0, 0, 0, 0, 1016, 1022, 1000, 1022, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4092, 4101, 4099, 4066, 4069, 4133, 4064] avg=4096.0000 max=4144 max_rank=0 min=4064 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1001, 1005, 1069, 1000, 4, 0, 0, 0, 1018, 1011, 1018, 1007, 0, 0, 0, 0, 987, 1011, 1001, 1000, 27, 0, 0, 0, 996, 1046, 1043, 1029, 0, 0, 0, 0, 1056, 973, 1009, 1033, 7, 0, 0, 0, 1019, 1051, 1087, 1024, 0, 0, 0, 0, 952, 1046, 1008, 1046, 0, 0, 0, 0, 1045, 1076, 1031, 1028, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4054, 4026, 4114, 4078, 4181, 4052, 4184] avg=4096.0000 max=4184 max_rank=7 min=4026 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1005, 989, 994, 1035, 9, 0, 0, 0, 1044, 1021, 934, 1068, 2, 0, 0, 0, 1015, 996, 996, 985, 33, 0, 0, 0, 1037, 1039, 1058, 1037, 0, 0, 0, 0, 1022, 1048, 1043, 1015, 10, 2, 0, 0, 1025, 1025, 997, 1034, 0, 0, 0, 0, 1047, 1022, 1036, 1048, 0, 0, 0, 0, 981, 1068, 1033, 1009, 4, 2, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4069, 4025, 4171, 4140, 4081, 4153, 4097] avg=4096.0000 max=4171 max_rank=3 min=4025 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[998, 1020, 1009, 1032, 4, 0, 0, 0, 1016, 1028, 1038, 1087, 0, 0, 0, 0, 998, 1044, 974, 1020, 27, 0, 0, 0, 1039, 1034, 1022, 1008, 0, 0, 0, 0, 1028, 1009, 1015, 1037, 7, 0, 0, 0, 1004, 1055, 1007, 1012, 0, 0, 0, 0, 1040, 1036, 1011, 1041, 0, 0, 0, 0, 995, 1048, 1009, 1012, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4169, 4063, 4103, 4096, 4078, 4128, 4068] avg=4096.0000 max=4169 max_rank=1 min=4063 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[994, 1062, 1042, 1030, 4, 0, 0, 0, 1046, 1055, 1001, 1046, 0, 0, 0, 0, 1003, 1016, 999, 1067, 25, 0, 0, 0, 1051, 1030, 1094, 953, 0, 0, 0, 0, 1004, 974, 1063, 1037, 6, 0, 0, 0, 1002, 1033, 1026, 1034, 0, 0, 0, 0, 1071, 1018, 1033, 977, 0, 0, 0, 0, 967, 1019, 1003, 979, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4148, 4110, 4128, 4084, 4095, 4099, 3972] avg=4096.0000 max=4148 max_rank=1 min=3972 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1017, 1063, 1025, 968, 4, 0, 0, 0, 1018, 1003, 1031, 996, 0, 0, 0, 0, 1050, 1012, 1044, 1038, 26, 0, 0, 0, 1007, 1014, 1036, 992, 0, 0, 0, 0, 1025, 1058, 1039, 1032, 7, 0, 0, 0, 989, 1061, 1025, 996, 0, 0, 0, 0, 1016, 1020, 985, 990, 0, 0, 0, 0, 1058, 1021, 1084, 1014, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4048, 4170, 4049, 4161, 4071, 4011, 4181] avg=4096.0000 max=4181 max_rank=7 min=4011 min_rank=6 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[984, 1066, 1001, 1068, 4, 0, 0, 0, 1059, 1000, 1031, 1000, 0, 0, 0, 0, 1024, 1031, 1001, 1043, 27, 0, 0, 0, 1058, 1064, 960, 1005, 0, 0, 0, 0, 997, 1010, 978, 1039, 7, 0, 0, 0, 1001, 1003, 1075, 1041, 0, 0, 0, 0, 1058, 1007, 993, 1023, 0, 0, 0, 0, 1017, 1029, 1025, 1035, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4090, 4126, 4087, 4031, 4120, 4081, 4110] avg=4096.0000 max=4126 max_rank=2 min=4031 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8448, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8064, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1033, 969, 1005, 1052, 1041, 1025, 983, 1041, 1067, 1045, 974, 968, 1025, 997, 1012, 1050, 1002, 1042, 1034, 1068, 1002, 1009, 1004, 1025, 1034, 994, 1040, 1044, 1060, 1014, 1085, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1036, 1047, 1055, 996, 1031, 1060, 1039, 1003, 1027, 1034, 1002, 1058, 1026, 1007, 1025, 998, 989, 1034, 1043, 973, 1046, 1000, 1009, 987, 1041, 1031, 1065, 1010, 1031, 1044, 1015, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1049, 1075, 1040, 1076, 1043, 1026, 1001, 1035, 989, 1016, 1020, 1022, 999, 1089, 1026, 997, 1034, 1012, 1061, 998, 1030, 1034, 1050, 999, 1025, 987, 999, 971, 1023, 990, 1043, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1062, 1058, 989, 1031, 1006, 1031, 981, 1017, 1047, 1043, 1022, 1022, 1060, 1032, 1064, 1016, 1005, 1002, 984, 1023, 1051, 1021, 1009, 1033, 1025, 1012, 1037, 1011, 1008, 1027, 998, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[999, 1008, 1057, 1033, 1025, 1020, 1015, 1034, 1106, 960, 1031, 1028, 1038, 997, 1015, 978, 1051, 1017, 1059, 1028, 1067, 1010, 996, 1005, 1002, 987, 1080, 1021, 1048, 998, 1029, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[997, 1021, 1046, 982, 1004, 1092, 1043, 1028, 1098, 1016, 1059, 999, 983, 1030, 1002, 1044, 1028, 1039, 1040, 977, 1019, 1050, 996, 1033, 1015, 1004, 1054, 1001, 1014, 1020, 1023, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1031, 1001, 1030, 1061, 1054, 1035, 1010, 1017, 1030, 1021, 1029, 1007, 1006, 1021, 1008, 1027, 1019, 982, 1030, 1041, 988, 1040, 1001, 992, 1043, 1024, 1076, 1005, 1070, 1030, 996, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1031, 1021, 976, 1013, 1056, 984, 988, 1019, 1040, 1076, 1021, 1041, 1069, 1040, 1022, 1022, 1029, 1029, 1049, 985, 1089, 1042, 998, 994, 995, 996, 987, 1043, 1028, 1019, 1042, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1036, 1047, 1055, 984, 0, 0, 0, 0, 1031, 1058, 1039, 1003, 0, 0, 0, 0, 1010, 1034, 1002, 1058, 0, 0, 0, 0, 1026, 1007, 1025, 998, 2, 2, 0, 0, 989, 1034, 1043, 973, 5, 1, 0, 0, 1046, 1000, 1009, 987, 16, 0, 0, 0, 1041, 1031, 1065, 1010, 11, 1, 0, 0, 1024, 1044, 1015, 1006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4131, 4104, 4060, 4045, 4058, 4159, 4089] avg=4096.0000 max=4159 max_rank=6 min=4045 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[997, 1021, 1046, 970, 0, 0, 0, 0, 1004, 1090, 1043, 1028, 0, 0, 0, 0, 1079, 1016, 1059, 999, 0, 0, 0, 0, 983, 1030, 1002, 1044, 2, 2, 0, 0, 1028, 1039, 1040, 977, 5, 1, 0, 0, 1019, 1050, 996, 1033, 18, 0, 0, 0, 1015, 1004, 1054, 1001, 11, 1, 0, 0, 1007, 1020, 1023, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4165, 4153, 4063, 4090, 4116, 4086, 4061] avg=4096.0000 max=4165 max_rank=1 min=4034 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1049, 1075, 1040, 1063, 0, 0, 0, 0, 1043, 1024, 1001, 1035, 0, 0, 0, 0, 972, 1016, 1020, 1022, 0, 0, 0, 0, 999, 1089, 1026, 997, 2, 2, 0, 0, 1034, 1012, 1061, 998, 5, 1, 0, 0, 1030, 1034, 1050, 999, 16, 0, 0, 0, 1025, 987, 999, 971, 12, 1, 0, 0, 1016, 990, 1043, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4227, 4103, 4030, 4115, 4111, 4129, 3995, 4058] avg=4096.0000 max=4227 max_rank=0 min=3995 min_rank=6 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[999, 1008, 1057, 1020, 0, 0, 0, 0, 1025, 1018, 1015, 1034, 0, 0, 0, 0, 1087, 960, 1031, 1028, 0, 0, 0, 0, 1038, 997, 1015, 978, 3, 2, 0, 0, 1051, 1017, 1059, 1028, 5, 1, 0, 0, 1067, 1010, 996, 1005, 18, 0, 0, 0, 1002, 987, 1080, 1021, 12, 1, 0, 0, 1040, 998, 1029, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4092, 4106, 4033, 4161, 4096, 4103, 4093] avg=4096.0000 max=4161 max_rank=4 min=4033 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1033, 969, 1005, 1039, 0, 0, 0, 0, 1041, 1023, 983, 1041, 0, 0, 0, 0, 1049, 1045, 974, 968, 0, 0, 0, 0, 1025, 997, 1012, 1050, 3, 2, 0, 0, 1002, 1042, 1034, 1068, 6, 1, 0, 0, 1002, 1009, 1004, 1025, 17, 0, 0, 0, 1034, 994, 1040, 1044, 12, 1, 0, 0, 1051, 1014, 1085, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4088, 4036, 4089, 4153, 4057, 4125, 4174] avg=4096.0000 max=4174 max_rank=7 min=4036 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1031, 1001, 1030, 1048, 0, 0, 0, 0, 1054, 1033, 1010, 1017, 0, 0, 0, 0, 1012, 1021, 1029, 1007, 0, 0, 0, 0, 1006, 1021, 1008, 1027, 3, 2, 0, 0, 1019, 982, 1030, 1041, 6, 1, 0, 0, 988, 1040, 1001, 992, 17, 0, 0, 0, 1043, 1024, 1076, 1005, 12, 1, 0, 0, 1061, 1030, 996, 1043, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4114, 4069, 4067, 4079, 4038, 4161, 4130] avg=4096.0000 max=4161 max_rank=6 min=4038 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1031, 1021, 976, 1000, 0, 0, 0, 0, 1056, 982, 988, 1019, 0, 0, 0, 0, 1022, 1076, 1021, 1041, 0, 0, 0, 0, 1069, 1040, 1022, 1022, 2, 2, 0, 0, 1029, 1029, 1049, 985, 5, 1, 0, 0, 1089, 1042, 998, 994, 17, 0, 0, 0, 995, 996, 987, 1043, 12, 1, 0, 0, 1021, 1019, 1042, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4045, 4160, 4157, 4098, 4140, 4034, 4106] avg=4096.0000 max=4160 max_rank=2 min=4028 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1062, 1058, 989, 1008, 0, 0, 0, 0, 1006, 1026, 981, 1017, 0, 0, 0, 0, 1023, 1043, 1022, 1022, 0, 0, 0, 0, 1060, 1032, 1064, 1016, 7, 5, 0, 0, 1005, 1002, 984, 1023, 10, 7, 0, 0, 1051, 1021, 1009, 1033, 20, 0, 0, 0, 1025, 1012, 1037, 1011, 16, 4, 0, 0, 991, 1027, 998, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4030, 4110, 4184, 4031, 4134, 4105, 4057] avg=4096.0000 max=4184 max_rank=3 min=4030 min_rank=1 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8448, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1007, 1020, 1084, 1019, 1028, 1029, 1022, 1089, 1008, 1025, 1072, 1002, 1030, 1046, 1033, 1028, 1019, 978, 1008, 1051, 997, 1016, 1007, 1044, 1039, 1021, 1001, 1065, 994, 991, 1004, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1029, 1010, 1044, 1052, 1051, 1028, 1020, 1019, 1004, 1039, 1003, 1002, 1042, 1019, 1040, 1044, 1007, 995, 990, 1046, 1002, 1029, 1055, 1028, 1020, 1058, 1005, 1004, 1067, 986, 1011, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1064, 1044, 1020, 1006, 1029, 981, 1022, 1050, 1024, 1049, 991, 1022, 1012, 1022, 1050, 1030, 1002, 1048, 1013, 961, 1000, 1068, 994, 1029, 1069, 1036, 1108, 986, 1038, 1044, 979, 977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[973, 985, 1024, 1019, 1016, 1081, 1044, 1029, 1026, 1048, 1022, 921, 1040, 1035, 1035, 1052, 1040, 1008, 1052, 1016, 1014, 1008, 1055, 1039, 1042, 1009, 961, 983, 1079, 1055, 1046, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1021, 980, 1077, 998, 1030, 1047, 1021, 1024, 997, 1019, 1062, 1001, 1015, 1010, 1002, 1018, 985, 1026, 1059, 1078, 1049, 1033, 1018, 1037, 985, 1056, 960, 1037, 1028, 1033, 1034, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[996, 1058, 1083, 993, 1040, 989, 1023, 1031, 1032, 1019, 1014, 1029, 1018, 977, 1035, 1081, 1050, 1005, 1050, 1071, 1016, 1023, 972, 1023, 1032, 1018, 963, 1055, 1008, 1024, 1025, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1022, 1029, 1001, 1017, 1012, 1073, 1025, 965, 1048, 1041, 1034, 988, 1010, 1038, 1097, 998, 1004, 1039, 1007, 1034, 990, 1008, 1040, 1073, 1032, 1029, 1006, 1043, 987, 1055, 1020, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1013, 1011, 1038, 977, 1053, 1014, 1034, 1011, 1004, 1032, 1035, 1015, 1062, 1023, 1018, 1017, 1031, 1028, 1001, 1007, 1014, 1063, 1069, 1055, 997, 1075, 1017, 1021, 986, 1026, 1011, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1021, 980, 1077, 998, 6, 0, 0, 0, 1011, 1047, 1021, 1024, 0, 0, 0, 0, 997, 1019, 1062, 1001, 14, 3, 0, 0, 1015, 1010, 977, 1018, 0, 0, 0, 0, 985, 1026, 1059, 1078, 5, 1, 0, 0, 1049, 1033, 1018, 1026, 0, 0, 0, 0, 985, 1056, 960, 1037, 4, 0, 0, 0, 1028, 1033, 1034, 1028, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4103, 4096, 4020, 4154, 4126, 4042, 4145] avg=4096.0000 max=4154 max_rank=4 min=4020 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1007, 1020, 1084, 1019, 6, 0, 0, 0, 1009, 1029, 1022, 1089, 0, 0, 0, 0, 1008, 1025, 1072, 1002, 14, 3, 0, 0, 1030, 1046, 1008, 1028, 0, 0, 0, 0, 1019, 978, 1008, 1051, 5, 1, 0, 0, 997, 1016, 1007, 1033, 0, 0, 0, 0, 1039, 1021, 1001, 1065, 4, 0, 0, 0, 994, 991, 1004, 991, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4149, 4124, 4112, 4062, 4053, 4130, 4002] avg=4096.0000 max=4149 max_rank=1 min=4002 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16247 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1029, 1010, 1044, 1052, 6, 0, 0, 0, 1031, 1028, 1020, 1019, 0, 0, 0, 0, 1004, 1039, 1003, 1002, 14, 3, 0, 0, 1042, 1019, 1015, 1044, 0, 0, 0, 0, 1007, 995, 990, 1046, 6, 1, 0, 0, 1002, 1029, 1055, 1017, 0, 0, 0, 0, 1020, 1058, 1005, 1004, 4, 0, 0, 0, 1067, 986, 1011, 1019, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4098, 4065, 4120, 4045, 4103, 4091, 4105] avg=4096.0000 max=4141 max_rank=0 min=4045 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1013, 1011, 1038, 977, 12, 0, 0, 0, 1025, 1014, 1034, 1011, 0, 0, 0, 0, 1004, 1032, 1035, 1015, 17, 5, 0, 0, 1062, 1023, 987, 1017, 0, 0, 0, 0, 1031, 1028, 1001, 1007, 11, 4, 0, 0, 1014, 1063, 1069, 1032, 0, 0, 0, 0, 997, 1075, 1017, 1021, 7, 0, 0, 0, 986, 1026, 1011, 1010, 26, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4084, 4108, 4089, 4082, 4178, 4117, 4059] avg=4096.0000 max=4178 max_rank=5 min=4051 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[973, 985, 1024, 1019, 6, 0, 0, 0, 997, 1081, 1044, 1029, 0, 0, 0, 0, 1026, 1048, 1022, 921, 14, 3, 0, 0, 1040, 1035, 1010, 1052, 0, 0, 0, 0, 1040, 1008, 1052, 1016, 5, 1, 0, 0, 1014, 1008, 1055, 1028, 0, 0, 0, 0, 1042, 1009, 961, 983, 4, 0, 0, 0, 1079, 1055, 1046, 1011, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4151, 4034, 4137, 4122, 4105, 3999, 4213] avg=4096.0000 max=4213 max_rank=7 min=3999 min_rank=6 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1022, 1029, 1001, 1017, 6, 0, 0, 0, 994, 1073, 1025, 965, 0, 0, 0, 0, 1048, 1041, 1034, 988, 13, 3, 0, 0, 1010, 1038, 1070, 998, 0, 0, 0, 0, 1004, 1039, 1007, 1034, 5, 1, 0, 0, 990, 1008, 1040, 1062, 0, 0, 0, 0, 1032, 1029, 1006, 1043, 4, 0, 0, 0, 987, 1055, 1020, 1003, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4057, 4127, 4116, 4090, 4100, 4114, 4089] avg=4096.0000 max=4127 max_rank=2 min=4057 min_rank=1 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[996, 1058, 1083, 993, 6, 0, 0, 0, 1020, 989, 1023, 1031, 0, 0, 0, 0, 1032, 1019, 1014, 1029, 14, 3, 0, 0, 1018, 977, 1010, 1081, 0, 0, 0, 0, 1050, 1005, 1050, 1071, 6, 1, 0, 0, 1016, 1023, 972, 1012, 0, 0, 0, 0, 1032, 1018, 963, 1055, 4, 0, 0, 0, 1008, 1024, 1025, 1015, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4063, 4111, 4086, 4183, 4023, 4072, 4094] avg=4096.0000 max=4183 max_rank=4 min=4023 min_rank=5 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1064, 1044, 1020, 1006, 6, 0, 0, 0, 1010, 981, 1022, 1050, 0, 0, 0, 0, 1024, 1049, 991, 1022, 14, 3, 0, 0, 1012, 1022, 1024, 1030, 0, 0, 0, 0, 1002, 1048, 1013, 961, 5, 1, 0, 0, 1000, 1068, 994, 1018, 0, 0, 0, 0, 1069, 1036, 1108, 986, 4, 0, 0, 0, 1038, 1044, 979, 977, 23, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4063, 4103, 4088, 4030, 4080, 4203, 4061] avg=4096.0000 max=4203 max_rank=6 min=4030 min_rank=4 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8064, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8448, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[996, 1138, 1045, 1032, 1032, 1011, 996, 1069, 1009, 974, 1056, 1034, 992, 969, 1077, 1006, 1009, 1057, 1051, 995, 1037, 997, 1040, 1015, 1035, 987, 958, 1025, 1043, 1050, 1003, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1041, 1035, 1020, 1045, 1044, 1018, 979, 1053, 993, 1045, 1037, 1050, 1033, 1021, 1002, 1024, 1063, 1000, 1012, 990, 1064, 1019, 1031, 972, 1008, 1016, 1000, 1018, 1053, 1081, 1016, 985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[964, 1003, 1094, 997, 1042, 974, 1026, 1043, 1022, 1024, 1047, 1015, 1053, 1057, 1007, 1030, 1022, 1045, 1034, 1033, 1041, 1000, 1041, 1068, 965, 1005, 990, 1025, 1011, 1024, 1045, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1026, 1055, 1029, 1029, 992, 1026, 998, 1070, 1038, 970, 1021, 1021, 1066, 981, 1042, 1026, 1022, 1020, 1029, 1014, 1022, 1014, 1047, 960, 996, 1014, 1053, 1051, 1069, 1019, 983, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1024, 1031, 1032, 1033, 1040, 1059, 988, 1040, 1023, 1010, 1022, 1058, 1033, 1010, 1051, 1082, 979, 1012, 993, 1038, 1024, 996, 1025, 959, 989, 1024, 994, 1070, 1013, 1053, 1015, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1095, 1019, 1041, 1028, 1021, 1000, 1019, 1041, 984, 996, 1042, 1001, 961, 1051, 1043, 1024, 973, 1050, 1035, 987, 971, 1055, 1033, 1016, 1060, 989, 1034, 1066, 1064, 975, 1066, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1049, 1016, 999, 1060, 1003, 1004, 1002, 962, 1047, 1018, 1032, 1002, 993, 1048, 1026, 1090, 984, 1044, 1072, 998, 986, 1057, 1019, 958, 1057, 1031, 993, 1019, 1028, 1049, 1085, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1050, 1090, 1001, 1020, 1043, 994, 1009, 1019, 1005, 1055, 1063, 1059, 1013, 1000, 979, 1018, 1037, 994, 1068, 1027, 1082, 1045, 1018, 1028, 959, 1018, 1031, 983, 1054, 994, 953, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1049, 973, 999, 1060, 0, 0, 0, 0, 1003, 1004, 1002, 962, 10, 7, 0, 0, 1047, 1018, 1032, 1002, 0, 0, 0, 0, 993, 1048, 1026, 1085, 0, 0, 0, 0, 984, 1044, 1072, 998, 5, 4, 0, 0, 986, 1057, 1019, 958, 15, 0, 0, 0, 1057, 1031, 993, 1019, 36, 0, 0, 0, 999, 1049, 1085, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 3988, 4099, 4152, 4107, 4035, 4136, 4170] avg=4096.0000 max=4170 max_rank=7 min=3988 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1095, 975, 1041, 1028, 0, 0, 0, 0, 1021, 1000, 1019, 1041, 11, 7, 0, 0, 984, 996, 1042, 1001, 0, 0, 0, 0, 961, 1051, 1043, 1020, 0, 0, 0, 0, 973, 1050, 1035, 987, 4, 4, 0, 0, 971, 1055, 1033, 1016, 16, 0, 0, 0, 1060, 989, 1034, 1066, 37, 0, 0, 0, 1033, 975, 1066, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4099, 4023, 4075, 4053, 4091, 4186, 4102] avg=4096.0000 max=4186 max_rank=6 min=4023 min_rank=2 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1041, 991, 1020, 1045, 0, 0, 0, 0, 1044, 1018, 979, 1053, 10, 7, 0, 0, 993, 1045, 1037, 1050, 0, 0, 0, 0, 1033, 1021, 1002, 1020, 0, 0, 0, 0, 1063, 1000, 1012, 990, 4, 4, 0, 0, 1064, 1019, 1031, 972, 16, 0, 0, 0, 1008, 1016, 1000, 1018, 37, 0, 0, 0, 1023, 1081, 1016, 985, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4111, 4125, 4076, 4073, 4102, 4079, 4105] avg=4096.0000 max=4125 max_rank=2 min=4073 min_rank=4 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[996, 1089, 1045, 1032, 0, 0, 0, 0, 1032, 1011, 996, 1069, 10, 8, 0, 0, 1009, 974, 1056, 1034, 0, 0, 0, 0, 992, 969, 1077, 1002, 0, 0, 0, 0, 1009, 1057, 1051, 995, 4, 4, 0, 0, 1037, 997, 1040, 1015, 16, 0, 0, 0, 1035, 987, 958, 1025, 41, 0, 0, 0, 1013, 1050, 1003, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4126, 4073, 4040, 4120, 4105, 4046, 4096] avg=4096.0000 max=4162 max_rank=0 min=4040 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1050, 1043, 1001, 1020, 0, 0, 0, 0, 1043, 994, 1009, 1019, 11, 8, 0, 0, 1005, 1055, 1063, 1059, 0, 0, 0, 0, 1013, 1000, 979, 1014, 0, 0, 0, 0, 1037, 994, 1068, 1027, 4, 4, 0, 0, 1082, 1045, 1018, 1028, 16, 0, 0, 0, 959, 1018, 1031, 983, 39, 0, 0, 0, 1023, 994, 953, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4084, 4182, 4006, 4134, 4189, 4030, 4029] avg=4096.0000 max=4189 max_rank=5 min=4006 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 1009, 1029, 1029, 0, 0, 0, 0, 992, 1026, 998, 1070, 11, 8, 0, 0, 1038, 970, 1021, 1021, 0, 0, 0, 0, 1066, 981, 1042, 1022, 0, 0, 0, 0, 1022, 1020, 1029, 1014, 4, 4, 0, 0, 1022, 1014, 1047, 960, 16, 0, 0, 0, 996, 1014, 1053, 1051, 38, 0, 0, 0, 1038, 1019, 983, 1065, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4105, 4050, 4111, 4093, 4059, 4152, 4105] avg=4096.0000 max=4152 max_rank=6 min=4050 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 978, 1032, 1033, 0, 0, 0, 0, 1040, 1059, 988, 1040, 14, 12, 0, 0, 1023, 1010, 1017, 1058, 0, 0, 0, 0, 1033, 1010, 1051, 1071, 0, 0, 0, 0, 979, 1012, 993, 1038, 11, 8, 5, 0, 1024, 996, 1025, 959, 18, 0, 0, 0, 989, 1024, 994, 1070, 41, 0, 0, 0, 973, 1053, 1015, 1048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4153, 4108, 4165, 4046, 4022, 4118, 4089] avg=4096.0000 max=4165 max_rank=3 min=4022 min_rank=5 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[964, 960, 1094, 997, 0, 0, 0, 0, 1042, 974, 1026, 1043, 10, 7, 0, 0, 1022, 1024, 1047, 1015, 0, 0, 0, 0, 1053, 1057, 1007, 1026, 0, 0, 0, 0, 1022, 1045, 1034, 1033, 4, 4, 0, 0, 1041, 1000, 1041, 1068, 15, 0, 0, 0, 965, 1005, 990, 1025, 36, 0, 0, 0, 982, 1024, 1045, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4102, 4108, 4143, 4142, 4165, 4021, 4072] avg=4096.0000 max=4165 max_rank=5 min=4015 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8064, 8320, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1048, 990, 1033, 1002, 1034, 998, 1025, 998, 1046, 1011, 1016, 1023, 987, 1035, 1006, 1002, 1024, 1039, 1007, 1012, 1046, 1024, 1028, 1069, 1048, 1031, 1012, 1031, 1063, 1043, 1018, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1013, 1016, 1046, 988, 1048, 1002, 1046, 1022, 1039, 1022, 983, 1021, 1026, 1009, 992, 1018, 1055, 1000, 1070, 1043, 1052, 1023, 1019, 1015, 1013, 1001, 1035, 1036, 1013, 1036, 1022, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1037, 1045, 1000, 988, 1016, 1061, 987, 1003, 1031, 1071, 1036, 1015, 990, 1037, 997, 1011, 1015, 989, 1054, 1019, 1020, 1075, 984, 1053, 1057, 989, 1035, 1056, 991, 1071, 1022, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1015, 1005, 1035, 1101, 1032, 1038, 1062, 987, 987, 1033, 1037, 967, 1001, 1071, 1023, 1031, 989, 1064, 1024, 1032, 1037, 1029, 1015, 1067, 1031, 995, 1003, 1021, 1043, 999, 988, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1012, 1009, 1010, 1054, 1046, 987, 992, 1028, 987, 1011, 960, 1006, 980, 975, 1024, 1063, 1042, 1036, 996, 1068, 1087, 1004, 993, 1018, 1025, 1057, 1053, 1053, 1047, 1060, 1035, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1011, 1021, 1050, 1011, 1069, 1012, 1013, 1024, 1053, 1021, 1010, 1028, 1046, 1039, 1025, 1035, 1003, 995, 1035, 1007, 1061, 1029, 1012, 1065, 1020, 1031, 1018, 1035, 980, 997, 993, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1034, 985, 1034, 1004, 1057, 1041, 1024, 994, 1043, 1075, 1046, 1052, 1032, 1060, 1033, 1013, 1016, 1002, 1012, 1021, 982, 996, 1074, 991, 1012, 1015, 983, 1045, 1029, 1071, 981, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1068, 1024, 1031, 1004, 1043, 1007, 975, 1016, 986, 978, 1043, 1046, 1002, 993, 1008, 1046, 1044, 1041, 1036, 999, 1047, 1054, 1035, 1003, 1010, 1004, 1015, 1038, 1027, 1071, 1025, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1037, 1045, 1000, 988, 2, 2, 0, 0, 1016, 1061, 987, 1003, 8, 2, 0, 0, 1031, 1071, 1036, 1015, 9, 0, 0, 0, 990, 1037, 997, 1011, 19, 0, 0, 0, 1015, 989, 1052, 1019, 0, 0, 0, 0, 992, 1075, 984, 1053, 0, 0, 0, 0, 1057, 989, 1035, 1052, 0, 0, 0, 0, 991, 1063, 1022, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4077, 4162, 4054, 4075, 4104, 4133, 4089] avg=4096.0000 max=4162 max_rank=2 min=4054 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1048, 990, 1033, 1002, 2, 2, 0, 0, 1034, 998, 1025, 998, 7, 2, 0, 0, 1046, 1011, 1016, 1023, 10, 0, 0, 0, 987, 1035, 1006, 1002, 19, 0, 0, 0, 1024, 1039, 1005, 1012, 0, 0, 0, 0, 1017, 1024, 1028, 1069, 0, 0, 0, 0, 1048, 1031, 1012, 1027, 0, 0, 0, 0, 1063, 1036, 1018, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4064, 4106, 4049, 4080, 4138, 4118, 4136] avg=4096.0000 max=4138 max_rank=5 min=4049 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1015, 1005, 1035, 1101, 9, 7, 0, 0, 1032, 1038, 1062, 987, 11, 3, 0, 0, 987, 1033, 1037, 967, 13, 4, 0, 0, 1001, 1071, 1023, 1031, 24, 0, 0, 0, 989, 1064, 1017, 1032, 0, 0, 0, 0, 1000, 1029, 1015, 1067, 0, 0, 0, 0, 1031, 995, 1003, 1009, 0, 0, 0, 0, 1043, 984, 988, 1006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4133, 4041, 4150, 4102, 4111, 4038, 4021] avg=4096.0000 max=4172 max_rank=0 min=4021 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1013, 1016, 1046, 988, 2, 2, 0, 0, 1048, 1002, 1046, 1022, 7, 2, 0, 0, 1039, 1022, 983, 1021, 10, 0, 0, 0, 1026, 1009, 992, 1018, 19, 0, 0, 0, 1055, 1000, 1068, 1043, 0, 0, 0, 0, 1023, 1023, 1019, 1015, 0, 0, 0, 0, 1013, 1001, 1035, 1032, 0, 0, 0, 0, 1013, 1029, 1022, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4127, 4075, 4064, 4166, 4080, 4081, 4108] avg=4096.0000 max=4166 max_rank=4 min=4064 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1034, 985, 1034, 1004, 2, 2, 0, 0, 1057, 1041, 1024, 994, 8, 2, 0, 0, 1043, 1075, 1046, 1052, 9, 0, 0, 0, 1032, 1060, 1033, 1013, 18, 0, 0, 0, 1016, 1002, 1010, 1021, 0, 0, 0, 0, 955, 996, 1074, 991, 0, 0, 0, 0, 1012, 1015, 983, 1041, 0, 0, 0, 0, 1029, 1063, 981, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4126, 4225, 4156, 4049, 4016, 4051, 4084] avg=4096.0000 max=4225 max_rank=2 min=4016 min_rank=5 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16200 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1011, 1021, 1050, 1011, 2, 2, 0, 0, 1069, 1012, 1013, 1024, 7, 2, 0, 0, 1053, 1021, 1010, 1028, 10, 0, 0, 0, 1046, 1039, 1025, 1035, 20, 0, 0, 0, 1003, 995, 1033, 1007, 0, 0, 0, 0, 1031, 1029, 1012, 1065, 0, 0, 0, 0, 1020, 1031, 1018, 1031, 0, 0, 0, 0, 980, 990, 993, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4127, 4122, 4165, 4038, 4137, 4100, 3982] avg=4096.0000 max=4165 max_rank=3 min=3982 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1068, 1024, 1031, 1004, 2, 2, 0, 0, 1043, 1007, 975, 1016, 8, 2, 0, 0, 986, 978, 1043, 1046, 10, 0, 0, 0, 1002, 993, 1008, 1046, 19, 0, 0, 0, 1044, 1041, 1034, 999, 0, 0, 0, 0, 1018, 1054, 1035, 1003, 0, 0, 0, 0, 1010, 1004, 1015, 1034, 0, 0, 0, 0, 1027, 1063, 1025, 1049, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4051, 4063, 4068, 4118, 4110, 4063, 4164] avg=4096.0000 max=4164 max_rank=7 min=4051 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1012, 1009, 1010, 1054, 2, 2, 0, 0, 1046, 987, 992, 1028, 8, 2, 0, 0, 987, 1011, 960, 1006, 10, 0, 0, 0, 980, 975, 1024, 1063, 20, 0, 0, 0, 1042, 1036, 994, 1068, 0, 0, 0, 0, 1057, 1004, 993, 1018, 0, 0, 0, 0, 1025, 1057, 1053, 1049, 0, 0, 0, 0, 1047, 1052, 1035, 1050, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4063, 3974, 4062, 4140, 4072, 4184, 4184] avg=4096.0000 max=4184 max_rank=6 min=3974 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8448, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1017, 1045, 1045, 1002, 1019, 992, 988, 1049, 1033, 1067, 1082, 1001, 1037, 1065, 1044, 1031, 1023, 1006, 1019, 1018, 1009, 988, 1002, 1000, 1011, 1028, 1001, 1009, 1055, 1030, 1025, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1004, 1015, 992, 1028, 1041, 1015, 1032, 984, 1055, 1006, 1068, 1007, 970, 1021, 1024, 1066, 1041, 1067, 1005, 1030, 1022, 1045, 1066, 1031, 1005, 1022, 1034, 1022, 995, 1033, 1004, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1028, 1042, 1026, 1005, 1038, 1023, 1053, 1066, 1011, 977, 1043, 970, 1043, 1056, 967, 1107, 1008, 1030, 1013, 1021, 1026, 996, 1003, 1046, 1005, 1001, 1017, 1030, 1036, 1008, 1062, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1022, 1072, 1041, 1030, 1004, 1008, 1049, 1025, 1027, 1004, 963, 1027, 1030, 1073, 1044, 981, 1043, 995, 970, 1002, 1064, 1016, 1036, 1003, 1009, 1062, 1039, 1000, 1043, 988, 1045, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1017, 1025, 1043, 993, 1002, 1073, 1011, 1046, 1040, 940, 1031, 1018, 990, 1037, 1019, 1022, 1029, 1037, 1019, 1027, 1017, 1011, 1014, 1085, 1099, 1018, 995, 1012, 977, 1060, 1019, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[991, 1050, 1013, 1046, 1023, 985, 1035, 1073, 1007, 1016, 990, 1005, 1038, 1056, 964, 1005, 989, 1039, 1062, 1003, 1006, 1013, 999, 1057, 1070, 1044, 1073, 1006, 1042, 1006, 1029, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1047, 1013, 1028, 1008, 1034, 1048, 981, 1048, 1035, 1012, 981, 1050, 1033, 1041, 997, 1016, 1059, 1048, 1045, 1004, 1046, 1012, 1010, 1042, 1034, 1011, 995, 969, 1072, 966, 1040, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[996, 1019, 1044, 1034, 1048, 1041, 1038, 1005, 1033, 1023, 996, 1025, 1060, 984, 1002, 1022, 1036, 1013, 1026, 1064, 1032, 1008, 969, 1053, 1000, 1048, 1015, 996, 1046, 1008, 1031, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1004, 1014, 992, 1028, 0, 0, 0, 0, 1041, 1015, 1032, 972, 0, 0, 0, 0, 1055, 1006, 1068, 1007, 16, 11, 0, 0, 970, 1013, 1024, 1066, 0, 0, 0, 0, 1041, 1065, 1005, 1030, 0, 0, 0, 0, 1022, 1045, 1066, 1031, 2, 1, 0, 0, 1005, 1022, 1034, 1022, 8, 1, 0, 0, 995, 1033, 1004, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4060, 4163, 4073, 4141, 4167, 4092, 4034] avg=4096.0000 max=4167 max_rank=5 min=4034 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1028, 1041, 1026, 1005, 0, 0, 0, 0, 1038, 1023, 1053, 1053, 0, 0, 0, 0, 1011, 977, 1043, 970, 16, 11, 0, 0, 1043, 1047, 967, 1107, 0, 0, 0, 0, 1008, 1028, 1013, 1021, 0, 0, 0, 0, 1026, 996, 1003, 1046, 2, 1, 0, 0, 1005, 1001, 1017, 1030, 9, 2, 0, 0, 1036, 1008, 1062, 995, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4167, 4028, 4164, 4070, 4074, 4064, 4101] avg=4096.0000 max=4167 max_rank=1 min=4028 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1017, 1044, 1045, 1002, 0, 0, 0, 0, 1019, 992, 988, 1036, 0, 0, 0, 0, 1033, 1067, 1082, 1001, 16, 11, 0, 0, 1037, 1056, 1044, 1031, 0, 0, 0, 0, 1023, 1004, 1019, 1018, 0, 0, 0, 0, 1009, 988, 1002, 1000, 2, 1, 0, 0, 1011, 1028, 1001, 1009, 9, 2, 0, 0, 1055, 1030, 1025, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4035, 4210, 4168, 4064, 4002, 4060, 4121] avg=4096.0000 max=4210 max_rank=2 min=4002 min_rank=5 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16247 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1022, 1071, 1041, 1030, 0, 0, 0, 0, 1004, 1008, 1049, 1013, 0, 0, 0, 0, 1027, 1004, 963, 1027, 16, 11, 0, 0, 1030, 1064, 1044, 981, 0, 0, 0, 0, 1043, 993, 970, 1002, 0, 0, 0, 0, 1064, 1016, 1036, 1003, 2, 1, 0, 0, 1009, 1062, 1039, 1000, 9, 1, 0, 0, 1043, 988, 1045, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4074, 4048, 4119, 4008, 4122, 4120, 4113] avg=4096.0000 max=4164 max_rank=0 min=4008 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1047, 1012, 1028, 1008, 0, 0, 0, 0, 1034, 1048, 981, 1035, 0, 0, 0, 0, 1035, 1012, 981, 1050, 16, 11, 0, 0, 1033, 1033, 997, 1016, 0, 0, 0, 0, 1059, 1046, 1045, 1004, 0, 0, 0, 0, 1046, 1012, 1010, 1042, 2, 1, 0, 0, 1034, 1011, 995, 969, 8, 2, 0, 0, 1072, 966, 1040, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4098, 4105, 4079, 4154, 4113, 4019, 4105] avg=4096.0000 max=4154 max_rank=4 min=4019 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1017, 1019, 1043, 993, 0, 0, 0, 0, 1002, 1073, 1011, 1026, 0, 0, 0, 0, 1040, 940, 1031, 1018, 20, 15, 0, 0, 990, 1020, 1019, 1022, 0, 0, 0, 0, 1029, 1028, 1019, 1027, 0, 0, 0, 0, 1017, 1011, 1014, 1085, 9, 6, 5, 0, 1099, 1018, 995, 1012, 12, 5, 0, 0, 977, 1060, 1019, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4112, 4064, 4051, 4103, 4147, 4141, 4078] avg=4096.0000 max=4147 max_rank=5 min=4051 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[991, 1049, 1013, 1046, 0, 0, 0, 0, 1023, 985, 1035, 1059, 0, 0, 0, 0, 1007, 1016, 990, 1005, 16, 12, 0, 0, 1038, 1047, 964, 1005, 0, 0, 0, 0, 989, 1037, 1062, 1003, 0, 0, 0, 0, 1006, 1013, 999, 1057, 2, 1, 0, 0, 1070, 1044, 1073, 1006, 9, 2, 0, 0, 1042, 1006, 1029, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4102, 4046, 4054, 4091, 4078, 4204, 4094] avg=4096.0000 max=4204 max_rank=6 min=4046 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[996, 1018, 1044, 1034, 0, 0, 0, 0, 1048, 1041, 1038, 993, 0, 0, 0, 0, 1033, 1023, 996, 1025, 16, 11, 0, 0, 1060, 976, 1002, 1022, 0, 0, 0, 0, 1036, 1011, 1026, 1064, 0, 0, 0, 0, 1032, 1008, 969, 1053, 2, 1, 0, 0, 1000, 1048, 1015, 996, 8, 1, 0, 0, 1046, 1008, 1031, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4120, 4104, 4060, 4137, 4065, 4068, 4122] avg=4096.0000 max=4137 max_rank=4 min=4060 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1025, 1029, 1117, 1054, 1018, 1051, 1004, 1020, 1046, 994, 1021, 1014, 1041, 1033, 1004, 1015, 1023, 993, 1005, 998, 1014, 1043, 986, 1026, 1052, 1025, 1016, 1025, 1025, 1033, 1043, 975] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1023, 978, 987, 984, 1057, 1045, 1031, 1048, 998, 988, 1006, 1037, 1020, 998, 1042, 1053, 1006, 972, 1061, 977, 1078, 1067, 1038, 1040, 1023, 1016, 1002, 1045, 1067, 1005, 1034, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1040, 993, 1054, 1068, 991, 991, 1045, 1026, 1045, 989, 1058, 982, 1027, 999, 1017, 1091, 987, 1025, 1059, 1055, 1025, 1011, 1024, 1019, 998, 1018, 977, 1033, 1000, 1048, 1030, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1010, 1015, 976, 1037, 1054, 1051, 1023, 1047, 975, 1026, 1039, 1052, 1058, 1047, 1046, 1038, 951, 1026, 997, 1050, 1036, 994, 1035, 996, 1014, 989, 1012, 1023, 1044, 1016, 1057, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[995, 1060, 1068, 1031, 991, 1027, 1036, 1014, 1057, 1060, 997, 1009, 1041, 1011, 1026, 1015, 976, 1013, 976, 1022, 1033, 992, 1005, 1026, 1056, 1071, 1060, 1039, 987, 987, 1047, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[990, 1025, 1066, 1062, 1033, 989, 1000, 1003, 1017, 1015, 1006, 996, 1042, 994, 1053, 990, 992, 1055, 994, 1043, 1024, 1034, 1059, 1025, 1061, 1047, 981, 1052, 1011, 1012, 1063, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1050, 1034, 1013, 982, 1027, 1037, 1020, 1011, 1036, 1023, 1047, 1025, 1044, 1013, 987, 1027, 1034, 1025, 976, 1042, 1011, 1013, 983, 1014, 1062, 1036, 1029, 1037, 1055, 1014, 1015, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1019, 1042, 1027, 1009, 1087, 1066, 1022, 1043, 1043, 1028, 1048, 974, 1021, 1054, 998, 1053, 1078, 940, 1023, 1016, 980, 1004, 1014, 977, 985, 1024, 986, 1040, 1048, 1068, 988, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[995, 1060, 1057, 1031, 0, 0, 0, 0, 975, 1027, 1036, 1014, 0, 0, 0, 0, 1057, 1060, 997, 1009, 8, 6, 0, 0, 1025, 1011, 1026, 1015, 0, 0, 0, 0, 976, 1013, 976, 1022, 25, 16, 4, 0, 1033, 992, 1005, 1026, 12, 5, 0, 0, 1056, 1071, 1060, 1031, 0, 0, 0, 0, 987, 987, 1047, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4052, 4137, 4077, 4032, 4073, 4218, 4036] avg=4096.0000 max=4218 max_rank=6 min=4032 min_rank=4 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1023, 978, 976, 984, 0, 0, 0, 0, 1040, 1045, 1031, 1048, 0, 0, 0, 0, 998, 988, 1006, 1037, 8, 6, 0, 0, 1005, 998, 1042, 1053, 0, 0, 0, 0, 1006, 972, 1061, 977, 25, 17, 3, 0, 1078, 1067, 1038, 1040, 12, 5, 0, 0, 1023, 1016, 1002, 1037, 0, 0, 0, 0, 1067, 1005, 1034, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3961, 4164, 4043, 4098, 4061, 4240, 4078, 4123] avg=4096.0000 max=4240 max_rank=5 min=3961 min_rank=0 imbalance_ratio=1.035156 max_load_reduction=28528 max_load_reduction_pct=0.870605 moved_to_echo=16502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1019, 1042, 1016, 1009, 0, 0, 0, 0, 1069, 1066, 1022, 1043, 0, 0, 0, 0, 1043, 1028, 1048, 974, 8, 6, 0, 0, 1006, 1054, 998, 1053, 0, 0, 0, 0, 1078, 940, 1023, 1016, 26, 18, 3, 0, 980, 1004, 1014, 977, 12, 5, 0, 0, 985, 1024, 986, 1032, 0, 0, 0, 0, 1048, 1068, 988, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4200, 4107, 4111, 4104, 3992, 4027, 4141] avg=4096.0000 max=4200 max_rank=1 min=3992 min_rank=5 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16264 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1010, 1015, 958, 1037, 0, 0, 0, 0, 1032, 1051, 1023, 1047, 0, 0, 0, 0, 975, 1026, 1039, 1052, 10, 9, 0, 0, 1037, 1047, 1046, 1038, 0, 0, 0, 0, 951, 1026, 997, 1050, 30, 22, 7, 0, 1036, 994, 1035, 996, 14, 9, 0, 0, 1014, 989, 1012, 1013, 0, 0, 0, 0, 1044, 1016, 1057, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4153, 4111, 4168, 4083, 4084, 4028, 4121] avg=4096.0000 max=4168 max_rank=3 min=4020 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1040, 993, 1043, 1068, 0, 0, 0, 0, 975, 991, 1045, 1026, 0, 0, 0, 0, 1045, 989, 1058, 982, 8, 6, 0, 0, 1012, 999, 1017, 1091, 0, 0, 0, 0, 987, 1025, 1059, 1055, 25, 16, 3, 0, 1025, 1011, 1024, 1019, 12, 5, 0, 0, 998, 1018, 977, 1025, 0, 0, 0, 0, 1000, 1048, 1030, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4037, 4088, 4119, 4170, 4096, 4018, 4096] avg=4096.0000 max=4170 max_rank=4 min=4018 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1025, 1029, 1106, 1054, 0, 0, 0, 0, 1001, 1051, 1004, 1020, 0, 0, 0, 0, 1046, 994, 1021, 1014, 8, 6, 0, 0, 1025, 1033, 1004, 1015, 0, 0, 0, 0, 1023, 993, 1005, 998, 24, 17, 4, 0, 1014, 1043, 986, 1026, 12, 5, 0, 0, 1052, 1025, 1016, 1017, 0, 0, 0, 0, 1025, 1033, 1043, 951, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4214, 4076, 4089, 4077, 4064, 4086, 4110, 4052] avg=4096.0000 max=4214 max_rank=0 min=4052 min_rank=7 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[990, 1025, 1055, 1062, 0, 0, 0, 0, 1016, 989, 1000, 1003, 0, 0, 0, 0, 1017, 1015, 1006, 996, 8, 6, 0, 0, 1026, 994, 1053, 990, 0, 0, 0, 0, 992, 1055, 994, 1043, 25, 17, 4, 0, 1024, 1034, 1059, 1025, 12, 5, 0, 0, 1061, 1047, 981, 1044, 0, 0, 0, 0, 1011, 1012, 1063, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4008, 4048, 4063, 4130, 4159, 4133, 4095] avg=4096.0000 max=4159 max_rank=5 min=4008 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1050, 1034, 1002, 982, 0, 0, 0, 0, 1010, 1037, 1020, 1011, 0, 0, 0, 0, 1036, 1023, 1047, 1025, 8, 6, 0, 0, 1028, 1013, 987, 1027, 0, 0, 0, 0, 1034, 1025, 976, 1042, 26, 17, 4, 0, 1011, 1013, 983, 1014, 12, 5, 0, 0, 1062, 1036, 1029, 1029, 0, 0, 0, 0, 1055, 1014, 1015, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4078, 4145, 4055, 4124, 4038, 4156, 4104] avg=4096.0000 max=4156 max_rank=6 min=4038 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8064, 8192, 8320, 256, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1035, 1015, 1070, 1013, 1071, 986, 1003, 1050, 972, 1057, 1078, 1022, 1013, 1035, 998, 989, 970, 996, 1008, 1030, 1061, 996, 1010, 1030, 1068, 1022, 1046, 1027, 1064, 1020, 1025, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1000, 1030, 1008, 1043, 1039, 1073, 1089, 1014, 1038, 1042, 972, 1027, 1031, 1056, 1036, 1044, 1003, 979, 1013, 1063, 995, 1000, 996, 995, 1021, 1011, 1000, 1004, 1042, 1043, 999, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1076, 1051, 980, 1027, 1015, 1002, 1000, 1108, 1087, 1024, 1056, 1039, 1017, 1040, 1039, 1005, 1008, 1028, 1006, 1032, 1007, 1034, 969, 1066, 1050, 987, 974, 979, 998, 1017, 1028, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[988, 990, 1042, 1008, 1004, 995, 1011, 1075, 1045, 1052, 1030, 1017, 1018, 1064, 975, 1066, 1069, 1001, 1007, 1015, 1017, 989, 1033, 1017, 1029, 1047, 1020, 1012, 997, 1059, 1031, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1012, 1048, 1043, 1029, 1041, 1043, 1017, 975, 1017, 1042, 1023, 1048, 1044, 1029, 1043, 1029, 1006, 955, 1019, 1031, 1029, 987, 1001, 1037, 1039, 1090, 1017, 1037, 1020, 987, 1003, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[981, 1071, 1050, 1083, 973, 1004, 1052, 987, 1053, 1026, 1034, 1042, 1058, 1033, 1026, 1031, 1030, 980, 1068, 971, 970, 1002, 987, 1034, 1019, 1009, 1010, 1002, 1054, 1057, 1030, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1027, 1016, 1018, 1010, 1041, 1041, 1029, 1031, 1001, 968, 1075, 1027, 1027, 1054, 1034, 1012, 988, 1073, 996, 1014, 1017, 1045, 985, 981, 1035, 999, 1056, 1023, 1010, 1075, 1032, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[967, 1013, 1067, 1033, 1048, 1023, 1069, 1019, 1009, 997, 1057, 1016, 976, 1045, 1013, 1021, 991, 1009, 1029, 1013, 1043, 981, 1040, 1017, 1047, 1021, 1015, 1061, 994, 1033, 1044, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1027, 1016, 1009, 1010, 0, 0, 0, 0, 1041, 1041, 1010, 1031, 0, 0, 0, 0, 1001, 968, 1046, 1027, 0, 0, 0, 0, 1027, 1038, 1034, 1012, 0, 0, 0, 0, 988, 1073, 996, 1014, 18, 16, 9, 1, 1017, 1045, 985, 981, 29, 20, 1, 0, 1034, 999, 1056, 1023, 0, 0, 0, 0, 1010, 1055, 1032, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4123, 4042, 4111, 4115, 4078, 4112, 4125] avg=4096.0000 max=4125 max_rank=7 min=4042 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1076, 1051, 972, 1027, 0, 0, 0, 0, 1015, 1002, 981, 1108, 0, 0, 0, 0, 1087, 1024, 1028, 1039, 0, 0, 0, 0, 1017, 1024, 1039, 1005, 0, 0, 0, 0, 1008, 1028, 1006, 1032, 18, 16, 8, 1, 1007, 1034, 969, 1066, 28, 19, 1, 0, 1049, 987, 974, 979, 0, 0, 0, 0, 998, 998, 1028, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4106, 4178, 4085, 4117, 4124, 3989, 4043] avg=4096.0000 max=4178 max_rank=2 min=3989 min_rank=6 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[967, 1013, 1058, 1033, 0, 0, 0, 0, 1048, 1023, 1049, 1019, 0, 0, 0, 0, 1009, 997, 1029, 1016, 0, 0, 0, 0, 976, 1029, 1013, 1021, 0, 0, 0, 0, 991, 1009, 1029, 1013, 19, 16, 9, 1, 1043, 981, 1040, 1017, 28, 20, 1, 0, 1046, 1021, 1015, 1061, 0, 0, 0, 0, 994, 1013, 1044, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4139, 4051, 4039, 4087, 4130, 4143, 4108] avg=4096.0000 max=4143 max_rank=6 min=4039 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1035, 1015, 1061, 1013, 0, 0, 0, 0, 1071, 986, 984, 1050, 0, 0, 0, 0, 972, 1057, 1049, 1022, 0, 0, 0, 0, 1013, 1019, 998, 989, 0, 0, 0, 0, 970, 996, 1008, 1030, 18, 16, 9, 1, 1061, 996, 1010, 1030, 29, 19, 1, 0, 1067, 1022, 1046, 1027, 0, 0, 0, 0, 1064, 1001, 1025, 988, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4091, 4100, 4019, 4048, 4146, 4162, 4078] avg=4096.0000 max=4162 max_rank=6 min=4019 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1000, 1030, 994, 1043, 0, 0, 0, 0, 1039, 1073, 1063, 1014, 0, 0, 0, 0, 1038, 1042, 942, 1027, 0, 0, 0, 0, 1031, 1035, 1036, 1044, 0, 0, 0, 0, 1003, 979, 1013, 1063, 22, 21, 14, 2, 995, 1000, 996, 995, 30, 24, 4, 0, 1019, 1011, 1000, 1004, 0, 0, 0, 0, 1042, 1019, 999, 1062, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4189, 4049, 4146, 4117, 4044, 4034, 4122] avg=4096.0000 max=4189 max_rank=1 min=4034 min_rank=6 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1012, 1048, 1034, 1029, 0, 0, 0, 0, 1041, 1043, 998, 975, 0, 0, 0, 0, 1017, 1042, 996, 1048, 0, 0, 0, 0, 1044, 1013, 1043, 1029, 0, 0, 0, 0, 1006, 955, 1019, 1031, 18, 16, 9, 1, 1029, 987, 1001, 1037, 27, 19, 1, 0, 1038, 1090, 1017, 1037, 0, 0, 0, 0, 1020, 968, 1003, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4057, 4103, 4129, 4055, 4101, 4182, 4018] avg=4096.0000 max=4182 max_rank=6 min=4018 min_rank=7 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[981, 1071, 1041, 1083, 0, 0, 0, 0, 973, 1004, 1033, 987, 0, 0, 0, 0, 1053, 1026, 1007, 1042, 0, 0, 0, 0, 1058, 1017, 1026, 1031, 0, 0, 0, 0, 1030, 980, 1068, 971, 18, 16, 9, 1, 970, 1002, 987, 1034, 27, 20, 1, 0, 1018, 1009, 1010, 1002, 0, 0, 0, 0, 1054, 1037, 1030, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 3997, 4128, 4132, 4093, 4041, 4039, 4162] avg=4096.0000 max=4176 max_rank=0 min=3997 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[988, 990, 1033, 1008, 0, 0, 0, 0, 1004, 995, 992, 1075, 0, 0, 0, 0, 1045, 1052, 1003, 1017, 0, 0, 0, 0, 1018, 1048, 975, 1066, 0, 0, 0, 0, 1069, 1001, 1007, 1015, 18, 16, 9, 1, 1017, 989, 1033, 1017, 27, 20, 1, 0, 1028, 1047, 1020, 1012, 0, 0, 0, 0, 997, 1039, 1031, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4066, 4117, 4107, 4136, 4104, 4107, 4112] avg=4096.0000 max=4136 max_rank=4 min=4019 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8064, 8192, 8192, 256, 256, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8064, 8192, 256, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1030, 1073, 1012, 1025, 1021, 1052, 1024, 1014, 1043, 1059, 1041, 1029, 1016, 1029, 957, 1042, 1048, 964, 1038, 1064, 1002, 983, 1016, 1010, 998, 1051, 1097, 1008, 1066, 989, 977, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[992, 1041, 1051, 1048, 1041, 1023, 1036, 968, 1041, 1046, 1024, 1022, 1041, 1022, 977, 1014, 973, 1057, 999, 955, 1051, 1036, 987, 1015, 1099, 1045, 1040, 980, 1051, 1048, 1032, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1041, 1030, 1024, 980, 1098, 1042, 1047, 1031, 1012, 1029, 1013, 1037, 1046, 1069, 1017, 1011, 1047, 1038, 1056, 993, 1057, 1011, 1017, 1005, 1027, 1057, 984, 1014, 951, 992, 989, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1029, 1064, 1044, 1043, 1011, 1018, 994, 1043, 1030, 1028, 1047, 1064, 1033, 1047, 1029, 996, 1029, 1017, 981, 1002, 974, 999, 977, 1060, 1058, 1022, 1009, 1019, 1025, 1024, 994, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1055, 1044, 1012, 1007, 1006, 983, 1044, 995, 1004, 1034, 1033, 1030, 1013, 1087, 1014, 1017, 1029, 979, 1006, 1020, 1017, 1009, 1054, 1037, 1028, 1068, 1007, 1035, 1028, 989, 1055, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1064, 1040, 995, 1022, 1011, 1034, 1005, 1049, 1064, 1007, 992, 1039, 1034, 953, 1018, 1020, 1009, 1004, 1047, 1043, 1023, 1000, 1000, 1004, 1034, 1029, 1057, 1033, 1080, 996, 1030, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1022, 1005, 1025, 1062, 992, 1023, 1047, 1019, 1000, 999, 1026, 961, 996, 1031, 1026, 1045, 1072, 1032, 995, 1077, 1046, 1002, 1049, 998, 1015, 1023, 1003, 1004, 1047, 1017, 1056, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1019, 993, 1037, 1036, 980, 1046, 1025, 994, 1033, 995, 990, 1029, 1007, 1014, 997, 1033, 1062, 1005, 1081, 1074, 952, 1021, 989, 1032, 1038, 1068, 1041, 1049, 1026, 1025, 1045, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1030, 1048, 1012, 1025, 0, 0, 0, 0, 1021, 1052, 1024, 1014, 4, 2, 0, 0, 1039, 1059, 1041, 1029, 0, 0, 0, 0, 1016, 1029, 957, 1042, 15, 0, 0, 0, 1045, 964, 1038, 1064, 0, 0, 0, 0, 1002, 983, 1016, 1010, 34, 8, 0, 0, 998, 1017, 1097, 1008, 0, 0, 0, 0, 1066, 989, 977, 990, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4117, 4168, 4059, 4111, 4053, 4120, 4025] avg=4096.0000 max=4168 max_rank=2 min=4025 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1041, 1007, 1024, 980, 0, 0, 0, 0, 1098, 1042, 1047, 1031, 4, 2, 0, 0, 1008, 1029, 1013, 1037, 0, 0, 0, 0, 1046, 1069, 1017, 1011, 14, 0, 0, 0, 1044, 1038, 1056, 993, 0, 0, 0, 0, 1057, 1011, 1017, 1005, 34, 7, 0, 0, 1027, 1023, 984, 1014, 0, 0, 0, 0, 951, 992, 989, 1003, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4224, 4087, 4157, 4131, 4131, 4048, 3938] avg=4096.0000 max=4224 max_rank=1 min=3938 min_rank=7 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16248 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[992, 1018, 1051, 1048, 0, 0, 0, 0, 1041, 1023, 1036, 968, 4, 2, 0, 0, 1037, 1046, 1024, 1022, 0, 0, 0, 0, 1041, 1022, 977, 1014, 14, 0, 0, 0, 970, 1057, 999, 955, 0, 0, 0, 0, 1051, 1036, 987, 1015, 33, 7, 0, 0, 1099, 1012, 1040, 980, 0, 0, 0, 0, 1051, 1048, 1032, 1013, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4074, 4129, 4068, 3981, 4129, 4131, 4147] avg=4096.0000 max=4147 max_rank=7 min=3981 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1022, 982, 1025, 1062, 0, 0, 0, 0, 992, 1023, 1047, 1019, 4, 2, 0, 0, 996, 999, 1026, 961, 0, 0, 0, 0, 996, 1031, 1026, 1045, 14, 0, 0, 0, 1069, 1032, 995, 1077, 0, 0, 0, 0, 1046, 1002, 1049, 998, 33, 7, 0, 0, 1015, 990, 1003, 1004, 0, 0, 0, 0, 1047, 1017, 1056, 1053, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4087, 3982, 4112, 4173, 4135, 4012, 4176] avg=4096.0000 max=4176 max_rank=7 min=3982 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1055, 1021, 1012, 1007, 0, 0, 0, 0, 1006, 983, 1044, 995, 4, 2, 0, 0, 1000, 1034, 1033, 1030, 0, 0, 0, 0, 1013, 1087, 1014, 1017, 14, 0, 0, 0, 1026, 979, 1006, 1020, 0, 0, 0, 0, 1017, 1009, 1054, 1037, 34, 7, 0, 0, 1028, 1034, 1007, 1035, 0, 0, 0, 0, 1028, 989, 1055, 1029, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4034, 4097, 4145, 4031, 4158, 4104, 4104] avg=4096.0000 max=4158 max_rank=5 min=4031 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1019, 970, 1037, 1036, 0, 0, 0, 0, 980, 1046, 1025, 994, 4, 2, 0, 0, 1029, 995, 990, 1029, 0, 0, 0, 0, 1007, 1014, 997, 1033, 14, 0, 0, 0, 1059, 1005, 1081, 1074, 0, 0, 0, 0, 952, 1021, 989, 1032, 34, 7, 0, 0, 1038, 1034, 1041, 1049, 0, 0, 0, 0, 1026, 1025, 1045, 1032, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4051, 4043, 4065, 4219, 4035, 4162, 4131] avg=4096.0000 max=4219 max_rank=4 min=4035 min_rank=5 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16547 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1064, 1008, 995, 1022, 0, 0, 0, 0, 1011, 1034, 1005, 1049, 5, 3, 2, 0, 1059, 1007, 992, 1039, 0, 0, 0, 0, 1034, 953, 1018, 1020, 17, 0, 0, 0, 1002, 1004, 1047, 1043, 0, 0, 0, 0, 1023, 1000, 1000, 1004, 37, 12, 0, 0, 1034, 992, 1057, 1033, 0, 0, 0, 0, 1080, 996, 1030, 1032, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4109, 4097, 4042, 4096, 4076, 4116, 4143] avg=4096.0000 max=4143 max_rank=7 min=4042 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1029, 1039, 1044, 1043, 0, 0, 0, 0, 1011, 1018, 994, 1043, 4, 2, 0, 0, 1026, 1028, 1047, 1064, 0, 0, 0, 0, 1033, 1047, 1029, 996, 15, 0, 0, 0, 1026, 1017, 981, 1002, 0, 0, 0, 0, 974, 999, 977, 1060, 33, 8, 0, 0, 1058, 989, 1009, 1019, 0, 0, 0, 0, 1025, 1024, 994, 1058, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4072, 4165, 4120, 4026, 4051, 4075, 4104] avg=4096.0000 max=4165 max_rank=2 min=4026 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8064, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1011, 996, 1034, 1022, 1052, 1069, 1041, 1040, 1046, 1045, 1053, 1028, 1035, 1014, 1046, 987, 995, 1054, 987, 997, 1027, 979, 991, 1005, 1028, 991, 1047, 1041, 1025, 1022, 1066, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1047, 1045, 992, 1088, 1030, 1040, 1046, 1009, 1007, 1044, 994, 1034, 953, 1017, 967, 1057, 1002, 1082, 1035, 1040, 1018, 1008, 993, 1011, 1051, 1026, 1028, 1043, 1008, 1036, 966, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1023, 1057, 1066, 1020, 1006, 992, 1056, 1029, 984, 1003, 1038, 1027, 1060, 1034, 1023, 1004, 995, 1050, 1002, 999, 975, 1037, 1036, 1037, 1007, 997, 1035, 983, 1067, 1022, 1057, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1061, 1027, 1003, 989, 990, 998, 1033, 1046, 1037, 1009, 1012, 1038, 1014, 1020, 1048, 1012, 1062, 1051, 1024, 1057, 1028, 986, 1042, 1025, 1041, 995, 1028, 996, 1016, 1032, 985, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1040, 1040, 991, 1028, 1019, 1009, 1038, 1012, 1024, 994, 1062, 1019, 1014, 988, 992, 1009, 1041, 1015, 1031, 1051, 1058, 1003, 990, 1041, 1019, 1068, 1070, 1044, 1007, 982, 1024, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1051, 1032, 1027, 1016, 1062, 1066, 1048, 997, 1040, 990, 1035, 1011, 1019, 1015, 1032, 1040, 1010, 999, 1017, 1011, 989, 1036, 998, 969, 988, 1071, 1030, 1015, 1059, 1000, 1057, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1053, 1001, 1055, 1032, 1049, 1030, 1016, 1002, 981, 975, 995, 1060, 983, 1010, 1045, 1029, 1026, 1006, 1017, 1075, 967, 1002, 1035, 1015, 1006, 1072, 1003, 1051, 1077, 1056, 1005, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1028, 1035, 1024, 1046, 1057, 1017, 1003, 998, 1015, 1014, 1016, 1089, 980, 1007, 992, 1027, 1049, 1018, 1022, 1006, 1051, 1022, 1020, 1019, 1009, 979, 1050, 1044, 1046, 995, 1027, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[986, 996, 1034, 1022, 0, 0, 0, 0, 1052, 1069, 1025, 1040, 0, 0, 0, 0, 1046, 1045, 1053, 1028, 6, 0, 0, 0, 1035, 1014, 1046, 987, 16, 11, 7, 1, 995, 1047, 987, 997, 0, 0, 0, 0, 1027, 979, 991, 1005, 25, 17, 0, 0, 1028, 991, 1036, 1041, 0, 0, 0, 0, 1025, 1022, 1066, 970, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4186, 4178, 4117, 4026, 4044, 4096, 4083] avg=4096.0000 max=4186 max_rank=1 min=4026 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16249 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1021, 1045, 992, 1088, 0, 0, 0, 0, 1030, 1040, 1030, 1009, 0, 0, 0, 0, 1007, 1044, 994, 1034, 6, 0, 0, 0, 953, 1017, 967, 1057, 16, 10, 8, 1, 1002, 1075, 1035, 1040, 0, 0, 0, 0, 1018, 1008, 993, 1011, 26, 18, 0, 0, 1051, 1026, 1018, 1043, 0, 0, 0, 0, 1008, 1036, 966, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4109, 4085, 4029, 4152, 4074, 4138, 4035] avg=4096.0000 max=4152 max_rank=4 min=4029 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1002, 1035, 1024, 1046, 0, 0, 0, 0, 1057, 1017, 988, 998, 0, 0, 0, 0, 1015, 1014, 1016, 1089, 6, 0, 0, 0, 980, 1007, 992, 1027, 15, 11, 8, 1, 1049, 1011, 1022, 1006, 0, 0, 0, 0, 1051, 1022, 1020, 1019, 26, 18, 0, 0, 1009, 979, 1039, 1044, 0, 0, 0, 0, 1046, 995, 1027, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4060, 4140, 4041, 4088, 4156, 4071, 4105] avg=4096.0000 max=4156 max_rank=5 min=4041 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[997, 1057, 1066, 1020, 0, 0, 0, 0, 1006, 992, 1040, 1029, 0, 0, 0, 0, 984, 1003, 1038, 1027, 6, 0, 0, 0, 1060, 1034, 1023, 1004, 16, 10, 8, 1, 995, 1043, 1002, 999, 0, 0, 0, 0, 975, 1037, 1036, 1037, 26, 17, 0, 0, 1007, 997, 1025, 983, 0, 0, 0, 0, 1067, 1022, 1057, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4067, 4058, 4156, 4039, 4128, 4012, 4168] avg=4096.0000 max=4168 max_rank=7 min=4012 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1034, 1027, 1003, 989, 0, 0, 0, 0, 990, 998, 1017, 1046, 0, 0, 0, 0, 1037, 1009, 1012, 1038, 6, 0, 0, 0, 1014, 1020, 1048, 1012, 16, 10, 8, 1, 1062, 1044, 1024, 1057, 0, 0, 0, 0, 1028, 986, 1042, 1025, 27, 18, 0, 0, 1041, 995, 1018, 996, 0, 0, 0, 0, 1016, 1032, 985, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4051, 4102, 4129, 4187, 4126, 4050, 4070] avg=4096.0000 max=4187 max_rank=4 min=4050 min_rank=6 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1014, 1040, 991, 1028, 0, 0, 0, 0, 1019, 1009, 1022, 1012, 0, 0, 0, 0, 1024, 994, 1062, 1019, 6, 0, 0, 0, 1014, 988, 992, 1009, 16, 11, 8, 1, 1041, 1008, 1031, 1051, 0, 0, 0, 0, 1058, 1003, 990, 1041, 26, 17, 0, 0, 1019, 1068, 1059, 1044, 0, 0, 0, 0, 1007, 982, 1024, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4062, 4105, 4039, 4131, 4135, 4190, 4033] avg=4096.0000 max=4190 max_rank=6 min=4033 min_rank=7 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 1032, 1027, 1016, 0, 0, 0, 0, 1062, 1066, 1032, 997, 0, 0, 0, 0, 1040, 990, 1035, 1011, 5, 0, 0, 0, 1019, 1015, 1032, 1040, 16, 10, 8, 1, 1010, 993, 1017, 1011, 0, 0, 0, 0, 989, 1036, 998, 969, 26, 17, 0, 0, 988, 1071, 1020, 1015, 0, 0, 0, 0, 1059, 1000, 1057, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4157, 4081, 4141, 4031, 4035, 4094, 4129] avg=4096.0000 max=4157 max_rank=1 min=4031 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1023, 1001, 1055, 1032, 0, 0, 0, 0, 1049, 1030, 995, 1002, 0, 0, 0, 0, 981, 975, 995, 1060, 8, 0, 0, 0, 983, 1010, 1045, 1029, 21, 15, 11, 2, 1026, 996, 1017, 1075, 0, 0, 0, 0, 967, 1002, 1035, 1015, 30, 21, 0, 0, 1006, 1072, 988, 1051, 0, 0, 0, 0, 1077, 1056, 1005, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4076, 4019, 4116, 4114, 4070, 4117, 4145] avg=4096.0000 max=4145 max_rank=7 min=4019 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8192, 256, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1039, 1034, 1062, 993, 1008, 1032, 1011, 1011, 1025, 1021, 1019, 1027, 1047, 1044, 1032, 956, 1062, 1042, 1038, 1067, 1022, 1037, 1021, 952, 1048, 1036, 1015, 1025, 1023, 975, 1001, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1028, 998, 1035, 1063, 1052, 1020, 1062, 993, 987, 1015, 1055, 979, 1006, 964, 1033, 1047, 1020, 1030, 1028, 1079, 1039, 1008, 1023, 1053, 1031, 1013, 991, 1030, 1011, 1056, 980, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[997, 1049, 1017, 990, 1002, 974, 1095, 1049, 977, 1050, 1037, 958, 1038, 1003, 1081, 1073, 1027, 996, 1000, 1022, 1083, 989, 1064, 949, 976, 1012, 1025, 1054, 1079, 1027, 1078, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1033, 1046, 1047, 1046, 985, 1026, 945, 1047, 987, 1044, 995, 994, 1042, 1037, 994, 997, 1045, 1026, 1009, 1006, 1055, 1006, 1005, 1077, 1030, 971, 1047, 1032, 1032, 1031, 1066, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1008, 1008, 1005, 1014, 1042, 1048, 1005, 995, 1033, 1014, 1082, 1009, 1014, 1005, 1052, 993, 950, 990, 1035, 1089, 1048, 1019, 967, 1041, 991, 1039, 1094, 1029, 1023, 1044, 1046, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1012, 1033, 1043, 1004, 1017, 1031, 1054, 987, 1020, 977, 984, 1017, 1005, 1009, 1051, 1042, 1075, 1050, 1074, 1017, 996, 998, 1059, 1040, 1065, 992, 980, 1004, 1021, 1038, 1011, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1051, 1019, 994, 1002, 1023, 1029, 990, 1033, 1049, 1042, 1037, 1070, 1043, 1031, 1021, 990, 1009, 967, 1073, 1074, 973, 1053, 990, 996, 1022, 1004, 1063, 1013, 1045, 1015, 1022, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1030, 1018, 1027, 1043, 1018, 996, 1002, 1064, 994, 1051, 998, 1060, 1029, 986, 1057, 1072, 960, 1044, 1027, 1064, 1014, 966, 1049, 1065, 1017, 1014, 1000, 997, 966, 1040, 1034, 1066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1028, 998, 1033, 1063, 0, 0, 0, 0, 1052, 1020, 1062, 993, 8, 6, 0, 0, 987, 1015, 1055, 979, 20, 0, 0, 0, 1006, 964, 1030, 1047, 0, 0, 0, 0, 1020, 1030, 1028, 1052, 0, 0, 0, 0, 1039, 1008, 1023, 1053, 14, 0, 0, 0, 1031, 1013, 991, 1030, 7, 3, 2, 0, 1011, 1056, 980, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4141, 4056, 4047, 4130, 4137, 4077, 4058] avg=4096.0000 max=4141 max_rank=1 min=4047 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1008, 1008, 1003, 1014, 0, 0, 0, 0, 1042, 1048, 1005, 995, 8, 6, 0, 0, 1033, 1014, 1082, 1009, 20, 0, 0, 0, 1014, 1005, 1049, 993, 0, 0, 0, 0, 950, 990, 1035, 1061, 0, 0, 0, 0, 1048, 1019, 967, 1041, 14, 0, 0, 0, 991, 1039, 1094, 1029, 8, 3, 2, 0, 1023, 1044, 1046, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4104, 4158, 4061, 4036, 4089, 4166, 4121] avg=4096.0000 max=4166 max_rank=6 min=4033 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1039, 1034, 1060, 993, 0, 0, 0, 0, 1008, 1032, 1011, 1011, 8, 6, 0, 0, 1025, 1021, 1019, 1027, 20, 0, 0, 0, 1047, 1044, 1029, 956, 0, 0, 0, 0, 1062, 1042, 1038, 1040, 0, 0, 0, 0, 1022, 1037, 1021, 952, 14, 0, 0, 0, 1048, 1036, 1015, 1025, 7, 3, 2, 0, 1023, 975, 1001, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4076, 4112, 4076, 4182, 4046, 4136, 4014] avg=4096.0000 max=4182 max_rank=4 min=4014 min_rank=7 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[997, 1049, 1015, 990, 0, 0, 0, 0, 1002, 974, 1095, 1049, 8, 6, 0, 0, 977, 1050, 1037, 958, 19, 0, 0, 0, 1038, 1003, 1078, 1073, 0, 0, 0, 0, 1027, 996, 1000, 996, 0, 0, 0, 0, 1083, 989, 1064, 949, 13, 0, 0, 0, 976, 1012, 1025, 1054, 7, 3, 2, 0, 1079, 1027, 1078, 970, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4134, 4041, 4192, 4019, 4098, 4079, 4154] avg=4096.0000 max=4192 max_rank=3 min=4019 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1033, 1046, 1045, 1046, 0, 0, 0, 0, 985, 1026, 945, 1047, 8, 6, 0, 0, 987, 1044, 995, 994, 20, 0, 0, 0, 1042, 1037, 991, 997, 0, 0, 0, 0, 1045, 1026, 1009, 980, 0, 0, 0, 0, 1055, 1006, 1005, 1077, 13, 0, 0, 0, 1030, 971, 1047, 1032, 7, 3, 2, 0, 1032, 1031, 1066, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4017, 4040, 4067, 4060, 4156, 4092, 4166] avg=4096.0000 max=4170 max_rank=0 min=4017 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1012, 1033, 1037, 1004, 0, 0, 0, 0, 1017, 1031, 1054, 987, 12, 12, 0, 0, 1020, 977, 984, 1017, 23, 0, 0, 0, 1005, 1009, 1046, 1042, 0, 0, 0, 0, 1075, 1050, 1074, 978, 0, 0, 0, 0, 996, 998, 1059, 1040, 15, 0, 0, 0, 1065, 992, 980, 1004, 12, 5, 6, 0, 1021, 1038, 1011, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4113, 4021, 4102, 4177, 4108, 4064, 4097] avg=4096.0000 max=4177 max_rank=4 min=4021 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1030, 1018, 1025, 1043, 0, 0, 0, 0, 1018, 996, 1002, 1064, 8, 6, 0, 0, 994, 1051, 998, 1060, 20, 0, 0, 0, 1029, 986, 1054, 1072, 0, 0, 0, 0, 960, 1044, 1027, 1037, 0, 0, 0, 0, 1014, 966, 1049, 1065, 14, 0, 0, 0, 1017, 1014, 1000, 997, 7, 3, 2, 0, 966, 1040, 1034, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4094, 4123, 4141, 4068, 4108, 4040, 4078] avg=4096.0000 max=4141 max_rank=3 min=4040 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1051, 1019, 992, 1002, 0, 0, 0, 0, 1023, 1029, 990, 1033, 8, 6, 0, 0, 1049, 1042, 1037, 1070, 19, 0, 0, 0, 1043, 1031, 1018, 990, 0, 0, 0, 0, 1009, 967, 1073, 1047, 0, 0, 0, 0, 973, 1053, 990, 996, 14, 0, 0, 0, 1022, 1004, 1063, 1013, 7, 3, 2, 0, 1045, 1015, 1022, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4089, 4217, 4082, 4096, 4026, 4114, 4080] avg=4096.0000 max=4217 max_rank=2 min=4026 min_rank=5 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1004, 1007, 1021, 1056, 984, 1035, 990, 985, 1027, 1023, 998, 1074, 1006, 980, 1075, 1049, 1089, 1016, 1048, 1028, 971, 1020, 984, 1038, 992, 1011, 971, 1046, 1074, 1041, 1068, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1049, 1050, 1019, 1042, 1040, 1039, 1111, 997, 1041, 1011, 996, 1006, 1006, 1012, 1010, 1036, 1064, 1036, 1028, 975, 1044, 991, 1018, 1030, 994, 1026, 1043, 1017, 1022, 969, 1059, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1022, 1003, 1034, 1027, 978, 1026, 1046, 1029, 1035, 974, 1077, 1038, 1056, 987, 1005, 1076, 1011, 1008, 1060, 1014, 1058, 1035, 1021, 1042, 1068, 1014, 984, 1022, 988, 1024, 1037, 969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[997, 1061, 1033, 1049, 978, 1037, 999, 1053, 992, 1043, 1052, 1046, 1018, 1030, 1033, 982, 1033, 1006, 1019, 1021, 1000, 1059, 965, 1011, 1072, 1034, 1005, 1029, 1020, 1028, 1015, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1062, 1019, 1010, 1044, 987, 1032, 1077, 1033, 1058, 1006, 1009, 1011, 985, 1004, 983, 1039, 1023, 1055, 967, 1033, 1052, 1050, 1014, 1038, 983, 1003, 1075, 999, 1057, 1023, 1031, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[994, 1086, 977, 1045, 1033, 1032, 994, 985, 1013, 1016, 1039, 1002, 1030, 1022, 1062, 1031, 1059, 990, 1010, 1043, 1056, 1101, 1042, 1020, 1027, 1027, 981, 1026, 1042, 1009, 988, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1002, 1043, 981, 995, 1017, 1000, 1011, 1035, 1054, 1054, 984, 1100, 1019, 1044, 1063, 1046, 1020, 1057, 955, 1028, 1070, 1037, 1039, 988, 1012, 1019, 1031, 1047, 986, 983, 983, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1002, 997, 981, 1012, 1063, 1051, 1029, 1011, 1021, 1024, 985, 1006, 1053, 992, 1056, 1046, 983, 1055, 1042, 1048, 1019, 1042, 1048, 1016, 1040, 1027, 1059, 1042, 1015, 980, 1005, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1022, 1003, 1034, 1027, 4, 0, 0, 0, 978, 1026, 1046, 1029, 4, 2, 0, 0, 1035, 974, 1077, 1033, 0, 0, 0, 0, 1056, 987, 1005, 1068, 0, 0, 0, 0, 1005, 1008, 1060, 1014, 0, 0, 0, 0, 1058, 1017, 1021, 1042, 0, 0, 0, 0, 1068, 1014, 984, 1022, 5, 0, 0, 0, 988, 1024, 1037, 969, 18, 4, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4085, 4119, 4116, 4087, 4138, 4093, 4040] avg=4096.0000 max=4138 max_rank=5 min=4040 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[994, 1086, 977, 1045, 4, 0, 0, 0, 1033, 1032, 994, 985, 4, 2, 0, 0, 1013, 1016, 1039, 997, 0, 0, 0, 0, 1030, 1022, 1062, 1023, 0, 0, 0, 0, 1053, 990, 1010, 1043, 0, 0, 0, 0, 1056, 1082, 1042, 1020, 0, 0, 0, 0, 1027, 1027, 981, 1026, 5, 0, 0, 0, 1042, 1009, 988, 986, 19, 4, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4050, 4065, 4137, 4096, 4200, 4066, 4048] avg=4096.0000 max=4200 max_rank=5 min=4048 min_rank=7 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1004, 1007, 1021, 1056, 5, 0, 0, 0, 984, 1035, 990, 985, 4, 2, 0, 0, 1027, 1023, 998, 1069, 0, 0, 0, 0, 1006, 980, 1075, 1041, 0, 0, 0, 0, 1082, 1016, 1048, 1028, 0, 0, 0, 0, 971, 1002, 984, 1038, 0, 0, 0, 0, 992, 1011, 971, 1046, 5, 0, 0, 0, 1074, 1041, 1068, 1057, 18, 4, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4000, 4117, 4102, 4174, 3995, 4025, 4262] avg=4096.0000 max=4262 max_rank=7 min=3995 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=28506 max_load_reduction_pct=0.869934 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1049, 1050, 1019, 1042, 10, 5, 0, 0, 1040, 1039, 1111, 997, 6, 3, 0, 0, 1041, 1011, 996, 994, 0, 0, 0, 0, 1006, 1012, 1010, 1024, 0, 0, 0, 0, 1051, 1036, 1028, 975, 0, 0, 0, 0, 1044, 969, 1018, 1030, 0, 0, 0, 0, 994, 1026, 1043, 1017, 7, 0, 0, 0, 1022, 969, 1059, 987, 22, 6, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4196, 4042, 4052, 4090, 4061, 4087, 4065] avg=4096.0000 max=4196 max_rank=1 min=4042 min_rank=2 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1062, 1019, 1010, 1044, 4, 0, 0, 0, 987, 1032, 1077, 1033, 4, 2, 0, 0, 1058, 1006, 1009, 1006, 0, 0, 0, 0, 985, 1004, 983, 1031, 0, 0, 0, 0, 1017, 1055, 967, 1033, 0, 0, 0, 0, 1052, 1031, 1014, 1038, 0, 0, 0, 0, 983, 1003, 1075, 999, 5, 0, 0, 0, 1057, 1023, 1031, 1006, 19, 4, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4135, 4079, 4003, 4072, 4135, 4065, 4140] avg=4096.0000 max=4140 max_rank=7 min=4003 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[997, 1061, 1033, 1049, 4, 0, 0, 0, 978, 1037, 999, 1053, 4, 2, 0, 0, 992, 1043, 1052, 1041, 0, 0, 0, 0, 1018, 1030, 1033, 974, 0, 0, 0, 0, 1027, 1006, 1019, 1021, 0, 0, 0, 0, 1000, 1040, 965, 1011, 0, 0, 0, 0, 1072, 1034, 1005, 1029, 5, 0, 0, 0, 1020, 1028, 1015, 1048, 19, 4, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4073, 4128, 4055, 4073, 4016, 4145, 4134] avg=4096.0000 max=4145 max_rank=6 min=4016 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1002, 1043, 981, 995, 4, 0, 0, 0, 1017, 1000, 1011, 1035, 4, 2, 0, 0, 1054, 1054, 984, 1095, 0, 0, 0, 0, 1019, 1044, 1063, 1038, 0, 0, 0, 0, 1014, 1057, 955, 1028, 0, 0, 0, 0, 1070, 1019, 1039, 988, 0, 0, 0, 0, 1012, 1019, 1031, 1047, 5, 0, 0, 0, 986, 983, 983, 1065, 18, 4, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4069, 4187, 4164, 4054, 4116, 4114, 4039] avg=4096.0000 max=4187 max_rank=2 min=4025 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1002, 997, 981, 1012, 4, 0, 0, 0, 1063, 1051, 1029, 1011, 4, 2, 0, 0, 1021, 1024, 985, 1001, 0, 0, 0, 0, 1053, 992, 1056, 1038, 0, 0, 0, 0, 977, 1055, 1042, 1048, 0, 0, 0, 0, 1019, 1024, 1048, 1016, 0, 0, 0, 0, 1040, 1027, 1059, 1042, 5, 0, 0, 0, 1015, 980, 1005, 1018, 18, 4, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4160, 4031, 4139, 4122, 4107, 4173, 4040] avg=4096.0000 max=4173 max_rank=6 min=3996 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True + [2026-08-30 04:58:02] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 38451.3 | throughput per GPU (TFLOP/s/GPU): 8.7 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102435E+01 | loss scale: 1.0 | grad norm: 43.563 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 23.22 + +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 63622.4423828125 | max allocated: 85569.9892578125 | reserved: 88146.0 | max reserved: 88146.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1070, 1047, 1011, 988, 1039, 1030, 1037, 1010, 1009, 1048, 988, 1001, 1074, 1031, 1011, 985, 986, 1041, 1054, 1009, 1035, 1009, 1052, 1041, 1047, 1021, 1003, 1000, 1002, 1038, 1052, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1075, 986, 984, 1004, 1029, 1026, 1041, 1001, 1044, 1005, 994, 1019, 1052, 1044, 1043, 1061, 944, 985, 1046, 1032, 1048, 1027, 1047, 1010, 1018, 986, 1046, 1008, 1039, 1075, 1033, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1024, 1062, 1027, 1023, 1023, 1058, 974, 1033, 1020, 1026, 1040, 1028, 1066, 1026, 1022, 1001, 1046, 1017, 1036, 1030, 1049, 968, 1029, 1017, 1019, 970, 970, 1007, 1049, 1028, 1040, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1035, 1030, 1025, 973, 994, 1017, 1025, 1016, 1021, 1020, 1055, 1060, 1057, 1027, 1025, 1036, 1035, 1004, 1002, 1030, 1022, 1017, 1038, 998, 1010, 997, 1039, 1072, 1029, 1027, 1005, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1022, 1022, 980, 973, 1050, 1029, 986, 1029, 1062, 1010, 1039, 971, 1036, 993, 1018, 1054, 1040, 1010, 1053, 1023, 1034, 1063, 1009, 1030, 1015, 1059, 1030, 1028, 1057, 1072, 999, 972] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1043, 1010, 1009, 1037, 1052, 1069, 1052, 1030, 964, 1031, 997, 1040, 1020, 1019, 1038, 1028, 1033, 994, 1020, 988, 1065, 1019, 1030, 1049, 1054, 1018, 966, 1005, 1024, 1032, 1041, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1061, 1041, 1048, 1005, 1017, 1015, 1011, 1040, 993, 1080, 1042, 1019, 1031, 1039, 1008, 1051, 1010, 1010, 998, 1045, 998, 1052, 989, 1074, 1004, 1004, 1010, 999, 1015, 1026, 1015, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1048, 1022, 1028, 1046, 1031, 1074, 1007, 1036, 1024, 1065, 1013, 1025, 1011, 1063, 1010, 965, 1032, 1003, 1039, 999, 1019, 1051, 1014, 1004, 1020, 1043, 1042, 1008, 1001, 1026, 998, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1061, 1041, 1048, 1005, 1, 0, 0, 0, 1017, 1002, 1011, 1040, 0, 0, 0, 0, 993, 1080, 1042, 1019, 1, 0, 0, 0, 1010, 1039, 1008, 1051, 0, 0, 0, 0, 1010, 1010, 998, 1045, 13, 7, 0, 0, 983, 1052, 989, 1074, 0, 0, 0, 0, 1004, 1004, 1010, 999, 21, 8, 0, 0, 1015, 1024, 1015, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4070, 4135, 4108, 4083, 4098, 4046, 4072] avg=4096.0000 max=4156 max_rank=0 min=4046 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1070, 1047, 1011, 988, 1, 0, 0, 0, 1039, 1017, 1037, 1010, 0, 0, 0, 0, 1009, 1048, 988, 1001, 1, 0, 0, 0, 1052, 1031, 1011, 985, 0, 0, 0, 0, 986, 1041, 1054, 1009, 13, 8, 0, 0, 1018, 1009, 1052, 1041, 0, 0, 0, 0, 1047, 1021, 1003, 1000, 22, 9, 0, 0, 1002, 1036, 1052, 999, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4103, 4047, 4079, 4111, 4120, 4102, 4089] avg=4096.0000 max=4120 max_rank=5 min=4047 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1075, 986, 984, 1004, 2, 0, 0, 0, 1029, 1004, 1041, 1001, 0, 0, 0, 0, 1044, 1005, 994, 1019, 3, 5, 0, 0, 1025, 1044, 1043, 1061, 0, 0, 0, 0, 944, 985, 1046, 1032, 17, 11, 0, 0, 1025, 1027, 1047, 1010, 0, 0, 0, 0, 1018, 986, 1046, 1008, 27, 12, 0, 0, 1039, 1070, 1033, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4075, 4070, 4173, 4035, 4109, 4097, 4158] avg=4096.0000 max=4173 max_rank=3 min=4035 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1022, 1022, 980, 973, 1, 0, 0, 0, 1050, 1016, 986, 1029, 0, 0, 0, 0, 1062, 1010, 1039, 971, 1, 0, 0, 0, 1015, 993, 1018, 1054, 0, 0, 0, 0, 1040, 1010, 1053, 1023, 13, 8, 0, 0, 1017, 1063, 1009, 1030, 0, 0, 0, 0, 1015, 1059, 1030, 1028, 21, 9, 0, 0, 1057, 1070, 999, 972, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 4081, 4083, 4080, 4147, 4119, 4162, 4098] avg=4096.0000 max=4162 max_rank=6 min=3998 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16526 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1024, 1062, 1027, 1023, 1, 0, 0, 0, 1023, 1045, 974, 1033, 0, 0, 0, 0, 1020, 1026, 1040, 1028, 1, 0, 0, 0, 1044, 1026, 1022, 1001, 0, 0, 0, 0, 1046, 1017, 1036, 1030, 13, 8, 0, 0, 1032, 968, 1029, 1017, 0, 0, 0, 0, 1019, 970, 970, 1007, 22, 9, 0, 0, 1049, 1026, 1040, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4075, 4115, 4093, 4150, 4046, 3997, 4155] avg=4096.0000 max=4155 max_rank=7 min=3997 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1035, 1030, 1025, 973, 1, 0, 0, 0, 994, 1004, 1025, 1016, 0, 0, 0, 0, 1021, 1020, 1055, 1060, 1, 0, 0, 0, 1035, 1027, 1025, 1036, 0, 0, 0, 0, 1035, 1004, 1002, 1030, 13, 8, 0, 0, 1005, 1017, 1038, 998, 0, 0, 0, 0, 1010, 997, 1039, 1072, 22, 9, 0, 0, 1029, 1025, 1005, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4039, 4157, 4123, 4092, 4058, 4149, 4086] avg=4096.0000 max=4157 max_rank=2 min=4039 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1048, 1022, 1028, 1046, 1, 0, 0, 0, 1031, 1061, 1007, 1036, 0, 0, 0, 0, 1024, 1065, 1013, 1025, 1, 0, 0, 0, 990, 1063, 1010, 965, 0, 0, 0, 0, 1032, 1003, 1039, 999, 13, 8, 0, 0, 1003, 1051, 1014, 1004, 0, 0, 0, 0, 1020, 1043, 1042, 1008, 21, 8, 0, 0, 1001, 1024, 998, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4135, 4128, 4028, 4094, 4072, 4142, 4024] avg=4096.0000 max=4145 max_rank=0 min=4024 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1043, 1010, 1009, 1037, 1, 0, 0, 0, 1052, 1056, 1052, 1030, 0, 0, 0, 0, 964, 1031, 997, 1040, 1, 0, 0, 0, 999, 1019, 1038, 1028, 0, 0, 0, 0, 1033, 994, 1020, 988, 13, 8, 0, 0, 1048, 1019, 1030, 1049, 0, 0, 0, 0, 1054, 1018, 966, 1005, 21, 9, 0, 0, 1024, 1030, 1041, 991, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4190, 4033, 4084, 4056, 4146, 4073, 4086] avg=4096.0000 max=4190 max_rank=1 min=4033 min_rank=2 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8320, 8192, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8320, 8192, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1062, 1006, 1020, 973, 1048, 1056, 970, 1019, 1050, 1047, 1024, 1033, 1010, 1031, 1017, 1033, 1083, 1054, 1008, 1077, 1008, 997, 1000, 1019, 1022, 1024, 1066, 973, 1003, 993, 1043, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1062, 1026, 1050, 1042, 1012, 1007, 1022, 977, 1047, 967, 1041, 976, 1047, 989, 999, 1037, 975, 1012, 1064, 1016, 994, 1063, 1064, 1023, 1046, 1055, 990, 1052, 1067, 1033, 1027, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1034, 980, 1037, 1051, 1028, 1049, 1023, 1039, 1048, 1027, 1013, 1009, 1046, 970, 1047, 988, 991, 1049, 1017, 1053, 1039, 1020, 1015, 1031, 1019, 1046, 975, 1026, 1022, 1030, 1001, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1016, 1041, 1048, 1062, 1007, 1013, 1041, 1030, 1029, 1067, 1000, 1055, 1072, 973, 1049, 1027, 952, 983, 1047, 974, 1038, 1049, 1045, 1058, 975, 1033, 1012, 1018, 985, 1021, 1032, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1062, 978, 1063, 1005, 1057, 1053, 1054, 1006, 1018, 1008, 1036, 1014, 1066, 1025, 965, 1056, 978, 1015, 1073, 1025, 1045, 942, 994, 1017, 1003, 1026, 1035, 1018, 1021, 1030, 1040, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1000, 1004, 1007, 1040, 1082, 1035, 1004, 1071, 1034, 1013, 1015, 1037, 973, 1028, 1010, 1024, 1069, 1090, 1014, 1014, 1053, 1019, 986, 950, 1052, 1001, 996, 1022, 1043, 1012, 1059, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1066, 1005, 1026, 1060, 952, 1010, 1053, 996, 1014, 1027, 1068, 1064, 1036, 1000, 1025, 1023, 1017, 1032, 1050, 1019, 965, 1036, 1054, 959, 1039, 1022, 1005, 1006, 1013, 1077, 1030, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1026, 1012, 1016, 1071, 1034, 1070, 993, 1057, 1041, 1048, 1096, 1012, 1021, 994, 992, 1052, 1021, 1004, 1017, 992, 1039, 1098, 1032, 997, 981, 966, 1041, 1036, 1033, 1019, 976, 981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1039, 978, 1063, 1005, 0, 0, 0, 0, 1057, 1041, 1054, 1006, 0, 0, 0, 0, 1018, 1008, 1011, 1014, 0, 0, 0, 0, 1066, 1025, 965, 1056, 15, 2, 0, 0, 978, 1015, 1071, 1025, 0, 0, 0, 0, 1045, 942, 994, 1017, 8, 7, 0, 0, 1003, 1026, 1035, 1018, 23, 0, 0, 0, 1021, 1030, 1040, 1040, 5, 2, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4158, 4051, 4129, 4089, 4013, 4105, 4138] avg=4096.0000 max=4158 max_rank=1 min=4013 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1043, 1005, 1026, 1060, 0, 0, 0, 0, 952, 999, 1053, 996, 0, 0, 0, 0, 1014, 1027, 1042, 1064, 0, 0, 0, 0, 1036, 1000, 1025, 1023, 15, 2, 0, 0, 1017, 1032, 1048, 1019, 0, 0, 0, 0, 965, 1036, 1054, 959, 8, 6, 0, 0, 1039, 1022, 1005, 1006, 24, 0, 0, 0, 1013, 1077, 1030, 1019, 5, 2, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4000, 4147, 4101, 4116, 4028, 4096, 4146] avg=4096.0000 max=4147 max_rank=2 min=4000 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1039, 1026, 1050, 1042, 0, 0, 0, 0, 1012, 996, 1022, 977, 0, 0, 0, 0, 1047, 967, 1016, 976, 0, 0, 0, 0, 1047, 989, 999, 1037, 15, 2, 0, 0, 975, 1012, 1062, 1016, 0, 0, 0, 0, 994, 1063, 1064, 1023, 8, 6, 0, 0, 1046, 1055, 990, 1052, 23, 0, 0, 0, 1067, 1033, 1027, 986, 5, 2, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4007, 4006, 4089, 4065, 4158, 4166, 4120] avg=4096.0000 max=4166 max_rank=6 min=4006 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1013, 980, 1037, 1051, 0, 0, 0, 0, 1028, 1037, 1023, 1039, 0, 0, 0, 0, 1048, 1027, 989, 1009, 0, 0, 0, 0, 1046, 970, 1047, 988, 14, 2, 0, 0, 991, 1049, 1015, 1053, 0, 0, 0, 0, 1039, 1020, 1015, 1031, 7, 7, 0, 0, 1019, 1046, 975, 1026, 22, 0, 0, 0, 1022, 1030, 1001, 1045, 5, 2, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4127, 4073, 4067, 4108, 4119, 4088, 4105] avg=4096.0000 max=4127 max_rank=1 min=4067 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1039, 1006, 1020, 973, 0, 0, 0, 0, 1048, 1044, 970, 1019, 0, 0, 0, 0, 1050, 1047, 999, 1033, 0, 0, 0, 0, 1010, 1031, 1017, 1033, 15, 2, 0, 0, 1083, 1054, 1006, 1077, 0, 0, 0, 0, 1008, 997, 1000, 1019, 8, 7, 0, 0, 1022, 1024, 1066, 973, 23, 0, 0, 0, 1003, 993, 1043, 999, 5, 2, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4081, 4129, 4108, 4220, 4039, 4108, 4045] avg=4096.0000 max=4220 max_rank=4 min=4038 min_rank=0 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[979, 1004, 1007, 1040, 0, 0, 0, 0, 1082, 1024, 1004, 1071, 0, 0, 0, 0, 1034, 1013, 991, 1037, 0, 0, 0, 0, 973, 1028, 1010, 1024, 14, 2, 0, 0, 1069, 1090, 1012, 1014, 0, 0, 0, 0, 1053, 1019, 986, 950, 7, 6, 0, 0, 1052, 1001, 996, 1022, 22, 0, 0, 0, 1043, 1012, 1059, 1011, 5, 2, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4181, 4075, 4051, 4185, 4021, 4093, 4132] avg=4096.0000 max=4185 max_rank=4 min=4021 min_rank=5 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[988, 1041, 1048, 1062, 0, 0, 0, 0, 1007, 994, 1041, 1030, 0, 0, 0, 0, 1029, 1067, 966, 1055, 0, 0, 0, 0, 1072, 973, 1049, 1027, 18, 8, 0, 0, 952, 983, 1044, 974, 0, 0, 0, 0, 1038, 1049, 1045, 1058, 10, 10, 0, 0, 975, 1033, 1012, 1018, 26, 0, 0, 0, 985, 1021, 1032, 1016, 9, 3, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4072, 4117, 4147, 3953, 4210, 4064, 4066] avg=4096.0000 max=4210 max_rank=5 min=3953 min_rank=4 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1005, 1012, 1016, 1071, 0, 0, 0, 0, 1034, 1058, 993, 1057, 0, 0, 0, 0, 1041, 1048, 1069, 1012, 0, 0, 0, 0, 1021, 994, 992, 1052, 14, 3, 0, 0, 1021, 1004, 1015, 992, 0, 0, 0, 0, 1039, 1098, 1032, 997, 7, 7, 0, 0, 981, 966, 1041, 1036, 24, 0, 0, 0, 1033, 1019, 976, 981, 5, 2, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4142, 4170, 4076, 4032, 4180, 4048, 4016] avg=4096.0000 max=4180 max_rank=5 min=4016 min_rank=7 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16276 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1019, 1015, 997, 997, 1016, 1069, 1022, 978, 1017, 1019, 1019, 1002, 999, 1026, 1079, 1047, 1010, 1028, 994, 985, 1020, 1031, 1023, 1014, 1045, 1083, 1033, 1057, 972, 1029, 1068, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1064, 1002, 1031, 1048, 1035, 1060, 1019, 1035, 1020, 1021, 1040, 995, 1011, 1027, 1059, 1020, 1014, 1019, 994, 1020, 1013, 1044, 983, 985, 1025, 1059, 1027, 1012, 1005, 1022, 1051, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1022, 1021, 1007, 1013, 1052, 1025, 1003, 1036, 1031, 1089, 1045, 1081, 1030, 1010, 1003, 1044, 1051, 1005, 1042, 975, 1029, 952, 1024, 1039, 982, 993, 1038, 1027, 1049, 1047, 1002, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1014, 1025, 980, 1032, 1035, 1021, 1017, 1024, 982, 1017, 1013, 967, 1017, 1040, 1028, 1061, 1013, 1057, 997, 1010, 1051, 1036, 1029, 985, 1035, 1093, 1065, 1045, 990, 1007, 1029, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[992, 1007, 1025, 1021, 1030, 993, 1082, 1063, 1008, 1050, 997, 983, 1037, 979, 1010, 1048, 1003, 1038, 1052, 1056, 1047, 987, 1013, 1027, 1052, 995, 1038, 1022, 1002, 1085, 1011, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1057, 1049, 1015, 1083, 1010, 1007, 968, 1031, 1026, 1008, 998, 953, 1031, 1007, 1004, 1022, 1029, 1067, 1064, 1078, 1063, 973, 1023, 1024, 1031, 995, 1050, 1041, 1042, 987, 999, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1042, 1007, 1101, 1084, 991, 1039, 1000, 996, 990, 1056, 1005, 1054, 1029, 1082, 998, 1051, 970, 1061, 1054, 1001, 1004, 1029, 1030, 969, 1046, 1004, 1036, 1011, 994, 1036, 1015, 983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1064, 1011, 1017, 1009, 1018, 1068, 1068, 1025, 1026, 1000, 1000, 997, 1016, 993, 1012, 1024, 998, 1063, 1020, 1082, 994, 1022, 1005, 1032, 1017, 1045, 1009, 1032, 1020, 996, 1027, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[992, 1007, 1025, 1009, 0, 0, 0, 0, 1030, 985, 1082, 1063, 0, 0, 0, 0, 1008, 1050, 997, 983, 12, 10, 8, 0, 1037, 979, 1010, 1039, 0, 0, 0, 0, 1003, 1028, 1052, 1056, 0, 0, 0, 0, 1047, 987, 1013, 1027, 33, 0, 0, 0, 1052, 995, 1005, 1022, 0, 0, 0, 0, 1002, 1085, 1011, 1015, 8, 1, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4160, 4068, 4065, 4139, 4107, 4074, 4122] avg=4096.0000 max=4160 max_rank=1 min=4033 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1057, 1049, 1015, 1070, 0, 0, 0, 0, 1010, 999, 968, 1031, 0, 0, 0, 0, 1026, 1008, 998, 953, 13, 10, 8, 0, 1031, 1007, 1004, 1013, 0, 0, 0, 0, 1029, 1057, 1064, 1078, 0, 0, 0, 0, 1063, 973, 1023, 1024, 33, 0, 0, 0, 1031, 995, 1017, 1041, 0, 0, 0, 0, 1042, 987, 999, 1033, 8, 1, 0, 0] rerouted_tokens_per_ep_rank=[4191, 4008, 4016, 4055, 4228, 4116, 4084, 4070] avg=4096.0000 max=4228 max_rank=4 min=4008 min_rank=1 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1064, 1002, 1031, 1035, 0, 0, 0, 0, 1035, 1052, 1019, 1035, 0, 0, 0, 0, 1020, 1021, 1040, 995, 13, 10, 8, 0, 1011, 1027, 1059, 1011, 0, 0, 0, 0, 1014, 1009, 994, 1020, 0, 0, 0, 0, 1013, 1044, 983, 985, 33, 0, 0, 0, 1025, 1059, 994, 1012, 0, 0, 0, 0, 1005, 1022, 1051, 1008, 8, 1, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4141, 4107, 4108, 4037, 4058, 4090, 4095] avg=4096.0000 max=4141 max_rank=1 min=4037 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1014, 1025, 980, 1020, 0, 0, 0, 0, 1035, 1013, 1017, 1024, 0, 0, 0, 0, 982, 1017, 1013, 967, 12, 10, 8, 0, 1017, 1040, 1028, 1052, 0, 0, 0, 0, 1013, 1047, 997, 1010, 0, 0, 0, 0, 1051, 1036, 1029, 985, 34, 0, 0, 0, 1035, 1093, 1031, 1045, 0, 0, 0, 0, 990, 1007, 1029, 1053, 8, 1, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4089, 4009, 4137, 4067, 4135, 4204, 4088] avg=4096.0000 max=4204 max_rank=6 min=4009 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1019, 1015, 997, 981, 0, 0, 0, 0, 1016, 1057, 1022, 978, 0, 0, 0, 0, 1017, 1019, 1019, 1002, 16, 13, 11, 7, 999, 1026, 1079, 1034, 0, 0, 0, 0, 1010, 1015, 994, 985, 0, 0, 0, 0, 1020, 1031, 1023, 1014, 37, 0, 0, 0, 1045, 1083, 989, 1057, 0, 0, 0, 0, 972, 1029, 1068, 1055, 12, 2, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4073, 4104, 4138, 4004, 4125, 4174, 4138] avg=4096.0000 max=4174 max_rank=6 min=4004 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1064, 1011, 1017, 997, 0, 0, 0, 0, 1018, 1060, 1068, 1025, 0, 0, 0, 0, 1026, 1000, 1000, 997, 12, 10, 8, 0, 1016, 993, 1012, 1015, 0, 0, 0, 0, 998, 1053, 1020, 1082, 0, 0, 0, 0, 994, 1022, 1005, 1032, 32, 0, 0, 0, 1017, 1045, 977, 1032, 0, 0, 0, 0, 1020, 996, 1027, 1058, 8, 1, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4171, 4053, 4036, 4153, 4085, 4071, 4110] avg=4096.0000 max=4171 max_rank=1 min=4036 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1022, 1021, 1007, 1001, 0, 0, 0, 0, 1052, 1017, 1003, 1036, 0, 0, 0, 0, 1031, 1089, 1045, 1081, 12, 9, 8, 0, 1030, 1010, 1003, 1035, 0, 0, 0, 0, 1051, 996, 1042, 975, 0, 0, 0, 0, 1029, 952, 1024, 1039, 33, 0, 0, 0, 982, 993, 1005, 1027, 0, 0, 0, 0, 1049, 1047, 1002, 1001, 8, 1, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4108, 4275, 4078, 4064, 4077, 4007, 4108] avg=4096.0000 max=4275 max_rank=2 min=4007 min_rank=6 imbalance_ratio=1.043701 max_load_reduction=28493 max_load_reduction_pct=0.869537 moved_to_echo=16256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1042, 1007, 1101, 1071, 0, 0, 0, 0, 991, 1031, 1000, 996, 0, 0, 0, 0, 990, 1056, 1005, 1054, 13, 10, 8, 0, 1029, 1082, 998, 1042, 0, 0, 0, 0, 970, 1051, 1054, 1001, 0, 0, 0, 0, 1004, 1029, 1030, 969, 33, 0, 0, 0, 1046, 1004, 1003, 1011, 0, 0, 0, 0, 994, 1036, 1015, 983, 8, 1, 0, 0] rerouted_tokens_per_ep_rank=[4221, 4018, 4136, 4151, 4076, 4065, 4064, 4037] avg=4096.0000 max=4221 max_rank=0 min=4018 min_rank=1 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8064, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[975, 999, 1075, 1085, 1011, 1009, 1018, 1051, 1032, 1000, 1011, 993, 1021, 986, 993, 1060, 1041, 999, 972, 992, 1049, 990, 1029, 1095, 1023, 1070, 1048, 1017, 1031, 1034, 1045, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1043, 998, 1017, 1022, 1072, 1031, 1041, 1033, 967, 985, 1012, 1017, 1053, 1040, 1019, 1019, 1009, 1008, 1065, 1039, 1026, 950, 1043, 1033, 1026, 1050, 999, 1050, 1034, 1064, 1021, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1011, 994, 1071, 1023, 1057, 1026, 999, 1037, 1008, 1010, 1019, 1020, 1003, 1038, 1018, 963, 1025, 1053, 999, 1042, 1087, 995, 1045, 1037, 1037, 1063, 1036, 1012, 1046, 963, 1043, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1063, 1025, 995, 1021, 1029, 1043, 1035, 1003, 1009, 1012, 1074, 1005, 991, 1046, 1039, 1047, 996, 1054, 1002, 985, 991, 1048, 987, 993, 1000, 1028, 1040, 1036, 1067, 1022, 1050, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1073, 1028, 1009, 989, 1016, 1006, 1007, 1044, 1016, 999, 1034, 1052, 1046, 1017, 1059, 988, 983, 982, 982, 1037, 998, 1053, 979, 1058, 1103, 1072, 1010, 1024, 988, 1027, 1037, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1029, 1036, 994, 1049, 1040, 1033, 1041, 1031, 1061, 1020, 1012, 1032, 1034, 1005, 1023, 1010, 976, 1006, 977, 1045, 1017, 1064, 1014, 1072, 983, 1009, 1075, 1043, 981, 1060, 985, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1029, 1043, 1002, 1040, 994, 1029, 1044, 1059, 1036, 1046, 1028, 1008, 1101, 993, 1028, 989, 1050, 989, 1022, 1038, 1003, 1013, 1034, 1039, 1012, 1018, 998, 1042, 1009, 1018, 1006, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1063, 1049, 1041, 1011, 1011, 1034, 1041, 1039, 1028, 1043, 1016, 1086, 985, 1003, 998, 1059, 967, 1013, 998, 1026, 1040, 1007, 1002, 1041, 1009, 1049, 982, 1048, 984, 1070, 998, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[961, 999, 1075, 1085, 0, 0, 0, 0, 1011, 1009, 1018, 1027, 0, 0, 0, 0, 1032, 1000, 1011, 993, 9, 0, 0, 0, 1021, 986, 993, 1060, 5, 5, 0, 0, 1041, 999, 972, 992, 31, 19, 0, 0, 1049, 990, 1029, 1087, 0, 0, 0, 0, 1023, 1039, 1048, 1017, 0, 0, 0, 0, 1031, 1034, 1045, 1014, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4065, 4045, 4070, 4054, 4155, 4127, 4132] avg=4096.0000 max=4155 max_rank=5 min=4045 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1028, 998, 1017, 1022, 0, 0, 0, 0, 1072, 1031, 1041, 1010, 0, 0, 0, 0, 967, 985, 1012, 1017, 9, 0, 0, 0, 1053, 1040, 1019, 1019, 6, 5, 0, 0, 1009, 1008, 1065, 1039, 30, 18, 0, 0, 1026, 950, 1043, 1026, 0, 0, 0, 0, 1026, 1020, 999, 1050, 0, 0, 0, 0, 1034, 1064, 1021, 982, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4154, 3990, 4142, 4169, 4045, 4095, 4108] avg=4096.0000 max=4169 max_rank=4 min=3990 min_rank=2 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[996, 994, 1071, 1023, 0, 0, 0, 0, 1057, 1026, 999, 1014, 0, 0, 0, 0, 1008, 1010, 1019, 1020, 9, 0, 0, 0, 1003, 1038, 1018, 963, 6, 5, 0, 0, 1025, 1053, 999, 1042, 31, 18, 0, 0, 1087, 995, 1045, 1030, 0, 0, 0, 0, 1037, 1032, 1036, 1012, 0, 0, 0, 0, 1046, 963, 1043, 988, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4096, 4066, 4033, 4168, 4157, 4117, 4047] avg=4096.0000 max=4168 max_rank=4 min=4033 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1058, 1028, 1009, 989, 0, 0, 0, 0, 1016, 1006, 1007, 1020, 0, 0, 0, 0, 1016, 999, 1034, 1052, 9, 0, 0, 0, 1046, 1017, 1059, 988, 6, 5, 0, 0, 983, 982, 982, 1037, 31, 19, 0, 0, 998, 1053, 979, 1050, 0, 0, 0, 0, 1103, 1041, 1010, 1024, 0, 0, 0, 0, 988, 1027, 1037, 1052, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4049, 4110, 4121, 4034, 4080, 4178, 4112] avg=4096.0000 max=4178 max_rank=6 min=4034 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1014, 1043, 1002, 1040, 0, 0, 0, 0, 994, 1029, 1044, 1035, 0, 0, 0, 0, 1036, 1046, 1028, 1008, 9, 0, 0, 0, 1101, 993, 1028, 989, 6, 5, 0, 0, 1050, 989, 1022, 1038, 29, 19, 0, 0, 1003, 1013, 1034, 1032, 0, 0, 0, 0, 1012, 989, 998, 1042, 0, 0, 0, 0, 1009, 1018, 1006, 1008, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4102, 4127, 4122, 4147, 4082, 4041, 4048] avg=4096.0000 max=4147 max_rank=4 min=4041 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1033, 1025, 995, 1021, 0, 0, 0, 0, 1029, 1043, 1035, 972, 0, 0, 0, 0, 1009, 1012, 1074, 1005, 14, 0, 0, 0, 991, 1046, 1039, 1047, 9, 9, 0, 0, 996, 1054, 1002, 985, 33, 22, 0, 0, 991, 1048, 987, 981, 0, 0, 0, 0, 1000, 995, 1040, 1036, 0, 0, 0, 0, 1067, 1022, 1050, 1032, 12, 7, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4079, 4114, 4141, 4092, 4007, 4071, 4190] avg=4096.0000 max=4190 max_rank=7 min=4007 min_rank=5 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1014, 1036, 994, 1049, 0, 0, 0, 0, 1040, 1033, 1041, 1008, 0, 0, 0, 0, 1061, 1020, 1012, 1032, 9, 0, 0, 0, 1034, 1005, 1023, 1010, 6, 5, 0, 0, 976, 1006, 977, 1045, 29, 18, 0, 0, 1017, 1064, 1014, 1064, 0, 0, 0, 0, 983, 980, 1075, 1043, 0, 0, 0, 0, 981, 1060, 985, 1011, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4122, 4134, 4083, 4051, 4159, 4081, 4045] avg=4096.0000 max=4159 max_rank=5 min=4045 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1048, 1049, 1041, 1011, 0, 0, 0, 0, 1011, 1034, 1041, 1015, 0, 0, 0, 0, 1028, 1043, 1016, 1086, 9, 0, 0, 0, 985, 1003, 998, 1059, 6, 5, 0, 0, 967, 1013, 998, 1026, 30, 19, 0, 0, 1040, 1007, 1002, 1034, 0, 0, 0, 0, 1009, 1019, 982, 1048, 0, 0, 0, 0, 984, 1070, 998, 1027, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4101, 4182, 4056, 4053, 4083, 4058, 4086] avg=4096.0000 max=4182 max_rank=2 min=4053 min_rank=4 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8064, 8320, 256, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1046, 1014, 1040, 1026, 1063, 1031, 1028, 1010, 1003, 1054, 1001, 1038, 1028, 1040, 1048, 990, 1041, 1044, 1004, 945, 981, 1071, 1021, 1019, 1026, 972, 1057, 1045, 1020, 1008, 1038, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[996, 1046, 1054, 1013, 1018, 1027, 999, 998, 1024, 995, 996, 1005, 1050, 1044, 1046, 1058, 998, 1018, 1038, 984, 1000, 1020, 1039, 1067, 1002, 996, 996, 1021, 1098, 1063, 1020, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[989, 1006, 1069, 1056, 1054, 1013, 1041, 1005, 1031, 1012, 1011, 1010, 1019, 999, 1068, 988, 1003, 1014, 1054, 1021, 1015, 992, 996, 1050, 1042, 1018, 1035, 1069, 1007, 1021, 1046, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1025, 1041, 1005, 1013, 1010, 1035, 1066, 1056, 1000, 1032, 1004, 1064, 1022, 1014, 1015, 1048, 1025, 992, 1085, 1029, 1007, 1010, 1067, 1016, 1012, 1020, 1008, 1064, 961, 1021, 1011, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1055, 1025, 1030, 1021, 1018, 988, 1023, 1048, 1000, 1036, 1044, 1049, 1025, 1016, 1029, 1046, 1025, 990, 1026, 1026, 1027, 1016, 1053, 1036, 1036, 964, 1014, 1065, 1026, 1005, 982, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1038, 1024, 1022, 995, 976, 1019, 1071, 1021, 1046, 997, 999, 1065, 1008, 990, 1025, 1052, 1019, 1048, 1021, 1087, 1000, 981, 999, 1032, 1031, 1035, 1058, 1003, 1042, 998, 1067, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[980, 1056, 1053, 1057, 1027, 1065, 1033, 1060, 1035, 963, 1030, 1036, 997, 977, 1025, 1025, 997, 1060, 984, 986, 1056, 1020, 973, 1065, 1026, 985, 1018, 1064, 1003, 1020, 1033, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1043, 1052, 1016, 978, 1022, 1042, 1091, 982, 989, 1027, 993, 1014, 1036, 1029, 985, 1028, 1051, 1027, 1053, 1026, 999, 1037, 1043, 984, 1025, 1012, 1036, 1058, 1018, 983, 1046, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[989, 1006, 1055, 1056, 0, 0, 0, 0, 1054, 1013, 1021, 1005, 0, 0, 0, 0, 1031, 1012, 1011, 1010, 20, 0, 0, 0, 1019, 999, 1068, 988, 0, 0, 0, 0, 1003, 1014, 1054, 1021, 6, 0, 0, 0, 1015, 992, 996, 1050, 8, 0, 0, 0, 1042, 1018, 1035, 1064, 0, 0, 0, 0, 1007, 1021, 1046, 1014, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4093, 4084, 4074, 4098, 4061, 4159, 4093] avg=4096.0000 max=4159 max_rank=6 min=4061 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16411 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1055, 1025, 1017, 1021, 0, 0, 0, 0, 1018, 988, 1003, 1048, 0, 0, 0, 0, 1000, 1036, 1044, 1049, 20, 0, 0, 0, 1025, 1016, 1029, 1046, 0, 0, 0, 0, 1025, 990, 1026, 1026, 5, 0, 0, 0, 1027, 1016, 1053, 1036, 8, 0, 0, 0, 1036, 964, 1014, 1060, 0, 0, 0, 0, 1026, 1005, 982, 1024, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4057, 4149, 4116, 4072, 4140, 4074, 4042] avg=4096.0000 max=4149 max_rank=2 min=4042 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16328 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1046, 1014, 1027, 1026, 0, 0, 0, 0, 1063, 1031, 1008, 1010, 0, 0, 0, 0, 1003, 1054, 1001, 1038, 20, 0, 0, 0, 1028, 1040, 1048, 990, 0, 0, 0, 0, 1041, 1044, 1004, 945, 5, 0, 0, 0, 981, 1071, 1021, 1019, 8, 0, 0, 0, 1026, 972, 1057, 1040, 0, 0, 0, 0, 1020, 1008, 1038, 1016, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4112, 4116, 4106, 4039, 4100, 4095, 4087] avg=4096.0000 max=4116 max_rank=2 min=4039 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16321 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[996, 1046, 1041, 1013, 0, 0, 0, 0, 1018, 1027, 980, 998, 0, 0, 0, 0, 1024, 995, 996, 1005, 19, 0, 0, 0, 1050, 1044, 1046, 1058, 0, 0, 0, 0, 998, 1018, 1038, 984, 5, 0, 0, 0, 1000, 1020, 1039, 1067, 8, 0, 0, 0, 1002, 996, 996, 1016, 0, 0, 0, 0, 1098, 1063, 1020, 1039, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4023, 4039, 4198, 4043, 4134, 4010, 4225] avg=4096.0000 max=4225 max_rank=7 min=4010 min_rank=6 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16412 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1025, 1041, 992, 1013, 0, 0, 0, 0, 1010, 1035, 1045, 1056, 0, 0, 0, 0, 1000, 1032, 1004, 1064, 21, 0, 0, 0, 1022, 1014, 1015, 1048, 0, 0, 0, 0, 1025, 992, 1085, 1029, 5, 0, 0, 0, 1007, 1010, 1067, 1016, 8, 0, 0, 0, 1012, 1020, 1008, 1059, 0, 0, 0, 0, 961, 1021, 1011, 990, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4146, 4121, 4099, 4136, 4108, 4099, 3988] avg=4096.0000 max=4146 max_rank=1 min=3988 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16331 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[980, 1056, 1040, 1057, 0, 0, 0, 0, 1027, 1065, 1013, 1060, 0, 0, 0, 0, 1035, 963, 1030, 1036, 20, 0, 0, 0, 997, 977, 1025, 1025, 0, 0, 0, 0, 997, 1060, 984, 986, 5, 0, 0, 0, 1056, 1020, 973, 1065, 8, 0, 0, 0, 1026, 985, 1018, 1059, 0, 0, 0, 0, 1003, 1020, 1033, 1059, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4165, 4084, 4024, 4032, 4122, 4088, 4120] avg=4096.0000 max=4165 max_rank=1 min=4024 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16362 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1038, 1024, 998, 995, 0, 0, 0, 0, 976, 1019, 1040, 1021, 0, 0, 0, 0, 1046, 997, 999, 1065, 24, 0, 0, 0, 1008, 990, 1023, 1052, 0, 0, 0, 0, 1019, 1048, 1021, 1087, 11, 0, 0, 0, 1000, 981, 999, 1032, 13, 7, 0, 0, 1031, 1035, 1058, 993, 0, 0, 0, 0, 1042, 998, 1067, 999, 10, 2, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4056, 4131, 4073, 4186, 4032, 4117, 4118] avg=4096.0000 max=4186 max_rank=4 min=4032 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16453 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1043, 1052, 1003, 978, 0, 0, 0, 0, 1022, 1042, 1070, 982, 0, 0, 0, 0, 989, 1027, 993, 1014, 21, 0, 0, 0, 1036, 1029, 985, 1028, 0, 0, 0, 0, 1051, 1027, 1053, 1026, 5, 0, 0, 0, 999, 1037, 1043, 984, 8, 0, 0, 0, 1025, 1012, 1036, 1053, 0, 0, 0, 0, 1018, 983, 1046, 1043, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4116, 4044, 4078, 4162, 4071, 4126, 4095] avg=4096.0000 max=4162 max_rank=4 min=4044 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16454 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1014, 1049, 1027, 954, 1048, 950, 1049, 1038, 1015, 1012, 1024, 997, 1003, 1054, 1019, 1015, 1034, 1000, 1046, 1052, 1029, 1037, 996, 1084, 1079, 1017, 994, 1009, 1057, 1036, 1035, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1049, 1025, 1043, 1101, 973, 982, 986, 1048, 1053, 1038, 994, 1040, 1010, 1052, 1018, 1004, 1024, 1004, 993, 998, 1051, 1033, 1048, 1033, 1037, 982, 985, 1027, 988, 1040, 1031, 1078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[984, 988, 1033, 976, 1045, 1063, 1038, 985, 1031, 1073, 1013, 1036, 989, 1001, 1034, 1026, 1029, 1064, 976, 1013, 1061, 1008, 1075, 1035, 992, 1083, 1004, 1035, 990, 1011, 1052, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1030, 1010, 1056, 1005, 990, 1009, 1076, 1009, 1045, 1093, 1043, 1007, 1004, 1025, 1063, 1017, 1022, 973, 969, 1044, 1025, 1036, 989, 1022, 1005, 1037, 1020, 1040, 1010, 1017, 1052, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1055, 1009, 1059, 1073, 1018, 1006, 998, 1106, 1045, 1044, 1051, 1015, 1025, 1017, 1017, 1033, 1020, 1008, 981, 981, 1015, 1012, 995, 1010, 1028, 1042, 1001, 992, 1046, 997, 998, 1071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[986, 984, 1004, 1022, 1055, 1016, 1072, 1014, 1037, 996, 1008, 1057, 1008, 990, 985, 1042, 1039, 1056, 1005, 1025, 1071, 1000, 1036, 1016, 1003, 1021, 1044, 1033, 1063, 1072, 1026, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1023, 993, 1050, 1034, 1018, 1074, 1064, 1054, 959, 1018, 1022, 1037, 1065, 1056, 1001, 977, 1038, 1037, 967, 1040, 1045, 1025, 1006, 1050, 1038, 994, 1044, 967, 961, 1041, 1007, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1018, 1004, 1033, 1044, 1003, 1033, 1023, 975, 949, 1051, 1035, 1072, 1029, 1052, 1031, 1057, 1023, 1011, 1027, 1035, 985, 1032, 1008, 1053, 1026, 1001, 1003, 1036, 1043, 1022, 1056, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1030, 1010, 1056, 1005, 4, 0, 0, 0, 990, 1009, 1070, 1009, 0, 0, 0, 0, 1045, 1075, 1043, 1007, 0, 0, 0, 0, 1004, 1025, 1063, 1017, 4, 2, 0, 0, 1022, 973, 969, 1044, 18, 10, 0, 0, 1025, 1036, 989, 1004, 0, 0, 0, 0, 1005, 1037, 1020, 1040, 11, 8, 0, 0, 1010, 1017, 1037, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4078, 4170, 4115, 4036, 4054, 4121, 4089] avg=4096.0000 max=4170 max_rank=2 min=4036 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1014, 1049, 1027, 954, 4, 0, 0, 0, 1048, 950, 1043, 1038, 0, 0, 0, 0, 1015, 996, 1024, 997, 0, 0, 0, 0, 1003, 1054, 1019, 1015, 4, 2, 0, 0, 1034, 1000, 1046, 1052, 19, 9, 0, 0, 1029, 1037, 996, 1065, 0, 0, 0, 0, 1079, 1017, 994, 1009, 11, 7, 0, 0, 1057, 1036, 1020, 995, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4079, 4032, 4097, 4160, 4127, 4117, 4108] avg=4096.0000 max=4160 max_rank=4 min=4032 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[984, 988, 1033, 976, 4, 0, 0, 0, 1045, 1063, 1032, 985, 0, 0, 0, 0, 1031, 1056, 1013, 1036, 0, 0, 0, 0, 989, 1001, 1034, 1026, 4, 2, 0, 0, 1029, 1064, 976, 1013, 19, 10, 0, 0, 1061, 1008, 1075, 1016, 0, 0, 0, 0, 992, 1083, 1004, 1035, 11, 7, 0, 0, 990, 1011, 1037, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3985, 4125, 4136, 4056, 4111, 4160, 4132, 4063] avg=4096.0000 max=4160 max_rank=5 min=3985 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1049, 1025, 1043, 1101, 3, 0, 0, 0, 973, 982, 981, 1048, 0, 0, 0, 0, 1053, 1021, 994, 1040, 0, 0, 0, 0, 1010, 1052, 1018, 1004, 3, 2, 0, 0, 1024, 1004, 993, 998, 19, 10, 0, 0, 1051, 1033, 1048, 1014, 0, 0, 0, 0, 1037, 982, 985, 1027, 10, 7, 0, 0, 988, 1040, 1018, 1078, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4221, 3984, 4108, 4089, 4048, 4146, 4048, 4124] avg=4096.0000 max=4221 max_rank=0 min=3984 min_rank=1 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1055, 1009, 1059, 1073, 3, 0, 0, 0, 1018, 1006, 993, 1106, 0, 0, 0, 0, 1045, 1027, 1051, 1015, 0, 0, 0, 0, 1025, 1017, 1017, 1033, 3, 2, 0, 0, 1020, 1008, 981, 981, 18, 10, 0, 0, 1015, 1012, 995, 992, 0, 0, 0, 0, 1028, 1042, 1001, 992, 10, 7, 0, 0, 1046, 997, 985, 1071, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4199, 4123, 4138, 4097, 4018, 4014, 4080, 4099] avg=4096.0000 max=4199 max_rank=0 min=4014 min_rank=5 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16211 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1023, 993, 1050, 1034, 4, 0, 0, 0, 1018, 1074, 1058, 1054, 0, 0, 0, 0, 959, 1002, 1022, 1037, 0, 0, 0, 0, 1065, 1056, 1001, 977, 3, 2, 0, 0, 1038, 1037, 967, 1040, 19, 9, 0, 0, 1045, 1025, 1006, 1031, 0, 0, 0, 0, 1038, 994, 1044, 967, 10, 7, 0, 0, 961, 1041, 994, 1063, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4204, 4020, 4104, 4110, 4107, 4060, 4059] avg=4096.0000 max=4204 max_rank=1 min=4020 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1018, 1004, 1033, 1044, 4, 0, 0, 0, 1003, 1033, 1017, 975, 0, 0, 0, 0, 949, 1034, 1035, 1072, 0, 0, 0, 0, 1029, 1052, 1031, 1057, 4, 2, 0, 0, 1023, 1011, 1027, 1035, 19, 10, 0, 0, 985, 1032, 1008, 1034, 0, 0, 0, 0, 1026, 1001, 1003, 1036, 11, 7, 0, 0, 1043, 1022, 1041, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4028, 4090, 4175, 4125, 4059, 4084, 4104] avg=4096.0000 max=4175 max_rank=3 min=4028 min_rank=1 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[986, 984, 1004, 1022, 7, 0, 0, 0, 1055, 1016, 1062, 1014, 0, 0, 0, 0, 1037, 972, 1008, 1057, 0, 0, 0, 0, 1008, 990, 985, 1042, 7, 3, 0, 0, 1039, 1056, 1005, 1025, 22, 13, 0, 0, 1071, 1000, 1036, 994, 0, 0, 0, 0, 1003, 1021, 1044, 1033, 14, 11, 0, 0, 1063, 1072, 1005, 982, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4003, 4147, 4074, 4035, 4160, 4101, 4126, 4122] avg=4096.0000 max=4160 max_rank=4 min=4003 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1023, 973, 1002, 1032, 1043, 982, 1032, 1052, 1018, 1000, 1044, 992, 1024, 999, 976, 1000, 1045, 1051, 978, 1108, 1037, 1032, 1042, 1030, 1014, 1061, 1012, 1061, 1052, 1023, 1025, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1008, 1038, 1043, 1016, 1004, 1029, 1026, 1024, 1062, 1004, 1035, 1006, 1038, 977, 1002, 1051, 1007, 995, 1048, 1025, 1086, 1011, 998, 1042, 1023, 1011, 1023, 1030, 993, 1051, 1025, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1043, 1036, 1044, 1003, 1032, 1042, 1008, 989, 998, 1009, 1039, 1046, 1001, 1024, 1025, 1038, 953, 1001, 989, 1015, 1061, 1004, 1029, 1052, 1091, 1002, 1074, 1031, 1030, 1000, 1042, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[955, 1014, 1056, 1050, 1044, 1065, 1033, 1006, 1028, 1040, 1039, 991, 1032, 999, 996, 1036, 1035, 1058, 1013, 1038, 1032, 1041, 991, 1017, 997, 1000, 985, 1051, 1000, 1040, 1065, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1042, 1005, 1024, 954, 1088, 972, 1013, 1039, 1021, 1054, 992, 1112, 1066, 1031, 998, 1001, 1007, 988, 1017, 983, 1014, 1030, 1044, 1044, 1057, 1000, 1025, 1032, 1020, 1045, 1029, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1026, 1045, 1061, 1007, 981, 1027, 1043, 1020, 1043, 1037, 997, 1019, 1047, 980, 1015, 1021, 1036, 1012, 1032, 1058, 1015, 1048, 960, 1015, 970, 999, 1053, 1047, 1059, 1025, 1008, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1044, 1026, 1006, 1014, 1038, 1020, 1025, 991, 1057, 1083, 1001, 1049, 1038, 1017, 947, 978, 1066, 1042, 1030, 1089, 1001, 1004, 978, 1015, 1008, 1016, 1041, 1028, 1021, 967, 1096, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[965, 1044, 1012, 1040, 1040, 1043, 992, 1028, 1008, 989, 1029, 1039, 994, 1052, 1023, 1033, 1027, 1009, 995, 1028, 1012, 1046, 1043, 1069, 1012, 1049, 1024, 1029, 1049, 1032, 1011, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1008, 1038, 1043, 1016, 9, 3, 1, 0, 1004, 1029, 1026, 1024, 0, 0, 0, 0, 1062, 1004, 1035, 993, 0, 0, 0, 0, 1038, 977, 1002, 1051, 16, 13, 7, 0, 1007, 995, 1048, 1024, 0, 0, 0, 0, 1086, 1011, 998, 1033, 0, 0, 0, 0, 1023, 1011, 1023, 1020, 0, 0, 0, 0, 993, 1051, 1009, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4083, 4094, 4104, 4074, 4128, 4077, 4090] avg=4096.0000 max=4128 max_rank=5 min=4074 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1023, 973, 1002, 1032, 9, 3, 1, 0, 1043, 982, 1032, 1052, 0, 0, 0, 0, 1018, 1000, 1044, 979, 0, 0, 0, 0, 1024, 999, 976, 1000, 16, 13, 7, 0, 1045, 1051, 978, 1107, 0, 0, 0, 0, 1037, 1032, 1042, 1021, 0, 0, 0, 0, 1014, 1061, 1012, 1051, 0, 0, 0, 0, 1052, 1023, 1009, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4109, 4041, 4035, 4181, 4132, 4138, 4089] avg=4096.0000 max=4181 max_rank=4 min=4035 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[955, 1014, 1056, 1050, 9, 3, 1, 0, 1044, 1065, 1033, 1006, 0, 0, 0, 0, 1028, 1040, 1039, 978, 0, 0, 0, 0, 1032, 999, 996, 1036, 17, 13, 7, 0, 1035, 1058, 1013, 1037, 0, 0, 0, 0, 1032, 1041, 991, 1008, 0, 0, 0, 0, 997, 1000, 985, 1041, 0, 0, 0, 0, 1000, 1040, 1048, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4148, 4085, 4100, 4143, 4072, 4023, 4109] avg=4096.0000 max=4148 max_rank=1 min=4023 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 1045, 1061, 1007, 9, 3, 1, 0, 981, 1027, 1043, 1020, 0, 0, 0, 0, 1043, 1037, 997, 1006, 0, 0, 0, 0, 1047, 980, 1015, 1021, 16, 13, 7, 0, 1036, 1012, 1032, 1057, 0, 0, 0, 0, 1015, 1048, 960, 1006, 0, 0, 0, 0, 970, 999, 1053, 1037, 0, 0, 0, 0, 1059, 1025, 992, 1062, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4071, 4083, 4099, 4137, 4029, 4059, 4138] avg=4096.0000 max=4152 max_rank=0 min=4029 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[965, 1044, 1012, 1040, 9, 3, 1, 0, 1040, 1043, 992, 1028, 0, 0, 0, 0, 1008, 989, 1029, 1025, 0, 0, 0, 0, 994, 1052, 1023, 1033, 16, 14, 7, 0, 1027, 1009, 995, 1027, 0, 0, 0, 0, 1012, 1046, 1043, 1060, 0, 0, 0, 0, 1012, 1049, 1024, 1019, 0, 0, 0, 0, 1049, 1032, 995, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4103, 4051, 4139, 4058, 4161, 4104, 4078] avg=4096.0000 max=4161 max_rank=5 min=4051 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1043, 1036, 1044, 1003, 12, 8, 3, 3, 1029, 1042, 1008, 989, 0, 0, 0, 0, 998, 1009, 1039, 1028, 0, 0, 0, 0, 1001, 1024, 1025, 1038, 22, 18, 10, 0, 953, 1001, 989, 1012, 0, 0, 0, 0, 1061, 1004, 1029, 1040, 0, 0, 0, 0, 1091, 1002, 1074, 1013, 0, 0, 0, 0, 1030, 1000, 1020, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4068, 4074, 4138, 3955, 4134, 4180, 4067] avg=4096.0000 max=4180 max_rank=6 min=3955 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1044, 1026, 1006, 1014, 9, 3, 1, 0, 1038, 1020, 1025, 991, 0, 0, 0, 0, 1057, 1083, 1001, 1035, 0, 0, 0, 0, 1038, 1017, 947, 978, 18, 14, 7, 0, 1066, 1042, 1030, 1088, 0, 0, 0, 0, 1001, 1004, 978, 1006, 0, 0, 0, 0, 1008, 1016, 1041, 1018, 0, 0, 0, 0, 1021, 967, 1078, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4074, 4176, 4019, 4226, 3989, 4083, 4098] avg=4096.0000 max=4226 max_rank=4 min=3989 min_rank=5 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1042, 1005, 1024, 954, 9, 3, 1, 0, 1088, 972, 1013, 1039, 0, 0, 0, 0, 1021, 1054, 992, 1097, 0, 0, 0, 0, 1066, 1031, 998, 1001, 16, 15, 7, 0, 1007, 988, 1017, 982, 0, 0, 0, 0, 1014, 1030, 1044, 1035, 0, 0, 0, 0, 1057, 1000, 1025, 1022, 0, 0, 0, 0, 1020, 1045, 1013, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4112, 4164, 4134, 3994, 4123, 4104, 4099] avg=4096.0000 max=4164 max_rank=2 min=3994 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1028, 997, 999, 1064, 1030, 976, 1012, 982, 1002, 1049, 1058, 1051, 1043, 1061, 1037, 1044, 1023, 995, 1012, 1031, 1001, 1005, 1010, 978, 1025, 1044, 1050, 1006, 1045, 1052, 1033, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1017, 1040, 959, 1029, 1023, 1073, 994, 1041, 1006, 1000, 1032, 1078, 1001, 1008, 1001, 1042, 999, 1032, 981, 1051, 1051, 999, 1069, 1028, 997, 1061, 1015, 1067, 1016, 1044, 990, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[998, 1035, 1025, 1046, 1042, 990, 1044, 1016, 975, 988, 1027, 1006, 1048, 1039, 1071, 1027, 1032, 1078, 1010, 1003, 1043, 964, 1053, 979, 1036, 1037, 1053, 1037, 1053, 999, 997, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1028, 992, 1043, 1041, 1053, 1018, 1080, 1032, 1082, 1002, 1030, 989, 1030, 960, 1048, 971, 1034, 1017, 1050, 1009, 1003, 1008, 1024, 990, 1011, 1028, 1084, 1008, 1027, 1035, 1022, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1002, 1035, 1006, 1060, 1038, 1010, 1039, 1054, 984, 989, 1028, 1039, 1032, 1076, 1022, 995, 1041, 1030, 977, 1017, 1013, 1073, 1008, 1040, 1024, 1017, 1017, 1019, 993, 1024, 1092, 974] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1042, 1085, 939, 1019, 1026, 1015, 981, 1025, 1067, 1047, 1031, 1009, 1000, 990, 1035, 1015, 1089, 964, 1004, 1056, 1043, 1025, 1021, 1047, 1026, 986, 1050, 1010, 1060, 1040, 1014, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1009, 1009, 1038, 1038, 1002, 988, 1013, 1024, 1016, 1056, 1040, 1037, 1039, 1012, 1058, 989, 1047, 1016, 1006, 1019, 1078, 1032, 1019, 1048, 977, 1026, 1033, 1016, 1020, 978, 1032, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1033, 1002, 1034, 1049, 996, 1001, 990, 1034, 985, 1019, 1030, 1016, 1009, 1043, 1042, 1044, 1027, 1022, 1072, 1043, 1032, 993, 1040, 996, 985, 1034, 1076, 1015, 969, 1041, 1086, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[998, 1035, 1025, 1046, 2, 0, 0, 0, 1042, 990, 1044, 1016, 12, 3, 0, 0, 975, 988, 1027, 1006, 0, 0, 0, 0, 1048, 1039, 1063, 1027, 0, 0, 0, 0, 1030, 1078, 1010, 1003, 0, 0, 0, 0, 1043, 964, 1053, 979, 5, 1, 0, 0, 1036, 1037, 1041, 1037, 0, 0, 0, 0, 1053, 999, 996, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4107, 3996, 4177, 4121, 4045, 4151, 4065] avg=4096.0000 max=4177 max_rank=3 min=3996 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1028, 992, 1043, 1041, 2, 0, 0, 0, 1053, 1018, 1080, 1032, 13, 3, 0, 0, 1082, 1002, 1030, 989, 0, 0, 0, 0, 1030, 960, 1040, 971, 0, 0, 0, 0, 1032, 1017, 1050, 1009, 0, 0, 0, 0, 1003, 1008, 1024, 990, 5, 1, 0, 0, 1011, 1028, 1071, 1008, 0, 0, 0, 0, 1027, 1035, 1021, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4199, 4103, 4001, 4108, 4031, 4118, 4102] avg=4096.0000 max=4199 max_rank=1 min=4001 min_rank=3 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16359 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1002, 1035, 1006, 1060, 2, 1, 0, 0, 1038, 1010, 1039, 1054, 12, 2, 0, 0, 984, 989, 1028, 1039, 0, 0, 0, 0, 1032, 1076, 1016, 995, 0, 0, 0, 0, 1039, 1030, 977, 1017, 0, 0, 0, 0, 1013, 1073, 1008, 1040, 4, 1, 0, 0, 1024, 1017, 1005, 1019, 0, 0, 0, 0, 993, 1024, 1090, 974, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4155, 4040, 4119, 4063, 4139, 4065, 4081] avg=4096.0000 max=4155 max_rank=1 min=4040 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16348 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1017, 1040, 959, 1029, 2, 0, 0, 0, 1023, 1073, 994, 1041, 12, 2, 0, 0, 1006, 1000, 1032, 1078, 0, 0, 0, 0, 1001, 1008, 995, 1042, 0, 0, 0, 0, 997, 1032, 981, 1051, 0, 0, 0, 0, 1051, 999, 1069, 1028, 4, 1, 0, 0, 997, 1061, 1003, 1067, 0, 0, 0, 0, 1016, 1044, 989, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4145, 4116, 4046, 4061, 4152, 4128, 4073] avg=4096.0000 max=4152 max_rank=5 min=4046 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16414 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1042, 1085, 939, 1019, 2, 0, 0, 0, 1026, 1015, 981, 1025, 12, 2, 0, 0, 1067, 1047, 1031, 1009, 0, 0, 0, 0, 1000, 990, 1029, 1015, 0, 0, 0, 0, 1087, 964, 1004, 1056, 0, 0, 0, 0, 1043, 1025, 1021, 1047, 4, 1, 0, 0, 1026, 986, 1038, 1010, 0, 0, 0, 0, 1060, 1040, 1013, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4061, 4154, 4034, 4111, 4141, 4060, 4120] avg=4096.0000 max=4154 max_rank=2 min=4034 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16432 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1028, 997, 999, 1064, 5, 6, 0, 0, 1030, 976, 1012, 982, 16, 6, 0, 0, 1002, 1049, 1058, 1051, 0, 0, 0, 0, 1043, 1061, 1023, 1044, 0, 0, 0, 0, 1018, 995, 1012, 1031, 0, 0, 0, 0, 1001, 1005, 1010, 978, 8, 8, 0, 0, 1025, 1044, 1034, 1006, 0, 0, 0, 0, 1045, 1052, 1019, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4022, 4160, 4171, 4056, 4010, 4109, 4141] avg=4096.0000 max=4171 max_rank=3 min=4010 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16316 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1009, 1009, 1038, 1038, 2, 0, 0, 0, 1002, 988, 1013, 1024, 12, 3, 0, 0, 1016, 1056, 1040, 1037, 0, 0, 0, 0, 1039, 1012, 1050, 989, 0, 0, 0, 0, 1045, 1016, 1006, 1019, 0, 0, 0, 0, 1078, 1032, 1019, 1048, 5, 1, 0, 0, 977, 1026, 1021, 1016, 0, 0, 0, 0, 1020, 978, 1031, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4042, 4149, 4090, 4086, 4183, 4040, 4082] avg=4096.0000 max=4183 max_rank=5 min=4040 min_rank=6 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16391 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1033, 1002, 1034, 1049, 2, 1, 0, 0, 996, 1001, 990, 1034, 13, 3, 0, 0, 985, 1019, 1030, 1016, 0, 0, 0, 0, 1009, 1043, 1034, 1044, 0, 0, 0, 0, 1025, 1022, 1072, 1043, 0, 0, 0, 0, 1032, 993, 1040, 996, 5, 1, 0, 0, 985, 1034, 1063, 1015, 0, 0, 0, 0, 969, 1041, 1084, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4037, 4050, 4130, 4162, 4067, 4097, 4104] avg=4096.0000 max=4162 max_rank=4 min=4037 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16430 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8448, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1031, 1023, 1048, 1033, 1027, 1034, 976, 1036, 1017, 1036, 1002, 1073, 1014, 1025, 1025, 1037, 1020, 1029, 1003, 994, 1064, 1042, 999, 970, 1019, 1065, 1050, 1011, 996, 1026, 992, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1031, 1032, 1021, 1023, 1052, 988, 1019, 983, 1042, 1108, 946, 1014, 1001, 1032, 1044, 1003, 999, 1010, 1046, 1046, 1075, 1036, 1029, 1021, 1027, 1028, 1019, 1042, 1023, 986, 1023, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1043, 1029, 1018, 977, 1036, 1015, 1036, 1012, 1081, 1039, 1021, 1024, 1050, 1010, 1068, 1023, 1039, 1023, 981, 1013, 1068, 1014, 1038, 1010, 1003, 1071, 1002, 1020, 1007, 1025, 988, 984] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1026, 1022, 1041, 976, 1060, 988, 1014, 1066, 1050, 1038, 1045, 1005, 974, 1025, 1057, 1000, 1021, 1055, 1034, 1043, 1019, 1038, 1089, 999, 1035, 1044, 1027, 981, 1025, 1017, 970, 984] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1028, 1040, 1037, 1028, 1030, 1029, 1047, 1006, 1030, 1062, 1042, 983, 1008, 1024, 1074, 993, 1011, 1008, 1073, 1071, 993, 1026, 984, 1022, 1040, 995, 1026, 1031, 1006, 972, 996, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1068, 1021, 1035, 1016, 995, 1038, 1053, 1005, 1032, 1018, 1008, 1036, 1013, 997, 972, 1034, 1032, 1041, 1063, 1033, 1006, 1013, 1015, 1002, 1033, 1057, 1026, 1008, 1036, 1030, 1016, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1034, 989, 1006, 993, 1026, 1056, 988, 1033, 1002, 1057, 1020, 1049, 1020, 1047, 1005, 1003, 1007, 1026, 1043, 1045, 964, 1018, 1020, 1038, 1048, 1035, 1060, 1023, 1045, 991, 1037, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1068, 1025, 1037, 1037, 1058, 975, 990, 1008, 1019, 1003, 1026, 1021, 1012, 1038, 1010, 1037, 991, 1032, 1041, 1038, 1038, 1038, 1039, 1019, 1034, 1019, 1058, 1003, 1022, 992, 1042, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1036, 1029, 1018, 977, 0, 0, 0, 0, 1036, 1015, 1036, 1012, 5, 5, 0, 0, 1081, 1017, 1021, 1024, 0, 0, 0, 0, 1050, 1010, 1068, 1023, 11, 0, 0, 0, 1039, 1023, 965, 1013, 0, 0, 0, 0, 1068, 1014, 1038, 1010, 2, 0, 0, 0, 1003, 1049, 1002, 1020, 0, 0, 0, 0, 1007, 1025, 988, 984, 22, 22, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4109, 4143, 4162, 4040, 4132, 4074, 4048] avg=4096.0000 max=4162 max_rank=3 min=4040 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1024, 1023, 1048, 1033, 0, 0, 0, 0, 1027, 1034, 976, 1036, 5, 5, 0, 0, 1017, 1014, 1002, 1073, 0, 0, 0, 0, 1014, 1025, 1025, 1037, 11, 0, 0, 0, 1020, 1029, 987, 994, 0, 0, 0, 0, 1064, 1042, 999, 970, 2, 0, 0, 0, 1019, 1043, 1050, 1011, 0, 0, 0, 0, 996, 1026, 992, 1051, 22, 22, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4083, 4106, 4112, 4030, 4077, 4123, 4109] avg=4096.0000 max=4128 max_rank=0 min=4030 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1024, 1032, 1021, 1023, 0, 0, 0, 0, 1052, 988, 1019, 983, 5, 5, 0, 0, 1042, 1085, 946, 1014, 0, 0, 0, 0, 1001, 1032, 1044, 1003, 11, 0, 0, 0, 999, 1010, 1030, 1046, 0, 0, 0, 0, 1075, 1036, 1029, 1021, 2, 0, 0, 0, 1027, 1007, 1019, 1042, 0, 0, 0, 0, 1023, 986, 1023, 1019, 23, 21, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4052, 4087, 4091, 4085, 4163, 4095, 4095] avg=4096.0000 max=4163 max_rank=5 min=4052 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1019, 1022, 1041, 976, 0, 0, 0, 0, 1060, 988, 1014, 1066, 5, 5, 0, 0, 1050, 1016, 1045, 1005, 0, 0, 0, 0, 974, 1025, 1057, 1000, 11, 0, 0, 0, 1021, 1055, 1018, 1043, 0, 0, 0, 0, 1019, 1038, 1089, 999, 2, 0, 0, 0, 1035, 1023, 1027, 981, 0, 0, 0, 0, 1025, 1017, 970, 984, 22, 21, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4138, 4116, 4067, 4137, 4147, 4066, 4039] avg=4096.0000 max=4147 max_rank=5 min=4039 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1061, 1021, 1035, 1016, 0, 0, 0, 0, 995, 1038, 1053, 1005, 5, 5, 0, 0, 1032, 996, 1008, 1036, 0, 0, 0, 0, 1013, 997, 972, 1034, 11, 0, 0, 0, 1032, 1041, 1047, 1033, 0, 0, 0, 0, 1006, 1013, 1015, 1002, 2, 0, 0, 0, 1033, 1036, 1026, 1008, 0, 0, 0, 0, 1036, 1030, 1016, 1016, 22, 21, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4101, 4072, 4027, 4153, 4038, 4103, 4141] avg=4096.0000 max=4153 max_rank=4 min=4027 min_rank=3 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1021, 1040, 1037, 1028, 0, 0, 0, 0, 1030, 1029, 1047, 1006, 5, 5, 0, 0, 1030, 1040, 1042, 983, 0, 0, 0, 0, 1008, 1024, 1074, 993, 12, 0, 0, 0, 1011, 1008, 1056, 1071, 0, 0, 0, 0, 993, 1026, 984, 1022, 2, 0, 0, 0, 1040, 975, 1026, 1031, 0, 0, 0, 0, 1006, 972, 996, 1053, 22, 20, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4122, 4095, 4111, 4146, 4027, 4072, 4069] avg=4096.0000 max=4146 max_rank=4 min=4027 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1015, 989, 1006, 993, 0, 0, 0, 0, 1026, 1056, 988, 1033, 11, 8, 0, 0, 1002, 1030, 1020, 1049, 0, 0, 0, 0, 1020, 1047, 1005, 1003, 15, 0, 0, 0, 1007, 1026, 1013, 1045, 0, 0, 0, 0, 964, 1018, 1020, 1038, 8, 0, 0, 0, 1048, 1011, 1060, 1023, 0, 0, 0, 0, 1045, 991, 1037, 1040, 27, 24, 7, 0] rerouted_tokens_per_ep_rank=[4003, 4122, 4101, 4090, 4091, 4048, 4142, 4171] avg=4096.0000 max=4171 max_rank=7 min=4003 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1061, 1025, 1037, 1037, 0, 0, 0, 0, 1058, 975, 990, 1008, 5, 5, 0, 0, 1019, 982, 1026, 1021, 0, 0, 0, 0, 1012, 1038, 1010, 1037, 11, 0, 0, 0, 991, 1032, 1025, 1038, 0, 0, 0, 0, 1038, 1038, 1039, 1019, 2, 0, 0, 0, 1034, 998, 1058, 1003, 0, 0, 0, 0, 1022, 992, 1042, 998, 21, 21, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4041, 4048, 4108, 4086, 4136, 4093, 4096] avg=4096.0000 max=4160 max_rank=0 min=4041 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8064, 8192, 256, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1054, 1026, 1021, 1010, 1036, 1027, 1053, 1051, 1030, 1019, 983, 1018, 1029, 1020, 980, 1041, 1049, 990, 1052, 1025, 1042, 1037, 1014, 958, 1035, 980, 993, 1040, 1019, 1097, 975, 1064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[965, 1031, 1010, 1021, 1024, 1018, 1047, 998, 1021, 1040, 1037, 1023, 1044, 1036, 998, 1023, 1018, 1028, 1029, 1011, 1082, 1061, 1011, 1029, 1027, 1049, 1000, 1039, 1006, 984, 1054, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1024, 1019, 986, 1007, 1066, 1007, 1013, 1028, 997, 1024, 1040, 1027, 1054, 1025, 1053, 976, 1052, 1033, 999, 1020, 1004, 998, 1027, 990, 1022, 989, 1079, 1020, 1067, 1046, 1034, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1021, 1002, 1052, 1052, 1043, 992, 1045, 1062, 1014, 984, 985, 1006, 1021, 1020, 1007, 1036, 1047, 1046, 1016, 992, 1014, 1026, 1037, 996, 1025, 1072, 1011, 1027, 1047, 1030, 1029, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1024, 1045, 1028, 1039, 1034, 1021, 1064, 995, 979, 1011, 980, 1008, 1044, 1033, 1045, 1034, 989, 1008, 1030, 1000, 1051, 991, 1051, 1031, 1020, 1036, 1019, 1033, 982, 1037, 1055, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1019, 1099, 1040, 970, 995, 1057, 1029, 1016, 1059, 1040, 975, 985, 966, 978, 987, 1022, 991, 1014, 1037, 1028, 1047, 1072, 974, 1008, 1061, 1087, 1044, 1029, 1053, 961, 1063, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1005, 1004, 1026, 1076, 1014, 1007, 1005, 1021, 1050, 1007, 1035, 1051, 1050, 1034, 1009, 949, 985, 1062, 1005, 1081, 1021, 1071, 996, 1001, 1060, 992, 1035, 1064, 1020, 999, 990, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1026, 1038, 1008, 1028, 1028, 1031, 1032, 1035, 1002, 990, 1026, 967, 1040, 1007, 1066, 1028, 1054, 997, 1003, 1047, 1017, 990, 1069, 1033, 1019, 1028, 1072, 1049, 998, 1019, 988, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[965, 1031, 1010, 1021, 0, 0, 0, 0, 1024, 1018, 1032, 998, 0, 0, 0, 0, 1021, 1040, 1037, 1023, 36, 8, 0, 0, 1044, 1036, 998, 1023, 7, 6, 0, 0, 1018, 1028, 1029, 1011, 3, 0, 0, 0, 1082, 1061, 1011, 1029, 1, 0, 0, 0, 1027, 1049, 1000, 1003, 0, 0, 0, 0, 1006, 984, 1054, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4072, 4165, 4114, 4089, 4184, 4079, 4038] avg=4096.0000 max=4184 max_rank=5 min=4027 min_rank=0 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1054, 1026, 1021, 1010, 0, 0, 0, 0, 1036, 1027, 1038, 1051, 0, 0, 0, 0, 1030, 1019, 983, 1018, 36, 8, 0, 0, 1029, 1020, 980, 1041, 7, 6, 0, 0, 1049, 990, 1052, 1025, 3, 0, 0, 0, 1042, 1037, 1014, 958, 1, 0, 0, 0, 1035, 980, 993, 1004, 0, 0, 0, 0, 1019, 1097, 975, 1054, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4152, 4094, 4083, 4119, 4052, 4012, 4145] avg=4096.0000 max=4152 max_rank=1 min=4012 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1024, 1044, 1028, 1039, 0, 0, 0, 0, 1034, 1021, 1049, 995, 0, 0, 0, 0, 979, 1011, 980, 1008, 35, 8, 0, 0, 1044, 1033, 1045, 1034, 7, 6, 0, 0, 989, 1008, 1030, 1000, 3, 0, 0, 0, 1051, 991, 1051, 1031, 1, 1, 0, 0, 1020, 1036, 1019, 998, 0, 0, 0, 0, 982, 1037, 1055, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4099, 4021, 4169, 4030, 4126, 4073, 4115] avg=4096.0000 max=4169 max_rank=3 min=4021 min_rank=2 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1021, 1002, 1052, 1052, 0, 0, 0, 0, 1043, 992, 1030, 1062, 0, 0, 0, 0, 1014, 984, 985, 1006, 35, 8, 0, 0, 1021, 1020, 1007, 1036, 7, 6, 0, 0, 1047, 1046, 1016, 992, 3, 0, 0, 0, 1014, 1026, 1037, 996, 1, 0, 0, 0, 1025, 1072, 1011, 992, 0, 0, 0, 0, 1047, 1030, 1029, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4127, 4032, 4097, 4104, 4074, 4100, 4107] avg=4096.0000 max=4127 max_rank=0 min=4032 min_rank=2 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1024, 1019, 986, 1007, 0, 0, 0, 0, 1066, 1007, 998, 1028, 0, 0, 0, 0, 997, 1024, 1040, 1027, 35, 8, 0, 0, 1054, 1025, 1053, 976, 7, 6, 0, 0, 1052, 1033, 999, 1020, 3, 0, 0, 0, 1004, 998, 1027, 990, 1, 0, 0, 0, 1022, 989, 1079, 985, 0, 0, 0, 0, 1067, 1046, 1034, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4099, 4131, 4121, 4107, 4020, 4075, 4179] avg=4096.0000 max=4179 max_rank=7 min=4020 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1019, 1098, 1040, 970, 0, 0, 0, 0, 995, 1057, 1014, 1016, 0, 0, 0, 0, 1059, 1040, 975, 985, 35, 8, 0, 0, 966, 978, 987, 1022, 7, 6, 0, 0, 991, 1014, 1037, 1028, 3, 0, 0, 0, 1047, 1072, 974, 1008, 1, 1, 0, 0, 1061, 1087, 1044, 994, 0, 0, 0, 0, 1053, 961, 1063, 1052, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4082, 4102, 3966, 4073, 4103, 4186, 4129] avg=4096.0000 max=4186 max_rank=6 min=3966 min_rank=3 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1005, 999, 1026, 1076, 0, 0, 0, 0, 1014, 1007, 984, 1021, 0, 0, 0, 0, 1050, 1007, 1035, 1051, 40, 11, 0, 0, 1050, 1034, 1009, 949, 10, 12, 0, 0, 985, 1062, 1005, 1081, 9, 0, 0, 0, 1021, 1071, 996, 1001, 4, 5, 0, 0, 1060, 992, 1035, 1024, 0, 0, 0, 0, 1020, 999, 990, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4026, 4194, 4064, 4142, 4098, 4111, 4027] avg=4096.0000 max=4194 max_rank=2 min=4026 min_rank=1 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1026, 1037, 1008, 1028, 0, 0, 0, 0, 1028, 1031, 1017, 1035, 0, 0, 0, 0, 1002, 990, 1026, 967, 36, 8, 0, 0, 1040, 1007, 1066, 1028, 7, 6, 0, 0, 1054, 997, 1003, 1047, 3, 0, 0, 0, 1017, 990, 1069, 1033, 1, 1, 0, 0, 1019, 1028, 1072, 1013, 0, 0, 0, 0, 998, 1019, 988, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4111, 4029, 4154, 4104, 4111, 4132, 4028] avg=4096.0000 max=4154 max_rank=3 min=4028 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8064, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1049, 1043, 987, 1015, 1045, 1016, 1027, 1058, 955, 1031, 1008, 1081, 1047, 991, 1039, 1019, 1025, 1042, 994, 1017, 1030, 1055, 1050, 1066, 1018, 978, 1040, 1024, 1007, 1040, 984, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1001, 1007, 1041, 1039, 1036, 998, 1001, 1062, 962, 1040, 1029, 1064, 1034, 1058, 991, 1033, 1046, 964, 1006, 1058, 1029, 1013, 1010, 993, 1035, 1112, 1007, 975, 1049, 1047, 988, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1017, 1014, 1040, 1041, 1075, 1031, 1015, 959, 1060, 1007, 1030, 1012, 1016, 1046, 1033, 1025, 1006, 1093, 1042, 997, 1043, 974, 1028, 1000, 1067, 1018, 1006, 1029, 1014, 1006, 983, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[997, 990, 1023, 1031, 1011, 1067, 1019, 1035, 1012, 1048, 1031, 1008, 996, 1024, 1024, 1043, 1029, 1050, 1053, 1014, 968, 997, 1043, 1002, 987, 1036, 1021, 1028, 1050, 1058, 1061, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[993, 1008, 1031, 1003, 1068, 1016, 1010, 1061, 1023, 1010, 1062, 1003, 1099, 1013, 1022, 1018, 1024, 974, 1070, 1061, 1039, 1039, 1002, 1056, 1031, 1027, 976, 1028, 992, 1011, 994, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1019, 1040, 1067, 1049, 1012, 1048, 1063, 1014, 1042, 1016, 918, 988, 1061, 1054, 1026, 986, 1061, 1009, 1066, 1068, 992, 1032, 1006, 961, 1009, 1035, 967, 1039, 1043, 1052, 1017, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1036, 1057, 1028, 1005, 1058, 1059, 1017, 998, 1043, 1090, 1024, 1011, 1018, 1000, 957, 1028, 1011, 989, 968, 1008, 1024, 1058, 1027, 1016, 1027, 1041, 995, 1008, 1051, 1053, 1055, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[962, 1031, 1060, 991, 1053, 1019, 978, 940, 1056, 1057, 1013, 998, 1037, 1026, 1038, 1032, 1057, 1093, 1030, 1057, 1000, 1000, 1034, 1054, 1031, 1012, 988, 1045, 1024, 1016, 1047, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1049, 1043, 987, 1015, 6, 0, 0, 0, 1034, 1016, 1027, 1058, 0, 0, 0, 0, 955, 1031, 1008, 1081, 4, 0, 0, 0, 1040, 991, 1039, 1019, 0, 0, 0, 0, 1025, 1042, 994, 992, 0, 0, 0, 0, 1030, 1055, 1050, 1066, 10, 5, 0, 0, 1018, 978, 1040, 1024, 15, 0, 0, 0, 1007, 1040, 984, 987, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4135, 4079, 4089, 4053, 4216, 4075, 4021] avg=4096.0000 max=4216 max_rank=5 min=4021 min_rank=7 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1019, 1040, 1067, 1049, 6, 0, 0, 0, 1002, 1048, 1063, 1014, 0, 0, 0, 0, 1042, 1016, 918, 988, 4, 0, 0, 0, 1054, 1054, 1026, 986, 0, 0, 0, 0, 1061, 1009, 1066, 1041, 0, 0, 0, 0, 992, 1032, 1006, 961, 11, 4, 0, 0, 1009, 1035, 967, 1039, 16, 0, 0, 0, 1043, 1052, 1017, 1008, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4127, 3968, 4120, 4177, 4006, 4066, 4123] avg=4096.0000 max=4181 max_rank=0 min=3968 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1001, 1007, 1041, 1039, 6, 0, 0, 0, 1025, 998, 1001, 1062, 0, 0, 0, 0, 962, 1040, 1029, 1064, 4, 0, 0, 0, 1027, 1058, 991, 1033, 0, 0, 0, 0, 1046, 964, 1006, 1032, 0, 0, 0, 0, 1029, 1013, 1010, 993, 10, 5, 0, 0, 1035, 1112, 1007, 975, 16, 0, 0, 0, 1049, 1047, 988, 1040, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4086, 4099, 4109, 4048, 4060, 4145, 4127] avg=4096.0000 max=4145 max_rank=6 min=4048 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[997, 990, 1023, 1031, 6, 0, 0, 0, 1001, 1067, 1019, 1035, 0, 0, 0, 0, 1012, 1048, 1031, 1008, 4, 0, 0, 0, 989, 1024, 1024, 1043, 0, 0, 0, 0, 1029, 1050, 1053, 989, 0, 0, 0, 0, 968, 997, 1043, 1002, 10, 4, 0, 0, 987, 1036, 1021, 1028, 15, 0, 0, 0, 1050, 1058, 1061, 1012, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4122, 4103, 4080, 4121, 4024, 4087, 4184] avg=4096.0000 max=4184 max_rank=7 min=4024 min_rank=5 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[993, 1008, 1031, 1003, 6, 0, 0, 0, 1057, 1016, 1010, 1061, 0, 0, 0, 0, 1023, 1010, 1062, 1003, 4, 0, 0, 0, 1092, 1013, 1022, 1018, 0, 0, 0, 0, 1024, 974, 1070, 1034, 0, 0, 0, 0, 1039, 1039, 1002, 1056, 11, 5, 0, 0, 1031, 1027, 976, 1028, 16, 0, 0, 0, 992, 1011, 994, 1004, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4144, 4102, 4145, 4102, 4152, 4078, 4004] avg=4096.0000 max=4152 max_rank=5 min=4004 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1036, 1057, 1028, 1005, 6, 0, 0, 0, 1047, 1059, 1017, 998, 0, 0, 0, 0, 1043, 1090, 1024, 1011, 4, 0, 0, 0, 1011, 1000, 957, 1028, 0, 0, 0, 0, 1011, 989, 968, 983, 0, 0, 0, 0, 1024, 1058, 1027, 1016, 10, 5, 0, 0, 1027, 1041, 995, 1008, 15, 0, 0, 0, 1051, 1053, 1055, 1008, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4121, 4172, 3996, 3951, 4140, 4086, 4170] avg=4096.0000 max=4172 max_rank=2 min=3951 min_rank=4 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1017, 1014, 1040, 1041, 11, 0, 0, 0, 1049, 1031, 1015, 959, 0, 0, 0, 0, 1060, 1007, 1030, 1012, 8, 0, 0, 0, 999, 1046, 1033, 1025, 0, 0, 0, 0, 1006, 1093, 1042, 964, 0, 0, 0, 0, 1043, 974, 1028, 1000, 14, 8, 0, 0, 1067, 1018, 1006, 1029, 19, 0, 0, 0, 1014, 1006, 983, 1041, 9, 7, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4054, 4117, 4103, 4105, 4067, 4139, 4060] avg=4096.0000 max=4139 max_rank=6 min=4054 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[962, 1031, 1060, 991, 6, 0, 0, 0, 1042, 1019, 978, 940, 0, 0, 0, 0, 1056, 1057, 1013, 998, 4, 0, 0, 0, 1030, 1026, 1038, 1032, 0, 0, 0, 0, 1057, 1093, 1030, 1031, 0, 0, 0, 0, 1000, 1000, 1034, 1054, 10, 5, 0, 0, 1031, 1012, 988, 1045, 16, 0, 0, 0, 1024, 1016, 1047, 989, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 3979, 4128, 4126, 4211, 4103, 4092, 4079] avg=4096.0000 max=4211 max_rank=4 min=3979 min_rank=1 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1014, 1029, 1038, 986, 987, 1069, 1023, 1018, 1062, 1006, 1006, 1065, 981, 991, 1016, 1065, 1000, 1039, 1066, 988, 1011, 1025, 1021, 1071, 1055, 1049, 1009, 1025, 990, 1008, 1053, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1044, 1036, 989, 1037, 1049, 1014, 993, 1040, 1032, 1061, 955, 1034, 1056, 1024, 983, 1047, 1062, 1020, 1040, 968, 989, 1079, 1065, 1001, 997, 997, 1036, 999, 1035, 1048, 1054, 984] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1015, 1014, 1049, 1006, 1058, 1045, 1026, 973, 1027, 996, 1024, 999, 1098, 1055, 1023, 969, 1047, 1044, 1015, 1003, 1011, 990, 1026, 983, 1026, 984, 1026, 1069, 1020, 1032, 1062, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1051, 1027, 987, 1045, 1052, 1052, 1017, 1061, 1002, 1061, 979, 1050, 1066, 1018, 1028, 1036, 996, 1021, 1015, 1039, 1033, 1049, 1020, 1004, 1005, 1005, 1021, 990, 1014, 987, 1055, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[976, 983, 1031, 984, 1026, 1045, 1020, 1000, 986, 996, 1016, 1068, 1036, 1068, 1006, 1048, 1029, 1032, 1087, 1013, 1073, 1048, 1059, 1060, 1040, 1017, 1004, 977, 1041, 1033, 973, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[998, 1053, 1033, 1035, 1010, 1028, 1000, 1017, 1037, 996, 1027, 994, 1041, 1030, 1016, 1001, 1030, 1004, 1019, 1076, 1021, 1025, 983, 1045, 1045, 1029, 1033, 1049, 1011, 1034, 1019, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1003, 1033, 994, 1023, 1058, 1034, 1039, 1001, 1032, 1028, 1027, 976, 982, 1033, 1074, 987, 1044, 1072, 1041, 1033, 1063, 1027, 1073, 1073, 1066, 962, 1027, 1049, 990, 964, 960, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1029, 1039, 981, 1019, 1026, 1034, 1022, 988, 1058, 984, 1019, 1020, 1074, 1008, 1040, 1012, 1057, 1054, 971, 1039, 1037, 1040, 1011, 1039, 1028, 991, 1047, 1026, 1004, 1049, 992, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1014, 1029, 1038, 986, 20, 2, 0, 0, 987, 1062, 1023, 1018, 0, 0, 0, 0, 1062, 1006, 1006, 1065, 13, 3, 0, 0, 965, 991, 1016, 1065, 0, 0, 0, 0, 1000, 1016, 1066, 988, 0, 0, 0, 0, 1011, 990, 1021, 1071, 0, 0, 0, 0, 1055, 1049, 1009, 1025, 7, 3, 0, 0, 990, 1008, 1053, 1002, 33, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4090, 4155, 4037, 4070, 4093, 4148, 4086] avg=4096.0000 max=4155 max_rank=2 min=4037 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1015, 1014, 1049, 1006, 21, 2, 0, 0, 1058, 1038, 1026, 973, 0, 0, 0, 0, 1027, 996, 1024, 999, 15, 3, 0, 0, 1080, 1055, 1023, 969, 0, 0, 0, 0, 1047, 1020, 1015, 1003, 0, 0, 0, 0, 1011, 957, 1026, 983, 0, 0, 0, 0, 1026, 984, 1026, 1069, 7, 3, 0, 0, 1020, 1032, 1062, 1053, 31, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4095, 4064, 4127, 4085, 3977, 4115, 4198] avg=4096.0000 max=4198 max_rank=7 min=3977 min_rank=5 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1044, 1036, 989, 1037, 20, 2, 0, 0, 1049, 1008, 993, 1040, 0, 0, 0, 0, 1032, 1061, 955, 1034, 14, 3, 0, 0, 1039, 1024, 983, 1047, 0, 0, 0, 0, 1062, 997, 1040, 968, 0, 0, 0, 0, 989, 1043, 1065, 1001, 0, 0, 0, 0, 997, 997, 1036, 999, 6, 3, 0, 0, 1035, 1048, 1054, 984, 34, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4090, 4099, 4093, 4067, 4098, 4038, 4155] avg=4096.0000 max=4155 max_rank=7 min=4038 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1051, 1027, 987, 1045, 20, 2, 0, 0, 1052, 1045, 1017, 1061, 0, 0, 0, 0, 1002, 1061, 979, 1050, 14, 3, 0, 0, 1049, 1018, 1028, 1036, 0, 0, 0, 0, 996, 998, 1015, 1039, 0, 0, 0, 0, 1033, 1014, 1020, 1004, 0, 0, 0, 0, 1005, 1005, 1021, 990, 7, 3, 0, 0, 1014, 987, 1055, 982, 33, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4175, 4109, 4131, 4048, 4071, 4031, 4071] avg=4096.0000 max=4175 max_rank=1 min=4031 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16221 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1003, 1033, 994, 1023, 21, 2, 0, 0, 1058, 1027, 1039, 1001, 0, 0, 0, 0, 1032, 1028, 1027, 976, 13, 3, 0, 0, 966, 1033, 1074, 987, 0, 0, 0, 0, 1044, 1048, 1041, 1033, 0, 0, 0, 0, 1063, 992, 1073, 1073, 0, 0, 0, 0, 1066, 962, 1027, 1049, 7, 3, 0, 0, 990, 964, 960, 1000, 33, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4125, 4079, 4060, 4166, 4201, 4114, 3947] avg=4096.0000 max=4201 max_rank=5 min=3947 min_rank=7 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[998, 1053, 1033, 1035, 24, 6, 0, 0, 1010, 1019, 1000, 1017, 0, 0, 0, 0, 1037, 996, 1027, 994, 19, 8, 0, 0, 1015, 1030, 1016, 1001, 0, 0, 0, 0, 1030, 972, 1019, 1076, 0, 0, 0, 0, 1021, 982, 983, 1045, 0, 0, 0, 0, 1045, 1029, 1033, 1049, 9, 7, 0, 0, 1011, 1034, 1019, 1029, 37, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4046, 4081, 4062, 4097, 4031, 4172, 4130] avg=4096.0000 max=4172 max_rank=6 min=4031 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[976, 983, 1031, 984, 20, 2, 0, 0, 1026, 1038, 1020, 1000, 0, 0, 0, 0, 986, 996, 1016, 1068, 14, 3, 0, 0, 1019, 1068, 1006, 1048, 0, 0, 0, 0, 1029, 1009, 1087, 1013, 0, 0, 0, 0, 1073, 1013, 1059, 1060, 0, 0, 0, 0, 1040, 1017, 1004, 977, 7, 3, 0, 0, 1041, 1033, 973, 993, 33, 0, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4084, 4083, 4141, 4138, 4205, 4048, 4073] avg=4096.0000 max=4205 max_rank=5 min=3996 min_rank=0 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1029, 1039, 981, 1019, 21, 2, 0, 0, 1026, 1027, 1022, 988, 0, 0, 0, 0, 1058, 984, 1019, 1020, 14, 3, 0, 0, 1057, 1008, 1040, 1012, 0, 0, 0, 0, 1057, 1030, 971, 1039, 0, 0, 0, 0, 1037, 1005, 1011, 1039, 0, 0, 0, 0, 1028, 991, 1047, 1026, 7, 3, 0, 0, 1004, 1049, 992, 1030, 33, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4063, 4098, 4117, 4097, 4092, 4102, 4108] avg=4096.0000 max=4117 max_rank=3 min=4063 min_rank=1 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1047, 975, 1003, 1037, 1052, 1023, 978, 1015, 1051, 1024, 966, 979, 1032, 1029, 1053, 966, 1065, 994, 1087, 1020, 1016, 1026, 1046, 1017, 1034, 1029, 1063, 1088, 984, 1028, 991, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1036, 1018, 1002, 1037, 1010, 953, 1025, 1000, 1047, 1049, 1060, 992, 1064, 1061, 970, 1036, 1006, 1076, 1007, 992, 1010, 1011, 1044, 1016, 1059, 1043, 1051, 1001, 985, 1037, 1050, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1048, 1045, 1053, 1031, 1036, 999, 1010, 1023, 1029, 1004, 1040, 988, 1052, 976, 1010, 1019, 1042, 1030, 1058, 1037, 1007, 1057, 1015, 1003, 983, 1020, 1039, 993, 1047, 1045, 1026, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1045, 1071, 1020, 995, 1062, 1026, 1018, 1017, 1031, 1046, 1035, 1058, 1031, 1018, 973, 1008, 1051, 1026, 984, 1008, 1007, 956, 1021, 1060, 1045, 1007, 1018, 995, 1057, 1014, 1008, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1090, 1051, 978, 1015, 993, 1016, 1062, 1003, 1089, 1036, 1029, 1037, 978, 1069, 960, 1021, 991, 1000, 1051, 1010, 1013, 1020, 1025, 1029, 1015, 1010, 1052, 1034, 1031, 1043, 1001, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1000, 1023, 1022, 1063, 1022, 992, 1050, 992, 1006, 973, 991, 1040, 998, 1007, 1034, 1018, 1002, 1091, 1046, 1019, 1095, 969, 1053, 1008, 1027, 1056, 1004, 986, 1014, 1052, 1102, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1041, 1040, 1041, 1031, 1013, 1033, 968, 1047, 988, 1035, 1070, 979, 1013, 1033, 1012, 1043, 1040, 1050, 1034, 985, 1002, 1010, 995, 984, 1031, 1065, 1034, 1019, 1042, 991, 1029, 1070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1017, 959, 1039, 997, 1032, 1038, 1057, 1023, 1044, 1057, 1017, 984, 1033, 1058, 1026, 1063, 968, 1024, 1007, 1048, 1048, 1009, 999, 1095, 1027, 1048, 1003, 991, 978, 1056, 1011, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1035, 975, 1003, 1037, 0, 0, 0, 0, 1052, 1023, 978, 1015, 12, 9, 0, 0, 1051, 1024, 966, 979, 0, 0, 0, 0, 1032, 1029, 1053, 966, 9, 3, 0, 0, 1065, 985, 1087, 1020, 0, 0, 0, 0, 1016, 1026, 1046, 1017, 9, 1, 0, 0, 1034, 1017, 1063, 1088, 0, 0, 0, 0, 984, 1018, 991, 1050, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4089, 4020, 4092, 4157, 4115, 4202, 4043] avg=4096.0000 max=4202 max_rank=6 min=4020 min_rank=2 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1025, 1018, 1002, 1037, 0, 0, 0, 0, 1010, 953, 1025, 1000, 12, 9, 0, 0, 1047, 1049, 1060, 992, 0, 0, 0, 0, 1064, 1061, 970, 1036, 10, 2, 0, 0, 1006, 1066, 1007, 992, 0, 0, 0, 0, 1010, 1011, 1044, 1016, 10, 1, 0, 0, 1059, 1031, 1051, 1001, 0, 0, 0, 0, 985, 1026, 1050, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4009, 4148, 4143, 4071, 4092, 4142, 4081] avg=4096.0000 max=4148 max_rank=2 min=4009 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1036, 1045, 1053, 1031, 0, 0, 0, 0, 1036, 999, 1010, 1023, 12, 9, 0, 0, 1029, 1004, 1040, 988, 0, 0, 0, 0, 1052, 976, 1010, 1019, 10, 3, 0, 0, 1042, 1020, 1058, 1037, 0, 0, 0, 0, 1007, 1057, 1015, 1003, 10, 1, 0, 0, 983, 1008, 1039, 993, 0, 0, 0, 0, 1047, 1034, 1026, 1003, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 4089, 4061, 4070, 4157, 4093, 4023, 4110] avg=4096.0000 max=4165 max_rank=0 min=4023 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1077, 1051, 978, 1015, 0, 0, 0, 0, 993, 1016, 1062, 1003, 12, 10, 0, 0, 1089, 1036, 1029, 1037, 0, 0, 0, 0, 978, 1069, 960, 1021, 10, 3, 0, 0, 991, 991, 1051, 1010, 0, 0, 0, 0, 1013, 1020, 1025, 1029, 9, 1, 0, 0, 1015, 998, 1052, 1034, 0, 0, 0, 0, 1031, 1032, 1001, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4096, 4191, 4041, 4043, 4097, 4099, 4080] avg=4096.0000 max=4191 max_rank=2 min=4041 min_rank=3 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1029, 1040, 1041, 1031, 0, 0, 0, 0, 1013, 1033, 968, 1047, 13, 9, 0, 0, 988, 1035, 1070, 979, 0, 0, 0, 0, 1013, 1033, 1012, 1043, 9, 3, 0, 0, 1040, 1040, 1034, 985, 0, 0, 0, 0, 1002, 1010, 995, 984, 10, 1, 0, 0, 1031, 1052, 1034, 1019, 0, 0, 0, 0, 1042, 981, 1029, 1070, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4083, 4072, 4113, 4099, 4002, 4136, 4122] avg=4096.0000 max=4141 max_rank=0 min=4002 min_rank=5 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[981, 1023, 1022, 1063, 0, 0, 0, 0, 1022, 992, 1050, 992, 17, 14, 0, 0, 1000, 973, 991, 1040, 0, 0, 0, 0, 998, 1007, 1034, 1018, 13, 5, 0, 0, 1002, 1078, 1046, 1019, 0, 0, 0, 0, 1095, 969, 1053, 1008, 13, 8, 6, 0, 1027, 1039, 1004, 986, 0, 0, 0, 0, 1014, 1031, 1102, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4087, 4004, 4075, 4145, 4152, 4056, 4160] avg=4096.0000 max=4160 max_rank=7 min=4004 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1033, 1071, 1020, 995, 0, 0, 0, 0, 1062, 1026, 1018, 1017, 12, 9, 0, 0, 1031, 1046, 1035, 1058, 0, 0, 0, 0, 1031, 1018, 973, 1008, 9, 3, 0, 0, 1051, 1016, 984, 1008, 0, 0, 0, 0, 1007, 956, 1021, 1060, 10, 1, 0, 0, 1045, 995, 1018, 995, 0, 0, 0, 0, 1057, 1004, 1008, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4144, 4170, 4042, 4059, 4055, 4053, 4126] avg=4096.0000 max=4170 max_rank=2 min=4042 min_rank=3 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1006, 959, 1039, 997, 0, 0, 0, 0, 1032, 1038, 1057, 1023, 12, 9, 0, 0, 1044, 1057, 1017, 984, 0, 0, 0, 0, 1033, 1058, 1026, 1063, 10, 2, 0, 0, 968, 1014, 1007, 1048, 0, 0, 0, 0, 1048, 1009, 999, 1095, 10, 1, 0, 0, 1027, 1036, 1003, 991, 0, 0, 0, 0, 978, 1045, 1011, 1012, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4001, 4171, 4102, 4192, 4037, 4162, 4057, 4046] avg=4096.0000 max=4192 max_rank=3 min=4001 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[994, 1042, 1025, 1021, 1027, 993, 1050, 1005, 1014, 1039, 1015, 1052, 1037, 985, 1044, 1001, 1004, 1008, 1031, 1015, 1026, 1076, 1021, 1023, 1023, 1007, 982, 1028, 1054, 1018, 1025, 1083] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1037, 986, 1024, 1022, 1044, 1057, 1064, 1027, 1036, 978, 1058, 956, 980, 1031, 1002, 1050, 1015, 1030, 1046, 1026, 1015, 1049, 1056, 1015, 1035, 1006, 1014, 951, 1058, 1041, 998, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1062, 1039, 1020, 1038, 1034, 1017, 1013, 1010, 1045, 1004, 1030, 1058, 1048, 996, 997, 1046, 1011, 1021, 979, 1005, 1071, 996, 1030, 1043, 995, 1054, 1030, 1058, 1010, 995, 1009, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1000, 1018, 1040, 1024, 1027, 1001, 1043, 1043, 1027, 999, 1054, 1007, 1017, 1040, 1048, 969, 1026, 1019, 1001, 997, 1033, 988, 1069, 1015, 1030, 1082, 1033, 1037, 1041, 1034, 972, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1075, 1010, 1004, 1052, 1017, 1006, 1040, 998, 1017, 1042, 1059, 1042, 1011, 979, 1041, 1013, 1005, 1022, 1053, 1032, 1007, 1023, 1005, 1047, 989, 1004, 1029, 975, 1037, 1019, 1069, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1018, 1010, 1032, 1072, 1023, 1015, 1025, 999, 1021, 1045, 1074, 1029, 997, 1017, 1060, 991, 973, 1032, 1021, 1056, 1051, 1060, 1003, 1011, 967, 1019, 1045, 1025, 1034, 1017, 1025, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1008, 1064, 1059, 1050, 1030, 1046, 1025, 1018, 1038, 1024, 1015, 1064, 1027, 988, 1009, 1071, 978, 1055, 1027, 1027, 1022, 950, 1013, 1000, 1091, 964, 993, 1012, 1028, 1073, 994, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1010, 1006, 1013, 966, 993, 1037, 1039, 1070, 1054, 1021, 1015, 1001, 1043, 1054, 1052, 1008, 1005, 1020, 1074, 1020, 1046, 1028, 1047, 1010, 1054, 1020, 1000, 1032, 1011, 1003, 989, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1000, 1018, 1040, 1016, 0, 0, 0, 0, 1027, 1001, 1035, 1043, 0, 0, 0, 0, 1027, 999, 1034, 1007, 0, 0, 0, 0, 1017, 1040, 1048, 969, 8, 5, 0, 0, 1026, 1019, 1001, 997, 8, 7, 0, 0, 1024, 988, 1069, 1015, 0, 0, 0, 0, 1030, 1082, 1033, 1037, 20, 2, 0, 0, 1036, 1034, 972, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4106, 4067, 4087, 4058, 4096, 4204, 4076] avg=4096.0000 max=4204 max_rank=6 min=4058 min_rank=4 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[994, 1042, 1025, 1013, 0, 0, 0, 0, 1027, 993, 1042, 1005, 0, 0, 0, 0, 1014, 1039, 995, 1052, 0, 0, 0, 0, 1037, 985, 1044, 1001, 8, 5, 0, 0, 1004, 1008, 1031, 1015, 8, 7, 0, 0, 1017, 1076, 1021, 1023, 0, 0, 0, 0, 1023, 1007, 982, 1028, 20, 2, 0, 0, 1049, 1018, 1025, 1083, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4067, 4100, 4080, 4073, 4137, 4062, 4175] avg=4096.0000 max=4175 max_rank=7 min=4062 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1037, 986, 1024, 1014, 0, 0, 0, 0, 1044, 1057, 1056, 1027, 0, 0, 0, 0, 1036, 978, 1038, 956, 0, 0, 0, 0, 980, 1031, 1002, 1050, 8, 6, 0, 0, 1015, 1030, 1046, 1026, 8, 7, 0, 0, 1006, 1049, 1056, 1015, 0, 0, 0, 0, 1035, 1006, 1014, 951, 20, 2, 0, 0, 1052, 1041, 998, 1061, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4184, 4008, 4077, 4132, 4126, 4028, 4152] avg=4096.0000 max=4184 max_rank=1 min=4008 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1062, 1039, 1020, 1029, 0, 0, 0, 0, 1034, 1017, 1005, 1010, 0, 0, 0, 0, 1045, 1004, 1010, 1058, 0, 0, 0, 0, 1048, 996, 997, 1046, 8, 5, 0, 0, 1011, 1021, 979, 1005, 9, 8, 0, 0, 1061, 996, 1030, 1043, 0, 0, 0, 0, 995, 1054, 1030, 1058, 20, 2, 0, 0, 1005, 995, 1009, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4066, 4117, 4100, 4033, 4130, 4159, 4013] avg=4096.0000 max=4159 max_rank=6 min=4013 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1075, 1010, 1004, 1043, 0, 0, 0, 0, 1017, 1006, 1032, 998, 0, 0, 0, 0, 1017, 1042, 1038, 1042, 0, 0, 0, 0, 1011, 979, 1041, 1013, 8, 5, 0, 0, 1005, 1022, 1053, 1032, 9, 7, 0, 0, 998, 1023, 1005, 1047, 0, 0, 0, 0, 989, 1004, 1029, 975, 21, 2, 0, 0, 1032, 1019, 1069, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4053, 4139, 4057, 4128, 4073, 4020, 4166] avg=4096.0000 max=4166 max_rank=7 min=4020 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1018, 1010, 1032, 1063, 0, 0, 0, 0, 1023, 1015, 1017, 999, 0, 0, 0, 0, 1021, 1045, 1053, 1029, 0, 0, 0, 0, 997, 1017, 1060, 991, 8, 5, 0, 0, 973, 1032, 1021, 1056, 9, 7, 0, 0, 1042, 1060, 1003, 1011, 0, 0, 0, 0, 967, 1019, 1045, 1025, 21, 2, 0, 0, 1029, 1017, 1025, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4054, 4148, 4078, 4098, 4116, 4079, 4072] avg=4096.0000 max=4148 max_rank=2 min=4054 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1008, 1064, 1059, 1036, 0, 0, 0, 0, 1030, 1046, 1013, 1018, 0, 0, 0, 0, 1038, 1024, 992, 1064, 0, 0, 0, 0, 1027, 988, 1009, 1071, 12, 11, 1, 0, 978, 1055, 1027, 1027, 13, 12, 0, 0, 1005, 950, 1013, 1000, 0, 0, 0, 0, 1091, 964, 993, 1012, 23, 5, 0, 0, 1017, 1073, 994, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4107, 4118, 4119, 4112, 3968, 4088, 4089] avg=4096.0000 max=4167 max_rank=0 min=3968 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1010, 1006, 1013, 958, 0, 0, 0, 0, 993, 1037, 1031, 1070, 0, 0, 0, 0, 1054, 1021, 995, 1001, 0, 0, 0, 0, 1043, 1054, 1052, 1008, 8, 5, 0, 0, 1005, 1020, 1074, 1020, 8, 7, 0, 0, 1037, 1028, 1047, 1010, 0, 0, 0, 0, 1054, 1020, 1000, 1032, 20, 2, 0, 0, 1006, 1003, 989, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3987, 4131, 4071, 4170, 4134, 4122, 4128, 4025] avg=4096.0000 max=4170 max_rank=3 min=3987 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1028, 1031, 1070, 1046, 1002, 1067, 1039, 1029, 1003, 1012, 1002, 1014, 1067, 1046, 1037, 1012, 1022, 1036, 993, 987, 1028, 1039, 1040, 1003, 1007, 1009, 1009, 1018, 1005, 1030, 1008, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1011, 1004, 1040, 1017, 1014, 1032, 1036, 1045, 1016, 1040, 992, 1025, 1034, 1054, 1047, 957, 1012, 1015, 1039, 1017, 1037, 1014, 1026, 1019, 1056, 1036, 1050, 982, 1025, 1045, 1033, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1056, 989, 1057, 1048, 1053, 967, 1016, 1041, 997, 1047, 1036, 1047, 1008, 1020, 1024, 1041, 1055, 1008, 1000, 1058, 1018, 1060, 1025, 955, 1003, 1031, 1035, 1017, 997, 1056, 1020, 983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1003, 1002, 1049, 1028, 1018, 972, 1081, 1036, 1032, 1046, 1076, 1029, 1016, 1017, 1072, 1077, 1042, 1001, 1021, 1046, 1032, 1009, 1044, 1031, 1000, 968, 1018, 993, 1022, 985, 979, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1009, 1024, 1014, 1057, 1002, 1070, 1015, 1031, 1006, 1052, 1019, 1066, 1053, 1034, 1031, 994, 1018, 1027, 1036, 952, 1016, 1047, 1011, 1038, 1020, 991, 1049, 1051, 1005, 1036, 951, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1043, 1018, 1015, 996, 1051, 1085, 974, 1013, 1013, 1055, 1064, 997, 1024, 1007, 1018, 1011, 1039, 1030, 1051, 1021, 1042, 1027, 1045, 1053, 1028, 1026, 986, 1004, 990, 988, 989, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1021, 1030, 1031, 1026, 1026, 1034, 993, 1028, 1039, 1043, 1043, 1025, 1027, 979, 978, 1043, 1070, 1010, 1021, 1055, 1019, 1007, 1050, 1028, 1021, 980, 1058, 992, 1033, 1041, 984, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1038, 964, 1000, 1050, 1052, 1012, 1044, 1011, 1042, 1026, 984, 1024, 1015, 1044, 1026, 1038, 1014, 1008, 1034, 1002, 1066, 1017, 1009, 1094, 1000, 1004, 1066, 1034, 1045, 996, 998, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1011, 1004, 1035, 1017, 0, 0, 0, 0, 1014, 1017, 1036, 1045, 0, 0, 0, 0, 1016, 1023, 992, 1025, 0, 0, 0, 0, 1024, 1054, 1047, 957, 0, 0, 0, 0, 1012, 1015, 1039, 1017, 3, 0, 0, 0, 1015, 1014, 1026, 1019, 0, 0, 0, 0, 1056, 1036, 1050, 982, 15, 10, 2, 0, 1025, 1045, 1033, 998, 22, 17, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4112, 4056, 4082, 4086, 4074, 4151, 4140] avg=4096.0000 max=4151 max_rank=6 min=4056 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1028, 1031, 1065, 1046, 0, 0, 0, 0, 1002, 1052, 1039, 1029, 0, 0, 0, 0, 1003, 995, 1002, 1014, 0, 0, 0, 0, 1057, 1046, 1037, 1012, 0, 0, 0, 0, 1022, 1036, 993, 987, 3, 0, 0, 0, 1006, 1039, 1040, 1003, 0, 0, 0, 0, 1007, 1009, 1009, 1018, 15, 10, 2, 0, 1005, 1030, 1008, 1029, 22, 17, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4122, 4014, 4152, 4041, 4088, 4070, 4111] avg=4096.0000 max=4170 max_rank=0 min=4014 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1003, 1002, 1044, 1028, 0, 0, 0, 0, 1018, 958, 1081, 1036, 0, 0, 0, 0, 1032, 1029, 1076, 1029, 0, 0, 0, 0, 1006, 1017, 1072, 1077, 0, 0, 0, 0, 1042, 1001, 1021, 1046, 3, 0, 0, 0, 1010, 1009, 1044, 1031, 0, 0, 0, 0, 1000, 968, 1018, 993, 14, 10, 2, 0, 1022, 985, 979, 1023, 22, 17, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4093, 4166, 4172, 4113, 4094, 4005, 4048] avg=4096.0000 max=4172 max_rank=3 min=4005 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1056, 989, 1045, 1048, 0, 0, 0, 0, 1053, 949, 1016, 1041, 0, 0, 0, 0, 997, 1022, 1036, 1047, 0, 0, 0, 0, 995, 1020, 1024, 1041, 0, 0, 0, 0, 1055, 1008, 1000, 1058, 7, 0, 0, 0, 992, 1060, 1025, 955, 0, 0, 0, 0, 1003, 1031, 1035, 1017, 18, 13, 5, 3, 997, 1056, 1020, 983, 26, 22, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4059, 4102, 4080, 4128, 4032, 4125, 4104] avg=4096.0000 max=4138 max_rank=0 min=4032 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1043, 1018, 1010, 996, 0, 0, 0, 0, 1051, 1070, 974, 1013, 0, 0, 0, 0, 1013, 1038, 1064, 997, 0, 0, 0, 0, 1014, 1007, 1018, 1011, 0, 0, 0, 0, 1039, 1030, 1051, 1021, 3, 0, 0, 0, 1020, 1027, 1045, 1053, 0, 0, 0, 0, 1028, 1026, 986, 1004, 15, 10, 2, 0, 990, 988, 989, 1065, 22, 17, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4108, 4112, 4050, 4144, 4145, 4071, 4071] avg=4096.0000 max=4145 max_rank=5 min=4050 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1009, 1024, 1009, 1057, 0, 0, 0, 0, 1002, 1055, 1015, 1031, 0, 0, 0, 0, 1006, 1035, 1019, 1066, 0, 0, 0, 0, 1043, 1034, 1031, 994, 0, 0, 0, 0, 1018, 1027, 1036, 952, 3, 0, 0, 0, 994, 1047, 1011, 1038, 0, 0, 0, 0, 1020, 991, 1049, 1051, 15, 10, 2, 0, 1005, 1036, 951, 1043, 22, 17, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4103, 4126, 4102, 4036, 4090, 4138, 4074] avg=4096.0000 max=4138 max_rank=6 min=4036 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1021, 1030, 1026, 1026, 0, 0, 0, 0, 1026, 1019, 993, 1028, 0, 0, 0, 0, 1039, 1026, 1043, 1025, 0, 0, 0, 0, 1017, 979, 978, 1043, 0, 0, 0, 0, 1070, 1010, 1021, 1055, 3, 0, 0, 0, 997, 1007, 1050, 1028, 0, 0, 0, 0, 1021, 980, 1058, 992, 15, 10, 2, 0, 1033, 1041, 984, 1033, 22, 17, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4066, 4133, 4017, 4159, 4082, 4078, 4130] avg=4096.0000 max=4159 max_rank=4 min=4017 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1038, 964, 995, 1050, 0, 0, 0, 0, 1052, 998, 1044, 1011, 0, 0, 0, 0, 1042, 1009, 984, 1024, 0, 0, 0, 0, 1005, 1044, 1026, 1038, 0, 0, 0, 0, 1014, 1008, 1034, 1002, 3, 0, 0, 0, 1043, 1017, 1009, 1094, 0, 0, 0, 0, 1000, 1004, 1066, 1034, 14, 10, 2, 0, 1045, 996, 998, 1011, 23, 17, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4105, 4059, 4113, 4061, 4163, 4130, 4090] avg=4096.0000 max=4163 max_rank=5 min=4047 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8064, 8192, 256, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1050, 1014, 1033, 1049, 1015, 1049, 1001, 1031, 1032, 1008, 1044, 1034, 987, 1019, 1107, 975, 1045, 1007, 1053, 993, 984, 1073, 1009, 1065, 1002, 1011, 958, 1008, 1043, 997, 1016, 1056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1029, 1052, 1060, 1007, 1009, 1022, 986, 1044, 998, 1026, 1058, 1032, 1056, 1050, 1022, 1008, 1075, 1021, 953, 1031, 1013, 1028, 1040, 1039, 1068, 979, 986, 1014, 1015, 1007, 1024, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1023, 1077, 1079, 1013, 1019, 1025, 1077, 1019, 1020, 1003, 1025, 1074, 1030, 1054, 942, 1011, 1032, 1018, 989, 978, 1000, 996, 1065, 989, 1001, 1066, 977, 1032, 1008, 1024, 1052, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1002, 1001, 1021, 1000, 1105, 1031, 986, 985, 1022, 1002, 1008, 1002, 1034, 1047, 1064, 997, 1044, 1038, 1045, 1002, 1025, 1033, 1002, 1014, 1020, 1083, 996, 997, 1030, 1009, 1037, 1086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1011, 1025, 1024, 1034, 996, 999, 1052, 989, 1043, 1002, 1034, 1020, 1011, 1031, 1063, 1010, 1040, 1065, 1009, 1032, 1003, 987, 1043, 974, 1019, 1071, 1015, 1018, 1059, 1015, 1040, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1025, 1037, 1000, 998, 1051, 1024, 1028, 1005, 1003, 1025, 1089, 1015, 1011, 1083, 1010, 1028, 1050, 944, 1025, 995, 1057, 1062, 991, 1017, 1000, 1021, 1034, 1008, 1029, 1039, 1003, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1030, 1038, 1028, 1016, 1030, 1032, 1000, 1044, 1087, 1015, 1034, 1014, 1005, 980, 976, 1040, 1034, 1049, 1008, 1037, 981, 991, 1013, 1014, 1033, 1076, 1023, 1015, 1116, 1044, 973, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1031, 991, 1052, 1077, 985, 1010, 1033, 1028, 1041, 1025, 1017, 1018, 990, 1033, 1008, 1002, 1045, 1059, 1050, 1007, 1049, 1059, 1045, 1016, 1009, 1008, 1020, 1005, 1013, 998, 1036, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1050, 1014, 1014, 1049, 0, 0, 0, 0, 1015, 1049, 1001, 1031, 6, 0, 0, 0, 1032, 1008, 1032, 1034, 0, 0, 0, 0, 987, 1019, 1107, 975, 6, 4, 0, 0, 1045, 1007, 1053, 993, 0, 0, 0, 0, 984, 1073, 1009, 1065, 11, 0, 0, 0, 1002, 1011, 958, 1008, 20, 4, 0, 0, 1023, 997, 1016, 1056, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4102, 4106, 4098, 4098, 4142, 4003, 4092] avg=4096.0000 max=4142 max_rank=5 min=4003 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1029, 1052, 1041, 1007, 0, 0, 0, 0, 1009, 1022, 986, 1044, 6, 0, 0, 0, 998, 1026, 1046, 1032, 0, 0, 0, 0, 1056, 1050, 1022, 1008, 6, 4, 0, 0, 1075, 1021, 953, 1031, 0, 0, 0, 0, 1013, 1028, 1040, 1039, 11, 0, 0, 0, 1068, 979, 986, 1014, 19, 4, 0, 0, 996, 1007, 1024, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4067, 4102, 4146, 4080, 4131, 4070, 4043] avg=4096.0000 max=4146 max_rank=3 min=4043 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1011, 1025, 1005, 1034, 0, 0, 0, 0, 996, 999, 1052, 989, 6, 0, 0, 0, 1043, 1002, 1022, 1020, 0, 0, 0, 0, 1011, 1031, 1063, 1010, 6, 4, 0, 0, 1040, 1065, 1009, 1032, 0, 0, 0, 0, 1003, 987, 1043, 974, 11, 0, 0, 0, 1019, 1071, 1015, 1018, 20, 4, 0, 0, 1039, 1015, 1040, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4042, 4087, 4125, 4146, 4018, 4147, 4128] avg=4096.0000 max=4147 max_rank=6 min=4018 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1023, 1077, 1060, 1013, 0, 0, 0, 0, 1019, 1025, 1077, 1019, 6, 0, 0, 0, 1020, 1003, 1013, 1074, 0, 0, 0, 0, 1030, 1054, 942, 1011, 6, 4, 0, 0, 1032, 1018, 989, 978, 0, 0, 0, 0, 1000, 996, 1065, 989, 11, 0, 0, 0, 1001, 1066, 977, 1032, 19, 4, 0, 0, 989, 1024, 1052, 1050, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4146, 4110, 4047, 4017, 4061, 4099, 4115] avg=4096.0000 max=4173 max_rank=0 min=4017 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1002, 1001, 1002, 1000, 0, 0, 0, 0, 1105, 1031, 986, 985, 6, 0, 0, 0, 1022, 1002, 997, 1002, 0, 0, 0, 0, 1034, 1047, 1064, 997, 5, 4, 0, 0, 1044, 1038, 1045, 1002, 0, 0, 0, 0, 1025, 1033, 1002, 1014, 11, 0, 0, 0, 1020, 1083, 996, 997, 20, 4, 0, 0, 1010, 1009, 1037, 1086, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 4113, 4023, 4151, 4129, 4085, 4120, 4142] avg=4096.0000 max=4151 max_rank=3 min=4005 min_rank=0 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 1037, 974, 998, 0, 0, 0, 0, 1051, 1024, 1028, 1005, 11, 5, 0, 0, 1003, 1025, 1069, 1015, 0, 0, 0, 0, 1011, 1083, 1010, 1028, 9, 7, 0, 0, 1045, 944, 1025, 995, 0, 0, 0, 0, 1057, 1062, 991, 1017, 14, 0, 0, 0, 1000, 1021, 1034, 1008, 24, 5, 0, 0, 1005, 1039, 1003, 1061, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4124, 4112, 4148, 4009, 4141, 4092, 4108] avg=4096.0000 max=4148 max_rank=3 min=4009 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1030, 1038, 1009, 1016, 0, 0, 0, 0, 1030, 1032, 1000, 1044, 6, 0, 0, 0, 1087, 1015, 1022, 1014, 0, 0, 0, 0, 1005, 980, 976, 1040, 6, 4, 0, 0, 1034, 1049, 1008, 1037, 0, 0, 0, 0, 981, 991, 1013, 1014, 11, 0, 0, 0, 1033, 1076, 1023, 1015, 21, 4, 0, 0, 1095, 1044, 973, 992, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4112, 4138, 4011, 4128, 4010, 4172, 4104] avg=4096.0000 max=4172 max_rank=6 min=4010 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1031, 991, 1033, 1077, 0, 0, 0, 0, 985, 1010, 1033, 1028, 6, 0, 0, 0, 1041, 1025, 1006, 1018, 0, 0, 0, 0, 990, 1033, 1008, 1002, 5, 4, 0, 0, 1045, 1059, 1050, 1007, 0, 0, 0, 0, 1049, 1059, 1045, 1016, 11, 0, 0, 0, 1009, 1008, 1020, 1005, 19, 4, 0, 0, 994, 998, 1036, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4062, 4090, 4042, 4161, 4180, 4065, 4036] avg=4096.0000 max=4180 max_rank=5 min=4036 min_rank=7 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[949, 989, 1061, 989, 1022, 1060, 1030, 1022, 1038, 1009, 1040, 1000, 1051, 1043, 1096, 1000, 1034, 1004, 1097, 965, 1042, 972, 1060, 1058, 1033, 1039, 1072, 994, 1016, 972, 996, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1055, 1035, 1052, 1085, 1027, 997, 1048, 1056, 1056, 986, 963, 1073, 1003, 1014, 1017, 1009, 1020, 955, 1042, 950, 982, 963, 1033, 986, 1109, 1064, 1078, 1012, 1072, 1004, 986, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1011, 1027, 1051, 988, 1016, 1006, 1024, 1065, 1041, 1029, 1029, 1027, 1060, 1003, 1032, 1016, 1041, 1022, 1036, 1018, 999, 989, 1004, 992, 1009, 1017, 1020, 1034, 1005, 1052, 1040, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[990, 1018, 1004, 1039, 989, 1033, 1026, 989, 1034, 1024, 1057, 995, 1054, 1021, 985, 1062, 1047, 1051, 1039, 1033, 1029, 1044, 1076, 1034, 998, 998, 1018, 1042, 1012, 1069, 997, 961] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1013, 1013, 1042, 986, 1030, 1051, 1032, 1032, 1063, 1039, 1001, 1031, 993, 1047, 991, 1018, 991, 1052, 1058, 1081, 977, 1008, 984, 986, 983, 1010, 1068, 984, 1037, 1042, 1048, 1077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[975, 1022, 1003, 1007, 986, 1016, 1052, 1008, 1032, 1027, 1051, 1031, 1038, 1034, 1007, 1037, 1018, 1015, 1081, 1025, 1034, 998, 1038, 998, 1030, 1048, 1041, 1049, 1001, 998, 1035, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1024, 1022, 1055, 1012, 1034, 1024, 984, 1009, 1049, 1018, 1047, 1026, 1027, 1045, 991, 1055, 981, 1067, 1062, 1033, 985, 987, 1070, 1057, 1015, 986, 1015, 977, 1075, 1012, 999, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1039, 1001, 1017, 1040, 1028, 1008, 1046, 982, 982, 989, 1052, 1070, 1019, 1024, 1029, 1038, 1017, 1006, 1046, 989, 940, 1028, 1009, 1030, 1062, 1024, 1016, 1031, 1080, 1027, 1045, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1011, 1027, 1051, 988, 12, 7, 0, 0, 1016, 1006, 1024, 1065, 4, 0, 0, 0, 1024, 1029, 1029, 1027, 0, 0, 0, 0, 1049, 1003, 1032, 1016, 0, 0, 0, 0, 1041, 1022, 1023, 1018, 0, 0, 0, 0, 999, 989, 1004, 992, 17, 14, 13, 1, 1009, 1017, 1007, 1034, 0, 0, 0, 0, 991, 1052, 1040, 1065, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4115, 4109, 4100, 4104, 4029, 4067, 4148] avg=4096.0000 max=4148 max_rank=7 min=4029 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1055, 1035, 1052, 1085, 12, 6, 0, 0, 1027, 997, 1048, 1056, 4, 0, 0, 0, 1039, 986, 963, 1073, 0, 0, 0, 0, 993, 1014, 1017, 1009, 0, 0, 0, 0, 1020, 955, 1029, 950, 0, 0, 0, 0, 982, 963, 1033, 986, 17, 15, 13, 1, 1109, 1064, 1065, 1012, 0, 0, 0, 0, 1057, 1004, 986, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4245, 4132, 4061, 4033, 3954, 4010, 4250, 4083] avg=4096.0000 max=4250 max_rank=6 min=3954 min_rank=4 imbalance_ratio=1.037598 max_load_reduction=28518 max_load_reduction_pct=0.870300 moved_to_echo=16297 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[949, 989, 1061, 989, 15, 11, 0, 0, 1022, 1060, 1030, 1022, 8, 0, 0, 0, 1016, 1009, 1040, 1000, 0, 0, 0, 0, 1032, 1043, 1096, 1000, 0, 0, 0, 0, 1034, 1004, 1080, 965, 0, 0, 0, 0, 1042, 972, 1060, 1058, 22, 17, 17, 2, 1033, 1039, 1055, 994, 0, 0, 0, 0, 999, 972, 996, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4014, 4142, 4065, 4171, 4083, 4190, 4121, 3982] avg=4096.0000 max=4190 max_rank=5 min=3982 min_rank=7 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[990, 1018, 1004, 1039, 12, 7, 0, 0, 989, 1033, 1026, 989, 4, 0, 0, 0, 1017, 1024, 1057, 995, 0, 0, 0, 0, 1043, 1021, 985, 1062, 0, 0, 0, 0, 1047, 1051, 1026, 1033, 0, 0, 0, 0, 1029, 1044, 1076, 1034, 17, 14, 13, 1, 998, 998, 1005, 1042, 0, 0, 0, 0, 998, 1069, 997, 961, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4041, 4093, 4111, 4157, 4228, 4043, 4025] avg=4096.0000 max=4228 max_rank=5 min=4025 min_rank=7 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[975, 1022, 1003, 1007, 12, 6, 0, 0, 986, 1016, 1052, 1008, 4, 0, 0, 0, 1015, 1027, 1051, 1031, 0, 0, 0, 0, 1028, 1034, 1007, 1037, 0, 0, 0, 0, 1018, 1015, 1068, 1025, 0, 0, 0, 0, 1034, 998, 1038, 998, 17, 14, 13, 1, 1030, 1048, 1028, 1049, 0, 0, 0, 0, 987, 998, 1035, 1033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4066, 4124, 4106, 4126, 4113, 4155, 4053] avg=4096.0000 max=4155 max_rank=6 min=4025 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1013, 1013, 1042, 986, 12, 6, 0, 0, 1030, 1051, 1032, 1032, 4, 0, 0, 0, 1045, 1039, 1001, 1031, 0, 0, 0, 0, 983, 1047, 991, 1018, 0, 0, 0, 0, 991, 1052, 1045, 1081, 0, 0, 0, 0, 977, 1008, 984, 986, 18, 14, 13, 1, 983, 1010, 1055, 984, 0, 0, 0, 0, 1023, 1042, 1048, 1077, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4149, 4116, 4039, 4169, 4001, 4032, 4190] avg=4096.0000 max=4190 max_rank=7 min=4001 min_rank=5 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1024, 1022, 1055, 1012, 12, 6, 0, 0, 1034, 1024, 984, 1009, 4, 0, 0, 0, 1032, 1018, 1047, 1026, 0, 0, 0, 0, 1017, 1045, 991, 1055, 0, 0, 0, 0, 981, 1067, 1049, 1033, 0, 0, 0, 0, 985, 987, 1070, 1057, 17, 15, 13, 1, 1015, 986, 1002, 977, 0, 0, 0, 0, 1060, 1012, 999, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4055, 4123, 4108, 4130, 4145, 3980, 4096] avg=4096.0000 max=4145 max_rank=5 min=3980 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1039, 1001, 1017, 1040, 12, 6, 0, 0, 1028, 1008, 1046, 982, 4, 0, 0, 0, 966, 989, 1052, 1070, 0, 0, 0, 0, 1009, 1024, 1029, 1038, 0, 0, 0, 0, 1017, 1006, 1033, 989, 0, 0, 0, 0, 940, 1028, 1009, 1030, 16, 15, 13, 1, 1062, 1024, 1003, 1031, 0, 0, 0, 0, 1065, 1027, 1045, 1054, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4068, 4077, 4100, 4045, 4052, 4120, 4191] avg=4096.0000 max=4191 max_rank=7 min=4045 min_rank=4 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8448, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8064, 8320, 8192, 256, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1030, 995, 1022, 1057, 1020, 1019, 1034, 1038, 1032, 1063, 997, 994, 1058, 953, 998, 1009, 988, 1045, 1019, 1014, 1014, 965, 1006, 1034, 1064, 1041, 1072, 1042, 1032, 1031, 1051, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1021, 1045, 1049, 997, 1030, 1026, 1018, 1069, 1015, 1034, 1034, 1017, 1019, 1043, 1029, 1039, 1019, 990, 1042, 998, 1013, 1057, 976, 1027, 978, 1052, 1002, 1024, 1074, 1001, 1024, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1064, 1051, 983, 1003, 1053, 1018, 1036, 1017, 968, 1026, 993, 1010, 996, 989, 1093, 1040, 1044, 1007, 999, 1002, 1011, 1047, 982, 1025, 989, 1077, 1050, 1023, 1025, 1044, 1051, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1015, 989, 1030, 1011, 1040, 1019, 1066, 978, 1010, 1031, 1000, 1036, 1005, 1043, 1024, 1095, 970, 1104, 1029, 1011, 1023, 1042, 1047, 1043, 996, 1021, 991, 1055, 1022, 1038, 979, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1023, 995, 1023, 1041, 1037, 1077, 1021, 1040, 1006, 1009, 975, 998, 992, 1016, 1021, 1053, 981, 1071, 1040, 981, 1062, 1024, 1033, 1013, 1000, 1019, 1028, 1030, 1019, 1055, 1036, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1021, 1026, 1045, 1040, 1046, 987, 1075, 1025, 997, 1019, 1049, 1025, 1005, 1057, 983, 1022, 1045, 984, 993, 1057, 1032, 1049, 1019, 1060, 1028, 1018, 1036, 962, 999, 1035, 1036, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1010, 1032, 993, 1018, 1015, 1010, 1029, 1022, 1061, 1054, 1065, 1020, 1083, 1031, 1047, 1018, 1010, 1045, 991, 1016, 999, 1034, 1013, 994, 1012, 1015, 1029, 999, 1027, 1057, 963, 1056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[956, 1009, 1041, 1028, 1051, 1072, 995, 1070, 1011, 1013, 1035, 1033, 1053, 1053, 1029, 1033, 1033, 1000, 998, 988, 1051, 1019, 1015, 1023, 1025, 1001, 1086, 999, 1043, 1004, 1003, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1030, 995, 1022, 1057, 6, 6, 0, 0, 986, 1019, 1034, 1038, 0, 0, 0, 0, 1032, 1063, 997, 994, 12, 3, 0, 0, 1058, 953, 998, 991, 0, 0, 0, 0, 988, 1045, 1019, 1014, 31, 0, 0, 0, 1014, 965, 1006, 1034, 1, 0, 0, 0, 1064, 1041, 1072, 1042, 0, 0, 0, 0, 1032, 1024, 1051, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4077, 4101, 4000, 4097, 4020, 4219, 4138] avg=4096.0000 max=4219 max_rank=6 min=4000 min_rank=3 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1021, 1045, 1049, 997, 6, 6, 0, 0, 996, 1026, 1018, 1069, 0, 0, 0, 0, 1015, 1034, 1034, 1017, 13, 3, 0, 0, 1019, 1043, 1029, 1020, 0, 0, 0, 0, 1019, 990, 1042, 998, 31, 0, 0, 0, 1013, 1057, 976, 1027, 1, 0, 0, 0, 978, 1052, 1002, 1024, 0, 0, 0, 0, 1074, 994, 1024, 1006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4109, 4116, 4111, 4080, 4074, 4056, 4098] avg=4096.0000 max=4124 max_rank=0 min=4056 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1064, 1051, 983, 1003, 6, 6, 0, 0, 1018, 1018, 1036, 1017, 0, 0, 0, 0, 968, 1026, 993, 1010, 13, 3, 0, 0, 996, 989, 1093, 1021, 0, 0, 0, 0, 1044, 1007, 999, 1002, 32, 0, 0, 0, 1011, 1047, 982, 1025, 2, 0, 0, 0, 989, 1077, 1050, 1023, 0, 0, 0, 0, 1025, 1036, 1051, 1052, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4089, 4013, 4099, 4084, 4067, 4139, 4164] avg=4096.0000 max=4164 max_rank=7 min=4013 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1015, 989, 1030, 1011, 7, 6, 0, 0, 1006, 1019, 1066, 978, 0, 0, 0, 0, 1010, 1031, 1000, 1036, 14, 3, 0, 0, 1005, 1043, 1024, 1074, 0, 0, 0, 0, 970, 1104, 1029, 1011, 31, 0, 0, 0, 1023, 1042, 1047, 1043, 2, 0, 0, 0, 996, 1021, 991, 1055, 0, 0, 0, 0, 1022, 1030, 979, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4069, 4094, 4146, 4145, 4157, 4063, 4036] avg=4096.0000 max=4157 max_rank=5 min=4036 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1010, 1032, 993, 1018, 6, 6, 0, 0, 981, 1010, 1029, 1022, 0, 0, 0, 0, 1061, 1054, 1065, 1020, 13, 3, 0, 0, 1083, 1031, 1047, 999, 0, 0, 0, 0, 1010, 1045, 991, 1016, 31, 0, 0, 0, 999, 1034, 1013, 994, 2, 0, 0, 0, 1012, 1015, 1029, 999, 0, 0, 0, 0, 1027, 1049, 963, 1056, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4042, 4216, 4160, 4093, 4042, 4055, 4095] avg=4096.0000 max=4216 max_rank=2 min=4042 min_rank=1 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1023, 995, 1023, 1041, 11, 9, 0, 0, 993, 1077, 1021, 1040, 0, 0, 0, 0, 1006, 1009, 975, 998, 16, 10, 0, 0, 992, 1016, 1021, 1026, 0, 0, 0, 0, 981, 1071, 1040, 981, 34, 0, 0, 0, 1062, 1024, 1033, 1013, 5, 0, 0, 0, 1000, 1019, 1028, 1030, 4, 0, 0, 0, 1019, 1037, 1036, 1049, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4131, 4014, 4055, 4107, 4137, 4081, 4141] avg=4096.0000 max=4141 max_rank=7 min=4014 min_rank=2 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[956, 1009, 1041, 1028, 6, 6, 0, 0, 1016, 1072, 995, 1070, 0, 0, 0, 0, 1011, 1013, 1035, 1033, 13, 3, 0, 0, 1053, 1053, 1029, 1014, 0, 0, 0, 0, 1033, 1000, 998, 988, 32, 0, 0, 0, 1051, 1019, 1015, 1023, 1, 0, 0, 0, 1025, 1001, 1086, 999, 0, 0, 0, 0, 1043, 997, 1003, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4153, 4108, 4149, 4051, 4109, 4111, 4041] avg=4096.0000 max=4153 max_rank=1 min=4041 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1021, 1026, 1045, 1040, 6, 6, 0, 0, 1011, 987, 1075, 1025, 0, 0, 0, 0, 997, 1019, 1049, 1025, 13, 3, 0, 0, 1005, 1057, 983, 1003, 0, 0, 0, 0, 1045, 984, 993, 1057, 32, 0, 0, 0, 1032, 1049, 1019, 1060, 2, 0, 0, 0, 1028, 1018, 1036, 962, 0, 0, 0, 0, 999, 1027, 1036, 993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4098, 4106, 4048, 4111, 4162, 4044, 4055] avg=4096.0000 max=4162 max_rank=5 min=4044 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1010, 1040, 966, 1017, 1013, 1010, 1041, 1037, 1026, 1001, 1016, 1013, 1021, 995, 1032, 992, 1059, 1017, 1075, 1037, 1005, 1042, 1009, 1031, 1000, 1076, 1032, 1073, 1043, 1027, 1024, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1066, 1016, 1013, 1056, 1026, 999, 1060, 1019, 1024, 1003, 996, 1015, 1021, 1002, 1037, 1018, 1020, 1021, 1047, 1027, 1057, 1030, 1004, 1032, 1085, 1043, 1034, 981, 1013, 981, 1006, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1018, 1015, 972, 1042, 1040, 1029, 1003, 1005, 1041, 1023, 1011, 993, 993, 1032, 1047, 1030, 1011, 1050, 1013, 1038, 1060, 995, 1053, 1054, 1017, 1029, 1053, 1036, 1008, 1041, 1034, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1008, 1067, 1002, 1027, 987, 1045, 1043, 1003, 1057, 1034, 1057, 1021, 1072, 1008, 1024, 1028, 962, 960, 1045, 1060, 1021, 999, 1031, 1017, 1036, 1003, 984, 1017, 1074, 1032, 1038, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1037, 996, 1022, 1028, 1038, 1012, 1051, 1028, 1010, 1061, 1027, 1027, 1024, 1020, 1055, 977, 1068, 1053, 1037, 1047, 984, 988, 1061, 1052, 991, 1011, 977, 1042, 977, 1046, 997, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1036, 1044, 972, 1018, 1043, 1014, 1051, 1054, 987, 1053, 1010, 1039, 1071, 1032, 984, 1059, 1045, 1012, 1020, 993, 1025, 1029, 1023, 1003, 1028, 1061, 995, 1012, 1027, 1045, 969, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1029, 1039, 1069, 1042, 1028, 1035, 993, 1030, 1049, 980, 998, 1016, 1042, 1034, 1028, 1045, 996, 1018, 1003, 991, 1075, 947, 1026, 1051, 1023, 1002, 1049, 1044, 981, 1033, 1040, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1062, 1060, 1023, 999, 1037, 1006, 1012, 990, 1058, 1012, 979, 1010, 1057, 1002, 1078, 1036, 1064, 1074, 978, 1052, 1012, 1048, 969, 1012, 1053, 1063, 1003, 1014, 1003, 1034, 1006, 962] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1010, 1035, 966, 1017, 0, 0, 0, 0, 1013, 1010, 1040, 1037, 0, 0, 0, 0, 1026, 1001, 1016, 1013, 11, 4, 0, 0, 1006, 995, 1032, 992, 0, 0, 0, 0, 1059, 1017, 1075, 1022, 0, 0, 0, 0, 1005, 1042, 1009, 1031, 1, 1, 0, 0, 1000, 1064, 1032, 1073, 0, 0, 0, 0, 1043, 1027, 1024, 988, 15, 15, 1, 0] rerouted_tokens_per_ep_rank=[4028, 4100, 4071, 4025, 4173, 4089, 4169, 4113] avg=4096.0000 max=4173 max_rank=4 min=4025 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16544 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1008, 1062, 1002, 1027, 0, 0, 0, 0, 987, 1045, 1042, 1003, 0, 0, 0, 0, 1057, 1034, 1057, 1021, 10, 4, 0, 0, 1056, 1008, 1024, 1028, 0, 0, 0, 0, 962, 960, 1045, 1044, 0, 0, 0, 0, 1021, 999, 1031, 1017, 1, 1, 0, 0, 1036, 992, 984, 1017, 0, 0, 0, 0, 1074, 1032, 1038, 1006, 16, 16, 1, 0] rerouted_tokens_per_ep_rank=[4099, 4077, 4183, 4116, 4011, 4070, 4029, 4183] avg=4096.0000 max=4183 max_rank=2 min=4011 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1066, 1011, 1013, 1056, 0, 0, 0, 0, 1026, 999, 1059, 1019, 0, 0, 0, 0, 1024, 1003, 996, 1015, 10, 4, 0, 0, 1006, 1002, 1037, 1018, 0, 0, 0, 0, 1020, 1021, 1047, 1012, 0, 0, 0, 0, 1057, 1030, 1004, 1032, 1, 1, 0, 0, 1085, 1032, 1034, 981, 0, 0, 0, 0, 1013, 981, 1006, 1016, 15, 15, 1, 0] rerouted_tokens_per_ep_rank=[4146, 4103, 4052, 4063, 4100, 4125, 4132, 4047] avg=4096.0000 max=4146 max_rank=0 min=4047 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1018, 1010, 972, 1042, 0, 0, 0, 0, 1040, 1029, 1002, 1005, 0, 0, 0, 0, 1041, 1023, 1011, 993, 10, 4, 0, 0, 978, 1032, 1047, 1030, 0, 0, 0, 0, 1011, 1050, 1013, 1023, 0, 0, 0, 0, 1060, 995, 1053, 1054, 1, 1, 0, 0, 1017, 1018, 1053, 1036, 0, 0, 0, 0, 1008, 1041, 1034, 982, 15, 15, 1, 0] rerouted_tokens_per_ep_rank=[4042, 4076, 4082, 4087, 4097, 4164, 4124, 4096] avg=4096.0000 max=4164 max_rank=5 min=4042 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1029, 1034, 1069, 1042, 0, 0, 0, 0, 1028, 1035, 992, 1030, 0, 0, 0, 0, 1049, 980, 998, 1016, 10, 4, 0, 0, 1026, 1034, 1028, 1045, 0, 0, 0, 0, 996, 1018, 1003, 976, 0, 0, 0, 0, 1075, 947, 1026, 1051, 1, 1, 0, 0, 1023, 991, 1049, 1044, 0, 0, 0, 0, 981, 1033, 1040, 1032, 16, 15, 1, 0] rerouted_tokens_per_ep_rank=[4174, 4085, 4057, 4133, 3993, 4101, 4107, 4118] avg=4096.0000 max=4174 max_rank=0 min=3993 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1037, 991, 1022, 1028, 0, 0, 0, 0, 1038, 1012, 1050, 1028, 0, 0, 0, 0, 1010, 1061, 1027, 1027, 10, 4, 0, 0, 1009, 1020, 1055, 977, 0, 0, 0, 0, 1068, 1053, 1037, 1032, 0, 0, 0, 0, 984, 988, 1061, 1052, 1, 1, 0, 0, 991, 1000, 977, 1042, 0, 0, 0, 0, 977, 1046, 997, 1024, 15, 15, 1, 0] rerouted_tokens_per_ep_rank=[4078, 4128, 4139, 4061, 4190, 4087, 4010, 4075] avg=4096.0000 max=4190 max_rank=4 min=4010 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1036, 1036, 972, 1018, 0, 0, 0, 0, 1043, 1014, 1044, 1054, 0, 0, 0, 0, 987, 1053, 1010, 1039, 15, 6, 0, 0, 1051, 1032, 984, 1059, 0, 0, 0, 0, 1045, 1012, 1020, 974, 0, 0, 0, 0, 1025, 1029, 1023, 1003, 7, 2, 0, 0, 1028, 1041, 995, 1012, 0, 0, 0, 0, 1027, 1045, 969, 1014, 20, 19, 5, 0] rerouted_tokens_per_ep_rank=[4062, 4155, 4110, 4126, 4051, 4089, 4076, 4099] avg=4096.0000 max=4155 max_rank=1 min=4051 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1062, 1055, 1023, 999, 0, 0, 0, 0, 1037, 1006, 1011, 990, 0, 0, 0, 0, 1058, 1012, 979, 1010, 11, 4, 0, 0, 1041, 1002, 1078, 1036, 0, 0, 0, 0, 1064, 1074, 978, 1037, 0, 0, 0, 0, 1012, 1048, 969, 1012, 1, 1, 0, 0, 1053, 1051, 1003, 1014, 0, 0, 0, 0, 1003, 1034, 1006, 962, 16, 15, 1, 0] rerouted_tokens_per_ep_rank=[4139, 4044, 4074, 4157, 4153, 4043, 4121, 4037] avg=4096.0000 max=4157 max_rank=3 min=4037 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1010, 1060, 1068, 1046, 1064, 1015, 996, 1086, 978, 1092, 1041, 1045, 1034, 995, 992, 970, 998, 1019, 1006, 1049, 1026, 940, 1015, 1019, 1058, 1034, 1021, 1034, 999, 980, 1050, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[985, 1017, 1000, 1026, 1022, 1013, 1027, 988, 1019, 1034, 1046, 1072, 1037, 1060, 1056, 1078, 1058, 1000, 1031, 984, 1015, 1006, 955, 966, 1050, 1065, 1043, 1008, 1031, 1031, 1015, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1052, 1045, 1002, 1036, 1051, 980, 1009, 1024, 1007, 1060, 1029, 1024, 1014, 1000, 1012, 1022, 1030, 1016, 1013, 1027, 1038, 1046, 1016, 1029, 1035, 989, 1054, 1036, 975, 1039, 1027, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1018, 987, 1027, 1007, 1029, 1011, 1028, 959, 1026, 1017, 1007, 1037, 1049, 1049, 1074, 981, 1013, 1022, 1055, 1024, 995, 1034, 1056, 1025, 1042, 1051, 981, 1057, 1014, 1073, 1027, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1007, 1032, 1003, 1017, 1015, 1013, 1024, 1001, 1016, 1035, 1013, 1021, 1009, 1047, 1039, 1024, 1022, 1084, 988, 1029, 1060, 970, 1028, 1030, 1014, 1014, 1032, 1034, 1056, 1019, 1020, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1044, 1037, 1032, 1013, 1014, 1011, 975, 1031, 1069, 1023, 1029, 1019, 996, 1056, 1070, 984, 1043, 1037, 1016, 1009, 1028, 1028, 1042, 1061, 998, 984, 1047, 989, 1003, 1033, 1021, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1032, 1011, 988, 1032, 1058, 1006, 1029, 1036, 1042, 1030, 1017, 975, 993, 1052, 1018, 1043, 1053, 982, 1034, 1016, 994, 1070, 1019, 1029, 1060, 1078, 996, 1001, 1039, 1008, 989, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1048, 988, 1060, 1018, 1024, 1025, 1060, 1039, 1009, 1022, 981, 1002, 1030, 1017, 1024, 1013, 998, 1010, 1027, 1011, 1002, 964, 1047, 960, 1065, 1014, 1018, 1051, 1043, 1079, 1001, 1118] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1010, 1060, 1068, 1046, 2, 0, 0, 0, 1064, 1015, 996, 1086, 6, 6, 0, 0, 978, 1084, 1041, 1045, 0, 0, 0, 0, 1034, 995, 985, 970, 0, 0, 0, 0, 998, 1019, 1006, 1049, 6, 1, 0, 0, 1026, 940, 1015, 1019, 23, 8, 0, 0, 1035, 1034, 1021, 1034, 0, 0, 0, 0, 999, 980, 1050, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4173, 4148, 3984, 4079, 4031, 4124, 4043] avg=4096.0000 max=4186 max_rank=0 min=3984 min_rank=3 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1018, 987, 1027, 1007, 2, 0, 0, 0, 1029, 1011, 1028, 959, 7, 5, 0, 0, 1026, 1010, 1007, 1037, 0, 0, 0, 0, 1049, 1049, 1066, 981, 0, 0, 0, 0, 1013, 1022, 1055, 1024, 5, 1, 0, 0, 995, 1034, 1056, 1025, 23, 8, 0, 0, 1019, 1051, 981, 1057, 0, 0, 0, 0, 1014, 1073, 1027, 980, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4039, 4080, 4145, 4120, 4141, 4108, 4094] avg=4096.0000 max=4145 max_rank=3 min=4039 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1032, 1011, 988, 1032, 2, 0, 0, 0, 1058, 1006, 1029, 1036, 6, 6, 0, 0, 1042, 1022, 1017, 975, 0, 0, 0, 0, 993, 1052, 1011, 1043, 0, 0, 0, 0, 1053, 982, 1034, 1016, 6, 1, 0, 0, 994, 1070, 1019, 1029, 23, 8, 0, 0, 1037, 1078, 996, 1001, 0, 0, 0, 0, 1039, 1008, 989, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4141, 4056, 4099, 4092, 4143, 4112, 4060] avg=4096.0000 max=4143 max_rank=5 min=4056 min_rank=2 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1052, 1045, 1002, 1036, 2, 0, 0, 0, 1051, 980, 1009, 1024, 6, 6, 0, 0, 1007, 1052, 1029, 1024, 0, 0, 0, 0, 1014, 1000, 1005, 1022, 0, 0, 0, 0, 1030, 1016, 1013, 1027, 6, 1, 0, 0, 1038, 1046, 1016, 1029, 23, 8, 0, 0, 1012, 989, 1054, 1036, 0, 0, 0, 0, 975, 1039, 1027, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4076, 4112, 4041, 4093, 4160, 4091, 4058] avg=4096.0000 max=4160 max_rank=5 min=4041 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[985, 1017, 1000, 1026, 2, 0, 0, 0, 1022, 1013, 1027, 988, 7, 6, 0, 0, 1019, 1026, 1046, 1072, 0, 0, 0, 0, 1037, 1060, 1048, 1078, 0, 0, 0, 0, 1058, 1000, 1031, 984, 6, 1, 0, 0, 1015, 1006, 955, 966, 23, 8, 0, 0, 1027, 1065, 1043, 1008, 0, 0, 0, 0, 1031, 1031, 1015, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4063, 4163, 4223, 4080, 3973, 4143, 4093] avg=4096.0000 max=4223 max_rank=3 min=3973 min_rank=5 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1007, 1032, 1003, 1017, 2, 0, 0, 0, 1015, 1013, 1024, 1001, 6, 6, 0, 0, 1016, 1027, 1013, 1021, 0, 0, 0, 0, 1009, 1047, 1032, 1024, 0, 0, 0, 0, 1022, 1084, 988, 1029, 6, 1, 0, 0, 1060, 970, 1028, 1030, 22, 8, 0, 0, 992, 1014, 1032, 1034, 0, 0, 0, 0, 1056, 1019, 1020, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4065, 4077, 4112, 4130, 4118, 4072, 4133] avg=4096.0000 max=4133 max_rank=7 min=4061 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1048, 988, 1060, 1018, 2, 0, 0, 0, 1024, 1025, 1060, 1039, 6, 6, 0, 0, 1009, 1014, 981, 1002, 0, 0, 0, 0, 1030, 1017, 1017, 1013, 0, 0, 0, 0, 998, 1010, 1027, 1011, 6, 1, 0, 0, 1002, 964, 1047, 960, 23, 9, 0, 0, 1042, 1014, 1018, 1051, 0, 0, 0, 0, 1043, 1079, 1001, 1103, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4160, 4006, 4077, 4053, 4005, 4125, 4226] avg=4096.0000 max=4226 max_rank=7 min=4005 min_rank=5 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1044, 1037, 1032, 1013, 6, 0, 0, 0, 1014, 1011, 975, 1031, 11, 9, 0, 0, 1069, 1009, 1029, 1019, 0, 0, 0, 0, 996, 1056, 1051, 984, 0, 0, 0, 0, 1043, 1037, 1016, 1009, 8, 8, 0, 0, 1028, 1028, 1042, 1061, 25, 13, 0, 0, 973, 984, 1047, 989, 0, 0, 0, 0, 1003, 1033, 1021, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4051, 4126, 4087, 4121, 4197, 3993, 4061] avg=4096.0000 max=4197 max_rank=5 min=3993 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1023, 1019, 1053, 1030, 1060, 1037, 1004, 1006, 974, 1065, 1012, 1013, 976, 991, 1007, 1063, 1031, 1021, 1053, 1053, 1041, 997, 1057, 1030, 1046, 1011, 1043, 993, 1031, 1039, 976, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1020, 1071, 1050, 1003, 1050, 994, 1036, 1057, 1034, 1025, 1036, 1023, 1007, 1069, 1006, 1026, 1056, 1018, 1003, 1047, 1023, 1026, 1019, 1012, 991, 1048, 1042, 1033, 1007, 936, 993, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[977, 1064, 1056, 1049, 959, 1042, 1030, 1006, 1067, 1022, 1018, 1032, 1026, 1047, 1029, 947, 1053, 1009, 1015, 1006, 1024, 1037, 1036, 1038, 1016, 981, 1040, 1060, 1031, 1025, 976, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1014, 997, 1034, 1032, 997, 1033, 1048, 988, 1026, 1030, 1030, 989, 989, 1045, 1044, 1064, 1022, 1023, 1031, 1003, 1054, 1069, 1066, 1070, 1021, 1003, 1005, 1019, 1028, 1014, 1007, 973] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1059, 1028, 1023, 1022, 1024, 1026, 1006, 990, 1041, 1003, 1016, 1013, 1015, 1041, 1040, 1059, 1025, 1065, 1075, 1023, 1015, 981, 1000, 965, 973, 1057, 1088, 1076, 970, 1024, 1020, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[999, 1031, 985, 1032, 968, 1020, 1061, 1041, 1071, 1021, 1051, 963, 937, 990, 1027, 1103, 1039, 1048, 1095, 1076, 1045, 982, 1025, 1024, 997, 1042, 1053, 1024, 1016, 960, 1030, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1005, 1015, 1056, 1036, 1018, 997, 1042, 1046, 1028, 1040, 1051, 1042, 1016, 1031, 995, 1029, 1004, 995, 1049, 1033, 1025, 1060, 1025, 1001, 1014, 1017, 1039, 1015, 1009, 999, 1010, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1011, 1039, 1000, 1029, 974, 1029, 1016, 1035, 1030, 986, 1048, 976, 1005, 1015, 994, 1037, 1040, 1079, 1070, 1074, 958, 1017, 1025, 1044, 1039, 1006, 1027, 1035, 1032, 1006, 1044, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1023, 1008, 1053, 1030, 0, 0, 0, 0, 1060, 1037, 1004, 1006, 11, 2, 1, 0, 974, 1065, 1012, 1013, 0, 0, 0, 0, 976, 991, 1007, 1063, 8, 2, 0, 0, 1031, 1021, 996, 1053, 0, 0, 0, 0, 1041, 997, 1055, 1030, 0, 0, 0, 0, 1046, 1011, 1033, 993, 0, 0, 0, 0, 1031, 1039, 976, 1013, 56, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4121, 4064, 4047, 4101, 4123, 4083, 4115] avg=4096.0000 max=4123 max_rank=5 min=4047 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[977, 1052, 1056, 1049, 0, 0, 0, 0, 959, 1042, 1030, 1006, 12, 2, 1, 0, 1066, 1022, 1018, 1032, 0, 0, 0, 0, 1026, 1047, 1029, 947, 8, 2, 1, 0, 1053, 1009, 960, 1006, 0, 0, 0, 0, 1024, 1037, 1034, 1038, 0, 0, 0, 0, 1016, 981, 1030, 1060, 0, 0, 0, 0, 1031, 1025, 976, 1050, 54, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4052, 4138, 4060, 4028, 4133, 4087, 4136] avg=4096.0000 max=4138 max_rank=2 min=4028 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1014, 986, 1034, 1032, 0, 0, 0, 0, 997, 1033, 1048, 988, 11, 2, 1, 0, 1026, 1030, 1030, 989, 0, 0, 0, 0, 989, 1045, 1044, 1064, 8, 2, 0, 0, 1022, 1023, 975, 1003, 0, 0, 0, 0, 1054, 1069, 1064, 1070, 0, 0, 0, 0, 1021, 1003, 995, 1019, 0, 0, 0, 0, 1028, 1014, 1007, 973, 55, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4080, 4075, 4152, 4023, 4257, 4038, 4077] avg=4096.0000 max=4257 max_rank=5 min=4023 min_rank=4 imbalance_ratio=1.039307 max_load_reduction=28511 max_load_reduction_pct=0.870087 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1020, 1055, 1050, 1003, 0, 0, 0, 0, 1050, 994, 1036, 1057, 16, 5, 8, 0, 1029, 1025, 1036, 1023, 0, 0, 0, 0, 1007, 1069, 1006, 1026, 11, 9, 5, 0, 1056, 1018, 937, 1047, 0, 0, 0, 0, 1023, 1026, 1010, 1012, 0, 0, 0, 0, 991, 1048, 1026, 1033, 0, 0, 0, 0, 1007, 936, 993, 1007, 58, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4166, 4113, 4133, 4058, 4071, 4098, 4001] avg=4096.0000 max=4166 max_rank=1 min=4001 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16228 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[999, 1020, 985, 1032, 0, 0, 0, 0, 968, 1020, 1061, 1041, 11, 2, 1, 0, 1070, 1021, 1051, 963, 0, 0, 0, 0, 937, 990, 1027, 1103, 8, 2, 1, 0, 1039, 1048, 1036, 1076, 0, 0, 0, 0, 1045, 982, 1023, 1024, 0, 0, 0, 0, 997, 1042, 1043, 1024, 0, 0, 0, 0, 1016, 960, 1030, 1012, 58, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4104, 4105, 4068, 4199, 4074, 4106, 4076] avg=4096.0000 max=4199 max_rank=4 min=4036 min_rank=0 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1005, 1004, 1056, 1036, 0, 0, 0, 0, 1018, 997, 1042, 1046, 11, 2, 1, 0, 1028, 1040, 1051, 1042, 0, 0, 0, 0, 1016, 1031, 995, 1029, 8, 2, 0, 0, 1004, 995, 992, 1033, 0, 0, 0, 0, 1025, 1060, 1023, 1001, 0, 0, 0, 0, 1014, 1017, 1029, 1015, 0, 0, 0, 0, 1009, 999, 1010, 1026, 56, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4117, 4161, 4081, 4024, 4109, 4075, 4100] avg=4096.0000 max=4161 max_rank=2 min=4024 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1059, 1017, 1023, 1022, 0, 0, 0, 0, 1024, 1026, 1006, 990, 11, 2, 1, 0, 1040, 1003, 1016, 1013, 0, 0, 0, 0, 1015, 1041, 1040, 1059, 8, 2, 1, 0, 1025, 1065, 1017, 1023, 0, 0, 0, 0, 1015, 981, 998, 965, 0, 0, 0, 0, 973, 1057, 1078, 1076, 0, 0, 0, 0, 970, 1024, 1020, 1005, 57, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4060, 4072, 4166, 4130, 3959, 4184, 4076] avg=4096.0000 max=4184 max_rank=6 min=3959 min_rank=5 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1011, 1028, 1000, 1029, 0, 0, 0, 0, 974, 1029, 1016, 1035, 11, 2, 1, 0, 1030, 986, 1048, 976, 0, 0, 0, 0, 1005, 1015, 994, 1037, 8, 2, 0, 0, 1040, 1079, 1012, 1074, 0, 0, 0, 0, 958, 1017, 1023, 1044, 0, 0, 0, 0, 1039, 1006, 1017, 1035, 0, 0, 0, 0, 1032, 1006, 1044, 1048, 57, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4068, 4040, 4061, 4205, 4042, 4097, 4187] avg=4096.0000 max=4205 max_rank=4 min=4040 min_rank=2 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8064, 8192, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8448, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8320, 7936, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1016, 1019, 1001, 971, 1047, 1010, 1041, 1058, 1042, 1040, 1032, 1049, 1010, 1014, 985, 1071, 1050, 1014, 1031, 1015, 973, 1049, 1106, 1020, 1048, 981, 1018, 1030, 1031, 995, 1013, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1028, 975, 1034, 1025, 1045, 1024, 1053, 1031, 1031, 1006, 1005, 1000, 1067, 1037, 956, 1082, 1071, 1058, 982, 1016, 1038, 1086, 997, 1018, 1039, 996, 1055, 995, 998, 1029, 1014, 977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1030, 1014, 990, 1035, 1076, 1025, 1031, 1034, 1030, 984, 1020, 1026, 1014, 1034, 1047, 1042, 1008, 990, 1068, 1047, 980, 1001, 1042, 1022, 1054, 991, 1049, 1013, 1020, 1014, 993, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1056, 1001, 977, 1023, 1001, 1060, 1079, 982, 1007, 1048, 984, 1073, 1014, 977, 1005, 1021, 1018, 1044, 1010, 1080, 1051, 1015, 1000, 1013, 1027, 1040, 1035, 1114, 1014, 964, 1025, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[968, 1058, 1016, 1008, 1040, 1036, 1034, 1067, 1019, 1036, 1033, 994, 983, 1064, 987, 1030, 1038, 1022, 1022, 1010, 1074, 1050, 967, 1016, 1039, 1061, 992, 1033, 998, 1024, 1046, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1011, 1048, 996, 1072, 985, 998, 1043, 1042, 986, 1050, 1037, 1009, 1045, 1048, 1006, 1048, 981, 1052, 1014, 994, 1008, 1034, 1049, 1080, 1040, 1001, 1026, 972, 1003, 1036, 1020, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1034, 1017, 1033, 1027, 980, 1015, 1050, 1004, 1046, 982, 1072, 1060, 1001, 1018, 1031, 994, 1040, 1022, 1069, 1084, 956, 1032, 967, 1043, 1053, 1012, 1076, 1040, 993, 1028, 982, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1034, 1032, 1043, 1023, 1025, 1039, 1009, 1001, 1011, 1051, 1018, 984, 1030, 1000, 1047, 1037, 1003, 970, 1053, 1031, 1033, 1027, 1021, 1047, 1020, 1032, 1027, 1058, 971, 1033, 1017, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1028, 975, 1034, 1025, 16, 2, 0, 0, 1045, 1024, 1029, 1031, 0, 0, 0, 0, 1031, 1006, 1005, 1000, 0, 0, 0, 0, 1067, 1037, 956, 1082, 3, 0, 0, 0, 1071, 1058, 982, 1000, 0, 0, 0, 0, 1038, 1081, 997, 1018, 0, 0, 0, 0, 1015, 996, 1055, 995, 0, 0, 0, 0, 998, 1029, 1014, 977, 24, 24, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4129, 4042, 4145, 4111, 4134, 4061, 4066] avg=4096.0000 max=4145 max_rank=3 min=4042 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1030, 1014, 990, 1035, 16, 2, 0, 0, 1076, 1025, 1007, 1034, 0, 0, 0, 0, 1030, 984, 1020, 1026, 0, 0, 0, 0, 1014, 1034, 1047, 1042, 2, 0, 0, 0, 1008, 990, 1068, 1031, 0, 0, 0, 0, 980, 997, 1042, 1022, 0, 0, 0, 0, 1029, 991, 1049, 1013, 0, 0, 0, 0, 1020, 1014, 993, 1044, 25, 24, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4142, 4060, 4139, 4097, 4041, 4082, 4120] avg=4096.0000 max=4142 max_rank=1 min=4041 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1016, 1019, 1001, 971, 16, 2, 0, 0, 1047, 1010, 1017, 1058, 0, 0, 0, 0, 1042, 1040, 1032, 1049, 0, 0, 0, 0, 1010, 1014, 985, 1071, 2, 0, 0, 0, 1050, 1014, 1031, 999, 0, 0, 0, 0, 973, 1045, 1106, 1020, 0, 0, 0, 0, 1023, 981, 1018, 1030, 0, 0, 0, 0, 1031, 995, 1013, 988, 25, 24, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4132, 4163, 4082, 4094, 4144, 4052, 4076] avg=4096.0000 max=4163 max_rank=2 min=4025 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1011, 1048, 996, 1072, 15, 2, 0, 0, 985, 998, 1019, 1042, 0, 0, 0, 0, 986, 1050, 1037, 1009, 0, 0, 0, 0, 1045, 1048, 1006, 1048, 2, 0, 0, 0, 981, 1052, 1014, 979, 0, 0, 0, 0, 1008, 1030, 1049, 1080, 0, 0, 0, 0, 1016, 1001, 1026, 972, 0, 0, 0, 0, 1003, 1036, 1020, 1034, 24, 24, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4044, 4082, 4149, 4026, 4167, 4015, 4141] avg=4096.0000 max=4167 max_rank=5 min=4015 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1056, 1001, 977, 1023, 20, 7, 0, 0, 1001, 1060, 1049, 982, 0, 0, 0, 0, 1007, 1048, 984, 1073, 3, 0, 0, 0, 1014, 977, 1005, 1021, 8, 0, 0, 0, 1018, 1044, 1010, 1053, 0, 0, 0, 0, 1051, 997, 1000, 1013, 0, 0, 0, 0, 999, 1040, 1035, 1114, 0, 0, 0, 0, 1014, 964, 1025, 1010, 28, 30, 7, 0] rerouted_tokens_per_ep_rank=[4084, 4092, 4115, 4025, 4125, 4061, 4188, 4078] avg=4096.0000 max=4188 max_rank=6 min=4025 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[968, 1058, 1016, 1008, 16, 2, 0, 0, 1040, 1036, 1010, 1067, 0, 0, 0, 0, 1019, 1036, 1033, 994, 0, 0, 0, 0, 983, 1064, 987, 1030, 2, 0, 0, 0, 1038, 1022, 1022, 994, 0, 0, 0, 0, 1074, 1046, 967, 1016, 0, 0, 0, 0, 1015, 1061, 992, 1033, 0, 0, 0, 0, 998, 1024, 1046, 1003, 24, 24, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4153, 4082, 4066, 4076, 4103, 4101, 4119] avg=4096.0000 max=4153 max_rank=1 min=4066 min_rank=3 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1034, 1017, 1033, 1027, 17, 2, 0, 0, 980, 1015, 1026, 1004, 0, 0, 0, 0, 1046, 982, 1072, 1060, 0, 0, 0, 0, 1001, 1018, 1031, 994, 2, 0, 0, 0, 1040, 1022, 1069, 1067, 0, 0, 0, 0, 956, 1028, 967, 1043, 0, 0, 0, 0, 1028, 1012, 1076, 1040, 0, 0, 0, 0, 993, 1028, 982, 1007, 25, 24, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4025, 4160, 4046, 4198, 3994, 4156, 4059] avg=4096.0000 max=4198 max_rank=4 min=3994 min_rank=5 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1034, 1032, 1043, 1023, 16, 2, 0, 0, 1025, 1039, 986, 1001, 0, 0, 0, 0, 1011, 1051, 1018, 984, 0, 0, 0, 0, 1030, 1000, 1047, 1037, 2, 0, 0, 0, 1003, 970, 1053, 1015, 0, 0, 0, 0, 1033, 1023, 1021, 1047, 0, 0, 0, 0, 996, 1032, 1027, 1058, 0, 0, 0, 0, 971, 1033, 1017, 1041, 24, 23, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4051, 4064, 4116, 4041, 4124, 4113, 4109] avg=4096.0000 max=4150 max_rank=0 min=4041 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8064, 8448, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8192, 256, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1054, 991, 996, 1038, 937, 1046, 1047, 1009, 1057, 999, 1057, 1046, 1026, 1021, 1017, 1094, 1029, 1012, 976, 1011, 997, 1026, 978, 1023, 1030, 988, 1075, 1002, 1040, 1079, 1071, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1018, 1038, 1030, 1042, 1012, 1026, 1010, 1027, 1071, 1013, 1068, 1024, 1029, 1010, 1027, 1061, 1052, 1040, 1043, 993, 1000, 974, 956, 974, 1005, 997, 1048, 1016, 1055, 1091, 1012, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1015, 1006, 1049, 984, 998, 1005, 1024, 1056, 1060, 1030, 996, 1019, 1019, 1044, 1016, 1003, 1028, 1029, 999, 989, 1029, 994, 1051, 1070, 1033, 1030, 1000, 1064, 1016, 997, 1037, 1078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[932, 1015, 1062, 1015, 1056, 1065, 1057, 1056, 991, 1005, 1019, 1004, 1005, 1031, 972, 1058, 1007, 1033, 1046, 1075, 969, 1042, 1051, 1023, 1032, 1051, 1003, 945, 983, 1061, 1041, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1015, 993, 1014, 1001, 1043, 1033, 1017, 980, 1018, 981, 1029, 1073, 1008, 996, 1018, 1021, 1017, 987, 1063, 1016, 1080, 1047, 1059, 1030, 1011, 1081, 1007, 1058, 1058, 997, 1011, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1022, 1026, 1040, 1024, 1016, 1034, 1039, 998, 1045, 1012, 999, 1031, 1029, 976, 1034, 1051, 985, 945, 1029, 1092, 1017, 1007, 1013, 1054, 1056, 1114, 1025, 1025, 1035, 1012, 981, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1052, 977, 1031, 1046, 1079, 1025, 1032, 1067, 1026, 1010, 1048, 1038, 1039, 998, 1030, 1016, 1001, 966, 1008, 985, 1039, 1032, 1004, 1070, 1005, 1024, 1003, 997, 1027, 1023, 1006, 1064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1014, 1036, 1033, 1067, 1027, 971, 1040, 1033, 979, 1027, 1039, 1023, 1000, 1043, 1011, 1026, 994, 1027, 969, 1078, 1069, 1020, 1079, 1019, 1027, 1014, 1015, 1019, 1038, 988, 999, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1015, 1006, 1049, 984, 8, 2, 0, 0, 998, 1005, 1013, 1056, 0, 0, 0, 0, 1060, 1030, 996, 1011, 0, 0, 0, 0, 1019, 1044, 1016, 1003, 3, 1, 0, 0, 1028, 1029, 999, 989, 19, 11, 0, 0, 1029, 994, 1051, 1067, 0, 0, 0, 0, 1033, 1027, 1000, 1064, 0, 0, 0, 0, 1016, 997, 1037, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4072, 4097, 4086, 4075, 4141, 4124, 4109] avg=4096.0000 max=4141 max_rank=5 min=4064 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1054, 991, 996, 1038, 8, 2, 0, 0, 937, 1046, 1035, 1009, 0, 0, 0, 0, 1057, 999, 1057, 1038, 0, 0, 0, 0, 1026, 1021, 1017, 1094, 3, 1, 0, 0, 1029, 1012, 976, 1011, 17, 12, 0, 0, 997, 1026, 978, 1020, 0, 0, 0, 0, 1030, 985, 1075, 1002, 0, 0, 0, 0, 1040, 1079, 1071, 979, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4027, 4151, 4162, 4057, 4021, 4092, 4169] avg=4096.0000 max=4169 max_rank=7 min=4021 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[932, 1015, 1062, 1015, 8, 2, 0, 0, 1056, 1065, 1045, 1056, 0, 0, 0, 0, 991, 1005, 1019, 996, 0, 0, 0, 0, 1005, 1031, 972, 1058, 3, 1, 0, 0, 1007, 1033, 1046, 1075, 19, 12, 0, 0, 969, 1042, 1051, 1020, 0, 0, 0, 0, 1032, 1048, 1003, 945, 0, 0, 0, 0, 983, 1061, 1041, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4222, 4011, 4070, 4192, 4082, 4028, 4129] avg=4096.0000 max=4222 max_rank=1 min=4011 min_rank=2 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1018, 1038, 1030, 1042, 8, 2, 0, 0, 1012, 1026, 999, 1027, 0, 0, 0, 0, 1071, 1013, 1068, 1016, 0, 0, 0, 0, 1029, 1010, 1027, 1061, 3, 1, 0, 0, 1052, 1040, 1043, 993, 18, 11, 0, 0, 1000, 974, 956, 971, 0, 0, 0, 0, 1005, 994, 1048, 1016, 0, 0, 0, 0, 1055, 1091, 1012, 988, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4064, 4168, 4131, 4157, 3901, 4063, 4146] avg=4096.0000 max=4168 max_rank=2 min=3901 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16267 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1015, 993, 1014, 1001, 8, 2, 0, 0, 1043, 1033, 1006, 980, 0, 0, 0, 0, 1018, 981, 1029, 1065, 0, 0, 0, 0, 1008, 996, 1018, 1021, 3, 1, 0, 0, 1017, 987, 1063, 1016, 18, 11, 0, 0, 1080, 1047, 1059, 1027, 0, 0, 0, 0, 1011, 1078, 1007, 1058, 0, 0, 0, 0, 1058, 997, 1011, 988, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4062, 4093, 4047, 4112, 4213, 4154, 4054] avg=4096.0000 max=4213 max_rank=5 min=4033 min_rank=0 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1022, 1026, 1040, 1024, 8, 2, 0, 0, 1016, 1034, 1028, 998, 0, 0, 0, 0, 1045, 1012, 999, 1023, 0, 0, 0, 0, 1029, 976, 1034, 1051, 3, 1, 0, 0, 985, 945, 1029, 1092, 18, 11, 0, 0, 1017, 1007, 1013, 1051, 0, 0, 0, 0, 1056, 1111, 1025, 1025, 0, 0, 0, 0, 1035, 1012, 981, 984, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4076, 4079, 4094, 4080, 4088, 4217, 4012] avg=4096.0000 max=4217 max_rank=6 min=4012 min_rank=7 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1052, 977, 1031, 1046, 13, 7, 2, 0, 1079, 1025, 1014, 1067, 0, 0, 0, 0, 1026, 1010, 1048, 1025, 0, 0, 0, 0, 1039, 998, 1030, 1016, 7, 4, 0, 0, 1001, 966, 1008, 985, 22, 16, 0, 0, 1039, 1032, 1004, 1063, 0, 0, 0, 0, 1005, 1013, 1003, 997, 0, 0, 0, 0, 1027, 1023, 1006, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4185, 4109, 4094, 3998, 4138, 4018, 4098] avg=4096.0000 max=4185 max_rank=1 min=3998 min_rank=4 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16252 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1014, 1036, 1033, 1067, 8, 2, 0, 0, 1027, 971, 1029, 1033, 0, 0, 0, 0, 979, 1027, 1039, 1015, 0, 0, 0, 0, 1000, 1043, 1011, 1026, 3, 1, 0, 0, 994, 1027, 969, 1078, 18, 11, 0, 0, 1069, 1020, 1079, 1016, 0, 0, 0, 0, 1027, 1011, 1015, 1019, 0, 0, 0, 0, 1038, 988, 999, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4060, 4060, 4084, 4097, 4184, 4072, 4051] avg=4096.0000 max=4184 max_rank=5 min=4051 min_rank=7 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8448, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1026, 1009, 989, 1023, 998, 1027, 1066, 1019, 1042, 1026, 1066, 1017, 1012, 1037, 1034, 1029, 983, 1074, 994, 1027, 1042, 1001, 1074, 1036, 1081, 989, 1017, 1005, 976, 1004, 1026, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[997, 1018, 1027, 972, 1076, 1033, 969, 1005, 1024, 1054, 1039, 1101, 1002, 964, 1016, 1019, 1025, 1009, 1016, 1033, 1032, 1047, 1061, 1053, 1020, 1046, 1022, 989, 1031, 1050, 1013, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1044, 1051, 1036, 1021, 987, 1020, 1023, 1025, 1033, 1019, 1053, 949, 1032, 1013, 1014, 1031, 1043, 1042, 1049, 1027, 1001, 1030, 1019, 1030, 1020, 1020, 1001, 1040, 1010, 994, 1015, 1076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[999, 1012, 985, 995, 1054, 1009, 1019, 1012, 1042, 1018, 998, 1005, 1037, 1020, 990, 1042, 1023, 1024, 1020, 1020, 1050, 1014, 1032, 1066, 1038, 1016, 1058, 1003, 1056, 1069, 1021, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1058, 1002, 1031, 974, 991, 1010, 1002, 1004, 1051, 993, 1060, 1088, 1052, 1038, 1021, 1039, 1009, 1031, 1100, 1013, 1022, 1021, 1019, 1020, 1023, 999, 981, 1070, 1028, 1011, 978, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[991, 1037, 1063, 958, 981, 1049, 1025, 1038, 1023, 1025, 1005, 973, 1075, 987, 1056, 1003, 1030, 1066, 1009, 1043, 1011, 1040, 1046, 1041, 1096, 1013, 1001, 1004, 1042, 1044, 991, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1025, 1000, 1074, 1034, 1025, 1044, 1001, 1061, 997, 1008, 1000, 1012, 988, 1004, 1058, 1067, 992, 1021, 1053, 996, 1072, 961, 1062, 985, 1038, 980, 1039, 1030, 1025, 1082, 1000, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1037, 1010, 1063, 1002, 1034, 978, 1053, 1020, 1031, 1047, 1017, 983, 983, 1029, 1035, 997, 1021, 1028, 1031, 1027, 1031, 1046, 997, 1020, 1007, 1065, 1067, 1038, 1046, 1002, 1017, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 1009, 989, 1023, 26, 0, 0, 0, 998, 1027, 1066, 1019, 13, 1, 0, 0, 1039, 1026, 1066, 1017, 0, 0, 0, 0, 1012, 1037, 1034, 1029, 3, 1, 0, 0, 983, 1060, 994, 1027, 0, 0, 0, 0, 1042, 1001, 1047, 1036, 0, 0, 0, 0, 1076, 989, 1017, 1005, 0, 0, 0, 0, 976, 1004, 1026, 1019, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4124, 4148, 4116, 4064, 4126, 4087, 4030] avg=4096.0000 max=4148 max_rank=2 min=4030 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[999, 1012, 985, 995, 25, 0, 0, 0, 1054, 1009, 1019, 1012, 12, 1, 0, 0, 1039, 1018, 998, 1005, 0, 0, 0, 0, 1037, 1020, 990, 1042, 3, 1, 0, 0, 1023, 1011, 1020, 1020, 0, 0, 0, 0, 1050, 1014, 1006, 1066, 0, 0, 0, 0, 1033, 1016, 1058, 1003, 0, 0, 0, 0, 1056, 1069, 1021, 1021, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4107, 4060, 4093, 4074, 4136, 4110, 4172] avg=4096.0000 max=4172 max_rank=7 min=4016 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1044, 1051, 1036, 1021, 25, 0, 0, 0, 987, 1020, 1023, 1025, 12, 1, 0, 0, 1030, 1019, 1053, 949, 0, 0, 0, 0, 1032, 1013, 1014, 1031, 3, 1, 0, 0, 1043, 1029, 1049, 1027, 0, 0, 0, 0, 1001, 1030, 993, 1030, 0, 0, 0, 0, 1015, 1020, 1001, 1040, 0, 0, 0, 0, 1010, 994, 1015, 1076, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4177, 4068, 4051, 4094, 4148, 4054, 4076, 4100] avg=4096.0000 max=4177 max_rank=0 min=4051 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1058, 1002, 1031, 974, 25, 0, 0, 0, 991, 1010, 1002, 1004, 12, 1, 0, 0, 1048, 993, 1060, 1088, 0, 0, 0, 0, 1052, 1038, 1021, 1039, 3, 1, 0, 0, 1009, 1018, 1100, 1013, 0, 0, 0, 0, 1022, 1021, 993, 1020, 0, 0, 0, 0, 1018, 999, 981, 1070, 0, 0, 0, 0, 1028, 1011, 978, 1029, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4020, 4189, 4154, 4140, 4056, 4068, 4051] avg=4096.0000 max=4189 max_rank=2 min=4020 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[997, 1018, 1027, 972, 26, 0, 0, 0, 1076, 1033, 969, 1005, 12, 1, 0, 0, 1021, 1054, 1039, 1101, 0, 0, 0, 0, 1002, 964, 1016, 1019, 3, 1, 0, 0, 1025, 996, 1016, 1033, 0, 0, 0, 0, 1032, 1047, 1034, 1053, 0, 0, 0, 0, 1015, 1046, 1022, 989, 0, 0, 0, 0, 1031, 1050, 1013, 1005, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4096, 4215, 4005, 4070, 4166, 4072, 4104] avg=4096.0000 max=4215 max_rank=2 min=4005 min_rank=3 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[991, 1037, 1063, 958, 28, 0, 0, 0, 981, 1049, 1025, 1038, 16, 2, 0, 0, 1013, 1025, 1005, 973, 0, 0, 0, 0, 1075, 987, 1056, 1003, 10, 6, 0, 0, 1030, 1047, 1009, 1043, 0, 0, 0, 0, 1011, 1040, 1016, 1041, 0, 0, 0, 0, 1083, 1013, 1001, 1004, 0, 0, 0, 0, 1042, 1044, 991, 1002, 7, 3, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4111, 4016, 4137, 4129, 4108, 4101, 4089] avg=4096.0000 max=4137 max_rank=3 min=4016 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 1000, 1074, 1034, 26, 0, 0, 0, 1025, 1044, 1001, 1061, 12, 1, 0, 0, 994, 1008, 1000, 1012, 0, 0, 0, 0, 988, 1004, 1058, 1067, 3, 1, 0, 0, 992, 1008, 1053, 996, 0, 0, 0, 0, 1072, 961, 1035, 985, 0, 0, 0, 0, 1033, 980, 1039, 1030, 0, 0, 0, 0, 1025, 1082, 1000, 1034, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4144, 4014, 4121, 4049, 4053, 4082, 4146] avg=4096.0000 max=4159 max_rank=0 min=4014 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1037, 1010, 1063, 1002, 24, 0, 0, 0, 1034, 978, 1053, 1020, 12, 1, 0, 0, 1028, 1047, 1017, 983, 0, 0, 0, 0, 983, 1029, 1035, 997, 3, 1, 0, 0, 1021, 1015, 1031, 1027, 0, 0, 0, 0, 1031, 1046, 972, 1020, 0, 0, 0, 0, 1002, 1065, 1067, 1038, 0, 0, 0, 0, 1046, 1002, 1017, 1006, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4098, 4075, 4048, 4094, 4069, 4172, 4076] avg=4096.0000 max=4172 max_rank=6 min=4048 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8064, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1002, 1017, 1021, 1005, 1031, 1040, 1010, 1061, 1025, 1041, 1072, 1021, 1015, 994, 1061, 978, 1038, 1033, 1032, 969, 973, 984, 1023, 1001, 1038, 1053, 1016, 1042, 1017, 1038, 1059, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1048, 1022, 1024, 982, 1007, 960, 1015, 1022, 1059, 1046, 1019, 1071, 1027, 1038, 1004, 1007, 1023, 1056, 1018, 1024, 1037, 989, 1029, 1008, 1027, 1039, 1006, 1024, 1028, 1080, 1025, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1000, 1015, 1025, 1063, 1063, 1019, 1020, 1046, 992, 995, 1049, 1033, 1026, 1028, 1041, 1025, 1016, 1030, 996, 1018, 1021, 1008, 978, 989, 1028, 1009, 1007, 1108, 1022, 1063, 1058, 977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1046, 1017, 982, 1028, 1039, 1035, 1029, 1034, 969, 1028, 1043, 1016, 1054, 1014, 1008, 1003, 1043, 1040, 1041, 1067, 1047, 1038, 1011, 998, 1006, 996, 1029, 1004, 1010, 1047, 989, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1014, 1058, 986, 1044, 971, 1051, 1077, 1019, 1026, 1003, 991, 1058, 1043, 1042, 1027, 1031, 964, 1060, 1005, 1058, 1025, 1001, 1039, 1026, 1011, 1055, 1003, 1009, 1034, 1016, 1018, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1036, 1032, 972, 1010, 957, 993, 1055, 1045, 1058, 993, 1056, 945, 993, 1030, 1080, 1046, 1019, 1035, 1035, 1066, 967, 1035, 1037, 1055, 1058, 1012, 1035, 1017, 990, 1035, 1029, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1035, 1044, 1050, 969, 993, 988, 1036, 1028, 1027, 994, 1040, 1020, 1038, 1026, 1095, 994, 1051, 1076, 1030, 994, 985, 1018, 1075, 1016, 983, 1062, 1018, 1042, 979, 1030, 1016, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1028, 1041, 1035, 1019, 1066, 989, 983, 978, 1003, 1065, 1041, 999, 1022, 1018, 997, 1062, 1004, 1029, 1014, 1016, 1049, 1061, 1011, 1013, 1023, 1008, 1012, 1028, 1055, 1058, 1030, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1002, 1017, 1021, 1005, 5, 3, 3, 0, 1031, 1040, 1010, 1061, 9, 4, 0, 0, 1025, 1041, 1069, 1021, 0, 0, 0, 0, 1015, 994, 1049, 978, 0, 0, 0, 0, 1038, 1017, 1032, 969, 0, 0, 0, 0, 973, 984, 1023, 1001, 16, 11, 0, 0, 1038, 1053, 1016, 1037, 0, 0, 0, 0, 1017, 1023, 1059, 1058, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4155, 4156, 4036, 4056, 4008, 4144, 4157] avg=4096.0000 max=4157 max_rank=7 min=4008 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1000, 1015, 1025, 1063, 5, 3, 3, 0, 1063, 1019, 1020, 1046, 8, 4, 0, 0, 992, 995, 1046, 1033, 0, 0, 0, 0, 1026, 1028, 1030, 1025, 0, 0, 0, 0, 1016, 1014, 996, 1018, 0, 0, 0, 0, 1021, 1008, 978, 989, 16, 11, 0, 0, 1028, 1009, 1007, 1103, 0, 0, 0, 0, 1022, 1048, 1058, 977, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4160, 4066, 4109, 4044, 4023, 4147, 4105] avg=4096.0000 max=4160 max_rank=1 min=4023 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1048, 1022, 1024, 982, 4, 3, 3, 0, 1007, 960, 1015, 1022, 8, 4, 0, 0, 1059, 1046, 1016, 1071, 0, 0, 0, 0, 1027, 1038, 993, 1007, 0, 0, 0, 0, 1023, 1040, 1018, 1024, 0, 0, 0, 0, 1037, 989, 1029, 1008, 16, 11, 0, 0, 1027, 1039, 1006, 1020, 0, 0, 0, 0, 1028, 1065, 1025, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4016, 4192, 4065, 4105, 4090, 4092, 4122] avg=4096.0000 max=4192 max_rank=2 min=4016 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1014, 1058, 986, 1044, 4, 3, 3, 0, 971, 1051, 1077, 1019, 8, 4, 0, 0, 1026, 1003, 988, 1058, 0, 0, 0, 0, 1043, 1042, 1016, 1031, 0, 0, 0, 0, 964, 1044, 1005, 1058, 0, 0, 0, 0, 1025, 1001, 1039, 1026, 16, 10, 0, 0, 1011, 1055, 1003, 1005, 0, 0, 0, 0, 1034, 1002, 1018, 1003, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4130, 4075, 4132, 4071, 4117, 4074, 4057] avg=4096.0000 max=4132 max_rank=3 min=4057 min_rank=7 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1046, 1017, 982, 1028, 9, 9, 7, 0, 1039, 1035, 1029, 1034, 12, 9, 0, 0, 969, 1028, 1034, 1016, 0, 0, 0, 0, 1054, 1014, 989, 1003, 0, 0, 0, 0, 1043, 1020, 1041, 1067, 0, 0, 0, 0, 1047, 1038, 1011, 998, 20, 14, 0, 0, 1006, 996, 1029, 995, 0, 0, 0, 0, 1010, 1024, 989, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4158, 4047, 4060, 4171, 4128, 4026, 4080] avg=4096.0000 max=4171 max_rank=4 min=4026 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1036, 1032, 972, 1010, 4, 3, 3, 0, 957, 993, 1055, 1045, 9, 4, 0, 0, 1058, 993, 1053, 945, 0, 0, 0, 0, 993, 1030, 1068, 1046, 0, 0, 0, 0, 1019, 1019, 1035, 1066, 0, 0, 0, 0, 967, 1035, 1037, 1055, 16, 11, 0, 0, 1058, 1012, 1035, 1013, 0, 0, 0, 0, 990, 1020, 1029, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4063, 4049, 4137, 4139, 4121, 4118, 4081] avg=4096.0000 max=4139 max_rank=4 min=4049 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1035, 1044, 1050, 969, 5, 3, 3, 0, 993, 988, 1036, 1028, 9, 4, 0, 0, 1027, 994, 1037, 1020, 0, 0, 0, 0, 1038, 1026, 1083, 994, 0, 0, 0, 0, 1051, 1060, 1030, 994, 0, 0, 0, 0, 985, 1018, 1075, 1016, 16, 10, 0, 0, 983, 1062, 1018, 1037, 0, 0, 0, 0, 979, 1016, 1016, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4058, 4078, 4141, 4135, 4120, 4100, 4027] avg=4096.0000 max=4141 max_rank=3 min=4027 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1028, 1041, 1035, 1019, 4, 3, 3, 0, 1066, 989, 983, 978, 8, 4, 0, 0, 1003, 1065, 1038, 999, 0, 0, 0, 0, 1022, 1018, 986, 1062, 0, 0, 0, 0, 1004, 1013, 1014, 1016, 0, 0, 0, 0, 1049, 1061, 1011, 1013, 16, 11, 0, 0, 1023, 1008, 1012, 1024, 0, 0, 0, 0, 1055, 1043, 1030, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4028, 4105, 4088, 4047, 4161, 4067, 4139] avg=4096.0000 max=4161 max_rank=5 min=4028 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[970, 1040, 1006, 1007, 1068, 1023, 1041, 1057, 1014, 1040, 1044, 1029, 1019, 1021, 1028, 1023, 1016, 1066, 1018, 1012, 1019, 1016, 1048, 1032, 1053, 1033, 998, 1004, 1009, 994, 988, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1041, 1046, 1003, 1042, 976, 1014, 1035, 982, 995, 1042, 1026, 1036, 1052, 1021, 1002, 1066, 1022, 1057, 1073, 1045, 994, 1005, 1008, 1001, 1006, 996, 1046, 1035, 1020, 1004, 1055, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1053, 1020, 1053, 978, 1002, 1001, 1038, 1045, 1036, 1039, 1035, 1020, 1015, 1048, 1028, 1051, 991, 998, 1060, 1050, 1005, 1049, 1025, 1028, 1009, 1027, 999, 1028, 981, 1061, 1006, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[983, 1004, 1026, 1024, 1007, 1052, 1055, 995, 1024, 1028, 1032, 1039, 1018, 1054, 1045, 1033, 978, 1037, 1017, 989, 1014, 1013, 1023, 1028, 1055, 1042, 1022, 1003, 1036, 1025, 1000, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[988, 1023, 1019, 1067, 1051, 996, 1067, 1058, 1001, 1041, 1032, 997, 1056, 1058, 969, 963, 1042, 1008, 1054, 1027, 1032, 986, 987, 1019, 1025, 1006, 1032, 1055, 1020, 1069, 1009, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1058, 1066, 989, 1000, 1019, 1012, 1002, 1027, 1049, 1019, 1010, 1055, 1047, 1010, 995, 1009, 1035, 1079, 1076, 977, 993, 1019, 986, 1036, 1032, 1030, 1013, 1037, 1052, 990, 1023, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1022, 1012, 1056, 1050, 1021, 963, 1005, 1022, 1017, 1018, 976, 1016, 1033, 1054, 1069, 995, 1044, 1045, 1013, 1023, 1018, 1039, 1041, 1066, 1034, 1040, 985, 993, 1039, 1058, 985, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1030, 1001, 1078, 1016, 1006, 1018, 996, 1026, 1024, 1019, 1024, 1048, 1000, 1045, 1007, 1073, 1019, 1016, 1045, 1001, 1021, 1012, 1027, 987, 1086, 1004, 1017, 986, 1059, 1021, 992, 1064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[970, 1040, 1006, 1007, 0, 0, 0, 0, 1068, 1023, 1041, 1057, 10, 0, 0, 0, 1014, 1034, 1044, 1029, 0, 0, 0, 0, 1019, 1005, 1028, 1023, 0, 0, 0, 0, 1016, 1066, 998, 1012, 0, 0, 0, 0, 1019, 1016, 1048, 1032, 20, 3, 0, 0, 1053, 1033, 998, 1004, 4, 0, 0, 0, 1009, 994, 988, 1032, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4199, 4121, 4075, 4092, 4138, 4092, 4028] avg=4096.0000 max=4199 max_rank=1 min=4023 min_rank=0 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1041, 1046, 1003, 1042, 0, 0, 0, 0, 976, 1014, 1035, 982, 10, 0, 0, 0, 995, 1036, 1026, 1036, 0, 0, 0, 0, 1052, 1005, 1002, 1066, 0, 0, 0, 0, 1022, 1057, 1052, 1045, 0, 0, 0, 0, 994, 1005, 1008, 1001, 21, 3, 0, 0, 1006, 996, 1046, 1035, 4, 0, 0, 0, 1020, 1004, 1055, 1022, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4017, 4093, 4125, 4176, 4032, 4087, 4106] avg=4096.0000 max=4176 max_rank=4 min=4017 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[988, 1023, 1019, 1067, 0, 0, 0, 0, 1051, 996, 1067, 1058, 11, 0, 0, 0, 1001, 1035, 1032, 997, 0, 0, 0, 0, 1056, 1041, 969, 963, 0, 0, 0, 0, 1042, 1008, 1034, 1027, 0, 0, 0, 0, 1032, 986, 987, 1019, 20, 3, 0, 0, 1025, 1006, 1032, 1055, 4, 0, 0, 0, 1020, 1069, 1009, 1011, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4183, 4065, 4029, 4111, 4047, 4122, 4114] avg=4096.0000 max=4183 max_rank=1 min=4029 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[983, 1004, 1026, 1024, 0, 0, 0, 0, 1007, 1052, 1055, 995, 11, 0, 0, 0, 1024, 1022, 1032, 1039, 0, 0, 0, 0, 1018, 1037, 1045, 1033, 0, 0, 0, 0, 978, 1037, 997, 989, 0, 0, 0, 0, 1014, 1013, 1023, 1028, 20, 3, 0, 0, 1055, 1042, 1022, 1003, 4, 0, 0, 0, 1036, 1025, 1000, 1067, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4120, 4117, 4133, 4001, 4101, 4126, 4133] avg=4096.0000 max=4133 max_rank=3 min=4001 min_rank=4 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1053, 1020, 1053, 978, 0, 0, 0, 0, 1002, 1001, 1038, 1045, 11, 0, 0, 0, 1036, 1033, 1035, 1020, 0, 0, 0, 0, 1015, 1031, 1028, 1051, 0, 0, 0, 0, 991, 998, 1040, 1050, 0, 0, 0, 0, 1005, 1049, 1025, 1028, 20, 3, 0, 0, 1009, 1027, 999, 1028, 4, 0, 0, 0, 981, 1061, 1006, 989, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4097, 4124, 4125, 4079, 4130, 4067, 4042] avg=4096.0000 max=4130 max_rank=5 min=4042 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1058, 1066, 989, 1000, 0, 0, 0, 0, 1019, 1012, 1002, 1027, 10, 0, 0, 0, 1049, 1013, 1010, 1055, 0, 0, 0, 0, 1047, 994, 995, 1009, 0, 0, 0, 0, 1035, 1079, 1055, 977, 0, 0, 0, 0, 993, 1019, 986, 1036, 21, 3, 0, 0, 1032, 1030, 1013, 1037, 4, 0, 0, 0, 1052, 990, 1023, 1023, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4070, 4127, 4045, 4146, 4058, 4116, 4093] avg=4096.0000 max=4146 max_rank=4 min=4045 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1022, 1012, 1056, 1050, 0, 0, 0, 0, 1021, 963, 1005, 1022, 11, 0, 0, 0, 1017, 1012, 976, 1016, 0, 0, 0, 0, 1033, 1037, 1069, 995, 0, 0, 0, 0, 1044, 1045, 993, 1023, 0, 0, 0, 0, 1018, 1039, 1041, 1066, 20, 3, 0, 0, 1034, 1040, 985, 993, 4, 0, 0, 0, 1039, 1058, 985, 1016, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4022, 4021, 4134, 4105, 4187, 4056, 4103] avg=4096.0000 max=4187 max_rank=5 min=4021 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1030, 1001, 1075, 1016, 0, 0, 0, 0, 1006, 1018, 996, 1026, 14, 0, 0, 0, 1024, 1004, 1024, 1048, 0, 0, 0, 0, 1000, 1022, 1007, 1073, 0, 0, 0, 0, 1019, 1016, 1022, 1001, 0, 0, 0, 0, 1021, 1012, 1027, 987, 23, 5, 0, 0, 1086, 1004, 1017, 986, 6, 3, 0, 0, 1059, 1021, 992, 1064, 4, 9, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4060, 4100, 4102, 4058, 4075, 4102, 4149] avg=4096.0000 max=4149 max_rank=7 min=4058 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1025, 1008, 1037, 1090, 1005, 1031, 983, 983, 1045, 962, 1052, 1044, 1033, 1038, 1025, 990, 1047, 1005, 1033, 989, 1049, 1001, 1082, 1028, 1039, 997, 993, 998, 1027, 1039, 1058, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[998, 1085, 1081, 1064, 986, 1025, 982, 1005, 1044, 989, 997, 1030, 995, 1090, 1019, 1037, 1046, 1036, 1015, 1011, 1030, 1029, 1021, 1031, 1037, 995, 1016, 1048, 989, 983, 1041, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1032, 1012, 1029, 997, 1045, 1005, 992, 1020, 997, 1018, 1009, 1005, 1045, 1063, 1074, 970, 1022, 1044, 1024, 1011, 1046, 1029, 1059, 1019, 1022, 1030, 1036, 1008, 1048, 1042, 997, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[995, 1007, 1032, 1035, 1041, 1041, 1035, 1056, 1037, 1061, 1011, 995, 1000, 978, 984, 1020, 1017, 1006, 998, 1010, 1068, 1052, 1039, 1033, 1001, 1018, 1065, 965, 1058, 1048, 1040, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1014, 1019, 969, 1019, 1020, 1056, 977, 1047, 950, 999, 1021, 1082, 983, 1041, 999, 1069, 1036, 1005, 994, 1011, 1016, 1051, 1010, 1063, 1053, 1048, 1036, 1035, 1002, 1021, 1063, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1030, 1013, 1033, 1003, 1043, 988, 1026, 1012, 963, 1043, 1003, 1001, 1026, 1031, 1041, 1030, 994, 1050, 1046, 1056, 1055, 987, 1013, 1083, 1004, 1036, 1070, 1011, 1029, 1018, 1015, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1003, 1048, 1040, 1023, 1000, 1027, 985, 1015, 1007, 1051, 1043, 1013, 1077, 1025, 1059, 1003, 998, 1076, 1014, 1047, 982, 1030, 990, 1014, 1060, 1044, 999, 1006, 969, 1059, 1018, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[995, 967, 1045, 1003, 1024, 1021, 1050, 1012, 1028, 1057, 1001, 1043, 1029, 1037, 997, 965, 986, 1057, 1023, 1002, 1040, 1056, 1027, 1039, 1000, 1044, 949, 1031, 1069, 1056, 1039, 1076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1025, 1008, 1037, 1090, 1, 0, 0, 0, 1005, 1031, 983, 983, 28, 0, 0, 0, 1045, 962, 1052, 1044, 11, 9, 0, 0, 1033, 1038, 1025, 990, 0, 0, 0, 0, 1047, 1005, 1033, 989, 7, 0, 0, 0, 1049, 1001, 1082, 991, 0, 0, 0, 0, 1039, 997, 993, 998, 9, 0, 0, 0, 1027, 1039, 1058, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4030, 4123, 4086, 4081, 4123, 4036, 4128] avg=4096.0000 max=4161 max_rank=0 min=4030 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1014, 1019, 969, 1019, 1, 0, 0, 0, 1020, 1056, 977, 1047, 29, 0, 0, 0, 950, 999, 1021, 1082, 11, 9, 0, 0, 983, 1041, 999, 1069, 0, 0, 0, 0, 1036, 1005, 994, 1011, 7, 0, 0, 0, 1016, 1051, 1010, 1025, 0, 0, 0, 0, 1053, 1048, 1036, 1035, 9, 0, 0, 0, 1002, 1021, 1063, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4129, 4072, 4092, 4053, 4102, 4181, 4117] avg=4096.0000 max=4181 max_rank=6 min=4022 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1032, 1012, 1029, 997, 1, 0, 0, 0, 1045, 1005, 992, 1020, 28, 0, 0, 0, 997, 1018, 1009, 1005, 11, 9, 0, 0, 1045, 1063, 1074, 970, 0, 0, 0, 0, 1022, 1044, 1024, 1011, 7, 0, 0, 0, 1046, 1029, 1059, 982, 0, 0, 0, 0, 1022, 1030, 1036, 1008, 9, 0, 0, 0, 1048, 1042, 997, 990, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4090, 4049, 4152, 4108, 4116, 4105, 4077] avg=4096.0000 max=4152 max_rank=3 min=4049 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[998, 1085, 1081, 1064, 1, 0, 0, 0, 986, 1025, 982, 1005, 28, 0, 0, 0, 1044, 989, 997, 1030, 11, 9, 0, 0, 995, 1090, 1019, 1037, 0, 0, 0, 0, 1046, 1036, 1015, 1011, 7, 0, 0, 0, 1030, 1029, 1021, 994, 0, 0, 0, 0, 1037, 995, 1016, 1048, 9, 0, 0, 0, 989, 983, 1041, 985, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4229, 4026, 4080, 4141, 4115, 4074, 4105, 3998] avg=4096.0000 max=4229 max_rank=0 min=3998 min_rank=7 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1003, 1048, 1040, 1023, 1, 0, 0, 0, 1000, 1027, 985, 1015, 28, 0, 0, 0, 1007, 1051, 1043, 1013, 11, 9, 0, 0, 1077, 1025, 1059, 1003, 0, 0, 0, 0, 998, 1076, 1014, 1047, 7, 0, 0, 0, 982, 1030, 990, 977, 0, 0, 0, 0, 1060, 1044, 999, 1006, 9, 0, 0, 0, 969, 1059, 1018, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4055, 4134, 4164, 4142, 3979, 4118, 4061] avg=4096.0000 max=4164 max_rank=3 min=3979 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1030, 1013, 1033, 1003, 1, 0, 0, 0, 1043, 988, 1026, 1012, 29, 0, 0, 0, 963, 1043, 1003, 1001, 11, 9, 0, 0, 1026, 1031, 1041, 1030, 0, 0, 0, 0, 994, 1050, 1046, 1056, 7, 0, 0, 0, 1055, 987, 1013, 1045, 0, 0, 0, 0, 1004, 1036, 1070, 1011, 9, 0, 0, 0, 1029, 1018, 1015, 987, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4098, 4030, 4128, 4153, 4100, 4130, 4049] avg=4096.0000 max=4153 max_rank=4 min=4030 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[995, 967, 1045, 1003, 1, 0, 0, 0, 1024, 1021, 1050, 1012, 28, 0, 0, 0, 1028, 1057, 1001, 1043, 12, 9, 0, 0, 1029, 1037, 997, 965, 0, 0, 0, 0, 986, 1057, 1023, 1002, 7, 0, 0, 0, 1040, 1056, 1027, 1002, 0, 0, 0, 0, 1000, 1044, 949, 1031, 9, 0, 0, 0, 1069, 1056, 1039, 1047, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4135, 4150, 4028, 4075, 4125, 4033, 4211] avg=4096.0000 max=4211 max_rank=7 min=4011 min_rank=0 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[995, 1007, 1032, 1035, 5, 5, 0, 0, 1041, 1041, 1035, 1056, 32, 0, 0, 0, 1037, 1061, 1011, 995, 15, 11, 0, 0, 1000, 973, 984, 1020, 0, 0, 0, 0, 1017, 1006, 998, 1010, 10, 0, 0, 0, 1068, 1052, 1039, 990, 0, 0, 0, 0, 1001, 1018, 1065, 965, 11, 0, 0, 0, 1058, 1048, 1040, 981, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4205, 4130, 3977, 4041, 4149, 4060, 4127] avg=4096.0000 max=4205 max_rank=1 min=3977 min_rank=3 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8064, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1034, 1008, 1045, 1001, 1011, 983, 1014, 1010, 1061, 1068, 1013, 1006, 1012, 1005, 1034, 1025, 1040, 1029, 1034, 1054, 1016, 1027, 1015, 1027, 1029, 996, 1048, 1036, 1045, 992, 1053, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1002, 1018, 1024, 1076, 1003, 1030, 1012, 1047, 1008, 1043, 1042, 1008, 1007, 1058, 1071, 1001, 999, 1028, 1058, 1000, 1050, 1004, 994, 979, 1016, 1020, 1021, 1029, 1040, 1023, 1069, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[972, 1045, 1007, 1009, 1002, 1052, 1018, 1056, 1028, 1043, 1020, 1043, 1037, 1022, 1011, 1019, 1024, 1029, 953, 1021, 992, 1056, 1048, 1055, 1007, 1029, 1037, 1020, 1017, 1015, 1054, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1015, 1009, 1035, 1045, 997, 1001, 975, 1036, 1033, 1064, 1009, 1052, 1028, 1018, 1062, 1038, 1001, 1011, 1032, 1052, 1027, 980, 1037, 1064, 1037, 1070, 1002, 1006, 1044, 986, 999, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1054, 1098, 1046, 1028, 1071, 1023, 1070, 1010, 1020, 1015, 1006, 1036, 1045, 1014, 1049, 1036, 1039, 978, 987, 1005, 1043, 1011, 1050, 1001, 1015, 963, 955, 1012, 1002, 1058, 990, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[988, 1036, 1055, 1054, 1013, 1010, 1039, 1076, 1003, 974, 1045, 987, 1039, 1002, 997, 1042, 1040, 1016, 1039, 1025, 1045, 1006, 1034, 951, 1016, 1003, 1034, 1021, 1004, 1032, 1083, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1037, 1017, 1025, 1008, 1016, 1018, 1012, 991, 1000, 967, 1044, 1001, 1033, 1040, 1031, 977, 1000, 1067, 990, 1063, 996, 1063, 1019, 989, 989, 1089, 1018, 1051, 1025, 1071, 1022, 1099] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[999, 979, 1022, 1046, 1071, 1007, 1061, 1011, 1012, 984, 1032, 1044, 1004, 1018, 1062, 1044, 996, 996, 1037, 1001, 1076, 1042, 1001, 984, 1038, 1068, 1018, 1008, 1053, 1041, 1008, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1002, 1018, 1024, 1068, 0, 0, 0, 0, 1003, 1030, 1012, 1047, 2, 0, 0, 0, 1008, 1043, 1042, 1008, 6, 1, 0, 0, 1007, 1058, 1058, 1001, 0, 0, 0, 0, 999, 1028, 1058, 1000, 16, 0, 0, 0, 1050, 1004, 994, 979, 6, 4, 0, 0, 1016, 1020, 1021, 1029, 8, 0, 0, 0, 1040, 1023, 1047, 988, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4094, 4108, 4124, 4101, 4037, 4094, 4098] avg=4096.0000 max=4124 max_rank=3 min=4037 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1054, 1098, 1046, 1021, 0, 0, 0, 0, 1071, 1023, 1070, 1010, 2, 0, 0, 0, 1020, 1015, 1006, 1036, 5, 1, 0, 0, 1045, 1014, 1036, 1036, 0, 0, 0, 0, 1039, 978, 987, 1005, 14, 0, 0, 0, 1043, 1011, 1050, 1001, 5, 4, 0, 0, 1015, 963, 955, 1012, 8, 0, 0, 0, 1002, 1058, 971, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4219, 4176, 4083, 4131, 4023, 4114, 3953, 4069] avg=4096.0000 max=4219 max_rank=0 min=3953 min_rank=6 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16159 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1034, 1008, 1045, 994, 0, 0, 0, 0, 1011, 983, 1014, 1010, 2, 0, 0, 0, 1061, 1068, 1013, 1006, 5, 1, 0, 0, 1012, 1005, 1021, 1025, 0, 0, 0, 0, 1040, 1029, 1034, 1054, 15, 0, 0, 0, 1016, 1027, 1015, 1027, 6, 4, 0, 0, 1029, 996, 1048, 1036, 8, 0, 0, 0, 1045, 992, 1032, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4020, 4154, 4063, 4172, 4095, 4117, 4066] avg=4096.0000 max=4172 max_rank=4 min=4020 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[988, 1036, 1055, 1047, 0, 0, 0, 0, 1013, 1010, 1039, 1076, 2, 0, 0, 0, 1003, 974, 1045, 987, 5, 1, 0, 0, 1039, 1002, 984, 1042, 0, 0, 0, 0, 1040, 1016, 1039, 1025, 16, 0, 0, 0, 1045, 1006, 1034, 951, 6, 4, 0, 0, 1016, 1003, 1034, 1021, 8, 0, 0, 0, 1004, 1032, 1061, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4140, 4015, 4067, 4136, 4046, 4082, 4156] avg=4096.0000 max=4156 max_rank=7 min=4015 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[972, 1045, 1007, 1002, 0, 0, 0, 0, 1002, 1052, 1018, 1056, 2, 0, 0, 0, 1028, 1043, 1020, 1043, 5, 1, 0, 0, 1037, 1022, 998, 1019, 0, 0, 0, 0, 1024, 1029, 953, 1021, 15, 0, 0, 0, 992, 1056, 1048, 1055, 6, 4, 0, 0, 1007, 1029, 1037, 1020, 8, 0, 0, 0, 1017, 1015, 1033, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4130, 4140, 4076, 4042, 4161, 4101, 4092] avg=4096.0000 max=4161 max_rank=5 min=4026 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1015, 1009, 1035, 1038, 0, 0, 0, 0, 997, 1001, 975, 1036, 2, 0, 0, 0, 1033, 1064, 1009, 1052, 5, 1, 0, 0, 1028, 1018, 1049, 1038, 0, 0, 0, 0, 1001, 1011, 1032, 1052, 14, 0, 0, 0, 1027, 980, 1037, 1064, 6, 4, 0, 0, 1037, 1070, 1002, 1006, 8, 0, 0, 0, 1044, 986, 979, 1003, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4011, 4164, 4133, 4110, 4118, 4123, 4012] avg=4096.0000 max=4164 max_rank=2 min=4011 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[999, 979, 1022, 1039, 0, 0, 0, 0, 1071, 1007, 1061, 1011, 2, 0, 0, 0, 1012, 984, 1032, 1044, 5, 1, 0, 0, 1004, 1018, 1049, 1044, 0, 0, 0, 0, 996, 996, 1037, 1001, 15, 0, 0, 0, 1076, 1042, 1001, 984, 6, 4, 0, 0, 1038, 1068, 1018, 1008, 8, 0, 0, 0, 1053, 1041, 987, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4152, 4078, 4115, 4045, 4113, 4140, 4086] avg=4096.0000 max=4152 max_rank=1 min=4039 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1037, 1017, 1025, 989, 0, 0, 0, 0, 1016, 1018, 1012, 991, 8, 0, 0, 0, 1000, 967, 1044, 1001, 11, 3, 0, 0, 1033, 1040, 1009, 977, 0, 0, 0, 0, 1000, 1067, 990, 1063, 19, 0, 0, 0, 996, 1063, 1019, 989, 9, 8, 0, 0, 989, 1089, 1018, 1051, 11, 0, 0, 0, 1025, 1071, 994, 1099, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4045, 4026, 4059, 4139, 4084, 4158, 4189] avg=4096.0000 max=4189 max_rank=7 min=4026 min_rank=2 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1024, 1014, 1009, 1030, 1014, 987, 1014, 1013, 978, 1028, 1016, 1022, 1031, 1022, 1024, 1019, 1088, 1008, 1028, 1009, 1003, 1025, 1058, 1019, 1063, 1011, 1044, 1074, 1023, 985, 1025, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1007, 1040, 1017, 994, 1074, 1065, 1002, 997, 974, 994, 981, 1042, 1050, 1054, 1078, 1067, 1045, 982, 1054, 1006, 1067, 1011, 1033, 1008, 1017, 1031, 1069, 1006, 1001, 999, 990, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[991, 977, 1019, 1049, 1019, 1032, 1013, 1036, 1027, 1021, 1077, 1007, 1037, 1027, 1014, 1025, 1040, 1117, 996, 1003, 1007, 1035, 1029, 997, 991, 1017, 1003, 1053, 989, 1012, 1056, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1060, 1022, 1042, 1039, 1013, 998, 1022, 1032, 993, 1041, 1046, 967, 1047, 991, 1053, 1032, 1046, 1056, 1006, 998, 1056, 1020, 1024, 1066, 978, 1023, 1031, 978, 1025, 989, 1025, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[963, 1010, 1033, 990, 1024, 1086, 986, 1023, 1050, 1061, 1037, 1061, 993, 1021, 1022, 1000, 1035, 1051, 1038, 1062, 1009, 1060, 1052, 1015, 1016, 1014, 1043, 1010, 1029, 965, 992, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1089, 1029, 1002, 1048, 1057, 997, 1007, 1030, 982, 992, 1070, 1048, 1016, 1029, 996, 1027, 991, 1020, 1045, 1050, 984, 1047, 1020, 1095, 1019, 1027, 1036, 964, 1023, 999, 1000, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1029, 1000, 1021, 1045, 976, 1033, 1025, 1029, 1042, 997, 1022, 1032, 1016, 1037, 960, 1059, 1063, 1019, 990, 1061, 1082, 1008, 995, 998, 1084, 1050, 1036, 1020, 1001, 1045, 983, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1012, 1027, 1035, 984, 1044, 1042, 1006, 1054, 991, 1044, 1019, 998, 963, 1030, 988, 1040, 1052, 1015, 1015, 1091, 1001, 1020, 996, 1006, 1066, 1066, 1045, 1020, 1041, 1040, 1026, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1089, 1029, 1002, 1048, 11, 3, 0, 0, 1057, 997, 1007, 1030, 2, 0, 0, 0, 982, 992, 1070, 1048, 7, 5, 0, 0, 1016, 1029, 996, 1027, 0, 0, 0, 0, 955, 1020, 1045, 1050, 0, 0, 0, 0, 984, 1038, 1020, 1095, 0, 0, 0, 0, 1019, 1027, 1020, 964, 0, 0, 0, 0, 1023, 999, 1000, 1029, 33, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 4093, 4104, 4068, 4070, 4137, 4030, 4084] avg=4096.0000 max=4182 max_rank=0 min=4030 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16321 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1024, 1014, 1009, 1030, 11, 3, 0, 0, 1014, 987, 1014, 1013, 2, 0, 0, 0, 978, 1028, 1016, 1022, 7, 6, 0, 0, 1031, 1022, 1024, 1019, 0, 0, 0, 0, 1049, 1008, 1028, 1009, 0, 0, 0, 0, 1003, 1016, 1058, 1019, 0, 0, 0, 0, 1063, 1011, 1027, 1074, 0, 0, 0, 0, 1023, 985, 1025, 1060, 36, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4030, 4057, 4096, 4094, 4096, 4175, 4129] avg=4096.0000 max=4175 max_rank=6 min=4030 min_rank=1 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16494 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[963, 1010, 1033, 990, 11, 3, 0, 0, 1024, 1086, 986, 1023, 2, 0, 0, 0, 1050, 1061, 1037, 1061, 7, 6, 0, 0, 993, 1021, 1022, 1000, 0, 0, 0, 0, 997, 1051, 1038, 1062, 0, 0, 0, 0, 1009, 1051, 1052, 1015, 0, 0, 0, 0, 1016, 1014, 1026, 1010, 0, 0, 0, 0, 1029, 965, 992, 1017, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4121, 4222, 4036, 4148, 4127, 4066, 4038] avg=4096.0000 max=4222 max_rank=2 min=4010 min_rank=0 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16379 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[991, 977, 1019, 1049, 10, 3, 0, 0, 1019, 1032, 1013, 1036, 2, 0, 0, 0, 1027, 1021, 1077, 1007, 7, 5, 0, 0, 1037, 1027, 1014, 1025, 0, 0, 0, 0, 1002, 1117, 996, 1003, 0, 0, 0, 0, 1007, 1026, 1029, 997, 0, 0, 0, 0, 991, 1017, 988, 1053, 0, 0, 0, 0, 989, 1012, 1056, 1052, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4102, 4144, 4103, 4118, 4059, 4049, 4144] avg=4096.0000 max=4144 max_rank=2 min=4049 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16370 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1007, 1040, 1017, 994, 11, 3, 0, 0, 1074, 1065, 1002, 997, 2, 0, 0, 0, 974, 994, 981, 1042, 7, 6, 0, 0, 1050, 1054, 1078, 1067, 0, 0, 0, 0, 1007, 982, 1054, 1006, 0, 0, 0, 0, 1067, 1002, 1033, 1008, 0, 0, 0, 0, 1017, 1031, 1052, 1006, 0, 0, 0, 0, 1001, 999, 990, 1013, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4140, 4004, 4249, 4049, 4110, 4106, 4038] avg=4096.0000 max=4249 max_rank=3 min=4004 min_rank=2 imbalance_ratio=1.037354 max_load_reduction=28519 max_load_reduction_pct=0.870331 moved_to_echo=16303 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1029, 1000, 1021, 1045, 11, 3, 0, 0, 976, 1033, 1025, 1029, 2, 0, 0, 0, 1042, 997, 1022, 1032, 7, 5, 0, 0, 1016, 1037, 960, 1059, 0, 0, 0, 0, 1024, 1019, 990, 1061, 0, 0, 0, 0, 1082, 999, 995, 998, 0, 0, 0, 0, 1084, 1050, 1020, 1020, 0, 0, 0, 0, 1001, 1045, 983, 1010, 36, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4065, 4105, 4072, 4094, 4074, 4174, 4075] avg=4096.0000 max=4174 max_rank=6 min=4065 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16417 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1012, 1027, 1035, 984, 13, 7, 0, 0, 1044, 1042, 1006, 1054, 4, 0, 0, 0, 991, 1044, 1019, 998, 11, 10, 0, 0, 963, 1030, 988, 1040, 0, 0, 0, 0, 1006, 1015, 1015, 1091, 0, 0, 0, 0, 1001, 1005, 996, 1006, 0, 0, 0, 0, 1066, 1066, 1022, 1020, 0, 0, 0, 0, 1041, 1040, 1026, 991, 39, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4150, 4073, 4021, 4127, 4008, 4174, 4137] avg=4096.0000 max=4174 max_rank=6 min=4008 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16446 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1060, 1022, 1042, 1039, 11, 3, 0, 0, 1013, 998, 1022, 1032, 2, 0, 0, 0, 993, 1041, 1046, 967, 7, 5, 0, 0, 1047, 991, 1053, 1032, 0, 0, 0, 0, 1008, 1056, 1006, 998, 0, 0, 0, 0, 1056, 1011, 1024, 1066, 0, 0, 0, 0, 978, 1023, 1015, 978, 0, 0, 0, 0, 1025, 989, 1025, 1049, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[4177, 4067, 4059, 4123, 4068, 4157, 3994, 4123] avg=4096.0000 max=4177 max_rank=0 min=3994 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16342 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8320, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1017, 1020, 954, 998, 1055, 1020, 1019, 1036, 1091, 1023, 1047, 1036, 1056, 1057, 1046, 995, 1026, 967, 1041, 1023, 1016, 1036, 1034, 1043, 986, 984, 1005, 1034, 1033, 1022, 1002, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1024, 1008, 1025, 1050, 959, 1100, 1017, 1052, 1051, 1028, 1029, 1003, 1037, 1037, 971, 1039, 1055, 1031, 965, 987, 1022, 1021, 1051, 1055, 1006, 1053, 987, 999, 1044, 1010, 1033, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[967, 993, 1028, 977, 979, 1052, 980, 1092, 1061, 1039, 1056, 1017, 1051, 1024, 1007, 1020, 1016, 1016, 1040, 1042, 992, 1092, 987, 1072, 1000, 1015, 1049, 1004, 1039, 1047, 988, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[997, 1039, 994, 1010, 1043, 1033, 1011, 977, 1031, 984, 1037, 1021, 1026, 1036, 1010, 1017, 983, 1048, 979, 1076, 1049, 1073, 1034, 1063, 1018, 1006, 1016, 995, 1036, 1052, 1050, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1039, 1044, 1050, 1007, 1037, 975, 1046, 1004, 1008, 1008, 1061, 1048, 1046, 1037, 1072, 1038, 983, 983, 1033, 992, 1027, 1005, 1009, 1003, 1082, 1005, 1022, 1007, 985, 1037, 1052, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1065, 1010, 1026, 992, 1031, 1021, 1018, 932, 1051, 1012, 990, 1062, 1066, 1031, 1015, 1053, 993, 993, 1061, 1025, 1023, 1024, 1047, 1039, 1057, 991, 1030, 1062, 987, 987, 1066, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1040, 1039, 1059, 1035, 1029, 1008, 1027, 1032, 961, 1004, 1095, 987, 1082, 1021, 1030, 1026, 1030, 1007, 1041, 1041, 1041, 984, 1004, 1034, 1001, 1017, 1032, 988, 1015, 1006, 1017, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1022, 1022, 1029, 1015, 982, 973, 1029, 1030, 1004, 1028, 1010, 1038, 986, 1045, 1069, 990, 1013, 1051, 1023, 1033, 1067, 1042, 1015, 1034, 1034, 1053, 1015, 1019, 1008, 1059, 1052, 978] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1024, 1008, 1025, 1050, 12, 9, 0, 0, 959, 1100, 1017, 1052, 18, 2, 0, 0, 1051, 1028, 1011, 1003, 0, 0, 0, 0, 1005, 1037, 971, 1039, 0, 0, 0, 0, 1055, 1031, 965, 987, 20, 0, 0, 0, 1022, 1021, 1051, 1022, 0, 0, 0, 0, 1006, 1053, 987, 999, 24, 0, 0, 0, 1044, 1010, 1031, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4148, 4093, 4052, 4058, 4116, 4069, 4104] avg=4096.0000 max=4148 max_rank=1 min=4052 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1017, 1020, 954, 998, 12, 9, 0, 0, 1055, 1020, 1019, 1036, 18, 2, 0, 0, 1091, 1023, 1029, 1036, 0, 0, 0, 0, 1023, 1057, 1046, 995, 0, 0, 0, 0, 1026, 967, 1041, 1023, 21, 0, 0, 0, 1016, 1036, 1034, 1010, 0, 0, 0, 0, 986, 984, 1005, 1034, 24, 0, 0, 0, 1033, 1022, 1000, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4150, 4179, 4121, 4078, 4096, 4033, 4101] avg=4096.0000 max=4179 max_rank=2 min=4010 min_rank=0 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[967, 993, 1028, 977, 12, 9, 0, 0, 979, 1052, 980, 1092, 19, 2, 0, 0, 1061, 1039, 1037, 1017, 0, 0, 0, 0, 1018, 1024, 1007, 1020, 0, 0, 0, 0, 1016, 1016, 1040, 1042, 21, 0, 0, 0, 992, 1092, 987, 1038, 0, 0, 0, 0, 1000, 1015, 1049, 1004, 25, 0, 0, 0, 1039, 1047, 986, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3986, 4124, 4154, 4069, 4135, 4109, 4093, 4098] avg=4096.0000 max=4154 max_rank=2 min=3986 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1039, 1044, 1050, 1007, 12, 8, 0, 0, 1037, 975, 1046, 1004, 19, 2, 0, 0, 1008, 1008, 1042, 1048, 0, 0, 0, 0, 1013, 1037, 1072, 1038, 0, 0, 0, 0, 983, 983, 1033, 992, 21, 0, 0, 0, 1027, 1005, 1009, 972, 0, 0, 0, 0, 1082, 1005, 1022, 1007, 23, 0, 0, 0, 985, 1037, 1050, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4083, 4106, 4160, 4012, 4013, 4139, 4095] avg=4096.0000 max=4160 max_rank=0 min=4012 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[997, 1039, 994, 1010, 12, 9, 0, 0, 1043, 1033, 1011, 977, 18, 2, 0, 0, 1031, 984, 1019, 1021, 0, 0, 0, 0, 994, 1036, 1010, 1017, 0, 0, 0, 0, 983, 1048, 979, 1076, 20, 0, 0, 0, 1049, 1073, 1034, 1030, 0, 0, 0, 0, 1018, 1006, 1016, 995, 24, 0, 0, 0, 1036, 1052, 1048, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4084, 4055, 4057, 4106, 4186, 4059, 4160] avg=4096.0000 max=4186 max_rank=5 min=4055 min_rank=2 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1022, 1022, 1029, 1015, 11, 9, 0, 0, 982, 973, 1029, 1030, 18, 2, 0, 0, 1004, 1028, 992, 1038, 0, 0, 0, 0, 956, 1045, 1069, 990, 0, 0, 0, 0, 1013, 1051, 1023, 1033, 19, 0, 0, 0, 1067, 1042, 1015, 1001, 0, 0, 0, 0, 1034, 1053, 1015, 1019, 24, 0, 0, 0, 1008, 1059, 1050, 978, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4034, 4062, 4060, 4139, 4125, 4145, 4095] avg=4096.0000 max=4145 max_rank=6 min=4034 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1065, 1010, 1026, 992, 12, 9, 0, 0, 1031, 1021, 1018, 932, 17, 2, 0, 0, 1051, 1012, 973, 1062, 0, 0, 0, 0, 1033, 1031, 1015, 1053, 0, 0, 0, 0, 993, 993, 1061, 1025, 21, 0, 0, 0, 1023, 1024, 1047, 1006, 0, 0, 0, 0, 1057, 991, 1030, 1062, 24, 0, 0, 0, 987, 987, 1064, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4021, 4098, 4132, 4093, 4100, 4164, 4046] avg=4096.0000 max=4164 max_rank=6 min=4021 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1040, 1039, 1059, 1035, 16, 12, 0, 0, 1029, 1008, 1027, 1032, 24, 4, 0, 0, 961, 1004, 1069, 987, 0, 0, 0, 0, 1040, 1021, 1030, 1026, 0, 0, 0, 0, 1030, 1007, 1041, 1041, 26, 2, 0, 0, 1041, 984, 1004, 994, 0, 0, 0, 0, 1001, 1017, 1032, 988, 28, 0, 0, 0, 1015, 1006, 1013, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4201, 4124, 4021, 4117, 4147, 4023, 4066, 4069] avg=4096.0000 max=4201 max_rank=0 min=4021 min_rank=2 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1006, 977, 1001, 1022, 1030, 1021, 999, 1006, 1052, 997, 994, 1044, 1062, 989, 1030, 1001, 1008, 1080, 1033, 1019, 1042, 1073, 1062, 1005, 1027, 1016, 1037, 1022, 973, 1018, 1039, 1083] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1029, 1007, 991, 1014, 1038, 1064, 1046, 998, 1019, 1030, 1053, 1016, 1006, 1061, 983, 998, 1049, 1024, 1024, 993, 1012, 989, 1016, 1093, 1021, 1041, 1033, 981, 1006, 1037, 1054, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1073, 1042, 965, 1053, 1012, 1050, 1044, 1023, 997, 1038, 1029, 1079, 1058, 1037, 1069, 978, 1009, 988, 1008, 998, 970, 1023, 1005, 986, 1012, 1052, 992, 1024, 1036, 1094, 996, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1018, 1054, 1035, 1019, 1031, 1051, 994, 1056, 1003, 1018, 1010, 1023, 1031, 1062, 1033, 1018, 1030, 1060, 953, 1010, 1035, 1003, 1058, 1037, 1024, 1030, 990, 1026, 1019, 1004, 984, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1025, 1038, 1026, 1036, 1018, 1012, 967, 966, 1031, 1029, 998, 1001, 1021, 974, 995, 1066, 1030, 1028, 1041, 1058, 1048, 1030, 1029, 1069, 992, 989, 1071, 1006, 1066, 1042, 1040, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1062, 1027, 986, 1054, 1047, 990, 939, 1010, 1002, 968, 1009, 1026, 1033, 1036, 1050, 1014, 1057, 1031, 965, 985, 1062, 1032, 1060, 1014, 1081, 1071, 995, 1055, 1031, 1014, 1029, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[998, 1032, 1010, 1028, 1025, 1031, 1023, 1036, 1060, 1009, 1021, 1048, 1020, 964, 1035, 1045, 1031, 972, 981, 1049, 996, 1020, 1010, 1008, 1034, 1072, 1039, 994, 1025, 1103, 1019, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[998, 1030, 1043, 1035, 1073, 1068, 1043, 1016, 950, 1063, 1068, 977, 1014, 1017, 1005, 1043, 1002, 1073, 1053, 1019, 1062, 992, 1028, 1008, 997, 1050, 1026, 1019, 990, 999, 972, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1029, 1007, 991, 1014, 3, 0, 0, 0, 1038, 1064, 1046, 998, 5, 0, 0, 0, 1019, 1030, 1053, 1016, 7, 5, 0, 0, 1006, 1061, 983, 998, 2, 0, 0, 0, 1049, 1024, 1024, 993, 13, 0, 0, 0, 1012, 989, 1004, 1093, 0, 0, 0, 0, 1021, 1036, 1033, 981, 0, 0, 0, 0, 1006, 1037, 1054, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4151, 4130, 4050, 4103, 4098, 4071, 4121] avg=4096.0000 max=4151 max_rank=1 min=4044 min_rank=0 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1006, 977, 1001, 1022, 3, 0, 0, 0, 1030, 1021, 999, 1006, 5, 0, 0, 0, 1052, 997, 994, 1044, 8, 5, 0, 0, 1062, 989, 1030, 1001, 2, 0, 0, 0, 1008, 1080, 1033, 1019, 13, 0, 0, 0, 1042, 1073, 1049, 1005, 0, 0, 0, 0, 1027, 1011, 1037, 1022, 0, 0, 0, 0, 973, 1018, 1039, 1065, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4009, 4061, 4100, 4084, 4153, 4169, 4097, 4095] avg=4096.0000 max=4169 max_rank=5 min=4009 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16514 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1073, 1042, 965, 1053, 3, 0, 0, 0, 1012, 1050, 1044, 1023, 4, 0, 0, 0, 997, 1038, 1029, 1079, 7, 5, 0, 0, 1058, 1037, 1069, 978, 2, 0, 0, 0, 1009, 988, 1008, 998, 13, 0, 0, 0, 970, 1023, 994, 986, 0, 0, 0, 0, 1012, 1047, 992, 1024, 0, 0, 0, 0, 1036, 1094, 996, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4133, 4155, 4144, 4016, 3973, 4075, 4136] avg=4096.0000 max=4155 max_rank=2 min=3973 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16200 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1062, 1027, 986, 1054, 3, 0, 0, 0, 1047, 990, 939, 1010, 5, 0, 0, 0, 1002, 968, 1009, 1026, 8, 5, 0, 0, 1033, 1036, 1050, 1014, 2, 0, 0, 0, 1057, 1031, 965, 985, 13, 0, 0, 0, 1062, 1032, 1047, 1014, 0, 0, 0, 0, 1081, 1066, 995, 1055, 0, 0, 0, 0, 1031, 1014, 1029, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 3991, 4018, 4135, 4051, 4155, 4197, 4089] avg=4096.0000 max=4197 max_rank=6 min=3991 min_rank=1 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1025, 1038, 1026, 1036, 3, 0, 0, 0, 1018, 1012, 967, 966, 5, 0, 0, 0, 1031, 1029, 998, 1001, 8, 5, 0, 0, 1021, 974, 995, 1066, 2, 0, 0, 0, 1030, 1028, 1041, 1058, 13, 0, 0, 0, 1048, 1030, 1016, 1069, 0, 0, 0, 0, 992, 984, 1071, 1006, 0, 0, 0, 0, 1066, 1042, 1040, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 3968, 4072, 4058, 4170, 4163, 4053, 4156] avg=4096.0000 max=4170 max_rank=4 min=3968 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1018, 1054, 1035, 1019, 3, 0, 0, 0, 1031, 1051, 994, 1056, 5, 0, 0, 0, 1003, 1018, 1010, 1023, 8, 5, 0, 0, 1031, 1062, 1033, 1018, 2, 0, 0, 0, 1030, 1060, 953, 1010, 13, 0, 0, 0, 1035, 1003, 1045, 1037, 0, 0, 0, 0, 1024, 1025, 990, 1026, 0, 0, 0, 0, 1019, 1004, 984, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4137, 4067, 4146, 4066, 4120, 4065, 4038] avg=4096.0000 max=4146 max_rank=3 min=4038 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[998, 1030, 1043, 1035, 3, 0, 0, 0, 1073, 1068, 1043, 1016, 5, 0, 0, 0, 950, 1063, 1068, 977, 8, 5, 0, 0, 1014, 1017, 1005, 1043, 2, 0, 0, 0, 1002, 1073, 1053, 1019, 13, 0, 0, 0, 1062, 992, 1015, 1008, 0, 0, 0, 0, 997, 1045, 1026, 1019, 0, 0, 0, 0, 990, 999, 972, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4205, 4071, 4081, 4160, 4077, 4087, 3978] avg=4096.0000 max=4205 max_rank=1 min=3978 min_rank=7 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[998, 1032, 1010, 1028, 10, 3, 0, 0, 1025, 1031, 1023, 1036, 7, 0, 0, 0, 1060, 1009, 1021, 1048, 11, 6, 0, 0, 1020, 964, 1035, 1045, 6, 0, 0, 0, 1031, 972, 981, 1049, 16, 0, 0, 0, 996, 1020, 989, 1008, 0, 0, 0, 0, 1034, 1056, 1039, 994, 0, 0, 0, 0, 1025, 1103, 1019, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4122, 4155, 4070, 4049, 4013, 4123, 4155] avg=4096.0000 max=4155 max_rank=2 min=4013 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1023, 1005, 1063, 1047, 996, 1073, 986, 1019, 1039, 1012, 962, 1011, 1045, 1037, 1049, 1009, 1016, 1026, 1016, 1006, 1011, 1006, 1002, 1028, 1016, 1002, 1083, 1023, 1022, 1074, 1036, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1056, 1077, 977, 981, 1002, 992, 1035, 1030, 1064, 1027, 1046, 1033, 1011, 968, 992, 1018, 1043, 1038, 982, 1039, 1045, 1032, 1003, 1073, 1005, 1021, 1019, 1057, 1028, 1035, 1027, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1044, 1005, 981, 991, 1029, 1028, 997, 1047, 1005, 1043, 1037, 1013, 1005, 1039, 1053, 1025, 1073, 1019, 1030, 976, 1062, 990, 1052, 954, 1072, 1071, 1007, 1008, 1017, 1052, 1073, 970] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1036, 1012, 1026, 1023, 1010, 1060, 1019, 1044, 1073, 985, 1032, 1037, 964, 1058, 1015, 1045, 1055, 982, 1047, 990, 1015, 1004, 1058, 1045, 1038, 1046, 1030, 982, 1005, 1020, 972, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1022, 974, 1020, 1005, 1065, 1021, 1002, 1061, 1052, 1003, 1038, 1035, 1039, 1078, 1067, 1008, 989, 1043, 1006, 968, 1026, 1080, 1084, 981, 1029, 994, 1012, 1001, 967, 1024, 1030, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[984, 998, 1007, 1033, 1033, 1042, 1020, 1030, 1102, 1020, 1011, 999, 1035, 1003, 1033, 1000, 1043, 1023, 1006, 1036, 1015, 1036, 1016, 1002, 1045, 1063, 1015, 1039, 1021, 965, 1059, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1042, 1034, 1049, 983, 1043, 1022, 970, 1051, 1030, 1001, 1043, 1050, 1041, 1014, 1044, 1030, 1049, 985, 996, 979, 1010, 1086, 1033, 1030, 1007, 993, 1029, 1010, 1012, 1006, 1041, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1047, 1010, 1049, 1022, 1038, 1041, 1043, 1079, 1044, 1015, 966, 1009, 1015, 1064, 1031, 1014, 1034, 999, 1046, 1031, 1011, 983, 1038, 1008, 999, 1041, 1029, 1005, 1011, 993, 1030, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1023, 1005, 1063, 1047, 8, 5, 3, 0, 996, 1073, 986, 1000, 0, 0, 0, 0, 1031, 1012, 962, 1011, 0, 0, 0, 0, 1045, 1037, 1040, 1009, 0, 0, 0, 0, 1016, 1026, 1016, 1006, 19, 4, 0, 0, 1011, 1006, 997, 1028, 0, 0, 0, 0, 1016, 1000, 1083, 1023, 0, 0, 0, 0, 1022, 1074, 1036, 1025, 2, 2, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4055, 4016, 4131, 4087, 4042, 4122, 4161] avg=4096.0000 max=4161 max_rank=7 min=4016 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1056, 1077, 977, 981, 8, 5, 3, 0, 1002, 992, 1035, 1011, 0, 0, 0, 0, 1056, 1027, 1046, 1033, 0, 0, 0, 0, 1011, 968, 983, 1018, 0, 0, 0, 0, 1043, 1038, 982, 1039, 19, 4, 0, 0, 1045, 1032, 998, 1073, 0, 0, 0, 0, 1005, 1019, 1019, 1057, 0, 0, 0, 0, 1028, 1035, 1027, 1012, 2, 2, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4040, 4162, 3980, 4125, 4148, 4100, 4106] avg=4096.0000 max=4162 max_rank=2 min=3980 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1044, 1005, 981, 991, 8, 5, 3, 0, 1029, 1028, 997, 1027, 0, 0, 0, 0, 997, 1043, 1037, 1013, 0, 0, 0, 0, 1005, 1039, 1044, 1025, 0, 0, 0, 0, 1073, 1019, 1030, 976, 20, 4, 0, 0, 1062, 990, 1047, 954, 0, 0, 0, 0, 1072, 1069, 1007, 1008, 0, 0, 0, 0, 1017, 1052, 1073, 970, 2, 2, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4081, 4090, 4113, 4122, 4053, 4156, 4116] avg=4096.0000 max=4156 max_rank=6 min=4037 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1042, 1034, 1049, 983, 8, 5, 3, 0, 1043, 1022, 970, 1031, 0, 0, 0, 0, 1022, 1001, 1043, 1050, 0, 0, 0, 0, 1041, 1014, 1035, 1030, 0, 0, 0, 0, 1049, 985, 996, 979, 20, 4, 0, 0, 1010, 1086, 1028, 1030, 0, 0, 0, 0, 1007, 991, 1029, 1010, 0, 0, 0, 0, 1012, 1006, 1041, 1055, 2, 2, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4066, 4116, 4120, 4033, 4154, 4037, 4118] avg=4096.0000 max=4154 max_rank=5 min=4033 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1022, 974, 1020, 1005, 8, 5, 3, 0, 1065, 1021, 1002, 1041, 0, 0, 0, 0, 1044, 1003, 1038, 1035, 0, 0, 0, 0, 1039, 1078, 1058, 1008, 0, 0, 0, 0, 989, 1043, 1006, 968, 20, 4, 0, 0, 1026, 1080, 1079, 981, 0, 0, 0, 0, 1029, 992, 1012, 1001, 0, 0, 0, 0, 967, 1024, 1030, 1044, 2, 2, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4129, 4120, 4183, 4030, 4166, 4034, 4069] avg=4096.0000 max=4183 max_rank=3 min=4030 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[984, 998, 1007, 1033, 13, 9, 8, 0, 1033, 1042, 1020, 1007, 0, 0, 0, 0, 1089, 1020, 1011, 999, 0, 0, 0, 0, 1035, 1003, 1015, 1000, 0, 0, 0, 0, 1043, 1023, 1006, 1036, 23, 9, 0, 0, 1015, 1036, 1000, 1002, 0, 0, 0, 0, 1045, 1054, 1015, 1039, 0, 0, 0, 0, 1021, 965, 1059, 1034, 9, 8, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4102, 4119, 4053, 4140, 4053, 4153, 4096] avg=4096.0000 max=4153 max_rank=6 min=4052 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1047, 1010, 1049, 1022, 8, 5, 3, 0, 1038, 1041, 1043, 1059, 0, 0, 0, 0, 1036, 1015, 966, 1009, 0, 0, 0, 0, 1015, 1064, 1022, 1014, 0, 0, 0, 0, 1034, 999, 1046, 1031, 20, 4, 0, 0, 1011, 983, 1033, 1008, 0, 0, 0, 0, 999, 1039, 1029, 1005, 0, 0, 0, 0, 1011, 993, 1030, 1023, 2, 2, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4181, 4026, 4115, 4134, 4035, 4072, 4061] avg=4096.0000 max=4181 max_rank=1 min=4026 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1036, 1012, 1026, 1023, 8, 5, 3, 0, 1010, 1060, 1019, 1025, 0, 0, 0, 0, 1065, 985, 1032, 1037, 0, 0, 0, 0, 964, 1058, 1006, 1045, 0, 0, 0, 0, 1055, 982, 1047, 990, 19, 4, 0, 0, 1015, 1004, 1053, 1045, 0, 0, 0, 0, 1038, 1044, 1030, 982, 0, 0, 0, 0, 1005, 1020, 972, 1040, 2, 2, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4114, 4119, 4073, 4097, 4117, 4094, 4041] avg=4096.0000 max=4119 max_rank=2 min=4041 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True + [2026-08-30 04:58:03] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 816.6 | throughput per GPU (TFLOP/s/GPU): 408.3 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101992E+01 | loss scale: 1.0 | grad norm: 45.255 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1011, 1018, 1039, 1029, 982, 1054, 1009, 1004, 1057, 1031, 1036, 1000, 1052, 1043, 1018, 1034, 972, 1007, 1034, 1019, 1020, 1002, 1033, 1023, 1103, 1027, 1022, 998, 997, 1035, 1032, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1059, 1038, 1031, 1011, 1015, 1030, 1001, 1033, 1030, 1002, 1011, 978, 1043, 1033, 1010, 1040, 1009, 998, 1055, 1028, 1020, 981, 1044, 996, 1065, 1022, 995, 1047, 1041, 1098, 1005, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[992, 1068, 1009, 1029, 1039, 1036, 994, 1057, 1047, 1035, 1006, 1002, 1087, 1001, 1030, 1029, 997, 1023, 1006, 1042, 1009, 1036, 1029, 1012, 980, 1026, 1030, 1056, 1020, 999, 1011, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1006, 989, 1010, 1039, 1043, 1057, 987, 1042, 1016, 1052, 1002, 1036, 996, 1028, 1012, 1044, 1014, 1047, 1023, 1032, 1003, 1069, 1039, 1070, 979, 1061, 996, 1015, 1028, 966, 1040, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1035, 1024, 1062, 1034, 1047, 995, 1027, 1057, 1028, 1040, 1077, 1020, 1014, 1001, 1069, 999, 1005, 1047, 1004, 1043, 1022, 1033, 1043, 1038, 1002, 1011, 1037, 1021, 958, 1015, 986, 974] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1014, 1029, 1018, 995, 1014, 1005, 1027, 1062, 1007, 1044, 1013, 1005, 1054, 1056, 999, 1005, 1030, 1032, 1001, 1029, 1033, 1009, 1063, 1085, 1028, 1017, 1026, 1010, 1014, 1012, 1003, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1025, 1035, 1022, 1036, 1034, 998, 1019, 1057, 990, 1045, 943, 985, 1034, 1026, 1041, 984, 1027, 1099, 999, 1024, 1013, 1050, 980, 1033, 1051, 1052, 1014, 1047, 1050, 1014, 1017, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1029, 1037, 1015, 1004, 1021, 1074, 1020, 1000, 982, 969, 1027, 1007, 1038, 1093, 1010, 1020, 967, 1043, 1022, 1000, 1071, 1020, 993, 988, 1072, 991, 1011, 1049, 1025, 1046, 1023, 1101] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1011, 1016, 1039, 1029, 0, 0, 0, 0, 982, 1054, 1009, 997, 0, 0, 0, 0, 1057, 1031, 1036, 1000, 22, 9, 0, 0, 1030, 1043, 1018, 1034, 0, 0, 0, 0, 972, 1007, 1034, 1019, 7, 2, 0, 0, 1020, 1002, 1033, 1012, 0, 0, 0, 0, 1091, 1027, 1022, 998, 0, 0, 0, 0, 997, 1035, 1032, 1027, 11, 3, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4042, 4155, 4125, 4041, 4067, 4138, 4105] avg=4096.0000 max=4155 max_rank=2 min=4041 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1059, 1035, 1031, 1011, 0, 0, 0, 0, 1015, 1030, 1001, 1025, 0, 0, 0, 0, 1030, 1002, 1011, 978, 21, 9, 0, 0, 1022, 1033, 1010, 1040, 0, 0, 0, 0, 1009, 998, 1055, 1028, 8, 3, 0, 0, 1020, 981, 1044, 985, 0, 0, 0, 0, 1054, 1022, 995, 1047, 0, 0, 0, 0, 1041, 1098, 1005, 999, 11, 2, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4071, 4051, 4105, 4101, 4030, 4118, 4156] avg=4096.0000 max=4156 max_rank=7 min=4030 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1006, 987, 1010, 1039, 0, 0, 0, 0, 1043, 1057, 987, 1034, 0, 0, 0, 0, 1016, 1052, 1002, 1036, 20, 8, 0, 0, 976, 1028, 1012, 1044, 0, 0, 0, 0, 1014, 1047, 1023, 1032, 8, 2, 0, 0, 1003, 1069, 1039, 1059, 0, 0, 0, 0, 969, 1061, 996, 1015, 0, 0, 0, 0, 1028, 966, 1040, 1027, 11, 2, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4121, 4134, 4060, 4126, 4170, 4041, 4074] avg=4096.0000 max=4170 max_rank=5 min=4041 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[992, 1065, 1009, 1029, 0, 0, 0, 0, 1039, 1036, 994, 1049, 0, 0, 0, 0, 1047, 1035, 1006, 1002, 22, 8, 0, 0, 1065, 1001, 1030, 1029, 0, 0, 0, 0, 997, 1023, 1006, 1042, 8, 3, 0, 0, 1009, 1036, 1029, 1001, 0, 0, 0, 0, 970, 1026, 1030, 1056, 0, 0, 0, 0, 1020, 999, 1011, 1031, 11, 2, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4118, 4120, 4125, 4079, 4075, 4082, 4074] avg=4096.0000 max=4125 max_rank=3 min=4074 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1035, 1022, 1062, 1034, 0, 0, 0, 0, 1047, 995, 1027, 1049, 0, 0, 0, 0, 1028, 1040, 1077, 1020, 21, 8, 0, 0, 993, 1001, 1069, 999, 0, 0, 0, 0, 1005, 1047, 1004, 1043, 8, 2, 0, 0, 1022, 1033, 1043, 1027, 0, 0, 0, 0, 992, 1011, 1037, 1021, 0, 0, 0, 0, 958, 1015, 986, 974, 11, 2, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4118, 4194, 4062, 4109, 4125, 4061, 3946] avg=4096.0000 max=4194 max_rank=2 min=3946 min_rank=7 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16241 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1029, 1034, 1015, 1004, 0, 0, 0, 0, 1021, 1074, 1020, 993, 0, 0, 0, 0, 982, 969, 1027, 1007, 21, 9, 0, 0, 1017, 1093, 1010, 1020, 0, 0, 0, 0, 967, 1043, 1022, 1000, 7, 3, 0, 0, 1071, 1020, 993, 977, 0, 0, 0, 0, 1061, 991, 1011, 1049, 0, 0, 0, 0, 1025, 1046, 1023, 1101, 11, 2, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4108, 4015, 4140, 4042, 4061, 4112, 4208] avg=4096.0000 max=4208 max_rank=7 min=4015 min_rank=2 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1014, 1027, 1018, 995, 0, 0, 0, 0, 1014, 1005, 1027, 1054, 0, 0, 0, 0, 1007, 1044, 1013, 1005, 22, 8, 0, 0, 1032, 1056, 999, 1005, 0, 0, 0, 0, 1030, 1032, 1001, 1029, 8, 2, 0, 0, 1033, 1009, 1063, 1073, 0, 0, 0, 0, 1018, 1017, 1026, 1010, 0, 0, 0, 0, 1014, 1012, 1003, 1029, 12, 2, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4100, 4099, 4092, 4102, 4178, 4071, 4072] avg=4096.0000 max=4178 max_rank=5 min=4054 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1025, 1028, 1022, 1036, 0, 0, 0, 0, 1034, 998, 1019, 1039, 0, 0, 0, 0, 990, 1045, 943, 985, 26, 11, 0, 0, 1008, 1026, 1041, 984, 0, 0, 0, 0, 1027, 1099, 999, 1024, 12, 7, 0, 0, 1013, 1050, 980, 1019, 0, 0, 0, 0, 1032, 1052, 1014, 1047, 0, 0, 0, 0, 1050, 1014, 1017, 1024, 14, 8, 6, 0] rerouted_tokens_per_ep_rank=[4111, 4090, 4000, 4059, 4168, 4062, 4145, 4133] avg=4096.0000 max=4168 max_rank=4 min=4000 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1020, 1012, 1036, 1027, 1045, 1001, 1012, 1006, 1016, 1017, 1041, 1022, 1056, 993, 1030, 992, 1040, 1034, 981, 1016, 1044, 1020, 1010, 999, 1002, 994, 1053, 996, 1067, 1038, 1080, 1068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1052, 1031, 989, 1045, 1044, 988, 1033, 1016, 1001, 1031, 997, 1030, 1021, 1046, 1006, 1059, 1048, 1073, 992, 1009, 1016, 1030, 1062, 1010, 973, 1062, 1030, 1039, 1037, 1002, 989, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1095, 981, 1016, 1058, 1036, 1036, 1052, 1017, 982, 1014, 980, 1025, 1018, 1039, 1017, 1049, 1008, 1057, 1022, 987, 1048, 992, 1012, 1003, 982, 1021, 1037, 1012, 1026, 1072, 1022, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1016, 1058, 968, 1080, 985, 1045, 1015, 1020, 1074, 1059, 1007, 1021, 992, 1003, 1030, 1037, 998, 1037, 984, 1065, 1045, 1023, 1006, 986, 965, 1062, 1026, 1068, 1050, 1013, 1000, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1030, 1064, 986, 1027, 1008, 1013, 1032, 1036, 1004, 1008, 1005, 979, 1011, 1030, 983, 1034, 1006, 1045, 1106, 1008, 1062, 1024, 1039, 1044, 970, 997, 1035, 1056, 1040, 1013, 1012, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1081, 1030, 999, 1039, 1017, 972, 1021, 1009, 965, 1031, 1004, 1025, 1083, 998, 1010, 1009, 1013, 1023, 1037, 990, 1016, 1023, 1068, 1091, 1025, 1007, 1036, 1041, 1017, 1013, 1019, 1056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1010, 1065, 1010, 1020, 1022, 1013, 1037, 1021, 1036, 1002, 1016, 994, 1007, 999, 1031, 1025, 1019, 1006, 1076, 1064, 1036, 1052, 1000, 1025, 979, 997, 1039, 1014, 1065, 986, 1042, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1045, 1041, 1031, 1040, 1024, 1010, 990, 1044, 1009, 1028, 998, 1040, 1003, 1035, 1065, 1036, 1024, 976, 1009, 1074, 1055, 1003, 997, 1042, 1003, 1054, 1040, 963, 1007, 1026, 1038, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[985, 1012, 1036, 1027, 0, 0, 0, 0, 1045, 1001, 1012, 1006, 18, 10, 0, 0, 1016, 1017, 1041, 1022, 37, 9, 0, 0, 1056, 993, 1030, 992, 7, 0, 0, 0, 1040, 1017, 981, 1016, 0, 0, 0, 0, 1021, 1020, 1010, 999, 0, 0, 0, 0, 1002, 994, 1053, 996, 26, 5, 0, 0, 1067, 1038, 1080, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4092, 4142, 4078, 4054, 4050, 4076, 4216] avg=4096.0000 max=4216 max_rank=7 min=4050 min_rank=5 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1023, 1031, 989, 1045, 0, 0, 0, 0, 1044, 988, 1033, 1016, 13, 4, 0, 0, 1001, 1031, 997, 1030, 31, 6, 0, 0, 1021, 1046, 1006, 1059, 2, 0, 0, 0, 1048, 1067, 992, 1009, 0, 0, 0, 0, 1003, 1030, 1062, 1010, 0, 0, 0, 0, 973, 1062, 1030, 1039, 23, 0, 0, 0, 1037, 1002, 989, 976, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4098, 4096, 4134, 4116, 4105, 4127, 4004] avg=4096.0000 max=4134 max_rank=3 min=4004 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1065, 981, 1016, 1058, 0, 0, 0, 0, 1036, 1036, 1052, 1017, 13, 4, 0, 0, 982, 1014, 980, 1025, 32, 6, 0, 0, 1018, 1039, 1017, 1049, 2, 0, 0, 0, 1008, 1051, 1022, 987, 0, 0, 0, 0, 1035, 992, 1012, 1003, 0, 0, 0, 0, 982, 1021, 1037, 1012, 24, 0, 0, 0, 1026, 1072, 1022, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4158, 4039, 4125, 4068, 4042, 4076, 4140] avg=4096.0000 max=4158 max_rank=1 min=4039 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1002, 1064, 986, 1027, 0, 0, 0, 0, 1008, 1013, 1032, 1036, 14, 4, 0, 0, 1004, 1008, 1005, 979, 32, 6, 0, 0, 1011, 1030, 983, 1034, 2, 0, 0, 0, 1006, 1039, 1106, 1008, 0, 0, 0, 0, 1048, 1024, 1039, 1044, 0, 0, 0, 0, 970, 997, 1035, 1056, 22, 0, 0, 0, 1040, 1013, 1012, 1029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4107, 4034, 4060, 4159, 4155, 4080, 4094] avg=4096.0000 max=4159 max_rank=4 min=4034 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1016, 1041, 1031, 1040, 0, 0, 0, 0, 1024, 1010, 990, 1044, 13, 4, 0, 0, 1009, 1028, 998, 1040, 31, 6, 0, 0, 1003, 1035, 1065, 1036, 2, 0, 0, 0, 1024, 970, 1009, 1074, 0, 0, 0, 0, 1042, 1003, 997, 1042, 0, 0, 0, 0, 1003, 1054, 1040, 963, 23, 0, 0, 0, 1007, 1026, 1038, 987, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4085, 4112, 4141, 4077, 4084, 4083, 4058] avg=4096.0000 max=4141 max_rank=3 min=4058 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1052, 1030, 999, 1039, 0, 0, 0, 0, 1017, 972, 1021, 1009, 13, 4, 0, 0, 965, 1031, 1004, 1025, 32, 6, 0, 0, 1083, 998, 1010, 1009, 2, 0, 0, 0, 1013, 1017, 1037, 990, 0, 0, 0, 0, 1003, 1023, 1068, 1091, 0, 0, 0, 0, 1025, 1007, 1036, 1041, 23, 0, 0, 0, 1017, 1013, 1019, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4036, 4063, 4102, 4057, 4185, 4132, 4073] avg=4096.0000 max=4185 max_rank=5 min=4036 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[983, 1065, 1010, 1020, 0, 0, 0, 0, 1022, 1013, 1037, 1021, 13, 4, 0, 0, 1036, 1002, 1016, 994, 32, 5, 0, 0, 1007, 999, 1031, 1025, 2, 0, 0, 0, 1019, 1000, 1076, 1064, 0, 0, 0, 0, 1023, 1052, 1000, 1025, 0, 0, 0, 0, 979, 997, 1039, 1014, 22, 0, 0, 0, 1065, 986, 1042, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4110, 4085, 4064, 4159, 4100, 4051, 4121] avg=4096.0000 max=4159 max_rank=4 min=4051 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[989, 1058, 968, 1080, 0, 0, 0, 0, 985, 1045, 1015, 1020, 13, 4, 0, 0, 1074, 1059, 1007, 1021, 31, 5, 0, 0, 992, 1003, 1030, 1037, 2, 0, 0, 0, 998, 1031, 984, 1065, 0, 0, 0, 0, 1032, 1023, 1006, 986, 0, 0, 0, 0, 965, 1062, 1026, 1068, 22, 0, 0, 0, 1050, 1013, 1000, 999, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4082, 4197, 4064, 4078, 4047, 4143, 4062] avg=4096.0000 max=4197 max_rank=2 min=4047 min_rank=5 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8064, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7936, 8320, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8064, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1068, 1039, 1018, 978, 1003, 1029, 988, 1020, 996, 1058, 1050, 999, 981, 1033, 1025, 1040, 1063, 1059, 1031, 993, 983, 1032, 1051, 1036, 1031, 1016, 1012, 1025, 1006, 1012, 1017, 1076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1043, 982, 1029, 1010, 996, 1026, 1083, 998, 1061, 1033, 998, 1039, 1030, 1075, 1046, 1043, 998, 1032, 1018, 986, 1044, 1031, 1018, 1068, 1014, 1070, 989, 1010, 1018, 964, 1003, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1031, 1045, 986, 1023, 1023, 997, 1061, 1029, 1085, 1017, 1020, 1054, 1001, 1037, 1054, 1051, 1025, 1066, 1016, 1016, 972, 1021, 1041, 983, 979, 1012, 1014, 1080, 1032, 1030, 961, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1071, 1047, 1014, 1026, 1051, 1003, 1041, 1004, 1068, 1011, 1033, 1000, 1043, 1007, 1044, 975, 983, 1008, 1028, 1020, 1012, 996, 1008, 1075, 1029, 989, 1054, 1007, 1005, 1014, 1046, 1056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1022, 994, 1059, 1009, 1028, 1089, 996, 1003, 1004, 1016, 1041, 1025, 1003, 980, 1007, 1034, 1039, 1072, 1027, 1031, 1043, 1016, 1015, 1016, 1020, 1032, 1035, 1015, 1008, 1031, 1052, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1026, 995, 1031, 1047, 999, 1052, 968, 990, 989, 1008, 1046, 1023, 977, 1070, 1044, 1065, 1067, 1005, 969, 1039, 1055, 1020, 1045, 1042, 1037, 1054, 1027, 1046, 1007, 1028, 1012, 985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[982, 1088, 994, 1085, 1048, 1024, 996, 1050, 1062, 1056, 1073, 1035, 1019, 1010, 967, 1046, 987, 978, 1039, 1027, 1033, 1024, 1028, 1000, 987, 1022, 989, 996, 1036, 1038, 1005, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1018, 1030, 1052, 994, 1037, 975, 1023, 1005, 1007, 1004, 1050, 1037, 1062, 999, 1023, 1061, 1049, 1065, 1021, 996, 1005, 1035, 1062, 1065, 1009, 1001, 970, 1093, 1038, 1025, 973, 984] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1061, 1039, 1018, 978, 0, 0, 0, 0, 1003, 1029, 988, 1020, 12, 4, 0, 0, 996, 1058, 1021, 999, 0, 0, 0, 0, 981, 1033, 1025, 1030, 0, 0, 0, 0, 1063, 1059, 1031, 993, 1, 0, 0, 0, 983, 1032, 1051, 1024, 0, 0, 0, 0, 1031, 1016, 1012, 1025, 6, 6, 0, 0, 1006, 1012, 1017, 1076, 29, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4056, 4074, 4069, 4147, 4090, 4096, 4140] avg=4096.0000 max=4147 max_rank=4 min=4056 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1024, 1045, 986, 1023, 0, 0, 0, 0, 1023, 997, 1061, 1029, 11, 4, 0, 0, 1085, 1017, 992, 1054, 0, 0, 0, 0, 1001, 1037, 1054, 1041, 0, 0, 0, 0, 1025, 1066, 1016, 1016, 1, 0, 0, 0, 972, 1021, 1041, 972, 0, 0, 0, 0, 979, 1012, 1014, 1080, 6, 6, 0, 0, 1032, 1030, 961, 1006, 28, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4125, 4148, 4133, 4124, 4006, 4097, 4057] avg=4096.0000 max=4148 max_rank=2 min=4006 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1036, 982, 1029, 1010, 0, 0, 0, 0, 996, 1026, 1083, 998, 12, 4, 0, 0, 1061, 1033, 971, 1039, 0, 0, 0, 0, 1030, 1075, 1046, 1033, 0, 0, 0, 0, 998, 1032, 1018, 986, 1, 0, 0, 0, 1044, 1031, 1018, 1056, 0, 0, 0, 0, 1014, 1070, 989, 1010, 6, 6, 0, 0, 1018, 964, 1003, 1013, 27, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4119, 4104, 4184, 4035, 4149, 4095, 4025] avg=4096.0000 max=4184 max_rank=3 min=4025 min_rank=7 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1015, 994, 1059, 1009, 0, 0, 0, 0, 1028, 1089, 996, 1003, 12, 4, 0, 0, 1004, 1016, 1013, 1025, 0, 0, 0, 0, 1003, 980, 1007, 1024, 0, 0, 0, 0, 1039, 1072, 1027, 1031, 1, 0, 0, 0, 1043, 1016, 1015, 1004, 0, 0, 0, 0, 1020, 1032, 1035, 1015, 6, 6, 0, 0, 1008, 1031, 1052, 1006, 28, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4132, 4058, 4014, 4170, 4078, 4114, 4125] avg=4096.0000 max=4170 max_rank=4 min=4014 min_rank=3 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1011, 1030, 1052, 994, 0, 0, 0, 0, 1037, 975, 1023, 1005, 12, 4, 0, 0, 1007, 1004, 1021, 1037, 0, 0, 0, 0, 1062, 999, 1023, 1051, 0, 0, 0, 0, 1049, 1065, 1021, 996, 1, 0, 0, 0, 1005, 1035, 1062, 1053, 0, 0, 0, 0, 1009, 1001, 970, 1093, 6, 6, 0, 0, 1038, 1025, 973, 984, 29, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4056, 4069, 4135, 4132, 4155, 4085, 4049] avg=4096.0000 max=4155 max_rank=5 min=4049 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1052, 1047, 1014, 1026, 0, 0, 0, 0, 1051, 1003, 1041, 1004, 17, 5, 0, 0, 1068, 1011, 1001, 1000, 0, 0, 0, 0, 1043, 1007, 1044, 961, 0, 0, 0, 0, 983, 1008, 1028, 1020, 8, 0, 0, 0, 1012, 996, 1008, 1051, 0, 0, 0, 0, 1029, 989, 1054, 1007, 11, 9, 0, 0, 1005, 1014, 1046, 1056, 32, 7, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4121, 4080, 4055, 4047, 4067, 4099, 4160] avg=4096.0000 max=4160 max_rank=7 min=4047 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[975, 1088, 994, 1085, 0, 0, 0, 0, 1048, 1024, 996, 1050, 12, 4, 0, 0, 1062, 1056, 1044, 1035, 0, 0, 0, 0, 1019, 1010, 967, 1036, 0, 0, 0, 0, 987, 978, 1039, 1027, 1, 0, 0, 0, 1033, 1024, 1028, 988, 0, 0, 0, 0, 987, 1022, 989, 996, 6, 6, 0, 0, 1036, 1038, 1005, 1044, 29, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4134, 4197, 4032, 4032, 4073, 4006, 4152] avg=4096.0000 max=4197 max_rank=2 min=4006 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1019, 995, 1031, 1047, 0, 0, 0, 0, 999, 1052, 968, 990, 12, 4, 0, 0, 989, 1008, 1018, 1023, 0, 0, 0, 0, 977, 1070, 1044, 1055, 0, 0, 0, 0, 1067, 1005, 969, 1039, 1, 0, 0, 0, 1055, 1020, 1045, 1030, 0, 0, 0, 0, 1037, 1054, 1027, 1046, 6, 6, 0, 0, 1007, 1028, 1012, 985, 28, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4025, 4038, 4146, 4081, 4150, 4176, 4060] avg=4096.0000 max=4176 max_rank=6 min=4025 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1006, 1036, 1062, 1038, 1076, 1023, 1054, 1024, 1015, 1004, 984, 997, 1068, 1018, 1036, 1030, 980, 1036, 1044, 1015, 1030, 1027, 1072, 999, 1014, 1064, 952, 1022, 1015, 1018, 1014, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1038, 1010, 1013, 1026, 1028, 1021, 1037, 1000, 1004, 1059, 1040, 1045, 1063, 1069, 1039, 1044, 985, 999, 1023, 1040, 997, 1050, 999, 1009, 1009, 1015, 1017, 1022, 1016, 990, 1009, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1033, 1043, 972, 1047, 1021, 1021, 1015, 1000, 1030, 1013, 1018, 951, 1068, 1039, 1062, 1016, 955, 985, 1081, 1010, 1008, 991, 1029, 1032, 1036, 1010, 1029, 1054, 1086, 1070, 993, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1035, 1041, 1045, 1008, 1000, 1043, 1015, 1033, 1046, 1037, 1038, 1016, 1026, 1038, 1025, 1005, 1010, 1004, 1018, 1042, 1034, 1008, 978, 1041, 1024, 1009, 1039, 1062, 1000, 1008, 1003, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[978, 1036, 1034, 1019, 1028, 1047, 1054, 1039, 1016, 1028, 1025, 985, 1062, 1038, 1016, 1042, 1022, 983, 1007, 1050, 1035, 1032, 1038, 1005, 1003, 1006, 1012, 1055, 996, 1014, 1040, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1077, 1034, 1027, 1028, 1008, 1018, 971, 1048, 1050, 1042, 1050, 973, 1028, 1000, 1009, 1025, 976, 1077, 1006, 1043, 1011, 1018, 1060, 1021, 1026, 989, 1026, 1014, 1061, 992, 1028, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1071, 989, 1023, 1011, 1032, 978, 1024, 1042, 1003, 991, 1060, 1061, 1001, 1053, 1021, 1023, 1015, 1043, 978, 1019, 993, 988, 1037, 1025, 1064, 987, 1088, 995, 1005, 1028, 1048, 1072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1041, 1012, 998, 1024, 986, 1049, 1005, 993, 1026, 1041, 1032, 1006, 1013, 1025, 1004, 1077, 989, 1055, 1021, 1005, 1012, 1003, 1032, 1028, 1042, 1034, 1000, 1020, 1061, 1065, 1086, 983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[997, 1036, 1062, 1038, 0, 0, 0, 0, 1076, 1023, 1054, 1024, 4, 0, 0, 0, 1015, 1004, 984, 997, 7, 2, 0, 0, 1028, 1018, 1036, 1030, 0, 0, 0, 0, 980, 1036, 1044, 1015, 32, 0, 0, 0, 1030, 1027, 1072, 999, 7, 8, 0, 0, 1014, 1064, 952, 1022, 3, 0, 0, 0, 1015, 1018, 1014, 981, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4181, 4009, 4112, 4107, 4143, 4055, 4028] avg=4096.0000 max=4181 max_rank=1 min=4009 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[969, 1036, 1034, 1019, 0, 0, 0, 0, 1028, 1047, 1054, 1039, 4, 0, 0, 0, 1016, 1028, 1025, 985, 7, 2, 0, 0, 1022, 1038, 1016, 1042, 0, 0, 0, 0, 1022, 983, 1007, 1050, 32, 0, 0, 0, 1035, 1032, 1038, 1005, 7, 8, 0, 0, 1003, 1006, 1012, 1055, 3, 0, 0, 0, 996, 1014, 1040, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4172, 4063, 4118, 4094, 4125, 4079, 4059] avg=4096.0000 max=4172 max_rank=1 min=4058 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1029, 1010, 1013, 1026, 0, 0, 0, 0, 1028, 1021, 1037, 1000, 4, 0, 0, 0, 1004, 1059, 1040, 1045, 7, 2, 0, 0, 1023, 1069, 1039, 1044, 0, 0, 0, 0, 985, 999, 1023, 1040, 32, 0, 0, 0, 997, 1050, 999, 1009, 8, 8, 0, 0, 1009, 1015, 1017, 1022, 3, 0, 0, 0, 1016, 990, 1009, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4090, 4157, 4175, 4079, 4071, 4066, 4052] avg=4096.0000 max=4175 max_rank=3 min=4052 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16268 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1026, 1041, 1045, 1008, 0, 0, 0, 0, 1000, 1043, 1015, 1033, 4, 0, 0, 0, 1046, 1037, 1038, 1016, 7, 2, 0, 0, 988, 1038, 1025, 1005, 0, 0, 0, 0, 1010, 1004, 1018, 1042, 31, 0, 0, 0, 1034, 1008, 978, 1041, 7, 7, 0, 0, 1024, 1009, 1039, 1062, 3, 0, 0, 0, 1000, 1008, 1003, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4095, 4146, 4056, 4105, 4075, 4137, 4034] avg=4096.0000 max=4146 max_rank=2 min=4034 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1032, 1012, 998, 1024, 0, 0, 0, 0, 986, 1049, 1005, 993, 4, 0, 0, 0, 1026, 1041, 1032, 1006, 7, 2, 0, 0, 976, 1025, 1004, 1077, 0, 0, 0, 0, 989, 1055, 1021, 1005, 30, 0, 0, 0, 1012, 1003, 1032, 1028, 7, 7, 0, 0, 1042, 1034, 1000, 1020, 3, 0, 0, 0, 1061, 1065, 1086, 969, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4037, 4114, 4082, 4100, 4089, 4099, 4181] avg=4096.0000 max=4181 max_rank=7 min=4037 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1024, 1043, 972, 1047, 0, 0, 0, 0, 1021, 1021, 1015, 1000, 4, 0, 0, 0, 1030, 1013, 1018, 951, 7, 2, 0, 0, 1028, 1039, 1062, 1016, 0, 0, 0, 0, 955, 985, 1081, 1010, 32, 0, 0, 0, 1008, 991, 1029, 1032, 8, 8, 0, 0, 1036, 1010, 1029, 1054, 3, 0, 0, 0, 1086, 1070, 993, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4061, 4021, 4145, 4063, 4076, 4132, 4184] avg=4096.0000 max=4184 max_rank=7 min=4021 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1068, 1034, 1027, 1028, 0, 0, 0, 0, 1008, 1018, 971, 1048, 4, 0, 0, 0, 1050, 1042, 1050, 973, 7, 2, 0, 0, 990, 1000, 1009, 1025, 0, 0, 0, 0, 976, 1077, 1006, 1043, 31, 0, 0, 0, 1011, 1018, 1060, 1021, 7, 7, 0, 0, 1026, 989, 1026, 1014, 3, 0, 0, 0, 1061, 992, 1028, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4049, 4124, 4024, 4133, 4124, 4058, 4099] avg=4096.0000 max=4157 max_rank=0 min=4024 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1047, 989, 1023, 1011, 0, 0, 0, 0, 1032, 978, 1024, 1042, 7, 0, 0, 0, 1003, 991, 1060, 1061, 10, 9, 0, 0, 959, 1053, 1021, 1023, 0, 0, 0, 0, 1015, 1043, 978, 1019, 32, 0, 0, 0, 993, 988, 1037, 1025, 12, 10, 0, 0, 1064, 987, 1088, 995, 8, 0, 0, 0, 1005, 1028, 1048, 1050, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4083, 4134, 4056, 4087, 4065, 4142, 4131] avg=4096.0000 max=4142 max_rank=6 min=4056 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[7936, 8192, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1014, 1055, 1003, 1024, 1064, 974, 1048, 991, 985, 1026, 1053, 1000, 969, 1047, 1022, 994, 1043, 1036, 1052, 1010, 1051, 1055, 1061, 1025, 1043, 997, 995, 987, 986, 1075, 1017, 1066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[985, 975, 1027, 1061, 1035, 1007, 1019, 1065, 968, 1019, 1022, 1005, 999, 1063, 999, 998, 1022, 1033, 1084, 1060, 1100, 1054, 956, 1069, 1000, 1015, 997, 983, 993, 1051, 1036, 1068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1067, 996, 1031, 1026, 1022, 1063, 1013, 1012, 1050, 1008, 1011, 1036, 1025, 1005, 982, 1020, 1058, 1016, 1008, 1044, 1035, 1041, 981, 1000, 1022, 1010, 1026, 1019, 1048, 1053, 981, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1009, 1028, 967, 996, 1070, 1070, 1002, 1005, 1002, 1051, 989, 1062, 1022, 1075, 1014, 1031, 1013, 1017, 1027, 1050, 1043, 967, 1051, 1006, 1025, 1080, 1023, 1043, 1011, 1015, 1014, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1037, 1004, 1049, 1048, 990, 1049, 1020, 955, 1058, 1019, 1027, 1075, 1020, 992, 1033, 1052, 1020, 942, 1058, 986, 1065, 1044, 1039, 1019, 990, 995, 1080, 1029, 1058, 1020, 1018, 977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1032, 1041, 1052, 1025, 977, 1006, 968, 986, 1072, 1035, 1006, 1047, 1006, 1025, 1059, 1021, 1031, 1006, 1014, 1024, 1048, 1032, 982, 1051, 1010, 962, 1052, 1007, 1084, 1003, 1031, 1073] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1018, 996, 971, 1034, 1016, 976, 1076, 1011, 1019, 1071, 1032, 971, 1066, 1003, 1067, 977, 1017, 987, 1050, 1037, 1052, 1032, 1043, 1041, 964, 981, 1032, 1053, 1040, 1064, 1032, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1032, 1049, 974, 1043, 1036, 1007, 1004, 1009, 1032, 1056, 1065, 1011, 1038, 1044, 994, 1037, 1005, 1003, 1057, 1087, 994, 1056, 1021, 991, 1058, 1022, 1002, 992, 1012, 1022, 1005, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[985, 975, 1027, 1061, 6, 5, 0, 0, 1035, 1007, 1019, 1065, 18, 9, 0, 0, 968, 1006, 1022, 1005, 0, 0, 0, 0, 999, 1063, 999, 998, 8, 0, 0, 0, 1022, 1033, 1069, 1060, 0, 0, 0, 0, 1069, 1054, 956, 1069, 0, 0, 0, 0, 1000, 1015, 997, 983, 31, 4, 0, 0, 993, 1029, 1036, 1068, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4153, 4001, 4067, 4184, 4148, 4030, 4126] avg=4096.0000 max=4184 max_rank=4 min=4001 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1037, 1004, 1049, 1048, 6, 5, 0, 0, 990, 1049, 1020, 955, 17, 9, 0, 0, 1058, 1006, 1027, 1075, 0, 0, 0, 0, 1020, 992, 1033, 1052, 8, 0, 0, 0, 1020, 942, 1043, 986, 0, 0, 0, 0, 1035, 1044, 1039, 1019, 0, 0, 0, 0, 990, 995, 1080, 1029, 30, 4, 0, 0, 1058, 999, 1018, 977, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4040, 4166, 4105, 3991, 4137, 4128, 4052] avg=4096.0000 max=4166 max_rank=2 min=3991 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1014, 1055, 1003, 1024, 6, 5, 0, 0, 1064, 974, 1048, 991, 18, 9, 0, 0, 985, 1013, 1053, 1000, 0, 0, 0, 0, 969, 1047, 1022, 994, 8, 0, 0, 0, 1043, 1036, 1037, 1010, 0, 0, 0, 0, 1022, 1055, 1061, 1025, 0, 0, 0, 0, 1043, 997, 995, 987, 29, 4, 0, 0, 986, 1053, 1017, 1066, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4104, 4051, 4040, 4126, 4163, 4055, 4122] avg=4096.0000 max=4163 max_rank=5 min=4040 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1067, 996, 1031, 1026, 6, 5, 0, 0, 1022, 1063, 1013, 1012, 18, 9, 0, 0, 1050, 995, 1011, 1036, 0, 0, 0, 0, 1025, 1005, 982, 1020, 8, 0, 0, 0, 1058, 1016, 993, 1044, 0, 0, 0, 0, 1006, 1041, 981, 1000, 0, 0, 0, 0, 1022, 1010, 1026, 1019, 29, 4, 0, 0, 1048, 1031, 981, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4137, 4092, 4040, 4111, 4028, 4110, 4119] avg=4096.0000 max=4137 max_rank=1 min=4028 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1032, 1041, 1052, 1025, 6, 5, 0, 0, 977, 1006, 968, 986, 17, 9, 0, 0, 1072, 1022, 1006, 1047, 0, 0, 0, 0, 1006, 1025, 1059, 1021, 8, 0, 0, 0, 1031, 1006, 999, 1024, 0, 0, 0, 0, 1019, 1032, 982, 1051, 0, 0, 0, 0, 1010, 962, 1052, 1007, 29, 4, 0, 0, 1084, 982, 1031, 1073, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 3963, 4147, 4119, 4060, 4084, 4064, 4170] avg=4096.0000 max=4170 max_rank=7 min=3963 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1018, 996, 971, 1034, 6, 5, 0, 0, 1016, 976, 1076, 1011, 18, 9, 0, 0, 1019, 1058, 1032, 971, 0, 0, 0, 0, 1066, 1003, 1067, 977, 8, 0, 0, 0, 1017, 987, 1035, 1037, 0, 0, 0, 0, 1023, 1032, 1043, 1041, 0, 0, 0, 0, 964, 981, 1032, 1053, 29, 4, 0, 0, 1040, 1042, 1032, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4106, 4080, 4121, 4076, 4139, 4063, 4153] avg=4096.0000 max=4153 max_rank=7 min=4030 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1009, 1028, 967, 996, 11, 11, 0, 0, 1070, 1070, 1002, 1005, 23, 13, 0, 0, 1002, 1027, 989, 1062, 0, 0, 0, 0, 1022, 1075, 1014, 1031, 13, 0, 0, 0, 1013, 1017, 1003, 1050, 0, 0, 0, 0, 1011, 967, 1051, 1006, 0, 0, 0, 0, 1025, 1080, 1023, 1043, 32, 9, 0, 0, 1011, 983, 1014, 990, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4183, 4080, 4155, 4083, 4035, 4212, 3998] avg=4096.0000 max=4212 max_rank=6 min=3998 min_rank=7 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1032, 1049, 974, 1043, 6, 5, 0, 0, 1036, 1007, 1004, 1009, 17, 9, 0, 0, 1032, 1043, 1065, 1011, 0, 0, 0, 0, 1038, 1044, 994, 1037, 8, 0, 0, 0, 1005, 1003, 1042, 1087, 0, 0, 0, 0, 966, 1056, 1021, 991, 0, 0, 0, 0, 1058, 1022, 1002, 992, 28, 4, 0, 0, 1012, 1001, 1005, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4082, 4151, 4121, 4137, 4034, 4106, 4028] avg=4096.0000 max=4151 max_rank=2 min=4028 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1027, 1027, 1046, 1023, 1069, 1006, 1065, 1034, 1046, 1011, 1013, 1031, 1035, 1037, 1042, 1043, 1045, 991, 1012, 1035, 985, 1017, 1002, 1012, 1009, 1047, 991, 1003, 941, 1034, 1029, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[984, 1062, 988, 1075, 1078, 1019, 1036, 1024, 991, 1025, 1011, 1020, 1006, 994, 960, 997, 1038, 988, 1050, 1045, 1012, 1023, 998, 1003, 1039, 1019, 1028, 1037, 1055, 1024, 1066, 1073] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1082, 1013, 1069, 1077, 1046, 970, 1030, 1021, 983, 985, 973, 976, 1039, 1035, 1014, 1035, 1013, 1036, 1056, 1004, 1046, 1018, 1073, 969, 981, 1041, 1037, 1036, 1035, 1011, 1030, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1026, 990, 1027, 1030, 1052, 1038, 1054, 970, 1049, 1056, 1001, 1045, 1041, 1038, 1029, 997, 987, 1030, 1021, 1045, 992, 1014, 1040, 980, 1020, 1035, 1046, 1013, 1035, 1026, 998, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1005, 1095, 1053, 1043, 992, 1049, 1024, 1005, 1042, 1077, 1009, 1018, 1018, 1043, 1016, 1028, 1009, 1026, 965, 1022, 1014, 1007, 1018, 1039, 1008, 1023, 1042, 996, 1015, 1018, 992, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1055, 1007, 1014, 994, 1037, 996, 1048, 1048, 1002, 1022, 1039, 1059, 1014, 1034, 1000, 1032, 1058, 1014, 1000, 993, 1007, 996, 1022, 1048, 1030, 1031, 1029, 1055, 1027, 975, 1057, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[983, 1047, 1064, 1008, 1075, 1021, 1058, 1036, 1024, 1004, 969, 994, 1040, 1001, 1047, 1020, 1038, 1001, 1021, 1005, 1002, 1044, 1010, 1043, 1007, 1029, 1057, 1047, 992, 1045, 992, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1015, 1028, 1017, 1010, 1033, 1017, 1017, 1029, 1011, 1036, 1028, 983, 1013, 1002, 988, 1046, 966, 1044, 1017, 1020, 1059, 1039, 1028, 1048, 1034, 1026, 1031, 1018, 1010, 1016, 1088, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1027, 1027, 1021, 1023, 0, 0, 0, 0, 1040, 1006, 1065, 1034, 0, 0, 0, 0, 1046, 1011, 1013, 1031, 29, 0, 0, 0, 1035, 1037, 1042, 1043, 9, 0, 0, 0, 1045, 991, 1012, 1035, 21, 0, 0, 0, 985, 1017, 1002, 1012, 15, 4, 0, 0, 1009, 1047, 982, 1003, 0, 0, 0, 0, 941, 1034, 1029, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4145, 4130, 4166, 4104, 4035, 4041, 4049] avg=4096.0000 max=4166 max_rank=3 min=4035 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16229 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1026, 990, 1002, 1030, 0, 0, 0, 0, 1024, 1038, 1054, 970, 0, 0, 0, 0, 1049, 1056, 1001, 1045, 28, 0, 0, 0, 1041, 1038, 1029, 997, 9, 0, 0, 0, 987, 1030, 1021, 1045, 21, 0, 0, 0, 992, 1014, 1040, 980, 15, 4, 0, 0, 1020, 1035, 1037, 1013, 0, 0, 0, 0, 1035, 1026, 998, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4086, 4179, 4114, 4104, 4045, 4105, 4087] avg=4096.0000 max=4179 max_rank=2 min=4045 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[984, 1062, 964, 1075, 0, 0, 0, 0, 1049, 1019, 1036, 1024, 0, 0, 0, 0, 991, 1025, 1011, 1020, 29, 0, 0, 0, 1006, 994, 960, 997, 9, 0, 0, 0, 1038, 988, 1050, 1045, 20, 0, 0, 0, 1012, 1023, 998, 1003, 15, 4, 0, 0, 1039, 1019, 1019, 1037, 0, 0, 0, 0, 1055, 1024, 1066, 1058, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4128, 4076, 3966, 4141, 4055, 4114, 4203] avg=4096.0000 max=4203 max_rank=7 min=3966 min_rank=3 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1082, 1013, 1043, 1077, 0, 0, 0, 0, 1018, 970, 1030, 1021, 0, 0, 0, 0, 983, 985, 973, 976, 28, 0, 0, 0, 1039, 1035, 1014, 1035, 9, 0, 0, 0, 1013, 1036, 1056, 1004, 22, 0, 0, 0, 1046, 1018, 1073, 969, 15, 4, 0, 0, 981, 1041, 1028, 1036, 0, 0, 0, 0, 1035, 1011, 1030, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4215, 4039, 3945, 4132, 4131, 4125, 4086, 4095] avg=4096.0000 max=4215 max_rank=0 min=3945 min_rank=2 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[983, 1047, 1038, 1008, 0, 0, 0, 0, 1046, 1021, 1058, 1036, 0, 0, 0, 0, 1024, 1004, 969, 994, 29, 0, 0, 0, 1040, 1001, 1047, 1020, 9, 0, 0, 0, 1038, 1001, 1021, 1005, 22, 0, 0, 0, 1002, 1044, 1010, 1043, 15, 4, 0, 0, 1007, 1029, 1048, 1047, 0, 0, 0, 0, 992, 1045, 992, 1029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4161, 4020, 4117, 4087, 4118, 4131, 4058] avg=4096.0000 max=4161 max_rank=1 min=4020 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1005, 1095, 1013, 1043, 0, 0, 0, 0, 962, 1049, 1024, 1005, 0, 0, 0, 0, 1042, 1077, 1009, 1018, 30, 6, 0, 0, 1018, 1043, 1016, 1028, 14, 7, 0, 0, 1009, 1026, 965, 1022, 25, 0, 0, 0, 1014, 1007, 1018, 1039, 18, 9, 0, 0, 1008, 1023, 1028, 996, 0, 0, 0, 0, 1015, 1018, 992, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4040, 4182, 4126, 4047, 4105, 4055, 4057] avg=4096.0000 max=4182 max_rank=2 min=4040 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16264 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1055, 1007, 989, 994, 0, 0, 0, 0, 1009, 996, 1048, 1048, 0, 0, 0, 0, 1002, 1022, 1039, 1059, 28, 0, 0, 0, 1014, 1034, 1000, 1032, 9, 0, 0, 0, 1058, 1014, 1000, 993, 21, 0, 0, 0, 1007, 996, 1022, 1048, 15, 4, 0, 0, 1030, 1031, 1020, 1055, 0, 0, 0, 0, 1027, 975, 1057, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4101, 4150, 4089, 4086, 4092, 4136, 4069] avg=4096.0000 max=4150 max_rank=2 min=4045 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1015, 1028, 992, 1010, 0, 0, 0, 0, 1005, 1017, 1017, 1029, 0, 0, 0, 0, 1011, 1036, 1028, 983, 28, 0, 0, 0, 1013, 1002, 988, 1046, 9, 0, 0, 0, 966, 1044, 1017, 1020, 21, 0, 0, 0, 1059, 1039, 1028, 1048, 15, 4, 0, 0, 1034, 1026, 1022, 1018, 0, 0, 0, 0, 1010, 1016, 1088, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4068, 4086, 4058, 4068, 4193, 4100, 4150] avg=4096.0000 max=4193 max_rank=5 min=4045 min_rank=0 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8448, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1095, 975, 1028, 991, 973, 1025, 990, 1018, 1012, 989, 1067, 1050, 1017, 1007, 1018, 1027, 1100, 1003, 1012, 1017, 1062, 1017, 1017, 1037, 1027, 1053, 1054, 1034, 958, 1016, 1077, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1031, 1082, 1050, 1021, 974, 1040, 977, 992, 1002, 1012, 980, 1031, 973, 1017, 1050, 1026, 988, 1070, 1029, 997, 1034, 1050, 1004, 1037, 1019, 1060, 1091, 1033, 1032, 1023, 1065, 978] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[982, 1049, 974, 999, 1048, 1016, 1011, 1023, 1015, 1036, 965, 1054, 1025, 1008, 1046, 977, 1048, 1001, 1050, 1040, 1056, 1003, 1068, 1001, 1032, 1053, 1060, 1040, 1047, 986, 1037, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1057, 1072, 999, 1031, 1002, 1054, 1018, 1061, 1006, 983, 1033, 1044, 1024, 1035, 1018, 1032, 1014, 982, 973, 985, 1017, 1009, 1053, 1056, 1004, 998, 1070, 1017, 998, 1086, 1035, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1042, 1050, 1027, 1063, 1089, 1006, 1013, 957, 1055, 986, 993, 1027, 1049, 1014, 974, 974, 1016, 1079, 1048, 1018, 990, 1020, 1016, 1026, 1019, 1015, 1032, 1042, 1056, 1026, 1059, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1034, 1016, 1012, 1010, 1017, 1006, 1035, 1025, 1033, 1033, 1068, 996, 1018, 1035, 975, 1056, 1097, 994, 1059, 965, 1008, 1014, 1010, 1030, 1063, 990, 1044, 1033, 1000, 1034, 1013, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1051, 1034, 962, 1035, 1006, 1057, 1006, 1023, 1044, 1046, 1042, 1008, 1043, 1018, 1046, 1031, 1066, 1098, 997, 1028, 974, 995, 1016, 988, 1054, 1032, 982, 1010, 1010, 1006, 1026, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1030, 1032, 977, 1040, 1058, 1005, 1028, 1071, 1014, 994, 1056, 1051, 1017, 1019, 1046, 1019, 1005, 1038, 1000, 948, 1022, 1047, 976, 1052, 1023, 1034, 1029, 1016, 1055, 1023, 1006, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1085, 975, 1028, 991, 0, 0, 0, 0, 973, 1025, 990, 1018, 18, 0, 0, 0, 1012, 989, 1067, 1050, 4, 0, 0, 0, 1017, 1007, 1018, 1027, 16, 0, 0, 0, 1100, 1003, 1012, 1017, 0, 0, 0, 0, 1062, 1017, 1017, 1037, 6, 2, 0, 0, 1027, 1053, 1018, 1034, 0, 0, 0, 0, 958, 1016, 1077, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4024, 4122, 4085, 4132, 4141, 4132, 4053] avg=4096.0000 max=4141 max_rank=5 min=4024 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1022, 1082, 1050, 1021, 0, 0, 0, 0, 974, 1040, 977, 992, 18, 0, 0, 0, 1002, 1012, 980, 1031, 4, 0, 0, 0, 973, 1017, 1050, 1026, 17, 0, 0, 0, 988, 1070, 1029, 997, 0, 0, 0, 0, 1034, 1050, 1004, 1037, 5, 2, 0, 0, 1019, 1060, 1054, 1033, 0, 0, 0, 0, 1032, 1023, 1065, 978, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4001, 4029, 4083, 4084, 4132, 4166, 4098] avg=4096.0000 max=4175 max_rank=0 min=4001 min_rank=1 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1048, 1072, 999, 1031, 0, 0, 0, 0, 1002, 1054, 1018, 1061, 18, 0, 0, 0, 1006, 983, 1033, 1044, 4, 0, 0, 0, 1024, 1035, 1018, 1032, 17, 0, 0, 0, 1014, 982, 973, 985, 0, 0, 0, 0, 1017, 1009, 1053, 1056, 5, 2, 0, 0, 1004, 998, 1033, 1017, 0, 0, 0, 0, 998, 1086, 1035, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4153, 4070, 4126, 3954, 4142, 4052, 4121] avg=4096.0000 max=4153 max_rank=1 min=3954 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16269 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[973, 1049, 974, 999, 0, 0, 0, 0, 1048, 1016, 1011, 1023, 18, 0, 0, 0, 1015, 1036, 965, 1054, 4, 0, 0, 0, 1025, 1008, 1046, 977, 16, 0, 0, 0, 1048, 1001, 1050, 1040, 0, 0, 0, 0, 1056, 1003, 1068, 1001, 5, 2, 0, 0, 1032, 1053, 1024, 1040, 0, 0, 0, 0, 1047, 986, 1037, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3995, 4116, 4074, 4072, 4139, 4135, 4149, 4088] avg=4096.0000 max=4149 max_rank=6 min=3995 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1033, 1050, 1027, 1063, 0, 0, 0, 0, 1089, 1006, 1013, 957, 17, 0, 0, 0, 1055, 986, 993, 1027, 4, 0, 0, 0, 1049, 1014, 974, 974, 16, 0, 0, 0, 1016, 1079, 1048, 1018, 0, 0, 0, 0, 990, 1020, 1016, 1026, 5, 2, 0, 0, 1019, 1015, 997, 1042, 0, 0, 0, 0, 1056, 1026, 1059, 987, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4082, 4065, 4027, 4161, 4059, 4073, 4128] avg=4096.0000 max=4173 max_rank=0 min=4027 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1025, 1016, 1012, 1010, 0, 0, 0, 0, 1017, 1006, 1035, 1025, 17, 0, 0, 0, 1033, 1033, 1068, 996, 4, 0, 0, 0, 1018, 1035, 975, 1056, 16, 0, 0, 0, 1097, 994, 1059, 965, 0, 0, 0, 0, 1008, 1014, 1010, 1030, 5, 2, 0, 0, 1063, 990, 1009, 1033, 0, 0, 0, 0, 1000, 1034, 1013, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4100, 4134, 4100, 4115, 4069, 4095, 4092] avg=4096.0000 max=4134 max_rank=2 min=4063 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1032, 1034, 962, 1035, 0, 0, 0, 0, 1006, 1057, 1006, 1023, 21, 0, 0, 0, 1044, 1046, 1042, 1008, 9, 6, 0, 0, 1043, 1018, 1046, 1031, 20, 0, 0, 0, 1066, 1098, 997, 1028, 3, 0, 0, 0, 974, 995, 1016, 988, 10, 3, 0, 0, 1054, 1032, 938, 1010, 0, 0, 0, 0, 1010, 1006, 1017, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4113, 4155, 4158, 4192, 3986, 4034, 4067] avg=4096.0000 max=4192 max_rank=4 min=3986 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1021, 1032, 977, 1040, 0, 0, 0, 0, 1058, 1005, 1028, 1071, 17, 0, 0, 0, 1014, 994, 1056, 1051, 4, 0, 0, 0, 1017, 1019, 1046, 1019, 16, 0, 0, 0, 1005, 1038, 1000, 948, 0, 0, 0, 0, 1022, 1047, 976, 1052, 5, 2, 0, 0, 1023, 1034, 994, 1016, 0, 0, 0, 0, 1055, 1023, 1006, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4179, 4119, 4117, 3991, 4104, 4067, 4121] avg=4096.0000 max=4179 max_rank=1 min=3991 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8320, 8064, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8320, 8192, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1080, 1048, 973, 1034, 1056, 1019, 956, 1025, 1029, 1004, 992, 989, 1044, 1015, 1015, 1044, 1021, 1042, 1025, 1031, 1024, 1026, 1000, 1007, 1035, 1055, 1055, 1026, 1029, 1008, 1013, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1020, 1025, 1040, 1025, 1030, 1024, 972, 974, 1044, 1019, 1052, 1030, 1027, 992, 1027, 1022, 1069, 1056, 1052, 1032, 1087, 1044, 998, 1001, 979, 1052, 987, 1014, 1004, 1030, 1001, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1059, 1050, 1053, 1046, 1024, 1014, 1059, 995, 983, 1023, 1003, 956, 1006, 1003, 1018, 1035, 992, 1055, 1043, 1082, 1039, 994, 1044, 1057, 1043, 1002, 996, 998, 1024, 1001, 1011, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1047, 990, 1061, 1081, 1028, 1037, 1040, 987, 1008, 1070, 1014, 1042, 986, 1030, 987, 996, 1053, 1017, 1000, 986, 1046, 1041, 1079, 1023, 1023, 1028, 1008, 996, 990, 985, 1053, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1039, 1017, 1055, 998, 1005, 948, 1016, 1009, 1013, 1046, 1042, 1001, 999, 1026, 1024, 1049, 1046, 1090, 1039, 1002, 1063, 1001, 1051, 1045, 1010, 995, 1012, 1029, 1014, 1036, 1038, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1043, 1054, 961, 1010, 1065, 1016, 1040, 1034, 1022, 1001, 1021, 1018, 1016, 1002, 1029, 1018, 1053, 996, 1042, 1011, 1051, 1020, 1033, 960, 1023, 1023, 1020, 1078, 1003, 997, 1058, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[995, 1041, 964, 1022, 983, 1054, 1052, 1010, 1009, 1046, 1018, 1020, 1020, 1027, 1010, 1042, 1030, 1032, 1073, 973, 1037, 1023, 1048, 1008, 1026, 1073, 1013, 1069, 1067, 988, 1010, 985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1037, 1011, 1019, 1060, 1093, 1041, 1066, 1031, 1032, 996, 1040, 1017, 1031, 1009, 1030, 1033, 985, 1052, 1034, 1041, 998, 1007, 1003, 1034, 993, 1011, 1011, 1010, 1041, 974, 1038, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1057, 1048, 973, 1034, 0, 0, 0, 0, 1056, 1019, 956, 1025, 7, 0, 0, 0, 1029, 1004, 992, 989, 20, 0, 0, 0, 1044, 1015, 1015, 1044, 14, 4, 0, 0, 1021, 1008, 1025, 1031, 0, 0, 0, 0, 1010, 1026, 1000, 1007, 0, 0, 0, 0, 1035, 1055, 1055, 1026, 7, 2, 0, 0, 1029, 1008, 1013, 1048, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4063, 4034, 4136, 4085, 4043, 4180, 4115] avg=4096.0000 max=4180 max_rank=6 min=4034 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1017, 1017, 1055, 998, 0, 0, 0, 0, 1005, 948, 1016, 1009, 8, 0, 0, 0, 1013, 1046, 1042, 1001, 21, 0, 0, 0, 999, 1026, 1024, 1049, 15, 4, 0, 0, 1046, 1054, 1039, 1002, 0, 0, 0, 0, 1048, 1001, 1051, 1045, 0, 0, 0, 0, 1010, 995, 1012, 1029, 7, 1, 0, 0, 1014, 1036, 1038, 1010, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 3986, 4123, 4117, 4141, 4145, 4054, 4115] avg=4096.0000 max=4145 max_rank=5 min=3986 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 990, 1061, 1081, 0, 0, 0, 0, 1028, 1037, 1040, 987, 8, 0, 0, 0, 1008, 1070, 1014, 1042, 20, 0, 0, 0, 986, 1030, 987, 996, 14, 4, 0, 0, 1053, 983, 1000, 986, 0, 0, 0, 0, 1031, 1041, 1079, 1023, 0, 0, 0, 0, 1023, 1028, 1008, 996, 7, 2, 0, 0, 990, 985, 1053, 1036, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4100, 4154, 4017, 4022, 4174, 4064, 4081] avg=4096.0000 max=4174 max_rank=5 min=4017 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1036, 1050, 1053, 1046, 0, 0, 0, 0, 1024, 1014, 1059, 995, 8, 0, 0, 0, 983, 1023, 1003, 956, 21, 0, 0, 0, 1006, 1003, 1018, 1035, 15, 4, 0, 0, 992, 1019, 1043, 1082, 0, 0, 0, 0, 1024, 994, 1044, 1057, 0, 0, 0, 0, 1043, 1002, 996, 998, 7, 2, 0, 0, 1024, 1001, 1011, 1060, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 4100, 3986, 4081, 4136, 4119, 4048, 4113] avg=4096.0000 max=4185 max_rank=0 min=3986 min_rank=2 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1020, 1054, 961, 1010, 0, 0, 0, 0, 1065, 1016, 1040, 1034, 8, 0, 0, 0, 1022, 1001, 1021, 1018, 20, 0, 0, 0, 1016, 1002, 1029, 1018, 14, 4, 0, 0, 1053, 962, 1042, 1011, 0, 0, 0, 0, 1036, 1020, 1033, 960, 0, 0, 0, 0, 1023, 1023, 1020, 1078, 7, 2, 0, 0, 1003, 997, 1058, 1050, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4163, 4082, 4083, 4068, 4049, 4153, 4125] avg=4096.0000 max=4163 max_rank=1 min=4045 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[974, 1041, 964, 1022, 0, 0, 0, 0, 983, 1054, 1052, 1010, 8, 0, 0, 0, 1009, 1046, 1018, 1020, 20, 0, 0, 0, 1020, 1027, 1010, 1042, 14, 4, 0, 0, 1030, 998, 1073, 973, 0, 0, 0, 0, 1022, 1023, 1048, 1008, 0, 0, 0, 0, 1026, 1073, 1013, 1069, 7, 1, 0, 0, 1067, 988, 1010, 985, 16, 0, 0, 0] rerouted_tokens_per_ep_rank=[4001, 4107, 4113, 4117, 4074, 4101, 4189, 4066] avg=4096.0000 max=4189 max_rank=6 min=4001 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[987, 1025, 1040, 1025, 0, 0, 0, 0, 1030, 1024, 972, 974, 11, 0, 0, 0, 1044, 1019, 1052, 1030, 25, 0, 0, 0, 1027, 992, 1027, 1022, 18, 9, 0, 0, 1069, 1013, 1052, 1032, 0, 0, 0, 0, 1066, 1044, 998, 1001, 0, 0, 0, 0, 979, 1052, 987, 1014, 10, 5, 0, 0, 1004, 1030, 1001, 1039, 19, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4011, 4170, 4095, 4166, 4109, 4047, 4093] avg=4096.0000 max=4170 max_rank=2 min=4011 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1015, 1011, 1019, 1060, 0, 0, 0, 0, 1093, 1041, 1066, 1031, 7, 0, 0, 0, 1032, 996, 1040, 1017, 21, 0, 0, 0, 1031, 1009, 1030, 1033, 15, 4, 0, 0, 985, 1016, 1034, 1041, 0, 0, 0, 0, 984, 1007, 1003, 1034, 0, 0, 0, 0, 993, 1011, 1011, 1010, 7, 1, 0, 0, 1041, 974, 1038, 990, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4238, 4106, 4122, 4076, 4028, 4033, 4060] avg=4096.0000 max=4238 max_rank=1 min=4028 min_rank=5 imbalance_ratio=1.034668 max_load_reduction=28530 max_load_reduction_pct=0.870667 moved_to_echo=16197 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8064, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1006, 995, 1055, 994, 1095, 1030, 1057, 1014, 1028, 1046, 996, 1015, 1002, 997, 973, 1065, 1028, 1022, 981, 1042, 1004, 987, 1014, 1008, 981, 1001, 1017, 1048, 1027, 1068, 1071, 1101] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1041, 1041, 1058, 995, 1028, 997, 1038, 1003, 1009, 1062, 1036, 995, 976, 1060, 970, 1069, 1010, 1022, 998, 1040, 1012, 1002, 977, 1075, 1030, 1064, 1004, 1034, 1040, 1011, 1044, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1075, 1014, 991, 1071, 1007, 1053, 1023, 1026, 1034, 1031, 1029, 1033, 970, 1032, 991, 1050, 968, 1010, 988, 1012, 1068, 1061, 1016, 1027, 1028, 996, 1067, 1031, 1019, 1040, 1025, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1008, 1041, 1031, 1009, 1023, 1082, 1042, 1026, 1003, 1017, 997, 956, 990, 1013, 1003, 983, 1066, 1077, 1027, 1023, 1026, 1036, 1036, 1039, 1036, 1041, 1037, 1048, 986, 1047, 1011, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1016, 1027, 1014, 1065, 980, 1077, 1028, 1007, 1053, 1006, 1025, 1041, 947, 1010, 1004, 1025, 1056, 1051, 993, 964, 1053, 1055, 995, 1033, 1018, 1005, 1019, 1090, 1070, 1019, 993, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1056, 991, 1029, 1042, 1066, 990, 1056, 1023, 1000, 1025, 1043, 1005, 1079, 1007, 989, 982, 1052, 1015, 1039, 987, 1004, 1006, 988, 1056, 1073, 983, 1001, 1047, 1050, 1074, 1000, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1071, 1019, 982, 1029, 1043, 1026, 999, 1033, 1059, 1028, 1038, 979, 1012, 1090, 1033, 1025, 1048, 989, 1031, 1014, 992, 946, 1016, 1049, 1012, 1022, 986, 1070, 982, 1024, 1043, 1078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1045, 1044, 1071, 1040, 1032, 949, 999, 1021, 997, 1006, 1032, 1036, 1016, 1032, 1026, 1021, 1026, 1039, 969, 1013, 1066, 967, 1058, 1007, 1022, 1057, 1068, 1052, 1004, 1002, 1051, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[983, 995, 1055, 994, 0, 0, 0, 0, 1082, 1030, 1057, 1014, 0, 0, 0, 0, 1028, 1046, 996, 1015, 11, 2, 0, 0, 1002, 997, 973, 1065, 27, 12, 0, 0, 1028, 1022, 981, 1042, 11, 9, 0, 0, 1004, 987, 1014, 1008, 11, 0, 0, 0, 981, 1001, 1017, 1021, 0, 0, 0, 0, 1027, 1048, 1071, 1101, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4183, 4098, 4076, 4093, 4024, 4020, 4247] avg=4096.0000 max=4247 max_rank=7 min=4020 min_rank=6 imbalance_ratio=1.036865 max_load_reduction=28521 max_load_reduction_pct=0.870392 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1017, 1041, 1058, 995, 0, 0, 0, 0, 1016, 997, 1038, 1003, 0, 0, 0, 0, 1009, 1062, 1036, 995, 11, 1, 0, 0, 976, 1060, 970, 1069, 27, 13, 0, 0, 1010, 1022, 998, 1040, 11, 9, 0, 0, 1012, 1002, 977, 1075, 11, 0, 0, 0, 1030, 1064, 1004, 1007, 0, 0, 0, 0, 1040, 991, 1044, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4054, 4114, 4115, 4090, 4077, 4105, 4102] avg=4096.0000 max=4115 max_rank=3 min=4054 min_rank=1 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[985, 1041, 1031, 1009, 0, 0, 0, 0, 1011, 1082, 1042, 1026, 0, 0, 0, 0, 1003, 1017, 997, 956, 11, 1, 0, 0, 990, 1013, 1003, 983, 27, 12, 0, 0, 1066, 1077, 1027, 1023, 11, 9, 0, 0, 1026, 1036, 1036, 1039, 11, 0, 0, 0, 1036, 1041, 1037, 1021, 0, 0, 0, 0, 986, 1027, 1011, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4161, 3985, 4028, 4213, 4148, 4135, 4032] avg=4096.0000 max=4213 max_rank=4 min=3985 min_rank=2 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16528 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1051, 1014, 991, 1071, 0, 0, 0, 0, 996, 1053, 1023, 1026, 0, 0, 0, 0, 1034, 1031, 1029, 1033, 11, 1, 0, 0, 970, 1032, 991, 1050, 26, 13, 0, 0, 968, 1010, 988, 1012, 11, 9, 0, 0, 1068, 1061, 1016, 1027, 10, 0, 0, 0, 1028, 996, 1067, 1005, 0, 0, 0, 0, 1019, 1020, 1025, 982, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4098, 4139, 4082, 3998, 4182, 4096, 4046] avg=4096.0000 max=4182 max_rank=5 min=3998 min_rank=4 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1032, 991, 1029, 1042, 0, 0, 0, 0, 1053, 990, 1056, 1023, 0, 0, 0, 0, 1000, 1025, 1043, 1005, 11, 2, 0, 0, 1079, 1007, 989, 982, 27, 13, 0, 0, 1052, 1015, 1039, 987, 11, 9, 0, 0, 1004, 1006, 988, 1056, 11, 0, 0, 0, 1073, 983, 1001, 1020, 0, 0, 0, 0, 1050, 1054, 1000, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4122, 4086, 4097, 4113, 4065, 4077, 4114] avg=4096.0000 max=4122 max_rank=1 min=4065 min_rank=5 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[984, 1027, 1014, 1065, 0, 0, 0, 0, 961, 1077, 1028, 1007, 0, 0, 0, 0, 1053, 1006, 1025, 1041, 15, 6, 0, 0, 947, 1010, 1004, 1025, 32, 17, 0, 0, 1056, 1051, 993, 964, 15, 13, 0, 0, 1053, 1055, 995, 1033, 13, 0, 0, 0, 1018, 1005, 1019, 1058, 0, 0, 0, 0, 1070, 991, 993, 1029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4073, 4146, 4035, 4092, 4149, 4100, 4083] avg=4096.0000 max=4149 max_rank=5 min=4035 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1021, 1044, 1071, 1040, 0, 0, 0, 0, 1020, 949, 999, 1021, 0, 0, 0, 0, 997, 1006, 1032, 1036, 11, 1, 0, 0, 1016, 1032, 1026, 1021, 27, 13, 0, 0, 1026, 1039, 969, 1013, 11, 9, 0, 0, 1066, 967, 1058, 1007, 11, 0, 0, 0, 1022, 1057, 1068, 1025, 0, 0, 0, 0, 1004, 982, 1051, 1000, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 3989, 4083, 4135, 4067, 4109, 4172, 4037] avg=4096.0000 max=4176 max_rank=0 min=3989 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1047, 1019, 982, 1029, 0, 0, 0, 0, 1030, 1026, 999, 1033, 0, 0, 0, 0, 1059, 1028, 1038, 979, 11, 2, 0, 0, 1012, 1090, 1033, 1025, 27, 13, 0, 0, 1048, 989, 1031, 1014, 11, 9, 0, 0, 992, 946, 1016, 1049, 11, 0, 0, 0, 1012, 1022, 986, 1043, 0, 0, 0, 0, 982, 1004, 1043, 1078, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4088, 4117, 4200, 4102, 4014, 4063, 4107] avg=4096.0000 max=4200 max_rank=3 min=4014 min_rank=5 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8064, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1021, 1037, 1037, 1037, 984, 1039, 1017, 1036, 1025, 1018, 1002, 1038, 1021, 1034, 1010, 1034, 1088, 1047, 1007, 1052, 1010, 973, 1043, 1012, 1007, 1022, 1020, 1007, 1034, 1031, 998, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1019, 1029, 993, 1014, 1013, 1073, 1003, 1004, 1045, 1033, 1014, 1025, 1011, 1050, 1006, 1008, 1037, 1064, 989, 1009, 966, 1077, 1089, 1043, 1018, 993, 1074, 975, 1083, 989, 1038, 984] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[996, 1057, 973, 1043, 967, 1074, 951, 963, 1046, 1040, 966, 1029, 1033, 986, 1073, 1000, 1043, 1038, 1062, 1046, 998, 1031, 1100, 1026, 1060, 983, 1050, 1040, 1033, 993, 1045, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1034, 1002, 1051, 1029, 1041, 1018, 1023, 1022, 981, 990, 979, 1014, 1012, 1055, 982, 1058, 1067, 1056, 1019, 1000, 1045, 1012, 1052, 1021, 1068, 1002, 1011, 1026, 999, 1021, 1018, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1057, 1030, 1038, 977, 1028, 1000, 1051, 1087, 983, 1002, 980, 1045, 1026, 1010, 1011, 1028, 1029, 1013, 989, 1030, 1035, 1006, 1054, 1020, 1025, 1013, 1026, 1071, 1048, 1013, 989, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1063, 1006, 993, 1015, 1059, 1048, 1003, 1038, 993, 1067, 1037, 1040, 1006, 1056, 1003, 995, 1087, 1028, 1029, 1012, 1028, 1077, 1039, 1049, 975, 1019, 993, 984, 1009, 981, 1021, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1016, 1012, 997, 1021, 1024, 1015, 1050, 1029, 1028, 994, 1040, 1033, 1054, 1050, 1016, 1055, 1031, 1010, 1035, 1006, 1050, 1036, 1004, 1069, 1015, 1011, 1000, 980, 1063, 956, 1027, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1031, 982, 971, 999, 1037, 1019, 1082, 1005, 1060, 1014, 1013, 994, 1038, 998, 993, 1028, 1021, 1046, 1061, 1026, 1052, 1025, 990, 1038, 1059, 1017, 1053, 1005, 1065, 1027, 1008, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1019, 1029, 993, 1014, 13, 10, 0, 0, 1013, 1070, 1003, 1004, 0, 0, 0, 0, 1045, 1033, 1014, 1025, 24, 0, 0, 0, 1011, 1050, 1006, 1008, 3, 0, 0, 0, 1000, 1064, 989, 1009, 0, 0, 0, 0, 966, 1077, 1051, 1043, 0, 0, 0, 0, 1018, 993, 1074, 975, 21, 0, 0, 0, 1083, 989, 1038, 984, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4090, 4141, 4078, 4062, 4137, 4081, 4101] avg=4096.0000 max=4141 max_rank=2 min=4062 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1034, 1002, 1051, 1029, 13, 9, 0, 0, 1041, 1015, 1023, 1022, 0, 0, 0, 0, 981, 990, 979, 1014, 25, 0, 0, 0, 1012, 1055, 982, 1058, 3, 0, 0, 0, 1029, 1056, 1019, 1000, 0, 0, 0, 0, 1045, 1012, 1016, 1021, 0, 0, 0, 0, 1068, 1002, 1011, 1026, 20, 0, 0, 0, 999, 1021, 1018, 1060, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4101, 3989, 4110, 4104, 4094, 4127, 4105] avg=4096.0000 max=4138 max_rank=0 min=3989 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[996, 1057, 973, 1043, 13, 10, 0, 0, 967, 1071, 951, 963, 0, 0, 0, 0, 1046, 1040, 966, 1029, 24, 0, 0, 0, 1033, 986, 1073, 1000, 3, 0, 0, 0, 1006, 1038, 1062, 1046, 0, 0, 0, 0, 998, 1031, 1062, 1026, 0, 0, 0, 0, 1060, 983, 1050, 1040, 21, 0, 0, 0, 1033, 993, 1045, 1023, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 3952, 4105, 4095, 4152, 4117, 4154, 4101] avg=4096.0000 max=4154 max_rank=6 min=3952 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16524 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1021, 1037, 1037, 1037, 14, 9, 0, 0, 984, 1036, 1017, 1036, 0, 0, 0, 0, 1025, 1018, 1002, 1038, 25, 0, 0, 0, 1021, 1034, 1010, 1034, 3, 0, 0, 0, 1049, 1047, 1007, 1052, 0, 0, 0, 0, 1010, 973, 1007, 1012, 0, 0, 0, 0, 1007, 1022, 1020, 1007, 20, 0, 0, 0, 1034, 1031, 998, 1027, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4073, 4108, 4102, 4155, 4002, 4076, 4097] avg=4096.0000 max=4155 max_rank=0 min=4002 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1057, 1030, 1038, 977, 13, 9, 0, 0, 1028, 997, 1051, 1087, 0, 0, 0, 0, 983, 1002, 980, 1045, 24, 0, 0, 0, 1026, 1010, 1011, 1028, 3, 0, 0, 0, 992, 1013, 989, 1030, 0, 0, 0, 0, 1035, 1006, 1018, 1020, 0, 0, 0, 0, 1025, 1013, 1026, 1071, 20, 0, 0, 0, 1048, 1013, 989, 1054, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4163, 4034, 4078, 4024, 4079, 4155, 4111] avg=4096.0000 max=4163 max_rank=1 min=4024 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1016, 1012, 997, 1021, 13, 9, 0, 0, 1024, 1012, 1050, 1029, 0, 0, 0, 0, 1028, 994, 1040, 1033, 24, 0, 0, 0, 1054, 1050, 1016, 1055, 3, 0, 0, 0, 994, 1010, 1035, 1006, 0, 0, 0, 0, 1050, 1036, 970, 1069, 0, 0, 0, 0, 1015, 1011, 1000, 980, 19, 0, 0, 0, 1063, 956, 1027, 1041, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4115, 4119, 4178, 4045, 4125, 4025, 4093] avg=4096.0000 max=4178 max_rank=3 min=4025 min_rank=6 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16288 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1063, 1006, 993, 1015, 17, 14, 0, 0, 1059, 1034, 1003, 1038, 0, 0, 0, 0, 993, 1067, 1037, 1040, 30, 0, 0, 0, 1006, 1056, 1003, 995, 7, 0, 0, 0, 1040, 1028, 1029, 1012, 0, 0, 0, 0, 1028, 1077, 989, 1049, 0, 0, 0, 0, 975, 1019, 993, 984, 26, 0, 0, 0, 1009, 981, 1021, 1015, 10, 7, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4134, 4167, 4067, 4109, 4143, 3997, 4043] avg=4096.0000 max=4167 max_rank=2 min=3997 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1031, 982, 971, 999, 13, 9, 0, 0, 1037, 1016, 1082, 1005, 0, 0, 0, 0, 1060, 1014, 1013, 994, 24, 0, 0, 0, 1038, 998, 993, 1028, 3, 0, 0, 0, 984, 1046, 1061, 1026, 0, 0, 0, 0, 1052, 1025, 956, 1038, 0, 0, 0, 0, 1059, 1017, 1053, 1005, 19, 0, 0, 0, 1065, 1027, 1008, 1011, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 4140, 4105, 4060, 4117, 4071, 4153, 4117] avg=4096.0000 max=4153 max_rank=6 min=4005 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8064, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8448, 8064, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8064, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1015, 1049, 1040, 1012, 997, 998, 1032, 991, 1025, 1004, 1012, 1002, 992, 1062, 1044, 1044, 1006, 1047, 1020, 986, 1022, 1036, 986, 1052, 1066, 1047, 1019, 1030, 1038, 1031, 1038, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1025, 1032, 1006, 979, 1019, 1002, 1030, 1011, 1029, 976, 1055, 1052, 1051, 1078, 1046, 1033, 1049, 1008, 1029, 995, 1045, 1055, 1023, 1021, 1036, 992, 1002, 994, 1008, 1016, 1044, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1044, 1023, 990, 1030, 1035, 988, 1027, 1051, 1019, 1027, 1026, 997, 1025, 1019, 1034, 1014, 1010, 1093, 1023, 1040, 1011, 1003, 1022, 1001, 1061, 995, 1006, 1011, 1024, 1054, 1022, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1060, 1024, 1032, 1038, 1068, 961, 991, 1026, 1002, 1024, 1004, 1015, 1041, 1036, 990, 1039, 1041, 978, 1020, 999, 1042, 1075, 1013, 1041, 995, 1030, 1010, 1019, 1056, 1040, 1069, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1001, 1039, 998, 1022, 1027, 992, 1003, 1014, 1018, 1036, 1022, 1071, 1037, 1057, 1044, 1034, 1003, 1032, 1037, 1030, 1060, 1018, 938, 1055, 997, 1051, 1015, 1066, 981, 1014, 1014, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1028, 1012, 1056, 966, 992, 1021, 1035, 1059, 1024, 1040, 1018, 1038, 1019, 978, 1039, 1034, 1006, 1033, 1030, 1032, 1046, 1041, 1010, 985, 1015, 998, 1011, 1055, 1035, 1015, 1060, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1075, 999, 1029, 1038, 971, 1093, 1003, 1021, 1007, 1038, 1032, 1009, 1017, 1021, 1066, 999, 1051, 981, 1058, 1065, 985, 994, 1099, 983, 1006, 988, 1036, 1055, 1040, 977, 984, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1029, 1028, 1052, 1055, 1024, 965, 1032, 1011, 1021, 1029, 1047, 1002, 1019, 1009, 1019, 1014, 1031, 987, 1070, 1010, 1043, 999, 1038, 1032, 1017, 999, 1011, 1032, 996, 1056, 1045, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1018, 1032, 1006, 979, 0, 0, 0, 0, 1019, 1002, 1030, 1011, 23, 11, 0, 0, 1029, 976, 1055, 1052, 3, 1, 0, 0, 1051, 1078, 1023, 1033, 0, 0, 0, 0, 1049, 1008, 1026, 995, 0, 0, 0, 0, 1045, 1055, 1023, 1021, 0, 0, 0, 0, 1036, 992, 1002, 994, 6, 6, 0, 0, 1008, 1016, 1027, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4096, 4116, 4185, 4078, 4144, 4036, 4078] avg=4096.0000 max=4185 max_rank=3 min=4035 min_rank=0 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1053, 1024, 1032, 1038, 0, 0, 0, 0, 1068, 961, 991, 1026, 22, 11, 0, 0, 1002, 1024, 1004, 1015, 3, 1, 0, 0, 1041, 1036, 968, 1039, 0, 0, 0, 0, 1041, 978, 1017, 999, 0, 0, 0, 0, 1042, 1075, 1013, 1041, 0, 0, 0, 0, 995, 1030, 1010, 1019, 6, 6, 0, 0, 1056, 1040, 1052, 989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4079, 4049, 4084, 4035, 4171, 4066, 4137] avg=4096.0000 max=4171 max_rank=5 min=4035 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1008, 1049, 1040, 1012, 0, 0, 0, 0, 997, 998, 1032, 991, 23, 11, 0, 0, 1025, 1004, 1012, 1002, 3, 1, 0, 0, 992, 1062, 1021, 1044, 0, 0, 0, 0, 1006, 1047, 1017, 986, 0, 0, 0, 0, 1022, 1036, 986, 1052, 0, 0, 0, 0, 1066, 1047, 1019, 1030, 6, 6, 0, 0, 1038, 1031, 1021, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4052, 4047, 4119, 4056, 4096, 4174, 4115] avg=4096.0000 max=4174 max_rank=6 min=4047 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1037, 1023, 990, 1030, 0, 0, 0, 0, 1035, 988, 1027, 1051, 23, 11, 0, 0, 1019, 1027, 1026, 997, 3, 1, 0, 0, 1025, 1019, 1011, 1014, 0, 0, 0, 0, 1010, 1093, 1020, 1040, 0, 0, 0, 0, 1011, 1003, 1022, 1001, 0, 0, 0, 0, 1061, 995, 1006, 1011, 6, 6, 0, 0, 1024, 1054, 1005, 1043, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4135, 4073, 4069, 4163, 4037, 4085, 4126] avg=4096.0000 max=4163 max_rank=4 min=4037 min_rank=5 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[992, 1039, 998, 1022, 0, 0, 0, 0, 1027, 992, 1003, 1014, 27, 16, 0, 0, 1018, 1036, 1022, 1071, 9, 2, 6, 0, 1037, 1057, 1017, 1034, 0, 0, 0, 0, 1003, 1032, 1028, 1030, 0, 0, 0, 0, 1054, 1018, 938, 1055, 0, 0, 0, 0, 997, 1051, 1015, 1066, 12, 7, 0, 0, 981, 1014, 986, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4079, 4164, 4145, 4093, 4065, 4148, 4023] avg=4096.0000 max=4164 max_rank=2 min=4023 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1021, 1012, 1056, 966, 0, 0, 0, 0, 992, 1021, 1035, 1059, 23, 11, 0, 0, 1024, 1040, 1018, 1038, 3, 1, 0, 0, 1019, 978, 1016, 1034, 0, 0, 0, 0, 1006, 1033, 1027, 1032, 0, 0, 0, 0, 1046, 1041, 1010, 985, 0, 0, 0, 0, 1015, 998, 1011, 1055, 6, 6, 0, 0, 1035, 1015, 1043, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4141, 4124, 4047, 4098, 4082, 4091, 4130] avg=4096.0000 max=4141 max_rank=1 min=4047 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1068, 999, 1029, 1038, 0, 0, 0, 0, 971, 1093, 1003, 1021, 23, 10, 0, 0, 1007, 1038, 1032, 1009, 4, 1, 0, 0, 1017, 1021, 1043, 999, 0, 0, 0, 0, 1051, 981, 1054, 1065, 0, 0, 0, 0, 985, 994, 1099, 983, 0, 0, 0, 0, 1006, 988, 1036, 1055, 6, 6, 0, 0, 1040, 977, 968, 1048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4121, 4091, 4080, 4151, 4061, 4097, 4033] avg=4096.0000 max=4151 max_rank=4 min=4033 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1022, 1028, 1052, 1055, 0, 0, 0, 0, 1024, 965, 1032, 1011, 22, 11, 0, 0, 1021, 1029, 1047, 1002, 4, 1, 0, 0, 1019, 1009, 997, 1014, 0, 0, 0, 0, 1031, 987, 1066, 1010, 0, 0, 0, 0, 1043, 999, 1038, 1032, 0, 0, 0, 0, 1017, 999, 1011, 1032, 6, 6, 0, 0, 996, 1056, 1028, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4065, 4104, 4039, 4094, 4112, 4071, 4126] avg=4096.0000 max=4157 max_rank=0 min=4039 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1029, 1026, 985, 1012, 1071, 1008, 1003, 1051, 1031, 989, 997, 1060, 1008, 1027, 1027, 1028, 1008, 998, 1001, 1043, 1001, 1037, 1031, 1048, 1004, 1026, 1059, 1023, 1014, 1006, 1072, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1012, 1040, 1078, 1021, 995, 1062, 1038, 1074, 1038, 1011, 1031, 1002, 975, 1038, 993, 995, 1024, 977, 1050, 1020, 1003, 1032, 1027, 1006, 1015, 1016, 1041, 1017, 1011, 1065, 1032, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1016, 1011, 1029, 1007, 1039, 960, 1005, 1076, 1038, 1078, 1090, 983, 1008, 995, 1027, 1012, 1020, 990, 1072, 1011, 1041, 993, 987, 1026, 1017, 1049, 1043, 1025, 1016, 1022, 1080, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[992, 1033, 1009, 980, 1036, 1022, 1081, 1014, 1012, 1012, 1038, 1061, 1036, 962, 1020, 1050, 1040, 1009, 1036, 1082, 986, 1057, 1073, 979, 1005, 979, 1006, 1067, 994, 1009, 1042, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1008, 1020, 988, 1011, 1041, 994, 1044, 1001, 1017, 1080, 1092, 979, 1037, 1017, 978, 1050, 1028, 1085, 990, 1026, 1068, 1065, 992, 1047, 995, 1003, 1041, 993, 973, 1073, 967, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1008, 1067, 989, 1032, 1033, 1047, 1015, 1044, 1031, 1049, 1040, 998, 994, 1004, 998, 1000, 1041, 978, 1065, 992, 1043, 1007, 983, 1067, 1041, 1097, 1006, 1017, 1073, 1012, 997, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1060, 1029, 1001, 994, 1012, 1008, 1020, 1106, 1025, 1064, 1022, 1027, 1046, 1010, 971, 1040, 1036, 1022, 1019, 1075, 1047, 1038, 1027, 1060, 1003, 1038, 1010, 1013, 976, 960, 1023, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1068, 988, 1037, 1008, 1017, 1033, 1059, 1033, 1065, 1030, 1085, 977, 996, 1067, 1066, 995, 1084, 975, 1010, 1041, 989, 1033, 1031, 1002, 1019, 1000, 1005, 1024, 1026, 1013, 1013, 979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1012, 1040, 1078, 1021, 23, 0, 0, 0, 995, 1062, 1038, 1040, 0, 0, 0, 0, 1038, 1011, 997, 1002, 0, 0, 0, 0, 975, 1038, 993, 995, 34, 1, 0, 0, 1024, 977, 1050, 1011, 0, 0, 0, 0, 1003, 1025, 1027, 1006, 0, 0, 0, 0, 1015, 1016, 1041, 1017, 7, 1, 0, 0, 1011, 1065, 1032, 1029, 10, 8, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4135, 4048, 4036, 4062, 4061, 4097, 4155] avg=4096.0000 max=4174 max_rank=0 min=4036 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1016, 1011, 1029, 1007, 23, 0, 0, 0, 1039, 960, 1005, 1042, 0, 0, 0, 0, 1038, 1078, 1054, 983, 0, 0, 0, 0, 1008, 995, 1027, 1012, 36, 1, 0, 0, 1020, 990, 1072, 1002, 0, 0, 0, 0, 1041, 987, 987, 1026, 0, 0, 0, 0, 1017, 1049, 1043, 1025, 6, 1, 0, 0, 1016, 1022, 1080, 1002, 10, 8, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4046, 4153, 4079, 4084, 4041, 4141, 4138] avg=4096.0000 max=4153 max_rank=2 min=4041 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1029, 1026, 985, 1012, 22, 0, 0, 0, 1071, 1008, 1003, 1018, 0, 0, 0, 0, 1031, 989, 964, 1060, 0, 0, 0, 0, 1008, 1027, 1027, 1028, 33, 1, 0, 0, 1008, 998, 1001, 1034, 0, 0, 0, 0, 1001, 1030, 1031, 1048, 0, 0, 0, 0, 1004, 1026, 1059, 1023, 7, 1, 0, 0, 1014, 1006, 1072, 1045, 10, 8, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4100, 4044, 4124, 4041, 4110, 4120, 4155] avg=4096.0000 max=4155 max_rank=7 min=4041 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[992, 1033, 1009, 980, 21, 0, 0, 0, 1036, 1022, 1081, 983, 0, 0, 0, 0, 1012, 1012, 1003, 1061, 0, 0, 0, 0, 1036, 962, 1020, 1050, 35, 1, 0, 0, 1040, 1009, 1036, 1073, 0, 0, 0, 0, 986, 1050, 1073, 979, 0, 0, 0, 0, 1005, 979, 1006, 1067, 7, 1, 0, 0, 994, 1009, 1042, 1046, 9, 8, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4122, 4088, 4104, 4158, 4088, 4065, 4108] avg=4096.0000 max=4158 max_rank=4 min=4035 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1008, 1067, 989, 1032, 22, 0, 0, 0, 1033, 1047, 1015, 1012, 0, 0, 0, 0, 1031, 1049, 1005, 998, 0, 0, 0, 0, 994, 1004, 998, 1000, 35, 1, 0, 0, 1041, 978, 1065, 983, 0, 0, 0, 0, 1043, 1000, 983, 1067, 0, 0, 0, 0, 1041, 1097, 1006, 1017, 7, 1, 0, 0, 1073, 1012, 997, 1000, 9, 8, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4107, 4083, 4032, 4067, 4093, 4169, 4099] avg=4096.0000 max=4169 max_rank=6 min=4032 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1008, 1020, 988, 1011, 24, 0, 0, 0, 1041, 994, 1044, 957, 0, 0, 0, 0, 1017, 1080, 1051, 979, 0, 0, 0, 0, 1037, 1017, 978, 1050, 41, 7, 0, 0, 1028, 1085, 990, 1009, 0, 0, 0, 0, 1068, 1055, 992, 1047, 0, 0, 0, 0, 995, 1003, 1041, 993, 10, 6, 0, 0, 973, 1073, 967, 1065, 13, 11, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4036, 4127, 4130, 4112, 4162, 4048, 4102] avg=4096.0000 max=4162 max_rank=5 min=4036 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1060, 1029, 1001, 994, 23, 0, 0, 0, 1012, 1008, 1020, 1072, 0, 0, 0, 0, 1025, 1064, 988, 1027, 0, 0, 0, 0, 1046, 1010, 971, 1040, 34, 1, 0, 0, 1036, 1022, 1019, 1066, 0, 0, 0, 0, 1047, 1031, 1027, 1060, 0, 0, 0, 0, 1003, 1038, 1010, 1013, 7, 1, 0, 0, 976, 960, 1023, 986, 10, 8, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4112, 4104, 4102, 4143, 4165, 4072, 3963] avg=4096.0000 max=4165 max_rank=5 min=3963 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1068, 988, 1037, 1008, 22, 0, 0, 0, 1017, 1033, 1059, 1001, 0, 0, 0, 0, 1065, 1030, 1049, 977, 0, 0, 0, 0, 996, 1067, 1066, 995, 36, 1, 0, 0, 1084, 975, 1010, 1032, 0, 0, 0, 0, 989, 1026, 1031, 1002, 0, 0, 0, 0, 1019, 1000, 1005, 1024, 7, 1, 0, 0, 1026, 1013, 1013, 979, 9, 8, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4110, 4121, 4161, 4101, 4048, 4056, 4048] avg=4096.0000 max=4161 max_rank=3 min=4048 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16253 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1045, 1042, 1035, 988, 953, 1012, 1054, 1005, 1035, 982, 1034, 1036, 987, 1069, 1042, 1018, 1002, 1061, 1008, 989, 985, 1058, 1033, 1050, 1061, 1014, 1049, 1055, 1028, 1029, 1000, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1045, 1029, 1040, 1002, 1000, 1009, 1097, 1049, 1011, 1053, 1071, 1027, 973, 1066, 1026, 1003, 1028, 1018, 957, 973, 1006, 1041, 1006, 1040, 1030, 1033, 1008, 992, 1021, 1045, 1039, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1055, 1030, 1048, 998, 985, 1026, 1038, 1044, 1029, 1044, 1029, 1003, 1070, 998, 959, 1022, 1032, 1001, 992, 1052, 1065, 1027, 988, 1021, 1015, 1051, 1048, 1023, 1021, 1037, 1015, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[987, 1028, 1055, 1055, 1029, 995, 1030, 1072, 1050, 1058, 1045, 1023, 972, 1040, 1030, 1029, 1002, 1027, 1066, 1035, 1000, 1048, 991, 1004, 996, 1021, 976, 1033, 1055, 1016, 992, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1065, 1077, 1015, 973, 971, 1011, 1040, 1013, 986, 1031, 1052, 1049, 996, 1052, 1049, 1032, 1035, 997, 1026, 1036, 1013, 1010, 1050, 1004, 998, 1060, 1018, 1040, 992, 994, 1071, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1077, 992, 1032, 1030, 1034, 1030, 1033, 1101, 1012, 1013, 994, 978, 1024, 1004, 1034, 994, 1061, 1046, 1022, 1017, 1005, 1074, 1045, 986, 1039, 976, 1006, 1013, 1053, 1013, 1030, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1008, 999, 1036, 1018, 977, 1030, 997, 1043, 1040, 1010, 1021, 1045, 1055, 1028, 1024, 1002, 989, 1042, 988, 1034, 1031, 1027, 1030, 1049, 967, 1036, 1052, 1060, 1012, 1064, 992, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1032, 1013, 1028, 1013, 984, 1005, 1102, 1027, 1031, 989, 1070, 1021, 1033, 1055, 1025, 1010, 1075, 1037, 1008, 1002, 1012, 1069, 996, 974, 1049, 978, 992, 1031, 1047, 1013, 1033, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1031, 1042, 1035, 988, 0, 0, 0, 0, 953, 1012, 1051, 1005, 0, 0, 0, 0, 1035, 982, 1023, 1036, 0, 0, 0, 0, 987, 1069, 1042, 1018, 5, 0, 0, 0, 1002, 1061, 1008, 989, 13, 0, 0, 0, 985, 1058, 1033, 1050, 2, 1, 0, 0, 1061, 1014, 1049, 1055, 4, 1, 0, 0, 1028, 1029, 1000, 1009, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4021, 4076, 4121, 4073, 4129, 4184, 4068] avg=4096.0000 max=4184 max_rank=6 min=4021 min_rank=1 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1031, 1029, 1040, 1002, 0, 0, 0, 0, 1000, 1009, 1094, 1049, 0, 0, 0, 0, 1011, 1053, 1059, 1027, 0, 0, 0, 0, 973, 1066, 1026, 1003, 6, 0, 0, 0, 1028, 1018, 957, 973, 13, 0, 0, 0, 1006, 1041, 1006, 1040, 2, 1, 0, 0, 1030, 1033, 1008, 992, 4, 1, 0, 0, 1021, 1045, 1039, 1030, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4152, 4150, 4074, 3989, 4096, 4068, 4137] avg=4096.0000 max=4152 max_rank=1 min=3989 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16290 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1041, 1030, 1048, 998, 0, 0, 0, 0, 985, 1026, 1035, 1044, 0, 0, 0, 0, 1029, 1044, 1018, 1003, 0, 0, 0, 0, 1070, 998, 959, 1022, 5, 0, 0, 0, 1032, 1001, 992, 1052, 13, 0, 0, 0, 1065, 1027, 988, 1021, 2, 1, 0, 0, 1015, 1051, 1048, 1023, 4, 1, 0, 0, 1021, 1037, 1015, 1002, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4090, 4094, 4054, 4090, 4104, 4142, 4077] avg=4096.0000 max=4142 max_rank=6 min=4054 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1050, 1077, 1015, 973, 0, 0, 0, 0, 971, 1011, 1037, 1013, 0, 0, 0, 0, 986, 1031, 1041, 1049, 0, 0, 0, 0, 996, 1052, 1049, 1032, 5, 0, 0, 0, 1035, 997, 1026, 1036, 14, 0, 0, 0, 1013, 1010, 1050, 1004, 2, 1, 0, 0, 998, 1060, 1018, 1040, 4, 1, 0, 0, 992, 994, 1071, 1012, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4032, 4107, 4134, 4108, 4080, 4121, 4071] avg=4096.0000 max=4134 max_rank=3 min=4032 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[973, 1028, 1055, 1055, 0, 0, 0, 0, 1029, 995, 1027, 1072, 0, 0, 0, 0, 1050, 1058, 1034, 1023, 0, 0, 0, 0, 972, 1040, 1030, 1029, 5, 0, 0, 0, 1002, 1027, 1066, 1035, 13, 0, 0, 0, 1000, 1048, 991, 1004, 2, 1, 0, 0, 996, 1021, 976, 1033, 4, 1, 0, 0, 1055, 1016, 992, 1008, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4123, 4165, 4076, 4143, 4046, 4031, 4073] avg=4096.0000 max=4165 max_rank=2 min=4031 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[994, 999, 1036, 1018, 0, 0, 0, 0, 977, 1030, 994, 1043, 0, 0, 0, 0, 1040, 1010, 1010, 1045, 0, 0, 0, 0, 1055, 1028, 1024, 1002, 5, 0, 0, 0, 989, 1042, 988, 1034, 13, 0, 0, 0, 1031, 1027, 1030, 1049, 2, 1, 0, 0, 967, 1036, 1052, 1060, 4, 1, 0, 0, 1012, 1064, 992, 1062, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4044, 4105, 4114, 4066, 4140, 4120, 4132] avg=4096.0000 max=4140 max_rank=5 min=4044 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1018, 1013, 1028, 1013, 0, 0, 0, 0, 984, 1005, 1099, 1027, 0, 0, 0, 0, 1031, 989, 1058, 1021, 0, 0, 0, 0, 1033, 1055, 1025, 1010, 6, 0, 0, 0, 1075, 1037, 1008, 1002, 13, 0, 0, 0, 1012, 1069, 996, 974, 2, 1, 0, 0, 1049, 978, 992, 1031, 4, 1, 0, 0, 1047, 1013, 1033, 1014, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4115, 4099, 4129, 4135, 4054, 4055, 4109] avg=4096.0000 max=4135 max_rank=4 min=4054 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1054, 992, 1032, 1030, 0, 0, 0, 0, 1034, 1030, 1026, 1101, 0, 0, 0, 0, 1012, 1013, 969, 978, 0, 0, 0, 0, 1024, 1004, 1034, 994, 10, 0, 0, 0, 1061, 1046, 1022, 1017, 18, 0, 0, 0, 1005, 1074, 1045, 986, 7, 2, 0, 0, 1039, 976, 1006, 1013, 8, 5, 0, 0, 1053, 1013, 1030, 1000, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4191, 3972, 4066, 4164, 4119, 4047, 4101] avg=4096.0000 max=4191 max_rank=1 min=3972 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8448, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7936, 8192, 8448, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1062, 1008, 988, 1017, 1040, 1076, 1047, 1040, 988, 1002, 1019, 1008, 1042, 1043, 1045, 1036, 1025, 1017, 1013, 1062, 990, 1025, 1020, 1046, 1004, 1030, 996, 1027, 983, 1067, 1007, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1000, 1031, 989, 1064, 1019, 1035, 1011, 1069, 1035, 1021, 991, 1042, 1003, 1034, 980, 1020, 975, 1048, 1032, 1014, 1003, 1037, 1021, 1073, 1051, 1012, 1025, 1025, 1047, 1009, 1037, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1010, 1036, 1006, 1046, 1031, 1012, 1008, 1030, 995, 995, 1033, 1021, 991, 1019, 1006, 980, 1062, 1055, 1062, 1007, 1069, 1045, 1029, 997, 1002, 1052, 1018, 1038, 1031, 1014, 1003, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[979, 1065, 1026, 1014, 1049, 1030, 1061, 1006, 1036, 1032, 1041, 1005, 1037, 1037, 1002, 1018, 1026, 1042, 1032, 1026, 1047, 986, 987, 1009, 1027, 1000, 1020, 1026, 1012, 1024, 1058, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1033, 1022, 1066, 988, 1018, 1030, 1017, 1000, 1000, 1062, 987, 1011, 999, 1086, 1069, 1014, 1033, 979, 1025, 1001, 1033, 1002, 1004, 979, 1001, 1031, 1055, 1031, 1072, 1011, 1044, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[989, 994, 1038, 983, 1030, 987, 1060, 1035, 970, 1033, 1027, 1097, 1059, 1003, 1036, 996, 1034, 996, 980, 1040, 1065, 1043, 1034, 1066, 996, 1060, 1067, 991, 982, 1006, 1047, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1011, 1031, 1041, 993, 1010, 1082, 1031, 1045, 995, 1032, 997, 1047, 1040, 1017, 1038, 1015, 1042, 999, 1041, 1021, 962, 1087, 981, 1004, 996, 1005, 1011, 1025, 1046, 1031, 1031, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1048, 1045, 1025, 1020, 982, 1041, 1006, 1056, 1072, 1073, 993, 1049, 976, 1027, 1002, 1050, 972, 1021, 1028, 1003, 1037, 1039, 1027, 1045, 1030, 1003, 1042, 941, 1036, 1008, 1061, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1010, 1036, 1006, 1046, 11, 0, 0, 0, 1031, 986, 1008, 1030, 0, 0, 0, 0, 995, 995, 1033, 1021, 7, 0, 0, 0, 991, 1019, 1006, 980, 3, 2, 0, 0, 1062, 1055, 1062, 1007, 6, 0, 0, 0, 1069, 1042, 1029, 997, 0, 0, 0, 0, 1002, 1052, 1018, 1038, 15, 0, 0, 0, 1031, 1014, 988, 1065, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4055, 4051, 4001, 4192, 4137, 4125, 4098] avg=4096.0000 max=4192 max_rank=4 min=4001 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16552 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[989, 994, 1038, 983, 11, 0, 0, 0, 1030, 961, 1060, 1035, 0, 0, 0, 0, 970, 1033, 1027, 1097, 7, 0, 0, 0, 1059, 1003, 1036, 996, 3, 3, 0, 0, 1034, 996, 980, 1040, 6, 0, 0, 0, 1065, 1040, 1034, 1066, 0, 0, 0, 0, 996, 1060, 1067, 991, 15, 0, 0, 0, 982, 1006, 1031, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4086, 4134, 4100, 4056, 4205, 4129, 4043] avg=4096.0000 max=4205 max_rank=5 min=4015 min_rank=0 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1062, 1008, 988, 1017, 12, 0, 0, 0, 1040, 1048, 1047, 1040, 0, 0, 0, 0, 988, 1002, 1019, 1008, 7, 0, 0, 0, 1042, 1043, 1045, 1036, 2, 2, 0, 0, 1025, 1017, 1013, 1062, 6, 0, 0, 0, 990, 1023, 1020, 1046, 0, 0, 0, 0, 1004, 1030, 996, 1027, 16, 0, 0, 0, 983, 1067, 992, 995, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4175, 4024, 4170, 4123, 4079, 4073, 4037] avg=4096.0000 max=4175 max_rank=1 min=4024 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[979, 1065, 1026, 1014, 11, 0, 0, 0, 1049, 1003, 1061, 1006, 0, 0, 0, 0, 1036, 1032, 1041, 1005, 7, 0, 0, 0, 1037, 1037, 1002, 1018, 2, 3, 0, 0, 1026, 1042, 1032, 1026, 7, 0, 0, 0, 1047, 984, 987, 1009, 0, 0, 0, 0, 1027, 1000, 1020, 1026, 16, 0, 0, 0, 1012, 1024, 1041, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4119, 4121, 4099, 4133, 4027, 4089, 4085] avg=4096.0000 max=4133 max_rank=4 min=4027 min_rank=5 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1000, 1031, 989, 1064, 11, 0, 0, 0, 1019, 1008, 1011, 1069, 0, 0, 0, 0, 1035, 1021, 991, 1042, 7, 0, 0, 0, 1003, 1034, 980, 1020, 3, 3, 0, 0, 975, 1048, 1032, 1014, 6, 0, 0, 0, 1003, 1034, 1021, 1073, 0, 0, 0, 0, 1051, 1012, 1025, 1025, 16, 0, 0, 0, 1047, 1009, 1021, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4107, 4096, 4043, 4075, 4131, 4129, 4092] avg=4096.0000 max=4131 max_rank=5 min=4043 min_rank=3 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1033, 1022, 1066, 988, 16, 4, 0, 0, 1018, 994, 1017, 1000, 0, 0, 0, 0, 1000, 1062, 987, 1011, 10, 0, 0, 0, 999, 1086, 1069, 1014, 5, 6, 0, 0, 1033, 979, 1025, 1001, 11, 0, 0, 0, 1033, 997, 1004, 979, 0, 0, 0, 0, 1001, 1031, 1055, 1031, 20, 0, 0, 0, 1072, 1011, 1013, 1065, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4029, 4070, 4179, 4049, 4013, 4138, 4161] avg=4096.0000 max=4179 max_rank=3 min=4013 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1011, 1031, 1041, 993, 12, 0, 0, 0, 1010, 1054, 1031, 1045, 0, 0, 0, 0, 995, 1032, 997, 1047, 7, 0, 0, 0, 1040, 1017, 1038, 1015, 3, 2, 0, 0, 1042, 999, 1041, 1021, 6, 0, 0, 0, 962, 1084, 981, 1004, 0, 0, 0, 0, 996, 1005, 1011, 1025, 16, 0, 0, 0, 1046, 1031, 1016, 1061, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4140, 4078, 4115, 4109, 4031, 4053, 4154] avg=4096.0000 max=4154 max_rank=7 min=4031 min_rank=5 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1048, 1045, 1025, 1020, 12, 0, 0, 0, 982, 1013, 1006, 1056, 0, 0, 0, 0, 1072, 1073, 993, 1049, 7, 0, 0, 0, 976, 1027, 1002, 1050, 3, 3, 0, 0, 972, 1021, 1028, 1003, 7, 0, 0, 0, 1037, 1036, 1027, 1045, 0, 0, 0, 0, 1030, 1003, 1042, 941, 16, 0, 0, 0, 1036, 1008, 1044, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4057, 4194, 4061, 4031, 4145, 4032, 4098] avg=4096.0000 max=4194 max_rank=2 min=4031 min_rank=4 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1027, 966, 1024, 1038, 1039, 1080, 1019, 1005, 988, 1011, 1005, 1029, 1038, 1007, 1052, 1056, 1035, 961, 1023, 1005, 1041, 1007, 983, 1018, 1002, 1075, 1001, 1061, 1072, 997, 1045, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1056, 1019, 1018, 1046, 1010, 1054, 1008, 998, 1105, 1014, 1010, 996, 1002, 1043, 1033, 1017, 1051, 1027, 1059, 1061, 1032, 1004, 982, 1010, 996, 1040, 982, 1022, 994, 1025, 1013, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1012, 990, 1026, 1041, 1082, 1028, 1014, 1003, 1081, 1020, 1049, 1006, 1017, 999, 1002, 1029, 1018, 1020, 1024, 1021, 1064, 1000, 1007, 1031, 1064, 987, 1001, 1029, 1021, 1037, 1003, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[978, 1043, 1054, 1020, 1047, 1052, 988, 1042, 1015, 1073, 1004, 989, 985, 1094, 1028, 1013, 1044, 1013, 1012, 955, 1004, 1039, 1028, 1033, 1068, 1000, 1079, 1032, 986, 992, 1003, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1062, 1030, 1024, 1052, 1038, 980, 1017, 1040, 1020, 1030, 991, 1077, 965, 995, 1047, 1030, 1105, 1016, 1048, 1022, 1044, 977, 1036, 1021, 997, 1072, 1038, 993, 1053, 962, 972, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[990, 994, 1036, 1028, 983, 1024, 1011, 1038, 1027, 1059, 1050, 985, 1047, 1021, 1040, 1005, 1042, 1048, 1005, 1041, 1044, 991, 1044, 1020, 993, 1032, 1022, 1035, 1027, 1000, 1038, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[999, 1028, 997, 1057, 1079, 1018, 1024, 1042, 1035, 1037, 1012, 1033, 1014, 1043, 1018, 1051, 967, 1026, 1059, 1030, 1015, 973, 1051, 1032, 983, 983, 1005, 1051, 1029, 1046, 1016, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1012, 993, 977, 1017, 1049, 1013, 1054, 1030, 971, 1054, 1031, 1041, 1035, 1028, 1045, 1047, 1056, 1005, 1025, 1061, 988, 967, 1072, 1013, 1048, 973, 1078, 991, 1028, 1057, 955, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1027, 966, 1024, 1038, 7, 6, 0, 0, 1022, 1080, 1019, 1005, 0, 0, 0, 0, 988, 1002, 1005, 1029, 0, 0, 0, 0, 1038, 1007, 1043, 1056, 0, 0, 0, 0, 1022, 961, 1023, 1005, 0, 0, 0, 0, 1041, 1007, 983, 1018, 17, 6, 0, 0, 1002, 1075, 1001, 1061, 4, 0, 0, 0, 1072, 997, 1045, 1058, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4126, 4024, 4144, 4011, 4072, 4143, 4180] avg=4096.0000 max=4180 max_rank=7 min=4011 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[978, 1043, 1054, 1020, 7, 6, 0, 0, 1030, 1052, 988, 1042, 0, 0, 0, 0, 1015, 1064, 1004, 989, 0, 0, 0, 0, 985, 1094, 1019, 1013, 0, 0, 0, 0, 1030, 1013, 1012, 955, 0, 0, 0, 0, 1004, 1039, 1028, 1033, 17, 7, 0, 0, 1068, 1000, 1079, 1032, 4, 0, 0, 0, 986, 992, 1003, 1055, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4112, 4072, 4111, 4010, 4128, 4183, 4044] avg=4096.0000 max=4183 max_rank=6 min=4010 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1012, 990, 1026, 1041, 7, 6, 0, 0, 1064, 1028, 1014, 1003, 0, 0, 0, 0, 1081, 1011, 1049, 1006, 0, 0, 0, 0, 1017, 999, 993, 1029, 0, 0, 0, 0, 1005, 1020, 1024, 1021, 0, 0, 0, 0, 1064, 1000, 1007, 1031, 18, 6, 0, 0, 1064, 987, 1001, 1029, 4, 0, 0, 0, 1021, 1037, 1003, 1042, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4109, 4147, 4038, 4070, 4126, 4085, 4111] avg=4096.0000 max=4147 max_rank=2 min=4038 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1056, 1019, 1018, 1046, 7, 6, 0, 0, 993, 1054, 1008, 998, 0, 0, 0, 0, 1105, 1005, 1010, 996, 0, 0, 0, 0, 1002, 1043, 1024, 1017, 0, 0, 0, 0, 1037, 1027, 1059, 1061, 0, 0, 0, 0, 1032, 1004, 982, 1010, 17, 7, 0, 0, 996, 1040, 982, 1022, 4, 0, 0, 0, 994, 1025, 1013, 1041, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4053, 4116, 4086, 4184, 4052, 4044, 4081] avg=4096.0000 max=4184 max_rank=4 min=4044 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[990, 994, 1036, 1028, 7, 6, 0, 0, 967, 1024, 1011, 1038, 0, 0, 0, 0, 1027, 1050, 1050, 985, 0, 0, 0, 0, 1047, 1021, 1031, 1005, 0, 0, 0, 0, 1028, 1048, 1005, 1041, 0, 0, 0, 0, 1044, 991, 1044, 1020, 16, 7, 0, 0, 993, 1032, 1022, 1035, 4, 0, 0, 0, 1027, 1000, 1038, 1048, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4040, 4112, 4104, 4122, 4122, 4086, 4121] avg=4096.0000 max=4122 max_rank=4 min=4040 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1062, 1030, 1024, 1052, 12, 11, 0, 0, 1017, 980, 1017, 1040, 0, 0, 0, 0, 1020, 1013, 991, 1077, 0, 0, 0, 0, 965, 995, 1032, 1030, 0, 0, 0, 0, 1083, 1016, 1048, 1022, 0, 0, 0, 0, 1044, 977, 1036, 1021, 21, 10, 0, 0, 997, 1072, 1038, 993, 7, 0, 0, 0, 1053, 962, 972, 1014, 8, 6, 0, 0] rerouted_tokens_per_ep_rank=[4191, 4054, 4101, 4022, 4169, 4109, 4107, 4015] avg=4096.0000 max=4191 max_rank=0 min=4015 min_rank=7 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[999, 1028, 997, 1057, 7, 6, 0, 0, 1061, 1018, 1024, 1042, 0, 0, 0, 0, 1035, 1028, 1012, 1033, 0, 0, 0, 0, 1014, 1043, 1009, 1051, 0, 0, 0, 0, 954, 1026, 1059, 1030, 0, 0, 0, 0, 1015, 973, 1051, 1032, 18, 6, 0, 0, 983, 983, 1005, 1051, 4, 0, 0, 0, 1029, 1046, 1016, 1015, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4145, 4108, 4117, 4069, 4095, 4026, 4114] avg=4096.0000 max=4145 max_rank=1 min=4026 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1012, 993, 977, 1017, 7, 6, 0, 0, 1032, 1013, 1054, 1030, 0, 0, 0, 0, 971, 1045, 1031, 1041, 0, 0, 0, 0, 1035, 1028, 1036, 1047, 0, 0, 0, 0, 1042, 1005, 1025, 1061, 0, 0, 0, 0, 988, 967, 1072, 1013, 17, 7, 0, 0, 1048, 973, 1078, 991, 4, 0, 0, 0, 1028, 1057, 955, 1054, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4129, 4088, 4146, 4133, 4064, 4094, 4102] avg=4096.0000 max=4146 max_rank=3 min=4012 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8448, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1003, 1026, 1010, 1026, 1074, 1008, 1052, 1061, 1022, 979, 1036, 988, 1016, 997, 1044, 1049, 993, 1005, 1046, 1047, 1021, 1096, 1032, 997, 986, 1016, 995, 984, 990, 1047, 1055, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1042, 984, 1035, 1046, 1007, 969, 1026, 1049, 1034, 1019, 1001, 1042, 1011, 1034, 1022, 961, 1026, 1022, 1045, 1010, 1060, 997, 1081, 1045, 1007, 1030, 1036, 1062, 1023, 1040, 995, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1016, 1020, 1013, 1119, 981, 1059, 1074, 1061, 1056, 967, 982, 971, 1015, 1019, 1044, 1023, 1077, 1018, 1017, 991, 1044, 990, 1027, 984, 977, 1027, 1074, 1045, 1031, 1020, 1040, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1015, 988, 1023, 1017, 1055, 1023, 1017, 1082, 1025, 1000, 978, 1022, 1026, 1014, 1059, 1041, 992, 1033, 1016, 1057, 1065, 989, 1066, 1001, 1031, 1007, 976, 1037, 1030, 1016, 1022, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1013, 1019, 1073, 1033, 1050, 1013, 984, 1020, 1051, 999, 984, 1026, 996, 992, 1014, 1020, 1053, 1032, 1012, 1023, 1023, 1004, 982, 1066, 984, 1041, 1038, 1061, 1034, 1037, 1036, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1059, 1047, 1034, 1003, 1021, 1003, 1015, 1032, 1013, 1011, 1003, 986, 1048, 1022, 1058, 1008, 1047, 984, 1042, 1018, 1014, 1083, 998, 1082, 1027, 1014, 1008, 1031, 974, 1035, 1031, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[991, 1042, 1012, 1018, 995, 1023, 978, 1033, 1037, 1047, 1000, 1018, 1021, 1036, 987, 1034, 1023, 1014, 1036, 1014, 1042, 1026, 1019, 1070, 1021, 1009, 1021, 1054, 1039, 1040, 1013, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1021, 961, 1024, 1017, 1041, 1013, 995, 975, 1068, 1082, 1025, 992, 1033, 1011, 1008, 1062, 1024, 1009, 1065, 982, 1007, 1025, 1054, 1066, 1061, 1045, 984, 1056, 1018, 1027, 1000, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1016, 1020, 1013, 1119, 1, 0, 0, 0, 981, 1059, 1074, 1059, 0, 0, 0, 0, 1056, 967, 982, 971, 34, 1, 0, 0, 1015, 1019, 1044, 1023, 5, 0, 0, 0, 1077, 1018, 1017, 991, 0, 0, 0, 0, 1044, 990, 1027, 950, 0, 0, 0, 0, 977, 1027, 1074, 1045, 1, 1, 0, 0, 1031, 1013, 1040, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4173, 4011, 4106, 4103, 4011, 4125, 4070] avg=4096.0000 max=4173 max_rank=1 min=4011 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1003, 1026, 1010, 1026, 1, 0, 0, 0, 1074, 1008, 1052, 1059, 0, 0, 0, 0, 1022, 979, 1036, 988, 34, 2, 0, 0, 1016, 997, 1044, 1049, 5, 0, 0, 0, 993, 1005, 1046, 1047, 0, 0, 0, 0, 1021, 1096, 1032, 963, 0, 0, 0, 0, 986, 1016, 995, 984, 1, 1, 0, 0, 990, 1039, 1055, 1067, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4193, 4061, 4111, 4091, 4112, 3983, 4151] avg=4096.0000 max=4193 max_rank=1 min=3983 min_rank=6 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1013, 1019, 1073, 1033, 1, 0, 0, 0, 1050, 1013, 984, 1019, 0, 0, 0, 0, 1051, 999, 984, 1026, 36, 2, 0, 0, 996, 992, 1014, 1020, 5, 0, 0, 0, 1053, 1032, 1012, 1023, 0, 0, 0, 0, 1023, 1004, 982, 1030, 0, 0, 0, 0, 984, 1041, 1038, 1061, 1, 0, 0, 0, 1034, 1029, 1036, 1055, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4066, 4098, 4027, 4120, 4039, 4125, 4154] avg=4096.0000 max=4154 max_rank=7 min=4027 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1042, 984, 1035, 1046, 1, 0, 0, 0, 1007, 969, 1026, 1047, 0, 0, 0, 0, 1034, 1019, 1001, 1042, 36, 2, 0, 0, 1011, 1034, 1022, 961, 5, 0, 0, 0, 1026, 1022, 1045, 1010, 0, 0, 0, 0, 1060, 997, 1081, 1009, 0, 0, 0, 0, 1007, 1030, 1036, 1062, 1, 1, 0, 0, 1023, 1032, 995, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4049, 4134, 4033, 4103, 4147, 4137, 4057] avg=4096.0000 max=4147 max_rank=5 min=4033 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1021, 961, 1024, 1017, 1, 0, 0, 0, 1041, 1013, 995, 974, 0, 0, 0, 0, 1068, 1082, 1025, 992, 36, 1, 0, 0, 1033, 1011, 1008, 1062, 5, 0, 0, 0, 1024, 1009, 1065, 982, 0, 0, 0, 0, 1007, 1025, 1054, 1030, 0, 0, 0, 0, 1061, 1045, 984, 1056, 1, 0, 0, 0, 1018, 1020, 1000, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4023, 4204, 4119, 4080, 4116, 4147, 4055] avg=4096.0000 max=4204 max_rank=2 min=4023 min_rank=1 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1059, 1047, 1034, 1003, 1, 0, 0, 0, 1021, 1003, 1015, 1031, 0, 0, 0, 0, 1013, 1011, 1003, 986, 37, 2, 0, 0, 1048, 1022, 1058, 1008, 5, 0, 0, 0, 1047, 984, 1042, 1018, 0, 0, 0, 0, 1014, 1083, 998, 1045, 0, 0, 0, 0, 1027, 1014, 1008, 1031, 1, 0, 0, 0, 974, 1027, 1031, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4070, 4052, 4141, 4091, 4140, 4081, 4049] avg=4096.0000 max=4144 max_rank=0 min=4049 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[991, 1042, 1012, 1018, 6, 5, 0, 0, 995, 1023, 978, 1023, 0, 0, 0, 0, 1037, 1047, 1000, 1018, 41, 5, 0, 0, 1021, 1036, 987, 1034, 8, 0, 0, 0, 1023, 1014, 1031, 1014, 0, 0, 0, 0, 1042, 1026, 1019, 1029, 0, 0, 0, 0, 1021, 1009, 1021, 1054, 8, 4, 0, 0, 1039, 1019, 1013, 1055, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4019, 4148, 4086, 4082, 4116, 4117, 4126] avg=4096.0000 max=4148 max_rank=2 min=4019 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1015, 988, 1023, 1017, 1, 0, 0, 0, 1055, 1023, 1017, 1080, 0, 0, 0, 0, 1025, 1000, 978, 1022, 34, 1, 0, 0, 1026, 1014, 1059, 1041, 5, 0, 0, 0, 992, 1033, 1016, 1057, 0, 0, 0, 0, 1065, 989, 1066, 967, 0, 0, 0, 0, 1031, 1007, 976, 1037, 1, 1, 0, 0, 1030, 1009, 1022, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4175, 4060, 4145, 4098, 4087, 4053, 4106] avg=4096.0000 max=4175 max_rank=1 min=4044 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8448, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8064, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1015, 1048, 1080, 1055, 983, 982, 1002, 987, 991, 1031, 978, 1046, 990, 1024, 1023, 1052, 1050, 1055, 1046, 1024, 1020, 1010, 1026, 1038, 1033, 1062, 1051, 976, 1005, 1027, 1032, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1064, 985, 1032, 1022, 1082, 1026, 1057, 1041, 1004, 1011, 1015, 979, 992, 1002, 1033, 1005, 1028, 986, 1060, 1022, 1041, 1022, 981, 1014, 1034, 1013, 1036, 1076, 1049, 1019, 1007, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[962, 998, 1040, 1079, 1018, 1053, 1043, 947, 1032, 1080, 1022, 987, 1011, 1025, 980, 1054, 1036, 1043, 1026, 1027, 1028, 1009, 1066, 1060, 1007, 1006, 990, 1038, 1004, 1037, 1047, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1005, 977, 1043, 976, 1051, 1019, 1058, 1031, 1062, 1014, 1065, 1010, 1037, 965, 1034, 1039, 1009, 1006, 1035, 1005, 1024, 997, 1001, 1035, 1028, 1056, 1067, 984, 1035, 1042, 1012, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1014, 1015, 1025, 1028, 1031, 1014, 1066, 993, 953, 1039, 1046, 1041, 1068, 1046, 941, 1007, 998, 1052, 1043, 1003, 996, 1013, 1018, 1051, 1036, 1080, 1008, 1017, 1025, 1000, 1073, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1046, 1054, 1057, 1038, 1045, 1013, 1003, 1056, 997, 999, 1037, 1014, 955, 1025, 1069, 1063, 1013, 1071, 957, 1004, 1018, 1020, 1086, 1032, 1002, 1005, 996, 1076, 1030, 996, 1008, 983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1043, 1006, 1011, 1057, 1008, 995, 1015, 1027, 1081, 1052, 980, 1042, 1013, 1016, 1032, 1031, 1049, 1012, 1060, 991, 980, 1014, 1049, 1016, 1019, 1016, 1032, 1043, 1003, 1035, 1023, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[983, 1031, 971, 1028, 1032, 1055, 1048, 1006, 1036, 982, 1047, 1032, 1004, 991, 1066, 1002, 1006, 997, 998, 1014, 1038, 1060, 1040, 1043, 1010, 1065, 1060, 1007, 1020, 1034, 1025, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1015, 1048, 1080, 1053, 0, 0, 0, 0, 983, 982, 1000, 987, 0, 0, 0, 0, 991, 1031, 978, 1046, 7, 0, 0, 0, 990, 1024, 1023, 1052, 20, 1, 0, 0, 1050, 1055, 1046, 1024, 2, 2, 0, 0, 1020, 1010, 1026, 1030, 0, 0, 0, 0, 1033, 1042, 1051, 976, 0, 0, 0, 0, 1005, 1027, 1032, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4196, 3952, 4053, 4110, 4179, 4086, 4102, 4090] avg=4096.0000 max=4196 max_rank=0 min=3952 min_rank=1 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16457 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1005, 977, 1043, 974, 0, 0, 0, 0, 1051, 1019, 1056, 1031, 0, 0, 0, 0, 1062, 1014, 1065, 1010, 7, 0, 0, 0, 1037, 965, 1034, 1039, 20, 1, 0, 0, 1009, 1006, 1035, 1005, 2, 2, 0, 0, 1024, 997, 1001, 1027, 0, 0, 0, 0, 1028, 1036, 1067, 984, 0, 0, 0, 0, 1035, 1042, 1012, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3999, 4157, 4158, 4096, 4059, 4049, 4115, 4135] avg=4096.0000 max=4158 max_rank=2 min=3999 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16358 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[962, 998, 1040, 1077, 0, 0, 0, 0, 1018, 1053, 1041, 947, 0, 0, 0, 0, 1032, 1080, 1022, 987, 8, 0, 0, 0, 1011, 1025, 980, 1054, 19, 1, 0, 0, 1036, 1043, 1026, 1027, 2, 2, 0, 0, 1028, 1009, 1066, 1051, 0, 0, 0, 0, 1007, 987, 990, 1038, 0, 0, 0, 0, 1004, 1037, 1047, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4059, 4129, 4090, 4136, 4154, 4022, 4101] avg=4096.0000 max=4154 max_rank=5 min=4022 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16413 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1014, 1015, 1025, 1026, 0, 0, 0, 0, 1031, 1014, 1064, 993, 0, 0, 0, 0, 953, 1039, 1046, 1041, 7, 0, 0, 0, 1068, 1046, 941, 1007, 20, 1, 0, 0, 998, 1052, 1043, 1003, 2, 2, 0, 0, 996, 1013, 1018, 1043, 0, 0, 0, 0, 1036, 1060, 1008, 1017, 0, 0, 0, 0, 1025, 1000, 1073, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4102, 4086, 4083, 4100, 4070, 4121, 4126] avg=4096.0000 max=4126 max_rank=7 min=4070 min_rank=5 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16417 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1064, 985, 1032, 1016, 0, 0, 0, 0, 1082, 1026, 1052, 1041, 0, 0, 0, 0, 1004, 1011, 1015, 979, 13, 0, 0, 0, 992, 1002, 1033, 1005, 24, 5, 0, 0, 1028, 986, 1060, 1022, 6, 5, 3, 0, 1041, 1022, 981, 993, 0, 0, 0, 0, 1034, 989, 1036, 1076, 0, 0, 0, 0, 1049, 1019, 1007, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4201, 4022, 4061, 4110, 4037, 4135, 4105] avg=4096.0000 max=4201 max_rank=1 min=4022 min_rank=2 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1046, 1054, 1057, 1036, 0, 0, 0, 0, 1045, 1013, 1001, 1056, 0, 0, 0, 0, 997, 999, 1037, 1014, 7, 0, 0, 0, 955, 1025, 1069, 1063, 19, 1, 0, 0, 1013, 1071, 957, 1004, 2, 2, 0, 0, 1018, 1020, 1086, 1024, 0, 0, 0, 0, 1002, 986, 996, 1076, 0, 0, 0, 0, 1030, 996, 1008, 983, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4193, 4115, 4054, 4132, 4049, 4148, 4060, 4017] avg=4096.0000 max=4193 max_rank=0 min=4017 min_rank=7 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16274 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1043, 1006, 1011, 1055, 0, 0, 0, 0, 1008, 995, 1013, 1027, 0, 0, 0, 0, 1081, 1052, 980, 1042, 7, 0, 0, 0, 1013, 1016, 1032, 1031, 19, 1, 0, 0, 1049, 1012, 1060, 991, 2, 2, 0, 0, 980, 1014, 1049, 1008, 0, 0, 0, 0, 1019, 997, 1032, 1043, 0, 0, 0, 0, 1003, 1035, 1023, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4043, 4162, 4112, 4116, 4051, 4091, 4078] avg=4096.0000 max=4162 max_rank=2 min=4043 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16336 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[983, 1031, 971, 1026, 0, 0, 0, 0, 1032, 1055, 1046, 1006, 0, 0, 0, 0, 1036, 982, 1047, 1032, 7, 0, 0, 0, 1004, 991, 1066, 1002, 20, 1, 0, 0, 1006, 997, 998, 1014, 2, 2, 0, 0, 1038, 1060, 1040, 1035, 0, 0, 0, 0, 1010, 1045, 1060, 1007, 0, 0, 0, 0, 1020, 1034, 1025, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4139, 4104, 4084, 4019, 4173, 4122, 4116] avg=4096.0000 max=4173 max_rank=5 min=4011 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16430 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1042, 954, 1052, 1016, 1037, 1026, 1067, 1002, 1046, 1056, 1006, 1023, 983, 1051, 1036, 1111, 992, 999, 1025, 1030, 1018, 1026, 1017, 1032, 991, 1019, 1023, 1014, 1038, 1030, 1008, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1074, 1016, 1002, 1037, 1010, 999, 1024, 1016, 982, 1048, 1037, 1035, 1053, 1008, 1006, 1001, 1038, 1055, 973, 1022, 1032, 998, 998, 1040, 1049, 1065, 1047, 991, 1030, 1036, 1009, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1005, 965, 1015, 1030, 1022, 1032, 1027, 992, 1051, 1031, 1068, 990, 1045, 1014, 1031, 1066, 1014, 1023, 999, 1057, 1040, 1046, 1005, 1016, 1009, 1033, 1052, 1063, 988, 1003, 1030, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1050, 1032, 1033, 995, 1040, 1015, 1017, 978, 1051, 972, 1042, 996, 993, 1059, 1020, 1009, 1067, 1069, 1010, 1052, 1061, 1014, 1062, 1011, 1050, 1011, 1006, 1039, 1023, 1003, 974, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[986, 1017, 1027, 1041, 977, 979, 1005, 1071, 1049, 1045, 1060, 1033, 1025, 1015, 1028, 1066, 1010, 977, 1022, 1029, 1031, 1001, 970, 1037, 1023, 1012, 1038, 1027, 1021, 1025, 1037, 1084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1047, 1019, 1028, 1000, 1025, 1084, 1018, 1033, 1023, 977, 1049, 1062, 1041, 1022, 1068, 1051, 1009, 1024, 1026, 1012, 983, 1018, 999, 998, 1064, 1020, 1032, 997, 1045, 1014, 986, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1046, 1036, 1070, 1006, 976, 993, 958, 1060, 994, 996, 1056, 1034, 974, 1042, 1009, 1006, 1053, 1018, 1024, 1001, 1064, 990, 1010, 1049, 1000, 1074, 1030, 1031, 1035, 1058, 1053, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1017, 1035, 1002, 1025, 998, 1038, 1016, 1019, 997, 1048, 1012, 1053, 1048, 1096, 1048, 1036, 1041, 1019, 1017, 996, 1009, 1085, 1013, 1048, 1017, 991, 984, 1017, 998, 981, 1028, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1042, 954, 1052, 1016, 5, 0, 0, 0, 1037, 1026, 1067, 1002, 28, 0, 0, 0, 1046, 1056, 989, 1023, 0, 0, 0, 0, 983, 1051, 1036, 1073, 0, 0, 0, 0, 992, 999, 1025, 1030, 7, 0, 0, 0, 1018, 1026, 1017, 1032, 5, 0, 0, 0, 991, 1014, 1023, 1014, 0, 0, 0, 0, 1038, 1030, 1008, 998, 10, 5, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4160, 4114, 4143, 4053, 4098, 4042, 4089] avg=4096.0000 max=4160 max_rank=1 min=4042 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1005, 965, 1015, 1030, 5, 0, 0, 0, 1022, 1032, 1027, 992, 27, 0, 0, 0, 1051, 1031, 1050, 990, 0, 0, 0, 0, 1045, 1014, 1031, 1029, 0, 0, 0, 0, 1014, 1023, 999, 1057, 8, 0, 0, 0, 1040, 1046, 1005, 1016, 5, 0, 0, 0, 1009, 1028, 1052, 1063, 0, 0, 0, 0, 988, 1003, 1030, 1006, 10, 5, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4100, 4122, 4119, 4101, 4112, 4152, 4042] avg=4096.0000 max=4152 max_rank=6 min=4020 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1050, 1032, 1033, 995, 5, 0, 0, 0, 1040, 1015, 1017, 978, 25, 0, 0, 0, 1051, 972, 1024, 996, 0, 0, 0, 0, 993, 1059, 1020, 975, 0, 0, 0, 0, 1067, 1069, 1010, 1052, 8, 0, 0, 0, 1061, 1014, 1062, 1011, 5, 0, 0, 0, 1050, 1006, 1006, 1039, 0, 0, 0, 0, 1023, 1003, 974, 1014, 9, 5, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4075, 4043, 4047, 4206, 4153, 4101, 4028] avg=4096.0000 max=4206 max_rank=4 min=4028 min_rank=7 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1074, 1016, 1002, 1037, 5, 0, 0, 0, 1010, 999, 1024, 1016, 25, 0, 0, 0, 982, 1048, 1019, 1035, 0, 0, 0, 0, 1053, 1008, 1006, 967, 0, 0, 0, 0, 1038, 1055, 973, 1022, 8, 0, 0, 0, 1032, 998, 998, 1040, 6, 0, 0, 0, 1049, 1059, 1047, 991, 0, 0, 0, 0, 1030, 1036, 1009, 1037, 9, 5, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4074, 4084, 4034, 4096, 4074, 4146, 4126] avg=4096.0000 max=4146 max_rank=6 min=4034 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1046, 1036, 1070, 1006, 5, 0, 0, 0, 976, 993, 958, 1060, 25, 0, 0, 0, 994, 996, 1038, 1034, 0, 0, 0, 0, 974, 1042, 1009, 972, 0, 0, 0, 0, 1053, 1018, 1024, 1001, 8, 0, 0, 0, 1064, 990, 1010, 1049, 6, 0, 0, 0, 1000, 1068, 1030, 1031, 0, 0, 0, 0, 1035, 1058, 1053, 1022, 9, 5, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4012, 4062, 3997, 4104, 4119, 4129, 4182] avg=4096.0000 max=4182 max_rank=7 min=3997 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1047, 1019, 1028, 1000, 5, 0, 0, 0, 1025, 1084, 1018, 1033, 26, 0, 0, 0, 1023, 977, 1031, 1062, 0, 0, 0, 0, 1041, 1022, 1068, 1016, 0, 0, 0, 0, 1009, 1024, 1026, 1012, 8, 0, 0, 0, 983, 1018, 999, 998, 5, 0, 0, 0, 1064, 1015, 1032, 997, 0, 0, 0, 0, 1045, 1014, 986, 994, 9, 5, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4186, 4093, 4147, 4079, 4003, 4108, 4053] avg=4096.0000 max=4186 max_rank=1 min=4003 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16243 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[986, 1017, 1027, 1041, 9, 4, 0, 0, 977, 979, 1005, 1071, 32, 0, 0, 0, 1049, 1045, 1030, 1033, 0, 0, 0, 0, 1025, 1015, 1028, 1020, 0, 0, 0, 0, 1010, 977, 1022, 1029, 11, 0, 0, 0, 1031, 1001, 970, 1037, 10, 0, 0, 0, 1023, 998, 1038, 1027, 0, 0, 0, 0, 1021, 1025, 1037, 1084, 14, 10, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4064, 4157, 4088, 4049, 4049, 4086, 4191] avg=4096.0000 max=4191 max_rank=7 min=4049 min_rank=4 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1017, 1035, 1002, 1025, 5, 0, 0, 0, 998, 1038, 1016, 1019, 26, 0, 0, 0, 997, 1048, 995, 1053, 0, 0, 0, 0, 1048, 1096, 1048, 1001, 0, 0, 0, 0, 1041, 1019, 1017, 996, 7, 0, 0, 0, 1009, 1085, 1013, 1048, 5, 0, 0, 0, 1017, 986, 984, 1017, 0, 0, 0, 0, 998, 981, 1028, 1036, 9, 5, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4097, 4093, 4193, 4080, 4160, 4004, 4057] avg=4096.0000 max=4193 max_rank=3 min=4004 min_rank=6 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1028, 989, 1029, 1012, 1038, 1060, 947, 988, 1021, 1086, 1036, 987, 1004, 1046, 1046, 1031, 986, 1008, 979, 1011, 1020, 999, 1033, 1067, 1040, 1021, 1026, 1057, 1058, 1082, 991, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1016, 1029, 1030, 1071, 974, 1046, 1079, 1053, 998, 1029, 1079, 980, 974, 1016, 1027, 991, 1033, 1019, 1087, 1014, 1023, 1046, 1058, 1009, 984, 1033, 1020, 1021, 992, 967, 1039, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1028, 1007, 1041, 1033, 1029, 1006, 1057, 1028, 1023, 1048, 996, 1016, 987, 982, 1052, 1001, 996, 1065, 1002, 1017, 1034, 988, 1082, 1022, 1027, 1008, 1045, 1017, 1028, 1029, 1049, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1053, 1027, 937, 1018, 1003, 1012, 1041, 1056, 1037, 1071, 1013, 1019, 1081, 1000, 1016, 1005, 1048, 1053, 1011, 1028, 982, 992, 1077, 1024, 1063, 1017, 983, 952, 1036, 1039, 1043, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1029, 1022, 1010, 1017, 1011, 1067, 990, 1036, 986, 1031, 1041, 995, 969, 1051, 1006, 1038, 943, 1003, 1007, 1048, 1039, 1066, 1028, 997, 1066, 1011, 1044, 1037, 1050, 1047, 1051, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1057, 1019, 1023, 1019, 1020, 1020, 1034, 975, 1033, 997, 1056, 1020, 1071, 1054, 977, 1081, 1043, 1023, 978, 1021, 959, 1048, 1021, 1067, 1015, 1037, 1018, 999, 1029, 1071, 997, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1037, 1059, 994, 1023, 1011, 1024, 1039, 1007, 1017, 994, 1023, 1002, 1049, 969, 1055, 1071, 1015, 1002, 1083, 1041, 1047, 982, 1075, 973, 1055, 1011, 995, 1043, 1046, 1009, 978, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1020, 999, 1001, 1045, 981, 1004, 987, 1073, 1032, 1021, 1076, 1020, 1012, 1013, 1009, 1009, 1060, 983, 1061, 1027, 1018, 1044, 1033, 1049, 1006, 1005, 1016, 1000, 1056, 1060, 1018, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1028, 989, 1029, 1012, 3, 1, 0, 0, 1038, 1060, 947, 988, 8, 0, 0, 0, 1021, 1086, 1035, 987, 0, 0, 0, 0, 1004, 1046, 1046, 1031, 9, 0, 0, 0, 986, 1008, 979, 1011, 5, 3, 0, 0, 1020, 999, 1019, 1067, 0, 0, 0, 0, 1040, 1021, 1026, 1057, 12, 0, 0, 0, 1058, 1056, 991, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4041, 4129, 4136, 3992, 4105, 4156, 4147] avg=4096.0000 max=4156 max_rank=6 min=3992 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1016, 1029, 1030, 1071, 3, 1, 0, 0, 974, 1046, 1079, 1053, 9, 0, 0, 0, 998, 1029, 1078, 980, 0, 0, 0, 0, 974, 1016, 1027, 991, 8, 0, 0, 0, 1033, 1019, 1087, 1014, 4, 3, 0, 0, 1023, 1046, 1043, 1009, 0, 0, 0, 0, 984, 1033, 1020, 1021, 11, 0, 0, 0, 992, 944, 1039, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4161, 4085, 4016, 4160, 4121, 4069, 4006] avg=4096.0000 max=4161 max_rank=1 min=4006 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1028, 1007, 1041, 1033, 3, 1, 0, 0, 1029, 1006, 1057, 1028, 9, 0, 0, 0, 1023, 1048, 995, 1016, 0, 0, 0, 0, 987, 982, 1052, 1001, 9, 0, 0, 0, 996, 1065, 1002, 1017, 5, 3, 0, 0, 1034, 988, 1067, 1022, 0, 0, 0, 0, 1027, 1008, 1045, 1017, 11, 0, 0, 0, 1028, 1004, 1049, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4129, 4082, 4031, 4088, 4111, 4108, 4106] avg=4096.0000 max=4129 max_rank=1 min=4031 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1053, 1027, 937, 1018, 3, 1, 0, 0, 1003, 1012, 1041, 1056, 9, 0, 0, 0, 1037, 1071, 1012, 1019, 0, 0, 0, 0, 1081, 1000, 1016, 1005, 9, 0, 0, 0, 1048, 1053, 1011, 1028, 5, 3, 0, 0, 982, 992, 1062, 1024, 0, 0, 0, 0, 1063, 1017, 983, 952, 12, 0, 0, 0, 1036, 1013, 1043, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4121, 4139, 4111, 4148, 4060, 4027, 4123] avg=4096.0000 max=4148 max_rank=4 min=4027 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1029, 1022, 1010, 1017, 3, 1, 0, 0, 1011, 1067, 990, 1036, 8, 0, 0, 0, 986, 1031, 1040, 995, 0, 0, 0, 0, 969, 1051, 1006, 1038, 9, 0, 0, 0, 943, 1003, 1007, 1048, 5, 3, 0, 0, 1039, 1066, 1014, 997, 0, 0, 0, 0, 1066, 1011, 1044, 1037, 12, 0, 0, 0, 1050, 1021, 1051, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4112, 4052, 4073, 4009, 4116, 4170, 4154] avg=4096.0000 max=4170 max_rank=6 min=4009 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1057, 1019, 1023, 1019, 3, 1, 0, 0, 1020, 1020, 1034, 975, 8, 0, 0, 0, 1033, 997, 1055, 1020, 0, 0, 0, 0, 1071, 1054, 977, 1081, 9, 0, 0, 0, 1043, 1023, 978, 1021, 5, 3, 0, 0, 959, 1048, 1007, 1067, 0, 0, 0, 0, 1015, 1037, 1018, 999, 12, 0, 0, 0, 1029, 1045, 997, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4057, 4105, 4192, 4073, 4081, 4081, 4057] avg=4096.0000 max=4192 max_rank=3 min=4057 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1020, 999, 1001, 1045, 3, 1, 0, 0, 981, 1004, 987, 1073, 8, 0, 0, 0, 1032, 1021, 1075, 1020, 0, 0, 0, 0, 1012, 1013, 1009, 1009, 9, 0, 0, 0, 1060, 983, 1061, 1027, 5, 3, 0, 0, 1018, 1044, 1019, 1049, 0, 0, 0, 0, 1006, 1005, 1016, 1000, 12, 0, 0, 0, 1056, 1034, 1018, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4053, 4148, 4052, 4139, 4130, 4039, 4138] avg=4096.0000 max=4148 max_rank=2 min=4039 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1037, 1059, 994, 1023, 10, 8, 0, 0, 1011, 1024, 1039, 1007, 13, 0, 0, 0, 1017, 994, 1015, 1002, 0, 0, 0, 0, 1049, 969, 1055, 1071, 13, 0, 0, 0, 1015, 1002, 1083, 1041, 8, 10, 0, 0, 1047, 982, 1042, 973, 0, 0, 0, 0, 1055, 1011, 995, 1043, 14, 0, 0, 0, 1046, 974, 978, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4094, 4028, 4157, 4159, 4044, 4118, 4037] avg=4096.0000 max=4159 max_rank=4 min=4028 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1015, 1022, 1041, 1014, 968, 1059, 1005, 1032, 1022, 1022, 1035, 1044, 997, 1010, 1024, 1038, 970, 1054, 1069, 1020, 1054, 1032, 1034, 1012, 1008, 1061, 1026, 982, 1064, 1041, 991, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1039, 1017, 1076, 1060, 992, 1018, 1013, 1036, 998, 1008, 1033, 1025, 1039, 969, 1011, 1039, 1078, 1054, 1045, 1029, 1002, 977, 1041, 1030, 1028, 1036, 1072, 1025, 1023, 960, 987, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1021, 1004, 1020, 993, 1037, 995, 984, 1029, 1032, 1012, 1023, 1098, 1012, 1033, 982, 1018, 957, 1063, 1037, 1030, 1033, 1040, 1058, 986, 1058, 1023, 1013, 996, 1072, 987, 1085, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1028, 942, 1021, 1064, 1019, 1022, 993, 1028, 1024, 1020, 1015, 989, 1039, 1043, 1059, 1021, 1024, 995, 1005, 1033, 1057, 1027, 1036, 1038, 1060, 1033, 1018, 1028, 1017, 1004, 1031, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1054, 975, 1050, 999, 1024, 1018, 1026, 993, 1043, 1043, 1061, 1041, 980, 994, 1026, 1080, 1018, 1049, 1025, 1046, 1070, 975, 1029, 1013, 1029, 1031, 1003, 1010, 1066, 1010, 1021, 966] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1007, 1025, 1035, 1010, 1039, 1054, 1047, 1021, 1003, 1022, 1077, 1072, 1044, 986, 1026, 1048, 1040, 1048, 1003, 943, 1024, 1045, 1039, 982, 999, 1029, 1025, 992, 991, 1040, 1004, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1028, 1003, 1013, 1037, 1053, 1007, 1046, 1001, 1001, 1071, 1043, 1029, 1017, 1030, 1013, 1030, 1015, 1009, 1021, 1033, 1041, 1039, 989, 991, 1027, 1019, 1026, 1045, 1048, 1005, 1049, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1044, 1030, 1068, 1093, 1043, 1028, 1092, 991, 995, 1036, 1012, 986, 1010, 1008, 1021, 1079, 983, 1013, 1007, 1029, 994, 955, 1040, 1005, 1006, 1039, 977, 1002, 1086, 1017, 1048, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1015, 1022, 1033, 1014, 0, 0, 0, 0, 968, 1059, 1005, 1032, 6, 0, 0, 0, 1022, 1022, 1017, 1044, 0, 0, 0, 0, 997, 1010, 1024, 1038, 3, 1, 0, 0, 970, 1054, 1069, 1020, 2, 0, 0, 0, 1054, 1032, 1034, 1012, 9, 0, 0, 0, 1008, 1061, 1026, 982, 5, 0, 0, 0, 1064, 1041, 991, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4070, 4105, 4073, 4115, 4141, 4082, 4098] avg=4096.0000 max=4141 max_rank=5 min=4070 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1028, 942, 1013, 1064, 0, 0, 0, 0, 1019, 1022, 993, 1028, 6, 0, 0, 0, 1024, 1020, 997, 989, 0, 0, 0, 0, 1039, 1043, 1059, 1021, 3, 1, 0, 0, 1024, 995, 1005, 1033, 2, 0, 0, 0, 1057, 1027, 1036, 1038, 9, 0, 0, 0, 1060, 1033, 1018, 1028, 5, 0, 0, 0, 1017, 1004, 1031, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4068, 4030, 4166, 4059, 4167, 4144, 4087] avg=4096.0000 max=4167 max_rank=5 min=4030 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1039, 1017, 1068, 1060, 0, 0, 0, 0, 992, 1018, 1013, 1036, 6, 0, 0, 0, 998, 1008, 1015, 1025, 0, 0, 0, 0, 1039, 969, 1011, 1039, 3, 1, 0, 0, 1078, 1054, 1045, 1029, 2, 0, 0, 0, 1002, 977, 1041, 1030, 9, 0, 0, 0, 1028, 1036, 1072, 1025, 5, 0, 0, 0, 1023, 960, 987, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4184, 4065, 4046, 4062, 4208, 4059, 4166, 3978] avg=4096.0000 max=4208 max_rank=4 min=3978 min_rank=7 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1021, 1004, 1012, 993, 0, 0, 0, 0, 1037, 995, 984, 1029, 6, 0, 0, 0, 1032, 1012, 1005, 1098, 0, 0, 0, 0, 1012, 1033, 982, 1018, 3, 1, 0, 0, 957, 1063, 1037, 1030, 2, 0, 0, 0, 1033, 1040, 1058, 986, 9, 0, 0, 0, 1058, 1023, 1013, 996, 5, 0, 0, 0, 1072, 987, 1085, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4051, 4147, 4049, 4089, 4126, 4095, 4181] avg=4096.0000 max=4181 max_rank=7 min=4030 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1054, 975, 1042, 999, 0, 0, 0, 0, 1024, 1018, 1026, 993, 7, 0, 0, 0, 1043, 1043, 1040, 1041, 0, 0, 0, 0, 980, 994, 1026, 1080, 4, 1, 0, 0, 1018, 1049, 1025, 1046, 2, 0, 0, 0, 1070, 975, 1029, 1013, 10, 0, 0, 0, 1029, 1031, 1003, 1010, 5, 0, 0, 0, 1066, 1010, 1021, 966, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4068, 4167, 4085, 4140, 4097, 4078, 4063] avg=4096.0000 max=4167 max_rank=2 min=4063 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1028, 1003, 1005, 1037, 0, 0, 0, 0, 1053, 1007, 1046, 1001, 6, 0, 0, 0, 1001, 1071, 1023, 1029, 0, 0, 0, 0, 1017, 1030, 1013, 1030, 4, 1, 0, 0, 1015, 1009, 1021, 1033, 2, 0, 0, 0, 1041, 1039, 989, 991, 10, 0, 0, 0, 1027, 1019, 1026, 1045, 5, 0, 0, 0, 1048, 1005, 1049, 989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4113, 4124, 4095, 4080, 4070, 4122, 4091] avg=4096.0000 max=4124 max_rank=2 min=4070 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1044, 1030, 1060, 1093, 0, 0, 0, 0, 1043, 1028, 1092, 991, 6, 0, 0, 0, 995, 1036, 994, 986, 0, 0, 0, 0, 1010, 1008, 1021, 1079, 3, 1, 0, 0, 983, 1013, 1007, 1029, 2, 0, 0, 0, 994, 955, 1040, 1005, 9, 0, 0, 0, 1006, 1039, 977, 1002, 5, 0, 0, 0, 1086, 1017, 1048, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4227, 4160, 4011, 4122, 4034, 4003, 4029, 4182] avg=4096.0000 max=4227 max_rank=0 min=4003 min_rank=5 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16248 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1007, 1025, 1011, 1010, 0, 0, 0, 0, 1039, 1054, 1047, 1021, 12, 0, 0, 0, 1003, 1022, 1041, 1072, 0, 0, 0, 0, 1044, 986, 1026, 1048, 9, 3, 0, 0, 1040, 1048, 1003, 943, 9, 0, 0, 0, 1024, 1045, 1039, 982, 15, 0, 0, 0, 999, 1029, 1025, 992, 7, 0, 0, 0, 991, 1040, 1004, 1048, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4173, 4138, 4116, 4043, 4105, 4052, 4088] avg=4096.0000 max=4173 max_rank=1 min=4043 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16288 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8064, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8448, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1039, 1045, 1093, 985, 1053, 994, 1076, 1027, 1012, 1042, 1054, 1078, 994, 1010, 1014, 1077, 991, 1041, 993, 1006, 1049, 1004, 1018, 1005, 1013, 1018, 1022, 1012, 1017, 984, 1039, 963] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1017, 1033, 1000, 995, 1000, 1047, 1053, 1035, 1033, 960, 1064, 1033, 1045, 1027, 1017, 1044, 1027, 1056, 993, 1031, 1045, 961, 1020, 1028, 1059, 1028, 1043, 1017, 1030, 987, 1010, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1008, 1004, 971, 991, 1012, 1073, 1049, 1077, 969, 1047, 1031, 997, 1048, 1025, 1029, 1021, 1057, 1047, 1003, 1013, 1058, 1016, 1041, 1029, 998, 1022, 1111, 1009, 1026, 981, 953, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1020, 1031, 1022, 1009, 1035, 990, 1090, 1054, 1049, 1066, 1030, 1012, 1013, 1043, 1008, 996, 993, 1033, 1002, 1007, 1041, 999, 1081, 1004, 952, 1037, 1040, 1036, 1061, 1001, 979, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[948, 1009, 1051, 1042, 1032, 1027, 1034, 1038, 1055, 1033, 1023, 1049, 1044, 1008, 1051, 1016, 1012, 1008, 1028, 990, 1013, 1004, 1036, 1011, 1050, 1041, 1049, 1020, 1028, 1009, 978, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1021, 1063, 986, 998, 1018, 1015, 1011, 1033, 1063, 1063, 1011, 1018, 1014, 1081, 1045, 1006, 1051, 1029, 995, 1028, 1004, 1054, 997, 1044, 993, 1037, 1076, 975, 997, 1020, 1021, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1010, 1024, 1007, 1051, 1017, 1028, 1039, 1044, 1021, 1011, 1052, 1008, 1012, 1025, 1021, 1026, 995, 988, 1029, 1056, 1023, 1050, 1057, 1019, 1004, 1048, 1016, 1020, 1029, 1047, 976, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1048, 982, 1028, 1013, 1036, 993, 1050, 1004, 977, 1021, 991, 996, 1042, 1030, 979, 1010, 1050, 1048, 1032, 1016, 1016, 1033, 1036, 1025, 1059, 1022, 982, 998, 1060, 1046, 1065, 1080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1017, 1033, 1000, 995, 28, 0, 0, 0, 1000, 1047, 1014, 1035, 0, 0, 0, 0, 1033, 960, 1051, 1033, 0, 0, 0, 0, 1045, 1021, 1017, 1044, 0, 0, 0, 0, 1027, 1056, 993, 1031, 6, 4, 3, 0, 1045, 961, 1014, 1028, 0, 0, 0, 0, 1059, 1028, 1039, 1017, 0, 0, 0, 0, 1030, 987, 1010, 1030, 13, 11, 3, 0] rerouted_tokens_per_ep_rank=[4073, 4096, 4077, 4127, 4120, 4048, 4143, 4084] avg=4096.0000 max=4143 max_rank=6 min=4048 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1039, 1045, 1093, 985, 28, 0, 0, 0, 1053, 994, 1037, 1027, 0, 0, 0, 0, 1012, 1042, 1042, 1078, 0, 0, 0, 0, 994, 1004, 1014, 1077, 0, 0, 0, 0, 991, 1041, 993, 1006, 6, 4, 3, 0, 1049, 1004, 1012, 1005, 0, 0, 0, 0, 1013, 1018, 1018, 1012, 0, 0, 0, 0, 1017, 984, 1039, 963, 12, 11, 3, 0] rerouted_tokens_per_ep_rank=[4190, 4111, 4174, 4089, 4044, 4070, 4061, 4029] avg=4096.0000 max=4190 max_rank=0 min=4029 min_rank=7 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16204 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1008, 1004, 971, 991, 27, 0, 0, 0, 1012, 1073, 1011, 1077, 0, 0, 0, 0, 969, 1047, 1019, 997, 0, 0, 0, 0, 1048, 1019, 1029, 1021, 0, 0, 0, 0, 1057, 1047, 1003, 1013, 6, 5, 3, 0, 1058, 1016, 1035, 1029, 0, 0, 0, 0, 998, 1022, 1106, 1009, 0, 0, 0, 0, 1026, 981, 953, 1052, 12, 11, 3, 0] rerouted_tokens_per_ep_rank=[4001, 4173, 4032, 4117, 4134, 4138, 4135, 4038] avg=4096.0000 max=4173 max_rank=1 min=4001 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[948, 1009, 1051, 1042, 27, 0, 0, 0, 1032, 1027, 996, 1038, 0, 0, 0, 0, 1055, 1033, 1011, 1049, 0, 0, 0, 0, 1044, 1002, 1051, 1016, 0, 0, 0, 0, 1012, 1008, 1028, 990, 6, 4, 3, 0, 1013, 1004, 1030, 1011, 0, 0, 0, 0, 1050, 1041, 1045, 1020, 0, 0, 0, 0, 1028, 1009, 978, 1031, 12, 11, 3, 0] rerouted_tokens_per_ep_rank=[4077, 4093, 4148, 4113, 4051, 4058, 4156, 4072] avg=4096.0000 max=4156 max_rank=6 min=4051 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1020, 1031, 1022, 1009, 29, 0, 0, 0, 1035, 990, 1050, 1054, 0, 0, 0, 0, 1049, 1066, 1018, 1012, 0, 0, 0, 0, 1013, 1037, 1008, 996, 0, 0, 0, 0, 993, 1033, 1002, 1007, 6, 4, 3, 0, 1041, 999, 1075, 1004, 0, 0, 0, 0, 952, 1037, 1036, 1036, 0, 0, 0, 0, 1061, 1001, 979, 1034, 12, 11, 3, 0] rerouted_tokens_per_ep_rank=[4111, 4129, 4145, 4054, 4048, 4119, 4061, 4101] avg=4096.0000 max=4145 max_rank=2 min=4048 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1021, 1063, 986, 998, 31, 0, 0, 0, 1018, 1015, 965, 1033, 0, 0, 0, 0, 1063, 1063, 995, 1018, 0, 0, 0, 0, 1014, 1070, 1045, 1006, 0, 0, 0, 0, 1051, 1029, 995, 1028, 11, 10, 7, 0, 1004, 1054, 986, 1044, 0, 0, 0, 0, 993, 1037, 1066, 975, 0, 0, 0, 0, 997, 1020, 1021, 1001, 16, 15, 4, 0] rerouted_tokens_per_ep_rank=[4099, 4031, 4139, 4135, 4131, 4088, 4071, 4074] avg=4096.0000 max=4139 max_rank=2 min=4031 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1048, 982, 1028, 1013, 27, 0, 0, 0, 1036, 993, 1012, 1004, 0, 0, 0, 0, 977, 1021, 979, 996, 0, 0, 0, 0, 1042, 1024, 979, 1010, 0, 0, 0, 0, 1050, 1048, 1032, 1016, 6, 4, 3, 0, 1016, 1033, 1030, 1025, 0, 0, 0, 0, 1059, 1022, 978, 998, 0, 0, 0, 0, 1060, 1046, 1065, 1080, 12, 11, 3, 0] rerouted_tokens_per_ep_rank=[4098, 4045, 3973, 4055, 4159, 4104, 4057, 4277] avg=4096.0000 max=4277 max_rank=7 min=3973 min_rank=2 imbalance_ratio=1.044189 max_load_reduction=28491 max_load_reduction_pct=0.869476 moved_to_echo=16597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1010, 1024, 1007, 1051, 27, 0, 0, 0, 1017, 1028, 1001, 1044, 0, 0, 0, 0, 1021, 1011, 1040, 1008, 0, 0, 0, 0, 1012, 1019, 1021, 1026, 0, 0, 0, 0, 995, 988, 1029, 1056, 6, 4, 3, 0, 1023, 1050, 1051, 1019, 0, 0, 0, 0, 1004, 1048, 1012, 1020, 0, 0, 0, 0, 1029, 1047, 976, 1015, 12, 11, 3, 0] rerouted_tokens_per_ep_rank=[4119, 4090, 4080, 4078, 4081, 4143, 4084, 4093] avg=4096.0000 max=4143 max_rank=5 min=4078 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1007, 1003, 1044, 1051, 1043, 1037, 1038, 1033, 1059, 1027, 1053, 989, 982, 1021, 1003, 1037, 1034, 1036, 999, 1035, 1015, 1031, 1040, 1007, 1048, 1009, 1017, 992, 1024, 998, 1055, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[993, 975, 1047, 1071, 1027, 993, 1018, 1049, 1030, 1039, 1035, 1030, 1008, 996, 1012, 1044, 979, 997, 1039, 1042, 1055, 1032, 1036, 1006, 1030, 1035, 1023, 1032, 1007, 1079, 1007, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[981, 1047, 984, 1031, 992, 975, 1006, 1065, 1051, 1033, 1008, 991, 1034, 1047, 1012, 1036, 1068, 1042, 1036, 1072, 1024, 1075, 1034, 1046, 982, 953, 1024, 1021, 991, 1024, 1042, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1043, 1001, 988, 1019, 998, 1036, 1045, 974, 1055, 1043, 1032, 996, 1025, 1033, 1034, 1001, 994, 1015, 1078, 990, 1031, 1042, 1007, 1064, 1074, 1006, 1007, 1043, 1049, 996, 1029, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1032, 1071, 1017, 1033, 1067, 1017, 980, 1062, 1006, 1011, 1030, 1042, 1003, 998, 1022, 1033, 1039, 1005, 1002, 1050, 995, 1019, 1052, 1023, 1051, 1047, 1002, 1010, 1015, 1056, 987, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1020, 970, 1070, 1028, 1034, 1006, 1000, 1003, 1095, 992, 1028, 999, 1037, 968, 1086, 1021, 1058, 994, 1052, 1012, 1064, 1021, 1012, 1038, 1056, 1065, 1021, 1000, 987, 1015, 1023, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1017, 995, 1018, 1011, 1029, 1023, 1081, 983, 987, 976, 1023, 1022, 980, 1030, 998, 1094, 1050, 1050, 1076, 1043, 1023, 1032, 997, 1051, 1047, 1031, 1016, 1023, 982, 1051, 998, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1000, 1000, 1044, 1044, 1058, 977, 1042, 1047, 1010, 1008, 1011, 1040, 1016, 1065, 1039, 997, 983, 1027, 975, 1015, 1061, 1045, 1035, 1046, 1036, 1019, 1019, 1032, 997, 1069, 999, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1007, 1003, 1044, 1051, 11, 2, 0, 0, 1043, 1037, 1038, 1033, 3, 0, 0, 0, 1059, 1027, 1053, 989, 1, 0, 0, 0, 982, 1021, 1003, 1037, 7, 0, 0, 0, 1034, 1036, 999, 1022, 0, 0, 0, 0, 1015, 996, 1040, 1007, 0, 0, 0, 0, 1048, 1009, 1017, 992, 0, 0, 0, 0, 1024, 998, 1055, 1001, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4154, 4129, 4050, 4091, 4058, 4066, 4102] avg=4096.0000 max=4154 max_rank=1 min=4050 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1043, 1001, 988, 1019, 11, 2, 0, 0, 998, 1036, 1045, 974, 3, 0, 0, 0, 1055, 1043, 1032, 996, 1, 0, 0, 0, 1025, 1033, 1034, 1001, 6, 0, 0, 0, 994, 1015, 1078, 978, 0, 0, 0, 0, 1031, 1007, 1007, 1064, 0, 0, 0, 0, 1074, 1006, 1007, 1043, 0, 0, 0, 0, 1049, 996, 1029, 1020, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4056, 4127, 4099, 4065, 4109, 4130, 4118] avg=4096.0000 max=4130 max_rank=6 min=4056 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[981, 1047, 984, 1031, 12, 2, 0, 0, 992, 975, 1006, 1065, 3, 0, 0, 0, 1051, 1033, 1008, 991, 1, 0, 0, 0, 1034, 1047, 1012, 1036, 7, 0, 0, 0, 1068, 1042, 1036, 1059, 0, 0, 0, 0, 1024, 1038, 1034, 1046, 0, 0, 0, 0, 982, 953, 1024, 1021, 0, 0, 0, 0, 991, 1024, 1042, 1041, 25, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4041, 4084, 4136, 4205, 4142, 3980, 4123] avg=4096.0000 max=4205 max_rank=4 min=3980 min_rank=6 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[993, 975, 1047, 1071, 11, 2, 0, 0, 1027, 993, 1018, 1049, 3, 0, 0, 0, 1030, 1039, 1035, 1030, 1, 0, 0, 0, 1008, 996, 1012, 1044, 7, 0, 0, 0, 979, 997, 1039, 1029, 0, 0, 0, 0, 1055, 997, 1036, 1006, 0, 0, 0, 0, 1030, 1035, 1023, 1032, 0, 0, 0, 0, 1007, 1079, 1007, 1002, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4090, 4135, 4067, 4044, 4094, 4120, 4119] avg=4096.0000 max=4135 max_rank=2 min=4044 min_rank=4 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1020, 970, 1070, 1028, 11, 2, 0, 0, 1034, 1006, 1000, 1003, 3, 0, 0, 0, 1095, 992, 1028, 999, 1, 0, 0, 0, 1037, 968, 1086, 1021, 6, 0, 0, 0, 1058, 994, 1052, 1000, 0, 0, 0, 0, 1064, 986, 1012, 1038, 0, 0, 0, 0, 1056, 1065, 1021, 1000, 0, 0, 0, 0, 987, 1015, 1023, 993, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4046, 4115, 4118, 4104, 4100, 4142, 4042] avg=4096.0000 max=4142 max_rank=6 min=4042 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1017, 995, 1018, 1011, 11, 2, 0, 0, 1029, 1023, 1081, 983, 3, 0, 0, 0, 987, 976, 1023, 1022, 1, 0, 0, 0, 980, 1030, 998, 1094, 7, 0, 0, 0, 1050, 1050, 1076, 1030, 0, 0, 0, 0, 1023, 997, 997, 1051, 0, 0, 0, 0, 1047, 1031, 1016, 1023, 0, 0, 0, 0, 982, 1051, 998, 1031, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4119, 4009, 4109, 4206, 4068, 4117, 4086] avg=4096.0000 max=4206 max_rank=4 min=4009 min_rank=2 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16477 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1032, 1071, 1017, 1033, 16, 5, 0, 0, 1067, 1017, 980, 1062, 9, 0, 0, 0, 1006, 1011, 1030, 1042, 7, 3, 0, 0, 1003, 998, 1022, 1033, 10, 0, 0, 0, 1039, 1005, 1002, 1019, 0, 0, 0, 0, 995, 975, 1052, 1023, 0, 0, 0, 0, 1048, 1047, 1002, 1010, 0, 0, 0, 0, 1015, 1056, 987, 991, 28, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4135, 4099, 4066, 4065, 4045, 4107, 4077] avg=4096.0000 max=4174 max_rank=0 min=4045 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1000, 1000, 1044, 1044, 11, 2, 0, 0, 1058, 977, 1042, 1047, 3, 0, 0, 0, 1010, 1008, 1011, 1040, 1, 0, 0, 0, 1016, 1065, 1039, 997, 6, 0, 0, 0, 983, 1027, 975, 1003, 0, 0, 0, 0, 1061, 1010, 1035, 1046, 0, 0, 0, 0, 1036, 1019, 1019, 1032, 0, 0, 0, 0, 997, 1069, 999, 1012, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4127, 4070, 4123, 3988, 4152, 4106, 4101] avg=4096.0000 max=4152 max_rank=5 min=3988 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[998, 1045, 1021, 1035, 1022, 1008, 988, 992, 1075, 1009, 1023, 1018, 1033, 1024, 1060, 1044, 1000, 1034, 1047, 1000, 1027, 1039, 1008, 999, 1025, 1054, 1018, 1046, 1006, 998, 1041, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1009, 1043, 995, 1045, 1065, 984, 1021, 1021, 1002, 1033, 1034, 984, 1043, 973, 1022, 1016, 970, 1075, 986, 1034, 1025, 1000, 1009, 1056, 1083, 1049, 1020, 1062, 1041, 1030, 1004, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1009, 1040, 1016, 1001, 1055, 1047, 1013, 1017, 965, 1023, 1016, 1014, 1007, 1014, 1036, 1052, 936, 1029, 1017, 1044, 1057, 1058, 995, 1040, 1010, 1034, 1063, 1060, 992, 1008, 977, 1123] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1030, 1023, 982, 1024, 1050, 995, 1003, 1036, 1032, 998, 992, 1002, 985, 1038, 1014, 1007, 1011, 1052, 1032, 1054, 1037, 998, 1004, 1010, 993, 1053, 1044, 1089, 1096, 1059, 1026, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1074, 988, 976, 1071, 1041, 993, 1039, 1054, 1037, 1014, 1053, 1084, 1002, 1007, 1036, 1018, 1035, 990, 1042, 999, 1025, 1011, 1044, 1008, 1032, 1023, 1026, 1012, 1003, 984, 1024, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1027, 1000, 1084, 1038, 1077, 972, 1010, 1026, 1001, 1035, 1025, 1055, 1016, 1051, 1002, 992, 952, 1033, 1033, 1065, 1035, 1021, 1033, 1020, 1014, 994, 996, 1019, 1082, 979, 1052, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1043, 1043, 1033, 1022, 1017, 1027, 1007, 1030, 1028, 1005, 1091, 1002, 1006, 968, 992, 978, 1042, 1044, 1070, 1039, 1062, 1023, 1052, 1034, 946, 997, 1078, 1013, 1026, 1032, 981, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1014, 955, 983, 1026, 1033, 1014, 1025, 1008, 985, 1090, 1028, 997, 1029, 1023, 1046, 1004, 1022, 1008, 1037, 1061, 1018, 1036, 1044, 1011, 1030, 1050, 1038, 1032, 1033, 1013, 1063, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[998, 1045, 1021, 1035, 8, 0, 0, 0, 1022, 1008, 988, 992, 8, 0, 0, 0, 1075, 1009, 1023, 1018, 2, 0, 0, 0, 1033, 1024, 1060, 1044, 28, 0, 0, 0, 1000, 1034, 1047, 998, 0, 0, 0, 0, 1019, 1039, 1008, 999, 0, 0, 0, 0, 1025, 1054, 1018, 1018, 0, 0, 0, 0, 1006, 998, 1041, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4018, 4127, 4189, 4079, 4065, 4115, 4068] avg=4096.0000 max=4189 max_rank=3 min=4018 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1009, 1043, 995, 1045, 8, 0, 0, 0, 1065, 984, 1021, 1021, 8, 0, 0, 0, 1002, 1033, 1034, 984, 2, 0, 0, 0, 1043, 973, 1022, 1016, 29, 0, 0, 0, 970, 1075, 986, 1032, 0, 0, 0, 0, 1017, 1000, 1009, 1056, 0, 0, 0, 0, 1083, 1049, 1020, 1033, 0, 0, 0, 0, 1041, 1030, 1004, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4099, 4055, 4083, 4063, 4082, 4185, 4101] avg=4096.0000 max=4185 max_rank=6 min=4055 min_rank=2 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1074, 988, 976, 1071, 8, 0, 0, 0, 1041, 993, 1039, 1054, 8, 0, 0, 0, 1037, 1014, 1053, 1084, 2, 0, 0, 0, 1002, 1007, 1036, 1018, 27, 0, 0, 0, 1035, 990, 1042, 997, 0, 0, 0, 0, 1017, 1011, 1044, 1008, 0, 0, 0, 0, 1032, 1023, 1026, 985, 0, 0, 0, 0, 1003, 984, 1024, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4135, 4190, 4090, 4064, 4080, 4066, 4026] avg=4096.0000 max=4190 max_rank=2 min=4026 min_rank=7 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16236 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1030, 1023, 982, 1024, 8, 0, 0, 0, 1050, 995, 1003, 1036, 8, 0, 0, 0, 1032, 998, 992, 1002, 2, 0, 0, 0, 985, 1038, 1014, 1007, 30, 0, 0, 0, 1011, 1052, 1032, 1052, 0, 0, 0, 0, 1029, 998, 1004, 1010, 0, 0, 0, 0, 993, 1053, 1044, 1059, 0, 0, 0, 0, 1096, 1059, 1026, 991, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4092, 4026, 4074, 4147, 4041, 4149, 4172] avg=4096.0000 max=4172 max_rank=7 min=4026 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1009, 1040, 1016, 1001, 9, 0, 0, 0, 1055, 1047, 1013, 1017, 9, 0, 0, 0, 965, 1023, 1016, 1014, 2, 0, 0, 0, 1007, 1014, 1036, 1052, 29, 0, 0, 0, 936, 1029, 1017, 1042, 0, 0, 0, 0, 1048, 1058, 995, 1040, 0, 0, 0, 0, 1010, 1034, 1063, 1031, 0, 0, 0, 0, 992, 1008, 977, 1114, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4141, 4020, 4138, 4024, 4141, 4138, 4091] avg=4096.0000 max=4141 max_rank=1 min=4020 min_rank=2 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1027, 1000, 1084, 1038, 11, 7, 0, 0, 1077, 972, 1010, 1026, 13, 5, 2, 0, 1001, 1035, 1025, 1055, 4, 0, 0, 0, 1016, 1051, 1002, 992, 32, 0, 0, 0, 952, 1033, 1033, 1054, 0, 0, 0, 0, 1022, 1021, 1033, 1020, 0, 0, 0, 0, 1014, 994, 996, 982, 0, 0, 0, 0, 1082, 979, 1052, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4105, 4120, 4093, 4072, 4096, 3986, 4129] avg=4096.0000 max=4167 max_rank=0 min=3986 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1043, 1043, 1033, 1022, 8, 0, 0, 0, 1017, 1027, 1007, 1030, 9, 0, 0, 0, 1028, 1005, 1091, 1002, 2, 0, 0, 0, 1006, 968, 992, 978, 27, 0, 0, 0, 1042, 1044, 1070, 1037, 0, 0, 0, 0, 1053, 1023, 1052, 1034, 0, 0, 0, 0, 946, 997, 1078, 986, 0, 0, 0, 0, 1026, 1032, 981, 1029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4090, 4128, 3971, 4193, 4162, 4007, 4068] avg=4096.0000 max=4193 max_rank=4 min=3971 min_rank=3 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1014, 955, 983, 1026, 8, 0, 0, 0, 1033, 1014, 1025, 1008, 8, 0, 0, 0, 985, 1090, 1028, 997, 2, 0, 0, 0, 1029, 1023, 1046, 1004, 28, 0, 0, 0, 1022, 1008, 1037, 1059, 0, 0, 0, 0, 1010, 1036, 1044, 1011, 0, 0, 0, 0, 1030, 1050, 1038, 1004, 0, 0, 0, 0, 1033, 1013, 1063, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3986, 4088, 4102, 4130, 4126, 4101, 4122, 4113] avg=4096.0000 max=4130 max_rank=3 min=3986 min_rank=0 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8448, 8064, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1060, 1019, 1003, 1079, 1041, 992, 1032, 1026, 965, 1017, 1032, 1038, 1119, 1024, 1003, 1032, 1063, 1054, 989, 1008, 976, 987, 1019, 976, 1023, 1043, 1035, 1003, 1012, 1018, 1032, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[976, 979, 1017, 1023, 1019, 1068, 1020, 1004, 998, 1053, 1018, 1044, 1034, 998, 1086, 1002, 1024, 1023, 1030, 1089, 1020, 1017, 1071, 1051, 1023, 1029, 996, 999, 1008, 993, 1041, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1061, 1017, 988, 1031, 1001, 1038, 945, 1038, 1026, 1004, 1002, 1014, 1054, 1016, 1023, 997, 1064, 1016, 1056, 1062, 990, 994, 1013, 1095, 1015, 1073, 1019, 1005, 1021, 1029, 1031, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[993, 1004, 996, 1023, 1028, 995, 1027, 1050, 985, 1023, 1048, 1024, 963, 1017, 1092, 1067, 1017, 1075, 1010, 1025, 1027, 1034, 1007, 1024, 1021, 1045, 982, 997, 1033, 1035, 1057, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1030, 1001, 1048, 1055, 1035, 1034, 1024, 1035, 1007, 1008, 1040, 992, 996, 1032, 1035, 1042, 1052, 1019, 1031, 1045, 959, 1039, 1048, 1001, 1037, 1026, 1032, 1048, 1042, 979, 992, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[978, 1061, 1047, 1014, 990, 996, 1012, 978, 1041, 996, 972, 1093, 1105, 1042, 980, 1031, 1026, 1047, 1085, 1033, 1055, 1020, 1033, 1005, 980, 1028, 1026, 1008, 1006, 1035, 1025, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1017, 1013, 1003, 1059, 1067, 1044, 1047, 1038, 1016, 1034, 1016, 1067, 992, 1009, 1031, 1025, 1048, 1033, 1042, 997, 986, 1019, 1046, 993, 997, 1050, 1007, 1024, 1016, 1001, 1008, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1021, 979, 1025, 1022, 1030, 1064, 981, 1013, 1022, 1025, 1037, 997, 1014, 1012, 1024, 1002, 1015, 993, 1059, 919, 1066, 1060, 983, 1084, 1061, 1041, 1017, 1095, 1034, 1039, 1030, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1060, 1019, 1003, 1079, 16, 0, 0, 0, 1041, 992, 1032, 1026, 5, 2, 0, 0, 965, 1017, 1032, 1038, 14, 0, 0, 0, 1103, 1024, 1003, 1032, 0, 0, 0, 0, 1028, 1054, 989, 1008, 0, 0, 0, 0, 976, 987, 1019, 976, 5, 3, 0, 0, 1023, 1041, 1035, 1003, 0, 0, 0, 0, 1012, 1018, 1032, 1048, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4177, 4098, 4066, 4162, 4079, 3966, 4102, 4118] avg=4096.0000 max=4177 max_rank=0 min=3966 min_rank=5 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[976, 979, 1017, 1023, 15, 0, 0, 0, 1019, 1068, 1020, 1004, 4, 2, 0, 0, 998, 1053, 1018, 1044, 14, 0, 0, 0, 1020, 998, 1086, 1002, 0, 0, 0, 0, 990, 1023, 1030, 1089, 0, 0, 0, 0, 1020, 1017, 1071, 1051, 5, 3, 0, 0, 1023, 1027, 996, 999, 0, 0, 0, 0, 1008, 993, 1041, 1015, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4117, 4127, 4106, 4132, 4167, 4045, 4064] avg=4096.0000 max=4167 max_rank=5 min=4010 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1061, 1017, 988, 1031, 16, 0, 0, 0, 1001, 1038, 945, 1038, 4, 2, 0, 0, 1026, 1004, 1002, 1014, 14, 0, 0, 0, 1039, 1016, 1023, 997, 0, 0, 0, 0, 1029, 1016, 1056, 1062, 0, 0, 0, 0, 990, 994, 1013, 1095, 5, 3, 0, 0, 1015, 1071, 1019, 1005, 0, 0, 0, 0, 1021, 1029, 1031, 1030, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4028, 4060, 4075, 4163, 4100, 4110, 4119] avg=4096.0000 max=4163 max_rank=4 min=4028 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[993, 1004, 996, 1023, 15, 0, 0, 0, 1028, 995, 1027, 1050, 4, 2, 0, 0, 985, 1023, 1048, 1024, 13, 0, 0, 0, 949, 1017, 1092, 1067, 0, 0, 0, 0, 984, 1075, 1010, 1025, 0, 0, 0, 0, 1027, 1034, 1007, 1024, 5, 3, 0, 0, 1021, 1043, 982, 997, 0, 0, 0, 0, 1033, 1035, 1057, 1044, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4106, 4093, 4125, 4094, 4100, 4043, 4176] avg=4096.0000 max=4176 max_rank=7 min=4031 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[978, 1061, 1047, 1014, 15, 0, 0, 0, 990, 996, 1012, 978, 5, 2, 0, 0, 1041, 996, 972, 1093, 14, 0, 0, 0, 1089, 1042, 980, 1031, 0, 0, 0, 0, 992, 1047, 1085, 1033, 0, 0, 0, 0, 1055, 1020, 1033, 1005, 5, 3, 0, 0, 980, 1026, 1026, 1008, 0, 0, 0, 0, 1006, 1035, 1025, 1020, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 3983, 4116, 4142, 4157, 4121, 4040, 4094] avg=4096.0000 max=4157 max_rank=4 min=3983 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1030, 1001, 1048, 1055, 19, 0, 0, 0, 1035, 1034, 1024, 1035, 9, 3, 0, 0, 1007, 1008, 1040, 992, 18, 0, 0, 0, 968, 1032, 1035, 1042, 0, 0, 0, 0, 1009, 1019, 1031, 1045, 0, 0, 0, 0, 959, 1039, 1048, 1001, 6, 8, 0, 0, 1037, 1023, 1032, 1048, 0, 0, 0, 0, 1042, 979, 992, 1004, 11, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4140, 4065, 4077, 4104, 4061, 4140, 4028] avg=4096.0000 max=4153 max_rank=0 min=4028 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1021, 979, 1025, 1022, 15, 0, 0, 0, 1030, 1064, 981, 1013, 4, 2, 0, 0, 1022, 1025, 1037, 997, 13, 0, 0, 0, 1000, 1012, 1024, 1002, 0, 0, 0, 0, 982, 993, 1059, 919, 0, 0, 0, 0, 1066, 1060, 983, 1084, 5, 3, 0, 0, 1061, 1039, 1017, 1095, 0, 0, 0, 0, 1034, 1039, 1030, 1004, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4094, 4094, 4038, 3953, 4201, 4212, 4114] avg=4096.0000 max=4212 max_rank=6 min=3953 min_rank=4 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1017, 1013, 1003, 1059, 15, 0, 0, 0, 1067, 1044, 1047, 1038, 4, 2, 0, 0, 1016, 1034, 1016, 1067, 14, 0, 0, 0, 978, 1009, 1031, 1025, 0, 0, 0, 0, 1014, 1033, 1042, 997, 0, 0, 0, 0, 986, 1019, 1046, 993, 5, 3, 0, 0, 997, 1048, 1007, 1024, 0, 0, 0, 0, 1016, 1001, 1008, 1023, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4202, 4147, 4043, 4086, 4052, 4076, 4055] avg=4096.0000 max=4202 max_rank=1 min=4043 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16269 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1046, 1001, 1033, 1045, 1026, 1007, 1012, 1036, 1022, 1029, 998, 1001, 1022, 1064, 1044, 1034, 1053, 1017, 1038, 1013, 979, 1029, 1030, 1006, 1095, 957, 1022, 1033, 983, 1027, 1019, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1031, 1050, 1072, 1013, 1042, 1017, 1043, 1001, 1042, 966, 1018, 999, 930, 1028, 1052, 1047, 981, 1021, 1037, 1016, 1078, 1054, 1029, 991, 1006, 1001, 984, 1102, 1031, 1017, 1030, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1007, 1038, 1100, 1016, 1083, 1048, 1012, 1013, 1011, 1036, 1010, 1048, 1043, 992, 990, 1003, 1019, 1038, 1003, 1005, 988, 1026, 1032, 1019, 1023, 1048, 1020, 1024, 1052, 1003, 989, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1040, 1106, 1013, 967, 1020, 998, 1006, 1000, 1028, 1045, 1020, 1065, 1025, 1051, 1001, 1009, 1013, 1036, 1103, 1041, 1076, 1034, 1003, 1027, 1021, 1008, 980, 1002, 1007, 1030, 972, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1018, 1047, 1038, 1044, 1026, 965, 1022, 1025, 1003, 1022, 1027, 1044, 1117, 986, 1025, 1052, 1026, 984, 980, 989, 1013, 1047, 1023, 1024, 1041, 1035, 1055, 995, 1007, 1020, 1049, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[988, 1026, 1012, 1035, 1070, 1031, 985, 1068, 1012, 1025, 974, 1023, 1059, 1015, 1050, 1015, 1030, 1012, 1052, 998, 1051, 1047, 1011, 1033, 1003, 1016, 1022, 967, 1060, 1028, 1015, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1025, 1009, 982, 1038, 1087, 1072, 979, 1051, 1044, 1039, 990, 1009, 1019, 1014, 996, 992, 1019, 1006, 1065, 1029, 1044, 1063, 1025, 1020, 1049, 1048, 1020, 1001, 973, 1001, 1039, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[997, 1049, 1016, 1028, 1024, 1017, 1033, 1026, 1033, 1047, 1022, 1043, 1027, 1012, 1026, 1087, 1035, 1047, 978, 1023, 966, 1025, 983, 1060, 1017, 1036, 1038, 982, 1039, 1007, 1043, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1046, 982, 1033, 1045, 0, 0, 0, 0, 1018, 1007, 1012, 1036, 0, 0, 0, 0, 1022, 1029, 998, 1001, 8, 0, 0, 0, 1015, 1064, 1044, 1034, 0, 0, 0, 0, 1053, 1017, 1038, 1013, 7, 0, 0, 0, 979, 1021, 1030, 1006, 0, 0, 0, 0, 1095, 957, 1022, 1033, 14, 0, 0, 0, 983, 1027, 1019, 1047, 8, 5, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4073, 4058, 4157, 4128, 4036, 4121, 4089] avg=4096.0000 max=4157 max_rank=3 min=4036 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1007, 1019, 1100, 1016, 0, 0, 0, 0, 1074, 1048, 1012, 1013, 0, 0, 0, 0, 1011, 1036, 1010, 1048, 8, 0, 0, 0, 1036, 992, 990, 1003, 0, 0, 0, 0, 1019, 1038, 1003, 1005, 7, 0, 0, 0, 988, 1018, 1032, 1019, 0, 0, 0, 0, 1023, 1048, 1020, 1024, 14, 0, 0, 0, 1052, 1003, 989, 1029, 9, 5, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4147, 4113, 4021, 4072, 4057, 4129, 4087] avg=4096.0000 max=4147 max_rank=1 min=4021 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1040, 1086, 1013, 967, 0, 0, 0, 0, 1012, 998, 1006, 1000, 0, 0, 0, 0, 1028, 1045, 1020, 1065, 8, 0, 0, 0, 1018, 1051, 1001, 1009, 0, 0, 0, 0, 1013, 1036, 1103, 1041, 7, 0, 0, 0, 1076, 1026, 1003, 1027, 0, 0, 0, 0, 1021, 1008, 980, 1002, 15, 0, 0, 0, 1007, 1030, 972, 1021, 8, 5, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4016, 4166, 4079, 4200, 4132, 4026, 4043] avg=4096.0000 max=4200 max_rank=4 min=4016 min_rank=1 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1031, 1031, 1072, 1013, 0, 0, 0, 0, 1034, 1017, 1043, 1001, 0, 0, 0, 0, 1042, 966, 1018, 999, 8, 0, 0, 0, 924, 1028, 1052, 1047, 0, 0, 0, 0, 981, 1021, 1037, 1016, 6, 0, 0, 0, 1078, 1046, 1029, 991, 0, 0, 0, 0, 1006, 1001, 984, 1102, 14, 0, 0, 0, 1031, 1017, 1030, 1039, 8, 5, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4095, 4033, 4051, 4061, 4144, 4107, 4130] avg=4096.0000 max=4147 max_rank=0 min=4033 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1018, 1028, 1038, 1044, 0, 0, 0, 0, 1018, 965, 1022, 1025, 0, 0, 0, 0, 1003, 1022, 1027, 1044, 8, 0, 0, 0, 1110, 986, 1025, 1052, 0, 0, 0, 0, 1026, 984, 980, 989, 7, 0, 0, 0, 1013, 1039, 1023, 1024, 0, 0, 0, 0, 1041, 1035, 1055, 995, 14, 0, 0, 0, 1007, 1020, 1049, 1019, 8, 5, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4030, 4104, 4173, 3986, 4099, 4140, 4108] avg=4096.0000 max=4173 max_rank=3 min=3986 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 990, 982, 1038, 0, 0, 0, 0, 1078, 1072, 979, 1051, 0, 0, 0, 0, 1044, 1039, 990, 1009, 8, 0, 0, 0, 1012, 1014, 996, 992, 0, 0, 0, 0, 1019, 1006, 1065, 1029, 7, 0, 0, 0, 1044, 1055, 1025, 1020, 0, 0, 0, 0, 1049, 1048, 1020, 1001, 14, 0, 0, 0, 973, 1001, 1039, 1020, 9, 5, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4180, 4090, 4014, 4126, 4144, 4132, 4047] avg=4096.0000 max=4180 max_rank=1 min=4014 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[988, 998, 1012, 1035, 0, 0, 0, 0, 1051, 1031, 985, 1068, 0, 0, 0, 0, 1012, 1025, 974, 1023, 10, 7, 0, 0, 1048, 1015, 1050, 1015, 0, 0, 0, 0, 1030, 1012, 1052, 998, 11, 2, 0, 0, 1051, 1035, 1011, 1033, 0, 0, 0, 0, 1003, 1016, 1022, 967, 18, 0, 0, 0, 1060, 1028, 1015, 1035, 12, 10, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4135, 4051, 4128, 4105, 4130, 4026, 4160] avg=4096.0000 max=4160 max_rank=7 min=4026 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[997, 1030, 1016, 1028, 0, 0, 0, 0, 1016, 1017, 1033, 1026, 0, 0, 0, 0, 1033, 1047, 1022, 1043, 8, 0, 0, 0, 1020, 1012, 1026, 1087, 0, 0, 0, 0, 1035, 1047, 978, 1023, 7, 0, 0, 0, 966, 1017, 983, 1060, 0, 0, 0, 0, 1017, 1036, 1038, 982, 14, 0, 0, 0, 1039, 1007, 1043, 1002, 8, 5, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4092, 4153, 4145, 4090, 4026, 4087, 4104] avg=4096.0000 max=4153 max_rank=2 min=4026 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1121, 1023, 1020, 1041, 1047, 997, 956, 1011, 1058, 1007, 997, 1020, 1091, 993, 1045, 997, 1069, 1008, 1057, 1035, 985, 962, 984, 1055, 1020, 1006, 1008, 1028, 1036, 1076, 1019, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1074, 1022, 1026, 1008, 1030, 996, 1008, 1020, 1091, 1053, 1030, 985, 978, 1019, 1017, 1054, 988, 1056, 1045, 1000, 1026, 1059, 1004, 998, 1019, 1018, 1042, 1003, 1032, 1012, 1023, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1047, 983, 1029, 1028, 1039, 1002, 1004, 1001, 1013, 1054, 1036, 1037, 1035, 974, 978, 1058, 1036, 1021, 1017, 1063, 1010, 1041, 1104, 1018, 990, 1007, 1024, 1047, 984, 1037, 994, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1021, 1039, 1026, 1050, 1040, 1026, 966, 1059, 977, 994, 1038, 1035, 1002, 1025, 1071, 1069, 1027, 999, 1025, 1021, 1039, 1029, 1077, 1053, 1044, 976, 1037, 991, 1004, 1019, 992, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1058, 1003, 991, 1044, 1025, 1006, 1073, 991, 1063, 1017, 1068, 1018, 1028, 1023, 1007, 1013, 1004, 1051, 1064, 1007, 1024, 992, 1030, 1010, 1016, 1020, 984, 1043, 1026, 1013, 1050, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[986, 1031, 1042, 1011, 1022, 1074, 1045, 981, 1046, 1024, 1038, 1051, 1020, 1019, 976, 1105, 990, 1053, 1020, 996, 1075, 1043, 985, 1005, 1007, 963, 1057, 980, 1020, 1028, 1015, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[980, 1031, 995, 1047, 1056, 981, 1035, 1036, 996, 1039, 1081, 964, 1025, 1002, 1040, 1028, 1029, 998, 1032, 1065, 978, 1044, 1031, 1039, 1063, 988, 1034, 1036, 1027, 1016, 1010, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1058, 1022, 1026, 1008, 0, 0, 0, 0, 1030, 996, 1008, 1020, 1, 0, 0, 0, 1091, 1053, 1015, 985, 0, 0, 0, 0, 978, 1019, 1017, 1054, 3, 0, 0, 0, 988, 1056, 1036, 1000, 0, 0, 0, 0, 1026, 1059, 1004, 998, 1, 0, 0, 0, 1019, 1018, 1042, 1003, 16, 9, 0, 0, 1032, 1012, 1023, 1032, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4055, 4144, 4071, 4080, 4088, 4107, 4109] avg=4096.0000 max=4144 max_rank=2 min=4055 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1104, 1023, 1020, 1041, 0, 0, 0, 0, 1047, 997, 956, 1011, 1, 0, 0, 0, 1058, 1007, 983, 1020, 0, 0, 0, 0, 1091, 993, 1045, 997, 3, 0, 0, 0, 1069, 1008, 1048, 1035, 0, 0, 0, 0, 985, 962, 984, 1055, 1, 0, 0, 0, 1020, 1006, 1008, 1028, 17, 9, 0, 0, 1036, 1076, 1019, 996, 5, 4, 0, 0] rerouted_tokens_per_ep_rank=[4188, 4012, 4068, 4129, 4160, 3987, 4088, 4136] avg=4096.0000 max=4188 max_rank=0 min=3987 min_rank=5 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1031, 983, 1029, 1028, 0, 0, 0, 0, 1039, 1002, 1004, 1001, 1, 0, 0, 0, 1013, 1054, 1021, 1037, 0, 0, 0, 0, 1035, 974, 978, 1058, 3, 0, 0, 0, 1036, 1021, 1008, 1063, 0, 0, 0, 0, 1010, 1041, 1104, 1018, 1, 0, 0, 0, 990, 1007, 1024, 1047, 16, 9, 0, 0, 984, 1037, 994, 1057, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4047, 4125, 4048, 4128, 4174, 4093, 4082] avg=4096.0000 max=4174 max_rank=5 min=4047 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16477 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1006, 1039, 1026, 1050, 0, 0, 0, 0, 1040, 1026, 966, 1059, 1, 0, 0, 0, 977, 994, 1023, 1035, 0, 0, 0, 0, 1002, 1025, 1071, 1069, 3, 0, 0, 0, 1027, 999, 1016, 1021, 0, 0, 0, 0, 1039, 1029, 1077, 1053, 1, 0, 0, 0, 1044, 976, 1037, 991, 15, 9, 0, 0, 1004, 1019, 992, 997, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4092, 4029, 4170, 4063, 4199, 4072, 4022] avg=4096.0000 max=4199 max_rank=5 min=4022 min_rank=7 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[971, 1031, 1042, 1011, 0, 0, 0, 0, 1022, 1074, 1045, 981, 1, 0, 0, 0, 1046, 1024, 1023, 1051, 0, 0, 0, 0, 1020, 1019, 976, 1105, 3, 0, 0, 0, 990, 1053, 1011, 996, 0, 0, 0, 0, 1075, 1043, 985, 1005, 1, 0, 0, 0, 1007, 963, 1057, 980, 15, 9, 0, 0, 1020, 1028, 1015, 1060, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4123, 4144, 4123, 4050, 4109, 4031, 4133] avg=4096.0000 max=4144 max_rank=2 min=4031 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1039, 1003, 991, 1044, 0, 0, 0, 0, 1025, 1006, 1073, 991, 2, 0, 0, 0, 1063, 1017, 1044, 1018, 0, 0, 0, 0, 1028, 1023, 1007, 1013, 5, 0, 0, 0, 1004, 1051, 1043, 1007, 0, 0, 0, 0, 1024, 992, 1030, 1010, 3, 0, 0, 0, 1016, 1020, 984, 1043, 19, 16, 0, 0, 1026, 1013, 1050, 1006, 8, 11, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4097, 4142, 4076, 4105, 4059, 4098, 4114] avg=4096.0000 max=4142 max_rank=2 min=4059 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[965, 1031, 995, 1047, 0, 0, 0, 0, 1056, 981, 1035, 1036, 1, 0, 0, 0, 996, 1039, 1065, 964, 0, 0, 0, 0, 1025, 1002, 1040, 1028, 3, 0, 0, 0, 1029, 998, 1023, 1065, 0, 0, 0, 0, 978, 1044, 1031, 1039, 1, 0, 0, 0, 1063, 988, 1034, 1036, 15, 10, 0, 0, 1027, 1016, 1010, 1042, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4109, 4064, 4098, 4115, 4093, 4146, 4105] avg=4096.0000 max=4146 max_rank=6 min=4038 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1089, 1051, 978, 1002, 1060, 1052, 1082, 1038, 1024, 1058, 1019, 966, 1023, 1017, 991, 1019, 1023, 1021, 1000, 1032, 1035, 1021, 1013, 989, 1051, 1063, 1029, 965, 1022, 984, 1041, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1073, 1051, 978, 1002, 0, 0, 0, 0, 1060, 1052, 1082, 1038, 1, 0, 0, 0, 1024, 1058, 1004, 966, 0, 0, 0, 0, 1023, 1017, 991, 1019, 3, 0, 0, 0, 1023, 1021, 991, 1032, 0, 0, 0, 0, 1035, 1021, 1013, 989, 1, 0, 0, 0, 1051, 1063, 1029, 965, 16, 9, 0, 0, 1022, 984, 1041, 1010, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4233, 4052, 4053, 4067, 4059, 4133, 4067] avg=4096.0000 max=4233 max_rank=1 min=4052 min_rank=2 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8448, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1032, 990, 1057, 961, 997, 975, 1037, 961, 1027, 989, 999, 1052, 1040, 970, 1004, 1048, 999, 1024, 1076, 1036, 1031, 1061, 1128, 958, 1046, 1044, 971, 1025, 1046, 1068, 1059, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1022, 1038, 1063, 1060, 1058, 960, 1058, 1012, 1032, 1044, 999, 1008, 1044, 996, 992, 1016, 960, 1022, 1048, 1012, 1003, 976, 1031, 1021, 1036, 950, 1027, 1059, 1069, 1031, 1039, 1082] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1012, 961, 1079, 1038, 1014, 1025, 1038, 1010, 1066, 984, 1041, 962, 1037, 1002, 1018, 1014, 1038, 1003, 1070, 1009, 963, 1051, 1020, 1030, 1048, 999, 1032, 1007, 1089, 1063, 1025, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1012, 1049, 1053, 1003, 1015, 985, 1015, 1020, 1040, 1046, 1045, 1001, 1000, 1011, 996, 1018, 1029, 1066, 1048, 1074, 1005, 1034, 982, 986, 1028, 1027, 1050, 1021, 1026, 996, 1001, 1086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1046, 994, 1015, 1044, 1066, 1053, 1028, 1053, 1057, 1035, 1041, 1006, 1025, 988, 1049, 965, 1009, 1042, 1032, 983, 1037, 1037, 1026, 1052, 1018, 989, 949, 1050, 1020, 1030, 1007, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1010, 1073, 1052, 1016, 1023, 1007, 1016, 1002, 1058, 1062, 1057, 998, 1027, 973, 1080, 1042, 1005, 1024, 993, 958, 1015, 1030, 1027, 1017, 1064, 1052, 1023, 1059, 962, 1001, 1047, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1010, 1053, 1019, 1065, 983, 1046, 1036, 1001, 1014, 999, 1046, 1042, 949, 1029, 1025, 1033, 1002, 1027, 1040, 1049, 1036, 993, 1032, 999, 1026, 1049, 1019, 1026, 1014, 1052, 1022, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1037, 1057, 1028, 1007, 988, 988, 1033, 1044, 1036, 1011, 993, 1022, 1017, 1047, 1042, 991, 1003, 989, 1004, 1041, 1006, 1001, 1064, 1042, 1037, 1059, 989, 1029, 1018, 1074, 1051, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1032, 990, 1034, 961, 0, 0, 0, 0, 997, 975, 1037, 961, 18, 9, 0, 0, 1023, 989, 999, 1052, 0, 0, 0, 0, 1040, 970, 1004, 1048, 35, 0, 0, 0, 999, 1024, 1076, 1036, 5, 1, 0, 0, 1031, 1061, 1128, 958, 5, 3, 0, 0, 1041, 1044, 971, 1025, 0, 0, 0, 0, 1046, 1024, 1059, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 3997, 4063, 4097, 4141, 4186, 4081, 4186] avg=4096.0000 max=4186 max_rank=5 min=3997 min_rank=1 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1022, 1038, 1040, 1060, 0, 0, 0, 0, 1058, 960, 1058, 1012, 18, 9, 0, 0, 1028, 1044, 999, 1008, 0, 0, 0, 0, 1044, 996, 992, 1016, 34, 0, 0, 0, 960, 1022, 1048, 1012, 4, 1, 0, 0, 1003, 976, 1031, 1021, 5, 3, 0, 0, 1032, 950, 1027, 1059, 0, 0, 0, 0, 1069, 988, 1039, 1082, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4115, 4079, 4082, 4047, 4039, 4068, 4178] avg=4096.0000 max=4178 max_rank=7 min=4039 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1012, 961, 1056, 1038, 0, 0, 0, 0, 1014, 1025, 1038, 1010, 18, 9, 0, 0, 1062, 984, 1041, 962, 0, 0, 0, 0, 1037, 1002, 1018, 1014, 35, 0, 0, 0, 1038, 1003, 1070, 1009, 5, 1, 0, 0, 963, 1051, 1020, 1030, 5, 3, 0, 0, 1043, 999, 1032, 1007, 0, 0, 0, 0, 1089, 1019, 1025, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4114, 4049, 4106, 4126, 4072, 4081, 4153] avg=4096.0000 max=4153 max_rank=7 min=4049 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1046, 994, 993, 1044, 0, 0, 0, 0, 1066, 1053, 1028, 1053, 17, 9, 0, 0, 1053, 1035, 1041, 1006, 0, 0, 0, 0, 1025, 988, 1049, 965, 34, 0, 0, 0, 1009, 1042, 1032, 983, 4, 1, 0, 0, 1037, 1037, 1026, 1052, 5, 3, 0, 0, 1014, 989, 949, 1050, 0, 0, 0, 0, 1020, 987, 1007, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4226, 4135, 4061, 4071, 4160, 4002, 4036] avg=4096.0000 max=4226 max_rank=1 min=4002 min_rank=6 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16269 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1012, 1049, 1030, 1003, 0, 0, 0, 0, 1015, 985, 1015, 1020, 18, 9, 0, 0, 1036, 1046, 1045, 1001, 0, 0, 0, 0, 1000, 1011, 996, 1018, 33, 0, 0, 0, 1029, 1066, 1048, 1074, 4, 1, 0, 0, 1005, 1034, 982, 986, 5, 3, 0, 0, 1024, 1027, 1050, 1021, 0, 0, 0, 0, 1026, 954, 1001, 1086, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4062, 4128, 4058, 4222, 4015, 4122, 4067] avg=4096.0000 max=4222 max_rank=4 min=4015 min_rank=5 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1010, 1053, 997, 1065, 0, 0, 0, 0, 983, 1046, 1036, 1001, 17, 9, 0, 0, 1010, 999, 1046, 1042, 0, 0, 0, 0, 949, 1029, 1025, 1033, 35, 0, 0, 0, 1002, 1027, 1040, 1049, 4, 1, 0, 0, 1036, 993, 1032, 999, 5, 3, 0, 0, 1022, 1049, 1019, 1026, 0, 0, 0, 0, 1014, 1008, 1022, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4092, 4097, 4071, 4123, 4068, 4116, 4076] avg=4096.0000 max=4125 max_rank=0 min=4068 min_rank=5 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1037, 1057, 1006, 1007, 0, 0, 0, 0, 988, 988, 1033, 1044, 17, 9, 0, 0, 1032, 1011, 993, 1022, 0, 0, 0, 0, 1017, 1047, 1042, 991, 36, 0, 0, 0, 1003, 989, 1004, 1041, 4, 1, 0, 0, 1006, 1001, 1064, 1042, 5, 3, 0, 0, 1033, 1059, 989, 1029, 0, 0, 0, 0, 1018, 1029, 1051, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4079, 4058, 4133, 4042, 4121, 4110, 4118] avg=4096.0000 max=4133 max_rank=3 min=4042 min_rank=4 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1010, 1073, 1022, 1016, 0, 0, 0, 0, 1023, 1007, 1016, 1002, 22, 13, 0, 0, 1042, 1062, 1057, 998, 0, 0, 0, 0, 1027, 973, 1080, 1042, 38, 0, 0, 0, 1005, 1024, 993, 958, 10, 6, 0, 0, 1015, 1030, 1027, 1017, 8, 10, 0, 0, 1054, 1052, 1023, 1059, 0, 0, 0, 0, 962, 950, 1047, 995, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4083, 4159, 4160, 3996, 4107, 4188, 3954] avg=4096.0000 max=4188 max_rank=6 min=3954 min_rank=7 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16245 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1008, 992, 1041, 1028, 1047, 1012, 984, 1072, 996, 1024, 1034, 1073, 1032, 1077, 1020, 975, 1024, 987, 1011, 1042, 1070, 968, 988, 1116, 1049, 1063, 1046, 1006, 1007, 967, 975, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1054, 1013, 1059, 1019, 1072, 1018, 1011, 1013, 1020, 993, 1040, 981, 1020, 1017, 1024, 1078, 1021, 1044, 1044, 1022, 1054, 1037, 976, 968, 1017, 1028, 1033, 1020, 972, 1063, 1040, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1041, 1015, 1035, 1022, 1010, 994, 1021, 1015, 1067, 1025, 1042, 1036, 1038, 979, 1035, 1008, 1066, 1014, 1031, 978, 1040, 1075, 981, 1029, 1024, 1011, 1046, 1008, 1011, 1047, 994, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1024, 998, 1058, 1015, 1051, 1028, 1061, 1048, 1012, 987, 1004, 1066, 1000, 1024, 956, 975, 1012, 1019, 1067, 1026, 1028, 1035, 1011, 1079, 996, 1036, 1023, 1000, 1034, 1075, 991, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1036, 994, 1016, 946, 1044, 1047, 1014, 1045, 1015, 1015, 1042, 1069, 1026, 1023, 979, 1032, 1007, 1032, 1007, 1061, 997, 1047, 1034, 1025, 1070, 995, 1014, 1053, 991, 1029, 1051, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[983, 1022, 984, 1041, 1034, 1054, 1094, 1046, 1044, 972, 1024, 1067, 1031, 1035, 1014, 1002, 957, 962, 984, 1034, 957, 1007, 1013, 1103, 1061, 1071, 1038, 1049, 1001, 984, 1060, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[941, 1028, 982, 1023, 1001, 1033, 992, 1056, 1021, 1025, 1032, 1003, 1005, 1012, 1058, 1015, 1049, 1014, 996, 1034, 1069, 1021, 1006, 1015, 1041, 1079, 1058, 1064, 1046, 1031, 996, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1001, 973, 1055, 981, 1023, 1011, 1050, 1087, 1035, 1032, 957, 1050, 994, 1027, 1008, 1037, 1063, 985, 1009, 1000, 1040, 1035, 1024, 1027, 1001, 1040, 1019, 1028, 1058, 1017, 1067, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1008, 992, 1041, 1028, 40, 2, 0, 0, 1047, 1012, 984, 1032, 0, 0, 0, 0, 996, 1024, 1034, 1069, 0, 0, 0, 0, 1032, 1077, 1020, 975, 27, 0, 0, 0, 1024, 987, 1011, 1042, 11, 10, 0, 0, 1070, 968, 988, 1103, 0, 0, 0, 0, 1049, 1023, 1046, 1006, 0, 0, 0, 0, 1007, 967, 975, 1034, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4075, 4123, 4131, 4085, 4129, 4124, 3990] avg=4096.0000 max=4131 max_rank=3 min=3990 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1041, 1015, 1035, 1022, 38, 2, 0, 0, 1010, 994, 1021, 977, 0, 0, 0, 0, 1067, 1025, 1042, 1032, 0, 0, 0, 0, 1038, 979, 1035, 1008, 25, 0, 0, 0, 1066, 1014, 1031, 978, 10, 9, 0, 0, 1040, 1075, 981, 1017, 0, 0, 0, 0, 1024, 974, 1046, 1008, 0, 0, 0, 0, 1011, 1047, 994, 1030, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4002, 4166, 4085, 4108, 4113, 4052, 4089] avg=4096.0000 max=4166 max_rank=2 min=4002 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[983, 1022, 984, 1041, 39, 2, 0, 0, 1034, 1054, 1094, 1007, 0, 0, 0, 0, 1044, 972, 1024, 1063, 0, 0, 0, 0, 1031, 1035, 1014, 1002, 27, 0, 0, 0, 957, 962, 984, 1034, 11, 10, 0, 0, 957, 1007, 1013, 1090, 0, 0, 0, 0, 1061, 1031, 1038, 1049, 0, 0, 0, 0, 1001, 984, 1060, 1040, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4189, 4103, 4109, 3958, 4067, 4179, 4092] avg=4096.0000 max=4189 max_rank=1 min=3958 min_rank=4 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1024, 998, 1058, 1015, 40, 2, 0, 0, 1051, 1028, 1061, 1008, 0, 0, 0, 0, 1012, 987, 1004, 1062, 0, 0, 0, 0, 1000, 1024, 956, 975, 26, 0, 0, 0, 1012, 1019, 1067, 1026, 10, 10, 0, 0, 1028, 1035, 1011, 1066, 0, 0, 0, 0, 996, 998, 1023, 1000, 0, 0, 0, 0, 1034, 1075, 991, 1029, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4148, 4065, 3981, 4144, 4140, 4017, 4136] avg=4096.0000 max=4148 max_rank=1 min=3981 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1036, 994, 1016, 946, 39, 2, 0, 0, 1044, 1047, 1014, 1006, 0, 0, 0, 0, 1015, 1015, 1042, 1065, 0, 0, 0, 0, 1026, 1023, 979, 1032, 25, 0, 0, 0, 1007, 1032, 1007, 1061, 10, 9, 0, 0, 997, 1047, 1034, 1013, 0, 0, 0, 0, 1070, 958, 1014, 1053, 0, 0, 0, 0, 991, 1029, 1051, 1012, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4111, 4137, 4085, 4126, 4091, 4095, 4090] avg=4096.0000 max=4137 max_rank=2 min=4033 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[941, 1028, 982, 1023, 40, 2, 0, 0, 1001, 1033, 992, 1016, 0, 0, 0, 0, 1021, 1025, 1032, 999, 0, 0, 0, 0, 1005, 1012, 1058, 1015, 27, 0, 0, 0, 1049, 1014, 996, 1034, 11, 9, 0, 0, 1069, 1021, 1006, 1003, 0, 0, 0, 0, 1041, 1039, 1058, 1064, 0, 0, 0, 0, 1046, 1031, 996, 1022, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4042, 4077, 4117, 4113, 4099, 4202, 4102] avg=4096.0000 max=4202 max_rank=6 min=4016 min_rank=0 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1054, 1013, 1059, 1019, 43, 6, 0, 0, 1072, 1018, 1011, 970, 0, 0, 0, 0, 1020, 993, 1040, 974, 0, 0, 0, 0, 1020, 1017, 1024, 1078, 29, 0, 0, 0, 1021, 1044, 1044, 1022, 14, 13, 0, 0, 1054, 1037, 976, 948, 0, 0, 0, 0, 1017, 979, 1033, 1020, 0, 0, 0, 0, 972, 1063, 1040, 997, 7, 7, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4071, 4027, 4168, 4158, 4015, 4049, 4086] avg=4096.0000 max=4194 max_rank=0 min=4015 min_rank=5 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1001, 973, 1055, 981, 41, 2, 0, 0, 1023, 1011, 1050, 1046, 0, 0, 0, 0, 1035, 1032, 957, 1046, 0, 0, 0, 0, 994, 1027, 1008, 1037, 26, 0, 0, 0, 1063, 985, 1009, 1000, 10, 9, 0, 0, 1040, 1035, 1024, 1015, 0, 0, 0, 0, 1001, 1002, 1019, 1028, 0, 0, 0, 0, 1058, 1017, 1067, 1034, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4130, 4070, 4092, 4076, 4114, 4050, 4183] avg=4096.0000 max=4183 max_rank=7 min=4050 min_rank=6 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1056, 953, 1033, 1009, 1034, 1012, 1070, 1033, 1001, 990, 1055, 1033, 1038, 1068, 999, 1038, 999, 1027, 1025, 992, 1032, 1002, 1034, 985, 1018, 1065, 1034, 1050, 1001, 1025, 1016, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1020, 972, 1007, 1016, 1010, 1059, 1011, 1069, 1003, 984, 998, 1054, 1034, 1044, 1010, 1024, 1035, 1035, 1035, 1027, 1051, 1017, 1009, 1055, 1025, 1028, 1006, 1025, 1049, 1047, 963, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1021, 987, 1008, 989, 1005, 1015, 1017, 1049, 1009, 1008, 1021, 993, 1038, 1009, 1046, 1030, 1054, 1049, 1021, 1035, 1024, 1043, 1022, 1047, 1072, 985, 1024, 1066, 1024, 999, 1040, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1029, 1038, 967, 1056, 1029, 1014, 1049, 1050, 1044, 986, 1048, 1023, 999, 1038, 1044, 996, 983, 993, 1060, 1068, 1063, 1037, 1022, 1043, 994, 1041, 1010, 1035, 1016, 998, 983, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[977, 1021, 1029, 999, 1062, 1056, 1036, 1031, 1047, 1028, 963, 1035, 1011, 1021, 1076, 1048, 1059, 1025, 1065, 1048, 1043, 980, 1040, 1005, 1022, 1021, 1009, 1051, 1014, 975, 968, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[979, 1017, 1014, 1014, 1012, 1082, 1011, 1084, 978, 1032, 1033, 1018, 1026, 1032, 1054, 1009, 967, 1012, 1033, 991, 1013, 1069, 1056, 1012, 982, 1070, 1028, 1034, 1072, 1004, 1049, 981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1013, 1067, 1053, 1031, 1004, 1036, 1019, 1008, 1045, 1046, 1035, 1009, 1014, 1012, 1022, 1010, 989, 1028, 977, 1010, 1025, 1045, 1086, 1014, 990, 983, 1056, 1013, 1048, 1035, 985, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1039, 991, 982, 1010, 1033, 1043, 1005, 1082, 1060, 1031, 1023, 1027, 1016, 1000, 1015, 1018, 1001, 1045, 998, 1014, 1016, 1030, 1050, 1032, 1043, 1009, 1026, 1041, 1031, 992, 1050, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1056, 953, 1033, 1009, 44, 1, 0, 0, 1034, 1012, 1070, 989, 0, 0, 0, 0, 1001, 990, 1055, 1033, 11, 2, 0, 0, 1038, 1068, 991, 1038, 0, 0, 0, 0, 999, 1027, 1025, 992, 8, 0, 0, 0, 1032, 1002, 1006, 985, 0, 0, 0, 0, 1018, 1065, 1034, 1039, 0, 0, 0, 0, 1001, 1025, 1016, 1041, 25, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4105, 4092, 4135, 4051, 4025, 4156, 4108] avg=4096.0000 max=4156 max_rank=6 min=4025 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1020, 972, 1007, 1016, 46, 1, 0, 0, 1010, 1059, 1011, 1023, 0, 0, 0, 0, 1003, 984, 998, 1054, 10, 2, 0, 0, 1034, 1044, 1002, 1024, 0, 0, 0, 0, 1035, 1035, 1035, 1027, 8, 0, 0, 0, 1051, 1017, 981, 1055, 0, 0, 0, 0, 1025, 1028, 1006, 1015, 0, 0, 0, 0, 1049, 1047, 963, 1046, 25, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4103, 4051, 4104, 4140, 4104, 4074, 4130] avg=4096.0000 max=4140 max_rank=4 min=4051 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[977, 1021, 1029, 999, 44, 1, 0, 0, 1062, 1056, 1036, 987, 0, 0, 0, 0, 1047, 1028, 963, 1035, 11, 2, 0, 0, 1011, 1021, 1068, 1048, 0, 0, 0, 0, 1059, 1025, 1065, 1048, 8, 0, 0, 0, 1043, 980, 1011, 1005, 0, 0, 0, 0, 1022, 1021, 1009, 1040, 0, 0, 0, 0, 1014, 975, 968, 1003, 26, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4141, 4086, 4148, 4205, 4039, 4092, 3986] avg=4096.0000 max=4205 max_rank=4 min=3986 min_rank=7 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1021, 987, 1008, 989, 45, 1, 0, 0, 1005, 1015, 1017, 1004, 0, 0, 0, 0, 1009, 1008, 1021, 993, 11, 2, 0, 0, 1038, 1009, 1038, 1030, 0, 0, 0, 0, 1054, 1049, 1021, 1035, 8, 0, 0, 0, 1024, 1043, 994, 1047, 0, 0, 0, 0, 1072, 985, 1024, 1055, 0, 0, 0, 0, 1024, 999, 1040, 1018, 25, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4041, 4044, 4115, 4167, 4108, 4136, 4106] avg=4096.0000 max=4167 max_rank=4 min=4041 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[979, 1017, 1014, 1014, 46, 1, 0, 0, 1012, 1082, 1011, 1038, 0, 0, 0, 0, 978, 1032, 1033, 1018, 10, 2, 0, 0, 1026, 1032, 1046, 1009, 0, 0, 0, 0, 967, 1012, 1033, 991, 8, 0, 0, 0, 1013, 1069, 1027, 1012, 0, 0, 0, 0, 982, 1070, 1028, 1024, 0, 0, 0, 0, 1072, 1004, 1049, 981, 26, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4143, 4073, 4113, 4011, 4121, 4104, 4132] avg=4096.0000 max=4143 max_rank=1 min=4011 min_rank=4 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1029, 1038, 967, 1056, 48, 2, 0, 0, 1029, 1014, 1049, 1002, 0, 0, 0, 0, 1044, 986, 1048, 1023, 14, 3, 3, 0, 999, 1038, 1029, 996, 0, 0, 0, 0, 983, 993, 1060, 1068, 12, 0, 0, 0, 1063, 1037, 989, 1043, 0, 0, 0, 0, 994, 1041, 1010, 1021, 0, 0, 0, 0, 1016, 998, 983, 1012, 28, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4094, 4121, 4062, 4116, 4132, 4066, 4037] avg=4096.0000 max=4140 max_rank=0 min=4037 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1039, 991, 982, 1010, 46, 1, 0, 0, 1033, 1043, 1005, 1036, 0, 0, 0, 0, 1060, 1031, 1023, 1027, 11, 2, 0, 0, 1016, 1000, 1007, 1018, 0, 0, 0, 0, 1001, 1045, 998, 1014, 8, 0, 0, 0, 1016, 1030, 1021, 1032, 0, 0, 0, 0, 1043, 1009, 1026, 1030, 0, 0, 0, 0, 1031, 992, 1050, 1015, 26, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4117, 4154, 4041, 4066, 4099, 4108, 4114] avg=4096.0000 max=4154 max_rank=2 min=4041 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1013, 1067, 1053, 1031, 43, 1, 0, 0, 1004, 1036, 1019, 965, 0, 0, 0, 0, 1045, 1046, 1035, 1009, 10, 2, 0, 0, 1014, 1012, 1014, 1010, 0, 0, 0, 0, 989, 1028, 977, 1010, 8, 0, 0, 0, 1025, 1045, 1056, 1014, 0, 0, 0, 0, 990, 983, 1056, 1003, 0, 0, 0, 0, 1048, 1035, 985, 1060, 27, 0, 0, 0] rerouted_tokens_per_ep_rank=[4208, 4024, 4147, 4050, 4012, 4140, 4032, 4155] avg=4096.0000 max=4208 max_rank=0 min=4012 min_rank=4 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8064, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1059, 1001, 1047, 1100, 1040, 1025, 1009, 976, 1029, 1059, 1065, 1019, 973, 1050, 1056, 978, 997, 1012, 1008, 1004, 982, 987, 998, 1048, 1028, 1045, 1013, 989, 1028, 1050, 1054, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1034, 998, 1033, 1019, 1083, 1039, 1038, 1049, 982, 1038, 1028, 1019, 1051, 1053, 998, 1003, 967, 987, 1030, 1063, 1021, 1034, 995, 1015, 1035, 991, 1066, 1021, 1031, 1011, 1019, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1026, 1032, 966, 1049, 1014, 1021, 1047, 1025, 1012, 1054, 1031, 1087, 1058, 1017, 988, 1016, 999, 1070, 1046, 1015, 1046, 1013, 1021, 1002, 979, 1048, 980, 1015, 995, 1018, 1008, 1070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1020, 1043, 998, 1056, 1032, 1039, 1013, 1028, 1081, 1010, 990, 1015, 1024, 1017, 1044, 988, 1009, 1029, 988, 1056, 1012, 1019, 1028, 1017, 983, 993, 1048, 1035, 1078, 1016, 1033, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[956, 997, 1026, 1052, 1022, 969, 1052, 1028, 989, 1047, 1014, 1025, 1037, 1016, 1045, 1071, 1035, 992, 1032, 1004, 1021, 1085, 968, 1001, 1041, 1043, 1043, 984, 1028, 1049, 1084, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1043, 1058, 1035, 1044, 1010, 1026, 998, 1094, 1007, 1017, 1078, 1028, 993, 1049, 1050, 1020, 1068, 1009, 992, 972, 986, 1052, 986, 998, 1016, 1036, 1027, 996, 1030, 978, 1031, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[981, 1013, 1023, 1021, 1046, 1010, 1042, 1021, 1017, 1003, 1072, 1007, 1001, 1047, 986, 997, 1038, 1020, 1000, 1018, 1012, 1031, 1057, 1056, 1049, 1006, 1040, 1028, 1030, 1001, 1075, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1021, 1036, 1034, 1059, 1004, 996, 1028, 1024, 969, 1013, 1036, 1052, 1086, 1018, 993, 1055, 1032, 1018, 1019, 1017, 970, 1010, 1021, 988, 1071, 1040, 996, 1001, 1024, 1036, 1050, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1059, 1001, 1047, 1086, 0, 0, 0, 0, 1030, 1025, 1009, 976, 0, 0, 0, 0, 1029, 1059, 1050, 1019, 0, 0, 0, 0, 973, 1049, 1056, 978, 0, 0, 0, 0, 997, 1012, 1008, 1004, 14, 13, 1, 0, 982, 987, 998, 1048, 33, 2, 0, 0, 1028, 1045, 1013, 989, 9, 1, 0, 0, 1028, 1050, 1021, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4193, 4040, 4157, 4056, 4049, 4050, 4085, 4138] avg=4096.0000 max=4193 max_rank=0 min=4040 min_rank=1 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 1032, 966, 1036, 0, 0, 0, 0, 1006, 1021, 1047, 1025, 0, 0, 0, 0, 1012, 1054, 1017, 1087, 0, 0, 0, 0, 1058, 1016, 988, 1016, 0, 0, 0, 0, 999, 1070, 1046, 1015, 13, 12, 0, 0, 1046, 1013, 1021, 1002, 31, 2, 0, 0, 979, 1048, 980, 1015, 8, 1, 0, 0, 995, 1018, 977, 1070, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4099, 4170, 4078, 4155, 4115, 4031, 4060] avg=4096.0000 max=4170 max_rank=2 min=4031 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1020, 1043, 998, 1042, 0, 0, 0, 0, 1022, 1039, 1013, 1028, 0, 0, 0, 0, 1081, 1010, 976, 1015, 0, 0, 0, 0, 1024, 1016, 1044, 988, 0, 0, 0, 0, 1009, 1029, 988, 1056, 14, 12, 1, 0, 1012, 1019, 1028, 1017, 32, 2, 0, 0, 983, 993, 1048, 1035, 9, 1, 0, 0, 1078, 1016, 1001, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4102, 4082, 4072, 4109, 4110, 4069, 4121] avg=4096.0000 max=4121 max_rank=7 min=4069 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1034, 998, 1033, 1006, 0, 0, 0, 0, 1073, 1039, 1038, 1049, 0, 0, 0, 0, 982, 1038, 1014, 1019, 0, 0, 0, 0, 1051, 1052, 998, 1003, 0, 0, 0, 0, 967, 987, 1030, 1063, 13, 12, 1, 0, 1021, 1034, 995, 1015, 32, 2, 0, 0, 1035, 991, 1066, 1021, 9, 1, 0, 0, 1031, 1011, 987, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4199, 4053, 4104, 4073, 4099, 4123, 4046] avg=4096.0000 max=4199 max_rank=1 min=4046 min_rank=7 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[956, 997, 1026, 1038, 0, 0, 0, 0, 1014, 969, 1052, 1028, 0, 0, 0, 0, 989, 1047, 1000, 1025, 0, 0, 0, 0, 1037, 1015, 1045, 1071, 0, 0, 0, 0, 1035, 992, 1032, 1004, 14, 12, 0, 0, 1021, 1085, 968, 1001, 34, 2, 0, 0, 1041, 1043, 1043, 984, 8, 1, 0, 0, 1028, 1049, 1050, 1012, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4063, 4061, 4168, 4089, 4111, 4120, 4139] avg=4096.0000 max=4168 max_rank=3 min=4017 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1043, 1058, 1035, 1031, 0, 0, 0, 0, 1002, 1026, 998, 1094, 0, 0, 0, 0, 1007, 1017, 1063, 1028, 0, 0, 0, 0, 993, 1048, 1050, 1020, 0, 0, 0, 0, 1068, 1009, 992, 972, 13, 13, 0, 0, 986, 1052, 986, 998, 32, 2, 0, 0, 1016, 1036, 1027, 996, 8, 1, 0, 0, 1030, 978, 999, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4120, 4115, 4111, 4067, 4056, 4084, 4048] avg=4096.0000 max=4167 max_rank=0 min=4048 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16255 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[981, 1013, 1023, 1004, 0, 0, 0, 0, 1028, 1010, 1042, 1021, 0, 0, 0, 0, 1017, 1003, 1047, 1007, 0, 0, 0, 0, 1001, 1044, 986, 997, 0, 0, 0, 0, 1038, 1020, 1000, 1018, 17, 16, 5, 0, 1012, 1031, 1057, 1056, 38, 9, 0, 0, 1049, 1006, 1040, 1028, 13, 3, 0, 0, 1030, 1001, 1037, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 4101, 4074, 4028, 4114, 4203, 4139, 4088] avg=4096.0000 max=4203 max_rank=5 min=4021 min_rank=0 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16544 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1021, 1036, 1034, 1045, 0, 0, 0, 0, 996, 996, 1028, 1024, 0, 0, 0, 0, 969, 1013, 1022, 1052, 0, 0, 0, 0, 1086, 1017, 993, 1055, 0, 0, 0, 0, 1032, 1018, 1019, 1017, 14, 12, 0, 0, 970, 1010, 1021, 988, 33, 2, 0, 0, 1071, 1040, 996, 1001, 8, 1, 0, 0, 1024, 1036, 1017, 1051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4044, 4056, 4151, 4112, 4024, 4117, 4128] avg=4096.0000 max=4151 max_rank=3 min=4024 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[966, 991, 1080, 1033, 1039, 1007, 1033, 1077, 1024, 1041, 1004, 1008, 1020, 1013, 1018, 983, 1023, 1003, 1031, 1010, 1029, 1069, 1001, 1045, 1050, 1015, 1014, 1051, 1037, 1038, 992, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1048, 1073, 1028, 1056, 1033, 941, 1010, 1019, 1024, 1005, 1043, 1074, 1026, 998, 1047, 1007, 1038, 958, 1024, 993, 1035, 1021, 1005, 1036, 944, 1035, 1041, 1070, 1070, 1022, 1009, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1021, 1016, 1024, 999, 1042, 1073, 1023, 957, 1033, 1036, 1108, 993, 1005, 1009, 1022, 1058, 1008, 1023, 1030, 1083, 1031, 989, 1052, 1001, 1060, 1027, 1028, 1000, 975, 1019, 1022, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[996, 1026, 987, 1043, 991, 1006, 1026, 1053, 1065, 1013, 1028, 1004, 1050, 1030, 1024, 1029, 1020, 1021, 1012, 1010, 1016, 1015, 1046, 1006, 997, 1036, 1032, 1040, 1037, 1018, 1019, 1072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1017, 1012, 1047, 1061, 1095, 1035, 1024, 1023, 1006, 1004, 1017, 1049, 1037, 1026, 1041, 1062, 1005, 1021, 1013, 1034, 975, 988, 1027, 1053, 982, 985, 1001, 1010, 1022, 1013, 1045, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1072, 1092, 1037, 984, 1034, 1084, 1001, 1002, 999, 1026, 1022, 1059, 1058, 1032, 1034, 1043, 994, 1001, 1022, 1003, 1021, 1064, 991, 1021, 988, 1022, 980, 1042, 1030, 970, 1053, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1006, 1006, 1037, 1015, 1095, 984, 987, 1026, 1018, 1001, 1010, 988, 1034, 1011, 1003, 1033, 1069, 1013, 999, 1034, 991, 1006, 1025, 1020, 1053, 1020, 1023, 1053, 1030, 1040, 1103, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1041, 1017, 1056, 1009, 1016, 1003, 1019, 992, 993, 1087, 1026, 1032, 1010, 1049, 1051, 997, 1083, 1071, 1025, 1005, 1068, 983, 970, 1024, 1026, 1020, 1018, 1024, 1013, 1032, 983, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[966, 991, 1064, 1033, 0, 0, 0, 0, 1039, 1007, 1033, 1077, 2, 0, 0, 0, 1024, 1041, 996, 1008, 0, 0, 0, 0, 1010, 1013, 1018, 983, 0, 0, 0, 0, 1023, 1003, 1031, 1010, 9, 1, 0, 0, 1029, 1069, 1001, 1045, 16, 1, 0, 0, 1050, 1015, 1014, 1051, 7, 2, 0, 0, 1037, 1038, 988, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4158, 4069, 4024, 4077, 4161, 4139, 4086] avg=4096.0000 max=4161 max_rank=5 min=4024 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[996, 1026, 972, 1043, 0, 0, 0, 0, 991, 1006, 1026, 1053, 2, 0, 0, 0, 1065, 1013, 1020, 1004, 0, 0, 0, 0, 1039, 1030, 1024, 1029, 0, 0, 0, 0, 1020, 1021, 1012, 1010, 9, 1, 0, 0, 1016, 1015, 1046, 1006, 15, 2, 0, 0, 997, 1036, 1032, 1040, 7, 2, 0, 0, 1037, 1018, 1015, 1072, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4078, 4102, 4122, 4073, 4100, 4114, 4142] avg=4096.0000 max=4142 max_rank=7 min=4037 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1048, 1073, 1013, 1056, 0, 0, 0, 0, 1033, 941, 1010, 1019, 2, 0, 0, 0, 1024, 1005, 1035, 1074, 0, 0, 0, 0, 1016, 998, 1047, 1007, 0, 0, 0, 0, 1038, 958, 1024, 993, 9, 1, 0, 0, 1035, 1021, 1005, 1036, 15, 1, 0, 0, 944, 1035, 1041, 1070, 7, 2, 0, 0, 1070, 1022, 1005, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4005, 4138, 4068, 4023, 4113, 4099, 4132] avg=4096.0000 max=4190 max_rank=0 min=4005 min_rank=1 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1021, 1016, 1009, 999, 0, 0, 0, 0, 1042, 1073, 1023, 957, 2, 0, 0, 0, 1033, 1036, 1100, 993, 0, 0, 0, 0, 995, 1009, 1022, 1058, 0, 0, 0, 0, 1008, 1023, 1030, 1083, 9, 1, 0, 0, 1031, 989, 1052, 1001, 15, 1, 0, 0, 1060, 1027, 1028, 1000, 7, 2, 0, 0, 975, 1019, 1018, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4097, 4162, 4084, 4154, 4089, 4124, 4013] avg=4096.0000 max=4162 max_rank=2 min=4013 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1017, 1012, 1031, 1061, 0, 0, 0, 0, 1095, 1035, 1024, 1023, 2, 0, 0, 0, 1006, 1004, 1009, 1049, 0, 0, 0, 0, 1026, 1026, 1041, 1062, 0, 0, 0, 0, 1005, 1021, 1013, 1034, 9, 1, 0, 0, 975, 988, 1027, 1053, 16, 2, 0, 0, 982, 985, 1001, 1010, 7, 2, 0, 0, 1022, 1013, 1041, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4179, 4068, 4155, 4083, 4061, 3987, 4114] avg=4096.0000 max=4179 max_rank=1 min=3987 min_rank=6 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16245 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1072, 1092, 1018, 984, 0, 0, 0, 0, 1034, 1084, 1001, 1002, 4, 0, 0, 0, 999, 1026, 1006, 1059, 0, 0, 0, 0, 1039, 1032, 1034, 1043, 0, 0, 0, 0, 994, 1001, 1022, 1003, 13, 6, 0, 0, 1021, 1064, 991, 1021, 19, 6, 0, 0, 988, 1022, 980, 1042, 10, 8, 0, 0, 1030, 970, 1041, 987, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4125, 4090, 4148, 4039, 4122, 4050, 4028] avg=4096.0000 max=4166 max_rank=0 min=4028 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16239 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1006, 1006, 1021, 1015, 0, 0, 0, 0, 1095, 984, 987, 1026, 2, 0, 0, 0, 1018, 1001, 1002, 988, 0, 0, 0, 0, 1023, 1011, 1003, 1033, 0, 0, 0, 0, 1069, 1013, 999, 1034, 9, 1, 0, 0, 991, 1006, 1025, 1020, 16, 2, 0, 0, 1053, 1020, 1023, 1053, 7, 2, 0, 0, 1030, 1040, 1099, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4094, 4009, 4070, 4125, 4060, 4158, 4204] avg=4096.0000 max=4204 max_rank=7 min=4009 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16547 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1041, 1017, 1040, 1009, 0, 0, 0, 0, 1016, 1003, 1019, 992, 2, 0, 0, 0, 993, 1087, 1018, 1032, 0, 0, 0, 0, 1000, 1049, 1051, 997, 0, 0, 0, 0, 1083, 1071, 1025, 1005, 9, 1, 0, 0, 1068, 983, 970, 1024, 16, 1, 0, 0, 1026, 1020, 1018, 1024, 7, 2, 0, 0, 1013, 1032, 979, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4032, 4130, 4097, 4194, 4062, 4097, 4049] avg=4096.0000 max=4194 max_rank=4 min=4032 min_rank=1 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1070, 985, 1048, 1041, 1054, 1036, 1006, 1011, 1043, 1094, 1028, 1007, 1002, 1042, 965, 1034, 1031, 1014, 1039, 955, 1040, 1031, 989, 1014, 1009, 1018, 988, 1042, 1024, 1059, 1025, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[986, 1030, 992, 1005, 1021, 1053, 1067, 970, 984, 1011, 996, 975, 999, 1015, 1050, 1022, 1029, 1015, 1087, 1010, 997, 1033, 1044, 1020, 1045, 1037, 1065, 1005, 1002, 1055, 1030, 1118] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1041, 1036, 984, 981, 1020, 1002, 1020, 1022, 1014, 1031, 976, 1036, 1013, 1043, 1055, 1074, 1063, 1072, 1035, 980, 1002, 1026, 1040, 1019, 1003, 1036, 1022, 1022, 985, 1032, 1032, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1016, 998, 1002, 1013, 1027, 1046, 1026, 1082, 1010, 1052, 1006, 1031, 1036, 995, 1011, 1025, 1034, 989, 1003, 1030, 1014, 1061, 1038, 1029, 1067, 1026, 1013, 1018, 1042, 1031, 1023, 974] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1002, 1009, 1020, 936, 1024, 1006, 1053, 1011, 1047, 1091, 1054, 1035, 1039, 1010, 1039, 1068, 1030, 1009, 976, 1012, 1030, 1025, 1004, 1034, 1023, 1029, 1058, 1018, 1020, 1007, 996, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1034, 1004, 1009, 1061, 1033, 969, 1014, 1063, 1007, 1002, 1015, 987, 1005, 1011, 1063, 1026, 1027, 1053, 1022, 1085, 1000, 1004, 1009, 1051, 1089, 976, 1039, 1033, 1030, 1004, 1042, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1034, 1000, 1025, 1065, 1031, 1014, 1028, 1047, 1049, 950, 1027, 982, 1037, 1010, 1036, 1038, 1008, 1027, 1043, 1054, 1009, 1070, 992, 1009, 1008, 1027, 1045, 972, 1014, 1042, 1047, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[996, 1034, 1033, 1052, 1027, 1031, 1007, 958, 1069, 1063, 1076, 1090, 943, 1045, 1056, 1019, 1010, 1084, 1027, 993, 1037, 1000, 1001, 1020, 980, 1031, 1024, 1022, 994, 1072, 986, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1070, 985, 1048, 1041, 9, 9, 8, 1, 1053, 1036, 1006, 1011, 0, 0, 0, 0, 1043, 1085, 1028, 1007, 0, 0, 0, 0, 1002, 1042, 965, 1028, 0, 0, 0, 0, 1031, 1005, 1039, 955, 0, 0, 0, 0, 1040, 1031, 989, 1014, 5, 2, 1, 0, 1009, 1018, 986, 1042, 0, 0, 0, 0, 1024, 1051, 1025, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4106, 4163, 4037, 4030, 4082, 4055, 4124] avg=4096.0000 max=4171 max_rank=0 min=4030 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[986, 1030, 992, 1005, 9, 8, 8, 1, 1020, 1053, 1067, 970, 0, 0, 0, 0, 984, 1003, 996, 975, 0, 0, 0, 0, 999, 1015, 1050, 1016, 0, 0, 0, 0, 1029, 1006, 1087, 1010, 0, 0, 0, 0, 997, 1033, 1044, 1020, 5, 2, 1, 0, 1045, 1037, 1063, 1005, 0, 0, 0, 0, 1002, 1047, 1030, 1118, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4110, 3958, 4080, 4132, 4102, 4150, 4197] avg=4096.0000 max=4197 max_rank=7 min=3958 min_rank=2 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1041, 1036, 984, 981, 10, 8, 7, 1, 1019, 1002, 1020, 1022, 0, 0, 0, 0, 1014, 1023, 976, 1036, 0, 0, 0, 0, 1013, 1043, 1055, 1068, 0, 0, 0, 0, 1063, 1062, 1035, 980, 0, 0, 0, 0, 1002, 1026, 1040, 1019, 5, 2, 1, 0, 1003, 1036, 1020, 1022, 0, 0, 0, 0, 985, 1025, 1032, 1051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4063, 4049, 4179, 4140, 4095, 4081, 4093] avg=4096.0000 max=4179 max_rank=3 min=4049 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1002, 1009, 1020, 936, 9, 9, 7, 1, 1023, 1006, 1053, 1011, 0, 0, 0, 0, 1047, 1082, 1054, 1035, 0, 0, 0, 0, 1039, 1010, 1039, 1062, 0, 0, 0, 0, 1030, 1000, 976, 1012, 0, 0, 0, 0, 1030, 1025, 1004, 1034, 5, 2, 1, 0, 1023, 1029, 1056, 1018, 0, 0, 0, 0, 1020, 1000, 996, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3993, 4093, 4218, 4150, 4018, 4101, 4126, 4069] avg=4096.0000 max=4218 max_rank=2 min=3993 min_rank=0 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1016, 998, 1002, 1013, 13, 12, 11, 8, 1023, 1046, 1026, 1082, 0, 0, 0, 0, 1010, 1040, 1006, 1031, 0, 0, 0, 0, 1036, 995, 1011, 1009, 0, 0, 0, 0, 1034, 976, 1003, 1030, 0, 0, 0, 0, 1014, 1061, 1038, 1029, 8, 8, 4, 0, 1067, 1026, 1005, 1018, 0, 0, 0, 0, 1042, 1020, 1023, 974, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4177, 4087, 4051, 4043, 4162, 4116, 4059] avg=4096.0000 max=4177 max_rank=1 min=4043 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1034, 1004, 1009, 1061, 9, 8, 7, 1, 1032, 969, 1014, 1063, 0, 0, 0, 0, 1007, 994, 1015, 987, 0, 0, 0, 0, 1005, 1011, 1063, 1020, 0, 0, 0, 0, 1027, 1044, 1022, 1085, 0, 0, 0, 0, 1000, 1004, 1009, 1051, 5, 2, 1, 0, 1089, 976, 1037, 1033, 0, 0, 0, 0, 1030, 997, 1042, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4078, 4003, 4099, 4178, 4072, 4135, 4070] avg=4096.0000 max=4178 max_rank=4 min=4003 min_rank=2 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[996, 1034, 1033, 1052, 10, 8, 8, 1, 1026, 1031, 1007, 958, 0, 0, 0, 0, 1069, 1055, 1076, 1090, 0, 0, 0, 0, 943, 1045, 1056, 1013, 0, 0, 0, 0, 1010, 1074, 1027, 993, 0, 0, 0, 0, 1037, 1000, 1001, 1020, 5, 2, 1, 0, 980, 1031, 1022, 1022, 0, 0, 0, 0, 994, 1064, 986, 988, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4022, 4290, 4057, 4104, 4066, 4055, 4032] avg=4096.0000 max=4290 max_rank=2 min=4022 min_rank=1 imbalance_ratio=1.047363 max_load_reduction=28478 max_load_reduction_pct=0.869080 moved_to_echo=16257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1034, 1000, 1025, 1065, 9, 8, 7, 1, 1030, 1014, 1028, 1047, 0, 0, 0, 0, 1049, 942, 1027, 982, 0, 0, 0, 0, 1037, 1010, 1036, 1032, 0, 0, 0, 0, 1008, 1018, 1043, 1054, 0, 0, 0, 0, 1009, 1070, 992, 1009, 5, 2, 1, 0, 1008, 1027, 1043, 972, 0, 0, 0, 0, 1014, 1035, 1047, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4119, 4000, 4115, 4123, 4088, 4050, 4124] avg=4096.0000 max=4149 max_rank=0 min=4000 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-08-30 04:58:04] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 623.5 | throughput per GPU (TFLOP/s/GPU): 534.7 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.035000E+01 | loss scale: 1.0 | grad norm: 210.120 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1004, 1013, 1033, 1064, 1033, 984, 1008, 1015, 1061, 1009, 1006, 997, 1062, 992, 1013, 1049, 1039, 1016, 1006, 995, 1016, 1024, 1009, 1036, 1059, 992, 1019, 1025, 1039, 1053, 1079, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1037, 1005, 1030, 1031, 959, 1063, 1040, 1030, 989, 1002, 1047, 995, 1020, 1078, 1012, 1026, 1014, 1006, 1070, 977, 1041, 1047, 1048, 1072, 980, 1002, 1020, 1018, 1085, 995, 1005, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1081, 1006, 1012, 1008, 993, 1033, 1036, 1038, 1042, 1007, 1011, 1022, 998, 1008, 1034, 1014, 1048, 1008, 1012, 993, 1062, 976, 1057, 966, 1046, 1021, 1004, 1071, 1016, 1057, 1055, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1036, 1020, 986, 1034, 1025, 1045, 1024, 1046, 1014, 1026, 1029, 1060, 1045, 1007, 987, 965, 1000, 1096, 1047, 1030, 1042, 1003, 1067, 946, 1063, 1036, 1029, 976, 1087, 1015, 986, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1033, 1012, 1016, 993, 991, 1009, 996, 1047, 1005, 1023, 1066, 1011, 1011, 1007, 1047, 1017, 1004, 1063, 1015, 978, 1014, 1041, 1032, 1012, 992, 1065, 1024, 1057, 1062, 1069, 1041, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[991, 1001, 1056, 1020, 1036, 1030, 1009, 1063, 1048, 1029, 997, 993, 1013, 1000, 1063, 1021, 1028, 1035, 1014, 1057, 964, 1057, 999, 962, 1028, 988, 1011, 1042, 1053, 1082, 1027, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[996, 982, 1047, 1039, 1043, 1007, 1031, 970, 1035, 992, 1012, 1024, 1028, 1025, 1017, 1037, 1023, 1011, 1068, 1035, 1028, 1018, 1034, 1045, 1039, 1062, 1039, 977, 1053, 1031, 1011, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1005, 1031, 993, 988, 1043, 1052, 1044, 992, 1035, 1011, 985, 952, 1060, 1032, 1024, 1040, 995, 1025, 1035, 1123, 1050, 1004, 1002, 1052, 1032, 1038, 1034, 1030, 991, 1024, 1067, 979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1037, 1005, 1030, 1031, 13, 7, 0, 0, 959, 1063, 1040, 1030, 3, 0, 0, 0, 989, 1002, 1047, 995, 30, 0, 0, 0, 1020, 1078, 1012, 1026, 1, 0, 0, 0, 1014, 1006, 1058, 977, 0, 0, 0, 0, 1041, 1047, 1048, 1072, 5, 0, 0, 0, 976, 1002, 1020, 1018, 0, 0, 0, 0, 1042, 995, 1005, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4095, 4063, 4137, 4055, 4213, 4016, 4066] avg=4096.0000 max=4213 max_rank=5 min=4016 min_rank=6 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1036, 1020, 986, 1034, 13, 7, 0, 0, 1025, 1045, 1024, 1046, 4, 0, 0, 0, 1014, 1026, 1029, 1060, 30, 0, 0, 0, 1045, 1007, 987, 965, 2, 0, 0, 0, 1000, 1096, 1035, 1030, 0, 0, 0, 0, 1042, 1003, 1067, 946, 5, 0, 0, 0, 1057, 1036, 1029, 976, 0, 0, 0, 0, 1044, 1015, 986, 996, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4144, 4159, 4006, 4161, 4063, 4098, 4041] avg=4096.0000 max=4161 max_rank=4 min=4006 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1081, 1006, 1012, 1008, 12, 7, 0, 0, 993, 1033, 1036, 1038, 4, 0, 0, 0, 1042, 1007, 1011, 1022, 28, 0, 0, 0, 998, 1008, 1034, 1014, 2, 0, 0, 0, 1048, 1008, 1001, 993, 0, 0, 0, 0, 1062, 976, 1057, 966, 4, 0, 0, 0, 1040, 1021, 1004, 1071, 0, 0, 0, 0, 976, 1057, 1055, 1033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4104, 4110, 4056, 4050, 4065, 4136, 4121] avg=4096.0000 max=4136 max_rank=6 min=4050 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1004, 1013, 1033, 1064, 13, 7, 0, 0, 1033, 984, 1008, 1015, 4, 0, 0, 0, 1061, 1009, 1006, 997, 28, 0, 0, 0, 1062, 992, 1013, 1049, 2, 0, 0, 0, 1039, 1016, 995, 995, 0, 0, 0, 0, 1016, 1024, 1009, 1036, 4, 0, 0, 0, 1053, 992, 1019, 1025, 0, 0, 0, 0, 998, 1053, 1079, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4044, 4101, 4118, 4045, 4089, 4089, 4148] avg=4096.0000 max=4148 max_rank=7 min=4044 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[991, 1001, 1056, 1020, 13, 7, 0, 0, 1036, 1030, 1009, 1063, 4, 0, 0, 0, 1048, 1029, 997, 993, 29, 0, 0, 0, 1013, 1000, 1063, 1021, 1, 0, 0, 0, 1028, 1035, 1003, 1057, 0, 0, 0, 0, 964, 1057, 999, 962, 4, 0, 0, 0, 1023, 988, 1011, 1042, 0, 0, 0, 0, 1011, 1082, 1027, 1051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4142, 4096, 4098, 4123, 3986, 4064, 4171] avg=4096.0000 max=4171 max_rank=7 min=3986 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1005, 1031, 993, 988, 17, 9, 0, 0, 1043, 1052, 1044, 992, 7, 0, 0, 0, 1035, 1011, 985, 952, 30, 0, 0, 0, 1060, 1032, 1024, 1040, 5, 0, 0, 0, 995, 1025, 1017, 1123, 0, 0, 0, 0, 1050, 1004, 1002, 1052, 9, 2, 0, 0, 1018, 1038, 1034, 1030, 0, 0, 0, 0, 944, 1024, 1067, 979, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4138, 4013, 4161, 4160, 4119, 4120, 4014] avg=4096.0000 max=4161 max_rank=3 min=4013 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[996, 982, 1047, 1039, 13, 7, 0, 0, 1043, 1007, 1031, 970, 4, 0, 0, 0, 1035, 992, 1012, 1024, 29, 0, 0, 0, 1028, 1025, 1017, 1037, 2, 0, 0, 0, 1023, 1011, 1056, 1035, 0, 0, 0, 0, 1028, 1018, 1034, 1045, 5, 0, 0, 0, 1033, 1062, 1039, 977, 0, 0, 0, 0, 1011, 1031, 1011, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4055, 4092, 4109, 4125, 4130, 4111, 4062] avg=4096.0000 max=4130 max_rank=5 min=4055 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1033, 1012, 1016, 993, 13, 7, 0, 0, 991, 1009, 996, 1047, 3, 0, 0, 0, 1005, 1023, 1066, 1011, 29, 0, 0, 0, 1011, 1007, 1047, 1017, 1, 0, 0, 0, 1004, 1063, 1004, 978, 0, 0, 0, 0, 1014, 1041, 1032, 1012, 4, 0, 0, 0, 988, 1065, 1024, 1057, 0, 0, 0, 0, 1020, 1069, 1041, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4046, 4134, 4083, 4049, 4103, 4134, 4145] avg=4096.0000 max=4145 max_rank=7 min=4046 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8448, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8064, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1022, 1001, 1026, 1021, 1019, 1066, 986, 1006, 1052, 1038, 1031, 1016, 1069, 1004, 1057, 1090, 1021, 1002, 995, 1021, 1085, 962, 1020, 1070, 1019, 969, 1055, 981, 1035, 994, 1027, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1016, 1017, 1016, 1023, 1030, 1057, 1017, 1006, 1046, 983, 1057, 1060, 1033, 1051, 1025, 1006, 1045, 1007, 988, 1012, 1054, 1027, 1067, 1014, 1038, 994, 970, 1007, 1026, 1025, 1031, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[950, 1054, 1057, 989, 1007, 1054, 1011, 1046, 963, 1000, 1051, 1082, 1077, 1008, 1047, 993, 993, 1007, 999, 1053, 1061, 1026, 1013, 1073, 1057, 992, 1010, 1008, 1035, 992, 1010, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[972, 1010, 1036, 1109, 1044, 983, 1013, 1060, 1026, 995, 1041, 979, 1054, 1024, 1016, 991, 1045, 1005, 1077, 994, 968, 1031, 1084, 1009, 1027, 1030, 990, 1058, 1018, 1034, 1029, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1022, 975, 1032, 1025, 1031, 1039, 1008, 978, 1023, 1033, 1034, 1052, 983, 1033, 1029, 1039, 1060, 1017, 994, 994, 1031, 1030, 1047, 1045, 1067, 1004, 1008, 1067, 1018, 1047, 1006, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1074, 1010, 1048, 1035, 1025, 1014, 999, 1027, 1046, 1021, 1043, 1019, 1031, 992, 1066, 1054, 962, 1042, 1019, 1057, 1048, 1014, 1012, 999, 1027, 983, 1018, 1020, 1013, 1014, 986, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1037, 1021, 1064, 1007, 1069, 1065, 1037, 990, 960, 982, 987, 1023, 1045, 1042, 1019, 1049, 1081, 983, 1085, 1038, 1054, 1024, 1059, 965, 1024, 1017, 992, 982, 1033, 990, 1019, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1069, 1037, 1010, 996, 1004, 989, 1052, 1005, 1030, 1059, 1005, 1017, 1028, 1034, 1056, 993, 1011, 989, 1015, 1002, 1059, 1045, 1043, 1005, 1013, 968, 1024, 1056, 1011, 1064, 1008, 1071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1022, 1001, 1025, 1021, 0, 0, 0, 0, 1019, 1066, 986, 1006, 3, 0, 0, 0, 1052, 1038, 1031, 1016, 1, 0, 0, 0, 1037, 1004, 1057, 1090, 0, 0, 0, 0, 1021, 1002, 995, 1021, 19, 0, 0, 0, 1050, 962, 1020, 1070, 0, 0, 0, 0, 1019, 969, 1055, 981, 35, 2, 0, 0, 1035, 994, 1027, 1008, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4080, 4138, 4188, 4058, 4102, 4061, 4072] avg=4096.0000 max=4188 max_rank=3 min=4058 min_rank=4 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[950, 1054, 1056, 989, 0, 0, 0, 0, 1007, 1054, 1011, 1046, 4, 0, 0, 0, 963, 1000, 1051, 1082, 1, 0, 0, 0, 1043, 1008, 1047, 993, 0, 0, 0, 0, 993, 1007, 999, 1053, 20, 0, 0, 0, 1026, 1026, 1013, 1073, 0, 0, 0, 0, 1057, 992, 1010, 1008, 35, 2, 0, 0, 1035, 992, 1010, 1050, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4122, 4097, 4091, 4072, 4138, 4104, 4095] avg=4096.0000 max=4138 max_rank=5 min=4049 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1016, 1017, 1010, 1023, 0, 0, 0, 0, 1030, 1057, 1017, 1006, 9, 0, 0, 0, 1046, 983, 1057, 1060, 6, 1, 0, 0, 987, 1051, 1025, 1006, 0, 0, 0, 0, 1045, 1007, 988, 1012, 22, 0, 0, 0, 1015, 1027, 1067, 1014, 0, 0, 0, 0, 1038, 994, 970, 1007, 39, 3, 0, 0, 1026, 1025, 1031, 1020, 11, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4119, 4153, 4069, 4074, 4123, 4051, 4113] avg=4096.0000 max=4153 max_rank=2 min=4051 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1022, 975, 1031, 1025, 0, 0, 0, 0, 1031, 1039, 1008, 978, 3, 0, 0, 0, 1023, 1033, 1034, 1052, 1, 0, 0, 0, 953, 1033, 1029, 1039, 0, 0, 0, 0, 1060, 1017, 994, 994, 18, 0, 0, 0, 997, 1030, 1047, 1045, 0, 0, 0, 0, 1067, 1004, 1008, 1067, 34, 2, 0, 0, 1018, 1047, 1006, 997, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4059, 4143, 4054, 4083, 4119, 4182, 4075] avg=4096.0000 max=4182 max_rank=6 min=4053 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1037, 1021, 1063, 1007, 0, 0, 0, 0, 1069, 1065, 1037, 990, 3, 0, 0, 0, 960, 982, 987, 1023, 1, 0, 0, 0, 1013, 1042, 1019, 1049, 0, 0, 0, 0, 1081, 983, 1085, 1038, 19, 0, 0, 0, 1020, 1024, 1059, 965, 0, 0, 0, 0, 1024, 1017, 992, 982, 34, 2, 0, 0, 1033, 990, 1019, 1025, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4164, 3953, 4123, 4206, 4068, 4051, 4075] avg=4096.0000 max=4206 max_rank=4 min=3953 min_rank=2 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[972, 1010, 1035, 1109, 0, 0, 0, 0, 1044, 983, 1013, 1060, 3, 0, 0, 0, 1026, 995, 1041, 979, 1, 0, 0, 0, 1022, 1024, 1016, 991, 0, 0, 0, 0, 1045, 1005, 1077, 994, 19, 0, 0, 0, 937, 1031, 1084, 1009, 0, 0, 0, 0, 1027, 1030, 990, 1058, 31, 2, 0, 0, 1018, 1034, 1029, 1016, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4103, 4042, 4053, 4140, 4061, 4138, 4105] avg=4096.0000 max=4140 max_rank=4 min=4042 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1074, 1010, 1047, 1035, 0, 0, 0, 0, 1025, 1014, 999, 1027, 3, 0, 0, 0, 1046, 1021, 1043, 1019, 1, 0, 0, 0, 999, 992, 1066, 1054, 0, 0, 0, 0, 962, 1042, 1019, 1057, 19, 0, 0, 0, 1014, 1014, 1012, 999, 0, 0, 0, 0, 1027, 983, 1018, 1020, 34, 2, 0, 0, 1013, 1014, 986, 1050, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4068, 4130, 4111, 4099, 4039, 4084, 4071] avg=4096.0000 max=4166 max_rank=0 min=4039 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1069, 1037, 1009, 996, 0, 0, 0, 0, 1004, 989, 1052, 1005, 3, 0, 0, 0, 1030, 1059, 1005, 1017, 1, 0, 0, 0, 996, 1034, 1056, 993, 0, 0, 0, 0, 1011, 989, 1015, 1002, 19, 0, 0, 0, 1025, 1045, 1043, 1005, 0, 0, 0, 0, 1013, 968, 1024, 1056, 34, 2, 0, 0, 1011, 1064, 1008, 1071, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4053, 4112, 4079, 4036, 4118, 4097, 4162] avg=4096.0000 max=4162 max_rank=7 min=4036 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8448, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1029, 1004, 1017, 980, 999, 1026, 1059, 1008, 1053, 1043, 1004, 1077, 1001, 983, 1016, 1065, 1019, 1002, 1073, 1009, 998, 1012, 994, 998, 988, 1045, 1083, 1027, 978, 1039, 1079, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[996, 1004, 1048, 1037, 1045, 1023, 1028, 1019, 1023, 1028, 1020, 940, 1039, 1066, 1074, 965, 1043, 1008, 972, 1032, 1019, 969, 1068, 1044, 994, 1012, 1045, 1020, 1067, 1065, 1026, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1014, 1020, 1008, 969, 998, 1033, 1040, 990, 1046, 1045, 1046, 1046, 1043, 1009, 1023, 997, 1027, 1005, 1030, 1032, 997, 1051, 1027, 1013, 1032, 1038, 1008, 1098, 1034, 1036, 969, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1003, 992, 1009, 1033, 1042, 1058, 966, 1049, 1020, 1014, 1056, 986, 1012, 1044, 1032, 1033, 1042, 1000, 954, 1027, 975, 993, 1034, 1048, 1013, 1047, 1060, 1069, 1052, 1045, 1009, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[997, 1012, 1051, 1020, 1014, 1024, 1034, 989, 1004, 999, 1039, 1044, 1034, 981, 1015, 1041, 1023, 1004, 1014, 1062, 1062, 1002, 1054, 1015, 1034, 1046, 1043, 1069, 991, 1052, 1012, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[974, 1040, 982, 1004, 1006, 1011, 1012, 1046, 1032, 1067, 1080, 1054, 1036, 1042, 1000, 999, 1018, 1053, 972, 1009, 1022, 1013, 1033, 1064, 1045, 991, 1053, 993, 948, 1031, 1066, 1072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1050, 1087, 993, 1004, 980, 1012, 1013, 1049, 1055, 1017, 1019, 999, 1024, 1058, 1021, 1046, 1035, 1024, 993, 998, 995, 1043, 1042, 1022, 1012, 1027, 989, 1091, 1017, 1045, 1018, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1041, 1053, 1005, 1014, 1013, 1018, 979, 1002, 988, 998, 1043, 1032, 1040, 1031, 992, 1009, 1047, 1055, 1002, 1037, 1074, 1008, 1034, 1046, 1008, 1072, 1011, 1011, 1087, 1001, 1031, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1003, 992, 1009, 1033, 35, 0, 0, 0, 1042, 1058, 966, 1049, 18, 3, 0, 0, 1020, 1014, 1038, 986, 0, 0, 0, 0, 1012, 1044, 1032, 1033, 0, 0, 0, 0, 1042, 1000, 954, 1027, 18, 0, 0, 0, 975, 993, 1034, 1048, 0, 0, 0, 0, 1013, 1047, 1060, 1031, 0, 0, 0, 0, 1052, 1027, 1009, 1051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4136, 4058, 4121, 4041, 4050, 4151, 4139] avg=4096.0000 max=4151 max_rank=6 min=4041 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1014, 1020, 1008, 969, 36, 0, 0, 0, 998, 1033, 1040, 990, 18, 3, 0, 0, 1046, 1045, 1028, 1046, 0, 0, 0, 0, 1043, 1009, 1023, 997, 0, 0, 0, 0, 1027, 1005, 1030, 1032, 18, 0, 0, 0, 997, 1051, 1027, 1013, 0, 0, 0, 0, 1032, 1038, 1008, 1059, 0, 0, 0, 0, 1034, 1018, 969, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4082, 4165, 4072, 4112, 4088, 4137, 4065] avg=4096.0000 max=4165 max_rank=2 min=4047 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[997, 1012, 1051, 1020, 35, 0, 0, 0, 1014, 1024, 1034, 989, 18, 3, 0, 0, 1004, 999, 1022, 1044, 0, 0, 0, 0, 1034, 981, 1015, 1041, 0, 0, 0, 0, 1023, 1004, 1014, 1062, 17, 0, 0, 0, 1062, 1002, 1054, 1015, 0, 0, 0, 0, 1034, 1046, 1043, 1031, 0, 0, 0, 0, 991, 1034, 1012, 987, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4082, 4069, 4071, 4120, 4133, 4154, 4024] avg=4096.0000 max=4154 max_rank=6 min=4024 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1029, 1004, 1017, 980, 34, 0, 0, 0, 999, 1026, 1059, 1008, 18, 3, 0, 0, 1053, 1043, 987, 1077, 0, 0, 0, 0, 1001, 983, 1016, 1065, 0, 0, 0, 0, 1019, 1002, 1073, 1009, 17, 0, 0, 0, 998, 1012, 994, 998, 0, 0, 0, 0, 988, 1045, 1083, 990, 0, 0, 0, 0, 978, 1021, 1079, 1060, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4113, 4160, 4065, 4120, 4002, 4106, 4138] avg=4096.0000 max=4160 max_rank=2 min=4002 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[974, 1040, 982, 1004, 32, 0, 0, 0, 1006, 1011, 1012, 1046, 18, 3, 0, 0, 1032, 1067, 1062, 1054, 0, 0, 0, 0, 1036, 1042, 1000, 999, 0, 0, 0, 0, 1018, 1053, 972, 1009, 18, 0, 0, 0, 1022, 1013, 1033, 1064, 0, 0, 0, 0, 1045, 991, 1053, 958, 0, 0, 0, 0, 948, 1013, 1066, 1072, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4096, 4215, 4077, 4070, 4132, 4047, 4099] avg=4096.0000 max=4215 max_rank=2 min=4032 min_rank=0 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[996, 1004, 1048, 1037, 33, 0, 0, 0, 1045, 1023, 1028, 1019, 19, 3, 0, 0, 1023, 1028, 1003, 940, 0, 0, 0, 0, 1039, 1066, 1074, 965, 0, 0, 0, 0, 1043, 1008, 972, 1032, 17, 0, 0, 0, 1019, 969, 1068, 1044, 0, 0, 0, 0, 994, 1012, 1045, 984, 0, 0, 0, 0, 1067, 1046, 1026, 1029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4137, 3994, 4144, 4072, 4100, 4035, 4168] avg=4096.0000 max=4168 max_rank=7 min=3994 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1041, 1053, 1005, 1014, 33, 0, 0, 0, 1013, 1018, 979, 1002, 17, 3, 0, 0, 988, 998, 1026, 1032, 0, 0, 0, 0, 1040, 1031, 992, 1009, 0, 0, 0, 0, 1047, 1055, 1002, 1037, 17, 0, 0, 0, 1074, 1008, 1034, 1046, 0, 0, 0, 0, 1008, 1072, 1011, 975, 0, 0, 0, 0, 1087, 984, 1031, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4032, 4044, 4072, 4158, 4162, 4066, 4088] avg=4096.0000 max=4162 max_rank=5 min=4032 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1050, 1087, 993, 1004, 40, 0, 0, 0, 980, 1012, 1013, 1049, 23, 7, 6, 0, 1055, 1017, 992, 999, 0, 0, 0, 0, 1021, 1058, 1021, 1046, 0, 0, 0, 0, 1035, 1024, 993, 998, 21, 3, 1, 0, 995, 1043, 1041, 1022, 0, 0, 0, 0, 1012, 1027, 989, 1044, 0, 0, 0, 0, 1017, 1022, 1018, 990, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4090, 4063, 4146, 4075, 4101, 4072, 4047] avg=4096.0000 max=4174 max_rank=0 min=4047 min_rank=7 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8064, 8320, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1072, 994, 1049, 1024, 1005, 1067, 1059, 1017, 1055, 990, 1020, 1000, 1062, 993, 1041, 968, 1032, 1040, 966, 1016, 1037, 1011, 1027, 1002, 1058, 993, 1003, 1016, 1038, 1002, 1069, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1041, 1015, 1062, 1013, 1044, 1042, 1076, 1039, 968, 1054, 1005, 1010, 1044, 1041, 1077, 1018, 999, 983, 1001, 985, 1005, 1064, 1048, 966, 1053, 1009, 998, 1016, 1048, 1003, 1018, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[973, 1003, 963, 1037, 988, 987, 983, 1054, 1054, 1034, 1009, 1009, 973, 1024, 1045, 1069, 1058, 1048, 1050, 994, 1013, 1023, 1032, 1088, 991, 1025, 1044, 1033, 1001, 1062, 1058, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1023, 988, 1046, 1024, 985, 987, 1028, 1036, 1032, 1054, 1047, 981, 1006, 1012, 1049, 1017, 1030, 1015, 1045, 1072, 1059, 1016, 1026, 1064, 1026, 1048, 1006, 1023, 1027, 999, 999, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1008, 1022, 1015, 1050, 1011, 1068, 1013, 1020, 1041, 1004, 1000, 1018, 1051, 1047, 1058, 1000, 1004, 1036, 1048, 1059, 1029, 1050, 986, 1037, 1016, 1020, 994, 1022, 995, 1030, 1011, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1001, 989, 992, 997, 984, 1015, 1058, 1044, 995, 1055, 1050, 1034, 1029, 992, 1051, 1009, 1015, 1027, 1015, 1054, 1072, 1031, 1041, 980, 982, 1012, 1018, 1058, 1027, 1052, 1054, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1004, 1017, 1046, 1015, 1054, 1044, 1024, 977, 1023, 998, 1037, 1030, 1022, 1013, 1043, 1100, 1022, 981, 1028, 1006, 1007, 1017, 1002, 1095, 959, 1016, 986, 1091, 1036, 1001, 1021, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[983, 987, 1021, 1046, 1009, 1019, 1052, 1054, 1040, 1072, 1022, 1041, 1009, 1048, 1037, 995, 986, 1022, 1013, 997, 1055, 1029, 1024, 1044, 1009, 998, 1024, 1030, 994, 1023, 1024, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1041, 1015, 1062, 1013, 25, 4, 0, 0, 1044, 1042, 1067, 1039, 0, 0, 0, 0, 968, 1053, 1005, 1010, 0, 0, 0, 0, 1044, 1041, 1056, 1018, 0, 0, 0, 0, 999, 983, 1001, 985, 9, 3, 1, 0, 980, 1064, 1048, 966, 0, 0, 0, 0, 1053, 1009, 998, 1016, 17, 6, 0, 0, 1048, 1003, 1018, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4192, 4036, 4159, 3981, 4058, 4099, 4083] avg=4096.0000 max=4192 max_rank=1 min=3981 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16221 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[973, 1003, 963, 1037, 25, 4, 0, 0, 988, 987, 975, 1054, 0, 0, 0, 0, 1054, 1033, 1009, 1009, 0, 0, 0, 0, 973, 1024, 1024, 1069, 0, 0, 0, 0, 1058, 1048, 1050, 994, 8, 4, 1, 0, 988, 1023, 1032, 1088, 0, 0, 0, 0, 991, 1025, 1044, 1033, 17, 6, 0, 0, 1001, 1062, 1058, 1033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 4004, 4105, 4090, 4163, 4131, 4116, 4154] avg=4096.0000 max=4163 max_rank=4 min=4004 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16564 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1023, 988, 1046, 1024, 27, 4, 0, 0, 985, 987, 1019, 1036, 0, 0, 0, 0, 1032, 1053, 1047, 981, 0, 0, 0, 0, 1006, 1012, 1028, 1017, 0, 0, 0, 0, 1030, 1015, 1045, 1072, 9, 3, 1, 0, 1032, 1016, 1026, 1064, 0, 0, 0, 0, 1026, 1048, 1006, 1023, 17, 6, 0, 0, 1027, 999, 999, 989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4027, 4113, 4063, 4175, 4138, 4126, 4014] avg=4096.0000 max=4175 max_rank=4 min=4014 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1072, 994, 1049, 1024, 26, 4, 0, 0, 1005, 1067, 1050, 1017, 0, 0, 0, 0, 1055, 989, 1020, 1000, 0, 0, 0, 0, 1062, 993, 1020, 968, 0, 0, 0, 0, 1032, 1040, 966, 1016, 9, 4, 1, 0, 1011, 1011, 1027, 1002, 0, 0, 0, 0, 1058, 993, 1003, 1016, 17, 6, 0, 0, 1038, 1002, 1069, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4139, 4064, 4043, 4068, 4051, 4093, 4141] avg=4096.0000 max=4169 max_rank=0 min=4043 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1008, 1022, 1015, 1050, 26, 4, 0, 0, 1011, 1068, 1004, 1020, 0, 0, 0, 0, 1041, 1003, 1000, 1018, 0, 0, 0, 0, 1051, 1047, 1037, 1000, 0, 0, 0, 0, 1004, 1036, 1048, 1059, 9, 3, 1, 0, 1003, 1050, 986, 1037, 0, 0, 0, 0, 1016, 1020, 994, 1022, 17, 6, 0, 0, 995, 1030, 1011, 996, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4103, 4062, 4135, 4160, 4076, 4075, 4032] avg=4096.0000 max=4160 max_rank=4 min=4032 min_rank=7 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1004, 1017, 1046, 1015, 25, 4, 0, 0, 1054, 1044, 1015, 977, 0, 0, 0, 0, 1023, 997, 1037, 1030, 0, 0, 0, 0, 1022, 1013, 1022, 1100, 0, 0, 0, 0, 1022, 981, 1028, 1006, 9, 4, 1, 0, 982, 1017, 1002, 1095, 0, 0, 0, 0, 959, 1016, 986, 1091, 17, 6, 0, 0, 1036, 1001, 1021, 1043, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4090, 4087, 4157, 4051, 4096, 4075, 4101] avg=4096.0000 max=4157 max_rank=3 min=4051 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1001, 989, 992, 997, 31, 8, 0, 0, 984, 1015, 1045, 1044, 0, 0, 0, 0, 995, 1048, 1050, 1034, 0, 0, 0, 0, 1029, 992, 1023, 1009, 0, 0, 0, 0, 1015, 1027, 1015, 1054, 13, 7, 7, 0, 1041, 1031, 1041, 980, 0, 0, 0, 0, 982, 1012, 1018, 1058, 20, 10, 0, 0, 1027, 1052, 1054, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4088, 4127, 4053, 4138, 4093, 4100, 4151] avg=4096.0000 max=4151 max_rank=7 min=4018 min_rank=0 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[983, 987, 1021, 1046, 27, 4, 0, 0, 1009, 1019, 1043, 1054, 0, 0, 0, 0, 1040, 1071, 1022, 1041, 0, 0, 0, 0, 1009, 1048, 1016, 995, 0, 0, 0, 0, 986, 1022, 1013, 997, 9, 4, 1, 0, 1028, 1029, 1024, 1044, 0, 0, 0, 0, 1009, 998, 1024, 1030, 17, 6, 0, 0, 994, 1023, 1024, 1051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4125, 4174, 4068, 4032, 4125, 4084, 4092] avg=4096.0000 max=4174 max_rank=2 min=4032 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1034, 1002, 1063, 1050, 981, 982, 995, 1027, 998, 976, 1040, 962, 1010, 1010, 998, 1004, 1002, 1008, 1052, 988, 1076, 1094, 1061, 1030, 1048, 1064, 1068, 1075, 1036, 971, 1029, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1007, 1061, 985, 1040, 1071, 1036, 1016, 988, 988, 1024, 1015, 1095, 1019, 1086, 1004, 1004, 1055, 1012, 1012, 1049, 1032, 1025, 998, 1000, 1023, 1001, 1046, 1021, 1004, 1012, 1021, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1010, 1043, 965, 1043, 1036, 1041, 993, 1060, 1031, 1029, 1056, 1039, 997, 1036, 937, 1032, 1010, 1041, 1039, 1001, 1051, 1052, 1041, 1061, 1034, 977, 1030, 1019, 1054, 979, 1017, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[952, 1022, 1062, 1056, 1015, 1053, 1059, 1006, 1000, 1012, 1032, 1019, 1036, 1025, 1010, 1040, 1028, 1042, 1017, 1004, 1041, 1046, 990, 1005, 1046, 1058, 997, 1037, 1029, 1010, 1044, 975] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1019, 961, 1049, 1010, 1037, 999, 998, 1018, 1035, 1054, 1043, 996, 1039, 1017, 1084, 997, 1050, 1020, 1026, 1055, 1022, 1026, 1043, 1009, 1044, 1063, 997, 1016, 1005, 1048, 1008, 980] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1015, 1027, 1028, 1038, 1012, 983, 1042, 1071, 1027, 981, 1047, 1036, 1058, 1021, 993, 1029, 1016, 1009, 1027, 1011, 1068, 1048, 994, 1009, 993, 1050, 1040, 1039, 1060, 1014, 989, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1022, 1016, 1050, 1017, 1016, 997, 1041, 1014, 1046, 1004, 952, 1044, 980, 1019, 1063, 1043, 994, 1070, 1091, 1042, 979, 1010, 986, 1041, 1028, 1021, 1012, 1011, 1024, 1050, 1021, 1064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1042, 1046, 1036, 1037, 989, 1035, 1039, 1032, 987, 1057, 1053, 1004, 1020, 1000, 1014, 1032, 1050, 1024, 1022, 999, 1014, 1023, 1035, 1000, 1037, 999, 994, 1027, 1092, 1000, 1022, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1007, 1061, 985, 1035, 0, 0, 0, 0, 1071, 1036, 1016, 988, 5, 5, 0, 0, 988, 1024, 1015, 1095, 6, 4, 0, 0, 1019, 1086, 1004, 1004, 13, 0, 0, 0, 1055, 1012, 1001, 1049, 0, 0, 0, 0, 1032, 1008, 998, 1000, 0, 0, 0, 0, 1006, 1001, 1046, 1021, 0, 0, 0, 0, 1004, 1012, 1021, 1018, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4121, 4132, 4126, 4117, 4038, 4074, 4072] avg=4096.0000 max=4132 max_rank=2 min=4038 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1034, 1002, 1063, 1045, 0, 0, 0, 0, 981, 982, 995, 1027, 5, 5, 0, 0, 998, 976, 1040, 962, 6, 4, 0, 0, 1010, 1010, 998, 1004, 14, 0, 0, 0, 1002, 1008, 1041, 988, 0, 0, 0, 0, 1076, 1076, 1061, 1030, 0, 0, 0, 0, 1030, 1064, 1068, 1075, 0, 0, 0, 0, 1036, 971, 1029, 1034, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 3995, 3986, 4036, 4039, 4243, 4237, 4088] avg=4096.0000 max=4243 max_rank=5 min=3986 min_rank=2 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16607 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1010, 1043, 965, 1038, 0, 0, 0, 0, 1036, 1041, 993, 1060, 5, 5, 0, 0, 1031, 1029, 1056, 1039, 6, 4, 0, 0, 997, 1036, 937, 1032, 13, 0, 0, 0, 1010, 1041, 1028, 1001, 0, 0, 0, 0, 1051, 1035, 1041, 1061, 0, 0, 0, 0, 1016, 977, 1030, 1019, 0, 0, 0, 0, 1054, 979, 1017, 1014, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4140, 4165, 4015, 4080, 4188, 4042, 4082] avg=4096.0000 max=4188 max_rank=5 min=4015 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[952, 1022, 1062, 1051, 0, 0, 0, 0, 1015, 1053, 1059, 1006, 5, 5, 0, 0, 1000, 1012, 1032, 1019, 6, 4, 0, 0, 1036, 1025, 1010, 1040, 13, 0, 0, 0, 1028, 1042, 1006, 1004, 0, 0, 0, 0, 1041, 1029, 990, 1005, 0, 0, 0, 0, 1028, 1058, 997, 1037, 0, 0, 0, 0, 1029, 1010, 1044, 975, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4143, 4073, 4124, 4080, 4065, 4120, 4076] avg=4096.0000 max=4143 max_rank=1 min=4065 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1015, 1027, 1028, 1033, 0, 0, 0, 0, 1012, 983, 1042, 1071, 5, 5, 0, 0, 1027, 981, 1047, 1036, 6, 4, 0, 0, 1058, 1021, 993, 1029, 13, 0, 0, 0, 1016, 1009, 1016, 1011, 0, 0, 0, 0, 1068, 1031, 994, 1009, 0, 0, 0, 0, 976, 1050, 1040, 1039, 0, 0, 0, 0, 1060, 1014, 989, 993, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4118, 4101, 4114, 4052, 4102, 4105, 4073] avg=4096.0000 max=4118 max_rank=1 min=4052 min_rank=4 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1019, 961, 1049, 1006, 0, 0, 0, 0, 1037, 999, 998, 1018, 5, 4, 0, 0, 1035, 1054, 1043, 996, 6, 4, 0, 0, 1039, 1017, 1084, 997, 13, 0, 0, 0, 1050, 1020, 1015, 1055, 0, 0, 0, 0, 1022, 1009, 1043, 1009, 0, 0, 0, 0, 1026, 1063, 997, 1016, 0, 0, 0, 0, 1005, 1048, 1008, 980, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4061, 4138, 4150, 4140, 4083, 4102, 4059] avg=4096.0000 max=4150 max_rank=3 min=4035 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1022, 1016, 1050, 1013, 0, 0, 0, 0, 1016, 997, 1041, 1014, 6, 4, 0, 0, 1046, 1004, 952, 1044, 6, 4, 0, 0, 980, 1019, 1063, 1043, 13, 0, 0, 0, 994, 1070, 1079, 1042, 0, 0, 0, 0, 979, 993, 986, 1041, 0, 0, 0, 0, 1011, 1021, 1012, 1011, 0, 0, 0, 0, 1024, 1050, 1021, 1064, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4078, 4056, 4118, 4185, 3999, 4055, 4176] avg=4096.0000 max=4185 max_rank=4 min=3999 min_rank=5 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1042, 1046, 1036, 1030, 0, 0, 0, 0, 989, 1035, 1039, 1032, 10, 7, 0, 0, 987, 1057, 1053, 1004, 10, 6, 0, 0, 1020, 1000, 1014, 1032, 16, 3, 0, 0, 1050, 1024, 1002, 999, 0, 0, 0, 0, 1014, 1001, 1035, 1000, 0, 0, 0, 0, 1013, 999, 994, 1027, 0, 0, 0, 0, 1092, 1000, 1022, 1007, 21, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4112, 4117, 4085, 4075, 4050, 4033, 4142] avg=4096.0000 max=4154 max_rank=0 min=4033 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[988, 991, 1015, 1038, 1042, 1055, 983, 1060, 1019, 1056, 1023, 1048, 1039, 1008, 1029, 992, 1036, 1023, 971, 1063, 1022, 1003, 967, 1027, 1044, 1054, 1026, 1028, 998, 1030, 1033, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1022, 1022, 992, 1081, 1008, 1043, 1062, 1011, 1034, 1024, 1048, 1038, 977, 1028, 1040, 1004, 1040, 1032, 992, 1008, 1011, 1055, 1036, 989, 1023, 1073, 1043, 1027, 1037, 992, 938, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1025, 1064, 1047, 1000, 990, 1023, 1042, 1006, 989, 1026, 1083, 1004, 1015, 1023, 981, 1005, 1023, 995, 1059, 1013, 1012, 1021, 1030, 1022, 1060, 1024, 1039, 1062, 1011, 1016, 1037, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1079, 979, 1000, 1057, 987, 1090, 1044, 1061, 1034, 1032, 1006, 960, 1039, 968, 985, 1055, 1028, 1022, 1031, 1039, 1045, 1018, 1073, 1021, 1016, 1018, 1026, 972, 1019, 1016, 1006, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[991, 971, 1047, 1000, 1002, 999, 1039, 1038, 1071, 1029, 1021, 1045, 999, 989, 1110, 1009, 968, 1009, 1034, 1021, 1029, 1049, 1044, 1062, 992, 1071, 1068, 993, 1024, 997, 1019, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1040, 986, 995, 1027, 1062, 1019, 1037, 1028, 1053, 1053, 1039, 1048, 1036, 1021, 994, 998, 1032, 992, 1015, 1010, 1043, 1044, 1044, 965, 1037, 1032, 1049, 998, 1008, 997, 1035, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1043, 980, 1000, 1021, 975, 1007, 1054, 946, 1072, 1061, 1014, 1050, 1072, 1054, 1036, 1060, 1017, 1047, 1069, 987, 989, 1042, 1030, 1047, 1001, 1008, 1047, 1045, 1012, 1004, 995, 983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1024, 1001, 991, 1008, 1071, 1023, 1079, 1037, 1015, 1031, 988, 1025, 1001, 1009, 984, 1004, 1016, 1009, 1036, 1023, 1030, 1031, 1029, 1054, 1039, 967, 1078, 1048, 1013, 1041, 1049, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1025, 1064, 1047, 1000, 29, 0, 0, 0, 990, 1023, 1023, 1006, 0, 0, 0, 0, 989, 994, 1083, 1004, 0, 0, 0, 0, 1015, 1023, 981, 1005, 19, 5, 0, 0, 1023, 995, 1059, 1013, 13, 0, 0, 0, 1012, 1008, 1030, 1022, 0, 0, 0, 0, 1060, 1024, 1010, 1062, 0, 0, 0, 0, 1011, 1016, 1037, 1021, 24, 3, 0, 0] rerouted_tokens_per_ep_rank=[4165, 4042, 4070, 4048, 4103, 4072, 4156, 4112] avg=4096.0000 max=4165 max_rank=0 min=4042 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1022, 1022, 992, 1081, 29, 0, 0, 0, 1008, 1043, 1043, 1011, 0, 0, 0, 0, 1034, 992, 1048, 1038, 0, 0, 0, 0, 977, 1028, 1040, 1004, 19, 5, 1, 0, 1040, 1032, 992, 1008, 13, 0, 0, 0, 1011, 1041, 1036, 989, 0, 0, 0, 0, 1023, 1073, 1014, 1027, 0, 0, 0, 0, 1037, 992, 938, 1038, 24, 3, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4105, 4112, 4074, 4085, 4077, 4137, 4032] avg=4096.0000 max=4146 max_rank=0 min=4032 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[991, 971, 1047, 1000, 30, 0, 0, 0, 1002, 999, 1020, 1038, 0, 0, 0, 0, 1071, 996, 1021, 1045, 0, 0, 0, 0, 999, 989, 1110, 1009, 19, 5, 1, 0, 968, 1009, 1034, 1021, 13, 0, 0, 0, 1029, 1035, 1044, 1062, 0, 0, 0, 0, 992, 1071, 1038, 993, 0, 0, 0, 0, 1024, 997, 1019, 1028, 25, 3, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4059, 4133, 4132, 4045, 4170, 4094, 4096] avg=4096.0000 max=4170 max_rank=5 min=4039 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[988, 991, 1015, 1038, 30, 0, 0, 0, 1042, 1055, 965, 1060, 0, 0, 0, 0, 1019, 1023, 1023, 1048, 0, 0, 0, 0, 1039, 1008, 1029, 992, 18, 5, 0, 0, 1036, 1023, 971, 1063, 13, 0, 0, 0, 1022, 990, 967, 1027, 0, 0, 0, 0, 1044, 1054, 997, 1028, 0, 0, 0, 0, 998, 1030, 1033, 1057, 24, 3, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4122, 4113, 4091, 4106, 4006, 4123, 4145] avg=4096.0000 max=4145 max_rank=7 min=4006 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1079, 979, 1000, 1057, 30, 0, 0, 0, 987, 1090, 1025, 1061, 0, 0, 0, 0, 1034, 999, 1006, 960, 0, 0, 0, 0, 1039, 968, 985, 1055, 19, 5, 0, 0, 1028, 1022, 1031, 1039, 13, 0, 0, 0, 1045, 1005, 1073, 1021, 0, 0, 0, 0, 1016, 1018, 997, 972, 0, 0, 0, 0, 1019, 1016, 1006, 1042, 24, 3, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4163, 3999, 4071, 4133, 4144, 4003, 4110] avg=4096.0000 max=4163 max_rank=1 min=3999 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1040, 986, 995, 1027, 34, 0, 0, 0, 1062, 1019, 1015, 1028, 0, 0, 0, 0, 1053, 1012, 1039, 1048, 0, 0, 0, 0, 1036, 1021, 994, 998, 22, 6, 5, 0, 1032, 992, 1015, 1010, 17, 0, 0, 0, 1043, 1022, 1044, 965, 0, 0, 0, 0, 1037, 1032, 1014, 998, 0, 0, 0, 0, 1008, 997, 1035, 1031, 29, 7, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4124, 4152, 4082, 4066, 4074, 4081, 4107] avg=4096.0000 max=4152 max_rank=2 min=4066 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1024, 1001, 991, 1008, 30, 0, 0, 0, 1071, 1023, 1059, 1037, 0, 0, 0, 0, 1015, 998, 988, 1025, 0, 0, 0, 0, 1001, 1009, 984, 1004, 20, 5, 0, 0, 1016, 1009, 1036, 1023, 13, 0, 0, 0, 1030, 1018, 1029, 1054, 0, 0, 0, 0, 1039, 967, 1048, 1048, 0, 0, 0, 0, 1013, 1041, 1049, 1014, 25, 3, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4190, 4026, 4023, 4097, 4131, 4102, 4145] avg=4096.0000 max=4190 max_rank=1 min=4023 min_rank=3 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16475 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1043, 980, 1000, 1021, 31, 0, 0, 0, 975, 1007, 1035, 946, 0, 0, 0, 0, 1072, 1027, 1014, 1050, 0, 0, 0, 0, 1072, 1054, 1036, 1060, 19, 5, 1, 0, 1017, 1047, 1069, 987, 13, 0, 0, 0, 989, 1028, 1030, 1047, 0, 0, 0, 0, 1001, 1008, 1018, 1045, 0, 0, 0, 0, 1012, 1004, 995, 983, 24, 3, 0, 0] rerouted_tokens_per_ep_rank=[4075, 3963, 4163, 4247, 4133, 4094, 4072, 4021] avg=4096.0000 max=4247 max_rank=3 min=3963 min_rank=1 imbalance_ratio=1.036865 max_load_reduction=28521 max_load_reduction_pct=0.870392 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1007, 1085, 1082, 980, 964, 1032, 1009, 1013, 1032, 1041, 997, 985, 1044, 1044, 1055, 1025, 1024, 996, 1050, 1061, 1023, 1016, 1043, 1034, 1029, 1001, 996, 1026, 1048, 1043, 990, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[987, 1034, 1007, 1083, 1060, 1017, 989, 1013, 1046, 1028, 1060, 1023, 1045, 1010, 1038, 1049, 1013, 1056, 1013, 975, 982, 954, 1051, 1031, 1047, 1002, 1019, 1041, 1035, 996, 1022, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1005, 1040, 978, 1058, 1007, 1078, 1020, 993, 976, 1050, 1013, 1032, 1045, 1082, 1032, 1016, 1008, 1079, 999, 1069, 1100, 1020, 1006, 1016, 1020, 1042, 966, 1017, 991, 1011, 988, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[990, 1059, 971, 1038, 1061, 1022, 1061, 1027, 1004, 1001, 1064, 1030, 1034, 1065, 995, 999, 1010, 1053, 1047, 1008, 1017, 1006, 1042, 982, 1052, 988, 1029, 986, 1007, 1048, 1061, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1028, 971, 1014, 951, 1008, 1008, 1028, 1044, 1061, 1033, 1053, 1056, 1005, 1033, 1018, 1038, 1028, 996, 1011, 1032, 1078, 982, 1012, 1007, 1049, 1040, 945, 1054, 999, 1051, 1091, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1076, 1045, 1047, 1010, 959, 1072, 977, 1021, 979, 1028, 1010, 1041, 1014, 1026, 1012, 1046, 963, 1020, 1077, 1018, 1013, 1008, 996, 1018, 1050, 1031, 1038, 1052, 1042, 1023, 998, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1032, 1046, 980, 1031, 978, 1009, 1024, 1018, 1001, 1074, 1037, 1018, 1010, 1025, 1014, 1013, 1009, 1008, 1044, 1063, 983, 1007, 996, 1003, 1050, 1065, 1035, 1029, 1069, 1004, 1012, 1081] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1014, 993, 1039, 1040, 1007, 1030, 1002, 950, 1025, 1005, 1075, 1050, 1026, 1042, 994, 1014, 1029, 1039, 1002, 1066, 1055, 1042, 1029, 1059, 1033, 1011, 1024, 1010, 1016, 1025, 1018, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[987, 1034, 1007, 1083, 4, 1, 0, 0, 1060, 1017, 989, 1013, 20, 12, 0, 0, 1046, 1028, 1040, 1023, 0, 0, 0, 0, 1045, 994, 1038, 1049, 0, 0, 0, 0, 1013, 1056, 1013, 964, 0, 0, 0, 0, 982, 954, 1051, 1031, 11, 4, 3, 0, 1046, 1002, 1019, 1041, 0, 0, 0, 0, 1035, 996, 1022, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4111, 4137, 4126, 4046, 4036, 4108, 4088] avg=4096.0000 max=4137 max_rank=2 min=4036 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1005, 1040, 978, 1058, 4, 1, 0, 0, 1007, 1078, 1020, 993, 19, 13, 0, 0, 976, 1050, 994, 1032, 0, 0, 0, 0, 1045, 1065, 1032, 1016, 0, 0, 0, 0, 1008, 1079, 999, 1057, 0, 0, 0, 0, 1100, 1020, 1006, 1016, 12, 4, 3, 0, 1019, 1042, 966, 1017, 0, 0, 0, 0, 991, 1011, 988, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4130, 4052, 4158, 4143, 4161, 4044, 3994] avg=4096.0000 max=4161 max_rank=5 min=3994 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1028, 971, 1014, 951, 4, 1, 0, 0, 1008, 1008, 1028, 1044, 20, 13, 0, 0, 1061, 1033, 1033, 1056, 0, 0, 0, 0, 1005, 1016, 1018, 1038, 0, 0, 0, 0, 1028, 996, 1011, 1020, 0, 0, 0, 0, 1078, 982, 1012, 1007, 12, 4, 3, 0, 1048, 1040, 945, 1054, 0, 0, 0, 0, 999, 1051, 1091, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3969, 4121, 4183, 4077, 4055, 4098, 4087, 4178] avg=4096.0000 max=4183 max_rank=2 min=3969 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1007, 1085, 1082, 980, 4, 1, 0, 0, 964, 1032, 1009, 1013, 19, 13, 0, 0, 1032, 1041, 978, 985, 0, 0, 0, 0, 1044, 1027, 1055, 1025, 0, 0, 0, 0, 1024, 996, 1050, 1049, 0, 0, 0, 0, 1023, 1016, 1043, 1034, 12, 4, 3, 0, 1028, 1001, 996, 1026, 0, 0, 0, 0, 1048, 1043, 990, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4050, 4036, 4151, 4119, 4135, 4051, 4067] avg=4096.0000 max=4159 max_rank=0 min=4036 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[990, 1059, 971, 1038, 4, 1, 0, 0, 1061, 1022, 1061, 1027, 20, 13, 0, 0, 1004, 1001, 1044, 1030, 0, 0, 0, 0, 1034, 1048, 995, 999, 0, 0, 0, 0, 1010, 1053, 1047, 997, 0, 0, 0, 0, 1017, 1006, 1042, 982, 11, 4, 3, 0, 1051, 988, 1029, 986, 0, 0, 0, 0, 1007, 1048, 1061, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4204, 4079, 4076, 4107, 4065, 4054, 4120] avg=4096.0000 max=4204 max_rank=1 min=4054 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1032, 1046, 980, 1031, 4, 1, 0, 0, 978, 1009, 1024, 1018, 19, 13, 0, 0, 1001, 1074, 1018, 1018, 0, 0, 0, 0, 1010, 1008, 1014, 1013, 0, 0, 0, 0, 1009, 1008, 1044, 1051, 0, 0, 0, 0, 983, 1007, 996, 1003, 12, 4, 3, 0, 1049, 1065, 1035, 1029, 0, 0, 0, 0, 1069, 1004, 1012, 1074, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4061, 4111, 4045, 4112, 4008, 4178, 4159] avg=4096.0000 max=4178 max_rank=6 min=4008 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1076, 1045, 1047, 1010, 10, 2, 0, 0, 959, 1072, 977, 1021, 23, 17, 0, 0, 979, 1028, 987, 1041, 0, 0, 0, 0, 1014, 1004, 1012, 1046, 0, 0, 0, 0, 963, 1020, 1077, 1002, 0, 0, 0, 0, 1013, 1008, 996, 1018, 16, 5, 5, 0, 1048, 1031, 1038, 1052, 0, 0, 0, 0, 1042, 1023, 998, 1043, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4069, 4035, 4076, 4062, 4061, 4169, 4106] avg=4096.0000 max=4190 max_rank=0 min=4035 min_rank=2 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1014, 993, 1039, 1040, 4, 1, 0, 0, 1007, 1030, 1002, 950, 20, 13, 0, 0, 1025, 1005, 1055, 1050, 0, 0, 0, 0, 1026, 1025, 994, 1014, 0, 0, 0, 0, 1029, 1039, 1002, 1054, 0, 0, 0, 0, 1055, 1042, 1029, 1059, 12, 4, 3, 0, 1032, 1011, 1024, 1010, 0, 0, 0, 0, 1016, 1025, 1018, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4022, 4135, 4059, 4124, 4204, 4077, 4056] avg=4096.0000 max=4204 max_rank=5 min=4022 min_rank=1 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 8192, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1056, 997, 1015, 1011, 1022, 1030, 1051, 1002, 1035, 1061, 1031, 1045, 1007, 984, 1060, 996, 1047, 1044, 998, 1041, 1020, 1009, 981, 978, 1068, 1045, 1020, 1055, 1024, 1010, 1031, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1051, 1051, 999, 1011, 1018, 1057, 993, 965, 993, 1007, 995, 1024, 1031, 1040, 1054, 1030, 958, 1026, 1012, 1031, 1010, 1072, 1019, 1066, 1013, 1027, 1007, 1061, 996, 1073, 1052, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1031, 1025, 988, 985, 1074, 1017, 1023, 1029, 993, 1059, 1036, 1046, 986, 1032, 1013, 1065, 952, 1017, 1012, 1015, 1004, 1035, 954, 1058, 1050, 1062, 1031, 1006, 1057, 1026, 1052, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1087, 1023, 985, 1022, 1051, 1004, 1034, 1061, 1021, 1057, 1047, 973, 1011, 1033, 977, 1079, 1030, 1015, 970, 1060, 993, 1016, 1014, 1018, 1023, 1018, 1023, 1044, 997, 1023, 1037, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1010, 1057, 1031, 1062, 1025, 985, 1032, 1048, 1035, 989, 979, 1033, 1040, 1003, 1018, 1055, 1016, 954, 1036, 963, 1103, 1040, 1083, 1068, 1056, 1010, 1000, 996, 1044, 1006, 969, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[972, 1000, 1025, 1037, 1031, 1068, 1033, 1027, 1014, 1025, 1049, 1039, 1034, 920, 1026, 995, 1048, 1018, 989, 985, 1020, 1008, 1081, 1091, 966, 1046, 1013, 1035, 976, 1082, 1048, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1017, 1015, 982, 986, 1006, 957, 1036, 954, 1054, 1031, 1032, 1040, 1038, 1003, 1015, 1014, 1028, 1071, 1055, 1031, 1017, 1045, 1004, 990, 1025, 1060, 1086, 1038, 1022, 1068, 1035, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1051, 1095, 1015, 1011, 1031, 990, 1031, 980, 1010, 1078, 1034, 1024, 1005, 1048, 978, 982, 1013, 1017, 1023, 1033, 1053, 1012, 1059, 987, 972, 1021, 1062, 1063, 1060, 1029, 1016, 985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1051, 1051, 999, 1011, 7, 0, 0, 0, 1018, 1057, 993, 965, 6, 6, 0, 0, 993, 994, 995, 1024, 0, 0, 0, 0, 1031, 1040, 1054, 1030, 14, 10, 0, 0, 958, 1026, 1012, 1031, 29, 3, 0, 0, 1010, 1072, 1019, 1049, 0, 0, 0, 0, 1013, 1027, 1007, 1032, 0, 0, 0, 0, 996, 1057, 1052, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4045, 4006, 4179, 4059, 4150, 4079, 4131] avg=4096.0000 max=4179 max_rank=3 min=4006 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1056, 997, 1015, 1011, 8, 0, 0, 0, 1022, 1030, 1051, 1002, 7, 5, 0, 0, 1035, 1046, 1031, 1045, 0, 0, 0, 0, 1007, 984, 1060, 996, 13, 9, 0, 0, 1047, 1044, 998, 1041, 29, 3, 0, 0, 1020, 1009, 981, 962, 0, 0, 0, 0, 1068, 1045, 1020, 1026, 0, 0, 0, 0, 1024, 996, 1031, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4117, 4157, 4069, 4162, 3972, 4159, 4045] avg=4096.0000 max=4162 max_rank=4 min=3972 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1031, 1025, 988, 985, 8, 0, 0, 0, 1074, 1017, 1023, 1029, 7, 5, 0, 0, 993, 1044, 1036, 1046, 0, 0, 0, 0, 986, 1032, 1013, 1065, 14, 10, 0, 0, 952, 1017, 1012, 1015, 28, 3, 0, 0, 1004, 1035, 954, 1041, 0, 0, 0, 0, 1050, 1062, 1031, 978, 0, 0, 0, 0, 1057, 1011, 1052, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4155, 4119, 4120, 4027, 4034, 4121, 4155] avg=4096.0000 max=4155 max_rank=1 min=4027 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1017, 1015, 982, 986, 8, 0, 0, 0, 1006, 957, 1036, 954, 6, 6, 0, 0, 1054, 1017, 1032, 1040, 0, 0, 0, 0, 1038, 1003, 1015, 1014, 13, 10, 0, 0, 1028, 1071, 1055, 1031, 29, 3, 0, 0, 1017, 1045, 1004, 974, 0, 0, 0, 0, 1025, 1060, 1086, 1009, 0, 0, 0, 0, 1022, 1052, 1035, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4008, 3965, 4143, 4093, 4217, 4040, 4180, 4122] avg=4096.0000 max=4217 max_rank=4 min=3965 min_rank=1 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[972, 1000, 1025, 1037, 8, 0, 0, 0, 1031, 1068, 1033, 1027, 6, 6, 0, 0, 1014, 1011, 1049, 1039, 0, 0, 0, 0, 1034, 920, 1026, 995, 15, 10, 0, 0, 1048, 1018, 989, 985, 29, 3, 0, 0, 1020, 1008, 1081, 1073, 0, 0, 0, 0, 966, 1046, 1013, 1006, 0, 0, 0, 0, 976, 1066, 1048, 1067, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4171, 4113, 4000, 4072, 4182, 4031, 4157] avg=4096.0000 max=4182 max_rank=5 min=4000 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1087, 1023, 985, 1022, 11, 0, 0, 0, 1051, 1004, 1034, 1061, 11, 9, 0, 0, 1021, 1035, 1047, 973, 0, 0, 0, 0, 1011, 1033, 977, 1079, 18, 14, 0, 0, 1030, 1015, 970, 1060, 33, 5, 0, 0, 993, 1016, 1014, 995, 0, 0, 0, 0, 1023, 1018, 1023, 1011, 0, 0, 0, 0, 997, 1000, 1037, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4170, 4076, 4132, 4113, 4018, 4075, 4056] avg=4096.0000 max=4170 max_rank=1 min=4018 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16262 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1051, 1095, 1015, 1011, 8, 0, 0, 0, 1031, 990, 1031, 980, 7, 5, 0, 0, 1010, 1063, 1034, 1024, 0, 0, 0, 0, 1005, 1048, 978, 982, 13, 10, 0, 0, 1013, 1017, 1023, 1033, 29, 3, 0, 0, 1053, 1012, 1059, 971, 0, 0, 0, 0, 972, 1021, 1062, 1034, 0, 0, 0, 0, 1060, 1014, 1016, 985, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4044, 4131, 4036, 4118, 4095, 4089, 4075] avg=4096.0000 max=4180 max_rank=0 min=4036 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1010, 1057, 1031, 1062, 7, 0, 0, 0, 1025, 985, 1032, 1048, 6, 5, 0, 0, 1035, 976, 979, 1033, 0, 0, 0, 0, 1040, 1003, 1018, 1055, 14, 9, 0, 0, 1016, 954, 1036, 963, 28, 3, 0, 0, 1103, 1040, 1083, 1051, 0, 0, 0, 0, 1056, 1010, 1000, 968, 0, 0, 0, 0, 1044, 992, 969, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4101, 4023, 4139, 4000, 4277, 4034, 4027] avg=4096.0000 max=4277 max_rank=5 min=4000 min_rank=4 imbalance_ratio=1.044189 max_load_reduction=28491 max_load_reduction_pct=0.869476 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1064, 989, 1016, 1027, 1001, 1009, 979, 1063, 1013, 1038, 1003, 1015, 1015, 1023, 1040, 1031, 1034, 1034, 1015, 1033, 1004, 991, 987, 1053, 1073, 1014, 1015, 1034, 1023, 1065, 1028, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1014, 1066, 996, 1052, 1022, 951, 1041, 1035, 1061, 1041, 1069, 947, 1019, 1025, 1054, 1038, 1038, 1013, 1037, 1036, 1037, 1037, 1038, 1007, 1052, 1049, 1045, 1036, 935, 1003, 1008, 966] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1025, 1002, 1011, 1015, 986, 1041, 1061, 1017, 1004, 969, 1014, 1026, 1055, 1037, 1046, 1079, 1018, 1067, 1053, 1031, 1046, 1000, 1040, 1033, 1011, 1019, 1001, 971, 1004, 1031, 1001, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1033, 1049, 1020, 979, 989, 1018, 1029, 1042, 1057, 1024, 1047, 1027, 1025, 997, 1004, 1025, 1006, 1034, 1023, 1031, 1038, 977, 965, 1037, 1013, 1009, 1015, 1064, 1034, 1062, 1022, 1073] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1014, 1041, 971, 1022, 1045, 995, 983, 1034, 985, 1032, 981, 1009, 1053, 997, 958, 1009, 1043, 1025, 1013, 1040, 1025, 1047, 1015, 1027, 1077, 1102, 990, 1079, 1034, 1035, 1041, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1032, 988, 1054, 1024, 1027, 1015, 1027, 1045, 981, 1000, 1032, 1035, 1042, 1024, 990, 1050, 1049, 995, 1001, 1041, 1024, 1032, 1013, 958, 1055, 1025, 1091, 1031, 1023, 1018, 1039, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1030, 1027, 976, 986, 1041, 1074, 1039, 985, 994, 980, 1014, 1056, 1038, 1042, 1047, 1003, 1068, 1056, 973, 983, 1062, 1037, 1024, 1008, 1079, 966, 1038, 1000, 1031, 1040, 1066, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1013, 989, 1042, 1071, 1027, 1016, 1025, 1037, 1054, 1025, 986, 1034, 1035, 1074, 1015, 1030, 1046, 976, 1038, 1042, 1060, 997, 1031, 955, 1046, 1069, 988, 993, 957, 1024, 1043, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1064, 989, 1016, 1027, 9, 6, 0, 0, 1001, 1009, 979, 1063, 6, 2, 0, 0, 1013, 1038, 1003, 1015, 27, 0, 0, 0, 998, 1023, 1040, 1031, 0, 0, 0, 0, 1019, 1034, 1015, 1033, 0, 0, 0, 0, 1004, 991, 987, 1053, 11, 8, 0, 0, 1038, 1014, 1015, 1034, 0, 0, 0, 0, 1023, 1063, 1028, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4060, 4096, 4092, 4101, 4054, 4101, 4153] avg=4096.0000 max=4153 max_rank=7 min=4054 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1014, 1066, 996, 1052, 9, 6, 0, 0, 1022, 951, 1041, 1035, 6, 2, 0, 0, 1061, 1041, 1069, 947, 26, 0, 0, 0, 1002, 1025, 1054, 1038, 0, 0, 0, 0, 1023, 1013, 1037, 1036, 0, 0, 0, 0, 1037, 1037, 1038, 1007, 11, 8, 0, 0, 1018, 1049, 1045, 1036, 0, 0, 0, 0, 935, 1001, 1008, 966, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4057, 4144, 4119, 4109, 4138, 4148, 3910] avg=4096.0000 max=4148 max_rank=6 min=3910 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1033, 1049, 1020, 979, 8, 6, 0, 0, 989, 1018, 1029, 1042, 6, 2, 0, 0, 1057, 1024, 1047, 1027, 25, 0, 0, 0, 1008, 997, 1004, 1025, 0, 0, 0, 0, 992, 1034, 1023, 1031, 0, 0, 0, 0, 1038, 977, 965, 1037, 11, 8, 0, 0, 980, 1009, 1015, 1064, 0, 0, 0, 0, 1034, 1060, 1022, 1073, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4086, 4180, 4034, 4080, 4036, 4068, 4189] avg=4096.0000 max=4189 max_rank=7 min=4034 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 1002, 1011, 1015, 9, 7, 0, 0, 986, 1041, 1061, 1017, 6, 2, 0, 0, 1004, 969, 1014, 1026, 25, 0, 0, 0, 1036, 1037, 1046, 1079, 0, 0, 0, 0, 1003, 1067, 1053, 1031, 0, 0, 0, 0, 1046, 1000, 1040, 1033, 12, 8, 0, 0, 978, 1019, 1001, 971, 0, 0, 0, 0, 1004, 1029, 1001, 1054, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4113, 4038, 4198, 4154, 4139, 3969, 4088] avg=4096.0000 max=4198 max_rank=3 min=3969 min_rank=6 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1014, 1041, 971, 1022, 9, 7, 0, 0, 1045, 995, 983, 1034, 6, 2, 0, 0, 985, 1032, 981, 1009, 27, 0, 0, 0, 1034, 997, 958, 1009, 0, 0, 0, 0, 1028, 1025, 1013, 1040, 0, 0, 0, 0, 1025, 1047, 1015, 1027, 12, 8, 0, 0, 1042, 1102, 990, 1079, 0, 0, 0, 0, 1034, 1033, 1041, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4065, 4034, 3998, 4106, 4134, 4213, 4154] avg=4096.0000 max=4213 max_rank=6 min=3998 min_rank=3 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16607 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1032, 988, 1054, 1024, 12, 11, 0, 0, 1027, 1015, 1027, 1045, 8, 5, 0, 0, 981, 1000, 1032, 1035, 32, 0, 0, 0, 1015, 1024, 990, 1050, 0, 0, 0, 0, 1029, 995, 1001, 1041, 0, 0, 0, 0, 1024, 1032, 1013, 958, 16, 11, 0, 0, 1012, 1025, 1091, 1031, 0, 0, 0, 0, 1023, 1013, 1039, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4127, 4080, 4079, 4066, 4054, 4159, 4082] avg=4096.0000 max=4159 max_rank=6 min=4054 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1013, 989, 1042, 1071, 9, 6, 0, 0, 1027, 1016, 1025, 1037, 6, 2, 0, 0, 1054, 1025, 986, 1034, 26, 0, 0, 0, 1018, 1074, 1015, 1030, 0, 0, 0, 0, 1031, 976, 1038, 1042, 0, 0, 0, 0, 1060, 997, 1031, 955, 11, 8, 0, 0, 1012, 1069, 988, 993, 0, 0, 0, 0, 957, 1022, 1043, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4113, 4125, 4137, 4087, 4062, 4062, 4052] avg=4096.0000 max=4137 max_rank=3 min=4052 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1030, 1027, 976, 986, 9, 7, 0, 0, 1041, 1074, 1039, 985, 6, 2, 0, 0, 994, 980, 1014, 1056, 27, 0, 0, 0, 1019, 1042, 1047, 1003, 0, 0, 0, 0, 1053, 1056, 973, 983, 0, 0, 0, 0, 1062, 1037, 1024, 1008, 12, 8, 0, 0, 1044, 966, 1038, 1000, 0, 0, 0, 0, 1031, 1038, 1066, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4147, 4071, 4111, 4065, 4151, 4048, 4140] avg=4096.0000 max=4151 max_rank=5 min=4035 min_rank=0 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1015, 1020, 1069, 1097, 1024, 1014, 1002, 1045, 1073, 1019, 1060, 1032, 985, 1003, 973, 1037, 989, 1021, 1005, 1058, 1006, 1009, 1018, 1041, 1043, 1052, 993, 1032, 1017, 1024, 1055, 937] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1046, 1045, 1019, 1052, 1022, 1041, 979, 1024, 1024, 1011, 1026, 1020, 1005, 1047, 1032, 1058, 1046, 1054, 1041, 1044, 1014, 1021, 1030, 967, 997, 992, 989, 1019, 1031, 1021, 1043, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1083, 1000, 969, 1020, 1027, 1006, 1033, 1051, 1044, 1049, 1036, 1054, 1027, 1010, 1028, 1011, 1023, 1020, 1038, 989, 1067, 1023, 988, 1006, 1057, 1072, 994, 993, 1013, 1034, 962, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1016, 1010, 1043, 1019, 1006, 1038, 982, 1022, 1065, 1035, 1017, 954, 1002, 1028, 1047, 1080, 1036, 1052, 1012, 1052, 1069, 1010, 1056, 989, 1087, 969, 1018, 1007, 1010, 1049, 960, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[987, 1048, 998, 1035, 1025, 1052, 1004, 1020, 1054, 985, 1050, 1016, 1075, 998, 1025, 958, 1023, 989, 1042, 1040, 1024, 999, 1057, 1044, 1071, 1001, 1024, 1035, 1016, 1005, 1017, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1008, 1008, 1006, 1052, 1039, 1037, 1018, 1033, 972, 1000, 1073, 1032, 1013, 1001, 1070, 1028, 1047, 1021, 1045, 983, 1000, 990, 992, 1041, 1033, 984, 1062, 1015, 1061, 1012, 1022, 1070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1050, 1030, 1005, 982, 1006, 1031, 1061, 1021, 991, 1017, 1048, 1011, 1028, 1064, 955, 988, 996, 1014, 1080, 1065, 1029, 1031, 1015, 1014, 1041, 1046, 998, 1031, 1055, 1013, 983, 1069] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1057, 1030, 988, 1020, 1036, 1071, 1003, 1038, 1018, 1047, 1019, 1034, 1050, 1019, 1028, 1054, 1005, 1009, 963, 1003, 1039, 1017, 1036, 1014, 1039, 994, 1024, 999, 1065, 1050, 990, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1015, 1020, 1069, 1090, 0, 0, 0, 0, 1024, 1010, 1002, 1045, 0, 0, 0, 0, 1073, 1019, 1046, 1032, 0, 0, 0, 0, 985, 1003, 973, 1037, 4, 0, 0, 0, 989, 1021, 1005, 1054, 0, 0, 0, 0, 1006, 1009, 1018, 1041, 14, 0, 0, 0, 1043, 1052, 993, 1032, 6, 0, 0, 0, 1017, 1024, 1055, 937, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4081, 4170, 4002, 4069, 4088, 4126, 4038] avg=4096.0000 max=4194 max_rank=0 min=4002 min_rank=3 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16321 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1016, 1010, 1043, 1013, 0, 0, 0, 0, 1006, 1034, 982, 1022, 0, 0, 0, 0, 1065, 1035, 1004, 954, 0, 0, 0, 0, 1002, 1028, 1047, 1080, 4, 0, 0, 0, 1036, 1052, 1012, 1048, 0, 0, 0, 0, 1069, 1010, 1056, 989, 13, 0, 0, 0, 1087, 969, 1018, 1007, 6, 0, 0, 0, 1010, 1049, 960, 1028, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4044, 4058, 4161, 4148, 4137, 4087, 4051] avg=4096.0000 max=4161 max_rank=3 min=4044 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1008, 1008, 1006, 1045, 0, 0, 0, 0, 1039, 1033, 1018, 1033, 0, 0, 0, 0, 972, 1000, 1059, 1032, 0, 0, 0, 0, 1013, 1001, 1070, 1028, 4, 0, 0, 0, 1047, 1021, 1045, 979, 0, 0, 0, 0, 1000, 990, 992, 1041, 14, 0, 0, 0, 1033, 984, 1062, 1015, 6, 0, 0, 0, 1061, 1012, 1022, 1070, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4123, 4063, 4116, 4092, 4037, 4100, 4170] avg=4096.0000 max=4170 max_rank=7 min=4037 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1050, 1030, 1005, 976, 0, 0, 0, 0, 1006, 1027, 1061, 1021, 0, 0, 0, 0, 991, 1017, 1034, 1011, 0, 0, 0, 0, 1028, 1064, 955, 988, 4, 0, 0, 0, 996, 1014, 1080, 1061, 0, 0, 0, 0, 1029, 1031, 1015, 1014, 14, 0, 0, 0, 1041, 1046, 998, 1031, 6, 0, 0, 0, 1055, 1013, 983, 1069, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4115, 4053, 4039, 4151, 4103, 4122, 4124] avg=4096.0000 max=4151 max_rank=4 min=4039 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1083, 1000, 969, 1014, 0, 0, 0, 0, 1027, 1002, 1033, 1051, 0, 0, 0, 0, 1044, 1049, 1023, 1054, 0, 0, 0, 0, 1027, 1010, 1028, 1011, 4, 0, 0, 0, 1023, 1020, 1038, 985, 0, 0, 0, 0, 1067, 1023, 988, 1006, 13, 0, 0, 0, 1057, 1072, 994, 993, 6, 0, 0, 0, 1013, 1034, 962, 1041, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4113, 4170, 4080, 4066, 4097, 4122, 4054] avg=4096.0000 max=4170 max_rank=2 min=4054 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1046, 1045, 1019, 1038, 0, 0, 0, 0, 1022, 1027, 979, 1024, 0, 0, 0, 0, 1024, 1011, 1003, 1020, 0, 0, 0, 0, 1005, 1047, 1032, 1058, 9, 4, 0, 0, 1046, 1054, 1041, 1035, 0, 0, 0, 0, 1014, 1021, 1030, 967, 17, 0, 0, 0, 997, 992, 989, 1019, 9, 6, 0, 0, 1031, 1021, 1043, 1008, 10, 5, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4052, 4058, 4155, 4176, 4049, 4012, 4118] avg=4096.0000 max=4176 max_rank=4 min=4012 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[987, 1048, 998, 1028, 0, 0, 0, 0, 1025, 1048, 1004, 1020, 0, 0, 0, 0, 1054, 985, 1036, 1016, 0, 0, 0, 0, 1075, 998, 1025, 958, 4, 0, 0, 0, 1023, 989, 1042, 1036, 0, 0, 0, 0, 1024, 999, 1057, 1044, 14, 0, 0, 0, 1071, 1001, 1024, 1035, 6, 0, 0, 0, 1016, 1005, 1017, 1051, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4097, 4091, 4060, 4090, 4138, 4137, 4094] avg=4096.0000 max=4138 max_rank=5 min=4060 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1057, 1030, 988, 1014, 0, 0, 0, 0, 1036, 1066, 1003, 1038, 0, 0, 0, 0, 1018, 1047, 1006, 1034, 0, 0, 0, 0, 1050, 1019, 1028, 1054, 4, 0, 0, 0, 1005, 1009, 963, 999, 0, 0, 0, 0, 1039, 1017, 1036, 1014, 13, 0, 0, 0, 1039, 994, 1024, 999, 6, 0, 0, 0, 1065, 1050, 990, 1009, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4143, 4105, 4155, 3976, 4119, 4062, 4119] avg=4096.0000 max=4155 max_rank=3 min=3976 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16276 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8448, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1042, 1043, 1020, 1018, 1021, 1035, 1038, 953, 1006, 978, 981, 1011, 1043, 1003, 1083, 995, 1024, 1019, 1065, 1026, 1036, 1031, 1023, 1010, 1056, 1008, 1014, 1005, 1059, 1032, 1073, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1027, 1006, 1015, 1019, 991, 1015, 1046, 1074, 1051, 985, 1016, 1042, 1006, 1011, 995, 1014, 1038, 1021, 1052, 1072, 1020, 1063, 977, 1032, 1027, 1041, 991, 1071, 1033, 940, 1059, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1092, 1004, 1029, 1012, 1048, 1030, 1002, 1079, 1043, 1032, 1030, 975, 993, 1028, 1029, 972, 1037, 971, 1048, 1013, 1014, 1026, 1039, 1016, 1044, 998, 1025, 994, 1044, 1020, 1074, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1005, 1043, 1005, 1048, 1027, 990, 1072, 1028, 1018, 1062, 998, 1011, 1006, 1032, 1027, 1019, 1020, 1058, 1038, 1001, 999, 1045, 1046, 1013, 1047, 1035, 995, 1008, 972, 1067, 996, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[999, 995, 1055, 1009, 985, 999, 1044, 1011, 989, 1045, 1016, 999, 1022, 1042, 1040, 1087, 1034, 992, 1053, 1064, 956, 1021, 1007, 1003, 1009, 1040, 1039, 1019, 1059, 1092, 1006, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1001, 993, 1026, 1024, 1060, 1015, 1004, 1024, 1013, 1011, 999, 1016, 994, 1012, 1038, 1017, 1048, 1008, 1059, 1023, 1040, 1083, 1051, 1042, 1034, 1021, 1002, 1030, 1027, 1015, 1015, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[984, 1018, 1015, 1088, 1049, 1046, 1025, 1014, 961, 1025, 1077, 986, 980, 1001, 988, 1006, 1012, 1022, 1029, 1029, 1077, 1051, 995, 1058, 1038, 1040, 1064, 1034, 991, 988, 1028, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1007, 1023, 1056, 1074, 1052, 996, 1047, 1019, 1017, 995, 1038, 1005, 1013, 1076, 1081, 980, 1031, 1013, 1021, 1010, 1026, 1025, 1007, 997, 1004, 1014, 953, 1012, 1028, 1070, 1050, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1042, 1043, 1020, 1015, 0, 0, 0, 0, 1021, 1035, 1030, 953, 0, 0, 0, 0, 1006, 978, 981, 1011, 24, 19, 0, 0, 1043, 1003, 1083, 995, 8, 3, 4, 0, 1024, 1019, 1042, 1026, 0, 0, 0, 0, 1036, 1025, 1023, 1010, 0, 0, 0, 0, 1056, 1008, 1014, 1005, 3, 3, 0, 0, 1059, 1032, 1049, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4039, 4019, 4139, 4111, 4094, 4089, 4157] avg=4096.0000 max=4157 max_rank=7 min=4019 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1092, 1004, 1029, 1009, 0, 0, 0, 0, 1048, 1030, 994, 1079, 0, 0, 0, 0, 1043, 1032, 1030, 975, 24, 18, 0, 0, 993, 1028, 1029, 972, 8, 3, 4, 0, 1037, 971, 1026, 1013, 0, 0, 0, 0, 1014, 1020, 1039, 1016, 0, 0, 0, 0, 1044, 998, 1025, 994, 3, 3, 0, 0, 1044, 1020, 1050, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4151, 4122, 4037, 4047, 4089, 4067, 4121] avg=4096.0000 max=4151 max_rank=1 min=4037 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1027, 1006, 1015, 1016, 0, 0, 0, 0, 991, 1015, 1038, 1074, 0, 0, 0, 0, 1051, 985, 1016, 1042, 23, 18, 0, 0, 1006, 1011, 995, 1014, 8, 4, 4, 0, 1038, 1021, 1030, 1072, 0, 0, 0, 0, 1020, 1056, 977, 1032, 0, 0, 0, 0, 1027, 1041, 991, 1071, 3, 3, 0, 0, 1033, 940, 1036, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4118, 4135, 4042, 4161, 4085, 4136, 4027] avg=4096.0000 max=4161 max_rank=4 min=4027 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1001, 993, 1026, 1021, 0, 0, 0, 0, 1060, 1015, 996, 1024, 0, 0, 0, 0, 1013, 1011, 999, 1016, 22, 19, 0, 0, 994, 1012, 1038, 1017, 8, 4, 4, 0, 1048, 1008, 1036, 1023, 0, 0, 0, 0, 1040, 1076, 1051, 1042, 0, 0, 0, 0, 1034, 1021, 1002, 1030, 3, 3, 0, 0, 1027, 1015, 993, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4095, 4080, 4077, 4115, 4209, 4093, 4058] avg=4096.0000 max=4209 max_rank=5 min=4041 min_rank=0 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16475 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[984, 1018, 1015, 1085, 0, 0, 0, 0, 1049, 1046, 1017, 1014, 0, 0, 0, 0, 961, 1025, 1077, 986, 23, 18, 0, 0, 980, 1001, 988, 1006, 8, 4, 3, 0, 1012, 1022, 1008, 1029, 0, 0, 0, 0, 1077, 1044, 995, 1058, 0, 0, 0, 0, 1038, 1040, 1064, 1034, 3, 3, 0, 0, 991, 988, 1005, 1049, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4126, 4090, 3990, 4071, 4174, 4182, 4033] avg=4096.0000 max=4182 max_rank=6 min=3990 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[999, 995, 1055, 1006, 0, 0, 0, 0, 985, 999, 1036, 1011, 0, 0, 0, 0, 989, 1045, 1016, 999, 22, 19, 0, 0, 1022, 1042, 1040, 1087, 8, 3, 4, 0, 1034, 992, 1030, 1064, 0, 0, 0, 0, 956, 1015, 1007, 1003, 0, 0, 0, 0, 1009, 1040, 1039, 1019, 3, 3, 0, 0, 1059, 1092, 984, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4031, 4090, 4206, 4120, 3981, 4113, 4172] avg=4096.0000 max=4206 max_rank=3 min=3981 min_rank=5 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1005, 1043, 1005, 1042, 0, 0, 0, 0, 1027, 990, 1057, 1028, 0, 0, 0, 0, 1018, 1062, 998, 1011, 25, 22, 0, 0, 1006, 1032, 1027, 1019, 15, 8, 6, 0, 1020, 1058, 1010, 1001, 0, 0, 0, 0, 999, 1029, 1046, 1013, 0, 0, 0, 0, 1047, 1035, 995, 1008, 8, 6, 0, 0, 972, 1067, 971, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4102, 4136, 4113, 4089, 4087, 4099, 4047] avg=4096.0000 max=4136 max_rank=2 min=4047 min_rank=7 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1007, 1023, 1056, 1071, 0, 0, 0, 0, 1052, 996, 1039, 1019, 0, 0, 0, 0, 1017, 995, 1038, 1005, 23, 18, 0, 0, 1013, 1076, 1081, 980, 8, 3, 3, 0, 1031, 1013, 1000, 1010, 0, 0, 0, 0, 1026, 1019, 1007, 997, 0, 0, 0, 0, 1004, 1014, 953, 1012, 3, 3, 0, 0, 1028, 1070, 1027, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4106, 4096, 4164, 4054, 4049, 3989, 4153] avg=4096.0000 max=4164 max_rank=3 min=3989 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16245 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8064, 256, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1034, 1055, 1050, 1051, 1009, 1010, 1005, 1023, 1014, 1035, 1015, 1023, 1022, 1028, 1027, 1050, 999, 1022, 1032, 1036, 983, 1011, 1007, 1033, 996, 995, 1012, 1008, 1047, 1014, 1025, 1097] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1026, 1010, 1023, 1013, 1031, 1021, 974, 1049, 988, 1052, 1053, 1051, 1006, 1027, 1042, 1064, 994, 1036, 1042, 1012, 1068, 1028, 1038, 1019, 1011, 1027, 1016, 1006, 993, 964, 1028, 1056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1004, 1055, 1086, 1079, 1097, 995, 1020, 1017, 1012, 1024, 1029, 989, 1012, 1026, 976, 989, 1058, 1015, 1014, 991, 1044, 1013, 992, 999, 1101, 1044, 994, 1055, 1026, 1013, 1024, 975] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1017, 1021, 1044, 1044, 1057, 1035, 1053, 1023, 993, 1054, 996, 1006, 1026, 1047, 1025, 1041, 991, 1049, 1056, 1021, 1005, 978, 1008, 1029, 979, 1031, 1044, 1052, 1028, 1031, 987, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1029, 1024, 1023, 998, 1005, 988, 1077, 1029, 1002, 993, 1049, 1020, 1018, 1013, 1008, 1083, 970, 990, 1039, 1039, 982, 1042, 1050, 1053, 1031, 1014, 973, 1044, 1000, 1082, 1078, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1019, 1004, 1056, 1009, 1053, 1001, 978, 1026, 1030, 988, 999, 1017, 1001, 1036, 992, 1030, 1062, 1052, 995, 1043, 975, 1025, 1007, 1038, 1023, 1028, 1105, 1055, 1018, 985, 1070, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[998, 1042, 993, 1024, 1009, 970, 1025, 1056, 1007, 973, 1036, 1013, 1045, 1015, 1006, 1032, 1010, 992, 1048, 1035, 1035, 978, 1028, 1060, 1044, 1075, 1004, 1078, 1026, 1071, 1013, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[993, 1005, 1057, 1039, 1025, 1020, 997, 1051, 1037, 1026, 1026, 1013, 1006, 1019, 1023, 999, 1031, 1022, 1081, 1073, 1020, 1011, 993, 1044, 1038, 988, 1042, 997, 1051, 1023, 1008, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 1010, 1004, 1013, 0, 0, 0, 0, 1031, 1021, 974, 1049, 4, 0, 0, 0, 988, 1052, 1053, 1051, 19, 5, 0, 0, 1006, 1027, 1042, 1064, 4, 0, 0, 0, 994, 1036, 1033, 1012, 0, 0, 0, 0, 1068, 1028, 1038, 1019, 11, 9, 0, 0, 1011, 1027, 1016, 990, 0, 0, 0, 0, 993, 964, 1020, 1056, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4079, 4168, 4143, 4075, 4173, 4044, 4033] avg=4096.0000 max=4173 max_rank=5 min=4033 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1004, 1055, 1066, 1079, 0, 0, 0, 0, 1097, 995, 1020, 1017, 4, 0, 0, 0, 1012, 1024, 1029, 989, 20, 6, 0, 0, 1012, 1026, 976, 989, 4, 0, 0, 0, 1058, 1015, 1005, 991, 0, 0, 0, 0, 1044, 1013, 992, 999, 11, 9, 0, 0, 1101, 1044, 994, 1038, 0, 0, 0, 0, 1026, 1013, 1016, 975, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4204, 4133, 4080, 4007, 4069, 4068, 4177, 4030] avg=4096.0000 max=4204 max_rank=0 min=4007 min_rank=3 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1034, 1055, 1031, 1051, 0, 0, 0, 0, 1009, 1010, 1005, 1023, 4, 0, 0, 0, 1014, 1035, 1015, 1023, 19, 5, 0, 0, 1022, 1028, 1027, 1050, 4, 0, 0, 0, 999, 1022, 1023, 1036, 0, 0, 0, 0, 983, 1011, 1007, 1033, 11, 9, 0, 0, 996, 995, 1012, 992, 0, 0, 0, 0, 1047, 1014, 1017, 1097, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4051, 4111, 4131, 4080, 4054, 3995, 4175] avg=4096.0000 max=4175 max_rank=7 min=3995 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1017, 1021, 1025, 1044, 0, 0, 0, 0, 1057, 1035, 1053, 1023, 4, 0, 0, 0, 993, 1054, 996, 1006, 19, 6, 0, 0, 1026, 1047, 1025, 1041, 4, 0, 0, 0, 991, 1049, 1047, 1021, 0, 0, 0, 0, 1005, 978, 1008, 1029, 11, 9, 0, 0, 979, 1031, 1044, 1035, 0, 0, 0, 0, 1028, 1031, 979, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4172, 4074, 4143, 4108, 4040, 4089, 4035] avg=4096.0000 max=4172 max_rank=1 min=4035 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1029, 1024, 1004, 998, 0, 0, 0, 0, 1005, 988, 1077, 1029, 4, 0, 0, 0, 1002, 993, 1049, 1020, 19, 6, 0, 0, 1018, 1013, 1008, 1083, 4, 0, 0, 0, 970, 990, 1030, 1039, 0, 0, 0, 0, 982, 1042, 1050, 1053, 11, 9, 0, 0, 1031, 1014, 973, 1027, 0, 0, 0, 0, 1000, 1082, 1070, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4103, 4089, 4126, 4029, 4147, 4045, 4174] avg=4096.0000 max=4174 max_rank=7 min=4029 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[998, 1042, 975, 1024, 0, 0, 0, 0, 1009, 970, 1025, 1056, 4, 0, 0, 0, 1007, 973, 1036, 1013, 18, 6, 0, 0, 1045, 1015, 1006, 1032, 4, 0, 0, 0, 1010, 992, 1039, 1035, 0, 0, 0, 0, 1035, 978, 1028, 1060, 12, 9, 0, 0, 1044, 1075, 1004, 1060, 0, 0, 0, 0, 1026, 1071, 1005, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4064, 4053, 4102, 4076, 4122, 4183, 4129] avg=4096.0000 max=4183 max_rank=6 min=4039 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[993, 1005, 1038, 1039, 0, 0, 0, 0, 1025, 1020, 997, 1051, 4, 0, 0, 0, 1037, 1026, 1026, 1013, 19, 5, 0, 0, 1006, 1019, 1023, 999, 4, 0, 0, 0, 1031, 1022, 1071, 1073, 0, 0, 0, 0, 1020, 1011, 993, 1044, 11, 10, 0, 0, 1038, 988, 1042, 981, 0, 0, 0, 0, 1051, 1023, 1000, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4097, 4126, 4051, 4197, 4089, 4049, 4084] avg=4096.0000 max=4197 max_rank=4 min=4049 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1019, 1004, 1032, 1009, 0, 0, 0, 0, 1053, 1001, 978, 1026, 7, 4, 0, 0, 1030, 988, 999, 1017, 24, 9, 0, 0, 1001, 1036, 992, 1030, 6, 0, 0, 0, 1062, 1052, 977, 1043, 0, 0, 0, 0, 975, 1025, 1007, 1038, 16, 14, 0, 0, 1023, 1028, 1105, 1030, 0, 0, 0, 0, 1018, 985, 1057, 1048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4069, 4067, 4065, 4134, 4075, 4186, 4108] avg=4096.0000 max=4186 max_rank=6 min=4064 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1032, 1026, 1007, 1001, 1053, 1061, 1069, 1027, 1060, 997, 994, 1051, 1089, 1000, 1034, 1018, 979, 1027, 1006, 1047, 1010, 980, 1044, 1045, 1050, 1015, 1040, 985, 1015, 1047, 1007, 952] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1090, 1044, 1005, 995, 957, 1021, 999, 1031, 1031, 1051, 1026, 1042, 1027, 1033, 1061, 961, 1006, 991, 1005, 956, 1000, 1020, 1042, 1017, 1017, 1042, 1062, 1018, 1071, 1040, 1046, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1025, 1069, 1002, 995, 1056, 1000, 1000, 1026, 1011, 970, 1056, 1070, 1007, 1026, 1045, 1017, 1037, 978, 1059, 1002, 1031, 1021, 1032, 1068, 1030, 1015, 961, 1023, 1056, 1030, 1031, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1061, 995, 1037, 1003, 1022, 1022, 1031, 1022, 1036, 1038, 1058, 1047, 1005, 1036, 999, 1038, 1051, 1053, 997, 1033, 1031, 1007, 964, 1031, 1020, 1040, 991, 972, 1050, 1028, 1029, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1059, 1014, 1058, 1001, 1051, 1009, 1011, 1000, 1020, 1003, 1048, 1020, 1028, 1006, 1026, 995, 1080, 1044, 973, 966, 1027, 1029, 1007, 1081, 1004, 1033, 1044, 1033, 1011, 1056, 1022, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1003, 1002, 1000, 1015, 963, 1050, 1011, 1054, 1035, 1007, 996, 1035, 1015, 992, 1017, 993, 1047, 1049, 1024, 1083, 1000, 1017, 1050, 1040, 1050, 1008, 1062, 1030, 1023, 993, 1073, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1037, 1022, 1046, 1029, 1026, 1040, 991, 1022, 1007, 1014, 1055, 1013, 1016, 1040, 1071, 1029, 1049, 1063, 1026, 1022, 986, 1042, 958, 1064, 1005, 1051, 1010, 1024, 1079, 993, 1002, 936] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1029, 1002, 985, 1027, 1050, 1041, 1043, 1036, 1004, 1015, 1051, 1046, 1011, 998, 1057, 1060, 1041, 1021, 1026, 998, 1027, 1013, 1011, 996, 1034, 1077, 1018, 1050, 1010, 997, 1003, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1032, 1026, 1007, 1001, 6, 0, 0, 0, 1053, 1058, 1069, 1027, 0, 0, 0, 0, 1060, 997, 994, 1035, 0, 0, 0, 0, 1089, 1000, 1034, 1018, 2, 0, 0, 0, 979, 1027, 1006, 1047, 2, 1, 0, 0, 1010, 980, 1044, 1045, 9, 0, 0, 0, 1050, 1010, 1040, 985, 0, 0, 0, 0, 1015, 1047, 1007, 952, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4207, 4086, 4143, 4062, 4088, 4085, 4025] avg=4096.0000 max=4207 max_rank=1 min=4025 min_rank=7 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1025, 1069, 1002, 995, 6, 0, 0, 0, 1056, 997, 1000, 1026, 0, 0, 0, 0, 1011, 970, 1056, 1054, 0, 0, 0, 0, 1007, 1026, 1045, 1017, 2, 0, 0, 0, 1037, 978, 1059, 1002, 2, 1, 0, 0, 1031, 1021, 1032, 1068, 9, 0, 0, 0, 1030, 1010, 961, 1023, 0, 0, 0, 0, 1056, 1030, 1031, 1019, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4079, 4091, 4097, 4079, 4161, 4024, 4140] avg=4096.0000 max=4161 max_rank=5 min=4024 min_rank=6 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1090, 1044, 1005, 995, 6, 0, 0, 0, 957, 1018, 999, 1031, 0, 0, 0, 0, 1031, 1051, 1026, 1026, 0, 0, 0, 0, 1027, 1033, 1061, 961, 2, 0, 0, 0, 1006, 991, 1005, 956, 2, 1, 0, 0, 1000, 1020, 1042, 1017, 9, 0, 0, 0, 1017, 1037, 1062, 1018, 0, 0, 0, 0, 1071, 1040, 1046, 1061, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4005, 4134, 4084, 3961, 4088, 4134, 4222] avg=4096.0000 max=4222 max_rank=7 min=3961 min_rank=4 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1061, 995, 1037, 1003, 6, 0, 0, 0, 1022, 1019, 1031, 1022, 0, 0, 0, 0, 1036, 1038, 1058, 1031, 0, 0, 0, 0, 1005, 1036, 999, 1038, 2, 0, 0, 0, 1051, 1053, 997, 1033, 2, 1, 0, 0, 1031, 1007, 964, 1031, 9, 0, 0, 0, 1020, 1035, 991, 972, 0, 0, 0, 0, 1050, 1028, 1029, 1021, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4094, 4163, 4080, 4137, 4042, 4018, 4132] avg=4096.0000 max=4163 max_rank=2 min=4018 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1059, 1014, 1058, 1001, 6, 0, 0, 0, 1051, 1006, 1011, 1000, 0, 0, 0, 0, 1020, 1003, 1048, 1004, 0, 0, 0, 0, 1028, 1006, 1026, 995, 2, 0, 0, 0, 1080, 1044, 973, 966, 2, 1, 0, 0, 1027, 1029, 1007, 1081, 9, 0, 0, 0, 1004, 1028, 1044, 1033, 0, 0, 0, 0, 1011, 1056, 1022, 1009, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4068, 4075, 4057, 4066, 4153, 4109, 4102] avg=4096.0000 max=4153 max_rank=5 min=4057 min_rank=3 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1003, 1002, 1000, 1015, 6, 0, 0, 0, 963, 1047, 1011, 1054, 0, 0, 0, 0, 1035, 1007, 996, 1019, 0, 0, 0, 0, 1015, 992, 1017, 993, 2, 0, 0, 0, 1047, 1049, 1024, 1083, 2, 1, 0, 0, 1000, 1017, 1050, 1040, 9, 0, 0, 0, 1050, 1003, 1062, 1030, 0, 0, 0, 0, 1023, 993, 1073, 1031, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4075, 4057, 4019, 4206, 4116, 4145, 4124] avg=4096.0000 max=4206 max_rank=4 min=4019 min_rank=3 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1037, 1022, 1046, 1029, 6, 0, 0, 0, 1026, 1037, 991, 1022, 0, 0, 0, 0, 1007, 1014, 1055, 997, 0, 0, 0, 0, 1016, 1040, 1071, 1029, 2, 0, 0, 0, 1049, 1063, 1026, 1022, 2, 1, 0, 0, 986, 1042, 958, 1064, 9, 0, 0, 0, 1005, 1046, 1010, 1024, 0, 0, 0, 0, 1079, 993, 1002, 936, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4076, 4073, 4158, 4163, 4059, 4085, 4014] avg=4096.0000 max=4163 max_rank=4 min=4014 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16321 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1029, 1002, 985, 1027, 10, 0, 0, 0, 1050, 1035, 1043, 1036, 0, 0, 0, 0, 1004, 1015, 1051, 1019, 0, 0, 0, 0, 1011, 998, 1057, 1060, 4, 0, 0, 0, 1041, 1021, 1026, 998, 6, 2, 0, 0, 1027, 1013, 1011, 996, 14, 0, 0, 0, 1034, 1066, 1018, 1050, 0, 0, 0, 0, 1010, 997, 1003, 991, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4164, 4089, 4130, 4094, 4061, 4168, 4009] avg=4096.0000 max=4168 max_rank=6 min=4009 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8448, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1044, 1006, 995, 1006, 1019, 996, 1052, 948, 1032, 1061, 1067, 1070, 986, 1034, 982, 1002, 1030, 1041, 975, 1011, 1041, 1048, 1011, 1038, 1063, 980, 1010, 1058, 1054, 1027, 1082, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[992, 1041, 1056, 1034, 1004, 1006, 1027, 1042, 1063, 1079, 1053, 1019, 1051, 1052, 1018, 1021, 988, 1038, 1001, 1003, 1030, 1009, 1040, 973, 1046, 968, 1069, 997, 991, 1003, 1030, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1046, 1049, 1023, 1030, 996, 1014, 951, 997, 1043, 1045, 1033, 1046, 1047, 999, 1042, 1030, 993, 1050, 1067, 1007, 1006, 1045, 972, 966, 1044, 979, 1066, 1025, 1051, 1083, 979, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1056, 1023, 1050, 1037, 1006, 1064, 1007, 981, 1046, 1023, 1028, 1013, 1015, 1012, 1037, 1003, 1015, 1030, 1029, 976, 1023, 997, 993, 1071, 1056, 1068, 1017, 1025, 978, 1050, 999, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1049, 975, 995, 997, 1009, 1055, 1069, 979, 1050, 999, 1081, 972, 1043, 1023, 1033, 1053, 1102, 1034, 1047, 993, 1017, 1035, 1025, 990, 1009, 1038, 986, 1033, 1014, 1025, 1018, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1020, 1068, 1072, 1056, 1040, 1068, 992, 988, 987, 982, 1039, 1038, 967, 1016, 998, 1047, 984, 1039, 984, 1042, 1061, 998, 1038, 1030, 990, 1002, 1040, 1030, 1034, 1030, 1048, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1021, 1015, 1009, 1027, 1013, 1006, 1046, 999, 1053, 1038, 1046, 1037, 1030, 1007, 1060, 1023, 989, 1080, 998, 1048, 1017, 1025, 989, 1051, 1002, 1059, 1022, 1029, 1001, 1018, 997, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[970, 1026, 1046, 1024, 986, 1025, 1012, 1056, 1041, 1075, 1016, 1009, 1052, 985, 1056, 992, 1033, 991, 1047, 1011, 1015, 1081, 999, 1061, 963, 996, 997, 983, 1066, 1027, 1080, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1044, 1006, 985, 1006, 0, 0, 0, 0, 1019, 996, 1052, 948, 40, 0, 0, 0, 1032, 1061, 1015, 1070, 0, 0, 0, 0, 986, 1034, 982, 1002, 6, 0, 0, 0, 1030, 1041, 975, 1011, 11, 0, 0, 0, 1041, 1048, 1011, 1038, 4, 4, 0, 0, 1063, 980, 1010, 1058, 12, 2, 0, 0, 1054, 1010, 1082, 999, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4055, 4178, 4010, 4068, 4146, 4125, 4145] avg=4096.0000 max=4178 max_rank=2 min=4010 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[992, 1041, 1046, 1034, 0, 0, 0, 0, 1004, 1006, 1027, 1042, 39, 0, 0, 0, 1063, 1079, 1002, 1019, 0, 0, 0, 0, 1051, 1052, 1018, 1021, 6, 0, 0, 0, 988, 1038, 1001, 1003, 11, 0, 0, 0, 1030, 1009, 1040, 973, 4, 4, 0, 0, 1046, 968, 1069, 997, 12, 2, 0, 0, 991, 986, 1030, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4118, 4163, 4148, 4041, 4060, 4094, 4031] avg=4096.0000 max=4163 max_rank=2 min=4031 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16226 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1046, 1049, 1013, 1030, 0, 0, 0, 0, 996, 1014, 951, 997, 38, 0, 0, 0, 1043, 1045, 983, 1046, 0, 0, 0, 0, 1047, 999, 1042, 1030, 6, 0, 0, 0, 993, 1050, 1067, 1007, 12, 0, 0, 0, 1006, 1045, 972, 966, 4, 4, 0, 0, 1044, 979, 1066, 1025, 12, 2, 0, 0, 1051, 1065, 979, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 3996, 4117, 4124, 4129, 3997, 4128, 4139] avg=4096.0000 max=4139 max_rank=7 min=3996 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1056, 1023, 1040, 1037, 0, 0, 0, 0, 1006, 1064, 1007, 981, 38, 0, 0, 0, 1046, 1023, 978, 1013, 0, 0, 0, 0, 1015, 1012, 1037, 1003, 6, 0, 0, 0, 1015, 1030, 1029, 976, 11, 0, 0, 0, 1023, 997, 993, 1071, 4, 4, 0, 0, 1056, 1068, 1017, 1025, 12, 2, 0, 0, 978, 1033, 999, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4096, 4060, 4073, 4061, 4092, 4180, 4050] avg=4096.0000 max=4180 max_rank=6 min=4050 min_rank=7 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1020, 1068, 1052, 1056, 0, 0, 0, 0, 1040, 1068, 992, 988, 43, 0, 0, 0, 987, 982, 980, 1038, 0, 0, 0, 0, 967, 1016, 998, 1047, 10, 0, 0, 0, 984, 1039, 984, 1042, 14, 0, 0, 0, 1061, 998, 1038, 1030, 10, 7, 0, 0, 990, 1002, 1040, 1030, 16, 3, 0, 0, 1034, 1006, 1048, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4196, 4131, 3987, 4038, 4063, 4144, 4081, 4128] avg=4096.0000 max=4196 max_rank=0 min=3987 min_rank=2 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1049, 975, 985, 997, 0, 0, 0, 0, 1009, 1055, 1069, 979, 40, 0, 0, 0, 1050, 999, 1028, 972, 0, 0, 0, 0, 1043, 1023, 1033, 1053, 6, 0, 0, 0, 1102, 1034, 1047, 993, 11, 0, 0, 0, 1017, 1035, 1025, 990, 4, 4, 0, 0, 1009, 1038, 986, 1033, 13, 2, 0, 0, 1014, 1008, 1018, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4152, 4049, 4158, 4187, 4075, 4081, 4060] avg=4096.0000 max=4187 max_rank=4 min=4006 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1021, 1015, 999, 1027, 0, 0, 0, 0, 1013, 1006, 1046, 999, 39, 0, 0, 0, 1053, 1038, 995, 1037, 0, 0, 0, 0, 1030, 1007, 1060, 1023, 6, 0, 0, 0, 989, 1080, 998, 1048, 11, 0, 0, 0, 1017, 1025, 989, 1051, 4, 4, 0, 0, 1002, 1059, 1022, 1029, 12, 2, 0, 0, 1001, 1001, 997, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4103, 4123, 4126, 4126, 4090, 4126, 4012] avg=4096.0000 max=4126 max_rank=3 min=4012 min_rank=7 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[970, 1026, 1036, 1024, 0, 0, 0, 0, 986, 1025, 1012, 1056, 38, 0, 0, 0, 1041, 1075, 966, 1009, 0, 0, 0, 0, 1052, 985, 1056, 992, 6, 0, 0, 0, 1033, 991, 1047, 1011, 11, 0, 0, 0, 1015, 1081, 999, 1061, 4, 4, 0, 0, 963, 996, 997, 983, 12, 2, 0, 0, 1066, 1010, 1080, 1047, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4117, 4091, 4091, 4093, 4164, 3953, 4203] avg=4096.0000 max=4203 max_rank=7 min=3953 min_rank=6 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1006, 1065, 996, 1010, 1036, 1021, 1019, 1087, 999, 979, 1026, 1034, 984, 1057, 972, 968, 1102, 990, 1104, 1028, 1042, 1048, 1013, 999, 1011, 1007, 1024, 1055, 1025, 1014, 1007, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1017, 1079, 1030, 1039, 1038, 1049, 1037, 1065, 990, 1010, 1029, 1025, 1016, 1058, 1019, 1047, 1011, 1004, 1031, 1057, 1012, 997, 1057, 945, 1001, 986, 1036, 1063, 969, 1018, 1045, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[999, 1019, 1054, 1022, 1035, 975, 1052, 1022, 1011, 1005, 1055, 1029, 982, 1044, 1037, 1016, 1023, 1005, 1089, 1012, 1021, 974, 1055, 987, 1009, 1025, 1002, 1003, 1070, 1050, 1037, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[992, 1023, 984, 1053, 1040, 995, 1052, 1006, 1045, 1074, 1020, 1009, 1040, 1041, 1085, 1003, 1051, 986, 984, 1010, 1063, 1026, 974, 1013, 999, 1025, 1035, 991, 1057, 1050, 1036, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1003, 1010, 1005, 992, 1043, 1025, 1089, 1069, 957, 983, 1072, 1000, 1024, 988, 1023, 1049, 1030, 1087, 1054, 1017, 1026, 1033, 1020, 1024, 1040, 1061, 994, 1034, 1020, 1006, 994, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1018, 1035, 999, 1004, 1024, 998, 1043, 983, 1040, 1038, 1049, 990, 1056, 1024, 1019, 1035, 1007, 1056, 1021, 1009, 1001, 1043, 1015, 1047, 1039, 985, 1017, 1048, 1015, 1024, 1050, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[981, 1047, 1046, 976, 1024, 976, 1009, 1019, 1007, 1057, 1007, 972, 1012, 1041, 984, 1065, 1046, 1010, 1014, 1024, 1017, 1081, 1014, 1007, 1037, 1082, 1009, 1003, 1085, 1025, 1051, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1040, 1019, 1020, 1005, 1066, 950, 1042, 1019, 1008, 1048, 991, 988, 1019, 1003, 1022, 1078, 1005, 1036, 1038, 1059, 1002, 966, 1035, 1030, 1058, 1026, 1048, 1041, 1053, 1053, 987, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1017, 1079, 1030, 1039, 13, 5, 3, 0, 1038, 1049, 1021, 1065, 0, 0, 0, 0, 990, 1010, 1029, 1025, 27, 0, 0, 0, 1016, 1058, 1019, 1042, 0, 0, 0, 0, 1011, 1004, 1003, 1057, 0, 0, 0, 0, 1012, 997, 1057, 945, 16, 3, 1, 0, 1001, 986, 1036, 1060, 0, 0, 0, 0, 953, 1018, 1045, 988, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4173, 4081, 4135, 4075, 4031, 4083, 4004] avg=4096.0000 max=4186 max_rank=0 min=4004 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[999, 1019, 1054, 1022, 13, 5, 3, 0, 1035, 975, 1036, 1022, 0, 0, 0, 0, 1011, 1005, 1055, 1029, 28, 0, 0, 0, 982, 1044, 1037, 1011, 0, 0, 0, 0, 1023, 1005, 1060, 1012, 0, 0, 0, 0, 1021, 974, 1055, 987, 18, 3, 1, 0, 1009, 1025, 1002, 1000, 0, 0, 0, 0, 1052, 1050, 1037, 1049, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4068, 4128, 4074, 4100, 4059, 4036, 4188] avg=4096.0000 max=4188 max_rank=7 min=4036 min_rank=6 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1003, 1010, 1005, 992, 14, 5, 3, 0, 1043, 1025, 1072, 1069, 0, 0, 0, 0, 957, 983, 1072, 1000, 27, 0, 0, 0, 1024, 988, 1023, 1044, 0, 0, 0, 0, 1030, 1087, 1026, 1017, 0, 0, 0, 0, 1026, 1033, 1020, 1024, 17, 3, 1, 0, 1040, 1061, 994, 1031, 0, 0, 0, 0, 1003, 1006, 994, 996, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4209, 4039, 4079, 4160, 4124, 4126, 3999] avg=4096.0000 max=4209 max_rank=1 min=3999 min_rank=7 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1006, 1065, 996, 1010, 13, 5, 3, 0, 1036, 1021, 1003, 1087, 0, 0, 0, 0, 999, 979, 1026, 1034, 29, 0, 0, 0, 984, 1057, 972, 963, 0, 0, 0, 0, 1102, 990, 1074, 1028, 0, 0, 0, 0, 1042, 1048, 1013, 999, 17, 3, 1, 0, 1011, 1007, 1024, 1052, 0, 0, 0, 0, 1008, 1014, 1007, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4147, 4067, 3976, 4194, 4123, 4094, 4069] avg=4096.0000 max=4194 max_rank=4 min=3976 min_rank=3 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1018, 1035, 999, 1004, 13, 5, 3, 0, 1024, 998, 1027, 983, 0, 0, 0, 0, 1040, 1038, 1049, 990, 27, 0, 0, 0, 1056, 1024, 1019, 1030, 0, 0, 0, 0, 1007, 1056, 993, 1009, 0, 0, 0, 0, 1001, 1043, 1015, 1047, 17, 3, 1, 0, 1039, 985, 1017, 1045, 0, 0, 0, 0, 998, 1024, 1050, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4032, 4144, 4129, 4065, 4127, 4086, 4108] avg=4096.0000 max=4144 max_rank=2 min=4032 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[992, 1023, 984, 1053, 19, 8, 5, 0, 1040, 995, 1025, 1006, 0, 0, 0, 0, 1045, 1074, 1020, 1009, 30, 0, 0, 0, 1040, 1041, 1085, 995, 0, 0, 0, 0, 1051, 986, 950, 1010, 0, 0, 0, 0, 1063, 1026, 974, 1013, 21, 8, 4, 0, 999, 1025, 1035, 986, 0, 0, 0, 0, 1036, 1050, 1036, 1006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4066, 4178, 4161, 3997, 4109, 4045, 4128] avg=4096.0000 max=4178 max_rank=2 min=3997 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[981, 1047, 1046, 976, 13, 5, 3, 0, 1024, 976, 993, 1019, 0, 0, 0, 0, 1007, 1057, 1007, 972, 26, 0, 0, 0, 1012, 1041, 984, 1060, 0, 0, 0, 0, 1046, 1010, 987, 1024, 0, 0, 0, 0, 1017, 1081, 1014, 1007, 18, 3, 1, 0, 1037, 1082, 1009, 1000, 0, 0, 0, 0, 1067, 1025, 1051, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4012, 4069, 4097, 4067, 4141, 4128, 4183] avg=4096.0000 max=4183 max_rank=7 min=4012 min_rank=1 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1040, 1019, 1020, 1005, 13, 5, 3, 0, 1066, 950, 1026, 1019, 0, 0, 0, 0, 1008, 1048, 991, 988, 27, 0, 0, 0, 1019, 1003, 1022, 1073, 0, 0, 0, 0, 1005, 1036, 1010, 1059, 0, 0, 0, 0, 1002, 966, 1035, 1030, 17, 3, 1, 0, 1058, 1026, 1048, 1038, 0, 0, 0, 0, 1036, 1053, 987, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4061, 4062, 4117, 4110, 4054, 4170, 4089] avg=4096.0000 max=4170 max_rank=6 min=4054 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8064, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8064, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1032, 1037, 1100, 1002, 1107, 1055, 1029, 999, 1030, 1019, 1065, 1009, 1029, 1016, 1039, 984, 1008, 1045, 979, 1013, 1000, 986, 1028, 1019, 1042, 1024, 998, 1026, 994, 1051, 951, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1035, 1016, 1019, 998, 1038, 1056, 1034, 1049, 1010, 1004, 1062, 1063, 1011, 1042, 986, 1040, 1028, 1014, 1000, 1026, 935, 1029, 994, 1006, 969, 1038, 1035, 1039, 1045, 1055, 1086, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1022, 1004, 975, 1016, 1042, 1037, 1093, 984, 1054, 1032, 1042, 1025, 1022, 983, 1034, 995, 1021, 1060, 1032, 1023, 1040, 986, 1008, 1023, 1024, 1039, 1043, 986, 1001, 1038, 1032, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1052, 1020, 1052, 1022, 1008, 1050, 1039, 1019, 1020, 1005, 984, 1083, 996, 1015, 1036, 1058, 1054, 1044, 982, 991, 1035, 1059, 989, 995, 1006, 996, 1032, 1013, 1061, 1039, 994, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[994, 1045, 1030, 1045, 1023, 1014, 1000, 1004, 994, 1041, 1032, 978, 1004, 996, 995, 1009, 1027, 1005, 1010, 1014, 1072, 1006, 1006, 1023, 1072, 1016, 1002, 1001, 1063, 1067, 1077, 1103] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1058, 1013, 1027, 974, 1083, 1027, 1022, 1037, 1023, 1034, 988, 1019, 1020, 1052, 1069, 1059, 998, 1048, 1050, 945, 1021, 1022, 1021, 1047, 1023, 980, 1041, 1011, 1078, 1007, 1013, 958] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1048, 1026, 1037, 1042, 1010, 984, 1051, 1027, 1037, 1041, 1052, 1026, 981, 998, 1046, 1014, 1037, 989, 997, 994, 1003, 982, 1023, 991, 1038, 1094, 1041, 1028, 1027, 1005, 1031, 1068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1042, 1063, 1057, 945, 1064, 1056, 1040, 1042, 1015, 983, 991, 1036, 1016, 1008, 1027, 977, 1039, 962, 1042, 1021, 1051, 1052, 1027, 1020, 1010, 1034, 992, 1010, 985, 1049, 1034, 1078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1032, 1037, 1091, 1002, 0, 0, 0, 0, 1061, 1055, 1029, 999, 0, 0, 0, 0, 1030, 1019, 1065, 1006, 0, 0, 0, 0, 1029, 1016, 1039, 984, 21, 5, 0, 0, 1008, 1045, 979, 1013, 35, 0, 0, 0, 1000, 986, 1028, 1019, 23, 11, 0, 0, 1042, 1024, 998, 1026, 4, 3, 0, 0, 994, 1051, 951, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4144, 4120, 4094, 4080, 4067, 4097, 4004] avg=4096.0000 max=4162 max_rank=0 min=4004 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16248 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1035, 1016, 1010, 998, 0, 0, 0, 0, 995, 1056, 1034, 1049, 0, 0, 0, 0, 1010, 1004, 1062, 1060, 0, 0, 0, 0, 1011, 1042, 986, 1040, 20, 5, 0, 0, 1028, 1014, 1000, 1026, 33, 0, 0, 0, 935, 1029, 994, 1006, 22, 10, 0, 0, 969, 1038, 1035, 1039, 4, 3, 0, 0, 1045, 1055, 1086, 964, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4134, 4136, 4104, 4101, 3996, 4088, 4150] avg=4096.0000 max=4150 max_rank=7 min=3996 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[994, 1045, 1021, 1045, 0, 0, 0, 0, 981, 1014, 1000, 1004, 0, 0, 0, 0, 994, 1041, 1032, 975, 0, 0, 0, 0, 1004, 996, 995, 1009, 22, 5, 0, 0, 1027, 1005, 1010, 1014, 32, 0, 0, 0, 1072, 1006, 1006, 1023, 24, 10, 0, 0, 1072, 1016, 1002, 1001, 4, 3, 0, 0, 1063, 1067, 1077, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 3999, 4042, 4031, 4088, 4141, 4098, 4264] avg=4096.0000 max=4264 max_rank=7 min=3999 min_rank=1 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1022, 1004, 966, 1016, 0, 0, 0, 0, 999, 1037, 1093, 984, 0, 0, 0, 0, 1054, 1032, 1042, 1022, 0, 0, 0, 0, 1022, 983, 1034, 995, 21, 5, 0, 0, 1021, 1060, 1032, 1023, 33, 0, 0, 0, 1040, 986, 1008, 1023, 23, 10, 0, 0, 1024, 1039, 1043, 986, 4, 3, 0, 0, 1001, 1038, 1032, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4008, 4113, 4150, 4060, 4169, 4090, 4099, 4079] avg=4096.0000 max=4169 max_rank=4 min=4008 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1048, 1026, 1028, 1042, 0, 0, 0, 0, 968, 984, 1051, 1027, 0, 0, 0, 0, 1037, 1041, 1052, 1023, 0, 0, 0, 0, 981, 998, 1046, 1014, 21, 5, 0, 0, 1037, 989, 997, 994, 32, 0, 0, 0, 1003, 982, 1023, 991, 23, 10, 0, 0, 1038, 1094, 1041, 1028, 4, 3, 0, 0, 1027, 1005, 1031, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4030, 4153, 4065, 4049, 4032, 4208, 4087] avg=4096.0000 max=4208 max_rank=6 min=4030 min_rank=1 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1052, 1020, 1035, 1022, 0, 0, 0, 0, 957, 1050, 1039, 1019, 0, 0, 0, 0, 1020, 1005, 984, 1075, 0, 0, 0, 0, 996, 1015, 1036, 1058, 22, 9, 0, 0, 1054, 1044, 982, 991, 37, 0, 0, 0, 1035, 1059, 989, 995, 25, 14, 0, 0, 1006, 996, 1032, 1013, 8, 8, 0, 0, 1061, 1039, 994, 972, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4065, 4084, 4136, 4108, 4117, 4063, 4066] avg=4096.0000 max=4136 max_rank=3 min=4063 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1042, 1063, 1048, 945, 0, 0, 0, 0, 1020, 1056, 1040, 1042, 0, 0, 0, 0, 1015, 983, 991, 1033, 0, 0, 0, 0, 1016, 1008, 1027, 977, 21, 5, 0, 0, 1039, 962, 1042, 1021, 34, 0, 0, 0, 1051, 1052, 1027, 1020, 23, 10, 0, 0, 1010, 1034, 992, 1010, 4, 3, 0, 0, 985, 1049, 1034, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4158, 4022, 4054, 4098, 4183, 4053, 4102] avg=4096.0000 max=4183 max_rank=5 min=4022 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1058, 1013, 1018, 974, 0, 0, 0, 0, 1039, 1027, 1022, 1037, 0, 0, 0, 0, 1023, 1034, 988, 1016, 0, 0, 0, 0, 1020, 1052, 1069, 1059, 19, 5, 0, 0, 998, 1048, 1050, 945, 34, 0, 0, 0, 1021, 1022, 1021, 1047, 21, 10, 0, 0, 1023, 980, 1041, 1011, 4, 3, 0, 0, 1078, 1007, 1013, 918, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4125, 4061, 4224, 4075, 4142, 4062, 4016] avg=4096.0000 max=4224 max_rank=3 min=4016 min_rank=7 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8064, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1012, 1067, 1064, 1034, 1012, 1049, 998, 1020, 978, 1067, 1028, 1034, 1011, 984, 996, 1039, 1048, 1053, 1042, 1035, 1045, 1038, 985, 965, 1014, 1007, 1037, 988, 1024, 1031, 1004, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1014, 1033, 1036, 1099, 1007, 972, 985, 975, 981, 1065, 1033, 1029, 1024, 1042, 1026, 1061, 1040, 1026, 1008, 1038, 1002, 987, 1027, 1014, 1042, 1028, 1054, 999, 1023, 1021, 1035, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1025, 1009, 1032, 1016, 1024, 990, 1033, 1038, 1013, 1004, 1042, 1010, 974, 1005, 987, 1032, 983, 1030, 1077, 1082, 1033, 1013, 1008, 1001, 1014, 1031, 1048, 1029, 1095, 1058, 1018, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1017, 994, 1064, 1053, 988, 1029, 1014, 997, 1050, 1048, 1062, 1025, 1015, 1009, 1011, 1018, 1019, 1023, 986, 1021, 1022, 1013, 1021, 1058, 1003, 1004, 1112, 1026, 1030, 999, 1023, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1048, 995, 989, 1050, 1075, 1035, 1032, 1040, 1034, 1032, 1023, 1021, 1000, 1024, 975, 981, 1008, 976, 1003, 1044, 1037, 1009, 1055, 1051, 1079, 1006, 989, 1052, 1018, 1046, 1001, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1012, 1014, 1029, 1042, 1028, 1034, 1076, 1027, 1062, 987, 1037, 1034, 992, 1037, 996, 1005, 1081, 999, 1052, 1041, 1007, 970, 967, 1008, 1012, 1065, 1067, 999, 1020, 1041, 989, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1052, 1027, 990, 1006, 982, 1050, 1071, 1036, 1025, 991, 1011, 1009, 1019, 1071, 1004, 1051, 1025, 1040, 1012, 1032, 970, 1032, 1015, 1007, 995, 1034, 1060, 1047, 1073, 1029, 987, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[996, 1050, 1051, 1045, 1007, 1041, 1054, 973, 1064, 1099, 1070, 1043, 995, 987, 967, 1021, 1041, 1030, 996, 1002, 1020, 1027, 996, 1025, 1017, 1010, 1026, 1039, 995, 1062, 1016, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1014, 1033, 1036, 1074, 0, 0, 0, 0, 1007, 972, 985, 975, 9, 0, 0, 0, 981, 1065, 1003, 1029, 0, 0, 0, 0, 1024, 1042, 1026, 1061, 30, 21, 0, 0, 1040, 1026, 1008, 1023, 0, 0, 0, 0, 1002, 987, 1027, 1014, 20, 15, 4, 2, 1042, 1028, 1034, 999, 0, 0, 0, 0, 1023, 1010, 1035, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 3948, 4078, 4204, 4097, 4071, 4103, 4110] avg=4096.0000 max=4204 max_rank=3 min=3948 min_rank=1 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1012, 1067, 1064, 1011, 0, 0, 0, 0, 1012, 1049, 998, 1020, 9, 0, 0, 0, 978, 1067, 998, 1034, 0, 0, 0, 0, 1011, 984, 996, 1039, 30, 20, 0, 0, 1048, 1053, 1042, 1020, 0, 0, 0, 0, 1045, 1038, 985, 965, 20, 15, 3, 2, 1014, 1007, 1017, 988, 0, 0, 0, 0, 1024, 1020, 1004, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4088, 4077, 4080, 4163, 4073, 4026, 4107] avg=4096.0000 max=4163 max_rank=4 min=4026 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1025, 1009, 1032, 993, 0, 0, 0, 0, 1024, 990, 1033, 1038, 9, 0, 0, 0, 1013, 1004, 1012, 1010, 0, 0, 0, 0, 974, 1005, 987, 1032, 30, 20, 0, 0, 983, 1030, 1077, 1066, 0, 0, 0, 0, 1033, 1013, 1008, 1001, 20, 16, 3, 2, 1014, 1031, 1028, 1029, 0, 0, 0, 0, 1095, 1047, 1018, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4094, 4039, 4048, 4156, 4096, 4102, 4174] avg=4096.0000 max=4174 max_rank=7 min=4039 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16528 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1048, 995, 989, 1027, 0, 0, 0, 0, 1075, 1035, 1032, 1040, 9, 0, 0, 0, 1034, 1032, 994, 1021, 0, 0, 0, 0, 1000, 1024, 975, 981, 29, 20, 0, 0, 1008, 976, 1003, 1029, 0, 0, 0, 0, 1037, 1009, 1055, 1051, 19, 15, 3, 2, 1079, 1006, 970, 1052, 0, 0, 0, 0, 1018, 1035, 1001, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4191, 4081, 4029, 4016, 4191, 4107, 4094] avg=4096.0000 max=4191 max_rank=1 min=4016 min_rank=4 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1017, 994, 1064, 1030, 0, 0, 0, 0, 988, 1029, 1014, 997, 9, 0, 0, 0, 1050, 1048, 1031, 1025, 0, 0, 0, 0, 1015, 1009, 1011, 1018, 31, 20, 0, 0, 1019, 1023, 986, 1006, 0, 0, 0, 0, 1022, 1013, 1021, 1058, 21, 15, 3, 2, 1003, 1004, 1091, 1026, 0, 0, 0, 0, 1030, 988, 1023, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4037, 4154, 4104, 4034, 4155, 4124, 4055] avg=4096.0000 max=4155 max_rank=5 min=4034 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1052, 1027, 990, 983, 0, 0, 0, 0, 982, 1050, 1071, 1036, 9, 0, 0, 0, 1025, 991, 982, 1009, 0, 0, 0, 0, 1019, 1071, 1004, 1051, 29, 20, 0, 0, 1025, 1040, 1012, 1017, 0, 0, 0, 0, 970, 1032, 1015, 1007, 20, 15, 3, 2, 995, 1034, 1040, 1047, 0, 0, 0, 0, 1073, 1018, 987, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4148, 4007, 4194, 4094, 4064, 4116, 4093] avg=4096.0000 max=4194 max_rank=3 min=4007 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1012, 1014, 1029, 1019, 0, 0, 0, 0, 1028, 1034, 1076, 1027, 9, 0, 0, 0, 1062, 987, 1007, 1034, 0, 0, 0, 0, 992, 1037, 996, 1005, 30, 20, 0, 0, 1081, 999, 1052, 1026, 0, 0, 0, 0, 1007, 970, 967, 1008, 20, 15, 3, 2, 1012, 1065, 1047, 999, 0, 0, 0, 0, 1020, 1030, 989, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4174, 4090, 4080, 4158, 3992, 4123, 4077] avg=4096.0000 max=4174 max_rank=1 min=3992 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[996, 1050, 1051, 1011, 0, 0, 0, 0, 1007, 1041, 1054, 973, 13, 0, 0, 0, 1064, 1099, 1036, 1043, 0, 0, 0, 0, 995, 987, 967, 1021, 34, 25, 0, 0, 1041, 1030, 996, 983, 0, 0, 0, 0, 1020, 1027, 996, 1025, 25, 19, 9, 5, 1017, 1010, 1001, 1039, 0, 0, 0, 0, 995, 1044, 1016, 1003, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4088, 4242, 4029, 4050, 4126, 4067, 4058] avg=4096.0000 max=4242 max_rank=2 min=4029 min_rank=3 imbalance_ratio=1.035645 max_load_reduction=28526 max_load_reduction_pct=0.870544 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8064, 8320, 256, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 128, 128] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1011, 1056, 998, 1073, 1053, 981, 982, 980, 1034, 1025, 1029, 988, 991, 1030, 1071, 1043, 1018, 997, 1020, 1051, 1040, 1031, 1013, 1033, 1052, 1060, 968, 1033, 979, 1055, 1027, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1042, 1039, 1031, 1061, 1011, 1037, 1060, 996, 1046, 1008, 1061, 1013, 990, 1012, 1051, 1076, 1119, 970, 1025, 1043, 965, 1027, 1019, 1019, 985, 1028, 1002, 988, 980, 1018, 1023, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1045, 1006, 1030, 1027, 1010, 1062, 1027, 1014, 1031, 1039, 1029, 1031, 1025, 1038, 1050, 1028, 1047, 1025, 969, 1013, 1016, 1011, 1038, 1029, 1039, 1013, 1022, 1025, 1021, 1026, 1005, 977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1058, 1002, 979, 1035, 1024, 1012, 1032, 1019, 1040, 1042, 1056, 1024, 1020, 999, 1008, 1055, 1043, 1036, 1018, 1039, 986, 1024, 1034, 983, 1032, 1048, 987, 1037, 1016, 1043, 1012, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1041, 1080, 1046, 1042, 1032, 982, 1015, 972, 976, 1027, 971, 1027, 1022, 1016, 1008, 1037, 1048, 1076, 1039, 1010, 1039, 1008, 1095, 990, 989, 1004, 1092, 1005, 1045, 1046, 1011, 977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[983, 1047, 1018, 1016, 1084, 1036, 1062, 1000, 1054, 980, 1017, 1051, 1068, 1039, 1016, 1042, 1046, 1046, 1002, 987, 1014, 1019, 964, 1018, 1048, 1024, 1018, 986, 1041, 1004, 1020, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1049, 1032, 1033, 991, 1054, 982, 953, 1013, 1031, 1004, 957, 1034, 1016, 993, 1061, 1050, 1052, 990, 986, 1095, 1033, 1011, 1022, 1029, 1055, 1031, 985, 1067, 1054, 1030, 1030, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1044, 1017, 965, 1053, 1035, 1021, 1028, 1053, 1022, 1015, 1026, 1064, 1002, 1018, 1032, 1038, 970, 997, 992, 994, 1033, 1028, 1042, 1050, 1031, 1033, 1036, 1009, 1022, 1033, 1021, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1042, 1039, 1031, 1039, 0, 0, 0, 0, 1011, 1037, 1060, 996, 18, 0, 0, 0, 1046, 1008, 1061, 1013, 2, 0, 0, 0, 990, 1012, 1051, 1056, 0, 0, 0, 0, 1119, 970, 1025, 1043, 0, 0, 0, 0, 965, 1027, 1019, 1019, 8, 4, 0, 0, 985, 1028, 1002, 988, 4, 0, 0, 0, 980, 1018, 1023, 1023, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4122, 4130, 4109, 4157, 4042, 4007, 4050] avg=4096.0000 max=4157 max_rank=4 min=4007 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1058, 1002, 979, 1013, 0, 0, 0, 0, 1024, 1012, 1032, 1019, 18, 0, 0, 0, 1040, 1042, 1056, 1024, 2, 0, 0, 0, 1020, 999, 1008, 1035, 0, 0, 0, 0, 1043, 1036, 1018, 1039, 0, 0, 0, 0, 986, 1024, 1034, 983, 8, 4, 0, 0, 1032, 1048, 987, 1037, 4, 0, 0, 0, 1016, 1043, 1012, 1025, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4105, 4164, 4062, 4136, 4039, 4108, 4102] avg=4096.0000 max=4164 max_rank=2 min=4039 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1011, 1056, 998, 1051, 0, 0, 0, 0, 1053, 981, 982, 980, 18, 0, 0, 0, 1034, 1025, 1029, 988, 1, 0, 0, 0, 991, 1030, 1071, 1024, 0, 0, 0, 0, 1018, 997, 1020, 1051, 0, 0, 0, 0, 1040, 1031, 1013, 1033, 8, 4, 0, 0, 1052, 1060, 968, 1033, 4, 0, 0, 0, 979, 1055, 1027, 1046, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4014, 4077, 4116, 4086, 4129, 4117, 4113] avg=4096.0000 max=4129 max_rank=5 min=4014 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1045, 1006, 1030, 1005, 0, 0, 0, 0, 1010, 1062, 1027, 1014, 18, 0, 0, 0, 1031, 1039, 1029, 1031, 1, 0, 0, 0, 1025, 1038, 1050, 1009, 0, 0, 0, 0, 1047, 1025, 969, 1013, 0, 0, 0, 0, 1016, 1011, 1038, 1029, 8, 4, 0, 0, 1039, 1013, 1022, 1025, 4, 0, 0, 0, 1021, 1026, 1005, 977, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4131, 4131, 4122, 4054, 4106, 4103, 4035] avg=4096.0000 max=4131 max_rank=1 min=4035 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1041, 1080, 1046, 1020, 0, 0, 0, 0, 1032, 982, 1015, 972, 18, 0, 0, 0, 976, 1027, 971, 1027, 1, 0, 0, 0, 1022, 1016, 1008, 1018, 0, 0, 0, 0, 1048, 1076, 1039, 1010, 0, 0, 0, 0, 1039, 1008, 1095, 990, 8, 4, 0, 0, 989, 1004, 1092, 1005, 4, 0, 0, 0, 1045, 1046, 1011, 977, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4187, 4019, 4002, 4064, 4173, 4144, 4094, 4085] avg=4096.0000 max=4187 max_rank=0 min=4002 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[983, 1047, 1018, 987, 0, 0, 0, 0, 1084, 1036, 1062, 1000, 21, 0, 0, 0, 1054, 980, 1017, 1051, 6, 0, 0, 0, 1068, 1039, 1016, 1001, 0, 0, 0, 0, 1046, 1046, 1002, 987, 5, 0, 0, 0, 1014, 1019, 964, 1018, 13, 8, 0, 0, 1048, 1024, 1018, 986, 8, 0, 0, 0, 1041, 1004, 1020, 1018, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4203, 4108, 4124, 4086, 4036, 4084, 4092] avg=4096.0000 max=4203 max_rank=1 min=4035 min_rank=0 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1049, 1032, 1033, 970, 0, 0, 0, 0, 1054, 982, 953, 1013, 17, 0, 0, 0, 1031, 1004, 957, 1034, 2, 0, 0, 0, 1016, 993, 1061, 1030, 0, 0, 0, 0, 1052, 990, 986, 1095, 0, 0, 0, 0, 1033, 1011, 1022, 1029, 8, 4, 0, 0, 1055, 1031, 985, 1067, 4, 0, 0, 0, 1054, 1030, 1030, 1045, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4019, 4028, 4100, 4123, 4107, 4142, 4165] avg=4096.0000 max=4165 max_rank=7 min=4019 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1044, 1017, 965, 1031, 0, 0, 0, 0, 1035, 1021, 1028, 1053, 18, 0, 0, 0, 1022, 1015, 1026, 1064, 1, 0, 0, 0, 1002, 1018, 1032, 1019, 0, 0, 0, 0, 970, 997, 992, 994, 0, 0, 0, 0, 1033, 1028, 1042, 1050, 8, 4, 0, 0, 1031, 1033, 1036, 1009, 4, 0, 0, 0, 1022, 1033, 1021, 1044, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4155, 4128, 4071, 3953, 4165, 4113, 4126] avg=4096.0000 max=4165 max_rank=5 min=3953 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8192, 8064, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8064, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1040, 1064, 1018, 994, 1019, 1017, 1048, 1011, 989, 1011, 1013, 1031, 978, 984, 1032, 1010, 1036, 1026, 1058, 1084, 1007, 1017, 1020, 998, 1027, 994, 1012, 1051, 1087, 1009, 1024, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[991, 1071, 1043, 1002, 1038, 1038, 1056, 1011, 1069, 961, 1028, 941, 1061, 993, 1018, 1051, 993, 974, 1025, 1006, 1034, 1018, 1062, 1004, 1019, 1036, 1032, 1029, 1062, 1039, 1043, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[977, 1028, 1041, 1027, 1026, 1060, 975, 998, 1033, 1031, 1030, 1020, 998, 998, 1012, 1000, 988, 1047, 1050, 1044, 1065, 978, 982, 1029, 1045, 1065, 1011, 997, 1049, 1040, 1088, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1008, 981, 1011, 1001, 1057, 1042, 1087, 962, 1016, 1051, 1074, 1011, 1047, 985, 1082, 1010, 992, 1004, 1040, 1077, 1025, 1051, 995, 970, 1009, 1015, 1046, 1048, 981, 1033, 1041, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1023, 1038, 975, 1007, 1021, 1019, 1046, 1051, 963, 1032, 1007, 1015, 1051, 1008, 980, 996, 999, 1064, 1008, 1025, 1037, 992, 1015, 1041, 1080, 1013, 1038, 1037, 1029, 1050, 1075, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1055, 1022, 1020, 1068, 1040, 1018, 1025, 1003, 1026, 1028, 1102, 1034, 1003, 999, 990, 990, 1025, 1054, 1040, 1005, 1077, 1009, 998, 1008, 1002, 999, 1042, 1015, 1046, 1017, 1036, 972] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[990, 1021, 1021, 1036, 1022, 1039, 1033, 1018, 1037, 979, 1013, 1056, 1071, 1016, 995, 1004, 1004, 1021, 1004, 970, 1015, 1034, 1069, 1041, 1028, 1057, 1035, 1003, 1069, 1024, 1019, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1034, 1021, 1030, 1016, 1008, 1007, 1003, 1054, 1053, 1009, 992, 1068, 1024, 1059, 1061, 979, 1001, 1011, 1053, 1058, 996, 989, 1042, 1042, 1001, 998, 1013, 1030, 997, 1070, 1033, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[991, 1071, 1043, 1002, 7, 4, 0, 0, 1038, 1038, 1047, 1011, 0, 0, 0, 0, 1069, 961, 1028, 941, 3, 2, 0, 0, 1061, 993, 1018, 1051, 35, 0, 0, 0, 993, 974, 1023, 1006, 0, 0, 0, 0, 1034, 1018, 1062, 1004, 10, 2, 0, 0, 1019, 1036, 1025, 1029, 0, 0, 0, 0, 1062, 1039, 998, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4134, 4004, 4158, 3996, 4130, 4109, 4119] avg=4096.0000 max=4158 max_rank=3 min=3996 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[977, 1028, 1041, 1027, 7, 3, 0, 0, 1026, 1060, 966, 998, 0, 0, 0, 0, 1033, 1031, 1030, 1020, 3, 2, 0, 0, 998, 998, 1012, 1000, 36, 0, 0, 0, 988, 1047, 1048, 1044, 0, 0, 0, 0, 1065, 978, 982, 1029, 11, 2, 0, 0, 1045, 1065, 1005, 997, 0, 0, 0, 0, 1049, 1040, 1041, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4050, 4119, 4044, 4127, 4067, 4112, 4166] avg=4096.0000 max=4166 max_rank=7 min=4044 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1040, 1064, 1018, 994, 7, 3, 0, 0, 1019, 1017, 1039, 1011, 0, 0, 0, 0, 989, 1011, 1013, 1031, 3, 2, 0, 0, 978, 984, 1032, 1010, 34, 0, 0, 0, 1036, 1026, 1056, 1084, 0, 0, 0, 0, 1007, 1017, 1020, 998, 10, 2, 0, 0, 1027, 994, 1006, 1051, 0, 0, 0, 0, 1087, 1009, 980, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4086, 4049, 4038, 4202, 4054, 4078, 4135] avg=4096.0000 max=4202 max_rank=4 min=4038 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1008, 981, 1011, 1001, 8, 4, 0, 0, 1057, 1042, 1077, 962, 0, 0, 0, 0, 1016, 1051, 1074, 1011, 3, 2, 0, 0, 1047, 985, 1082, 1010, 35, 0, 0, 0, 992, 1004, 1038, 1077, 0, 0, 0, 0, 1025, 1051, 995, 970, 10, 2, 0, 0, 1009, 1015, 1039, 1048, 0, 0, 0, 0, 981, 1033, 996, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4138, 4157, 4159, 4111, 4053, 4111, 4026] avg=4096.0000 max=4159 max_rank=3 min=4013 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1055, 1022, 1020, 1068, 7, 4, 0, 0, 1040, 1018, 1016, 1003, 0, 0, 0, 0, 1026, 1028, 1102, 1034, 3, 2, 0, 0, 1003, 999, 990, 990, 35, 0, 0, 0, 1025, 1054, 1038, 1005, 0, 0, 0, 0, 1077, 1009, 998, 1008, 10, 2, 0, 0, 1002, 999, 1035, 1015, 0, 0, 0, 0, 1046, 1017, 991, 972, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4077, 4195, 4017, 4122, 4104, 4051, 4026] avg=4096.0000 max=4195 max_rank=2 min=4017 min_rank=3 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1023, 1038, 975, 1007, 12, 7, 0, 0, 1021, 1019, 1026, 1051, 0, 0, 0, 0, 963, 1032, 1007, 1015, 6, 4, 0, 0, 1051, 1008, 980, 996, 40, 0, 0, 0, 999, 1064, 1004, 1025, 0, 0, 0, 0, 1037, 992, 1015, 1041, 15, 8, 0, 0, 1080, 1013, 1025, 1037, 0, 0, 0, 0, 1029, 1050, 1020, 1033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4117, 4027, 4075, 4092, 4108, 4155, 4132] avg=4096.0000 max=4155 max_rank=6 min=4027 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[990, 1021, 1021, 1036, 7, 4, 0, 0, 1022, 1039, 1024, 1018, 0, 0, 0, 0, 1037, 979, 1013, 1056, 3, 2, 0, 0, 1071, 1016, 995, 1004, 34, 0, 0, 0, 1004, 1021, 1002, 970, 0, 0, 0, 0, 1015, 1034, 1069, 1041, 10, 2, 0, 0, 1028, 1057, 1028, 1003, 0, 0, 0, 0, 1069, 1024, 975, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4103, 4090, 4120, 3997, 4171, 4116, 4092] avg=4096.0000 max=4171 max_rank=5 min=3997 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1034, 1021, 1030, 1016, 7, 3, 0, 0, 1008, 1007, 994, 1054, 0, 0, 0, 0, 1053, 1009, 992, 1068, 3, 2, 0, 0, 1024, 1059, 1061, 979, 34, 0, 0, 0, 1001, 1011, 1051, 1058, 0, 0, 0, 0, 996, 989, 1042, 1042, 10, 2, 0, 0, 1001, 998, 1007, 1030, 0, 0, 0, 0, 997, 1070, 989, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4063, 4127, 4157, 4121, 4081, 4036, 4072] avg=4096.0000 max=4157 max_rank=3 min=4036 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8320, 8064, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8192, 8064, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1039, 1000, 1023, 1045, 1049, 1036, 1006, 1025, 1046, 1034, 984, 1020, 1009, 1053, 1016, 1028, 1044, 1053, 1024, 1003, 1027, 1046, 997, 1063, 992, 1006, 1057, 1001, 1026, 1019, 972, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1034, 982, 993, 986, 1072, 1081, 1021, 1032, 1021, 1028, 1023, 1027, 999, 1037, 1026, 1031, 1060, 1064, 998, 1032, 1019, 1057, 1004, 1032, 1040, 1011, 1033, 1030, 1013, 986, 961, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1008, 1018, 1013, 1027, 1000, 1016, 978, 1000, 1051, 1004, 1026, 968, 1036, 1020, 1042, 1089, 1031, 1059, 1029, 1113, 1021, 1005, 1027, 1002, 1037, 1029, 978, 1053, 1029, 959, 1043, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1047, 1026, 987, 1046, 1014, 1007, 1070, 1004, 1005, 1044, 977, 1030, 997, 1041, 1051, 972, 995, 1021, 1032, 1049, 1091, 1077, 1062, 958, 1013, 1049, 984, 1061, 976, 1016, 1056, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1018, 1022, 997, 1002, 1029, 1013, 1063, 1030, 1003, 1040, 1001, 997, 1016, 1037, 1049, 962, 1014, 1030, 1031, 1027, 1059, 1064, 999, 1048, 1056, 1008, 1035, 1063, 1049, 1029, 1003, 974] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1017, 1041, 964, 1039, 946, 989, 1043, 1047, 1015, 1015, 1070, 1038, 1005, 1042, 1054, 1027, 1018, 1032, 986, 1052, 1001, 1066, 1018, 1013, 1079, 1042, 1022, 997, 1014, 1025, 1015, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1030, 1033, 1032, 971, 1063, 1020, 1066, 1103, 1002, 1040, 1023, 1032, 977, 1056, 1015, 1005, 1041, 1015, 1004, 1005, 1023, 1047, 1014, 1028, 1031, 1003, 1031, 1028, 1027, 984, 985, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1033, 1022, 1046, 1071, 1011, 1008, 1043, 1000, 1077, 1036, 1001, 1046, 1009, 1070, 1023, 1015, 1018, 1008, 1050, 1003, 1050, 1025, 979, 1028, 1007, 1023, 988, 1017, 976, 1033, 1054, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1047, 1026, 987, 1046, 15, 4, 0, 0, 1014, 1007, 1055, 1004, 0, 0, 0, 0, 1005, 1044, 977, 1030, 4, 0, 0, 0, 997, 1037, 1051, 972, 0, 0, 0, 0, 995, 1021, 1032, 1028, 0, 0, 0, 0, 1091, 1054, 1062, 958, 0, 0, 0, 0, 1009, 1049, 984, 1061, 0, 0, 0, 0, 976, 1016, 1056, 1010, 23, 21, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4080, 4060, 4057, 4076, 4165, 4103, 4102] avg=4096.0000 max=4165 max_rank=5 min=4057 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1018, 1022, 997, 1002, 15, 4, 0, 0, 1029, 1013, 1048, 1030, 0, 0, 0, 0, 1003, 1040, 1001, 997, 4, 0, 0, 0, 1016, 1033, 1049, 962, 0, 0, 0, 0, 1014, 1030, 1031, 1007, 0, 0, 0, 0, 1059, 1041, 999, 1048, 0, 0, 0, 0, 1052, 1008, 1035, 1063, 0, 0, 0, 0, 1049, 1029, 1003, 974, 23, 20, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4120, 4045, 4060, 4082, 4147, 4158, 4098] avg=4096.0000 max=4158 max_rank=6 min=4045 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1030, 1033, 1032, 971, 15, 4, 0, 0, 1063, 1020, 1051, 1103, 0, 0, 0, 0, 1002, 1040, 1023, 1032, 4, 1, 0, 0, 977, 1051, 1015, 1005, 0, 0, 0, 0, 1041, 1015, 1004, 985, 0, 0, 0, 0, 1023, 1025, 1014, 1028, 0, 0, 0, 0, 1027, 1003, 1031, 1028, 0, 0, 0, 0, 1027, 984, 985, 1034, 22, 20, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4237, 4102, 4048, 4045, 4090, 4089, 4072] avg=4096.0000 max=4237 max_rank=1 min=4045 min_rank=4 imbalance_ratio=1.034424 max_load_reduction=28531 max_load_reduction_pct=0.870697 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1034, 982, 993, 986, 14, 4, 0, 0, 1072, 1081, 1007, 1032, 0, 0, 0, 0, 1021, 1028, 1023, 1027, 4, 0, 0, 0, 999, 1033, 1026, 1031, 0, 0, 0, 0, 1060, 1064, 998, 1012, 0, 0, 0, 0, 1019, 1035, 1004, 1032, 0, 0, 0, 0, 1036, 1011, 1033, 1030, 0, 0, 0, 0, 1013, 986, 961, 1035, 22, 20, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4192, 4103, 4089, 4134, 4090, 4110, 4037] avg=4096.0000 max=4192 max_rank=1 min=4013 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1039, 1000, 1023, 1045, 14, 4, 0, 0, 1049, 1036, 992, 1025, 0, 0, 0, 0, 1046, 1034, 984, 1020, 4, 1, 0, 0, 1009, 1048, 1016, 1028, 0, 0, 0, 0, 1044, 1053, 1024, 983, 0, 0, 0, 0, 1027, 1024, 997, 1063, 0, 0, 0, 0, 988, 1006, 1057, 1001, 0, 0, 0, 0, 1026, 1019, 972, 1025, 22, 20, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4102, 4089, 4101, 4104, 4111, 4052, 4084] avg=4096.0000 max=4125 max_rank=0 min=4052 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1017, 1041, 964, 1039, 14, 4, 0, 0, 946, 989, 1029, 1047, 0, 0, 0, 0, 1015, 1015, 1070, 1038, 4, 0, 0, 0, 1005, 1038, 1054, 1027, 0, 0, 0, 0, 1018, 1032, 986, 1031, 0, 0, 0, 0, 1001, 1043, 1018, 1013, 0, 0, 0, 0, 1075, 1042, 1022, 997, 0, 0, 0, 0, 1014, 1025, 1015, 1036, 23, 21, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4011, 4142, 4124, 4067, 4075, 4136, 4134] avg=4096.0000 max=4142 max_rank=2 min=4011 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1008, 1018, 1013, 1027, 16, 5, 6, 0, 1000, 1016, 962, 1000, 0, 0, 0, 0, 1051, 1004, 1026, 968, 8, 5, 0, 0, 1036, 1010, 1042, 1089, 0, 0, 0, 0, 1031, 1059, 1029, 1082, 0, 0, 0, 0, 1021, 980, 1027, 1002, 0, 0, 0, 0, 1029, 1029, 978, 1053, 0, 0, 0, 0, 1029, 959, 1043, 1057, 25, 25, 0, 0] rerouted_tokens_per_ep_rank=[4093, 3978, 4062, 4177, 4201, 4030, 4089, 4138] avg=4096.0000 max=4201 max_rank=4 min=3978 min_rank=1 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1033, 1022, 1046, 1071, 14, 4, 0, 0, 1011, 1008, 1029, 1000, 0, 0, 0, 0, 1077, 1036, 1001, 1046, 4, 1, 0, 0, 1009, 1065, 1023, 1015, 0, 0, 0, 0, 1018, 1008, 1050, 983, 0, 0, 0, 0, 1050, 1003, 979, 1028, 0, 0, 0, 0, 1003, 1023, 988, 1017, 0, 0, 0, 0, 976, 1033, 1054, 998, 22, 20, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4048, 4165, 4112, 4059, 4060, 4031, 4103] avg=4096.0000 max=4190 max_rank=0 min=4031 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16253 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8192, 256, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1054, 978, 1061, 1071, 1019, 1059, 934, 1011, 971, 1047, 1065, 1052, 1059, 1037, 983, 970, 1017, 1038, 1068, 1022, 1061, 981, 1018, 1057, 1042, 989, 1030, 994, 1029, 1020, 1014, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1010, 1006, 1064, 1052, 947, 1011, 1061, 1008, 1024, 1036, 949, 1019, 1014, 1027, 999, 1023, 1026, 1051, 1005, 1073, 1028, 1033, 1027, 993, 1024, 1038, 1009, 1028, 1061, 1024, 994, 1104] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1030, 1036, 1062, 1080, 1110, 989, 1033, 1038, 1029, 1040, 989, 1024, 1008, 1017, 1021, 1059, 985, 1042, 1027, 991, 1002, 964, 1041, 1014, 1045, 1049, 1024, 1021, 981, 953, 1033, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1015, 1020, 1018, 1015, 1040, 1024, 1004, 1078, 1051, 1054, 986, 957, 1003, 1010, 1045, 1034, 973, 1019, 1046, 1041, 1022, 1093, 1019, 1008, 1043, 1052, 1009, 985, 1039, 1029, 1045, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1019, 1057, 1019, 1044, 994, 1051, 977, 1047, 1025, 1010, 1024, 1028, 979, 1020, 1041, 949, 1034, 1005, 1077, 1048, 964, 1024, 1056, 1032, 1056, 1018, 1061, 1067, 1016, 999, 968, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[995, 1046, 1043, 1009, 1022, 1008, 1023, 1025, 990, 994, 1005, 1046, 1023, 975, 992, 1012, 1008, 1076, 1020, 1000, 1030, 1026, 1050, 1048, 1017, 1029, 1051, 1015, 1080, 1049, 1053, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1025, 1037, 982, 1063, 1000, 1017, 1015, 1011, 1064, 974, 1067, 1013, 1027, 1012, 1023, 1067, 1047, 1020, 987, 1031, 1035, 1040, 1044, 1032, 1019, 1000, 1031, 1009, 1024, 1052, 1013, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1130, 1051, 1010, 1035, 1053, 1038, 1046, 1003, 1022, 1035, 1005, 1016, 1035, 1015, 953, 990, 1036, 1012, 1036, 989, 1035, 1009, 1031, 995, 1075, 1012, 986, 1006, 1074, 1028, 1027, 980] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1054, 978, 1061, 1025, 0, 0, 0, 0, 1019, 1059, 934, 1011, 5, 1, 1, 0, 971, 1047, 1065, 1052, 10, 6, 2, 0, 1059, 1037, 983, 970, 44, 0, 0, 0, 1017, 1038, 1058, 1022, 0, 0, 0, 0, 1061, 981, 1013, 1057, 0, 0, 0, 0, 1035, 989, 1030, 994, 0, 0, 0, 0, 1028, 1020, 1014, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4030, 4153, 4093, 4135, 4112, 4048, 4079] avg=4096.0000 max=4153 max_rank=2 min=4030 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1030, 1036, 1062, 1034, 0, 0, 0, 0, 1110, 989, 1033, 1038, 5, 1, 1, 0, 1029, 1040, 989, 1024, 10, 6, 2, 0, 1008, 1017, 1021, 1059, 44, 0, 0, 0, 985, 1042, 1017, 991, 0, 0, 0, 0, 1002, 964, 1036, 1014, 0, 0, 0, 0, 1038, 1049, 1024, 1021, 0, 0, 0, 0, 980, 953, 1033, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4177, 4100, 4149, 4035, 4016, 4132, 3997] avg=4096.0000 max=4177 max_rank=1 min=3997 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16180 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1010, 1006, 1064, 1007, 0, 0, 0, 0, 947, 1011, 1061, 1008, 5, 1, 1, 0, 1024, 1036, 949, 1019, 9, 6, 2, 0, 1014, 1027, 999, 1023, 43, 0, 0, 0, 1026, 1051, 996, 1073, 0, 0, 0, 0, 1028, 1033, 1022, 993, 0, 0, 0, 0, 1017, 1038, 1009, 1028, 0, 0, 0, 0, 1060, 1024, 994, 1104, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4034, 4045, 4106, 4146, 4076, 4092, 4182] avg=4096.0000 max=4182 max_rank=7 min=4034 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1015, 1020, 1018, 972, 0, 0, 0, 0, 1040, 1024, 1004, 1078, 5, 1, 1, 0, 1051, 1054, 986, 957, 10, 6, 2, 0, 1003, 1010, 1045, 1034, 41, 0, 0, 0, 973, 1019, 1036, 1041, 0, 0, 0, 0, 1022, 1093, 1014, 1008, 0, 0, 0, 0, 1036, 1052, 1009, 985, 0, 0, 0, 0, 1038, 1029, 1045, 991, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4153, 4066, 4133, 4069, 4137, 4082, 4103] avg=4096.0000 max=4153 max_rank=1 min=4025 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1019, 1057, 1019, 999, 0, 0, 0, 0, 994, 1051, 977, 1047, 5, 1, 1, 0, 1025, 1010, 1024, 1028, 10, 6, 2, 0, 979, 1020, 1041, 949, 43, 0, 0, 0, 1034, 1005, 1067, 1048, 0, 0, 0, 0, 964, 1024, 1051, 1032, 0, 0, 0, 0, 1049, 1018, 1061, 1067, 0, 0, 0, 0, 1015, 999, 968, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4076, 4105, 4032, 4154, 4071, 4195, 4041] avg=4096.0000 max=4195 max_rank=6 min=4032 min_rank=3 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[995, 1046, 1043, 966, 0, 0, 0, 0, 1022, 1008, 1023, 1025, 5, 1, 1, 0, 990, 994, 1005, 1046, 10, 6, 2, 0, 1023, 975, 992, 1012, 41, 0, 0, 0, 1008, 1076, 1010, 1000, 0, 0, 0, 0, 1030, 1026, 1045, 1048, 0, 0, 0, 0, 1010, 1029, 1051, 1015, 0, 0, 0, 0, 1079, 1049, 1053, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4085, 4053, 4043, 4094, 4149, 4105, 4189] avg=4096.0000 max=4189 max_rank=7 min=4043 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1025, 1037, 982, 1018, 0, 0, 0, 0, 1000, 1017, 1015, 1011, 5, 1, 1, 0, 1064, 974, 1067, 1013, 9, 6, 2, 0, 1027, 1012, 1023, 1067, 43, 0, 0, 0, 1047, 1020, 978, 1031, 0, 0, 0, 0, 1035, 1040, 1039, 1032, 0, 0, 0, 0, 1012, 1000, 1031, 1009, 0, 0, 0, 0, 1023, 1052, 1013, 987, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4050, 4135, 4172, 4076, 4146, 4052, 4075] avg=4096.0000 max=4172 max_rank=3 min=4050 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1130, 1051, 1010, 979, 0, 0, 0, 0, 1053, 1038, 1046, 1003, 9, 7, 7, 0, 1022, 1035, 1005, 1016, 14, 10, 9, 0, 1035, 1015, 953, 990, 47, 0, 0, 0, 1036, 1012, 1022, 989, 0, 0, 0, 0, 1035, 1009, 1022, 995, 0, 0, 0, 0, 1058, 1012, 986, 1006, 0, 0, 0, 0, 1067, 1028, 1027, 980, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4163, 4111, 4040, 4059, 4061, 4062, 4102] avg=4096.0000 max=4170 max_rank=0 min=4040 min_rank=3 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8064, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1005, 1033, 984, 1031, 1016, 1012, 1037, 1030, 1030, 1042, 1030, 1050, 1010, 1018, 1016, 1033, 1032, 1028, 1028, 1017, 1055, 1027, 1019, 1021, 999, 1036, 1043, 1041, 1008, 998, 1016, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1030, 1011, 1018, 1016, 1032, 1044, 1014, 1012, 1006, 1009, 1088, 1017, 1023, 1055, 998, 1021, 977, 1031, 1048, 1033, 1029, 1052, 991, 1036, 1016, 1063, 1010, 1050, 991, 1041, 1020, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[962, 1029, 1007, 985, 1007, 1008, 990, 1014, 1000, 1005, 1036, 1033, 1032, 959, 1010, 1019, 1095, 1057, 1040, 1054, 1059, 1050, 1024, 1026, 1059, 989, 1008, 1028, 1068, 998, 1032, 1085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1044, 1010, 1059, 1032, 1050, 1038, 963, 999, 1031, 993, 1008, 999, 1000, 996, 1056, 1079, 985, 1045, 1033, 1001, 1020, 1079, 1017, 1113, 997, 1040, 963, 1053, 950, 1045, 1051, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[968, 1017, 1050, 1033, 1030, 1054, 1011, 964, 1053, 1059, 1024, 1031, 1010, 1020, 1042, 992, 1036, 988, 1020, 1039, 1055, 1038, 1035, 1095, 1026, 1013, 1073, 1003, 1013, 976, 992, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1020, 1010, 1039, 979, 1027, 1045, 1043, 964, 1009, 1068, 1009, 1018, 1068, 1054, 991, 1057, 994, 1051, 1030, 1019, 1008, 1051, 1023, 1056, 1041, 1008, 996, 1012, 1000, 1049, 1009, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1032, 1038, 981, 1013, 1039, 1001, 1028, 1001, 1010, 1014, 1021, 1023, 1017, 983, 1127, 1003, 1062, 1000, 1043, 1019, 1021, 1044, 1016, 1030, 1050, 1005, 1014, 986, 1030, 1046, 1039, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1074, 1019, 1023, 1045, 1016, 1049, 990, 981, 1040, 994, 1058, 992, 1006, 1014, 1024, 1023, 1048, 1028, 1078, 967, 1075, 1060, 992, 1047, 1011, 1009, 1030, 1012, 993, 987, 1066, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[962, 1029, 1007, 985, 20, 0, 0, 0, 1007, 1008, 990, 1014, 31, 0, 0, 0, 1000, 1005, 1033, 1033, 0, 0, 0, 0, 1032, 959, 1010, 1019, 1, 0, 0, 0, 1095, 1057, 1021, 1054, 0, 0, 0, 0, 1059, 1050, 1024, 967, 0, 0, 0, 0, 1059, 989, 1008, 1028, 8, 2, 0, 0, 1068, 998, 1032, 1085, 11, 8, 0, 0] rerouted_tokens_per_ep_rank=[4003, 4050, 4071, 4021, 4227, 4100, 4094, 4202] avg=4096.0000 max=4227 max_rank=4 min=4003 min_rank=0 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[968, 1017, 1050, 1033, 22, 0, 0, 0, 1030, 1054, 1011, 964, 33, 0, 0, 0, 1053, 1059, 1021, 1031, 0, 0, 0, 0, 1010, 1020, 1042, 992, 1, 0, 0, 0, 1036, 988, 1002, 1039, 0, 0, 0, 0, 1055, 1038, 1035, 1032, 0, 0, 0, 0, 1026, 1013, 1073, 1003, 7, 2, 0, 0, 1013, 976, 992, 1008, 11, 8, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4092, 4164, 4065, 4065, 4160, 4124, 4008] avg=4096.0000 max=4164 max_rank=2 min=4008 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1044, 1010, 1059, 1032, 22, 0, 0, 0, 1050, 1038, 963, 999, 34, 0, 0, 0, 1031, 993, 1005, 999, 0, 0, 0, 0, 1000, 996, 1056, 1079, 1, 0, 0, 0, 985, 1045, 1015, 1001, 0, 0, 0, 0, 1020, 1079, 1017, 1049, 0, 0, 0, 0, 997, 1040, 963, 1053, 7, 2, 0, 0, 950, 1045, 1051, 1019, 11, 8, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4084, 4028, 4132, 4046, 4165, 4062, 4084] avg=4096.0000 max=4167 max_rank=0 min=4028 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1005, 1033, 984, 1031, 20, 0, 0, 0, 1016, 1012, 1037, 1030, 31, 0, 0, 0, 1030, 1042, 1027, 1050, 0, 0, 0, 0, 1010, 1018, 1016, 1033, 1, 0, 0, 0, 1032, 1028, 1010, 1017, 0, 0, 0, 0, 1055, 1027, 1019, 963, 0, 0, 0, 0, 999, 1036, 1043, 1041, 7, 2, 0, 0, 1008, 998, 1016, 1023, 11, 7, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4126, 4149, 4078, 4087, 4064, 4128, 4063] avg=4096.0000 max=4149 max_rank=2 min=4063 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1030, 1011, 1018, 1016, 25, 0, 0, 0, 1032, 1044, 1014, 1012, 35, 0, 0, 0, 1006, 1009, 1077, 1017, 0, 0, 0, 0, 1023, 1055, 998, 1021, 5, 0, 0, 0, 977, 1031, 1020, 1033, 0, 0, 0, 0, 1029, 1052, 991, 965, 0, 0, 0, 0, 1016, 1063, 1010, 1050, 12, 6, 0, 0, 991, 1041, 1020, 986, 16, 11, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4137, 4109, 4102, 4061, 4037, 4157, 4065] avg=4096.0000 max=4157 max_rank=6 min=4037 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1020, 1010, 1039, 979, 21, 0, 0, 0, 1027, 1045, 1043, 964, 32, 0, 0, 0, 1009, 1068, 1006, 1018, 0, 0, 0, 0, 1068, 1054, 991, 1057, 1, 0, 0, 0, 994, 1051, 1012, 1019, 0, 0, 0, 0, 1008, 1051, 1023, 995, 0, 0, 0, 0, 1041, 1008, 996, 1012, 7, 2, 0, 0, 1000, 1049, 1009, 1020, 11, 8, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4111, 4101, 4171, 4076, 4077, 4066, 4097] avg=4096.0000 max=4171 max_rank=3 min=4066 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1032, 1038, 981, 1013, 20, 0, 0, 0, 1039, 1001, 1028, 1001, 31, 0, 0, 0, 1010, 1014, 1018, 1023, 0, 0, 0, 0, 1017, 983, 1127, 1003, 1, 0, 0, 0, 1062, 1000, 1024, 1019, 0, 0, 0, 0, 1021, 1044, 1016, 971, 0, 0, 0, 0, 1050, 1005, 1014, 986, 8, 2, 0, 0, 1030, 1046, 1039, 1032, 11, 8, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4100, 4065, 4131, 4105, 4052, 4065, 4166] avg=4096.0000 max=4166 max_rank=7 min=4052 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1074, 1019, 1023, 1045, 21, 0, 0, 0, 1016, 1049, 990, 981, 32, 0, 0, 0, 1040, 994, 1055, 992, 0, 0, 0, 0, 1006, 1014, 1024, 1023, 1, 0, 0, 0, 1048, 1028, 1058, 967, 0, 0, 0, 0, 1075, 1060, 992, 986, 0, 0, 0, 0, 1011, 1009, 1030, 1012, 8, 2, 0, 0, 993, 987, 1066, 1017, 12, 8, 0, 0] rerouted_tokens_per_ep_rank=[4182, 4068, 4081, 4068, 4101, 4113, 4072, 4083] avg=4096.0000 max=4182 max_rank=0 min=4068 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 8448, 8192, 7936, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8064, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1050, 1046, 1009, 981, 1049, 1052, 1001, 1012, 988, 1065, 1004, 1012, 958, 1040, 1041, 1059, 991, 1020, 1015, 1000, 1051, 1044, 1020, 1021, 1020, 1031, 1054, 1041, 1034, 1034, 1029, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[981, 1053, 1068, 1041, 1012, 1055, 991, 1022, 1018, 1024, 1003, 1012, 1029, 1072, 1015, 995, 1004, 987, 984, 1017, 1013, 1023, 985, 1035, 999, 1038, 1025, 1038, 1039, 1097, 1043, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[991, 1013, 1027, 1021, 1017, 1000, 1016, 1005, 989, 1020, 1064, 1014, 1008, 1033, 1018, 997, 1031, 1064, 1048, 1043, 1059, 1016, 993, 1049, 1065, 1014, 1018, 1026, 1010, 1005, 1043, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1018, 978, 1013, 1063, 1049, 980, 1030, 1002, 1063, 971, 1052, 1019, 1009, 1062, 1045, 1026, 1006, 1000, 1017, 1018, 981, 1058, 987, 1035, 1060, 975, 1041, 997, 1055, 1041, 1040, 1077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1018, 1052, 1031, 1044, 1032, 1000, 1001, 1039, 1067, 1031, 1000, 1006, 1057, 1031, 1030, 1000, 994, 962, 1071, 1021, 1019, 1027, 1008, 1030, 1012, 1013, 1072, 1028, 985, 1047, 1031, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1027, 1034, 980, 1038, 1020, 1033, 1005, 964, 980, 1027, 966, 1050, 1052, 1008, 1027, 1050, 1061, 989, 1033, 1041, 977, 1065, 1037, 1067, 980, 1051, 1033, 1019, 1073, 1026, 1023, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1033, 1047, 1016, 1007, 990, 1024, 1029, 1000, 1029, 1101, 1042, 1024, 1071, 996, 1047, 1034, 1006, 1052, 1019, 1031, 1005, 1024, 1050, 974, 998, 987, 993, 1018, 1046, 960, 1061, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1013, 1053, 1037, 1036, 1003, 1017, 988, 1044, 1072, 1004, 1040, 1098, 988, 1002, 1058, 1068, 998, 1034, 1015, 1019, 1017, 1023, 1035, 995, 1015, 974, 1003, 1022, 1014, 1020, 1019, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1050, 1041, 1009, 981, 0, 0, 0, 0, 1049, 1052, 1001, 1012, 34, 0, 0, 0, 988, 1054, 1004, 1012, 0, 0, 0, 0, 958, 1040, 1023, 1059, 0, 0, 0, 0, 991, 1020, 1015, 1000, 17, 4, 0, 0, 1051, 1044, 1020, 1021, 5, 0, 0, 0, 1020, 1031, 1054, 1041, 11, 1, 0, 0, 1034, 1034, 1029, 958, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4148, 4058, 4080, 4047, 4141, 4158, 4055] avg=4096.0000 max=4158 max_rank=6 min=4047 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[981, 1048, 1068, 1041, 0, 0, 0, 0, 1012, 1055, 991, 1022, 36, 0, 0, 0, 1018, 1014, 1003, 1012, 0, 0, 0, 0, 1029, 1072, 997, 995, 0, 0, 0, 0, 1004, 987, 984, 1017, 17, 4, 0, 0, 1013, 1023, 985, 1035, 5, 0, 0, 0, 999, 1038, 1025, 1038, 10, 1, 0, 0, 1039, 1097, 1043, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4116, 4047, 4093, 4013, 4061, 4111, 4189] avg=4096.0000 max=4189 max_rank=7 min=4013 min_rank=4 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1018, 973, 1013, 1063, 0, 0, 0, 0, 1049, 980, 1030, 1002, 37, 0, 0, 0, 1063, 961, 1052, 1019, 0, 0, 0, 0, 1009, 1062, 1026, 1026, 0, 0, 0, 0, 1006, 1000, 1017, 1018, 18, 4, 0, 0, 981, 1058, 987, 1035, 5, 0, 0, 0, 1060, 975, 1041, 997, 10, 1, 0, 0, 1055, 1041, 1040, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4098, 4095, 4123, 4063, 4066, 4084, 4172] avg=4096.0000 max=4172 max_rank=7 min=4063 min_rank=4 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[991, 1008, 1027, 1021, 0, 0, 0, 0, 1017, 1000, 1016, 1005, 36, 0, 0, 0, 989, 1010, 1064, 1014, 0, 0, 0, 0, 1008, 1033, 1000, 997, 0, 0, 0, 0, 1031, 1064, 1048, 1043, 17, 4, 0, 0, 1059, 1016, 993, 1049, 5, 0, 0, 0, 1065, 1014, 1018, 1026, 10, 1, 0, 0, 1010, 1005, 1043, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4074, 4077, 4038, 4207, 4122, 4134, 4069] avg=4096.0000 max=4207 max_rank=4 min=4038 min_rank=3 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16532 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1018, 1036, 1031, 1044, 0, 0, 0, 0, 1032, 1000, 1001, 1039, 37, 0, 0, 0, 1067, 1016, 1000, 1006, 0, 0, 0, 0, 1057, 1031, 1001, 1000, 0, 0, 0, 0, 994, 962, 1071, 1021, 21, 6, 0, 0, 1019, 1027, 1008, 1030, 10, 0, 0, 0, 1012, 1013, 1072, 1028, 15, 8, 6, 0, 985, 1047, 1031, 966, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4109, 4089, 4089, 4075, 4094, 4154, 4029] avg=4096.0000 max=4154 max_rank=6 min=4029 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1027, 1029, 980, 1038, 0, 0, 0, 0, 1020, 1033, 1005, 964, 35, 0, 0, 0, 980, 1017, 966, 1050, 0, 0, 0, 0, 1052, 1008, 1009, 1050, 0, 0, 0, 0, 1061, 989, 1033, 1041, 17, 4, 0, 0, 977, 1065, 1037, 1067, 5, 0, 0, 0, 980, 1051, 1033, 1019, 10, 1, 0, 0, 1073, 1026, 1023, 993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4057, 4013, 4119, 4145, 4151, 4094, 4115] avg=4096.0000 max=4151 max_rank=5 min=4013 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1013, 1048, 1037, 1036, 0, 0, 0, 0, 1003, 1017, 988, 1044, 35, 0, 0, 0, 1072, 994, 1040, 1098, 0, 0, 0, 0, 988, 1002, 1039, 1068, 0, 0, 0, 0, 998, 1034, 1015, 1019, 18, 4, 0, 0, 1017, 1023, 1035, 995, 5, 0, 0, 0, 1015, 974, 1003, 1022, 10, 1, 0, 0, 1014, 1020, 1019, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4087, 4204, 4097, 4088, 4075, 4025, 4058] avg=4096.0000 max=4204 max_rank=2 min=4025 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1033, 1042, 1016, 1007, 0, 0, 0, 0, 990, 1024, 1029, 1000, 36, 0, 0, 0, 1029, 1090, 1042, 1024, 0, 0, 0, 0, 1071, 996, 1028, 1034, 0, 0, 0, 0, 1006, 1052, 1019, 1031, 18, 4, 0, 0, 1005, 1024, 1050, 974, 5, 0, 0, 0, 998, 987, 993, 1018, 11, 1, 0, 0, 1046, 960, 1061, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4079, 4185, 4129, 4130, 4058, 4008, 4081] avg=4096.0000 max=4185 max_rank=2 min=4008 min_rank=6 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8064, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1041, 1015, 1060, 1013, 1015, 1009, 1023, 1037, 998, 1110, 999, 1025, 1048, 974, 1020, 1055, 1017, 998, 1027, 1010, 996, 1045, 987, 1024, 1009, 1042, 1027, 1030, 1017, 1039, 1060, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1042, 1056, 987, 996, 1003, 1022, 994, 1052, 1028, 1019, 991, 1006, 1008, 994, 1013, 1060, 1054, 1037, 1058, 1029, 1023, 1037, 975, 1014, 1033, 1027, 1012, 1031, 1040, 1035, 1067, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1034, 997, 1068, 996, 1040, 970, 1058, 1022, 1008, 1056, 1052, 1037, 1004, 1041, 989, 1064, 964, 985, 1029, 1084, 1049, 952, 1044, 1026, 1064, 1019, 1006, 997, 1008, 1054, 1012, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1041, 1004, 988, 1065, 1059, 1070, 1040, 1013, 1023, 1026, 986, 1002, 1071, 1009, 1001, 991, 1014, 1036, 1005, 1055, 1019, 1022, 1014, 1051, 1028, 1021, 982, 986, 1069, 1038, 1021, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1015, 1045, 1007, 980, 1007, 1063, 1060, 1063, 995, 1005, 1005, 999, 1049, 1009, 1020, 1020, 1018, 1020, 975, 1087, 1033, 1041, 1020, 1082, 992, 996, 1042, 1051, 1018, 1020, 998, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1052, 1038, 995, 1005, 1064, 990, 1009, 1007, 1072, 1019, 1060, 1015, 995, 1059, 1039, 1042, 1050, 997, 1035, 992, 978, 1056, 1027, 1000, 1018, 1008, 1008, 1028, 1056, 1006, 1046, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1021, 1036, 1060, 1001, 1030, 1062, 986, 1020, 1031, 1015, 1020, 1041, 1058, 983, 1062, 1043, 1021, 1031, 1038, 973, 1025, 974, 1027, 1063, 1026, 1018, 1050, 1032, 976, 1012, 1043, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[997, 992, 994, 1040, 1004, 1011, 1004, 1027, 982, 1030, 1044, 1042, 1034, 1015, 1017, 1025, 1007, 1035, 1023, 1019, 1009, 1048, 1084, 1008, 1020, 1075, 1034, 1017, 1023, 1058, 1036, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1042, 1056, 987, 996, 11, 0, 0, 0, 1003, 1022, 994, 1045, 0, 0, 0, 0, 1028, 1019, 991, 1006, 3, 0, 0, 0, 1008, 994, 1013, 1056, 0, 0, 0, 0, 1054, 1037, 1058, 1029, 3, 2, 0, 0, 1023, 1037, 975, 1014, 1, 0, 0, 0, 1033, 1027, 1012, 1031, 4, 0, 0, 0, 1040, 1035, 1054, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4064, 4047, 4071, 4183, 4050, 4107, 4154] avg=4096.0000 max=4183 max_rank=4 min=4047 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1034, 997, 1068, 996, 10, 0, 0, 0, 1040, 970, 1058, 1015, 0, 0, 0, 0, 1008, 1056, 1052, 1037, 3, 0, 0, 0, 1004, 1041, 989, 1060, 0, 0, 0, 0, 964, 985, 1029, 1084, 3, 1, 0, 0, 1049, 952, 1044, 1026, 1, 0, 0, 0, 1064, 1019, 1006, 997, 4, 0, 0, 0, 1008, 1054, 1001, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4083, 4156, 4094, 4066, 4072, 4090, 4102] avg=4096.0000 max=4156 max_rank=2 min=4066 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1041, 1015, 1060, 1013, 11, 0, 0, 0, 1015, 1009, 1023, 1030, 0, 0, 0, 0, 998, 1110, 999, 1025, 3, 0, 0, 0, 1048, 974, 1020, 1051, 0, 0, 0, 0, 1017, 998, 1027, 1010, 3, 2, 0, 0, 996, 1045, 987, 1024, 1, 0, 0, 0, 1009, 1042, 1027, 1030, 4, 0, 0, 0, 1017, 1039, 1047, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4077, 4135, 4093, 4057, 4053, 4112, 4101] avg=4096.0000 max=4140 max_rank=0 min=4053 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1041, 1004, 988, 1065, 10, 0, 0, 0, 1059, 1070, 1040, 1006, 0, 0, 0, 0, 1023, 1026, 986, 1002, 3, 0, 0, 0, 1071, 1009, 1001, 987, 0, 0, 0, 0, 1014, 1036, 1005, 1055, 3, 1, 0, 0, 1019, 1022, 1014, 1051, 1, 0, 0, 0, 1028, 1021, 982, 986, 4, 0, 0, 0, 1069, 1038, 1010, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4175, 4040, 4068, 4114, 4107, 4021, 4135] avg=4096.0000 max=4175 max_rank=1 min=4021 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1015, 1045, 1007, 980, 15, 0, 0, 0, 1007, 1063, 1060, 1046, 0, 0, 0, 0, 995, 1005, 1005, 999, 6, 0, 0, 0, 1049, 1009, 1020, 1004, 0, 0, 0, 0, 1018, 1020, 975, 1087, 8, 4, 0, 0, 1033, 1041, 1020, 1082, 8, 0, 0, 0, 992, 996, 1042, 1051, 9, 2, 0, 0, 1018, 1020, 979, 1033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4176, 4010, 4082, 4112, 4184, 4092, 4050] avg=4096.0000 max=4184 max_rank=5 min=4010 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1052, 1038, 995, 1005, 10, 0, 0, 0, 1064, 990, 1009, 1000, 0, 0, 0, 0, 1072, 1019, 1060, 1015, 3, 0, 0, 0, 995, 1059, 1039, 1038, 0, 0, 0, 0, 1050, 997, 1035, 992, 3, 2, 0, 0, 978, 1056, 1027, 1000, 1, 0, 0, 0, 1018, 1008, 1008, 1028, 4, 0, 0, 0, 1056, 1006, 1034, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4063, 4169, 4131, 4079, 4062, 4066, 4098] avg=4096.0000 max=4169 max_rank=2 min=4062 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[997, 992, 994, 1040, 10, 0, 0, 0, 1004, 1011, 1004, 1020, 0, 0, 0, 0, 982, 1030, 1044, 1042, 3, 0, 0, 0, 1034, 1015, 1017, 1021, 0, 0, 0, 0, 1007, 1035, 1023, 1019, 3, 2, 0, 0, 1009, 1048, 1084, 1008, 1, 0, 0, 0, 1020, 1075, 1034, 1017, 4, 0, 0, 0, 1023, 1058, 1024, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4039, 4101, 4087, 4089, 4150, 4150, 4119] avg=4096.0000 max=4150 max_rank=5 min=4033 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1021, 1036, 1060, 1001, 10, 0, 0, 0, 1030, 1062, 986, 1013, 0, 0, 0, 0, 1031, 1015, 1020, 1041, 3, 0, 0, 0, 1058, 983, 1062, 1039, 0, 0, 0, 0, 1021, 1031, 1038, 973, 3, 2, 0, 0, 1025, 974, 1027, 1063, 1, 0, 0, 0, 1026, 1018, 1050, 1032, 4, 0, 0, 0, 976, 1012, 1031, 990, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4091, 4110, 4142, 4068, 4090, 4130, 4009] avg=4096.0000 max=4142 max_rank=3 min=4009 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16297 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1025, 995, 1019, 1064, 1002, 998, 995, 1026, 957, 1039, 1006, 1004, 1018, 1007, 1011, 1039, 1039, 1025, 1069, 1094, 1053, 1059, 1020, 1040, 1049, 1026, 986, 1036, 1068, 998, 1006, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[999, 973, 981, 1039, 1038, 1060, 1046, 1011, 1045, 1029, 1087, 1000, 1014, 1031, 1004, 1021, 1037, 1012, 1063, 988, 1016, 1033, 1045, 1084, 1070, 966, 977, 1038, 962, 1031, 1064, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1031, 1048, 1027, 1020, 1016, 1010, 1039, 1003, 1048, 1028, 961, 1037, 1004, 1090, 1033, 988, 1039, 1022, 1092, 988, 976, 1017, 1003, 1024, 1042, 1058, 1007, 1041, 1043, 1006, 1000, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1064, 960, 980, 967, 1034, 1028, 1015, 1029, 998, 1015, 1031, 1047, 1018, 1023, 1031, 1009, 1053, 1063, 988, 1026, 1013, 1039, 1016, 1012, 1019, 1058, 1096, 1026, 997, 1058, 1009, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1014, 969, 1059, 1024, 1005, 1018, 967, 1026, 1061, 1051, 967, 1046, 1056, 1022, 997, 1044, 1027, 1044, 997, 1026, 1034, 1041, 1008, 994, 1054, 1045, 988, 974, 1080, 1010, 1049, 1071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1049, 1032, 1033, 1026, 1053, 1012, 1097, 992, 1035, 1064, 1013, 1070, 1044, 967, 1026, 975, 1026, 1007, 996, 970, 1079, 990, 1031, 1036, 1014, 1006, 1020, 1035, 1005, 1023, 1020, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1032, 1000, 1068, 1050, 1017, 1027, 998, 1021, 968, 991, 1026, 1012, 1010, 985, 1004, 1080, 1015, 1013, 1044, 1060, 980, 1034, 1056, 1028, 1048, 1062, 1057, 1008, 1011, 1058, 985, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1010, 1056, 981, 988, 1029, 1042, 1059, 1040, 1005, 1014, 1030, 1001, 1035, 1032, 987, 981, 1042, 1027, 1034, 1070, 970, 1054, 1052, 993, 972, 1052, 1030, 1059, 1043, 1006, 1021, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1031, 1048, 1027, 1020, 24, 0, 0, 0, 1016, 1010, 1039, 1003, 1, 0, 0, 0, 1048, 1028, 961, 1037, 7, 1, 0, 0, 1004, 1090, 1033, 988, 17, 5, 0, 0, 1039, 1022, 1063, 988, 0, 0, 0, 0, 976, 1010, 1003, 1024, 0, 0, 0, 0, 1042, 1040, 1007, 1041, 0, 0, 0, 0, 1043, 1006, 1000, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4069, 4082, 4137, 4112, 4013, 4130, 4075] avg=4096.0000 max=4150 max_rank=0 min=4013 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 995, 1019, 1064, 23, 0, 0, 0, 1002, 998, 995, 1026, 1, 0, 0, 0, 957, 1039, 1006, 1004, 7, 1, 0, 0, 1018, 1007, 1011, 1039, 17, 5, 0, 0, 1039, 1025, 1041, 1094, 0, 0, 0, 0, 1053, 1052, 1020, 1040, 0, 0, 0, 0, 1049, 1008, 986, 1036, 0, 0, 0, 0, 1068, 998, 1006, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4022, 4014, 4097, 4199, 4165, 4079, 4066] avg=4096.0000 max=4199 max_rank=4 min=4014 min_rank=2 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[999, 973, 981, 1039, 23, 0, 0, 0, 1038, 1060, 1046, 1011, 1, 0, 0, 0, 1045, 1029, 1087, 1000, 7, 1, 0, 0, 1014, 1031, 1004, 1021, 16, 5, 0, 0, 1037, 1012, 1035, 988, 0, 0, 0, 0, 1016, 1026, 1045, 1084, 0, 0, 0, 0, 1070, 949, 977, 1038, 0, 0, 0, 0, 962, 1031, 1064, 1003, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4156, 4169, 4091, 4072, 4171, 4034, 4060] avg=4096.0000 max=4171 max_rank=5 min=4015 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1064, 960, 980, 967, 22, 0, 0, 0, 1034, 1028, 1015, 1029, 1, 0, 0, 0, 998, 1015, 1031, 1047, 7, 1, 1, 0, 1018, 1023, 1031, 1009, 17, 5, 0, 0, 1053, 1063, 961, 1026, 0, 0, 0, 0, 1013, 1032, 1016, 1012, 0, 0, 0, 0, 1019, 1040, 1096, 1026, 0, 0, 0, 0, 997, 1058, 1009, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3993, 4107, 4100, 4103, 4103, 4073, 4181, 4108] avg=4096.0000 max=4181 max_rank=6 min=3993 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1049, 1032, 1033, 1026, 22, 0, 0, 0, 1053, 1012, 1097, 992, 1, 0, 0, 0, 1035, 1064, 1013, 1070, 7, 1, 0, 0, 1044, 967, 1026, 975, 16, 5, 0, 0, 1026, 1007, 969, 970, 0, 0, 0, 0, 1079, 983, 1031, 1036, 0, 0, 0, 0, 1014, 989, 1020, 1035, 0, 0, 0, 0, 1005, 1023, 1020, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4155, 4190, 4033, 3972, 4129, 4058, 4069] avg=4096.0000 max=4190 max_rank=2 min=3972 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16228 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1014, 969, 1059, 1024, 22, 0, 0, 0, 1005, 1018, 967, 1026, 1, 0, 0, 0, 1061, 1051, 967, 1046, 7, 1, 1, 0, 1056, 1022, 997, 1044, 17, 5, 0, 0, 1027, 1044, 970, 1026, 0, 0, 0, 0, 1034, 1034, 1008, 994, 0, 0, 0, 0, 1054, 1027, 988, 974, 0, 0, 0, 0, 1080, 1010, 1049, 1069, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4017, 4134, 4141, 4067, 4070, 4043, 4208] avg=4096.0000 max=4208 max_rank=7 min=4017 min_rank=1 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1032, 1000, 1068, 1050, 26, 0, 0, 0, 1017, 1027, 998, 1021, 8, 0, 0, 0, 968, 991, 1026, 1012, 13, 5, 5, 0, 1010, 985, 1004, 1080, 22, 8, 0, 0, 1015, 1013, 1010, 1060, 0, 0, 0, 0, 980, 1021, 1056, 1028, 0, 0, 0, 0, 1048, 1035, 1057, 1008, 0, 0, 0, 0, 1011, 1058, 985, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4071, 4020, 4109, 4098, 4085, 4148, 4061] avg=4096.0000 max=4176 max_rank=0 min=4020 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1010, 1056, 981, 988, 23, 0, 0, 0, 1029, 1042, 1059, 1040, 1, 0, 0, 0, 1005, 1014, 1030, 1001, 7, 1, 1, 0, 1035, 1032, 987, 981, 17, 5, 0, 0, 1042, 1027, 1006, 1070, 0, 0, 0, 0, 970, 1047, 1052, 993, 0, 0, 0, 0, 972, 1034, 1030, 1059, 0, 0, 0, 0, 1043, 1006, 1021, 1051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4171, 4059, 4057, 4145, 4062, 4095, 4121] avg=4096.0000 max=4171 max_rank=1 min=4057 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1037, 1024, 993, 1002, 1027, 1005, 1033, 1019, 1039, 1019, 1015, 1025, 1031, 1061, 1052, 1034, 1002, 1025, 1020, 1000, 1028, 971, 1007, 981, 1064, 1021, 1046, 1063, 1040, 1001, 1011, 1072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1002, 976, 1074, 990, 1009, 1060, 1018, 1045, 1051, 1005, 1012, 1030, 1053, 1039, 1034, 1077, 1065, 985, 1014, 1019, 1030, 1010, 987, 978, 1046, 1042, 1021, 984, 1003, 995, 1085, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1021, 1056, 1025, 1054, 984, 1022, 1028, 1042, 1022, 1024, 1024, 996, 1021, 1079, 983, 1025, 998, 989, 990, 1056, 1034, 1026, 1060, 1019, 1040, 1064, 1028, 1010, 1000, 1037, 1013, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1053, 994, 1021, 1061, 996, 1016, 1048, 1020, 1001, 1021, 1022, 1061, 1074, 966, 1076, 1054, 1033, 1038, 1015, 959, 1020, 996, 1009, 1037, 998, 1066, 1018, 1030, 1022, 1018, 1023, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1015, 996, 1024, 1070, 1043, 1019, 970, 1081, 1028, 1020, 1005, 980, 1015, 1043, 1022, 989, 1008, 1069, 1025, 1034, 1050, 1029, 1027, 1043, 1038, 1006, 1046, 1026, 993, 1018, 1029, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1051, 1013, 999, 1078, 1021, 1003, 1023, 972, 1008, 1004, 1038, 988, 1020, 1082, 1047, 1039, 1045, 1024, 1019, 1002, 990, 1019, 1006, 1015, 1025, 1044, 1036, 977, 1074, 1075, 990, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1005, 1007, 1051, 1015, 1023, 1055, 1002, 989, 1013, 1000, 1045, 981, 1020, 1039, 1016, 1046, 1014, 1011, 1081, 979, 1010, 1038, 1029, 995, 1037, 1038, 1013, 1011, 1062, 1023, 1053, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1010, 1029, 1018, 1029, 1057, 1029, 1093, 1008, 1019, 1010, 1029, 1049, 1036, 1000, 1009, 1016, 980, 1068, 1004, 1048, 1031, 1062, 1027, 1011, 1008, 1005, 1018, 1032, 1007, 1014, 997, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1037, 1024, 993, 1000, 0, 0, 0, 0, 1027, 1005, 1033, 1019, 0, 0, 0, 0, 1039, 1019, 1015, 1025, 17, 5, 0, 0, 1031, 1020, 1052, 1034, 0, 0, 0, 0, 1002, 1025, 1020, 1000, 10, 5, 2, 0, 1028, 971, 1007, 981, 24, 0, 0, 0, 1064, 1006, 1046, 1063, 0, 0, 0, 0, 1040, 1001, 1011, 1067, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4084, 4120, 4137, 4064, 4011, 4179, 4119] avg=4096.0000 max=4179 max_rank=6 min=4011 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1021, 1056, 1025, 1051, 0, 0, 0, 0, 984, 1022, 1028, 1042, 1, 0, 0, 0, 1022, 1024, 1024, 996, 17, 6, 0, 0, 1021, 1037, 983, 1025, 0, 0, 0, 0, 998, 989, 990, 1056, 11, 5, 2, 0, 1034, 1026, 1060, 1019, 25, 0, 0, 0, 1040, 1047, 1028, 1010, 0, 0, 0, 0, 1000, 1037, 1013, 993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4077, 4089, 4066, 4051, 4164, 4125, 4043] avg=4096.0000 max=4164 max_rank=5 min=4043 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1002, 976, 1074, 988, 0, 0, 0, 0, 1009, 1060, 1018, 1045, 0, 0, 0, 0, 1051, 1005, 1012, 1030, 17, 5, 0, 0, 1053, 998, 1034, 1077, 0, 0, 0, 0, 1065, 985, 1014, 1019, 10, 5, 2, 0, 1030, 1010, 987, 978, 24, 0, 0, 0, 1046, 1027, 1021, 984, 0, 0, 0, 0, 1003, 995, 1085, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4132, 4120, 4162, 4100, 4029, 4078, 4107] avg=4096.0000 max=4162 max_rank=3 min=4029 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1053, 994, 1021, 1058, 0, 0, 0, 0, 996, 1016, 1048, 1020, 1, 0, 0, 0, 1001, 1021, 1022, 1061, 15, 6, 0, 0, 1074, 929, 1076, 1054, 0, 0, 0, 0, 1033, 1038, 1015, 959, 11, 5, 2, 0, 1020, 996, 1009, 1037, 22, 0, 0, 0, 998, 1049, 1018, 1030, 0, 0, 0, 0, 1022, 1018, 1023, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4081, 4126, 4133, 4063, 4084, 4095, 4060] avg=4096.0000 max=4133 max_rank=3 min=4060 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1015, 996, 1024, 1067, 0, 0, 0, 0, 1043, 1019, 970, 1081, 1, 0, 0, 0, 1028, 1020, 1005, 980, 17, 5, 0, 0, 1015, 1002, 1022, 989, 0, 0, 0, 0, 1008, 1069, 1025, 1034, 10, 5, 2, 0, 1050, 1029, 1027, 1043, 24, 0, 0, 0, 1038, 991, 1046, 1026, 0, 0, 0, 0, 993, 1018, 1029, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4114, 4055, 4028, 4153, 4173, 4101, 4042] avg=4096.0000 max=4173 max_rank=5 min=4028 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1005, 1007, 1051, 1013, 0, 0, 0, 0, 1023, 1055, 1002, 989, 0, 0, 0, 0, 1013, 1000, 1045, 981, 17, 5, 0, 0, 1020, 998, 1016, 1046, 0, 0, 0, 0, 1014, 1011, 1081, 979, 10, 5, 2, 0, 1010, 1038, 1029, 995, 24, 0, 0, 0, 1037, 1023, 1013, 1011, 0, 0, 0, 0, 1062, 1023, 1053, 1062, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4069, 4061, 4080, 4102, 4096, 4084, 4200] avg=4096.0000 max=4200 max_rank=7 min=4061 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1051, 1013, 999, 1070, 0, 0, 0, 0, 1021, 1003, 1023, 972, 5, 0, 0, 0, 1008, 1004, 1038, 988, 21, 10, 0, 0, 1020, 1034, 1047, 1039, 0, 0, 0, 0, 1045, 1024, 1019, 1002, 14, 11, 3, 0, 990, 1019, 1006, 1015, 27, 0, 0, 0, 1025, 1020, 1036, 977, 0, 0, 0, 0, 1074, 1075, 990, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4024, 4069, 4140, 4118, 4057, 4058, 4169] avg=4096.0000 max=4169 max_rank=7 min=4024 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1010, 1029, 1018, 1027, 0, 0, 0, 0, 1057, 1029, 1093, 1008, 0, 0, 0, 0, 1019, 1010, 1029, 1049, 16, 5, 0, 0, 1036, 961, 1009, 1016, 0, 0, 0, 0, 980, 1068, 1004, 1048, 10, 5, 2, 0, 1031, 1062, 1027, 1011, 23, 0, 0, 0, 1008, 990, 1018, 1032, 0, 0, 0, 0, 1007, 1014, 997, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4187, 4128, 4022, 4117, 4154, 4048, 4028] avg=4096.0000 max=4187 max_rank=1 min=4022 min_rank=3 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1032, 996, 1037, 1021, 1010, 1027, 1081, 1000, 1008, 1055, 1001, 1010, 988, 1038, 988, 1034, 1018, 999, 1009, 1038, 1021, 1006, 1033, 1069, 1045, 966, 1056, 985, 1046, 1032, 1022, 1097] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1020, 1011, 1058, 1024, 991, 1037, 1055, 1031, 1019, 1038, 1022, 966, 984, 1078, 1023, 992, 1038, 1024, 1032, 1067, 1044, 1022, 1028, 957, 1031, 1039, 1021, 1026, 1025, 1046, 995, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1029, 999, 1014, 1029, 1014, 1063, 980, 1072, 1022, 1056, 1032, 1049, 983, 1019, 1029, 1015, 1057, 1023, 1057, 1028, 1002, 1007, 1000, 1041, 988, 1038, 989, 1023, 1004, 1056, 998, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1023, 1054, 1008, 999, 1028, 1074, 1016, 980, 973, 1058, 971, 1048, 1029, 1050, 1099, 1012, 1030, 1027, 978, 1005, 992, 990, 1038, 1071, 1000, 1009, 1028, 1044, 1006, 1019, 1033, 1076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1049, 1034, 1001, 1004, 1047, 1077, 997, 1076, 973, 1025, 997, 1035, 1014, 997, 1047, 1028, 1014, 1054, 992, 979, 1032, 1016, 1026, 1044, 1011, 1017, 1020, 1021, 1074, 1015, 1041, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1001, 995, 978, 1015, 1045, 1018, 987, 1051, 1004, 989, 1039, 1003, 1021, 1063, 1072, 1010, 1031, 1048, 1022, 1039, 1047, 1052, 1037, 1055, 1034, 1024, 1001, 1033, 1008, 977, 1040, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1033, 1033, 989, 978, 1014, 1052, 1028, 995, 1041, 1016, 1042, 1033, 981, 1034, 995, 1045, 1051, 1031, 1035, 1022, 1023, 1018, 1057, 996, 1055, 1039, 985, 1025, 1022, 1039, 1016, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1034, 1025, 1041, 1018, 997, 1039, 1082, 1054, 1029, 1052, 996, 1035, 986, 1027, 1024, 1020, 1037, 1015, 1011, 1018, 1016, 1013, 1053, 981, 1074, 1013, 994, 1033, 1040, 1005, 1000, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1029, 999, 1014, 1029, 23, 0, 0, 0, 1014, 1032, 980, 1072, 0, 0, 0, 0, 1022, 1056, 1032, 1049, 8, 8, 0, 0, 983, 1019, 1029, 1015, 3, 2, 0, 0, 1050, 1023, 1057, 1028, 0, 0, 0, 0, 1002, 1007, 998, 1041, 0, 0, 0, 0, 988, 1038, 989, 1023, 8, 4, 0, 0, 1004, 1056, 998, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4098, 4175, 4051, 4158, 4048, 4050, 4094] avg=4096.0000 max=4175 max_rank=2 min=4048 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1020, 1011, 1058, 1024, 22, 0, 0, 0, 991, 1008, 1055, 1031, 0, 0, 0, 0, 1019, 1038, 1022, 966, 8, 7, 0, 0, 984, 1078, 1023, 992, 3, 2, 0, 0, 1031, 1024, 1032, 1067, 0, 0, 0, 0, 1044, 1022, 1026, 957, 0, 0, 0, 0, 1031, 1039, 1021, 1026, 7, 4, 0, 0, 1025, 1046, 995, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4085, 4060, 4082, 4154, 4049, 4128, 4075] avg=4096.0000 max=4154 max_rank=4 min=4049 min_rank=5 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1032, 996, 1037, 1021, 22, 0, 0, 0, 1010, 998, 1081, 1000, 0, 0, 0, 0, 1008, 1055, 1001, 1010, 8, 7, 0, 0, 988, 1038, 988, 1034, 2, 2, 0, 0, 1012, 999, 1009, 1038, 0, 0, 0, 0, 1021, 1006, 1031, 1069, 0, 0, 0, 0, 1045, 966, 1056, 985, 8, 4, 0, 0, 1046, 1032, 1022, 1081, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4089, 4089, 4052, 4058, 4127, 4064, 4181] avg=4096.0000 max=4181 max_rank=7 min=4052 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1049, 1034, 1001, 1004, 23, 0, 0, 0, 1047, 1046, 997, 1076, 0, 0, 0, 0, 973, 1025, 997, 1035, 8, 8, 0, 0, 1014, 997, 1047, 1028, 2, 2, 0, 0, 1008, 1054, 992, 979, 0, 0, 0, 0, 1032, 1016, 1024, 1044, 0, 0, 0, 0, 1011, 1017, 1020, 1021, 7, 4, 0, 0, 1074, 1015, 1041, 996, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4166, 4046, 4090, 4033, 4116, 4080, 4126] avg=4096.0000 max=4166 max_rank=1 min=4033 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1023, 1054, 1008, 999, 23, 0, 0, 0, 1028, 1043, 1016, 980, 0, 0, 0, 0, 973, 1058, 971, 1048, 8, 8, 0, 0, 1029, 1050, 1099, 1012, 2, 2, 0, 0, 1024, 1027, 978, 1005, 0, 0, 0, 0, 992, 990, 1036, 1071, 0, 0, 0, 0, 1000, 1009, 1028, 1044, 8, 4, 0, 0, 1006, 1019, 1033, 1060, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4067, 4066, 4194, 4034, 4089, 4093, 4118] avg=4096.0000 max=4194 max_rank=3 min=4034 min_rank=4 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1033, 1033, 989, 978, 23, 0, 0, 0, 1014, 1021, 1028, 995, 0, 0, 0, 0, 1041, 1016, 1042, 1033, 8, 8, 0, 0, 981, 1034, 995, 1045, 3, 2, 0, 0, 1044, 1031, 1035, 1022, 0, 0, 0, 0, 1023, 1018, 1055, 996, 0, 0, 0, 0, 1055, 1039, 985, 1025, 8, 4, 0, 0, 1022, 1039, 1016, 1029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4058, 4148, 4060, 4132, 4092, 4116, 4106] avg=4096.0000 max=4148 max_rank=2 min=4056 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1001, 995, 978, 1015, 27, 0, 0, 0, 1045, 980, 987, 1051, 0, 0, 0, 0, 1004, 989, 1039, 1003, 11, 11, 0, 0, 1021, 1063, 1072, 1010, 6, 5, 0, 0, 1016, 1048, 1022, 1039, 0, 0, 0, 0, 1047, 1052, 1032, 1055, 0, 0, 0, 0, 1034, 1024, 1001, 1033, 11, 9, 0, 0, 1008, 977, 1040, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4063, 4057, 4177, 4125, 4186, 4112, 4032] avg=4096.0000 max=4186 max_rank=5 min=4016 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1034, 1025, 1041, 1018, 23, 0, 0, 0, 997, 1009, 1082, 1054, 0, 0, 0, 0, 1029, 1052, 996, 1035, 8, 7, 0, 0, 986, 1027, 1024, 1020, 3, 2, 0, 0, 1030, 1015, 1011, 1018, 0, 0, 0, 0, 1016, 1013, 1051, 981, 0, 0, 0, 0, 1074, 1013, 994, 1033, 7, 4, 0, 0, 1040, 1005, 1000, 991, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4142, 4127, 4062, 4074, 4061, 4125, 4036] avg=4096.0000 max=4142 max_rank=1 min=4036 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[966, 1015, 1016, 1012, 1005, 1035, 1060, 1028, 1053, 1050, 977, 1031, 1031, 983, 1031, 1001, 1008, 1038, 1010, 1022, 986, 1017, 996, 1022, 1038, 1056, 1034, 1045, 1065, 1065, 1024, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1005, 1013, 1022, 1065, 1002, 982, 1006, 1039, 1027, 1061, 1024, 1043, 1009, 1024, 996, 1008, 993, 1005, 1045, 997, 1029, 983, 1055, 1046, 1033, 1047, 1085, 1000, 1028, 984, 1027, 1085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1067, 1039, 1028, 1005, 1026, 1011, 1001, 1075, 1032, 992, 1040, 1016, 1055, 1024, 1020, 1067, 987, 1004, 995, 1042, 985, 1023, 1031, 1011, 1067, 973, 1060, 1007, 1020, 1032, 991, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1045, 1039, 1044, 1034, 1024, 1063, 1074, 1062, 1018, 1043, 975, 955, 1011, 964, 1044, 1005, 1031, 1023, 1031, 1058, 1046, 1009, 981, 1003, 1031, 1002, 1047, 990, 1044, 1062, 1003, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1030, 998, 997, 1022, 1014, 1020, 1030, 990, 999, 1021, 1014, 1039, 1055, 1021, 1036, 1048, 1007, 1086, 1058, 1006, 982, 1022, 1022, 1014, 1007, 1006, 1069, 1006, 1038, 1084, 1003, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[975, 1009, 1003, 1030, 1043, 1001, 1021, 973, 1019, 1054, 1078, 1038, 1033, 990, 1022, 1028, 999, 1073, 1018, 1019, 1053, 993, 1032, 1069, 976, 1024, 1040, 1026, 985, 1031, 1032, 1081] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1005, 1014, 950, 1038, 1054, 1027, 1024, 1017, 1050, 1056, 1017, 1033, 1028, 1006, 1067, 1010, 983, 1010, 1074, 970, 1052, 1040, 1047, 1048, 1047, 1058, 998, 1000, 1032, 959, 1012, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1008, 1029, 981, 1065, 1038, 1044, 988, 1069, 1035, 1047, 1031, 1026, 1034, 1052, 1002, 1040, 1008, 1014, 1020, 1007, 1017, 1038, 1026, 1007, 1018, 1049, 984, 1054, 987, 980, 1003, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[966, 1015, 1016, 1012, 15, 9, 0, 0, 1005, 1035, 1060, 1020, 0, 0, 0, 0, 1053, 1035, 977, 1031, 0, 0, 0, 0, 1031, 983, 1031, 1001, 2, 0, 0, 0, 1008, 1038, 1010, 1022, 10, 5, 0, 0, 986, 1017, 996, 1022, 6, 3, 0, 0, 1038, 1056, 1021, 1045, 0, 0, 0, 0, 1065, 1065, 1024, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4120, 4096, 4048, 4093, 4030, 4160, 4188] avg=4096.0000 max=4188 max_rank=7 min=4030 min_rank=5 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1067, 1039, 1028, 1005, 15, 9, 0, 0, 1026, 1011, 1001, 1066, 0, 0, 0, 0, 1032, 977, 1040, 1016, 0, 0, 0, 0, 1055, 1024, 1020, 1067, 2, 0, 0, 0, 987, 1004, 995, 1042, 10, 5, 0, 0, 985, 1023, 1031, 1011, 7, 3, 0, 0, 1067, 973, 1047, 1007, 0, 0, 0, 0, 1020, 1032, 991, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4104, 4065, 4168, 4043, 4060, 4094, 4071] avg=4096.0000 max=4168 max_rank=3 min=4043 min_rank=4 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16268 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1005, 1014, 950, 1038, 15, 9, 0, 0, 1054, 1027, 1024, 1009, 0, 0, 0, 0, 1050, 1041, 1017, 1033, 0, 0, 0, 0, 1028, 1006, 1067, 1010, 2, 0, 0, 0, 983, 1010, 1074, 970, 9, 5, 0, 0, 1052, 1040, 1047, 1048, 6, 3, 0, 0, 1047, 1058, 986, 1000, 0, 0, 0, 0, 1032, 959, 1012, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4114, 4141, 4113, 4051, 4196, 4091, 4031] avg=4096.0000 max=4196 max_rank=5 min=4031 min_rank=0 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1005, 1013, 1022, 1065, 16, 9, 0, 0, 1002, 982, 1006, 1031, 0, 0, 0, 0, 1027, 1045, 1024, 1043, 0, 0, 0, 0, 1009, 1024, 996, 1008, 2, 0, 0, 0, 993, 1005, 1045, 997, 10, 5, 0, 0, 1029, 983, 1055, 1046, 6, 3, 0, 0, 1033, 1047, 1072, 1000, 0, 0, 0, 0, 1028, 984, 1027, 1071, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4021, 4139, 4039, 4055, 4122, 4152, 4110] avg=4096.0000 max=4152 max_rank=6 min=4021 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1030, 998, 997, 1022, 15, 8, 0, 0, 1014, 1020, 1030, 982, 0, 0, 0, 0, 999, 1006, 1014, 1039, 0, 0, 0, 0, 1055, 1021, 1036, 1048, 2, 0, 0, 0, 1007, 1086, 1058, 1006, 10, 5, 0, 0, 982, 1022, 1022, 1014, 6, 3, 0, 0, 1007, 1006, 1056, 1006, 0, 0, 0, 0, 1038, 1084, 1003, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4046, 4058, 4162, 4172, 4049, 4075, 4136] avg=4096.0000 max=4172 max_rank=4 min=4046 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1045, 1039, 1044, 1034, 20, 11, 0, 0, 1024, 1063, 1074, 1042, 0, 0, 0, 0, 1018, 1023, 975, 955, 0, 0, 0, 0, 1011, 964, 1044, 1005, 9, 0, 0, 0, 1031, 1023, 1031, 1058, 13, 11, 0, 0, 1046, 1009, 981, 1003, 11, 7, 0, 0, 1031, 1002, 1027, 990, 0, 0, 0, 0, 1044, 1062, 1003, 985, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4193, 4203, 3971, 4033, 4167, 4057, 4050, 4094] avg=4096.0000 max=4203 max_rank=1 min=3971 min_rank=2 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[975, 1009, 1003, 1030, 15, 9, 0, 0, 1043, 1001, 1021, 965, 0, 0, 0, 0, 1019, 1039, 1078, 1038, 0, 0, 0, 0, 1033, 990, 1022, 1028, 2, 0, 0, 0, 999, 1073, 1018, 1019, 10, 5, 0, 0, 1053, 993, 1032, 1069, 6, 3, 0, 0, 976, 1024, 1027, 1026, 0, 0, 0, 0, 985, 1031, 1032, 1067, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4030, 4174, 4075, 4124, 4156, 4053, 4115] avg=4096.0000 max=4174 max_rank=2 min=4030 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1008, 1029, 981, 1065, 15, 9, 0, 0, 1038, 1044, 988, 1060, 0, 0, 0, 0, 1035, 1032, 1031, 1026, 0, 0, 0, 0, 1034, 1052, 1002, 1040, 2, 0, 0, 0, 1008, 1014, 1020, 1007, 9, 5, 0, 0, 1017, 1038, 1026, 1007, 7, 3, 0, 0, 1018, 1049, 972, 1054, 0, 0, 0, 0, 987, 980, 1003, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4130, 4124, 4130, 4063, 4098, 4093, 4023] avg=4096.0000 max=4130 max_rank=1 min=4023 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1073, 1035, 1009, 1005, 984, 1043, 993, 990, 1031, 1021, 1031, 1004, 1045, 977, 1062, 1031, 1017, 1002, 1016, 1085, 1026, 1049, 1056, 1052, 1000, 1014, 998, 1044, 1018, 1067, 980, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1007, 1069, 1007, 1014, 998, 1008, 1033, 1006, 1012, 1089, 1085, 983, 1025, 989, 997, 1046, 1046, 1027, 1039, 1038, 1073, 980, 997, 985, 1046, 1062, 1016, 1059, 975, 977, 1032, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[982, 1094, 1041, 992, 1010, 989, 1007, 987, 1037, 1075, 1072, 1060, 1025, 1003, 1024, 1018, 1038, 1009, 985, 982, 1041, 996, 1008, 1019, 1042, 1077, 1035, 999, 1034, 1027, 1007, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1021, 1050, 1038, 1006, 1008, 1081, 1045, 990, 995, 1031, 1046, 993, 1000, 1033, 999, 997, 1010, 1016, 1026, 1036, 1022, 1033, 1051, 996, 1047, 1026, 1029, 1058, 1013, 1009, 1025, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1008, 976, 1008, 1064, 1011, 1018, 985, 1036, 1087, 994, 1028, 1032, 1000, 996, 1048, 1024, 1013, 1009, 1035, 997, 1045, 1038, 1032, 1035, 997, 1071, 1046, 1020, 1031, 1034, 1025, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1081, 1001, 1003, 1044, 1018, 1033, 1000, 1040, 1014, 1015, 1031, 1028, 1030, 1008, 977, 1050, 938, 1010, 1029, 1057, 1016, 981, 1003, 1052, 1037, 1088, 1031, 1052, 1043, 984, 1050, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1038, 1012, 1043, 984, 990, 1028, 1043, 1012, 1062, 1001, 971, 1040, 1037, 1066, 1029, 1064, 1033, 1034, 1032, 993, 1033, 974, 1036, 1031, 996, 1011, 1027, 986, 1011, 1026, 1062, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1045, 1030, 1034, 1040, 979, 1040, 1013, 978, 965, 1035, 1040, 988, 1041, 1014, 1038, 988, 1010, 999, 1076, 1063, 1035, 1026, 1078, 1002, 1032, 1056, 1058, 995, 1012, 1004, 1053, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1073, 1025, 1009, 1005, 0, 0, 0, 0, 984, 1043, 993, 990, 34, 10, 0, 0, 1031, 1021, 1016, 1004, 0, 0, 0, 0, 1045, 977, 1062, 1031, 5, 5, 0, 0, 1017, 1002, 1016, 1085, 5, 3, 0, 0, 1023, 1049, 1056, 1052, 0, 0, 0, 0, 1000, 980, 998, 1044, 0, 0, 0, 0, 1018, 1067, 980, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4054, 4072, 4125, 4128, 4180, 4022, 4075] avg=4096.0000 max=4180 max_rank=5 min=4022 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1007, 1059, 1007, 1014, 0, 0, 0, 0, 998, 1008, 1033, 1006, 36, 11, 0, 0, 1012, 1089, 1069, 983, 0, 0, 0, 0, 1025, 989, 997, 1046, 5, 5, 0, 0, 1046, 1027, 1039, 1038, 5, 3, 0, 0, 1070, 980, 997, 985, 0, 0, 0, 0, 1046, 1026, 1016, 1059, 0, 0, 0, 0, 975, 977, 1032, 1048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4092, 4153, 4067, 4158, 4032, 4147, 4032] avg=4096.0000 max=4158 max_rank=4 min=4032 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1008, 967, 1008, 1064, 0, 0, 0, 0, 1011, 1018, 985, 1036, 36, 10, 0, 0, 1087, 994, 1013, 1032, 0, 0, 0, 0, 1000, 996, 1048, 1024, 5, 5, 0, 0, 1013, 1009, 1035, 997, 4, 3, 0, 0, 1042, 1038, 1032, 1035, 0, 0, 0, 0, 997, 1035, 1046, 1020, 0, 0, 0, 0, 1031, 1034, 1025, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4096, 4126, 4078, 4061, 4147, 4098, 4115] avg=4096.0000 max=4147 max_rank=5 min=4047 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[982, 1084, 1041, 992, 0, 0, 0, 0, 1010, 989, 1007, 987, 36, 10, 0, 0, 1037, 1075, 1057, 1060, 0, 0, 0, 0, 1025, 1003, 1024, 1018, 5, 5, 0, 0, 1038, 1009, 985, 982, 5, 3, 0, 0, 1038, 996, 1008, 1019, 0, 0, 0, 0, 1042, 1041, 1035, 999, 0, 0, 0, 0, 1034, 1027, 1007, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4039, 4229, 4080, 4022, 4061, 4117, 4121] avg=4096.0000 max=4229 max_rank=2 min=4022 min_rank=4 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16321 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1021, 1040, 1038, 1006, 0, 0, 0, 0, 1008, 1081, 1045, 990, 35, 10, 0, 0, 995, 1031, 1031, 993, 0, 0, 0, 0, 1000, 1033, 999, 997, 5, 5, 0, 0, 1010, 1016, 1026, 1036, 5, 3, 0, 0, 1019, 1033, 1051, 996, 0, 0, 0, 0, 1047, 991, 1029, 1058, 0, 0, 0, 0, 1013, 1009, 1025, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4169, 4050, 4039, 4096, 4099, 4125, 4085] avg=4096.0000 max=4169 max_rank=1 min=4039 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1081, 984, 1003, 1044, 0, 0, 0, 0, 1018, 1033, 1000, 1040, 40, 15, 0, 0, 1014, 1015, 1008, 1028, 0, 0, 0, 0, 1030, 1008, 977, 1050, 9, 8, 0, 0, 938, 1010, 1029, 1057, 8, 5, 0, 0, 1004, 981, 1003, 1052, 0, 0, 0, 0, 1037, 1048, 1031, 1052, 0, 0, 0, 0, 1043, 984, 1050, 1024, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4146, 4065, 4082, 4047, 4040, 4168, 4108] avg=4096.0000 max=4168 max_rank=6 min=4040 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1038, 1002, 1043, 984, 0, 0, 0, 0, 990, 1028, 1043, 1012, 34, 9, 0, 0, 1062, 1001, 957, 1040, 0, 0, 0, 0, 1037, 1066, 1029, 1064, 5, 5, 0, 0, 1033, 1034, 1032, 993, 5, 3, 0, 0, 1030, 974, 1036, 1031, 0, 0, 0, 0, 996, 977, 1027, 986, 0, 0, 0, 0, 1011, 1026, 1062, 1063, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4116, 4060, 4206, 4100, 4071, 3986, 4162] avg=4096.0000 max=4206 max_rank=3 min=3986 min_rank=6 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1045, 1020, 1034, 1040, 0, 0, 0, 0, 979, 1040, 1013, 978, 36, 10, 0, 0, 965, 1035, 1025, 988, 0, 0, 0, 0, 1041, 1014, 1038, 988, 5, 5, 0, 0, 1010, 999, 1076, 1063, 5, 3, 0, 0, 1032, 1026, 1078, 1002, 0, 0, 0, 0, 1032, 1020, 1058, 995, 0, 0, 0, 0, 1012, 1004, 1053, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4056, 4013, 4091, 4156, 4138, 4105, 4070] avg=4096.0000 max=4156 max_rank=4 min=4013 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8064, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1036, 1022, 1013, 982, 1078, 1038, 1016, 1005, 1020, 1025, 1027, 1072, 1040, 992, 982, 1049, 1023, 996, 996, 1032, 977, 1066, 1037, 1035, 1022, 1043, 1054, 967, 1046, 1011, 1040, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[983, 1011, 987, 1038, 1022, 1085, 1056, 1002, 1027, 1010, 1042, 1034, 1061, 1068, 1068, 1027, 956, 961, 990, 1003, 1024, 1041, 1093, 1017, 1064, 1030, 1004, 997, 1057, 1038, 1020, 952] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[948, 1040, 1015, 1024, 1039, 1057, 1051, 1028, 967, 1023, 1066, 1072, 1060, 990, 1049, 977, 1031, 1034, 1050, 1010, 990, 1004, 1018, 1066, 1061, 1018, 1045, 999, 1032, 992, 1008, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1025, 1065, 997, 1021, 973, 1035, 1013, 1014, 1046, 1009, 1015, 1059, 1053, 998, 1042, 1046, 1023, 1061, 1000, 959, 1023, 1039, 1036, 1046, 1063, 978, 1016, 978, 1046, 975, 1051, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1009, 1033, 1011, 1003, 979, 1002, 1032, 1047, 1005, 1032, 1036, 1023, 999, 996, 1032, 1042, 1010, 1046, 1045, 1047, 1062, 1006, 1036, 1002, 1050, 987, 1043, 1002, 1030, 1045, 1041, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[986, 1021, 1043, 1059, 1044, 1033, 1023, 1033, 1003, 1046, 1047, 1061, 1017, 1074, 1010, 947, 1005, 974, 1027, 1011, 1023, 1044, 990, 1051, 1080, 1016, 1066, 1026, 971, 1010, 1009, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[999, 1042, 1028, 977, 1016, 968, 1032, 999, 1007, 1023, 1033, 1067, 1039, 1047, 1001, 1053, 1025, 979, 1025, 985, 1054, 1019, 980, 1096, 1059, 1062, 1027, 1062, 1058, 1005, 990, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1009, 1007, 1018, 1078, 1009, 1080, 989, 982, 1021, 991, 1074, 1014, 1007, 1048, 989, 1036, 1027, 992, 1021, 1020, 1053, 1027, 1046, 978, 1026, 1008, 1070, 1056, 1013, 1027, 1018, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1036, 1022, 1013, 982, 25, 4, 0, 0, 1078, 1037, 1016, 1005, 0, 0, 0, 0, 1020, 1025, 1027, 1043, 0, 0, 0, 0, 1032, 992, 982, 1049, 0, 0, 0, 0, 1023, 996, 996, 1032, 29, 21, 0, 0, 977, 1066, 1037, 1009, 0, 0, 0, 0, 997, 1043, 1054, 967, 0, 0, 0, 0, 1046, 1011, 1040, 1026, 8, 1, 1, 0] rerouted_tokens_per_ep_rank=[4082, 4136, 4115, 4055, 4097, 4089, 4061, 4133] avg=4096.0000 max=4136 max_rank=1 min=4055 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[983, 1011, 987, 1038, 24, 4, 0, 0, 1022, 1084, 1056, 1002, 0, 0, 0, 0, 1027, 1010, 1042, 1006, 0, 0, 0, 0, 1052, 1068, 1068, 1027, 0, 0, 0, 0, 956, 961, 990, 1003, 28, 22, 0, 0, 1024, 1041, 1093, 992, 0, 0, 0, 0, 1038, 1030, 1004, 997, 0, 0, 0, 0, 1057, 1038, 1020, 952, 9, 1, 1, 0] rerouted_tokens_per_ep_rank=[4047, 4164, 4085, 4215, 3960, 4150, 4069, 4078] avg=4096.0000 max=4215 max_rank=3 min=3960 min_rank=4 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[948, 1040, 1015, 1024, 25, 4, 0, 0, 1039, 1056, 1051, 1028, 0, 0, 0, 0, 967, 1023, 1066, 1043, 0, 0, 0, 0, 1051, 990, 1049, 977, 0, 0, 0, 0, 1031, 1034, 1050, 1010, 29, 22, 0, 0, 990, 1004, 1018, 1040, 0, 0, 0, 0, 1035, 1018, 1045, 999, 0, 0, 0, 0, 1032, 992, 1008, 1004, 9, 1, 1, 0] rerouted_tokens_per_ep_rank=[4056, 4174, 4099, 4067, 4176, 4052, 4097, 4047] avg=4096.0000 max=4176 max_rank=4 min=4047 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 1065, 997, 1021, 25, 4, 0, 0, 973, 1034, 1013, 1014, 0, 0, 0, 0, 1046, 1009, 1015, 1031, 0, 0, 0, 0, 1044, 998, 1042, 1046, 0, 0, 0, 0, 1023, 1061, 1000, 959, 28, 22, 0, 0, 1023, 1039, 1036, 1020, 0, 0, 0, 0, 1037, 978, 1016, 978, 0, 0, 0, 0, 1046, 975, 1051, 1063, 9, 1, 1, 0] rerouted_tokens_per_ep_rank=[4137, 4034, 4101, 4130, 4093, 4118, 4009, 4146] avg=4096.0000 max=4146 max_rank=7 min=4009 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1009, 1033, 1011, 1003, 24, 4, 0, 0, 979, 1001, 1032, 1047, 0, 0, 0, 0, 1005, 1032, 1036, 996, 0, 0, 0, 0, 991, 996, 1032, 1042, 0, 0, 0, 0, 1010, 1046, 1045, 1047, 27, 21, 0, 0, 1062, 1006, 1036, 977, 0, 0, 0, 0, 1025, 987, 1043, 1002, 0, 0, 0, 0, 1030, 1045, 1041, 1035, 8, 1, 1, 0] rerouted_tokens_per_ep_rank=[4084, 4059, 4069, 4061, 4196, 4081, 4057, 4161] avg=4096.0000 max=4196 max_rank=4 min=4057 min_rank=6 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[986, 1021, 1043, 1059, 25, 4, 0, 0, 1044, 1032, 1023, 1033, 0, 0, 0, 0, 1003, 1046, 1047, 1033, 0, 0, 0, 0, 1009, 1074, 1010, 947, 0, 0, 0, 0, 1005, 974, 1027, 1011, 28, 22, 0, 0, 1023, 1044, 990, 1025, 0, 0, 0, 0, 1054, 1016, 1066, 1026, 0, 0, 0, 0, 971, 1010, 1009, 1018, 8, 1, 1, 0] rerouted_tokens_per_ep_rank=[4138, 4132, 4129, 4040, 4067, 4082, 4162, 4018] avg=4096.0000 max=4162 max_rank=6 min=4018 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[999, 1042, 1028, 977, 31, 8, 0, 0, 1016, 963, 1032, 999, 0, 0, 0, 0, 1007, 1023, 1033, 1034, 0, 0, 0, 0, 1027, 1047, 1001, 1053, 0, 0, 0, 0, 1025, 979, 1025, 985, 33, 24, 0, 0, 1054, 1019, 980, 1063, 0, 0, 0, 0, 1027, 1062, 1027, 1062, 0, 0, 0, 0, 1058, 1005, 990, 1011, 12, 5, 2, 0] rerouted_tokens_per_ep_rank=[4085, 4010, 4097, 4128, 4071, 4116, 4178, 4083] avg=4096.0000 max=4178 max_rank=6 min=4010 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1009, 1007, 1018, 1078, 23, 4, 0, 0, 1009, 1079, 989, 982, 0, 0, 0, 0, 1021, 991, 1074, 987, 0, 0, 0, 0, 999, 1048, 989, 1036, 0, 0, 0, 0, 1027, 992, 1021, 1020, 27, 21, 0, 0, 1053, 1027, 1046, 954, 0, 0, 0, 0, 1001, 1008, 1070, 1056, 0, 0, 0, 0, 1013, 1027, 1018, 1034, 8, 1, 1, 0] rerouted_tokens_per_ep_rank=[4139, 4059, 4073, 4072, 4108, 4080, 4135, 4102] avg=4096.0000 max=4139 max_rank=0 min=4059 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8448, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8448, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8192, 256, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1035, 994, 1009, 986, 1001, 1060, 1045, 988, 1009, 1015, 994, 1041, 1001, 1024, 1025, 1050, 1045, 980, 1057, 986, 1052, 1046, 1026, 1035, 1001, 1101, 1018, 965, 1035, 1073, 1040, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1021, 1060, 1048, 1045, 983, 1047, 1016, 1042, 1039, 977, 1047, 986, 999, 1029, 1010, 1023, 1007, 1016, 1012, 1032, 1016, 1050, 1012, 1041, 1011, 996, 1038, 993, 1027, 1058, 1024, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1043, 1014, 1027, 1028, 1036, 1026, 996, 1020, 1037, 1013, 987, 1023, 1006, 1010, 1041, 1015, 1033, 1034, 1056, 1019, 1031, 1029, 1038, 1015, 1012, 1047, 1036, 992, 1025, 1026, 1016, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1026, 990, 997, 1061, 1070, 1029, 1052, 980, 1011, 1018, 1003, 1032, 1020, 972, 1042, 1052, 1008, 1027, 1020, 1063, 1100, 1006, 999, 1045, 1039, 1081, 978, 1021, 1024, 1002, 1016, 984] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[966, 1038, 1028, 1027, 1064, 1021, 1028, 1048, 1022, 1003, 1037, 1007, 1053, 1035, 1018, 1016, 1051, 1033, 1025, 1034, 977, 991, 1045, 1000, 1040, 1027, 1034, 1015, 1011, 998, 1041, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1084, 1031, 1019, 1036, 1040, 1027, 1043, 1064, 1017, 994, 984, 1048, 1046, 1011, 1040, 1019, 970, 982, 1037, 1038, 1033, 1035, 1048, 1053, 1013, 1034, 1056, 974, 996, 996, 994, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1011, 1002, 1051, 1036, 1025, 1026, 1025, 1056, 1019, 988, 1063, 1050, 1078, 1007, 1019, 1076, 1016, 1008, 1031, 1081, 1027, 1020, 1004, 986, 1035, 1026, 1023, 1004, 993, 971, 1006, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1013, 1003, 1042, 1042, 1030, 1048, 1029, 1041, 1040, 1007, 1040, 1005, 1046, 992, 991, 1035, 1019, 970, 1059, 1041, 1042, 994, 991, 990, 1032, 1040, 1069, 1042, 1026, 1022, 1058, 969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1035, 994, 1009, 981, 0, 0, 0, 0, 1001, 1030, 1045, 988, 0, 0, 0, 0, 1009, 1015, 994, 1041, 27, 0, 0, 0, 1001, 1024, 1025, 1046, 0, 0, 0, 0, 1045, 980, 1057, 986, 0, 0, 0, 0, 1052, 1046, 1026, 1035, 0, 0, 0, 0, 1001, 1098, 1018, 965, 0, 0, 0, 0, 1035, 1073, 1040, 1031, 5, 4, 3, 3] rerouted_tokens_per_ep_rank=[4019, 4064, 4086, 4096, 4068, 4159, 4082, 4194] avg=4096.0000 max=4194 max_rank=7 min=4019 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16503 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1021, 1060, 1048, 1040, 0, 0, 0, 0, 983, 1018, 1016, 1042, 0, 0, 0, 0, 1039, 977, 1047, 986, 26, 0, 0, 0, 999, 1029, 1010, 1019, 0, 0, 0, 0, 1007, 1016, 1012, 1032, 0, 0, 0, 0, 1016, 1050, 1012, 1041, 0, 0, 0, 0, 1011, 994, 1038, 993, 0, 0, 0, 0, 1027, 1058, 1024, 1063, 5, 4, 3, 2] rerouted_tokens_per_ep_rank=[4169, 4059, 4075, 4057, 4067, 4119, 4036, 4186] avg=4096.0000 max=4186 max_rank=7 min=4036 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16408 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1043, 1014, 1027, 1023, 0, 0, 0, 0, 1036, 997, 996, 1020, 0, 0, 0, 0, 1037, 1013, 987, 1023, 26, 0, 0, 0, 1006, 1010, 1041, 1011, 0, 0, 0, 0, 1033, 1034, 1056, 1019, 0, 0, 0, 0, 1031, 1029, 1038, 1015, 0, 0, 0, 0, 1012, 1044, 1036, 992, 0, 0, 0, 0, 1025, 1026, 1016, 1037, 5, 4, 3, 3] rerouted_tokens_per_ep_rank=[4107, 4049, 4086, 4068, 4142, 4113, 4084, 4119] avg=4096.0000 max=4142 max_rank=4 min=4049 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16458 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1026, 990, 997, 1056, 0, 0, 0, 0, 1070, 1000, 1052, 980, 0, 0, 0, 0, 1011, 1018, 1003, 1032, 26, 0, 0, 0, 1020, 972, 1042, 1048, 0, 0, 0, 0, 1008, 1027, 1020, 1063, 0, 0, 0, 0, 1100, 1006, 999, 1045, 0, 0, 0, 0, 1039, 1078, 978, 1021, 0, 0, 0, 0, 1024, 1002, 1016, 984, 5, 4, 3, 3] rerouted_tokens_per_ep_rank=[4069, 4102, 4090, 4082, 4118, 4150, 4116, 4041] avg=4096.0000 max=4150 max_rank=5 min=4041 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16425 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1084, 1031, 1019, 1031, 0, 0, 0, 0, 1040, 998, 1043, 1064, 0, 0, 0, 0, 1017, 994, 984, 1048, 26, 0, 0, 0, 1046, 1011, 1040, 1015, 0, 0, 0, 0, 970, 982, 1037, 1038, 0, 0, 0, 0, 1033, 1035, 1048, 1053, 0, 0, 0, 0, 1013, 1031, 1056, 974, 0, 0, 0, 0, 996, 996, 994, 1006, 5, 4, 3, 3] rerouted_tokens_per_ep_rank=[4165, 4145, 4069, 4112, 4027, 4169, 4074, 4007] avg=4096.0000 max=4169 max_rank=5 min=4007 min_rank=7 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16277 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1011, 1002, 1051, 1031, 0, 0, 0, 0, 1025, 997, 1025, 1056, 0, 0, 0, 0, 1019, 988, 1063, 1050, 26, 0, 0, 0, 1078, 1007, 1019, 1072, 0, 0, 0, 0, 1016, 1008, 1031, 1081, 0, 0, 0, 0, 1027, 1020, 1004, 986, 0, 0, 0, 0, 1035, 1023, 1023, 1004, 0, 0, 0, 0, 993, 971, 1006, 1005, 5, 4, 3, 3] rerouted_tokens_per_ep_rank=[4095, 4103, 4146, 4176, 4136, 4037, 4085, 3990] avg=4096.0000 max=4176 max_rank=3 min=3990 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16248 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[966, 1038, 1028, 1017, 0, 0, 0, 0, 1064, 987, 1028, 1048, 0, 0, 0, 0, 1022, 1003, 1037, 1007, 29, 0, 0, 0, 1053, 1035, 1018, 1011, 0, 0, 0, 0, 1051, 1033, 1025, 1034, 0, 0, 0, 0, 977, 991, 1045, 1000, 0, 0, 0, 0, 1040, 1022, 1034, 1015, 0, 0, 0, 0, 1011, 998, 1041, 1035, 10, 5, 5, 5] rerouted_tokens_per_ep_rank=[4049, 4127, 4098, 4117, 4143, 4013, 4111, 4110] avg=4096.0000 max=4143 max_rank=4 min=4013 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16377 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1013, 1003, 1042, 1037, 0, 0, 0, 0, 1030, 1019, 1029, 1041, 0, 0, 0, 0, 1040, 1007, 1040, 1005, 26, 0, 0, 0, 1046, 992, 991, 1031, 0, 0, 0, 0, 1019, 970, 1059, 1041, 0, 0, 0, 0, 1042, 994, 991, 990, 0, 0, 0, 0, 1032, 1037, 1069, 1042, 0, 0, 0, 0, 1026, 1022, 1058, 969, 5, 4, 3, 3] rerouted_tokens_per_ep_rank=[4095, 4119, 4118, 4060, 4089, 4017, 4180, 4090] avg=4096.0000 max=4180 max_rank=6 min=4017 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16376 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32790, 32777, 32768, 32737] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32790, 32777, 32768, 32737] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32790, 32777, 32768, 32737] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32790, 32777, 32768, 32737] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32790, 32777, 32768, 32737] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32790, 32777, 32768, 32737] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32790, 32777, 32768, 32737] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32790, 32777, 32768, 32737] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32790 actual_valid_recv=32896 observed_loss=-106 tokens_per_expert=[8192, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32777 actual_valid_recv=32896 observed_loss=-119 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32737 actual_valid_recv=33408 observed_loss=-671 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8448, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1014, 1021, 1025, 1022, 997, 1035, 1016, 1037, 1036, 1021, 973, 1021, 1019, 1044, 1050, 1043, 1011, 1048, 1019, 1040, 1013, 1064, 1028, 1011, 1010, 1035, 1015, 994, 1055, 1028, 1013, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[992, 1044, 1013, 1001, 996, 1064, 993, 1010, 998, 1032, 1060, 1012, 1017, 1003, 1052, 1023, 1057, 1057, 998, 1037, 1059, 1033, 1040, 1055, 991, 1076, 965, 977, 1056, 1023, 1042, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1003, 1039, 1021, 1012, 1012, 1048, 976, 1025, 1013, 1017, 1066, 1039, 1006, 1082, 1002, 1035, 1079, 1035, 1019, 1023, 992, 1015, 1016, 1057, 1005, 1040, 1021, 1058, 953, 975, 1039, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[984, 1081, 1004, 1023, 1078, 1068, 1059, 1027, 1049, 1044, 997, 1001, 1063, 1008, 1047, 998, 975, 1062, 1000, 1064, 984, 1031, 975, 1020, 971, 1035, 1051, 1014, 1008, 995, 988, 1064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1018, 984, 1039, 1040, 986, 1034, 1022, 1032, 1027, 1022, 1027, 1044, 1000, 1022, 1014, 1061, 1042, 1049, 1016, 1036, 1002, 1056, 1019, 1037, 1007, 1017, 1019, 1005, 1001, 1005, 1027, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1035, 1003, 1051, 1042, 1031, 1035, 1081, 1037, 964, 999, 1045, 1019, 993, 1007, 1042, 975, 1064, 980, 1046, 992, 1052, 1078, 1023, 1003, 1003, 1026, 1037, 1018, 1031, 980, 1022, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1033, 1026, 1033, 1018, 1058, 1003, 1033, 1020, 1023, 1073, 990, 1052, 1042, 1049, 1054, 1017, 1021, 1010, 1011, 996, 994, 1023, 1026, 1029, 973, 1036, 1004, 979, 1047, 1056, 1037, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1009, 1081, 1035, 1060, 1002, 1011, 1038, 1033, 979, 1026, 1070, 1010, 950, 1021, 1044, 1060, 1046, 1016, 1052, 993, 1036, 1029, 1052, 1092, 1066, 990, 1019, 1027, 993, 953, 978, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1014, 1017, 1025, 1022, 0, 0, 0, 0, 997, 1020, 1016, 1037, 0, 0, 0, 0, 1036, 1021, 973, 1021, 2, 0, 0, 0, 1019, 1044, 1041, 1043, 0, 0, 0, 0, 996, 1048, 1019, 1040, 0, 0, 0, 0, 1013, 1042, 1028, 1011, 0, 0, 0, 0, 1010, 1035, 1015, 994, 22, 13, 0, 0, 1055, 1028, 1013, 1010, 15, 9, 2, 2] rerouted_tokens_per_ep_rank=[4078, 4070, 4053, 4147, 4103, 4094, 4089, 4134] avg=4096.0000 max=4147 max_rank=3 min=4053 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1003, 1035, 1021, 1012, 0, 0, 0, 0, 1012, 1033, 976, 1025, 0, 0, 0, 0, 1013, 1017, 1066, 1039, 2, 0, 0, 0, 1006, 1082, 993, 1035, 0, 0, 0, 0, 1063, 1035, 1019, 1023, 0, 0, 0, 0, 992, 994, 1016, 1057, 0, 0, 0, 0, 1005, 1040, 1021, 1058, 21, 13, 0, 0, 953, 975, 1039, 1045, 16, 9, 2, 2] rerouted_tokens_per_ep_rank=[4071, 4046, 4137, 4116, 4140, 4059, 4158, 4041] avg=4096.0000 max=4158 max_rank=6 min=4041 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[984, 1077, 1004, 1023, 0, 0, 0, 0, 1078, 1053, 1059, 1027, 0, 0, 0, 0, 1049, 1044, 997, 1001, 2, 0, 0, 0, 1063, 1008, 1038, 998, 0, 0, 0, 0, 961, 1062, 1000, 1064, 0, 0, 0, 0, 984, 1010, 975, 1020, 0, 0, 0, 0, 971, 1035, 1051, 1014, 21, 13, 0, 0, 1008, 995, 988, 1064, 14, 9, 2, 2] rerouted_tokens_per_ep_rank=[4088, 4217, 4093, 4107, 4087, 3989, 4105, 4082] avg=4096.0000 max=4217 max_rank=1 min=3989 min_rank=5 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[992, 1040, 1013, 1001, 0, 0, 0, 0, 996, 1049, 993, 1010, 0, 0, 0, 0, 998, 1032, 1060, 1012, 2, 0, 0, 0, 1017, 1003, 1043, 1023, 0, 0, 0, 0, 1041, 1057, 998, 1037, 0, 0, 0, 0, 1059, 1012, 1040, 1055, 0, 0, 0, 0, 991, 1076, 965, 977, 21, 13, 0, 0, 1056, 1023, 1042, 992, 16, 9, 2, 2] rerouted_tokens_per_ep_rank=[4046, 4048, 4104, 4086, 4133, 4166, 4043, 4142] avg=4096.0000 max=4166 max_rank=5 min=4043 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1009, 1077, 1035, 1060, 0, 0, 0, 0, 1002, 996, 1038, 1033, 0, 0, 0, 0, 979, 1026, 1070, 1010, 2, 0, 0, 0, 950, 1021, 1035, 1060, 0, 0, 0, 0, 1031, 1016, 1052, 993, 0, 0, 0, 0, 1036, 1008, 1052, 1092, 0, 0, 0, 0, 1066, 990, 1019, 1027, 21, 13, 0, 0, 993, 953, 978, 997, 15, 9, 2, 2] rerouted_tokens_per_ep_rank=[4181, 4069, 4087, 4066, 4092, 4188, 4136, 3949] avg=4096.0000 max=4188 max_rank=5 min=3949 min_rank=7 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1018, 974, 1039, 1040, 0, 0, 0, 0, 986, 1010, 1022, 1032, 0, 0, 0, 0, 1027, 1022, 1027, 1044, 5, 0, 0, 0, 1000, 1022, 1002, 1061, 0, 0, 0, 0, 1023, 1049, 1016, 1036, 0, 0, 0, 0, 1002, 1029, 1019, 1037, 0, 0, 0, 0, 1007, 1017, 1019, 1005, 27, 17, 0, 0, 1001, 1005, 1027, 1058, 19, 12, 7, 5] rerouted_tokens_per_ep_rank=[4071, 4050, 4125, 4085, 4124, 4087, 4092, 4134] avg=4096.0000 max=4134 max_rank=7 min=4050 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1033, 1022, 1033, 1018, 0, 0, 0, 0, 1058, 988, 1033, 1020, 0, 0, 0, 0, 1023, 1073, 990, 1052, 2, 0, 0, 0, 1042, 1049, 1045, 1017, 0, 0, 0, 0, 1006, 1010, 1011, 996, 0, 0, 0, 0, 994, 1002, 1026, 1029, 0, 0, 0, 0, 973, 1036, 1004, 979, 21, 13, 0, 0, 1047, 1056, 1037, 1002, 15, 9, 2, 2] rerouted_tokens_per_ep_rank=[4106, 4099, 4140, 4153, 4023, 4051, 4026, 4170] avg=4096.0000 max=4170 max_rank=7 min=4023 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1035, 999, 1051, 1042, 0, 0, 0, 0, 1031, 1020, 1081, 1037, 0, 0, 0, 0, 964, 999, 1045, 1019, 2, 0, 0, 0, 993, 1007, 1033, 975, 0, 0, 0, 0, 1048, 980, 1046, 992, 0, 0, 0, 0, 1052, 1056, 1023, 1003, 0, 0, 0, 0, 1003, 1026, 1037, 1018, 22, 13, 0, 0, 1031, 980, 1022, 1054, 16, 9, 2, 2] rerouted_tokens_per_ep_rank=[4127, 4169, 4029, 4008, 4066, 4134, 4119, 4116] avg=4096.0000 max=4169 max_rank=1 min=4008 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8320, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True + [2026-08-30 04:58:04] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 900.3 | throughput per GPU (TFLOP/s/GPU): 370.3 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.328803E+01 | loss scale: 1.0 | grad norm: 27.162 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[994, 1020, 1002, 1002, 1086, 1045, 1005, 1018, 1045, 1046, 1035, 1033, 1016, 1037, 1041, 1029, 1009, 1036, 1018, 1010, 1024, 986, 1007, 984, 1039, 1050, 1034, 986, 1028, 1020, 1027, 1056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[983, 1010, 987, 1077, 1024, 1013, 1027, 978, 1023, 1033, 1060, 1043, 1082, 1032, 1001, 1007, 1033, 1089, 1081, 979, 1065, 1000, 974, 1027, 1041, 1028, 987, 1029, 967, 1034, 1047, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1032, 951, 1009, 1054, 1006, 1057, 1017, 964, 1013, 1047, 1032, 989, 1138, 1065, 1006, 1056, 1014, 987, 1061, 1064, 1019, 1040, 1083, 985, 991, 1034, 1046, 1024, 956, 1024, 1009, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1071, 990, 1011, 1021, 1045, 1054, 1029, 1047, 1009, 1047, 1045, 1016, 973, 1074, 1043, 1002, 1038, 1039, 1006, 1044, 949, 1014, 1007, 1024, 1006, 1019, 991, 1027, 1058, 1049, 1026, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[997, 1039, 995, 1055, 1023, 1009, 1045, 1042, 1006, 979, 1010, 950, 1030, 1063, 1039, 1054, 1022, 1021, 1033, 1025, 1065, 1031, 1024, 1052, 1053, 1069, 983, 1042, 1034, 1006, 967, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1018, 995, 1022, 966, 1011, 1062, 1025, 1000, 1027, 1043, 995, 1045, 1020, 1022, 1052, 1042, 1039, 987, 962, 1054, 1030, 1022, 1058, 1004, 1018, 1016, 1033, 1036, 1051, 1070, 1034, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1011, 1023, 1035, 1035, 1031, 1017, 1026, 977, 1029, 1042, 1034, 1019, 1036, 997, 1054, 980, 1010, 1019, 1030, 986, 988, 1033, 984, 1031, 1032, 1065, 1074, 1010, 1054, 1043, 1020, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1040, 1018, 1014, 1001, 987, 1033, 1037, 1088, 998, 999, 1108, 1017, 1024, 1019, 1045, 979, 979, 1035, 1022, 1047, 993, 1052, 1021, 1064, 1003, 1040, 1014, 992, 991, 1023, 1055, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[994, 1020, 1002, 1002, 35, 0, 0, 0, 1086, 1038, 1005, 1018, 0, 0, 0, 0, 1045, 1046, 1029, 1033, 0, 0, 0, 0, 981, 1037, 1041, 1029, 0, 0, 0, 0, 1009, 1035, 1018, 1010, 0, 0, 0, 0, 1024, 986, 1007, 984, 7, 6, 2, 0, 1039, 1045, 1034, 986, 0, 0, 0, 0, 1028, 1020, 1027, 1056, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4147, 4153, 4088, 4072, 4016, 4104, 4135] avg=4096.0000 max=4153 max_rank=2 min=4016 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16327 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1032, 951, 1009, 1054, 39, 0, 0, 0, 1006, 1050, 1017, 964, 0, 0, 0, 0, 1013, 1047, 1026, 989, 0, 0, 0, 0, 1099, 1065, 1006, 1056, 0, 0, 0, 0, 1014, 986, 1061, 1064, 0, 0, 0, 0, 1019, 1040, 1083, 985, 7, 6, 2, 0, 991, 1029, 1046, 1024, 0, 0, 0, 0, 956, 1024, 1009, 995, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4037, 4075, 4226, 4125, 4142, 4090, 3988] avg=4096.0000 max=4226 max_rank=3 min=3988 min_rank=7 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16345 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[983, 1010, 987, 1077, 37, 0, 0, 0, 1024, 1006, 1027, 978, 0, 0, 0, 0, 1023, 1033, 1054, 1043, 0, 0, 0, 0, 1045, 1032, 1001, 1007, 0, 0, 0, 0, 1033, 1088, 1081, 979, 0, 0, 0, 0, 1065, 1000, 974, 1027, 7, 6, 2, 0, 1041, 1023, 987, 1029, 0, 0, 0, 0, 967, 1034, 1047, 1007, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4035, 4153, 4085, 4181, 4081, 4080, 4059] avg=4096.0000 max=4181 max_rank=4 min=4035 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16401 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1071, 990, 1011, 1021, 33, 0, 0, 0, 1045, 1047, 1029, 1047, 0, 0, 0, 0, 1009, 1047, 1039, 1016, 0, 0, 0, 0, 940, 1074, 1043, 1002, 0, 0, 0, 0, 1038, 1038, 1006, 1044, 0, 0, 0, 0, 949, 1014, 1007, 1024, 7, 6, 2, 0, 1006, 1014, 991, 1027, 0, 0, 0, 0, 1058, 1049, 1026, 994, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4168, 4111, 4059, 4126, 4009, 4038, 4131] avg=4096.0000 max=4168 max_rank=1 min=4009 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16304 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[997, 1039, 995, 1055, 40, 0, 0, 0, 1023, 998, 1045, 1042, 0, 0, 0, 0, 1006, 979, 1002, 950, 0, 0, 0, 0, 990, 1063, 1039, 1054, 0, 0, 0, 0, 1022, 1017, 1033, 1025, 0, 0, 0, 0, 1065, 1031, 1024, 1052, 11, 8, 5, 0, 1053, 1060, 983, 1042, 0, 0, 0, 0, 1034, 1006, 967, 1005, 4, 4, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4108, 3937, 4146, 4097, 4196, 4138, 4020] avg=4096.0000 max=4196 max_rank=5 min=3937 min_rank=2 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16451 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1018, 995, 1022, 966, 35, 0, 0, 0, 1011, 1055, 1025, 1000, 0, 0, 0, 0, 1027, 1043, 990, 1045, 0, 0, 0, 0, 985, 1022, 1052, 1042, 0, 0, 0, 0, 1039, 986, 962, 1054, 0, 0, 0, 0, 1030, 1022, 1058, 1004, 7, 5, 2, 0, 1018, 1011, 1033, 1036, 0, 0, 0, 0, 1051, 1070, 1034, 1009, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4091, 4105, 4101, 4041, 4128, 4098, 4168] avg=4096.0000 max=4168 max_rank=7 min=4036 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16435 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1011, 1023, 1035, 1035, 36, 0, 0, 0, 1031, 1010, 1026, 977, 0, 0, 0, 0, 1029, 1042, 1028, 1019, 0, 0, 0, 0, 1000, 997, 1054, 980, 0, 0, 0, 0, 1010, 1018, 1030, 986, 0, 0, 0, 0, 988, 1033, 984, 1031, 7, 6, 2, 0, 1032, 1060, 1074, 1010, 0, 0, 0, 0, 1054, 1043, 1020, 1043, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4044, 4118, 4031, 4044, 4051, 4176, 4164] avg=4096.0000 max=4176 max_rank=6 min=4031 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16435 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1040, 1018, 1014, 1001, 35, 0, 0, 0, 987, 1026, 1037, 1088, 0, 0, 0, 0, 998, 999, 1102, 1017, 0, 0, 0, 0, 989, 1019, 1045, 979, 0, 0, 0, 0, 979, 1034, 1022, 1047, 0, 0, 0, 0, 993, 1052, 1021, 1064, 7, 6, 2, 0, 1003, 1035, 1014, 992, 0, 0, 0, 0, 991, 1023, 1055, 1030, 3, 1, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4138, 4116, 4032, 4082, 4145, 4044, 4103] avg=4096.0000 max=4145 max_rank=5 min=4032 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16374 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8320, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1004, 1028, 997, 1052, 1010, 1023, 1009, 1006, 1039, 1004, 1016, 1043, 1030, 1005, 1040, 1026, 1010, 1031, 1030, 1023, 1069, 1049, 1050, 1043, 986, 991, 985, 1044, 1027, 1009, 1018, 1071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1039, 1026, 1047, 1036, 1038, 1029, 1012, 1026, 1009, 1045, 1003, 998, 1000, 994, 1039, 1052, 971, 1066, 1048, 1013, 995, 994, 1041, 1017, 999, 1058, 1009, 1060, 1045, 1052, 990, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1042, 1061, 1014, 1072, 1009, 1041, 1050, 1000, 1016, 1001, 1070, 992, 1036, 970, 1011, 1010, 1012, 1014, 1070, 1072, 1003, 1022, 1024, 1025, 1026, 993, 1013, 1051, 1007, 988, 1034, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[982, 1008, 1037, 1026, 994, 1005, 1030, 1028, 1104, 1032, 1048, 1067, 1060, 982, 1042, 1066, 1024, 1036, 972, 991, 1015, 961, 1033, 1002, 985, 1063, 1008, 1002, 1065, 1029, 1027, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1102, 1058, 991, 1039, 1019, 1041, 1022, 1017, 1014, 1030, 1002, 964, 1054, 1034, 1006, 1043, 1021, 991, 1018, 1009, 986, 1069, 1008, 1017, 1052, 1032, 1030, 1056, 984, 1004, 1015, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1012, 1100, 994, 994, 1013, 1026, 1042, 1053, 962, 1017, 972, 1022, 1011, 1097, 1002, 1029, 1029, 1055, 1020, 1037, 1047, 957, 1036, 1015, 1066, 1000, 989, 1008, 1018, 1046, 1032, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1017, 1025, 1035, 1023, 998, 989, 1011, 1014, 1015, 1002, 1058, 1009, 1025, 1004, 1001, 1024, 1032, 1045, 979, 1007, 1069, 1024, 1020, 1075, 1043, 1070, 1018, 1026, 995, 1039, 1062, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1011, 1031, 985, 1028, 981, 1069, 1050, 1070, 1021, 1024, 1093, 1041, 1013, 998, 1020, 1050, 999, 1053, 971, 1028, 1008, 1049, 995, 1028, 1026, 1059, 1001, 984, 1012, 1023, 1015, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1004, 1011, 997, 1052, 0, 0, 0, 0, 1010, 1023, 1009, 1006, 5, 0, 0, 0, 1039, 1004, 1016, 1043, 2, 1, 0, 0, 1030, 1005, 1040, 1026, 0, 0, 0, 0, 1010, 1031, 1030, 1023, 11, 0, 0, 0, 1069, 1049, 1050, 1043, 2, 0, 0, 0, 986, 991, 985, 1044, 4, 0, 0, 0, 1027, 1009, 1018, 1063, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4053, 4105, 4101, 4105, 4213, 4010, 4117] avg=4096.0000 max=4213 max_rank=5 min=4010 min_rank=6 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[982, 991, 1037, 1026, 0, 0, 0, 0, 994, 1005, 1030, 1028, 5, 0, 0, 0, 1104, 1032, 1048, 1067, 2, 1, 0, 0, 1060, 982, 1042, 1066, 0, 0, 0, 0, 1024, 1036, 972, 991, 11, 0, 0, 0, 1015, 961, 1033, 1002, 2, 0, 0, 0, 985, 1063, 1008, 1002, 4, 0, 0, 0, 1065, 1029, 1027, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4062, 4254, 4150, 4034, 4013, 4062, 4157] avg=4096.0000 max=4254 max_rank=2 min=4013 min_rank=5 imbalance_ratio=1.038574 max_load_reduction=28514 max_load_reduction_pct=0.870178 moved_to_echo=16266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1039, 1009, 1047, 1036, 0, 0, 0, 0, 1038, 1029, 1012, 1026, 5, 0, 0, 0, 1009, 1045, 1003, 998, 2, 1, 0, 0, 1000, 994, 1039, 1052, 0, 0, 0, 0, 971, 1066, 1048, 1013, 11, 0, 0, 0, 995, 994, 1041, 1017, 1, 0, 0, 0, 999, 1058, 1009, 1060, 4, 0, 0, 0, 1045, 1052, 990, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4110, 4058, 4085, 4109, 4048, 4130, 4097] avg=4096.0000 max=4131 max_rank=0 min=4048 min_rank=5 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1042, 1044, 1014, 1072, 0, 0, 0, 0, 1009, 1041, 1050, 1000, 5, 0, 0, 0, 1016, 1001, 1070, 992, 2, 1, 0, 0, 1036, 970, 1011, 1010, 0, 0, 0, 0, 1012, 1014, 1070, 1072, 11, 0, 0, 0, 1003, 1022, 1024, 1025, 1, 0, 0, 0, 1026, 993, 1013, 1051, 4, 0, 0, 0, 1007, 988, 1034, 1012, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4105, 4082, 4027, 4179, 4075, 4087, 4041] avg=4096.0000 max=4179 max_rank=4 min=4027 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1012, 1082, 994, 994, 0, 0, 0, 0, 1013, 1026, 1042, 1053, 5, 0, 0, 0, 962, 1017, 972, 1022, 2, 1, 0, 0, 1011, 1097, 1002, 1029, 0, 0, 0, 0, 1029, 1055, 1020, 1037, 12, 0, 0, 0, 1047, 957, 1036, 1015, 2, 0, 0, 0, 1066, 1000, 989, 1008, 4, 0, 0, 0, 1018, 1046, 1032, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4139, 3976, 4139, 4153, 4057, 4067, 4155] avg=4096.0000 max=4155 max_rank=7 min=3976 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1102, 1030, 991, 1039, 0, 0, 0, 0, 1019, 1041, 1022, 1017, 8, 0, 0, 0, 1014, 1030, 1002, 964, 7, 7, 0, 0, 1054, 1034, 1006, 1037, 0, 0, 0, 0, 1021, 991, 1018, 1009, 13, 0, 0, 0, 986, 1069, 1008, 1017, 6, 6, 0, 0, 1052, 1032, 1030, 1056, 7, 0, 0, 0, 984, 1004, 1015, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4107, 4024, 4131, 4052, 4092, 4177, 4023] avg=4096.0000 max=4177 max_rank=6 min=4023 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1011, 1014, 985, 1028, 0, 0, 0, 0, 981, 1069, 1050, 1070, 5, 0, 0, 0, 1021, 1024, 1093, 1041, 2, 1, 0, 0, 1013, 998, 1020, 1050, 0, 0, 0, 0, 999, 1053, 971, 1028, 11, 0, 0, 0, 1008, 1049, 995, 1028, 1, 0, 0, 0, 1026, 1059, 1001, 984, 4, 0, 0, 0, 1012, 1023, 1015, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4175, 4182, 4081, 4062, 4081, 4074, 4075] avg=4096.0000 max=4182 max_rank=2 min=4038 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1017, 1008, 1035, 1023, 0, 0, 0, 0, 998, 989, 1011, 1014, 5, 0, 0, 0, 1015, 1002, 1058, 1009, 2, 1, 0, 0, 1025, 1004, 1001, 1024, 0, 0, 0, 0, 1032, 1045, 979, 1007, 11, 0, 0, 0, 1069, 1024, 1020, 1075, 1, 0, 0, 0, 1043, 1070, 1018, 1026, 4, 0, 0, 0, 995, 1039, 1062, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4017, 4087, 4054, 4074, 4189, 4161, 4103] avg=4096.0000 max=4189 max_rank=5 min=4017 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8064, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1062, 1013, 1041, 985, 996, 1061, 1007, 1036, 1027, 1032, 982, 1056, 992, 1022, 977, 1058, 1045, 1059, 1026, 1027, 1024, 999, 1025, 1003, 976, 1059, 983, 1063, 1028, 1039, 1026, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1082, 1074, 980, 995, 1024, 1029, 1033, 1025, 1011, 966, 1040, 1011, 1027, 1010, 1062, 1008, 1008, 982, 1004, 1034, 1045, 1063, 1020, 1024, 1028, 1046, 1026, 1010, 1046, 983, 1041, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1045, 1023, 997, 1060, 1011, 1039, 1022, 1041, 1030, 980, 1064, 1012, 1078, 991, 1052, 1024, 999, 1035, 1066, 1020, 1024, 996, 1046, 996, 1022, 1007, 994, 999, 1033, 1030, 990, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[988, 1026, 1089, 1018, 1006, 1016, 957, 1087, 1007, 1020, 1040, 1023, 1024, 1072, 1036, 1032, 1032, 1041, 1030, 1042, 982, 1055, 996, 1034, 1023, 1034, 971, 1018, 1001, 1058, 983, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1108, 1022, 1044, 1004, 1026, 1007, 1017, 1016, 1029, 1051, 988, 1032, 1027, 1046, 1017, 990, 1047, 1016, 1001, 1052, 1007, 1010, 1000, 1006, 1039, 1014, 1008, 1063, 1034, 979, 1042, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1037, 977, 1012, 1065, 1005, 1014, 1018, 1034, 1026, 1026, 1021, 1049, 1043, 1003, 1020, 972, 1084, 1044, 1018, 978, 1008, 1044, 1025, 1068, 981, 1017, 1018, 1011, 1015, 1066, 998, 1071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1028, 1031, 1032, 1041, 1020, 995, 1023, 1002, 1048, 1020, 1077, 979, 1055, 1018, 1009, 1019, 1046, 1005, 985, 1020, 1014, 1052, 1048, 1008, 1008, 1031, 1022, 1001, 1033, 1049, 1024, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1005, 1024, 1025, 1017, 975, 1051, 1058, 975, 1012, 1006, 1065, 1100, 1007, 1017, 1049, 1082, 959, 1028, 1067, 989, 953, 1038, 1028, 1029, 1072, 1017, 1057, 1000, 1017, 1005, 1010, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1039, 1013, 1041, 985, 0, 0, 0, 0, 996, 1061, 1007, 1036, 8, 4, 4, 0, 1027, 1032, 976, 1056, 0, 0, 0, 0, 984, 1022, 977, 1058, 0, 0, 0, 0, 1043, 1059, 1026, 1027, 0, 0, 0, 0, 1024, 999, 1025, 1003, 6, 2, 2, 0, 976, 1059, 983, 1063, 19, 0, 0, 0, 1028, 1039, 1026, 1033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4116, 4091, 4041, 4155, 4061, 4100, 4126] avg=4096.0000 max=4155 max_rank=4 min=4041 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1085, 1022, 1044, 1004, 0, 0, 0, 0, 1026, 1007, 1017, 1016, 8, 4, 4, 0, 1029, 1051, 982, 1032, 0, 0, 0, 0, 1019, 1046, 1017, 990, 0, 0, 0, 0, 1045, 1016, 1001, 1052, 0, 0, 0, 0, 1007, 1010, 1000, 1006, 6, 2, 2, 0, 1039, 1014, 1008, 1063, 19, 0, 0, 0, 1034, 979, 1042, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4082, 4094, 4072, 4114, 4033, 4143, 4075] avg=4096.0000 max=4155 max_rank=0 min=4033 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1059, 1074, 980, 995, 0, 0, 0, 0, 1024, 1029, 1033, 1025, 8, 4, 4, 0, 1011, 966, 1034, 1011, 0, 0, 0, 0, 1019, 1010, 1062, 1008, 0, 0, 0, 0, 1006, 982, 1004, 1034, 0, 0, 0, 0, 1045, 1063, 1020, 1024, 6, 2, 2, 0, 1028, 1046, 1026, 1010, 19, 0, 0, 0, 1046, 983, 1041, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4127, 4022, 4099, 4026, 4162, 4129, 4095] avg=4096.0000 max=4162 max_rank=5 min=4022 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1007, 1031, 1032, 1041, 0, 0, 0, 0, 1020, 995, 1023, 1002, 9, 5, 3, 0, 1048, 1020, 1070, 979, 0, 0, 0, 0, 1046, 1018, 1009, 1019, 0, 0, 0, 0, 1044, 1005, 985, 1020, 0, 0, 0, 0, 1014, 1052, 1048, 1008, 6, 2, 2, 0, 1008, 1031, 1022, 1001, 18, 0, 0, 0, 1033, 1049, 1024, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4057, 4117, 4092, 4054, 4132, 4080, 4125] avg=4096.0000 max=4132 max_rank=5 min=4054 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[960, 1026, 1089, 1018, 0, 0, 0, 0, 1006, 1016, 957, 1087, 13, 8, 7, 0, 1007, 1020, 1023, 1023, 0, 0, 0, 0, 1011, 1072, 1036, 1032, 0, 0, 0, 0, 1025, 1041, 1030, 1042, 0, 0, 0, 0, 982, 1055, 996, 1034, 12, 9, 7, 0, 1023, 1034, 971, 1018, 21, 0, 0, 0, 1001, 1058, 983, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4094, 4073, 4151, 4138, 4095, 4067, 4057] avg=4096.0000 max=4151 max_rank=3 min=4057 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[984, 1024, 1025, 1017, 0, 0, 0, 0, 975, 1051, 1058, 975, 8, 5, 3, 0, 1012, 1006, 1058, 1100, 0, 0, 0, 0, 999, 1017, 1049, 1082, 0, 0, 0, 0, 957, 1028, 1067, 989, 0, 0, 0, 0, 953, 1038, 1028, 1029, 6, 2, 2, 0, 1072, 1017, 1057, 1000, 18, 0, 0, 0, 1017, 1005, 1010, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4075, 4176, 4147, 4041, 4058, 4164, 4057] avg=4096.0000 max=4176 max_rank=2 min=4041 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1016, 977, 1012, 1065, 0, 0, 0, 0, 1005, 1014, 1018, 1034, 8, 4, 3, 0, 1026, 1026, 1015, 1049, 0, 0, 0, 0, 1035, 1003, 1020, 972, 0, 0, 0, 0, 1082, 1044, 1018, 978, 0, 0, 0, 0, 1008, 1044, 1025, 1068, 6, 2, 2, 0, 981, 1017, 1018, 1011, 18, 0, 0, 0, 1015, 1066, 998, 1065, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4086, 4116, 4030, 4122, 4155, 4045, 4144] avg=4096.0000 max=4155 max_rank=5 min=4030 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1023, 1023, 997, 1060, 0, 0, 0, 0, 1011, 1039, 1022, 1041, 9, 5, 4, 0, 1030, 980, 1057, 1012, 0, 0, 0, 0, 1069, 991, 1052, 1024, 0, 0, 0, 0, 997, 1035, 1066, 1020, 0, 0, 0, 0, 1024, 996, 1046, 996, 6, 2, 2, 0, 1022, 1007, 994, 999, 18, 0, 0, 0, 1033, 1030, 990, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4131, 4079, 4136, 4118, 4072, 4040, 4089] avg=4096.0000 max=4136 max_rank=3 min=4040 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1061, 1043, 1022, 1080, 1007, 1021, 1029, 1028, 1009, 1014, 1079, 1028, 992, 972, 1045, 1006, 1029, 1042, 1005, 1048, 1030, 1056, 1043, 1001, 989, 1014, 988, 1003, 1048, 997, 1028, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1035, 1049, 1003, 1000, 1009, 1056, 1016, 979, 1012, 1008, 1041, 981, 1006, 1017, 1051, 1015, 1040, 1066, 953, 1080, 1055, 1021, 1062, 1042, 1047, 1047, 1037, 1009, 1018, 987, 1014, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1013, 1042, 1009, 988, 1037, 1030, 989, 997, 996, 1011, 996, 1020, 1040, 981, 1071, 1037, 1013, 1002, 1015, 1072, 1010, 1030, 1069, 981, 1050, 1058, 1031, 999, 1013, 1058, 1054, 1056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1010, 1003, 1067, 995, 1020, 1027, 1001, 963, 1001, 1044, 1045, 1047, 1040, 1070, 1034, 1020, 1002, 1016, 1019, 987, 1005, 1037, 1039, 1014, 1005, 1022, 1038, 1041, 1035, 1026, 1044, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1017, 1098, 1009, 1015, 997, 1087, 1006, 1043, 1039, 987, 1062, 1031, 1015, 983, 1003, 1054, 1047, 1023, 1016, 1003, 1046, 959, 1016, 1007, 1015, 1058, 987, 1068, 1033, 1017, 992, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1019, 977, 1044, 1056, 1021, 954, 996, 1048, 1019, 979, 1003, 1043, 1081, 996, 1031, 1037, 1061, 1002, 1037, 1011, 1028, 1044, 1010, 1027, 1051, 1063, 1018, 1032, 1010, 1013, 1045, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[970, 1035, 1006, 1057, 1053, 998, 958, 1015, 1048, 1045, 1024, 1039, 1041, 1034, 1080, 1023, 1043, 996, 1028, 1045, 1012, 1032, 1061, 1006, 1024, 1008, 1050, 976, 1038, 1052, 982, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1054, 1002, 1001, 1036, 1010, 1024, 1059, 1014, 1054, 1022, 1023, 1047, 1005, 997, 1063, 1049, 1050, 1018, 1027, 1072, 1023, 1005, 982, 980, 1047, 1035, 991, 1015, 1029, 1030, 973, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1061, 1037, 1022, 1080, 0, 0, 0, 0, 1007, 1021, 1029, 1028, 15, 12, 6, 0, 1009, 1014, 1076, 1028, 0, 0, 0, 0, 992, 972, 1030, 1006, 0, 0, 0, 0, 1029, 1042, 1005, 1036, 0, 0, 0, 0, 1030, 1056, 1043, 1001, 3, 0, 0, 0, 989, 1010, 988, 1003, 0, 0, 0, 0, 1048, 997, 1028, 1011, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4200, 4118, 4127, 4000, 4112, 4133, 3990, 4088] avg=4096.0000 max=4200 max_rank=0 min=3990 min_rank=6 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1035, 1043, 1003, 1000, 0, 0, 0, 0, 1009, 1056, 1016, 979, 15, 12, 6, 0, 1012, 1008, 1038, 981, 0, 0, 0, 0, 1006, 1017, 1036, 1015, 0, 0, 0, 0, 1040, 1066, 953, 1068, 0, 0, 0, 0, 1055, 1021, 1062, 1042, 3, 0, 0, 0, 1047, 1043, 1037, 1009, 0, 0, 0, 0, 1018, 987, 1014, 1012, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4093, 4039, 4074, 4127, 4183, 4136, 4035] avg=4096.0000 max=4183 max_rank=5 min=4035 min_rank=7 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1010, 998, 1067, 995, 0, 0, 0, 0, 1020, 1027, 1001, 963, 14, 11, 5, 0, 1001, 1044, 1042, 1047, 0, 0, 0, 0, 1040, 1070, 1020, 1020, 0, 0, 0, 0, 1002, 1016, 1019, 976, 0, 0, 0, 0, 1005, 1037, 1039, 1014, 3, 0, 0, 0, 1005, 1018, 1038, 1041, 0, 0, 0, 0, 1035, 1026, 1044, 1051, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4041, 4134, 4150, 4013, 4098, 4102, 4160] avg=4096.0000 max=4160 max_rank=7 min=4013 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1013, 1036, 1009, 988, 0, 0, 0, 0, 1037, 1030, 989, 997, 15, 12, 6, 0, 996, 1011, 993, 1020, 0, 0, 0, 0, 1040, 981, 1056, 1037, 0, 0, 0, 0, 1013, 1002, 1015, 1060, 0, 0, 0, 0, 1010, 1030, 1069, 981, 3, 0, 0, 0, 1050, 1054, 1031, 999, 0, 0, 0, 0, 1013, 1058, 1054, 1056, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4086, 4020, 4114, 4090, 4093, 4134, 4185] avg=4096.0000 max=4185 max_rank=7 min=4020 min_rank=2 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1017, 1092, 1009, 1015, 0, 0, 0, 0, 997, 1087, 1006, 1043, 14, 12, 6, 0, 1039, 987, 1059, 1031, 0, 0, 0, 0, 1015, 983, 989, 1054, 0, 0, 0, 0, 1047, 1023, 1016, 991, 0, 0, 0, 0, 1046, 959, 1016, 1007, 3, 0, 0, 0, 1015, 1054, 987, 1068, 0, 0, 0, 0, 1033, 1017, 992, 1035, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4165, 4116, 4041, 4077, 4031, 4124, 4081] avg=4096.0000 max=4165 max_rank=1 min=4031 min_rank=5 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1019, 972, 1044, 1056, 0, 0, 0, 0, 1021, 954, 996, 1048, 14, 12, 5, 0, 1019, 979, 1000, 1043, 0, 0, 0, 0, 1081, 996, 1017, 1037, 0, 0, 0, 0, 1061, 1002, 1037, 999, 0, 0, 0, 0, 1028, 1044, 1010, 1027, 3, 0, 0, 0, 1051, 1059, 1018, 1032, 0, 0, 0, 0, 1010, 1013, 1045, 1012, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4050, 4041, 4131, 4099, 4112, 4160, 4084] avg=4096.0000 max=4160 max_rank=6 min=4041 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1054, 994, 1001, 1036, 0, 0, 0, 0, 1010, 1024, 1059, 1014, 19, 17, 8, 7, 1054, 1022, 1015, 1047, 0, 0, 0, 0, 1005, 997, 1044, 1049, 0, 0, 0, 0, 1050, 1018, 1027, 1055, 0, 0, 0, 0, 1023, 1005, 982, 980, 8, 6, 0, 0, 1047, 1015, 991, 1015, 0, 0, 0, 0, 1029, 1030, 973, 1031, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4158, 4138, 4095, 4150, 4004, 4068, 4070] avg=4096.0000 max=4158 max_rank=1 min=4004 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[970, 1029, 1006, 1057, 0, 0, 0, 0, 1053, 998, 958, 1015, 15, 12, 6, 0, 1048, 1045, 1021, 1039, 0, 0, 0, 0, 1041, 1034, 1065, 1023, 0, 0, 0, 0, 1043, 996, 1028, 1033, 0, 0, 0, 0, 1012, 1032, 1061, 1006, 3, 0, 0, 0, 1024, 1004, 1050, 976, 0, 0, 0, 0, 1038, 1052, 982, 989, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4057, 4153, 4163, 4100, 4114, 4054, 4065] avg=4096.0000 max=4163 max_rank=3 min=4054 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[973, 1021, 1029, 1022, 1059, 1035, 1050, 1043, 1044, 993, 1048, 1011, 1005, 1023, 1056, 954, 1001, 1078, 1008, 1023, 1047, 1032, 1008, 1023, 1018, 1065, 1034, 1057, 1016, 1008, 976, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1006, 1019, 996, 1046, 1042, 999, 1089, 1065, 1007, 987, 1036, 1046, 1019, 1050, 1019, 983, 1046, 1012, 1008, 1058, 986, 1008, 1005, 1035, 1015, 1032, 1052, 1001, 1033, 1017, 1006, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1044, 1013, 1023, 1006, 1003, 973, 1067, 999, 1051, 1047, 1029, 1006, 1024, 1066, 1077, 1046, 1012, 1018, 992, 987, 1035, 1013, 977, 1036, 1013, 1007, 1074, 977, 1074, 1026, 1038, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[982, 1007, 1000, 1078, 1061, 1019, 1020, 1008, 996, 1047, 1004, 1000, 1037, 1000, 1100, 1071, 1024, 1027, 1041, 1002, 1037, 1011, 1050, 1026, 1041, 1052, 975, 986, 966, 1014, 1005, 1081] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1011, 1044, 1005, 1042, 1032, 1036, 973, 995, 1034, 986, 1057, 1057, 1038, 1023, 996, 1001, 1056, 1026, 1017, 1029, 983, 1017, 1023, 1029, 1064, 1015, 1035, 998, 1028, 1036, 1041, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1051, 1037, 1017, 995, 1004, 1040, 1048, 978, 992, 1069, 1047, 1010, 1011, 1016, 1009, 1018, 1015, 1016, 1024, 1040, 1058, 1006, 1024, 1020, 1000, 1058, 998, 1088, 991, 1056, 1025, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1003, 1010, 988, 1013, 1060, 1020, 1015, 1041, 1001, 1029, 1007, 1027, 983, 1084, 998, 1031, 1054, 1042, 971, 1043, 1051, 1079, 1027, 1005, 1034, 1031, 1031, 1016, 1022, 1027, 1053, 972] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1056, 997, 1023, 1075, 1061, 1080, 1011, 1070, 986, 994, 969, 996, 1083, 1039, 976, 1040, 1018, 1016, 1063, 988, 1036, 1034, 1017, 1021, 1020, 1039, 1028, 1015, 1030, 994, 1004, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1006, 1019, 996, 1046, 9, 7, 0, 0, 1014, 999, 1089, 1065, 0, 0, 0, 0, 1007, 987, 1036, 1046, 19, 0, 0, 0, 1019, 1038, 1019, 983, 0, 0, 0, 0, 1046, 1012, 1008, 1058, 1, 0, 0, 0, 986, 1008, 1005, 1035, 1, 0, 0, 0, 1015, 1021, 1052, 1001, 0, 0, 0, 0, 1033, 1017, 1006, 1045, 9, 5, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4167, 4095, 4059, 4125, 4035, 4089, 4115] avg=4096.0000 max=4167 max_rank=1 min=4035 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[973, 1021, 1029, 1022, 9, 7, 0, 0, 1031, 1035, 1050, 1043, 0, 0, 0, 0, 1044, 993, 1048, 1011, 19, 0, 0, 0, 1005, 1011, 1056, 954, 0, 0, 0, 0, 1001, 1078, 1008, 1023, 1, 0, 0, 0, 1047, 1032, 1008, 1023, 1, 0, 0, 0, 1018, 1053, 1034, 1057, 0, 0, 0, 0, 1016, 1008, 976, 1008, 10, 5, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4159, 4115, 4026, 4111, 4111, 4162, 4023] avg=4096.0000 max=4162 max_rank=6 min=4023 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[982, 1007, 1000, 1078, 9, 7, 0, 0, 1033, 1019, 1020, 1008, 0, 0, 0, 0, 996, 1047, 1004, 1000, 19, 0, 0, 0, 1037, 988, 1100, 1071, 0, 0, 0, 0, 1024, 1027, 1041, 1002, 1, 0, 0, 0, 1037, 1011, 1050, 1026, 1, 0, 0, 0, 1041, 1040, 975, 986, 0, 0, 0, 0, 966, 1014, 1005, 1081, 10, 5, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4080, 4066, 4196, 4095, 4125, 4042, 4081] avg=4096.0000 max=4196 max_rank=3 min=4042 min_rank=6 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1044, 1013, 1023, 1006, 8, 8, 0, 0, 977, 973, 1067, 999, 0, 0, 0, 0, 1051, 1047, 1029, 1006, 18, 0, 0, 0, 1024, 1053, 1077, 1046, 0, 0, 0, 0, 1012, 1018, 992, 987, 1, 0, 0, 0, 1035, 1013, 977, 1036, 1, 0, 0, 0, 1013, 996, 1074, 977, 0, 0, 0, 0, 1074, 1026, 1038, 1015, 9, 5, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4016, 4151, 4200, 4010, 4062, 4060, 4167] avg=4096.0000 max=4200 max_rank=3 min=4010 min_rank=4 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1011, 1044, 1005, 1042, 9, 7, 0, 0, 1004, 1036, 973, 995, 0, 0, 0, 0, 1034, 986, 1057, 1057, 19, 0, 0, 0, 1038, 1011, 996, 1001, 0, 0, 0, 0, 1056, 1026, 1017, 1029, 1, 0, 0, 0, 983, 1017, 1023, 1029, 1, 0, 0, 0, 1064, 1004, 1035, 998, 0, 0, 0, 0, 1028, 1036, 1041, 1041, 9, 5, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4008, 4153, 4046, 4129, 4053, 4101, 4160] avg=4096.0000 max=4160 max_rank=7 min=4008 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1051, 1037, 1017, 995, 8, 7, 0, 0, 978, 1040, 1048, 978, 0, 0, 0, 0, 992, 1069, 1047, 1010, 18, 0, 0, 0, 1011, 1004, 1009, 1018, 0, 0, 0, 0, 1015, 1016, 1024, 1040, 1, 0, 0, 0, 1058, 1006, 1024, 1020, 1, 0, 0, 0, 1000, 1046, 998, 1088, 0, 0, 0, 0, 991, 1056, 1025, 1007, 10, 5, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4044, 4136, 4042, 4096, 4109, 4132, 4094] avg=4096.0000 max=4136 max_rank=2 min=4042 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1003, 1010, 988, 1013, 12, 13, 0, 0, 1024, 1020, 1015, 1041, 0, 0, 0, 0, 1001, 1029, 1007, 1027, 24, 0, 0, 0, 983, 1061, 998, 1031, 0, 0, 0, 0, 1054, 1042, 971, 1043, 6, 0, 0, 0, 1051, 1079, 1027, 1005, 2, 0, 0, 0, 1034, 1010, 1031, 1016, 0, 0, 0, 0, 1022, 1027, 1053, 972, 13, 10, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4100, 4088, 4073, 4116, 4164, 4091, 4097] avg=4096.0000 max=4164 max_rank=5 min=4039 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1056, 997, 1023, 1075, 9, 7, 0, 0, 1033, 1080, 1011, 1070, 0, 0, 0, 0, 986, 994, 969, 996, 19, 0, 0, 0, 1083, 1027, 976, 1040, 0, 0, 0, 0, 1018, 1016, 1063, 988, 1, 0, 0, 0, 1036, 1034, 1017, 1021, 1, 0, 0, 0, 1020, 1028, 1028, 1015, 0, 0, 0, 0, 1030, 994, 1004, 989, 9, 5, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4194, 3964, 4126, 4086, 4109, 4091, 4031] avg=4096.0000 max=4194 max_rank=1 min=3964 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1053, 1052, 1037, 1083, 989, 1032, 1017, 1045, 1033, 984, 1011, 988, 1026, 1023, 1017, 1039, 1049, 1035, 1029, 1032, 1050, 1028, 1010, 1033, 988, 988, 1024, 1010, 1034, 1026, 1014, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1043, 1016, 982, 982, 1022, 1070, 1011, 1066, 1025, 1046, 1022, 1058, 998, 1006, 1018, 1044, 956, 1009, 1045, 1050, 1016, 1059, 994, 1033, 1030, 1026, 1010, 1029, 1029, 1016, 1026, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1002, 981, 1028, 1025, 990, 997, 1007, 1020, 1041, 1031, 1073, 1021, 1045, 1001, 992, 987, 976, 1049, 1032, 1024, 1036, 1017, 1062, 1015, 1032, 1057, 1068, 1030, 1029, 1028, 1021, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1029, 1054, 1039, 1032, 1016, 1062, 1021, 995, 1004, 1019, 1013, 997, 993, 996, 1027, 1019, 1003, 1030, 1037, 1069, 1037, 974, 1087, 1035, 999, 1056, 1041, 1025, 995, 1034, 1004, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1036, 1061, 1007, 1022, 1035, 1006, 1070, 1072, 1027, 1071, 998, 980, 1036, 1017, 1029, 975, 1012, 1034, 965, 971, 1063, 1034, 997, 1077, 1036, 1063, 1012, 1003, 1038, 1030, 965, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1045, 1000, 986, 1028, 1034, 1029, 1040, 971, 1006, 1083, 1033, 1055, 1022, 1048, 1021, 1033, 956, 1033, 1016, 1027, 1060, 1026, 1007, 1007, 1015, 1002, 1021, 1066, 1034, 994, 1029, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1024, 992, 1029, 1024, 1041, 1009, 1009, 1012, 1015, 1021, 1071, 1017, 1039, 975, 1034, 1053, 1008, 1048, 1071, 1033, 977, 1071, 1058, 1020, 1019, 980, 1057, 1025, 992, 1007, 1038, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1095, 1007, 984, 1011, 1038, 975, 1057, 1075, 1016, 987, 1022, 1019, 1030, 1033, 1054, 958, 1045, 1059, 1036, 1052, 995, 1014, 1072, 1024, 1013, 1002, 997, 1033, 1014, 993, 1025, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1051, 1052, 1037, 1083, 0, 0, 0, 0, 989, 1032, 1017, 1037, 0, 0, 0, 0, 1033, 984, 1010, 988, 0, 0, 0, 0, 1026, 1023, 1017, 1039, 21, 0, 0, 0, 1049, 1033, 1029, 1032, 0, 0, 0, 0, 1050, 1028, 985, 1033, 0, 0, 0, 0, 988, 988, 1024, 1010, 1, 0, 0, 0, 1034, 1026, 1014, 989, 8, 4, 2, 2] rerouted_tokens_per_ep_rank=[4223, 4075, 4015, 4126, 4143, 4096, 4011, 4079] avg=4096.0000 max=4223 max_rank=0 min=4011 min_rank=6 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16329 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1041, 1016, 982, 982, 0, 0, 0, 0, 1022, 1070, 1011, 1058, 0, 0, 0, 0, 1025, 1046, 1021, 1058, 0, 0, 0, 0, 998, 1006, 1018, 1044, 21, 0, 0, 0, 956, 1007, 1045, 1050, 0, 0, 0, 0, 1016, 1059, 969, 1033, 0, 0, 0, 0, 1030, 1026, 1010, 1029, 1, 0, 0, 0, 1029, 1016, 1026, 1031, 8, 4, 2, 2] rerouted_tokens_per_ep_rank=[4021, 4161, 4150, 4087, 4058, 4077, 4096, 4118] avg=4096.0000 max=4161 max_rank=1 min=4021 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16349 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1000, 981, 1028, 1025, 0, 0, 0, 0, 990, 997, 1007, 1012, 0, 0, 0, 0, 1041, 1031, 1072, 1021, 0, 0, 0, 0, 1045, 1001, 992, 987, 23, 0, 0, 0, 976, 1047, 1032, 1024, 0, 0, 0, 0, 1036, 1017, 1034, 1015, 0, 0, 0, 0, 1032, 1057, 1068, 1030, 1, 0, 0, 0, 1029, 1028, 1021, 1051, 8, 5, 2, 2] rerouted_tokens_per_ep_rank=[4034, 4006, 4165, 4048, 4079, 4102, 4188, 4146] avg=4096.0000 max=4188 max_rank=6 min=4006 min_rank=1 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16515 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1027, 1054, 1039, 1032, 0, 0, 0, 0, 1016, 1062, 1021, 988, 0, 0, 0, 0, 1004, 1019, 1012, 997, 0, 0, 0, 0, 993, 996, 1027, 1019, 23, 0, 0, 0, 1003, 1028, 1037, 1069, 0, 0, 0, 0, 1037, 974, 1059, 1035, 0, 0, 0, 0, 999, 1056, 1041, 1025, 1, 0, 0, 0, 995, 1034, 1004, 1026, 7, 5, 2, 2] rerouted_tokens_per_ep_rank=[4152, 4087, 4032, 4058, 4137, 4105, 4122, 4075] avg=4096.0000 max=4152 max_rank=0 min=4032 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16439 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1043, 1000, 986, 1028, 0, 0, 0, 0, 1034, 1029, 1040, 964, 0, 0, 0, 0, 1006, 1083, 1032, 1055, 0, 0, 0, 0, 1022, 1048, 1021, 1033, 21, 0, 0, 0, 956, 1031, 1016, 1027, 0, 0, 0, 0, 1060, 1026, 982, 1007, 0, 0, 0, 0, 1015, 1002, 1021, 1066, 1, 0, 0, 0, 1034, 994, 1029, 1041, 7, 4, 2, 2] rerouted_tokens_per_ep_rank=[4057, 4067, 4176, 4145, 4030, 4075, 4105, 4113] avg=4096.0000 max=4176 max_rank=2 min=4030 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16323 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1022, 992, 1029, 1024, 0, 0, 0, 0, 1041, 1009, 1009, 1005, 0, 0, 0, 0, 1015, 1021, 1070, 1017, 0, 0, 0, 0, 1039, 975, 1034, 1053, 22, 0, 0, 0, 1008, 1046, 1071, 1033, 0, 0, 0, 0, 977, 1071, 1031, 1020, 0, 0, 0, 0, 1019, 980, 1057, 1025, 1, 0, 0, 0, 992, 1007, 1038, 999, 7, 5, 2, 2] rerouted_tokens_per_ep_rank=[4067, 4064, 4123, 4123, 4158, 4099, 4082, 4052] avg=4096.0000 max=4158 max_rank=4 min=4052 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16391 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1093, 1007, 984, 1011, 0, 0, 0, 0, 1038, 975, 1057, 1067, 0, 0, 0, 0, 1016, 987, 1021, 1019, 0, 0, 0, 0, 1030, 1033, 1054, 958, 23, 0, 0, 0, 1045, 1057, 1036, 1052, 0, 0, 0, 0, 995, 1014, 1044, 1024, 0, 0, 0, 0, 1013, 1002, 997, 1033, 1, 0, 0, 0, 1014, 993, 1025, 1033, 8, 5, 2, 2] rerouted_tokens_per_ep_rank=[4095, 4137, 4043, 4098, 4190, 4077, 4046, 4082] avg=4096.0000 max=4190 max_rank=4 min=4043 min_rank=2 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16395 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1029, 1061, 1007, 1022, 0, 0, 0, 0, 1035, 1006, 1070, 1060, 0, 0, 0, 0, 1027, 1071, 994, 980, 0, 0, 0, 0, 1036, 1017, 1029, 975, 26, 0, 0, 0, 1012, 1025, 965, 971, 0, 0, 0, 0, 1063, 1034, 963, 1077, 0, 0, 0, 0, 1036, 1063, 1012, 1003, 4, 0, 0, 0, 1038, 1030, 965, 1026, 12, 8, 9, 7] rerouted_tokens_per_ep_rank=[4119, 4171, 4072, 4083, 3973, 4137, 4118, 4095] avg=4096.0000 max=4171 max_rank=1 min=3973 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16323 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32776, 32768, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32776, 32768, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32776, 32768, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32776, 32768, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32776, 32768, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32776, 32768, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32776, 32768, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32776, 32768, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32776 actual_valid_recv=33024 observed_loss=-248 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32760 actual_valid_recv=33408 observed_loss=-648 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[995, 1029, 1061, 1001, 1031, 968, 995, 1013, 1068, 1018, 1008, 1042, 1082, 1039, 990, 947, 1010, 997, 1076, 990, 1099, 1027, 1046, 1013, 1023, 1057, 1053, 1038, 999, 1004, 1017, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1002, 1072, 1003, 986, 989, 1027, 1003, 1008, 1028, 1039, 1058, 1048, 1038, 1020, 1011, 1028, 1039, 1005, 1044, 1074, 1040, 947, 1034, 1046, 976, 1014, 1023, 1002, 1032, 1052, 1034, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1005, 1012, 982, 1025, 1021, 995, 1037, 1040, 1003, 1012, 1007, 1016, 1041, 1032, 1097, 1050, 1032, 1025, 996, 1018, 1064, 1051, 1062, 986, 1001, 1026, 1035, 1020, 997, 994, 1059, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1006, 991, 1052, 1041, 995, 1067, 1018, 1036, 1016, 995, 984, 1017, 1007, 973, 1035, 1037, 1029, 1060, 1041, 1056, 984, 1046, 1069, 1076, 1014, 954, 1002, 1019, 981, 1047, 1049, 1071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1031, 1062, 962, 1028, 1043, 1048, 1042, 1045, 993, 1066, 1018, 1001, 1007, 1010, 1055, 985, 1037, 1052, 1011, 986, 1011, 1025, 1013, 1014, 1050, 1054, 1053, 1000, 964, 1025, 1043, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1047, 989, 1039, 1011, 983, 1023, 1046, 1011, 991, 1040, 1011, 1031, 966, 1014, 1002, 1061, 1058, 1051, 1080, 1089, 1004, 1048, 1049, 1016, 1005, 1037, 977, 1053, 1040, 1008, 986, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1006, 1012, 1007, 992, 1016, 1015, 1021, 1037, 1021, 1047, 1015, 1059, 1002, 983, 1019, 1007, 1020, 1057, 1074, 1005, 1031, 1042, 1008, 1026, 1060, 1018, 1011, 1051, 1047, 1006, 1035, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1047, 1000, 1033, 986, 986, 1019, 1047, 1086, 1022, 1023, 1019, 1036, 1016, 1025, 988, 1017, 1019, 996, 1034, 999, 1075, 1030, 1076, 1052, 1047, 998, 1029, 1013, 988, 1017, 1006, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[995, 1029, 1061, 1001, 31, 0, 0, 0, 1031, 968, 995, 1013, 7, 0, 0, 0, 1068, 1018, 1008, 1042, 2, 0, 0, 0, 1082, 1039, 990, 947, 12, 11, 0, 0, 1010, 997, 1040, 990, 0, 0, 0, 0, 1099, 1027, 1004, 1013, 0, 0, 0, 0, 1023, 1057, 1053, 1038, 7, 0, 0, 0, 999, 1004, 1017, 1032, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4014, 4138, 4081, 4037, 4143, 4178, 4060] avg=4096.0000 max=4178 max_rank=6 min=4014 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1005, 1012, 982, 1025, 32, 0, 0, 0, 1021, 995, 1037, 1040, 6, 0, 0, 0, 1003, 1012, 1007, 1016, 1, 0, 0, 0, 1041, 1032, 1097, 1050, 11, 11, 0, 0, 1032, 1025, 964, 1018, 0, 0, 0, 0, 1064, 1051, 1019, 986, 0, 0, 0, 0, 1001, 1026, 1035, 1020, 6, 0, 0, 0, 997, 994, 1059, 1027, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4099, 4039, 4242, 4039, 4120, 4088, 4085] avg=4096.0000 max=4242 max_rank=3 min=4039 min_rank=2 imbalance_ratio=1.035645 max_load_reduction=28526 max_load_reduction_pct=0.870544 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1031, 1062, 962, 1028, 30, 0, 0, 0, 1043, 1048, 1042, 1045, 6, 0, 0, 0, 993, 1066, 1018, 1001, 1, 0, 0, 0, 1007, 1010, 1055, 985, 11, 10, 0, 0, 1037, 1052, 979, 986, 0, 0, 0, 0, 1011, 1025, 973, 1014, 0, 0, 0, 0, 1050, 1054, 1053, 1000, 6, 0, 0, 0, 964, 1025, 1043, 1034, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4184, 4079, 4078, 4054, 4023, 4163, 4074] avg=4096.0000 max=4184 max_rank=1 min=4023 min_rank=5 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1002, 1072, 1003, 986, 31, 0, 0, 0, 989, 1027, 1003, 1008, 7, 0, 0, 0, 1028, 1039, 1058, 1048, 1, 0, 0, 0, 1038, 1020, 1011, 1028, 11, 11, 0, 0, 1039, 1005, 1011, 1074, 0, 0, 0, 0, 1040, 947, 992, 1046, 0, 0, 0, 0, 976, 1014, 1023, 1002, 6, 0, 0, 0, 1032, 1052, 1034, 1046, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4034, 4174, 4119, 4129, 4025, 4021, 4172] avg=4096.0000 max=4174 max_rank=2 min=4021 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1047, 989, 1039, 1011, 31, 0, 0, 0, 983, 1023, 1046, 1011, 7, 0, 0, 0, 991, 1040, 1011, 1031, 2, 0, 0, 0, 966, 1014, 1002, 1061, 12, 11, 0, 0, 1058, 1051, 1044, 1089, 0, 0, 0, 0, 1004, 1048, 1007, 1016, 0, 0, 0, 0, 1005, 1037, 977, 1053, 7, 0, 0, 0, 1040, 1008, 986, 1002, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4070, 4075, 4066, 4242, 4075, 4079, 4044] avg=4096.0000 max=4242 max_rank=4 min=4044 min_rank=7 imbalance_ratio=1.035645 max_load_reduction=28526 max_load_reduction_pct=0.870544 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1006, 991, 1052, 1041, 36, 0, 0, 0, 995, 1067, 1018, 1036, 10, 0, 0, 0, 1016, 995, 984, 1017, 6, 0, 0, 0, 1007, 973, 1035, 1037, 15, 14, 0, 0, 1029, 1060, 987, 1056, 0, 0, 0, 0, 984, 1046, 1019, 1076, 0, 0, 0, 0, 1014, 954, 1002, 1019, 10, 0, 0, 0, 981, 1047, 1049, 1071, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4126, 4018, 4081, 4132, 4125, 3999, 4161] avg=4096.0000 max=4161 max_rank=7 min=3999 min_rank=6 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1006, 1012, 1007, 992, 30, 0, 0, 0, 1016, 1015, 1021, 1037, 7, 0, 0, 0, 1021, 1047, 1015, 1059, 2, 0, 0, 0, 1002, 983, 1019, 1007, 12, 10, 0, 0, 1020, 1057, 1038, 1005, 0, 0, 0, 0, 1031, 1042, 968, 1026, 0, 0, 0, 0, 1060, 1018, 1011, 1051, 7, 0, 0, 0, 1047, 1006, 1035, 1018, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4096, 4144, 4033, 4120, 4067, 4147, 4114] avg=4096.0000 max=4147 max_rank=6 min=4033 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1047, 1000, 1033, 986, 32, 0, 0, 0, 986, 1019, 1047, 1086, 7, 0, 0, 0, 1022, 1023, 1019, 1036, 1, 0, 0, 0, 1016, 1025, 988, 1017, 11, 11, 0, 0, 1019, 996, 1001, 999, 0, 0, 0, 0, 1075, 1030, 1033, 1052, 0, 0, 0, 0, 1047, 998, 1029, 1013, 6, 0, 0, 0, 988, 1017, 1006, 1039, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4145, 4101, 4068, 4015, 4190, 4093, 4058] avg=4096.0000 max=4190 max_rank=5 min=4015 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1034, 1002, 1065, 975, 1021, 1008, 1033, 1047, 1020, 1035, 1010, 1068, 1036, 974, 1033, 1050, 1040, 1040, 962, 1070, 1004, 1060, 982, 1030, 1003, 1038, 1014, 1067, 987, 1015, 1049, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1044, 1012, 980, 1007, 1058, 1036, 1013, 1058, 1004, 1003, 978, 1030, 1027, 1046, 1026, 1061, 1009, 1049, 1030, 1010, 1006, 1045, 1061, 1003, 1030, 998, 1052, 1039, 995, 1016, 1017, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1014, 1010, 1001, 1046, 1029, 1017, 967, 981, 1009, 1019, 1037, 1004, 1099, 1000, 1040, 1016, 1059, 985, 1027, 1045, 1020, 1064, 1034, 1038, 1016, 1016, 1039, 1005, 1045, 1012, 995, 1079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1020, 967, 1028, 1008, 1103, 1078, 1062, 1027, 1072, 997, 1048, 1034, 1007, 1012, 1009, 1003, 1073, 1034, 1026, 1058, 1061, 1009, 1060, 985, 995, 1002, 994, 1028, 987, 967, 970, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1014, 985, 1029, 1014, 1067, 1011, 1017, 1056, 1051, 1005, 1021, 998, 1015, 1023, 1019, 1036, 1093, 1048, 1025, 1030, 983, 1026, 1027, 1023, 1030, 999, 1026, 1040, 993, 1005, 1053, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1034, 1061, 1041, 1059, 1021, 1006, 1035, 1054, 1024, 997, 997, 1012, 1002, 1060, 999, 1009, 1030, 1018, 1011, 1040, 1002, 1001, 1014, 957, 1059, 1074, 1079, 1054, 956, 993, 1012, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1039, 1020, 1037, 1050, 1009, 1034, 990, 1019, 986, 1012, 987, 1076, 1045, 1028, 1028, 1034, 1054, 1020, 1036, 1027, 990, 999, 998, 996, 1051, 1052, 1039, 1035, 1047, 1034, 1035, 961] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[993, 1011, 974, 1046, 1053, 965, 1007, 1027, 1046, 1024, 1014, 1028, 1019, 1006, 1002, 1006, 1084, 955, 1042, 1054, 1053, 1026, 1057, 1014, 1055, 1001, 1032, 1029, 1039, 1033, 1029, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1034, 1002, 1065, 975, 13, 5, 0, 0, 1005, 1008, 1033, 1047, 0, 0, 0, 0, 1020, 1035, 1010, 1068, 14, 0, 0, 0, 1036, 974, 1033, 1050, 0, 0, 0, 0, 1002, 1040, 962, 1070, 0, 0, 0, 0, 1004, 1060, 982, 1030, 15, 2, 0, 0, 1003, 1038, 1014, 1039, 0, 0, 0, 0, 987, 1015, 1049, 996, 33, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4093, 4147, 4093, 4074, 4093, 4094, 4080] avg=4096.0000 max=4147 max_rank=2 min=4074 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1044, 1012, 980, 1007, 13, 5, 0, 0, 1041, 1036, 1013, 1058, 0, 0, 0, 0, 1004, 1003, 978, 1030, 15, 0, 0, 0, 1027, 1046, 1026, 1061, 0, 0, 0, 0, 972, 1049, 1030, 1010, 0, 0, 0, 0, 1006, 1045, 1061, 1003, 14, 2, 0, 0, 1030, 998, 1052, 1012, 0, 0, 0, 0, 995, 1016, 1017, 1025, 32, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4148, 4030, 4160, 4061, 4131, 4092, 4085] avg=4096.0000 max=4160 max_rank=3 min=4030 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1014, 1010, 1001, 1046, 12, 5, 0, 0, 1013, 1017, 967, 981, 0, 0, 0, 0, 1009, 1019, 1037, 1004, 14, 0, 0, 0, 1099, 1000, 1040, 1016, 0, 0, 0, 0, 1020, 985, 1027, 1045, 0, 0, 0, 0, 1020, 1064, 1034, 1038, 14, 2, 0, 0, 1016, 1016, 1039, 979, 0, 0, 0, 0, 1045, 1012, 995, 1079, 34, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 3978, 4083, 4155, 4077, 4172, 4050, 4165] avg=4096.0000 max=4172 max_rank=5 min=3978 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1020, 967, 1028, 1008, 12, 5, 0, 0, 1086, 1078, 1062, 1027, 0, 0, 0, 0, 1072, 997, 1048, 1034, 15, 0, 0, 0, 1007, 1012, 1009, 1003, 0, 0, 0, 0, 1034, 1034, 1026, 1058, 0, 0, 0, 0, 1061, 1009, 1060, 985, 14, 2, 0, 0, 995, 1002, 994, 1002, 0, 0, 0, 0, 987, 967, 970, 1044, 34, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4253, 4166, 4031, 4152, 4131, 3993, 4002] avg=4096.0000 max=4253 max_rank=1 min=3993 min_rank=6 imbalance_ratio=1.038330 max_load_reduction=28515 max_load_reduction_pct=0.870209 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1014, 985, 1029, 1014, 13, 5, 0, 0, 1050, 1011, 1017, 1056, 0, 0, 0, 0, 1051, 1005, 1021, 998, 15, 0, 0, 0, 1015, 1023, 1019, 1036, 0, 0, 0, 0, 1053, 1048, 1025, 1030, 0, 0, 0, 0, 983, 1026, 1027, 1023, 14, 2, 0, 0, 1030, 999, 1026, 1013, 0, 0, 0, 0, 993, 1005, 1053, 1006, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4134, 4090, 4093, 4156, 4075, 4068, 4092] avg=4096.0000 max=4156 max_rank=4 min=4060 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1034, 1061, 1041, 1059, 17, 9, 0, 0, 996, 1006, 1035, 1054, 0, 0, 0, 0, 1024, 997, 997, 1012, 18, 2, 0, 0, 1000, 1060, 999, 1009, 0, 0, 0, 0, 984, 1018, 1011, 1040, 0, 0, 0, 0, 1002, 1001, 1014, 957, 20, 7, 0, 0, 1059, 1074, 1079, 1017, 0, 0, 0, 0, 956, 993, 1012, 1057, 37, 0, 0, 0] rerouted_tokens_per_ep_rank=[4221, 4091, 4050, 4068, 4053, 4001, 4229, 4055] avg=4096.0000 max=4229 max_rank=6 min=4001 min_rank=5 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1039, 1020, 1037, 1050, 12, 5, 0, 0, 993, 1034, 990, 1019, 0, 0, 0, 0, 986, 1012, 987, 1076, 14, 0, 0, 0, 1045, 1028, 1028, 1034, 0, 0, 0, 0, 1016, 1020, 1036, 1027, 0, 0, 0, 0, 990, 999, 998, 996, 14, 2, 0, 0, 1051, 1052, 1039, 1009, 0, 0, 0, 0, 1047, 1034, 1035, 961, 33, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4036, 4075, 4135, 4099, 3999, 4151, 4110] avg=4096.0000 max=4163 max_rank=0 min=3999 min_rank=5 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[993, 1011, 974, 1046, 12, 5, 0, 0, 1036, 965, 1007, 1027, 0, 0, 0, 0, 1046, 1024, 1014, 1028, 15, 0, 0, 0, 1019, 1006, 1002, 1006, 0, 0, 0, 0, 1045, 955, 1042, 1054, 0, 0, 0, 0, 1053, 1026, 1057, 1014, 14, 2, 0, 0, 1055, 1001, 1032, 1003, 0, 0, 0, 0, 1039, 1033, 1029, 1044, 34, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4035, 4127, 4033, 4096, 4166, 4091, 4179] avg=4096.0000 max=4179 max_rank=7 min=4033 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16532 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8320, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1007, 1049, 1047, 1027, 1014, 1039, 1005, 1007, 995, 1036, 996, 1002, 1028, 1023, 1005, 1048, 1026, 1074, 1029, 1046, 1009, 1014, 999, 991, 1035, 1016, 1026, 1003, 1056, 994, 1080, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1011, 995, 1018, 1002, 1022, 1074, 1008, 1016, 1020, 998, 1039, 1055, 1053, 1023, 1043, 995, 1019, 1030, 1037, 1040, 1049, 1014, 1047, 1017, 1001, 986, 1013, 1022, 1046, 1020, 1012, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1023, 1065, 1054, 1040, 985, 990, 1050, 992, 958, 1018, 1024, 1064, 1042, 1014, 1054, 1043, 1046, 1008, 1035, 1035, 1015, 1036, 1023, 1031, 1000, 1052, 974, 1023, 1032, 1043, 999, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1020, 1057, 1039, 1034, 986, 1019, 1024, 1038, 1014, 1056, 1038, 1018, 1046, 992, 1054, 1008, 1036, 1029, 1028, 1047, 1030, 1001, 995, 996, 1070, 1023, 1029, 1038, 980, 1018, 995, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1036, 1074, 1031, 1003, 1039, 1053, 1044, 985, 1031, 1023, 1015, 1014, 1022, 1012, 1012, 1005, 1025, 1025, 1029, 1011, 1007, 1035, 1046, 965, 997, 1029, 1026, 1044, 1049, 1028, 1009, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1080, 1005, 1057, 1023, 1029, 984, 1064, 1034, 1045, 1057, 1022, 992, 967, 987, 1050, 1006, 1042, 1014, 1015, 1021, 1032, 988, 1016, 1029, 1015, 1040, 1050, 1014, 1038, 988, 1030, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1076, 1042, 1031, 1000, 1091, 1031, 1016, 1025, 1076, 1045, 1033, 1022, 1069, 1019, 985, 1014, 992, 1007, 1012, 969, 1046, 1029, 935, 1011, 1084, 1010, 1049, 990, 1034, 1004, 1006, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1067, 1014, 998, 988, 1022, 1035, 1000, 962, 1022, 1056, 1002, 1045, 1006, 1040, 931, 1058, 1048, 1036, 1027, 1028, 1037, 1049, 1019, 990, 1015, 1034, 994, 1024, 1043, 1114, 1023, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[982, 995, 1018, 1002, 0, 0, 0, 0, 1022, 1074, 1008, 1016, 7, 2, 0, 0, 1020, 991, 1039, 1055, 0, 0, 0, 0, 1053, 1023, 1043, 995, 10, 4, 0, 0, 1008, 1030, 1037, 1040, 0, 0, 0, 0, 1049, 1014, 1047, 1017, 29, 2, 0, 0, 1001, 986, 1013, 1022, 5, 0, 0, 0, 1034, 1020, 1012, 1043, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3997, 4129, 4105, 4128, 4115, 4158, 4027, 4109] avg=4096.0000 max=4158 max_rank=5 min=3997 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[978, 1049, 1047, 1027, 0, 0, 0, 0, 1014, 1039, 1005, 1007, 7, 2, 0, 0, 995, 1029, 996, 1002, 0, 0, 0, 0, 1028, 1023, 1005, 1048, 10, 4, 0, 0, 1015, 1074, 1029, 1046, 0, 0, 0, 0, 1009, 1014, 999, 991, 29, 2, 0, 0, 1035, 1016, 1026, 1003, 5, 0, 0, 0, 1044, 994, 1080, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4074, 4022, 4118, 4164, 4044, 4085, 4160] avg=4096.0000 max=4164 max_rank=4 min=4022 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[990, 1057, 1039, 1034, 0, 0, 0, 0, 986, 1019, 1024, 1038, 8, 2, 0, 0, 1014, 1049, 1038, 1018, 0, 0, 0, 0, 1046, 992, 1054, 1008, 9, 4, 0, 0, 1024, 1029, 1028, 1047, 0, 0, 0, 0, 1030, 1001, 995, 996, 30, 2, 0, 0, 1070, 1023, 1029, 1038, 5, 0, 0, 0, 969, 1018, 995, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4077, 4119, 4113, 4128, 4054, 4165, 3992] avg=4096.0000 max=4165 max_rank=6 min=3992 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[993, 1065, 1054, 1040, 0, 0, 0, 0, 985, 990, 1050, 992, 8, 2, 0, 0, 958, 1011, 1024, 1064, 0, 0, 0, 0, 1042, 1014, 1054, 1043, 9, 4, 0, 0, 1034, 1008, 1035, 1035, 0, 0, 0, 0, 1015, 1036, 1023, 1031, 30, 2, 0, 0, 1000, 1052, 974, 1023, 5, 0, 0, 0, 1021, 1043, 999, 1000, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4027, 4057, 4166, 4112, 4137, 4054, 4063] avg=4096.0000 max=4166 max_rank=3 min=4027 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1049, 1005, 1057, 1023, 0, 0, 0, 0, 1029, 984, 1064, 1034, 8, 2, 0, 0, 1045, 1050, 1022, 992, 0, 0, 0, 0, 967, 987, 1050, 1006, 10, 4, 0, 0, 1030, 1014, 1015, 1021, 0, 0, 0, 0, 1032, 988, 1016, 1029, 31, 2, 0, 0, 1015, 1040, 1050, 1014, 5, 0, 0, 0, 1026, 988, 1030, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4121, 4109, 4024, 4080, 4098, 4124, 4078] avg=4096.0000 max=4134 max_rank=0 min=4024 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1002, 1074, 1031, 1003, 0, 0, 0, 0, 1039, 1053, 1044, 985, 11, 7, 0, 0, 1031, 1009, 1015, 1014, 0, 0, 0, 0, 1022, 1012, 1012, 1005, 13, 6, 0, 0, 1008, 1025, 1029, 1011, 0, 0, 0, 0, 1007, 1035, 1046, 965, 34, 8, 0, 0, 997, 1029, 1026, 1044, 7, 0, 0, 0, 1028, 1028, 1009, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4139, 4069, 4070, 4073, 4095, 4103, 4109] avg=4096.0000 max=4139 max_rank=1 min=4069 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1045, 1042, 1031, 1000, 0, 0, 0, 0, 1091, 1031, 1016, 1025, 7, 2, 0, 0, 1076, 1038, 1033, 1022, 0, 0, 0, 0, 1069, 1019, 985, 1014, 9, 4, 0, 0, 981, 1007, 1012, 969, 0, 0, 0, 0, 1046, 1029, 935, 1011, 31, 2, 0, 0, 1084, 1010, 1049, 990, 5, 0, 0, 0, 1023, 1004, 1006, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4172, 4169, 4100, 3969, 4054, 4138, 4048] avg=4096.0000 max=4172 max_rank=1 min=3969 min_rank=4 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16209 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1036, 1014, 998, 988, 0, 0, 0, 0, 1022, 1035, 1000, 962, 8, 2, 0, 0, 1022, 1049, 1002, 1045, 0, 0, 0, 0, 1006, 1040, 931, 1058, 10, 4, 0, 0, 1036, 1036, 1027, 1028, 0, 0, 0, 0, 1037, 1049, 1019, 990, 31, 2, 0, 0, 1015, 1034, 994, 1024, 5, 0, 0, 0, 1031, 1114, 1023, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4029, 4118, 4049, 4127, 4128, 4072, 4209] avg=4096.0000 max=4209 max_rank=7 min=4029 min_rank=1 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16536 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1043, 967, 1010, 1032, 1011, 1054, 1043, 1011, 1031, 1031, 1060, 991, 1050, 1047, 978, 1046, 1003, 1035, 1020, 1012, 992, 1028, 1022, 1057, 1007, 1003, 1031, 1023, 1004, 1072, 1026, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1033, 1053, 1001, 1041, 1025, 1057, 1045, 1030, 1073, 1026, 1009, 1010, 946, 1016, 1009, 1002, 1008, 1018, 1070, 1011, 1051, 1054, 1006, 982, 1088, 935, 1023, 962, 1025, 1056, 1055, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1031, 1042, 1025, 979, 1009, 1054, 1008, 1029, 1047, 1050, 1010, 1031, 1070, 1014, 1036, 1055, 948, 1007, 1026, 1040, 1030, 991, 1016, 1052, 1046, 1024, 1025, 1003, 977, 1032, 998, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[947, 1027, 1023, 1004, 1078, 1030, 981, 1024, 1067, 955, 971, 1034, 1055, 1049, 1020, 991, 995, 1026, 1065, 1055, 1088, 1056, 1028, 1056, 1039, 1004, 1028, 1025, 1016, 984, 1027, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[975, 1032, 1021, 1096, 1030, 986, 1050, 1030, 1035, 1069, 1026, 995, 1064, 1064, 1003, 1040, 1013, 1048, 1001, 1044, 987, 1038, 1010, 1015, 991, 1055, 982, 1084, 1053, 964, 1022, 945] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1048, 1002, 1040, 1007, 1010, 1013, 1000, 1058, 1025, 1016, 1074, 1019, 1037, 1001, 1063, 1014, 1003, 992, 1036, 1067, 1039, 1049, 998, 1038, 1010, 1018, 991, 994, 1004, 1018, 1070, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[986, 1021, 1050, 1019, 998, 1027, 1031, 1027, 1029, 1020, 1002, 1000, 1066, 977, 1034, 1026, 1004, 1023, 998, 1007, 1002, 994, 1045, 1076, 1065, 1014, 997, 1034, 1075, 1043, 1034, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[972, 1025, 1061, 1033, 1085, 1014, 1017, 1016, 999, 1021, 1014, 978, 994, 1023, 1053, 1013, 1039, 1002, 1041, 1021, 1054, 1062, 1051, 1014, 996, 1046, 996, 1026, 1074, 1022, 997, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1043, 967, 1010, 1032, 13, 1, 0, 0, 998, 1054, 1043, 1011, 0, 0, 0, 0, 1031, 1031, 1060, 991, 6, 0, 0, 0, 1039, 1047, 978, 1046, 0, 0, 0, 0, 1003, 1035, 1020, 1012, 11, 0, 0, 0, 992, 1028, 1022, 1031, 0, 0, 0, 0, 1007, 1003, 1031, 1023, 25, 0, 0, 0, 1004, 1072, 1020, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4106, 4119, 4110, 4081, 4073, 4089, 4124] avg=4096.0000 max=4124 max_rank=7 min=4066 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1033, 1053, 1001, 1041, 13, 1, 0, 0, 1012, 1057, 1045, 1030, 0, 0, 0, 0, 1073, 1026, 1009, 1010, 6, 0, 0, 0, 936, 1016, 1009, 1002, 0, 0, 0, 0, 1008, 1018, 1070, 1011, 10, 0, 0, 0, 1051, 1054, 1006, 957, 0, 0, 0, 0, 1088, 935, 1023, 962, 24, 0, 0, 0, 1025, 1056, 1049, 1048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4144, 4124, 3963, 4117, 4068, 4032, 4178] avg=4096.0000 max=4178 max_rank=7 min=3963 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1048, 1002, 1040, 1007, 13, 1, 0, 0, 997, 1013, 1000, 1058, 0, 0, 0, 0, 1025, 1016, 1074, 1019, 6, 0, 0, 0, 1026, 1001, 1063, 1014, 0, 0, 0, 0, 1003, 992, 1036, 1067, 11, 0, 0, 0, 1039, 1049, 998, 1012, 0, 0, 0, 0, 1010, 1018, 991, 994, 25, 0, 0, 0, 1004, 1018, 1064, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4068, 4140, 4104, 4109, 4098, 4038, 4100] avg=4096.0000 max=4140 max_rank=2 min=4038 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1031, 1042, 1025, 979, 19, 3, 0, 0, 989, 1054, 1008, 1029, 0, 0, 0, 0, 1047, 1050, 1010, 1031, 8, 0, 0, 0, 1057, 1014, 1036, 1055, 0, 0, 0, 0, 948, 1007, 1026, 1040, 13, 1, 1, 0, 1030, 991, 1016, 1019, 0, 0, 0, 0, 1046, 1024, 1025, 1003, 30, 0, 0, 0, 977, 1032, 989, 1063, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4080, 4146, 4162, 4036, 4056, 4128, 4061] avg=4096.0000 max=4162 max_rank=3 min=4036 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[986, 1021, 1050, 1019, 13, 1, 0, 0, 985, 1027, 1031, 1027, 0, 0, 0, 0, 1029, 1020, 1002, 1000, 6, 0, 0, 0, 1055, 977, 1034, 1026, 0, 0, 0, 0, 1004, 1023, 998, 1007, 11, 0, 0, 0, 1002, 994, 1045, 1049, 0, 0, 0, 0, 1065, 1014, 997, 1034, 26, 0, 0, 0, 1075, 1043, 1028, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4070, 4057, 4092, 4043, 4090, 4136, 4190] avg=4096.0000 max=4190 max_rank=7 min=4043 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[947, 1027, 1023, 1004, 14, 1, 0, 0, 1064, 1030, 981, 1024, 0, 0, 0, 0, 1067, 955, 971, 1034, 6, 0, 0, 0, 1044, 1049, 1020, 991, 0, 0, 0, 0, 995, 1026, 1065, 1055, 11, 0, 0, 0, 1088, 1056, 1028, 1030, 0, 0, 0, 0, 1039, 1004, 1028, 1025, 25, 0, 0, 0, 1016, 984, 1021, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4099, 4033, 4104, 4152, 4202, 4121, 4041] avg=4096.0000 max=4202 max_rank=5 min=4016 min_rank=0 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[975, 1032, 1021, 1096, 13, 1, 0, 0, 1017, 986, 1050, 1030, 0, 0, 0, 0, 1035, 1069, 1026, 995, 6, 0, 0, 0, 1053, 1064, 1003, 1040, 0, 0, 0, 0, 1013, 1048, 1001, 1044, 11, 0, 0, 0, 987, 1038, 1010, 990, 0, 0, 0, 0, 991, 1055, 982, 1084, 24, 0, 0, 0, 1053, 964, 1016, 945, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4083, 4131, 4160, 4117, 4025, 4136, 3978] avg=4096.0000 max=4160 max_rank=3 min=3978 min_rank=7 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[972, 1025, 1061, 1033, 14, 1, 0, 0, 1071, 1014, 1017, 1016, 0, 0, 0, 0, 999, 1021, 1014, 978, 6, 0, 0, 0, 984, 1023, 1053, 1013, 0, 0, 0, 0, 1039, 1002, 1041, 1021, 10, 0, 0, 0, 1054, 1062, 1051, 989, 0, 0, 0, 0, 996, 1046, 996, 1026, 24, 0, 0, 0, 1074, 1022, 991, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4118, 4018, 4073, 4113, 4156, 4088, 4096] avg=4096.0000 max=4156 max_rank=5 min=4018 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1071, 978, 995, 1022, 994, 1004, 1022, 1091, 985, 1042, 1041, 1056, 1037, 1021, 1021, 1042, 1023, 1015, 1031, 989, 1090, 984, 1056, 992, 1046, 1003, 1031, 1043, 984, 1048, 1029, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1072, 1028, 1007, 1040, 1032, 1018, 1014, 1104, 983, 1028, 1001, 1022, 962, 999, 1029, 1061, 985, 985, 1017, 984, 1004, 1011, 1059, 1081, 1040, 1033, 1004, 1034, 1066, 1024, 1032, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[992, 1024, 1019, 1038, 1036, 1008, 1035, 1008, 1063, 1054, 1000, 992, 1037, 1032, 1029, 1087, 1063, 1020, 1014, 1039, 1020, 960, 1059, 977, 1047, 988, 1011, 1018, 1014, 1050, 994, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1009, 1005, 1050, 1018, 1064, 1077, 1008, 1007, 1055, 1055, 1006, 1049, 1018, 1042, 1031, 1008, 1053, 1035, 986, 999, 1039, 983, 1043, 981, 992, 1052, 984, 999, 992, 1018, 1049, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1046, 1011, 1052, 1063, 1064, 1022, 1009, 1003, 1023, 1007, 1022, 998, 1015, 1016, 1025, 974, 1060, 1029, 1054, 999, 1041, 997, 1045, 1012, 1004, 1028, 1069, 1049, 990, 1029, 1024, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1037, 1021, 1037, 1083, 1057, 1030, 1028, 1013, 1022, 1032, 1041, 983, 1046, 1029, 1020, 1000, 980, 1005, 1039, 967, 1007, 1065, 1032, 1033, 1029, 1036, 1033, 1027, 1002, 971, 997, 1066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1063, 1038, 1021, 1012, 1025, 1039, 1024, 1019, 1033, 1040, 1042, 1022, 1012, 1034, 996, 1052, 1003, 989, 994, 1022, 1025, 1010, 1034, 1065, 1019, 1053, 1004, 1020, 1002, 1050, 1021, 985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1005, 1003, 1030, 1009, 1072, 1039, 1053, 999, 973, 1006, 1058, 1005, 1030, 1039, 1045, 1054, 1029, 1000, 1044, 1055, 1042, 1070, 989, 940, 1017, 1041, 1053, 1024, 1011, 983, 1048, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1055, 978, 995, 1022, 0, 0, 0, 0, 965, 1004, 1022, 1091, 0, 0, 0, 0, 985, 1042, 1041, 1056, 3, 0, 0, 0, 1037, 1021, 1021, 1033, 0, 0, 0, 0, 1023, 1015, 1031, 989, 29, 1, 0, 0, 1090, 984, 1056, 992, 2, 0, 0, 0, 1046, 997, 1031, 1043, 0, 0, 0, 0, 984, 1048, 1029, 982, 15, 9, 1, 0] rerouted_tokens_per_ep_rank=[4050, 4082, 4127, 4112, 4088, 4124, 4117, 4068] avg=4096.0000 max=4127 max_rank=2 min=4050 min_rank=0 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[977, 1024, 1019, 1038, 0, 0, 0, 0, 1005, 1008, 1035, 1008, 0, 0, 0, 0, 1063, 1054, 1000, 992, 3, 0, 0, 0, 1037, 1032, 1029, 1078, 0, 0, 0, 0, 1063, 1020, 1014, 1039, 31, 1, 0, 0, 1020, 960, 1059, 977, 2, 0, 0, 0, 1047, 982, 1011, 1018, 0, 0, 0, 0, 1014, 1050, 994, 1040, 14, 9, 1, 0] rerouted_tokens_per_ep_rank=[4058, 4056, 4112, 4176, 4168, 4018, 4058, 4122] avg=4096.0000 max=4176 max_rank=3 min=4018 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1056, 1028, 1007, 1040, 0, 0, 0, 0, 1002, 1018, 1014, 1104, 0, 0, 0, 0, 983, 1028, 1001, 1022, 3, 0, 0, 0, 962, 999, 1029, 1052, 0, 0, 0, 0, 985, 985, 1017, 984, 30, 1, 0, 0, 1004, 1011, 1059, 1081, 2, 0, 0, 0, 1040, 1027, 1004, 1034, 0, 0, 0, 0, 1066, 1024, 1032, 1009, 15, 9, 1, 0] rerouted_tokens_per_ep_rank=[4131, 4138, 4037, 4042, 4002, 4157, 4105, 4156] avg=4096.0000 max=4157 max_rank=5 min=4002 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[988, 1005, 1050, 1018, 0, 0, 0, 0, 1028, 1077, 1008, 1007, 0, 0, 0, 0, 1055, 1055, 1006, 1049, 8, 0, 0, 0, 1018, 1042, 1031, 995, 0, 0, 0, 0, 1053, 1035, 986, 999, 36, 4, 0, 0, 1039, 983, 1043, 981, 8, 0, 0, 0, 992, 1031, 984, 999, 0, 0, 0, 0, 992, 1018, 1049, 1061, 17, 13, 5, 0] rerouted_tokens_per_ep_rank=[4061, 4120, 4173, 4086, 4113, 4054, 4006, 4155] avg=4096.0000 max=4173 max_rank=2 min=4006 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1047, 1038, 1021, 1012, 0, 0, 0, 0, 995, 1039, 1024, 1019, 0, 0, 0, 0, 1033, 1040, 1042, 1022, 3, 0, 0, 0, 1012, 1034, 996, 1043, 0, 0, 0, 0, 1003, 989, 994, 1022, 30, 1, 0, 0, 1025, 1010, 1034, 1065, 2, 0, 0, 0, 1019, 1047, 1004, 1020, 0, 0, 0, 0, 1002, 1050, 1021, 985, 15, 9, 1, 0] rerouted_tokens_per_ep_rank=[4118, 4077, 4140, 4085, 4039, 4136, 4090, 4083] avg=4096.0000 max=4140 max_rank=2 min=4039 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1030, 1011, 1052, 1063, 0, 0, 0, 0, 1033, 1022, 1009, 1003, 0, 0, 0, 0, 1023, 1007, 1022, 998, 3, 0, 0, 0, 1015, 1016, 1025, 966, 0, 0, 0, 0, 1060, 1029, 1054, 999, 31, 1, 0, 0, 1041, 997, 1045, 1012, 2, 0, 0, 0, 1004, 1022, 1069, 1049, 0, 0, 0, 0, 990, 1029, 1024, 988, 15, 8, 1, 0] rerouted_tokens_per_ep_rank=[4156, 4067, 4053, 4022, 4174, 4097, 4144, 4055] avg=4096.0000 max=4174 max_rank=4 min=4022 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1021, 1021, 1037, 1083, 0, 0, 0, 0, 1026, 1030, 1028, 1013, 0, 0, 0, 0, 1022, 1032, 1041, 983, 3, 0, 0, 0, 1046, 1029, 1020, 991, 0, 0, 0, 0, 980, 1005, 1039, 967, 31, 1, 0, 0, 1007, 1065, 1032, 1033, 2, 0, 0, 0, 1029, 1030, 1033, 1027, 0, 0, 0, 0, 1002, 971, 997, 1066, 15, 9, 1, 0] rerouted_tokens_per_ep_rank=[4162, 4097, 4081, 4086, 4023, 4139, 4119, 4061] avg=4096.0000 max=4162 max_rank=0 min=4023 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[990, 1003, 1030, 1009, 0, 0, 0, 0, 1040, 1039, 1053, 999, 0, 0, 0, 0, 973, 1006, 1058, 1005, 3, 0, 0, 0, 1030, 1039, 1045, 1045, 0, 0, 0, 0, 1029, 1000, 1044, 1055, 32, 1, 0, 0, 1042, 1070, 989, 940, 2, 0, 0, 0, 1017, 1035, 1053, 1024, 0, 0, 0, 0, 1011, 983, 1048, 1002, 14, 9, 1, 0] rerouted_tokens_per_ep_rank=[4032, 4131, 4045, 4159, 4161, 4043, 4129, 4068] avg=4096.0000 max=4161 max_rank=4 min=4032 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[950, 1040, 984, 1052, 991, 1083, 1042, 1036, 1063, 1055, 958, 1021, 1020, 1028, 1030, 1043, 969, 991, 1071, 1014, 1031, 997, 988, 1036, 1039, 1054, 994, 1026, 1034, 1059, 1014, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1063, 1032, 1007, 1025, 1084, 972, 1065, 1002, 1065, 1068, 1048, 1006, 1017, 1015, 992, 1010, 1024, 1002, 989, 1044, 1021, 1008, 981, 1028, 1003, 1012, 1060, 1077, 979, 1002, 1054, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1009, 1017, 987, 1000, 1031, 1044, 1010, 1044, 1048, 1055, 1040, 982, 1015, 1032, 1062, 1030, 1003, 1033, 1001, 1015, 970, 1019, 1050, 1002, 1082, 1026, 1017, 1056, 1055, 1042, 1004, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1062, 1041, 1020, 1066, 993, 1024, 1039, 1038, 968, 983, 1011, 1043, 1020, 999, 1005, 1049, 1022, 1006, 988, 1049, 1020, 989, 1029, 1009, 1069, 1034, 1025, 1003, 1004, 1050, 1049, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1058, 1076, 1049, 1044, 1065, 1002, 1005, 1021, 1029, 1045, 1011, 986, 1010, 1011, 983, 1035, 1048, 1012, 1009, 1081, 1020, 1023, 1055, 999, 1044, 990, 1031, 962, 1033, 969, 1068, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1051, 1036, 1002, 1005, 1003, 1035, 1040, 1043, 1052, 1050, 988, 1095, 1005, 1006, 980, 997, 1064, 1043, 1014, 1005, 992, 1080, 1046, 1026, 1034, 933, 1026, 1023, 1008, 1037, 1021, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[971, 993, 1025, 1008, 1017, 994, 1006, 1020, 990, 1029, 958, 1038, 1127, 1074, 1016, 1004, 1011, 1002, 1028, 1014, 1037, 1092, 1081, 1043, 989, 1041, 1045, 1019, 985, 1018, 1022, 1071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[995, 1016, 1029, 1043, 1029, 1026, 1041, 1030, 1042, 1064, 968, 1057, 989, 1010, 1029, 1015, 1046, 1029, 1012, 1065, 1062, 1003, 1047, 1027, 1035, 1020, 1002, 1035, 997, 998, 973, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1009, 1017, 987, 1000, 1, 0, 0, 0, 1031, 1044, 997, 1044, 0, 0, 0, 0, 1048, 1050, 1040, 982, 0, 0, 0, 0, 1015, 1032, 1062, 1030, 13, 0, 0, 0, 1003, 1033, 1001, 1015, 5, 2, 0, 0, 970, 1019, 1045, 1002, 0, 0, 0, 0, 1078, 1026, 1017, 1056, 0, 0, 0, 0, 1055, 1042, 1004, 987, 3, 3, 0, 0] rerouted_tokens_per_ep_rank=[4014, 4116, 4120, 4152, 4059, 4036, 4177, 4094] avg=4096.0000 max=4177 max_rank=6 min=4014 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[950, 1040, 984, 1052, 1, 0, 0, 0, 991, 1083, 1029, 1036, 0, 0, 0, 0, 1063, 1050, 958, 1021, 0, 0, 0, 0, 1020, 1028, 1030, 1043, 13, 0, 0, 0, 969, 991, 1071, 1014, 5, 2, 0, 0, 1031, 997, 984, 1036, 0, 0, 0, 0, 1035, 1054, 994, 1026, 0, 0, 0, 0, 1034, 1059, 1014, 1055, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4139, 4092, 4134, 4052, 4048, 4109, 4167] avg=4096.0000 max=4167 max_rank=7 min=4027 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1051, 1036, 1002, 1005, 1, 0, 0, 0, 1003, 1035, 1027, 1043, 0, 0, 0, 0, 1052, 1045, 988, 1095, 0, 0, 0, 0, 1005, 1006, 980, 997, 13, 0, 0, 0, 1064, 1043, 1014, 1005, 5, 2, 0, 0, 992, 1080, 1041, 1026, 0, 0, 0, 0, 1030, 933, 1026, 1023, 0, 0, 0, 0, 1008, 1037, 1021, 1028, 3, 3, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4108, 4180, 4001, 4133, 4139, 4012, 4100] avg=4096.0000 max=4180 max_rank=2 min=4001 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1063, 1032, 1007, 1025, 1, 0, 0, 0, 1084, 972, 1052, 1002, 0, 0, 0, 0, 1065, 1063, 1048, 1006, 0, 0, 0, 0, 1017, 1015, 992, 1010, 13, 0, 0, 0, 1024, 1002, 989, 1044, 5, 2, 0, 0, 1021, 1008, 977, 1028, 0, 0, 0, 0, 999, 1012, 1060, 1077, 0, 0, 0, 0, 979, 1002, 1054, 1013, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4110, 4182, 4047, 4066, 4034, 4148, 4053] avg=4096.0000 max=4182 max_rank=2 min=4034 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1062, 1041, 1020, 1066, 1, 0, 0, 0, 993, 1024, 1026, 1038, 0, 0, 0, 0, 968, 978, 1011, 1043, 0, 0, 0, 0, 1020, 999, 1005, 1049, 13, 0, 0, 0, 1022, 1006, 988, 1049, 5, 2, 0, 0, 1020, 989, 1025, 1009, 0, 0, 0, 0, 1065, 1034, 1025, 1003, 0, 0, 0, 0, 1004, 1050, 1049, 1061, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4081, 4000, 4086, 4072, 4043, 4127, 4169] avg=4096.0000 max=4190 max_rank=0 min=4000 min_rank=2 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1058, 1076, 1049, 1044, 1, 0, 0, 0, 1065, 1002, 992, 1021, 0, 0, 0, 0, 1029, 1040, 1011, 986, 0, 0, 0, 0, 1010, 1011, 983, 1035, 13, 0, 0, 0, 1048, 1012, 1009, 1081, 5, 2, 0, 0, 1020, 1023, 1050, 999, 0, 0, 0, 0, 1040, 990, 1031, 962, 0, 0, 0, 0, 1033, 969, 1068, 994, 3, 3, 0, 0] rerouted_tokens_per_ep_rank=[4228, 4080, 4066, 4052, 4157, 4092, 4023, 4070] avg=4096.0000 max=4228 max_rank=0 min=4023 min_rank=6 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[971, 993, 1025, 1008, 5, 0, 0, 0, 1017, 994, 990, 1020, 0, 0, 0, 0, 990, 1016, 958, 1038, 0, 0, 0, 0, 1127, 1074, 1016, 1004, 16, 3, 0, 0, 1011, 1002, 1028, 1014, 10, 5, 0, 0, 1037, 1092, 1070, 1043, 0, 0, 0, 0, 979, 1041, 1045, 1019, 0, 0, 0, 0, 985, 1018, 1022, 1071, 5, 6, 0, 0] rerouted_tokens_per_ep_rank=[4002, 4021, 4002, 4240, 4070, 4242, 4084, 4107] avg=4096.0000 max=4242 max_rank=5 min=4002 min_rank=0 imbalance_ratio=1.035645 max_load_reduction=28526 max_load_reduction_pct=0.870544 moved_to_echo=16503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[995, 1016, 1029, 1043, 1, 0, 0, 0, 1029, 1026, 1028, 1030, 0, 0, 0, 0, 1042, 1059, 968, 1057, 0, 0, 0, 0, 989, 1010, 1029, 1015, 13, 0, 0, 0, 1046, 1029, 1012, 1065, 5, 2, 0, 0, 1062, 1003, 1042, 1027, 0, 0, 0, 0, 1031, 1020, 1002, 1035, 0, 0, 0, 0, 997, 998, 973, 1034, 3, 3, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4113, 4126, 4056, 4159, 4134, 4088, 4008] avg=4096.0000 max=4159 max_rank=4 min=4008 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1043, 1040, 1041, 1071, 1008, 1058, 969, 1002, 1047, 1007, 1022, 979, 1032, 1057, 1039, 968, 1037, 1029, 1056, 997, 989, 1046, 1059, 997, 1025, 1009, 1023, 1012, 1030, 1029, 1011, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1053, 1040, 1031, 1014, 1027, 997, 1096, 1002, 1050, 1070, 999, 1010, 999, 1013, 1007, 1006, 1034, 1020, 1057, 992, 986, 1026, 1021, 1031, 1037, 1024, 1010, 1033, 1069, 1008, 1036, 970] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1059, 1068, 1087, 1049, 1077, 1066, 1026, 999, 1026, 975, 947, 971, 1035, 972, 1017, 1045, 1014, 1053, 1003, 1031, 1009, 1002, 1008, 1029, 1094, 996, 1016, 1016, 984, 1037, 1017, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1025, 1004, 1034, 1059, 1043, 1071, 1064, 1042, 1030, 1021, 944, 1051, 1011, 1019, 969, 1047, 1064, 940, 1043, 1017, 1034, 1036, 1039, 1055, 1024, 1064, 985, 1012, 1024, 995, 998, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1042, 1037, 1020, 1001, 980, 1018, 1022, 991, 1048, 1037, 1066, 1054, 1053, 987, 1059, 1067, 972, 964, 1020, 1006, 1071, 1077, 936, 995, 1043, 1008, 1023, 1051, 1032, 1045, 1051, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[997, 1017, 1013, 1044, 984, 967, 973, 1030, 1009, 1008, 1064, 1033, 1013, 1057, 1073, 1055, 1049, 1050, 1004, 960, 1027, 1058, 1009, 1054, 1023, 1014, 1034, 1060, 1023, 1005, 1042, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1025, 1011, 1043, 980, 1006, 1000, 1033, 1038, 1054, 1047, 1027, 999, 1026, 973, 1040, 1010, 1064, 1029, 964, 1016, 1017, 1061, 1015, 1026, 1024, 1023, 1033, 1001, 1057, 1016, 1082, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1047, 1000, 1034, 1008, 1006, 1042, 976, 1040, 1037, 993, 1026, 1013, 996, 1037, 999, 1006, 1064, 1027, 1023, 1043, 981, 1048, 1009, 1022, 1047, 1026, 1041, 1050, 988, 1051, 1079, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1059, 1068, 1053, 1049, 0, 0, 0, 0, 1077, 1066, 1026, 999, 15, 0, 0, 0, 1026, 975, 947, 971, 9, 3, 0, 0, 1035, 972, 1017, 1045, 4, 4, 0, 0, 1014, 1053, 1003, 1031, 16, 0, 0, 0, 1009, 1002, 1008, 1029, 0, 0, 0, 0, 1081, 996, 1016, 1016, 0, 0, 0, 0, 984, 1037, 1013, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4229, 4183, 3931, 4077, 4117, 4048, 4109, 4074] avg=4096.0000 max=4229 max_rank=0 min=3931 min_rank=2 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1053, 1040, 999, 1014, 0, 0, 0, 0, 1027, 997, 1096, 1002, 14, 0, 0, 0, 1050, 1070, 999, 1010, 9, 3, 0, 0, 999, 1013, 1007, 1006, 4, 4, 0, 0, 1034, 1020, 1057, 992, 15, 0, 0, 0, 986, 1026, 1021, 1031, 0, 0, 0, 0, 1024, 1024, 1010, 1033, 0, 0, 0, 0, 1069, 1008, 1032, 970, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4136, 4141, 4033, 4118, 4064, 4091, 4079] avg=4096.0000 max=4141 max_rank=2 min=4033 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1043, 1040, 1009, 1071, 0, 0, 0, 0, 1008, 1058, 969, 1002, 14, 0, 0, 0, 1047, 1007, 1022, 979, 9, 3, 0, 0, 1032, 1057, 1039, 968, 4, 4, 0, 0, 1037, 1029, 1056, 997, 15, 0, 0, 0, 989, 1046, 1059, 997, 0, 0, 0, 0, 1012, 1009, 1023, 1012, 0, 0, 0, 0, 1030, 1029, 1007, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4051, 4067, 4104, 4134, 4091, 4056, 4102] avg=4096.0000 max=4163 max_rank=0 min=4051 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1025, 1004, 1002, 1059, 0, 0, 0, 0, 1043, 1071, 1064, 1042, 14, 0, 0, 0, 1030, 1021, 944, 1051, 9, 3, 0, 0, 1011, 1019, 969, 1047, 4, 4, 0, 0, 1064, 940, 1043, 1017, 15, 0, 0, 0, 1034, 1036, 1039, 1055, 0, 0, 0, 0, 1011, 1064, 985, 1012, 0, 0, 0, 0, 1024, 995, 994, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4234, 4058, 4054, 4079, 4164, 4072, 4017] avg=4096.0000 max=4234 max_rank=1 min=4017 min_rank=7 imbalance_ratio=1.033691 max_load_reduction=28534 max_load_reduction_pct=0.870789 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1042, 1037, 988, 1001, 0, 0, 0, 0, 980, 1018, 1022, 991, 14, 0, 0, 0, 1048, 1037, 1066, 1054, 9, 3, 0, 0, 1053, 987, 1059, 1067, 4, 4, 0, 0, 972, 964, 1020, 1006, 15, 0, 0, 0, 1071, 1077, 936, 995, 0, 0, 0, 0, 1030, 1008, 1023, 1051, 0, 0, 0, 0, 1032, 1045, 1047, 992, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4025, 4217, 4174, 3977, 4079, 4112, 4116] avg=4096.0000 max=4217 max_rank=2 min=3977 min_rank=4 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[997, 1017, 981, 1044, 0, 0, 0, 0, 984, 967, 973, 1030, 14, 0, 0, 0, 1009, 1008, 1064, 1033, 9, 3, 0, 0, 1013, 1057, 1073, 1055, 4, 4, 0, 0, 1049, 1050, 1004, 960, 15, 0, 0, 0, 1027, 1058, 1009, 1054, 0, 0, 0, 0, 1010, 1014, 1034, 1060, 0, 0, 0, 0, 1023, 1005, 1038, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 3968, 4126, 4206, 4078, 4148, 4118, 4085] avg=4096.0000 max=4206 max_rank=3 min=3968 min_rank=1 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1025, 1011, 1000, 980, 0, 0, 0, 0, 1006, 1000, 1033, 1038, 16, 0, 0, 0, 1054, 1047, 1027, 999, 13, 7, 0, 0, 1026, 973, 1040, 1010, 10, 9, 5, 0, 1064, 1029, 964, 1016, 20, 0, 0, 0, 1017, 1056, 1015, 1026, 0, 0, 0, 0, 1002, 1023, 1033, 1001, 0, 0, 0, 0, 1057, 1016, 1072, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4093, 4147, 4073, 4093, 4114, 4059, 4173] avg=4096.0000 max=4173 max_rank=7 min=4016 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1047, 1000, 1002, 1008, 0, 0, 0, 0, 1006, 1042, 976, 1040, 14, 0, 0, 0, 1037, 993, 1026, 1013, 9, 3, 0, 0, 996, 1037, 999, 1006, 5, 4, 0, 0, 1064, 1027, 1023, 1043, 15, 0, 0, 0, 981, 1048, 1009, 1022, 0, 0, 0, 0, 1034, 1026, 1041, 1050, 0, 0, 0, 0, 988, 1051, 1074, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4078, 4081, 4047, 4172, 4060, 4151, 4122] avg=4096.0000 max=4172 max_rank=4 min=4047 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8448, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1002, 1043, 1001, 1063, 1083, 1038, 973, 1046, 976, 1021, 1012, 1010, 1042, 1015, 1079, 1027, 1007, 1044, 1026, 1007, 1025, 1028, 1022, 958, 1026, 991, 1043, 1023, 1074, 1022, 1032, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[988, 1033, 1026, 1000, 1019, 1026, 1070, 1010, 1072, 1018, 1052, 1022, 985, 1039, 1009, 973, 1021, 997, 1058, 1034, 972, 1023, 1064, 998, 1008, 1029, 1032, 1050, 1057, 1073, 990, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1016, 1019, 1001, 965, 1048, 1027, 1039, 1079, 1017, 1020, 1029, 997, 1033, 1047, 1033, 1039, 1044, 1001, 1020, 994, 1037, 1028, 1019, 999, 1015, 1050, 983, 1015, 1026, 1072, 1048, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1019, 1038, 1092, 1046, 1026, 975, 988, 1017, 1024, 1029, 1032, 1019, 1041, 1028, 1038, 1017, 975, 1027, 1041, 1017, 1032, 1027, 1052, 1002, 1037, 994, 1018, 1037, 1031, 1020, 970, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1004, 1010, 1022, 1001, 1010, 1055, 1030, 1040, 1008, 1042, 1077, 980, 1039, 1032, 1041, 1055, 1030, 1069, 1085, 1004, 996, 1002, 1022, 1023, 996, 996, 996, 1007, 1037, 1038, 1027, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[991, 1014, 992, 1051, 1020, 1040, 1047, 1026, 1047, 1060, 989, 1012, 1030, 1035, 1028, 1019, 1005, 1057, 991, 1004, 1050, 1095, 996, 1016, 1023, 978, 1001, 1021, 1044, 1013, 1044, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1009, 1004, 1018, 1036, 980, 1052, 1057, 1023, 1019, 1012, 1034, 1047, 1015, 1059, 988, 994, 1072, 1004, 1004, 1008, 1038, 994, 1060, 1029, 1007, 988, 1062, 1010, 1049, 1039, 1016, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1005, 1095, 1008, 979, 993, 1031, 943, 1001, 1024, 1058, 1024, 1042, 1023, 1019, 1060, 1027, 1050, 1032, 1049, 1036, 1052, 1019, 988, 1087, 1043, 1020, 1027, 1007, 1017, 1035, 998, 976] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1002, 1043, 1001, 1063, 7, 5, 5, 3, 1083, 1034, 973, 1046, 0, 0, 0, 0, 976, 1014, 1012, 1010, 0, 0, 0, 0, 1042, 1015, 1061, 1027, 0, 0, 0, 0, 1007, 1044, 1021, 1007, 0, 0, 0, 0, 1025, 1028, 1022, 958, 1, 0, 0, 0, 1026, 991, 1043, 1023, 18, 12, 0, 0, 1057, 1022, 1032, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4136, 4012, 4145, 4079, 4034, 4113, 4120] avg=4096.0000 max=4145 max_rank=3 min=4012 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1004, 1010, 1022, 1001, 7, 5, 5, 3, 1010, 1051, 1030, 1040, 0, 0, 0, 0, 1008, 1035, 1077, 980, 0, 0, 0, 0, 1039, 1032, 1024, 1055, 0, 0, 0, 0, 1030, 1069, 1080, 1004, 0, 0, 0, 0, 996, 1002, 1022, 1023, 1, 0, 0, 0, 996, 996, 996, 1007, 17, 11, 0, 0, 1021, 1038, 1027, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4131, 4100, 4150, 4183, 4044, 4023, 4080] avg=4096.0000 max=4183 max_rank=4 min=4023 min_rank=6 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[991, 1014, 992, 1051, 7, 5, 5, 3, 1020, 1036, 1047, 1026, 0, 0, 0, 0, 1047, 1053, 989, 1012, 0, 0, 0, 0, 1030, 1035, 1011, 1019, 0, 0, 0, 0, 1005, 1057, 986, 1004, 0, 0, 0, 0, 1050, 1095, 996, 1016, 1, 0, 0, 0, 1023, 978, 1001, 1021, 17, 11, 0, 0, 1028, 1013, 1044, 1029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4129, 4101, 4095, 4052, 4158, 4051, 4114] avg=4096.0000 max=4158 max_rank=5 min=4051 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[988, 1033, 1026, 1000, 7, 5, 5, 3, 1019, 1022, 1070, 1010, 0, 0, 0, 0, 1072, 1011, 1052, 1022, 0, 0, 0, 0, 985, 1039, 992, 973, 0, 0, 0, 0, 1021, 997, 1053, 1034, 0, 0, 0, 0, 972, 1023, 1064, 998, 1, 0, 0, 0, 1008, 1029, 1032, 1050, 17, 11, 0, 0, 1041, 1073, 990, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4121, 4157, 3989, 4105, 4058, 4147, 4124] avg=4096.0000 max=4157 max_rank=2 min=3989 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1016, 1019, 1001, 965, 7, 5, 5, 3, 1048, 1023, 1039, 1079, 0, 0, 0, 0, 1017, 1013, 1029, 997, 0, 0, 0, 0, 1033, 1047, 1016, 1039, 0, 0, 0, 0, 1044, 1001, 1015, 994, 0, 0, 0, 0, 1037, 1028, 1019, 999, 1, 0, 0, 0, 1015, 1050, 983, 1015, 17, 11, 0, 0, 1010, 1072, 1048, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 4189, 4056, 4135, 4054, 4084, 4091, 4138] avg=4096.0000 max=4189 max_rank=1 min=4021 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1019, 1038, 1092, 1046, 9, 11, 10, 8, 1026, 959, 988, 1017, 0, 0, 0, 0, 1024, 1020, 1032, 1019, 0, 0, 0, 0, 1041, 1028, 1017, 1017, 0, 0, 0, 0, 975, 1027, 1031, 1017, 0, 0, 0, 0, 1032, 1027, 1052, 1002, 8, 0, 0, 0, 1037, 994, 1018, 1037, 21, 16, 0, 0, 1004, 1020, 970, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4233, 3990, 4095, 4103, 4050, 4121, 4123, 4053] avg=4096.0000 max=4233 max_rank=0 min=3990 min_rank=1 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1009, 1004, 1018, 1036, 6, 5, 5, 3, 980, 1048, 1057, 1023, 0, 0, 0, 0, 1019, 1006, 1034, 1047, 0, 0, 0, 0, 1015, 1059, 972, 994, 0, 0, 0, 0, 1072, 1004, 999, 1008, 0, 0, 0, 0, 1038, 994, 1060, 1029, 1, 0, 0, 0, 1007, 988, 1062, 1010, 16, 11, 0, 0, 1033, 1039, 1016, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4108, 4106, 4040, 4083, 4122, 4094, 4129] avg=4096.0000 max=4129 max_rank=7 min=4040 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1005, 1095, 1008, 979, 7, 5, 5, 3, 993, 1027, 943, 1001, 0, 0, 0, 0, 1024, 1051, 1024, 1042, 0, 0, 0, 0, 1023, 1019, 1042, 1027, 0, 0, 0, 0, 1050, 1032, 1044, 1036, 0, 0, 0, 0, 1052, 1019, 988, 1087, 1, 0, 0, 0, 1043, 1020, 1027, 1007, 18, 11, 0, 0, 1001, 1035, 998, 976, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 3964, 4141, 4111, 4162, 4147, 4126, 4010] avg=4096.0000 max=4162 max_rank=4 min=3964 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1004, 1000, 1006, 1050, 1038, 1004, 1035, 1045, 1031, 1037, 1059, 1016, 1037, 983, 1052, 1011, 997, 1027, 1045, 1048, 1016, 1010, 1071, 1019, 1026, 1031, 998, 967, 1025, 973, 1055, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1062, 1002, 1053, 1051, 1028, 1032, 988, 974, 1005, 1014, 996, 1018, 1044, 1000, 994, 1023, 1044, 1048, 1073, 1061, 1028, 1043, 1008, 987, 1000, 1041, 991, 1014, 1040, 1061, 988, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1042, 998, 1057, 1038, 1044, 979, 1026, 1013, 969, 1012, 1013, 1005, 989, 1003, 1010, 1045, 1031, 992, 1020, 990, 1053, 1006, 1065, 1014, 1081, 1029, 1008, 1082, 1038, 1028, 1041, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[946, 1031, 1015, 1062, 1023, 1051, 1040, 996, 1014, 1062, 1011, 999, 1022, 1037, 1042, 978, 1060, 1041, 1029, 1061, 1048, 1037, 1019, 1017, 977, 1044, 969, 1061, 1037, 1043, 1004, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1041, 1046, 1033, 1022, 1010, 1042, 1020, 1062, 1017, 1072, 1003, 1037, 1039, 998, 1013, 1037, 993, 1017, 1037, 994, 1021, 981, 1013, 1053, 1023, 990, 1080, 1019, 1051, 986, 989, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1055, 1045, 1064, 994, 1010, 1021, 1061, 1035, 1030, 1027, 1028, 1033, 1007, 1011, 1011, 1037, 1019, 1048, 998, 970, 1008, 975, 1016, 1043, 1005, 997, 1017, 1068, 1007, 1058, 1031, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[960, 1057, 1027, 1019, 998, 1021, 1070, 1061, 984, 1046, 1044, 1022, 1026, 1035, 998, 1020, 1073, 1103, 1006, 927, 1024, 1056, 1010, 988, 1069, 1015, 1003, 977, 1005, 1065, 979, 1080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1030, 1033, 1044, 970, 1008, 1063, 1015, 970, 1011, 1036, 1078, 1050, 1051, 984, 982, 1051, 1027, 1052, 1025, 1011, 983, 1041, 1021, 1034, 976, 1002, 1086, 1037, 1033, 999, 1055, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1004, 1000, 996, 1050, 0, 0, 0, 0, 1038, 1004, 1034, 1045, 0, 0, 0, 0, 1031, 1036, 1059, 1016, 0, 0, 0, 0, 1037, 983, 1052, 1011, 16, 8, 0, 0, 997, 1016, 1045, 1048, 0, 0, 0, 0, 1016, 1010, 1071, 1019, 4, 1, 1, 0, 1026, 1031, 998, 967, 6, 3, 0, 0, 1025, 973, 1055, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4121, 4142, 4107, 4106, 4122, 4031, 4089] avg=4096.0000 max=4142 max_rank=2 min=4031 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1062, 1002, 1043, 1051, 0, 0, 0, 0, 1028, 1032, 987, 974, 0, 0, 0, 0, 1005, 1013, 996, 1018, 0, 0, 0, 0, 1044, 1000, 994, 1023, 16, 8, 0, 0, 1044, 1037, 1073, 1061, 0, 0, 0, 0, 1028, 1043, 1008, 987, 4, 1, 1, 0, 1000, 1041, 991, 1014, 6, 3, 0, 0, 1040, 1061, 988, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4021, 4032, 4085, 4215, 4072, 4055, 4130] avg=4096.0000 max=4215 max_rank=4 min=4021 min_rank=1 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[946, 1031, 1005, 1062, 0, 0, 0, 0, 1023, 1051, 1039, 996, 0, 0, 0, 0, 1014, 1061, 1011, 999, 0, 0, 0, 0, 1022, 1037, 1042, 978, 15, 8, 0, 0, 1060, 1030, 1029, 1061, 0, 0, 0, 0, 1048, 1037, 1019, 1017, 4, 1, 1, 0, 977, 1044, 969, 1061, 6, 3, 0, 0, 1037, 1043, 1004, 977, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4109, 4085, 4102, 4180, 4127, 4060, 4061] avg=4096.0000 max=4180 max_rank=4 min=4044 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1042, 998, 1046, 1038, 0, 0, 0, 0, 1044, 979, 1025, 1013, 0, 0, 0, 0, 969, 1011, 1013, 1005, 0, 0, 0, 0, 989, 1003, 1010, 1045, 16, 8, 0, 0, 1031, 981, 1020, 990, 0, 0, 0, 0, 1053, 1006, 1065, 1014, 4, 1, 1, 0, 1081, 1029, 1008, 1082, 7, 3, 0, 0, 1038, 1028, 1041, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4061, 3998, 4071, 4022, 4144, 4210, 4138] avg=4096.0000 max=4210 max_rank=6 min=3998 min_rank=2 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16514 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1041, 1046, 1023, 1022, 0, 0, 0, 0, 1010, 1042, 1019, 1062, 0, 0, 0, 0, 1017, 1071, 1003, 1037, 0, 0, 0, 0, 1039, 998, 1013, 1037, 15, 8, 0, 0, 993, 1006, 1037, 994, 0, 0, 0, 0, 1021, 981, 1013, 1053, 4, 1, 1, 0, 1023, 990, 1080, 1019, 6, 3, 0, 0, 1051, 986, 989, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4133, 4128, 4110, 4030, 4074, 4121, 4040] avg=4096.0000 max=4133 max_rank=1 min=4030 min_rank=4 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1055, 1045, 1053, 994, 0, 0, 0, 0, 1010, 1021, 1060, 1035, 0, 0, 0, 0, 1030, 1026, 1028, 1033, 0, 0, 0, 0, 1007, 1011, 1011, 1037, 16, 8, 0, 0, 1019, 1037, 998, 970, 0, 0, 0, 0, 1008, 975, 1016, 1043, 4, 1, 1, 0, 1005, 997, 1017, 1068, 7, 3, 0, 0, 1007, 1058, 1031, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4126, 4117, 4090, 4024, 4048, 4097, 4119] avg=4096.0000 max=4147 max_rank=0 min=4024 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16288 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[960, 1057, 1010, 1019, 0, 0, 0, 0, 998, 1021, 1062, 1061, 0, 0, 0, 0, 984, 1042, 1044, 1022, 0, 0, 0, 0, 1026, 1035, 998, 1020, 20, 13, 0, 0, 1073, 1081, 1006, 927, 0, 0, 0, 0, 1024, 1056, 1010, 988, 6, 8, 4, 0, 1069, 1015, 1003, 977, 11, 9, 0, 0, 1005, 1065, 979, 1060, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4142, 4092, 4112, 4087, 4096, 4084, 4109] avg=4096.0000 max=4142 max_rank=1 min=4046 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1030, 1033, 1034, 970, 0, 0, 0, 0, 1008, 1063, 1014, 970, 0, 0, 0, 0, 1011, 1035, 1078, 1050, 0, 0, 0, 0, 1051, 984, 982, 1051, 15, 8, 0, 0, 1027, 1041, 1025, 1011, 0, 0, 0, 0, 983, 1041, 1021, 1034, 4, 1, 1, 0, 976, 1002, 1086, 1037, 6, 3, 0, 0, 1033, 999, 1055, 995, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4055, 4174, 4091, 4104, 4085, 4110, 4082] avg=4096.0000 max=4174 max_rank=2 min=4055 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[993, 1013, 975, 1005, 1022, 1001, 1062, 1000, 1066, 951, 1066, 988, 1078, 1018, 1035, 1046, 1024, 1032, 1022, 1041, 1051, 1030, 1019, 1024, 993, 1061, 1030, 1036, 982, 1051, 1010, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1047, 1009, 1015, 1013, 941, 1014, 1021, 1001, 1013, 1013, 1028, 1067, 1005, 1053, 1019, 1035, 1030, 1014, 994, 1023, 994, 1051, 1026, 1024, 1076, 1050, 1058, 1035, 1028, 1052, 995, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1068, 1038, 1007, 1005, 1041, 990, 1066, 996, 1025, 1014, 1035, 1053, 988, 1017, 1016, 1038, 1037, 1020, 1067, 1012, 1011, 1007, 1000, 1020, 1012, 959, 1043, 1004, 1021, 1083, 1048, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1024, 1065, 1027, 1057, 1047, 1016, 968, 1047, 1006, 1054, 1036, 1043, 1003, 1028, 982, 1027, 994, 1002, 1040, 1044, 987, 1003, 1049, 1028, 1039, 1051, 1010, 1000, 1006, 1042, 1005, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1025, 981, 982, 1008, 1058, 980, 1001, 998, 1007, 1043, 1043, 1046, 1059, 1045, 1010, 975, 1021, 1067, 988, 1030, 1028, 1024, 1003, 1000, 1037, 1024, 1024, 1016, 1058, 1032, 1088, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1010, 1027, 1013, 1015, 1028, 1037, 1035, 1067, 1017, 975, 1040, 986, 996, 1048, 1043, 1011, 1048, 1047, 1008, 1019, 1047, 1041, 1013, 1076, 993, 1076, 1006, 950, 1063, 988, 999, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1021, 1016, 1033, 985, 1042, 1015, 1040, 1046, 1043, 1038, 1048, 1019, 993, 1013, 1043, 1041, 969, 1048, 1024, 1023, 1026, 1022, 1019, 1029, 1001, 1003, 1010, 1039, 1015, 1041, 1047, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1002, 950, 1053, 1044, 1055, 993, 996, 1014, 1040, 1019, 1011, 1015, 1069, 974, 1013, 1046, 1040, 1001, 1064, 1045, 1052, 1021, 999, 1080, 1050, 1053, 1001, 1020, 1022, 1026, 1007, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1047, 1009, 1015, 1013, 24, 5, 0, 0, 941, 1014, 1021, 1001, 8, 4, 3, 0, 1013, 1013, 1019, 1067, 0, 0, 0, 0, 1005, 1053, 1019, 1035, 0, 0, 0, 0, 1030, 1014, 994, 1015, 0, 0, 0, 0, 994, 1051, 1026, 1021, 0, 0, 0, 0, 1076, 1050, 1058, 1035, 0, 0, 0, 0, 1028, 1028, 995, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 3992, 4112, 4112, 4053, 4092, 4219, 4075] avg=4096.0000 max=4219 max_rank=6 min=3992 min_rank=1 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[993, 1013, 975, 1005, 24, 6, 0, 0, 1022, 1001, 1062, 1000, 8, 4, 3, 0, 1066, 951, 1056, 988, 0, 0, 0, 0, 1078, 1018, 1035, 1046, 0, 0, 0, 0, 1024, 1032, 1022, 1033, 0, 0, 0, 0, 1051, 1030, 1019, 1021, 0, 0, 0, 0, 993, 1061, 1030, 1036, 0, 0, 0, 0, 982, 1027, 1010, 1043, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4100, 4061, 4177, 4111, 4121, 4120, 4062] avg=4096.0000 max=4177 max_rank=3 min=4016 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1068, 1038, 1007, 1005, 25, 5, 0, 0, 1041, 990, 1066, 996, 8, 4, 3, 0, 1025, 1014, 1026, 1053, 0, 0, 0, 0, 988, 1017, 1016, 1038, 0, 0, 0, 0, 1037, 1020, 1067, 1004, 0, 0, 0, 0, 1011, 1007, 1000, 1017, 0, 0, 0, 0, 1012, 959, 1043, 1004, 0, 0, 0, 0, 1021, 1058, 1048, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4108, 4118, 4059, 4128, 4035, 4018, 4154] avg=4096.0000 max=4154 max_rank=7 min=4018 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1021, 1016, 1033, 985, 24, 5, 0, 0, 1042, 1015, 1040, 1046, 8, 4, 3, 0, 1043, 1038, 1039, 1019, 0, 0, 0, 0, 993, 1013, 1043, 1041, 0, 0, 0, 0, 969, 1048, 1024, 1015, 0, 0, 0, 0, 1026, 1022, 1019, 1026, 0, 0, 0, 0, 1001, 1003, 1010, 1039, 0, 0, 0, 0, 1015, 1017, 1047, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4158, 4139, 4090, 4056, 4093, 4053, 4095] avg=4096.0000 max=4158 max_rank=1 min=4053 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16297 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 981, 982, 1008, 24, 5, 0, 0, 1058, 980, 1001, 998, 8, 4, 3, 0, 1007, 1043, 1034, 1046, 0, 0, 0, 0, 1059, 1045, 1010, 975, 0, 0, 0, 0, 1021, 1067, 988, 1022, 0, 0, 0, 0, 1028, 1024, 1003, 997, 0, 0, 0, 0, 1037, 1024, 1024, 1016, 0, 0, 0, 0, 1058, 1008, 1088, 1067, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4052, 4130, 4089, 4098, 4052, 4101, 4221] avg=4096.0000 max=4221 max_rank=7 min=4025 min_rank=0 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1024, 1065, 1027, 1057, 27, 11, 0, 0, 1047, 1016, 968, 1047, 14, 5, 6, 0, 1006, 1054, 1020, 1043, 0, 0, 0, 0, 1003, 1028, 982, 1027, 1, 0, 0, 0, 994, 1002, 1040, 1030, 0, 0, 0, 0, 987, 1003, 1049, 1015, 0, 0, 0, 0, 1039, 1051, 1010, 1000, 6, 0, 0, 0, 1006, 1015, 1005, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4211, 4103, 4123, 4041, 4066, 4054, 4106, 4064] avg=4096.0000 max=4211 max_rank=0 min=4041 min_rank=3 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16290 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1010, 1027, 1013, 1015, 23, 5, 0, 0, 1028, 1037, 1035, 1067, 8, 4, 3, 0, 1017, 975, 1031, 986, 0, 0, 0, 0, 996, 1048, 1043, 1011, 0, 0, 0, 0, 1048, 1047, 1008, 1011, 0, 0, 0, 0, 1047, 1041, 1013, 1072, 0, 0, 0, 0, 993, 1076, 1006, 950, 1, 0, 0, 0, 1063, 965, 999, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4182, 4009, 4098, 4114, 4173, 4026, 4073] avg=4096.0000 max=4182 max_rank=1 min=4009 min_rank=2 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1002, 950, 1053, 1044, 24, 5, 0, 0, 1055, 993, 996, 1014, 8, 4, 3, 0, 1040, 1019, 1002, 1015, 0, 0, 0, 0, 1069, 974, 1013, 1046, 0, 0, 0, 0, 1040, 1001, 1064, 1037, 0, 0, 0, 0, 1052, 1021, 999, 1076, 0, 0, 0, 0, 1050, 1053, 1001, 1020, 1, 0, 0, 0, 1022, 1002, 1007, 993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4073, 4076, 4102, 4142, 4148, 4125, 4024] avg=4096.0000 max=4148 max_rank=5 min=4024 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1033, 1027, 1042, 1028, 1075, 1029, 1024, 1014, 1014, 1022, 1023, 1033, 989, 1047, 1016, 1049, 998, 1018, 1013, 1057, 1065, 1007, 947, 1002, 1046, 1046, 987, 1047, 1029, 1025, 1015, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1042, 979, 1022, 966, 1024, 1024, 994, 1080, 1015, 988, 1015, 1005, 1049, 1033, 1030, 1008, 971, 1085, 1019, 1010, 1018, 1012, 1037, 1026, 1050, 1013, 1053, 1067, 1048, 1030, 1021, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1069, 991, 1093, 999, 1023, 991, 1015, 1019, 1003, 1037, 1017, 1044, 1021, 1015, 1004, 1047, 1013, 1030, 1035, 1032, 1007, 1000, 1079, 1010, 1035, 1027, 1036, 995, 1029, 1031, 985, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1041, 1016, 1028, 1041, 1003, 1040, 1041, 1006, 1038, 1032, 998, 1013, 1017, 1063, 1048, 1022, 1001, 1012, 1047, 1053, 1008, 994, 1035, 990, 1034, 1073, 986, 1047, 990, 999, 1047, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1046, 1066, 1052, 1026, 1015, 1008, 1083, 984, 1037, 1094, 1010, 1025, 978, 1031, 1021, 1071, 1004, 984, 1043, 1036, 966, 1033, 1041, 990, 1018, 1050, 1033, 1010, 992, 1008, 1042, 971] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1035, 1052, 1049, 999, 1041, 1078, 1027, 1057, 1024, 977, 982, 1041, 1028, 1038, 1035, 1063, 975, 1028, 1004, 1005, 1032, 1038, 1046, 1001, 1039, 1003, 997, 997, 1016, 992, 995, 1074] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1034, 992, 1024, 1011, 1038, 979, 1041, 959, 1029, 1055, 1003, 1035, 1075, 1028, 1059, 1063, 1057, 964, 1044, 1065, 990, 1037, 989, 1016, 1023, 1046, 1071, 979, 1007, 1017, 1021, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1005, 1051, 1044, 1031, 984, 972, 1013, 991, 1049, 1073, 975, 1058, 1016, 1014, 990, 1033, 1035, 1048, 1031, 1053, 1041, 1006, 1010, 1087, 1016, 1018, 1031, 1013, 1011, 965, 1075, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1042, 979, 1002, 966, 0, 0, 0, 0, 1024, 1024, 994, 1080, 11, 0, 0, 0, 1015, 988, 1015, 1005, 0, 0, 0, 0, 1049, 1033, 1030, 981, 0, 0, 0, 0, 971, 1085, 1019, 1010, 0, 0, 0, 0, 1018, 1012, 1037, 1026, 20, 2, 2, 0, 1050, 1000, 1053, 1067, 0, 0, 0, 0, 1048, 1030, 1021, 1034, 25, 0, 0, 0] rerouted_tokens_per_ep_rank=[3989, 4133, 4023, 4093, 4085, 4117, 4170, 4158] avg=4096.0000 max=4170 max_rank=6 min=3989 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16530 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1069, 991, 1072, 999, 0, 0, 0, 0, 1023, 991, 1015, 1019, 11, 0, 0, 0, 1003, 1037, 1017, 1044, 0, 0, 0, 0, 1021, 1015, 1004, 1019, 0, 0, 0, 0, 1013, 1030, 1035, 1032, 0, 0, 0, 0, 1007, 1000, 1079, 1010, 21, 2, 2, 0, 1035, 1014, 1036, 995, 0, 0, 0, 0, 1029, 1031, 985, 1036, 26, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4059, 4101, 4059, 4110, 4121, 4080, 4107] avg=4096.0000 max=4131 max_rank=0 min=4059 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1033, 1027, 1022, 1028, 0, 0, 0, 0, 1075, 1029, 1024, 1014, 12, 0, 0, 0, 1014, 1022, 1023, 1033, 0, 0, 0, 0, 989, 1047, 1016, 1021, 0, 0, 0, 0, 998, 1018, 1013, 1057, 0, 0, 0, 0, 1065, 1007, 947, 1002, 20, 2, 2, 0, 1046, 1032, 987, 1047, 0, 0, 0, 0, 1029, 1025, 1015, 1001, 26, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4154, 4092, 4073, 4086, 4045, 4112, 4096] avg=4096.0000 max=4154 max_rank=1 min=4045 min_rank=5 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1046, 1066, 1032, 1026, 0, 0, 0, 0, 1015, 1008, 1083, 984, 12, 0, 0, 0, 1037, 1094, 1010, 1025, 0, 0, 0, 0, 978, 1031, 1021, 1042, 0, 0, 0, 0, 1004, 984, 1043, 1036, 0, 0, 0, 0, 966, 1033, 1041, 990, 20, 2, 2, 0, 1018, 1036, 1033, 1010, 0, 0, 0, 0, 992, 1008, 1042, 971, 27, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4102, 4166, 4072, 4067, 4054, 4097, 4040] avg=4096.0000 max=4170 max_rank=0 min=4040 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16258 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1041, 1016, 1008, 1041, 0, 0, 0, 0, 1003, 1040, 1041, 1006, 12, 0, 0, 0, 1038, 1032, 998, 1013, 0, 0, 0, 0, 1017, 1063, 1048, 995, 0, 0, 0, 0, 1001, 1012, 1047, 1053, 0, 0, 0, 0, 1008, 994, 1035, 990, 20, 2, 2, 0, 1034, 1059, 986, 1047, 0, 0, 0, 0, 990, 999, 1047, 1005, 25, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4102, 4081, 4123, 4113, 4051, 4126, 4066] avg=4096.0000 max=4126 max_rank=6 min=4051 min_rank=5 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1035, 1052, 1024, 999, 0, 0, 0, 0, 1041, 1078, 1027, 1057, 15, 0, 0, 0, 1024, 977, 982, 1041, 2, 2, 0, 0, 1028, 1038, 1035, 1025, 0, 0, 0, 0, 975, 1028, 1004, 1003, 0, 0, 0, 0, 1032, 1038, 1046, 1001, 25, 8, 6, 0, 1039, 980, 997, 997, 0, 0, 0, 0, 1016, 992, 995, 1074, 30, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4218, 4028, 4126, 4010, 4156, 4013, 4107] avg=4096.0000 max=4218 max_rank=1 min=4010 min_rank=4 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1034, 992, 1004, 1011, 0, 0, 0, 0, 1038, 979, 1041, 959, 12, 0, 0, 0, 1029, 1055, 1003, 1035, 0, 0, 0, 0, 1075, 1028, 1059, 1035, 0, 0, 0, 0, 1057, 964, 1044, 1065, 0, 0, 0, 0, 990, 1037, 989, 1016, 20, 2, 2, 0, 1023, 1032, 1071, 979, 0, 0, 0, 0, 1007, 1017, 1021, 1017, 26, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4029, 4122, 4197, 4130, 4056, 4105, 4088] avg=4096.0000 max=4197 max_rank=3 min=4029 min_rank=1 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1005, 1051, 1024, 1031, 0, 0, 0, 0, 984, 972, 1013, 991, 11, 0, 0, 0, 1049, 1073, 975, 1058, 0, 0, 0, 0, 1016, 1014, 990, 1005, 0, 0, 0, 0, 1035, 1048, 1031, 1053, 0, 0, 0, 0, 1041, 1006, 1010, 1087, 20, 2, 2, 0, 1016, 1005, 1031, 1013, 0, 0, 0, 0, 1011, 965, 1075, 1029, 26, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 3971, 4155, 4025, 4167, 4168, 4065, 4106] avg=4096.0000 max=4168 max_rank=5 min=3971 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1010, 1023, 1038, 966, 983, 1069, 1038, 1072, 1053, 963, 1018, 1000, 1027, 1035, 1001, 1001, 1043, 1026, 1009, 1040, 1070, 1019, 1074, 1016, 1041, 1043, 1032, 1009, 1017, 1010, 1004, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1009, 996, 1009, 1031, 1047, 1010, 1057, 1016, 1065, 991, 1024, 995, 994, 987, 1020, 986, 946, 1020, 1098, 1031, 1032, 1072, 1043, 979, 1028, 968, 1068, 1049, 1075, 1071, 1001, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1036, 1018, 971, 1008, 1019, 995, 1008, 1024, 1078, 1002, 1067, 1027, 1012, 1059, 1028, 1032, 1049, 1050, 1059, 1031, 1000, 1037, 985, 998, 1024, 1017, 1010, 1017, 1062, 981, 1016, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[992, 1025, 1030, 1001, 979, 978, 1047, 1036, 1067, 1020, 1024, 1005, 1021, 1032, 1025, 1055, 999, 1032, 1024, 1092, 1010, 1036, 1034, 1050, 1046, 998, 1011, 995, 990, 1068, 1011, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1021, 1065, 1045, 987, 1000, 1032, 995, 1037, 1054, 984, 1072, 1018, 1010, 1050, 980, 1025, 1020, 1001, 1004, 1024, 1020, 1041, 1041, 1042, 1016, 1025, 988, 995, 1028, 1047, 1066, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1003, 1007, 1055, 1079, 1025, 1068, 982, 1005, 965, 1034, 983, 1047, 1054, 1001, 992, 1045, 994, 1017, 1054, 996, 1003, 1005, 1063, 1015, 1001, 1044, 1048, 1089, 991, 1036, 1012, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1009, 1021, 991, 1011, 1010, 983, 1048, 1018, 1048, 992, 1075, 995, 997, 979, 1064, 1036, 1039, 1011, 1062, 1006, 1053, 1051, 1045, 1028, 996, 1046, 1039, 1054, 1013, 1035, 1017, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1053, 1020, 1069, 1020, 1062, 1024, 1032, 1053, 1052, 1035, 971, 1023, 1013, 1030, 1021, 1026, 1046, 1010, 967, 1006, 1030, 1055, 1029, 1014, 1019, 1002, 982, 1007, 1019, 993, 1044, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1010, 1023, 1038, 966, 18, 0, 0, 0, 983, 1069, 1038, 1072, 1, 0, 0, 0, 1053, 963, 1018, 1000, 2, 0, 0, 0, 1027, 1035, 1001, 1001, 8, 7, 0, 0, 1043, 1026, 1006, 1040, 0, 0, 0, 0, 1070, 993, 1074, 1016, 0, 0, 0, 0, 1041, 1043, 1032, 1009, 7, 0, 0, 0, 1017, 1010, 1004, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4163, 4036, 4079, 4115, 4153, 4132, 4035] avg=4096.0000 max=4163 max_rank=1 min=4035 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1036, 1018, 971, 1008, 18, 0, 0, 0, 1019, 995, 1008, 1024, 2, 0, 0, 0, 1078, 1002, 1067, 1027, 2, 0, 0, 0, 1012, 1059, 1028, 1032, 9, 7, 0, 0, 1049, 1050, 1055, 1031, 0, 0, 0, 0, 1000, 1010, 985, 998, 0, 0, 0, 0, 1024, 1017, 1010, 1017, 7, 0, 0, 0, 1062, 981, 1016, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4048, 4176, 4147, 4185, 3993, 4075, 4093] avg=4096.0000 max=4185 max_rank=4 min=3993 min_rank=5 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[992, 1025, 1030, 1001, 18, 0, 0, 0, 979, 978, 1047, 1036, 1, 0, 0, 0, 1067, 1020, 1024, 1005, 2, 0, 0, 0, 1021, 1032, 1025, 1055, 9, 7, 0, 0, 999, 1032, 1021, 1092, 0, 0, 0, 0, 1010, 1009, 1034, 1050, 0, 0, 0, 0, 1046, 998, 1011, 995, 7, 0, 0, 0, 990, 1068, 1011, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4041, 4118, 4149, 4144, 4103, 4057, 4090] avg=4096.0000 max=4149 max_rank=3 min=4041 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1053, 1020, 1069, 1020, 18, 0, 0, 0, 1062, 1024, 1032, 1053, 1, 0, 0, 0, 1052, 1035, 971, 1023, 2, 0, 0, 0, 1013, 1030, 1021, 1026, 9, 7, 0, 0, 1046, 1010, 964, 1006, 0, 0, 0, 0, 1030, 1028, 1029, 1014, 0, 0, 0, 0, 1019, 1002, 982, 1007, 7, 0, 0, 0, 1019, 993, 1044, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4172, 4083, 4106, 4026, 4101, 4017, 4083] avg=4096.0000 max=4180 max_rank=0 min=4017 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16227 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1003, 1007, 1055, 1079, 18, 0, 0, 0, 1025, 1068, 982, 1005, 2, 0, 0, 0, 965, 1034, 983, 1047, 2, 0, 0, 0, 1054, 1001, 992, 1045, 8, 7, 0, 0, 994, 1017, 1050, 996, 0, 0, 0, 0, 1003, 979, 1063, 1015, 0, 0, 0, 0, 1001, 1044, 1048, 1089, 7, 0, 0, 0, 991, 1036, 1012, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4082, 4031, 4107, 4057, 4060, 4189, 4080] avg=4096.0000 max=4189 max_rank=6 min=4031 min_rank=2 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1009, 1021, 991, 1011, 18, 0, 0, 0, 1010, 983, 1048, 1018, 2, 0, 0, 0, 1048, 992, 1075, 995, 2, 0, 0, 0, 997, 979, 1064, 1036, 9, 6, 0, 0, 1039, 1011, 1058, 1006, 0, 0, 0, 0, 1053, 1024, 1045, 1028, 0, 0, 0, 0, 996, 1046, 1039, 1054, 7, 0, 0, 0, 1013, 1035, 1017, 983, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4061, 4112, 4091, 4114, 4150, 4142, 4048] avg=4096.0000 max=4150 max_rank=5 min=4048 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1021, 1065, 1045, 987, 18, 0, 0, 0, 1000, 1032, 995, 1037, 1, 0, 0, 0, 1054, 984, 1072, 1018, 2, 0, 0, 0, 1010, 1050, 980, 1025, 9, 7, 0, 0, 1020, 1001, 1001, 1024, 0, 0, 0, 0, 1020, 1014, 1041, 1042, 0, 0, 0, 0, 1016, 1025, 988, 995, 7, 0, 0, 0, 1028, 1047, 1066, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4065, 4130, 4081, 4046, 4117, 4031, 4162] avg=4096.0000 max=4162 max_rank=7 min=4031 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1009, 996, 1009, 1031, 23, 0, 0, 0, 1047, 1010, 1057, 1016, 6, 0, 0, 0, 1065, 991, 1024, 995, 7, 0, 0, 0, 994, 987, 1020, 986, 12, 9, 0, 0, 946, 1020, 1084, 1031, 0, 0, 0, 0, 1032, 1037, 1043, 979, 0, 0, 0, 0, 1028, 968, 1068, 1049, 11, 1, 0, 0, 1075, 1071, 1001, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4136, 4082, 4008, 4081, 4091, 4125, 4177] avg=4096.0000 max=4177 max_rank=7 min=4008 min_rank=3 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8064, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1022, 1011, 991, 1015, 1042, 1042, 1029, 1024, 1027, 989, 1063, 1030, 1052, 1036, 1025, 1024, 1029, 1018, 1004, 994, 1015, 981, 1035, 1022, 1027, 1045, 1048, 1023, 1032, 1002, 1029, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1006, 1027, 1055, 1032, 988, 1006, 984, 1015, 1007, 1025, 1037, 1029, 1032, 1088, 1071, 998, 1045, 983, 1002, 1075, 1026, 1005, 963, 1070, 1031, 998, 1021, 1003, 981, 1033, 1074, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1003, 985, 1036, 1026, 1084, 1006, 1077, 1072, 983, 1031, 1022, 1014, 1018, 1002, 1021, 1034, 1005, 998, 999, 1005, 1065, 1007, 1069, 1050, 1013, 998, 1010, 1061, 1055, 967, 1073, 979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1009, 1032, 1026, 1023, 996, 1049, 1035, 1010, 1014, 1060, 1009, 1032, 1008, 1006, 1004, 1048, 1068, 983, 1016, 1020, 1011, 1047, 1056, 1023, 987, 1052, 1056, 1033, 1031, 999, 1019, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[978, 1044, 1015, 1041, 1040, 1046, 987, 1021, 956, 1088, 1037, 1048, 988, 1036, 1041, 1036, 1019, 1037, 1080, 981, 1023, 975, 1052, 1033, 981, 1029, 1029, 983, 1036, 1010, 1044, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1018, 1028, 979, 1016, 1013, 1037, 1030, 1060, 1080, 1003, 1006, 1037, 1062, 1032, 1079, 1002, 1039, 1020, 963, 997, 1011, 993, 1010, 1006, 1088, 1041, 997, 1005, 1040, 1063, 996, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1069, 977, 975, 1022, 987, 1045, 1034, 1009, 1036, 1067, 1041, 1035, 1034, 983, 1062, 1001, 1007, 961, 1024, 1018, 999, 1045, 1057, 1007, 966, 1018, 1066, 1061, 1064, 1046, 1018, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1054, 1050, 967, 1033, 1054, 1070, 944, 983, 992, 1045, 998, 1004, 1026, 1055, 1048, 1023, 1027, 993, 1048, 1030, 1035, 991, 1076, 1043, 1007, 1061, 956, 1030, 1052, 1053, 1022, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1022, 1011, 991, 1015, 10, 9, 5, 0, 1042, 1036, 1029, 1024, 0, 0, 0, 0, 1027, 980, 1063, 1030, 0, 0, 0, 0, 1052, 1036, 1000, 1024, 0, 0, 0, 0, 1029, 1018, 1004, 994, 25, 9, 0, 0, 1015, 981, 1031, 1022, 0, 0, 0, 0, 1027, 1045, 1048, 1023, 4, 1, 0, 0, 1013, 1002, 1029, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4131, 4100, 4112, 4079, 4049, 4148, 4086] avg=4096.0000 max=4148 max_rank=6 min=4049 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1006, 1027, 1055, 1032, 10, 9, 4, 0, 988, 1001, 984, 1015, 0, 0, 0, 0, 1007, 1016, 1037, 1029, 0, 0, 0, 0, 1032, 1088, 1045, 998, 0, 0, 0, 0, 1045, 983, 1002, 1075, 26, 8, 0, 0, 1026, 1005, 960, 1070, 0, 0, 0, 0, 1031, 998, 1021, 1003, 3, 1, 0, 0, 963, 1033, 1074, 1058, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 3988, 4089, 4163, 4139, 4061, 4057, 4128] avg=4096.0000 max=4163 max_rank=3 min=3988 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1054, 1050, 967, 1033, 10, 9, 5, 0, 1054, 1064, 944, 983, 0, 0, 0, 0, 992, 1036, 998, 1004, 0, 0, 0, 0, 1026, 1055, 1023, 1023, 0, 0, 0, 0, 1027, 993, 1048, 1030, 25, 9, 0, 0, 1035, 991, 1072, 1043, 0, 0, 0, 0, 1007, 1061, 956, 1030, 4, 1, 0, 0, 1033, 1053, 1022, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4045, 4030, 4127, 4132, 4141, 4059, 4106] avg=4096.0000 max=4141 max_rank=5 min=4030 min_rank=2 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1003, 985, 1036, 1026, 10, 9, 4, 0, 1084, 1001, 1077, 1072, 0, 0, 0, 0, 983, 1022, 1022, 1014, 0, 0, 0, 0, 1018, 1002, 996, 1034, 0, 0, 0, 0, 1005, 998, 999, 1005, 25, 9, 0, 0, 1065, 1007, 1065, 1050, 0, 0, 0, 0, 1013, 998, 1010, 1061, 4, 1, 0, 0, 1036, 967, 1073, 979, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4234, 4041, 4050, 4041, 4187, 4087, 4055] avg=4096.0000 max=4234 max_rank=1 min=4041 min_rank=2 imbalance_ratio=1.033691 max_load_reduction=28534 max_load_reduction_pct=0.870789 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1009, 1032, 1026, 1023, 10, 9, 5, 0, 996, 1043, 1035, 1010, 0, 0, 0, 0, 1014, 1051, 1009, 1032, 0, 0, 0, 0, 1008, 1006, 980, 1048, 0, 0, 0, 0, 1068, 983, 1016, 1020, 24, 9, 0, 0, 1011, 1047, 1052, 1023, 0, 0, 0, 0, 987, 1052, 1056, 1033, 4, 1, 0, 0, 1012, 999, 1019, 1006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4084, 4106, 4042, 4120, 4133, 4133, 4036] avg=4096.0000 max=4133 max_rank=5 min=4036 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[978, 1044, 1015, 1041, 10, 10, 5, 0, 1040, 1040, 987, 1021, 0, 0, 0, 0, 956, 1078, 1037, 1048, 0, 0, 0, 0, 988, 1036, 1016, 1036, 0, 0, 0, 0, 1019, 1037, 1080, 981, 25, 9, 0, 0, 1023, 975, 1048, 1033, 0, 0, 0, 0, 981, 1029, 1029, 983, 4, 1, 0, 0, 1017, 1010, 1044, 1054, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4088, 4119, 4076, 4151, 4079, 4027, 4125] avg=4096.0000 max=4151 max_rank=4 min=4027 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1018, 1028, 979, 1016, 10, 9, 4, 0, 1013, 1032, 1030, 1060, 0, 0, 0, 0, 1080, 994, 1006, 1037, 0, 0, 0, 0, 1062, 1032, 1053, 1002, 0, 0, 0, 0, 1039, 1020, 963, 997, 26, 9, 0, 0, 1011, 993, 1006, 1006, 0, 0, 0, 0, 1088, 1041, 997, 1005, 4, 1, 0, 0, 1021, 1063, 996, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4135, 4117, 4149, 4054, 4016, 4136, 4097] avg=4096.0000 max=4149 max_rank=3 min=4016 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1069, 977, 975, 1022, 16, 13, 8, 0, 987, 1033, 1034, 1009, 0, 0, 0, 0, 1036, 1054, 1041, 1035, 0, 0, 0, 0, 1034, 983, 1031, 1001, 0, 0, 0, 0, 1007, 961, 1024, 1018, 31, 11, 0, 0, 999, 1045, 1051, 1007, 0, 0, 0, 0, 966, 1018, 1066, 1061, 6, 4, 0, 0, 1037, 1046, 1018, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4063, 4166, 4049, 4052, 4102, 4121, 4135] avg=4096.0000 max=4166 max_rank=2 min=4049 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8064, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1039, 1019, 1052, 1024, 1024, 1051, 1055, 1018, 1012, 1055, 1013, 1022, 1045, 1004, 997, 1045, 1048, 1059, 1040, 1012, 995, 1028, 1024, 1028, 1032, 1033, 1018, 988, 954, 1012, 964, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1024, 1015, 1088, 1029, 1030, 1049, 1054, 1019, 999, 988, 1003, 1057, 1032, 1009, 1005, 1055, 988, 989, 1029, 1082, 1013, 1031, 1021, 995, 993, 1010, 1077, 1056, 1008, 981, 1048, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[990, 999, 1033, 1019, 1014, 1073, 1047, 983, 1071, 1039, 1012, 1046, 1057, 1058, 1039, 1008, 1000, 1006, 1013, 1066, 1029, 973, 971, 992, 1013, 1060, 1062, 1011, 1025, 1055, 1007, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1017, 1011, 1067, 1035, 1010, 994, 1043, 1012, 995, 1013, 1014, 1038, 1088, 987, 1045, 1020, 1025, 1046, 1084, 1029, 1028, 1040, 991, 1053, 996, 1018, 963, 1026, 1029, 1041, 985, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1026, 1057, 1000, 1013, 990, 1026, 1072, 1010, 1061, 1013, 1016, 1025, 1030, 1009, 986, 1072, 1022, 1064, 998, 966, 1001, 1073, 1034, 1054, 1020, 1040, 1012, 964, 1057, 1015, 999, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1018, 1017, 1013, 1043, 1065, 1002, 1063, 1004, 1000, 1048, 1012, 973, 1015, 973, 957, 1045, 1024, 1010, 1003, 1048, 1017, 1078, 1016, 1030, 1033, 1065, 1031, 1038, 1028, 1020, 1054, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[982, 1017, 1023, 1010, 1027, 1020, 1014, 1044, 1042, 1011, 1030, 1022, 978, 1009, 948, 1071, 1020, 1025, 1062, 999, 1008, 1052, 1062, 1071, 1039, 1035, 989, 1002, 1031, 1041, 1026, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1035, 1077, 1036, 1079, 1034, 971, 1040, 992, 958, 1063, 998, 999, 980, 1016, 1020, 1073, 966, 1044, 1095, 1015, 979, 1005, 1005, 991, 1074, 1030, 1007, 1029, 1029, 981, 1056, 1091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1039, 1019, 1035, 1024, 0, 0, 0, 0, 1024, 1051, 1046, 1018, 0, 0, 0, 0, 1012, 1055, 1013, 1022, 15, 0, 0, 0, 1045, 1004, 997, 1045, 9, 2, 0, 0, 1048, 1059, 1027, 1012, 0, 0, 0, 0, 995, 1028, 1024, 1028, 5, 4, 0, 0, 1032, 1033, 1018, 988, 0, 0, 0, 0, 954, 1012, 964, 1058, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4139, 4117, 4102, 4146, 4084, 4071, 3992] avg=4096.0000 max=4146 max_rank=4 min=3992 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[990, 999, 1017, 1019, 0, 0, 0, 0, 1014, 1073, 1038, 983, 0, 0, 0, 0, 1071, 1039, 1012, 1046, 14, 0, 0, 0, 1057, 1058, 1039, 1008, 8, 2, 0, 0, 1000, 1006, 1001, 1066, 0, 0, 0, 0, 1029, 973, 971, 992, 5, 4, 0, 0, 1013, 1060, 1062, 1011, 0, 0, 0, 0, 1025, 1055, 1007, 997, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4108, 4182, 4172, 4073, 3974, 4146, 4088] avg=4096.0000 max=4182 max_rank=2 min=3974 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1024, 1015, 1071, 1029, 0, 0, 0, 0, 1030, 1049, 1045, 1019, 0, 0, 0, 0, 999, 988, 1003, 1057, 15, 0, 0, 0, 1032, 1009, 1005, 1055, 9, 2, 0, 0, 988, 989, 1016, 1082, 0, 0, 0, 0, 1013, 1031, 1021, 995, 5, 4, 0, 0, 993, 1010, 1077, 1056, 0, 0, 0, 0, 1008, 981, 1048, 991, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4143, 4062, 4112, 4075, 4069, 4136, 4032] avg=4096.0000 max=4143 max_rank=1 min=4032 min_rank=7 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[982, 1017, 1007, 1010, 0, 0, 0, 0, 1027, 1020, 1005, 1044, 0, 0, 0, 0, 1042, 1011, 1030, 1022, 14, 0, 0, 0, 978, 1009, 948, 1071, 9, 2, 0, 0, 1020, 1025, 1049, 999, 0, 0, 0, 0, 1008, 1052, 1062, 1071, 5, 4, 0, 0, 1039, 1035, 989, 1002, 0, 0, 0, 0, 1031, 1041, 1026, 1058, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4096, 4119, 4017, 4093, 4202, 4065, 4160] avg=4096.0000 max=4202 max_rank=5 min=4016 min_rank=0 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1017, 1011, 1050, 1035, 0, 0, 0, 0, 1010, 994, 1034, 1012, 0, 0, 0, 0, 995, 1013, 1014, 1038, 15, 0, 0, 0, 1088, 987, 1045, 1020, 9, 2, 0, 0, 1025, 1046, 1071, 1029, 0, 0, 0, 0, 1028, 1040, 991, 1053, 5, 4, 0, 0, 996, 1018, 963, 1026, 0, 0, 0, 0, 1029, 1041, 985, 1025, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4050, 4075, 4151, 4171, 4121, 4003, 4084] avg=4096.0000 max=4171 max_rank=4 min=4003 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1035, 1077, 1020, 1079, 0, 0, 0, 0, 1034, 971, 1031, 992, 0, 0, 0, 0, 958, 1063, 998, 999, 14, 0, 0, 0, 980, 1016, 1020, 1073, 9, 2, 0, 0, 966, 1044, 1082, 1015, 0, 0, 0, 0, 979, 1005, 1005, 991, 5, 4, 0, 0, 1074, 1030, 1007, 1029, 0, 0, 0, 0, 1029, 981, 1056, 1091, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4211, 4028, 4032, 4100, 4107, 3989, 4140, 4161] avg=4096.0000 max=4211 max_rank=0 min=3989 min_rank=5 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1026, 1057, 984, 1013, 0, 0, 0, 0, 990, 1026, 1063, 1010, 0, 0, 0, 0, 1061, 1013, 1016, 1025, 14, 0, 0, 0, 1030, 1009, 986, 1072, 8, 2, 0, 0, 1022, 1064, 986, 966, 0, 0, 0, 0, 1001, 1073, 1034, 1054, 5, 4, 0, 0, 1020, 1040, 1012, 964, 0, 0, 0, 0, 1057, 1015, 999, 1043, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4089, 4129, 4107, 4038, 4171, 4036, 4118] avg=4096.0000 max=4171 max_rank=5 min=4036 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1018, 1017, 989, 1043, 0, 0, 0, 0, 1065, 1002, 1044, 1004, 0, 0, 0, 0, 1000, 1048, 1012, 973, 19, 0, 0, 0, 1015, 973, 957, 1045, 12, 5, 0, 0, 1024, 1010, 983, 1048, 0, 0, 0, 0, 1017, 1078, 1016, 1030, 9, 8, 0, 0, 1033, 1065, 1031, 1038, 4, 0, 0, 0, 1028, 1020, 1054, 1025, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4115, 4052, 4007, 4065, 4158, 4171, 4133] avg=4096.0000 max=4171 max_rank=6 min=4007 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8448, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[999, 994, 1074, 1052, 1000, 1010, 1016, 1001, 1059, 984, 1033, 1007, 1054, 999, 1015, 996, 1067, 989, 1104, 1016, 1020, 1045, 1018, 1070, 1031, 994, 985, 1028, 1047, 1036, 1024, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1036, 1052, 994, 974, 1031, 1080, 1037, 1039, 1051, 1008, 1051, 1016, 1018, 1056, 1033, 998, 1007, 1053, 991, 1031, 984, 1002, 1026, 1067, 1024, 978, 1017, 1021, 1008, 1002, 1044, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1036, 1030, 1030, 1032, 960, 1041, 1045, 1053, 1007, 1014, 1043, 1027, 1049, 994, 1025, 1028, 1036, 990, 978, 993, 1001, 996, 1052, 1019, 1053, 1018, 1071, 1044, 1039, 1034, 1063, 967] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[960, 1040, 1016, 1002, 986, 1042, 1058, 1007, 988, 1068, 1012, 985, 1058, 1030, 998, 1004, 1055, 1028, 1020, 1027, 1032, 1019, 1035, 1012, 1027, 1036, 1024, 1090, 1011, 1063, 1024, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1001, 1052, 1042, 1039, 1093, 1046, 1006, 1032, 995, 1036, 999, 1046, 1011, 1033, 1030, 1005, 1012, 960, 1050, 1031, 1049, 991, 1045, 995, 1010, 999, 1045, 986, 988, 1082, 1028, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1034, 996, 1041, 1013, 1003, 1011, 1008, 1031, 1024, 969, 1024, 1014, 1038, 1021, 1056, 1027, 1020, 1000, 1050, 1007, 1018, 1059, 1031, 1062, 1057, 969, 1000, 1013, 1051, 1052, 1045, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1025, 1029, 1046, 1032, 1046, 1035, 1029, 1054, 1017, 1015, 1021, 1018, 1033, 1024, 1015, 1032, 1007, 1041, 1040, 1075, 1054, 1041, 994, 996, 1011, 1042, 979, 982, 1004, 955, 1071, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1026, 1016, 1053, 1026, 976, 999, 1032, 1047, 963, 1043, 1018, 1030, 1010, 1029, 992, 1030, 1069, 1042, 1060, 998, 1056, 1024, 1007, 1048, 1023, 1036, 1046, 981, 1061, 1003, 1015, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[999, 994, 1071, 1052, 0, 0, 0, 0, 1000, 1000, 1016, 1001, 0, 0, 0, 0, 1059, 984, 1033, 1007, 12, 10, 0, 0, 1054, 999, 1015, 996, 3, 0, 0, 0, 1067, 989, 1094, 1016, 0, 0, 0, 0, 1020, 1045, 1018, 1058, 0, 0, 0, 0, 1031, 994, 985, 1028, 10, 8, 0, 0, 1047, 1036, 1016, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4017, 4105, 4067, 4166, 4141, 4056, 4100] avg=4096.0000 max=4166 max_rank=4 min=4017 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1036, 1030, 1028, 1032, 0, 0, 0, 0, 960, 1031, 1045, 1053, 0, 0, 0, 0, 1007, 1014, 1043, 1027, 12, 10, 0, 0, 1049, 994, 1025, 1028, 2, 0, 0, 0, 1036, 990, 969, 993, 0, 0, 0, 0, 1001, 996, 1052, 1007, 0, 0, 0, 0, 1053, 1018, 1071, 1044, 9, 8, 0, 0, 1039, 1034, 1055, 967, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4089, 4113, 4098, 3988, 4056, 4203, 4095] avg=4096.0000 max=4203 max_rank=6 min=3988 min_rank=4 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1036, 1052, 992, 974, 0, 0, 0, 0, 1031, 1070, 1037, 1039, 0, 0, 0, 0, 1051, 1008, 1051, 1016, 12, 10, 0, 0, 1018, 1056, 1033, 998, 2, 0, 0, 0, 1007, 1053, 982, 1031, 0, 0, 0, 0, 984, 1002, 1026, 1055, 0, 0, 0, 0, 1024, 978, 1017, 1021, 9, 8, 0, 0, 1008, 1002, 1036, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4177, 4148, 4107, 4073, 4067, 4057, 4085] avg=4096.0000 max=4177 max_rank=1 min=4054 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[960, 1040, 1014, 1002, 0, 0, 0, 0, 986, 1032, 1058, 1007, 0, 0, 0, 0, 988, 1068, 1012, 985, 12, 10, 0, 0, 1058, 1030, 998, 1004, 2, 0, 0, 0, 1055, 1028, 1011, 1027, 0, 0, 0, 0, 1032, 1019, 1035, 1000, 0, 0, 0, 0, 1027, 1036, 1024, 1090, 9, 8, 0, 0, 1011, 1063, 1016, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4083, 4075, 4092, 4121, 4086, 4194, 4101] avg=4096.0000 max=4194 max_rank=6 min=4016 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1034, 996, 1038, 1013, 0, 0, 0, 0, 1003, 1001, 1008, 1031, 0, 0, 0, 0, 1024, 969, 1024, 1014, 12, 10, 0, 0, 1038, 1021, 1056, 1027, 3, 0, 0, 0, 1020, 1000, 1040, 1007, 0, 0, 0, 0, 1018, 1059, 1031, 1050, 0, 0, 0, 0, 1057, 969, 1000, 1013, 10, 8, 0, 0, 1051, 1052, 1037, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4043, 4053, 4145, 4067, 4158, 4057, 4164] avg=4096.0000 max=4164 max_rank=7 min=4043 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1001, 1052, 1036, 1039, 0, 0, 0, 0, 1093, 1030, 1006, 1032, 0, 0, 0, 0, 995, 1036, 999, 1046, 16, 13, 0, 0, 1011, 1033, 1030, 1005, 6, 3, 0, 0, 1012, 960, 1037, 1031, 0, 0, 0, 0, 1049, 991, 1045, 979, 0, 0, 0, 0, 1010, 999, 1045, 986, 13, 10, 3, 0, 988, 1082, 1015, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4161, 4105, 4088, 4040, 4064, 4066, 4116] avg=4096.0000 max=4161 max_rank=1 min=4040 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1026, 1016, 1050, 1026, 0, 0, 0, 0, 976, 989, 1032, 1047, 0, 0, 0, 0, 963, 1043, 1018, 1030, 12, 10, 0, 0, 1010, 1029, 992, 1030, 3, 0, 0, 0, 1069, 1042, 1050, 998, 0, 0, 0, 0, 1056, 1024, 1007, 1036, 0, 0, 0, 0, 1023, 1036, 1046, 981, 10, 8, 0, 0, 1061, 1003, 1007, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4044, 4076, 4064, 4159, 4123, 4104, 4080] avg=4096.0000 max=4159 max_rank=4 min=4044 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1025, 1029, 1043, 1032, 0, 0, 0, 0, 1046, 1025, 1029, 1054, 0, 0, 0, 0, 1017, 1015, 1021, 1018, 12, 10, 0, 0, 1033, 1024, 1015, 1032, 3, 0, 0, 0, 1007, 1041, 1031, 1075, 0, 0, 0, 0, 1054, 1041, 994, 984, 0, 0, 0, 0, 1011, 1042, 979, 982, 9, 8, 0, 0, 1004, 955, 1063, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4154, 4093, 4107, 4154, 4073, 4031, 4027] avg=4096.0000 max=4154 max_rank=1 min=4027 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1023, 972, 1053, 1005, 1026, 1068, 1054, 1039, 984, 1065, 999, 1054, 1033, 1015, 1056, 1061, 1009, 963, 1000, 1051, 1021, 1004, 1013, 1003, 1028, 1032, 1012, 1008, 1052, 1030, 1027, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[992, 1080, 1016, 942, 1017, 1004, 1031, 1042, 1030, 1037, 980, 1017, 1068, 1047, 1016, 1025, 1048, 1011, 1003, 1046, 1032, 1009, 1014, 1028, 977, 1095, 1020, 1022, 1060, 1012, 1024, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1019, 1013, 957, 1049, 986, 980, 1013, 993, 1069, 1029, 999, 1037, 1035, 1023, 1018, 990, 1036, 1049, 1033, 1008, 1053, 1070, 1046, 1001, 1047, 1051, 1042, 992, 1019, 1046, 1046, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[968, 1056, 1054, 1041, 1046, 1037, 1031, 1034, 972, 1044, 1043, 1056, 1053, 1019, 1041, 1000, 1020, 993, 1001, 1023, 1020, 991, 1012, 974, 1035, 1039, 1021, 1040, 1000, 1055, 1034, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1040, 1017, 1020, 1053, 1017, 1085, 1025, 1010, 1015, 1020, 1009, 1013, 968, 1028, 1022, 983, 1054, 1029, 1036, 1047, 1030, 1029, 967, 1008, 1066, 986, 1044, 1028, 1061, 996, 1044, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1037, 1038, 984, 1011, 977, 1058, 1035, 1021, 1055, 1038, 1036, 1027, 988, 1010, 1018, 1048, 975, 1016, 1046, 999, 1029, 1001, 1055, 1013, 1071, 1023, 1010, 1046, 997, 1020, 1046, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1032, 1046, 1062, 1050, 1028, 1010, 1029, 1075, 1027, 995, 1012, 1001, 970, 1035, 1022, 1001, 990, 1008, 1020, 1042, 997, 1028, 1001, 1021, 989, 1056, 984, 1011, 1038, 1038, 1075, 1075] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[976, 1003, 1052, 1028, 1029, 995, 1020, 986, 1036, 998, 1008, 1058, 1033, 999, 1057, 1025, 992, 1039, 999, 1117, 989, 1034, 1033, 1001, 1059, 1043, 990, 1014, 1003, 1043, 1072, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[992, 1080, 1016, 942, 7, 1, 0, 0, 1017, 1004, 1028, 1042, 0, 0, 0, 0, 1030, 1037, 980, 1017, 0, 0, 0, 0, 1068, 1047, 1016, 1025, 4, 3, 0, 0, 1048, 1011, 1003, 1046, 8, 0, 0, 0, 1032, 1009, 1014, 1028, 29, 0, 0, 0, 977, 1082, 1020, 1022, 0, 0, 0, 0, 1060, 1012, 988, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4091, 4064, 4163, 4116, 4112, 4101, 4083] avg=4096.0000 max=4163 max_rank=3 min=4038 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1023, 972, 1053, 1005, 7, 1, 0, 0, 1026, 1068, 1051, 1039, 0, 0, 0, 0, 984, 1065, 999, 1054, 0, 0, 0, 0, 1033, 1015, 1056, 1061, 4, 3, 0, 0, 1009, 963, 1000, 1051, 8, 0, 0, 0, 1021, 1004, 1013, 1003, 29, 0, 0, 0, 1028, 1019, 1012, 1008, 0, 0, 0, 0, 1052, 1030, 991, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4184, 4102, 4172, 4031, 4070, 4067, 4081] avg=4096.0000 max=4184 max_rank=1 min=4031 min_rank=4 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16249 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1019, 1013, 957, 1049, 8, 1, 0, 0, 986, 980, 1010, 993, 0, 0, 0, 0, 1069, 1029, 999, 1037, 0, 0, 0, 0, 1035, 1023, 1018, 990, 4, 3, 0, 0, 1036, 1049, 1033, 1008, 8, 0, 0, 0, 1053, 1070, 1046, 1001, 30, 0, 0, 0, 1047, 1038, 1042, 992, 0, 0, 0, 0, 1019, 1046, 1008, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 3969, 4134, 4073, 4134, 4200, 4119, 4092] avg=4096.0000 max=4200 max_rank=5 min=3969 min_rank=1 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16545 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[968, 1056, 1054, 1041, 7, 1, 0, 0, 1046, 1037, 1028, 1034, 0, 0, 0, 0, 972, 1044, 1043, 1056, 0, 0, 0, 0, 1053, 1019, 1041, 1000, 4, 3, 0, 0, 1020, 993, 1001, 1023, 8, 0, 0, 0, 1020, 991, 1012, 974, 29, 0, 0, 0, 1035, 1026, 1021, 1040, 0, 0, 0, 0, 1000, 1055, 998, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4145, 4115, 4120, 4045, 4026, 4122, 4068] avg=4096.0000 max=4145 max_rank=1 min=4026 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[976, 1003, 1052, 1028, 8, 1, 0, 0, 1029, 995, 1017, 986, 0, 0, 0, 0, 1036, 998, 1008, 1058, 0, 0, 0, 0, 1033, 999, 1057, 1025, 4, 3, 0, 0, 992, 1039, 999, 1117, 8, 0, 0, 0, 989, 1034, 1033, 1001, 30, 0, 0, 0, 1059, 1030, 990, 1014, 0, 0, 0, 0, 1003, 1043, 1034, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4027, 4100, 4121, 4155, 4087, 4093, 4117] avg=4096.0000 max=4155 max_rank=4 min=4027 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1037, 1038, 984, 1011, 8, 1, 0, 0, 977, 1058, 1032, 1021, 0, 0, 0, 0, 1055, 1038, 1036, 1027, 0, 0, 0, 0, 988, 1010, 1018, 1048, 4, 3, 0, 0, 975, 1016, 1046, 999, 7, 0, 0, 0, 1029, 1001, 1055, 1013, 30, 0, 0, 0, 1071, 1011, 1010, 1046, 0, 0, 0, 0, 997, 1020, 1008, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4088, 4156, 4071, 4043, 4128, 4138, 4065] avg=4096.0000 max=4156 max_rank=2 min=4043 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1040, 1017, 1020, 1053, 11, 8, 0, 0, 1017, 1085, 1013, 1010, 0, 0, 0, 0, 1015, 1020, 1009, 1013, 5, 0, 0, 0, 968, 1028, 1022, 983, 5, 7, 0, 0, 1054, 1029, 1036, 1047, 10, 0, 0, 0, 1030, 1029, 967, 1008, 34, 0, 0, 0, 1066, 963, 1044, 1028, 0, 0, 0, 0, 1061, 996, 999, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4125, 4062, 4013, 4176, 4068, 4101, 4074] avg=4096.0000 max=4176 max_rank=4 min=4013 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1032, 1046, 1062, 1050, 8, 1, 0, 0, 1028, 1010, 1026, 1075, 0, 0, 0, 0, 1027, 995, 1012, 1001, 0, 0, 0, 0, 970, 1035, 1022, 1001, 4, 3, 0, 0, 990, 1008, 1020, 1042, 8, 0, 0, 0, 997, 1028, 1001, 1021, 30, 0, 0, 0, 989, 1043, 984, 1011, 0, 0, 0, 0, 1038, 1038, 1037, 1075, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4199, 4139, 4035, 4035, 4068, 4077, 4027, 4188] avg=4096.0000 max=4199 max_rank=0 min=4027 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8064, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8448, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1017, 1054, 1084, 1063, 1013, 1048, 1010, 1029, 988, 1028, 1034, 1018, 992, 1005, 1105, 987, 1038, 1011, 1047, 997, 1000, 1038, 1016, 1051, 992, 1023, 1051, 977, 980, 1001, 1031, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1035, 1032, 1044, 1034, 978, 1034, 1013, 995, 982, 1038, 1042, 984, 1037, 1000, 1060, 1018, 994, 1067, 992, 1054, 1011, 1041, 1018, 1068, 962, 1090, 1021, 1062, 1013, 1014, 975, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1012, 1050, 1018, 1060, 1029, 1075, 1026, 1060, 1027, 988, 1032, 967, 1050, 1047, 1015, 1101, 1044, 934, 1028, 1067, 1032, 1032, 1046, 943, 1004, 1034, 994, 1045, 1001, 987, 1017, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1017, 1003, 1056, 1022, 1071, 996, 1003, 1037, 1007, 1045, 1051, 983, 986, 974, 1055, 1009, 1066, 992, 1049, 1064, 1036, 1061, 1047, 996, 994, 1017, 1050, 1004, 1029, 998, 1028, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[997, 1036, 1036, 1016, 1033, 1038, 965, 1013, 998, 1007, 1026, 1062, 1046, 1018, 1052, 998, 1024, 1031, 1054, 1023, 1066, 1031, 994, 1029, 1001, 1001, 943, 1039, 1024, 1044, 1102, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1033, 1006, 1028, 1014, 1039, 1049, 1073, 1004, 1025, 1015, 985, 1051, 1072, 1030, 960, 1014, 1013, 994, 1018, 1043, 1033, 1046, 990, 1050, 1000, 1030, 1016, 1068, 1105, 981, 1010, 973] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1004, 1053, 1023, 1011, 981, 999, 1047, 1003, 1059, 1018, 1073, 1020, 970, 1086, 1026, 1052, 1030, 1016, 1072, 1039, 1032, 1026, 992, 998, 1028, 1011, 985, 1053, 1000, 1053, 989, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1031, 1036, 1070, 1025, 1004, 1068, 1032, 1053, 1010, 1022, 1020, 989, 1026, 1030, 1000, 1051, 970, 1005, 1103, 1024, 1021, 1058, 1020, 1018, 993, 1001, 1018, 1019, 1043, 998, 994, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1017, 1054, 1052, 1063, 0, 0, 0, 0, 1013, 1042, 1010, 1029, 0, 0, 0, 0, 988, 1028, 1034, 1018, 8, 6, 6, 0, 992, 1005, 1093, 987, 0, 0, 0, 0, 1038, 1011, 1031, 997, 0, 0, 0, 0, 1000, 1030, 1016, 1051, 0, 0, 0, 0, 992, 1023, 1051, 977, 31, 0, 0, 0, 980, 1001, 1031, 1040, 16, 6, 1, 0] rerouted_tokens_per_ep_rank=[4186, 4094, 4088, 4077, 4077, 4097, 4074, 4075] avg=4096.0000 max=4186 max_rank=0 min=4074 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1012, 1050, 988, 1060, 0, 0, 0, 0, 1029, 1069, 1026, 1060, 0, 0, 0, 0, 1027, 988, 1032, 967, 8, 6, 6, 0, 1050, 1047, 1003, 1101, 0, 0, 0, 0, 1044, 934, 1012, 1067, 0, 0, 0, 0, 1032, 1024, 1046, 943, 0, 0, 0, 0, 1004, 1034, 994, 1045, 29, 0, 0, 0, 1001, 987, 1017, 1003, 16, 6, 1, 0] rerouted_tokens_per_ep_rank=[4110, 4184, 4034, 4201, 4057, 4045, 4106, 4031] avg=4096.0000 max=4201 max_rank=3 min=4031 min_rank=7 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16239 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1035, 1032, 1013, 1034, 0, 0, 0, 0, 978, 1028, 1013, 995, 0, 0, 0, 0, 982, 1038, 1042, 984, 8, 6, 6, 0, 1037, 1000, 1048, 1018, 0, 0, 0, 0, 994, 1067, 976, 1054, 0, 0, 0, 0, 1011, 1033, 1018, 1068, 0, 0, 0, 0, 962, 1090, 1021, 1062, 30, 0, 0, 0, 1013, 1014, 975, 1060, 16, 6, 1, 0] rerouted_tokens_per_ep_rank=[4114, 4014, 4066, 4103, 4091, 4130, 4165, 4085] avg=4096.0000 max=4165 max_rank=6 min=4014 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1033, 1006, 998, 1014, 0, 0, 0, 0, 1039, 1043, 1073, 1004, 0, 0, 0, 0, 1025, 1015, 985, 1051, 9, 6, 6, 0, 1072, 1030, 948, 1014, 0, 0, 0, 0, 1013, 994, 1002, 1043, 0, 0, 0, 0, 1033, 1037, 990, 1050, 0, 0, 0, 0, 1000, 1030, 1016, 1068, 29, 0, 0, 0, 1105, 981, 1010, 973, 16, 6, 1, 0] rerouted_tokens_per_ep_rank=[4051, 4159, 4097, 4064, 4052, 4110, 4143, 4092] avg=4096.0000 max=4159 max_rank=1 min=4051 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[997, 1036, 1006, 1016, 0, 0, 0, 0, 1033, 1032, 965, 1013, 0, 0, 0, 0, 998, 1007, 1026, 1062, 8, 6, 6, 0, 1046, 1018, 1040, 998, 0, 0, 0, 0, 1024, 1031, 1037, 1023, 0, 0, 0, 0, 1066, 1023, 994, 1029, 0, 0, 0, 0, 1001, 1001, 943, 1039, 29, 0, 0, 0, 1024, 1044, 1102, 1021, 17, 6, 1, 0] rerouted_tokens_per_ep_rank=[4055, 4043, 4113, 4102, 4115, 4112, 4013, 4215] avg=4096.0000 max=4215 max_rank=7 min=4013 min_rank=6 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1017, 1003, 1018, 1022, 0, 0, 0, 0, 1071, 988, 1003, 1037, 0, 0, 0, 0, 1007, 1045, 1051, 983, 14, 10, 8, 0, 986, 974, 1035, 1009, 0, 0, 0, 0, 1066, 992, 1029, 1064, 0, 0, 0, 0, 1036, 1047, 1047, 996, 0, 0, 0, 0, 994, 1017, 1050, 1004, 35, 0, 0, 0, 1029, 998, 1028, 1022, 20, 10, 3, 0] rerouted_tokens_per_ep_rank=[4060, 4099, 4118, 4004, 4151, 4126, 4100, 4110] avg=4096.0000 max=4151 max_rank=4 min=4004 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1004, 1053, 993, 1011, 0, 0, 0, 0, 981, 993, 1047, 1003, 0, 0, 0, 0, 1059, 1018, 1073, 1020, 8, 6, 6, 0, 970, 1086, 1014, 1052, 0, 0, 0, 0, 1030, 1016, 1055, 1039, 0, 0, 0, 0, 1032, 1018, 992, 998, 0, 0, 0, 0, 1028, 1011, 985, 1053, 29, 0, 0, 0, 1000, 1053, 989, 1019, 17, 6, 1, 0] rerouted_tokens_per_ep_rank=[4061, 4024, 4190, 4122, 4140, 4040, 4106, 4085] avg=4096.0000 max=4190 max_rank=2 min=4024 min_rank=1 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1031, 1036, 1039, 1025, 0, 0, 0, 0, 1004, 1062, 1032, 1053, 0, 0, 0, 0, 1010, 1022, 1020, 989, 9, 6, 6, 0, 1026, 1030, 988, 1051, 0, 0, 0, 0, 970, 1005, 1086, 1024, 0, 0, 0, 0, 1021, 1049, 1020, 1018, 0, 0, 0, 0, 993, 1001, 1018, 1019, 30, 0, 0, 0, 1043, 998, 994, 1016, 17, 6, 1, 0] rerouted_tokens_per_ep_rank=[4131, 4151, 4062, 4095, 4085, 4108, 4061, 4075] avg=4096.0000 max=4151 max_rank=1 min=4061 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[995, 1023, 1043, 1000, 1068, 990, 1013, 1011, 1041, 1023, 1024, 994, 1033, 1022, 1030, 1002, 1035, 1041, 1057, 1012, 1014, 1053, 976, 1039, 1020, 1084, 1033, 1005, 1040, 1009, 1061, 977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1025, 1005, 1012, 996, 1058, 1079, 1012, 987, 1061, 1048, 1057, 967, 1003, 1009, 1001, 1002, 1050, 971, 1025, 1006, 1011, 1016, 1014, 1043, 1049, 1010, 1061, 1039, 1043, 1018, 1045, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1037, 1026, 1039, 984, 1048, 960, 1062, 1023, 1026, 1054, 1031, 1037, 1060, 1015, 1059, 1014, 1031, 1009, 1003, 1038, 995, 1074, 1009, 1000, 1027, 1016, 992, 978, 1033, 1055, 1033, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1028, 1006, 1010, 994, 1007, 953, 1022, 1013, 1027, 1052, 1047, 1000, 1071, 1033, 1013, 1048, 1044, 1017, 1042, 1003, 948, 1051, 1029, 1013, 1046, 1112, 992, 1043, 1030, 998, 1024, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1002, 1048, 1002, 1050, 1031, 1065, 981, 1083, 1005, 1064, 1044, 1051, 1008, 970, 1020, 996, 1054, 1013, 1075, 969, 1050, 1004, 1013, 1017, 1015, 1017, 1014, 1030, 1029, 1006, 1041, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1033, 995, 1067, 1044, 1088, 1049, 1003, 1011, 1007, 1013, 1042, 1022, 1045, 1009, 1054, 1033, 1029, 1033, 988, 1016, 996, 1009, 986, 1046, 1049, 968, 1028, 1023, 1008, 1033, 1034, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1010, 996, 1034, 1011, 1034, 1009, 1058, 987, 1045, 1051, 1015, 1029, 984, 1032, 1035, 1011, 1017, 1034, 990, 1012, 1101, 1009, 1048, 998, 1068, 1043, 1038, 1011, 1013, 997, 1021, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1034, 981, 1028, 1029, 1014, 1047, 982, 1008, 991, 936, 1057, 1040, 1005, 1028, 1028, 1031, 1037, 1047, 1028, 1054, 998, 1052, 1001, 1043, 1053, 996, 1033, 1010, 1132, 1016, 1043, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 1005, 1012, 996, 17, 5, 0, 0, 1058, 1079, 1012, 987, 1, 0, 0, 0, 1061, 1048, 1041, 967, 0, 0, 0, 0, 1003, 1009, 1001, 1002, 8, 0, 0, 0, 1049, 971, 1025, 1006, 0, 0, 0, 0, 1011, 1016, 1014, 1043, 11, 3, 0, 0, 1032, 1010, 1061, 1039, 0, 0, 0, 0, 1032, 1018, 1045, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4137, 4117, 4023, 4051, 4098, 4142, 4140] avg=4096.0000 max=4142 max_rank=6 min=4023 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16431 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1037, 1026, 1039, 984, 16, 5, 0, 0, 1048, 960, 1062, 1023, 1, 0, 0, 0, 1026, 1054, 1016, 1037, 0, 0, 0, 0, 1060, 1015, 1059, 1014, 7, 0, 0, 0, 1030, 1009, 1003, 1038, 0, 0, 0, 0, 995, 1074, 1009, 1000, 10, 3, 0, 0, 1011, 1016, 992, 978, 0, 0, 0, 0, 1023, 1055, 1033, 1000, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4094, 4133, 4155, 4080, 4091, 3997, 4111] avg=4096.0000 max=4155 max_rank=3 min=3997 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16279 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1033, 995, 1067, 1044, 17, 5, 0, 0, 1088, 1049, 1003, 1011, 1, 0, 0, 0, 1007, 1013, 1027, 1022, 0, 0, 0, 0, 1045, 1009, 1054, 1033, 7, 0, 0, 0, 1028, 1033, 988, 1016, 0, 0, 0, 0, 996, 1009, 986, 1046, 10, 3, 0, 0, 1032, 968, 1028, 1023, 0, 0, 0, 0, 998, 1033, 1034, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4152, 4069, 4148, 4065, 4050, 4051, 4072] avg=4096.0000 max=4161 max_rank=0 min=4050 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16238 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[995, 1023, 1043, 1000, 16, 5, 0, 0, 1068, 990, 1013, 1011, 1, 0, 0, 0, 1041, 1023, 1009, 994, 0, 0, 0, 0, 1033, 1022, 1030, 1002, 7, 0, 0, 0, 1034, 1041, 1057, 1012, 0, 0, 0, 0, 1014, 1053, 976, 1039, 10, 3, 0, 0, 1004, 1084, 1033, 1005, 0, 0, 0, 0, 1030, 1009, 1061, 977, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4083, 4067, 4094, 4144, 4095, 4126, 4077] avg=4096.0000 max=4144 max_rank=4 min=4067 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16442 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1028, 1006, 1010, 994, 16, 5, 0, 0, 1007, 953, 1022, 1013, 1, 0, 0, 0, 1027, 1052, 1032, 1000, 0, 0, 0, 0, 1071, 1033, 1013, 1048, 7, 0, 0, 0, 1043, 1017, 1042, 1003, 0, 0, 0, 0, 948, 1051, 1029, 1013, 10, 3, 0, 0, 1030, 1112, 992, 1043, 0, 0, 0, 0, 1020, 998, 1024, 1052, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 3996, 4111, 4172, 4105, 4054, 4177, 4094] avg=4096.0000 max=4177 max_rank=6 min=3996 min_rank=1 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16430 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1002, 1048, 1002, 1050, 19, 11, 0, 0, 1031, 1065, 981, 1083, 5, 0, 0, 0, 1005, 1064, 1018, 1051, 0, 0, 0, 0, 1008, 970, 1020, 996, 13, 0, 0, 0, 1049, 1013, 1075, 969, 0, 0, 0, 0, 1050, 1004, 1013, 1017, 15, 4, 0, 0, 996, 1017, 1014, 1030, 0, 0, 0, 0, 1012, 1006, 1041, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4165, 4138, 4007, 4106, 4103, 4057, 4060] avg=4096.0000 max=4165 max_rank=1 min=4007 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16326 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1010, 996, 1034, 1011, 17, 5, 0, 0, 1034, 1009, 1058, 987, 1, 0, 0, 0, 1045, 1051, 1000, 1029, 0, 0, 0, 0, 984, 1032, 1035, 1011, 7, 0, 0, 0, 1016, 1034, 990, 1012, 0, 0, 0, 0, 1101, 1009, 1048, 998, 10, 3, 0, 0, 1051, 1043, 1038, 1011, 0, 0, 0, 0, 1003, 997, 1021, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4089, 4125, 4069, 4052, 4169, 4143, 4048] avg=4096.0000 max=4169 max_rank=5 min=4048 min_rank=7 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16412 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1034, 981, 1028, 1029, 17, 5, 0, 0, 1014, 1047, 982, 1008, 1, 0, 0, 0, 991, 936, 1041, 1040, 0, 0, 0, 0, 1005, 1028, 1028, 1031, 8, 0, 0, 0, 1036, 1047, 1028, 1054, 0, 0, 0, 0, 998, 1052, 1001, 1043, 11, 3, 0, 0, 1036, 996, 1033, 1010, 0, 0, 0, 0, 1121, 1016, 1043, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4052, 4008, 4100, 4165, 4108, 4075, 4166] avg=4096.0000 max=4166 max_rank=7 min=4008 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16514 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1070, 1033, 998, 1026, 1021, 1049, 1047, 961, 1012, 1020, 1024, 1069, 1004, 1010, 1104, 1016, 1025, 1014, 1025, 1064, 982, 1025, 1021, 1030, 1027, 992, 1003, 1014, 1031, 990, 1019, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1052, 1024, 1017, 1029, 1038, 1022, 1051, 1023, 1022, 1067, 972, 998, 1060, 1052, 999, 1056, 983, 1004, 1009, 1042, 1006, 1014, 1005, 1039, 1028, 988, 1031, 1015, 1010, 1081, 1050, 981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1051, 1001, 1043, 1025, 1057, 1071, 1016, 993, 1043, 997, 1008, 988, 1010, 1056, 976, 966, 1060, 1009, 1024, 1072, 1038, 1015, 1007, 1080, 993, 1004, 1041, 1013, 1011, 1037, 1025, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1062, 1035, 1036, 1071, 1055, 1044, 1032, 1056, 1074, 1053, 1027, 1045, 982, 971, 1008, 1052, 1025, 1036, 1005, 1031, 1009, 1004, 987, 996, 1031, 1056, 994, 1037, 995, 1003, 994, 962] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1046, 1017, 1040, 1082, 1019, 1014, 1028, 1034, 1033, 993, 1011, 1006, 1030, 1026, 992, 1083, 1006, 1045, 1016, 965, 1026, 1014, 1018, 1087, 1001, 1031, 999, 1004, 982, 1061, 1016, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1027, 1081, 1012, 1018, 1021, 1033, 1003, 1032, 1013, 949, 1018, 1048, 1045, 1040, 1019, 1007, 1065, 1037, 977, 1058, 982, 1045, 1048, 1005, 1075, 1018, 995, 1065, 1032, 1017, 1037, 946] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1044, 989, 1030, 1003, 1016, 1009, 1024, 1003, 1003, 1078, 1041, 1059, 1041, 1023, 1056, 1022, 1020, 1045, 1020, 1082, 976, 1021, 1004, 1018, 998, 1023, 978, 1029, 1038, 1045, 1026, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1061, 1070, 1062, 1043, 1016, 994, 1016, 1054, 1059, 979, 1029, 1083, 1000, 1050, 1011, 1010, 969, 1038, 1024, 1015, 1024, 1029, 1024, 1000, 1002, 1017, 1011, 1040, 1014, 1056, 958, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1016, 1033, 998, 1026, 0, 0, 0, 0, 1011, 1049, 1047, 961, 0, 0, 0, 0, 1012, 1020, 1024, 1063, 0, 0, 0, 0, 1004, 1009, 1104, 1016, 0, 0, 0, 0, 1025, 1014, 1025, 1059, 0, 0, 0, 0, 982, 1025, 1021, 1030, 10, 6, 5, 1, 1027, 992, 1003, 1014, 27, 0, 0, 0, 1031, 990, 1019, 1042, 27, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4068, 4119, 4133, 4123, 4080, 4063, 4109] avg=4096.0000 max=4133 max_rank=3 min=4063 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16375 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1000, 1024, 1017, 1029, 0, 0, 0, 0, 1028, 1022, 1051, 1023, 0, 0, 0, 0, 1022, 1067, 972, 992, 0, 0, 0, 0, 1060, 1051, 999, 1056, 0, 0, 0, 0, 983, 1004, 1009, 1037, 0, 0, 0, 0, 1006, 1014, 1005, 1039, 10, 6, 5, 1, 1028, 988, 1031, 1015, 26, 0, 0, 0, 1010, 1081, 1050, 981, 26, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4124, 4053, 4166, 4033, 4086, 4088, 4148] avg=4096.0000 max=4166 max_rank=3 min=4033 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16355 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1008, 1035, 1036, 1071, 0, 0, 0, 0, 1045, 1044, 1032, 1056, 0, 0, 0, 0, 1074, 1053, 1027, 1039, 0, 0, 0, 0, 982, 970, 1008, 1052, 0, 0, 0, 0, 1025, 1036, 1005, 1026, 0, 0, 0, 0, 1009, 1004, 987, 996, 10, 6, 5, 1, 1031, 1056, 994, 1037, 27, 0, 0, 0, 995, 1003, 994, 962, 27, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4177, 4193, 4012, 4092, 4018, 4145, 3981] avg=4096.0000 max=4193 max_rank=2 min=3981 min_rank=7 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16236 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[994, 1017, 1040, 1082, 0, 0, 0, 0, 1009, 1014, 1028, 1034, 0, 0, 0, 0, 1033, 993, 1011, 1000, 0, 0, 0, 0, 1030, 1025, 992, 1083, 0, 0, 0, 0, 1006, 1045, 1016, 961, 0, 0, 0, 0, 1026, 1014, 1018, 1087, 10, 6, 4, 1, 1001, 1031, 999, 1004, 26, 0, 0, 0, 982, 1061, 1016, 1043, 26, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4085, 4037, 4130, 4028, 4166, 4061, 4128] avg=4096.0000 max=4166 max_rank=5 min=4028 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16383 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[991, 1001, 1043, 1025, 0, 0, 0, 0, 1043, 1071, 1016, 993, 0, 0, 0, 0, 1043, 997, 1008, 977, 0, 0, 0, 0, 1010, 1054, 976, 966, 0, 0, 0, 0, 1060, 1009, 1024, 1064, 0, 0, 0, 0, 1038, 1015, 1007, 1080, 14, 11, 8, 2, 993, 1004, 1041, 1013, 30, 0, 0, 0, 1011, 1037, 1025, 1038, 30, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4123, 4025, 4006, 4157, 4175, 4081, 4141] avg=4096.0000 max=4175 max_rank=5 min=4006 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16554 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[975, 1081, 1012, 1018, 0, 0, 0, 0, 1011, 1033, 1003, 1032, 0, 0, 0, 0, 1013, 949, 1018, 1042, 0, 0, 0, 0, 1045, 1039, 1019, 1007, 0, 0, 0, 0, 1065, 1037, 977, 1053, 0, 0, 0, 0, 982, 1045, 1048, 1005, 10, 6, 5, 1, 1075, 1018, 995, 1065, 26, 0, 0, 0, 1032, 1017, 1037, 946, 26, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4079, 4022, 4110, 4132, 4102, 4179, 4058] avg=4096.0000 max=4179 max_rank=6 min=4022 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16471 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1008, 1070, 1062, 1043, 0, 0, 0, 0, 1006, 994, 1016, 1054, 0, 0, 0, 0, 1059, 979, 1029, 1077, 0, 0, 0, 0, 1000, 1049, 1011, 1010, 0, 0, 0, 0, 969, 1038, 1024, 1010, 0, 0, 0, 0, 1024, 1029, 1024, 1000, 10, 6, 5, 1, 1002, 1017, 1011, 1040, 27, 0, 0, 0, 1014, 1056, 958, 1010, 26, 0, 0, 0] rerouted_tokens_per_ep_rank=[4183, 4070, 4144, 4070, 4041, 4099, 4097, 4064] avg=4096.0000 max=4183 max_rank=0 min=4041 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16301 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[992, 989, 1030, 1003, 0, 0, 0, 0, 1006, 1009, 1024, 1003, 0, 0, 0, 0, 1003, 1078, 1041, 1053, 0, 0, 0, 0, 1041, 1022, 1056, 1022, 0, 0, 0, 0, 1020, 1045, 1020, 1077, 0, 0, 0, 0, 976, 1021, 1004, 1018, 10, 6, 5, 1, 998, 1023, 978, 1029, 26, 0, 0, 0, 1038, 1045, 1026, 1004, 26, 0, 0, 0] rerouted_tokens_per_ep_rank=[4014, 4042, 4175, 4141, 4162, 4041, 4054, 4139] avg=4096.0000 max=4175 max_rank=2 min=4014 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16396 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32769, 32768, 32768, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32769 actual_valid_recv=33024 observed_loss=-255 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8064, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8064, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32767 actual_valid_recv=33280 observed_loss=-513 tokens_per_expert=[8064, 8192, 8192, 8320, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[972, 1026, 1026, 998, 992, 1003, 1016, 1034, 1059, 1007, 1013, 978, 1014, 1049, 1026, 1037, 1029, 985, 1037, 1035, 1049, 1015, 1003, 1022, 1014, 1034, 1057, 1057, 1049, 1033, 1033, 1066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1029, 1051, 1040, 1072, 973, 1069, 1014, 1060, 1049, 1106, 981, 1007, 1039, 1018, 997, 1002, 995, 1008, 997, 1015, 1004, 1053, 1013, 1018, 1017, 1000, 1002, 1018, 1003, 999, 1073, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1020, 1021, 1008, 1022, 1004, 1053, 995, 1026, 1040, 1010, 1040, 1034, 1028, 1058, 1036, 1007, 1065, 1028, 1001, 1054, 1028, 1000, 1020, 1042, 1017, 1083, 1015, 1032, 980, 989, 974, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1038, 1018, 1012, 1026, 1042, 1014, 1024, 1000, 1023, 1027, 1042, 989, 1013, 1012, 1038, 1010, 978, 1089, 1040, 1017, 1039, 1016, 1018, 1031, 1035, 1019, 1038, 1014, 1007, 1066, 1017, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1025, 1067, 1048, 1057, 1055, 1007, 1014, 1032, 1043, 1073, 1037, 983, 977, 1024, 1028, 1030, 1049, 1041, 1001, 1002, 997, 1017, 981, 1008, 1021, 1025, 1019, 1001, 1051, 1029, 988, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[993, 1033, 1034, 1011, 973, 1005, 1008, 1063, 1070, 1046, 1012, 1005, 1030, 994, 1039, 967, 969, 1037, 1062, 1025, 1022, 1048, 1059, 1054, 1041, 1014, 1064, 998, 1037, 983, 997, 1075] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[989, 1083, 954, 1047, 1019, 1003, 1015, 963, 1010, 1059, 1027, 997, 1101, 997, 1009, 1027, 1035, 998, 1023, 967, 1029, 1060, 1023, 1018, 961, 1033, 1005, 1003, 1108, 1089, 1071, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1021, 1076, 1051, 1005, 969, 1052, 1044, 1035, 1028, 1009, 1015, 1075, 1081, 1050, 1002, 979, 1047, 1038, 1014, 934, 990, 1019, 1025, 1000, 1008, 991, 1026, 1062, 1001, 1005, 1051, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[972, 1015, 1026, 998, 0, 0, 0, 0, 992, 1003, 1016, 1034, 24, 0, 0, 0, 1059, 993, 1013, 978, 0, 0, 0, 0, 1014, 1049, 1026, 1037, 4, 1, 0, 0, 1029, 985, 1037, 1035, 10, 7, 0, 0, 1049, 1015, 1003, 1022, 5, 0, 0, 0, 1014, 1034, 1057, 1057, 5, 0, 0, 0, 1049, 1033, 1033, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4069, 4043, 4131, 4103, 4094, 4167, 4150] avg=4096.0000 max=4167 max_rank=6 min=4011 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16514 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1029, 1040, 1040, 1072, 0, 0, 0, 0, 973, 1069, 1014, 1060, 23, 0, 0, 0, 1049, 1090, 981, 1007, 0, 0, 0, 0, 1039, 1018, 997, 1002, 4, 1, 0, 0, 995, 1008, 997, 1015, 12, 7, 0, 0, 1004, 1053, 1013, 1018, 5, 0, 0, 0, 1017, 1000, 1002, 1018, 5, 0, 0, 0, 1003, 999, 1073, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4139, 4127, 4061, 4034, 4093, 4042, 4091] avg=4096.0000 max=4181 max_rank=0 min=4034 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1038, 1007, 1012, 1026, 0, 0, 0, 0, 1042, 1014, 1024, 1000, 23, 0, 0, 0, 1023, 1012, 1042, 989, 0, 0, 0, 0, 1013, 1012, 1038, 1010, 4, 1, 0, 0, 978, 1089, 1040, 1017, 11, 7, 0, 0, 1039, 1016, 1018, 1031, 5, 0, 0, 0, 1035, 1019, 1038, 1014, 5, 0, 0, 0, 1007, 1066, 1017, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4103, 4066, 4078, 4142, 4109, 4111, 4076] avg=4096.0000 max=4142 max_rank=4 min=4066 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1025, 1056, 1048, 1057, 0, 0, 0, 0, 1055, 1007, 1014, 1032, 23, 0, 0, 0, 1043, 1058, 1037, 983, 0, 0, 0, 0, 977, 1024, 1028, 1030, 4, 1, 0, 0, 1049, 1041, 1001, 1002, 11, 7, 0, 0, 997, 1017, 981, 1008, 5, 0, 0, 0, 1021, 1025, 1019, 1001, 5, 0, 0, 0, 1051, 1029, 988, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4131, 4121, 4064, 4111, 4008, 4071, 4076] avg=4096.0000 max=4186 max_rank=0 min=4008 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1020, 995, 1008, 1022, 0, 0, 0, 0, 1004, 1053, 995, 1026, 28, 0, 0, 0, 1040, 989, 1040, 1034, 0, 0, 0, 0, 1028, 1058, 1036, 1007, 7, 7, 0, 0, 1065, 1028, 1001, 1054, 14, 13, 0, 0, 1028, 1000, 1020, 1042, 10, 0, 0, 0, 1017, 1083, 1015, 1032, 9, 0, 0, 0, 980, 989, 974, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4106, 4103, 4143, 4175, 4100, 4156, 3940] avg=4096.0000 max=4175 max_rank=4 min=3940 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[993, 1022, 1034, 1011, 0, 0, 0, 0, 973, 1005, 1008, 1063, 24, 0, 0, 0, 1070, 1031, 1012, 1005, 0, 0, 0, 0, 1030, 994, 1039, 967, 4, 1, 0, 0, 969, 1037, 1062, 1025, 11, 7, 0, 0, 1022, 1048, 1059, 1054, 5, 0, 0, 0, 1041, 1014, 1064, 998, 5, 0, 0, 0, 1037, 983, 997, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4073, 4118, 4035, 4111, 4188, 4122, 4061] avg=4096.0000 max=4188 max_rank=5 min=4035 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[989, 1071, 954, 1047, 0, 0, 0, 0, 1019, 1003, 1015, 963, 23, 0, 0, 0, 1010, 1044, 1027, 997, 0, 0, 0, 0, 1101, 997, 1009, 1027, 4, 1, 0, 0, 1035, 998, 1023, 967, 11, 7, 0, 0, 1029, 1060, 1023, 1018, 6, 0, 0, 0, 961, 1033, 1005, 1003, 5, 0, 0, 0, 1108, 1089, 1071, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4023, 4078, 4139, 4041, 4136, 4007, 4283] avg=4096.0000 max=4283 max_rank=7 min=4007 min_rank=6 imbalance_ratio=1.045654 max_load_reduction=28485 max_load_reduction_pct=0.869293 moved_to_echo=16467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1021, 1064, 1051, 1005, 0, 0, 0, 0, 969, 1052, 1044, 1035, 24, 0, 0, 0, 1028, 994, 1015, 1075, 0, 0, 0, 0, 1081, 1050, 1002, 979, 4, 1, 0, 0, 1047, 1038, 1014, 934, 11, 7, 0, 0, 990, 1019, 1025, 1000, 6, 0, 0, 0, 1008, 991, 1026, 1062, 5, 0, 0, 0, 1001, 1005, 1051, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4124, 4112, 4117, 4051, 4040, 4092, 4091] avg=4096.0000 max=4141 max_rank=0 min=4040 min_rank=5 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1034, 1019, 1013, 1043, 1032, 1040, 987, 1007, 1017, 1048, 1002, 1021, 1071, 1003, 994, 1015, 1041, 1028, 1027, 982, 1009, 1046, 1052, 1042, 1042, 1020, 1031, 1033, 1018, 990, 1074, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1002, 1039, 985, 993, 1041, 1041, 1046, 1036, 1041, 1029, 1060, 996, 1023, 1010, 1015, 1012, 1010, 1011, 1000, 958, 1100, 1047, 992, 1025, 1008, 983, 1052, 1060, 1015, 1059, 1068, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1002, 1028, 1034, 1020, 1007, 1043, 1058, 1039, 1051, 992, 1011, 1014, 966, 1042, 1061, 1012, 1056, 1032, 1014, 1008, 1024, 1031, 1110, 1016, 1012, 996, 997, 1011, 998, 1017, 1041, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1020, 963, 1030, 1032, 1056, 982, 1009, 995, 1019, 999, 1056, 1019, 1018, 983, 1046, 1006, 1043, 1004, 994, 1024, 1032, 1025, 1043, 1066, 1058, 1041, 1045, 999, 1074, 1008, 1059, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1028, 1031, 987, 1012, 1029, 1026, 1041, 1020, 998, 1048, 1033, 1004, 1005, 1008, 1059, 1042, 1041, 1013, 1057, 1061, 1016, 1027, 1040, 1031, 959, 1027, 1042, 1028, 1002, 1018, 1011, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[996, 1046, 1041, 1002, 1039, 1001, 1061, 1010, 983, 1079, 1036, 998, 1050, 1035, 1010, 1003, 1027, 1036, 1035, 967, 1005, 1019, 1024, 1012, 1064, 961, 1023, 1040, 1063, 1089, 1044, 969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1030, 1025, 1099, 1025, 1065, 1007, 1050, 946, 1031, 985, 1012, 1038, 1045, 1022, 1017, 1021, 1061, 1016, 1029, 1039, 1006, 1044, 1013, 1003, 984, 978, 1027, 1017, 996, 1067, 1047, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1024, 979, 999, 1057, 1034, 1024, 1018, 1012, 1008, 1012, 1048, 1019, 1064, 1014, 1029, 1041, 1011, 1038, 995, 1020, 999, 1020, 1038, 1042, 1022, 1034, 1035, 1042, 1040, 1001, 1017, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1034, 1019, 1013, 1043, 16, 0, 0, 0, 1029, 1040, 987, 1007, 0, 0, 0, 0, 1017, 1048, 1002, 1021, 6, 0, 0, 0, 1071, 1003, 994, 1015, 3, 0, 0, 0, 1041, 1028, 1027, 982, 11, 0, 0, 0, 1009, 1046, 1024, 1042, 0, 0, 0, 0, 1042, 1020, 1031, 1033, 12, 0, 0, 0, 1018, 990, 1057, 987, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4063, 4094, 4086, 4089, 4121, 4138, 4052] avg=4096.0000 max=4138 max_rank=6 min=4052 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1020, 963, 1030, 1032, 16, 0, 0, 0, 1052, 982, 1009, 995, 0, 0, 0, 0, 1019, 999, 1056, 1019, 6, 1, 0, 0, 1018, 983, 1046, 1006, 3, 0, 0, 0, 1043, 1004, 994, 1024, 10, 0, 0, 0, 1032, 1025, 1015, 1066, 0, 0, 0, 0, 1058, 1041, 1045, 999, 12, 0, 0, 0, 1074, 1008, 1043, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4038, 4100, 4056, 4075, 4138, 4155, 4145] avg=4096.0000 max=4155 max_rank=6 min=4038 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1002, 1039, 985, 993, 15, 0, 0, 0, 1037, 1041, 1046, 1036, 0, 0, 0, 0, 1041, 1029, 1060, 996, 6, 1, 0, 0, 1023, 1010, 1015, 1012, 3, 0, 0, 0, 1010, 1011, 1000, 958, 10, 0, 0, 0, 1100, 1047, 966, 1025, 0, 0, 0, 0, 1008, 983, 1052, 1060, 11, 0, 0, 0, 1015, 1059, 1052, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4160, 4133, 4063, 3989, 4138, 4114, 4137] avg=4096.0000 max=4160 max_rank=1 min=3989 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1002, 1028, 1034, 1020, 17, 0, 0, 0, 1004, 1043, 1058, 1039, 0, 0, 0, 0, 1051, 992, 1011, 1014, 6, 0, 0, 0, 966, 1042, 1061, 1012, 3, 0, 0, 0, 1056, 1032, 1014, 1008, 10, 0, 0, 0, 1024, 1031, 1081, 1016, 0, 0, 0, 0, 1012, 996, 997, 1011, 12, 0, 0, 0, 998, 1017, 1025, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4144, 4074, 4084, 4120, 4152, 4028, 4065] avg=4096.0000 max=4152 max_rank=5 min=4028 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1030, 1025, 1099, 1025, 15, 0, 0, 0, 1061, 1007, 1050, 946, 0, 0, 0, 0, 1031, 985, 1012, 1038, 6, 1, 0, 0, 1045, 1022, 1017, 1021, 3, 0, 0, 0, 1061, 1016, 1029, 1039, 10, 0, 0, 0, 1006, 1044, 987, 1003, 0, 0, 0, 0, 984, 978, 1027, 1017, 11, 0, 0, 0, 996, 1067, 1031, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4064, 4073, 4108, 4155, 4040, 4017, 4117] avg=4096.0000 max=4194 max_rank=0 min=4017 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[996, 1046, 1041, 1002, 16, 0, 0, 0, 1035, 1001, 1061, 1010, 0, 0, 0, 0, 983, 1079, 1036, 998, 6, 1, 0, 0, 1050, 1035, 1010, 1003, 3, 0, 0, 0, 1027, 1036, 1035, 967, 10, 0, 0, 0, 1005, 1019, 997, 1012, 0, 0, 0, 0, 1064, 961, 1023, 1040, 11, 0, 0, 0, 1063, 1089, 1028, 969, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4107, 4103, 4101, 4075, 4033, 4099, 4149] avg=4096.0000 max=4149 max_rank=7 min=4033 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1028, 1031, 987, 1012, 19, 0, 0, 0, 1020, 1026, 1041, 1020, 0, 0, 0, 0, 998, 1048, 1033, 1004, 11, 4, 0, 0, 1005, 1008, 1059, 1042, 5, 0, 0, 0, 1041, 1013, 1057, 1061, 15, 4, 0, 0, 1016, 1027, 1001, 1031, 0, 0, 0, 0, 959, 1027, 1042, 1028, 16, 0, 0, 0, 1002, 1018, 985, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4107, 4098, 4119, 4191, 4075, 4072, 4029] avg=4096.0000 max=4191 max_rank=4 min=4029 min_rank=7 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1024, 979, 999, 1057, 16, 0, 0, 0, 1031, 1024, 1018, 1012, 0, 0, 0, 0, 1008, 1012, 1048, 1019, 6, 0, 0, 0, 1064, 1014, 1029, 1041, 3, 0, 0, 0, 1011, 1038, 995, 1020, 10, 0, 0, 0, 999, 1020, 1010, 1042, 0, 0, 0, 0, 1022, 1034, 1035, 1042, 12, 0, 0, 0, 1040, 1001, 1001, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4085, 4093, 4151, 4074, 4071, 4145, 4074] avg=4096.0000 max=4151 max_rank=3 min=4071 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1126, 1041, 975, 988, 1018, 1057, 1073, 1017, 984, 1024, 1041, 1022, 1045, 1038, 1022, 1001, 1037, 1016, 1024, 997, 1042, 1016, 994, 988, 1021, 1001, 982, 1064, 985, 1038, 1041, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1071, 1040, 1006, 1040, 995, 1038, 1043, 1038, 998, 1000, 1027, 1001, 1022, 1065, 1023, 1015, 970, 1063, 978, 991, 1086, 1072, 1011, 1020, 1033, 1030, 1014, 986, 1023, 995, 1042, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1060, 1051, 1006, 968, 994, 1006, 1067, 1056, 1038, 1048, 1003, 985, 1007, 1037, 1025, 1021, 1032, 1045, 992, 971, 1029, 1022, 1020, 1024, 1004, 1061, 1054, 1021, 1040, 1023, 1049, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[984, 1031, 1065, 986, 965, 1043, 1063, 1038, 984, 1051, 1017, 1024, 1021, 1031, 988, 1032, 1009, 1090, 1061, 1042, 1022, 1025, 999, 1023, 993, 979, 1023, 1036, 1064, 1032, 1064, 983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[976, 989, 1017, 1047, 1013, 1045, 1018, 1048, 1017, 1018, 1041, 1014, 1065, 999, 1049, 1087, 975, 980, 976, 961, 1036, 992, 1047, 1052, 1050, 1010, 994, 1059, 1083, 1009, 1036, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1070, 1049, 992, 1033, 997, 1085, 988, 1036, 1020, 1030, 1061, 975, 1033, 1058, 1016, 1027, 1013, 1002, 1036, 1041, 1010, 972, 1052, 991, 992, 1013, 1070, 1005, 1011, 1015, 1034, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1062, 980, 1068, 1013, 1056, 1034, 1014, 1008, 1012, 1005, 1037, 1016, 1038, 1043, 1025, 1048, 994, 969, 1009, 1014, 1048, 1045, 1020, 1012, 1023, 992, 1052, 1030, 987, 1033, 1023, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1016, 1030, 985, 1026, 1011, 983, 1007, 999, 1051, 1019, 1054, 1003, 1049, 1025, 1000, 1016, 1051, 1051, 1004, 1008, 1091, 1010, 1002, 1016, 1004, 1040, 1005, 1052, 1056, 1057, 1031, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1123, 1041, 975, 988, 0, 0, 0, 0, 1018, 1048, 1073, 1017, 0, 0, 0, 0, 984, 1024, 1041, 1022, 11, 6, 0, 0, 1045, 1014, 1022, 1001, 0, 0, 0, 0, 1037, 1016, 1024, 997, 32, 13, 0, 0, 1040, 1016, 994, 988, 0, 0, 0, 0, 1021, 1001, 982, 1064, 3, 3, 2, 0, 985, 1038, 1009, 1050, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4156, 4088, 4082, 4119, 4038, 4076, 4082] avg=4096.0000 max=4156 max_rank=1 min=4038 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1058, 1051, 1006, 968, 0, 0, 0, 0, 994, 997, 1067, 1056, 0, 0, 0, 0, 1038, 1048, 1003, 985, 11, 6, 0, 0, 1007, 1013, 1025, 1021, 0, 0, 0, 0, 1032, 1045, 992, 971, 32, 13, 0, 0, 1027, 1022, 1020, 1024, 0, 0, 0, 0, 1004, 1061, 1054, 1021, 3, 2, 2, 0, 1040, 1023, 1017, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4114, 4091, 4066, 4085, 4093, 4147, 4089] avg=4096.0000 max=4147 max_rank=6 min=4066 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1068, 1049, 992, 1033, 0, 0, 0, 0, 997, 1074, 988, 1036, 0, 0, 0, 0, 1020, 1030, 1061, 975, 11, 7, 0, 0, 1033, 1034, 1016, 1027, 0, 0, 0, 0, 1013, 1002, 1036, 1041, 31, 13, 0, 0, 1008, 972, 1052, 991, 0, 0, 0, 0, 992, 1013, 1070, 1005, 4, 2, 2, 0, 1011, 1015, 1003, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4095, 4104, 4110, 4136, 4023, 4088, 4070] avg=4096.0000 max=4142 max_rank=0 min=4023 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1069, 1040, 1006, 1040, 0, 0, 0, 0, 995, 1029, 1043, 1038, 0, 0, 0, 0, 998, 1000, 1027, 1001, 12, 6, 0, 0, 1022, 1040, 1023, 1015, 0, 0, 0, 0, 970, 1063, 978, 991, 32, 13, 0, 0, 1084, 1072, 1011, 1020, 0, 0, 0, 0, 1033, 1030, 1014, 986, 3, 2, 2, 0, 1023, 995, 1010, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4105, 4044, 4100, 4047, 4187, 4070, 4060] avg=4096.0000 max=4187 max_rank=5 min=4044 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[982, 1031, 1065, 986, 0, 0, 0, 0, 965, 1034, 1063, 1038, 0, 0, 0, 0, 984, 1051, 1017, 1024, 11, 6, 0, 0, 1021, 1007, 988, 1032, 0, 0, 0, 0, 1009, 1090, 1061, 1042, 32, 13, 0, 0, 1020, 1025, 999, 1023, 0, 0, 0, 0, 993, 979, 1023, 1036, 3, 2, 2, 0, 1064, 1032, 1032, 983, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4100, 4093, 4048, 4247, 4067, 4038, 4111] avg=4096.0000 max=4247 max_rank=4 min=4038 min_rank=6 imbalance_ratio=1.036865 max_load_reduction=28521 max_load_reduction_pct=0.870392 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1060, 980, 1068, 1013, 0, 0, 0, 0, 1056, 1025, 1014, 1008, 0, 0, 0, 0, 1012, 1005, 1037, 1016, 11, 6, 0, 0, 1038, 1019, 1025, 1048, 0, 0, 0, 0, 994, 969, 1009, 1014, 31, 13, 0, 0, 1046, 1045, 1020, 1012, 0, 0, 0, 0, 1023, 992, 1052, 1030, 3, 2, 2, 0, 987, 1033, 992, 1058, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4103, 4087, 4130, 4030, 4123, 4104, 4070] avg=4096.0000 max=4130 max_rank=3 min=4030 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[968, 989, 1017, 1047, 0, 0, 0, 0, 1013, 1025, 1018, 1048, 0, 0, 0, 0, 1017, 1018, 1041, 1014, 16, 11, 0, 0, 1065, 965, 1049, 1087, 0, 0, 0, 0, 975, 980, 976, 961, 36, 18, 0, 0, 1029, 992, 1047, 1052, 0, 0, 0, 0, 1050, 1010, 994, 1059, 9, 8, 7, 0, 1083, 1009, 1000, 1065, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 4104, 4117, 4166, 3946, 4120, 4137, 4157] avg=4096.0000 max=4166 max_rank=3 min=3946 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1014, 1030, 985, 1026, 0, 0, 0, 0, 1011, 974, 1007, 999, 0, 0, 0, 0, 1051, 1019, 1054, 1003, 11, 6, 0, 0, 1049, 1001, 1000, 1016, 0, 0, 0, 0, 1051, 1051, 1004, 1008, 31, 13, 0, 0, 1089, 1010, 1002, 1016, 0, 0, 0, 0, 1004, 1040, 1005, 1052, 3, 2, 2, 0, 1056, 1057, 1000, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 3991, 4144, 4066, 4158, 4117, 4108, 4129] avg=4096.0000 max=4158 max_rank=4 min=3991 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8064, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1040, 986, 1036, 1050, 993, 1049, 1027, 1052, 1065, 1017, 1016, 989, 1059, 968, 968, 1033, 1055, 1022, 1011, 1017, 1025, 1049, 1045, 1017, 1031, 1017, 978, 1077, 967, 1015, 1054, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1019, 1022, 995, 1014, 1045, 1003, 1037, 1049, 1044, 1053, 1029, 1007, 1053, 1007, 1004, 1026, 1053, 1004, 1009, 1032, 997, 1061, 1049, 1052, 1021, 1017, 1029, 1008, 998, 1001, 999, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1031, 1040, 1052, 1024, 1008, 971, 1049, 1034, 1008, 1026, 1023, 1031, 1011, 1059, 988, 994, 1028, 1016, 1054, 989, 1038, 1030, 991, 1056, 974, 1025, 1085, 1002, 1006, 1000, 1000, 1125] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1039, 1011, 1038, 1045, 1027, 950, 1048, 1058, 987, 992, 1017, 1019, 1014, 1067, 1057, 1001, 1026, 1014, 1073, 984, 1034, 1008, 1047, 1038, 1035, 1021, 1005, 1017, 1005, 1023, 1050, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1048, 1059, 1031, 978, 1060, 1031, 991, 1052, 1041, 1045, 986, 1061, 1038, 1026, 1012, 1009, 1014, 975, 990, 989, 972, 989, 1011, 1053, 1052, 1013, 1038, 1060, 1075, 1051, 1043, 975] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1013, 1039, 973, 995, 1015, 971, 1000, 1048, 1071, 1043, 1061, 1027, 1024, 1030, 1086, 1009, 1004, 1018, 1002, 1058, 1012, 1066, 991, 1065, 989, 1056, 985, 1007, 1050, 1004, 1020, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1030, 997, 1047, 1035, 1037, 1017, 1040, 1032, 1005, 1001, 1045, 1036, 1035, 1003, 976, 993, 1012, 1037, 1036, 1037, 1021, 1020, 1078, 1042, 981, 988, 985, 1045, 1045, 1051, 1051, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1029, 1048, 1032, 1012, 1007, 1068, 976, 1055, 978, 1028, 1031, 986, 1053, 1039, 1012, 988, 984, 1013, 994, 1008, 1025, 1028, 1043, 1036, 1017, 1007, 1061, 1066, 999, 1060, 1044, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1035, 986, 1036, 1050, 0, 0, 0, 0, 993, 1049, 1027, 1048, 0, 0, 0, 0, 1065, 1017, 1016, 989, 0, 0, 0, 0, 1059, 968, 968, 1033, 14, 1, 0, 0, 1055, 1022, 1011, 1017, 25, 0, 0, 0, 1025, 1049, 1045, 991, 0, 0, 0, 0, 1031, 1017, 978, 1077, 5, 4, 0, 0, 967, 1015, 1054, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4117, 4087, 4043, 4130, 4110, 4112, 4062] avg=4096.0000 max=4130 max_rank=4 min=4043 min_rank=3 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16414 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1027, 1040, 1052, 1024, 0, 0, 0, 0, 1008, 971, 1049, 1031, 0, 0, 0, 0, 1008, 1026, 1023, 1031, 0, 0, 0, 0, 1011, 1059, 988, 994, 15, 1, 0, 0, 1028, 1016, 1054, 989, 26, 0, 0, 0, 1038, 1030, 991, 1029, 0, 0, 0, 0, 974, 1025, 1085, 1002, 4, 3, 0, 0, 1006, 1000, 1000, 1110, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4059, 4088, 4068, 4113, 4088, 4093, 4116] avg=4096.0000 max=4143 max_rank=0 min=4059 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16410 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1043, 1059, 1031, 978, 0, 0, 0, 0, 1060, 1031, 991, 1048, 0, 0, 0, 0, 1041, 1045, 986, 1061, 0, 0, 0, 0, 1038, 1026, 1012, 1009, 13, 1, 0, 0, 1014, 975, 990, 989, 26, 0, 0, 0, 972, 989, 1011, 1026, 0, 0, 0, 0, 1052, 1013, 1038, 1060, 5, 4, 0, 0, 1075, 1051, 1043, 962, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4130, 4133, 4099, 3994, 3998, 4172, 4131] avg=4096.0000 max=4172 max_rank=6 min=3994 min_rank=4 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16295 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1010, 1022, 995, 1014, 0, 0, 0, 0, 1045, 1003, 1037, 1043, 0, 0, 0, 0, 1044, 1053, 1029, 1007, 0, 0, 0, 0, 1053, 1007, 1004, 1026, 17, 4, 0, 0, 1053, 1004, 1009, 1032, 29, 0, 0, 0, 997, 1061, 1049, 1019, 0, 0, 0, 0, 1021, 1017, 1029, 1008, 9, 6, 4, 0, 998, 1001, 999, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4128, 4133, 4111, 4127, 4126, 4094, 4008] avg=4096.0000 max=4133 max_rank=2 min=4008 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16355 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1034, 1011, 1038, 1045, 0, 0, 0, 0, 1027, 950, 1048, 1054, 0, 0, 0, 0, 987, 992, 1017, 1019, 0, 0, 0, 0, 1014, 1067, 1057, 1001, 14, 1, 0, 0, 1026, 1014, 1073, 984, 26, 0, 0, 0, 1034, 1008, 1047, 1011, 0, 0, 0, 0, 1035, 1021, 1005, 1017, 5, 4, 0, 0, 1005, 1023, 1050, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4079, 4015, 4154, 4123, 4100, 4087, 4082] avg=4096.0000 max=4154 max_rank=3 min=4015 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16392 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1009, 1039, 973, 995, 0, 0, 0, 0, 1015, 971, 1000, 1044, 0, 0, 0, 0, 1071, 1043, 1061, 1027, 0, 0, 0, 0, 1024, 1030, 1086, 1009, 14, 1, 0, 0, 1004, 1018, 1002, 1058, 26, 0, 0, 0, 1012, 1066, 991, 1038, 0, 0, 0, 0, 989, 1056, 985, 1007, 4, 4, 0, 0, 1050, 1004, 1020, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4030, 4202, 4164, 4108, 4107, 4045, 4096] avg=4096.0000 max=4202 max_rank=2 min=4016 min_rank=0 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16356 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1026, 997, 1047, 1035, 0, 0, 0, 0, 1037, 1017, 1040, 1029, 0, 0, 0, 0, 1005, 1001, 1045, 1036, 0, 0, 0, 0, 1035, 1003, 976, 993, 14, 1, 0, 0, 1012, 1037, 1036, 1037, 26, 0, 0, 0, 1021, 1020, 1078, 1015, 0, 0, 0, 0, 981, 988, 985, 1045, 4, 3, 0, 0, 1045, 1051, 1051, 996, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4123, 4087, 4022, 4148, 4134, 4006, 4143] avg=4096.0000 max=4148 max_rank=4 min=4006 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16431 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1025, 1048, 1032, 1012, 0, 0, 0, 0, 1007, 1068, 976, 1051, 0, 0, 0, 0, 978, 1028, 1031, 986, 0, 0, 0, 0, 1053, 1039, 1012, 988, 14, 1, 0, 0, 984, 1013, 994, 1008, 26, 0, 0, 0, 1025, 1028, 1043, 1009, 0, 0, 0, 0, 1017, 1007, 1061, 1066, 4, 4, 0, 0, 999, 1060, 1044, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4102, 4023, 4107, 4025, 4105, 4159, 4130] avg=4096.0000 max=4159 max_rank=6 min=4023 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16419 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8064, 8192, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1038, 1087, 1031, 1049, 1037, 996, 996, 1023, 983, 1006, 968, 1008, 1031, 1013, 1071, 984, 986, 988, 1022, 1051, 1066, 999, 1040, 1035, 1034, 994, 1023, 1064, 1044, 1053, 1016, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1074, 987, 1029, 1070, 1026, 1004, 1035, 1009, 977, 1006, 1052, 1046, 1025, 993, 1016, 1028, 1024, 1010, 1053, 988, 1012, 1023, 1051, 996, 998, 1017, 1086, 1065, 1012, 1002, 1032, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1090, 1034, 1024, 1010, 978, 1060, 1052, 986, 1021, 990, 1053, 1045, 1023, 1014, 1092, 1039, 999, 965, 981, 1038, 1026, 1043, 1010, 1029, 1013, 1010, 1042, 999, 1027, 1037, 1005, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1044, 999, 1036, 1001, 983, 996, 1050, 1016, 1053, 950, 1046, 982, 1088, 1029, 1034, 1038, 1025, 1021, 1024, 1102, 1076, 1026, 1034, 987, 1008, 964, 1045, 1002, 1007, 1037, 1052, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1025, 985, 1036, 1031, 1008, 1058, 1026, 1062, 1011, 1002, 1060, 993, 1059, 1010, 1015, 977, 1032, 1022, 1019, 996, 1034, 1053, 1017, 1013, 1043, 1000, 1007, 1033, 1056, 1010, 1026, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[999, 1030, 1041, 1047, 1034, 1088, 1011, 1066, 1012, 1034, 1036, 980, 983, 964, 977, 1009, 1011, 1037, 1064, 1027, 1053, 1027, 1023, 1000, 1021, 1021, 992, 997, 1027, 1064, 1012, 1081] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[973, 985, 973, 1026, 1018, 1027, 1016, 1043, 1076, 1111, 1004, 1008, 1053, 1050, 1041, 995, 1058, 1018, 1026, 1011, 979, 1055, 1067, 1047, 1029, 1035, 996, 1030, 959, 998, 1062, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1022, 1028, 1005, 1058, 1034, 1046, 1054, 1002, 977, 1003, 1041, 959, 991, 1046, 1059, 1020, 1023, 1031, 1017, 965, 1073, 1043, 965, 1066, 1018, 1036, 1025, 1019, 1041, 1036, 1038, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1074, 987, 1029, 1058, 0, 0, 0, 0, 1026, 996, 1035, 1009, 0, 0, 0, 0, 977, 1006, 1052, 1046, 24, 9, 0, 0, 1025, 993, 1016, 1028, 0, 0, 0, 0, 1024, 1010, 1053, 988, 8, 8, 0, 0, 988, 1023, 1051, 996, 0, 0, 0, 0, 998, 1017, 1086, 1065, 8, 4, 0, 0, 1012, 1002, 1032, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4066, 4114, 4062, 4091, 4058, 4178, 4051] avg=4096.0000 max=4178 max_rank=6 min=4051 min_rank=7 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1038, 1087, 1031, 1038, 0, 0, 0, 0, 1037, 988, 996, 1023, 0, 0, 0, 0, 983, 1006, 968, 1008, 25, 9, 0, 0, 1031, 1013, 1071, 984, 0, 0, 0, 0, 986, 988, 1022, 1051, 8, 8, 0, 0, 1041, 999, 1040, 1035, 0, 0, 0, 0, 1034, 994, 1023, 1064, 8, 3, 0, 0, 1044, 1053, 1016, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4044, 3999, 4099, 4063, 4115, 4126, 4128] avg=4096.0000 max=4194 max_rank=0 min=3999 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1044, 999, 1036, 990, 0, 0, 0, 0, 983, 988, 1050, 1016, 0, 0, 0, 0, 1053, 950, 1046, 982, 25, 9, 0, 0, 1088, 1029, 1034, 1038, 0, 0, 0, 0, 1025, 1021, 1024, 1102, 8, 8, 0, 0, 1051, 1026, 1034, 987, 0, 0, 0, 0, 1008, 964, 1045, 1002, 8, 3, 0, 0, 1007, 1037, 1052, 996, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4037, 4065, 4189, 4188, 4098, 4030, 4092] avg=4096.0000 max=4189 max_rank=3 min=4030 min_rank=6 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[973, 985, 973, 1015, 0, 0, 0, 0, 1018, 1019, 1016, 1043, 0, 0, 0, 0, 1076, 1111, 1004, 1008, 23, 9, 0, 0, 1053, 1050, 1041, 995, 0, 0, 0, 0, 1058, 1018, 1026, 1011, 8, 7, 0, 0, 956, 1055, 1067, 1047, 0, 0, 0, 0, 1029, 1035, 996, 1030, 8, 3, 0, 0, 959, 998, 1062, 983, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3946, 4096, 4231, 4139, 4128, 4125, 4101, 4002] avg=4096.0000 max=4231 max_rank=2 min=3946 min_rank=0 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1090, 1034, 1024, 989, 0, 0, 0, 0, 978, 1046, 1052, 986, 0, 0, 0, 0, 1021, 990, 1053, 1045, 29, 12, 0, 0, 1023, 1014, 1092, 1039, 1, 0, 0, 0, 999, 965, 981, 1038, 12, 11, 0, 0, 997, 1043, 1010, 1029, 0, 0, 0, 0, 1013, 1010, 1042, 999, 13, 9, 0, 0, 1027, 1037, 1005, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4062, 4150, 4169, 4006, 4079, 4086, 4079] avg=4096.0000 max=4169 max_rank=3 min=4006 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 985, 1036, 1020, 0, 0, 0, 0, 1008, 1049, 1026, 1062, 0, 0, 0, 0, 1011, 1002, 1060, 993, 24, 9, 0, 0, 1059, 1010, 1015, 977, 0, 0, 0, 0, 1032, 1022, 1019, 996, 8, 8, 0, 0, 1010, 1053, 1017, 1013, 0, 0, 0, 0, 1043, 1000, 1007, 1033, 9, 3, 0, 0, 1056, 1010, 1026, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4145, 4099, 4061, 4085, 4093, 4095, 4124] avg=4096.0000 max=4145 max_rank=1 min=4061 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[999, 1030, 1041, 1036, 0, 0, 0, 0, 1034, 1079, 1011, 1066, 0, 0, 0, 0, 1012, 1034, 1036, 980, 25, 9, 0, 0, 983, 964, 977, 1009, 0, 0, 0, 0, 1011, 1037, 1064, 1027, 8, 8, 0, 0, 1028, 1027, 1023, 1000, 0, 0, 0, 0, 1021, 1021, 992, 997, 9, 3, 0, 0, 1027, 1064, 1012, 1064, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4190, 4096, 3933, 4155, 4078, 4043, 4167] avg=4096.0000 max=4190 max_rank=1 min=3933 min_rank=3 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1022, 1028, 1005, 1047, 0, 0, 0, 0, 1034, 1038, 1054, 1002, 0, 0, 0, 0, 977, 1003, 1041, 959, 25, 9, 0, 0, 991, 1046, 1059, 1020, 0, 0, 0, 0, 1023, 1031, 1017, 965, 8, 8, 0, 0, 1048, 1043, 965, 1066, 0, 0, 0, 0, 1018, 1036, 1025, 1019, 8, 3, 0, 0, 1041, 1036, 1038, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4128, 4014, 4116, 4052, 4122, 4109, 4125] avg=4096.0000 max=4128 max_rank=1 min=4014 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1042, 1045, 989, 973, 1017, 1053, 1051, 978, 1031, 1035, 1020, 987, 1014, 1043, 987, 1058, 1038, 988, 1005, 1020, 1007, 1038, 1033, 1077, 1050, 987, 1025, 1080, 1040, 1011, 1053, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1010, 1045, 987, 1049, 1032, 1071, 992, 1021, 1074, 1018, 1028, 968, 1030, 998, 1023, 987, 1014, 1031, 1060, 1008, 1052, 1014, 1015, 981, 1017, 1019, 1042, 1018, 1067, 1049, 1014, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1064, 979, 987, 1010, 1000, 1057, 994, 980, 984, 989, 1050, 1057, 1041, 1011, 1034, 1079, 1035, 1005, 1007, 1038, 1014, 1011, 1058, 1070, 1022, 1013, 998, 1013, 1065, 1058, 1009, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1084, 1013, 1042, 1015, 1067, 1038, 961, 1064, 1030, 1015, 984, 1038, 1049, 1025, 1019, 1043, 995, 1002, 1052, 1009, 1003, 1026, 1034, 1004, 1001, 1044, 979, 1020, 1020, 1045, 1005, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1045, 1012, 988, 1005, 1002, 972, 1052, 1011, 958, 1035, 1047, 1035, 990, 993, 993, 1056, 1051, 1063, 1044, 1051, 1059, 1009, 1036, 1062, 1003, 1041, 1026, 1033, 980, 1018, 1017, 1081] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[946, 1009, 1053, 989, 1052, 1011, 1039, 1035, 998, 1019, 954, 1034, 1043, 1057, 1029, 997, 1058, 1037, 1009, 1052, 1048, 1061, 1076, 1017, 1029, 1029, 1039, 998, 991, 1037, 988, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1027, 1033, 1007, 990, 1021, 1051, 1036, 1001, 1042, 986, 1025, 1017, 1048, 1030, 1010, 1044, 1034, 1006, 1020, 1025, 1012, 1072, 1027, 935, 993, 1059, 1051, 1118, 1013, 995, 1028, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[970, 1018, 1103, 1021, 1023, 1014, 975, 1048, 1048, 1019, 1020, 1039, 1028, 1019, 1021, 1008, 1020, 1019, 1016, 1052, 1014, 1047, 1022, 1038, 1061, 1027, 996, 1044, 988, 1022, 993, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1042, 1045, 989, 973, 25, 1, 0, 0, 1017, 1053, 1051, 978, 4, 0, 0, 0, 1031, 1035, 1020, 987, 12, 11, 0, 0, 1014, 1043, 987, 1054, 0, 0, 0, 0, 1038, 988, 1005, 1009, 0, 0, 0, 0, 1007, 1038, 1008, 1077, 0, 0, 0, 0, 1050, 987, 1025, 1067, 0, 0, 0, 0, 1040, 1011, 1053, 993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4103, 4096, 4098, 4040, 4130, 4129, 4097] avg=4096.0000 max=4130 max_rank=5 min=4040 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1064, 979, 987, 1010, 26, 1, 0, 0, 1000, 1057, 994, 980, 5, 0, 0, 0, 984, 989, 1050, 1057, 11, 11, 0, 0, 1041, 1011, 1034, 1074, 0, 0, 0, 0, 1035, 1005, 1007, 1027, 0, 0, 0, 0, 1014, 1011, 1032, 1070, 0, 0, 0, 0, 1022, 1013, 998, 1001, 0, 0, 0, 0, 1065, 1058, 1009, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4036, 4102, 4160, 4074, 4127, 4034, 4168] avg=4096.0000 max=4168 max_rank=7 min=4034 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1010, 1045, 987, 1049, 24, 1, 0, 0, 1032, 1071, 992, 1021, 4, 0, 0, 0, 1074, 1018, 1028, 968, 11, 11, 0, 0, 1030, 998, 1023, 983, 0, 0, 0, 0, 1014, 1031, 1060, 997, 0, 0, 0, 0, 1052, 1014, 991, 981, 0, 0, 0, 0, 1017, 1019, 1042, 1006, 0, 0, 0, 0, 1067, 1049, 1014, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4120, 4110, 4034, 4102, 4038, 4084, 4164] avg=4096.0000 max=4164 max_rank=7 min=4034 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1045, 1012, 988, 1005, 25, 1, 0, 0, 1002, 972, 1052, 1011, 4, 0, 0, 0, 958, 1035, 1047, 1035, 11, 11, 0, 0, 990, 993, 993, 1052, 0, 0, 0, 0, 1051, 1063, 1044, 1040, 0, 0, 0, 0, 1059, 1009, 1011, 1062, 0, 0, 0, 0, 1003, 1041, 1026, 1021, 0, 0, 0, 0, 980, 1018, 1017, 1081, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4041, 4097, 4028, 4198, 4141, 4091, 4096] avg=4096.0000 max=4198 max_rank=4 min=4028 min_rank=3 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16526 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1084, 1013, 1042, 1015, 28, 7, 0, 0, 1067, 1038, 961, 1064, 10, 5, 5, 0, 1030, 1015, 984, 1038, 14, 14, 0, 0, 1049, 1025, 1019, 1033, 0, 0, 0, 0, 995, 1002, 1052, 990, 0, 0, 0, 0, 1003, 1026, 1006, 1004, 0, 0, 0, 0, 1001, 1044, 979, 999, 0, 0, 0, 0, 1020, 1045, 1005, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4189, 4150, 4095, 4126, 4039, 4039, 4023, 4107] avg=4096.0000 max=4189 max_rank=0 min=4023 min_rank=6 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16208 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[946, 1009, 1053, 989, 26, 1, 0, 0, 1052, 1011, 1039, 1035, 4, 0, 0, 0, 998, 1019, 954, 1034, 11, 11, 0, 0, 1043, 1057, 1029, 993, 0, 0, 0, 0, 1058, 1037, 1009, 1041, 0, 0, 0, 0, 1048, 1061, 1050, 1017, 0, 0, 0, 0, 1029, 1029, 1039, 986, 0, 0, 0, 0, 991, 1037, 988, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4141, 4027, 4122, 4145, 4176, 4083, 4050] avg=4096.0000 max=4176 max_rank=5 min=4024 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1027, 1033, 1007, 990, 25, 1, 0, 0, 1021, 1051, 1036, 1001, 4, 0, 0, 0, 1042, 986, 1025, 1017, 12, 11, 0, 0, 1048, 1030, 1010, 1040, 0, 0, 0, 0, 1034, 1006, 1020, 1014, 0, 0, 0, 0, 1012, 1072, 1002, 935, 0, 0, 0, 0, 993, 1059, 1051, 1105, 0, 0, 0, 0, 1013, 995, 1028, 1012, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4113, 4093, 4128, 4074, 4021, 4208, 4048] avg=4096.0000 max=4208 max_rank=6 min=4021 min_rank=5 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[970, 1018, 1103, 1021, 25, 1, 0, 0, 1023, 1014, 975, 1048, 4, 0, 0, 0, 1048, 1019, 1020, 1039, 11, 11, 0, 0, 1028, 1019, 1021, 1004, 0, 0, 0, 0, 1020, 1019, 1016, 1041, 0, 0, 0, 0, 1014, 1047, 997, 1038, 0, 0, 0, 0, 1061, 1027, 996, 1032, 0, 0, 0, 0, 988, 1022, 993, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4064, 4148, 4072, 4096, 4096, 4116, 4038] avg=4096.0000 max=4148 max_rank=2 min=4038 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[996, 998, 959, 1037, 979, 1055, 998, 1042, 1054, 1047, 1005, 1045, 986, 1017, 1048, 1014, 1014, 1010, 1012, 1003, 1047, 1087, 1047, 1042, 1055, 990, 985, 1072, 1064, 1015, 1033, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1000, 995, 1047, 1047, 1029, 1026, 990, 1021, 1004, 1018, 1045, 1015, 1019, 1040, 1066, 1019, 1010, 1068, 1024, 983, 1025, 990, 1001, 1047, 1054, 1023, 989, 1030, 1026, 1046, 1046, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[979, 941, 1025, 978, 1050, 1072, 1031, 1008, 1040, 1001, 1040, 1042, 1090, 1057, 1032, 1072, 988, 1057, 1052, 1023, 1004, 988, 1019, 1023, 1003, 1029, 1055, 1006, 976, 1022, 1037, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1025, 1022, 986, 1083, 1045, 1066, 1043, 999, 1042, 1004, 984, 1066, 1000, 997, 1043, 1011, 1024, 1068, 1012, 1056, 1033, 992, 1036, 1038, 1017, 1008, 1011, 975, 1001, 1034, 996, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1041, 1022, 1005, 1054, 1064, 1047, 1065, 1019, 1029, 1025, 1020, 1039, 1057, 976, 1045, 987, 1014, 1047, 1029, 1029, 981, 995, 1080, 1043, 1051, 987, 1030, 1016, 1018, 971, 993, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1021, 1065, 1042, 1013, 1029, 1008, 1053, 1015, 996, 1029, 1054, 983, 997, 1048, 1054, 1007, 1027, 1061, 994, 1036, 1063, 944, 984, 1023, 1017, 1082, 1003, 1004, 1039, 972, 1038, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1025, 1045, 1037, 1042, 1035, 966, 1034, 1043, 997, 1010, 1053, 982, 1034, 1031, 1045, 967, 1024, 1011, 998, 1022, 1015, 992, 1040, 1067, 1082, 1047, 1035, 997, 989, 992, 1041, 1070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[998, 1038, 1009, 1039, 1013, 1033, 1004, 994, 1020, 1046, 1020, 1042, 1010, 1011, 993, 1046, 1071, 1029, 978, 1019, 991, 1061, 945, 1013, 1054, 1030, 1010, 1043, 1098, 1016, 1018, 1076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1000, 995, 1047, 1047, 13, 5, 0, 0, 1029, 1013, 990, 1021, 0, 0, 0, 0, 1004, 1018, 1042, 1015, 0, 0, 0, 0, 1019, 1040, 1061, 1019, 0, 0, 0, 0, 1010, 1065, 1024, 983, 0, 0, 0, 0, 1025, 990, 1001, 1047, 3, 3, 3, 2, 1052, 1023, 989, 1030, 0, 0, 0, 0, 1026, 1046, 1046, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4053, 4079, 4139, 4082, 4074, 4094, 4140] avg=4096.0000 max=4140 max_rank=7 min=4053 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16390 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[996, 998, 959, 1037, 13, 5, 0, 0, 979, 1042, 998, 1042, 0, 0, 0, 0, 1054, 1047, 1002, 1045, 0, 0, 0, 0, 986, 1017, 1043, 1014, 0, 0, 0, 0, 1014, 1007, 1012, 1003, 0, 0, 0, 0, 1047, 1087, 1047, 1042, 3, 3, 3, 2, 1053, 990, 985, 1072, 0, 0, 0, 0, 1064, 1015, 1033, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4008, 4061, 4148, 4060, 4036, 4234, 4100, 4121] avg=4096.0000 max=4234 max_rank=5 min=4008 min_rank=0 imbalance_ratio=1.033691 max_load_reduction=28534 max_load_reduction_pct=0.870789 moved_to_echo=16491 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[979, 941, 1025, 978, 13, 5, 0, 0, 1050, 1059, 1031, 1008, 0, 0, 0, 0, 1040, 1001, 1037, 1042, 0, 0, 0, 0, 1090, 1057, 1027, 1072, 0, 0, 0, 0, 988, 1054, 1052, 1023, 0, 0, 0, 0, 1004, 988, 1019, 1023, 3, 3, 3, 2, 1001, 1029, 1055, 1006, 0, 0, 0, 0, 976, 1022, 1037, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3941, 4148, 4120, 4246, 4117, 4045, 4091, 4060] avg=4096.0000 max=4246 max_rank=3 min=3941 min_rank=0 imbalance_ratio=1.036621 max_load_reduction=28522 max_load_reduction_pct=0.870422 moved_to_echo=16313 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 1022, 986, 1083, 13, 5, 0, 0, 1045, 1053, 1043, 999, 0, 0, 0, 0, 1042, 1004, 981, 1066, 0, 0, 0, 0, 1000, 997, 1038, 1011, 0, 0, 0, 0, 1024, 1065, 1012, 1056, 0, 0, 0, 0, 1033, 992, 1036, 1038, 3, 3, 3, 2, 1015, 1008, 1011, 975, 0, 0, 0, 0, 1001, 1034, 996, 1048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4140, 4093, 4046, 4157, 4110, 4009, 4079] avg=4096.0000 max=4157 max_rank=4 min=4009 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16355 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 1045, 1037, 1042, 12, 5, 0, 0, 1035, 954, 1034, 1043, 0, 0, 0, 0, 997, 1010, 1050, 982, 0, 0, 0, 0, 1034, 1031, 1040, 967, 0, 0, 0, 0, 1024, 1008, 998, 1022, 0, 0, 0, 0, 1015, 992, 1040, 1067, 3, 3, 3, 2, 1080, 1047, 1035, 997, 0, 0, 0, 0, 989, 992, 1041, 1067, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4066, 4039, 4072, 4052, 4125, 4159, 4089] avg=4096.0000 max=4166 max_rank=0 min=4039 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16425 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1041, 1022, 1005, 1054, 18, 11, 0, 0, 1064, 1029, 1065, 1019, 0, 0, 0, 0, 1029, 1025, 1012, 1039, 0, 0, 0, 0, 1057, 976, 1034, 987, 0, 0, 0, 0, 1014, 1043, 1029, 1029, 0, 0, 0, 0, 981, 995, 1080, 1043, 9, 8, 4, 8, 1043, 987, 1030, 1016, 0, 0, 0, 0, 1018, 971, 993, 980, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4177, 4105, 4054, 4115, 4128, 4076, 3962] avg=4096.0000 max=4177 max_rank=1 min=3962 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16281 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1021, 1065, 1042, 1013, 13, 5, 0, 0, 1029, 995, 1053, 1015, 0, 0, 0, 0, 996, 1029, 1051, 983, 0, 0, 0, 0, 997, 1048, 1049, 1007, 0, 0, 0, 0, 1027, 1058, 994, 1036, 0, 0, 0, 0, 1063, 944, 984, 1023, 3, 3, 3, 2, 1015, 1082, 1003, 1004, 0, 0, 0, 0, 1039, 972, 1038, 1064, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4092, 4059, 4101, 4115, 4025, 4104, 4113] avg=4096.0000 max=4159 max_rank=0 min=4025 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16357 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[998, 1038, 1009, 1039, 13, 5, 0, 0, 1013, 1020, 1004, 994, 0, 0, 0, 0, 1020, 1046, 1017, 1042, 0, 0, 0, 0, 1010, 1011, 988, 1046, 0, 0, 0, 0, 1071, 1026, 978, 1019, 0, 0, 0, 0, 991, 1061, 945, 1013, 4, 3, 3, 2, 1052, 1030, 1010, 1043, 0, 0, 0, 0, 1098, 1016, 1018, 1072, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4031, 4125, 4055, 4094, 4022, 4135, 4204] avg=4096.0000 max=4204 max_rank=7 min=4022 min_rank=5 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16455 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32773, 32768, 32763, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32773, 32768, 32763, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32773, 32768, 32763, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32773, 32768, 32763, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32773, 32768, 32763, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32773, 32768, 32763, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32773, 32768, 32763, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32773, 32768, 32763, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32773 actual_valid_recv=33024 observed_loss=-251 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8448, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32763 actual_valid_recv=33280 observed_loss=-517 tokens_per_expert=[8192, 8064, 8192, 8320, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-08-30 04:58:05] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 629.5 | throughput per GPU (TFLOP/s/GPU): 529.6 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.036231E+01 | loss scale: 1.0 | grad norm: 12.720 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[987, 1020, 1044, 1043, 981, 1055, 1037, 1064, 1025, 1063, 1013, 1001, 959, 1011, 1007, 1072, 1001, 1044, 1040, 992, 1039, 1057, 997, 1018, 1024, 1025, 1030, 1019, 987, 1036, 1041, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1013, 1045, 994, 1038, 1014, 1001, 1005, 1071, 1024, 982, 972, 984, 1045, 1097, 989, 1046, 1000, 1046, 1013, 1047, 1007, 1078, 1053, 1020, 988, 1001, 1045, 1064, 1045, 992, 1011, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1087, 1000, 1057, 1014, 1018, 1025, 1016, 1038, 1045, 1062, 1029, 986, 1026, 997, 1002, 1033, 1047, 1039, 992, 1017, 1013, 999, 992, 1023, 1033, 1026, 1032, 1019, 1049, 1032, 1027, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1043, 995, 1049, 989, 1071, 992, 1047, 988, 1029, 1053, 1051, 982, 1048, 1032, 1003, 1014, 996, 998, 985, 1040, 1034, 1017, 1080, 1086, 1061, 1008, 1015, 993, 1013, 1028, 969, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1031, 993, 1001, 967, 1012, 988, 983, 1044, 1010, 986, 1046, 1026, 1016, 993, 1010, 1035, 1049, 1022, 1102, 975, 1090, 1069, 1004, 1029, 1072, 1031, 1032, 1012, 1067, 1010, 1043, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1029, 1016, 1046, 1011, 1002, 1015, 1048, 1068, 1051, 1033, 1033, 988, 1051, 1022, 1002, 1027, 1031, 989, 998, 1009, 1010, 1039, 1037, 1044, 992, 1028, 1035, 1027, 1029, 1004, 1045, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1049, 1036, 1010, 1037, 965, 1058, 1027, 955, 1033, 1046, 1072, 1017, 1077, 1013, 1004, 1033, 1043, 973, 1004, 1049, 1027, 1027, 1011, 999, 999, 1034, 1025, 1027, 1038, 1033, 1013, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1048, 1005, 1058, 1019, 996, 1079, 1041, 1008, 1015, 1071, 981, 1029, 1074, 970, 1009, 1045, 1064, 1009, 1015, 1026, 988, 998, 961, 1012, 967, 993, 1028, 1072, 1067, 1027, 1059, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[987, 1020, 1044, 1043, 0, 0, 0, 0, 981, 1055, 1037, 1064, 6, 0, 0, 0, 1025, 1063, 1013, 1001, 3, 0, 0, 0, 959, 1011, 1007, 1072, 0, 0, 0, 0, 1001, 1044, 1040, 992, 13, 0, 0, 0, 1039, 1047, 997, 1018, 0, 0, 0, 0, 1024, 1025, 1030, 1019, 1, 0, 0, 0, 974, 1036, 1041, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4143, 4105, 4049, 4090, 4101, 4099, 4087] avg=4096.0000 max=4143 max_rank=1 min=4049 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16377 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1087, 1000, 1057, 1014, 0, 0, 0, 0, 1018, 1025, 1016, 1038, 5, 0, 0, 0, 1045, 1062, 1029, 986, 3, 0, 0, 0, 1026, 997, 1002, 1033, 0, 0, 0, 0, 1047, 1039, 992, 1017, 14, 0, 0, 0, 1013, 990, 992, 1023, 0, 0, 0, 0, 1033, 1026, 1032, 1019, 1, 0, 0, 0, 1035, 1032, 1027, 993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4102, 4125, 4058, 4109, 4018, 4111, 4087] avg=4096.0000 max=4158 max_rank=0 min=4018 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16325 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1013, 1045, 994, 1038, 0, 0, 0, 0, 1014, 1001, 1005, 1071, 6, 0, 0, 0, 1024, 982, 972, 984, 3, 0, 0, 0, 1045, 1097, 989, 1046, 0, 0, 0, 0, 1000, 1046, 1013, 1047, 14, 0, 0, 0, 1007, 1068, 1053, 1020, 0, 0, 0, 0, 988, 1001, 1045, 1064, 1, 0, 0, 0, 1031, 992, 1011, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4097, 3965, 4177, 4120, 4148, 4099, 4072] avg=4096.0000 max=4177 max_rank=3 min=3965 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16439 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1048, 1005, 1058, 1019, 0, 0, 0, 0, 996, 1079, 1041, 1008, 5, 0, 0, 0, 1015, 1071, 981, 1029, 3, 0, 0, 0, 1074, 970, 1009, 1045, 0, 0, 0, 0, 1064, 1009, 1015, 1026, 14, 0, 0, 0, 988, 989, 961, 1012, 0, 0, 0, 0, 967, 993, 1028, 1072, 1, 0, 0, 0, 1053, 1027, 1059, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4129, 4099, 4098, 4128, 3950, 4061, 4173] avg=4096.0000 max=4173 max_rank=7 min=3950 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16312 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1029, 1016, 1046, 1011, 0, 0, 0, 0, 1002, 1015, 1048, 1068, 6, 0, 0, 0, 1051, 1033, 1033, 988, 3, 0, 0, 0, 1051, 1022, 1002, 1027, 0, 0, 0, 0, 1031, 989, 998, 1009, 14, 0, 0, 0, 1010, 1029, 1037, 1044, 0, 0, 0, 0, 992, 1028, 1035, 1027, 1, 0, 0, 0, 1015, 1004, 1045, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4139, 4108, 4102, 4041, 4120, 4083, 4073] avg=4096.0000 max=4139 max_rank=1 min=4041 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16317 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1025, 993, 1001, 967, 0, 0, 0, 0, 1012, 988, 983, 1044, 9, 7, 0, 0, 1010, 986, 1046, 1026, 9, 0, 0, 0, 1016, 993, 1010, 1035, 6, 0, 0, 0, 1049, 1022, 1102, 975, 17, 0, 0, 0, 1090, 1047, 1004, 1029, 0, 0, 0, 0, 1072, 1031, 1032, 1012, 4, 0, 0, 0, 1043, 1010, 1043, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3986, 4043, 4077, 4060, 4165, 4170, 4151, 4116] avg=4096.0000 max=4170 max_rank=5 min=3986 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16602 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1049, 1036, 1010, 1037, 0, 0, 0, 0, 965, 1058, 1027, 955, 6, 0, 0, 0, 1033, 1046, 1072, 1017, 3, 0, 0, 0, 1077, 1013, 1004, 1033, 0, 0, 0, 0, 1043, 973, 1004, 1049, 14, 0, 0, 0, 1027, 1017, 1011, 999, 0, 0, 0, 0, 999, 1034, 1025, 1027, 1, 0, 0, 0, 1024, 1033, 1013, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4011, 4171, 4127, 4083, 4054, 4086, 4104] avg=4096.0000 max=4171 max_rank=2 min=4011 min_rank=1 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16327 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1043, 995, 1049, 989, 0, 0, 0, 0, 1071, 992, 1047, 988, 6, 0, 0, 0, 1029, 1053, 1051, 982, 3, 0, 0, 0, 1048, 1032, 1003, 1014, 0, 0, 0, 0, 996, 998, 985, 1040, 13, 0, 0, 0, 1034, 1007, 1080, 1086, 0, 0, 0, 0, 1061, 1008, 1015, 993, 1, 0, 0, 0, 1000, 1028, 969, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4104, 4118, 4097, 4032, 4207, 4078, 4056] avg=4096.0000 max=4207 max_rank=5 min=4032 min_rank=4 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16373 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8064, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1019, 995, 1016, 1020, 1054, 1055, 983, 1076, 1003, 1062, 1018, 1026, 1004, 1080, 1020, 1057, 1041, 1047, 1000, 1023, 1018, 1046, 1013, 1021, 1015, 969, 1036, 1010, 963, 1031, 1028, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1032, 989, 1075, 1072, 1061, 1002, 1015, 1014, 1057, 1008, 993, 961, 1035, 1031, 1045, 1047, 1022, 986, 947, 1029, 1023, 1011, 1031, 1007, 1008, 1041, 1043, 1045, 1010, 1031, 1051, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1010, 1020, 1045, 982, 1016, 980, 992, 984, 995, 1023, 1034, 1054, 1031, 1098, 1026, 1038, 1035, 1084, 1031, 1004, 1063, 990, 1018, 1022, 1038, 963, 1045, 1071, 991, 1055, 1053, 977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1081, 992, 1027, 1056, 1018, 996, 1030, 1013, 999, 1087, 1017, 1041, 965, 1058, 969, 1050, 971, 1078, 1034, 983, 1004, 1049, 1025, 1050, 1029, 1007, 994, 1045, 1049, 1041, 1047, 963] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1043, 1012, 1039, 1059, 1028, 1048, 1002, 1004, 1017, 1039, 1042, 1028, 1046, 1055, 1027, 995, 981, 1009, 1001, 1038, 963, 1035, 1030, 1026, 1076, 1011, 1044, 1023, 1019, 1023, 1000, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[991, 1019, 1033, 980, 1001, 1038, 997, 995, 1036, 1041, 1051, 1016, 1000, 1013, 1055, 983, 1015, 1002, 1012, 1029, 1030, 1086, 1088, 1068, 1039, 1069, 998, 1040, 1003, 995, 1041, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1015, 1015, 1057, 1033, 1019, 999, 1014, 1001, 1026, 1020, 997, 1045, 1082, 1008, 1020, 965, 1047, 1052, 975, 1022, 1051, 1012, 1016, 1060, 1023, 968, 1017, 1084, 984, 1018, 1053, 1070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1011, 1013, 1009, 1001, 1002, 1033, 1034, 1058, 1054, 1034, 1072, 992, 1019, 980, 1030, 1022, 1034, 1020, 1016, 1059, 968, 1013, 1054, 1019, 1046, 1059, 1050, 1006, 1023, 973, 1052, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1043, 1012, 1039, 1059, 0, 0, 0, 0, 1028, 1048, 1002, 1004, 17, 7, 0, 0, 1017, 1025, 1042, 1028, 0, 0, 0, 0, 1046, 1045, 1027, 995, 0, 0, 0, 0, 981, 1009, 1001, 1038, 9, 7, 0, 0, 963, 1035, 1014, 1026, 0, 0, 0, 0, 1076, 1011, 1044, 1006, 0, 0, 0, 0, 1019, 1023, 1000, 1005, 10, 7, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4106, 4112, 4113, 4045, 4038, 4137, 4064] avg=4096.0000 max=4153 max_rank=0 min=4038 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1081, 992, 1027, 1056, 0, 0, 0, 0, 1018, 996, 1030, 1013, 18, 7, 0, 0, 999, 1073, 1017, 1041, 0, 0, 0, 0, 965, 1048, 969, 1050, 0, 0, 0, 0, 971, 1078, 1034, 983, 9, 7, 0, 0, 1004, 1049, 1009, 1050, 0, 0, 0, 0, 1029, 1007, 994, 1027, 0, 0, 0, 0, 1049, 1041, 1047, 963, 10, 7, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4082, 4130, 4032, 4082, 4112, 4057, 4117] avg=4096.0000 max=4156 max_rank=0 min=4032 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1010, 1020, 1045, 982, 0, 0, 0, 0, 1016, 980, 992, 984, 18, 7, 0, 0, 995, 1009, 1034, 1054, 0, 0, 0, 0, 1031, 1088, 1026, 1038, 0, 0, 0, 0, 1035, 1084, 1031, 1004, 9, 7, 0, 0, 1063, 990, 1002, 1022, 0, 0, 0, 0, 1038, 963, 1045, 1053, 0, 0, 0, 0, 991, 1055, 1053, 977, 10, 7, 0, 0] rerouted_tokens_per_ep_rank=[4057, 3997, 4092, 4183, 4170, 4077, 4099, 4093] avg=4096.0000 max=4183 max_rank=3 min=3997 min_rank=1 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1032, 989, 1075, 1072, 0, 0, 0, 0, 1061, 1002, 1015, 1014, 18, 7, 0, 0, 1057, 994, 993, 961, 0, 0, 0, 0, 1035, 1022, 1045, 1047, 0, 0, 0, 0, 1022, 986, 947, 1029, 9, 7, 0, 0, 1023, 1011, 1015, 1007, 0, 0, 0, 0, 1008, 1041, 1043, 1027, 0, 0, 0, 0, 1010, 1031, 1051, 1046, 9, 7, 0, 0] rerouted_tokens_per_ep_rank=[4168, 4117, 4005, 4149, 4000, 4056, 4119, 4154] avg=4096.0000 max=4168 max_rank=0 min=4000 min_rank=4 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1019, 995, 1016, 1020, 0, 0, 0, 0, 1054, 1055, 983, 1076, 17, 7, 0, 0, 1003, 1048, 1018, 1026, 0, 0, 0, 0, 1004, 1070, 1020, 1057, 0, 0, 0, 0, 1041, 1047, 1000, 1023, 9, 7, 0, 0, 1018, 1046, 997, 1021, 0, 0, 0, 0, 1015, 969, 1036, 993, 0, 0, 0, 0, 963, 1031, 1028, 1019, 10, 7, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4192, 4095, 4151, 4127, 4082, 4013, 4058] avg=4096.0000 max=4192 max_rank=1 min=4013 min_rank=6 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[991, 1019, 1033, 980, 7, 0, 0, 0, 1001, 1038, 997, 995, 21, 13, 0, 0, 1036, 1019, 1051, 1016, 0, 0, 0, 0, 1000, 994, 1055, 983, 0, 0, 0, 0, 1015, 1002, 1012, 1029, 16, 12, 0, 0, 1030, 1086, 1059, 1068, 0, 0, 0, 0, 1039, 1069, 998, 1019, 0, 0, 0, 0, 1003, 995, 1041, 1004, 12, 10, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4065, 4122, 4032, 4086, 4243, 4125, 4065] avg=4096.0000 max=4243 max_rank=5 min=4030 min_rank=0 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1015, 1015, 1057, 1033, 0, 0, 0, 0, 1019, 999, 1014, 1001, 18, 7, 0, 0, 1026, 1006, 997, 1045, 0, 0, 0, 0, 1082, 999, 1020, 965, 0, 0, 0, 0, 1047, 1052, 975, 1022, 9, 7, 0, 0, 1051, 1012, 1000, 1060, 0, 0, 0, 0, 1023, 968, 1017, 1066, 0, 0, 0, 0, 984, 1018, 1053, 1070, 9, 7, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4058, 4074, 4066, 4112, 4123, 4074, 4141] avg=4096.0000 max=4141 max_rank=7 min=4058 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1011, 1013, 1009, 1001, 0, 0, 0, 0, 1002, 1033, 1034, 1058, 17, 7, 0, 0, 1054, 1020, 1072, 992, 0, 0, 0, 0, 1019, 971, 1030, 1022, 0, 0, 0, 0, 1034, 1020, 1016, 1059, 10, 7, 0, 0, 968, 1013, 1037, 1019, 0, 0, 0, 0, 1046, 1059, 1050, 989, 0, 0, 0, 0, 1023, 973, 1052, 1012, 9, 7, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4151, 4138, 4042, 4146, 4037, 4144, 4076] avg=4096.0000 max=4151 max_rank=1 min=4034 min_rank=0 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8064, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8448, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1025, 1013, 1032, 1041, 1014, 995, 1029, 1035, 1031, 1037, 1002, 1110, 1005, 1041, 1022, 1005, 1019, 1054, 981, 1048, 1021, 1009, 1032, 985, 977, 985, 1035, 1039, 1016, 1037, 1049, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[996, 1030, 998, 919, 1029, 1042, 1026, 999, 1004, 994, 1062, 1015, 1054, 992, 1061, 1069, 1002, 1039, 1017, 1022, 1024, 1036, 1019, 1040, 1039, 1053, 1058, 1011, 1090, 980, 1002, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1024, 1068, 1063, 1056, 985, 989, 996, 1014, 1051, 1025, 970, 1056, 1030, 978, 980, 1002, 1050, 974, 1042, 1056, 1067, 958, 1003, 1034, 1051, 1061, 1024, 1028, 1048, 1009, 1029, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1065, 1020, 1024, 996, 992, 1022, 1051, 1020, 1053, 1020, 1042, 1016, 1047, 984, 1050, 995, 1045, 1034, 1062, 992, 1024, 1027, 1028, 1024, 1017, 1057, 1018, 1010, 997, 992, 1041, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[979, 1064, 1033, 996, 1037, 979, 1010, 1043, 993, 1027, 1003, 1031, 1025, 1049, 1036, 980, 1036, 1096, 1034, 1009, 979, 1007, 1043, 1055, 974, 1050, 1079, 1020, 1016, 1062, 1038, 985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[986, 1035, 1036, 1037, 1045, 1005, 1023, 1028, 1000, 924, 1042, 1059, 1022, 1004, 1025, 1016, 1033, 1011, 1021, 1002, 1019, 1048, 1037, 1053, 1026, 1027, 1031, 1059, 1025, 1023, 1044, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[988, 1025, 959, 1025, 1017, 1014, 1020, 1014, 1019, 975, 1056, 1000, 1035, 1074, 1069, 1009, 1061, 1032, 997, 1051, 1003, 1032, 1052, 1038, 1039, 1037, 1067, 1027, 983, 1005, 1039, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1006, 1032, 1032, 1028, 1046, 1052, 1005, 1059, 1046, 1039, 1046, 980, 1031, 1032, 1046, 989, 1029, 1023, 971, 1008, 1051, 1043, 1006, 1053, 1022, 1003, 1064, 1014, 1038, 971, 999, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[996, 1030, 998, 919, 17, 0, 0, 0, 1029, 1042, 1026, 999, 12, 4, 0, 0, 1004, 994, 1062, 1015, 5, 0, 0, 0, 1054, 992, 1061, 1069, 1, 0, 0, 0, 993, 1039, 1017, 1022, 0, 0, 0, 0, 1024, 1036, 1019, 1031, 0, 0, 0, 0, 1039, 1053, 1029, 1011, 0, 0, 0, 0, 1090, 980, 1002, 1046, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[3960, 4112, 4080, 4177, 4071, 4110, 4132, 4126] avg=4096.0000 max=4177 max_rank=3 min=3960 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1024, 1068, 1063, 1056, 16, 0, 0, 0, 985, 989, 996, 1014, 11, 4, 0, 0, 1051, 1025, 970, 1056, 4, 0, 0, 0, 1030, 978, 980, 1002, 1, 0, 0, 0, 1041, 974, 1042, 1056, 0, 0, 0, 0, 1067, 958, 1003, 1026, 0, 0, 0, 0, 1051, 1061, 997, 1028, 0, 0, 0, 0, 1048, 1009, 1029, 1047, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4227, 3999, 4106, 3991, 4113, 4054, 4137, 4141] avg=4096.0000 max=4227 max_rank=0 min=3991 min_rank=3 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1025, 1013, 1032, 1041, 16, 0, 0, 0, 1014, 995, 1029, 1035, 11, 4, 0, 0, 1031, 1037, 1002, 1110, 4, 0, 0, 0, 1005, 1041, 1022, 1005, 1, 0, 0, 0, 1010, 1054, 981, 1048, 0, 0, 0, 0, 1021, 1009, 1032, 977, 0, 0, 0, 0, 977, 985, 1008, 1039, 0, 0, 0, 0, 1016, 1037, 1049, 1044, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4088, 4184, 4074, 4093, 4039, 4009, 4154] avg=4096.0000 max=4184 max_rank=2 min=4009 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[979, 1064, 1033, 996, 17, 0, 0, 0, 1037, 979, 1010, 1043, 12, 4, 0, 0, 993, 1027, 1003, 1031, 5, 0, 0, 0, 1025, 1049, 1036, 980, 1, 0, 0, 0, 1027, 1096, 1034, 1009, 0, 0, 0, 0, 979, 1007, 1043, 1046, 0, 0, 0, 0, 974, 1050, 1050, 1020, 0, 0, 0, 0, 1016, 1062, 1038, 985, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4085, 4059, 4091, 4166, 4075, 4094, 4109] avg=4096.0000 max=4166 max_rank=4 min=4059 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[988, 1025, 959, 1025, 17, 0, 0, 0, 1017, 1014, 1020, 1014, 12, 4, 0, 0, 1019, 975, 1056, 1000, 5, 0, 0, 0, 1035, 1074, 1069, 1009, 1, 0, 0, 0, 1051, 1032, 997, 1051, 0, 0, 0, 0, 1003, 1032, 1052, 1029, 0, 0, 0, 0, 1039, 1037, 1038, 1027, 0, 0, 0, 0, 983, 1005, 1039, 1006, 6, 3, 0, 0] rerouted_tokens_per_ep_rank=[4014, 4081, 4055, 4188, 4131, 4116, 4141, 4042] avg=4096.0000 max=4188 max_rank=3 min=4014 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1065, 1020, 1024, 996, 16, 0, 0, 0, 992, 1022, 1051, 1020, 11, 4, 0, 0, 1053, 1020, 1042, 1016, 4, 0, 0, 0, 1047, 984, 1050, 995, 1, 0, 0, 0, 1036, 1034, 1062, 992, 0, 0, 0, 0, 1024, 1027, 1028, 1016, 0, 0, 0, 0, 1017, 1057, 991, 1010, 0, 0, 0, 0, 997, 992, 1041, 1003, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4100, 4135, 4077, 4124, 4095, 4075, 4041] avg=4096.0000 max=4135 max_rank=2 min=4041 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[986, 1035, 1036, 1037, 16, 0, 0, 0, 1045, 1005, 1023, 1028, 11, 4, 0, 0, 1000, 924, 1042, 1059, 5, 0, 0, 0, 1022, 1004, 1025, 1016, 1, 0, 0, 0, 1024, 1011, 1021, 1002, 0, 0, 0, 0, 1019, 1048, 1037, 1044, 0, 0, 0, 0, 1026, 1027, 1004, 1059, 0, 0, 0, 0, 1025, 1023, 1044, 1022, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4116, 4030, 4068, 4058, 4148, 4116, 4122] avg=4096.0000 max=4148 max_rank=5 min=4030 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1006, 1032, 1032, 1028, 22, 0, 0, 0, 1046, 1052, 1005, 1059, 17, 8, 0, 0, 1046, 1039, 1046, 980, 8, 0, 0, 0, 1031, 1032, 1046, 989, 4, 0, 0, 0, 1010, 1023, 971, 1008, 0, 0, 0, 0, 1051, 1043, 1006, 1031, 0, 0, 0, 0, 1022, 1003, 1025, 1014, 0, 0, 0, 0, 1038, 971, 999, 1004, 11, 10, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4187, 4119, 4102, 4012, 4131, 4064, 4033] avg=4096.0000 max=4187 max_rank=1 min=4012 min_rank=4 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1044, 961, 987, 1022, 1027, 1070, 982, 957, 982, 1017, 1013, 1019, 1050, 1020, 1033, 1038, 1039, 1036, 1012, 1044, 1054, 1022, 1043, 1025, 1003, 1011, 1085, 1112, 1011, 1016, 996, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1059, 993, 1040, 987, 1026, 1018, 1111, 1018, 990, 951, 1033, 997, 1038, 1023, 1042, 984, 1022, 1075, 1012, 1047, 1048, 1012, 1053, 1022, 1046, 1058, 1010, 1018, 1023, 1002, 1009, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1008, 1042, 1015, 1024, 1018, 987, 1034, 1028, 1041, 1044, 993, 1009, 1039, 1044, 1046, 1040, 1008, 1008, 1077, 1016, 1019, 1027, 1021, 1017, 954, 1053, 1027, 1019, 1022, 1029, 1020, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1003, 1018, 958, 1033, 1085, 1038, 1083, 1014, 1036, 1067, 1023, 981, 1026, 999, 1023, 994, 1033, 1058, 1059, 993, 974, 996, 1049, 1021, 1009, 1031, 1042, 1006, 997, 1026, 1014, 1079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1027, 998, 1027, 1066, 1023, 1002, 1003, 1114, 995, 1029, 1016, 1012, 1055, 1021, 998, 1043, 1004, 986, 988, 1034, 1018, 993, 1048, 1011, 1021, 1005, 1023, 1047, 1016, 1037, 1098, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1071, 970, 1030, 1002, 1047, 1008, 1054, 1029, 997, 1023, 1049, 981, 1005, 1017, 1040, 1033, 1021, 1037, 975, 1060, 1049, 1014, 1013, 1086, 1012, 987, 1003, 1024, 1034, 1034, 1012, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1028, 993, 1008, 996, 1078, 1036, 993, 1019, 1042, 1011, 1045, 1036, 1040, 1047, 1005, 1028, 1012, 992, 1028, 1027, 1029, 1057, 983, 1019, 1068, 1038, 989, 1035, 1073, 1008, 983, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1017, 1011, 1032, 1032, 1049, 1019, 1052, 1057, 1035, 1035, 1052, 983, 1059, 1008, 1005, 1001, 1022, 1035, 1046, 1039, 1036, 1022, 1009, 1019, 994, 1027, 1003, 1017, 995, 1044, 991, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1044, 961, 987, 1022, 32, 0, 0, 0, 990, 1070, 982, 957, 0, 0, 0, 0, 982, 1017, 1013, 1019, 9, 9, 5, 4, 1041, 1020, 1033, 1038, 0, 0, 0, 0, 1039, 1036, 1012, 1035, 0, 0, 0, 0, 1049, 1022, 1043, 1025, 0, 0, 0, 0, 1003, 1011, 1085, 1111, 0, 0, 0, 0, 1011, 1016, 996, 1037, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4046, 3999, 4058, 4132, 4122, 4139, 4210, 4062] avg=4096.0000 max=4210 max_rank=6 min=3999 min_rank=1 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1027, 998, 1027, 1066, 32, 0, 0, 0, 986, 1002, 1003, 1114, 0, 0, 0, 0, 995, 1029, 1016, 1012, 9, 9, 5, 4, 1046, 1021, 998, 1043, 0, 0, 0, 0, 1004, 986, 988, 1025, 0, 0, 0, 0, 1014, 993, 1048, 1011, 0, 0, 0, 0, 1021, 1005, 1023, 1046, 0, 0, 0, 0, 1016, 1037, 1098, 1010, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4105, 4079, 4108, 4003, 4066, 4095, 4162] avg=4096.0000 max=4162 max_rank=7 min=4003 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1059, 993, 1040, 987, 32, 0, 0, 0, 989, 1018, 1111, 1018, 0, 0, 0, 0, 990, 951, 1033, 997, 9, 9, 5, 4, 1029, 1023, 1042, 984, 0, 0, 0, 0, 1022, 1075, 1012, 1038, 0, 0, 0, 0, 1043, 1012, 1053, 1022, 0, 0, 0, 0, 1046, 1058, 1010, 1017, 0, 0, 0, 0, 1023, 1002, 1009, 1001, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4136, 3998, 4078, 4147, 4130, 4131, 4037] avg=4096.0000 max=4147 max_rank=4 min=3998 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1008, 1042, 1015, 1024, 32, 0, 0, 0, 981, 987, 1034, 1028, 0, 0, 0, 0, 1041, 1044, 993, 1009, 9, 9, 5, 4, 1030, 1044, 1046, 1040, 0, 0, 0, 0, 1008, 1008, 1077, 1007, 0, 0, 0, 0, 1015, 1027, 1021, 1017, 0, 0, 0, 0, 954, 1053, 1027, 1018, 0, 0, 0, 0, 1022, 1029, 1020, 1039, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4030, 4114, 4160, 4100, 4080, 4052, 4111] avg=4096.0000 max=4160 max_rank=3 min=4030 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1003, 1018, 958, 1033, 38, 0, 0, 0, 1037, 1038, 1083, 1014, 0, 0, 0, 0, 1036, 1067, 1023, 981, 14, 13, 10, 5, 1013, 999, 1023, 994, 0, 0, 0, 0, 1033, 1058, 1059, 979, 0, 0, 0, 0, 966, 996, 1049, 1021, 0, 0, 0, 0, 1009, 1031, 1042, 1004, 0, 0, 0, 0, 997, 1026, 1014, 1079, 2, 3, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4172, 4149, 4029, 4129, 4032, 4086, 4121] avg=4096.0000 max=4172 max_rank=1 min=4029 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1028, 993, 1008, 996, 34, 0, 0, 0, 1039, 1036, 993, 1019, 0, 0, 0, 0, 1042, 1011, 1045, 1036, 9, 9, 5, 4, 1031, 1047, 1005, 1028, 0, 0, 0, 0, 1012, 992, 1028, 1018, 0, 0, 0, 0, 1024, 1057, 983, 1019, 0, 0, 0, 0, 1068, 1038, 989, 1034, 0, 0, 0, 0, 1073, 1008, 983, 1022, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4087, 4161, 4111, 4050, 4083, 4129, 4088] avg=4096.0000 max=4161 max_rank=2 min=4050 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1071, 970, 1030, 1002, 33, 0, 0, 0, 1009, 1008, 1054, 1029, 0, 0, 0, 0, 997, 1023, 1049, 981, 9, 9, 5, 4, 996, 1017, 1040, 1033, 0, 0, 0, 0, 1021, 1037, 975, 1051, 0, 0, 0, 0, 1044, 1014, 1013, 1086, 0, 0, 0, 0, 1012, 987, 1003, 1023, 0, 0, 0, 0, 1034, 1034, 1012, 1051, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4100, 4077, 4086, 4084, 4157, 4025, 4133] avg=4096.0000 max=4157 max_rank=5 min=4025 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1017, 1011, 1032, 1032, 33, 0, 0, 0, 1011, 1019, 1052, 1057, 0, 0, 0, 0, 1035, 1035, 1052, 983, 9, 9, 5, 4, 1050, 1008, 1005, 1001, 0, 0, 0, 0, 1022, 1035, 1046, 1030, 0, 0, 0, 0, 1031, 1022, 1009, 1019, 0, 0, 0, 0, 994, 1027, 1003, 1016, 0, 0, 0, 0, 995, 1044, 991, 1022, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4139, 4132, 4064, 4133, 4081, 4040, 4054] avg=4096.0000 max=4139 max_rank=1 min=4040 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8064, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1015, 1017, 1000, 1012, 996, 965, 1064, 997, 1063, 980, 1006, 1027, 1021, 1075, 1041, 996, 1031, 1070, 1074, 1047, 1045, 1030, 1053, 1038, 986, 1035, 1000, 1054, 985, 1011, 1042, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1062, 1073, 1076, 1015, 1068, 986, 1023, 1001, 981, 978, 1019, 1034, 974, 978, 1038, 1001, 1004, 993, 1019, 1019, 1003, 1060, 1005, 1081, 1077, 1037, 1005, 1006, 984, 1080, 1023, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1028, 1020, 1017, 1083, 1022, 1000, 1062, 1092, 1024, 1055, 988, 1021, 1056, 963, 1064, 972, 1020, 1042, 1029, 984, 1012, 1078, 1013, 1019, 995, 999, 978, 1025, 1042, 1033, 1012, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1013, 1016, 1047, 1006, 1047, 1006, 1023, 1045, 1007, 1025, 1011, 1019, 1046, 1016, 1014, 1067, 1041, 999, 1055, 1011, 1006, 1052, 1014, 995, 1002, 1008, 1077, 1044, 1014, 1023, 1013, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1053, 1064, 989, 1073, 1035, 1075, 1014, 966, 1060, 986, 1035, 1001, 948, 1005, 1066, 1019, 1032, 1037, 1047, 1008, 1010, 985, 1019, 1006, 1038, 1014, 1012, 1045, 1027, 1031, 1032, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[979, 1026, 1050, 1005, 1009, 1015, 1013, 1022, 984, 1007, 1046, 1067, 985, 1023, 1058, 992, 1014, 1013, 1010, 1057, 1039, 1059, 1046, 981, 1069, 1071, 1034, 1048, 994, 1013, 1008, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1031, 987, 1003, 1005, 1022, 989, 1005, 1042, 1054, 1054, 996, 1007, 1046, 1025, 1035, 1090, 1041, 964, 1001, 1073, 1015, 1031, 1033, 1035, 1036, 1086, 1004, 1015, 1044, 1000, 1021, 978] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1008, 998, 1066, 1032, 983, 1020, 1048, 1024, 1085, 1063, 991, 990, 1043, 1002, 1080, 1001, 999, 1020, 991, 1046, 1038, 1057, 993, 1020, 1042, 1037, 1029, 992, 1010, 1045, 1016, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1062, 1073, 1065, 1015, 0, 0, 0, 0, 1068, 986, 1023, 1001, 11, 0, 0, 0, 981, 978, 1019, 1034, 12, 0, 0, 0, 974, 978, 1038, 1001, 2, 0, 0, 0, 1004, 993, 1019, 1017, 0, 0, 0, 0, 1003, 1048, 1005, 1081, 0, 0, 0, 0, 1077, 1021, 1005, 1006, 0, 0, 0, 0, 984, 1080, 1023, 1065, 16, 0, 0, 0] rerouted_tokens_per_ep_rank=[4215, 4089, 4024, 3993, 4033, 4137, 4109, 4168] avg=4096.0000 max=4215 max_rank=0 min=3993 min_rank=3 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1028, 1020, 1007, 1083, 0, 0, 0, 0, 1022, 1000, 1062, 1092, 10, 0, 0, 0, 1024, 1055, 988, 1021, 12, 0, 0, 0, 1056, 963, 1064, 972, 2, 0, 0, 0, 1020, 1042, 1029, 982, 0, 0, 0, 0, 1012, 1066, 1013, 1019, 0, 0, 0, 0, 995, 984, 978, 1025, 0, 0, 0, 0, 1042, 1033, 1012, 1020, 15, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4186, 4100, 4057, 4073, 4110, 3982, 4122] avg=4096.0000 max=4186 max_rank=1 min=3982 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1013, 1016, 1036, 1006, 0, 0, 0, 0, 1047, 1006, 1023, 1045, 11, 0, 0, 0, 1007, 1025, 1011, 1019, 12, 0, 0, 0, 1046, 1016, 1014, 1067, 2, 0, 0, 0, 1041, 999, 1055, 1009, 0, 0, 0, 0, 1006, 1040, 1014, 995, 0, 0, 0, 0, 1002, 992, 1077, 1044, 0, 0, 0, 0, 1014, 1023, 1013, 1006, 16, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4132, 4074, 4145, 4104, 4055, 4115, 4072] avg=4096.0000 max=4145 max_rank=3 min=4055 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1015, 1017, 990, 1012, 0, 0, 0, 0, 996, 965, 1064, 997, 10, 0, 0, 0, 1063, 980, 1006, 1027, 11, 0, 0, 0, 1021, 1075, 1041, 996, 2, 0, 0, 0, 1031, 1070, 1074, 1045, 0, 0, 0, 0, 1045, 1019, 1053, 1038, 0, 0, 0, 0, 986, 1019, 1000, 1054, 0, 0, 0, 0, 985, 1011, 1042, 992, 16, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4032, 4087, 4135, 4220, 4155, 4059, 4046] avg=4096.0000 max=4220 max_rank=4 min=4032 min_rank=1 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1053, 1064, 962, 1073, 0, 0, 0, 0, 1035, 1075, 1014, 966, 15, 0, 0, 0, 1060, 986, 1035, 1001, 15, 7, 0, 0, 948, 1005, 1066, 1019, 9, 5, 0, 0, 1032, 1037, 1047, 999, 0, 0, 0, 0, 1010, 964, 1019, 1006, 0, 0, 0, 0, 1038, 995, 1012, 1045, 0, 0, 0, 0, 1027, 1031, 1032, 1036, 19, 6, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4105, 4104, 4052, 4115, 3999, 4090, 4151] avg=4096.0000 max=4152 max_rank=0 min=3999 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[979, 1026, 1039, 1005, 0, 0, 0, 0, 1009, 1015, 1013, 1022, 11, 0, 0, 0, 984, 1007, 1046, 1067, 12, 0, 0, 0, 985, 1023, 1058, 992, 2, 0, 0, 0, 1014, 1013, 1010, 1055, 0, 0, 0, 0, 1039, 1047, 1046, 981, 0, 0, 0, 0, 1069, 1054, 1034, 1048, 0, 0, 0, 0, 994, 1013, 1008, 1031, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4070, 4116, 4060, 4092, 4113, 4205, 4063] avg=4096.0000 max=4205 max_rank=6 min=4049 min_rank=0 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1031, 987, 993, 1005, 0, 0, 0, 0, 1022, 989, 1005, 1042, 10, 0, 0, 0, 1054, 1054, 996, 1007, 11, 0, 0, 0, 1046, 1025, 1035, 1090, 2, 0, 0, 0, 1041, 964, 1001, 1071, 0, 0, 0, 0, 1015, 1020, 1033, 1035, 0, 0, 0, 0, 1036, 1069, 1004, 1015, 0, 0, 0, 0, 1044, 1000, 1021, 978, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4068, 4122, 4198, 4077, 4103, 4124, 4060] avg=4096.0000 max=4198 max_rank=3 min=4016 min_rank=0 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1008, 998, 1055, 1032, 0, 0, 0, 0, 983, 1020, 1048, 1024, 11, 0, 0, 0, 1085, 1063, 991, 990, 12, 0, 0, 0, 1043, 1002, 1080, 1001, 2, 0, 0, 0, 999, 1020, 991, 1044, 0, 0, 0, 0, 1038, 1045, 993, 1020, 0, 0, 0, 0, 1042, 1021, 1029, 992, 0, 0, 0, 0, 1010, 1045, 1016, 999, 16, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4086, 4141, 4128, 4054, 4096, 4084, 4086] avg=4096.0000 max=4141 max_rank=2 min=4054 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8448, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8064, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1039, 1096, 1019, 1047, 1059, 1014, 1048, 1030, 997, 1041, 1023, 1035, 1014, 1020, 1010, 1015, 972, 991, 1017, 1031, 1013, 1001, 1008, 1039, 1013, 1040, 1003, 1027, 1010, 1044, 998, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1069, 1003, 1008, 1025, 1029, 1049, 967, 1027, 1069, 1018, 1036, 1055, 993, 1042, 1029, 1005, 1020, 1000, 1038, 1039, 1034, 1023, 1028, 1054, 1028, 995, 1039, 1007, 972, 1079, 971, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1064, 1014, 999, 1001, 1034, 1016, 1017, 1061, 1039, 1061, 1013, 1000, 999, 1033, 1032, 1002, 1028, 1014, 1018, 1032, 1007, 1014, 960, 1056, 990, 1052, 999, 1023, 1063, 1018, 1064, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1079, 1008, 1017, 989, 1008, 980, 1059, 982, 1013, 991, 1070, 994, 1052, 1030, 1018, 1020, 1030, 1021, 1101, 1013, 975, 1018, 1037, 1035, 1030, 1001, 1048, 1038, 1038, 1018, 1029, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1070, 1082, 1037, 1033, 1052, 1048, 1009, 1007, 1057, 978, 1001, 1035, 1051, 1011, 984, 1011, 1025, 999, 1029, 1002, 1049, 1069, 1002, 1030, 996, 1003, 986, 1024, 1053, 982, 1028, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[996, 945, 1055, 980, 1029, 999, 1025, 977, 999, 1107, 1083, 994, 1040, 1052, 995, 1071, 1025, 1020, 1028, 997, 1065, 1056, 1002, 1039, 1054, 1024, 1057, 1009, 1018, 1003, 1011, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1021, 1038, 1018, 1077, 991, 1029, 1006, 1032, 1005, 1054, 989, 1042, 1035, 1063, 1006, 1052, 1024, 991, 1050, 1032, 976, 1001, 1002, 984, 1061, 1045, 999, 975, 1051, 1027, 1074, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[969, 1020, 1036, 1010, 1047, 1062, 1015, 1030, 991, 986, 1030, 1032, 993, 1061, 1008, 1003, 1049, 972, 1038, 1039, 1033, 1064, 1038, 1052, 1024, 1051, 980, 1036, 1014, 995, 1060, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1057, 1003, 1008, 1025, 0, 0, 0, 0, 1029, 1049, 967, 1027, 3, 0, 0, 0, 1069, 1018, 1029, 1055, 0, 0, 0, 0, 993, 1042, 1029, 1005, 2, 0, 0, 0, 1020, 1000, 1038, 1039, 7, 2, 0, 0, 1034, 1023, 1028, 1054, 0, 0, 0, 0, 1028, 995, 1039, 1007, 12, 1, 0, 0, 972, 1079, 963, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4075, 4171, 4071, 4106, 4139, 4082, 4031] avg=4096.0000 max=4171 max_rank=2 min=4031 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1067, 1008, 1017, 989, 0, 0, 0, 0, 1008, 980, 1059, 982, 3, 0, 0, 0, 1013, 991, 1063, 994, 0, 0, 0, 0, 1052, 1030, 1018, 1020, 2, 0, 0, 0, 1030, 1021, 1101, 1013, 8, 2, 0, 0, 975, 1018, 1037, 1035, 0, 0, 0, 0, 1030, 1001, 1048, 1038, 12, 1, 0, 0, 1038, 1018, 1020, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4032, 4061, 4122, 4175, 4065, 4130, 4102] avg=4096.0000 max=4175 max_rank=4 min=4032 min_rank=1 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1052, 1014, 999, 1001, 0, 0, 0, 0, 1034, 1016, 1017, 1061, 3, 0, 0, 0, 1039, 1061, 1006, 1000, 0, 0, 0, 0, 999, 1033, 1032, 1002, 2, 0, 0, 0, 1028, 1014, 1018, 1032, 8, 2, 0, 0, 1007, 1014, 960, 1056, 0, 0, 0, 0, 990, 1052, 999, 1023, 12, 1, 0, 0, 1063, 1018, 1055, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4131, 4106, 4068, 4102, 4037, 4077, 4181] avg=4096.0000 max=4181 max_rank=7 min=4037 min_rank=5 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1027, 1096, 1019, 1047, 0, 0, 0, 0, 1059, 1014, 1048, 1030, 3, 0, 0, 0, 997, 1041, 1016, 1035, 0, 0, 0, 0, 1014, 1020, 1010, 1015, 2, 0, 0, 0, 972, 991, 1017, 1031, 7, 2, 0, 0, 1013, 1001, 1008, 1039, 0, 0, 0, 0, 1013, 1040, 1003, 1027, 12, 1, 0, 0, 1010, 1044, 990, 1054, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4189, 4154, 4089, 4061, 4020, 4061, 4096, 4098] avg=4096.0000 max=4189 max_rank=0 min=4020 min_rank=4 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1058, 1082, 1037, 1033, 0, 0, 0, 0, 1052, 1048, 1009, 1007, 3, 0, 0, 0, 1057, 978, 994, 1035, 0, 0, 0, 0, 1051, 1011, 984, 1011, 2, 0, 0, 0, 1025, 999, 1029, 1002, 8, 2, 0, 0, 1049, 1069, 1002, 1030, 0, 0, 0, 0, 996, 1003, 986, 1024, 12, 1, 0, 0, 1053, 982, 1019, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4210, 4119, 4064, 4059, 4065, 4150, 4022, 4079] avg=4096.0000 max=4210 max_rank=0 min=4022 min_rank=6 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1010, 1038, 1018, 1077, 0, 0, 0, 0, 991, 1029, 1006, 1032, 3, 0, 0, 0, 1005, 1054, 982, 1042, 0, 0, 0, 0, 1035, 1063, 1006, 1052, 2, 0, 0, 0, 1024, 991, 1050, 1032, 8, 2, 0, 0, 976, 1001, 1002, 984, 0, 0, 0, 0, 1061, 1045, 999, 975, 11, 1, 0, 0, 1051, 1027, 1065, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4061, 4083, 4158, 4107, 3963, 4092, 4161] avg=4096.0000 max=4161 max_rank=7 min=3963 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[982, 945, 1055, 980, 0, 0, 0, 0, 1029, 999, 1025, 977, 9, 0, 0, 0, 999, 1107, 1062, 994, 0, 0, 0, 0, 1040, 1052, 995, 1071, 4, 0, 0, 0, 1025, 1020, 1028, 997, 11, 4, 0, 0, 1065, 1056, 1002, 1039, 4, 0, 0, 0, 1054, 1024, 1057, 1009, 14, 8, 0, 0, 1018, 1003, 992, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3962, 4039, 4162, 4162, 4085, 4166, 4166, 4026] avg=4096.0000 max=4166 max_rank=5 min=3962 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[958, 1020, 1036, 1010, 0, 0, 0, 0, 1047, 1062, 1015, 1030, 3, 0, 0, 0, 991, 986, 1023, 1032, 0, 0, 0, 0, 993, 1061, 1008, 1003, 2, 0, 0, 0, 1049, 972, 1038, 1039, 8, 2, 0, 0, 1033, 1064, 1038, 1052, 0, 0, 0, 0, 1024, 1051, 980, 1036, 11, 1, 0, 0, 1014, 995, 1051, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4157, 4032, 4067, 4108, 4187, 4103, 4090] avg=4096.0000 max=4187 max_rank=5 min=4024 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1061, 1000, 1045, 988, 1065, 1073, 995, 1030, 1063, 1025, 1047, 1002, 1004, 1028, 1004, 1041, 1007, 1028, 1031, 981, 1033, 997, 999, 1025, 1037, 1045, 993, 1053, 1025, 989, 1012, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1041, 1043, 1010, 982, 1007, 1036, 1022, 1051, 1011, 1037, 1037, 1053, 984, 1024, 1002, 1055, 1032, 1042, 1021, 1001, 999, 1015, 1016, 1023, 1030, 1008, 1035, 1000, 1074, 1008, 996, 1073] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1070, 998, 974, 1021, 1009, 1020, 1065, 998, 1013, 974, 1039, 1057, 996, 1059, 1074, 1022, 1068, 1042, 1044, 994, 997, 1042, 1018, 1001, 1004, 1014, 1009, 1039, 1044, 1028, 1036, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1020, 1038, 990, 986, 1035, 1021, 1029, 1047, 1039, 1086, 1111, 989, 1011, 1015, 1032, 1026, 1038, 1013, 1017, 1035, 944, 1027, 988, 1004, 1050, 983, 1017, 1065, 1039, 1022, 992, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[998, 1000, 1053, 1051, 1036, 1043, 1002, 953, 1051, 1043, 1042, 1065, 1034, 1017, 989, 966, 1036, 1073, 1032, 1001, 1046, 1054, 1033, 1010, 1001, 984, 1021, 1026, 1011, 1028, 997, 1072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1007, 980, 1023, 1046, 1020, 1019, 1033, 1052, 1030, 1026, 1009, 1033, 985, 1047, 1036, 1036, 1032, 1012, 1066, 1045, 1008, 998, 1008, 1086, 1009, 994, 1033, 994, 1010, 994, 1053, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1086, 1047, 1045, 1041, 1031, 988, 1029, 1024, 1028, 1053, 1076, 988, 1029, 1002, 968, 1026, 994, 1027, 1074, 991, 1035, 1072, 1007, 996, 1000, 1012, 1058, 993, 1024, 968, 1006, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1034, 1026, 1007, 969, 1026, 1020, 1059, 1060, 972, 1035, 1055, 1007, 1042, 1019, 1059, 1027, 1013, 1003, 1021, 1002, 942, 1019, 1007, 1043, 1029, 1076, 1029, 1078, 1046, 1001, 1015, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1070, 998, 974, 1021, 9, 1, 0, 0, 1009, 1020, 1049, 998, 0, 0, 0, 0, 1013, 974, 999, 1057, 0, 0, 0, 0, 996, 1059, 1074, 1022, 7, 6, 0, 0, 1068, 1042, 1039, 994, 0, 0, 0, 0, 997, 1042, 1018, 1001, 34, 0, 0, 0, 1004, 1014, 1009, 1039, 4, 1, 0, 0, 1044, 1028, 1036, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4076, 4043, 4164, 4143, 4092, 4071, 4106] avg=4096.0000 max=4164 max_rank=3 min=4043 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1020, 1038, 990, 986, 9, 1, 0, 0, 1035, 1021, 1013, 1047, 0, 0, 0, 0, 1039, 1086, 1069, 989, 0, 0, 0, 0, 1011, 1015, 1032, 1026, 7, 6, 0, 0, 1038, 1013, 1012, 1035, 0, 0, 0, 0, 944, 1027, 988, 1004, 36, 0, 0, 0, 1050, 983, 1017, 1065, 4, 2, 0, 0, 1039, 1022, 992, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4116, 4183, 4097, 4098, 3999, 4121, 4110] avg=4096.0000 max=4183 max_rank=2 min=3999 min_rank=5 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1061, 1000, 1045, 988, 8, 1, 0, 0, 1065, 1073, 981, 1030, 0, 0, 0, 0, 1063, 1025, 1007, 1002, 0, 0, 0, 0, 1004, 1028, 1004, 1041, 6, 6, 0, 0, 1007, 1028, 1026, 981, 0, 0, 0, 0, 1033, 997, 999, 1025, 34, 0, 0, 0, 1037, 1045, 993, 1053, 4, 1, 0, 0, 1025, 989, 1012, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4149, 4097, 4089, 4042, 4088, 4133, 4067] avg=4096.0000 max=4149 max_rank=1 min=4042 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1041, 1043, 1010, 982, 9, 1, 0, 0, 1007, 1036, 1006, 1051, 0, 0, 0, 0, 1011, 1037, 997, 1053, 0, 0, 0, 0, 984, 1024, 1002, 1055, 7, 6, 0, 0, 1032, 1042, 1016, 1001, 0, 0, 0, 0, 999, 1015, 1016, 1023, 34, 0, 0, 0, 1030, 1008, 1035, 1000, 4, 2, 0, 0, 1074, 1008, 996, 1071, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4100, 4098, 4078, 4091, 4087, 4079, 4149] avg=4096.0000 max=4149 max_rank=7 min=4078 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1007, 980, 1023, 1046, 9, 1, 0, 0, 1020, 1019, 1017, 1052, 0, 0, 0, 0, 1030, 1026, 970, 1033, 0, 0, 0, 0, 985, 1047, 1036, 1036, 7, 6, 0, 0, 1032, 1012, 1061, 1045, 0, 0, 0, 0, 1008, 998, 1008, 1086, 33, 0, 0, 0, 1009, 994, 1033, 994, 4, 1, 0, 0, 1010, 994, 1053, 1043, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4108, 4059, 4117, 4150, 4133, 4035, 4100] avg=4096.0000 max=4150 max_rank=4 min=4035 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[998, 1000, 1053, 1051, 8, 1, 0, 0, 1036, 1043, 988, 953, 0, 0, 0, 0, 1051, 1043, 1002, 1065, 0, 0, 0, 0, 1034, 1017, 989, 966, 6, 6, 0, 0, 1036, 1073, 1027, 1001, 0, 0, 0, 0, 1046, 1054, 1033, 1010, 34, 0, 0, 0, 1001, 984, 1021, 1026, 4, 2, 0, 0, 1011, 1028, 997, 1070, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4020, 4161, 4018, 4137, 4177, 4038, 4106] avg=4096.0000 max=4177 max_rank=5 min=4018 min_rank=3 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1086, 1047, 1045, 1041, 12, 8, 0, 0, 1031, 988, 1007, 1024, 0, 0, 0, 0, 1028, 1053, 1029, 988, 0, 0, 0, 0, 1029, 1002, 968, 1026, 10, 10, 0, 0, 994, 1027, 1061, 991, 0, 0, 0, 0, 1035, 1072, 1007, 996, 37, 0, 0, 0, 1000, 1012, 1058, 993, 5, 6, 0, 0, 1024, 968, 1006, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4239, 4050, 4098, 4045, 4073, 4147, 4074, 4042] avg=4096.0000 max=4239 max_rank=0 min=4042 min_rank=7 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1034, 1026, 1007, 969, 9, 1, 0, 0, 1026, 1020, 1043, 1060, 0, 0, 0, 0, 972, 1035, 1015, 1007, 0, 0, 0, 0, 1042, 1019, 1059, 1027, 7, 6, 0, 0, 1013, 1003, 1016, 1002, 0, 0, 0, 0, 942, 1019, 1007, 1043, 34, 0, 0, 0, 1029, 1076, 1029, 1078, 4, 1, 0, 0, 1046, 1001, 1015, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4149, 4029, 4160, 4034, 4045, 4217, 4088] avg=4096.0000 max=4217 max_rank=6 min=4029 min_rank=2 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8192, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1008, 1038, 1043, 1018, 1037, 1028, 1002, 980, 982, 988, 1086, 1012, 1007, 1006, 1030, 1045, 980, 1053, 1022, 1016, 1012, 1044, 1023, 1048, 1013, 1076, 982, 1052, 1015, 1025, 1020, 1077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1063, 1012, 1045, 1025, 1040, 1055, 1034, 1018, 1006, 1022, 972, 1020, 1013, 1011, 1007, 1020, 1047, 1047, 1010, 1024, 974, 1040, 1044, 1080, 1014, 1000, 1013, 1000, 1034, 1055, 967, 1056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1048, 1061, 1005, 1047, 1079, 1036, 1017, 1080, 1008, 1031, 993, 1019, 979, 988, 1023, 1082, 1027, 984, 1058, 1038, 1064, 978, 981, 1003, 1030, 1061, 1033, 1025, 974, 981, 1031, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[983, 1018, 1008, 1051, 1059, 1025, 1030, 1035, 1046, 1032, 1027, 1023, 1050, 993, 1038, 1049, 981, 1045, 995, 1051, 1027, 1014, 1036, 966, 1049, 1032, 1058, 1044, 1027, 983, 1004, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1046, 1012, 981, 1017, 977, 1034, 1047, 1036, 1071, 1034, 981, 958, 1042, 1048, 1082, 1026, 1015, 969, 1043, 1005, 1011, 1053, 1050, 1000, 964, 1063, 1048, 1034, 1001, 1007, 1037, 1076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[986, 1017, 1031, 1005, 1050, 1069, 1040, 1033, 1023, 1032, 1077, 1018, 1033, 1056, 1020, 992, 990, 1039, 1003, 1029, 989, 1015, 960, 988, 1054, 1042, 1049, 1034, 1039, 1026, 1021, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1031, 1006, 1029, 1041, 1056, 1023, 1021, 993, 1046, 1024, 983, 995, 1008, 1013, 1061, 1012, 1036, 1003, 1055, 1061, 1031, 986, 1056, 1045, 975, 999, 1029, 1023, 953, 1032, 1104, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1005, 989, 1028, 1028, 1017, 1061, 1048, 1024, 1061, 1023, 1035, 1021, 1086, 1005, 995, 983, 998, 999, 995, 1050, 1009, 1039, 1044, 1020, 1014, 1046, 1017, 999, 1056, 1059, 1011, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1008, 1038, 1043, 1018, 4, 1, 0, 0, 1037, 990, 1002, 980, 0, 0, 0, 0, 982, 988, 1086, 1012, 14, 0, 0, 0, 1007, 1006, 1026, 1045, 0, 0, 0, 0, 980, 1053, 1022, 1016, 7, 5, 0, 0, 1012, 1044, 1023, 1048, 17, 0, 0, 0, 1013, 1063, 982, 1052, 0, 0, 0, 0, 1015, 1025, 1020, 1077, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4009, 4082, 4084, 4083, 4144, 4110, 4144] avg=4096.0000 max=4144 max_rank=5 min=4009 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1048, 1061, 1005, 1047, 4, 1, 0, 0, 1079, 998, 1017, 1080, 0, 0, 0, 0, 1008, 1031, 993, 1019, 14, 0, 0, 0, 979, 988, 1019, 1082, 0, 0, 0, 0, 1027, 984, 1058, 1038, 7, 5, 0, 0, 1064, 978, 981, 1003, 17, 0, 0, 0, 1030, 1048, 1033, 1025, 0, 0, 0, 0, 974, 981, 1031, 1004, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4174, 4065, 4068, 4119, 4043, 4136, 3997] avg=4096.0000 max=4174 max_rank=1 min=3997 min_rank=7 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[983, 1018, 1008, 1051, 4, 0, 0, 0, 1059, 988, 1030, 1035, 0, 0, 0, 0, 1046, 1032, 1027, 1023, 14, 0, 0, 0, 1050, 993, 1034, 1049, 0, 0, 0, 0, 981, 1045, 995, 1051, 7, 5, 0, 0, 1027, 1014, 1036, 966, 16, 0, 0, 0, 1049, 1021, 1058, 1044, 0, 0, 0, 0, 1027, 983, 1004, 989, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4112, 4142, 4126, 4084, 4059, 4172, 4009] avg=4096.0000 max=4172 max_rank=6 min=4009 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1063, 1012, 1045, 1025, 4, 0, 0, 0, 1040, 1016, 1034, 1018, 0, 0, 0, 0, 1006, 1022, 972, 1020, 15, 0, 0, 0, 1013, 1011, 1003, 1020, 0, 0, 0, 0, 1047, 1047, 1010, 1024, 7, 4, 0, 0, 974, 1040, 1044, 1080, 17, 0, 0, 0, 1014, 990, 1013, 1000, 0, 0, 0, 0, 1034, 1055, 967, 1056, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4108, 4035, 4047, 4139, 4155, 4017, 4118] avg=4096.0000 max=4155 max_rank=5 min=4017 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[986, 1017, 1031, 1005, 4, 1, 0, 0, 1050, 1030, 1040, 1033, 0, 0, 0, 0, 1023, 1032, 1077, 1018, 15, 0, 0, 0, 1033, 1056, 1016, 992, 0, 0, 0, 0, 990, 1039, 1003, 1029, 7, 5, 0, 0, 989, 1015, 960, 988, 17, 0, 0, 0, 1054, 1030, 1049, 1034, 0, 0, 0, 0, 1039, 1026, 1021, 1008, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4153, 4165, 4097, 4073, 3969, 4167, 4100] avg=4096.0000 max=4167 max_rank=6 min=3969 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1031, 1006, 1029, 1041, 4, 0, 0, 0, 1056, 986, 1021, 993, 0, 0, 0, 0, 1046, 1024, 983, 995, 14, 0, 0, 0, 1008, 1013, 1057, 1012, 0, 0, 0, 0, 1036, 1003, 1055, 1061, 7, 4, 0, 0, 1031, 986, 1056, 1045, 16, 0, 0, 0, 975, 989, 1029, 1023, 0, 0, 0, 0, 953, 1032, 1104, 1038, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4056, 4062, 4090, 4166, 4134, 4016, 4133] avg=4096.0000 max=4166 max_rank=4 min=4016 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1046, 1012, 981, 1017, 7, 4, 0, 0, 977, 985, 1047, 1036, 0, 0, 0, 0, 1071, 1034, 981, 958, 18, 0, 0, 0, 1042, 1048, 1075, 1026, 0, 0, 0, 0, 1015, 969, 1043, 1005, 10, 8, 0, 0, 1011, 1053, 1050, 1000, 21, 0, 0, 0, 964, 1040, 1048, 1034, 0, 0, 0, 0, 1001, 1007, 1037, 1076, 11, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4045, 4062, 4191, 4050, 4135, 4086, 4132] avg=4096.0000 max=4191 max_rank=3 min=4045 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1005, 989, 1028, 1028, 4, 1, 0, 0, 1017, 1022, 1048, 1024, 0, 0, 0, 0, 1061, 1023, 1035, 1021, 15, 0, 0, 0, 1086, 1005, 991, 983, 0, 0, 0, 0, 998, 999, 995, 1050, 7, 5, 0, 0, 1009, 1039, 1044, 1020, 17, 0, 0, 0, 1014, 1034, 1017, 999, 0, 0, 0, 0, 1056, 1059, 1011, 1003, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4111, 4155, 4065, 4054, 4129, 4064, 4135] avg=4096.0000 max=4155 max_rank=2 min=4054 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1012, 1034, 1030, 1026, 1005, 1068, 1025, 1014, 1017, 1032, 1027, 983, 1081, 1023, 1009, 1014, 1057, 986, 1033, 1000, 1015, 1009, 995, 1087, 1014, 1036, 986, 1061, 1031, 994, 1062, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1037, 1007, 1016, 992, 1007, 1023, 960, 947, 1037, 1034, 1038, 1010, 1015, 1053, 1012, 1005, 1025, 1031, 1036, 1021, 1011, 1065, 1098, 1032, 1057, 1021, 1026, 1018, 1054, 1045, 1014, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1009, 1038, 1015, 969, 1012, 984, 995, 1059, 1032, 1016, 1027, 1053, 1061, 1013, 1042, 1016, 1065, 1050, 1015, 1055, 994, 1039, 1007, 1011, 1105, 1005, 996, 1006, 1007, 1021, 1014, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1009, 1040, 1059, 1014, 1042, 1035, 1022, 1021, 996, 1041, 1024, 1001, 1065, 979, 974, 1047, 1010, 978, 1061, 991, 1038, 1022, 1046, 1030, 1044, 978, 1040, 1040, 1018, 994, 1019, 1090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[998, 998, 1014, 1000, 1006, 1074, 1027, 1040, 981, 980, 1052, 1104, 1001, 984, 1034, 1017, 1028, 1027, 1104, 1034, 1006, 998, 1058, 1053, 997, 1058, 1035, 995, 1006, 993, 1029, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1012, 1017, 1022, 1043, 1017, 1043, 1031, 1079, 999, 979, 1018, 1056, 1013, 977, 1013, 1035, 1099, 993, 991, 1068, 1011, 991, 1024, 950, 983, 1044, 1062, 1055, 1027, 1066, 1028, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[990, 986, 1040, 1069, 1000, 1067, 975, 1032, 1042, 1067, 1044, 1041, 1033, 1026, 1004, 1044, 1048, 1044, 1024, 960, 1055, 999, 1023, 1014, 1022, 1033, 989, 1008, 1012, 998, 1038, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1048, 1040, 1000, 1005, 1009, 1013, 1040, 1076, 1035, 1018, 1050, 1009, 1006, 1029, 1010, 993, 1012, 1024, 1035, 1000, 1065, 1040, 1028, 982, 996, 1001, 1070, 1033, 1032, 1006, 1032, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1037, 1007, 1016, 992, 16, 5, 0, 0, 1007, 1023, 960, 947, 2, 0, 0, 0, 1037, 1034, 1029, 1010, 0, 0, 0, 0, 1015, 1053, 1012, 1005, 6, 5, 4, 1, 1009, 1031, 1036, 1021, 0, 0, 0, 0, 1011, 1065, 1095, 1032, 0, 0, 0, 0, 1052, 1021, 1026, 1018, 0, 0, 0, 0, 1054, 1045, 1014, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 3939, 4110, 4101, 4097, 4203, 4117, 4128] avg=4096.0000 max=4203 max_rank=5 min=3939 min_rank=1 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16545 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1009, 1040, 1059, 1014, 16, 5, 0, 0, 1042, 1035, 1022, 1021, 2, 0, 0, 0, 996, 1041, 1015, 1001, 0, 0, 0, 0, 1065, 979, 974, 1047, 6, 5, 4, 1, 994, 978, 1061, 991, 0, 0, 0, 0, 1038, 1022, 1043, 1030, 0, 0, 0, 0, 1039, 978, 1040, 1040, 0, 0, 0, 0, 1018, 994, 1019, 1084, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4122, 4053, 4081, 4024, 4133, 4097, 4115] avg=4096.0000 max=4143 max_rank=0 min=4024 min_rank=4 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1012, 1034, 1030, 1026, 17, 5, 0, 0, 1005, 1068, 1025, 1014, 2, 0, 0, 0, 1017, 1032, 1018, 983, 0, 0, 0, 0, 1081, 1023, 1009, 1014, 6, 5, 4, 0, 1040, 986, 1033, 1000, 0, 0, 0, 0, 1015, 1009, 993, 1087, 0, 0, 0, 0, 1009, 1036, 986, 1061, 0, 0, 0, 0, 1031, 994, 1062, 996, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4114, 4050, 4142, 4059, 4104, 4092, 4083] avg=4096.0000 max=4142 max_rank=3 min=4050 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1009, 1038, 1015, 969, 17, 5, 0, 0, 1012, 984, 995, 1059, 2, 0, 0, 0, 1032, 1016, 1018, 1053, 0, 0, 0, 0, 1061, 1013, 1042, 1016, 6, 6, 4, 0, 1048, 1050, 1015, 1055, 0, 0, 0, 0, 994, 1039, 1005, 1011, 0, 0, 0, 0, 1099, 1005, 996, 1006, 0, 0, 0, 0, 1007, 1021, 1014, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4052, 4119, 4148, 4168, 4049, 4106, 4073] avg=4096.0000 max=4168 max_rank=4 min=4049 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[998, 998, 1014, 1000, 16, 5, 0, 0, 1006, 1074, 1027, 1040, 2, 0, 0, 0, 981, 980, 1043, 1104, 0, 0, 0, 0, 1001, 984, 1034, 1017, 6, 5, 4, 1, 1012, 1027, 1104, 1034, 0, 0, 0, 0, 1006, 998, 1055, 1053, 0, 0, 0, 0, 992, 1058, 1035, 995, 0, 0, 0, 0, 1006, 993, 1029, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4149, 4108, 4052, 4177, 4112, 4080, 4059] avg=4096.0000 max=4177 max_rank=4 min=4031 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[990, 986, 1040, 1069, 17, 5, 0, 0, 1000, 1067, 975, 1032, 2, 0, 0, 0, 1042, 1067, 1035, 1041, 0, 0, 0, 0, 1033, 1026, 1004, 1044, 6, 5, 4, 0, 1031, 1044, 1024, 960, 0, 0, 0, 0, 1055, 999, 1021, 1014, 0, 0, 0, 0, 1017, 1033, 989, 1008, 0, 0, 0, 0, 1012, 998, 1038, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4076, 4185, 4122, 4059, 4089, 4047, 4083] avg=4096.0000 max=4185 max_rank=2 min=4047 min_rank=6 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1012, 1017, 1022, 1043, 22, 7, 0, 0, 1017, 1043, 1031, 1079, 6, 0, 0, 0, 999, 979, 1006, 1056, 0, 0, 0, 0, 1013, 977, 1013, 1035, 11, 10, 5, 5, 1077, 993, 991, 1068, 0, 0, 0, 0, 1011, 991, 1013, 950, 0, 0, 0, 0, 973, 1044, 1062, 1055, 0, 0, 0, 0, 1027, 1066, 1028, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4176, 4040, 4069, 4129, 3965, 4134, 4132] avg=4096.0000 max=4176 max_rank=1 min=3965 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1048, 1040, 1000, 1005, 16, 5, 0, 0, 1009, 1013, 1040, 1076, 2, 0, 0, 0, 1035, 1018, 1041, 1009, 0, 0, 0, 0, 1006, 1029, 1010, 993, 6, 5, 4, 0, 996, 1024, 1035, 1000, 0, 0, 0, 0, 1065, 1040, 1026, 982, 0, 0, 0, 0, 991, 1001, 1070, 1033, 0, 0, 0, 0, 1032, 1006, 1032, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4140, 4103, 4053, 4055, 4113, 4095, 4095] avg=4096.0000 max=4140 max_rank=1 min=4053 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1000, 1044, 1029, 1004, 999, 1011, 1021, 990, 1012, 1064, 1014, 1010, 1017, 1023, 1040, 1014, 1013, 1051, 1040, 1075, 997, 1056, 1010, 997, 1004, 1026, 1019, 1081, 1017, 994, 1052, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1003, 1041, 1023, 990, 988, 1005, 1021, 1047, 1040, 1052, 994, 1052, 1015, 1001, 989, 1051, 1066, 1041, 1003, 1013, 1014, 1030, 1010, 1023, 1035, 1053, 1051, 1044, 1035, 1024, 1004, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1035, 1039, 1054, 1019, 1069, 1059, 989, 1030, 957, 1051, 970, 986, 1015, 997, 1001, 1020, 992, 1017, 1011, 1042, 1074, 1029, 993, 1026, 1071, 1101, 1009, 980, 1045, 1000, 1026, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1017, 1035, 997, 1017, 1016, 996, 1016, 1043, 1015, 1028, 1031, 1021, 1027, 990, 1026, 1033, 1066, 1001, 1034, 1054, 1020, 997, 1027, 1044, 1044, 1033, 1030, 1001, 1027, 1006, 1013, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1016, 990, 1038, 1005, 969, 1045, 1012, 1043, 1076, 1032, 1026, 1024, 1017, 1011, 1018, 1033, 1041, 1049, 982, 1017, 1041, 1057, 992, 1058, 1083, 1005, 1015, 1018, 1025, 1021, 1004, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1025, 975, 987, 1047, 1011, 1047, 999, 1106, 961, 1062, 1010, 1022, 1064, 984, 988, 1047, 1018, 1036, 1018, 1008, 992, 939, 1027, 1022, 1073, 999, 1058, 1019, 1063, 1020, 1055, 1086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1043, 972, 1033, 1029, 1049, 1013, 1010, 1017, 1038, 999, 1029, 992, 1063, 989, 1012, 998, 1046, 1054, 1041, 1022, 1044, 1036, 1015, 1040, 1008, 1021, 1023, 1020, 1009, 1019, 1068, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1043, 1003, 1010, 1021, 1051, 1080, 1092, 1095, 1016, 1029, 1047, 970, 1032, 955, 989, 1011, 1024, 1040, 1024, 1029, 1026, 988, 1021, 1056, 1016, 1022, 1018, 1073, 1001, 974, 1026, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1003, 1041, 1023, 990, 23, 0, 0, 0, 988, 1005, 1021, 1027, 0, 0, 0, 0, 1040, 1052, 994, 1052, 16, 0, 0, 0, 1015, 1001, 989, 1051, 35, 1, 0, 0, 1066, 1017, 1003, 1013, 0, 0, 0, 0, 1014, 1030, 1010, 1023, 4, 3, 0, 0, 1000, 1053, 1051, 1044, 0, 0, 0, 0, 1035, 1024, 1004, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4041, 4154, 4092, 4099, 4084, 4148, 4070] avg=4096.0000 max=4154 max_rank=2 min=4041 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1000, 1044, 1029, 1004, 23, 0, 0, 0, 999, 1011, 1021, 971, 0, 0, 0, 0, 1012, 1064, 1014, 1010, 15, 0, 0, 0, 1017, 1023, 1040, 1014, 34, 1, 0, 0, 1013, 1027, 1040, 1075, 0, 0, 0, 0, 997, 1056, 1010, 997, 4, 3, 0, 0, 970, 1026, 1019, 1081, 0, 0, 0, 0, 1017, 994, 1052, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4002, 4115, 4129, 4155, 4067, 4096, 4104] avg=4096.0000 max=4155 max_rank=4 min=4002 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1035, 1039, 1054, 1019, 22, 0, 0, 0, 1069, 1059, 989, 1010, 0, 0, 0, 0, 957, 1051, 970, 986, 16, 0, 0, 0, 1015, 997, 1001, 1020, 36, 1, 0, 0, 992, 994, 1011, 1042, 0, 0, 0, 0, 1074, 1029, 993, 1026, 4, 3, 0, 0, 1035, 1101, 1009, 980, 0, 0, 0, 0, 1045, 1000, 1026, 1058, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4127, 3980, 4070, 4039, 4129, 4125, 4129] avg=4096.0000 max=4169 max_rank=0 min=3980 min_rank=2 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1017, 1035, 997, 1017, 22, 0, 0, 0, 1016, 996, 1016, 1023, 0, 0, 0, 0, 1015, 1028, 1031, 1021, 16, 0, 0, 0, 1027, 990, 1026, 1033, 35, 1, 0, 0, 1066, 978, 1034, 1054, 0, 0, 0, 0, 1020, 997, 1027, 1044, 4, 3, 0, 0, 1009, 1033, 1030, 1001, 0, 0, 0, 0, 1027, 1006, 1013, 1060, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4051, 4111, 4112, 4132, 4095, 4073, 4106] avg=4096.0000 max=4132 max_rank=4 min=4051 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1016, 990, 1038, 1005, 23, 0, 0, 0, 969, 1045, 1012, 1023, 0, 0, 0, 0, 1076, 1032, 1026, 1024, 16, 0, 0, 0, 1017, 1011, 1018, 1033, 37, 1, 0, 0, 1041, 1025, 982, 1017, 0, 0, 0, 0, 1041, 1057, 992, 1058, 4, 3, 0, 0, 1046, 1005, 1015, 1018, 0, 0, 0, 0, 1025, 1021, 1004, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4049, 4174, 4117, 4065, 4155, 4084, 4052] avg=4096.0000 max=4174 max_rank=2 min=4049 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1025, 975, 987, 1047, 22, 0, 0, 0, 1011, 1047, 999, 1085, 0, 0, 0, 0, 961, 1062, 1010, 1022, 17, 0, 0, 0, 1064, 984, 988, 1047, 36, 1, 0, 0, 1018, 1013, 1018, 1008, 0, 0, 0, 0, 992, 939, 1027, 1022, 4, 4, 0, 0, 1037, 999, 1058, 1019, 0, 0, 0, 0, 1063, 1020, 1055, 1082, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4142, 4072, 4120, 4057, 3988, 4113, 4220] avg=4096.0000 max=4220 max_rank=7 min=3988 min_rank=5 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1043, 972, 1033, 1029, 26, 0, 0, 0, 1049, 1013, 1010, 987, 0, 0, 0, 0, 1038, 999, 1029, 992, 22, 3, 0, 0, 1063, 989, 1012, 998, 38, 6, 0, 0, 1046, 1019, 1041, 1022, 0, 0, 0, 0, 1044, 1036, 1015, 1040, 8, 9, 0, 0, 970, 1021, 1023, 1020, 0, 0, 0, 0, 1009, 1019, 1068, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4059, 4083, 4106, 4128, 4152, 4034, 4103] avg=4096.0000 max=4152 max_rank=5 min=4034 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1043, 1003, 1010, 1021, 23, 0, 0, 0, 1051, 1080, 1092, 1074, 0, 0, 0, 0, 1016, 1029, 1047, 970, 17, 0, 0, 0, 1032, 955, 989, 1011, 34, 1, 0, 0, 1024, 1016, 1024, 1029, 0, 0, 0, 0, 1026, 988, 1021, 1056, 4, 3, 0, 0, 982, 1022, 1018, 1073, 0, 0, 0, 0, 1001, 974, 1026, 983, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4297, 4079, 4022, 4093, 4098, 4095, 3984] avg=4096.0000 max=4297 max_rank=1 min=3984 min_rank=7 imbalance_ratio=1.049072 max_load_reduction=28471 max_load_reduction_pct=0.868866 moved_to_echo=16270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8064, 8064, 8320, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1056, 1014, 1046, 1070, 1031, 1038, 1009, 987, 1009, 1038, 1052, 1040, 1053, 1039, 1003, 1038, 1000, 1049, 1008, 986, 1050, 1014, 1014, 977, 964, 1019, 1032, 1016, 1051, 987, 1057, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1045, 1020, 1020, 1025, 1026, 1008, 1003, 1027, 1025, 1078, 988, 997, 1047, 1037, 1063, 981, 1061, 1042, 1035, 1064, 1049, 1013, 991, 1006, 1004, 991, 1028, 1057, 1041, 988, 1003, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1058, 1039, 1023, 1003, 1069, 1000, 1036, 988, 1052, 996, 1044, 1000, 1000, 1034, 1000, 1013, 981, 1060, 1034, 1023, 1049, 1020, 1005, 1008, 1033, 1001, 1040, 1001, 1083, 1062, 1032, 981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1057, 1051, 1016, 1034, 985, 1035, 1028, 1036, 1003, 1005, 1012, 992, 1032, 1042, 1049, 1028, 1016, 1006, 1017, 1031, 1002, 1010, 1013, 1052, 1031, 1007, 1031, 1020, 1025, 1033, 1037, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1053, 1004, 986, 1007, 1017, 1044, 1042, 1029, 1028, 1013, 1018, 1026, 1042, 1021, 965, 1049, 1017, 1104, 1010, 1013, 1019, 986, 976, 1046, 1081, 1026, 1010, 1050, 1000, 1003, 1053, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1071, 1023, 1032, 1027, 1037, 1001, 1004, 1018, 997, 1044, 1032, 1001, 994, 975, 1013, 1009, 1005, 1033, 1022, 1047, 1019, 1056, 1016, 1015, 1062, 1063, 975, 1035, 1037, 1057, 1042, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1029, 1041, 1030, 1022, 1009, 1010, 998, 1036, 1018, 1045, 990, 1074, 1067, 986, 1042, 1037, 1066, 1011, 1045, 1026, 1045, 1005, 1043, 998, 1058, 986, 1008, 989, 1037, 1011, 1005, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[984, 1029, 1018, 1027, 984, 1044, 985, 1029, 1037, 1026, 1051, 1018, 998, 1057, 978, 1035, 1024, 1028, 1021, 1032, 1081, 1059, 1042, 1002, 1018, 1048, 1021, 1024, 1001, 1009, 1018, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1022, 1020, 1020, 1025, 0, 0, 0, 0, 1026, 1008, 1003, 1027, 21, 0, 0, 0, 1025, 1078, 988, 997, 2, 0, 0, 0, 1047, 1037, 1063, 981, 5, 0, 0, 0, 1061, 1025, 1035, 1064, 0, 0, 0, 0, 1049, 1013, 991, 1006, 8, 2, 0, 0, 1004, 991, 1028, 1057, 4, 0, 0, 0, 1039, 988, 1003, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4085, 4090, 4133, 4185, 4069, 4084, 4035] avg=4096.0000 max=4185 max_rank=4 min=4035 min_rank=7 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1032, 1014, 1046, 1070, 0, 0, 0, 0, 1031, 1038, 1009, 987, 22, 0, 0, 0, 1009, 1038, 1052, 1040, 2, 0, 0, 0, 1053, 1039, 1003, 1038, 5, 0, 0, 0, 1000, 1032, 1008, 986, 0, 0, 0, 0, 1050, 1014, 1014, 977, 8, 2, 0, 0, 964, 1019, 1032, 1016, 4, 0, 0, 0, 1049, 987, 1057, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4087, 4141, 4138, 4026, 4065, 4035, 4114] avg=4096.0000 max=4162 max_rank=0 min=4026 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1034, 1039, 1023, 1003, 0, 0, 0, 0, 1069, 1000, 1036, 988, 22, 0, 0, 0, 1052, 996, 1044, 1000, 2, 0, 0, 0, 1000, 1034, 1000, 1013, 5, 0, 0, 0, 981, 1043, 1034, 1023, 0, 0, 0, 0, 1049, 1020, 1005, 1008, 8, 2, 0, 0, 1033, 1001, 1040, 1001, 4, 0, 0, 0, 1081, 1062, 1032, 981, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4115, 4094, 4052, 4081, 4092, 4079, 4156] avg=4096.0000 max=4156 max_rank=7 min=4052 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1033, 1051, 1016, 1034, 0, 0, 0, 0, 985, 1035, 1028, 1036, 22, 0, 0, 0, 1003, 1005, 1012, 992, 2, 0, 0, 0, 1032, 1042, 1049, 1028, 4, 0, 0, 0, 1016, 990, 1017, 1031, 0, 0, 0, 0, 1002, 1010, 1013, 1052, 8, 2, 0, 0, 1031, 1007, 1031, 1020, 4, 0, 0, 0, 1023, 1033, 1037, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4106, 4014, 4155, 4054, 4087, 4093, 4125] avg=4096.0000 max=4155 max_rank=3 min=4014 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1029, 1004, 986, 1007, 0, 0, 0, 0, 1017, 1044, 1042, 1029, 22, 0, 0, 0, 1028, 1013, 1018, 1026, 2, 0, 0, 0, 1042, 1021, 965, 1049, 5, 0, 0, 0, 1017, 1085, 1010, 1013, 0, 0, 0, 0, 1019, 986, 976, 1046, 9, 2, 0, 0, 1081, 1026, 1010, 1050, 5, 0, 0, 0, 998, 1003, 1053, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4154, 4087, 4082, 4125, 4038, 4172, 4084] avg=4096.0000 max=4172 max_rank=6 min=4026 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1047, 1023, 1032, 1027, 0, 0, 0, 0, 1037, 1001, 1004, 1018, 22, 0, 0, 0, 997, 1044, 1032, 1001, 2, 0, 0, 0, 994, 975, 1013, 1009, 5, 0, 0, 0, 1005, 1016, 1022, 1047, 0, 0, 0, 0, 1019, 1056, 1016, 1015, 8, 2, 0, 0, 1062, 1063, 975, 1035, 4, 0, 0, 0, 1035, 1057, 1042, 1006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4082, 4076, 3996, 4090, 4116, 4139, 4140] avg=4096.0000 max=4140 max_rank=7 min=3996 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1002, 1041, 1030, 1022, 0, 0, 0, 0, 1009, 1010, 998, 1036, 24, 0, 0, 0, 1018, 1045, 990, 1074, 5, 0, 0, 0, 1067, 986, 1042, 1037, 7, 0, 0, 0, 1066, 982, 1045, 1026, 0, 0, 0, 0, 1045, 1005, 1043, 998, 13, 3, 0, 0, 1058, 986, 1008, 989, 9, 1, 0, 0, 1031, 1011, 1005, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4077, 4132, 4139, 4119, 4107, 4051, 4048] avg=4096.0000 max=4139 max_rank=3 min=4048 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[962, 1029, 1018, 1027, 0, 0, 0, 0, 984, 1044, 985, 1029, 20, 0, 0, 0, 1037, 1026, 1051, 1018, 2, 0, 0, 0, 998, 1057, 978, 1035, 5, 0, 0, 0, 1024, 1011, 1021, 1032, 0, 0, 0, 0, 1081, 1059, 1042, 1002, 8, 2, 0, 0, 1018, 1048, 1021, 1024, 4, 0, 0, 0, 999, 1009, 1018, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4062, 4134, 4073, 4088, 4194, 4115, 4066] avg=4096.0000 max=4194 max_rank=5 min=4036 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1029, 990, 1004, 1016, 1004, 1064, 1005, 1031, 1049, 1047, 1017, 1026, 980, 1063, 1022, 1029, 1062, 994, 1005, 1021, 1010, 998, 1038, 1048, 1045, 999, 977, 1077, 1045, 1054, 1020, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1047, 1077, 1024, 1049, 1016, 1013, 1026, 994, 1024, 1071, 1057, 1001, 1022, 981, 1043, 1011, 994, 985, 1029, 1077, 1000, 1012, 981, 1053, 1022, 1010, 1015, 1030, 1005, 1006, 1060, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[936, 993, 1030, 966, 971, 1059, 1009, 1046, 1038, 1084, 1000, 1048, 1040, 1044, 1050, 985, 1025, 994, 1085, 1021, 987, 1028, 1030, 1088, 1001, 1055, 1058, 1058, 1004, 1041, 977, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[994, 1099, 1049, 991, 1032, 995, 1024, 1007, 1042, 1034, 1033, 1030, 1032, 998, 1022, 1044, 1018, 1020, 1018, 1021, 1020, 1064, 977, 1009, 1030, 991, 1041, 1038, 988, 1061, 996, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1035, 975, 1049, 1021, 1014, 1064, 978, 1029, 1059, 1002, 1003, 1044, 1033, 1017, 1072, 1008, 1009, 1049, 1034, 1021, 971, 1063, 1044, 988, 1033, 1001, 1050, 1017, 1042, 1009, 1028, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1034, 1022, 995, 1051, 995, 1020, 1045, 1035, 1026, 1020, 1068, 981, 1008, 1010, 1045, 1059, 1032, 1036, 1056, 1041, 1007, 1039, 973, 1018, 1037, 975, 1048, 1032, 1015, 997, 1010, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[975, 1019, 1084, 1032, 1029, 1016, 1018, 1004, 1025, 1006, 1018, 1025, 1063, 1042, 1075, 985, 1023, 989, 1027, 1001, 1019, 1046, 1064, 1010, 1015, 1021, 1041, 1033, 1029, 994, 991, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1009, 1023, 1026, 1054, 1050, 1040, 1037, 1013, 1034, 996, 1061, 999, 1014, 1021, 980, 1040, 1020, 1008, 1005, 1014, 1004, 999, 1049, 1041, 1017, 1076, 1015, 1022, 1076, 1051, 950, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1029, 990, 1004, 1016, 4, 4, 0, 0, 1004, 1064, 1005, 1031, 10, 0, 0, 0, 1025, 1047, 1017, 1026, 0, 0, 0, 0, 980, 1063, 1014, 1029, 0, 0, 0, 0, 1062, 994, 1005, 1021, 4, 0, 0, 0, 1010, 998, 1038, 1048, 11, 0, 0, 0, 1045, 999, 977, 1063, 0, 0, 0, 0, 1045, 1054, 1020, 999, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4114, 4115, 4086, 4086, 4105, 4084, 4131] avg=4096.0000 max=4131 max_rank=7 min=4047 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1047, 1077, 1024, 1049, 4, 4, 0, 0, 1016, 1013, 1026, 994, 9, 0, 0, 0, 1001, 1071, 1057, 1001, 0, 0, 0, 0, 1022, 981, 1035, 1011, 0, 0, 0, 0, 994, 985, 1029, 1077, 4, 0, 0, 0, 1000, 1012, 981, 1053, 11, 0, 0, 0, 1022, 1010, 1015, 1017, 0, 0, 0, 0, 1005, 1006, 1060, 1033, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4205, 4058, 4130, 4049, 4089, 4057, 4064, 4116] avg=4096.0000 max=4205 max_rank=0 min=4049 min_rank=3 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[975, 1019, 1084, 1032, 4, 4, 0, 0, 1029, 1016, 1018, 1004, 9, 0, 0, 0, 1002, 1006, 1018, 1025, 0, 0, 0, 0, 1063, 1042, 1067, 985, 0, 0, 0, 0, 1023, 989, 1027, 1001, 4, 0, 0, 0, 1019, 1046, 1064, 1010, 11, 0, 0, 0, 1015, 1021, 1041, 1020, 0, 0, 0, 0, 1029, 994, 991, 1049, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4076, 4051, 4157, 4044, 4150, 4097, 4075] avg=4096.0000 max=4157 max_rank=3 min=4044 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1035, 975, 1049, 1021, 4, 4, 0, 0, 1014, 1064, 978, 1029, 9, 0, 0, 0, 1035, 1002, 1003, 1044, 0, 0, 0, 0, 1033, 1017, 1064, 1008, 0, 0, 0, 0, 1009, 1049, 1034, 1021, 4, 0, 0, 0, 971, 1063, 1044, 988, 11, 0, 0, 0, 1033, 1001, 1050, 1004, 0, 0, 0, 0, 1042, 1009, 1028, 1006, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4094, 4084, 4122, 4117, 4077, 4088, 4098] avg=4096.0000 max=4122 max_rank=3 min=4077 min_rank=5 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[936, 993, 1030, 966, 4, 4, 0, 0, 971, 1059, 1009, 1046, 9, 0, 0, 0, 1015, 1084, 1000, 1048, 0, 0, 0, 0, 1040, 1044, 1042, 985, 0, 0, 0, 0, 1025, 994, 1085, 1021, 4, 0, 0, 0, 987, 1028, 1030, 1088, 11, 0, 0, 0, 1001, 1055, 1058, 1045, 0, 0, 0, 0, 1004, 1041, 977, 1017, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[3933, 4094, 4147, 4111, 4129, 4144, 4159, 4051] avg=4096.0000 max=4159 max_rank=6 min=3933 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16483 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[994, 1099, 1049, 991, 4, 4, 0, 0, 1032, 995, 1024, 1007, 9, 0, 0, 0, 1019, 1034, 1033, 1030, 0, 0, 0, 0, 1032, 998, 1014, 1044, 0, 0, 0, 0, 1018, 1020, 1018, 1021, 4, 0, 0, 0, 1020, 1064, 977, 1009, 11, 0, 0, 0, 1030, 991, 1041, 1025, 0, 0, 0, 0, 988, 1061, 996, 1050, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4067, 4116, 4088, 4081, 4081, 4087, 4107] avg=4096.0000 max=4141 max_rank=0 min=4067 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1034, 1022, 995, 1051, 4, 4, 0, 0, 995, 1020, 1045, 1035, 9, 0, 0, 0, 1003, 1020, 1068, 981, 0, 0, 0, 0, 1008, 1010, 1037, 1059, 0, 0, 0, 0, 1032, 1036, 1056, 1041, 4, 0, 0, 0, 1007, 1039, 973, 1018, 11, 0, 0, 0, 1037, 975, 1048, 1019, 0, 0, 0, 0, 1015, 997, 1010, 1038, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4104, 4072, 4114, 4169, 4048, 4079, 4072] avg=4096.0000 max=4169 max_rank=4 min=4048 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1009, 1023, 1026, 1054, 8, 6, 0, 0, 1050, 1040, 1037, 1013, 14, 7, 0, 0, 997, 996, 1061, 999, 0, 0, 0, 0, 1014, 1021, 966, 1040, 0, 0, 0, 0, 1020, 1008, 1005, 1014, 6, 0, 0, 0, 1004, 999, 1049, 1041, 13, 0, 0, 0, 1017, 1076, 1015, 1002, 0, 0, 0, 0, 1076, 1051, 950, 1024, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4161, 4053, 4041, 4053, 4106, 4110, 4118] avg=4096.0000 max=4161 max_rank=1 min=4041 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1026, 1050, 1076, 1063, 1024, 994, 1019, 1016, 1004, 1012, 966, 1066, 1048, 1016, 1011, 1012, 997, 995, 1032, 1000, 1074, 1028, 1007, 1036, 1041, 996, 1002, 1054, 1071, 1028, 1007, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1056, 1060, 1034, 1019, 999, 1011, 1038, 1049, 1013, 1020, 1048, 1022, 971, 1040, 1036, 1021, 1033, 1044, 999, 1001, 1047, 1022, 966, 1037, 1037, 1034, 988, 1099, 1003, 1034, 1014, 973] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1025, 986, 1017, 1038, 1026, 1050, 1021, 1018, 1025, 1031, 974, 1026, 1045, 1034, 1027, 1038, 967, 1065, 1076, 1023, 1033, 1057, 1004, 1077, 1064, 983, 1005, 1052, 1005, 985, 971, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1064, 1025, 1007, 1000, 1039, 993, 1074, 1035, 1012, 1013, 1003, 1041, 1026, 1042, 1006, 1027, 1027, 1050, 1028, 1005, 1026, 1052, 1028, 1017, 1036, 1045, 1003, 1024, 1028, 1035, 986, 971] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1015, 1047, 1049, 979, 1011, 1008, 997, 996, 1018, 1012, 1017, 1042, 1030, 1024, 1023, 1021, 1052, 1003, 1016, 1066, 1041, 1032, 1033, 991, 1033, 1036, 1033, 1048, 1026, 1011, 1055, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1050, 1053, 1041, 1033, 1009, 1022, 1021, 1066, 1008, 970, 1022, 1055, 1033, 993, 982, 1015, 1053, 1047, 990, 1025, 1018, 1022, 999, 1068, 1002, 999, 1073, 1008, 1038, 1007, 1065, 981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1069, 1013, 990, 1015, 1048, 1090, 1013, 1027, 998, 1064, 1026, 1062, 1102, 1024, 958, 1033, 1019, 1017, 1022, 960, 1009, 1044, 1011, 998, 1018, 997, 1005, 1035, 1037, 1042, 1024, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[968, 1013, 1070, 1044, 1033, 1027, 1018, 1013, 1024, 1068, 1009, 1038, 1006, 1004, 1078, 1014, 1049, 1044, 979, 1029, 1010, 1009, 1013, 1032, 1031, 986, 1071, 1014, 1030, 1015, 1047, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1056, 1060, 1006, 1019, 0, 0, 0, 0, 999, 1011, 1038, 1045, 0, 0, 0, 0, 1013, 1020, 1048, 1022, 4, 3, 0, 0, 971, 1040, 1036, 1021, 3, 0, 0, 0, 1033, 1044, 999, 1001, 5, 1, 0, 0, 1047, 1014, 966, 1037, 0, 0, 0, 0, 1037, 1034, 988, 1089, 0, 0, 0, 0, 1003, 1034, 1014, 973, 28, 6, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4093, 4110, 4071, 4083, 4064, 4148, 4058] avg=4096.0000 max=4148 max_rank=6 min=4058 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1026, 1050, 1047, 1063, 0, 0, 0, 0, 1024, 994, 1019, 1012, 0, 0, 0, 0, 1004, 1012, 966, 1066, 4, 3, 0, 0, 1048, 1016, 1011, 1012, 3, 0, 0, 0, 997, 995, 1032, 1000, 5, 1, 0, 0, 1074, 1020, 1007, 1036, 0, 0, 0, 0, 1041, 996, 1002, 1044, 0, 0, 0, 0, 1071, 1028, 1007, 997, 29, 6, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4049, 4055, 4090, 4030, 4137, 4083, 4138] avg=4096.0000 max=4186 max_rank=0 min=4030 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1064, 1025, 980, 1000, 0, 0, 0, 0, 1039, 993, 1074, 1031, 0, 0, 0, 0, 1012, 1013, 1003, 1041, 3, 3, 0, 0, 1026, 1042, 1006, 1027, 3, 0, 0, 0, 1027, 1050, 1028, 1005, 5, 1, 0, 0, 1026, 1044, 1028, 1017, 0, 0, 0, 0, 1036, 1045, 1003, 1015, 0, 0, 0, 0, 1028, 1035, 986, 971, 27, 6, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4137, 4075, 4104, 4116, 4115, 4099, 4053] avg=4096.0000 max=4137 max_rank=1 min=4053 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1025, 986, 990, 1038, 0, 0, 0, 0, 1026, 1050, 1021, 1014, 0, 0, 0, 0, 1025, 1031, 974, 1026, 4, 3, 0, 0, 1045, 1034, 1027, 1038, 3, 0, 0, 0, 967, 1065, 1076, 1023, 5, 1, 0, 0, 1033, 1049, 1004, 1077, 0, 0, 0, 0, 1064, 983, 1005, 1042, 0, 0, 0, 0, 1005, 985, 971, 1020, 27, 6, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4111, 4063, 4147, 4137, 4163, 4094, 4014] avg=4096.0000 max=4163 max_rank=5 min=4014 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1015, 1047, 1021, 979, 0, 0, 0, 0, 1011, 1008, 997, 992, 0, 0, 0, 0, 1018, 1012, 1017, 1042, 4, 3, 0, 0, 1030, 1024, 1023, 1021, 3, 0, 0, 0, 1052, 1003, 1016, 1066, 5, 1, 0, 0, 1041, 1024, 1033, 991, 0, 0, 0, 0, 1033, 1036, 1033, 1038, 0, 0, 0, 0, 1026, 1011, 1055, 1003, 28, 6, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4008, 4096, 4101, 4143, 4089, 4140, 4129] avg=4096.0000 max=4143 max_rank=4 min=4008 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1050, 1053, 1013, 1033, 0, 0, 0, 0, 1009, 1022, 1021, 1062, 0, 0, 0, 0, 1008, 970, 1022, 1055, 3, 3, 0, 0, 1033, 993, 982, 1015, 3, 0, 0, 0, 1053, 1047, 990, 1025, 5, 1, 0, 0, 1018, 1014, 999, 1068, 0, 0, 0, 0, 1002, 999, 1073, 999, 0, 0, 0, 0, 1038, 1007, 1065, 981, 28, 6, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4114, 4061, 4026, 4121, 4099, 4073, 4125] avg=4096.0000 max=4149 max_rank=0 min=4026 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1069, 1013, 959, 1015, 0, 0, 0, 0, 1048, 1090, 1013, 1018, 0, 0, 0, 0, 998, 1064, 1026, 1062, 7, 6, 0, 0, 1102, 1024, 958, 1033, 7, 0, 0, 0, 1019, 1017, 1022, 960, 11, 2, 0, 0, 1009, 1027, 1011, 998, 0, 0, 0, 0, 1018, 997, 1005, 1018, 0, 0, 0, 0, 1037, 1042, 1024, 998, 31, 10, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4169, 4163, 4124, 4031, 4045, 4038, 4142] avg=4096.0000 max=4169 max_rank=1 min=4031 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[968, 1013, 1041, 1044, 0, 0, 0, 0, 1033, 1027, 1018, 1009, 0, 0, 0, 0, 1024, 1068, 1009, 1038, 3, 3, 0, 0, 1006, 1004, 1078, 1014, 3, 0, 0, 0, 1049, 1044, 979, 1029, 5, 1, 0, 0, 1010, 1001, 1013, 1032, 0, 0, 0, 0, 1031, 986, 1071, 1005, 0, 0, 0, 0, 1030, 1015, 1047, 982, 29, 6, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4087, 4145, 4105, 4107, 4056, 4093, 4109] avg=4096.0000 max=4145 max_rank=2 min=4056 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8320, 8064, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8448, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 7936, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1015, 1081, 1043, 973, 1070, 1007, 1043, 1025, 1047, 1067, 1014, 1026, 1042, 1023, 1014, 998, 1046, 1011, 986, 1016, 1046, 1012, 958, 1025, 1045, 987, 993, 1021, 994, 1058, 1028, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1039, 1044, 1015, 1044, 1050, 1070, 1043, 1007, 1031, 1008, 1014, 992, 1013, 1045, 980, 995, 998, 1013, 1059, 1024, 999, 1028, 996, 1056, 1020, 963, 1053, 1060, 1001, 1041, 1025, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1032, 1043, 1051, 1040, 1038, 1044, 981, 1021, 1006, 1005, 1007, 1026, 1046, 1014, 982, 1013, 1031, 1032, 1029, 1059, 1034, 1057, 1024, 995, 986, 1035, 1049, 944, 1095, 977, 1025, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1039, 1009, 1031, 1016, 1031, 983, 999, 998, 1057, 1072, 1048, 1035, 997, 1031, 1042, 1071, 1003, 1013, 1039, 1023, 1087, 974, 1059, 1000, 1015, 993, 1010, 994, 1010, 1049, 1012, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1032, 1010, 1037, 998, 1053, 1038, 1049, 1012, 1071, 1075, 1056, 1010, 960, 1025, 1001, 1050, 1001, 1003, 1026, 1064, 1035, 1053, 1021, 999, 1028, 1010, 1054, 982, 1036, 940, 1003, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1043, 959, 957, 1034, 1018, 1049, 1012, 1017, 995, 1024, 1030, 1062, 1040, 1052, 1037, 1074, 1028, 979, 1010, 1031, 1041, 1043, 1015, 1026, 1074, 1007, 1014, 1054, 1028, 1017, 996, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1034, 1021, 964, 958, 1025, 1013, 986, 1006, 1014, 1027, 1005, 1065, 1009, 1055, 1038, 1092, 1037, 1053, 1082, 1006, 1025, 1012, 1004, 965, 1047, 1008, 1030, 1034, 1105, 1022, 1057, 969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1046, 990, 1090, 1013, 1036, 941, 1038, 966, 1000, 1029, 1018, 1039, 1031, 1001, 989, 1006, 1076, 1045, 1023, 1087, 1054, 1001, 1018, 1049, 1021, 1030, 1045, 1024, 1033, 993, 1036, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1015, 1081, 1043, 973, 8, 0, 0, 0, 1070, 1007, 1043, 1025, 8, 3, 0, 0, 1047, 1042, 1014, 1026, 0, 0, 0, 0, 1042, 1023, 1014, 998, 0, 0, 0, 0, 1046, 1011, 986, 998, 0, 0, 0, 0, 1046, 1012, 958, 1025, 6, 0, 0, 0, 1045, 987, 993, 1021, 17, 0, 0, 0, 994, 1058, 1028, 1054, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4156, 4129, 4077, 4041, 4047, 4063, 4135] avg=4096.0000 max=4156 max_rank=1 min=4041 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1039, 1009, 1031, 1016, 8, 0, 0, 0, 1031, 983, 999, 998, 8, 3, 0, 0, 1057, 1047, 1048, 1035, 0, 0, 0, 0, 997, 1031, 1042, 1071, 0, 0, 0, 0, 1003, 1013, 1039, 1004, 0, 0, 0, 0, 1087, 974, 1059, 1000, 7, 0, 0, 0, 1015, 993, 1010, 994, 17, 0, 0, 0, 1010, 1049, 1012, 1028, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4022, 4187, 4141, 4059, 4127, 4029, 4100] avg=4096.0000 max=4187 max_rank=2 min=4022 min_rank=1 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1039, 1044, 1015, 1044, 8, 0, 0, 0, 1050, 1070, 1043, 1007, 8, 3, 0, 0, 1031, 984, 1014, 992, 0, 0, 0, 0, 1013, 1045, 980, 995, 0, 0, 0, 0, 998, 1013, 1059, 1005, 0, 0, 0, 0, 999, 1028, 996, 1056, 7, 0, 0, 0, 1020, 963, 1053, 1060, 16, 0, 0, 0, 1001, 1041, 1025, 1042, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4181, 4021, 4033, 4075, 4086, 4112, 4110] avg=4096.0000 max=4181 max_rank=1 min=4021 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1032, 1043, 1051, 1040, 8, 0, 0, 0, 1038, 1044, 981, 1021, 8, 3, 0, 0, 1006, 981, 1007, 1026, 0, 0, 0, 0, 1046, 1014, 982, 1013, 0, 0, 0, 0, 1031, 1032, 1029, 1040, 0, 0, 0, 0, 1034, 1057, 1024, 995, 7, 0, 0, 0, 986, 1035, 1049, 944, 16, 0, 0, 0, 1095, 977, 1025, 1047, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4095, 4020, 4055, 4132, 4117, 4030, 4145] avg=4096.0000 max=4174 max_rank=0 min=4020 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1032, 1010, 1037, 998, 8, 0, 0, 0, 1053, 1038, 1049, 1012, 8, 3, 0, 0, 1071, 1050, 1056, 1010, 0, 0, 0, 0, 960, 1025, 1001, 1050, 0, 0, 0, 0, 1001, 1003, 1026, 1045, 0, 0, 0, 0, 1035, 1053, 1021, 999, 7, 0, 0, 0, 1028, 1010, 1054, 982, 17, 0, 0, 0, 1036, 940, 1003, 1036, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4163, 4187, 4036, 4075, 4115, 4091, 4016] avg=4096.0000 max=4187 max_rank=2 min=4016 min_rank=7 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16297 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1043, 959, 957, 1034, 8, 0, 0, 0, 1018, 1049, 1012, 1017, 8, 3, 0, 0, 995, 999, 1030, 1062, 0, 0, 0, 0, 1040, 1052, 1037, 1074, 0, 0, 0, 0, 1028, 979, 1010, 1012, 0, 0, 0, 0, 1041, 1043, 1015, 1026, 7, 0, 0, 0, 1074, 1007, 1014, 1054, 17, 0, 0, 0, 1028, 1017, 996, 1002, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4001, 4107, 4086, 4203, 4029, 4132, 4166, 4044] avg=4096.0000 max=4203 max_rank=3 min=4001 min_rank=0 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1034, 1021, 964, 958, 8, 0, 0, 0, 1025, 1013, 986, 1006, 8, 3, 0, 0, 1014, 1002, 1005, 1065, 0, 0, 0, 0, 1009, 1055, 1038, 1092, 0, 0, 0, 0, 1037, 1053, 1082, 988, 0, 0, 0, 0, 1025, 1012, 1004, 965, 6, 0, 0, 0, 1047, 1008, 1030, 1034, 17, 0, 0, 0, 1105, 1022, 1057, 969, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3985, 4041, 4086, 4194, 4160, 4012, 4136, 4154] avg=4096.0000 max=4194 max_rank=3 min=3985 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1046, 990, 1090, 1013, 11, 0, 0, 0, 1036, 941, 1038, 966, 13, 9, 0, 0, 1000, 995, 1018, 1039, 0, 0, 0, 0, 1031, 1001, 989, 1006, 2, 0, 0, 0, 1076, 1045, 1023, 1053, 0, 0, 0, 0, 1054, 1001, 1018, 1049, 10, 0, 0, 0, 1021, 1030, 1045, 1024, 21, 0, 0, 0, 1033, 993, 1036, 1000, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4003, 4052, 4029, 4197, 4132, 4141, 4064] avg=4096.0000 max=4197 max_rank=4 min=4003 min_rank=1 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8192, 8192, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[995, 998, 1015, 1023, 1033, 1010, 1036, 1050, 1051, 1042, 1003, 1026, 1007, 988, 1024, 1056, 1023, 1056, 1075, 1012, 994, 1019, 1028, 1040, 1031, 1006, 985, 1002, 1073, 1015, 994, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[977, 1075, 1044, 976, 1016, 1036, 1048, 1050, 1008, 1033, 992, 1068, 1000, 1029, 1006, 1037, 968, 995, 1021, 1055, 998, 1036, 1010, 1106, 1014, 1033, 1004, 1046, 1069, 1045, 1001, 972] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1046, 1036, 1017, 1022, 977, 1028, 1042, 1032, 1031, 967, 1020, 1040, 1067, 1011, 1037, 1058, 1063, 1051, 981, 995, 981, 992, 1017, 1024, 1024, 1038, 987, 1068, 1052, 1020, 1064, 980] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[999, 1027, 1013, 1042, 1055, 982, 1017, 1041, 1005, 1025, 1028, 1036, 1013, 1001, 1053, 981, 1030, 1007, 1055, 1033, 1068, 1015, 1049, 1017, 1026, 1020, 1000, 1060, 1024, 1041, 994, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1001, 1027, 988, 997, 1034, 1033, 1058, 963, 1017, 949, 1071, 1050, 1034, 990, 1028, 996, 1018, 1030, 1050, 1019, 1022, 1011, 1022, 1038, 1060, 1009, 1032, 1058, 1028, 1084, 979, 1072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1039, 1078, 1015, 1011, 1044, 1027, 998, 963, 1045, 1019, 1063, 1089, 1054, 991, 1020, 1022, 986, 988, 1065, 1057, 1070, 1042, 1010, 999, 1018, 1098, 989, 956, 1039, 966, 1010, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1042, 1013, 1009, 1003, 1063, 1010, 977, 1040, 994, 1027, 1045, 1058, 1038, 1011, 1017, 1031, 1009, 1027, 1037, 1031, 1025, 1000, 1021, 1014, 1013, 1061, 1057, 1041, 1011, 1033, 1032, 978] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1062, 1004, 1045, 1049, 989, 986, 984, 1062, 1017, 1009, 983, 1055, 1003, 1040, 984, 1045, 1029, 1044, 1046, 1004, 1029, 1010, 1044, 1036, 1038, 1047, 1026, 955, 1060, 1029, 1038, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[995, 998, 1015, 1023, 4, 2, 2, 1, 1033, 1010, 1036, 1050, 10, 0, 0, 0, 1051, 1042, 1003, 1015, 0, 0, 0, 0, 1007, 988, 1024, 1056, 11, 0, 0, 0, 1023, 1056, 1064, 1012, 0, 0, 0, 0, 994, 1019, 1028, 1038, 0, 0, 0, 0, 1031, 1002, 985, 1002, 0, 0, 0, 0, 1071, 1015, 994, 1058, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4139, 4111, 4086, 4155, 4079, 4020, 4138] avg=4096.0000 max=4155 max_rank=4 min=4020 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1046, 1036, 1017, 1022, 4, 2, 2, 1, 977, 1028, 1042, 1032, 9, 0, 0, 0, 1031, 967, 1020, 1029, 0, 0, 0, 0, 1067, 1011, 1037, 1058, 11, 0, 0, 0, 1063, 1051, 971, 995, 0, 0, 0, 0, 981, 992, 1017, 1022, 0, 0, 0, 0, 1024, 1034, 987, 1068, 0, 0, 0, 0, 1050, 1020, 1064, 980, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4088, 4047, 4184, 4080, 4012, 4113, 4114] avg=4096.0000 max=4184 max_rank=3 min=4012 min_rank=5 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[999, 1027, 1013, 1042, 4, 2, 2, 1, 1055, 982, 1017, 1041, 10, 0, 0, 0, 1005, 1025, 1028, 1025, 0, 0, 0, 0, 1013, 1001, 1053, 981, 11, 0, 0, 0, 1030, 1007, 1044, 1033, 0, 0, 0, 0, 1068, 1015, 1049, 1015, 0, 0, 0, 0, 1026, 1016, 1000, 1060, 0, 0, 0, 0, 1022, 1041, 994, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4105, 4083, 4059, 4114, 4147, 4102, 4068] avg=4096.0000 max=4147 max_rank=5 min=4059 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[977, 1075, 1044, 976, 4, 2, 2, 1, 1016, 1036, 1048, 1050, 10, 0, 0, 0, 1008, 1033, 992, 1056, 0, 0, 0, 0, 1000, 1029, 1006, 1037, 12, 0, 0, 0, 968, 995, 1010, 1055, 0, 0, 0, 0, 998, 1036, 1010, 1104, 0, 0, 0, 0, 1014, 1029, 1004, 1046, 0, 0, 0, 0, 1067, 1045, 1001, 972, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4160, 4089, 4084, 4028, 4148, 4093, 4085] avg=4096.0000 max=4160 max_rank=1 min=4028 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1001, 1027, 988, 997, 4, 2, 2, 1, 1034, 1033, 1058, 963, 10, 0, 0, 0, 1017, 949, 1071, 1039, 0, 0, 0, 0, 1034, 990, 1028, 996, 11, 0, 0, 0, 1018, 1030, 1039, 1019, 0, 0, 0, 0, 1022, 1011, 1022, 1036, 0, 0, 0, 0, 1060, 1005, 1032, 1058, 0, 0, 0, 0, 1026, 1084, 979, 1072, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4098, 4076, 4059, 4106, 4091, 4155, 4161] avg=4096.0000 max=4161 max_rank=7 min=4022 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1039, 1078, 1015, 1011, 6, 5, 3, 3, 1044, 1027, 998, 963, 13, 2, 0, 0, 1045, 1019, 1063, 1071, 0, 0, 0, 0, 1054, 991, 1020, 1022, 16, 0, 0, 0, 986, 988, 1049, 1057, 0, 0, 0, 0, 1070, 1042, 1010, 994, 0, 0, 0, 0, 1018, 1092, 989, 956, 0, 0, 0, 0, 1036, 966, 1010, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4047, 4198, 4103, 4080, 4116, 4055, 4009] avg=4096.0000 max=4198 max_rank=2 min=4009 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1042, 1013, 1009, 1003, 4, 2, 2, 1, 1063, 1010, 977, 1040, 10, 0, 0, 0, 994, 1027, 1045, 1046, 0, 0, 0, 0, 1038, 1011, 1017, 1031, 12, 0, 0, 0, 1009, 1027, 1026, 1031, 0, 0, 0, 0, 1025, 1000, 1021, 1012, 0, 0, 0, 0, 1013, 1057, 1057, 1041, 0, 0, 0, 0, 1009, 1033, 1032, 978, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4100, 4112, 4109, 4093, 4058, 4168, 4052] avg=4096.0000 max=4168 max_rank=6 min=4052 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1062, 1004, 1045, 1049, 4, 2, 2, 1, 989, 986, 984, 1062, 10, 0, 0, 0, 1017, 1009, 983, 1043, 0, 0, 0, 0, 1003, 1040, 984, 1045, 12, 0, 0, 0, 1029, 1044, 1035, 1004, 0, 0, 0, 0, 1029, 1010, 1044, 1034, 0, 0, 0, 0, 1038, 1043, 1026, 955, 0, 0, 0, 0, 1058, 1029, 1038, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4031, 4052, 4084, 4112, 4117, 4062, 4141] avg=4096.0000 max=4169 max_rank=0 min=4031 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1043, 1056, 1007, 1057, 1050, 1055, 991, 1032, 999, 1004, 1067, 1036, 1032, 1038, 989, 1019, 1010, 1013, 1025, 1051, 1007, 1009, 1035, 1004, 1033, 1004, 1050, 958, 1001, 974, 1079, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1019, 1007, 1019, 1045, 1023, 1046, 1005, 1004, 994, 1074, 1052, 989, 1028, 1013, 1019, 1048, 994, 1036, 1037, 1060, 1003, 1014, 1034, 1016, 1035, 1030, 1022, 1043, 1016, 1048, 1026, 969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1034, 1072, 992, 1058, 1057, 1010, 1011, 1029, 1057, 1050, 1050, 1019, 1017, 983, 1032, 995, 1035, 1010, 1005, 1082, 980, 1059, 1050, 988, 985, 1009, 1040, 1035, 998, 950, 1012, 1064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1004, 1047, 1005, 1014, 980, 1055, 1027, 1031, 1011, 1061, 1052, 1046, 1017, 1027, 1021, 1004, 995, 1025, 981, 1083, 1021, 992, 1012, 1038, 989, 1030, 1038, 1039, 1036, 1028, 1016, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1037, 1049, 1015, 1046, 1010, 1003, 1012, 1044, 1008, 1049, 1029, 1039, 1064, 1025, 1062, 1016, 1043, 1041, 989, 995, 996, 1027, 979, 1034, 1057, 1000, 1043, 999, 1044, 1014, 1006, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[992, 974, 990, 1004, 985, 1025, 1073, 1046, 1071, 1032, 1035, 981, 945, 1007, 1015, 1014, 1055, 1064, 1057, 1066, 1032, 1048, 1014, 1001, 1060, 1050, 1003, 1056, 997, 1029, 1005, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[999, 1033, 1015, 994, 1061, 990, 977, 1023, 968, 1005, 1021, 999, 1015, 1035, 1015, 1014, 1054, 1039, 1022, 1013, 1076, 1028, 1045, 1041, 1010, 1053, 1061, 1053, 1021, 1052, 1048, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1000, 1013, 1025, 987, 1051, 990, 1017, 1029, 1021, 1008, 1014, 1064, 1048, 994, 1004, 1035, 1012, 1039, 1026, 1024, 1023, 1033, 1043, 1027, 1024, 1044, 1035, 1032, 1018, 998, 1018, 1072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1004, 1047, 1005, 1014, 8, 6, 0, 0, 980, 1055, 1027, 1031, 3, 0, 0, 0, 1011, 1061, 1036, 1046, 0, 0, 0, 0, 1017, 1027, 1021, 1004, 23, 0, 0, 0, 995, 1025, 981, 1056, 0, 0, 0, 0, 1021, 992, 1012, 1038, 7, 0, 0, 0, 989, 1030, 1019, 1039, 0, 0, 0, 0, 1036, 1028, 1016, 1043, 11, 4, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4096, 4154, 4092, 4057, 4070, 4077, 4138] avg=4096.0000 max=4154 max_rank=2 min=4057 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1043, 1056, 1007, 1057, 8, 6, 0, 0, 1050, 1055, 991, 1032, 3, 0, 0, 0, 999, 1004, 1051, 1036, 0, 0, 0, 0, 1032, 1038, 989, 1019, 22, 0, 0, 0, 1010, 1013, 1025, 1025, 0, 0, 0, 0, 1007, 1009, 1035, 1004, 7, 0, 0, 0, 1033, 1004, 1031, 958, 0, 0, 0, 0, 1001, 974, 1079, 1040, 11, 4, 0, 0] rerouted_tokens_per_ep_rank=[4177, 4131, 4090, 4100, 4073, 4062, 4026, 4109] avg=4096.0000 max=4177 max_rank=0 min=4026 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1019, 1007, 1019, 1045, 7, 6, 0, 0, 1023, 1046, 1005, 1004, 3, 0, 0, 0, 994, 1074, 1036, 989, 0, 0, 0, 0, 1028, 1013, 1019, 1048, 22, 0, 0, 0, 994, 1036, 1037, 1034, 0, 0, 0, 0, 1003, 1014, 1034, 1016, 7, 0, 0, 0, 1035, 1030, 1005, 1043, 0, 0, 0, 0, 1016, 1048, 1026, 969, 10, 4, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4081, 4093, 4130, 4101, 4074, 4113, 4073] avg=4096.0000 max=4130 max_rank=3 min=4073 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1034, 1072, 992, 1058, 8, 6, 0, 0, 1057, 1010, 1011, 1029, 3, 0, 0, 0, 1057, 1050, 1034, 1019, 0, 0, 0, 0, 1017, 983, 1032, 995, 22, 0, 0, 0, 1035, 1010, 1005, 1056, 0, 0, 0, 0, 980, 1059, 1050, 988, 7, 0, 0, 0, 985, 1009, 1021, 1035, 0, 0, 0, 0, 998, 950, 1012, 1064, 11, 4, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4110, 4160, 4049, 4106, 4084, 4050, 4039] avg=4096.0000 max=4170 max_rank=0 min=4039 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[992, 974, 990, 1004, 7, 6, 0, 0, 985, 1025, 1073, 1046, 3, 0, 0, 0, 1071, 1032, 1019, 981, 0, 0, 0, 0, 945, 1007, 1015, 1014, 22, 0, 0, 0, 1055, 1064, 1057, 1040, 0, 0, 0, 0, 1032, 1048, 1014, 1001, 7, 0, 0, 0, 1060, 1050, 986, 1056, 0, 0, 0, 0, 997, 1029, 1005, 1042, 10, 4, 0, 0] rerouted_tokens_per_ep_rank=[3973, 4132, 4103, 4003, 4216, 4102, 4152, 4087] avg=4096.0000 max=4216 max_rank=4 min=3973 min_rank=0 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16557 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[999, 1033, 1015, 994, 8, 6, 0, 0, 1061, 990, 977, 1023, 3, 0, 0, 0, 968, 1005, 1005, 999, 0, 0, 0, 0, 1015, 1035, 1015, 1014, 21, 0, 0, 0, 1054, 1039, 1022, 988, 0, 0, 0, 0, 1076, 1028, 1045, 1041, 7, 0, 0, 0, 1010, 1053, 1042, 1053, 0, 0, 0, 0, 1021, 1052, 1048, 988, 11, 4, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4054, 3977, 4100, 4103, 4197, 4158, 4124] avg=4096.0000 max=4197 max_rank=5 min=3977 min_rank=2 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1037, 1049, 1015, 1046, 8, 6, 0, 0, 1010, 1003, 1012, 1044, 3, 0, 0, 0, 1008, 1049, 1013, 1039, 0, 0, 0, 0, 1064, 1025, 1062, 1016, 21, 0, 0, 0, 1043, 1041, 989, 970, 0, 0, 0, 0, 996, 1027, 979, 1034, 7, 0, 0, 0, 1057, 1000, 1024, 999, 0, 0, 0, 0, 1044, 1014, 1006, 993, 11, 4, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4072, 4109, 4188, 4043, 4043, 4080, 4072] avg=4096.0000 max=4188 max_rank=3 min=4043 min_rank=4 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16238 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1000, 1013, 1025, 987, 10, 10, 0, 0, 1051, 990, 1017, 1029, 5, 0, 0, 0, 1021, 1008, 989, 1064, 0, 0, 0, 0, 1048, 994, 1004, 1035, 25, 0, 0, 0, 1012, 1039, 1026, 992, 0, 0, 0, 0, 1023, 1033, 1043, 1027, 10, 0, 0, 0, 1024, 1044, 1012, 1032, 0, 0, 0, 0, 1018, 998, 1018, 1072, 13, 7, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4092, 4082, 4106, 4069, 4136, 4112, 4126] avg=4096.0000 max=4136 max_rank=5 min=4045 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1054, 993, 978, 1060, 1019, 1066, 1029, 1080, 977, 1057, 1007, 999, 983, 990, 1027, 1033, 1001, 1050, 1013, 970, 1036, 990, 1005, 1078, 1067, 1052, 1005, 982, 1048, 1059, 1044, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1020, 1013, 1055, 994, 1033, 996, 1028, 970, 1063, 1039, 1013, 1103, 1019, 1048, 982, 1001, 1098, 1026, 1040, 1021, 1003, 1004, 1044, 1009, 1041, 965, 1022, 1064, 1022, 1020, 1029, 983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1007, 992, 1032, 1012, 1048, 1024, 1091, 1023, 1049, 1059, 1016, 986, 1013, 1041, 1044, 972, 993, 1015, 1011, 1052, 981, 1011, 1056, 1007, 1042, 1018, 1072, 1045, 999, 994, 1073, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1076, 993, 1059, 1002, 1021, 1045, 975, 1030, 992, 1003, 1074, 1043, 1031, 1044, 1088, 1031, 973, 1031, 1010, 1033, 999, 1017, 1034, 990, 1031, 1061, 1009, 1008, 1047, 996, 967, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1052, 1014, 1014, 974, 1018, 1069, 1031, 997, 1052, 1044, 934, 981, 1040, 1072, 1045, 1037, 1007, 1054, 1035, 1057, 994, 980, 1055, 948, 1058, 996, 1017, 995, 1067, 1003, 1097, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[950, 998, 1056, 1006, 1063, 1028, 997, 1010, 1029, 1025, 996, 980, 1051, 1049, 1081, 1040, 1020, 1049, 1059, 1041, 1033, 1050, 1007, 1039, 1026, 1009, 1024, 982, 1054, 965, 1016, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1039, 986, 1036, 1002, 984, 1066, 1001, 997, 992, 1033, 1013, 993, 1004, 1043, 1040, 1006, 1037, 1013, 1019, 1019, 1027, 1046, 1079, 1032, 1076, 1020, 1027, 1002, 1011, 1058, 1047, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1029, 1013, 1005, 1019, 1011, 981, 1081, 1032, 983, 1037, 1034, 1018, 1008, 1043, 998, 1008, 1035, 1064, 1043, 1083, 1047, 1045, 1038, 1001, 1004, 1018, 997, 972, 1019, 1055, 1002, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1054, 993, 978, 1060, 25, 3, 0, 0, 1019, 1058, 1029, 1080, 0, 0, 0, 0, 977, 1057, 1007, 999, 13, 3, 0, 0, 983, 974, 1027, 1033, 0, 0, 0, 0, 1001, 1025, 1013, 970, 0, 0, 0, 0, 1036, 990, 1005, 1078, 8, 1, 0, 0, 1067, 1052, 1005, 982, 7, 0, 0, 0, 1048, 1059, 1033, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4186, 4056, 4017, 4009, 4118, 4113, 4156] avg=4096.0000 max=4186 max_rank=1 min=4009 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1020, 1013, 1055, 994, 25, 3, 0, 0, 1033, 988, 1028, 970, 0, 0, 0, 0, 1063, 1039, 1013, 1103, 14, 3, 0, 0, 1019, 1031, 982, 1001, 0, 0, 0, 0, 1098, 1001, 1040, 1021, 0, 0, 0, 0, 1003, 1004, 1044, 1009, 8, 1, 0, 0, 1041, 965, 1022, 1064, 7, 0, 0, 0, 1022, 1020, 1018, 983, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4019, 4235, 4033, 4160, 4069, 4099, 4043] avg=4096.0000 max=4235 max_rank=2 min=4019 min_rank=1 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1007, 992, 1032, 1012, 24, 3, 0, 0, 1048, 1016, 1091, 1023, 0, 0, 0, 0, 1049, 1059, 1016, 986, 14, 4, 0, 0, 1013, 1024, 1044, 972, 0, 0, 0, 0, 993, 991, 1011, 1052, 0, 0, 0, 0, 981, 1011, 1056, 1007, 8, 1, 0, 0, 1042, 1018, 1072, 1045, 7, 0, 0, 0, 999, 994, 1061, 990, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4178, 4128, 4053, 4047, 4064, 4184, 4044] avg=4096.0000 max=4184 max_rank=6 min=4044 min_rank=7 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[950, 998, 1056, 1006, 25, 3, 0, 0, 1063, 1020, 997, 1010, 0, 0, 0, 0, 1029, 1025, 996, 980, 14, 3, 0, 0, 1051, 1032, 1081, 1040, 0, 0, 0, 0, 1020, 1024, 1059, 1041, 0, 0, 0, 0, 1033, 1050, 1007, 1039, 8, 1, 0, 0, 1026, 1009, 1024, 982, 7, 0, 0, 0, 1054, 965, 1005, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4090, 4047, 4204, 4144, 4138, 4048, 4059] avg=4096.0000 max=4204 max_rank=3 min=4038 min_rank=0 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1076, 993, 1059, 1002, 25, 3, 0, 0, 1021, 1037, 975, 1030, 0, 0, 0, 0, 992, 1003, 1074, 1043, 14, 3, 0, 0, 1031, 1027, 1088, 1031, 0, 0, 0, 0, 973, 1006, 1010, 1033, 0, 0, 0, 0, 999, 1017, 1034, 990, 7, 1, 0, 0, 1031, 1061, 1009, 1008, 7, 0, 0, 0, 1047, 996, 957, 1055, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4063, 4129, 4177, 4022, 4048, 4116, 4055] avg=4096.0000 max=4177 max_rank=3 min=4022 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16241 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1039, 986, 1036, 1002, 24, 3, 0, 0, 984, 1058, 1001, 997, 0, 0, 0, 0, 992, 1033, 1013, 993, 14, 3, 0, 0, 1004, 1026, 1040, 1006, 0, 0, 0, 0, 1037, 989, 1019, 1019, 0, 0, 0, 0, 1027, 1046, 1079, 1032, 8, 1, 0, 0, 1076, 1020, 1027, 1002, 7, 0, 0, 0, 1011, 1058, 1036, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4040, 4048, 4076, 4064, 4193, 4132, 4125] avg=4096.0000 max=4193 max_rank=5 min=4040 min_rank=1 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16514 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1052, 1014, 1014, 974, 30, 10, 0, 0, 1018, 1049, 1031, 997, 0, 0, 0, 0, 1052, 1044, 934, 981, 16, 9, 0, 0, 1040, 1046, 1045, 1037, 0, 0, 0, 0, 1007, 1024, 1035, 1057, 0, 0, 0, 0, 994, 980, 1055, 948, 13, 8, 0, 0, 1058, 996, 1017, 995, 12, 0, 0, 0, 1067, 1003, 1075, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4095, 4036, 4168, 4123, 3998, 4078, 4176] avg=4096.0000 max=4176 max_rank=7 min=3998 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1029, 1013, 1005, 1019, 26, 3, 0, 0, 1011, 973, 1081, 1032, 0, 0, 0, 0, 983, 1037, 1034, 1018, 14, 3, 0, 0, 1008, 1026, 998, 1008, 0, 0, 0, 0, 1035, 1038, 1043, 1083, 0, 0, 0, 0, 1047, 1045, 1038, 1001, 8, 1, 0, 0, 1004, 1018, 997, 972, 7, 0, 0, 0, 1019, 1055, 991, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4097, 4089, 4040, 4199, 4140, 3998, 4110] avg=4096.0000 max=4199 max_rank=4 min=3998 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 8192, 8192, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1068, 1026, 972, 1026, 1047, 1043, 1052, 1085, 1030, 1032, 1007, 1010, 999, 1060, 1019, 985, 1006, 1072, 1024, 1044, 1003, 1032, 1017, 999, 977, 1032, 1027, 985, 1020, 1004, 1042, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1033, 1021, 1041, 1015, 1067, 1043, 1043, 1012, 1002, 1010, 985, 984, 1021, 1006, 992, 1031, 1006, 988, 1021, 1000, 1029, 995, 1044, 1063, 1031, 1012, 1046, 1017, 1038, 1031, 1049, 1092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1015, 1029, 1017, 988, 1034, 1026, 1046, 1025, 1030, 1050, 1017, 1019, 1029, 1050, 1047, 1038, 1041, 1056, 1018, 1010, 1010, 1024, 989, 973, 1031, 1029, 1001, 1029, 1025, 987, 1082, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1065, 1012, 1057, 1008, 1024, 1051, 1036, 1011, 1023, 1017, 1022, 1070, 997, 953, 965, 1022, 994, 1028, 1045, 1025, 1036, 1078, 1086, 1031, 1000, 1022, 1090, 985, 984, 1021, 1008, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1010, 1018, 1033, 1067, 1042, 1067, 1017, 1042, 1064, 1035, 995, 1018, 1082, 1023, 1033, 1022, 1007, 1031, 1038, 992, 978, 1033, 984, 1023, 1045, 986, 1028, 982, 1038, 1001, 1026, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[996, 991, 1035, 1014, 1059, 1042, 1059, 1035, 1053, 1040, 993, 1023, 998, 1007, 1018, 1089, 1023, 1015, 1015, 1023, 1024, 993, 1050, 1013, 1034, 1085, 1054, 1000, 959, 1044, 994, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1041, 1019, 1023, 1015, 1028, 995, 1008, 986, 1096, 1024, 1033, 1036, 1051, 1023, 1007, 1056, 1014, 999, 1039, 1028, 981, 1019, 1030, 1053, 1067, 985, 1012, 1003, 1028, 1002, 1053, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1006, 1003, 1005, 1034, 1024, 992, 1044, 994, 1029, 1007, 1006, 1006, 1030, 1018, 990, 1009, 1060, 1006, 1060, 1002, 1028, 1053, 1053, 1045, 1032, 1028, 1038, 1016, 1060, 1042, 1050, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1068, 1026, 972, 1026, 8, 0, 0, 0, 1010, 1043, 1052, 1085, 0, 0, 0, 0, 1030, 1032, 1007, 1010, 0, 0, 0, 0, 999, 1060, 1019, 985, 12, 0, 0, 0, 1006, 1072, 1024, 1044, 4, 0, 0, 0, 1003, 1032, 1017, 999, 0, 0, 0, 0, 977, 1032, 1027, 985, 7, 0, 0, 0, 1020, 1004, 1042, 1023, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4190, 4079, 4075, 4150, 4051, 4028, 4095] avg=4096.0000 max=4190 max_rank=1 min=4028 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1015, 1029, 1017, 988, 8, 0, 0, 0, 997, 1026, 1046, 1025, 0, 0, 0, 0, 1030, 1050, 1017, 1019, 0, 0, 0, 0, 1029, 1050, 1047, 1038, 12, 0, 0, 0, 1041, 1056, 1018, 1010, 4, 0, 0, 0, 1010, 1024, 989, 973, 0, 0, 0, 0, 1031, 1029, 1001, 1029, 7, 0, 0, 0, 1025, 987, 1082, 1003, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4094, 4116, 4176, 4129, 3996, 4097, 4103] avg=4096.0000 max=4176 max_rank=3 min=3996 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1010, 1018, 1033, 1067, 8, 0, 0, 0, 1005, 1067, 1017, 1042, 0, 0, 0, 0, 1064, 1035, 995, 1018, 0, 0, 0, 0, 1082, 1023, 1033, 1022, 12, 0, 0, 0, 1007, 1031, 1038, 992, 4, 0, 0, 0, 978, 1033, 984, 1023, 0, 0, 0, 0, 1045, 986, 1028, 982, 7, 0, 0, 0, 1038, 1001, 1026, 1008, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4131, 4112, 4172, 4072, 4018, 4048, 4079] avg=4096.0000 max=4172 max_rank=3 min=4018 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16217 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1065, 1012, 1057, 1008, 7, 0, 0, 0, 988, 1051, 1036, 1011, 0, 0, 0, 0, 1023, 1017, 1022, 1070, 0, 0, 0, 0, 997, 953, 965, 1022, 12, 0, 0, 0, 994, 1028, 1045, 1025, 4, 0, 0, 0, 1036, 1078, 1086, 1031, 0, 0, 0, 0, 1000, 1022, 1090, 985, 7, 0, 0, 0, 984, 1021, 1008, 1002, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4086, 4132, 3949, 4096, 4231, 4104, 4021] avg=4096.0000 max=4231 max_rank=5 min=3949 min_rank=3 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1033, 1021, 1041, 1015, 11, 0, 0, 0, 1013, 1043, 1043, 1012, 0, 0, 0, 0, 1002, 1010, 985, 984, 1, 1, 0, 0, 1021, 1006, 992, 1031, 14, 0, 0, 0, 1006, 988, 1021, 1000, 10, 0, 0, 0, 1029, 995, 1043, 1063, 0, 0, 0, 0, 1031, 1012, 1046, 1017, 10, 0, 0, 0, 1038, 1031, 1049, 1092, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4111, 3983, 4064, 4025, 4130, 4116, 4218] avg=4096.0000 max=4218 max_rank=7 min=3983 min_rank=2 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[996, 991, 1035, 1014, 8, 0, 0, 0, 1022, 1042, 1059, 1035, 0, 0, 0, 0, 1053, 1040, 993, 1023, 0, 0, 0, 0, 998, 1007, 1018, 1089, 12, 0, 0, 0, 1023, 1015, 1015, 1023, 4, 0, 0, 0, 1024, 993, 1050, 1013, 0, 0, 0, 0, 1034, 1085, 1054, 1000, 7, 0, 0, 0, 959, 1044, 994, 990, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4158, 4109, 4124, 4080, 4080, 4180, 3993] avg=4096.0000 max=4180 max_rank=6 min=3993 min_rank=7 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1041, 1019, 1023, 1015, 8, 0, 0, 0, 991, 995, 1008, 986, 0, 0, 0, 0, 1096, 1024, 1033, 1036, 0, 0, 0, 0, 1051, 1023, 1007, 1056, 12, 0, 0, 0, 1014, 999, 1039, 1028, 4, 0, 0, 0, 981, 1019, 1030, 1053, 0, 0, 0, 0, 1067, 985, 1012, 1003, 7, 0, 0, 0, 1028, 1002, 1053, 1014, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 3980, 4189, 4149, 4084, 4083, 4074, 4103] avg=4096.0000 max=4189 max_rank=2 min=3980 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1006, 1003, 1005, 1034, 7, 0, 0, 0, 988, 992, 1044, 994, 0, 0, 0, 0, 1029, 1007, 1006, 1006, 0, 0, 0, 0, 1030, 1018, 990, 1009, 12, 0, 0, 0, 1060, 1006, 1060, 1002, 4, 0, 0, 0, 1028, 1053, 1053, 1045, 0, 0, 0, 0, 1032, 1028, 1038, 1016, 7, 0, 0, 0, 1060, 1042, 1050, 998, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4018, 4048, 4059, 4132, 4179, 4121, 4156] avg=4096.0000 max=4179 max_rank=5 min=4018 min_rank=1 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16588 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8448, 8320, 8064, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1042, 1042, 1011, 1000, 1078, 985, 1044, 1039, 1038, 1038, 985, 1034, 1006, 1048, 1045, 970, 993, 1034, 1063, 981, 1042, 1001, 1010, 997, 991, 1024, 1079, 1064, 1004, 1041, 1046, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1016, 970, 1040, 985, 995, 1038, 1084, 997, 1048, 1054, 1020, 995, 1005, 1046, 1000, 1065, 998, 1064, 1022, 1045, 1002, 1028, 1020, 1075, 1015, 1036, 1030, 1040, 1031, 1054, 997, 953] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[967, 964, 1096, 1072, 984, 1044, 1002, 1003, 1076, 1060, 970, 1039, 948, 1051, 1011, 1005, 1010, 1019, 1063, 1003, 1043, 1010, 1053, 1028, 1047, 1016, 973, 1025, 1070, 1056, 1027, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1003, 1039, 1028, 1023, 1021, 1045, 1002, 1060, 1053, 971, 1067, 1016, 946, 996, 1060, 1032, 1007, 1005, 1002, 1020, 1050, 1059, 1022, 1066, 1032, 1057, 991, 997, 1031, 991, 1042, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1016, 1047, 1028, 1053, 1015, 975, 1042, 982, 1035, 1023, 1046, 1013, 1028, 1024, 1023, 1048, 1021, 1033, 986, 1050, 1047, 1030, 1064, 1009, 1030, 1020, 999, 1031, 1015, 1004, 1041, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1047, 1032, 1007, 1064, 1052, 1045, 998, 983, 1062, 1022, 1000, 1021, 1015, 975, 1007, 1030, 987, 1059, 1049, 1053, 1017, 1028, 1019, 1026, 1078, 1035, 998, 1027, 954, 1039, 1009, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1015, 1021, 1003, 1023, 989, 1022, 1028, 972, 1034, 1038, 1016, 1066, 995, 1047, 959, 1063, 1007, 1011, 1019, 1080, 1055, 1029, 1060, 1073, 995, 1038, 1031, 1017, 966, 1046, 991, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1015, 1047, 1002, 1054, 995, 968, 1011, 1023, 1040, 1030, 1045, 991, 1033, 1017, 1038, 1041, 1005, 1009, 976, 1079, 1031, 1021, 1023, 1037, 1010, 1015, 1023, 1024, 1066, 1043, 1059, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1042, 1042, 1011, 1000, 0, 0, 0, 0, 1078, 985, 1044, 1039, 29, 0, 0, 0, 1017, 1038, 985, 1034, 0, 0, 0, 0, 1006, 1048, 1045, 970, 16, 7, 0, 0, 993, 1034, 1063, 981, 1, 0, 0, 0, 1042, 1001, 1010, 961, 0, 0, 0, 0, 991, 1022, 1079, 1064, 0, 0, 0, 0, 1004, 1041, 1046, 993, 5, 1, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4175, 4074, 4092, 4072, 4014, 4156, 4090] avg=4096.0000 max=4175 max_rank=1 min=4014 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1016, 970, 1040, 985, 0, 0, 0, 0, 995, 1038, 1084, 997, 31, 0, 0, 0, 1027, 1054, 1020, 995, 0, 0, 0, 0, 1005, 1046, 1000, 1065, 16, 7, 0, 0, 998, 1064, 1022, 1045, 1, 0, 0, 0, 1002, 1028, 1020, 1037, 0, 0, 0, 0, 1015, 1034, 1030, 1040, 0, 0, 0, 0, 1031, 1054, 997, 953, 5, 1, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4145, 4096, 4139, 4130, 4087, 4119, 4041] avg=4096.0000 max=4145 max_rank=1 min=4011 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[967, 964, 1096, 1072, 0, 0, 0, 0, 984, 1044, 1002, 1003, 30, 0, 0, 0, 1054, 1060, 970, 1039, 0, 0, 0, 0, 948, 1051, 1011, 1005, 16, 7, 0, 0, 1010, 1019, 1063, 1003, 1, 0, 0, 0, 1043, 1010, 1053, 991, 0, 0, 0, 0, 1047, 1014, 973, 1025, 0, 0, 0, 0, 1070, 1056, 1027, 1033, 6, 1, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4063, 4123, 4038, 4096, 4097, 4059, 4193] avg=4096.0000 max=4193 max_rank=7 min=4038 min_rank=3 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1003, 1039, 1028, 1023, 0, 0, 0, 0, 1021, 1045, 1002, 1060, 31, 0, 0, 0, 1032, 971, 1067, 1016, 0, 0, 0, 0, 946, 996, 1060, 1032, 16, 7, 0, 0, 1007, 1005, 1002, 1020, 1, 0, 0, 0, 1050, 1059, 1022, 1028, 0, 0, 0, 0, 1032, 1055, 991, 997, 0, 0, 0, 0, 1031, 991, 1042, 1034, 5, 1, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4159, 4086, 4057, 4035, 4159, 4075, 4104] avg=4096.0000 max=4159 max_rank=1 min=4035 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1015, 1021, 1003, 1023, 0, 0, 0, 0, 989, 1022, 1028, 972, 31, 0, 0, 0, 1013, 1038, 1016, 1066, 0, 0, 0, 0, 995, 1047, 959, 1063, 16, 7, 0, 0, 1007, 1011, 1019, 1080, 1, 0, 0, 0, 1055, 1029, 1060, 1035, 0, 0, 0, 0, 995, 1036, 1031, 1017, 0, 0, 0, 0, 966, 1046, 991, 1059, 5, 1, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4042, 4133, 4087, 4118, 4179, 4079, 4068] avg=4096.0000 max=4179 max_rank=5 min=4042 min_rank=1 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1016, 1047, 1028, 1053, 0, 0, 0, 0, 1015, 975, 1042, 982, 29, 0, 0, 0, 1014, 1023, 1046, 1013, 0, 0, 0, 0, 1028, 1024, 1023, 1048, 16, 7, 0, 0, 1021, 1033, 986, 1050, 1, 0, 0, 0, 1047, 1030, 1064, 973, 0, 0, 0, 0, 1030, 1018, 999, 1031, 0, 0, 0, 0, 1015, 1004, 1041, 990, 5, 1, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4043, 4096, 4146, 4091, 4114, 4078, 4056] avg=4096.0000 max=4146 max_rank=3 min=4043 min_rank=1 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1047, 1032, 1007, 1060, 0, 0, 0, 0, 1052, 1045, 998, 983, 36, 0, 0, 0, 1032, 1022, 1000, 1021, 0, 0, 0, 0, 1015, 975, 1007, 1030, 19, 11, 0, 0, 987, 1059, 1049, 1053, 4, 4, 0, 0, 1017, 1028, 1019, 979, 0, 0, 0, 0, 1078, 1029, 998, 1027, 0, 0, 0, 0, 954, 1039, 1009, 1030, 11, 2, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4114, 4075, 4057, 4156, 4043, 4132, 4045] avg=4096.0000 max=4156 max_rank=4 min=4043 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1015, 1047, 1002, 1054, 0, 0, 0, 0, 995, 968, 1011, 1023, 30, 0, 0, 0, 1019, 1030, 1045, 991, 0, 0, 0, 0, 1033, 1017, 1038, 1041, 16, 7, 0, 0, 1005, 1009, 976, 1079, 1, 0, 0, 0, 1031, 1021, 1023, 1000, 0, 0, 0, 0, 1010, 1013, 1023, 1024, 0, 0, 0, 0, 1066, 1043, 1059, 997, 5, 1, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4027, 4085, 4152, 4070, 4075, 4070, 4171] avg=4096.0000 max=4171 max_rank=7 min=4027 min_rank=1 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8064, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1018, 983, 1014, 1024, 1032, 1021, 984, 1036, 1064, 1064, 1026, 1049, 1018, 1044, 1017, 1022, 1023, 1016, 1013, 1047, 1003, 972, 1058, 1029, 1039, 1017, 975, 1027, 1046, 1047, 1023, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1038, 995, 1051, 1014, 1013, 1031, 1017, 998, 1041, 1036, 1032, 1021, 1055, 1026, 993, 1024, 1026, 1029, 1020, 1032, 1048, 1004, 1038, 1024, 989, 1006, 1065, 1012, 1041, 1029, 1013, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1021, 1042, 1022, 1043, 1042, 1050, 1018, 1034, 1045, 1050, 1028, 995, 1045, 1029, 1041, 1026, 1071, 1028, 1014, 1049, 992, 1042, 1010, 969, 969, 995, 1050, 999, 1039, 970, 1047, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[999, 1017, 1058, 1017, 1050, 1038, 1034, 1068, 994, 959, 1039, 1041, 1055, 998, 1024, 1025, 1011, 991, 1011, 1052, 1007, 1024, 1045, 982, 1024, 1029, 1054, 1026, 1033, 1003, 1027, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1036, 988, 976, 1050, 1007, 1076, 1015, 1034, 1041, 1018, 1016, 998, 990, 1031, 1038, 1014, 1062, 987, 1011, 1012, 1009, 998, 996, 1036, 1023, 1071, 1076, 1037, 957, 1086, 1058, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1008, 996, 1002, 989, 1078, 973, 1044, 1024, 1046, 1032, 1014, 1000, 1053, 1025, 1016, 1044, 1053, 1076, 1031, 978, 1038, 1026, 952, 1030, 1063, 1018, 1027, 1013, 1017, 1054, 1036, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[993, 999, 995, 1034, 1003, 1043, 975, 1051, 1009, 1017, 1036, 1062, 1043, 1023, 1023, 1015, 1020, 1033, 1007, 1075, 1039, 1007, 1010, 1008, 1046, 1033, 1053, 991, 1031, 999, 1055, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[971, 1010, 1005, 1034, 1005, 1104, 1056, 1041, 1016, 1019, 1043, 985, 1054, 985, 1009, 1035, 1043, 1007, 1004, 1032, 1038, 1022, 1025, 1087, 1060, 1032, 1003, 1003, 980, 1068, 969, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1018, 983, 1014, 1024, 27, 11, 0, 0, 1032, 994, 984, 1036, 0, 0, 0, 0, 1056, 1064, 1026, 1049, 0, 0, 0, 0, 1010, 1044, 1017, 1022, 0, 0, 0, 0, 1012, 1016, 1013, 1047, 0, 0, 0, 0, 1003, 972, 1058, 1029, 8, 8, 6, 0, 1039, 1017, 969, 1027, 0, 0, 0, 0, 1046, 1047, 1023, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4046, 4195, 4093, 4088, 4084, 4052, 4133] avg=4096.0000 max=4195 max_rank=2 min=4046 min_rank=1 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1038, 995, 1051, 1014, 28, 11, 0, 0, 1013, 1003, 1017, 998, 0, 0, 0, 0, 1033, 1036, 1032, 1021, 0, 0, 0, 0, 1047, 1026, 993, 1024, 0, 0, 0, 0, 1015, 1029, 1020, 1032, 0, 0, 0, 0, 1048, 1004, 1038, 1024, 8, 8, 7, 0, 989, 1006, 1058, 1012, 0, 0, 0, 0, 1041, 1029, 1013, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4031, 4122, 4090, 4096, 4137, 4065, 4090] avg=4096.0000 max=4137 max_rank=0 min=4031 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[999, 1017, 1058, 1017, 28, 11, 0, 0, 1050, 1010, 1034, 1068, 0, 0, 0, 0, 986, 959, 1039, 1041, 0, 0, 0, 0, 1047, 998, 1024, 1025, 0, 0, 0, 0, 1000, 991, 1011, 1052, 0, 0, 0, 0, 1007, 1024, 1045, 982, 8, 8, 7, 0, 1024, 1029, 1047, 1026, 0, 0, 0, 0, 1033, 1003, 1027, 1033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4162, 4025, 4094, 4054, 4081, 4126, 4096] avg=4096.0000 max=4162 max_rank=1 min=4025 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1021, 1042, 1022, 1043, 28, 12, 0, 0, 1042, 1022, 1018, 1034, 0, 0, 0, 0, 1037, 1050, 1028, 995, 0, 0, 0, 0, 1037, 1029, 1041, 1026, 0, 0, 0, 0, 1059, 1028, 1014, 1049, 0, 0, 0, 0, 992, 1042, 1010, 969, 8, 8, 7, 0, 969, 995, 1043, 999, 0, 0, 0, 0, 1039, 970, 1047, 993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4168, 4116, 4110, 4133, 4150, 4036, 4006, 4049] avg=4096.0000 max=4168 max_rank=0 min=4006 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16241 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1008, 996, 1002, 989, 26, 12, 0, 0, 1078, 947, 1044, 1024, 0, 0, 0, 0, 1038, 1032, 1014, 1000, 0, 0, 0, 0, 1045, 1025, 1016, 1044, 0, 0, 0, 0, 1041, 1076, 1031, 978, 0, 0, 0, 0, 1038, 1026, 952, 1030, 8, 8, 7, 0, 1063, 1018, 1020, 1013, 0, 0, 0, 0, 1017, 1054, 1036, 1012, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4093, 4084, 4130, 4126, 4069, 4114, 4119] avg=4096.0000 max=4130 max_rank=3 min=4033 min_rank=0 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1036, 988, 976, 1050, 29, 12, 0, 0, 1007, 1047, 1015, 1034, 0, 0, 0, 0, 1033, 1018, 1016, 998, 0, 0, 0, 0, 982, 1031, 1038, 1014, 0, 0, 0, 0, 1050, 987, 1011, 1012, 0, 0, 0, 0, 1009, 998, 996, 1036, 8, 8, 7, 0, 1023, 1071, 1069, 1037, 0, 0, 0, 0, 957, 1086, 1058, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4103, 4065, 4065, 4060, 4062, 4200, 4122] avg=4096.0000 max=4200 max_rank=6 min=4060 min_rank=4 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[993, 999, 995, 1034, 31, 15, 3, 0, 1003, 1012, 975, 1051, 0, 0, 0, 0, 997, 1017, 1036, 1062, 0, 0, 0, 0, 1027, 1023, 1023, 1015, 0, 0, 0, 0, 1005, 1033, 1007, 1075, 0, 0, 0, 0, 1039, 1007, 1010, 1008, 13, 12, 10, 6, 1046, 1033, 1043, 991, 0, 0, 0, 0, 1031, 993, 1055, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4041, 4112, 4088, 4120, 4105, 4113, 4119] avg=4096.0000 max=4120 max_rank=4 min=4041 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[971, 1010, 1005, 1034, 30, 12, 0, 0, 1005, 1074, 1056, 1041, 0, 0, 0, 0, 1008, 1019, 1043, 985, 0, 0, 0, 0, 1046, 985, 1009, 1035, 0, 0, 0, 0, 1031, 1007, 1004, 1032, 0, 0, 0, 0, 1038, 1022, 1025, 1087, 8, 8, 6, 0, 1060, 1032, 997, 1003, 0, 0, 0, 0, 980, 1068, 969, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4176, 4055, 4075, 4074, 4194, 4092, 4040] avg=4096.0000 max=4194 max_rank=5 min=4040 min_rank=7 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8320, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1040, 1032, 1021, 1032, 1028, 1055, 967, 1016, 1046, 1040, 1009, 1021, 1015, 1080, 1018, 991, 974, 1010, 1029, 1069, 1001, 1008, 1045, 997, 1030, 1049, 1026, 1014, 1040, 1058, 992, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1021, 999, 1037, 982, 1009, 991, 999, 1087, 1023, 1053, 1017, 1029, 977, 1035, 1032, 1067, 996, 1083, 1035, 1010, 1045, 1091, 1060, 993, 1001, 1036, 1012, 1004, 998, 1068, 947, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1045, 1019, 1055, 1044, 1025, 992, 1025, 1001, 1045, 1017, 1016, 993, 1027, 957, 1018, 999, 1004, 1043, 1065, 1009, 1048, 1067, 1021, 1006, 1007, 1038, 1014, 1055, 1054, 996, 1066, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[989, 1047, 1025, 985, 1043, 1020, 1003, 1002, 1053, 1041, 1016, 1034, 962, 1014, 1057, 1032, 1016, 1020, 1020, 1026, 1013, 1042, 1035, 991, 1004, 1045, 1023, 1061, 1081, 1003, 1015, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1004, 1037, 1035, 1005, 1021, 1047, 1045, 1043, 1017, 1008, 1053, 1021, 996, 1007, 1000, 1012, 1057, 1015, 1039, 1016, 1026, 1054, 1029, 1023, 968, 1004, 1056, 1057, 1044, 1026, 991, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[993, 1024, 1031, 1044, 1049, 1012, 1034, 1061, 1023, 1029, 1003, 1036, 1077, 1031, 958, 1023, 1017, 983, 991, 1019, 1076, 1010, 1045, 1021, 1020, 1027, 993, 1065, 1042, 1013, 1047, 971] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1018, 1026, 1033, 1012, 996, 965, 1063, 1043, 1012, 1025, 1012, 1031, 1019, 1083, 1034, 999, 979, 1076, 1036, 995, 1034, 995, 1004, 1015, 1046, 1108, 1030, 1005, 990, 1051, 998, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[990, 1069, 1046, 1044, 1008, 1018, 1021, 1033, 1012, 1018, 1031, 982, 1047, 1064, 1019, 1059, 997, 987, 1035, 1028, 1049, 998, 1030, 1074, 1020, 997, 1011, 994, 1022, 990, 1043, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1040, 1032, 1020, 1032, 0, 0, 0, 0, 1028, 1055, 967, 1016, 4, 1, 0, 0, 1046, 1040, 1009, 1021, 0, 0, 0, 0, 1015, 1080, 1018, 991, 7, 0, 0, 0, 974, 1010, 1029, 1069, 10, 0, 0, 0, 981, 1008, 1045, 997, 0, 0, 0, 0, 1030, 1043, 1026, 1014, 0, 0, 0, 0, 1040, 1058, 992, 1015, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4071, 4116, 4111, 4092, 4031, 4113, 4110] avg=4096.0000 max=4124 max_rank=0 min=4031 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[989, 1047, 1024, 985, 0, 0, 0, 0, 1043, 1020, 1003, 1002, 4, 1, 0, 0, 1053, 1041, 1016, 1034, 0, 0, 0, 0, 962, 1014, 1057, 1032, 7, 0, 0, 0, 1016, 1020, 1020, 1026, 10, 0, 0, 0, 993, 1042, 1035, 991, 0, 0, 0, 0, 1004, 1039, 1023, 1061, 0, 0, 0, 0, 1081, 1003, 1015, 1050, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4073, 4144, 4072, 4092, 4061, 4127, 4154] avg=4096.0000 max=4154 max_rank=7 min=4045 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1045, 1019, 1054, 1044, 0, 0, 0, 0, 1025, 992, 1025, 1001, 4, 1, 0, 0, 1045, 1017, 1016, 993, 0, 0, 0, 0, 1027, 957, 1018, 999, 7, 0, 0, 0, 1004, 1043, 1065, 1009, 11, 0, 0, 0, 1027, 1067, 1021, 1006, 0, 0, 0, 0, 1007, 1032, 1014, 1055, 0, 0, 0, 0, 1054, 996, 1066, 997, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4048, 4071, 4008, 4132, 4121, 4108, 4118] avg=4096.0000 max=4162 max_rank=0 min=4008 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1021, 999, 1036, 982, 0, 0, 0, 0, 1009, 991, 999, 1087, 3, 1, 0, 0, 1023, 1053, 1017, 1029, 0, 0, 0, 0, 977, 1035, 1032, 1067, 7, 0, 0, 0, 996, 1083, 1035, 1010, 11, 0, 0, 0, 1024, 1091, 1060, 993, 0, 0, 0, 0, 1001, 1031, 1012, 1004, 0, 0, 0, 0, 998, 1068, 947, 1031, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4090, 4122, 4118, 4135, 4168, 4048, 4049] avg=4096.0000 max=4168 max_rank=5 min=4038 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[993, 1024, 1030, 1044, 0, 0, 0, 0, 1049, 1012, 1034, 1061, 3, 1, 0, 0, 1023, 1029, 1003, 1036, 0, 0, 0, 0, 1077, 1031, 958, 1023, 7, 0, 0, 0, 1017, 983, 991, 1019, 11, 0, 0, 0, 1055, 1010, 1045, 1021, 0, 0, 0, 0, 1020, 1022, 993, 1065, 0, 0, 0, 0, 1042, 1013, 1047, 971, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4160, 4091, 4096, 4021, 4131, 4100, 4078] avg=4096.0000 max=4160 max_rank=1 min=4021 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1004, 1037, 1034, 1005, 0, 0, 0, 0, 1021, 1047, 1045, 1043, 3, 1, 0, 0, 1017, 1008, 1053, 1021, 0, 0, 0, 0, 996, 1007, 1000, 1012, 7, 0, 0, 0, 1057, 1015, 1039, 1016, 11, 0, 0, 0, 1005, 1054, 1029, 1023, 0, 0, 0, 0, 968, 999, 1056, 1057, 0, 0, 0, 0, 1044, 1026, 991, 1012, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4160, 4099, 4022, 4138, 4111, 4080, 4078] avg=4096.0000 max=4160 max_rank=1 min=4022 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[990, 1069, 1045, 1044, 0, 0, 0, 0, 1008, 1018, 1021, 1033, 3, 1, 0, 0, 1012, 1018, 1031, 982, 0, 0, 0, 0, 1047, 1064, 1019, 1059, 7, 0, 0, 0, 997, 987, 1035, 1028, 11, 0, 0, 0, 1028, 998, 1030, 1074, 0, 0, 0, 0, 1020, 992, 1011, 994, 0, 0, 0, 0, 1022, 990, 1043, 1032, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4084, 4043, 4196, 4058, 4130, 4017, 4092] avg=4096.0000 max=4196 max_rank=3 min=4017 min_rank=6 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16297 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1018, 1026, 1024, 1012, 0, 0, 0, 0, 996, 965, 1063, 1043, 8, 7, 0, 0, 1012, 1025, 1012, 1031, 2, 0, 0, 0, 1019, 1083, 1034, 999, 10, 0, 0, 0, 979, 1076, 1036, 995, 14, 0, 0, 0, 1001, 995, 1004, 1015, 0, 0, 0, 0, 1046, 1094, 1030, 1005, 0, 0, 0, 0, 990, 1051, 998, 1035, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4082, 4082, 4145, 4100, 4015, 4175, 4089] avg=4096.0000 max=4175 max_rank=6 min=4015 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[990, 1043, 1001, 1020, 1049, 1078, 1043, 1037, 1042, 990, 1020, 1036, 1043, 1004, 1007, 1019, 1022, 1004, 1013, 1035, 1040, 1004, 996, 1007, 1031, 1019, 1018, 1034, 1047, 1035, 1016, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1010, 987, 1035, 990, 1008, 1037, 1056, 1021, 996, 1034, 1027, 1023, 969, 1027, 994, 1057, 1068, 999, 1056, 1058, 1031, 1050, 1027, 992, 1012, 1063, 988, 1048, 993, 1089, 987, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1030, 1061, 1044, 1002, 1041, 1029, 1010, 965, 1070, 1023, 990, 1062, 1001, 1004, 1037, 989, 1020, 1000, 1029, 1012, 1066, 1027, 1052, 1011, 1069, 1027, 993, 995, 1008, 1017, 1098, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1023, 1038, 1046, 1006, 1048, 1060, 1050, 1021, 1033, 1028, 1053, 1030, 995, 1072, 994, 964, 1028, 1015, 966, 1003, 1009, 1053, 1022, 1050, 1012, 1028, 1062, 994, 1010, 1024, 1020, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1006, 1068, 1045, 993, 1054, 1026, 1039, 1010, 1034, 986, 1002, 990, 1065, 1042, 1035, 987, 1036, 1013, 1036, 1011, 1037, 1014, 1051, 1022, 990, 1044, 1065, 1016, 1025, 1036, 999, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1002, 1025, 955, 1022, 999, 1084, 1058, 1093, 978, 1030, 986, 1060, 975, 1056, 1013, 1020, 962, 1052, 1050, 1062, 1009, 999, 1015, 998, 1039, 1038, 1056, 1048, 1018, 996, 1023, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1057, 985, 1010, 1033, 1024, 1034, 1025, 1019, 992, 1079, 1026, 1016, 984, 1003, 1015, 1018, 1026, 1064, 1073, 1029, 985, 1051, 983, 1012, 1043, 993, 1038, 980, 998, 1073, 1068, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1028, 1046, 964, 1030, 992, 1022, 1010, 1034, 1045, 1037, 1074, 1004, 1040, 1037, 1064, 962, 1018, 1013, 988, 1042, 1011, 1051, 1076, 1019, 1041, 1011, 1013, 1039, 1025, 1029, 1017, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[990, 1043, 1001, 1020, 9, 4, 4, 3, 1049, 1039, 1043, 1037, 0, 0, 0, 0, 1042, 990, 1020, 1033, 0, 0, 0, 0, 1043, 1004, 1007, 1019, 35, 0, 0, 0, 1022, 1004, 1013, 1031, 0, 0, 0, 0, 1040, 1004, 996, 1007, 0, 0, 0, 0, 1022, 1019, 1018, 1034, 0, 0, 0, 0, 1047, 1035, 1016, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4168, 4085, 4108, 4070, 4047, 4093, 4123] avg=4096.0000 max=4168 max_rank=1 min=4047 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1023, 1038, 1046, 1006, 9, 4, 4, 3, 1048, 1022, 1050, 1021, 0, 0, 0, 0, 1033, 1028, 1053, 1027, 0, 0, 0, 0, 995, 1072, 994, 964, 34, 0, 0, 0, 1028, 1015, 966, 999, 0, 0, 0, 0, 1009, 1053, 1022, 1050, 0, 0, 0, 0, 1003, 1028, 1062, 994, 0, 0, 0, 0, 1010, 1024, 1020, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4141, 4141, 4059, 4008, 4134, 4087, 4065] avg=4096.0000 max=4141 max_rank=1 min=4008 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1010, 987, 1035, 990, 9, 4, 3, 3, 1008, 1000, 1056, 1021, 0, 0, 0, 0, 996, 1034, 1027, 1020, 0, 0, 0, 0, 969, 1027, 994, 1057, 34, 0, 0, 0, 1068, 999, 1056, 1054, 0, 0, 0, 0, 1031, 1050, 1027, 992, 0, 0, 0, 0, 1003, 1063, 988, 1048, 0, 0, 0, 0, 993, 1089, 987, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4085, 4077, 4081, 4177, 4100, 4102, 4105] avg=4096.0000 max=4177 max_rank=4 min=4041 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1030, 1061, 1044, 1002, 10, 4, 3, 3, 1041, 993, 1010, 965, 0, 0, 0, 0, 1070, 1023, 990, 1059, 0, 0, 0, 0, 1001, 1004, 1037, 989, 33, 0, 0, 0, 1020, 1000, 1029, 1008, 0, 0, 0, 0, 1066, 1027, 1052, 1011, 0, 0, 0, 0, 1059, 1027, 993, 995, 0, 0, 0, 0, 1008, 1017, 1098, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4009, 4142, 4064, 4057, 4156, 4074, 4109] avg=4096.0000 max=4157 max_rank=0 min=4009 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1002, 1025, 955, 1022, 9, 4, 4, 3, 999, 1045, 1058, 1093, 0, 0, 0, 0, 978, 1030, 986, 1057, 0, 0, 0, 0, 975, 1056, 1013, 1020, 35, 0, 0, 0, 962, 1052, 1050, 1058, 0, 0, 0, 0, 1009, 999, 1015, 998, 0, 0, 0, 0, 1030, 1038, 1056, 1048, 0, 0, 0, 0, 1018, 996, 1023, 1047, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4195, 4051, 4099, 4122, 4021, 4172, 4084] avg=4096.0000 max=4195 max_rank=1 min=4021 min_rank=5 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1057, 985, 1010, 1033, 10, 4, 3, 3, 1024, 997, 1025, 1019, 0, 0, 0, 0, 992, 1079, 1026, 1013, 0, 0, 0, 0, 984, 1003, 1015, 1018, 34, 0, 0, 0, 1026, 1064, 1073, 1025, 0, 0, 0, 0, 985, 1051, 983, 1012, 0, 0, 0, 0, 1033, 993, 1038, 980, 0, 0, 0, 0, 998, 1073, 1068, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4065, 4110, 4054, 4188, 4031, 4044, 4171] avg=4096.0000 max=4188 max_rank=4 min=4031 min_rank=5 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1028, 1046, 964, 1030, 9, 4, 3, 3, 992, 986, 1010, 1034, 0, 0, 0, 0, 1045, 1037, 1074, 1001, 0, 0, 0, 0, 1040, 1037, 1064, 962, 33, 0, 0, 0, 1018, 1013, 988, 1038, 0, 0, 0, 0, 1011, 1051, 1076, 1019, 0, 0, 0, 0, 1032, 1011, 1013, 1039, 0, 0, 0, 0, 1025, 1029, 1017, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4022, 4157, 4136, 4057, 4157, 4095, 4057] avg=4096.0000 max=4157 max_rank=2 min=4022 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1006, 1068, 1045, 993, 14, 7, 8, 6, 1054, 980, 1039, 1010, 0, 0, 0, 0, 1034, 986, 1002, 983, 0, 0, 0, 0, 1065, 1042, 1035, 987, 38, 0, 0, 0, 1036, 1013, 1036, 1004, 0, 0, 0, 0, 1037, 1012, 1051, 1022, 0, 0, 0, 0, 976, 1044, 1065, 1016, 0, 0, 0, 0, 1025, 1036, 999, 991, 2, 1, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4083, 4005, 4167, 4089, 4122, 4101, 4054] avg=4096.0000 max=4167 max_rank=3 min=4005 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1027, 1068, 1010, 1025, 1021, 1027, 1041, 973, 1046, 1008, 1025, 940, 1021, 1040, 1046, 1043, 1040, 1024, 1035, 1034, 987, 994, 1086, 1013, 1024, 1025, 1023, 998, 1015, 1017, 1054, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[996, 1038, 1031, 1022, 1038, 967, 1007, 995, 1045, 1009, 1030, 1032, 964, 1043, 1018, 1025, 1040, 1046, 1054, 1041, 1056, 1053, 994, 1016, 1004, 1075, 1032, 1035, 1065, 1030, 974, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1069, 1019, 1022, 1072, 997, 1009, 955, 1002, 1006, 1045, 1029, 1083, 1037, 1003, 1041, 988, 1047, 973, 1075, 1033, 1066, 1052, 983, 1010, 1010, 1016, 1005, 1054, 1018, 1006, 1023, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1053, 993, 1090, 997, 1059, 1016, 1049, 1022, 1025, 1032, 1039, 984, 1019, 1023, 1054, 1023, 998, 1016, 1054, 1009, 999, 1052, 1028, 1014, 1019, 1035, 977, 1054, 1021, 1000, 1021, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1035, 1079, 1040, 945, 1040, 1009, 1065, 1019, 1026, 1012, 1029, 1008, 1050, 1034, 1029, 1005, 1022, 1001, 1052, 1045, 1062, 957, 1082, 1017, 1019, 990, 1024, 1030, 1034, 981, 1029, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[990, 1022, 987, 1061, 1027, 1040, 1007, 1048, 1072, 1050, 980, 974, 1022, 1027, 1059, 1033, 990, 1037, 1035, 1007, 1055, 1019, 1021, 1013, 1047, 1009, 981, 1024, 1031, 1047, 1014, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[998, 1006, 1093, 1002, 1019, 1037, 1032, 1010, 1053, 1000, 994, 993, 995, 1035, 978, 1046, 1010, 1028, 1044, 1052, 1004, 1072, 1010, 1019, 1053, 1017, 990, 1013, 1027, 1015, 1031, 1092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1069, 1030, 1019, 1021, 1015, 1047, 1050, 1056, 1036, 1009, 1002, 1063, 1028, 1015, 1004, 1028, 1022, 1072, 988, 1048, 1018, 1026, 968, 1022, 1000, 1035, 1056, 941, 1039, 997, 1012, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[996, 1038, 1013, 1022, 0, 0, 0, 0, 1038, 967, 1007, 995, 8, 0, 0, 0, 1045, 1009, 1030, 1032, 6, 4, 0, 0, 964, 1043, 1018, 1025, 0, 0, 0, 0, 1040, 1046, 1029, 1041, 0, 0, 0, 0, 1056, 1053, 994, 1016, 0, 0, 0, 0, 1004, 1075, 1032, 1035, 19, 0, 0, 0, 1065, 1030, 974, 993, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4015, 4126, 4050, 4156, 4119, 4165, 4068] avg=4096.0000 max=4165 max_rank=6 min=4015 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16508 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1027, 1068, 992, 1025, 0, 0, 0, 0, 1021, 1027, 1041, 973, 8, 0, 0, 0, 1046, 1008, 1025, 940, 6, 4, 0, 0, 1021, 1040, 1045, 1043, 0, 0, 0, 0, 1040, 1024, 1011, 1034, 0, 0, 0, 0, 987, 994, 1086, 1013, 0, 0, 0, 0, 1024, 1025, 1023, 998, 18, 0, 0, 0, 1015, 1017, 1054, 1038, 6, 1, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4070, 4029, 4149, 4109, 4080, 4088, 4131] avg=4096.0000 max=4149 max_rank=3 min=4029 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16408 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1069, 1019, 1004, 1072, 0, 0, 0, 0, 997, 1009, 955, 1002, 8, 0, 0, 0, 1006, 1045, 1029, 1083, 6, 4, 0, 0, 1037, 1003, 1040, 988, 0, 0, 0, 0, 1047, 973, 1050, 1033, 0, 0, 0, 0, 1066, 1052, 983, 1010, 0, 0, 0, 0, 1010, 1016, 1005, 1054, 19, 0, 0, 0, 1018, 1006, 1023, 1020, 6, 1, 0, 0] rerouted_tokens_per_ep_rank=[4164, 3971, 4173, 4068, 4103, 4111, 4104, 4074] avg=4096.0000 max=4173 max_rank=2 min=3971 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16392 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1053, 993, 1070, 997, 0, 0, 0, 0, 1059, 1016, 1049, 1022, 9, 0, 0, 0, 1025, 1032, 1039, 984, 6, 4, 0, 0, 1019, 1023, 1053, 1023, 0, 0, 0, 0, 998, 1016, 1029, 1009, 0, 0, 0, 0, 999, 1052, 1028, 1014, 0, 0, 0, 0, 1019, 1035, 977, 1054, 19, 0, 0, 0, 1021, 1000, 1021, 993, 7, 1, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4155, 4090, 4118, 4052, 4093, 4104, 4043] avg=4096.0000 max=4155 max_rank=1 min=4043 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16292 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[990, 1022, 969, 1061, 0, 0, 0, 0, 1027, 1040, 1007, 1048, 8, 0, 0, 0, 1072, 1050, 980, 974, 6, 4, 0, 0, 1022, 1027, 1058, 1033, 0, 0, 0, 0, 990, 1037, 1011, 1007, 0, 0, 0, 0, 1055, 1019, 1021, 1013, 0, 0, 0, 0, 1047, 1009, 981, 1024, 18, 0, 0, 0, 1031, 1047, 1014, 1039, 6, 1, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4130, 4086, 4140, 4045, 4108, 4079, 4138] avg=4096.0000 max=4140 max_rank=3 min=4042 min_rank=0 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16370 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[998, 1006, 1072, 1002, 0, 0, 0, 0, 1019, 1037, 1032, 1010, 9, 0, 0, 0, 1053, 1000, 994, 993, 6, 5, 0, 0, 995, 1035, 978, 1046, 0, 0, 0, 0, 1010, 1028, 1019, 1052, 0, 0, 0, 0, 1004, 1072, 1010, 1019, 0, 0, 0, 0, 1053, 1017, 990, 1013, 19, 0, 0, 0, 1027, 1015, 1031, 1092, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4107, 4051, 4054, 4109, 4105, 4092, 4172] avg=4096.0000 max=4172 max_rank=7 min=4051 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16478 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1035, 1079, 1010, 945, 0, 0, 0, 0, 1040, 1009, 1065, 1019, 11, 0, 0, 0, 1026, 1012, 1029, 1008, 9, 9, 0, 0, 1050, 1034, 1025, 1005, 0, 0, 0, 0, 1022, 1001, 1020, 1045, 0, 0, 0, 0, 1062, 957, 1082, 1017, 0, 0, 0, 0, 1019, 990, 1024, 1030, 23, 0, 0, 0, 1034, 981, 1029, 998, 10, 4, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4144, 4093, 4114, 4088, 4118, 4086, 4056] avg=4096.0000 max=4144 max_rank=1 min=4056 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16348 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1069, 1030, 1001, 1021, 0, 0, 0, 0, 1015, 1047, 1050, 1056, 8, 0, 0, 0, 1036, 1009, 1002, 1063, 6, 4, 0, 0, 1028, 1015, 1004, 1028, 0, 0, 0, 0, 1022, 1072, 964, 1048, 0, 0, 0, 0, 1018, 1026, 968, 1022, 0, 0, 0, 0, 1000, 1035, 1056, 941, 18, 0, 0, 0, 1039, 997, 1012, 1032, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4176, 4120, 4075, 4106, 4034, 4050, 4086] avg=4096.0000 max=4176 max_rank=1 min=4034 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16276 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[948, 1027, 1061, 1059, 1020, 998, 1048, 1059, 1013, 997, 1048, 1005, 1022, 984, 1007, 1040, 995, 1014, 1027, 1022, 1044, 1052, 992, 1018, 1025, 996, 999, 1053, 1035, 1059, 1080, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1048, 962, 1067, 1048, 1034, 1016, 1017, 981, 1005, 1024, 1030, 1035, 1033, 1049, 1042, 1051, 1010, 1024, 993, 1068, 1024, 1015, 1016, 1009, 993, 1012, 1029, 1053, 1036, 1002, 1030, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1029, 986, 987, 992, 1036, 1026, 993, 1061, 1042, 1019, 1055, 953, 991, 1054, 1011, 1024, 1061, 1022, 1018, 1021, 1057, 1026, 1038, 1018, 1018, 1036, 1054, 1038, 1032, 1040, 1003, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1012, 1017, 985, 1027, 972, 1021, 1039, 998, 1048, 1039, 1040, 1023, 1008, 1002, 1020, 1033, 1033, 1007, 1027, 1015, 1027, 1051, 1043, 1026, 1026, 1010, 1011, 1058, 1065, 1024, 977, 1084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[987, 1051, 1044, 1035, 1009, 1038, 981, 1061, 1026, 1036, 1032, 1064, 1040, 1021, 1035, 1039, 1092, 1059, 1074, 1040, 987, 992, 991, 1043, 975, 1023, 1008, 971, 983, 1037, 1034, 960] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1026, 1011, 1016, 1039, 1032, 1061, 1037, 1017, 1025, 1020, 1000, 1068, 1005, 1009, 1024, 1034, 1003, 1050, 1004, 984, 1014, 1026, 1018, 1062, 1022, 1059, 1028, 1019, 1040, 1014, 1012, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[985, 1026, 1025, 1030, 987, 1054, 1012, 1057, 1023, 1031, 1018, 1085, 997, 1059, 1034, 1022, 1005, 1024, 991, 1024, 1008, 1011, 1014, 1049, 1034, 1022, 1001, 1049, 974, 1055, 1046, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1010, 1017, 1017, 942, 1055, 1026, 1054, 1056, 1013, 1053, 1019, 1022, 1034, 1025, 1006, 1039, 976, 1013, 1054, 1014, 1032, 1067, 1044, 1044, 1012, 980, 1072, 1013, 1055, 998, 980, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[948, 1027, 1061, 1059, 17, 11, 2, 0, 1020, 998, 1048, 1049, 0, 0, 0, 0, 1013, 997, 1048, 988, 0, 0, 0, 0, 1022, 984, 1007, 1038, 0, 0, 0, 0, 995, 1014, 1027, 1022, 0, 0, 0, 0, 1044, 1052, 992, 1007, 0, 0, 0, 0, 1025, 996, 999, 1053, 8, 0, 0, 0, 1035, 1059, 1080, 1021, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4115, 4046, 4051, 4058, 4095, 4081, 4197] avg=4096.0000 max=4197 max_rank=7 min=4046 min_rank=2 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16431 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1048, 962, 1067, 1048, 17, 10, 2, 0, 1034, 1016, 1017, 971, 0, 0, 0, 0, 1005, 1024, 1030, 1018, 0, 0, 0, 0, 1033, 1049, 1042, 1049, 0, 0, 0, 0, 1010, 1024, 993, 1068, 0, 0, 0, 0, 1024, 1015, 1016, 999, 0, 0, 0, 0, 993, 1012, 1029, 1053, 8, 0, 0, 0, 1036, 1002, 1030, 1012, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4038, 4077, 4173, 4095, 4054, 4095, 4082] avg=4096.0000 max=4173 max_rank=3 min=4038 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16326 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1012, 1017, 985, 1027, 17, 11, 2, 0, 972, 1021, 1039, 988, 0, 0, 0, 0, 1048, 1039, 1040, 1006, 0, 0, 0, 0, 1008, 1002, 1020, 1031, 0, 0, 0, 0, 1033, 1007, 1027, 1015, 0, 0, 0, 0, 1027, 1051, 1043, 1015, 0, 0, 0, 0, 1026, 1010, 1011, 1058, 8, 0, 0, 0, 1065, 1024, 977, 1084, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4020, 4133, 4061, 4082, 4136, 4113, 4152] avg=4096.0000 max=4152 max_rank=7 min=4020 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16483 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[987, 1051, 1044, 1035, 18, 11, 2, 0, 1009, 1038, 981, 1051, 0, 0, 0, 0, 1026, 1036, 1032, 1046, 0, 0, 0, 0, 1040, 1021, 1035, 1037, 0, 0, 0, 0, 1092, 1059, 1074, 1040, 0, 0, 0, 0, 987, 992, 991, 1032, 0, 0, 0, 0, 975, 1023, 1008, 971, 8, 0, 0, 0, 983, 1037, 1034, 960, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4079, 4140, 4133, 4265, 4002, 3985, 4016] avg=4096.0000 max=4265 max_rank=4 min=3985 min_rank=6 imbalance_ratio=1.041260 max_load_reduction=28503 max_load_reduction_pct=0.869843 moved_to_echo=16268 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1029, 986, 987, 992, 16, 11, 2, 0, 1036, 1026, 993, 1051, 0, 0, 0, 0, 1042, 1019, 1055, 937, 0, 0, 0, 0, 991, 1054, 1011, 1022, 0, 0, 0, 0, 1061, 1022, 1018, 1021, 0, 0, 0, 0, 1057, 1026, 1038, 1007, 0, 0, 0, 0, 1018, 1036, 1054, 1038, 8, 0, 0, 0, 1032, 1040, 1003, 1027, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4106, 4053, 4078, 4122, 4128, 4154, 4104] avg=4096.0000 max=4154 max_rank=6 min=4023 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16508 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 1011, 1016, 1039, 18, 11, 2, 0, 1032, 1061, 1037, 1007, 0, 0, 0, 0, 1025, 1020, 1000, 1050, 0, 0, 0, 0, 1005, 1009, 1024, 1032, 0, 0, 0, 0, 1003, 1050, 1004, 984, 0, 0, 0, 0, 1014, 1026, 1018, 1051, 0, 0, 0, 0, 1022, 1059, 1028, 1019, 8, 0, 0, 0, 1040, 1014, 1012, 989, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4137, 4095, 4070, 4041, 4109, 4136, 4057] avg=4096.0000 max=4137 max_rank=1 min=4041 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16343 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[985, 1026, 1025, 1030, 18, 11, 2, 0, 987, 1054, 1012, 1047, 0, 0, 0, 0, 1023, 1031, 1018, 1067, 0, 0, 0, 0, 997, 1059, 1034, 1020, 0, 0, 0, 0, 1005, 1024, 991, 1024, 0, 0, 0, 0, 1008, 1011, 1014, 1038, 0, 0, 0, 0, 1034, 1022, 1001, 1049, 8, 0, 0, 0, 974, 1055, 1046, 1016, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4100, 4139, 4110, 4044, 4071, 4114, 4093] avg=4096.0000 max=4139 max_rank=2 min=4044 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16322 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1010, 1017, 1017, 942, 22, 14, 5, 0, 1055, 1026, 1054, 1038, 0, 0, 0, 0, 1013, 1053, 1019, 1000, 0, 0, 0, 0, 1034, 1025, 1006, 1027, 0, 0, 0, 0, 976, 1013, 1054, 1014, 4, 0, 0, 0, 1032, 1067, 1044, 1030, 0, 0, 0, 0, 1012, 980, 1072, 1013, 13, 0, 0, 0, 1055, 998, 980, 1026, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4173, 4085, 4092, 4061, 4173, 4090, 4067] avg=4096.0000 max=4173 max_rank=1 min=4027 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16391 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1014, 954, 1014, 1029, 993, 1043, 1007, 1019, 989, 1044, 1052, 1007, 1021, 1089, 1025, 1042, 1061, 1007, 1008, 1010, 1023, 1026, 996, 1017, 1041, 1055, 1032, 1060, 1055, 1022, 998, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[992, 1014, 1004, 1064, 1007, 1024, 963, 1091, 1012, 1026, 1031, 998, 1062, 1003, 989, 1017, 1040, 1048, 1049, 1049, 998, 1007, 1093, 1030, 1020, 999, 997, 1040, 1020, 1006, 1038, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[974, 1049, 1036, 1050, 1068, 1017, 999, 1035, 1023, 1044, 1068, 1012, 1069, 1024, 997, 983, 1005, 1010, 1037, 1014, 1079, 1020, 990, 1037, 997, 1017, 1006, 993, 1041, 1054, 995, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1023, 1038, 1023, 1006, 1070, 1063, 1066, 1042, 1037, 971, 1022, 1060, 978, 1019, 1048, 1011, 990, 1022, 1068, 1023, 1017, 1027, 1005, 1029, 988, 1050, 1016, 998, 1030, 1066, 1006, 956] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1054, 1036, 1018, 1061, 1046, 980, 1021, 1109, 1034, 1070, 995, 1009, 1028, 1022, 1001, 997, 1036, 1074, 980, 1001, 1012, 1005, 1020, 1013, 1024, 1015, 999, 1052, 1027, 1014, 1001, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1007, 1030, 1011, 1030, 1005, 1043, 983, 1011, 1040, 1035, 1009, 967, 1022, 1077, 1008, 997, 1027, 1028, 1046, 984, 1030, 1061, 1004, 1087, 1006, 1076, 1017, 962, 1000, 1089, 1039, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1012, 1026, 1024, 972, 1043, 1038, 987, 1040, 1059, 1016, 996, 978, 968, 1086, 996, 1047, 1005, 1018, 1023, 1055, 1032, 1018, 1016, 1018, 1037, 999, 1036, 1095, 1035, 1030, 1046, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1073, 996, 993, 1042, 1064, 1042, 1028, 1024, 1017, 1029, 1029, 1003, 986, 993, 1036, 1048, 1059, 1025, 1020, 1011, 1036, 1069, 1006, 1054, 993, 999, 1004, 1018, 1018, 1047, 1005, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1014, 954, 1014, 1029, 9, 2, 0, 0, 993, 1043, 1007, 995, 0, 0, 0, 0, 989, 1044, 1052, 1007, 10, 0, 0, 0, 1021, 1089, 1025, 1042, 7, 1, 0, 0, 1061, 1000, 1008, 1010, 0, 0, 0, 0, 1023, 1026, 996, 1005, 0, 0, 0, 0, 1041, 1055, 1032, 1060, 15, 0, 0, 0, 1055, 1021, 998, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4038, 4102, 4185, 4079, 4050, 4203, 4089] avg=4096.0000 max=4203 max_rank=6 min=4022 min_rank=0 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[992, 1014, 1004, 1064, 9, 2, 0, 0, 1007, 1024, 963, 1066, 0, 0, 0, 0, 1012, 1026, 1031, 998, 10, 0, 0, 0, 1062, 1003, 989, 1017, 8, 1, 0, 0, 1040, 1040, 1049, 1049, 0, 0, 0, 0, 998, 1007, 1093, 1018, 0, 0, 0, 0, 1020, 999, 997, 1040, 16, 0, 0, 0, 1020, 1005, 1038, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4060, 4077, 4080, 4178, 4116, 4072, 4100] avg=4096.0000 max=4178 max_rank=4 min=4060 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[974, 1049, 1036, 1050, 9, 2, 0, 0, 1068, 1017, 999, 1011, 0, 0, 0, 0, 1023, 1044, 1068, 1012, 10, 0, 0, 0, 1069, 1024, 997, 983, 7, 2, 0, 0, 1005, 1003, 1037, 1014, 0, 0, 0, 0, 1079, 1020, 990, 1025, 0, 0, 0, 0, 997, 1017, 1006, 993, 15, 0, 0, 0, 1041, 1052, 995, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4095, 4157, 4082, 4059, 4114, 4028, 4113] avg=4096.0000 max=4157 max_rank=2 min=4028 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1023, 1038, 1023, 1006, 9, 2, 0, 0, 1070, 1063, 1066, 1018, 0, 0, 0, 0, 1037, 971, 1022, 1060, 10, 0, 0, 0, 978, 1019, 1048, 1011, 7, 2, 0, 0, 990, 1015, 1068, 1023, 0, 0, 0, 0, 1017, 1027, 1005, 1017, 0, 0, 0, 0, 988, 1050, 1016, 998, 15, 0, 0, 0, 1030, 1064, 1006, 956, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4217, 4100, 4065, 4096, 4066, 4067, 4056] avg=4096.0000 max=4217 max_rank=1 min=4056 min_rank=7 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1054, 1036, 1018, 1061, 10, 2, 0, 0, 1046, 980, 1021, 1083, 0, 0, 0, 0, 1034, 1070, 995, 1009, 10, 0, 0, 0, 1028, 1022, 1001, 997, 8, 1, 0, 0, 1036, 1066, 980, 1001, 0, 0, 0, 0, 1012, 1005, 1020, 1001, 0, 0, 0, 0, 1024, 1015, 999, 1052, 16, 0, 0, 0, 1027, 1013, 1001, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4130, 4118, 4057, 4083, 4038, 4106, 4055] avg=4096.0000 max=4181 max_rank=0 min=4038 min_rank=5 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1007, 1030, 1011, 1030, 9, 3, 0, 0, 1005, 1043, 983, 987, 0, 0, 0, 0, 1040, 1035, 1009, 967, 11, 0, 0, 0, 1022, 1077, 1008, 997, 7, 2, 0, 0, 1027, 1021, 1046, 984, 0, 0, 0, 0, 1030, 1061, 1004, 1073, 0, 0, 0, 0, 1006, 1076, 1017, 962, 15, 0, 0, 0, 1000, 1087, 1039, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4018, 4062, 4113, 4078, 4168, 4076, 4163] avg=4096.0000 max=4168 max_rank=5 min=4018 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1012, 1026, 1024, 972, 12, 8, 0, 0, 1043, 1038, 987, 1008, 0, 0, 0, 0, 1059, 1016, 996, 978, 13, 2, 0, 0, 968, 1086, 996, 1047, 12, 5, 0, 0, 1005, 1004, 1023, 1055, 0, 0, 0, 0, 1032, 1018, 1016, 997, 0, 0, 0, 0, 1037, 999, 1036, 1095, 20, 0, 0, 0, 1035, 1025, 1046, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4076, 4064, 4114, 4087, 4063, 4187, 4123] avg=4096.0000 max=4187 max_rank=6 min=4054 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1073, 996, 993, 1042, 9, 2, 0, 0, 1064, 1042, 1028, 1000, 0, 0, 0, 0, 1017, 1029, 1029, 1003, 10, 0, 0, 0, 986, 993, 1036, 1048, 7, 2, 0, 0, 1059, 1018, 1020, 1011, 0, 0, 0, 0, 1036, 1069, 1006, 1042, 0, 0, 0, 0, 993, 999, 1004, 1018, 15, 0, 0, 0, 1018, 1045, 1005, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4134, 4088, 4072, 4108, 4153, 4029, 4069] avg=4096.0000 max=4153 max_rank=5 min=4029 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8320, 8064, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1034, 1057, 954, 990, 1003, 1044, 987, 985, 999, 1048, 1015, 1053, 1047, 981, 1012, 1058, 1002, 1063, 1073, 1045, 1016, 1030, 1053, 1028, 1063, 1041, 1071, 991, 1016, 1042, 994, 973] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1042, 992, 1064, 960, 973, 1005, 1056, 1042, 1053, 1033, 1009, 991, 1046, 1020, 1025, 981, 1058, 1028, 997, 1049, 1027, 997, 1001, 1041, 983, 1017, 1090, 1046, 1046, 1006, 1033, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1009, 1055, 1033, 1030, 1003, 1026, 1036, 980, 1020, 1000, 1021, 1014, 998, 1008, 1018, 1019, 1031, 988, 1020, 1033, 1030, 1027, 1058, 1077, 1061, 1037, 1017, 986, 1041, 1061, 1012, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[980, 999, 1029, 1010, 1029, 1048, 954, 1011, 1041, 1059, 1044, 1030, 1029, 996, 1035, 1051, 1036, 1027, 1045, 1035, 1075, 1074, 1014, 1036, 1035, 971, 1026, 1000, 1015, 985, 999, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1002, 1071, 1046, 1072, 971, 1072, 1034, 1019, 1015, 1033, 1082, 1106, 1022, 978, 1001, 995, 998, 1035, 971, 1004, 1047, 991, 1009, 1049, 992, 994, 1071, 1018, 996, 1031, 1012, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1094, 1040, 958, 993, 1011, 1025, 976, 1026, 999, 1024, 1032, 1065, 1027, 1031, 1029, 1003, 994, 1039, 1007, 1044, 1014, 1056, 1068, 1034, 987, 998, 1022, 1053, 1024, 1032, 1023, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1054, 1041, 993, 989, 1071, 1004, 993, 1019, 1033, 1009, 997, 1025, 1070, 1026, 1033, 1011, 1024, 984, 1071, 1032, 1040, 1031, 1057, 1010, 1027, 992, 966, 1035, 1015, 1018, 1035, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1027, 1011, 1019, 1048, 1051, 1056, 1030, 1039, 1055, 1083, 1010, 1036, 993, 964, 1004, 1071, 1011, 982, 1032, 1007, 1001, 1021, 996, 979, 1034, 1035, 1027, 1012, 1055, 986, 1033, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1034, 1057, 954, 990, 9, 0, 0, 0, 1003, 1044, 987, 985, 23, 0, 0, 0, 999, 1048, 1015, 1021, 0, 0, 0, 0, 1047, 981, 1012, 1058, 13, 9, 0, 0, 1002, 1063, 1073, 1045, 0, 0, 0, 0, 1016, 1030, 1027, 1028, 0, 0, 0, 0, 1063, 1041, 1071, 991, 4, 3, 0, 0, 1016, 1042, 994, 970, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4042, 4083, 4120, 4183, 4101, 4173, 4022] avg=4096.0000 max=4183 max_rank=4 min=4022 min_rank=7 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[980, 999, 1029, 1010, 8, 0, 0, 0, 1029, 1048, 954, 1011, 23, 0, 0, 0, 1041, 1059, 1044, 998, 0, 0, 0, 0, 1029, 996, 1035, 1051, 13, 9, 0, 0, 1036, 1027, 1045, 1035, 0, 0, 0, 0, 1075, 1074, 989, 1036, 0, 0, 0, 0, 1035, 971, 1026, 1000, 4, 4, 0, 0, 1015, 985, 999, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4065, 4142, 4133, 4143, 4174, 4040, 4045] avg=4096.0000 max=4174 max_rank=5 min=4026 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1009, 1055, 1033, 1030, 9, 0, 0, 0, 1003, 1026, 1036, 980, 23, 0, 0, 0, 1020, 1000, 1021, 982, 0, 0, 0, 0, 998, 1008, 1018, 1019, 13, 9, 0, 0, 1031, 988, 1020, 1033, 0, 0, 0, 0, 1030, 1027, 1032, 1077, 0, 0, 0, 0, 1061, 1037, 1017, 986, 4, 3, 0, 0, 1041, 1061, 1012, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4068, 4023, 4065, 4072, 4166, 4108, 4130] avg=4096.0000 max=4166 max_rank=5 min=4023 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1042, 992, 1064, 960, 8, 0, 0, 0, 973, 1005, 1056, 1042, 22, 0, 0, 0, 1053, 1033, 1009, 960, 0, 0, 0, 0, 1046, 1020, 1025, 981, 13, 9, 0, 0, 1058, 1028, 997, 1049, 0, 0, 0, 0, 1027, 997, 976, 1041, 0, 0, 0, 0, 983, 1017, 1090, 1046, 4, 4, 0, 0, 1046, 1006, 1033, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4098, 4055, 4094, 4132, 4041, 4144, 4138] avg=4096.0000 max=4144 max_rank=6 min=4041 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1094, 1040, 958, 993, 9, 0, 0, 0, 1011, 1025, 976, 1026, 24, 0, 0, 0, 999, 1024, 1032, 1032, 0, 0, 0, 0, 1027, 1031, 1029, 1003, 14, 9, 0, 0, 994, 1039, 1007, 1044, 0, 0, 0, 0, 1014, 1056, 1041, 1034, 0, 0, 0, 0, 987, 998, 1022, 1053, 4, 4, 0, 0, 1024, 1032, 1023, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4062, 4087, 4113, 4084, 4145, 4068, 4115] avg=4096.0000 max=4145 max_rank=5 min=4062 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1002, 1071, 1046, 1072, 8, 0, 0, 0, 971, 1072, 1034, 1019, 25, 0, 0, 0, 1015, 1033, 1082, 1071, 0, 0, 0, 0, 1022, 978, 1001, 995, 13, 10, 0, 0, 998, 1035, 971, 1004, 0, 0, 0, 0, 1047, 991, 984, 1049, 0, 0, 0, 0, 992, 994, 1071, 1018, 4, 3, 0, 0, 996, 1031, 1012, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4199, 4121, 4201, 4019, 4008, 4071, 4082, 4067] avg=4096.0000 max=4201 max_rank=2 min=4008 min_rank=4 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16228 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1054, 1041, 993, 989, 13, 0, 0, 0, 1071, 1004, 993, 1019, 26, 0, 0, 0, 1033, 1009, 997, 986, 0, 0, 0, 0, 1070, 1026, 1033, 1011, 17, 13, 0, 0, 1024, 984, 1071, 1032, 3, 0, 0, 0, 1040, 1031, 1017, 1010, 0, 0, 0, 0, 1027, 992, 966, 1035, 10, 7, 0, 0, 1015, 1018, 1035, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4113, 4025, 4170, 4114, 4098, 4037, 4121] avg=4096.0000 max=4170 max_rank=3 min=4025 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1027, 1011, 1019, 1048, 8, 0, 0, 0, 1051, 1056, 1030, 1039, 23, 0, 0, 0, 1055, 1083, 1010, 1004, 0, 0, 0, 0, 993, 964, 1004, 1071, 13, 9, 0, 0, 1011, 982, 1032, 1007, 0, 0, 0, 0, 1001, 1021, 971, 979, 0, 0, 0, 0, 1034, 1035, 1027, 1012, 4, 4, 0, 0, 1055, 986, 1033, 1056, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4199, 4152, 4054, 4032, 3972, 4116, 4130] avg=4096.0000 max=4199 max_rank=1 min=3972 min_rank=5 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1021, 1042, 1037, 990, 1042, 995, 1041, 1012, 1035, 1025, 963, 1061, 1005, 1041, 1023, 1035, 1048, 1001, 1077, 1035, 1011, 997, 975, 1044, 1002, 1042, 977, 1038, 1052, 1023, 1029, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[996, 973, 1065, 1026, 1027, 1028, 1080, 1013, 1069, 1085, 1006, 1059, 1032, 1045, 1005, 1075, 1012, 988, 1004, 1052, 1011, 1020, 1004, 1015, 1059, 1013, 968, 984, 1021, 1011, 1043, 979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1023, 1002, 998, 1019, 1013, 1087, 1047, 1039, 959, 1034, 1003, 1003, 1068, 994, 1072, 1026, 1026, 1056, 1019, 979, 1021, 1010, 1023, 1023, 992, 993, 1041, 1067, 1002, 1036, 1034, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1045, 1011, 1011, 1014, 1040, 1076, 1057, 1003, 1029, 1036, 994, 1026, 1061, 1026, 1004, 1037, 1027, 967, 1054, 1005, 973, 1056, 1042, 1027, 1055, 1024, 1025, 999, 1066, 996, 975, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[972, 1053, 1047, 960, 1024, 1030, 1034, 989, 1042, 1032, 1032, 1021, 1032, 971, 964, 1051, 1070, 997, 1045, 987, 1018, 1012, 1017, 1062, 986, 1064, 1113, 1011, 1012, 1080, 1001, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1019, 1052, 1029, 1015, 1071, 1036, 1070, 1003, 1015, 1044, 1025, 1074, 1020, 1039, 1025, 1028, 1001, 1049, 984, 1014, 996, 1029, 1037, 970, 990, 1010, 994, 999, 1013, 1073, 1030, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1042, 1039, 1009, 1081, 1001, 1064, 1039, 1014, 1022, 1026, 1040, 992, 1019, 1040, 987, 945, 1042, 1055, 996, 990, 1019, 995, 1000, 1011, 1023, 1051, 1001, 1042, 1043, 1041, 1064, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[988, 1039, 1064, 1060, 1015, 989, 1026, 1024, 997, 1041, 1035, 1040, 1012, 1061, 1059, 1030, 999, 1084, 1051, 998, 1013, 979, 1009, 1075, 1050, 988, 975, 1060, 974, 998, 1009, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1021, 1042, 1037, 990, 3, 0, 0, 0, 1042, 995, 1009, 1012, 0, 0, 0, 0, 1035, 1014, 963, 1061, 0, 0, 0, 0, 998, 1041, 1023, 1035, 0, 0, 0, 0, 1048, 1001, 1077, 1035, 4, 2, 0, 0, 1011, 997, 975, 1044, 32, 1, 0, 0, 1002, 1042, 977, 1038, 10, 5, 0, 0, 1052, 1016, 1029, 1049, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4058, 4073, 4097, 4167, 4060, 4074, 4146] avg=4096.0000 max=4167 max_rank=4 min=4058 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[996, 973, 1065, 1026, 3, 0, 0, 0, 1027, 1028, 1047, 1013, 0, 0, 0, 0, 1069, 1074, 1006, 1059, 0, 0, 0, 0, 1025, 1045, 1005, 1075, 0, 0, 0, 0, 1012, 988, 1004, 1052, 4, 2, 0, 0, 1011, 1020, 1004, 1015, 33, 1, 0, 0, 1059, 1013, 968, 984, 10, 5, 0, 0, 1021, 1004, 1043, 979, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4115, 4208, 4150, 4062, 4084, 4039, 4047] avg=4096.0000 max=4208 max_rank=2 min=4039 min_rank=6 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16232 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1045, 1011, 1011, 1014, 3, 0, 0, 0, 1040, 1076, 1025, 1003, 0, 0, 0, 0, 1029, 1025, 994, 1026, 0, 0, 0, 0, 1054, 1026, 1004, 1037, 0, 0, 0, 0, 1027, 967, 1054, 1005, 4, 2, 0, 0, 973, 1056, 1042, 1027, 32, 1, 0, 0, 1055, 1024, 1025, 999, 10, 5, 0, 0, 1066, 989, 975, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4144, 4074, 4121, 4059, 4131, 4118, 4037] avg=4096.0000 max=4144 max_rank=1 min=4037 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[972, 1053, 1047, 960, 3, 0, 0, 0, 1024, 1030, 1002, 989, 0, 0, 0, 0, 1042, 1021, 1032, 1021, 0, 0, 0, 0, 1025, 971, 964, 1051, 0, 0, 0, 0, 1070, 997, 1045, 987, 4, 2, 0, 0, 1018, 1012, 1017, 1062, 32, 1, 0, 0, 986, 1064, 1113, 1011, 10, 6, 0, 0, 1012, 1072, 1001, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4045, 4116, 4011, 4105, 4142, 4190, 4124] avg=4096.0000 max=4190 max_rank=6 min=4011 min_rank=3 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1042, 1039, 1009, 1081, 3, 0, 0, 0, 1001, 1064, 1007, 1014, 0, 0, 0, 0, 1022, 1015, 1040, 992, 0, 0, 0, 0, 1012, 1040, 987, 945, 0, 0, 0, 0, 1042, 1055, 996, 990, 4, 2, 0, 0, 1019, 995, 1000, 1011, 32, 1, 0, 0, 1023, 1051, 1001, 1042, 10, 6, 0, 0, 1043, 1033, 1064, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4086, 4069, 3984, 4089, 4058, 4133, 4175] avg=4096.0000 max=4175 max_rank=7 min=3984 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1023, 1002, 998, 1019, 3, 0, 0, 0, 1013, 1087, 1015, 1039, 0, 0, 0, 0, 959, 1023, 1003, 1003, 0, 0, 0, 0, 1061, 994, 1072, 1026, 0, 0, 0, 0, 1026, 1056, 1019, 979, 4, 2, 0, 0, 1021, 1010, 1023, 1023, 32, 1, 0, 0, 992, 993, 1041, 1067, 10, 6, 0, 0, 1002, 1028, 1034, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4154, 3988, 4153, 4086, 4110, 4109, 4123] avg=4096.0000 max=4154 max_rank=1 min=3988 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1019, 1052, 1029, 1015, 5, 0, 0, 0, 1071, 1036, 1033, 1003, 0, 0, 0, 0, 1015, 1024, 1025, 1074, 0, 0, 0, 0, 1005, 1039, 1025, 1028, 0, 0, 0, 0, 1001, 1049, 984, 1014, 10, 4, 0, 0, 996, 1029, 1037, 970, 37, 6, 0, 0, 990, 1010, 994, 999, 14, 10, 0, 0, 1013, 1059, 1030, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4143, 4138, 4097, 4062, 4075, 4017, 4116] avg=4096.0000 max=4143 max_rank=1 min=4017 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[988, 1039, 1064, 1060, 3, 0, 0, 0, 1015, 989, 995, 1024, 0, 0, 0, 0, 997, 1030, 1035, 1040, 0, 0, 0, 0, 1005, 1061, 1059, 1030, 0, 0, 0, 0, 999, 1084, 1051, 998, 4, 2, 0, 0, 1013, 979, 1009, 1075, 31, 1, 0, 0, 1050, 988, 975, 1060, 10, 5, 0, 0, 974, 991, 1009, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4023, 4102, 4155, 4138, 4108, 4088, 4000] avg=4096.0000 max=4155 max_rank=3 min=4000 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8320, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1046, 1051, 1011, 1087, 1040, 1050, 1020, 955, 974, 970, 1007, 1017, 1023, 1034, 1052, 997, 1012, 985, 1056, 1097, 1030, 1013, 1015, 1008, 1013, 1046, 1014, 1027, 1064, 1006, 1007, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1030, 1010, 1054, 1020, 1057, 1046, 1013, 1015, 1024, 994, 1032, 974, 1031, 1016, 1057, 1006, 1003, 1014, 1081, 1052, 1039, 987, 1044, 969, 1041, 1001, 1027, 1046, 1044, 975, 1030, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1026, 1023, 1037, 1057, 1041, 1031, 991, 1048, 1059, 1034, 1043, 1029, 1011, 1098, 985, 1054, 1019, 1032, 1017, 970, 997, 1028, 1025, 1039, 1044, 1001, 988, 1015, 1011, 973, 1026, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1055, 1037, 1054, 1036, 1065, 1007, 980, 994, 1016, 1053, 998, 1027, 1034, 993, 997, 1004, 995, 1058, 1000, 1061, 1016, 1008, 1044, 1014, 1040, 986, 1012, 1011, 1039, 1021, 1082, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1058, 1023, 1015, 1044, 1036, 1065, 1008, 1033, 998, 1028, 1063, 1031, 1010, 1032, 1003, 1069, 1043, 1023, 1003, 1009, 1038, 1067, 964, 1008, 983, 1052, 1014, 1011, 1001, 1017, 1018, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1003, 1020, 1043, 1056, 1033, 1027, 1027, 1003, 1041, 984, 997, 999, 1057, 1004, 1037, 1007, 1023, 1036, 1025, 1024, 1041, 1070, 1021, 1028, 1035, 1043, 1025, 984, 1029, 988, 1005, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1041, 1049, 976, 998, 1039, 995, 1034, 1033, 1023, 1002, 1067, 1058, 1022, 994, 1042, 1036, 1035, 1052, 1018, 1001, 1000, 1034, 1005, 1018, 1006, 1036, 1043, 1051, 1013, 1028, 976, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1031, 1005, 1031, 1007, 1008, 1036, 1050, 1029, 979, 1065, 1018, 1030, 1066, 994, 1039, 1021, 1020, 992, 1030, 1023, 1000, 1027, 1048, 1038, 1020, 1043, 995, 1021, 1065, 1001, 1016, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1030, 1010, 1054, 989, 0, 0, 0, 0, 1052, 1046, 1013, 1015, 0, 0, 0, 0, 1024, 994, 1032, 974, 16, 0, 0, 0, 1025, 1016, 1057, 1006, 0, 0, 0, 0, 1003, 1014, 1081, 1047, 0, 0, 0, 0, 1039, 987, 1044, 969, 5, 5, 0, 0, 1041, 1001, 1027, 1046, 11, 0, 0, 0, 1044, 975, 1030, 1036, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4126, 4040, 4104, 4145, 4049, 4126, 4095] avg=4096.0000 max=4145 max_rank=4 min=4040 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1046, 1051, 1011, 1054, 0, 0, 0, 0, 1035, 1050, 1020, 955, 0, 0, 0, 0, 974, 970, 1007, 1017, 17, 0, 0, 0, 1017, 1034, 1052, 997, 0, 0, 0, 0, 1012, 985, 1056, 1092, 0, 0, 0, 0, 1030, 1013, 1015, 1008, 5, 5, 0, 0, 1013, 1046, 1014, 1027, 12, 0, 0, 0, 1064, 1006, 1007, 1041, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4060, 3985, 4100, 4145, 4076, 4112, 4128] avg=4096.0000 max=4162 max_rank=0 min=3985 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 1023, 1037, 1025, 0, 0, 0, 0, 1036, 1031, 991, 1048, 0, 0, 0, 0, 1059, 1034, 1043, 1029, 17, 0, 0, 0, 1005, 1098, 985, 1054, 0, 0, 0, 0, 1019, 1032, 1017, 966, 0, 0, 0, 0, 997, 1028, 1025, 1039, 4, 5, 0, 0, 1044, 1001, 988, 1015, 11, 0, 0, 0, 1011, 973, 1026, 1016, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4106, 4182, 4142, 4034, 4098, 4059, 4036] avg=4096.0000 max=4182 max_rank=2 min=4034 min_rank=4 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16227 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1058, 1023, 1015, 1013, 0, 0, 0, 0, 1031, 1065, 1008, 1033, 0, 0, 0, 0, 998, 1028, 1063, 1031, 16, 0, 0, 0, 1004, 1032, 1003, 1069, 0, 0, 0, 0, 1043, 1023, 1003, 1004, 0, 0, 0, 0, 1038, 1067, 964, 1008, 5, 5, 0, 0, 983, 1052, 1014, 1011, 11, 0, 0, 0, 1001, 1017, 1018, 1001, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4137, 4136, 4108, 4073, 4087, 4071, 4047] avg=4096.0000 max=4137 max_rank=1 min=4047 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1055, 1037, 1054, 1005, 0, 0, 0, 0, 1060, 1007, 980, 994, 0, 0, 0, 0, 1016, 1053, 998, 1027, 16, 0, 0, 0, 1028, 993, 997, 1004, 0, 0, 0, 0, 995, 1058, 1000, 1056, 0, 0, 0, 0, 1016, 1008, 1044, 1014, 5, 5, 0, 0, 1040, 986, 1012, 1011, 11, 0, 0, 0, 1039, 1021, 1082, 1031, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4041, 4110, 4022, 4109, 4092, 4060, 4183] avg=4096.0000 max=4183 max_rank=7 min=4022 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1003, 1020, 1043, 1024, 0, 0, 0, 0, 1028, 1027, 1027, 1003, 0, 0, 0, 0, 1041, 984, 997, 999, 17, 0, 0, 0, 1051, 1004, 1037, 1007, 0, 0, 0, 0, 1023, 1036, 1025, 1019, 0, 0, 0, 0, 1041, 1070, 1021, 1028, 5, 5, 0, 0, 1035, 1043, 1025, 984, 11, 0, 0, 0, 1029, 988, 1005, 1053, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4085, 4038, 4099, 4103, 4170, 4098, 4085] avg=4096.0000 max=4170 max_rank=5 min=4038 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1031, 1005, 1031, 976, 0, 0, 0, 0, 1004, 1036, 1050, 1029, 0, 0, 0, 0, 979, 1065, 1018, 1030, 16, 0, 0, 0, 1060, 994, 1039, 1021, 0, 0, 0, 0, 1020, 992, 1030, 1018, 0, 0, 0, 0, 1000, 1027, 1048, 1038, 5, 4, 0, 0, 1020, 1043, 995, 1021, 11, 0, 0, 0, 1065, 1001, 1016, 1020, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4119, 4108, 4114, 4060, 4122, 4090, 4112] avg=4096.0000 max=4122 max_rank=5 min=4043 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1041, 1049, 976, 953, 0, 0, 0, 0, 1032, 995, 1034, 1033, 0, 0, 0, 0, 1023, 1002, 1067, 1058, 19, 0, 0, 0, 1007, 994, 1042, 1036, 0, 0, 0, 0, 1035, 1052, 1018, 994, 0, 0, 0, 0, 1000, 1034, 1005, 1018, 7, 7, 3, 0, 1006, 1036, 1043, 1051, 16, 0, 0, 0, 1013, 1028, 976, 1043, 12, 10, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4094, 4169, 4079, 4099, 4074, 4152, 4082] avg=4096.0000 max=4169 max_rank=2 min=4019 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1034, 1042, 1009, 1042, 1038, 1062, 1032, 1009, 1026, 1036, 1052, 1007, 1043, 1049, 988, 981, 1049, 1069, 1046, 995, 1039, 982, 1009, 1011, 1040, 1012, 1021, 991, 1002, 1015, 1021, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1020, 1018, 1024, 978, 1042, 1023, 1028, 1019, 1028, 1057, 999, 1017, 1060, 1029, 1034, 1019, 1045, 1052, 971, 1028, 1015, 1045, 1019, 1038, 1028, 1023, 1036, 1014, 992, 1018, 1002, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1041, 1001, 1051, 1024, 999, 1014, 1057, 1029, 992, 1038, 1000, 985, 1035, 1054, 991, 1035, 1024, 1034, 1006, 991, 1024, 1032, 1038, 1018, 1013, 1040, 1022, 1075, 1020, 981, 1062, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1055, 1027, 1008, 1026, 1018, 1021, 1044, 1058, 1036, 1027, 1014, 1038, 1002, 997, 1039, 1026, 1070, 1025, 1024, 1046, 1007, 967, 1010, 971, 1038, 1058, 1036, 992, 1035, 1008, 1011, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1034, 1015, 1002, 1010, 1031, 1021, 990, 1029, 1023, 1057, 1011, 1053, 984, 1011, 1088, 1041, 1066, 1050, 1020, 1027, 998, 971, 974, 1028, 1061, 1002, 1026, 1027, 1023, 1037, 1029, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1051, 980, 1009, 1042, 1025, 1037, 1017, 1035, 1034, 1021, 967, 1017, 1013, 1032, 1044, 1046, 1007, 1036, 1030, 1031, 1027, 1030, 1022, 1027, 1016, 1020, 1020, 1010, 1038, 1035, 1027, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1005, 1115, 984, 1024, 1005, 1076, 1045, 1036, 1016, 1073, 973, 1006, 1007, 988, 1000, 970, 1045, 960, 1017, 1043, 998, 1026, 1014, 1040, 1058, 1022, 1052, 1041, 1022, 1074, 950, 1083] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[980, 1053, 1036, 1008, 1062, 1051, 983, 1030, 1003, 995, 1069, 1043, 1029, 1057, 1035, 1030, 1039, 985, 979, 1000, 1061, 1046, 1040, 1048, 1079, 1024, 1041, 1017, 980, 966, 1004, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1034, 1042, 1009, 1042, 2, 0, 0, 0, 1038, 1038, 1032, 1009, 0, 0, 0, 0, 1026, 1036, 1052, 1007, 5, 1, 0, 0, 1043, 1049, 988, 981, 1, 0, 0, 0, 1045, 1069, 1046, 995, 0, 0, 0, 0, 1039, 982, 1009, 1011, 24, 0, 0, 0, 1018, 1012, 1021, 991, 0, 0, 0, 0, 1002, 1015, 1021, 1016, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4117, 4127, 4062, 4155, 4065, 4042, 4071] avg=4096.0000 max=4155 max_rank=4 min=4042 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1020, 1018, 1024, 978, 2, 0, 0, 0, 1042, 1000, 1028, 1019, 0, 0, 0, 0, 1028, 1057, 999, 1017, 5, 1, 0, 0, 1060, 1029, 1034, 1019, 1, 0, 0, 0, 1041, 1052, 971, 1028, 0, 0, 0, 0, 1015, 1045, 1019, 1038, 23, 0, 0, 0, 1006, 1023, 1036, 1014, 0, 0, 0, 0, 992, 1018, 1002, 1047, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4089, 4107, 4143, 4092, 4140, 4079, 4076] avg=4096.0000 max=4143 max_rank=3 min=4042 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1051, 980, 1009, 1042, 2, 0, 0, 0, 1025, 1013, 1017, 1035, 0, 0, 0, 0, 1034, 1021, 967, 1017, 5, 1, 0, 0, 1013, 1032, 1044, 1046, 1, 0, 0, 0, 1003, 1036, 1030, 1031, 0, 0, 0, 0, 1027, 1030, 1022, 1027, 24, 0, 0, 0, 994, 1020, 1020, 1010, 0, 0, 0, 0, 1038, 1035, 1027, 1022, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4090, 4045, 4136, 4100, 4130, 4044, 4139] avg=4096.0000 max=4139 max_rank=7 min=4044 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1034, 1015, 1002, 1010, 2, 0, 0, 0, 1031, 998, 990, 1029, 0, 0, 0, 0, 1023, 1057, 1011, 1053, 5, 1, 0, 0, 984, 1011, 1088, 1041, 1, 0, 0, 0, 1062, 1050, 1020, 1027, 0, 0, 0, 0, 998, 971, 974, 1028, 23, 0, 0, 0, 1038, 1002, 1026, 1027, 0, 0, 0, 0, 1023, 1037, 1029, 1029, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4048, 4150, 4125, 4159, 3994, 4093, 4136] avg=4096.0000 max=4159 max_rank=4 min=3994 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1055, 1027, 1008, 1026, 2, 0, 0, 0, 1018, 998, 1044, 1058, 0, 0, 0, 0, 1036, 1027, 1014, 1038, 5, 1, 0, 0, 1002, 997, 1039, 1026, 1, 0, 0, 0, 1066, 1025, 1024, 1046, 0, 0, 0, 0, 1007, 967, 1010, 971, 23, 0, 0, 0, 1016, 1058, 1036, 992, 0, 0, 0, 0, 1035, 1008, 1011, 1034, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4118, 4121, 4065, 4161, 3978, 4102, 4105] avg=4096.0000 max=4161 max_rank=4 min=3978 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1041, 1001, 1051, 1024, 2, 0, 0, 0, 999, 991, 1057, 1029, 0, 0, 0, 0, 992, 1038, 1000, 985, 5, 1, 0, 0, 1035, 1054, 991, 1035, 1, 0, 0, 0, 1020, 1034, 1006, 991, 0, 0, 0, 0, 1024, 1032, 1038, 1018, 23, 0, 0, 0, 991, 1040, 1022, 1075, 0, 0, 0, 0, 1020, 981, 1062, 1042, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4076, 4021, 4116, 4051, 4135, 4128, 4122] avg=4096.0000 max=4135 max_rank=5 min=4021 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1005, 1115, 984, 1024, 2, 0, 0, 0, 1005, 1051, 1045, 1036, 0, 0, 0, 0, 1016, 1073, 973, 1006, 5, 1, 0, 0, 1007, 988, 1000, 970, 1, 0, 0, 0, 1041, 960, 1017, 1043, 0, 0, 0, 0, 998, 1026, 1014, 1040, 25, 0, 0, 0, 1035, 1022, 1052, 1041, 0, 0, 0, 0, 1022, 1074, 950, 1083, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4137, 4074, 3966, 4061, 4103, 4150, 4147] avg=4096.0000 max=4150 max_rank=6 min=3966 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[980, 1053, 1036, 1008, 6, 0, 0, 0, 1062, 1018, 983, 1030, 0, 0, 0, 0, 1003, 995, 1069, 1043, 9, 4, 0, 0, 1029, 1057, 1035, 1030, 4, 0, 0, 0, 1025, 985, 979, 1000, 0, 0, 0, 0, 1061, 1046, 1040, 1048, 28, 0, 0, 0, 1048, 1024, 1041, 1017, 0, 0, 0, 0, 980, 966, 1004, 995, 22, 5, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4093, 4123, 4155, 3989, 4223, 4130, 3972] avg=4096.0000 max=4223 max_rank=5 min=3972 min_rank=7 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1063, 966, 1019, 1028, 1003, 1038, 989, 1062, 1003, 999, 1034, 1010, 1028, 1033, 1015, 1077, 1021, 1035, 1024, 1040, 1068, 1034, 1063, 1039, 980, 998, 1020, 1019, 1061, 991, 993, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[986, 1040, 1069, 1006, 1031, 1019, 1060, 1021, 1024, 1048, 1066, 988, 1017, 1026, 1025, 1039, 1021, 996, 992, 1027, 1035, 1043, 994, 1020, 1037, 1021, 1035, 990, 1028, 1042, 1017, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1043, 990, 1077, 1058, 1055, 1052, 1025, 1035, 1013, 1032, 1039, 1037, 994, 1028, 1027, 996, 1059, 1005, 1069, 1010, 984, 979, 1012, 1014, 1037, 992, 940, 1032, 1067, 1054, 991, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1056, 1024, 1010, 989, 1035, 1027, 1070, 988, 1051, 1004, 1039, 1077, 1019, 1015, 1032, 1047, 1010, 975, 1026, 1006, 1030, 1023, 1041, 992, 1035, 1045, 1011, 998, 1036, 995, 1022, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[954, 958, 994, 1049, 1048, 983, 1050, 1050, 1046, 1004, 1007, 1017, 1034, 1003, 1070, 1012, 1015, 1038, 1048, 1010, 1068, 1006, 1005, 1026, 1035, 1084, 1034, 1011, 1050, 1027, 1043, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1019, 1067, 1036, 1073, 1013, 1006, 1029, 1016, 1021, 1007, 1036, 1028, 1023, 999, 1000, 1013, 978, 1055, 997, 982, 982, 1029, 1031, 1034, 1051, 1040, 1085, 1025, 984, 1049, 1056, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1062, 965, 1021, 976, 1000, 993, 1056, 1028, 999, 1059, 1040, 1071, 1023, 1023, 984, 1060, 1012, 1053, 1032, 987, 994, 1061, 1086, 1023, 988, 1020, 991, 1018, 1002, 1053, 1019, 1069] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1054, 1022, 1075, 996, 1037, 1003, 1022, 1013, 1022, 1062, 1033, 961, 1036, 1027, 1022, 1014, 991, 1002, 1020, 1055, 1052, 988, 1004, 999, 1059, 1097, 996, 1008, 1024, 976, 1034, 1064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[986, 1040, 1069, 1006, 2, 0, 0, 0, 1031, 1019, 1050, 1021, 0, 0, 0, 0, 1024, 1048, 1052, 988, 0, 0, 0, 0, 1017, 1026, 1025, 1039, 0, 0, 0, 0, 1021, 996, 992, 1027, 14, 8, 0, 0, 1035, 1043, 994, 1020, 1, 0, 0, 0, 1037, 1021, 1035, 990, 2, 1, 0, 0, 1024, 1042, 1017, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4121, 4112, 4107, 4058, 4093, 4086, 4088] avg=4096.0000 max=4121 max_rank=1 min=4058 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1056, 1024, 1010, 989, 2, 0, 0, 0, 1035, 1027, 1060, 988, 0, 0, 0, 0, 1051, 1004, 1026, 1077, 0, 0, 0, 0, 1019, 1015, 1032, 1047, 0, 0, 0, 0, 1010, 975, 1026, 1006, 13, 8, 0, 0, 1030, 1023, 1041, 992, 1, 0, 0, 0, 1035, 1045, 1011, 998, 2, 1, 0, 0, 1032, 995, 1022, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4110, 4158, 4113, 4038, 4087, 4092, 4089] avg=4096.0000 max=4158 max_rank=2 min=4038 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1063, 966, 1019, 1028, 2, 0, 0, 0, 1003, 1038, 979, 1062, 0, 0, 0, 0, 1003, 999, 1021, 1010, 0, 0, 0, 0, 1028, 1033, 1015, 1077, 0, 0, 0, 0, 1021, 1035, 1024, 1040, 13, 8, 0, 0, 1068, 1034, 1063, 1039, 1, 0, 0, 0, 980, 998, 1020, 1019, 2, 1, 0, 0, 1057, 991, 993, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4082, 4033, 4153, 4141, 4205, 4020, 4056] avg=4096.0000 max=4205 max_rank=5 min=4020 min_rank=6 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1043, 990, 1077, 1058, 2, 0, 0, 0, 1055, 1052, 1015, 1035, 0, 0, 0, 0, 1013, 1032, 1026, 1037, 0, 0, 0, 0, 994, 1028, 1027, 996, 0, 0, 0, 0, 1059, 1005, 1069, 1010, 13, 8, 0, 0, 984, 979, 1012, 1014, 1, 0, 0, 0, 1037, 992, 940, 1032, 2, 1, 0, 0, 1063, 1054, 991, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4157, 4108, 4045, 4164, 3990, 4004, 4130] avg=4096.0000 max=4170 max_rank=0 min=3990 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16288 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[954, 958, 994, 1049, 2, 0, 0, 0, 1048, 983, 1040, 1050, 0, 0, 0, 0, 1046, 1004, 994, 1017, 0, 0, 0, 0, 1034, 1003, 1070, 1012, 0, 0, 0, 0, 1015, 1038, 1048, 1010, 13, 8, 0, 0, 1068, 1006, 1005, 1026, 1, 0, 0, 0, 1035, 1084, 1034, 1011, 2, 1, 0, 0, 1046, 1027, 1043, 989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3957, 4121, 4061, 4119, 4132, 4106, 4167, 4105] avg=4096.0000 max=4167 max_rank=6 min=3957 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1019, 1067, 1036, 1073, 2, 0, 0, 0, 1013, 1006, 1019, 1016, 0, 0, 0, 0, 1021, 1007, 1023, 1028, 0, 0, 0, 0, 1023, 999, 1000, 1013, 0, 0, 0, 0, 978, 1055, 997, 982, 13, 8, 0, 0, 982, 1029, 1031, 1034, 1, 0, 0, 0, 1051, 1040, 1085, 1025, 2, 1, 0, 0, 980, 1049, 1056, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4197, 4054, 4079, 4035, 4033, 4077, 4204, 4089] avg=4096.0000 max=4204 max_rank=6 min=4033 min_rank=4 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1062, 965, 1021, 976, 2, 0, 0, 0, 1000, 993, 1046, 1028, 0, 0, 0, 0, 999, 1059, 1027, 1071, 0, 0, 0, 0, 1023, 1023, 984, 1060, 0, 0, 0, 0, 1012, 1053, 1032, 987, 13, 8, 0, 0, 994, 1061, 1086, 1023, 1, 0, 0, 0, 988, 1020, 991, 1018, 2, 1, 0, 0, 998, 1053, 1019, 1069, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4067, 4156, 4090, 4105, 4165, 4020, 4139] avg=4096.0000 max=4165 max_rank=5 min=4020 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1054, 1022, 1075, 996, 9, 0, 0, 0, 1037, 1003, 1003, 1013, 0, 0, 0, 0, 1022, 1062, 1016, 961, 0, 0, 0, 0, 1036, 1027, 1022, 1014, 7, 0, 0, 0, 991, 1002, 1020, 1055, 17, 12, 0, 0, 1052, 988, 1004, 999, 2, 0, 0, 0, 1059, 1097, 996, 1008, 7, 8, 0, 0, 998, 976, 1034, 1064, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4056, 4061, 4106, 4097, 4045, 4175, 4072] avg=4096.0000 max=4175 max_rank=6 min=4045 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1046, 1011, 999, 1049, 988, 1012, 1010, 988, 1010, 1045, 1025, 1000, 1031, 991, 1042, 1014, 1056, 1105, 992, 990, 1015, 1042, 1022, 964, 1011, 1057, 1036, 1024, 1057, 1044, 1054, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1057, 963, 1025, 1032, 1003, 1081, 1026, 1042, 1015, 1029, 1029, 1037, 998, 1015, 1090, 1032, 975, 1008, 1030, 969, 1044, 1059, 1055, 1048, 1032, 990, 987, 1057, 998, 1023, 980, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1018, 1063, 1034, 1004, 1038, 1000, 1017, 1069, 1001, 998, 987, 1065, 998, 1020, 1098, 1061, 1031, 1034, 1024, 1067, 977, 993, 1024, 1028, 1016, 1041, 992, 1007, 977, 1027, 1013, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1056, 1030, 995, 991, 1017, 1048, 997, 1048, 1042, 1051, 1047, 1008, 1015, 1048, 1009, 1022, 996, 1080, 999, 1017, 1038, 1008, 1037, 1061, 980, 996, 1016, 1024, 986, 1085, 996, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1019, 998, 1010, 968, 950, 1051, 1032, 997, 1034, 1040, 1004, 1036, 998, 1032, 1011, 1058, 1015, 1007, 1056, 979, 1061, 1019, 1031, 1048, 1069, 1084, 990, 1051, 1046, 1046, 1015, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1005, 981, 1022, 1020, 1056, 977, 1048, 1023, 1051, 1024, 967, 1052, 1047, 996, 1041, 982, 1043, 984, 1054, 955, 1050, 1054, 1025, 1037, 1028, 1059, 1038, 1024, 1036, 1048, 1025, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1043, 997, 1066, 1012, 1032, 1013, 1010, 996, 1012, 1066, 1012, 1020, 1033, 1047, 1066, 1050, 1021, 949, 1034, 1037, 972, 988, 1013, 1022, 1066, 1076, 996, 983, 1035, 1069, 1008, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1038, 1056, 1020, 1028, 1064, 1033, 976, 1086, 1018, 1025, 993, 1039, 982, 950, 1032, 1033, 1044, 1042, 996, 985, 1019, 1094, 1053, 1030, 1025, 988, 986, 997, 1065, 994, 1059, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1046, 1011, 999, 1049, 13, 0, 0, 0, 988, 1012, 1010, 988, 3, 1, 0, 0, 1010, 1044, 1025, 1000, 0, 0, 0, 0, 1031, 991, 1034, 1014, 0, 0, 0, 0, 1056, 1105, 992, 990, 20, 3, 0, 0, 1015, 1042, 1002, 964, 0, 0, 0, 0, 1011, 1057, 1036, 1024, 5, 0, 0, 0, 1057, 1028, 1054, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4002, 4079, 4070, 4166, 4023, 4133, 4177] avg=4096.0000 max=4177 max_rank=7 min=4002 min_rank=1 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1057, 963, 1025, 1032, 13, 0, 0, 0, 1003, 1081, 1026, 1042, 3, 1, 0, 0, 1015, 1028, 1029, 1037, 0, 0, 0, 0, 998, 1015, 1082, 1032, 0, 0, 0, 0, 975, 1008, 1030, 969, 20, 3, 0, 0, 1044, 1059, 1035, 1048, 0, 0, 0, 0, 1032, 990, 987, 1057, 5, 0, 0, 0, 998, 1007, 980, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4156, 4109, 4127, 4005, 4186, 4071, 4024] avg=4096.0000 max=4186 max_rank=5 min=4005 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1038, 1056, 1020, 1028, 12, 0, 0, 0, 1064, 1033, 976, 1086, 3, 1, 0, 0, 1018, 1024, 993, 1039, 0, 0, 0, 0, 982, 950, 1024, 1033, 0, 0, 0, 0, 1044, 1042, 996, 985, 20, 3, 0, 0, 1019, 1094, 1033, 1030, 0, 0, 0, 0, 1025, 988, 986, 997, 5, 0, 0, 0, 1065, 979, 1059, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4163, 4074, 3989, 4090, 4176, 4001, 4121] avg=4096.0000 max=4176 max_rank=5 min=3989 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1056, 1030, 995, 991, 13, 0, 0, 0, 1017, 1048, 997, 1048, 3, 1, 0, 0, 1042, 1050, 1047, 1008, 0, 0, 0, 0, 1015, 1048, 1001, 1022, 0, 0, 0, 0, 996, 1080, 999, 1017, 20, 4, 0, 0, 1038, 1008, 1017, 1061, 0, 0, 0, 0, 980, 996, 1016, 1024, 5, 0, 0, 0, 986, 1068, 996, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4114, 4147, 4086, 4116, 4124, 4021, 4075] avg=4096.0000 max=4147 max_rank=2 min=4021 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1019, 998, 1010, 968, 13, 0, 0, 0, 950, 1051, 1032, 997, 3, 1, 0, 0, 1034, 1039, 1004, 1036, 0, 0, 0, 0, 998, 1032, 1003, 1058, 0, 0, 0, 0, 1015, 1007, 1056, 979, 20, 3, 0, 0, 1061, 1019, 1011, 1048, 0, 0, 0, 0, 1069, 1084, 990, 1051, 5, 0, 0, 0, 1046, 1030, 1015, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4008, 4034, 4113, 4091, 4080, 4139, 4199, 4104] avg=4096.0000 max=4199 max_rank=6 min=4008 min_rank=0 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16522 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1018, 1063, 1034, 1004, 16, 6, 0, 0, 1038, 1000, 1017, 1069, 5, 7, 0, 0, 1001, 991, 987, 1065, 0, 0, 0, 0, 998, 1020, 1080, 1061, 0, 0, 0, 0, 1031, 1034, 1024, 1067, 24, 9, 0, 0, 977, 993, 1000, 1028, 0, 0, 0, 0, 1016, 1041, 992, 1007, 7, 0, 0, 0, 977, 1002, 1013, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4136, 4044, 4159, 4189, 3998, 4063, 4038] avg=4096.0000 max=4189 max_rank=4 min=3998 min_rank=5 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16288 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1005, 981, 1022, 1020, 13, 0, 0, 0, 1056, 977, 1048, 1023, 3, 1, 0, 0, 1051, 1023, 967, 1052, 0, 0, 0, 0, 1047, 996, 1033, 982, 0, 0, 0, 0, 1043, 984, 1054, 955, 20, 3, 0, 0, 1050, 1054, 1005, 1037, 0, 0, 0, 0, 1028, 1059, 1038, 1024, 5, 0, 0, 0, 1036, 1032, 1025, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4108, 4093, 4058, 4059, 4146, 4154, 4109] avg=4096.0000 max=4154 max_rank=6 min=4041 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1043, 997, 1066, 1012, 13, 0, 0, 0, 1032, 1013, 1010, 996, 3, 1, 0, 0, 1012, 1065, 1012, 1020, 0, 0, 0, 0, 1033, 1047, 1058, 1050, 0, 0, 0, 0, 1021, 949, 1034, 1037, 19, 3, 0, 0, 972, 988, 994, 1022, 0, 0, 0, 0, 1066, 1076, 996, 983, 5, 0, 0, 0, 1035, 1053, 1008, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4055, 4109, 4188, 4063, 3976, 4126, 4120] avg=4096.0000 max=4188 max_rank=3 min=3976 min_rank=5 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1038, 1025, 1062, 1018, 1078, 1049, 1054, 988, 1039, 1061, 1037, 1001, 1010, 976, 1001, 993, 1001, 1043, 1005, 1010, 970, 997, 1047, 1032, 1006, 1039, 1017, 1076, 1059, 1008, 1022, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1016, 1029, 1068, 1027, 1045, 1031, 1027, 1022, 1037, 991, 1065, 1007, 1074, 1057, 981, 1029, 1052, 1044, 1025, 1029, 1037, 1033, 994, 1005, 989, 969, 986, 998, 1030, 1017, 1026, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1027, 1010, 977, 1002, 1027, 1022, 1010, 1043, 1049, 1016, 1018, 976, 1028, 1002, 999, 1027, 1022, 1035, 1028, 1002, 1005, 1027, 1051, 1084, 1064, 1038, 1029, 1055, 1023, 1019, 998, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[998, 1025, 1030, 978, 1014, 1011, 985, 1014, 1038, 1072, 1011, 1030, 1022, 1006, 1049, 1011, 1017, 1021, 1034, 1000, 1015, 1016, 1053, 1009, 1045, 1014, 1052, 1023, 1055, 1046, 1054, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1065, 1026, 1014, 1045, 982, 1016, 1039, 1002, 1003, 1027, 1017, 1060, 1016, 1055, 1029, 1088, 979, 1017, 956, 1056, 980, 1034, 1021, 1026, 1050, 1053, 1038, 990, 999, 1080, 1012, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1055, 1043, 986, 1054, 1012, 988, 1015, 1042, 982, 1033, 1017, 1005, 1064, 1059, 1024, 1007, 1041, 1048, 1043, 1027, 1039, 1044, 1019, 1012, 1084, 1028, 1002, 978, 984, 990, 978, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[993, 1055, 1016, 1021, 1036, 1001, 1058, 1032, 1064, 979, 1008, 969, 990, 1071, 1022, 1051, 1021, 1057, 1036, 1016, 1032, 1010, 1010, 991, 1024, 1001, 1045, 1042, 1096, 991, 1008, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1060, 996, 994, 980, 1050, 1001, 1026, 1014, 1056, 1057, 1031, 1019, 1031, 1009, 1055, 1031, 1056, 1040, 1007, 1038, 1069, 950, 1026, 1033, 1026, 1031, 973, 990, 1000, 1042, 1045, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1016, 1029, 1068, 1027, 4, 0, 0, 0, 1045, 1031, 1027, 1022, 3, 0, 0, 0, 1037, 991, 1065, 1007, 0, 0, 0, 0, 1074, 1057, 981, 1017, 0, 0, 0, 0, 1052, 1040, 1025, 1029, 0, 0, 0, 0, 1037, 1033, 994, 1005, 12, 0, 0, 0, 989, 969, 986, 998, 0, 0, 0, 0, 1027, 1017, 1026, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4128, 4100, 4129, 4146, 4081, 3942, 4098] avg=4096.0000 max=4146 max_rank=4 min=3942 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16267 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1038, 1025, 1062, 1018, 4, 0, 0, 0, 1078, 1049, 1054, 988, 3, 0, 0, 0, 1039, 1061, 1037, 1001, 0, 0, 0, 0, 1010, 976, 1001, 982, 0, 0, 0, 0, 1001, 1039, 1005, 1010, 0, 0, 0, 0, 970, 997, 1047, 1032, 11, 0, 0, 0, 1006, 1039, 1017, 1076, 0, 0, 0, 0, 1056, 1008, 1022, 1006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4172, 4138, 3969, 4055, 4057, 4138, 4092] avg=4096.0000 max=4172 max_rank=1 min=3969 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1027, 1010, 977, 1002, 4, 0, 0, 0, 1027, 1022, 1010, 1043, 3, 0, 0, 0, 1049, 1016, 1018, 976, 0, 0, 0, 0, 1028, 1002, 999, 1015, 0, 0, 0, 0, 1022, 1031, 1028, 1002, 0, 0, 0, 0, 1005, 1027, 1051, 1084, 12, 0, 0, 0, 1064, 1038, 1029, 1055, 0, 0, 0, 0, 1020, 1019, 998, 1055, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4105, 4059, 4044, 4083, 4179, 4186, 4092] avg=4096.0000 max=4186 max_rank=6 min=4020 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[998, 1025, 1030, 978, 4, 0, 0, 0, 1014, 1011, 985, 1014, 3, 0, 0, 0, 1038, 1072, 1011, 1030, 0, 0, 0, 0, 1022, 1006, 1049, 1000, 0, 0, 0, 0, 1017, 1017, 1034, 1000, 0, 0, 0, 0, 1015, 1016, 1053, 1009, 11, 0, 0, 0, 1045, 1014, 1052, 1023, 0, 0, 0, 0, 1052, 1046, 1054, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4027, 4151, 4077, 4068, 4104, 4134, 4172] avg=4096.0000 max=4172 max_rank=7 min=4027 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1065, 1026, 1014, 1045, 4, 0, 0, 0, 982, 1016, 1039, 1002, 3, 0, 0, 0, 1003, 1027, 1017, 1060, 0, 0, 0, 0, 1016, 1055, 1029, 1076, 0, 0, 0, 0, 979, 1013, 956, 1056, 0, 0, 0, 0, 980, 1034, 1021, 1026, 12, 0, 0, 0, 1050, 1053, 1038, 990, 0, 0, 0, 0, 996, 1080, 1012, 993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4042, 4107, 4176, 4004, 4073, 4131, 4081] avg=4096.0000 max=4176 max_rank=3 min=4004 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1055, 1043, 986, 1054, 5, 2, 0, 0, 1012, 988, 1015, 1042, 8, 5, 0, 0, 975, 1033, 1017, 1005, 0, 0, 0, 0, 1064, 1059, 1024, 990, 0, 0, 0, 0, 1041, 1038, 1043, 1027, 0, 0, 0, 0, 1039, 1044, 1019, 1012, 15, 0, 0, 0, 1084, 1028, 1002, 978, 7, 6, 0, 0, 970, 990, 978, 1065, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4070, 4030, 4137, 4149, 4129, 4105, 4003] avg=4096.0000 max=4149 max_rank=4 min=4003 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1060, 996, 994, 980, 4, 0, 0, 0, 1050, 1001, 1026, 1014, 3, 0, 0, 0, 1056, 1057, 1031, 1019, 0, 0, 0, 0, 1031, 1009, 1055, 1019, 0, 0, 0, 0, 1056, 1036, 1007, 1038, 0, 0, 0, 0, 1069, 950, 1026, 1033, 12, 0, 0, 0, 1026, 1031, 973, 990, 0, 0, 0, 0, 997, 1042, 1045, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4094, 4163, 4114, 4137, 4090, 4020, 4116] avg=4096.0000 max=4163 max_rank=2 min=4020 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[993, 1055, 1016, 1021, 4, 0, 0, 0, 1036, 1001, 1058, 1032, 3, 0, 0, 0, 1064, 979, 1008, 969, 0, 0, 0, 0, 990, 1071, 1022, 1039, 0, 0, 0, 0, 1021, 1053, 1036, 1016, 0, 0, 0, 0, 1032, 1010, 1010, 991, 12, 0, 0, 0, 1024, 1001, 1045, 1042, 0, 0, 0, 0, 1093, 991, 1008, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4130, 4020, 4122, 4126, 4055, 4112, 4114] avg=4096.0000 max=4130 max_rank=1 min=4020 min_rank=2 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True + [2026-08-30 04:58:06] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 641.7 | throughput per GPU (TFLOP/s/GPU): 519.5 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.048906E+01 | loss scale: 1.0 | grad norm: 44.659 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1055, 1010, 955, 1035, 1024, 1021, 1068, 1076, 1048, 1024, 1028, 1014, 1023, 1024, 1045, 995, 1021, 1003, 1012, 1029, 1021, 985, 1025, 1083, 1035, 1030, 1035, 1046, 1004, 984, 1017, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1080, 978, 1044, 1057, 1006, 1057, 1035, 1041, 1054, 1029, 986, 1011, 1039, 1041, 1023, 1034, 1017, 1063, 1037, 1026, 982, 1027, 982, 1015, 987, 1060, 987, 984, 1083, 970, 1036, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1040, 1053, 1027, 1023, 998, 1095, 1015, 1036, 987, 1025, 1054, 1024, 1062, 964, 996, 1040, 1004, 1065, 1026, 1025, 1010, 1013, 1055, 1028, 1022, 993, 990, 993, 1023, 1042, 1028, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1000, 974, 1026, 992, 1030, 1031, 1012, 1078, 1007, 1042, 1025, 1021, 1027, 998, 1056, 1042, 984, 1076, 1069, 993, 1031, 1014, 1048, 993, 1087, 1047, 1005, 984, 1028, 1037, 1019, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1032, 1009, 989, 1028, 1047, 1041, 1026, 1074, 1030, 1028, 993, 1028, 996, 1037, 1034, 1029, 1013, 1074, 964, 1010, 1044, 1089, 1035, 1035, 1041, 1024, 1008, 979, 996, 1028, 988, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1025, 999, 995, 1041, 1086, 1036, 1018, 1013, 1069, 1068, 1011, 994, 977, 997, 1012, 1036, 973, 1051, 1024, 1028, 1012, 1064, 1037, 1020, 992, 1033, 1025, 1044, 977, 1045, 1041, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1018, 1053, 965, 987, 1023, 1021, 1020, 1037, 1004, 1060, 1041, 1026, 991, 992, 1003, 1043, 1002, 1086, 1039, 1063, 1085, 1021, 1057, 1016, 1041, 954, 1043, 1054, 1028, 1026, 1010, 959] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1036, 1022, 1003, 1044, 1036, 1024, 1013, 1001, 996, 1027, 1019, 997, 1014, 1035, 1048, 1023, 1051, 1079, 1047, 1049, 963, 1023, 1027, 1071, 1048, 950, 1022, 1023, 1078, 974, 1039, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1055, 1010, 955, 1035, 14, 6, 0, 0, 1024, 1021, 1068, 1029, 0, 0, 0, 0, 1048, 1024, 1028, 1014, 0, 0, 0, 0, 1023, 1024, 1045, 995, 11, 0, 0, 0, 1021, 976, 1012, 1029, 0, 0, 0, 0, 1021, 985, 1008, 1083, 0, 0, 0, 0, 1035, 1030, 1035, 1046, 13, 11, 0, 0, 1004, 984, 1017, 993, 36, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4142, 4114, 4098, 4038, 4097, 4170, 4034] avg=4096.0000 max=4170 max_rank=6 min=4034 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1040, 1053, 1027, 1023, 15, 6, 0, 0, 998, 1095, 1015, 991, 0, 0, 0, 0, 987, 1025, 1054, 1024, 0, 0, 0, 0, 1062, 964, 996, 1040, 11, 0, 0, 0, 1004, 1036, 1026, 1025, 0, 0, 0, 0, 1010, 1013, 1038, 1028, 0, 0, 0, 0, 1022, 993, 990, 993, 14, 10, 0, 0, 1023, 1042, 1028, 1012, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4099, 4090, 4073, 4091, 4089, 4022, 4140] avg=4096.0000 max=4164 max_rank=0 min=4022 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1000, 974, 1026, 992, 15, 6, 0, 0, 1030, 1031, 1012, 1031, 0, 0, 0, 0, 1007, 1042, 1025, 1021, 0, 0, 0, 0, 1027, 998, 1056, 1042, 11, 0, 0, 0, 984, 1047, 1069, 993, 0, 0, 0, 0, 1031, 1014, 1031, 993, 0, 0, 0, 0, 1087, 1047, 1005, 984, 14, 11, 0, 0, 1028, 1037, 1019, 992, 36, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4104, 4095, 4134, 4093, 4069, 4148, 4112] avg=4096.0000 max=4148 max_rank=6 min=4013 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1036, 1022, 1003, 1044, 15, 6, 0, 0, 1036, 1024, 1013, 957, 0, 0, 0, 0, 996, 1027, 1019, 997, 0, 0, 0, 0, 1014, 1035, 1048, 1023, 11, 0, 0, 0, 1051, 1050, 1047, 1049, 0, 0, 0, 0, 963, 1023, 1010, 1071, 0, 0, 0, 0, 1048, 950, 1022, 1023, 14, 10, 0, 0, 1078, 974, 1039, 986, 34, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4030, 4039, 4131, 4197, 4067, 4067, 4111] avg=4096.0000 max=4197 max_rank=4 min=4030 min_rank=1 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1080, 978, 1044, 1057, 15, 6, 0, 0, 1006, 1057, 1035, 996, 0, 0, 0, 0, 1054, 1029, 986, 1011, 0, 0, 0, 0, 1039, 1041, 1023, 1034, 10, 0, 0, 0, 1017, 1034, 1037, 1026, 0, 0, 0, 0, 982, 1027, 966, 1015, 0, 0, 0, 0, 987, 1060, 987, 984, 14, 10, 0, 0, 1083, 970, 1036, 997, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4094, 4080, 4147, 4114, 3990, 4042, 4121] avg=4096.0000 max=4180 max_rank=0 min=3990 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16267 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1032, 1009, 989, 1028, 15, 6, 0, 0, 1047, 1041, 1026, 1027, 0, 0, 0, 0, 1030, 1028, 993, 1028, 0, 0, 0, 0, 996, 1037, 1034, 1029, 11, 0, 0, 0, 1013, 1045, 964, 1010, 0, 0, 0, 0, 1044, 1089, 1018, 1035, 0, 0, 0, 0, 1041, 1024, 1008, 979, 14, 11, 0, 0, 996, 1028, 988, 1019, 36, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4141, 4079, 4107, 4032, 4186, 4077, 4067] avg=4096.0000 max=4186 max_rank=5 min=4032 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1018, 1053, 965, 987, 15, 6, 0, 0, 1023, 1021, 1020, 992, 0, 0, 0, 0, 1004, 1060, 1041, 1026, 0, 0, 0, 0, 991, 992, 1003, 1043, 11, 0, 0, 0, 1002, 1057, 1039, 1063, 0, 0, 0, 0, 1085, 1021, 1040, 1016, 0, 0, 0, 0, 1041, 954, 1043, 1054, 14, 10, 0, 0, 1028, 1026, 1010, 959, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4056, 4131, 4040, 4161, 4162, 4116, 4058] avg=4096.0000 max=4162 max_rank=5 min=4040 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1025, 999, 995, 1041, 16, 11, 0, 0, 1086, 1036, 1018, 962, 0, 0, 0, 0, 1069, 1066, 1011, 994, 0, 0, 0, 0, 977, 997, 1012, 1036, 14, 2, 0, 0, 973, 1017, 1024, 1028, 0, 0, 0, 0, 1012, 1064, 1012, 1020, 0, 0, 0, 0, 992, 1033, 1025, 1044, 18, 14, 0, 0, 977, 1045, 1041, 1025, 37, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4102, 4140, 4038, 4042, 4108, 4126, 4125] avg=4096.0000 max=4140 max_rank=2 min=4038 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8064, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8448, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[989, 990, 1028, 1036, 1014, 1037, 1013, 1040, 1017, 1016, 1051, 1008, 1031, 1062, 1041, 1050, 1019, 994, 1016, 1026, 1038, 1011, 1042, 1036, 1055, 1020, 962, 999, 1035, 1020, 1041, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1035, 1011, 1024, 1010, 983, 1022, 1034, 1009, 1010, 976, 1027, 1031, 1009, 1041, 1030, 1056, 1016, 1051, 1018, 1048, 1019, 1014, 1009, 1029, 1026, 1068, 1029, 1010, 1046, 1053, 1001, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1028, 1013, 997, 1017, 1009, 1003, 1022, 1077, 1021, 1046, 1023, 988, 1005, 1036, 1025, 1053, 1052, 1017, 1037, 1034, 1002, 1059, 997, 1020, 1015, 1030, 1018, 1035, 1023, 1058, 994, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1021, 1001, 1052, 1053, 1031, 1067, 1106, 1000, 984, 971, 964, 1057, 1020, 1019, 1042, 1052, 1016, 976, 1020, 1057, 1048, 999, 971, 1024, 1036, 1015, 1020, 1003, 1035, 1044, 1027, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1027, 1071, 1053, 1025, 1063, 1034, 1047, 1046, 1004, 1009, 1020, 970, 1024, 992, 991, 1061, 1030, 1029, 1059, 974, 1037, 997, 1032, 1030, 996, 1006, 1016, 1002, 1039, 1015, 1034, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[958, 1037, 1030, 1027, 1025, 1009, 1026, 1083, 1041, 1013, 1031, 1008, 994, 1024, 1002, 985, 1048, 1012, 1035, 1034, 1003, 1051, 991, 1061, 1049, 1044, 1013, 1001, 1048, 995, 1058, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[979, 1000, 1024, 1038, 1059, 1022, 1043, 990, 1027, 1012, 977, 1041, 1040, 1047, 1003, 1011, 1082, 1063, 1013, 1059, 1039, 999, 1039, 1026, 973, 1080, 1000, 1014, 1014, 1038, 1014, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1030, 1007, 1015, 965, 986, 1058, 1002, 1017, 1063, 997, 1022, 993, 1031, 1041, 981, 1019, 1032, 1059, 1055, 1108, 974, 1020, 1033, 1027, 1059, 1029, 1010, 1061, 997, 1029, 1009, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1035, 1011, 1024, 1010, 22, 0, 0, 0, 983, 1022, 1009, 1009, 0, 0, 0, 0, 1010, 976, 1027, 1031, 40, 3, 0, 0, 1009, 1041, 1030, 1050, 0, 0, 0, 0, 1016, 1051, 1018, 1008, 0, 0, 0, 0, 1019, 1014, 1009, 1029, 11, 0, 0, 0, 1026, 1068, 1029, 1010, 6, 3, 0, 0, 1046, 1039, 1001, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4023, 4087, 4130, 4093, 4082, 4142, 4109] avg=4096.0000 max=4142 max_rank=6 min=4023 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1027, 1071, 1053, 1025, 22, 0, 0, 0, 1063, 1034, 1022, 1046, 0, 0, 0, 0, 1004, 1009, 1020, 970, 37, 3, 0, 0, 1024, 992, 991, 1055, 0, 0, 0, 0, 1030, 1029, 1059, 937, 0, 0, 0, 0, 1037, 997, 1032, 1030, 10, 0, 0, 0, 996, 1006, 1016, 1002, 6, 2, 0, 0, 1039, 1003, 1034, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 4165, 4043, 4062, 4055, 4106, 4028, 4111] avg=4096.0000 max=4198 max_rank=0 min=4028 min_rank=6 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1028, 1013, 997, 1017, 25, 0, 0, 0, 1009, 1003, 986, 1077, 0, 0, 0, 0, 1021, 1046, 1023, 988, 44, 8, 0, 0, 1005, 1036, 1025, 1044, 0, 0, 0, 0, 1052, 1017, 1037, 990, 0, 0, 0, 0, 1002, 1059, 997, 1020, 15, 3, 0, 0, 1015, 1030, 1018, 1035, 9, 7, 0, 0, 1023, 1036, 994, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4075, 4130, 4110, 4096, 4096, 4114, 4067] avg=4096.0000 max=4130 max_rank=2 min=4067 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1021, 1001, 1052, 1053, 23, 0, 0, 0, 1031, 1067, 1080, 1000, 0, 0, 0, 0, 984, 971, 964, 1057, 40, 3, 0, 0, 1020, 1019, 1042, 1046, 0, 0, 0, 0, 1016, 976, 1020, 1017, 0, 0, 0, 0, 1048, 999, 971, 1024, 11, 0, 0, 0, 1036, 1015, 1020, 1003, 6, 3, 0, 0, 1035, 1030, 1027, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4178, 4019, 4127, 4029, 4053, 4083, 4129] avg=4096.0000 max=4178 max_rank=1 min=4019 min_rank=2 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[979, 1000, 1024, 1038, 22, 0, 0, 0, 1059, 1022, 1018, 990, 0, 0, 0, 0, 1027, 1012, 977, 1041, 40, 3, 0, 0, 1040, 1047, 1003, 1005, 0, 0, 0, 0, 1082, 1063, 1013, 1019, 0, 0, 0, 0, 1039, 999, 1039, 1026, 11, 0, 0, 0, 973, 1080, 1000, 1014, 6, 3, 0, 0, 1014, 1024, 1014, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4089, 4100, 4095, 4177, 4114, 4076, 4054] avg=4096.0000 max=4177 max_rank=4 min=4054 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1030, 1007, 1015, 965, 21, 0, 0, 0, 986, 1058, 978, 1017, 0, 0, 0, 0, 1063, 997, 1022, 993, 42, 3, 0, 0, 1031, 1041, 981, 1013, 0, 0, 0, 0, 1032, 1059, 1055, 1066, 0, 0, 0, 0, 974, 1020, 1033, 1027, 10, 0, 0, 0, 1059, 1029, 1010, 1061, 6, 2, 0, 0, 997, 1017, 1009, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4039, 4120, 4066, 4212, 4064, 4167, 4062] avg=4096.0000 max=4212 max_rank=4 min=4038 min_rank=0 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[958, 1037, 1030, 1027, 21, 0, 0, 0, 1025, 1009, 1002, 1083, 0, 0, 0, 0, 1041, 1013, 1031, 1008, 39, 3, 0, 0, 994, 1024, 1002, 980, 0, 0, 0, 0, 1048, 1012, 1035, 995, 0, 0, 0, 0, 1003, 1051, 991, 1061, 10, 0, 0, 0, 1049, 1044, 1013, 1001, 5, 2, 0, 0, 1048, 983, 1058, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4119, 4135, 4000, 4090, 4116, 4114, 4121] avg=4096.0000 max=4135 max_rank=2 min=4000 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[989, 990, 1028, 1036, 21, 0, 0, 0, 1014, 1037, 989, 1040, 0, 0, 0, 0, 1017, 1016, 1051, 1008, 39, 3, 0, 0, 1031, 1062, 1041, 1044, 0, 0, 0, 0, 1019, 994, 1016, 987, 0, 0, 0, 0, 1038, 1011, 1042, 1036, 10, 0, 0, 0, 1055, 1020, 962, 999, 6, 2, 0, 0, 1035, 1008, 1041, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4080, 4134, 4178, 4016, 4137, 4044, 4115] avg=4096.0000 max=4178 max_rank=3 min=4016 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[996, 947, 1015, 1075, 967, 1009, 1073, 985, 1027, 1048, 979, 1032, 990, 1034, 1026, 1014, 1011, 1030, 998, 1056, 1002, 1034, 1080, 1106, 1017, 1017, 1034, 1036, 1033, 1054, 1017, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[975, 1020, 1049, 1052, 1053, 1010, 1005, 1026, 1104, 1014, 988, 1022, 1063, 1064, 1006, 1034, 1025, 996, 1052, 1014, 996, 991, 1029, 1097, 1008, 1022, 1019, 1032, 988, 1034, 967, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[990, 1013, 1030, 1073, 1007, 1021, 1016, 1027, 1036, 952, 1079, 965, 1045, 1062, 1029, 1088, 1004, 1057, 997, 1054, 1020, 977, 1091, 1034, 1063, 985, 1033, 1026, 1001, 1002, 977, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1002, 1068, 1067, 1007, 1071, 1046, 1063, 1028, 991, 1042, 1051, 980, 1078, 1027, 1030, 1023, 978, 971, 1032, 1004, 1063, 1043, 1040, 1048, 1052, 1004, 967, 951, 982, 1036, 1043, 980] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1038, 961, 1049, 1026, 1017, 993, 1057, 1070, 1020, 965, 1013, 1043, 1032, 1006, 1056, 1012, 1015, 1043, 1038, 1007, 1045, 1011, 1100, 1052, 1006, 992, 1023, 975, 1021, 1025, 1028, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1061, 984, 1010, 1038, 1016, 1007, 1022, 994, 1010, 1047, 975, 1053, 967, 1057, 1028, 1038, 1005, 1026, 1058, 1078, 1048, 1031, 1060, 1031, 1019, 956, 1031, 1005, 984, 1023, 1044, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1027, 1018, 1038, 1017, 1010, 1041, 995, 1038, 1002, 991, 1049, 1037, 1016, 1044, 1047, 1019, 985, 979, 1054, 1008, 1022, 1031, 1055, 1005, 1016, 1024, 1018, 1010, 1056, 1050, 1046, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1047, 1059, 1071, 1051, 1026, 1034, 1031, 1057, 1024, 1014, 1056, 1065, 1034, 989, 1005, 997, 1047, 998, 1052, 1008, 1025, 1009, 1045, 1019, 1021, 1022, 972, 968, 957, 980, 1050, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[996, 947, 1015, 1062, 0, 0, 0, 0, 967, 1009, 1067, 985, 0, 0, 0, 0, 1027, 1048, 979, 1032, 8, 3, 0, 0, 990, 1011, 1026, 1014, 0, 0, 0, 0, 1011, 1030, 998, 1056, 6, 5, 0, 0, 1002, 1034, 1021, 1106, 0, 0, 0, 0, 1017, 1017, 1034, 1036, 56, 0, 0, 0, 1033, 1054, 1017, 1026, 20, 3, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4028, 4097, 4041, 4106, 4163, 4160, 4153] avg=4096.0000 max=4163 max_rank=5 min=4020 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[975, 1020, 1049, 1039, 0, 0, 0, 0, 1053, 1010, 1000, 1026, 0, 0, 0, 0, 1104, 1014, 988, 1022, 8, 3, 0, 0, 1063, 1041, 1006, 1034, 0, 0, 0, 0, 1025, 996, 1052, 1014, 5, 5, 0, 0, 996, 991, 973, 1097, 0, 0, 0, 0, 1008, 1022, 1019, 1032, 53, 0, 0, 0, 988, 1034, 967, 1013, 20, 3, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4089, 4139, 4144, 4097, 4057, 4134, 4025] avg=4096.0000 max=4144 max_rank=3 min=4025 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1038, 961, 1049, 1014, 0, 0, 0, 0, 1017, 993, 1051, 1070, 0, 0, 0, 0, 1020, 965, 1013, 1043, 7, 3, 0, 0, 1032, 984, 1056, 1012, 0, 0, 0, 0, 1015, 1043, 1038, 1007, 6, 5, 0, 0, 1045, 1011, 1040, 1052, 0, 0, 0, 0, 1006, 992, 1023, 975, 57, 0, 0, 0, 1021, 1025, 1028, 1029, 19, 3, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4131, 4051, 4084, 4114, 4148, 4053, 4125] avg=4096.0000 max=4148 max_rank=5 min=4051 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1061, 984, 1010, 1025, 0, 0, 0, 0, 1016, 1007, 1017, 994, 0, 0, 0, 0, 1010, 1047, 975, 1053, 8, 3, 0, 0, 967, 1034, 1028, 1038, 0, 0, 0, 0, 1005, 1026, 1058, 1078, 5, 5, 0, 0, 1048, 1031, 1002, 1031, 0, 0, 0, 0, 1019, 956, 1031, 1005, 55, 0, 0, 0, 984, 1023, 1044, 1062, 20, 3, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4034, 4096, 4067, 4177, 4112, 4066, 4136] avg=4096.0000 max=4177 max_rank=4 min=4034 min_rank=1 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1002, 1068, 1067, 996, 0, 0, 0, 0, 1071, 1046, 1057, 1028, 0, 0, 0, 0, 991, 1042, 1051, 980, 7, 3, 0, 0, 1078, 1004, 1030, 1023, 0, 0, 0, 0, 978, 971, 1032, 1004, 6, 4, 0, 0, 1063, 1043, 983, 1048, 0, 0, 0, 0, 1052, 1004, 967, 951, 54, 0, 0, 0, 982, 1036, 1043, 980, 20, 3, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4202, 4074, 4135, 3995, 4137, 4028, 4064] avg=4096.0000 max=4202 max_rank=1 min=3995 min_rank=4 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16224 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1027, 1018, 1038, 1005, 0, 0, 0, 0, 1010, 1041, 990, 1038, 0, 0, 0, 0, 1002, 991, 1049, 1037, 7, 3, 0, 0, 1016, 1021, 1047, 1019, 0, 0, 0, 0, 985, 979, 1054, 1008, 5, 5, 0, 0, 1022, 1031, 997, 1005, 0, 0, 0, 0, 1016, 1024, 1018, 1010, 55, 0, 0, 0, 1056, 1050, 1046, 1020, 20, 3, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4079, 4089, 4103, 4036, 4055, 4123, 4195] avg=4096.0000 max=4195 max_rank=7 min=4036 min_rank=4 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[990, 1013, 1030, 1060, 0, 0, 0, 0, 1007, 1021, 1011, 1027, 0, 0, 0, 0, 1036, 952, 1079, 965, 8, 3, 0, 0, 1045, 1039, 1029, 1088, 0, 0, 0, 0, 1004, 1057, 997, 1054, 5, 5, 0, 0, 1020, 977, 1032, 1034, 0, 0, 0, 0, 1063, 985, 1033, 1026, 56, 0, 0, 0, 1001, 1002, 977, 1014, 20, 3, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4066, 4043, 4201, 4122, 4063, 4163, 4017] avg=4096.0000 max=4201 max_rank=3 min=4017 min_rank=7 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1047, 1059, 1071, 1032, 0, 0, 0, 0, 1026, 1034, 1022, 1057, 0, 0, 0, 0, 1024, 1014, 1056, 1065, 12, 8, 0, 0, 1034, 957, 1005, 997, 0, 0, 0, 0, 1047, 998, 1052, 1008, 9, 7, 0, 0, 1025, 1009, 980, 1019, 0, 0, 0, 0, 1021, 1022, 972, 968, 58, 0, 0, 0, 957, 980, 1050, 1035, 24, 7, 0, 0] rerouted_tokens_per_ep_rank=[4209, 4139, 4179, 3993, 4121, 4033, 4041, 4053] avg=4096.0000 max=4209 max_rank=0 min=3993 min_rank=3 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16248 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8064, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8064, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8448, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1016, 1011, 1020, 1042, 1028, 1001, 1018, 995, 991, 1059, 1038, 1035, 996, 1061, 1031, 1039, 1030, 1069, 1043, 991, 967, 1001, 1007, 1082, 1006, 1073, 972, 1053, 1019, 1026, 1038, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[989, 1023, 1019, 1055, 1018, 989, 1028, 1025, 1031, 1072, 971, 987, 1037, 1061, 983, 1059, 1059, 1036, 1077, 1019, 1012, 988, 1044, 978, 1049, 1049, 1023, 1050, 981, 1048, 1008, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1001, 1009, 1042, 1037, 1023, 1042, 1055, 1016, 1005, 1022, 989, 1033, 1036, 1024, 1019, 1032, 1032, 1040, 1055, 1022, 1013, 1043, 1001, 1070, 995, 1031, 1023, 1019, 987, 1019, 1032, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1028, 1010, 1062, 1032, 974, 1046, 1014, 1027, 1013, 1006, 1027, 1064, 1042, 1043, 979, 1049, 1017, 1058, 1026, 1000, 978, 1019, 1016, 1050, 1056, 1056, 1050, 1018, 1007, 969, 1003, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1021, 982, 1024, 1001, 929, 1070, 1027, 1071, 1048, 1044, 998, 1079, 1016, 1065, 1039, 1027, 1042, 982, 1016, 1003, 1069, 1030, 1028, 960, 997, 989, 1028, 1040, 1052, 1056, 1047, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1051, 992, 1064, 1031, 1041, 1043, 1054, 1003, 1002, 1064, 1040, 992, 1050, 1040, 1004, 1042, 1002, 1041, 1013, 974, 1061, 1065, 988, 1020, 982, 1005, 1017, 1016, 1029, 1006, 1006, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1030, 1021, 1055, 1000, 1042, 1038, 1034, 984, 1024, 1034, 1021, 1031, 1005, 933, 1016, 1020, 1043, 1049, 1083, 1018, 988, 986, 1069, 1013, 1053, 994, 1016, 1015, 1082, 1017, 1055, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[999, 1032, 1005, 1023, 1043, 1027, 1022, 1022, 1048, 1016, 956, 1044, 1039, 1015, 1054, 1017, 1006, 1025, 1005, 1012, 1021, 1061, 978, 1047, 1015, 1033, 994, 1022, 1093, 1042, 1005, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1016, 1011, 1020, 1042, 4, 0, 0, 0, 1028, 1001, 1018, 995, 2, 0, 0, 0, 991, 1057, 1038, 1035, 0, 0, 0, 0, 996, 1061, 1031, 1028, 0, 0, 0, 0, 1030, 1069, 1029, 991, 0, 0, 0, 0, 967, 1001, 1007, 1082, 14, 0, 0, 0, 1006, 1073, 972, 1053, 3, 0, 0, 0, 1019, 1026, 1038, 1010, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4044, 4121, 4116, 4119, 4071, 4107, 4097] avg=4096.0000 max=4121 max_rank=2 min=4044 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[989, 1023, 1019, 1055, 4, 0, 0, 0, 1018, 989, 1028, 1025, 2, 0, 0, 0, 1031, 1070, 971, 987, 0, 0, 0, 0, 1037, 1061, 983, 1048, 0, 0, 0, 0, 1059, 1036, 1063, 1019, 0, 0, 0, 0, 1012, 988, 1044, 978, 14, 0, 0, 0, 1049, 1049, 1023, 1050, 3, 0, 0, 0, 981, 1048, 1008, 1000, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4062, 4059, 4129, 4177, 4036, 4174, 4041] avg=4096.0000 max=4177 max_rank=4 min=4036 min_rank=5 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1021, 982, 1024, 1001, 4, 0, 0, 0, 929, 1070, 1027, 1071, 2, 0, 0, 0, 1048, 1042, 998, 1079, 0, 0, 0, 0, 1016, 1065, 1039, 1016, 0, 0, 0, 0, 1042, 982, 1002, 1003, 0, 0, 0, 0, 1069, 1030, 1028, 960, 14, 0, 0, 0, 997, 989, 1028, 1040, 3, 0, 0, 0, 1052, 1056, 1047, 988, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4099, 4167, 4136, 4029, 4101, 4057, 4147] avg=4096.0000 max=4167 max_rank=2 min=4029 min_rank=4 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1001, 1009, 1042, 1037, 4, 0, 0, 0, 1023, 1042, 1055, 1016, 1, 0, 0, 0, 1005, 1021, 989, 1033, 0, 0, 0, 0, 1036, 1024, 1019, 1021, 0, 0, 0, 0, 1032, 1040, 1041, 1022, 0, 0, 0, 0, 1013, 1043, 1001, 1070, 14, 0, 0, 0, 995, 1031, 1023, 1019, 3, 0, 0, 0, 987, 1019, 1032, 1001, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4137, 4048, 4100, 4135, 4141, 4071, 4043] avg=4096.0000 max=4141 max_rank=5 min=4043 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1028, 1010, 1062, 1032, 4, 0, 0, 0, 974, 1046, 1014, 1027, 1, 0, 0, 0, 1013, 1005, 1027, 1064, 0, 0, 0, 0, 1042, 1043, 979, 1038, 0, 0, 0, 0, 1017, 1058, 1012, 1000, 0, 0, 0, 0, 978, 1019, 1016, 1050, 14, 0, 0, 0, 1056, 1056, 1050, 1018, 3, 0, 0, 0, 1007, 969, 1003, 1029, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4062, 4109, 4102, 4087, 4077, 4183, 4012] avg=4096.0000 max=4183 max_rank=6 min=4012 min_rank=7 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1030, 1021, 1055, 1000, 4, 0, 0, 0, 1042, 1038, 1034, 984, 1, 0, 0, 0, 1024, 1033, 1021, 1031, 0, 0, 0, 0, 1005, 933, 1016, 1009, 0, 0, 0, 0, 1043, 1049, 1069, 1018, 0, 0, 0, 0, 988, 986, 1069, 1013, 14, 0, 0, 0, 1053, 994, 1016, 1015, 3, 0, 0, 0, 1082, 1017, 1055, 999, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4099, 4109, 3963, 4179, 4070, 4081, 4157] avg=4096.0000 max=4179 max_rank=4 min=3963 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1051, 992, 1064, 1031, 8, 5, 0, 0, 1041, 1043, 1054, 1003, 6, 3, 0, 0, 1002, 1058, 1040, 992, 0, 0, 0, 0, 1050, 1040, 1004, 1014, 0, 0, 0, 0, 1002, 1041, 990, 974, 0, 0, 0, 0, 1061, 1065, 988, 1020, 18, 0, 0, 0, 982, 1005, 1017, 1016, 8, 0, 0, 0, 1029, 1006, 1006, 1030, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4150, 4092, 4108, 4007, 4152, 4028, 4080] avg=4096.0000 max=4152 max_rank=5 min=4007 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16267 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[999, 1032, 1005, 1023, 4, 0, 0, 0, 1043, 1027, 1022, 1022, 1, 0, 0, 0, 1048, 1015, 956, 1044, 0, 0, 0, 0, 1039, 1015, 1054, 1006, 0, 0, 0, 0, 1006, 1025, 992, 1012, 0, 0, 0, 0, 1021, 1061, 978, 1047, 13, 0, 0, 0, 1015, 1033, 994, 1022, 3, 0, 0, 0, 1093, 1042, 1005, 1047, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4115, 4063, 4114, 4035, 4120, 4067, 4191] avg=4096.0000 max=4191 max_rank=7 min=4035 min_rank=4 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1037, 1018, 1032, 1027, 1033, 983, 989, 1050, 1065, 992, 1035, 1045, 1006, 1023, 982, 1047, 1026, 1038, 1037, 1031, 992, 1009, 1002, 1041, 978, 1089, 994, 1048, 1030, 1053, 1034, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1016, 1031, 1036, 1072, 1046, 1046, 1035, 1057, 960, 1050, 1020, 1020, 999, 989, 1011, 1061, 1006, 1081, 1086, 999, 993, 1025, 1072, 1021, 979, 1010, 1034, 1027, 1004, 976, 969, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1006, 1015, 992, 1031, 1034, 1033, 993, 1026, 1017, 966, 998, 1027, 1041, 1027, 1078, 1034, 1084, 1002, 1012, 1015, 1042, 1046, 1018, 1040, 996, 1017, 1009, 1051, 1035, 1005, 999, 1079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[984, 1056, 1064, 1069, 1021, 1046, 968, 994, 983, 1021, 1036, 1091, 1001, 1073, 1035, 998, 984, 1024, 1033, 969, 1007, 1011, 980, 1021, 1035, 1045, 1046, 1012, 1072, 1019, 1032, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1039, 1050, 1018, 1031, 1044, 1052, 1031, 983, 1064, 1003, 994, 1014, 1008, 980, 1008, 1092, 1040, 1031, 1020, 1024, 1069, 992, 1014, 1093, 1034, 990, 972, 1006, 1047, 1005, 1008, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[986, 1024, 1028, 1028, 990, 1036, 1003, 1015, 1053, 1002, 1030, 1054, 1042, 1014, 1067, 1035, 1018, 1061, 1022, 1031, 1045, 982, 1049, 1021, 1047, 1036, 1044, 1042, 981, 1016, 979, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1060, 1006, 988, 1026, 1070, 1043, 1002, 1055, 997, 986, 1008, 1039, 1033, 1081, 1009, 1054, 1035, 984, 1005, 982, 1027, 1046, 1041, 1039, 982, 997, 1007, 999, 1014, 1025, 1082, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1014, 1052, 1057, 1021, 1050, 981, 1017, 1040, 1012, 1060, 1013, 1078, 1009, 1019, 953, 1045, 1017, 1036, 1041, 977, 1050, 1032, 1025, 1010, 1032, 1012, 995, 1018, 1036, 1019, 1029, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1037, 1018, 1032, 1009, 0, 0, 0, 0, 1033, 983, 989, 1050, 0, 0, 0, 0, 1065, 992, 1035, 1045, 4, 0, 0, 0, 1006, 1023, 982, 1037, 0, 0, 0, 0, 1026, 1038, 1037, 1031, 2, 0, 0, 0, 992, 1009, 1002, 1031, 0, 0, 0, 0, 978, 1089, 994, 1048, 18, 4, 0, 0, 1030, 1053, 1034, 1002, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4055, 4141, 4048, 4134, 4034, 4131, 4129] avg=4096.0000 max=4141 max_rank=2 min=4034 min_rank=5 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[984, 1056, 1064, 1051, 0, 0, 0, 0, 1021, 1046, 968, 994, 0, 0, 0, 0, 983, 1021, 1036, 1091, 4, 0, 0, 0, 1001, 1073, 1035, 989, 0, 0, 0, 0, 984, 1024, 1033, 969, 2, 0, 0, 0, 1007, 1011, 980, 1012, 0, 0, 0, 0, 1035, 1045, 1046, 1012, 18, 4, 0, 0, 1072, 1019, 1032, 1038, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4029, 4135, 4098, 4012, 4010, 4160, 4169] avg=4096.0000 max=4169 max_rank=7 min=4010 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1006, 1015, 992, 1013, 0, 0, 0, 0, 1034, 1033, 993, 1026, 0, 0, 0, 0, 1017, 966, 998, 1027, 4, 0, 0, 0, 1041, 1027, 1078, 1025, 0, 0, 0, 0, 1084, 1002, 1012, 1015, 2, 0, 0, 0, 1042, 1046, 1018, 1030, 0, 0, 0, 0, 996, 1017, 1009, 1051, 18, 4, 0, 0, 1035, 1005, 999, 1079, 6, 3, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4086, 4012, 4171, 4115, 4136, 4095, 4127] avg=4096.0000 max=4171 max_rank=3 min=4012 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1016, 1031, 1036, 1054, 0, 0, 0, 0, 1046, 1046, 1035, 1057, 0, 0, 0, 0, 960, 1050, 1020, 1020, 4, 0, 0, 0, 999, 989, 1011, 1051, 0, 0, 0, 0, 1006, 1081, 1086, 999, 2, 0, 0, 0, 993, 1025, 1072, 1012, 0, 0, 0, 0, 979, 1010, 1034, 1027, 18, 4, 0, 0, 1004, 976, 969, 1037, 5, 4, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4184, 4054, 4050, 4174, 4102, 4072, 3995] avg=4096.0000 max=4184 max_rank=1 min=3995 min_rank=7 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[986, 1024, 1028, 1010, 0, 0, 0, 0, 990, 1036, 1003, 1015, 0, 0, 0, 0, 1053, 1002, 1030, 1054, 4, 0, 0, 0, 1042, 1014, 1067, 1026, 0, 0, 0, 0, 1018, 1061, 1022, 1031, 2, 0, 0, 0, 1045, 982, 1049, 1012, 0, 0, 0, 0, 1047, 1036, 1044, 1042, 18, 4, 0, 0, 981, 1016, 979, 987, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4044, 4143, 4149, 4134, 4088, 4191, 3971] avg=4096.0000 max=4191 max_rank=6 min=3971 min_rank=7 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1014, 1052, 1057, 1004, 0, 0, 0, 0, 1050, 981, 1017, 1040, 0, 0, 0, 0, 1012, 1060, 1013, 1078, 4, 0, 0, 0, 1009, 1019, 953, 1036, 0, 0, 0, 0, 1017, 1036, 1041, 977, 2, 0, 0, 0, 1050, 1032, 1025, 1001, 0, 0, 0, 0, 1032, 1012, 995, 1018, 17, 4, 0, 0, 1036, 1019, 1029, 1018, 5, 3, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4088, 4167, 4017, 4073, 4108, 4078, 4110] avg=4096.0000 max=4167 max_rank=2 min=4017 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1060, 1006, 988, 1008, 0, 0, 0, 0, 1070, 1043, 1002, 1055, 0, 0, 0, 0, 997, 986, 1008, 1039, 4, 0, 0, 0, 1033, 1081, 1009, 1044, 0, 0, 0, 0, 1035, 984, 1005, 982, 2, 0, 0, 0, 1027, 1046, 1041, 1029, 0, 0, 0, 0, 982, 997, 1007, 999, 18, 4, 0, 0, 1014, 1025, 1082, 1046, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4170, 4034, 4167, 4008, 4143, 4007, 4177] avg=4096.0000 max=4177 max_rank=7 min=4007 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1039, 1050, 1018, 1010, 0, 0, 0, 0, 1044, 1052, 1031, 983, 2, 0, 0, 0, 1064, 1003, 994, 1014, 7, 0, 0, 0, 1008, 980, 1008, 1072, 0, 0, 0, 0, 1040, 1031, 1020, 1024, 3, 0, 0, 0, 1069, 992, 1014, 1072, 0, 0, 0, 0, 1034, 990, 972, 1006, 21, 11, 0, 0, 1047, 1005, 1008, 1012, 10, 8, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4112, 4082, 4068, 4118, 4147, 4034, 4090] avg=4096.0000 max=4147 max_rank=5 min=4034 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8448, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1045, 946, 1073, 1002, 989, 1002, 1019, 1017, 1033, 1046, 1009, 1015, 1064, 1019, 1011, 1050, 1011, 1043, 1047, 1059, 1018, 1084, 1029, 1000, 1020, 1021, 1031, 1044, 960, 1039, 1010, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1035, 1028, 1020, 1052, 1037, 999, 1036, 1024, 1009, 1032, 1045, 1021, 1063, 1007, 1044, 1036, 1064, 1007, 1011, 949, 1031, 1016, 1034, 989, 1030, 1043, 1035, 969, 1034, 1038, 1022, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1025, 1043, 1024, 1043, 1052, 1007, 1005, 1014, 991, 1033, 968, 1049, 1066, 1096, 1036, 997, 1005, 1012, 1031, 1033, 1071, 1012, 1021, 990, 997, 987, 1063, 1069, 994, 1014, 990, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[996, 993, 1019, 1018, 1095, 1000, 1000, 1041, 1036, 1027, 1026, 1015, 1023, 986, 1018, 1004, 1056, 1023, 994, 980, 963, 993, 1038, 1036, 1096, 1010, 1093, 1034, 1029, 1025, 1008, 1093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[989, 1026, 1041, 1080, 972, 1021, 1032, 995, 1011, 1028, 1059, 1014, 1024, 1039, 1036, 981, 1050, 991, 1006, 1028, 1044, 1054, 1000, 1053, 968, 1041, 1009, 1062, 1034, 1019, 1054, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1053, 1003, 1012, 1059, 1083, 991, 992, 1036, 1055, 991, 1010, 994, 1056, 1024, 1013, 987, 1035, 1043, 1048, 1059, 1002, 963, 1001, 1066, 1023, 1044, 1045, 1066, 1032, 989, 1004, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1026, 1025, 1070, 1033, 971, 1002, 1021, 1003, 1030, 1077, 1040, 1019, 979, 997, 1030, 1061, 1056, 1006, 1044, 1012, 1003, 976, 1056, 1061, 1045, 977, 997, 1000, 1000, 1079, 1044, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1016, 1015, 1037, 1062, 1018, 985, 1078, 1055, 1034, 1031, 1027, 1013, 1016, 1002, 1020, 991, 1043, 957, 1038, 1043, 1009, 1016, 1014, 985, 1062, 1042, 991, 1025, 1006, 1039, 1042, 1056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1045, 946, 1073, 986, 0, 0, 0, 0, 989, 1002, 1019, 1017, 21, 0, 0, 0, 1033, 1044, 1009, 1015, 0, 0, 0, 0, 1063, 1019, 1011, 1050, 0, 0, 0, 0, 1010, 1043, 1047, 1059, 0, 0, 0, 0, 1018, 1084, 1029, 1000, 16, 0, 0, 0, 1020, 1021, 1031, 1023, 0, 0, 0, 0, 960, 1039, 1010, 1012, 1, 2, 1, 0] rerouted_tokens_per_ep_rank=[4050, 4048, 4101, 4143, 4159, 4147, 4095, 4025] avg=4096.0000 max=4159 max_rank=4 min=4025 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1025, 1043, 1024, 1027, 0, 0, 0, 0, 1052, 1007, 1005, 1014, 22, 0, 0, 0, 991, 1032, 968, 1049, 0, 0, 0, 0, 1065, 1096, 1036, 997, 0, 0, 0, 0, 1004, 1012, 1031, 1033, 0, 0, 0, 0, 1071, 1012, 1021, 990, 16, 0, 0, 0, 997, 987, 1063, 1047, 0, 0, 0, 0, 994, 1014, 990, 1030, 1, 1, 1, 0] rerouted_tokens_per_ep_rank=[4119, 4100, 4040, 4194, 4080, 4110, 4094, 4031] avg=4096.0000 max=4194 max_rank=3 min=4031 min_rank=7 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1035, 1028, 1020, 1035, 0, 0, 0, 0, 1037, 999, 1036, 1024, 20, 0, 0, 0, 1009, 1031, 1045, 1021, 0, 0, 0, 0, 1062, 1007, 1044, 1036, 0, 0, 0, 0, 1062, 1007, 1011, 949, 0, 0, 0, 0, 1031, 1016, 1034, 989, 17, 0, 0, 0, 1030, 1043, 1035, 949, 0, 0, 0, 0, 1034, 1038, 1022, 1008, 2, 1, 1, 0] rerouted_tokens_per_ep_rank=[4118, 4116, 4106, 4149, 4029, 4087, 4057, 4106] avg=4096.0000 max=4149 max_rank=3 min=4029 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[996, 993, 1019, 1002, 0, 0, 0, 0, 1095, 1000, 1000, 1041, 21, 0, 0, 0, 1036, 1026, 1026, 1015, 0, 0, 0, 0, 1023, 986, 1018, 1004, 0, 0, 0, 0, 1054, 1023, 994, 980, 0, 0, 0, 0, 963, 993, 1038, 1036, 16, 0, 0, 0, 1096, 1010, 1093, 1013, 0, 0, 0, 0, 1029, 1025, 1008, 1093, 2, 1, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4157, 4103, 4031, 4051, 4046, 4212, 4158] avg=4096.0000 max=4212 max_rank=6 min=4010 min_rank=0 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1053, 1003, 1012, 1042, 0, 0, 0, 0, 1083, 991, 992, 1036, 22, 0, 0, 0, 1055, 990, 1010, 994, 0, 0, 0, 0, 1055, 1024, 1013, 987, 0, 0, 0, 0, 1034, 1043, 1048, 1059, 0, 0, 0, 0, 1002, 963, 1001, 1066, 17, 0, 0, 0, 1023, 1044, 1045, 1044, 0, 0, 0, 0, 1032, 989, 1004, 989, 1, 1, 1, 0] rerouted_tokens_per_ep_rank=[4110, 4124, 4049, 4079, 4184, 4049, 4156, 4017] avg=4096.0000 max=4184 max_rank=4 min=4017 min_rank=7 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[989, 1026, 1041, 1063, 0, 0, 0, 0, 972, 1021, 1032, 995, 21, 0, 0, 0, 1011, 1027, 1059, 1014, 0, 0, 0, 0, 1024, 1039, 1036, 981, 0, 0, 0, 0, 1048, 991, 1006, 1028, 0, 0, 0, 0, 1044, 1054, 1000, 1053, 17, 0, 0, 0, 968, 1041, 1009, 1041, 0, 0, 0, 0, 1034, 1019, 1054, 1007, 2, 1, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4041, 4111, 4080, 4073, 4168, 4059, 4117] avg=4096.0000 max=4168 max_rank=5 min=4041 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1026, 1025, 1070, 1008, 0, 0, 0, 0, 971, 1002, 1021, 1003, 23, 5, 0, 0, 1030, 1065, 1040, 1019, 0, 0, 0, 0, 975, 997, 1030, 1061, 0, 0, 0, 0, 1051, 1006, 1044, 1012, 0, 0, 0, 0, 1003, 976, 1056, 1061, 20, 4, 0, 0, 1045, 977, 997, 977, 0, 0, 0, 0, 1000, 1079, 1044, 1028, 5, 8, 4, 0] rerouted_tokens_per_ep_rank=[4129, 4025, 4154, 4063, 4113, 4120, 3996, 4168] avg=4096.0000 max=4168 max_rank=7 min=3996 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1016, 1015, 1037, 1045, 0, 0, 0, 0, 1018, 985, 1078, 1055, 21, 0, 0, 0, 1034, 1030, 1027, 1013, 0, 0, 0, 0, 1016, 1002, 1020, 991, 0, 0, 0, 0, 1041, 957, 1038, 1043, 0, 0, 0, 0, 1009, 1016, 1014, 985, 17, 0, 0, 0, 1062, 1042, 991, 1004, 0, 0, 0, 0, 1006, 1039, 1042, 1056, 2, 1, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4157, 4104, 4029, 4079, 4041, 4099, 4146] avg=4096.0000 max=4157 max_rank=1 min=4029 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1088, 1015, 1006, 997, 1026, 992, 1057, 1056, 1007, 1064, 1096, 1006, 1009, 1018, 1018, 1003, 997, 1033, 1030, 1013, 1009, 1005, 1055, 1029, 1005, 1007, 1034, 1008, 1047, 1000, 1000, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1021, 996, 1011, 1044, 1042, 1058, 1071, 1031, 982, 989, 993, 1028, 1023, 1038, 1043, 1034, 1058, 1039, 1009, 1035, 985, 1023, 1067, 1080, 992, 1016, 991, 995, 1035, 1001, 1013, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1025, 992, 1050, 1014, 1075, 1078, 992, 1050, 1035, 1007, 997, 1039, 1023, 1057, 1014, 1061, 1005, 1024, 1029, 1005, 1005, 1016, 995, 1075, 1053, 1020, 1027, 1002, 954, 996, 1009, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1004, 977, 1087, 1039, 1043, 1052, 1029, 1026, 978, 1003, 1006, 1019, 984, 998, 1047, 1030, 1018, 1076, 1042, 1031, 1020, 978, 990, 999, 1009, 1063, 1021, 1042, 1072, 1027, 1035, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[977, 1051, 1041, 1052, 1032, 1000, 1034, 1013, 1023, 1038, 1032, 1029, 1007, 1064, 1065, 1011, 987, 1034, 1002, 1021, 992, 1046, 1021, 1051, 1043, 981, 1002, 1019, 1048, 1006, 1006, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1060, 1031, 1047, 997, 1001, 1034, 1019, 1050, 1019, 1050, 1049, 1027, 1036, 1049, 1047, 1003, 1032, 1040, 993, 984, 1043, 978, 996, 1035, 1032, 964, 1053, 1045, 1005, 1015, 989, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1046, 1009, 1024, 1040, 1043, 1054, 1022, 1027, 1003, 1007, 1012, 1070, 1007, 1017, 1056, 998, 1060, 1012, 1034, 1038, 1005, 965, 976, 1006, 1030, 1010, 1014, 1054, 1050, 1024, 995, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1029, 1025, 1103, 1006, 969, 996, 1062, 1009, 986, 1067, 1013, 994, 1037, 980, 1041, 1040, 997, 985, 1059, 1079, 1026, 998, 1042, 1036, 1058, 976, 1025, 974, 1046, 1025, 1041, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1021, 996, 995, 1044, 0, 0, 0, 0, 1042, 1058, 1037, 1031, 0, 0, 0, 0, 982, 989, 993, 1028, 9, 2, 0, 0, 1023, 1038, 1033, 1034, 0, 0, 0, 0, 1058, 1036, 1009, 1035, 0, 0, 0, 0, 985, 1023, 1067, 1080, 28, 0, 0, 0, 992, 1016, 991, 995, 16, 6, 1, 0, 1035, 1001, 1013, 1025, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4168, 4003, 4128, 4138, 4183, 4017, 4075] avg=4096.0000 max=4183 max_rank=5 min=4003 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1025, 992, 1033, 1014, 0, 0, 0, 0, 1075, 1078, 960, 1050, 0, 0, 0, 0, 1035, 1007, 997, 1039, 9, 2, 0, 0, 1023, 1057, 1004, 1061, 0, 0, 0, 0, 1005, 1021, 1029, 1005, 0, 0, 0, 0, 1005, 1016, 995, 1075, 26, 0, 0, 0, 1053, 1020, 1027, 1002, 17, 6, 1, 0, 954, 996, 1009, 1044, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4163, 4089, 4145, 4060, 4117, 4126, 4004] avg=4096.0000 max=4163 max_rank=1 min=4004 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1004, 977, 1070, 1039, 0, 0, 0, 0, 1043, 1052, 996, 1026, 0, 0, 0, 0, 978, 1003, 1006, 1019, 9, 3, 0, 0, 984, 998, 1037, 1030, 0, 0, 0, 0, 1018, 1072, 1042, 1031, 0, 0, 0, 0, 1020, 978, 990, 999, 27, 0, 0, 0, 1009, 1063, 1021, 1042, 17, 6, 1, 0, 1072, 1027, 1035, 1023, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4117, 4018, 4049, 4163, 4014, 4159, 4158] avg=4096.0000 max=4163 max_rank=4 min=4014 min_rank=5 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1088, 1015, 990, 997, 0, 0, 0, 0, 1026, 992, 1023, 1056, 0, 0, 0, 0, 1007, 1064, 1096, 1006, 9, 2, 0, 0, 1009, 1018, 1008, 1003, 0, 0, 0, 0, 997, 1030, 1030, 1013, 0, 0, 0, 0, 1009, 1005, 1055, 1029, 28, 0, 0, 0, 1005, 1007, 1034, 1008, 16, 6, 1, 0, 1047, 1000, 1000, 1038, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4097, 4184, 4038, 4070, 4126, 4077, 4086] avg=4096.0000 max=4184 max_rank=2 min=4038 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[977, 1051, 1025, 1052, 0, 0, 0, 0, 1032, 1000, 1001, 1013, 0, 0, 0, 0, 1023, 1038, 1032, 1029, 9, 2, 0, 0, 1007, 1064, 1055, 1011, 0, 0, 0, 0, 987, 1031, 1002, 1021, 0, 0, 0, 0, 992, 1046, 1021, 1051, 27, 0, 0, 0, 1043, 981, 1002, 1019, 16, 6, 1, 0, 1048, 1006, 1006, 1040, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4046, 4133, 4137, 4041, 4137, 4068, 4101] avg=4096.0000 max=4137 max_rank=3 min=4041 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1060, 1031, 1030, 997, 0, 0, 0, 0, 1001, 1034, 986, 1050, 0, 0, 0, 0, 1019, 1050, 1049, 1027, 9, 2, 0, 0, 1036, 1049, 1037, 1003, 0, 0, 0, 0, 1032, 1037, 993, 984, 0, 0, 0, 0, 1043, 978, 996, 1035, 27, 0, 0, 0, 1032, 964, 1053, 1045, 17, 6, 1, 0, 1005, 1015, 989, 1045, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4071, 4156, 4125, 4046, 4079, 4118, 4055] avg=4096.0000 max=4156 max_rank=2 min=4046 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1046, 1009, 1004, 1040, 0, 0, 0, 0, 1043, 1054, 980, 1027, 0, 0, 0, 0, 1003, 1007, 1012, 1070, 14, 8, 0, 0, 1007, 1017, 1036, 998, 0, 0, 0, 0, 1060, 1001, 1034, 1038, 0, 0, 0, 0, 1005, 965, 976, 1006, 30, 0, 0, 0, 1030, 1010, 1014, 1054, 20, 12, 6, 0, 1050, 1024, 995, 1060, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4104, 4114, 4058, 4133, 3982, 4146, 4132] avg=4096.0000 max=4146 max_rank=6 min=3982 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1029, 1025, 1086, 1006, 0, 0, 0, 0, 969, 996, 1028, 1009, 0, 0, 0, 0, 986, 1067, 1013, 994, 9, 2, 0, 0, 1037, 980, 1031, 1040, 0, 0, 0, 0, 997, 982, 1059, 1079, 0, 0, 0, 0, 1026, 998, 1042, 1036, 28, 0, 0, 0, 1058, 976, 1025, 974, 17, 6, 1, 0, 1046, 1025, 1041, 1044, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4002, 4071, 4088, 4117, 4130, 4057, 4157] avg=4096.0000 max=4157 max_rank=7 min=4002 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1066, 976, 1014, 1030, 1058, 1056, 1065, 988, 1043, 1021, 1015, 1065, 1041, 1042, 1006, 1026, 1023, 996, 1042, 1003, 982, 1068, 1000, 991, 977, 1030, 1031, 1065, 1010, 1017, 998, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[976, 986, 1010, 1013, 1005, 1051, 1001, 981, 1049, 1066, 994, 1054, 998, 1067, 963, 983, 1058, 1052, 995, 1045, 1041, 1040, 1104, 1029, 983, 981, 1085, 999, 974, 1066, 1039, 1080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1084, 1069, 1038, 996, 998, 1006, 1016, 985, 990, 1038, 1010, 984, 1050, 1026, 1077, 951, 989, 1060, 1025, 973, 1021, 981, 1032, 1044, 1038, 1014, 1062, 1016, 1095, 1040, 1019, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1024, 1037, 1046, 1067, 1043, 992, 1053, 1013, 1050, 976, 996, 1027, 1006, 1004, 1063, 1003, 1051, 1035, 1028, 1039, 1040, 1057, 1005, 1015, 998, 1072, 943, 1038, 1033, 1016, 994, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1007, 1037, 1015, 998, 1006, 947, 1029, 1012, 1003, 1053, 1060, 1046, 1026, 1037, 1032, 1048, 981, 988, 1039, 1032, 1035, 1026, 997, 1044, 1023, 1047, 991, 1037, 1047, 1064, 1020, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1016, 998, 994, 1053, 1045, 1018, 1045, 1021, 1060, 1053, 1035, 1030, 994, 1072, 1005, 999, 1053, 1019, 1033, 961, 1056, 1043, 1011, 986, 987, 1012, 1023, 1019, 1029, 1024, 1029, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[952, 1034, 1015, 1050, 1015, 1089, 1030, 1085, 1029, 977, 996, 1049, 1054, 1087, 1061, 1007, 989, 1040, 1040, 953, 1000, 1049, 1009, 1018, 1012, 1061, 1036, 1001, 1015, 1040, 964, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1021, 1009, 1046, 1034, 989, 1015, 1005, 1062, 1052, 993, 1059, 998, 1029, 1007, 1028, 1009, 1027, 1013, 1026, 1032, 1027, 989, 1064, 1060, 994, 1053, 1007, 1025, 1037, 1005, 1071, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1066, 976, 1014, 1030, 7, 0, 0, 0, 1058, 1056, 1065, 988, 4, 1, 0, 0, 1031, 1021, 1015, 1065, 0, 0, 0, 0, 1041, 1038, 1006, 1026, 0, 0, 0, 0, 1023, 996, 1042, 1003, 12, 2, 0, 0, 982, 1057, 1000, 991, 0, 0, 0, 0, 977, 1030, 1031, 1065, 10, 3, 0, 0, 1010, 1005, 998, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4172, 4132, 4111, 4078, 4030, 4116, 4036] avg=4096.0000 max=4172 max_rank=1 min=4030 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1016, 998, 994, 1053, 7, 0, 0, 0, 1045, 1018, 1045, 1021, 4, 1, 0, 0, 1048, 1053, 1035, 1030, 0, 0, 0, 0, 994, 1068, 1005, 999, 0, 0, 0, 0, 1053, 1019, 1033, 961, 12, 2, 0, 0, 1056, 1032, 1011, 986, 0, 0, 0, 0, 987, 1012, 1023, 1019, 10, 3, 0, 0, 1029, 1012, 1029, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4134, 4166, 4066, 4080, 4085, 4054, 4115] avg=4096.0000 max=4166 max_rank=2 min=4054 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 1037, 1046, 1067, 7, 0, 0, 0, 1043, 992, 1053, 1013, 3, 1, 0, 0, 1038, 976, 996, 1027, 0, 0, 0, 0, 1006, 1001, 1063, 1003, 0, 0, 0, 0, 1051, 1035, 1028, 1039, 12, 2, 0, 0, 1040, 1046, 1005, 1015, 0, 0, 0, 0, 998, 1072, 943, 1038, 10, 3, 0, 0, 1033, 1004, 994, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4105, 4037, 4073, 4167, 4106, 4064, 4035] avg=4096.0000 max=4181 max_rank=0 min=4035 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1007, 1037, 1015, 998, 7, 0, 0, 0, 1006, 947, 1029, 1012, 4, 1, 0, 0, 992, 1053, 1060, 1046, 0, 0, 0, 0, 1026, 1033, 1032, 1048, 0, 0, 0, 0, 981, 988, 1039, 1032, 13, 2, 0, 0, 1035, 1015, 997, 1044, 0, 0, 0, 0, 1023, 1047, 991, 1037, 9, 3, 0, 0, 1047, 1051, 1020, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 3999, 4151, 4139, 4055, 4091, 4110, 4159] avg=4096.0000 max=4159 max_rank=7 min=3999 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1084, 1069, 1038, 996, 6, 0, 0, 0, 998, 1006, 1016, 985, 4, 1, 0, 0, 979, 1038, 1010, 984, 0, 0, 0, 0, 1050, 1022, 1077, 951, 0, 0, 0, 0, 989, 1060, 1025, 973, 13, 2, 0, 0, 1021, 971, 1032, 1044, 0, 0, 0, 0, 1038, 1014, 1062, 1016, 9, 3, 0, 0, 1095, 1027, 1019, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4193, 4010, 4011, 4100, 4062, 4068, 4142, 4182] avg=4096.0000 max=4193 max_rank=0 min=4010 min_rank=1 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[976, 986, 1010, 1013, 7, 0, 0, 0, 1005, 1051, 1001, 981, 4, 1, 0, 0, 1037, 1066, 994, 1054, 0, 0, 0, 0, 998, 1063, 963, 983, 0, 0, 0, 0, 1058, 1052, 995, 1045, 13, 2, 0, 0, 1041, 1029, 1104, 1029, 0, 0, 0, 0, 983, 981, 1085, 999, 10, 3, 0, 0, 974, 1053, 1039, 1080, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3992, 4043, 4151, 4007, 4165, 4203, 4061, 4146] avg=4096.0000 max=4203 max_rank=5 min=3992 min_rank=0 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[952, 1034, 1015, 1050, 10, 0, 0, 0, 1015, 1089, 1030, 1085, 7, 4, 0, 0, 1008, 977, 996, 1049, 0, 0, 0, 0, 1054, 1080, 1061, 1007, 0, 0, 0, 0, 989, 1040, 1040, 953, 18, 9, 0, 0, 1000, 1028, 1009, 1018, 0, 0, 0, 0, 1012, 1061, 1036, 1001, 12, 7, 0, 0, 1015, 1022, 964, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4230, 4030, 4202, 4049, 4055, 4129, 4012] avg=4096.0000 max=4230 max_rank=1 min=4012 min_rank=7 imbalance_ratio=1.032715 max_load_reduction=28538 max_load_reduction_pct=0.870911 moved_to_echo=16245 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1021, 1009, 1046, 1034, 6, 0, 0, 0, 989, 1015, 1005, 1062, 3, 1, 0, 0, 1040, 993, 1059, 998, 0, 0, 0, 0, 1029, 1004, 1028, 1009, 0, 0, 0, 0, 1027, 1013, 1026, 1032, 12, 2, 0, 0, 1027, 979, 1064, 1060, 0, 0, 0, 0, 994, 1053, 1007, 1025, 10, 3, 0, 0, 1037, 993, 1071, 982, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4075, 4090, 4070, 4112, 4130, 4092, 4083] avg=4096.0000 max=4130 max_rank=5 min=4070 min_rank=3 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[996, 1053, 1017, 1076, 1012, 1019, 1025, 979, 1059, 1035, 1004, 1029, 1017, 1028, 1020, 1043, 1000, 1010, 1062, 1026, 990, 1038, 983, 1024, 1059, 996, 1013, 1041, 1017, 1049, 1010, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1024, 1054, 1033, 1040, 1033, 1025, 1026, 985, 1030, 983, 1010, 1030, 991, 1044, 1049, 969, 1030, 1020, 1016, 1010, 1070, 965, 1071, 1049, 1060, 1036, 1010, 1028, 992, 1025, 1036, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[989, 1033, 1025, 987, 1014, 1089, 1024, 1056, 1014, 1018, 1026, 1066, 1024, 1031, 1002, 976, 1021, 989, 1043, 1037, 1044, 1019, 1039, 1044, 1007, 1012, 1048, 1009, 1041, 1048, 988, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1023, 1026, 1047, 1021, 1051, 1063, 967, 1015, 1020, 1060, 1044, 1024, 1024, 981, 1027, 1003, 1031, 1027, 1028, 1040, 1028, 1000, 1063, 1038, 978, 1013, 1002, 1038, 1007, 1001, 1010, 1068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1022, 1046, 994, 1037, 1026, 1038, 998, 1016, 974, 1033, 1011, 1023, 1008, 1025, 977, 1085, 1067, 985, 1009, 1051, 1043, 1037, 1044, 1013, 1061, 1025, 1016, 1029, 1019, 1043, 1022, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1039, 1063, 1062, 1014, 1033, 979, 1013, 1032, 1052, 1018, 1023, 1057, 1083, 958, 1002, 998, 1029, 1005, 1051, 1035, 1038, 1004, 1041, 1030, 1005, 1047, 1024, 967, 983, 1001, 1036, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[991, 1041, 1007, 1021, 1028, 990, 1076, 1019, 1006, 996, 1059, 941, 1027, 990, 1033, 1027, 1024, 1038, 993, 1052, 1013, 1026, 994, 1057, 1038, 1028, 1052, 1059, 1021, 1048, 1023, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1053, 1012, 1057, 1009, 1024, 1053, 1076, 1049, 1025, 961, 996, 1055, 1026, 1011, 1075, 999, 1044, 1028, 1040, 1012, 953, 1056, 960, 1065, 997, 1038, 1044, 996, 1006, 1018, 1016, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[996, 1035, 1017, 1076, 0, 0, 0, 0, 1012, 1012, 1025, 979, 0, 0, 0, 0, 1059, 1035, 1004, 1029, 8, 1, 0, 0, 1017, 1028, 1020, 1043, 18, 8, 0, 0, 1000, 1010, 1062, 1017, 0, 0, 0, 0, 990, 1038, 983, 1016, 0, 0, 0, 0, 1059, 996, 1013, 1041, 0, 0, 0, 0, 1017, 1049, 1010, 1038, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4028, 4136, 4134, 4089, 4027, 4109, 4121] avg=4096.0000 max=4136 max_rank=2 min=4027 min_rank=5 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1024, 1036, 1033, 1040, 0, 0, 0, 0, 1033, 1018, 1026, 985, 0, 0, 0, 0, 1030, 983, 1010, 1030, 8, 1, 0, 0, 991, 1044, 1049, 969, 18, 8, 0, 0, 1030, 1020, 1016, 1001, 0, 0, 0, 0, 1070, 965, 1071, 1041, 0, 0, 0, 0, 1060, 1036, 1010, 1028, 0, 0, 0, 0, 992, 1025, 1036, 1024, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4062, 4062, 4079, 4067, 4147, 4134, 4084] avg=4096.0000 max=4147 max_rank=5 min=4062 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[989, 1016, 1025, 987, 0, 0, 0, 0, 1014, 1081, 1024, 1056, 0, 0, 0, 0, 1014, 1018, 1026, 1066, 8, 1, 0, 0, 1024, 1031, 1002, 976, 17, 8, 0, 0, 1021, 989, 1043, 1028, 0, 0, 0, 0, 1044, 1019, 1039, 1036, 0, 0, 0, 0, 1007, 1012, 1047, 1009, 0, 0, 0, 0, 1041, 1048, 988, 1005, 8, 1, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4175, 4133, 4058, 4081, 4138, 4075, 4091] avg=4096.0000 max=4175 max_rank=1 min=4017 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1023, 1009, 1047, 1021, 0, 0, 0, 0, 1051, 1055, 967, 1015, 0, 0, 0, 0, 1020, 1060, 1044, 1024, 8, 1, 0, 0, 1024, 981, 1027, 1003, 17, 8, 0, 0, 1031, 1027, 1028, 1031, 0, 0, 0, 0, 1028, 1000, 1063, 1030, 0, 0, 0, 0, 978, 1013, 1002, 1038, 0, 0, 0, 0, 1007, 1001, 1010, 1068, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4088, 4157, 4060, 4117, 4121, 4031, 4094] avg=4096.0000 max=4157 max_rank=2 min=4031 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1022, 1028, 994, 1037, 0, 0, 0, 0, 1026, 1030, 998, 1016, 0, 0, 0, 0, 974, 1033, 1011, 1023, 8, 1, 0, 0, 1008, 1025, 977, 1085, 18, 9, 0, 0, 1067, 985, 1009, 1041, 0, 0, 0, 0, 1043, 1037, 1044, 1005, 0, 0, 0, 0, 1061, 1025, 1016, 1029, 0, 0, 0, 0, 1019, 1043, 1022, 991, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4070, 4050, 4122, 4102, 4129, 4131, 4083] avg=4096.0000 max=4131 max_rank=6 min=4050 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1039, 1042, 1062, 1014, 0, 0, 0, 0, 1033, 967, 1013, 1032, 0, 0, 0, 0, 1052, 1018, 1023, 1057, 13, 7, 1, 0, 1083, 958, 1002, 998, 21, 13, 0, 0, 1029, 1005, 1051, 1015, 0, 0, 0, 0, 1038, 1004, 1041, 1017, 0, 0, 0, 0, 1005, 1047, 1019, 967, 0, 0, 0, 0, 983, 1001, 1036, 1046, 11, 5, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4045, 4171, 4075, 4100, 4100, 4038, 4082] avg=4096.0000 max=4171 max_rank=2 min=4038 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1053, 995, 1057, 1009, 0, 0, 0, 0, 1024, 1045, 1076, 1049, 0, 0, 0, 0, 1025, 961, 996, 1055, 9, 1, 0, 0, 1026, 1011, 1075, 999, 17, 8, 0, 0, 1044, 1028, 1040, 1003, 0, 0, 0, 0, 953, 1056, 960, 1056, 0, 0, 0, 0, 997, 1038, 1043, 996, 0, 0, 0, 0, 1006, 1018, 1016, 1014, 8, 1, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4194, 4047, 4136, 4115, 4025, 4074, 4063] avg=4096.0000 max=4194 max_rank=1 min=4025 min_rank=5 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[991, 1023, 1007, 1021, 0, 0, 0, 0, 1028, 983, 1076, 1019, 0, 0, 0, 0, 1006, 996, 1059, 941, 9, 1, 0, 0, 1027, 990, 1033, 1027, 18, 9, 0, 0, 1024, 1038, 993, 1042, 0, 0, 0, 0, 1013, 1026, 994, 1048, 0, 0, 0, 0, 1038, 1028, 1051, 1059, 0, 0, 0, 0, 1021, 1048, 1023, 1050, 7, 1, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4106, 4012, 4104, 4097, 4081, 4176, 4150] avg=4096.0000 max=4176 max_rank=6 min=4012 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1031, 1040, 1012, 1026, 1037, 1012, 995, 1042, 1051, 1043, 999, 1015, 1036, 1048, 1001, 999, 1038, 1062, 1004, 1016, 1032, 1012, 1052, 979, 1000, 1052, 989, 1012, 1029, 1048, 1053, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1045, 1030, 1034, 1018, 981, 1027, 1005, 1045, 1048, 996, 1046, 1061, 1008, 1020, 1035, 1076, 990, 988, 1010, 1038, 1027, 1041, 1042, 1048, 1002, 1003, 1023, 993, 975, 1043, 1030, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1035, 1050, 1010, 1057, 992, 972, 1032, 1020, 1023, 1023, 1033, 984, 1018, 1058, 1030, 999, 1039, 1046, 1029, 1015, 1037, 1003, 1060, 1030, 1044, 991, 992, 1014, 1050, 1016, 1046, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1069, 1041, 1028, 1059, 1047, 1061, 1024, 1005, 1006, 1045, 1066, 984, 989, 1036, 1023, 1030, 1039, 967, 1000, 1050, 1008, 1029, 1059, 1014, 945, 1026, 1031, 1007, 1010, 1029, 1047, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1018, 977, 1027, 1008, 1054, 1062, 1024, 1030, 974, 1033, 1035, 1030, 998, 1071, 1025, 1050, 996, 1049, 981, 1029, 1007, 1034, 1033, 1007, 1005, 985, 1011, 1050, 1003, 1012, 1054, 1096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1014, 1015, 1076, 1065, 1012, 1023, 1030, 1057, 1028, 1032, 993, 997, 1070, 1052, 987, 1007, 1061, 1010, 1008, 1010, 1020, 984, 1021, 1018, 1050, 980, 1054, 1057, 1013, 1015, 1001, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1035, 1032, 1046, 1017, 1036, 1066, 1054, 1074, 982, 998, 1053, 1020, 1001, 1047, 1011, 1058, 1043, 1044, 1042, 965, 1082, 1003, 950, 1009, 984, 1038, 1027, 992, 982, 1057, 993, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1032, 1073, 1060, 1057, 1014, 1043, 988, 1024, 967, 1021, 978, 1053, 983, 1020, 989, 1028, 1042, 1047, 1003, 999, 1053, 993, 1085, 1018, 1009, 1017, 1007, 1021, 1051, 1074, 1016, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1031, 1040, 1012, 981, 0, 0, 0, 0, 1037, 1012, 995, 1027, 0, 0, 0, 0, 1051, 1043, 999, 1015, 15, 2, 1, 0, 1036, 1044, 1001, 999, 0, 0, 0, 0, 1038, 1062, 1004, 1016, 6, 4, 2, 0, 1032, 1012, 1050, 979, 0, 0, 0, 0, 1000, 1052, 989, 1012, 44, 0, 0, 0, 1029, 1040, 1053, 1003, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4071, 4126, 4080, 4132, 4073, 4097, 4125] avg=4096.0000 max=4132 max_rank=4 min=4064 min_rank=0 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1069, 1041, 1028, 1013, 0, 0, 0, 0, 1047, 1061, 1024, 991, 0, 0, 0, 0, 1006, 1045, 1066, 984, 14, 2, 1, 0, 989, 1032, 1023, 1030, 0, 0, 0, 0, 1039, 967, 1000, 1050, 6, 4, 2, 0, 1008, 1029, 1057, 1014, 0, 0, 0, 0, 945, 1026, 1031, 1007, 45, 0, 0, 0, 1010, 1021, 1047, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4123, 4118, 4074, 4068, 4108, 4054, 4072] avg=4096.0000 max=4151 max_rank=0 min=4054 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1035, 1050, 1010, 1011, 0, 0, 0, 0, 992, 972, 1032, 1006, 0, 0, 0, 0, 1023, 1023, 1033, 984, 14, 2, 1, 0, 1018, 1054, 1030, 999, 0, 0, 0, 0, 1039, 1046, 1029, 1015, 6, 4, 2, 0, 1037, 1003, 1058, 1030, 0, 0, 0, 0, 1044, 991, 992, 1014, 45, 0, 0, 0, 1050, 1008, 1046, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4002, 4080, 4101, 4141, 4128, 4086, 4124] avg=4096.0000 max=4141 max_rank=4 min=4002 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1045, 1030, 1034, 974, 0, 0, 0, 0, 981, 1027, 1005, 1030, 0, 0, 0, 0, 1048, 996, 1046, 1061, 15, 2, 1, 0, 1008, 1016, 1035, 1076, 0, 0, 0, 0, 990, 988, 1010, 1038, 6, 4, 2, 0, 1027, 1041, 1040, 1048, 0, 0, 0, 0, 1002, 1003, 1023, 993, 43, 0, 0, 0, 975, 1035, 1030, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4043, 4169, 4135, 4038, 4156, 4064, 4080] avg=4096.0000 max=4169 max_rank=2 min=4038 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1014, 1015, 1076, 1019, 0, 0, 0, 0, 1012, 1023, 1030, 1042, 0, 0, 0, 0, 1028, 1032, 993, 997, 15, 2, 1, 0, 1070, 1048, 987, 1007, 0, 0, 0, 0, 1061, 1010, 1008, 1010, 6, 4, 2, 0, 1020, 984, 1019, 1018, 0, 0, 0, 0, 1050, 980, 1054, 1057, 45, 0, 0, 0, 1013, 1007, 1001, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4107, 4068, 4112, 4101, 4041, 4186, 4029] avg=4096.0000 max=4186 max_rank=6 min=4029 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1018, 977, 1027, 964, 0, 0, 0, 0, 1054, 1062, 1024, 1016, 0, 0, 0, 0, 974, 1033, 1035, 1030, 14, 2, 1, 0, 998, 1067, 1025, 1050, 0, 0, 0, 0, 996, 1049, 981, 1029, 6, 4, 2, 0, 1007, 1034, 1031, 1007, 0, 0, 0, 0, 1005, 985, 1011, 1050, 43, 0, 0, 0, 1003, 1004, 1054, 1096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3986, 4156, 4089, 4140, 4067, 4079, 4094, 4157] avg=4096.0000 max=4157 max_rank=7 min=3986 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1035, 1032, 1046, 965, 0, 0, 0, 0, 1036, 1066, 1054, 1055, 0, 0, 0, 0, 982, 998, 1053, 1020, 19, 5, 5, 0, 1001, 1040, 1011, 1058, 0, 0, 0, 0, 1043, 1044, 1042, 965, 9, 7, 8, 0, 1082, 1003, 942, 1009, 0, 0, 0, 0, 984, 1038, 1027, 992, 47, 0, 0, 0, 982, 1043, 993, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4211, 4082, 4110, 4118, 4036, 4088, 4045] avg=4096.0000 max=4211 max_rank=1 min=4036 min_rank=5 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1032, 1073, 1060, 1011, 0, 0, 0, 0, 1014, 1043, 988, 1010, 0, 0, 0, 0, 967, 1021, 978, 1053, 14, 2, 1, 0, 983, 1016, 989, 1028, 0, 0, 0, 0, 1042, 1047, 1003, 999, 6, 4, 2, 0, 1053, 993, 1083, 1018, 0, 0, 0, 0, 1009, 1017, 1007, 1021, 45, 0, 0, 0, 1051, 1066, 1016, 1003, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4055, 4036, 4016, 4103, 4147, 4099, 4136] avg=4096.0000 max=4176 max_rank=0 min=4016 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1020, 1106, 1011, 1010, 1023, 1027, 1012, 1071, 1007, 1022, 1034, 1032, 1016, 1001, 1001, 1030, 1031, 1003, 997, 1036, 1005, 1041, 998, 1053, 1048, 995, 1042, 1035, 1019, 1020, 995, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1035, 1007, 1056, 1075, 1020, 1004, 1060, 1041, 994, 1003, 1053, 1027, 1046, 1038, 1006, 1034, 985, 1023, 1043, 992, 997, 1079, 1029, 1011, 1031, 1035, 1037, 992, 1012, 997, 1002, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1023, 1017, 1042, 1008, 1049, 1040, 1052, 1044, 1032, 1029, 1012, 1000, 1018, 990, 1038, 1013, 1011, 1024, 996, 1065, 1022, 1037, 1020, 990, 1011, 1056, 1034, 1076, 991, 1044, 1001, 983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[998, 1004, 1070, 1079, 1003, 1056, 1058, 1040, 1038, 1020, 995, 1012, 987, 967, 1021, 1078, 1004, 1034, 1039, 1033, 1017, 1043, 999, 1024, 993, 990, 986, 1057, 1032, 996, 1057, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1011, 1046, 1037, 1062, 1029, 985, 1052, 1057, 973, 977, 1073, 1002, 1045, 1063, 1058, 1039, 1014, 1044, 1083, 1004, 990, 1026, 1026, 1024, 994, 1006, 1037, 1049, 972, 968, 1029, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1083, 1009, 1046, 1024, 1044, 1017, 1008, 1041, 1000, 1044, 990, 1002, 1044, 1035, 1031, 1014, 976, 971, 1026, 1032, 1042, 1033, 1010, 1006, 1006, 1060, 982, 1037, 1073, 991, 1048, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1062, 976, 995, 996, 1025, 995, 1007, 1011, 1073, 993, 1006, 1015, 1051, 1033, 1046, 978, 1035, 1068, 1037, 1009, 1045, 1071, 1065, 1062, 1079, 1005, 991, 1023, 1003, 995, 1041, 977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1052, 1040, 1035, 983, 1017, 1014, 1038, 1057, 1071, 981, 1014, 1049, 1024, 1033, 1031, 1034, 989, 1049, 986, 1063, 1014, 1028, 1071, 1017, 1007, 1032, 1026, 1023, 975, 999, 997, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1035, 1007, 1025, 1075, 0, 0, 0, 0, 1020, 1004, 1060, 1013, 0, 0, 0, 0, 994, 1003, 1053, 1027, 16, 6, 1, 0, 1046, 1038, 998, 1034, 0, 0, 0, 0, 985, 1023, 1043, 992, 7, 0, 0, 0, 997, 1063, 1029, 1011, 0, 0, 0, 0, 1031, 1035, 1037, 992, 0, 0, 0, 0, 1012, 997, 1002, 1004, 31, 22, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4097, 4100, 4116, 4050, 4100, 4095, 4068] avg=4096.0000 max=4142 max_rank=0 min=4050 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1020, 1106, 981, 1010, 0, 0, 0, 0, 1023, 1027, 1012, 1043, 0, 0, 0, 0, 1007, 1022, 1034, 1032, 15, 6, 1, 0, 1016, 1001, 993, 1030, 0, 0, 0, 0, 1031, 1003, 997, 1036, 7, 0, 0, 0, 1005, 1026, 998, 1053, 0, 0, 0, 0, 1048, 995, 1042, 1035, 0, 0, 0, 0, 1019, 1020, 995, 1027, 30, 22, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4105, 4117, 4040, 4074, 4082, 4120, 4113] avg=4096.0000 max=4120 max_rank=6 min=4040 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1023, 1017, 1011, 1008, 0, 0, 0, 0, 1049, 1040, 1052, 1016, 0, 0, 0, 0, 1032, 1029, 1012, 1000, 15, 6, 2, 0, 1018, 990, 1029, 1013, 0, 0, 0, 0, 1011, 1024, 996, 1065, 7, 0, 0, 0, 1022, 1022, 1020, 990, 0, 0, 0, 0, 1011, 1056, 1034, 1076, 0, 0, 0, 0, 991, 1044, 1001, 983, 31, 22, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4157, 4096, 4050, 4103, 4054, 4177, 4072] avg=4096.0000 max=4177 max_rank=6 min=4050 min_rank=3 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[998, 1004, 1038, 1079, 0, 0, 0, 0, 1003, 1056, 1058, 1012, 0, 0, 0, 0, 1038, 1020, 995, 1012, 15, 6, 1, 0, 987, 967, 1013, 1078, 0, 0, 0, 0, 1004, 1034, 1039, 1033, 7, 0, 0, 0, 1017, 1028, 999, 1024, 0, 0, 0, 0, 993, 990, 986, 1057, 0, 0, 0, 0, 1032, 996, 1057, 1038, 32, 22, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4129, 4087, 4045, 4117, 4068, 4026, 4177] avg=4096.0000 max=4177 max_rank=7 min=4026 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1011, 1046, 1006, 1062, 0, 0, 0, 0, 1029, 985, 1052, 1029, 0, 0, 0, 0, 973, 977, 1073, 1002, 15, 6, 2, 0, 1045, 1063, 1049, 1039, 0, 0, 0, 0, 1014, 1044, 1083, 1004, 7, 0, 0, 0, 990, 1011, 1026, 1024, 0, 0, 0, 0, 994, 1006, 1037, 1049, 0, 0, 0, 0, 972, 968, 1029, 993, 31, 22, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4095, 4048, 4196, 4152, 4051, 4086, 4015] avg=4096.0000 max=4196 max_rank=3 min=4015 min_rank=7 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1083, 1009, 1015, 1024, 0, 0, 0, 0, 1044, 1017, 1008, 1013, 0, 0, 0, 0, 1000, 1044, 990, 1002, 15, 6, 2, 0, 1044, 1035, 1022, 1014, 0, 0, 0, 0, 976, 971, 1026, 1032, 7, 0, 0, 0, 1042, 1018, 1010, 1006, 0, 0, 0, 0, 1006, 1060, 982, 1037, 0, 0, 0, 0, 1073, 991, 1048, 1043, 31, 22, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4082, 4059, 4115, 4012, 4076, 4085, 4208] avg=4096.0000 max=4208 max_rank=7 min=4012 min_rank=4 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1062, 976, 962, 996, 0, 0, 0, 0, 1025, 995, 1007, 978, 0, 0, 0, 0, 1073, 993, 1006, 1015, 21, 10, 5, 0, 1051, 1033, 1031, 978, 0, 0, 0, 0, 1035, 1068, 1037, 1009, 10, 7, 0, 0, 1045, 1050, 1065, 1062, 0, 0, 0, 0, 1079, 1005, 991, 1016, 0, 0, 0, 0, 1003, 995, 1041, 977, 33, 23, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4005, 4123, 4093, 4166, 4222, 4091, 4072] avg=4096.0000 max=4222 max_rank=5 min=3996 min_rank=0 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1052, 1040, 1004, 983, 0, 0, 0, 0, 1017, 1014, 1038, 1029, 0, 0, 0, 0, 1071, 981, 1014, 1049, 15, 6, 2, 0, 1024, 1033, 1022, 1034, 0, 0, 0, 0, 989, 1049, 986, 1063, 7, 0, 0, 0, 1014, 1013, 1071, 1017, 0, 0, 0, 0, 1007, 1032, 1026, 1023, 0, 0, 0, 0, 975, 999, 997, 1019, 31, 22, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4098, 4138, 4113, 4094, 4115, 4088, 4043] avg=4096.0000 max=4138 max_rank=2 min=4043 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8192, 256, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1012, 970, 1049, 1029, 967, 1040, 1023, 1085, 982, 1039, 1031, 1015, 1013, 1050, 1021, 1020, 1028, 1013, 1009, 1085, 1002, 1016, 1020, 1075, 1006, 1017, 1042, 1017, 1029, 1024, 1015, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1026, 1044, 1053, 1047, 1019, 1063, 983, 999, 1013, 1021, 996, 1055, 1008, 1053, 1003, 1002, 1056, 1004, 1004, 1001, 1013, 1071, 1039, 1016, 1006, 1000, 1057, 1019, 1042, 1069, 992, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[972, 1022, 1004, 1047, 1034, 987, 1038, 1040, 1057, 1035, 986, 1032, 1057, 1046, 1026, 1003, 1044, 992, 1011, 1045, 1053, 1084, 1028, 1015, 1017, 1017, 1012, 1016, 1025, 983, 1012, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1037, 992, 1017, 993, 1062, 1027, 997, 1031, 1007, 1042, 1082, 1049, 1048, 1039, 1034, 1027, 1008, 981, 1033, 975, 969, 1017, 1048, 1023, 1052, 1019, 1007, 998, 1058, 1056, 1017, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1007, 986, 1006, 963, 986, 1031, 1034, 1017, 1016, 1059, 1048, 1063, 990, 1090, 1023, 1043, 985, 1025, 1062, 1004, 1038, 997, 1037, 1036, 1042, 1001, 988, 985, 1071, 1021, 1087, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1030, 1036, 1008, 1026, 1019, 1011, 1007, 1025, 1034, 999, 1044, 1086, 989, 994, 1085, 1007, 985, 993, 1013, 1017, 1058, 1012, 1027, 1038, 1018, 1076, 1008, 976, 1076, 1016, 1046, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1031, 998, 1010, 1017, 1035, 980, 1060, 1030, 1036, 1004, 1035, 1044, 996, 1013, 1043, 1022, 1005, 1020, 1021, 1018, 1017, 1032, 1007, 1054, 1040, 1049, 1007, 1067, 1008, 988, 1044, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1043, 1043, 980, 987, 1019, 1009, 987, 1057, 1029, 1038, 997, 1091, 1020, 993, 1055, 1059, 1035, 1057, 1016, 1040, 1040, 1030, 1069, 968, 995, 1041, 998, 1028, 1030, 982, 1026, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[972, 1022, 1004, 1047, 34, 0, 0, 0, 1034, 987, 1038, 1040, 8, 0, 0, 0, 1057, 1035, 986, 997, 0, 0, 0, 0, 1057, 1046, 1014, 1003, 0, 0, 0, 0, 1044, 992, 1011, 1045, 20, 1, 0, 0, 1053, 1084, 1007, 1015, 0, 0, 0, 0, 1017, 1017, 1012, 1016, 12, 3, 1, 0, 1014, 983, 1012, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4107, 4075, 4120, 4113, 4159, 4078, 4037] avg=4096.0000 max=4159 max_rank=5 min=4037 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1012, 970, 1049, 1029, 34, 0, 0, 0, 967, 1040, 1023, 1085, 8, 0, 0, 0, 982, 1039, 1031, 980, 0, 0, 0, 0, 1013, 1050, 1009, 1020, 0, 0, 0, 0, 1028, 1013, 1009, 1085, 20, 1, 0, 0, 1002, 1016, 999, 1075, 0, 0, 0, 0, 1006, 1017, 1042, 1017, 12, 3, 1, 0, 1018, 1024, 1015, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4123, 4032, 4092, 4156, 4092, 4098, 4081] avg=4096.0000 max=4156 max_rank=4 min=4032 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1030, 1036, 1008, 1026, 36, 0, 0, 0, 1019, 1011, 1007, 1025, 8, 0, 0, 0, 1034, 999, 1044, 1049, 0, 0, 0, 0, 989, 994, 1072, 1007, 0, 0, 0, 0, 985, 993, 1013, 1017, 20, 1, 0, 0, 1058, 1012, 1006, 1038, 0, 0, 0, 0, 1018, 1076, 1008, 976, 13, 4, 1, 0, 1064, 1016, 1046, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4070, 4126, 4062, 4029, 4114, 4096, 4135] avg=4096.0000 max=4136 max_rank=0 min=4029 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 1044, 1053, 1047, 35, 0, 0, 0, 1019, 1063, 983, 999, 8, 0, 0, 0, 1013, 1021, 996, 1019, 0, 0, 0, 0, 1008, 1053, 991, 1002, 0, 0, 0, 0, 1056, 1004, 1004, 1001, 21, 1, 0, 0, 1013, 1071, 1017, 1016, 0, 0, 0, 0, 1006, 1000, 1057, 1019, 12, 3, 1, 0, 1031, 1069, 992, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4205, 4072, 4049, 4054, 4087, 4117, 4098, 4086] avg=4096.0000 max=4205 max_rank=0 min=4049 min_rank=2 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1037, 992, 1017, 993, 35, 0, 0, 0, 1062, 1027, 997, 1031, 8, 0, 0, 0, 1007, 1042, 1082, 1013, 0, 0, 0, 0, 1048, 1039, 1022, 1027, 0, 0, 0, 0, 1008, 981, 1033, 975, 21, 1, 0, 0, 969, 1017, 1026, 1023, 0, 0, 0, 0, 1052, 1019, 1007, 998, 12, 3, 1, 0, 1047, 1056, 1017, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4125, 4144, 4136, 4019, 4035, 4092, 4143] avg=4096.0000 max=4144 max_rank=2 min=4019 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1007, 986, 1006, 963, 38, 0, 0, 0, 986, 1031, 1034, 1017, 11, 0, 0, 0, 1016, 1059, 1048, 1018, 0, 0, 0, 0, 990, 1090, 1006, 1043, 0, 0, 0, 0, 985, 1025, 1062, 1004, 26, 7, 0, 0, 1038, 997, 1006, 1036, 0, 0, 0, 0, 1042, 1001, 988, 985, 17, 9, 5, 0, 1051, 1021, 1087, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4000, 4079, 4141, 4129, 4109, 4077, 4047, 4186] avg=4096.0000 max=4186 max_rank=7 min=4000 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1043, 1043, 980, 987, 36, 0, 0, 0, 1019, 1009, 987, 1057, 8, 0, 0, 0, 1029, 1038, 997, 1054, 0, 0, 0, 0, 1020, 993, 1042, 1059, 0, 0, 0, 0, 1035, 1057, 1016, 1040, 21, 1, 0, 0, 1040, 1030, 1047, 968, 0, 0, 0, 0, 995, 1041, 998, 1028, 13, 3, 1, 0, 1019, 982, 1026, 1006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4080, 4118, 4114, 4170, 4085, 4079, 4033] avg=4096.0000 max=4170 max_rank=4 min=4033 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1031, 998, 1010, 1017, 35, 0, 0, 0, 1035, 980, 1060, 1030, 7, 0, 0, 0, 1036, 1004, 1035, 1008, 0, 0, 0, 0, 996, 1013, 1030, 1022, 0, 0, 0, 0, 1005, 1020, 1021, 1018, 20, 1, 0, 0, 1017, 1032, 986, 1054, 0, 0, 0, 0, 1040, 1049, 1007, 1067, 13, 3, 1, 0, 998, 988, 1044, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4112, 4083, 4061, 4085, 4089, 4180, 4067] avg=4096.0000 max=4180 max_rank=6 min=4061 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1046, 1038, 1031, 987, 1018, 1042, 1012, 1026, 1052, 1060, 1044, 991, 1032, 1028, 1032, 1063, 1014, 998, 1030, 972, 1004, 1031, 981, 1010, 1068, 1033, 1017, 1048, 992, 1007, 1022, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1018, 997, 1042, 952, 1007, 1041, 1020, 987, 1009, 1064, 1053, 1024, 982, 999, 1059, 1024, 985, 1037, 1010, 1050, 1000, 1023, 1080, 1065, 1037, 1001, 1045, 1043, 1017, 1015, 1046, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1021, 1033, 1033, 982, 1038, 1045, 1036, 1038, 1051, 1001, 1041, 1028, 1001, 1016, 1078, 1035, 1009, 1010, 993, 1059, 972, 1045, 1020, 1025, 1035, 993, 996, 1069, 1042, 1023, 996, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1029, 1054, 1014, 991, 1026, 1061, 1028, 1042, 989, 1015, 1038, 1015, 1018, 1025, 1014, 1011, 980, 1012, 997, 1051, 1031, 1016, 1035, 1032, 1054, 1032, 1038, 1011, 1014, 1018, 1035, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1015, 1044, 1067, 970, 1035, 1019, 1064, 1003, 1063, 1037, 991, 1004, 1007, 993, 1039, 972, 1048, 1030, 1014, 1022, 1031, 1003, 1033, 1035, 1058, 1006, 987, 1014, 1052, 1043, 1056, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1066, 1013, 1012, 1015, 1022, 1051, 1014, 1001, 1032, 1037, 1007, 1071, 1032, 1019, 1030, 1047, 1050, 1046, 989, 1019, 995, 1032, 1019, 1049, 998, 982, 1017, 1004, 1017, 999, 1069, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1011, 1022, 967, 1029, 1025, 980, 1010, 1038, 1004, 988, 1059, 1001, 1018, 1018, 1040, 1008, 961, 1044, 996, 1061, 1011, 1012, 1070, 1033, 1042, 1012, 1023, 1059, 1058, 993, 1089, 1086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[982, 1038, 1038, 987, 1001, 1069, 1017, 1081, 1036, 1021, 1043, 1038, 975, 1028, 1031, 947, 1022, 1050, 1038, 972, 1041, 1029, 987, 1092, 1008, 1032, 1052, 1050, 975, 983, 1048, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1029, 1054, 1014, 991, 17, 10, 0, 0, 1026, 1045, 1028, 1042, 0, 0, 0, 0, 989, 1015, 1021, 1015, 0, 0, 0, 0, 1018, 1025, 1014, 1011, 9, 9, 0, 0, 980, 1012, 997, 1051, 16, 5, 2, 0, 1031, 1016, 1035, 1023, 0, 0, 0, 0, 1043, 1032, 1038, 1011, 0, 0, 0, 0, 1014, 1018, 1020, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4141, 4040, 4086, 4063, 4105, 4124, 4094] avg=4096.0000 max=4141 max_rank=1 min=4040 min_rank=2 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1046, 1038, 1031, 987, 17, 10, 0, 0, 1018, 1026, 1012, 1026, 0, 0, 0, 0, 1052, 1060, 1027, 991, 0, 0, 0, 0, 1032, 1028, 1032, 1063, 8, 9, 0, 0, 1014, 998, 1030, 972, 16, 5, 2, 0, 1004, 1031, 981, 1002, 0, 0, 0, 0, 1057, 1033, 1017, 1048, 0, 0, 0, 0, 992, 1007, 1007, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4082, 4130, 4172, 4037, 4018, 4155, 4045] avg=4096.0000 max=4172 max_rank=3 min=4018 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16255 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1021, 1033, 1033, 982, 17, 10, 0, 0, 1038, 1029, 1036, 1038, 0, 0, 0, 0, 1051, 1001, 1024, 1028, 0, 0, 0, 0, 1001, 1016, 1078, 1035, 9, 9, 0, 0, 1009, 1010, 993, 1059, 16, 5, 2, 0, 972, 1045, 1020, 1016, 0, 0, 0, 0, 1024, 993, 996, 1069, 0, 0, 0, 0, 1042, 1023, 981, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4141, 4104, 4148, 4094, 4053, 4082, 4050] avg=4096.0000 max=4148 max_rank=3 min=4050 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1018, 997, 1042, 952, 17, 10, 0, 0, 1007, 1025, 1020, 987, 0, 0, 0, 0, 1009, 1064, 1036, 1024, 0, 0, 0, 0, 982, 999, 1059, 1024, 9, 9, 0, 0, 985, 1037, 1010, 1050, 16, 5, 2, 0, 1000, 1023, 1080, 1056, 0, 0, 0, 0, 1026, 1001, 1045, 1043, 0, 0, 0, 0, 1017, 1015, 1031, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4039, 4133, 4082, 4105, 4159, 4115, 4099] avg=4096.0000 max=4159 max_rank=5 min=4036 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1015, 1044, 1067, 970, 16, 10, 0, 0, 1035, 1004, 1064, 1003, 0, 0, 0, 0, 1063, 1037, 975, 1004, 0, 0, 0, 0, 1007, 993, 1039, 972, 9, 9, 0, 0, 1048, 1030, 1014, 1022, 15, 5, 2, 0, 1031, 1003, 1033, 1026, 0, 0, 0, 0, 1047, 1006, 987, 1014, 0, 0, 0, 0, 1052, 1043, 1041, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4106, 4079, 4029, 4136, 4093, 4054, 4149] avg=4096.0000 max=4149 max_rank=7 min=4029 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1066, 1013, 1012, 1015, 16, 10, 0, 0, 1022, 1035, 1014, 1001, 0, 0, 0, 0, 1032, 1037, 991, 1071, 0, 0, 0, 0, 1032, 1019, 1030, 1047, 9, 8, 0, 0, 1050, 1046, 989, 1019, 16, 6, 2, 0, 995, 1032, 1019, 1040, 0, 0, 0, 0, 988, 982, 1017, 1004, 0, 0, 0, 0, 1017, 999, 1053, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4072, 4131, 4145, 4128, 4086, 3991, 4083] avg=4096.0000 max=4145 max_rank=3 min=3991 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16288 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[982, 1038, 1038, 987, 22, 14, 0, 0, 1001, 1050, 1017, 1081, 0, 0, 0, 0, 1036, 1021, 1021, 1038, 0, 0, 0, 0, 975, 1028, 1031, 947, 12, 11, 0, 0, 1022, 1050, 1038, 972, 19, 10, 9, 0, 1041, 1029, 987, 1080, 0, 0, 0, 0, 988, 1032, 1052, 1050, 0, 0, 0, 0, 975, 983, 1024, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4149, 4116, 4004, 4120, 4137, 4122, 4039] avg=4096.0000 max=4149 max_rank=1 min=4004 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1011, 1022, 967, 1029, 17, 11, 0, 0, 1025, 965, 1010, 1038, 0, 0, 0, 0, 1004, 988, 1042, 1001, 0, 0, 0, 0, 1018, 1018, 1040, 1008, 9, 9, 0, 0, 961, 1044, 996, 1061, 15, 6, 2, 0, 1011, 1012, 1070, 1024, 0, 0, 0, 0, 1031, 1012, 1023, 1059, 0, 0, 0, 0, 1058, 993, 1072, 1086, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4038, 4035, 4102, 4085, 4117, 4125, 4209] avg=4096.0000 max=4209 max_rank=7 min=4035 min_rank=2 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16536 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 7936, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8448, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1016, 954, 1009, 999, 1027, 959, 1068, 1019, 993, 1066, 1086, 1049, 1037, 994, 996, 1031, 987, 985, 1025, 1072, 1048, 1057, 1000, 1072, 1016, 1084, 961, 1073, 1016, 1045, 1012, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[982, 1009, 1068, 1049, 1056, 1050, 1048, 1062, 1007, 1006, 1063, 1016, 1016, 1034, 1026, 998, 967, 1030, 1038, 1033, 1055, 993, 1047, 1000, 1041, 1003, 993, 1040, 1002, 988, 1051, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1040, 1027, 1003, 1012, 1018, 1010, 993, 971, 1037, 1049, 1039, 996, 1025, 1027, 1023, 1022, 1049, 1017, 1022, 948, 1050, 1041, 1035, 1018, 1037, 1076, 1062, 993, 980, 1044, 1063, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1020, 1044, 1015, 1010, 1020, 1037, 1083, 1007, 1049, 1026, 1050, 1034, 1044, 1056, 1021, 1041, 1026, 1003, 974, 1011, 1025, 1076, 1025, 934, 1017, 1024, 990, 1049, 978, 1013, 1060, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1023, 992, 1014, 1045, 1016, 1026, 995, 1031, 1027, 1070, 1006, 966, 1032, 1016, 1008, 1043, 1048, 1029, 1006, 1054, 1052, 1012, 1085, 1038, 1038, 1010, 1012, 1015, 1044, 1008, 981, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[974, 1087, 1028, 1045, 970, 1019, 1010, 1052, 1008, 1010, 1082, 1069, 1011, 1023, 982, 1005, 1037, 1024, 1025, 1006, 999, 992, 1088, 967, 1001, 1060, 1097, 1097, 989, 992, 1017, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[984, 1004, 1032, 1049, 993, 1021, 1023, 1000, 1022, 998, 1012, 980, 1037, 1001, 1080, 1002, 1015, 1012, 993, 1059, 1064, 1062, 1053, 1047, 1067, 1005, 1060, 1041, 998, 1045, 986, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1022, 1026, 1010, 1065, 1025, 1010, 1055, 1003, 1024, 1081, 1004, 1039, 1031, 1039, 1075, 996, 1060, 1064, 993, 1031, 982, 992, 984, 1028, 1070, 989, 1010, 1036, 1053, 963, 1015, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1016, 954, 1009, 999, 7, 5, 0, 0, 1027, 959, 1068, 1019, 10, 0, 0, 0, 993, 1066, 1062, 1049, 0, 0, 0, 0, 1037, 994, 996, 1031, 0, 0, 0, 0, 987, 985, 1025, 1072, 16, 0, 0, 0, 1048, 1057, 983, 1072, 0, 0, 0, 0, 1016, 1084, 961, 1035, 0, 0, 0, 0, 1016, 1045, 1012, 1012, 38, 3, 0, 0] rerouted_tokens_per_ep_rank=[3990, 4083, 4170, 4058, 4085, 4160, 4096, 4126] avg=4096.0000 max=4170 max_rank=2 min=3990 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1040, 1027, 1003, 1012, 8, 5, 0, 0, 1018, 1010, 993, 971, 10, 0, 0, 0, 1037, 1049, 1016, 996, 0, 0, 0, 0, 1025, 1027, 1023, 1022, 0, 0, 0, 0, 1049, 1017, 1022, 948, 15, 0, 0, 0, 1050, 1041, 1017, 1018, 0, 0, 0, 0, 1037, 1076, 1062, 958, 0, 0, 0, 0, 980, 1044, 1063, 1041, 35, 3, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4002, 4098, 4097, 4051, 4126, 4133, 4166] avg=4096.0000 max=4166 max_rank=7 min=4002 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1020, 1044, 1015, 1010, 8, 5, 0, 0, 1020, 1037, 1083, 1007, 10, 0, 0, 0, 1049, 1026, 1027, 1034, 0, 0, 0, 0, 1044, 1056, 1021, 1041, 0, 0, 0, 0, 1026, 1003, 974, 1011, 15, 0, 0, 0, 1025, 1076, 1007, 934, 0, 0, 0, 0, 1017, 1024, 990, 1012, 0, 0, 0, 0, 978, 1013, 1060, 1006, 37, 3, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4157, 4136, 4162, 4029, 4042, 4043, 4097] avg=4096.0000 max=4162 max_rank=3 min=4029 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16211 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[974, 1087, 1028, 1045, 8, 5, 0, 0, 970, 1019, 1010, 1052, 11, 0, 0, 0, 1008, 1010, 1058, 1069, 0, 0, 0, 0, 1011, 1023, 982, 1005, 0, 0, 0, 0, 1037, 1024, 1025, 1006, 16, 0, 0, 0, 999, 992, 1069, 967, 0, 0, 0, 0, 1001, 1060, 1097, 1058, 0, 0, 0, 0, 989, 992, 1017, 1002, 39, 3, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4062, 4145, 4021, 4108, 4027, 4216, 4042] avg=4096.0000 max=4216 max_rank=6 min=4021 min_rank=3 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[982, 1009, 1068, 1049, 12, 10, 0, 0, 1056, 1050, 1048, 1062, 14, 4, 0, 0, 1007, 1006, 1030, 1016, 0, 0, 0, 0, 1012, 1034, 1026, 998, 0, 0, 0, 0, 967, 1030, 1038, 1033, 18, 0, 0, 0, 1055, 993, 1021, 1000, 0, 0, 0, 0, 1041, 1003, 993, 1000, 0, 0, 0, 0, 1002, 988, 1051, 997, 40, 5, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4234, 4059, 4070, 4086, 4069, 4037, 4083] avg=4096.0000 max=4234 max_rank=1 min=4037 min_rank=6 imbalance_ratio=1.033691 max_load_reduction=28534 max_load_reduction_pct=0.870789 moved_to_echo=16275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1023, 992, 1014, 1045, 8, 5, 0, 0, 1016, 1026, 995, 1031, 11, 0, 0, 0, 1027, 1070, 983, 966, 0, 0, 0, 0, 1032, 1016, 1008, 1043, 0, 0, 0, 0, 1048, 1029, 1006, 1054, 15, 0, 0, 0, 1052, 1012, 1066, 1038, 0, 0, 0, 0, 1038, 1010, 1012, 979, 0, 0, 0, 0, 1044, 1008, 981, 1026, 36, 3, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4079, 4046, 4099, 4152, 4168, 4039, 4098] avg=4096.0000 max=4168 max_rank=5 min=4039 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[984, 1004, 1032, 1049, 8, 5, 0, 0, 993, 1021, 1023, 1000, 11, 0, 0, 0, 1022, 998, 989, 980, 0, 0, 0, 0, 1037, 1001, 1080, 1002, 0, 0, 0, 0, 1015, 1012, 993, 1059, 15, 0, 0, 0, 1064, 1062, 1034, 1047, 0, 0, 0, 0, 1067, 1005, 1060, 1004, 0, 0, 0, 0, 998, 1045, 986, 1023, 37, 3, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4048, 3989, 4120, 4094, 4207, 4136, 4092] avg=4096.0000 max=4207 max_rank=5 min=3989 min_rank=2 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16529 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1022, 1026, 1010, 1065, 7, 5, 0, 0, 1025, 1010, 1055, 1003, 10, 0, 0, 0, 1024, 1081, 981, 1039, 0, 0, 0, 0, 1031, 1039, 1075, 996, 0, 0, 0, 0, 1060, 1064, 993, 1031, 15, 0, 0, 0, 982, 992, 967, 1028, 0, 0, 0, 0, 1070, 989, 1010, 999, 0, 0, 0, 0, 1053, 963, 1015, 993, 37, 3, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4103, 4125, 4141, 4163, 3969, 4068, 4064] avg=4096.0000 max=4163 max_rank=4 min=3969 min_rank=5 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16264 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8320, 8192, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1019, 1085, 1059, 1022, 994, 1004, 1044, 995, 992, 1032, 1030, 950, 1011, 1025, 1009, 1061, 1003, 1081, 967, 1009, 996, 1025, 1081, 1079, 1018, 1012, 1034, 1029, 1043, 1006, 1027, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1032, 983, 1012, 1053, 1030, 1042, 1061, 1001, 1036, 1039, 1027, 1016, 1035, 1006, 1041, 1006, 1006, 1073, 1058, 1022, 1050, 1028, 989, 971, 1010, 1007, 1028, 1035, 1028, 1020, 985, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1053, 1002, 1038, 1028, 994, 1064, 1031, 1006, 1025, 989, 1006, 993, 969, 1044, 1051, 1027, 1041, 1085, 984, 1036, 1040, 1033, 1031, 990, 1059, 1018, 1007, 1032, 1017, 1000, 1039, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1058, 994, 1034, 1033, 1080, 1000, 994, 968, 1033, 1035, 1009, 994, 1036, 1025, 982, 1028, 1031, 1056, 992, 1022, 1004, 1014, 1023, 1005, 1011, 1025, 1025, 1054, 1085, 1105, 1027, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1044, 1012, 1011, 975, 968, 1012, 1011, 1023, 1046, 1032, 1029, 998, 1104, 1020, 1003, 990, 1019, 1014, 1033, 980, 974, 1038, 1043, 1035, 1040, 1024, 1048, 1041, 1030, 1062, 1038, 1071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1042, 1040, 982, 1040, 999, 1024, 1025, 1030, 1017, 1067, 1051, 1032, 1033, 1043, 1063, 1017, 1017, 1048, 1026, 978, 1102, 1026, 1028, 1020, 1025, 1006, 972, 1009, 1013, 1005, 982, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[988, 975, 1001, 1101, 1010, 992, 987, 996, 1038, 1038, 986, 1045, 1072, 1038, 1048, 1078, 982, 1032, 1045, 987, 1067, 991, 1038, 1051, 1003, 1061, 1012, 987, 1044, 1014, 1027, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1022, 1036, 1039, 1009, 973, 1020, 980, 1044, 1055, 1059, 1008, 1013, 1041, 1064, 1033, 1028, 986, 982, 994, 1035, 1038, 1047, 995, 965, 1052, 1031, 1030, 1022, 1039, 1053, 1053, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1019, 1085, 1059, 1016, 0, 0, 0, 0, 994, 1004, 1044, 995, 32, 12, 0, 0, 992, 1032, 1030, 950, 5, 0, 0, 0, 979, 1025, 1009, 1061, 0, 0, 0, 0, 1003, 1081, 967, 1009, 11, 6, 0, 0, 991, 1025, 1081, 1079, 0, 0, 0, 0, 1018, 1012, 1034, 1029, 0, 0, 0, 0, 1020, 1006, 1027, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4081, 4009, 4074, 4077, 4176, 4093, 4079] avg=4096.0000 max=4179 max_rank=0 min=4009 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16425 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1032, 983, 1012, 1047, 0, 0, 0, 0, 1030, 1042, 1061, 1001, 32, 12, 0, 0, 1036, 1039, 1027, 1016, 6, 0, 0, 0, 1003, 1006, 1041, 1006, 0, 0, 0, 0, 1006, 1073, 1058, 1022, 11, 6, 0, 0, 1044, 1028, 989, 971, 0, 0, 0, 0, 1010, 1007, 1028, 1035, 0, 0, 0, 0, 1005, 1020, 985, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4178, 4124, 4056, 4176, 4032, 4080, 4048] avg=4096.0000 max=4178 max_rank=1 min=4032 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16336 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[988, 975, 1001, 1094, 0, 0, 0, 0, 1010, 992, 987, 996, 33, 12, 0, 0, 1038, 1038, 986, 1045, 6, 0, 0, 0, 1039, 1038, 1048, 1078, 0, 0, 0, 0, 982, 1032, 1045, 987, 11, 7, 0, 0, 1061, 991, 1038, 1051, 0, 0, 0, 0, 1003, 1061, 1012, 987, 0, 0, 0, 0, 1021, 1014, 1027, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4030, 4113, 4203, 4064, 4141, 4063, 4096] avg=4096.0000 max=4203 max_rank=3 min=4030 min_rank=1 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16364 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1053, 1002, 1038, 1022, 0, 0, 0, 0, 994, 1064, 1031, 1006, 30, 12, 0, 0, 1025, 989, 1006, 993, 6, 0, 0, 0, 939, 1044, 1051, 1027, 0, 0, 0, 0, 1041, 1085, 984, 1036, 11, 6, 0, 0, 1034, 1033, 1031, 990, 0, 0, 0, 0, 1059, 1018, 1007, 1032, 0, 0, 0, 0, 994, 1000, 1039, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4137, 4019, 4061, 4163, 4088, 4116, 4069] avg=4096.0000 max=4163 max_rank=4 min=4019 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16436 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1044, 1012, 1011, 969, 0, 0, 0, 0, 968, 1012, 1011, 1023, 34, 12, 0, 0, 1046, 1032, 1029, 998, 5, 0, 0, 0, 1070, 1020, 1003, 990, 0, 0, 0, 0, 1019, 1014, 1033, 980, 11, 6, 0, 0, 969, 1038, 1043, 1035, 0, 0, 0, 0, 1040, 1024, 1048, 1041, 0, 0, 0, 0, 1007, 1062, 1038, 1071, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4060, 4110, 4083, 4063, 4085, 4153, 4178] avg=4096.0000 max=4178 max_rank=7 min=4036 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16479 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1058, 994, 1034, 1027, 0, 0, 0, 0, 1080, 1000, 994, 968, 32, 13, 0, 0, 1033, 1035, 1009, 994, 5, 0, 0, 0, 1004, 1025, 982, 1028, 0, 0, 0, 0, 1031, 1056, 992, 1022, 11, 6, 0, 0, 999, 1014, 1023, 1005, 0, 0, 0, 0, 1011, 1025, 1025, 1054, 0, 0, 0, 0, 1061, 1105, 1027, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4087, 4076, 4039, 4118, 4041, 4115, 4179] avg=4096.0000 max=4179 max_rank=7 min=4039 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16453 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1042, 1040, 982, 1029, 0, 0, 0, 0, 999, 1024, 1025, 1030, 38, 18, 0, 0, 1017, 1067, 1051, 1032, 9, 0, 0, 0, 995, 1043, 1063, 1017, 0, 0, 0, 0, 1017, 1048, 1026, 978, 13, 11, 0, 0, 1092, 1026, 1028, 1020, 0, 0, 0, 0, 1025, 1006, 972, 1009, 1, 0, 0, 0, 982, 1005, 982, 1006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4134, 4176, 4118, 4093, 4166, 4013, 3975] avg=4096.0000 max=4176 max_rank=2 min=3975 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16247 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1022, 1036, 1039, 1003, 0, 0, 0, 0, 973, 1020, 980, 1044, 32, 12, 0, 0, 1055, 1059, 1008, 1013, 6, 0, 0, 0, 1009, 1064, 1033, 1028, 0, 0, 0, 0, 986, 982, 994, 1035, 11, 6, 0, 0, 1032, 1047, 995, 965, 0, 0, 0, 0, 1052, 1031, 1030, 1022, 0, 0, 0, 0, 1016, 1053, 1053, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4061, 4141, 4134, 4014, 4039, 4135, 4144] avg=4096.0000 max=4144 max_rank=7 min=4014 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16332 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8064, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8448, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1017, 999, 1028, 1013, 1047, 1057, 1012, 969, 1010, 995, 1034, 1026, 1053, 1015, 1020, 1073, 1045, 1040, 1010, 1006, 1035, 1006, 966, 1014, 1036, 971, 1090, 1081, 1014, 1039, 999, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1000, 987, 1037, 1032, 1060, 1019, 1055, 1022, 1009, 1047, 1003, 1033, 1019, 988, 1017, 1096, 1045, 1021, 1096, 983, 1024, 1060, 1026, 984, 1028, 1029, 980, 1025, 1013, 1023, 1008, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1043, 1021, 1036, 1010, 1026, 1033, 961, 945, 1020, 1014, 1016, 1030, 1038, 1053, 1059, 1033, 1024, 1093, 1048, 1047, 1033, 1008, 951, 1031, 1024, 997, 1061, 1035, 959, 1013, 1056, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[986, 1031, 1014, 999, 1008, 1021, 1044, 1038, 1037, 992, 1053, 959, 977, 1017, 990, 1015, 1018, 990, 1057, 1038, 993, 991, 1107, 1066, 1018, 1030, 1068, 1010, 1041, 1031, 1098, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[994, 999, 1059, 1055, 1014, 1046, 1024, 1006, 1039, 1013, 1041, 972, 1035, 1028, 973, 1008, 1049, 1018, 991, 1076, 1004, 1056, 1029, 1027, 989, 1025, 1049, 1033, 1004, 1019, 1071, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1045, 1060, 1037, 1028, 1000, 988, 1054, 985, 1025, 1050, 1073, 979, 1036, 1024, 951, 1053, 1010, 1053, 1031, 1000, 1020, 1032, 1049, 1004, 1030, 1033, 1045, 1007, 974, 1051, 1059, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1029, 1065, 1013, 1000, 1026, 1014, 1049, 996, 991, 1028, 1066, 1030, 1041, 991, 1060, 990, 1051, 1070, 1032, 997, 1031, 1015, 1017, 1014, 1025, 1029, 961, 1050, 1024, 1012, 1015, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[973, 1032, 1003, 1046, 1002, 1050, 1036, 974, 1026, 1023, 982, 1059, 1022, 993, 1030, 1004, 1017, 1033, 980, 1034, 1013, 1052, 1049, 1062, 961, 1044, 1054, 1057, 1050, 1031, 1053, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1017, 999, 1028, 1013, 8, 1, 0, 0, 1047, 1057, 1012, 969, 23, 0, 0, 0, 1010, 995, 1034, 1026, 6, 5, 0, 0, 1053, 1015, 1020, 1073, 7, 0, 0, 0, 1045, 1011, 1010, 1006, 0, 0, 0, 0, 1035, 1006, 966, 1014, 0, 0, 0, 0, 1036, 971, 1077, 1081, 0, 0, 0, 0, 1014, 1039, 991, 1048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4108, 4076, 4168, 4072, 4021, 4165, 4092] avg=4096.0000 max=4168 max_rank=3 min=4021 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1000, 987, 1037, 1032, 7, 1, 0, 0, 1060, 1019, 1055, 1022, 22, 0, 0, 0, 1009, 1047, 1003, 1033, 5, 5, 0, 0, 1019, 988, 1017, 1096, 7, 0, 0, 0, 1045, 994, 1096, 983, 0, 0, 0, 0, 1024, 1060, 1026, 984, 0, 0, 0, 0, 1028, 1029, 968, 1025, 0, 0, 0, 0, 1013, 1023, 1000, 999, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4178, 4102, 4127, 4118, 4094, 4050, 4035] avg=4096.0000 max=4178 max_rank=1 min=4035 min_rank=7 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16297 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1043, 1021, 1036, 1010, 7, 1, 0, 0, 1026, 1033, 961, 945, 24, 0, 0, 0, 1020, 1014, 1016, 1030, 6, 5, 0, 0, 1038, 1053, 1059, 1033, 8, 0, 0, 0, 1024, 1063, 1048, 1047, 0, 0, 0, 0, 1033, 1008, 951, 1031, 0, 0, 0, 0, 1024, 997, 1049, 1035, 0, 0, 0, 0, 959, 1013, 1047, 1050, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 3989, 4091, 4191, 4182, 4023, 4105, 4069] avg=4096.0000 max=4191 max_rank=3 min=3989 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[986, 1031, 1014, 999, 7, 1, 0, 0, 1008, 1021, 1044, 1038, 22, 0, 0, 0, 1037, 992, 1053, 959, 5, 5, 0, 0, 977, 1017, 990, 1015, 8, 0, 0, 0, 1018, 963, 1057, 1038, 0, 0, 0, 0, 993, 991, 1107, 1066, 0, 0, 0, 0, 1018, 1030, 1056, 1010, 0, 0, 0, 0, 1041, 1031, 1089, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4133, 4051, 4007, 4076, 4157, 4114, 4192] avg=4096.0000 max=4192 max_rank=7 min=4007 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1045, 1060, 1037, 1028, 7, 1, 0, 0, 1000, 988, 1054, 985, 23, 0, 0, 0, 1025, 1050, 1073, 979, 6, 5, 0, 0, 1036, 1024, 951, 1053, 8, 0, 0, 0, 1010, 1024, 1031, 1000, 0, 0, 0, 0, 1020, 1032, 1049, 1004, 0, 0, 0, 0, 1030, 1033, 1033, 1007, 0, 0, 0, 0, 974, 1051, 1050, 982, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4050, 4138, 4072, 4065, 4105, 4103, 4057] avg=4096.0000 max=4178 max_rank=0 min=4050 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[994, 999, 1059, 1055, 12, 8, 0, 0, 1014, 1046, 1024, 1006, 26, 0, 0, 0, 1039, 1013, 1041, 972, 9, 10, 0, 0, 1035, 1028, 973, 1008, 12, 1, 0, 0, 1049, 983, 991, 1076, 0, 0, 0, 0, 1004, 1055, 1029, 1027, 0, 0, 0, 0, 989, 1025, 1027, 1033, 0, 0, 0, 0, 1004, 1019, 1051, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4116, 4084, 4057, 4099, 4115, 4074, 4096] avg=4096.0000 max=4127 max_rank=0 min=4057 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[973, 1032, 1003, 1046, 7, 1, 0, 0, 1002, 1050, 1036, 974, 23, 0, 0, 0, 1026, 1023, 982, 1059, 5, 5, 0, 0, 1022, 993, 1030, 1004, 8, 0, 0, 0, 1017, 1005, 980, 1034, 0, 0, 0, 0, 1013, 1052, 1049, 1062, 0, 0, 0, 0, 961, 1044, 1042, 1057, 0, 0, 0, 0, 1050, 1031, 1044, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4085, 4100, 4057, 4036, 4176, 4104, 4148] avg=4096.0000 max=4176 max_rank=5 min=4036 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1029, 1065, 1013, 1000, 7, 1, 0, 0, 1026, 1014, 1049, 996, 24, 0, 0, 0, 991, 1028, 1066, 1030, 6, 5, 0, 0, 1041, 991, 1060, 990, 7, 0, 0, 0, 1051, 1040, 1032, 997, 0, 0, 0, 0, 1031, 1015, 1017, 1014, 0, 0, 0, 0, 1025, 1029, 949, 1050, 0, 0, 0, 0, 1024, 1012, 1007, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4109, 4126, 4089, 4120, 4077, 4053, 4079] avg=4096.0000 max=4126 max_rank=2 min=4053 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 7936, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1023, 1058, 966, 1030, 988, 1022, 1037, 1026, 1007, 1069, 1022, 1030, 1017, 1008, 1045, 1034, 1027, 1077, 1065, 1066, 1070, 997, 991, 1030, 1042, 971, 989, 1004, 1013, 1003, 1005, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1046, 1017, 983, 1039, 970, 1039, 1046, 1017, 1040, 1036, 1021, 1015, 1034, 992, 1057, 996, 1023, 1009, 1064, 1014, 1034, 1022, 1010, 1023, 1009, 1058, 1010, 1010, 1031, 1034, 1046, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1041, 1013, 1026, 1070, 1026, 1032, 1011, 1003, 1022, 1054, 950, 1049, 1012, 1014, 1033, 1005, 1061, 1017, 1036, 1029, 1048, 1043, 1059, 978, 1007, 976, 1009, 961, 994, 1093, 1023, 1073] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[978, 1048, 1052, 944, 1043, 994, 978, 996, 1073, 1040, 1045, 1089, 1038, 1039, 1032, 1032, 1035, 1019, 1041, 1052, 1030, 1025, 1046, 1065, 1027, 981, 1017, 1011, 1016, 996, 1032, 954] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1027, 994, 999, 1015, 1055, 1013, 1023, 1020, 989, 1019, 1030, 996, 1036, 1028, 1023, 1026, 994, 1040, 1029, 1072, 1002, 1044, 1009, 1053, 1052, 1010, 1002, 1096, 1047, 1015, 966, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[989, 1064, 971, 1068, 1025, 1032, 1016, 1028, 1001, 997, 1014, 1043, 1026, 1009, 954, 1033, 1012, 1038, 999, 1038, 1046, 1038, 1049, 1056, 1002, 1041, 982, 1037, 1076, 1012, 1036, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1031, 980, 1072, 1035, 1013, 1046, 1044, 981, 1052, 1003, 1022, 1042, 1010, 1057, 1023, 1010, 996, 1029, 1028, 1034, 1067, 1032, 1018, 1031, 995, 975, 1032, 1036, 1032, 1030, 968, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1010, 1016, 1036, 1037, 1029, 1021, 1061, 1064, 1042, 1067, 1041, 984, 1053, 1041, 1025, 1013, 1005, 975, 1067, 1040, 989, 977, 993, 997, 1031, 1028, 993, 1017, 1042, 1046, 1035, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1046, 1017, 983, 1039, 5, 5, 0, 0, 970, 1039, 1046, 1017, 6, 1, 0, 0, 1040, 1020, 1021, 1015, 0, 0, 0, 0, 1034, 992, 1057, 996, 1, 0, 0, 0, 1023, 1009, 1064, 983, 0, 0, 0, 0, 1023, 1022, 1010, 1023, 0, 0, 0, 0, 1009, 1058, 1010, 1010, 31, 11, 0, 0, 1029, 1034, 1046, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4079, 4096, 4080, 4079, 4078, 4129, 4132] avg=4096.0000 max=4132 max_rank=7 min=4078 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1023, 1058, 966, 1030, 10, 7, 0, 0, 988, 1022, 1037, 1026, 11, 6, 0, 0, 1007, 1046, 1022, 1030, 0, 0, 0, 0, 1017, 1008, 1045, 1034, 6, 0, 0, 0, 1027, 1077, 1065, 1028, 0, 0, 0, 0, 1049, 997, 991, 1030, 0, 0, 0, 0, 1042, 971, 989, 1004, 38, 16, 0, 0, 1001, 1003, 1005, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4090, 4105, 4110, 4197, 4067, 4060, 4045] avg=4096.0000 max=4197 max_rank=4 min=4045 min_rank=7 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[978, 1048, 1052, 944, 5, 5, 0, 0, 1043, 994, 978, 996, 6, 1, 0, 0, 1073, 1024, 1045, 1089, 0, 0, 0, 0, 1038, 1039, 1032, 1032, 1, 0, 0, 0, 1035, 1019, 1041, 1019, 0, 0, 0, 0, 1019, 1025, 1046, 1065, 0, 0, 0, 0, 1027, 981, 1017, 1011, 33, 11, 0, 0, 1014, 996, 1032, 954, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4018, 4231, 4142, 4114, 4155, 4080, 3996] avg=4096.0000 max=4231 max_rank=2 min=3996 min_rank=7 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1027, 994, 999, 1015, 5, 5, 0, 0, 1055, 1013, 1023, 1020, 6, 1, 0, 0, 989, 1003, 1030, 996, 0, 0, 0, 0, 1036, 1028, 1023, 1026, 1, 0, 0, 0, 994, 1040, 1029, 1039, 0, 0, 0, 0, 991, 1044, 1009, 1053, 0, 0, 0, 0, 1052, 1010, 1002, 1096, 33, 11, 0, 0, 1045, 1015, 966, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4118, 4018, 4114, 4102, 4097, 4204, 4070] avg=4096.0000 max=4204 max_rank=6 min=4018 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1041, 1013, 1026, 1070, 6, 5, 0, 0, 1026, 1032, 1011, 1003, 7, 1, 0, 0, 1022, 1038, 950, 1049, 0, 0, 0, 0, 1012, 1014, 1033, 1005, 1, 0, 0, 0, 1061, 1017, 1036, 997, 0, 0, 0, 0, 1035, 1043, 1059, 978, 0, 0, 0, 0, 1007, 976, 1009, 961, 32, 11, 0, 0, 992, 1093, 1023, 1073, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4080, 4059, 4065, 4111, 4115, 3996, 4181] avg=4096.0000 max=4181 max_rank=7 min=3996 min_rank=6 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[989, 1064, 971, 1068, 6, 5, 0, 0, 1025, 1032, 1016, 1028, 7, 1, 0, 0, 1001, 981, 1014, 1043, 0, 0, 0, 0, 1026, 1009, 954, 1033, 1, 0, 0, 0, 1012, 1038, 999, 1006, 0, 0, 0, 0, 1033, 1038, 1049, 1056, 0, 0, 0, 0, 1002, 1041, 982, 1037, 32, 11, 0, 0, 1074, 1012, 1036, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4109, 4039, 4023, 4055, 4176, 4105, 4158] avg=4096.0000 max=4176 max_rank=5 min=4023 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1031, 980, 1072, 1035, 6, 5, 0, 0, 1013, 1046, 1044, 981, 7, 1, 0, 0, 1052, 987, 1022, 1042, 0, 0, 0, 0, 1010, 1057, 1023, 1010, 1, 0, 0, 0, 996, 1029, 1028, 1002, 0, 0, 0, 0, 1054, 1032, 1018, 1031, 0, 0, 0, 0, 995, 975, 1032, 1036, 32, 11, 0, 0, 1030, 1030, 968, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4092, 4103, 4101, 4055, 4135, 4081, 4072] avg=4096.0000 max=4135 max_rank=5 min=4055 min_rank=4 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1010, 1016, 1036, 1037, 5, 5, 0, 0, 1029, 1021, 1061, 1064, 6, 1, 0, 0, 1042, 1050, 1041, 984, 0, 0, 0, 0, 1053, 1041, 1025, 1013, 1, 0, 0, 0, 1005, 975, 1067, 1008, 0, 0, 0, 0, 978, 977, 993, 997, 0, 0, 0, 0, 1031, 1028, 993, 1017, 32, 12, 0, 0, 1040, 1046, 1035, 993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4182, 4117, 4133, 4055, 3945, 4113, 4114] avg=4096.0000 max=4182 max_rank=1 min=3945 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16227 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8064, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8448, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1036, 1030, 1039, 993, 1033, 1024, 1010, 1024, 1028, 1008, 1017, 1019, 1024, 1051, 1064, 1018, 1007, 973, 1053, 1035, 1054, 1018, 949, 998, 1014, 1014, 1079, 1001, 1064, 1034, 1030, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1003, 1012, 995, 1050, 1069, 1001, 1088, 1021, 1019, 1032, 973, 1011, 1017, 1033, 1035, 1022, 1004, 1091, 997, 1055, 1002, 999, 992, 1053, 1022, 1054, 1035, 1016, 1005, 1026, 1000, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1024, 996, 1009, 1032, 999, 1016, 1028, 1070, 1009, 1002, 981, 1005, 1013, 1075, 1039, 1023, 1055, 1078, 1004, 1055, 1012, 1010, 1043, 1055, 998, 998, 1033, 1024, 1014, 1044, 1003, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1038, 1007, 1055, 968, 980, 1018, 1029, 1005, 1048, 1017, 1073, 1023, 1032, 1041, 1038, 974, 1043, 1057, 1042, 976, 990, 1054, 1033, 1041, 1050, 1033, 993, 1029, 1023, 975, 1048, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1063, 1017, 1045, 994, 1041, 1053, 1049, 1033, 987, 976, 1060, 1088, 1021, 988, 1004, 1032, 1013, 1032, 1008, 1023, 1051, 1041, 997, 964, 1036, 1016, 986, 1063, 1044, 1022, 1002, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1010, 1024, 994, 1013, 1050, 965, 993, 1052, 1040, 1030, 1060, 1036, 994, 952, 1036, 1043, 1031, 1057, 1038, 1052, 1036, 1080, 1021, 999, 986, 1020, 1031, 1019, 1029, 995, 1062, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1034, 1023, 1046, 1032, 985, 1039, 1030, 987, 1028, 1036, 1020, 987, 998, 1015, 1061, 1058, 960, 1016, 1023, 1073, 1008, 975, 1067, 998, 1058, 1054, 1045, 1027, 995, 1057, 985, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1016, 1041, 1017, 977, 1010, 1019, 1004, 1089, 1052, 1025, 994, 1021, 1020, 1051, 1042, 1054, 1009, 1006, 1037, 1026, 1007, 1010, 1026, 1048, 1035, 1014, 999, 1045, 1040, 1020, 1017, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1036, 1030, 1039, 993, 12, 2, 1, 0, 1033, 1024, 1010, 1020, 0, 0, 0, 0, 1028, 1008, 1017, 1019, 5, 1, 0, 0, 1024, 1051, 1052, 1018, 0, 0, 0, 0, 1007, 955, 1053, 1035, 0, 0, 0, 0, 1054, 1018, 949, 998, 16, 0, 0, 0, 1014, 1014, 1079, 995, 0, 0, 0, 0, 1064, 1034, 1030, 1027, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4087, 4078, 4145, 4050, 4035, 4102, 4158] avg=4096.0000 max=4158 max_rank=7 min=4035 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1003, 1012, 995, 1050, 12, 3, 1, 0, 1069, 1001, 1088, 1017, 0, 0, 0, 0, 1019, 1032, 973, 1011, 5, 1, 0, 0, 1017, 1033, 1023, 1022, 0, 0, 0, 0, 1004, 1071, 997, 1055, 0, 0, 0, 0, 1002, 999, 992, 1053, 17, 0, 0, 0, 1022, 1054, 1035, 1010, 0, 0, 0, 0, 1005, 1026, 1000, 1036, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4175, 4041, 4095, 4127, 4063, 4121, 4070] avg=4096.0000 max=4175 max_rank=1 min=4041 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1024, 996, 1009, 1032, 12, 3, 1, 0, 999, 1016, 1028, 1065, 0, 0, 0, 0, 1009, 1002, 981, 1005, 5, 1, 0, 0, 1013, 1075, 1027, 1023, 0, 0, 0, 0, 1055, 1058, 1004, 1055, 0, 0, 0, 0, 1012, 1010, 1043, 1055, 17, 0, 0, 0, 998, 998, 1033, 1018, 0, 0, 0, 0, 1014, 1044, 1003, 1021, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4108, 4003, 4138, 4172, 4137, 4047, 4086] avg=4096.0000 max=4172 max_rank=4 min=4003 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1038, 1007, 1055, 968, 12, 3, 1, 0, 980, 1018, 1029, 1001, 0, 0, 0, 0, 1048, 1017, 1073, 1023, 6, 1, 0, 0, 1032, 1041, 1026, 974, 0, 0, 0, 0, 1043, 1037, 1042, 976, 0, 0, 0, 0, 990, 1054, 1033, 1041, 17, 0, 0, 0, 1050, 1033, 993, 1022, 0, 0, 0, 0, 1023, 975, 1048, 1035, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4028, 4168, 4073, 4098, 4135, 4098, 4084] avg=4096.0000 max=4168 max_rank=2 min=4028 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1034, 1023, 1046, 1032, 12, 2, 1, 0, 985, 1039, 1030, 983, 0, 0, 0, 0, 1028, 1036, 1020, 987, 5, 1, 0, 0, 998, 1015, 1049, 1058, 0, 0, 0, 0, 960, 998, 1023, 1073, 0, 0, 0, 0, 1008, 975, 1067, 998, 16, 0, 0, 0, 1058, 1054, 1045, 1021, 0, 0, 0, 0, 995, 1057, 985, 1048, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4037, 4077, 4120, 4054, 4064, 4178, 4088] avg=4096.0000 max=4178 max_rank=6 min=4037 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1010, 1024, 994, 1013, 16, 7, 4, 0, 1050, 965, 993, 1036, 0, 0, 0, 0, 1040, 1030, 1060, 1036, 10, 8, 0, 0, 994, 952, 1020, 1043, 0, 0, 0, 0, 1031, 1029, 1038, 1052, 0, 0, 0, 0, 1036, 1080, 1021, 999, 21, 0, 0, 0, 986, 1020, 1031, 1005, 0, 0, 0, 0, 1029, 995, 1062, 1020, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4044, 4184, 4009, 4150, 4157, 4042, 4114] avg=4096.0000 max=4184 max_rank=2 min=4009 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1063, 1017, 1045, 994, 12, 2, 1, 0, 1041, 1053, 1049, 1029, 0, 0, 0, 0, 987, 976, 1060, 1088, 6, 1, 0, 0, 1021, 988, 992, 1032, 0, 0, 0, 0, 1013, 1013, 1008, 1023, 0, 0, 0, 0, 1051, 1041, 997, 964, 17, 0, 0, 0, 1036, 1016, 986, 1056, 0, 0, 0, 0, 1044, 1022, 1002, 1019, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4172, 4118, 4033, 4057, 4070, 4094, 4090] avg=4096.0000 max=4172 max_rank=1 min=4033 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1016, 1041, 1017, 977, 12, 2, 1, 0, 1010, 1019, 1004, 1084, 0, 0, 0, 0, 1052, 1025, 994, 1021, 6, 1, 0, 0, 1020, 1051, 1030, 1054, 0, 0, 0, 0, 1009, 988, 1037, 1026, 0, 0, 0, 0, 1007, 1010, 1026, 1048, 16, 0, 0, 0, 1035, 1014, 999, 1038, 0, 0, 0, 0, 1040, 1020, 1017, 997, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4117, 4099, 4155, 4060, 4107, 4086, 4078] avg=4096.0000 max=4155 max_rank=3 min=4060 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8064, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1015, 1048, 1010, 1063, 1021, 983, 1046, 1078, 1015, 1019, 1026, 1039, 1075, 1045, 1032, 981, 1030, 994, 1059, 983, 1029, 1029, 990, 1039, 1021, 1051, 992, 1030, 1027, 968, 1004, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1007, 992, 1026, 1005, 989, 1061, 1033, 1037, 1027, 1019, 1011, 1058, 1047, 1028, 1026, 1033, 1010, 1003, 1025, 1060, 1078, 1026, 982, 1026, 1002, 1013, 1025, 1031, 1005, 1032, 1024, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1070, 929, 1022, 1044, 1009, 1029, 1039, 1020, 1081, 1000, 1022, 1043, 1043, 1034, 1016, 1005, 1048, 989, 1019, 997, 1034, 1053, 1011, 1019, 1020, 990, 1010, 1024, 1038, 1026, 1021, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1029, 990, 1050, 1038, 1080, 990, 997, 1026, 1062, 1045, 1033, 1041, 1029, 1051, 1021, 1026, 977, 1061, 997, 1018, 987, 993, 1044, 1012, 1024, 992, 1051, 1012, 1009, 1012, 1057, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1027, 1061, 1032, 1002, 1042, 1011, 1073, 1051, 1014, 1029, 1063, 959, 1049, 1037, 1004, 1019, 1030, 1021, 1018, 988, 984, 1037, 1005, 994, 1015, 1015, 1013, 1006, 1040, 1053, 1064, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1024, 1074, 1036, 1032, 1038, 1053, 1002, 966, 1051, 1059, 1032, 1022, 1015, 992, 1048, 1038, 974, 1035, 1023, 1062, 1006, 1038, 1005, 1011, 1036, 975, 1037, 1011, 1019, 1011, 1021, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1018, 1035, 1024, 953, 1005, 1046, 1052, 1033, 986, 1030, 1017, 1026, 1012, 1014, 1033, 1048, 994, 991, 1011, 1034, 1061, 1045, 991, 1051, 998, 1020, 1005, 1058, 1060, 1073, 1024, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1003, 1033, 1028, 1015, 1093, 1005, 1060, 1054, 1039, 996, 1015, 1026, 1013, 1063, 955, 1037, 996, 1022, 1052, 1078, 1033, 997, 1018, 1005, 1032, 993, 1042, 1012, 1023, 1046, 1010, 974] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1015, 1048, 1010, 1063, 4, 0, 0, 0, 1021, 983, 1015, 1078, 0, 0, 0, 0, 999, 1019, 1026, 1039, 0, 0, 0, 0, 1062, 1045, 1032, 981, 0, 0, 0, 0, 1030, 994, 1059, 983, 15, 5, 0, 0, 1029, 1029, 990, 1039, 13, 2, 1, 0, 1021, 1051, 992, 1030, 26, 0, 0, 0, 1027, 968, 998, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4097, 4083, 4120, 4086, 4103, 4120, 4019] avg=4096.0000 max=4140 max_rank=0 min=4019 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1007, 992, 1026, 1005, 4, 0, 0, 0, 989, 1061, 1002, 1037, 0, 0, 0, 0, 1011, 1019, 1011, 1058, 0, 0, 0, 0, 1035, 1028, 1026, 1033, 0, 0, 0, 0, 1010, 1003, 1025, 1060, 15, 5, 0, 0, 1078, 1026, 982, 1026, 12, 2, 1, 0, 1002, 1013, 1025, 1031, 26, 0, 0, 0, 1005, 1032, 1018, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4089, 4099, 4122, 4118, 4127, 4097, 4082] avg=4096.0000 max=4127 max_rank=5 min=4034 min_rank=0 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1018, 1035, 1024, 953, 4, 0, 0, 0, 1005, 1046, 1021, 1033, 0, 0, 0, 0, 970, 1030, 1017, 1026, 0, 0, 0, 0, 1000, 1014, 1033, 1048, 0, 0, 0, 0, 994, 991, 1011, 1034, 15, 5, 0, 0, 1061, 1045, 991, 1051, 12, 2, 1, 0, 998, 1020, 1005, 1058, 26, 0, 0, 0, 1060, 1073, 1018, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4105, 4043, 4095, 4050, 4163, 4107, 4171] avg=4096.0000 max=4171 max_rank=7 min=4034 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1070, 929, 1022, 1044, 4, 0, 0, 0, 1009, 1029, 1008, 1020, 0, 0, 0, 0, 1064, 1000, 1022, 1043, 0, 0, 0, 0, 1031, 1034, 1016, 1005, 0, 0, 0, 0, 1048, 989, 1019, 997, 16, 5, 0, 0, 1034, 1053, 1011, 1019, 12, 2, 1, 0, 1020, 990, 1010, 1024, 26, 0, 0, 0, 1038, 1026, 1015, 1063, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4066, 4129, 4086, 4074, 4132, 4070, 4142] avg=4096.0000 max=4142 max_rank=7 min=4066 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1024, 1074, 1036, 1032, 4, 0, 0, 0, 1038, 1053, 972, 966, 0, 0, 0, 0, 1034, 1059, 1032, 1022, 0, 0, 0, 0, 1003, 992, 1048, 1038, 0, 0, 0, 0, 974, 1035, 1023, 1062, 16, 5, 0, 0, 1006, 1038, 1005, 1011, 12, 2, 1, 0, 1036, 975, 1037, 1011, 25, 0, 0, 0, 1019, 1011, 1015, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4029, 4147, 4081, 4115, 4075, 4084, 4067] avg=4096.0000 max=4170 max_rank=0 min=4029 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1029, 990, 1050, 1038, 5, 0, 0, 0, 1080, 990, 961, 1026, 0, 0, 0, 0, 1039, 1045, 1033, 1041, 0, 0, 0, 0, 1013, 1051, 1021, 1026, 0, 0, 0, 0, 977, 1061, 997, 1018, 20, 7, 0, 0, 987, 993, 1044, 1012, 16, 9, 3, 0, 1024, 992, 1051, 1012, 29, 0, 0, 0, 1009, 1012, 1043, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4057, 4158, 4111, 4080, 4064, 4108, 4078] avg=4096.0000 max=4158 max_rank=2 min=4057 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1027, 1061, 1032, 1002, 4, 0, 0, 0, 1042, 1011, 1041, 1051, 0, 0, 0, 0, 998, 1029, 1063, 959, 0, 0, 0, 0, 1037, 1037, 1004, 1019, 0, 0, 0, 0, 1030, 1021, 1018, 988, 15, 5, 0, 0, 984, 1037, 1005, 994, 12, 3, 1, 0, 1015, 1015, 1013, 1006, 27, 0, 0, 0, 1040, 1053, 1057, 1012, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4145, 4049, 4097, 4077, 4036, 4076, 4162] avg=4096.0000 max=4162 max_rank=7 min=4036 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1003, 1033, 1028, 1015, 4, 0, 0, 0, 1093, 1005, 1028, 1054, 0, 0, 0, 0, 1023, 996, 1015, 1026, 0, 0, 0, 0, 1001, 1063, 955, 1037, 0, 0, 0, 0, 996, 1022, 1052, 1078, 15, 5, 0, 0, 1033, 997, 1018, 1005, 12, 2, 1, 0, 1032, 993, 1042, 1012, 27, 0, 0, 0, 1023, 1046, 1004, 974, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4180, 4060, 4056, 4168, 4068, 4106, 4047] avg=4096.0000 max=4180 max_rank=1 min=4047 min_rank=7 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8064, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1010, 1017, 1025, 1060, 1049, 1003, 1027, 1010, 992, 1012, 1071, 1068, 992, 1012, 992, 1019, 1038, 1014, 1100, 1013, 1008, 1019, 992, 1039, 1012, 1020, 1017, 1034, 1026, 1048, 1029, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1023, 1004, 1039, 992, 1046, 1044, 1069, 1001, 978, 1035, 1037, 991, 1033, 1012, 1010, 1016, 1001, 1063, 1008, 1020, 1003, 1035, 1051, 1053, 1004, 1061, 1026, 993, 1078, 1004, 1038, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1001, 992, 991, 1019, 1022, 1038, 1046, 1062, 1025, 1025, 1024, 1003, 1082, 1019, 1013, 1032, 978, 1045, 1039, 991, 1021, 1023, 1036, 1034, 1027, 1053, 1005, 1036, 1013, 1017, 1004, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1034, 1024, 1037, 1047, 984, 999, 1013, 1011, 994, 1035, 979, 1031, 1026, 1079, 1032, 1026, 1014, 1077, 1027, 994, 1011, 1045, 1018, 1023, 1028, 1040, 1034, 1011, 1011, 982, 1041, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1047, 1064, 1021, 999, 1042, 990, 1020, 1094, 996, 1061, 1029, 1002, 1017, 1047, 1060, 999, 1047, 1014, 1010, 1037, 1058, 1029, 997, 992, 1034, 999, 973, 1038, 988, 1050, 1017, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1006, 965, 1017, 1031, 1047, 1048, 1035, 1032, 1047, 1013, 1018, 1054, 1010, 982, 985, 1042, 1034, 1022, 1034, 1055, 1016, 1087, 1008, 1013, 1002, 1025, 989, 1003, 1033, 1024, 1059, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1023, 1023, 1009, 1012, 1020, 997, 986, 1031, 1040, 1012, 955, 1016, 1018, 1013, 1082, 991, 1055, 1085, 1067, 1018, 1057, 1069, 1006, 1049, 1018, 1015, 982, 1030, 1020, 1049, 1000, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1012, 1038, 1003, 1024, 1025, 994, 1054, 996, 1014, 1034, 1043, 1016, 1027, 1043, 1043, 1042, 1035, 1005, 982, 1032, 987, 979, 1068, 1076, 1053, 1054, 962, 979, 1014, 1027, 1068, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1010, 1017, 1025, 1060, 13, 6, 0, 0, 1049, 1003, 1019, 1010, 0, 0, 0, 0, 992, 1012, 1071, 1068, 8, 3, 2, 0, 992, 1012, 989, 1019, 0, 0, 0, 0, 1038, 992, 1100, 1013, 0, 0, 0, 0, 1008, 1003, 992, 1039, 0, 0, 0, 0, 1012, 1020, 1017, 1034, 22, 3, 0, 0, 1026, 1048, 1021, 1000, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4081, 4156, 4012, 4143, 4042, 4108, 4095] avg=4096.0000 max=4156 max_rank=2 min=4012 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1001, 992, 991, 1019, 13, 6, 0, 0, 1022, 1038, 1038, 1062, 0, 0, 0, 0, 1025, 1025, 1024, 1003, 8, 3, 2, 0, 1082, 1019, 1010, 1032, 0, 0, 0, 0, 978, 1022, 1039, 991, 0, 0, 0, 0, 1021, 1007, 1036, 1034, 0, 0, 0, 0, 1027, 1053, 1005, 1036, 23, 3, 0, 0, 1013, 1017, 996, 1052, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4160, 4090, 4143, 4030, 4098, 4147, 4078] avg=4096.0000 max=4160 max_rank=1 min=4022 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1023, 1023, 1009, 1012, 14, 6, 0, 0, 1020, 997, 978, 1031, 0, 0, 0, 0, 1040, 1012, 955, 1016, 8, 3, 2, 0, 1018, 1013, 1079, 991, 0, 0, 0, 0, 1055, 1061, 1067, 1018, 0, 0, 0, 0, 1057, 1052, 1006, 1049, 0, 0, 0, 0, 1018, 1015, 982, 1030, 24, 3, 0, 0, 1020, 1049, 992, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4026, 4036, 4101, 4201, 4164, 4072, 4081] avg=4096.0000 max=4201 max_rank=4 min=4026 min_rank=1 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1034, 1024, 1037, 1047, 13, 6, 0, 0, 984, 999, 1005, 1011, 0, 0, 0, 0, 994, 1035, 979, 1031, 8, 3, 2, 0, 1026, 1079, 1029, 1026, 0, 0, 0, 0, 1014, 1053, 1027, 994, 0, 0, 0, 0, 1011, 1029, 1018, 1023, 0, 0, 0, 0, 1028, 1040, 1034, 1011, 24, 3, 0, 0, 1011, 982, 1033, 1061, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 3999, 4052, 4160, 4088, 4081, 4140, 4087] avg=4096.0000 max=4161 max_rank=0 min=3999 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1047, 1064, 1021, 999, 13, 6, 0, 0, 1042, 990, 1012, 1094, 0, 0, 0, 0, 996, 1061, 1029, 1002, 8, 3, 2, 0, 1017, 1047, 1057, 999, 0, 0, 0, 0, 1047, 992, 1010, 1037, 0, 0, 0, 0, 1058, 1013, 997, 992, 0, 0, 0, 0, 1034, 999, 973, 1038, 22, 3, 0, 0, 988, 1050, 1009, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4138, 4101, 4120, 4086, 4060, 4069, 4044] avg=4096.0000 max=4150 max_rank=0 min=4044 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1023, 1004, 1039, 992, 13, 6, 0, 0, 1046, 1044, 1061, 1001, 0, 0, 0, 0, 978, 1035, 1037, 991, 8, 3, 2, 0, 1033, 1012, 1007, 1016, 0, 0, 0, 0, 1001, 1040, 1008, 1020, 0, 0, 0, 0, 1003, 1019, 1051, 1053, 0, 0, 0, 0, 1004, 1061, 1026, 993, 23, 3, 0, 0, 1078, 1004, 1030, 1000, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4152, 4054, 4068, 4069, 4126, 4110, 4112] avg=4096.0000 max=4152 max_rank=1 min=4054 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1006, 965, 1017, 1031, 18, 8, 0, 0, 1047, 1048, 1024, 1032, 0, 0, 0, 0, 1047, 1013, 1018, 1054, 11, 7, 9, 0, 1010, 982, 978, 1042, 0, 0, 0, 0, 1034, 996, 1034, 1055, 0, 0, 0, 0, 1016, 1064, 1008, 1013, 0, 0, 0, 0, 1002, 1025, 989, 1003, 26, 5, 0, 0, 1033, 1024, 1042, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4151, 4159, 4012, 4119, 4101, 4050, 4131] avg=4096.0000 max=4159 max_rank=2 min=4012 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1012, 1038, 1003, 1024, 12, 6, 0, 0, 1025, 994, 1046, 996, 0, 0, 0, 0, 1014, 1034, 1043, 1016, 8, 3, 2, 0, 1027, 1043, 1040, 1042, 0, 0, 0, 0, 1035, 983, 982, 1032, 0, 0, 0, 0, 987, 965, 1068, 1076, 0, 0, 0, 0, 1053, 1054, 962, 979, 22, 2, 0, 0, 1014, 1027, 1060, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4061, 4120, 4152, 4032, 4096, 4072, 4140] avg=4096.0000 max=4152 max_rank=3 min=4032 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1008, 1041, 1024, 1049, 1005, 1018, 1022, 1017, 1021, 1057, 1046, 1051, 1089, 1022, 1062, 1036, 1021, 1071, 997, 974, 1023, 1006, 1003, 1002, 998, 1001, 1016, 1004, 1016, 1006, 1037, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1035, 1039, 1026, 1033, 1018, 979, 994, 1038, 1012, 979, 1064, 1052, 1041, 1039, 1033, 966, 1060, 1018, 993, 973, 1016, 1002, 998, 1045, 1043, 1040, 1000, 1032, 1053, 1037, 1045, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[967, 1020, 1038, 1028, 1007, 993, 1018, 1038, 1065, 1059, 998, 1014, 1082, 1005, 1023, 1015, 998, 1034, 1054, 1023, 1041, 1059, 1067, 985, 1004, 975, 1014, 1020, 1026, 1010, 1027, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1065, 1041, 1066, 1023, 1070, 975, 1034, 992, 1027, 1025, 1001, 990, 1018, 1004, 1018, 1011, 996, 1041, 986, 1000, 1032, 1024, 1054, 1037, 994, 1080, 1028, 1025, 1000, 1035, 1044, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1017, 1008, 1059, 1070, 991, 1013, 1032, 989, 1027, 1002, 1055, 1060, 1008, 1040, 1030, 1045, 963, 1004, 1021, 1019, 985, 983, 1050, 1036, 1015, 1045, 1023, 1033, 1014, 1063, 1051, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1046, 1045, 1031, 1055, 1025, 1038, 1033, 1051, 1041, 1018, 1018, 1034, 998, 983, 1020, 1044, 1021, 1033, 955, 996, 1022, 1015, 970, 1072, 991, 1034, 1033, 1010, 1049, 1051, 1035, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[998, 1046, 1032, 1029, 995, 1017, 967, 1044, 1023, 1050, 1065, 1025, 1060, 1032, 1040, 1037, 982, 1022, 1072, 980, 1019, 1033, 1039, 978, 1018, 1024, 987, 1018, 1029, 1061, 1043, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1041, 1034, 1060, 1020, 1056, 1034, 966, 1028, 986, 1010, 1029, 978, 1020, 1013, 997, 1071, 1013, 1024, 1003, 1069, 1027, 1030, 1034, 1023, 1013, 1008, 1062, 1011, 1013, 1058, 1023, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1035, 1039, 986, 1033, 0, 0, 0, 0, 1018, 979, 994, 1038, 31, 2, 0, 0, 1012, 979, 1050, 1052, 0, 0, 0, 0, 1025, 1039, 1033, 966, 0, 0, 0, 0, 1060, 1018, 993, 973, 40, 3, 0, 0, 1016, 1002, 998, 1045, 7, 0, 0, 0, 1043, 1040, 1000, 1032, 14, 7, 0, 0, 1053, 1003, 1045, 1065, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4062, 4093, 4063, 4087, 4068, 4136, 4166] avg=4096.0000 max=4166 max_rank=7 min=4062 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[967, 1020, 998, 1028, 0, 0, 0, 0, 1007, 993, 1018, 1038, 30, 2, 0, 0, 1065, 1059, 986, 1014, 0, 0, 0, 0, 1066, 1005, 1023, 1015, 0, 0, 0, 0, 998, 1034, 1054, 1023, 40, 3, 0, 0, 1041, 1059, 1067, 985, 6, 0, 0, 0, 1004, 975, 1014, 1020, 14, 6, 0, 0, 1026, 977, 1027, 1061, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4088, 4124, 4109, 4152, 4158, 4033, 4091] avg=4096.0000 max=4158 max_rank=5 min=4013 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1008, 1041, 984, 1049, 0, 0, 0, 0, 1005, 1018, 1022, 1017, 30, 2, 0, 0, 1021, 1057, 1032, 1051, 0, 0, 0, 0, 1073, 1022, 1062, 1036, 0, 0, 0, 0, 1021, 1071, 997, 974, 40, 3, 0, 0, 1023, 1006, 1003, 1002, 7, 0, 0, 0, 998, 1001, 1016, 1004, 14, 7, 0, 0, 1016, 973, 1037, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4094, 4161, 4193, 4106, 4041, 4040, 4051] avg=4096.0000 max=4193 max_rank=3 min=4040 min_rank=6 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16238 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1017, 1008, 1018, 1070, 0, 0, 0, 0, 991, 1013, 1032, 989, 31, 2, 0, 0, 1027, 1002, 1041, 1060, 0, 0, 0, 0, 993, 1040, 1030, 1045, 0, 0, 0, 0, 963, 1004, 1021, 1019, 41, 3, 0, 0, 985, 983, 1050, 1036, 7, 0, 0, 0, 1015, 1045, 1023, 1033, 13, 7, 0, 0, 1014, 1029, 1051, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4058, 4130, 4108, 4051, 4061, 4136, 4111] avg=4096.0000 max=4136 max_rank=6 min=4051 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1046, 1045, 991, 1055, 0, 0, 0, 0, 1025, 1038, 1033, 1051, 31, 2, 0, 0, 1041, 1018, 1005, 1034, 0, 0, 0, 0, 983, 983, 1020, 1044, 0, 0, 0, 0, 1021, 1033, 955, 996, 40, 3, 0, 0, 1022, 1015, 970, 1072, 7, 0, 0, 0, 991, 1034, 1033, 1010, 13, 6, 0, 0, 1049, 1017, 1035, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4180, 4098, 4030, 4048, 4086, 4087, 4102] avg=4096.0000 max=4180 max_rank=1 min=4030 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1065, 1041, 1022, 1023, 0, 0, 0, 0, 1070, 975, 1034, 992, 35, 7, 0, 0, 1027, 1025, 981, 990, 0, 0, 0, 0, 993, 1004, 1018, 1011, 0, 0, 0, 0, 996, 1041, 986, 1000, 44, 5, 0, 0, 1032, 1024, 1054, 1037, 10, 0, 0, 0, 994, 1080, 1028, 1025, 18, 10, 0, 0, 1000, 995, 1044, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4113, 4023, 4026, 4072, 4157, 4155, 4071] avg=4096.0000 max=4157 max_rank=5 min=4023 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[998, 1046, 992, 1029, 0, 0, 0, 0, 995, 1017, 967, 1044, 31, 2, 0, 0, 1023, 1050, 1051, 1025, 0, 0, 0, 0, 1044, 1032, 1040, 1037, 0, 0, 0, 0, 982, 1022, 1072, 980, 40, 3, 0, 0, 1019, 1033, 1039, 978, 7, 0, 0, 0, 1018, 1024, 987, 1018, 14, 7, 0, 0, 1029, 1027, 1043, 1003, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4056, 4149, 4153, 4099, 4076, 4068, 4102] avg=4096.0000 max=4153 max_rank=3 min=4056 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1041, 1034, 1019, 1020, 0, 0, 0, 0, 1056, 1034, 966, 1028, 31, 2, 0, 0, 986, 1010, 1016, 978, 0, 0, 0, 0, 1005, 1013, 997, 1071, 0, 0, 0, 0, 1013, 1024, 1003, 1069, 41, 3, 0, 0, 1027, 1030, 1034, 1023, 7, 0, 0, 0, 1013, 1008, 1062, 1011, 13, 6, 0, 0, 1013, 1024, 1023, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4117, 3990, 4086, 4153, 4121, 4113, 4074] avg=4096.0000 max=4153 max_rank=4 min=3990 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8064, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1022, 1053, 1028, 1050, 1031, 1044, 1004, 1024, 1015, 987, 1071, 1080, 985, 1021, 1030, 1021, 1071, 1022, 985, 1032, 1013, 998, 1010, 966, 1017, 1041, 1001, 1042, 1049, 974, 1035, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1025, 1026, 1030, 1069, 1035, 1056, 1063, 984, 1045, 989, 1047, 1035, 1031, 1038, 949, 1065, 1001, 985, 1019, 981, 992, 996, 1008, 1087, 1070, 998, 977, 1044, 1001, 1022, 1064, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1024, 1041, 958, 1050, 1023, 1065, 996, 1010, 997, 1022, 1037, 1046, 1046, 1007, 1030, 1034, 981, 1053, 1014, 1014, 1026, 1030, 1020, 1079, 1046, 1023, 1045, 1028, 998, 1000, 1025, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1019, 1022, 1075, 1011, 1022, 1036, 1027, 1042, 1004, 1016, 1005, 1042, 1001, 1035, 1009, 1015, 1011, 1006, 1036, 1034, 983, 1010, 1011, 1035, 1067, 1006, 1030, 1031, 1030, 1031, 1043, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[999, 1000, 1017, 1013, 999, 1066, 1034, 1020, 1018, 1048, 991, 982, 1001, 981, 1038, 1014, 1002, 1007, 997, 1035, 1022, 1062, 1022, 1021, 1096, 1043, 1092, 1023, 996, 1088, 986, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1076, 1039, 962, 1040, 999, 989, 1030, 993, 1098, 1056, 1068, 1021, 1047, 1013, 990, 1008, 1006, 1000, 1004, 1026, 975, 1034, 1031, 1035, 1048, 1035, 1012, 961, 1036, 1071, 1033, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[966, 1035, 996, 1045, 982, 1056, 1010, 1046, 1005, 1005, 991, 1039, 1043, 1001, 1036, 1017, 1057, 1009, 1060, 1096, 995, 1042, 1026, 1054, 1019, 1045, 1021, 996, 1038, 987, 1053, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1009, 1050, 1040, 1042, 1039, 1025, 1001, 1023, 1033, 1012, 975, 1069, 992, 1014, 1036, 1036, 1028, 1001, 1004, 1018, 1029, 1033, 1042, 1016, 1067, 1040, 1027, 986, 1041, 1019, 1006, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1025, 1026, 1030, 1061, 0, 0, 0, 0, 1035, 1056, 1063, 984, 0, 0, 0, 0, 1045, 989, 1047, 1025, 0, 0, 0, 0, 1031, 1038, 949, 1065, 23, 0, 0, 0, 1001, 985, 1019, 981, 10, 8, 3, 0, 992, 996, 1008, 1087, 7, 0, 0, 0, 1044, 998, 977, 1044, 0, 0, 0, 0, 1001, 1022, 1057, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4138, 4106, 4106, 4007, 4090, 4063, 4116] avg=4096.0000 max=4142 max_rank=0 min=4007 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16276 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1022, 1053, 1028, 1042, 0, 0, 0, 0, 1031, 1044, 1004, 1024, 0, 0, 0, 0, 1015, 987, 1071, 1070, 0, 0, 0, 0, 985, 1021, 1030, 1021, 22, 0, 0, 0, 1071, 1022, 985, 1032, 10, 8, 3, 0, 1013, 998, 1010, 966, 7, 0, 0, 0, 992, 1041, 1001, 1042, 0, 0, 0, 0, 1049, 974, 1028, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4103, 4143, 4079, 4131, 3994, 4076, 4097] avg=4096.0000 max=4145 max_rank=0 min=3994 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1024, 1041, 958, 1042, 0, 0, 0, 0, 1023, 1065, 996, 1010, 0, 0, 0, 0, 997, 1022, 1037, 1036, 0, 0, 0, 0, 1046, 1007, 1030, 1034, 22, 0, 0, 0, 981, 1053, 1014, 1014, 10, 8, 3, 0, 1026, 1030, 1020, 1079, 7, 0, 0, 0, 1021, 1023, 1045, 1028, 0, 0, 0, 0, 998, 1000, 1018, 1000, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4094, 4092, 4139, 4083, 4162, 4117, 4016] avg=4096.0000 max=4162 max_rank=5 min=4016 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[999, 1000, 1017, 1006, 0, 0, 0, 0, 999, 1066, 1034, 1020, 0, 0, 0, 0, 1018, 1048, 991, 973, 0, 0, 0, 0, 1001, 981, 1038, 1014, 23, 0, 0, 0, 1002, 1007, 997, 1035, 9, 7, 3, 0, 1022, 1062, 1022, 1021, 7, 0, 0, 0, 1070, 1043, 1092, 1023, 0, 0, 0, 0, 996, 1088, 979, 1055, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4119, 4030, 4057, 4060, 4134, 4228, 4118] avg=4096.0000 max=4228 max_rank=6 min=4022 min_rank=0 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1019, 1022, 1075, 1002, 0, 0, 0, 0, 1022, 1030, 1027, 1042, 0, 0, 0, 0, 1004, 1016, 1005, 1029, 0, 0, 0, 0, 1001, 1035, 1009, 1015, 27, 0, 0, 0, 1011, 1006, 1036, 1034, 13, 9, 4, 3, 983, 1010, 1011, 1035, 10, 6, 0, 0, 1036, 1006, 1030, 1031, 0, 0, 0, 0, 1030, 1031, 1030, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4121, 4054, 4087, 4116, 4055, 4103, 4114] avg=4096.0000 max=4121 max_rank=1 min=4054 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1076, 1039, 962, 1032, 0, 0, 0, 0, 999, 989, 1030, 993, 0, 0, 0, 0, 1098, 1056, 1068, 1012, 0, 0, 0, 0, 1047, 1013, 990, 1008, 22, 0, 0, 0, 1006, 1000, 1004, 1026, 9, 8, 3, 0, 975, 1034, 1031, 1035, 7, 0, 0, 0, 1023, 1035, 1012, 961, 0, 0, 0, 0, 1036, 1071, 1026, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4011, 4234, 4080, 4056, 4082, 4031, 4165] avg=4096.0000 max=4234 max_rank=2 min=4011 min_rank=1 imbalance_ratio=1.033691 max_load_reduction=28534 max_load_reduction_pct=0.870789 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[966, 1035, 996, 1037, 0, 0, 0, 0, 982, 1056, 1010, 1046, 0, 0, 0, 0, 1005, 1005, 991, 1029, 0, 0, 0, 0, 1043, 1001, 1036, 1017, 22, 0, 0, 0, 1057, 1009, 1060, 1096, 10, 8, 3, 0, 995, 1042, 1026, 1054, 7, 0, 0, 0, 994, 1045, 1021, 996, 0, 0, 0, 0, 1038, 987, 1046, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4094, 4030, 4119, 4243, 4124, 4056, 4068] avg=4096.0000 max=4243 max_rank=4 min=4030 min_rank=2 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1009, 1050, 1040, 1034, 0, 0, 0, 0, 1039, 1025, 1001, 1023, 0, 0, 0, 0, 1033, 1012, 975, 1059, 0, 0, 0, 0, 992, 1014, 1036, 1036, 23, 0, 0, 0, 1028, 1001, 1004, 1018, 10, 8, 3, 0, 1029, 1033, 1042, 1016, 7, 0, 0, 0, 1041, 1040, 1027, 986, 0, 0, 0, 0, 1041, 1019, 999, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4088, 4079, 4101, 4072, 4127, 4094, 4074] avg=4096.0000 max=4133 max_rank=0 min=4072 min_rank=4 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8448, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1010, 1046, 1065, 1040, 1039, 978, 1013, 1046, 1041, 1020, 1008, 933, 1029, 1042, 998, 1038, 1005, 1013, 1018, 1027, 1030, 1037, 970, 1067, 963, 1014, 1045, 1044, 1062, 1052, 1047, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[984, 1033, 1053, 1029, 988, 1016, 1057, 1002, 1032, 1031, 1039, 1075, 1000, 970, 1031, 1019, 1056, 981, 977, 999, 1012, 1045, 1050, 1013, 1051, 1008, 1041, 1083, 982, 1034, 1064, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1030, 996, 1046, 1057, 1056, 1052, 1036, 1014, 1045, 1011, 1013, 963, 1043, 1009, 1028, 987, 1057, 989, 984, 1017, 1059, 1063, 999, 1056, 1059, 1018, 969, 1018, 1010, 1038, 1018, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1014, 997, 988, 1039, 1034, 1046, 1013, 1000, 1035, 994, 1055, 1042, 1070, 1034, 1027, 1008, 1030, 1026, 967, 969, 1007, 1048, 1028, 1049, 1051, 994, 983, 1067, 1026, 1057, 986, 1084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1034, 1007, 1064, 1001, 1026, 1046, 1045, 991, 1009, 1027, 1018, 993, 1032, 1053, 1060, 1066, 988, 1032, 994, 1022, 974, 1051, 1053, 1003, 1026, 1022, 1059, 990, 1068, 974, 1007, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1032, 1007, 1040, 971, 1044, 987, 1006, 1067, 1036, 1087, 1019, 1067, 1083, 998, 1068, 1028, 1009, 1063, 1051, 1006, 1024, 1045, 1011, 997, 989, 969, 987, 994, 994, 1026, 1043, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[974, 1007, 1067, 1030, 1025, 985, 1045, 1012, 1065, 1051, 1025, 1029, 1064, 1003, 992, 1020, 1055, 1042, 1033, 985, 1027, 1027, 995, 1033, 1051, 1032, 1042, 946, 987, 1067, 1058, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1011, 1031, 1032, 1011, 1038, 1050, 1036, 1055, 1054, 1029, 1080, 1025, 1024, 1048, 991, 1056, 1009, 1028, 1040, 998, 986, 1004, 987, 997, 1016, 1018, 1020, 1047, 1008, 999, 1013, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1010, 1046, 1065, 1040, 2, 0, 0, 0, 1039, 978, 1004, 1046, 0, 0, 0, 0, 1019, 1020, 1008, 933, 0, 0, 0, 0, 1011, 1042, 998, 1038, 0, 0, 0, 0, 1005, 1013, 1018, 1027, 22, 14, 0, 0, 1030, 1037, 970, 1067, 2, 0, 0, 0, 963, 1014, 1045, 1044, 9, 4, 4, 0, 1062, 1044, 1047, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4067, 3980, 4089, 4099, 4106, 4083, 4181] avg=4096.0000 max=4181 max_rank=7 min=3980 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1030, 996, 1046, 1057, 2, 0, 0, 0, 1056, 1052, 1026, 1014, 0, 0, 0, 0, 1023, 1011, 1013, 963, 0, 0, 0, 0, 1025, 1009, 1028, 987, 0, 0, 0, 0, 1057, 989, 984, 1017, 22, 14, 0, 0, 1059, 1063, 999, 1056, 2, 0, 0, 0, 1059, 1018, 969, 1018, 10, 4, 4, 0, 1010, 1030, 1018, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4148, 4010, 4049, 4083, 4179, 4082, 4086] avg=4096.0000 max=4179 max_rank=5 min=4010 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[984, 1033, 1053, 1029, 2, 0, 0, 0, 988, 1016, 1047, 1002, 0, 0, 0, 0, 1010, 1031, 1039, 1075, 0, 0, 0, 0, 983, 970, 1031, 1019, 0, 0, 0, 0, 1056, 981, 977, 999, 22, 13, 0, 0, 1012, 1045, 1050, 1013, 2, 0, 0, 0, 1051, 1008, 1041, 1083, 10, 4, 4, 0, 982, 1026, 1064, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4053, 4155, 4003, 4048, 4122, 4201, 4085] avg=4096.0000 max=4201 max_rank=6 min=4003 min_rank=3 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1034, 1007, 1064, 1001, 2, 0, 0, 0, 1026, 1046, 1035, 991, 0, 0, 0, 0, 987, 1027, 1018, 993, 0, 0, 0, 0, 1014, 1053, 1060, 1066, 0, 0, 0, 0, 988, 1032, 994, 1022, 22, 14, 0, 0, 974, 1051, 1053, 1003, 2, 0, 0, 0, 1026, 1022, 1059, 990, 10, 4, 4, 0, 1068, 966, 1007, 1033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4098, 4025, 4193, 4072, 4083, 4115, 4074] avg=4096.0000 max=4193 max_rank=3 min=4025 min_rank=2 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[974, 1007, 1067, 1030, 2, 0, 0, 0, 1025, 985, 1035, 1012, 0, 0, 0, 0, 1042, 1051, 1025, 1029, 0, 0, 0, 0, 1046, 1003, 992, 1020, 0, 0, 0, 0, 1055, 1042, 1033, 985, 23, 14, 0, 0, 1027, 1027, 995, 1033, 2, 0, 0, 0, 1051, 1032, 1042, 946, 10, 4, 4, 0, 987, 1059, 1058, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4057, 4147, 4061, 4152, 4084, 4089, 4098] avg=4096.0000 max=4152 max_rank=4 min=4057 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1014, 997, 988, 1039, 8, 0, 0, 0, 1034, 1046, 1001, 1000, 0, 0, 0, 0, 1008, 994, 1055, 1042, 0, 0, 0, 0, 1044, 1034, 1027, 1008, 0, 0, 0, 0, 1030, 1026, 967, 969, 27, 18, 0, 0, 1007, 1048, 1028, 1049, 7, 0, 0, 0, 1051, 994, 983, 1067, 12, 8, 8, 0, 1026, 1034, 986, 1084, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4081, 4099, 4113, 4037, 4139, 4123, 4130] avg=4096.0000 max=4139 max_rank=5 min=4037 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1011, 1031, 1032, 1011, 2, 0, 0, 0, 1038, 1050, 1026, 1055, 0, 0, 0, 0, 1031, 1029, 1080, 1025, 0, 0, 0, 0, 1006, 1048, 991, 1056, 0, 0, 0, 0, 1009, 1028, 1040, 998, 23, 14, 0, 0, 986, 1004, 987, 997, 2, 0, 0, 0, 1016, 1018, 1020, 1047, 10, 4, 4, 0, 1008, 991, 1013, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4169, 4165, 4101, 4112, 3976, 4119, 4039] avg=4096.0000 max=4169 max_rank=1 min=3976 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1032, 1007, 1040, 971, 2, 0, 0, 0, 1044, 987, 997, 1067, 0, 0, 0, 0, 1014, 1087, 1019, 1067, 0, 0, 0, 0, 1065, 998, 1068, 1028, 0, 0, 0, 0, 1009, 1063, 1051, 1006, 22, 14, 0, 0, 1024, 1045, 1011, 997, 2, 0, 0, 0, 989, 969, 987, 994, 9, 4, 4, 0, 994, 1018, 1043, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4095, 4187, 4159, 4165, 4079, 3956, 4075] avg=4096.0000 max=4187 max_rank=2 min=3956 min_rank=6 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8448, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8064, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1004, 1018, 1062, 988, 995, 1057, 1025, 1017, 1028, 1022, 1020, 980, 1029, 1008, 1032, 1059, 1035, 993, 994, 1073, 1035, 1008, 999, 1000, 1024, 1017, 1040, 1000, 1016, 1005, 1085, 1100] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1026, 991, 1030, 992, 1061, 1027, 1008, 1065, 1015, 1021, 1058, 1019, 1010, 1013, 995, 1056, 970, 1051, 1069, 1023, 988, 987, 1056, 1079, 1046, 1009, 1045, 1007, 1022, 997, 1006, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1000, 1077, 1025, 1030, 1007, 1018, 1022, 1032, 1007, 995, 1011, 1012, 1007, 1051, 1085, 1029, 962, 1034, 1014, 1044, 1067, 1010, 1043, 1016, 1015, 1005, 1069, 1011, 1048, 1061, 960, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1019, 1008, 1018, 1047, 1066, 1003, 1034, 1012, 1001, 1031, 1031, 1016, 1068, 1032, 1037, 1027, 991, 1020, 1038, 1034, 1021, 1025, 963, 1033, 1015, 1005, 983, 1008, 1053, 1032, 1021, 1076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1021, 1041, 1053, 1033, 1025, 1003, 1030, 1080, 1076, 1021, 1023, 1046, 987, 998, 1042, 1005, 944, 1021, 1072, 1006, 1040, 995, 1005, 1016, 1011, 1025, 1054, 995, 1033, 1019, 1044, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1022, 1059, 1020, 1034, 986, 1091, 1062, 1006, 1031, 997, 1012, 1046, 1064, 998, 991, 993, 1022, 1048, 1000, 965, 1018, 1027, 1017, 986, 1050, 1021, 1019, 1060, 1029, 1052, 1064, 978] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1027, 1025, 1002, 1058, 1033, 1011, 1057, 978, 1053, 975, 987, 1017, 1061, 1034, 1011, 1061, 995, 1064, 1010, 1049, 1039, 1030, 1036, 958, 1033, 1011, 1010, 1050, 999, 1050, 982, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1040, 997, 1036, 1020, 1011, 996, 1037, 1036, 1044, 992, 1052, 1024, 1032, 998, 1018, 1070, 990, 1074, 1043, 1049, 975, 999, 982, 990, 1068, 1049, 1012, 1038, 1027, 1055, 1022, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1004, 1018, 1056, 988, 0, 0, 0, 0, 995, 1057, 1011, 1017, 0, 0, 0, 0, 1028, 1022, 1020, 980, 10, 2, 0, 0, 1029, 1008, 1032, 1043, 0, 0, 0, 0, 1035, 993, 994, 1073, 4, 4, 0, 0, 1035, 1008, 999, 1000, 18, 16, 4, 0, 1020, 1017, 1040, 1000, 0, 0, 0, 0, 1016, 987, 1085, 1100, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4080, 4062, 4112, 4103, 4080, 4077, 4188] avg=4096.0000 max=4188 max_rank=7 min=4062 min_rank=2 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1000, 1077, 1019, 1030, 0, 0, 0, 0, 1007, 1018, 1008, 1032, 0, 0, 0, 0, 1007, 995, 1011, 1012, 10, 2, 0, 0, 1007, 1051, 1085, 1013, 0, 0, 0, 0, 962, 1034, 1014, 1044, 4, 4, 0, 0, 1067, 1010, 1043, 1016, 19, 16, 4, 0, 1011, 1005, 1069, 1011, 0, 0, 0, 0, 1048, 1042, 960, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4065, 4037, 4156, 4062, 4175, 4096, 4051] avg=4096.0000 max=4175 max_rank=5 min=4037 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1019, 1008, 1012, 1047, 0, 0, 0, 0, 1066, 1003, 1020, 1012, 0, 0, 0, 0, 1001, 1031, 1031, 1016, 10, 2, 0, 0, 1068, 1032, 1037, 1011, 0, 0, 0, 0, 991, 1020, 1038, 1034, 4, 4, 0, 0, 1021, 1025, 963, 1033, 19, 16, 4, 0, 1011, 1005, 983, 1008, 0, 0, 0, 0, 1053, 1013, 1021, 1076, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4101, 4091, 4148, 4091, 4081, 4007, 4163] avg=4096.0000 max=4163 max_rank=7 min=4007 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1021, 1041, 1047, 1033, 0, 0, 0, 0, 1025, 1003, 1016, 1080, 0, 0, 0, 0, 1076, 1021, 1023, 1046, 10, 2, 0, 0, 987, 998, 1042, 989, 0, 0, 0, 0, 944, 1021, 1072, 1006, 4, 4, 0, 0, 1040, 995, 1005, 1016, 18, 16, 4, 0, 1007, 1025, 1054, 995, 0, 0, 0, 0, 1033, 1001, 1044, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4124, 4178, 4016, 4051, 4094, 4081, 4082] avg=4096.0000 max=4178 max_rank=2 min=4016 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 991, 1017, 992, 0, 0, 0, 0, 1061, 1027, 984, 1065, 0, 0, 0, 0, 1015, 1021, 1058, 1019, 14, 7, 0, 0, 1010, 1013, 995, 1036, 0, 0, 0, 0, 970, 1051, 1069, 1023, 9, 6, 0, 0, 988, 987, 1056, 1079, 22, 20, 10, 0, 1037, 1009, 1045, 1007, 0, 0, 0, 0, 1022, 975, 1006, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4137, 4134, 4054, 4128, 4162, 4098, 4029] avg=4096.0000 max=4162 max_rank=5 min=4026 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1022, 1059, 1014, 1034, 0, 0, 0, 0, 986, 1091, 1047, 1006, 0, 0, 0, 0, 1031, 997, 1012, 1046, 10, 2, 0, 0, 1064, 998, 991, 978, 0, 0, 0, 0, 1022, 1048, 1000, 965, 4, 4, 0, 0, 1018, 1027, 1017, 986, 19, 15, 5, 0, 1046, 1021, 1019, 1060, 0, 0, 0, 0, 1029, 1033, 1064, 978, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4130, 4098, 4031, 4043, 4087, 4146, 4104] avg=4096.0000 max=4146 max_rank=6 min=4031 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1027, 1025, 996, 1058, 0, 0, 0, 0, 1033, 1011, 1043, 978, 0, 0, 0, 0, 1053, 975, 987, 1017, 10, 2, 0, 0, 1061, 1034, 1011, 1044, 0, 0, 0, 0, 995, 1064, 1010, 1049, 4, 4, 0, 0, 1039, 1030, 1036, 958, 19, 17, 4, 0, 1029, 1011, 1010, 1050, 0, 0, 0, 0, 999, 1031, 982, 1062, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4065, 4044, 4150, 4126, 4103, 4100, 4074] avg=4096.0000 max=4150 max_rank=3 min=4044 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1040, 997, 1030, 1020, 0, 0, 0, 0, 1011, 996, 1023, 1036, 0, 0, 0, 0, 1044, 992, 1052, 1024, 10, 2, 0, 0, 1032, 998, 1018, 1053, 0, 0, 0, 0, 990, 1074, 1043, 1049, 4, 4, 0, 0, 975, 999, 982, 990, 19, 17, 4, 0, 1064, 1049, 1012, 1038, 0, 0, 0, 0, 1027, 1036, 1022, 992, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4066, 4124, 4101, 4164, 3986, 4163, 4077] avg=4096.0000 max=4164 max_rank=4 min=3986 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7936, 8320, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[990, 1021, 1020, 1010, 1000, 995, 998, 1028, 994, 1060, 1025, 1055, 1035, 1021, 1024, 1007, 1057, 978, 1068, 1057, 1036, 1020, 1033, 1025, 1003, 1026, 1082, 1031, 980, 1010, 1027, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[999, 1019, 1053, 1045, 1027, 1032, 1044, 1015, 1031, 1065, 995, 1020, 1022, 1008, 1065, 1051, 1019, 998, 1003, 1001, 1021, 1063, 1014, 1061, 1039, 994, 1085, 1000, 983, 963, 1042, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[951, 1006, 1013, 984, 1013, 1015, 1033, 1006, 1016, 1013, 1037, 1042, 1012, 1024, 1054, 1018, 1092, 1050, 1018, 1019, 1030, 1016, 1062, 971, 1008, 1058, 1080, 1030, 1046, 1013, 1001, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1033, 1025, 1030, 1005, 969, 1055, 993, 1054, 974, 1079, 1025, 1072, 1006, 1028, 976, 1057, 1027, 1031, 1006, 1066, 1019, 1042, 1083, 1050, 1016, 1027, 972, 1018, 1014, 989, 1025, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1061, 1045, 1007, 1049, 986, 1048, 1000, 990, 1050, 990, 998, 1036, 1043, 1051, 996, 1003, 1010, 1037, 974, 1018, 1032, 965, 1029, 1053, 1089, 1059, 997, 1030, 1067, 1012, 1034, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1023, 1024, 1009, 1009, 1045, 1002, 988, 1004, 1028, 1030, 1057, 1018, 1034, 1014, 1004, 1009, 1032, 987, 989, 1073, 1009, 1033, 1035, 991, 1045, 1092, 1008, 1049, 1042, 1017, 1018, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1052, 1055, 950, 1040, 1045, 997, 1029, 1036, 1041, 1001, 1025, 1002, 1055, 989, 1049, 1049, 1044, 1037, 1102, 1043, 1039, 1034, 949, 985, 994, 1029, 1033, 976, 1043, 1034, 1013, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[972, 1030, 1010, 1020, 1031, 1006, 1095, 982, 1008, 1036, 961, 984, 1025, 1053, 1020, 1017, 1028, 1049, 999, 1037, 1039, 1015, 1034, 1071, 1027, 979, 1060, 988, 1043, 1056, 1043, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[999, 1019, 1053, 1045, 21, 3, 0, 0, 1027, 1032, 1044, 1015, 16, 9, 0, 0, 1031, 1065, 995, 1020, 0, 0, 0, 0, 1016, 1008, 1065, 1051, 0, 0, 0, 0, 1019, 998, 1003, 980, 0, 0, 0, 0, 1021, 1063, 1004, 1061, 0, 0, 0, 0, 1039, 994, 1066, 1000, 0, 0, 0, 0, 983, 963, 1042, 991, 6, 1, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4143, 4111, 4140, 4000, 4149, 4099, 3986] avg=4096.0000 max=4149 max_rank=5 min=3986 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16234 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[990, 1021, 1020, 1010, 23, 3, 0, 0, 1000, 995, 998, 1028, 16, 9, 0, 0, 994, 1060, 1025, 1055, 0, 0, 0, 0, 1029, 1021, 1024, 1007, 0, 0, 0, 0, 1057, 978, 1068, 1034, 0, 0, 0, 0, 1036, 1020, 1023, 1025, 0, 0, 0, 0, 1003, 1026, 1063, 1031, 0, 0, 0, 0, 980, 1010, 1027, 1052, 6, 1, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4046, 4134, 4081, 4137, 4104, 4123, 4076] avg=4096.0000 max=4137 max_rank=4 min=4046 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16440 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[951, 1006, 1013, 984, 22, 3, 0, 0, 1013, 1015, 1033, 1006, 16, 10, 0, 0, 1016, 1013, 1037, 1042, 0, 0, 0, 0, 1006, 1024, 1054, 1018, 0, 0, 0, 0, 1092, 1050, 1018, 997, 0, 0, 0, 0, 1030, 1016, 1051, 971, 0, 0, 0, 0, 1008, 1058, 1061, 1030, 0, 0, 0, 0, 1046, 1013, 1001, 1037, 6, 1, 0, 0] rerouted_tokens_per_ep_rank=[3979, 4093, 4108, 4102, 4157, 4068, 4157, 4104] avg=4096.0000 max=4157 max_rank=4 min=3979 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16486 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1061, 1045, 1007, 1049, 22, 3, 0, 0, 986, 1048, 1000, 990, 15, 9, 0, 0, 1050, 990, 998, 1036, 0, 0, 0, 0, 1037, 1051, 996, 1003, 0, 0, 0, 0, 1010, 1037, 974, 996, 0, 0, 0, 0, 1032, 965, 1019, 1053, 0, 0, 0, 0, 1089, 1059, 979, 1030, 0, 0, 0, 0, 1067, 1012, 1034, 1009, 6, 1, 0, 0] rerouted_tokens_per_ep_rank=[4187, 4048, 4074, 4087, 4017, 4069, 4157, 4129] avg=4096.0000 max=4187 max_rank=0 min=4017 min_rank=4 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16372 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1023, 1024, 1009, 1009, 23, 3, 0, 0, 1045, 1002, 988, 1004, 15, 9, 0, 0, 1028, 1030, 1057, 1018, 0, 0, 0, 0, 1028, 1014, 1004, 1009, 0, 0, 0, 0, 1032, 987, 989, 1050, 0, 0, 0, 0, 1009, 1033, 1025, 991, 0, 0, 0, 0, 1045, 1092, 990, 1049, 0, 0, 0, 0, 1042, 1017, 1018, 1050, 6, 1, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4063, 4133, 4055, 4058, 4058, 4176, 4134] avg=4096.0000 max=4176 max_rank=6 min=4055 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16426 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[972, 1030, 1010, 1020, 25, 6, 0, 0, 1031, 1006, 1095, 982, 20, 14, 0, 0, 1008, 1036, 961, 984, 0, 0, 0, 0, 1016, 1053, 1020, 1017, 0, 0, 0, 0, 1028, 1049, 999, 1012, 0, 0, 0, 0, 1039, 1015, 1014, 1071, 0, 0, 0, 0, 1027, 979, 1034, 988, 0, 0, 0, 0, 1043, 1056, 1043, 1050, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4148, 3989, 4106, 4088, 4139, 4028, 4207] avg=4096.0000 max=4207 max_rank=7 min=3989 min_rank=2 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16462 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1052, 1055, 950, 1040, 22, 3, 0, 0, 1045, 997, 1029, 1036, 16, 8, 0, 0, 1041, 1001, 1025, 1002, 0, 0, 0, 0, 1049, 989, 1049, 1049, 0, 0, 0, 0, 1044, 1037, 1102, 1021, 0, 0, 0, 0, 1039, 1034, 940, 985, 0, 0, 0, 0, 994, 1029, 1014, 976, 0, 0, 0, 0, 1043, 1034, 1013, 998, 6, 1, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4131, 4069, 4136, 4204, 3998, 4013, 4095] avg=4096.0000 max=4204 max_rank=4 min=3998 min_rank=5 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16310 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1033, 1025, 1030, 1005, 23, 3, 0, 0, 969, 1055, 993, 1054, 15, 10, 0, 0, 974, 1079, 1025, 1072, 0, 0, 0, 0, 1000, 1028, 976, 1057, 0, 0, 0, 0, 1027, 1031, 1006, 1043, 0, 0, 0, 0, 1019, 1042, 1072, 1050, 0, 0, 0, 0, 1016, 1027, 954, 1018, 0, 0, 0, 0, 1014, 989, 1025, 1002, 6, 1, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4096, 4150, 4061, 4107, 4183, 4015, 4037] avg=4096.0000 max=4183 max_rank=5 min=4015 min_rank=6 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16342 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1086, 1014, 1017, 991, 1017, 1044, 1031, 1014, 1026, 1027, 1043, 1033, 1041, 1017, 1027, 1041, 1000, 988, 987, 951, 1035, 1035, 1012, 1046, 1045, 964, 1033, 1045, 1058, 1050, 1032, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1029, 1086, 1012, 1056, 980, 1018, 978, 1020, 1059, 1055, 994, 1017, 1047, 1025, 950, 1049, 1019, 979, 1045, 1039, 1019, 1011, 1064, 1042, 1000, 1036, 1021, 994, 1022, 1035, 1027, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1045, 1032, 967, 1002, 1057, 1016, 1013, 965, 984, 1034, 1045, 973, 1043, 992, 1025, 1050, 1092, 1085, 1115, 1039, 946, 1019, 1006, 1012, 1055, 982, 1003, 1004, 1022, 1054, 1063, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[975, 978, 1010, 1047, 1019, 1060, 1030, 1003, 971, 1058, 1004, 1045, 1049, 992, 1048, 1057, 1043, 1016, 1048, 1070, 1030, 1025, 1002, 995, 1011, 1038, 1039, 1038, 1020, 1019, 1004, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1041, 1029, 1044, 1010, 999, 1006, 1042, 1036, 995, 1026, 1028, 1019, 1032, 1012, 991, 1036, 1002, 1024, 1043, 1013, 999, 1002, 987, 1091, 1023, 1011, 1007, 1028, 1105, 987, 1047, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1033, 1044, 1046, 1067, 1048, 1049, 1013, 1026, 1018, 1032, 1031, 1039, 1047, 1019, 1029, 991, 1009, 1009, 997, 1055, 1038, 1077, 1016, 1024, 961, 1013, 1034, 962, 1028, 977, 1018, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1003, 1033, 1015, 1057, 1044, 987, 1033, 1036, 1017, 1007, 1070, 1018, 1015, 1044, 1055, 1014, 1019, 982, 1049, 1019, 985, 1064, 1020, 1022, 1018, 1050, 1035, 1016, 1008, 1005, 1010, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1031, 1052, 1025, 1013, 1036, 1041, 1051, 1036, 996, 1042, 1047, 1021, 999, 1002, 994, 1025, 1009, 977, 1022, 1038, 1014, 1058, 980, 1086, 1008, 1009, 1019, 1050, 1006, 998, 1073, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1086, 1000, 1017, 991, 0, 0, 0, 0, 1017, 1044, 1031, 1014, 1, 0, 0, 0, 1026, 1027, 1043, 1033, 0, 0, 0, 0, 1041, 1017, 1027, 1041, 1, 0, 0, 0, 1000, 988, 986, 951, 0, 0, 0, 0, 1035, 1035, 1012, 1046, 0, 0, 0, 0, 1045, 964, 1033, 1045, 14, 11, 0, 0, 1058, 1050, 1020, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4107, 4129, 4127, 3925, 4128, 4112, 4146] avg=4096.0000 max=4146 max_rank=7 min=3925 min_rank=4 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16311 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[975, 964, 1010, 1047, 0, 0, 0, 0, 1019, 1060, 1030, 1003, 1, 0, 0, 0, 971, 1058, 1004, 1045, 0, 0, 0, 0, 1049, 992, 1048, 1057, 1, 0, 0, 0, 1043, 1016, 1047, 1070, 0, 0, 0, 0, 1030, 1025, 1002, 995, 0, 0, 0, 0, 1011, 1038, 1039, 1038, 14, 11, 0, 0, 1020, 1019, 992, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4113, 4078, 4147, 4176, 4052, 4151, 4055] avg=4096.0000 max=4176 max_rank=4 min=3996 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16434 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1045, 1017, 967, 1002, 0, 0, 0, 0, 1057, 1016, 1013, 965, 1, 0, 0, 0, 984, 1034, 1045, 973, 0, 0, 0, 0, 1043, 992, 1025, 1050, 1, 0, 0, 0, 1092, 1085, 1114, 1039, 0, 0, 0, 0, 946, 1019, 1006, 1012, 0, 0, 0, 0, 1055, 982, 1003, 1004, 15, 12, 0, 0, 1022, 1054, 1050, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4052, 4036, 4111, 4330, 3983, 4071, 4154] avg=4096.0000 max=4330 max_rank=4 min=3983 min_rank=5 imbalance_ratio=1.057129 max_load_reduction=28438 max_load_reduction_pct=0.867859 moved_to_echo=16538 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1029, 1067, 1012, 1056, 0, 0, 0, 0, 980, 1018, 978, 1020, 8, 5, 0, 0, 1059, 1049, 994, 1017, 0, 0, 0, 0, 1047, 1025, 950, 1049, 3, 0, 0, 0, 1019, 979, 1037, 1039, 0, 0, 0, 0, 1019, 1011, 1064, 1042, 6, 0, 0, 0, 1000, 1036, 1021, 994, 19, 15, 0, 0, 1022, 1035, 1004, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4009, 4119, 4074, 4074, 4142, 4085, 4101] avg=4096.0000 max=4164 max_rank=0 min=4009 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16402 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1003, 1018, 1015, 1057, 0, 0, 0, 0, 1044, 987, 1033, 1036, 1, 0, 0, 0, 1017, 1007, 1070, 1018, 0, 0, 0, 0, 1015, 1044, 1055, 1014, 1, 0, 0, 0, 1019, 982, 1048, 1019, 0, 0, 0, 0, 985, 1064, 1020, 1022, 0, 0, 0, 0, 1018, 1050, 1035, 1016, 15, 11, 0, 0, 1008, 1005, 998, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4101, 4112, 4129, 4068, 4091, 4145, 4029] avg=4096.0000 max=4145 max_rank=6 min=4029 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16333 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1041, 1014, 1044, 1010, 0, 0, 0, 0, 999, 1006, 1042, 1036, 1, 0, 0, 0, 995, 1026, 1028, 1019, 0, 0, 0, 0, 1032, 1012, 991, 1036, 1, 0, 0, 0, 1002, 1024, 1042, 1013, 0, 0, 0, 0, 999, 1002, 987, 1091, 0, 0, 0, 0, 1023, 1011, 1007, 1028, 15, 11, 0, 0, 1105, 987, 1035, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4084, 4068, 4072, 4081, 4079, 4095, 4180] avg=4096.0000 max=4180 max_rank=7 min=4068 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16435 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1033, 1029, 1046, 1067, 0, 0, 0, 0, 1048, 1049, 1013, 1026, 1, 0, 0, 0, 1018, 1032, 1031, 1039, 0, 0, 0, 0, 1047, 1019, 1029, 991, 1, 0, 0, 0, 1009, 1009, 996, 1055, 0, 0, 0, 0, 1038, 1077, 1016, 1024, 0, 0, 0, 0, 961, 1013, 1034, 962, 15, 11, 0, 0, 1028, 977, 1006, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4137, 4120, 4087, 4069, 4155, 3996, 4029] avg=4096.0000 max=4175 max_rank=0 min=3996 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16249 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1031, 1037, 1025, 1013, 0, 0, 0, 0, 1036, 1041, 1051, 1036, 1, 0, 0, 0, 996, 1042, 1047, 1021, 0, 0, 0, 0, 999, 1002, 994, 1025, 1, 0, 0, 0, 1009, 977, 1021, 1038, 0, 0, 0, 0, 1014, 1058, 980, 1086, 0, 0, 0, 0, 1008, 1009, 1019, 1050, 15, 12, 0, 0, 1006, 998, 1060, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4165, 4106, 4021, 4045, 4138, 4113, 4074] avg=4096.0000 max=4165 max_rank=1 min=4021 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16370 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1032, 1070, 1059, 1055, 1031, 1035, 1067, 995, 987, 1045, 1057, 997, 983, 950, 1050, 999, 988, 1012, 1019, 1012, 1029, 1039, 1048, 1039, 1038, 1034, 986, 1051, 1046, 1046, 962, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1029, 1023, 987, 1007, 1004, 1045, 1006, 1021, 1042, 1031, 1094, 1033, 1048, 1003, 973, 1026, 1001, 977, 1040, 1022, 1029, 1009, 1085, 1047, 1022, 1018, 1017, 993, 1059, 1031, 1049, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1086, 1066, 994, 1025, 1001, 1011, 1052, 1009, 1088, 1027, 999, 988, 1071, 1025, 1049, 1051, 1024, 991, 1039, 993, 1017, 995, 1031, 1002, 1016, 1034, 1043, 1039, 997, 998, 1000, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1080, 1023, 1082, 1035, 1012, 1013, 1033, 1016, 977, 1018, 1012, 1053, 1016, 1070, 1031, 1000, 1001, 1017, 964, 970, 1043, 1029, 1017, 1043, 1023, 1006, 1001, 1084, 1016, 1020, 1035, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1027, 972, 1003, 1037, 1036, 1027, 1019, 995, 1046, 1044, 1057, 1074, 1000, 1062, 1055, 988, 1036, 1003, 1002, 1025, 1060, 1017, 1000, 1011, 1029, 1065, 1029, 1014, 978, 984, 1025, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1018, 1053, 1009, 1021, 1024, 1015, 1018, 1037, 1041, 1015, 1064, 989, 1054, 997, 1011, 1063, 1019, 1051, 1036, 1028, 972, 988, 1027, 1039, 1030, 1041, 1051, 1017, 1007, 1044, 1003, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1010, 1023, 1008, 1034, 1039, 1022, 1006, 1002, 1011, 1050, 1010, 1019, 1060, 1067, 983, 1079, 1045, 1031, 1061, 987, 966, 967, 1029, 1045, 1046, 1089, 1042, 978, 1021, 982, 1045, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1041, 1046, 1068, 993, 1054, 1026, 1038, 1006, 981, 1017, 1026, 1017, 1043, 1026, 1011, 1049, 1042, 1039, 991, 1040, 1064, 976, 993, 1082, 997, 991, 1019, 1010, 972, 1062, 1040, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1054, 1066, 994, 1025, 0, 0, 0, 0, 1001, 1011, 1052, 1009, 6, 0, 0, 0, 1088, 1027, 983, 988, 0, 0, 0, 0, 1055, 1025, 1049, 1051, 0, 0, 0, 0, 1024, 991, 1039, 993, 32, 1, 0, 0, 1017, 995, 1031, 1002, 3, 0, 0, 0, 1016, 1025, 1043, 1039, 0, 0, 0, 0, 997, 998, 1000, 1007, 15, 16, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4079, 4086, 4180, 4080, 4048, 4123, 4033] avg=4096.0000 max=4180 max_rank=3 min=4033 min_rank=7 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[988, 1053, 1009, 1021, 0, 0, 0, 0, 1024, 1015, 1018, 1037, 6, 0, 0, 0, 1041, 1015, 1047, 989, 0, 0, 0, 0, 1039, 997, 1011, 1063, 0, 0, 0, 0, 1019, 1051, 1036, 1028, 30, 1, 0, 0, 972, 988, 1027, 1039, 3, 0, 0, 0, 1030, 1032, 1051, 1017, 0, 0, 0, 0, 1007, 1044, 1003, 986, 16, 15, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4100, 4092, 4110, 4165, 4029, 4130, 4071] avg=4096.0000 max=4165 max_rank=4 min=4029 min_rank=5 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1002, 1070, 1059, 1055, 0, 0, 0, 0, 1031, 1035, 1067, 995, 6, 0, 0, 0, 987, 1045, 1040, 997, 0, 0, 0, 0, 969, 950, 1050, 999, 0, 0, 0, 0, 988, 1012, 1019, 1012, 30, 1, 0, 0, 1029, 1039, 1048, 1039, 3, 0, 0, 0, 1038, 1025, 986, 1051, 0, 0, 0, 0, 1046, 1046, 962, 1007, 16, 14, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4134, 4069, 3968, 4062, 4158, 4100, 4091] avg=4096.0000 max=4186 max_rank=0 min=3968 min_rank=3 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[997, 972, 1003, 1037, 0, 0, 0, 0, 1036, 1027, 1019, 995, 6, 0, 0, 0, 1046, 1044, 1040, 1074, 0, 0, 0, 0, 985, 1062, 1055, 988, 0, 0, 0, 0, 1036, 1003, 1002, 1025, 30, 1, 0, 0, 1060, 1017, 1000, 1011, 3, 0, 0, 0, 1029, 1056, 1029, 1014, 0, 0, 0, 0, 978, 984, 1025, 1048, 16, 15, 0, 0] rerouted_tokens_per_ep_rank=[4009, 4083, 4204, 4090, 4097, 4091, 4128, 4066] avg=4096.0000 max=4204 max_rank=2 min=4009 min_rank=0 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1048, 1023, 1082, 1035, 0, 0, 0, 0, 1012, 1013, 1033, 1016, 6, 0, 0, 0, 977, 1018, 996, 1053, 0, 0, 0, 0, 1001, 1070, 1031, 1000, 0, 0, 0, 0, 1001, 1017, 964, 970, 32, 1, 0, 0, 1043, 1029, 1017, 1043, 3, 0, 0, 0, 1023, 997, 1001, 1084, 0, 0, 0, 0, 1016, 1020, 1035, 1028, 15, 15, 0, 0] rerouted_tokens_per_ep_rank=[4188, 4080, 4044, 4102, 3985, 4135, 4105, 4129] avg=4096.0000 max=4188 max_rank=0 min=3985 min_rank=4 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[999, 1023, 987, 1007, 0, 0, 0, 0, 1004, 1045, 1006, 1021, 6, 0, 0, 0, 1042, 1031, 1077, 1033, 0, 0, 0, 0, 1033, 1003, 973, 1026, 0, 0, 0, 0, 1001, 977, 1040, 1022, 30, 1, 0, 0, 1029, 1009, 1085, 1047, 3, 0, 0, 0, 1022, 1009, 1017, 993, 0, 0, 0, 0, 1059, 1031, 1049, 997, 16, 15, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4082, 4183, 4035, 4071, 4173, 4041, 4167] avg=4096.0000 max=4183 max_rank=2 min=4016 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[977, 1023, 1008, 1034, 0, 0, 0, 0, 1039, 1022, 1006, 1002, 11, 0, 0, 0, 1011, 1050, 985, 1019, 0, 0, 0, 0, 1040, 1067, 983, 1079, 0, 0, 0, 0, 1045, 1031, 1061, 987, 33, 7, 0, 0, 966, 967, 1029, 1045, 9, 0, 0, 0, 1046, 1067, 1042, 978, 0, 0, 0, 0, 1021, 982, 1045, 1011, 18, 20, 2, 0] rerouted_tokens_per_ep_rank=[4042, 4080, 4065, 4169, 4164, 4016, 4133, 4099] avg=4096.0000 max=4169 max_rank=3 min=4016 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1010, 1046, 1068, 993, 0, 0, 0, 0, 1054, 1026, 1038, 1006, 6, 0, 0, 0, 981, 1017, 1010, 1017, 0, 0, 0, 0, 1028, 1026, 1011, 1049, 0, 0, 0, 0, 1042, 1039, 991, 1040, 31, 1, 0, 0, 1064, 976, 993, 1082, 3, 0, 0, 0, 997, 982, 1019, 1010, 0, 0, 0, 0, 972, 1062, 1040, 1008, 15, 15, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4130, 4025, 4114, 4144, 4118, 4008, 4112] avg=4096.0000 max=4144 max_rank=4 min=4008 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1002, 1022, 1069, 1057, 999, 997, 1039, 1045, 1043, 1027, 1045, 1001, 1063, 997, 1005, 1013, 991, 1078, 994, 1035, 1014, 972, 1051, 1020, 1031, 1035, 1015, 1018, 1011, 1044, 1012, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1040, 1016, 1059, 1023, 1020, 1055, 1007, 1019, 1002, 1042, 1000, 1038, 1058, 1003, 1049, 1035, 1043, 1026, 1031, 992, 1036, 1045, 1054, 1012, 1032, 1020, 1028, 1001, 1001, 964, 1003, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1053, 978, 1018, 1042, 1031, 1029, 979, 1044, 1048, 1064, 1027, 1008, 1075, 1001, 1022, 1039, 1038, 1030, 1004, 994, 1025, 977, 1031, 1049, 1020, 1013, 1037, 1054, 1032, 992, 1008, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[996, 1048, 1030, 986, 995, 1038, 1058, 1013, 1026, 1035, 965, 1035, 1021, 1051, 1013, 1080, 1031, 981, 1025, 1010, 1038, 1014, 1030, 1089, 1032, 986, 1059, 1055, 1028, 1003, 1003, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[994, 1021, 998, 1086, 1006, 1005, 1064, 1012, 978, 1043, 999, 1008, 1011, 1022, 1034, 1007, 1035, 1030, 1027, 1080, 1033, 1036, 1035, 997, 1043, 1034, 1055, 1006, 1049, 1036, 995, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1040, 985, 1035, 1095, 1026, 984, 1023, 1030, 1027, 1035, 1050, 1004, 996, 1048, 1016, 1026, 1040, 1034, 1024, 1039, 1006, 983, 1023, 1004, 1066, 1009, 1028, 1041, 1038, 1028, 987, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1002, 1027, 979, 1041, 1016, 1004, 1017, 1065, 1043, 1080, 1087, 1073, 1006, 1053, 1034, 1079, 1041, 1004, 1009, 1048, 1014, 1029, 1014, 984, 1011, 1026, 954, 1008, 983, 1003, 1026, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1026, 1049, 1040, 1036, 1039, 1021, 981, 992, 962, 980, 1082, 1000, 1035, 1007, 1018, 1040, 1001, 994, 1038, 1005, 1065, 1028, 976, 1045, 976, 1019, 1032, 1075, 1001, 1108, 1082, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1002, 1022, 1069, 1042, 0, 0, 0, 0, 999, 997, 1039, 1045, 10, 3, 0, 0, 1043, 1017, 1045, 1001, 0, 0, 0, 0, 1063, 997, 1005, 990, 0, 0, 0, 0, 991, 1078, 994, 1035, 1, 0, 0, 0, 1014, 972, 1051, 1020, 4, 0, 0, 0, 1031, 1035, 1015, 1013, 0, 0, 0, 0, 1011, 1044, 1012, 1023, 23, 12, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4093, 4106, 4055, 4099, 4061, 4094, 4125] avg=4096.0000 max=4135 max_rank=0 min=4055 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1053, 978, 1018, 1028, 0, 0, 0, 0, 1031, 1029, 979, 1044, 10, 3, 0, 0, 1048, 1054, 1027, 1008, 0, 0, 0, 0, 1075, 1001, 1022, 1016, 0, 0, 0, 0, 1038, 1030, 1004, 994, 2, 0, 0, 0, 1025, 977, 1031, 1049, 4, 0, 0, 0, 1020, 1013, 1037, 1048, 0, 0, 0, 0, 1032, 992, 1008, 1006, 23, 11, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4096, 4137, 4114, 4068, 4086, 4118, 4072] avg=4096.0000 max=4137 max_rank=2 min=4068 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1040, 1016, 1059, 1009, 0, 0, 0, 0, 1020, 1055, 1007, 1019, 10, 3, 0, 0, 1002, 1032, 1000, 1038, 0, 0, 0, 0, 1058, 1003, 1049, 1012, 0, 0, 0, 0, 1043, 1026, 1031, 992, 1, 0, 0, 0, 1036, 1045, 1054, 1012, 4, 0, 0, 0, 1032, 1020, 1028, 996, 0, 0, 0, 0, 1001, 964, 1003, 1014, 23, 11, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4114, 4072, 4122, 4093, 4151, 4076, 4016] avg=4096.0000 max=4151 max_rank=5 min=4016 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[996, 1048, 1030, 972, 0, 0, 0, 0, 995, 1038, 1058, 1013, 10, 3, 0, 0, 1026, 1025, 965, 1035, 0, 0, 0, 0, 1021, 1051, 1013, 1056, 0, 0, 0, 0, 1031, 981, 1025, 1010, 2, 0, 0, 0, 1038, 1014, 1030, 1089, 4, 0, 0, 0, 1032, 986, 1059, 1049, 0, 0, 0, 0, 1028, 1003, 1003, 994, 24, 11, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4117, 4051, 4141, 4049, 4175, 4126, 4063] avg=4096.0000 max=4175 max_rank=5 min=4046 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[994, 1021, 998, 1061, 0, 0, 0, 0, 1006, 1005, 1064, 1012, 14, 9, 0, 0, 978, 1025, 999, 1008, 0, 0, 0, 0, 1011, 1022, 1034, 981, 0, 0, 0, 0, 1035, 1030, 1027, 1080, 5, 0, 0, 0, 1033, 1036, 1035, 997, 7, 4, 0, 0, 1043, 1034, 1055, 994, 0, 0, 0, 0, 1049, 1036, 995, 989, 26, 16, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4110, 4010, 4048, 4177, 4112, 4126, 4111] avg=4096.0000 max=4177 max_rank=4 min=4010 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16526 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 1049, 1040, 1022, 0, 0, 0, 0, 1039, 1021, 981, 992, 10, 3, 0, 0, 962, 970, 1082, 1000, 0, 0, 0, 0, 1035, 1007, 1018, 1017, 0, 0, 0, 0, 1001, 994, 1038, 1005, 2, 0, 0, 0, 1065, 1028, 976, 1045, 4, 0, 0, 0, 976, 1019, 1032, 1069, 0, 0, 0, 0, 1001, 1108, 1082, 1015, 23, 11, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4046, 4014, 4077, 4040, 4118, 4096, 4240] avg=4096.0000 max=4240 max_rank=7 min=4014 min_rank=2 imbalance_ratio=1.035156 max_load_reduction=28528 max_load_reduction_pct=0.870605 moved_to_echo=16494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1040, 985, 1035, 1080, 0, 0, 0, 0, 1026, 984, 1023, 1030, 10, 3, 0, 0, 1027, 1025, 1050, 1004, 0, 0, 0, 0, 996, 1048, 1016, 1003, 0, 0, 0, 0, 1040, 1034, 1024, 1039, 2, 0, 0, 0, 1006, 983, 1023, 1004, 4, 0, 0, 0, 1066, 1009, 1028, 1035, 0, 0, 0, 0, 1038, 1028, 987, 998, 23, 12, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4076, 4106, 4063, 4139, 4020, 4138, 4086] avg=4096.0000 max=4140 max_rank=0 min=4020 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1002, 1027, 979, 1027, 0, 0, 0, 0, 1016, 1004, 1017, 1065, 11, 3, 0, 0, 1043, 1069, 1087, 1073, 0, 0, 0, 0, 1006, 1053, 1034, 1055, 0, 0, 0, 0, 1041, 1004, 1009, 1048, 1, 0, 0, 0, 1014, 1029, 1014, 984, 4, 0, 0, 0, 1011, 1026, 954, 1003, 0, 0, 0, 0, 983, 1003, 1026, 1008, 24, 11, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4116, 4272, 4148, 4103, 4045, 3994, 4055] avg=4096.0000 max=4272 max_rank=2 min=3994 min_rank=6 imbalance_ratio=1.042969 max_load_reduction=28496 max_load_reduction_pct=0.869629 moved_to_echo=16197 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[970, 999, 1050, 1003, 969, 1059, 1012, 1054, 1053, 1042, 1036, 1030, 1065, 958, 1026, 986, 1034, 1031, 1027, 1007, 1021, 1023, 1025, 1019, 1011, 1037, 1040, 1059, 1020, 1046, 999, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1013, 1037, 1035, 1010, 987, 1026, 1031, 997, 1014, 1003, 1059, 985, 1058, 988, 1016, 991, 998, 1014, 1063, 1016, 1073, 1018, 994, 1072, 1057, 1062, 1092, 1030, 1005, 967, 1023, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1047, 1007, 1067, 1024, 1016, 1048, 1044, 1019, 1039, 1047, 994, 1045, 1047, 993, 1026, 1043, 970, 1021, 1020, 1002, 971, 1040, 981, 1028, 1020, 995, 1040, 1030, 1034, 1045, 1028, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1056, 1003, 1017, 1010, 1011, 1048, 1034, 1040, 1066, 1016, 1000, 1042, 984, 1047, 1010, 985, 1044, 1003, 1046, 1017, 1002, 1024, 1003, 1001, 1041, 1049, 1069, 1024, 1023, 998, 1051, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1004, 1034, 987, 1014, 1019, 1042, 1017, 1032, 1014, 1049, 1019, 1030, 1017, 996, 973, 1047, 1041, 1006, 1000, 1023, 1068, 1036, 1008, 999, 993, 1021, 1080, 1043, 1020, 1015, 1095, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1028, 1048, 1050, 1029, 1021, 1044, 1015, 982, 1054, 1049, 1002, 991, 1009, 1060, 1006, 1018, 1038, 1047, 998, 1005, 1055, 997, 1074, 996, 1038, 1012, 1047, 1004, 1012, 974, 1029, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1005, 1058, 988, 1032, 1059, 1017, 1054, 1017, 995, 1048, 1016, 1036, 1017, 965, 1041, 988, 994, 1029, 1013, 1039, 1061, 1042, 1039, 996, 1033, 1022, 1057, 993, 1043, 1043, 1030, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1043, 1014, 1013, 1063, 1019, 992, 1015, 1006, 1052, 1038, 1009, 1033, 1052, 961, 1067, 1035, 1057, 1011, 1030, 997, 960, 1067, 1044, 1061, 970, 1064, 1019, 1003, 1020, 1080, 1018, 955] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1047, 1007, 1067, 1024, 1, 0, 0, 0, 1016, 1048, 1044, 1019, 2, 0, 0, 0, 1039, 1031, 994, 1045, 0, 0, 0, 0, 1047, 993, 1026, 1043, 35, 0, 0, 0, 970, 1021, 1020, 1002, 16, 0, 0, 0, 971, 1037, 981, 1028, 0, 0, 0, 0, 1020, 995, 1005, 1030, 0, 0, 0, 0, 1034, 1045, 1028, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4129, 4109, 4144, 4029, 4017, 4050, 4144] avg=4096.0000 max=4146 max_rank=0 min=4017 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1013, 1037, 1035, 1010, 1, 0, 0, 0, 987, 1026, 1031, 997, 2, 0, 0, 0, 1014, 988, 1059, 985, 0, 0, 0, 0, 1058, 988, 1016, 991, 37, 0, 0, 0, 998, 1014, 1063, 1016, 15, 0, 0, 0, 1073, 1015, 994, 1072, 0, 0, 0, 0, 1057, 1062, 1055, 1030, 0, 0, 0, 0, 1005, 967, 1023, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4043, 4046, 4090, 4106, 4154, 4204, 4029] avg=4096.0000 max=4204 max_rank=6 min=4029 min_rank=7 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[970, 999, 1050, 1003, 1, 0, 0, 0, 969, 1059, 1012, 1054, 2, 0, 0, 0, 1053, 1027, 1036, 1030, 0, 0, 0, 0, 1065, 958, 1026, 986, 35, 0, 0, 0, 1034, 1031, 1027, 1007, 15, 0, 0, 0, 1021, 1020, 1025, 1019, 0, 0, 0, 0, 1011, 1037, 1005, 1059, 0, 0, 0, 0, 1020, 1046, 999, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4096, 4146, 4070, 4114, 4085, 4112, 4122] avg=4096.0000 max=4146 max_rank=2 min=4023 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1028, 1048, 1050, 1029, 1, 0, 0, 0, 1021, 1044, 1015, 982, 2, 0, 0, 0, 1054, 1033, 1002, 991, 0, 0, 0, 0, 1009, 1060, 1006, 1018, 35, 0, 0, 0, 1038, 1047, 998, 1005, 16, 0, 0, 0, 1055, 994, 1074, 996, 0, 0, 0, 0, 1038, 1012, 1012, 1004, 0, 0, 0, 0, 1012, 974, 1029, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4064, 4080, 4128, 4104, 4119, 4066, 4051] avg=4096.0000 max=4156 max_rank=0 min=4051 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1004, 1034, 987, 1014, 1, 0, 0, 0, 1019, 1042, 1017, 1032, 2, 0, 0, 0, 1014, 1033, 1019, 1030, 0, 0, 0, 0, 1017, 996, 973, 1047, 36, 0, 0, 0, 1041, 1006, 1000, 1023, 16, 0, 0, 0, 1068, 1033, 1008, 999, 0, 0, 0, 0, 993, 1021, 1044, 1043, 0, 0, 0, 0, 1020, 1015, 1095, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4112, 4096, 4069, 4086, 4108, 4101, 4156] avg=4096.0000 max=4156 max_rank=7 min=4040 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1056, 1003, 1017, 1010, 3, 0, 0, 0, 1011, 1048, 1034, 1040, 3, 5, 0, 0, 1066, 987, 1000, 1042, 0, 0, 0, 0, 984, 1047, 1010, 985, 40, 6, 0, 0, 1044, 1003, 1046, 1017, 18, 0, 0, 0, 1002, 1015, 1003, 1001, 0, 0, 0, 0, 1041, 1049, 1029, 1024, 0, 0, 0, 0, 1023, 998, 1051, 1004, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4141, 4095, 4072, 4128, 4021, 4143, 4079] avg=4096.0000 max=4143 max_rank=6 min=4021 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1005, 1058, 988, 1032, 1, 0, 0, 0, 1059, 1017, 1054, 1017, 2, 0, 0, 0, 995, 1032, 1016, 1036, 0, 0, 0, 0, 1017, 965, 1041, 988, 35, 0, 0, 0, 994, 1029, 1013, 1039, 16, 0, 0, 0, 1061, 1039, 1039, 996, 0, 0, 0, 0, 1033, 1022, 1022, 993, 0, 0, 0, 0, 1043, 1043, 1030, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4149, 4079, 4046, 4091, 4135, 4070, 4114] avg=4096.0000 max=4149 max_rank=1 min=4046 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1043, 1014, 1013, 1063, 1, 0, 0, 0, 1019, 992, 1015, 1006, 2, 0, 0, 0, 1052, 1023, 1009, 1033, 0, 0, 0, 0, 1052, 961, 1067, 1035, 34, 0, 0, 0, 1057, 1011, 1030, 997, 15, 0, 0, 0, 960, 1064, 1044, 1061, 0, 0, 0, 0, 970, 1064, 985, 1003, 0, 0, 0, 0, 1020, 1080, 1018, 955, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4034, 4117, 4149, 4110, 4129, 4022, 4073] avg=4096.0000 max=4149 max_rank=3 min=4022 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1047, 1079, 1022, 1061, 1019, 1040, 1027, 1037, 970, 984, 1038, 972, 986, 1012, 1041, 1051, 1008, 1101, 1034, 1021, 1014, 988, 1031, 1052, 1002, 1028, 1025, 1035, 1000, 1003, 1013, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1051, 997, 991, 1037, 1065, 989, 1079, 1015, 1039, 1055, 1006, 983, 992, 1042, 1033, 1010, 991, 1039, 1017, 1033, 1035, 1068, 994, 1004, 986, 1008, 991, 1052, 1038, 1043, 1072, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1031, 1032, 1046, 1039, 1008, 1018, 1032, 991, 1025, 989, 1040, 1006, 1013, 994, 1032, 1001, 981, 1048, 1079, 1029, 1027, 1038, 1059, 1056, 1048, 1045, 1009, 1004, 1054, 1028, 998, 968] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1011, 1049, 1043, 981, 1053, 1048, 981, 1062, 1054, 1037, 1118, 1005, 1007, 1023, 1061, 1000, 1031, 1038, 1013, 969, 972, 1050, 1040, 1011, 971, 1028, 1028, 991, 1052, 968, 1059, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1008, 1031, 1008, 1051, 1025, 1023, 1021, 1017, 1067, 1040, 1031, 1013, 1041, 986, 1039, 1005, 1010, 1013, 1010, 1002, 1003, 1019, 1025, 1051, 1029, 1032, 1044, 991, 1066, 986, 1055, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1040, 1008, 1034, 1038, 986, 1000, 1021, 1003, 1022, 1037, 1042, 1034, 980, 997, 1050, 1057, 999, 1039, 1077, 1058, 1035, 1032, 1062, 992, 1052, 1057, 1046, 1001, 999, 993, 1000, 977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1020, 979, 1015, 1045, 1046, 1060, 1042, 1046, 1040, 985, 998, 1059, 1048, 1049, 1029, 1014, 1036, 1010, 1055, 1011, 1005, 1023, 1025, 1036, 996, 1025, 1010, 1003, 1017, 1065, 972, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1010, 1046, 1044, 1077, 1004, 1044, 973, 1044, 973, 1077, 1018, 1017, 1020, 1027, 998, 975, 1043, 1018, 994, 1067, 1068, 1042, 1071, 1006, 1007, 1013, 1021, 985, 1001, 1047, 1031, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1047, 1079, 1022, 1036, 0, 0, 0, 0, 1019, 1034, 1027, 1037, 0, 0, 0, 0, 970, 984, 1038, 972, 0, 0, 0, 0, 986, 1012, 1041, 1051, 12, 6, 0, 0, 1008, 1088, 1034, 1021, 0, 0, 0, 0, 1014, 988, 1011, 1052, 0, 0, 0, 0, 1002, 1028, 1025, 1035, 25, 0, 0, 0, 1000, 1003, 1013, 1027, 20, 1, 0, 0] rerouted_tokens_per_ep_rank=[4184, 4117, 3964, 4108, 4151, 4065, 4115, 4064] avg=4096.0000 max=4184 max_rank=0 min=3964 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1051, 997, 991, 1012, 0, 0, 0, 0, 1065, 983, 1079, 1015, 0, 0, 0, 0, 1039, 1055, 1006, 983, 0, 0, 0, 0, 992, 1042, 1033, 1010, 12, 6, 0, 0, 991, 1026, 1017, 1033, 0, 0, 0, 0, 1035, 1068, 975, 1004, 0, 0, 0, 0, 986, 1008, 991, 1052, 25, 0, 0, 0, 1038, 1043, 1072, 1013, 19, 1, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4142, 4083, 4095, 4067, 4082, 4062, 4186] avg=4096.0000 max=4186 max_rank=7 min=4051 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1031, 1032, 1046, 1014, 0, 0, 0, 0, 1008, 1012, 1032, 991, 0, 0, 0, 0, 1025, 989, 1040, 1006, 0, 0, 0, 0, 1013, 994, 1032, 1001, 12, 6, 0, 0, 981, 1035, 1079, 1029, 0, 0, 0, 0, 1027, 1038, 1039, 1056, 0, 0, 0, 0, 1048, 1045, 1009, 1004, 25, 0, 0, 0, 1054, 1028, 998, 968, 20, 1, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4043, 4060, 4058, 4124, 4160, 4131, 4069] avg=4096.0000 max=4160 max_rank=5 min=4043 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1011, 1049, 1043, 958, 0, 0, 0, 0, 1053, 1042, 981, 1062, 0, 0, 0, 0, 1054, 1037, 1118, 1005, 0, 0, 0, 0, 1007, 1023, 1061, 1000, 12, 6, 0, 0, 1031, 1026, 1013, 969, 0, 0, 0, 0, 972, 1050, 1020, 1011, 0, 0, 0, 0, 971, 1028, 1028, 991, 23, 0, 0, 0, 1052, 968, 1059, 1014, 20, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4138, 4214, 4109, 4039, 4053, 4041, 4113] avg=4096.0000 max=4214 max_rank=2 min=4039 min_rank=4 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1008, 1031, 1008, 1026, 0, 0, 0, 0, 1025, 1017, 1021, 1017, 0, 0, 0, 0, 1067, 1040, 1031, 1013, 0, 0, 0, 0, 1041, 986, 1039, 1005, 11, 6, 0, 0, 1010, 1002, 1010, 1002, 0, 0, 0, 0, 1003, 1019, 1005, 1051, 0, 0, 0, 0, 1029, 1032, 1044, 991, 25, 0, 0, 0, 1066, 986, 1055, 1026, 20, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4080, 4151, 4088, 4024, 4078, 4121, 4153] avg=4096.0000 max=4153 max_rank=7 min=4024 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1020, 979, 1015, 1016, 0, 0, 0, 0, 1046, 1051, 1042, 1046, 0, 0, 0, 0, 1040, 985, 992, 1059, 0, 0, 0, 0, 1048, 1049, 1029, 1014, 15, 9, 6, 0, 1036, 991, 1055, 1011, 0, 0, 0, 0, 1005, 1023, 999, 1036, 0, 0, 0, 0, 996, 1025, 1010, 1003, 29, 2, 0, 0, 1017, 1065, 972, 1004, 24, 4, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4185, 4076, 4170, 4093, 4063, 4065, 4086] avg=4096.0000 max=4185 max_rank=1 min=4030 min_rank=0 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1040, 1008, 1034, 1013, 0, 0, 0, 0, 986, 994, 1021, 1003, 0, 0, 0, 0, 1022, 1037, 1042, 1034, 0, 0, 0, 0, 980, 997, 1050, 1057, 12, 6, 0, 0, 999, 1026, 1077, 1058, 0, 0, 0, 0, 1035, 1032, 1042, 992, 0, 0, 0, 0, 1052, 1057, 1046, 1001, 25, 0, 0, 0, 999, 993, 1000, 977, 20, 1, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4004, 4135, 4102, 4160, 4101, 4181, 3990] avg=4096.0000 max=4181 max_rank=6 min=3990 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1010, 1046, 1044, 1051, 0, 0, 0, 0, 1004, 1038, 973, 1044, 0, 0, 0, 0, 973, 1077, 1018, 1017, 0, 0, 0, 0, 1020, 1027, 998, 975, 12, 6, 0, 0, 1043, 1006, 994, 1067, 0, 0, 0, 0, 1068, 1042, 1050, 1006, 0, 0, 0, 0, 1007, 1013, 1021, 985, 26, 0, 0, 0, 1001, 1047, 1031, 1007, 21, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4059, 4085, 4038, 4110, 4166, 4052, 4107] avg=4096.0000 max=4166 max_rank=5 min=4038 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1059, 1025, 1012, 1055, 1006, 1016, 1014, 1017, 1025, 1030, 1039, 1026, 1010, 1004, 1050, 980, 1037, 1028, 1055, 963, 989, 1055, 1018, 1060, 1030, 1012, 1051, 1034, 1014, 1024, 1022, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[968, 1035, 1037, 996, 1080, 1026, 1015, 1008, 1000, 954, 1020, 1012, 1034, 1022, 1013, 967, 998, 1010, 1057, 1030, 1063, 1033, 1051, 1040, 1014, 1054, 1099, 1033, 1029, 1042, 996, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1062, 992, 981, 1038, 983, 1060, 997, 1015, 1013, 1008, 1049, 1009, 1040, 971, 1002, 1040, 1029, 1008, 1053, 971, 1039, 1043, 1020, 1042, 1067, 1058, 1059, 1054, 1024, 1024, 1026, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1019, 1062, 1046, 977, 1050, 1031, 1015, 1060, 1032, 1004, 1001, 1024, 1051, 1086, 1016, 967, 988, 967, 1013, 1057, 986, 1048, 1021, 987, 1053, 1021, 1044, 1022, 1002, 1036, 1030, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1011, 1052, 1019, 1057, 1042, 1057, 1027, 1030, 1001, 1028, 1034, 992, 1056, 1017, 1011, 1065, 997, 1062, 986, 986, 1045, 970, 1019, 1078, 1007, 1030, 976, 1023, 1014, 1059, 1016, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1073, 1013, 1042, 974, 999, 1047, 972, 1044, 981, 1042, 1013, 1049, 1011, 1029, 996, 1020, 1048, 1078, 1023, 1035, 1051, 989, 1005, 985, 1003, 1012, 1012, 1016, 1063, 1067, 1027, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1051, 1023, 1014, 1009, 1051, 1046, 1042, 989, 1051, 1005, 1028, 1050, 1053, 1012, 1055, 995, 1023, 1039, 1050, 1021, 1011, 1066, 1030, 989, 984, 1022, 969, 974, 1026, 1045, 999, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1015, 1039, 1040, 1032, 1017, 1048, 1002, 951, 983, 1006, 998, 1057, 1011, 989, 1023, 1029, 986, 991, 1062, 1038, 1045, 1081, 1053, 1032, 1040, 1047, 1036, 1024, 1001, 1076, 1006, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1012, 1062, 1046, 977, 0, 0, 0, 0, 1050, 1031, 1015, 1060, 1, 0, 0, 0, 1032, 1004, 1001, 1024, 22, 3, 0, 0, 1051, 1086, 1016, 967, 10, 7, 0, 0, 988, 967, 1013, 1057, 6, 3, 0, 0, 986, 1026, 1021, 987, 0, 0, 0, 0, 1053, 1008, 1044, 1022, 0, 0, 0, 0, 1002, 1026, 1030, 1052, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4157, 4086, 4137, 4034, 4020, 4127, 4110] avg=4096.0000 max=4157 max_rank=1 min=4020 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[962, 1035, 1037, 996, 0, 0, 0, 0, 1080, 1026, 1015, 1008, 1, 0, 0, 0, 1000, 954, 1020, 1012, 21, 3, 0, 0, 1034, 1022, 1013, 967, 10, 7, 0, 0, 998, 1010, 1057, 1030, 5, 3, 0, 0, 1063, 1012, 1051, 1040, 0, 0, 0, 0, 1014, 1041, 1099, 1033, 0, 0, 0, 0, 1029, 1032, 996, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4130, 4010, 4053, 4103, 4166, 4187, 4089] avg=4096.0000 max=4187 max_rank=6 min=4010 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16545 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1055, 992, 981, 1038, 0, 0, 0, 0, 983, 1060, 997, 1015, 1, 0, 0, 0, 1013, 1008, 1049, 1009, 22, 3, 0, 0, 1040, 971, 1002, 1040, 10, 7, 0, 0, 1029, 1008, 1053, 971, 6, 3, 0, 0, 1039, 1021, 1020, 1042, 0, 0, 0, 0, 1067, 1045, 1059, 1054, 0, 0, 0, 0, 1024, 1014, 1026, 991, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4056, 4104, 4070, 4070, 4122, 4225, 4055] avg=4096.0000 max=4225 max_rank=6 min=4055 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1052, 1025, 1012, 1055, 0, 0, 0, 0, 1006, 1016, 1014, 1017, 1, 0, 0, 0, 1025, 1030, 1039, 1026, 22, 3, 0, 0, 1010, 1004, 1050, 980, 10, 7, 0, 0, 1037, 1028, 1055, 963, 6, 3, 0, 0, 989, 1033, 1018, 1060, 0, 0, 0, 0, 1030, 999, 1051, 1034, 0, 0, 0, 0, 1014, 1014, 1022, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4054, 4145, 4061, 4092, 4100, 4114, 4058] avg=4096.0000 max=4145 max_rank=2 min=4054 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[999, 1052, 1019, 1057, 0, 0, 0, 0, 1042, 1057, 1027, 1030, 4, 0, 0, 0, 1001, 1028, 1034, 992, 24, 7, 0, 0, 1056, 1017, 1011, 1065, 14, 10, 0, 0, 997, 1062, 986, 986, 8, 9, 0, 0, 1045, 946, 1019, 1078, 0, 0, 0, 0, 1007, 1009, 976, 1023, 0, 0, 0, 0, 1014, 1040, 1016, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4160, 4086, 4173, 4048, 4088, 4015, 4071] avg=4096.0000 max=4173 max_rank=3 min=4015 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16222 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1066, 1013, 1042, 974, 0, 0, 0, 0, 999, 1047, 972, 1044, 1, 0, 0, 0, 981, 1042, 1013, 1049, 21, 3, 0, 0, 1011, 1029, 996, 1020, 10, 7, 0, 0, 1048, 1078, 1023, 1035, 6, 3, 0, 0, 1051, 968, 1005, 985, 0, 0, 0, 0, 1003, 999, 1012, 1016, 0, 0, 0, 0, 1063, 1057, 1027, 1049, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4063, 4109, 4073, 4193, 4009, 4030, 4196] avg=4096.0000 max=4196 max_rank=7 min=4009 min_rank=5 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1044, 1023, 1014, 1009, 0, 0, 0, 0, 1051, 1046, 1042, 989, 1, 0, 0, 0, 1051, 1005, 1028, 1050, 22, 3, 0, 0, 1053, 1012, 1055, 995, 10, 7, 0, 0, 1023, 1039, 1050, 1021, 6, 3, 0, 0, 1011, 1044, 1030, 989, 0, 0, 0, 0, 984, 1009, 969, 974, 0, 0, 0, 0, 1026, 1035, 999, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4129, 4159, 4132, 4142, 4074, 3936, 4106] avg=4096.0000 max=4159 max_rank=2 min=3936 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16258 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1008, 1039, 1040, 1032, 0, 0, 0, 0, 1017, 1048, 1002, 951, 1, 0, 0, 0, 983, 1006, 998, 1057, 22, 3, 0, 0, 1011, 989, 1023, 1029, 10, 7, 0, 0, 986, 991, 1062, 1038, 6, 3, 0, 0, 1045, 1059, 1053, 1032, 0, 0, 0, 0, 1040, 1034, 1036, 1024, 0, 0, 0, 0, 1001, 1066, 1006, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4019, 4069, 4069, 4086, 4189, 4134, 4083] avg=4096.0000 max=4189 max_rank=5 min=4019 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8448, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1012, 1030, 990, 1017, 1049, 1033, 1001, 1000, 1005, 1004, 1094, 1072, 997, 1018, 1071, 1026, 1009, 1006, 987, 1022, 1012, 1008, 1074, 997, 1011, 1049, 984, 1048, 1057, 1013, 1036, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[997, 1054, 1067, 1029, 1050, 1048, 990, 1056, 1014, 1117, 995, 1014, 1017, 997, 981, 1011, 982, 991, 1075, 1039, 1025, 998, 1021, 1048, 1015, 1004, 1036, 1006, 982, 1032, 1018, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1021, 1092, 1003, 1044, 996, 1006, 1009, 1022, 1010, 1002, 1016, 1015, 1020, 1013, 977, 1018, 985, 1039, 1059, 1023, 1033, 1032, 1020, 1001, 1014, 1003, 1060, 1069, 1030, 1044, 1031, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1046, 1018, 1068, 1054, 1012, 1036, 1022, 1028, 983, 990, 1001, 1026, 1010, 998, 1004, 1066, 1010, 987, 1054, 1007, 1059, 1033, 1031, 1012, 1042, 1005, 1022, 1026, 1054, 999, 1008, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1051, 1035, 989, 1020, 1035, 1024, 1056, 1023, 1093, 1015, 1008, 1032, 1078, 1031, 1010, 1022, 1042, 978, 1038, 1056, 984, 1068, 1043, 1028, 1034, 959, 984, 998, 1021, 1013, 1026, 974] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1024, 1001, 1038, 995, 1070, 1012, 992, 991, 1030, 1035, 986, 1028, 1061, 1030, 1048, 1021, 1023, 1049, 981, 1010, 983, 1003, 1034, 1045, 1044, 1020, 1031, 1034, 1043, 1051, 1014, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1002, 998, 1009, 1000, 1011, 1065, 1028, 1044, 1046, 1028, 1002, 1018, 1006, 1040, 1013, 1083, 1030, 1057, 985, 1036, 979, 1001, 1023, 1052, 1035, 1041, 1050, 1028, 966, 975, 1083, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1057, 990, 1059, 1067, 1016, 1022, 1018, 993, 1016, 985, 1013, 1076, 1012, 1019, 1015, 976, 1037, 1048, 1002, 999, 1076, 1024, 1069, 1022, 1079, 1055, 1016, 1031, 997, 1012, 982, 985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1012, 1030, 990, 1004, 0, 0, 0, 0, 1049, 1033, 1001, 1000, 1, 0, 0, 0, 1005, 1004, 1094, 1072, 0, 0, 0, 0, 997, 1018, 1071, 1026, 7, 2, 0, 0, 1009, 1006, 987, 1022, 13, 1, 0, 0, 1012, 1008, 1066, 997, 0, 0, 0, 0, 1004, 1049, 984, 1048, 0, 0, 0, 0, 1057, 1013, 1036, 1036, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4084, 4175, 4121, 4038, 4083, 4085, 4146] avg=4096.0000 max=4175 max_rank=2 min=4036 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[997, 1054, 1067, 1016, 0, 0, 0, 0, 1050, 1048, 990, 1056, 1, 0, 0, 0, 1014, 1117, 995, 1014, 0, 0, 0, 0, 1017, 997, 981, 1011, 6, 2, 0, 0, 982, 991, 1075, 1039, 13, 1, 0, 0, 1025, 998, 1014, 1048, 0, 0, 0, 0, 1008, 1004, 1036, 1006, 0, 0, 0, 0, 982, 1032, 1018, 1059, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4145, 4140, 4014, 4101, 4085, 4054, 4095] avg=4096.0000 max=4145 max_rank=1 min=4014 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1002, 998, 1009, 987, 0, 0, 0, 0, 1011, 1065, 1028, 1044, 1, 0, 0, 0, 1046, 1028, 1002, 1018, 0, 0, 0, 0, 1006, 1040, 1013, 1083, 7, 2, 0, 0, 1030, 1057, 985, 1036, 13, 1, 0, 0, 979, 1001, 1015, 1052, 0, 0, 0, 0, 1028, 1041, 1050, 1028, 0, 0, 0, 0, 966, 975, 1083, 1034, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4149, 4094, 4151, 4122, 4047, 4147, 4062] avg=4096.0000 max=4151 max_rank=3 min=3996 min_rank=0 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1051, 1035, 989, 1007, 0, 0, 0, 0, 1035, 1024, 1056, 1023, 1, 0, 0, 0, 1093, 1015, 1008, 1032, 0, 0, 0, 0, 1078, 1031, 1010, 1022, 7, 2, 0, 0, 1042, 978, 1038, 1056, 13, 1, 0, 0, 984, 1068, 1035, 1028, 0, 0, 0, 0, 1027, 959, 984, 998, 0, 0, 0, 0, 1021, 1013, 1026, 974, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4139, 4148, 4150, 4128, 4115, 3968, 4038] avg=4096.0000 max=4150 max_rank=3 min=3968 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16249 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1021, 1092, 1003, 1027, 0, 0, 0, 0, 996, 1006, 1009, 1022, 2, 1, 0, 0, 1010, 1002, 1016, 1014, 0, 0, 0, 0, 1020, 1013, 977, 1018, 9, 8, 0, 0, 985, 1039, 1059, 1023, 17, 6, 0, 0, 1033, 1032, 1005, 1001, 0, 0, 0, 0, 998, 1003, 1060, 1069, 0, 0, 0, 0, 1030, 1044, 1031, 1061, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4036, 4042, 4045, 4129, 4071, 4130, 4172] avg=4096.0000 max=4172 max_rank=7 min=4036 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1046, 1018, 1068, 1041, 0, 0, 0, 0, 1012, 1036, 1022, 1028, 1, 0, 0, 0, 983, 990, 1001, 1026, 0, 0, 0, 0, 1010, 998, 1004, 1066, 7, 2, 0, 0, 1010, 987, 1054, 1007, 13, 1, 0, 0, 1059, 1033, 1023, 1012, 0, 0, 0, 0, 1035, 1005, 1022, 1026, 0, 0, 0, 0, 1054, 999, 1008, 1057, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4099, 4000, 4087, 4072, 4127, 4088, 4122] avg=4096.0000 max=4173 max_rank=0 min=4000 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1024, 1001, 1038, 982, 0, 0, 0, 0, 1070, 1012, 992, 991, 1, 0, 0, 0, 1030, 1035, 986, 1028, 0, 0, 0, 0, 1061, 1030, 1048, 1021, 7, 2, 0, 0, 1023, 1049, 981, 1010, 13, 1, 0, 0, 983, 1003, 1026, 1045, 0, 0, 0, 0, 1037, 1020, 1031, 1034, 0, 0, 0, 0, 1043, 1051, 1014, 1041, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4066, 4079, 4169, 4077, 4057, 4122, 4153] avg=4096.0000 max=4169 max_rank=3 min=4045 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1057, 990, 1059, 1053, 0, 0, 0, 0, 1016, 1022, 1018, 993, 1, 0, 0, 0, 1016, 985, 1013, 1076, 0, 0, 0, 0, 1012, 1019, 1015, 976, 7, 2, 0, 0, 1037, 1048, 1002, 999, 14, 1, 0, 0, 1076, 1024, 1061, 1022, 0, 0, 0, 0, 1072, 1055, 1016, 1031, 0, 0, 0, 0, 997, 1012, 982, 985, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4050, 4090, 4031, 4101, 4183, 4174, 3980] avg=4096.0000 max=4183 max_rank=5 min=3980 min_rank=7 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True + [2026-08-30 04:58:06] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 627.6 | throughput per GPU (TFLOP/s/GPU): 531.2 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.954794E+00 | loss scale: 1.0 | grad norm: 9.192 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1004, 967, 1034, 1003, 1066, 1075, 1036, 1041, 982, 1027, 1064, 1043, 1027, 987, 1065, 986, 1062, 1024, 1040, 1023, 1027, 1020, 1068, 1006, 1004, 1082, 1018, 1000, 1009, 961, 1007, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1026, 1095, 1052, 1045, 1004, 1044, 1013, 1057, 1096, 1022, 1023, 968, 1043, 1035, 995, 943, 1025, 1005, 1013, 1021, 987, 1019, 980, 1032, 1023, 1059, 1008, 1013, 1000, 1077, 1043, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1029, 1045, 1055, 970, 1025, 1031, 1012, 1024, 1009, 978, 1030, 1050, 1039, 977, 1042, 973, 1015, 1057, 1078, 1029, 1026, 1061, 1006, 1010, 1024, 1035, 1047, 1027, 1027, 981, 1013, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1053, 1009, 1040, 1024, 1024, 1009, 1000, 1013, 1013, 1010, 1047, 1003, 973, 1010, 988, 1052, 1043, 1004, 1022, 1009, 1041, 1045, 1056, 1045, 1045, 1015, 1076, 999, 1034, 1027, 1054, 985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[994, 1028, 1036, 982, 1016, 1094, 988, 1053, 1023, 975, 1026, 999, 1000, 1054, 1014, 1037, 1063, 1048, 1052, 1022, 1003, 987, 1020, 1041, 1048, 1042, 1061, 1032, 1015, 1035, 991, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1099, 997, 1045, 979, 1060, 1093, 1014, 984, 1032, 1005, 1062, 1064, 1022, 1049, 992, 1008, 983, 1000, 1001, 1021, 1027, 1005, 1010, 1034, 989, 994, 1026, 1013, 1069, 1030, 1052, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1002, 1042, 1047, 1028, 976, 1056, 994, 1029, 1018, 1063, 1004, 1057, 1032, 1060, 993, 970, 1040, 1083, 1018, 1030, 1022, 1046, 1035, 1034, 1000, 988, 984, 1036, 961, 1029, 1040, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1066, 1030, 1042, 1011, 1026, 997, 1061, 1038, 1037, 985, 1031, 1066, 1031, 1051, 973, 1021, 1037, 994, 988, 1024, 1035, 1050, 982, 1063, 1018, 1028, 965, 1023, 976, 1040, 1059, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1029, 1045, 1041, 970, 0, 0, 0, 0, 1025, 1009, 1012, 1024, 0, 0, 0, 0, 1009, 978, 1025, 1050, 0, 0, 0, 0, 1039, 977, 1042, 973, 22, 14, 3, 0, 1003, 1057, 1078, 1029, 0, 0, 0, 0, 1026, 1061, 1006, 1004, 0, 0, 0, 0, 1024, 1035, 1047, 1027, 5, 0, 0, 0, 1027, 981, 1013, 1043, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4070, 4062, 4070, 4167, 4097, 4138, 4079] avg=4096.0000 max=4167 max_rank=4 min=4062 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1053, 1009, 1027, 1024, 0, 0, 0, 0, 1024, 987, 1000, 1013, 0, 0, 0, 0, 1013, 1010, 1042, 1003, 0, 0, 0, 0, 973, 1010, 988, 1052, 22, 13, 3, 0, 1031, 1004, 1022, 1009, 0, 0, 0, 0, 1041, 1045, 1056, 1039, 0, 0, 0, 0, 1045, 1015, 1076, 999, 5, 0, 0, 0, 1034, 1027, 1054, 985, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4024, 4068, 4061, 4066, 4181, 4140, 4115] avg=4096.0000 max=4181 max_rank=5 min=4024 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1004, 967, 1021, 1003, 0, 0, 0, 0, 1066, 1052, 1036, 1041, 0, 0, 0, 0, 982, 1027, 1059, 1043, 0, 0, 0, 0, 1027, 987, 1065, 986, 23, 13, 3, 0, 1050, 1024, 1040, 1023, 0, 0, 0, 0, 1027, 1020, 1068, 1000, 0, 0, 0, 0, 1004, 1082, 1018, 1000, 5, 0, 0, 0, 1009, 961, 1007, 1010, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[3995, 4195, 4111, 4104, 4137, 4115, 4109, 4002] avg=4096.0000 max=4195 max_rank=1 min=3995 min_rank=0 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1026, 1095, 1039, 1045, 0, 0, 0, 0, 1004, 1022, 1013, 1057, 0, 0, 0, 0, 1096, 1022, 1018, 968, 0, 0, 0, 0, 1043, 1035, 995, 943, 22, 13, 3, 0, 1013, 1005, 1013, 1021, 0, 0, 0, 0, 987, 1019, 980, 1026, 0, 0, 0, 0, 1023, 1059, 1008, 1013, 5, 0, 0, 0, 1000, 1077, 1043, 1002, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4205, 4096, 4104, 4054, 4052, 4012, 4108, 4137] avg=4096.0000 max=4205 max_rank=0 min=4012 min_rank=5 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1099, 997, 1026, 979, 0, 0, 0, 0, 1060, 1065, 1014, 984, 0, 0, 0, 0, 1032, 1005, 1053, 1064, 0, 0, 0, 0, 1022, 1049, 992, 1008, 28, 19, 9, 0, 961, 1000, 1001, 1021, 0, 0, 0, 0, 1027, 1005, 1010, 1021, 0, 0, 0, 0, 989, 994, 1026, 1013, 9, 2, 0, 0, 1069, 1030, 1052, 1009, 13, 11, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4123, 4154, 4127, 3983, 4063, 4033, 4184] avg=4096.0000 max=4184 max_rank=7 min=3983 min_rank=4 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1002, 1042, 1034, 1028, 0, 0, 0, 0, 976, 1033, 994, 1029, 0, 0, 0, 0, 1018, 1063, 999, 1057, 0, 0, 0, 0, 1032, 1060, 993, 970, 23, 13, 3, 0, 1028, 1083, 1018, 1030, 0, 0, 0, 0, 1022, 1046, 1035, 1028, 0, 0, 0, 0, 1000, 988, 984, 1036, 5, 0, 0, 0, 961, 1029, 1040, 1051, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4032, 4137, 4094, 4159, 4131, 4013, 4096] avg=4096.0000 max=4159 max_rank=4 min=4013 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[994, 1028, 1023, 982, 0, 0, 0, 0, 1016, 1070, 988, 1053, 0, 0, 0, 0, 1023, 975, 1021, 999, 0, 0, 0, 0, 1000, 1054, 1014, 1037, 24, 13, 3, 0, 1051, 1048, 1052, 1022, 0, 0, 0, 0, 1003, 987, 1020, 1035, 0, 0, 0, 0, 1048, 1042, 1061, 1032, 5, 0, 0, 0, 1015, 1035, 991, 989, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4127, 4018, 4145, 4173, 4045, 4188, 4045] avg=4096.0000 max=4188 max_rank=6 min=4018 min_rank=2 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1066, 1030, 1029, 1011, 0, 0, 0, 0, 1026, 976, 1061, 1038, 0, 0, 0, 0, 1037, 985, 1026, 1066, 0, 0, 0, 0, 1031, 1051, 973, 1021, 21, 13, 3, 0, 1025, 994, 988, 1024, 0, 0, 0, 0, 1035, 1050, 982, 1057, 0, 0, 0, 0, 1018, 1028, 965, 1023, 5, 0, 0, 0, 976, 1040, 1059, 1020, 9, 6, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4101, 4114, 4113, 4031, 4124, 4039, 4110] avg=4096.0000 max=4136 max_rank=0 min=4031 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8064, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1016, 1010, 1036, 972, 1018, 1006, 1047, 962, 1009, 1008, 1048, 1038, 1030, 1034, 1044, 1021, 1008, 1038, 992, 1002, 1014, 1057, 1043, 1021, 1058, 1052, 1038, 1037, 1065, 1051, 979, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1029, 988, 1054, 1058, 981, 1042, 1055, 1009, 1036, 1024, 1024, 1000, 1042, 999, 1077, 986, 1048, 1071, 1040, 977, 1057, 1055, 961, 1018, 1020, 980, 1023, 1002, 1038, 1022, 1014, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1040, 987, 1041, 1032, 983, 1068, 1034, 1067, 1050, 1029, 961, 966, 1036, 1021, 1018, 1058, 989, 1019, 1023, 1028, 1029, 1042, 1056, 1014, 977, 958, 1026, 1011, 1058, 1074, 1018, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1070, 1060, 1026, 985, 1079, 993, 1009, 1039, 1026, 962, 995, 1023, 1039, 988, 1002, 984, 1035, 1067, 1056, 1064, 1068, 1054, 1000, 1068, 1007, 993, 1020, 1006, 1002, 1055, 984, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1002, 1027, 1040, 1048, 1004, 1019, 1016, 990, 1020, 1052, 1053, 1004, 1027, 1082, 997, 1041, 1005, 1046, 1043, 1011, 1008, 1066, 1058, 999, 1038, 1023, 1007, 1048, 981, 983, 1021, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1008, 1027, 1034, 1060, 1045, 1021, 1048, 1087, 979, 1021, 1029, 1018, 1065, 1031, 1005, 1019, 1004, 1020, 991, 1031, 1029, 1017, 1014, 1026, 1006, 1032, 1024, 1035, 1059, 1005, 1002, 976] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[981, 1022, 1025, 1005, 1029, 1046, 1041, 1031, 1005, 1018, 1016, 1072, 1014, 1012, 1036, 985, 1040, 1029, 1005, 1001, 983, 1051, 984, 1024, 1028, 1057, 1046, 1037, 1016, 1045, 1027, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[987, 1018, 1040, 1015, 1018, 1050, 982, 1006, 1038, 1021, 1023, 939, 1052, 1007, 1001, 1023, 1020, 1019, 1049, 1054, 1058, 1011, 1019, 1062, 990, 1052, 1039, 1035, 1011, 1056, 1027, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1016, 1010, 1036, 972, 2, 0, 0, 0, 1018, 1000, 1047, 962, 0, 0, 0, 0, 1009, 1008, 1048, 1038, 25, 7, 0, 0, 1030, 1034, 1044, 1021, 0, 0, 0, 0, 1008, 1031, 992, 1002, 0, 0, 0, 0, 1014, 1032, 1043, 1021, 0, 0, 0, 0, 1058, 1052, 1038, 1037, 6, 1, 0, 0, 1065, 1048, 979, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4027, 4135, 4129, 4033, 4110, 4192, 4106] avg=4096.0000 max=4192 max_rank=6 min=4027 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1029, 988, 1054, 1058, 2, 1, 0, 0, 981, 1036, 1055, 1009, 0, 0, 0, 0, 1036, 1024, 1024, 1000, 25, 7, 0, 0, 1041, 999, 1077, 986, 0, 0, 0, 0, 1048, 1064, 1040, 977, 0, 0, 0, 0, 1057, 1030, 961, 1018, 0, 0, 0, 0, 1020, 980, 1023, 1002, 6, 1, 0, 0, 1038, 1019, 1014, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4081, 4116, 4103, 4129, 4066, 4032, 4109] avg=4096.0000 max=4132 max_rank=0 min=4032 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1040, 987, 1041, 1032, 2, 0, 0, 0, 983, 1062, 1034, 1067, 0, 0, 0, 0, 1050, 1029, 961, 966, 24, 6, 0, 0, 1036, 1021, 1018, 1058, 0, 0, 0, 0, 989, 1013, 1023, 1028, 0, 0, 0, 0, 1029, 1018, 1056, 1014, 0, 0, 0, 0, 977, 958, 1026, 1011, 6, 1, 0, 0, 1058, 1071, 1018, 1055, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4146, 4036, 4133, 4053, 4117, 3979, 4202] avg=4096.0000 max=4202 max_rank=7 min=3979 min_rank=6 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[981, 1022, 1025, 1005, 2, 0, 0, 0, 1029, 1040, 1041, 1031, 0, 0, 0, 0, 1005, 1018, 1016, 1072, 24, 7, 0, 0, 1014, 1012, 1036, 985, 0, 0, 0, 0, 1040, 1022, 1005, 1001, 0, 0, 0, 0, 983, 1027, 984, 1024, 0, 0, 0, 0, 1028, 1057, 1046, 1037, 6, 1, 0, 0, 1016, 1042, 1027, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4141, 4142, 4047, 4068, 4018, 4175, 4142] avg=4096.0000 max=4175 max_rank=6 min=4018 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1008, 1027, 1034, 1060, 2, 1, 0, 0, 1045, 1015, 1048, 1087, 0, 0, 0, 0, 979, 1021, 1029, 1018, 24, 6, 0, 0, 1064, 1031, 1005, 1019, 0, 0, 0, 0, 1004, 1014, 991, 1031, 0, 0, 0, 0, 1029, 993, 1014, 1026, 0, 0, 0, 0, 1006, 1032, 1024, 1035, 6, 1, 0, 0, 1059, 1002, 1002, 976, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4195, 4077, 4119, 4040, 4062, 4104, 4039] avg=4096.0000 max=4195 max_rank=1 min=4039 min_rank=7 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16245 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1070, 1060, 1026, 985, 3, 5, 0, 0, 1079, 978, 1009, 1039, 0, 0, 0, 0, 1026, 962, 995, 1023, 28, 11, 6, 0, 1034, 988, 1002, 984, 0, 0, 0, 0, 1035, 1056, 1056, 1064, 0, 0, 0, 0, 1068, 1026, 1000, 1068, 0, 0, 0, 0, 1007, 993, 1020, 1006, 9, 5, 0, 0, 1002, 1047, 984, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4105, 4051, 4008, 4211, 4162, 4040, 4042] avg=4096.0000 max=4211 max_rank=4 min=4008 min_rank=3 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1002, 1027, 1040, 1048, 2, 0, 0, 0, 1004, 1013, 1016, 990, 0, 0, 0, 0, 1020, 1052, 1053, 1004, 25, 7, 0, 0, 1027, 1082, 997, 1041, 0, 0, 0, 0, 1005, 1039, 1043, 1011, 0, 0, 0, 0, 1008, 1041, 1058, 999, 0, 0, 0, 0, 1038, 1023, 1007, 1048, 6, 1, 0, 0, 981, 980, 1021, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4023, 4161, 4147, 4098, 4106, 4123, 3991] avg=4096.0000 max=4161 max_rank=2 min=3991 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[987, 1018, 1040, 1015, 2, 1, 0, 0, 1018, 1044, 982, 1006, 0, 0, 0, 0, 1038, 1021, 1023, 939, 23, 6, 0, 0, 1051, 1007, 1001, 1023, 0, 0, 0, 0, 1020, 1013, 1049, 1054, 0, 0, 0, 0, 1058, 988, 1019, 1062, 0, 0, 0, 0, 990, 1052, 1039, 1035, 6, 1, 0, 0, 1011, 1053, 1027, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4050, 4050, 4082, 4136, 4127, 4123, 4137] avg=4096.0000 max=4137 max_rank=7 min=4050 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16523 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8064, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[997, 1033, 1057, 1058, 1011, 1038, 1011, 1039, 1020, 1037, 1030, 998, 1027, 994, 970, 1053, 1060, 1020, 1026, 1039, 994, 1080, 995, 1049, 1028, 1043, 1060, 1027, 939, 1001, 1028, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[979, 1026, 1042, 987, 1080, 987, 1021, 1016, 1020, 1032, 1017, 1021, 1017, 1039, 1034, 1062, 1030, 1041, 1025, 1072, 1038, 1037, 1004, 943, 1031, 1024, 999, 1002, 1030, 1046, 1032, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[960, 1056, 984, 1010, 1062, 1001, 1057, 1036, 996, 1078, 1038, 1067, 1067, 992, 1037, 989, 1069, 1034, 1054, 997, 1022, 1038, 1032, 980, 1035, 1048, 1022, 1017, 1017, 993, 1014, 966] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1009, 1037, 958, 1019, 1029, 1031, 1002, 1015, 1047, 1069, 1024, 1037, 1026, 1024, 1054, 1047, 1009, 1022, 1050, 1001, 1008, 1000, 1019, 1052, 1022, 1021, 1035, 1055, 1015, 1008, 1016, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1031, 1016, 985, 1036, 1102, 1065, 979, 1067, 1044, 984, 1031, 1057, 1048, 1015, 1013, 1018, 1017, 1010, 1029, 988, 1011, 1004, 983, 1054, 1030, 1045, 999, 1024, 1034, 999, 995, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[962, 1042, 1001, 1013, 1018, 1016, 1013, 1022, 1022, 1020, 1012, 1025, 1026, 1055, 1005, 1043, 1012, 1059, 1012, 1023, 1030, 1036, 1020, 1050, 1014, 1020, 1014, 1070, 1033, 1001, 1012, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1025, 1031, 1041, 1007, 1024, 1039, 1030, 1008, 1011, 1027, 989, 1009, 1060, 1041, 1032, 981, 1037, 1001, 1027, 1010, 1025, 1028, 1028, 1022, 1018, 1034, 1054, 1062, 1006, 1024, 1003, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1055, 1007, 1067, 1030, 1042, 988, 1043, 967, 985, 983, 1010, 1026, 978, 1070, 1059, 1047, 989, 1040, 1041, 1048, 1062, 1030, 1021, 1020, 976, 1065, 999, 998, 1021, 1028, 1026, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[979, 1026, 1042, 987, 15, 5, 4, 0, 1070, 987, 1021, 1016, 0, 0, 0, 0, 1020, 1032, 1017, 1021, 0, 0, 0, 0, 998, 1039, 1034, 1062, 0, 0, 0, 0, 1030, 1041, 1011, 1072, 0, 0, 0, 0, 1038, 1037, 1004, 943, 6, 0, 0, 0, 1031, 1009, 999, 1002, 0, 0, 0, 0, 1030, 1046, 1032, 1034, 19, 9, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4094, 4090, 4133, 4154, 4028, 4041, 4170] avg=4096.0000 max=4170 max_rank=7 min=4028 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[997, 1033, 1057, 1058, 15, 5, 4, 0, 1001, 1038, 1011, 1039, 0, 0, 0, 0, 1020, 1037, 1030, 998, 0, 0, 0, 0, 1008, 994, 970, 1053, 0, 0, 0, 0, 1060, 1020, 1012, 1039, 0, 0, 0, 0, 994, 1080, 995, 1049, 6, 0, 0, 0, 1028, 1028, 1060, 1027, 0, 0, 0, 0, 939, 1001, 1028, 1006, 19, 9, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4089, 4085, 4025, 4131, 4124, 4143, 4002] avg=4096.0000 max=4169 max_rank=0 min=4002 min_rank=7 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1055, 1007, 1067, 1030, 15, 6, 4, 0, 1032, 988, 1043, 967, 0, 0, 0, 0, 985, 983, 1010, 1026, 0, 0, 0, 0, 960, 1070, 1059, 1047, 0, 0, 0, 0, 989, 1040, 1026, 1048, 0, 0, 0, 0, 1062, 1030, 1021, 1020, 6, 0, 0, 0, 976, 1050, 999, 998, 0, 0, 0, 0, 1021, 1028, 1026, 1047, 18, 9, 0, 0] rerouted_tokens_per_ep_rank=[4184, 4030, 4004, 4136, 4103, 4139, 4023, 4149] avg=4096.0000 max=4184 max_rank=0 min=4004 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1031, 1016, 985, 1036, 15, 5, 4, 0, 1092, 1065, 979, 1067, 0, 0, 0, 0, 1044, 984, 1031, 1057, 0, 0, 0, 0, 1029, 1015, 1013, 1018, 0, 0, 0, 0, 1017, 1010, 1015, 988, 0, 0, 0, 0, 1011, 1004, 983, 1054, 6, 0, 0, 0, 1030, 1030, 999, 1024, 0, 0, 0, 0, 1034, 999, 995, 1055, 19, 9, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4203, 4116, 4075, 4030, 4058, 4083, 4111] avg=4096.0000 max=4203 max_rank=1 min=4030 min_rank=4 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[962, 1042, 1001, 1013, 15, 5, 4, 0, 1008, 1016, 1013, 1022, 0, 0, 0, 0, 1022, 1020, 1012, 1025, 0, 0, 0, 0, 1007, 1055, 1005, 1043, 0, 0, 0, 0, 1012, 1059, 998, 1023, 0, 0, 0, 0, 1030, 1036, 1020, 1050, 6, 0, 0, 0, 1014, 1005, 1014, 1070, 0, 0, 0, 0, 1033, 1001, 1012, 1067, 19, 9, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4059, 4079, 4110, 4092, 4142, 4103, 4141] avg=4096.0000 max=4142 max_rank=5 min=4042 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[960, 1056, 984, 1010, 18, 11, 8, 0, 1041, 1001, 1057, 1036, 0, 0, 0, 0, 996, 1078, 1038, 1067, 2, 0, 0, 0, 1044, 992, 1037, 989, 0, 0, 0, 0, 1069, 1034, 1030, 997, 0, 0, 0, 0, 1022, 1038, 1032, 980, 11, 0, 0, 0, 1035, 1030, 1022, 1017, 0, 0, 0, 0, 1017, 993, 1014, 966, 23, 13, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4135, 4181, 4062, 4130, 4083, 4104, 4026] avg=4096.0000 max=4181 max_rank=2 min=4026 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1009, 1037, 958, 1019, 15, 6, 4, 0, 1019, 1031, 1002, 1015, 0, 0, 0, 0, 1047, 1069, 1024, 1037, 0, 0, 0, 0, 1007, 1024, 1054, 1047, 0, 0, 0, 0, 1009, 1022, 1035, 1001, 0, 0, 0, 0, 1008, 1000, 1019, 1052, 6, 0, 0, 0, 1022, 1006, 1035, 1055, 0, 0, 0, 0, 1015, 1008, 1016, 1007, 19, 9, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4067, 4177, 4132, 4067, 4085, 4118, 4074] avg=4096.0000 max=4177 max_rank=2 min=4048 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1025, 1031, 1041, 1007, 15, 5, 4, 0, 1014, 1039, 1030, 1008, 0, 0, 0, 0, 1011, 1027, 989, 1009, 0, 0, 0, 0, 1041, 1041, 1032, 981, 0, 0, 0, 0, 1037, 1001, 1013, 1010, 0, 0, 0, 0, 1025, 1028, 1028, 1022, 6, 0, 0, 0, 1018, 1019, 1054, 1062, 0, 0, 0, 0, 1006, 1024, 1003, 1034, 19, 9, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4091, 4036, 4095, 4061, 4109, 4153, 4095] avg=4096.0000 max=4153 max_rank=6 min=4036 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1041, 1052, 1086, 990, 1061, 1045, 1038, 1016, 1009, 1028, 1035, 1029, 1023, 1036, 1005, 1006, 987, 1013, 1015, 1091, 1030, 1007, 1006, 1043, 963, 1034, 976, 1009, 990, 1056, 1031, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1010, 996, 1011, 1049, 1071, 1001, 1036, 1046, 1008, 991, 1021, 1009, 1013, 1026, 976, 1033, 1027, 1010, 1012, 1044, 994, 1034, 968, 1049, 1076, 1049, 1020, 1014, 1052, 1037, 1042, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1051, 990, 1004, 1031, 1016, 1037, 1043, 1055, 1065, 984, 1081, 1032, 987, 1042, 1034, 1026, 982, 1016, 1041, 1016, 1018, 1046, 954, 1020, 1012, 1017, 1025, 1026, 1033, 1007, 1031, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1021, 1013, 961, 996, 1070, 1039, 980, 1016, 993, 1094, 1008, 1044, 1031, 1003, 1063, 1007, 1002, 996, 1059, 1048, 1029, 1022, 1036, 1004, 1050, 1034, 1074, 1069, 983, 1027, 969, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[994, 992, 975, 1057, 1057, 965, 993, 1033, 1049, 1010, 1030, 1067, 1025, 1083, 1017, 1001, 1062, 955, 1046, 1000, 1059, 1048, 1027, 990, 1001, 1046, 1021, 1016, 1076, 1032, 1011, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1007, 1023, 957, 1024, 1058, 1036, 1009, 1015, 1007, 1010, 1040, 1074, 1048, 1069, 1008, 1025, 937, 1019, 1040, 1010, 1035, 1041, 1002, 1019, 990, 1071, 1014, 1107, 1023, 1009, 1049, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1045, 999, 999, 1057, 1011, 1006, 1059, 1052, 1015, 1026, 1018, 1007, 1000, 990, 1056, 955, 1038, 1034, 1025, 1029, 1013, 987, 1075, 1018, 1022, 1018, 1051, 1047, 1058, 1031, 989, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1013, 1016, 1090, 1014, 1036, 1033, 992, 1028, 999, 1030, 1083, 1016, 1014, 956, 1050, 1042, 1042, 1034, 1020, 1051, 1013, 1022, 1004, 1029, 1035, 1019, 1043, 1036, 1038, 994, 961, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1041, 1052, 1086, 990, 24, 0, 0, 0, 1038, 1045, 1038, 1016, 0, 0, 0, 0, 1009, 1028, 1019, 1029, 0, 0, 0, 0, 1023, 1036, 1005, 1006, 9, 5, 0, 0, 987, 1013, 1015, 1091, 8, 0, 0, 0, 1030, 1007, 1006, 1043, 14, 1, 0, 0, 963, 1034, 976, 984, 0, 0, 0, 0, 990, 1056, 1031, 1017, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4193, 4137, 4085, 4084, 4114, 4101, 3957, 4097] avg=4096.0000 max=4193 max_rank=0 min=3957 min_rank=6 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16269 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1010, 996, 1011, 1049, 24, 0, 0, 0, 1048, 1001, 1036, 1046, 0, 0, 0, 0, 1008, 991, 1005, 1009, 0, 0, 0, 0, 1013, 1026, 976, 1033, 9, 5, 0, 0, 1027, 1010, 1012, 1044, 8, 0, 0, 0, 994, 1034, 968, 1049, 14, 1, 0, 0, 1076, 1049, 1020, 989, 0, 0, 0, 0, 1052, 1037, 1042, 1043, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4131, 4013, 4062, 4101, 4060, 4134, 4177] avg=4096.0000 max=4177 max_rank=7 min=4013 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1045, 999, 999, 1057, 25, 0, 0, 0, 990, 1006, 1059, 1052, 0, 0, 0, 0, 1015, 1026, 1002, 1007, 0, 0, 0, 0, 1000, 990, 1056, 955, 8, 5, 0, 0, 1038, 1034, 1025, 1029, 8, 0, 0, 0, 1013, 987, 1075, 1018, 13, 1, 0, 0, 1022, 1018, 1051, 1021, 0, 0, 0, 0, 1058, 1031, 989, 1038, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4107, 4050, 4014, 4134, 4107, 4112, 4119] avg=4096.0000 max=4134 max_rank=4 min=4014 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1021, 1013, 961, 996, 26, 0, 0, 0, 1047, 1039, 980, 1016, 0, 0, 0, 0, 993, 1094, 992, 1044, 0, 0, 0, 0, 1031, 1003, 1063, 1007, 9, 5, 0, 0, 1002, 996, 1059, 1048, 8, 0, 0, 0, 1029, 1022, 1036, 1004, 14, 1, 0, 0, 1050, 1034, 1074, 1042, 0, 0, 0, 0, 983, 1027, 969, 1027, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4082, 4123, 4118, 4113, 4106, 4200, 4009] avg=4096.0000 max=4200 max_rank=6 min=4009 min_rank=7 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1051, 990, 1004, 1031, 25, 0, 0, 0, 995, 1037, 1043, 1055, 0, 0, 0, 0, 1065, 984, 1064, 1032, 0, 0, 0, 0, 987, 1042, 1034, 1026, 8, 5, 0, 0, 982, 1016, 1041, 1016, 8, 0, 0, 0, 1018, 1046, 954, 1020, 13, 1, 0, 0, 1012, 1017, 1025, 1000, 0, 0, 0, 0, 1033, 1007, 1031, 1046, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4130, 4145, 4102, 4063, 4052, 4054, 4121] avg=4096.0000 max=4145 max_rank=2 min=4052 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16290 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1007, 1023, 957, 1024, 27, 0, 0, 0, 1035, 1036, 1009, 1015, 0, 0, 0, 0, 1007, 1010, 1024, 1074, 0, 0, 0, 0, 1048, 1069, 1008, 1025, 9, 5, 0, 0, 937, 1019, 1040, 1010, 8, 0, 0, 0, 1035, 1041, 1002, 1019, 14, 1, 0, 0, 990, 1071, 1014, 1079, 0, 0, 0, 0, 1023, 1009, 1049, 992, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4095, 4115, 4164, 4014, 4112, 4154, 4076] avg=4096.0000 max=4164 max_rank=3 min=4014 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[994, 992, 975, 1057, 28, 0, 0, 0, 1027, 965, 993, 1033, 0, 0, 0, 0, 1049, 1010, 1000, 1067, 0, 0, 0, 0, 1025, 1083, 1017, 1001, 12, 11, 0, 0, 1062, 955, 1046, 1000, 11, 0, 0, 0, 1059, 1048, 1027, 990, 18, 6, 0, 0, 1001, 1046, 1021, 982, 0, 0, 0, 0, 1076, 1032, 1011, 1030, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4018, 4126, 4149, 4074, 4148, 4050, 4157] avg=4096.0000 max=4157 max_rank=7 min=4018 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1013, 1016, 1090, 1014, 25, 0, 0, 0, 1015, 1033, 992, 1028, 0, 0, 0, 0, 999, 1030, 1066, 1016, 0, 0, 0, 0, 1014, 956, 1050, 1042, 8, 5, 0, 0, 1042, 1034, 1020, 1051, 8, 0, 0, 0, 1013, 1022, 1004, 1029, 13, 1, 0, 0, 1035, 1019, 1043, 1010, 0, 0, 0, 0, 1038, 994, 961, 1015, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4068, 4111, 4075, 4155, 4082, 4107, 4012] avg=4096.0000 max=4158 max_rank=0 min=4012 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1019, 965, 1024, 999, 1015, 996, 1019, 1019, 1071, 1011, 991, 1024, 1009, 1029, 1061, 1008, 1009, 1018, 1047, 1015, 1027, 1038, 986, 1034, 1027, 1085, 1078, 1043, 996, 1051, 1024, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[997, 1045, 1025, 1048, 1007, 989, 980, 1004, 1024, 1056, 1076, 996, 1048, 1030, 1010, 996, 1060, 1016, 1050, 1052, 1031, 1003, 1017, 997, 1026, 1044, 1043, 972, 1034, 1034, 1040, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1021, 1034, 1059, 1033, 997, 978, 1000, 1030, 1013, 988, 1027, 1048, 1005, 1013, 1054, 1037, 1036, 994, 1052, 1014, 1085, 1048, 1012, 976, 1047, 988, 1105, 1024, 999, 980, 1020, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1005, 1037, 978, 1046, 1016, 994, 1011, 1062, 977, 1007, 1001, 1087, 1052, 987, 1045, 1060, 996, 1040, 1027, 1031, 998, 983, 1044, 1044, 1031, 1021, 1060, 1040, 1006, 1047, 997, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1077, 1017, 968, 1010, 1037, 992, 1047, 1002, 1047, 1011, 1051, 1022, 979, 1043, 1023, 1016, 1092, 1031, 1011, 1057, 1010, 1054, 1040, 972, 969, 1032, 1048, 1010, 1065, 1001, 1018, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1017, 1020, 1054, 1022, 1055, 1004, 1037, 983, 1025, 1003, 994, 997, 1044, 1038, 1046, 970, 1061, 1042, 1000, 1039, 1048, 1020, 1011, 1011, 1074, 998, 1039, 1019, 1010, 1046, 1013, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1042, 1004, 976, 1027, 963, 1025, 1006, 958, 1057, 1019, 1039, 1004, 1020, 1069, 1068, 1036, 1057, 1029, 1051, 996, 1056, 1000, 1007, 1043, 1086, 1046, 1041, 963, 1053, 990, 1041, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1036, 1008, 1041, 978, 1010, 1042, 997, 1002, 1055, 1040, 1025, 1029, 1075, 1003, 1072, 993, 1019, 1049, 1031, 1004, 1069, 993, 1047, 1035, 1044, 1063, 1031, 974, 1061, 997, 978, 967] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[997, 1045, 1025, 1048, 8, 7, 0, 0, 1007, 989, 980, 1004, 37, 23, 0, 0, 1019, 1056, 1076, 996, 0, 0, 0, 0, 1048, 1030, 991, 996, 0, 0, 0, 0, 1029, 1016, 1050, 1052, 0, 0, 0, 0, 1031, 1003, 1017, 997, 3, 0, 0, 0, 1026, 1044, 1006, 972, 0, 0, 0, 0, 1034, 1034, 1040, 1018, 12, 2, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4040, 4147, 4065, 4147, 4051, 4048, 4140] avg=4096.0000 max=4147 max_rank=2 min=4040 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1021, 1034, 1059, 1033, 8, 8, 0, 0, 997, 978, 1000, 1030, 39, 23, 0, 0, 1008, 988, 1027, 1048, 0, 0, 0, 0, 1005, 1013, 1033, 1037, 0, 0, 0, 0, 1005, 994, 1052, 1014, 0, 0, 0, 0, 1085, 1048, 1012, 976, 3, 0, 0, 0, 1047, 988, 1066, 1024, 0, 0, 0, 0, 999, 980, 1020, 1051, 13, 2, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4067, 4071, 4088, 4065, 4124, 4125, 4065] avg=4096.0000 max=4163 max_rank=0 min=4065 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1019, 965, 1024, 999, 8, 8, 0, 0, 1015, 996, 1019, 1019, 38, 22, 0, 0, 1066, 1011, 991, 1024, 0, 0, 0, 0, 1009, 1029, 1040, 1008, 0, 0, 0, 0, 979, 1018, 1047, 1015, 0, 0, 0, 0, 1027, 1038, 986, 1034, 3, 0, 0, 0, 1027, 1085, 1040, 1043, 0, 0, 0, 0, 996, 1051, 1024, 1030, 13, 2, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4109, 4092, 4086, 4059, 4088, 4195, 4116] avg=4096.0000 max=4195 max_rank=6 min=4023 min_rank=0 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1005, 1037, 978, 1046, 8, 8, 0, 0, 1016, 994, 1011, 1062, 38, 22, 0, 0, 972, 1007, 1001, 1087, 0, 0, 0, 0, 1052, 987, 1024, 1060, 0, 0, 0, 0, 966, 1040, 1027, 1031, 0, 0, 0, 0, 998, 983, 1044, 1044, 3, 0, 0, 0, 1031, 1021, 1022, 1040, 0, 0, 0, 0, 1006, 1047, 997, 1038, 13, 2, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4143, 4067, 4123, 4064, 4072, 4114, 4103] avg=4096.0000 max=4143 max_rank=1 min=4064 min_rank=4 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1077, 1017, 968, 1010, 14, 11, 0, 0, 1037, 992, 1047, 1002, 41, 30, 0, 0, 1035, 1011, 1051, 1022, 0, 0, 0, 0, 979, 1043, 997, 1016, 0, 0, 0, 0, 1048, 1031, 1011, 1057, 0, 0, 0, 0, 1010, 1054, 1040, 972, 8, 0, 0, 0, 969, 1032, 1007, 1010, 0, 0, 0, 0, 1065, 1001, 1018, 1016, 15, 4, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4149, 4119, 4035, 4147, 4084, 4018, 4119] avg=4096.0000 max=4149 max_rank=1 min=4018 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1017, 1020, 1054, 1022, 8, 8, 0, 0, 1055, 1004, 1037, 983, 37, 23, 0, 0, 1020, 1003, 994, 997, 0, 0, 0, 0, 1044, 1038, 1025, 970, 0, 0, 0, 0, 1030, 1042, 1000, 1039, 0, 0, 0, 0, 1048, 1020, 1011, 1011, 3, 0, 0, 0, 1074, 998, 1002, 1019, 0, 0, 0, 0, 1010, 1046, 1013, 1028, 13, 2, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4139, 4014, 4077, 4111, 4093, 4093, 4112] avg=4096.0000 max=4139 max_rank=1 min=4014 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1042, 1004, 976, 1027, 8, 8, 0, 0, 963, 1025, 1006, 958, 37, 23, 0, 0, 1052, 1019, 1039, 1004, 0, 0, 0, 0, 1020, 1069, 1047, 1036, 0, 0, 0, 0, 1026, 1029, 1051, 996, 0, 0, 0, 0, 1056, 1000, 1007, 1043, 3, 0, 0, 0, 1086, 1046, 1004, 963, 0, 0, 0, 0, 1053, 990, 1041, 996, 13, 2, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4012, 4114, 4172, 4102, 4109, 4099, 4095] avg=4096.0000 max=4172 max_rank=3 min=4012 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1036, 1008, 1041, 978, 8, 8, 0, 0, 1010, 1042, 997, 1002, 36, 22, 0, 0, 1050, 1040, 1025, 1029, 0, 0, 0, 0, 1075, 1003, 1051, 993, 0, 0, 0, 0, 989, 1049, 1031, 1004, 0, 0, 0, 0, 1069, 993, 1047, 1035, 3, 0, 0, 0, 1044, 1063, 995, 974, 0, 0, 0, 0, 1061, 997, 978, 967, 13, 2, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4109, 4144, 4122, 4073, 4147, 4076, 4018] avg=4096.0000 max=4147 max_rank=5 min=4018 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8064, 384, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8320, 8192, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[996, 1003, 1031, 1014, 1007, 996, 1080, 1002, 1043, 1040, 1034, 1014, 1027, 1045, 1088, 989, 989, 1009, 1064, 1034, 1053, 1030, 1021, 1027, 1007, 1043, 1037, 1049, 980, 1029, 1020, 967] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[973, 1055, 1028, 1066, 1001, 1007, 1008, 1027, 972, 1032, 1023, 1035, 988, 1061, 1062, 1057, 1037, 1017, 976, 1046, 1051, 954, 980, 1023, 1039, 1013, 998, 1024, 1066, 1052, 1040, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1050, 1025, 982, 1042, 1034, 1018, 986, 1045, 1036, 1013, 1047, 1008, 1054, 1040, 1023, 1019, 1021, 988, 1020, 1048, 994, 1006, 1037, 1009, 1014, 1040, 1064, 1003, 1036, 1045, 1017, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1011, 1000, 1023, 1050, 1056, 1028, 1003, 999, 1027, 1042, 1063, 1028, 984, 1033, 1008, 1024, 1025, 1057, 1040, 1020, 1038, 1030, 1024, 981, 1008, 1025, 1026, 1008, 1062, 1041, 1027, 977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1039, 1027, 1013, 1000, 1038, 1005, 1032, 1030, 1031, 988, 1011, 1045, 1021, 1085, 1022, 972, 1042, 1085, 991, 1017, 1022, 1029, 983, 1064, 1024, 1064, 1046, 981, 1060, 972, 1053, 976] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1027, 981, 1001, 1083, 1018, 1026, 1009, 1005, 1043, 1027, 1052, 990, 1021, 995, 1059, 1024, 971, 1029, 998, 1042, 1014, 1056, 1100, 1017, 1028, 1037, 1030, 1022, 1009, 1016, 1005, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1007, 1053, 1073, 1018, 1049, 1037, 1064, 1028, 1019, 1041, 1011, 1026, 944, 1001, 1033, 1021, 991, 1074, 1036, 987, 991, 1042, 999, 1020, 1059, 1019, 1048, 990, 1019, 1039, 1011, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1000, 1002, 1019, 1049, 1006, 1029, 1007, 1031, 1033, 1016, 1048, 1029, 1019, 993, 1028, 1043, 1028, 1014, 1003, 1046, 1021, 1016, 1050, 1010, 1031, 1035, 1002, 1027, 1039, 1005, 1050, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[973, 1055, 1028, 1066, 3, 0, 0, 0, 1001, 1007, 1008, 1027, 4, 2, 0, 0, 972, 1032, 1012, 1035, 0, 0, 0, 0, 988, 1061, 1061, 1057, 0, 0, 0, 0, 1037, 1017, 976, 1046, 1, 1, 0, 0, 1051, 954, 980, 1023, 9, 0, 0, 0, 1039, 1005, 998, 1024, 0, 0, 0, 0, 1066, 1052, 1040, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4049, 4051, 4167, 4078, 4017, 4066, 4215] avg=4096.0000 max=4215 max_rank=7 min=4017 min_rank=5 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1039, 1027, 1013, 1000, 3, 0, 0, 0, 1038, 1005, 1032, 1030, 4, 2, 0, 0, 1031, 988, 1000, 1045, 0, 0, 0, 0, 1021, 1085, 1021, 972, 0, 0, 0, 0, 1042, 1085, 991, 1017, 1, 1, 0, 0, 1022, 1029, 983, 1064, 9, 0, 0, 0, 1024, 1056, 1046, 981, 0, 0, 0, 0, 1060, 972, 1053, 976, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4111, 4064, 4099, 4137, 4107, 4107, 4061] avg=4096.0000 max=4137 max_rank=4 min=4061 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[996, 1003, 1031, 1014, 3, 0, 0, 0, 1007, 996, 1080, 1002, 4, 2, 0, 0, 1043, 1040, 1023, 1014, 0, 0, 0, 0, 1027, 1045, 1087, 989, 0, 0, 0, 0, 989, 1009, 1064, 1034, 1, 1, 0, 0, 1053, 1030, 1021, 1027, 9, 0, 0, 0, 1007, 1035, 1037, 1049, 0, 0, 0, 0, 980, 1029, 1020, 967, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4091, 4120, 4148, 4098, 4140, 4128, 3996] avg=4096.0000 max=4148 max_rank=3 min=3996 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1050, 1025, 982, 1042, 6, 0, 0, 0, 1034, 1018, 986, 1045, 6, 9, 0, 0, 1036, 1013, 1025, 1008, 0, 0, 0, 0, 1054, 1040, 1015, 1019, 0, 0, 0, 0, 1021, 988, 1020, 1048, 5, 4, 0, 0, 994, 1006, 1037, 1009, 13, 0, 0, 0, 1014, 1023, 1064, 1003, 0, 0, 0, 0, 1036, 1045, 1017, 1004, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4098, 4082, 4128, 4086, 4059, 4104, 4106] avg=4096.0000 max=4128 max_rank=3 min=4059 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1027, 981, 1001, 1083, 3, 0, 0, 0, 1018, 1026, 1009, 1005, 4, 2, 0, 0, 1043, 1027, 1041, 990, 0, 0, 0, 0, 1021, 995, 1058, 1024, 0, 0, 0, 0, 971, 1029, 998, 1042, 1, 1, 0, 0, 1014, 1056, 1100, 1017, 9, 0, 0, 0, 1028, 1029, 1030, 1022, 0, 0, 0, 0, 1009, 1016, 1005, 1033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4064, 4101, 4098, 4042, 4196, 4109, 4063] avg=4096.0000 max=4196 max_rank=5 min=4042 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1011, 1000, 1023, 1050, 3, 0, 0, 0, 1056, 1028, 1003, 999, 4, 2, 0, 0, 1027, 1042, 1052, 1028, 0, 0, 0, 0, 984, 1033, 1007, 1024, 0, 0, 0, 0, 1025, 1057, 1040, 1020, 1, 1, 0, 0, 1038, 1030, 1024, 981, 9, 0, 0, 0, 1008, 1017, 1026, 1008, 0, 0, 0, 0, 1062, 1041, 1027, 977, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4092, 4149, 4048, 4144, 4082, 4059, 4107] avg=4096.0000 max=4149 max_rank=2 min=4048 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1007, 1053, 1073, 1018, 3, 0, 0, 0, 1049, 1037, 1064, 1028, 4, 2, 0, 0, 1019, 1041, 1000, 1026, 0, 0, 0, 0, 944, 1001, 1032, 1021, 0, 0, 0, 0, 991, 1074, 1036, 987, 1, 1, 0, 0, 991, 1042, 999, 1020, 9, 0, 0, 0, 1059, 1011, 1048, 990, 0, 0, 0, 0, 1019, 1039, 1011, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4184, 4086, 3998, 4090, 4061, 4108, 4087] avg=4096.0000 max=4184 max_rank=1 min=3998 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1000, 1002, 1019, 1049, 3, 0, 0, 0, 1006, 1029, 1007, 1031, 4, 2, 0, 0, 1033, 1016, 1037, 1029, 0, 0, 0, 0, 1019, 993, 1027, 1043, 0, 0, 0, 0, 1028, 1014, 1003, 1046, 1, 1, 0, 0, 1021, 1016, 1050, 1010, 9, 0, 0, 0, 1031, 1027, 1002, 1027, 0, 0, 0, 0, 1039, 1005, 1050, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4079, 4115, 4082, 4093, 4106, 4087, 4133] avg=4096.0000 max=4133 max_rank=7 min=4073 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8448, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1036, 1036, 1034, 1008, 998, 1058, 1027, 1021, 1051, 1064, 1021, 1005, 1022, 1012, 1034, 990, 1007, 1024, 991, 1045, 1003, 1033, 1080, 1067, 1010, 1002, 1038, 1005, 1034, 1023, 1019, 970] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1045, 1009, 1005, 984, 1030, 999, 1035, 1029, 1019, 1056, 1002, 1015, 1049, 1000, 1042, 1031, 1053, 1052, 1008, 996, 1025, 1011, 1078, 988, 1035, 1025, 1008, 1024, 1030, 1034, 1028, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1066, 1031, 979, 1034, 990, 1052, 1000, 1021, 1063, 1000, 1069, 995, 973, 1019, 1015, 1053, 1014, 1036, 1026, 1020, 989, 1063, 1020, 1016, 1114, 1017, 994, 1010, 982, 1074, 1043, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1030, 992, 1014, 981, 1100, 1027, 1069, 1032, 1040, 1066, 1014, 1052, 1041, 1015, 964, 984, 1078, 1013, 992, 1035, 1019, 988, 1020, 1007, 1029, 1010, 1067, 1011, 1004, 1012, 1025, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1027, 1058, 1018, 984, 1046, 1027, 1052, 1027, 1030, 1038, 1010, 1015, 1022, 1009, 1054, 1017, 1050, 1047, 974, 1018, 978, 1039, 1045, 1004, 1047, 1001, 1026, 1010, 983, 1059, 1027, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1068, 1010, 1025, 1060, 1003, 1035, 1006, 1006, 997, 991, 994, 1053, 1010, 990, 1077, 1046, 982, 1012, 1004, 1058, 1057, 997, 1048, 1038, 1057, 1027, 1021, 1011, 1042, 1036, 1030, 977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1049, 1032, 1002, 1015, 992, 1031, 1017, 1063, 1015, 1035, 1033, 1000, 1014, 1065, 1018, 991, 1027, 988, 982, 1021, 1020, 1048, 1067, 1010, 1046, 1012, 1052, 1059, 982, 1057, 1005, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1052, 984, 1019, 1001, 1009, 1066, 1046, 1027, 1051, 1020, 1043, 1038, 1012, 1052, 1049, 1036, 1012, 991, 972, 992, 1043, 1025, 1033, 1010, 1078, 1009, 1032, 1020, 1029, 1009, 986, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1036, 1036, 1034, 1008, 5, 5, 0, 0, 998, 1036, 1027, 1021, 0, 0, 0, 0, 1051, 1048, 1021, 1005, 0, 0, 0, 0, 1022, 1012, 1034, 990, 7, 0, 0, 0, 1007, 1024, 991, 1045, 22, 9, 0, 0, 1003, 1033, 1068, 1067, 0, 0, 0, 0, 994, 1002, 1038, 1005, 0, 0, 0, 0, 1034, 1023, 1019, 970, 11, 7, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4082, 4125, 4065, 4098, 4171, 4039, 4064] avg=4096.0000 max=4171 max_rank=5 min=4039 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1045, 1009, 1005, 984, 5, 5, 0, 0, 1030, 979, 1035, 1029, 0, 0, 0, 0, 1019, 1041, 1002, 1015, 0, 0, 0, 0, 1049, 1000, 1042, 1031, 7, 0, 0, 0, 1053, 1052, 1008, 996, 20, 9, 0, 0, 1025, 1011, 1066, 988, 0, 0, 0, 0, 1019, 1025, 1008, 1024, 0, 0, 0, 0, 1030, 1034, 1028, 1023, 10, 7, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4073, 4077, 4129, 4138, 4090, 4076, 4132] avg=4096.0000 max=4138 max_rank=4 min=4053 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1066, 1031, 979, 1034, 4, 4, 0, 0, 990, 1031, 1000, 1021, 0, 0, 0, 0, 1063, 986, 1069, 995, 0, 0, 0, 0, 973, 1019, 1015, 1053, 7, 0, 0, 0, 1014, 1036, 1026, 1020, 21, 9, 0, 0, 989, 1063, 1009, 1016, 0, 0, 0, 0, 1097, 1017, 994, 1010, 0, 0, 0, 0, 982, 1074, 1043, 990, 10, 8, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4042, 4113, 4067, 4126, 4077, 4118, 4107] avg=4096.0000 max=4126 max_rank=4 min=4042 min_rank=1 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1030, 992, 1014, 981, 5, 4, 0, 0, 1100, 1006, 1069, 1032, 0, 0, 0, 0, 1040, 1050, 1014, 1052, 0, 0, 0, 0, 1041, 1015, 964, 984, 7, 0, 0, 0, 1078, 1013, 992, 1035, 21, 9, 0, 0, 1019, 988, 1009, 1007, 0, 0, 0, 0, 1013, 1010, 1067, 1011, 0, 0, 0, 0, 1004, 1012, 1025, 1037, 11, 7, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4207, 4156, 4011, 4148, 4023, 4101, 4096] avg=4096.0000 max=4207 max_rank=1 min=4011 min_rank=3 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1027, 1058, 1018, 984, 8, 9, 0, 0, 1046, 1002, 1052, 1027, 0, 0, 0, 0, 1030, 1016, 1010, 1015, 0, 0, 0, 0, 1022, 1009, 1054, 1017, 13, 0, 0, 0, 1050, 1047, 974, 1018, 25, 12, 0, 0, 978, 1039, 1023, 1004, 0, 0, 0, 0, 1023, 1001, 1026, 1010, 0, 0, 0, 0, 983, 1059, 1027, 1026, 14, 12, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4127, 4071, 4115, 4126, 4044, 4060, 4121] avg=4096.0000 max=4127 max_rank=1 min=4044 min_rank=5 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1068, 1010, 1025, 1060, 4, 4, 0, 0, 1003, 1014, 1006, 1006, 0, 0, 0, 0, 997, 977, 994, 1053, 0, 0, 0, 0, 1010, 990, 1077, 1046, 7, 0, 0, 0, 982, 1012, 1004, 1058, 21, 9, 0, 0, 1057, 997, 1037, 1038, 0, 0, 0, 0, 1040, 1027, 1021, 1011, 0, 0, 0, 0, 1042, 1036, 1030, 977, 10, 8, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4029, 4021, 4130, 4086, 4129, 4099, 4103] avg=4096.0000 max=4171 max_rank=0 min=4021 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1049, 1032, 1002, 1015, 5, 4, 0, 0, 992, 1010, 1017, 1063, 0, 0, 0, 0, 1015, 1020, 1033, 1000, 0, 0, 0, 0, 1014, 1065, 1018, 991, 7, 0, 0, 0, 1027, 988, 982, 1021, 21, 9, 0, 0, 1020, 1048, 1056, 1010, 0, 0, 0, 0, 1030, 1012, 1052, 1059, 0, 0, 0, 0, 982, 1057, 1005, 1020, 10, 7, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4082, 4068, 4095, 4048, 4134, 4153, 4081] avg=4096.0000 max=4153 max_rank=6 min=4048 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1052, 984, 1019, 1001, 5, 4, 0, 0, 1009, 1044, 1046, 1027, 0, 0, 0, 0, 1051, 1005, 1043, 1038, 0, 0, 0, 0, 1012, 1052, 1049, 1036, 7, 0, 0, 0, 1012, 991, 972, 992, 22, 9, 0, 0, 1043, 1025, 1022, 1010, 0, 0, 0, 0, 1061, 1009, 1032, 1020, 0, 0, 0, 0, 1029, 1009, 986, 1022, 10, 8, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4126, 4137, 4156, 3998, 4100, 4122, 4064] avg=4096.0000 max=4156 max_rank=3 min=3998 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8448, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1076, 1044, 1042, 1052, 1030, 1026, 1002, 970, 1080, 1051, 1005, 1015, 1038, 1025, 1028, 1010, 1042, 1023, 1019, 982, 1021, 1027, 992, 1004, 991, 1012, 1036, 1023, 1021, 1013, 1036, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1010, 1024, 1040, 1008, 1012, 1005, 1040, 1035, 1056, 1038, 1022, 1036, 1021, 1035, 1041, 965, 1047, 1001, 1038, 1024, 1019, 1047, 1037, 1021, 1041, 1065, 1023, 1006, 1008, 1006, 991, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[987, 1012, 987, 1043, 1032, 1024, 1018, 1011, 1049, 1000, 1029, 1044, 1009, 1003, 1057, 1024, 1085, 1020, 959, 1060, 986, 1011, 1022, 1054, 1002, 1015, 1050, 1061, 1026, 1020, 1026, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1081, 1008, 1057, 1019, 1009, 1054, 1010, 984, 997, 987, 1073, 998, 1077, 1033, 1011, 1028, 1047, 1013, 1078, 988, 1047, 1010, 1021, 1011, 1021, 1002, 1035, 1006, 988, 1008, 1038, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1037, 975, 990, 1024, 1033, 1028, 1023, 1017, 1013, 1010, 1000, 1001, 1021, 1001, 1032, 1027, 1029, 1027, 1054, 1069, 1037, 1014, 1017, 1028, 1045, 1036, 1057, 990, 1063, 973, 1079, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1041, 1008, 1038, 1032, 1037, 1036, 1036, 990, 1053, 998, 1000, 1020, 1079, 1014, 1013, 1066, 1032, 1035, 996, 1046, 1030, 1003, 973, 1032, 990, 1086, 1028, 1001, 1008, 970, 1048, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1018, 1011, 1055, 1041, 974, 1071, 1026, 1037, 1033, 1041, 985, 977, 989, 996, 1056, 1051, 995, 1085, 1043, 1062, 1001, 994, 1044, 1073, 1001, 1034, 987, 1026, 974, 998, 1044, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1046, 1060, 1047, 1022, 1046, 1052, 1039, 1011, 1042, 996, 1039, 1072, 1038, 1040, 1026, 989, 1029, 1039, 1020, 966, 1012, 1038, 1012, 996, 1005, 1026, 1023, 997, 1023, 999, 1010, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1056, 1044, 1042, 1052, 0, 0, 0, 0, 1030, 1026, 1002, 970, 3, 2, 0, 0, 1080, 1051, 1005, 1015, 1, 0, 0, 0, 1030, 1025, 1028, 1010, 0, 0, 0, 0, 1019, 1023, 1019, 982, 0, 0, 0, 0, 1021, 1027, 992, 1004, 17, 0, 0, 0, 991, 1012, 1036, 1023, 5, 0, 0, 0, 1021, 1013, 1036, 1032, 23, 0, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4033, 4152, 4093, 4043, 4061, 4067, 4125] avg=4096.0000 max=4194 max_rank=0 min=4033 min_rank=1 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[991, 1024, 1040, 1008, 0, 0, 0, 0, 1012, 1005, 1040, 1035, 3, 2, 0, 0, 1056, 1038, 1022, 1036, 0, 0, 0, 0, 1014, 1035, 1041, 965, 0, 0, 0, 0, 1024, 1001, 1038, 1024, 0, 0, 0, 0, 1019, 1047, 1037, 1021, 16, 0, 0, 0, 1041, 1065, 1023, 1006, 5, 0, 0, 0, 1008, 1006, 991, 1006, 23, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4097, 4152, 4055, 4087, 4140, 4140, 4034] avg=4096.0000 max=4152 max_rank=2 min=4034 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1061, 1008, 1057, 1019, 0, 0, 0, 0, 1009, 1054, 1010, 984, 3, 2, 0, 0, 997, 987, 1073, 998, 1, 0, 0, 0, 1068, 1033, 1011, 1028, 0, 0, 0, 0, 1024, 1013, 1078, 988, 0, 0, 0, 0, 1047, 1010, 1021, 1011, 17, 0, 0, 0, 1021, 1002, 1035, 1006, 6, 0, 0, 0, 988, 1008, 1038, 1029, 23, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4062, 4056, 4140, 4103, 4106, 4070, 4086] avg=4096.0000 max=4145 max_rank=0 min=4056 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1018, 975, 990, 1024, 0, 0, 0, 0, 1033, 1028, 1023, 1017, 3, 2, 0, 0, 1013, 1010, 1000, 1001, 0, 0, 0, 0, 1014, 1001, 1032, 1027, 0, 0, 0, 0, 1007, 1027, 1054, 1069, 0, 0, 0, 0, 1037, 1014, 1017, 1028, 16, 0, 0, 0, 1045, 1036, 1057, 990, 5, 0, 0, 0, 1063, 973, 1079, 1018, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4106, 4024, 4074, 4157, 4112, 4133, 4155] avg=4096.0000 max=4157 max_rank=4 min=4007 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16557 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[955, 1012, 987, 1043, 0, 0, 0, 0, 1032, 1024, 1018, 1011, 9, 6, 0, 0, 1049, 1000, 1029, 1044, 4, 0, 0, 0, 989, 1003, 1057, 1024, 0, 0, 0, 0, 1057, 1020, 959, 1060, 0, 0, 0, 0, 986, 1011, 1022, 1054, 20, 0, 0, 0, 1002, 1015, 1050, 1061, 10, 0, 0, 0, 1026, 1020, 1026, 1042, 28, 3, 0, 0] rerouted_tokens_per_ep_rank=[3997, 4100, 4126, 4073, 4096, 4093, 4138, 4145] avg=4096.0000 max=4145 max_rank=7 min=3997 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1022, 1008, 1038, 1032, 0, 0, 0, 0, 1037, 1036, 1036, 990, 3, 2, 0, 0, 1053, 998, 1000, 1020, 1, 0, 0, 0, 1070, 1014, 1013, 1066, 0, 0, 0, 0, 1010, 1035, 996, 1046, 0, 0, 0, 0, 1030, 1003, 973, 1032, 16, 0, 0, 0, 990, 1086, 1028, 1001, 6, 0, 0, 0, 1008, 970, 1048, 1029, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4104, 4072, 4163, 4087, 4054, 4111, 4077] avg=4096.0000 max=4163 max_rank=3 min=4054 min_rank=5 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[999, 1011, 1055, 1041, 0, 0, 0, 0, 974, 1071, 1026, 1037, 3, 2, 0, 0, 1033, 1041, 985, 977, 0, 0, 0, 0, 982, 996, 1056, 1051, 0, 0, 0, 0, 973, 1085, 1043, 1062, 0, 0, 0, 0, 1001, 994, 1044, 1073, 16, 0, 0, 0, 1001, 1034, 987, 1026, 5, 0, 0, 0, 974, 998, 1044, 1046, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4113, 4036, 4085, 4163, 4128, 4053, 4084] avg=4096.0000 max=4163 max_rank=4 min=4036 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1027, 1060, 1047, 1022, 0, 0, 0, 0, 1046, 1052, 1039, 1011, 3, 2, 0, 0, 1042, 996, 1039, 1072, 1, 0, 0, 0, 1030, 1040, 1026, 989, 0, 0, 0, 0, 1007, 1039, 1020, 966, 0, 0, 0, 0, 1012, 1038, 1012, 996, 16, 0, 0, 0, 1005, 1026, 1023, 997, 5, 0, 0, 0, 1023, 999, 1010, 1008, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4153, 4150, 4085, 4032, 4074, 4056, 4062] avg=4096.0000 max=4156 max_rank=0 min=4032 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16224 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1000, 1041, 1002, 1007, 1045, 993, 958, 1018, 1080, 1055, 1057, 1038, 1019, 991, 1035, 1010, 977, 1032, 1024, 1052, 979, 1057, 1017, 1017, 1015, 1076, 979, 1035, 1040, 1054, 1031, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1015, 1017, 1007, 1010, 1035, 1032, 1019, 1009, 1000, 1045, 1097, 985, 1059, 982, 1021, 1028, 997, 1013, 1047, 1067, 1030, 994, 988, 998, 998, 1052, 1033, 1024, 1018, 1060, 1063, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1022, 1027, 1033, 1042, 1081, 1022, 1056, 1009, 1089, 1045, 990, 1056, 996, 1009, 1074, 1043, 1014, 1022, 1003, 1043, 942, 999, 1013, 1054, 999, 969, 1022, 1024, 1000, 994, 1059, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[998, 1025, 998, 1034, 1083, 980, 1042, 1028, 1016, 1048, 1035, 1045, 1048, 955, 1007, 1057, 1031, 980, 1037, 1024, 1013, 1019, 1006, 1017, 1050, 1009, 1049, 1044, 1012, 997, 1066, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1033, 1068, 1035, 1038, 1028, 1007, 978, 1009, 989, 991, 1043, 999, 1016, 1085, 1017, 1041, 1013, 982, 1052, 1063, 990, 1037, 1033, 1015, 1045, 1013, 1034, 1058, 1028, 990, 1032, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1026, 1063, 1036, 1039, 1020, 1009, 1019, 1041, 1028, 1047, 1036, 1023, 989, 1051, 988, 1014, 1094, 1025, 1010, 1025, 1034, 1043, 1021, 996, 1024, 1066, 976, 1030, 1026, 988, 994, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[999, 1032, 1029, 1076, 1003, 998, 1031, 1098, 1024, 1045, 1068, 1025, 1000, 1038, 1003, 994, 1000, 973, 1005, 979, 1050, 1034, 1025, 995, 1073, 1006, 1040, 1021, 1056, 1020, 994, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1007, 1010, 1050, 1028, 1008, 1057, 1019, 1035, 947, 1053, 1086, 1006, 952, 1028, 995, 1057, 1042, 1024, 1024, 1023, 1044, 993, 988, 1086, 976, 1031, 1023, 1049, 1047, 1041, 1023, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1015, 1008, 1007, 1010, 0, 0, 0, 0, 1035, 1032, 1019, 1009, 0, 0, 0, 0, 1000, 1045, 1056, 985, 0, 0, 0, 0, 1059, 982, 1021, 1028, 10, 9, 0, 0, 997, 1013, 1047, 1067, 8, 0, 0, 0, 1030, 994, 988, 998, 31, 0, 0, 0, 998, 1052, 1033, 1016, 0, 0, 0, 0, 1018, 1060, 1063, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4095, 4086, 4109, 4132, 4041, 4099, 4166] avg=4096.0000 max=4166 max_rank=7 min=4040 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1022, 1018, 1033, 1042, 0, 0, 0, 0, 1081, 1022, 1056, 1009, 0, 0, 0, 0, 1089, 1045, 953, 1056, 0, 0, 0, 0, 996, 1009, 1074, 1043, 9, 9, 0, 0, 1014, 1022, 1003, 1043, 8, 0, 0, 0, 942, 999, 1013, 1054, 28, 0, 0, 0, 999, 969, 1022, 1016, 0, 0, 0, 0, 1000, 994, 1059, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4168, 4143, 4140, 4090, 4036, 4006, 4070] avg=4096.0000 max=4168 max_rank=1 min=4006 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16202 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1000, 1032, 1002, 1007, 0, 0, 0, 0, 1045, 993, 958, 1018, 0, 0, 0, 0, 1080, 1055, 1017, 1038, 0, 0, 0, 0, 1019, 991, 1035, 1010, 10, 9, 0, 0, 977, 1032, 1024, 1052, 8, 0, 0, 0, 979, 1057, 1017, 1017, 30, 0, 0, 0, 1015, 1076, 979, 1027, 0, 0, 0, 0, 1040, 1054, 1031, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4014, 4190, 4074, 4093, 4100, 4097, 4159] avg=4096.0000 max=4190 max_rank=2 min=4014 min_rank=1 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[998, 1016, 998, 1034, 0, 0, 0, 0, 1083, 980, 1042, 1028, 0, 0, 0, 0, 1016, 1048, 996, 1045, 0, 0, 0, 0, 1048, 955, 1007, 1057, 10, 9, 0, 0, 1031, 980, 1037, 1024, 8, 0, 0, 0, 1013, 1019, 1006, 1017, 29, 0, 0, 0, 1050, 1009, 1049, 1036, 0, 0, 0, 0, 1012, 997, 1066, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4133, 4105, 4086, 4080, 4084, 4144, 4090] avg=4096.0000 max=4144 max_rank=6 min=4046 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[999, 1023, 1029, 1076, 0, 0, 0, 0, 1003, 998, 1031, 1098, 0, 0, 0, 0, 1024, 1045, 1028, 1025, 0, 0, 0, 0, 1000, 1038, 1003, 994, 10, 9, 0, 0, 1000, 973, 1005, 979, 8, 0, 0, 0, 1050, 1034, 1025, 995, 30, 0, 0, 0, 1073, 1006, 1040, 1013, 0, 0, 0, 0, 1056, 1020, 994, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4130, 4122, 4054, 3965, 4134, 4132, 4104] avg=4096.0000 max=4134 max_rank=5 min=3965 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 1053, 1036, 1039, 0, 0, 0, 0, 1020, 1009, 1019, 1041, 0, 0, 0, 0, 1028, 1047, 997, 1023, 0, 0, 0, 0, 989, 1051, 988, 1014, 10, 10, 0, 0, 1094, 1025, 1010, 1025, 8, 0, 0, 0, 1034, 1043, 1021, 996, 29, 0, 0, 0, 1024, 1066, 976, 1022, 0, 0, 0, 0, 1026, 988, 994, 987, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4089, 4095, 4062, 4162, 4123, 4088, 3995] avg=4096.0000 max=4162 max_rank=4 min=3995 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1033, 1053, 1035, 1038, 0, 0, 0, 0, 1026, 1007, 978, 1009, 0, 0, 0, 0, 989, 991, 997, 999, 0, 0, 0, 0, 1016, 1085, 1017, 1041, 13, 15, 5, 0, 1013, 982, 1052, 1063, 14, 1, 0, 0, 990, 1037, 1033, 1015, 33, 0, 0, 0, 1045, 1013, 1034, 1039, 0, 0, 0, 0, 1028, 990, 1032, 1006, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4020, 3976, 4192, 4125, 4108, 4131, 4057] avg=4096.0000 max=4192 max_rank=3 min=3976 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1007, 1001, 1050, 1028, 0, 0, 0, 0, 1008, 1057, 1019, 1035, 0, 0, 0, 0, 947, 1053, 1045, 1006, 0, 0, 0, 0, 952, 1028, 995, 1057, 10, 9, 0, 0, 1042, 1024, 1024, 1023, 8, 0, 0, 0, 1044, 993, 988, 1086, 31, 0, 0, 0, 976, 1031, 1023, 1041, 0, 0, 0, 0, 1047, 1041, 1023, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4119, 4051, 4051, 4121, 4142, 4071, 4127] avg=4096.0000 max=4142 max_rank=5 min=4051 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8448, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1038, 1025, 1026, 1035, 1017, 1045, 1059, 985, 1004, 1004, 1056, 1051, 1053, 1040, 968, 1005, 1031, 1005, 1015, 1018, 1027, 1047, 1063, 991, 980, 1053, 1073, 1044, 1019, 1009, 981, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[954, 1000, 991, 1032, 1058, 1053, 1015, 1034, 1029, 1044, 1025, 981, 1009, 1036, 1045, 1051, 1022, 1005, 1030, 1024, 1055, 992, 994, 1010, 1066, 1043, 1022, 1022, 961, 1077, 1049, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1046, 1070, 1008, 1007, 1034, 950, 989, 986, 1008, 993, 1079, 1082, 1083, 1031, 1060, 1015, 993, 1025, 1025, 1048, 1025, 1040, 1044, 953, 1004, 1006, 1019, 1016, 1019, 1037, 1044, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[951, 1059, 1061, 1000, 1004, 999, 1015, 1075, 1016, 1106, 988, 1048, 969, 1059, 1006, 1028, 1015, 1052, 1094, 1046, 1037, 1045, 986, 1019, 1035, 912, 1013, 960, 1088, 1012, 1023, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1017, 1081, 1069, 1004, 1040, 992, 1004, 1027, 1009, 936, 1012, 1063, 1002, 1103, 1088, 1003, 1031, 996, 1031, 971, 1017, 1029, 1019, 1049, 1026, 1019, 968, 1038, 1035, 996, 1079, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1004, 1029, 1018, 996, 992, 1076, 972, 986, 1037, 1051, 993, 1058, 1000, 1014, 1106, 1009, 1006, 976, 1034, 972, 1052, 1001, 1041, 1056, 1036, 976, 1006, 1026, 985, 1079, 1087, 1094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1010, 1001, 1026, 1067, 1043, 1055, 1042, 990, 988, 999, 987, 1016, 1030, 1037, 1020, 1087, 1012, 1074, 1021, 1028, 1051, 1002, 1015, 1066, 991, 992, 1024, 1021, 997, 1005, 1023, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1056, 1019, 963, 1007, 1025, 1048, 1004, 994, 965, 992, 1058, 975, 1041, 983, 1077, 1027, 1059, 1069, 1012, 1044, 1016, 1041, 1004, 1068, 1028, 1035, 1040, 1033, 1009, 1009, 1002, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1038, 1025, 1026, 1035, 10, 1, 0, 0, 1017, 1045, 1059, 985, 10, 8, 0, 0, 1004, 1004, 1056, 1051, 13, 0, 0, 0, 1053, 1040, 933, 1005, 0, 0, 0, 0, 1031, 1005, 1014, 1018, 0, 0, 0, 0, 1017, 1047, 1063, 991, 0, 0, 0, 0, 980, 1053, 1073, 1044, 27, 0, 0, 0, 1019, 1009, 981, 978, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4124, 4128, 4031, 4068, 4118, 4177, 3987] avg=4096.0000 max=4177 max_rank=6 min=3987 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[954, 1000, 991, 1032, 10, 1, 0, 0, 1058, 1053, 1015, 1034, 10, 9, 0, 0, 1029, 1044, 1025, 981, 13, 0, 0, 0, 1009, 1036, 1006, 1051, 0, 0, 0, 0, 1022, 1005, 1029, 1024, 0, 0, 0, 0, 1045, 992, 994, 1010, 0, 0, 0, 0, 1066, 1043, 1022, 1022, 30, 0, 0, 0, 961, 1077, 1049, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3988, 4179, 4092, 4102, 4080, 4041, 4183, 4103] avg=4096.0000 max=4183 max_rank=6 min=3988 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[951, 1059, 1061, 1000, 10, 2, 0, 0, 1004, 999, 1015, 1075, 10, 9, 0, 0, 1016, 1106, 988, 1048, 13, 0, 0, 0, 969, 1059, 969, 1028, 0, 0, 0, 0, 1015, 1052, 1092, 1046, 0, 0, 0, 0, 1027, 1045, 986, 1019, 0, 0, 0, 0, 1035, 912, 1013, 960, 28, 0, 0, 0, 1088, 1012, 1023, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4112, 4171, 4025, 4205, 4077, 3948, 4147] avg=4096.0000 max=4205 max_rank=4 min=3948 min_rank=6 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1046, 1070, 1008, 1007, 10, 1, 0, 0, 1034, 950, 989, 986, 10, 9, 0, 0, 1008, 993, 1079, 1082, 13, 0, 0, 0, 1083, 1031, 1021, 1015, 0, 0, 0, 0, 993, 1025, 1024, 1048, 0, 0, 0, 0, 1015, 1040, 1044, 953, 0, 0, 0, 0, 1004, 1006, 1019, 1016, 30, 0, 0, 0, 1019, 1037, 1044, 1006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 3978, 4175, 4150, 4090, 4052, 4075, 4106] avg=4096.0000 max=4175 max_rank=2 min=3978 min_rank=1 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1004, 1029, 1018, 996, 10, 2, 0, 0, 992, 1076, 972, 986, 11, 10, 0, 0, 1037, 1051, 993, 1058, 14, 0, 0, 0, 1000, 1014, 1065, 1009, 0, 0, 0, 0, 1006, 976, 1032, 972, 0, 0, 0, 0, 1042, 1001, 1041, 1056, 0, 0, 0, 0, 1036, 976, 1006, 1026, 31, 0, 0, 0, 985, 1079, 1087, 1069, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4047, 4153, 4088, 3986, 4140, 4075, 4220] avg=4096.0000 max=4220 max_rank=7 min=3986 min_rank=4 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1017, 1081, 1069, 1004, 12, 7, 0, 0, 1040, 992, 1004, 1027, 13, 13, 0, 0, 1009, 936, 1012, 1063, 17, 5, 0, 0, 1002, 1103, 1040, 1003, 0, 0, 0, 0, 1031, 996, 1024, 971, 0, 0, 0, 0, 1000, 1029, 1019, 1049, 0, 0, 0, 0, 1026, 1019, 968, 1038, 35, 0, 0, 0, 1035, 996, 1079, 984, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4089, 4042, 4148, 4022, 4097, 4086, 4094] avg=4096.0000 max=4190 max_rank=0 min=4022 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1010, 1001, 1026, 1067, 10, 1, 0, 0, 1043, 1055, 1042, 990, 10, 9, 0, 0, 988, 999, 987, 1016, 13, 0, 0, 0, 1030, 1037, 982, 1087, 0, 0, 0, 0, 1012, 1074, 1020, 1028, 0, 0, 0, 0, 1041, 1002, 1015, 1066, 0, 0, 0, 0, 991, 992, 1024, 1021, 29, 0, 0, 0, 997, 1005, 1023, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4149, 4003, 4136, 4134, 4124, 4057, 4050] avg=4096.0000 max=4149 max_rank=1 min=4003 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1056, 1019, 963, 1007, 10, 1, 0, 0, 1025, 1048, 1004, 994, 10, 9, 0, 0, 965, 992, 1058, 975, 14, 0, 0, 0, 1041, 983, 1037, 1027, 0, 0, 0, 0, 1059, 1069, 1011, 1044, 0, 0, 0, 0, 1006, 1041, 1004, 1068, 0, 0, 0, 0, 1028, 1035, 1040, 1033, 31, 0, 0, 0, 1009, 1009, 1002, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4090, 4004, 4088, 4183, 4119, 4167, 4061] avg=4096.0000 max=4183 max_rank=4 min=4004 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16530 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8064, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1010, 1003, 1024, 1038, 1028, 1060, 971, 1021, 1049, 1064, 1056, 1017, 1041, 994, 1016, 1008, 1051, 1058, 1014, 1024, 986, 994, 1023, 1017, 1002, 1007, 1057, 1027, 1023, 1018, 1030, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1061, 951, 1009, 1030, 1041, 999, 1079, 1023, 1002, 1002, 1032, 1034, 1062, 947, 1032, 1021, 1035, 1030, 1008, 998, 1034, 1040, 1019, 1028, 1062, 1003, 1020, 987, 1038, 1046, 1031, 1064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1027, 1019, 1011, 1012, 1056, 1059, 1006, 1011, 997, 1024, 1025, 1001, 1035, 1054, 998, 1042, 1025, 1041, 1004, 1074, 1050, 1014, 970, 1043, 992, 1013, 1034, 995, 1036, 1011, 1036, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1036, 1027, 1116, 991, 1017, 1024, 1027, 988, 1025, 1002, 1028, 1053, 1023, 1015, 1028, 1024, 1050, 1022, 1008, 990, 1037, 1053, 983, 999, 993, 1067, 996, 1051, 1029, 1015, 1009, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1008, 1040, 1057, 1033, 995, 1001, 1003, 1042, 977, 1022, 1124, 1040, 1016, 975, 1034, 1001, 1061, 1019, 1012, 1069, 1056, 1031, 1006, 1035, 991, 1028, 1019, 1032, 1023, 1041, 1015, 962] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1012, 1017, 987, 1067, 996, 1012, 1026, 994, 1057, 1016, 1000, 1019, 1026, 1057, 1024, 1034, 1024, 1013, 1043, 1074, 1018, 1041, 979, 1041, 1007, 982, 1065, 1005, 1045, 1014, 1027, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[965, 1013, 1053, 1009, 1018, 997, 982, 1019, 1012, 974, 1072, 1010, 999, 1029, 997, 1060, 1009, 1001, 1040, 1058, 1018, 1016, 1051, 1056, 1009, 1003, 1025, 1015, 1080, 1037, 1078, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1048, 1034, 1070, 999, 1001, 1064, 1042, 1017, 1023, 994, 996, 977, 1018, 1013, 1001, 1013, 990, 1028, 1027, 1033, 1042, 991, 1044, 1029, 1041, 1009, 1008, 1065, 1027, 1035, 1045, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1010, 1003, 1023, 1038, 0, 0, 0, 0, 1028, 1060, 971, 1021, 18, 0, 0, 0, 1049, 1064, 1056, 1017, 5, 0, 0, 0, 1041, 994, 1016, 1008, 13, 3, 0, 0, 1051, 1058, 1014, 1005, 0, 0, 0, 0, 986, 994, 1023, 1017, 1, 1, 0, 0, 1002, 1007, 1057, 1027, 19, 0, 0, 0, 1023, 1018, 1030, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4098, 4191, 4075, 4128, 4022, 4112, 4068] avg=4096.0000 max=4191 max_rank=2 min=4022 min_rank=5 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1036, 1027, 1115, 991, 0, 0, 0, 0, 1017, 1024, 1027, 988, 18, 0, 0, 0, 1025, 1002, 1028, 1053, 5, 0, 0, 0, 1023, 1015, 1028, 1024, 12, 3, 0, 0, 1050, 1022, 1008, 972, 0, 0, 0, 0, 1037, 1053, 983, 999, 1, 1, 0, 0, 993, 1067, 996, 1051, 19, 0, 0, 0, 1029, 1015, 1009, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4074, 4113, 4105, 4052, 4074, 4126, 4055] avg=4096.0000 max=4169 max_rank=0 min=4052 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1027, 1019, 1010, 1012, 0, 0, 0, 0, 1056, 1059, 1006, 1011, 18, 0, 0, 0, 997, 1024, 1025, 1001, 5, 0, 0, 0, 1035, 1054, 998, 1042, 13, 3, 0, 0, 1025, 1041, 1004, 1055, 0, 0, 0, 0, 1050, 1014, 970, 1043, 1, 1, 0, 0, 992, 1013, 1034, 995, 20, 0, 0, 0, 1036, 1011, 1036, 1012, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4150, 4052, 4145, 4125, 4079, 4054, 4095] avg=4096.0000 max=4150 max_rank=1 min=4052 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1061, 951, 1008, 1030, 0, 0, 0, 0, 1041, 999, 1079, 1023, 19, 0, 0, 0, 1002, 1002, 1032, 1034, 5, 0, 0, 0, 1062, 947, 1032, 1021, 12, 3, 0, 0, 1035, 1030, 1008, 980, 0, 0, 0, 0, 1034, 1040, 1019, 1028, 1, 1, 0, 0, 1062, 1003, 1020, 987, 20, 0, 0, 0, 1038, 1046, 1031, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4161, 4075, 4077, 4053, 4123, 4092, 4137] avg=4096.0000 max=4161 max_rank=1 min=4050 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1012, 1017, 986, 1067, 0, 0, 0, 0, 996, 1012, 1026, 994, 18, 0, 0, 0, 1057, 1016, 1000, 1019, 5, 0, 0, 0, 1026, 1057, 1024, 1034, 13, 3, 0, 0, 1024, 1013, 1043, 1055, 0, 0, 0, 0, 1018, 1041, 979, 1041, 1, 1, 0, 0, 1007, 982, 1065, 1005, 19, 0, 0, 0, 1045, 1014, 1027, 1006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4046, 4097, 4157, 4135, 4081, 4078, 4092] avg=4096.0000 max=4157 max_rank=3 min=4046 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[965, 1013, 1052, 1009, 0, 0, 0, 0, 1018, 997, 982, 1019, 19, 0, 0, 0, 1012, 974, 1072, 1010, 5, 0, 0, 0, 999, 1029, 997, 1060, 13, 3, 0, 0, 1009, 1001, 1040, 1039, 0, 0, 0, 0, 1018, 1016, 1051, 1056, 1, 1, 0, 0, 1009, 1003, 1025, 1015, 20, 0, 0, 0, 1080, 1037, 1078, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4035, 4073, 4101, 4089, 4143, 4072, 4216] avg=4096.0000 max=4216 max_rank=7 min=4035 min_rank=1 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1008, 1040, 1055, 1033, 0, 0, 0, 0, 995, 1001, 1003, 1042, 21, 0, 0, 0, 977, 1022, 1124, 1040, 9, 0, 0, 0, 1016, 975, 1034, 1001, 17, 4, 0, 0, 1061, 1019, 1012, 1035, 0, 0, 0, 0, 1056, 1031, 1006, 1035, 8, 2, 0, 0, 991, 1028, 1019, 1032, 22, 0, 0, 0, 1023, 1041, 1015, 915, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4062, 4172, 4047, 4127, 4138, 4092, 3994] avg=4096.0000 max=4172 max_rank=2 min=3994 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1048, 1034, 1069, 999, 0, 0, 0, 0, 1001, 1064, 1042, 1017, 18, 0, 0, 0, 1023, 994, 996, 977, 5, 0, 0, 0, 1018, 1013, 1001, 1013, 13, 3, 0, 0, 990, 1028, 1027, 1014, 0, 0, 0, 0, 1042, 991, 1044, 1029, 1, 1, 0, 0, 1041, 1009, 1008, 1065, 19, 0, 0, 0, 1027, 1035, 1045, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4142, 3995, 4061, 4059, 4108, 4142, 4111] avg=4096.0000 max=4150 max_rank=0 min=3995 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8448, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[995, 1013, 999, 1029, 990, 1055, 1039, 989, 1029, 985, 1020, 1016, 1034, 1006, 1096, 1006, 1012, 1043, 1028, 1030, 1096, 1007, 999, 1037, 977, 1025, 1040, 994, 1056, 1036, 1031, 1056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[999, 1024, 974, 1006, 1014, 994, 991, 1025, 1010, 1050, 1019, 1069, 996, 1058, 1019, 1030, 1017, 1019, 1049, 1043, 1032, 1046, 1004, 1049, 1061, 1020, 1063, 1006, 980, 1032, 1019, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1084, 1064, 1045, 980, 1026, 988, 974, 1023, 1010, 1053, 999, 999, 996, 1048, 1047, 979, 988, 1054, 1054, 1029, 981, 1032, 1038, 1020, 1081, 1051, 1044, 1003, 1025, 1003, 1024, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[966, 991, 1015, 1061, 1015, 1011, 1013, 1012, 1032, 1020, 1052, 1016, 1021, 1001, 1021, 1019, 1006, 1024, 1022, 1096, 949, 1041, 1071, 1027, 1032, 1086, 1030, 1082, 1043, 1015, 1004, 974] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1011, 1061, 1033, 966, 975, 999, 1007, 1008, 996, 988, 1080, 1011, 1059, 1037, 1014, 1058, 1058, 1016, 1045, 1017, 1041, 1032, 1029, 1015, 1019, 1036, 1025, 1060, 1031, 1002, 1001, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1080, 1051, 984, 975, 1040, 1001, 996, 1019, 1070, 982, 1017, 1013, 1047, 1043, 1008, 1041, 1000, 1073, 1046, 1071, 1038, 1028, 1050, 973, 1032, 1045, 1008, 1017, 980, 993, 1071, 976] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[994, 978, 1044, 1045, 1028, 1029, 1036, 1076, 983, 977, 1040, 1017, 1037, 1060, 1042, 1064, 938, 1010, 1002, 1028, 1012, 1010, 1020, 1051, 1069, 1008, 1046, 999, 989, 1031, 1047, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1019, 1029, 974, 1007, 1003, 1020, 1009, 1010, 984, 1042, 978, 999, 1063, 1035, 1003, 1064, 1064, 1036, 1002, 985, 1056, 1074, 1003, 1050, 1073, 1001, 1031, 996, 1070, 1035, 1020, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[995, 1013, 999, 1029, 27, 5, 0, 0, 990, 1055, 1039, 989, 34, 7, 0, 0, 1029, 985, 1020, 1016, 14, 11, 0, 0, 1034, 972, 1096, 1006, 0, 0, 0, 0, 1012, 1043, 1028, 1014, 0, 0, 0, 0, 1096, 991, 999, 1037, 0, 0, 0, 0, 943, 1025, 1040, 994, 0, 0, 0, 0, 1056, 1036, 1031, 1056, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4114, 4075, 4108, 4097, 4123, 4002, 4181] avg=4096.0000 max=4181 max_rank=7 min=4002 min_rank=6 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[999, 1024, 974, 1006, 28, 6, 0, 0, 1014, 994, 991, 1025, 37, 7, 0, 0, 1010, 1050, 1019, 1069, 14, 11, 0, 0, 996, 1023, 1019, 1030, 0, 0, 0, 0, 1017, 1019, 1049, 1027, 0, 0, 0, 0, 1032, 1029, 1004, 1049, 0, 0, 0, 0, 1024, 1020, 1063, 1006, 0, 0, 0, 0, 980, 1032, 1019, 1050, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4068, 4173, 4068, 4112, 4114, 4113, 4083] avg=4096.0000 max=4173 max_rank=2 min=4037 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[966, 991, 1015, 1061, 26, 6, 0, 0, 1015, 1011, 1013, 1012, 36, 7, 0, 0, 1032, 1020, 1052, 1016, 15, 11, 0, 0, 1021, 968, 1021, 1019, 0, 0, 0, 0, 1006, 1024, 1022, 1079, 0, 0, 0, 0, 949, 1024, 1071, 1027, 0, 0, 0, 0, 996, 1086, 1030, 1082, 0, 0, 0, 0, 1043, 1015, 1004, 974, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4094, 4146, 4029, 4131, 4071, 4194, 4038] avg=4096.0000 max=4194 max_rank=6 min=4029 min_rank=3 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1084, 1064, 1045, 980, 28, 6, 0, 0, 1026, 988, 974, 1023, 37, 7, 0, 0, 1010, 1053, 999, 999, 14, 11, 0, 0, 996, 1013, 1047, 979, 0, 0, 0, 0, 988, 1054, 1054, 1013, 0, 0, 0, 0, 981, 1015, 1038, 1020, 0, 0, 0, 0, 1044, 1051, 1044, 1003, 0, 0, 0, 0, 1025, 1003, 1024, 1026, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4207, 4055, 4086, 4035, 4109, 4054, 4142, 4080] avg=4096.0000 max=4207 max_rank=0 min=4035 min_rank=3 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1011, 1061, 1033, 966, 27, 6, 0, 0, 975, 999, 1007, 1008, 35, 7, 0, 0, 996, 988, 1080, 1011, 14, 11, 0, 0, 1059, 1003, 1014, 1058, 0, 0, 0, 0, 1058, 1016, 1045, 1001, 0, 0, 0, 0, 1041, 1015, 1029, 1015, 0, 0, 0, 0, 984, 1036, 1025, 1060, 0, 0, 0, 0, 1031, 1002, 1001, 1038, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4031, 4100, 4134, 4120, 4100, 4105, 4074] avg=4096.0000 max=4134 max_rank=3 min=4031 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[994, 978, 1044, 1045, 28, 5, 0, 0, 1028, 1029, 1036, 1076, 37, 7, 0, 0, 983, 977, 1040, 1017, 14, 11, 0, 0, 1037, 1025, 1042, 1064, 0, 0, 0, 0, 938, 1010, 1002, 1012, 0, 0, 0, 0, 1012, 994, 1020, 1051, 0, 0, 0, 0, 1032, 1008, 1046, 999, 0, 0, 0, 0, 989, 1031, 1047, 1058, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4213, 4042, 4168, 3962, 4077, 4085, 4127] avg=4096.0000 max=4213 max_rank=1 min=3962 min_rank=4 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16251 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1080, 1051, 984, 975, 32, 9, 0, 0, 1040, 1001, 996, 1019, 39, 12, 0, 0, 1070, 982, 1017, 1013, 19, 16, 0, 0, 1047, 999, 1008, 1041, 0, 0, 0, 0, 1000, 1073, 1046, 1047, 0, 0, 0, 0, 1038, 1003, 1050, 973, 0, 0, 0, 0, 993, 1045, 1008, 1017, 0, 0, 0, 0, 980, 993, 1071, 976, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4107, 4117, 4095, 4166, 4064, 4063, 4025] avg=4096.0000 max=4166 max_rank=4 min=4025 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1019, 1029, 974, 1007, 27, 6, 0, 0, 1003, 1020, 1009, 1010, 37, 7, 0, 0, 984, 1042, 978, 999, 14, 12, 0, 0, 1063, 1001, 1003, 1064, 0, 0, 0, 0, 1064, 1036, 1002, 969, 0, 0, 0, 0, 1056, 1056, 1003, 1050, 0, 0, 0, 0, 1036, 1001, 1031, 996, 0, 0, 0, 0, 1070, 1035, 1020, 1033, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4086, 4029, 4131, 4071, 4165, 4064, 4160] avg=4096.0000 max=4165 max_rank=5 min=4029 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[993, 1011, 1001, 1027, 1031, 1050, 1015, 988, 996, 1096, 1011, 1037, 1040, 995, 1009, 1019, 1011, 1041, 967, 1031, 1037, 1006, 1009, 1103, 1049, 1057, 1022, 1063, 1018, 1004, 1013, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1003, 1047, 1005, 1007, 1008, 1050, 1035, 1037, 997, 1082, 1043, 1046, 1047, 1030, 1019, 1024, 1013, 1018, 1023, 1004, 1066, 978, 1072, 997, 1045, 993, 954, 1036, 1010, 1075, 973, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1005, 1060, 1035, 1024, 1054, 956, 1035, 1041, 992, 1010, 1038, 990, 1069, 1044, 1018, 1027, 986, 1056, 1035, 994, 1037, 1001, 1049, 1007, 1038, 1018, 1032, 974, 1045, 1054, 1020, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1008, 1000, 977, 1063, 1050, 995, 1021, 1047, 1055, 1024, 1050, 1055, 1002, 997, 1037, 1034, 1060, 1022, 1054, 1046, 1026, 1038, 1009, 1010, 993, 993, 976, 1004, 1051, 1051, 1019, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1009, 1015, 1044, 1039, 1086, 993, 1032, 1071, 989, 1045, 1045, 1030, 1029, 1052, 989, 1029, 1012, 1019, 984, 1028, 991, 1017, 974, 1025, 1030, 1055, 1027, 1049, 1030, 1017, 1049, 964] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1049, 1040, 980, 1008, 1022, 1014, 1022, 1002, 1062, 1036, 1028, 1025, 968, 1013, 994, 1008, 1051, 1069, 979, 1026, 1044, 1001, 1060, 1049, 1005, 1069, 993, 1035, 1021, 1031, 1037, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1052, 1062, 986, 1024, 1008, 1006, 995, 964, 1001, 1007, 1024, 1048, 1020, 1016, 1024, 1050, 983, 1035, 1058, 1031, 1038, 1054, 1043, 1003, 1055, 1009, 1053, 1022, 961, 1039, 1074, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1044, 1024, 1025, 981, 1039, 971, 1017, 1073, 1032, 997, 1031, 1023, 997, 1047, 989, 1024, 977, 1026, 1013, 1012, 1037, 1032, 1039, 1053, 1002, 1055, 1057, 1051, 1022, 1050, 1030, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1005, 1060, 1035, 1024, 14, 0, 0, 0, 1054, 956, 1035, 1041, 3, 1, 0, 0, 992, 990, 1038, 990, 0, 0, 0, 0, 1069, 1044, 1018, 1027, 6, 6, 0, 0, 986, 1056, 1035, 994, 10, 2, 0, 0, 1021, 1001, 1049, 1007, 0, 0, 0, 0, 1038, 1013, 1032, 974, 0, 0, 0, 0, 1045, 1053, 1020, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4090, 4010, 4170, 4083, 4078, 4057, 4142] avg=4096.0000 max=4170 max_rank=3 min=4010 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[993, 1011, 1001, 1027, 15, 0, 0, 0, 1031, 1050, 1015, 988, 3, 1, 0, 0, 996, 1074, 1011, 1037, 0, 0, 0, 0, 1040, 995, 1009, 1019, 7, 6, 0, 0, 1011, 1041, 967, 1031, 10, 2, 0, 0, 1021, 1006, 1009, 1103, 0, 0, 0, 0, 1049, 1052, 1022, 1063, 0, 0, 0, 0, 1018, 1003, 1013, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4088, 4118, 4076, 4062, 4139, 4186, 4052] avg=4096.0000 max=4186 max_rank=6 min=4047 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1008, 1000, 977, 1063, 14, 0, 0, 0, 1050, 995, 1021, 1047, 3, 1, 0, 0, 1055, 1003, 1050, 1055, 0, 0, 0, 0, 1002, 997, 1037, 1034, 7, 6, 0, 0, 1060, 1022, 1054, 1046, 10, 2, 0, 0, 1010, 1038, 1009, 1010, 0, 0, 0, 0, 993, 988, 976, 1004, 0, 0, 0, 0, 1051, 1050, 1019, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4117, 4163, 4083, 4194, 4067, 3961, 4121] avg=4096.0000 max=4194 max_rank=4 min=3961 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1003, 1047, 1005, 1007, 20, 0, 0, 0, 1008, 1050, 1035, 1037, 7, 5, 0, 0, 997, 1051, 1043, 1046, 0, 0, 0, 0, 1047, 1030, 1019, 1024, 11, 9, 0, 0, 1013, 1018, 1023, 1004, 16, 4, 0, 0, 1041, 978, 1072, 997, 0, 0, 0, 0, 1045, 982, 954, 1036, 0, 0, 0, 0, 1010, 1070, 973, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4142, 4137, 4140, 4078, 4088, 4017, 4084] avg=4096.0000 max=4142 max_rank=1 min=4017 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16267 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1052, 1062, 986, 1024, 14, 0, 0, 0, 1008, 1006, 995, 964, 3, 1, 0, 0, 1001, 987, 1024, 1048, 0, 0, 0, 0, 1020, 1016, 1024, 1050, 6, 6, 0, 0, 983, 1035, 1058, 1031, 10, 2, 0, 0, 1022, 1054, 1043, 1003, 0, 0, 0, 0, 1055, 1004, 1053, 1022, 0, 0, 0, 0, 961, 1038, 1074, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 3977, 4060, 4122, 4119, 4122, 4134, 4096] avg=4096.0000 max=4138 max_rank=0 min=3977 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1049, 1040, 980, 1008, 14, 0, 0, 0, 1022, 1014, 1022, 1002, 3, 1, 0, 0, 1062, 1015, 1028, 1025, 0, 0, 0, 0, 968, 1013, 994, 1008, 7, 6, 0, 0, 1051, 1069, 979, 1026, 10, 2, 0, 0, 1028, 1001, 1060, 1049, 0, 0, 0, 0, 1005, 1064, 993, 1035, 0, 0, 0, 0, 1021, 1030, 1037, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4064, 4130, 3996, 4137, 4138, 4097, 4115] avg=4096.0000 max=4138 max_rank=5 min=3996 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1009, 1015, 1044, 1039, 15, 0, 0, 0, 1086, 993, 1032, 1071, 3, 1, 0, 0, 989, 1023, 1045, 1030, 0, 0, 0, 0, 1029, 1052, 989, 1029, 7, 6, 0, 0, 1012, 1019, 984, 1028, 10, 2, 0, 0, 975, 1017, 974, 1025, 0, 0, 0, 0, 1030, 1050, 1027, 1049, 0, 0, 0, 0, 1030, 1016, 1049, 964, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4186, 4087, 4112, 4055, 3991, 4156, 4059] avg=4096.0000 max=4186 max_rank=1 min=3991 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1044, 1024, 1025, 981, 14, 0, 0, 0, 1039, 971, 1017, 1073, 3, 1, 0, 0, 1032, 977, 1031, 1023, 0, 0, 0, 0, 997, 1047, 989, 1024, 6, 6, 0, 0, 977, 1026, 1013, 1012, 10, 2, 0, 0, 1021, 1032, 1039, 1053, 0, 0, 0, 0, 1002, 1050, 1057, 1051, 0, 0, 0, 0, 1022, 1049, 1030, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4104, 4063, 4069, 4040, 4145, 4160, 4099] avg=4096.0000 max=4160 max_rank=6 min=4040 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8064, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[993, 982, 1005, 1005, 1010, 1045, 1073, 1036, 1062, 1019, 1050, 987, 1008, 995, 1071, 999, 1014, 999, 1071, 1092, 1011, 990, 1059, 1055, 1058, 997, 995, 1059, 1029, 991, 984, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1045, 1023, 1007, 1014, 1023, 994, 1040, 988, 999, 1054, 1017, 1030, 1003, 1046, 967, 1111, 979, 1015, 1053, 1015, 1043, 988, 1043, 1023, 1025, 1034, 1016, 1076, 1044, 977, 1040, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1014, 1003, 1077, 1037, 997, 1038, 997, 1061, 989, 1016, 999, 1028, 1049, 1009, 1016, 1066, 1059, 1025, 1015, 1029, 1050, 1031, 1026, 1011, 994, 1033, 1034, 1046, 972, 1051, 1016, 980] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1021, 1075, 1003, 1015, 1049, 1034, 1039, 994, 1047, 1016, 1048, 1035, 1042, 1058, 990, 984, 1044, 1037, 1019, 1028, 1032, 1021, 997, 992, 1052, 1016, 1013, 1029, 1029, 993, 1017, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1009, 1004, 1054, 984, 1070, 1023, 1057, 1010, 1015, 1054, 1004, 1020, 1031, 1024, 955, 1072, 1051, 1047, 1036, 1022, 1020, 1007, 1043, 1029, 970, 1067, 1002, 1000, 1002, 1046, 1027, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1017, 1004, 1065, 1015, 1018, 1054, 1041, 1023, 1042, 1005, 1015, 1029, 1040, 1026, 990, 1019, 1034, 1043, 1037, 993, 1039, 1018, 1043, 1015, 1030, 1014, 1016, 1013, 1047, 1001, 1003, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[994, 1031, 1020, 1069, 1062, 1016, 1050, 1022, 1048, 1037, 980, 1013, 1052, 990, 949, 1016, 1020, 1066, 1069, 1004, 1036, 1019, 993, 1033, 1001, 999, 1043, 1048, 1061, 976, 1028, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[981, 1004, 1034, 1050, 984, 999, 1045, 977, 1036, 1009, 1045, 1006, 1062, 1035, 1039, 1029, 1077, 974, 1058, 1018, 1021, 1001, 1052, 1009, 996, 980, 1052, 1049, 1040, 1062, 991, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1014, 1003, 1077, 1037, 9, 3, 0, 0, 997, 1038, 988, 1061, 0, 0, 0, 0, 989, 1016, 999, 1028, 1, 0, 0, 0, 1049, 1009, 1016, 1066, 2, 0, 0, 0, 1059, 1025, 983, 1029, 0, 0, 0, 0, 1050, 1031, 1026, 1011, 2, 0, 0, 0, 994, 1033, 1034, 1046, 1, 0, 0, 0, 972, 1051, 1016, 980, 23, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4084, 4033, 4142, 4096, 4120, 4108, 4042] avg=4096.0000 max=4143 max_rank=0 min=4033 min_rank=2 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1021, 1075, 1003, 1015, 9, 4, 0, 0, 1049, 1034, 1028, 994, 0, 0, 0, 0, 1047, 1016, 1048, 1035, 1, 0, 0, 0, 1042, 1058, 990, 984, 3, 0, 0, 0, 1044, 1037, 987, 1028, 0, 0, 0, 0, 1032, 1021, 997, 992, 2, 0, 0, 0, 1052, 1016, 1013, 1029, 1, 0, 0, 0, 1029, 993, 1017, 999, 23, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4105, 4147, 4077, 4096, 4044, 4111, 4061] avg=4096.0000 max=4147 max_rank=2 min=4044 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[993, 982, 1005, 1005, 10, 4, 0, 0, 1010, 1045, 1062, 1036, 0, 0, 0, 0, 1062, 1019, 1050, 987, 1, 0, 0, 0, 1008, 995, 1071, 999, 3, 0, 0, 0, 1014, 999, 1037, 1092, 0, 0, 0, 0, 1011, 990, 1059, 1055, 2, 0, 0, 0, 1058, 997, 995, 1059, 1, 0, 0, 0, 1029, 991, 984, 1024, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[3999, 4153, 4119, 4076, 4142, 4117, 4110, 4052] avg=4096.0000 max=4153 max_rank=1 min=3999 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[994, 1031, 1020, 1069, 10, 4, 0, 0, 1062, 1016, 1039, 1022, 0, 0, 0, 0, 1048, 1037, 980, 1013, 1, 0, 0, 0, 1052, 990, 949, 1016, 3, 0, 0, 0, 1020, 1066, 1035, 1004, 0, 0, 0, 0, 1036, 1019, 993, 1033, 2, 0, 0, 0, 1001, 999, 1043, 1048, 1, 0, 0, 0, 1061, 976, 1028, 1023, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4139, 4079, 4010, 4125, 4083, 4092, 4112] avg=4096.0000 max=4139 max_rank=1 min=4010 min_rank=3 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1045, 1023, 1007, 1014, 10, 4, 0, 0, 1023, 994, 1029, 988, 0, 0, 0, 0, 999, 1054, 1017, 1030, 1, 0, 0, 0, 1003, 1046, 967, 1111, 3, 0, 0, 0, 979, 1015, 1019, 1015, 0, 0, 0, 0, 1043, 988, 1043, 1023, 2, 0, 0, 0, 1025, 1034, 1016, 1076, 1, 0, 0, 0, 1044, 977, 1040, 1036, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4034, 4101, 4130, 4028, 4099, 4152, 4121] avg=4096.0000 max=4152 max_rank=6 min=4028 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[981, 1004, 1034, 1050, 10, 4, 0, 0, 984, 999, 1034, 977, 0, 0, 0, 0, 1036, 1009, 1045, 1006, 1, 0, 0, 0, 1062, 1035, 1039, 1029, 3, 0, 0, 0, 1077, 974, 1024, 1018, 0, 0, 0, 0, 1021, 1001, 1052, 1009, 2, 0, 0, 0, 996, 980, 1052, 1049, 1, 0, 0, 0, 1040, 1062, 991, 1053, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 3994, 4097, 4168, 4093, 4085, 4078, 4170] avg=4096.0000 max=4170 max_rank=7 min=3994 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1017, 1004, 1065, 1015, 10, 4, 0, 0, 1018, 1054, 1030, 1023, 0, 0, 0, 0, 1042, 1005, 1015, 1029, 1, 0, 0, 0, 1040, 1026, 990, 1019, 3, 0, 0, 0, 1034, 1043, 1003, 993, 0, 0, 0, 0, 1039, 1018, 1043, 1015, 2, 0, 0, 0, 1030, 1014, 1016, 1013, 1, 0, 0, 0, 1047, 1001, 1003, 1019, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4125, 4092, 4078, 4073, 4117, 4074, 4094] avg=4096.0000 max=4125 max_rank=1 min=4073 min_rank=4 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1009, 1004, 1054, 984, 13, 6, 0, 0, 1070, 1023, 1031, 1010, 0, 0, 0, 0, 1015, 1054, 1004, 1020, 7, 0, 0, 0, 1031, 1024, 955, 1072, 5, 0, 0, 0, 1051, 1047, 995, 1022, 0, 0, 0, 0, 1020, 1007, 1043, 1029, 4, 0, 0, 0, 970, 1067, 1002, 1000, 4, 0, 0, 0, 1002, 1046, 1027, 1013, 28, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4134, 4100, 4087, 4115, 4103, 4043, 4116] avg=4096.0000 max=4134 max_rank=1 min=4043 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8064, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1011, 1037, 1028, 1000, 1031, 1042, 1054, 1016, 990, 1063, 1038, 1015, 1055, 998, 1046, 1014, 1023, 1005, 1008, 1013, 969, 1000, 1003, 1018, 1047, 1027, 1014, 1006, 1062, 1083, 1007, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1007, 1006, 1034, 1031, 999, 1012, 1058, 1052, 1026, 1078, 1016, 1026, 999, 1046, 1036, 1023, 1021, 971, 1038, 1022, 978, 1016, 1052, 1051, 1056, 993, 1033, 966, 1052, 1033, 1020, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1050, 1106, 992, 1089, 1016, 1044, 1006, 992, 988, 979, 995, 1006, 1057, 994, 1013, 1012, 1023, 1012, 1013, 992, 1044, 1020, 1076, 1038, 1019, 1001, 1051, 1003, 1031, 1014, 1030, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1014, 1003, 1015, 1033, 998, 1051, 1025, 1021, 1025, 1046, 1060, 1018, 996, 1001, 986, 1007, 1031, 1058, 1020, 1040, 1014, 1038, 1059, 1001, 1052, 1019, 1012, 1016, 1035, 982, 1005, 1087] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1019, 1018, 1018, 1034, 1026, 1026, 1009, 984, 1069, 983, 1037, 990, 1026, 1009, 973, 1017, 1003, 1043, 1099, 1049, 1054, 1013, 1033, 1020, 1064, 1019, 1028, 1031, 1010, 1055, 1018, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1028, 998, 1023, 1038, 1014, 1039, 1032, 1007, 1054, 1033, 1057, 1050, 1021, 1027, 981, 967, 1010, 1039, 1017, 1045, 1019, 997, 1042, 1002, 995, 1026, 1032, 1053, 1064, 1030, 1037, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1012, 1024, 998, 1006, 1005, 1001, 1021, 1035, 1023, 1005, 1054, 994, 1032, 1063, 1042, 1017, 1082, 1048, 992, 1014, 1016, 1078, 1016, 1025, 1013, 1067, 971, 1038, 1031, 1035, 980, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1037, 1012, 1063, 1053, 980, 1042, 999, 1021, 1037, 1014, 1015, 1014, 1019, 996, 1026, 1019, 1036, 980, 1016, 1089, 1008, 1052, 1020, 1019, 1011, 984, 1069, 1030, 1014, 1025, 1079, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1007, 1006, 1034, 1023, 0, 0, 0, 0, 999, 1012, 1058, 1052, 8, 3, 1, 0, 1026, 1078, 1013, 1026, 0, 0, 0, 0, 999, 1046, 1036, 1023, 22, 9, 0, 0, 1021, 971, 1038, 1012, 0, 0, 0, 0, 978, 1016, 1050, 1051, 0, 0, 0, 0, 1056, 993, 1033, 966, 2, 0, 0, 0, 1030, 1033, 1020, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4133, 4143, 4135, 4042, 4095, 4050, 4100] avg=4096.0000 max=4143 max_rank=2 min=4042 min_rank=4 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1050, 1106, 992, 1080, 0, 0, 0, 0, 1016, 1044, 1006, 992, 9, 3, 1, 0, 988, 979, 992, 1006, 0, 0, 0, 0, 1057, 994, 1013, 1012, 21, 8, 0, 0, 1023, 1012, 1013, 983, 0, 0, 0, 0, 1044, 1020, 1074, 1038, 0, 0, 0, 0, 1019, 1001, 1051, 1003, 2, 0, 0, 0, 1010, 1014, 1030, 1062, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4228, 4071, 3965, 4105, 4031, 4176, 4076, 4116] avg=4096.0000 max=4228 max_rank=0 min=3965 min_rank=2 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1014, 1003, 1015, 1025, 0, 0, 0, 0, 998, 1051, 1025, 1021, 8, 3, 1, 0, 1025, 1046, 1057, 1018, 0, 0, 0, 0, 996, 1001, 986, 1007, 21, 9, 0, 0, 1031, 1058, 1020, 1030, 0, 0, 0, 0, 1014, 1038, 1057, 1001, 0, 0, 0, 0, 1052, 1019, 1012, 1016, 2, 0, 0, 0, 1014, 982, 1005, 1087, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4107, 4146, 4020, 4139, 4110, 4101, 4088] avg=4096.0000 max=4146 max_rank=2 min=4020 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1011, 1037, 1028, 992, 0, 0, 0, 0, 1031, 1042, 1054, 1016, 8, 3, 1, 0, 990, 1063, 1035, 1015, 0, 0, 0, 0, 1055, 998, 1046, 1014, 22, 9, 0, 0, 1023, 1005, 1008, 1003, 0, 0, 0, 0, 969, 1000, 1001, 1018, 0, 0, 0, 0, 1047, 1027, 1014, 1006, 2, 0, 0, 0, 1040, 1083, 1007, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4155, 4103, 4144, 4039, 3988, 4096, 4175] avg=4096.0000 max=4175 max_rank=7 min=3988 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1019, 1018, 1018, 1026, 0, 0, 0, 0, 1026, 1026, 1009, 984, 8, 3, 1, 0, 1069, 983, 1034, 990, 0, 0, 0, 0, 1026, 1009, 973, 1017, 21, 9, 0, 0, 1003, 1043, 1099, 1039, 0, 0, 0, 0, 1054, 1013, 1031, 1020, 0, 0, 0, 0, 1064, 1019, 1028, 1031, 2, 0, 0, 0, 989, 1055, 1018, 991, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4057, 4076, 4055, 4184, 4118, 4144, 4053] avg=4096.0000 max=4184 max_rank=4 min=4053 min_rank=7 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1028, 998, 1023, 1030, 0, 0, 0, 0, 1014, 1039, 1032, 1007, 8, 3, 1, 0, 1054, 1033, 1054, 1050, 0, 0, 0, 0, 1021, 1027, 981, 967, 22, 9, 0, 0, 1010, 1039, 1017, 1035, 0, 0, 0, 0, 1019, 997, 1040, 1002, 0, 0, 0, 0, 995, 1026, 1032, 1053, 2, 0, 0, 0, 1042, 1030, 1037, 991, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4104, 4191, 4027, 4101, 4058, 4108, 4100] avg=4096.0000 max=4191 max_rank=2 min=4027 min_rank=3 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1012, 1024, 998, 994, 0, 0, 0, 0, 1005, 1001, 1021, 1035, 12, 9, 3, 1, 1023, 1005, 1045, 994, 0, 0, 0, 0, 1032, 1063, 1042, 1017, 26, 12, 0, 0, 1082, 1048, 992, 999, 0, 0, 0, 0, 1016, 1078, 1007, 1025, 0, 0, 0, 0, 1013, 1067, 971, 1038, 8, 0, 0, 0, 1005, 1035, 980, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4087, 4067, 4192, 4121, 4126, 4097, 4050] avg=4096.0000 max=4192 max_rank=3 min=4028 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1037, 1012, 1063, 1045, 0, 0, 0, 0, 980, 1042, 999, 1021, 8, 3, 1, 0, 1037, 1014, 1012, 1014, 0, 0, 0, 0, 1019, 996, 1026, 1019, 21, 9, 0, 0, 1036, 980, 1016, 1079, 0, 0, 0, 0, 1008, 1052, 1018, 1019, 0, 0, 0, 0, 1011, 984, 1069, 1030, 2, 0, 0, 0, 993, 1025, 1079, 989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4054, 4077, 4090, 4111, 4097, 4096, 4086] avg=4096.0000 max=4157 max_rank=0 min=4054 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[985, 1012, 1047, 1045, 1016, 1033, 1007, 1010, 1026, 995, 1062, 1078, 1036, 1051, 1050, 1000, 1029, 1006, 1004, 1002, 1018, 1010, 968, 1095, 1036, 988, 999, 1000, 1051, 1038, 1011, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[962, 1013, 1047, 1030, 1031, 1015, 1015, 1001, 989, 926, 1057, 997, 993, 1010, 1056, 1060, 1014, 1032, 1086, 996, 1054, 993, 1054, 1019, 1060, 1033, 1087, 1030, 1033, 1012, 1037, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[999, 1026, 1019, 994, 1087, 1067, 1052, 1028, 1000, 1011, 1011, 1011, 1040, 1028, 973, 1019, 1020, 988, 1011, 1035, 1041, 1035, 1005, 1028, 1010, 1050, 1006, 1032, 1062, 1016, 1015, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1066, 1021, 1051, 1032, 997, 1043, 995, 1002, 1045, 1060, 1007, 1038, 945, 1052, 1025, 982, 1014, 1003, 1048, 998, 1074, 990, 1017, 1073, 1036, 1055, 1038, 1006, 1035, 1010, 1044, 966] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1025, 1011, 985, 1013, 1090, 1067, 1083, 1007, 994, 1025, 1012, 1018, 1029, 1030, 973, 1029, 1026, 973, 1005, 1002, 1011, 1060, 1029, 1012, 1044, 981, 1023, 1058, 1062, 1009, 1023, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1030, 1049, 978, 1018, 1029, 1034, 1028, 1066, 1021, 1010, 1020, 1025, 1011, 1023, 1018, 1004, 1006, 991, 1036, 991, 1058, 1032, 1045, 1077, 1014, 1042, 1013, 1047, 1030, 1050, 1033, 939] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[994, 1034, 1007, 1010, 1016, 1077, 1006, 1093, 989, 1020, 1030, 1061, 1045, 1010, 1010, 1047, 979, 978, 999, 1024, 1007, 998, 1014, 1003, 1027, 1068, 986, 1050, 992, 1025, 1128, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1031, 1074, 1027, 1035, 1019, 972, 993, 1023, 1025, 1017, 1045, 1026, 1022, 998, 1019, 1032, 982, 997, 1037, 1057, 1042, 1043, 1056, 1048, 1008, 996, 987, 1021, 1043, 1042, 1041, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[985, 1012, 1047, 1045, 7, 4, 0, 0, 1016, 1005, 1007, 1010, 0, 0, 0, 0, 1026, 995, 1062, 1078, 14, 0, 0, 0, 1036, 1051, 1050, 1000, 9, 8, 0, 0, 1029, 1006, 1004, 1002, 31, 19, 0, 0, 1018, 1010, 968, 1064, 0, 0, 0, 0, 1036, 988, 999, 993, 0, 0, 0, 0, 1051, 1038, 985, 1060, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4038, 4175, 4154, 4091, 4060, 4016, 4134] avg=4096.0000 max=4175 max_rank=2 min=4016 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[999, 1026, 1019, 994, 7, 4, 0, 0, 1087, 1038, 1052, 1028, 0, 0, 0, 0, 1000, 1011, 1011, 1011, 14, 0, 0, 0, 1040, 1028, 973, 1019, 9, 8, 0, 0, 1020, 988, 1011, 1035, 29, 20, 0, 0, 1041, 1035, 1005, 999, 0, 0, 0, 0, 1010, 1050, 1006, 1025, 0, 0, 0, 0, 1062, 1016, 989, 1049, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4205, 4047, 4077, 4103, 4080, 4091, 4116] avg=4096.0000 max=4205 max_rank=1 min=4047 min_rank=2 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[962, 1013, 1047, 1030, 7, 4, 0, 0, 1031, 987, 1015, 1001, 0, 0, 0, 0, 989, 926, 1057, 997, 14, 0, 0, 0, 993, 1010, 1056, 1060, 9, 8, 0, 0, 1014, 1032, 1086, 996, 29, 19, 0, 0, 1054, 993, 1054, 990, 0, 0, 0, 0, 1060, 1033, 1087, 1023, 0, 0, 0, 0, 1033, 1012, 1011, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4034, 3983, 4136, 4176, 4091, 4203, 4082] avg=4096.0000 max=4203 max_rank=6 min=3983 min_rank=2 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16552 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1066, 1021, 1051, 1032, 7, 4, 0, 0, 997, 1015, 995, 1002, 0, 0, 0, 0, 1045, 1060, 1007, 1038, 14, 0, 0, 0, 945, 1052, 1025, 982, 9, 9, 0, 0, 1014, 1003, 1048, 998, 30, 19, 0, 0, 1074, 990, 1017, 1043, 0, 0, 0, 0, 1036, 1055, 1038, 999, 0, 0, 0, 0, 1035, 1010, 1017, 966, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4009, 4164, 4022, 4112, 4124, 4128, 4028] avg=4096.0000 max=4181 max_rank=0 min=4009 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[994, 1034, 1007, 1010, 8, 4, 0, 0, 1016, 1048, 1006, 1093, 0, 0, 0, 0, 989, 1020, 1030, 1061, 15, 0, 0, 0, 1045, 1010, 1010, 1047, 9, 9, 0, 0, 979, 978, 999, 1024, 28, 20, 0, 0, 1007, 998, 1014, 975, 0, 0, 0, 0, 1027, 1068, 986, 1042, 0, 0, 0, 0, 992, 1025, 1100, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4163, 4115, 4130, 4028, 3994, 4123, 4158] avg=4096.0000 max=4163 max_rank=1 min=3994 min_rank=5 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1030, 1049, 978, 1018, 8, 4, 0, 0, 1029, 1006, 1028, 1066, 0, 0, 0, 0, 1021, 1010, 1020, 1025, 14, 0, 0, 0, 1011, 1023, 1018, 1004, 9, 8, 0, 0, 1006, 991, 1036, 991, 31, 19, 0, 0, 1058, 1032, 1045, 1046, 0, 0, 0, 0, 1014, 1042, 1013, 1039, 0, 0, 0, 0, 1030, 1050, 1007, 939, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4129, 4090, 4073, 4074, 4181, 4108, 4026] avg=4096.0000 max=4181 max_rank=5 min=4026 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 1011, 985, 1013, 12, 7, 0, 0, 1090, 1029, 1083, 1007, 0, 0, 0, 0, 994, 1025, 1012, 1018, 18, 0, 0, 0, 1029, 1030, 973, 1029, 14, 14, 0, 0, 1026, 973, 1005, 1002, 33, 24, 0, 0, 1011, 1060, 1029, 979, 0, 0, 0, 0, 1044, 981, 1023, 1046, 0, 0, 0, 0, 1062, 1009, 984, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4209, 4067, 4089, 4063, 4079, 4094, 4114] avg=4096.0000 max=4209 max_rank=1 min=4053 min_rank=0 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1031, 1074, 1027, 1035, 7, 4, 0, 0, 1019, 946, 993, 1023, 0, 0, 0, 0, 1025, 1017, 1045, 1026, 14, 0, 0, 0, 1022, 998, 1019, 1032, 8, 8, 0, 0, 982, 997, 1037, 1057, 30, 18, 0, 0, 1042, 1043, 1056, 1018, 0, 0, 0, 0, 1008, 996, 987, 1014, 0, 0, 0, 0, 1043, 1042, 1015, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 3981, 4127, 4087, 4121, 4159, 4005, 4110] avg=4096.0000 max=4178 max_rank=0 min=3981 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8192, 256, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1023, 1020, 1018, 1041, 1015, 966, 1037, 1031, 999, 1069, 1008, 1016, 1023, 1000, 980, 1008, 1013, 1085, 1046, 985, 971, 1082, 1005, 1016, 1028, 1064, 998, 1021, 1073, 1038, 1032, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1001, 989, 986, 987, 1008, 992, 1041, 1091, 1049, 1046, 1020, 1008, 1008, 1011, 1006, 1021, 1043, 1047, 1029, 1055, 1067, 1032, 1036, 971, 1096, 1046, 1046, 989, 1021, 1007, 1004, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1099, 1074, 1063, 990, 956, 1016, 1045, 979, 973, 1033, 1016, 1026, 1022, 1094, 1021, 1018, 963, 1042, 1054, 1041, 1027, 1004, 1066, 1064, 1029, 1017, 989, 1032, 1007, 1002, 1052, 954] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1005, 996, 989, 1027, 1014, 1037, 1042, 997, 1049, 1028, 1015, 1072, 966, 1041, 948, 1027, 1004, 1031, 1041, 995, 1057, 1021, 1032, 1038, 1025, 1065, 1047, 1014, 1013, 1049, 1037, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1052, 1029, 1017, 1040, 1038, 995, 1027, 1028, 1015, 968, 1070, 1060, 1027, 1030, 988, 1057, 985, 1007, 1011, 1049, 1013, 1051, 1003, 1030, 1013, 1014, 1032, 1043, 1019, 1024, 997, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1023, 977, 1033, 1012, 1087, 1021, 1073, 1027, 1006, 1022, 1011, 1015, 1002, 999, 994, 1039, 1042, 1009, 963, 1044, 982, 1004, 1005, 1020, 1097, 1043, 1059, 1038, 1060, 1008, 1016, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[979, 1039, 974, 1017, 1072, 1069, 1041, 1044, 1026, 1053, 1045, 1039, 1049, 1007, 1019, 1009, 1057, 1029, 1033, 982, 1001, 998, 1038, 983, 1026, 1023, 1051, 1038, 1014, 953, 1036, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1032, 1000, 1018, 995, 994, 1046, 987, 1020, 1059, 1065, 1003, 1008, 1022, 1003, 1087, 991, 1065, 980, 1061, 1013, 1043, 1059, 1039, 1034, 1028, 1074, 1027, 1051, 976, 1013, 1009, 966] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1023, 1020, 1018, 1041, 18, 10, 0, 0, 1015, 966, 1029, 1031, 0, 0, 0, 0, 999, 1054, 1008, 1016, 0, 0, 0, 0, 1023, 1000, 980, 1008, 31, 0, 0, 0, 1013, 1085, 1042, 985, 0, 0, 0, 0, 971, 1079, 1005, 1016, 0, 0, 0, 0, 1028, 1015, 998, 1021, 0, 0, 0, 0, 1073, 1038, 1032, 1057, 8, 5, 4, 3] rerouted_tokens_per_ep_rank=[4130, 4041, 4077, 4042, 4125, 4071, 4062, 4220] avg=4096.0000 max=4220 max_rank=7 min=4041 min_rank=1 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1001, 989, 986, 987, 18, 9, 0, 0, 1008, 992, 1033, 1091, 0, 0, 0, 0, 1049, 1032, 1020, 1008, 0, 0, 0, 0, 1008, 1011, 1006, 1021, 31, 0, 0, 0, 1043, 1047, 1025, 1055, 0, 0, 0, 0, 1067, 1029, 1036, 971, 0, 0, 0, 0, 1096, 997, 1046, 989, 0, 0, 0, 0, 1021, 1007, 1004, 1015, 8, 5, 4, 3] rerouted_tokens_per_ep_rank=[3990, 4124, 4109, 4077, 4170, 4103, 4128, 4067] avg=4096.0000 max=4170 max_rank=4 min=3990 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1052, 1029, 1017, 1040, 17, 9, 0, 0, 1038, 995, 1019, 1028, 0, 0, 0, 0, 1015, 954, 1070, 1060, 0, 0, 0, 0, 1027, 1030, 988, 1057, 30, 0, 0, 0, 985, 1007, 1007, 1049, 0, 0, 0, 0, 1013, 1048, 1003, 1030, 0, 0, 0, 0, 1013, 967, 1032, 1043, 0, 0, 0, 0, 1019, 1024, 997, 1036, 8, 5, 4, 3] rerouted_tokens_per_ep_rank=[4164, 4080, 4099, 4132, 4048, 4094, 4055, 4096] avg=4096.0000 max=4164 max_rank=0 min=4048 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1023, 977, 1033, 1012, 17, 9, 0, 0, 1087, 1021, 1065, 1027, 0, 0, 0, 0, 1006, 1008, 1011, 1015, 0, 0, 0, 0, 1002, 999, 994, 1039, 31, 0, 0, 0, 1042, 1009, 959, 1044, 0, 0, 0, 0, 982, 1002, 1005, 1020, 0, 0, 0, 0, 1097, 995, 1059, 1038, 0, 0, 0, 0, 1060, 1008, 1016, 1037, 8, 5, 4, 2] rerouted_tokens_per_ep_rank=[4071, 4200, 4040, 4065, 4054, 4009, 4189, 4140] avg=4096.0000 max=4200 max_rank=1 min=4009 min_rank=5 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1005, 996, 989, 1027, 22, 13, 0, 0, 1014, 1037, 1030, 997, 0, 0, 0, 0, 1049, 1005, 1015, 1072, 0, 0, 0, 0, 966, 1041, 948, 1027, 36, 0, 0, 0, 1004, 1031, 1033, 995, 0, 0, 0, 0, 1057, 1015, 1032, 1038, 0, 0, 0, 0, 1025, 1007, 1047, 1014, 0, 0, 0, 0, 1013, 1049, 1037, 1046, 12, 10, 8, 6] rerouted_tokens_per_ep_rank=[4052, 4078, 4141, 4018, 4063, 4142, 4093, 4181] avg=4096.0000 max=4181 max_rank=7 min=4018 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1099, 1074, 1063, 990, 17, 9, 0, 0, 956, 1016, 1037, 979, 0, 0, 0, 0, 973, 1019, 1016, 1026, 0, 0, 0, 0, 1022, 1094, 1021, 1018, 30, 0, 0, 0, 963, 1042, 1050, 1041, 0, 0, 0, 0, 1027, 1002, 1066, 1064, 0, 0, 0, 0, 1029, 970, 989, 1032, 0, 0, 0, 0, 1007, 1002, 1052, 954, 8, 5, 4, 2] rerouted_tokens_per_ep_rank=[4252, 3988, 4034, 4185, 4096, 4159, 4020, 4034] avg=4096.0000 max=4252 max_rank=0 min=3988 min_rank=1 imbalance_ratio=1.038086 max_load_reduction=28516 max_load_reduction_pct=0.870239 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1032, 1000, 1018, 995, 18, 10, 0, 0, 994, 1046, 979, 1020, 0, 0, 0, 0, 1059, 1050, 1003, 1008, 0, 0, 0, 0, 1022, 1003, 1087, 991, 32, 0, 0, 0, 1065, 980, 1057, 1013, 0, 0, 0, 0, 1043, 1056, 1039, 1034, 0, 0, 0, 0, 1028, 1024, 1027, 1051, 0, 0, 0, 0, 976, 1013, 1009, 966, 8, 5, 4, 3] rerouted_tokens_per_ep_rank=[4073, 4039, 4120, 4135, 4115, 4172, 4130, 3984] avg=4096.0000 max=4172 max_rank=5 min=3984 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[979, 1039, 974, 1017, 17, 10, 0, 0, 1072, 1069, 1033, 1044, 0, 0, 0, 0, 1026, 1038, 1045, 1039, 0, 0, 0, 0, 1049, 1007, 1019, 1009, 30, 0, 0, 0, 1057, 1029, 1029, 982, 0, 0, 0, 0, 1001, 996, 1038, 983, 0, 0, 0, 0, 1026, 976, 1051, 1038, 0, 0, 0, 0, 1014, 953, 1036, 1024, 8, 5, 4, 2] rerouted_tokens_per_ep_rank=[4036, 4218, 4148, 4114, 4097, 4018, 4091, 4046] avg=4096.0000 max=4218 max_rank=1 min=4018 min_rank=5 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16252 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8064, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1033, 1020, 1072, 1000, 1057, 1058, 1009, 974, 1017, 1045, 1035, 997, 1039, 1033, 988, 1030, 1039, 1005, 1060, 1060, 982, 1019, 975, 961, 976, 1014, 1010, 1064, 1057, 1049, 1031, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1019, 1029, 1024, 1042, 1054, 1003, 1025, 996, 992, 1010, 1004, 1030, 1010, 1049, 1011, 1018, 1036, 1004, 1039, 1046, 1010, 1051, 1054, 942, 1000, 1069, 1063, 1042, 1006, 1037, 1005, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[986, 1041, 1031, 1040, 1008, 1026, 1069, 1034, 1061, 1080, 1001, 1035, 1005, 1024, 974, 1003, 992, 1020, 1007, 995, 1032, 1020, 1019, 1012, 1051, 1054, 1018, 978, 1049, 1021, 1027, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[984, 1037, 1026, 1007, 1025, 1062, 1058, 966, 994, 986, 1025, 979, 1014, 1081, 1032, 1038, 1034, 1000, 1081, 963, 1069, 1080, 1000, 1015, 1017, 1013, 1036, 1035, 1002, 1073, 1010, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1034, 999, 1020, 1018, 1068, 1049, 977, 1029, 1017, 991, 996, 1012, 1043, 1080, 1035, 1007, 1045, 1058, 1046, 1028, 1089, 1056, 1032, 992, 978, 1011, 1005, 1006, 986, 1010, 1024, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1026, 1019, 1049, 1022, 1059, 1010, 1011, 1029, 1034, 1012, 1022, 1042, 1006, 1085, 1013, 1011, 1015, 1010, 1024, 1007, 1046, 989, 989, 1013, 1011, 1065, 1023, 1033, 1004, 1021, 1016, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[988, 1019, 1038, 1009, 1018, 983, 1019, 1021, 997, 1044, 1069, 1041, 1014, 1022, 1031, 1069, 1028, 1044, 1043, 1014, 1010, 1018, 1000, 1070, 1049, 1002, 1035, 1041, 948, 1048, 1029, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1033, 1022, 1042, 1041, 1014, 994, 1021, 1035, 1022, 1036, 1032, 1003, 1011, 1003, 1051, 1016, 949, 1009, 1019, 1035, 1076, 1051, 1040, 1005, 1013, 1000, 1043, 1048, 981, 1043, 1064, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[986, 1041, 1031, 1040, 0, 0, 0, 0, 1008, 1026, 1069, 1034, 0, 0, 0, 0, 1061, 1080, 1001, 1035, 9, 3, 0, 0, 1005, 1015, 974, 1003, 0, 0, 0, 0, 992, 1020, 1007, 995, 1, 0, 0, 0, 1032, 1020, 1019, 1012, 4, 2, 0, 0, 1051, 1054, 1018, 975, 0, 0, 0, 0, 1049, 1014, 1027, 1055, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4137, 4189, 3997, 4015, 4089, 4098, 4145] avg=4096.0000 max=4189 max_rank=2 min=3997 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[984, 1037, 1026, 1007, 0, 0, 0, 0, 1025, 1062, 1058, 966, 0, 0, 0, 0, 994, 986, 1025, 979, 10, 3, 0, 0, 1014, 1071, 1032, 1038, 0, 0, 0, 0, 1034, 1000, 1081, 963, 1, 0, 0, 0, 1069, 1080, 1000, 1015, 4, 2, 0, 0, 1017, 1013, 1036, 1032, 0, 0, 0, 0, 1002, 1066, 1010, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4111, 3997, 4155, 4079, 4170, 4098, 4104] avg=4096.0000 max=4170 max_rank=5 min=3997 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1033, 1020, 1072, 1000, 0, 0, 0, 0, 1057, 1058, 1009, 974, 0, 0, 0, 0, 1017, 1045, 1035, 997, 9, 3, 0, 0, 1039, 1024, 988, 1030, 0, 0, 0, 0, 1039, 1005, 1060, 1060, 1, 0, 0, 0, 982, 1019, 975, 961, 4, 2, 0, 0, 976, 1014, 1010, 1061, 0, 0, 0, 0, 1057, 1042, 1031, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4098, 4106, 4081, 4165, 3943, 4061, 4189] avg=4096.0000 max=4189 max_rank=7 min=3943 min_rank=5 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1034, 999, 1020, 1018, 0, 0, 0, 0, 1068, 1049, 977, 1029, 0, 0, 0, 0, 1017, 991, 996, 1012, 10, 3, 0, 0, 1043, 1070, 1035, 1007, 0, 0, 0, 0, 1045, 1058, 1046, 1028, 1, 0, 0, 0, 1089, 1056, 1032, 992, 4, 2, 0, 0, 978, 1011, 1005, 1003, 0, 0, 0, 0, 986, 1003, 1024, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4123, 4029, 4155, 4178, 4175, 3997, 4040] avg=4096.0000 max=4178 max_rank=4 min=3997 min_rank=6 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 1019, 1049, 1022, 0, 0, 0, 0, 1059, 1010, 1011, 1029, 0, 0, 0, 0, 1034, 1012, 1022, 1042, 10, 3, 0, 0, 1006, 1075, 1013, 1011, 0, 0, 0, 0, 1015, 1010, 1024, 1007, 1, 0, 0, 0, 1046, 989, 989, 1013, 4, 2, 0, 0, 1011, 1065, 1023, 1030, 0, 0, 0, 0, 1004, 1014, 1016, 1052, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4109, 4123, 4105, 4057, 4043, 4129, 4086] avg=4096.0000 max=4129 max_rank=6 min=4043 min_rank=5 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1019, 1029, 1022, 1042, 0, 0, 0, 0, 1054, 1003, 1025, 996, 5, 2, 0, 0, 992, 1010, 1004, 1030, 12, 8, 0, 0, 1010, 1037, 1011, 1018, 0, 0, 0, 0, 1036, 1004, 1039, 1046, 6, 0, 0, 0, 1010, 1051, 1054, 942, 6, 3, 0, 0, 1000, 1069, 1063, 1028, 0, 0, 0, 0, 1006, 1023, 1005, 1048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4085, 4056, 4076, 4131, 4066, 4160, 4082] avg=4096.0000 max=4160 max_rank=6 min=4056 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1033, 1022, 1042, 1041, 0, 0, 0, 0, 1014, 994, 1021, 1035, 0, 0, 0, 0, 1022, 1036, 1032, 1003, 9, 3, 0, 0, 1011, 994, 1051, 1016, 0, 0, 0, 0, 949, 1009, 1019, 1035, 1, 0, 0, 0, 1076, 1051, 1040, 1005, 4, 2, 0, 0, 1013, 1000, 1043, 1045, 0, 0, 0, 0, 981, 1036, 1064, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4064, 4105, 4072, 4013, 4178, 4101, 4097] avg=4096.0000 max=4178 max_rank=5 min=4013 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[988, 1019, 1038, 1009, 0, 0, 0, 0, 1018, 983, 1019, 1021, 0, 0, 0, 0, 997, 1044, 1069, 1041, 9, 3, 0, 0, 1014, 1013, 1031, 1069, 0, 0, 0, 0, 1028, 1044, 1043, 1014, 1, 0, 0, 0, 1010, 1018, 1000, 1070, 4, 2, 0, 0, 1049, 1002, 1035, 1038, 0, 0, 0, 0, 948, 1041, 1029, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4041, 4163, 4127, 4130, 4104, 4124, 4025] avg=4096.0000 max=4163 max_rank=2 min=4025 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1039, 1009, 1017, 1026, 1015, 1029, 1016, 1088, 1025, 1050, 1017, 1028, 1053, 1015, 1013, 1084, 1045, 992, 1066, 1018, 991, 989, 984, 1059, 1070, 943, 974, 1037, 988, 1044, 1033, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1013, 985, 1015, 963, 1038, 1021, 1028, 1003, 997, 1076, 1029, 1024, 989, 1037, 1042, 1040, 1029, 1083, 998, 1091, 1003, 1000, 1015, 1069, 1051, 1027, 1022, 1014, 1052, 1031, 1004, 979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1034, 1067, 1024, 1025, 1029, 1040, 1055, 1041, 1028, 1051, 1013, 1038, 1023, 1065, 1079, 981, 979, 1007, 995, 1011, 965, 1014, 1014, 1011, 1012, 1005, 1028, 1007, 1023, 1053, 1005, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1012, 1031, 1002, 1059, 1025, 1019, 1024, 1016, 1016, 1016, 999, 1043, 1011, 1019, 1036, 1040, 1043, 1004, 1024, 1044, 980, 1030, 1049, 1042, 1061, 1018, 1080, 976, 1017, 1023, 1001, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[973, 1054, 1049, 997, 985, 995, 1000, 991, 1037, 1015, 1056, 1037, 1021, 1054, 1013, 1005, 1026, 1059, 1091, 1023, 1014, 996, 1040, 1012, 1032, 1047, 995, 1032, 1033, 1002, 1006, 1078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1024, 1038, 1000, 1034, 1013, 1022, 1013, 1027, 1001, 1013, 1043, 1031, 1066, 1032, 1029, 998, 1008, 1016, 1017, 1010, 1019, 1044, 1064, 1030, 1013, 1034, 1054, 1007, 1084, 1006, 1014, 964] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1067, 1010, 959, 978, 1045, 1025, 1006, 1062, 1026, 1027, 1042, 1041, 1030, 1003, 1024, 1000, 992, 1020, 1016, 1023, 1092, 1075, 1088, 996, 979, 1022, 991, 998, 1018, 1049, 1018, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1022, 1065, 996, 1018, 1001, 1018, 1028, 985, 1024, 1049, 1008, 1008, 989, 1059, 1033, 974, 1028, 1013, 1104, 1004, 1025, 1009, 1021, 1056, 999, 996, 1003, 1053, 1010, 1059, 1069, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1013, 985, 1015, 963, 11, 7, 1, 0, 1038, 1021, 1028, 1003, 4, 3, 0, 0, 997, 1058, 1029, 1024, 0, 0, 0, 0, 989, 1026, 1042, 1040, 0, 0, 0, 0, 1029, 1083, 986, 1091, 0, 0, 0, 0, 1003, 1000, 1015, 1066, 0, 0, 0, 0, 1051, 1027, 1022, 1014, 18, 5, 0, 0, 1052, 1026, 1004, 979, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3995, 4097, 4108, 4097, 4189, 4084, 4137, 4061] avg=4096.0000 max=4189 max_rank=4 min=3995 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1039, 1009, 1017, 1026, 10, 8, 1, 0, 1015, 1029, 1016, 1088, 4, 3, 0, 0, 1025, 1033, 1017, 1028, 0, 0, 0, 0, 1053, 1005, 1013, 1084, 0, 0, 0, 0, 1045, 992, 1052, 1018, 0, 0, 0, 0, 991, 989, 984, 1056, 0, 0, 0, 0, 1070, 943, 974, 1037, 17, 6, 0, 0, 988, 1039, 1033, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4155, 4103, 4155, 4107, 4020, 4047, 4071] avg=4096.0000 max=4155 max_rank=1 min=4020 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16245 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1067, 1010, 959, 978, 10, 7, 1, 0, 1045, 1025, 1006, 1062, 4, 3, 0, 0, 1026, 1010, 1042, 1041, 0, 0, 0, 0, 1030, 993, 1024, 1000, 0, 0, 0, 0, 992, 1020, 1004, 1023, 0, 0, 0, 0, 1092, 1075, 1088, 993, 0, 0, 0, 0, 979, 1022, 991, 998, 17, 5, 0, 0, 1018, 1044, 1018, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4145, 4119, 4047, 4039, 4248, 4012, 4126] avg=4096.0000 max=4248 max_rank=5 min=4012 min_rank=6 imbalance_ratio=1.037109 max_load_reduction=28520 max_load_reduction_pct=0.870361 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1012, 1031, 1002, 1059, 10, 7, 1, 0, 1025, 1019, 1024, 1016, 4, 3, 0, 0, 1016, 999, 999, 1043, 0, 0, 0, 0, 1011, 1009, 1036, 1040, 0, 0, 0, 0, 1043, 1004, 1012, 1044, 0, 0, 0, 0, 980, 1030, 1049, 1039, 0, 0, 0, 0, 1061, 1018, 1080, 976, 17, 5, 0, 0, 1017, 1018, 1001, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4091, 4057, 4096, 4103, 4098, 4157, 4044] avg=4096.0000 max=4157 max_rank=6 min=4044 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1034, 1067, 1024, 1025, 11, 7, 1, 0, 1029, 1040, 1055, 1041, 4, 3, 0, 0, 1028, 1034, 1013, 1038, 0, 0, 0, 0, 1023, 1054, 1079, 981, 0, 0, 0, 0, 979, 1007, 983, 1011, 0, 0, 0, 0, 965, 1014, 1014, 1008, 0, 0, 0, 0, 1012, 1005, 1028, 1007, 17, 5, 0, 0, 1023, 1048, 1005, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4172, 4113, 4137, 3980, 4001, 4074, 4122] avg=4096.0000 max=4172 max_rank=1 min=3980 min_rank=4 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16177 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[973, 1054, 1049, 997, 11, 8, 1, 0, 985, 995, 1000, 991, 4, 3, 0, 0, 1037, 998, 1056, 1037, 0, 0, 0, 0, 1021, 1043, 1013, 1005, 0, 0, 0, 0, 1026, 1059, 1077, 1023, 0, 0, 0, 0, 1014, 996, 1040, 1009, 0, 0, 0, 0, 1032, 1047, 995, 1032, 17, 6, 0, 0, 1033, 997, 1006, 1078, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 3978, 4128, 4082, 4185, 4059, 4129, 4114] avg=4096.0000 max=4185 max_rank=4 min=3978 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1024, 1038, 1000, 1034, 11, 7, 1, 0, 1013, 1022, 1013, 1027, 4, 3, 0, 0, 1001, 996, 1043, 1031, 0, 0, 0, 0, 1066, 1021, 1029, 998, 0, 0, 0, 0, 1008, 1016, 1005, 1010, 0, 0, 0, 0, 1019, 1044, 1064, 1027, 0, 0, 0, 0, 1013, 1034, 1054, 1007, 17, 5, 0, 0, 1084, 1001, 1014, 964, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4082, 4071, 4114, 4039, 4154, 4130, 4063] avg=4096.0000 max=4154 max_rank=5 min=4039 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1022, 1065, 996, 1018, 15, 12, 4, 0, 1001, 1018, 1028, 985, 9, 7, 0, 0, 1024, 1029, 1008, 1008, 0, 0, 0, 0, 989, 1044, 1033, 974, 0, 0, 0, 0, 1028, 1013, 1081, 1004, 0, 0, 0, 0, 1025, 1009, 1021, 1049, 0, 0, 0, 0, 999, 996, 1003, 1053, 20, 11, 0, 0, 1010, 1046, 1069, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4048, 4069, 4040, 4126, 4104, 4082, 4167] avg=4096.0000 max=4167 max_rank=7 min=4040 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1018, 978, 1038, 1041, 1030, 984, 977, 1034, 1017, 969, 1045, 1021, 1059, 1013, 1065, 1030, 999, 1004, 1021, 1087, 974, 1059, 1013, 1002, 997, 1002, 1076, 1028, 1038, 1083, 1026, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1046, 1020, 978, 975, 1026, 1069, 1030, 1055, 1043, 1088, 984, 989, 1020, 1015, 1001, 1014, 1025, 1032, 1057, 1017, 1019, 1044, 1039, 980, 982, 992, 1020, 1033, 1057, 1029, 1071, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1045, 957, 1031, 1037, 1047, 1040, 993, 1012, 982, 1052, 992, 984, 1022, 1084, 1021, 1029, 1079, 991, 1021, 1064, 1033, 1048, 1016, 992, 1050, 1051, 964, 1044, 1039, 1029, 1021, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[973, 1012, 1048, 1070, 1022, 1024, 1051, 1022, 1037, 1070, 1013, 1022, 984, 1047, 1008, 993, 1024, 994, 1033, 1052, 1002, 1029, 1036, 1025, 1066, 1006, 992, 1027, 1053, 990, 1014, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1019, 1039, 1005, 978, 1045, 1007, 1068, 1014, 1056, 1003, 1050, 1060, 1048, 1013, 1080, 987, 1050, 1019, 1024, 995, 1074, 1020, 1079, 981, 1032, 966, 1002, 991, 1026, 1065, 987, 985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1015, 993, 1005, 1020, 1025, 1024, 1015, 1064, 1038, 1018, 1043, 1001, 1004, 1066, 991, 1070, 1033, 1040, 1039, 1045, 1024, 1021, 1045, 1022, 988, 1047, 1016, 986, 1001, 1004, 1014, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1019, 1042, 1040, 1020, 1015, 1050, 1026, 1022, 1020, 1070, 998, 962, 1021, 1047, 998, 1013, 968, 1035, 1060, 1029, 969, 1016, 1050, 1002, 1045, 1018, 1066, 993, 1018, 1058, 1018, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[980, 1054, 1015, 1046, 1023, 1007, 1004, 1038, 1018, 1011, 1045, 998, 1000, 1042, 1101, 1022, 1007, 1061, 1056, 1008, 1008, 992, 998, 1010, 1030, 1030, 1090, 1059, 1030, 987, 979, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1046, 1020, 978, 975, 25, 1, 0, 0, 1026, 1069, 1030, 1043, 0, 0, 0, 0, 1043, 1088, 984, 989, 8, 0, 0, 0, 1020, 999, 1001, 1014, 0, 0, 0, 0, 1025, 1032, 1032, 1017, 0, 0, 0, 0, 1019, 1044, 1039, 980, 15, 2, 0, 0, 982, 992, 1020, 1033, 10, 0, 0, 0, 1049, 1029, 1071, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4168, 4112, 4034, 4106, 4099, 4037, 4167] avg=4096.0000 max=4168 max_rank=1 min=4034 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16409 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[973, 1012, 1048, 1070, 24, 1, 0, 0, 1022, 1024, 1051, 1012, 0, 0, 0, 0, 1037, 1070, 1013, 1022, 8, 0, 0, 0, 984, 1030, 1008, 993, 0, 0, 0, 0, 1024, 994, 1009, 1052, 0, 0, 0, 0, 1002, 1029, 1036, 1025, 16, 1, 0, 0, 1066, 1006, 992, 1027, 9, 0, 0, 0, 1045, 990, 1014, 1029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4109, 4150, 4015, 4079, 4109, 4100, 4078] avg=4096.0000 max=4150 max_rank=2 min=4015 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16366 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1045, 957, 1031, 1037, 24, 1, 0, 0, 1047, 1040, 993, 1002, 0, 0, 0, 0, 982, 1052, 992, 984, 8, 0, 0, 0, 1022, 1067, 1021, 1029, 0, 0, 0, 0, 1079, 991, 997, 1064, 0, 0, 0, 0, 1033, 1048, 1016, 992, 16, 1, 0, 0, 1050, 1051, 964, 1044, 9, 0, 0, 0, 1031, 1029, 1021, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4082, 4018, 4139, 4131, 4106, 4118, 4079] avg=4096.0000 max=4139 max_rank=3 min=4018 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16434 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1018, 978, 1038, 1041, 24, 1, 0, 0, 1030, 984, 977, 1023, 0, 0, 0, 0, 1017, 969, 1045, 1021, 8, 0, 0, 0, 1059, 997, 1065, 1030, 0, 0, 0, 0, 999, 1004, 997, 1087, 0, 0, 0, 0, 974, 1059, 1013, 1002, 15, 2, 0, 0, 997, 1002, 1076, 1028, 9, 0, 0, 0, 1030, 1083, 1026, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4014, 4060, 4151, 4087, 4065, 4112, 4179] avg=4096.0000 max=4179 max_rank=7 min=4014 min_rank=1 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16443 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1019, 1039, 1005, 978, 24, 1, 0, 0, 1045, 1007, 1068, 1004, 0, 0, 0, 0, 1056, 1003, 1050, 1060, 8, 0, 0, 0, 1048, 997, 1080, 987, 0, 0, 0, 0, 1050, 1019, 1000, 995, 0, 0, 0, 0, 1074, 1020, 1079, 981, 15, 1, 0, 0, 1032, 966, 1002, 991, 9, 0, 0, 0, 1018, 1065, 987, 985, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4124, 4177, 4112, 4064, 4170, 4000, 4055] avg=4096.0000 max=4177 max_rank=2 min=4000 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16289 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1019, 1042, 1040, 1020, 25, 1, 0, 0, 1015, 1050, 1026, 1012, 0, 0, 0, 0, 1020, 1070, 998, 962, 8, 0, 0, 0, 1021, 1030, 998, 1013, 0, 0, 0, 0, 968, 1035, 1035, 1029, 0, 0, 0, 0, 969, 1016, 1050, 1002, 16, 1, 0, 0, 1045, 1018, 1066, 993, 9, 0, 0, 0, 1010, 1058, 1018, 1060, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4103, 4058, 4062, 4067, 4054, 4131, 4146] avg=4096.0000 max=4147 max_rank=0 min=4054 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16398 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1015, 993, 1005, 1020, 30, 3, 0, 0, 1025, 1024, 1015, 1043, 0, 0, 0, 0, 1038, 1018, 1043, 1001, 13, 0, 0, 0, 1004, 1042, 991, 1070, 0, 0, 0, 0, 1033, 1040, 1009, 1045, 0, 0, 0, 0, 1024, 1021, 1045, 1022, 21, 6, 0, 0, 988, 1047, 1016, 986, 15, 0, 0, 0, 988, 1004, 1014, 1051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4107, 4113, 4107, 4127, 4139, 4052, 4057] avg=4096.0000 max=4139 max_rank=5 min=4052 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16375 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[980, 1054, 1015, 1046, 25, 1, 0, 0, 1023, 1007, 1004, 1027, 0, 0, 0, 0, 1018, 1011, 1045, 998, 8, 0, 0, 0, 1000, 1025, 1101, 1022, 0, 0, 0, 0, 1007, 1061, 1031, 1008, 0, 0, 0, 0, 1008, 992, 998, 1010, 16, 2, 0, 0, 1030, 1030, 1090, 1059, 9, 0, 0, 0, 1022, 987, 979, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4061, 4080, 4148, 4107, 4026, 4218, 4007] avg=4096.0000 max=4218 max_rank=6 min=4007 min_rank=7 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16358 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1051, 1035, 1027, 1012, 1000, 1037, 973, 1069, 1075, 1045, 1009, 1045, 990, 996, 1047, 1032, 1006, 1022, 1005, 1019, 1087, 986, 1030, 1017, 986, 1059, 1007, 1042, 981, 1035, 1022, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1035, 986, 1012, 1065, 978, 1021, 1017, 1024, 998, 979, 1011, 1015, 1029, 1038, 1056, 1020, 990, 1030, 1008, 1011, 1058, 1004, 1042, 1081, 985, 1024, 1033, 1026, 1034, 1031, 1052, 1075] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1023, 1044, 1015, 984, 1039, 1060, 984, 1021, 1013, 1022, 1026, 1025, 1039, 1053, 974, 1006, 1013, 1031, 1019, 1033, 1063, 1054, 1027, 1096, 999, 979, 1009, 1034, 1023, 1024, 977, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1025, 1007, 1046, 1029, 1027, 1025, 1056, 1009, 1028, 1048, 1060, 968, 992, 1001, 1033, 1065, 1009, 994, 994, 1044, 1010, 1019, 1026, 1036, 1021, 1069, 1048, 1031, 980, 1013, 1021, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1067, 997, 1061, 1024, 1040, 1004, 1061, 991, 985, 1039, 1004, 1031, 1040, 1037, 1022, 1048, 1009, 1044, 992, 1039, 1021, 1024, 1008, 1023, 1019, 1008, 1026, 1041, 1021, 991, 1043, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1047, 1043, 1018, 1002, 991, 996, 1047, 1036, 1024, 1044, 1012, 1032, 996, 1005, 976, 1011, 1021, 1070, 986, 1003, 1043, 1002, 1017, 1086, 1030, 1022, 1053, 1024, 985, 1089, 1031, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1019, 1022, 1038, 978, 1045, 1012, 1034, 1019, 989, 995, 1021, 1062, 1006, 1034, 1024, 1014, 1024, 1013, 1015, 1057, 1034, 1043, 1020, 1047, 1067, 987, 1006, 979, 1060, 1006, 1012, 1086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1054, 1008, 1037, 1010, 965, 995, 1006, 1080, 996, 1002, 1037, 1045, 997, 1072, 1010, 1054, 1053, 1059, 983, 998, 1005, 1016, 1046, 1014, 1032, 1003, 1029, 1040, 1061, 961, 1046, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1045, 1035, 1027, 1012, 0, 0, 0, 0, 1000, 1037, 973, 1069, 12, 0, 0, 0, 1075, 1045, 1009, 1045, 5, 4, 0, 0, 990, 996, 1047, 1032, 5, 0, 0, 0, 1006, 1022, 1005, 1019, 21, 0, 0, 0, 1087, 986, 1030, 980, 0, 0, 0, 0, 986, 1059, 1007, 1042, 6, 0, 0, 0, 981, 1035, 1022, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4091, 4183, 4070, 4073, 4083, 4100, 4049] avg=4096.0000 max=4183 max_rank=2 min=4049 min_rank=7 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1017, 1044, 1015, 984, 0, 0, 0, 0, 1039, 1060, 984, 1021, 13, 0, 0, 0, 1013, 1022, 1026, 1025, 5, 4, 0, 0, 1039, 1053, 974, 1006, 6, 0, 0, 0, 1013, 1031, 1019, 1033, 22, 0, 0, 0, 1063, 1054, 1027, 1057, 0, 0, 0, 0, 999, 979, 1009, 1034, 6, 0, 0, 0, 1023, 1024, 977, 1048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4117, 4095, 4078, 4118, 4201, 4027, 4072] avg=4096.0000 max=4201 max_rank=5 min=4027 min_rank=6 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1019, 1007, 1046, 1029, 0, 0, 0, 0, 1027, 1025, 1056, 1009, 13, 0, 0, 0, 1028, 1048, 1060, 968, 5, 4, 0, 0, 992, 1001, 1033, 1065, 5, 0, 0, 0, 1009, 994, 994, 1044, 21, 0, 0, 0, 1010, 1019, 1026, 998, 0, 0, 0, 0, 1021, 1069, 1048, 1031, 6, 0, 0, 0, 980, 1013, 1021, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4130, 4113, 4096, 4062, 4053, 4175, 4038] avg=4096.0000 max=4175 max_rank=6 min=4038 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1029, 986, 1012, 1065, 0, 0, 0, 0, 978, 1021, 1017, 1024, 13, 0, 0, 0, 998, 979, 1011, 1015, 5, 4, 0, 0, 1029, 1038, 1056, 1020, 6, 0, 0, 0, 990, 1030, 1008, 1011, 22, 0, 0, 0, 1058, 1004, 1042, 1042, 0, 0, 0, 0, 985, 1024, 1033, 1026, 6, 0, 0, 0, 1034, 1031, 1052, 1064, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4053, 4012, 4149, 4061, 4146, 4074, 4181] avg=4096.0000 max=4181 max_rank=7 min=4012 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1041, 1043, 1018, 1002, 0, 0, 0, 0, 991, 996, 1047, 1036, 13, 0, 0, 0, 1024, 1044, 1012, 1032, 5, 4, 0, 0, 996, 1005, 976, 1011, 5, 0, 0, 0, 1021, 1070, 986, 1003, 22, 0, 0, 0, 1043, 1002, 1017, 1047, 0, 0, 0, 0, 1030, 1022, 1053, 1024, 6, 0, 0, 0, 985, 1089, 1031, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4083, 4121, 3993, 4102, 4109, 4135, 4121] avg=4096.0000 max=4135 max_rank=6 min=3993 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1013, 1022, 1038, 978, 0, 0, 0, 0, 1045, 1012, 1034, 1019, 13, 0, 0, 0, 989, 995, 1021, 1062, 5, 4, 0, 0, 1006, 1034, 1024, 1014, 6, 0, 0, 0, 1024, 1013, 1015, 1057, 21, 0, 0, 0, 1034, 1043, 1020, 1009, 0, 0, 0, 0, 1067, 987, 1006, 979, 6, 0, 0, 0, 1060, 1006, 1012, 1075, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4123, 4076, 4084, 4130, 4106, 4045, 4153] avg=4096.0000 max=4153 max_rank=7 min=4045 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1056, 997, 1061, 1024, 0, 0, 0, 0, 1040, 1004, 1061, 991, 17, 0, 0, 0, 985, 1039, 1004, 1031, 11, 9, 0, 0, 1040, 1037, 1022, 1048, 9, 3, 0, 0, 1009, 1044, 992, 1039, 24, 0, 0, 0, 1021, 1024, 1008, 973, 0, 0, 0, 0, 1019, 1008, 1026, 1041, 8, 0, 0, 0, 1021, 991, 1043, 988, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4113, 4079, 4159, 4108, 4026, 4102, 4043] avg=4096.0000 max=4159 max_rank=3 min=4026 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1048, 1008, 1037, 1010, 0, 0, 0, 0, 965, 995, 1006, 1080, 12, 0, 0, 0, 996, 1002, 1037, 1045, 5, 4, 0, 0, 997, 1072, 1010, 1054, 6, 0, 0, 0, 1053, 1059, 983, 998, 21, 0, 0, 0, 1005, 1016, 1046, 977, 0, 0, 0, 0, 1032, 1003, 1029, 1040, 6, 0, 0, 0, 1061, 961, 1046, 1043, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4058, 4089, 4139, 4114, 4044, 4110, 4111] avg=4096.0000 max=4139 max_rank=3 min=4044 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1034, 982, 1036, 1084, 1063, 1006, 1025, 1006, 1022, 1032, 1025, 1028, 1034, 1050, 991, 1004, 1051, 1038, 1044, 1037, 1064, 997, 957, 1018, 1019, 1037, 989, 1065, 1028, 1027, 1002, 973] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1036, 1037, 999, 1051, 1021, 972, 1040, 1048, 1055, 1005, 998, 1062, 1034, 1066, 1034, 1003, 995, 1008, 1050, 1010, 1018, 1036, 1012, 990, 1009, 1010, 1040, 1084, 979, 1035, 1024, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1040, 1052, 1017, 1058, 964, 1017, 1042, 1052, 1029, 1016, 1043, 983, 1029, 1047, 1005, 1039, 990, 1031, 1007, 1015, 1003, 1053, 1007, 1022, 1067, 990, 1009, 1042, 994, 1071, 1055, 979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1029, 1007, 1032, 1047, 977, 956, 1011, 1014, 1006, 1007, 1030, 1057, 1020, 1037, 1066, 1044, 1069, 1043, 1064, 1018, 988, 995, 1033, 1010, 1011, 1007, 1000, 1071, 1019, 1004, 1039, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1058, 1063, 979, 1070, 990, 980, 1045, 1015, 1023, 1023, 1053, 1013, 1030, 1006, 1031, 1038, 1023, 1008, 1029, 1052, 1000, 989, 1037, 1087, 1034, 1018, 1026, 998, 1005, 1012, 984, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1003, 996, 1030, 1058, 993, 990, 1048, 1056, 1006, 1027, 1038, 1020, 1054, 963, 1055, 1037, 1021, 1070, 1076, 1028, 1032, 1026, 1005, 1051, 1034, 1026, 975, 1013, 1046, 1015, 1013, 963] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1027, 1055, 1081, 1039, 1028, 1049, 1028, 1045, 1034, 991, 1035, 1019, 1056, 1002, 1005, 957, 1041, 1025, 1004, 1031, 1013, 1024, 971, 1048, 945, 1050, 1016, 1033, 1022, 1024, 1028, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1059, 1061, 1048, 989, 1055, 992, 984, 1038, 1028, 979, 995, 1023, 1043, 1025, 985, 1025, 1035, 1006, 1012, 1009, 1067, 1040, 1028, 1013, 1080, 1003, 1041, 1073, 1004, 1004, 1004, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1034, 982, 1036, 1035, 0, 0, 0, 0, 1063, 1006, 1025, 1006, 19, 8, 0, 0, 1022, 1032, 1025, 1028, 5, 1, 0, 0, 1030, 1050, 991, 1004, 0, 0, 0, 0, 1051, 1038, 1024, 1037, 0, 0, 0, 0, 1064, 997, 957, 1018, 12, 3, 0, 0, 1019, 1037, 989, 1060, 0, 0, 0, 0, 1028, 1027, 1002, 973, 30, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4127, 4113, 4075, 4150, 4051, 4105, 4060] avg=4096.0000 max=4150 max_rank=4 min=4051 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1036, 1037, 999, 1003, 0, 0, 0, 0, 1021, 972, 1040, 1048, 18, 8, 0, 0, 1055, 1005, 998, 1062, 6, 1, 0, 0, 1030, 1066, 1034, 1003, 0, 0, 0, 0, 995, 1008, 1029, 1010, 0, 0, 0, 0, 1018, 1036, 1012, 990, 13, 3, 0, 0, 1009, 1010, 1040, 1078, 0, 0, 0, 0, 979, 1035, 1024, 1007, 30, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4107, 4127, 4133, 4042, 4072, 4137, 4075] avg=4096.0000 max=4137 max_rank=6 min=4042 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1040, 1052, 1017, 1010, 0, 0, 0, 0, 964, 1017, 1042, 1052, 18, 8, 0, 0, 1029, 1016, 1043, 983, 5, 1, 0, 0, 1025, 1047, 1005, 1039, 0, 0, 0, 0, 990, 1031, 987, 1015, 0, 0, 0, 0, 1003, 1053, 1007, 1022, 12, 3, 0, 0, 1067, 990, 1009, 1037, 0, 0, 0, 0, 994, 1071, 1055, 979, 30, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4101, 4077, 4116, 4023, 4100, 4103, 4129] avg=4096.0000 max=4129 max_rank=7 min=4023 min_rank=4 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1029, 1007, 1032, 1000, 0, 0, 0, 0, 977, 956, 1011, 1014, 18, 8, 0, 0, 1006, 1007, 1030, 1057, 6, 1, 0, 0, 1016, 1037, 1066, 1044, 0, 0, 0, 0, 1069, 1043, 1043, 1018, 0, 0, 0, 0, 988, 995, 1033, 1010, 13, 3, 0, 0, 1011, 1007, 1000, 1065, 0, 0, 0, 0, 1019, 1004, 1039, 1057, 29, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 3984, 4107, 4163, 4173, 4042, 4083, 4148] avg=4096.0000 max=4173 max_rank=4 min=3984 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1058, 1063, 979, 1022, 0, 0, 0, 0, 990, 980, 1045, 1015, 18, 8, 0, 0, 1023, 1023, 1053, 1013, 5, 1, 0, 0, 1026, 1006, 1031, 1038, 0, 0, 0, 0, 1023, 1008, 1009, 1052, 0, 0, 0, 0, 1000, 989, 1037, 1087, 12, 3, 0, 0, 1034, 1018, 1026, 993, 0, 0, 0, 0, 1005, 1012, 984, 1049, 30, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4056, 4118, 4101, 4092, 4128, 4071, 4080] avg=4096.0000 max=4128 max_rank=5 min=4056 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1027, 1055, 1081, 992, 0, 0, 0, 0, 1028, 1049, 1028, 1045, 18, 8, 0, 0, 1034, 991, 1035, 1019, 5, 2, 0, 0, 1051, 1002, 1005, 957, 0, 0, 0, 0, 1041, 1025, 984, 1031, 0, 0, 0, 0, 1013, 1024, 971, 1048, 12, 3, 0, 0, 945, 1050, 1016, 1028, 0, 0, 0, 0, 1022, 1024, 1028, 1042, 29, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4176, 4086, 4015, 4081, 4071, 4039, 4145] avg=4096.0000 max=4176 max_rank=1 min=4015 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1003, 996, 1030, 1001, 0, 0, 0, 0, 993, 990, 1048, 1056, 23, 13, 0, 0, 1006, 1027, 1038, 1020, 9, 7, 0, 0, 1037, 963, 1055, 1037, 0, 0, 0, 0, 1021, 1070, 1046, 1028, 0, 0, 0, 0, 1032, 1026, 1005, 1051, 17, 10, 0, 0, 1034, 1026, 975, 1004, 0, 0, 0, 0, 1046, 1015, 1013, 963, 34, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4123, 4107, 4092, 4165, 4141, 4039, 4071] avg=4096.0000 max=4165 max_rank=4 min=4030 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1059, 1061, 1048, 944, 0, 0, 0, 0, 1055, 992, 984, 1038, 17, 8, 0, 0, 1028, 979, 995, 1023, 6, 2, 0, 0, 1038, 1025, 985, 1025, 0, 0, 0, 0, 1035, 1006, 992, 1009, 0, 0, 0, 0, 1067, 1040, 1028, 1013, 12, 3, 0, 0, 1080, 1003, 1041, 1067, 0, 0, 0, 0, 1004, 1004, 1004, 1020, 28, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4094, 4033, 4073, 4042, 4163, 4191, 4060] avg=4096.0000 max=4191 max_rank=6 min=4033 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1005, 1034, 1022, 1043, 960, 1049, 991, 1080, 1027, 1011, 1011, 1046, 1065, 1026, 993, 1017, 1040, 1039, 952, 980, 1036, 1027, 1024, 966, 1070, 1044, 1013, 1045, 1033, 1063, 1036, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1002, 985, 1009, 995, 1032, 1070, 1044, 1051, 1036, 1075, 1060, 1050, 1056, 1046, 986, 1048, 978, 1005, 1006, 1017, 1066, 1000, 1053, 1038, 963, 1080, 1005, 1013, 1005, 977, 1002, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[999, 1048, 998, 1024, 1057, 1014, 1041, 1016, 1029, 993, 1041, 1039, 1093, 989, 1062, 1031, 1006, 1003, 1037, 1005, 1031, 1016, 1010, 1029, 1037, 997, 1006, 1064, 1049, 1007, 973, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1024, 1003, 1005, 994, 965, 1018, 1036, 1001, 1056, 976, 991, 1061, 1036, 1003, 1025, 1053, 1078, 998, 1004, 1035, 952, 1093, 1051, 1034, 1018, 1082, 1042, 1018, 1013, 1037, 999, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1003, 1072, 1029, 962, 1019, 1013, 1007, 1031, 1014, 1029, 1039, 1019, 1038, 1075, 1027, 1066, 1036, 1015, 961, 1036, 1068, 993, 994, 1023, 1056, 1050, 1078, 1021, 1030, 1051, 944, 969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[988, 1034, 1052, 1027, 1050, 992, 1024, 1026, 1026, 1030, 1030, 1041, 1051, 1064, 1016, 1062, 1023, 999, 1019, 988, 1006, 951, 1051, 1004, 999, 1024, 1002, 1055, 995, 1018, 1053, 1068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1033, 1046, 996, 1031, 971, 1019, 1025, 1077, 974, 966, 962, 1027, 1067, 1040, 1051, 1035, 1066, 995, 1064, 994, 1053, 1003, 998, 1034, 1024, 1025, 1021, 1008, 1036, 1060, 1048, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1028, 1046, 1017, 1021, 1033, 1042, 1055, 1034, 1060, 1045, 1045, 999, 1027, 976, 1021, 1046, 1060, 996, 995, 1032, 1005, 1013, 1035, 980, 1020, 1042, 987, 1055, 952, 997, 1063, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[999, 1048, 998, 1024, 15, 9, 0, 0, 1057, 1014, 1041, 1008, 0, 0, 0, 0, 1029, 993, 1041, 1034, 0, 0, 0, 0, 1039, 989, 1062, 1031, 0, 0, 0, 0, 1006, 1003, 1037, 1005, 39, 0, 0, 0, 1031, 1016, 1010, 1029, 14, 1, 0, 0, 1037, 974, 1006, 1064, 0, 0, 0, 0, 1049, 1007, 973, 1024, 7, 5, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4120, 4097, 4121, 4090, 4101, 4081, 4065] avg=4096.0000 max=4121 max_rank=3 min=4065 min_rank=7 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[988, 1034, 1052, 1027, 14, 9, 0, 0, 1050, 992, 1024, 1018, 0, 0, 0, 0, 1026, 1030, 1030, 1036, 0, 0, 0, 0, 999, 1064, 1016, 1062, 0, 0, 0, 0, 1023, 999, 1019, 988, 38, 0, 0, 0, 1006, 951, 1051, 1004, 14, 1, 0, 0, 999, 1001, 1002, 1055, 0, 0, 0, 0, 995, 1018, 1053, 1068, 7, 5, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4084, 4122, 4141, 4067, 4027, 4057, 4146] avg=4096.0000 max=4146 max_rank=7 min=4027 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16297 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1003, 1072, 1029, 962, 14, 9, 0, 0, 1019, 1013, 1007, 1023, 0, 0, 0, 0, 1014, 1029, 1039, 1015, 0, 0, 0, 0, 987, 1075, 1027, 1066, 0, 0, 0, 0, 1036, 1015, 961, 1036, 37, 0, 0, 0, 1068, 993, 994, 1023, 15, 1, 0, 0, 1056, 1026, 1078, 1021, 0, 0, 0, 0, 1030, 1051, 944, 969, 7, 4, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4062, 4097, 4155, 4085, 4094, 4181, 4005] avg=4096.0000 max=4181 max_rank=6 min=4005 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1005, 1034, 1022, 1043, 14, 9, 0, 0, 960, 1049, 991, 1071, 0, 0, 0, 0, 1027, 1011, 1011, 1041, 0, 0, 0, 0, 1013, 1026, 993, 1017, 0, 0, 0, 0, 1040, 1039, 952, 980, 38, 0, 0, 0, 1036, 1027, 1024, 966, 15, 1, 0, 0, 1070, 1020, 1013, 1045, 0, 0, 0, 0, 1033, 1063, 1036, 1020, 8, 5, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4071, 4090, 4049, 4049, 4069, 4148, 4165] avg=4096.0000 max=4165 max_rank=7 min=4049 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1002, 985, 1009, 995, 14, 9, 0, 0, 1032, 1070, 1044, 1042, 0, 0, 0, 0, 1036, 1075, 1060, 1045, 0, 0, 0, 0, 1004, 1046, 986, 1048, 0, 0, 0, 0, 978, 1005, 1006, 1017, 38, 0, 0, 0, 1066, 1000, 1053, 1038, 15, 1, 0, 0, 963, 1056, 1005, 1013, 0, 0, 0, 0, 1005, 977, 1002, 1015, 8, 5, 0, 0] rerouted_tokens_per_ep_rank=[4014, 4188, 4216, 4084, 4044, 4173, 4037, 4012] avg=4096.0000 max=4216 max_rank=2 min=4012 min_rank=7 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1024, 1003, 1005, 994, 14, 9, 0, 0, 965, 1018, 1036, 993, 0, 0, 0, 0, 1056, 976, 991, 1056, 0, 0, 0, 0, 985, 1003, 1025, 1053, 0, 0, 0, 0, 1078, 998, 1004, 1035, 37, 0, 0, 0, 952, 1093, 1051, 1034, 15, 1, 0, 0, 1018, 1058, 1042, 1018, 0, 0, 0, 0, 1013, 1037, 999, 1067, 7, 5, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4012, 4079, 4066, 4152, 4146, 4136, 4128] avg=4096.0000 max=4152 max_rank=4 min=4012 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16562 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1033, 1046, 996, 1031, 18, 13, 0, 0, 971, 1019, 1025, 1060, 0, 0, 0, 0, 974, 966, 962, 1020, 0, 0, 0, 0, 1007, 1040, 1051, 1035, 0, 0, 0, 0, 1066, 995, 1064, 994, 42, 0, 0, 0, 1053, 1003, 998, 1034, 18, 4, 0, 0, 1024, 994, 1021, 1008, 0, 0, 0, 0, 1036, 1060, 1048, 1019, 13, 7, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4075, 3922, 4133, 4161, 4110, 4047, 4183] avg=4096.0000 max=4183 max_rank=7 min=3922 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1028, 1046, 1017, 1021, 14, 9, 0, 0, 1033, 1042, 1055, 1026, 0, 0, 0, 0, 1060, 1045, 1045, 995, 0, 0, 0, 0, 976, 976, 1021, 1046, 0, 0, 0, 0, 1060, 996, 995, 1032, 37, 0, 0, 0, 1005, 1013, 1035, 980, 14, 1, 0, 0, 1020, 1019, 987, 1055, 0, 0, 0, 0, 952, 997, 1063, 1041, 7, 4, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4156, 4145, 4019, 4120, 4048, 4081, 4064] avg=4096.0000 max=4156 max_rank=1 min=4019 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8192, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8064, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1033, 1012, 1034, 983, 997, 1002, 981, 1055, 1016, 1051, 1036, 997, 1022, 1016, 1032, 1028, 981, 1024, 1014, 1078, 1045, 1006, 1010, 1044, 1008, 1047, 1039, 1046, 1061, 1004, 1051, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1029, 1089, 1019, 1024, 980, 1038, 987, 1015, 999, 1035, 1029, 987, 994, 1038, 980, 1061, 1055, 1016, 1047, 1057, 1007, 999, 967, 1044, 1020, 1005, 1004, 1027, 1036, 1033, 1062, 1085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1052, 1005, 1042, 999, 1029, 1035, 1003, 1083, 1020, 1029, 984, 1029, 1042, 1045, 1036, 962, 1034, 1002, 1038, 1019, 1059, 1025, 1013, 1040, 1030, 998, 1006, 1025, 1023, 990, 1055, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[999, 981, 1035, 977, 1014, 1028, 1023, 1025, 1015, 1029, 1015, 1024, 1018, 1029, 1013, 1016, 1039, 1058, 1035, 1038, 1018, 1046, 1069, 1070, 1009, 977, 998, 1034, 1052, 1020, 1017, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1032, 1036, 998, 1009, 1063, 1011, 1063, 1033, 1008, 1000, 1039, 1020, 1045, 985, 1033, 1028, 1043, 978, 999, 996, 1028, 1006, 1049, 1081, 991, 1029, 1029, 1035, 1023, 1017, 1086, 975] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1017, 1024, 1037, 1071, 986, 967, 1005, 983, 1032, 1016, 1012, 1051, 1033, 1009, 1070, 1014, 1028, 1013, 1015, 1044, 1050, 1025, 1042, 1064, 1025, 1006, 971, 1042, 1063, 1034, 984, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1010, 1027, 1102, 1004, 1033, 1020, 984, 1004, 1016, 1035, 996, 1035, 1029, 1063, 1036, 1005, 1019, 1027, 967, 1044, 1017, 1053, 1029, 999, 1027, 1008, 1002, 1064, 1030, 1008, 1041, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[995, 1012, 1013, 1012, 983, 1058, 1011, 1011, 1030, 1009, 1024, 1062, 1002, 1027, 1002, 1064, 1055, 1064, 1045, 1018, 1021, 1023, 984, 1065, 1033, 1019, 982, 1076, 1052, 1011, 1011, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1033, 1012, 1034, 983, 6, 1, 0, 0, 997, 1002, 981, 1055, 28, 3, 0, 0, 1016, 1051, 1036, 997, 9, 1, 0, 0, 1022, 1015, 1032, 1028, 0, 0, 0, 0, 981, 1024, 1014, 1063, 0, 0, 0, 0, 1045, 1006, 1010, 1016, 0, 0, 0, 0, 1008, 1047, 1039, 1046, 19, 0, 0, 0, 1038, 1004, 1051, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4066, 4110, 4097, 4082, 4077, 4159, 4108] avg=4096.0000 max=4159 max_rank=6 min=4066 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1052, 1005, 1042, 999, 5, 1, 0, 0, 1029, 1035, 1003, 1083, 28, 3, 0, 0, 1020, 1029, 984, 1029, 9, 1, 0, 0, 1042, 1044, 1036, 962, 0, 0, 0, 0, 1034, 1002, 1038, 1005, 0, 0, 0, 0, 1059, 1025, 1013, 1012, 0, 0, 0, 0, 1030, 998, 1006, 1025, 19, 0, 0, 0, 1000, 990, 1055, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4181, 4072, 4084, 4079, 4109, 4078, 4061] avg=4096.0000 max=4181 max_rank=1 min=4061 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1032, 1036, 998, 1009, 5, 1, 0, 0, 1063, 1011, 1063, 1033, 29, 3, 0, 0, 1008, 1000, 1039, 1020, 9, 1, 0, 0, 1045, 984, 1033, 1028, 0, 0, 0, 0, 1043, 978, 999, 982, 0, 0, 0, 0, 1028, 1006, 1049, 1052, 0, 0, 0, 0, 991, 1029, 1029, 1035, 19, 0, 0, 0, 1000, 1017, 1086, 975, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4202, 4077, 4090, 4002, 4135, 4103, 4078] avg=4096.0000 max=4202 max_rank=1 min=4002 min_rank=4 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[999, 981, 1035, 977, 5, 1, 0, 0, 1014, 1028, 1023, 1025, 29, 3, 0, 0, 1015, 1029, 1015, 1024, 9, 1, 0, 0, 1018, 1028, 1013, 1016, 0, 0, 0, 0, 1039, 1058, 1035, 1024, 0, 0, 0, 0, 1018, 1046, 1069, 1041, 0, 0, 0, 0, 1009, 977, 998, 1034, 19, 0, 0, 0, 1029, 1020, 1017, 1047, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 4122, 4093, 4075, 4156, 4174, 4037, 4113] avg=4096.0000 max=4174 max_rank=5 min=3998 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1029, 1089, 1019, 1024, 11, 2, 0, 0, 980, 1038, 987, 1015, 31, 7, 0, 0, 999, 1035, 1029, 987, 12, 6, 0, 0, 994, 1036, 980, 1061, 0, 0, 0, 0, 1055, 1016, 1047, 1034, 0, 0, 0, 0, 1007, 999, 967, 1013, 0, 0, 0, 0, 1020, 1005, 1004, 1027, 23, 0, 0, 0, 1000, 1033, 1062, 1085, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4058, 4068, 4071, 4152, 3986, 4079, 4180] avg=4096.0000 max=4180 max_rank=7 min=3986 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1010, 1027, 1102, 1004, 5, 1, 0, 0, 1033, 1020, 984, 1004, 27, 3, 0, 0, 1016, 1035, 996, 1035, 9, 1, 0, 0, 1029, 1062, 1036, 1005, 0, 0, 0, 0, 1019, 1027, 967, 1030, 0, 0, 0, 0, 1017, 1053, 1029, 972, 0, 0, 0, 0, 1027, 1008, 1002, 1064, 19, 0, 0, 0, 1007, 1008, 1041, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4071, 4092, 4132, 4043, 4071, 4120, 4090] avg=4096.0000 max=4149 max_rank=0 min=4043 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1017, 1024, 1037, 1071, 5, 1, 0, 0, 986, 967, 1005, 983, 29, 3, 0, 0, 1032, 1016, 1012, 1051, 9, 1, 0, 0, 1033, 1008, 1070, 1014, 0, 0, 0, 0, 1028, 1013, 1015, 1030, 0, 0, 0, 0, 1050, 1025, 1042, 1035, 0, 0, 0, 0, 1025, 1006, 971, 1042, 19, 0, 0, 0, 1040, 1034, 984, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 3973, 4121, 4125, 4086, 4152, 4063, 4093] avg=4096.0000 max=4155 max_rank=0 min=3973 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[995, 1012, 1013, 1012, 5, 1, 0, 0, 983, 1058, 1011, 1011, 29, 3, 0, 0, 1030, 1009, 1024, 1062, 9, 1, 0, 0, 1002, 1026, 1002, 1064, 0, 0, 0, 0, 1055, 1064, 1045, 1004, 0, 0, 0, 0, 1021, 1023, 984, 1036, 0, 0, 0, 0, 1033, 1019, 982, 1076, 19, 0, 0, 0, 1029, 1011, 1011, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4095, 4135, 4094, 4168, 4064, 4129, 4045] avg=4096.0000 max=4168 max_rank=4 min=4038 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8064, 8448, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8064, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1056, 942, 1067, 1039, 1028, 1064, 1022, 1046, 1010, 1003, 1005, 1056, 1034, 1031, 1044, 976, 1032, 1057, 958, 1003, 1055, 1041, 965, 1048, 1030, 1050, 992, 966, 1038, 1023, 1027, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1030, 1018, 986, 1020, 1004, 993, 1042, 1028, 1019, 1093, 978, 1005, 1032, 1027, 1001, 993, 1020, 1022, 1040, 982, 983, 1084, 1009, 1081, 1058, 997, 1010, 1023, 1042, 1073, 1060, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1005, 1005, 993, 1049, 972, 1005, 1027, 1011, 1031, 1008, 1025, 955, 1070, 1041, 1064, 1022, 1041, 995, 1037, 1076, 1029, 1015, 1065, 1038, 1021, 1000, 1022, 1038, 1017, 1054, 1001, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[951, 1065, 1048, 1045, 1030, 1048, 1054, 1034, 1009, 1002, 1063, 1023, 1002, 1002, 953, 997, 1040, 998, 1045, 1053, 1028, 1035, 1014, 1048, 1022, 1033, 983, 1042, 980, 1024, 1043, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1026, 1015, 998, 1078, 1056, 1015, 1019, 1025, 1013, 1055, 1056, 1036, 1031, 1024, 1082, 1010, 1020, 1018, 982, 984, 1011, 1024, 1018, 1025, 1029, 1021, 1071, 1005, 997, 1028, 981, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1000, 1031, 1042, 1032, 1022, 989, 1054, 1014, 979, 1027, 973, 1030, 999, 1046, 994, 1026, 1069, 1007, 1017, 1042, 1047, 1037, 1009, 1018, 1044, 1032, 1003, 1013, 1015, 1089, 1025, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1059, 957, 1040, 1074, 1019, 1062, 1046, 1032, 1000, 1022, 1028, 972, 1025, 1035, 985, 1051, 977, 1026, 971, 1002, 1060, 1035, 1051, 1022, 1062, 995, 1030, 1091, 1023, 1004, 1019, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[975, 1066, 1066, 1015, 1001, 1012, 1006, 1014, 973, 1025, 1001, 1069, 1025, 1050, 1027, 1039, 1070, 1040, 1044, 1044, 1044, 1020, 1022, 1029, 1025, 1017, 978, 1009, 1021, 1060, 995, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1056, 942, 1067, 1036, 0, 0, 0, 0, 1028, 1064, 1020, 1046, 0, 0, 0, 0, 1010, 1003, 1005, 1056, 28, 0, 0, 0, 1034, 1031, 1044, 976, 3, 0, 0, 0, 1032, 1057, 958, 1003, 4, 2, 0, 0, 1055, 1041, 965, 1018, 0, 0, 0, 0, 1030, 1050, 992, 966, 4, 2, 0, 0, 1038, 1015, 1027, 1060, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4158, 4102, 4088, 4056, 4079, 4044, 4140] avg=4096.0000 max=4158 max_rank=1 min=4044 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[951, 1065, 1048, 1042, 0, 0, 0, 0, 1030, 1048, 1052, 1034, 0, 0, 0, 0, 1009, 1002, 1063, 1023, 28, 0, 0, 0, 1002, 1002, 953, 997, 3, 0, 0, 0, 1040, 998, 1045, 1053, 4, 2, 0, 0, 1028, 1035, 1014, 1018, 0, 0, 0, 0, 1022, 1033, 983, 1042, 4, 2, 0, 0, 980, 1016, 1043, 1054, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4164, 4125, 3957, 4142, 4095, 4086, 4093] avg=4096.0000 max=4164 max_rank=1 min=3957 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1030, 1018, 986, 1017, 0, 0, 0, 0, 1004, 993, 1040, 1028, 0, 0, 0, 0, 1019, 1093, 978, 1005, 29, 0, 0, 0, 1032, 1027, 1001, 993, 3, 0, 0, 0, 1020, 1022, 1040, 982, 4, 2, 0, 0, 983, 1084, 1009, 1050, 0, 0, 0, 0, 1058, 997, 1010, 1023, 4, 2, 0, 0, 1042, 1065, 1060, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4065, 4124, 4056, 4070, 4126, 4094, 4182] avg=4096.0000 max=4182 max_rank=7 min=4051 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1005, 1005, 993, 1046, 0, 0, 0, 0, 972, 1005, 1025, 1011, 0, 0, 0, 0, 1031, 1008, 1025, 955, 27, 0, 0, 0, 1070, 1041, 1064, 1022, 3, 0, 0, 0, 1041, 995, 1037, 1076, 4, 2, 0, 0, 1029, 1015, 1065, 1009, 0, 0, 0, 0, 1021, 1000, 1022, 1038, 4, 2, 0, 0, 1017, 1046, 1001, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4013, 4046, 4200, 4155, 4118, 4087, 4100] avg=4096.0000 max=4200 max_rank=3 min=4013 min_rank=1 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1000, 1031, 1042, 1029, 0, 0, 0, 0, 1022, 989, 1052, 1014, 0, 0, 0, 0, 979, 1027, 973, 1030, 27, 0, 0, 0, 999, 1046, 994, 1026, 3, 0, 0, 0, 1069, 1007, 1017, 1042, 4, 2, 0, 0, 1047, 1037, 1009, 989, 0, 0, 0, 0, 1044, 1032, 1003, 1013, 4, 2, 0, 0, 1015, 1081, 1025, 1043, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4077, 4036, 4068, 4141, 4082, 4098, 4164] avg=4096.0000 max=4164 max_rank=7 min=4036 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 1015, 998, 1074, 0, 0, 0, 0, 1056, 1015, 1007, 1025, 0, 0, 0, 0, 1013, 1055, 1056, 1036, 31, 0, 0, 0, 1031, 1024, 1082, 1010, 4, 5, 0, 0, 1020, 1018, 982, 984, 7, 7, 0, 0, 1011, 1024, 1018, 990, 0, 0, 0, 0, 1029, 1021, 1071, 1005, 10, 4, 0, 0, 997, 1011, 981, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4103, 4191, 4156, 4018, 4043, 4140, 4004] avg=4096.0000 max=4191 max_rank=2 min=4004 min_rank=7 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16205 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1059, 957, 1040, 1071, 0, 0, 0, 0, 1019, 1062, 1044, 1032, 0, 0, 0, 0, 1000, 1022, 1028, 972, 27, 0, 0, 0, 1025, 1035, 985, 1051, 3, 0, 0, 0, 977, 1026, 971, 1002, 4, 2, 0, 0, 1060, 1035, 1051, 993, 0, 0, 0, 0, 1062, 995, 1030, 1091, 4, 2, 0, 0, 1023, 996, 1019, 993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4157, 4049, 4099, 3982, 4139, 4184, 4031] avg=4096.0000 max=4184 max_rank=6 min=3982 min_rank=4 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[975, 1066, 1066, 1012, 0, 0, 0, 0, 1001, 1012, 1004, 1014, 0, 0, 0, 0, 973, 1025, 1001, 1069, 27, 0, 0, 0, 1025, 1050, 1027, 1039, 3, 0, 0, 0, 1070, 1040, 1044, 1044, 4, 2, 0, 0, 1044, 1020, 1022, 1000, 0, 0, 0, 0, 1025, 1017, 978, 1009, 4, 2, 0, 0, 1021, 1052, 995, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4031, 4095, 4144, 4204, 4086, 4035, 4054] avg=4096.0000 max=4204 max_rank=4 min=4031 min_rank=1 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1005, 1028, 1041, 999, 1015, 1032, 1016, 1018, 1033, 1052, 1050, 1031, 1006, 1006, 998, 1018, 1036, 1053, 1094, 1026, 1021, 985, 1019, 1015, 981, 954, 1058, 1036, 1027, 1029, 1043, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1034, 1013, 999, 1066, 1018, 1026, 1023, 987, 1053, 1070, 1058, 1016, 1016, 1053, 1075, 1031, 1050, 1013, 995, 1031, 989, 992, 1002, 969, 1033, 1055, 1040, 1025, 1013, 1017, 1024, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1043, 1061, 1017, 1010, 1013, 1022, 1010, 1040, 1017, 993, 1042, 1099, 978, 1014, 1013, 1026, 1053, 997, 1022, 1008, 1020, 1042, 1014, 1018, 1020, 1005, 1015, 1050, 1016, 1044, 1037, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1024, 988, 983, 1013, 977, 986, 1070, 998, 1025, 1058, 1052, 1028, 1060, 994, 1017, 1015, 1036, 1033, 1021, 1061, 1033, 996, 976, 1057, 1010, 1042, 1080, 1069, 1010, 982, 996, 1078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1055, 1013, 1102, 1007, 989, 999, 1014, 973, 1030, 995, 1058, 1073, 995, 1060, 1005, 981, 1010, 1030, 1029, 1022, 1024, 1008, 1013, 976, 1048, 1029, 1043, 1023, 1051, 1028, 1011, 1074] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1053, 1016, 1014, 1017, 1008, 1019, 991, 994, 1021, 1077, 1065, 1094, 1090, 1011, 1042, 1008, 978, 1005, 1048, 1063, 1005, 1068, 1008, 1013, 1047, 972, 1020, 980, 986, 1017, 1020, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1017, 1053, 1027, 1054, 1044, 1065, 1058, 1024, 1036, 1022, 1039, 994, 955, 988, 1008, 1024, 1035, 1059, 1002, 1071, 998, 992, 987, 1014, 1056, 1016, 1007, 1024, 981, 1033, 1011, 1074] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1003, 989, 1050, 1013, 1004, 1010, 1038, 1044, 1012, 1001, 1024, 1000, 988, 1021, 1047, 1056, 1024, 1013, 1037, 1026, 1047, 1040, 1052, 960, 1054, 1072, 1039, 1007, 999, 1004, 1026, 1068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1039, 1061, 1017, 1010, 0, 0, 0, 0, 1013, 1022, 1010, 1040, 25, 4, 0, 0, 1017, 993, 987, 1099, 0, 0, 0, 0, 978, 1014, 1013, 1026, 17, 2, 0, 0, 1053, 997, 1022, 983, 0, 0, 0, 0, 1020, 1042, 1014, 1018, 51, 0, 0, 0, 1020, 1005, 998, 1050, 0, 0, 0, 0, 1016, 1044, 1037, 1009, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4114, 4096, 4050, 4055, 4145, 4073, 4108] avg=4096.0000 max=4145 max_rank=5 min=4050 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1001, 1028, 1041, 999, 0, 0, 0, 0, 1015, 1032, 1016, 1018, 25, 4, 0, 0, 1033, 1052, 995, 1031, 0, 0, 0, 0, 1006, 1006, 998, 1018, 18, 2, 0, 0, 1036, 1053, 1094, 1001, 0, 0, 0, 0, 1021, 985, 1019, 1015, 51, 0, 0, 0, 981, 954, 1040, 1036, 0, 0, 0, 0, 1027, 1029, 1043, 1043, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4110, 4111, 4048, 4184, 4091, 4011, 4144] avg=4096.0000 max=4184 max_rank=4 min=4011 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1020, 988, 983, 1013, 0, 0, 0, 0, 977, 986, 1070, 998, 26, 4, 0, 0, 1025, 1058, 996, 1028, 0, 0, 0, 0, 1060, 994, 1017, 1015, 18, 2, 0, 0, 1036, 1033, 1021, 1035, 0, 0, 0, 0, 1033, 996, 976, 1057, 52, 0, 0, 0, 1010, 1042, 1062, 1069, 0, 0, 0, 0, 1010, 982, 996, 1078, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4004, 4061, 4107, 4106, 4125, 4114, 4183, 4068] avg=4096.0000 max=4183 max_rank=6 min=4004 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1030, 1013, 999, 1066, 0, 0, 0, 0, 1018, 1026, 1023, 987, 25, 4, 0, 0, 1053, 1070, 1002, 1016, 0, 0, 0, 0, 1016, 1053, 1075, 1031, 17, 2, 0, 0, 1050, 1013, 995, 1006, 0, 0, 0, 0, 989, 992, 1002, 969, 52, 0, 0, 0, 1033, 1055, 1023, 1025, 0, 0, 0, 0, 1013, 1017, 1024, 982, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4083, 4141, 4194, 4064, 4004, 4136, 4038] avg=4096.0000 max=4194 max_rank=3 min=4004 min_rank=5 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1051, 1013, 1102, 1007, 0, 0, 0, 0, 989, 999, 1014, 973, 25, 4, 0, 0, 1030, 995, 1002, 1073, 0, 0, 0, 0, 995, 1060, 1005, 981, 17, 2, 0, 0, 1010, 1030, 1029, 997, 0, 0, 0, 0, 1024, 1008, 1013, 976, 52, 0, 0, 0, 1048, 1029, 1026, 1023, 0, 0, 0, 0, 1051, 1028, 1011, 1074, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4004, 4100, 4060, 4066, 4073, 4126, 4166] avg=4096.0000 max=4173 max_rank=0 min=4004 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1013, 1053, 1027, 1054, 0, 0, 0, 0, 1044, 1065, 1058, 1024, 26, 4, 0, 0, 1036, 1022, 984, 994, 0, 0, 0, 0, 955, 988, 1008, 1024, 17, 2, 0, 0, 1035, 1059, 1002, 1045, 0, 0, 0, 0, 998, 992, 987, 1014, 51, 0, 0, 0, 1056, 1016, 990, 1024, 0, 0, 0, 0, 981, 1033, 1011, 1074, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4221, 4036, 3994, 4141, 4042, 4086, 4101] avg=4096.0000 max=4221 max_rank=1 min=3994 min_rank=3 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1042, 1016, 1014, 1017, 0, 0, 0, 0, 1008, 1019, 991, 994, 31, 7, 0, 0, 1021, 1077, 1002, 1094, 0, 0, 0, 0, 1090, 1011, 1042, 1008, 21, 8, 5, 0, 978, 1005, 1048, 1027, 0, 0, 0, 0, 1005, 1068, 1008, 1013, 56, 0, 0, 0, 1047, 972, 999, 980, 0, 0, 0, 0, 986, 1017, 1020, 1018, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4050, 4194, 4185, 4058, 4150, 3998, 4044] avg=4096.0000 max=4194 max_rank=2 min=3998 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[999, 989, 1050, 1013, 0, 0, 0, 0, 1004, 1010, 1038, 1044, 25, 4, 0, 0, 1012, 1001, 970, 1000, 0, 0, 0, 0, 988, 1021, 1047, 1056, 17, 2, 0, 0, 1024, 1013, 1037, 1001, 0, 0, 0, 0, 1047, 1040, 1052, 960, 50, 0, 0, 0, 1054, 1072, 1022, 1007, 0, 0, 0, 0, 999, 1004, 1026, 1068, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4125, 3983, 4131, 4075, 4149, 4155, 4099] avg=4096.0000 max=4155 max_rank=6 min=3983 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8064, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8448, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1021, 1015, 1019, 1025, 992, 1065, 1008, 1053, 1023, 1001, 1049, 1033, 1084, 985, 1044, 1044, 1008, 998, 1033, 1078, 1037, 1020, 1011, 978, 1029, 1037, 1030, 960, 1054, 989, 1010, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1060, 1042, 984, 1051, 1006, 1081, 1037, 992, 1016, 978, 996, 1026, 1021, 1063, 1011, 1005, 987, 1069, 1030, 1029, 986, 1058, 1035, 1003, 1050, 974, 1034, 1038, 1020, 1019, 1005, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[993, 1016, 1017, 1012, 1008, 1005, 1017, 1025, 1023, 1046, 1044, 1068, 1015, 1056, 1022, 1034, 1058, 1006, 1005, 1061, 1060, 1029, 1000, 991, 988, 1004, 1064, 1029, 973, 1038, 1033, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1019, 1034, 1034, 1059, 1026, 967, 1000, 1077, 1048, 1052, 1021, 1014, 1036, 1037, 997, 1023, 986, 1013, 983, 1020, 999, 1060, 1052, 1018, 1032, 984, 1041, 1051, 1005, 1004, 1050, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1030, 992, 1017, 1016, 1004, 1027, 999, 959, 1038, 1014, 1025, 1014, 1011, 1054, 1022, 1011, 1055, 1025, 1021, 993, 1073, 1072, 1021, 998, 1055, 1051, 1064, 1000, 1005, 1043, 1011, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[990, 1051, 1014, 1010, 1012, 1037, 1021, 1050, 1039, 1028, 993, 1026, 1051, 1008, 977, 1081, 1031, 1047, 1037, 1026, 1005, 1004, 1054, 1049, 1036, 1005, 994, 1020, 1025, 1023, 1009, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1042, 1063, 1048, 1034, 1028, 1026, 1014, 1015, 1034, 1068, 1040, 969, 980, 986, 1063, 991, 1011, 1027, 1018, 1036, 1065, 1029, 985, 1059, 1036, 1011, 1037, 1025, 987, 1020, 992, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1003, 985, 1004, 1038, 1023, 1053, 1007, 1000, 1015, 1089, 1036, 1012, 1053, 1027, 1065, 1016, 1064, 1002, 1027, 1001, 1029, 1023, 1041, 1009, 1012, 1040, 1025, 949, 994, 1062, 1027, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1021, 1015, 1019, 1025, 2, 1, 0, 0, 992, 1065, 1008, 1053, 13, 3, 0, 0, 1023, 988, 1049, 1033, 0, 0, 0, 0, 1071, 985, 1044, 1044, 0, 0, 0, 0, 1008, 998, 1033, 1076, 0, 0, 0, 0, 1037, 1011, 1011, 978, 0, 0, 0, 0, 1029, 1037, 1030, 960, 7, 0, 0, 0, 1054, 989, 1010, 1035, 10, 1, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4134, 4093, 4144, 4115, 4037, 4063, 4099] avg=4096.0000 max=4144 max_rank=3 min=4037 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[993, 1016, 1017, 1012, 2, 1, 0, 0, 1008, 1005, 1017, 1025, 13, 2, 0, 0, 1023, 1033, 1044, 1068, 0, 0, 0, 0, 1004, 1056, 1022, 1034, 0, 0, 0, 0, 1058, 1006, 1005, 1059, 0, 0, 0, 0, 1060, 1020, 1000, 991, 0, 0, 0, 0, 988, 1004, 1064, 1029, 7, 0, 0, 0, 973, 1038, 1033, 1028, 9, 1, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4070, 4168, 4116, 4128, 4071, 4092, 4082] avg=4096.0000 max=4168 max_rank=2 min=4041 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1019, 1034, 1034, 1059, 2, 1, 0, 0, 1026, 967, 1000, 1077, 13, 3, 0, 0, 1048, 1039, 1021, 1014, 0, 0, 0, 0, 1023, 1037, 997, 1023, 0, 0, 0, 0, 986, 1013, 983, 1018, 0, 0, 0, 0, 999, 1050, 1052, 1018, 0, 0, 0, 0, 1032, 984, 1041, 1051, 8, 0, 0, 0, 1005, 1004, 1050, 1026, 10, 1, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4086, 4122, 4080, 4000, 4119, 4116, 4096] avg=4096.0000 max=4149 max_rank=0 min=4000 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1030, 992, 1017, 1016, 2, 1, 0, 0, 1004, 1027, 999, 959, 13, 2, 0, 0, 1038, 1001, 1025, 1014, 0, 0, 0, 0, 1000, 1054, 1022, 1011, 0, 0, 0, 0, 1055, 1025, 1021, 991, 0, 0, 0, 0, 1073, 1062, 1021, 998, 0, 0, 0, 0, 1055, 1051, 1064, 1000, 8, 0, 0, 0, 1005, 1043, 1011, 1048, 9, 1, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4004, 4078, 4087, 4092, 4154, 4178, 4117] avg=4096.0000 max=4178 max_rank=6 min=4004 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16541 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1060, 1042, 984, 1051, 2, 1, 0, 0, 1006, 1081, 1037, 992, 12, 2, 0, 0, 1016, 966, 996, 1026, 0, 0, 0, 0, 1009, 1063, 1011, 1005, 0, 0, 0, 0, 987, 1069, 1030, 1027, 0, 0, 0, 0, 986, 1048, 1035, 1003, 0, 0, 0, 0, 1050, 974, 1034, 1038, 8, 0, 0, 0, 1020, 1019, 1005, 1062, 10, 1, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4130, 4004, 4088, 4113, 4072, 4104, 4117] avg=4096.0000 max=4140 max_rank=0 min=4004 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1042, 1063, 1048, 1034, 2, 1, 0, 0, 1028, 1026, 1014, 1015, 14, 2, 0, 0, 1034, 1054, 1040, 969, 0, 0, 0, 0, 969, 986, 1063, 991, 0, 0, 0, 0, 1011, 1027, 1018, 1034, 0, 0, 0, 0, 1065, 1020, 985, 1059, 0, 0, 0, 0, 1036, 1011, 1037, 1025, 7, 0, 0, 0, 987, 1020, 992, 1029, 9, 1, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4099, 4097, 4009, 4090, 4129, 4116, 4038] avg=4096.0000 max=4190 max_rank=0 min=4009 min_rank=3 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[990, 1051, 1014, 1010, 3, 6, 0, 0, 1012, 1037, 1021, 1050, 18, 7, 0, 0, 1039, 1010, 993, 1026, 0, 0, 0, 0, 1030, 1008, 977, 1081, 0, 0, 0, 0, 1031, 1047, 1037, 1023, 0, 0, 0, 0, 1005, 985, 1054, 1049, 0, 0, 0, 0, 1036, 1005, 994, 1020, 11, 0, 0, 0, 1025, 1023, 1009, 1015, 14, 2, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4145, 4068, 4096, 4138, 4093, 4066, 4088] avg=4096.0000 max=4145 max_rank=1 min=4066 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1003, 985, 1004, 1038, 2, 1, 0, 0, 1023, 1053, 1007, 1000, 14, 3, 0, 0, 1015, 1075, 1036, 1012, 0, 0, 0, 0, 1040, 1027, 1065, 1016, 0, 0, 0, 0, 1064, 1002, 1027, 999, 0, 0, 0, 0, 1029, 1014, 1041, 1009, 0, 0, 0, 0, 1012, 1040, 1025, 949, 7, 0, 0, 0, 994, 1062, 1027, 1037, 10, 1, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4100, 4138, 4148, 4092, 4093, 4033, 4131] avg=4096.0000 max=4148 max_rank=3 min=4033 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1012, 1006, 1036, 1004, 1009, 1011, 1059, 1053, 1017, 1017, 1026, 1024, 1091, 1014, 993, 1042, 1061, 1028, 1079, 1009, 1003, 1005, 1008, 1042, 1005, 1012, 1010, 1010, 995, 970, 1052, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1013, 1034, 1050, 1041, 1063, 1043, 1024, 1022, 1027, 1008, 1004, 979, 1009, 1014, 996, 1021, 989, 1024, 1013, 1009, 954, 1037, 1052, 1037, 1014, 1057, 1034, 1009, 1020, 1074, 1017, 1080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[998, 1000, 1005, 969, 1016, 1047, 1020, 1069, 991, 1040, 1009, 1054, 1069, 1041, 1043, 1079, 1028, 1011, 1011, 1033, 1048, 1028, 996, 1013, 1047, 1028, 1035, 987, 1025, 980, 973, 1075] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1033, 1044, 1034, 1072, 1043, 1057, 978, 1058, 1023, 1004, 1002, 1011, 1062, 1080, 1009, 981, 996, 983, 1000, 1044, 1031, 1026, 1013, 1020, 971, 1044, 1042, 997, 1034, 1031, 1038, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[999, 1038, 988, 1035, 1052, 1065, 1036, 1044, 1047, 1015, 988, 1015, 1009, 1073, 999, 1025, 1007, 1054, 976, 1021, 1064, 1029, 1079, 1031, 1009, 992, 1030, 951, 1012, 1031, 992, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1035, 983, 1044, 1010, 988, 1058, 1002, 1041, 1012, 1014, 1026, 984, 1012, 1011, 1047, 1048, 1030, 1042, 1051, 1020, 1065, 978, 1028, 1056, 1043, 1021, 1044, 1009, 1007, 1033, 1009, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1000, 1044, 1043, 1020, 1020, 999, 1005, 1016, 1038, 1050, 1002, 1006, 1041, 966, 1011, 1002, 1036, 995, 1016, 1019, 990, 1007, 1016, 1051, 1021, 1039, 1094, 1043, 1080, 1063, 1001, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1011, 1036, 1022, 1047, 1061, 1022, 1003, 1006, 1024, 1008, 991, 970, 1042, 1013, 1003, 1083, 1015, 1027, 1020, 1078, 1013, 997, 1006, 1057, 1050, 1029, 1000, 1025, 1015, 1056, 1016, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1013, 1034, 1050, 1041, 8, 0, 0, 0, 1063, 1043, 1024, 995, 0, 0, 0, 0, 1027, 1008, 1004, 979, 27, 14, 0, 0, 991, 1014, 996, 1021, 0, 0, 0, 0, 989, 1024, 1013, 1009, 4, 1, 0, 0, 954, 1037, 1052, 1036, 0, 0, 0, 0, 1014, 1057, 1034, 1009, 4, 3, 0, 0, 1020, 1074, 1017, 1065, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4125, 4059, 4022, 4040, 4079, 4121, 4176] avg=4096.0000 max=4176 max_rank=7 min=4022 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1012, 1006, 1036, 1004, 7, 0, 0, 0, 1009, 1011, 1059, 1025, 0, 0, 0, 0, 1017, 1017, 1026, 1024, 28, 15, 0, 0, 1071, 1014, 993, 1042, 0, 0, 0, 0, 1061, 1028, 1079, 1009, 3, 1, 0, 0, 1003, 1005, 1008, 1041, 0, 0, 0, 0, 1005, 1012, 1010, 1010, 5, 3, 0, 0, 995, 970, 1052, 1052, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4104, 4127, 4120, 4181, 4057, 4045, 4069] avg=4096.0000 max=4181 max_rank=4 min=4045 min_rank=6 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[998, 1000, 1005, 969, 7, 0, 0, 0, 1016, 1047, 1020, 1041, 0, 0, 0, 0, 991, 1040, 1009, 1054, 28, 15, 0, 0, 1049, 1041, 1043, 1079, 0, 0, 0, 0, 1028, 1011, 1011, 1033, 3, 1, 0, 0, 1048, 1028, 996, 1012, 0, 0, 0, 0, 1047, 1028, 1035, 987, 5, 3, 0, 0, 1025, 980, 973, 1062, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3979, 4124, 4137, 4212, 4087, 4084, 4105, 4040] avg=4096.0000 max=4212 max_rank=3 min=3979 min_rank=0 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[999, 1038, 988, 1035, 7, 0, 0, 0, 1052, 1065, 1036, 1017, 0, 0, 0, 0, 1047, 1015, 988, 1015, 27, 14, 0, 0, 991, 1073, 999, 1025, 0, 0, 0, 0, 1007, 1054, 976, 1021, 3, 1, 0, 0, 1064, 1029, 1079, 1030, 0, 0, 0, 0, 1009, 992, 1030, 951, 4, 3, 0, 0, 1012, 1031, 992, 1049, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4170, 4106, 4088, 4062, 4202, 3989, 4084] avg=4096.0000 max=4202 max_rank=5 min=3989 min_rank=6 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1033, 1044, 1034, 1072, 7, 0, 0, 0, 1043, 1057, 978, 1030, 0, 0, 0, 0, 1023, 1004, 1002, 1011, 28, 15, 0, 0, 1042, 1080, 1009, 981, 0, 0, 0, 0, 996, 983, 1000, 1044, 3, 1, 0, 0, 1031, 1026, 1013, 1019, 0, 0, 0, 0, 971, 1044, 1042, 997, 5, 3, 0, 0, 1034, 1031, 1038, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4108, 4083, 4112, 4027, 4089, 4062, 4097] avg=4096.0000 max=4190 max_rank=0 min=4027 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1000, 1044, 1043, 1020, 7, 0, 0, 0, 1020, 999, 1005, 989, 0, 0, 0, 0, 1038, 1050, 1002, 1006, 27, 14, 0, 0, 1022, 966, 1011, 1002, 0, 0, 0, 0, 1036, 995, 1016, 1019, 3, 1, 0, 0, 990, 1007, 1016, 1050, 0, 0, 0, 0, 1021, 1039, 1094, 1043, 5, 3, 0, 0, 1080, 1063, 1001, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4013, 4137, 4001, 4070, 4063, 4205, 4165] avg=4096.0000 max=4205 max_rank=6 min=4001 min_rank=3 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1035, 983, 1044, 1010, 12, 0, 0, 0, 988, 1058, 1002, 1010, 0, 0, 0, 0, 1012, 1014, 1026, 984, 31, 18, 0, 0, 986, 1011, 1047, 1048, 0, 0, 0, 0, 1030, 1042, 1051, 1020, 9, 5, 0, 0, 1065, 978, 1028, 1051, 0, 0, 0, 0, 1043, 1021, 1044, 1009, 8, 4, 0, 0, 1007, 1033, 1009, 992, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4058, 4085, 4092, 4157, 4122, 4129, 4041] avg=4096.0000 max=4157 max_rank=4 min=4041 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1011, 1036, 1022, 1047, 7, 0, 0, 0, 1061, 1022, 1003, 980, 0, 0, 0, 0, 1024, 1008, 991, 970, 26, 15, 0, 0, 1022, 1013, 1003, 1083, 0, 0, 0, 0, 1015, 1027, 1020, 1078, 3, 1, 0, 0, 1013, 997, 1006, 1056, 0, 0, 0, 0, 1050, 1029, 1000, 1025, 5, 3, 0, 0, 1015, 1056, 1016, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4066, 4034, 4121, 4144, 4072, 4112, 4096] avg=4096.0000 max=4144 max_rank=4 min=4034 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8064, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1070, 1014, 996, 1009, 968, 1024, 995, 1036, 1063, 1069, 998, 994, 998, 945, 999, 1006, 994, 1052, 1065, 1021, 1057, 1088, 1062, 981, 1056, 1031, 1046, 1020, 1026, 1039, 1026, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1042, 1014, 1048, 1005, 1016, 1055, 1026, 1001, 1058, 973, 1054, 1056, 1009, 1021, 1036, 1050, 1025, 1027, 1043, 1017, 1003, 985, 967, 1017, 1025, 1042, 1012, 1007, 1053, 1040, 1039, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[974, 1056, 983, 1007, 1058, 1074, 1010, 997, 997, 1017, 1005, 1039, 1025, 1025, 1007, 1037, 1034, 1047, 1007, 1017, 1083, 1062, 1000, 1071, 1049, 1026, 973, 1025, 1025, 1032, 1038, 968] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1032, 1068, 1026, 1035, 1076, 1025, 1026, 1011, 974, 1007, 1049, 1041, 989, 1038, 1006, 1023, 1052, 999, 1011, 1051, 994, 1050, 1016, 995, 1067, 1005, 1022, 1028, 970, 1015, 1029, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1064, 985, 970, 1030, 1010, 1028, 960, 1016, 1101, 1045, 1059, 979, 1001, 1050, 1034, 1004, 1049, 1074, 993, 1015, 1006, 1028, 1017, 1049, 1022, 1043, 988, 1033, 1026, 1028, 1032, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1007, 1089, 1002, 988, 1061, 1049, 1033, 1004, 1016, 1064, 1020, 992, 1008, 976, 1057, 970, 983, 1025, 1045, 1066, 1001, 1040, 1017, 1071, 1002, 984, 1054, 1064, 1026, 1073, 972, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1092, 1041, 1000, 1005, 998, 1024, 1037, 1059, 1014, 1021, 1016, 1040, 1033, 981, 1060, 1030, 1016, 1028, 1028, 977, 1002, 1026, 1003, 1015, 1018, 996, 1019, 1075, 1040, 1001, 1035, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1013, 1014, 1048, 1011, 1019, 1056, 1042, 994, 1021, 1020, 1082, 1049, 1030, 1018, 1027, 987, 1096, 1061, 1054, 1018, 1032, 979, 992, 985, 1013, 1038, 1026, 1028, 1001, 989, 990, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1032, 1068, 1026, 1035, 2, 1, 0, 0, 1076, 1023, 1026, 1011, 0, 0, 0, 0, 974, 1007, 1030, 1041, 0, 0, 0, 0, 989, 1038, 1006, 1023, 26, 8, 0, 0, 1052, 973, 1011, 1051, 0, 0, 0, 0, 994, 1050, 1016, 995, 7, 1, 0, 0, 1067, 1005, 1022, 1020, 0, 0, 0, 0, 970, 1015, 1029, 1038, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4136, 4052, 4090, 4087, 4063, 4114, 4062] avg=4096.0000 max=4164 max_rank=0 min=4052 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1042, 1014, 1048, 1005, 2, 1, 0, 0, 1016, 1053, 1026, 1001, 0, 0, 0, 0, 1058, 973, 1035, 1056, 0, 0, 0, 0, 1009, 1021, 1036, 1050, 27, 8, 0, 0, 1025, 1000, 1043, 1017, 0, 0, 0, 0, 1003, 985, 967, 1017, 7, 1, 0, 0, 1025, 1042, 1012, 999, 0, 0, 0, 0, 1053, 1040, 1039, 1002, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4096, 4122, 4151, 4085, 3980, 4078, 4144] avg=4096.0000 max=4151 max_rank=3 min=3980 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1070, 1014, 996, 1009, 2, 1, 0, 0, 968, 1022, 995, 1036, 0, 0, 0, 0, 1063, 1069, 980, 994, 0, 0, 0, 0, 998, 945, 999, 1006, 28, 7, 0, 0, 994, 1024, 1065, 1021, 0, 0, 0, 0, 1057, 1088, 1062, 981, 7, 1, 0, 0, 1056, 1031, 1046, 1012, 0, 0, 0, 0, 1026, 1039, 1026, 1020, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4021, 4106, 3983, 4104, 4196, 4145, 4121] avg=4096.0000 max=4196 max_rank=5 min=3983 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[974, 1056, 983, 1007, 2, 1, 0, 0, 1058, 1072, 1010, 997, 0, 0, 0, 0, 997, 1017, 986, 1039, 0, 0, 0, 0, 1025, 1025, 1007, 1037, 27, 8, 0, 0, 1034, 1020, 1007, 1017, 0, 0, 0, 0, 1083, 1062, 1000, 1071, 7, 1, 0, 0, 1049, 1026, 973, 1017, 0, 0, 0, 0, 1025, 1032, 1038, 968, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4137, 4039, 4129, 4078, 4224, 4065, 4073] avg=4096.0000 max=4224 max_rank=5 min=4023 min_rank=0 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1092, 1041, 1000, 1005, 2, 1, 0, 0, 998, 1022, 1037, 1059, 0, 0, 0, 0, 1014, 1021, 997, 1040, 0, 0, 0, 0, 1033, 981, 1060, 1030, 27, 8, 0, 0, 1016, 1001, 1028, 977, 0, 0, 0, 0, 1002, 1026, 1003, 1015, 7, 1, 0, 0, 1018, 996, 1019, 1067, 0, 0, 0, 0, 1040, 1001, 1035, 1038, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4116, 4072, 4139, 4022, 4054, 4100, 4124] avg=4096.0000 max=4141 max_rank=0 min=4022 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1007, 1089, 1002, 988, 2, 1, 0, 0, 1061, 1047, 1033, 1004, 0, 0, 0, 0, 1016, 1064, 1001, 992, 0, 0, 0, 0, 1008, 976, 1057, 970, 27, 8, 0, 0, 983, 998, 1045, 1066, 0, 0, 0, 0, 1001, 1040, 1017, 1071, 7, 1, 0, 0, 1002, 984, 1054, 1056, 0, 0, 0, 0, 1026, 1073, 972, 1009, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4145, 4073, 4046, 4092, 4137, 4096, 4090] avg=4096.0000 max=4145 max_rank=1 min=4046 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1064, 985, 970, 1030, 6, 3, 0, 0, 1010, 1022, 960, 1016, 0, 0, 0, 0, 1101, 1045, 1026, 979, 0, 0, 0, 0, 1001, 1050, 1034, 1004, 32, 11, 0, 0, 1049, 1042, 993, 1015, 0, 0, 0, 0, 1006, 1028, 1017, 1049, 10, 8, 0, 0, 1022, 1043, 988, 1020, 0, 0, 0, 0, 1026, 1028, 1032, 1029, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4008, 4151, 4132, 4099, 4118, 4073, 4129] avg=4096.0000 max=4151 max_rank=2 min=4008 min_rank=1 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1013, 1014, 1048, 1011, 2, 1, 0, 0, 1019, 1054, 1042, 994, 0, 0, 0, 0, 1021, 1020, 1063, 1049, 0, 0, 0, 0, 1030, 1018, 1027, 987, 28, 8, 0, 0, 1096, 1033, 1054, 1018, 0, 0, 0, 0, 1032, 979, 992, 985, 7, 1, 0, 0, 1013, 1038, 1026, 1020, 0, 0, 0, 0, 1001, 989, 990, 1035, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4109, 4153, 4098, 4201, 3996, 4097, 4025] avg=4096.0000 max=4201 max_rank=4 min=3996 min_rank=5 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1013, 1005, 1044, 1062, 1029, 1000, 1060, 1000, 1037, 1027, 968, 1018, 1011, 1024, 1050, 1052, 1011, 1002, 1054, 1015, 1029, 1014, 975, 1032, 1052, 1028, 1040, 1026, 1018, 1073, 981, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1079, 981, 1039, 996, 1036, 1013, 998, 1031, 995, 1001, 1051, 976, 1026, 1061, 1029, 990, 1028, 969, 1020, 1096, 1046, 991, 1062, 1007, 1004, 1051, 1008, 1091, 1008, 984, 1057, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1063, 1020, 996, 993, 1001, 996, 1037, 1062, 1020, 1009, 1034, 997, 1040, 1049, 1035, 1037, 979, 1017, 1067, 992, 997, 1042, 1085, 1046, 1013, 996, 1032, 1008, 1064, 1035, 1043, 963] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1027, 977, 1073, 990, 1044, 995, 1056, 1062, 1021, 1015, 1044, 961, 1059, 1070, 1044, 1016, 1009, 984, 1042, 1030, 969, 1045, 1050, 1008, 1026, 1017, 1007, 1056, 1039, 982, 1021, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1010, 1024, 1011, 1017, 1018, 1001, 1039, 982, 978, 1074, 1059, 980, 1055, 1041, 1065, 1040, 1071, 1001, 992, 1024, 1069, 1014, 990, 1059, 976, 1036, 1044, 1064, 1044, 975, 1002, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1043, 1043, 1019, 989, 1002, 1032, 1030, 1023, 1027, 1045, 1013, 958, 1032, 1049, 1024, 1048, 1041, 1033, 960, 1042, 1052, 1016, 983, 1021, 1044, 994, 1002, 1004, 1058, 1062, 1051, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1045, 1038, 1007, 1025, 1012, 975, 1021, 1017, 1059, 1032, 1019, 1016, 1011, 1005, 981, 1025, 1057, 1084, 1028, 1007, 1019, 1018, 1013, 996, 989, 1022, 1030, 1036, 1015, 1073, 1040, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[996, 1058, 1014, 1056, 1011, 1108, 1030, 1027, 1011, 1037, 1045, 981, 1048, 1021, 1026, 992, 970, 1060, 1043, 1064, 1007, 998, 1016, 982, 1031, 1050, 1044, 980, 1019, 1010, 1019, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1013, 1005, 1044, 1062, 1, 0, 0, 0, 1029, 1000, 1060, 1000, 1, 1, 0, 0, 1037, 1027, 968, 1018, 32, 0, 0, 0, 1011, 989, 1050, 1052, 0, 0, 0, 0, 1011, 1002, 1054, 1013, 0, 0, 0, 0, 1029, 1014, 975, 1032, 8, 3, 2, 0, 1052, 1028, 1040, 1023, 0, 0, 0, 0, 1010, 1073, 981, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4091, 4082, 4102, 4080, 4063, 4143, 4082] avg=4096.0000 max=4143 max_rank=6 min=4063 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1010, 1024, 1011, 1017, 1, 0, 0, 0, 1018, 1001, 1039, 982, 1, 1, 0, 0, 978, 1074, 1059, 980, 32, 0, 0, 0, 1055, 1006, 1065, 1040, 0, 0, 0, 0, 1071, 1001, 992, 1022, 0, 0, 0, 0, 1069, 1014, 990, 1059, 8, 3, 2, 0, 976, 1036, 1044, 1061, 0, 0, 0, 0, 1036, 975, 1002, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4042, 4123, 4166, 4086, 4145, 4117, 4026] avg=4096.0000 max=4166 max_rank=3 min=4026 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1027, 977, 1073, 990, 1, 0, 0, 0, 1044, 995, 1056, 1062, 1, 1, 0, 0, 1021, 1015, 1044, 961, 33, 0, 0, 0, 1059, 1034, 1044, 1016, 0, 0, 0, 0, 1009, 984, 1042, 1028, 0, 0, 0, 0, 969, 1045, 1050, 1008, 8, 3, 2, 0, 1026, 1017, 1007, 1053, 0, 0, 0, 0, 1031, 982, 1021, 1029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4159, 4074, 4153, 4063, 4085, 4103, 4063] avg=4096.0000 max=4159 max_rank=1 min=4063 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1063, 1020, 996, 993, 1, 0, 0, 0, 1001, 996, 1037, 1062, 1, 1, 0, 0, 1020, 1009, 1034, 997, 32, 0, 0, 0, 1040, 1014, 1035, 1037, 0, 0, 0, 0, 979, 1017, 1067, 990, 0, 0, 0, 0, 997, 1042, 1085, 1046, 8, 3, 2, 0, 1013, 996, 1032, 1005, 0, 0, 0, 0, 1056, 1035, 1043, 963, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4098, 4092, 4126, 4053, 4183, 4046, 4097] avg=4096.0000 max=4183 max_rank=5 min=4046 min_rank=6 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1079, 981, 1039, 996, 8, 0, 0, 0, 1036, 1013, 998, 1031, 4, 2, 0, 0, 995, 1001, 1051, 976, 37, 0, 0, 0, 1026, 1017, 1029, 990, 0, 0, 0, 0, 1028, 969, 1020, 1086, 0, 0, 0, 0, 1046, 991, 1062, 1007, 11, 7, 8, 0, 1004, 1051, 1008, 1079, 0, 0, 0, 0, 997, 984, 1057, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4084, 4060, 4062, 4103, 4132, 4142, 4082] avg=4096.0000 max=4142 max_rank=6 min=4060 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1043, 1043, 1019, 989, 1, 0, 0, 0, 1002, 1032, 1030, 1023, 1, 1, 0, 0, 1027, 1045, 1013, 958, 32, 0, 0, 0, 1032, 1014, 1024, 1048, 0, 0, 0, 0, 1041, 1033, 960, 1040, 0, 0, 0, 0, 1052, 1016, 983, 1021, 8, 3, 2, 0, 1044, 994, 1002, 1001, 0, 0, 0, 0, 1050, 1062, 1051, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4089, 4075, 4118, 4074, 4085, 4041, 4191] avg=4096.0000 max=4191 max_rank=7 min=4041 min_rank=6 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1045, 1038, 1007, 1025, 1, 0, 0, 0, 1012, 975, 1021, 1017, 1, 1, 0, 0, 1059, 1032, 1019, 1016, 31, 0, 0, 0, 1011, 971, 981, 1025, 0, 0, 0, 0, 1057, 1084, 1028, 1005, 0, 0, 0, 0, 1019, 1018, 1013, 996, 8, 3, 2, 0, 989, 1022, 1030, 1033, 0, 0, 0, 0, 1007, 1073, 1040, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4027, 4157, 3988, 4174, 4059, 4074, 4173] avg=4096.0000 max=4174 max_rank=4 min=3988 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[996, 1058, 1014, 1056, 1, 0, 0, 0, 1011, 1108, 1030, 1027, 1, 1, 0, 0, 1011, 1037, 1045, 981, 31, 0, 0, 0, 1048, 987, 1026, 992, 0, 0, 0, 0, 970, 1060, 1043, 1062, 0, 0, 0, 0, 1007, 998, 1016, 982, 8, 3, 2, 0, 1031, 1050, 1044, 977, 0, 0, 0, 0, 1011, 1010, 1019, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4178, 4105, 4053, 4135, 4016, 4102, 4054] avg=4096.0000 max=4178 max_rank=1 min=4016 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 7936, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1028, 1046, 1056, 1031, 1046, 1020, 992, 1003, 1040, 976, 1003, 1023, 996, 1046, 1062, 1012, 1034, 1000, 1025, 1022, 1047, 985, 1006, 1039, 1036, 993, 1046, 1035, 1005, 1031, 1030, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1055, 1004, 1046, 1039, 1034, 983, 968, 1033, 997, 997, 1024, 1035, 982, 1059, 1071, 1058, 1034, 1035, 1023, 1032, 994, 1012, 1043, 1013, 1075, 1012, 1020, 1026, 993, 1041, 1049, 981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1027, 1001, 1019, 986, 1042, 1022, 1025, 1000, 1069, 1078, 1044, 1065, 1014, 980, 982, 996, 994, 1040, 1078, 1060, 954, 1025, 1028, 977, 1035, 1038, 1019, 1035, 1027, 1045, 977, 1086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[998, 1017, 1021, 1016, 1042, 1024, 1023, 1061, 1076, 1011, 1011, 1032, 1025, 1019, 1017, 1042, 1027, 977, 1055, 1041, 1007, 979, 1035, 978, 1017, 1008, 1059, 1041, 1020, 1037, 1031, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[983, 1054, 975, 1054, 1032, 1027, 1025, 1043, 984, 991, 1041, 1042, 1022, 1063, 1071, 1057, 1001, 1025, 1031, 1034, 1023, 1036, 1032, 1034, 1033, 1013, 987, 1009, 1008, 1040, 1018, 980] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1007, 1066, 1033, 1007, 1036, 1042, 984, 976, 1044, 1026, 1012, 976, 1029, 1046, 1017, 1051, 998, 1015, 1039, 956, 1070, 1057, 1067, 988, 1025, 990, 1015, 1037, 1045, 1045, 1020, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[996, 1021, 1052, 1012, 993, 1007, 1035, 1029, 1079, 1024, 1023, 1042, 995, 1000, 1007, 1030, 1047, 1024, 1052, 1074, 963, 1033, 1028, 1054, 999, 1003, 1032, 1042, 1035, 1000, 1015, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1055, 963, 1013, 1030, 1051, 1018, 985, 992, 1010, 1003, 1079, 1036, 1018, 1032, 1040, 1015, 1040, 1016, 1020, 1041, 1023, 1032, 1017, 1061, 1048, 1044, 1012, 984, 1023, 1001, 1027, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1055, 1004, 1046, 1039, 3, 3, 0, 0, 1034, 983, 968, 1033, 15, 6, 0, 0, 982, 997, 1024, 1035, 0, 0, 0, 0, 982, 1059, 1061, 1058, 0, 0, 0, 0, 1034, 1035, 1009, 1032, 0, 0, 0, 0, 994, 1012, 1043, 1013, 8, 7, 0, 0, 1075, 1012, 1020, 1026, 0, 0, 0, 0, 993, 1038, 1049, 981, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4039, 4038, 4160, 4110, 4077, 4133, 4061] avg=4096.0000 max=4160 max_rank=3 min=4038 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1028, 1046, 1056, 1031, 3, 3, 0, 0, 1046, 1020, 992, 1003, 15, 6, 0, 0, 1025, 976, 1003, 1023, 0, 0, 0, 0, 996, 1046, 1052, 1012, 0, 0, 0, 0, 1034, 1000, 1011, 1022, 0, 0, 0, 0, 1047, 985, 1006, 1039, 8, 7, 0, 0, 1036, 993, 1046, 1035, 0, 0, 0, 0, 1005, 1028, 1030, 1054, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4082, 4027, 4106, 4067, 4092, 4110, 4117] avg=4096.0000 max=4167 max_rank=0 min=4027 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16386 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[983, 1054, 975, 1054, 3, 3, 0, 0, 1032, 1027, 1025, 1043, 14, 6, 0, 0, 970, 991, 1041, 1042, 0, 0, 0, 0, 1022, 1063, 1061, 1057, 0, 0, 0, 0, 1001, 1025, 1017, 1034, 0, 0, 0, 0, 1023, 1036, 1032, 1034, 8, 7, 0, 0, 1033, 1013, 987, 1009, 0, 0, 0, 0, 1008, 1037, 1018, 980, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4147, 4044, 4203, 4077, 4140, 4042, 4043] avg=4096.0000 max=4203 max_rank=3 min=4042 min_rank=6 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16302 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1027, 1001, 1019, 986, 3, 3, 0, 0, 1042, 1022, 1025, 1000, 16, 6, 0, 0, 1053, 1078, 1044, 1065, 0, 0, 0, 0, 1014, 980, 973, 996, 0, 0, 0, 0, 994, 1040, 1063, 1060, 0, 0, 0, 0, 954, 1025, 1028, 977, 9, 6, 0, 0, 1035, 1038, 1019, 1035, 0, 0, 0, 0, 1027, 1042, 977, 1086, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4111, 4240, 3963, 4157, 3999, 4127, 4132] avg=4096.0000 max=4240 max_rank=2 min=3963 min_rank=3 imbalance_ratio=1.035156 max_load_reduction=28528 max_load_reduction_pct=0.870605 moved_to_echo=16415 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[998, 1017, 1021, 1016, 3, 3, 0, 0, 1042, 1024, 1023, 1061, 16, 6, 0, 0, 1060, 1011, 1011, 1032, 0, 0, 0, 0, 1025, 1019, 1008, 1042, 0, 0, 0, 0, 1027, 977, 1040, 1041, 0, 0, 0, 0, 1007, 979, 1035, 978, 9, 6, 0, 0, 1017, 1008, 1059, 1041, 0, 0, 0, 0, 1020, 1034, 1031, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4172, 4114, 4094, 4085, 4014, 4125, 4106] avg=4096.0000 max=4172 max_rank=1 min=4014 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16330 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[996, 1021, 1052, 1012, 3, 3, 0, 0, 993, 1007, 1035, 1029, 16, 6, 0, 0, 1063, 1024, 1023, 1042, 0, 0, 0, 0, 995, 1000, 998, 1030, 0, 0, 0, 0, 1047, 1024, 1037, 1074, 0, 0, 0, 0, 963, 1033, 1028, 1054, 9, 6, 0, 0, 999, 1003, 1032, 1042, 0, 0, 0, 0, 1035, 997, 1015, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4086, 4152, 4023, 4182, 4093, 4076, 4069] avg=4096.0000 max=4182 max_rank=4 min=4023 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16420 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1007, 1066, 1033, 1007, 9, 6, 0, 0, 1036, 1042, 984, 976, 18, 8, 0, 0, 1026, 1026, 1012, 976, 0, 0, 0, 0, 1029, 1046, 998, 1051, 0, 0, 0, 0, 998, 1015, 1018, 956, 0, 0, 0, 0, 1070, 1057, 1067, 988, 13, 10, 0, 0, 1025, 990, 1015, 1037, 0, 0, 0, 0, 1045, 1039, 1020, 1049, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4064, 4040, 4124, 3987, 4205, 4067, 4153] avg=4096.0000 max=4205 max_rank=5 min=3987 min_rank=4 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16412 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1055, 963, 1013, 1030, 3, 3, 0, 0, 1051, 1018, 985, 992, 15, 6, 0, 0, 995, 1003, 1079, 1036, 0, 0, 0, 0, 1018, 1032, 1030, 1015, 0, 0, 0, 0, 1040, 1016, 1006, 1041, 0, 0, 0, 0, 1023, 1032, 1017, 1061, 8, 7, 0, 0, 1048, 1044, 1012, 984, 0, 0, 0, 0, 1023, 998, 1027, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4067, 4113, 4095, 4103, 4148, 4088, 4087] avg=4096.0000 max=4148 max_rank=5 min=4067 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16426 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8064, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1064, 1025, 1037, 1062, 1056, 984, 1050, 992, 1029, 1004, 1046, 1016, 1044, 1017, 1077, 984, 1013, 1016, 1047, 1037, 1020, 991, 1004, 1016, 998, 1006, 1051, 1053, 1006, 1018, 996, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1028, 1020, 1013, 976, 936, 1038, 1062, 1040, 1029, 1018, 1010, 1030, 968, 1010, 1039, 1037, 1014, 1031, 1063, 1010, 1078, 1044, 1039, 1019, 1041, 1011, 997, 1018, 1012, 1034, 1069, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1061, 963, 1065, 1026, 992, 1067, 1025, 1062, 1062, 1018, 1020, 1009, 1020, 980, 1034, 1014, 993, 1007, 1018, 989, 1007, 1021, 976, 1074, 982, 1070, 1063, 1066, 1016, 1044, 1001, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[992, 1055, 1016, 998, 1070, 1040, 1043, 1069, 1063, 1004, 1015, 1073, 1006, 1055, 1014, 1058, 970, 949, 1018, 1015, 1039, 1043, 978, 1029, 1043, 1052, 1001, 974, 1032, 1045, 995, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1045, 996, 1011, 996, 970, 1024, 1034, 1034, 1010, 1014, 992, 1058, 1040, 1018, 1036, 1013, 978, 1023, 1034, 1032, 1057, 1094, 1063, 991, 1028, 1001, 1051, 1008, 991, 1046, 1006, 1074] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1005, 1022, 1035, 1013, 1050, 1037, 1058, 1032, 964, 995, 1033, 980, 1023, 1035, 1034, 985, 1039, 1001, 1004, 1057, 1057, 1015, 1033, 1052, 1021, 1009, 1053, 1012, 1044, 1005, 1056, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1034, 998, 1002, 963, 1004, 981, 1029, 1091, 1031, 1037, 1073, 1030, 1045, 1025, 987, 1066, 1015, 1049, 1053, 1017, 1019, 992, 990, 1028, 1038, 972, 1095, 1019, 997, 1022, 1031, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1015, 984, 1005, 1009, 999, 1083, 975, 1082, 1000, 1028, 1042, 1012, 1025, 1022, 1016, 1007, 1006, 1041, 1017, 1014, 1060, 1055, 1018, 1009, 1009, 1020, 1016, 1024, 1055, 1076, 1022, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1061, 963, 1065, 1026, 29, 0, 0, 0, 992, 1067, 1025, 1033, 0, 0, 0, 0, 1062, 1018, 1020, 1009, 2, 0, 0, 0, 1020, 980, 1034, 1014, 2, 1, 0, 0, 993, 1007, 1018, 989, 17, 6, 0, 0, 990, 1021, 976, 1074, 0, 0, 0, 0, 982, 1070, 1060, 1066, 0, 0, 0, 0, 1016, 1036, 1001, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4117, 4111, 4051, 4030, 4061, 4178, 4076] avg=4096.0000 max=4178 max_rank=6 min=4030 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[992, 1055, 1016, 998, 29, 0, 0, 0, 1070, 1040, 1043, 1040, 0, 0, 0, 0, 1063, 1004, 1015, 1073, 2, 0, 0, 0, 1006, 1055, 1014, 1058, 2, 1, 0, 0, 970, 949, 1018, 1015, 17, 6, 0, 0, 1022, 1043, 978, 1029, 0, 0, 0, 0, 1043, 1052, 998, 974, 0, 0, 0, 0, 1032, 1037, 995, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4193, 4157, 4136, 3975, 4072, 4067, 4078] avg=4096.0000 max=4193 max_rank=1 min=3975 min_rank=4 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16192 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1028, 1020, 1013, 976, 28, 0, 0, 0, 936, 1038, 1062, 1012, 0, 0, 0, 0, 1029, 1018, 1010, 1030, 2, 0, 0, 0, 968, 1010, 1039, 1037, 2, 1, 0, 0, 1014, 1031, 1063, 1010, 18, 5, 0, 0, 1060, 1044, 1039, 1019, 0, 0, 0, 0, 1041, 1011, 994, 1018, 0, 0, 0, 0, 1012, 1027, 1069, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4048, 4089, 4057, 4141, 4162, 4064, 4142] avg=4096.0000 max=4162 max_rank=5 min=4048 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1064, 1025, 1037, 1062, 27, 0, 0, 0, 1056, 984, 1050, 965, 0, 0, 0, 0, 1029, 1004, 1046, 1016, 2, 0, 0, 0, 1044, 1017, 1077, 984, 2, 1, 0, 0, 1013, 1016, 1047, 1037, 17, 5, 0, 0, 1003, 991, 1004, 1016, 0, 0, 0, 0, 998, 1006, 1048, 1053, 0, 0, 0, 0, 1006, 1011, 996, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4215, 4055, 4097, 4125, 4135, 4014, 4105, 4022] avg=4096.0000 max=4215 max_rank=0 min=4014 min_rank=5 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16276 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1045, 996, 1011, 996, 28, 0, 0, 0, 970, 1024, 1034, 1006, 0, 0, 0, 0, 1010, 1014, 992, 1058, 2, 0, 0, 0, 1040, 1018, 1036, 1013, 2, 1, 0, 0, 978, 1023, 1034, 1032, 18, 6, 0, 0, 1039, 1094, 1063, 991, 0, 0, 0, 0, 1028, 1001, 1048, 1008, 0, 0, 0, 0, 991, 1038, 1006, 1074, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4034, 4076, 4110, 4091, 4187, 4085, 4109] avg=4096.0000 max=4187 max_rank=5 min=4034 min_rank=1 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1005, 1022, 1035, 1013, 28, 0, 0, 0, 1050, 1037, 1058, 1004, 0, 0, 0, 0, 964, 995, 1033, 980, 2, 0, 0, 0, 1023, 1035, 1034, 985, 2, 1, 0, 0, 1039, 1001, 1004, 1057, 18, 5, 0, 0, 1039, 1015, 1033, 1052, 0, 0, 0, 0, 1021, 1009, 1050, 1012, 0, 0, 0, 0, 1044, 998, 1056, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4149, 3974, 4080, 4124, 4139, 4092, 4107] avg=4096.0000 max=4149 max_rank=1 min=3974 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1034, 998, 1002, 963, 35, 0, 0, 0, 1004, 981, 1029, 1049, 0, 0, 0, 0, 1031, 1037, 1073, 1030, 9, 0, 0, 0, 1045, 1025, 987, 1066, 9, 4, 0, 0, 1015, 1049, 1053, 1017, 20, 9, 7, 0, 999, 992, 990, 1028, 0, 0, 0, 0, 1038, 972, 1082, 1019, 0, 0, 0, 0, 997, 1004, 1031, 1035, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4063, 4180, 4136, 4170, 4009, 4111, 4067] avg=4096.0000 max=4180 max_rank=2 min=4009 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1015, 984, 1005, 1009, 30, 0, 0, 0, 999, 1083, 975, 1052, 0, 0, 0, 0, 1000, 1028, 1042, 1012, 2, 0, 0, 0, 1025, 1022, 1016, 1007, 2, 1, 0, 0, 1006, 1041, 1017, 1014, 18, 6, 0, 0, 1042, 1055, 1018, 1009, 0, 0, 0, 0, 1009, 1020, 1013, 1024, 0, 0, 0, 0, 1055, 1068, 1022, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4109, 4084, 4073, 4102, 4124, 4066, 4167] avg=4096.0000 max=4167 max_rank=7 min=4043 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8064, 8192, 8064, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True + [2026-08-30 04:58:07] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 616.3 | throughput per GPU (TFLOP/s/GPU): 540.9 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.486655E+00 | loss scale: 1.0 | grad norm: 5.612 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1037, 996, 1053, 1035, 1018, 1037, 1028, 1031, 1024, 1009, 979, 1038, 990, 1047, 1025, 1029, 1045, 1024, 1013, 1018, 1046, 1060, 1004, 1014, 1068, 944, 1079, 1020, 1031, 1040, 977, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1020, 1031, 980, 993, 1003, 1048, 1084, 1018, 1007, 1036, 1016, 1082, 1012, 1000, 1053, 1033, 1047, 1020, 1033, 1038, 971, 1089, 1030, 977, 1014, 1005, 1052, 1018, 1020, 999, 1049, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1037, 1028, 1029, 1038, 1016, 1067, 1006, 1024, 1022, 1060, 1003, 1004, 1035, 1100, 977, 1051, 1043, 1033, 1010, 1042, 966, 1056, 1059, 978, 1049, 1008, 951, 992, 1050, 1026, 1000, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1002, 1059, 1040, 1050, 1017, 1024, 1038, 1025, 1061, 1009, 1025, 1038, 1041, 1021, 1020, 1020, 1007, 1034, 966, 1045, 1025, 1060, 1022, 1003, 1004, 990, 1007, 1007, 1030, 1036, 999, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[992, 1029, 942, 1003, 1007, 1054, 1069, 1047, 1037, 1067, 994, 1056, 1041, 1034, 997, 982, 1018, 1059, 1028, 1050, 1059, 1005, 1009, 1042, 970, 986, 1048, 1047, 1084, 952, 1069, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1061, 1038, 1008, 1017, 989, 1025, 993, 1025, 1011, 1045, 1048, 1049, 1046, 986, 1066, 1043, 1013, 975, 1033, 1041, 1028, 1016, 1000, 1032, 1044, 1048, 982, 988, 1023, 1021, 1051, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1013, 986, 984, 1030, 1015, 1025, 1059, 1052, 964, 1068, 1009, 1048, 981, 984, 1025, 1005, 1058, 1019, 1015, 1022, 1033, 1030, 1034, 1013, 1021, 1054, 1018, 1082, 1043, 1005, 1017, 1056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1040, 1030, 1048, 1052, 991, 1015, 1086, 983, 968, 1000, 1042, 1043, 1027, 1021, 1070, 1047, 999, 1023, 1072, 1032, 1051, 1009, 1066, 977, 1027, 1041, 1003, 977, 1036, 1008, 1012, 972] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1002, 1059, 1040, 1050, 4, 3, 0, 0, 1017, 1024, 1020, 1025, 0, 0, 0, 0, 1061, 1009, 1025, 1028, 0, 0, 0, 0, 1041, 1021, 1016, 1020, 0, 0, 0, 0, 1007, 1034, 966, 1032, 0, 0, 0, 0, 1025, 1060, 1022, 1003, 0, 0, 0, 0, 1004, 990, 1007, 1007, 18, 9, 0, 0, 1030, 1036, 999, 1043, 7, 4, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4086, 4123, 4098, 4039, 4110, 4035, 4119] avg=4096.0000 max=4158 max_rank=0 min=4035 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1037, 1028, 1029, 1038, 4, 3, 0, 0, 1016, 1067, 988, 1024, 0, 0, 0, 0, 1022, 1060, 1003, 994, 0, 0, 0, 0, 1035, 1100, 973, 1051, 0, 0, 0, 0, 1043, 1033, 1010, 1029, 0, 0, 0, 0, 966, 1056, 1059, 978, 0, 0, 0, 0, 1049, 1008, 951, 992, 18, 9, 0, 0, 1050, 1026, 1000, 1008, 7, 4, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4095, 4079, 4159, 4115, 4059, 4027, 4095] avg=4096.0000 max=4159 max_rank=3 min=4027 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1037, 996, 1053, 1035, 4, 3, 0, 0, 1018, 1037, 1010, 1031, 0, 0, 0, 0, 1024, 1009, 979, 1028, 0, 0, 0, 0, 990, 1047, 1021, 1029, 0, 0, 0, 0, 1045, 1024, 1013, 1006, 0, 0, 0, 0, 1046, 1060, 1004, 1014, 0, 0, 0, 0, 1068, 944, 1079, 1020, 18, 8, 0, 0, 1031, 1040, 977, 1009, 7, 4, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4096, 4040, 4087, 4088, 4124, 4137, 4068] avg=4096.0000 max=4137 max_rank=6 min=4040 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1020, 1031, 980, 993, 4, 3, 0, 0, 1003, 1048, 1065, 1018, 0, 0, 0, 0, 1007, 1036, 1016, 1071, 0, 0, 0, 0, 1012, 1000, 1049, 1033, 0, 0, 0, 0, 1047, 1020, 1033, 1025, 0, 0, 0, 0, 971, 1089, 1030, 977, 0, 0, 0, 0, 1014, 1005, 1052, 1018, 19, 9, 0, 0, 1020, 999, 1049, 990, 8, 4, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4134, 4130, 4094, 4125, 4067, 4117, 4070] avg=4096.0000 max=4134 max_rank=1 min=4031 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1061, 1038, 1008, 1017, 4, 3, 0, 0, 989, 1025, 976, 1025, 0, 0, 0, 0, 1011, 1045, 1048, 1038, 0, 0, 0, 0, 1046, 986, 1062, 1043, 0, 0, 0, 0, 1013, 975, 1033, 1028, 0, 0, 0, 0, 1028, 1016, 1000, 1032, 0, 0, 0, 0, 1044, 1048, 982, 988, 17, 9, 0, 0, 1023, 1021, 1051, 1023, 8, 4, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4015, 4142, 4137, 4049, 4076, 4088, 4130] avg=4096.0000 max=4142 max_rank=2 min=4015 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[992, 1029, 942, 1003, 4, 3, 0, 0, 1007, 1054, 1050, 1047, 0, 0, 0, 0, 1037, 1067, 994, 1045, 0, 0, 0, 0, 1041, 1034, 993, 982, 0, 0, 0, 0, 1018, 1059, 1028, 1037, 0, 0, 0, 0, 1059, 1005, 1009, 1042, 0, 0, 0, 0, 970, 986, 1048, 1047, 19, 9, 0, 0, 1084, 952, 1069, 991, 8, 4, 0, 0] rerouted_tokens_per_ep_rank=[3973, 4158, 4143, 4050, 4142, 4115, 4079, 4108] avg=4096.0000 max=4158 max_rank=1 min=3973 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1013, 986, 984, 1030, 9, 9, 0, 0, 1015, 1025, 1036, 1052, 0, 0, 0, 0, 964, 1068, 1009, 1027, 0, 0, 0, 0, 981, 984, 1012, 1005, 0, 0, 0, 0, 1058, 1019, 1015, 1005, 0, 0, 0, 0, 1033, 1030, 1034, 1013, 4, 0, 0, 0, 1021, 1054, 1018, 1082, 23, 11, 0, 0, 1043, 1005, 1017, 1056, 12, 6, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4128, 4068, 3982, 4097, 4114, 4209, 4139] avg=4096.0000 max=4209 max_rank=6 min=3982 min_rank=3 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1040, 1030, 1048, 1052, 4, 3, 0, 0, 991, 1015, 1067, 983, 0, 0, 0, 0, 968, 1000, 1042, 1033, 0, 0, 0, 0, 1027, 1021, 1066, 1047, 0, 0, 0, 0, 999, 1023, 1072, 1019, 0, 0, 0, 0, 1051, 1009, 1066, 977, 0, 0, 0, 0, 1027, 1041, 1003, 977, 19, 9, 0, 0, 1036, 1008, 1012, 972, 7, 4, 0, 0] rerouted_tokens_per_ep_rank=[4177, 4056, 4043, 4161, 4113, 4103, 4076, 4039] avg=4096.0000 max=4177 max_rank=0 min=4039 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8448, 8320, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1028, 1013, 1022, 1005, 1005, 976, 1008, 1039, 1039, 1040, 1003, 981, 1083, 1016, 1037, 1035, 1066, 1003, 1013, 1037, 1001, 1014, 1039, 1054, 1080, 1034, 983, 1078, 994, 1041, 992, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1032, 1039, 1016, 1029, 1035, 1047, 1035, 1049, 1015, 1055, 1040, 1045, 1001, 1072, 1027, 996, 1028, 966, 1023, 958, 1026, 1006, 1028, 1055, 1059, 1031, 1025, 999, 1018, 1054, 1004, 955] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1041, 1066, 997, 989, 986, 1078, 1075, 1014, 1031, 985, 1052, 1033, 1031, 1025, 1011, 1005, 1019, 990, 1034, 1010, 1052, 1067, 1026, 1032, 1015, 1051, 968, 1050, 1025, 1030, 976, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[982, 1058, 1009, 1025, 1014, 1027, 1082, 1054, 1052, 1008, 1018, 1052, 1017, 1037, 998, 1047, 955, 1031, 1060, 1020, 1020, 1030, 981, 1027, 998, 1031, 1023, 1015, 1017, 1024, 1026, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1019, 1010, 1047, 1011, 1040, 1040, 1029, 1015, 1018, 994, 1002, 1032, 1023, 1053, 1032, 1032, 1006, 1024, 1028, 1011, 1055, 1021, 1068, 1019, 1034, 1012, 973, 1023, 1066, 1016, 1016, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1036, 1007, 1052, 988, 1010, 1025, 1032, 963, 1019, 1032, 981, 1048, 1001, 1081, 1057, 1012, 1015, 1008, 1047, 1002, 1023, 1024, 1016, 1015, 1024, 1040, 1052, 1029, 1032, 1085, 1037, 975] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1014, 1066, 992, 1039, 1017, 1010, 1004, 1001, 1022, 1037, 1054, 1054, 1038, 1009, 1008, 1020, 983, 1005, 1086, 1042, 1041, 958, 1048, 1039, 997, 1041, 1003, 1062, 1036, 991, 1024, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[993, 1023, 1034, 1024, 1037, 1059, 1048, 1070, 1047, 994, 973, 1023, 1029, 1002, 999, 1057, 1021, 1015, 1035, 1036, 977, 1022, 1051, 1020, 994, 1059, 1032, 1039, 1026, 1032, 1006, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1032, 1039, 1016, 1029, 6, 1, 0, 0, 1035, 1047, 1016, 1049, 0, 0, 0, 0, 1015, 1055, 1040, 1044, 0, 0, 0, 0, 1001, 1058, 1027, 996, 0, 0, 0, 0, 1028, 966, 1023, 958, 11, 8, 4, 0, 1026, 1006, 1028, 1044, 0, 0, 0, 0, 1059, 1021, 1025, 999, 0, 0, 0, 0, 1018, 1054, 1004, 955, 19, 6, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4147, 4154, 4082, 3998, 4104, 4104, 4056] avg=4096.0000 max=4154 max_rank=2 min=3998 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16262 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1028, 1013, 1022, 1005, 6, 1, 0, 0, 1005, 976, 990, 1039, 0, 0, 0, 0, 1039, 1040, 1003, 980, 0, 0, 0, 0, 1083, 1002, 1037, 1035, 0, 0, 0, 0, 1066, 1003, 1013, 1037, 11, 8, 4, 0, 1001, 1014, 1039, 1043, 0, 0, 0, 0, 1080, 1024, 983, 1078, 0, 0, 0, 0, 994, 1041, 992, 1009, 18, 6, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4010, 4062, 4157, 4142, 4097, 4165, 4060] avg=4096.0000 max=4165 max_rank=6 min=4010 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[982, 1058, 1009, 1025, 6, 1, 0, 0, 1014, 1027, 1062, 1054, 0, 0, 0, 0, 1052, 1008, 1018, 1051, 0, 0, 0, 0, 1017, 1023, 998, 1047, 0, 0, 0, 0, 955, 1031, 1060, 1020, 10, 8, 4, 0, 1020, 1030, 981, 1017, 0, 0, 0, 0, 998, 1021, 1023, 1015, 0, 0, 0, 0, 1017, 1024, 1026, 1030, 20, 6, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4157, 4129, 4085, 4088, 4048, 4057, 4123] avg=4096.0000 max=4157 max_rank=1 min=4048 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1014, 1066, 992, 1039, 6, 1, 0, 0, 1017, 1010, 986, 1001, 0, 0, 0, 0, 1022, 1037, 1054, 1053, 0, 0, 0, 0, 1038, 995, 1008, 1020, 0, 0, 0, 0, 983, 1005, 1086, 1042, 10, 8, 4, 0, 1041, 958, 1048, 1029, 0, 0, 0, 0, 997, 1031, 1003, 1062, 0, 0, 0, 0, 1036, 991, 1024, 1027, 18, 6, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4014, 4166, 4061, 4138, 4076, 4093, 4102] avg=4096.0000 max=4166 max_rank=2 min=4014 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[993, 1023, 1034, 1024, 6, 1, 0, 0, 1037, 1059, 1029, 1070, 0, 0, 0, 0, 1047, 994, 973, 1022, 0, 0, 0, 0, 1029, 988, 999, 1057, 0, 0, 0, 0, 1021, 1015, 1035, 1036, 10, 8, 4, 0, 977, 1022, 1051, 1010, 0, 0, 0, 0, 994, 1049, 1032, 1039, 0, 0, 0, 0, 1026, 1032, 1006, 991, 19, 6, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4195, 4036, 4073, 4129, 4060, 4114, 4080] avg=4096.0000 max=4195 max_rank=1 min=4036 min_rank=2 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1041, 1066, 997, 989, 6, 1, 0, 0, 986, 1078, 1055, 1014, 0, 0, 0, 0, 1031, 985, 1052, 1032, 0, 0, 0, 0, 1031, 1011, 1011, 1005, 0, 0, 0, 0, 1019, 990, 1034, 1010, 10, 8, 4, 0, 1052, 1067, 1026, 1022, 0, 0, 0, 0, 1015, 1041, 968, 1050, 0, 0, 0, 0, 1025, 1030, 976, 1004, 20, 6, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4133, 4100, 4058, 4075, 4167, 4074, 4061] avg=4096.0000 max=4167 max_rank=5 min=4058 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1019, 1010, 1047, 1011, 6, 1, 0, 0, 1040, 1040, 1010, 1015, 0, 0, 0, 0, 1018, 994, 1002, 1031, 0, 0, 0, 0, 1023, 1039, 1032, 1032, 0, 0, 0, 0, 1006, 1024, 1028, 1011, 10, 8, 4, 0, 1055, 1021, 1068, 1009, 0, 0, 0, 0, 1034, 1002, 973, 1023, 0, 0, 0, 0, 1066, 1016, 1016, 999, 19, 6, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4105, 4045, 4126, 4091, 4153, 4032, 4122] avg=4096.0000 max=4153 max_rank=5 min=4032 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1036, 1007, 1052, 988, 9, 4, 0, 0, 1010, 1025, 1009, 963, 0, 0, 0, 0, 1019, 1032, 981, 1044, 0, 0, 0, 0, 1001, 1056, 1057, 1012, 0, 0, 0, 0, 1015, 1008, 1047, 1002, 15, 13, 7, 0, 1023, 1024, 1016, 1000, 0, 0, 0, 0, 1024, 1024, 1052, 1029, 0, 0, 0, 0, 1032, 1085, 1037, 975, 23, 12, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4007, 4076, 4126, 4107, 4063, 4129, 4164] avg=4096.0000 max=4164 max_rank=7 min=4007 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8064, 8448, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1053, 1078, 1011, 1040, 992, 997, 971, 1052, 1108, 1011, 1053, 1072, 1010, 1044, 1033, 1062, 991, 1020, 987, 1017, 991, 993, 1030, 970, 1025, 1020, 1002, 1023, 997, 1016, 1064, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1036, 1029, 1060, 965, 1013, 1017, 1052, 1048, 1019, 1020, 1050, 1022, 1020, 1040, 1058, 1036, 990, 1030, 1041, 1021, 1026, 980, 995, 1004, 1023, 1020, 1043, 1018, 1046, 1006, 1076, 964] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1045, 1001, 1024, 1019, 1009, 992, 1052, 1071, 1007, 1028, 1011, 1009, 1019, 995, 1042, 1036, 1066, 1034, 1016, 1083, 1040, 1008, 1018, 1045, 1004, 1034, 1006, 1017, 1008, 1022, 1003, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1029, 1059, 1020, 1066, 989, 1000, 1025, 998, 1032, 1021, 1055, 1056, 1030, 970, 1014, 1020, 948, 1000, 1015, 1014, 1022, 1059, 1024, 995, 1073, 1031, 1040, 1060, 1007, 1021, 1048, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1048, 971, 1042, 1034, 1053, 971, 1025, 983, 1035, 993, 1039, 1011, 1051, 1052, 1005, 1030, 1074, 1016, 1003, 1033, 1000, 1012, 1030, 1032, 1043, 1024, 1011, 1014, 1044, 1059, 1055, 975] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[970, 1015, 1032, 1008, 1047, 1022, 1001, 1039, 1005, 1035, 1038, 1027, 1042, 1076, 981, 1017, 1031, 1073, 1036, 1016, 1043, 1056, 1042, 969, 1080, 1021, 981, 1001, 1040, 998, 996, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1045, 982, 998, 1042, 1000, 1052, 1046, 1021, 989, 1041, 1045, 1029, 998, 1040, 1052, 1047, 1021, 1058, 1026, 1016, 1047, 1001, 1028, 982, 990, 1047, 990, 986, 1010, 1041, 1025, 1073] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1075, 1058, 1010, 1031, 1076, 1072, 1075, 1041, 1021, 1053, 1033, 1028, 1049, 1051, 987, 1019, 952, 976, 1021, 996, 1012, 998, 979, 1054, 1031, 992, 1027, 1053, 1013, 1012, 983, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1038, 1078, 1011, 1040, 0, 0, 0, 0, 992, 997, 971, 1048, 0, 0, 0, 0, 1108, 1011, 1025, 1072, 0, 0, 0, 0, 1010, 1025, 1033, 1062, 0, 0, 0, 0, 991, 1020, 987, 1017, 13, 5, 0, 0, 991, 993, 1030, 970, 28, 6, 0, 0, 1025, 1020, 1002, 1023, 4, 0, 0, 0, 997, 1016, 1064, 1035, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4008, 4216, 4130, 4033, 4018, 4074, 4122] avg=4096.0000 max=4216 max_rank=2 min=4008 min_rank=1 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16247 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1022, 1029, 1060, 965, 0, 0, 0, 0, 1013, 1017, 1052, 1044, 0, 0, 0, 0, 1019, 1020, 1022, 1022, 0, 0, 0, 0, 1020, 1022, 1058, 1036, 0, 0, 0, 0, 990, 1030, 1041, 1021, 12, 5, 0, 0, 1026, 980, 995, 1004, 28, 6, 0, 0, 1023, 1020, 1043, 1018, 4, 0, 0, 0, 1046, 1006, 1076, 964, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4126, 4083, 4136, 4099, 4039, 4108, 4101] avg=4096.0000 max=4136 max_rank=3 min=4039 min_rank=5 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1033, 971, 1042, 1034, 0, 0, 0, 0, 1053, 971, 1025, 979, 0, 0, 0, 0, 1035, 993, 1011, 1011, 0, 0, 0, 0, 1051, 1033, 1005, 1030, 0, 0, 0, 0, 1074, 1016, 1003, 1033, 13, 5, 0, 0, 1000, 1012, 1030, 1032, 28, 6, 0, 0, 1043, 1024, 1011, 1014, 4, 0, 0, 0, 1044, 1059, 1055, 975, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4028, 4050, 4119, 4144, 4108, 4096, 4143] avg=4096.0000 max=4144 max_rank=4 min=4028 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1015, 1059, 1020, 1066, 0, 0, 0, 0, 989, 1000, 1025, 994, 0, 0, 0, 0, 1032, 1021, 1027, 1056, 0, 0, 0, 0, 1030, 952, 1014, 1020, 0, 0, 0, 0, 948, 1000, 1015, 1014, 12, 5, 0, 0, 1022, 1059, 1024, 995, 28, 6, 0, 0, 1073, 1031, 1040, 1060, 4, 0, 0, 0, 1007, 1021, 1048, 1027, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4008, 4136, 4016, 3994, 4134, 4208, 4112] avg=4096.0000 max=4208 max_rank=6 min=3994 min_rank=4 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[956, 1015, 1032, 1008, 0, 0, 0, 0, 1047, 1022, 1001, 1035, 0, 0, 0, 0, 1005, 1035, 1010, 1027, 0, 0, 0, 0, 1042, 1056, 981, 1017, 0, 0, 0, 0, 1031, 1073, 1036, 1016, 13, 5, 0, 0, 1043, 1056, 1042, 969, 28, 7, 0, 0, 1080, 1021, 981, 1001, 4, 0, 0, 0, 1040, 998, 996, 1030, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4105, 4077, 4096, 4174, 4145, 4087, 4073] avg=4096.0000 max=4174 max_rank=4 min=4011 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1030, 1001, 1024, 1019, 0, 0, 0, 0, 1009, 992, 1052, 1067, 0, 0, 0, 0, 1007, 1028, 984, 1009, 0, 0, 0, 0, 1019, 977, 1042, 1036, 0, 0, 0, 0, 1066, 1034, 1016, 1083, 12, 5, 0, 0, 1040, 1008, 1018, 1045, 27, 6, 0, 0, 1004, 1034, 1006, 1017, 4, 0, 0, 0, 1008, 1022, 1003, 1004, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4120, 4028, 4074, 4216, 4144, 4065, 4047] avg=4096.0000 max=4216 max_rank=4 min=4028 min_rank=2 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1060, 1058, 1010, 1031, 0, 0, 0, 0, 1076, 1072, 1075, 1037, 0, 0, 0, 0, 1021, 1053, 1005, 1028, 0, 0, 0, 0, 1049, 1032, 987, 1019, 0, 0, 0, 0, 952, 976, 1021, 996, 13, 5, 0, 0, 1012, 998, 979, 1054, 28, 6, 0, 0, 1031, 992, 1027, 1053, 4, 0, 0, 0, 1013, 1012, 983, 990, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4260, 4107, 4087, 3963, 4077, 4107, 4008] avg=4096.0000 max=4260 max_rank=1 min=3963 min_rank=4 imbalance_ratio=1.040039 max_load_reduction=28508 max_load_reduction_pct=0.869995 moved_to_echo=16155 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1019, 982, 998, 1042, 0, 0, 0, 0, 1000, 1052, 1046, 1015, 0, 0, 0, 0, 989, 1041, 1012, 1029, 0, 0, 0, 0, 998, 1013, 1052, 1047, 0, 0, 0, 0, 1021, 1058, 1026, 1016, 15, 9, 0, 0, 1047, 1001, 1028, 982, 33, 12, 0, 0, 990, 1047, 990, 986, 6, 4, 0, 0, 1010, 1041, 1025, 1073, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4113, 4071, 4110, 4145, 4103, 4023, 4162] avg=4096.0000 max=4162 max_rank=7 min=4023 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1023, 1008, 1015, 997, 1050, 1031, 1024, 1038, 1017, 980, 1014, 1034, 1072, 1014, 1006, 1053, 991, 997, 991, 1015, 994, 1058, 1042, 996, 1026, 1056, 1035, 1077, 1056, 1041, 987, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1004, 940, 1049, 1044, 1004, 1029, 1022, 1026, 1075, 1048, 1031, 1030, 1053, 980, 1025, 1044, 980, 1038, 1002, 1030, 1001, 1007, 1043, 1051, 1066, 1018, 1035, 1002, 1042, 1009, 1031, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1015, 999, 1008, 1052, 1003, 1037, 1026, 1013, 1105, 1034, 950, 1023, 1011, 982, 969, 1043, 1043, 1070, 1063, 1033, 1056, 990, 1050, 1016, 1019, 1045, 1001, 1013, 974, 1028, 1065, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1025, 984, 1014, 1029, 991, 1014, 1025, 999, 1031, 1039, 1100, 1087, 947, 964, 1035, 1028, 1046, 978, 1022, 1086, 1020, 1035, 999, 1006, 1032, 1010, 1025, 1032, 1058, 1052, 1017, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1059, 1080, 1025, 1012, 989, 1026, 977, 964, 1054, 1018, 995, 1050, 1077, 984, 1033, 1037, 1019, 1033, 1080, 1018, 1018, 981, 1033, 1018, 1003, 1035, 1005, 1035, 1066, 1002, 1031, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[987, 1068, 1029, 1061, 1065, 1043, 1023, 1016, 1006, 1046, 1058, 994, 991, 1058, 989, 993, 1030, 977, 1044, 1049, 1001, 1054, 1001, 1045, 1037, 1007, 1042, 998, 1016, 1023, 995, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[991, 999, 1030, 1049, 1014, 1036, 1056, 992, 1016, 988, 959, 1034, 1019, 1077, 1049, 1001, 1008, 1068, 1042, 1012, 1130, 977, 1010, 1026, 970, 1026, 1030, 1062, 1025, 1007, 1059, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1043, 1001, 1003, 993, 996, 1061, 1002, 1049, 1056, 1042, 1029, 1012, 1017, 1094, 1030, 957, 989, 1012, 1069, 1034, 1035, 1057, 1005, 1001, 1081, 1012, 1023, 1033, 1008, 1012, 1017, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1023, 1008, 1015, 997, 10, 6, 0, 0, 1050, 1031, 1024, 1038, 10, 5, 0, 0, 995, 980, 1014, 1034, 0, 0, 0, 0, 1072, 1014, 1006, 1053, 16, 0, 0, 0, 991, 997, 980, 1015, 0, 0, 0, 0, 994, 1058, 1042, 996, 1, 0, 0, 0, 1026, 1056, 1035, 1062, 0, 0, 0, 0, 1056, 1041, 987, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4158, 4023, 4161, 3983, 4091, 4179, 4114] avg=4096.0000 max=4179 max_rank=6 min=3983 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1015, 999, 1008, 1052, 9, 6, 0, 0, 1003, 1037, 1026, 1013, 10, 5, 0, 0, 1082, 1034, 950, 1023, 0, 0, 0, 0, 1011, 982, 969, 1043, 17, 0, 0, 0, 1043, 1070, 1052, 1033, 0, 0, 0, 0, 1056, 990, 1050, 1016, 1, 0, 0, 0, 1019, 1045, 1001, 999, 0, 0, 0, 0, 974, 1028, 1065, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4094, 4089, 4022, 4198, 4113, 4064, 4099] avg=4096.0000 max=4198 max_rank=4 min=4022 min_rank=3 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1025, 984, 1014, 1029, 10, 6, 0, 0, 991, 1014, 1025, 999, 10, 5, 0, 0, 1009, 1039, 1100, 1087, 0, 0, 0, 0, 947, 964, 1035, 1028, 16, 0, 0, 0, 1046, 978, 1011, 1086, 0, 0, 0, 0, 1020, 1035, 999, 1006, 1, 0, 0, 0, 1032, 1010, 1025, 1017, 0, 0, 0, 0, 1058, 1052, 1017, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4044, 4235, 3990, 4121, 4061, 4084, 4165] avg=4096.0000 max=4235 max_rank=2 min=3990 min_rank=3 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1004, 940, 1049, 1044, 9, 6, 0, 0, 1004, 1029, 1022, 1026, 10, 5, 0, 0, 1052, 1048, 1031, 1030, 0, 0, 0, 0, 1053, 980, 1025, 1044, 17, 0, 0, 0, 980, 1038, 991, 1030, 0, 0, 0, 0, 1001, 1007, 1043, 1051, 1, 0, 0, 0, 1066, 1018, 1035, 988, 0, 0, 0, 0, 1042, 1009, 1031, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4096, 4161, 4119, 4039, 4103, 4107, 4091] avg=4096.0000 max=4161 max_rank=2 min=4039 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[987, 1068, 1029, 1061, 9, 6, 0, 0, 1065, 1043, 1023, 1016, 10, 5, 0, 0, 984, 1046, 1058, 994, 0, 0, 0, 0, 991, 1058, 989, 993, 16, 0, 0, 0, 1030, 977, 1033, 1049, 0, 0, 0, 0, 1001, 1054, 1001, 1045, 1, 0, 0, 0, 1037, 1007, 1042, 984, 0, 0, 0, 0, 1016, 1023, 995, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4162, 4082, 4047, 4089, 4102, 4070, 4056] avg=4096.0000 max=4162 max_rank=1 min=4047 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[991, 999, 1030, 1049, 10, 6, 0, 0, 1014, 1036, 1056, 992, 10, 5, 0, 0, 994, 988, 959, 1034, 0, 0, 0, 0, 1019, 1077, 1049, 1001, 16, 0, 0, 0, 1008, 1068, 1031, 1012, 0, 0, 0, 0, 1130, 977, 1010, 1026, 1, 0, 0, 0, 970, 1026, 1030, 1047, 0, 0, 0, 0, 1025, 1007, 1059, 1006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4113, 3975, 4162, 4119, 4144, 4073, 4097] avg=4096.0000 max=4162 max_rank=3 min=3975 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1059, 1080, 1025, 1012, 14, 9, 0, 0, 989, 1026, 977, 964, 15, 7, 0, 0, 1024, 1018, 995, 1050, 0, 0, 0, 0, 1077, 984, 1033, 1037, 21, 0, 0, 0, 1019, 1033, 1056, 1018, 0, 0, 0, 0, 1018, 981, 1033, 1018, 5, 0, 0, 0, 1003, 1035, 1005, 1014, 0, 0, 0, 0, 1066, 1002, 1031, 1011, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4199, 3978, 4087, 4152, 4126, 4055, 4057, 4114] avg=4096.0000 max=4199 max_rank=0 min=3978 min_rank=1 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1043, 1001, 1003, 993, 10, 6, 0, 0, 996, 1061, 1002, 1049, 10, 5, 0, 0, 1033, 1042, 1029, 1012, 0, 0, 0, 0, 1017, 1094, 1030, 957, 17, 0, 0, 0, 989, 1012, 1058, 1034, 0, 0, 0, 0, 1035, 1057, 1005, 1001, 1, 0, 0, 0, 1081, 1012, 1023, 1018, 0, 0, 0, 0, 1008, 1012, 1017, 995, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4123, 4116, 4115, 4093, 4099, 4134, 4032] avg=4096.0000 max=4134 max_rank=6 min=4032 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1020, 1016, 1000, 1049, 1001, 1013, 1082, 976, 1060, 1071, 975, 1019, 1010, 979, 1073, 1006, 1025, 1041, 1038, 1064, 1006, 993, 1019, 1016, 1002, 1006, 1042, 1077, 1017, 1018, 1030, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1030, 1059, 1008, 990, 1002, 1032, 1012, 1005, 1064, 1065, 1041, 1006, 981, 1043, 1026, 1057, 1053, 1011, 1048, 1009, 1016, 1002, 1000, 1054, 989, 998, 1021, 1034, 1014, 1011, 1046, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[979, 1051, 1025, 999, 1035, 1014, 1052, 999, 1005, 1014, 991, 984, 1042, 1084, 1086, 1040, 1010, 1005, 1015, 1030, 1069, 1021, 971, 998, 1041, 1003, 1058, 1064, 1002, 1023, 1020, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1021, 1023, 1045, 1064, 1022, 1042, 999, 1004, 988, 1059, 1003, 989, 993, 1001, 1006, 1076, 1041, 1005, 1035, 1039, 1031, 1010, 1003, 1055, 1001, 1056, 1011, 1032, 1038, 1016, 1011, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1037, 998, 1035, 1044, 1014, 977, 1011, 1031, 965, 1002, 1004, 1027, 1061, 1050, 1048, 1019, 1053, 1030, 1048, 1051, 1011, 1076, 998, 1026, 1002, 998, 1025, 1035, 996, 1036, 1030, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[996, 956, 1044, 1062, 999, 1024, 1015, 1053, 1029, 1027, 1041, 1041, 1029, 1069, 1005, 1062, 1060, 1036, 1019, 1025, 1039, 1015, 1004, 1023, 972, 1027, 1042, 985, 1015, 1013, 971, 1070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1020, 1012, 1030, 1028, 1092, 1024, 1063, 970, 976, 1008, 1034, 1057, 1016, 1017, 1008, 1007, 1023, 1053, 1034, 1053, 1082, 999, 1004, 1030, 1017, 1004, 1016, 985, 1053, 1036, 1012, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[995, 1016, 1039, 1015, 979, 1002, 1024, 1071, 1052, 1060, 1070, 1038, 1011, 1029, 1010, 1014, 1059, 994, 998, 1023, 1044, 1080, 1006, 1055, 1000, 1027, 998, 975, 1038, 1062, 1005, 979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1020, 1016, 1000, 1049, 7, 0, 0, 0, 1001, 1013, 1082, 976, 11, 4, 0, 0, 1060, 1071, 975, 1019, 0, 0, 0, 0, 1010, 979, 1073, 985, 0, 0, 0, 0, 994, 1041, 1038, 1064, 0, 0, 0, 0, 1006, 993, 1019, 1016, 1, 0, 0, 0, 1002, 1006, 1042, 1077, 27, 0, 0, 0, 1017, 1018, 1030, 1024, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4087, 4125, 4047, 4137, 4035, 4154, 4091] avg=4096.0000 max=4154 max_rank=6 min=4035 min_rank=5 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1021, 1023, 1045, 1064, 8, 0, 0, 0, 1022, 1042, 999, 1004, 12, 4, 0, 0, 988, 1059, 1003, 989, 0, 0, 0, 0, 993, 1001, 1006, 1053, 0, 0, 0, 0, 1009, 1005, 1035, 1039, 0, 0, 0, 0, 1031, 1010, 1003, 1055, 1, 0, 0, 0, 1001, 1056, 1011, 1032, 28, 0, 0, 0, 1038, 1016, 1011, 1049, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4083, 4039, 4053, 4088, 4100, 4128, 4116] avg=4096.0000 max=4161 max_rank=0 min=4039 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[979, 1051, 1025, 999, 7, 0, 0, 0, 1035, 1014, 1052, 999, 11, 4, 0, 0, 1005, 1014, 991, 984, 0, 0, 0, 0, 1042, 1084, 1086, 1019, 0, 0, 0, 0, 979, 1005, 1015, 1030, 0, 0, 0, 0, 1069, 1021, 971, 998, 1, 0, 0, 0, 1041, 1003, 1058, 1064, 27, 0, 0, 0, 1002, 1023, 1020, 1038, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4115, 3994, 4231, 4029, 4060, 4193, 4085] avg=4096.0000 max=4231 max_rank=3 min=3994 min_rank=2 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1030, 1059, 1008, 990, 7, 0, 0, 0, 1002, 1032, 1012, 1005, 11, 4, 0, 0, 1064, 1065, 1041, 1006, 0, 0, 0, 0, 981, 1043, 1026, 1036, 0, 0, 0, 0, 1021, 1011, 1048, 1009, 0, 0, 0, 0, 1016, 1002, 1000, 1054, 1, 0, 0, 0, 989, 998, 1021, 1034, 28, 0, 0, 0, 1014, 1011, 1046, 1041, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4066, 4176, 4086, 4089, 4073, 4070, 4114] avg=4096.0000 max=4176 max_rank=2 min=4066 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[996, 956, 1044, 1062, 12, 0, 0, 0, 999, 1024, 1015, 1053, 16, 7, 0, 0, 1029, 1027, 1041, 1041, 3, 0, 0, 0, 1029, 1069, 1005, 1021, 0, 0, 0, 0, 1021, 1036, 1019, 1025, 0, 0, 0, 0, 1039, 1015, 1004, 1023, 5, 0, 0, 0, 972, 1027, 1042, 985, 32, 0, 0, 0, 1015, 1013, 971, 1070, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4114, 4141, 4124, 4101, 4086, 4058, 4074] avg=4096.0000 max=4141 max_rank=2 min=4058 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1020, 1012, 1030, 1028, 7, 0, 0, 0, 1092, 1024, 1063, 970, 11, 4, 0, 0, 976, 1008, 1034, 1057, 0, 0, 0, 0, 1016, 1017, 1008, 986, 0, 0, 0, 0, 992, 1053, 1034, 1053, 0, 0, 0, 0, 1082, 999, 1004, 1030, 1, 0, 0, 0, 1017, 1004, 1016, 985, 27, 0, 0, 0, 1053, 1036, 1012, 1005, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4164, 4075, 4027, 4132, 4116, 4049, 4108] avg=4096.0000 max=4164 max_rank=1 min=4027 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1037, 998, 1035, 1044, 7, 0, 0, 0, 1014, 977, 1011, 1031, 11, 4, 0, 0, 965, 1002, 1004, 1027, 0, 0, 0, 0, 1061, 1050, 1048, 998, 0, 0, 0, 0, 1021, 1030, 1048, 1051, 0, 0, 0, 0, 1011, 1076, 998, 1026, 1, 0, 0, 0, 1002, 998, 1025, 1035, 28, 0, 0, 0, 996, 1036, 1030, 1030, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4048, 3998, 4157, 4150, 4112, 4088, 4094] avg=4096.0000 max=4157 max_rank=3 min=3998 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[995, 1016, 1039, 1015, 7, 0, 0, 0, 979, 1002, 1024, 1071, 11, 4, 0, 0, 1052, 1060, 1070, 1038, 0, 0, 0, 0, 1011, 1029, 1010, 993, 0, 0, 0, 0, 1027, 994, 998, 1023, 0, 0, 0, 0, 1044, 1080, 1006, 1055, 1, 0, 0, 0, 1000, 1027, 998, 975, 28, 0, 0, 0, 1038, 1062, 1005, 979, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4091, 4220, 4043, 4042, 4186, 4028, 4086] avg=4096.0000 max=4220 max_rank=2 min=4028 min_rank=6 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1007, 1026, 975, 1027, 1020, 997, 1000, 1036, 1030, 1015, 1021, 1011, 1039, 1031, 1019, 1036, 1067, 987, 1018, 980, 1079, 981, 1043, 1082, 1007, 1031, 1046, 1024, 988, 1022, 1094, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1012, 1043, 992, 1043, 1013, 1087, 973, 1001, 1048, 1058, 991, 1015, 1022, 1064, 1002, 964, 1021, 1022, 1004, 1022, 1083, 1034, 1077, 983, 983, 1048, 1044, 1020, 1031, 1005, 1031, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1024, 1003, 1023, 1025, 1049, 1040, 978, 1087, 998, 1029, 1035, 1060, 1031, 1049, 1010, 996, 1050, 1061, 996, 1023, 998, 1020, 1054, 983, 1062, 993, 1046, 1061, 948, 1009, 979, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[984, 1021, 1030, 1083, 1043, 1050, 1059, 1010, 940, 996, 1034, 1002, 1017, 970, 1042, 1072, 1023, 1050, 1049, 974, 1005, 1047, 1042, 1044, 1019, 1009, 1022, 1054, 1026, 1031, 988, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1095, 1032, 1043, 1082, 1005, 1006, 1046, 1016, 1007, 1047, 1009, 1044, 1002, 1022, 969, 1064, 1016, 1022, 976, 980, 1026, 1039, 1008, 1019, 1041, 1062, 1039, 1011, 1010, 1044, 969, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1011, 1036, 1022, 1015, 1062, 980, 1045, 1030, 1030, 1044, 1013, 969, 1029, 1051, 1033, 1011, 1070, 1023, 1041, 999, 1001, 1022, 1014, 1015, 1018, 997, 1010, 973, 1067, 1065, 1041, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1101, 1066, 990, 1015, 983, 1037, 998, 1023, 1050, 1067, 1017, 1060, 1003, 994, 1051, 990, 1035, 1026, 1015, 997, 1020, 1055, 1082, 969, 1033, 1040, 1010, 1052, 1013, 968, 996, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1035, 1026, 1003, 1087, 1011, 989, 987, 984, 1055, 1021, 1050, 1059, 988, 1042, 1061, 994, 1026, 999, 1014, 1038, 1074, 1007, 1029, 1033, 949, 1032, 1052, 981, 1022, 1071, 1019, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1007, 1026, 975, 1003, 0, 0, 0, 0, 1020, 997, 1000, 1036, 7, 7, 0, 0, 1030, 1009, 1021, 1011, 0, 0, 0, 0, 1039, 1031, 1019, 1036, 12, 0, 0, 0, 1067, 987, 1018, 980, 17, 0, 0, 0, 1079, 981, 1019, 1082, 0, 0, 0, 0, 1007, 1031, 1046, 1024, 0, 0, 0, 0, 988, 1022, 1094, 1029, 6, 5, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4067, 4071, 4137, 4069, 4161, 4108, 4144] avg=4096.0000 max=4161 max_rank=5 min=4011 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1012, 1043, 992, 1018, 0, 0, 0, 0, 1013, 1087, 973, 1001, 8, 7, 0, 0, 1048, 1051, 991, 1015, 0, 0, 0, 0, 1022, 1064, 1002, 964, 12, 0, 0, 0, 1021, 1022, 1004, 1022, 17, 0, 0, 0, 1083, 1034, 1053, 983, 0, 0, 0, 0, 983, 1048, 1044, 1020, 0, 0, 0, 0, 1031, 1005, 1031, 1032, 7, 5, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4089, 4105, 4064, 4086, 4153, 4095, 4111] avg=4096.0000 max=4153 max_rank=5 min=4064 min_rank=3 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1024, 1003, 1023, 1001, 0, 0, 0, 0, 1049, 1040, 978, 1087, 7, 7, 0, 0, 998, 1022, 1035, 1060, 0, 0, 0, 0, 1031, 1049, 1010, 996, 12, 0, 0, 0, 1050, 1061, 996, 1023, 17, 0, 0, 0, 998, 1020, 1030, 983, 0, 0, 0, 0, 1062, 993, 1046, 1061, 0, 0, 0, 0, 948, 1009, 979, 1048, 7, 5, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4168, 4115, 4098, 4147, 4031, 4162, 3996] avg=4096.0000 max=4168 max_rank=1 min=3996 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[984, 1021, 1030, 1057, 0, 0, 0, 0, 1043, 1050, 1059, 1010, 8, 7, 0, 0, 940, 990, 1034, 1002, 0, 0, 0, 0, 1017, 970, 1042, 1072, 12, 0, 0, 0, 1023, 1050, 1049, 974, 18, 0, 0, 0, 1005, 1047, 1018, 1044, 0, 0, 0, 0, 1019, 1009, 1022, 1054, 0, 0, 0, 0, 1026, 1031, 988, 1032, 6, 5, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4177, 3966, 4113, 4114, 4114, 4104, 4088] avg=4096.0000 max=4177 max_rank=1 min=3966 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1095, 1032, 1043, 1056, 0, 0, 0, 0, 1005, 1006, 1046, 1016, 8, 7, 0, 0, 1007, 1040, 1009, 1044, 0, 0, 0, 0, 1002, 1022, 969, 1064, 12, 0, 0, 0, 1016, 1022, 976, 980, 18, 0, 0, 0, 1026, 1039, 984, 1019, 0, 0, 0, 0, 1041, 1062, 1039, 1011, 0, 0, 0, 0, 1010, 1044, 969, 1017, 7, 5, 0, 0] rerouted_tokens_per_ep_rank=[4226, 4088, 4100, 4069, 4012, 4068, 4153, 4052] avg=4096.0000 max=4226 max_rank=0 min=4012 min_rank=4 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1101, 1066, 990, 991, 0, 0, 0, 0, 983, 1037, 998, 1023, 7, 7, 0, 0, 1050, 1060, 1017, 1060, 0, 0, 0, 0, 1003, 994, 1051, 990, 12, 0, 0, 0, 1035, 1026, 1015, 997, 17, 0, 0, 0, 1020, 1055, 1058, 969, 0, 0, 0, 0, 1033, 1040, 1010, 1052, 0, 0, 0, 0, 1013, 968, 996, 1012, 7, 5, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4055, 4187, 4050, 4090, 4102, 4135, 4001] avg=4096.0000 max=4187 max_rank=2 min=4001 min_rank=7 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1011, 1036, 1022, 991, 0, 0, 0, 0, 1062, 980, 1045, 1030, 7, 7, 0, 0, 1030, 1037, 1013, 969, 0, 0, 0, 0, 1029, 1051, 1033, 1011, 12, 0, 0, 0, 1070, 1023, 1041, 999, 17, 0, 0, 0, 1001, 1022, 990, 1015, 0, 0, 0, 0, 1018, 997, 1010, 973, 0, 0, 0, 0, 1067, 1065, 1041, 1031, 7, 5, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4131, 4049, 4136, 4150, 4028, 3998, 4216] avg=4096.0000 max=4216 max_rank=7 min=3998 min_rank=6 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1035, 1026, 1003, 1051, 0, 0, 0, 0, 1011, 989, 987, 984, 13, 9, 0, 0, 1055, 1011, 1050, 1059, 0, 0, 0, 0, 988, 1042, 1061, 994, 16, 0, 0, 0, 1026, 999, 1014, 1038, 23, 0, 0, 0, 1074, 1007, 997, 1033, 0, 0, 0, 0, 949, 1032, 1051, 981, 0, 0, 0, 0, 1022, 1071, 1019, 1030, 10, 7, 1, 0] rerouted_tokens_per_ep_rank=[4115, 3993, 4175, 4101, 4100, 4111, 4013, 4160] avg=4096.0000 max=4175 max_rank=2 min=3993 min_rank=1 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8064, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[999, 977, 1051, 995, 1068, 1068, 1015, 1010, 1026, 1059, 983, 1057, 1018, 1046, 1018, 1023, 1000, 996, 1090, 1055, 1001, 1071, 1013, 1011, 1017, 989, 1004, 1042, 1014, 1056, 989, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1014, 971, 1068, 1021, 1017, 1068, 1036, 984, 1083, 996, 993, 998, 1040, 1058, 1042, 959, 1039, 1009, 1032, 1041, 974, 1006, 991, 1061, 1031, 1036, 1045, 1019, 1072, 1053, 979, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1051, 1076, 1026, 993, 1009, 1056, 1016, 1006, 1029, 1053, 1000, 954, 1035, 989, 1047, 993, 1072, 1046, 1000, 1038, 1037, 1006, 1049, 1068, 1011, 1015, 1016, 1000, 1022, 1023, 1013, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1009, 1047, 1028, 1003, 1057, 1024, 998, 1000, 996, 1013, 1033, 987, 1023, 1032, 1038, 1061, 1034, 1008, 1044, 1045, 1047, 1035, 993, 1056, 987, 977, 1042, 1073, 978, 1029, 1060, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1049, 1029, 1051, 1005, 1029, 1031, 1010, 995, 1096, 1017, 1000, 1036, 1011, 1017, 1033, 968, 1012, 1034, 967, 1057, 1000, 1041, 979, 1078, 1034, 1074, 1002, 1018, 1037, 1052, 1009, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[993, 1006, 1011, 1023, 1061, 972, 994, 1052, 1027, 1038, 1041, 1041, 1045, 1046, 999, 1019, 1016, 997, 976, 1045, 1045, 1026, 982, 1023, 1041, 1022, 1035, 1005, 1031, 1057, 1073, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[991, 1033, 981, 1053, 1025, 951, 1034, 1047, 1044, 1013, 1055, 1086, 1065, 1014, 993, 1043, 992, 990, 1051, 1017, 1037, 1052, 1005, 1023, 992, 1033, 1007, 1016, 1068, 1004, 1035, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1007, 1000, 1031, 1057, 1005, 1085, 1031, 1031, 1027, 1074, 1041, 999, 1051, 981, 1059, 1033, 1023, 1007, 1040, 999, 1054, 1008, 1049, 1063, 961, 972, 1063, 1026, 977, 994, 1012, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1014, 971, 1068, 1021, 10, 3, 0, 0, 1016, 1068, 1036, 984, 0, 0, 0, 0, 1067, 996, 993, 998, 0, 0, 0, 0, 1037, 1058, 1042, 959, 0, 0, 0, 0, 1039, 1009, 1032, 1041, 0, 0, 0, 0, 974, 1006, 991, 1047, 0, 0, 0, 0, 1031, 1036, 1045, 1019, 16, 4, 0, 0, 1072, 1053, 979, 1032, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4104, 4054, 4096, 4121, 4018, 4151, 4137] avg=4096.0000 max=4151 max_rank=6 min=4018 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1009, 1047, 1028, 1003, 10, 3, 1, 0, 1055, 1024, 998, 1000, 0, 0, 0, 0, 981, 1013, 1033, 987, 0, 0, 0, 0, 1020, 1032, 1038, 1061, 0, 0, 0, 0, 1034, 1008, 1044, 1045, 0, 0, 0, 0, 1047, 1035, 993, 1042, 0, 0, 0, 0, 987, 977, 1042, 1073, 15, 4, 0, 0, 978, 1029, 1060, 1011, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4077, 4014, 4151, 4131, 4117, 4098, 4079] avg=4096.0000 max=4151 max_rank=3 min=4014 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[999, 977, 1051, 995, 9, 3, 1, 0, 1066, 1068, 1015, 1010, 0, 0, 0, 0, 1011, 1059, 983, 1057, 0, 0, 0, 0, 1015, 1046, 1018, 1023, 0, 0, 0, 0, 1000, 996, 1090, 1055, 0, 0, 0, 0, 1001, 1071, 1013, 998, 0, 0, 0, 0, 1017, 989, 1004, 1042, 15, 4, 0, 0, 1014, 1056, 989, 1007, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4159, 4110, 4102, 4141, 4083, 4071, 4067] avg=4096.0000 max=4159 max_rank=1 min=4035 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1051, 1076, 1026, 993, 10, 3, 0, 0, 1008, 1056, 1016, 1006, 0, 0, 0, 0, 1014, 1053, 1000, 954, 0, 0, 0, 0, 1032, 989, 1047, 993, 0, 0, 0, 0, 1072, 1046, 1000, 1038, 0, 0, 0, 0, 1037, 1006, 1049, 1054, 0, 0, 0, 0, 1011, 1015, 1016, 1000, 15, 4, 0, 0, 1022, 1023, 1013, 1019, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4086, 4021, 4061, 4156, 4146, 4061, 4078] avg=4096.0000 max=4159 max_rank=0 min=4021 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[993, 1006, 1011, 1023, 9, 3, 1, 0, 1059, 972, 994, 1052, 0, 0, 0, 0, 1012, 1038, 1041, 1041, 0, 0, 0, 0, 1042, 1046, 999, 1019, 0, 0, 0, 0, 1016, 997, 976, 1045, 0, 0, 0, 0, 1045, 1026, 982, 1010, 0, 0, 0, 0, 1041, 1022, 1035, 1005, 15, 4, 0, 0, 1031, 1057, 1073, 1026, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4077, 4132, 4106, 4034, 4063, 4122, 4188] avg=4096.0000 max=4188 max_rank=7 min=4034 min_rank=4 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1049, 1029, 1051, 1005, 10, 3, 0, 0, 1028, 1031, 1010, 995, 0, 0, 0, 0, 1080, 1017, 1000, 1036, 0, 0, 0, 0, 1008, 1017, 1033, 968, 0, 0, 0, 0, 1012, 1034, 967, 1057, 0, 0, 0, 0, 1000, 1041, 979, 1064, 0, 0, 0, 0, 1034, 1074, 1002, 1018, 16, 4, 0, 0, 1037, 1052, 1009, 997, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4064, 4133, 4026, 4070, 4084, 4148, 4096] avg=4096.0000 max=4148 max_rank=6 min=4026 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[991, 1033, 981, 1053, 12, 10, 5, 0, 1018, 951, 1034, 1047, 0, 0, 0, 0, 1024, 1013, 1055, 1086, 0, 0, 0, 0, 1055, 1014, 993, 1043, 0, 0, 0, 0, 992, 990, 1051, 1013, 0, 0, 0, 0, 1037, 1052, 1005, 1003, 0, 0, 0, 0, 992, 1033, 1007, 1016, 20, 8, 0, 0, 1068, 1004, 1035, 1018, 2, 4, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4050, 4178, 4105, 4046, 4097, 4076, 4131] avg=4096.0000 max=4178 max_rank=2 min=4046 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1007, 1000, 1031, 1057, 10, 3, 0, 0, 1004, 1085, 1031, 1031, 0, 0, 0, 0, 1012, 1074, 1041, 999, 0, 0, 0, 0, 1048, 981, 1059, 1033, 0, 0, 0, 0, 1023, 1007, 1040, 999, 0, 0, 0, 0, 1054, 1008, 1049, 1049, 0, 0, 0, 0, 961, 972, 1063, 1026, 15, 4, 0, 0, 977, 994, 1012, 1008, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4151, 4126, 4121, 4069, 4160, 4041, 3992] avg=4096.0000 max=4160 max_rank=5 min=3992 min_rank=7 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16262 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1030, 1056, 1014, 1026, 1031, 1041, 1031, 1035, 1014, 1013, 960, 1047, 1007, 1019, 1030, 1000, 1055, 1029, 1052, 1002, 1047, 1042, 1014, 1081, 1022, 1020, 1033, 1015, 1047, 983, 972, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1066, 1041, 978, 1052, 1013, 1027, 1004, 1014, 1041, 1056, 1032, 1022, 1009, 1034, 1028, 1042, 1069, 1036, 1003, 1086, 951, 1041, 998, 986, 991, 990, 1042, 1075, 1021, 1016, 1009, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1000, 1071, 1032, 1018, 1017, 1042, 1012, 1077, 977, 1026, 994, 1055, 1104, 1046, 1044, 1059, 1017, 1019, 993, 975, 1024, 996, 1034, 1035, 1048, 1011, 1043, 998, 1033, 981, 1015, 972] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1055, 1025, 1045, 1011, 1016, 1000, 1014, 1040, 1061, 1066, 988, 963, 1061, 1023, 1006, 1052, 1022, 1002, 1071, 1019, 1027, 999, 1021, 987, 1024, 1006, 1023, 1059, 1062, 1028, 992, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1007, 1000, 1038, 997, 1050, 1032, 1036, 1000, 1073, 1081, 1041, 1019, 985, 1039, 1079, 1017, 1041, 988, 1004, 1005, 992, 1060, 997, 1011, 1016, 1005, 961, 1009, 1087, 1041, 1025, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1053, 1007, 1029, 1047, 984, 999, 983, 1059, 1025, 1016, 992, 993, 1044, 1033, 1003, 1023, 996, 1047, 1004, 1014, 1048, 1040, 1062, 1003, 1033, 1017, 1006, 1041, 1004, 1040, 1052, 1071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1012, 1020, 1019, 1037, 1017, 1056, 1001, 992, 1055, 1027, 1037, 1017, 1014, 1040, 1025, 1008, 1042, 1019, 1016, 1051, 1016, 1005, 1076, 992, 1063, 1021, 1030, 1029, 1017, 1006, 986, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1028, 1048, 1063, 1068, 1028, 1011, 1011, 1014, 1048, 983, 1059, 1054, 1015, 1017, 1066, 1059, 1043, 1001, 998, 951, 1066, 1009, 1006, 1034, 1017, 1056, 1024, 1018, 955, 1047, 1012, 959] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1066, 1015, 978, 1052, 0, 0, 0, 0, 1013, 1027, 1004, 1014, 10, 0, 0, 0, 1034, 1056, 1032, 1022, 0, 0, 0, 0, 1009, 1034, 996, 1042, 0, 0, 0, 0, 1069, 1036, 1003, 1086, 12, 0, 0, 0, 951, 1041, 998, 986, 5, 2, 0, 0, 991, 990, 1042, 1075, 2, 0, 0, 0, 1021, 1016, 1009, 995, 32, 2, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4068, 4144, 4081, 4206, 3983, 4100, 4075] avg=4096.0000 max=4206 max_rank=4 min=3983 min_rank=5 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1055, 999, 1045, 1011, 0, 0, 0, 0, 1016, 1000, 1014, 1040, 10, 0, 0, 0, 1054, 1066, 988, 963, 0, 0, 0, 0, 1061, 1023, 975, 1052, 0, 0, 0, 0, 1022, 1002, 1071, 1019, 12, 0, 0, 0, 1027, 999, 1021, 987, 5, 2, 0, 0, 1024, 1006, 1023, 1059, 2, 0, 0, 0, 1062, 1028, 992, 1000, 31, 2, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4080, 4071, 4111, 4126, 4041, 4114, 4115] avg=4096.0000 max=4126 max_rank=4 min=4041 min_rank=5 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1000, 1045, 1032, 1018, 0, 0, 0, 0, 1017, 1042, 1012, 1077, 10, 0, 0, 0, 970, 1026, 994, 1055, 0, 0, 0, 0, 1104, 1046, 1011, 1059, 0, 0, 0, 0, 1017, 1019, 993, 975, 12, 0, 0, 0, 1024, 996, 1034, 1035, 5, 2, 0, 0, 1048, 1011, 1043, 998, 2, 0, 0, 0, 1033, 981, 1015, 972, 33, 2, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4158, 4045, 4220, 4016, 4096, 4102, 4036] avg=4096.0000 max=4220 max_rank=3 min=4016 min_rank=4 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1030, 1030, 1014, 1026, 0, 0, 0, 0, 1031, 1041, 1031, 1035, 10, 0, 0, 0, 1007, 1013, 960, 1047, 0, 0, 0, 0, 1007, 1019, 998, 1000, 0, 0, 0, 0, 1055, 1029, 1052, 1002, 12, 0, 0, 0, 1047, 1042, 1014, 1081, 5, 2, 0, 0, 1022, 1020, 1033, 1015, 2, 0, 0, 0, 1047, 983, 972, 1000, 32, 2, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4148, 4027, 4024, 4150, 4191, 4092, 4036] avg=4096.0000 max=4191 max_rank=5 min=4024 min_rank=3 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1007, 976, 1038, 997, 0, 0, 0, 0, 1050, 1032, 1036, 1000, 9, 0, 0, 0, 1066, 1081, 1041, 1019, 0, 0, 0, 0, 985, 1039, 1045, 1017, 0, 0, 0, 0, 1041, 988, 1004, 1005, 11, 0, 0, 0, 992, 1060, 997, 1011, 5, 2, 0, 0, 1016, 1005, 961, 1009, 2, 0, 0, 0, 1087, 1041, 1025, 1032, 34, 2, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4127, 4207, 4086, 4049, 4067, 3993, 4221] avg=4096.0000 max=4221 max_rank=7 min=3993 min_rank=6 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1012, 995, 1019, 1037, 0, 0, 0, 0, 1017, 1056, 1001, 992, 9, 0, 0, 0, 1048, 1027, 1037, 1017, 0, 0, 0, 0, 1014, 1040, 993, 1008, 0, 0, 0, 0, 1042, 1019, 1016, 1051, 12, 0, 0, 0, 1016, 1005, 1076, 992, 5, 2, 0, 0, 1063, 1021, 1030, 1029, 2, 0, 0, 0, 1017, 1006, 986, 1022, 32, 2, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4075, 4129, 4055, 4140, 4096, 4145, 4065] avg=4096.0000 max=4145 max_rank=6 min=4055 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1053, 961, 1029, 1047, 0, 0, 0, 0, 984, 999, 983, 1059, 13, 0, 0, 0, 1007, 1016, 992, 993, 0, 0, 0, 0, 1044, 1033, 967, 1023, 0, 0, 0, 0, 996, 1047, 1004, 1014, 15, 0, 0, 0, 1048, 1040, 1062, 1003, 10, 9, 0, 0, 1033, 1017, 1006, 1041, 8, 0, 0, 0, 1004, 1040, 1052, 1071, 36, 9, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4038, 4008, 4067, 4076, 4172, 4105, 4212] avg=4096.0000 max=4212 max_rank=7 min=4008 min_rank=2 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1028, 1022, 1063, 1068, 0, 0, 0, 0, 1028, 1011, 1011, 1014, 10, 0, 0, 0, 1041, 983, 1059, 1054, 0, 0, 0, 0, 1015, 1017, 1033, 1059, 0, 0, 0, 0, 1043, 1001, 998, 951, 12, 0, 0, 0, 1066, 1009, 1006, 1034, 5, 2, 0, 0, 1017, 1056, 1024, 1018, 2, 0, 0, 0, 955, 1047, 1012, 959, 33, 2, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4074, 4137, 4124, 4005, 4122, 4117, 4008] avg=4096.0000 max=4181 max_rank=0 min=4005 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16252 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8064, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1002, 1038, 1005, 1064, 1058, 1033, 983, 1026, 1010, 1010, 996, 1096, 1067, 1004, 1061, 1021, 1000, 1031, 1002, 1048, 986, 1000, 966, 994, 1083, 1097, 1045, 1010, 1058, 971, 1005, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1054, 1014, 1017, 1034, 1055, 1025, 1029, 1009, 1069, 1014, 1001, 1031, 1057, 1014, 1033, 1023, 986, 972, 1039, 1021, 988, 1010, 1036, 1013, 1029, 1085, 1002, 1052, 994, 1049, 1004, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1013, 1033, 1016, 1031, 1031, 1031, 983, 1039, 1039, 1024, 1028, 1005, 985, 1014, 981, 986, 1022, 1011, 1027, 1070, 1012, 1031, 1031, 1052, 1033, 1039, 1041, 1040, 1036, 1041, 1016, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1003, 1031, 1023, 1080, 1016, 1034, 1001, 1055, 1039, 992, 987, 1023, 1032, 980, 1012, 1047, 1077, 1032, 1078, 1012, 1028, 993, 996, 1036, 972, 1018, 1030, 1029, 1017, 1036, 1028, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1027, 981, 1053, 1003, 1061, 1017, 1040, 1035, 988, 1021, 1057, 1029, 1068, 1005, 1037, 1038, 1029, 1041, 1060, 980, 1036, 1037, 994, 1074, 1027, 1043, 1011, 1033, 979, 975, 1007, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1066, 943, 1019, 995, 1033, 1029, 1038, 1037, 1027, 1012, 999, 1019, 1012, 968, 991, 1018, 981, 1024, 1002, 1047, 1063, 1066, 1043, 1061, 1044, 1051, 1039, 1043, 1023, 1009, 1042, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1000, 1010, 1009, 1031, 1032, 996, 985, 1008, 1003, 1047, 987, 1021, 998, 1034, 988, 1016, 1037, 1011, 1023, 1020, 1061, 1022, 1010, 1028, 1058, 1057, 1060, 1035, 1078, 1045, 1041, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1019, 1028, 1041, 1012, 980, 1027, 1008, 999, 1062, 1008, 1022, 1031, 988, 1015, 1091, 983, 1051, 1034, 1007, 987, 1000, 1038, 1003, 1031, 1086, 1080, 1029, 1029, 1034, 1025, 1034, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1002, 1038, 1005, 1064, 9, 0, 0, 0, 1058, 1033, 983, 1026, 4, 0, 0, 0, 1010, 1010, 996, 1096, 9, 0, 0, 0, 1067, 1004, 1061, 1021, 26, 0, 0, 0, 1000, 1031, 1002, 1048, 0, 0, 0, 0, 986, 1000, 966, 994, 3, 0, 0, 0, 1083, 1027, 1045, 1010, 0, 0, 0, 0, 1058, 971, 1005, 998, 19, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4104, 4121, 4179, 4081, 3949, 4165, 4051] avg=4096.0000 max=4179 max_rank=3 min=3949 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=1 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1013, 1033, 1016, 1031, 8, 0, 0, 0, 1031, 1031, 983, 1039, 4, 0, 0, 0, 1039, 1024, 1028, 1005, 8, 0, 0, 0, 985, 1014, 981, 986, 24, 0, 0, 0, 1022, 1011, 1027, 1070, 0, 0, 0, 0, 1012, 1031, 1031, 1052, 3, 0, 0, 0, 1033, 974, 1041, 1040, 0, 0, 0, 0, 1036, 1041, 1016, 1027, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4088, 4104, 3990, 4130, 4129, 4088, 4138] avg=4096.0000 max=4138 max_rank=7 min=3990 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=1 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1054, 1014, 1017, 1034, 9, 0, 0, 0, 1055, 1025, 1029, 1009, 4, 0, 0, 0, 1069, 1014, 1001, 1031, 9, 0, 0, 0, 1057, 1014, 1033, 1023, 25, 0, 0, 0, 986, 972, 1039, 1021, 0, 0, 0, 0, 988, 1010, 1036, 1013, 3, 0, 0, 0, 1029, 1017, 1002, 1052, 0, 0, 0, 0, 994, 1049, 1004, 1009, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4122, 4124, 4152, 4018, 4050, 4100, 4074] avg=4096.0000 max=4152 max_rank=3 min=4018 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=1 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1003, 1031, 1023, 1080, 8, 0, 0, 0, 1016, 1034, 1001, 1055, 4, 0, 0, 0, 1039, 992, 987, 1023, 8, 0, 0, 0, 1032, 980, 1012, 1047, 24, 0, 0, 0, 1077, 1032, 1078, 1012, 0, 0, 0, 0, 1028, 993, 996, 1036, 3, 0, 0, 0, 972, 954, 1030, 1029, 0, 0, 0, 0, 1017, 1036, 1028, 1031, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4110, 4049, 4095, 4199, 4056, 3985, 4129] avg=4096.0000 max=4199 max_rank=4 min=3985 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=1 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1027, 981, 1053, 1003, 8, 0, 0, 0, 1061, 1017, 1040, 1035, 4, 0, 0, 0, 988, 1021, 1057, 1029, 8, 0, 0, 0, 1068, 1005, 1037, 1038, 24, 0, 0, 0, 1029, 1041, 1060, 980, 0, 0, 0, 0, 1036, 1037, 994, 1074, 3, 0, 0, 0, 1027, 978, 1011, 1033, 0, 0, 0, 0, 979, 975, 1007, 982, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4157, 4103, 4172, 4110, 4144, 4049, 3961] avg=4096.0000 max=4172 max_rank=3 min=3961 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16264 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=1 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1000, 1010, 1009, 1031, 13, 0, 0, 0, 1032, 996, 985, 1008, 7, 0, 0, 0, 1003, 1047, 987, 1021, 12, 0, 0, 0, 998, 1034, 988, 1016, 29, 0, 0, 0, 1037, 1011, 1023, 1020, 6, 0, 0, 0, 1061, 1022, 1010, 1028, 8, 0, 0, 0, 1058, 961, 1060, 1035, 0, 0, 0, 0, 1078, 1045, 1041, 1017, 21, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4028, 4070, 4065, 4097, 4129, 4114, 4202] avg=4096.0000 max=4202 max_rank=7 min=4028 min_rank=1 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=1 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1066, 943, 1019, 995, 9, 0, 0, 0, 1033, 1029, 1038, 1037, 4, 0, 0, 0, 1027, 1012, 999, 1019, 8, 0, 0, 0, 1012, 968, 991, 1018, 24, 0, 0, 0, 981, 1024, 1002, 1047, 0, 0, 0, 0, 1063, 1066, 1043, 1061, 3, 0, 0, 0, 1044, 985, 1039, 1043, 0, 0, 0, 0, 1023, 1009, 1042, 1024, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4141, 4065, 4013, 4054, 4236, 4111, 4116] avg=4096.0000 max=4236 max_rank=5 min=4013 min_rank=3 imbalance_ratio=1.034180 max_load_reduction=28532 max_load_reduction_pct=0.870728 moved_to_echo=16517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=1 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1019, 1028, 1041, 1012, 9, 0, 0, 0, 980, 1027, 1008, 999, 4, 0, 0, 0, 1062, 1008, 1022, 1031, 9, 0, 0, 0, 988, 1015, 1091, 983, 25, 0, 0, 0, 1051, 1034, 1007, 987, 0, 0, 0, 0, 1000, 1038, 1003, 1031, 3, 0, 0, 0, 1086, 1012, 1029, 1029, 0, 0, 0, 0, 1034, 1025, 1034, 986, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4018, 4132, 4102, 4079, 4075, 4156, 4097] avg=4096.0000 max=4156 max_rank=6 min=4018 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=1 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 7936, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1006, 1036, 1000, 979, 1016, 1013, 1012, 988, 960, 1044, 1065, 972, 1069, 1043, 1044, 1008, 1016, 1046, 1003, 1068, 986, 1043, 1021, 1067, 967, 1022, 1022, 1069, 1073, 1028, 1009, 1073] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[966, 1063, 1027, 972, 1062, 975, 1009, 1029, 958, 993, 984, 1010, 1002, 1033, 1065, 1064, 1028, 1019, 1051, 1040, 1032, 1039, 1042, 1044, 998, 1031, 1004, 1059, 1017, 1046, 1047, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1042, 1023, 1023, 1037, 1026, 1028, 997, 1050, 1034, 1048, 1029, 1068, 979, 1011, 1000, 1060, 1039, 993, 996, 1002, 1013, 1032, 1026, 986, 1020, 1029, 1068, 1022, 1051, 1024, 1003, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[979, 1014, 1002, 1070, 1001, 1059, 1033, 1026, 1033, 1107, 973, 1063, 992, 1042, 1030, 1042, 1026, 988, 1019, 987, 1039, 985, 1041, 1043, 1069, 1051, 1047, 998, 997, 965, 990, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[976, 1031, 1011, 1036, 997, 996, 1016, 1033, 1091, 1050, 1034, 1093, 977, 1011, 1015, 1055, 1062, 1058, 1023, 1044, 1039, 989, 1014, 1007, 1010, 1002, 1054, 1008, 1008, 1009, 1033, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1046, 978, 1014, 1024, 1033, 1006, 1053, 1015, 1045, 1076, 960, 1066, 1047, 1024, 1034, 1033, 1056, 1066, 1009, 1023, 1052, 1002, 1066, 1003, 1009, 956, 1016, 1050, 1038, 977, 1018, 973] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1024, 1041, 1024, 1005, 1054, 988, 982, 1053, 1067, 1035, 961, 1026, 1026, 1016, 996, 1029, 1010, 1051, 1014, 1054, 1033, 1001, 1040, 1031, 1029, 1023, 1030, 1014, 1032, 990, 1082, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1010, 985, 1013, 981, 1050, 1039, 986, 996, 1027, 1005, 1032, 1056, 969, 1094, 1081, 1039, 1045, 971, 1037, 1031, 1041, 967, 1009, 1047, 988, 1054, 1064, 1037, 1021, 1014, 1026, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1006, 1036, 1000, 979, 24, 16, 0, 0, 1016, 1013, 1012, 988, 13, 3, 1, 0, 960, 1020, 1065, 972, 0, 0, 0, 0, 1069, 1043, 1044, 989, 0, 0, 0, 0, 1003, 1046, 1003, 1068, 0, 0, 0, 0, 986, 1043, 1020, 1067, 0, 0, 0, 0, 967, 1022, 1016, 1069, 0, 0, 0, 0, 1073, 1028, 1009, 1073, 5, 1, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4046, 4017, 4145, 4120, 4116, 4074, 4189] avg=4096.0000 max=4189 max_rank=7 min=4017 min_rank=2 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[966, 1063, 1027, 972, 23, 16, 0, 0, 1062, 975, 1009, 1029, 13, 3, 1, 0, 958, 970, 984, 1010, 0, 0, 0, 0, 1002, 1033, 1065, 1045, 0, 0, 0, 0, 1015, 1019, 1051, 1040, 0, 0, 0, 0, 1032, 1039, 1041, 1044, 0, 0, 0, 0, 998, 1031, 999, 1059, 0, 0, 0, 0, 1017, 1046, 1047, 1059, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4092, 3922, 4145, 4125, 4156, 4087, 4174] avg=4096.0000 max=4174 max_rank=7 min=3922 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[976, 1031, 1011, 1036, 24, 16, 0, 0, 997, 996, 1016, 1033, 13, 3, 1, 0, 1091, 1026, 1034, 1093, 0, 0, 0, 0, 977, 1011, 1015, 1036, 0, 0, 0, 0, 1049, 1058, 1023, 1044, 0, 0, 0, 0, 1039, 989, 1013, 1007, 0, 0, 0, 0, 1010, 1002, 1048, 1008, 0, 0, 0, 0, 1008, 1009, 1033, 986, 5, 1, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4059, 4244, 4039, 4174, 4048, 4068, 4042] avg=4096.0000 max=4244 max_rank=2 min=4039 min_rank=3 imbalance_ratio=1.036133 max_load_reduction=28524 max_load_reduction_pct=0.870483 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1042, 1023, 1023, 1037, 24, 16, 0, 0, 1026, 1028, 997, 1050, 13, 3, 1, 0, 1034, 1024, 1029, 1068, 0, 0, 0, 0, 979, 1011, 1000, 1041, 0, 0, 0, 0, 1026, 993, 996, 1002, 0, 0, 0, 0, 1013, 1032, 1025, 986, 0, 0, 0, 0, 1020, 1029, 1062, 1022, 0, 0, 0, 0, 1051, 1024, 1003, 1009, 5, 1, 0, 0] rerouted_tokens_per_ep_rank=[4165, 4118, 4155, 4031, 4017, 4056, 4133, 4093] avg=4096.0000 max=4165 max_rank=0 min=4017 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1046, 978, 1014, 1024, 25, 16, 0, 0, 1033, 1006, 1053, 1015, 13, 3, 1, 0, 1045, 1051, 960, 1066, 0, 0, 0, 0, 1047, 1024, 1034, 1014, 0, 0, 0, 0, 1043, 1066, 1009, 1023, 0, 0, 0, 0, 1052, 1002, 1065, 1003, 0, 0, 0, 0, 1009, 956, 1010, 1050, 0, 0, 0, 0, 1038, 977, 1018, 973, 5, 1, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4124, 4122, 4119, 4141, 4122, 4025, 4012] avg=4096.0000 max=4141 max_rank=4 min=4012 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[979, 1014, 1002, 1070, 30, 21, 0, 0, 1001, 1059, 1033, 1026, 16, 8, 4, 0, 1033, 1077, 973, 1063, 0, 0, 0, 0, 992, 1042, 1030, 1013, 0, 0, 0, 0, 1010, 988, 1019, 987, 0, 0, 0, 0, 1039, 985, 1036, 1043, 0, 0, 0, 0, 1069, 1051, 1036, 998, 0, 0, 0, 0, 997, 965, 990, 1057, 7, 5, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4147, 4146, 4077, 4004, 4103, 4154, 4021] avg=4096.0000 max=4154 max_rank=6 min=4004 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1010, 985, 1013, 981, 23, 16, 0, 0, 1050, 1039, 986, 996, 13, 3, 1, 0, 1027, 982, 1032, 1056, 0, 0, 0, 0, 969, 1094, 1081, 1020, 0, 0, 0, 0, 1032, 971, 1037, 1031, 0, 0, 0, 0, 1041, 967, 1008, 1047, 0, 0, 0, 0, 988, 1054, 1058, 1037, 0, 0, 0, 0, 1021, 1014, 1026, 1053, 5, 1, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4088, 4097, 4164, 4071, 4063, 4137, 4120] avg=4096.0000 max=4164 max_rank=3 min=4028 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1024, 1041, 1024, 1005, 24, 16, 0, 0, 1054, 988, 982, 1053, 13, 3, 1, 0, 1067, 1011, 961, 1026, 0, 0, 0, 0, 1026, 1016, 996, 1010, 0, 0, 0, 0, 997, 1051, 1014, 1054, 0, 0, 0, 0, 1033, 1001, 1039, 1031, 0, 0, 0, 0, 1029, 1023, 1024, 1014, 0, 0, 0, 0, 1032, 990, 1082, 1007, 5, 1, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4094, 4065, 4048, 4116, 4104, 4090, 4117] avg=4096.0000 max=4134 max_rank=0 min=4048 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8192, 256, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8064, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1018, 1002, 1026, 1034, 1038, 1052, 989, 1048, 1002, 1009, 1052, 1031, 1036, 1029, 992, 1026, 1035, 1093, 1021, 1069, 1009, 972, 980, 984, 1042, 1035, 996, 1013, 1079, 1048, 1011, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1071, 1067, 1014, 978, 999, 983, 984, 998, 1004, 1051, 1020, 1040, 1043, 1026, 1048, 1047, 1084, 1011, 1053, 998, 1034, 1044, 1010, 1005, 1055, 1000, 1026, 1008, 1073, 1010, 972, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1023, 1030, 1011, 1053, 1004, 1015, 1032, 1010, 1039, 1060, 977, 1028, 1014, 1072, 1080, 1067, 993, 980, 980, 1034, 1034, 1056, 1012, 985, 1022, 1018, 1031, 1068, 1022, 980, 1030, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[995, 947, 1062, 1042, 1015, 1045, 1025, 1056, 1050, 1041, 999, 1052, 994, 973, 983, 1005, 1008, 1075, 1016, 959, 1036, 1040, 1036, 1043, 994, 1002, 1022, 1037, 1054, 1105, 1038, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1021, 1052, 1066, 1026, 1022, 1067, 1003, 1039, 1067, 1015, 1014, 1081, 1003, 1009, 994, 991, 994, 945, 1017, 1016, 1026, 1054, 1062, 1027, 966, 1076, 1013, 1006, 1043, 1050, 979, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1013, 1027, 1009, 1049, 986, 1046, 1044, 1014, 985, 1028, 1009, 1008, 1015, 1051, 1013, 1029, 1070, 990, 1001, 1000, 1068, 1078, 1029, 1004, 983, 1016, 1065, 994, 1070, 1025, 1035, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1011, 1040, 1014, 1019, 1049, 1028, 1016, 1029, 1038, 1034, 1072, 1054, 1036, 1029, 1035, 985, 999, 1049, 1055, 981, 988, 992, 1021, 1050, 1030, 962, 1054, 1010, 1067, 1059, 974, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1010, 1060, 1018, 1046, 1029, 1060, 1007, 987, 1011, 1026, 1059, 990, 1021, 1065, 983, 1007, 1039, 1022, 1034, 1021, 1024, 1002, 990, 1056, 1022, 1032, 1000, 1043, 1038, 1021, 1051, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1018, 1002, 1026, 1024, 0, 0, 0, 0, 1038, 1052, 989, 1048, 6, 0, 0, 0, 1002, 1009, 1052, 1010, 0, 0, 0, 0, 1036, 1029, 992, 1026, 6, 2, 0, 0, 1035, 1093, 1021, 1069, 9, 6, 0, 0, 1009, 972, 980, 984, 2, 0, 0, 0, 1042, 1035, 996, 1013, 15, 0, 0, 0, 1064, 1048, 1011, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4133, 4073, 4091, 4233, 3947, 4101, 4120] avg=4096.0000 max=4233 max_rank=4 min=3947 min_rank=5 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1023, 1030, 1011, 1043, 0, 0, 0, 0, 1004, 1015, 1032, 1010, 6, 0, 0, 0, 1039, 1060, 977, 1007, 0, 0, 0, 0, 1014, 1072, 1080, 1067, 5, 2, 0, 0, 993, 980, 980, 1034, 9, 6, 0, 0, 1034, 1056, 1012, 985, 2, 0, 0, 0, 1022, 1018, 1031, 1068, 15, 0, 0, 0, 1008, 980, 1030, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4067, 4083, 4240, 4002, 4089, 4154, 4026] avg=4096.0000 max=4240 max_rank=3 min=4002 min_rank=4 imbalance_ratio=1.035156 max_load_reduction=28528 max_load_reduction_pct=0.870605 moved_to_echo=16271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1071, 1067, 1014, 969, 0, 0, 0, 0, 999, 983, 984, 998, 5, 0, 0, 0, 1004, 1051, 1020, 1019, 0, 0, 0, 0, 1043, 1026, 1048, 1047, 5, 2, 0, 0, 1084, 1011, 1053, 998, 9, 6, 0, 0, 1034, 1044, 1010, 1005, 2, 0, 0, 0, 1055, 1000, 1026, 1008, 15, 0, 0, 0, 1059, 1010, 972, 1012, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 3969, 4094, 4171, 4161, 4095, 4104, 4053] avg=4096.0000 max=4171 max_rank=3 min=3969 min_rank=1 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[995, 947, 1062, 1032, 0, 0, 0, 0, 1015, 1045, 1025, 1056, 6, 0, 0, 0, 1050, 1041, 999, 1030, 0, 0, 0, 0, 994, 973, 983, 1005, 5, 2, 0, 0, 1008, 1075, 1016, 959, 9, 6, 0, 0, 1036, 1040, 1036, 1043, 2, 0, 0, 0, 994, 1002, 1022, 1037, 16, 0, 0, 0, 1040, 1105, 1038, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4147, 4120, 3962, 4073, 4157, 4071, 4202] avg=4096.0000 max=4202 max_rank=7 min=3962 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1011, 1040, 1014, 1009, 0, 0, 0, 0, 1049, 1028, 1016, 1029, 6, 0, 0, 0, 1038, 1034, 1072, 1032, 0, 0, 0, 0, 1036, 1029, 1035, 985, 5, 2, 0, 0, 999, 1049, 1055, 981, 9, 6, 0, 0, 988, 992, 1021, 1050, 2, 0, 0, 0, 1030, 962, 1054, 1010, 16, 0, 0, 0, 1053, 1059, 974, 988, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4128, 4176, 4092, 4099, 4053, 4072, 4074] avg=4096.0000 max=4176 max_rank=2 min=4053 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1013, 1027, 1009, 1039, 0, 0, 0, 0, 986, 1046, 1044, 1014, 6, 0, 0, 0, 985, 1028, 1009, 987, 0, 0, 0, 0, 1015, 1051, 1013, 1029, 5, 2, 0, 0, 1070, 990, 1001, 1000, 9, 6, 0, 0, 1068, 1078, 1029, 1004, 2, 0, 0, 0, 983, 1016, 1065, 994, 15, 0, 0, 0, 1056, 1025, 1035, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4096, 4009, 4115, 4076, 4181, 4073, 4130] avg=4096.0000 max=4181 max_rank=5 min=4009 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1021, 1052, 1066, 1009, 0, 0, 0, 0, 1022, 1067, 1003, 1039, 8, 0, 0, 0, 1067, 1015, 1014, 1052, 0, 0, 0, 0, 1003, 1009, 994, 991, 11, 6, 0, 0, 994, 945, 1017, 1016, 12, 9, 0, 0, 1026, 1054, 1062, 1027, 3, 0, 0, 0, 966, 1076, 1013, 1006, 20, 0, 0, 0, 1020, 1050, 979, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4139, 4148, 4014, 3993, 4172, 4081, 4073] avg=4096.0000 max=4172 max_rank=5 min=3993 min_rank=4 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1010, 1060, 1018, 1036, 0, 0, 0, 0, 1029, 1060, 1007, 987, 6, 0, 0, 0, 1011, 1026, 1059, 969, 0, 0, 0, 0, 1021, 1065, 983, 1007, 5, 2, 0, 0, 1039, 1022, 1034, 1021, 9, 6, 0, 0, 1024, 1002, 990, 1056, 2, 0, 0, 0, 1022, 1032, 1000, 1043, 15, 0, 0, 0, 1024, 1021, 1051, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4089, 4065, 4083, 4131, 4074, 4112, 4090] avg=4096.0000 max=4131 max_rank=4 min=4065 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 8320, 8192, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[982, 1031, 1005, 1023, 986, 1027, 1070, 1020, 989, 1009, 1030, 985, 1056, 1027, 997, 1040, 1077, 1014, 1059, 994, 1042, 1002, 1029, 1031, 1034, 1005, 1040, 1060, 985, 1051, 1032, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[952, 1039, 1044, 1019, 1005, 1002, 1027, 999, 999, 996, 1028, 1023, 1046, 1037, 1037, 974, 1057, 1019, 1018, 1051, 1060, 1053, 1028, 1045, 1024, 1055, 1022, 1018, 994, 1017, 1034, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[996, 1028, 1037, 985, 1055, 1018, 988, 1015, 1000, 1050, 1032, 1033, 1037, 994, 1044, 1060, 1051, 990, 1012, 1023, 1056, 995, 1022, 965, 1021, 1065, 1033, 1043, 1002, 1011, 1076, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1062, 995, 1082, 1051, 970, 1037, 1009, 1008, 987, 991, 1040, 1021, 1040, 1057, 1009, 1028, 1021, 998, 1023, 1032, 1018, 979, 1031, 1005, 1019, 1040, 1030, 1012, 1054, 1053, 1047, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1032, 1038, 1026, 1008, 1002, 1023, 1025, 1005, 1067, 1043, 1007, 997, 1076, 1053, 982, 1033, 1035, 1033, 991, 1044, 996, 996, 1081, 1058, 1008, 1000, 1026, 1061, 1039, 985, 975, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1030, 1039, 993, 1008, 977, 1057, 1019, 1034, 992, 1022, 1012, 1022, 1022, 1026, 993, 1063, 1037, 1005, 1061, 997, 1039, 1004, 1022, 1049, 1032, 997, 1008, 1009, 1005, 1052, 1042, 1100] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[979, 1049, 1045, 987, 1030, 1034, 1024, 1010, 1027, 1056, 1024, 982, 1022, 977, 1085, 993, 1003, 1008, 1034, 957, 1038, 1028, 1025, 1029, 1012, 1042, 1066, 1021, 1028, 1053, 1052, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1027, 1039, 1006, 1019, 1061, 981, 1009, 1018, 1037, 1050, 1029, 1045, 994, 1022, 1030, 1051, 1011, 1052, 1054, 1020, 1025, 1046, 1003, 1051, 1012, 1049, 1002, 981, 981, 1003, 1047, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[982, 1031, 1005, 1023, 9, 2, 1, 0, 986, 1027, 1070, 1020, 20, 7, 0, 0, 989, 1009, 1030, 985, 10, 7, 0, 0, 1039, 1027, 997, 1040, 0, 0, 0, 0, 1076, 1014, 1059, 994, 0, 0, 0, 0, 1033, 1002, 1029, 1031, 0, 0, 0, 0, 1034, 996, 1040, 1060, 0, 0, 0, 0, 985, 1051, 1032, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4130, 4030, 4103, 4143, 4095, 4130, 4084] avg=4096.0000 max=4143 max_rank=4 min=4030 min_rank=2 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[952, 1039, 1044, 1019, 10, 2, 1, 0, 1005, 1002, 1027, 999, 20, 7, 0, 0, 999, 996, 1028, 1023, 10, 8, 0, 0, 1029, 1037, 1037, 974, 0, 0, 0, 0, 1056, 1019, 1018, 1051, 0, 0, 0, 0, 1050, 1053, 1028, 1045, 0, 0, 0, 0, 1024, 1045, 1022, 1018, 0, 0, 0, 0, 994, 1017, 1034, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4060, 4064, 4077, 4144, 4176, 4109, 4071] avg=4096.0000 max=4176 max_rank=5 min=4060 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1032, 1038, 1026, 1008, 9, 2, 1, 0, 1002, 1023, 1025, 1005, 20, 7, 0, 0, 1067, 1043, 1007, 997, 10, 7, 0, 0, 1059, 1053, 982, 1033, 0, 0, 0, 0, 1034, 1033, 991, 1044, 0, 0, 0, 0, 987, 996, 1081, 1058, 0, 0, 0, 0, 1008, 991, 1026, 1061, 0, 0, 0, 0, 1039, 985, 975, 1003, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4082, 4131, 4127, 4102, 4122, 4086, 4002] avg=4096.0000 max=4131 max_rank=2 min=4002 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[996, 1028, 1037, 985, 10, 2, 1, 0, 1055, 1018, 988, 1015, 20, 7, 0, 0, 1000, 1050, 1032, 1033, 10, 8, 0, 0, 1020, 994, 1044, 1060, 0, 0, 0, 0, 1050, 990, 1012, 1023, 0, 0, 0, 0, 1046, 995, 1022, 965, 0, 0, 0, 0, 1021, 1055, 1033, 1043, 0, 0, 0, 0, 1002, 1011, 1076, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4103, 4133, 4118, 4075, 4028, 4152, 4100] avg=4096.0000 max=4152 max_rank=6 min=4028 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1062, 995, 1082, 1051, 9, 2, 1, 0, 970, 1037, 1009, 1008, 20, 7, 0, 0, 987, 991, 1040, 1021, 10, 7, 0, 0, 1023, 1057, 1009, 1028, 0, 0, 0, 0, 1020, 998, 1023, 1032, 0, 0, 0, 0, 1009, 979, 1031, 1005, 0, 0, 0, 0, 1019, 1031, 1030, 1012, 0, 0, 0, 0, 1054, 1053, 1047, 999, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4202, 4051, 4056, 4117, 4073, 4024, 4092, 4153] avg=4096.0000 max=4202 max_rank=0 min=4024 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1030, 1039, 993, 1008, 9, 2, 1, 0, 977, 1057, 1019, 1034, 21, 7, 0, 0, 992, 1022, 1012, 1022, 9, 7, 0, 0, 1006, 1026, 993, 1063, 0, 0, 0, 0, 1036, 1005, 1061, 997, 0, 0, 0, 0, 1030, 1004, 1022, 1049, 0, 0, 0, 0, 1032, 988, 1008, 1009, 0, 0, 0, 0, 1005, 1052, 1042, 1079, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4115, 4064, 4088, 4099, 4105, 4037, 4178] avg=4096.0000 max=4178 max_rank=7 min=4037 min_rank=6 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[979, 1049, 1045, 987, 9, 2, 1, 0, 1030, 1034, 1024, 1010, 20, 7, 0, 0, 1027, 1056, 1024, 982, 9, 7, 0, 0, 1006, 977, 1085, 993, 0, 0, 0, 0, 1002, 1008, 1034, 957, 0, 0, 0, 0, 1029, 1028, 1025, 1029, 0, 0, 0, 0, 1012, 1033, 1066, 1021, 0, 0, 0, 0, 1028, 1053, 1052, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4125, 4105, 4061, 4001, 4111, 4132, 4161] avg=4096.0000 max=4161 max_rank=7 min=4001 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1027, 1039, 1006, 1019, 14, 6, 6, 0, 1061, 981, 1009, 1018, 25, 8, 0, 0, 1037, 1050, 1029, 1045, 12, 12, 0, 0, 974, 1022, 1030, 1051, 0, 0, 0, 0, 1005, 1052, 1054, 1020, 0, 0, 0, 0, 1007, 1046, 1003, 1051, 0, 0, 0, 0, 1012, 1035, 1002, 981, 0, 0, 0, 0, 981, 1003, 1047, 988, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4102, 4185, 4077, 4131, 4107, 4030, 4019] avg=4096.0000 max=4185 max_rank=2 min=4019 min_rank=7 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1020, 1014, 1018, 1086, 1037, 1020, 1018, 960, 1016, 1063, 1007, 1033, 1029, 1026, 1016, 986, 1017, 1016, 1028, 1036, 1009, 1021, 972, 1027, 1009, 1038, 1078, 1028, 1022, 1002, 1085, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[956, 1023, 1037, 990, 1040, 1047, 1065, 1048, 1036, 1011, 983, 1042, 1067, 983, 1077, 1086, 967, 1044, 1032, 991, 1049, 964, 1024, 1010, 1068, 1057, 1013, 1016, 1011, 1020, 1021, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1014, 1052, 1031, 1007, 1030, 1018, 979, 1047, 1034, 1061, 1033, 1008, 1017, 1020, 1009, 1023, 1040, 1005, 1053, 1039, 1039, 1044, 1060, 975, 998, 1047, 990, 988, 1059, 1008, 1031, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1044, 1010, 1044, 1047, 1046, 1043, 995, 1060, 1001, 1076, 1018, 995, 993, 991, 1034, 1071, 1017, 1016, 1019, 985, 1093, 1028, 1030, 971, 968, 1008, 1081, 1048, 977, 1018, 997, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1030, 1018, 1032, 1049, 1009, 967, 1056, 1091, 945, 996, 1008, 1061, 1049, 1007, 1050, 1026, 1038, 1057, 1034, 996, 993, 981, 1021, 1065, 984, 1011, 1067, 1036, 1040, 980, 1006, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1037, 1015, 1007, 1051, 1017, 1068, 1018, 1033, 1040, 1030, 1012, 1007, 1047, 1022, 1074, 1053, 1047, 1088, 1023, 994, 978, 995, 958, 1021, 1015, 1045, 1039, 975, 977, 1012, 1005, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[980, 1041, 1002, 1068, 984, 978, 1047, 1000, 1085, 1021, 1048, 1033, 1008, 1038, 1054, 991, 1035, 999, 1018, 1032, 993, 1027, 1046, 1011, 1087, 1034, 1008, 1016, 1000, 1056, 1028, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1040, 995, 1070, 1034, 1028, 1035, 1000, 1007, 971, 1010, 978, 990, 1026, 1013, 999, 1074, 1056, 1029, 1010, 967, 1018, 984, 1065, 1032, 1046, 1004, 1077, 1052, 1057, 1015, 1027, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1020, 1014, 1018, 1075, 0, 0, 0, 0, 1037, 1020, 1018, 958, 0, 0, 0, 0, 1016, 1063, 1007, 1033, 11, 3, 0, 0, 1029, 1026, 993, 986, 0, 0, 0, 0, 1017, 1016, 1028, 1036, 2, 2, 0, 0, 1009, 1021, 972, 1027, 23, 9, 0, 0, 1009, 1038, 1058, 1028, 0, 0, 0, 0, 1022, 1002, 1085, 1031, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4033, 4133, 4034, 4101, 4061, 4133, 4146] avg=4096.0000 max=4146 max_rank=7 min=4033 min_rank=1 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1044, 1010, 1044, 1036, 0, 0, 0, 0, 1046, 1043, 995, 1058, 0, 0, 0, 0, 1001, 1076, 1018, 995, 11, 3, 0, 0, 993, 991, 1011, 1071, 0, 0, 0, 0, 1017, 1016, 1019, 985, 2, 2, 0, 0, 1093, 1028, 1030, 971, 23, 9, 0, 0, 968, 1008, 1061, 1048, 0, 0, 0, 0, 977, 1018, 997, 1044, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4142, 4104, 4066, 4041, 4154, 4085, 4042] avg=4096.0000 max=4154 max_rank=5 min=4041 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[956, 1023, 1037, 979, 0, 0, 0, 0, 1040, 1047, 1065, 1046, 0, 0, 0, 0, 1036, 1011, 983, 1042, 10, 3, 0, 0, 1067, 983, 1053, 1086, 0, 0, 0, 0, 967, 1044, 1032, 991, 2, 2, 0, 0, 1049, 964, 1024, 1010, 24, 8, 0, 0, 1068, 1057, 995, 1016, 0, 0, 0, 0, 1011, 1020, 1021, 990, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[3995, 4198, 4085, 4189, 4038, 4079, 4136, 4048] avg=4096.0000 max=4198 max_rank=1 min=3995 min_rank=0 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1014, 1052, 1031, 996, 0, 0, 0, 0, 1030, 1018, 979, 1045, 0, 0, 0, 0, 1034, 1061, 1033, 1008, 10, 3, 0, 0, 1017, 1020, 986, 1023, 0, 0, 0, 0, 1040, 1005, 1053, 1039, 2, 2, 0, 0, 1039, 1044, 1060, 975, 23, 8, 0, 0, 998, 1047, 972, 988, 0, 0, 0, 0, 1059, 1008, 1031, 1009, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4072, 4149, 4046, 4141, 4149, 4005, 4113] avg=4096.0000 max=4149 max_rank=2 min=4005 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1037, 1015, 1007, 1040, 0, 0, 0, 0, 1017, 1068, 1018, 1031, 0, 0, 0, 0, 1040, 1030, 1012, 1007, 11, 3, 0, 0, 1047, 1022, 1050, 1053, 0, 0, 0, 0, 1047, 1088, 1023, 994, 2, 2, 0, 0, 978, 995, 958, 1021, 24, 9, 0, 0, 1015, 1045, 1019, 975, 0, 0, 0, 0, 977, 1012, 1005, 1065, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4134, 4103, 4172, 4156, 3985, 4054, 4065] avg=4096.0000 max=4172 max_rank=3 min=3985 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1030, 1018, 1032, 1038, 0, 0, 0, 0, 1009, 967, 1056, 1088, 0, 0, 0, 0, 945, 996, 1008, 1061, 11, 3, 0, 0, 1049, 1007, 1026, 1026, 0, 0, 0, 0, 1038, 1057, 1034, 996, 3, 2, 0, 0, 993, 981, 1021, 1065, 24, 9, 0, 0, 984, 1011, 1047, 1036, 0, 0, 0, 0, 1040, 980, 1006, 1065, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4120, 4024, 4108, 4130, 4093, 4078, 4097] avg=4096.0000 max=4130 max_rank=4 min=4024 min_rank=2 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[980, 1041, 1002, 1051, 0, 0, 0, 0, 984, 978, 1047, 992, 0, 0, 0, 0, 1085, 1021, 1048, 1033, 15, 5, 0, 0, 1008, 1038, 1026, 991, 0, 0, 0, 0, 1035, 999, 1018, 1032, 8, 3, 0, 0, 993, 1027, 1046, 1011, 28, 12, 0, 0, 1087, 1034, 981, 1016, 0, 0, 0, 0, 1000, 1056, 1028, 1000, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4001, 4207, 4063, 4095, 4117, 4118, 4093] avg=4096.0000 max=4207 max_rank=2 min=4001 min_rank=1 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1040, 995, 1070, 1023, 0, 0, 0, 0, 1028, 1035, 1000, 1005, 0, 0, 0, 0, 971, 1010, 978, 990, 11, 3, 0, 0, 1026, 1013, 977, 1074, 0, 0, 0, 0, 1056, 1029, 1010, 967, 2, 2, 0, 0, 1018, 984, 1065, 1032, 22, 9, 0, 0, 1046, 1004, 1057, 1052, 0, 0, 0, 0, 1057, 1015, 1027, 1059, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4068, 3963, 4090, 4066, 4130, 4159, 4164] avg=4096.0000 max=4164 max_rank=7 min=3963 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1044, 1028, 1066, 1028, 1022, 1007, 1046, 1050, 1003, 1018, 1013, 1019, 1017, 1057, 1037, 972, 971, 999, 1011, 1027, 1064, 981, 1040, 1036, 1022, 1014, 1034, 983, 1055, 1058, 1009, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1087, 1032, 1026, 1001, 1027, 1018, 1038, 1006, 982, 1037, 1061, 1063, 1039, 1020, 1073, 1031, 1054, 1026, 1010, 1045, 986, 980, 1034, 962, 984, 1046, 968, 1031, 1043, 1008, 995, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[993, 1037, 1063, 1035, 1022, 1017, 1014, 1005, 1059, 981, 1024, 1042, 1058, 1003, 980, 1014, 1025, 1045, 1056, 973, 1068, 1044, 1054, 1077, 981, 978, 1017, 988, 987, 1050, 1028, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1031, 1045, 1006, 994, 1017, 1006, 1018, 985, 1007, 1037, 1033, 977, 1040, 1067, 1030, 978, 1072, 1050, 1026, 1033, 1099, 1011, 1047, 1038, 992, 1022, 1013, 1010, 998, 1030, 1010, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1055, 1001, 1000, 1024, 1051, 979, 1045, 1031, 1011, 1056, 1034, 1019, 1052, 978, 996, 1048, 988, 1020, 1046, 1064, 1046, 1047, 1021, 994, 1028, 1014, 1047, 1010, 981, 1039, 1042, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1050, 1010, 1091, 1027, 1003, 1018, 999, 1023, 986, 978, 982, 1021, 990, 1066, 1045, 1024, 1024, 1041, 986, 1030, 1042, 997, 1057, 1010, 1064, 1039, 985, 1055, 1008, 1057, 1021, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1047, 1008, 985, 1040, 1018, 1043, 1069, 983, 1001, 962, 1029, 1037, 1060, 1033, 1051, 1044, 1044, 1029, 1040, 1068, 1013, 1058, 1010, 1037, 1016, 1017, 1021, 966, 1038, 988, 1001, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1076, 1022, 1017, 1001, 1053, 985, 1049, 1010, 1052, 1036, 970, 979, 1023, 1040, 1030, 1023, 991, 1034, 1026, 1020, 1081, 1035, 1011, 981, 1061, 1043, 1035, 1010, 1015, 1029, 964, 1066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1007, 1045, 1006, 994, 0, 0, 0, 0, 1017, 1006, 1018, 985, 12, 1, 0, 0, 1007, 1037, 1033, 977, 17, 15, 0, 0, 1022, 1067, 1030, 978, 0, 0, 0, 0, 1072, 1050, 1026, 1020, 0, 0, 0, 0, 1075, 1011, 1047, 1038, 0, 0, 0, 0, 992, 1022, 1013, 1010, 24, 9, 0, 0, 998, 1030, 1010, 1046, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4039, 4086, 4097, 4168, 4171, 4070, 4085] avg=4096.0000 max=4171 max_rank=5 min=4039 min_rank=1 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1019, 1028, 1066, 1028, 0, 0, 0, 0, 1022, 1007, 1046, 1050, 12, 1, 0, 0, 1003, 1018, 1013, 1019, 16, 15, 0, 0, 1000, 1057, 1037, 972, 0, 0, 0, 0, 971, 999, 1011, 1015, 0, 0, 0, 0, 1040, 981, 1040, 1036, 0, 0, 0, 0, 1022, 1014, 1034, 983, 25, 9, 0, 0, 1055, 1058, 1009, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4138, 4084, 4066, 3996, 4097, 4087, 4159] avg=4096.0000 max=4159 max_rank=7 min=3996 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[970, 1037, 1063, 1035, 0, 0, 0, 0, 1022, 1017, 1014, 1005, 11, 1, 0, 0, 1059, 981, 1024, 1042, 17, 15, 0, 0, 1040, 1003, 980, 1014, 0, 0, 0, 0, 1025, 1045, 1056, 962, 0, 0, 0, 0, 1044, 1044, 1054, 1077, 0, 0, 0, 0, 981, 978, 1017, 988, 23, 9, 0, 0, 987, 1050, 1028, 1050, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4070, 4138, 4037, 4088, 4219, 3996, 4115] avg=4096.0000 max=4219 max_rank=5 min=3996 min_rank=6 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1061, 1032, 1026, 1001, 0, 0, 0, 0, 1027, 1018, 1038, 1006, 12, 1, 0, 0, 982, 1037, 1061, 1063, 17, 14, 0, 0, 1021, 1020, 1073, 1031, 0, 0, 0, 0, 1054, 1026, 1010, 1032, 0, 0, 0, 0, 964, 980, 1034, 962, 0, 0, 0, 0, 984, 1046, 968, 1031, 26, 8, 0, 0, 1043, 1008, 995, 1055, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4102, 4174, 4145, 4122, 3940, 4063, 4102] avg=4096.0000 max=4174 max_rank=2 min=3940 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16227 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1030, 1001, 1000, 1024, 0, 0, 0, 0, 1051, 979, 1045, 1031, 12, 1, 0, 0, 1011, 1056, 1034, 1019, 17, 15, 0, 0, 1034, 978, 996, 1048, 0, 0, 0, 0, 988, 1020, 1046, 1051, 0, 0, 0, 0, 1023, 1047, 1021, 994, 0, 0, 0, 0, 1028, 1014, 1047, 1010, 25, 8, 0, 0, 981, 1039, 1042, 1001, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4119, 4152, 4056, 4105, 4085, 4132, 4064] avg=4096.0000 max=4152 max_rank=2 min=4055 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1021, 1010, 1091, 1027, 0, 0, 0, 0, 1003, 1018, 999, 1023, 15, 6, 0, 0, 986, 978, 982, 1021, 20, 18, 0, 0, 964, 1066, 1045, 1024, 0, 0, 0, 0, 1024, 1041, 986, 1011, 0, 0, 0, 0, 1012, 997, 1057, 1010, 0, 0, 0, 0, 1064, 1039, 985, 1055, 29, 12, 0, 0, 1008, 1057, 1021, 1039, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4064, 4005, 4099, 4062, 4076, 4184, 4129] avg=4096.0000 max=4184 max_rank=6 min=4005 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1051, 1022, 1017, 1001, 0, 0, 0, 0, 1053, 985, 1049, 1010, 12, 1, 0, 0, 1052, 1036, 970, 979, 17, 15, 0, 0, 1005, 1040, 1030, 1023, 0, 0, 0, 0, 991, 1034, 1026, 1008, 0, 0, 0, 0, 1057, 1035, 1011, 981, 0, 0, 0, 0, 1061, 1043, 1035, 1010, 25, 9, 0, 0, 1015, 1029, 964, 1066, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4110, 4069, 4098, 4059, 4084, 4183, 4074] avg=4096.0000 max=4183 max_rank=6 min=4059 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1022, 1008, 985, 1040, 0, 0, 0, 0, 1018, 1043, 1069, 983, 12, 1, 0, 0, 1001, 962, 1029, 1037, 17, 14, 0, 0, 1042, 1033, 1051, 1044, 0, 0, 0, 0, 1044, 1029, 1040, 1055, 0, 0, 0, 0, 991, 1058, 1010, 1037, 0, 0, 0, 0, 1016, 1017, 1021, 966, 25, 8, 0, 0, 1038, 988, 1001, 1012, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4126, 4060, 4170, 4168, 4096, 4053, 4040] avg=4096.0000 max=4170 max_rank=3 min=4040 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1026, 998, 1000, 1026, 1029, 1058, 1012, 1005, 1054, 1014, 1011, 1096, 1014, 1077, 961, 992, 1060, 1044, 1045, 1023, 1005, 1051, 996, 1008, 1053, 1067, 993, 1031, 988, 964, 1015, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1076, 977, 1036, 1017, 1018, 1013, 1099, 975, 1019, 1029, 995, 1063, 953, 1063, 1004, 1022, 1036, 1047, 1007, 1010, 989, 1072, 1053, 1039, 994, 1033, 1022, 960, 1016, 1050, 1034, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1004, 1052, 1050, 1019, 996, 1048, 1069, 1005, 999, 1002, 1029, 1034, 1023, 1032, 1010, 995, 985, 1032, 1004, 1051, 1024, 1008, 1015, 1013, 1046, 1032, 1015, 1016, 1014, 1042, 1071, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1006, 1081, 1009, 1042, 1025, 1031, 968, 1053, 999, 1012, 1038, 988, 1002, 1066, 1005, 1026, 1044, 1058, 1026, 1017, 1003, 1044, 990, 1059, 1006, 1051, 1047, 1011, 1009, 1025, 1014, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1042, 1018, 1012, 1037, 978, 1017, 1009, 992, 1047, 1009, 1027, 1067, 1072, 999, 1068, 1046, 1042, 1031, 1066, 1009, 1013, 998, 979, 1004, 1026, 1035, 1001, 1012, 1009, 1074, 1013, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1028, 960, 1048, 1007, 1069, 1079, 1058, 1048, 1023, 1027, 1062, 976, 1006, 981, 1058, 1016, 1061, 1031, 1024, 1041, 1037, 1052, 1023, 1018, 980, 1052, 986, 1003, 1014, 976, 1000, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1098, 1044, 1032, 1035, 996, 998, 1022, 1019, 1060, 993, 1009, 1016, 1014, 1083, 1061, 1029, 1041, 1006, 1038, 1008, 1037, 1043, 1036, 1024, 1032, 961, 1031, 1013, 988, 1014, 972, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1027, 1068, 995, 1052, 1010, 1061, 1063, 1046, 992, 1008, 1019, 1041, 1018, 1030, 1083, 1033, 1003, 1030, 993, 1029, 1046, 1007, 983, 1015, 979, 1034, 1000, 1008, 1034, 1035, 1010, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1057, 977, 1036, 1017, 0, 0, 0, 0, 1018, 1002, 1099, 975, 0, 0, 0, 0, 1019, 1029, 995, 1063, 1, 0, 0, 0, 953, 1054, 1004, 1022, 0, 0, 0, 0, 1036, 1025, 1007, 1010, 0, 0, 0, 0, 989, 1072, 1053, 1039, 8, 2, 0, 0, 994, 1033, 1022, 960, 22, 8, 0, 0, 1016, 1050, 1034, 1047, 11, 9, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4094, 4107, 4033, 4078, 4163, 4039, 4167] avg=4096.0000 max=4167 max_rank=7 min=4033 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1010, 960, 1048, 1007, 0, 0, 0, 0, 1069, 1067, 1058, 1048, 0, 0, 0, 0, 1023, 1027, 1062, 976, 1, 0, 0, 0, 1006, 973, 1058, 1016, 0, 0, 0, 0, 1061, 1010, 1024, 1041, 0, 0, 0, 0, 1037, 1052, 1023, 1018, 7, 2, 0, 0, 980, 1052, 986, 1003, 21, 7, 0, 0, 1014, 976, 1000, 1024, 11, 10, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4242, 4089, 4053, 4136, 4139, 4049, 4035] avg=4096.0000 max=4242 max_rank=1 min=4025 min_rank=0 imbalance_ratio=1.035645 max_load_reduction=28526 max_load_reduction_pct=0.870544 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1008, 998, 1000, 1026, 0, 0, 0, 0, 1029, 1046, 1012, 1005, 0, 0, 0, 0, 1054, 1014, 1011, 1096, 1, 0, 0, 0, 1014, 1068, 961, 992, 0, 0, 0, 0, 1060, 1023, 1045, 1023, 0, 0, 0, 0, 1005, 1051, 996, 1008, 8, 2, 0, 0, 1053, 1067, 993, 1031, 21, 7, 0, 0, 988, 964, 1015, 1052, 11, 10, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4092, 4176, 4035, 4151, 4070, 4172, 4040] avg=4096.0000 max=4176 max_rank=2 min=4032 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[989, 1081, 1009, 1042, 0, 0, 0, 0, 1025, 1019, 968, 1053, 0, 0, 0, 0, 999, 1012, 1038, 988, 1, 0, 0, 0, 1002, 1057, 1005, 1026, 0, 0, 0, 0, 1044, 1036, 1026, 1017, 0, 0, 0, 0, 1003, 1044, 990, 1059, 8, 2, 0, 0, 1006, 1051, 1047, 1011, 22, 7, 0, 0, 1009, 1025, 1014, 1013, 10, 10, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4065, 4038, 4090, 4123, 4106, 4144, 4081] avg=4096.0000 max=4144 max_rank=6 min=4038 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[987, 1052, 1050, 1019, 0, 0, 0, 0, 996, 1036, 1069, 1005, 0, 0, 0, 0, 999, 1002, 1029, 1034, 1, 0, 0, 0, 1023, 1024, 1010, 995, 0, 0, 0, 0, 985, 1011, 1004, 1051, 0, 0, 0, 0, 1024, 1008, 1015, 1013, 7, 2, 0, 0, 1046, 1032, 1015, 1016, 21, 7, 0, 0, 1014, 1042, 1071, 1033, 10, 10, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4106, 4065, 4052, 4051, 4069, 4137, 4180] avg=4096.0000 max=4180 max_rank=7 min=4051 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1023, 1018, 1012, 1037, 0, 0, 0, 0, 978, 1006, 1009, 992, 0, 0, 0, 0, 1047, 1009, 1027, 1067, 1, 0, 0, 0, 1072, 991, 1068, 1046, 0, 0, 0, 0, 1042, 1010, 1066, 1009, 0, 0, 0, 0, 1013, 998, 979, 1004, 7, 2, 0, 0, 1026, 1035, 1001, 1012, 21, 8, 0, 0, 1009, 1074, 1013, 1016, 11, 9, 0, 0] rerouted_tokens_per_ep_rank=[4090, 3985, 4151, 4177, 4127, 4003, 4103, 4132] avg=4096.0000 max=4177 max_rank=3 min=3985 min_rank=1 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1070, 1044, 1032, 1035, 0, 0, 0, 0, 996, 979, 1022, 1019, 0, 0, 0, 0, 1060, 993, 1009, 1016, 3, 0, 0, 0, 1014, 1068, 1061, 1029, 0, 0, 0, 0, 1041, 981, 1038, 1008, 0, 0, 0, 0, 1037, 1043, 1036, 1024, 12, 6, 0, 0, 1032, 961, 1031, 1013, 25, 13, 0, 0, 988, 1014, 972, 1015, 15, 13, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4016, 4081, 4172, 4068, 4158, 4075, 4017] avg=4096.0000 max=4181 max_rank=0 min=4016 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1009, 1068, 995, 1052, 0, 0, 0, 0, 1010, 1049, 1063, 1046, 0, 0, 0, 0, 992, 1008, 1019, 1041, 1, 0, 0, 0, 1018, 1022, 1083, 1033, 0, 0, 0, 0, 1003, 1009, 993, 1029, 0, 0, 0, 0, 1046, 1007, 983, 1015, 7, 2, 0, 0, 979, 1034, 1000, 1008, 21, 7, 0, 0, 1034, 1035, 1010, 1016, 11, 10, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4168, 4061, 4156, 4034, 4060, 4049, 4116] avg=4096.0000 max=4168 max_rank=1 min=4034 min_rank=4 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1056, 1031, 1048, 1010, 1056, 1025, 1034, 982, 1004, 970, 1056, 1031, 1013, 1026, 1006, 1008, 1019, 1032, 1048, 1005, 1054, 1022, 1031, 1058, 1020, 1006, 1027, 1012, 1019, 984, 1073, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[998, 999, 996, 994, 1002, 1035, 1021, 982, 1010, 971, 1070, 1008, 1082, 977, 1036, 1043, 1046, 992, 996, 1089, 1022, 980, 1011, 1072, 1035, 1044, 1034, 1072, 1038, 1076, 1019, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1028, 1027, 1037, 1001, 1019, 987, 1043, 1015, 982, 1061, 1039, 970, 993, 999, 1092, 1029, 1061, 1030, 1045, 1051, 1013, 997, 1031, 1004, 1061, 984, 1006, 1054, 1023, 1044, 1006, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[977, 1024, 1061, 1068, 984, 1071, 1021, 1027, 1024, 1021, 1027, 1023, 998, 1040, 1029, 1050, 1015, 1047, 1075, 999, 1011, 1020, 1051, 1005, 1005, 1032, 1027, 1018, 1020, 993, 1026, 979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1005, 1071, 1040, 1020, 1027, 1021, 1091, 1037, 978, 1016, 1020, 1046, 1056, 1030, 1014, 1000, 996, 1027, 1025, 1001, 1035, 1045, 1050, 1021, 1047, 1028, 997, 1008, 1039, 984, 986, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1050, 1028, 1043, 1024, 1005, 1033, 1006, 1013, 1056, 999, 1015, 1023, 1024, 1005, 1026, 1028, 1035, 1019, 1080, 1018, 1068, 999, 1006, 1017, 1069, 962, 1008, 990, 1038, 1034, 1051, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[984, 1035, 960, 1016, 1002, 992, 1013, 1010, 1015, 1033, 1046, 1017, 1022, 1061, 1068, 1065, 1056, 1046, 1055, 1023, 1004, 1015, 1047, 1029, 998, 1039, 1003, 963, 1064, 1012, 1047, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1054, 1068, 1011, 1008, 1039, 1097, 1015, 1007, 932, 1030, 1000, 1058, 999, 1037, 1011, 1024, 1063, 1011, 1013, 1019, 960, 1028, 1046, 999, 1045, 1039, 1024, 1015, 1010, 1061, 1025, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1056, 1031, 1048, 1010, 0, 0, 0, 0, 1056, 1025, 1034, 982, 6, 0, 0, 0, 1004, 970, 1056, 1031, 27, 0, 0, 0, 1013, 1026, 993, 1008, 0, 0, 0, 0, 1019, 1032, 1015, 1005, 0, 0, 0, 0, 1054, 1022, 1031, 1058, 2, 0, 0, 0, 1020, 1006, 1027, 1012, 6, 5, 0, 0, 1019, 984, 1073, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4103, 4088, 4040, 4071, 4167, 4076, 4078] avg=4096.0000 max=4167 max_rank=5 min=4040 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1028, 1027, 1037, 1001, 0, 0, 0, 0, 1019, 987, 1043, 1015, 7, 0, 0, 0, 982, 1061, 1039, 970, 27, 0, 0, 0, 993, 999, 1078, 1029, 0, 0, 0, 0, 1061, 1030, 1012, 1051, 0, 0, 0, 0, 1013, 997, 1031, 1004, 2, 0, 0, 0, 1061, 984, 1006, 1054, 6, 5, 0, 0, 1023, 1044, 1006, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4071, 4079, 4099, 4154, 4047, 4116, 4109] avg=4096.0000 max=4154 max_rank=4 min=4047 min_rank=5 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1005, 1071, 1040, 1020, 0, 0, 0, 0, 1027, 1021, 1091, 1037, 6, 0, 0, 0, 978, 1016, 1020, 1046, 26, 0, 0, 0, 1056, 1030, 1001, 1000, 0, 0, 0, 0, 996, 1027, 993, 1001, 0, 0, 0, 0, 1035, 1045, 1050, 1021, 2, 0, 0, 0, 1047, 1028, 997, 1008, 6, 5, 0, 0, 1039, 984, 986, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4182, 4086, 4087, 4017, 4153, 4091, 4016] avg=4096.0000 max=4182 max_rank=1 min=4016 min_rank=7 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[998, 999, 996, 994, 0, 0, 0, 0, 1002, 1035, 1021, 982, 7, 0, 0, 0, 1010, 971, 1070, 1008, 25, 0, 0, 0, 1082, 977, 1022, 1043, 0, 0, 0, 0, 1046, 992, 965, 1089, 0, 0, 0, 0, 1022, 980, 1011, 1072, 2, 0, 0, 0, 1035, 1044, 1034, 1072, 6, 5, 0, 0, 1038, 1076, 1019, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3987, 4047, 4084, 4124, 4092, 4087, 4196, 4151] avg=4096.0000 max=4196 max_rank=6 min=3987 min_rank=0 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16526 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[984, 1035, 960, 1016, 0, 0, 0, 0, 1002, 992, 1013, 1010, 7, 0, 0, 0, 1015, 1033, 1046, 1017, 27, 0, 0, 0, 1022, 1061, 1054, 1065, 0, 0, 0, 0, 1056, 1046, 1022, 1023, 0, 0, 0, 0, 1004, 1015, 1047, 1029, 2, 0, 0, 0, 998, 1039, 1003, 963, 6, 5, 0, 0, 1064, 1012, 1047, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3995, 4024, 4138, 4202, 4147, 4097, 4014, 4151] avg=4096.0000 max=4202 max_rank=3 min=3995 min_rank=0 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1054, 1068, 1011, 1008, 0, 0, 0, 0, 1039, 1097, 1015, 1007, 6, 0, 0, 0, 932, 1030, 1000, 1058, 26, 0, 0, 0, 999, 1037, 998, 1024, 0, 0, 0, 0, 1063, 1011, 981, 1019, 0, 0, 0, 0, 960, 1028, 1046, 999, 2, 0, 0, 0, 1045, 1039, 1024, 1015, 6, 5, 0, 0, 1010, 1061, 1025, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4164, 4046, 4058, 4074, 4035, 4134, 4116] avg=4096.0000 max=4164 max_rank=1 min=4035 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[977, 1020, 1061, 1068, 0, 0, 0, 0, 984, 1071, 1021, 1027, 11, 0, 0, 0, 1024, 1021, 1027, 1023, 31, 0, 0, 0, 998, 1040, 1000, 1050, 0, 0, 0, 0, 1015, 1047, 1034, 999, 0, 0, 0, 0, 1011, 1020, 1051, 1005, 3, 0, 0, 0, 1005, 1032, 1027, 1018, 10, 9, 0, 0, 1020, 993, 1026, 979, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4114, 4126, 4088, 4095, 4090, 4101, 4028] avg=4096.0000 max=4126 max_rank=0 min=4028 min_rank=7 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1050, 1028, 1043, 1024, 0, 0, 0, 0, 1005, 1033, 1006, 1013, 6, 0, 0, 0, 1056, 999, 1015, 1023, 28, 0, 0, 0, 1024, 1005, 1013, 1028, 0, 0, 0, 0, 1035, 1019, 1046, 1018, 0, 0, 0, 0, 1068, 999, 1006, 1017, 2, 0, 0, 0, 1069, 962, 1008, 990, 6, 5, 0, 0, 1038, 1034, 1051, 996, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4063, 4121, 4070, 4118, 4092, 4040, 4119] avg=4096.0000 max=4145 max_rank=0 min=4040 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1035, 979, 984, 982, 1017, 1032, 1027, 1042, 1019, 976, 1026, 1010, 1015, 1050, 1038, 1017, 1052, 1034, 1045, 1005, 1025, 1046, 1016, 1043, 1050, 1044, 995, 1080, 1021, 1025, 1007, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1024, 1047, 1030, 1021, 1014, 1073, 1023, 1007, 1024, 1021, 1106, 1028, 1047, 970, 924, 998, 1040, 1056, 1041, 1028, 1020, 1062, 1068, 992, 978, 1062, 1009, 976, 1023, 1000, 1037, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1049, 1051, 1022, 1040, 1009, 1005, 1022, 1038, 982, 1029, 1022, 1011, 1032, 1023, 1026, 1044, 992, 1026, 1065, 999, 987, 998, 1068, 1042, 1048, 1066, 1013, 1010, 991, 1051, 986, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1007, 1017, 1087, 1052, 1019, 1087, 1037, 1055, 1035, 1070, 981, 1022, 1030, 1003, 1004, 1041, 1038, 1003, 975, 996, 1004, 995, 1029, 1001, 1094, 1035, 966, 1013, 1041, 992, 1034, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1040, 1018, 1007, 973, 1003, 1036, 1023, 1058, 1011, 1020, 1023, 1049, 1068, 1013, 998, 1056, 1026, 989, 1029, 1045, 1023, 987, 991, 1025, 1039, 1011, 1095, 1034, 1000, 1026, 1040, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1097, 1051, 1020, 1024, 1019, 1027, 1044, 1035, 986, 1046, 995, 1059, 1079, 1020, 1054, 1009, 1023, 997, 1017, 1024, 1011, 1037, 1022, 1006, 1028, 979, 1010, 997, 1005, 995, 1047, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[973, 1077, 1068, 1019, 1007, 1009, 1003, 1053, 977, 1010, 1042, 1016, 1046, 1044, 1023, 1004, 1034, 1030, 1053, 1073, 989, 978, 1015, 1061, 1044, 1053, 980, 1012, 1025, 1020, 982, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1025, 1014, 1001, 996, 1021, 1027, 1047, 1016, 1003, 1048, 950, 1077, 1035, 1048, 1030, 1047, 1029, 936, 1031, 1020, 1024, 1009, 1015, 1032, 1056, 1069, 1024, 1040, 1014, 1027, 1021, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1024, 1040, 1030, 1021, 0, 0, 0, 0, 1014, 1073, 1023, 987, 0, 0, 0, 0, 1024, 1021, 1106, 1028, 6, 5, 0, 0, 1039, 970, 924, 998, 0, 0, 0, 0, 1040, 1056, 1041, 1028, 2, 0, 0, 0, 1020, 1062, 1068, 992, 15, 2, 0, 0, 962, 1062, 1009, 976, 0, 0, 0, 0, 1023, 1000, 1037, 1019, 20, 1, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4097, 4190, 3931, 4167, 4159, 4009, 4100] avg=4096.0000 max=4190 max_rank=2 min=3931 min_rank=3 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1049, 1044, 1022, 1040, 0, 0, 0, 0, 1009, 1005, 1022, 1018, 0, 0, 0, 0, 982, 1029, 1022, 1011, 6, 5, 0, 0, 1024, 1023, 1026, 1044, 0, 0, 0, 0, 992, 1026, 1065, 999, 2, 0, 0, 0, 987, 998, 1068, 1042, 16, 2, 0, 0, 1031, 1066, 1013, 1010, 0, 0, 0, 0, 991, 1051, 986, 1021, 20, 1, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4054, 4055, 4117, 4084, 4113, 4120, 4070] avg=4096.0000 max=4155 max_rank=0 min=4054 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1007, 1010, 1087, 1052, 0, 0, 0, 0, 1019, 1087, 1037, 1034, 0, 0, 0, 0, 1035, 1070, 981, 1022, 6, 5, 0, 0, 1022, 1003, 1004, 1041, 0, 0, 0, 0, 1038, 1003, 975, 996, 2, 0, 0, 0, 1004, 995, 1029, 1001, 16, 2, 0, 0, 1077, 1035, 966, 1013, 0, 0, 0, 0, 1041, 992, 1034, 1005, 21, 1, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4177, 4119, 4070, 4014, 4047, 4091, 4094] avg=4096.0000 max=4177 max_rank=1 min=4014 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1035, 966, 984, 982, 0, 0, 0, 0, 1017, 1032, 1027, 1018, 0, 0, 0, 0, 1019, 976, 1026, 1010, 7, 10, 0, 0, 1003, 1050, 1038, 1017, 0, 0, 0, 0, 1052, 1034, 1045, 1005, 3, 0, 0, 0, 1025, 1046, 1016, 1043, 19, 5, 0, 0, 1024, 1044, 995, 1080, 0, 0, 0, 0, 1021, 1025, 1007, 1031, 24, 7, 0, 0] rerouted_tokens_per_ep_rank=[3967, 4094, 4048, 4108, 4139, 4154, 4143, 4115] avg=4096.0000 max=4154 max_rank=5 min=3967 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1097, 1044, 1020, 1024, 0, 0, 0, 0, 1019, 1027, 1044, 1015, 0, 0, 0, 0, 986, 1046, 995, 1059, 6, 5, 0, 0, 1071, 1020, 1054, 1009, 0, 0, 0, 0, 1023, 997, 1017, 1024, 2, 0, 0, 0, 1011, 1037, 1022, 1006, 15, 2, 0, 0, 1012, 979, 1010, 997, 0, 0, 0, 0, 1005, 995, 1047, 1005, 20, 1, 0, 0] rerouted_tokens_per_ep_rank=[4185, 4105, 4097, 4154, 4063, 4093, 3998, 4073] avg=4096.0000 max=4185 max_rank=0 min=3998 min_rank=6 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16227 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[973, 1070, 1068, 1019, 0, 0, 0, 0, 1007, 1009, 1003, 1032, 0, 0, 0, 0, 977, 1010, 1042, 1016, 6, 5, 0, 0, 1038, 1044, 1023, 1004, 0, 0, 0, 0, 1034, 1030, 1053, 1073, 2, 0, 0, 0, 989, 978, 1015, 1061, 16, 2, 0, 0, 1027, 1053, 980, 1012, 0, 0, 0, 0, 1025, 1020, 982, 1048, 21, 1, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4051, 4056, 4109, 4192, 4061, 4072, 4097] avg=4096.0000 max=4192 max_rank=4 min=4051 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1040, 1011, 1007, 973, 0, 0, 0, 0, 1003, 1036, 1023, 1037, 0, 0, 0, 0, 1011, 1020, 1023, 1049, 6, 5, 0, 0, 1060, 1013, 998, 1056, 0, 0, 0, 0, 1026, 989, 1029, 1045, 2, 0, 0, 0, 1023, 987, 991, 1025, 15, 2, 0, 0, 1023, 1011, 1095, 1034, 0, 0, 0, 0, 1000, 1026, 1040, 1012, 21, 1, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4099, 4114, 4127, 4091, 4043, 4163, 4100] avg=4096.0000 max=4163 max_rank=6 min=4031 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1025, 1007, 1001, 996, 0, 0, 0, 0, 1021, 1027, 1047, 996, 0, 0, 0, 0, 1003, 1048, 950, 1077, 6, 5, 0, 0, 1027, 1048, 1030, 1047, 0, 0, 0, 0, 1029, 936, 1031, 1020, 2, 0, 0, 0, 1024, 1009, 1015, 1032, 16, 2, 0, 0, 1039, 1069, 1024, 1040, 0, 0, 0, 0, 1014, 1027, 1021, 1036, 20, 1, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4091, 4089, 4152, 4018, 4098, 4172, 4119] avg=4096.0000 max=4172 max_rank=6 min=4018 min_rank=4 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1043, 1008, 1083, 1017, 1058, 1006, 978, 973, 1046, 1020, 1039, 1027, 996, 1050, 1000, 979, 1045, 1029, 982, 1031, 1027, 1004, 1037, 1060, 1095, 986, 1022, 1029, 973, 1036, 1076, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1006, 978, 1076, 1035, 1030, 1060, 1026, 1006, 998, 1013, 1046, 1023, 1034, 1074, 1003, 1021, 1073, 979, 1007, 992, 1033, 1027, 1054, 1047, 1050, 987, 1021, 1040, 1032, 1039, 985, 973] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1082, 1009, 1029, 994, 1036, 1052, 978, 979, 1027, 1038, 1017, 1026, 1057, 1051, 1075, 1029, 996, 1037, 964, 977, 1046, 1046, 999, 993, 1017, 1024, 1022, 1047, 1037, 1024, 1035, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[997, 1068, 971, 1007, 1007, 1035, 1007, 1005, 1041, 1059, 1013, 1028, 997, 1039, 1051, 1106, 1015, 1027, 1014, 1064, 983, 1039, 983, 1079, 1040, 992, 1018, 995, 1023, 1021, 1018, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1021, 998, 1059, 1019, 1043, 1053, 1025, 1004, 1010, 1069, 1042, 1026, 1009, 1034, 1028, 1051, 1011, 1015, 1020, 1037, 989, 981, 1024, 1014, 1018, 1033, 996, 1014, 1052, 1026, 983, 1064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1044, 1094, 1047, 1007, 998, 1040, 1016, 986, 1057, 1025, 1062, 1086, 1015, 1014, 1020, 1039, 1027, 997, 1004, 1017, 1036, 1052, 1028, 997, 1027, 1026, 967, 1019, 1026, 1006, 1007, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[964, 1002, 989, 1036, 1055, 1039, 1030, 1029, 1002, 1036, 1021, 1018, 1025, 1034, 991, 1005, 1072, 1039, 1063, 1017, 1017, 1032, 985, 1059, 1037, 1019, 1001, 1013, 999, 1037, 1039, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1017, 1034, 1037, 1010, 1014, 1012, 1034, 1041, 1029, 1021, 963, 988, 1004, 1082, 1009, 1012, 992, 1050, 1074, 1033, 958, 1045, 1025, 1040, 995, 1077, 1024, 1037, 1043, 996, 1043, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1043, 1008, 1082, 1017, 0, 0, 0, 0, 1058, 1006, 978, 973, 14, 0, 0, 0, 1046, 1004, 1039, 1027, 0, 0, 0, 0, 996, 1030, 1000, 979, 0, 0, 0, 0, 1045, 1029, 982, 1031, 8, 0, 0, 0, 1027, 1004, 1037, 1060, 1, 1, 0, 0, 1095, 986, 1022, 1029, 6, 3, 0, 0, 973, 1036, 1076, 1013, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4029, 4116, 4005, 4095, 4130, 4141, 4102] avg=4096.0000 max=4150 max_rank=0 min=4005 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[997, 1068, 970, 1007, 0, 0, 0, 0, 1007, 1035, 1007, 1005, 14, 0, 0, 0, 1041, 1042, 1013, 1028, 0, 0, 0, 0, 997, 1019, 1051, 1106, 0, 0, 0, 0, 1015, 1027, 1014, 1064, 8, 0, 0, 0, 983, 1039, 983, 1079, 2, 1, 0, 0, 1040, 992, 1018, 995, 6, 3, 0, 0, 1023, 1021, 1018, 1026, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4068, 4124, 4173, 4128, 4087, 4054, 4092] avg=4096.0000 max=4173 max_rank=3 min=4042 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1082, 1009, 1028, 994, 0, 0, 0, 0, 1036, 1052, 978, 979, 14, 0, 0, 0, 1027, 1021, 1017, 1026, 0, 0, 0, 0, 1057, 1031, 1075, 1029, 0, 0, 0, 0, 996, 1037, 964, 977, 8, 0, 0, 0, 1046, 1046, 999, 993, 2, 1, 0, 0, 1017, 1024, 1022, 1047, 6, 3, 0, 0, 1037, 1024, 1035, 1025, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4059, 4091, 4192, 3982, 4087, 4119, 4125] avg=4096.0000 max=4192 max_rank=3 min=3982 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1006, 978, 1075, 1035, 0, 0, 0, 0, 1030, 1060, 1026, 1006, 14, 0, 0, 0, 998, 997, 1046, 1023, 0, 0, 0, 0, 1034, 1054, 1003, 1021, 0, 0, 0, 0, 1073, 979, 1007, 992, 8, 0, 0, 0, 1033, 1027, 1054, 1047, 1, 1, 0, 0, 1050, 987, 1021, 1040, 6, 3, 0, 0, 1032, 1039, 985, 973, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4136, 4064, 4112, 4059, 4163, 4107, 4033] avg=4096.0000 max=4163 max_rank=5 min=4033 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[964, 1002, 988, 1036, 0, 0, 0, 0, 1055, 1039, 1030, 1029, 13, 0, 0, 0, 1002, 1019, 1021, 1018, 0, 0, 0, 0, 1025, 1015, 991, 1005, 0, 0, 0, 0, 1072, 1039, 1063, 1017, 8, 0, 0, 0, 1017, 1032, 985, 1059, 2, 1, 0, 0, 1037, 1019, 1001, 1013, 6, 3, 0, 0, 999, 1037, 1039, 1063, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[3990, 4166, 4060, 4036, 4199, 4096, 4079, 4142] avg=4096.0000 max=4199 max_rank=4 min=3990 min_rank=0 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1021, 998, 1058, 1019, 0, 0, 0, 0, 1043, 1053, 1025, 1004, 13, 0, 0, 0, 1010, 1052, 1042, 1026, 0, 0, 0, 0, 1009, 1015, 1028, 1051, 0, 0, 0, 0, 1011, 1015, 1020, 1037, 8, 0, 0, 0, 989, 981, 1024, 1014, 2, 1, 0, 0, 1018, 1033, 996, 1014, 6, 3, 0, 0, 1052, 1026, 983, 1064, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4138, 4130, 4103, 4091, 4011, 4070, 4129] avg=4096.0000 max=4138 max_rank=1 min=4011 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1044, 1094, 1041, 1007, 0, 0, 0, 0, 998, 1040, 1016, 986, 17, 0, 0, 0, 1057, 993, 1062, 1086, 0, 0, 0, 0, 1015, 986, 1020, 1039, 0, 0, 0, 0, 1027, 997, 1004, 1017, 12, 0, 0, 0, 1036, 1052, 1028, 997, 5, 6, 0, 0, 1027, 1026, 967, 1019, 11, 6, 0, 0, 1026, 1006, 1007, 982, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4057, 4198, 4060, 4057, 4124, 4056, 4030] avg=4096.0000 max=4198 max_rank=2 min=4030 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16267 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1017, 1034, 1036, 1010, 0, 0, 0, 0, 1014, 1012, 1034, 1041, 14, 0, 0, 0, 1029, 1005, 963, 988, 0, 0, 0, 0, 1004, 1062, 1009, 1012, 0, 0, 0, 0, 992, 1050, 1074, 1033, 8, 0, 0, 0, 958, 1045, 1025, 1040, 1, 1, 0, 0, 995, 1077, 1024, 1037, 6, 3, 0, 0, 1043, 996, 1043, 1029, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4115, 3985, 4087, 4157, 4070, 4142, 4115] avg=4096.0000 max=4157 max_rank=4 min=3985 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1041, 1060, 1064, 967, 980, 998, 1057, 1025, 1006, 1003, 1096, 1034, 1045, 993, 1009, 1053, 1047, 1018, 1040, 1035, 983, 1018, 978, 992, 1012, 1036, 995, 1013, 1026, 1064, 1063, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1026, 1011, 961, 1028, 1035, 1042, 1039, 1053, 1054, 1021, 986, 1051, 987, 1007, 1086, 1030, 979, 1053, 983, 1094, 1076, 997, 1016, 1019, 1009, 1032, 1033, 997, 1016, 1054, 1005, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1060, 1077, 975, 1047, 1030, 1046, 1059, 1045, 1017, 1012, 971, 1011, 1007, 1011, 1011, 1005, 1033, 1058, 1030, 975, 1020, 1029, 1023, 1065, 1061, 1006, 974, 977, 1098, 975, 1042, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1019, 1058, 1033, 1024, 1007, 986, 1009, 1051, 986, 1055, 1010, 997, 1024, 1021, 1006, 1074, 1055, 1030, 1033, 1005, 1065, 1032, 1009, 1041, 997, 997, 991, 1020, 1019, 1058, 1029, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[978, 1023, 1039, 1055, 1025, 1042, 1027, 1037, 1021, 1023, 1030, 1017, 987, 1014, 1054, 1011, 1017, 985, 991, 1027, 1065, 1078, 1010, 1023, 999, 1047, 978, 1045, 1111, 979, 994, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1018, 993, 1044, 1044, 1031, 1008, 1022, 1017, 996, 1026, 1037, 1010, 1019, 1038, 1060, 1004, 999, 1018, 1030, 1050, 1024, 1005, 1047, 1015, 1091, 1013, 1018, 1060, 1015, 972, 1023, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1016, 1013, 979, 1019, 1017, 1052, 963, 1025, 1067, 1044, 1019, 1052, 1057, 1043, 1047, 989, 999, 982, 1026, 1039, 975, 1048, 999, 1027, 1046, 1041, 1038, 1030, 1027, 1030, 1021, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1013, 966, 1007, 1016, 990, 1059, 1025, 1047, 1031, 1050, 1027, 995, 1026, 1042, 1012, 1071, 980, 1043, 1059, 1049, 1031, 1025, 1033, 1066, 1058, 1050, 985, 1004, 1002, 1010, 969, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1041, 1060, 1064, 967, 6, 4, 0, 0, 980, 998, 1057, 1015, 0, 0, 0, 0, 1006, 1003, 1096, 1034, 1, 0, 0, 0, 1045, 993, 1001, 1053, 0, 0, 0, 0, 1047, 1018, 1040, 1035, 0, 0, 0, 0, 983, 1018, 978, 985, 0, 0, 0, 0, 1012, 1036, 995, 1013, 10, 4, 0, 0, 1026, 1064, 1063, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4050, 4140, 4092, 4140, 3964, 4070, 4170] avg=4096.0000 max=4170 max_rank=7 min=3964 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16344 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1026, 1011, 961, 1028, 6, 5, 0, 0, 1035, 1042, 1039, 1043, 0, 0, 0, 0, 1054, 1021, 986, 1051, 1, 0, 0, 0, 987, 1007, 1077, 1030, 0, 0, 0, 0, 979, 1053, 983, 1094, 0, 0, 0, 0, 1076, 997, 1016, 1012, 0, 0, 0, 0, 1009, 1032, 1033, 997, 10, 4, 0, 0, 1016, 1054, 1005, 988, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4159, 4113, 4101, 4109, 4101, 4085, 4063] avg=4096.0000 max=4159 max_rank=1 min=4037 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16358 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1060, 1077, 975, 1047, 6, 5, 0, 0, 1030, 1046, 1059, 1035, 0, 0, 0, 0, 1017, 1012, 971, 1011, 1, 0, 0, 0, 1007, 1011, 1002, 1005, 0, 0, 0, 0, 1033, 1058, 1030, 975, 0, 0, 0, 0, 1020, 1029, 1023, 1058, 0, 0, 0, 0, 1061, 1006, 974, 977, 10, 4, 0, 0, 1098, 975, 1042, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4170, 4012, 4025, 4096, 4130, 4032, 4133] avg=4096.0000 max=4170 max_rank=0 min=4012 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16391 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1019, 1058, 1033, 1024, 6, 4, 0, 0, 1007, 986, 1009, 1041, 0, 0, 0, 0, 986, 1055, 1010, 997, 1, 0, 0, 0, 1024, 1021, 998, 1074, 0, 0, 0, 0, 1055, 1030, 1033, 1005, 0, 0, 0, 0, 1065, 1032, 1009, 1034, 0, 0, 0, 0, 997, 997, 991, 1020, 10, 4, 0, 0, 1019, 1058, 1029, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4043, 4049, 4117, 4123, 4140, 4019, 4133] avg=4096.0000 max=4144 max_rank=0 min=4019 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16415 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1018, 993, 1044, 1044, 6, 5, 0, 0, 1031, 1008, 1022, 1008, 0, 0, 0, 0, 996, 1026, 1037, 1010, 1, 0, 0, 0, 1019, 1038, 1051, 1004, 0, 0, 0, 0, 999, 1018, 1030, 1050, 0, 0, 0, 0, 1024, 1005, 1047, 1008, 0, 0, 0, 0, 1091, 1013, 1018, 1060, 9, 4, 0, 0, 1015, 972, 1023, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4069, 4070, 4112, 4097, 4084, 4195, 4031] avg=4096.0000 max=4195 max_rank=6 min=4031 min_rank=7 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16407 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[978, 1023, 1039, 1055, 11, 8, 0, 0, 1025, 1042, 1027, 1025, 0, 0, 0, 0, 1021, 1023, 1030, 1017, 6, 0, 0, 0, 987, 1014, 1040, 1011, 0, 0, 0, 0, 1017, 985, 991, 1027, 6, 0, 0, 0, 1065, 1078, 1010, 1006, 0, 0, 0, 0, 999, 1047, 978, 1045, 12, 6, 0, 0, 1105, 979, 994, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4119, 4097, 4052, 4026, 4159, 4087, 4114] avg=4096.0000 max=4159 max_rank=5 min=4026 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16386 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1013, 966, 1007, 1016, 6, 5, 0, 0, 990, 1059, 1025, 1037, 0, 0, 0, 0, 1031, 1050, 1027, 995, 1, 0, 0, 0, 1026, 1042, 1003, 1071, 0, 0, 0, 0, 980, 1043, 1059, 1049, 0, 0, 0, 0, 1031, 1025, 1033, 1059, 0, 0, 0, 0, 1058, 1050, 985, 1004, 10, 4, 0, 0, 1002, 1010, 969, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4111, 4104, 4142, 4131, 4148, 4111, 4008] avg=4096.0000 max=4148 max_rank=5 min=4008 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16398 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1016, 1013, 979, 1019, 6, 5, 0, 0, 1017, 1052, 963, 1015, 0, 0, 0, 0, 1067, 1044, 1019, 1052, 1, 0, 0, 0, 1057, 1043, 1038, 989, 0, 0, 0, 0, 999, 982, 1026, 1039, 0, 0, 0, 0, 975, 1048, 999, 1020, 0, 0, 0, 0, 1046, 1041, 1038, 1030, 10, 4, 0, 0, 1027, 1030, 1021, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4047, 4183, 4127, 4046, 4042, 4169, 4116] avg=4096.0000 max=4183 max_rank=2 min=4038 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16373 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1051, 1021, 1058, 967, 1006, 1022, 1019, 1022, 1074, 1006, 1003, 1062, 1031, 1036, 1047, 1053, 1032, 1020, 1019, 1066, 1017, 1018, 1042, 982, 1013, 994, 1005, 1063, 1000, 1008, 993, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1014, 1045, 1020, 1018, 1045, 1058, 1002, 1056, 1046, 1042, 997, 1000, 1004, 1022, 1031, 1026, 1005, 1066, 1028, 1043, 1003, 1036, 994, 1046, 1044, 1036, 968, 1015, 1020, 1026, 969, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[979, 998, 1082, 1036, 1036, 964, 1014, 1006, 1079, 1013, 1029, 998, 1051, 1021, 1030, 1026, 995, 958, 1035, 1014, 1043, 1049, 1021, 1029, 1037, 1015, 1039, 1001, 1041, 1044, 1059, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1017, 1044, 1019, 1006, 1035, 1025, 1025, 1035, 1053, 1044, 1041, 1018, 1033, 1009, 1075, 1020, 1036, 991, 1009, 1019, 1033, 976, 989, 1043, 1025, 1045, 1067, 1013, 1016, 1007, 967, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1099, 979, 1012, 1070, 1019, 988, 1047, 1021, 1009, 1050, 998, 1050, 1050, 1040, 968, 991, 1046, 1012, 1006, 1060, 980, 1028, 1026, 1062, 1009, 986, 1014, 1014, 1043, 1013, 1023, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1042, 1015, 1049, 1011, 1039, 1005, 1021, 986, 1062, 1001, 1071, 1039, 1013, 1024, 1050, 948, 1049, 1064, 1028, 1072, 1033, 1015, 977, 1023, 1031, 996, 1032, 1033, 1016, 948, 1048, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1029, 1019, 1014, 1019, 1030, 1004, 1015, 1004, 1011, 980, 1025, 1001, 1006, 1046, 996, 968, 1075, 1004, 1012, 1069, 1027, 1010, 1061, 1076, 1009, 1063, 1017, 1007, 1059, 1059, 1054, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1015, 1016, 1016, 1044, 964, 992, 1036, 1003, 1071, 1019, 995, 1103, 1015, 1014, 1059, 1008, 1015, 1013, 1042, 1014, 1016, 1007, 1061, 1056, 1049, 1056, 1004, 1079, 978, 981, 1022, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1051, 1021, 1052, 967, 0, 0, 0, 0, 1006, 1022, 1019, 1022, 28, 0, 0, 0, 1046, 1006, 1003, 1062, 0, 0, 0, 0, 1031, 1036, 1047, 1053, 4, 1, 1, 0, 1032, 1020, 1019, 1048, 0, 0, 0, 0, 1017, 1018, 1042, 981, 0, 0, 0, 0, 1013, 994, 1005, 1062, 0, 0, 0, 0, 1000, 1008, 993, 1018, 18, 2, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4097, 4117, 4173, 4119, 4058, 4074, 4039] avg=4096.0000 max=4173 max_rank=3 min=4039 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16290 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1014, 1045, 1014, 1018, 0, 0, 0, 0, 1045, 1058, 1002, 1056, 27, 0, 0, 0, 1019, 1042, 997, 1000, 0, 0, 0, 0, 1004, 1022, 1031, 1026, 4, 1, 1, 0, 1005, 1066, 1028, 1025, 0, 0, 0, 0, 1003, 1036, 994, 1045, 0, 0, 0, 0, 1044, 1036, 968, 1014, 0, 0, 0, 0, 1020, 1026, 969, 1043, 18, 2, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4188, 4058, 4089, 4124, 4078, 4062, 4078] avg=4096.0000 max=4188 max_rank=1 min=4058 min_rank=2 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[979, 998, 1076, 1036, 0, 0, 0, 0, 1036, 964, 1014, 1006, 28, 0, 0, 0, 1051, 1013, 1029, 998, 0, 0, 0, 0, 1051, 1021, 1030, 1026, 4, 1, 1, 0, 995, 958, 1035, 997, 0, 0, 0, 0, 1043, 1049, 1021, 1028, 0, 0, 0, 0, 1037, 1015, 1039, 1000, 0, 0, 0, 0, 1041, 1044, 1059, 1026, 17, 2, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4048, 4091, 4134, 3985, 4141, 4091, 4189] avg=4096.0000 max=4189 max_rank=7 min=3985 min_rank=4 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1017, 1044, 1013, 1006, 0, 0, 0, 0, 1035, 1025, 1025, 1035, 27, 0, 0, 0, 1026, 1044, 1041, 1018, 0, 0, 0, 0, 1033, 1009, 1075, 1020, 4, 1, 1, 0, 1036, 991, 1009, 1002, 0, 0, 0, 0, 1033, 976, 989, 1042, 0, 0, 0, 0, 1025, 1045, 1067, 1012, 0, 0, 0, 0, 1016, 1007, 967, 1033, 17, 2, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4147, 4129, 4143, 4038, 4040, 4149, 4042] avg=4096.0000 max=4149 max_rank=6 min=4038 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16269 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1042, 1015, 1043, 1011, 0, 0, 0, 0, 1039, 1005, 1021, 986, 28, 0, 0, 0, 1034, 1001, 1071, 1039, 0, 0, 0, 0, 1013, 1024, 1050, 948, 4, 1, 1, 0, 1049, 1064, 1028, 1054, 0, 0, 0, 0, 1033, 1015, 977, 1022, 0, 0, 0, 0, 1031, 996, 1032, 1032, 0, 0, 0, 0, 1016, 948, 1048, 1027, 18, 2, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4079, 4145, 4041, 4195, 4047, 4091, 4059] avg=4096.0000 max=4195 max_rank=4 min=4041 min_rank=3 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1029, 1019, 1008, 1019, 0, 0, 0, 0, 1030, 1004, 1015, 1004, 26, 0, 0, 0, 985, 980, 1025, 1001, 0, 0, 0, 0, 1006, 1046, 996, 968, 4, 1, 1, 0, 1075, 1004, 1012, 1051, 0, 0, 0, 0, 1027, 1010, 1061, 1075, 0, 0, 0, 0, 1009, 1063, 1017, 1006, 0, 0, 0, 0, 1059, 1059, 1054, 999, 18, 2, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4079, 3991, 4022, 4142, 4173, 4095, 4191] avg=4096.0000 max=4191 max_rank=7 min=3991 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1099, 979, 998, 1070, 0, 0, 0, 0, 1019, 988, 1047, 1021, 30, 2, 0, 0, 979, 1050, 998, 1050, 0, 0, 0, 0, 1050, 1040, 968, 991, 7, 4, 4, 0, 1046, 1012, 1006, 1034, 0, 0, 0, 0, 980, 1028, 1026, 1058, 0, 0, 0, 0, 1009, 986, 1014, 1010, 0, 0, 0, 0, 1043, 1013, 1023, 1055, 24, 7, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4107, 4077, 4064, 4098, 4092, 4019, 4165] avg=4096.0000 max=4165 max_rank=7 min=4019 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1015, 1016, 1010, 1044, 0, 0, 0, 0, 964, 992, 1036, 1003, 28, 0, 0, 0, 1043, 1019, 995, 1103, 0, 0, 0, 0, 1015, 1014, 1059, 1008, 4, 1, 1, 0, 1015, 1013, 1042, 997, 0, 0, 0, 0, 1016, 1007, 1061, 1055, 0, 0, 0, 0, 1049, 1056, 1004, 1078, 0, 0, 0, 0, 978, 981, 1022, 1005, 17, 2, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4023, 4160, 4102, 4067, 4139, 4187, 4005] avg=4096.0000 max=4187 max_rank=6 min=4005 min_rank=7 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8064, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1038, 1013, 1064, 1038, 971, 1026, 1000, 1035, 1039, 1035, 1048, 1006, 995, 1025, 1039, 1035, 1006, 1026, 1028, 1034, 988, 1036, 1013, 1054, 1033, 1060, 1015, 1012, 1010, 1039, 984, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[977, 1014, 998, 1001, 1016, 999, 1032, 964, 1002, 1035, 1068, 1028, 1063, 1054, 1033, 1016, 1015, 1007, 1074, 999, 1050, 1054, 1030, 1043, 1070, 1033, 1020, 1072, 995, 1009, 1007, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1051, 1024, 1019, 1019, 1049, 1052, 1017, 1029, 1009, 1018, 1001, 970, 1027, 1002, 1084, 985, 1028, 999, 1028, 984, 1053, 1014, 1052, 1000, 1029, 1058, 1028, 1037, 1029, 1041, 1010, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1021, 1006, 1037, 998, 986, 1037, 1021, 998, 1046, 1057, 1055, 1015, 1022, 985, 1000, 1020, 1004, 1049, 1055, 1003, 1059, 1042, 1036, 999, 1060, 1056, 1015, 988, 1021, 1070, 997, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1020, 1022, 1037, 984, 954, 996, 1003, 1026, 1046, 1011, 1065, 1056, 1055, 1002, 1008, 1053, 1010, 1048, 992, 1051, 1058, 996, 1043, 1033, 1065, 991, 1070, 1059, 980, 1006, 1028, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[971, 1035, 1014, 1059, 1014, 992, 1012, 1026, 1017, 1019, 988, 1017, 980, 1013, 983, 1042, 1040, 1016, 1078, 1060, 1016, 983, 1082, 1036, 1022, 1037, 1046, 1032, 1039, 999, 1039, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1040, 1044, 1025, 1029, 991, 984, 1041, 997, 1037, 1066, 1013, 1000, 961, 1041, 1046, 1017, 1027, 1011, 1032, 1042, 1089, 1028, 998, 1020, 1050, 1018, 1043, 997, 1012, 1010, 1035, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[986, 974, 1016, 1051, 1022, 1064, 1043, 1030, 1024, 981, 1033, 1012, 1060, 1036, 964, 1059, 997, 1013, 1022, 1021, 1038, 985, 1000, 1061, 1031, 1030, 1021, 1040, 1036, 1030, 1048, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1038, 1013, 1064, 1038, 13, 3, 0, 0, 971, 1026, 1000, 1035, 42, 0, 0, 0, 1039, 1035, 1042, 1006, 0, 0, 0, 0, 995, 1025, 1039, 1035, 4, 3, 0, 0, 1006, 1026, 1025, 1034, 0, 0, 0, 0, 963, 1036, 1013, 1054, 0, 0, 0, 0, 988, 1060, 1015, 1012, 0, 0, 0, 0, 1010, 1039, 984, 1023, 12, 2, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4074, 4122, 4101, 4091, 4066, 4075, 4070] avg=4096.0000 max=4169 max_rank=0 min=4066 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[977, 1014, 998, 1001, 14, 3, 0, 0, 1016, 999, 1032, 964, 43, 0, 0, 0, 1002, 1035, 1062, 1028, 0, 0, 0, 0, 1063, 1054, 1033, 1016, 4, 4, 0, 0, 1015, 1007, 1070, 999, 0, 0, 0, 0, 1023, 1054, 1030, 1043, 0, 0, 0, 0, 1024, 1033, 1020, 1072, 0, 0, 0, 0, 995, 1009, 1007, 990, 13, 2, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4054, 4127, 4174, 4091, 4150, 4149, 4016] avg=4096.0000 max=4174 max_rank=3 min=4007 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1051, 1024, 1019, 1019, 14, 3, 0, 0, 1049, 1052, 1017, 1029, 41, 0, 0, 0, 1009, 1018, 996, 970, 0, 0, 0, 0, 1027, 1002, 1084, 985, 3, 3, 0, 0, 1028, 999, 1025, 984, 0, 0, 0, 0, 1026, 1014, 1052, 1000, 0, 0, 0, 0, 985, 1058, 1028, 1037, 0, 0, 0, 0, 1029, 1041, 1010, 1022, 13, 2, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4188, 3993, 4104, 4036, 4092, 4108, 4117] avg=4096.0000 max=4188 max_rank=1 min=3993 min_rank=2 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[971, 1035, 1014, 1059, 13, 3, 0, 0, 1014, 992, 1012, 1026, 41, 0, 0, 0, 1017, 1019, 983, 1017, 0, 0, 0, 0, 980, 1013, 983, 1042, 3, 4, 0, 0, 1040, 1016, 1074, 1060, 0, 0, 0, 0, 990, 983, 1082, 1036, 0, 0, 0, 0, 978, 1037, 1046, 1032, 0, 0, 0, 0, 1039, 999, 1039, 1061, 13, 2, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4085, 4036, 4025, 4190, 4091, 4093, 4153] avg=4096.0000 max=4190 max_rank=4 min=4025 min_rank=3 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1040, 1044, 1025, 1029, 14, 3, 0, 0, 991, 984, 1041, 997, 42, 0, 0, 0, 1037, 1066, 1008, 1000, 0, 0, 0, 0, 961, 1041, 1046, 1017, 3, 3, 0, 0, 1027, 1011, 1029, 1042, 0, 0, 0, 0, 1062, 1028, 998, 1020, 0, 0, 0, 0, 1005, 1018, 1043, 997, 0, 0, 0, 0, 1012, 1010, 1035, 1024, 13, 2, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4055, 4111, 4071, 4109, 4108, 4063, 4096] avg=4096.0000 max=4155 max_rank=0 min=4055 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1021, 1006, 1037, 998, 14, 3, 0, 0, 986, 1037, 1021, 998, 43, 0, 0, 0, 1046, 1057, 1049, 1015, 0, 0, 0, 0, 1022, 985, 1000, 1020, 4, 3, 0, 0, 1004, 1049, 1052, 1003, 0, 0, 0, 0, 1032, 1042, 1036, 999, 0, 0, 0, 0, 1014, 1056, 1015, 988, 0, 0, 0, 0, 1021, 1070, 997, 1010, 13, 2, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4085, 4167, 4034, 4108, 4109, 4073, 4113] avg=4096.0000 max=4167 max_rank=2 min=4034 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1020, 1022, 1037, 984, 18, 8, 0, 0, 954, 996, 1003, 1026, 47, 0, 0, 0, 1046, 1011, 1054, 1056, 0, 0, 0, 0, 1055, 1002, 1008, 1053, 8, 8, 0, 0, 1010, 1048, 984, 1051, 0, 0, 0, 0, 1023, 996, 1043, 1033, 0, 0, 0, 0, 1010, 991, 1070, 1059, 0, 0, 0, 0, 980, 1006, 1028, 1000, 17, 3, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4026, 4167, 4134, 4093, 4095, 4130, 4034] avg=4096.0000 max=4167 max_rank=2 min=4026 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[986, 974, 1016, 1051, 14, 3, 0, 0, 1022, 1064, 1043, 1030, 42, 0, 0, 0, 1024, 981, 1028, 1012, 0, 0, 0, 0, 1060, 1036, 964, 1059, 3, 3, 0, 0, 997, 1013, 1019, 1021, 0, 0, 0, 0, 1011, 985, 1000, 1061, 0, 0, 0, 0, 986, 1030, 1021, 1040, 0, 0, 0, 0, 1036, 1030, 1048, 1040, 13, 2, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4201, 4045, 4125, 4050, 4057, 4077, 4169] avg=4096.0000 max=4201 max_rank=1 min=4044 min_rank=0 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1000, 1013, 973, 993, 1000, 1023, 1036, 1028, 1020, 1033, 1069, 1033, 1049, 1105, 1061, 1002, 1085, 1041, 1015, 978, 955, 1061, 1021, 1052, 1019, 1038, 988, 1051, 1041, 1009, 1010, 966] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[981, 998, 1029, 1012, 999, 1038, 999, 1013, 1036, 1035, 1025, 1056, 1008, 984, 1051, 1031, 1042, 1002, 1033, 1047, 1020, 1019, 1003, 1012, 1097, 1015, 1042, 983, 1074, 1067, 976, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[953, 1028, 1026, 1035, 1053, 1019, 1070, 1066, 1007, 1003, 1028, 1034, 1058, 993, 1097, 1051, 1033, 966, 1028, 1044, 1009, 1015, 1009, 1031, 989, 1045, 1041, 1011, 1040, 1005, 979, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1019, 993, 1019, 1011, 1099, 1037, 1015, 1074, 974, 992, 1003, 1014, 996, 1020, 1048, 1016, 1060, 1034, 1042, 1015, 1039, 1040, 1074, 1009, 983, 978, 1035, 1040, 980, 1047, 1029, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1002, 968, 1008, 1029, 996, 1058, 1030, 1052, 996, 1044, 1034, 1030, 1001, 964, 1036, 1045, 999, 1021, 986, 1027, 1040, 996, 1056, 1050, 1040, 1051, 1052, 1057, 1021, 1046, 1035, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1022, 1037, 1065, 1017, 1055, 1054, 1033, 998, 994, 1043, 1039, 980, 1053, 1067, 986, 1006, 1013, 999, 1012, 1011, 1039, 953, 1055, 1028, 996, 981, 1010, 1033, 1048, 1069, 1078, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1013, 1092, 1009, 1048, 1023, 974, 1041, 990, 999, 1057, 1037, 961, 1087, 1051, 1042, 1049, 983, 1028, 1054, 1001, 1005, 989, 1030, 1011, 1061, 1014, 1023, 1052, 1001, 1025, 1013, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1008, 1034, 1030, 981, 980, 1037, 1028, 984, 1042, 1050, 1071, 1008, 1025, 1007, 1025, 1021, 1051, 1056, 1030, 1001, 988, 1010, 1023, 1014, 1060, 1057, 1043, 1023, 1042, 1028, 1031, 980] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[953, 1028, 1026, 1035, 35, 6, 0, 0, 1053, 1019, 1054, 1066, 0, 0, 0, 0, 1007, 1003, 1028, 1034, 2, 0, 0, 0, 1058, 993, 1062, 1051, 0, 0, 0, 0, 1033, 966, 1028, 1044, 4, 0, 0, 0, 1009, 1015, 1009, 1031, 10, 3, 0, 0, 989, 1045, 1041, 995, 0, 0, 0, 0, 1040, 1005, 979, 1002, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4192, 4074, 4164, 4075, 4077, 4070, 4033] avg=4096.0000 max=4192 max_rank=1 min=4033 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16255 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1019, 993, 1019, 1011, 33, 6, 0, 0, 1099, 1037, 1001, 1074, 0, 0, 0, 0, 974, 992, 1003, 1014, 2, 0, 0, 0, 996, 1020, 1015, 1016, 0, 0, 0, 0, 1060, 1034, 1042, 1015, 3, 0, 0, 0, 1039, 1040, 1074, 1009, 10, 3, 0, 0, 983, 978, 1035, 1024, 0, 0, 0, 0, 980, 1047, 1029, 1033, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4211, 3985, 4047, 4154, 4175, 4020, 4095] avg=4096.0000 max=4211 max_rank=1 min=3985 min_rank=2 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1002, 968, 1008, 1029, 33, 7, 0, 0, 996, 1058, 1016, 1052, 0, 0, 0, 0, 996, 1044, 1034, 1030, 2, 0, 0, 0, 1001, 964, 1003, 1045, 0, 0, 0, 0, 999, 1021, 986, 1027, 3, 0, 0, 0, 1040, 996, 1056, 1050, 10, 3, 0, 0, 1040, 1051, 1052, 1040, 0, 0, 0, 0, 1021, 1046, 1035, 998, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4122, 4106, 4013, 4036, 4155, 4183, 4106] avg=4096.0000 max=4183 max_rank=6 min=4013 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1000, 1013, 973, 993, 33, 7, 0, 0, 1000, 1023, 1022, 1028, 0, 0, 0, 0, 1020, 1033, 1069, 1033, 2, 0, 0, 0, 1049, 1105, 1028, 1002, 0, 0, 0, 0, 1085, 1041, 1015, 978, 3, 0, 0, 0, 955, 1061, 1021, 1052, 10, 3, 0, 0, 1019, 1038, 988, 1034, 0, 0, 0, 0, 1041, 1009, 1010, 966, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4073, 4157, 4184, 4122, 4102, 4079, 4032] avg=4096.0000 max=4184 max_rank=3 min=4019 min_rank=0 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[981, 998, 1029, 1012, 33, 6, 0, 0, 999, 1038, 985, 1013, 0, 0, 0, 0, 1036, 1035, 1025, 1056, 2, 0, 0, 0, 1008, 984, 1018, 1031, 0, 0, 0, 0, 1042, 1002, 1033, 1047, 3, 0, 0, 0, 1020, 1019, 1003, 1012, 10, 3, 0, 0, 1097, 1015, 1042, 967, 0, 0, 0, 0, 1074, 1067, 976, 1041, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4035, 4154, 4041, 4127, 4067, 4121, 4164] avg=4096.0000 max=4164 max_rank=7 min=4035 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1013, 1092, 1009, 1048, 33, 7, 0, 0, 1023, 974, 1027, 990, 0, 0, 0, 0, 999, 1057, 1037, 961, 2, 0, 0, 0, 1087, 1051, 1009, 1049, 0, 0, 0, 0, 983, 1028, 1054, 1001, 3, 0, 0, 0, 1005, 989, 1030, 1011, 10, 3, 0, 0, 1061, 1014, 1023, 1035, 0, 0, 0, 0, 1001, 1025, 1013, 1005, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4202, 4014, 4056, 4196, 4069, 4048, 4133, 4050] avg=4096.0000 max=4202 max_rank=0 min=4014 min_rank=1 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1022, 1037, 1065, 1017, 35, 10, 0, 0, 1055, 1054, 999, 998, 0, 0, 0, 0, 994, 1043, 1039, 980, 7, 0, 0, 0, 1053, 1067, 951, 1006, 0, 0, 0, 0, 1013, 999, 1012, 1011, 9, 0, 0, 0, 1039, 953, 1055, 1028, 15, 6, 0, 0, 996, 981, 1010, 1008, 0, 0, 0, 0, 1048, 1069, 1078, 994, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4106, 4063, 4077, 4044, 4096, 3995, 4201] avg=4096.0000 max=4201 max_rank=7 min=3995 min_rank=6 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1008, 1034, 1030, 981, 32, 6, 0, 0, 980, 1037, 1014, 984, 0, 0, 0, 0, 1042, 1050, 1071, 1008, 2, 0, 0, 0, 1025, 1007, 993, 1021, 0, 0, 0, 0, 1051, 1056, 1030, 1001, 3, 0, 0, 0, 988, 1010, 1023, 1014, 10, 3, 0, 0, 1060, 1057, 1043, 1007, 0, 0, 0, 0, 1042, 1028, 1031, 980, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4015, 4173, 4046, 4141, 4048, 4167, 4087] avg=4096.0000 max=4173 max_rank=2 min=4015 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1074, 1013, 1038, 980, 1116, 1022, 1009, 993, 981, 1030, 998, 1061, 1025, 1064, 1045, 1001, 1043, 1007, 997, 1022, 1010, 1031, 1001, 993, 1054, 1015, 1028, 1045, 1038, 959, 1055, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1074, 964, 1033, 1048, 1027, 1031, 1028, 1056, 997, 1035, 1018, 1018, 1061, 1032, 988, 1051, 966, 1038, 1062, 1040, 1015, 999, 1006, 977, 1065, 1000, 997, 1011, 987, 1050, 1012, 1082] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1032, 1041, 1043, 1049, 1016, 1057, 1034, 1010, 1044, 1027, 1030, 1030, 1041, 1022, 1002, 1027, 1037, 1021, 1012, 1017, 1018, 1005, 1010, 1034, 1044, 987, 1008, 999, 1006, 1015, 1038, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1081, 965, 1071, 1054, 1043, 1051, 1018, 1018, 1034, 1029, 1047, 1019, 998, 1044, 991, 1048, 999, 1007, 1029, 1060, 1002, 1057, 1028, 928, 985, 1028, 1001, 1043, 1045, 1048, 988, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1032, 1057, 1027, 1027, 1021, 1028, 1045, 1041, 1001, 1012, 971, 1023, 1026, 1023, 1034, 1033, 1016, 1011, 995, 1060, 997, 1010, 986, 1015, 1023, 1064, 1033, 1092, 1031, 1013, 1027, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1011, 1001, 1005, 983, 984, 1047, 1055, 1034, 1024, 1019, 1031, 1078, 1029, 1012, 1044, 1013, 969, 1056, 1040, 1019, 1006, 968, 1035, 1062, 1076, 1082, 1006, 1010, 1032, 994, 1049, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1030, 1038, 1064, 1011, 1008, 1025, 999, 1040, 1026, 1046, 1019, 997, 1045, 1031, 1031, 999, 1018, 1031, 1074, 1045, 1028, 1007, 1017, 997, 1031, 994, 1043, 1006, 981, 1003, 1016, 1068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1036, 1027, 991, 1014, 1043, 1080, 1027, 1019, 1019, 1008, 1071, 1021, 1018, 1047, 1022, 1008, 1020, 1038, 1001, 1020, 1032, 1039, 983, 1026, 1005, 1014, 1008, 1014, 1030, 1022, 1051, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1056, 1013, 1038, 980, 0, 0, 0, 0, 1116, 991, 1009, 993, 0, 0, 0, 0, 981, 1030, 998, 1061, 0, 0, 0, 0, 1025, 1054, 1045, 1001, 0, 0, 0, 0, 1043, 1007, 997, 1022, 0, 0, 0, 0, 1010, 1031, 1001, 993, 31, 18, 5, 0, 1049, 1015, 1028, 1045, 0, 0, 0, 0, 1038, 959, 1055, 1020, 5, 5, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4109, 4070, 4125, 4069, 4089, 4137, 4082] avg=4096.0000 max=4137 max_rank=6 min=4069 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1056, 964, 1033, 1048, 0, 0, 0, 0, 1027, 1000, 1028, 1056, 0, 0, 0, 0, 997, 1035, 1018, 1018, 0, 0, 0, 0, 1061, 1022, 988, 1051, 0, 0, 0, 0, 966, 1038, 1062, 1040, 0, 0, 0, 0, 1015, 999, 1006, 977, 31, 18, 5, 0, 1060, 1000, 997, 1011, 0, 0, 0, 0, 987, 1050, 1012, 1082, 5, 5, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4111, 4068, 4122, 4106, 4051, 4068, 4141] avg=4096.0000 max=4141 max_rank=7 min=4051 min_rank=5 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1063, 965, 1071, 1054, 0, 0, 0, 0, 1043, 1019, 1018, 1018, 0, 0, 0, 0, 1034, 1029, 1047, 1019, 0, 0, 0, 0, 998, 1034, 991, 1048, 0, 0, 0, 0, 999, 1007, 1029, 1060, 0, 0, 0, 0, 1002, 1057, 1028, 928, 32, 18, 5, 0, 981, 1028, 1001, 1043, 0, 0, 0, 0, 1045, 1048, 988, 1009, 5, 4, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4098, 4129, 4071, 4095, 4070, 4053, 4099] avg=4096.0000 max=4153 max_rank=0 min=4053 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1014, 1041, 1043, 1049, 0, 0, 0, 0, 1016, 1025, 1034, 1010, 0, 0, 0, 0, 1044, 1027, 1030, 1030, 0, 0, 0, 0, 1041, 1012, 1002, 1027, 0, 0, 0, 0, 1037, 1021, 1012, 1017, 0, 0, 0, 0, 1018, 1005, 1010, 1034, 32, 18, 5, 0, 1039, 987, 1008, 999, 0, 0, 0, 0, 1006, 1015, 1038, 1012, 5, 5, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4085, 4131, 4082, 4087, 4122, 4033, 4081] avg=4096.0000 max=4147 max_rank=0 min=4033 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[994, 1001, 1005, 983, 0, 0, 0, 0, 984, 1015, 1055, 1034, 0, 0, 0, 0, 1024, 1019, 1031, 1078, 0, 0, 0, 0, 1029, 1002, 1044, 1013, 0, 0, 0, 0, 969, 1056, 1040, 1019, 0, 0, 0, 0, 1006, 968, 1035, 1062, 32, 17, 5, 0, 1071, 1082, 1006, 1010, 0, 0, 0, 0, 1032, 994, 1049, 994, 5, 5, 0, 0] rerouted_tokens_per_ep_rank=[3983, 4088, 4152, 4088, 4084, 4125, 4169, 4079] avg=4096.0000 max=4169 max_rank=6 min=3983 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1014, 1057, 1027, 1027, 0, 0, 0, 0, 1021, 997, 1045, 1041, 0, 0, 0, 0, 1001, 1012, 971, 1023, 0, 0, 0, 0, 1026, 1013, 1034, 1033, 0, 0, 0, 0, 1016, 1011, 995, 1060, 0, 0, 0, 0, 997, 1010, 986, 1015, 31, 18, 5, 0, 1018, 1064, 1033, 1092, 0, 0, 0, 0, 1031, 1013, 1027, 994, 5, 5, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4104, 4007, 4106, 4082, 4062, 4207, 4075] avg=4096.0000 max=4207 max_rank=6 min=4007 min_rank=2 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1009, 1038, 1064, 1011, 0, 0, 0, 0, 1008, 990, 999, 1040, 0, 0, 0, 0, 1026, 1046, 1019, 997, 2, 2, 0, 0, 1045, 1014, 1031, 999, 0, 0, 0, 0, 1018, 1031, 1074, 1043, 0, 0, 0, 0, 1028, 1007, 1017, 997, 35, 21, 8, 0, 1021, 994, 1043, 1006, 0, 0, 0, 0, 981, 1003, 1016, 1068, 9, 8, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4037, 4092, 4089, 4166, 4113, 4064, 4085] avg=4096.0000 max=4166 max_rank=4 min=4037 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1018, 1027, 991, 1014, 0, 0, 0, 0, 1043, 1047, 1027, 1019, 0, 0, 0, 0, 1019, 1008, 1071, 1021, 0, 0, 0, 0, 1018, 1037, 1022, 1008, 0, 0, 0, 0, 1020, 1038, 1001, 1020, 0, 0, 0, 0, 1032, 1039, 983, 1026, 33, 18, 5, 0, 1001, 1014, 1008, 1014, 0, 0, 0, 0, 1030, 1022, 1051, 1014, 5, 4, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4136, 4119, 4085, 4079, 4136, 4037, 4126] avg=4096.0000 max=4136 max_rank=1 min=4037 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8064, 384, 256, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1027, 1018, 1019, 1085, 1018, 1033, 1017, 1026, 1006, 963, 990, 1000, 1060, 1042, 1017, 1052, 1069, 1042, 970, 1048, 997, 1005, 1026, 1015, 1079, 984, 998, 1020, 1017, 1071, 1031, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1039, 1000, 1032, 1054, 1002, 1052, 1089, 1009, 1020, 985, 1017, 980, 1003, 1091, 1016, 1040, 986, 985, 1019, 1046, 1042, 1052, 1007, 1027, 1021, 1014, 1014, 1042, 1025, 1005, 1039, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1025, 1074, 982, 1017, 1010, 1022, 1045, 1065, 1014, 1013, 1032, 1000, 1012, 1029, 1030, 988, 979, 1062, 1037, 1010, 1001, 1023, 1017, 1045, 1051, 1057, 1053, 1046, 996, 1034, 1005, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1073, 1073, 1007, 1061, 1015, 990, 986, 1064, 1009, 1037, 1034, 1003, 1028, 1026, 1037, 979, 1053, 1034, 1039, 1014, 998, 1023, 998, 960, 1049, 1066, 1033, 1031, 975, 999, 1016, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[999, 969, 999, 1060, 1006, 1034, 1062, 1026, 1035, 1006, 1032, 997, 1065, 995, 1025, 1037, 1045, 1013, 1060, 1009, 1031, 978, 973, 1052, 992, 1028, 1059, 1071, 1034, 1015, 1036, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1037, 1048, 1043, 1021, 982, 1006, 1058, 1019, 1012, 1017, 1056, 986, 989, 1002, 995, 1035, 1025, 1009, 1041, 1037, 1018, 1011, 1050, 1032, 1020, 993, 1037, 1048, 1032, 1030, 1049, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1061, 1071, 949, 1040, 1029, 961, 1027, 989, 1039, 979, 1015, 1035, 988, 1016, 1040, 1068, 1069, 983, 1070, 1029, 1000, 1030, 973, 1090, 1009, 1013, 936, 1047, 1005, 1019, 1084, 1104] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1003, 994, 1041, 1039, 1032, 1003, 1023, 1000, 1057, 1046, 1023, 978, 1104, 1010, 996, 1018, 996, 1013, 1035, 1073, 987, 1001, 1008, 1044, 1017, 1070, 1029, 1014, 1052, 1016, 1046, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1039, 1000, 1032, 1030, 0, 0, 0, 0, 1002, 1052, 1089, 1009, 7, 0, 0, 0, 1020, 985, 1017, 980, 24, 20, 0, 0, 996, 1091, 1016, 1040, 0, 0, 0, 0, 986, 985, 1003, 1046, 0, 0, 0, 0, 1042, 1052, 1007, 1027, 16, 13, 1, 0, 1021, 1014, 1014, 1021, 0, 0, 0, 0, 1025, 1005, 1026, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4159, 4046, 4143, 4020, 4158, 4070, 4071] avg=4096.0000 max=4159 max_rank=1 min=4020 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1027, 1018, 1019, 1061, 0, 0, 0, 0, 1018, 1033, 1017, 1026, 8, 0, 0, 0, 1006, 963, 990, 1000, 24, 19, 0, 0, 1052, 1042, 1017, 1052, 0, 0, 0, 0, 1069, 1042, 955, 1048, 0, 0, 0, 0, 997, 1005, 1026, 1015, 15, 13, 1, 0, 1079, 984, 998, 1000, 0, 0, 0, 0, 1017, 1071, 1018, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4102, 4002, 4163, 4114, 4072, 4061, 4129] avg=4096.0000 max=4163 max_rank=3 min=4002 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1073, 1073, 1007, 1037, 0, 0, 0, 0, 1015, 990, 986, 1064, 8, 0, 0, 0, 1009, 1037, 1034, 1003, 24, 19, 0, 0, 1020, 1026, 1037, 979, 0, 0, 0, 0, 1053, 1034, 1023, 1014, 0, 0, 0, 0, 998, 1023, 998, 960, 16, 13, 1, 0, 1049, 1066, 1033, 1011, 0, 0, 0, 0, 975, 999, 1003, 1058, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4063, 4126, 4062, 4124, 4009, 4159, 4035] avg=4096.0000 max=4190 max_rank=0 min=4009 min_rank=5 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 1074, 982, 994, 0, 0, 0, 0, 1010, 1022, 1045, 1065, 7, 0, 0, 0, 1014, 1013, 1032, 1000, 23, 20, 0, 0, 1005, 1029, 1030, 988, 0, 0, 0, 0, 979, 1062, 1021, 1010, 0, 0, 0, 0, 1001, 1023, 1017, 1045, 16, 13, 1, 0, 1051, 1057, 1053, 1025, 0, 0, 0, 0, 996, 1034, 992, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4149, 4102, 4052, 4072, 4116, 4186, 4016] avg=4096.0000 max=4186 max_rank=6 min=4016 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[999, 969, 999, 1036, 0, 0, 0, 0, 1006, 1034, 1062, 1026, 8, 0, 0, 0, 1035, 1006, 1032, 997, 24, 20, 0, 0, 1057, 995, 1025, 1037, 0, 0, 0, 0, 1045, 1013, 1044, 1009, 0, 0, 0, 0, 1031, 978, 973, 1052, 16, 13, 1, 0, 992, 1028, 1059, 1050, 0, 0, 0, 0, 1034, 1015, 1023, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4003, 4136, 4114, 4114, 4111, 4064, 4129, 4097] avg=4096.0000 max=4136 max_rank=1 min=4003 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1037, 1048, 1043, 994, 0, 0, 0, 0, 982, 1006, 1058, 1019, 12, 3, 0, 0, 1012, 1017, 1056, 986, 27, 23, 0, 0, 977, 1002, 995, 1035, 0, 0, 0, 0, 1025, 1009, 1021, 1037, 0, 0, 0, 0, 1018, 1011, 1050, 1032, 20, 17, 6, 0, 1020, 993, 1037, 1019, 0, 0, 0, 0, 1032, 1030, 1029, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4080, 4121, 4009, 4092, 4154, 4069, 4121] avg=4096.0000 max=4154 max_rank=5 min=4009 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1003, 994, 1041, 1016, 0, 0, 0, 0, 1032, 1003, 1023, 1000, 8, 0, 0, 0, 1057, 1046, 1023, 978, 23, 19, 0, 0, 1096, 1010, 996, 1018, 0, 0, 0, 0, 996, 1013, 1019, 1073, 0, 0, 0, 0, 987, 1001, 1008, 1044, 16, 13, 1, 0, 1017, 1070, 1029, 994, 0, 0, 0, 0, 1052, 1016, 1033, 1000, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4066, 4146, 4120, 4101, 4070, 4110, 4101] avg=4096.0000 max=4146 max_rank=2 min=4054 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1061, 1071, 949, 1017, 0, 0, 0, 0, 1029, 961, 1027, 989, 7, 0, 0, 0, 1039, 979, 1015, 1035, 23, 20, 0, 0, 981, 1016, 1040, 1068, 0, 0, 0, 0, 1069, 983, 1053, 1029, 0, 0, 0, 0, 1000, 1030, 973, 1090, 17, 14, 1, 0, 1009, 1013, 936, 1026, 0, 0, 0, 0, 1005, 1019, 1070, 1104, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4013, 4111, 4105, 4134, 4125, 3984, 4198] avg=4096.0000 max=4198 max_rank=7 min=3984 min_rank=6 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8064, 256, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8064, 8320, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1049, 1045, 1037, 938, 1069, 998, 1010, 1029, 1018, 1015, 1050, 1024, 1028, 1007, 990, 1066, 1021, 1036, 1004, 1018, 993, 1081, 1016, 1007, 988, 1055, 1075, 1059, 1006, 1021, 993, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1032, 1040, 960, 1052, 1013, 1041, 1040, 1014, 976, 954, 1070, 1009, 1006, 1022, 1095, 1067, 1002, 1017, 1038, 1002, 1056, 1031, 1045, 1061, 1017, 956, 1029, 998, 1060, 1007, 1037, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[990, 1039, 1039, 1056, 1025, 996, 1032, 988, 1032, 1002, 1068, 1013, 1038, 1025, 971, 1048, 1020, 1018, 1043, 945, 1016, 1043, 1040, 1049, 1055, 1018, 998, 1016, 1054, 1034, 1044, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1011, 1056, 980, 1017, 1006, 978, 1031, 1031, 1017, 961, 1073, 1048, 982, 978, 1000, 993, 1056, 1077, 1008, 1027, 1017, 998, 1037, 1058, 1022, 1049, 1006, 1062, 1050, 1044, 1081, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[976, 1124, 1048, 991, 957, 1017, 1047, 1069, 1004, 1018, 1013, 970, 1050, 1044, 1030, 1038, 1058, 1047, 1021, 978, 976, 1013, 1054, 1065, 1035, 992, 1009, 1022, 1057, 1006, 1034, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1024, 1030, 1042, 986, 1075, 998, 993, 996, 982, 1046, 1018, 1005, 997, 1004, 993, 1059, 1091, 997, 1054, 1074, 1047, 1023, 1053, 1005, 969, 1038, 1054, 1020, 1043, 1010, 1040, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1011, 1025, 1046, 1048, 1029, 1028, 1012, 946, 996, 1020, 1045, 1004, 1051, 1034, 1006, 1071, 1056, 1008, 1028, 996, 1086, 978, 1000, 1045, 1014, 1048, 981, 1037, 1044, 996, 1059, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1001, 1023, 980, 1009, 1028, 1023, 1068, 995, 1087, 1013, 1044, 1041, 1063, 945, 1029, 974, 1069, 1050, 992, 1045, 1010, 1044, 952, 1024, 1047, 997, 1052, 1062, 1022, 1036, 1007, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1032, 1040, 960, 1052, 6, 1, 0, 0, 1013, 1041, 1040, 1014, 19, 3, 0, 0, 976, 954, 1070, 1009, 15, 1, 0, 0, 1006, 1022, 1095, 1067, 7, 0, 0, 0, 988, 1017, 1038, 1002, 0, 0, 0, 0, 1056, 1031, 1045, 1042, 0, 0, 0, 0, 1017, 956, 1029, 997, 0, 0, 0, 0, 1042, 1007, 1037, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4130, 4025, 4197, 4045, 4174, 3999, 4107] avg=4096.0000 max=4197 max_rank=3 min=3999 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1011, 1056, 980, 1017, 6, 1, 0, 0, 1006, 978, 1031, 1031, 19, 3, 0, 0, 1017, 961, 1073, 1048, 14, 1, 0, 0, 982, 978, 1000, 993, 8, 0, 0, 0, 1041, 1077, 1008, 1027, 0, 0, 0, 0, 1017, 998, 1037, 1039, 0, 0, 0, 0, 1022, 1049, 1006, 1061, 0, 0, 0, 0, 1033, 1044, 1081, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4068, 4114, 3961, 4153, 4091, 4138, 4172] avg=4096.0000 max=4172 max_rank=7 min=3961 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[990, 1039, 1039, 1056, 6, 1, 0, 0, 1025, 996, 1032, 988, 19, 3, 0, 0, 1032, 1002, 1068, 1013, 15, 1, 0, 0, 1038, 1025, 971, 1048, 7, 0, 0, 0, 1006, 1018, 1043, 945, 0, 0, 0, 0, 1016, 1043, 1040, 1030, 0, 0, 0, 0, 1055, 1018, 998, 1015, 0, 0, 0, 0, 1036, 1034, 1044, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4063, 4131, 4089, 4012, 4129, 4086, 4127] avg=4096.0000 max=4131 max_rank=0 min=4012 min_rank=4 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1049, 1045, 1037, 938, 6, 1, 0, 0, 1069, 998, 1010, 1029, 18, 3, 0, 0, 1018, 1015, 1050, 1024, 14, 1, 0, 0, 1028, 1007, 990, 1066, 7, 0, 0, 0, 1007, 1036, 1004, 1018, 0, 0, 0, 0, 993, 1081, 1016, 989, 0, 0, 0, 0, 988, 1055, 1075, 1058, 0, 0, 0, 0, 989, 1021, 993, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4127, 4122, 4098, 4065, 4079, 4176, 4025] avg=4096.0000 max=4176 max_rank=6 min=4025 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[976, 1124, 1048, 991, 6, 1, 0, 0, 957, 1017, 1047, 1069, 19, 3, 0, 0, 1004, 1018, 1013, 970, 15, 1, 0, 0, 1050, 1044, 1030, 1038, 8, 0, 0, 0, 1043, 1047, 1021, 978, 0, 0, 0, 0, 976, 1013, 1054, 1046, 0, 0, 0, 0, 1035, 992, 1009, 1021, 0, 0, 0, 0, 1039, 1006, 1034, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4112, 4021, 4170, 4089, 4089, 4057, 4084] avg=4096.0000 max=4170 max_rank=3 min=4021 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1011, 1025, 1046, 1048, 6, 1, 0, 0, 1029, 1028, 1012, 946, 19, 3, 0, 0, 996, 1020, 1045, 1004, 14, 1, 0, 0, 1051, 1034, 1006, 1071, 8, 0, 0, 0, 1041, 1008, 1028, 996, 0, 0, 0, 0, 1086, 978, 1000, 1026, 0, 0, 0, 0, 1014, 1048, 981, 1036, 0, 0, 0, 0, 1027, 996, 1059, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4037, 4080, 4170, 4073, 4090, 4079, 4102] avg=4096.0000 max=4170 max_rank=3 min=4037 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1024, 1030, 1042, 986, 9, 5, 0, 0, 1075, 998, 993, 996, 23, 10, 0, 0, 982, 1046, 1018, 1005, 18, 6, 0, 0, 997, 1004, 993, 1059, 11, 0, 0, 0, 1065, 997, 1054, 1074, 0, 0, 0, 0, 1047, 1023, 1053, 982, 0, 0, 0, 0, 969, 1038, 1054, 1015, 0, 0, 0, 0, 1015, 1010, 1040, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4075, 4064, 4190, 4105, 4076, 4067] avg=4096.0000 max=4190 max_rank=4 min=4064 min_rank=3 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1001, 1023, 980, 1009, 6, 1, 0, 0, 1028, 1023, 1068, 995, 19, 3, 0, 0, 1087, 1013, 1044, 1041, 14, 1, 0, 0, 1063, 945, 1029, 974, 8, 0, 0, 0, 1054, 1050, 992, 1045, 0, 0, 0, 0, 1010, 1044, 952, 1005, 0, 0, 0, 0, 1047, 997, 1052, 1061, 0, 0, 0, 0, 1005, 1036, 1007, 1036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4136, 4200, 4019, 4141, 4011, 4157, 4084] avg=4096.0000 max=4200 max_rank=2 min=4011 min_rank=5 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8448, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8448, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1043, 1037, 1058, 1022, 1002, 1027, 1006, 980, 995, 1049, 1041, 1055, 988, 1055, 1072, 999, 1060, 1023, 1021, 959, 1024, 1008, 960, 1013, 1015, 1080, 1030, 1038, 1040, 1036, 1024, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1028, 1037, 1007, 1019, 1038, 998, 1035, 972, 1012, 1038, 1034, 1000, 1056, 1044, 981, 1004, 1076, 1060, 1034, 1042, 1008, 989, 961, 1016, 1060, 1002, 1074, 930, 1116, 1058, 1047, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1051, 1026, 1027, 978, 967, 1046, 1028, 947, 982, 1033, 1037, 1063, 1074, 998, 1022, 1038, 1085, 1032, 1043, 994, 1008, 1012, 1035, 1008, 1022, 1042, 1035, 980, 1096, 1010, 1026, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1007, 1047, 1032, 1061, 1003, 1033, 1054, 991, 1054, 1004, 1058, 1010, 994, 1060, 1029, 1033, 1054, 1033, 1022, 1015, 959, 981, 1060, 1060, 1006, 1030, 1056, 1015, 987, 976, 1025, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1026, 1018, 1018, 990, 989, 1036, 1033, 1007, 1026, 1064, 993, 1059, 1046, 982, 1055, 1055, 989, 1002, 1036, 1025, 1052, 957, 995, 1033, 1066, 1051, 998, 1023, 1081, 1061, 1016, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[998, 1065, 1017, 997, 1001, 1037, 1047, 1044, 998, 1026, 1007, 960, 1066, 977, 994, 990, 1039, 994, 1049, 1028, 1030, 1110, 1038, 1059, 1053, 1031, 1028, 988, 1043, 1027, 1017, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1024, 1050, 1028, 995, 1042, 961, 1032, 1034, 1041, 1008, 1067, 966, 1001, 1022, 987, 1077, 1021, 990, 973, 1035, 1014, 1018, 1094, 1003, 1076, 1044, 1069, 1019, 969, 1050, 1024, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[979, 1005, 1093, 1044, 1037, 1042, 1025, 1000, 975, 1028, 1067, 1034, 1054, 1043, 1033, 1007, 1025, 1058, 1001, 1038, 1017, 1001, 1008, 978, 1033, 1025, 983, 987, 1084, 1042, 1024, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1028, 1030, 1007, 1019, 0, 0, 0, 0, 1038, 998, 1035, 972, 24, 11, 0, 0, 1012, 1038, 1034, 1000, 0, 0, 0, 0, 1048, 1044, 981, 1004, 0, 0, 0, 0, 1065, 1060, 1034, 1042, 0, 0, 0, 0, 1008, 989, 961, 1016, 11, 8, 7, 3, 1046, 1002, 1074, 930, 0, 0, 0, 0, 1092, 1058, 1047, 992, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4078, 4084, 4077, 4201, 4003, 4052, 4189] avg=4096.0000 max=4201 max_rank=4 min=4003 min_rank=5 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16445 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1043, 1030, 1058, 1022, 0, 0, 0, 0, 1002, 1027, 1006, 980, 22, 11, 0, 0, 995, 1049, 1041, 1055, 0, 0, 0, 0, 980, 1055, 1072, 999, 0, 0, 0, 0, 1049, 1023, 1021, 959, 0, 0, 0, 0, 1024, 1008, 960, 1013, 11, 8, 7, 3, 1001, 1080, 1030, 1038, 0, 0, 0, 0, 1018, 1036, 1024, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4048, 4140, 4106, 4052, 4034, 4149, 4086] avg=4096.0000 max=4153 max_rank=0 min=4034 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16321 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1051, 1019, 1027, 978, 0, 0, 0, 0, 967, 1046, 1028, 947, 23, 11, 0, 0, 982, 1033, 1037, 1063, 0, 0, 0, 0, 1066, 998, 1022, 1038, 0, 0, 0, 0, 1074, 1032, 1043, 994, 0, 0, 0, 0, 1008, 1012, 1035, 1008, 11, 8, 7, 3, 1008, 1042, 1035, 980, 0, 0, 0, 0, 1073, 1010, 1026, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4022, 4115, 4124, 4143, 4092, 4065, 4132] avg=4096.0000 max=4143 max_rank=4 min=4022 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16432 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1026, 1011, 1018, 990, 0, 0, 0, 0, 989, 1036, 1033, 1007, 23, 11, 0, 0, 1026, 1064, 993, 1059, 0, 0, 0, 0, 1038, 982, 1055, 1055, 0, 0, 0, 0, 979, 1002, 1036, 1025, 0, 0, 0, 0, 1052, 957, 995, 1033, 10, 8, 7, 3, 1052, 1051, 998, 1023, 0, 0, 0, 0, 1058, 1061, 1016, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4099, 4142, 4130, 4042, 4065, 4124, 4121] avg=4096.0000 max=4142 max_rank=2 min=4042 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16352 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1007, 1037, 1032, 1061, 0, 0, 0, 0, 1003, 1033, 1054, 991, 24, 15, 0, 0, 1054, 1004, 1058, 1010, 0, 0, 0, 0, 982, 1060, 1029, 1033, 0, 0, 0, 0, 1039, 1033, 1022, 1015, 0, 0, 0, 0, 959, 981, 1060, 1060, 15, 12, 10, 7, 984, 1030, 1056, 1015, 0, 0, 0, 0, 963, 976, 1025, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4120, 4126, 4104, 4109, 4104, 4085, 3983] avg=4096.0000 max=4137 max_rank=0 min=3983 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16281 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[998, 1058, 1017, 997, 0, 0, 0, 0, 1001, 1037, 1047, 1044, 22, 11, 0, 0, 998, 1026, 1007, 960, 0, 0, 0, 0, 1058, 977, 994, 990, 0, 0, 0, 0, 1029, 994, 1049, 1028, 0, 0, 0, 0, 1030, 1110, 1038, 1059, 10, 8, 7, 3, 1039, 1031, 1028, 988, 0, 0, 0, 0, 1021, 1027, 1017, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4162, 3991, 4019, 4100, 4265, 4086, 4075] avg=4096.0000 max=4265 max_rank=5 min=3991 min_rank=2 imbalance_ratio=1.041260 max_load_reduction=28503 max_load_reduction_pct=0.869843 moved_to_echo=16526 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1024, 1043, 1028, 995, 0, 0, 0, 0, 1042, 961, 1032, 1034, 20, 12, 0, 0, 1041, 1008, 1067, 966, 0, 0, 0, 0, 993, 1022, 987, 1077, 0, 0, 0, 0, 1011, 990, 973, 1035, 0, 0, 0, 0, 1014, 1018, 1094, 1003, 10, 8, 7, 3, 1061, 1044, 1069, 1019, 0, 0, 0, 0, 949, 1050, 1024, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4101, 4082, 4079, 4009, 4157, 4193, 4057] avg=4096.0000 max=4193 max_rank=6 min=4009 min_rank=4 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16416 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[979, 998, 1093, 1044, 0, 0, 0, 0, 1037, 1042, 1025, 1000, 23, 11, 0, 0, 975, 1028, 1067, 1034, 0, 0, 0, 0, 1046, 1043, 1033, 1007, 0, 0, 0, 0, 1015, 1058, 1001, 1038, 0, 0, 0, 0, 1017, 1001, 1008, 978, 10, 8, 7, 3, 1019, 1025, 983, 987, 0, 0, 0, 0, 1061, 1042, 1024, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4138, 4104, 4129, 4112, 4032, 4014, 4125] avg=4096.0000 max=4138 max_rank=1 min=4014 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16283 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32784, 32768, 32768, 32752, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32784, 32768, 32768, 32752, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32784, 32768, 32768, 32752, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32784, 32768, 32768, 32752, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32784, 32768, 32768, 32752, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32784, 32768, 32768, 32752, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32784, 32768, 32768, 32752, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32784, 32768, 32768, 32752, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32752 actual_valid_recv=33280 observed_loss=-528 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32784 actual_valid_recv=33024 observed_loss=-240 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1003, 1017, 995, 1015, 1036, 988, 1067, 1026, 1016, 1027, 999, 996, 994, 980, 1061, 1091, 997, 1035, 1027, 1055, 1081, 1003, 1013, 1010, 1029, 995, 1023, 997, 1076, 1066, 1021, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1030, 1041, 1007, 1059, 1086, 1022, 1035, 1080, 1006, 1033, 1004, 1051, 1015, 1017, 1033, 1048, 1009, 989, 969, 1005, 1005, 1004, 1031, 1049, 1022, 1048, 1020, 1041, 950, 1020, 1050, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[988, 1017, 1040, 1037, 1057, 1011, 1025, 1006, 1029, 1015, 1043, 1074, 1012, 986, 1041, 1027, 1001, 976, 1017, 1042, 994, 1002, 990, 1063, 1012, 1046, 1015, 1042, 1069, 1020, 1020, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[998, 1020, 1031, 1074, 1027, 1016, 996, 1031, 1036, 1037, 1006, 1015, 1028, 975, 981, 1078, 1058, 1069, 1050, 1051, 1002, 1008, 968, 1027, 1020, 1027, 1024, 978, 1047, 1025, 1019, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1030, 1027, 1038, 1035, 1014, 1047, 998, 997, 969, 1000, 1026, 1025, 1090, 1010, 1071, 1026, 984, 1000, 1034, 1019, 957, 1016, 983, 1002, 1055, 1021, 1002, 1056, 1099, 1076, 1014, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1010, 1003, 1102, 998, 1028, 1066, 996, 1004, 1016, 1034, 1008, 1007, 1032, 1001, 1031, 1052, 997, 992, 1065, 972, 1006, 1084, 1023, 1039, 1006, 1055, 992, 1035, 1038, 995, 1035, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1023, 1056, 1062, 1021, 1059, 981, 994, 1055, 1023, 1039, 1047, 997, 1058, 1056, 1005, 1007, 1024, 994, 1036, 1028, 1014, 1022, 1030, 1031, 1016, 1020, 1031, 985, 1001, 1018, 1011, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1017, 1001, 1042, 1007, 1004, 1027, 1030, 1022, 1010, 1057, 1042, 990, 1029, 1000, 1026, 1039, 1012, 1054, 1020, 1038, 990, 1002, 1030, 1053, 986, 1074, 1015, 1009, 1078, 1024, 998, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1003, 1017, 987, 1015, 0, 0, 0, 0, 1029, 988, 1067, 1026, 0, 0, 0, 0, 1016, 1027, 999, 996, 8, 2, 0, 0, 994, 980, 1061, 1075, 0, 0, 0, 0, 997, 1035, 1027, 1055, 14, 5, 0, 0, 1081, 1003, 1013, 1010, 30, 0, 0, 0, 1029, 995, 1023, 997, 7, 0, 0, 0, 1041, 1066, 1021, 1029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4110, 4048, 4110, 4133, 4137, 4051, 4157] avg=4096.0000 max=4157 max_rank=7 min=4022 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1030, 1041, 999, 1059, 0, 0, 0, 0, 1078, 1022, 1035, 1080, 0, 0, 0, 0, 1006, 1033, 1004, 1051, 8, 2, 0, 0, 1015, 1017, 1033, 1033, 0, 0, 0, 0, 1009, 989, 969, 1005, 13, 4, 0, 0, 1005, 1004, 1031, 1049, 26, 0, 0, 0, 1022, 1048, 1020, 1041, 8, 0, 0, 0, 920, 1020, 1050, 989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4215, 4104, 4098, 3989, 4115, 4139, 3979] avg=4096.0000 max=4215 max_rank=1 min=3979 min_rank=7 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16222 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[998, 1020, 1023, 1074, 0, 0, 0, 0, 1020, 1016, 996, 1031, 0, 0, 0, 0, 1036, 1037, 1006, 1015, 8, 2, 0, 0, 1028, 975, 981, 1062, 0, 0, 0, 0, 1058, 1069, 1050, 1051, 14, 5, 0, 0, 1002, 1008, 968, 1027, 29, 0, 0, 0, 1020, 1027, 1024, 978, 7, 0, 0, 0, 1013, 1025, 1019, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4063, 4104, 4046, 4247, 4034, 4056, 4103] avg=4096.0000 max=4247 max_rank=4 min=4034 min_rank=5 imbalance_ratio=1.036865 max_load_reduction=28521 max_load_reduction_pct=0.870392 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[988, 1017, 1031, 1037, 0, 0, 0, 0, 1049, 1011, 1025, 1006, 0, 0, 0, 0, 1029, 1015, 1043, 1074, 8, 2, 0, 0, 1012, 986, 1041, 1012, 0, 0, 0, 0, 1001, 976, 1017, 1042, 13, 5, 0, 0, 994, 1002, 990, 1063, 30, 0, 0, 0, 1012, 1046, 1015, 1042, 8, 1, 0, 0, 1034, 1020, 1020, 1051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4091, 4171, 4051, 4054, 4079, 4124, 4125] avg=4096.0000 max=4171 max_rank=2 min=4051 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1030, 1027, 1030, 1035, 0, 0, 0, 0, 1007, 1047, 998, 997, 0, 0, 0, 0, 969, 1000, 1026, 1025, 8, 2, 0, 0, 1090, 1010, 1071, 1011, 0, 0, 0, 0, 984, 1000, 1034, 1019, 13, 5, 0, 0, 957, 1016, 983, 1002, 31, 0, 0, 0, 1055, 1021, 1002, 1056, 7, 0, 0, 0, 1063, 1076, 1014, 1047, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4049, 4030, 4182, 4055, 3989, 4141, 4200] avg=4096.0000 max=4200 max_rank=7 min=3989 min_rank=5 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1023, 1056, 1053, 1021, 0, 0, 0, 0, 1051, 981, 994, 1055, 0, 0, 0, 0, 1023, 1039, 1047, 997, 8, 2, 0, 0, 1058, 1056, 1005, 992, 0, 0, 0, 0, 1024, 994, 1036, 1028, 13, 4, 0, 0, 1014, 1022, 1030, 1031, 28, 0, 0, 0, 1016, 1020, 1031, 985, 8, 1, 0, 0, 969, 1018, 1011, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4081, 4116, 4111, 4099, 4125, 4061, 4022] avg=4096.0000 max=4153 max_rank=0 min=4022 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1010, 1003, 1085, 998, 0, 0, 0, 0, 1017, 1066, 996, 1004, 0, 0, 0, 0, 1016, 1034, 1008, 1007, 12, 9, 0, 0, 1032, 1001, 1031, 1027, 0, 0, 0, 0, 997, 992, 1065, 972, 16, 7, 0, 0, 1006, 1084, 1023, 1039, 32, 0, 0, 0, 1006, 1055, 992, 1035, 11, 5, 0, 0, 999, 995, 1035, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4083, 4086, 4091, 4049, 4184, 4104, 4075] avg=4096.0000 max=4184 max_rank=5 min=4049 min_rank=4 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1017, 1001, 1033, 1007, 0, 0, 0, 0, 997, 1027, 1030, 1022, 0, 0, 0, 0, 1010, 1057, 1042, 990, 8, 2, 0, 0, 1029, 1000, 1026, 1024, 0, 0, 0, 0, 1012, 1054, 1020, 1038, 13, 5, 0, 0, 990, 1002, 1030, 1053, 30, 0, 0, 0, 986, 1074, 1015, 1009, 7, 1, 0, 0, 1043, 1024, 998, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4076, 4109, 4079, 4142, 4105, 4092, 4107] avg=4096.0000 max=4142 max_rank=4 min=4058 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[981, 1044, 1010, 1001, 1009, 1012, 1010, 984, 1028, 1042, 1028, 1038, 1072, 1011, 1066, 1050, 1000, 995, 997, 969, 996, 1063, 1000, 972, 1036, 1079, 1068, 1056, 999, 1069, 1020, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1060, 995, 1022, 986, 1011, 1043, 978, 1010, 1051, 1038, 1037, 974, 985, 1039, 1024, 1032, 995, 1001, 1006, 1075, 1030, 1059, 967, 1051, 1075, 1047, 1037, 1029, 1029, 1041, 1016, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1044, 1040, 1019, 1011, 1037, 976, 1036, 1040, 1062, 948, 1019, 1002, 1039, 1029, 1010, 1009, 1068, 995, 1035, 1000, 1001, 1014, 1046, 1031, 1008, 1040, 1060, 1061, 995, 1027, 1020, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[961, 1028, 1065, 1080, 1022, 1015, 981, 1075, 1007, 1042, 1000, 997, 1021, 997, 1088, 1033, 1011, 1030, 1045, 1068, 995, 1025, 1012, 1016, 1017, 1001, 958, 1010, 1036, 992, 1093, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1041, 1015, 1010, 1048, 1055, 1053, 1002, 978, 966, 1011, 1024, 1036, 1013, 1024, 995, 993, 1038, 1037, 979, 1072, 1031, 989, 1024, 966, 1011, 1066, 1064, 1086, 1040, 1039, 1048, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1040, 1008, 964, 1011, 985, 1069, 1032, 977, 1030, 1018, 1003, 1027, 1032, 1059, 996, 1040, 1024, 1002, 1031, 1038, 1064, 1059, 1042, 1027, 1017, 1049, 1015, 1064, 1029, 995, 989, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1009, 1066, 1049, 1075, 1018, 1041, 1024, 979, 1029, 1036, 1000, 1004, 1029, 1034, 1019, 1053, 1048, 1038, 1046, 1010, 1049, 958, 1035, 1033, 987, 1035, 999, 952, 1076, 1021, 973, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1003, 1011, 1033, 1023, 1018, 1024, 1004, 998, 1020, 974, 1046, 1031, 1027, 1003, 991, 1055, 1053, 1004, 1043, 1049, 990, 1059, 1064, 1070, 1043, 1008, 987, 992, 1001, 1043, 1023, 1078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[981, 1044, 1010, 1001, 1, 0, 0, 0, 1009, 1012, 1010, 984, 24, 10, 0, 0, 1028, 1042, 1028, 1038, 14, 11, 0, 0, 1072, 1011, 1066, 1038, 0, 0, 0, 0, 1000, 995, 997, 966, 0, 0, 0, 0, 996, 1063, 1000, 972, 2, 1, 0, 0, 1036, 1055, 1068, 1056, 0, 0, 0, 0, 999, 1069, 1020, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4049, 4161, 4187, 3958, 4034, 4215, 4127] avg=4096.0000 max=4215 max_rank=6 min=3958 min_rank=4 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1044, 1040, 1019, 1011, 1, 0, 0, 0, 1037, 976, 1036, 1040, 24, 9, 0, 0, 1062, 948, 1019, 1002, 13, 10, 0, 0, 1039, 1029, 1010, 998, 0, 0, 0, 0, 1068, 995, 1035, 997, 0, 0, 0, 0, 1001, 1014, 1046, 1031, 2, 1, 0, 0, 1008, 1018, 1060, 1061, 0, 0, 0, 0, 995, 1027, 1020, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4122, 4054, 4076, 4095, 4095, 4147, 4064] avg=4096.0000 max=4147 max_rank=6 min=4054 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[961, 1028, 1065, 1080, 1, 0, 0, 0, 1022, 1015, 981, 1075, 24, 9, 0, 0, 1007, 1042, 1000, 997, 13, 11, 0, 0, 1021, 997, 1088, 1021, 0, 0, 0, 0, 1011, 1030, 1045, 1065, 0, 0, 0, 0, 995, 1025, 1012, 1016, 2, 1, 0, 0, 1017, 979, 958, 1010, 0, 0, 0, 0, 1036, 992, 1093, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4126, 4070, 4127, 4151, 4051, 3964, 4144] avg=4096.0000 max=4151 max_rank=4 min=3964 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1060, 995, 1022, 986, 1, 0, 0, 0, 1011, 1043, 978, 1010, 23, 9, 0, 0, 1051, 1038, 1037, 974, 13, 11, 0, 0, 985, 1039, 1024, 1020, 0, 0, 0, 0, 995, 1001, 1006, 1072, 0, 0, 0, 0, 1030, 1059, 967, 1051, 2, 1, 0, 0, 1075, 1025, 1037, 1029, 0, 0, 0, 0, 1029, 1041, 1016, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4074, 4124, 4068, 4074, 4110, 4166, 4088] avg=4096.0000 max=4166 max_rank=6 min=4064 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1041, 1015, 1010, 1048, 1, 0, 0, 0, 1055, 1053, 1002, 978, 23, 9, 0, 0, 966, 1011, 1024, 1036, 14, 10, 0, 0, 1013, 1024, 995, 982, 0, 0, 0, 0, 1038, 1037, 979, 1069, 0, 0, 0, 0, 1031, 989, 1024, 966, 2, 1, 0, 0, 1011, 1043, 1064, 1086, 0, 0, 0, 0, 1040, 1039, 1048, 991, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4120, 4061, 4014, 4123, 4013, 4204, 4118] avg=4096.0000 max=4204 max_rank=6 min=4013 min_rank=5 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1009, 1066, 1049, 1075, 1, 0, 0, 0, 1018, 1041, 1024, 979, 24, 9, 0, 0, 1029, 1036, 1000, 1004, 13, 11, 0, 0, 1029, 1034, 1019, 1041, 0, 0, 0, 0, 1048, 1038, 1046, 1007, 0, 0, 0, 0, 1049, 958, 1035, 1033, 2, 1, 0, 0, 987, 1013, 999, 952, 0, 0, 0, 0, 1076, 1021, 973, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4200, 4095, 4093, 4123, 4139, 4078, 3951, 4089] avg=4096.0000 max=4200 max_rank=0 min=3951 min_rank=6 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1040, 1008, 964, 1011, 8, 0, 0, 0, 985, 1069, 1032, 977, 27, 14, 0, 0, 1030, 1018, 1003, 1027, 18, 14, 0, 0, 1032, 1059, 996, 1022, 0, 0, 0, 0, 1024, 1002, 1031, 1025, 0, 0, 0, 0, 1064, 1059, 1042, 1027, 5, 4, 0, 0, 1017, 1017, 1015, 1064, 0, 0, 0, 0, 1029, 995, 989, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4104, 4110, 4109, 4082, 4201, 4113, 4018] avg=4096.0000 max=4201 max_rank=5 min=4018 min_rank=7 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1003, 1011, 1033, 1023, 1, 0, 0, 0, 1018, 1024, 1004, 998, 25, 9, 0, 0, 1020, 974, 1046, 1031, 13, 11, 0, 0, 1027, 1003, 991, 1043, 0, 0, 0, 0, 1053, 1004, 1043, 1046, 0, 0, 0, 0, 990, 1059, 1064, 1070, 2, 1, 0, 0, 1043, 986, 987, 992, 0, 0, 0, 0, 1001, 1043, 1023, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4078, 4095, 4064, 4146, 4186, 4008, 4120] avg=4096.0000 max=4186 max_rank=5 min=4008 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[969, 1017, 1012, 1018, 1031, 1071, 1040, 1042, 1027, 1005, 1059, 1063, 1032, 988, 1008, 987, 1033, 1040, 1040, 1079, 1013, 1031, 1004, 1013, 991, 1023, 1024, 1015, 1014, 1011, 986, 1082] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1040, 1078, 988, 1032, 1034, 1008, 1059, 1026, 1049, 983, 1030, 993, 1026, 1034, 1010, 1002, 1010, 1018, 1030, 1015, 1026, 1004, 1042, 1063, 1075, 1058, 1019, 994, 989, 1013, 1033, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[999, 1041, 1019, 999, 1034, 1032, 1001, 972, 989, 1026, 1004, 1029, 1045, 1040, 1011, 1012, 1044, 1020, 1037, 1047, 1047, 1025, 1030, 1020, 1044, 1030, 1016, 1047, 1005, 1031, 1056, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[979, 1045, 1019, 1040, 1049, 996, 1054, 1043, 1016, 1050, 1015, 1070, 1044, 1008, 1031, 1028, 1054, 976, 1010, 1023, 1020, 1047, 1004, 1024, 979, 1019, 1023, 1035, 1014, 1038, 1005, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1057, 1026, 1060, 1068, 1012, 997, 995, 1004, 1066, 1051, 1050, 994, 1004, 1025, 1032, 1046, 989, 1046, 1024, 1042, 1061, 973, 971, 1003, 1040, 1039, 992, 1021, 1011, 1033, 1004, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1057, 1018, 999, 1044, 1005, 1062, 1074, 1012, 965, 962, 1059, 1013, 1006, 1002, 1020, 1032, 1077, 1007, 1010, 1013, 998, 1063, 1039, 985, 1050, 1044, 1026, 1033, 1048, 1010, 996, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1083, 997, 1070, 1015, 1054, 1062, 1018, 1017, 1055, 991, 1040, 1003, 1004, 998, 1029, 1001, 1004, 1049, 983, 1069, 982, 995, 1046, 984, 1027, 1000, 1007, 1056, 1048, 1055, 1013, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1008, 1020, 998, 1011, 1068, 1026, 987, 1031, 1055, 1011, 1040, 1031, 1081, 1016, 1053, 1002, 1023, 1010, 1039, 1041, 980, 975, 981, 1009, 999, 1025, 1025, 1045, 1071, 1056, 1013, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[969, 1011, 1012, 1018, 0, 0, 0, 0, 1013, 1071, 1040, 1042, 0, 0, 0, 0, 1027, 1005, 1056, 1063, 0, 0, 0, 0, 1032, 988, 1008, 987, 6, 3, 3, 0, 1033, 1040, 1040, 1062, 0, 0, 0, 0, 1013, 1031, 1004, 1013, 18, 17, 3, 0, 991, 1023, 1024, 1009, 0, 0, 0, 0, 1014, 1011, 986, 1082, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4166, 4151, 4027, 4175, 4099, 4047, 4093] avg=4096.0000 max=4175 max_rank=4 min=4010 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1040, 1072, 988, 1032, 0, 0, 0, 0, 1016, 1008, 1059, 1026, 0, 0, 0, 0, 1049, 983, 1027, 993, 0, 0, 0, 0, 1026, 1034, 1010, 1002, 6, 3, 3, 0, 1010, 1018, 1030, 999, 0, 0, 0, 0, 1026, 1004, 1042, 1063, 18, 16, 3, 0, 1075, 1058, 1019, 988, 0, 0, 0, 0, 989, 1013, 1033, 987, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4109, 4052, 4084, 4057, 4172, 4140, 4022] avg=4096.0000 max=4172 max_rank=5 min=4022 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[999, 1035, 1019, 999, 0, 0, 0, 0, 1016, 1032, 1001, 972, 0, 0, 0, 0, 989, 1026, 1001, 1029, 0, 0, 0, 0, 1045, 1040, 1011, 1012, 6, 3, 3, 0, 1044, 1020, 1037, 1031, 0, 0, 0, 0, 1047, 1025, 1030, 1020, 18, 16, 3, 0, 1044, 1030, 1016, 1041, 0, 0, 0, 0, 1005, 1031, 1056, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4021, 4045, 4120, 4132, 4159, 4131, 4108] avg=4096.0000 max=4159 max_rank=5 min=4021 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16530 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1057, 1012, 999, 1044, 0, 0, 0, 0, 988, 1062, 1074, 1012, 0, 0, 0, 0, 965, 962, 1056, 1013, 0, 0, 0, 0, 1006, 1002, 1020, 1032, 6, 3, 3, 0, 1077, 1007, 1010, 997, 0, 0, 0, 0, 998, 1063, 1039, 985, 17, 16, 3, 0, 1050, 1044, 1026, 1027, 0, 0, 0, 0, 1048, 1010, 996, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4136, 3996, 4072, 4091, 4121, 4147, 4093] avg=4096.0000 max=4147 max_rank=6 min=3996 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1057, 1020, 1060, 1068, 0, 0, 0, 0, 994, 997, 995, 1004, 0, 0, 0, 0, 1066, 1051, 1047, 994, 0, 0, 0, 0, 1004, 1025, 1032, 1046, 6, 3, 3, 0, 989, 1046, 1024, 1026, 0, 0, 0, 0, 1061, 973, 971, 1003, 18, 16, 3, 0, 1040, 1039, 992, 1015, 0, 0, 0, 0, 1011, 1033, 1004, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4205, 3990, 4158, 4119, 4085, 4045, 4086, 4080] avg=4096.0000 max=4205 max_rank=0 min=3990 min_rank=1 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[979, 1029, 1019, 1040, 0, 0, 0, 0, 1027, 996, 1054, 1043, 0, 0, 0, 0, 1016, 1050, 1010, 1070, 0, 0, 0, 0, 1044, 1008, 1031, 1028, 11, 9, 5, 2, 1054, 976, 1010, 1003, 0, 0, 0, 0, 1020, 1047, 1004, 1024, 22, 20, 7, 0, 979, 1019, 1023, 1024, 0, 0, 0, 0, 1014, 1036, 1005, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4120, 4146, 4138, 4043, 4144, 4045, 4065] avg=4096.0000 max=4146 max_rank=2 min=4043 min_rank=4 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16297 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1083, 991, 1070, 1015, 0, 0, 0, 0, 1036, 1062, 1018, 1017, 0, 0, 0, 0, 1055, 991, 1037, 1003, 0, 0, 0, 0, 1004, 998, 1029, 1001, 6, 3, 3, 0, 1004, 1049, 983, 1052, 0, 0, 0, 0, 982, 995, 1046, 984, 18, 17, 3, 0, 1027, 1000, 1007, 1050, 0, 0, 0, 0, 1048, 1055, 1013, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4133, 4086, 4044, 4088, 4045, 4084, 4129] avg=4096.0000 max=4159 max_rank=0 min=4044 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1008, 1014, 998, 1011, 0, 0, 0, 0, 1049, 1026, 987, 1031, 0, 0, 0, 0, 1055, 1011, 1037, 1031, 0, 0, 0, 0, 1081, 1016, 1053, 1002, 6, 3, 3, 0, 1023, 1010, 1039, 1025, 0, 0, 0, 0, 980, 975, 981, 1009, 19, 16, 3, 0, 999, 1025, 1025, 1039, 0, 0, 0, 0, 1071, 1056, 1013, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4093, 4134, 4164, 4097, 3983, 4088, 4178] avg=4096.0000 max=4178 max_rank=7 min=3983 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33536 slack_slots=1920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[965, 1038, 1057, 1036, 1006, 1003, 1049, 1017, 1028, 1020, 1022, 1029, 1040, 1030, 1055, 1041, 1019, 1052, 1061, 1041, 1022, 1001, 1012, 1026, 1012, 965, 964, 1022, 1000, 1067, 1054, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1029, 1057, 1036, 992, 1012, 1008, 1028, 1034, 1048, 1100, 1011, 980, 1084, 1029, 1050, 1019, 1001, 1050, 993, 984, 1036, 1012, 994, 1040, 1018, 1050, 982, 996, 1076, 997, 1044, 978] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[986, 1047, 1036, 1027, 1048, 975, 1021, 1029, 1009, 1037, 1051, 1028, 1017, 1031, 952, 1046, 1080, 1037, 1023, 1009, 1022, 999, 1021, 1031, 1023, 1045, 1045, 994, 1007, 1010, 1039, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[989, 1013, 993, 1046, 1006, 1053, 975, 1037, 1023, 1027, 1008, 988, 1055, 1026, 1016, 989, 1062, 1078, 1044, 1046, 996, 1012, 1054, 959, 1026, 1063, 1047, 1045, 1006, 1013, 1022, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[978, 1036, 1085, 1030, 1014, 1054, 1004, 995, 1011, 1004, 1031, 1051, 1041, 1036, 1041, 992, 1048, 984, 1052, 990, 1021, 1019, 996, 1022, 1053, 1039, 1083, 962, 1030, 1075, 1005, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1031, 972, 1026, 1066, 1033, 972, 990, 1011, 1055, 1030, 1044, 1023, 1038, 1074, 991, 1030, 962, 1027, 1002, 1033, 1071, 998, 996, 1062, 1108, 950, 1047, 1073, 1008, 1046, 998, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1019, 1010, 1008, 1005, 1074, 1021, 1083, 973, 998, 998, 1040, 1070, 1053, 1057, 1017, 1048, 1041, 1004, 1014, 996, 955, 1058, 1069, 981, 1024, 1027, 973, 1084, 1046, 1029, 986, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[976, 1045, 1049, 1062, 1063, 1010, 977, 1039, 1012, 991, 1030, 1031, 1001, 1015, 1071, 1049, 1007, 1038, 988, 1063, 1039, 1020, 1018, 976, 1060, 1079, 1004, 1037, 1031, 1013, 998, 976] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[965, 1038, 1057, 1036, 2, 0, 0, 0, 1006, 1003, 1049, 1017, 14, 4, 0, 0, 1028, 1020, 1016, 1029, 0, 0, 0, 0, 1008, 1030, 1055, 1041, 0, 0, 0, 0, 1019, 1045, 1061, 1041, 0, 0, 0, 0, 1022, 1001, 1012, 1026, 28, 0, 0, 0, 997, 965, 964, 1022, 0, 0, 0, 0, 1000, 1067, 1054, 1014, 7, 4, 1, 0] rerouted_tokens_per_ep_rank=[4098, 4093, 4093, 4134, 4166, 4089, 3948, 4147] avg=4096.0000 max=4166 max_rank=4 min=3948 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1029, 1057, 1036, 992, 2, 0, 0, 0, 1012, 1008, 1028, 1034, 14, 4, 0, 0, 1048, 1100, 1005, 980, 0, 0, 0, 0, 1051, 1029, 1050, 1019, 0, 0, 0, 0, 1001, 1043, 993, 984, 0, 0, 0, 0, 1036, 1012, 994, 1040, 29, 0, 0, 0, 1003, 1050, 982, 996, 0, 0, 0, 0, 1076, 997, 1044, 978, 7, 4, 1, 0] rerouted_tokens_per_ep_rank=[4116, 4100, 4133, 4149, 4021, 4111, 4031, 4107] avg=4096.0000 max=4149 max_rank=3 min=4021 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[989, 1013, 993, 1046, 2, 0, 0, 0, 1006, 1053, 975, 1037, 14, 4, 0, 0, 1023, 1027, 1002, 988, 0, 0, 0, 0, 1022, 1026, 1016, 989, 0, 0, 0, 0, 1062, 1071, 1044, 1046, 0, 0, 0, 0, 996, 1012, 1054, 959, 29, 0, 0, 0, 1011, 1063, 1047, 1045, 0, 0, 0, 0, 1006, 1013, 1022, 1051, 7, 4, 1, 0] rerouted_tokens_per_ep_rank=[4043, 4089, 4040, 4053, 4223, 4050, 4166, 4104] avg=4096.0000 max=4223 max_rank=4 min=4040 min_rank=2 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[986, 1047, 1036, 1027, 2, 0, 0, 0, 1048, 975, 1021, 1029, 14, 4, 0, 0, 1009, 1037, 1045, 1028, 0, 0, 0, 0, 985, 1031, 952, 1046, 0, 0, 0, 0, 1080, 1030, 1023, 1009, 0, 0, 0, 0, 1022, 999, 1021, 1031, 28, 0, 0, 0, 1008, 1045, 1045, 994, 0, 0, 0, 0, 1007, 1010, 1039, 1043, 7, 4, 1, 0] rerouted_tokens_per_ep_rank=[4098, 4091, 4119, 4014, 4142, 4101, 4092, 4111] avg=4096.0000 max=4142 max_rank=4 min=4014 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[978, 1036, 1085, 1030, 2, 0, 0, 0, 1014, 1054, 1004, 995, 14, 4, 0, 0, 1011, 1004, 1025, 1051, 0, 0, 0, 0, 1009, 1036, 1041, 992, 0, 0, 0, 0, 1048, 977, 1052, 990, 0, 0, 0, 0, 1021, 1019, 996, 1022, 28, 0, 0, 0, 1038, 1039, 1083, 962, 0, 0, 0, 0, 1030, 1075, 1005, 986, 7, 4, 1, 0] rerouted_tokens_per_ep_rank=[4131, 4085, 4091, 4078, 4067, 4086, 4122, 4108] avg=4096.0000 max=4131 max_rank=0 min=4067 min_rank=4 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1019, 1010, 1008, 1005, 2, 0, 0, 0, 1074, 1021, 1083, 973, 14, 4, 0, 0, 998, 998, 1034, 1070, 0, 0, 0, 0, 1020, 1057, 1017, 1048, 0, 0, 0, 0, 1041, 997, 1014, 996, 0, 0, 0, 0, 955, 1058, 1069, 981, 29, 0, 0, 0, 1009, 1027, 973, 1084, 0, 0, 0, 0, 1046, 1029, 986, 1007, 7, 4, 1, 0] rerouted_tokens_per_ep_rank=[4044, 4169, 4100, 4142, 4048, 4092, 4093, 4080] avg=4096.0000 max=4169 max_rank=1 min=4044 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1031, 972, 1026, 1066, 9, 0, 0, 0, 1033, 972, 990, 1011, 19, 8, 0, 0, 1055, 1030, 1026, 1023, 0, 0, 0, 0, 998, 1074, 991, 1030, 0, 0, 0, 0, 962, 1015, 1002, 1033, 0, 0, 0, 0, 1071, 998, 996, 1062, 32, 0, 0, 0, 1082, 950, 1047, 1073, 0, 0, 0, 0, 1008, 1046, 998, 1001, 12, 9, 7, 0] rerouted_tokens_per_ep_rank=[4104, 4033, 4134, 4093, 4012, 4159, 4152, 4081] avg=4096.0000 max=4159 max_rank=5 min=4012 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[976, 1045, 1049, 1062, 2, 0, 0, 0, 1063, 1010, 977, 1039, 15, 4, 0, 0, 1012, 991, 1024, 1031, 0, 0, 0, 0, 970, 1015, 1071, 1049, 0, 0, 0, 0, 1007, 1031, 988, 1063, 0, 0, 0, 0, 1039, 1020, 1018, 976, 27, 0, 0, 0, 1044, 1079, 1004, 1037, 0, 0, 0, 0, 1031, 1013, 998, 976, 7, 4, 1, 0] rerouted_tokens_per_ep_rank=[4134, 4108, 4058, 4105, 4089, 4080, 4164, 4030] avg=4096.0000 max=4164 max_rank=6 min=4030 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8064, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1038, 981, 1008, 1040, 1008, 1040, 1093, 1027, 1023, 1038, 1092, 1021, 983, 1024, 1028, 1056, 1028, 998, 1042, 1028, 1006, 1022, 1057, 963, 1040, 1015, 1015, 983, 1010, 1038, 1023, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1019, 1020, 1043, 1048, 988, 1002, 1068, 995, 1009, 1014, 1060, 1046, 1068, 1016, 1048, 987, 1023, 1059, 1053, 992, 1007, 1059, 985, 1018, 1009, 1015, 1018, 1025, 1006, 1025, 1002, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1077, 1022, 990, 987, 1074, 1027, 1042, 1018, 1016, 954, 994, 976, 1021, 1014, 1074, 992, 1001, 996, 1072, 1006, 1038, 1021, 1054, 1068, 1001, 1013, 1054, 1030, 1055, 1003, 1025, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1048, 1024, 1065, 1029, 997, 1035, 1062, 1059, 1014, 991, 1017, 995, 1019, 1015, 1004, 1029, 993, 980, 1019, 1045, 1022, 1018, 999, 1004, 1022, 1042, 1034, 1032, 1036, 1036, 1053, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1033, 1008, 974, 979, 1039, 997, 1056, 1014, 1029, 1083, 1058, 1004, 1079, 985, 1023, 1035, 996, 1003, 1010, 1078, 1031, 1015, 1043, 979, 1052, 998, 1005, 1034, 1022, 1021, 1053, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1018, 1000, 1030, 1030, 1049, 1037, 1032, 1034, 964, 1061, 1000, 1040, 1001, 1044, 1018, 1023, 990, 1021, 1014, 1051, 1003, 1062, 1061, 1049, 993, 1038, 1026, 1019, 970, 1004, 1037, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1020, 1023, 1024, 1004, 1016, 1009, 1077, 991, 1103, 1034, 1028, 1060, 989, 1029, 981, 1078, 1015, 1026, 1008, 1028, 1047, 1034, 1000, 1014, 1049, 990, 1004, 1012, 1030, 973, 1031, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[977, 1051, 1017, 1018, 1019, 1015, 1027, 1029, 1019, 1020, 1063, 1026, 1037, 1031, 1015, 1044, 999, 1040, 924, 1021, 1044, 1016, 1052, 1012, 1043, 1041, 1028, 1025, 1008, 998, 1070, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1038, 981, 1008, 1040, 10, 4, 0, 0, 1008, 1040, 1067, 1027, 0, 0, 0, 0, 1023, 1038, 1082, 1021, 0, 0, 0, 0, 983, 1024, 1028, 1054, 0, 0, 0, 0, 1028, 998, 1042, 1028, 26, 0, 0, 0, 1006, 1022, 1053, 963, 0, 0, 0, 0, 1040, 1015, 1015, 983, 4, 2, 0, 0, 1010, 1038, 1019, 1000, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4142, 4164, 4089, 4122, 4044, 4059, 4067] avg=4096.0000 max=4164 max_rank=2 min=4044 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1019, 1020, 1043, 1048, 10, 4, 0, 0, 988, 1002, 1042, 995, 0, 0, 0, 0, 1009, 1014, 1050, 1046, 0, 0, 0, 0, 1068, 1016, 1048, 985, 0, 0, 0, 0, 1023, 1059, 1053, 992, 26, 0, 0, 0, 1007, 1059, 981, 1018, 0, 0, 0, 0, 1009, 1015, 1018, 1025, 4, 2, 0, 0, 1006, 1025, 998, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4027, 4119, 4117, 4153, 4065, 4073, 4070] avg=4096.0000 max=4153 max_rank=4 min=4027 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1048, 1024, 1065, 1029, 10, 5, 0, 0, 997, 1035, 1036, 1059, 0, 0, 0, 0, 1014, 991, 1007, 995, 0, 0, 0, 0, 1019, 1015, 1004, 1027, 0, 0, 0, 0, 993, 980, 1019, 1045, 26, 0, 0, 0, 1022, 1018, 995, 1004, 0, 0, 0, 0, 1022, 1042, 1034, 1032, 4, 2, 0, 0, 1036, 1036, 1048, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4127, 4007, 4065, 4063, 4039, 4136, 4150] avg=4096.0000 max=4181 max_rank=0 min=4007 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1077, 1022, 990, 987, 9, 4, 0, 0, 1074, 1027, 1017, 1018, 0, 0, 0, 0, 1016, 954, 985, 976, 0, 0, 0, 0, 1021, 1014, 1074, 990, 0, 0, 0, 0, 1001, 996, 1072, 1006, 25, 0, 0, 0, 1038, 1021, 1050, 1068, 0, 0, 0, 0, 1001, 1013, 1054, 1030, 4, 2, 0, 0, 1055, 1003, 1021, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4136, 3931, 4099, 4100, 4177, 4104, 4132] avg=4096.0000 max=4177 max_rank=5 min=3931 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1033, 1008, 974, 979, 10, 5, 0, 0, 1039, 997, 1031, 1014, 0, 0, 0, 0, 1029, 1083, 1048, 1004, 0, 0, 0, 0, 1079, 985, 1023, 1033, 0, 0, 0, 0, 996, 1003, 1010, 1078, 25, 0, 0, 0, 1031, 1015, 1039, 979, 0, 0, 0, 0, 1052, 998, 1005, 1034, 4, 2, 0, 0, 1022, 1021, 1048, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4009, 4081, 4164, 4120, 4112, 4064, 4095, 4123] avg=4096.0000 max=4164 max_rank=2 min=4009 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1020, 1023, 1024, 1004, 10, 4, 0, 0, 1016, 1009, 1051, 991, 0, 0, 0, 0, 1103, 1034, 1018, 1060, 0, 0, 0, 0, 989, 1029, 981, 1076, 0, 0, 0, 0, 1015, 1026, 1008, 1028, 26, 0, 0, 0, 1047, 1034, 996, 1014, 0, 0, 0, 0, 1049, 990, 1004, 1012, 4, 2, 0, 0, 1030, 973, 1027, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4067, 4215, 4075, 4103, 4091, 4061, 4071] avg=4096.0000 max=4215 max_rank=2 min=4061 min_rank=6 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1018, 1000, 1030, 1030, 14, 9, 0, 0, 1049, 1037, 1003, 1034, 0, 0, 0, 0, 964, 1061, 985, 1040, 0, 0, 0, 0, 1001, 1044, 1018, 1015, 0, 0, 0, 0, 990, 1021, 1014, 1051, 29, 1, 0, 0, 1003, 1062, 1054, 1049, 0, 0, 0, 0, 993, 1038, 1026, 1019, 7, 8, 6, 0, 970, 1004, 1022, 1049, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4123, 4050, 4078, 4106, 4168, 4097, 4045] avg=4096.0000 max=4168 max_rank=5 min=4045 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[977, 1051, 1017, 1018, 10, 5, 0, 0, 1019, 1015, 1002, 1029, 0, 0, 0, 0, 1019, 1020, 1053, 1026, 0, 0, 0, 0, 1037, 1031, 1015, 1042, 0, 0, 0, 0, 999, 1040, 924, 1021, 25, 0, 0, 0, 1044, 1016, 1048, 1012, 0, 0, 0, 0, 1043, 1041, 1028, 1025, 4, 2, 0, 0, 1008, 998, 1065, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4065, 4118, 4125, 4009, 4120, 4143, 4110] avg=4096.0000 max=4143 max_rank=6 min=4009 min_rank=4 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1000, 1056, 1013, 1036, 1036, 1062, 1034, 1007, 997, 1004, 1016, 1063, 1015, 1039, 1013, 993, 1060, 1004, 1058, 1015, 1019, 995, 1039, 1014, 1030, 963, 1005, 1033, 1055, 1019, 1036, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1056, 1029, 1003, 1040, 1041, 1010, 1062, 1075, 997, 1025, 1017, 1019, 1046, 1043, 1082, 1048, 986, 1020, 970, 1011, 1048, 996, 1066, 1006, 985, 1011, 991, 1036, 990, 984, 1000, 1075] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[988, 1029, 979, 977, 1024, 1021, 1086, 1018, 1049, 1039, 1023, 1023, 1061, 1015, 1047, 1019, 1008, 1007, 1003, 1019, 1023, 1034, 999, 1027, 1057, 1011, 1054, 1005, 1016, 972, 1101, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1037, 1021, 1028, 1020, 1042, 1044, 1026, 1107, 1026, 972, 1022, 1016, 1040, 1005, 1045, 1051, 1026, 942, 1030, 1012, 1024, 1018, 1075, 982, 1076, 1004, 968, 1013, 1075, 1008, 968, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1032, 1037, 1026, 1001, 1015, 1000, 1035, 1032, 1027, 1063, 1021, 993, 1018, 1016, 988, 997, 1030, 1060, 1021, 1019, 1045, 1036, 1054, 989, 1059, 1026, 1026, 1037, 984, 1008, 1036, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[995, 1011, 989, 1024, 1085, 998, 1045, 1059, 1060, 1026, 1001, 990, 1004, 1010, 1012, 1072, 1023, 1013, 1009, 1019, 1063, 1047, 1037, 1001, 1020, 1037, 1037, 1031, 993, 1027, 1005, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[989, 997, 1041, 991, 1008, 967, 1048, 1055, 1025, 1009, 1043, 1029, 1011, 1090, 1046, 1004, 1094, 1031, 1061, 986, 976, 1027, 1029, 1049, 1001, 1052, 1011, 1057, 1030, 1021, 1018, 972] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[998, 1050, 1040, 1015, 991, 1040, 968, 1007, 965, 1045, 1045, 1029, 1004, 1048, 1015, 979, 1025, 1083, 1094, 1066, 1019, 1021, 950, 1013, 1025, 1035, 1043, 1035, 1065, 1093, 971, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1056, 1029, 1003, 1040, 28, 0, 0, 0, 1041, 1010, 1062, 1040, 0, 0, 0, 0, 997, 1025, 1017, 1019, 7, 3, 0, 0, 1046, 1031, 1082, 1048, 0, 0, 0, 0, 982, 1020, 970, 1011, 0, 0, 0, 0, 1048, 996, 1066, 1006, 4, 0, 0, 0, 985, 1011, 991, 1036, 0, 0, 0, 0, 990, 984, 1000, 1075, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4153, 4068, 4207, 3983, 4120, 4023, 4058] avg=4096.0000 max=4207 max_rank=3 min=3983 min_rank=4 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16184 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1000, 1056, 1013, 1036, 26, 0, 0, 0, 1036, 1062, 1034, 974, 0, 0, 0, 0, 997, 1004, 1016, 1063, 7, 3, 0, 0, 1015, 1027, 1013, 993, 0, 0, 0, 0, 1056, 1004, 1058, 1015, 0, 0, 0, 0, 1019, 995, 1039, 1014, 4, 0, 0, 0, 1030, 963, 1005, 1033, 0, 0, 0, 0, 1055, 1019, 1036, 1039, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4106, 4090, 4048, 4133, 4071, 4031, 4158] avg=4096.0000 max=4158 max_rank=7 min=4031 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[988, 1029, 979, 977, 26, 0, 0, 0, 1024, 1021, 1086, 985, 0, 0, 0, 0, 1049, 1039, 1023, 1023, 7, 3, 0, 0, 1061, 1003, 1047, 1019, 0, 0, 0, 0, 1004, 1007, 1003, 1019, 0, 0, 0, 0, 1023, 1034, 999, 1027, 4, 0, 0, 0, 1057, 1011, 1054, 1005, 0, 0, 0, 0, 1016, 972, 1101, 1034, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[3999, 4116, 4144, 4130, 4033, 4087, 4127, 4132] avg=4096.0000 max=4144 max_rank=2 min=3999 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1032, 1037, 1026, 1001, 27, 0, 0, 0, 1015, 1000, 1035, 998, 0, 0, 0, 0, 1027, 1063, 1021, 993, 7, 3, 0, 0, 1018, 1004, 988, 997, 0, 0, 0, 0, 1026, 1060, 1021, 1019, 0, 0, 0, 0, 1045, 1036, 1054, 989, 4, 0, 0, 0, 1059, 1026, 1026, 1037, 0, 0, 0, 0, 984, 1008, 1036, 1037, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4048, 4114, 4007, 4126, 4128, 4148, 4074] avg=4096.0000 max=4148 max_rank=6 min=4007 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1037, 1021, 1028, 1020, 29, 0, 0, 0, 1042, 1044, 1026, 1071, 0, 0, 0, 0, 1026, 972, 1022, 1016, 7, 3, 0, 0, 1040, 993, 1045, 1051, 0, 0, 0, 0, 1022, 942, 1030, 1012, 0, 0, 0, 0, 1024, 1018, 1075, 982, 4, 0, 0, 0, 1076, 1004, 968, 1013, 0, 0, 0, 0, 1075, 1008, 968, 1045, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4183, 4046, 4129, 4006, 4103, 4061, 4105] avg=4096.0000 max=4183 max_rank=1 min=4006 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[995, 1011, 989, 1024, 31, 0, 0, 0, 1085, 998, 1045, 1017, 0, 0, 0, 0, 1060, 1026, 1001, 990, 11, 8, 0, 0, 1004, 986, 1012, 1072, 0, 0, 0, 0, 1014, 1013, 1009, 1019, 0, 0, 0, 0, 1063, 1047, 1037, 1001, 9, 6, 4, 0, 1014, 1037, 1037, 1031, 0, 0, 0, 0, 993, 1027, 1005, 1025, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4145, 4096, 4074, 4055, 4167, 4119, 4062] avg=4096.0000 max=4167 max_rank=5 min=4050 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[998, 1050, 1040, 1015, 26, 0, 0, 0, 991, 1040, 968, 974, 0, 0, 0, 0, 965, 1045, 1045, 1029, 7, 3, 0, 0, 1004, 1036, 1015, 979, 0, 0, 0, 0, 1021, 1083, 1094, 1066, 0, 0, 0, 0, 1019, 1021, 950, 1013, 4, 0, 0, 0, 1025, 1035, 1043, 1035, 0, 0, 0, 0, 1065, 1093, 971, 991, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 3973, 4094, 4034, 4264, 4007, 4138, 4129] avg=4096.0000 max=4264 max_rank=4 min=3973 min_rank=1 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16538 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[989, 997, 1041, 991, 27, 0, 0, 0, 1008, 967, 1048, 1021, 0, 0, 0, 0, 1025, 1009, 1043, 1029, 7, 3, 0, 0, 1011, 1078, 1046, 1004, 0, 0, 0, 0, 1090, 1031, 1061, 986, 0, 0, 0, 0, 976, 1027, 1029, 1049, 4, 0, 0, 0, 1001, 1052, 1011, 1057, 0, 0, 0, 0, 1030, 1021, 1018, 972, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4044, 4116, 4139, 4168, 4085, 4121, 4050] avg=4096.0000 max=4168 max_rank=4 min=4044 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True + [2026-08-30 04:58:08] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 616.1 | throughput per GPU (TFLOP/s/GPU): 541.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.276508E+00 | loss scale: 1.0 | grad norm: 5.102 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1022, 1029, 1019, 961, 1004, 1022, 1003, 957, 1022, 1041, 1023, 1011, 1033, 1058, 1064, 991, 1008, 997, 1047, 1067, 1066, 980, 1046, 1006, 1044, 991, 1059, 999, 1012, 1041, 1048, 1097] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1011, 1001, 1022, 1013, 979, 1011, 1032, 1040, 1043, 1011, 1031, 1050, 1032, 995, 1068, 1015, 1045, 1041, 1033, 999, 1013, 1070, 1025, 1016, 1029, 1010, 992, 1021, 1031, 1011, 1077, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1027, 1051, 998, 996, 1041, 1075, 1055, 1023, 983, 1012, 1009, 1074, 999, 990, 1035, 1020, 1038, 975, 1011, 994, 1035, 1061, 1032, 1023, 1065, 1004, 1035, 1056, 1018, 1007, 1015, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1040, 1026, 1034, 1007, 1017, 988, 1014, 1050, 1012, 999, 1026, 986, 1041, 1033, 1030, 1034, 1038, 1016, 1052, 991, 1003, 1042, 1040, 1055, 1040, 1037, 971, 1074, 1006, 1001, 1008, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[988, 980, 1057, 1020, 991, 986, 1011, 1033, 984, 1004, 962, 1046, 1029, 1051, 1091, 1025, 1056, 1052, 993, 994, 1028, 1045, 987, 1047, 1004, 1085, 1015, 1052, 1018, 1037, 1040, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1000, 1026, 995, 1020, 1018, 1049, 1059, 1053, 1009, 1051, 1029, 1048, 1066, 1012, 1028, 1011, 1014, 1000, 949, 1011, 1042, 1019, 1052, 1034, 1004, 1031, 1005, 1017, 1020, 1001, 1053, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[992, 978, 1041, 1020, 1057, 1032, 1102, 1039, 1060, 1010, 1052, 979, 1028, 1001, 1045, 1018, 1002, 1002, 1006, 1000, 1040, 1065, 976, 1093, 1031, 1000, 988, 1040, 1037, 1057, 1004, 973] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[975, 1060, 1045, 1048, 1016, 1030, 1030, 1029, 1034, 1057, 1032, 1061, 1015, 1053, 1059, 961, 1001, 1042, 999, 1011, 1016, 1043, 1062, 1037, 1061, 1007, 976, 1027, 961, 1068, 933, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1022, 1029, 1019, 961, 20, 6, 5, 0, 1004, 1022, 995, 957, 0, 0, 0, 0, 1022, 1041, 1023, 1011, 2, 0, 0, 0, 1033, 1058, 1044, 991, 0, 0, 0, 0, 1008, 997, 1047, 1067, 33, 0, 0, 0, 1066, 942, 1046, 1006, 0, 0, 0, 0, 1044, 991, 1059, 999, 0, 0, 0, 0, 1012, 1041, 1048, 1097, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 3978, 4099, 4126, 4152, 4060, 4093, 4198] avg=4096.0000 max=4198 max_rank=7 min=3978 min_rank=1 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1011, 1001, 1022, 1013, 20, 6, 6, 0, 979, 1011, 1024, 1040, 0, 0, 0, 0, 1043, 1011, 1031, 1050, 2, 0, 0, 0, 1032, 995, 1048, 1015, 0, 0, 0, 0, 1045, 1041, 1033, 999, 36, 0, 0, 0, 1013, 1028, 1025, 1016, 0, 0, 0, 0, 1029, 1010, 992, 1021, 0, 0, 0, 0, 1031, 1011, 1077, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4054, 4137, 4090, 4154, 4082, 4052, 4120] avg=4096.0000 max=4154 max_rank=4 min=4052 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1027, 1051, 998, 996, 20, 7, 5, 0, 1041, 1075, 1046, 1023, 0, 0, 0, 0, 983, 1012, 1009, 1074, 2, 0, 0, 0, 999, 990, 1015, 1020, 0, 0, 0, 0, 1038, 975, 1011, 994, 36, 0, 0, 0, 1035, 1020, 1032, 1023, 0, 0, 0, 0, 1065, 1004, 1035, 1056, 0, 0, 0, 0, 1018, 1007, 1015, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4185, 4080, 4024, 4054, 4110, 4160, 4051] avg=4096.0000 max=4185 max_rank=1 min=4024 min_rank=3 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1040, 1026, 1034, 1007, 19, 6, 5, 0, 1017, 988, 1006, 1050, 0, 0, 0, 0, 1012, 999, 1026, 986, 2, 0, 0, 0, 1041, 1033, 1011, 1034, 0, 0, 0, 0, 1038, 1016, 1052, 991, 35, 0, 0, 0, 1003, 1002, 1040, 1055, 0, 0, 0, 0, 1040, 1037, 971, 1074, 0, 0, 0, 0, 1006, 1001, 1008, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4061, 4025, 4119, 4132, 4100, 4122, 4072] avg=4096.0000 max=4137 max_rank=0 min=4025 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1000, 1026, 995, 1020, 23, 12, 10, 0, 1018, 1049, 1039, 1053, 0, 0, 0, 0, 1009, 1051, 1029, 1048, 3, 0, 0, 0, 1066, 1012, 1005, 1011, 0, 0, 0, 0, 1014, 1000, 949, 1011, 38, 0, 0, 0, 1042, 971, 1052, 1034, 0, 0, 0, 0, 1004, 1031, 1005, 1015, 0, 0, 0, 0, 1020, 1001, 1053, 1042, 5, 2, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4159, 4140, 4094, 4012, 4099, 4055, 4123] avg=4096.0000 max=4159 max_rank=1 min=4012 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[988, 980, 1057, 1020, 21, 6, 5, 0, 991, 986, 1003, 1033, 0, 0, 0, 0, 984, 1004, 962, 1046, 2, 0, 0, 0, 1029, 1051, 1070, 1025, 0, 0, 0, 0, 1056, 1052, 993, 994, 35, 0, 0, 0, 1028, 1005, 987, 1047, 0, 0, 0, 0, 1004, 1085, 1015, 1052, 0, 0, 0, 0, 1018, 1037, 1040, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4013, 3998, 4175, 4130, 4067, 4156, 4152] avg=4096.0000 max=4175 max_rank=3 min=3998 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[992, 978, 1041, 1020, 20, 7, 6, 0, 1057, 1032, 1093, 1039, 0, 0, 0, 0, 1060, 1010, 1052, 979, 2, 0, 0, 0, 1028, 1001, 1025, 1018, 0, 0, 0, 0, 1002, 1002, 1006, 1000, 36, 0, 0, 0, 1040, 1023, 976, 1093, 0, 0, 0, 0, 1031, 1000, 988, 1040, 0, 0, 0, 0, 1037, 1057, 1004, 973, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4221, 4103, 4072, 4046, 4132, 4059, 4071] avg=4096.0000 max=4221 max_rank=1 min=4046 min_rank=4 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[975, 1060, 1045, 1048, 20, 6, 5, 0, 1016, 1030, 1022, 1029, 0, 0, 0, 0, 1034, 1057, 1032, 1061, 2, 0, 0, 0, 1015, 1053, 1039, 961, 0, 0, 0, 0, 1001, 1042, 999, 1011, 35, 0, 0, 0, 1016, 1003, 1062, 1037, 0, 0, 0, 0, 1061, 1007, 976, 1027, 0, 0, 0, 0, 961, 1068, 933, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4097, 4186, 4068, 4088, 4118, 4071, 3981] avg=4096.0000 max=4186 max_rank=2 min=3981 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16258 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[997, 977, 1057, 1041, 1039, 1028, 1029, 1032, 1056, 985, 1015, 1039, 1049, 1017, 981, 1001, 1026, 1041, 1003, 1063, 1023, 972, 1007, 1070, 1023, 1035, 1064, 1007, 1026, 1037, 1031, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1036, 1040, 1052, 1035, 1010, 1029, 1011, 1002, 978, 1060, 1011, 1025, 1046, 1058, 973, 1043, 1014, 1051, 1034, 996, 1066, 983, 1065, 1072, 977, 1028, 1023, 1030, 1028, 995, 996, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1037, 1008, 1027, 1003, 1011, 1099, 1005, 1037, 1038, 1009, 1049, 1063, 1000, 1015, 1025, 985, 1036, 1009, 985, 1002, 989, 1064, 982, 1010, 1044, 996, 1059, 993, 1074, 1022, 1074, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1028, 1007, 1007, 1040, 972, 1010, 1026, 1055, 1032, 986, 1011, 1038, 1059, 1069, 1040, 1049, 954, 1081, 1020, 959, 1030, 1021, 1077, 1059, 1035, 1023, 1039, 1037, 1014, 1035, 994, 961] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1057, 1076, 1055, 1005, 976, 1065, 1049, 1015, 1015, 1056, 992, 1023, 1002, 1031, 1012, 1027, 1022, 996, 1011, 1087, 1014, 1046, 1019, 975, 1054, 964, 1010, 1011, 1038, 1047, 1021, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1023, 1028, 1019, 1053, 1011, 1057, 1068, 1032, 1057, 1015, 1045, 1049, 1019, 1038, 1023, 1018, 974, 1014, 995, 997, 1027, 1007, 1024, 961, 1035, 1066, 1025, 992, 1025, 1066, 997, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1084, 1048, 993, 1079, 1042, 977, 994, 1027, 1028, 1044, 968, 1040, 1041, 1007, 998, 1039, 1028, 1050, 1036, 1026, 1018, 1001, 993, 1047, 1033, 1006, 1031, 973, 1037, 1048, 1037, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1080, 1035, 1054, 1019, 1053, 985, 1043, 1067, 1038, 1002, 1045, 1039, 940, 1049, 986, 1056, 1028, 1002, 998, 1015, 957, 1024, 1054, 1054, 1029, 1003, 1057, 1034, 999, 983, 1031, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[997, 1008, 1027, 1003, 0, 0, 0, 0, 1011, 1099, 1005, 1027, 0, 0, 0, 0, 1038, 1009, 1049, 1053, 0, 0, 0, 0, 1000, 1015, 1025, 985, 9, 0, 0, 0, 1036, 1009, 985, 1002, 26, 0, 0, 0, 989, 1064, 982, 1010, 6, 0, 0, 0, 1044, 996, 1059, 993, 4, 0, 0, 0, 1074, 1022, 1074, 1018, 14, 1, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4142, 4149, 4034, 4058, 4051, 4096, 4203] avg=4096.0000 max=4203 max_rank=7 min=4034 min_rank=3 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[996, 1040, 1052, 1035, 0, 0, 0, 0, 1010, 1029, 1011, 993, 0, 0, 0, 0, 978, 1060, 1011, 1016, 0, 0, 0, 0, 1046, 1058, 973, 1043, 8, 0, 0, 0, 1014, 1051, 1034, 996, 26, 0, 0, 0, 1066, 983, 1065, 1072, 6, 0, 0, 0, 977, 1028, 1023, 1030, 3, 0, 0, 0, 1028, 995, 996, 1001, 14, 1, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4043, 4065, 4128, 4121, 4192, 4061, 4035] avg=4096.0000 max=4192 max_rank=5 min=4035 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1016, 1076, 1055, 1005, 0, 0, 0, 0, 976, 1065, 1049, 1006, 0, 0, 0, 0, 1015, 1056, 992, 1014, 0, 0, 0, 0, 1002, 1031, 1012, 1027, 8, 0, 0, 0, 1022, 996, 1011, 1087, 27, 0, 0, 0, 1014, 1046, 1019, 975, 6, 0, 0, 0, 1054, 964, 1010, 1011, 3, 0, 0, 0, 1038, 1047, 1021, 997, 14, 1, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4096, 4077, 4080, 4143, 4060, 4042, 4118] avg=4096.0000 max=4152 max_rank=0 min=4042 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[988, 1007, 1007, 1040, 0, 0, 0, 0, 972, 1010, 1026, 1045, 0, 0, 0, 0, 1032, 986, 1011, 1029, 0, 0, 0, 0, 1059, 1069, 1040, 1049, 9, 0, 0, 0, 954, 1081, 1020, 959, 26, 0, 0, 0, 1030, 1021, 1077, 1059, 6, 0, 0, 0, 1035, 1023, 1039, 1037, 3, 0, 0, 0, 1014, 1035, 994, 961, 14, 1, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4053, 4058, 4226, 4040, 4193, 4137, 4019] avg=4096.0000 max=4226 max_rank=3 min=4019 min_rank=7 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[974, 1028, 1019, 1053, 0, 0, 0, 0, 1011, 1057, 1068, 1010, 0, 0, 0, 0, 1057, 1015, 1045, 1031, 0, 0, 0, 0, 1019, 1038, 1023, 1018, 13, 0, 0, 0, 974, 1014, 995, 997, 31, 0, 0, 0, 1027, 1007, 1024, 961, 9, 6, 0, 0, 1035, 1066, 1025, 992, 9, 0, 0, 0, 1025, 1066, 997, 1008, 18, 3, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4146, 4148, 4111, 4011, 4034, 4127, 4117] avg=4096.0000 max=4148 max_rank=2 min=4011 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1042, 1048, 993, 1079, 0, 0, 0, 0, 1042, 977, 994, 1018, 0, 0, 0, 0, 1028, 1044, 968, 1030, 0, 0, 0, 0, 1041, 1007, 998, 1039, 8, 0, 0, 0, 1028, 1050, 1036, 1026, 27, 0, 0, 0, 1018, 1001, 993, 1047, 6, 0, 0, 0, 1033, 1006, 1031, 973, 4, 0, 0, 0, 1037, 1048, 1037, 995, 15, 1, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4031, 4070, 4093, 4167, 4065, 4047, 4133] avg=4096.0000 max=4167 max_rank=4 min=4031 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1038, 1035, 1054, 1019, 0, 0, 0, 0, 1053, 985, 1043, 1057, 0, 0, 0, 0, 1038, 1002, 1045, 1030, 0, 0, 0, 0, 940, 1049, 986, 1056, 9, 0, 0, 0, 1028, 1002, 998, 1015, 27, 0, 0, 0, 957, 1024, 1054, 1054, 6, 0, 0, 0, 1029, 1003, 1057, 1034, 3, 0, 0, 0, 999, 983, 1031, 1009, 15, 1, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4138, 4115, 4040, 4070, 4095, 4126, 4038] avg=4096.0000 max=4146 max_rank=0 min=4038 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[959, 977, 1057, 1041, 0, 0, 0, 0, 1039, 1028, 1029, 1023, 0, 0, 0, 0, 1056, 985, 1015, 1030, 0, 0, 0, 0, 1049, 1017, 981, 1001, 8, 0, 0, 0, 1026, 1041, 1003, 1063, 25, 0, 0, 0, 1023, 972, 1007, 1070, 6, 0, 0, 0, 1023, 1035, 1064, 1007, 3, 0, 0, 0, 1026, 1037, 1031, 997, 13, 1, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4119, 4086, 4056, 4158, 4078, 4132, 4105] avg=4096.0000 max=4158 max_rank=4 min=4034 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8064, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1033, 1055, 1079, 1004, 1012, 1013, 1025, 985, 1010, 967, 1003, 1025, 951, 1009, 983, 1031, 1061, 985, 1056, 1041, 1013, 1070, 1037, 1011, 1045, 1070, 989, 1030, 1016, 1006, 1066, 1087] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1037, 1005, 1027, 1033, 1001, 997, 1025, 1042, 998, 1000, 1033, 1020, 1052, 1037, 991, 1040, 1060, 1007, 1068, 987, 995, 1013, 1034, 1038, 1019, 1026, 1009, 1051, 1039, 1069, 1012, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1036, 1027, 976, 1031, 1000, 1054, 1018, 1003, 1002, 1038, 1045, 1015, 1072, 1057, 1042, 1055, 1030, 1023, 1010, 991, 1014, 1049, 992, 1041, 1071, 1013, 963, 1048, 1024, 1015, 985, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1031, 1042, 1070, 980, 996, 1030, 1049, 1025, 980, 1037, 984, 1042, 1031, 987, 1034, 1032, 992, 1064, 1006, 1011, 1006, 1038, 1026, 1048, 1043, 1017, 1039, 1072, 1016, 991, 1042, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[991, 990, 1016, 1032, 960, 992, 1099, 1026, 1042, 1042, 1005, 1004, 992, 1009, 1100, 991, 1024, 1019, 995, 1014, 1061, 1017, 1038, 1028, 1083, 1027, 1041, 1006, 1031, 1032, 1023, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1025, 1007, 978, 954, 1052, 1007, 971, 1042, 991, 1042, 1040, 1030, 1076, 1063, 1049, 1078, 1006, 985, 1052, 1018, 1035, 1039, 994, 1023, 1034, 1035, 1049, 1014, 999, 1019, 1047, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[975, 1026, 1055, 1028, 1057, 1074, 1019, 998, 1032, 991, 1031, 1065, 1006, 1028, 1032, 1005, 1015, 963, 1016, 1000, 1060, 973, 1049, 1072, 1034, 997, 1002, 1026, 1010, 1057, 1019, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[973, 1017, 1004, 1084, 998, 1024, 1026, 1009, 1053, 1050, 993, 1004, 1063, 1047, 1007, 981, 977, 1041, 1063, 1066, 1005, 977, 1005, 1051, 1016, 1017, 1052, 1034, 976, 1081, 1003, 1071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1033, 1055, 1079, 1004, 11, 6, 0, 0, 1012, 1013, 1025, 985, 9, 7, 0, 0, 1010, 967, 1003, 1025, 19, 0, 0, 0, 933, 1009, 983, 1031, 0, 0, 0, 0, 1061, 985, 1056, 1041, 10, 4, 0, 0, 1013, 1070, 1037, 1001, 0, 0, 0, 0, 1020, 1070, 989, 1030, 0, 0, 0, 0, 1016, 1006, 1066, 1074, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4188, 4051, 4024, 3956, 4157, 4121, 4109, 4162] avg=4096.0000 max=4188 max_rank=0 min=3956 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1037, 1005, 1027, 1033, 12, 6, 0, 0, 1001, 997, 1025, 1042, 8, 8, 0, 0, 998, 1000, 1033, 1020, 18, 0, 0, 0, 1032, 1037, 991, 1040, 0, 0, 0, 0, 1060, 1007, 1068, 987, 10, 4, 0, 0, 995, 1013, 1034, 1028, 0, 0, 0, 0, 995, 1026, 1009, 1051, 0, 0, 0, 0, 1039, 1069, 1012, 991, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4081, 4069, 4100, 4136, 4070, 4081, 4111] avg=4096.0000 max=4136 max_rank=4 min=4069 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1036, 1027, 976, 1031, 12, 6, 0, 0, 1000, 1054, 1018, 1003, 9, 8, 0, 0, 1002, 1038, 1045, 1015, 19, 0, 0, 0, 1052, 1057, 1042, 1055, 0, 0, 0, 0, 1030, 1023, 1010, 991, 10, 4, 0, 0, 1014, 1049, 992, 1031, 0, 0, 0, 0, 1046, 1013, 963, 1048, 0, 0, 0, 0, 1024, 1015, 985, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4092, 4119, 4206, 4068, 4086, 4070, 4039] avg=4096.0000 max=4206 max_rank=3 min=4039 min_rank=7 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1031, 1042, 1070, 980, 12, 6, 0, 0, 996, 1030, 1049, 1025, 8, 8, 0, 0, 980, 1037, 984, 1042, 19, 0, 0, 0, 1011, 987, 1034, 1032, 0, 0, 0, 0, 992, 1064, 1006, 1011, 10, 4, 0, 0, 1006, 1038, 1026, 1038, 0, 0, 0, 0, 1018, 1017, 1039, 1072, 0, 0, 0, 0, 1016, 991, 1042, 995, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4116, 4062, 4064, 4087, 4108, 4146, 4044] avg=4096.0000 max=4146 max_rank=6 min=4044 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[991, 990, 1016, 1032, 11, 6, 0, 0, 960, 992, 1099, 1026, 9, 7, 0, 0, 1042, 1042, 1005, 1004, 19, 0, 0, 0, 974, 1009, 1100, 991, 0, 0, 0, 0, 1024, 1019, 995, 1014, 10, 4, 0, 0, 1061, 1017, 1038, 1018, 0, 0, 0, 0, 1058, 1027, 1041, 1006, 0, 0, 0, 0, 1031, 1032, 1023, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4093, 4112, 4074, 4066, 4134, 4132, 4111] avg=4096.0000 max=4134 max_rank=5 min=4046 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[973, 1017, 1004, 1084, 12, 6, 0, 0, 998, 1024, 1026, 1009, 9, 8, 0, 0, 1053, 1050, 993, 1004, 18, 0, 0, 0, 1043, 1047, 1007, 981, 0, 0, 0, 0, 977, 1041, 1063, 1066, 10, 4, 0, 0, 1005, 977, 1005, 1041, 0, 0, 0, 0, 992, 1017, 1052, 1034, 0, 0, 0, 0, 976, 1081, 1003, 1058, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4074, 4118, 4078, 4161, 4028, 4095, 4118] avg=4096.0000 max=4161 max_rank=4 min=4028 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1025, 1007, 978, 954, 12, 6, 0, 0, 1052, 1007, 971, 1042, 8, 8, 0, 0, 991, 1042, 1040, 1030, 19, 0, 0, 0, 1056, 1063, 1049, 1078, 0, 0, 0, 0, 1006, 985, 1052, 1018, 10, 4, 0, 0, 1035, 1039, 994, 1013, 0, 0, 0, 0, 1009, 1035, 1049, 1014, 0, 0, 0, 0, 999, 1019, 1047, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3982, 4088, 4122, 4246, 4075, 4081, 4107, 4067] avg=4096.0000 max=4246 max_rank=3 min=3982 min_rank=0 imbalance_ratio=1.036621 max_load_reduction=28522 max_load_reduction_pct=0.870422 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[975, 1026, 1055, 1028, 15, 8, 0, 0, 1057, 1074, 1019, 998, 13, 12, 0, 0, 1032, 991, 1031, 1065, 23, 0, 0, 0, 979, 1028, 1032, 1005, 0, 0, 0, 0, 1015, 963, 1016, 1000, 14, 10, 0, 0, 1060, 973, 1049, 1058, 0, 0, 0, 0, 1003, 997, 1002, 1026, 0, 0, 0, 0, 1010, 1057, 1019, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4173, 4142, 4044, 4018, 4140, 4028, 4116] avg=4096.0000 max=4173 max_rank=1 min=4018 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1067, 1068, 1008, 1029, 1036, 1011, 1057, 999, 1035, 1009, 974, 1043, 1040, 1002, 1007, 1017, 1039, 1045, 1019, 1005, 1011, 1045, 1021, 1051, 1040, 979, 1069, 1001, 1006, 1011, 1031, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1033, 1042, 1066, 1017, 975, 1004, 982, 1053, 1076, 1004, 1046, 985, 1025, 1052, 1004, 1052, 1041, 1010, 968, 999, 1044, 1049, 1021, 1043, 1025, 1030, 1057, 1014, 1003, 1035, 1027, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1026, 1002, 1018, 1065, 1015, 1018, 1044, 1014, 1011, 981, 1023, 1017, 1024, 1037, 1021, 1004, 995, 1044, 1020, 1020, 1057, 1037, 1034, 976, 1041, 1058, 980, 1088, 1015, 1083, 978, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1002, 1028, 1074, 1016, 1030, 1044, 1051, 1003, 1016, 1005, 1057, 1020, 1043, 1027, 1050, 983, 1001, 1027, 1052, 1034, 1005, 1051, 1039, 997, 1030, 1025, 996, 1069, 995, 1034, 958, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1040, 1002, 1034, 984, 1020, 1044, 967, 1029, 1053, 1029, 1072, 1003, 1027, 985, 1037, 1047, 997, 1038, 994, 1101, 1033, 1016, 1026, 968, 1052, 1010, 1034, 1020, 1016, 1036, 1030, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1020, 1039, 1009, 1003, 1008, 978, 1040, 1029, 1079, 1026, 1064, 1044, 996, 1055, 983, 985, 1023, 1055, 974, 1006, 1055, 1005, 1012, 1063, 1052, 1005, 1011, 1034, 1058, 1021, 1031, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1020, 1024, 1072, 1021, 1005, 1054, 999, 1024, 1045, 1072, 980, 992, 1025, 1030, 988, 991, 971, 970, 1064, 1074, 1010, 986, 1033, 1016, 1000, 1003, 1031, 1051, 1056, 1037, 1057, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1008, 1022, 1012, 1025, 1084, 1069, 988, 1050, 1001, 1041, 1048, 1036, 1006, 1019, 1026, 1019, 996, 964, 1017, 1033, 1027, 976, 1011, 1052, 1036, 1011, 1068, 1017, 1081, 970, 1036, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1033, 1042, 1050, 1017, 0, 0, 0, 0, 975, 1004, 982, 1053, 5, 0, 0, 0, 1062, 1004, 1046, 985, 0, 0, 0, 0, 1025, 1052, 1004, 1052, 16, 4, 0, 0, 1041, 1010, 968, 999, 20, 0, 0, 0, 1044, 1049, 1021, 1043, 0, 0, 0, 0, 1025, 1030, 1057, 994, 0, 0, 0, 0, 1003, 1035, 1027, 986, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4019, 4097, 4153, 4038, 4157, 4106, 4056] avg=4096.0000 max=4157 max_rank=5 min=4019 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1067, 1068, 993, 1029, 0, 0, 0, 0, 1036, 1011, 1057, 999, 5, 0, 0, 0, 1022, 1009, 974, 1043, 0, 0, 0, 0, 1040, 1002, 1007, 1017, 15, 4, 0, 0, 1039, 1045, 1019, 1005, 20, 0, 0, 0, 1011, 1045, 1021, 1051, 0, 0, 0, 0, 1040, 979, 1069, 981, 0, 0, 0, 0, 1006, 1011, 1031, 993, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4108, 4048, 4085, 4128, 4128, 4069, 4045] avg=4096.0000 max=4157 max_rank=0 min=4045 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1020, 1039, 994, 1003, 0, 0, 0, 0, 1008, 978, 1040, 1029, 5, 0, 0, 0, 1065, 1026, 1064, 1044, 0, 0, 0, 0, 996, 1055, 983, 985, 15, 4, 0, 0, 1023, 1055, 974, 1006, 21, 0, 0, 0, 1055, 1005, 1012, 1063, 0, 0, 0, 0, 1052, 1005, 1011, 1013, 0, 0, 0, 0, 1058, 1021, 1031, 1005, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4060, 4199, 4038, 4079, 4135, 4081, 4120] avg=4096.0000 max=4199 max_rank=2 min=4038 min_rank=3 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1026, 1002, 1003, 1065, 0, 0, 0, 0, 1015, 1018, 1044, 1014, 5, 0, 0, 0, 998, 981, 1023, 1017, 0, 0, 0, 0, 1024, 1037, 1021, 1004, 15, 4, 0, 0, 995, 1044, 1020, 1020, 22, 0, 0, 0, 1057, 1037, 1034, 976, 0, 0, 0, 0, 1041, 1058, 980, 1066, 0, 0, 0, 0, 1015, 1083, 978, 1022, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4019, 4105, 4101, 4104, 4145, 4102] avg=4096.0000 max=4145 max_rank=6 min=4019 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1040, 1002, 1019, 984, 0, 0, 0, 0, 1020, 1044, 967, 1029, 5, 0, 0, 0, 1040, 1029, 1072, 1003, 0, 0, 0, 0, 1027, 985, 1037, 1047, 15, 4, 0, 0, 997, 1038, 994, 1101, 20, 0, 0, 0, 1033, 1016, 1026, 968, 0, 0, 0, 0, 1052, 1010, 1034, 1000, 0, 0, 0, 0, 1016, 1036, 1030, 1024, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4065, 4144, 4115, 4150, 4043, 4096, 4110] avg=4096.0000 max=4150 max_rank=4 min=4043 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1020, 1024, 1056, 1021, 0, 0, 0, 0, 1005, 1054, 999, 1024, 5, 0, 0, 0, 1032, 1072, 980, 992, 0, 0, 0, 0, 1025, 1030, 988, 991, 16, 4, 0, 0, 971, 970, 1064, 1074, 21, 0, 0, 0, 1010, 986, 1033, 1016, 0, 0, 0, 0, 1000, 1003, 1031, 1030, 0, 0, 0, 0, 1056, 1037, 1057, 1067, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4087, 4076, 4054, 4100, 4045, 4064, 4221] avg=4096.0000 max=4221 max_rank=7 min=4045 min_rank=5 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1002, 1028, 1053, 1016, 0, 0, 0, 0, 1030, 1044, 1051, 1003, 9, 0, 0, 0, 990, 1005, 1057, 1020, 0, 0, 0, 0, 1043, 1027, 1050, 983, 18, 8, 0, 0, 1001, 1027, 1052, 1034, 25, 3, 0, 0, 1003, 1051, 1039, 997, 0, 0, 0, 0, 1030, 1025, 996, 1044, 0, 0, 0, 0, 995, 1034, 958, 1006, 9, 2, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4137, 4072, 4129, 4142, 4090, 4095, 4004] avg=4096.0000 max=4142 max_rank=4 min=4004 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1008, 1022, 997, 1025, 0, 0, 0, 0, 1084, 1069, 988, 1050, 5, 0, 0, 0, 988, 1041, 1048, 1036, 0, 0, 0, 0, 1006, 1019, 1026, 1019, 15, 4, 0, 0, 996, 964, 1017, 1033, 20, 0, 0, 0, 1027, 976, 1011, 1052, 0, 0, 0, 0, 1036, 1011, 1068, 997, 0, 0, 0, 0, 1081, 970, 1036, 1019, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4196, 4113, 4089, 4030, 4066, 4112, 4110] avg=4096.0000 max=4196 max_rank=1 min=4030 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1041, 1006, 1024, 1002, 1023, 1005, 1021, 1064, 1055, 976, 1033, 1030, 1047, 1031, 1008, 1021, 1033, 1038, 1054, 1019, 1019, 1016, 993, 1034, 1009, 1028, 987, 1024, 1072, 1058, 986, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1036, 1055, 1090, 1024, 980, 981, 1020, 1027, 1033, 1033, 1064, 1027, 1061, 1039, 998, 1024, 1029, 966, 966, 1053, 1018, 1010, 986, 986, 1044, 1006, 1029, 1025, 1054, 1067, 1052, 985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1017, 1039, 1020, 1031, 1058, 1021, 1030, 998, 1010, 992, 1067, 1033, 1064, 987, 965, 1000, 1049, 1042, 1011, 1021, 1050, 1000, 992, 1045, 1018, 999, 1058, 1032, 1040, 1023, 1003, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1058, 1071, 1042, 1018, 1047, 1017, 1046, 1000, 1018, 1028, 1028, 1048, 1040, 1040, 1028, 999, 1097, 1001, 998, 1047, 1023, 1013, 1043, 1008, 991, 1012, 980, 1001, 970, 985, 1049, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1035, 1028, 1016, 986, 994, 1058, 989, 999, 1028, 1024, 996, 1050, 995, 1038, 1039, 1012, 1004, 1058, 1022, 968, 1018, 1050, 1044, 976, 1011, 1034, 1096, 1045, 1011, 1028, 1075, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1061, 1036, 1002, 1000, 1013, 1065, 1032, 1007, 984, 1026, 964, 995, 967, 991, 970, 1031, 1070, 1029, 1018, 1060, 1047, 1043, 1044, 1005, 1023, 1029, 1073, 1028, 1040, 1056, 1011, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1025, 1062, 996, 961, 1040, 1036, 1001, 1017, 1040, 1018, 1044, 1050, 1058, 966, 1003, 1012, 1023, 1043, 1077, 1073, 991, 1011, 1045, 1049, 1035, 1043, 1010, 1017, 996, 1005, 1038, 983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1036, 1059, 1000, 1046, 1048, 1021, 1040, 1016, 1028, 1010, 1052, 1028, 1044, 1039, 962, 1030, 1032, 966, 1074, 979, 1017, 1017, 1063, 1035, 1046, 1016, 1021, 970, 991, 1023, 1002, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1036, 1036, 1090, 1024, 0, 0, 0, 0, 980, 981, 1020, 1027, 4, 1, 0, 0, 1033, 1033, 1064, 1023, 0, 0, 0, 0, 1061, 1039, 998, 1024, 19, 12, 0, 0, 1012, 966, 966, 1053, 0, 0, 0, 0, 1018, 1010, 986, 986, 5, 3, 0, 0, 1044, 1006, 1029, 1025, 3, 0, 0, 0, 1054, 1060, 1052, 985, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4013, 4153, 4153, 3997, 4008, 4107, 4151] avg=4096.0000 max=4186 max_rank=0 min=3997 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1041, 988, 1024, 1002, 0, 0, 0, 0, 1023, 1005, 1021, 1064, 4, 1, 0, 0, 1055, 976, 1033, 1026, 0, 0, 0, 0, 1047, 1031, 1008, 1021, 18, 12, 0, 0, 1016, 1038, 1054, 1019, 0, 0, 0, 0, 1019, 1016, 993, 1034, 5, 3, 0, 0, 1009, 1028, 987, 1024, 3, 0, 0, 0, 1072, 1051, 986, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4118, 4090, 4137, 4127, 4070, 4051, 4120] avg=4096.0000 max=4137 max_rank=3 min=4051 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1058, 1048, 1042, 1018, 0, 0, 0, 0, 1047, 1017, 1046, 1000, 10, 6, 0, 0, 1018, 1028, 1028, 1035, 0, 0, 0, 0, 1040, 1040, 1028, 999, 23, 18, 0, 0, 1068, 1001, 998, 1047, 0, 0, 0, 0, 1023, 1013, 1043, 1008, 11, 8, 0, 0, 991, 1012, 980, 1001, 7, 0, 0, 0, 970, 967, 1049, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4126, 4109, 4148, 4114, 4106, 3991, 4008] avg=4096.0000 max=4166 max_rank=0 min=3991 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16219 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1017, 1020, 1020, 1031, 0, 0, 0, 0, 1058, 1021, 1030, 998, 4, 2, 0, 0, 1010, 992, 1067, 1028, 0, 0, 0, 0, 1064, 987, 965, 1000, 19, 13, 0, 0, 1030, 1042, 1011, 1021, 0, 0, 0, 0, 1050, 1000, 992, 1045, 6, 3, 0, 0, 1018, 999, 1058, 1032, 3, 0, 0, 0, 1040, 1016, 1003, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4113, 4097, 4048, 4104, 4096, 4110, 4112] avg=4096.0000 max=4113 max_rank=1 min=4048 min_rank=3 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1035, 1009, 1016, 986, 0, 0, 0, 0, 994, 1058, 989, 999, 4, 2, 0, 0, 1028, 1024, 996, 1045, 0, 0, 0, 0, 995, 1038, 1039, 1012, 19, 12, 0, 0, 987, 1058, 1022, 968, 0, 0, 0, 0, 1018, 1050, 1044, 976, 5, 3, 0, 0, 1011, 1034, 1096, 1045, 3, 0, 0, 0, 1011, 1021, 1075, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4046, 4093, 4115, 4035, 4096, 4189, 4148] avg=4096.0000 max=4189 max_rank=6 min=4035 min_rank=4 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1061, 1017, 1002, 1000, 0, 0, 0, 0, 1013, 1065, 1032, 1007, 4, 1, 0, 0, 984, 1026, 964, 991, 0, 0, 0, 0, 967, 991, 970, 1031, 19, 13, 0, 0, 1051, 1029, 1018, 1060, 0, 0, 0, 0, 1047, 1043, 1044, 1005, 6, 3, 0, 0, 1023, 1029, 1073, 1028, 3, 0, 0, 0, 1040, 1049, 1011, 1048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4122, 3965, 3991, 4158, 4148, 4156, 4148] avg=4096.0000 max=4158 max_rank=4 min=3965 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16610 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1025, 1043, 996, 961, 0, 0, 0, 0, 1040, 1036, 1001, 1017, 4, 2, 0, 0, 1040, 1018, 1044, 1045, 0, 0, 0, 0, 1058, 966, 1003, 1012, 19, 12, 0, 0, 1006, 1043, 1077, 1073, 0, 0, 0, 0, 991, 1011, 1045, 1049, 5, 3, 0, 0, 1035, 1043, 1010, 1017, 3, 0, 0, 0, 996, 998, 1038, 983, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4100, 4147, 4070, 4199, 4104, 4108, 4015] avg=4096.0000 max=4199 max_rank=4 min=4015 min_rank=7 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1036, 1040, 1000, 1046, 0, 0, 0, 0, 1048, 1021, 1040, 1016, 4, 1, 0, 0, 1028, 1010, 1052, 1024, 0, 0, 0, 0, 1044, 1039, 962, 1030, 19, 12, 0, 0, 1015, 966, 1074, 979, 0, 0, 0, 0, 1017, 1017, 1063, 1035, 5, 3, 0, 0, 1046, 1016, 1021, 970, 3, 0, 0, 0, 991, 1016, 1002, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4130, 4114, 4106, 4034, 4140, 4056, 4066] avg=4096.0000 max=4140 max_rank=5 min=4034 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1034, 1030, 1025, 1007, 1018, 1022, 1023, 993, 1070, 1066, 969, 1054, 1008, 1024, 1029, 996, 990, 1046, 1032, 1017, 1045, 980, 1049, 1012, 1007, 977, 1046, 1047, 1012, 1047, 1047, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1036, 1050, 1059, 1013, 1008, 1024, 987, 1060, 1019, 1035, 974, 1033, 1008, 981, 1039, 1022, 995, 1037, 1008, 1076, 1003, 1021, 1016, 1035, 1004, 1045, 1062, 1031, 1003, 1045, 1017, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[981, 1027, 1018, 982, 994, 1031, 1069, 1034, 983, 994, 1007, 1037, 1034, 1038, 1026, 1020, 1048, 1058, 1038, 1038, 978, 1060, 1058, 995, 1014, 1073, 982, 988, 1101, 1059, 991, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1040, 1032, 1022, 1051, 1002, 1023, 1014, 970, 1032, 1079, 1008, 1017, 1074, 984, 1039, 973, 1033, 1070, 1042, 1049, 1005, 1040, 973, 1057, 1024, 1031, 1013, 1011, 1056, 1017, 976, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1070, 1068, 1005, 970, 980, 1020, 1043, 1005, 993, 1042, 1019, 1020, 1026, 1022, 1041, 1041, 1019, 1011, 1020, 1050, 1023, 1007, 1022, 1037, 1005, 1040, 993, 1029, 1072, 1047, 1005, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1041, 1038, 1053, 1001, 1018, 1042, 992, 1021, 994, 1078, 1023, 1054, 1043, 1002, 1044, 985, 995, 1071, 1040, 981, 1010, 1020, 1080, 991, 1013, 1031, 1001, 979, 993, 1019, 1083, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1060, 995, 1000, 1037, 1054, 1001, 1017, 1011, 993, 1072, 1023, 1022, 1073, 1003, 1014, 1034, 1038, 1011, 1064, 975, 1077, 969, 1008, 979, 1071, 1008, 988, 1042, 1027, 1033, 1051, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1035, 1053, 992, 969, 999, 1009, 1082, 1011, 1026, 1005, 1078, 1020, 1084, 1015, 1043, 1008, 1037, 1043, 989, 1080, 1021, 1000, 1071, 976, 995, 968, 1019, 1039, 1034, 1032, 1014, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1033, 1050, 1059, 1013, 0, 0, 0, 0, 1008, 1024, 987, 1060, 18, 5, 0, 0, 1019, 1027, 974, 1033, 0, 0, 0, 0, 1008, 981, 1039, 1022, 0, 0, 0, 0, 995, 1009, 1008, 1076, 0, 0, 0, 0, 1003, 1021, 1016, 1035, 8, 6, 3, 0, 1004, 1045, 1062, 1031, 23, 0, 0, 0, 1003, 1021, 1017, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4102, 4053, 4050, 4088, 4092, 4165, 4063] avg=4096.0000 max=4165 max_rank=6 min=4050 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1037, 1032, 1022, 1051, 0, 0, 0, 0, 1002, 1023, 1014, 970, 18, 5, 0, 0, 1032, 1070, 1008, 1017, 0, 0, 0, 0, 1074, 984, 1039, 973, 0, 0, 0, 0, 1033, 1041, 1042, 1049, 0, 0, 0, 0, 1005, 1040, 973, 1057, 9, 5, 3, 0, 1024, 1031, 1013, 1011, 24, 0, 0, 0, 1056, 994, 976, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4032, 4127, 4070, 4165, 4092, 4103, 4037] avg=4096.0000 max=4165 max_rank=4 min=4032 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1031, 1030, 1025, 1007, 0, 0, 0, 0, 1018, 1022, 1023, 993, 18, 5, 0, 0, 1070, 1057, 969, 1054, 0, 0, 0, 0, 1008, 1024, 1029, 996, 0, 0, 0, 0, 990, 1017, 1032, 1017, 0, 0, 0, 0, 1045, 980, 1049, 1012, 9, 6, 3, 0, 1007, 977, 1046, 1047, 24, 0, 0, 0, 1012, 1023, 1047, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4079, 4150, 4057, 4056, 4104, 4101, 4128] avg=4096.0000 max=4150 max_rank=2 min=4056 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1057, 995, 1000, 1037, 0, 0, 0, 0, 1054, 1001, 1017, 1011, 18, 4, 0, 0, 993, 1063, 1023, 1022, 0, 0, 0, 0, 1073, 1003, 1014, 1034, 0, 0, 0, 0, 1038, 984, 1064, 975, 0, 0, 0, 0, 1077, 969, 1008, 979, 9, 5, 3, 0, 1071, 1008, 988, 1042, 23, 0, 0, 0, 1027, 1010, 1051, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4105, 4101, 4124, 4061, 4050, 4132, 4106] avg=4096.0000 max=4132 max_rank=6 min=4050 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[978, 1027, 1018, 982, 0, 0, 0, 0, 994, 1031, 1069, 1034, 19, 5, 0, 0, 983, 986, 1007, 1037, 0, 0, 0, 0, 1034, 1038, 1026, 1020, 0, 0, 0, 0, 1048, 1029, 1038, 1038, 0, 0, 0, 0, 978, 1060, 1058, 995, 8, 6, 3, 0, 1014, 1073, 982, 988, 24, 0, 0, 0, 1101, 1034, 991, 1012, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 4152, 4013, 4118, 4153, 4108, 4081, 4138] avg=4096.0000 max=4153 max_rank=4 min=4005 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1067, 1068, 1005, 970, 0, 0, 0, 0, 980, 1020, 1043, 1005, 18, 4, 0, 0, 993, 1034, 1019, 1020, 0, 0, 0, 0, 1026, 1022, 1041, 1041, 0, 0, 0, 0, 1019, 984, 1020, 1050, 0, 0, 0, 0, 1023, 1007, 1022, 1037, 8, 6, 3, 0, 1005, 1040, 993, 1029, 23, 0, 0, 0, 1072, 1023, 1005, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4070, 4066, 4130, 4073, 4106, 4090, 4123] avg=4096.0000 max=4130 max_rank=3 min=4066 min_rank=2 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1038, 1038, 1053, 1001, 0, 0, 0, 0, 1018, 1042, 992, 1021, 18, 5, 0, 0, 994, 1069, 1023, 1054, 0, 0, 0, 0, 1043, 1002, 1044, 985, 0, 0, 0, 0, 995, 1042, 1040, 981, 0, 0, 0, 0, 1010, 1020, 1080, 991, 9, 5, 3, 0, 1013, 1031, 1001, 979, 24, 0, 0, 0, 993, 996, 1083, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4096, 4140, 4074, 4058, 4118, 4048, 4104] avg=4096.0000 max=4140 max_rank=2 min=4048 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1030, 1053, 992, 969, 0, 0, 0, 0, 999, 1009, 1082, 1011, 23, 8, 0, 0, 1026, 994, 1078, 1020, 0, 0, 0, 0, 1079, 1015, 1043, 1008, 0, 0, 0, 0, 1037, 1008, 989, 1080, 0, 0, 0, 0, 1021, 1000, 1071, 976, 11, 9, 5, 5, 995, 968, 1019, 1039, 27, 0, 0, 0, 1034, 1000, 1014, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4132, 4118, 4145, 4114, 4098, 4048, 4069] avg=4096.0000 max=4145 max_rank=3 min=4044 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8320, 8192, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1000, 1053, 1043, 978, 1011, 973, 989, 999, 1038, 1043, 998, 1043, 1022, 986, 1066, 1044, 1094, 1010, 1008, 1035, 1062, 1019, 1013, 1026, 1015, 1039, 1025, 997, 1023, 1060, 1017, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1058, 1070, 1004, 1027, 973, 1022, 1002, 969, 1041, 1050, 984, 1009, 1021, 1002, 1047, 1002, 1014, 1055, 1024, 1011, 1027, 1060, 1031, 1036, 998, 1042, 1025, 1012, 1033, 1069, 1052, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1064, 1032, 996, 1027, 1028, 994, 1030, 1024, 1005, 1002, 1045, 1077, 998, 1000, 1035, 1024, 1008, 1035, 1016, 1036, 1043, 1026, 986, 1025, 1011, 1032, 1067, 1027, 1040, 990, 1033, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[993, 1015, 996, 1088, 1035, 1059, 1016, 1092, 1077, 1044, 968, 1020, 1001, 960, 984, 1025, 960, 1055, 1010, 1015, 1051, 1009, 1069, 1043, 1008, 968, 1050, 1060, 1013, 1001, 1017, 1066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1012, 1012, 1051, 1022, 1012, 1015, 1087, 1001, 1007, 1020, 1029, 1027, 1025, 1040, 1051, 1020, 998, 1064, 1013, 976, 980, 1011, 1052, 1004, 1006, 1059, 999, 1028, 987, 1052, 1031, 1077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1036, 1003, 1023, 1020, 981, 986, 1067, 1045, 1066, 1013, 1102, 1018, 990, 997, 994, 1028, 998, 1055, 1002, 997, 998, 1045, 1057, 1043, 1019, 1063, 1005, 1024, 1058, 966, 1025, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1026, 1029, 982, 1045, 1050, 1066, 1059, 999, 1006, 1017, 1047, 1022, 990, 1008, 1031, 1018, 993, 1019, 1014, 1053, 984, 1066, 1045, 1044, 1081, 976, 1012, 1015, 1053, 1026, 995, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[953, 1060, 1042, 1062, 1017, 1008, 1052, 1036, 1040, 1053, 1009, 1022, 1000, 1064, 993, 1029, 1055, 996, 953, 1038, 1019, 1035, 1060, 989, 1017, 1019, 1015, 1043, 1047, 1065, 978, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1000, 1047, 1043, 978, 0, 0, 0, 0, 1011, 973, 989, 999, 5, 3, 0, 0, 1017, 1043, 998, 1043, 0, 0, 0, 0, 1022, 986, 1066, 1044, 23, 10, 0, 0, 1094, 1010, 1008, 1035, 11, 8, 0, 0, 1062, 1019, 990, 1026, 0, 0, 0, 0, 1015, 1039, 1025, 997, 1, 0, 0, 0, 1012, 1060, 1017, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 3980, 4101, 4151, 4166, 4097, 4077, 4128] avg=4096.0000 max=4166 max_rank=4 min=3980 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1058, 1064, 1004, 1027, 0, 0, 0, 0, 973, 1022, 1002, 969, 5, 3, 0, 0, 1020, 1050, 984, 1009, 0, 0, 0, 0, 1021, 1002, 1047, 1002, 23, 10, 0, 0, 1014, 1055, 1024, 1011, 11, 8, 0, 0, 1027, 1060, 1008, 1036, 0, 0, 0, 0, 998, 1042, 1025, 1012, 1, 0, 0, 0, 1022, 1069, 1052, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 3974, 4063, 4105, 4123, 4131, 4078, 4141] avg=4096.0000 max=4153 max_rank=0 min=3974 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[993, 1009, 996, 1088, 0, 0, 0, 0, 1035, 1059, 1016, 1092, 5, 3, 0, 0, 1056, 1044, 968, 1020, 0, 0, 0, 0, 1001, 960, 984, 1025, 24, 10, 0, 0, 960, 1055, 1010, 1015, 11, 8, 0, 0, 1051, 1009, 1045, 1043, 0, 0, 0, 0, 1008, 968, 1050, 1060, 1, 0, 0, 0, 1002, 1001, 1017, 1066, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4210, 4088, 4004, 4059, 4148, 4087, 4086] avg=4096.0000 max=4210 max_rank=1 min=4004 min_rank=3 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1012, 1006, 1051, 1022, 0, 0, 0, 0, 1012, 1015, 1087, 1001, 5, 3, 0, 0, 986, 1020, 1029, 1027, 0, 0, 0, 0, 1025, 1040, 1051, 1020, 24, 10, 0, 0, 998, 1064, 1013, 976, 11, 8, 0, 0, 980, 1011, 1028, 1004, 0, 0, 0, 0, 1006, 1059, 999, 1028, 1, 0, 0, 0, 976, 1052, 1031, 1077, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4123, 4062, 4170, 4070, 4023, 4093, 4136] avg=4096.0000 max=4170 max_rank=3 min=4023 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1064, 1020, 996, 1027, 0, 0, 0, 0, 1028, 994, 1030, 1024, 8, 7, 0, 0, 978, 1002, 1045, 1077, 0, 0, 0, 0, 998, 1000, 1035, 1024, 25, 13, 0, 0, 1008, 1035, 1016, 1036, 14, 11, 0, 0, 1043, 1026, 961, 1025, 0, 0, 0, 0, 1011, 1032, 1067, 1027, 4, 0, 0, 0, 1022, 990, 1033, 1012, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4091, 4102, 4095, 4120, 4055, 4141, 4057] avg=4096.0000 max=4141 max_rank=6 min=4055 min_rank=5 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1036, 997, 1023, 1020, 0, 0, 0, 0, 981, 986, 1067, 1045, 5, 3, 0, 0, 1045, 1013, 1102, 1018, 0, 0, 0, 0, 990, 997, 994, 1028, 24, 10, 0, 0, 998, 1055, 1002, 997, 11, 8, 0, 0, 998, 1045, 1033, 1043, 0, 0, 0, 0, 1019, 1063, 1005, 1024, 1, 0, 0, 0, 1047, 966, 1025, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4087, 4178, 4043, 4071, 4119, 4112, 4082] avg=4096.0000 max=4178 max_rank=2 min=4043 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 1023, 982, 1045, 0, 0, 0, 0, 1050, 1066, 1059, 999, 5, 3, 0, 0, 985, 1017, 1047, 1022, 0, 0, 0, 0, 990, 1008, 1031, 1018, 23, 10, 0, 0, 993, 1019, 1014, 1053, 11, 8, 0, 0, 984, 1066, 1022, 1044, 0, 0, 0, 0, 1081, 976, 1012, 1015, 1, 0, 0, 0, 1042, 1026, 995, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4182, 4071, 4080, 4098, 4116, 4085, 4060] avg=4096.0000 max=4182 max_rank=1 min=4060 min_rank=7 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[953, 1054, 1042, 1062, 0, 0, 0, 0, 1017, 1008, 1052, 1036, 5, 3, 0, 0, 1019, 1053, 1009, 1022, 0, 0, 0, 0, 1000, 1064, 993, 1029, 24, 10, 0, 0, 1055, 996, 953, 1038, 11, 8, 0, 0, 1019, 1035, 1036, 989, 0, 0, 0, 0, 1017, 1019, 1015, 1043, 1, 0, 0, 0, 1036, 1065, 978, 999, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4121, 4103, 4120, 4061, 4079, 4095, 4078] avg=4096.0000 max=4121 max_rank=1 min=4061 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8064, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8064, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1072, 1002, 1017, 1016, 990, 1009, 992, 1080, 1013, 1026, 992, 968, 994, 1063, 1040, 1067, 1022, 1035, 994, 1028, 1021, 1043, 1026, 1013, 1028, 1088, 1005, 1001, 1056, 1027, 1035, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1007, 1052, 1010, 1028, 1075, 979, 1033, 1005, 988, 1004, 1010, 1046, 1002, 1014, 1053, 1041, 1015, 1039, 1012, 1048, 1020, 1051, 1017, 1023, 1045, 1018, 1052, 1012, 999, 1029, 1025, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[989, 1080, 1031, 1044, 1027, 1059, 1023, 1025, 1056, 977, 1040, 1003, 1001, 1047, 1019, 1041, 1017, 995, 980, 1038, 1026, 1020, 1024, 967, 1054, 998, 1011, 1031, 1055, 1018, 1002, 1070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1009, 1007, 1006, 975, 1056, 1023, 1033, 1051, 1072, 997, 1040, 1021, 1021, 1079, 1017, 1013, 1068, 1027, 1006, 983, 1041, 957, 1059, 1008, 1008, 1017, 1039, 1004, 1038, 1026, 1029, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[998, 1031, 1014, 1053, 1022, 1050, 1055, 1070, 992, 1047, 1000, 985, 987, 1043, 1036, 1017, 1018, 1043, 1030, 1077, 1007, 992, 1026, 1048, 1012, 1019, 1014, 1008, 1018, 1033, 984, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1046, 1063, 1050, 1013, 1059, 1027, 1067, 1036, 1004, 1014, 1042, 1031, 1011, 1069, 1036, 1031, 1003, 1060, 975, 1050, 986, 1048, 1012, 1031, 988, 1031, 1005, 989, 1003, 1004, 973, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1000, 1081, 1013, 1063, 1024, 1031, 1013, 1028, 967, 1082, 1044, 997, 1014, 1020, 1048, 1027, 1074, 976, 1026, 1041, 971, 1029, 1038, 1075, 986, 1034, 999, 1011, 1037, 1020, 1009, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[970, 963, 1011, 974, 1003, 1043, 1044, 1009, 1018, 1028, 975, 1030, 1028, 1052, 1037, 1044, 1055, 1060, 1073, 1081, 1038, 1034, 1018, 995, 1032, 1029, 1050, 1008, 1013, 988, 1054, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1072, 1002, 1017, 1016, 9, 0, 0, 0, 990, 1009, 992, 1046, 0, 0, 0, 0, 1013, 1026, 992, 968, 33, 0, 0, 0, 994, 1033, 1040, 1067, 0, 0, 0, 0, 1022, 1035, 994, 1007, 0, 0, 0, 0, 1021, 1043, 1026, 1013, 12, 0, 0, 0, 1028, 1088, 1005, 1001, 16, 1, 0, 0, 1056, 1027, 1035, 1005, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4037, 4032, 4134, 4058, 4115, 4139, 4137] avg=4096.0000 max=4139 max_rank=6 min=4032 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[989, 1080, 1031, 1044, 9, 0, 0, 0, 1027, 1059, 1023, 993, 0, 0, 0, 0, 1056, 977, 1040, 1003, 31, 0, 0, 0, 1001, 1018, 1019, 1041, 0, 0, 0, 0, 1017, 995, 980, 1017, 0, 0, 0, 0, 1026, 1020, 1024, 967, 12, 0, 0, 0, 1054, 998, 1011, 1031, 15, 1, 0, 0, 1055, 1018, 1002, 1070, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4102, 4107, 4079, 4009, 4049, 4110, 4159] avg=4096.0000 max=4159 max_rank=7 min=4009 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1009, 1007, 1006, 975, 9, 0, 0, 0, 1056, 1023, 1033, 1018, 0, 0, 0, 0, 1072, 997, 1040, 1021, 32, 0, 0, 0, 1021, 1049, 1017, 1013, 0, 0, 0, 0, 1068, 1027, 1006, 963, 0, 0, 0, 0, 1041, 957, 1059, 1008, 11, 0, 0, 0, 1008, 1017, 1039, 1004, 16, 1, 0, 0, 1038, 1026, 1029, 1038, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4130, 4162, 4100, 4064, 4076, 4085, 4145] avg=4096.0000 max=4162 max_rank=2 min=4006 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1007, 1052, 1010, 1028, 14, 0, 0, 0, 1075, 979, 1033, 963, 0, 0, 0, 0, 988, 1004, 1010, 1046, 35, 0, 0, 0, 1002, 976, 1053, 1041, 0, 0, 0, 0, 1015, 1039, 1012, 1018, 0, 0, 0, 0, 1020, 1051, 1017, 1023, 16, 3, 0, 0, 1045, 1018, 1052, 1012, 19, 7, 0, 0, 999, 1029, 1025, 1016, 16, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4050, 4083, 4072, 4084, 4130, 4153, 4085] avg=4096.0000 max=4153 max_rank=6 min=4050 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[998, 1031, 1014, 1053, 10, 0, 0, 0, 1022, 1050, 1055, 1036, 0, 0, 0, 0, 992, 1047, 1000, 985, 33, 0, 0, 0, 987, 1014, 1036, 1017, 0, 0, 0, 0, 1018, 1043, 1030, 1054, 0, 0, 0, 0, 1007, 992, 1026, 1048, 13, 0, 0, 0, 1012, 1019, 1014, 1008, 15, 1, 0, 0, 1018, 1033, 984, 1039, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4163, 4057, 4054, 4145, 4086, 4069, 4088] avg=4096.0000 max=4163 max_rank=1 min=4054 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1046, 1063, 1050, 1013, 10, 0, 0, 0, 1059, 1027, 1067, 1003, 0, 0, 0, 0, 1004, 1014, 1042, 1031, 32, 0, 0, 0, 1011, 1039, 1036, 1031, 0, 0, 0, 0, 1003, 1060, 975, 1028, 0, 0, 0, 0, 986, 1048, 1012, 1031, 12, 0, 0, 0, 988, 1031, 1005, 989, 16, 1, 0, 0, 1003, 1004, 973, 1011, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 4156, 4123, 4117, 4066, 4089, 4030, 4005] avg=4096.0000 max=4182 max_rank=0 min=4005 min_rank=7 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16190 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1000, 1081, 1013, 1063, 9, 0, 0, 0, 1024, 1031, 1013, 995, 0, 0, 0, 0, 967, 1082, 1044, 997, 32, 0, 0, 0, 1014, 992, 1048, 1027, 0, 0, 0, 0, 1074, 976, 1026, 1020, 0, 0, 0, 0, 971, 1029, 1038, 1075, 12, 0, 0, 0, 986, 1034, 999, 1011, 15, 1, 0, 0, 1037, 1020, 1009, 990, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4063, 4122, 4081, 4096, 4125, 4046, 4069] avg=4096.0000 max=4166 max_rank=0 min=4046 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[970, 963, 1011, 974, 10, 0, 0, 0, 1003, 1043, 1044, 977, 0, 0, 0, 0, 1018, 1028, 975, 1030, 31, 0, 0, 0, 1028, 1022, 1037, 1044, 0, 0, 0, 0, 1055, 1060, 1073, 1058, 0, 0, 0, 0, 1038, 1034, 1018, 995, 13, 0, 0, 0, 1032, 1029, 1050, 1008, 16, 1, 0, 0, 1013, 988, 1054, 1011, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[3928, 4067, 4082, 4131, 4246, 4098, 4136, 4080] avg=4096.0000 max=4246 max_rank=4 min=3928 min_rank=0 imbalance_ratio=1.036621 max_load_reduction=28522 max_load_reduction_pct=0.870422 moved_to_echo=16560 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[989, 1021, 1056, 1039, 1013, 1026, 1049, 1020, 1069, 1021, 1016, 993, 1088, 1008, 1024, 1034, 989, 1012, 996, 1039, 1030, 993, 1056, 1053, 1016, 1004, 1005, 1038, 1044, 1021, 1004, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1017, 1037, 1032, 1053, 1053, 1036, 987, 1042, 1009, 1052, 978, 1011, 1081, 1006, 1024, 1053, 1014, 990, 1036, 1033, 1026, 990, 1046, 1027, 1023, 1013, 1036, 988, 1041, 982, 1019, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1043, 994, 1059, 976, 1008, 1010, 1019, 1048, 1000, 1038, 1011, 1039, 1029, 1068, 1039, 1020, 1029, 1058, 1029, 1054, 1024, 1005, 1003, 1053, 989, 1020, 1031, 1033, 1000, 976, 1037, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1019, 1069, 971, 995, 1033, 1053, 1003, 1004, 1035, 1009, 1076, 1038, 971, 1024, 1031, 1010, 998, 1080, 1052, 1048, 952, 1029, 1024, 1055, 1001, 1013, 1020, 1074, 1016, 1053, 980, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1000, 1045, 1054, 1028, 983, 1008, 1026, 1012, 1037, 1010, 1006, 977, 1014, 1013, 1015, 1034, 1049, 1037, 1054, 1000, 1060, 1009, 988, 1019, 1053, 1038, 1070, 997, 1051, 1014, 1023, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1007, 998, 1044, 1030, 1029, 1067, 1023, 1023, 1010, 1042, 1038, 1057, 1016, 1020, 1043, 1042, 1070, 1043, 1037, 1006, 1018, 1022, 1011, 1054, 1018, 978, 1041, 978, 977, 1020, 1013, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1034, 1052, 1035, 1060, 1086, 1019, 981, 1030, 1059, 1029, 1044, 1006, 1061, 1009, 1013, 957, 988, 997, 1012, 1030, 1012, 1018, 1019, 1030, 1040, 1033, 1064, 988, 1010, 982, 1046, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1002, 1016, 1052, 1116, 995, 1043, 1055, 1020, 1008, 1010, 1004, 1018, 1014, 1000, 1045, 1039, 1056, 996, 993, 978, 1036, 997, 1003, 1041, 1045, 1064, 968, 977, 1027, 1066, 1062, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[989, 1021, 1035, 1039, 0, 0, 0, 0, 1013, 1022, 1049, 1020, 0, 0, 0, 0, 1069, 1021, 1016, 993, 2, 0, 0, 0, 1079, 1008, 1024, 1034, 0, 0, 0, 0, 989, 1008, 996, 1039, 0, 0, 0, 0, 1030, 993, 1056, 1053, 4, 2, 1, 0, 1016, 1004, 1005, 1038, 8, 5, 0, 0, 1044, 1021, 1004, 1002, 16, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4104, 4101, 4145, 4032, 4139, 4076, 4087] avg=4096.0000 max=4145 max_rank=3 min=4032 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1043, 994, 1038, 976, 0, 0, 0, 0, 1008, 1006, 1019, 1048, 0, 0, 0, 0, 1000, 1038, 1011, 1039, 2, 0, 0, 0, 1020, 1068, 1039, 1020, 0, 0, 0, 0, 1029, 1054, 1029, 1054, 0, 0, 0, 0, 1024, 1005, 1003, 1053, 4, 2, 1, 0, 989, 1020, 1031, 1033, 8, 5, 0, 0, 1000, 976, 1037, 1026, 16, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4081, 4090, 4147, 4166, 4092, 4086, 4055] avg=4096.0000 max=4166 max_rank=4 min=4051 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1019, 1069, 952, 995, 0, 0, 0, 0, 1033, 1049, 1003, 1004, 0, 0, 0, 0, 1035, 1009, 1076, 1038, 2, 0, 0, 0, 963, 1024, 1031, 1010, 0, 0, 0, 0, 998, 1076, 1052, 1048, 0, 0, 0, 0, 952, 1029, 1024, 1055, 4, 2, 1, 0, 1001, 1013, 1020, 1074, 7, 5, 0, 0, 1016, 1053, 980, 1032, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4089, 4160, 4028, 4174, 4067, 4120, 4095] avg=4096.0000 max=4174 max_rank=4 min=4028 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1017, 1037, 1001, 1053, 0, 0, 0, 0, 1053, 1028, 987, 1042, 0, 0, 0, 0, 1009, 1052, 978, 1011, 4, 0, 0, 0, 1063, 1006, 1024, 1053, 0, 0, 0, 0, 1014, 983, 1036, 1033, 0, 0, 0, 0, 1026, 990, 1046, 1027, 8, 3, 5, 0, 1023, 1013, 1036, 988, 13, 12, 0, 0, 1041, 982, 1019, 1033, 19, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4110, 4054, 4146, 4066, 4105, 4085, 4094] avg=4096.0000 max=4146 max_rank=3 min=4054 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1000, 1045, 1033, 1028, 0, 0, 0, 0, 983, 1004, 1026, 1012, 0, 0, 0, 0, 1037, 1010, 1006, 977, 2, 0, 0, 0, 1006, 1013, 1015, 1034, 0, 0, 0, 0, 1049, 1033, 1054, 1000, 0, 0, 0, 0, 1060, 1009, 988, 1019, 4, 2, 1, 0, 1053, 1038, 1070, 997, 7, 5, 0, 0, 1051, 1014, 1023, 1044, 16, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4025, 4032, 4068, 4136, 4083, 4170, 4148] avg=4096.0000 max=4170 max_rank=6 min=4025 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1034, 1052, 1015, 1060, 0, 0, 0, 0, 1086, 1015, 981, 1030, 0, 0, 0, 0, 1059, 1029, 1044, 1006, 2, 0, 0, 0, 1052, 1009, 1013, 957, 0, 0, 0, 0, 988, 993, 1012, 1030, 0, 0, 0, 0, 1012, 1018, 1019, 1030, 4, 2, 1, 0, 1040, 1033, 1064, 988, 8, 5, 0, 0, 1010, 982, 1046, 1024, 15, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4112, 4140, 4031, 4023, 4086, 4138, 4077] avg=4096.0000 max=4161 max_rank=0 min=4023 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1007, 998, 1023, 1030, 0, 0, 0, 0, 1029, 1063, 1023, 1023, 0, 0, 0, 0, 1010, 1042, 1038, 1057, 2, 0, 0, 0, 1008, 1020, 1043, 1042, 0, 0, 0, 0, 1070, 1039, 1037, 1006, 0, 0, 0, 0, 1018, 1022, 1011, 1054, 4, 2, 1, 0, 1018, 978, 1041, 978, 7, 5, 0, 0, 977, 1020, 1013, 993, 16, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4138, 4149, 4113, 4152, 4112, 4027, 4019] avg=4096.0000 max=4152 max_rank=4 min=4019 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1002, 1016, 1031, 1116, 0, 0, 0, 0, 995, 1039, 1055, 1020, 0, 0, 0, 0, 1008, 1010, 1004, 1018, 2, 0, 0, 0, 1006, 1000, 1045, 1039, 0, 0, 0, 0, 1056, 992, 993, 978, 0, 0, 0, 0, 1036, 997, 1003, 1041, 4, 2, 1, 0, 1045, 1064, 968, 977, 7, 5, 0, 0, 1027, 1066, 1062, 1022, 16, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 4109, 4042, 4090, 4019, 4084, 4066, 4193] avg=4096.0000 max=4193 max_rank=7 min=4019 min_rank=4 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8448, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[991, 997, 1009, 951, 1026, 1021, 999, 1050, 1041, 1058, 1020, 1022, 1015, 1048, 1003, 1043, 1018, 975, 1040, 1049, 1018, 1040, 982, 1057, 1003, 1049, 1038, 1048, 1047, 1007, 1056, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1043, 1020, 995, 1011, 1014, 1037, 1031, 1048, 1004, 1010, 975, 1034, 1009, 1041, 1042, 1045, 1018, 1069, 962, 1028, 995, 1006, 1005, 1023, 1043, 1079, 1025, 1052, 1017, 1001, 1070, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1007, 1081, 1061, 1019, 997, 1055, 1009, 1013, 1012, 1056, 1031, 1004, 1043, 1022, 1018, 989, 1032, 973, 1034, 1049, 1000, 1076, 1031, 1009, 982, 1015, 997, 1036, 1015, 1054, 1006, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1003, 1004, 993, 1034, 1070, 1011, 1017, 1039, 1038, 1070, 1068, 981, 1010, 981, 982, 1009, 1083, 1023, 1085, 948, 984, 1039, 994, 1017, 1051, 1072, 1054, 988, 1028, 1046, 1038, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1009, 1067, 1022, 1038, 1028, 1050, 1040, 1024, 1026, 996, 1015, 1019, 1050, 1003, 1081, 1034, 1071, 1085, 986, 1009, 1052, 1004, 976, 1042, 974, 1001, 976, 1051, 1024, 999, 965, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[971, 1031, 993, 978, 1023, 973, 1002, 1042, 982, 1040, 1022, 1007, 1009, 1045, 1011, 1049, 1079, 971, 997, 1061, 1055, 1087, 1018, 1015, 1028, 1066, 1004, 1027, 1065, 1046, 1019, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1025, 1001, 1003, 1041, 1034, 948, 1038, 1015, 1031, 1035, 1024, 1073, 988, 1041, 994, 1001, 1003, 1042, 996, 1023, 1019, 999, 1068, 1028, 1076, 985, 1044, 1005, 1068, 1038, 1053, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1050, 1030, 987, 997, 1052, 1031, 1013, 1040, 1077, 1029, 1011, 1028, 1038, 1033, 1019, 1047, 976, 995, 1049, 1037, 972, 1033, 1011, 1027, 1029, 1007, 1030, 1041, 1027, 1016, 1041, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1003, 1004, 993, 1034, 27, 11, 0, 0, 1070, 1011, 1017, 1037, 0, 0, 0, 0, 1038, 1062, 1068, 981, 0, 0, 0, 0, 1010, 981, 982, 1009, 2, 0, 0, 0, 1083, 1023, 1085, 948, 0, 0, 0, 0, 984, 1039, 994, 1017, 8, 2, 0, 0, 1051, 1059, 1054, 988, 0, 0, 0, 0, 1001, 1046, 1038, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4135, 4149, 3984, 4139, 4044, 4152, 4093] avg=4096.0000 max=4152 max_rank=6 min=3984 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[991, 997, 1009, 951, 27, 11, 0, 0, 1026, 1021, 999, 1048, 0, 0, 0, 0, 1041, 1050, 1020, 1022, 0, 0, 0, 0, 1015, 1048, 1003, 1043, 2, 0, 0, 0, 1018, 975, 1040, 1049, 0, 0, 0, 0, 1018, 1040, 982, 1057, 8, 2, 0, 0, 1003, 1036, 1038, 1048, 0, 0, 0, 0, 1020, 1007, 1056, 1047, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3986, 4094, 4133, 4111, 4082, 4107, 4125, 4130] avg=4096.0000 max=4133 max_rank=2 min=3986 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[971, 1031, 993, 978, 28, 11, 0, 0, 1023, 973, 1002, 1040, 0, 0, 0, 0, 982, 1032, 1022, 1007, 0, 0, 0, 0, 1009, 1045, 1011, 1049, 2, 0, 0, 0, 1079, 971, 997, 1061, 0, 0, 0, 0, 1055, 1087, 1018, 1015, 8, 2, 0, 0, 1028, 1053, 1004, 1027, 0, 0, 0, 0, 1037, 1046, 1019, 1052, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4038, 4043, 4116, 4108, 4185, 4112, 4154] avg=4096.0000 max=4185 max_rank=5 min=4012 min_rank=0 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1025, 1001, 1003, 1041, 28, 10, 0, 0, 1034, 948, 1038, 1013, 0, 0, 0, 0, 1031, 1027, 1024, 1073, 0, 0, 0, 0, 988, 1041, 994, 1001, 2, 0, 0, 0, 1003, 1042, 996, 1023, 0, 0, 0, 0, 1019, 999, 1068, 1028, 8, 2, 0, 0, 1076, 973, 1044, 1005, 0, 0, 0, 0, 1040, 1038, 1053, 1029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4033, 4155, 4026, 4064, 4124, 4098, 4160] avg=4096.0000 max=4160 max_rank=7 min=4026 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1007, 1081, 1061, 1019, 26, 10, 0, 0, 997, 1055, 1009, 1011, 0, 0, 0, 0, 1012, 1048, 1031, 1004, 0, 0, 0, 0, 1043, 1022, 1018, 989, 2, 0, 0, 0, 1032, 973, 1034, 1049, 0, 0, 0, 0, 1000, 1076, 1031, 1009, 8, 2, 0, 0, 982, 1003, 997, 1036, 0, 0, 0, 0, 989, 1054, 1006, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4204, 4072, 4095, 4074, 4088, 4126, 4018, 4091] avg=4096.0000 max=4204 max_rank=0 min=4018 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1043, 1020, 995, 1011, 26, 11, 0, 0, 1014, 1037, 1031, 1046, 0, 0, 0, 0, 1004, 1002, 975, 1034, 0, 0, 0, 0, 1009, 1041, 1042, 1045, 2, 0, 0, 0, 1018, 1069, 962, 1028, 0, 0, 0, 0, 995, 1006, 1005, 1023, 8, 2, 0, 0, 1043, 1066, 1025, 1052, 0, 0, 0, 0, 991, 1001, 1070, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4128, 4015, 4139, 4077, 4039, 4186, 4078] avg=4096.0000 max=4186 max_rank=6 min=4015 min_rank=2 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1050, 1030, 987, 997, 27, 10, 0, 0, 1052, 1031, 1013, 1038, 0, 0, 0, 0, 1077, 1021, 1011, 1028, 0, 0, 0, 0, 1038, 1033, 1019, 1047, 2, 0, 0, 0, 976, 995, 1049, 1037, 0, 0, 0, 0, 972, 1033, 1011, 1027, 8, 2, 0, 0, 1029, 995, 1030, 1041, 0, 0, 0, 0, 1000, 1016, 1041, 995, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4134, 4137, 4139, 4057, 4053, 4095, 4052] avg=4096.0000 max=4139 max_rank=3 min=4052 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1009, 1067, 1022, 1038, 29, 14, 0, 0, 1028, 1050, 1040, 1016, 0, 0, 0, 0, 1026, 981, 1015, 1019, 0, 0, 0, 0, 1050, 1003, 1081, 1034, 6, 5, 0, 0, 1071, 1085, 986, 1009, 2, 0, 0, 0, 1052, 1004, 976, 1042, 10, 6, 0, 0, 974, 981, 976, 1051, 0, 0, 0, 0, 995, 999, 965, 1051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4134, 4041, 4179, 4153, 4090, 3982, 4010] avg=4096.0000 max=4179 max_rank=0 min=3982 min_rank=6 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16235 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1014, 1022, 1037, 1037, 1021, 1052, 1015, 1035, 992, 1027, 990, 1047, 968, 1017, 1048, 1062, 1022, 1050, 1009, 1060, 1083, 1019, 1005, 979, 1030, 1008, 1009, 1075, 998, 1023, 1015, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1028, 984, 1034, 984, 999, 1016, 1009, 1050, 1049, 1040, 1056, 1000, 1008, 1041, 1038, 1038, 973, 995, 1053, 1026, 1030, 1027, 1046, 1045, 1073, 1051, 1026, 1016, 989, 1009, 1034, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1054, 1024, 1048, 997, 958, 1051, 987, 1020, 1027, 1024, 998, 1033, 991, 1027, 1050, 1073, 998, 1041, 1028, 1029, 1041, 1044, 997, 1054, 1026, 1027, 1042, 944, 1025, 1049, 1021, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1009, 1083, 1082, 1041, 1029, 1006, 1046, 999, 1033, 970, 1027, 996, 1028, 1004, 1010, 1000, 989, 1004, 1012, 1074, 1034, 1006, 1032, 1058, 998, 1036, 974, 1042, 1057, 1028, 1038, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1053, 1025, 990, 1012, 1029, 1042, 1029, 1005, 1000, 1037, 1011, 1034, 1016, 1015, 1022, 1030, 1020, 1039, 1042, 1025, 1038, 1052, 1041, 1005, 1008, 1006, 1038, 1036, 1042, 952, 1029, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[994, 1024, 1006, 1001, 975, 1009, 1052, 1029, 1013, 1002, 1062, 1037, 1007, 1075, 1020, 1035, 1085, 995, 962, 1038, 1032, 1016, 1011, 1038, 1015, 982, 1038, 1072, 1066, 1012, 1027, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1073, 1073, 998, 1045, 987, 1024, 1022, 1011, 996, 1040, 1007, 978, 1023, 1073, 1034, 1037, 1045, 1023, 1008, 1024, 1015, 996, 1017, 1034, 1004, 1049, 1028, 994, 1001, 1012, 1032, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1048, 1046, 1033, 981, 1008, 1028, 997, 997, 1028, 965, 1087, 985, 998, 1029, 1031, 1014, 1034, 1058, 1031, 1067, 1023, 1067, 1041, 1018, 976, 1023, 995, 1037, 1088, 1011, 993, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1014, 1009, 1037, 1037, 0, 0, 0, 0, 1021, 1052, 1015, 1035, 22, 6, 0, 0, 992, 1027, 990, 1047, 12, 7, 3, 0, 968, 1017, 1048, 1050, 0, 0, 0, 0, 1022, 1050, 1009, 1049, 0, 0, 0, 0, 1061, 1019, 1005, 979, 0, 0, 0, 0, 1030, 1008, 1009, 1075, 8, 3, 0, 0, 995, 1023, 1015, 999, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4151, 4078, 4083, 4130, 4064, 4133, 4032] avg=4096.0000 max=4151 max_rank=1 min=4032 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1053, 1012, 990, 1012, 0, 0, 0, 0, 1029, 1042, 1029, 1005, 22, 6, 0, 0, 1000, 1037, 1011, 1034, 11, 7, 3, 0, 1016, 1015, 1022, 1019, 0, 0, 0, 0, 1020, 1039, 1042, 1015, 0, 0, 0, 0, 1016, 1052, 1041, 1005, 0, 0, 0, 0, 1008, 1006, 1038, 1036, 7, 3, 0, 0, 1039, 952, 1029, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4133, 4103, 4072, 4116, 4114, 4098, 4065] avg=4096.0000 max=4133 max_rank=1 min=4065 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1009, 1069, 1082, 1041, 0, 0, 0, 0, 1029, 1006, 1046, 999, 21, 7, 0, 0, 1033, 970, 1027, 996, 11, 7, 3, 0, 1028, 1004, 1010, 989, 0, 0, 0, 0, 989, 1004, 1012, 1063, 0, 0, 0, 0, 1013, 1006, 1032, 1058, 0, 0, 0, 0, 998, 1036, 974, 1042, 8, 3, 0, 0, 1054, 1028, 1038, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4201, 4108, 4047, 4031, 4068, 4109, 4061, 4143] avg=4096.0000 max=4201 max_rank=0 min=4031 min_rank=3 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1054, 1011, 1048, 997, 0, 0, 0, 0, 958, 1051, 987, 1020, 22, 6, 0, 0, 1027, 1024, 998, 1033, 12, 7, 3, 0, 991, 1027, 1050, 1061, 0, 0, 0, 0, 998, 1041, 1028, 1018, 0, 0, 0, 0, 1019, 1044, 997, 1054, 0, 0, 0, 0, 1026, 1027, 1042, 944, 8, 3, 0, 0, 1022, 1049, 1021, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4044, 4104, 4129, 4085, 4114, 4050, 4132] avg=4096.0000 max=4132 max_rank=7 min=4044 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1073, 1059, 998, 1045, 0, 0, 0, 0, 987, 1024, 1022, 1011, 21, 7, 0, 0, 996, 1040, 1007, 978, 11, 7, 3, 0, 1023, 1073, 1034, 1026, 0, 0, 0, 0, 1045, 1023, 1008, 1014, 0, 0, 0, 0, 994, 996, 1017, 1034, 0, 0, 0, 0, 1004, 1049, 1028, 994, 7, 3, 0, 0, 998, 1012, 1032, 1065, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4072, 4042, 4156, 4090, 4041, 4085, 4107] avg=4096.0000 max=4175 max_rank=0 min=4041 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1028, 972, 1034, 984, 0, 0, 0, 0, 999, 1016, 1009, 1050, 21, 6, 0, 0, 1049, 1040, 1056, 1000, 11, 6, 3, 0, 1008, 1041, 1038, 1027, 0, 0, 0, 0, 973, 995, 1053, 1016, 0, 0, 0, 0, 1009, 1027, 1046, 1045, 0, 0, 0, 0, 1073, 1051, 1026, 1016, 7, 2, 0, 0, 987, 1009, 1034, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4101, 4165, 4114, 4037, 4127, 4175, 4031] avg=4096.0000 max=4175 max_rank=6 min=4018 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[994, 1004, 1006, 1001, 0, 0, 0, 0, 975, 1009, 1052, 1029, 26, 11, 0, 0, 1013, 1002, 1062, 1037, 15, 9, 5, 0, 1007, 1075, 1020, 1020, 0, 0, 0, 0, 1085, 995, 962, 1021, 0, 0, 0, 0, 1006, 1016, 1011, 1038, 0, 0, 0, 0, 1015, 982, 1038, 1072, 12, 5, 0, 0, 1061, 1012, 1027, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 4102, 4143, 4122, 4063, 4071, 4124, 4138] avg=4096.0000 max=4143 max_rank=2 min=4005 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1048, 1033, 1033, 981, 0, 0, 0, 0, 1008, 1028, 997, 997, 21, 6, 0, 0, 1028, 965, 1087, 985, 11, 7, 3, 0, 998, 1029, 1031, 1003, 0, 0, 0, 0, 1034, 1058, 1031, 1056, 0, 0, 0, 0, 1002, 1067, 1041, 1018, 0, 0, 0, 0, 976, 1023, 995, 1037, 8, 3, 0, 0, 1085, 1011, 993, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4057, 4086, 4061, 4179, 4128, 4042, 4120] avg=4096.0000 max=4179 max_rank=4 min=4042 min_rank=6 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1051, 978, 1015, 1036, 1023, 1022, 1053, 984, 1034, 1037, 1019, 1043, 1033, 1013, 1038, 1019, 1000, 1064, 1030, 1024, 1001, 1050, 1070, 993, 1058, 1022, 1045, 1024, 972, 1000, 1011, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1017, 1051, 1013, 1022, 996, 1068, 1105, 1048, 1042, 999, 1029, 1024, 1046, 1008, 1043, 1038, 1030, 1000, 969, 1020, 988, 1016, 1009, 985, 1021, 1076, 981, 1047, 998, 1037, 1025, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[996, 1036, 1085, 1027, 985, 1061, 1023, 1026, 1025, 1016, 982, 1024, 979, 1034, 1014, 985, 1033, 1016, 1007, 1037, 1014, 1030, 1031, 998, 1001, 1000, 1048, 1028, 1056, 1059, 1103, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[983, 985, 1056, 1030, 1072, 1018, 998, 954, 1006, 1079, 1012, 1046, 1017, 1033, 1045, 986, 1058, 1019, 1010, 1038, 985, 1020, 1023, 1066, 1037, 1055, 1035, 1020, 997, 1060, 1043, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1018, 1005, 1000, 1009, 1024, 1029, 1015, 1024, 1053, 1065, 1014, 1040, 1030, 1050, 1043, 982, 1067, 1051, 1010, 1049, 1026, 1059, 998, 1017, 1050, 1006, 1027, 997, 1020, 1066, 964, 960] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1038, 1099, 1037, 1046, 967, 1009, 1051, 984, 1003, 1029, 1019, 1008, 1049, 1045, 1012, 1040, 1039, 1055, 1026, 1023, 1031, 998, 1052, 998, 1023, 1052, 999, 990, 1013, 1032, 997, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1012, 1049, 1046, 1021, 1008, 1056, 1027, 1033, 1007, 1066, 993, 1029, 1029, 975, 1074, 1060, 1010, 1006, 1009, 1042, 1019, 1005, 1060, 1049, 1002, 1019, 1032, 999, 1009, 1007, 986, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1045, 1031, 1007, 975, 1017, 1047, 1093, 1029, 991, 1046, 1042, 1015, 1029, 1063, 999, 1005, 1022, 1059, 1034, 1002, 1008, 989, 990, 1047, 1075, 966, 1016, 1022, 1058, 1046, 992, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1051, 978, 1009, 1036, 0, 0, 0, 0, 1023, 1022, 1043, 984, 0, 0, 0, 0, 1034, 1030, 1019, 1043, 0, 0, 0, 0, 1033, 1013, 1032, 1019, 0, 0, 0, 0, 1000, 1053, 1030, 1024, 0, 0, 0, 0, 1001, 1050, 1070, 993, 6, 6, 4, 0, 1058, 1022, 1045, 1024, 0, 0, 0, 0, 972, 1000, 1011, 1006, 11, 10, 3, 0] rerouted_tokens_per_ep_rank=[4074, 4072, 4126, 4097, 4107, 4130, 4149, 4013] avg=4096.0000 max=4149 max_rank=6 min=4013 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1017, 1051, 1007, 1022, 0, 0, 0, 0, 996, 1068, 1095, 1048, 0, 0, 0, 0, 1042, 992, 1029, 1024, 0, 0, 0, 0, 1046, 1008, 1037, 1038, 0, 0, 0, 0, 1030, 989, 969, 1020, 0, 0, 0, 0, 988, 1016, 1009, 985, 6, 6, 4, 0, 1021, 1076, 981, 1047, 0, 0, 0, 0, 998, 1037, 1025, 1017, 11, 10, 3, 0] rerouted_tokens_per_ep_rank=[4097, 4207, 4087, 4129, 4008, 4014, 4125, 4101] avg=4096.0000 max=4207 max_rank=1 min=4008 min_rank=4 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16248 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[996, 1036, 1079, 1027, 0, 0, 0, 0, 985, 1061, 1014, 1026, 0, 0, 0, 0, 1025, 1009, 982, 1024, 0, 0, 0, 0, 979, 1034, 1009, 985, 0, 0, 0, 0, 1033, 1005, 1007, 1037, 0, 0, 0, 0, 1014, 1030, 1031, 998, 6, 5, 4, 0, 1001, 1000, 1048, 1028, 0, 0, 0, 0, 1056, 1059, 1103, 1009, 11, 9, 3, 0] rerouted_tokens_per_ep_rank=[4138, 4086, 4040, 4007, 4082, 4088, 4077, 4250] avg=4096.0000 max=4250 max_rank=7 min=4007 min_rank=3 imbalance_ratio=1.037598 max_load_reduction=28518 max_load_reduction_pct=0.870300 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[983, 985, 1050, 1030, 0, 0, 0, 0, 1072, 1018, 989, 954, 0, 0, 0, 0, 1006, 1071, 1012, 1046, 0, 0, 0, 0, 1017, 1033, 1039, 986, 0, 0, 0, 0, 1058, 1008, 1010, 1038, 0, 0, 0, 0, 985, 1020, 1023, 1066, 6, 6, 5, 0, 1037, 1055, 1035, 1020, 0, 0, 0, 0, 997, 1060, 1043, 982, 11, 9, 3, 0] rerouted_tokens_per_ep_rank=[4048, 4033, 4135, 4075, 4114, 4111, 4147, 4105] avg=4096.0000 max=4147 max_rank=6 min=4033 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16477 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1038, 1099, 1031, 1046, 0, 0, 0, 0, 967, 1009, 1041, 984, 0, 0, 0, 0, 1003, 1022, 1019, 1008, 0, 0, 0, 0, 1049, 1045, 1007, 1040, 0, 0, 0, 0, 1039, 1044, 1026, 1023, 0, 0, 0, 0, 1031, 998, 1052, 998, 6, 5, 4, 0, 1023, 1052, 999, 990, 0, 0, 0, 0, 1013, 1032, 997, 1004, 11, 10, 3, 0] rerouted_tokens_per_ep_rank=[4214, 4001, 4052, 4141, 4132, 4094, 4064, 4070] avg=4096.0000 max=4214 max_rank=0 min=4001 min_rank=1 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1012, 1049, 1040, 1021, 0, 0, 0, 0, 1008, 1056, 1018, 1033, 0, 0, 0, 0, 1007, 1059, 993, 1029, 0, 0, 0, 0, 1029, 975, 1068, 1060, 0, 0, 0, 0, 1010, 995, 1009, 1042, 0, 0, 0, 0, 1019, 1005, 1060, 1049, 6, 6, 4, 0, 1002, 1019, 1032, 999, 0, 0, 0, 0, 1009, 1007, 986, 1029, 11, 9, 3, 0] rerouted_tokens_per_ep_rank=[4122, 4115, 4088, 4132, 4056, 4149, 4052, 4054] avg=4096.0000 max=4149 max_rank=5 min=4052 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1018, 1005, 991, 1009, 0, 0, 0, 0, 1024, 1029, 1001, 1024, 0, 0, 0, 0, 1053, 1046, 1014, 1040, 0, 0, 0, 0, 1030, 1050, 1034, 982, 0, 0, 0, 0, 1067, 1037, 1010, 1049, 0, 0, 0, 0, 1026, 1059, 998, 1017, 9, 9, 10, 5, 1045, 1006, 1027, 997, 0, 0, 0, 0, 1020, 1066, 964, 960, 14, 14, 9, 0] rerouted_tokens_per_ep_rank=[4023, 4078, 4153, 4096, 4163, 4133, 4075, 4047] avg=4096.0000 max=4163 max_rank=4 min=4023 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1045, 1031, 1001, 975, 0, 0, 0, 0, 1017, 1047, 1083, 1029, 0, 0, 0, 0, 991, 1039, 1042, 1015, 0, 0, 0, 0, 1029, 1063, 994, 1005, 0, 0, 0, 0, 1022, 1048, 1034, 1002, 0, 0, 0, 0, 1008, 989, 990, 1047, 6, 5, 4, 0, 1075, 966, 1016, 1022, 0, 0, 0, 0, 1058, 1046, 992, 1008, 11, 10, 3, 0] rerouted_tokens_per_ep_rank=[4052, 4176, 4087, 4091, 4106, 4049, 4079, 4128] avg=4096.0000 max=4176 max_rank=1 min=4049 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1013, 1038, 1051, 1000, 1062, 1083, 1039, 1040, 1024, 1006, 1048, 1063, 980, 988, 1005, 994, 1020, 1007, 998, 961, 993, 1017, 1062, 1074, 1042, 1006, 1062, 1023, 1037, 1005, 1023, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1046, 1042, 1046, 1041, 1035, 1032, 961, 1030, 1074, 1045, 1035, 1054, 1019, 1042, 1004, 1047, 988, 1008, 1043, 1014, 1014, 1023, 1028, 1047, 965, 998, 1008, 1001, 1028, 1035, 1025, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1025, 1103, 1034, 1047, 1013, 973, 1078, 977, 1024, 1011, 1023, 1033, 1021, 1040, 1022, 997, 969, 1060, 981, 1007, 998, 1008, 1077, 1017, 1062, 1060, 1026, 1033, 1036, 1016, 977, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1001, 965, 1016, 1017, 1022, 1037, 1031, 1034, 1014, 1049, 1026, 1035, 1039, 1038, 1018, 996, 1013, 1022, 1034, 983, 1061, 1013, 1065, 1032, 1038, 1045, 1036, 988, 983, 1072, 1030, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1027, 1035, 1006, 1003, 994, 1025, 1017, 1082, 1068, 1014, 996, 1035, 1021, 1078, 970, 1022, 1035, 1023, 1060, 1013, 974, 1040, 1039, 1010, 1010, 1005, 1046, 1005, 1018, 1021, 1035, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[985, 1034, 1038, 1070, 1040, 1038, 1024, 1040, 1023, 1049, 1069, 1038, 996, 998, 1014, 1026, 1006, 997, 1052, 1057, 1045, 977, 994, 987, 1039, 1044, 960, 1038, 980, 1001, 1081, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1040, 993, 1008, 990, 1028, 1016, 1003, 1041, 978, 1025, 1005, 1022, 991, 1092, 1011, 1031, 1038, 1025, 995, 1013, 1058, 1089, 1025, 1066, 983, 1055, 1029, 1000, 1029, 1032, 1015, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[971, 991, 1062, 1044, 1017, 1007, 1029, 1051, 987, 1034, 1007, 1021, 1025, 1031, 959, 1065, 1030, 1000, 1075, 984, 1024, 1029, 1052, 1070, 1042, 1027, 963, 1017, 1024, 1069, 1066, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1046, 1042, 1045, 1041, 0, 0, 0, 0, 1035, 1032, 961, 1015, 0, 0, 0, 0, 1074, 1045, 1035, 1033, 0, 0, 0, 0, 1019, 1042, 1004, 1047, 19, 4, 0, 0, 988, 1008, 1043, 1014, 29, 2, 0, 0, 1014, 1023, 999, 1047, 0, 0, 0, 0, 965, 998, 1008, 1001, 11, 1, 0, 0, 1028, 1035, 1025, 990, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4043, 4187, 4135, 4084, 4083, 3984, 4078] avg=4096.0000 max=4187 max_rank=2 min=3984 min_rank=6 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16229 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1013, 1038, 1050, 1000, 0, 0, 0, 0, 1062, 1083, 1039, 1025, 0, 0, 0, 0, 1024, 1006, 1048, 1042, 0, 0, 0, 0, 980, 988, 1005, 994, 19, 4, 0, 0, 1020, 1007, 998, 961, 30, 2, 0, 0, 993, 1017, 1032, 1074, 0, 0, 0, 0, 1042, 1006, 1062, 1023, 11, 1, 0, 0, 1037, 1005, 1023, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4209, 4120, 3990, 4018, 4116, 4145, 4069] avg=4096.0000 max=4209 max_rank=1 min=3990 min_rank=3 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1001, 965, 1015, 1017, 0, 0, 0, 0, 1022, 1037, 1031, 1019, 0, 0, 0, 0, 1014, 1049, 1026, 1015, 0, 0, 0, 0, 1039, 1038, 1018, 996, 18, 4, 0, 0, 1013, 1022, 1034, 983, 30, 2, 0, 0, 1061, 1013, 1035, 1032, 0, 0, 0, 0, 1038, 1045, 1036, 988, 11, 1, 0, 0, 983, 1072, 1030, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 4109, 4104, 4113, 4084, 4141, 4119, 4100] avg=4096.0000 max=4141 max_rank=5 min=3998 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1025, 1103, 1033, 1047, 0, 0, 0, 0, 1013, 973, 1078, 963, 0, 0, 0, 0, 1024, 1011, 1023, 1013, 0, 0, 0, 0, 1021, 1040, 1022, 997, 18, 4, 0, 0, 969, 1060, 981, 1007, 30, 2, 0, 0, 998, 1008, 1047, 1017, 0, 0, 0, 0, 1062, 1060, 1026, 1033, 10, 1, 0, 0, 1036, 1016, 977, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4208, 4027, 4071, 4102, 4049, 4070, 4192, 4049] avg=4096.0000 max=4208 max_rank=0 min=4027 min_rank=1 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1040, 993, 1007, 990, 0, 0, 0, 0, 1028, 1016, 1003, 1026, 0, 0, 0, 0, 978, 1025, 1005, 1002, 0, 0, 0, 0, 991, 1092, 1011, 1031, 18, 4, 0, 0, 1038, 1025, 995, 1013, 29, 2, 0, 0, 1058, 1089, 996, 1066, 0, 0, 0, 0, 983, 1055, 1029, 1000, 11, 1, 0, 0, 1029, 1032, 1015, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4073, 4010, 4147, 4102, 4209, 4079, 4118] avg=4096.0000 max=4209 max_rank=5 min=4010 min_rank=2 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[985, 1034, 1037, 1070, 0, 0, 0, 0, 1040, 1038, 1024, 1025, 0, 0, 0, 0, 1023, 1049, 1069, 1018, 0, 0, 0, 0, 996, 998, 1014, 1026, 18, 4, 0, 0, 1006, 997, 1052, 1057, 28, 2, 0, 0, 1045, 977, 966, 987, 0, 0, 0, 0, 1039, 1044, 960, 1038, 11, 1, 0, 0, 980, 1001, 1081, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4127, 4159, 4056, 4142, 3975, 4093, 4090] avg=4096.0000 max=4159 max_rank=2 min=3975 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1027, 1035, 999, 1003, 0, 0, 0, 0, 994, 1025, 1017, 1055, 0, 0, 0, 0, 1068, 1014, 996, 1010, 0, 0, 0, 0, 1021, 1078, 970, 1022, 22, 10, 0, 0, 1035, 1023, 1060, 1013, 34, 3, 0, 0, 974, 1040, 1005, 1010, 0, 0, 0, 0, 1010, 1005, 1046, 1005, 17, 7, 5, 0, 1018, 1021, 1030, 1041, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4091, 4088, 4123, 4168, 4029, 4095, 4110] avg=4096.0000 max=4168 max_rank=4 min=4029 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[971, 991, 1061, 1044, 0, 0, 0, 0, 1017, 1007, 1029, 1036, 0, 0, 0, 0, 987, 1034, 1007, 1001, 0, 0, 0, 0, 1025, 1031, 959, 1065, 18, 4, 0, 0, 1030, 1000, 1075, 984, 30, 2, 0, 0, 1024, 1029, 1022, 1070, 0, 0, 0, 0, 1042, 1027, 963, 1017, 11, 1, 0, 0, 1024, 1069, 1066, 995, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4089, 4029, 4102, 4121, 4145, 4061, 4154] avg=4096.0000 max=4154 max_rank=7 min=4029 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1025, 997, 1022, 1077, 1017, 1065, 1019, 1071, 1004, 1021, 992, 1030, 1028, 1040, 1028, 1041, 1036, 1041, 1016, 1023, 1034, 1041, 1009, 1055, 1040, 1010, 1031, 978, 964, 1026, 976, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1060, 974, 1035, 1043, 986, 1006, 1031, 1027, 1012, 1058, 1016, 1004, 1075, 1004, 1022, 1078, 1024, 1044, 994, 1019, 1053, 1010, 1081, 1035, 987, 995, 1006, 1012, 996, 1044, 998, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1009, 1021, 1005, 1019, 1086, 1015, 1006, 1055, 995, 1002, 1032, 1009, 1085, 1039, 1016, 1000, 989, 1041, 973, 1031, 1048, 990, 1061, 1031, 1057, 1020, 1054, 1048, 987, 1013, 1028, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[995, 1008, 1039, 1047, 1068, 978, 1001, 1060, 1038, 986, 1016, 1055, 1068, 1029, 1018, 1051, 1025, 996, 1002, 1024, 1016, 1012, 992, 1030, 1034, 1015, 1057, 1002, 1050, 1049, 1013, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1056, 990, 1042, 984, 1040, 928, 1047, 1014, 1001, 1063, 987, 1042, 1022, 1015, 1043, 1012, 1032, 1070, 1047, 1072, 982, 1070, 1021, 1020, 1064, 1047, 964, 1030, 1074, 1027, 997, 965] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[997, 1070, 1016, 999, 974, 1028, 1033, 1034, 1057, 980, 971, 1002, 1016, 1017, 1029, 1050, 1046, 992, 1038, 1006, 1064, 971, 987, 1058, 1041, 1078, 989, 1003, 1023, 1051, 1080, 1068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[989, 1043, 1018, 1074, 1034, 988, 1031, 1050, 1017, 1034, 1056, 1027, 1029, 1038, 971, 1023, 991, 1017, 977, 1012, 1004, 1055, 1025, 1026, 1045, 981, 1044, 1049, 982, 1085, 1012, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[994, 1018, 1030, 1021, 1043, 1018, 1050, 1049, 1089, 1014, 1016, 1015, 1038, 1009, 1057, 1042, 1029, 1052, 989, 1044, 1021, 1014, 1020, 1023, 1046, 1020, 1033, 1046, 969, 968, 960, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 997, 1022, 1077, 6, 0, 0, 0, 1017, 1065, 1019, 1062, 0, 0, 0, 0, 1004, 1021, 992, 1030, 11, 2, 1, 0, 996, 1040, 1028, 1041, 0, 0, 0, 0, 1036, 1041, 1016, 1023, 8, 0, 0, 0, 1034, 1041, 1009, 1044, 0, 0, 0, 0, 1034, 1010, 1031, 978, 0, 0, 0, 0, 964, 1026, 976, 1011, 30, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4163, 4061, 4105, 4124, 4128, 4053, 4007] avg=4096.0000 max=4163 max_rank=1 min=4007 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1009, 1021, 1005, 1019, 6, 0, 0, 0, 1086, 1015, 1006, 1046, 0, 0, 0, 0, 995, 1002, 1032, 1009, 11, 2, 1, 0, 1052, 1039, 1016, 1000, 0, 0, 0, 0, 989, 1041, 973, 1031, 8, 0, 0, 0, 1048, 990, 1061, 1020, 0, 0, 0, 0, 1051, 1020, 1054, 1048, 0, 0, 0, 0, 987, 1013, 1028, 1003, 31, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4153, 4052, 4107, 4042, 4119, 4173, 4062] avg=4096.0000 max=4173 max_rank=6 min=4042 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1056, 990, 1042, 984, 6, 0, 0, 0, 1040, 928, 1047, 1005, 0, 0, 0, 0, 1001, 1063, 987, 1042, 11, 2, 1, 0, 991, 1015, 1043, 1012, 0, 0, 0, 0, 1032, 1070, 1047, 1072, 8, 0, 0, 0, 982, 1070, 1021, 1009, 0, 0, 0, 0, 1058, 1047, 964, 1030, 0, 0, 0, 0, 1074, 1027, 997, 965, 29, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4020, 4107, 4061, 4229, 4082, 4099, 4092] avg=4096.0000 max=4229 max_rank=4 min=4020 min_rank=1 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1060, 974, 1035, 1043, 9, 0, 0, 0, 986, 1006, 1031, 1006, 0, 0, 0, 0, 1012, 1058, 1016, 1004, 14, 7, 8, 0, 1034, 1004, 1022, 1078, 0, 0, 0, 0, 1024, 1044, 994, 1019, 13, 7, 0, 0, 1053, 1010, 1081, 1021, 0, 0, 0, 0, 971, 995, 1006, 1012, 0, 0, 0, 0, 996, 1044, 998, 1039, 34, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4029, 4119, 4138, 4101, 4165, 3984, 4111] avg=4096.0000 max=4165 max_rank=5 min=3984 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[995, 1008, 1039, 1047, 6, 0, 0, 0, 1068, 978, 1001, 1051, 0, 0, 0, 0, 1038, 986, 1016, 1055, 11, 2, 1, 0, 1035, 1029, 1018, 1051, 0, 0, 0, 0, 1025, 996, 1002, 1024, 8, 0, 0, 0, 1016, 1012, 992, 1019, 0, 0, 0, 0, 1028, 1015, 1057, 1002, 0, 0, 0, 0, 1050, 1049, 1013, 994, 31, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4098, 4109, 4133, 4055, 4039, 4102, 4137] avg=4096.0000 max=4137 max_rank=7 min=4039 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[997, 1070, 1016, 999, 6, 0, 0, 0, 974, 1028, 1033, 1025, 0, 0, 0, 0, 1057, 980, 971, 1002, 11, 2, 1, 0, 985, 1017, 1029, 1050, 0, 0, 0, 0, 1046, 992, 1038, 1006, 8, 0, 0, 0, 1064, 971, 987, 1047, 0, 0, 0, 0, 1035, 1078, 989, 1003, 0, 0, 0, 0, 1023, 1051, 1080, 1068, 29, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4060, 4024, 4081, 4090, 4069, 4105, 4251] avg=4096.0000 max=4251 max_rank=7 min=4024 min_rank=2 imbalance_ratio=1.037842 max_load_reduction=28517 max_load_reduction_pct=0.870270 moved_to_echo=16515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[994, 1018, 1030, 1021, 6, 0, 0, 0, 1043, 1018, 1050, 1040, 0, 0, 0, 0, 1089, 1014, 1016, 1015, 11, 2, 1, 0, 1006, 1009, 1057, 1042, 0, 0, 0, 0, 1029, 1052, 989, 1044, 8, 0, 0, 0, 1021, 1014, 1020, 1012, 0, 0, 0, 0, 1040, 1020, 1033, 1046, 0, 0, 0, 0, 969, 968, 960, 1031, 30, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4151, 4148, 4114, 4122, 4067, 4139, 3958] avg=4096.0000 max=4151 max_rank=1 min=3958 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[989, 1043, 1018, 1074, 6, 0, 0, 0, 1034, 988, 1031, 1041, 0, 0, 0, 0, 1017, 1034, 1056, 1027, 11, 2, 1, 0, 997, 1038, 971, 1023, 0, 0, 0, 0, 991, 1017, 977, 1012, 8, 0, 0, 0, 1004, 1055, 1025, 1015, 0, 0, 0, 0, 1039, 981, 1044, 1049, 0, 0, 0, 0, 982, 1085, 1012, 1041, 30, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4094, 4148, 4029, 4005, 4099, 4113, 4150] avg=4096.0000 max=4150 max_rank=7 min=4005 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8064, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1017, 1029, 1006, 1038, 1056, 1099, 1049, 1052, 1039, 984, 981, 1001, 1039, 993, 995, 1009, 995, 1058, 1044, 991, 1051, 978, 1015, 979, 1053, 982, 1036, 1045, 1040, 1037, 1029, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1008, 1073, 985, 1035, 996, 993, 993, 1023, 1062, 1012, 1040, 1031, 967, 991, 1033, 1036, 1047, 1014, 1015, 1015, 1025, 960, 1096, 1066, 1044, 1057, 1040, 1020, 1041, 992, 1031, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[977, 1042, 1003, 1030, 1060, 1035, 1019, 1062, 992, 968, 1050, 1022, 1016, 989, 1039, 1004, 1002, 991, 1057, 1009, 1036, 1042, 1038, 1069, 1048, 984, 1085, 1022, 1010, 997, 1061, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[995, 1050, 1070, 991, 1033, 1061, 1039, 1023, 998, 1057, 1068, 1015, 1055, 1004, 1027, 1066, 1003, 969, 977, 1011, 1024, 1084, 1035, 988, 1022, 983, 1050, 1006, 1017, 987, 1017, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1004, 1068, 997, 1039, 975, 1057, 1035, 1021, 1037, 1022, 1029, 1055, 1022, 1057, 1039, 1038, 1051, 1017, 1061, 1019, 1018, 998, 1048, 1011, 1018, 1026, 995, 1011, 971, 981, 1054, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1055, 1030, 1018, 1042, 1064, 1018, 1078, 968, 976, 1037, 987, 1021, 1005, 1056, 1082, 1000, 1013, 1021, 1003, 1003, 1041, 1043, 1034, 1028, 1032, 1027, 992, 995, 1043, 980, 1069, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[978, 1023, 1027, 1043, 1024, 1032, 1032, 1012, 1004, 1055, 1025, 1005, 1019, 1021, 989, 1003, 1048, 1008, 1072, 1063, 1060, 1058, 1027, 1047, 1054, 1009, 1001, 1000, 1007, 1015, 1001, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1003, 1055, 1023, 1014, 1089, 1021, 1059, 1008, 1058, 982, 1017, 1043, 1025, 1037, 1020, 1053, 995, 1012, 1004, 1033, 1003, 1053, 1021, 972, 1025, 1001, 1047, 1005, 1003, 1101, 985, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1017, 1029, 1006, 1038, 0, 0, 0, 0, 1056, 1059, 1049, 1052, 0, 0, 0, 0, 1039, 984, 981, 1001, 10, 0, 0, 0, 1039, 993, 995, 1009, 4, 0, 0, 0, 995, 1058, 1044, 991, 19, 0, 0, 0, 1051, 978, 995, 979, 0, 0, 0, 0, 1053, 982, 1036, 1045, 6, 0, 0, 0, 1040, 1037, 1029, 1048, 21, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4216, 4015, 4040, 4107, 4003, 4122, 4175] avg=4096.0000 max=4216 max_rank=1 min=4003 min_rank=5 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16407 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[977, 1042, 1003, 1030, 0, 0, 0, 0, 1060, 997, 1019, 1062, 0, 0, 0, 0, 992, 968, 1050, 1022, 10, 0, 0, 0, 1016, 989, 1039, 1004, 4, 0, 0, 0, 1002, 991, 1057, 1009, 18, 0, 0, 0, 1036, 1042, 1018, 1069, 0, 0, 0, 0, 1048, 984, 1085, 1022, 6, 0, 0, 0, 1010, 997, 1061, 1009, 20, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4138, 4042, 4052, 4077, 4165, 4145, 4097] avg=4096.0000 max=4165 max_rank=5 min=4042 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16484 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1008, 1073, 985, 1035, 0, 0, 0, 0, 996, 957, 993, 1023, 0, 0, 0, 0, 1062, 1012, 1040, 1031, 11, 0, 0, 0, 967, 991, 1033, 1036, 5, 0, 0, 0, 1047, 1014, 1015, 1015, 17, 0, 0, 0, 1025, 960, 1074, 1066, 0, 0, 0, 0, 1044, 1057, 1040, 1020, 6, 0, 0, 0, 1041, 992, 1031, 1027, 19, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 3969, 4156, 4032, 4108, 4125, 4167, 4110] avg=4096.0000 max=4167 max_rank=6 min=3969 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16510 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[995, 1050, 1070, 991, 0, 0, 0, 0, 1033, 1023, 1039, 1023, 0, 0, 0, 0, 998, 1057, 1068, 1015, 10, 0, 0, 0, 1055, 1004, 1027, 1066, 4, 0, 0, 0, 1003, 969, 977, 1011, 18, 0, 0, 0, 1024, 1084, 1015, 988, 0, 0, 0, 0, 1022, 983, 1050, 1006, 6, 0, 0, 0, 1017, 987, 1017, 1043, 20, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4118, 4148, 4156, 3978, 4111, 4067, 4084] avg=4096.0000 max=4156 max_rank=3 min=3978 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16240 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1004, 1068, 997, 1039, 0, 0, 0, 0, 975, 1019, 1035, 1021, 0, 0, 0, 0, 1037, 1022, 1029, 1055, 11, 0, 0, 0, 1022, 1057, 1039, 1038, 4, 0, 0, 0, 1051, 1017, 1061, 1019, 18, 0, 0, 0, 1018, 998, 1027, 1011, 0, 0, 0, 0, 1018, 1026, 995, 1011, 6, 0, 0, 0, 971, 981, 1054, 994, 20, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4050, 4154, 4160, 4166, 4054, 4056, 4020] avg=4096.0000 max=4166 max_rank=4 min=4020 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16296 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1055, 1030, 1018, 1042, 0, 0, 0, 0, 1064, 981, 1078, 968, 0, 0, 0, 0, 976, 1037, 987, 1021, 10, 0, 0, 0, 1005, 1056, 1082, 1000, 4, 0, 0, 0, 1013, 1021, 1003, 1003, 17, 0, 0, 0, 1041, 1043, 1014, 1028, 0, 0, 0, 0, 1032, 1027, 992, 995, 6, 0, 0, 0, 1043, 980, 1069, 1007, 20, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4091, 4031, 4147, 4057, 4126, 4052, 4119] avg=4096.0000 max=4147 max_rank=3 min=4031 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16354 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[978, 1023, 1027, 1043, 0, 0, 0, 0, 1024, 979, 1032, 1012, 0, 0, 0, 0, 1004, 1055, 1025, 1005, 16, 7, 0, 0, 1019, 1021, 989, 1003, 10, 0, 0, 0, 1048, 1008, 1072, 1063, 22, 0, 0, 0, 1060, 1058, 990, 1047, 0, 0, 0, 0, 1054, 1009, 1001, 1000, 11, 0, 0, 0, 1007, 1015, 1001, 1006, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4047, 4112, 4042, 4213, 4155, 4075, 4053] avg=4096.0000 max=4213 max_rank=4 min=4042 min_rank=3 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16496 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1003, 1055, 1023, 1014, 0, 0, 0, 0, 1089, 983, 1059, 1008, 0, 0, 0, 0, 1058, 982, 1017, 1043, 10, 0, 0, 0, 1025, 1037, 1020, 1053, 4, 0, 0, 0, 995, 1012, 1004, 1033, 18, 0, 0, 0, 1003, 1053, 1001, 972, 0, 0, 0, 0, 1025, 1001, 1047, 1005, 6, 0, 0, 0, 1003, 1101, 985, 1001, 20, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4139, 4110, 4139, 4062, 4029, 4084, 4110] avg=4096.0000 max=4139 max_rank=1 min=4029 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16285 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8448, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8064, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1004, 1089, 1001, 1040, 1018, 1017, 1000, 1027, 1038, 1061, 1022, 1010, 958, 1057, 1030, 1024, 1000, 1009, 999, 1032, 1073, 1035, 1028, 997, 1009, 1053, 1048, 1023, 1054, 1009, 1015, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1023, 1023, 1007, 1069, 998, 1042, 1020, 1013, 1045, 985, 1017, 1067, 1014, 1059, 1008, 1028, 1061, 1002, 1030, 990, 991, 1059, 1018, 1006, 1057, 1101, 992, 1034, 980, 1015, 991, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1053, 1066, 1039, 1040, 1002, 995, 1030, 1060, 1023, 998, 1057, 962, 959, 1039, 1046, 1054, 977, 1049, 992, 1022, 1019, 998, 999, 1023, 1024, 1058, 1015, 990, 1051, 1034, 1040, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1064, 1043, 1017, 1007, 1026, 1056, 1001, 1003, 1038, 1023, 1084, 1006, 1037, 1019, 1039, 1011, 1087, 1004, 1051, 1008, 1036, 1006, 1004, 1041, 1017, 1000, 1057, 993, 996, 991, 983, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1081, 1058, 1036, 995, 1036, 982, 989, 1001, 1004, 1033, 997, 1077, 1060, 1000, 1056, 1023, 1030, 1011, 1028, 1010, 1062, 1025, 1023, 975, 1026, 970, 1007, 1101, 1023, 1025, 1021, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1017, 986, 1016, 1001, 1046, 1033, 1015, 1019, 995, 1042, 1010, 1066, 1042, 1059, 1081, 1060, 1036, 990, 1042, 975, 1001, 1015, 1010, 1032, 1045, 1041, 996, 1032, 989, 1034, 1004, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1027, 1034, 1037, 1057, 1026, 974, 1066, 996, 1031, 999, 1020, 1013, 1007, 1039, 1023, 1027, 1044, 1096, 1017, 1021, 1041, 1023, 1002, 1022, 1030, 991, 962, 1087, 1006, 980, 1052, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1041, 1061, 1058, 1033, 961, 999, 1022, 1020, 1081, 1068, 1004, 1048, 981, 1011, 1049, 1041, 1048, 985, 1037, 1016, 1045, 986, 1007, 1032, 1026, 1046, 1040, 993, 989, 994, 1029, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1004, 1044, 1001, 1040, 0, 0, 0, 0, 1018, 1017, 1000, 1027, 21, 6, 6, 0, 1019, 1061, 1022, 1010, 0, 0, 0, 0, 958, 1057, 1009, 1024, 0, 0, 0, 0, 1000, 1009, 999, 1032, 8, 0, 0, 0, 1073, 1035, 1028, 997, 13, 3, 0, 0, 1009, 1042, 1048, 1023, 0, 0, 0, 0, 1054, 1009, 1015, 988, 39, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4095, 4112, 4048, 4048, 4149, 4122, 4105] avg=4096.0000 max=4149 max_rank=5 min=4048 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1053, 1022, 1039, 1040, 0, 0, 0, 0, 1002, 995, 1030, 1060, 21, 6, 6, 0, 1004, 998, 1057, 962, 0, 0, 0, 0, 959, 1039, 1025, 1054, 0, 0, 0, 0, 977, 1049, 992, 1022, 8, 0, 0, 0, 1019, 998, 999, 1023, 13, 3, 0, 0, 1024, 1047, 1015, 990, 0, 0, 0, 0, 1051, 1034, 1040, 1054, 38, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4120, 4021, 4077, 4048, 4055, 4076, 4217] avg=4096.0000 max=4217 max_rank=7 min=4021 min_rank=2 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1023, 981, 1007, 1069, 0, 0, 0, 0, 998, 1042, 1020, 1013, 20, 6, 6, 0, 1026, 985, 1017, 1067, 0, 0, 0, 0, 1014, 1059, 988, 1028, 0, 0, 0, 0, 1061, 1002, 1030, 990, 9, 0, 0, 0, 991, 1059, 1018, 1006, 13, 3, 0, 0, 1057, 1089, 992, 1034, 0, 0, 0, 0, 980, 1015, 991, 1023, 36, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4105, 4095, 4089, 4092, 4090, 4172, 4045] avg=4096.0000 max=4172 max_rank=6 min=4045 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1081, 1014, 1036, 995, 0, 0, 0, 0, 1036, 982, 989, 1001, 21, 6, 5, 0, 986, 1033, 997, 1077, 0, 0, 0, 0, 1060, 1000, 1035, 1023, 0, 0, 0, 0, 1030, 1011, 1028, 1010, 8, 0, 0, 0, 1062, 1025, 1023, 975, 13, 3, 0, 0, 1026, 959, 1007, 1101, 0, 0, 0, 0, 1023, 1025, 1021, 1003, 38, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4040, 4093, 4118, 4087, 4101, 4093, 4110] avg=4096.0000 max=4126 max_rank=0 min=4040 min_rank=1 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1017, 945, 1016, 1001, 0, 0, 0, 0, 1046, 1033, 1015, 1019, 22, 6, 5, 0, 978, 1042, 1010, 1066, 0, 0, 0, 0, 1042, 1059, 1059, 1060, 0, 0, 0, 0, 1036, 990, 1042, 975, 8, 0, 0, 0, 1001, 1015, 1010, 1032, 12, 3, 0, 0, 1045, 1030, 996, 1032, 0, 0, 0, 0, 989, 1034, 1004, 1038, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[3979, 4146, 4096, 4220, 4051, 4073, 4103, 4100] avg=4096.0000 max=4220 max_rank=3 min=3979 min_rank=0 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1064, 991, 1017, 1007, 0, 0, 0, 0, 1026, 1056, 1001, 1003, 26, 11, 9, 0, 1013, 1023, 1084, 1006, 0, 0, 0, 0, 1037, 1019, 1013, 1011, 0, 0, 0, 0, 1087, 1004, 1051, 1008, 12, 0, 0, 0, 1036, 1006, 1004, 1041, 16, 6, 0, 0, 1017, 982, 1057, 993, 0, 0, 0, 0, 996, 991, 983, 1020, 41, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4132, 4126, 4080, 4162, 4109, 4049, 4031] avg=4096.0000 max=4162 max_rank=4 min=4031 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1041, 1017, 1058, 1033, 0, 0, 0, 0, 961, 999, 1022, 1020, 21, 6, 6, 0, 1061, 1068, 1004, 1048, 0, 0, 0, 0, 981, 1011, 1028, 1041, 0, 0, 0, 0, 1048, 985, 1037, 1016, 8, 0, 0, 0, 1045, 986, 1007, 1032, 14, 3, 0, 0, 1026, 1035, 1040, 993, 0, 0, 0, 0, 989, 994, 1029, 1017, 38, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4035, 4181, 4061, 4094, 4087, 4094, 4067] avg=4096.0000 max=4181 max_rank=2 min=4035 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1027, 991, 1037, 1057, 0, 0, 0, 0, 1026, 974, 1066, 996, 21, 6, 6, 0, 1012, 999, 1020, 1013, 0, 0, 0, 0, 1007, 1039, 1002, 1027, 0, 0, 0, 0, 1044, 1096, 1017, 1021, 8, 0, 0, 0, 1041, 1023, 1002, 1022, 13, 3, 0, 0, 1030, 980, 962, 1087, 0, 0, 0, 0, 1006, 980, 1052, 1018, 37, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4095, 4044, 4075, 4186, 4104, 4059, 4093] avg=4096.0000 max=4186 max_rank=4 min=4044 min_rank=2 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1017, 994, 971, 1049, 1003, 985, 1002, 1022, 1057, 1056, 1056, 1045, 990, 1045, 991, 1096, 1010, 1045, 1025, 992, 1016, 1023, 1046, 1056, 1011, 1018, 1024, 1100, 997, 1013, 999, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[998, 1052, 1021, 1009, 1039, 1044, 1061, 977, 1004, 1045, 985, 981, 1031, 1020, 1016, 1035, 1038, 1051, 1046, 1054, 1020, 984, 1012, 1022, 1075, 994, 1089, 1020, 1009, 1007, 1016, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1025, 998, 978, 1009, 1018, 991, 1021, 1072, 1047, 1007, 976, 1030, 1048, 994, 1002, 1065, 1036, 1047, 1029, 1034, 963, 1053, 1046, 1003, 1047, 1019, 1030, 1050, 1021, 1009, 1072, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1011, 1030, 1029, 1059, 1054, 1075, 1013, 993, 1008, 1035, 1036, 1042, 1016, 993, 1078, 1026, 1006, 1043, 987, 1019, 1095, 1025, 997, 994, 1039, 1011, 964, 1023, 985, 1041, 1021, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[984, 1059, 1040, 1023, 1027, 1042, 1055, 1021, 1044, 1011, 992, 1026, 1042, 1045, 1021, 1046, 1002, 1013, 1004, 1035, 1038, 1032, 1028, 957, 1047, 1065, 1030, 1007, 995, 1003, 1019, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[968, 1037, 1028, 994, 1002, 1040, 1036, 1098, 1017, 992, 1055, 993, 1012, 1006, 1041, 1064, 1003, 1054, 1075, 982, 1037, 1010, 1027, 1016, 1012, 1045, 1061, 1029, 1016, 1005, 1020, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1035, 1013, 1048, 1000, 1074, 1028, 1009, 998, 1015, 1036, 1046, 999, 1060, 1032, 978, 1015, 1047, 1027, 1034, 1026, 1040, 1005, 1038, 998, 1014, 1008, 1032, 1033, 1023, 999, 1044, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1035, 1059, 1060, 1013, 1020, 1023, 1005, 1024, 1051, 1063, 1014, 1028, 999, 1025, 1019, 1004, 1036, 961, 996, 1042, 1024, 1012, 1076, 1039, 989, 1026, 1032, 1020, 1004, 982, 1024, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[998, 1052, 1021, 1009, 10, 2, 2, 0, 1026, 1044, 1061, 977, 0, 0, 0, 0, 1004, 1042, 985, 981, 0, 0, 0, 0, 1031, 1020, 1016, 1025, 0, 0, 0, 0, 1038, 1048, 1046, 1054, 0, 0, 0, 0, 1020, 984, 1012, 1022, 3, 1, 0, 0, 1075, 994, 1089, 996, 0, 0, 0, 0, 1009, 1007, 1016, 1013, 24, 11, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4108, 4012, 4092, 4186, 4042, 4154, 4080] avg=4096.0000 max=4186 max_rank=4 min=4012 min_rank=2 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1017, 994, 971, 1049, 11, 2, 1, 0, 992, 985, 1002, 1022, 0, 0, 0, 0, 1057, 1053, 1056, 1045, 0, 0, 0, 0, 990, 1045, 991, 1085, 0, 0, 0, 0, 1010, 1042, 1025, 992, 0, 0, 0, 0, 1016, 1023, 1046, 1056, 3, 1, 0, 0, 1011, 1018, 1024, 1074, 0, 0, 0, 0, 997, 1013, 999, 1014, 26, 10, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4001, 4211, 4111, 4069, 4145, 4127, 4059] avg=4096.0000 max=4211 max_rank=2 min=4001 min_rank=1 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1025, 998, 978, 1009, 11, 2, 1, 0, 1007, 991, 1021, 1072, 0, 0, 0, 0, 1047, 1005, 976, 1030, 0, 0, 0, 0, 1048, 994, 1002, 1054, 0, 0, 0, 0, 1036, 1044, 1029, 1034, 0, 0, 0, 0, 963, 1053, 1046, 1003, 2, 1, 0, 0, 1047, 1019, 1030, 1026, 0, 0, 0, 0, 1021, 1009, 1072, 1028, 24, 10, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4091, 4058, 4098, 4143, 4068, 4122, 4164] avg=4096.0000 max=4164 max_rank=7 min=4024 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1035, 1013, 1048, 1000, 10, 2, 2, 0, 1061, 1028, 1009, 998, 0, 0, 0, 0, 1015, 1033, 1046, 999, 0, 0, 0, 0, 1060, 1032, 978, 1005, 0, 0, 0, 0, 1047, 1024, 1034, 1026, 0, 0, 0, 0, 1040, 1005, 1038, 998, 3, 1, 0, 0, 1014, 1008, 1032, 1009, 0, 0, 0, 0, 1023, 999, 1044, 1014, 24, 11, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4096, 4093, 4075, 4131, 4085, 4063, 4115] avg=4096.0000 max=4131 max_rank=4 min=4063 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[968, 1037, 1028, 994, 11, 2, 1, 0, 991, 1040, 1036, 1098, 0, 0, 0, 0, 1017, 990, 1055, 993, 0, 0, 0, 0, 1012, 1006, 1041, 1053, 0, 0, 0, 0, 1003, 1051, 1075, 982, 0, 0, 0, 0, 1037, 1010, 1027, 1016, 2, 1, 0, 0, 1012, 1045, 1061, 1005, 0, 0, 0, 0, 1016, 1005, 1020, 993, 24, 10, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4165, 4055, 4112, 4111, 4093, 4123, 4068] avg=4096.0000 max=4165 max_rank=1 min=4041 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1011, 1030, 1029, 1059, 10, 2, 2, 0, 1041, 1075, 1013, 993, 0, 0, 0, 0, 1008, 1032, 1036, 1042, 0, 0, 0, 0, 1016, 993, 1078, 1016, 0, 0, 0, 0, 1006, 1040, 987, 1019, 0, 0, 0, 0, 1095, 1025, 997, 994, 3, 1, 0, 0, 1039, 1011, 964, 999, 0, 0, 0, 0, 985, 1041, 1021, 1020, 24, 11, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4122, 4118, 4103, 4052, 4115, 4013, 4102] avg=4096.0000 max=4143 max_rank=0 min=4013 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[984, 1059, 1040, 1023, 10, 2, 1, 0, 1016, 1042, 1055, 1021, 0, 0, 0, 0, 1044, 1009, 992, 1026, 0, 0, 0, 0, 1042, 1045, 1021, 1036, 0, 0, 0, 0, 1002, 1010, 1004, 1035, 0, 0, 0, 0, 1038, 1032, 1028, 957, 2, 1, 0, 0, 1047, 1065, 1030, 984, 0, 0, 0, 0, 995, 1003, 1019, 1015, 23, 10, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4134, 4071, 4144, 4051, 4058, 4126, 4065] avg=4096.0000 max=4144 max_rank=3 min=4051 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1035, 1059, 1060, 1013, 14, 5, 6, 0, 999, 1023, 1005, 1024, 0, 0, 0, 0, 1051, 1057, 1014, 1028, 0, 0, 0, 0, 999, 1025, 1019, 990, 0, 0, 0, 0, 1036, 951, 996, 1042, 0, 0, 0, 0, 1024, 1012, 1076, 1039, 6, 5, 0, 0, 989, 1026, 1032, 993, 0, 0, 0, 0, 1004, 982, 1024, 1063, 27, 15, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4051, 4150, 4033, 4025, 4162, 4040, 4115] avg=4096.0000 max=4192 max_rank=0 min=4025 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8064, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1069, 1050, 1026, 1016, 1035, 1025, 1035, 1022, 1018, 1007, 1010, 997, 996, 1005, 1070, 1031, 1019, 1000, 1056, 1030, 1007, 1037, 1022, 1050, 981, 965, 1026, 1030, 1047, 1050, 1039, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1051, 955, 1043, 1012, 1020, 1005, 1070, 987, 1041, 1020, 1016, 1008, 979, 1055, 1026, 1016, 1044, 1048, 1001, 1106, 1017, 1010, 985, 1021, 1002, 1009, 1046, 1025, 1014, 1053, 1016, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1005, 1001, 1019, 1050, 1036, 1066, 1004, 977, 1022, 1021, 1060, 1023, 1070, 1012, 986, 1018, 979, 1055, 1034, 981, 1025, 1025, 1029, 1024, 1010, 1020, 1054, 1018, 1033, 1096, 1034, 981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[970, 1017, 981, 1012, 997, 1046, 1008, 1044, 1019, 1053, 1045, 1028, 1017, 1045, 1046, 1019, 1034, 993, 1001, 1099, 1056, 973, 1041, 1010, 1091, 1046, 997, 973, 1028, 1077, 1002, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[993, 1046, 1004, 968, 1029, 1040, 1022, 1019, 1020, 1019, 973, 1004, 1008, 971, 1057, 1013, 1025, 1112, 1028, 1067, 1052, 1019, 1012, 1024, 1021, 1046, 1038, 1027, 1009, 1059, 1022, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1015, 1057, 1074, 975, 1044, 1022, 1015, 999, 1034, 1007, 1066, 1041, 994, 1021, 1046, 1025, 1028, 998, 1011, 1004, 1020, 1038, 992, 1042, 1023, 1011, 1052, 1109, 1021, 996, 990, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1027, 1030, 995, 1066, 971, 1031, 989, 1033, 1038, 1003, 990, 1003, 1028, 1041, 1048, 1038, 1049, 1038, 1020, 1056, 966, 1019, 1068, 1024, 1000, 998, 1003, 1027, 1043, 1012, 1033, 1081] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1061, 1010, 1042, 1052, 1004, 1038, 1068, 1059, 1014, 1046, 1000, 1005, 1004, 995, 987, 1025, 1015, 1048, 1064, 1092, 1033, 1009, 973, 976, 1079, 1002, 1041, 961, 1020, 988, 1056, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1051, 955, 1043, 1012, 8, 0, 0, 0, 1020, 1005, 1070, 987, 1, 0, 0, 0, 1041, 1020, 1016, 1008, 15, 0, 0, 0, 979, 1055, 1026, 1016, 9, 0, 0, 0, 1044, 1048, 1001, 1060, 0, 0, 0, 0, 1017, 1010, 985, 1021, 22, 0, 0, 0, 1002, 1009, 1046, 1025, 4, 0, 0, 0, 1014, 1040, 1016, 1067, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4083, 4100, 4085, 4153, 4055, 4086, 4137] avg=4096.0000 max=4153 max_rank=4 min=4055 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1069, 1050, 1026, 1016, 8, 0, 0, 0, 1035, 1025, 1035, 1022, 1, 0, 0, 0, 1018, 1007, 1010, 997, 14, 0, 0, 0, 996, 1005, 1070, 1031, 9, 0, 0, 0, 1019, 1000, 1056, 987, 0, 0, 0, 0, 1007, 1037, 1022, 1050, 20, 0, 0, 0, 981, 965, 1026, 1030, 4, 0, 0, 0, 1047, 1037, 1039, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4118, 4046, 4111, 4062, 4136, 4006, 4120] avg=4096.0000 max=4169 max_rank=0 min=4006 min_rank=6 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[970, 1017, 981, 1012, 9, 0, 0, 0, 997, 1046, 1008, 1044, 1, 0, 0, 0, 1019, 1053, 1045, 1028, 15, 0, 0, 0, 1017, 1045, 1046, 1019, 9, 0, 0, 0, 1034, 993, 1001, 1053, 0, 0, 0, 0, 1056, 973, 1041, 1010, 22, 0, 0, 0, 1091, 1046, 997, 973, 4, 0, 0, 0, 1028, 1063, 1002, 1000, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3989, 4096, 4160, 4136, 4081, 4102, 4111, 4093] avg=4096.0000 max=4160 max_rank=2 min=3989 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1005, 1001, 1019, 1050, 9, 0, 0, 0, 1036, 1066, 1004, 977, 1, 0, 0, 0, 1022, 1021, 1060, 1023, 13, 0, 0, 0, 1070, 1012, 986, 1018, 8, 0, 0, 0, 979, 1055, 1034, 941, 0, 0, 0, 0, 1025, 1025, 1029, 1024, 19, 0, 0, 0, 1010, 1020, 1054, 1018, 4, 0, 0, 0, 1033, 1082, 1034, 981, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4084, 4139, 4094, 4009, 4122, 4106, 4130] avg=4096.0000 max=4139 max_rank=2 min=4009 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1027, 1030, 995, 1066, 8, 0, 0, 0, 971, 1031, 989, 1033, 0, 0, 0, 0, 1038, 1003, 990, 1003, 14, 0, 0, 0, 1028, 1041, 1048, 1038, 9, 0, 0, 0, 1049, 1038, 1020, 1012, 0, 0, 0, 0, 966, 1019, 1068, 1024, 21, 0, 0, 0, 1000, 998, 1003, 1027, 4, 0, 0, 0, 1043, 1000, 1033, 1081, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4024, 4048, 4164, 4119, 4098, 4032, 4157] avg=4096.0000 max=4164 max_rank=3 min=4024 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1015, 1057, 1074, 975, 8, 0, 0, 0, 1044, 1022, 1015, 999, 0, 0, 0, 0, 1034, 1007, 1066, 1041, 13, 0, 0, 0, 994, 1021, 1046, 1025, 9, 0, 0, 0, 1028, 998, 1011, 962, 0, 0, 0, 0, 1020, 1038, 992, 1042, 20, 0, 0, 0, 1023, 1011, 1052, 1109, 4, 0, 0, 0, 1021, 984, 990, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4080, 4161, 4095, 3999, 4112, 4199, 3993] avg=4096.0000 max=4199 max_rank=6 min=3993 min_rank=7 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[993, 1046, 1004, 968, 13, 5, 0, 0, 1029, 1040, 1022, 1019, 4, 0, 0, 0, 1020, 1019, 973, 1004, 18, 0, 0, 0, 1008, 971, 1057, 1013, 14, 0, 0, 0, 1025, 1112, 1028, 1006, 0, 0, 0, 0, 1052, 1019, 1012, 1024, 24, 0, 0, 0, 1021, 1046, 1038, 1027, 9, 0, 0, 0, 1009, 1033, 1022, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4114, 4034, 4063, 4171, 4131, 4141, 4085] avg=4096.0000 max=4171 max_rank=4 min=4029 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16528 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1061, 1010, 1042, 1052, 8, 0, 0, 0, 1004, 1038, 1068, 1059, 0, 0, 0, 0, 1014, 1046, 1000, 1005, 15, 0, 0, 0, 1004, 995, 987, 1025, 9, 0, 0, 0, 1015, 1048, 1064, 1047, 0, 0, 0, 0, 1033, 1009, 973, 976, 21, 0, 0, 0, 1079, 1002, 1041, 961, 4, 0, 0, 0, 1020, 976, 1056, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4169, 4080, 4020, 4174, 4012, 4087, 4053] avg=4096.0000 max=4174 max_rank=4 min=4012 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1014, 1042, 1007, 1005, 1011, 1062, 1072, 1002, 973, 988, 1018, 1021, 1000, 1047, 1060, 1023, 992, 1034, 1027, 1040, 1068, 1029, 998, 1022, 1008, 1033, 1046, 992, 1027, 1041, 1037, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1024, 1037, 979, 996, 1015, 1069, 1045, 1005, 1001, 1051, 1016, 1048, 1062, 1016, 1048, 997, 1029, 1033, 1008, 1044, 1057, 1037, 999, 1025, 984, 1012, 1013, 1037, 1004, 1046, 1016, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1037, 1051, 1099, 979, 1017, 998, 983, 1035, 1047, 1024, 1011, 1044, 1024, 1017, 1047, 1012, 1016, 1031, 1046, 1003, 975, 1031, 995, 1005, 1036, 1002, 1063, 1012, 1014, 1050, 1015, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[954, 995, 1000, 1026, 1033, 1056, 1079, 1028, 1033, 998, 1004, 1035, 1045, 1047, 1058, 982, 1039, 1031, 1030, 1040, 994, 1000, 1033, 1045, 1015, 1016, 1082, 1040, 1000, 1008, 991, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1010, 1064, 994, 1018, 1051, 1039, 1063, 1009, 1006, 1043, 1029, 1022, 1025, 1040, 1017, 991, 1004, 1012, 1018, 1027, 1023, 1027, 1032, 1030, 1016, 1044, 1027, 996, 1041, 992, 1041, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1051, 988, 1007, 1020, 1063, 1009, 1031, 1033, 1058, 959, 1038, 1027, 996, 996, 1089, 1000, 1009, 1013, 1071, 1042, 1034, 1004, 994, 1052, 981, 1023, 1018, 1052, 1055, 1034, 1037, 984] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1013, 1081, 1024, 1058, 1022, 988, 984, 1038, 1030, 1008, 1076, 994, 1054, 1012, 1002, 1022, 1007, 1022, 1025, 955, 1013, 1043, 973, 1051, 1033, 1039, 1020, 1044, 1059, 1103, 991, 984] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[987, 1052, 1007, 1016, 1010, 1039, 985, 1007, 972, 1061, 996, 1018, 1040, 991, 1017, 1058, 1019, 1041, 1100, 1013, 1013, 1075, 1028, 1020, 1037, 1053, 1002, 1025, 993, 985, 1053, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1024, 1037, 979, 996, 14, 2, 0, 0, 1015, 1055, 1045, 1005, 0, 0, 0, 0, 1001, 1051, 1016, 1048, 6, 5, 2, 0, 1062, 1016, 1041, 997, 0, 0, 0, 0, 1029, 1033, 1002, 1044, 0, 0, 0, 0, 1057, 1037, 999, 1025, 3, 1, 0, 0, 984, 1012, 1010, 1037, 0, 0, 0, 0, 1004, 1043, 1016, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4120, 4129, 4116, 4108, 4122, 4043, 4078] avg=4096.0000 max=4129 max_rank=2 min=4043 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1014, 1042, 1007, 1005, 14, 2, 0, 0, 1011, 1048, 1072, 1002, 0, 0, 0, 0, 973, 988, 1018, 1021, 6, 5, 2, 0, 1000, 1047, 1053, 1023, 0, 0, 0, 0, 992, 1034, 1021, 1040, 0, 0, 0, 0, 1068, 1029, 998, 1022, 3, 1, 0, 0, 1008, 1033, 1043, 992, 0, 0, 0, 0, 1027, 1038, 1037, 1029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4133, 4013, 4123, 4087, 4121, 4076, 4131] avg=4096.0000 max=4133 max_rank=1 min=4013 min_rank=2 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1037, 1051, 1099, 979, 13, 2, 0, 0, 1017, 985, 983, 1035, 0, 0, 0, 0, 1047, 1024, 1011, 1044, 6, 5, 2, 0, 1024, 1017, 1040, 1012, 0, 0, 0, 0, 1016, 1031, 1040, 1003, 0, 0, 0, 0, 975, 1031, 995, 1005, 3, 1, 0, 0, 1036, 1002, 1060, 1012, 0, 0, 0, 0, 1014, 1047, 1015, 1049, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4020, 4139, 4093, 4090, 4010, 4110, 4125] avg=4096.0000 max=4181 max_rank=0 min=4010 min_rank=5 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1010, 1064, 994, 1018, 14, 2, 0, 0, 1051, 1025, 1063, 1009, 0, 0, 0, 0, 1006, 1043, 1029, 1022, 6, 5, 2, 0, 1025, 1040, 1010, 991, 0, 0, 0, 0, 1004, 1012, 1012, 1027, 0, 0, 0, 0, 1023, 1027, 1032, 1030, 3, 1, 0, 0, 1016, 1044, 1024, 996, 0, 0, 0, 0, 1041, 989, 1041, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4148, 4113, 4066, 4055, 4116, 4080, 4088] avg=4096.0000 max=4148 max_rank=1 min=4055 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[954, 995, 1000, 1026, 14, 2, 0, 0, 1033, 1042, 1079, 1028, 0, 0, 0, 0, 1033, 998, 1004, 1035, 6, 5, 2, 0, 1045, 1047, 1051, 982, 0, 0, 0, 0, 1039, 1031, 1024, 1040, 0, 0, 0, 0, 994, 1000, 1033, 1045, 3, 1, 0, 0, 1015, 1016, 1079, 1040, 0, 0, 0, 0, 1000, 1005, 991, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3991, 4182, 4083, 4125, 4134, 4076, 4150, 4027] avg=4096.0000 max=4182 max_rank=1 min=3991 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[987, 1052, 1007, 1016, 14, 2, 0, 0, 1010, 1025, 985, 1007, 0, 0, 0, 0, 972, 1061, 996, 1018, 7, 5, 2, 0, 1040, 991, 1010, 1058, 0, 0, 0, 0, 1019, 1041, 1093, 1013, 0, 0, 0, 0, 1013, 1075, 1028, 1020, 3, 1, 0, 0, 1037, 1053, 999, 1025, 0, 0, 0, 0, 993, 982, 1053, 1055, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4027, 4061, 4099, 4166, 4140, 4114, 4083] avg=4096.0000 max=4166 max_rank=4 min=4027 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1013, 1081, 1024, 1058, 17, 6, 0, 0, 1022, 971, 984, 1038, 0, 0, 0, 0, 1030, 1008, 1076, 994, 10, 11, 5, 0, 1054, 1012, 985, 1022, 0, 0, 0, 0, 1007, 1022, 1015, 955, 0, 0, 0, 0, 1013, 1043, 973, 1051, 8, 7, 0, 0, 1033, 1039, 1008, 1044, 0, 0, 0, 0, 1059, 1095, 991, 984, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4199, 4015, 4134, 4073, 3999, 4095, 4124, 4129] avg=4096.0000 max=4199 max_rank=0 min=3999 min_rank=4 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1051, 988, 1007, 1020, 13, 2, 0, 0, 1063, 996, 1031, 1033, 0, 0, 0, 0, 1058, 959, 1038, 1027, 6, 6, 2, 0, 996, 996, 1081, 1000, 0, 0, 0, 0, 1009, 1013, 1065, 1042, 0, 0, 0, 0, 1034, 1004, 994, 1052, 3, 1, 0, 0, 981, 1023, 1015, 1052, 0, 0, 0, 0, 1055, 1031, 1037, 984, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4123, 4096, 4073, 4129, 4088, 4071, 4107] avg=4096.0000 max=4129 max_rank=4 min=4071 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1018, 1059, 1082, 997, 1006, 1007, 971, 1020, 1011, 1107, 1016, 1023, 1041, 1011, 1042, 1031, 1008, 1023, 1013, 1002, 1045, 1020, 964, 1081, 978, 1042, 1042, 1041, 1029, 986, 1033, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1019, 1028, 999, 1006, 1018, 982, 1047, 939, 1047, 1057, 1027, 1030, 1029, 1054, 1016, 1037, 993, 1042, 1016, 1006, 1040, 1020, 1035, 1024, 1029, 1084, 995, 995, 1043, 1017, 1048, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[963, 968, 1061, 1065, 1054, 1015, 1038, 978, 1041, 1049, 1053, 1020, 1023, 1028, 1029, 999, 1012, 1012, 1030, 1045, 1024, 1052, 1039, 996, 1032, 993, 1054, 989, 1015, 1038, 993, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1002, 1059, 1039, 1019, 943, 1028, 1007, 1024, 1043, 1020, 1038, 1037, 1042, 1048, 1035, 1013, 1047, 1005, 1049, 1005, 1004, 1058, 1038, 1050, 1007, 1029, 998, 1011, 1000, 1005, 1018, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1011, 1023, 1008, 996, 1066, 1017, 1030, 1012, 1033, 980, 1003, 1041, 1009, 1032, 1030, 1024, 1066, 1031, 1014, 1009, 1020, 1025, 1001, 1032, 1021, 1058, 1041, 988, 1019, 1072, 1015, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1006, 1012, 1013, 1031, 1024, 1007, 993, 1044, 1023, 1019, 1051, 1022, 1026, 1071, 1054, 1032, 1011, 1015, 1016, 971, 1050, 1041, 1037, 1030, 1003, 1002, 1037, 1013, 1026, 1031, 1002, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1002, 1075, 1015, 1004, 1051, 1014, 1012, 999, 1006, 1011, 1085, 1075, 1049, 1062, 1021, 1039, 1042, 998, 981, 1075, 1039, 1020, 996, 1022, 985, 991, 987, 1005, 1086, 1009, 984, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1046, 1045, 1001, 1040, 1029, 1015, 1046, 996, 1032, 1026, 1029, 1037, 990, 1054, 1034, 1012, 1020, 1037, 1007, 1052, 988, 1011, 1019, 1049, 1003, 1060, 1023, 1001, 992, 1016, 1029, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1018, 1059, 1082, 997, 6, 0, 0, 0, 1006, 1007, 971, 1020, 39, 1, 0, 0, 1011, 1107, 977, 1023, 0, 0, 0, 0, 1041, 983, 1042, 1031, 0, 0, 0, 0, 1008, 1023, 1013, 1002, 13, 0, 0, 0, 1045, 1020, 964, 1068, 0, 0, 0, 0, 978, 1042, 1042, 1041, 27, 0, 0, 0, 1029, 986, 1033, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4044, 4118, 4097, 4059, 4097, 4130, 4061] avg=4096.0000 max=4162 max_rank=0 min=4044 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1019, 1028, 999, 1006, 7, 0, 0, 0, 1018, 982, 1047, 939, 40, 1, 0, 0, 1047, 1057, 987, 1030, 0, 0, 0, 0, 1029, 1024, 1016, 1037, 0, 0, 0, 0, 993, 1042, 1016, 1006, 12, 0, 0, 0, 1040, 1020, 1035, 1012, 0, 0, 0, 0, 1029, 1084, 995, 995, 29, 0, 0, 0, 1043, 1017, 1048, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4027, 4121, 4106, 4069, 4107, 4132, 4147] avg=4096.0000 max=4147 max_rank=7 min=4027 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1002, 1059, 1039, 1019, 7, 0, 0, 0, 943, 1028, 1007, 1024, 40, 1, 0, 0, 1043, 1020, 998, 1037, 0, 0, 0, 0, 1042, 1019, 1035, 1013, 0, 0, 0, 0, 1047, 1005, 1049, 1005, 12, 0, 0, 0, 1004, 1058, 1038, 1038, 0, 0, 0, 0, 1007, 1029, 998, 1011, 28, 0, 0, 0, 1000, 1005, 1018, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4043, 4098, 4109, 4118, 4138, 4073, 4063] avg=4096.0000 max=4138 max_rank=5 min=4043 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[963, 968, 1061, 1065, 7, 0, 0, 0, 1054, 1015, 1038, 978, 41, 1, 0, 0, 1041, 1049, 1012, 1020, 0, 0, 0, 0, 1023, 999, 1029, 999, 0, 0, 0, 0, 1012, 1012, 1030, 1045, 12, 0, 0, 0, 1024, 1052, 1039, 984, 0, 0, 0, 0, 1032, 993, 1054, 989, 28, 0, 0, 0, 1015, 1038, 993, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4127, 4122, 4050, 4111, 4099, 4096, 4099] avg=4096.0000 max=4127 max_rank=1 min=4050 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1011, 1023, 1008, 996, 10, 0, 0, 0, 1066, 1017, 1030, 1012, 41, 5, 0, 0, 1033, 980, 962, 1041, 0, 0, 0, 0, 1009, 989, 1030, 1024, 0, 0, 0, 0, 1066, 1031, 1014, 1009, 17, 7, 6, 0, 1020, 1025, 1001, 1015, 0, 0, 0, 0, 1021, 1058, 1041, 988, 32, 0, 0, 0, 1019, 1072, 1015, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4171, 4016, 4052, 4150, 4061, 4140, 4130] avg=4096.0000 max=4171 max_rank=1 min=4016 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1006, 1012, 1013, 1031, 7, 0, 0, 0, 1024, 1007, 993, 1044, 41, 1, 0, 0, 1023, 1019, 1010, 1022, 0, 0, 0, 0, 1026, 1041, 1054, 1032, 0, 0, 0, 0, 1011, 1015, 1016, 971, 12, 0, 0, 0, 1050, 1041, 1037, 1018, 0, 0, 0, 0, 1003, 1002, 1037, 1013, 29, 0, 0, 0, 1026, 1031, 1002, 1048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4110, 4074, 4153, 4025, 4146, 4084, 4107] avg=4096.0000 max=4153 max_rank=3 min=4025 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1002, 1075, 1015, 1004, 6, 0, 0, 0, 1051, 1014, 1012, 999, 42, 1, 0, 0, 1006, 1011, 1043, 1075, 0, 0, 0, 0, 1049, 1032, 1021, 1039, 0, 0, 0, 0, 1042, 998, 981, 1075, 12, 0, 0, 0, 1039, 1020, 996, 1010, 0, 0, 0, 0, 985, 991, 987, 1005, 29, 0, 0, 0, 1086, 1009, 984, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4119, 4135, 4141, 4108, 4065, 3997, 4101] avg=4096.0000 max=4141 max_rank=3 min=3997 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1046, 1045, 1001, 1040, 6, 0, 0, 0, 1029, 1015, 1046, 996, 40, 1, 0, 0, 1032, 1026, 989, 1037, 0, 0, 0, 0, 990, 1024, 1034, 1012, 0, 0, 0, 0, 1020, 1037, 1007, 1052, 12, 0, 0, 0, 988, 1011, 1019, 1037, 0, 0, 0, 0, 1003, 1060, 1023, 1001, 29, 0, 0, 0, 992, 1016, 1029, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4127, 4084, 4060, 4128, 4055, 4116, 4060] avg=4096.0000 max=4138 max_rank=0 min=4055 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8192, 8064, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8064, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1018, 1046, 1069, 1048, 1030, 1022, 1066, 1034, 1012, 1038, 1043, 991, 1051, 1048, 1009, 997, 1017, 1045, 983, 992, 1015, 1070, 990, 1051, 1037, 1029, 988, 1011, 981, 980, 1063, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1015, 1054, 1046, 1037, 945, 1066, 1029, 1022, 1001, 1013, 1005, 1088, 1011, 1046, 1046, 1005, 1028, 1054, 1013, 1034, 994, 1037, 1021, 1026, 1023, 1028, 1014, 1066, 1012, 1028, 962, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1002, 1008, 1037, 1064, 1031, 979, 1041, 990, 1057, 1049, 1022, 965, 1007, 1009, 1056, 1059, 1054, 1058, 1033, 1045, 1002, 1049, 1008, 1067, 1015, 1044, 1019, 1002, 1027, 1004, 977, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1024, 1024, 1025, 1024, 1010, 1047, 995, 1087, 1028, 1058, 1024, 1064, 1011, 1031, 1001, 1042, 1044, 932, 1037, 1015, 1028, 997, 1031, 984, 1017, 1030, 1032, 1026, 965, 1051, 1029, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1016, 1014, 1028, 1028, 1036, 1012, 1054, 1034, 1067, 1031, 998, 997, 999, 977, 1018, 1026, 1049, 1015, 1010, 1049, 999, 1048, 1037, 1026, 1088, 964, 1014, 1025, 999, 1014, 1070, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1029, 1034, 1043, 991, 1015, 970, 1014, 1042, 1034, 1008, 1079, 1051, 990, 1041, 1041, 1083, 1026, 1024, 1053, 1014, 1018, 991, 1000, 1010, 1003, 1053, 1003, 984, 1027, 1029, 1018, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1001, 946, 1090, 1014, 1057, 1049, 1014, 1037, 1071, 1030, 1033, 1023, 1032, 1024, 1019, 1060, 1017, 1039, 952, 1056, 1000, 1032, 1035, 1011, 1034, 1021, 1016, 993, 1011, 1054, 1023, 974] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[990, 1038, 1007, 1042, 982, 1049, 997, 1022, 1041, 1073, 1021, 963, 1047, 1002, 1016, 994, 993, 1009, 1023, 1026, 1108, 1045, 1045, 1032, 1047, 1016, 1021, 1045, 1019, 990, 1067, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1018, 1046, 1059, 1048, 0, 0, 0, 0, 1030, 1022, 1066, 1033, 0, 0, 0, 0, 987, 1038, 1043, 991, 0, 0, 0, 0, 1051, 1048, 1009, 994, 0, 0, 0, 0, 1017, 1045, 983, 992, 2, 1, 0, 0, 1015, 1065, 990, 1051, 0, 0, 0, 0, 1037, 1029, 988, 1011, 5, 1, 1, 0, 981, 980, 1063, 994, 25, 9, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4151, 4059, 4102, 4040, 4121, 4072, 4052] avg=4096.0000 max=4171 max_rank=0 min=4040 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1002, 1008, 1027, 1064, 0, 0, 0, 0, 1031, 979, 1041, 989, 0, 0, 0, 0, 1031, 1049, 1022, 965, 0, 0, 0, 0, 1007, 1009, 1056, 1056, 0, 0, 0, 0, 1054, 1058, 1033, 1045, 2, 1, 0, 0, 1002, 1045, 1008, 1067, 0, 0, 0, 0, 1015, 1044, 1019, 1002, 4, 1, 1, 0, 1027, 1004, 977, 988, 26, 9, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4040, 4067, 4128, 4193, 4122, 4086, 4031] avg=4096.0000 max=4193 max_rank=4 min=4031 min_rank=7 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1015, 1054, 1036, 1037, 0, 0, 0, 0, 945, 1066, 1029, 1021, 0, 0, 0, 0, 976, 1013, 1005, 1088, 0, 0, 0, 0, 1011, 1046, 1046, 1002, 0, 0, 0, 0, 1028, 1054, 1013, 1034, 2, 1, 0, 0, 994, 1033, 1021, 1026, 0, 0, 0, 0, 1023, 1028, 1014, 1066, 4, 1, 1, 0, 1012, 1028, 962, 999, 25, 9, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4061, 4082, 4105, 4132, 4074, 4137, 4035] avg=4096.0000 max=4142 max_rank=0 min=4035 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1024, 1024, 1015, 1024, 0, 0, 0, 0, 1010, 1047, 995, 1086, 0, 0, 0, 0, 1003, 1058, 1024, 1064, 0, 0, 0, 0, 1011, 1031, 1001, 1039, 0, 0, 0, 0, 1044, 932, 1037, 1015, 2, 1, 0, 0, 1028, 993, 1031, 984, 0, 0, 0, 0, 1017, 1030, 1032, 1026, 4, 1, 1, 0, 965, 1051, 1029, 1055, 25, 9, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4138, 4149, 4082, 4031, 4036, 4111, 4134] avg=4096.0000 max=4149 max_rank=2 min=4031 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1029, 1034, 1033, 991, 0, 0, 0, 0, 1015, 970, 1014, 1041, 0, 0, 0, 0, 1008, 1008, 1079, 1051, 0, 0, 0, 0, 990, 1041, 1041, 1080, 0, 0, 0, 0, 1026, 1024, 1053, 1014, 2, 1, 0, 0, 1018, 987, 1000, 1010, 0, 0, 0, 0, 1003, 1053, 1003, 984, 4, 1, 1, 0, 1027, 1029, 1018, 1050, 26, 9, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4040, 4146, 4152, 4120, 4015, 4049, 4159] avg=4096.0000 max=4159 max_rank=7 min=4015 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1016, 1014, 1013, 1028, 0, 0, 0, 0, 1036, 1012, 1054, 1031, 0, 0, 0, 0, 1037, 1031, 998, 997, 0, 0, 0, 0, 999, 977, 1018, 1017, 0, 0, 0, 0, 1049, 1015, 1010, 1049, 5, 3, 0, 0, 999, 1038, 1037, 1026, 0, 0, 0, 0, 1088, 964, 1014, 1025, 10, 4, 3, 0, 999, 1014, 1070, 1026, 30, 12, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4133, 4063, 4011, 4131, 4100, 4108, 4151] avg=4096.0000 max=4151 max_rank=7 min=4011 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1001, 946, 1080, 1014, 0, 0, 0, 0, 1057, 1049, 1014, 1036, 0, 0, 0, 0, 1044, 1030, 1033, 1023, 0, 0, 0, 0, 1032, 1024, 1019, 1057, 0, 0, 0, 0, 1017, 1039, 952, 1056, 2, 1, 0, 0, 1000, 1028, 1035, 1011, 0, 0, 0, 0, 1034, 1021, 1016, 993, 4, 1, 1, 0, 1011, 1054, 1023, 974, 27, 9, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4156, 4130, 4132, 4067, 4074, 4070, 4098] avg=4096.0000 max=4156 max_rank=1 min=4041 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[990, 1038, 998, 1042, 0, 0, 0, 0, 982, 1049, 997, 1021, 0, 0, 0, 0, 1015, 1073, 1021, 963, 0, 0, 0, 0, 1047, 1002, 1016, 991, 0, 0, 0, 0, 993, 1009, 1023, 1026, 2, 1, 0, 0, 1108, 1041, 1045, 1032, 0, 0, 0, 0, 1047, 1016, 1021, 1045, 4, 1, 1, 0, 1019, 990, 1067, 998, 26, 8, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4049, 4072, 4056, 4054, 4226, 4135, 4108] avg=4096.0000 max=4226 max_rank=5 min=4049 min_rank=1 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16523 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1089, 1013, 997, 1024, 1020, 1009, 1005, 1023, 1019, 1027, 1064, 1024, 1038, 1039, 1029, 1029, 1028, 1060, 983, 1043, 1009, 1045, 1036, 1023, 1054, 1036, 952, 1045, 1031, 977, 1007, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1008, 1087, 992, 998, 948, 1042, 1004, 1007, 1011, 1049, 1065, 1032, 1019, 1021, 1041, 1074, 1001, 1072, 988, 1023, 1005, 1017, 1016, 1013, 996, 1002, 1033, 1070, 1047, 1054, 1019, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1047, 1005, 972, 974, 1001, 1041, 1048, 1039, 1018, 978, 1038, 1010, 1012, 1010, 977, 1030, 1007, 1028, 1031, 1016, 1050, 1040, 1044, 1019, 1068, 1058, 1018, 1023, 1006, 1050, 1058, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1037, 1054, 1053, 1024, 993, 1071, 1038, 1040, 1026, 1042, 1061, 1074, 1000, 960, 1036, 964, 977, 1044, 1032, 1004, 1013, 1018, 1019, 1061, 1022, 1030, 1040, 1033, 1022, 947, 1010, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1048, 997, 1007, 1028, 1009, 1014, 1072, 1025, 1038, 1008, 1057, 964, 959, 975, 1023, 1002, 1044, 1057, 1050, 999, 1021, 1035, 1021, 1031, 1026, 987, 1046, 1065, 991, 1063, 1056, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[996, 1013, 1022, 1022, 1003, 1033, 1007, 1008, 1002, 1025, 952, 1035, 1088, 1025, 1052, 1063, 1047, 979, 1037, 1003, 1073, 999, 1076, 1053, 1030, 985, 1019, 1009, 1021, 1043, 1025, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1030, 1005, 994, 1050, 1043, 994, 1017, 1021, 992, 993, 1061, 1043, 1011, 1022, 974, 1087, 1065, 1008, 1016, 1014, 1048, 1035, 1058, 1039, 1010, 1022, 1059, 975, 1034, 1036, 978, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1051, 966, 1004, 991, 1031, 1022, 1067, 994, 1047, 1073, 956, 1014, 1077, 1014, 1029, 1042, 1020, 1041, 1011, 1073, 965, 1029, 1051, 1039, 1011, 1024, 1030, 1008, 1018, 1018, 989, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1089, 1013, 997, 1024, 21, 0, 0, 0, 1020, 1009, 1005, 1023, 7, 2, 0, 0, 1019, 1027, 1061, 1024, 0, 0, 0, 0, 1038, 1039, 1029, 1029, 3, 2, 0, 0, 1028, 1056, 983, 1043, 0, 0, 0, 0, 1009, 1045, 1008, 1023, 0, 0, 0, 0, 1054, 1036, 952, 1043, 0, 0, 0, 0, 1031, 977, 1007, 990, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4066, 4131, 4140, 4110, 4085, 4085, 4007] avg=4096.0000 max=4144 max_rank=0 min=4007 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1008, 1087, 992, 998, 20, 0, 0, 0, 948, 1042, 1004, 1007, 7, 2, 0, 0, 1011, 1049, 1062, 1032, 0, 0, 0, 0, 1019, 1021, 1041, 1074, 3, 2, 0, 0, 1001, 1068, 988, 1023, 0, 0, 0, 0, 1005, 1017, 989, 1013, 0, 0, 0, 0, 996, 1002, 1033, 1068, 0, 0, 0, 0, 1047, 1054, 1019, 1014, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4010, 4154, 4160, 4080, 4024, 4099, 4136] avg=4096.0000 max=4160 max_rank=3 min=4010 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1037, 1054, 1053, 1024, 20, 0, 0, 0, 993, 1071, 1038, 1040, 7, 2, 0, 0, 1026, 1042, 1058, 1074, 0, 0, 0, 0, 1000, 960, 1036, 964, 3, 2, 0, 0, 977, 1040, 1032, 1004, 0, 0, 0, 0, 1013, 1018, 992, 1061, 0, 0, 0, 0, 1022, 1030, 1040, 1031, 0, 0, 0, 0, 1022, 947, 1010, 1023, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4188, 4151, 4200, 3965, 4053, 4084, 4123, 4004] avg=4096.0000 max=4200 max_rank=2 min=3965 min_rank=3 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16264 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1047, 1005, 972, 974, 21, 0, 0, 0, 1001, 1041, 1048, 1039, 7, 1, 0, 0, 1018, 978, 1035, 1010, 0, 0, 0, 0, 1012, 1010, 977, 1030, 3, 2, 0, 0, 1007, 1025, 1031, 1016, 0, 0, 0, 0, 1050, 1040, 1016, 1019, 0, 0, 0, 0, 1068, 1058, 1018, 1021, 0, 0, 0, 0, 1006, 1050, 1058, 1052, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4137, 4041, 4034, 4079, 4125, 4165, 4168] avg=4096.0000 max=4168 max_rank=7 min=4019 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1048, 997, 1007, 1028, 25, 0, 0, 0, 1009, 1014, 1072, 1025, 12, 5, 0, 0, 1038, 1008, 1048, 964, 0, 0, 0, 0, 959, 975, 1023, 1002, 8, 3, 0, 0, 1044, 1049, 1050, 999, 0, 0, 0, 0, 1021, 1035, 984, 1031, 0, 0, 0, 0, 1026, 987, 1046, 1061, 0, 0, 0, 0, 991, 1063, 1056, 1050, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4137, 4058, 3970, 4142, 4071, 4120, 4165] avg=4096.0000 max=4165 max_rank=7 min=3970 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1030, 1005, 994, 1050, 21, 0, 0, 0, 1043, 994, 1017, 1021, 8, 1, 0, 0, 992, 993, 1058, 1043, 0, 0, 0, 0, 1011, 1022, 974, 1087, 3, 2, 0, 0, 1065, 1005, 1016, 1014, 0, 0, 0, 0, 1048, 1035, 1029, 1039, 0, 0, 0, 0, 1010, 1022, 1059, 973, 0, 0, 0, 0, 1034, 1036, 978, 1034, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4084, 4086, 4099, 4100, 4151, 4064, 4084] avg=4096.0000 max=4151 max_rank=5 min=4064 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[996, 1013, 1022, 1022, 21, 0, 0, 0, 1003, 1033, 1007, 1008, 8, 1, 0, 0, 1002, 1025, 949, 1035, 0, 0, 0, 0, 1088, 1025, 1052, 1063, 3, 2, 0, 0, 1047, 976, 1037, 1003, 0, 0, 0, 0, 1073, 999, 1047, 1053, 0, 0, 0, 0, 1030, 985, 1019, 1007, 0, 0, 0, 0, 1021, 1043, 1025, 1023, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4060, 4011, 4233, 4063, 4172, 4041, 4114] avg=4096.0000 max=4233 max_rank=3 min=4011 min_rank=2 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1051, 966, 1004, 991, 21, 0, 0, 0, 1031, 1022, 1067, 994, 7, 2, 0, 0, 1047, 1073, 953, 1014, 0, 0, 0, 0, 1077, 1014, 1029, 1042, 3, 2, 0, 0, 1020, 1037, 1011, 1073, 0, 0, 0, 0, 965, 1029, 1023, 1039, 0, 0, 0, 0, 1011, 1024, 1030, 1006, 0, 0, 0, 0, 1018, 1018, 989, 1063, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4123, 4087, 4167, 4141, 4056, 4071, 4090] avg=4096.0000 max=4167 max_rank=3 min=4033 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8064, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1039, 1045, 984, 998, 1028, 1035, 1047, 1020, 1033, 1002, 1057, 1006, 1017, 1055, 1065, 970, 1030, 1053, 985, 1056, 1021, 1004, 1024, 1017, 984, 1030, 994, 1010, 1035, 1060, 1003, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1060, 1010, 1045, 991, 1004, 1055, 1041, 1018, 1053, 986, 1042, 1043, 1024, 1022, 997, 1038, 1083, 957, 1048, 993, 1021, 999, 1017, 1020, 1015, 982, 1039, 1056, 1078, 1032, 1039, 960] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1011, 1015, 1002, 1040, 1018, 1000, 1035, 1074, 1018, 1032, 1022, 1062, 1046, 1013, 1055, 1034, 1010, 1025, 977, 998, 1020, 1049, 1006, 1044, 1037, 1005, 1033, 979, 1092, 1000, 990, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[975, 1049, 1046, 989, 1024, 1042, 1074, 1001, 1016, 1037, 995, 1079, 1030, 1032, 970, 1027, 1073, 1057, 966, 1022, 1035, 998, 978, 1026, 1067, 1011, 1033, 996, 1031, 1036, 1023, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1026, 1020, 1037, 1014, 1002, 1049, 962, 1061, 1041, 1024, 1037, 1010, 1008, 998, 1032, 1026, 1022, 982, 1022, 1079, 1036, 1031, 1002, 1038, 1034, 1030, 1042, 1007, 1006, 1032, 1015, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1020, 997, 1026, 991, 982, 1017, 1052, 1034, 980, 1038, 1002, 1039, 1020, 1044, 994, 1037, 1026, 1078, 1032, 1016, 1030, 1006, 1034, 1018, 1004, 1014, 1063, 1020, 1034, 1030, 1002, 1088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1024, 1027, 988, 995, 1043, 1002, 993, 1057, 1023, 1023, 1026, 1066, 1016, 1010, 1009, 1014, 1054, 1002, 1031, 1020, 977, 1026, 1002, 1031, 993, 1084, 1045, 1014, 1011, 1076, 1077, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1064, 1023, 1016, 1010, 990, 999, 1054, 1048, 1026, 1012, 1018, 1050, 1019, 1022, 1015, 1017, 965, 1009, 1064, 1005, 990, 1055, 1082, 1008, 1027, 1034, 991, 1040, 1083, 1020, 1030, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1039, 1045, 984, 998, 23, 0, 0, 0, 1028, 1035, 1047, 1010, 0, 0, 0, 0, 1033, 1002, 1057, 992, 0, 0, 0, 0, 1017, 1055, 1065, 970, 9, 1, 0, 0, 1030, 1053, 985, 1056, 3, 0, 0, 0, 1021, 1004, 1024, 1017, 9, 5, 0, 0, 984, 1030, 994, 1010, 6, 0, 0, 0, 1003, 1060, 1003, 1061, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4120, 4084, 4117, 4127, 4080, 4024, 4127] avg=4096.0000 max=4127 max_rank=4 min=4024 min_rank=6 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1011, 1015, 1002, 1040, 24, 0, 0, 0, 1018, 1000, 1035, 1063, 0, 0, 0, 0, 1018, 1032, 1022, 1046, 0, 0, 0, 0, 1046, 1013, 1055, 1034, 10, 1, 0, 0, 1010, 1025, 977, 998, 3, 0, 0, 0, 1020, 1049, 1006, 1044, 9, 6, 0, 0, 1037, 1005, 1033, 979, 7, 0, 0, 0, 1059, 1000, 990, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4116, 4118, 4159, 4013, 4134, 4061, 4075] avg=4096.0000 max=4159 max_rank=3 min=4013 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 1020, 1037, 1014, 22, 0, 0, 0, 1002, 1049, 962, 1050, 0, 0, 0, 0, 1041, 1024, 1037, 996, 0, 0, 0, 0, 1008, 998, 1032, 1026, 9, 1, 0, 0, 1022, 982, 1022, 1079, 3, 0, 0, 0, 1036, 1031, 1002, 1038, 9, 5, 0, 0, 1034, 1030, 1042, 1007, 7, 0, 0, 0, 975, 1032, 1015, 1043, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4063, 4098, 4074, 4108, 4121, 4120, 4065] avg=4096.0000 max=4121 max_rank=5 min=4063 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1060, 1010, 1045, 991, 24, 0, 0, 0, 1004, 1055, 1041, 1008, 0, 0, 0, 0, 1053, 986, 1042, 1028, 0, 0, 0, 0, 1024, 1022, 997, 1038, 10, 1, 0, 0, 1083, 957, 1048, 993, 3, 0, 0, 0, 1021, 999, 1017, 1020, 9, 5, 0, 0, 1015, 982, 1039, 1056, 6, 0, 0, 0, 1045, 1032, 1039, 960, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4108, 4109, 4092, 4084, 4071, 4098, 4076] avg=4096.0000 max=4130 max_rank=0 min=4071 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[975, 1049, 1046, 989, 23, 0, 0, 0, 1024, 1042, 1074, 991, 0, 0, 0, 0, 1016, 1037, 995, 1063, 0, 0, 0, 0, 1030, 1032, 970, 1027, 10, 1, 0, 0, 1073, 1057, 966, 1022, 3, 0, 0, 0, 1035, 998, 978, 1026, 9, 6, 0, 0, 1067, 1011, 1033, 996, 6, 0, 0, 0, 999, 1036, 1023, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4131, 4111, 4070, 4121, 4052, 4113, 4088] avg=4096.0000 max=4131 max_rank=1 min=4052 min_rank=5 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1020, 997, 1026, 991, 23, 0, 0, 0, 982, 1017, 1052, 1024, 0, 0, 0, 0, 980, 1038, 1002, 1024, 0, 0, 0, 0, 1020, 1044, 994, 1037, 10, 1, 0, 0, 1026, 1078, 1032, 1016, 3, 0, 0, 0, 1030, 1006, 1034, 1018, 9, 5, 0, 0, 1004, 1014, 1063, 1020, 6, 0, 0, 0, 1002, 1030, 1002, 1088, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4075, 4044, 4106, 4155, 4102, 4107, 4122] avg=4096.0000 max=4155 max_rank=4 min=4044 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1024, 1027, 988, 995, 28, 0, 0, 0, 1043, 1002, 993, 1036, 0, 0, 0, 0, 1023, 1023, 1026, 1042, 0, 0, 0, 0, 1016, 1010, 1009, 1014, 14, 3, 0, 0, 1054, 1002, 1031, 1020, 7, 0, 0, 0, 977, 1026, 1002, 1031, 12, 10, 0, 0, 993, 1084, 1045, 1014, 11, 0, 0, 0, 971, 1076, 1077, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4074, 4114, 4066, 4114, 4058, 4147, 4133] avg=4096.0000 max=4147 max_rank=6 min=4058 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1064, 1023, 1016, 1010, 24, 0, 0, 0, 990, 999, 1054, 1038, 0, 0, 0, 0, 1026, 1012, 1018, 1034, 0, 0, 0, 0, 1019, 1022, 1015, 1017, 10, 1, 0, 0, 965, 1009, 1064, 1005, 3, 0, 0, 0, 990, 1055, 1082, 1008, 9, 6, 0, 0, 1027, 1034, 991, 1040, 6, 0, 0, 0, 1050, 1020, 1030, 982, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4081, 4090, 4084, 4046, 4150, 4098, 4082] avg=4096.0000 max=4150 max_rank=5 min=4046 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8064, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[984, 1035, 1027, 1040, 1008, 1006, 998, 996, 1042, 1035, 1050, 1012, 1047, 1060, 984, 1006, 986, 1034, 999, 1023, 1111, 1004, 1028, 1025, 1024, 1046, 986, 1015, 1045, 1056, 993, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1024, 1043, 1037, 1018, 999, 1040, 1037, 1019, 1019, 1009, 1043, 1021, 988, 1024, 1026, 994, 1104, 1034, 1019, 1006, 1042, 1026, 1010, 1036, 1010, 1006, 993, 1035, 1042, 1044, 1026, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1072, 999, 1027, 1046, 1044, 1027, 1046, 1022, 1040, 994, 1006, 1018, 1024, 978, 994, 1036, 1018, 985, 987, 1008, 1000, 1007, 995, 1020, 1042, 1044, 1034, 1073, 1036, 1034, 1075, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1038, 1057, 1030, 1044, 967, 1015, 1011, 1007, 1025, 1031, 1055, 1053, 987, 1011, 1044, 1023, 997, 1000, 1052, 1026, 1007, 985, 1038, 1048, 978, 1025, 1054, 1008, 1026, 1013, 1091, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1050, 1033, 1073, 978, 1058, 1022, 1041, 987, 997, 1040, 978, 983, 1002, 1055, 989, 1021, 1019, 1033, 995, 1048, 1058, 1027, 1039, 973, 1017, 1075, 1020, 1036, 1020, 1027, 1080, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[998, 984, 1012, 1064, 994, 1083, 1083, 1016, 1011, 982, 1048, 1045, 1043, 1001, 1021, 1052, 1005, 1031, 1043, 1012, 996, 1014, 1048, 1046, 998, 1003, 1037, 1003, 1008, 1068, 1017, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1032, 1017, 1011, 1053, 1035, 1031, 1019, 1055, 1047, 1037, 1032, 1001, 1081, 991, 1033, 1039, 1052, 1008, 998, 1014, 1033, 1013, 1014, 971, 1006, 1012, 992, 1027, 1029, 1052, 1015, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1046, 1028, 1012, 1047, 1010, 1031, 1020, 1057, 1034, 960, 1037, 1031, 1004, 1015, 1008, 1054, 1044, 1066, 1061, 1005, 948, 1024, 1028, 999, 1041, 1038, 1037, 981, 1034, 1033, 1006, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[984, 1035, 1027, 1017, 0, 0, 0, 0, 1008, 1005, 998, 996, 0, 0, 0, 0, 1042, 1035, 1050, 1012, 4, 1, 0, 0, 1047, 1060, 984, 1006, 13, 3, 0, 0, 986, 1034, 999, 1023, 7, 0, 0, 0, 1111, 1004, 1028, 1025, 19, 0, 0, 0, 1024, 1046, 986, 1015, 9, 0, 0, 0, 1045, 1024, 993, 1063, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4007, 4144, 4113, 4049, 4187, 4080, 4125] avg=4096.0000 max=4187 max_rank=5 min=4007 min_rank=1 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 1043, 1037, 997, 0, 0, 0, 0, 999, 1038, 1037, 1019, 0, 0, 0, 0, 1019, 1009, 1043, 1021, 4, 2, 0, 0, 988, 1024, 1026, 994, 13, 3, 0, 0, 1104, 1034, 1019, 1006, 6, 0, 0, 0, 1042, 1026, 1010, 1036, 19, 0, 0, 0, 1010, 1006, 993, 1035, 8, 0, 0, 0, 1042, 1012, 1026, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4093, 4098, 4048, 4169, 4133, 4052, 4074] avg=4096.0000 max=4169 max_rank=4 min=4048 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1072, 999, 1027, 1023, 0, 0, 0, 0, 1044, 1026, 1046, 1022, 0, 0, 0, 0, 1040, 994, 1006, 1018, 4, 1, 0, 0, 1024, 978, 994, 1036, 13, 3, 0, 0, 1018, 985, 987, 1008, 7, 0, 0, 0, 1000, 1007, 995, 1020, 19, 0, 0, 0, 1042, 1044, 1034, 1073, 9, 0, 0, 0, 1036, 1002, 1075, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4138, 4063, 4048, 4005, 4041, 4202, 4150] avg=4096.0000 max=4202 max_rank=6 min=4005 min_rank=4 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1038, 1057, 1030, 1010, 0, 0, 0, 0, 967, 1008, 1011, 1007, 0, 0, 0, 0, 1025, 1031, 1055, 1053, 8, 7, 0, 0, 987, 1011, 1044, 1023, 15, 6, 0, 0, 997, 1000, 1052, 1026, 9, 0, 0, 0, 1007, 985, 1038, 1048, 22, 0, 0, 0, 978, 1025, 1054, 1008, 11, 0, 0, 0, 1026, 976, 1091, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 3993, 4179, 4086, 4084, 4100, 4076, 4115] avg=4096.0000 max=4179 max_rank=2 min=3993 min_rank=1 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[998, 984, 1012, 1041, 0, 0, 0, 0, 994, 1081, 1083, 1016, 0, 0, 0, 0, 1011, 982, 1048, 1045, 4, 2, 0, 0, 1043, 1001, 1021, 1052, 13, 3, 0, 0, 1005, 1031, 1043, 1012, 7, 0, 0, 0, 996, 1014, 1048, 1046, 19, 0, 0, 0, 998, 1003, 1037, 1003, 9, 0, 0, 0, 1008, 1036, 1017, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4174, 4092, 4133, 4098, 4123, 4050, 4063] avg=4096.0000 max=4174 max_rank=1 min=4035 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1050, 1033, 1073, 957, 0, 0, 0, 0, 1058, 1021, 1041, 987, 0, 0, 0, 0, 997, 1040, 978, 983, 4, 1, 0, 0, 1002, 1055, 989, 1021, 13, 3, 0, 0, 1019, 1033, 995, 1048, 6, 0, 0, 0, 1058, 1027, 1039, 973, 19, 0, 0, 0, 1017, 1075, 1020, 1036, 8, 0, 0, 0, 1020, 995, 1080, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4107, 4003, 4083, 4101, 4116, 4156, 4089] avg=4096.0000 max=4156 max_rank=6 min=4003 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1032, 1017, 1011, 1030, 0, 0, 0, 0, 1035, 1030, 1019, 1055, 0, 0, 0, 0, 1047, 1037, 1032, 1001, 4, 1, 0, 0, 1081, 991, 1033, 1039, 13, 3, 0, 0, 1052, 1008, 998, 1014, 7, 0, 0, 0, 1033, 1013, 1014, 971, 19, 0, 0, 0, 1006, 1012, 992, 1027, 9, 0, 0, 0, 1029, 1020, 1015, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4139, 4122, 4160, 4079, 4050, 4046, 4082] avg=4096.0000 max=4160 max_rank=3 min=4046 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1046, 1028, 1012, 1024, 0, 0, 0, 0, 1010, 1030, 1020, 1057, 0, 0, 0, 0, 1034, 960, 1037, 1031, 4, 1, 0, 0, 1004, 1015, 1008, 1054, 13, 3, 0, 0, 1044, 1066, 1061, 1005, 7, 0, 0, 0, 948, 1024, 1028, 999, 19, 0, 0, 0, 1041, 1038, 1037, 981, 9, 0, 0, 0, 1034, 1001, 1006, 1029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4117, 4067, 4097, 4183, 4018, 4106, 4070] avg=4096.0000 max=4183 max_rank=4 min=4018 min_rank=5 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1072, 1025, 984, 1071, 1024, 1030, 1007, 1018, 1053, 1006, 1020, 1038, 1074, 1023, 1017, 982, 983, 996, 1035, 1027, 971, 1030, 1042, 977, 1065, 991, 1040, 1027, 1021, 1031, 1066, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[968, 1048, 1023, 1032, 1038, 1048, 1002, 976, 1050, 988, 973, 995, 1048, 1039, 1023, 1006, 1027, 1000, 1083, 1080, 1061, 999, 1055, 1018, 1005, 1053, 1049, 1046, 993, 1023, 1022, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1080, 1055, 1005, 1003, 1001, 1073, 1069, 1026, 1023, 1018, 1042, 1010, 1015, 1066, 1026, 1051, 1002, 1020, 1007, 1036, 1005, 1070, 1021, 1001, 1028, 979, 1041, 994, 1029, 992, 960, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1038, 1015, 1066, 989, 996, 1002, 1042, 998, 963, 1019, 995, 1049, 1003, 995, 1027, 1018, 1010, 1042, 1031, 1046, 1021, 1016, 1041, 1047, 1064, 1034, 1023, 1012, 1013, 1046, 1045, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1021, 1061, 1033, 1031, 1079, 997, 1017, 1051, 1066, 1032, 992, 962, 1027, 962, 1042, 1018, 1058, 1014, 1006, 1017, 1003, 1023, 1022, 987, 1048, 1049, 1011, 1016, 1024, 1037, 1010, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1020, 1016, 944, 1019, 1006, 1018, 1032, 1021, 1085, 1079, 1012, 1062, 962, 1027, 1046, 1016, 1029, 1010, 1012, 1058, 985, 1001, 1000, 1045, 1013, 1003, 1053, 1062, 1054, 1038, 1013, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1041, 1048, 1015, 1052, 989, 1038, 997, 1039, 978, 1004, 1052, 1021, 1042, 1015, 999, 1077, 1023, 993, 1059, 1024, 1019, 1025, 993, 1015, 1002, 1087, 1068, 991, 1024, 1019, 1016, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[992, 1015, 1007, 1042, 998, 1083, 1015, 1029, 1080, 1017, 1038, 1005, 1026, 1032, 1019, 976, 1046, 1032, 1026, 1019, 1066, 1018, 1053, 1015, 1064, 1034, 1007, 1027, 1005, 1022, 985, 975] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1072, 1018, 984, 1071, 0, 0, 0, 0, 1024, 1030, 1007, 1018, 1, 0, 0, 0, 1053, 1006, 1020, 1038, 5, 0, 0, 0, 1074, 1023, 1017, 982, 6, 1, 0, 0, 983, 996, 1035, 1018, 0, 0, 0, 0, 971, 1030, 1042, 977, 15, 0, 0, 0, 1065, 991, 1014, 1027, 0, 0, 0, 0, 1021, 1031, 1066, 1022, 11, 3, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4080, 4122, 4103, 4032, 4035, 4097, 4154] avg=4096.0000 max=4154 max_rank=7 min=4032 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1080, 1048, 1005, 1003, 0, 0, 0, 0, 1001, 1073, 1069, 1026, 1, 0, 0, 0, 1023, 1018, 1042, 1010, 5, 0, 0, 0, 1015, 1066, 1026, 1051, 6, 1, 0, 0, 1002, 1020, 1007, 1027, 0, 0, 0, 0, 1005, 1070, 1021, 1001, 15, 0, 0, 0, 1028, 979, 1015, 994, 0, 0, 0, 0, 1029, 992, 960, 1020, 11, 3, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4170, 4098, 4165, 4056, 4112, 4016, 4015] avg=4096.0000 max=4170 max_rank=1 min=4015 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16199 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1038, 1008, 1066, 989, 0, 0, 0, 0, 996, 1002, 1042, 998, 1, 0, 0, 0, 963, 1019, 995, 1049, 5, 0, 0, 0, 1003, 995, 1027, 1018, 7, 1, 0, 0, 1010, 1042, 1031, 1036, 0, 0, 0, 0, 1021, 1016, 1041, 1047, 15, 0, 0, 0, 1064, 1034, 997, 1012, 0, 0, 0, 0, 1013, 1046, 1045, 1062, 11, 3, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4039, 4031, 4051, 4119, 4140, 4107, 4180] avg=4096.0000 max=4180 max_rank=7 min=4031 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16546 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[968, 1041, 1023, 1032, 0, 0, 0, 0, 1038, 1048, 1002, 976, 1, 0, 0, 0, 1050, 988, 973, 995, 5, 0, 0, 0, 1048, 1039, 1023, 1006, 7, 1, 0, 0, 1027, 1000, 1083, 1070, 0, 0, 0, 0, 1061, 999, 1055, 1018, 15, 0, 0, 0, 1005, 1053, 1022, 1046, 0, 0, 0, 0, 993, 1023, 1022, 997, 12, 3, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4065, 4011, 4124, 4180, 4148, 4126, 4050] avg=4096.0000 max=4180 max_rank=4 min=4011 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1021, 1047, 1033, 1031, 0, 0, 0, 0, 1079, 997, 1017, 1051, 2, 0, 0, 0, 1066, 1032, 992, 962, 6, 0, 0, 0, 1027, 962, 1042, 1018, 11, 6, 0, 0, 1058, 1014, 1006, 1000, 0, 0, 0, 0, 1003, 1023, 1022, 987, 19, 0, 0, 0, 1048, 1049, 977, 1016, 0, 0, 0, 0, 1024, 1037, 1010, 1052, 15, 6, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4146, 4058, 4066, 4078, 4054, 4090, 4144] avg=4096.0000 max=4146 max_rank=1 min=4054 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1041, 1041, 1015, 1052, 0, 0, 0, 0, 989, 1038, 997, 1039, 1, 0, 0, 0, 978, 1004, 1052, 1021, 5, 0, 0, 0, 1042, 1015, 999, 1077, 6, 1, 0, 0, 1023, 993, 1059, 1015, 0, 0, 0, 0, 1019, 1025, 993, 1015, 15, 0, 0, 0, 1002, 1087, 1041, 991, 0, 0, 0, 0, 1024, 1019, 1016, 1003, 12, 3, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4064, 4060, 4140, 4090, 4067, 4121, 4077] avg=4096.0000 max=4149 max_rank=0 min=4060 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1020, 1009, 944, 1019, 0, 0, 0, 0, 1006, 1018, 1032, 1021, 1, 0, 0, 0, 1085, 1079, 1012, 1062, 5, 0, 0, 0, 962, 1027, 1046, 1016, 7, 1, 0, 0, 1029, 1010, 1012, 1048, 0, 0, 0, 0, 985, 1001, 1000, 1045, 15, 0, 0, 0, 1013, 1003, 1026, 1062, 0, 0, 0, 0, 1054, 1038, 1013, 1027, 12, 3, 0, 0] rerouted_tokens_per_ep_rank=[3992, 4078, 4243, 4059, 4099, 4046, 4104, 4147] avg=4096.0000 max=4243 max_rank=2 min=3992 min_rank=0 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[992, 1008, 1007, 1042, 0, 0, 0, 0, 998, 1083, 1015, 1029, 1, 0, 0, 0, 1080, 1017, 1038, 1005, 5, 0, 0, 0, 1026, 1032, 1019, 976, 6, 1, 0, 0, 1046, 1032, 1026, 1010, 0, 0, 0, 0, 1066, 1018, 1053, 1015, 14, 0, 0, 0, 1064, 1034, 982, 1027, 0, 0, 0, 0, 1005, 1022, 985, 975, 11, 3, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4126, 4145, 4060, 4114, 4166, 4107, 4001] avg=4096.0000 max=4166 max_rank=5 min=4001 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1039, 1059, 989, 1051, 1021, 1050, 992, 1037, 1017, 1004, 994, 1037, 996, 1021, 995, 1061, 990, 993, 1059, 1031, 1024, 1025, 1048, 1028, 1039, 1012, 983, 1024, 1007, 1031, 1034, 1077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[989, 986, 1009, 1053, 1034, 994, 1028, 1034, 1047, 1059, 1053, 1056, 1030, 1041, 1005, 1013, 1034, 1039, 1016, 1041, 1018, 1001, 1044, 997, 1017, 1028, 1016, 1034, 1034, 970, 1033, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1041, 1005, 1010, 970, 1070, 1015, 1034, 1030, 1014, 1009, 952, 1067, 1008, 1018, 1021, 1046, 1047, 1014, 1002, 1004, 1075, 1055, 1002, 1036, 1039, 1032, 1025, 1013, 1059, 1031, 1004, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1019, 1046, 1022, 986, 1010, 1051, 1006, 1044, 1020, 1013, 1019, 1057, 1025, 1029, 1021, 1065, 1027, 1024, 1036, 1032, 1034, 986, 1035, 986, 1024, 1014, 1026, 1024, 1040, 1048, 1000, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1007, 1028, 1013, 1029, 1018, 1026, 1014, 1054, 1078, 986, 970, 1017, 998, 1002, 1000, 1006, 1037, 1042, 1095, 999, 989, 1031, 1088, 1018, 1014, 1087, 1031, 1042, 977, 992, 1015, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1049, 982, 1034, 1023, 1007, 985, 1012, 1039, 1023, 1072, 1008, 1062, 992, 1073, 1004, 984, 1028, 1033, 1090, 1071, 1013, 1077, 1037, 973, 994, 1047, 1019, 1010, 1002, 1039, 971, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1014, 1034, 982, 1003, 1055, 1026, 1035, 1024, 1043, 997, 1011, 1037, 1032, 1010, 1025, 1054, 980, 1069, 999, 1000, 1006, 1024, 1011, 1035, 1061, 1057, 1054, 1015, 1036, 1021, 1016, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1063, 1006, 1033, 999, 986, 1041, 1037, 1017, 1032, 1005, 1065, 1057, 1055, 1036, 1025, 1019, 1029, 1005, 1013, 1004, 1031, 970, 989, 1028, 1033, 1004, 1028, 1033, 1041, 1070, 1041, 973] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1039, 1059, 989, 1051, 14, 9, 0, 0, 1021, 1050, 992, 1031, 0, 0, 0, 0, 1017, 1004, 994, 1024, 0, 0, 0, 0, 996, 1021, 995, 1061, 6, 0, 0, 0, 990, 993, 1045, 1031, 0, 0, 0, 0, 1024, 1025, 1048, 1028, 6, 0, 0, 0, 1039, 1000, 983, 1024, 0, 0, 0, 0, 1007, 1031, 1034, 1077, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4094, 4039, 4079, 4059, 4131, 4046, 4159] avg=4096.0000 max=4161 max_rank=0 min=4039 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1041, 1005, 1010, 970, 13, 10, 0, 0, 1070, 1015, 1034, 1024, 0, 0, 0, 0, 1014, 1009, 952, 1053, 0, 0, 0, 0, 1008, 1018, 1021, 1046, 6, 0, 0, 0, 1047, 1014, 989, 1004, 0, 0, 0, 0, 1075, 1055, 1002, 1036, 6, 0, 0, 0, 1039, 1019, 1025, 1013, 0, 0, 0, 0, 1059, 1031, 1004, 1020, 7, 4, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4143, 4028, 4099, 4054, 4174, 4096, 4125] avg=4096.0000 max=4174 max_rank=5 min=4028 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1007, 1028, 1013, 1029, 15, 9, 0, 0, 1018, 1026, 1014, 1048, 0, 0, 0, 0, 1078, 986, 970, 1004, 0, 0, 0, 0, 998, 1002, 1000, 1006, 7, 0, 0, 0, 1037, 1042, 1080, 999, 0, 0, 0, 0, 989, 1031, 1088, 1018, 6, 0, 0, 0, 1014, 1073, 1031, 1042, 0, 0, 0, 0, 977, 992, 1015, 1065, 7, 4, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4106, 4038, 4013, 4158, 4132, 4160, 4060] avg=4096.0000 max=4160 max_rank=6 min=4013 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[989, 986, 1009, 1053, 14, 10, 0, 0, 1034, 994, 1028, 1028, 0, 0, 0, 0, 1047, 1059, 1053, 1042, 0, 0, 0, 0, 1030, 1041, 1005, 1013, 6, 0, 0, 0, 1034, 1039, 1002, 1041, 0, 0, 0, 0, 1018, 1001, 1044, 997, 6, 0, 0, 0, 1017, 1015, 1016, 1034, 0, 0, 0, 0, 1034, 970, 1033, 1015, 7, 4, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4084, 4201, 4095, 4116, 4066, 4082, 4063] avg=4096.0000 max=4201 max_rank=2 min=4061 min_rank=0 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1019, 1046, 1022, 986, 17, 13, 0, 0, 1010, 1051, 1006, 1028, 0, 0, 0, 0, 1020, 1013, 1019, 1039, 0, 0, 0, 0, 1025, 1029, 1021, 1065, 11, 4, 0, 0, 1027, 1024, 1019, 1032, 0, 0, 0, 0, 1034, 986, 1035, 986, 12, 0, 0, 0, 1024, 993, 1026, 1024, 0, 0, 0, 0, 1040, 1048, 1000, 999, 10, 5, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4095, 4091, 4155, 4102, 4053, 4067, 4102] avg=4096.0000 max=4155 max_rank=3 min=4053 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1049, 982, 1034, 1023, 15, 10, 0, 0, 1007, 985, 1012, 1033, 0, 0, 0, 0, 1023, 1072, 1008, 1048, 0, 0, 0, 0, 992, 1073, 1004, 984, 6, 0, 0, 0, 1028, 1033, 1075, 1071, 0, 0, 0, 0, 1013, 1077, 1037, 973, 6, 0, 0, 0, 994, 1034, 1019, 1010, 0, 0, 0, 0, 1002, 1039, 971, 1015, 7, 4, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4037, 4151, 4059, 4207, 4106, 4057, 4038] avg=4096.0000 max=4207 max_rank=4 min=4037 min_rank=1 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1014, 1034, 982, 1003, 13, 9, 0, 0, 1055, 1026, 1035, 1018, 0, 0, 0, 0, 1043, 997, 1011, 1024, 0, 0, 0, 0, 1032, 1010, 1025, 1054, 6, 0, 0, 0, 980, 1069, 986, 1000, 0, 0, 0, 0, 1006, 1024, 1011, 1035, 6, 0, 0, 0, 1061, 1044, 1054, 1015, 0, 0, 0, 0, 1036, 1021, 1016, 1002, 7, 4, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4134, 4075, 4127, 4035, 4082, 4174, 4086] avg=4096.0000 max=4174 max_rank=6 min=4035 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1063, 1006, 1033, 999, 14, 10, 0, 0, 986, 1041, 1037, 1011, 0, 0, 0, 0, 1032, 1005, 1065, 1043, 0, 0, 0, 0, 1055, 1036, 1025, 1019, 6, 0, 0, 0, 1029, 1005, 999, 1004, 0, 0, 0, 0, 1031, 970, 989, 1028, 6, 0, 0, 0, 1033, 992, 1028, 1033, 0, 0, 0, 0, 1041, 1070, 1041, 973, 6, 4, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4075, 4145, 4141, 4037, 4024, 4086, 4135] avg=4096.0000 max=4145 max_rank=2 min=4024 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1006, 1012, 1044, 997, 1044, 1043, 1001, 1054, 1018, 1014, 1049, 1053, 1070, 995, 1033, 1065, 1028, 973, 968, 1013, 1026, 987, 999, 1066, 1023, 996, 1069, 1010, 1042, 1027, 1067, 976] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1060, 953, 1038, 1005, 1049, 1020, 970, 1057, 1060, 1035, 1029, 1008, 1067, 1014, 1049, 1033, 949, 1066, 1020, 1026, 1066, 1035, 982, 1006, 1002, 1061, 1046, 998, 986, 1055, 1013, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1060, 1028, 998, 985, 1022, 1061, 1038, 1031, 1029, 992, 979, 959, 1032, 1035, 1004, 1011, 1023, 1003, 988, 1042, 988, 1062, 1057, 1024, 1012, 1051, 1068, 1033, 1026, 1040, 1035, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1048, 1018, 1034, 1062, 993, 1025, 1029, 1034, 1035, 1023, 962, 1038, 1032, 980, 1044, 1050, 1029, 1027, 1012, 986, 1071, 1080, 1029, 970, 978, 1033, 1013, 994, 1051, 1065, 993, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[952, 998, 1070, 1023, 989, 1029, 1008, 1053, 1015, 1022, 972, 1044, 1004, 1048, 1003, 1069, 1052, 1003, 1036, 992, 1051, 1014, 1033, 1031, 1085, 1052, 997, 1016, 1012, 1011, 1040, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1004, 1026, 1038, 986, 1075, 1001, 953, 985, 1052, 1052, 980, 988, 1008, 1044, 1038, 1039, 1048, 1061, 1024, 986, 1028, 1023, 1042, 1035, 1040, 1047, 1033, 1041, 1009, 1026, 1016, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1087, 1048, 955, 1008, 1037, 994, 1009, 1053, 1037, 994, 1056, 1026, 1035, 1027, 1040, 1061, 968, 1032, 1018, 980, 1072, 1037, 1024, 1060, 997, 1030, 1041, 1056, 1015, 1022, 988, 961] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1034, 1034, 1056, 1018, 1014, 1035, 1065, 1047, 1043, 988, 1024, 1027, 1073, 1039, 1036, 997, 990, 1029, 1025, 1018, 987, 1039, 999, 1037, 998, 1015, 1027, 996, 1008, 1023, 1052, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1060, 953, 1038, 1005, 8, 1, 0, 0, 1049, 1020, 970, 1052, 0, 0, 0, 0, 1060, 1035, 1029, 1008, 18, 2, 0, 0, 1067, 1014, 1049, 995, 0, 0, 0, 0, 949, 1066, 1020, 1026, 38, 5, 0, 0, 1043, 1035, 982, 1006, 0, 0, 0, 0, 1002, 1061, 1036, 998, 0, 0, 0, 0, 986, 1055, 1013, 1010, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4091, 4152, 4125, 4104, 4066, 4097, 4068] avg=4096.0000 max=4152 max_rank=2 min=4065 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1006, 1012, 1044, 997, 9, 1, 0, 0, 1044, 1043, 1001, 1049, 0, 0, 0, 0, 1018, 1014, 1049, 1053, 18, 2, 0, 0, 1070, 995, 1033, 1026, 0, 0, 0, 0, 1028, 973, 968, 1013, 39, 5, 0, 0, 1003, 987, 999, 1066, 0, 0, 0, 0, 1023, 996, 1058, 1010, 0, 0, 0, 0, 1042, 1027, 1067, 976, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4137, 4154, 4124, 4026, 4055, 4087, 4116] avg=4096.0000 max=4154 max_rank=2 min=4026 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1048, 1018, 1034, 1062, 8, 1, 0, 0, 993, 1025, 1029, 1029, 0, 0, 0, 0, 1035, 1023, 962, 1038, 18, 2, 0, 0, 1032, 980, 1044, 1012, 0, 0, 0, 0, 1029, 1027, 1012, 986, 38, 5, 0, 0, 1048, 1080, 1029, 970, 0, 0, 0, 0, 978, 1033, 1003, 994, 0, 0, 0, 0, 1051, 1065, 993, 1030, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4076, 4078, 4068, 4097, 4127, 4008, 4143] avg=4096.0000 max=4171 max_rank=0 min=4008 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1060, 1028, 998, 985, 9, 1, 0, 0, 1022, 1061, 1038, 1026, 0, 0, 0, 0, 1029, 992, 979, 959, 17, 2, 0, 0, 1032, 1035, 1004, 974, 0, 0, 0, 0, 1023, 1003, 988, 1042, 37, 5, 0, 0, 966, 1062, 1057, 1024, 0, 0, 0, 0, 1012, 1051, 1057, 1033, 0, 0, 0, 0, 1026, 1040, 1035, 1052, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4147, 3978, 4045, 4098, 4109, 4153, 4157] avg=4096.0000 max=4157 max_rank=7 min=3978 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[952, 998, 1070, 1023, 8, 1, 0, 0, 989, 1029, 1008, 1048, 0, 0, 0, 0, 1015, 1022, 972, 1044, 18, 2, 0, 0, 1004, 1048, 1003, 1030, 0, 0, 0, 0, 1052, 1003, 1036, 992, 39, 5, 0, 0, 1028, 1014, 1033, 1031, 0, 0, 0, 0, 1085, 1052, 987, 1016, 0, 0, 0, 0, 1012, 1011, 1040, 1044, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4074, 4073, 4085, 4127, 4106, 4140, 4111] avg=4096.0000 max=4140 max_rank=6 min=4052 min_rank=0 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1004, 1026, 1038, 986, 13, 5, 0, 0, 1075, 1001, 953, 970, 0, 0, 0, 0, 1052, 1052, 980, 988, 22, 5, 0, 0, 1008, 1044, 1038, 998, 0, 0, 0, 0, 1048, 1061, 1024, 986, 41, 11, 0, 0, 995, 1023, 1042, 1035, 0, 0, 0, 0, 1040, 1047, 1015, 1041, 0, 0, 0, 0, 1009, 1026, 1016, 1040, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 3999, 4099, 4088, 4171, 4095, 4143, 4101] avg=4096.0000 max=4171 max_rank=4 min=3999 min_rank=1 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1087, 1048, 955, 1008, 8, 1, 0, 0, 1037, 994, 1009, 1048, 0, 0, 0, 0, 1037, 994, 1056, 1026, 18, 2, 0, 0, 1035, 1027, 1040, 1022, 0, 0, 0, 0, 968, 1032, 1018, 980, 39, 5, 0, 0, 1049, 1037, 1024, 1060, 0, 0, 0, 0, 997, 1030, 1031, 1056, 0, 0, 0, 0, 1015, 1022, 988, 961, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4088, 4133, 4124, 4042, 4170, 4114, 3990] avg=4096.0000 max=4170 max_rank=5 min=3990 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1034, 1034, 1056, 1018, 8, 1, 0, 0, 1014, 1035, 1065, 1042, 0, 0, 0, 0, 1043, 988, 1024, 1027, 17, 2, 0, 0, 1073, 1039, 1036, 961, 0, 0, 0, 0, 990, 1029, 1025, 1018, 36, 5, 0, 0, 965, 1039, 999, 1037, 0, 0, 0, 0, 998, 1015, 1017, 996, 0, 0, 0, 0, 1008, 1023, 1052, 995, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4156, 4101, 4109, 4103, 4040, 4026, 4082] avg=4096.0000 max=4156 max_rank=1 min=4026 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16251 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8064, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 8192, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1076, 986, 1055, 1048, 989, 966, 1017, 1056, 1024, 990, 1056, 1049, 1058, 1026, 1036, 1047, 1022, 997, 1047, 1032, 1066, 964, 1011, 1032, 1014, 988, 1013, 1058, 974, 1028, 1034, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1017, 1037, 1023, 1037, 1088, 1042, 998, 1001, 1024, 1011, 989, 1041, 1019, 1056, 1018, 992, 1048, 1017, 1023, 1040, 996, 968, 1062, 1013, 1053, 1043, 1017, 1014, 994, 992, 1043, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[993, 973, 1007, 966, 1018, 1045, 1027, 1005, 1023, 1032, 1028, 1007, 1011, 1005, 1001, 1042, 1033, 1051, 1035, 1029, 1068, 1038, 1082, 1080, 1041, 988, 1025, 1028, 982, 1047, 1035, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1041, 1009, 1057, 1073, 1013, 1027, 1083, 1039, 979, 996, 965, 1040, 1015, 1033, 988, 986, 1059, 1037, 1041, 1028, 1040, 980, 1035, 1034, 998, 1052, 1016, 1069, 1032, 987, 1017, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1009, 998, 1011, 1026, 1024, 1103, 992, 1074, 1034, 966, 1021, 1043, 947, 1042, 1034, 1026, 1033, 1020, 1025, 993, 1012, 1045, 1019, 1035, 1047, 1051, 1030, 1008, 1047, 1015, 1003, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1002, 993, 969, 1063, 1045, 969, 965, 1038, 1045, 1056, 1033, 1017, 1048, 1041, 1029, 1014, 998, 1002, 1041, 1038, 1037, 1012, 1046, 1019, 1070, 1019, 1084, 1002, 1006, 1033, 960, 1074] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[998, 1071, 1052, 1034, 999, 1026, 1076, 1029, 1049, 1033, 1028, 1011, 999, 981, 1048, 1071, 1050, 1027, 1013, 987, 976, 978, 1018, 1048, 1018, 1010, 996, 1045, 1026, 1012, 1024, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1049, 1013, 990, 1057, 1005, 1043, 975, 1049, 1038, 1039, 1002, 1045, 998, 1025, 994, 1036, 996, 1002, 1007, 1057, 1042, 1026, 1011, 1006, 1062, 1033, 997, 1039, 1035, 1077, 992, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1076, 986, 1055, 1048, 3, 0, 0, 0, 989, 966, 1017, 1050, 0, 0, 0, 0, 1024, 990, 1056, 1049, 6, 0, 0, 0, 1058, 1026, 1036, 1047, 7, 5, 0, 0, 1015, 997, 1047, 1032, 0, 0, 0, 0, 1066, 964, 1008, 1032, 0, 0, 0, 0, 995, 988, 1013, 1058, 0, 0, 0, 0, 974, 1028, 1034, 1009, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4168, 4022, 4125, 4179, 4091, 4070, 4054, 4059] avg=4096.0000 max=4179 max_rank=3 min=4022 min_rank=1 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16274 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1017, 1037, 1023, 1037, 3, 0, 0, 0, 1088, 1042, 998, 995, 0, 0, 0, 0, 1024, 1011, 989, 1041, 6, 0, 0, 0, 1019, 1056, 1018, 992, 7, 5, 0, 0, 1041, 1017, 1023, 1040, 0, 0, 0, 0, 996, 968, 1059, 1013, 0, 0, 0, 0, 1034, 1043, 1017, 1014, 0, 0, 0, 0, 994, 992, 1043, 1052, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4123, 4071, 4097, 4121, 4036, 4108, 4095] avg=4096.0000 max=4123 max_rank=1 min=4036 min_rank=5 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16360 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[993, 973, 1007, 966, 4, 0, 0, 0, 1018, 1045, 1027, 999, 0, 0, 0, 0, 1023, 1032, 1028, 1007, 6, 0, 0, 0, 1011, 1005, 1001, 1042, 7, 5, 0, 0, 1026, 1051, 1035, 1029, 0, 0, 0, 0, 1068, 1038, 1078, 1080, 0, 0, 0, 0, 1022, 988, 1025, 1028, 0, 0, 0, 0, 982, 1047, 1035, 1023, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[3943, 4089, 4096, 4071, 4141, 4264, 4063, 4101] avg=4096.0000 max=4264 max_rank=5 min=3943 min_rank=0 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16569 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1009, 998, 1011, 1026, 3, 0, 0, 0, 1024, 1103, 992, 1068, 0, 0, 0, 0, 1034, 966, 1021, 1043, 6, 0, 0, 0, 947, 1042, 1034, 1026, 7, 5, 0, 0, 1026, 1020, 1025, 993, 0, 0, 0, 0, 1012, 1045, 1016, 1035, 0, 0, 0, 0, 1028, 1051, 1030, 1008, 0, 0, 0, 0, 1047, 1015, 1003, 1035, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4187, 4070, 4061, 4064, 4108, 4117, 4114] avg=4096.0000 max=4187 max_rank=1 min=4047 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16403 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1041, 1009, 1057, 1073, 9, 4, 0, 0, 1013, 1027, 1083, 1023, 0, 0, 0, 0, 979, 996, 965, 1040, 12, 0, 0, 0, 1015, 1033, 988, 986, 11, 7, 0, 0, 1048, 1037, 1041, 1028, 0, 0, 0, 0, 1040, 980, 1026, 1034, 0, 0, 0, 0, 973, 1052, 1016, 1069, 0, 0, 0, 0, 1032, 987, 1017, 999, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4193, 4146, 3992, 4040, 4154, 4080, 4110, 4053] avg=4096.0000 max=4193 max_rank=0 min=3992 min_rank=2 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16397 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1002, 993, 969, 1063, 3, 0, 0, 0, 1045, 969, 965, 1032, 0, 0, 0, 0, 1045, 1056, 1033, 1017, 6, 0, 0, 0, 1048, 1041, 1029, 1014, 7, 5, 0, 0, 991, 1002, 1041, 1038, 0, 0, 0, 0, 1037, 1012, 1043, 1019, 0, 0, 0, 0, 1050, 1019, 1084, 1002, 0, 0, 0, 0, 1006, 1033, 960, 1074, 15, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4011, 4157, 4144, 4072, 4111, 4155, 4088] avg=4096.0000 max=4157 max_rank=2 min=4011 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16426 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[998, 1071, 1052, 1034, 3, 0, 0, 0, 999, 1026, 1076, 1023, 0, 0, 0, 0, 1049, 1033, 1028, 1011, 6, 0, 0, 0, 999, 981, 1048, 1071, 7, 5, 0, 0, 1043, 1027, 1013, 987, 0, 0, 0, 0, 976, 978, 1015, 1048, 0, 0, 0, 0, 999, 1010, 996, 1045, 0, 0, 0, 0, 1026, 1012, 1024, 1035, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4124, 4127, 4111, 4070, 4017, 4050, 4111] avg=4096.0000 max=4158 max_rank=0 min=4017 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16248 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1049, 1013, 990, 1057, 3, 0, 0, 0, 1005, 1043, 975, 1043, 0, 0, 0, 0, 1038, 1039, 1002, 1045, 6, 0, 0, 0, 998, 1025, 994, 1036, 7, 5, 0, 0, 989, 1002, 1007, 1057, 0, 0, 0, 0, 1042, 1026, 1008, 1006, 0, 0, 0, 0, 1042, 1033, 997, 1039, 0, 0, 0, 0, 1035, 1077, 992, 1028, 15, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4066, 4130, 4065, 4055, 4082, 4111, 4147] avg=4096.0000 max=4147 max_rank=7 min=4055 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16395 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1034, 1033, 1008, 999, 1059, 1032, 1031, 999, 1053, 1032, 953, 1007, 1013, 1046, 1048, 1062, 1030, 983, 990, 1036, 1000, 1006, 1029, 1016, 1013, 1096, 1066, 999, 1032, 1014, 1073, 976] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1056, 1074, 1009, 1049, 1014, 975, 1032, 1016, 1010, 962, 1026, 1032, 1059, 1038, 999, 1036, 1030, 1043, 1046, 958, 1048, 1033, 1028, 1021, 992, 1000, 1038, 996, 980, 1093, 1029, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1053, 1037, 1048, 1039, 999, 1033, 1044, 956, 1056, 1004, 1022, 1029, 1035, 1067, 1035, 1061, 949, 1069, 1022, 1019, 1000, 989, 1030, 1087, 989, 980, 1025, 1043, 1013, 1012, 977, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1048, 1002, 1058, 953, 1017, 1005, 997, 1018, 1033, 1011, 1041, 1056, 1033, 1035, 1015, 1026, 1021, 1006, 1012, 1068, 966, 1100, 1010, 1010, 1037, 990, 1064, 1017, 1047, 1058, 1010, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[976, 980, 1023, 1022, 1028, 1024, 1035, 1067, 1048, 1050, 1003, 1054, 1075, 1024, 1047, 1037, 1038, 1036, 989, 1030, 1071, 1037, 1071, 1015, 1034, 1016, 978, 991, 968, 1008, 992, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1022, 1013, 1047, 1045, 1020, 971, 1039, 1042, 1058, 1049, 985, 978, 1011, 1008, 1030, 1023, 1038, 1050, 976, 1060, 1042, 1082, 984, 1050, 1011, 981, 993, 1007, 1008, 1039, 1043, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1018, 1044, 1029, 1021, 1019, 1065, 1011, 1013, 1069, 1075, 1017, 973, 1025, 1010, 1039, 1004, 991, 997, 1015, 989, 1078, 1051, 989, 1037, 1041, 1045, 1067, 1014, 937, 1044, 996, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[983, 1057, 1006, 1053, 1046, 978, 1024, 977, 1068, 1028, 1078, 983, 969, 980, 990, 1058, 1054, 1033, 1034, 965, 1023, 1031, 1052, 997, 990, 1057, 1025, 1031, 1066, 1053, 1018, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1053, 1029, 1048, 1039, 0, 0, 0, 0, 999, 1033, 1044, 956, 20, 2, 0, 0, 1047, 1004, 1022, 1029, 0, 0, 0, 0, 1035, 1067, 1035, 1040, 0, 0, 0, 0, 949, 1069, 1022, 1019, 22, 0, 0, 0, 1000, 965, 1030, 1087, 0, 0, 0, 0, 989, 980, 1025, 1043, 9, 6, 1, 0, 1013, 1012, 977, 1046, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4054, 4102, 4177, 4081, 4082, 4053, 4050] avg=4096.0000 max=4177 max_rank=3 min=4050 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1056, 1065, 1009, 1049, 0, 0, 0, 0, 1014, 975, 1032, 1016, 19, 2, 0, 0, 1001, 962, 1026, 1032, 0, 0, 0, 0, 1059, 1038, 999, 1016, 0, 0, 0, 0, 1030, 1043, 1046, 958, 23, 0, 0, 0, 1048, 1008, 1028, 1021, 0, 0, 0, 0, 992, 1000, 1038, 996, 9, 7, 1, 0, 980, 1093, 1029, 1046, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4058, 4021, 4112, 4100, 4105, 4043, 4150] avg=4096.0000 max=4179 max_rank=0 min=4021 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[976, 973, 1023, 1022, 0, 0, 0, 0, 1028, 1024, 1035, 1067, 19, 2, 0, 0, 1039, 1050, 1003, 1054, 0, 0, 0, 0, 1075, 1024, 1047, 1017, 0, 0, 0, 0, 1038, 1036, 989, 1030, 23, 0, 0, 0, 1071, 1012, 1071, 1015, 0, 0, 0, 0, 1034, 1016, 978, 991, 9, 6, 1, 0, 968, 1008, 992, 1001, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 4175, 4146, 4163, 4116, 4169, 4035, 3970] avg=4096.0000 max=4175 max_rank=1 min=3970 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16290 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1034, 1025, 1008, 999, 0, 0, 0, 0, 1059, 1032, 1031, 999, 20, 2, 0, 0, 1044, 1032, 953, 1007, 0, 0, 0, 0, 1013, 1046, 1048, 1041, 0, 0, 0, 0, 1030, 983, 990, 1036, 23, 0, 0, 0, 1000, 981, 1029, 1016, 0, 0, 0, 0, 1013, 1096, 1066, 999, 9, 6, 1, 0, 1032, 1014, 1073, 976, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4143, 4036, 4148, 4062, 4026, 4190, 4097] avg=4096.0000 max=4190 max_rank=6 min=4026 min_rank=5 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1048, 987, 1058, 953, 0, 0, 0, 0, 1017, 1005, 997, 1018, 22, 8, 0, 0, 1021, 1011, 1041, 1056, 0, 0, 0, 0, 1033, 1035, 1015, 999, 0, 0, 0, 0, 1021, 1006, 1012, 1068, 29, 0, 0, 0, 966, 1063, 1010, 1010, 0, 0, 0, 0, 1037, 990, 1064, 1017, 12, 11, 5, 0, 1047, 1058, 1010, 1004, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4067, 4129, 4082, 4136, 4049, 4136, 4123] avg=4096.0000 max=4136 max_rank=4 min=4046 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1018, 1036, 1029, 1021, 0, 0, 0, 0, 1019, 1065, 1011, 1013, 19, 3, 0, 0, 1060, 1075, 1017, 973, 0, 0, 0, 0, 1025, 1010, 1039, 984, 0, 0, 0, 0, 991, 997, 1015, 989, 24, 0, 0, 0, 1078, 1024, 989, 1037, 0, 0, 0, 0, 1041, 1045, 1067, 1014, 9, 6, 1, 0, 937, 1044, 996, 1045, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4130, 4125, 4058, 4016, 4128, 4183, 4024] avg=4096.0000 max=4183 max_rank=6 min=4016 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1022, 1006, 1047, 1045, 0, 0, 0, 0, 1020, 971, 1039, 1042, 19, 3, 0, 0, 1049, 1049, 985, 978, 0, 0, 0, 0, 1011, 1008, 1030, 1003, 0, 0, 0, 0, 1038, 1050, 976, 1060, 24, 0, 0, 0, 1042, 1055, 984, 1050, 0, 0, 0, 0, 1011, 981, 993, 1007, 9, 6, 1, 0, 1008, 1039, 1043, 1063, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4094, 4061, 4052, 4148, 4131, 4008, 4154] avg=4096.0000 max=4154 max_rank=7 min=4008 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[983, 1048, 1006, 1053, 0, 0, 0, 0, 1046, 978, 1024, 977, 20, 2, 0, 0, 1059, 1028, 1078, 983, 0, 0, 0, 0, 969, 980, 990, 1037, 0, 0, 0, 0, 1054, 1033, 1034, 965, 23, 0, 0, 0, 1023, 1006, 1052, 997, 0, 0, 0, 0, 990, 1057, 1025, 1031, 9, 7, 1, 0, 1066, 1053, 1018, 1061, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4047, 4148, 3976, 4109, 4078, 4120, 4200] avg=4096.0000 max=4200 max_rank=7 min=3976 min_rank=3 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8448, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1036, 1018, 1056, 1039, 995, 1031, 1017, 1003, 1025, 1025, 1078, 1003, 1018, 1029, 1050, 1016, 984, 1023, 1029, 973, 1036, 1028, 1008, 1023, 968, 1029, 1060, 941, 1103, 1030, 1031, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1016, 1051, 1025, 991, 1005, 974, 975, 1036, 1061, 1065, 1012, 1031, 998, 983, 1017, 998, 1023, 1070, 1042, 1014, 1072, 1058, 1042, 1035, 1024, 1029, 1016, 1039, 1002, 1019, 1053, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1042, 989, 1011, 1006, 1026, 1042, 1052, 1036, 1014, 1014, 1042, 1018, 991, 995, 1015, 1046, 1006, 1062, 1006, 1085, 1047, 1019, 1040, 964, 1038, 1032, 988, 1050, 997, 1068, 973, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1009, 1052, 1053, 1020, 1063, 1037, 977, 989, 1005, 1003, 1024, 1054, 1018, 1039, 1033, 1038, 1030, 1020, 1033, 1066, 1064, 1011, 1038, 1039, 988, 990, 1001, 1000, 1033, 1020, 1013, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1007, 1045, 1059, 1051, 995, 973, 1034, 1038, 1009, 1052, 1064, 1004, 1034, 1032, 1046, 1052, 988, 1002, 1027, 966, 1002, 1016, 1037, 1077, 1025, 1038, 1002, 993, 994, 1033, 1056, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1005, 1015, 1047, 1056, 1008, 1003, 1052, 1067, 1013, 1064, 996, 1011, 1038, 1016, 1028, 1014, 1008, 1036, 1030, 1003, 1020, 1018, 1003, 994, 1084, 1046, 1039, 1036, 988, 1000, 1062, 968] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1055, 1049, 1009, 1015, 1027, 992, 998, 1075, 1053, 1013, 996, 1014, 1041, 1034, 1022, 1045, 1026, 1047, 1047, 1055, 975, 1027, 1040, 1008, 1009, 1028, 994, 1012, 1055, 1009, 1035, 963] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1034, 1011, 1015, 1007, 1007, 1005, 1005, 1068, 1030, 1022, 1057, 1033, 1087, 1059, 1062, 999, 1007, 1015, 1012, 1009, 1006, 1026, 991, 1021, 1006, 1029, 1037, 1032, 1020, 1004, 982, 1070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1036, 1018, 1041, 1039, 0, 0, 0, 0, 995, 1031, 1017, 1003, 12, 8, 0, 0, 1025, 1025, 1061, 1003, 0, 0, 0, 0, 1018, 1029, 1036, 1016, 0, 0, 0, 0, 984, 1023, 1029, 973, 2, 0, 0, 0, 1034, 1028, 1008, 1023, 0, 0, 0, 0, 968, 1029, 1060, 941, 17, 3, 0, 0, 1103, 1030, 1031, 1063, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4066, 4114, 4099, 4011, 4093, 4018, 4233] avg=4096.0000 max=4233 max_rank=7 min=4011 min_rank=4 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1016, 1051, 1010, 991, 0, 0, 0, 0, 1005, 974, 975, 1036, 12, 7, 0, 0, 1061, 1065, 996, 1031, 0, 0, 0, 0, 998, 983, 1004, 998, 0, 0, 0, 0, 1023, 1070, 1042, 1014, 2, 0, 0, 0, 1070, 1058, 1042, 1035, 0, 0, 0, 0, 1024, 1029, 1016, 1039, 16, 3, 0, 0, 1002, 1019, 1053, 992, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4009, 4153, 3983, 4151, 4205, 4127, 4072] avg=4096.0000 max=4205 max_rank=5 min=3983 min_rank=3 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16555 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1042, 989, 997, 1006, 0, 0, 0, 0, 1026, 1042, 1052, 1036, 11, 7, 0, 0, 1014, 1014, 1025, 1018, 0, 0, 0, 0, 991, 995, 1002, 1046, 0, 0, 0, 0, 1006, 1062, 1006, 1085, 2, 0, 0, 0, 1045, 1019, 1040, 964, 0, 0, 0, 0, 1038, 1032, 988, 1050, 17, 3, 0, 0, 997, 1068, 973, 1054, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4174, 4071, 4034, 4161, 4068, 4128, 4098] avg=4096.0000 max=4174 max_rank=1 min=4034 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1009, 1052, 1038, 1020, 0, 0, 0, 0, 1063, 1037, 977, 989, 12, 8, 0, 0, 1005, 1003, 1008, 1054, 0, 0, 0, 0, 1018, 1039, 1019, 1038, 0, 0, 0, 0, 1030, 1020, 1033, 1066, 2, 0, 0, 0, 1062, 1011, 1038, 1039, 0, 0, 0, 0, 988, 990, 1001, 1000, 16, 3, 0, 0, 1033, 1020, 1013, 1008, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4086, 4070, 4114, 4151, 4150, 3998, 4080] avg=4096.0000 max=4151 max_rank=4 min=3998 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1007, 1045, 1044, 1051, 0, 0, 0, 0, 995, 973, 1034, 1038, 12, 8, 0, 0, 1009, 1052, 1047, 1004, 0, 0, 0, 0, 1034, 1032, 1032, 1052, 0, 0, 0, 0, 988, 1002, 1027, 966, 2, 0, 0, 0, 1000, 1016, 1037, 1077, 0, 0, 0, 0, 1025, 1038, 1002, 993, 17, 3, 0, 0, 994, 1033, 1056, 1017, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4060, 4112, 4150, 3985, 4130, 4078, 4106] avg=4096.0000 max=4150 max_rank=3 min=3985 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1005, 1015, 1024, 1056, 0, 0, 0, 0, 1008, 1003, 1052, 1067, 16, 11, 0, 0, 1013, 1064, 975, 1011, 0, 0, 0, 0, 1038, 1016, 999, 1014, 0, 0, 0, 0, 1008, 1036, 1030, 1003, 3, 7, 0, 0, 1017, 1018, 1003, 994, 0, 0, 0, 0, 1084, 1046, 1039, 1036, 21, 7, 0, 0, 988, 1000, 1062, 968, 11, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4157, 4063, 4067, 4087, 4032, 4233, 4029] avg=4096.0000 max=4233 max_rank=6 min=4029 min_rank=7 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1055, 1049, 995, 1015, 0, 0, 0, 0, 1027, 992, 998, 1075, 11, 8, 0, 0, 1053, 1013, 980, 1014, 0, 0, 0, 0, 1041, 1034, 1008, 1045, 0, 0, 0, 0, 1026, 1047, 1047, 1055, 2, 0, 0, 0, 973, 1027, 1040, 1008, 0, 0, 0, 0, 1009, 1028, 994, 1012, 16, 3, 0, 0, 1055, 1009, 1035, 963, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4111, 4060, 4128, 4177, 4048, 4062, 4068] avg=4096.0000 max=4177 max_rank=4 min=4048 min_rank=5 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1034, 1011, 1000, 1007, 0, 0, 0, 0, 1007, 1005, 1005, 1068, 12, 8, 0, 0, 1030, 1022, 1040, 1033, 0, 0, 0, 0, 1087, 1059, 1048, 999, 0, 0, 0, 0, 1007, 1015, 1012, 1009, 2, 0, 0, 0, 1004, 1026, 991, 1021, 0, 0, 0, 0, 1006, 1029, 1037, 1032, 17, 3, 0, 0, 1020, 1004, 982, 1070, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4105, 4125, 4193, 4045, 4042, 4124, 4082] avg=4096.0000 max=4193 max_rank=3 min=4042 min_rank=5 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1010, 1043, 1014, 1064, 1023, 1010, 1031, 1035, 1000, 1061, 974, 1027, 1023, 1040, 1062, 979, 1028, 1041, 1020, 1019, 1062, 957, 1023, 1089, 969, 1026, 995, 982, 1068, 1017, 1034, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1012, 978, 967, 1013, 973, 1041, 1020, 1029, 988, 1006, 1062, 1062, 1098, 1007, 1021, 1022, 1022, 966, 1039, 1032, 1027, 1050, 1067, 1009, 1025, 1059, 1066, 976, 1058, 1032, 1017, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1047, 1046, 979, 1035, 1001, 1042, 969, 1026, 1000, 1089, 1016, 1020, 1015, 1008, 1065, 980, 1036, 1024, 1078, 1031, 1022, 1026, 1017, 983, 994, 992, 1032, 1060, 1004, 1087, 1030, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1005, 1011, 1046, 999, 1068, 1006, 1038, 993, 1057, 989, 1051, 1010, 1022, 1056, 1007, 980, 1066, 1037, 1013, 965, 1039, 1025, 1044, 1040, 1028, 1029, 962, 1016, 1021, 1035, 1064, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[980, 994, 1047, 1065, 1017, 1092, 981, 998, 984, 1040, 1011, 1014, 1013, 1048, 1017, 1029, 1022, 1035, 997, 1008, 1030, 1032, 1008, 1037, 1052, 987, 1029, 1047, 1007, 1056, 1022, 1069] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1028, 1075, 989, 1000, 1018, 1025, 1011, 1011, 1020, 974, 1071, 1033, 1026, 1051, 995, 1076, 1062, 968, 1037, 1008, 963, 1015, 1030, 1042, 1015, 1013, 1019, 1044, 1014, 1021, 1055, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1017, 988, 990, 1021, 998, 1062, 1038, 1031, 997, 1005, 1013, 1006, 1008, 977, 1015, 993, 1049, 963, 1056, 1079, 1013, 1092, 1055, 1012, 1035, 1041, 1044, 995, 1037, 1058, 1046, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1044, 1033, 1039, 994, 1027, 1014, 1001, 1106, 1026, 1057, 984, 1003, 1017, 972, 990, 1048, 1020, 1017, 1041, 976, 1064, 1036, 1021, 1020, 1086, 976, 1014, 1002, 1044, 988, 1108, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1047, 1046, 979, 1035, 23, 0, 0, 0, 1001, 1042, 969, 1026, 4, 0, 0, 0, 1000, 1089, 1016, 1020, 11, 3, 0, 0, 1015, 1008, 1065, 980, 13, 0, 0, 0, 1036, 1024, 1078, 1031, 1, 0, 0, 0, 1022, 1026, 996, 983, 0, 0, 0, 0, 994, 992, 1032, 1060, 19, 0, 0, 0, 1004, 1087, 977, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4042, 4139, 4081, 4170, 4027, 4097, 4082] avg=4096.0000 max=4170 max_rank=4 min=4027 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1010, 1043, 1014, 1064, 24, 0, 0, 0, 1023, 1010, 1031, 1035, 4, 0, 0, 0, 1000, 1061, 974, 1027, 11, 3, 0, 0, 1023, 1040, 1062, 979, 13, 0, 0, 0, 1028, 1041, 1020, 1019, 1, 0, 0, 0, 1062, 957, 1002, 1089, 0, 0, 0, 0, 969, 1026, 995, 982, 19, 0, 0, 0, 1068, 1017, 980, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4103, 4076, 4117, 4109, 4110, 3991, 4107] avg=4096.0000 max=4155 max_rank=0 min=3991 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1012, 978, 967, 1013, 23, 0, 0, 0, 973, 1041, 1020, 1029, 4, 0, 0, 0, 988, 1006, 1062, 1062, 10, 3, 0, 0, 1098, 1007, 1021, 1022, 13, 0, 0, 0, 1022, 966, 1039, 1032, 1, 0, 0, 0, 1027, 1050, 1046, 1009, 0, 0, 0, 0, 1025, 1059, 1066, 976, 19, 0, 0, 0, 1058, 1032, 965, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3993, 4067, 4131, 4161, 4060, 4132, 4145, 4079] avg=4096.0000 max=4161 max_rank=3 min=3993 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1005, 1011, 1046, 999, 24, 0, 0, 0, 1068, 1006, 1038, 993, 4, 0, 0, 0, 1057, 989, 1051, 1010, 11, 3, 0, 0, 1022, 1056, 1007, 980, 13, 0, 0, 0, 1066, 1037, 1013, 965, 1, 0, 0, 0, 1039, 1025, 1023, 1040, 0, 0, 0, 0, 1028, 1029, 962, 1016, 20, 0, 0, 0, 1021, 1035, 1009, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4109, 4121, 4078, 4082, 4127, 4055, 4111] avg=4096.0000 max=4127 max_rank=5 min=4055 min_rank=6 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1028, 1075, 989, 1000, 24, 0, 0, 0, 1018, 1025, 1011, 1011, 4, 0, 0, 0, 1020, 974, 1071, 1033, 11, 3, 0, 0, 1026, 1051, 995, 1076, 13, 0, 0, 0, 1062, 968, 1037, 1008, 1, 0, 0, 0, 963, 1015, 1009, 1042, 0, 0, 0, 0, 1015, 1013, 1019, 1044, 19, 0, 0, 0, 1014, 1021, 1001, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4069, 4112, 4161, 4076, 4029, 4110, 4095] avg=4096.0000 max=4161 max_rank=3 min=4029 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[980, 994, 1047, 1065, 23, 0, 0, 0, 1017, 1092, 981, 998, 4, 0, 0, 0, 984, 1040, 1011, 1014, 10, 3, 0, 0, 1013, 1048, 1017, 1029, 13, 0, 0, 0, 1022, 1035, 997, 1008, 1, 0, 0, 0, 1030, 1032, 987, 1037, 0, 0, 0, 0, 1052, 987, 1029, 1047, 19, 0, 0, 0, 1007, 1056, 970, 1069, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4092, 4062, 4120, 4063, 4086, 4134, 4102] avg=4096.0000 max=4134 max_rank=6 min=4062 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1017, 988, 990, 1021, 29, 0, 0, 0, 998, 1062, 1038, 1031, 5, 0, 0, 0, 997, 1005, 1013, 1006, 15, 7, 0, 0, 1008, 977, 1015, 993, 17, 0, 0, 0, 1049, 963, 1056, 1079, 6, 0, 0, 0, 1013, 1092, 1020, 1012, 0, 0, 0, 0, 1035, 1041, 1044, 995, 23, 0, 0, 0, 1037, 1058, 979, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4134, 4043, 4010, 4153, 4137, 4138, 4108] avg=4096.0000 max=4153 max_rank=4 min=4010 min_rank=3 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16536 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1044, 1033, 1039, 994, 25, 0, 0, 0, 1027, 1014, 1001, 1106, 4, 0, 0, 0, 1026, 1057, 984, 1003, 11, 3, 0, 0, 1017, 972, 990, 1048, 13, 0, 0, 0, 1020, 1017, 1041, 976, 1, 0, 0, 0, 1064, 1036, 1000, 1020, 0, 0, 0, 0, 1086, 976, 1014, 1002, 20, 0, 0, 0, 1044, 988, 1052, 1000, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4152, 4084, 4040, 4055, 4120, 4098, 4084] avg=4096.0000 max=4152 max_rank=1 min=4040 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8320, 7936, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1075, 1002, 1046, 978, 1012, 1096, 975, 1027, 984, 991, 1010, 981, 1049, 1046, 1027, 1010, 1008, 1035, 1029, 1005, 1081, 1029, 1027, 1047, 982, 1058, 1015, 1020, 1066, 979, 1014, 1064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1065, 1024, 1015, 997, 1110, 1023, 969, 1122, 1038, 1050, 1003, 1039, 1054, 972, 1030, 986, 1043, 1034, 972, 975, 997, 987, 988, 1044, 993, 1022, 1051, 1045, 1061, 1047, 1007, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[988, 1070, 994, 1019, 1036, 1035, 1011, 1023, 991, 1021, 1104, 1047, 982, 989, 1017, 1040, 1017, 1087, 1015, 1008, 1028, 1012, 1003, 1066, 1056, 1008, 1054, 1006, 1070, 999, 991, 981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1048, 1008, 1001, 1042, 1051, 1003, 1040, 1032, 1024, 975, 976, 1002, 1080, 1022, 989, 1042, 999, 1022, 1043, 991, 1065, 1032, 1029, 1049, 1017, 1048, 998, 1017, 996, 1050, 1063, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1047, 1029, 1078, 1011, 1038, 1013, 1018, 1022, 1016, 1031, 1050, 973, 999, 1061, 1019, 1006, 1013, 964, 1040, 1057, 1048, 1055, 993, 1031, 969, 1009, 1039, 1030, 1034, 1041, 1029, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1033, 1023, 1054, 1009, 1114, 1012, 1007, 1019, 1032, 1059, 1047, 995, 1015, 1001, 1030, 1034, 995, 1059, 1038, 1028, 1040, 977, 960, 1059, 1031, 958, 1032, 1026, 1011, 1054, 1017, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[999, 1013, 1071, 1035, 1034, 1072, 998, 1024, 998, 1046, 1026, 992, 1042, 1013, 1049, 965, 1030, 1043, 1008, 1011, 1020, 1044, 956, 987, 1051, 1026, 1042, 1014, 990, 1059, 1064, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1025, 1028, 1032, 1002, 1087, 1008, 977, 1009, 1019, 1041, 1025, 1009, 1057, 1057, 1056, 1060, 1011, 998, 1026, 979, 1017, 1019, 1088, 993, 969, 1117, 960, 1005, 1036, 989, 1039, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1075, 1002, 1035, 978, 0, 0, 0, 0, 983, 1096, 975, 1027, 0, 0, 0, 0, 984, 991, 1010, 981, 11, 7, 2, 0, 1046, 1046, 1027, 1010, 0, 0, 0, 0, 1008, 1035, 1029, 1005, 22, 0, 0, 0, 1081, 1029, 1027, 1047, 0, 0, 0, 0, 982, 1058, 1015, 1020, 8, 3, 0, 0, 1056, 979, 1014, 1064, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4081, 3986, 4129, 4099, 4184, 4086, 4113] avg=4096.0000 max=4184 max_rank=5 min=3986 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[988, 1070, 983, 1019, 0, 0, 0, 0, 1007, 1035, 1011, 1023, 0, 0, 0, 0, 991, 1021, 1104, 1047, 11, 7, 2, 0, 979, 989, 1017, 1040, 0, 0, 0, 0, 1017, 1087, 1015, 1008, 22, 0, 0, 0, 1028, 1012, 1003, 1066, 0, 0, 0, 0, 1056, 1008, 1054, 1006, 8, 3, 0, 0, 1060, 999, 991, 981, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4076, 4183, 4025, 4149, 4109, 4135, 4031] avg=4096.0000 max=4183 max_rank=2 min=4025 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1048, 1008, 990, 1042, 0, 0, 0, 0, 1022, 1003, 1040, 1032, 0, 0, 0, 0, 1024, 975, 976, 1002, 11, 7, 1, 0, 1076, 1022, 989, 1042, 0, 0, 0, 0, 999, 1022, 1043, 991, 22, 0, 0, 0, 1065, 1032, 1029, 1049, 0, 0, 0, 0, 1017, 1048, 998, 1017, 7, 4, 0, 0, 988, 1050, 1063, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4097, 3996, 4129, 4077, 4175, 4091, 4115] avg=4096.0000 max=4175 max_rank=5 min=3996 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1065, 1024, 1004, 997, 0, 0, 0, 0, 1078, 1023, 969, 1122, 0, 0, 0, 0, 1038, 1050, 1003, 1039, 11, 8, 2, 0, 1051, 972, 1030, 986, 0, 0, 0, 0, 1043, 1034, 972, 975, 24, 0, 0, 0, 997, 987, 988, 1044, 0, 0, 0, 0, 993, 1022, 1051, 1045, 8, 3, 0, 0, 1051, 1047, 1007, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4192, 4151, 4039, 4048, 4016, 4122, 4110] avg=4096.0000 max=4192 max_rank=1 min=4016 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1033, 1023, 1043, 1009, 0, 0, 0, 0, 1082, 1012, 1007, 1019, 0, 0, 0, 0, 1032, 1059, 1047, 995, 11, 8, 1, 0, 1012, 1001, 1030, 1034, 0, 0, 0, 0, 995, 1059, 1038, 1028, 24, 0, 0, 0, 1040, 977, 960, 1059, 0, 0, 0, 0, 1031, 958, 1032, 1026, 8, 3, 0, 0, 1002, 1054, 1017, 999, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4120, 4153, 4077, 4144, 4036, 4058, 4072] avg=4096.0000 max=4153 max_rank=2 min=4036 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[999, 1013, 1059, 1035, 0, 0, 0, 0, 1005, 1072, 998, 1024, 0, 0, 0, 0, 998, 1046, 1026, 992, 12, 7, 1, 0, 1039, 1013, 1049, 965, 0, 0, 0, 0, 1030, 1043, 1008, 1011, 22, 0, 0, 0, 1020, 1044, 956, 987, 0, 0, 0, 0, 1051, 1026, 1042, 1014, 7, 3, 0, 0, 982, 1059, 1064, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4099, 4082, 4066, 4114, 4007, 4143, 4151] avg=4096.0000 max=4151 max_rank=7 min=4007 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1047, 1029, 1063, 1011, 0, 0, 0, 0, 1000, 1013, 1018, 1022, 0, 0, 0, 0, 1016, 1031, 1050, 973, 15, 12, 5, 0, 990, 1061, 1019, 1006, 0, 0, 0, 0, 1013, 964, 1040, 1057, 26, 0, 0, 0, 1045, 1055, 993, 1031, 0, 0, 0, 0, 969, 1009, 1039, 1030, 12, 9, 3, 0, 1017, 1041, 1029, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4053, 4102, 4076, 4100, 4124, 4071, 4092] avg=4096.0000 max=4150 max_rank=0 min=4053 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 1028, 1021, 1002, 0, 0, 0, 0, 1056, 1008, 977, 1009, 0, 0, 0, 0, 1019, 1041, 1025, 1009, 11, 8, 2, 0, 1054, 1057, 1056, 1060, 0, 0, 0, 0, 1011, 998, 1026, 979, 23, 0, 0, 0, 1017, 1019, 1088, 993, 0, 0, 0, 0, 969, 1117, 960, 1005, 8, 3, 0, 0, 1026, 989, 1039, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4050, 4115, 4227, 4037, 4117, 4062, 4084] avg=4096.0000 max=4227 max_rank=3 min=4037 min_rank=4 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8064, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-08-30 04:58:08] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 628.0 | throughput per GPU (TFLOP/s/GPU): 530.9 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.134318E+00 | loss scale: 1.0 | grad norm: 4.564 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1029, 999, 1048, 1008, 995, 1021, 1017, 1003, 1008, 1018, 1084, 1063, 1079, 1020, 1052, 1010, 997, 1020, 1050, 1029, 1034, 1012, 1009, 1053, 994, 988, 1049, 1011, 984, 998, 1028, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1023, 1013, 1008, 1022, 978, 998, 980, 1029, 956, 1048, 1049, 1021, 1029, 1034, 1010, 1022, 1038, 1100, 1023, 1028, 1063, 1039, 1013, 1023, 1034, 1004, 1039, 1045, 1071, 1024, 1002, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1028, 999, 1016, 1018, 1046, 1023, 1030, 992, 1062, 1002, 1035, 1020, 1017, 1072, 1031, 992, 978, 1040, 1024, 1031, 1082, 1005, 1084, 1030, 1010, 1015, 951, 1009, 1029, 1031, 1024, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1026, 1044, 1042, 1032, 1026, 1049, 1022, 1046, 1001, 1007, 1029, 1017, 1041, 979, 992, 1083, 1019, 986, 983, 1018, 1060, 1020, 1021, 1034, 1013, 1049, 1031, 1033, 1016, 1021, 1057, 971] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1011, 1015, 1000, 1001, 997, 1045, 1025, 1022, 1016, 1050, 1028, 1028, 1060, 1001, 1060, 976, 1050, 1039, 1012, 1028, 1026, 1021, 1014, 1044, 1004, 1078, 995, 999, 1028, 1033, 1030, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[998, 1018, 1003, 1034, 1019, 1056, 1046, 1035, 1001, 1002, 1029, 1020, 1027, 1023, 1024, 1024, 1052, 1030, 1017, 1073, 1024, 1055, 949, 1021, 984, 1086, 1038, 1014, 1037, 1020, 999, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1045, 1036, 1025, 988, 985, 1044, 1011, 1014, 1046, 997, 1064, 989, 1016, 1015, 1075, 992, 972, 1022, 1057, 1019, 1069, 1036, 994, 1027, 1038, 1048, 1028, 1007, 1044, 1063, 965, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1010, 1000, 1053, 1022, 1051, 1036, 1059, 988, 1064, 1002, 1023, 1022, 1033, 1041, 1074, 1047, 1023, 1012, 1000, 1038, 1026, 1030, 1045, 1005, 995, 995, 996, 976, 1062, 1026, 1004, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1029, 999, 1048, 1008, 19, 0, 0, 0, 995, 1021, 1017, 1003, 4, 2, 2, 0, 1008, 1018, 1081, 1063, 0, 0, 0, 0, 1079, 1020, 1030, 1010, 0, 0, 0, 0, 997, 1020, 1050, 1025, 0, 0, 0, 0, 1010, 1012, 1009, 1053, 0, 0, 0, 0, 994, 988, 1049, 1011, 24, 1, 0, 0, 984, 998, 1028, 1058, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4044, 4170, 4139, 4092, 4084, 4067, 4069] avg=4096.0000 max=4170 max_rank=2 min=4044 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1026, 1044, 1042, 1032, 18, 0, 0, 0, 1026, 1049, 1022, 1046, 4, 2, 2, 0, 1001, 1007, 1026, 1017, 0, 0, 0, 0, 1041, 979, 971, 1083, 0, 0, 0, 0, 1019, 986, 983, 1014, 0, 0, 0, 0, 1035, 1020, 1021, 1034, 0, 0, 0, 0, 1013, 1049, 1031, 1033, 25, 1, 0, 0, 1016, 1021, 1057, 971, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4151, 4051, 4074, 4002, 4110, 4152, 4066] avg=4096.0000 max=4162 max_rank=0 min=4002 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1028, 999, 1016, 1018, 19, 0, 0, 0, 1046, 1023, 1030, 992, 4, 2, 2, 0, 1062, 1002, 1032, 1020, 0, 0, 0, 0, 1017, 1072, 1009, 992, 0, 0, 0, 0, 978, 1040, 1024, 1027, 0, 0, 0, 0, 1057, 1005, 1084, 1030, 0, 0, 0, 0, 1010, 1015, 951, 1009, 25, 1, 0, 0, 1029, 1031, 1024, 1042, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4099, 4116, 4090, 4069, 4176, 4011, 4127] avg=4096.0000 max=4176 max_rank=5 min=4011 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1023, 1013, 1008, 1022, 18, 0, 0, 0, 978, 998, 980, 1029, 4, 2, 2, 0, 956, 1048, 1046, 1021, 0, 0, 0, 0, 1029, 1034, 989, 1022, 0, 0, 0, 0, 1038, 1100, 1023, 1024, 0, 0, 0, 0, 1038, 1039, 1013, 1023, 0, 0, 0, 0, 1034, 1004, 1039, 1045, 25, 1, 0, 0, 1071, 1024, 1002, 1002, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 3993, 4071, 4074, 4185, 4113, 4148, 4100] avg=4096.0000 max=4185 max_rank=4 min=3993 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16546 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[998, 1018, 1003, 1034, 18, 0, 0, 0, 1019, 1056, 1046, 1035, 5, 2, 2, 0, 1001, 1002, 1026, 1020, 0, 0, 0, 0, 1027, 1023, 1003, 1024, 0, 0, 0, 0, 1052, 1030, 1017, 1068, 0, 0, 0, 0, 1000, 1055, 949, 1021, 0, 0, 0, 0, 984, 1086, 1038, 1014, 24, 1, 0, 0, 1037, 1020, 999, 1010, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4165, 4049, 4077, 4167, 4025, 4147, 4067] avg=4096.0000 max=4167 max_rank=4 min=4025 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1011, 1015, 1000, 1001, 24, 0, 0, 0, 997, 1045, 1025, 1022, 10, 9, 3, 0, 1016, 1050, 1016, 1028, 0, 0, 0, 0, 1060, 1001, 1028, 976, 0, 0, 0, 0, 1050, 1039, 1012, 1018, 0, 0, 0, 0, 998, 1021, 1014, 1044, 0, 0, 0, 0, 1004, 1078, 995, 999, 28, 5, 0, 0, 1028, 1033, 1030, 1032, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4111, 4110, 4065, 4119, 4077, 4109, 4126] avg=4096.0000 max=4126 max_rank=7 min=4051 min_rank=0 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1045, 1036, 1025, 988, 19, 0, 0, 0, 985, 1044, 1011, 1014, 4, 2, 2, 0, 1046, 997, 1061, 989, 0, 0, 0, 0, 1016, 1015, 1053, 992, 0, 0, 0, 0, 972, 1022, 1057, 1015, 0, 0, 0, 0, 1044, 1036, 994, 1027, 0, 0, 0, 0, 1038, 1048, 1028, 1007, 25, 1, 0, 0, 1044, 1063, 965, 1037, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4062, 4093, 4076, 4066, 4101, 4147, 4110] avg=4096.0000 max=4147 max_rank=6 min=4062 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1010, 1000, 1053, 1022, 19, 0, 0, 0, 1051, 1036, 1059, 988, 5, 2, 2, 0, 1064, 1002, 1020, 1022, 0, 0, 0, 0, 1033, 1041, 1052, 1047, 0, 0, 0, 0, 1023, 1012, 1000, 1033, 0, 0, 0, 0, 1002, 1030, 1045, 1005, 0, 0, 0, 0, 995, 995, 996, 976, 24, 1, 0, 0, 1062, 1026, 1004, 1010, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4143, 4108, 4173, 4068, 4082, 3987, 4103] avg=4096.0000 max=4173 max_rank=3 min=3987 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1046, 1025, 1093, 1059, 1004, 1009, 1040, 1045, 1036, 1085, 1012, 1000, 1039, 975, 1003, 1020, 981, 1024, 985, 994, 1028, 1040, 1023, 984, 1034, 1025, 1013, 1045, 1048, 998, 1037, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[973, 1056, 1024, 1067, 1010, 1056, 1001, 995, 1071, 1074, 1030, 998, 973, 1010, 967, 1029, 1051, 1047, 1005, 1017, 1069, 1001, 1044, 1041, 1010, 1054, 1063, 999, 999, 1058, 996, 980] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1001, 1038, 1029, 1053, 1056, 1009, 1038, 1028, 1004, 1015, 1010, 972, 1056, 967, 1022, 1016, 1070, 973, 1018, 1009, 1104, 1078, 1023, 1002, 1008, 1046, 1020, 1020, 950, 1016, 1046, 1071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1029, 1071, 1067, 1036, 999, 1008, 1052, 993, 1046, 995, 1071, 978, 1053, 1016, 991, 1022, 1034, 1031, 980, 1049, 1015, 1037, 983, 1052, 1054, 949, 1022, 1042, 1045, 995, 1043, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1057, 1013, 1008, 1040, 1024, 1047, 1007, 1018, 1007, 1007, 1021, 1030, 996, 1030, 1008, 1033, 1053, 1022, 1005, 1002, 974, 991, 1053, 1049, 1059, 1010, 1055, 1040, 1032, 1065, 994, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1022, 1041, 1001, 1060, 1032, 1011, 982, 1049, 1034, 1005, 1063, 1004, 1066, 1084, 1014, 1041, 1008, 1030, 1004, 1011, 1029, 987, 1023, 1023, 1030, 1009, 1016, 1003, 1027, 995, 1045, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1039, 1023, 1023, 1038, 983, 1023, 989, 1062, 1035, 1041, 1023, 1039, 1025, 1044, 1032, 1023, 1050, 1039, 1056, 1011, 999, 1014, 952, 1028, 1025, 991, 1001, 1043, 1054, 1007, 1036, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1020, 1065, 1022, 1005, 1036, 1044, 1024, 1032, 1031, 990, 1050, 1033, 999, 1024, 1050, 1040, 1035, 955, 1013, 1064, 999, 1035, 1034, 1006, 1044, 1003, 1023, 1047, 1010, 1006, 1031, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1001, 1038, 1029, 1008, 0, 0, 0, 0, 1056, 1009, 1038, 1028, 4, 3, 0, 0, 1004, 1015, 1006, 972, 0, 0, 0, 0, 1056, 967, 1022, 1016, 12, 0, 0, 0, 1070, 973, 1018, 1009, 17, 0, 0, 0, 1104, 1078, 1023, 1002, 4, 1, 0, 0, 1004, 1046, 1020, 1020, 0, 0, 0, 0, 950, 1016, 1046, 1071, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4138, 3997, 4073, 4087, 4212, 4090, 4095] avg=4096.0000 max=4212 max_rank=5 min=3997 min_rank=2 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[973, 1056, 1024, 1021, 0, 0, 0, 0, 1010, 1056, 1001, 995, 4, 3, 0, 0, 1071, 1074, 1026, 998, 0, 0, 0, 0, 973, 1010, 967, 1029, 12, 0, 0, 0, 1051, 1047, 1005, 1017, 18, 0, 0, 0, 1069, 1001, 1044, 1041, 4, 1, 0, 0, 1006, 1054, 1063, 999, 0, 0, 0, 0, 999, 1058, 996, 980, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4069, 4169, 3991, 4138, 4160, 4122, 4045] avg=4096.0000 max=4169 max_rank=2 min=3991 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1057, 1013, 1008, 995, 0, 0, 0, 0, 1024, 1047, 1007, 1018, 4, 3, 0, 0, 1007, 1007, 1017, 1030, 0, 0, 0, 0, 996, 1030, 1008, 1033, 12, 0, 0, 0, 1053, 1022, 1005, 1002, 17, 0, 0, 0, 974, 991, 1053, 1049, 4, 1, 0, 0, 1055, 1010, 1055, 1040, 0, 0, 0, 0, 1032, 1065, 994, 1018, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4103, 4061, 4079, 4099, 4072, 4160, 4121] avg=4096.0000 max=4160 max_rank=6 min=4061 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1029, 1071, 1067, 991, 0, 0, 0, 0, 999, 1008, 1052, 993, 4, 3, 0, 0, 1046, 995, 1067, 978, 0, 0, 0, 0, 1053, 1016, 991, 1022, 12, 0, 0, 0, 1034, 1031, 980, 1049, 17, 0, 0, 0, 1015, 1037, 983, 1052, 4, 1, 0, 0, 1050, 949, 1022, 1042, 0, 0, 0, 0, 1045, 995, 1043, 1010, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4059, 4086, 4094, 4111, 4092, 4063, 4105] avg=4096.0000 max=4158 max_rank=0 min=4059 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1046, 1025, 1093, 1014, 0, 0, 0, 0, 1004, 1009, 1040, 1045, 4, 3, 0, 0, 1036, 1085, 1008, 1000, 0, 0, 0, 0, 1039, 975, 1003, 1020, 12, 0, 0, 0, 981, 1024, 985, 994, 17, 0, 0, 0, 1028, 1040, 1023, 984, 4, 1, 0, 0, 1030, 1025, 1013, 1045, 0, 0, 0, 0, 1048, 998, 1037, 1018, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4105, 4129, 4049, 4001, 4080, 4113, 4113] avg=4096.0000 max=4178 max_rank=0 min=4001 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1022, 1041, 1001, 999, 0, 0, 0, 0, 1032, 1011, 982, 1049, 6, 8, 0, 0, 1034, 1005, 1049, 1004, 0, 0, 0, 0, 1066, 1084, 1014, 1041, 16, 0, 0, 0, 1008, 1030, 1004, 1011, 22, 0, 0, 0, 1029, 987, 1023, 1023, 7, 6, 0, 0, 1023, 1009, 1016, 1003, 0, 0, 0, 0, 1027, 995, 1045, 1019, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4088, 4092, 4221, 4075, 4075, 4051, 4103] avg=4096.0000 max=4221 max_rank=3 min=4051 min_rank=6 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1020, 1065, 1022, 961, 0, 0, 0, 0, 1036, 1044, 1024, 1032, 3, 3, 0, 0, 1031, 990, 1046, 1033, 0, 0, 0, 0, 999, 1024, 1050, 1040, 12, 0, 0, 0, 1035, 955, 1013, 1064, 17, 0, 0, 0, 999, 1035, 1034, 1006, 4, 1, 0, 0, 1040, 1003, 1023, 1047, 0, 0, 0, 0, 1010, 1006, 1031, 998, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4142, 4100, 4125, 4084, 4079, 4113, 4057] avg=4096.0000 max=4142 max_rank=1 min=4057 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1039, 1023, 1023, 993, 0, 0, 0, 0, 983, 1023, 989, 1062, 4, 3, 0, 0, 1035, 1041, 1019, 1039, 0, 0, 0, 0, 1025, 1044, 1032, 1023, 12, 0, 0, 0, 1050, 1039, 1056, 1011, 17, 0, 0, 0, 999, 1014, 952, 1028, 4, 1, 0, 0, 1021, 991, 1001, 1043, 0, 0, 0, 0, 1054, 1007, 1036, 1020, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4064, 4134, 4136, 4173, 3998, 4056, 4129] avg=4096.0000 max=4173 max_rank=4 min=3998 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8448, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[960, 1027, 1083, 1025, 1006, 1002, 982, 1094, 1045, 1045, 1041, 1014, 986, 1018, 1014, 1029, 983, 1048, 994, 1038, 1009, 1024, 1069, 1000, 997, 1046, 1034, 1046, 1016, 1044, 1011, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1038, 1023, 1021, 1019, 1015, 996, 1024, 1021, 1080, 1066, 1055, 983, 992, 1015, 1030, 1018, 980, 1023, 985, 1013, 1061, 1042, 1006, 1047, 1017, 1048, 993, 1070, 1025, 1032, 942, 1088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[990, 1046, 1043, 1013, 995, 1057, 1054, 1003, 984, 989, 1039, 1037, 1035, 968, 1059, 1049, 1029, 997, 1047, 1047, 1026, 1008, 1018, 1006, 1047, 985, 1020, 1052, 1001, 1028, 1032, 1064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1045, 1006, 1053, 1049, 973, 1043, 1007, 1084, 1048, 1039, 1022, 992, 942, 1045, 1066, 1037, 1013, 1012, 1024, 1030, 982, 1011, 984, 1059, 1036, 978, 1039, 1058, 1042, 1029, 987, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1081, 993, 1003, 1065, 1044, 1069, 955, 1005, 979, 1033, 1017, 1039, 1051, 999, 970, 1010, 1021, 1019, 1084, 1046, 1079, 996, 986, 1032, 1015, 996, 1053, 991, 1029, 1031, 1035, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1037, 973, 996, 1042, 1012, 1047, 1008, 1000, 1011, 1049, 1044, 1015, 1051, 998, 1062, 1050, 995, 1030, 1022, 1000, 996, 1053, 1002, 1014, 1020, 1060, 1064, 1021, 1059, 1009, 1000, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1038, 1003, 1032, 1036, 1023, 1049, 1003, 1038, 1036, 958, 1030, 1032, 983, 1071, 1006, 1032, 991, 1007, 1003, 1029, 1030, 1008, 1028, 1057, 1025, 1039, 1076, 1007, 1026, 1035, 1004, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1010, 992, 1037, 1054, 1017, 1050, 960, 1075, 1014, 1009, 1045, 1039, 1024, 1032, 1045, 1011, 1005, 1030, 1021, 1046, 977, 1016, 974, 1051, 1045, 1066, 991, 1044, 1035, 1005, 977, 1071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[960, 1027, 1083, 1018, 0, 0, 0, 0, 1006, 1002, 982, 1094, 7, 0, 0, 0, 1045, 1045, 1034, 1014, 0, 0, 0, 0, 986, 1018, 1014, 1029, 7, 0, 0, 0, 983, 1048, 994, 1038, 19, 0, 0, 0, 1009, 1024, 1069, 1000, 7, 6, 0, 0, 997, 1046, 1034, 1021, 0, 0, 0, 0, 1016, 1044, 1011, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4091, 4138, 4054, 4082, 4115, 4098, 4102] avg=4096.0000 max=4138 max_rank=2 min=4054 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1045, 1006, 1053, 1042, 0, 0, 0, 0, 973, 1043, 1007, 1084, 7, 0, 0, 0, 1048, 1039, 1015, 992, 0, 0, 0, 0, 942, 1045, 1066, 1037, 7, 0, 0, 0, 1013, 1012, 1024, 1030, 19, 0, 0, 0, 982, 1011, 984, 1059, 7, 7, 0, 0, 1036, 978, 1039, 1032, 0, 0, 0, 0, 1042, 1029, 987, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4114, 4094, 4097, 4098, 4050, 4085, 4084] avg=4096.0000 max=4146 max_rank=0 min=4050 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[990, 1046, 1043, 1006, 0, 0, 0, 0, 995, 1057, 1054, 1003, 7, 0, 0, 0, 984, 989, 1032, 1037, 0, 0, 0, 0, 1035, 968, 1059, 1049, 7, 0, 0, 0, 1029, 997, 1047, 1047, 19, 0, 0, 0, 1026, 1008, 1018, 1006, 7, 6, 0, 0, 1047, 985, 1020, 1027, 0, 0, 0, 0, 1001, 1028, 1032, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4116, 4042, 4118, 4139, 4071, 4079, 4118] avg=4096.0000 max=4139 max_rank=4 min=4042 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1038, 1023, 1021, 1012, 0, 0, 0, 0, 1015, 996, 1024, 1021, 7, 0, 0, 0, 1080, 1066, 1048, 983, 0, 0, 0, 0, 992, 1015, 1030, 1018, 8, 0, 0, 0, 980, 1023, 985, 1013, 20, 0, 0, 0, 1061, 1042, 1006, 1047, 7, 7, 0, 0, 1017, 1048, 993, 1043, 0, 0, 0, 0, 1025, 1032, 942, 1080, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4063, 4177, 4063, 4021, 4170, 4101, 4079] avg=4096.0000 max=4177 max_rank=2 min=4021 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1037, 973, 996, 1035, 0, 0, 0, 0, 1012, 1047, 1008, 1000, 7, 0, 0, 0, 1011, 1049, 1037, 1015, 0, 0, 0, 0, 1051, 998, 1062, 1050, 7, 0, 0, 0, 995, 1030, 1022, 1000, 19, 0, 0, 0, 996, 1053, 1002, 1014, 7, 6, 0, 0, 1020, 1060, 1064, 996, 0, 0, 0, 0, 1059, 1009, 1000, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4074, 4112, 4168, 4066, 4078, 4140, 4089] avg=4096.0000 max=4168 max_rank=3 min=4041 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1038, 1003, 1032, 1029, 0, 0, 0, 0, 1023, 1049, 1003, 1038, 7, 0, 0, 0, 1036, 958, 1023, 1032, 0, 0, 0, 0, 983, 1071, 1006, 1032, 7, 0, 0, 0, 991, 1007, 1003, 1029, 18, 0, 0, 0, 1030, 1008, 1028, 1057, 7, 6, 0, 0, 1025, 1039, 1076, 983, 0, 0, 0, 0, 1026, 1035, 1004, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4120, 4049, 4099, 4048, 4136, 4123, 4091] avg=4096.0000 max=4136 max_rank=5 min=4048 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1081, 993, 1003, 1049, 0, 0, 0, 0, 1044, 1069, 955, 1005, 8, 0, 0, 0, 979, 1033, 1005, 1039, 0, 0, 0, 0, 1051, 999, 970, 1010, 12, 4, 3, 0, 1021, 1019, 1084, 1046, 23, 0, 0, 0, 1079, 996, 986, 1032, 13, 11, 0, 0, 1015, 996, 1053, 957, 0, 0, 0, 0, 1029, 1031, 1035, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4081, 4056, 4049, 4193, 4117, 4021, 4125] avg=4096.0000 max=4193 max_rank=4 min=4021 min_rank=6 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1010, 992, 1037, 1047, 0, 0, 0, 0, 1017, 1050, 960, 1075, 7, 0, 0, 0, 1014, 1009, 1038, 1039, 0, 0, 0, 0, 1024, 1032, 1045, 1011, 8, 0, 0, 0, 1005, 1030, 1021, 1046, 19, 0, 0, 0, 977, 1016, 974, 1051, 7, 6, 0, 0, 1045, 1066, 991, 1019, 0, 0, 0, 0, 1035, 1005, 977, 1063, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4109, 4100, 4120, 4121, 4031, 4121, 4080] avg=4096.0000 max=4121 max_rank=4 min=4031 min_rank=5 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8064, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[990, 985, 1035, 984, 1021, 1025, 1038, 1041, 997, 1013, 1050, 1017, 1026, 1046, 1056, 1026, 1000, 1042, 1022, 1053, 999, 1014, 1046, 1008, 1018, 1018, 1017, 1002, 1050, 1017, 1045, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1039, 1015, 1044, 998, 1050, 1009, 1001, 1006, 1028, 1010, 1034, 1032, 1004, 997, 1013, 1013, 1088, 1005, 1060, 976, 1011, 1071, 1024, 1032, 989, 1084, 1035, 985, 1020, 1015, 1055, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1026, 1000, 1014, 949, 1043, 1028, 1014, 1030, 1053, 1029, 1048, 1002, 1050, 993, 1058, 1010, 1016, 1040, 1035, 995, 1054, 1003, 1031, 991, 1075, 1069, 1034, 1041, 1031, 1033, 986, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1025, 1042, 1040, 1062, 1023, 1003, 1000, 999, 1085, 1026, 1006, 1062, 1034, 984, 1022, 1011, 1055, 1033, 1011, 971, 1008, 1015, 1005, 1035, 1019, 1019, 1025, 1033, 1058, 1037, 1022, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1046, 1000, 998, 998, 1042, 986, 1054, 952, 1027, 980, 1037, 1045, 1014, 1040, 1037, 1046, 1029, 987, 987, 1041, 1033, 1062, 1008, 1030, 1059, 1062, 1036, 1033, 1003, 1022, 1026, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1051, 1009, 1019, 1010, 987, 961, 1044, 965, 1027, 1024, 992, 1055, 1056, 1079, 1041, 1018, 1051, 1046, 1013, 1046, 1009, 1020, 1053, 988, 990, 1028, 1023, 1033, 980, 1044, 1011, 1095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1002, 1031, 1079, 1002, 1021, 984, 996, 1001, 1000, 1053, 1003, 1011, 1035, 1010, 1086, 1007, 1072, 1013, 1034, 1036, 1026, 1019, 1055, 1053, 1008, 997, 1021, 1011, 1056, 1026, 987, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1025, 987, 1007, 1001, 1052, 1030, 1095, 1006, 1019, 1036, 1052, 1020, 1020, 970, 1032, 1016, 1021, 1029, 977, 1063, 1021, 1019, 980, 1083, 1021, 1025, 1031, 1023, 1018, 987, 1065, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[990, 985, 1035, 984, 10, 9, 6, 4, 1021, 1025, 1038, 1041, 14, 12, 4, 0, 997, 1013, 1050, 1005, 0, 0, 0, 0, 1026, 1046, 1046, 1026, 0, 0, 0, 0, 991, 1042, 1022, 1053, 0, 0, 0, 0, 999, 1010, 1046, 1008, 0, 0, 0, 0, 1018, 1008, 1017, 1002, 0, 0, 0, 0, 1050, 1017, 1045, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4155, 4065, 4144, 4108, 4063, 4045, 4165] avg=4096.0000 max=4165 max_rank=7 min=4023 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1025, 1042, 1040, 1062, 10, 10, 6, 4, 1023, 1003, 1000, 999, 13, 13, 4, 0, 1085, 1026, 1006, 1049, 0, 0, 0, 0, 1034, 984, 1012, 1011, 0, 0, 0, 0, 1045, 1033, 1011, 971, 0, 0, 0, 0, 1008, 1011, 1005, 1035, 0, 0, 0, 0, 1019, 1009, 1025, 1033, 0, 0, 0, 0, 1058, 1037, 1022, 985, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4199, 4055, 4166, 4041, 4060, 4059, 4086, 4102] avg=4096.0000 max=4199 max_rank=0 min=4041 min_rank=3 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1026, 1000, 1014, 949, 10, 9, 7, 4, 1043, 1028, 1014, 1030, 13, 12, 5, 0, 1053, 1029, 1048, 990, 0, 0, 0, 0, 1050, 993, 1048, 1010, 0, 0, 0, 0, 1007, 1040, 1035, 995, 0, 0, 0, 0, 1054, 999, 1031, 991, 0, 0, 0, 0, 1075, 1057, 1034, 1041, 0, 0, 0, 0, 1031, 1033, 986, 974, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4145, 4120, 4101, 4077, 4075, 4207, 4024] avg=4096.0000 max=4207 max_rank=6 min=4019 min_rank=0 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1046, 1000, 998, 998, 10, 9, 7, 4, 1042, 986, 1054, 952, 14, 13, 5, 0, 1027, 980, 1037, 1032, 0, 0, 0, 0, 1014, 1040, 1027, 1046, 0, 0, 0, 0, 1020, 987, 987, 1041, 0, 0, 0, 0, 1033, 1058, 1008, 1030, 0, 0, 0, 0, 1059, 1050, 1036, 1033, 0, 0, 0, 0, 1003, 1022, 1026, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4066, 4076, 4127, 4035, 4129, 4178, 4085] avg=4096.0000 max=4178 max_rank=6 min=4035 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1039, 1015, 1044, 998, 9, 10, 7, 4, 1050, 1009, 1001, 1006, 14, 13, 5, 0, 1028, 1010, 1034, 1019, 0, 0, 0, 0, 1004, 997, 1004, 1013, 0, 0, 0, 0, 1078, 1005, 1060, 976, 0, 0, 0, 0, 1011, 1067, 1024, 1032, 0, 0, 0, 0, 989, 1072, 1035, 985, 0, 0, 0, 0, 1020, 1015, 1055, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4098, 4091, 4018, 4119, 4134, 4081, 4101] avg=4096.0000 max=4134 max_rank=5 min=4018 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1002, 1031, 1079, 1002, 10, 10, 6, 4, 1021, 984, 996, 1001, 14, 12, 4, 0, 1000, 1053, 1003, 999, 0, 0, 0, 0, 1035, 1010, 1076, 1007, 0, 0, 0, 0, 1062, 1013, 1034, 1036, 0, 0, 0, 0, 1026, 1015, 1055, 1053, 0, 0, 0, 0, 1008, 987, 1021, 1011, 0, 0, 0, 0, 1056, 1026, 987, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4032, 4055, 4128, 4145, 4149, 4027, 4088] avg=4096.0000 max=4149 max_rank=5 min=4027 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1051, 1009, 1019, 1010, 13, 13, 11, 10, 987, 961, 1044, 965, 20, 18, 9, 0, 1027, 1024, 992, 1037, 0, 0, 0, 0, 1056, 1079, 1028, 1018, 0, 0, 0, 0, 1038, 1046, 1013, 1046, 0, 0, 0, 0, 1009, 1010, 1053, 988, 0, 0, 0, 0, 990, 1008, 1023, 1033, 0, 0, 0, 0, 980, 1044, 1011, 1075, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4004, 4080, 4181, 4143, 4060, 4054, 4110] avg=4096.0000 max=4181 max_rank=3 min=4004 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1025, 987, 1007, 1001, 10, 9, 6, 4, 1052, 1030, 1095, 1006, 14, 12, 4, 0, 1019, 1036, 1052, 1008, 0, 0, 0, 0, 1020, 970, 1022, 1016, 0, 0, 0, 0, 1012, 1029, 977, 1063, 0, 0, 0, 0, 1021, 1015, 980, 1083, 0, 0, 0, 0, 1021, 1015, 1031, 1023, 0, 0, 0, 0, 1018, 987, 1065, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4213, 4115, 4028, 4081, 4099, 4090, 4093] avg=4096.0000 max=4213 max_rank=1 min=4028 min_rank=3 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8064, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8064, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1090, 1050, 1011, 1039, 1014, 1043, 999, 1028, 1063, 1019, 997, 978, 985, 1016, 1056, 1029, 1014, 1026, 995, 1000, 1013, 1031, 1045, 1033, 1051, 1022, 1078, 1001, 1017, 1018, 1008, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1044, 1054, 1026, 1063, 1007, 1028, 1015, 1015, 1076, 997, 988, 1020, 973, 1053, 1042, 1028, 1049, 1070, 1076, 986, 1034, 1003, 1022, 1039, 1052, 1018, 1049, 985, 961, 1005, 1003, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1052, 1017, 1054, 1093, 1023, 1008, 1016, 1023, 1045, 974, 1000, 1016, 1003, 1033, 1012, 1060, 1036, 1015, 1016, 979, 1036, 1069, 1046, 1011, 1034, 1035, 980, 1039, 1027, 998, 1009, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1021, 1004, 1058, 987, 1067, 1005, 1014, 1011, 1050, 1063, 1029, 1002, 1027, 997, 999, 1048, 1036, 1036, 995, 1030, 1087, 1024, 999, 1062, 1047, 986, 992, 1041, 990, 1037, 1004, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[995, 1013, 1076, 1002, 1024, 1001, 1030, 1004, 1008, 1020, 1031, 1014, 1051, 1052, 1019, 1020, 985, 997, 1047, 992, 1012, 1027, 1000, 1004, 1057, 1012, 1083, 1029, 1035, 1048, 1070, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1031, 1042, 1067, 1016, 996, 1007, 991, 989, 1041, 1002, 975, 1015, 1047, 1005, 1034, 1039, 1049, 1034, 1043, 1039, 980, 1023, 996, 1036, 1068, 988, 1002, 1062, 1109, 977, 1018, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1005, 1032, 1072, 1056, 1031, 1008, 1009, 1026, 1019, 1001, 993, 1026, 994, 1016, 1058, 1039, 953, 953, 1025, 1041, 1006, 1023, 1026, 1066, 1002, 1069, 1022, 1008, 1056, 1086, 1012, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1039, 1062, 972, 1036, 988, 1052, 1061, 1031, 1073, 1034, 1036, 1029, 1018, 1003, 1019, 1044, 1029, 994, 1018, 1057, 1020, 1010, 1015, 996, 998, 1021, 1052, 1020, 1002, 1019, 998, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1044, 1054, 977, 1063, 0, 0, 0, 0, 1007, 1028, 1015, 1015, 25, 0, 0, 0, 1076, 997, 988, 1020, 10, 6, 0, 0, 973, 1053, 1042, 1022, 0, 0, 0, 0, 1049, 1070, 1076, 986, 18, 0, 0, 0, 1034, 1003, 1022, 1036, 0, 0, 0, 0, 1036, 1018, 1049, 985, 0, 0, 0, 0, 961, 1005, 1003, 987, 6, 6, 3, 0] rerouted_tokens_per_ep_rank=[4138, 4090, 4097, 4090, 4199, 4095, 4088, 3971] avg=4096.0000 max=4199 max_rank=4 min=3971 min_rank=7 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1090, 1050, 963, 1039, 0, 0, 0, 0, 1014, 1043, 999, 1028, 24, 0, 0, 0, 1063, 1019, 997, 978, 10, 6, 0, 0, 985, 1016, 1056, 1023, 0, 0, 0, 0, 1014, 1026, 995, 1000, 18, 0, 0, 0, 1013, 1031, 1045, 1030, 0, 0, 0, 0, 1035, 1022, 1078, 1001, 0, 0, 0, 0, 1017, 1018, 1008, 999, 6, 6, 3, 0] rerouted_tokens_per_ep_rank=[4142, 4108, 4073, 4080, 4053, 4119, 4136, 4057] avg=4096.0000 max=4142 max_rank=0 min=4053 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1021, 1004, 1008, 987, 0, 0, 0, 0, 1067, 1005, 1014, 1011, 25, 0, 0, 0, 1050, 1063, 1029, 1002, 10, 6, 0, 0, 1027, 997, 999, 1042, 0, 0, 0, 0, 1036, 1036, 995, 1030, 19, 0, 0, 0, 1087, 1024, 999, 1059, 0, 0, 0, 0, 1031, 986, 992, 1041, 0, 0, 0, 0, 990, 1037, 1004, 1020, 6, 6, 3, 0] rerouted_tokens_per_ep_rank=[4020, 4122, 4160, 4065, 4116, 4169, 4050, 4066] avg=4096.0000 max=4169 max_rank=5 min=4020 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1052, 1017, 1004, 1093, 0, 0, 0, 0, 1023, 1008, 1016, 1023, 25, 0, 0, 0, 1045, 974, 1000, 1016, 9, 6, 0, 0, 1003, 1033, 1012, 1054, 0, 0, 0, 0, 1036, 1015, 1016, 979, 19, 0, 0, 0, 1036, 1069, 1046, 1008, 0, 0, 0, 0, 1019, 1035, 980, 1039, 0, 0, 0, 0, 1027, 998, 1009, 1009, 6, 6, 3, 0] rerouted_tokens_per_ep_rank=[4166, 4095, 4050, 4102, 4065, 4159, 4073, 4058] avg=4096.0000 max=4166 max_rank=0 min=4050 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[995, 1013, 1025, 1002, 0, 0, 0, 0, 1024, 1001, 1030, 1004, 26, 0, 0, 0, 1008, 1020, 1031, 1014, 10, 6, 0, 0, 1051, 1052, 1019, 1014, 0, 0, 0, 0, 985, 997, 1047, 992, 19, 0, 0, 0, 1012, 1027, 1000, 1001, 0, 0, 0, 0, 1041, 1012, 1083, 1029, 0, 0, 0, 0, 1035, 1048, 1070, 1010, 6, 6, 3, 0] rerouted_tokens_per_ep_rank=[4035, 4085, 4089, 4136, 4040, 4040, 4165, 4178] avg=4096.0000 max=4178 max_rank=7 min=4035 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1005, 1032, 1006, 1056, 0, 0, 0, 0, 1031, 1008, 1009, 1026, 30, 0, 0, 0, 1019, 1001, 993, 1026, 12, 12, 0, 0, 994, 1016, 1058, 1030, 0, 0, 0, 0, 953, 953, 1025, 1041, 24, 0, 0, 0, 1006, 1023, 1026, 1061, 0, 0, 0, 0, 978, 1069, 1022, 1008, 0, 0, 0, 0, 1056, 1086, 1012, 1035, 12, 9, 5, 0] rerouted_tokens_per_ep_rank=[4099, 4104, 4063, 4098, 3996, 4116, 4077, 4215] avg=4096.0000 max=4215 max_rank=7 min=3996 min_rank=4 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1039, 1062, 926, 1036, 0, 0, 0, 0, 988, 1052, 1061, 1031, 23, 0, 0, 0, 1073, 1034, 1036, 1029, 9, 6, 0, 0, 1018, 1003, 1019, 1038, 0, 0, 0, 0, 1029, 994, 1018, 1057, 17, 0, 0, 0, 1020, 1010, 1015, 993, 0, 0, 0, 0, 983, 1021, 1052, 1020, 0, 0, 0, 0, 1002, 1019, 998, 1022, 6, 6, 3, 0] rerouted_tokens_per_ep_rank=[4063, 4155, 4187, 4078, 4115, 4038, 4076, 4056] avg=4096.0000 max=4187 max_rank=2 min=4038 min_rank=5 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1031, 1042, 1016, 1016, 0, 0, 0, 0, 996, 1007, 991, 989, 26, 0, 0, 0, 1041, 1002, 975, 1015, 10, 6, 0, 0, 1047, 1005, 1034, 1033, 0, 0, 0, 0, 1049, 1034, 1043, 1039, 19, 0, 0, 0, 980, 1023, 996, 1033, 0, 0, 0, 0, 1051, 988, 1002, 1062, 0, 0, 0, 0, 1109, 977, 1018, 1047, 7, 6, 3, 0] rerouted_tokens_per_ep_rank=[4105, 4009, 4049, 4119, 4184, 4032, 4103, 4167] avg=4096.0000 max=4184 max_rank=4 min=4009 min_rank=1 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8320, 7936, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8192, 8064, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1018, 1043, 1032, 1082, 1045, 978, 1006, 1020, 1018, 1091, 1062, 1021, 1030, 1049, 1013, 990, 1018, 1025, 1040, 990, 995, 1017, 973, 1052, 1039, 1015, 999, 1072, 977, 1043, 1020, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1041, 975, 1086, 1038, 1028, 1020, 989, 995, 1025, 1035, 1037, 1025, 1099, 1044, 994, 1053, 1085, 1006, 1002, 1016, 1045, 1032, 972, 1002, 986, 1019, 1004, 1075, 1031, 1030, 987, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1041, 1035, 1017, 1025, 1025, 1021, 990, 1062, 1014, 1006, 1082, 988, 1020, 982, 1042, 1054, 1059, 1004, 963, 1042, 1003, 1042, 1022, 1011, 970, 1055, 1064, 1024, 964, 1054, 1047, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1010, 985, 1014, 1034, 1024, 1068, 1061, 986, 1051, 1006, 1003, 993, 1057, 1053, 1043, 1053, 1045, 1037, 989, 1016, 1042, 994, 999, 1044, 1022, 1018, 998, 1022, 1010, 1030, 1037, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1084, 1017, 1049, 1034, 976, 1018, 1015, 1055, 981, 1065, 1026, 1028, 1024, 1076, 1008, 1015, 1084, 1009, 1008, 1008, 1052, 1003, 1026, 1016, 999, 1001, 1031, 1044, 981, 1024, 1000, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1026, 983, 1010, 1025, 1062, 1047, 1022, 976, 1044, 1019, 1021, 989, 1053, 1008, 1078, 1024, 1042, 1004, 1037, 1021, 969, 1027, 1044, 1011, 1021, 994, 978, 1027, 1056, 1018, 1073, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1059, 1010, 1017, 1096, 1022, 1032, 1030, 1031, 1047, 1030, 1038, 1001, 940, 1039, 982, 974, 1018, 1071, 1003, 1056, 1032, 1006, 1006, 1046, 1003, 1019, 1013, 1039, 1071, 1030, 996, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1037, 1000, 995, 1069, 975, 1111, 1021, 978, 988, 1002, 982, 1046, 1012, 1038, 1013, 984, 1004, 1055, 994, 1032, 1053, 1025, 1028, 1031, 1079, 1103, 1026, 1029, 985, 1034, 1051, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1041, 975, 1086, 1012, 0, 0, 0, 0, 1028, 1020, 989, 995, 6, 2, 1, 0, 1025, 1035, 1037, 1025, 0, 0, 0, 0, 1099, 1035, 994, 1053, 0, 0, 0, 0, 1084, 1006, 1002, 1016, 0, 0, 0, 0, 1045, 1032, 972, 1002, 18, 0, 0, 0, 986, 1019, 1004, 1073, 0, 0, 0, 0, 1031, 1030, 987, 992, 8, 3, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4041, 4122, 4181, 4108, 4069, 4082, 4051] avg=4096.0000 max=4181 max_rank=3 min=4041 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1018, 1043, 1032, 1054, 0, 0, 0, 0, 1045, 978, 1006, 1020, 6, 2, 1, 0, 1018, 1091, 1062, 1021, 0, 0, 0, 0, 1030, 1040, 1013, 990, 0, 0, 0, 0, 1017, 1025, 1040, 990, 0, 0, 0, 0, 995, 1017, 973, 1052, 19, 0, 0, 0, 1039, 1015, 999, 1070, 0, 0, 0, 0, 977, 1043, 1020, 995, 9, 3, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4058, 4192, 4073, 4072, 4056, 4123, 4047] avg=4096.0000 max=4192 max_rank=2 min=4047 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1041, 1035, 1017, 999, 0, 0, 0, 0, 1025, 1021, 990, 1062, 5, 2, 1, 0, 1014, 1006, 1082, 988, 0, 0, 0, 0, 1020, 974, 1042, 1054, 0, 0, 0, 0, 1058, 1004, 963, 1042, 0, 0, 0, 0, 1003, 1042, 1022, 1011, 18, 0, 0, 0, 970, 1055, 1064, 1022, 0, 0, 0, 0, 964, 1054, 1047, 1040, 8, 3, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4106, 4090, 4090, 4067, 4096, 4111, 4116] avg=4096.0000 max=4116 max_rank=7 min=4067 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1084, 1017, 1049, 1008, 0, 0, 0, 0, 976, 1018, 1015, 1055, 6, 2, 1, 0, 981, 1065, 1026, 1028, 0, 0, 0, 0, 1024, 1067, 1008, 1015, 0, 0, 0, 0, 1083, 1009, 1008, 1008, 0, 0, 0, 0, 1052, 1003, 1026, 1016, 18, 0, 0, 0, 999, 1001, 1031, 1042, 0, 0, 0, 0, 981, 1024, 1000, 1011, 8, 3, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4073, 4100, 4114, 4108, 4115, 4073, 4027] avg=4096.0000 max=4158 max_rank=0 min=4027 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1010, 985, 1014, 1008, 0, 0, 0, 0, 1024, 1068, 1061, 986, 6, 2, 1, 0, 1051, 1006, 1003, 993, 0, 0, 0, 0, 1057, 1044, 1043, 1053, 0, 0, 0, 0, 1044, 1037, 989, 1016, 0, 0, 0, 0, 1042, 994, 999, 1044, 18, 0, 0, 0, 1022, 1018, 998, 1020, 0, 0, 0, 0, 1010, 1030, 1037, 1024, 8, 3, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4148, 4053, 4197, 4086, 4097, 4058, 4112] avg=4096.0000 max=4197 max_rank=3 min=4017 min_rank=0 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1026, 983, 1010, 993, 0, 0, 0, 0, 1062, 1047, 1022, 976, 7, 6, 4, 0, 1044, 1019, 1021, 989, 4, 0, 0, 0, 1053, 994, 1078, 1024, 0, 0, 0, 0, 1034, 1004, 1037, 1021, 0, 0, 0, 0, 969, 1027, 1044, 1011, 20, 0, 0, 0, 1021, 994, 978, 1021, 0, 0, 0, 0, 1056, 1018, 1073, 1059, 12, 7, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4124, 4077, 4149, 4096, 4071, 4014, 4225] avg=4096.0000 max=4225 max_rank=7 min=4012 min_rank=0 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1059, 1010, 1017, 1068, 0, 0, 0, 0, 1022, 1032, 1030, 1031, 6, 2, 1, 0, 1047, 1030, 1038, 1001, 0, 0, 0, 0, 940, 1030, 982, 974, 0, 0, 0, 0, 1017, 1071, 1003, 1056, 0, 0, 0, 0, 1032, 1006, 1006, 1046, 19, 0, 0, 0, 1003, 1019, 1013, 1037, 0, 0, 0, 0, 1071, 1030, 996, 1011, 9, 3, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4124, 4116, 3926, 4147, 4109, 4072, 4120] avg=4096.0000 max=4154 max_rank=0 min=3926 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1037, 1000, 995, 1042, 0, 0, 0, 0, 975, 1111, 1021, 978, 6, 2, 1, 0, 988, 1002, 982, 1046, 0, 0, 0, 0, 1012, 1029, 1013, 984, 0, 0, 0, 0, 1003, 1055, 994, 1032, 0, 0, 0, 0, 1053, 1025, 1028, 1031, 18, 0, 0, 0, 1079, 1103, 1026, 1027, 0, 0, 0, 0, 985, 1034, 1051, 988, 9, 3, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4094, 4018, 4038, 4084, 4155, 4235, 4070] avg=4096.0000 max=4235 max_rank=6 min=4018 min_rank=2 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16544 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8064, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 8320, 8064, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1017, 1051, 1070, 1025, 1018, 996, 1026, 1013, 1016, 1072, 992, 1049, 1030, 971, 1051, 1019, 1032, 1058, 1017, 1046, 1020, 1002, 1048, 1009, 1013, 1031, 1019, 974, 1014, 1017, 1011, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[976, 999, 1048, 1115, 1030, 1057, 981, 1035, 1102, 955, 1074, 1000, 988, 986, 1036, 1052, 1038, 1035, 1036, 1001, 1026, 1023, 1048, 1016, 986, 1003, 979, 1024, 1043, 1042, 992, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1005, 1039, 989, 1081, 1011, 1045, 1065, 1048, 988, 1031, 1023, 1014, 1041, 1020, 1030, 973, 1030, 1018, 974, 983, 1043, 997, 1069, 1045, 1010, 1077, 1017, 997, 1030, 1009, 1030, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1017, 1043, 1032, 1031, 1011, 1072, 991, 1045, 1040, 1079, 1020, 1025, 1019, 1074, 1074, 972, 1056, 975, 983, 987, 1001, 1002, 1042, 976, 1053, 1012, 1013, 1019, 1032, 1010, 1046, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1049, 1040, 1023, 1017, 1037, 972, 1008, 1048, 996, 1049, 1045, 981, 1000, 1064, 1068, 1010, 1009, 1029, 983, 979, 1048, 1042, 1025, 1086, 1024, 1049, 1036, 1026, 983, 1023, 1019, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1014, 981, 1013, 987, 1010, 1049, 1057, 993, 993, 1037, 1018, 1052, 953, 1003, 1044, 1083, 1018, 1030, 1042, 1043, 1061, 1001, 1023, 995, 1001, 1063, 1035, 1053, 1091, 997, 1027, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[995, 969, 1025, 1065, 977, 1016, 991, 1044, 999, 1037, 989, 1027, 1001, 1076, 1043, 1028, 1011, 1033, 1042, 1031, 1043, 1007, 1026, 1034, 1036, 1028, 1057, 1016, 1031, 1034, 1032, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1055, 991, 1101, 1010, 1053, 1016, 1027, 1072, 1015, 1015, 1026, 990, 1008, 1024, 1010, 1054, 1037, 997, 1031, 1029, 1019, 1001, 1039, 1009, 1013, 1045, 1031, 1033, 1010, 985, 1019, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1017, 1043, 1032, 1019, 0, 0, 0, 0, 1011, 1072, 991, 1040, 0, 0, 0, 0, 1040, 1079, 1020, 1025, 1, 0, 0, 0, 1019, 1074, 1071, 972, 0, 0, 0, 0, 1056, 975, 983, 987, 12, 6, 0, 0, 1001, 1002, 1035, 976, 0, 0, 0, 0, 1053, 1012, 1013, 1019, 0, 0, 0, 0, 1032, 1010, 1046, 1016, 5, 2, 1, 0] rerouted_tokens_per_ep_rank=[4111, 4114, 4165, 4136, 4019, 4014, 4097, 4112] avg=4096.0000 max=4165 max_rank=2 min=4014 min_rank=5 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1017, 1051, 1070, 1013, 0, 0, 0, 0, 1018, 996, 1026, 1008, 0, 0, 0, 0, 1016, 1072, 992, 1049, 1, 0, 0, 0, 1030, 971, 1048, 1019, 0, 0, 0, 0, 1032, 1058, 1017, 1046, 12, 6, 0, 0, 1020, 1002, 1041, 1009, 0, 0, 0, 0, 1013, 1031, 1019, 974, 0, 0, 0, 0, 1014, 1017, 1011, 1041, 5, 2, 1, 0] rerouted_tokens_per_ep_rank=[4151, 4048, 4130, 4068, 4171, 4072, 4037, 4091] avg=4096.0000 max=4171 max_rank=4 min=4037 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1005, 1039, 989, 1068, 0, 0, 0, 0, 1011, 1045, 1065, 1043, 0, 0, 0, 0, 988, 1031, 1023, 1014, 1, 0, 0, 0, 1041, 1020, 1027, 973, 0, 0, 0, 0, 1030, 1018, 974, 983, 13, 6, 0, 0, 1043, 997, 1062, 1045, 0, 0, 0, 0, 1010, 1077, 1017, 997, 0, 0, 0, 0, 1030, 1009, 1030, 1036, 5, 2, 1, 0] rerouted_tokens_per_ep_rank=[4101, 4164, 4057, 4061, 4024, 4147, 4101, 4113] avg=4096.0000 max=4164 max_rank=1 min=4024 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[976, 999, 1048, 1101, 0, 0, 0, 0, 1030, 1057, 981, 1030, 0, 0, 0, 0, 1102, 955, 1074, 1000, 1, 0, 0, 0, 988, 986, 1033, 1052, 0, 0, 0, 0, 1038, 1035, 1036, 1001, 14, 6, 0, 0, 1026, 1023, 1041, 1016, 0, 0, 0, 0, 986, 1003, 979, 1024, 0, 0, 0, 0, 1043, 1042, 992, 1042, 5, 2, 1, 0] rerouted_tokens_per_ep_rank=[4124, 4098, 4132, 4059, 4130, 4106, 3992, 4127] avg=4096.0000 max=4132 max_rank=2 min=3992 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1049, 1040, 1023, 1000, 0, 0, 0, 0, 1037, 972, 1008, 1037, 0, 0, 0, 0, 996, 1049, 1045, 981, 7, 5, 0, 0, 1000, 1064, 1052, 1010, 0, 0, 0, 0, 1009, 1029, 983, 979, 17, 8, 0, 0, 1048, 1042, 1013, 1086, 0, 0, 0, 0, 1024, 1044, 1036, 1026, 0, 0, 0, 0, 983, 1023, 1019, 1000, 11, 9, 4, 0] rerouted_tokens_per_ep_rank=[4112, 4054, 4083, 4126, 4025, 4189, 4130, 4049] avg=4096.0000 max=4189 max_rank=5 min=4025 min_rank=4 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1014, 981, 1013, 975, 0, 0, 0, 0, 1010, 1049, 1057, 988, 0, 0, 0, 0, 993, 1037, 1018, 1052, 1, 0, 0, 0, 953, 1003, 1041, 1083, 0, 0, 0, 0, 1018, 1030, 1042, 1043, 12, 6, 0, 0, 1061, 1001, 1017, 995, 0, 0, 0, 0, 1001, 1063, 1035, 1053, 0, 0, 0, 0, 1091, 997, 1027, 1001, 5, 2, 0, 0] rerouted_tokens_per_ep_rank=[3983, 4104, 4101, 4080, 4151, 4074, 4152, 4123] avg=4096.0000 max=4152 max_rank=6 min=3983 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[995, 969, 1025, 1052, 0, 0, 0, 0, 977, 1016, 991, 1039, 0, 0, 0, 0, 999, 1037, 989, 1027, 1, 0, 0, 0, 1001, 1076, 1040, 1028, 0, 0, 0, 0, 1011, 1033, 1042, 1031, 13, 6, 0, 0, 1043, 1007, 1020, 1034, 0, 0, 0, 0, 1036, 1028, 1057, 1016, 0, 0, 0, 0, 1031, 1034, 1032, 1025, 5, 2, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4023, 4053, 4145, 4136, 4104, 4137, 4129] avg=4096.0000 max=4145 max_rank=3 min=4023 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1055, 991, 1101, 998, 0, 0, 0, 0, 1053, 1016, 1027, 1067, 0, 0, 0, 0, 1015, 1015, 1026, 990, 1, 0, 0, 0, 1008, 1024, 1007, 1054, 0, 0, 0, 0, 1037, 997, 1031, 1029, 12, 6, 0, 0, 1019, 1001, 1033, 1009, 0, 0, 0, 0, 1013, 1045, 1031, 1033, 0, 0, 0, 0, 1010, 985, 1019, 1003, 5, 2, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4163, 4047, 4093, 4112, 4062, 4122, 4024] avg=4096.0000 max=4163 max_rank=1 min=4024 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1014, 1041, 1046, 1027, 1019, 1028, 1026, 978, 1018, 980, 981, 1039, 1029, 972, 1038, 1037, 1040, 1021, 1026, 1006, 1033, 1033, 1007, 1054, 1022, 1021, 992, 1057, 1065, 1030, 1049, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1065, 1030, 1049, 1010, 1019, 986, 1034, 1012, 1011, 1030, 1012, 1061, 1045, 998, 982, 981, 1050, 1042, 1004, 1017, 1054, 1034, 1022, 1056, 1017, 989, 1001, 1039, 1053, 1019, 1054, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1033, 1009, 962, 1003, 1022, 994, 1001, 1022, 1026, 1074, 1039, 1029, 1023, 1016, 1059, 1066, 1037, 994, 1029, 1040, 1032, 1068, 1025, 1021, 1014, 1012, 987, 988, 1047, 987, 1058, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1023, 1020, 1038, 1011, 1035, 1035, 996, 1034, 986, 994, 1042, 1001, 1026, 1014, 1000, 1050, 1019, 1033, 1024, 1026, 1038, 1061, 1051, 1030, 1028, 982, 1021, 1042, 1030, 1000, 1047, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1010, 1021, 1055, 1008, 1021, 1032, 1002, 1018, 971, 1031, 979, 1040, 1032, 970, 1053, 997, 1055, 980, 1090, 1037, 1036, 1051, 1059, 1037, 983, 1020, 1010, 1046, 1043, 1056, 1021, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1012, 1030, 1020, 1047, 1023, 972, 1082, 973, 1023, 1014, 1004, 1042, 1053, 1042, 1053, 1040, 1061, 977, 1013, 1034, 1050, 1027, 1022, 1026, 1026, 992, 1000, 1064, 981, 1029, 1049, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[999, 1065, 993, 1050, 1049, 1055, 1055, 971, 1031, 1037, 1028, 1015, 1001, 1040, 1051, 1021, 991, 1044, 1042, 984, 1019, 1040, 994, 1057, 989, 1074, 978, 1024, 1017, 1074, 977, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1056, 1014, 998, 1004, 984, 1059, 1014, 1032, 1002, 1028, 1053, 1019, 1006, 1039, 1029, 1016, 1019, 1025, 1028, 1025, 1052, 961, 1051, 1022, 1071, 1047, 1010, 1051, 978, 986, 1012, 1077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1014, 1041, 1046, 1027, 0, 0, 0, 0, 1019, 1028, 1026, 978, 22, 0, 0, 0, 1018, 980, 981, 1039, 9, 5, 0, 0, 1029, 972, 1038, 1037, 0, 0, 0, 0, 1035, 1021, 1026, 1006, 0, 0, 0, 0, 990, 1033, 1007, 1054, 0, 0, 0, 0, 1022, 1021, 992, 1057, 21, 0, 0, 0, 1065, 1030, 1040, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4073, 4032, 4076, 4088, 4084, 4113, 4174] avg=4096.0000 max=4174 max_rank=7 min=4032 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1023, 1020, 1038, 1011, 0, 0, 0, 0, 1035, 1035, 996, 1034, 23, 0, 0, 0, 986, 994, 1042, 1001, 9, 5, 0, 0, 1026, 1014, 1000, 1050, 0, 0, 0, 0, 1014, 1033, 1024, 1026, 0, 0, 0, 0, 994, 1061, 1051, 1030, 0, 0, 0, 0, 1028, 982, 1021, 1042, 21, 0, 0, 0, 1030, 1000, 1038, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4123, 4037, 4090, 4097, 4136, 4094, 4099] avg=4096.0000 max=4136 max_rank=5 min=4037 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1033, 1009, 962, 1003, 0, 0, 0, 0, 1022, 994, 1001, 1022, 22, 0, 0, 0, 1026, 1074, 1039, 1029, 9, 5, 0, 0, 1023, 1016, 1059, 1066, 0, 0, 0, 0, 1032, 994, 1029, 1040, 0, 0, 0, 0, 989, 1068, 1025, 1021, 0, 0, 0, 0, 1014, 1012, 987, 988, 21, 0, 0, 0, 1047, 987, 1049, 1051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4061, 4182, 4164, 4095, 4103, 4022, 4134] avg=4096.0000 max=4182 max_rank=2 min=4007 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1065, 1030, 1049, 1010, 0, 0, 0, 0, 1019, 986, 1034, 1012, 23, 0, 0, 0, 1011, 1030, 1012, 1061, 9, 5, 0, 0, 1045, 998, 982, 981, 0, 0, 0, 0, 1045, 1042, 1004, 1017, 0, 0, 0, 0, 1010, 1034, 1022, 1056, 0, 0, 0, 0, 1017, 989, 1001, 1039, 21, 0, 0, 0, 1053, 1019, 1045, 992, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4074, 4128, 4006, 4108, 4122, 4067, 4109] avg=4096.0000 max=4154 max_rank=0 min=4006 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1010, 1021, 1055, 1008, 0, 0, 0, 0, 1021, 1032, 1002, 1018, 23, 0, 0, 0, 971, 1031, 979, 1040, 9, 5, 0, 0, 1032, 970, 1053, 997, 0, 0, 0, 0, 1050, 980, 1090, 1037, 0, 0, 0, 0, 992, 1051, 1059, 1037, 0, 0, 0, 0, 983, 1020, 1010, 1046, 21, 0, 0, 0, 1043, 1056, 1012, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4035, 4052, 4157, 4139, 4080, 4115] avg=4096.0000 max=4157 max_rank=4 min=4035 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[999, 1065, 993, 1050, 0, 0, 0, 0, 1049, 1055, 1055, 971, 22, 0, 0, 0, 1031, 1037, 1028, 1015, 9, 5, 0, 0, 1001, 1040, 1051, 1021, 0, 0, 0, 0, 986, 1044, 1042, 984, 0, 0, 0, 0, 977, 1040, 994, 1057, 0, 0, 0, 0, 989, 1074, 978, 1024, 20, 0, 0, 0, 1017, 1074, 968, 1003, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4152, 4125, 4113, 4056, 4068, 4085, 4062] avg=4096.0000 max=4152 max_rank=1 min=4056 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1012, 1030, 1020, 1047, 5, 0, 0, 0, 1023, 972, 1082, 973, 27, 0, 0, 0, 1023, 1014, 1004, 1042, 14, 10, 6, 0, 1053, 1042, 1042, 1040, 0, 0, 0, 0, 1051, 977, 1013, 1034, 0, 0, 0, 0, 999, 1027, 1022, 1026, 0, 0, 0, 0, 1026, 992, 1000, 1064, 24, 1, 0, 0, 981, 1029, 1034, 987, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4077, 4113, 4177, 4075, 4074, 4107, 4031] avg=4096.0000 max=4177 max_rank=3 min=4031 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1056, 1014, 998, 1004, 0, 0, 0, 0, 984, 1059, 1014, 1032, 23, 0, 0, 0, 1002, 1028, 1053, 1019, 9, 5, 0, 0, 1006, 1039, 1029, 1016, 0, 0, 0, 0, 1014, 1025, 1028, 1025, 0, 0, 0, 0, 1008, 961, 1051, 1022, 0, 0, 0, 0, 1071, 1047, 1010, 1051, 21, 0, 0, 0, 978, 986, 1003, 1077, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4112, 4116, 4090, 4092, 4042, 4200, 4044] avg=4096.0000 max=4200 max_rank=6 min=4042 min_rank=5 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8064, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8064, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1076, 1081, 1039, 996, 1041, 1000, 1016, 1077, 962, 1002, 1043, 996, 1073, 1033, 999, 1070, 999, 1010, 1011, 1046, 1055, 1022, 1013, 1008, 973, 1042, 996, 1028, 1039, 1006, 974, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1023, 1021, 1037, 1007, 984, 1016, 1005, 1038, 1040, 961, 989, 1055, 972, 1017, 1003, 1067, 999, 1058, 1044, 1013, 1015, 1065, 1019, 1009, 1032, 1043, 1075, 1019, 1052, 1030, 1047, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1036, 993, 991, 1027, 1011, 969, 994, 1001, 1022, 1044, 1053, 1004, 1044, 986, 1034, 1068, 1057, 1074, 987, 1009, 1050, 1036, 1044, 1049, 998, 1037, 1015, 1008, 1071, 1035, 1004, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1016, 1070, 1032, 1030, 979, 1036, 1023, 1078, 1024, 967, 1018, 992, 1045, 1000, 1044, 1010, 1043, 997, 1012, 1044, 998, 1027, 1029, 1043, 1019, 1027, 1005, 1010, 1018, 1021, 1065, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1004, 1049, 1007, 1026, 1067, 975, 1051, 998, 1039, 992, 993, 1002, 1065, 1080, 1023, 1004, 1039, 965, 1052, 1037, 1012, 970, 1034, 1017, 1025, 1031, 1092, 1031, 1003, 979, 1041, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1018, 1039, 1029, 1005, 1033, 990, 1017, 1062, 1004, 993, 1011, 1015, 1047, 1063, 1005, 1051, 1016, 988, 1031, 1057, 1043, 1054, 1028, 1057, 1021, 1022, 969, 1023, 1023, 1030, 1015, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1036, 980, 977, 1009, 1054, 995, 1049, 1039, 1038, 1040, 1056, 1024, 1050, 995, 1026, 1036, 1015, 988, 1026, 1026, 1039, 1062, 1049, 1036, 1011, 1025, 1083, 1008, 1020, 1018, 979, 979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1054, 1003, 993, 1001, 1050, 1060, 1042, 1057, 1048, 962, 982, 1049, 1048, 974, 1037, 1012, 969, 1049, 955, 1049, 1007, 1058, 1064, 1079, 1052, 972, 1029, 1061, 1008, 1011, 1012, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1076, 1081, 1039, 996, 5, 1, 1, 0, 1041, 1000, 1016, 1072, 0, 0, 0, 0, 962, 1002, 1043, 996, 39, 3, 0, 0, 1047, 1033, 999, 1070, 0, 0, 0, 0, 999, 1010, 1011, 1046, 13, 0, 0, 0, 1055, 1022, 1013, 969, 0, 0, 0, 0, 973, 1042, 995, 1028, 0, 0, 0, 0, 1039, 1006, 974, 1042, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4199, 4129, 4045, 4149, 4079, 4059, 4038, 4070] avg=4096.0000 max=4199 max_rank=0 min=4038 min_rank=6 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1016, 1070, 1032, 1030, 5, 1, 1, 0, 979, 1036, 1023, 1073, 0, 0, 0, 0, 1024, 967, 1018, 992, 40, 3, 0, 0, 1020, 1000, 1044, 1010, 0, 0, 0, 0, 1043, 997, 1012, 1044, 12, 0, 0, 0, 998, 1027, 1029, 1003, 0, 0, 0, 0, 1019, 1027, 1004, 1010, 0, 0, 0, 0, 1018, 1021, 1065, 1046, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4111, 4044, 4074, 4108, 4057, 4060, 4159] avg=4096.0000 max=4159 max_rank=7 min=4044 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1023, 1021, 1037, 1007, 4, 1, 1, 0, 984, 1016, 1005, 1034, 0, 0, 0, 0, 1040, 961, 989, 1055, 39, 2, 0, 0, 950, 1017, 1003, 1067, 0, 0, 0, 0, 999, 1058, 1044, 1013, 11, 0, 0, 0, 1015, 1065, 1019, 970, 0, 0, 0, 0, 1032, 1043, 1074, 1019, 0, 0, 0, 0, 1052, 1030, 1047, 1013, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4039, 4086, 4037, 4125, 4069, 4168, 4150] avg=4096.0000 max=4168 max_rank=6 min=4037 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1004, 1049, 1007, 1026, 4, 1, 1, 0, 1067, 975, 1051, 994, 0, 0, 0, 0, 1039, 992, 993, 1002, 39, 3, 0, 0, 1039, 1080, 1023, 1004, 0, 0, 0, 0, 1039, 965, 1052, 1037, 13, 0, 0, 0, 1012, 970, 1034, 978, 0, 0, 0, 0, 1025, 1031, 1091, 1031, 0, 0, 0, 0, 1003, 979, 1041, 1065, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4087, 4068, 4146, 4106, 3994, 4178, 4097] avg=4096.0000 max=4178 max_rank=6 min=3994 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1036, 980, 977, 1009, 4, 1, 1, 0, 1054, 995, 1049, 1035, 0, 0, 0, 0, 1038, 1040, 1056, 1024, 40, 3, 0, 0, 1025, 995, 1026, 1036, 0, 0, 0, 0, 1015, 988, 1026, 1026, 12, 0, 0, 0, 1039, 1062, 1049, 996, 0, 0, 0, 0, 1011, 1025, 1082, 1008, 0, 0, 0, 0, 1020, 1018, 979, 979, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4008, 4133, 4201, 4082, 4067, 4146, 4126, 4005] avg=4096.0000 max=4201 max_rank=2 min=4005 min_rank=7 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1018, 1039, 1029, 1005, 4, 1, 1, 0, 1033, 990, 1017, 1058, 0, 0, 0, 0, 1004, 993, 1011, 1015, 41, 3, 0, 0, 1022, 1063, 1005, 1051, 0, 0, 0, 0, 1016, 988, 1031, 1057, 12, 0, 0, 0, 1043, 1054, 1028, 1016, 0, 0, 0, 0, 1021, 1022, 968, 1023, 0, 0, 0, 0, 1023, 1030, 1015, 1009, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4098, 4067, 4141, 4104, 4141, 4034, 4086] avg=4096.0000 max=4141 max_rank=3 min=4034 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1036, 993, 991, 1027, 9, 7, 3, 0, 1011, 969, 994, 992, 0, 0, 0, 0, 1022, 1044, 1053, 1004, 43, 5, 0, 0, 1005, 986, 1034, 1068, 0, 0, 0, 0, 1057, 1074, 987, 1009, 18, 0, 0, 0, 1050, 1036, 1044, 1006, 0, 0, 0, 0, 998, 1037, 1008, 1008, 0, 0, 0, 0, 1071, 1035, 1004, 1017, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 3966, 4171, 4093, 4145, 4136, 4051, 4140] avg=4096.0000 max=4171 max_rank=2 min=3966 min_rank=1 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1054, 1003, 993, 1001, 4, 1, 1, 0, 1050, 1060, 1042, 1053, 0, 0, 0, 0, 1048, 962, 982, 1049, 42, 3, 0, 0, 1023, 974, 1037, 1012, 0, 0, 0, 0, 969, 1049, 955, 1049, 12, 0, 0, 0, 1007, 1058, 1064, 1037, 0, 0, 0, 0, 1052, 972, 1028, 1061, 0, 0, 0, 0, 1008, 1011, 1012, 1021, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4205, 4086, 4046, 4034, 4166, 4113, 4061] avg=4096.0000 max=4205 max_rank=1 min=4034 min_rank=4 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[973, 1032, 1006, 1053, 1054, 1068, 1024, 975, 1066, 1038, 984, 1031, 985, 1026, 1057, 1045, 984, 1018, 1005, 1033, 1024, 1049, 1024, 1024, 1020, 1032, 1008, 1073, 970, 1065, 990, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1035, 1018, 1060, 1022, 1028, 1013, 1068, 1049, 1029, 993, 1020, 1056, 1086, 1046, 1030, 1047, 1037, 1041, 1025, 973, 972, 1008, 1033, 928, 1051, 1019, 1022, 994, 1016, 994, 1029, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[991, 1054, 1005, 1015, 1066, 998, 992, 1029, 1015, 1008, 984, 1018, 1012, 1047, 1028, 1059, 1005, 1024, 1053, 1046, 1009, 1015, 1064, 1075, 1022, 1008, 1020, 1038, 1004, 1045, 1034, 985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1009, 1048, 1090, 1049, 1017, 995, 1041, 1036, 1031, 1020, 1013, 1016, 1040, 994, 1045, 1028, 1019, 1032, 982, 1013, 1026, 1009, 999, 1067, 1036, 990, 1049, 1027, 1026, 1019, 1053, 949] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1049, 1062, 1052, 1051, 981, 969, 1005, 1031, 1010, 1001, 1083, 989, 1010, 1030, 1038, 1021, 1026, 1016, 995, 1048, 1089, 994, 1036, 1029, 994, 1035, 1026, 1000, 1078, 1003, 1037, 980] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1039, 1039, 999, 1007, 990, 1029, 1101, 1051, 1044, 1056, 1065, 982, 1004, 1049, 1027, 991, 1050, 985, 1000, 1050, 992, 1024, 1047, 1030, 983, 1032, 1000, 977, 1037, 1045, 1029, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1043, 1057, 1046, 1004, 1039, 1030, 999, 1030, 1031, 1034, 1001, 1102, 1020, 1010, 1002, 1009, 1001, 1067, 1025, 1004, 988, 1029, 1012, 984, 1031, 1010, 1065, 1047, 1023, 993, 1051, 981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1021, 1015, 1061, 1039, 988, 1020, 1071, 1006, 1040, 1056, 984, 1040, 1027, 1004, 1048, 1008, 1063, 1106, 1018, 1017, 1037, 1011, 1009, 993, 1028, 983, 1026, 967, 1054, 1040, 1034, 954] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1035, 986, 1060, 1022, 0, 0, 0, 0, 1028, 1013, 1066, 1049, 0, 0, 0, 0, 1021, 993, 1020, 1056, 0, 0, 0, 0, 1086, 1046, 1018, 1047, 0, 0, 0, 0, 1037, 1041, 1025, 973, 0, 0, 0, 0, 972, 1008, 1033, 928, 8, 5, 2, 0, 1051, 1019, 1022, 994, 11, 7, 0, 0, 1016, 994, 1029, 1026, 21, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4156, 4090, 4197, 4076, 3956, 4104, 4086] avg=4096.0000 max=4197 max_rank=3 min=3956 min_rank=5 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16222 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1009, 1014, 1090, 1049, 0, 0, 0, 0, 1017, 995, 1039, 1036, 0, 0, 0, 0, 1023, 1020, 1013, 1016, 0, 0, 0, 0, 1040, 994, 1032, 1028, 0, 0, 0, 0, 1019, 1032, 982, 1013, 0, 0, 0, 0, 1026, 1009, 999, 1067, 8, 6, 2, 0, 1036, 990, 1049, 1027, 12, 7, 0, 0, 1026, 1019, 1053, 949, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4087, 4072, 4094, 4046, 4117, 4121, 4069] avg=4096.0000 max=4162 max_rank=0 min=4046 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[973, 998, 1006, 1053, 0, 0, 0, 0, 1054, 1068, 1022, 975, 0, 0, 0, 0, 1057, 1038, 984, 1031, 0, 0, 0, 0, 985, 1026, 1044, 1045, 0, 0, 0, 0, 984, 1018, 1005, 1033, 0, 0, 0, 0, 1024, 1049, 1024, 1024, 9, 6, 2, 0, 1020, 1032, 1008, 1073, 12, 7, 0, 0, 970, 1065, 990, 1032, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4119, 4110, 4100, 4040, 4138, 4152, 4079] avg=4096.0000 max=4152 max_rank=6 min=4030 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[991, 1020, 1005, 1015, 0, 0, 0, 0, 1066, 998, 990, 1029, 0, 0, 0, 0, 1007, 1008, 984, 1018, 0, 0, 0, 0, 1012, 1047, 1016, 1059, 0, 0, 0, 0, 1005, 1024, 1053, 1046, 0, 0, 0, 0, 1009, 1015, 1064, 1075, 8, 5, 2, 0, 1022, 1008, 1020, 1038, 12, 7, 0, 0, 1004, 1045, 1034, 985, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4083, 4017, 4134, 4128, 4178, 4107, 4090] avg=4096.0000 max=4178 max_rank=5 min=4017 min_rank=2 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1043, 1023, 1046, 1004, 0, 0, 0, 0, 1039, 1030, 997, 1030, 0, 0, 0, 0, 1023, 1034, 1001, 1102, 0, 0, 0, 0, 1020, 1010, 991, 1009, 0, 0, 0, 0, 1001, 1067, 1025, 1004, 0, 0, 0, 0, 988, 1029, 1012, 984, 8, 5, 2, 0, 1031, 1010, 1065, 1047, 12, 6, 0, 0, 1023, 993, 1051, 981, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4096, 4160, 4030, 4097, 4028, 4171, 4070] avg=4096.0000 max=4171 max_rank=6 min=4028 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1039, 997, 999, 1007, 0, 0, 0, 0, 990, 1029, 1090, 1051, 0, 0, 0, 0, 1030, 1056, 1065, 982, 0, 0, 0, 0, 1004, 1049, 1009, 991, 0, 0, 0, 0, 1050, 985, 1000, 1050, 7, 0, 0, 0, 992, 1024, 1047, 1030, 14, 9, 4, 0, 983, 1032, 1000, 977, 16, 9, 0, 0, 1037, 1045, 1029, 1014, 26, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4160, 4133, 4053, 4092, 4120, 4017, 4151] avg=4096.0000 max=4160 max_rank=1 min=4017 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1049, 1028, 1052, 1051, 0, 0, 0, 0, 981, 969, 1003, 1031, 0, 0, 0, 0, 1002, 1001, 1083, 989, 0, 0, 0, 0, 1010, 1030, 1025, 1021, 0, 0, 0, 0, 1026, 1016, 995, 1048, 0, 0, 0, 0, 1089, 994, 1036, 1029, 8, 6, 2, 0, 994, 1035, 1026, 1000, 12, 7, 0, 0, 1078, 1003, 1037, 980, 22, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 3984, 4075, 4086, 4085, 4164, 4074, 4120] avg=4096.0000 max=4180 max_rank=0 min=3984 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1021, 983, 1061, 1039, 0, 0, 0, 0, 988, 1020, 1069, 1006, 0, 0, 0, 0, 1031, 1056, 984, 1040, 0, 0, 0, 0, 1027, 1004, 1035, 1008, 0, 0, 0, 0, 1063, 1106, 1018, 1017, 0, 0, 0, 0, 1037, 1011, 1009, 993, 9, 6, 2, 0, 1028, 983, 1026, 967, 11, 7, 0, 0, 1054, 1040, 1034, 954, 21, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4083, 4111, 4074, 4204, 4067, 4022, 4103] avg=4096.0000 max=4204 max_rank=4 min=4022 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 7936, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1027, 1066, 1011, 1040, 985, 1036, 1048, 1038, 1044, 987, 1027, 1012, 963, 1003, 1027, 1030, 1050, 1043, 1029, 1044, 994, 1005, 1017, 1021, 1004, 1036, 988, 1037, 1032, 1045, 1018, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[987, 1003, 1095, 1007, 970, 1004, 1048, 989, 1036, 1008, 1040, 994, 1030, 1080, 1050, 1012, 1060, 1031, 1056, 1024, 1016, 987, 1059, 981, 1017, 1046, 986, 1022, 1046, 1026, 1007, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1070, 993, 1017, 1034, 1038, 997, 1043, 1015, 1023, 1074, 1066, 1031, 981, 1023, 1045, 1020, 998, 980, 1027, 1009, 1007, 1019, 1073, 1044, 993, 1023, 1048, 1011, 1005, 985, 1045, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1023, 1026, 1025, 1019, 1031, 1002, 1004, 1031, 1014, 941, 1005, 1085, 1002, 1021, 1040, 1029, 1004, 1013, 1030, 988, 1030, 1026, 1025, 1041, 1065, 1039, 1013, 1018, 976, 1035, 1081, 1086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1003, 1020, 1017, 1042, 1024, 1063, 1022, 1020, 1010, 1038, 1003, 975, 1034, 1029, 1084, 995, 1012, 1034, 1006, 1076, 1007, 1067, 1044, 1002, 1031, 1023, 990, 1019, 1009, 990, 1017, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1060, 988, 978, 1026, 1016, 1040, 1047, 1068, 1029, 1031, 1002, 990, 1044, 987, 1013, 1002, 999, 1053, 1051, 1045, 1031, 1052, 1003, 1038, 1049, 988, 1019, 1035, 1028, 1023, 1027, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1052, 1028, 1022, 1026, 1048, 996, 1024, 1042, 1003, 1045, 1029, 1023, 988, 1002, 1018, 1043, 1019, 1032, 987, 1037, 1075, 1049, 1001, 1021, 1044, 1026, 981, 995, 1007, 1003, 1053, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1043, 1074, 998, 979, 1012, 1046, 998, 1024, 1000, 1050, 1024, 1055, 1008, 1013, 1030, 1017, 979, 975, 1053, 1028, 1070, 1013, 1036, 991, 1022, 1036, 1006, 1027, 1016, 1048, 1033, 1064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1024, 1066, 1011, 1040, 0, 0, 0, 0, 985, 1036, 1048, 1038, 0, 0, 0, 0, 1044, 987, 1027, 1012, 4, 3, 0, 0, 963, 1003, 1027, 1030, 8, 4, 0, 0, 1050, 1043, 1029, 1044, 0, 0, 0, 0, 994, 1005, 1009, 1021, 0, 0, 0, 0, 1004, 1036, 988, 1037, 16, 0, 0, 0, 1032, 1045, 1018, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4107, 4077, 4035, 4166, 4029, 4081, 4132] avg=4096.0000 max=4166 max_rank=4 min=4029 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[984, 1003, 1095, 1007, 0, 0, 0, 0, 970, 1004, 1048, 989, 0, 0, 0, 0, 1036, 1008, 1040, 994, 4, 3, 0, 0, 1030, 1080, 1050, 1012, 8, 5, 0, 0, 1060, 1031, 1056, 1024, 0, 0, 0, 0, 1016, 987, 1050, 981, 0, 0, 0, 0, 1017, 1046, 986, 1022, 16, 0, 0, 0, 1046, 1026, 1007, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4011, 4085, 4185, 4171, 4034, 4087, 4106] avg=4096.0000 max=4185 max_rank=3 min=4011 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1000, 1020, 1017, 1042, 0, 0, 0, 0, 1024, 1063, 1022, 1020, 0, 0, 0, 0, 1010, 1038, 1003, 975, 4, 3, 0, 0, 1034, 1029, 1084, 995, 8, 4, 0, 0, 1012, 1034, 1006, 1076, 0, 0, 0, 0, 1007, 1067, 1036, 1002, 0, 0, 0, 0, 1031, 1023, 990, 1019, 16, 0, 0, 0, 1009, 990, 1017, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4129, 4033, 4154, 4128, 4112, 4079, 4054] avg=4096.0000 max=4154 max_rank=3 min=4033 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1057, 988, 978, 1026, 0, 0, 0, 0, 1016, 1040, 1047, 1068, 0, 0, 0, 0, 1029, 1031, 1002, 990, 4, 3, 0, 0, 1044, 987, 1013, 1002, 7, 4, 0, 0, 999, 1053, 1051, 1045, 0, 0, 0, 0, 1031, 1052, 995, 1038, 0, 0, 0, 0, 1049, 988, 1019, 1035, 15, 0, 0, 0, 1028, 1023, 1027, 984, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4171, 4059, 4057, 4148, 4116, 4106, 4062] avg=4096.0000 max=4171 max_rank=1 min=4049 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1066, 993, 1017, 1034, 0, 0, 0, 0, 1038, 997, 1043, 1015, 0, 0, 0, 0, 1023, 1074, 1066, 1031, 4, 4, 0, 0, 981, 1023, 1045, 1020, 8, 5, 0, 0, 998, 980, 1027, 1009, 0, 0, 0, 0, 1007, 1019, 1064, 1044, 0, 0, 0, 0, 993, 1023, 1048, 1011, 16, 0, 0, 0, 1005, 985, 1045, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4093, 4202, 4082, 4014, 4134, 4091, 4042] avg=4096.0000 max=4202 max_rank=2 min=4014 min_rank=4 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1020, 1026, 1025, 1019, 0, 0, 0, 0, 1031, 1002, 1004, 1031, 0, 0, 0, 0, 1014, 941, 1005, 1085, 4, 3, 0, 0, 1002, 1021, 1040, 1029, 8, 4, 0, 0, 1004, 1013, 1030, 988, 0, 0, 0, 0, 1030, 1026, 1017, 1041, 0, 0, 0, 0, 1065, 1039, 1013, 1018, 16, 0, 0, 0, 976, 1035, 1081, 1062, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4068, 4052, 4104, 4035, 4114, 4151, 4154] avg=4096.0000 max=4154 max_rank=7 min=4035 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1043, 1028, 1022, 1026, 0, 0, 0, 0, 1048, 996, 1023, 1042, 0, 0, 0, 0, 1003, 1045, 1029, 1023, 10, 9, 4, 1, 988, 1002, 1018, 1043, 11, 9, 0, 0, 1019, 1032, 987, 1033, 0, 0, 0, 0, 1075, 1049, 982, 1021, 0, 0, 0, 0, 1044, 1026, 981, 995, 20, 0, 0, 0, 1007, 1003, 1053, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4109, 4124, 4071, 4071, 4127, 4066, 4081] avg=4096.0000 max=4127 max_rank=5 min=4066 min_rank=6 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1040, 1074, 998, 979, 0, 0, 0, 0, 1012, 1046, 998, 1024, 0, 0, 0, 0, 1000, 1050, 1024, 1055, 4, 3, 0, 0, 1008, 1013, 1030, 1017, 8, 4, 0, 0, 979, 975, 1053, 1028, 0, 0, 0, 0, 1070, 1013, 1028, 991, 0, 0, 0, 0, 1022, 1036, 1006, 1027, 16, 0, 0, 0, 1016, 1048, 1033, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4080, 4136, 4080, 4035, 4102, 4107, 4137] avg=4096.0000 max=4137 max_rank=7 min=4035 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1040, 997, 1039, 1021, 1001, 1036, 1022, 1019, 1032, 990, 1021, 1076, 1019, 1031, 1073, 1031, 990, 1020, 1067, 1017, 1054, 1062, 1036, 1035, 974, 1053, 1006, 992, 1004, 1029, 978, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1043, 1047, 1018, 1018, 1013, 1036, 1011, 989, 1018, 1002, 976, 1020, 1032, 1006, 1021, 1039, 1054, 1034, 997, 1054, 1017, 1056, 991, 1025, 1024, 995, 1044, 1056, 1008, 1096, 1027, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1056, 1032, 1039, 1031, 991, 1013, 1023, 975, 1014, 1022, 1059, 1013, 1008, 1010, 1033, 1070, 981, 1011, 1019, 1044, 1068, 994, 1065, 1045, 1041, 1029, 955, 1027, 1012, 1035, 1031, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1014, 1029, 1023, 1017, 1070, 1027, 1028, 1027, 1050, 975, 1031, 1016, 1021, 1065, 1056, 1025, 1053, 1019, 973, 1070, 998, 1025, 1001, 1047, 1011, 1034, 1008, 978, 1037, 1044, 1009, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1005, 989, 1026, 1069, 1052, 975, 1028, 1070, 1012, 1039, 1038, 968, 1060, 1008, 1041, 1026, 1067, 967, 993, 1011, 1051, 1023, 1040, 1061, 1005, 1036, 997, 1011, 1035, 1038, 1023, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1030, 1045, 1082, 1055, 973, 1004, 1001, 946, 1001, 1039, 1028, 994, 1056, 999, 1042, 1037, 1063, 1027, 1039, 1017, 1077, 1014, 991, 1020, 1043, 1011, 1023, 984, 1049, 1049, 1043, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[992, 1010, 1003, 1049, 1019, 999, 1006, 997, 1015, 1028, 1044, 1052, 1015, 1029, 998, 1005, 1076, 1055, 1033, 1046, 1045, 1010, 1056, 1038, 1034, 1005, 1014, 1047, 1053, 1020, 977, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1003, 1026, 984, 1029, 1020, 1011, 1034, 1035, 1089, 1021, 1050, 1017, 1018, 1037, 1041, 1046, 1027, 977, 990, 971, 1004, 1021, 1023, 1005, 1039, 1026, 1065, 1026, 1030, 1040, 1039, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1040, 997, 1039, 1012, 0, 0, 0, 0, 1001, 1036, 1022, 1019, 29, 11, 0, 0, 1032, 990, 1021, 1076, 2, 0, 0, 0, 1019, 1031, 1044, 1031, 0, 0, 0, 0, 990, 1020, 1067, 1017, 0, 0, 0, 0, 1025, 1062, 1036, 1035, 0, 0, 0, 0, 974, 1053, 1006, 992, 18, 3, 0, 0, 1004, 1029, 978, 1003, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4118, 4121, 4125, 4094, 4158, 4046, 4018] avg=4096.0000 max=4158 max_rank=5 min=4018 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1043, 1047, 1018, 1009, 0, 0, 0, 0, 1013, 1036, 1011, 989, 28, 10, 0, 0, 1018, 1002, 976, 1020, 2, 0, 0, 0, 1032, 1006, 994, 1039, 0, 0, 0, 0, 1054, 1034, 997, 1054, 0, 0, 0, 0, 989, 1056, 991, 1025, 0, 0, 0, 0, 1024, 995, 1044, 1056, 17, 3, 0, 0, 1008, 1096, 1027, 1001, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4087, 4018, 4071, 4139, 4061, 4139, 4136] avg=4096.0000 max=4139 max_rank=4 min=4018 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16475 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1014, 1029, 1023, 1008, 0, 0, 0, 0, 1070, 1027, 1028, 1027, 27, 11, 0, 0, 1050, 975, 1031, 1016, 2, 0, 0, 0, 1021, 1065, 1027, 1025, 0, 0, 0, 0, 1053, 1019, 973, 1070, 0, 0, 0, 0, 971, 1025, 1001, 1047, 0, 0, 0, 0, 1011, 1034, 1008, 978, 18, 3, 0, 0, 1037, 1044, 1009, 987, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4190, 4074, 4138, 4115, 4044, 4052, 4081] avg=4096.0000 max=4190 max_rank=1 min=4044 min_rank=5 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1056, 1032, 1039, 1022, 0, 0, 0, 0, 991, 1013, 1023, 975, 29, 10, 0, 0, 1014, 1022, 1059, 1013, 2, 0, 0, 0, 1008, 1010, 1006, 1070, 0, 0, 0, 0, 981, 1011, 1019, 1044, 0, 0, 0, 0, 1039, 994, 1065, 1045, 0, 0, 0, 0, 1041, 1029, 955, 1027, 17, 3, 0, 0, 1012, 1035, 1031, 1022, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4041, 4110, 4094, 4055, 4143, 4072, 4104] avg=4096.0000 max=4149 max_rank=0 min=4041 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1005, 989, 1026, 1041, 0, 0, 0, 0, 1052, 975, 1028, 1070, 33, 15, 0, 0, 1012, 1039, 1038, 968, 4, 0, 0, 0, 1060, 1008, 1004, 1026, 0, 0, 0, 0, 1067, 967, 993, 1011, 6, 0, 0, 0, 1018, 1023, 1040, 1061, 0, 0, 0, 0, 1005, 1036, 997, 1011, 22, 9, 0, 0, 1035, 1038, 1023, 1004, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4173, 4061, 4098, 4044, 4142, 4080, 4109] avg=4096.0000 max=4173 max_rank=1 min=4044 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[992, 1010, 1003, 1039, 0, 0, 0, 0, 1019, 999, 1006, 997, 28, 10, 0, 0, 1015, 1028, 1044, 1052, 2, 0, 0, 0, 1015, 1029, 971, 1005, 0, 0, 0, 0, 1076, 1055, 1033, 1046, 0, 0, 0, 0, 1017, 1010, 1056, 1038, 0, 0, 0, 0, 1034, 1005, 1014, 1047, 17, 4, 0, 0, 1053, 1020, 977, 998, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4059, 4141, 4020, 4210, 4121, 4121, 4052] avg=4096.0000 max=4210 max_rank=4 min=4020 min_rank=3 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1030, 1045, 1082, 1045, 0, 0, 0, 0, 973, 1004, 1001, 946, 29, 10, 0, 0, 1001, 1039, 1028, 994, 2, 0, 0, 0, 1056, 999, 1015, 1037, 0, 0, 0, 0, 1063, 1027, 1039, 1017, 0, 0, 0, 0, 1048, 1014, 991, 1020, 0, 0, 0, 0, 1043, 1011, 1023, 984, 17, 4, 0, 0, 1049, 1049, 1043, 986, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4202, 3963, 4064, 4107, 4146, 4073, 4082, 4131] avg=4096.0000 max=4202 max_rank=0 min=3963 min_rank=1 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1003, 1026, 984, 1020, 0, 0, 0, 0, 1020, 1011, 1034, 1035, 27, 10, 0, 0, 1089, 1021, 1050, 1017, 2, 0, 0, 0, 1018, 1037, 1014, 1046, 0, 0, 0, 0, 1027, 977, 990, 971, 0, 0, 0, 0, 977, 1021, 1023, 1005, 0, 0, 0, 0, 1039, 1026, 1065, 1026, 17, 3, 0, 0, 1030, 1040, 1039, 1024, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4137, 4179, 4115, 3965, 4026, 4176, 4137] avg=4096.0000 max=4179 max_rank=2 min=3965 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8448, 8192, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1037, 978, 1028, 1026, 1073, 1053, 1022, 988, 974, 1011, 1022, 1030, 1034, 1033, 1026, 1017, 1000, 1077, 1017, 1005, 1021, 1003, 1006, 1021, 1014, 1054, 1045, 1003, 1034, 1002, 1061, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1019, 987, 1028, 1045, 1016, 1008, 1041, 1006, 970, 1028, 1038, 1005, 1037, 1038, 1031, 1037, 997, 1066, 1029, 1053, 976, 1041, 1043, 1024, 1015, 990, 1039, 1056, 1046, 988, 1041, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1006, 1086, 1054, 1021, 1014, 1022, 1071, 982, 996, 1020, 1045, 1031, 1051, 1006, 1021, 1055, 1029, 1052, 1052, 992, 1058, 1074, 1037, 1010, 964, 980, 999, 987, 1002, 1009, 1010, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[973, 1036, 1080, 1030, 1002, 1001, 1013, 1012, 1031, 1009, 1046, 1013, 1028, 1064, 1007, 1024, 1094, 1025, 1018, 1048, 1017, 985, 1009, 1022, 1027, 1032, 988, 1041, 1056, 1032, 1003, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1005, 1032, 1015, 1011, 1045, 1025, 1015, 1028, 1049, 989, 999, 1048, 1031, 987, 997, 1041, 1061, 978, 1040, 1035, 1022, 1062, 1032, 1034, 1031, 998, 1017, 1031, 1002, 1069, 1008, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1024, 1103, 1081, 1038, 1034, 1038, 1033, 1026, 1051, 1000, 1016, 1003, 981, 964, 999, 1021, 1021, 1014, 1012, 1070, 1004, 1034, 1003, 1014, 983, 986, 1038, 1066, 965, 1044, 1021, 1081] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1019, 1053, 1037, 1037, 1026, 1060, 1010, 1027, 1024, 1038, 1034, 1011, 1014, 1013, 1046, 1034, 1031, 1005, 1023, 1047, 1016, 1048, 1009, 1055, 987, 1016, 987, 1000, 1009, 1013, 997, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[985, 1004, 988, 1000, 1031, 1015, 1032, 1026, 1004, 998, 1047, 998, 1018, 1040, 1050, 994, 1030, 1060, 1007, 1026, 993, 1004, 1060, 1032, 1027, 1067, 1068, 1013, 1024, 990, 1049, 1088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1037, 978, 1016, 1026, 0, 0, 0, 0, 1070, 1053, 1022, 988, 0, 0, 0, 0, 974, 1011, 1022, 1030, 12, 8, 3, 0, 1034, 1033, 1026, 1017, 3, 0, 0, 0, 1000, 1046, 1017, 1005, 0, 0, 0, 0, 1021, 1003, 1006, 1021, 0, 0, 0, 0, 1014, 1054, 1045, 1003, 28, 0, 0, 0, 1034, 1002, 1061, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4133, 4060, 4113, 4068, 4051, 4144, 4142] avg=4096.0000 max=4144 max_rank=6 min=4051 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1019, 987, 1016, 1045, 0, 0, 0, 0, 1013, 1008, 1041, 1006, 0, 0, 0, 0, 970, 1028, 1038, 1005, 12, 8, 3, 0, 1037, 1038, 1031, 1037, 3, 0, 0, 0, 997, 1035, 1029, 1053, 0, 0, 0, 0, 976, 1041, 1043, 1024, 0, 0, 0, 0, 1015, 990, 1039, 1056, 28, 0, 0, 0, 1046, 988, 1041, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4068, 4064, 4146, 4114, 4084, 4128, 4097] avg=4096.0000 max=4146 max_rank=3 min=4064 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[973, 1036, 1068, 1030, 0, 0, 0, 0, 999, 1001, 1013, 1012, 0, 0, 0, 0, 1031, 1009, 1046, 1013, 12, 7, 3, 0, 1028, 1064, 1007, 1024, 3, 0, 0, 0, 1094, 995, 1018, 1048, 0, 0, 0, 0, 1017, 985, 1009, 1022, 0, 0, 0, 0, 1027, 1032, 988, 1041, 27, 0, 0, 0, 1056, 1032, 1003, 995, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4025, 4121, 4126, 4155, 4033, 4115, 4086] avg=4096.0000 max=4155 max_rank=4 min=4025 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1006, 1086, 1042, 1021, 0, 0, 0, 0, 1011, 1022, 1071, 982, 0, 0, 0, 0, 996, 1020, 1045, 1031, 12, 8, 3, 0, 1051, 1006, 1021, 1055, 3, 0, 0, 0, 1029, 1022, 1052, 992, 0, 0, 0, 0, 1058, 1074, 1037, 1010, 0, 0, 0, 0, 964, 980, 999, 987, 27, 0, 0, 0, 1002, 1009, 1010, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4086, 4115, 4136, 4095, 4179, 3957, 4045] avg=4096.0000 max=4179 max_rank=5 min=3957 min_rank=6 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16276 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1005, 1032, 1004, 1011, 0, 0, 0, 0, 1042, 1025, 1015, 1028, 0, 0, 0, 0, 1049, 989, 999, 1048, 11, 8, 3, 0, 1031, 987, 997, 1041, 3, 0, 0, 0, 1061, 950, 1040, 1035, 0, 0, 0, 0, 1022, 1062, 1032, 1034, 0, 0, 0, 0, 1031, 998, 1017, 1031, 25, 0, 0, 0, 1002, 1069, 1008, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4110, 4107, 4059, 4086, 4150, 4102, 4102] avg=4096.0000 max=4150 max_rank=5 min=4052 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1019, 1053, 1025, 1037, 0, 0, 0, 0, 1023, 1060, 1010, 1027, 0, 0, 0, 0, 1024, 1038, 1034, 1011, 12, 8, 3, 0, 1014, 1013, 1046, 1034, 3, 0, 0, 0, 1031, 976, 1023, 1047, 0, 0, 0, 0, 1016, 1048, 1009, 1055, 0, 0, 0, 0, 987, 1016, 987, 1000, 26, 0, 0, 0, 1009, 1013, 997, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4120, 4130, 4110, 4077, 4128, 4016, 4053] avg=4096.0000 max=4134 max_rank=0 min=4016 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1024, 1103, 1065, 1038, 0, 0, 0, 0, 1028, 1038, 1033, 1026, 0, 0, 0, 0, 1051, 1000, 1016, 1003, 16, 10, 6, 0, 981, 964, 999, 1021, 6, 1, 1, 0, 1021, 978, 1012, 1070, 0, 0, 0, 0, 1004, 1033, 1003, 1014, 0, 0, 0, 0, 983, 986, 1038, 1066, 30, 0, 0, 0, 965, 1044, 1021, 1070, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4230, 4125, 4102, 3973, 4081, 4054, 4103, 4100] avg=4096.0000 max=4230 max_rank=0 min=3973 min_rank=3 imbalance_ratio=1.032715 max_load_reduction=28538 max_load_reduction_pct=0.870911 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[985, 1004, 977, 1000, 0, 0, 0, 0, 1028, 1015, 1032, 1026, 0, 0, 0, 0, 1004, 998, 1047, 998, 11, 8, 3, 0, 1018, 1040, 1050, 994, 3, 0, 0, 0, 1030, 1029, 1007, 1026, 0, 0, 0, 0, 993, 1004, 1060, 1032, 0, 0, 0, 0, 1027, 1067, 1068, 1013, 28, 0, 0, 0, 1024, 990, 1049, 1080, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3966, 4101, 4069, 4105, 4092, 4089, 4203, 4143] avg=4096.0000 max=4203 max_rank=6 min=3966 min_rank=0 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[972, 1022, 1038, 1033, 1017, 1040, 1011, 990, 949, 1034, 1015, 1062, 1031, 1079, 1044, 993, 1070, 1024, 1022, 1014, 1024, 1053, 1024, 1031, 1025, 1008, 1020, 1038, 1049, 1014, 985, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[990, 1029, 1002, 1002, 1038, 1043, 1006, 1080, 1072, 1026, 990, 1001, 1043, 1049, 1025, 1019, 1008, 1034, 1027, 1077, 1043, 981, 1024, 1042, 1054, 985, 991, 1011, 1014, 1042, 1037, 983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[995, 1015, 992, 1001, 1029, 1070, 987, 1009, 960, 1023, 986, 993, 1026, 1031, 1008, 1045, 1029, 1008, 1067, 1101, 1038, 990, 1086, 1054, 1048, 1033, 985, 988, 1085, 1077, 965, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[997, 986, 1070, 1016, 994, 1040, 1056, 1000, 1058, 1046, 1036, 1021, 1069, 985, 1027, 996, 1019, 1017, 1075, 1079, 1020, 1005, 1017, 1050, 1035, 1015, 1024, 1034, 973, 1023, 973, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1082, 1037, 1038, 1021, 1027, 1026, 1022, 1022, 1052, 1022, 980, 1039, 992, 1013, 998, 998, 1043, 1012, 987, 1022, 1027, 1034, 1059, 1080, 1009, 1057, 1000, 1012, 1026, 1017, 1007, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[991, 986, 1014, 1022, 1012, 992, 974, 1024, 1062, 1041, 996, 1045, 1054, 991, 1022, 1060, 1062, 1028, 1054, 1001, 990, 1033, 1024, 1032, 1021, 1065, 1061, 1019, 1009, 1028, 1034, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1026, 1067, 1024, 976, 1015, 1070, 1045, 996, 977, 981, 1059, 992, 1060, 1060, 1021, 1053, 981, 1064, 1033, 1063, 1028, 1035, 1009, 1009, 958, 998, 1016, 1017, 1036, 1019, 1044, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[994, 1036, 978, 1027, 1038, 1036, 1039, 1038, 1050, 1033, 998, 1002, 988, 1017, 1056, 1029, 1016, 1037, 1028, 1066, 1021, 1014, 1027, 994, 1003, 1029, 1044, 1028, 1020, 1059, 1016, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[990, 1029, 1002, 1002, 36, 0, 0, 0, 1038, 1041, 1006, 1080, 0, 0, 0, 0, 1072, 1026, 990, 1001, 14, 7, 0, 0, 1030, 1049, 1025, 1019, 0, 0, 0, 0, 1008, 1034, 1027, 1027, 0, 0, 0, 0, 1043, 981, 1024, 1026, 0, 0, 0, 0, 1054, 985, 991, 1011, 9, 7, 0, 0, 1014, 1042, 1037, 983, 6, 2, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4165, 4110, 4123, 4096, 4074, 4057, 4084] avg=4096.0000 max=4165 max_rank=1 min=4057 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[972, 1022, 1038, 1033, 34, 0, 0, 0, 1017, 1038, 1011, 990, 0, 0, 0, 0, 949, 1034, 1015, 1062, 13, 6, 0, 0, 1018, 1079, 1044, 993, 0, 0, 0, 0, 1070, 1024, 1022, 967, 0, 0, 0, 0, 1024, 1053, 1024, 1016, 0, 0, 0, 0, 1025, 1008, 1020, 1038, 9, 7, 0, 0, 1049, 1014, 985, 1037, 6, 2, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4056, 4079, 4134, 4083, 4117, 4107, 4093] avg=4096.0000 max=4134 max_rank=3 min=4056 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[997, 986, 1070, 1016, 37, 0, 0, 0, 994, 1038, 1056, 1000, 0, 0, 0, 0, 1058, 1046, 1036, 1021, 14, 7, 0, 0, 1055, 985, 1027, 996, 0, 0, 0, 0, 1019, 1017, 1075, 1028, 0, 0, 0, 0, 1020, 1005, 1017, 1034, 0, 0, 0, 0, 1035, 1015, 1024, 1034, 9, 8, 0, 0, 973, 1023, 973, 1012, 6, 2, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4088, 4182, 4063, 4139, 4076, 4125, 3989] avg=4096.0000 max=4182 max_rank=2 min=3989 min_rank=7 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[995, 1015, 992, 1001, 37, 0, 0, 0, 1029, 1068, 987, 1009, 0, 0, 0, 0, 960, 1023, 986, 993, 14, 7, 0, 0, 1013, 1031, 1008, 1045, 0, 0, 0, 0, 1029, 1008, 1067, 1050, 0, 0, 0, 0, 1038, 990, 1086, 1038, 0, 0, 0, 0, 1048, 1033, 985, 988, 9, 7, 0, 0, 1085, 1077, 965, 1044, 6, 2, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4093, 3983, 4097, 4154, 4152, 4070, 4179] avg=4096.0000 max=4179 max_rank=7 min=3983 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16555 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1082, 1037, 1038, 1021, 35, 0, 0, 0, 1027, 1024, 1022, 1022, 0, 0, 0, 0, 1052, 1022, 980, 1039, 13, 7, 0, 0, 979, 1013, 998, 998, 0, 0, 0, 0, 1043, 1012, 987, 974, 0, 0, 0, 0, 1027, 1034, 1059, 1064, 0, 0, 0, 0, 1009, 1057, 1000, 1012, 9, 7, 0, 0, 1026, 1017, 1007, 1007, 6, 2, 0, 0] rerouted_tokens_per_ep_rank=[4213, 4095, 4113, 3988, 4016, 4184, 4094, 4065] avg=4096.0000 max=4213 max_rank=0 min=3988 min_rank=3 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[991, 986, 1014, 1022, 34, 0, 0, 0, 1012, 990, 974, 1024, 0, 0, 0, 0, 1062, 1041, 996, 1045, 13, 6, 0, 0, 1040, 991, 1022, 1060, 0, 0, 0, 0, 1062, 1028, 1054, 954, 0, 0, 0, 0, 990, 1033, 1024, 1017, 0, 0, 0, 0, 1021, 1065, 1061, 1019, 9, 8, 0, 0, 1009, 1028, 1034, 1021, 6, 2, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4000, 4163, 4113, 4098, 4064, 4183, 4100] avg=4096.0000 max=4183 max_rank=6 min=4000 min_rank=1 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 1067, 1024, 976, 40, 0, 0, 0, 1015, 1066, 1045, 996, 0, 0, 0, 0, 977, 981, 1059, 992, 16, 10, 0, 0, 1039, 1060, 1021, 1053, 0, 0, 0, 0, 981, 1064, 1033, 1007, 0, 0, 0, 0, 1028, 1035, 1009, 987, 0, 0, 0, 0, 958, 998, 1016, 1017, 12, 12, 0, 0, 1036, 1019, 1044, 1036, 9, 4, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4122, 4035, 4173, 4085, 4059, 4013, 4148] avg=4096.0000 max=4173 max_rank=3 min=4013 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[994, 1036, 978, 1027, 36, 0, 0, 0, 1038, 1034, 1039, 1038, 0, 0, 0, 0, 1050, 1033, 998, 1002, 14, 6, 0, 0, 975, 1017, 1056, 1029, 0, 0, 0, 0, 1016, 1037, 1028, 1016, 0, 0, 0, 0, 1021, 1014, 1027, 980, 0, 0, 0, 0, 1003, 1029, 1044, 1028, 8, 7, 0, 0, 1020, 1059, 1016, 1007, 6, 2, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4149, 4103, 4077, 4097, 4042, 4119, 4110] avg=4096.0000 max=4149 max_rank=1 min=4042 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8064, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1056, 1007, 996, 1023, 980, 989, 1031, 1055, 1023, 1049, 1037, 1034, 1042, 1013, 1030, 1071, 1025, 979, 1039, 1033, 993, 1007, 1027, 1025, 999, 1022, 1048, 1049, 1030, 1030, 989, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1028, 1026, 1040, 1039, 1007, 1030, 1068, 1017, 1021, 986, 992, 1029, 972, 1046, 1005, 1029, 1021, 1052, 984, 1060, 1074, 1022, 1045, 1075, 1035, 1006, 1038, 1010, 1010, 1006, 984, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1016, 1079, 976, 1051, 1053, 1050, 972, 977, 1006, 1052, 1002, 1005, 984, 1086, 1033, 1000, 1085, 1060, 1065, 1009, 1011, 1020, 1058, 1054, 1042, 1036, 991, 996, 1023, 966, 1023, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1029, 1036, 1031, 1061, 1031, 1037, 1015, 990, 1024, 997, 1025, 1021, 962, 1009, 1044, 1017, 1051, 1048, 1044, 1015, 1007, 1029, 1014, 1041, 1031, 1029, 1015, 1019, 1027, 1039, 1024, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1029, 1044, 1010, 993, 1014, 1044, 1018, 1078, 1062, 991, 1008, 1084, 1050, 1007, 1012, 1044, 1056, 933, 1040, 985, 1044, 994, 1006, 1040, 1025, 1012, 1026, 1029, 1071, 979, 989, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1057, 1036, 1036, 1021, 1068, 975, 990, 1021, 1068, 1045, 995, 1002, 1029, 1012, 1040, 1057, 1007, 1045, 1020, 1056, 1010, 1004, 1014, 1025, 1015, 998, 1008, 1008, 985, 1050, 1050, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1038, 1008, 1038, 1012, 1050, 1004, 1020, 1036, 1055, 1027, 1019, 1032, 1011, 1036, 1042, 1001, 1094, 1001, 1039, 1064, 1065, 1006, 965, 1003, 1006, 1016, 980, 1031, 1025, 971, 1056, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1069, 1034, 1014, 1013, 1001, 1062, 982, 1027, 1027, 1003, 1025, 1015, 1053, 998, 1045, 1027, 1062, 1037, 1030, 1027, 1002, 1031, 1036, 1044, 1015, 1044, 1012, 1044, 992, 1045, 985, 967] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1008, 1026, 1040, 1039, 0, 0, 0, 0, 1007, 1030, 1068, 1017, 4, 2, 2, 0, 1021, 986, 992, 1029, 0, 0, 0, 0, 972, 1046, 1001, 1029, 0, 0, 0, 0, 985, 1052, 984, 1060, 0, 0, 0, 0, 1074, 1022, 1045, 1073, 0, 0, 0, 0, 1035, 1006, 1038, 1010, 16, 0, 0, 0, 1010, 1006, 984, 1011, 36, 2, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4130, 4028, 4048, 4081, 4214, 4105, 4049] avg=4096.0000 max=4214 max_rank=5 min=4028 min_rank=2 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1035, 1007, 996, 1023, 0, 0, 0, 0, 980, 989, 1031, 1055, 4, 2, 1, 0, 1023, 1049, 1037, 1034, 0, 0, 0, 0, 1042, 1013, 1026, 1071, 0, 0, 0, 0, 989, 979, 1039, 1033, 0, 0, 0, 0, 993, 1007, 1027, 1024, 0, 0, 0, 0, 999, 1022, 1048, 1049, 16, 0, 0, 0, 1030, 1030, 989, 1037, 36, 3, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4062, 4143, 4152, 4040, 4051, 4134, 4125] avg=4096.0000 max=4152 max_rank=3 min=4040 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1009, 1036, 1031, 1061, 0, 0, 0, 0, 1031, 1037, 1015, 990, 4, 2, 2, 0, 1024, 997, 1025, 1021, 0, 0, 0, 0, 962, 1009, 1040, 1017, 0, 0, 0, 0, 1014, 1048, 1044, 1015, 0, 0, 0, 0, 1007, 1029, 1014, 1039, 0, 0, 0, 0, 1031, 1029, 1015, 1019, 16, 0, 0, 0, 1027, 1039, 1024, 1006, 37, 2, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4081, 4067, 4028, 4121, 4089, 4110, 4135] avg=4096.0000 max=4137 max_rank=0 min=4028 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1036, 1036, 1036, 1021, 0, 0, 0, 0, 1068, 975, 990, 1021, 4, 2, 1, 0, 1068, 1045, 995, 1002, 0, 0, 0, 0, 1029, 1012, 1036, 1057, 0, 0, 0, 0, 972, 1045, 1020, 1056, 0, 0, 0, 0, 1010, 1004, 1014, 1024, 0, 0, 0, 0, 1015, 998, 1008, 1008, 16, 0, 0, 0, 985, 1050, 1050, 1021, 35, 3, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4061, 4110, 4134, 4093, 4052, 4045, 4144] avg=4096.0000 max=4144 max_rank=7 min=4045 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1018, 1008, 1038, 1012, 0, 0, 0, 0, 1050, 1004, 1020, 1036, 4, 2, 1, 0, 1055, 1027, 1019, 1032, 0, 0, 0, 0, 1011, 1036, 1038, 1001, 0, 0, 0, 0, 1056, 1001, 1039, 1064, 0, 0, 0, 0, 1065, 1006, 965, 1002, 0, 0, 0, 0, 1006, 1016, 980, 1031, 16, 0, 0, 0, 1025, 971, 1056, 1017, 38, 2, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4117, 4133, 4086, 4160, 4038, 4049, 4109] avg=4096.0000 max=4160 max_rank=4 min=4038 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[996, 1079, 976, 1051, 0, 0, 0, 0, 1053, 1050, 972, 977, 4, 2, 2, 0, 1006, 1052, 1002, 1005, 0, 0, 0, 0, 984, 1086, 1029, 1000, 0, 0, 0, 0, 1047, 1060, 1065, 1009, 0, 0, 0, 0, 1011, 1020, 1058, 1052, 0, 0, 0, 0, 1042, 1036, 991, 996, 16, 0, 0, 0, 1023, 966, 1023, 987, 38, 2, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4060, 4065, 4099, 4181, 4141, 4081, 4039] avg=4096.0000 max=4181 max_rank=4 min=4039 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[995, 1044, 1010, 993, 0, 0, 0, 0, 1014, 1044, 1018, 1078, 11, 7, 5, 0, 1062, 991, 1008, 1084, 7, 0, 0, 0, 1050, 1007, 1001, 1044, 0, 0, 0, 0, 1015, 933, 1040, 985, 0, 0, 0, 0, 1044, 994, 1006, 1028, 0, 0, 0, 0, 1025, 1012, 1026, 1029, 20, 0, 0, 0, 1071, 979, 989, 1051, 41, 7, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4177, 4152, 4102, 3973, 4072, 4112, 4138] avg=4096.0000 max=4177 max_rank=1 min=3973 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1047, 1034, 1014, 1013, 0, 0, 0, 0, 1001, 1062, 982, 1027, 4, 2, 2, 0, 1027, 1003, 1025, 1015, 0, 0, 0, 0, 1053, 998, 1041, 1027, 0, 0, 0, 0, 1025, 1037, 1030, 1027, 0, 0, 0, 0, 1002, 1031, 1036, 1042, 0, 0, 0, 0, 1015, 1044, 1012, 1044, 17, 0, 0, 0, 992, 1045, 985, 967, 37, 3, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4080, 4070, 4119, 4119, 4111, 4132, 4029] avg=4096.0000 max=4132 max_rank=6 min=4029 min_rank=7 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[996, 1016, 1027, 1024, 1061, 986, 1044, 1053, 1027, 1037, 1045, 1007, 1038, 1002, 1057, 1005, 1054, 1064, 1059, 1012, 1002, 1045, 1009, 981, 1001, 1033, 1001, 1034, 1013, 966, 1057, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[992, 1042, 1073, 969, 1065, 995, 1055, 1007, 1076, 995, 1019, 1022, 1019, 1052, 1025, 1031, 1009, 1023, 1021, 1010, 1039, 1005, 1047, 967, 1010, 1033, 999, 1038, 988, 1057, 1057, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1037, 1003, 1005, 1064, 997, 1036, 1025, 1019, 981, 1075, 992, 1008, 1004, 1036, 1048, 1027, 1048, 1012, 1028, 1037, 988, 1038, 1041, 1054, 1027, 1017, 1019, 1032, 1005, 1047, 990, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1033, 1022, 999, 1003, 1049, 992, 1013, 1028, 1022, 1064, 1049, 1065, 1022, 952, 1054, 1021, 1012, 1007, 1011, 1024, 995, 1012, 1013, 1068, 1059, 1015, 1028, 1045, 1025, 1087, 968, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1041, 1021, 1022, 1041, 986, 991, 996, 1021, 1034, 997, 1021, 1058, 1024, 1033, 1068, 1011, 996, 1071, 1011, 1029, 1036, 999, 1037, 1026, 1034, 1040, 1008, 1022, 1021, 1050, 974, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1008, 1070, 1016, 1039, 1027, 1047, 1020, 1006, 1055, 976, 1003, 1004, 1034, 1020, 1017, 1018, 1011, 1079, 990, 1079, 1055, 1015, 984, 1051, 1059, 999, 1023, 1000, 1000, 1050, 1004, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1028, 997, 1030, 1014, 1047, 1018, 970, 1029, 1065, 1040, 1017, 1012, 1047, 1004, 1009, 1013, 1039, 1008, 987, 1026, 1016, 1019, 1059, 1020, 1003, 1016, 1082, 996, 1010, 1061, 1027, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1015, 1017, 1046, 1044, 974, 1031, 1070, 1055, 1051, 1045, 1033, 1065, 1068, 1031, 1026, 1029, 978, 1010, 998, 1039, 990, 1041, 957, 1030, 1015, 958, 1009, 1015, 1082, 1034, 1004, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1037, 1003, 1005, 1064, 1, 0, 0, 0, 997, 1036, 1025, 1019, 5, 1, 0, 0, 959, 1075, 992, 1008, 0, 0, 0, 0, 1004, 1036, 1039, 1027, 0, 0, 0, 0, 1048, 1011, 1028, 1037, 0, 0, 0, 0, 988, 1038, 1041, 1054, 15, 0, 0, 0, 1027, 1017, 1019, 1032, 7, 4, 0, 0, 1005, 1046, 990, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4083, 4034, 4106, 4124, 4136, 4106, 4069] avg=4096.0000 max=4136 max_rank=5 min=4034 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1041, 1021, 1022, 1041, 1, 0, 0, 0, 986, 991, 996, 1021, 5, 1, 0, 0, 1011, 997, 1021, 1058, 0, 0, 0, 0, 1024, 1033, 1059, 1011, 0, 0, 0, 0, 996, 1070, 1011, 1029, 0, 0, 0, 0, 1036, 999, 1037, 1026, 16, 0, 0, 0, 1034, 1040, 1008, 1022, 7, 4, 0, 0, 1021, 1049, 974, 1049, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4000, 4087, 4127, 4106, 4114, 4115, 4093] avg=4096.0000 max=4127 max_rank=3 min=4000 min_rank=1 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1033, 1022, 999, 1003, 1, 0, 0, 0, 1049, 992, 1013, 1028, 5, 1, 0, 0, 1000, 1064, 1049, 1065, 0, 0, 0, 0, 1022, 952, 1045, 1021, 0, 0, 0, 0, 1012, 1006, 1011, 1024, 0, 0, 0, 0, 995, 1012, 1013, 1068, 15, 0, 0, 0, 1059, 1015, 1028, 1045, 7, 4, 0, 0, 1025, 1086, 968, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4088, 4178, 4040, 4053, 4103, 4158, 4090] avg=4096.0000 max=4178 max_rank=2 min=4040 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[996, 1016, 1027, 1024, 1, 0, 0, 0, 1061, 986, 1044, 1053, 5, 1, 0, 0, 1005, 1037, 1045, 1007, 0, 0, 0, 0, 1038, 1002, 1048, 1005, 0, 0, 0, 0, 1054, 1063, 1059, 1012, 0, 0, 0, 0, 1002, 1045, 1009, 981, 15, 0, 0, 0, 1001, 1033, 1001, 1034, 7, 4, 0, 0, 1013, 965, 1057, 1012, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4150, 4094, 4093, 4188, 4052, 4080, 4047] avg=4096.0000 max=4188 max_rank=4 min=4047 min_rank=7 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[992, 1042, 1073, 969, 6, 1, 0, 0, 1065, 995, 1055, 1007, 7, 6, 0, 0, 1043, 995, 1019, 1022, 0, 0, 0, 0, 1019, 1052, 1011, 1031, 0, 0, 0, 0, 1009, 1016, 1021, 1010, 0, 0, 0, 0, 1039, 1005, 1047, 967, 20, 0, 0, 0, 1010, 1033, 999, 1038, 13, 7, 0, 0, 988, 1051, 1057, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4135, 4079, 4113, 4056, 4078, 4100, 4124] avg=4096.0000 max=4135 max_rank=1 min=4056 min_rank=4 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1008, 1070, 1016, 1039, 1, 0, 0, 0, 1027, 1047, 1020, 1006, 5, 1, 0, 0, 1032, 976, 1003, 1004, 0, 0, 0, 0, 1034, 1020, 1008, 1018, 0, 0, 0, 0, 1011, 1078, 990, 1079, 0, 0, 0, 0, 1055, 1015, 984, 1051, 16, 0, 0, 0, 1059, 999, 1023, 1000, 7, 4, 0, 0, 1000, 1049, 1004, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4106, 4015, 4080, 4158, 4121, 4092, 4062] avg=4096.0000 max=4158 max_rank=4 min=4015 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1028, 997, 1030, 1014, 1, 0, 0, 0, 1047, 1018, 970, 1029, 5, 1, 0, 0, 1041, 1040, 1017, 1012, 0, 0, 0, 0, 1047, 1004, 1000, 1013, 0, 0, 0, 0, 1039, 1007, 987, 1026, 0, 0, 0, 0, 1016, 1019, 1059, 1020, 16, 0, 0, 0, 1003, 1016, 1082, 996, 8, 4, 0, 0, 1010, 1060, 1027, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4070, 4110, 4064, 4059, 4130, 4109, 4156] avg=4096.0000 max=4156 max_rank=7 min=4059 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1015, 1017, 1046, 1044, 1, 0, 0, 0, 974, 1031, 1070, 1055, 5, 1, 0, 0, 1028, 1045, 1033, 1065, 0, 0, 0, 0, 1068, 1031, 1017, 1029, 0, 0, 0, 0, 978, 1009, 998, 1039, 0, 0, 0, 0, 990, 1041, 957, 1030, 16, 0, 0, 0, 1015, 958, 1009, 1015, 7, 4, 0, 0, 1082, 1033, 1004, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4136, 4171, 4145, 4024, 4034, 4008, 4127] avg=4096.0000 max=4171 max_rank=2 min=4008 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8448, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[992, 1003, 1025, 1033, 994, 1035, 1041, 1042, 1035, 1004, 1041, 995, 1021, 1022, 1013, 984, 1068, 1017, 1016, 1088, 1019, 1055, 1004, 1003, 1041, 1056, 1009, 1032, 1027, 1012, 978, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1005, 1014, 1069, 1031, 1003, 1052, 974, 1019, 976, 1041, 1018, 1066, 1039, 1035, 999, 1002, 1051, 1088, 1030, 1005, 1003, 1036, 1073, 1062, 967, 1000, 1037, 1003, 977, 1049, 1043, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[997, 1060, 1069, 1005, 1072, 1031, 1050, 1028, 987, 994, 1055, 1049, 1030, 1040, 1022, 1013, 992, 1050, 1069, 1005, 975, 1061, 1044, 1036, 976, 1015, 980, 989, 1019, 1029, 1016, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[960, 1002, 1039, 1014, 1044, 999, 1021, 1051, 1017, 1014, 1016, 1058, 997, 1037, 985, 1036, 1064, 1046, 1076, 994, 1054, 1026, 997, 969, 1024, 1039, 1012, 1022, 1013, 1057, 1055, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1052, 1028, 1087, 1029, 942, 1002, 1049, 990, 1042, 1065, 1002, 1026, 1016, 1004, 994, 1035, 1018, 1006, 1016, 1039, 1055, 1017, 1063, 1044, 1030, 999, 983, 1039, 1040, 1056, 1005, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1019, 1033, 979, 1030, 1058, 995, 1035, 1032, 1043, 1052, 951, 1018, 1032, 1031, 1012, 1048, 1015, 1065, 1041, 1016, 1015, 1026, 1018, 1079, 1040, 983, 962, 1031, 1040, 992, 1047, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1003, 980, 1029, 1038, 1023, 1036, 1043, 1027, 1006, 1025, 1005, 994, 1014, 1065, 1015, 1019, 1053, 1031, 1039, 1016, 1014, 1075, 1039, 1001, 1066, 986, 1005, 1024, 990, 1016, 1038, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1048, 990, 1017, 982, 994, 1036, 1029, 1025, 1003, 1053, 1004, 1003, 1046, 1036, 1015, 1003, 1022, 1029, 1006, 950, 992, 1029, 1068, 1015, 1063, 1105, 1050, 1041, 988, 1058, 1041, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[992, 1003, 1025, 1033, 9, 3, 0, 0, 994, 1035, 1041, 1042, 0, 0, 0, 0, 1035, 1004, 1041, 995, 11, 2, 0, 0, 1021, 1022, 1013, 984, 13, 0, 0, 0, 1068, 987, 1016, 1088, 0, 0, 0, 0, 1019, 1031, 1004, 1003, 0, 0, 0, 0, 1041, 1056, 1009, 1032, 19, 0, 0, 0, 1027, 1009, 978, 1063, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4112, 4088, 4053, 4159, 4057, 4157, 4077] avg=4096.0000 max=4159 max_rank=4 min=4053 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1019, 1033, 979, 1030, 9, 3, 0, 0, 1058, 995, 1035, 1032, 0, 0, 0, 0, 1043, 1052, 951, 1018, 12, 1, 0, 0, 1032, 1031, 1012, 1048, 13, 0, 0, 0, 1015, 1033, 1041, 1016, 0, 0, 0, 0, 1015, 1003, 1018, 1079, 0, 0, 0, 0, 1040, 983, 962, 1031, 20, 0, 0, 0, 1040, 989, 1047, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4120, 4077, 4136, 4105, 4115, 4036, 4106] avg=4096.0000 max=4136 max_rank=3 min=4036 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1048, 990, 1017, 982, 9, 3, 0, 0, 994, 1036, 1029, 1025, 0, 0, 0, 0, 1003, 1053, 1004, 1003, 11, 1, 0, 0, 1046, 1036, 1015, 1003, 13, 0, 0, 0, 1022, 999, 1006, 950, 0, 0, 0, 0, 992, 1006, 1068, 1015, 0, 0, 0, 0, 1063, 1105, 1050, 1041, 19, 0, 0, 0, 988, 1055, 1041, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4084, 4075, 4113, 3977, 4081, 4278, 4111] avg=4096.0000 max=4278 max_rank=6 min=3977 min_rank=4 imbalance_ratio=1.044434 max_load_reduction=28490 max_load_reduction_pct=0.869446 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[960, 1002, 1039, 1014, 9, 3, 0, 0, 1044, 999, 1021, 1051, 0, 0, 0, 0, 1017, 1014, 1016, 1058, 11, 1, 0, 0, 997, 1037, 985, 1036, 13, 0, 0, 0, 1064, 1016, 1076, 994, 0, 0, 0, 0, 1054, 1003, 997, 969, 0, 0, 0, 0, 1024, 1039, 1012, 1022, 19, 0, 0, 0, 1013, 1054, 1055, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4115, 4117, 4068, 4150, 4023, 4116, 4152] avg=4096.0000 max=4152 max_rank=7 min=4023 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[997, 1060, 1069, 1005, 12, 6, 4, 0, 1072, 1031, 1046, 1028, 0, 0, 0, 0, 987, 994, 1055, 1049, 15, 7, 0, 0, 1030, 1040, 1022, 1013, 17, 0, 0, 0, 992, 1011, 1069, 1005, 0, 0, 0, 0, 975, 1025, 1044, 1036, 0, 0, 0, 0, 976, 1015, 980, 989, 24, 0, 0, 0, 1019, 1023, 1016, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4177, 4107, 4122, 4077, 4080, 3984, 4068] avg=4096.0000 max=4177 max_rank=1 min=3984 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16209 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1052, 1028, 1087, 1029, 9, 3, 0, 0, 942, 1002, 1049, 990, 0, 0, 0, 0, 1042, 1065, 1002, 1026, 11, 1, 0, 0, 1016, 1004, 994, 1035, 13, 0, 0, 0, 1018, 976, 1016, 1039, 0, 0, 0, 0, 1055, 994, 1063, 1044, 0, 0, 0, 0, 1030, 999, 983, 1039, 19, 0, 0, 0, 1040, 1053, 1005, 995, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4208, 3983, 4147, 4062, 4049, 4156, 4070, 4093] avg=4096.0000 max=4208 max_rank=0 min=3983 min_rank=1 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1003, 980, 1029, 1038, 9, 3, 0, 0, 1023, 1036, 1043, 1027, 0, 0, 0, 0, 1006, 1025, 1005, 994, 11, 2, 0, 0, 1014, 1065, 1015, 1019, 13, 0, 0, 0, 1053, 1001, 1039, 1016, 0, 0, 0, 0, 1014, 1051, 1039, 1001, 0, 0, 0, 0, 1066, 986, 1005, 1024, 19, 0, 0, 0, 990, 1013, 1038, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4129, 4043, 4126, 4109, 4105, 4100, 4094] avg=4096.0000 max=4129 max_rank=1 min=4043 min_rank=2 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1005, 1014, 1069, 1031, 9, 3, 0, 0, 1003, 1052, 974, 1019, 0, 0, 0, 0, 976, 1041, 1018, 1066, 12, 1, 0, 0, 1039, 1035, 999, 1002, 13, 0, 0, 0, 1051, 1056, 1030, 1005, 0, 0, 0, 0, 1003, 1013, 1073, 1062, 0, 0, 0, 0, 967, 1000, 1037, 1003, 20, 0, 0, 0, 977, 1046, 1043, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4048, 4114, 4088, 4142, 4151, 4027, 4067] avg=4096.0000 max=4151 max_rank=5 min=4027 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8064, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1046, 1038, 1038, 984, 1044, 1006, 1040, 1026, 1034, 1024, 1029, 991, 1017, 1036, 1026, 1033, 1005, 1001, 1066, 1022, 991, 1017, 1024, 995, 1056, 1032, 997, 993, 1075, 1056, 1016, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1028, 1049, 1015, 1018, 1027, 1031, 1006, 1013, 1033, 1070, 1006, 1009, 1016, 1060, 1015, 999, 1057, 998, 1039, 1010, 1032, 982, 1030, 1040, 1001, 1013, 1045, 1011, 1054, 975, 1070, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1002, 1020, 1042, 1033, 1020, 1055, 984, 1014, 992, 1010, 998, 1011, 971, 1047, 1079, 1048, 1013, 1056, 1002, 1020, 1093, 982, 1057, 1018, 1006, 1015, 978, 1076, 1031, 1062, 1039, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1036, 1055, 1043, 1009, 1089, 1051, 1025, 1049, 968, 975, 1026, 1109, 1030, 1020, 1031, 1037, 953, 979, 1013, 1007, 1024, 1001, 987, 1031, 1031, 1022, 1033, 1077, 1044, 992, 995, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1068, 991, 1014, 1087, 1011, 999, 1026, 998, 1029, 1032, 1054, 1045, 1010, 981, 1002, 1035, 1053, 1019, 1033, 1024, 1039, 999, 1046, 1004, 1027, 1039, 1015, 1055, 1011, 998, 1006, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1016, 1023, 1061, 1031, 1077, 972, 991, 1061, 1086, 1024, 1075, 1010, 1054, 986, 992, 1003, 1009, 973, 1016, 1032, 1023, 1044, 1014, 973, 1011, 1023, 1058, 1010, 1046, 1041, 967, 1066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1014, 1026, 1017, 996, 1092, 1018, 1028, 1007, 1010, 1013, 1001, 1060, 1011, 1016, 979, 1059, 1039, 1035, 1060, 1022, 1014, 1039, 980, 1034, 1072, 1010, 1031, 1049, 1046, 1023, 969, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[998, 1059, 1054, 1045, 990, 1030, 1018, 1012, 1026, 1050, 1026, 1009, 1009, 936, 1023, 970, 962, 1032, 1023, 1002, 1066, 1048, 1010, 1032, 986, 1030, 1058, 1066, 1043, 999, 1095, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1046, 1038, 1015, 984, 0, 0, 0, 0, 1039, 1006, 1040, 1026, 0, 0, 0, 0, 1034, 1024, 1029, 984, 0, 0, 0, 0, 1017, 1036, 1026, 1033, 23, 5, 0, 0, 1005, 1001, 1066, 1022, 12, 9, 1, 0, 991, 1017, 1024, 995, 6, 5, 0, 0, 1056, 1032, 997, 976, 0, 0, 0, 0, 1066, 1056, 1016, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4111, 4071, 4140, 4116, 4038, 4061, 4148] avg=4096.0000 max=4148 max_rank=7 min=4038 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1028, 1049, 992, 1018, 0, 0, 0, 0, 1022, 1031, 1006, 1013, 0, 0, 0, 0, 1033, 1070, 1006, 1002, 0, 0, 0, 0, 1016, 1060, 1015, 999, 23, 5, 0, 0, 1057, 998, 1039, 1010, 13, 9, 1, 0, 1032, 982, 1030, 1040, 6, 5, 0, 0, 1001, 1013, 1045, 993, 0, 0, 0, 0, 1045, 975, 1070, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4072, 4111, 4118, 4127, 4095, 4052, 4106] avg=4096.0000 max=4127 max_rank=4 min=4052 min_rank=6 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1002, 1020, 1019, 1033, 0, 0, 0, 0, 1015, 1055, 984, 1014, 0, 0, 0, 0, 992, 1010, 998, 1004, 0, 0, 0, 0, 971, 1047, 1079, 1048, 23, 6, 0, 0, 1013, 1056, 1002, 1020, 14, 9, 1, 0, 1093, 982, 1057, 1018, 6, 5, 0, 0, 1006, 1015, 978, 1056, 0, 0, 0, 0, 1022, 1062, 1039, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4068, 4004, 4174, 4115, 4161, 4055, 4117] avg=4096.0000 max=4174 max_rank=3 min=4004 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1036, 1055, 1020, 1009, 0, 0, 0, 0, 1084, 1051, 1025, 1049, 0, 0, 0, 0, 968, 975, 1026, 1101, 0, 0, 0, 0, 1030, 1020, 1031, 1037, 23, 6, 0, 0, 953, 979, 1013, 1007, 14, 9, 1, 0, 1024, 1001, 987, 1031, 7, 5, 0, 0, 1031, 1022, 1033, 1057, 0, 0, 0, 0, 1035, 992, 995, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4209, 4070, 4147, 3976, 4055, 4143, 4048] avg=4096.0000 max=4209 max_rank=1 min=3976 min_rank=4 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16222 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1068, 991, 988, 1087, 0, 0, 0, 0, 1003, 999, 1026, 998, 0, 0, 0, 0, 1029, 1032, 1054, 1029, 0, 0, 0, 0, 1010, 981, 1002, 1035, 26, 10, 0, 0, 1053, 1019, 1033, 1024, 17, 11, 3, 0, 1039, 999, 1046, 1004, 13, 8, 0, 0, 1027, 1039, 1015, 1028, 0, 0, 0, 0, 1000, 998, 1006, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4026, 4144, 4064, 4160, 4109, 4109, 4022] avg=4096.0000 max=4160 max_rank=4 min=4022 min_rank=7 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1014, 1026, 994, 996, 0, 0, 0, 0, 1087, 1018, 1028, 1007, 0, 0, 0, 0, 1010, 1013, 1001, 1052, 0, 0, 0, 0, 1011, 1016, 979, 1059, 23, 6, 0, 0, 1039, 1035, 1060, 1022, 13, 9, 1, 0, 1014, 1039, 980, 1034, 7, 5, 0, 0, 1072, 1010, 1031, 1030, 0, 0, 0, 0, 1037, 1023, 969, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4140, 4076, 4094, 4179, 4079, 4143, 4027] avg=4096.0000 max=4179 max_rank=4 min=4027 min_rank=7 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[998, 1059, 1031, 1045, 0, 0, 0, 0, 986, 1030, 1018, 1012, 0, 0, 0, 0, 1026, 1050, 1026, 1002, 0, 0, 0, 0, 1009, 936, 1023, 970, 23, 6, 0, 0, 962, 1032, 1023, 1002, 13, 9, 1, 0, 1066, 1048, 1010, 1032, 6, 4, 0, 0, 986, 1030, 1058, 1047, 0, 0, 0, 0, 1034, 999, 1095, 1061, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4046, 4104, 3967, 4042, 4166, 4121, 4189] avg=4096.0000 max=4189 max_rank=7 min=3967 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1016, 1023, 1037, 1031, 0, 0, 0, 0, 1072, 972, 991, 1061, 0, 0, 0, 0, 1086, 1024, 1075, 1003, 0, 0, 0, 0, 1054, 986, 992, 1003, 24, 5, 0, 0, 1009, 973, 1016, 1032, 13, 9, 1, 0, 1023, 1044, 1014, 973, 6, 5, 0, 0, 1011, 1023, 1058, 992, 0, 0, 0, 0, 1037, 1041, 967, 1066, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4096, 4188, 4064, 4053, 4065, 4084, 4111] avg=4096.0000 max=4188 max_rank=2 min=4053 min_rank=4 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1023, 1044, 1053, 1052, 1029, 962, 1034, 1026, 1045, 1010, 1048, 1041, 998, 1014, 1059, 997, 1023, 1020, 996, 1018, 1077, 1015, 1020, 1034, 1008, 1085, 1007, 994, 1025, 981, 1011, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1038, 1063, 1031, 1009, 1011, 1044, 1084, 1005, 1045, 989, 954, 1014, 983, 1044, 982, 1000, 994, 980, 1040, 988, 1067, 1021, 1083, 996, 1010, 1060, 1041, 1044, 1023, 1032, 1056, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1017, 1022, 1001, 1036, 1047, 1020, 1023, 1000, 1021, 1025, 970, 1009, 1039, 1009, 1049, 1034, 1041, 1026, 994, 1063, 1040, 999, 1040, 1001, 1073, 982, 1043, 1053, 1022, 994, 1014, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[994, 1002, 1038, 999, 1013, 1045, 988, 1007, 1042, 1000, 1001, 1030, 1033, 1065, 1035, 1024, 1028, 1039, 1050, 993, 1029, 1027, 1015, 996, 1000, 1044, 1010, 1015, 1003, 1052, 1082, 1069] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1049, 1059, 961, 1026, 1009, 974, 1013, 1044, 1066, 1024, 1045, 1033, 1002, 1055, 1025, 1086, 982, 989, 983, 1043, 990, 1025, 1046, 1014, 1025, 1035, 1020, 1037, 1068, 1001, 1038, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1029, 995, 974, 1000, 1035, 1082, 1030, 1055, 1016, 1052, 1072, 1056, 1056, 1046, 1015, 1010, 1019, 1016, 1022, 974, 980, 1073, 998, 960, 976, 1038, 1023, 966, 1091, 1066, 1000, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1042, 999, 1032, 1029, 1025, 1026, 1092, 1010, 1021, 1000, 1018, 1046, 1021, 1013, 1011, 998, 1033, 1069, 984, 1060, 1035, 1073, 1042, 945, 989, 1009, 1030, 1045, 1012, 1039, 1011, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1026, 1032, 1040, 1041, 987, 1012, 1057, 1029, 1076, 973, 1049, 1038, 977, 1042, 1035, 1053, 1023, 1043, 1024, 1050, 972, 1034, 1030, 1002, 1020, 998, 1050, 1001, 995, 1013, 1019, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1023, 1044, 1053, 1052, 1, 0, 0, 0, 1029, 962, 1029, 1026, 0, 0, 0, 0, 1039, 1010, 1048, 1041, 0, 0, 0, 0, 998, 1010, 1059, 997, 0, 0, 0, 0, 1023, 1020, 996, 1018, 17, 1, 0, 0, 1077, 1015, 1020, 1034, 5, 5, 0, 0, 1008, 1085, 1007, 994, 3, 0, 0, 0, 1025, 981, 1011, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4046, 4138, 4064, 4075, 4156, 4097, 4019] avg=4096.0000 max=4173 max_rank=0 min=4019 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1038, 1063, 1031, 1009, 1, 0, 0, 0, 1011, 1044, 1079, 1005, 0, 0, 0, 0, 1039, 989, 954, 1014, 0, 0, 0, 0, 983, 1040, 982, 1000, 0, 0, 0, 0, 994, 980, 1040, 988, 18, 1, 0, 0, 1067, 1021, 1083, 996, 5, 5, 0, 0, 1010, 1060, 1041, 1044, 3, 0, 0, 0, 1023, 1032, 1056, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4139, 3996, 4005, 4021, 4177, 4158, 4130] avg=4096.0000 max=4177 max_rank=5 min=3996 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1049, 1059, 961, 1026, 1, 0, 0, 0, 1009, 974, 1008, 1044, 0, 0, 0, 0, 1060, 1024, 1045, 1033, 0, 0, 0, 0, 1002, 1051, 1025, 1086, 0, 0, 0, 0, 982, 989, 983, 1043, 17, 1, 0, 0, 990, 1025, 1046, 1014, 5, 5, 0, 0, 1025, 1035, 1020, 1037, 3, 0, 0, 0, 1068, 1001, 1038, 984, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4035, 4162, 4164, 4015, 4085, 4120, 4091] avg=4096.0000 max=4164 max_rank=3 min=4015 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1017, 1022, 1001, 1036, 1, 0, 0, 0, 1047, 1020, 1018, 1000, 0, 0, 0, 0, 1015, 1025, 970, 1009, 0, 0, 0, 0, 1039, 1005, 1049, 1034, 0, 0, 0, 0, 1041, 1026, 994, 1063, 18, 1, 0, 0, 1040, 999, 1040, 1001, 5, 5, 0, 0, 1073, 982, 1043, 1053, 3, 0, 0, 0, 1022, 994, 1014, 1043, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4085, 4019, 4127, 4143, 4090, 4154, 4073] avg=4096.0000 max=4154 max_rank=6 min=4019 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1029, 995, 974, 1000, 1, 0, 0, 0, 1035, 1082, 1025, 1055, 0, 0, 0, 0, 1010, 1052, 1072, 1056, 0, 0, 0, 0, 1056, 1042, 1015, 1010, 0, 0, 0, 0, 1019, 1016, 1022, 974, 18, 1, 0, 0, 980, 1073, 998, 960, 5, 5, 0, 0, 976, 1038, 1023, 966, 3, 0, 0, 0, 1091, 1066, 1000, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3999, 4197, 4190, 4123, 4050, 4021, 4006, 4182] avg=4096.0000 max=4197 max_rank=1 min=3999 min_rank=0 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[994, 1002, 1038, 999, 5, 0, 0, 0, 1013, 1045, 973, 1007, 0, 0, 0, 0, 1023, 1000, 1001, 1030, 0, 0, 0, 0, 1033, 1051, 1035, 1024, 0, 0, 0, 0, 1028, 1039, 1050, 993, 23, 8, 0, 0, 1029, 1027, 1015, 996, 11, 8, 0, 0, 1000, 1044, 1010, 1015, 9, 7, 0, 0, 1003, 1052, 1082, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4038, 4054, 4143, 4141, 4086, 4085, 4183] avg=4096.0000 max=4183 max_rank=7 min=4038 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1026, 1032, 1040, 1041, 1, 0, 0, 0, 987, 1012, 1052, 1029, 0, 0, 0, 0, 1070, 973, 1049, 1038, 0, 0, 0, 0, 977, 1038, 1035, 1053, 0, 0, 0, 0, 1023, 1043, 1024, 1050, 18, 1, 0, 0, 972, 1034, 1030, 1002, 5, 5, 0, 0, 1020, 998, 1050, 1001, 3, 0, 0, 0, 995, 1013, 1019, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4080, 4130, 4103, 4159, 4048, 4072, 4036] avg=4096.0000 max=4159 max_rank=4 min=4036 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1042, 999, 1032, 1029, 1, 0, 0, 0, 1025, 1026, 1087, 1010, 0, 0, 0, 0, 1015, 1000, 1018, 1046, 0, 0, 0, 0, 1021, 1009, 1011, 998, 0, 0, 0, 0, 1033, 1069, 984, 1060, 17, 1, 0, 0, 1035, 1073, 1042, 945, 5, 5, 0, 0, 989, 1009, 1030, 1045, 3, 0, 0, 0, 1012, 1039, 1011, 992, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4148, 4079, 4039, 4164, 4105, 4076, 4054] avg=4096.0000 max=4164 max_rank=4 min=4039 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[971, 980, 1015, 1050, 1049, 1009, 1018, 992, 1003, 1094, 1018, 980, 1044, 983, 1043, 1070, 1038, 1100, 1016, 1031, 1034, 991, 1040, 1032, 985, 1036, 1002, 1087, 1029, 998, 989, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1029, 1045, 1006, 1049, 1042, 1027, 997, 1026, 1008, 1052, 1024, 1026, 1017, 1031, 1051, 1046, 1027, 1030, 1035, 1012, 1082, 1001, 1058, 963, 1016, 984, 1029, 1049, 1044, 1022, 933, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1009, 1004, 1039, 1071, 1035, 1022, 983, 1046, 992, 1070, 1011, 1043, 1012, 1021, 1017, 997, 997, 1063, 991, 998, 1050, 1024, 1007, 1017, 1056, 1024, 1009, 1038, 1028, 1051, 1021, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1005, 969, 1004, 1037, 1083, 993, 992, 1026, 1057, 1028, 1025, 991, 1028, 1028, 1058, 1028, 1015, 1120, 1016, 1006, 999, 1065, 1036, 1041, 1027, 1017, 1009, 988, 1021, 1029, 1061, 966] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1074, 1017, 959, 1068, 1070, 980, 1026, 985, 1049, 1012, 1036, 1008, 1011, 1003, 1021, 1033, 1015, 1030, 1069, 1035, 1031, 1062, 977, 988, 998, 1069, 1018, 1022, 1024, 1050, 1022, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1040, 979, 1024, 1002, 1102, 997, 1054, 1020, 956, 997, 1013, 1002, 1031, 1080, 983, 998, 1022, 1008, 1013, 1049, 1033, 1051, 1030, 1031, 1028, 1008, 1022, 1009, 1064, 1097, 1015, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1017, 1034, 1017, 1031, 1014, 1032, 1039, 1062, 1063, 1023, 987, 1012, 1025, 1018, 969, 1062, 1000, 1058, 1016, 1001, 1011, 988, 1047, 991, 1010, 1047, 1043, 1016, 1019, 1053, 1031, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1001, 1062, 1004, 1037, 975, 1038, 1019, 1021, 1052, 993, 1021, 1077, 989, 1017, 1016, 1033, 1012, 984, 1036, 1001, 1060, 1065, 1012, 1028, 1078, 1048, 1027, 988, 990, 1037, 1037, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[971, 980, 1015, 1050, 9, 5, 0, 0, 1049, 1009, 1018, 992, 0, 0, 0, 0, 1003, 1094, 1018, 980, 4, 1, 0, 0, 1044, 983, 1043, 1070, 0, 0, 0, 0, 1038, 1091, 1016, 1031, 0, 0, 0, 0, 1025, 991, 1040, 1032, 0, 0, 0, 0, 985, 1034, 1002, 1087, 0, 0, 0, 0, 1029, 998, 989, 1041, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4068, 4100, 4140, 4176, 4088, 4108, 4058] avg=4096.0000 max=4176 max_rank=4 min=4030 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1029, 1045, 1006, 1049, 10, 5, 0, 0, 1042, 1027, 997, 1026, 0, 0, 0, 0, 1008, 1052, 1024, 1026, 4, 1, 0, 0, 1017, 1031, 1051, 1046, 0, 0, 0, 0, 1027, 1021, 1035, 1012, 0, 0, 0, 0, 1072, 1001, 1058, 963, 0, 0, 0, 0, 1016, 983, 1029, 1049, 0, 0, 0, 0, 1044, 1022, 933, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4092, 4115, 4145, 4095, 4094, 4077, 4006] avg=4096.0000 max=4145 max_rank=3 min=4006 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1017, 1034, 1017, 1031, 9, 5, 0, 0, 1014, 1032, 1039, 1062, 0, 0, 0, 0, 1063, 1023, 987, 1012, 4, 1, 0, 0, 1025, 1018, 969, 1062, 0, 0, 0, 0, 1000, 1049, 1016, 1001, 0, 0, 0, 0, 1002, 988, 1047, 991, 0, 0, 0, 0, 1010, 1045, 1043, 1016, 0, 0, 0, 0, 1019, 1053, 1031, 1032, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4147, 4090, 4074, 4066, 4028, 4114, 4136] avg=4096.0000 max=4147 max_rank=1 min=4028 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1009, 1004, 1039, 1071, 9, 5, 0, 0, 1035, 1022, 983, 1046, 0, 0, 0, 0, 992, 1070, 1011, 1043, 4, 1, 0, 0, 1012, 1021, 1017, 997, 0, 0, 0, 0, 997, 1054, 991, 998, 0, 0, 0, 0, 1041, 1024, 1007, 1017, 0, 0, 0, 0, 1056, 1023, 1009, 1038, 0, 0, 0, 0, 1028, 1051, 1021, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4086, 4121, 4047, 4040, 4089, 4126, 4122] avg=4096.0000 max=4137 max_rank=0 min=4040 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1005, 969, 1004, 1037, 9, 5, 0, 0, 1083, 993, 992, 1026, 0, 0, 0, 0, 1057, 1028, 1025, 991, 4, 1, 0, 0, 1028, 1028, 1058, 1028, 0, 0, 0, 0, 1015, 1111, 1016, 1006, 0, 0, 0, 0, 990, 1065, 1036, 1041, 0, 0, 0, 0, 1027, 1016, 1009, 988, 0, 0, 0, 0, 1021, 1029, 1061, 966, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4094, 4106, 4142, 4148, 4132, 4040, 4077] avg=4096.0000 max=4148 max_rank=4 min=4029 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1074, 1017, 959, 1068, 9, 5, 0, 0, 1070, 980, 1026, 985, 0, 0, 0, 0, 1049, 1012, 1036, 1008, 4, 1, 0, 0, 1011, 1003, 1021, 1033, 0, 0, 0, 0, 1015, 1021, 1069, 1035, 0, 0, 0, 0, 1022, 1062, 977, 988, 0, 0, 0, 0, 998, 1067, 1018, 1022, 0, 0, 0, 0, 1024, 1050, 1022, 1006, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4061, 4110, 4068, 4140, 4049, 4105, 4103] avg=4096.0000 max=4140 max_rank=4 min=4049 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1040, 979, 1024, 1002, 13, 8, 0, 0, 1096, 997, 1054, 1020, 0, 0, 0, 0, 956, 997, 1013, 1002, 7, 4, 0, 0, 1031, 1080, 983, 998, 5, 0, 0, 0, 1022, 993, 1013, 1049, 0, 0, 0, 0, 1020, 1051, 1030, 1031, 0, 0, 0, 0, 1028, 1000, 1022, 1009, 0, 0, 0, 0, 1064, 1097, 1015, 1010, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4167, 3979, 4097, 4077, 4132, 4059, 4191] avg=4096.0000 max=4191 max_rank=7 min=3979 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1001, 1062, 1004, 1037, 9, 4, 0, 0, 975, 1038, 1019, 1021, 0, 0, 0, 0, 1052, 993, 1021, 1077, 3, 1, 0, 0, 989, 1017, 1016, 1033, 0, 0, 0, 0, 1012, 977, 1036, 1001, 0, 0, 0, 0, 1051, 1065, 1012, 1028, 0, 0, 0, 0, 1078, 1046, 1027, 988, 0, 0, 0, 0, 990, 1037, 1037, 1010, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4053, 4147, 4055, 4026, 4156, 4139, 4075] avg=4096.0000 max=4156 max_rank=5 min=4026 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8448, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8448, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1052, 1025, 1006, 1026, 1065, 1031, 1078, 1033, 1026, 1062, 1050, 1041, 986, 1026, 1068, 940, 946, 1033, 1017, 1049, 1054, 1011, 978, 996, 1052, 1040, 961, 1049, 975, 1049, 1013, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1029, 994, 1049, 1048, 992, 1020, 997, 1047, 996, 974, 1041, 1067, 990, 1004, 1013, 1062, 1061, 1040, 985, 1051, 972, 1001, 1013, 1035, 1020, 1014, 1068, 963, 1067, 1034, 1087, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[999, 1045, 1030, 1028, 1023, 1002, 1014, 1011, 1016, 979, 993, 1026, 1049, 1025, 1020, 992, 1085, 1008, 1009, 1062, 1008, 1033, 1046, 1020, 1017, 992, 1039, 1012, 1042, 1055, 1073, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1006, 1011, 1026, 1049, 1002, 1042, 1033, 1004, 993, 1041, 1037, 981, 1053, 1016, 994, 1084, 1011, 1023, 986, 958, 1036, 1005, 1026, 1095, 1014, 1025, 1080, 1025, 1053, 993, 988, 1078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1096, 1035, 1020, 1023, 1008, 957, 1070, 1008, 1034, 992, 1030, 998, 999, 1019, 1007, 1027, 1004, 1045, 1034, 1011, 1018, 1022, 1022, 1026, 1057, 1008, 1052, 1004, 1020, 1044, 1051, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1028, 998, 994, 984, 1025, 1062, 1053, 1005, 1022, 1030, 1051, 970, 1029, 957, 1047, 1075, 1014, 1027, 1001, 1017, 1064, 1007, 1055, 987, 1030, 1009, 1033, 1071, 1029, 1038, 1030, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1053, 1030, 1012, 1035, 1049, 1048, 1054, 1023, 1039, 998, 1044, 1032, 989, 1046, 975, 1056, 1039, 1018, 979, 1000, 1016, 1002, 1014, 1043, 999, 1046, 997, 1047, 989, 1057, 1011, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1025, 1025, 1006, 1044, 1037, 1003, 1019, 1010, 1015, 1052, 1028, 999, 1026, 1052, 1040, 1010, 1066, 997, 1055, 1037, 992, 1046, 1070, 1050, 1006, 1036, 1024, 985, 991, 988, 996, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1045, 1025, 1006, 1026, 0, 0, 0, 0, 1065, 1031, 1071, 1033, 0, 0, 0, 0, 1026, 1062, 1050, 1041, 13, 0, 0, 0, 986, 1026, 1068, 940, 7, 4, 0, 0, 946, 1033, 1017, 1049, 8, 3, 0, 0, 1054, 1011, 978, 996, 0, 0, 0, 0, 1052, 1040, 961, 1049, 0, 0, 0, 0, 975, 1049, 1013, 1009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4200, 4192, 4031, 4056, 4039, 4102, 4046] avg=4096.0000 max=4200 max_rank=1 min=4031 min_rank=3 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16243 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[993, 1045, 1030, 1028, 0, 0, 0, 0, 1023, 1002, 1008, 1011, 0, 0, 0, 0, 1016, 979, 993, 1026, 13, 0, 0, 0, 1049, 1025, 1020, 992, 6, 3, 0, 0, 1085, 1008, 1009, 1062, 8, 3, 0, 0, 1008, 1033, 1046, 1020, 0, 0, 0, 0, 1017, 992, 1039, 1012, 0, 0, 0, 0, 1042, 1055, 1073, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4044, 4027, 4095, 4175, 4107, 4060, 4164] avg=4096.0000 max=4175 max_rank=4 min=4027 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1021, 998, 994, 984, 0, 0, 0, 0, 1025, 1062, 1046, 1005, 0, 0, 0, 0, 1022, 1030, 1051, 970, 13, 0, 0, 0, 1029, 957, 1047, 1075, 7, 4, 0, 0, 1014, 1027, 1001, 1017, 8, 3, 0, 0, 1064, 1007, 1055, 987, 0, 0, 0, 0, 1030, 1009, 1033, 1071, 0, 0, 0, 0, 1029, 1038, 1030, 1005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3997, 4138, 4086, 4119, 4070, 4113, 4143, 4102] avg=4096.0000 max=4143 max_rank=6 min=3997 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1022, 994, 1049, 1048, 0, 0, 0, 0, 992, 1020, 991, 1047, 0, 0, 0, 0, 996, 974, 1041, 1067, 13, 0, 0, 0, 990, 1004, 1013, 1062, 6, 4, 0, 0, 1061, 1040, 985, 1051, 8, 3, 0, 0, 972, 1001, 1013, 1035, 0, 0, 0, 0, 1020, 1014, 1068, 963, 0, 0, 0, 0, 1067, 1034, 1087, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4050, 4091, 4079, 4148, 4021, 4065, 4201] avg=4096.0000 max=4201 max_rank=7 min=4021 min_rank=5 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1089, 1035, 1020, 1023, 0, 0, 0, 0, 1008, 957, 1063, 1008, 0, 0, 0, 0, 1034, 992, 1030, 998, 13, 0, 0, 0, 999, 1019, 1007, 1027, 7, 4, 0, 0, 1004, 1045, 1034, 1011, 8, 3, 0, 0, 1018, 1022, 1022, 1026, 0, 0, 0, 0, 1057, 1008, 1052, 1004, 0, 0, 0, 0, 1020, 1044, 1051, 1006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4036, 4067, 4063, 4105, 4088, 4121, 4121] avg=4096.0000 max=4167 max_rank=0 min=4036 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[999, 1011, 1026, 1049, 0, 0, 0, 0, 1002, 1042, 1026, 1004, 0, 0, 0, 0, 993, 1041, 1037, 981, 14, 0, 0, 0, 1053, 1016, 994, 1084, 7, 4, 0, 0, 1011, 1023, 986, 958, 9, 3, 0, 0, 1036, 1005, 1026, 1095, 0, 0, 0, 0, 1014, 1025, 1080, 1025, 0, 0, 0, 0, 1053, 993, 988, 1055, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4074, 4066, 4158, 3990, 4162, 4144, 4089] avg=4096.0000 max=4162 max_rank=5 min=3990 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1038, 1030, 1012, 1035, 0, 0, 0, 0, 1049, 1048, 1043, 1023, 0, 0, 0, 0, 1039, 998, 1044, 1032, 19, 0, 0, 0, 989, 1046, 975, 1056, 11, 6, 2, 0, 1039, 1018, 979, 1000, 13, 9, 0, 0, 1016, 1002, 1014, 1043, 5, 0, 0, 0, 999, 1046, 990, 1047, 0, 0, 0, 0, 989, 1057, 1011, 996, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4163, 4132, 4085, 4058, 4080, 4082, 4053] avg=4096.0000 max=4163 max_rank=1 min=4053 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1018, 1025, 1006, 1044, 0, 0, 0, 0, 1037, 1003, 1013, 1010, 0, 0, 0, 0, 1015, 1052, 1028, 999, 13, 0, 0, 0, 1026, 1052, 1040, 1010, 6, 4, 0, 0, 1066, 997, 1055, 1037, 8, 3, 0, 0, 992, 1046, 1070, 1050, 0, 0, 0, 0, 1006, 1036, 1024, 985, 0, 0, 0, 0, 991, 988, 996, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4063, 4107, 4138, 4166, 4158, 4051, 3992] avg=4096.0000 max=4166 max_rank=4 min=3992 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1063, 1061, 1017, 1023, 1007, 981, 1000, 1014, 1037, 990, 1029, 1023, 1081, 1028, 1003, 977, 1060, 1041, 1014, 1038, 962, 1075, 982, 1091, 1065, 1063, 984, 1035, 1046, 965, 1025, 988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1048, 1022, 1053, 973, 1029, 1041, 1005, 1027, 991, 1041, 1070, 997, 1046, 999, 993, 1031, 1029, 1035, 1018, 1021, 991, 1038, 1031, 1000, 1079, 1037, 1015, 1025, 1032, 972, 1047, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1063, 1045, 978, 1068, 999, 991, 1016, 1000, 1010, 1029, 1020, 1055, 1079, 1031, 1012, 1006, 1051, 1044, 1017, 991, 1032, 1057, 1030, 1052, 975, 1005, 1021, 1009, 1029, 1018, 953, 1082] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1025, 1003, 1043, 1061, 1042, 1036, 1050, 1058, 1053, 984, 1000, 1033, 1005, 976, 1009, 976, 1064, 1041, 1012, 1037, 1081, 972, 1030, 1023, 1036, 957, 1031, 1047, 1085, 1006, 1000, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1004, 1023, 1033, 1022, 1081, 1015, 1029, 1011, 1058, 1027, 1003, 1021, 1043, 1068, 1045, 1052, 986, 1038, 1005, 1032, 1071, 1018, 1038, 1029, 1041, 1020, 944, 1038, 1001, 1001, 998, 973] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1018, 982, 1024, 1077, 998, 1049, 967, 991, 1017, 1078, 1000, 1051, 990, 1012, 1015, 1017, 1004, 1048, 1014, 1046, 1058, 1005, 1085, 1007, 1059, 1015, 1017, 1040, 1052, 1048, 997, 987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1013, 1031, 1015, 992, 1058, 1036, 1054, 1004, 1033, 1042, 1058, 1021, 1009, 1033, 1030, 1093, 1036, 1059, 1040, 1036, 1006, 999, 1056, 1003, 945, 979, 983, 1041, 1010, 976, 1044, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1078, 1028, 1071, 1040, 1028, 1007, 1011, 1063, 1037, 1032, 1028, 1056, 993, 1000, 1053, 1063, 1023, 1013, 1017, 1041, 1021, 1043, 973, 1050, 1028, 999, 995, 1030, 931, 1000, 1009, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1034, 1061, 1017, 1023, 0, 0, 0, 0, 1007, 981, 1000, 1014, 9, 0, 0, 0, 1037, 990, 1029, 1004, 0, 0, 0, 0, 1081, 1028, 1003, 977, 0, 0, 0, 0, 1060, 1019, 1014, 1038, 0, 0, 0, 0, 962, 1075, 982, 1073, 0, 0, 0, 0, 1065, 1063, 984, 1035, 17, 9, 0, 0, 1046, 965, 1025, 988, 29, 22, 2, 0] rerouted_tokens_per_ep_rank=[4135, 4011, 4060, 4089, 4131, 4092, 4173, 4077] avg=4096.0000 max=4173 max_rank=6 min=4011 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16473 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1034, 1045, 978, 1068, 0, 0, 0, 0, 999, 991, 1016, 1000, 8, 0, 0, 0, 1010, 1029, 1020, 1036, 0, 0, 0, 0, 1079, 1031, 1012, 1006, 0, 0, 0, 0, 1051, 1022, 1017, 991, 0, 0, 0, 0, 1032, 1057, 1030, 1035, 0, 0, 0, 0, 975, 1005, 1021, 1009, 17, 9, 0, 0, 1029, 1018, 953, 1082, 29, 22, 2, 0] rerouted_tokens_per_ep_rank=[4125, 4014, 4095, 4128, 4081, 4154, 4036, 4135] avg=4096.0000 max=4154 max_rank=5 min=4014 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16406 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1020, 1022, 1053, 973, 0, 0, 0, 0, 1029, 1041, 1005, 1027, 8, 0, 0, 0, 991, 1041, 1070, 979, 0, 0, 0, 0, 1046, 999, 993, 1031, 0, 0, 0, 0, 1029, 1013, 1018, 1021, 0, 0, 0, 0, 991, 1038, 1031, 984, 0, 0, 0, 0, 1079, 1037, 1015, 1025, 16, 8, 0, 0, 1032, 972, 1047, 1032, 28, 22, 2, 0] rerouted_tokens_per_ep_rank=[4068, 4110, 4081, 4069, 4081, 4044, 4180, 4135] avg=4096.0000 max=4180 max_rank=6 min=4044 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16440 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[997, 1003, 1043, 1061, 0, 0, 0, 0, 1042, 1036, 1050, 1058, 8, 0, 0, 0, 1053, 984, 1000, 1014, 0, 0, 0, 0, 1005, 976, 1009, 976, 0, 0, 0, 0, 1064, 1019, 1012, 1037, 0, 0, 0, 0, 1081, 972, 1030, 1007, 0, 0, 0, 0, 1036, 957, 1031, 1047, 17, 8, 0, 0, 1085, 1006, 1000, 992, 28, 22, 2, 0] rerouted_tokens_per_ep_rank=[4104, 4194, 4051, 3966, 4132, 4090, 4096, 4135] avg=4096.0000 max=4194 max_rank=1 min=3966 min_rank=3 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16453 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[977, 1023, 1033, 1022, 0, 0, 0, 0, 1081, 1015, 1029, 1011, 8, 0, 0, 0, 1058, 1027, 1003, 1002, 0, 0, 0, 0, 1043, 1068, 1045, 1052, 0, 0, 0, 0, 986, 1016, 1005, 1032, 0, 0, 0, 0, 1071, 1018, 1038, 1013, 0, 0, 0, 0, 1041, 1020, 944, 1038, 17, 8, 0, 0, 1001, 1001, 998, 973, 27, 22, 2, 0] rerouted_tokens_per_ep_rank=[4055, 4144, 4090, 4208, 4039, 4140, 4068, 4024] avg=4096.0000 max=4208 max_rank=3 min=4024 min_rank=7 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16271 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[986, 982, 1024, 1077, 0, 0, 0, 0, 998, 1049, 967, 991, 13, 0, 0, 0, 1017, 1078, 1000, 1027, 0, 0, 0, 0, 990, 1012, 1015, 1017, 0, 0, 0, 0, 1004, 1020, 1014, 1046, 0, 0, 0, 0, 1058, 1005, 1085, 982, 0, 0, 0, 0, 1059, 1015, 1017, 1040, 21, 12, 0, 0, 1052, 1048, 997, 987, 32, 28, 3, 0] rerouted_tokens_per_ep_rank=[4069, 4018, 4122, 4034, 4084, 4130, 4164, 4147] avg=4096.0000 max=4164 max_rank=6 min=4018 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16525 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[986, 1031, 1015, 992, 0, 0, 0, 0, 1058, 1036, 1054, 1004, 8, 0, 0, 0, 1033, 1042, 1058, 1002, 0, 0, 0, 0, 1009, 1033, 1030, 1093, 0, 0, 0, 0, 1036, 1036, 1040, 1036, 0, 0, 0, 0, 1006, 999, 1056, 987, 0, 0, 0, 0, 945, 979, 983, 1041, 17, 8, 0, 0, 1010, 976, 1044, 1033, 27, 23, 2, 0] rerouted_tokens_per_ep_rank=[4024, 4160, 4135, 4165, 4148, 4048, 3973, 4115] avg=4096.0000 max=4165 max_rank=3 min=3973 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16284 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1049, 1028, 1071, 1040, 0, 0, 0, 0, 1028, 1007, 1011, 1063, 8, 0, 0, 0, 1037, 1032, 1028, 1037, 0, 0, 0, 0, 993, 1000, 1053, 1063, 0, 0, 0, 0, 1023, 991, 1017, 1041, 0, 0, 0, 0, 1021, 1043, 973, 1033, 0, 0, 0, 0, 1028, 999, 995, 1030, 17, 9, 0, 0, 931, 1000, 1009, 1007, 29, 22, 2, 0] rerouted_tokens_per_ep_rank=[4188, 4117, 4134, 4109, 4072, 4070, 4078, 4000] avg=4096.0000 max=4188 max_rank=0 min=4000 min_rank=7 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16220 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8192, 256, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1057, 1025, 988, 1042, 1056, 1092, 1052, 1030, 962, 961, 1002, 985, 1020, 1060, 1018, 991, 1010, 1032, 1039, 1076, 1043, 1029, 1029, 1042, 1047, 1000, 1030, 990, 1042, 1034, 1007, 977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[968, 1005, 973, 1050, 1011, 1004, 1008, 1071, 1071, 1033, 1084, 1023, 1047, 1037, 963, 1022, 1034, 1059, 1043, 1037, 993, 1014, 1006, 1012, 997, 1047, 1020, 1029, 987, 1036, 1046, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1005, 1058, 1030, 1014, 996, 1036, 1006, 1011, 1030, 1017, 1008, 1011, 1043, 1019, 1003, 1032, 1044, 1031, 1034, 1070, 1018, 987, 1025, 1044, 1049, 999, 1018, 1019, 986, 1037, 1039, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1009, 1023, 1044, 974, 995, 1006, 1018, 1033, 1082, 975, 985, 1030, 1029, 1039, 1042, 1027, 1002, 1030, 1083, 1009, 1018, 1083, 1023, 1002, 1048, 1002, 1068, 1029, 1006, 1018, 981, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1051, 1014, 1012, 1049, 995, 1020, 1073, 1033, 995, 1023, 1061, 1033, 990, 998, 1008, 992, 1025, 1029, 1027, 1020, 1044, 972, 1018, 1035, 1020, 1047, 1105, 998, 1008, 1024, 1053, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1036, 999, 1046, 1009, 1047, 962, 1001, 1054, 1037, 1031, 993, 1017, 1093, 1061, 1065, 1034, 1029, 1030, 1043, 1015, 967, 993, 1019, 1020, 1035, 1012, 1039, 1005, 1054, 1005, 1038, 979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1007, 1058, 1013, 1035, 1044, 1043, 1053, 994, 996, 1025, 1034, 1045, 1022, 998, 1045, 1050, 1004, 1000, 966, 1053, 993, 991, 1079, 1043, 1035, 1022, 1047, 1028, 1034, 988, 978, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1023, 1032, 1018, 1006, 1017, 1052, 1017, 1059, 1020, 1028, 1030, 1053, 1028, 1041, 1018, 1003, 997, 1003, 1015, 1023, 977, 1036, 1004, 1022, 1039, 1028, 1028, 1044, 1027, 1006, 1017, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[968, 1005, 973, 1050, 12, 0, 0, 0, 1011, 1004, 1008, 1056, 0, 0, 0, 0, 1071, 1033, 1084, 1023, 8, 2, 0, 0, 1039, 1037, 963, 1022, 0, 0, 0, 0, 1034, 1059, 1043, 1020, 0, 0, 0, 0, 993, 1014, 1006, 1012, 19, 3, 0, 0, 997, 1047, 1001, 1029, 0, 0, 0, 0, 987, 1036, 1046, 1038, 14, 1, 0, 0] rerouted_tokens_per_ep_rank=[4008, 4079, 4221, 4061, 4156, 4047, 4074, 4122] avg=4096.0000 max=4221 max_rank=2 min=4008 min_rank=0 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1005, 1058, 1030, 1014, 11, 0, 0, 0, 996, 1036, 1006, 998, 0, 0, 0, 0, 1030, 1017, 1008, 1011, 8, 2, 0, 0, 1035, 1019, 1003, 1032, 0, 0, 0, 0, 1044, 1031, 1034, 1053, 0, 0, 0, 0, 1018, 987, 1025, 1044, 19, 3, 0, 0, 1049, 999, 999, 1019, 0, 0, 0, 0, 986, 1037, 1039, 1049, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4036, 4076, 4089, 4162, 4096, 4066, 4125] avg=4096.0000 max=4162 max_rank=4 min=4036 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1057, 1025, 988, 1042, 11, 0, 0, 0, 1056, 1092, 1052, 1017, 0, 0, 0, 0, 962, 961, 1002, 985, 8, 2, 0, 0, 1012, 1060, 1018, 991, 0, 0, 0, 0, 1010, 1032, 1039, 1058, 0, 0, 0, 0, 1043, 1029, 1029, 1042, 19, 4, 0, 0, 1047, 1000, 1011, 990, 0, 0, 0, 0, 1042, 1034, 1007, 977, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4217, 3920, 4081, 4139, 4166, 4048, 4074] avg=4096.0000 max=4217 max_rank=1 min=3920 min_rank=2 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1051, 1014, 1012, 1049, 11, 0, 0, 0, 995, 1020, 1073, 1020, 0, 0, 0, 0, 995, 1023, 1061, 1033, 8, 2, 0, 0, 982, 998, 1008, 992, 0, 0, 0, 0, 1025, 1029, 1027, 1004, 0, 0, 0, 0, 1044, 972, 1018, 1035, 20, 3, 0, 0, 1020, 1047, 1085, 998, 0, 0, 0, 0, 1008, 1024, 1053, 996, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4108, 4122, 3980, 4085, 4092, 4150, 4094] avg=4096.0000 max=4150 max_rank=6 min=3980 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1036, 999, 1046, 1009, 12, 0, 0, 0, 1047, 962, 1001, 1039, 0, 0, 0, 0, 1037, 1031, 993, 1017, 9, 2, 0, 0, 1084, 1061, 1065, 1034, 0, 0, 0, 0, 1029, 1030, 1043, 999, 0, 0, 0, 0, 967, 993, 1019, 1020, 19, 3, 0, 0, 1035, 1012, 1020, 1005, 0, 0, 0, 0, 1054, 1005, 1038, 979, 13, 1, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4049, 4089, 4244, 4101, 4021, 4072, 4090] avg=4096.0000 max=4244 max_rank=3 min=4021 min_rank=5 imbalance_ratio=1.036133 max_load_reduction=28524 max_load_reduction_pct=0.870483 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1009, 1023, 1044, 974, 15, 0, 0, 0, 995, 1006, 1018, 1009, 0, 0, 0, 0, 1082, 975, 985, 1030, 13, 4, 0, 0, 1016, 1039, 1042, 1027, 0, 0, 0, 0, 1002, 1030, 1083, 982, 0, 0, 0, 0, 1018, 1083, 1023, 1002, 22, 9, 0, 0, 1048, 1002, 1046, 1029, 0, 0, 0, 0, 1006, 1018, 981, 1055, 18, 5, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4028, 4089, 4124, 4097, 4157, 4125, 4083] avg=4096.0000 max=4157 max_rank=5 min=4028 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1007, 1058, 1013, 1035, 11, 0, 0, 0, 1044, 1043, 1053, 981, 0, 0, 0, 0, 996, 1025, 1034, 1045, 8, 2, 0, 0, 1014, 998, 1045, 1050, 0, 0, 0, 0, 1004, 1000, 966, 1036, 0, 0, 0, 0, 993, 991, 1079, 1043, 19, 3, 0, 0, 1035, 1022, 1028, 1028, 0, 0, 0, 0, 1034, 988, 978, 1045, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4121, 4110, 4107, 4006, 4128, 4113, 4059] avg=4096.0000 max=4128 max_rank=5 min=4006 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1023, 1032, 1018, 1006, 12, 0, 0, 0, 1017, 1052, 1017, 1044, 0, 0, 0, 0, 1020, 1028, 1030, 1053, 8, 2, 0, 0, 1020, 1041, 1018, 1003, 0, 0, 0, 0, 997, 1003, 1015, 1007, 0, 0, 0, 0, 977, 1036, 1004, 1022, 19, 3, 0, 0, 1039, 1028, 1009, 1044, 0, 0, 0, 0, 1027, 1006, 1017, 1057, 13, 1, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4130, 4141, 4082, 4022, 4061, 4120, 4121] avg=4096.0000 max=4141 max_rank=2 min=4022 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1029, 1058, 1011, 1023, 970, 999, 1015, 1034, 1049, 978, 1027, 1052, 1034, 1003, 1010, 1066, 1030, 1049, 1022, 1074, 1011, 1006, 1025, 1005, 1027, 1061, 1024, 981, 1026, 1057, 1036, 976] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1032, 1061, 1029, 1059, 1020, 998, 1077, 1097, 1038, 1033, 1011, 1005, 1052, 1016, 1041, 976, 990, 1022, 1020, 987, 1075, 1013, 1027, 1007, 1031, 1027, 986, 1015, 1022, 1054, 985, 962] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[986, 1014, 1024, 1035, 1017, 1036, 1027, 1046, 1047, 1035, 1029, 1050, 1012, 1043, 1011, 1049, 991, 1029, 1030, 1008, 995, 1006, 1082, 960, 1019, 1030, 1033, 1025, 1036, 1016, 1032, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1017, 1051, 1013, 1033, 977, 1024, 1042, 1005, 1013, 1004, 995, 1037, 1038, 1049, 1021, 1019, 1011, 1038, 1019, 1000, 1034, 1084, 1016, 1025, 1020, 1033, 997, 987, 1037, 1044, 1052, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1053, 1009, 1017, 1014, 1045, 1085, 1063, 1000, 998, 1044, 1052, 1061, 1029, 1006, 996, 1033, 1010, 983, 1030, 1018, 988, 1055, 997, 1003, 1009, 1043, 1037, 1031, 1010, 1043, 1020, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1039, 1015, 1032, 1051, 1016, 1019, 1042, 1013, 1011, 1029, 973, 1035, 991, 1052, 1040, 1047, 1037, 1041, 1035, 1030, 1023, 948, 970, 1057, 1017, 1037, 989, 1027, 1067, 1026, 1049, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1044, 1000, 1012, 1051, 993, 1054, 989, 1050, 986, 1046, 1052, 1006, 1054, 1019, 1053, 1040, 998, 1046, 1089, 1012, 994, 1007, 1004, 988, 987, 993, 1051, 1013, 1008, 1040, 1046, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1006, 1055, 1008, 1022, 1078, 1047, 1014, 1051, 1047, 1020, 1013, 1030, 1009, 1039, 1049, 1065, 991, 1028, 1020, 1027, 1003, 1001, 1012, 986, 995, 1026, 1044, 1025, 1009, 978, 1043, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1029, 1058, 1011, 1007, 0, 0, 0, 0, 970, 999, 1015, 1014, 0, 0, 0, 0, 1049, 978, 1027, 1048, 0, 0, 0, 0, 1034, 1003, 1010, 1042, 0, 0, 0, 0, 1030, 1049, 1022, 1074, 4, 2, 0, 0, 1011, 1006, 1025, 1005, 24, 20, 0, 0, 1027, 1061, 1024, 981, 16, 0, 0, 0, 1026, 1057, 1034, 976, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 3998, 4102, 4089, 4181, 4091, 4109, 4093] avg=4096.0000 max=4181 max_rank=4 min=3998 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1032, 1061, 1029, 1042, 0, 0, 0, 0, 1020, 998, 1077, 1074, 0, 0, 0, 0, 1038, 1033, 1011, 1001, 0, 0, 0, 0, 1052, 1016, 1041, 954, 0, 0, 0, 0, 990, 1022, 1020, 987, 4, 2, 0, 0, 1075, 1013, 1027, 1007, 22, 22, 0, 0, 1031, 1027, 986, 1015, 17, 1, 0, 0, 1022, 1054, 983, 962, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4169, 4083, 4063, 4025, 4166, 4077, 4021] avg=4096.0000 max=4169 max_rank=1 min=4021 min_rank=7 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[986, 1014, 1024, 1019, 0, 0, 0, 0, 1017, 1036, 1027, 1024, 0, 0, 0, 0, 1047, 1035, 1029, 1046, 0, 0, 0, 0, 1012, 1043, 1011, 1025, 0, 0, 0, 0, 991, 1029, 1030, 1008, 4, 2, 0, 0, 995, 1006, 1082, 960, 24, 21, 0, 0, 1019, 1030, 1033, 1025, 16, 1, 0, 0, 1036, 1016, 1030, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4104, 4157, 4091, 4064, 4088, 4124, 4097] avg=4096.0000 max=4157 max_rank=2 min=4043 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1053, 1009, 1017, 998, 0, 0, 0, 0, 1045, 1085, 1063, 980, 0, 0, 0, 0, 998, 1044, 1052, 1057, 0, 0, 0, 0, 1029, 1006, 996, 1009, 0, 0, 0, 0, 1010, 983, 1030, 1018, 4, 2, 0, 0, 988, 1055, 997, 1003, 24, 20, 0, 0, 1009, 1043, 1037, 1031, 16, 0, 0, 0, 1010, 1043, 1018, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4173, 4151, 4040, 4047, 4087, 4136, 4057] avg=4096.0000 max=4173 max_rank=1 min=4040 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1039, 1015, 1032, 1034, 0, 0, 0, 0, 1016, 1019, 1042, 993, 0, 0, 0, 0, 1011, 1029, 973, 1031, 0, 0, 0, 0, 991, 1052, 1040, 1023, 0, 0, 0, 0, 1037, 1041, 1035, 1030, 4, 2, 0, 0, 1023, 948, 970, 1057, 24, 20, 0, 0, 1017, 1037, 989, 1027, 17, 0, 0, 0, 1067, 1026, 1047, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4070, 4044, 4106, 4149, 4042, 4087, 4150] avg=4096.0000 max=4150 max_rank=7 min=4042 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1017, 1051, 1013, 1013, 0, 0, 0, 0, 977, 1024, 1042, 979, 0, 0, 0, 0, 1013, 1004, 995, 1027, 0, 0, 0, 0, 1038, 1049, 1021, 991, 0, 0, 0, 0, 1011, 1038, 1019, 1000, 5, 6, 0, 0, 1034, 1084, 1016, 1025, 28, 22, 0, 0, 1020, 1033, 997, 987, 20, 4, 5, 0, 1037, 1044, 1046, 1033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4022, 4039, 4099, 4079, 4209, 4066, 4160] avg=4096.0000 max=4209 max_rank=5 min=4022 min_rank=1 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16514 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1044, 1000, 1012, 1034, 0, 0, 0, 0, 993, 1054, 989, 1028, 0, 0, 0, 0, 986, 1046, 1052, 1002, 0, 0, 0, 0, 1054, 1019, 1053, 1016, 0, 0, 0, 0, 998, 1046, 1089, 1012, 4, 2, 0, 0, 994, 1007, 1004, 988, 24, 21, 0, 0, 987, 993, 1051, 1013, 17, 1, 0, 0, 1008, 1040, 1044, 1043, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4064, 4086, 4142, 4151, 4038, 4062, 4135] avg=4096.0000 max=4151 max_rank=4 min=4038 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1006, 1055, 1008, 1006, 0, 0, 0, 0, 1078, 1047, 1014, 1029, 0, 0, 0, 0, 1047, 1020, 1013, 1026, 0, 0, 0, 0, 1009, 1039, 1049, 1041, 0, 0, 0, 0, 991, 1028, 1020, 1027, 4, 2, 0, 0, 1003, 1001, 1012, 986, 24, 21, 0, 0, 995, 1026, 1044, 1025, 16, 1, 0, 0, 1009, 978, 1041, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4168, 4106, 4138, 4072, 4047, 4107, 4055] avg=4096.0000 max=4168 max_rank=1 min=4047 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8064, 256, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1002, 1011, 996, 1033, 1041, 1036, 1075, 1027, 1040, 979, 1048, 1017, 1058, 1015, 1025, 1005, 996, 987, 1057, 1054, 985, 1043, 1021, 1013, 1073, 1011, 969, 1026, 1021, 1000, 1037, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1005, 1018, 1038, 1055, 1009, 1034, 1026, 1011, 1007, 1040, 1010, 993, 1073, 1039, 1051, 1038, 971, 1038, 1072, 1009, 1049, 960, 1017, 1046, 1041, 1004, 1072, 1002, 1003, 992, 1013, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1055, 1049, 1013, 1012, 989, 1000, 988, 1029, 1028, 1011, 1004, 1084, 1051, 1021, 1075, 1009, 1044, 1031, 1019, 1042, 965, 1008, 1019, 1034, 1017, 1040, 991, 1031, 1021, 1019, 1022, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1083, 1039, 1071, 962, 979, 1021, 1031, 1063, 941, 1064, 1023, 1076, 979, 1063, 1051, 1059, 1027, 980, 962, 1041, 1024, 1017, 1006, 1000, 1017, 1023, 1028, 1026, 1014, 1023, 992, 1083] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[990, 973, 1058, 1039, 1027, 1016, 1042, 1053, 1036, 1094, 1002, 1018, 1053, 991, 1071, 1031, 997, 991, 1022, 1004, 1020, 1051, 1007, 1019, 1022, 1030, 1025, 1039, 977, 1034, 1029, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1032, 983, 1015, 1039, 1026, 1007, 977, 1018, 1051, 1044, 1093, 1030, 1032, 1083, 995, 1010, 1014, 1001, 1027, 1031, 1047, 1100, 1005, 1029, 965, 1038, 999, 1020, 998, 1020, 1048, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1012, 1060, 985, 998, 1052, 1013, 1044, 1034, 1039, 1026, 1042, 1037, 992, 1009, 1044, 992, 972, 1005, 1028, 984, 1019, 1018, 1061, 1044, 1017, 1027, 1056, 1021, 1030, 980, 1066, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1012, 1030, 1036, 1046, 1044, 1043, 1015, 1013, 1000, 1017, 995, 989, 1033, 1014, 1063, 1015, 987, 1041, 1056, 1006, 1028, 1048, 1025, 1016, 1004, 1041, 1076, 1014, 988, 1011, 997, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1002, 1011, 996, 1033, 1, 0, 0, 0, 1041, 1036, 1075, 1026, 0, 0, 0, 0, 1040, 968, 1048, 1017, 0, 0, 0, 0, 1058, 1015, 992, 1005, 0, 0, 0, 0, 996, 987, 1057, 1054, 33, 0, 0, 0, 985, 1043, 1021, 1013, 2, 0, 0, 0, 1073, 1011, 969, 1026, 0, 0, 0, 0, 1021, 1000, 1037, 1067, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4178, 4073, 4070, 4127, 4064, 4079, 4134] avg=4096.0000 max=4178 max_rank=1 min=4043 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16404 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1083, 1039, 1071, 962, 1, 0, 0, 0, 979, 1021, 1031, 1062, 0, 0, 0, 0, 941, 1051, 1023, 1076, 0, 0, 0, 0, 979, 1063, 1017, 1059, 0, 0, 0, 0, 1027, 980, 962, 1041, 34, 0, 0, 0, 1024, 1017, 1006, 1000, 3, 0, 0, 0, 1017, 1023, 1028, 1026, 0, 0, 0, 0, 1014, 1023, 992, 1083, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4093, 4091, 4118, 4044, 4050, 4094, 4122] avg=4096.0000 max=4156 max_rank=0 min=4044 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16310 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1055, 1049, 1013, 1012, 1, 0, 0, 0, 989, 1000, 988, 1028, 0, 0, 0, 0, 1028, 1000, 1004, 1084, 0, 0, 0, 0, 1051, 1021, 1041, 1009, 0, 0, 0, 0, 1044, 1031, 1019, 1042, 34, 0, 0, 0, 965, 1008, 1019, 1034, 2, 0, 0, 0, 1017, 1040, 991, 1031, 0, 0, 0, 0, 1021, 1019, 1022, 1047, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4005, 4116, 4122, 4170, 4028, 4079, 4118] avg=4096.0000 max=4170 max_rank=4 min=4005 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16395 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1005, 1018, 1038, 1055, 1, 0, 0, 0, 1009, 1034, 1026, 1010, 0, 0, 0, 0, 1007, 1027, 1010, 993, 0, 0, 0, 0, 1073, 1039, 1017, 1038, 0, 0, 0, 0, 971, 1038, 1072, 1009, 34, 0, 0, 0, 1049, 960, 1017, 1046, 3, 0, 0, 0, 1041, 1004, 1072, 1002, 0, 0, 0, 0, 1003, 992, 1013, 1032, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4079, 4037, 4167, 4124, 4075, 4119, 4050] avg=4096.0000 max=4167 max_rank=3 min=4037 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16368 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1032, 983, 1015, 1039, 1, 0, 0, 0, 1026, 1007, 977, 1017, 0, 0, 0, 0, 1051, 1031, 1093, 1030, 0, 0, 0, 0, 1032, 1083, 963, 1010, 0, 0, 0, 0, 1014, 1001, 1027, 1031, 32, 0, 0, 0, 1047, 1100, 1005, 1029, 3, 0, 0, 0, 965, 1038, 999, 1020, 0, 0, 0, 0, 998, 1020, 1048, 991, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4027, 4205, 4088, 4105, 4184, 4022, 4067] avg=4096.0000 max=4205 max_rank=2 min=4022 min_rank=6 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16378 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[990, 973, 1058, 1039, 5, 6, 0, 0, 1027, 1016, 1042, 1045, 0, 0, 0, 0, 1036, 1067, 1002, 1018, 0, 0, 0, 0, 1053, 991, 1033, 1031, 0, 0, 0, 0, 997, 991, 1022, 1004, 38, 0, 0, 0, 1020, 1051, 1007, 1019, 7, 0, 0, 0, 1022, 1030, 1025, 1039, 3, 0, 0, 0, 977, 1034, 1029, 1007, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4130, 4123, 4108, 4052, 4104, 4119, 4061] avg=4096.0000 max=4130 max_rank=1 min=4052 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16336 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1012, 1060, 985, 998, 1, 0, 0, 0, 1052, 1013, 1044, 1033, 0, 0, 0, 0, 1039, 1015, 1042, 1037, 0, 0, 0, 0, 992, 1009, 1011, 992, 0, 0, 0, 0, 972, 1005, 1028, 984, 33, 0, 0, 0, 1019, 1018, 1061, 1044, 2, 0, 0, 0, 1017, 1027, 1056, 1021, 0, 0, 0, 0, 1030, 980, 1066, 1061, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4142, 4133, 4004, 4022, 4144, 4121, 4146] avg=4096.0000 max=4146 max_rank=7 min=4004 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16433 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1012, 1030, 1036, 1046, 1, 0, 0, 0, 1044, 1043, 1015, 1012, 0, 0, 0, 0, 1000, 1006, 995, 989, 0, 0, 0, 0, 1033, 1014, 1029, 1015, 0, 0, 0, 0, 987, 1041, 1056, 1006, 34, 0, 0, 0, 1028, 1048, 1025, 1016, 2, 0, 0, 0, 1004, 1041, 1076, 1014, 0, 0, 0, 0, 988, 1011, 997, 1065, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4114, 3990, 4091, 4124, 4119, 4135, 4070] avg=4096.0000 max=4135 max_rank=6 min=3990 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16448 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8448, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1008, 1022, 1039, 1044, 993, 1020, 1014, 1051, 1020, 1010, 978, 1021, 1043, 1035, 1016, 1070, 1042, 1005, 1003, 1031, 1023, 1040, 1029, 1031, 1018, 971, 993, 999, 1028, 1081, 1068, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1004, 1064, 1043, 1021, 985, 1050, 1020, 1024, 1056, 1034, 1073, 1028, 1002, 1049, 1039, 997, 975, 1042, 1058, 989, 1007, 1023, 1002, 1022, 1033, 1048, 970, 997, 952, 1100, 1040, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1055, 1012, 1043, 983, 992, 1007, 1018, 1014, 1043, 1054, 1035, 986, 1051, 1030, 995, 998, 1011, 1051, 1001, 1002, 1030, 1065, 1033, 1015, 1025, 1096, 1011, 1006, 1012, 996, 1084, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1046, 1112, 1037, 1015, 1023, 1043, 993, 1030, 992, 1027, 1022, 1024, 984, 1021, 1006, 1046, 1028, 1021, 1023, 1019, 1010, 1059, 1054, 1030, 1059, 999, 1040, 1013, 981, 1022, 1014, 975] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1042, 1062, 1000, 1043, 1013, 1046, 1063, 1025, 1064, 1045, 986, 1008, 995, 1040, 1022, 1063, 974, 994, 1004, 1016, 1012, 1017, 997, 1052, 991, 1025, 999, 1042, 1044, 1029, 1028, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[996, 1053, 999, 1029, 1035, 1046, 1031, 956, 1036, 1009, 959, 1068, 1023, 1028, 1068, 1025, 1044, 1044, 991, 1005, 999, 1011, 1037, 1082, 1030, 1044, 1032, 993, 1040, 1012, 1001, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1023, 1017, 1018, 964, 1007, 1031, 985, 1052, 1007, 1028, 1017, 1013, 1089, 990, 1015, 1029, 1017, 1019, 970, 1064, 1066, 962, 1024, 1025, 1029, 1076, 1018, 1047, 1057, 1020, 1081, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1014, 1013, 1015, 1026, 1039, 987, 1001, 1068, 1018, 1021, 1040, 1024, 1012, 1006, 1057, 970, 1042, 1053, 1085, 1002, 1058, 1053, 1056, 1054, 1027, 989, 1022, 1031, 967, 1010, 1017, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1004, 1053, 1043, 1021, 0, 0, 0, 0, 985, 1050, 1020, 1024, 8, 4, 0, 0, 1056, 1034, 1073, 1028, 1, 0, 0, 0, 1002, 1049, 1034, 997, 0, 0, 0, 0, 975, 1042, 1058, 989, 17, 0, 0, 0, 1007, 1023, 1002, 997, 0, 0, 0, 0, 1033, 1048, 970, 997, 7, 5, 0, 0, 952, 1100, 1039, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4091, 4192, 4082, 4081, 4029, 4060, 4112] avg=4096.0000 max=4192 max_rank=2 min=4029 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1008, 1012, 1039, 1044, 0, 0, 0, 0, 993, 1020, 1014, 1051, 8, 4, 0, 0, 1020, 1010, 978, 1021, 2, 0, 0, 0, 1043, 1035, 1012, 1070, 0, 0, 0, 0, 1042, 1005, 1003, 1031, 17, 0, 0, 0, 1023, 1040, 1029, 1006, 0, 0, 0, 0, 1018, 971, 993, 999, 6, 4, 0, 0, 1028, 1081, 1066, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4090, 4031, 4160, 4098, 4098, 3991, 4197] avg=4096.0000 max=4197 max_rank=7 min=3991 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1042, 1052, 1000, 1043, 0, 0, 0, 0, 1013, 1046, 1063, 1025, 8, 4, 0, 0, 1064, 1045, 986, 1008, 1, 0, 0, 0, 995, 1040, 1018, 1063, 0, 0, 0, 0, 974, 994, 1004, 1016, 18, 0, 0, 0, 1012, 1017, 997, 1026, 0, 0, 0, 0, 991, 1025, 999, 1042, 6, 4, 0, 0, 1044, 1029, 1027, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4159, 4104, 4116, 4006, 4052, 4067, 4127] avg=4096.0000 max=4159 max_rank=1 min=4006 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16252 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1046, 1100, 1037, 1015, 0, 0, 0, 0, 1023, 1043, 993, 1030, 8, 5, 0, 0, 992, 1027, 1022, 1024, 1, 0, 0, 0, 984, 1021, 1002, 1046, 0, 0, 0, 0, 1028, 1021, 1023, 1019, 17, 0, 0, 0, 1010, 1059, 1054, 1005, 0, 0, 0, 0, 1059, 999, 1040, 1013, 7, 4, 0, 0, 981, 1022, 1013, 975, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4198, 4102, 4066, 4053, 4108, 4128, 4122, 3991] avg=4096.0000 max=4198 max_rank=0 min=3991 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1055, 991, 1043, 983, 0, 0, 0, 0, 992, 1007, 1018, 1014, 11, 10, 0, 0, 1043, 1054, 1035, 986, 7, 6, 0, 0, 1051, 1030, 980, 998, 0, 0, 0, 0, 1011, 1051, 1001, 1002, 21, 0, 0, 0, 1030, 1065, 1033, 983, 0, 0, 0, 0, 1025, 1096, 1011, 1006, 11, 9, 0, 0, 1012, 996, 1077, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4052, 4131, 4059, 4086, 4111, 4158, 4099] avg=4096.0000 max=4158 max_rank=6 min=4052 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1023, 1007, 1018, 964, 0, 0, 0, 0, 1007, 1031, 985, 1052, 8, 4, 0, 0, 1007, 1028, 1017, 1013, 2, 0, 0, 0, 1089, 990, 1011, 1029, 0, 0, 0, 0, 1017, 1019, 970, 1064, 17, 0, 0, 0, 1066, 962, 1024, 1000, 0, 0, 0, 0, 1029, 1076, 1018, 1047, 6, 4, 0, 0, 1057, 1020, 1079, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4087, 4067, 4119, 4087, 4052, 4180, 4164] avg=4096.0000 max=4180 max_rank=6 min=4012 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16483 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[996, 1043, 999, 1029, 0, 0, 0, 0, 1035, 1046, 1031, 956, 8, 4, 0, 0, 1036, 1009, 959, 1068, 1, 0, 0, 0, 1023, 1028, 1063, 1025, 0, 0, 0, 0, 1044, 1044, 991, 1005, 18, 0, 0, 0, 999, 1011, 1037, 1056, 0, 0, 0, 0, 1030, 1044, 1032, 993, 6, 5, 0, 0, 1040, 1012, 1000, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4080, 4073, 4139, 4102, 4103, 4110, 4094] avg=4096.0000 max=4139 max_rank=3 min=4067 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1014, 1003, 1015, 1026, 0, 0, 0, 0, 1039, 987, 1001, 1068, 8, 4, 0, 0, 1018, 1021, 1040, 1024, 1, 0, 0, 0, 1012, 1006, 1052, 970, 0, 0, 0, 0, 1042, 1053, 1085, 1002, 18, 0, 0, 0, 1058, 1053, 1056, 1028, 0, 0, 0, 0, 1027, 989, 1022, 1031, 6, 5, 0, 0, 967, 1010, 1016, 991, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4107, 4104, 4040, 4200, 4195, 4080, 3984] avg=4096.0000 max=4200 max_rank=4 min=3984 min_rank=7 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[954, 1014, 1003, 994, 1088, 1012, 1050, 986, 991, 1026, 1034, 1050, 993, 1046, 1018, 1071, 1040, 1005, 1044, 1015, 977, 1005, 1023, 1014, 997, 984, 1020, 1055, 1068, 1040, 1088, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1084, 1028, 1014, 962, 1046, 1041, 1043, 1032, 994, 1041, 995, 1005, 1025, 985, 978, 1028, 1059, 1022, 1041, 1018, 1005, 1015, 998, 1088, 996, 1061, 1003, 1045, 1022, 1043, 1055, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1004, 1021, 1018, 1016, 1030, 1025, 1066, 996, 1003, 1005, 1037, 1005, 1039, 1030, 1012, 1041, 1032, 983, 1085, 1039, 1024, 1049, 1024, 982, 1072, 1043, 1026, 1006, 1007, 1052, 965, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1040, 1058, 1052, 1037, 1031, 1021, 1022, 1027, 1004, 1045, 987, 1010, 1002, 1031, 1019, 1017, 1022, 998, 993, 1032, 1001, 1033, 1009, 1014, 1043, 1038, 1042, 1004, 1089, 988, 991, 1068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1031, 1029, 1015, 1018, 1015, 1027, 1029, 1042, 1014, 998, 1034, 1027, 1009, 1036, 1045, 1051, 1072, 996, 995, 1049, 1067, 1015, 1013, 1031, 994, 1031, 995, 1013, 1005, 1035, 1021, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1045, 1003, 991, 1013, 1013, 1060, 1015, 1003, 1003, 1025, 1002, 1075, 1019, 1004, 1056, 1048, 1050, 1068, 1052, 1041, 977, 1025, 980, 1026, 1008, 1042, 1027, 1006, 1056, 1003, 1034, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1077, 1015, 1011, 1009, 1053, 991, 992, 1003, 1031, 1001, 1012, 1037, 1028, 1005, 1037, 1055, 1001, 977, 1041, 1035, 1018, 1025, 1034, 1014, 1041, 1070, 988, 1039, 985, 1058, 1052, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1043, 1041, 1042, 1050, 1052, 986, 995, 1062, 1013, 975, 1033, 1033, 1005, 1071, 1041, 1055, 1024, 1002, 1009, 987, 1018, 1074, 1002, 1036, 1036, 1000, 988, 990, 971, 1062, 1000, 1072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[954, 1014, 1003, 994, 4, 0, 0, 0, 1078, 1012, 1050, 986, 0, 0, 0, 0, 991, 1026, 1034, 1050, 24, 3, 0, 0, 993, 1046, 1018, 1055, 0, 0, 0, 0, 1034, 1005, 1044, 1015, 0, 0, 0, 0, 977, 1005, 1023, 1014, 13, 5, 0, 0, 997, 984, 1020, 1055, 5, 2, 0, 0, 1068, 1016, 1088, 1063, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3969, 4126, 4128, 4112, 4098, 4037, 4063, 4235] avg=4096.0000 max=4235 max_rank=7 min=3969 min_rank=0 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1004, 1021, 1018, 1016, 4, 0, 0, 0, 1020, 1025, 1066, 996, 0, 0, 0, 0, 1003, 1005, 1037, 1005, 25, 2, 0, 0, 1039, 1030, 1012, 1026, 0, 0, 0, 0, 1026, 983, 1085, 1039, 0, 0, 0, 0, 1024, 1049, 1024, 982, 13, 5, 0, 0, 1072, 1043, 1026, 1006, 5, 2, 0, 0, 1007, 1027, 965, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4107, 4077, 4107, 4133, 4097, 4154, 4030] avg=4096.0000 max=4154 max_rank=6 min=4030 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1084, 1028, 1014, 962, 4, 0, 0, 0, 1036, 1041, 1043, 1032, 0, 0, 0, 0, 994, 1041, 995, 1005, 25, 2, 0, 0, 1025, 985, 978, 1013, 0, 0, 0, 0, 1053, 1022, 1041, 1018, 0, 0, 0, 0, 1005, 1015, 998, 1088, 13, 5, 0, 0, 996, 1061, 1003, 1045, 5, 2, 0, 0, 1022, 1018, 1055, 996, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4152, 4062, 4001, 4134, 4124, 4112, 4091] avg=4096.0000 max=4152 max_rank=1 min=4001 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1031, 1029, 1015, 1018, 4, 0, 0, 0, 1005, 1027, 1029, 1042, 0, 0, 0, 0, 1014, 998, 1034, 1027, 24, 3, 0, 0, 1009, 1036, 1045, 1035, 0, 0, 0, 0, 1066, 996, 995, 1049, 0, 0, 0, 0, 1067, 1015, 1013, 1031, 13, 5, 0, 0, 994, 1031, 995, 1013, 5, 2, 0, 0, 1005, 1011, 1021, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4103, 4100, 4125, 4106, 4144, 4040, 4053] avg=4096.0000 max=4144 max_rank=5 min=4040 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1077, 1015, 1011, 1009, 4, 0, 0, 0, 1043, 991, 992, 1003, 0, 0, 0, 0, 1031, 1001, 1012, 1037, 25, 3, 0, 0, 1028, 1005, 1037, 1039, 0, 0, 0, 0, 995, 977, 1041, 1035, 0, 0, 0, 0, 1018, 1025, 1034, 1014, 13, 5, 0, 0, 1041, 1070, 988, 1039, 5, 2, 0, 0, 985, 1033, 1052, 1033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4029, 4109, 4109, 4048, 4109, 4145, 4103] avg=4096.0000 max=4145 max_rank=6 min=4029 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1045, 1003, 991, 1013, 4, 0, 0, 0, 1003, 1060, 1015, 1003, 0, 0, 0, 0, 1003, 1025, 1002, 1075, 24, 3, 0, 0, 1019, 1004, 1056, 1032, 0, 0, 0, 0, 1044, 1068, 1052, 1041, 0, 0, 0, 0, 977, 1025, 980, 1026, 13, 5, 0, 0, 1008, 1042, 1027, 1006, 5, 2, 0, 0, 1056, 979, 1034, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4081, 4132, 4111, 4205, 4026, 4090, 4067] avg=4096.0000 max=4205 max_rank=4 min=4026 min_rank=5 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1043, 1041, 1042, 1050, 4, 0, 0, 0, 1042, 986, 995, 1062, 0, 0, 0, 0, 1013, 975, 1033, 1033, 25, 3, 0, 0, 1005, 1071, 1041, 1039, 0, 0, 0, 0, 1018, 1002, 1009, 987, 0, 0, 0, 0, 1018, 1074, 1002, 1036, 13, 5, 0, 0, 1036, 1000, 988, 990, 5, 2, 0, 0, 971, 1037, 1000, 1072, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4085, 4082, 4156, 4016, 4148, 4021, 4080] avg=4096.0000 max=4180 max_rank=0 min=4016 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1040, 1058, 1052, 1037, 8, 0, 0, 0, 1015, 1021, 1022, 1027, 0, 0, 0, 0, 1004, 1045, 987, 1010, 27, 5, 0, 0, 1002, 1031, 1019, 995, 0, 0, 0, 0, 1005, 998, 993, 1032, 0, 0, 0, 0, 1001, 1033, 1009, 1014, 17, 9, 0, 0, 1043, 1038, 1042, 1004, 7, 9, 0, 0, 1089, 961, 991, 1068, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4195, 4085, 4078, 4047, 4028, 4083, 4143, 4109] avg=4096.0000 max=4195 max_rank=0 min=4028 min_rank=4 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1032, 1034, 1058, 1057, 1069, 1027, 1005, 946, 996, 1035, 1009, 1041, 1013, 997, 1029, 1025, 1021, 1011, 1002, 1052, 1026, 958, 992, 985, 1094, 1029, 1027, 1050, 1045, 996, 1016, 1091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1057, 988, 1001, 1029, 1036, 993, 1050, 996, 1010, 1044, 1074, 1051, 1017, 1046, 1040, 997, 1021, 1050, 1000, 1019, 971, 1028, 1003, 1045, 1007, 1017, 1038, 951, 1054, 1051, 1056, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1060, 1031, 992, 987, 1050, 990, 1042, 1023, 1032, 1003, 1023, 1037, 1031, 1029, 1023, 1011, 1054, 970, 1014, 1017, 1054, 1010, 1025, 1017, 1011, 1022, 1058, 1033, 1017, 1029, 1015, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1003, 1018, 1024, 1034, 1009, 984, 1000, 1011, 1034, 1029, 1021, 1082, 998, 976, 1037, 1018, 1030, 1013, 1026, 1051, 1031, 1019, 1030, 1069, 1027, 1059, 956, 1022, 1077, 1018, 1001, 1061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1008, 1014, 1028, 1022, 1026, 1012, 1072, 1061, 1070, 1035, 1036, 1018, 1011, 1003, 996, 992, 1043, 998, 990, 1017, 1075, 1056, 1032, 1033, 1009, 991, 1011, 1035, 1039, 1015, 1034, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[967, 1043, 1033, 1026, 1037, 1002, 1004, 1045, 997, 1107, 1023, 1046, 970, 984, 1028, 1017, 1059, 1052, 1036, 978, 1033, 1009, 1000, 1038, 1009, 1046, 1073, 1019, 1018, 1042, 1004, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1025, 996, 1032, 1087, 1052, 1061, 1004, 1073, 981, 1074, 1038, 984, 993, 1015, 1044, 1053, 1006, 979, 1031, 979, 973, 1007, 1037, 1017, 1025, 1041, 992, 1021, 1010, 1048, 1046, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1013, 1025, 1029, 1005, 1002, 1017, 999, 1004, 1002, 987, 1044, 1026, 1073, 1059, 1015, 1047, 1043, 1046, 1040, 1060, 995, 1027, 1020, 1023, 1018, 1025, 1096, 997, 1038, 1014, 972, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1057, 988, 1001, 1029, 1, 0, 0, 0, 1036, 993, 1050, 996, 8, 0, 0, 0, 1010, 1017, 1074, 1051, 0, 0, 0, 0, 1017, 1046, 1040, 997, 22, 0, 0, 0, 1021, 1050, 1000, 1019, 3, 3, 0, 0, 971, 1028, 1003, 1045, 11, 5, 0, 0, 1007, 1017, 1034, 951, 0, 0, 0, 0, 1032, 1051, 1056, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4083, 4152, 4122, 4096, 4063, 4009, 4167] avg=4096.0000 max=4167 max_rank=7 min=4009 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1060, 1031, 992, 987, 1, 0, 0, 0, 1050, 990, 1042, 1023, 8, 0, 0, 0, 1032, 978, 1023, 1037, 0, 0, 0, 0, 1031, 1029, 1023, 1011, 21, 0, 0, 0, 1054, 970, 1014, 1017, 3, 3, 0, 0, 1054, 1010, 1025, 1017, 11, 4, 0, 0, 1011, 1022, 1054, 1033, 0, 0, 0, 0, 995, 1029, 1015, 1058, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4113, 4070, 4115, 4061, 4121, 4120, 4097] avg=4096.0000 max=4121 max_rank=5 min=4061 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1008, 1014, 1028, 1022, 1, 0, 0, 0, 1026, 1012, 1072, 1061, 8, 0, 0, 0, 1070, 1009, 1036, 1018, 0, 0, 0, 0, 1011, 1003, 996, 992, 22, 0, 0, 0, 1043, 998, 990, 1017, 3, 3, 0, 0, 1075, 1056, 1032, 1033, 11, 4, 0, 0, 1009, 991, 1007, 1035, 0, 0, 0, 0, 1017, 1015, 1034, 986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4179, 4133, 4024, 4054, 4211, 4042, 4052] avg=4096.0000 max=4211 max_rank=5 min=4024 min_rank=3 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1032, 1034, 1058, 1057, 1, 0, 0, 0, 1069, 1027, 1005, 946, 8, 0, 0, 0, 996, 1009, 1009, 1041, 0, 0, 0, 0, 1013, 997, 1029, 1025, 22, 0, 0, 0, 1021, 1011, 1002, 1052, 3, 3, 0, 0, 1026, 958, 992, 985, 11, 4, 0, 0, 1094, 1029, 1023, 1050, 0, 0, 0, 0, 1023, 996, 1016, 1091, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 4055, 4055, 4086, 4092, 3976, 4196, 4126] avg=4096.0000 max=4196 max_rank=6 min=3976 min_rank=5 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1003, 1018, 1024, 1034, 3, 0, 0, 0, 1009, 984, 1000, 1011, 10, 0, 0, 0, 1034, 994, 1021, 1082, 0, 0, 0, 0, 998, 976, 1037, 1018, 26, 0, 0, 0, 1030, 1013, 1026, 1051, 8, 8, 0, 0, 1031, 1019, 1030, 1069, 14, 9, 0, 0, 1027, 1059, 945, 1022, 0, 0, 0, 0, 1045, 1018, 1001, 1061, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4014, 4131, 4055, 4136, 4172, 4053, 4125] avg=4096.0000 max=4172 max_rank=5 min=4014 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[967, 1043, 1033, 1026, 1, 0, 0, 0, 1037, 1002, 1004, 1045, 8, 0, 0, 0, 997, 1078, 1023, 1046, 0, 0, 0, 0, 970, 984, 1028, 1017, 24, 0, 0, 0, 1059, 1052, 1036, 978, 4, 3, 0, 0, 1033, 1009, 1000, 1038, 11, 5, 0, 0, 1009, 1046, 1068, 1019, 0, 0, 0, 0, 996, 1042, 1004, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4096, 4144, 4023, 4132, 4096, 4142, 4065] avg=4096.0000 max=4144 max_rank=2 min=4023 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 996, 1032, 1087, 1, 0, 0, 0, 1052, 1061, 1004, 1073, 8, 0, 0, 0, 981, 1046, 1038, 984, 0, 0, 0, 0, 993, 1015, 1044, 1053, 23, 0, 0, 0, 1006, 979, 1031, 979, 3, 3, 0, 0, 973, 1007, 1037, 1017, 10, 5, 0, 0, 1025, 1041, 988, 1021, 0, 0, 0, 0, 989, 1048, 1046, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4198, 4049, 4128, 4001, 4049, 4075, 4127] avg=4096.0000 max=4198 max_rank=1 min=4001 min_rank=4 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16252 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1013, 1025, 1029, 1005, 1, 0, 0, 0, 1002, 1017, 999, 1004, 8, 0, 0, 0, 1002, 962, 1044, 1026, 0, 0, 0, 0, 1073, 1059, 1015, 1047, 21, 0, 0, 0, 1043, 1046, 1040, 1060, 4, 3, 0, 0, 995, 1027, 1020, 1023, 11, 4, 0, 0, 1018, 1025, 1091, 997, 0, 0, 0, 0, 1016, 1014, 972, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4030, 4034, 4215, 4196, 4080, 4131, 4009] avg=4096.0000 max=4215 max_rank=3 min=4009 min_rank=7 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1042, 1026, 1022, 1016, 1030, 1000, 988, 1025, 1018, 1014, 1027, 1053, 1034, 1031, 1021, 1010, 1024, 1005, 1012, 1048, 1010, 1020, 1045, 1055, 1008, 1041, 1018, 1045, 1075, 979, 972, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1034, 1013, 1006, 1028, 1040, 1041, 1011, 1074, 1063, 1031, 1026, 1059, 1015, 1017, 1018, 1042, 1010, 1039, 1031, 1029, 1051, 1035, 1026, 1014, 984, 1016, 1000, 979, 986, 1030, 996, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1064, 1032, 1017, 1040, 980, 1041, 1059, 1001, 971, 994, 1031, 1053, 1021, 1006, 986, 961, 1043, 1041, 1013, 1068, 1006, 1054, 992, 1053, 1033, 1051, 1008, 1046, 1037, 998, 1044, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1062, 1059, 1040, 1011, 1031, 974, 1040, 996, 1089, 1010, 1032, 1044, 1067, 1016, 1005, 1036, 1004, 1042, 980, 1031, 1013, 1002, 1035, 965, 1063, 1080, 1016, 1014, 1007, 977, 1033, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1042, 1022, 997, 1001, 1042, 1018, 983, 1027, 1011, 1026, 1050, 1085, 1000, 1014, 1043, 1025, 1018, 1007, 1048, 1016, 1013, 1006, 982, 1056, 1057, 1025, 1016, 1029, 991, 1024, 1059, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1031, 1033, 1003, 1069, 993, 1083, 1053, 1016, 995, 1041, 1017, 1036, 1025, 1027, 1017, 1048, 1045, 984, 1060, 1039, 982, 1035, 960, 1019, 998, 1047, 1000, 1028, 1016, 1061, 1008, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1029, 1012, 1040, 1034, 966, 984, 972, 1032, 1002, 1044, 1002, 1054, 1059, 1051, 1001, 974, 1007, 1056, 1002, 1051, 1050, 1039, 1049, 996, 1032, 1033, 1040, 1032, 1038, 1050, 998, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1042, 980, 1005, 1023, 981, 981, 1022, 978, 1021, 1067, 1008, 992, 1059, 1061, 965, 986, 1041, 1059, 1072, 1012, 1038, 972, 1008, 1058, 1068, 1034, 1021, 1033, 1071, 1042, 999, 1069] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1030, 1026, 1022, 1016, 0, 0, 0, 0, 1030, 1000, 988, 1025, 24, 13, 0, 0, 1018, 1014, 1027, 1029, 0, 0, 0, 0, 1034, 1031, 1021, 1010, 8, 7, 0, 0, 1024, 1005, 1012, 1028, 0, 0, 0, 0, 1010, 1020, 1045, 1055, 8, 7, 0, 0, 1008, 1026, 1018, 1045, 0, 0, 0, 0, 1075, 979, 972, 1054, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4080, 4088, 4111, 4069, 4145, 4097, 4084] avg=4096.0000 max=4145 max_rank=5 min=4069 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1052, 1032, 1017, 1040, 0, 0, 0, 0, 980, 1041, 1059, 1001, 24, 13, 0, 0, 971, 994, 1031, 1029, 0, 0, 0, 0, 1021, 1006, 986, 961, 8, 7, 0, 0, 1043, 1041, 1013, 1048, 0, 0, 0, 0, 1006, 1054, 992, 1053, 8, 7, 0, 0, 1033, 1036, 1008, 1046, 0, 0, 0, 0, 1037, 998, 1044, 1024, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4118, 4025, 3989, 4145, 4120, 4123, 4107] avg=4096.0000 max=4145 max_rank=4 min=3989 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1050, 1059, 1040, 1011, 0, 0, 0, 0, 1031, 974, 1040, 996, 24, 13, 0, 0, 1089, 1010, 1032, 1020, 0, 0, 0, 0, 1067, 1016, 1005, 1036, 8, 7, 0, 0, 1004, 1042, 980, 1011, 0, 0, 0, 0, 1013, 1002, 1035, 965, 8, 7, 0, 0, 1063, 1065, 1016, 1014, 0, 0, 0, 0, 1007, 977, 1033, 994, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4078, 4151, 4139, 4037, 4030, 4158, 4015] avg=4096.0000 max=4160 max_rank=0 min=4015 min_rank=7 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1022, 1013, 1006, 1028, 0, 0, 0, 0, 1040, 1041, 1011, 1074, 25, 13, 0, 0, 1063, 1031, 1026, 1034, 0, 0, 0, 0, 1015, 1017, 1018, 1042, 8, 7, 0, 0, 1010, 1039, 1031, 1009, 0, 0, 0, 0, 1051, 1035, 1026, 1014, 8, 7, 0, 0, 984, 1001, 1000, 979, 0, 0, 0, 0, 986, 1030, 996, 1024, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4204, 4154, 4107, 4089, 4141, 3964, 4040] avg=4096.0000 max=4204 max_rank=1 min=3964 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16234 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1030, 1022, 997, 1001, 0, 0, 0, 0, 1042, 1018, 983, 1027, 25, 13, 0, 0, 1011, 1026, 1050, 1060, 0, 0, 0, 0, 1000, 1014, 1043, 1025, 8, 7, 0, 0, 1018, 1007, 1048, 996, 0, 0, 0, 0, 1013, 1006, 982, 1056, 8, 7, 0, 0, 1057, 1010, 1016, 1029, 0, 0, 0, 0, 991, 1024, 1059, 1035, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4108, 4147, 4097, 4069, 4072, 4112, 4113] avg=4096.0000 max=4147 max_rank=2 min=4050 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1008, 1033, 1003, 1069, 0, 0, 0, 0, 993, 1083, 1053, 1016, 29, 17, 0, 0, 995, 1041, 1017, 1007, 0, 0, 0, 0, 1025, 1027, 1017, 1048, 12, 10, 0, 0, 1045, 984, 1060, 1010, 0, 0, 0, 0, 982, 1035, 960, 1019, 12, 12, 0, 0, 998, 1025, 1000, 1028, 0, 0, 0, 0, 1016, 1061, 1008, 999, 11, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4191, 4060, 4139, 4099, 4020, 4051, 4095] avg=4096.0000 max=4191 max_rank=1 min=4020 min_rank=5 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1017, 1012, 1040, 1034, 0, 0, 0, 0, 966, 984, 972, 1032, 24, 13, 0, 0, 1002, 1044, 1002, 1030, 0, 0, 0, 0, 1059, 1051, 1001, 974, 8, 7, 0, 0, 1007, 1056, 1002, 1031, 0, 0, 0, 0, 1050, 1039, 1049, 996, 8, 7, 0, 0, 1032, 1018, 1040, 1032, 0, 0, 0, 0, 1038, 1050, 998, 1039, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 3991, 4078, 4100, 4096, 4149, 4122, 4129] avg=4096.0000 max=4149 max_rank=5 min=3991 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1030, 980, 1005, 1023, 0, 0, 0, 0, 981, 981, 1022, 978, 23, 13, 0, 0, 1021, 1067, 1008, 969, 0, 0, 0, 0, 1059, 1061, 965, 986, 8, 7, 0, 0, 1041, 1059, 1072, 992, 0, 0, 0, 0, 1038, 972, 1008, 1058, 8, 7, 0, 0, 1068, 1019, 1021, 1033, 0, 0, 0, 0, 1071, 1042, 999, 1069, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 3998, 4065, 4086, 4164, 4091, 4141, 4185] avg=4096.0000 max=4185 max_rank=7 min=3998 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[986, 1062, 1037, 1040, 1005, 1072, 1013, 1051, 988, 1044, 1063, 1061, 995, 1045, 994, 1007, 1014, 971, 1006, 991, 1028, 1070, 962, 1025, 983, 1031, 1040, 999, 1000, 1058, 1064, 1063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1006, 1044, 1054, 978, 1029, 1007, 1032, 1026, 1018, 1028, 1058, 1032, 1013, 1047, 983, 1011, 1051, 980, 1041, 995, 1048, 1017, 1061, 1023, 1019, 1019, 1018, 1010, 1032, 1024, 1037, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[991, 990, 1004, 1006, 1044, 983, 1037, 1013, 1077, 1046, 1033, 1038, 1033, 1003, 1053, 1022, 1031, 1050, 1018, 1037, 987, 1045, 1042, 1026, 997, 1028, 990, 1009, 1052, 1047, 1014, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[960, 1077, 1043, 1030, 1039, 985, 988, 1002, 999, 1026, 996, 992, 997, 1041, 1094, 1012, 1054, 1067, 1002, 991, 1017, 1014, 1114, 1023, 1024, 1022, 978, 1045, 1046, 1046, 997, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1034, 1039, 1034, 1008, 978, 1042, 1026, 1043, 1010, 1018, 1001, 1012, 1001, 1000, 995, 1065, 995, 1028, 993, 1059, 1045, 1088, 1062, 1020, 1079, 987, 999, 1010, 1055, 1017, 1023, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1045, 1046, 1000, 1024, 1058, 1003, 975, 1019, 1052, 1025, 1054, 997, 1014, 987, 1043, 1039, 1040, 1012, 1036, 1000, 1017, 1075, 992, 1036, 1022, 998, 1029, 974, 1009, 1030, 1091, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1007, 1021, 978, 1007, 1001, 990, 1025, 997, 1040, 1048, 1001, 1059, 1020, 1066, 997, 1039, 1015, 1089, 1032, 999, 1019, 1060, 956, 1044, 1021, 1043, 1028, 999, 1003, 1008, 1055, 1101] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1037, 1039, 1043, 1005, 999, 1084, 999, 1049, 1027, 986, 1039, 1011, 1056, 1008, 1053, 1042, 1038, 992, 1025, 994, 996, 1040, 1043, 1020, 1000, 1056, 1042, 986, 1002, 1040, 1009, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[986, 1062, 1037, 1040, 11, 0, 0, 0, 1005, 1072, 1013, 1051, 19, 0, 0, 0, 988, 1044, 1049, 1061, 0, 0, 0, 0, 995, 1045, 994, 1007, 0, 0, 0, 0, 1014, 971, 1006, 991, 12, 3, 0, 0, 1028, 1039, 962, 1025, 0, 0, 0, 0, 983, 1031, 1040, 999, 36, 0, 0, 0, 1000, 1058, 1064, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4160, 4142, 4041, 3997, 4054, 4089, 4149] avg=4096.0000 max=4160 max_rank=1 min=3997 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[991, 990, 1004, 1006, 10, 0, 0, 0, 1044, 983, 1037, 1013, 18, 0, 0, 0, 1077, 1046, 1020, 1038, 0, 0, 0, 0, 1033, 1003, 1053, 1022, 0, 0, 0, 0, 1031, 1050, 1018, 1037, 12, 3, 0, 0, 987, 1015, 1042, 1026, 0, 0, 0, 0, 997, 1028, 990, 1009, 34, 0, 0, 0, 1052, 1047, 1014, 988, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4001, 4095, 4181, 4111, 4151, 4070, 4058, 4101] avg=4096.0000 max=4181 max_rank=2 min=4001 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1006, 1044, 1054, 978, 11, 0, 0, 0, 1029, 1007, 1032, 1026, 18, 0, 0, 0, 1018, 1028, 1044, 1032, 0, 0, 0, 0, 1013, 1047, 983, 1011, 0, 0, 0, 0, 1051, 980, 1041, 995, 11, 3, 0, 0, 1048, 988, 1061, 1023, 0, 0, 0, 0, 1019, 1019, 1018, 1010, 35, 0, 0, 0, 1032, 1024, 1037, 992, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4112, 4122, 4054, 4081, 4120, 4101, 4085] avg=4096.0000 max=4122 max_rank=2 min=4054 min_rank=3 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1037, 1039, 1043, 1005, 10, 0, 0, 0, 999, 1084, 999, 1049, 18, 0, 0, 0, 1027, 986, 1026, 1011, 0, 0, 0, 0, 1056, 1008, 1053, 1042, 0, 0, 0, 0, 1038, 992, 1025, 994, 11, 3, 0, 0, 996, 1011, 1043, 1020, 0, 0, 0, 0, 1000, 1056, 1042, 986, 34, 0, 0, 0, 1002, 1040, 1009, 974, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4149, 4050, 4159, 4063, 4070, 4118, 4025] avg=4096.0000 max=4159 max_rank=3 min=4025 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16276 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[960, 1077, 1043, 1030, 10, 0, 0, 0, 1039, 985, 988, 1002, 18, 0, 0, 0, 999, 1026, 983, 992, 0, 0, 0, 0, 997, 1041, 1094, 1012, 0, 0, 0, 0, 1054, 1067, 1002, 991, 11, 3, 0, 0, 1017, 985, 1114, 1023, 0, 0, 0, 0, 1024, 1022, 978, 1045, 35, 0, 0, 0, 1046, 1046, 997, 1012, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4032, 4000, 4144, 4128, 4139, 4104, 4101] avg=4096.0000 max=4144 max_rank=3 min=4000 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1007, 1021, 978, 1007, 14, 7, 0, 0, 1001, 990, 1025, 997, 21, 4, 0, 0, 1040, 1048, 983, 1059, 0, 0, 0, 0, 1020, 1066, 997, 1032, 0, 0, 0, 0, 1015, 1089, 1032, 999, 16, 4, 0, 0, 1019, 1023, 956, 1044, 0, 0, 0, 0, 1021, 1043, 1028, 999, 41, 0, 0, 0, 1003, 1008, 1055, 1056, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4038, 4130, 4115, 4155, 4042, 4132, 4122] avg=4096.0000 max=4155 max_rank=4 min=4034 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1045, 1046, 1000, 1024, 10, 0, 0, 0, 1058, 1003, 975, 1019, 19, 0, 0, 0, 1052, 1025, 1041, 997, 0, 0, 0, 0, 1014, 987, 1043, 1039, 0, 0, 0, 0, 1040, 1012, 1036, 1000, 12, 3, 0, 0, 1017, 1044, 992, 1036, 0, 0, 0, 0, 1022, 998, 1029, 974, 34, 0, 0, 0, 1009, 1030, 1091, 992, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4074, 4115, 4083, 4103, 4089, 4057, 4122] avg=4096.0000 max=4125 max_rank=0 min=4057 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1034, 1039, 1034, 1008, 10, 0, 0, 0, 978, 1042, 1026, 1043, 19, 0, 0, 0, 1010, 1018, 988, 1012, 0, 0, 0, 0, 1001, 1000, 995, 1065, 0, 0, 0, 0, 995, 1028, 993, 1059, 12, 3, 0, 0, 1045, 1057, 1062, 1020, 0, 0, 0, 0, 1079, 987, 999, 1010, 34, 0, 0, 0, 1055, 1017, 1023, 968, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4108, 4028, 4061, 4090, 4184, 4109, 4063] avg=4096.0000 max=4184 max_rank=5 min=4028 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8064, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1003, 1060, 981, 1065, 1007, 1023, 970, 1042, 1025, 1055, 995, 1030, 1005, 1064, 1030, 1017, 996, 991, 1037, 1027, 1014, 1045, 1013, 1027, 1074, 1053, 952, 1042, 1054, 1048, 948, 1075] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1004, 1019, 1041, 1075, 1044, 1042, 985, 948, 1008, 1026, 1032, 998, 1030, 1006, 1027, 1019, 1049, 1020, 1058, 1054, 1042, 1064, 1047, 1020, 1020, 1035, 948, 1049, 990, 1003, 1043, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[976, 1033, 1090, 996, 1053, 1023, 1012, 1004, 1065, 1039, 1019, 1014, 1026, 991, 1016, 1030, 940, 999, 1062, 1063, 1070, 997, 1011, 981, 975, 1063, 1051, 1082, 998, 1033, 1005, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1004, 1029, 1034, 997, 989, 1048, 1057, 1023, 1013, 1045, 1035, 1022, 1028, 1045, 981, 1044, 1039, 1014, 1051, 1010, 1010, 1012, 1044, 1017, 995, 1023, 992, 983, 1031, 1054, 1048, 1051] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1034, 993, 1062, 1049, 1052, 978, 1019, 1005, 1004, 1067, 1038, 1061, 1052, 988, 979, 1033, 998, 1014, 1008, 1037, 1029, 1033, 1013, 1017, 1034, 1051, 1014, 981, 1027, 1062, 1021, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1024, 1077, 1015, 1020, 1050, 984, 1015, 1028, 1063, 1037, 963, 1026, 1087, 1059, 1029, 1004, 1023, 1036, 982, 1020, 996, 980, 1042, 1028, 978, 1004, 1022, 1034, 1057, 1033, 1004, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[993, 1082, 1031, 993, 988, 1035, 1030, 1061, 1040, 999, 1005, 1054, 1001, 1032, 1027, 989, 1034, 1048, 993, 989, 970, 1017, 999, 1109, 1038, 991, 1027, 1046, 1073, 1004, 1022, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1063, 1057, 994, 1033, 1055, 1042, 1028, 1015, 974, 999, 1028, 958, 1000, 992, 1064, 1015, 986, 1072, 1032, 1034, 995, 1049, 1033, 1043, 1016, 1053, 1033, 1036, 1015, 989, 1029, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1003, 1042, 981, 1065, 0, 0, 0, 0, 1007, 1023, 970, 1042, 14, 0, 0, 0, 1025, 1055, 995, 1030, 3, 0, 0, 0, 1005, 1064, 1030, 1017, 7, 0, 0, 0, 996, 991, 1037, 1027, 6, 0, 0, 0, 1014, 1045, 1013, 1027, 0, 0, 0, 0, 1074, 1053, 952, 1042, 7, 2, 0, 0, 1054, 1048, 948, 1054, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4056, 4108, 4123, 4057, 4099, 4130, 4104] avg=4096.0000 max=4130 max_rank=6 min=4056 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[976, 1015, 1090, 996, 0, 0, 0, 0, 1053, 1023, 1012, 1004, 14, 0, 0, 0, 1065, 1039, 1019, 1014, 3, 0, 0, 0, 1026, 991, 1016, 1030, 7, 0, 0, 0, 940, 999, 1062, 1063, 6, 0, 0, 0, 1070, 997, 1011, 981, 0, 0, 0, 0, 975, 1063, 1051, 1082, 7, 2, 0, 0, 998, 1033, 1005, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4106, 4140, 4070, 4070, 4059, 4180, 4066] avg=4096.0000 max=4180 max_rank=6 min=4059 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1034, 976, 1062, 1049, 0, 0, 0, 0, 1052, 978, 1019, 1005, 13, 0, 0, 0, 1004, 1067, 1038, 1061, 3, 0, 0, 0, 1052, 988, 979, 1033, 6, 0, 0, 0, 998, 1014, 1008, 1037, 6, 0, 0, 0, 1029, 1033, 1013, 1017, 0, 0, 0, 0, 1034, 1051, 1014, 981, 6, 2, 0, 0, 1027, 1062, 1021, 996, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4067, 4173, 4058, 4063, 4092, 4088, 4106] avg=4096.0000 max=4173 max_rank=2 min=4058 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1004, 1011, 1034, 997, 0, 0, 0, 0, 989, 1048, 1057, 1023, 14, 0, 0, 0, 1013, 1045, 1035, 1022, 3, 0, 0, 0, 1028, 1045, 981, 1044, 7, 0, 0, 0, 1039, 1014, 1051, 1010, 6, 0, 0, 0, 1010, 1012, 1044, 1017, 0, 0, 0, 0, 995, 1023, 992, 983, 7, 2, 0, 0, 1031, 1054, 1048, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4131, 4118, 4105, 4120, 4083, 4002, 4163] avg=4096.0000 max=4163 max_rank=7 min=4002 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1004, 986, 1041, 1075, 0, 0, 0, 0, 1044, 1042, 985, 948, 16, 0, 0, 0, 1008, 1026, 1032, 998, 9, 0, 0, 0, 1030, 1006, 1027, 1019, 10, 0, 0, 0, 1049, 1020, 1058, 1054, 10, 0, 0, 0, 1042, 1064, 1047, 1020, 1, 0, 0, 0, 1020, 1035, 948, 1049, 9, 3, 0, 0, 990, 1003, 1043, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4035, 4073, 4092, 4191, 4174, 4064, 4033] avg=4096.0000 max=4191 max_rank=4 min=4033 min_rank=7 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[993, 1063, 1031, 993, 0, 0, 0, 0, 988, 1035, 1030, 1061, 14, 0, 0, 0, 1040, 999, 1005, 1054, 4, 0, 0, 0, 1001, 1032, 1027, 989, 7, 0, 0, 0, 1034, 1048, 993, 989, 6, 0, 0, 0, 970, 1017, 999, 1109, 0, 0, 0, 0, 1038, 991, 1027, 1046, 7, 2, 0, 0, 1073, 1004, 1022, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4128, 4102, 4056, 4070, 4095, 4111, 4126] avg=4096.0000 max=4128 max_rank=1 min=4056 min_rank=3 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1024, 1059, 1015, 1020, 0, 0, 0, 0, 1050, 984, 1015, 1028, 14, 0, 0, 0, 1063, 1037, 963, 1026, 3, 0, 0, 0, 1087, 1059, 1029, 1004, 7, 0, 0, 0, 1023, 1036, 982, 1020, 6, 0, 0, 0, 996, 980, 1042, 1028, 0, 0, 0, 0, 978, 1004, 1022, 1034, 7, 2, 0, 0, 1057, 1033, 1004, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4091, 4092, 4186, 4067, 4046, 4047, 4121] avg=4096.0000 max=4186 max_rank=3 min=4046 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1063, 1039, 994, 1033, 0, 0, 0, 0, 1055, 1042, 1028, 1015, 14, 0, 0, 0, 974, 999, 1028, 958, 3, 0, 0, 0, 1000, 992, 1064, 1015, 7, 0, 0, 0, 986, 1072, 1032, 1034, 6, 0, 0, 0, 995, 1049, 1033, 1043, 0, 0, 0, 0, 1016, 1053, 1033, 1036, 6, 2, 0, 0, 1015, 989, 1029, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4154, 3962, 4078, 4130, 4120, 4146, 4049] avg=4096.0000 max=4154 max_rank=1 min=3962 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1019, 986, 1058, 1045, 991, 995, 1008, 1038, 1048, 1029, 1031, 1019, 1107, 1038, 985, 997, 1005, 1018, 1052, 1099, 996, 994, 936, 1045, 1082, 1015, 1044, 1026, 1035, 1041, 1008, 978] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1039, 988, 1029, 1034, 984, 976, 1016, 1031, 1074, 1005, 1024, 1048, 1033, 1053, 996, 1068, 1059, 1032, 973, 1024, 1002, 1086, 1014, 1038, 999, 1025, 1028, 1003, 1033, 993, 1027, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[985, 991, 1018, 999, 976, 1028, 1037, 1040, 1022, 1083, 997, 1050, 999, 1033, 1019, 1028, 1016, 1025, 1085, 990, 1009, 986, 1033, 1048, 1057, 1046, 1031, 1059, 1054, 954, 1025, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[996, 1027, 1054, 1079, 999, 1033, 1080, 1066, 1032, 1052, 1008, 1029, 1035, 1005, 1001, 1021, 975, 1006, 1034, 1011, 1006, 1040, 1016, 1006, 1055, 1056, 992, 1011, 1006, 1012, 1003, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1013, 1090, 1033, 1002, 1001, 1023, 1047, 1032, 1031, 1000, 1052, 984, 1005, 1049, 1022, 995, 1006, 1029, 1081, 996, 1008, 1047, 1053, 1033, 1006, 1000, 998, 1040, 1020, 982, 1065, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1024, 1034, 1050, 1050, 1018, 1035, 987, 1046, 1008, 1009, 996, 1047, 991, 1054, 1037, 1031, 1018, 1023, 1015, 1033, 1024, 1045, 1013, 1014, 1040, 1014, 1003, 948, 1074, 1029, 1041, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[998, 972, 1061, 1059, 1019, 1012, 1061, 1008, 968, 1045, 1079, 999, 1026, 1023, 1037, 1034, 1025, 984, 1038, 1022, 1044, 1013, 968, 981, 1027, 1024, 1011, 1051, 1065, 1049, 1055, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1040, 1022, 956, 1015, 1045, 1005, 1037, 1033, 972, 994, 1025, 1043, 1023, 1041, 1034, 1062, 1029, 992, 1008, 1012, 1002, 999, 1043, 1019, 1062, 1041, 992, 1083, 1034, 1031, 1014, 1060] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1039, 988, 1029, 1034, 0, 0, 0, 0, 984, 976, 1016, 1031, 6, 0, 0, 0, 1074, 1005, 1024, 1044, 0, 0, 0, 0, 1033, 1039, 996, 1068, 0, 0, 0, 0, 1059, 1032, 973, 1024, 4, 2, 0, 0, 1002, 1086, 1014, 1038, 14, 11, 0, 0, 988, 1025, 1028, 1003, 0, 0, 0, 0, 1025, 993, 1027, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4013, 4147, 4136, 4094, 4165, 4044, 4079] avg=4096.0000 max=4165 max_rank=5 min=4013 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[996, 1027, 1054, 1079, 0, 0, 0, 0, 999, 1033, 1080, 1066, 6, 1, 0, 0, 1032, 1052, 1008, 1025, 0, 0, 0, 0, 1035, 992, 1001, 1021, 0, 0, 0, 0, 975, 1006, 1034, 1011, 4, 2, 0, 0, 1006, 1040, 1016, 1006, 13, 11, 0, 0, 1043, 1056, 992, 1011, 0, 0, 0, 0, 998, 1012, 1003, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4185, 4117, 4049, 4032, 4092, 4102, 4035] avg=4096.0000 max=4185 max_rank=1 min=4032 min_rank=4 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1024, 1034, 1050, 1050, 0, 0, 0, 0, 1018, 1035, 987, 1046, 6, 0, 0, 0, 1008, 1009, 996, 1043, 0, 0, 0, 0, 991, 1040, 1037, 1031, 0, 0, 0, 0, 1018, 1023, 1015, 1033, 4, 2, 0, 0, 1024, 1045, 1013, 1014, 14, 11, 0, 0, 1029, 1014, 1003, 948, 0, 0, 0, 0, 1066, 1029, 1041, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4092, 4056, 4099, 4095, 4121, 3994, 4153] avg=4096.0000 max=4158 max_rank=0 min=3994 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1019, 986, 1058, 1045, 0, 0, 0, 0, 991, 995, 1008, 1038, 6, 1, 0, 0, 1048, 1029, 1031, 1015, 0, 0, 0, 0, 1107, 1024, 985, 997, 0, 0, 0, 0, 1005, 1018, 1052, 1099, 4, 2, 0, 0, 996, 994, 936, 1045, 14, 12, 0, 0, 1069, 1015, 1044, 1026, 0, 0, 0, 0, 1027, 1041, 1008, 978, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4039, 4123, 4113, 4180, 3997, 4154, 4054] avg=4096.0000 max=4180 max_rank=4 min=3997 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[985, 991, 1018, 999, 0, 0, 0, 0, 976, 1028, 1037, 1040, 6, 1, 0, 0, 1022, 1083, 997, 1046, 0, 0, 0, 0, 999, 1019, 1019, 1028, 0, 0, 0, 0, 1016, 1025, 1085, 990, 4, 2, 0, 0, 1009, 986, 1033, 1048, 14, 11, 0, 0, 1045, 1046, 1031, 1059, 0, 0, 0, 0, 1046, 954, 1025, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3993, 4088, 4148, 4065, 4122, 4101, 4181, 4070] avg=4096.0000 max=4181 max_rank=6 min=3993 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1013, 1090, 1033, 1002, 0, 0, 0, 0, 1001, 1023, 1047, 1032, 6, 0, 0, 0, 1031, 1000, 1052, 981, 0, 0, 0, 0, 1005, 1035, 1022, 995, 0, 0, 0, 0, 1006, 1029, 1081, 996, 3, 2, 0, 0, 1008, 1047, 1053, 1033, 14, 11, 0, 0, 995, 1000, 998, 1040, 0, 0, 0, 0, 1012, 982, 1065, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4109, 4064, 4057, 4117, 4166, 4033, 4084] avg=4096.0000 max=4166 max_rank=5 min=4033 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[998, 972, 1061, 1059, 2, 0, 0, 0, 1019, 1012, 1061, 1008, 11, 4, 0, 0, 968, 1045, 1079, 991, 0, 0, 0, 0, 1026, 1006, 1037, 1034, 0, 0, 0, 0, 1025, 984, 1038, 1022, 6, 6, 0, 0, 1044, 1013, 968, 981, 17, 15, 0, 0, 1008, 1024, 1011, 1051, 0, 0, 0, 0, 1048, 1049, 1055, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4115, 4083, 4103, 4081, 4038, 4094, 4162] avg=4096.0000 max=4162 max_rank=7 min=4038 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1040, 1022, 956, 1015, 0, 0, 0, 0, 1045, 1005, 1037, 1033, 6, 1, 0, 0, 972, 994, 1025, 1039, 0, 0, 0, 0, 1023, 1027, 1034, 1062, 0, 0, 0, 0, 1029, 992, 1008, 1012, 4, 2, 0, 0, 1002, 999, 1043, 1019, 14, 11, 0, 0, 1050, 1041, 992, 1083, 0, 0, 0, 0, 1026, 1031, 1014, 1060, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4127, 4030, 4146, 4047, 4088, 4166, 4131] avg=4096.0000 max=4166 max_rank=6 min=4030 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-08-30 04:58:09] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 631.3 | throughput per GPU (TFLOP/s/GPU): 528.1 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.041442E+00 | loss scale: 1.0 | grad norm: 4.447 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1029, 1042, 1044, 1004, 1004, 977, 1011, 1022, 998, 1040, 1052, 1023, 1091, 941, 1041, 1009, 1089, 1049, 1008, 987, 1050, 1025, 1000, 1035, 1023, 1038, 1017, 1019, 1043, 1073, 954, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[991, 1039, 1014, 1068, 1032, 996, 1009, 1052, 1007, 992, 1041, 1000, 1031, 1065, 1029, 1018, 1011, 1019, 1055, 1052, 1027, 1045, 1043, 1039, 1024, 1035, 980, 1033, 1013, 1008, 992, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1015, 974, 1050, 1001, 1026, 1039, 1052, 1086, 982, 1008, 1051, 1070, 1040, 1032, 1000, 980, 1045, 1043, 1046, 1014, 1065, 1029, 992, 1065, 959, 943, 1028, 993, 1052, 1024, 1027, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1057, 1066, 1059, 1025, 961, 993, 985, 998, 986, 969, 1014, 1048, 1040, 1043, 1041, 1004, 1009, 1043, 1027, 1040, 1060, 1060, 1034, 1023, 1041, 1059, 1024, 981, 1022, 993, 1041, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1030, 1087, 1018, 1025, 1010, 1003, 1064, 1053, 981, 1029, 1043, 1088, 940, 1008, 1018, 1008, 1051, 1055, 1022, 1023, 993, 1033, 1034, 1073, 1055, 994, 1018, 1015, 998, 993, 997, 1009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1013, 1051, 1011, 1006, 1001, 1037, 1000, 1031, 1004, 1077, 1043, 1032, 1012, 1025, 1044, 1014, 1070, 1007, 1043, 1047, 1017, 988, 1017, 1027, 974, 1022, 1008, 1020, 1072, 1035, 989, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1024, 1051, 1030, 1044, 1031, 1018, 992, 1034, 1027, 1073, 1010, 1018, 972, 1060, 1027, 1023, 976, 980, 1012, 1033, 1085, 1035, 1057, 956, 978, 1028, 1056, 1007, 1031, 1060, 1042, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1072, 1022, 1007, 1048, 1052, 1028, 998, 994, 1006, 994, 1029, 1036, 1004, 1038, 1015, 1003, 1008, 992, 1007, 1024, 1040, 1014, 1044, 956, 1031, 1060, 1047, 1059, 1050, 1000, 1025, 1065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1029, 1012, 1044, 1004, 0, 0, 0, 0, 1004, 977, 1011, 1022, 16, 6, 0, 0, 998, 1040, 1052, 1023, 0, 0, 0, 0, 1091, 941, 1041, 1009, 11, 8, 0, 0, 1074, 1049, 1008, 987, 0, 0, 0, 0, 1026, 1025, 1000, 1035, 0, 0, 0, 0, 1023, 1038, 1017, 1019, 24, 0, 0, 0, 1043, 1073, 954, 1030, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4036, 4113, 4101, 4118, 4086, 4121, 4104] avg=4096.0000 max=4121 max_rank=6 min=4036 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[991, 1009, 1014, 1068, 0, 0, 0, 0, 1032, 996, 1009, 1052, 15, 6, 0, 0, 1007, 992, 1041, 1000, 0, 0, 0, 0, 1031, 1065, 1029, 1018, 10, 7, 0, 0, 998, 1019, 1055, 1052, 0, 0, 0, 0, 1005, 1045, 1043, 1039, 0, 0, 0, 0, 1024, 1035, 980, 1033, 24, 0, 0, 0, 1013, 1008, 992, 1008, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4110, 4040, 4160, 4124, 4132, 4096, 4024] avg=4096.0000 max=4160 max_rank=3 min=4024 min_rank=7 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1015, 946, 1050, 1001, 0, 0, 0, 0, 1026, 1039, 1052, 1086, 16, 5, 0, 0, 982, 1008, 1051, 1070, 0, 0, 0, 0, 1040, 1032, 1000, 980, 11, 8, 0, 0, 1031, 1043, 1046, 1014, 0, 0, 0, 0, 1041, 1029, 992, 1065, 0, 0, 0, 0, 959, 943, 1028, 993, 23, 0, 0, 0, 1052, 1024, 1027, 1037, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4224, 4111, 4071, 4134, 4127, 3946, 4143] avg=4096.0000 max=4224 max_rank=1 min=3946 min_rank=6 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1057, 1028, 1059, 1025, 0, 0, 0, 0, 961, 993, 985, 998, 20, 9, 0, 0, 986, 969, 1014, 1045, 0, 0, 0, 0, 1040, 1043, 1041, 1004, 14, 12, 0, 0, 987, 1043, 1027, 1040, 0, 0, 0, 0, 1028, 1060, 1034, 1023, 0, 0, 0, 0, 1041, 1059, 1024, 981, 29, 0, 0, 0, 1022, 993, 1041, 1022, 8, 3, 0, 0] rerouted_tokens_per_ep_rank=[4169, 3966, 4014, 4154, 4097, 4145, 4134, 4089] avg=4096.0000 max=4169 max_rank=0 min=3966 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1024, 1020, 1030, 1044, 0, 0, 0, 0, 1031, 1018, 992, 1034, 16, 6, 0, 0, 1027, 1073, 1010, 1018, 0, 0, 0, 0, 972, 1060, 1027, 1023, 10, 8, 0, 0, 963, 980, 1012, 1033, 0, 0, 0, 0, 1061, 1035, 1057, 956, 0, 0, 0, 0, 978, 1028, 1056, 1007, 25, 0, 0, 0, 1031, 1060, 1042, 998, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4097, 4128, 4100, 3988, 4109, 4094, 4134] avg=4096.0000 max=4134 max_rank=7 min=3988 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1013, 1020, 1011, 1006, 0, 0, 0, 0, 1001, 1037, 1000, 1031, 15, 6, 0, 0, 1004, 1077, 1043, 1032, 0, 0, 0, 0, 1012, 1025, 1044, 1014, 11, 7, 0, 0, 1055, 1007, 1043, 1047, 0, 0, 0, 0, 995, 988, 1017, 1027, 0, 0, 0, 0, 974, 1022, 1008, 1020, 25, 0, 0, 0, 1072, 1035, 989, 1031, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4090, 4156, 4113, 4152, 4027, 4049, 4131] avg=4096.0000 max=4156 max_rank=2 min=4027 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1030, 1056, 1018, 1025, 0, 0, 0, 0, 1010, 1003, 1064, 1053, 15, 6, 0, 0, 981, 1029, 1043, 1088, 0, 0, 0, 0, 940, 1008, 1018, 1008, 11, 7, 0, 0, 1037, 1055, 1022, 1023, 0, 0, 0, 0, 971, 1033, 1034, 1073, 0, 0, 0, 0, 1055, 994, 1018, 1015, 25, 0, 0, 0, 998, 993, 997, 1009, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4151, 4141, 3992, 4137, 4111, 4107, 4000] avg=4096.0000 max=4151 max_rank=1 min=3992 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1072, 992, 1007, 1048, 0, 0, 0, 0, 1052, 1028, 998, 994, 16, 6, 0, 0, 1006, 994, 1029, 1036, 0, 0, 0, 0, 1004, 1038, 1015, 1003, 10, 7, 0, 0, 995, 992, 1007, 1024, 0, 0, 0, 0, 1017, 1014, 1044, 956, 0, 0, 0, 0, 1031, 1060, 1047, 1059, 24, 0, 0, 0, 1050, 1000, 1025, 1065, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4094, 4065, 4077, 4018, 4031, 4221, 4143] avg=4096.0000 max=4221 max_rank=6 min=4018 min_rank=4 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1117, 1047, 951, 1045, 990, 1057, 979, 1052, 978, 984, 1025, 1030, 1009, 973, 983, 1033, 1052, 1009, 1033, 1025, 984, 1049, 1015, 1030, 1077, 1002, 1042, 1039, 1011, 1027, 1024, 1096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[983, 959, 1035, 1002, 995, 1077, 1011, 1017, 1004, 1071, 1022, 998, 1023, 1001, 999, 1043, 1028, 1041, 1017, 1023, 1048, 1071, 1041, 1017, 1026, 1019, 971, 1041, 1049, 1017, 1075, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1035, 1024, 1027, 1034, 1042, 1039, 981, 1019, 985, 1014, 1050, 1015, 1037, 1013, 1024, 1047, 1050, 1020, 1033, 1017, 1012, 1055, 1033, 1028, 1033, 989, 1009, 955, 1050, 1037, 1054, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1025, 1061, 1063, 978, 1026, 1002, 1044, 1053, 990, 997, 1017, 1051, 1003, 1002, 1052, 1014, 1057, 971, 994, 1042, 1024, 1036, 1016, 1048, 1021, 1032, 1036, 1049, 1032, 1007, 1006, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1033, 1026, 1043, 1027, 1031, 1019, 1028, 1019, 1062, 1054, 1061, 1027, 1006, 1017, 984, 972, 960, 989, 1030, 1036, 1045, 1022, 1052, 1043, 1031, 1053, 1009, 1030, 989, 1008, 1046, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1008, 1057, 1015, 997, 1030, 1062, 1037, 1042, 1004, 1060, 1038, 1050, 1046, 1020, 1012, 1030, 993, 1048, 996, 986, 1018, 1022, 1028, 941, 1012, 1047, 1024, 1022, 1048, 1027, 1033, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1047, 1089, 1035, 1065, 1034, 1005, 1035, 1040, 975, 988, 993, 1026, 1013, 1002, 1001, 975, 999, 1020, 982, 1053, 1077, 1044, 1004, 1014, 1019, 979, 1045, 1075, 1029, 999, 1023, 1083] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1011, 1029, 1042, 1064, 1078, 971, 1018, 1042, 1061, 1050, 1027, 986, 1062, 1063, 967, 973, 1048, 1029, 1065, 1024, 975, 1027, 1037, 1008, 988, 1018, 1038, 1021, 989, 1027, 997, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1033, 1001, 1043, 1027, 0, 0, 0, 0, 1031, 1019, 1028, 1007, 0, 0, 0, 0, 1062, 1054, 1061, 1027, 9, 0, 0, 0, 1006, 1017, 984, 972, 25, 18, 1, 0, 960, 989, 1030, 1036, 11, 0, 0, 0, 1045, 1012, 1052, 1043, 0, 0, 0, 0, 1031, 1053, 1009, 1030, 1, 0, 0, 0, 989, 1008, 1046, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4085, 4213, 4023, 4026, 4152, 4124, 4041] avg=4096.0000 max=4213 max_rank=2 min=4023 min_rank=3 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[983, 936, 1035, 1002, 0, 0, 0, 0, 995, 1077, 1011, 1005, 0, 0, 0, 0, 1004, 1071, 1022, 998, 9, 0, 0, 0, 1023, 1001, 999, 1043, 23, 18, 1, 0, 1028, 1041, 1017, 1023, 11, 0, 0, 0, 1048, 1060, 1041, 1017, 0, 0, 0, 0, 1026, 1019, 971, 1041, 2, 0, 0, 0, 1049, 1017, 1075, 1026, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3956, 4088, 4104, 4108, 4120, 4166, 4059, 4167] avg=4096.0000 max=4167 max_rank=7 min=3956 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1008, 1031, 1015, 997, 0, 0, 0, 0, 1030, 1062, 1037, 1030, 0, 0, 0, 0, 1004, 1060, 1038, 1050, 9, 0, 0, 0, 1046, 1020, 1012, 1030, 26, 18, 1, 0, 993, 1048, 996, 986, 11, 0, 0, 0, 1018, 1012, 1028, 941, 0, 0, 0, 0, 1012, 1047, 1024, 1022, 1, 0, 0, 0, 1048, 1027, 1033, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4159, 4161, 4153, 4034, 3999, 4106, 4105] avg=4096.0000 max=4161 max_rank=2 min=3999 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16244 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1035, 999, 1027, 1034, 0, 0, 0, 0, 1042, 1039, 981, 1007, 0, 0, 0, 0, 985, 1014, 1050, 1015, 9, 0, 0, 0, 1037, 1013, 1024, 1047, 25, 18, 1, 0, 1050, 1020, 1033, 1017, 11, 0, 0, 0, 1012, 1044, 1033, 1028, 0, 0, 0, 0, 1033, 989, 1009, 955, 2, 0, 0, 0, 1050, 1037, 1054, 989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4069, 4073, 4165, 4131, 4117, 3988, 4130] avg=4096.0000 max=4165 max_rank=3 min=3988 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1047, 1062, 1035, 1065, 0, 0, 0, 0, 1034, 1005, 1035, 1028, 0, 0, 0, 0, 975, 988, 993, 1026, 9, 0, 0, 0, 1013, 1002, 1001, 975, 27, 19, 1, 0, 999, 1020, 982, 1053, 11, 0, 0, 0, 1077, 1033, 1004, 1014, 0, 0, 0, 0, 1019, 979, 1045, 1075, 2, 0, 0, 0, 1029, 999, 1023, 1064, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4209, 4102, 3991, 4038, 4065, 4128, 4120, 4115] avg=4096.0000 max=4209 max_rank=0 min=3991 min_rank=2 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 1032, 1063, 978, 0, 0, 0, 0, 1026, 1002, 1044, 1030, 0, 0, 0, 0, 990, 997, 1017, 1051, 12, 0, 0, 0, 1003, 1002, 1052, 1014, 29, 21, 7, 0, 1057, 971, 994, 1042, 16, 5, 0, 0, 1024, 1014, 1016, 1048, 0, 0, 0, 0, 1021, 1032, 1036, 1049, 5, 0, 0, 0, 1032, 1007, 1006, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4102, 4067, 4128, 4085, 4102, 4143, 4043] avg=4096.0000 max=4143 max_rank=6 min=4043 min_rank=7 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1117, 1021, 951, 1045, 0, 0, 0, 0, 990, 1057, 979, 1040, 0, 0, 0, 0, 978, 984, 1025, 1030, 9, 0, 0, 0, 1009, 973, 983, 1033, 26, 19, 1, 0, 1052, 1009, 1033, 1025, 11, 0, 0, 0, 984, 1038, 1015, 1030, 0, 0, 0, 0, 1077, 1002, 1042, 1039, 2, 0, 0, 0, 1011, 1027, 1024, 1077, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4066, 4026, 4044, 4130, 4067, 4162, 4139] avg=4096.0000 max=4162 max_rank=6 min=4026 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1011, 1004, 1042, 1064, 0, 0, 0, 0, 1078, 971, 1018, 1030, 0, 0, 0, 0, 1061, 1050, 1027, 986, 9, 0, 0, 0, 1062, 1063, 967, 973, 25, 18, 1, 0, 1048, 1029, 1065, 1024, 11, 0, 0, 0, 975, 1017, 1037, 1008, 0, 0, 0, 0, 988, 1018, 1038, 1021, 1, 0, 0, 0, 989, 1027, 997, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4097, 4133, 4109, 4177, 4037, 4066, 4028] avg=4096.0000 max=4177 max_rank=4 min=4028 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8064, 8192, 256, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1059, 1010, 1060, 1012, 1015, 1015, 1000, 1069, 1054, 1032, 1054, 994, 1017, 1042, 984, 999, 1026, 1015, 1024, 1013, 1056, 1027, 985, 957, 1084, 1010, 1006, 1028, 1030, 1023, 1030, 1038] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1042, 1019, 996, 1034, 1053, 1009, 1011, 1024, 1002, 1021, 997, 1019, 1037, 1069, 1027, 1035, 994, 1002, 1079, 975, 1008, 1063, 1046, 985, 1034, 1056, 1008, 1003, 1003, 1057, 1018, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[989, 1043, 985, 1057, 1007, 1009, 1002, 967, 969, 1011, 997, 1052, 1045, 1030, 1061, 1003, 1035, 1008, 1023, 1033, 1029, 1095, 1035, 1029, 1009, 1047, 1088, 1067, 1033, 1002, 994, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1054, 1016, 1077, 1045, 994, 1045, 1015, 1030, 1004, 1034, 1019, 1022, 987, 1037, 1036, 1032, 990, 1035, 980, 1029, 1048, 995, 979, 1042, 1028, 1025, 1005, 1062, 1025, 1044, 1002, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1065, 1014, 1026, 997, 1013, 1007, 976, 1049, 995, 984, 1039, 1042, 1087, 1070, 993, 1054, 1037, 1062, 1033, 1022, 982, 1000, 1007, 1020, 982, 1036, 1020, 1051, 1048, 1026, 1008, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1010, 1006, 1009, 1035, 1045, 1016, 1039, 1003, 1047, 1043, 1003, 1061, 983, 1025, 1021, 1022, 976, 1048, 1051, 1011, 1015, 999, 1048, 983, 1071, 978, 1039, 1062, 1073, 1025, 1020, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1048, 1055, 1043, 1014, 1031, 1032, 962, 1054, 1015, 1063, 994, 1012, 1040, 1065, 999, 1004, 1005, 1006, 1016, 994, 980, 988, 1021, 1007, 1005, 1051, 961, 1041, 1136, 1029, 1083, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1031, 1062, 1007, 990, 984, 1023, 1010, 1053, 1008, 986, 1008, 999, 992, 1009, 1062, 1016, 1058, 1029, 1034, 987, 997, 1062, 1057, 1053, 1028, 1044, 999, 975, 1096, 1008, 1057, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1042, 1010, 1060, 1012, 0, 0, 0, 0, 1015, 1015, 1000, 1069, 25, 0, 0, 0, 1054, 1032, 1054, 994, 12, 10, 0, 0, 1017, 1028, 984, 999, 0, 0, 0, 0, 1026, 1015, 1024, 1013, 14, 2, 0, 0, 1056, 1027, 985, 957, 13, 7, 0, 0, 1084, 1010, 1006, 1013, 0, 0, 0, 0, 993, 1023, 1030, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4124, 4156, 4028, 4094, 4045, 4113, 4084] avg=4096.0000 max=4156 max_rank=2 min=4028 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1025, 1019, 996, 1034, 0, 0, 0, 0, 1053, 1009, 1011, 1024, 24, 0, 0, 0, 1002, 1021, 997, 1019, 12, 10, 0, 0, 1037, 1055, 1027, 1035, 0, 0, 0, 0, 994, 1002, 1079, 975, 14, 2, 0, 0, 1008, 1063, 1046, 985, 13, 7, 0, 0, 1034, 1056, 1008, 988, 0, 0, 0, 0, 967, 1057, 1018, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4121, 4061, 4154, 4066, 4122, 4086, 4084] avg=4096.0000 max=4154 max_rank=3 min=4061 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1037, 1016, 1077, 1045, 0, 0, 0, 0, 994, 1045, 1015, 1030, 25, 0, 0, 0, 1004, 1034, 1019, 1022, 12, 10, 0, 0, 987, 1023, 1036, 1032, 0, 0, 0, 0, 990, 1035, 980, 1029, 14, 2, 0, 0, 1048, 995, 979, 1042, 14, 7, 0, 0, 1028, 1025, 1005, 1046, 0, 0, 0, 0, 988, 1044, 1002, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4109, 4101, 4078, 4050, 4085, 4104, 4066] avg=4096.0000 max=4175 max_rank=0 min=4050 min_rank=4 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[993, 1006, 1009, 1035, 0, 0, 0, 0, 1045, 1016, 1039, 1003, 26, 0, 0, 0, 1047, 1043, 1003, 1061, 13, 10, 0, 0, 983, 1011, 1021, 1022, 0, 0, 0, 0, 976, 1048, 1051, 1011, 14, 2, 0, 0, 1015, 999, 1048, 983, 14, 7, 0, 0, 1071, 978, 1039, 1046, 0, 0, 0, 0, 1034, 1025, 1020, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4129, 4177, 4037, 4102, 4066, 4134, 4080] avg=4096.0000 max=4177 max_rank=2 min=4037 min_rank=3 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[966, 1043, 985, 1057, 0, 0, 0, 0, 1007, 1009, 1002, 967, 28, 0, 0, 0, 969, 1011, 997, 1052, 17, 14, 0, 0, 1045, 1012, 1061, 1003, 0, 0, 0, 0, 1035, 1008, 1023, 1033, 18, 9, 0, 0, 1029, 1095, 1035, 1029, 17, 9, 0, 0, 1009, 1047, 1088, 1041, 0, 0, 0, 0, 988, 1002, 994, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4013, 4060, 4121, 4126, 4214, 4185, 3998] avg=4096.0000 max=4214 max_rank=5 min=3998 min_rank=7 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16523 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1031, 1055, 1043, 1014, 0, 0, 0, 0, 1031, 1032, 962, 1054, 27, 0, 0, 0, 1015, 1063, 994, 1012, 13, 10, 0, 0, 1040, 1051, 999, 1004, 0, 0, 0, 0, 1005, 1006, 1016, 994, 14, 2, 0, 0, 980, 988, 1021, 1007, 14, 7, 0, 0, 1005, 1051, 961, 1025, 0, 0, 0, 0, 1096, 1029, 1083, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4106, 4107, 4094, 4037, 4017, 4042, 4222] avg=4096.0000 max=4222 max_rank=7 min=4017 min_rank=5 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1048, 1014, 1026, 997, 0, 0, 0, 0, 1013, 1007, 976, 1049, 25, 0, 0, 0, 995, 984, 1039, 1042, 12, 10, 0, 0, 1087, 1056, 993, 1054, 0, 0, 0, 0, 1037, 1062, 1033, 1022, 14, 2, 0, 0, 982, 1000, 1007, 1020, 14, 7, 0, 0, 982, 1036, 1020, 1035, 0, 0, 0, 0, 1011, 1026, 1008, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4070, 4082, 4190, 4170, 4030, 4073, 4068] avg=4096.0000 max=4190 max_rank=3 min=4030 min_rank=5 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1014, 1062, 1007, 990, 0, 0, 0, 0, 984, 1023, 1010, 1053, 26, 0, 0, 0, 1008, 986, 1008, 999, 13, 10, 0, 0, 992, 996, 1062, 1016, 0, 0, 0, 0, 1058, 1029, 1034, 987, 13, 2, 0, 0, 997, 1062, 1057, 1053, 13, 7, 0, 0, 1028, 1044, 999, 960, 0, 0, 0, 0, 1057, 1008, 1057, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4096, 4024, 4066, 4123, 4189, 4031, 4166] avg=4096.0000 max=4189 max_rank=5 min=4024 min_rank=2 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8064, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1073, 984, 1048, 1056, 1050, 1028, 1028, 989, 1017, 993, 1053, 1066, 1040, 980, 979, 1021, 1007, 1051, 1032, 981, 1054, 1042, 1015, 1022, 1021, 1041, 1002, 1058, 967, 979, 1058, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[937, 1059, 1002, 1058, 1047, 1075, 1016, 1045, 1036, 1019, 1049, 959, 998, 992, 965, 1042, 1072, 1055, 1065, 1001, 992, 1038, 977, 996, 1038, 1052, 1022, 1081, 996, 990, 1055, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1025, 1024, 1069, 1010, 1051, 1031, 1017, 999, 1047, 1025, 1007, 1004, 1029, 1052, 1010, 1027, 1032, 1018, 1031, 1018, 961, 1027, 1085, 993, 1018, 1008, 1060, 1047, 1011, 1024, 1009, 999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1047, 974, 1015, 977, 1008, 999, 1036, 1020, 1030, 1004, 1041, 992, 1050, 1063, 1047, 1013, 1066, 992, 985, 1025, 1066, 1037, 1041, 1036, 1059, 1002, 958, 1028, 1015, 1045, 1018, 1079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1042, 991, 1052, 1046, 1068, 1029, 1057, 957, 1030, 1039, 1004, 986, 1016, 1049, 980, 1016, 1024, 1019, 1042, 986, 1036, 1032, 1022, 1032, 1051, 1040, 1031, 1022, 994, 1003, 1083, 989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1026, 1041, 1018, 987, 1015, 1011, 993, 999, 1077, 1052, 1008, 1043, 1039, 998, 1025, 1028, 1057, 1007, 1042, 1038, 999, 1016, 1021, 1059, 1028, 1002, 1005, 1002, 1035, 993, 1090, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1008, 1043, 1060, 1006, 1040, 1029, 1035, 978, 1025, 1035, 1037, 1016, 965, 1012, 1022, 1017, 1018, 1018, 1041, 1050, 1052, 962, 1050, 1052, 1006, 1037, 979, 1043, 1034, 1018, 1000, 1080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[998, 1053, 1042, 1030, 986, 1023, 1070, 1017, 1034, 1025, 1077, 1029, 1061, 1021, 985, 1030, 1012, 955, 1008, 1017, 1001, 1024, 1039, 1012, 1025, 1015, 1030, 1031, 1022, 1060, 1008, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1073, 984, 1045, 1056, 0, 0, 0, 0, 1050, 1028, 1028, 989, 2, 0, 0, 0, 1006, 993, 1053, 1066, 0, 0, 0, 0, 1040, 980, 979, 1021, 11, 9, 3, 0, 1007, 1051, 1032, 981, 0, 0, 0, 0, 1054, 1042, 1013, 1022, 0, 0, 0, 0, 1021, 1041, 1002, 1049, 0, 0, 0, 0, 967, 979, 1058, 1033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4097, 4118, 4043, 4071, 4131, 4113, 4037] avg=4096.0000 max=4158 max_rank=0 min=4037 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16352 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1047, 974, 1012, 977, 0, 0, 0, 0, 1008, 999, 1036, 1020, 2, 0, 0, 0, 1019, 1004, 1041, 992, 0, 0, 0, 0, 1050, 1063, 1047, 1013, 11, 9, 3, 0, 1066, 992, 985, 1025, 0, 0, 0, 0, 1066, 1037, 1039, 1036, 0, 0, 0, 0, 1059, 1002, 958, 1019, 0, 0, 0, 0, 1015, 1045, 1018, 1079, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4065, 4056, 4196, 4068, 4178, 4038, 4157] avg=4096.0000 max=4196 max_rank=3 min=4010 min_rank=0 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16441 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1025, 1024, 1066, 1010, 0, 0, 0, 0, 1051, 1031, 1017, 999, 2, 0, 0, 0, 1036, 1025, 1007, 1004, 0, 0, 0, 0, 1029, 1052, 1010, 1027, 11, 9, 3, 0, 1032, 1018, 1031, 1018, 0, 0, 0, 0, 961, 1027, 1083, 993, 0, 0, 0, 0, 1018, 1008, 1060, 1038, 0, 0, 0, 0, 1011, 1024, 1009, 999, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4100, 4072, 4141, 4099, 4064, 4124, 4043] avg=4096.0000 max=4141 max_rank=3 min=4043 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16330 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[937, 1059, 999, 1058, 0, 0, 0, 0, 1047, 1075, 1016, 1045, 2, 0, 0, 0, 1025, 1019, 1049, 959, 0, 0, 0, 0, 998, 992, 965, 1042, 11, 9, 3, 0, 1072, 1055, 1065, 1001, 0, 0, 0, 0, 992, 1038, 975, 996, 0, 0, 0, 0, 1038, 1052, 1022, 1072, 0, 0, 0, 0, 996, 990, 1055, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4185, 4052, 4020, 4193, 4001, 4184, 4080] avg=4096.0000 max=4193 max_rank=4 min=4001 min_rank=5 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16458 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1042, 991, 1040, 1046, 0, 0, 0, 0, 1068, 1029, 1057, 957, 6, 2, 0, 0, 1016, 1039, 1004, 986, 0, 0, 0, 0, 1016, 1049, 980, 1016, 14, 12, 10, 0, 1024, 1019, 1042, 986, 3, 0, 0, 0, 1036, 1032, 1013, 1032, 0, 0, 0, 0, 1051, 1040, 1031, 1010, 0, 0, 0, 0, 994, 1003, 1083, 989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4119, 4045, 4097, 4074, 4113, 4132, 4069] avg=4096.0000 max=4132 max_rank=6 min=4045 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1008, 1043, 1057, 1006, 0, 0, 0, 0, 1040, 1029, 1035, 978, 2, 0, 0, 0, 1014, 1035, 1037, 1016, 0, 0, 0, 0, 965, 1012, 1022, 1017, 11, 9, 3, 0, 1018, 1018, 1041, 1050, 0, 0, 0, 0, 1052, 962, 1048, 1052, 0, 0, 0, 0, 1006, 1037, 979, 1034, 0, 0, 0, 0, 1034, 1018, 1000, 1080, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4084, 4102, 4039, 4127, 4114, 4056, 4132] avg=4096.0000 max=4132 max_rank=7 min=4039 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16429 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1026, 1041, 1015, 987, 0, 0, 0, 0, 1015, 1011, 993, 999, 2, 0, 0, 0, 1066, 1052, 1008, 1043, 0, 0, 0, 0, 1039, 998, 1025, 1028, 11, 8, 3, 0, 1057, 1007, 1042, 1038, 0, 0, 0, 0, 999, 1016, 1019, 1059, 0, 0, 0, 0, 1028, 1002, 1005, 994, 0, 0, 0, 0, 1035, 993, 1090, 1014, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4020, 4169, 4112, 4144, 4093, 4029, 4132] avg=4096.0000 max=4169 max_rank=2 min=4020 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16398 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[998, 1053, 1039, 1030, 0, 0, 0, 0, 986, 1023, 1070, 1017, 2, 0, 0, 0, 1023, 1025, 1077, 1029, 0, 0, 0, 0, 1061, 1021, 985, 1030, 11, 9, 3, 0, 1012, 955, 1008, 1017, 0, 0, 0, 0, 1001, 1024, 1037, 1012, 0, 0, 0, 0, 1025, 1015, 1030, 1022, 0, 0, 0, 0, 1022, 1060, 1008, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4098, 4154, 4120, 3992, 4074, 4092, 4118] avg=4096.0000 max=4154 max_rank=2 min=3992 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16276 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8448, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1040, 1005, 972, 1048, 1029, 1042, 1036, 1031, 1033, 1041, 1012, 1033, 993, 1034, 985, 1040, 990, 1042, 1015, 1073, 1035, 994, 1034, 1004, 997, 1006, 1019, 1039, 1021, 1086, 1036, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1011, 1049, 1015, 1013, 1031, 1003, 1043, 1055, 1037, 1002, 1070, 996, 985, 1035, 1032, 1027, 1037, 1055, 1016, 1004, 947, 1060, 990, 1015, 1019, 1043, 1024, 1042, 1075, 1059, 981, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1055, 999, 1001, 1038, 1048, 1084, 1019, 1004, 1026, 1017, 1043, 987, 982, 1030, 1036, 989, 1001, 1040, 1033, 1061, 1009, 1039, 1016, 992, 1017, 986, 1064, 997, 1023, 1061, 1001, 1070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1029, 1039, 1072, 1023, 1008, 1018, 998, 1015, 978, 1037, 1053, 1006, 997, 1049, 999, 998, 1084, 1049, 1037, 994, 1048, 1033, 1039, 1034, 1048, 996, 1014, 1016, 1010, 972, 1030, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1029, 966, 1047, 1018, 1020, 1017, 981, 1056, 1022, 1040, 997, 1023, 1039, 1038, 1052, 1047, 1049, 1005, 996, 1017, 1035, 1044, 1096, 1026, 990, 1061, 1013, 1022, 1012, 1005, 1001, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1039, 1031, 1026, 1005, 1029, 966, 1031, 1021, 1060, 984, 1032, 1025, 1063, 1034, 992, 1044, 1033, 1055, 1058, 1047, 1042, 1026, 1025, 1034, 1053, 968, 984, 1082, 992, 1027, 1008, 952] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1015, 1068, 989, 1037, 970, 965, 992, 1046, 1048, 1012, 1021, 1041, 1017, 1095, 1016, 1042, 998, 1000, 1038, 1065, 1096, 964, 998, 979, 1020, 1034, 990, 1048, 1065, 1061, 1044, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1051, 1016, 994, 1016, 1045, 1073, 996, 1067, 969, 1009, 1054, 1009, 1043, 1008, 1009, 1033, 987, 1041, 953, 1073, 1011, 1037, 1001, 1024, 1018, 1044, 996, 1013, 1018, 1078, 1063, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1040, 1005, 972, 1048, 1, 0, 0, 0, 1029, 1042, 1036, 1031, 3, 0, 0, 0, 1033, 1041, 1012, 1033, 6, 0, 0, 0, 993, 1033, 985, 1040, 0, 0, 0, 0, 990, 1042, 1015, 1052, 0, 0, 0, 0, 1035, 994, 1034, 1004, 2, 2, 0, 0, 997, 1006, 1019, 1039, 13, 0, 0, 0, 1021, 1081, 1036, 1003, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4141, 4125, 4051, 4099, 4071, 4074, 4141] avg=4096.0000 max=4141 max_rank=1 min=4051 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1029, 1039, 1072, 1023, 1, 0, 0, 0, 1008, 1018, 998, 1015, 3, 0, 0, 0, 978, 1037, 1053, 1006, 6, 0, 0, 0, 997, 1048, 999, 998, 0, 0, 0, 0, 1084, 1049, 1037, 974, 0, 0, 0, 0, 1048, 1033, 1039, 1034, 2, 2, 0, 0, 1048, 996, 1014, 1016, 12, 0, 0, 0, 1010, 967, 1030, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4042, 4080, 4042, 4144, 4158, 4086, 4052] avg=4096.0000 max=4164 max_rank=0 min=4042 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1055, 999, 1001, 1038, 1, 0, 0, 0, 1048, 1084, 1019, 1004, 3, 0, 0, 0, 1026, 1017, 1043, 987, 6, 0, 0, 0, 982, 1029, 1036, 989, 0, 0, 0, 0, 1001, 1040, 1033, 1040, 0, 0, 0, 0, 1009, 1039, 1016, 992, 2, 2, 0, 0, 1017, 986, 1064, 997, 13, 0, 0, 0, 1023, 1056, 1001, 1070, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4158, 4079, 4036, 4114, 4060, 4077, 4150] avg=4096.0000 max=4158 max_rank=1 min=4036 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1011, 1049, 1015, 1013, 1, 0, 0, 0, 1031, 1003, 1043, 1055, 3, 0, 0, 0, 1037, 1002, 1070, 996, 6, 0, 0, 0, 985, 1034, 1032, 1027, 0, 0, 0, 0, 1037, 1055, 1016, 984, 0, 0, 0, 0, 947, 1060, 990, 1015, 2, 2, 0, 0, 1019, 1043, 1024, 1042, 12, 0, 0, 0, 1075, 1054, 981, 997, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4135, 4111, 4078, 4092, 4016, 4140, 4107] avg=4096.0000 max=4140 max_rank=6 min=4016 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1029, 966, 1047, 1018, 1, 0, 0, 0, 1020, 1017, 981, 1056, 3, 0, 0, 0, 1022, 1040, 997, 1023, 6, 0, 0, 0, 1039, 1037, 1052, 1047, 0, 0, 0, 0, 1049, 1005, 996, 997, 0, 0, 0, 0, 1035, 1044, 1096, 1026, 2, 2, 0, 0, 990, 1061, 1013, 1022, 12, 0, 0, 0, 1012, 1000, 1001, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4077, 4088, 4175, 4047, 4205, 4098, 4017] avg=4096.0000 max=4205 max_rank=5 min=4017 min_rank=7 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1039, 1031, 1026, 1005, 5, 0, 0, 0, 1029, 966, 1031, 1021, 5, 3, 0, 0, 1060, 984, 1032, 1025, 9, 0, 0, 0, 1063, 1026, 992, 1044, 0, 0, 0, 0, 1033, 1055, 1058, 1013, 0, 0, 0, 0, 1042, 1026, 1025, 1034, 8, 5, 0, 0, 1053, 968, 984, 1082, 17, 0, 0, 0, 992, 1017, 1008, 952, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4055, 4110, 4125, 4159, 4140, 4104, 3969] avg=4096.0000 max=4159 max_rank=4 min=3969 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1051, 1016, 994, 1016, 1, 0, 0, 0, 1045, 1073, 996, 1067, 3, 0, 0, 0, 969, 1009, 1054, 1009, 6, 0, 0, 0, 1043, 1007, 1009, 1033, 0, 0, 0, 0, 987, 1041, 953, 1052, 0, 0, 0, 0, 1011, 1037, 1001, 1024, 2, 2, 0, 0, 1018, 1044, 996, 1013, 13, 0, 0, 0, 1018, 1073, 1063, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4184, 4047, 4092, 4033, 4077, 4084, 4173] avg=4096.0000 max=4184 max_rank=1 min=4033 min_rank=4 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1015, 1068, 989, 1037, 1, 0, 0, 0, 970, 965, 992, 1046, 3, 0, 0, 0, 1048, 1012, 1021, 1041, 6, 0, 0, 0, 1017, 1094, 1016, 1042, 0, 0, 0, 0, 998, 1000, 1038, 1044, 0, 0, 0, 0, 1096, 964, 998, 979, 2, 2, 0, 0, 1020, 1034, 990, 1048, 13, 0, 0, 0, 1065, 1056, 1044, 994, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 3976, 4128, 4169, 4080, 4041, 4105, 4159] avg=4096.0000 max=4169 max_rank=3 min=3976 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1010, 1051, 1031, 1023, 1062, 999, 1049, 1069, 1029, 1007, 993, 1002, 1042, 983, 1012, 996, 1017, 1041, 1067, 1045, 1023, 1013, 1065, 996, 1016, 1017, 1005, 1002, 1001, 1003, 1082, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1041, 1022, 1065, 1056, 1032, 1038, 1082, 1026, 1027, 1028, 1038, 1048, 962, 993, 1050, 1025, 1012, 1014, 1034, 1023, 1013, 1039, 967, 999, 993, 1052, 1006, 964, 1034, 1009, 1031, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[993, 991, 1025, 992, 1058, 1057, 1017, 989, 1032, 984, 1030, 1029, 1016, 1043, 1027, 1049, 1050, 1051, 1037, 1046, 977, 1053, 1011, 1001, 1034, 1011, 1021, 1099, 1008, 1055, 974, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1040, 1032, 985, 989, 991, 998, 1019, 1028, 1017, 1084, 1063, 1035, 1012, 1019, 1046, 1053, 1009, 952, 1040, 995, 1017, 1016, 1033, 1023, 1028, 1025, 1055, 1058, 1000, 1036, 1013, 1057] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1042, 999, 1027, 1000, 1059, 987, 1034, 1088, 1005, 1031, 989, 1036, 1038, 989, 1021, 1046, 1032, 1023, 1023, 1010, 1018, 1033, 1058, 982, 990, 1027, 1045, 995, 1055, 996, 1074, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1050, 1066, 1064, 1009, 1064, 1052, 991, 967, 998, 1032, 1055, 1008, 1020, 988, 1082, 1059, 979, 1024, 1044, 1035, 1026, 1012, 1032, 991, 1048, 972, 1009, 1031, 995, 1040, 979, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1033, 995, 986, 1029, 1043, 987, 1010, 1009, 991, 996, 1003, 1087, 1022, 1008, 1047, 977, 1002, 1049, 1041, 999, 1027, 1060, 1035, 1039, 1023, 997, 1066, 1087, 1040, 1015, 1048, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1019, 965, 1001, 1044, 1009, 1012, 986, 1014, 1070, 1026, 1050, 1020, 1015, 1075, 1067, 1030, 1027, 1026, 1090, 997, 1037, 976, 961, 999, 1003, 1013, 1058, 1006, 1009, 1061, 1075, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1010, 1051, 1031, 1023, 7, 3, 0, 0, 1055, 999, 1049, 1069, 0, 0, 0, 0, 1029, 1007, 993, 993, 0, 0, 0, 0, 1042, 983, 1008, 996, 0, 0, 0, 0, 1017, 1041, 1059, 1045, 0, 0, 0, 0, 1023, 1013, 1065, 996, 12, 9, 8, 0, 1016, 1017, 1005, 1002, 1, 0, 0, 0, 1001, 1003, 1070, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4172, 4022, 4029, 4162, 4126, 4041, 4091] avg=4096.0000 max=4172 max_rank=1 min=4022 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1041, 1022, 1065, 1056, 7, 4, 0, 0, 1025, 1038, 1082, 1026, 0, 0, 0, 0, 1027, 1028, 1038, 1039, 0, 0, 0, 0, 962, 993, 1045, 1025, 0, 0, 0, 0, 1012, 1014, 1027, 1023, 0, 0, 0, 0, 1013, 1039, 967, 999, 12, 9, 7, 0, 993, 1052, 1006, 964, 1, 0, 0, 0, 1034, 1009, 1019, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4195, 4171, 4132, 4025, 4076, 4046, 4016, 4107] avg=4096.0000 max=4195 max_rank=0 min=4016 min_rank=6 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16245 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1040, 1032, 985, 989, 6, 4, 0, 0, 985, 998, 1019, 1028, 0, 0, 0, 0, 1017, 1084, 1063, 1026, 0, 0, 0, 0, 1012, 1019, 1041, 1053, 0, 0, 0, 0, 1009, 952, 1033, 995, 0, 0, 0, 0, 1017, 1016, 1033, 1023, 11, 9, 7, 0, 1028, 1025, 1055, 1058, 1, 0, 0, 0, 1000, 1036, 1002, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4030, 4190, 4125, 3989, 4116, 4167, 4095] avg=4096.0000 max=4190 max_rank=2 min=3989 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1050, 1066, 1064, 1009, 7, 4, 0, 0, 1057, 1052, 991, 967, 0, 0, 0, 0, 998, 1032, 1055, 999, 0, 0, 0, 0, 1020, 988, 1077, 1059, 0, 0, 0, 0, 979, 1024, 1037, 1035, 0, 0, 0, 0, 1026, 1012, 1032, 991, 11, 9, 7, 0, 1048, 972, 1009, 1031, 1, 0, 0, 0, 995, 1040, 968, 1046, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4200, 4067, 4084, 4144, 4075, 4088, 4061, 4049] avg=4096.0000 max=4200 max_rank=0 min=4049 min_rank=7 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1042, 999, 1027, 1000, 7, 3, 0, 0, 1052, 987, 1034, 1088, 0, 0, 0, 0, 1005, 1031, 989, 1027, 0, 0, 0, 0, 1038, 989, 1017, 1046, 0, 0, 0, 0, 1032, 1023, 1016, 1010, 0, 0, 0, 0, 1018, 1033, 1058, 982, 12, 9, 7, 0, 990, 1027, 1045, 995, 1, 0, 0, 0, 1055, 996, 1062, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4161, 4052, 4090, 4081, 4119, 4058, 4129] avg=4096.0000 max=4161 max_rank=1 min=4052 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[993, 991, 1025, 992, 10, 6, 3, 0, 1048, 1057, 1017, 989, 0, 0, 0, 0, 1032, 984, 1030, 1017, 0, 0, 0, 0, 1016, 1043, 1016, 1049, 0, 0, 0, 0, 1050, 1051, 1022, 1046, 0, 0, 0, 0, 977, 1053, 1011, 1001, 16, 12, 12, 0, 1034, 1011, 1021, 1099, 5, 0, 0, 0, 1008, 1055, 958, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4111, 4063, 4124, 4169, 4082, 4170, 4029] avg=4096.0000 max=4170 max_rank=6 min=4020 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1033, 995, 986, 1029, 7, 4, 0, 0, 1036, 987, 1010, 1009, 0, 0, 0, 0, 991, 996, 1003, 1078, 0, 0, 0, 0, 1022, 1008, 1042, 977, 0, 0, 0, 0, 1002, 1049, 1034, 999, 0, 0, 0, 0, 1027, 1060, 1035, 1039, 12, 9, 7, 0, 1023, 997, 1066, 1087, 1, 0, 0, 0, 1040, 1015, 1036, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4042, 4068, 4049, 4084, 4189, 4174, 4108] avg=4096.0000 max=4189 max_rank=5 min=4042 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16555 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1019, 965, 1001, 1044, 7, 4, 0, 0, 1002, 1012, 986, 1014, 0, 0, 0, 0, 1070, 1026, 1050, 1011, 0, 0, 0, 0, 1015, 1075, 1062, 1030, 0, 0, 0, 0, 1027, 1026, 1082, 997, 0, 0, 0, 0, 1037, 976, 961, 999, 12, 9, 8, 0, 1003, 1013, 1058, 1006, 1, 0, 0, 0, 1009, 1061, 1063, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4014, 4157, 4182, 4132, 4002, 4081, 4160] avg=4096.0000 max=4182 max_rank=3 min=4002 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[949, 1023, 965, 995, 1059, 1026, 1002, 1016, 1075, 1033, 1026, 972, 1065, 1054, 1003, 1025, 1010, 1096, 1036, 1049, 1001, 985, 1008, 1015, 1032, 1040, 1046, 1028, 1025, 1034, 1055, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1025, 1040, 1009, 1025, 1020, 1025, 998, 1028, 1016, 1041, 1013, 1027, 1041, 1004, 1048, 1049, 1012, 990, 1021, 1054, 1007, 998, 1010, 1085, 1047, 1031, 1021, 1009, 1031, 993, 1008, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1031, 1018, 1044, 996, 981, 999, 1046, 1040, 1053, 1056, 1064, 1005, 1020, 1036, 1037, 1042, 986, 1040, 1028, 1031, 988, 1023, 938, 981, 1039, 1038, 1038, 978, 1029, 1102, 1013, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1009, 989, 1012, 1035, 1042, 1028, 1013, 1031, 1052, 1064, 1047, 1045, 1009, 1046, 1016, 1018, 1009, 964, 1037, 1025, 1037, 1002, 1005, 1021, 1041, 1019, 974, 1033, 1070, 1034, 1031, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1053, 1044, 1014, 1061, 1013, 1000, 1050, 1026, 1001, 1047, 1060, 1038, 1013, 990, 1004, 998, 1030, 1033, 1016, 1041, 1061, 1002, 995, 1032, 985, 997, 995, 1052, 1018, 1035, 1011, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1037, 1002, 1033, 984, 1017, 989, 1002, 1000, 1034, 1015, 1035, 1065, 999, 1031, 999, 1035, 1025, 974, 1107, 1069, 1006, 989, 1013, 1094, 1043, 1041, 1000, 1047, 1032, 1002, 1022, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1015, 1009, 1008, 1002, 1018, 1063, 1070, 1047, 976, 975, 1054, 1089, 1058, 1028, 1036, 1034, 1039, 981, 1029, 999, 1007, 1019, 991, 1056, 1027, 1002, 999, 1059, 970, 1046, 1035, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1018, 1010, 982, 1019, 1063, 1004, 1022, 1065, 1021, 1018, 960, 1041, 1069, 1013, 1025, 1029, 1079, 1034, 1053, 1002, 1044, 1038, 1060, 1021, 987, 1009, 1014, 1018, 1040, 1032, 1009, 969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[949, 1023, 965, 995, 29, 7, 0, 0, 1059, 1026, 1002, 1012, 0, 0, 0, 0, 1075, 1033, 1026, 943, 0, 0, 0, 0, 1052, 1054, 1003, 1025, 0, 0, 0, 0, 1010, 1096, 1021, 1049, 0, 0, 0, 0, 1001, 985, 1008, 1015, 13, 8, 7, 0, 1032, 1040, 1046, 1028, 5, 4, 0, 0, 1025, 1022, 1055, 1020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3968, 4099, 4077, 4134, 4176, 4037, 4155, 4122] avg=4096.0000 max=4176 max_rank=4 min=3968 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1025, 1040, 1009, 1025, 31, 7, 0, 0, 1020, 1025, 998, 1024, 0, 0, 0, 0, 1016, 1041, 1013, 996, 0, 0, 0, 0, 1028, 1004, 1048, 1049, 0, 0, 0, 0, 1012, 990, 1006, 1054, 0, 0, 0, 0, 1007, 998, 1010, 1085, 13, 8, 7, 0, 1047, 1031, 1021, 1009, 5, 4, 0, 0, 1031, 981, 1008, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4067, 4066, 4129, 4062, 4128, 4117, 4062] avg=4096.0000 max=4137 max_rank=0 min=4062 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1009, 989, 1012, 1035, 31, 7, 0, 0, 1042, 1028, 1013, 1027, 0, 0, 0, 0, 1052, 1064, 1047, 1014, 0, 0, 0, 0, 997, 1046, 1016, 1018, 0, 0, 0, 0, 1009, 964, 1022, 1025, 0, 0, 0, 0, 1037, 1002, 1005, 1021, 12, 8, 7, 0, 1041, 1019, 974, 1033, 5, 4, 0, 0, 1070, 1022, 1031, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4110, 4177, 4077, 4020, 4092, 4076, 4133] avg=4096.0000 max=4177 max_rank=2 min=4020 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16321 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1031, 1018, 1044, 996, 30, 7, 0, 0, 981, 999, 1046, 1036, 0, 0, 0, 0, 1053, 1056, 1064, 975, 0, 0, 0, 0, 1007, 1036, 1037, 1042, 0, 0, 0, 0, 986, 1040, 1013, 1031, 0, 0, 0, 0, 988, 1023, 938, 981, 13, 8, 8, 0, 1039, 1038, 1038, 978, 5, 4, 0, 0, 1029, 1089, 1013, 1048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4062, 4148, 4122, 4070, 3959, 4102, 4179] avg=4096.0000 max=4179 max_rank=7 min=3959 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1015, 1009, 1008, 1002, 32, 7, 0, 0, 1018, 1063, 1070, 1043, 0, 0, 0, 0, 976, 975, 1054, 1057, 0, 0, 0, 0, 1045, 1028, 1036, 1034, 0, 0, 0, 0, 1039, 981, 1014, 999, 0, 0, 0, 0, 1007, 1019, 991, 1056, 13, 8, 7, 0, 1027, 1002, 999, 1059, 5, 4, 0, 0, 970, 1034, 1035, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4194, 4062, 4143, 4033, 4101, 4096, 4066] avg=4096.0000 max=4194 max_rank=1 min=4033 min_rank=4 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1053, 1044, 1014, 1061, 34, 12, 0, 0, 1013, 1000, 1050, 1019, 0, 0, 0, 0, 1001, 1047, 1060, 1004, 0, 0, 0, 0, 996, 990, 1004, 998, 0, 0, 0, 0, 1030, 1033, 992, 1041, 0, 0, 0, 0, 1061, 1002, 995, 1032, 17, 12, 11, 0, 985, 997, 995, 1052, 9, 7, 0, 0, 1018, 1015, 1011, 1053, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4218, 4082, 4112, 3988, 4096, 4130, 4045, 4097] avg=4096.0000 max=4218 max_rank=0 min=3988 min_rank=3 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1037, 1002, 1033, 984, 32, 8, 0, 0, 1017, 989, 1002, 996, 0, 0, 0, 0, 1034, 1015, 1035, 1033, 0, 0, 0, 0, 987, 1031, 999, 1035, 0, 0, 0, 0, 1025, 974, 1090, 1069, 0, 0, 0, 0, 1006, 989, 1013, 1094, 12, 9, 7, 0, 1043, 1041, 1000, 1047, 5, 4, 0, 0, 1032, 990, 1022, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4004, 4117, 4052, 4158, 4130, 4140, 4071] avg=4096.0000 max=4158 max_rank=4 min=4004 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1018, 1010, 982, 1019, 31, 7, 0, 0, 1063, 1004, 1022, 1061, 0, 0, 0, 0, 1021, 1018, 960, 1010, 0, 0, 0, 0, 1056, 1013, 1025, 1029, 0, 0, 0, 0, 1079, 1034, 1038, 1002, 0, 0, 0, 0, 1044, 1038, 1060, 1021, 13, 8, 7, 0, 987, 1009, 1014, 1018, 5, 4, 0, 0, 1040, 1020, 1009, 969, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4150, 4009, 4123, 4153, 4191, 4037, 4038] avg=4096.0000 max=4191 max_rank=5 min=4009 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8064, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1082, 990, 1038, 962, 1058, 970, 1028, 1038, 1041, 1052, 1001, 1044, 1078, 1017, 1030, 978, 987, 1035, 1003, 1035, 1076, 1050, 1021, 1004, 957, 991, 1013, 1074, 1017, 984, 1046, 1068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[990, 1041, 1032, 991, 1057, 1019, 1038, 1058, 998, 1007, 993, 1054, 996, 1068, 1007, 1012, 1037, 1070, 1046, 990, 1061, 1005, 1072, 984, 1003, 1030, 1030, 1018, 968, 1074, 972, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[989, 1015, 1090, 1028, 1018, 995, 998, 1007, 1042, 1045, 1052, 1039, 1007, 982, 1021, 1079, 1052, 1034, 1043, 1017, 1018, 984, 1019, 1036, 990, 1020, 1037, 1055, 1044, 994, 1000, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1007, 992, 1080, 980, 1029, 1040, 1004, 1053, 1056, 1019, 1031, 1025, 1053, 1048, 1058, 981, 991, 1012, 1055, 1030, 1033, 997, 1071, 1032, 1048, 1046, 996, 986, 993, 970, 1028, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1067, 1028, 1018, 987, 1036, 1044, 1035, 1050, 1015, 1053, 1009, 1020, 964, 1022, 1049, 1029, 1010, 999, 1009, 1041, 1073, 1019, 1002, 1040, 1009, 975, 1039, 1068, 1007, 1022, 1009, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1005, 1032, 1042, 990, 1037, 1002, 1106, 1020, 1064, 1050, 984, 1031, 1012, 1018, 1069, 1045, 975, 1001, 1009, 996, 1040, 1012, 1011, 1023, 993, 1021, 1054, 1023, 1088, 1006, 1012, 997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1012, 1048, 1008, 1004, 1047, 997, 991, 1007, 1083, 1061, 1057, 1016, 1021, 1050, 962, 1054, 1010, 1072, 990, 1050, 1051, 1010, 965, 1057, 1011, 1053, 1006, 1036, 991, 1041, 1044, 963] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1019, 1033, 990, 975, 1023, 1048, 1044, 1072, 1039, 967, 1018, 1062, 998, 1038, 1046, 1015, 1020, 1033, 1035, 1002, 1023, 1016, 1003, 1006, 1049, 951, 1031, 1000, 999, 1060, 1086, 1067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1082, 990, 1038, 962, 25, 0, 0, 0, 1033, 970, 1028, 1038, 0, 0, 0, 0, 1009, 1052, 1001, 1044, 0, 0, 0, 0, 1078, 1013, 1030, 978, 0, 0, 0, 0, 987, 1035, 1003, 1035, 5, 4, 0, 0, 1071, 1050, 1021, 1004, 0, 0, 0, 0, 957, 991, 1013, 1074, 12, 7, 0, 0, 1017, 984, 1046, 1068, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4069, 4106, 4099, 4069, 4146, 4054, 4128] avg=4096.0000 max=4146 max_rank=5 min=4054 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[990, 1041, 1032, 991, 24, 0, 0, 0, 1032, 1019, 1038, 1058, 0, 0, 0, 0, 968, 1007, 993, 1054, 0, 0, 0, 0, 996, 1063, 1007, 1012, 0, 0, 0, 0, 1037, 1070, 1046, 990, 5, 5, 0, 0, 1056, 1005, 1072, 984, 0, 0, 0, 0, 1003, 1030, 1030, 1018, 12, 6, 0, 0, 968, 1074, 972, 1047, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4147, 4022, 4078, 4153, 4117, 4099, 4074] avg=4096.0000 max=4153 max_rank=4 min=4022 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[989, 1015, 1090, 1028, 25, 0, 0, 0, 994, 995, 998, 1007, 0, 0, 0, 0, 1010, 1045, 1052, 1039, 0, 0, 0, 0, 1007, 978, 1021, 1079, 0, 0, 0, 0, 1052, 1034, 1043, 1017, 4, 4, 0, 0, 1014, 984, 1019, 1036, 0, 0, 0, 0, 990, 1020, 1037, 1055, 12, 7, 0, 0, 1044, 994, 1000, 1018, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 3994, 4146, 4085, 4154, 4053, 4121, 4068] avg=4096.0000 max=4154 max_rank=4 min=3994 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1005, 1032, 1042, 990, 25, 0, 0, 0, 1013, 1002, 1106, 1020, 0, 0, 0, 0, 1032, 1050, 984, 1031, 0, 0, 0, 0, 1012, 1014, 1069, 1045, 0, 0, 0, 0, 975, 1001, 1009, 996, 4, 4, 0, 0, 1036, 1012, 1011, 1023, 0, 0, 0, 0, 993, 1021, 1054, 1023, 12, 7, 0, 0, 1088, 1006, 1012, 997, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4141, 4097, 4140, 3989, 4082, 4110, 4115] avg=4096.0000 max=4141 max_rank=1 min=3989 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1067, 1028, 1018, 987, 24, 0, 0, 0, 1012, 1044, 1035, 1050, 0, 0, 0, 0, 985, 1053, 1009, 1020, 0, 0, 0, 0, 964, 1018, 1049, 1029, 0, 0, 0, 0, 1010, 999, 1009, 1041, 5, 4, 0, 0, 1068, 1019, 1002, 1040, 0, 0, 0, 0, 1009, 975, 1039, 1068, 12, 6, 0, 0, 1007, 1022, 1009, 1020, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4141, 4067, 4060, 4068, 4129, 4109, 4070] avg=4096.0000 max=4141 max_rank=1 min=4060 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1007, 992, 1080, 980, 29, 0, 0, 0, 998, 1040, 1004, 1053, 0, 0, 0, 0, 1017, 1019, 1031, 1025, 0, 0, 0, 0, 1053, 1038, 1058, 981, 0, 0, 0, 0, 991, 1012, 1055, 1030, 8, 10, 0, 0, 1019, 997, 1071, 1032, 0, 0, 0, 0, 1048, 1046, 996, 986, 14, 10, 0, 0, 993, 970, 1028, 1024, 17, 6, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4095, 4092, 4130, 4106, 4119, 4100, 4038] avg=4096.0000 max=4130 max_rank=3 min=4038 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1012, 1048, 1008, 1004, 26, 0, 0, 0, 1023, 997, 991, 1007, 0, 0, 0, 0, 1050, 1061, 1057, 1016, 0, 0, 0, 0, 1021, 1046, 962, 1054, 0, 0, 0, 0, 1010, 1072, 990, 1050, 5, 4, 0, 0, 1046, 1010, 965, 1057, 0, 0, 0, 0, 1011, 1053, 1006, 1036, 12, 7, 0, 0, 991, 1041, 1044, 963, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4018, 4184, 4083, 4131, 4078, 4125, 4051] avg=4096.0000 max=4184 max_rank=2 min=4018 min_rank=1 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1019, 1033, 990, 975, 25, 0, 0, 0, 999, 1048, 1044, 1072, 0, 0, 0, 0, 1007, 967, 1018, 1062, 0, 0, 0, 0, 998, 1034, 1046, 1015, 0, 0, 0, 0, 1020, 1033, 1035, 1002, 4, 4, 0, 0, 1019, 1016, 1003, 1006, 0, 0, 0, 0, 1049, 951, 1031, 1000, 12, 7, 0, 0, 999, 1060, 1086, 1067, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4163, 4054, 4093, 4098, 4044, 4050, 4224] avg=4096.0000 max=4224 max_rank=7 min=4042 min_rank=0 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 7936, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[993, 1014, 1018, 1046, 1040, 1057, 1040, 981, 1009, 1065, 1071, 947, 991, 1053, 1054, 1029, 1032, 1081, 1017, 979, 1033, 1010, 984, 995, 1061, 1055, 1006, 1018, 1026, 984, 1034, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[994, 1047, 1056, 1007, 1087, 1001, 962, 1028, 1002, 1051, 974, 1045, 955, 1029, 1032, 1001, 1096, 1003, 1018, 1017, 1032, 1055, 1052, 994, 1042, 1024, 1005, 1028, 1046, 1019, 995, 1071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1052, 988, 1016, 1034, 1046, 1026, 978, 965, 1016, 993, 1060, 1043, 1004, 1029, 1021, 1029, 965, 1010, 1071, 1081, 1050, 1055, 1032, 984, 997, 1022, 1017, 1023, 1046, 1074, 1031, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1020, 1088, 997, 1053, 1067, 1026, 980, 1027, 1037, 1034, 1046, 998, 1043, 1060, 996, 971, 1101, 1013, 1015, 1043, 1043, 924, 999, 990, 994, 1000, 1049, 1025, 1039, 1044, 1064, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[980, 1006, 1033, 1040, 1035, 1046, 1056, 1056, 997, 1025, 1057, 1021, 1013, 1009, 1015, 1011, 1029, 1019, 997, 1041, 1004, 1010, 1029, 1004, 1056, 1025, 1045, 1027, 1054, 1027, 1000, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1012, 986, 1038, 1025, 1017, 985, 1019, 1019, 1031, 1029, 961, 1054, 1003, 1039, 1044, 1044, 1034, 1044, 1061, 1019, 1017, 1093, 1016, 1011, 1032, 1034, 1009, 1017, 1001, 1020, 1004, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1021, 1016, 1010, 1034, 1052, 1020, 1018, 1027, 1037, 1016, 1050, 989, 1062, 1070, 969, 999, 1056, 999, 1003, 1043, 1063, 1038, 1016, 1059, 1015, 970, 1046, 1032, 1007, 1036, 1003, 992] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[978, 1065, 1019, 1038, 1070, 985, 985, 1052, 1070, 1003, 1063, 979, 965, 1023, 1038, 1052, 1028, 1045, 1040, 1027, 1077, 1020, 1009, 1026, 1032, 991, 1014, 1015, 1013, 1002, 1031, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[993, 1014, 1018, 1046, 5, 0, 0, 0, 1040, 1057, 1040, 981, 1, 0, 0, 0, 1009, 1065, 1071, 947, 0, 0, 0, 0, 991, 1053, 1054, 1029, 14, 0, 0, 0, 1002, 1081, 1017, 979, 0, 0, 0, 0, 1033, 1010, 984, 995, 5, 0, 0, 0, 1061, 1055, 1006, 1018, 5, 0, 0, 0, 1026, 984, 1034, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4119, 4092, 4141, 4079, 4027, 4145, 4089] avg=4096.0000 max=4145 max_rank=6 min=4027 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[994, 1047, 1056, 1007, 5, 0, 0, 0, 1087, 1001, 962, 1028, 1, 0, 0, 0, 1002, 1051, 974, 1045, 0, 0, 0, 0, 955, 1029, 1032, 1001, 15, 0, 0, 0, 1065, 1003, 1018, 1017, 0, 0, 0, 0, 1032, 1055, 1052, 994, 5, 0, 0, 0, 1042, 1024, 1005, 1028, 5, 0, 0, 0, 1046, 1019, 995, 1071, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4079, 4072, 4032, 4103, 4138, 4104, 4131] avg=4096.0000 max=4138 max_rank=5 min=4032 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[980, 1006, 1033, 1040, 5, 0, 0, 0, 1035, 1046, 1056, 1056, 1, 0, 0, 0, 997, 1025, 1057, 1021, 0, 0, 0, 0, 1013, 1009, 1015, 1011, 14, 0, 0, 0, 999, 1019, 997, 1041, 0, 0, 0, 0, 1004, 1010, 1029, 1004, 5, 0, 0, 0, 1056, 1025, 1045, 1027, 5, 0, 0, 0, 1054, 1027, 1000, 1001, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4194, 4100, 4062, 4056, 4052, 4158, 4082] avg=4096.0000 max=4194 max_rank=1 min=4052 min_rank=5 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1052, 988, 1016, 1034, 5, 0, 0, 0, 1046, 1026, 978, 965, 1, 0, 0, 0, 1016, 993, 1060, 1043, 0, 0, 0, 0, 1004, 1029, 1021, 1029, 13, 0, 0, 0, 937, 1010, 1071, 1081, 0, 0, 0, 0, 1050, 1055, 1032, 984, 5, 0, 0, 0, 997, 1022, 1017, 1023, 4, 0, 0, 0, 1046, 1074, 1031, 1010, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4016, 4112, 4096, 4099, 4126, 4063, 4161] avg=4096.0000 max=4161 max_rank=7 min=4016 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1020, 1088, 997, 1053, 5, 0, 0, 0, 1067, 1026, 980, 1027, 1, 0, 0, 0, 1037, 1034, 1046, 998, 0, 0, 0, 0, 1043, 1060, 996, 971, 15, 0, 0, 0, 1070, 1013, 1015, 1043, 0, 0, 0, 0, 1043, 924, 999, 990, 5, 0, 0, 0, 994, 1000, 1049, 1025, 5, 0, 0, 0, 1039, 1044, 1064, 982, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4101, 4115, 4085, 4141, 3961, 4073, 4129] avg=4096.0000 max=4163 max_rank=0 min=3961 min_rank=5 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1012, 986, 1038, 1025, 9, 0, 0, 0, 1017, 985, 1019, 1019, 7, 1, 0, 0, 1031, 1029, 956, 1054, 0, 0, 0, 0, 1003, 1039, 1044, 1044, 16, 0, 0, 0, 985, 1044, 1061, 1019, 0, 0, 0, 0, 1017, 1093, 1016, 1011, 9, 0, 0, 0, 1032, 1034, 1009, 1017, 8, 0, 0, 0, 1001, 1020, 1004, 1050, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4048, 4070, 4146, 4109, 4146, 4100, 4079] avg=4096.0000 max=4146 max_rank=3 min=4048 min_rank=1 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1021, 1016, 1010, 1034, 5, 0, 0, 0, 1052, 1020, 1018, 1027, 1, 0, 0, 0, 1037, 1016, 1050, 989, 0, 0, 0, 0, 1062, 1070, 969, 999, 14, 0, 0, 0, 1026, 999, 1003, 1043, 0, 0, 0, 0, 1063, 1038, 1016, 1059, 5, 0, 0, 0, 1015, 970, 1046, 1032, 5, 0, 0, 0, 1007, 1036, 1003, 992, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4118, 4092, 4114, 4071, 4181, 4068, 4038] avg=4096.0000 max=4181 max_rank=5 min=4038 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[978, 1065, 1019, 1038, 5, 0, 0, 0, 1070, 985, 985, 1052, 1, 0, 0, 0, 1070, 1003, 1063, 979, 0, 0, 0, 0, 965, 1023, 1038, 1052, 14, 0, 0, 0, 998, 1045, 1040, 1027, 0, 0, 0, 0, 1077, 1020, 1009, 1026, 5, 0, 0, 0, 1032, 991, 1014, 1015, 5, 0, 0, 0, 1013, 1002, 1031, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4093, 4115, 4092, 4110, 4137, 4057, 4059] avg=4096.0000 max=4137 max_rank=5 min=4057 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8192, 8064, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[999, 1045, 1006, 974, 1039, 1025, 1037, 1002, 1100, 1003, 1028, 1017, 1027, 1013, 1044, 1002, 1026, 1005, 1041, 1004, 1025, 1042, 1071, 992, 1058, 995, 1003, 1054, 1041, 1002, 1027, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1041, 1048, 1035, 981, 985, 1029, 992, 978, 1072, 1025, 977, 1007, 1031, 1017, 1037, 1013, 1040, 1007, 1069, 1011, 1039, 1028, 1062, 1048, 1022, 1002, 1033, 972, 986, 1038, 1073, 1070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1011, 1070, 1043, 1041, 1036, 1011, 1053, 998, 1039, 1031, 1051, 1036, 1004, 1022, 1061, 1032, 1048, 1013, 966, 1034, 1043, 972, 985, 1052, 984, 1020, 1066, 1049, 962, 1032, 1003, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[999, 1023, 1013, 998, 996, 1054, 1008, 1016, 975, 1018, 1055, 1030, 1037, 1060, 1061, 1052, 991, 1039, 1044, 1037, 1017, 994, 1043, 1000, 1016, 1023, 1057, 1067, 1002, 1012, 1008, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1034, 1038, 1091, 1032, 1061, 1045, 1015, 1040, 975, 1012, 1029, 1010, 1052, 1011, 1002, 987, 1037, 1060, 1015, 1031, 981, 971, 1021, 1035, 1034, 1005, 1024, 975, 1073, 1038, 986, 1048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[977, 1009, 1014, 1068, 1026, 1020, 1028, 973, 1036, 1082, 1018, 1053, 1021, 1080, 957, 1028, 1038, 1010, 993, 1017, 1038, 1020, 1036, 1055, 1030, 988, 1063, 1038, 983, 1006, 1037, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1037, 1006, 1044, 1028, 971, 1099, 1014, 1009, 1043, 1025, 1043, 1011, 1007, 1040, 1020, 992, 995, 973, 992, 996, 1062, 1024, 1054, 1012, 1035, 1026, 1009, 1021, 1039, 1037, 1076, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1085, 991, 1033, 1030, 1059, 996, 1011, 1005, 999, 1003, 1000, 1004, 1017, 1061, 1024, 1042, 1029, 1033, 1027, 1021, 969, 1073, 999, 1039, 1004, 1049, 1032, 1008, 1044, 1056, 989, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[999, 1045, 997, 974, 0, 0, 0, 0, 1039, 1025, 1037, 1002, 10, 6, 0, 0, 1095, 1003, 1028, 1017, 0, 0, 0, 0, 1027, 1003, 1044, 1002, 0, 0, 0, 0, 1026, 1005, 1041, 1004, 5, 4, 3, 3, 1025, 1042, 1068, 992, 0, 0, 0, 0, 1058, 995, 1003, 1054, 0, 0, 0, 0, 1041, 1002, 1027, 1017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4119, 4143, 4076, 4091, 4127, 4110, 4087] avg=4096.0000 max=4143 max_rank=2 min=4015 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1011, 1070, 1034, 1041, 0, 0, 0, 0, 1036, 1011, 1053, 998, 10, 6, 0, 0, 1035, 1031, 1051, 1036, 0, 0, 0, 0, 1004, 1012, 1061, 1032, 0, 0, 0, 0, 1048, 1013, 966, 1034, 4, 4, 3, 3, 1043, 972, 982, 1052, 0, 0, 0, 0, 984, 1020, 1066, 1049, 0, 0, 0, 0, 962, 1032, 1003, 996, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4114, 4153, 4109, 4075, 4049, 4119, 3993] avg=4096.0000 max=4156 max_rank=0 min=3993 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16236 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1041, 1048, 1026, 981, 0, 0, 0, 0, 985, 1029, 992, 978, 10, 6, 0, 0, 1067, 1025, 977, 1007, 0, 0, 0, 0, 1031, 1007, 1037, 1013, 0, 0, 0, 0, 1040, 1007, 1069, 1011, 5, 4, 3, 3, 1039, 1028, 1059, 1048, 0, 0, 0, 0, 1022, 1002, 1033, 972, 0, 0, 0, 0, 986, 1038, 1073, 1066, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4000, 4076, 4088, 4142, 4174, 4029, 4163] avg=4096.0000 max=4174 max_rank=5 min=4000 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[999, 1023, 1004, 998, 0, 0, 0, 0, 996, 1054, 1008, 1016, 10, 6, 0, 0, 971, 1018, 1055, 1030, 0, 0, 0, 0, 1037, 1050, 1061, 1052, 0, 0, 0, 0, 991, 1039, 1044, 1037, 4, 4, 3, 3, 1017, 994, 1040, 1000, 0, 0, 0, 0, 1016, 1023, 1057, 1067, 0, 0, 0, 0, 1002, 1012, 1008, 1019, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4090, 4074, 4200, 4125, 4051, 4163, 4041] avg=4096.0000 max=4200 max_rank=3 min=4024 min_rank=0 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[977, 1009, 1005, 1068, 0, 0, 0, 0, 1026, 1020, 1028, 973, 11, 6, 0, 0, 1032, 1082, 1018, 1053, 0, 0, 0, 0, 1021, 1069, 957, 1028, 0, 0, 0, 0, 1038, 1010, 993, 1017, 4, 4, 3, 3, 1038, 1020, 1033, 1055, 0, 0, 0, 0, 1030, 988, 1063, 1038, 0, 0, 0, 0, 983, 1006, 1037, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4064, 4185, 4075, 4072, 4146, 4119, 4048] avg=4096.0000 max=4185 max_rank=2 min=4048 min_rank=7 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1034, 1038, 1078, 1032, 0, 0, 0, 0, 1061, 1045, 1015, 1040, 15, 9, 0, 0, 966, 1012, 1029, 1010, 0, 0, 0, 0, 1052, 996, 1002, 987, 0, 0, 0, 0, 1037, 1060, 1015, 1031, 9, 6, 7, 4, 981, 971, 1008, 1035, 0, 0, 0, 0, 1034, 1005, 1024, 975, 6, 0, 0, 0, 1073, 1038, 986, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 4185, 4017, 4037, 4169, 3995, 4044, 4139] avg=4096.0000 max=4185 max_rank=1 min=3995 min_rank=5 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1037, 1006, 1035, 1028, 0, 0, 0, 0, 971, 1099, 1014, 1009, 10, 6, 0, 0, 1039, 1025, 1043, 1011, 0, 0, 0, 0, 1007, 1030, 1020, 992, 0, 0, 0, 0, 995, 973, 992, 996, 4, 4, 3, 3, 1062, 1024, 1051, 1012, 0, 0, 0, 0, 1035, 1026, 1009, 1021, 0, 0, 0, 0, 1039, 1037, 1076, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4109, 4118, 4049, 3970, 4149, 4091, 4176] avg=4096.0000 max=4176 max_rank=7 min=3970 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1085, 991, 1024, 1030, 0, 0, 0, 0, 1059, 996, 1011, 1005, 10, 6, 0, 0, 995, 1003, 1000, 1004, 0, 0, 0, 0, 1017, 1051, 1024, 1042, 0, 0, 0, 0, 1029, 1033, 1027, 1021, 4, 4, 3, 3, 969, 1073, 996, 1039, 0, 0, 0, 0, 1004, 1049, 1032, 1008, 0, 0, 0, 0, 1044, 1056, 989, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4087, 4002, 4134, 4124, 4077, 4093, 4121] avg=4096.0000 max=4134 max_rank=3 min=4002 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8064, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[995, 1032, 1050, 1042, 1026, 990, 1036, 993, 1034, 974, 1015, 990, 1022, 989, 1013, 1053, 1018, 1050, 1030, 1013, 983, 967, 998, 1060, 1046, 1005, 1049, 1092, 1048, 1038, 1043, 1074] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1043, 1008, 1081, 1059, 1012, 1005, 1073, 1018, 1056, 1058, 1056, 1060, 1022, 1015, 1023, 1009, 1000, 1007, 1014, 1024, 974, 1004, 1007, 968, 1073, 1022, 983, 1017, 1021, 973, 1034, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1036, 1076, 1021, 1055, 988, 1008, 1058, 987, 1089, 1022, 1007, 1064, 1040, 1045, 1001, 999, 1011, 1048, 1024, 1023, 996, 1030, 999, 1025, 1030, 1025, 1014, 1017, 962, 1002, 1056, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[998, 1055, 1054, 989, 993, 1042, 1052, 992, 1012, 1035, 1040, 1046, 994, 990, 1053, 1005, 969, 1075, 999, 1049, 1008, 1050, 1051, 1020, 1037, 1081, 986, 999, 1021, 1002, 1016, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1044, 1055, 1044, 1034, 1042, 987, 1039, 1024, 981, 1056, 980, 1024, 1043, 1043, 982, 987, 1027, 1083, 1037, 1021, 996, 1021, 1063, 1036, 999, 992, 1024, 1026, 997, 1022, 1001, 1058] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1004, 1041, 1020, 1012, 1047, 1080, 1066, 996, 1026, 1014, 981, 1045, 1054, 1023, 1005, 1028, 1064, 1027, 1026, 1071, 959, 1045, 1011, 1060, 1015, 1051, 1005, 978, 996, 994, 991, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1030, 1018, 1047, 1021, 1020, 991, 986, 1069, 1023, 1030, 1023, 1022, 1091, 1034, 1002, 1013, 1017, 1028, 1085, 972, 1032, 1020, 1050, 996, 946, 996, 1065, 1017, 994, 1068, 985, 1077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1018, 1031, 964, 1045, 1041, 1066, 1044, 1002, 1011, 1012, 979, 1027, 1115, 992, 1100, 998, 1057, 1080, 1025, 1018, 993, 1018, 1062, 972, 1040, 1004, 1047, 975, 1019, 979, 966, 1068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[995, 1001, 1050, 1042, 0, 0, 0, 0, 1026, 990, 1036, 993, 0, 0, 0, 0, 1034, 974, 1015, 988, 0, 0, 0, 0, 1021, 989, 1013, 1053, 0, 0, 0, 0, 1018, 1023, 1030, 1013, 0, 0, 0, 0, 983, 967, 998, 1060, 31, 5, 0, 0, 1046, 1005, 1049, 1092, 8, 2, 1, 0, 1048, 1038, 1043, 1074, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4045, 4011, 4076, 4084, 4044, 4203, 4217] avg=4096.0000 max=4217 max_rank=7 min=4011 min_rank=2 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1036, 1044, 1021, 1055, 0, 0, 0, 0, 988, 1008, 1058, 987, 0, 0, 0, 0, 1089, 1022, 1007, 1061, 0, 0, 0, 0, 1039, 1045, 1001, 999, 0, 0, 0, 0, 1011, 1022, 1024, 1023, 0, 0, 0, 0, 996, 1030, 999, 1025, 32, 4, 0, 0, 1030, 1025, 1014, 1017, 8, 3, 1, 0, 962, 1002, 1056, 1010, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4041, 4179, 4084, 4080, 4086, 4098, 4044] avg=4096.0000 max=4179 max_rank=2 min=4041 min_rank=1 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1044, 1023, 1044, 1034, 0, 0, 0, 0, 1042, 987, 1039, 1024, 0, 0, 0, 0, 981, 1056, 980, 1022, 0, 0, 0, 0, 1042, 1043, 982, 987, 0, 0, 0, 0, 1027, 1056, 1037, 1021, 0, 0, 0, 0, 996, 1021, 1063, 1036, 32, 5, 0, 0, 999, 992, 1024, 1026, 8, 2, 1, 0, 997, 1022, 1001, 1058, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4092, 4039, 4054, 4141, 4153, 4052, 4092] avg=4096.0000 max=4153 max_rank=5 min=4039 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1043, 974, 1081, 1059, 0, 0, 0, 0, 1012, 1005, 1068, 1018, 0, 0, 0, 0, 1056, 1058, 1056, 1053, 0, 0, 0, 0, 1014, 1015, 1023, 1009, 0, 0, 0, 0, 1000, 969, 1014, 1024, 0, 0, 0, 0, 974, 1004, 1007, 968, 34, 8, 0, 0, 1073, 1022, 983, 1017, 12, 7, 8, 5, 1021, 973, 1034, 1049, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4103, 4223, 4061, 4007, 3995, 4127, 4095] avg=4096.0000 max=4223 max_rank=2 min=3995 min_rank=5 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16224 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[998, 1023, 1054, 989, 0, 0, 0, 0, 993, 1042, 1052, 992, 0, 0, 0, 0, 1012, 1035, 1040, 1043, 0, 0, 0, 0, 993, 990, 1053, 1005, 0, 0, 0, 0, 969, 1048, 999, 1049, 0, 0, 0, 0, 1008, 1050, 1051, 1020, 32, 5, 0, 0, 1037, 1081, 986, 999, 8, 3, 1, 0, 1021, 1002, 1016, 1055, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4079, 4130, 4041, 4065, 4166, 4115, 4108] avg=4096.0000 max=4166 max_rank=5 min=4041 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1004, 1010, 1020, 1012, 0, 0, 0, 0, 1047, 1080, 1066, 996, 0, 0, 0, 0, 1026, 1014, 981, 1042, 0, 0, 0, 0, 1053, 1023, 1005, 1028, 0, 0, 0, 0, 1064, 1001, 1026, 1071, 0, 0, 0, 0, 959, 1045, 1011, 1060, 31, 4, 0, 0, 1015, 1051, 1005, 978, 8, 3, 1, 0, 996, 994, 991, 1033, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4189, 4063, 4109, 4162, 4110, 4061, 4028] avg=4096.0000 max=4189 max_rank=1 min=4028 min_rank=7 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1030, 987, 1047, 1021, 0, 0, 0, 0, 1020, 991, 986, 1069, 0, 0, 0, 0, 1023, 1030, 1023, 1020, 0, 0, 0, 0, 1090, 1034, 1002, 1013, 0, 0, 0, 0, 1017, 1002, 1085, 972, 0, 0, 0, 0, 1032, 1020, 1050, 996, 31, 4, 0, 0, 946, 996, 1065, 1017, 8, 2, 1, 0, 994, 1068, 985, 1077, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4066, 4096, 4139, 4076, 4133, 4035, 4138] avg=4096.0000 max=4139 max_rank=3 min=4035 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1018, 1000, 964, 1045, 0, 0, 0, 0, 1041, 1066, 1044, 1002, 0, 0, 0, 0, 1011, 1012, 979, 1025, 0, 0, 0, 0, 1114, 992, 1100, 998, 0, 0, 0, 0, 1057, 1053, 1025, 1018, 0, 0, 0, 0, 993, 1018, 1062, 972, 31, 5, 0, 0, 1040, 1004, 1047, 975, 8, 2, 1, 0, 1019, 979, 966, 1068, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4153, 4027, 4204, 4153, 4081, 4077, 4046] avg=4096.0000 max=4204 max_rank=3 min=4027 min_rank=0 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8448, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8448, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1060, 1006, 1035, 1067, 1024, 1068, 1008, 991, 1023, 1030, 993, 1011, 1021, 1035, 977, 1050, 1012, 1055, 1041, 995, 1007, 1026, 1029, 1027, 1044, 992, 1053, 993, 1032, 1047, 1002, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[984, 1012, 1049, 1045, 1072, 1035, 994, 1021, 1018, 1003, 1050, 991, 1030, 1021, 1000, 1008, 997, 1021, 1076, 1029, 1055, 988, 1060, 1089, 1030, 989, 1002, 1005, 1041, 1038, 1015, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1026, 1033, 1034, 1031, 1040, 1039, 980, 1022, 1001, 1047, 1036, 1005, 1046, 1053, 1050, 991, 1010, 992, 1038, 1005, 1033, 1028, 1067, 1005, 1024, 1062, 1010, 1049, 991, 1001, 972, 1047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[977, 1030, 996, 1041, 1053, 1026, 959, 1050, 1011, 998, 994, 1052, 1022, 1039, 1039, 978, 995, 1032, 1067, 1005, 1057, 1066, 1027, 1065, 1058, 1042, 1056, 1062, 981, 1033, 990, 967] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[993, 1053, 987, 1039, 1027, 1055, 985, 996, 1013, 1047, 1047, 1018, 1043, 1007, 1038, 1019, 979, 1010, 1010, 1074, 1009, 1054, 1042, 1048, 1044, 1033, 1009, 1021, 1027, 1008, 1026, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1032, 1020, 1006, 994, 1060, 1053, 1047, 1090, 971, 1042, 1038, 1048, 1045, 996, 1041, 1054, 1027, 999, 1008, 1043, 1015, 998, 1022, 1033, 1045, 992, 979, 1022, 1005, 1024, 1005, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[997, 968, 985, 1018, 993, 933, 1064, 1049, 1004, 1035, 1084, 1035, 1038, 993, 1049, 1012, 1044, 999, 1008, 1052, 983, 1095, 1055, 1040, 1031, 1047, 1054, 1021, 1044, 1039, 972, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[991, 1035, 1051, 1022, 1026, 1038, 1023, 1019, 1051, 1036, 1002, 1028, 1026, 996, 973, 974, 1072, 1061, 1048, 1052, 966, 1076, 1007, 979, 1007, 1074, 974, 1011, 1001, 1029, 1043, 1077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1060, 1006, 1035, 1067, 11, 4, 3, 0, 1016, 1068, 1008, 991, 0, 0, 0, 0, 1023, 1030, 993, 1011, 0, 0, 0, 0, 1021, 1035, 977, 1050, 7, 5, 0, 0, 1012, 1055, 1030, 995, 0, 0, 0, 0, 1007, 992, 1029, 1027, 0, 0, 0, 0, 1037, 992, 1053, 993, 0, 0, 0, 0, 1032, 1047, 1002, 1014, 30, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4083, 4057, 4095, 4092, 4055, 4075, 4125] avg=4096.0000 max=4186 max_rank=0 min=4055 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[984, 1012, 1049, 1045, 11, 4, 3, 0, 1064, 1035, 994, 1021, 0, 0, 0, 0, 1018, 1003, 1050, 991, 0, 0, 0, 0, 1030, 1021, 1000, 1008, 7, 5, 0, 0, 997, 1021, 1065, 1029, 0, 0, 0, 0, 1055, 955, 1060, 1089, 0, 0, 0, 0, 1023, 989, 1002, 1005, 0, 0, 0, 0, 1041, 1038, 1015, 1000, 29, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4114, 4062, 4071, 4112, 4159, 4019, 4123] avg=4096.0000 max=4159 max_rank=5 min=4019 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 1033, 1034, 1031, 11, 4, 3, 0, 1032, 1039, 980, 1022, 0, 0, 0, 0, 1001, 1047, 1036, 1005, 0, 0, 0, 0, 1046, 1053, 1050, 991, 7, 5, 0, 0, 1010, 992, 1027, 1005, 0, 0, 0, 0, 1033, 994, 1067, 1005, 0, 0, 0, 0, 1017, 1062, 1010, 1049, 0, 0, 0, 0, 991, 1001, 972, 1047, 30, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4073, 4089, 4152, 4034, 4099, 4138, 4041] avg=4096.0000 max=4152 max_rank=3 min=4034 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[977, 1030, 996, 1041, 11, 4, 3, 0, 1045, 1026, 959, 1050, 0, 0, 0, 0, 1011, 998, 994, 1052, 0, 0, 0, 0, 1022, 1039, 1039, 978, 7, 5, 0, 0, 995, 1032, 1056, 1005, 0, 0, 0, 0, 1057, 1031, 1027, 1065, 0, 0, 0, 0, 1051, 1042, 1056, 1062, 0, 0, 0, 0, 981, 1033, 990, 967, 31, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4080, 4055, 4090, 4088, 4180, 4211, 4002] avg=4096.0000 max=4211 max_rank=6 min=4002 min_rank=7 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1032, 1020, 1006, 994, 10, 4, 3, 0, 1052, 1053, 1047, 1090, 0, 0, 0, 0, 971, 1042, 1038, 1048, 0, 0, 0, 0, 1045, 996, 1041, 1054, 7, 5, 0, 0, 1027, 999, 998, 1043, 0, 0, 0, 0, 1015, 965, 1022, 1033, 0, 0, 0, 0, 1038, 992, 979, 1022, 0, 0, 0, 0, 1005, 1024, 1005, 1014, 29, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4242, 4099, 4148, 4067, 4035, 4031, 4077] avg=4096.0000 max=4242 max_rank=1 min=4031 min_rank=6 imbalance_ratio=1.035645 max_load_reduction=28526 max_load_reduction_pct=0.870544 moved_to_echo=16210 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[993, 1053, 987, 1039, 13, 6, 8, 0, 1011, 1055, 985, 996, 0, 0, 0, 0, 1013, 1047, 1047, 1018, 6, 0, 0, 0, 1043, 1007, 1038, 1019, 12, 8, 0, 0, 979, 1010, 997, 1074, 0, 0, 0, 0, 1009, 1012, 1042, 1048, 0, 0, 0, 0, 1026, 1033, 1009, 1021, 0, 0, 0, 0, 1027, 1008, 1026, 1007, 36, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4047, 4131, 4127, 4060, 4111, 4089, 4104] avg=4096.0000 max=4131 max_rank=2 min=4047 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[997, 968, 985, 1018, 10, 4, 3, 0, 985, 933, 1064, 1049, 0, 0, 0, 0, 1004, 1035, 1084, 1035, 0, 0, 0, 0, 1038, 993, 1049, 1012, 7, 5, 0, 0, 1044, 999, 998, 1052, 0, 0, 0, 0, 983, 1059, 1055, 1040, 0, 0, 0, 0, 1024, 1047, 1054, 1021, 0, 0, 0, 0, 1044, 1039, 972, 1027, 32, 0, 0, 0] rerouted_tokens_per_ep_rank=[3985, 4031, 4158, 4104, 4093, 4137, 4146, 4114] avg=4096.0000 max=4158 max_rank=2 min=3985 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[991, 1035, 1051, 1022, 11, 4, 3, 0, 1018, 1038, 1023, 1019, 0, 0, 0, 0, 1051, 1036, 1002, 1028, 0, 0, 0, 0, 1026, 996, 973, 974, 7, 5, 0, 0, 1072, 1061, 1037, 1052, 0, 0, 0, 0, 966, 1040, 1007, 979, 0, 0, 0, 0, 1000, 1074, 974, 1011, 0, 0, 0, 0, 1001, 1029, 1043, 1077, 32, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4098, 4117, 3981, 4222, 3992, 4059, 4182] avg=4096.0000 max=4222 max_rank=4 min=3981 min_rank=3 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8064, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1042, 1026, 1077, 1021, 1018, 980, 989, 1027, 1046, 1004, 1005, 1018, 1026, 998, 1024, 1008, 1041, 1057, 1036, 995, 1016, 1022, 1046, 982, 1039, 1027, 1058, 1007, 1039, 994, 1056, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1003, 1021, 1009, 1027, 1040, 996, 982, 977, 1012, 1019, 1040, 1028, 1022, 1044, 962, 1026, 1079, 1041, 1028, 1021, 1021, 1027, 1044, 1035, 996, 1041, 1102, 1031, 1006, 1006, 1052, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1028, 1046, 1000, 1087, 1011, 1024, 988, 1051, 1006, 1033, 1009, 1026, 1000, 1047, 990, 968, 1004, 986, 1046, 1014, 1045, 1068, 1099, 1002, 1004, 1046, 1012, 1027, 993, 1077, 1018, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1010, 998, 981, 1052, 1036, 1000, 1045, 947, 1059, 1015, 994, 1039, 1090, 1021, 1004, 1042, 1085, 992, 1019, 1053, 1016, 1037, 1031, 1008, 1054, 987, 992, 996, 1062, 1021, 1048, 1034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1008, 1020, 978, 1034, 1040, 1067, 1066, 996, 1013, 1035, 1019, 1044, 1006, 1069, 1027, 1031, 1023, 971, 1015, 1037, 944, 1026, 1089, 1021, 975, 1042, 1061, 1065, 1004, 1006, 1028, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[998, 1007, 1011, 1029, 1017, 1010, 1064, 1033, 1002, 957, 1007, 1068, 1025, 1036, 999, 1061, 1049, 1016, 1082, 1025, 1026, 1040, 1031, 1015, 1011, 997, 1011, 1051, 1061, 1014, 997, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1040, 980, 1069, 1070, 1019, 1024, 996, 1054, 1012, 1015, 1022, 1001, 1016, 1000, 976, 1031, 1030, 988, 1047, 1027, 1021, 1015, 1034, 1050, 1036, 1040, 1044, 1025, 1046, 971, 1041, 1028] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1000, 1026, 1016, 1021, 1025, 1008, 1003, 1014, 1065, 1024, 1006, 1007, 1043, 1044, 1001, 1028, 1027, 1030, 1061, 995, 1078, 1000, 972, 1027, 1015, 1054, 1035, 1037, 1026, 1030, 1005, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1042, 1026, 1077, 1021, 4, 0, 0, 0, 1018, 980, 989, 1027, 18, 8, 0, 0, 1046, 1004, 1005, 1018, 10, 4, 0, 0, 1026, 998, 1024, 1008, 10, 2, 0, 0, 1023, 1057, 1036, 995, 0, 0, 0, 0, 1016, 1022, 1032, 982, 0, 0, 0, 0, 1039, 1027, 1040, 1007, 0, 0, 0, 0, 1039, 994, 1050, 1044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4040, 4087, 4068, 4111, 4052, 4113, 4127] avg=4096.0000 max=4170 max_rank=0 min=4040 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1028, 1046, 1000, 1087, 4, 0, 0, 0, 1011, 1024, 988, 1051, 18, 8, 0, 0, 1006, 1033, 1009, 1026, 9, 4, 0, 0, 1000, 1047, 990, 968, 10, 2, 0, 0, 986, 986, 1046, 1014, 0, 0, 0, 0, 1045, 1068, 1085, 1002, 0, 0, 0, 0, 1004, 1046, 995, 1027, 0, 0, 0, 0, 993, 1077, 1012, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 4100, 4087, 4017, 4032, 4200, 4072, 4095] avg=4096.0000 max=4200 max_rank=5 min=4017 min_rank=3 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1003, 1021, 1009, 1027, 6, 0, 0, 0, 1040, 996, 982, 977, 24, 13, 0, 0, 1012, 1019, 1040, 1028, 14, 9, 0, 0, 1022, 1044, 962, 1026, 14, 6, 0, 0, 1055, 1041, 1028, 1021, 0, 0, 0, 0, 1021, 1027, 1021, 1035, 0, 0, 0, 0, 996, 1041, 1075, 1031, 0, 0, 0, 0, 1006, 1006, 1040, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4032, 4122, 4074, 4145, 4104, 4143, 4082] avg=4096.0000 max=4145 max_rank=4 min=4032 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1010, 998, 981, 1052, 4, 0, 0, 0, 1036, 1000, 1045, 947, 19, 8, 0, 0, 1059, 1015, 994, 1039, 9, 4, 0, 0, 1090, 1021, 1004, 1042, 10, 2, 0, 0, 1066, 992, 1019, 1053, 0, 0, 0, 0, 1016, 1037, 1017, 1008, 0, 0, 0, 0, 1054, 987, 975, 996, 0, 0, 0, 0, 1062, 1021, 1042, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4055, 4120, 4169, 4130, 4078, 4012, 4159] avg=4096.0000 max=4169 max_rank=3 min=4012 min_rank=6 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1008, 1020, 978, 1034, 4, 0, 0, 0, 1040, 1067, 1066, 996, 18, 8, 0, 0, 1013, 1035, 1019, 1044, 10, 4, 0, 0, 1006, 1069, 1027, 1031, 10, 2, 0, 0, 1005, 971, 1015, 1037, 0, 0, 0, 0, 944, 1026, 1075, 1021, 0, 0, 0, 0, 975, 1042, 1043, 1065, 0, 0, 0, 0, 1004, 1006, 1022, 1008, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4195, 4125, 4145, 4028, 4066, 4125, 4040] avg=4096.0000 max=4195 max_rank=1 min=4028 min_rank=4 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[998, 1007, 1011, 1029, 3, 0, 0, 0, 1017, 1010, 1064, 1033, 19, 8, 0, 0, 1002, 957, 1007, 1068, 9, 4, 0, 0, 1025, 1036, 999, 1061, 10, 2, 0, 0, 1030, 1016, 1082, 1025, 0, 0, 0, 0, 1026, 1040, 1017, 1015, 0, 0, 0, 0, 1011, 997, 994, 1051, 0, 0, 0, 0, 1061, 1014, 992, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4151, 4047, 4133, 4153, 4098, 4053, 4085] avg=4096.0000 max=4153 max_rank=4 min=4047 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1040, 980, 1069, 1070, 4, 0, 0, 0, 1019, 1024, 996, 1054, 18, 8, 0, 0, 1012, 1015, 1022, 1001, 10, 4, 0, 0, 1016, 1000, 976, 1031, 10, 2, 0, 0, 1012, 988, 1047, 1027, 0, 0, 0, 0, 1021, 1015, 1020, 1050, 0, 0, 0, 0, 1036, 1040, 1026, 1025, 0, 0, 0, 0, 1046, 971, 1035, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4119, 4064, 4035, 4074, 4106, 4127, 4080] avg=4096.0000 max=4163 max_rank=0 min=4035 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1000, 1026, 1016, 1021, 4, 0, 0, 0, 1025, 1008, 1003, 1014, 18, 8, 0, 0, 1065, 1024, 1006, 1007, 10, 4, 0, 0, 1043, 1044, 1001, 1028, 9, 2, 0, 0, 1009, 1030, 1061, 995, 0, 0, 0, 0, 1078, 1000, 959, 1027, 0, 0, 0, 0, 1015, 1054, 1017, 1037, 0, 0, 0, 0, 1026, 1030, 999, 1045, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4076, 4116, 4127, 4095, 4064, 4123, 4100] avg=4096.0000 max=4127 max_rank=3 min=4064 min_rank=5 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8448, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8448, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[992, 1006, 1014, 1032, 1025, 1046, 1055, 1040, 978, 1017, 983, 995, 1066, 1076, 1021, 1048, 1014, 1002, 998, 1043, 1052, 1046, 1018, 995, 1026, 1030, 1009, 1049, 1004, 1020, 1051, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1034, 1031, 1023, 1014, 970, 1013, 1026, 1002, 1038, 1007, 1053, 1057, 986, 1068, 1086, 1078, 1029, 1019, 1010, 1005, 1012, 1019, 1037, 1042, 1026, 1026, 1039, 1028, 1011, 991, 997, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1056, 1002, 1075, 1053, 1042, 1008, 994, 973, 1038, 1043, 1062, 1059, 1052, 978, 1044, 1021, 1028, 1044, 1030, 1016, 1036, 993, 1066, 989, 1026, 1031, 991, 1012, 969, 997, 1007, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1024, 1071, 1049, 1064, 1012, 1006, 991, 1030, 1019, 979, 1032, 1032, 1019, 1052, 1033, 1012, 1021, 1062, 1046, 1065, 1006, 1044, 1017, 1012, 1004, 1000, 1005, 1001, 1020, 1059, 978, 1003] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1057, 1032, 1004, 1009, 1019, 1016, 995, 1118, 1010, 1058, 1014, 1055, 1068, 1027, 974, 1021, 1010, 1034, 1041, 975, 1039, 1010, 931, 1050, 1062, 1021, 1053, 1029, 1022, 1018, 972, 1024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1007, 988, 1021, 1018, 1061, 1011, 965, 1062, 1029, 1041, 1071, 1013, 1065, 1040, 1010, 1020, 1069, 987, 1003, 1033, 1001, 1068, 1038, 998, 992, 1004, 984, 1003, 1058, 1046, 1027, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1039, 1032, 1013, 1022, 1015, 1028, 1007, 1044, 1067, 1001, 991, 1052, 1052, 1001, 1002, 1049, 983, 1050, 1009, 1023, 1036, 1068, 1001, 1016, 1025, 1028, 1005, 1022, 1049, 1013, 999, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1011, 1004, 1060, 1032, 1051, 1047, 1028, 985, 961, 1032, 1093, 1047, 961, 1043, 1065, 1037, 1061, 1023, 1008, 1020, 996, 1028, 1030, 1007, 1020, 994, 1073, 1007, 1050, 1019, 994, 981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1034, 1031, 1010, 1014, 0, 0, 0, 0, 970, 1013, 1026, 1002, 5, 4, 0, 0, 1038, 1007, 1053, 1038, 0, 0, 0, 0, 986, 1068, 1086, 1039, 0, 0, 0, 0, 1029, 1019, 1010, 1005, 0, 0, 0, 0, 1012, 1019, 1037, 1042, 8, 0, 0, 0, 1026, 1026, 1039, 1028, 15, 2, 0, 0, 1011, 991, 997, 991, 37, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4020, 4136, 4179, 4063, 4118, 4136, 4027] avg=4096.0000 max=4179 max_rank=3 min=4020 min_rank=1 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[992, 1006, 1001, 1032, 0, 0, 0, 0, 1025, 1046, 1055, 1040, 5, 4, 0, 0, 978, 1017, 983, 977, 0, 0, 0, 0, 1066, 1076, 1021, 1010, 0, 0, 0, 0, 1014, 1002, 998, 1043, 0, 0, 0, 0, 1052, 1046, 1018, 995, 8, 0, 0, 0, 1026, 1030, 1009, 1049, 14, 2, 0, 0, 1004, 1020, 1051, 1017, 36, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4175, 3955, 4173, 4057, 4119, 4130, 4128] avg=4096.0000 max=4175 max_rank=1 min=3955 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1057, 1032, 991, 1009, 0, 0, 0, 0, 1019, 1016, 995, 1118, 5, 4, 0, 0, 1010, 1058, 1014, 1036, 0, 0, 0, 0, 1068, 1027, 974, 984, 0, 0, 0, 0, 1010, 1034, 1041, 975, 0, 0, 0, 0, 1039, 1010, 931, 1050, 8, 0, 0, 0, 1062, 1021, 1053, 1029, 15, 2, 0, 0, 1022, 1018, 972, 1024, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4157, 4118, 4053, 4060, 4038, 4182, 4071] avg=4096.0000 max=4182 max_rank=6 min=4038 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1024, 1071, 1036, 1064, 0, 0, 0, 0, 1012, 1006, 991, 1030, 5, 4, 0, 0, 1019, 979, 1032, 1013, 0, 0, 0, 0, 1019, 1052, 1033, 975, 0, 0, 0, 0, 1021, 1062, 1046, 1065, 0, 0, 0, 0, 1006, 1044, 1017, 1012, 8, 0, 0, 0, 1004, 1000, 1005, 1001, 15, 2, 0, 0, 1020, 1059, 978, 1003, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[4195, 4048, 4043, 4079, 4194, 4087, 4027, 4095] avg=4096.0000 max=4195 max_rank=0 min=4027 min_rank=6 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1056, 1002, 1046, 1053, 0, 0, 0, 0, 1042, 1008, 994, 973, 11, 8, 0, 0, 1038, 1043, 1062, 1031, 0, 0, 0, 0, 1052, 978, 1044, 977, 0, 0, 0, 0, 1028, 1044, 1030, 1016, 7, 0, 0, 0, 1036, 993, 1066, 989, 11, 0, 0, 0, 1026, 1031, 991, 1012, 20, 6, 0, 0, 969, 997, 1007, 1033, 38, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4036, 4174, 4051, 4125, 4095, 4086, 4044] avg=4096.0000 max=4174 max_rank=2 min=4036 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1007, 988, 1008, 1018, 0, 0, 0, 0, 1061, 1011, 965, 1062, 5, 4, 0, 0, 1029, 1041, 1071, 995, 0, 0, 0, 0, 1065, 1040, 1010, 983, 0, 0, 0, 0, 1069, 987, 1003, 1033, 0, 0, 0, 0, 1001, 1068, 1038, 998, 8, 0, 0, 0, 992, 1004, 984, 1003, 14, 2, 0, 0, 1058, 1046, 1027, 1035, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 4108, 4136, 4098, 4092, 4113, 3999, 4201] avg=4096.0000 max=4201 max_rank=7 min=3999 min_rank=6 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1039, 1032, 1000, 1022, 0, 0, 0, 0, 1015, 1028, 1007, 1044, 5, 4, 0, 0, 1067, 1001, 991, 1033, 0, 0, 0, 0, 1052, 1001, 1002, 1011, 0, 0, 0, 0, 983, 1050, 1009, 1023, 0, 0, 0, 0, 1036, 1068, 1001, 1016, 8, 0, 0, 0, 1025, 1028, 1005, 1022, 15, 2, 0, 0, 1049, 1013, 999, 1026, 36, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4103, 4092, 4066, 4065, 4129, 4097, 4123] avg=4096.0000 max=4129 max_rank=5 min=4065 min_rank=4 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1011, 1004, 1046, 1032, 0, 0, 0, 0, 1051, 1047, 1028, 985, 6, 4, 0, 0, 961, 1032, 1093, 1028, 0, 0, 0, 0, 961, 1043, 1065, 1000, 0, 0, 0, 0, 1061, 1023, 1008, 1020, 0, 0, 0, 0, 996, 1028, 1030, 1007, 8, 0, 0, 0, 1020, 994, 1073, 1007, 15, 2, 0, 0, 1050, 1019, 994, 981, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4121, 4114, 4069, 4112, 4069, 4111, 4079] avg=4096.0000 max=4121 max_rank=1 min=4069 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8064, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[959, 1005, 1026, 1054, 1053, 1008, 1017, 1046, 1050, 1055, 1031, 987, 1029, 980, 1032, 1045, 1053, 1032, 1050, 1013, 1046, 971, 1018, 996, 1047, 1038, 1045, 1027, 1030, 1034, 980, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[994, 1005, 993, 1024, 1014, 1019, 994, 1008, 1089, 1006, 1006, 963, 1088, 1080, 1060, 1047, 1029, 1018, 1000, 1024, 1097, 1073, 1021, 1015, 1041, 1059, 980, 984, 1001, 977, 1014, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1046, 1023, 1025, 986, 1003, 1020, 1031, 1023, 1036, 1020, 1014, 997, 1018, 1076, 1028, 1024, 1030, 1039, 994, 1051, 1010, 1030, 998, 1036, 1005, 1000, 1061, 1031, 1026, 1059, 1024, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[986, 1018, 1010, 1008, 1042, 1037, 1079, 1032, 1035, 1030, 1014, 1032, 1043, 1043, 996, 1068, 1034, 1033, 1004, 983, 993, 1033, 1089, 999, 1016, 1054, 1043, 1028, 994, 973, 1008, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[999, 1034, 1041, 1008, 1054, 1003, 1060, 983, 985, 1002, 1007, 1027, 1059, 1057, 1019, 1028, 1018, 1043, 1032, 1046, 1056, 1051, 984, 1028, 1095, 1008, 1012, 994, 958, 1028, 1014, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1051, 1048, 1046, 1000, 1015, 986, 1014, 1019, 1008, 1028, 1050, 1102, 974, 1000, 975, 1017, 1032, 963, 1022, 1072, 1039, 1044, 1025, 1036, 1034, 1021, 1052, 1013, 1023, 1020, 1043, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1040, 962, 1002, 1040, 1019, 1045, 1033, 1045, 1010, 1067, 1028, 1005, 1021, 1004, 960, 982, 1011, 1038, 1063, 1046, 983, 1040, 1075, 1019, 1031, 1037, 1033, 1025, 1004, 1040, 1037, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1017, 1036, 1027, 1043, 1005, 974, 1038, 1038, 1003, 1008, 994, 1052, 970, 1067, 1050, 1018, 1073, 1034, 1072, 1033, 1016, 1077, 999, 1039, 1038, 988, 1013, 1018, 977, 1035, 957, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[994, 1005, 993, 1024, 12, 7, 6, 0, 1014, 1019, 994, 1008, 1, 0, 0, 0, 1089, 1006, 1006, 963, 3, 0, 0, 0, 1088, 1070, 1060, 1047, 0, 0, 0, 0, 1003, 1018, 1000, 1024, 0, 0, 0, 0, 1097, 1052, 1021, 1015, 0, 0, 0, 0, 1029, 1059, 980, 984, 0, 0, 0, 0, 1001, 977, 1014, 1045, 26, 14, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4036, 4067, 4265, 4045, 4185, 4052, 4077] avg=4096.0000 max=4265 max_rank=3 min=4036 min_rank=1 imbalance_ratio=1.041260 max_load_reduction=28503 max_load_reduction_pct=0.869843 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[959, 1005, 1026, 1054, 12, 6, 6, 0, 1053, 1008, 1017, 1046, 1, 0, 0, 0, 1050, 1055, 1031, 987, 3, 0, 0, 0, 1029, 970, 1032, 1045, 0, 0, 0, 0, 1026, 1032, 1050, 1013, 0, 0, 0, 0, 1046, 953, 1018, 996, 0, 0, 0, 0, 1035, 1038, 1045, 1027, 0, 0, 0, 0, 1030, 1034, 980, 1011, 27, 12, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4125, 4126, 4076, 4121, 4013, 4145, 4094] avg=4096.0000 max=4145 max_rank=6 min=4013 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1046, 1023, 1025, 986, 12, 7, 6, 0, 1003, 1020, 1031, 1023, 1, 0, 0, 0, 1036, 1020, 1014, 997, 3, 0, 0, 0, 1018, 1066, 1028, 1024, 0, 0, 0, 0, 1004, 1039, 994, 1051, 0, 0, 0, 0, 1010, 1010, 998, 1036, 0, 0, 0, 0, 993, 1000, 1061, 1031, 0, 0, 0, 0, 1026, 1059, 1024, 1004, 26, 13, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4078, 4070, 4136, 4088, 4054, 4085, 4152] avg=4096.0000 max=4152 max_rank=7 min=4054 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[986, 1018, 1010, 1008, 12, 7, 6, 0, 1042, 1037, 1079, 1032, 1, 0, 0, 0, 1035, 1030, 1014, 1032, 3, 0, 0, 0, 1043, 1033, 996, 1068, 0, 0, 0, 0, 1007, 1033, 1004, 983, 0, 0, 0, 0, 993, 1013, 1089, 999, 0, 0, 0, 0, 1004, 1054, 1043, 1028, 0, 0, 0, 0, 994, 973, 1008, 1011, 27, 13, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4191, 4114, 4140, 4027, 4094, 4129, 4026] avg=4096.0000 max=4191 max_rank=1 min=4026 min_rank=7 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16276 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1040, 962, 1002, 1040, 12, 7, 6, 0, 1019, 1045, 1033, 1045, 1, 0, 0, 0, 1010, 1067, 1028, 1005, 3, 0, 0, 0, 1021, 994, 960, 982, 0, 0, 0, 0, 985, 1038, 1063, 1046, 0, 0, 0, 0, 983, 1020, 1075, 1019, 0, 0, 0, 0, 1019, 1037, 1033, 1025, 0, 0, 0, 0, 1004, 1040, 1037, 1023, 26, 13, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4143, 4113, 3957, 4132, 4097, 4114, 4143] avg=4096.0000 max=4143 max_rank=1 min=3957 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1051, 1048, 1046, 1000, 18, 9, 10, 0, 1015, 986, 1014, 1019, 4, 0, 0, 0, 1008, 1028, 1050, 1102, 6, 0, 0, 0, 974, 980, 975, 1017, 0, 0, 0, 0, 1001, 963, 1022, 1072, 0, 0, 0, 0, 1039, 1017, 1025, 1036, 0, 0, 0, 0, 1016, 1021, 1052, 1013, 0, 0, 0, 0, 1023, 1020, 1043, 996, 31, 18, 0, 0] rerouted_tokens_per_ep_rank=[4182, 4038, 4194, 3946, 4058, 4117, 4102, 4131] avg=4096.0000 max=4194 max_rank=2 min=3946 min_rank=3 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[999, 1034, 1041, 1008, 13, 7, 6, 0, 1054, 1003, 1060, 983, 1, 0, 0, 0, 985, 1002, 1007, 1027, 3, 0, 0, 0, 1059, 1047, 1019, 1028, 0, 0, 0, 0, 992, 1043, 1032, 1046, 0, 0, 0, 0, 1056, 1030, 984, 1028, 0, 0, 0, 0, 1082, 1008, 1012, 994, 0, 0, 0, 0, 958, 1028, 1014, 1035, 26, 14, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4101, 4024, 4153, 4113, 4098, 4096, 4075] avg=4096.0000 max=4153 max_rank=3 min=4024 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1017, 1036, 1027, 1043, 12, 7, 6, 0, 1005, 974, 1038, 1038, 1, 0, 0, 0, 1003, 1008, 994, 1052, 3, 0, 0, 0, 970, 1057, 1050, 1018, 0, 0, 0, 0, 1045, 1034, 1072, 1033, 0, 0, 0, 0, 1016, 1056, 999, 1039, 0, 0, 0, 0, 1026, 988, 1013, 1018, 0, 0, 0, 0, 977, 1035, 957, 1059, 28, 14, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4056, 4060, 4095, 4184, 4110, 4045, 4070] avg=4096.0000 max=4184 max_rank=4 min=4045 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1031, 1038, 1056, 1025, 1008, 980, 1008, 989, 1028, 1038, 1031, 1042, 975, 1072, 1009, 1086, 970, 1055, 1053, 1049, 1011, 1045, 1047, 1007, 1028, 1012, 994, 996, 1038, 1023, 1016, 1008] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1076, 1027, 981, 1065, 1034, 1022, 1065, 1010, 1017, 1018, 1002, 1069, 1055, 1017, 1005, 1026, 985, 1045, 1011, 986, 1004, 1038, 1017, 1027, 1013, 1053, 975, 1059, 1026, 1006, 984, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1047, 994, 1001, 1013, 1046, 1035, 1034, 1032, 1002, 1007, 1021, 1023, 1057, 1062, 1065, 1060, 979, 994, 1030, 1020, 1015, 1060, 992, 995, 1046, 1061, 995, 1017, 971, 1031, 1027, 1036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[985, 1027, 1044, 1016, 1019, 1036, 976, 1056, 1030, 1050, 1035, 1062, 1006, 973, 1047, 1014, 995, 1024, 1035, 998, 1038, 1025, 986, 1007, 1062, 1010, 1018, 1031, 1024, 1058, 1041, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1019, 973, 1039, 1015, 1028, 988, 1009, 1020, 994, 970, 1010, 1041, 1001, 1058, 1056, 1035, 1077, 997, 1002, 1058, 1039, 1008, 1053, 1031, 1042, 1052, 1062, 1091, 979, 952, 1067, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1010, 1046, 1010, 1011, 976, 1051, 1033, 1043, 1027, 1080, 1005, 1011, 1022, 1055, 1069, 1016, 1016, 1086, 1014, 980, 1020, 1010, 1036, 1035, 1033, 989, 1058, 948, 1012, 1036, 975, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1006, 1006, 1038, 1069, 995, 1044, 1078, 1010, 1034, 1028, 1040, 1022, 1025, 1029, 1016, 1046, 1004, 1036, 1015, 1017, 1040, 985, 1042, 986, 987, 1042, 1038, 1032, 1004, 1002, 1006, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1009, 1026, 1074, 1005, 1044, 1042, 1040, 1008, 976, 1051, 985, 1043, 993, 1052, 983, 1014, 1020, 1026, 1014, 1059, 1047, 1002, 979, 1035, 1043, 1054, 1043, 994, 1047, 1000, 982, 1078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1076, 1027, 980, 1065, 0, 0, 0, 0, 1034, 1022, 1065, 1010, 1, 0, 0, 0, 1017, 1018, 1002, 1066, 0, 0, 0, 0, 1055, 990, 1005, 1026, 0, 0, 0, 0, 985, 1045, 1011, 986, 10, 4, 0, 0, 1004, 1038, 1017, 1027, 9, 3, 0, 0, 1013, 1040, 975, 1059, 0, 0, 0, 0, 1026, 1006, 984, 1050, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4132, 4103, 4076, 4041, 4098, 4087, 4083] avg=4096.0000 max=4148 max_rank=0 min=4041 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1010, 1046, 1009, 1011, 0, 0, 0, 0, 976, 1051, 1033, 1043, 1, 0, 0, 0, 1027, 1080, 1005, 1009, 0, 0, 0, 0, 1022, 1027, 1069, 1016, 0, 0, 0, 0, 1016, 1086, 1014, 980, 10, 3, 0, 0, 1020, 1010, 1036, 1035, 9, 2, 0, 0, 1033, 977, 1058, 948, 0, 0, 0, 0, 1012, 1036, 975, 1055, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4104, 4121, 4134, 4109, 4112, 4016, 4096] avg=4096.0000 max=4134 max_rank=3 min=4016 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1047, 994, 1000, 1013, 0, 0, 0, 0, 1046, 1035, 1034, 1032, 1, 0, 0, 0, 1002, 1007, 1021, 1021, 0, 0, 0, 0, 1057, 1034, 1065, 1060, 0, 0, 0, 0, 979, 994, 1030, 1020, 10, 4, 0, 0, 1015, 1060, 992, 995, 9, 2, 0, 0, 1046, 1048, 995, 1017, 0, 0, 0, 0, 971, 1031, 1027, 1036, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4148, 4051, 4216, 4037, 4073, 4106, 4083] avg=4096.0000 max=4216 max_rank=3 min=4037 min_rank=4 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1031, 1038, 1055, 1025, 0, 0, 0, 0, 1008, 980, 1008, 989, 1, 0, 0, 0, 1028, 1038, 1031, 1039, 0, 0, 0, 0, 975, 1044, 1009, 1086, 0, 0, 0, 0, 970, 1055, 1053, 1049, 10, 4, 0, 0, 1011, 1045, 1047, 1007, 9, 3, 0, 0, 1028, 999, 994, 996, 0, 0, 0, 0, 1038, 1023, 1016, 1008, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 3986, 4136, 4114, 4141, 4122, 4017, 4103] avg=4096.0000 max=4149 max_rank=0 min=3986 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1006, 1006, 1037, 1069, 0, 0, 0, 0, 995, 1044, 1078, 1010, 1, 0, 0, 0, 1034, 1028, 1040, 1020, 0, 0, 0, 0, 1025, 1001, 1016, 1046, 0, 0, 0, 0, 1004, 1036, 1015, 1017, 10, 4, 0, 0, 1040, 985, 1042, 986, 9, 2, 0, 0, 987, 1029, 1038, 1032, 0, 0, 0, 0, 1004, 1002, 1006, 1046, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4128, 4122, 4088, 4086, 4064, 4086, 4076] avg=4096.0000 max=4128 max_rank=1 min=4064 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1019, 973, 1038, 1015, 0, 0, 0, 0, 1028, 988, 1009, 1020, 1, 0, 0, 0, 994, 970, 1010, 1038, 0, 0, 0, 0, 1001, 1030, 1056, 1035, 0, 0, 0, 0, 1077, 997, 1002, 1058, 10, 4, 0, 0, 1039, 1008, 1053, 1031, 9, 3, 0, 0, 1042, 1039, 1062, 1091, 0, 0, 0, 0, 979, 952, 1067, 1002, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4046, 4012, 4122, 4148, 4143, 4234, 4018] avg=4096.0000 max=4234 max_rank=6 min=4012 min_rank=2 imbalance_ratio=1.033691 max_load_reduction=28534 max_load_reduction_pct=0.870789 moved_to_echo=16543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[985, 1027, 1037, 1016, 0, 0, 0, 0, 1019, 1036, 976, 1056, 2, 0, 0, 0, 1030, 1050, 1035, 1056, 0, 0, 0, 0, 1006, 937, 1047, 1014, 0, 0, 0, 0, 995, 1024, 1035, 998, 15, 6, 0, 0, 1038, 1025, 986, 1007, 14, 6, 5, 0, 1062, 990, 1018, 1031, 0, 0, 0, 0, 1024, 1058, 1041, 1040, 21, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4089, 4171, 4004, 4073, 4081, 4101, 4184] avg=4096.0000 max=4184 max_rank=7 min=4004 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1009, 1026, 1073, 1005, 0, 0, 0, 0, 1044, 1042, 1040, 1008, 1, 0, 0, 0, 976, 1051, 985, 1040, 0, 0, 0, 0, 993, 1024, 983, 1014, 0, 0, 0, 0, 1020, 1026, 1014, 1059, 10, 4, 0, 0, 1047, 1002, 979, 1035, 9, 3, 0, 0, 1043, 1041, 1043, 994, 0, 0, 0, 0, 1047, 1000, 982, 1078, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4135, 4052, 4014, 4133, 4075, 4121, 4125] avg=4096.0000 max=4135 max_rank=1 min=4014 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1001, 996, 1070, 1021, 1015, 1042, 943, 1004, 1028, 1032, 1066, 1012, 997, 1001, 1004, 1045, 1049, 968, 1049, 1031, 1028, 997, 1044, 980, 1029, 1050, 1018, 1066, 1037, 1053, 1009, 1083] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1001, 1012, 1024, 969, 1036, 991, 1052, 1031, 1039, 1016, 1016, 1014, 1058, 1053, 1041, 1028, 1023, 1017, 1029, 1012, 985, 1026, 1015, 989, 1027, 1035, 1061, 1042, 1034, 1008, 1042, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1048, 1078, 1011, 1011, 1057, 990, 1025, 1067, 1010, 1062, 991, 1014, 1025, 991, 995, 1029, 1060, 995, 1000, 997, 1063, 1004, 1008, 1021, 1025, 1009, 1054, 1013, 990, 1050, 1054, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1080, 1053, 1012, 992, 1010, 1026, 1045, 974, 1076, 1022, 989, 1033, 1054, 1020, 983, 1003, 1043, 1024, 992, 1022, 1051, 988, 1027, 1034, 1012, 1049, 986, 999, 1004, 1023, 1078, 1064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1074, 974, 1021, 1037, 995, 1006, 978, 1088, 1021, 1098, 1012, 1030, 1046, 1044, 1048, 1033, 986, 1014, 1015, 1052, 986, 996, 993, 1037, 1040, 1018, 1054, 1008, 999, 1011, 1024, 1030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1029, 1018, 1011, 1012, 1077, 1036, 1001, 1048, 1073, 1038, 990, 988, 1019, 1044, 1076, 993, 1013, 1062, 949, 1033, 993, 948, 1016, 1026, 986, 992, 1048, 1059, 1069, 1022, 1068, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1014, 1035, 1011, 1035, 1001, 1025, 1033, 963, 1011, 995, 1087, 1011, 1018, 1004, 993, 1051, 1069, 1031, 1045, 1021, 1018, 972, 992, 1075, 1051, 1026, 996, 1059, 1000, 1062, 1041, 1023] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1002, 991, 990, 1007, 999, 1039, 1054, 1026, 1018, 1029, 988, 1086, 998, 1062, 975, 1070, 1045, 994, 1020, 1046, 1029, 1043, 1015, 1025, 1041, 1042, 1020, 1003, 1019, 1065, 1012, 1015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1001, 996, 1070, 1021, 15, 0, 0, 0, 1015, 1042, 943, 1004, 11, 0, 0, 0, 1028, 1018, 1066, 1012, 0, 0, 0, 0, 997, 1001, 1004, 1041, 0, 0, 0, 0, 1049, 968, 1049, 1031, 4, 3, 0, 0, 1028, 997, 1044, 980, 38, 3, 0, 0, 1029, 1050, 1018, 1048, 0, 0, 0, 0, 1037, 1053, 971, 1083, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4015, 4124, 4043, 4104, 4090, 4145, 4144] avg=4096.0000 max=4145 max_rank=6 min=4015 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16483 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1001, 1012, 1024, 969, 15, 0, 0, 0, 1036, 991, 1052, 1031, 11, 0, 0, 0, 1039, 1002, 1016, 1014, 0, 0, 0, 0, 1058, 1053, 1041, 1024, 0, 0, 0, 0, 1023, 1017, 1029, 1012, 4, 3, 0, 0, 985, 1026, 1015, 989, 39, 3, 0, 0, 1027, 1035, 1061, 1024, 0, 0, 0, 0, 1034, 1008, 1003, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 4121, 4071, 4176, 4088, 4057, 4147, 4087] avg=4096.0000 max=4176 max_rank=3 min=4021 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1074, 974, 1021, 1037, 14, 0, 0, 0, 995, 1006, 978, 1088, 12, 0, 0, 0, 1021, 1083, 1012, 1030, 0, 0, 0, 0, 1046, 1044, 1048, 1029, 0, 0, 0, 0, 986, 1014, 1015, 1052, 4, 3, 0, 0, 986, 996, 993, 1037, 38, 3, 0, 0, 1040, 1018, 1054, 991, 0, 0, 0, 0, 999, 1011, 986, 1030, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4079, 4146, 4167, 4074, 4053, 4103, 4026] avg=4096.0000 max=4167 max_rank=3 min=4026 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1048, 1078, 1011, 1011, 19, 7, 0, 0, 1057, 990, 1025, 1067, 14, 0, 0, 0, 1010, 1033, 991, 1014, 0, 0, 0, 0, 1025, 991, 995, 1023, 0, 0, 0, 0, 1060, 995, 1000, 997, 6, 8, 0, 0, 1063, 1004, 1008, 1021, 43, 8, 0, 0, 1025, 1009, 1054, 986, 0, 0, 0, 0, 990, 1050, 1011, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4153, 4048, 4034, 4066, 4147, 4074, 4072] avg=4096.0000 max=4174 max_rank=0 min=4034 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1029, 1018, 1011, 1012, 15, 0, 0, 0, 1077, 1036, 1001, 1048, 11, 0, 0, 0, 1073, 1024, 990, 988, 0, 0, 0, 0, 1019, 1044, 1076, 990, 0, 0, 0, 0, 1013, 1062, 949, 1033, 3, 3, 0, 0, 993, 948, 1016, 1026, 40, 3, 0, 0, 986, 992, 1048, 1041, 0, 0, 0, 0, 1069, 1022, 1028, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4173, 4075, 4129, 4063, 4026, 4067, 4150] avg=4096.0000 max=4173 max_rank=1 min=4026 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1080, 1053, 1012, 992, 14, 0, 0, 0, 1010, 1026, 1045, 974, 11, 0, 0, 0, 1076, 1008, 989, 1033, 0, 0, 0, 0, 1054, 1020, 983, 999, 0, 0, 0, 0, 1043, 1024, 992, 1022, 4, 3, 0, 0, 1051, 988, 1027, 1034, 40, 3, 0, 0, 1012, 1049, 986, 982, 0, 0, 0, 0, 1004, 1023, 1038, 1064, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4066, 4106, 4056, 4088, 4143, 4029, 4129] avg=4096.0000 max=4151 max_rank=0 min=4029 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1002, 991, 990, 1007, 14, 0, 0, 0, 999, 1039, 1054, 1026, 11, 0, 0, 0, 1018, 1015, 988, 1086, 0, 0, 0, 0, 998, 1062, 975, 1066, 0, 0, 0, 0, 1045, 994, 1020, 1046, 4, 3, 0, 0, 1029, 1043, 1015, 1025, 38, 3, 0, 0, 1041, 1042, 1020, 986, 0, 0, 0, 0, 1019, 1065, 974, 1015, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4004, 4129, 4107, 4101, 4112, 4153, 4089, 4073] avg=4096.0000 max=4153 max_rank=5 min=4004 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1014, 1035, 1011, 1035, 15, 0, 0, 0, 1001, 1025, 1033, 963, 10, 0, 0, 0, 1011, 982, 1087, 1011, 0, 0, 0, 0, 1018, 1004, 993, 1047, 0, 0, 0, 0, 1069, 1031, 1045, 1021, 4, 3, 0, 0, 1018, 972, 992, 1075, 39, 3, 0, 0, 1051, 1026, 996, 1041, 0, 0, 0, 0, 1000, 1062, 1002, 1023, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4032, 4091, 4062, 4173, 4099, 4114, 4087] avg=4096.0000 max=4173 max_rank=4 min=4032 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8064, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[991, 985, 1016, 1022, 1046, 1025, 1056, 1061, 1014, 1051, 1067, 1039, 991, 1064, 1000, 1042, 1047, 995, 1003, 1009, 1053, 1029, 970, 1010, 1042, 1023, 1004, 1007, 1013, 1009, 1068, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1060, 1035, 1051, 1026, 971, 1035, 1010, 1076, 1047, 1020, 1049, 1050, 1049, 1020, 1009, 1016, 1058, 1002, 1009, 998, 1010, 1003, 1011, 1002, 1035, 999, 1040, 1041, 973, 994, 1030, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1043, 1087, 1031, 1013, 1040, 1003, 1005, 1014, 1071, 1038, 1029, 995, 1107, 1034, 1011, 999, 1038, 1040, 1015, 1008, 982, 945, 1001, 1016, 1030, 1031, 1001, 1036, 1030, 1012, 1031, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1016, 1025, 1008, 1012, 1062, 1024, 1030, 1029, 1065, 1003, 1011, 1030, 977, 1073, 1002, 1037, 967, 1049, 1012, 1052, 1021, 1017, 1061, 1027, 1050, 1001, 1022, 1014, 1035, 990, 1021, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1016, 1065, 1008, 1042, 1026, 1064, 1050, 1015, 994, 1007, 996, 1027, 1040, 993, 1005, 1044, 1067, 994, 1019, 1043, 1046, 1021, 1001, 1044, 1008, 1011, 955, 1017, 1026, 1050, 1035, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1028, 1046, 1041, 1029, 1016, 1040, 997, 1073, 1022, 994, 996, 1024, 1044, 980, 1028, 1024, 1025, 1074, 1036, 1009, 979, 989, 1036, 963, 1054, 1046, 1053, 1026, 1050, 1006, 1022, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1008, 1005, 1044, 998, 1040, 1061, 987, 995, 1064, 1024, 1011, 1033, 1020, 1000, 1032, 1002, 1011, 1004, 992, 1053, 1043, 1061, 1062, 1034, 1018, 1010, 1012, 1036, 1057, 1027, 1022, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1062, 1045, 1038, 976, 1039, 1028, 996, 1022, 1099, 1014, 1088, 1080, 1006, 1064, 988, 977, 1001, 1016, 1029, 1070, 1024, 1016, 997, 993, 1014, 1021, 961, 1010, 1012, 1018, 1053, 1011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[991, 974, 1016, 1022, 0, 0, 0, 0, 1046, 1025, 1056, 1041, 0, 0, 0, 0, 980, 1051, 1067, 1039, 0, 0, 0, 0, 991, 1064, 1000, 1042, 9, 1, 0, 0, 1047, 995, 1003, 1009, 2, 0, 0, 0, 1053, 1029, 970, 1010, 34, 2, 0, 0, 1042, 1023, 1004, 1007, 17, 0, 0, 0, 1013, 1009, 1068, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4003, 4168, 4137, 4107, 4056, 4098, 4093, 4106] avg=4096.0000 max=4168 max_rank=1 min=4003 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1043, 1074, 1031, 1013, 0, 0, 0, 0, 1040, 1003, 1005, 994, 0, 0, 0, 0, 1035, 1038, 1029, 995, 0, 0, 0, 0, 1107, 1034, 1011, 999, 10, 1, 0, 0, 1038, 1040, 1015, 1008, 3, 0, 0, 0, 982, 945, 1001, 1016, 36, 2, 0, 0, 1030, 1031, 1001, 1036, 17, 0, 0, 0, 1030, 1012, 1031, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4042, 4097, 4162, 4104, 3982, 4115, 4105] avg=4096.0000 max=4162 max_rank=3 min=3982 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1028, 1035, 1041, 1029, 0, 0, 0, 0, 1016, 1040, 997, 1052, 0, 0, 0, 0, 988, 994, 996, 1024, 0, 0, 0, 0, 1044, 980, 1028, 1024, 9, 1, 0, 0, 1025, 1074, 1036, 1009, 2, 0, 0, 0, 979, 989, 1036, 963, 34, 2, 0, 0, 1054, 1046, 1053, 1026, 18, 0, 0, 0, 1050, 1006, 1022, 1018, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4105, 4002, 4086, 4146, 4003, 4197, 4096] avg=4096.0000 max=4197 max_rank=6 min=4002 min_rank=2 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1016, 1014, 1008, 1012, 0, 0, 0, 0, 1062, 1024, 1030, 1009, 0, 0, 0, 0, 1029, 1003, 1011, 1030, 0, 0, 0, 0, 977, 1073, 1002, 1037, 9, 1, 0, 0, 967, 1049, 1012, 1052, 2, 0, 0, 0, 1021, 1017, 1061, 1027, 36, 2, 0, 0, 1050, 1001, 1022, 1014, 17, 0, 0, 0, 1035, 990, 1021, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4125, 4073, 4099, 4082, 4164, 4104, 4071] avg=4096.0000 max=4164 max_rank=5 min=4050 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1016, 1040, 1008, 1042, 0, 0, 0, 0, 1026, 1064, 1050, 988, 0, 0, 0, 0, 958, 1007, 996, 1027, 0, 0, 0, 0, 1040, 993, 1005, 1044, 15, 4, 0, 0, 1067, 994, 1019, 1043, 8, 0, 0, 0, 1046, 1021, 1001, 1044, 36, 3, 0, 0, 1008, 1011, 955, 1017, 20, 0, 0, 0, 1026, 1050, 1035, 1039, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4128, 3988, 4101, 4131, 4151, 4011, 4152] avg=4096.0000 max=4152 max_rank=7 min=3988 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1060, 1024, 1051, 1026, 0, 0, 0, 0, 971, 1035, 1010, 1055, 0, 0, 0, 0, 1012, 1020, 1049, 1050, 0, 0, 0, 0, 1049, 1020, 1009, 1016, 9, 1, 0, 0, 1058, 1002, 1009, 998, 2, 0, 0, 0, 1010, 1003, 1011, 1002, 35, 2, 0, 0, 1035, 999, 1040, 1041, 18, 0, 0, 0, 973, 994, 1030, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4071, 4131, 4104, 4069, 4063, 4133, 4036] avg=4096.0000 max=4161 max_rank=0 min=4036 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1008, 994, 1044, 998, 0, 0, 0, 0, 1040, 1061, 987, 976, 0, 0, 0, 0, 1028, 1024, 1011, 1033, 0, 0, 0, 0, 1020, 1000, 1032, 1002, 9, 1, 0, 0, 1011, 1004, 992, 1053, 2, 0, 0, 0, 1043, 1061, 1062, 1034, 36, 2, 0, 0, 1018, 1010, 1012, 1036, 16, 0, 0, 0, 1057, 1027, 1022, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4064, 4096, 4064, 4062, 4238, 4092, 4108] avg=4096.0000 max=4238 max_rank=5 min=4044 min_rank=0 imbalance_ratio=1.034668 max_load_reduction=28530 max_load_reduction_pct=0.870667 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1062, 1034, 1038, 976, 0, 0, 0, 0, 1039, 1028, 996, 1002, 0, 0, 0, 0, 1062, 1014, 1088, 1080, 0, 0, 0, 0, 1006, 1064, 988, 977, 9, 1, 0, 0, 1001, 1016, 1029, 1070, 2, 0, 0, 0, 1024, 1016, 997, 993, 37, 2, 0, 0, 1014, 1021, 961, 1010, 17, 0, 0, 0, 1012, 1018, 1053, 1011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4065, 4244, 4045, 4118, 4069, 4023, 4094] avg=4096.0000 max=4244 max_rank=2 min=4023 min_rank=6 imbalance_ratio=1.036133 max_load_reduction=28524 max_load_reduction_pct=0.870483 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8064, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1067, 974, 980, 974, 1017, 1012, 1046, 1033, 1010, 1030, 1017, 1054, 990, 1031, 1073, 1023, 1044, 1015, 972, 1109, 988, 991, 1084, 1069, 1071, 1020, 994, 993, 996, 1038, 1012, 1041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[995, 1016, 1012, 992, 1038, 995, 1016, 1059, 1015, 1027, 1027, 1028, 1046, 1021, 1010, 1036, 1033, 1075, 1056, 1041, 1016, 1012, 1041, 1034, 1042, 986, 1053, 1039, 964, 1019, 1002, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1052, 1016, 1074, 1053, 1033, 1029, 1007, 1015, 1018, 1019, 1021, 1001, 1087, 990, 987, 1010, 1000, 1040, 989, 983, 1029, 1012, 990, 1016, 998, 1052, 1047, 1065, 1004, 1045, 1053, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1024, 1056, 1021, 1037, 990, 1001, 984, 1066, 1058, 1015, 1030, 992, 1042, 1013, 1029, 1049, 1015, 1044, 1004, 1026, 1055, 1007, 1032, 1067, 1035, 1057, 1077, 1000, 994, 1028, 938, 982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1017, 992, 1072, 1013, 1019, 1031, 1011, 961, 1022, 1025, 1055, 987, 977, 1028, 1042, 1026, 1039, 1066, 970, 1041, 1034, 1023, 1077, 989, 1068, 1047, 981, 1023, 982, 1043, 1076, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[995, 1021, 1037, 1027, 1001, 1020, 1055, 987, 1011, 1016, 1038, 1009, 1032, 1023, 1043, 1059, 1001, 1036, 989, 1013, 978, 1048, 994, 1058, 1073, 1004, 1014, 1067, 1038, 1037, 1007, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[986, 1003, 993, 1061, 1033, 999, 1027, 1007, 1032, 1027, 1031, 1009, 1031, 1006, 994, 1035, 1061, 970, 968, 1010, 1071, 1020, 1039, 1018, 1059, 991, 1083, 1048, 1042, 1041, 1028, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1045, 998, 996, 1024, 1033, 1009, 1042, 1020, 1043, 1078, 1031, 1031, 1067, 1020, 1005, 1025, 1037, 1010, 1036, 1005, 1029, 1018, 1010, 999, 986, 1007, 1041, 1002, 1012, 1032, 1021, 1056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1052, 1016, 1074, 1053, 9, 8, 0, 0, 1033, 1029, 1007, 1015, 20, 0, 0, 0, 1018, 1019, 1017, 1001, 0, 0, 0, 0, 1077, 990, 987, 1010, 0, 0, 0, 0, 1000, 1040, 989, 983, 5, 2, 0, 0, 1029, 1012, 982, 1016, 0, 0, 0, 0, 969, 1052, 1047, 1065, 0, 0, 0, 0, 1004, 1045, 1053, 1033, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[4212, 4104, 4055, 4064, 4019, 4039, 4133, 4142] avg=4096.0000 max=4212 max_rank=0 min=4019 min_rank=4 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1067, 974, 980, 974, 10, 7, 0, 0, 1017, 1012, 1046, 1033, 22, 0, 0, 0, 1010, 1030, 1013, 1054, 0, 0, 0, 0, 981, 1031, 1073, 1023, 0, 0, 0, 0, 1044, 1015, 972, 1109, 6, 2, 0, 0, 988, 991, 1075, 1069, 0, 0, 0, 0, 1039, 1020, 994, 993, 0, 0, 0, 0, 996, 1038, 1012, 1041, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4130, 4107, 4108, 4148, 4123, 4046, 4094] avg=4096.0000 max=4148 max_rank=4 min=4012 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1017, 992, 1072, 1013, 10, 7, 0, 0, 1019, 1031, 1011, 961, 22, 0, 0, 0, 1022, 1025, 1051, 987, 0, 0, 0, 0, 968, 1028, 1042, 1026, 0, 0, 0, 0, 1039, 1066, 970, 1041, 6, 2, 0, 0, 1034, 1023, 1068, 989, 0, 0, 0, 0, 1036, 1047, 981, 1023, 0, 0, 0, 0, 982, 1043, 1076, 1031, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4044, 4085, 4064, 4124, 4114, 4087, 4139] avg=4096.0000 max=4139 max_rank=7 min=4044 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[995, 1016, 1012, 992, 15, 12, 0, 0, 1038, 995, 1016, 1059, 24, 0, 0, 0, 1015, 1027, 1016, 1028, 0, 0, 0, 0, 1028, 1021, 1010, 1036, 0, 0, 0, 0, 1033, 1075, 1056, 1041, 9, 6, 0, 0, 1016, 1012, 1023, 1034, 0, 0, 0, 0, 1003, 986, 1053, 1039, 0, 0, 0, 0, 964, 1019, 1002, 1022, 11, 9, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4132, 4086, 4095, 4220, 4085, 4081, 4027] avg=4096.0000 max=4220 max_rank=4 min=4027 min_rank=7 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1024, 1056, 1021, 1037, 10, 7, 0, 0, 990, 1001, 984, 1066, 21, 0, 0, 0, 1058, 1015, 1026, 992, 0, 0, 0, 0, 1033, 1013, 1029, 1049, 0, 0, 0, 0, 1015, 1044, 1004, 1026, 6, 2, 0, 0, 1055, 1007, 1023, 1067, 0, 0, 0, 0, 1004, 1057, 1077, 1000, 0, 0, 0, 0, 994, 1028, 938, 982, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4062, 4091, 4124, 4097, 4152, 4138, 3949] avg=4096.0000 max=4155 max_rank=0 min=3949 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[995, 1021, 1037, 1027, 10, 7, 0, 0, 1001, 1020, 1055, 987, 22, 0, 0, 0, 1011, 1016, 1034, 1009, 0, 0, 0, 0, 1023, 1023, 1043, 1059, 0, 0, 0, 0, 1001, 1036, 989, 1013, 6, 2, 0, 0, 978, 1048, 985, 1058, 0, 0, 0, 0, 1041, 1004, 1014, 1067, 0, 0, 0, 0, 1038, 1037, 1007, 1037, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4085, 4070, 4148, 4047, 4069, 4126, 4126] avg=4096.0000 max=4148 max_rank=3 min=4047 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[986, 1003, 993, 1061, 10, 7, 0, 0, 1033, 999, 1027, 1007, 21, 0, 0, 0, 1032, 1027, 1027, 1009, 0, 0, 0, 0, 1022, 1006, 994, 1035, 0, 0, 0, 0, 1061, 970, 968, 1010, 6, 2, 0, 0, 1071, 1020, 1030, 1018, 0, 0, 0, 0, 1028, 991, 1083, 1048, 0, 0, 0, 0, 1042, 1041, 1028, 1045, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4087, 4095, 4057, 4017, 4139, 4150, 4163] avg=4096.0000 max=4163 max_rank=7 min=4017 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1045, 998, 996, 1024, 9, 7, 0, 0, 1033, 1009, 1042, 1020, 20, 0, 0, 0, 1043, 1078, 1027, 1031, 0, 0, 0, 0, 1058, 1020, 1005, 1025, 0, 0, 0, 0, 1037, 1010, 1036, 1005, 6, 2, 0, 0, 1029, 1018, 1001, 999, 0, 0, 0, 0, 957, 1007, 1041, 1002, 0, 0, 0, 0, 1012, 1032, 1021, 1056, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4124, 4179, 4108, 4096, 4047, 4007, 4128] avg=4096.0000 max=4179 max_rank=2 min=4007 min_rank=6 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1027, 979, 1078, 984, 1028, 996, 984, 1036, 1008, 1028, 1084, 1026, 1015, 1056, 1026, 1030, 1009, 1033, 1001, 1010, 1046, 1051, 1028, 1027, 1028, 1021, 1007, 1008, 1026, 1005, 1029, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1035, 1004, 994, 1005, 1051, 990, 1057, 1064, 1005, 1029, 1050, 1086, 1006, 1026, 1002, 1058, 1034, 1008, 999, 1065, 1069, 1006, 1004, 1075, 1031, 996, 1036, 1030, 1021, 976, 989, 967] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[996, 985, 1005, 962, 991, 1030, 1024, 1044, 1059, 1002, 1016, 1036, 994, 1024, 1031, 1032, 1065, 1012, 1037, 950, 1045, 996, 1028, 1083, 1050, 1056, 1046, 1013, 1076, 1036, 1001, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[962, 1018, 1041, 982, 948, 980, 1042, 989, 1047, 1041, 1027, 1048, 1076, 1081, 998, 1048, 1048, 1080, 1038, 1035, 997, 993, 1012, 1033, 1024, 1017, 1040, 1029, 1020, 1047, 1021, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1032, 957, 988, 984, 1008, 1049, 975, 1077, 1045, 1041, 1010, 1032, 1062, 1006, 1034, 1029, 1026, 994, 1034, 1051, 1032, 1028, 968, 1069, 1044, 1044, 996, 1053, 1063, 988, 1031, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1042, 1015, 1022, 1049, 1050, 1022, 1051, 1037, 993, 1032, 1050, 1022, 1011, 1049, 975, 1023, 1040, 1039, 1030, 1037, 1053, 1002, 985, 1048, 978, 1002, 1013, 1021, 1025, 1075, 1023, 954] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1054, 996, 1039, 1012, 987, 1065, 979, 1002, 1052, 1027, 1024, 1044, 1018, 1018, 995, 1021, 1024, 991, 1013, 1004, 1011, 1045, 1068, 1014, 1016, 1055, 991, 1065, 1015, 1022, 1059, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1012, 1030, 1021, 1007, 996, 1029, 1031, 994, 1030, 1070, 985, 1028, 1014, 1040, 1040, 1059, 1003, 1021, 1037, 1047, 1048, 1042, 1046, 1019, 1030, 1005, 1053, 1008, 1000, 1037, 1020, 966] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1027, 979, 1078, 984, 38, 17, 0, 0, 1028, 996, 984, 1036, 12, 7, 0, 0, 1008, 1028, 1084, 988, 0, 0, 0, 0, 1015, 1041, 1026, 1030, 0, 0, 0, 0, 1004, 1033, 1001, 1010, 0, 0, 0, 0, 1046, 1051, 1028, 1003, 0, 0, 0, 0, 1028, 1021, 1007, 1004, 0, 0, 0, 0, 1026, 1005, 1029, 1054, 5, 4, 3, 0] rerouted_tokens_per_ep_rank=[4123, 4063, 4108, 4112, 4048, 4128, 4060, 4126] avg=4096.0000 max=4128 max_rank=5 min=4048 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[996, 985, 1005, 962, 38, 18, 0, 0, 991, 1030, 1024, 1044, 12, 7, 0, 0, 1059, 1002, 1016, 998, 0, 0, 0, 0, 994, 1009, 1031, 1032, 0, 0, 0, 0, 1059, 1012, 1037, 950, 0, 0, 0, 0, 1045, 996, 1028, 1058, 0, 0, 0, 0, 1050, 1056, 1046, 1009, 0, 0, 0, 0, 1076, 1036, 1001, 1043, 6, 4, 3, 0] rerouted_tokens_per_ep_rank=[4004, 4108, 4075, 4066, 4058, 4127, 4161, 4169] avg=4096.0000 max=4169 max_rank=7 min=4004 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1035, 1004, 994, 1005, 40, 18, 0, 0, 1051, 990, 1057, 1064, 12, 7, 0, 0, 1005, 1029, 1050, 1046, 0, 0, 0, 0, 1006, 1011, 1002, 1058, 0, 0, 0, 0, 1029, 1008, 999, 1065, 0, 0, 0, 0, 1069, 1006, 1004, 1050, 0, 0, 0, 0, 1031, 996, 1036, 1026, 0, 0, 0, 0, 1021, 976, 989, 967, 5, 4, 3, 0] rerouted_tokens_per_ep_rank=[4096, 4181, 4130, 4077, 4101, 4129, 4089, 3965] avg=4096.0000 max=4181 max_rank=1 min=3965 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1032, 957, 988, 984, 38, 18, 0, 0, 1008, 1049, 975, 1077, 12, 7, 0, 0, 1045, 1041, 1010, 994, 0, 0, 0, 0, 1062, 991, 1034, 1029, 0, 0, 0, 0, 1021, 994, 1034, 1051, 0, 0, 0, 0, 1032, 1028, 968, 1044, 0, 0, 0, 0, 1044, 1044, 996, 1049, 0, 0, 0, 0, 1063, 988, 1031, 1018, 5, 4, 3, 0] rerouted_tokens_per_ep_rank=[4017, 4128, 4090, 4116, 4100, 4072, 4133, 4112] avg=4096.0000 max=4133 max_rank=6 min=4017 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1042, 1015, 1022, 1049, 37, 17, 0, 0, 1050, 1022, 1051, 1037, 12, 7, 0, 0, 993, 1032, 1050, 985, 0, 0, 0, 0, 1011, 1034, 975, 1023, 0, 0, 0, 0, 1035, 1039, 1030, 1037, 0, 0, 0, 0, 1053, 1002, 985, 1024, 0, 0, 0, 0, 978, 1002, 1013, 1017, 0, 0, 0, 0, 1025, 1075, 1023, 954, 5, 4, 3, 0] rerouted_tokens_per_ep_rank=[4182, 4179, 4060, 4043, 4141, 4064, 4010, 4089] avg=4096.0000 max=4182 max_rank=0 min=4010 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[962, 1018, 1041, 982, 42, 20, 0, 0, 948, 980, 1042, 989, 17, 12, 0, 0, 1047, 1041, 1027, 1006, 0, 0, 0, 0, 1076, 1057, 998, 1048, 0, 0, 0, 0, 1037, 1080, 1038, 1035, 0, 0, 0, 0, 997, 993, 1012, 1001, 0, 0, 0, 0, 1024, 1017, 1040, 1019, 0, 0, 0, 0, 1020, 1047, 1021, 1006, 11, 10, 7, 0] rerouted_tokens_per_ep_rank=[4065, 3988, 4121, 4179, 4190, 4003, 4100, 4122] avg=4096.0000 max=4190 max_rank=4 min=3988 min_rank=1 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1054, 996, 1039, 1012, 38, 17, 0, 0, 987, 1065, 979, 1002, 12, 7, 0, 0, 1052, 1027, 1024, 1006, 0, 0, 0, 0, 1018, 1003, 995, 1021, 0, 0, 0, 0, 1019, 991, 1013, 1004, 0, 0, 0, 0, 1011, 1045, 1068, 990, 0, 0, 0, 0, 1016, 1055, 991, 1061, 0, 0, 0, 0, 1015, 1022, 1059, 1042, 5, 4, 3, 0] rerouted_tokens_per_ep_rank=[4156, 4052, 4109, 4037, 4027, 4114, 4123, 4150] avg=4096.0000 max=4156 max_rank=0 min=4027 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1012, 1030, 1021, 1007, 38, 17, 0, 0, 996, 1029, 1031, 994, 12, 7, 0, 0, 1030, 1070, 985, 990, 0, 0, 0, 0, 1014, 1025, 1040, 1059, 0, 0, 0, 0, 998, 1021, 1037, 1047, 0, 0, 0, 0, 1048, 1042, 1046, 995, 0, 0, 0, 0, 1030, 1005, 1053, 1004, 0, 0, 0, 0, 1000, 1037, 1020, 966, 5, 4, 3, 0] rerouted_tokens_per_ep_rank=[4125, 4069, 4075, 4138, 4103, 4131, 4092, 4035] avg=4096.0000 max=4138 max_rank=3 min=4035 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8064, 384, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8064, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1078, 1043, 1058, 1009, 1059, 990, 1027, 1012, 995, 1062, 1014, 997, 1005, 1047, 1040, 1034, 1021, 987, 1031, 984, 1039, 1040, 1000, 994, 1002, 1032, 1051, 1032, 1026, 1003, 1031, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1021, 1023, 1033, 1048, 1006, 1031, 1033, 992, 946, 1020, 1020, 1074, 1067, 990, 1003, 1055, 1048, 1008, 1046, 1005, 998, 1055, 1002, 1000, 1074, 1036, 1005, 1000, 1072, 1025, 993, 1039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1026, 1003, 1011, 1017, 1052, 1041, 1074, 1023, 1003, 1075, 1029, 1030, 977, 1015, 1074, 1025, 999, 1027, 1032, 1004, 1004, 1048, 1038, 994, 1034, 987, 994, 1021, 1043, 1050, 1035, 983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1043, 1001, 1065, 1047, 1007, 1026, 1043, 1034, 1039, 980, 1047, 1009, 1031, 1027, 1036, 1014, 1017, 1046, 1028, 972, 1000, 1004, 997, 1013, 984, 1044, 1029, 1037, 1054, 1052, 1059, 983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1050, 1073, 1008, 1026, 998, 998, 1010, 1034, 1063, 1065, 1010, 1010, 1055, 1026, 983, 953, 1043, 1086, 1017, 979, 985, 1056, 1045, 984, 1018, 1047, 1031, 991, 1004, 1003, 1042, 1075] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[998, 1080, 1062, 1028, 1012, 1025, 1023, 1022, 1026, 1059, 1045, 980, 991, 1021, 1017, 1046, 1041, 1004, 1064, 1020, 1007, 1034, 1054, 1041, 976, 1008, 1008, 1015, 1015, 991, 1015, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[980, 1045, 1046, 967, 999, 1025, 1010, 1005, 1016, 1037, 1042, 999, 1015, 1028, 1019, 1036, 1005, 1007, 1072, 1006, 1044, 1023, 1022, 1061, 1087, 994, 1028, 1017, 1049, 995, 1035, 1054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1003, 1001, 993, 1028, 1043, 988, 1053, 973, 1040, 1021, 1061, 975, 1056, 1063, 1010, 1056, 1024, 1035, 1043, 1049, 1032, 1053, 1039, 1026, 1019, 1037, 981, 979, 1034, 1023, 990, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1021, 1023, 1015, 1048, 0, 0, 0, 0, 1006, 1031, 1033, 992, 11, 1, 0, 0, 946, 1018, 1020, 1074, 0, 0, 0, 0, 1067, 990, 1003, 1050, 0, 0, 0, 0, 1048, 1008, 1046, 1005, 2, 0, 0, 0, 998, 1055, 1002, 1000, 4, 0, 0, 0, 1074, 1036, 1005, 1000, 18, 3, 0, 0, 1058, 1025, 993, 1039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4074, 4058, 4110, 4109, 4059, 4136, 4115] avg=4096.0000 max=4136 max_rank=6 min=4058 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1078, 1043, 1040, 1009, 0, 0, 0, 0, 1059, 990, 1027, 1012, 10, 1, 0, 0, 995, 1060, 1014, 997, 0, 0, 0, 0, 1005, 1047, 1040, 1029, 0, 0, 0, 0, 1021, 987, 1031, 984, 2, 0, 0, 0, 1039, 1040, 1000, 994, 4, 0, 0, 0, 1002, 1032, 1051, 1032, 18, 2, 0, 0, 1014, 1003, 1031, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4099, 4066, 4121, 4025, 4077, 4137, 4073] avg=4096.0000 max=4170 max_rank=0 min=4025 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1050, 1073, 991, 1026, 0, 0, 0, 0, 998, 998, 1010, 1034, 10, 1, 0, 0, 1063, 1063, 1010, 1010, 0, 0, 0, 0, 1055, 1026, 983, 949, 0, 0, 0, 0, 1043, 1086, 1017, 979, 2, 0, 0, 0, 985, 1056, 1045, 984, 3, 0, 0, 0, 1018, 1047, 1031, 991, 17, 2, 0, 0, 992, 1003, 1042, 1075, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4051, 4146, 4013, 4127, 4073, 4106, 4112] avg=4096.0000 max=4146 max_rank=2 min=4013 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1026, 1003, 994, 1017, 0, 0, 0, 0, 1052, 1041, 1074, 1023, 10, 1, 0, 0, 1003, 1073, 1029, 1030, 0, 0, 0, 0, 977, 1015, 1074, 1020, 0, 0, 0, 0, 999, 1027, 1032, 1004, 2, 0, 0, 0, 1004, 1048, 1038, 994, 4, 0, 0, 0, 1034, 987, 994, 1021, 17, 3, 0, 0, 1030, 1050, 1035, 983, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4201, 4135, 4086, 4064, 4088, 4056, 4098] avg=4096.0000 max=4201 max_rank=1 min=4040 min_rank=0 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1043, 1001, 1042, 1047, 0, 0, 0, 0, 1007, 1026, 1043, 1034, 15, 7, 0, 0, 1039, 973, 1047, 1009, 0, 0, 0, 0, 1031, 1027, 1036, 1001, 0, 0, 0, 0, 1017, 1046, 1028, 972, 4, 0, 0, 0, 1000, 1004, 997, 1013, 6, 3, 0, 0, 984, 1044, 1029, 1037, 23, 7, 0, 0, 1032, 1052, 1059, 983, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4132, 4068, 4095, 4067, 4023, 4124, 4126] avg=4096.0000 max=4133 max_rank=0 min=4023 min_rank=5 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[998, 1080, 1044, 1028, 0, 0, 0, 0, 1012, 1025, 1023, 1022, 10, 1, 0, 0, 1026, 1057, 1045, 980, 0, 0, 0, 0, 991, 1021, 1017, 1041, 0, 0, 0, 0, 1041, 1004, 1064, 1020, 2, 0, 0, 0, 1007, 1034, 1054, 1041, 4, 0, 0, 0, 976, 1008, 1008, 1015, 18, 2, 0, 0, 1003, 991, 1015, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4093, 4108, 4070, 4131, 4140, 4027, 4049] avg=4096.0000 max=4150 max_rank=0 min=4027 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[980, 1045, 1028, 967, 0, 0, 0, 0, 999, 1025, 1010, 1005, 10, 1, 0, 0, 1016, 1035, 1042, 999, 0, 0, 0, 0, 1015, 1028, 1019, 1031, 0, 0, 0, 0, 1005, 1007, 1072, 1006, 2, 0, 0, 0, 1044, 1023, 1022, 1061, 4, 0, 0, 0, 1087, 994, 1028, 1017, 18, 3, 0, 0, 1036, 995, 1035, 1054, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4050, 4092, 4093, 4092, 4154, 4147, 4120] avg=4096.0000 max=4154 max_rank=5 min=4020 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1003, 1001, 976, 1028, 0, 0, 0, 0, 1043, 988, 1053, 973, 10, 1, 0, 0, 1040, 1019, 1061, 975, 0, 0, 0, 0, 1056, 1063, 1010, 1051, 0, 0, 0, 0, 1024, 1035, 1043, 1049, 2, 0, 0, 0, 1032, 1053, 1039, 1026, 4, 0, 0, 0, 1019, 1037, 981, 979, 17, 2, 0, 0, 1022, 1023, 990, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4008, 4068, 4095, 4180, 4153, 4154, 4035, 4075] avg=4096.0000 max=4180 max_rank=3 min=4008 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8320, 8448, 8064, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1035, 1039, 1043, 1038, 1018, 1015, 996, 1028, 1033, 1018, 990, 1020, 1030, 980, 1004, 1052, 1044, 1036, 1078, 1007, 1031, 1000, 1068, 1006, 985, 1005, 1072, 1024, 1016, 1014, 960, 1083] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1066, 1008, 1016, 1036, 1016, 1000, 1024, 991, 1022, 1013, 1049, 1000, 985, 1065, 1075, 1007, 1057, 974, 1035, 1003, 1012, 1018, 1062, 1003, 1053, 1031, 1056, 1056, 1018, 1001, 973, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[957, 1034, 1054, 1064, 996, 1036, 1013, 1033, 991, 1050, 1053, 1039, 1044, 1014, 1007, 1035, 1003, 1044, 993, 996, 1029, 1031, 1004, 1027, 1026, 1006, 1025, 1004, 1041, 1026, 1058, 1035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1017, 971, 972, 1066, 1022, 1053, 999, 1008, 1023, 1040, 1025, 996, 1044, 1018, 1009, 1049, 1047, 1006, 1061, 1044, 955, 1012, 1010, 1020, 1042, 962, 1041, 1056, 1062, 1043, 1043, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1009, 995, 1047, 987, 1051, 1063, 1001, 1008, 1017, 1020, 1029, 962, 975, 999, 1036, 1024, 1005, 1000, 1054, 1032, 1036, 1070, 1033, 1022, 1044, 1028, 1033, 1040, 1031, 1051, 1037, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[989, 1053, 1013, 1026, 1037, 1043, 1052, 1014, 1022, 1066, 1012, 1011, 1064, 1075, 1047, 996, 1070, 983, 1017, 1020, 1009, 1067, 1040, 970, 995, 1010, 1046, 1017, 1032, 973, 997, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[992, 1004, 1048, 1006, 1025, 1030, 1006, 1012, 1055, 1016, 1074, 1034, 1071, 1040, 993, 1002, 1013, 1003, 979, 1037, 1052, 1011, 1039, 1017, 1014, 1042, 1071, 994, 1032, 1010, 1014, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1013, 965, 983, 1007, 1073, 1042, 1061, 1004, 1014, 983, 1061, 1106, 977, 1001, 1049, 1037, 1025, 1026, 1002, 983, 1011, 1091, 1030, 1064, 1038, 1007, 1016, 1060, 1024, 999, 1002, 1014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1066, 1008, 1016, 1036, 16, 10, 0, 0, 1016, 1000, 1024, 991, 0, 0, 0, 0, 1022, 1013, 1040, 1000, 0, 0, 0, 0, 985, 1065, 1071, 1007, 0, 0, 0, 0, 1057, 974, 1035, 1003, 9, 2, 0, 0, 1012, 1008, 1062, 1003, 0, 0, 0, 0, 1053, 1031, 1040, 1056, 0, 0, 0, 0, 1018, 1001, 973, 1043, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4031, 4075, 4128, 4080, 4085, 4180, 4037] avg=4096.0000 max=4180 max_rank=6 min=4031 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1035, 1039, 1043, 1038, 16, 9, 0, 0, 1018, 1015, 996, 1028, 0, 0, 0, 0, 1033, 1018, 982, 1020, 0, 0, 0, 0, 1030, 980, 1000, 1052, 0, 0, 0, 0, 1044, 1036, 1078, 1007, 8, 2, 0, 0, 1031, 991, 1068, 1006, 0, 0, 0, 0, 985, 1005, 1056, 1024, 0, 0, 0, 0, 1016, 1014, 960, 1083, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4057, 4053, 4062, 4175, 4096, 4070, 4075] avg=4096.0000 max=4180 max_rank=0 min=4053 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1017, 971, 972, 1066, 16, 9, 0, 0, 1022, 1053, 999, 1008, 0, 0, 0, 0, 1023, 1040, 1016, 996, 0, 0, 0, 0, 1044, 1018, 1005, 1049, 0, 0, 0, 0, 1047, 1006, 1061, 1044, 9, 2, 0, 0, 955, 1003, 1010, 1020, 0, 0, 0, 0, 1042, 962, 1025, 1056, 0, 0, 0, 0, 1062, 1043, 1043, 1052, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4082, 4075, 4116, 4169, 3988, 4085, 4202] avg=4096.0000 max=4202 max_rank=7 min=3988 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[989, 1053, 1013, 1026, 16, 10, 0, 0, 1037, 1043, 1052, 1014, 0, 0, 0, 0, 1022, 1066, 1003, 1011, 0, 0, 0, 0, 1064, 1075, 1043, 996, 0, 0, 0, 0, 1070, 983, 1017, 1020, 9, 2, 0, 0, 1009, 1057, 1040, 970, 0, 0, 0, 0, 995, 1010, 1030, 1017, 0, 0, 0, 0, 1032, 973, 997, 1002, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4146, 4102, 4178, 4101, 4076, 4052, 4006] avg=4096.0000 max=4178 max_rank=3 min=4006 min_rank=7 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16235 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[957, 1034, 1054, 1064, 16, 10, 0, 0, 996, 1036, 1013, 1033, 0, 0, 0, 0, 991, 1050, 1044, 1039, 0, 0, 0, 0, 1044, 1014, 1003, 1035, 0, 0, 0, 0, 1003, 1044, 993, 996, 9, 2, 0, 0, 1029, 1021, 1004, 1027, 0, 0, 0, 0, 1026, 1006, 1009, 1004, 0, 0, 0, 0, 1041, 1026, 1058, 1035, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4078, 4124, 4096, 4047, 4081, 4045, 4162] avg=4096.0000 max=4162 max_rank=7 min=4045 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1009, 995, 1047, 987, 20, 15, 2, 0, 1051, 1061, 1001, 1008, 0, 0, 0, 0, 1017, 1020, 1015, 962, 0, 0, 0, 0, 975, 999, 1028, 1024, 0, 0, 0, 0, 1005, 1000, 1054, 1032, 12, 3, 0, 0, 1036, 1055, 1033, 1022, 0, 0, 0, 0, 1044, 1028, 1013, 1040, 0, 0, 0, 0, 1031, 1051, 1037, 1029, 5, 2, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4121, 4014, 4026, 4106, 4146, 4125, 4155] avg=4096.0000 max=4155 max_rank=7 min=4014 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16532 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[992, 1004, 1048, 1006, 16, 9, 0, 0, 1025, 1030, 1006, 1012, 0, 0, 0, 0, 1055, 1016, 1065, 1034, 0, 0, 0, 0, 1071, 1040, 989, 1002, 0, 0, 0, 0, 1013, 1003, 979, 1037, 9, 2, 0, 0, 1052, 1002, 1039, 1017, 0, 0, 0, 0, 1014, 1042, 1055, 994, 0, 0, 0, 0, 1032, 1010, 1014, 1032, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4073, 4170, 4102, 4043, 4110, 4105, 4090] avg=4096.0000 max=4170 max_rank=2 min=4043 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1013, 965, 983, 1007, 15, 10, 0, 0, 1073, 1042, 1061, 1004, 0, 0, 0, 0, 1014, 983, 1052, 1106, 0, 0, 0, 0, 977, 1001, 1045, 1037, 0, 0, 0, 0, 1025, 1026, 1002, 983, 9, 2, 0, 0, 1011, 1081, 1030, 1064, 0, 0, 0, 0, 1038, 1007, 1001, 1060, 0, 0, 0, 0, 1024, 999, 1002, 1014, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[3993, 4180, 4155, 4060, 4047, 4186, 4106, 4041] avg=4096.0000 max=4186 max_rank=5 min=3993 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[982, 1052, 993, 1044, 989, 994, 1038, 1074, 1016, 1028, 1040, 1005, 1000, 1030, 1037, 1042, 1004, 1036, 1045, 970, 1076, 1023, 1063, 1052, 1037, 986, 991, 1028, 1004, 1074, 993, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1068, 1019, 970, 1006, 1044, 1056, 993, 1100, 1037, 1008, 1004, 1001, 1003, 1042, 988, 1066, 1031, 997, 1006, 979, 996, 1030, 1079, 1067, 1025, 1020, 1048, 1021, 1012, 1027, 1013, 1012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1048, 1045, 980, 1051, 1024, 1027, 1039, 1037, 989, 1021, 1010, 1053, 1042, 1017, 1048, 1024, 1016, 998, 1033, 1022, 1008, 1011, 1020, 1028, 1066, 1051, 1012, 991, 977, 1042, 1017, 1021] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1027, 1014, 994, 1023, 1034, 989, 1032, 1010, 1083, 1041, 994, 1034, 1002, 1065, 1030, 1020, 1080, 1035, 979, 1023, 1018, 1004, 1030, 977, 1021, 1050, 960, 1030, 1045, 1020, 1030, 1074] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1038, 998, 1011, 1050, 984, 1021, 1053, 1039, 1047, 1027, 1049, 1039, 998, 1050, 1033, 1051, 1058, 1031, 1018, 1013, 1012, 987, 1000, 974, 1035, 1012, 1030, 1034, 975, 1054, 1015, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[983, 1031, 1053, 1011, 1015, 1041, 1011, 1070, 1032, 1061, 996, 1036, 1047, 1030, 1005, 1031, 944, 1004, 1031, 1010, 1034, 1009, 1041, 1102, 995, 1033, 988, 1028, 1023, 1045, 1012, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1038, 1034, 970, 1021, 996, 1024, 1061, 1017, 1010, 960, 1003, 1027, 1030, 1025, 1026, 1023, 1025, 1029, 1023, 1012, 1043, 1018, 1011, 1065, 1030, 1036, 1011, 1024, 1063, 1041, 1013, 1059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1032, 1032, 985, 994, 1024, 957, 992, 1051, 1010, 1008, 1032, 1002, 977, 1034, 1015, 1072, 978, 1045, 1061, 1026, 1032, 972, 1065, 990, 1059, 1074, 1070, 1012, 1061, 1006, 1057, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[982, 1052, 993, 1044, 8, 7, 5, 0, 989, 994, 1038, 1066, 0, 0, 0, 0, 1016, 1028, 1040, 1005, 5, 3, 0, 0, 1000, 1030, 1037, 1026, 0, 0, 0, 0, 1004, 1036, 1045, 970, 16, 9, 0, 0, 1076, 1023, 1055, 1052, 0, 0, 0, 0, 1032, 986, 991, 1028, 0, 0, 0, 0, 1004, 1058, 993, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4087, 4097, 4093, 4080, 4206, 4037, 4077] avg=4096.0000 max=4206 max_rank=5 min=4037 min_rank=6 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1038, 998, 1011, 1050, 8, 7, 5, 0, 984, 1021, 1053, 1031, 0, 0, 0, 0, 1047, 1027, 1049, 1039, 5, 3, 0, 0, 998, 1050, 1033, 1034, 0, 0, 0, 0, 1058, 1031, 1018, 1013, 17, 9, 0, 0, 1012, 987, 992, 974, 0, 0, 0, 0, 1030, 1012, 1030, 1034, 0, 0, 0, 0, 975, 1038, 1015, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4089, 4170, 4115, 4146, 3965, 4106, 4060] avg=4096.0000 max=4170 max_rank=2 min=3965 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[983, 1031, 1053, 1011, 8, 7, 5, 0, 1015, 1041, 1011, 1062, 0, 0, 0, 0, 1032, 1061, 996, 1036, 4, 3, 0, 0, 1047, 1030, 1005, 1015, 0, 0, 0, 0, 944, 1004, 1031, 1010, 16, 8, 0, 0, 1034, 1009, 1033, 1102, 0, 0, 0, 0, 991, 1033, 988, 1028, 0, 0, 0, 0, 1023, 1030, 1012, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4129, 4132, 4097, 4013, 4178, 4040, 4081] avg=4096.0000 max=4178 max_rank=5 min=4013 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1048, 1045, 980, 1051, 8, 7, 5, 0, 1024, 1027, 1039, 1029, 0, 0, 0, 0, 989, 1021, 1010, 1053, 5, 3, 0, 0, 1042, 1017, 1048, 1008, 0, 0, 0, 0, 1016, 998, 1033, 1022, 16, 8, 0, 0, 1008, 1011, 1012, 1028, 0, 0, 0, 0, 1061, 1051, 1012, 991, 0, 0, 0, 0, 977, 1027, 1017, 1021, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4119, 4081, 4115, 4093, 4059, 4115, 4042] avg=4096.0000 max=4144 max_rank=0 min=4042 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1068, 1019, 970, 1006, 13, 10, 8, 0, 1044, 1056, 993, 1088, 0, 0, 0, 0, 1037, 1008, 1004, 1001, 8, 4, 0, 0, 1003, 1042, 988, 1045, 0, 0, 0, 0, 1031, 997, 1006, 979, 21, 13, 0, 0, 996, 1030, 1066, 1067, 0, 0, 0, 0, 1017, 1020, 1048, 1021, 0, 0, 0, 0, 1012, 1004, 1013, 1012, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4181, 4062, 4078, 4047, 4159, 4106, 4041] avg=4096.0000 max=4181 max_rank=1 min=4041 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1027, 1014, 994, 1023, 8, 7, 5, 0, 1034, 989, 1032, 1002, 0, 0, 0, 0, 1083, 1041, 994, 1034, 4, 3, 0, 0, 1002, 1065, 1030, 1004, 0, 0, 0, 0, 1080, 1035, 979, 1023, 16, 8, 0, 0, 1018, 1004, 1022, 977, 0, 0, 0, 0, 1017, 1050, 960, 1030, 0, 0, 0, 0, 1045, 1005, 1030, 1074, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4057, 4159, 4101, 4141, 4021, 4057, 4154] avg=4096.0000 max=4159 max_rank=2 min=4021 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1038, 1034, 970, 1021, 8, 7, 5, 0, 996, 1024, 1061, 1009, 0, 0, 0, 0, 1010, 960, 1003, 1027, 4, 3, 0, 0, 1030, 1025, 1026, 1007, 0, 0, 0, 0, 1025, 1029, 1023, 1012, 16, 8, 0, 0, 1043, 1018, 1003, 1065, 0, 0, 0, 0, 1026, 1036, 1011, 1024, 0, 0, 0, 0, 1063, 1026, 1013, 1059, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4090, 4007, 4088, 4113, 4129, 4097, 4161] avg=4096.0000 max=4161 max_rank=7 min=4007 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1032, 1032, 985, 994, 8, 7, 5, 0, 1024, 957, 992, 1043, 0, 0, 0, 0, 1010, 1008, 1032, 1002, 5, 3, 0, 0, 977, 1034, 1015, 1055, 0, 0, 0, 0, 978, 1045, 1061, 1026, 17, 8, 0, 0, 1032, 972, 1057, 990, 0, 0, 0, 0, 1054, 1074, 1070, 1012, 0, 0, 0, 0, 1061, 991, 1057, 1043, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4016, 4060, 4081, 4135, 4051, 4210, 4152] avg=4096.0000 max=4210 max_rank=6 min=4016 min_rank=1 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1058, 1053, 1067, 1004, 1036, 978, 1025, 1030, 999, 1012, 1008, 1039, 1056, 1034, 1033, 1013, 1046, 1003, 1057, 1012, 1045, 1007, 1016, 994, 985, 1048, 1003, 1020, 983, 977, 1042, 1085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[990, 1056, 1044, 1020, 1007, 1022, 1027, 1030, 1036, 1014, 996, 1021, 999, 1034, 1060, 1040, 1010, 1027, 1003, 1021, 1029, 1020, 1046, 1032, 1020, 1039, 1075, 1017, 1032, 989, 1022, 990] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1065, 1033, 1069, 999, 1010, 1041, 1021, 997, 1000, 1046, 1037, 1032, 1007, 998, 1052, 1032, 1020, 1066, 1052, 1033, 997, 1013, 1006, 1046, 1012, 1029, 1026, 999, 978, 1021, 1035, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1000, 1065, 1034, 1013, 1054, 1011, 1020, 1011, 1020, 1050, 1031, 1000, 988, 1011, 1024, 1090, 1057, 1054, 1039, 968, 1095, 1003, 997, 1030, 1018, 1013, 1007, 1034, 978, 1019, 1015, 1019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1042, 1035, 997, 1030, 952, 1037, 1100, 1024, 999, 1057, 989, 1060, 1009, 976, 1009, 1008, 1033, 1044, 987, 1061, 975, 1046, 1034, 1017, 1005, 1036, 1038, 1011, 1035, 1024, 1024, 1074] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1040, 1047, 1027, 1042, 1082, 1023, 1073, 1010, 1015, 1022, 1068, 1009, 997, 1024, 987, 986, 1049, 1038, 1020, 1016, 1003, 1008, 1024, 1003, 1006, 1040, 1074, 1008, 1000, 1023, 1004, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1012, 1054, 974, 1017, 1021, 1030, 1044, 1014, 1047, 999, 1025, 1044, 1022, 995, 1038, 1026, 1051, 1015, 1075, 992, 1020, 978, 1043, 1001, 1023, 1034, 983, 999, 1073, 1045, 1042, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1024, 935, 1033, 1019, 1029, 1055, 999, 1038, 1054, 1009, 1019, 999, 1047, 1067, 1020, 1018, 1048, 1014, 1037, 1053, 1048, 1072, 1007, 1011, 1052, 1004, 1014, 1031, 981, 991, 996, 1044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[990, 1041, 1044, 1020, 0, 0, 0, 0, 1007, 1022, 1018, 1030, 0, 0, 0, 0, 1036, 1014, 996, 1021, 1, 0, 0, 0, 999, 1034, 1060, 1040, 7, 0, 0, 0, 982, 1027, 1003, 1021, 0, 0, 0, 0, 1029, 1020, 1046, 1032, 8, 4, 0, 0, 1020, 1039, 1075, 1017, 8, 0, 0, 0, 1032, 989, 1022, 990, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4077, 4068, 4140, 4033, 4139, 4159, 4057] avg=4096.0000 max=4159 max_rank=6 min=4033 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1058, 1038, 1067, 1004, 0, 0, 0, 0, 1036, 978, 1016, 1030, 0, 0, 0, 0, 999, 1012, 1008, 1039, 1, 0, 0, 0, 1056, 1034, 1033, 1013, 7, 0, 0, 0, 1017, 1003, 1057, 1012, 0, 0, 0, 0, 1045, 1007, 1016, 994, 8, 4, 0, 0, 985, 1048, 1003, 1020, 8, 0, 0, 0, 983, 977, 1042, 1085, 25, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4060, 4059, 4143, 4089, 4074, 4064, 4112] avg=4096.0000 max=4167 max_rank=0 min=4059 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1065, 1018, 1069, 999, 0, 0, 0, 0, 1010, 1041, 1013, 997, 0, 0, 0, 0, 1000, 1046, 1037, 1032, 1, 0, 0, 0, 1007, 998, 1052, 1032, 7, 0, 0, 0, 992, 1066, 1052, 1033, 0, 0, 0, 0, 997, 1013, 1006, 1046, 8, 4, 0, 0, 1012, 1029, 1026, 999, 7, 0, 0, 0, 978, 1021, 1035, 996, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4061, 4116, 4096, 4143, 4074, 4073, 4054] avg=4096.0000 max=4151 max_rank=0 min=4054 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1000, 1050, 1034, 1013, 0, 0, 0, 0, 1054, 1011, 1012, 1011, 0, 0, 0, 0, 1020, 1050, 1031, 1000, 1, 0, 0, 0, 988, 1011, 1024, 1090, 7, 0, 0, 0, 1028, 1054, 1039, 968, 0, 0, 0, 0, 1095, 1003, 997, 1030, 8, 4, 0, 0, 1018, 1013, 1007, 1034, 7, 0, 0, 0, 978, 1019, 1015, 1019, 25, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4088, 4102, 4120, 4089, 4137, 4079, 4056] avg=4096.0000 max=4137 max_rank=5 min=4056 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1012, 1028, 974, 1017, 0, 0, 0, 0, 1021, 1030, 1021, 1014, 0, 0, 0, 0, 1047, 999, 1025, 1044, 5, 0, 0, 0, 1022, 995, 1038, 1026, 13, 6, 0, 0, 1018, 1015, 1075, 992, 0, 0, 0, 0, 1020, 978, 1043, 1001, 13, 6, 0, 0, 1023, 1034, 983, 999, 12, 0, 0, 0, 1073, 1045, 1042, 1032, 27, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4086, 4120, 4100, 4100, 4061, 4051, 4219] avg=4096.0000 max=4219 max_rank=7 min=4031 min_rank=0 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1042, 1020, 997, 1030, 0, 0, 0, 0, 952, 1037, 1091, 1024, 0, 0, 0, 0, 999, 1057, 989, 1060, 1, 0, 0, 0, 1009, 976, 1009, 1008, 7, 0, 0, 0, 1005, 1044, 987, 1061, 0, 0, 0, 0, 975, 1046, 1034, 1017, 8, 4, 0, 0, 1005, 1036, 1038, 1011, 8, 0, 0, 0, 1035, 1024, 1024, 1074, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4104, 4106, 4009, 4097, 4084, 4098, 4181] avg=4096.0000 max=4181 max_rank=7 min=4009 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1040, 1032, 1027, 1042, 0, 0, 0, 0, 1082, 1023, 1064, 1010, 0, 0, 0, 0, 1015, 1022, 1068, 1009, 1, 0, 0, 0, 997, 1024, 987, 986, 7, 0, 0, 0, 1020, 1038, 1020, 1016, 0, 0, 0, 0, 1003, 1008, 1024, 1003, 8, 4, 0, 0, 1006, 1040, 1074, 1008, 8, 0, 0, 0, 1000, 1023, 1004, 1000, 25, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4179, 4115, 4001, 4094, 4050, 4136, 4052] avg=4096.0000 max=4179 max_rank=1 min=4001 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 921, 1033, 1019, 0, 0, 0, 0, 1029, 1055, 991, 1038, 0, 0, 0, 0, 1054, 1009, 1019, 999, 1, 0, 0, 0, 1047, 1067, 1020, 1018, 7, 0, 0, 0, 1019, 1014, 1037, 1053, 0, 0, 0, 0, 1048, 1072, 1007, 1011, 7, 4, 0, 0, 1052, 1004, 1014, 1031, 7, 0, 0, 0, 981, 991, 996, 1044, 25, 0, 0, 0] rerouted_tokens_per_ep_rank=[3997, 4113, 4082, 4159, 4123, 4149, 4108, 4037] avg=4096.0000 max=4159 max_rank=3 min=3997 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1056, 1003, 1062, 986, 1072, 1053, 1027, 1015, 1021, 976, 1020, 1040, 1005, 1037, 1045, 1073, 1074, 989, 1069, 1032, 1023, 997, 989, 1016, 1029, 977, 1055, 995, 1006, 986, 1022, 1018] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1052, 1075, 1006, 1035, 1029, 1054, 1028, 986, 1006, 1068, 988, 1014, 987, 1031, 977, 1076, 1022, 1022, 1002, 1026, 995, 1045, 1059, 1036, 1020, 991, 1012, 1025, 987, 1029, 1054, 1031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1023, 997, 1009, 951, 1008, 1076, 1034, 1037, 1010, 1082, 1042, 1063, 1038, 1003, 1029, 1013, 995, 1018, 1000, 990, 1091, 1056, 1052, 994, 1025, 1011, 1068, 1046, 1013, 1009, 1010, 975] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1019, 1024, 1053, 989, 1031, 995, 996, 1009, 1083, 1020, 1033, 1025, 1009, 1003, 1062, 1075, 1043, 1027, 1048, 1024, 1022, 1004, 1008, 1014, 1005, 1086, 1014, 1017, 983, 989, 1042, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[989, 1004, 994, 1054, 1024, 1034, 1081, 1037, 1070, 989, 1024, 1057, 1020, 1005, 1048, 1005, 1047, 997, 983, 1053, 1012, 989, 1025, 1022, 977, 1015, 1027, 1027, 1007, 1069, 1031, 1052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[963, 1036, 990, 1039, 1042, 1055, 1013, 1005, 1019, 1003, 1056, 1020, 1040, 1000, 956, 1023, 976, 1058, 1038, 1008, 1002, 984, 1002, 1061, 1052, 1018, 1028, 1039, 1045, 1095, 1047, 1055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1023, 1006, 999, 1004, 1018, 1024, 1045, 1038, 1024, 992, 1025, 1070, 997, 981, 1025, 1034, 1001, 1014, 1011, 1039, 1045, 1008, 989, 1098, 1035, 1106, 1020, 1095, 938, 1047, 1017, 1000] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1056, 1066, 1042, 1026, 1069, 1019, 998, 1028, 1036, 1083, 1023, 1025, 1009, 964, 1016, 1061, 1024, 1005, 999, 1049, 1014, 987, 1019, 1006, 1048, 978, 1017, 1037, 1003, 995, 1064, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1052, 1075, 1006, 1035, 16, 0, 0, 0, 1029, 1029, 1028, 986, 0, 0, 0, 0, 1006, 1068, 988, 986, 0, 0, 0, 0, 987, 1031, 977, 1076, 15, 0, 0, 0, 1022, 1022, 1002, 1026, 10, 0, 0, 0, 995, 1045, 1059, 1036, 7, 5, 0, 0, 1020, 991, 1012, 1010, 0, 0, 0, 0, 987, 1029, 1054, 1031, 12, 3, 0, 0] rerouted_tokens_per_ep_rank=[4184, 4072, 4048, 4086, 4082, 4147, 4033, 4116] avg=4096.0000 max=4184 max_rank=0 min=4033 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1023, 997, 1009, 951, 17, 0, 0, 0, 1008, 1050, 1034, 1037, 0, 0, 0, 0, 1010, 1082, 1042, 1033, 0, 0, 0, 0, 1038, 1003, 1029, 1013, 15, 0, 0, 0, 995, 1018, 1000, 990, 10, 0, 0, 0, 1091, 1056, 1052, 994, 8, 5, 0, 0, 1025, 1011, 1068, 1031, 0, 0, 0, 0, 1013, 1009, 1010, 975, 13, 3, 0, 0] rerouted_tokens_per_ep_rank=[3997, 4129, 4167, 4098, 4013, 4206, 4135, 4023] avg=4096.0000 max=4206 max_rank=5 min=3997 min_rank=0 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1056, 1003, 1062, 986, 17, 0, 0, 0, 1072, 1028, 1027, 1015, 0, 0, 0, 0, 1021, 976, 1020, 1011, 0, 0, 0, 0, 1005, 1037, 1045, 1073, 15, 0, 0, 0, 1074, 989, 1069, 1032, 10, 0, 0, 0, 1023, 997, 989, 1016, 7, 5, 0, 0, 1029, 977, 1055, 980, 0, 0, 0, 0, 1006, 986, 1022, 1018, 12, 3, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4142, 4028, 4175, 4174, 4037, 4041, 4047] avg=4096.0000 max=4175 max_rank=3 min=4028 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1019, 1024, 1053, 989, 16, 0, 0, 0, 1031, 971, 996, 1009, 0, 0, 0, 0, 1083, 1020, 1033, 997, 0, 0, 0, 0, 1009, 1003, 1062, 1075, 14, 0, 0, 0, 1043, 1027, 1048, 1024, 10, 0, 0, 0, 1022, 1004, 1008, 1014, 7, 5, 0, 0, 1005, 1086, 1014, 1002, 0, 0, 0, 0, 983, 989, 1042, 1016, 12, 3, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4007, 4133, 4163, 4152, 4060, 4107, 4045] avg=4096.0000 max=4163 max_rank=3 min=4007 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[989, 1004, 994, 1054, 22, 0, 0, 0, 1024, 995, 1081, 1037, 0, 0, 0, 0, 1070, 989, 1024, 1019, 0, 0, 0, 0, 1020, 1005, 1048, 1005, 19, 0, 0, 0, 1047, 997, 983, 1053, 14, 0, 0, 0, 1012, 989, 1025, 1022, 12, 7, 0, 0, 977, 1015, 1027, 1006, 0, 0, 0, 0, 1007, 1069, 1031, 1052, 16, 8, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4137, 4102, 4097, 4094, 4067, 4025, 4183] avg=4096.0000 max=4183 max_rank=7 min=4025 min_rank=6 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[963, 1036, 990, 1039, 16, 0, 0, 0, 1042, 1030, 1013, 1005, 0, 0, 0, 0, 1019, 1003, 1056, 992, 0, 0, 0, 0, 1040, 1000, 956, 1023, 15, 0, 0, 0, 976, 1058, 1038, 1008, 10, 0, 0, 0, 1002, 984, 1002, 1061, 7, 5, 0, 0, 1052, 1018, 1028, 1024, 0, 0, 0, 0, 1045, 1095, 1047, 1055, 12, 3, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4090, 4070, 4034, 4090, 4061, 4122, 4257] avg=4096.0000 max=4257 max_rank=7 min=4034 min_rank=3 imbalance_ratio=1.039307 max_load_reduction=28511 max_load_reduction_pct=0.870087 moved_to_echo=16530 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1056, 1066, 1042, 1026, 16, 0, 0, 0, 1069, 995, 998, 1028, 0, 0, 0, 0, 1036, 1083, 1023, 997, 0, 0, 0, 0, 1009, 964, 1016, 1061, 14, 0, 0, 0, 1024, 1005, 999, 1049, 10, 0, 0, 0, 1014, 987, 1019, 1006, 7, 5, 0, 0, 1048, 978, 1017, 1022, 0, 0, 0, 0, 1003, 995, 1064, 1002, 12, 3, 0, 0] rerouted_tokens_per_ep_rank=[4206, 4090, 4139, 4064, 4087, 4038, 4065, 4079] avg=4096.0000 max=4206 max_rank=0 min=4038 min_rank=5 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16269 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1023, 1006, 999, 1004, 17, 0, 0, 0, 1018, 1000, 1045, 1038, 0, 0, 0, 0, 1024, 992, 1025, 1040, 0, 0, 0, 0, 997, 981, 1025, 1034, 14, 0, 0, 0, 1001, 1014, 1011, 1039, 11, 0, 0, 0, 1045, 1008, 989, 1098, 7, 5, 0, 0, 1035, 1106, 1020, 1079, 0, 0, 0, 0, 938, 1047, 1017, 1000, 13, 3, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4101, 4081, 4051, 4076, 4152, 4240, 4018] avg=4096.0000 max=4240 max_rank=6 min=4018 min_rank=7 imbalance_ratio=1.035156 max_load_reduction=28528 max_load_reduction_pct=0.870605 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8064, 8192, 8448, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1040, 1011, 1015, 1066, 1036, 1046, 1067, 1011, 1014, 1053, 1008, 1020, 1031, 999, 1030, 1027, 1015, 1019, 977, 1034, 1015, 990, 1015, 1018, 990, 1000, 1060, 1017, 1008, 1057, 1030, 1049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1026, 1036, 1033, 975, 1066, 1048, 1032, 1025, 1070, 980, 1038, 1055, 1003, 989, 1041, 1034, 1015, 998, 1049, 1005, 1078, 1016, 991, 1058, 1037, 1024, 1021, 1030, 997, 985, 1033, 980] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1051, 1027, 1025, 1033, 998, 1048, 1045, 1034, 998, 1017, 1054, 1009, 1011, 1039, 1031, 1011, 993, 998, 1013, 1017, 993, 1016, 1011, 1023, 988, 1000, 1050, 1019, 1069, 1087, 1015, 1045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1038, 1023, 1016, 1068, 1012, 984, 1005, 991, 1043, 986, 1002, 998, 1029, 1051, 1019, 1029, 1038, 1082, 1055, 1040, 1015, 1014, 1040, 1011, 981, 978, 1053, 1036, 1029, 1018, 1034, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[998, 981, 1059, 1033, 1031, 1017, 1014, 1047, 1060, 1062, 1017, 1033, 1011, 1048, 1048, 1043, 973, 1048, 1083, 1074, 991, 989, 1024, 1059, 1015, 1030, 1013, 983, 1006, 982, 1019, 977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1053, 982, 1017, 986, 968, 1024, 1010, 1025, 978, 1049, 998, 1034, 1039, 1033, 1008, 1002, 1048, 1008, 1070, 1028, 1001, 1078, 1037, 1016, 1043, 1021, 1018, 1061, 1071, 1030, 1010, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1020, 1043, 1038, 983, 1025, 1036, 998, 1047, 1075, 999, 1031, 1023, 1100, 1066, 1062, 1034, 1007, 994, 1040, 991, 1048, 1037, 1035, 1004, 996, 966, 977, 1025, 970, 1019, 1046, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1005, 1008, 1024, 1007, 1026, 1072, 1028, 1007, 1014, 1012, 1076, 1006, 1081, 1014, 1041, 1045, 1021, 1033, 999, 1042, 1053, 1004, 1017, 985, 1038, 981, 1044, 1041, 1031, 960, 1021, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1040, 1011, 1015, 1066, 4, 1, 0, 0, 1036, 1040, 1067, 1011, 0, 0, 0, 0, 1009, 1053, 1008, 1020, 0, 0, 0, 0, 997, 999, 1030, 1027, 0, 0, 0, 0, 1015, 1019, 973, 1034, 0, 0, 0, 0, 1015, 990, 1015, 1018, 6, 4, 0, 0, 990, 1000, 1060, 1017, 28, 0, 0, 0, 1008, 1057, 1030, 1049, 4, 2, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4154, 4090, 4053, 4041, 4048, 4095, 4150] avg=4096.0000 max=4154 max_rank=1 min=4041 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1051, 1027, 1025, 1033, 4, 1, 0, 0, 998, 1042, 1045, 1034, 0, 0, 0, 0, 993, 1017, 1054, 1009, 0, 0, 0, 0, 978, 1039, 1031, 1011, 0, 0, 0, 0, 993, 998, 1009, 1017, 0, 0, 0, 0, 993, 1016, 1011, 1023, 5, 4, 0, 0, 988, 1000, 1050, 1019, 28, 0, 0, 0, 1069, 1087, 1015, 1045, 4, 2, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4119, 4073, 4059, 4017, 4052, 4085, 4222] avg=4096.0000 max=4222 max_rank=7 min=4017 min_rank=4 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1026, 1036, 1033, 975, 4, 1, 0, 0, 1066, 1042, 1032, 1025, 0, 0, 0, 0, 1065, 980, 1038, 1055, 0, 0, 0, 0, 970, 989, 1041, 1034, 0, 0, 0, 0, 1015, 998, 1045, 1005, 0, 0, 0, 0, 1078, 1016, 991, 1058, 5, 4, 0, 0, 1037, 1024, 1021, 1030, 28, 0, 0, 0, 997, 985, 1033, 980, 4, 2, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4165, 4138, 4034, 4063, 4152, 4140, 4001] avg=4096.0000 max=4165 max_rank=1 min=4001 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1038, 1023, 1016, 1068, 9, 2, 0, 0, 1012, 971, 1005, 991, 0, 0, 0, 0, 1034, 986, 1002, 998, 0, 0, 0, 0, 988, 1051, 1019, 1029, 0, 0, 0, 0, 1038, 1082, 1046, 1040, 0, 0, 0, 0, 1015, 1014, 1040, 1011, 10, 9, 0, 0, 981, 978, 1053, 1036, 31, 0, 0, 0, 1029, 1018, 1034, 1050, 7, 4, 0, 0] rerouted_tokens_per_ep_rank=[4156, 3979, 4020, 4087, 4206, 4099, 4079, 4142] avg=4096.0000 max=4206 max_rank=4 min=3979 min_rank=1 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16526 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1020, 1043, 1038, 983, 4, 1, 0, 0, 1025, 1030, 998, 1047, 0, 0, 0, 0, 1070, 999, 1031, 1023, 0, 0, 0, 0, 1064, 1066, 1062, 1034, 0, 0, 0, 0, 1007, 994, 1036, 991, 0, 0, 0, 0, 1048, 1037, 1035, 1004, 6, 4, 0, 0, 996, 966, 977, 1025, 30, 0, 0, 0, 970, 1019, 1046, 1033, 4, 2, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4100, 4123, 4226, 4028, 4134, 3994, 4074] avg=4096.0000 max=4226 max_rank=3 min=3994 min_rank=6 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16230 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[998, 981, 1059, 1033, 5, 1, 0, 0, 1031, 1011, 1014, 1047, 0, 0, 0, 0, 1055, 1062, 1017, 1033, 0, 0, 0, 0, 978, 1048, 1048, 1043, 0, 0, 0, 0, 973, 1048, 1078, 1074, 0, 0, 0, 0, 991, 989, 1024, 1059, 5, 4, 0, 0, 1015, 1030, 1013, 983, 28, 0, 0, 0, 1006, 982, 1019, 977, 4, 2, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4103, 4167, 4117, 4173, 4072, 4069, 3990] avg=4096.0000 max=4173 max_rank=4 min=3990 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1053, 982, 1017, 986, 5, 1, 0, 0, 968, 1018, 1010, 1025, 0, 0, 0, 0, 973, 1049, 998, 1034, 0, 0, 0, 0, 1004, 1033, 1008, 1002, 0, 0, 0, 0, 1048, 1008, 1065, 1028, 0, 0, 0, 0, 1001, 1078, 1037, 1016, 6, 4, 0, 0, 1043, 1021, 1018, 1061, 29, 0, 0, 0, 1071, 1030, 1010, 1022, 4, 2, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4021, 4054, 4047, 4149, 4142, 4172, 4139] avg=4096.0000 max=4172 max_rank=6 min=4021 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16602 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1005, 1008, 1024, 1007, 4, 1, 0, 0, 1026, 1066, 1028, 1007, 0, 0, 0, 0, 1009, 1012, 1076, 1006, 0, 0, 0, 0, 1045, 1014, 1041, 1045, 0, 0, 0, 0, 1021, 1033, 995, 1042, 0, 0, 0, 0, 1053, 1004, 1017, 985, 6, 4, 0, 0, 1038, 981, 1044, 1041, 30, 0, 0, 0, 1031, 960, 1021, 1032, 4, 2, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4127, 4103, 4145, 4091, 4069, 4134, 4050] avg=4096.0000 max=4145 max_rank=3 min=4049 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8320, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1001, 1016, 1007, 1033, 1010, 1073, 1006, 1012, 1050, 978, 993, 1003, 1058, 1035, 1015, 1081, 993, 1065, 1063, 978, 1010, 1026, 1036, 1005, 1015, 1013, 1056, 1008, 1063, 1028, 1021, 1017] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1013, 1003, 1021, 1039, 1050, 989, 1003, 1015, 1060, 1000, 1034, 1053, 1015, 1018, 1011, 1058, 1044, 1014, 1026, 1017, 1016, 984, 1022, 1056, 1046, 1009, 986, 1023, 1011, 1072, 991, 1069] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1021, 1075, 1020, 1039, 1039, 1034, 1011, 1059, 1009, 1012, 1068, 1017, 1016, 1020, 1017, 977, 1036, 999, 1010, 987, 979, 1070, 1057, 1073, 1012, 994, 1034, 970, 997, 1043, 1057, 1016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1032, 991, 1009, 1017, 1084, 996, 1074, 990, 1021, 971, 1056, 1053, 995, 1021, 1043, 1043, 1009, 1005, 1009, 1028, 1011, 1022, 1026, 1043, 1033, 1023, 1006, 1042, 1057, 978, 1047, 1033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[987, 1041, 1017, 989, 1008, 1011, 1037, 1073, 993, 1049, 1031, 1028, 1036, 1055, 1043, 1028, 1033, 983, 996, 1036, 1050, 1026, 1021, 1035, 1009, 1038, 1032, 989, 975, 1058, 1035, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1019, 980, 1019, 963, 1002, 1012, 1024, 1013, 1018, 1068, 1053, 1032, 996, 1049, 1013, 1014, 997, 1027, 1033, 1033, 1022, 1009, 991, 1035, 989, 1026, 1024, 1040, 1049, 1049, 1073, 1096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1016, 1014, 972, 1050, 1031, 1063, 1014, 1032, 1005, 1050, 1030, 1012, 981, 1015, 1007, 997, 986, 1047, 1058, 1033, 968, 1051, 1006, 1062, 1024, 1055, 1090, 961, 983, 1101, 1032, 1022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1009, 1042, 1018, 1034, 977, 1072, 987, 1026, 1007, 1051, 1027, 1089, 1021, 1066, 1043, 980, 1039, 984, 1013, 1029, 1008, 1026, 1024, 1013, 997, 1044, 1035, 1036, 1032, 985, 1014, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1013, 1003, 1021, 1039, 33, 0, 0, 0, 1050, 982, 1003, 1015, 0, 0, 0, 0, 1060, 1000, 1016, 1053, 0, 0, 0, 0, 1015, 1018, 1011, 1058, 0, 0, 0, 0, 1044, 1014, 1026, 1017, 14, 5, 0, 0, 1016, 984, 1022, 1055, 0, 0, 0, 0, 1046, 1009, 986, 1023, 7, 4, 1, 0, 1011, 1072, 991, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4050, 4129, 4102, 4120, 4077, 4076, 4105] avg=4096.0000 max=4129 max_rank=2 min=4050 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1001, 1016, 1007, 1033, 31, 0, 0, 0, 1010, 1066, 1006, 1012, 0, 0, 0, 0, 1050, 978, 975, 1003, 0, 0, 0, 0, 1058, 1035, 1015, 1081, 0, 0, 0, 0, 993, 1065, 1063, 978, 14, 5, 0, 0, 1010, 1026, 1036, 1004, 0, 0, 0, 0, 1015, 1013, 1056, 1008, 7, 4, 1, 0, 1063, 1028, 1021, 981, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4094, 4006, 4189, 4118, 4076, 4104, 4093] avg=4096.0000 max=4189 max_rank=3 min=4006 min_rank=2 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1021, 1075, 1020, 1039, 31, 0, 0, 0, 1039, 1027, 1011, 1059, 0, 0, 0, 0, 1009, 1012, 1049, 1017, 0, 0, 0, 0, 1016, 1020, 1017, 977, 0, 0, 0, 0, 1036, 999, 1010, 987, 15, 5, 0, 0, 979, 1070, 1057, 1072, 0, 0, 0, 0, 1012, 994, 1034, 970, 7, 4, 1, 0, 997, 1043, 1057, 980, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4136, 4087, 4030, 4052, 4178, 4022, 4077] avg=4096.0000 max=4186 max_rank=0 min=4022 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[987, 1041, 1017, 989, 32, 0, 0, 0, 1008, 1004, 1037, 1073, 0, 0, 0, 0, 993, 1049, 1013, 1028, 0, 0, 0, 0, 1036, 1055, 1043, 1028, 0, 0, 0, 0, 1033, 983, 996, 1036, 14, 5, 0, 0, 1050, 1026, 1021, 1034, 0, 0, 0, 0, 1009, 1038, 1032, 989, 7, 4, 1, 0, 975, 1058, 1035, 989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4122, 4083, 4162, 4067, 4131, 4080, 4057] avg=4096.0000 max=4162 max_rank=3 min=4057 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1009, 1042, 1018, 1034, 32, 0, 0, 0, 977, 1065, 987, 1026, 0, 0, 0, 0, 1007, 1051, 1009, 1089, 0, 0, 0, 0, 1021, 1066, 1043, 980, 0, 0, 0, 0, 1039, 984, 1013, 1029, 14, 5, 0, 0, 1008, 1026, 1024, 1012, 0, 0, 0, 0, 997, 1044, 1035, 1036, 7, 4, 1, 0, 1032, 985, 1014, 1003, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4055, 4156, 4110, 4084, 4070, 4124, 4034] avg=4096.0000 max=4156 max_rank=2 min=4034 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1016, 1014, 972, 1050, 32, 0, 0, 0, 1031, 1056, 1014, 1032, 0, 0, 0, 0, 1005, 1050, 1012, 1012, 0, 0, 0, 0, 981, 1015, 1007, 997, 0, 0, 0, 0, 986, 1047, 1058, 1033, 14, 5, 0, 0, 968, 1051, 1006, 1061, 0, 0, 0, 0, 1024, 1055, 1090, 961, 7, 4, 1, 0, 983, 1101, 1032, 985, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4133, 4079, 4000, 4143, 4086, 4142, 4101] avg=4096.0000 max=4143 max_rank=4 min=4000 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1032, 991, 1009, 1017, 36, 0, 0, 0, 1084, 986, 1074, 990, 0, 0, 0, 0, 1021, 971, 1030, 1053, 0, 0, 0, 0, 995, 1021, 1043, 1043, 1, 0, 0, 0, 1009, 1005, 1009, 1028, 18, 6, 0, 0, 1011, 1022, 1026, 1035, 0, 0, 0, 0, 1033, 1023, 1006, 1042, 10, 8, 7, 0, 1057, 978, 1047, 991, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4134, 4075, 4103, 4075, 4094, 4129, 4073] avg=4096.0000 max=4134 max_rank=1 min=4073 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1019, 980, 1019, 963, 34, 0, 0, 0, 1002, 1005, 1024, 1013, 0, 0, 0, 0, 1018, 1068, 1035, 1032, 0, 0, 0, 0, 996, 1049, 1013, 1014, 0, 0, 0, 0, 997, 1027, 1033, 1033, 14, 5, 0, 0, 1022, 1009, 991, 1034, 0, 0, 0, 0, 989, 1026, 1024, 1040, 7, 4, 1, 0, 1049, 1049, 1073, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4044, 4153, 4072, 4109, 4056, 4091, 4228] avg=4096.0000 max=4228 max_rank=7 min=4015 min_rank=0 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8192, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8320, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[975, 1002, 1033, 1063, 1028, 1011, 1077, 1059, 1016, 1053, 989, 989, 1040, 1068, 1033, 1014, 1019, 1004, 1049, 1034, 1030, 1025, 1010, 1067, 1043, 988, 993, 1012, 982, 1018, 988, 1056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1044, 1040, 993, 980, 986, 1031, 1018, 1032, 1077, 1047, 1048, 1003, 1026, 995, 959, 1014, 1027, 1028, 976, 1053, 1024, 997, 1044, 980, 1052, 1040, 1026, 1055, 1029, 1074, 1033, 1037] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1025, 1043, 1000, 970, 1026, 1003, 1025, 1034, 1013, 992, 1063, 1036, 979, 1033, 1065, 1029, 1029, 1029, 995, 1048, 1001, 1073, 1059, 1055, 994, 1028, 1004, 1035, 1045, 1004, 1029, 1004] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1050, 1055, 993, 1040, 1047, 988, 1021, 1029, 1012, 1018, 1026, 1042, 1043, 1012, 1056, 1031, 1001, 1025, 979, 1001, 990, 1007, 1046, 1082, 1019, 1029, 1022, 1026, 1044, 1047, 985, 1002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[998, 1037, 986, 1073, 1038, 1040, 1017, 1027, 1016, 957, 1024, 1047, 988, 1004, 1010, 1010, 1058, 1013, 1023, 1014, 1041, 1020, 1024, 1068, 1041, 1047, 1010, 1073, 980, 1037, 981, 1066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1049, 1001, 1038, 988, 1016, 1052, 991, 991, 1047, 998, 1021, 1030, 1070, 1047, 1034, 986, 1010, 1040, 1042, 1000, 983, 1013, 999, 1004, 1055, 1020, 1099, 1074, 990, 1031, 1022, 1027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1023, 1045, 1027, 1071, 1029, 1013, 1026, 968, 1026, 1024, 1063, 1029, 1010, 1040, 988, 1004, 1051, 982, 1039, 1054, 1011, 1055, 1010, 1008, 982, 1072, 1028, 1009, 1011, 999, 1021, 1050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1030, 1053, 1040, 1045, 1047, 1024, 989, 1074, 1024, 1011, 1045, 1018, 1012, 985, 1023, 1021, 1042, 1028, 1029, 1031, 993, 1000, 1013, 1011, 967, 1026, 1043, 969, 1030, 1006, 1045, 1094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[975, 998, 1033, 1063, 0, 0, 0, 0, 1028, 1011, 1077, 1059, 1, 0, 0, 0, 1016, 1053, 986, 989, 0, 0, 0, 0, 1040, 1068, 1033, 1014, 13, 3, 0, 0, 1019, 1004, 1049, 1034, 1, 0, 0, 0, 1030, 1025, 1010, 1067, 1, 1, 0, 0, 1043, 988, 993, 999, 0, 0, 0, 0, 982, 1018, 988, 1056, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4176, 4044, 4171, 4107, 4134, 4023, 4044] avg=4096.0000 max=4176 max_rank=1 min=4023 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1044, 1035, 993, 980, 0, 0, 0, 0, 986, 1031, 1018, 1032, 1, 0, 0, 0, 1077, 1047, 1045, 1003, 0, 0, 0, 0, 1026, 995, 959, 1014, 14, 3, 0, 0, 1027, 1028, 976, 1053, 1, 0, 0, 0, 1024, 997, 1044, 980, 2, 1, 0, 0, 1052, 1040, 1026, 1041, 0, 0, 0, 0, 1029, 1074, 1033, 1037, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4068, 4172, 4011, 4085, 4048, 4159, 4173] avg=4096.0000 max=4173 max_rank=7 min=4011 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1049, 997, 1038, 988, 0, 0, 0, 0, 1016, 1052, 991, 991, 1, 0, 0, 0, 1047, 998, 1018, 1030, 0, 0, 0, 0, 1070, 1047, 1034, 986, 14, 3, 0, 0, 1010, 1040, 1042, 1000, 1, 0, 0, 0, 983, 1013, 999, 1004, 1, 1, 0, 0, 1055, 1020, 1099, 1060, 0, 0, 0, 0, 990, 1031, 1022, 1027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4051, 4093, 4154, 4093, 4001, 4234, 4070] avg=4096.0000 max=4234 max_rank=6 min=4001 min_rank=5 imbalance_ratio=1.033691 max_load_reduction=28534 max_load_reduction_pct=0.870789 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1050, 1050, 993, 1040, 0, 0, 0, 0, 1047, 988, 1021, 1029, 1, 0, 0, 0, 1012, 1018, 1023, 1042, 0, 0, 0, 0, 1043, 1012, 1056, 1031, 14, 3, 0, 0, 1001, 1025, 979, 1001, 1, 0, 0, 0, 990, 1007, 1046, 1082, 2, 1, 0, 0, 1019, 1029, 1022, 1012, 0, 0, 0, 0, 1044, 1047, 985, 1002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4086, 4095, 4159, 4007, 4128, 4082, 4078] avg=4096.0000 max=4159 max_rank=3 min=4007 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 1038, 1000, 970, 0, 0, 0, 0, 1026, 1003, 1025, 1034, 1, 0, 0, 0, 1013, 992, 1060, 1036, 0, 0, 0, 0, 979, 1033, 1065, 1029, 14, 3, 0, 0, 1029, 1029, 995, 1048, 1, 0, 0, 0, 1001, 1073, 1059, 1055, 2, 1, 0, 0, 994, 1028, 1004, 1021, 0, 0, 0, 0, 1045, 1004, 1029, 1004, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4089, 4101, 4123, 4102, 4191, 4047, 4082] avg=4096.0000 max=4191 max_rank=5 min=4033 min_rank=0 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1023, 1040, 1027, 1071, 0, 0, 0, 0, 1029, 1013, 1026, 968, 1, 0, 0, 0, 1026, 1024, 1060, 1029, 0, 0, 0, 0, 1010, 1040, 988, 1004, 13, 3, 0, 0, 1051, 982, 1039, 1054, 1, 0, 0, 0, 1011, 1055, 1010, 1008, 2, 1, 0, 0, 982, 1072, 1028, 996, 0, 0, 0, 0, 1011, 999, 1021, 1050, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4037, 4139, 4058, 4127, 4087, 4078, 4081] avg=4096.0000 max=4161 max_rank=0 min=4037 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[998, 1032, 986, 1073, 0, 0, 0, 0, 1038, 1040, 1017, 1027, 1, 0, 0, 0, 1016, 957, 1021, 1047, 0, 0, 0, 0, 988, 1004, 1010, 1010, 14, 3, 0, 0, 1058, 1013, 1023, 1014, 1, 0, 0, 0, 1041, 1020, 1024, 1068, 2, 1, 0, 0, 1041, 1047, 1010, 1059, 0, 0, 0, 0, 980, 1037, 981, 1066, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4123, 4041, 4029, 4109, 4156, 4157, 4064] avg=4096.0000 max=4157 max_rank=6 min=4029 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1030, 1044, 1040, 1045, 0, 0, 0, 0, 1047, 1024, 989, 1074, 4, 0, 0, 0, 1024, 1011, 1030, 1018, 0, 0, 0, 0, 1012, 985, 1023, 1021, 17, 5, 0, 0, 1042, 1028, 1029, 1031, 8, 0, 0, 0, 993, 1000, 1013, 1011, 4, 2, 0, 0, 967, 1026, 1043, 952, 0, 0, 0, 0, 1030, 1006, 1045, 1094, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4138, 4083, 4063, 4138, 4023, 3988, 4176] avg=4096.0000 max=4176 max_rank=7 min=3988 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8448, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1074, 1014, 1018, 1022, 1038, 1013, 966, 1016, 1016, 996, 1044, 1053, 993, 1071, 1045, 1024, 1010, 1030, 1001, 1022, 1050, 1008, 1014, 1021, 1008, 1030, 1039, 1029, 1039, 1005, 1063, 996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[978, 1014, 1034, 1025, 996, 1008, 1037, 1013, 1053, 1037, 1062, 1040, 1024, 1005, 1031, 1004, 1011, 1036, 1023, 1065, 1076, 1028, 1023, 1036, 1035, 1045, 990, 1014, 1008, 1007, 1015, 995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1012, 1043, 1031, 984, 1051, 1082, 1022, 1027, 1011, 1023, 990, 1020, 1020, 1034, 1040, 993, 1026, 1087, 1041, 970, 1031, 1041, 1049, 954, 969, 1024, 997, 1032, 1017, 1041, 1040, 1066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[968, 975, 1004, 992, 1041, 1042, 1079, 1036, 1009, 1049, 1032, 1046, 997, 972, 1055, 1044, 1013, 974, 1029, 1055, 996, 1039, 1081, 992, 1011, 1018, 1019, 1056, 1026, 1018, 1071, 1029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1000, 1041, 1011, 1038, 1018, 1040, 1024, 1001, 985, 1024, 1023, 1081, 995, 1048, 1036, 1020, 1030, 1014, 1034, 1006, 1004, 1023, 1016, 1057, 1061, 1015, 989, 985, 1074, 1025, 1043, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1006, 1060, 1010, 987, 1049, 1020, 1018, 1035, 1067, 996, 1014, 1020, 1006, 1041, 993, 1040, 989, 980, 1005, 1022, 1033, 1033, 1060, 1051, 1037, 977, 1002, 1065, 1045, 985, 1054, 1068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1036, 1014, 1059, 978, 1012, 1010, 1020, 1082, 1074, 1028, 974, 970, 1032, 1066, 1002, 981, 1041, 1064, 1043, 990, 1027, 1011, 1010, 1000, 1044, 1004, 1046, 1041, 1011, 1031, 1025, 1042] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1006, 1018, 1012, 1043, 1021, 1067, 1007, 1083, 1046, 1047, 1002, 1001, 986, 1034, 1013, 1053, 999, 1046, 1058, 1049, 1002, 1037, 1087, 973, 1015, 1009, 1023, 1001, 1006, 1006, 1020, 998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[978, 1014, 1034, 1025, 25, 6, 0, 0, 996, 1008, 1037, 988, 0, 0, 0, 0, 1046, 1037, 1062, 1040, 0, 0, 0, 0, 1024, 1005, 1031, 1004, 7, 1, 0, 0, 1011, 1036, 1023, 1065, 0, 0, 0, 0, 1076, 1028, 1012, 1036, 0, 0, 0, 0, 1035, 1045, 990, 1014, 10, 6, 0, 0, 1008, 1007, 1003, 995, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4029, 4185, 4072, 4135, 4152, 4100, 4013] avg=4096.0000 max=4185 max_rank=2 min=4013 min_rank=7 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1074, 1014, 1018, 1022, 25, 7, 0, 0, 1038, 1013, 966, 991, 0, 0, 0, 0, 1009, 996, 1044, 1053, 0, 0, 0, 0, 993, 1071, 1045, 1024, 7, 1, 0, 0, 1010, 1030, 1001, 1022, 0, 0, 0, 0, 1050, 1008, 1003, 1021, 0, 0, 0, 0, 1008, 1030, 1039, 1029, 10, 6, 0, 0, 1039, 1005, 1050, 996, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4008, 4102, 4141, 4063, 4082, 4122, 4090] avg=4096.0000 max=4160 max_rank=0 min=4008 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[968, 975, 1004, 992, 25, 7, 0, 0, 1041, 1042, 1079, 1011, 0, 0, 0, 0, 1002, 1049, 1032, 1046, 0, 0, 0, 0, 997, 972, 1055, 1044, 7, 1, 0, 0, 1013, 974, 1029, 1055, 0, 0, 0, 0, 996, 1039, 1069, 992, 0, 0, 0, 0, 1011, 1018, 1019, 1056, 11, 6, 0, 0, 1026, 1018, 1058, 1029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3971, 4173, 4129, 4076, 4071, 4096, 4121, 4131] avg=4096.0000 max=4173 max_rank=1 min=3971 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1012, 1043, 1031, 984, 25, 6, 0, 0, 1051, 1082, 1022, 1002, 0, 0, 0, 0, 1004, 1023, 990, 1020, 0, 0, 0, 0, 1020, 1034, 1040, 993, 7, 1, 0, 0, 1026, 1087, 1041, 970, 0, 0, 0, 0, 1031, 1041, 1038, 954, 0, 0, 0, 0, 969, 1024, 997, 1032, 10, 6, 0, 0, 1017, 1041, 1028, 1066, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4157, 4037, 4095, 4124, 4064, 4038, 4152] avg=4096.0000 max=4157 max_rank=1 min=4037 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1000, 1041, 1011, 1038, 24, 6, 0, 0, 1018, 1040, 1024, 977, 0, 0, 0, 0, 978, 1024, 1023, 1081, 0, 0, 0, 0, 995, 1048, 1036, 1020, 7, 1, 0, 0, 1030, 1014, 1034, 1006, 0, 0, 0, 0, 1004, 1023, 1005, 1057, 0, 0, 0, 0, 1061, 1015, 989, 985, 10, 6, 0, 0, 1074, 1025, 1031, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4059, 4106, 4107, 4084, 4089, 4066, 4137] avg=4096.0000 max=4137 max_rank=7 min=4059 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1036, 1014, 1059, 978, 31, 11, 0, 0, 1012, 1010, 1020, 1051, 0, 0, 0, 0, 1058, 1028, 974, 970, 0, 0, 0, 0, 1032, 1066, 1002, 981, 11, 3, 0, 0, 1041, 1064, 1043, 990, 5, 0, 0, 0, 1027, 1011, 994, 1000, 0, 0, 0, 0, 1044, 1004, 1046, 1041, 13, 11, 0, 0, 1011, 1031, 1003, 1042, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4093, 4030, 4095, 4143, 4032, 4159, 4087] avg=4096.0000 max=4159 max_rank=6 min=4030 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1006, 1018, 1012, 1043, 26, 6, 0, 0, 1021, 1067, 1007, 1057, 0, 0, 0, 0, 1039, 1047, 1002, 1001, 0, 0, 0, 0, 986, 1034, 1013, 1053, 7, 1, 0, 0, 999, 1046, 1058, 1049, 0, 0, 0, 0, 1002, 1037, 1075, 973, 0, 0, 0, 0, 1015, 1009, 1023, 1001, 11, 6, 0, 0, 1006, 1006, 1008, 998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4152, 4089, 4094, 4152, 4087, 4065, 4018] avg=4096.0000 max=4152 max_rank=1 min=4018 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1006, 1060, 1010, 987, 25, 6, 0, 0, 1049, 1020, 1018, 1010, 0, 0, 0, 0, 1060, 996, 1014, 1020, 0, 0, 0, 0, 1006, 1041, 993, 1040, 7, 1, 0, 0, 989, 980, 1005, 1022, 0, 0, 0, 0, 1033, 1033, 1049, 1051, 0, 0, 0, 0, 1037, 977, 1002, 1065, 10, 6, 0, 0, 1045, 985, 1042, 1068, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4097, 4090, 4088, 3996, 4166, 4097, 4140] avg=4096.0000 max=4166 max_rank=5 min=3996 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1026, 1059, 1031, 1028, 1039, 1008, 1007, 1008, 1024, 1016, 1009, 1013, 1042, 1047, 996, 1019, 993, 1037, 1025, 1052, 1004, 968, 1029, 1014, 1050, 1020, 1048, 1065, 993, 1026, 1046, 1026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1031, 980, 1046, 1091, 1031, 1031, 1099, 1050, 1015, 1021, 988, 972, 1030, 989, 1035, 1040, 1041, 976, 1000, 1050, 1016, 1035, 968, 1063, 1057, 967, 1029, 1013, 981, 1025, 1045, 1053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1060, 1001, 1087, 1020, 1035, 981, 1021, 1040, 988, 984, 1042, 1018, 1036, 1014, 1072, 1042, 1014, 1002, 1083, 1033, 1032, 992, 1066, 1001, 1041, 1027, 1046, 996, 979, 964, 1008, 1043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1037, 1002, 1019, 991, 1004, 1033, 1021, 1045, 1004, 1020, 1036, 1017, 1099, 1017, 1015, 976, 1033, 1014, 1027, 1045, 1033, 1030, 1050, 1028, 985, 971, 1000, 1012, 1025, 1070, 996, 1113] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1011, 1075, 1030, 988, 1008, 994, 1003, 1031, 1021, 1054, 1022, 1018, 1048, 1069, 995, 1041, 1059, 1011, 1023, 997, 1066, 1046, 1005, 1020, 993, 1038, 1015, 1018, 959, 1061, 987, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1028, 966, 1045, 1070, 1032, 1051, 1016, 1024, 1014, 1016, 1078, 1010, 1044, 1034, 994, 1016, 1029, 1010, 994, 978, 1079, 1035, 1040, 1017, 1044, 1046, 1009, 955, 1051, 1016, 1033, 994] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1007, 1062, 1027, 1008, 1026, 1017, 1043, 1051, 1019, 1020, 1028, 1022, 1025, 1034, 1048, 1034, 969, 1017, 1020, 1008, 1060, 1006, 1042, 1001, 993, 1029, 1036, 1019, 1026, 997, 1054, 1020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1031, 1027, 984, 986, 1048, 1089, 1045, 1031, 1031, 1010, 970, 961, 993, 1011, 1038, 1060, 1040, 1048, 1081, 1005, 1009, 1012, 1014, 1019, 1034, 1002, 1013, 1015, 1048, 1022, 1059, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 1059, 1021, 1028, 0, 0, 0, 0, 1039, 1008, 1007, 985, 0, 0, 0, 0, 1024, 1016, 1009, 1013, 23, 14, 0, 0, 1019, 1047, 996, 1019, 0, 0, 0, 0, 993, 1037, 1025, 1052, 3, 2, 0, 0, 1001, 968, 1029, 1014, 0, 0, 0, 0, 1050, 1020, 1048, 1065, 10, 9, 2, 0, 993, 1026, 1046, 1022, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4039, 4099, 4081, 4112, 4012, 4204, 4087] avg=4096.0000 max=4204 max_rank=6 min=4012 min_rank=5 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1031, 980, 1036, 1091, 0, 0, 0, 0, 1031, 1031, 1099, 1026, 0, 0, 0, 0, 1015, 1021, 988, 972, 24, 13, 0, 0, 1009, 989, 1035, 1040, 0, 0, 0, 0, 1041, 976, 1000, 1050, 3, 2, 0, 0, 1013, 1035, 968, 1063, 0, 0, 0, 0, 1057, 967, 1029, 1013, 10, 8, 3, 0, 981, 1025, 1045, 1048, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4187, 4033, 4073, 4072, 4079, 4087, 4099] avg=4096.0000 max=4187 max_rank=1 min=4033 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1060, 1001, 1076, 1020, 0, 0, 0, 0, 1035, 981, 1021, 1016, 0, 0, 0, 0, 988, 984, 1042, 1018, 24, 14, 0, 0, 1014, 1014, 1072, 1042, 0, 0, 0, 0, 1014, 1002, 1083, 1033, 3, 2, 0, 0, 1029, 992, 1066, 1001, 0, 0, 0, 0, 1041, 1027, 1046, 996, 11, 8, 3, 0, 979, 964, 1008, 1038, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4053, 4070, 4142, 4137, 4088, 4132, 3989] avg=4096.0000 max=4157 max_rank=0 min=3989 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1011, 1075, 1020, 988, 0, 0, 0, 0, 1008, 994, 1003, 1007, 0, 0, 0, 0, 1021, 1054, 1022, 1018, 24, 14, 0, 0, 1025, 1069, 995, 1041, 0, 0, 0, 0, 1059, 1011, 1023, 997, 4, 2, 0, 0, 1062, 1046, 1005, 1020, 0, 0, 0, 0, 993, 1038, 1015, 1018, 10, 9, 3, 0, 959, 1061, 987, 1057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4012, 4153, 4130, 4096, 4133, 4086, 4064] avg=4096.0000 max=4153 max_rank=2 min=4012 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1037, 1002, 1004, 991, 0, 0, 0, 0, 1004, 1033, 1021, 1017, 0, 0, 0, 0, 1004, 1020, 1036, 1017, 28, 18, 0, 0, 1068, 1017, 1015, 976, 0, 0, 0, 0, 1033, 1014, 1027, 1045, 8, 8, 0, 0, 1025, 1030, 1050, 1028, 0, 0, 0, 0, 985, 971, 1000, 1012, 15, 13, 7, 0, 1025, 1070, 996, 1098, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4075, 4123, 4076, 4135, 4133, 4003, 4189] avg=4096.0000 max=4189 max_rank=7 min=4003 min_rank=6 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1031, 1027, 974, 986, 0, 0, 0, 0, 1048, 1089, 1045, 1007, 0, 0, 0, 0, 1031, 1010, 970, 961, 24, 13, 0, 0, 972, 1011, 1038, 1060, 0, 0, 0, 0, 1040, 1048, 1081, 1005, 3, 2, 0, 0, 1006, 1012, 1014, 1019, 0, 0, 0, 0, 1034, 1002, 1013, 1015, 10, 8, 2, 0, 1048, 1022, 1059, 1028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4189, 4009, 4081, 4179, 4051, 4084, 4157] avg=4096.0000 max=4189 max_rank=1 min=4009 min_rank=2 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1028, 966, 1035, 1070, 0, 0, 0, 0, 1032, 1051, 1016, 1001, 0, 0, 0, 0, 1014, 1016, 1078, 1010, 23, 14, 0, 0, 1021, 1034, 994, 1016, 0, 0, 0, 0, 1029, 1010, 994, 978, 4, 2, 0, 0, 1075, 1035, 1040, 1017, 0, 0, 0, 0, 1044, 1046, 1009, 955, 10, 9, 2, 0, 1051, 1016, 1033, 990, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4100, 4155, 4065, 4017, 4167, 4075, 4090] avg=4096.0000 max=4167 max_rank=5 min=4017 min_rank=4 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1007, 1062, 1017, 1008, 0, 0, 0, 0, 1026, 1017, 1043, 1027, 0, 0, 0, 0, 1019, 1020, 1028, 1022, 24, 13, 0, 0, 1004, 1034, 1048, 1034, 0, 0, 0, 0, 969, 1017, 1020, 1008, 4, 2, 0, 0, 1056, 1006, 1042, 1001, 0, 0, 0, 0, 993, 1029, 1036, 1019, 10, 8, 2, 0, 1026, 997, 1054, 1016, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4113, 4126, 4120, 4020, 4105, 4097, 4093] avg=4096.0000 max=4126 max_rank=2 min=4020 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8320, 8192, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8064, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8064, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[986, 1002, 1036, 979, 1027, 1012, 1027, 1038, 1000, 996, 1020, 1013, 1016, 1067, 1017, 1042, 1076, 1053, 1025, 1013, 1033, 1037, 1034, 1036, 1013, 1046, 1009, 1027, 1079, 1013, 1018, 978] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[968, 1051, 1021, 1005, 965, 1022, 999, 1036, 1045, 1043, 1033, 1021, 1034, 1045, 1001, 1076, 1009, 1024, 1045, 1055, 1063, 1025, 1028, 998, 1001, 1046, 1011, 989, 1034, 1009, 1081, 985] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1020, 1022, 1017, 1019, 1006, 1041, 1024, 969, 1029, 990, 1069, 982, 976, 1035, 1020, 1069, 1038, 1024, 1043, 1036, 1009, 959, 1004, 1045, 1069, 995, 1007, 1058, 1081, 996, 1109, 1007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1055, 1043, 1022, 1039, 1014, 1038, 1042, 1042, 1025, 1005, 1059, 968, 981, 1012, 1014, 1064, 1022, 1026, 1095, 1023, 977, 1032, 1010, 983, 985, 995, 1060, 1049, 1023, 1074, 978, 1013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1006, 1032, 1002, 1047, 1016, 1052, 1022, 1001, 990, 973, 1020, 1003, 1016, 1011, 1032, 1064, 1033, 1062, 1043, 999, 1030, 1008, 1016, 1032, 1052, 1029, 1042, 1045, 1015, 1021, 992, 1062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[998, 1016, 1041, 1060, 1066, 1051, 1034, 1009, 992, 1022, 995, 965, 1039, 1000, 1048, 1012, 1076, 993, 1038, 1042, 1031, 1007, 1068, 1025, 1061, 989, 998, 1034, 1063, 1003, 1012, 980] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1052, 1002, 1049, 1070, 1037, 1016, 1020, 1039, 1096, 1010, 1019, 958, 975, 1046, 1064, 1006, 1026, 1006, 969, 975, 1022, 1025, 992, 1076, 1036, 1022, 1039, 1001, 1056, 1032, 1007, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[987, 987, 1062, 1041, 1029, 1016, 1020, 1052, 1063, 1000, 1000, 1008, 1052, 1018, 1084, 1003, 1041, 1036, 986, 965, 1001, 1064, 992, 1069, 985, 1022, 1051, 1004, 1039, 995, 1056, 1040] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[986, 1002, 1036, 979, 1, 1, 0, 0, 1027, 1011, 1027, 1038, 0, 0, 0, 0, 1000, 996, 1020, 1013, 21, 16, 7, 0, 1016, 1067, 1017, 1021, 0, 0, 0, 0, 1060, 1053, 1025, 1013, 0, 0, 0, 0, 1033, 1037, 1034, 1036, 4, 0, 0, 0, 1013, 1046, 1009, 1027, 0, 0, 0, 0, 1067, 1013, 1018, 978, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 4103, 4073, 4121, 4151, 4144, 4095, 4076] avg=4096.0000 max=4151 max_rank=4 min=4005 min_rank=0 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[968, 1051, 1021, 1005, 1, 1, 0, 0, 965, 1021, 999, 1036, 0, 0, 0, 0, 1045, 1043, 1033, 1021, 22, 15, 6, 0, 1034, 1045, 1001, 1054, 0, 0, 0, 0, 994, 1024, 1045, 1055, 0, 0, 0, 0, 1063, 1025, 1028, 998, 4, 0, 0, 0, 1001, 1046, 1011, 989, 0, 0, 0, 0, 1023, 1009, 1081, 985, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4021, 4185, 4134, 4118, 4118, 4047, 4098] avg=4096.0000 max=4185 max_rank=2 min=4021 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1006, 1032, 1002, 1047, 1, 1, 0, 0, 1016, 1051, 1022, 1001, 0, 0, 0, 0, 990, 973, 1020, 1003, 21, 16, 6, 0, 1016, 1011, 1032, 1043, 0, 0, 0, 0, 1017, 1062, 1043, 999, 0, 0, 0, 0, 1030, 1008, 1016, 1032, 4, 0, 0, 0, 1052, 1029, 1042, 1045, 0, 0, 0, 0, 1004, 1021, 992, 1062, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4090, 4029, 4102, 4121, 4090, 4168, 4079] avg=4096.0000 max=4168 max_rank=6 min=4029 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[998, 1016, 1041, 1060, 1, 1, 0, 0, 1066, 1050, 1034, 1009, 0, 0, 0, 0, 992, 1022, 995, 965, 20, 16, 7, 0, 1039, 1000, 1048, 992, 0, 0, 0, 0, 1060, 993, 1038, 1042, 0, 0, 0, 0, 1031, 1007, 1068, 1025, 4, 0, 0, 0, 1061, 989, 998, 1034, 0, 0, 0, 0, 1051, 1003, 1012, 980, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4159, 4017, 4079, 4133, 4135, 4082, 4046] avg=4096.0000 max=4159 max_rank=1 min=4017 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1020, 1022, 1017, 1019, 1, 1, 0, 0, 1006, 1040, 1024, 969, 0, 0, 0, 0, 1029, 990, 1069, 982, 21, 16, 7, 0, 976, 1035, 1020, 1048, 0, 0, 0, 0, 1022, 1024, 1043, 1036, 0, 0, 0, 0, 1009, 959, 1004, 1045, 4, 0, 0, 0, 1069, 995, 1007, 1058, 0, 0, 0, 0, 1069, 996, 1109, 1007, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4039, 4114, 4079, 4125, 4021, 4129, 4181] avg=4096.0000 max=4181 max_rank=7 min=4021 min_rank=5 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1055, 1043, 1022, 1039, 8, 7, 2, 0, 1014, 1031, 1042, 1042, 0, 0, 0, 0, 1025, 1005, 1059, 968, 26, 19, 10, 0, 981, 1012, 1014, 1038, 0, 0, 0, 0, 1003, 1026, 1095, 1023, 0, 0, 0, 0, 977, 1032, 1010, 983, 9, 0, 0, 0, 985, 995, 1058, 1049, 0, 0, 0, 0, 996, 1074, 978, 1013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4129, 4112, 4045, 4147, 4011, 4087, 4061] avg=4096.0000 max=4176 max_rank=0 min=4011 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1052, 1002, 1049, 1070, 1, 1, 0, 0, 1037, 1015, 1020, 1039, 0, 0, 0, 0, 1096, 1010, 1019, 958, 20, 15, 7, 0, 975, 1046, 1064, 986, 0, 0, 0, 0, 1011, 1006, 969, 975, 0, 0, 0, 0, 1022, 1025, 992, 1076, 4, 0, 0, 0, 1036, 1022, 1039, 1001, 0, 0, 0, 0, 1044, 1032, 1007, 1025, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4111, 4125, 4071, 3961, 4119, 4098, 4108] avg=4096.0000 max=4175 max_rank=0 min=3961 min_rank=4 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[987, 987, 1062, 1041, 1, 1, 0, 0, 1029, 1015, 1020, 1052, 0, 0, 0, 0, 1063, 1000, 1000, 1008, 20, 16, 6, 0, 1052, 1018, 1084, 983, 0, 0, 0, 0, 1025, 1036, 986, 965, 0, 0, 0, 0, 1001, 1064, 992, 1069, 4, 0, 0, 0, 985, 1022, 1051, 1004, 0, 0, 0, 0, 1028, 995, 1056, 1040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4116, 4113, 4137, 4012, 4130, 4062, 4119] avg=4096.0000 max=4137 max_rank=3 min=4012 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8192, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 7936, 256, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=32896 slack_slots=2560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33408 slack_slots=2048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1004, 980, 1016, 1018, 1032, 1049, 979, 1036, 1065, 1043, 1041, 1028, 1018, 1006, 1028, 1009, 1003, 989, 1065, 1025, 989, 1103, 1065, 1038, 996, 1048, 987, 1003, 1041, 1018, 1014, 1032] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1037, 1055, 1002, 1050, 1005, 1049, 1011, 1040, 1031, 1013, 1019, 1023, 1062, 996, 1010, 1018, 1013, 1018, 1050, 1052, 1048, 1070, 946, 1037, 1020, 1035, 983, 976, 1041, 1011, 1041, 1006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1047, 998, 1011, 1004, 1043, 1038, 1025, 1021, 1022, 1018, 1053, 1031, 1084, 992, 1042, 1026, 1007, 999, 1015, 994, 1022, 991, 1034, 1018, 1042, 1024, 1052, 1069, 1017, 964, 1040, 1025] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1020, 1019, 1046, 1047, 1051, 1027, 1024, 1021, 1008, 1044, 1033, 1051, 1009, 1039, 1084, 1047, 1015, 999, 1000, 1061, 956, 1008, 1035, 982, 1034, 990, 1031, 1018, 991, 1021, 965, 1092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[997, 1070, 1033, 988, 1031, 1027, 1000, 1007, 1040, 1029, 1063, 999, 1006, 982, 1033, 1023, 1048, 1049, 1003, 1031, 999, 1042, 1052, 1062, 1042, 1058, 1001, 1030, 1020, 973, 1044, 986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1003, 997, 1079, 938, 1083, 964, 1023, 1039, 982, 1071, 1069, 1006, 994, 1020, 1029, 1051, 1046, 1045, 1025, 1028, 1120, 1012, 1003, 1031, 1084, 1001, 954, 1031, 1013, 994, 1032, 1001] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1048, 973, 1008, 1032, 1018, 1022, 992, 1010, 1008, 1022, 1034, 1034, 1028, 1002, 1037, 1039, 1041, 1036, 1010, 1028, 1068, 1016, 1027, 1002, 1000, 997, 1063, 1027, 1058, 1042, 1036, 1010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1025, 1063, 995, 1017, 1017, 1012, 1040, 974, 1009, 1066, 1018, 1038, 1015, 1030, 1038, 1034, 1008, 1007, 1022, 1014, 1022, 1081, 1021, 972, 997, 1022, 1020, 1014, 1045, 1045, 1041, 1046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1037, 1055, 1002, 1050, 18, 0, 0, 0, 1005, 1049, 1011, 1040, 4, 2, 0, 0, 1031, 1013, 990, 1023, 0, 0, 0, 0, 1062, 996, 1004, 1018, 0, 0, 0, 0, 1013, 1018, 1050, 1052, 2, 0, 0, 0, 1048, 1057, 946, 1037, 0, 0, 0, 0, 1020, 1035, 983, 976, 11, 3, 0, 0, 1041, 1011, 1041, 1006, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4111, 4057, 4080, 4135, 4088, 4028, 4107] avg=4096.0000 max=4162 max_rank=0 min=4028 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1004, 980, 1016, 1018, 18, 0, 0, 0, 1032, 1049, 979, 1036, 5, 2, 0, 0, 1065, 1043, 1012, 1028, 0, 0, 0, 0, 1018, 1006, 1021, 1009, 0, 0, 0, 0, 1003, 989, 1065, 1025, 2, 0, 0, 0, 989, 1090, 1065, 1038, 0, 0, 0, 0, 996, 1048, 987, 1003, 11, 3, 0, 0, 1041, 1018, 1014, 1032, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4103, 4148, 4054, 4084, 4182, 4048, 4113] avg=4096.0000 max=4182 max_rank=5 min=4036 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1047, 998, 1011, 1004, 18, 0, 0, 0, 1043, 1038, 1025, 1021, 5, 2, 0, 0, 1022, 1018, 1023, 1031, 0, 0, 0, 0, 1084, 992, 1035, 1026, 0, 0, 0, 0, 1007, 999, 1015, 994, 2, 0, 0, 0, 1022, 980, 1034, 1018, 0, 0, 0, 0, 1042, 1024, 1052, 1069, 12, 2, 0, 0, 1017, 964, 1040, 1025, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4134, 4094, 4137, 4017, 4054, 4201, 4053] avg=4096.0000 max=4201 max_rank=6 min=4017 min_rank=4 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1020, 1019, 1046, 1047, 22, 0, 0, 0, 1051, 1027, 1024, 1021, 7, 3, 0, 0, 1008, 1044, 996, 1051, 0, 0, 0, 0, 1009, 1039, 1069, 1047, 0, 0, 0, 0, 1015, 999, 1000, 1061, 8, 0, 0, 0, 956, 988, 1035, 982, 0, 0, 0, 0, 1034, 990, 1031, 1018, 15, 6, 0, 0, 991, 1021, 965, 1092, 11, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4133, 4099, 4164, 4083, 3961, 4094, 4080] avg=4096.0000 max=4164 max_rank=3 min=3961 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16218 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[997, 1070, 1033, 988, 18, 0, 0, 0, 1031, 1027, 1000, 1007, 5, 2, 0, 0, 1040, 1029, 1033, 999, 0, 0, 0, 0, 1006, 982, 1026, 1023, 0, 0, 0, 0, 1048, 1049, 1003, 1031, 2, 0, 0, 0, 999, 1030, 1052, 1062, 0, 0, 0, 0, 1042, 1058, 1001, 1030, 12, 3, 0, 0, 1020, 973, 1044, 986, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4072, 4101, 4037, 4133, 4143, 4146, 4030] avg=4096.0000 max=4146 max_rank=6 min=4030 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1003, 997, 1079, 938, 18, 0, 0, 0, 1083, 964, 1023, 1039, 5, 2, 0, 0, 982, 1071, 1039, 1006, 0, 0, 0, 0, 994, 1020, 1022, 1051, 0, 0, 0, 0, 1046, 1045, 1025, 1028, 2, 0, 0, 0, 1120, 1001, 1003, 1031, 0, 0, 0, 0, 1084, 1001, 954, 1031, 12, 2, 0, 0, 1013, 994, 1032, 1001, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4116, 4098, 4087, 4146, 4155, 4084, 4047] avg=4096.0000 max=4155 max_rank=5 min=4035 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1048, 973, 1008, 1032, 18, 0, 0, 0, 1018, 1022, 992, 1010, 5, 2, 0, 0, 1008, 1022, 1005, 1034, 0, 0, 0, 0, 1028, 1002, 1030, 1039, 0, 0, 0, 0, 1041, 1036, 1010, 1028, 2, 0, 0, 0, 1068, 1005, 1027, 1002, 0, 0, 0, 0, 1000, 997, 1063, 1027, 11, 2, 0, 0, 1058, 1042, 1036, 1010, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4049, 4069, 4099, 4117, 4102, 4100, 4153] avg=4096.0000 max=4153 max_rank=7 min=4049 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 1063, 995, 1017, 18, 0, 0, 0, 1017, 1012, 1040, 974, 5, 2, 0, 0, 1009, 1066, 989, 1038, 0, 0, 0, 0, 1015, 1030, 1031, 1034, 0, 0, 0, 0, 1008, 1007, 1022, 1014, 2, 0, 0, 0, 1022, 1068, 1021, 972, 0, 0, 0, 0, 997, 1022, 1020, 1014, 11, 3, 0, 0, 1045, 1045, 1041, 1046, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4050, 4102, 4110, 4053, 4083, 4067, 4185] avg=4096.0000 max=4185 max_rank=7 min=4050 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=35456 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=35456 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=35456 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=35456 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=35456 valid_num_tokens=33152 slack_slots=2304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=35456 valid_num_tokens=33280 slack_slots=2176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=35456 valid_num_tokens=33024 slack_slots=2432 device_initiated_grouped_gemm=True + [2026-08-30 04:58:09] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 635.5 | throughput per GPU (TFLOP/s/GPU): 524.6 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.949361E+00 | loss scale: 1.0 | grad norm: 4.932 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 04:58:09 +[rank0]:[W830 04:58:11.394318656 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..02ffa71c7f681f5a0fb049211ef163065804495e --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv @@ -0,0 +1,264 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2231 +1, 2231 +2, 2231 +3, 2231 +4, 2231 +5, 2231 +6, 2231 +7, 2231 +0, 39507 +1, 39503 +2, 39503 +3, 39503 +4, 39503 +5, 39503 +6, 39503 +7, 39503 +0, 39773 +1, 39773 +2, 39773 +3, 39773 +4, 39765 +5, 39765 +6, 39773 +7, 39773 +0, 42263 +1, 42255 +2, 42255 +3, 42255 +4, 42255 +5, 42255 +6, 42255 +7, 42255 +0, 42507 +1, 42499 +2, 42499 +3, 42499 +4, 42499 +5, 42499 +6, 42499 +7, 42499 +0, 42507 +1, 42499 +2, 42499 +3, 42499 +4, 42499 +5, 42499 +6, 42499 +7, 42499 +0, 43153 +1, 43145 +2, 43145 +3, 43145 +4, 43145 +5, 43145 +6, 43145 +7, 43145 +0, 43423 +1, 43415 +2, 43415 +3, 43415 +4, 43415 +5, 43415 +6, 43415 +7, 43415 +0, 43935 +1, 43927 +2, 43927 +3, 43927 +4, 43927 +5, 43927 +6, 43927 +7, 43927 +0, 46549 +1, 46541 +2, 46541 +3, 46541 +4, 46541 +5, 46541 +6, 46541 +7, 46541 +0, 46549 +1, 46541 +2, 46541 +3, 46541 +4, 46541 +5, 46541 +6, 46541 +7, 46541 +0, 86187 +1, 86163 +2, 86163 +3, 86163 +4, 86165 +5, 86163 +6, 86163 +7, 86163 +0, 86187 +1, 86163 +2, 86163 +3, 86163 +4, 86165 +5, 86163 +6, 86163 +7, 86163 +0, 87083 +1, 87059 +2, 87059 +3, 87059 +4, 87061 +5, 87059 +6, 87059 +7, 87059 +0, 87083 +1, 87059 +2, 87059 +3, 87059 +4, 87061 +5, 87059 +6, 87059 +7, 87059 +0, 90789 +1, 90765 +2, 90765 +3, 90765 +4, 90767 +5, 90765 +6, 90765 +7, 90765 +0, 104601 +1, 104605 +2, 104605 +3, 104605 +4, 104607 +5, 104605 +6, 104605 +7, 104605 +0, 106035 +1, 106011 +2, 106037 +3, 106029 +4, 106035 +5, 106055 +6, 106043 +7, 106031 +0, 106773 +1, 106807 +2, 106853 +3, 106857 +4, 106839 +5, 106869 +6, 106839 +7, 106925 +0, 125423 +1, 124993 +2, 125455 +3, 125665 +4, 125429 +5, 125679 +6, 125665 +7, 125665 +0, 130273 +1, 130275 +2, 130289 +3, 130275 +4, 130263 +5, 130275 +6, 130275 +7, 130275 +0, 130273 +1, 130275 +2, 130289 +3, 130275 +4, 130263 +5, 130275 +6, 130275 +7, 130275 +0, 130273 +1, 130275 +2, 130289 +3, 130275 +4, 130263 +5, 130275 +6, 130275 +7, 130275 +0, 129421 +1, 130005 +2, 129693 +3, 129359 +4, 129347 +5, 129423 +6, 130005 +7, 129679 +0, 2115 +1, 129359 +2, 3142 +3, 1862 +4, 1862 +5, 1862 +6, 1862 +7, 1862 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1/run.env b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..03fd3372ceac13e5dee2c64c88c28248492e897c --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1/run.env @@ -0,0 +1,70 @@ +backend=echo +config=G2 +family=tierA +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=3584 +num_experts=32 +topk=8 +experts_per_rank=4 +expert_params=44040192 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=32768 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=4 +capacity_mode=static +capacity=2.00 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..46b50f2f15f4247c3808a9186539185fe95e47e1 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log @@ -0,0 +1,16469 @@ +[04:54:14] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G2 / echo (slots/rank=4 policy=matched) + E=32 K=8 H'=3584 m=4 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=2.00 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0830 04:54:16.341000 444139 torch/distributed/run.py:766] +W0830 04:54:16.341000 444139 torch/distributed/run.py:766] ***************************************** +W0830 04:54:16.341000 444139 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 04:54:16.341000 444139 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 3584 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 32 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 2.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 32 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to [Gloo] Rank 7 peer ranks. Expected number of connected peer ranks is : 17 is connected to 7 + peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 2[Gloo] Rank 57 is connected to 6 is connected to is connected to 7 is connected to 77 peer ranks. 7 peer ranks. peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : +777 + + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 5 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W830 04:54:33.105251072 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 04:54:33.109347751 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 04:54:33.109482245 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 04:54:33.110155770 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 04:54:33.111610990 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 04:54:33.111636313 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 04:54:33.113442434 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 04:54:34.278473440 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.903 seconds +time to initialize megatron (seconds): 7.322 +[after megatron is initialized] datetime: 2026-08-30 04:54:39 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3903983616 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42086400 elements, 42086400 padded size): + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight +Params for bucket 3 (42082304 elements, 42082304 padded size): + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight +Params for bucket 4 (42082304 elements, 42082304 padded size): + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 5 (42082304 elements, 42082304 padded size): + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_qkv.weight +Params for bucket 6 (42082304 elements, 42082304 padded size): + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight +Params for bucket 7 (42082304 elements, 42082304 padded size): + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 8 (42082304 elements, 42082304 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight +Params for bucket 9 (42082304 elements, 42082304 padded size): + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight +Params for bucket 10 (42082304 elements, 42082304 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.weight +Params for bucket 11 (42082304 elements, 42082304 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight +Params for bucket 12 (42082304 elements, 42082304 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.weight +Params for bucket 13 (42082304 elements, 42082304 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 14 (42082304 elements, 42082304 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight +Params for bucket 15 (42082304 elements, 42082304 padded size): + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.weight +Params for bucket 16 (42082304 elements, 42082304 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight +Params for bucket 17 (42082304 elements, 42082304 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 56 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 3 (58720256 elements, 58720256 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 6 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 7 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 +Params for bucket 10 (58720256 elements, 58720256 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 +Params for bucket 13 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 +Params for bucket 14 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 +Params for bucket 20 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 24 (58720256 elements, 58720256 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 28 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 30 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 34 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 38 (58720256 elements, 58720256 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 41 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 42 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 43 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 45 (58720256 elements, 58720256 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 +Params for bucket 47 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 48 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 49 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 52 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 55 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 56 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 04:54:40 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 04:54:40 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (338.26, 342.17) + train/valid/test-data-iterators-setup ..........: (61.56, 76.75) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 04:54:40 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[56, 2890, 926, 456, 1435, 545, 827, 921, 984, 196, 994, 572, 1216, 655, 382, 552, 165, 474, 1182, 927, 1110, 103, 2981, 2197, 537, 958, 853, 1381, 1331, 427, 2033, 2502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[161, 3125, 861, 327, 1307, 667, 761, 1183, 863, 309, 1057, 464, 1189, 1067, 167, 450, 107, 565, 1481, 807, 917, 160, 2701, 1817, 864, 937, 1123, 1509, 1533, 360, 1810, 2119] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[68, 3107, 586, 237, 1848, 671, 1188, 817, 1101, 230, 1708, 378, 943, 869, 235, 347, 96, 1022, 670, 673, 1135, 213, 3308, 2120, 686, 1233, 972, 1331, 989, 381, 1609, 1997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[155, 2518, 951, 567, 1540, 284, 1128, 857, 1362, 194, 1623, 771, 874, 802, 215, 522, 153, 875, 669, 1111, 1230, 163, 2921, 2084, 688, 917, 1021, 1023, 1291, 213, 1984, 2062] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[218, 2853, 1229, 315, 1285, 586, 1176, 1212, 1297, 227, 1026, 209, 1055, 693, 255, 549, 336, 431, 1314, 1006, 1162, 140, 3062, 1749, 618, 833, 1355, 742, 1641, 724, 1676, 1794] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[236, 3079, 692, 641, 1529, 646, 1133, 640, 1225, 164, 1989, 146, 1189, 1211, 243, 519, 146, 450, 1336, 749, 831, 379, 2957, 2064, 602, 876, 1021, 1195, 1384, 319, 1665, 1512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[105, 3071, 689, 291, 1413, 531, 884, 1316, 1242, 381, 1171, 224, 1045, 1000, 383, 469, 166, 503, 413, 1356, 1180, 132, 2817, 1901, 362, 1161, 1225, 1243, 1564, 534, 1610, 2386] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[103, 3109, 814, 315, 1699, 775, 1139, 1132, 1210, 235, 1121, 237, 703, 660, 298, 620, 203, 805, 769, 1140, 1035, 178, 3067, 1918, 633, 856, 1113, 901, 1441, 516, 1843, 2180] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[56, 2650, 926, 456, 0, 0, 0, 0, 1399, 545, 827, 921, 0, 0, 0, 0, 984, 196, 994, 572, 1143, 35, 0, 0, 1216, 655, 382, 552, 1425, 0, 0, 0, 165, 474, 1182, 927, 697, 770, 0, 0, 1110, 103, 859, 2197, 0, 0, 0, 0, 537, 958, 853, 1381, 205, 36, 0, 0, 1331, 427, 2033, 589, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 3692, 3924, 4230, 4215, 4269, 3970, 4380] avg=4096.0000 max=4380 max_rank=7 min=3692 min_rank=1 imbalance_ratio=1.069336 max_load_reduction=28388 max_load_reduction_pct=0.866333 moved_to_echo=16834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[105, 2816, 689, 291, 0, 0, 0, 0, 1378, 531, 884, 1316, 0, 0, 0, 0, 1242, 381, 1171, 224, 1090, 37, 0, 0, 1045, 1000, 383, 469, 1347, 0, 0, 0, 166, 503, 413, 1356, 659, 734, 0, 0, 1180, 132, 811, 1901, 0, 0, 0, 0, 362, 1161, 1225, 1243, 218, 35, 0, 0, 1564, 534, 1610, 562, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3901, 4109, 4145, 4244, 3831, 4024, 4244, 4270] avg=4096.0000 max=4270 max_rank=7 min=3831 min_rank=4 imbalance_ratio=1.042480 max_load_reduction=28498 max_load_reduction_pct=0.869690 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[68, 2848, 586, 237, 0, 0, 0, 0, 1801, 671, 1188, 817, 0, 0, 0, 0, 1101, 230, 1708, 378, 912, 38, 0, 0, 943, 869, 235, 347, 1582, 0, 0, 0, 96, 1022, 670, 673, 774, 614, 0, 0, 1135, 213, 952, 2120, 0, 0, 0, 0, 686, 1233, 972, 1331, 221, 47, 0, 0, 989, 381, 1609, 471, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3739, 4477, 4367, 3976, 3849, 4420, 4490, 3450] avg=4096.0000 max=4490 max_rank=6 min=3450 min_rank=7 imbalance_ratio=1.096191 max_load_reduction=28278 max_load_reduction_pct=0.862976 moved_to_echo=16209 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[155, 2309, 951, 567, 0, 0, 0, 0, 1501, 284, 1128, 857, 0, 0, 0, 0, 1362, 194, 1623, 771, 942, 30, 0, 0, 874, 802, 215, 522, 1397, 0, 0, 0, 153, 875, 669, 1111, 683, 634, 0, 0, 1230, 163, 841, 2084, 0, 0, 0, 0, 688, 917, 1021, 1023, 179, 39, 0, 0, 1291, 213, 1984, 486, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3982, 3770, 4922, 3810, 4125, 4318, 3867, 3974] avg=4096.0000 max=4922 max_rank=2 min=3770 min_rank=1 imbalance_ratio=1.201660 max_load_reduction=27846 max_load_reduction_pct=0.849792 moved_to_echo=16284 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[218, 2608, 1229, 315, 0, 0, 0, 0, 1249, 586, 1176, 1212, 0, 0, 0, 0, 1297, 227, 1026, 209, 825, 38, 0, 0, 1055, 693, 255, 549, 1468, 0, 0, 0, 336, 431, 1314, 1006, 719, 555, 0, 0, 1162, 140, 875, 1749, 0, 0, 0, 0, 618, 833, 1355, 742, 207, 36, 0, 0, 1641, 724, 1676, 414, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4370, 4223, 3622, 4020, 4361, 3926, 3791, 4455] avg=4096.0000 max=4455 max_rank=7 min=3622 min_rank=2 imbalance_ratio=1.087646 max_load_reduction=28313 max_load_reduction_pct=0.864044 moved_to_echo=16533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[161, 2865, 861, 327, 0, 0, 0, 0, 1274, 667, 761, 1183, 0, 0, 0, 0, 863, 309, 1057, 464, 968, 38, 0, 0, 1189, 1067, 167, 450, 1291, 0, 0, 0, 107, 565, 1481, 807, 632, 652, 0, 0, 917, 160, 778, 1817, 0, 0, 0, 0, 864, 937, 1123, 1509, 222, 33, 0, 0, 1533, 360, 1810, 499, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4214, 3885, 3699, 4164, 4244, 3672, 4688, 4202] avg=4096.0000 max=4688 max_rank=6 min=3672 min_rank=5 imbalance_ratio=1.144531 max_load_reduction=28080 max_load_reduction_pct=0.856934 moved_to_echo=16806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[236, 2823, 692, 641, 0, 0, 0, 0, 1491, 646, 1133, 640, 0, 0, 0, 0, 1225, 164, 1989, 146, 691, 37, 0, 0, 1189, 1211, 243, 519, 1414, 0, 0, 0, 146, 450, 1336, 749, 692, 465, 0, 0, 831, 379, 851, 2064, 0, 0, 0, 0, 602, 876, 1021, 1195, 219, 38, 0, 0, 1384, 319, 1665, 356, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4392, 3910, 4252, 4576, 3838, 4125, 3951, 3724] avg=4096.0000 max=4576 max_rank=3 min=3724 min_rank=7 imbalance_ratio=1.117188 max_load_reduction=28192 max_load_reduction_pct=0.860352 moved_to_echo=15638 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[103, 2850, 814, 315, 0, 0, 0, 0, 1656, 775, 1139, 1132, 0, 0, 0, 0, 1210, 235, 1121, 237, 996, 38, 0, 0, 703, 660, 298, 620, 1467, 0, 0, 0, 203, 805, 769, 1140, 717, 671, 0, 0, 1035, 178, 883, 1918, 0, 0, 0, 0, 633, 856, 1113, 901, 221, 43, 0, 0, 1441, 516, 1843, 513, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4702, 3837, 3748, 4305, 4014, 3767, 4313] avg=4096.0000 max=4702 max_rank=1 min=3748 min_rank=3 imbalance_ratio=1.147949 max_load_reduction=28066 max_load_reduction_pct=0.856506 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11264, 3584, 14336, 3968, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9344, 2048, 10752, 3072, 7680, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1408, 5248, 7936, 7808, 5632, 5120, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 21888, 6784, 3200, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4992, 7808, 8704, 9344, 1792, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8704, 1536, 6912, 15872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 7040, 2304, 4096, 11392, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11776, 4736, 8320, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1026, 2, 1043, 1, 999, 555, 2560, 2357, 209, 0, 1718, 147, 22, 1855, 187, 1976, 455, 906, 0, 4096, 11, 0, 2, 280, 3602, 2, 639, 4096, 3952, 0, 59, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[128, 10, 576, 1, 2016, 3821, 634, 1102, 469, 2, 1220, 1893, 239, 3281, 72, 1263, 23, 1003, 1, 4086, 69, 183, 189, 1, 657, 62, 1596, 4096, 4006, 0, 61, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[277, 1, 994, 138, 3207, 1886, 309, 1054, 960, 2, 58, 3294, 13, 3750, 16, 1566, 267, 1267, 0, 4019, 549, 202, 0, 0, 296, 3, 569, 3925, 4050, 28, 25, 43] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[228, 0, 1280, 234, 3217, 895, 367, 1568, 3, 0, 239, 2371, 5, 3033, 19, 571, 8, 2190, 0, 4062, 149, 0, 4, 0, 2712, 49, 1391, 4036, 4085, 2, 50, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2313, 26, 217, 262, 899, 705, 2897, 2573, 16, 20, 256, 1456, 12, 3299, 0, 1365, 88, 774, 11, 4057, 213, 17, 1, 187, 2646, 1, 298, 3947, 4070, 7, 135, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1864, 0, 3131, 128, 1206, 1076, 2916, 3834, 10, 0, 814, 805, 12, 660, 1, 20, 13, 355, 0, 4092, 23, 0, 1, 13, 3549, 48, 240, 3923, 4000, 0, 34, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[472, 0, 2299, 6, 2394, 814, 70, 660, 4, 1, 2854, 3389, 330, 3549, 6, 960, 307, 241, 0, 4026, 14, 97, 2, 591, 437, 24, 1168, 3840, 4085, 18, 2, 108] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[168, 0, 1813, 4, 1177, 3862, 879, 1933, 28, 0, 2062, 245, 78, 1919, 0, 3051, 1, 17, 73, 4083, 16, 52, 11, 2, 147, 5, 3018, 4085, 4028, 0, 0, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[128, 10, 576, 1, 1009, 824, 0, 0, 0, 3821, 634, 623, 0, 0, 0, 0, 469, 2, 1220, 1893, 479, 149, 55, 13, 239, 3226, 72, 1263, 0, 0, 0, 0, 23, 1003, 1, 3113, 0, 0, 0, 0, 69, 183, 189, 1, 2864, 1007, 0, 0, 657, 62, 1596, 1232, 0, 0, 0, 0, 3993, 0, 61, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2548, 5078, 4280, 4800, 4140, 4313, 3547, 4062] avg=4096.0000 max=5078 max_rank=1 min=2548 min_rank=0 imbalance_ratio=1.239746 max_load_reduction=27690 max_load_reduction_pct=0.845032 moved_to_echo=16062 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[168, 0, 1813, 4, 590, 823, 0, 0, 0, 3862, 879, 1093, 0, 0, 0, 0, 28, 0, 2062, 245, 840, 149, 32, 13, 78, 1887, 0, 3051, 0, 0, 0, 0, 1, 17, 73, 3111, 0, 0, 0, 0, 16, 52, 11, 2, 2856, 587, 0, 0, 147, 5, 3018, 1229, 0, 0, 0, 0, 4015, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3398, 5834, 3369, 5016, 3202, 3524, 4399, 4026] avg=4096.0000 max=5834 max_rank=1 min=3202 min_rank=4 imbalance_ratio=1.424316 max_load_reduction=26934 max_load_reduction_pct=0.821960 moved_to_echo=15151 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1864, 0, 3131, 128, 604, 825, 0, 0, 0, 1076, 2916, 2167, 0, 0, 0, 0, 10, 0, 814, 805, 1667, 149, 11, 13, 12, 649, 1, 20, 0, 0, 0, 0, 13, 355, 0, 3118, 0, 0, 0, 0, 23, 0, 1, 13, 2743, 602, 0, 0, 3549, 48, 240, 1180, 0, 0, 0, 0, 3987, 0, 34, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6552, 6159, 3469, 682, 3486, 3382, 5017, 4021] avg=4096.0000 max=6552 max_rank=0 min=682 min_rank=3 imbalance_ratio=1.599609 max_load_reduction=26216 max_load_reduction_pct=0.800049 moved_to_echo=15906 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2313, 26, 217, 262, 449, 818, 0, 0, 0, 705, 2897, 1454, 0, 0, 0, 0, 16, 20, 256, 1456, 1119, 148, 55, 13, 12, 3244, 0, 1365, 0, 0, 0, 0, 88, 774, 11, 3091, 0, 0, 0, 0, 213, 17, 1, 187, 2759, 450, 0, 0, 2646, 1, 298, 1188, 0, 0, 0, 0, 4057, 7, 135, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 5056, 3083, 4621, 3964, 3627, 4133, 4199] avg=4096.0000 max=5056 max_rank=1 min=3083 min_rank=2 imbalance_ratio=1.234375 max_load_reduction=27712 max_load_reduction_pct=0.845703 moved_to_echo=15923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[228, 0, 1280, 234, 1610, 819, 0, 0, 0, 895, 367, 887, 0, 0, 0, 0, 3, 0, 239, 2371, 681, 148, 51, 13, 5, 2982, 19, 571, 0, 0, 0, 0, 8, 2190, 0, 3095, 0, 0, 0, 0, 149, 0, 4, 0, 2822, 1607, 0, 0, 2712, 49, 1391, 1214, 0, 0, 0, 0, 4072, 2, 50, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 2149, 3506, 3577, 5293, 4582, 5366, 4124] avg=4096.0000 max=5366 max_rank=6 min=2149 min_rank=1 imbalance_ratio=1.310059 max_load_reduction=27402 max_load_reduction_pct=0.836243 moved_to_echo=19365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[472, 0, 2299, 6, 1198, 812, 0, 0, 0, 814, 70, 373, 0, 0, 0, 0, 4, 1, 2854, 3389, 287, 147, 60, 13, 330, 3489, 6, 960, 0, 0, 0, 0, 307, 241, 0, 3067, 0, 0, 0, 0, 14, 97, 2, 591, 2685, 1196, 0, 0, 437, 24, 1168, 1155, 0, 0, 0, 0, 4072, 18, 2, 108, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4787, 1257, 6755, 4785, 3615, 4585, 2784, 4200] avg=4096.0000 max=6755 max_rank=2 min=1257 min_rank=1 imbalance_ratio=1.649170 max_load_reduction=26013 max_load_reduction_pct=0.793854 moved_to_echo=15184 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 2, 1043, 1, 500, 830, 0, 0, 0, 555, 2560, 1329, 0, 0, 0, 0, 209, 0, 1718, 147, 1028, 155, 35, 19, 22, 1820, 187, 1976, 0, 0, 0, 0, 455, 906, 0, 3111, 0, 0, 0, 0, 11, 0, 2, 280, 2866, 499, 0, 0, 3602, 2, 639, 1230, 0, 0, 0, 0, 3933, 0, 59, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3402, 4444, 3311, 4005, 4472, 3658, 5473, 4003] avg=4096.0000 max=5473 max_rank=6 min=3311 min_rank=2 imbalance_ratio=1.336182 max_load_reduction=27295 max_load_reduction_pct=0.832977 moved_to_echo=17606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[277, 1, 994, 138, 1605, 810, 0, 0, 0, 1886, 309, 596, 0, 0, 0, 0, 960, 2, 58, 3294, 458, 147, 63, 13, 13, 3687, 16, 1566, 0, 0, 0, 0, 267, 1267, 0, 3062, 0, 0, 0, 0, 549, 202, 0, 0, 2744, 1602, 0, 0, 296, 3, 569, 1181, 0, 0, 0, 0, 4037, 28, 25, 43, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3825, 2791, 4995, 5282, 4596, 5097, 2049, 4133] avg=4096.0000 max=5282 max_rank=3 min=2049 min_rank=6 imbalance_ratio=1.289551 max_load_reduction=27486 max_load_reduction_pct=0.838806 moved_to_echo=15875 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[6528, 128, 11392, 896, 7680, 6656, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 13696, 10752, 8576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1792, 128, 9344, 13696, 6656, 1280, 384, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14080, 256, 8960, 9728, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 20992, 384, 10880, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1152, 640, 256, 1152, 22400, 7552, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 6784, 128, 24832, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32256, 128, 384, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[10, 3652, 414, 889, 7, 807, 4096, 0, 0, 3786, 25, 112, 20, 1486, 0, 0, 4, 3932, 1286, 0, 69, 3204, 25, 986, 1194, 1, 0, 3, 3291, 611, 0, 2858] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4083, 829, 0, 2714, 0, 0, 3593, 194, 1, 1119, 3, 0, 9, 537, 196, 504, 14, 2872, 3772, 0, 0, 1589, 0, 3878, 0, 2, 438, 11, 3894, 0, 31, 2485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1481, 1033, 47, 1894, 27, 2001, 2680, 0, 204, 3313, 8, 171, 3328, 3824, 0, 0, 543, 4069, 2999, 308, 0, 3902, 261, 319, 128, 0, 0, 1, 199, 28, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[2474, 4, 0, 3798, 15, 8, 4089, 0, 0, 71, 44, 2, 0, 3799, 0, 885, 5, 4096, 2193, 11, 146, 0, 0, 3825, 66, 35, 6, 9, 3305, 80, 2, 3800] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1915, 2082, 12, 3192, 1, 0, 4078, 0, 0, 72, 0, 35, 2, 2615, 154, 108, 261, 3848, 3801, 389, 2, 3215, 274, 4058, 10, 0, 567, 0, 1415, 5, 356, 301] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[765, 676, 3, 3690, 422, 1079, 3989, 24, 0, 4093, 3915, 0, 2795, 3968, 20, 1448, 5, 4095, 296, 101, 8, 0, 162, 516, 159, 0, 14, 119, 103, 135, 0, 168] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3535, 1539, 6, 4007, 10, 0, 4095, 156, 0, 1328, 40, 28, 583, 3285, 17, 98, 0, 3904, 3628, 331, 288, 2257, 255, 2756, 299, 3, 5, 0, 0, 0, 79, 236] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[2995, 1336, 161, 1883, 0, 0, 4037, 1, 0, 3830, 91, 15, 2, 3997, 824, 0, 11, 3975, 716, 52, 814, 316, 0, 1532, 3494, 18, 35, 5, 2597, 10, 0, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2474, 4, 0, 641, 0, 0, 0, 0, 15, 8, 3737, 0, 0, 0, 0, 0, 0, 71, 44, 2, 1800, 0, 0, 0, 0, 3519, 0, 885, 0, 0, 0, 0, 5, 1602, 2193, 11, 0, 0, 0, 0, 146, 0, 0, 3421, 0, 0, 0, 0, 66, 35, 6, 9, 2494, 1273, 0, 0, 3305, 80, 2, 3800, 352, 404, 280, 84] rerouted_tokens_per_ep_rank=[3119, 3760, 1917, 4404, 3811, 3567, 3883, 8307] avg=4096.0000 max=8307 max_rank=7 min=1917 min_rank=2 imbalance_ratio=2.028076 max_load_reduction=24461 max_load_reduction_pct=0.746490 moved_to_echo=19568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[765, 676, 3, 623, 0, 0, 0, 0, 422, 1079, 3646, 24, 0, 0, 0, 0, 0, 4093, 3915, 0, 1749, 0, 0, 0, 2795, 3676, 20, 1448, 0, 0, 0, 0, 5, 1602, 296, 101, 0, 0, 0, 0, 8, 0, 162, 462, 0, 0, 0, 0, 159, 0, 14, 119, 2493, 1236, 0, 0, 103, 135, 0, 168, 343, 54, 292, 82] rerouted_tokens_per_ep_rank=[2067, 5171, 9757, 7939, 2004, 632, 4021, 1177] avg=4096.0000 max=9757 max_rank=2 min=632 min_rank=5 imbalance_ratio=2.382080 max_load_reduction=23011 max_load_reduction_pct=0.702240 moved_to_echo=7834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1481, 1033, 47, 321, 0, 0, 0, 0, 27, 2001, 2450, 0, 0, 0, 0, 0, 204, 3313, 8, 171, 897, 0, 0, 0, 3328, 3542, 0, 0, 0, 0, 0, 0, 543, 1592, 2999, 308, 0, 0, 0, 0, 0, 3902, 261, 286, 0, 0, 0, 0, 128, 0, 0, 1, 2477, 634, 0, 0, 199, 28, 0, 0, 230, 33, 282, 42] rerouted_tokens_per_ep_rank=[2882, 4478, 4593, 6870, 5442, 4449, 3240, 814] avg=4096.0000 max=6870 max_rank=3 min=814 min_rank=7 imbalance_ratio=1.677246 max_load_reduction=25898 max_load_reduction_pct=0.790344 moved_to_echo=13945 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4083, 829, 0, 459, 0, 0, 0, 0, 0, 0, 3284, 194, 0, 0, 0, 0, 1, 1119, 3, 0, 1286, 0, 0, 0, 9, 498, 196, 504, 0, 0, 0, 0, 14, 1124, 3772, 0, 0, 0, 0, 0, 0, 1589, 0, 3469, 0, 0, 0, 0, 0, 2, 438, 11, 1748, 909, 0, 0, 3894, 0, 31, 2485, 309, 409, 39, 60] rerouted_tokens_per_ep_rank=[5371, 3478, 2409, 1207, 4910, 5058, 3108, 7227] avg=4096.0000 max=7227 max_rank=7 min=1207 min_rank=3 imbalance_ratio=1.764404 max_load_reduction=25541 max_load_reduction_pct=0.779449 moved_to_echo=20303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1915, 2082, 12, 539, 0, 0, 0, 0, 1, 0, 3727, 0, 0, 0, 0, 0, 0, 72, 0, 35, 1513, 0, 0, 0, 2, 2422, 154, 108, 0, 0, 0, 0, 261, 1505, 3801, 389, 0, 0, 0, 0, 2, 3215, 274, 3630, 0, 0, 0, 0, 10, 0, 567, 0, 2343, 1069, 0, 0, 1415, 5, 356, 301, 351, 428, 193, 71] rerouted_tokens_per_ep_rank=[4548, 3728, 1620, 2686, 5956, 7121, 3989, 3120] avg=4096.0000 max=7121 max_rank=5 min=1620 min_rank=2 imbalance_ratio=1.738525 max_load_reduction=25647 max_load_reduction_pct=0.782684 moved_to_echo=20186 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[10, 3652, 414, 138, 0, 0, 0, 0, 7, 807, 3739, 0, 0, 0, 0, 0, 0, 3786, 25, 112, 426, 0, 0, 0, 20, 1373, 0, 0, 0, 0, 0, 0, 4, 1535, 1286, 0, 0, 0, 0, 0, 69, 3204, 25, 877, 0, 0, 0, 0, 1194, 1, 0, 3, 2397, 302, 0, 0, 3291, 611, 0, 2858, 357, 109, 113, 23] rerouted_tokens_per_ep_rank=[4214, 4553, 4349, 1393, 2825, 4175, 3897, 7362] avg=4096.0000 max=7362 max_rank=7 min=1393 min_rank=3 imbalance_ratio=1.797363 max_load_reduction=25406 max_load_reduction_pct=0.775330 moved_to_echo=18259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2995, 1336, 161, 319, 0, 0, 0, 0, 0, 0, 3690, 1, 0, 0, 0, 0, 0, 3830, 91, 15, 892, 0, 0, 0, 2, 3702, 824, 0, 0, 0, 0, 0, 11, 1555, 716, 52, 0, 0, 0, 0, 814, 316, 0, 1371, 0, 0, 0, 0, 3494, 18, 35, 5, 2420, 631, 0, 0, 2597, 10, 0, 21, 347, 161, 295, 41] rerouted_tokens_per_ep_rank=[4811, 3691, 4828, 4528, 2334, 2501, 6603, 3472] avg=4096.0000 max=6603 max_rank=6 min=2334 min_rank=4 imbalance_ratio=1.612061 max_load_reduction=26165 max_load_reduction_pct=0.798492 moved_to_echo=14910 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3535, 1539, 6, 676, 0, 0, 0, 0, 10, 0, 3743, 156, 0, 0, 0, 0, 0, 1328, 40, 28, 1899, 0, 0, 0, 583, 3043, 17, 98, 0, 0, 0, 0, 0, 1527, 3628, 331, 0, 0, 0, 0, 288, 2257, 255, 2465, 0, 0, 0, 0, 299, 3, 5, 0, 2377, 1343, 0, 0, 0, 0, 79, 236, 352, 291, 242, 89] rerouted_tokens_per_ep_rank=[5756, 3909, 3295, 3741, 5486, 5265, 4027, 1289] avg=4096.0000 max=5756 max_rank=0 min=1289 min_rank=7 imbalance_ratio=1.405273 max_load_reduction=27012 max_load_reduction_pct=0.824341 moved_to_echo=16067 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14848, 896, 512, 9984, 2688, 1920, 1792, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6784, 21888, 1280, 3072, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 17664, 4224, 384, 10496, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 12160, 18816, 1280, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1408, 14592, 1024, 16000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[17280, 11264, 768, 3840, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5376, 128, 1152, 256, 18816, 7424, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[512, 3968, 28032, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[3991, 70, 1, 2224, 2712, 342, 0, 650, 58, 2589, 62, 0, 4080, 4066, 30, 2282, 129, 28, 4, 553, 0, 0, 6, 4013, 0, 153, 8, 4064, 0, 653, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4072, 0, 0, 1, 113, 0, 2550, 0, 2222, 3478, 6, 18, 4088, 1553, 1286, 0, 0, 4086, 0, 3191, 0, 0, 0, 2614, 123, 718, 126, 411, 0, 0, 0, 2112] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[679, 0, 0, 4094, 2017, 1458, 6, 0, 78, 3366, 0, 0, 4093, 4, 0, 1728, 0, 3835, 0, 1841, 7, 42, 206, 4093, 0, 44, 680, 4049, 397, 1, 0, 50] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[3181, 0, 1, 3979, 7, 2867, 514, 4, 0, 4092, 70, 93, 4092, 1, 92, 4, 0, 17, 2686, 4079, 983, 0, 0, 578, 0, 17, 100, 898, 121, 24, 182, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2407, 42, 183, 2654, 445, 4071, 1, 0, 15, 2696, 14, 73, 432, 0, 759, 63, 2, 1811, 2942, 1598, 0, 0, 0, 3314, 1, 3, 0, 1346, 2166, 4, 1636, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[84, 4095, 108, 3844, 1700, 3243, 4049, 0, 1, 1091, 0, 7, 3288, 44, 907, 679, 166, 3806, 4, 725, 0, 0, 0, 2530, 0, 0, 0, 65, 1400, 8, 158, 766] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4087, 0, 0, 4076, 144, 174, 85, 0, 0, 134, 3, 11, 4096, 0, 30, 151, 59, 3750, 4076, 1, 97, 0, 0, 3781, 0, 3673, 2, 4085, 18, 0, 54, 181] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3977, 270, 11, 4070, 21, 42, 0, 0, 821, 3165, 0, 26, 3970, 0, 2797, 450, 0, 2613, 192, 2, 42, 0, 0, 4047, 2, 2316, 0, 51, 11, 17, 1, 3854] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 0, 901, 0, 0, 0, 0, 144, 174, 85, 0, 1044, 0, 0, 0, 0, 134, 3, 11, 1248, 0, 0, 0, 2308, 0, 30, 151, 0, 0, 0, 0, 59, 1978, 4076, 1, 0, 0, 0, 0, 97, 0, 0, 3781, 728, 369, 0, 0, 0, 3673, 2, 4085, 1414, 171, 0, 0, 18, 0, 54, 181, 1761, 0, 0, 0] rerouted_tokens_per_ep_rank=[4988, 1447, 1396, 2489, 6114, 4975, 9345, 2014] avg=4096.0000 max=9345 max_rank=6 min=1396 min_rank=2 imbalance_ratio=2.281494 max_load_reduction=23423 max_load_reduction_pct=0.714813 moved_to_echo=22448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3991, 70, 1, 492, 0, 0, 0, 0, 2712, 342, 0, 650, 7, 0, 0, 0, 58, 2589, 62, 0, 1243, 0, 0, 0, 2299, 4066, 30, 2282, 0, 0, 0, 0, 129, 16, 4, 553, 0, 0, 0, 0, 0, 0, 6, 4013, 5, 368, 0, 0, 0, 153, 8, 4064, 771, 170, 0, 0, 0, 653, 0, 0, 961, 0, 0, 0] rerouted_tokens_per_ep_rank=[4554, 3711, 3952, 8677, 702, 4392, 5166, 1614] avg=4096.0000 max=8677 max_rank=3 min=702 min_rank=4 imbalance_ratio=2.118408 max_load_reduction=24091 max_load_reduction_pct=0.735199 moved_to_echo=11874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3181, 0, 1, 872, 0, 0, 0, 0, 7, 2867, 514, 4, 8, 0, 0, 0, 0, 4092, 70, 93, 1252, 0, 0, 0, 2293, 1, 92, 4, 0, 0, 0, 0, 0, 2, 2686, 4079, 0, 0, 0, 0, 983, 0, 0, 578, 7, 373, 0, 0, 0, 17, 100, 898, 1385, 174, 0, 0, 121, 24, 182, 4086, 1722, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 3400, 5507, 2390, 6767, 1941, 2574, 6135] avg=4096.0000 max=6767 max_rank=4 min=1941 min_rank=5 imbalance_ratio=1.652100 max_load_reduction=26001 max_load_reduction_pct=0.793488 moved_to_echo=17417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4072, 0, 0, 1, 0, 0, 0, 0, 113, 0, 2550, 0, 1137, 0, 0, 0, 2222, 3478, 6, 18, 1246, 0, 0, 0, 2302, 1553, 1286, 0, 0, 0, 0, 0, 0, 2156, 0, 3191, 0, 0, 0, 0, 0, 0, 0, 2614, 793, 369, 0, 0, 123, 718, 126, 411, 0, 171, 0, 0, 0, 0, 0, 2112, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 3800, 6970, 5141, 5347, 3776, 1549, 2112] avg=4096.0000 max=6970 max_rank=2 min=1549 min_rank=6 imbalance_ratio=1.701660 max_load_reduction=25798 max_load_reduction_pct=0.787292 moved_to_echo=12784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[679, 0, 0, 905, 0, 0, 0, 0, 2017, 1458, 6, 0, 1067, 0, 0, 0, 78, 3366, 0, 0, 1247, 0, 0, 0, 2306, 4, 0, 1728, 0, 0, 0, 0, 0, 2023, 0, 1841, 0, 0, 0, 0, 7, 42, 206, 4093, 745, 369, 0, 0, 0, 44, 680, 4049, 1420, 171, 0, 0, 397, 1, 0, 50, 1769, 0, 0, 0] rerouted_tokens_per_ep_rank=[1584, 4548, 4691, 4038, 3864, 5462, 6364, 2217] avg=4096.0000 max=6364 max_rank=6 min=1584 min_rank=0 imbalance_ratio=1.553711 max_load_reduction=26404 max_load_reduction_pct=0.805786 moved_to_echo=17907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2407, 42, 183, 588, 0, 0, 0, 0, 445, 4071, 1, 0, 504, 0, 0, 0, 15, 2696, 14, 73, 131, 0, 0, 0, 245, 0, 759, 63, 0, 0, 0, 0, 2, 956, 2942, 1598, 0, 0, 0, 0, 0, 0, 0, 3314, 351, 38, 0, 0, 1, 3, 0, 1346, 920, 18, 0, 0, 2166, 4, 1636, 4090, 1146, 0, 0, 0] rerouted_tokens_per_ep_rank=[3220, 5021, 2929, 1067, 5498, 3703, 2288, 9042] avg=4096.0000 max=9042 max_rank=7 min=1067 min_rank=3 imbalance_ratio=2.207520 max_load_reduction=23726 max_load_reduction_pct=0.724060 moved_to_echo=20531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[84, 4095, 108, 850, 0, 0, 0, 0, 1700, 3243, 4049, 0, 1059, 0, 0, 0, 1, 1091, 0, 7, 1002, 0, 0, 0, 1853, 44, 907, 679, 0, 0, 0, 0, 166, 2008, 4, 725, 0, 0, 0, 0, 0, 0, 0, 2530, 739, 296, 0, 0, 0, 0, 0, 65, 1333, 137, 0, 0, 1400, 8, 158, 766, 1661, 0, 0, 0] rerouted_tokens_per_ep_rank=[5137, 10051, 2101, 3483, 2903, 3565, 1535, 3993] avg=4096.0000 max=10051 max_rank=1 min=1535 min_rank=6 imbalance_ratio=2.453857 max_load_reduction=22717 max_load_reduction_pct=0.693268 moved_to_echo=11996 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3977, 270, 11, 900, 0, 0, 0, 0, 21, 42, 0, 0, 727, 0, 0, 0, 821, 3165, 0, 26, 1210, 0, 0, 0, 2236, 0, 2797, 450, 0, 0, 0, 0, 0, 1379, 192, 2, 0, 0, 0, 0, 42, 0, 0, 4047, 507, 358, 0, 0, 2, 2316, 0, 51, 1412, 166, 0, 0, 11, 17, 1, 3854, 1758, 0, 0, 0] rerouted_tokens_per_ep_rank=[5158, 790, 5222, 5483, 1573, 4954, 3947, 5641] avg=4096.0000 max=5641 max_rank=7 min=790 min_rank=1 imbalance_ratio=1.377197 max_load_reduction=27127 max_load_reduction_pct=0.827850 moved_to_echo=16115 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 10624, 9984, 12032, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 768, 2048, 15232, 10880, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7168, 12288, 7296, 768, 5632, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15872, 5760, 6016, 5376, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7040, 1024, 14976, 8704, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[22528, 4480, 384, 5632, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3200, 20736, 256, 256, 8704, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1152, 128, 256, 25088, 3968, 2560, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[413, 1042, 794, 9, 1481, 0, 0, 1032, 0, 6, 3167, 3520, 4095, 377, 13, 3533, 11, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 3772, 0, 0, 167, 2928, 2548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[11, 490, 731, 19, 3036, 2, 13, 3628, 625, 0, 2843, 3139, 4073, 0, 4020, 261, 10, 0, 0, 916, 0, 878, 0, 2541, 407, 12, 1484, 176, 25, 1014, 2413, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[3909, 0, 3253, 7, 14, 0, 277, 3968, 4096, 140, 779, 3483, 0, 140, 165, 0, 0, 0, 140, 170, 4096, 0, 6, 20, 18, 450, 3714, 0, 1, 0, 3919, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4019, 162, 3956, 27, 823, 0, 0, 4096, 3923, 0, 1813, 3223, 136, 1548, 0, 0, 94, 0, 3985, 127, 0, 78, 1, 207, 21, 457, 68, 0, 25, 0, 3973, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 774, 8, 28, 138, 1336, 3290, 2, 35, 21, 3917, 1872, 6, 83, 1, 0, 0, 3320, 8, 2, 1079, 876, 2833, 98, 4096, 4067, 0, 11, 234, 1131, 3502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3556, 0, 3081, 56, 2500, 0, 0, 540, 844, 136, 4082, 4011, 164, 4076, 9, 0, 2201, 4, 46, 124, 0, 1045, 0, 451, 0, 4, 0, 0, 0, 1748, 4090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 66, 5, 39, 78, 2116, 2717, 3, 3646, 4037, 599, 0, 0, 2190, 4093, 51, 0, 0, 140, 1, 0, 4022, 1, 5, 436, 0, 0, 176, 4088, 163] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1394, 2, 4043, 471, 1250, 57, 2576, 4096, 68, 319, 10, 3963, 0, 229, 3902, 0, 124, 0, 3999, 36, 663, 161, 54, 12, 11, 12, 7, 3, 0, 1314, 3992, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3184, 162, 3956, 27, 0, 0, 0, 0, 823, 0, 0, 3440, 0, 0, 0, 0, 3923, 0, 1813, 386, 0, 0, 0, 0, 136, 1548, 0, 0, 294, 0, 0, 0, 94, 0, 3985, 127, 1358, 143, 0, 0, 0, 78, 1, 207, 560, 656, 541, 0, 21, 457, 68, 0, 1479, 0, 0, 0, 25, 0, 3270, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7329, 4263, 6122, 1978, 5707, 2043, 2025, 3301] avg=4096.0000 max=7329 max_rank=0 min=1978 min_rank=3 imbalance_ratio=1.789307 max_load_reduction=25439 max_load_reduction_pct=0.776337 moved_to_echo=13076 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2817, 0, 3081, 56, 0, 0, 0, 0, 2500, 0, 0, 454, 0, 0, 0, 0, 844, 136, 4082, 479, 0, 0, 0, 0, 164, 4076, 9, 0, 260, 0, 0, 0, 2201, 4, 46, 124, 1691, 147, 0, 0, 0, 1045, 0, 451, 577, 86, 479, 0, 0, 4, 0, 0, 1841, 0, 0, 0, 0, 1748, 3366, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5954, 2954, 5541, 4509, 4213, 2638, 1845, 5114] avg=4096.0000 max=5954 max_rank=0 min=1845 min_rank=6 imbalance_ratio=1.453613 max_load_reduction=26814 max_load_reduction_pct=0.818298 moved_to_echo=13810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3097, 0, 3253, 7, 0, 0, 0, 0, 14, 0, 277, 3332, 0, 0, 0, 0, 4096, 140, 779, 416, 0, 0, 0, 0, 0, 140, 165, 0, 286, 0, 0, 0, 0, 0, 140, 170, 1468, 141, 0, 0, 4096, 0, 6, 20, 553, 636, 526, 0, 18, 450, 3714, 0, 1599, 0, 0, 0, 1, 0, 3225, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6357, 3623, 5431, 591, 1919, 5837, 5781, 3229] avg=4096.0000 max=6357 max_rank=0 min=591 min_rank=3 imbalance_ratio=1.552002 max_load_reduction=26411 max_load_reduction_pct=0.806000 moved_to_echo=16766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[320, 1042, 794, 9, 0, 0, 0, 0, 1481, 0, 0, 862, 0, 0, 0, 0, 0, 6, 3167, 413, 0, 0, 0, 0, 4095, 377, 13, 3533, 34, 0, 0, 0, 11, 0, 0, 0, 1488, 109, 0, 0, 0, 0, 3860, 0, 417, 170, 59, 0, 0, 0, 3772, 0, 1619, 0, 0, 0, 0, 167, 2402, 2548, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2165, 2343, 3586, 8052, 1608, 4506, 5391, 5117] avg=4096.0000 max=8052 max_rank=3 min=1608 min_rank=4 imbalance_ratio=1.965820 max_load_reduction=24716 max_load_reduction_pct=0.754272 moved_to_echo=16622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[10, 490, 731, 19, 0, 0, 0, 0, 3036, 2, 13, 3047, 0, 0, 0, 0, 625, 0, 2843, 375, 0, 0, 0, 0, 4073, 0, 4020, 261, 0, 0, 0, 0, 10, 0, 0, 916, 1323, 87, 0, 0, 0, 878, 0, 2541, 340, 581, 1, 0, 407, 12, 1484, 176, 1441, 0, 0, 0, 25, 1014, 1986, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1250, 6098, 3843, 8354, 2336, 4341, 3520, 3026] avg=4096.0000 max=8354 max_rank=3 min=1250 min_rank=0 imbalance_ratio=2.039551 max_load_reduction=24414 max_load_reduction_pct=0.745056 moved_to_echo=13223 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3244, 0, 0, 66, 0, 0, 0, 0, 5, 39, 78, 1777, 0, 0, 0, 0, 2717, 3, 3646, 482, 0, 0, 0, 0, 599, 0, 0, 2190, 300, 0, 0, 0, 4093, 51, 0, 0, 1702, 147, 0, 0, 140, 1, 0, 4022, 577, 339, 552, 0, 1, 5, 436, 0, 1853, 0, 0, 0, 0, 176, 3364, 163, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3310, 1899, 6848, 3089, 5993, 5631, 2295, 3703] avg=4096.0000 max=6848 max_rank=2 min=1899 min_rank=1 imbalance_ratio=1.671875 max_load_reduction=25920 max_load_reduction_pct=0.791016 moved_to_echo=17622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 774, 8, 0, 0, 0, 0, 28, 138, 1336, 2763, 0, 0, 0, 0, 2, 35, 21, 468, 0, 0, 0, 0, 1872, 6, 83, 1, 0, 0, 0, 0, 0, 0, 3320, 8, 1651, 40, 0, 0, 2, 1079, 876, 2833, 159, 527, 0, 0, 98, 4096, 4067, 0, 1798, 0, 0, 0, 11, 234, 932, 3502, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[782, 4265, 526, 1962, 5019, 5476, 10059, 4679] avg=4096.0000 max=10059 max_rank=6 min=526 min_rank=2 imbalance_ratio=2.455811 max_load_reduction=22709 max_load_reduction_pct=0.693024 moved_to_echo=25233 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1105, 2, 4043, 471, 0, 0, 0, 0, 1250, 57, 2576, 3440, 0, 0, 0, 0, 68, 319, 10, 474, 0, 0, 0, 0, 0, 229, 3902, 0, 102, 0, 0, 0, 124, 0, 3999, 36, 1670, 144, 0, 0, 663, 161, 54, 12, 563, 656, 187, 0, 11, 12, 7, 3, 1819, 0, 0, 0, 0, 1314, 3285, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5621, 7323, 871, 4233, 5973, 2296, 1852, 4599] avg=4096.0000 max=7323 max_rank=1 min=871 min_rank=2 imbalance_ratio=1.787842 max_load_reduction=25445 max_load_reduction_pct=0.776520 moved_to_echo=14720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13824, 1792, 16640, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[11008, 6400, 8192, 6016, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 5120, 13568, 256, 13568, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6656, 128, 11520, 1408, 12416, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[12288, 640, 16384, 3584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4992, 3328, 4864, 10112, 3840, 3712, 2432, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 4736, 21888, 6272, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9216, 256, 4352, 19200, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[829, 0, 4076, 63, 104, 0, 0, 9, 1326, 0, 2653, 3482, 0, 3711, 1045, 732, 0, 0, 0, 3936, 475, 0, 0, 0, 892, 4027, 1, 2480, 2901, 0, 5, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[167, 0, 0, 0, 0, 892, 3894, 5, 14, 207, 10, 4006, 1, 4077, 4003, 46, 3662, 3813, 77, 0, 448, 89, 4, 0, 3, 3050, 59, 3718, 0, 160, 1, 362] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[29, 0, 564, 4, 352, 25, 1747, 0, 0, 4079, 15, 3173, 0, 3220, 3215, 2426, 0, 0, 139, 1483, 3167, 30, 0, 909, 0, 4094, 11, 3896, 187, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 209, 6, 0, 1626, 32, 3054, 1695, 61, 4091, 2963, 0, 3876, 2492, 4067, 0, 0, 0, 0, 0, 8, 0, 156, 0, 4094, 2, 782, 1460, 2094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 3, 10, 65, 0, 0, 3255, 4071, 0, 0, 3644, 4096, 0, 1639, 3, 0, 841, 0, 0, 0, 381, 0, 0, 4, 58, 2792, 1, 4096, 3731, 4071, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[5, 0, 4091, 0, 458, 0, 419, 4, 0, 1234, 0, 3847, 63, 4081, 3373, 3431, 6, 20, 1871, 0, 0, 96, 1, 0, 0, 4092, 12, 3920, 0, 1479, 53, 212] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[3647, 0, 0, 0, 2, 111, 2094, 0, 0, 0, 85, 4090, 26, 3989, 2813, 0, 5, 0, 0, 0, 276, 9, 1, 0, 0, 3528, 1327, 4003, 4056, 1, 0, 2705] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[506, 11, 0, 1, 0, 0, 3604, 518, 3402, 220, 3979, 4096, 0, 2823, 4085, 2, 0, 16, 62, 0, 590, 441, 218, 1600, 1870, 67, 3616, 827, 1, 213, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[167, 0, 0, 0, 1880, 387, 0, 0, 0, 892, 3894, 5, 874, 0, 0, 0, 14, 207, 10, 816, 0, 0, 0, 0, 1, 142, 4003, 46, 0, 0, 0, 0, 3662, 3813, 77, 0, 1027, 1099, 0, 0, 448, 89, 4, 0, 3548, 0, 0, 0, 3, 143, 59, 3718, 0, 0, 0, 0, 0, 160, 1, 362, 1217, 0, 0, 0] rerouted_tokens_per_ep_rank=[2434, 5665, 1047, 4192, 9678, 4089, 3923, 1740] avg=4096.0000 max=9678 max_rank=4 min=1047 min_rank=2 imbalance_ratio=2.362793 max_load_reduction=23090 max_load_reduction_pct=0.704651 moved_to_echo=19430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 4091, 0, 2523, 387, 0, 0, 458, 0, 419, 4, 839, 0, 0, 0, 0, 1234, 0, 784, 0, 0, 0, 0, 63, 142, 3373, 3431, 0, 0, 0, 0, 6, 20, 1871, 0, 1378, 1055, 0, 0, 0, 96, 1, 0, 3552, 0, 0, 0, 0, 191, 12, 3920, 0, 0, 0, 0, 0, 1479, 53, 212, 1169, 0, 0, 0] rerouted_tokens_per_ep_rank=[7006, 1720, 2018, 7009, 4330, 3649, 4123, 2913] avg=4096.0000 max=7009 max_rank=3 min=1720 min_rank=1 imbalance_ratio=1.711182 max_load_reduction=25759 max_load_reduction_pct=0.786102 moved_to_echo=15015 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[829, 0, 4076, 63, 2483, 352, 0, 0, 104, 0, 0, 9, 759, 0, 0, 0, 1326, 0, 2653, 710, 0, 0, 0, 0, 0, 129, 1045, 732, 0, 0, 0, 0, 0, 0, 0, 3936, 1356, 955, 0, 0, 475, 0, 0, 0, 3230, 0, 0, 0, 892, 188, 1, 2480, 0, 0, 0, 0, 2901, 0, 5, 21, 1058, 0, 0, 0] rerouted_tokens_per_ep_rank=[7803, 872, 4689, 1906, 6247, 3705, 3561, 3985] avg=4096.0000 max=7803 max_rank=0 min=872 min_rank=1 imbalance_ratio=1.905029 max_load_reduction=24965 max_load_reduction_pct=0.761871 moved_to_echo=17498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 10, 65, 1726, 159, 0, 0, 0, 0, 3255, 4071, 897, 0, 0, 0, 0, 0, 3644, 823, 0, 0, 0, 0, 0, 49, 3, 0, 0, 0, 0, 0, 841, 0, 0, 0, 945, 1127, 0, 0, 381, 0, 0, 4, 1431, 0, 0, 0, 58, 121, 1, 4096, 0, 0, 0, 0, 3731, 4071, 7, 0, 1249, 0, 0, 0] rerouted_tokens_per_ep_rank=[1963, 8223, 4467, 52, 2913, 1816, 4276, 9058] avg=4096.0000 max=9058 max_rank=7 min=52 min_rank=3 imbalance_ratio=2.211426 max_load_reduction=23710 max_load_reduction_pct=0.723572 moved_to_echo=18063 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 564, 4, 2524, 306, 0, 0, 352, 25, 1747, 0, 692, 0, 0, 0, 0, 4079, 15, 647, 0, 0, 0, 0, 0, 112, 3215, 2426, 0, 0, 0, 0, 0, 0, 139, 1483, 1379, 870, 0, 0, 3167, 30, 0, 909, 2802, 0, 0, 0, 0, 191, 11, 3896, 0, 0, 0, 0, 187, 0, 0, 3, 964, 0, 0, 0] rerouted_tokens_per_ep_rank=[3427, 2816, 4741, 5753, 3871, 6908, 4098, 1154] avg=4096.0000 max=6908 max_rank=5 min=1154 min_rank=7 imbalance_ratio=1.686523 max_load_reduction=25860 max_load_reduction_pct=0.789185 moved_to_echo=16031 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[506, 11, 0, 1, 41, 268, 0, 0, 0, 0, 3604, 518, 893, 0, 0, 0, 3402, 220, 3979, 835, 0, 0, 0, 0, 0, 98, 4085, 2, 0, 0, 0, 0, 0, 16, 62, 0, 22, 1123, 0, 0, 590, 441, 218, 1600, 2457, 0, 0, 0, 1870, 4, 3616, 827, 0, 0, 0, 0, 1, 213, 0, 0, 1245, 0, 0, 0] rerouted_tokens_per_ep_rank=[827, 5015, 8436, 4185, 1223, 5306, 6317, 1459] avg=4096.0000 max=8436 max_rank=2 min=827 min_rank=0 imbalance_ratio=2.059570 max_load_reduction=24332 max_load_reduction_pct=0.742554 moved_to_echo=14305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 209, 6, 2524, 368, 0, 0, 0, 1626, 32, 3054, 646, 0, 0, 0, 1695, 61, 4091, 605, 0, 0, 0, 0, 0, 135, 2492, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 1379, 812, 0, 0, 0, 8, 0, 156, 3373, 0, 0, 0, 0, 191, 2, 782, 0, 0, 0, 0, 1460, 2094, 0, 0, 900, 0, 0, 0] rerouted_tokens_per_ep_rank=[3107, 5358, 6452, 6694, 2191, 3537, 975, 4454] avg=4096.0000 max=6694 max_rank=3 min=975 min_rank=6 imbalance_ratio=1.634277 max_load_reduction=26074 max_load_reduction_pct=0.795715 moved_to_echo=11157 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3647, 0, 0, 0, 2175, 379, 0, 0, 2, 111, 2094, 0, 892, 0, 0, 0, 0, 0, 85, 833, 0, 0, 0, 0, 26, 138, 2813, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1188, 1122, 0, 0, 276, 9, 1, 0, 3472, 0, 0, 0, 0, 165, 1327, 4003, 0, 0, 0, 0, 4056, 1, 0, 2705, 1243, 0, 0, 0] rerouted_tokens_per_ep_rank=[6201, 3099, 918, 2977, 2315, 3758, 5495, 8005] avg=4096.0000 max=8005 max_rank=7 min=918 min_rank=2 imbalance_ratio=1.954346 max_load_reduction=24763 max_load_reduction_pct=0.755707 moved_to_echo=19573 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12416, 8064, 128, 3328, 9088, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5248, 128, 8960, 256, 16000, 2688, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6528, 5888, 14592, 6144, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4608, 3968, 2176, 5504, 8704, 8192, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1024, 2688, 15104, 7680, 6528, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 1024, 21120, 10752, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5376, 768, 256, 2688, 23936, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2944, 1280, 5120, 23808, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[16, 0, 0, 3866, 1958, 1, 1, 802, 23, 3917, 140, 3636, 1021, 1, 430, 3949, 4060, 231, 4095, 0, 1, 0, 283, 0, 98, 0, 2, 3484, 0, 719, 34, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 1101, 14, 667, 13, 1460, 1940, 3061, 0, 38, 3440, 29, 0, 503, 2091, 3852, 22, 307, 17, 156, 1, 0, 4045, 1, 0, 0, 3487, 3794, 2724, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[42, 0, 622, 972, 0, 878, 4091, 4048, 827, 19, 3767, 754, 0, 286, 0, 3633, 3747, 2224, 2097, 433, 5, 0, 57, 119, 0, 0, 0, 3849, 0, 0, 298, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 2, 2523, 3327, 233, 0, 1207, 3356, 4060, 668, 65, 1509, 1408, 0, 15, 487, 3734, 4080, 0, 0, 0, 0, 43, 3713, 2090, 0, 0, 246, 0, 1, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3850, 0, 0, 3995, 3263, 52, 125, 4, 3796, 1388, 4, 159, 0, 0, 894, 1, 4069, 1706, 0, 0, 309, 0, 48, 753, 340, 0, 0, 121, 13, 4081, 3797, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1023, 0, 0, 4096, 3, 0, 0, 4082, 3825, 5, 4084, 814, 3, 0, 3764, 0, 3664, 3014, 25, 0, 0, 0, 74, 63, 8, 0, 49, 3985, 176, 0, 11, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 10, 354, 0, 3037, 111, 4096, 0, 3188, 0, 7, 3924, 4089, 0, 542, 0, 9, 1, 1310, 0, 21, 2, 2, 4067, 0, 0, 0, 4095, 29, 600, 0, 3274] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 3, 72, 17, 0, 0, 0, 4096, 1000, 4088, 4095, 3404, 1883, 0, 3213, 0, 1858, 3558, 0, 72, 1, 0, 0, 2931, 391, 0, 2, 70, 0, 4, 23, 1987] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1101, 14, 903, 236, 0, 0, 667, 13, 1460, 1225, 0, 0, 0, 0, 0, 0, 38, 2045, 0, 0, 0, 0, 29, 0, 503, 2091, 479, 0, 0, 0, 1477, 22, 307, 17, 0, 0, 0, 0, 156, 1, 0, 4045, 2487, 0, 0, 0, 1, 0, 0, 3487, 1221, 492, 0, 0, 3794, 2724, 0, 5, 1154, 574, 0, 0] rerouted_tokens_per_ep_rank=[2254, 3365, 2083, 3102, 1823, 6689, 5201, 8251] avg=4096.0000 max=8251 max_rank=7 min=1823 min_rank=4 imbalance_ratio=2.014404 max_load_reduction=24517 max_load_reduction_pct=0.748199 moved_to_echo=21964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 3866, 954, 97, 0, 0, 1958, 1, 1, 507, 0, 0, 0, 0, 0, 3917, 140, 2162, 0, 0, 0, 0, 1021, 1, 430, 3949, 198, 0, 0, 0, 1557, 231, 4095, 0, 0, 0, 0, 0, 1, 0, 283, 0, 18, 0, 0, 0, 98, 0, 2, 3484, 1287, 520, 0, 0, 0, 719, 34, 0, 1216, 5, 0, 0] rerouted_tokens_per_ep_rank=[4933, 2467, 6219, 5599, 5883, 302, 5391, 1974] avg=4096.0000 max=6219 max_rank=2 min=302 min_rank=5 imbalance_ratio=1.518311 max_load_reduction=26549 max_load_reduction_pct=0.810211 moved_to_echo=13550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1023, 0, 0, 4096, 213, 496, 0, 0, 3, 0, 0, 2578, 0, 0, 0, 0, 0, 5, 4084, 485, 0, 0, 0, 0, 3, 0, 3764, 0, 1008, 0, 0, 0, 1405, 3014, 25, 0, 0, 0, 0, 0, 0, 0, 74, 63, 3109, 0, 0, 0, 8, 0, 49, 3985, 1161, 116, 0, 0, 176, 0, 11, 0, 1098, 716, 0, 0] rerouted_tokens_per_ep_rank=[5828, 2581, 4574, 4775, 4444, 3246, 5319, 2001] avg=4096.0000 max=5828 max_rank=0 min=2001 min_rank=7 imbalance_ratio=1.422852 max_load_reduction=26940 max_load_reduction_pct=0.822144 moved_to_echo=15010 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 354, 0, 1030, 0, 0, 0, 3037, 111, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 2333, 0, 0, 0, 0, 4089, 0, 542, 0, 0, 0, 0, 0, 5, 1, 1310, 0, 0, 0, 0, 0, 21, 2, 2, 4067, 2591, 0, 0, 0, 0, 0, 0, 4095, 2, 561, 0, 0, 29, 600, 0, 3274, 2, 597, 0, 0] rerouted_tokens_per_ep_rank=[1394, 7244, 2340, 4631, 1316, 6683, 4658, 4502] avg=4096.0000 max=7244 max_rank=1 min=1316 min_rank=4 imbalance_ratio=1.768555 max_load_reduction=25524 max_load_reduction_pct=0.778931 moved_to_echo=17159 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 72, 17, 898, 501, 0, 0, 0, 0, 0, 2581, 0, 0, 0, 0, 0, 4088, 4095, 2015, 0, 0, 0, 0, 1883, 0, 3213, 0, 1014, 0, 0, 0, 705, 3558, 0, 72, 0, 0, 0, 0, 1, 0, 0, 2931, 813, 0, 0, 0, 391, 0, 2, 70, 593, 491, 0, 0, 0, 4, 23, 1987, 560, 187, 0, 0] rerouted_tokens_per_ep_rank=[1491, 2581, 10198, 6110, 4335, 3745, 1547, 2761] avg=4096.0000 max=10198 max_rank=2 min=1491 min_rank=0 imbalance_ratio=2.489746 max_load_reduction=22570 max_load_reduction_pct=0.688782 moved_to_echo=12388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3850, 0, 0, 3995, 41, 0, 0, 0, 3263, 52, 125, 4, 0, 0, 0, 0, 0, 1388, 4, 96, 0, 0, 0, 0, 0, 0, 894, 1, 0, 0, 0, 0, 1560, 1706, 0, 0, 0, 0, 0, 0, 309, 0, 48, 753, 3084, 0, 0, 0, 340, 0, 0, 121, 1290, 22, 0, 0, 13, 4081, 3797, 0, 1219, 712, 0, 0] rerouted_tokens_per_ep_rank=[7886, 3444, 1488, 895, 3266, 4194, 1773, 9822] avg=4096.0000 max=9822 max_rank=7 min=895 min_rank=3 imbalance_ratio=2.397949 max_load_reduction=22946 max_load_reduction_pct=0.700256 moved_to_echo=19055 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 2523, 3327, 396, 408, 0, 0, 233, 0, 1207, 2120, 0, 0, 0, 0, 0, 668, 65, 898, 0, 0, 0, 0, 1408, 0, 15, 487, 828, 0, 0, 0, 1431, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 43, 3713, 3300, 0, 0, 0, 2090, 0, 0, 246, 1184, 215, 0, 0, 0, 1, 1, 0, 1119, 760, 0, 0] rerouted_tokens_per_ep_rank=[6656, 3560, 1631, 2738, 5511, 7056, 3735, 1881] avg=4096.0000 max=7056 max_rank=5 min=1631 min_rank=2 imbalance_ratio=1.722656 max_load_reduction=25712 max_load_reduction_pct=0.784668 moved_to_echo=18183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[42, 0, 622, 972, 198, 492, 0, 0, 0, 878, 4091, 2557, 0, 0, 0, 0, 0, 19, 3767, 449, 0, 0, 0, 0, 0, 286, 0, 3633, 999, 0, 0, 0, 1436, 2224, 2097, 433, 0, 0, 0, 0, 5, 0, 57, 119, 672, 0, 0, 0, 0, 0, 0, 3849, 1188, 107, 0, 0, 0, 0, 298, 0, 1123, 155, 0, 0] rerouted_tokens_per_ep_rank=[2326, 7526, 4235, 4918, 6190, 853, 5144, 1576] avg=4096.0000 max=7526 max_rank=1 min=853 min_rank=5 imbalance_ratio=1.837402 max_load_reduction=25242 max_load_reduction_pct=0.770325 moved_to_echo=13763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 8192, 4224, 5376, 7552, 3712, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4992, 128, 4736, 16384, 4736, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 128, 512, 15744, 16128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9216, 1152, 11008, 11648, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 10112, 12288, 10496, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 384, 9472, 10240, 4608, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2944, 0, 128, 19456, 7936, 2560, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9600, 14848, 7936, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[102, 0, 1, 4092, 0, 1931, 1, 0, 3540, 3831, 0, 0, 8, 0, 0, 0, 0, 1693, 0, 186, 2991, 3, 4096, 9, 1941, 3477, 834, 1, 1, 0, 0, 4030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[29, 0, 8, 80, 604, 3635, 3307, 170, 3951, 0, 0, 0, 3953, 1, 2571, 1, 252, 1, 18, 0, 654, 172, 36, 1, 4096, 0, 3599, 209, 426, 0, 4092, 902] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 148, 0, 1, 0, 5, 4096, 110, 0, 3024, 311, 826, 70, 0, 4096, 1, 104, 0, 4009, 0, 3684, 0, 4094, 0, 4072, 4083, 29, 0, 5, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 159, 0, 4004, 4008, 702, 0, 0, 10, 3667, 0, 4051, 69, 3561, 405, 2, 4088, 3, 0, 3942, 521, 0, 0, 3557, 0, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1099, 0, 4, 59, 0, 3636, 2783, 56, 3851, 0, 0, 0, 0, 0, 0, 6, 683, 333, 1419, 3255, 456, 4096, 2210, 0, 0, 212, 1385, 4072, 0, 0, 3119, 34] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[43, 11, 342, 3668, 101, 3488, 3647, 9, 3977, 1102, 5, 3, 62, 0, 0, 0, 0, 8, 1619, 2, 1118, 5, 2859, 2234, 3536, 4092, 240, 63, 0, 0, 533, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 4014, 0, 0, 0, 1, 3954, 0, 2235, 803, 1892, 891, 2, 0, 169, 19, 4088, 0, 1997, 0, 4067, 341, 4078, 0, 4092, 3, 0, 2, 120, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[2, 707, 0, 0, 0, 861, 3585, 4081, 1, 1091, 16, 0, 4081, 1, 120, 0, 0, 74, 4045, 0, 1, 3187, 3958, 97, 3, 1, 4096, 0, 2760, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 148, 0, 1, 2615, 354, 0, 0, 0, 5, 1601, 110, 0, 0, 0, 0, 0, 3024, 311, 826, 162, 0, 0, 0, 70, 0, 4096, 1, 2254, 0, 0, 0, 104, 0, 4009, 0, 212, 0, 0, 0, 3684, 0, 1479, 0, 0, 0, 0, 0, 647, 4083, 29, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2141, 797, 0, 0] rerouted_tokens_per_ep_rank=[3118, 1716, 4323, 6421, 4325, 5163, 4759, 2943] avg=4096.0000 max=6421 max_rank=3 min=1716 min_rank=1 imbalance_ratio=1.567627 max_load_reduction=26347 max_load_reduction_pct=0.804047 moved_to_echo=17190 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[102, 0, 1, 4092, 2620, 1, 0, 0, 0, 1931, 0, 0, 0, 0, 0, 0, 3540, 3831, 0, 0, 79, 0, 0, 0, 8, 0, 0, 0, 1076, 0, 0, 0, 0, 1693, 0, 186, 104, 0, 0, 0, 2991, 3, 1476, 9, 0, 0, 0, 0, 299, 3477, 834, 1, 0, 0, 0, 0, 1, 0, 0, 4030, 0, 383, 0, 0] rerouted_tokens_per_ep_rank=[6816, 1931, 7450, 1084, 1983, 4479, 4611, 4414] avg=4096.0000 max=7450 max_rank=2 min=1084 min_rank=3 imbalance_ratio=1.818848 max_load_reduction=25318 max_load_reduction_pct=0.772644 moved_to_echo=15487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 8, 80, 22, 286, 0, 0, 604, 3635, 1292, 170, 0, 0, 0, 0, 3951, 0, 0, 0, 163, 0, 0, 0, 3953, 1, 2571, 1, 2267, 0, 0, 0, 252, 1, 18, 0, 213, 0, 0, 0, 654, 172, 14, 1, 0, 0, 0, 0, 651, 0, 3599, 209, 0, 0, 0, 0, 426, 0, 4092, 902, 1729, 802, 0, 0] rerouted_tokens_per_ep_rank=[425, 5701, 4114, 8793, 484, 841, 4459, 7951] avg=4096.0000 max=8793 max_rank=3 min=425 min_rank=0 imbalance_ratio=2.146729 max_load_reduction=23975 max_load_reduction_pct=0.731659 moved_to_echo=13735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 4014, 0, 0, 2605, 342, 0, 0, 0, 1, 1545, 0, 0, 0, 0, 0, 2235, 803, 1892, 891, 162, 0, 0, 0, 2, 0, 169, 19, 2265, 0, 0, 0, 4088, 0, 1997, 0, 213, 0, 0, 0, 4067, 341, 1473, 0, 0, 0, 0, 0, 651, 3, 0, 2, 0, 0, 0, 0, 120, 0, 0, 0, 2067, 801, 0, 0] rerouted_tokens_per_ep_rank=[6961, 1546, 5983, 2455, 6298, 5881, 656, 2988] avg=4096.0000 max=6961 max_rank=0 min=656 min_rank=6 imbalance_ratio=1.699463 max_load_reduction=25807 max_load_reduction_pct=0.787567 moved_to_echo=15823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[43, 11, 342, 3668, 1826, 315, 0, 0, 101, 3488, 1426, 9, 0, 0, 0, 0, 3977, 1102, 5, 3, 140, 0, 0, 0, 62, 0, 0, 0, 1957, 0, 0, 0, 0, 8, 1619, 2, 184, 0, 0, 0, 1118, 5, 1033, 2234, 0, 0, 0, 0, 563, 4092, 240, 63, 0, 0, 0, 0, 0, 0, 533, 1, 1906, 692, 0, 0] rerouted_tokens_per_ep_rank=[6205, 5024, 5227, 2019, 1813, 4390, 4958, 3132] avg=4096.0000 max=6205 max_rank=0 min=1813 min_rank=4 imbalance_ratio=1.514893 max_load_reduction=26563 max_load_reduction_pct=0.810638 moved_to_echo=14293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 707, 0, 0, 2528, 312, 0, 0, 0, 861, 1396, 4081, 0, 0, 0, 0, 1, 1091, 16, 0, 0, 0, 0, 0, 4081, 1, 120, 0, 1, 0, 0, 0, 0, 74, 4045, 0, 0, 0, 0, 0, 1, 3187, 1430, 97, 0, 0, 0, 0, 2, 1, 4096, 0, 0, 0, 0, 0, 2760, 0, 0, 0, 1877, 0, 0, 0] rerouted_tokens_per_ep_rank=[3549, 6338, 1108, 4203, 4119, 4715, 4099, 4637] avg=4096.0000 max=6338 max_rank=1 min=1108 min_rank=2 imbalance_ratio=1.547363 max_load_reduction=26430 max_load_reduction_pct=0.806580 moved_to_echo=17570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1099, 0, 4, 59, 1411, 240, 0, 0, 0, 3636, 1088, 56, 0, 0, 0, 0, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 683, 333, 1419, 3255, 0, 0, 0, 0, 456, 4096, 799, 0, 0, 0, 0, 0, 0, 212, 1385, 4072, 0, 0, 0, 0, 0, 0, 3119, 34, 1455, 0, 0, 0] rerouted_tokens_per_ep_rank=[2813, 4780, 3851, 6, 5690, 5351, 5669, 4608] avg=4096.0000 max=5690 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.389160 max_load_reduction=27078 max_load_reduction_pct=0.826355 moved_to_echo=21318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 17, 0, 2518, 346, 0, 0, 159, 0, 1565, 4008, 0, 0, 0, 0, 702, 0, 0, 10, 0, 0, 0, 0, 3667, 0, 4051, 69, 0, 0, 0, 0, 3561, 405, 2, 4088, 0, 0, 0, 0, 3, 0, 1424, 521, 0, 0, 0, 0, 0, 0, 3557, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2093, 0, 0, 0] rerouted_tokens_per_ep_rank=[2881, 5732, 712, 7787, 8056, 1948, 3557, 2095] avg=4096.0000 max=8056 max_rank=4 min=712 min_rank=2 imbalance_ratio=1.966797 max_load_reduction=24712 max_load_reduction_pct=0.754150 moved_to_echo=15656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11904, 128, 11008, 128, 9856, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1280, 4992, 384, 7936, 16256, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8704, 2560, 13184, 7552, 1024, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3328, 128, 7808, 4992, 13312, 3584, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[896, 13568, 9984, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13056, 7808, 9216, 2944, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[18304, 9856, 2304, 1792, 768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[2816, 11904, 13824, 4352, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 17, 4070, 0, 4030, 0, 0, 0, 1573, 3586, 0, 3786, 4040, 0, 13, 457, 23, 3, 0, 1094, 57, 0, 69, 4063, 0, 3626, 0, 1, 0, 2259, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[67, 0, 3669, 0, 4094, 0, 17, 6, 408, 183, 0, 1486, 3973, 0, 3, 154, 3, 0, 0, 3696, 0, 2818, 0, 395, 5, 2199, 0, 4093, 90, 3924, 0, 1485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[49, 84, 4024, 0, 4075, 66, 1, 3, 0, 1832, 73, 3450, 3743, 0, 200, 836, 611, 0, 94, 0, 0, 0, 3242, 2321, 13, 4049, 0, 0, 9, 3885, 47, 61] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4094, 0, 146, 35, 3705, 0, 0, 25, 106, 4091, 77, 2173, 3992, 53, 0, 3392, 0, 14, 2, 754, 173, 3821, 0, 0, 1989, 0, 49, 4058, 18, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[3436, 0, 0, 0, 4084, 1, 0, 93, 0, 2, 0, 0, 709, 0, 18, 4066, 3968, 0, 1, 4096, 44, 0, 0, 8, 1, 4074, 0, 4048, 0, 3822, 7, 290] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[3201, 0, 2015, 40, 3686, 0, 0, 3860, 0, 4094, 1, 4075, 422, 0, 20, 0, 50, 654, 0, 345, 6, 4070, 0, 172, 0, 3846, 527, 0, 0, 156, 1444, 84] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3900, 194, 2981, 0, 1, 10, 0, 0, 0, 4090, 4094, 12, 0, 0, 1410, 0, 0, 4067, 185, 8, 0, 4025, 0, 3992, 0, 113, 0, 3686, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 39, 3832, 0, 4071, 4, 0, 4095, 66, 3465, 90, 3887, 3982, 0, 8, 342, 0, 0, 0, 4074, 42, 0, 0, 0, 1, 623, 4, 65, 0, 3790, 0, 288] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 4043, 0, 0, 0, 0, 0, 3226, 0, 0, 0, 0, 0, 0, 0, 1573, 3586, 0, 2166, 0, 0, 0, 0, 3755, 0, 13, 457, 0, 0, 0, 0, 23, 3, 0, 1094, 1288, 0, 0, 0, 57, 0, 69, 4063, 745, 285, 96, 27, 0, 2881, 0, 1, 0, 0, 0, 0, 0, 2259, 1, 0, 708, 332, 0, 0] rerouted_tokens_per_ep_rank=[4060, 3226, 7325, 4225, 2408, 5342, 2882, 3300] avg=4096.0000 max=7325 max_rank=2 min=2408 min_rank=4 imbalance_ratio=1.788330 max_load_reduction=25443 max_load_reduction_pct=0.776459 moved_to_echo=13932 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3436, 0, 0, 0, 0, 0, 0, 0, 3269, 1, 0, 93, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 659, 0, 18, 4066, 0, 0, 0, 0, 3968, 0, 1, 4096, 0, 0, 0, 0, 44, 0, 0, 8, 837, 50, 97, 0, 1, 3237, 0, 4048, 0, 0, 0, 0, 0, 3822, 7, 290, 718, 0, 0, 0] rerouted_tokens_per_ep_rank=[3436, 3363, 2, 4743, 8065, 1036, 7286, 4837] avg=4096.0000 max=8065 max_rank=4 min=2 min_rank=2 imbalance_ratio=1.968994 max_load_reduction=24703 max_load_reduction_pct=0.753876 moved_to_echo=21224 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[67, 0, 3645, 0, 0, 0, 0, 0, 3276, 0, 17, 6, 0, 0, 0, 0, 408, 183, 0, 851, 0, 0, 0, 0, 3693, 0, 3, 154, 0, 0, 0, 0, 3, 0, 0, 3696, 505, 0, 0, 0, 0, 2818, 0, 395, 452, 280, 98, 24, 5, 1747, 0, 4093, 0, 0, 0, 0, 90, 3924, 0, 1485, 720, 130, 0, 0] rerouted_tokens_per_ep_rank=[3712, 3299, 1442, 3850, 4204, 4067, 5845, 6349] avg=4096.0000 max=6349 max_rank=7 min=1442 min_rank=2 imbalance_ratio=1.550049 max_load_reduction=26419 max_load_reduction_pct=0.806244 moved_to_echo=20465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 146, 35, 0, 0, 0, 0, 2966, 0, 0, 25, 0, 0, 0, 0, 106, 4091, 77, 1243, 0, 0, 0, 0, 3711, 53, 0, 3392, 0, 0, 0, 0, 0, 14, 2, 754, 739, 0, 0, 0, 173, 3821, 0, 0, 0, 281, 88, 0, 1989, 0, 49, 4058, 0, 0, 0, 0, 18, 0, 1, 0, 651, 191, 0, 0] rerouted_tokens_per_ep_rank=[4275, 2991, 5517, 7156, 1509, 4363, 6096, 861] avg=4096.0000 max=7156 max_rank=3 min=861 min_rank=7 imbalance_ratio=1.747070 max_load_reduction=25612 max_load_reduction_pct=0.781616 moved_to_echo=12829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3201, 0, 2002, 40, 0, 0, 0, 0, 2950, 0, 0, 3860, 0, 0, 0, 0, 0, 4094, 1, 2331, 0, 0, 0, 0, 393, 0, 20, 0, 0, 0, 0, 0, 50, 654, 0, 345, 1386, 0, 0, 0, 6, 4070, 0, 172, 790, 29, 88, 13, 0, 3056, 527, 0, 0, 0, 0, 0, 0, 156, 1444, 84, 648, 358, 0, 0] rerouted_tokens_per_ep_rank=[5243, 6810, 6426, 413, 2435, 5168, 3583, 2690] avg=4096.0000 max=6810 max_rank=1 min=413 min_rank=3 imbalance_ratio=1.662598 max_load_reduction=25958 max_load_reduction_pct=0.792175 moved_to_echo=13876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 39, 3807, 0, 0, 0, 0, 0, 3259, 4, 0, 4095, 0, 0, 0, 0, 66, 3465, 90, 2224, 0, 0, 0, 0, 3702, 0, 8, 342, 0, 0, 0, 0, 0, 0, 0, 4074, 1322, 0, 0, 0, 42, 0, 0, 0, 128, 280, 97, 25, 1, 495, 4, 65, 0, 0, 0, 0, 0, 3790, 0, 288, 715, 341, 0, 0] rerouted_tokens_per_ep_rank=[3846, 7358, 5845, 4052, 5396, 572, 565, 5134] avg=4096.0000 max=7358 max_rank=1 min=565 min_rank=6 imbalance_ratio=1.796387 max_load_reduction=25410 max_load_reduction_pct=0.775452 moved_to_echo=11667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[49, 84, 3998, 0, 0, 0, 0, 0, 3262, 66, 1, 3, 0, 0, 0, 0, 0, 1832, 73, 1973, 0, 0, 0, 0, 3479, 0, 200, 836, 0, 0, 0, 0, 611, 0, 94, 0, 1174, 0, 0, 0, 0, 0, 3242, 2321, 832, 264, 97, 26, 13, 3217, 0, 0, 0, 0, 0, 0, 9, 3885, 47, 61, 716, 303, 0, 0] rerouted_tokens_per_ep_rank=[4131, 3332, 3878, 4515, 1879, 6782, 3230, 5021] avg=4096.0000 max=6782 max_rank=5 min=1879 min_rank=4 imbalance_ratio=1.655762 max_load_reduction=25986 max_load_reduction_pct=0.793030 moved_to_echo=16912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3871, 194, 0, 0, 0, 0, 2378, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 2333, 0, 0, 0, 0, 3802, 12, 0, 0, 0, 0, 0, 0, 1410, 0, 0, 4067, 1395, 0, 0, 0, 185, 8, 0, 4025, 824, 292, 75, 29, 0, 3168, 0, 113, 0, 0, 0, 0, 0, 3686, 0, 0, 528, 362, 0, 0] rerouted_tokens_per_ep_rank=[4065, 2389, 2333, 3814, 6872, 5438, 3281, 4576] avg=4096.0000 max=6872 max_rank=4 min=2333 min_rank=2 imbalance_ratio=1.677734 max_load_reduction=25896 max_load_reduction_pct=0.790283 moved_to_echo=20167 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6144, 768, 128, 18176, 7936, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[2176, 17280, 256, 13184, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 10752, 3328, 11008, 4608, 1792, 768, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 21632, 1536, 2304, 5504, 2048, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[24704, 128, 128, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2048, 17920, 640, 12416, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10880, 256, 21632, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[23296, 128, 384, 9344, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3585, 3313, 0, 54, 2224, 3720, 0, 0, 0, 582, 169, 0, 3916, 0, 38, 1011, 3, 3448, 110, 0, 0, 0, 2430, 0, 1937, 0, 0, 1, 16, 1, 2114, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 453, 979, 0, 4096, 0, 0, 4046, 3293, 143, 3407, 76, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3921, 0, 3725, 0, 110, 0, 6, 320, 4095, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4037, 0, 1, 0, 0, 4093, 5, 3980, 16, 12, 0, 2086, 4078, 219, 0, 124, 6, 635, 0, 7, 3753, 570, 13, 0, 1105, 3464, 0, 3, 0, 545, 0, 4016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[153, 1, 0, 0, 0, 3452, 244, 4033, 9, 4085, 3, 366, 3878, 1, 63, 14, 85, 2734, 0, 359, 0, 145, 3959, 3971, 38, 0, 420, 4071, 683, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[258, 0, 1884, 0, 175, 3, 0, 235, 4065, 4, 0, 4096, 3658, 3, 0, 18, 26, 4090, 0, 90, 1668, 0, 0, 5, 3866, 4090, 2028, 0, 0, 0, 2, 2504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4013, 0, 0, 2902, 0, 259, 0, 0, 0, 112, 0, 4095, 18, 0, 98, 0, 3759, 0, 0, 0, 0, 846, 3995, 3926, 19, 494, 1, 53, 0, 4083, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 1228, 1912, 0, 44, 1529, 0, 0, 4068, 0, 0, 2255, 4095, 0, 0, 18, 0, 0, 39, 0, 369, 0, 4035, 0, 3727, 3103, 2144, 0, 0, 0, 110, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 4018, 0, 1582, 0, 3849, 0, 0, 338, 398, 0, 2806, 2160, 349, 1058, 1, 18, 283, 0, 0, 4, 0, 2309, 3703, 5, 51, 0, 2, 0, 1478, 3746, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3585, 3313, 0, 54, 215, 0, 0, 0, 2224, 3349, 0, 0, 0, 0, 0, 0, 0, 582, 169, 0, 0, 0, 0, 0, 3916, 0, 38, 1011, 403, 156, 0, 0, 3, 3448, 110, 0, 1381, 587, 0, 3, 0, 0, 2024, 0, 0, 0, 0, 0, 1350, 0, 0, 1, 0, 0, 0, 0, 16, 1, 2114, 2715, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7167, 5573, 751, 5524, 5532, 2024, 1351, 4846] avg=4096.0000 max=7167 max_rank=0 min=751 min_rank=2 imbalance_ratio=1.749756 max_load_reduction=25601 max_load_reduction_pct=0.781281 moved_to_echo=13753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4013, 0, 0, 0, 0, 0, 0, 2902, 0, 259, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 4095, 18, 0, 98, 140, 0, 0, 0, 0, 3759, 0, 0, 1381, 1191, 0, 1, 0, 0, 705, 3995, 0, 0, 0, 0, 2735, 19, 494, 1, 0, 0, 0, 0, 53, 0, 4083, 2714, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 3161, 112, 4351, 6332, 4700, 3249, 6850] avg=4096.0000 max=6850 max_rank=7 min=112 min_rank=2 imbalance_ratio=1.672363 max_load_reduction=25918 max_load_reduction_pct=0.790955 moved_to_echo=21131 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4037, 0, 1, 0, 236, 0, 0, 0, 0, 3685, 5, 3980, 0, 0, 0, 0, 16, 12, 0, 1588, 0, 0, 0, 0, 4078, 219, 0, 124, 2, 172, 0, 0, 6, 635, 0, 7, 1354, 335, 498, 0, 3753, 570, 11, 0, 0, 0, 0, 0, 770, 3464, 0, 3, 0, 0, 0, 0, 0, 545, 0, 2662, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4274, 7670, 1616, 4595, 2835, 4334, 4237, 3207] avg=4096.0000 max=7670 max_rank=1 min=1616 min_rank=2 imbalance_ratio=1.872559 max_load_reduction=25098 max_load_reduction_pct=0.765930 moved_to_echo=14613 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 453, 979, 241, 0, 0, 0, 0, 3680, 0, 0, 0, 0, 0, 0, 4046, 3293, 143, 2590, 0, 0, 0, 0, 76, 0, 0, 0, 652, 175, 0, 0, 0, 0, 0, 2, 1385, 1134, 817, 6, 0, 0, 3263, 0, 0, 0, 0, 0, 2591, 0, 110, 0, 0, 0, 0, 0, 6, 320, 4095, 2711, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1673, 3680, 10072, 903, 3344, 3263, 2701, 7132] avg=4096.0000 max=10072 max_rank=2 min=903 min_rank=3 imbalance_ratio=2.458984 max_load_reduction=22696 max_load_reduction_pct=0.692627 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[153, 1, 0, 0, 199, 0, 0, 0, 0, 3108, 244, 4033, 0, 0, 0, 0, 9, 4085, 3, 279, 0, 0, 0, 0, 3878, 1, 63, 14, 656, 145, 0, 0, 85, 2734, 0, 359, 0, 11, 87, 5, 0, 145, 3298, 3971, 0, 0, 0, 0, 27, 0, 420, 4071, 0, 0, 0, 0, 683, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[353, 7385, 4376, 4757, 3281, 7414, 4518, 684] avg=4096.0000 max=7414 max_rank=5 min=353 min_rank=0 imbalance_ratio=1.810059 max_load_reduction=25354 max_load_reduction_pct=0.773743 moved_to_echo=15897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1228, 1912, 0, 88, 0, 0, 0, 44, 1377, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 1716, 0, 0, 0, 0, 4095, 0, 0, 18, 669, 64, 0, 0, 0, 0, 39, 0, 1380, 1130, 539, 5, 369, 0, 3361, 0, 0, 0, 0, 0, 2597, 3103, 2144, 0, 0, 0, 0, 0, 0, 0, 110, 2712, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3228, 1421, 5784, 4846, 3093, 3730, 7844, 2822] avg=4096.0000 max=7844 max_rank=6 min=1421 min_rank=1 imbalance_ratio=1.915039 max_load_reduction=24924 max_load_reduction_pct=0.760620 moved_to_echo=17489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 4018, 0, 1582, 222, 0, 0, 0, 0, 3465, 0, 0, 0, 0, 0, 0, 338, 398, 0, 2135, 0, 0, 0, 0, 2160, 349, 1058, 1, 383, 162, 0, 0, 18, 283, 0, 0, 173, 1, 671, 3, 4, 0, 1923, 3703, 0, 0, 0, 0, 4, 51, 0, 2, 0, 0, 0, 0, 0, 1478, 3746, 341, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9918, 3465, 2871, 4113, 1149, 5630, 57, 5565] avg=4096.0000 max=9918 max_rank=0 min=57 min_rank=6 imbalance_ratio=2.421387 max_load_reduction=22850 max_load_reduction_pct=0.697327 moved_to_echo=12401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[258, 0, 1884, 0, 0, 0, 0, 0, 175, 3, 0, 235, 0, 0, 0, 0, 4065, 4, 0, 3117, 0, 0, 0, 0, 3658, 3, 0, 18, 0, 0, 0, 0, 26, 4090, 0, 90, 844, 1173, 979, 0, 1668, 0, 0, 5, 0, 0, 0, 0, 2693, 4090, 2028, 0, 0, 0, 0, 0, 0, 0, 2, 1660, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2142, 413, 7186, 3679, 7202, 1673, 8811, 1662] avg=4096.0000 max=8811 max_rank=6 min=413 min_rank=1 imbalance_ratio=2.151123 max_load_reduction=23957 max_load_reduction_pct=0.731110 moved_to_echo=19348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12544, 8448, 512, 11520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 14976, 256, 512, 7936, 5632, 3712, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12160, 12672, 4352, 2688, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[5376, 18688, 512, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 2432, 14208, 15616, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[12800, 10752, 5248, 4096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5888, 768, 14592, 11776, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[25984, 640, 1280, 1408, 2944, 896, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 142, 0, 0, 1, 7, 100, 7, 4092, 0, 0, 0, 16, 57, 4, 3979, 4074, 0, 3397, 4077, 4093, 746, 2914, 1, 0, 762, 240, 3071, 988, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1893, 1, 0, 6, 0, 0, 0, 0, 3887, 0, 0, 0, 1943, 2657, 0, 67, 17, 3118, 3023, 4069, 3651, 2020, 78, 233, 670, 180, 32, 2113, 611, 2499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[882, 19, 0, 2, 0, 0, 0, 6, 22, 3, 4076, 0, 0, 2, 3921, 0, 3482, 3797, 43, 0, 0, 4013, 4086, 1657, 0, 0, 1, 0, 0, 0, 2720, 4036] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[3652, 462, 0, 24, 22, 386, 0, 0, 84, 3092, 3500, 15, 109, 2, 12, 0, 3844, 3779, 30, 0, 3936, 4065, 4, 0, 751, 1358, 0, 61, 0, 7, 3572, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1, 196, 0, 0, 0, 1128, 1, 1, 4012, 1726, 2361, 0, 2, 0, 0, 118, 3752, 4017, 0, 1900, 4095, 588, 1868, 2569, 63, 243, 17, 14, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 43, 0, 0, 0, 2, 1, 0, 183, 42, 3574, 0, 0, 195, 84, 752, 3186, 262, 3394, 0, 3878, 0, 4096, 0, 3991, 298, 0, 0, 606, 0, 4085, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 45, 0, 0, 0, 3, 2615, 4009, 4074, 0, 0, 17, 2024, 1410, 0, 4096, 12, 0, 4081, 3522, 3809, 0, 375, 0, 0, 1437, 0, 0, 1239, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[235, 886, 0, 10, 32, 0, 1, 4, 60, 2886, 4096, 0, 2843, 2, 4085, 472, 2, 311, 151, 38, 0, 4088, 6, 0, 3909, 383, 5, 3708, 479, 0, 4076, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 196, 0, 0, 1016, 2101, 0, 0, 0, 1128, 1, 1, 588, 1232, 0, 0, 4012, 1726, 1109, 0, 0, 0, 0, 0, 2, 0, 0, 118, 1794, 236, 0, 0, 3752, 1132, 0, 1900, 0, 0, 0, 0, 762, 0, 1868, 2569, 0, 0, 0, 0, 63, 243, 17, 14, 0, 1091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3314, 2950, 6847, 2150, 6784, 5199, 1428, 4096] avg=4096.0000 max=6847 max_rank=2 min=1428 min_rank=6 imbalance_ratio=1.671631 max_load_reduction=25921 max_load_reduction_pct=0.791046 moved_to_echo=17507 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3652, 462, 0, 24, 1507, 2020, 0, 0, 22, 386, 0, 0, 4065, 1184, 0, 0, 84, 3092, 1642, 15, 0, 0, 0, 0, 109, 2, 12, 0, 1687, 351, 0, 0, 3844, 1066, 30, 0, 0, 0, 0, 0, 732, 0, 4, 0, 0, 0, 0, 0, 751, 1358, 0, 61, 1197, 1026, 0, 0, 0, 7, 2375, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7665, 5657, 4833, 2161, 4940, 736, 4393, 2383] avg=4096.0000 max=7665 max_rank=0 min=736 min_rank=5 imbalance_ratio=1.871338 max_load_reduction=25103 max_load_reduction_pct=0.766083 moved_to_echo=12452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 0, 0, 1543, 1993, 0, 0, 0, 2, 1, 0, 0, 1170, 0, 0, 183, 42, 1668, 0, 0, 0, 0, 0, 0, 195, 84, 752, 121, 363, 0, 0, 3186, 66, 3394, 0, 0, 0, 0, 0, 715, 0, 4096, 0, 0, 0, 0, 0, 3991, 298, 0, 0, 1374, 75, 0, 0, 606, 0, 2711, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3579, 1173, 1893, 1515, 6646, 4811, 5738, 7413] avg=4096.0000 max=7413 max_rank=7 min=1173 min_rank=1 imbalance_ratio=1.809814 max_load_reduction=25355 max_load_reduction_pct=0.773773 moved_to_echo=24608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[882, 19, 0, 2, 1755, 0, 0, 0, 0, 0, 0, 6, 4013, 0, 0, 0, 22, 3, 1913, 0, 0, 0, 0, 0, 0, 2, 3921, 0, 1695, 408, 0, 0, 3482, 1071, 43, 0, 0, 0, 0, 0, 0, 0, 4086, 1657, 0, 0, 0, 0, 0, 0, 1, 0, 912, 1031, 0, 0, 0, 0, 1808, 4036, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2658, 4019, 1938, 6026, 4596, 5743, 1944, 5844] avg=4096.0000 max=6026 max_rank=3 min=1938 min_rank=2 imbalance_ratio=1.471191 max_load_reduction=26742 max_load_reduction_pct=0.816101 moved_to_echo=18127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[235, 886, 0, 10, 1763, 0, 0, 0, 32, 0, 1, 4, 4088, 0, 0, 0, 60, 2886, 1923, 0, 0, 0, 0, 0, 2843, 2, 4085, 472, 138, 410, 0, 0, 2, 89, 151, 38, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 3909, 383, 5, 3708, 1366, 84, 0, 0, 479, 0, 2710, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2894, 4125, 4869, 7950, 280, 6, 9455, 3189] avg=4096.0000 max=9455 max_rank=6 min=6 min_rank=5 imbalance_ratio=2.308350 max_load_reduction=23313 max_load_reduction_pct=0.711456 moved_to_echo=12930 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1893, 1, 1673, 1551, 0, 0, 0, 6, 0, 0, 4069, 909, 0, 0, 0, 0, 1825, 0, 0, 0, 0, 0, 0, 0, 1943, 2657, 29, 389, 0, 0, 0, 20, 17, 3118, 0, 0, 0, 0, 563, 0, 3651, 2020, 0, 0, 0, 0, 78, 233, 670, 180, 204, 18, 0, 0, 32, 2113, 407, 2499, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5118, 4984, 1825, 5018, 3155, 6234, 1383, 5051] avg=4096.0000 max=6234 max_rank=5 min=1383 min_rank=6 imbalance_ratio=1.521973 max_load_reduction=26534 max_load_reduction_pct=0.809753 moved_to_echo=15823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 142, 1762, 1743, 0, 0, 0, 0, 1, 7, 4077, 1022, 0, 0, 100, 7, 1920, 0, 0, 0, 0, 0, 0, 0, 16, 57, 1777, 410, 0, 0, 4, 1122, 4074, 0, 0, 0, 0, 0, 632, 0, 4093, 746, 0, 0, 0, 0, 2914, 1, 0, 762, 331, 1080, 0, 0, 240, 3071, 657, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3647, 5107, 2027, 2260, 5200, 5471, 5088, 3968] avg=4096.0000 max=5471 max_rank=5 min=2027 min_rank=2 imbalance_ratio=1.335693 max_load_reduction=27297 max_load_reduction_pct=0.833038 moved_to_echo=19727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 45, 1754, 2094, 0, 0, 0, 0, 0, 3, 3522, 1228, 0, 0, 2615, 4009, 1912, 0, 0, 0, 0, 0, 0, 17, 2024, 1410, 1829, 408, 0, 0, 0, 1155, 12, 0, 0, 0, 0, 0, 759, 0, 3809, 0, 0, 0, 0, 0, 375, 0, 0, 1437, 415, 1112, 0, 0, 0, 0, 824, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3893, 4753, 8536, 5688, 1167, 4568, 3339, 824] avg=4096.0000 max=8536 max_rank=2 min=824 min_rank=7 imbalance_ratio=2.083984 max_load_reduction=24232 max_load_reduction_pct=0.739502 moved_to_echo=9898 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4864, 1664, 1920, 256, 12800, 11520, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[12160, 2560, 768, 6272, 5888, 5632, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3072, 256, 12160, 5504, 9088, 3072, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7168, 11776, 13952, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14336, 5760, 7808, 5120, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[4224, 0, 21632, 7040, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 1536, 128, 128, 24448, 6784, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1408, 9344, 11520, 10752, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[54, 0, 0, 40, 0, 4095, 0, 3486, 4096, 1796, 2577, 2951, 0, 0, 1437, 369, 2798, 391, 532, 4077, 0, 494, 0, 5, 0, 0, 16, 1193, 0, 1, 2316, 44] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[114, 3478, 3997, 4091, 4088, 1, 0, 0, 0, 6, 184, 3818, 169, 224, 2200, 88, 1, 0, 1182, 15, 496, 51, 0, 0, 1669, 0, 0, 511, 4018, 0, 74, 2293] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1410, 0, 4088, 860, 4034, 3589, 1, 2017, 0, 4, 1727, 286, 0, 35, 4082, 7, 105, 2019, 4094, 89, 0, 0, 0, 0, 0, 782, 6, 0, 3509, 14, 10, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 4094, 0, 4074, 16, 4061, 461, 497, 0, 983, 2875, 1, 1416, 2838, 0, 3, 85, 3676, 0, 1393, 0, 1, 0, 10, 4033, 2211, 0, 33, 0, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3772, 177, 0, 46, 1, 4012, 50, 507, 2, 0, 4077, 3540, 0, 0, 3934, 0, 7, 0, 10, 4024, 4, 0, 49, 0, 169, 0, 4023, 154, 3910, 295, 1, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4068, 0, 1121, 324, 2372, 0, 0, 0, 0, 2, 1503, 4077, 280, 628, 3857, 1927, 0, 4089, 4096, 0, 0, 0, 982, 0, 1, 1110, 1964, 1, 0, 0, 0, 366] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 37, 10, 7, 4095, 0, 2919, 0, 0, 2, 587, 28, 0, 570, 0, 4096, 0, 4065, 4017, 0, 0, 4084, 9, 40, 0, 274, 0, 3795, 0, 15, 22] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[2, 3592, 12, 0, 4095, 0, 4094, 0, 0, 0, 259, 1554, 1620, 104, 2340, 3827, 0, 2731, 4096, 0, 42, 0, 1506, 1884, 0, 24, 0, 0, 0, 0, 5, 981] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[58, 3478, 3997, 4091, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6, 184, 2920, 0, 0, 0, 0, 169, 224, 2200, 88, 0, 29, 0, 0, 1, 0, 230, 15, 0, 0, 0, 0, 496, 51, 0, 0, 4088, 168, 0, 0, 1669, 0, 0, 511, 784, 1, 0, 0, 4018, 0, 74, 2293, 55, 869, 0, 0] rerouted_tokens_per_ep_rank=[11624, 1, 3110, 2710, 246, 4803, 2965, 7309] avg=4096.0000 max=11624 max_rank=0 min=1 min_rank=1 imbalance_ratio=2.837891 max_load_reduction=21144 max_load_reduction_pct=0.645264 moved_to_echo=15323 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2041, 0, 1121, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1503, 3111, 0, 0, 0, 0, 280, 628, 3857, 1927, 0, 34, 0, 0, 0, 4089, 788, 0, 0, 0, 0, 0, 0, 0, 982, 0, 2372, 587, 0, 0, 1, 1110, 1964, 1, 2721, 45, 0, 0, 0, 0, 0, 366, 1982, 932, 0, 0] rerouted_tokens_per_ep_rank=[3486, 0, 4616, 6726, 4877, 3941, 5842, 3280] avg=4096.0000 max=6726 max_rank=3 min=0 min_rank=1 imbalance_ratio=1.642090 max_load_reduction=26042 max_load_reduction_pct=0.794739 moved_to_echo=17940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4094, 0, 0, 0, 0, 0, 0, 13, 4061, 461, 0, 0, 0, 0, 497, 0, 983, 2199, 0, 0, 0, 0, 1, 1416, 2838, 0, 3, 22, 0, 0, 3, 85, 714, 0, 0, 0, 0, 0, 1393, 0, 1, 0, 4074, 523, 0, 0, 10, 4033, 2211, 0, 2439, 0, 0, 0, 33, 0, 6, 0, 0, 654, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4535, 3679, 4280, 802, 5991, 8693, 693] avg=4096.0000 max=8693 max_rank=6 min=693 min_rank=7 imbalance_ratio=2.122314 max_load_reduction=24075 max_load_reduction_pct=0.734711 moved_to_echo=16179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 3592, 12, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 259, 1190, 0, 0, 0, 0, 1620, 104, 2340, 3827, 0, 11, 0, 0, 0, 2731, 796, 0, 0, 0, 0, 0, 42, 0, 1506, 1884, 4095, 583, 0, 0, 0, 24, 0, 0, 2717, 0, 0, 0, 0, 0, 5, 981, 0, 353, 0, 0] rerouted_tokens_per_ep_rank=[3606, 4094, 1449, 7902, 3527, 8110, 2741, 1339] avg=4096.0000 max=8110 max_rank=5 min=1339 min_rank=7 imbalance_ratio=1.979980 max_load_reduction=24658 max_load_reduction_pct=0.752502 moved_to_echo=15717 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 0, 40, 0, 0, 0, 0, 0, 3931, 0, 3486, 0, 0, 0, 0, 4096, 1796, 2577, 2257, 0, 0, 0, 0, 0, 0, 1437, 369, 164, 22, 0, 0, 2798, 391, 104, 4077, 0, 0, 0, 0, 0, 494, 0, 5, 0, 75, 0, 0, 0, 0, 16, 1193, 353, 0, 0, 0, 0, 1, 2316, 44, 26, 672, 0, 0] rerouted_tokens_per_ep_rank=[68, 7417, 10726, 1992, 7370, 574, 1562, 3059] avg=4096.0000 max=10726 max_rank=2 min=68 min_rank=0 imbalance_ratio=2.618652 max_load_reduction=22042 max_load_reduction_pct=0.672668 moved_to_echo=12565 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1899, 177, 0, 46, 0, 0, 0, 0, 0, 3851, 50, 507, 0, 0, 0, 0, 2, 0, 4077, 2707, 0, 0, 0, 0, 0, 0, 3934, 0, 161, 27, 0, 0, 7, 0, 3, 4024, 0, 0, 0, 0, 4, 0, 49, 0, 1, 1, 0, 0, 169, 0, 4023, 154, 6, 40, 0, 0, 3910, 295, 1, 4, 1833, 806, 0, 0] rerouted_tokens_per_ep_rank=[2122, 4408, 6786, 4122, 4034, 55, 4392, 6849] avg=4096.0000 max=6849 max_rank=7 min=55 min_rank=5 imbalance_ratio=1.672119 max_load_reduction=25919 max_load_reduction_pct=0.790985 moved_to_echo=15330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2062, 0, 37, 10, 0, 0, 0, 0, 0, 3931, 0, 2919, 0, 0, 0, 0, 0, 0, 2, 450, 0, 0, 0, 0, 28, 0, 570, 0, 164, 4, 0, 0, 4096, 0, 790, 4017, 0, 0, 0, 0, 0, 0, 4084, 9, 7, 578, 0, 0, 40, 0, 274, 0, 2697, 43, 0, 0, 3795, 0, 15, 22, 1991, 133, 0, 0] rerouted_tokens_per_ep_rank=[2109, 6850, 452, 766, 8903, 4678, 3054, 5956] avg=4096.0000 max=8903 max_rank=4 min=452 min_rank=2 imbalance_ratio=2.173584 max_load_reduction=23865 max_load_reduction_pct=0.728302 moved_to_echo=22591 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[710, 0, 4088, 860, 0, 0, 0, 0, 0, 3445, 1, 2017, 0, 0, 0, 0, 0, 4, 1727, 219, 0, 0, 0, 0, 0, 35, 4082, 7, 144, 2, 0, 0, 105, 2019, 796, 89, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 582, 0, 0, 0, 782, 6, 0, 2716, 15, 0, 0, 3509, 14, 10, 0, 685, 65, 0, 0] rerouted_tokens_per_ep_rank=[5658, 5463, 1950, 4270, 3009, 4616, 3519, 4283] avg=4096.0000 max=5658 max_rank=0 min=1950 min_rank=2 imbalance_ratio=1.381348 max_load_reduction=27110 max_load_reduction_pct=0.827332 moved_to_echo=15427 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6912, 7296, 13440, 5376, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4608, 1920, 11392, 15104, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[7040, 9344, 4224, 12288, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2048, 640, 6656, 1920, 18688, 3200, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 15232, 8320, 9472, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[15360, 384, 2432, 3712, 6656, 4608, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1920, 6016, 8576, 1920, 14464, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2176, 2432, 21376, 6272, 640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4, 0, 4096, 3, 0, 3, 216, 59, 0, 4045, 0, 4031, 0, 0, 3, 8, 3389, 4096, 423, 4043, 0, 0, 0, 0, 10, 0, 69, 2858, 1230, 0, 3271, 911] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1941, 106, 4096, 27, 1796, 0, 182, 0, 1, 4096, 3944, 603, 1581, 0, 180, 0, 4095, 527, 11, 4078, 0, 0, 0, 28, 1, 22, 0, 1371, 4034, 0, 48, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 2796, 360, 634, 0, 0, 0, 3607, 3986, 3861, 3655, 525, 0, 53, 3798, 0, 0, 4095, 1, 0, 0, 16, 0, 213, 0, 0, 148, 943, 29, 4048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 21, 50, 1659, 15, 49, 3532, 4096, 2474, 0, 0, 264, 4092, 11, 0, 132, 4074, 1025, 3618, 32, 2131, 0, 0, 52, 0, 0, 383, 0, 37, 3381, 1640] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3578, 2211, 0, 0, 79, 0, 500, 3032, 3940, 3315, 0, 0, 257, 0, 4095, 0, 0, 3860, 0, 8, 0, 156, 0, 0, 0, 4093, 22, 0, 351, 3271] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 222, 52, 12, 4020, 90, 0, 0, 0, 4093, 1995, 132, 12, 0, 0, 237, 3881, 59, 124, 2584, 29, 2, 530, 0, 0, 3172, 0, 3794, 2337, 1253, 44, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3831, 0, 0, 3, 0, 0, 0, 4094, 4066, 3257, 4051, 40, 10, 0, 4090, 0, 243, 0, 0, 0, 0, 0, 0, 623, 0, 4096, 0, 0, 286, 4078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3215, 140, 85, 0, 0, 1, 2165, 4073, 4096, 3678, 58, 94, 1570, 0, 4089, 0, 481, 4095, 1, 0, 37, 0, 10, 51, 0, 4092, 0, 0, 55, 682] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3578, 2211, 598, 430, 0, 0, 0, 0, 79, 0, 2968, 0, 0, 0, 500, 0, 1277, 3315, 0, 0, 0, 0, 0, 0, 257, 0, 1567, 1084, 0, 0, 0, 0, 0, 3178, 0, 0, 0, 0, 0, 8, 0, 156, 3032, 43, 0, 0, 0, 0, 0, 4093, 1096, 252, 0, 0, 22, 0, 351, 2673, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6817, 3047, 5092, 2908, 3178, 3239, 5441, 3046] avg=4096.0000 max=6817 max_rank=0 min=2908 min_rank=3 imbalance_ratio=1.664307 max_load_reduction=25951 max_load_reduction_pct=0.791962 moved_to_echo=14904 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2796, 740, 456, 0, 0, 360, 634, 0, 0, 2753, 0, 0, 0, 0, 0, 1291, 3861, 0, 0, 0, 0, 3655, 525, 0, 53, 1586, 1005, 0, 0, 0, 0, 0, 3371, 0, 0, 0, 0, 1, 0, 0, 16, 3607, 40, 0, 0, 0, 213, 0, 0, 1109, 268, 0, 0, 148, 943, 29, 3308, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3992, 3747, 5152, 6824, 3371, 3664, 1590, 4428] avg=4096.0000 max=6824 max_rank=3 min=1590 min_rank=6 imbalance_ratio=1.666016 max_load_reduction=25944 max_load_reduction_pct=0.791748 moved_to_echo=13053 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1941, 106, 4096, 27, 0, 458, 0, 0, 1796, 0, 182, 0, 2969, 0, 0, 0, 1, 0, 1271, 603, 0, 0, 0, 0, 1581, 0, 180, 0, 1572, 1084, 0, 0, 0, 527, 11, 3351, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 42, 0, 0, 1, 22, 0, 1371, 1101, 269, 0, 0, 4034, 0, 48, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6628, 4947, 1875, 4417, 3889, 4166, 2764, 4082] avg=4096.0000 max=6628 max_rank=0 min=1875 min_rank=2 imbalance_ratio=1.618164 max_load_reduction=26140 max_load_reduction_pct=0.797729 moved_to_echo=14901 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3215, 140, 124, 456, 0, 0, 85, 0, 0, 1, 2964, 0, 0, 0, 2165, 0, 1326, 3678, 0, 0, 0, 0, 58, 94, 1570, 0, 1630, 1084, 0, 0, 0, 0, 481, 3371, 0, 0, 0, 0, 1, 0, 37, 0, 4073, 41, 0, 0, 10, 51, 0, 4092, 1140, 268, 0, 0, 0, 0, 55, 558, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3935, 3050, 7169, 4436, 3852, 4152, 5561, 613] avg=4096.0000 max=7169 max_rank=2 min=613 min_rank=7 imbalance_ratio=1.750244 max_load_reduction=25599 max_load_reduction_pct=0.781219 moved_to_echo=14178 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 222, 52, 12, 748, 288, 0, 0, 4020, 90, 0, 0, 2813, 0, 0, 0, 0, 0, 647, 132, 0, 0, 0, 0, 12, 0, 0, 237, 793, 1029, 0, 0, 0, 59, 124, 2127, 0, 0, 0, 0, 29, 2, 530, 0, 4093, 39, 0, 0, 0, 3172, 0, 3794, 555, 169, 0, 0, 2337, 1253, 44, 3346, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1322, 6923, 779, 2071, 2310, 4693, 7690, 6980] avg=4096.0000 max=7690 max_rank=6 min=779 min_rank=2 imbalance_ratio=1.877441 max_load_reduction=25078 max_load_reduction_pct=0.765320 moved_to_echo=21673 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 3, 166, 451, 0, 0, 0, 3, 216, 59, 2457, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 3, 8, 0, 898, 0, 0, 0, 4096, 423, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 34, 0, 0, 10, 0, 69, 2858, 0, 264, 0, 0, 1230, 0, 3271, 745, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4720, 2735, 4031, 909, 7847, 4079, 3201, 5246] avg=4096.0000 max=7847 max_rank=4 min=909 min_rank=3 imbalance_ratio=1.915771 max_load_reduction=24921 max_load_reduction_pct=0.760529 moved_to_echo=20373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 21, 50, 299, 403, 0, 0, 1659, 15, 49, 3532, 95, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 264, 4092, 11, 0, 0, 34, 0, 0, 0, 4074, 1025, 2979, 0, 0, 0, 0, 32, 2131, 0, 0, 2474, 3, 0, 0, 52, 0, 0, 383, 0, 236, 0, 0, 0, 37, 3381, 1341, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[773, 5350, 4096, 4401, 8078, 4640, 671, 4759] avg=4096.0000 max=8078 max_rank=4 min=671 min_rank=6 imbalance_ratio=1.972168 max_load_reduction=24690 max_load_reduction_pct=0.753479 moved_to_echo=18148 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3831, 0, 750, 0, 0, 0, 0, 3, 0, 0, 2966, 0, 0, 0, 0, 0, 1317, 3257, 0, 0, 0, 0, 4051, 40, 10, 0, 1618, 1083, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 41, 0, 0, 0, 623, 0, 4096, 1131, 0, 0, 0, 0, 0, 286, 3328, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4581, 2969, 4574, 6802, 243, 4135, 5850, 3614] avg=4096.0000 max=6802 max_rank=3 min=243 min_rank=4 imbalance_ratio=1.660645 max_load_reduction=25966 max_load_reduction_pct=0.792419 moved_to_echo=13842 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 4096, 128, 20736, 6144, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2048, 384, 18944, 5248, 3456, 2944, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[6784, 0, 7168, 18944, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 8832, 2432, 21760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7936, 768, 640, 3712, 20096, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7808, 2304, 7552, 15360, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9728, 4864, 2048, 384, 8832, 7424, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 2176, 640, 256, 29568, 384, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[194, 4060, 39, 289, 219, 0, 2, 0, 0, 4096, 1075, 0, 0, 0, 0, 3641, 5, 29, 0, 4094, 3465, 0, 488, 1, 0, 2827, 956, 4019, 31, 0, 0, 3238] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 2684, 8, 83, 3673, 33, 0, 772, 1, 0, 1822, 3430, 0, 7, 0, 7, 0, 1646, 10, 1008, 34, 1813, 4061, 0, 0, 1103, 2488, 0, 0, 3989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[69, 179, 0, 405, 573, 3933, 0, 0, 244, 4003, 24, 3, 1459, 14, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 81, 3, 2248, 0, 3667, 3728, 0, 4036, 4007] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 3887, 7, 60, 127, 0, 0, 0, 3012, 4056, 0, 39, 825, 164, 4, 0, 4028, 0, 0, 1, 4083, 0, 6, 0, 48, 1332, 0, 0, 4042, 2899, 52, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4012, 4064, 0, 0, 0, 2, 67, 0, 0, 3900, 4095, 6, 2, 2301, 3946, 1301, 1, 2727, 21, 0, 3012, 7, 0, 180, 0, 1, 648, 1394, 2, 2, 0, 1077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[111, 0, 0, 0, 3, 0, 1703, 0, 1, 4000, 27, 0, 1415, 0, 4084, 0, 2112, 0, 0, 0, 207, 0, 0, 4094, 271, 4070, 808, 3823, 1911, 76, 5, 4047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[3847, 169, 0, 2, 8, 14, 1887, 4077, 122, 0, 0, 0, 2230, 3123, 19, 3770, 92, 0, 164, 4096, 4094, 5, 0, 0, 82, 0, 0, 4071, 0, 896, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[22, 3049, 0, 2514, 1, 11, 0, 61, 286, 3658, 2643, 71, 3737, 0, 0, 4035, 3969, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4073, 0, 19, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3847, 161, 0, 2, 0, 0, 0, 0, 8, 14, 1887, 4077, 0, 1128, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 2230, 3123, 19, 1240, 0, 0, 0, 0, 92, 0, 164, 4096, 1402, 0, 375, 0, 4094, 5, 0, 0, 225, 8, 0, 0, 82, 0, 0, 3471, 0, 0, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 7114, 122, 6612, 6129, 4332, 3553, 896] avg=4096.0000 max=7114 max_rank=1 min=122 min_rank=2 imbalance_ratio=1.736816 max_load_reduction=25654 max_load_reduction_pct=0.782898 moved_to_echo=14910 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[69, 170, 0, 405, 0, 0, 0, 0, 573, 3933, 0, 0, 2381, 0, 0, 0, 244, 3453, 24, 3, 0, 0, 0, 0, 1459, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 338, 0, 4092, 0, 0, 81, 203, 9, 0, 0, 3, 2248, 0, 3126, 0, 0, 0, 0, 3728, 0, 4036, 1626, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[644, 6887, 3724, 1473, 888, 4385, 5377, 9390] avg=4096.0000 max=9390 max_rank=7 min=644 min_rank=0 imbalance_ratio=2.292480 max_load_reduction=23378 max_load_reduction_pct=0.713440 moved_to_echo=20040 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 3885, 0, 2684, 0, 0, 0, 0, 8, 83, 3673, 33, 2373, 5, 0, 0, 0, 663, 1, 0, 0, 0, 0, 0, 1822, 3430, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1646, 2, 109, 104, 0, 10, 1008, 34, 1813, 63, 211, 0, 0, 4061, 0, 0, 936, 0, 0, 0, 0, 2488, 0, 0, 1616, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6569, 6175, 664, 5252, 1868, 3139, 4997, 4104] avg=4096.0000 max=6569 max_rank=0 min=664 min_rank=2 imbalance_ratio=1.603760 max_load_reduction=26199 max_load_reduction_pct=0.799530 moved_to_echo=14108 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[194, 3854, 39, 289, 0, 0, 0, 0, 219, 0, 2, 0, 1924, 1089, 0, 0, 0, 3533, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1198, 0, 0, 0, 0, 5, 29, 0, 4094, 1354, 563, 370, 0, 3465, 0, 488, 1, 222, 206, 0, 0, 0, 2827, 956, 3427, 0, 0, 0, 0, 31, 0, 0, 1314, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4376, 3234, 4608, 1198, 6415, 4382, 7210, 1345] avg=4096.0000 max=7210 max_rank=6 min=1198 min_rank=3 imbalance_ratio=1.760254 max_load_reduction=25558 max_load_reduction_pct=0.779968 moved_to_echo=19352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 3690, 7, 60, 0, 0, 0, 0, 127, 0, 0, 0, 2434, 0, 0, 0, 3012, 3499, 0, 39, 0, 0, 0, 0, 825, 164, 4, 0, 0, 0, 0, 0, 4028, 0, 0, 1, 0, 557, 0, 0, 4083, 0, 6, 0, 0, 197, 0, 0, 48, 1332, 0, 0, 0, 0, 0, 0, 4042, 2899, 52, 1662, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3757, 2561, 6550, 993, 4586, 4286, 1380, 8655] avg=4096.0000 max=8655 max_rank=7 min=993 min_rank=3 imbalance_ratio=2.113037 max_load_reduction=24113 max_load_reduction_pct=0.735870 moved_to_echo=18907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4012, 3858, 0, 0, 0, 0, 0, 0, 0, 2, 67, 0, 640, 389, 0, 0, 0, 3364, 4095, 6, 0, 0, 0, 0, 2, 2301, 3946, 429, 0, 0, 0, 0, 1, 2727, 21, 0, 483, 536, 128, 0, 3012, 7, 0, 180, 77, 206, 0, 0, 0, 1, 648, 1189, 0, 0, 0, 0, 2, 2, 0, 437, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7870, 1098, 7465, 6678, 3896, 3482, 1838, 441] avg=4096.0000 max=7870 max_rank=0 min=441 min_rank=7 imbalance_ratio=1.921387 max_load_reduction=24898 max_load_reduction_pct=0.759827 moved_to_echo=9657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[111, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1703, 0, 2405, 0, 0, 0, 1, 3451, 27, 0, 0, 0, 0, 0, 1415, 0, 4084, 0, 0, 0, 0, 0, 2112, 0, 0, 0, 0, 549, 352, 0, 207, 0, 0, 4094, 211, 0, 0, 0, 271, 4070, 808, 3260, 0, 0, 0, 0, 1911, 76, 5, 1642, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[111, 4111, 3479, 5499, 3013, 4512, 8409, 3634] avg=4096.0000 max=8409 max_rank=6 min=111 min_rank=0 imbalance_ratio=2.052979 max_load_reduction=24359 max_load_reduction_pct=0.743378 moved_to_echo=19568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[22, 2895, 0, 2514, 0, 0, 0, 0, 1, 11, 0, 61, 308, 1207, 0, 0, 286, 3156, 2643, 71, 0, 0, 0, 0, 3737, 0, 0, 1326, 0, 0, 0, 0, 3969, 0, 0, 0, 1502, 502, 0, 0, 4096, 0, 0, 0, 0, 154, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4073, 0, 19, 211, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5431, 1588, 6156, 5063, 5973, 4250, 4, 4303] avg=4096.0000 max=6156 max_rank=2 min=4 min_rank=6 imbalance_ratio=1.502930 max_load_reduction=26612 max_load_reduction_pct=0.812134 moved_to_echo=14530 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 4096, 7424, 4224, 12544, 3840, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[11520, 9088, 8064, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8320, 18560, 128, 6016, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10240, 2816, 256, 9856, 4864, 3456, 1792, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[4480, 10496, 2432, 15488, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[3712, 21120, 7936, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[23168, 1024, 640, 6272, 1024, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16384, 3968, 4224, 8576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 102, 141, 0, 4035, 0, 97, 0, 0, 0, 1507, 11, 3, 3, 4054, 0, 0, 0, 3525, 3912, 440, 3734, 25, 30, 17, 9, 3974, 548, 2, 0, 2503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4077, 0, 1, 4096, 2, 1, 95, 522, 3650, 4080, 13, 0, 0, 1, 3886, 3484, 240, 0, 3908, 0, 28, 16, 10, 0, 390, 274, 3994, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[470, 0, 131, 0, 3052, 624, 10, 3074, 788, 0, 0, 0, 403, 3860, 0, 4031, 0, 0, 5, 3904, 0, 1, 4096, 3726, 201, 9, 0, 28, 0, 0, 260, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 3059, 3207, 4096, 4040, 0, 0, 0, 0, 0, 4077, 103, 14, 0, 3708, 197, 0, 0, 1598, 2, 0, 0, 3785, 0, 0, 0, 788] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4043, 0, 3866, 0, 3510, 0, 3235, 811, 3674, 7, 0, 0, 3980, 0, 0, 4090, 0, 232, 0, 4095, 134, 0, 0, 5, 0, 2, 408, 0, 0, 139, 537, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 0, 3061, 0, 91, 3, 793, 1720, 0, 3601, 0, 0, 0, 1521, 4068, 0, 0, 3850, 359, 0, 1542, 1618, 0, 2232, 24, 0, 14, 55, 23, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[2027, 15, 36, 0, 151, 328, 0, 2, 4079, 126, 3082, 11, 3462, 4094, 1, 4087, 0, 3, 200, 1726, 18, 0, 1855, 0, 0, 0, 1, 4096, 0, 0, 0, 3368] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 547, 4076, 0, 0, 4068, 0, 2762, 15, 253, 0, 3954, 0, 3542, 0, 1622, 3222, 3, 0, 4096, 10, 0, 608, 0, 0, 0, 54, 95, 0, 0, 1, 3840] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[470, 0, 131, 0, 478, 304, 0, 0, 556, 624, 10, 3074, 0, 0, 0, 0, 788, 0, 0, 0, 55, 0, 0, 0, 403, 3860, 0, 974, 0, 0, 0, 0, 0, 0, 5, 2893, 0, 0, 0, 0, 0, 1, 4096, 3726, 652, 0, 0, 0, 201, 9, 0, 28, 3046, 0, 0, 0, 0, 0, 260, 4095, 2018, 11, 0, 0] rerouted_tokens_per_ep_rank=[1383, 4264, 843, 5237, 2898, 8475, 3284, 6384] avg=4096.0000 max=8475 max_rank=5 min=843 min_rank=2 imbalance_ratio=2.069092 max_load_reduction=24293 max_load_reduction_pct=0.741364 moved_to_echo=21041 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 639, 271, 0, 0, 743, 0, 1, 4096, 0, 0, 0, 0, 2, 1, 95, 522, 49, 0, 0, 0, 3650, 4080, 13, 0, 0, 0, 0, 0, 0, 1, 3886, 2583, 0, 0, 0, 0, 240, 0, 3908, 0, 581, 0, 0, 0, 28, 16, 10, 0, 0, 0, 0, 0, 390, 274, 3994, 0, 2695, 0, 0, 0] rerouted_tokens_per_ep_rank=[910, 4840, 669, 7743, 6470, 4729, 54, 7353] avg=4096.0000 max=7743 max_rank=3 min=54 min_rank=6 imbalance_ratio=1.890381 max_load_reduction=25025 max_load_reduction_pct=0.763702 moved_to_echo=18606 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 102, 141, 0, 274, 0, 0, 0, 4035, 0, 97, 0, 0, 0, 0, 0, 0, 0, 1507, 49, 0, 0, 0, 11, 3, 3, 980, 0, 0, 0, 0, 0, 0, 0, 2614, 0, 0, 0, 0, 3912, 440, 3734, 25, 588, 0, 0, 0, 30, 17, 9, 3974, 3063, 0, 0, 0, 548, 2, 0, 2503, 0, 11, 0, 0] rerouted_tokens_per_ep_rank=[4613, 4132, 1556, 997, 2614, 8699, 7093, 3064] avg=4096.0000 max=8699 max_rank=5 min=997 min_rank=3 imbalance_ratio=2.123779 max_load_reduction=24069 max_load_reduction_pct=0.734528 moved_to_echo=21470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4095, 0, 483, 303, 0, 0, 551, 0, 91, 3, 0, 0, 0, 0, 793, 1720, 0, 3601, 58, 0, 0, 0, 0, 0, 0, 360, 0, 0, 0, 0, 4068, 0, 0, 2843, 0, 0, 0, 0, 359, 0, 1542, 1618, 646, 0, 0, 0, 0, 2232, 24, 0, 1153, 0, 0, 0, 14, 55, 23, 2, 2027, 8, 0, 0] rerouted_tokens_per_ep_rank=[8977, 645, 6172, 360, 6911, 4165, 3409, 2129] avg=4096.0000 max=8977 max_rank=0 min=360 min_rank=3 imbalance_ratio=2.191650 max_load_reduction=23791 max_load_reduction_pct=0.726044 moved_to_echo=16614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 641, 289, 0, 0, 746, 0, 3059, 3207, 0, 0, 0, 0, 4096, 4040, 0, 0, 52, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 0, 103, 14, 0, 2748, 0, 0, 0, 0, 197, 0, 0, 1598, 619, 0, 0, 0, 2, 0, 0, 3785, 3081, 0, 0, 0, 0, 0, 0, 788, 2707, 11, 0, 0] rerouted_tokens_per_ep_rank=[930, 7012, 8188, 985, 2865, 2414, 6868, 3506] avg=4096.0000 max=8188 max_rank=2 min=930 min_rank=0 imbalance_ratio=1.999023 max_load_reduction=24580 max_load_reduction_pct=0.750122 moved_to_echo=15653 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2027, 15, 36, 0, 23, 134, 0, 0, 29, 328, 0, 2, 0, 0, 0, 0, 4079, 126, 3082, 11, 24, 0, 0, 0, 3462, 4094, 1, 987, 0, 0, 0, 0, 0, 3, 200, 1280, 0, 0, 0, 0, 18, 0, 1855, 0, 288, 0, 0, 0, 0, 0, 1, 4096, 3088, 0, 0, 0, 0, 0, 0, 3368, 99, 12, 0, 0] rerouted_tokens_per_ep_rank=[2235, 359, 7322, 8544, 1483, 2161, 7185, 3479] avg=4096.0000 max=8544 max_rank=3 min=359 min_rank=1 imbalance_ratio=2.085938 max_load_reduction=24224 max_load_reduction_pct=0.739258 moved_to_echo=14308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4043, 0, 3866, 0, 550, 319, 0, 0, 640, 0, 3235, 811, 0, 0, 0, 0, 3674, 7, 0, 0, 57, 0, 0, 0, 3980, 0, 0, 987, 0, 0, 0, 0, 0, 232, 0, 3035, 0, 0, 0, 0, 134, 0, 0, 5, 684, 0, 0, 0, 0, 2, 408, 0, 3091, 0, 0, 0, 0, 139, 537, 0, 2320, 12, 0, 0] rerouted_tokens_per_ep_rank=[8778, 4686, 3738, 4967, 3267, 823, 3501, 3008] avg=4096.0000 max=8778 max_rank=0 min=823 min_rank=5 imbalance_ratio=2.143066 max_load_reduction=23990 max_load_reduction_pct=0.732117 moved_to_echo=10599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 547, 4076, 0, 0, 319, 0, 0, 0, 4068, 0, 2762, 0, 0, 0, 0, 15, 253, 0, 3954, 58, 0, 0, 0, 0, 3542, 0, 393, 0, 0, 0, 0, 3222, 3, 0, 3035, 0, 0, 0, 0, 10, 0, 608, 0, 684, 0, 0, 0, 0, 0, 54, 95, 1225, 0, 0, 0, 0, 0, 1, 3840, 0, 4, 0, 0] rerouted_tokens_per_ep_rank=[4942, 6830, 4280, 3935, 6260, 1302, 1374, 3845] avg=4096.0000 max=6830 max_rank=1 min=1302 min_rank=5 imbalance_ratio=1.667480 max_load_reduction=25938 max_load_reduction_pct=0.791565 moved_to_echo=12781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10752, 4736, 12416, 256, 2816, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 2304, 512, 12032, 17792, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13568, 6272, 3200, 9600, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4992, 512, 15744, 7040, 4864, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11520, 15616, 128, 5760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[7424, 256, 4096, 21120, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 512, 4864, 14720, 11904, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[3328, 9088, 6400, 14080, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 3204, 4096, 0, 332, 235, 0, 0, 222, 0, 2893, 2, 429, 13, 259, 44, 0, 4043, 0, 4073, 4091, 541, 0, 0, 2139, 0, 3420, 143, 2588, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 3965, 4065, 0, 202, 289, 0, 379, 98, 1438, 1930, 1, 643, 0, 553, 4079, 2583, 3448, 15, 0, 30, 2890, 0, 0, 0, 0, 3141, 35, 2984, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4066, 0, 0, 122, 4096, 654, 2, 7, 19, 0, 0, 104, 1825, 40, 0, 0, 4085, 0, 0, 4007, 3602, 66, 0, 0, 0, 4085, 0, 1910, 0, 4062, 16] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[306, 451, 0, 0, 0, 4096, 4013, 2, 31, 0, 0, 3768, 0, 0, 11, 2987, 2516, 1969, 4096, 3, 61, 3993, 0, 0, 3, 266, 434, 3703, 0, 10, 38, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4033, 1649, 1808, 41, 1, 2718, 0, 0, 3312, 358, 1, 0, 17, 0, 0, 163, 16, 3661, 412, 1, 2425, 0, 0, 161, 3799, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1480, 3640, 2619, 0, 0, 4032, 0, 117, 99, 4096, 81, 299, 0, 0, 0, 672, 0, 4, 197, 0, 340, 3890, 0, 0, 0, 0, 3754, 0, 4096, 0, 3352, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 3209, 0, 1, 1260, 16, 0, 0, 2835, 0, 22, 1, 0, 233, 953, 1811, 1986, 4095, 0, 88, 3738, 4096, 28, 0, 0, 0, 4089, 0, 596, 6, 3705, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4025, 0, 67, 280, 50, 47, 450, 0, 3278, 2, 259, 3832, 2684, 5, 0, 0, 1417, 0, 297, 3156, 4094, 0, 95, 0, 543, 4096, 4, 4087, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2010, 4096, 0, 0, 0, 0, 0, 332, 235, 0, 0, 142, 554, 0, 0, 222, 0, 2893, 2, 496, 801, 0, 0, 429, 13, 259, 44, 1194, 35, 0, 0, 0, 2688, 0, 4073, 0, 0, 0, 0, 4091, 364, 0, 0, 0, 0, 0, 0, 2139, 0, 2772, 143, 0, 0, 0, 0, 2588, 0, 0, 1, 152, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 1263, 4414, 1974, 6761, 4455, 5054, 2741] avg=4096.0000 max=6761 max_rank=4 min=1263 min_rank=1 imbalance_ratio=1.650635 max_load_reduction=26007 max_load_reduction_pct=0.793671 moved_to_echo=19011 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2526, 1649, 1808, 0, 0, 0, 0, 41, 1, 2718, 0, 642, 5, 0, 0, 0, 3312, 358, 1, 597, 7, 0, 0, 0, 17, 0, 0, 1507, 162, 0, 0, 163, 4, 3661, 412, 0, 0, 0, 0, 1, 1621, 0, 0, 0, 0, 0, 0, 161, 3799, 3313, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 186, 0, 0, 0] rerouted_tokens_per_ep_rank=[5983, 3407, 4275, 1686, 4240, 1622, 7273, 4282] avg=4096.0000 max=7273 max_rank=6 min=1622 min_rank=5 imbalance_ratio=1.775635 max_load_reduction=25495 max_load_reduction_pct=0.778046 moved_to_echo=17417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2525, 0, 67, 0, 0, 0, 0, 280, 50, 47, 450, 1077, 194, 0, 0, 0, 3278, 2, 259, 594, 280, 0, 0, 3832, 2684, 5, 0, 1500, 265, 0, 0, 0, 943, 0, 297, 0, 0, 0, 0, 3156, 2752, 0, 95, 0, 0, 0, 0, 0, 543, 3320, 4, 0, 0, 0, 0, 4087, 0, 0, 0, 182, 0, 0, 0] rerouted_tokens_per_ep_rank=[2592, 2098, 4413, 8286, 1240, 6003, 3867, 4269] avg=4096.0000 max=8286 max_rank=3 min=1240 min_rank=4 imbalance_ratio=2.022949 max_load_reduction=24482 max_load_reduction_pct=0.747131 moved_to_echo=15379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2550, 0, 0, 0, 0, 0, 0, 122, 4096, 654, 2, 947, 560, 0, 0, 7, 19, 0, 0, 592, 810, 0, 0, 104, 1825, 40, 0, 1516, 233, 0, 0, 0, 2715, 0, 0, 0, 0, 0, 0, 4007, 2422, 66, 0, 0, 0, 0, 0, 0, 0, 3311, 0, 0, 0, 0, 0, 1910, 0, 4062, 16, 182, 0, 0, 0] rerouted_tokens_per_ep_rank=[2550, 6381, 1428, 3718, 2715, 6495, 3311, 6170] avg=4096.0000 max=6495 max_rank=5 min=1428 min_rank=2 imbalance_ratio=1.585693 max_load_reduction=26273 max_load_reduction_pct=0.801788 moved_to_echo=18691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2013, 0, 1, 0, 0, 0, 0, 1260, 16, 0, 0, 1077, 561, 0, 0, 2835, 0, 22, 1, 593, 812, 0, 0, 0, 233, 953, 1811, 1196, 265, 0, 0, 1986, 2722, 0, 88, 0, 0, 0, 0, 3738, 2754, 28, 0, 0, 0, 0, 0, 0, 0, 3314, 0, 0, 0, 0, 0, 596, 6, 3705, 0, 182, 0, 0, 0] rerouted_tokens_per_ep_rank=[2014, 2914, 4263, 4458, 4796, 6520, 3314, 4489] avg=4096.0000 max=6520 max_rank=5 min=2014 min_rank=0 imbalance_ratio=1.591797 max_load_reduction=26248 max_load_reduction_pct=0.801025 moved_to_echo=19119 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2487, 4065, 0, 0, 0, 0, 0, 202, 289, 0, 379, 760, 472, 0, 0, 98, 1438, 1930, 1, 455, 683, 0, 0, 643, 0, 553, 4079, 1478, 187, 0, 0, 2583, 2293, 15, 0, 0, 0, 0, 0, 30, 1943, 0, 0, 0, 0, 0, 0, 0, 0, 2546, 35, 0, 0, 0, 0, 2984, 0, 0, 0, 140, 0, 0, 0] rerouted_tokens_per_ep_rank=[6552, 2102, 4605, 6940, 4891, 1973, 2581, 3124] avg=4096.0000 max=6940 max_rank=3 min=1973 min_rank=5 imbalance_ratio=1.694336 max_load_reduction=25828 max_load_reduction_pct=0.788208 moved_to_echo=12569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[306, 283, 0, 0, 0, 0, 0, 0, 0, 4096, 4013, 2, 1050, 270, 0, 0, 31, 0, 0, 3768, 62, 390, 0, 0, 0, 0, 11, 2987, 168, 259, 0, 0, 2516, 1309, 4096, 3, 0, 0, 0, 0, 61, 2684, 0, 0, 0, 0, 0, 0, 3, 266, 353, 3703, 0, 0, 0, 0, 0, 10, 38, 11, 19, 0, 0, 0] rerouted_tokens_per_ep_rank=[589, 9431, 4251, 3425, 7924, 2745, 4325, 78] avg=4096.0000 max=9431 max_rank=1 min=78 min_rank=7 imbalance_ratio=2.302490 max_load_reduction=23337 max_load_reduction_pct=0.712189 moved_to_echo=15072 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1480, 2283, 2619, 0, 0, 0, 0, 0, 0, 4032, 0, 117, 1023, 0, 0, 0, 99, 4096, 81, 299, 544, 0, 0, 0, 0, 0, 0, 672, 1357, 252, 0, 0, 0, 4, 197, 0, 0, 0, 0, 0, 340, 2615, 0, 0, 0, 0, 0, 0, 0, 0, 3043, 0, 0, 0, 0, 0, 4096, 0, 3352, 0, 167, 0, 0, 0] rerouted_tokens_per_ep_rank=[6382, 5172, 5119, 2281, 201, 2955, 3043, 7615] avg=4096.0000 max=7615 max_rank=7 min=201 min_rank=4 imbalance_ratio=1.859131 max_load_reduction=25153 max_load_reduction_pct=0.767609 moved_to_echo=13814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2304, 12928, 7552, 1024, 6784, 2688, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1792, 16768, 12544, 1920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5120, 4864, 1920, 9600, 9984, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7296, 12800, 8064, 4992, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15488, 17280, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[20480, 128, 11264, 128, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[2304, 4608, 22016, 3968, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3328, 12160, 5376, 4352, 3968, 3840, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[272, 3062, 785, 418, 1272, 492, 820, 1116, 1436, 198, 1601, 413, 1165, 759, 151, 400, 200, 793, 1612, 741, 996, 169, 2792, 1858, 364, 846, 1155, 1035, 1570, 612, 1822, 1843] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[51, 2838, 577, 460, 1942, 421, 802, 690, 1115, 128, 2059, 299, 1073, 731, 200, 561, 329, 1008, 648, 895, 718, 221, 2797, 2359, 667, 1567, 1119, 1287, 1027, 367, 1425, 2387] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[74, 2616, 598, 297, 1685, 341, 1179, 1150, 1155, 175, 1307, 369, 1160, 937, 430, 364, 70, 488, 628, 997, 1371, 100, 3477, 2062, 795, 967, 1091, 1277, 1421, 282, 1555, 2350] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[444, 2843, 402, 566, 1702, 349, 808, 1091, 1451, 179, 1569, 222, 1174, 1008, 245, 480, 105, 659, 1193, 751, 829, 249, 3176, 2338, 745, 861, 878, 1173, 1063, 617, 1913, 1685] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[69, 2868, 1123, 433, 1774, 609, 1202, 1014, 962, 170, 1873, 637, 1323, 583, 299, 490, 230, 429, 970, 825, 1074, 74, 3545, 2225, 734, 1076, 922, 987, 1171, 289, 1108, 1680] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[63, 3172, 553, 326, 1528, 330, 913, 1050, 1524, 108, 1672, 165, 1135, 1284, 559, 438, 73, 576, 875, 530, 1202, 199, 2832, 2195, 895, 951, 909, 1028, 1230, 484, 2000, 1969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[81, 2554, 819, 238, 1275, 316, 966, 708, 1425, 217, 1658, 326, 544, 1113, 367, 480, 137, 700, 1716, 839, 1049, 286, 2945, 2186, 655, 1288, 849, 1259, 1446, 417, 2142, 1767] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[155, 3006, 960, 333, 1630, 442, 882, 1251, 923, 75, 1268, 317, 1452, 777, 254, 525, 304, 555, 1475, 1628, 787, 90, 2853, 1570, 587, 727, 971, 1239, 1746, 621, 1290, 2075] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[74, 2584, 598, 297, 0, 0, 0, 0, 1685, 341, 1179, 1150, 108, 32, 0, 0, 1155, 175, 1307, 369, 860, 0, 0, 0, 1160, 937, 430, 364, 1467, 0, 0, 0, 70, 488, 628, 997, 1074, 334, 0, 0, 1371, 100, 936, 2062, 0, 0, 0, 0, 795, 967, 1091, 1277, 278, 0, 0, 0, 1421, 282, 1555, 770, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3553, 4495, 3866, 4358, 3591, 4469, 4408, 4028] avg=4096.0000 max=4495 max_rank=1 min=3553 min_rank=0 imbalance_ratio=1.097412 max_load_reduction=28273 max_load_reduction_pct=0.862823 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[69, 2833, 1123, 433, 0, 0, 0, 0, 1774, 609, 1202, 1014, 77, 35, 0, 0, 962, 170, 1873, 637, 615, 0, 0, 0, 1323, 583, 299, 490, 1496, 0, 0, 0, 230, 429, 970, 825, 1096, 238, 0, 0, 1074, 74, 953, 2225, 0, 0, 0, 0, 734, 1076, 922, 987, 198, 0, 0, 0, 1171, 289, 1108, 552, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4458, 4711, 4257, 4191, 3788, 4326, 3917, 3120] avg=4096.0000 max=4711 max_rank=1 min=3120 min_rank=7 imbalance_ratio=1.150146 max_load_reduction=28057 max_load_reduction_pct=0.856232 moved_to_echo=15151 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[272, 3024, 785, 418, 0, 0, 0, 0, 1272, 492, 820, 1116, 85, 38, 0, 0, 1436, 198, 1601, 413, 675, 0, 0, 0, 1165, 759, 151, 400, 1178, 0, 0, 0, 200, 793, 1612, 741, 863, 262, 0, 0, 996, 169, 751, 1858, 0, 0, 0, 0, 364, 846, 1155, 1035, 218, 0, 0, 0, 1570, 612, 1822, 603, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4499, 3823, 4323, 3653, 4471, 3774, 3618, 4607] avg=4096.0000 max=4607 max_rank=7 min=3618 min_rank=6 imbalance_ratio=1.124756 max_load_reduction=28161 max_load_reduction_pct=0.859406 moved_to_echo=16470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[444, 2803, 402, 566, 0, 0, 0, 0, 1702, 349, 808, 1091, 82, 40, 0, 0, 1451, 179, 1569, 222, 620, 0, 0, 0, 1174, 1008, 245, 480, 1344, 0, 0, 0, 105, 659, 1193, 751, 985, 243, 0, 0, 829, 249, 847, 2338, 0, 0, 0, 0, 745, 861, 878, 1173, 202, 0, 0, 0, 1063, 617, 1913, 538, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4215, 4072, 4041, 4251, 3936, 4263, 3859, 4131] avg=4096.0000 max=4263 max_rank=5 min=3859 min_rank=6 imbalance_ratio=1.040771 max_load_reduction=28505 max_load_reduction_pct=0.869904 moved_to_echo=16189 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[63, 3133, 553, 326, 0, 0, 0, 0, 1528, 330, 913, 1050, 91, 39, 0, 0, 1524, 108, 1672, 165, 721, 0, 0, 0, 1135, 1284, 559, 438, 1195, 0, 0, 0, 73, 576, 875, 530, 875, 279, 0, 0, 1202, 199, 762, 2195, 0, 0, 0, 0, 895, 951, 909, 1028, 232, 0, 0, 0, 1230, 484, 2000, 646, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 3951, 4190, 4611, 3208, 4358, 4015, 4360] avg=4096.0000 max=4611 max_rank=3 min=3208 min_rank=4 imbalance_ratio=1.125732 max_load_reduction=28157 max_load_reduction_pct=0.859283 moved_to_echo=15941 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[51, 2803, 577, 460, 0, 0, 0, 0, 1942, 421, 802, 690, 110, 35, 0, 0, 1115, 128, 2059, 299, 874, 0, 0, 0, 1073, 731, 200, 561, 1180, 0, 0, 0, 329, 1008, 648, 895, 864, 339, 0, 0, 718, 221, 753, 2359, 0, 0, 0, 0, 667, 1567, 1119, 1287, 282, 0, 0, 0, 1027, 367, 1425, 782, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3891, 4000, 4475, 3745, 4083, 4051, 4922, 3601] avg=4096.0000 max=4922 max_rank=6 min=3601 min_rank=7 imbalance_ratio=1.201660 max_load_reduction=27846 max_load_reduction_pct=0.849792 moved_to_echo=16657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[81, 2522, 819, 238, 0, 0, 0, 0, 1275, 316, 966, 708, 81, 32, 0, 0, 1425, 217, 1658, 326, 647, 0, 0, 0, 544, 1113, 367, 480, 1243, 0, 0, 0, 137, 700, 1716, 839, 910, 251, 0, 0, 1049, 286, 792, 2186, 0, 0, 0, 0, 655, 1288, 849, 1259, 209, 0, 0, 0, 1446, 417, 2142, 579, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3660, 3378, 4273, 3747, 4553, 4313, 4260, 4584] avg=4096.0000 max=4584 max_rank=7 min=3378 min_rank=1 imbalance_ratio=1.119141 max_load_reduction=28184 max_load_reduction_pct=0.860107 moved_to_echo=17710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[155, 2969, 960, 333, 0, 0, 0, 0, 1630, 442, 882, 1251, 96, 37, 0, 0, 923, 75, 1268, 317, 760, 0, 0, 0, 1452, 777, 254, 525, 1204, 0, 0, 0, 304, 555, 1475, 1628, 882, 294, 0, 0, 787, 90, 767, 1570, 0, 0, 0, 0, 587, 727, 971, 1239, 245, 0, 0, 0, 1746, 621, 1290, 680, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4417, 4338, 3343, 4212, 5138, 3214, 3769, 4337] avg=4096.0000 max=5138 max_rank=4 min=3214 min_rank=5 imbalance_ratio=1.254395 max_load_reduction=27630 max_load_reduction_pct=0.843201 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1536, 5248, 9216, 7296, 7552, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10752, 3712, 13312, 5248, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5504, 8320, 7936, 9344, 1920, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9088, 7296, 2560, 3840, 10368, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8064, 1408, 6656, 16896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[12928, 3328, 7680, 8192, 768, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 22784, 5888, 3072, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10112, 1280, 13056, 2816, 5888, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[265, 5, 87, 455, 2707, 3667, 2319, 3774, 7, 0, 284, 236, 7, 3518, 17, 46, 0, 694, 0, 3652, 86, 21, 6, 0, 2592, 12, 56, 4065, 4087, 0, 103, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2150, 0, 1766, 38, 3417, 1267, 519, 575, 129, 0, 145, 398, 25, 3573, 0, 2966, 114, 1648, 4, 3998, 57, 2, 90, 4, 850, 190, 1283, 3588, 3808, 0, 4, 160] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[350, 670, 1076, 1, 2155, 3493, 268, 3809, 3, 4, 432, 816, 458, 3526, 2, 2202, 2, 1402, 9, 4085, 1, 5, 0, 5, 72, 15, 490, 3533, 3878, 4, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[523, 16, 1264, 3, 1761, 3936, 469, 3138, 107, 0, 2381, 1527, 9, 1514, 7, 1635, 0, 11, 0, 4093, 2, 3, 27, 14, 1204, 220, 41, 4096, 4054, 1, 126, 586] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[362, 0, 2790, 2, 3301, 3825, 767, 2379, 251, 0, 104, 513, 14, 4049, 27, 1466, 0, 18, 0, 4046, 12, 277, 2, 29, 540, 41, 44, 4072, 3803, 1, 15, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[69, 0, 1092, 10, 3932, 3693, 746, 158, 6, 59, 886, 807, 92, 3880, 15, 372, 257, 1753, 0, 3463, 1268, 156, 14, 6, 1995, 33, 54, 3768, 4032, 19, 133, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[226, 7, 1681, 240, 1292, 745, 3185, 1317, 113, 0, 133, 880, 3, 2461, 1622, 457, 139, 2029, 4, 4063, 16, 0, 1, 10, 3789, 5, 356, 4060, 3908, 20, 2, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[123, 0, 888, 476, 3407, 3652, 542, 3331, 172, 0, 0, 910, 17, 3835, 5, 1736, 575, 72, 0, 4083, 1, 43, 0, 361, 237, 24, 257, 3921, 4090, 5, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[265, 5, 87, 455, 863, 906, 235, 12, 675, 0, 2319, 3774, 0, 0, 0, 0, 7, 0, 284, 236, 1429, 1297, 0, 0, 7, 2612, 17, 46, 0, 0, 0, 0, 0, 694, 0, 2789, 0, 0, 0, 0, 86, 21, 6, 0, 3667, 735, 0, 0, 2592, 12, 56, 2401, 0, 0, 0, 0, 4075, 0, 103, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2828, 6768, 3253, 2682, 3483, 4515, 5061, 4178] avg=4096.0000 max=6768 max_rank=1 min=2682 min_rank=3 imbalance_ratio=1.652344 max_load_reduction=26000 max_load_reduction_pct=0.793457 moved_to_echo=17237 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[350, 670, 1076, 1, 966, 908, 204, 12, 538, 0, 268, 3809, 0, 0, 0, 0, 3, 4, 432, 816, 1242, 1032, 0, 0, 458, 2618, 2, 2202, 0, 0, 0, 0, 2, 1402, 9, 3119, 0, 0, 0, 0, 1, 5, 0, 5, 3493, 585, 0, 0, 72, 15, 490, 2087, 0, 0, 0, 0, 3866, 4, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4187, 4615, 3529, 5280, 4532, 4089, 2664, 3872] avg=4096.0000 max=5280 max_rank=3 min=2664 min_rank=6 imbalance_ratio=1.289062 max_load_reduction=27488 max_load_reduction_pct=0.838867 moved_to_echo=15157 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2150, 0, 1766, 38, 945, 920, 207, 12, 852, 0, 519, 575, 0, 0, 0, 0, 129, 0, 145, 398, 1261, 1637, 0, 0, 25, 2653, 0, 2966, 0, 0, 0, 0, 114, 1648, 4, 3053, 0, 0, 0, 0, 57, 2, 90, 4, 1267, 928, 0, 0, 850, 190, 1283, 2120, 0, 0, 0, 0, 3796, 0, 4, 160, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6038, 1946, 3570, 5644, 4819, 2348, 4443, 3960] avg=4096.0000 max=6038 max_rank=0 min=1946 min_rank=1 imbalance_ratio=1.474121 max_load_reduction=26730 max_load_reduction_pct=0.815735 moved_to_echo=15570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[362, 0, 2790, 2, 956, 1042, 235, 12, 823, 0, 767, 2379, 0, 0, 0, 0, 251, 0, 104, 513, 1431, 1581, 0, 0, 14, 3007, 27, 1466, 0, 0, 0, 0, 0, 18, 0, 3090, 0, 0, 0, 0, 12, 277, 2, 29, 3825, 897, 0, 0, 540, 41, 44, 2406, 0, 0, 0, 0, 3791, 1, 15, 18, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5399, 3969, 3880, 4514, 3108, 5042, 3031, 3825] avg=4096.0000 max=5399 max_rank=0 min=3031 min_rank=6 imbalance_ratio=1.318115 max_load_reduction=27369 max_load_reduction_pct=0.835236 moved_to_echo=15006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[523, 16, 1264, 3, 968, 389, 236, 12, 440, 0, 469, 3138, 0, 0, 0, 0, 107, 0, 2381, 1527, 1440, 843, 0, 0, 9, 1125, 7, 1635, 0, 0, 0, 0, 0, 11, 0, 3125, 0, 0, 0, 0, 2, 3, 27, 14, 3936, 478, 0, 0, 1204, 220, 41, 2420, 0, 0, 0, 0, 4042, 1, 126, 586, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3411, 4047, 6298, 2776, 3136, 4460, 3885, 4755] avg=4096.0000 max=6298 max_rank=2 min=2776 min_rank=3 imbalance_ratio=1.537598 max_load_reduction=26470 max_load_reduction_pct=0.807800 moved_to_echo=16236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[226, 7, 1681, 240, 960, 633, 234, 12, 322, 0, 3185, 1317, 0, 0, 0, 0, 113, 0, 133, 880, 1427, 619, 0, 0, 3, 1828, 1622, 457, 0, 0, 0, 0, 139, 2029, 4, 3103, 0, 0, 0, 0, 16, 0, 1, 10, 745, 351, 0, 0, 3789, 5, 356, 2399, 0, 0, 0, 0, 3896, 20, 2, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3993, 4824, 3172, 3910, 5275, 1123, 6549, 3922] avg=4096.0000 max=6549 max_rank=6 min=1123 min_rank=5 imbalance_ratio=1.598877 max_load_reduction=26219 max_load_reduction_pct=0.800140 moved_to_echo=16869 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[123, 0, 888, 476, 969, 991, 231, 16, 842, 0, 542, 3331, 0, 0, 0, 0, 172, 0, 0, 910, 1382, 1636, 0, 0, 17, 2844, 5, 1736, 0, 0, 0, 0, 575, 72, 0, 3114, 0, 0, 0, 0, 1, 43, 0, 361, 3652, 929, 0, 0, 237, 24, 257, 2308, 0, 0, 0, 0, 4074, 5, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3694, 4715, 4100, 4602, 3761, 4986, 2826, 4084] avg=4096.0000 max=4986 max_rank=5 min=2826 min_rank=6 imbalance_ratio=1.217285 max_load_reduction=27782 max_load_reduction_pct=0.847839 moved_to_echo=15657 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[69, 0, 1092, 10, 819, 999, 217, 12, 980, 0, 746, 158, 0, 0, 0, 0, 6, 59, 886, 807, 1324, 1884, 0, 0, 92, 2881, 15, 372, 0, 0, 0, 0, 257, 1753, 0, 2644, 0, 0, 0, 0, 1268, 156, 14, 6, 3693, 1068, 0, 0, 1995, 33, 54, 2227, 0, 0, 0, 0, 4020, 19, 133, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3218, 1884, 4966, 3360, 4654, 6205, 4309, 4172] avg=4096.0000 max=6205 max_rank=5 min=1884 min_rank=1 imbalance_ratio=1.514893 max_load_reduction=26563 max_load_reduction_pct=0.810638 moved_to_echo=19340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 768, 10752, 1280, 7552, 6912, 1920, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[896, 128, 4480, 6144, 11008, 10624, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[640, 19584, 1792, 10880, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11392, 640, 2688, 18432, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 7680, 128, 24064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1536, 512, 256, 512, 24320, 6016, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[5504, 0, 8832, 18560, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31616, 128, 512, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3869, 666, 469, 2174, 484, 0, 3731, 5, 3, 3408, 3566, 0, 3166, 1530, 17, 36, 33, 4089, 151, 2336, 0, 361, 4, 2, 1192, 0, 0, 0, 4, 1472, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[2049, 2092, 2233, 2854, 1974, 14, 2224, 1, 2, 114, 3564, 0, 4037, 94, 277, 0, 25, 3850, 4079, 0, 291, 1005, 0, 7, 1331, 2, 0, 542, 10, 36, 60, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3008, 3451, 142, 4072, 76, 216, 4082, 0, 2, 4094, 105, 0, 28, 2504, 4, 1648, 7, 3494, 3812, 0, 3, 751, 0, 3, 0, 0, 4, 675, 587, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1150, 3775, 0, 2340, 18, 34, 4091, 0, 0, 584, 2824, 2, 78, 3551, 1, 220, 0, 4093, 3041, 2, 0, 110, 0, 3078, 125, 0, 62, 32, 143, 17, 36, 3361] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3852, 215, 19, 63, 173, 1, 4096, 91, 0, 3697, 3352, 135, 29, 940, 7, 0, 0, 4078, 2680, 112, 0, 3002, 33, 1434, 36, 352, 17, 0, 3371, 0, 301, 682] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4024, 1533, 1, 4023, 4, 150, 4096, 1, 0, 465, 3446, 0, 193, 82, 0, 2, 360, 4096, 27, 1731, 7, 183, 10, 0, 30, 0, 161, 51, 4070, 0, 104, 3918] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1401, 3837, 180, 288, 0, 92, 4085, 911, 0, 2788, 1695, 3, 1230, 399, 2496, 0, 0, 4062, 10, 0, 0, 2243, 183, 3891, 1941, 0, 3, 0, 940, 87, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1739, 3106, 0, 127, 0, 15, 1567, 0, 0, 4094, 3018, 149, 3386, 3087, 1, 0, 19, 4096, 3463, 0, 1064, 260, 35, 497, 1254, 0, 0, 16, 1744, 0, 0, 31] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2788, 0, 2340, 0, 0, 0, 0, 18, 34, 4091, 0, 108, 0, 0, 0, 0, 584, 1732, 2, 0, 0, 0, 0, 78, 3551, 1, 220, 752, 0, 0, 0, 0, 1400, 3041, 2, 0, 0, 0, 0, 0, 110, 0, 3078, 1838, 0, 0, 0, 125, 0, 62, 32, 1150, 494, 0, 0, 143, 17, 36, 3361, 1092, 361, 127, 0] rerouted_tokens_per_ep_rank=[5128, 4251, 2318, 4602, 4443, 5026, 1863, 5137] avg=4096.0000 max=5137 max_rank=7 min=1863 min_rank=6 imbalance_ratio=1.254150 max_load_reduction=27631 max_load_reduction_pct=0.843231 moved_to_echo=16469 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2548, 142, 4072, 0, 0, 0, 0, 76, 216, 4082, 0, 99, 0, 0, 0, 2, 4094, 65, 0, 0, 0, 0, 0, 28, 2504, 4, 1648, 688, 0, 0, 0, 7, 1195, 3812, 0, 0, 0, 0, 0, 3, 751, 0, 3, 1569, 0, 0, 0, 0, 0, 4, 675, 3008, 422, 0, 0, 587, 0, 0, 0, 40, 308, 116, 0] rerouted_tokens_per_ep_rank=[6762, 4473, 4161, 4872, 5014, 2326, 4109, 1051] avg=4096.0000 max=6762 max_rank=0 min=1051 min_rank=7 imbalance_ratio=1.650879 max_load_reduction=26006 max_load_reduction_pct=0.793640 moved_to_echo=12500 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1122, 1, 4023, 0, 0, 0, 0, 4, 150, 4096, 1, 45, 0, 0, 0, 0, 465, 2111, 0, 0, 0, 0, 0, 193, 82, 0, 2, 310, 0, 0, 0, 360, 1386, 27, 1731, 0, 0, 0, 0, 7, 183, 10, 0, 1845, 0, 0, 0, 30, 0, 161, 51, 4024, 499, 0, 0, 4070, 0, 104, 3918, 1335, 366, 56, 0] rerouted_tokens_per_ep_rank=[5146, 4296, 2576, 587, 3504, 2045, 4765, 9849] avg=4096.0000 max=9849 max_rank=7 min=587 min_rank=3 imbalance_ratio=2.404541 max_load_reduction=22919 max_load_reduction_pct=0.699432 moved_to_echo=20163 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 493, 469, 2174, 0, 0, 0, 0, 484, 0, 3731, 5, 19, 0, 0, 0, 3, 3408, 2187, 0, 0, 0, 0, 0, 3166, 1530, 17, 36, 132, 0, 0, 0, 33, 1398, 151, 2336, 0, 0, 0, 0, 0, 361, 4, 2, 1836, 0, 0, 0, 1192, 0, 0, 0, 3869, 494, 0, 0, 4, 1472, 0, 0, 1379, 361, 22, 0] rerouted_tokens_per_ep_rank=[3136, 4239, 5598, 4881, 3918, 2203, 5555, 3238] avg=4096.0000 max=5598 max_rank=2 min=2203 min_rank=5 imbalance_ratio=1.366699 max_load_reduction=27170 max_load_reduction_pct=0.829163 moved_to_echo=14914 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1545, 2233, 2854, 0, 0, 0, 0, 1974, 14, 2224, 1, 60, 0, 0, 0, 2, 114, 2186, 0, 0, 0, 0, 0, 4037, 94, 277, 0, 417, 0, 0, 0, 25, 1316, 4079, 0, 0, 0, 0, 0, 291, 1005, 0, 7, 1729, 0, 0, 0, 1331, 2, 0, 542, 2049, 465, 0, 0, 10, 36, 60, 1, 1378, 340, 70, 0] rerouted_tokens_per_ep_rank=[6632, 4273, 2302, 4825, 5420, 3032, 4389, 1895] avg=4096.0000 max=6632 max_rank=0 min=1895 min_rank=7 imbalance_ratio=1.619141 max_load_reduction=26136 max_load_reduction_pct=0.797607 moved_to_echo=14736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2833, 180, 288, 0, 0, 0, 0, 0, 92, 4085, 911, 110, 0, 0, 0, 0, 2788, 1040, 3, 0, 0, 0, 0, 1230, 399, 2496, 0, 765, 0, 0, 0, 0, 1389, 10, 0, 0, 0, 0, 0, 0, 2243, 183, 3891, 1824, 0, 0, 0, 1941, 0, 3, 0, 1401, 490, 0, 0, 940, 87, 0, 3, 655, 359, 129, 0] rerouted_tokens_per_ep_rank=[3301, 5198, 3831, 4890, 1399, 8141, 3835, 2173] avg=4096.0000 max=8141 max_rank=5 min=1399 min_rank=4 imbalance_ratio=1.987549 max_load_reduction=24627 max_load_reduction_pct=0.751556 moved_to_echo=15548 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2294, 0, 127, 0, 0, 0, 0, 0, 15, 1567, 0, 89, 0, 0, 0, 0, 4094, 1851, 149, 0, 0, 0, 0, 3386, 3087, 1, 0, 619, 0, 0, 0, 19, 1401, 3463, 0, 0, 0, 0, 0, 1064, 260, 35, 497, 1839, 0, 0, 0, 1254, 0, 0, 16, 1739, 494, 0, 0, 1744, 0, 0, 31, 1167, 362, 104, 0] rerouted_tokens_per_ep_rank=[2421, 1671, 6094, 7093, 4883, 3695, 3503, 3408] avg=4096.0000 max=7093 max_rank=3 min=1671 min_rank=1 imbalance_ratio=1.731689 max_load_reduction=25675 max_load_reduction_pct=0.783539 moved_to_echo=15489 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 160, 19, 63, 0, 0, 0, 0, 173, 1, 4096, 91, 6, 0, 0, 0, 0, 3697, 2056, 135, 0, 0, 0, 0, 29, 940, 7, 0, 42, 0, 0, 0, 0, 1395, 2680, 112, 0, 0, 0, 0, 0, 3002, 33, 1434, 1831, 0, 0, 0, 36, 352, 17, 0, 3852, 492, 0, 0, 3371, 0, 301, 682, 1296, 360, 7, 0] rerouted_tokens_per_ep_rank=[242, 4367, 5888, 1018, 4187, 6300, 4749, 6017] avg=4096.0000 max=6300 max_rank=5 min=242 min_rank=0 imbalance_ratio=1.538086 max_load_reduction=26468 max_load_reduction_pct=0.807739 moved_to_echo=21253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10880, 1664, 512, 8064, 8448, 2944, 640, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1408, 7936, 384, 8960, 14336, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12160, 12288, 2816, 1920, 3840, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 13824, 3072, 16000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 19328, 13312, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[512, 10880, 17280, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6016, 384, 256, 1408, 21120, 3968, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2816, 640, 28032, 1024, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[813, 7, 0, 2845, 3803, 0, 0, 876, 0, 3434, 2, 2929, 3401, 1, 4020, 2930, 0, 1, 642, 0, 0, 0, 0, 2, 0, 3950, 1, 32, 1, 362, 83, 2633] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[3423, 47, 0, 3946, 427, 0, 154, 38, 0, 4096, 12, 754, 3493, 14, 3, 4, 0, 2626, 59, 503, 0, 0, 3, 3555, 1427, 37, 444, 4040, 1, 0, 0, 3662] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 488, 0, 3567, 7, 0, 22, 0, 3, 3669, 717, 140, 3344, 0, 1997, 9, 0, 1682, 224, 2177, 0, 0, 0, 4095, 1337, 1536, 10, 3491, 122, 36, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3901, 0, 0, 4094, 234, 143, 671, 0, 1, 4081, 0, 870, 3890, 350, 226, 145, 0, 20, 3275, 20, 13, 0, 0, 2703, 0, 4043, 0, 4064, 0, 0, 0, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 108, 14, 2829, 4073, 0, 1, 0, 1, 3940, 879, 2886, 4089, 16, 3776, 74, 0, 4088, 0, 37, 0, 156, 0, 3495, 10, 1937, 0, 126, 0, 1, 0, 232] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4073, 0, 0, 75, 4011, 0, 0, 40, 13, 3650, 958, 872, 4089, 1, 0, 124, 0, 3952, 4, 3188, 6, 0, 0, 4, 0, 2552, 0, 88, 414, 569, 8, 4077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4036, 7, 16, 1006, 16, 4089, 3, 16, 2, 2172, 3, 0, 4090, 1, 175, 814, 0, 125, 5, 3960, 0, 317, 393, 4035, 0, 116, 354, 3490, 0, 0, 27, 3500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[61, 47, 0, 4095, 4078, 452, 1425, 0, 0, 1231, 273, 22, 4041, 8, 11, 2149, 7, 301, 0, 0, 0, 2, 1, 2409, 2481, 1263, 184, 4067, 0, 15, 65, 4080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[813, 7, 0, 1986, 0, 0, 0, 0, 3803, 0, 0, 876, 724, 8, 0, 0, 0, 2798, 2, 2929, 0, 0, 0, 0, 1771, 1, 4020, 2930, 0, 0, 0, 0, 0, 1, 642, 0, 636, 135, 0, 0, 0, 0, 0, 2, 1298, 0, 0, 0, 0, 3950, 1, 9, 0, 0, 0, 0, 1, 362, 83, 2633, 15, 332, 0, 0] rerouted_tokens_per_ep_rank=[2806, 5411, 5729, 8722, 1414, 1300, 3960, 3426] avg=4096.0000 max=8722 max_rank=3 min=1300 min_rank=5 imbalance_ratio=2.129395 max_load_reduction=24046 max_load_reduction_pct=0.733826 moved_to_echo=10100 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 488, 0, 2499, 0, 0, 0, 0, 7, 0, 22, 0, 905, 447, 0, 0, 3, 2993, 717, 140, 0, 0, 0, 0, 1750, 0, 1997, 9, 0, 0, 0, 0, 0, 1682, 224, 2177, 676, 163, 0, 0, 0, 0, 0, 4095, 1272, 0, 0, 0, 1337, 1536, 10, 1997, 0, 0, 0, 0, 122, 36, 0, 4095, 1047, 322, 0, 0] rerouted_tokens_per_ep_rank=[2987, 1381, 3853, 3756, 4922, 5367, 4880, 5622] avg=4096.0000 max=5622 max_rank=7 min=1381 min_rank=1 imbalance_ratio=1.372559 max_load_reduction=27146 max_load_reduction_pct=0.828430 moved_to_echo=20791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 108, 14, 1982, 0, 0, 0, 0, 4073, 0, 1, 0, 718, 16, 0, 0, 1, 3214, 879, 2886, 0, 0, 0, 0, 2140, 16, 3776, 74, 0, 0, 0, 0, 0, 4088, 0, 37, 726, 129, 0, 0, 0, 156, 0, 3495, 1555, 0, 0, 0, 10, 1937, 0, 73, 0, 0, 0, 0, 0, 1, 0, 232, 37, 394, 0, 0] rerouted_tokens_per_ep_rank=[2104, 4808, 6980, 6006, 4980, 5206, 2020, 664] avg=4096.0000 max=6980 max_rank=2 min=664 min_rank=7 imbalance_ratio=1.704102 max_load_reduction=25788 max_load_reduction_pct=0.786987 moved_to_echo=12870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3423, 47, 0, 2765, 0, 0, 0, 0, 427, 0, 154, 38, 1001, 518, 0, 0, 0, 3342, 12, 754, 0, 0, 0, 0, 1828, 14, 3, 4, 0, 0, 0, 0, 0, 2626, 59, 503, 754, 180, 0, 0, 0, 0, 3, 3555, 1328, 0, 0, 0, 1427, 37, 444, 2310, 0, 0, 0, 0, 1, 0, 0, 3662, 1212, 337, 0, 0] rerouted_tokens_per_ep_rank=[6235, 2138, 4108, 1849, 4122, 4886, 4218, 5212] avg=4096.0000 max=6235 max_rank=0 min=1849 min_rank=3 imbalance_ratio=1.522217 max_load_reduction=26533 max_load_reduction_pct=0.809723 moved_to_echo=18438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3901, 0, 0, 2868, 0, 0, 0, 0, 234, 143, 671, 0, 1039, 521, 0, 0, 1, 3329, 0, 870, 0, 0, 0, 0, 2036, 350, 226, 145, 0, 0, 0, 0, 0, 20, 3275, 20, 752, 187, 0, 0, 13, 0, 0, 2703, 1479, 0, 0, 0, 0, 4043, 0, 2324, 0, 0, 0, 0, 0, 0, 0, 24, 1219, 375, 0, 0] rerouted_tokens_per_ep_rank=[6769, 2608, 4200, 2757, 4254, 4195, 6367, 1618] avg=4096.0000 max=6769 max_rank=0 min=1618 min_rank=7 imbalance_ratio=1.652588 max_load_reduction=25999 max_load_reduction_pct=0.793427 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[61, 47, 0, 2869, 0, 0, 0, 0, 4078, 452, 1425, 0, 1039, 521, 0, 0, 0, 1005, 273, 22, 0, 0, 0, 0, 2114, 8, 11, 2149, 0, 0, 0, 0, 7, 301, 0, 0, 226, 187, 0, 0, 0, 2, 1, 2409, 1537, 0, 0, 0, 2481, 1263, 184, 2326, 0, 0, 0, 0, 0, 15, 65, 4080, 1220, 390, 0, 0] rerouted_tokens_per_ep_rank=[2977, 7515, 1300, 4282, 721, 3949, 6254, 5770] avg=4096.0000 max=7515 max_rank=1 min=721 min_rank=4 imbalance_ratio=1.834717 max_load_reduction=25253 max_load_reduction_pct=0.770660 moved_to_echo=16694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4036, 7, 16, 705, 0, 0, 0, 0, 16, 4089, 3, 16, 255, 447, 0, 0, 2, 1772, 3, 0, 0, 0, 0, 0, 2141, 1, 175, 814, 0, 0, 0, 0, 0, 125, 5, 3960, 400, 46, 0, 0, 0, 317, 393, 4035, 1555, 0, 0, 0, 0, 116, 354, 1996, 0, 0, 0, 0, 0, 0, 27, 3500, 1047, 394, 0, 0] rerouted_tokens_per_ep_rank=[4764, 4826, 1777, 3131, 4536, 6300, 2466, 4968] avg=4096.0000 max=6300 max_rank=5 min=1777 min_rank=2 imbalance_ratio=1.538086 max_load_reduction=26468 max_load_reduction_pct=0.807739 moved_to_echo=18270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 0, 53, 0, 0, 0, 0, 4011, 0, 0, 40, 19, 11, 0, 0, 13, 2978, 958, 872, 0, 0, 0, 0, 2140, 1, 0, 124, 0, 0, 0, 0, 0, 3952, 4, 3188, 672, 3, 0, 0, 6, 0, 0, 4, 1555, 0, 0, 0, 0, 2552, 0, 51, 0, 0, 0, 0, 414, 569, 8, 4077, 26, 394, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4081, 4821, 2265, 7819, 1565, 2603, 5488] avg=4096.0000 max=7819 max_rank=4 min=1565 min_rank=5 imbalance_ratio=1.908936 max_load_reduction=24949 max_load_reduction_pct=0.761383 moved_to_echo=17475 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16768, 4736, 2304, 1024, 5760, 2560, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 1024, 256, 22400, 5888, 2944, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 12800, 4224, 9984, 4864, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 21504, 2944, 8576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5376, 15488, 1024, 11136, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16000, 512, 10240, 6272, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 512, 20352, 11648, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16384, 768, 128, 15744, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 5, 2982, 0, 0, 0, 5, 2805, 6, 252, 0, 2376, 3870, 385, 1025, 0, 0, 0, 1321, 144, 4046, 25, 0, 3093, 12, 0, 4014, 0, 2, 2305, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1535, 24, 602, 1022, 2711, 87, 0, 35, 4079, 0, 4096, 3169, 449, 2, 2597, 2750, 61, 1, 4095, 5, 1123, 1141, 550, 24, 0, 27, 595, 397, 2, 1160, 427, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[5, 0, 4083, 0, 0, 721, 0, 3897, 100, 0, 11, 4027, 1426, 173, 288, 2, 125, 8, 1512, 7, 8, 0, 1265, 3267, 1, 806, 4065, 0, 0, 4000, 2970, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4064, 1, 4093, 0, 9, 1620, 0, 4091, 6, 3, 0, 1341, 1468, 0, 31, 128, 0, 11, 3846, 0, 2247, 499, 0, 1517, 2375, 72, 3, 0, 2, 4096, 983, 262] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[2651, 8, 4061, 2, 52, 1, 681, 3161, 0, 14, 1, 3554, 3578, 1249, 360, 359, 1189, 0, 355, 6, 26, 216, 0, 2544, 0, 2736, 0, 0, 46, 3204, 2520, 194] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4084, 1, 144, 1434, 1378, 0, 127, 3259, 0, 0, 860, 3363, 0, 22, 681, 417, 4052, 21, 316, 1, 0, 0, 496, 4087, 106, 2973, 923, 106, 25, 5, 3766, 121] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3039, 3, 4094, 0, 52, 0, 1684, 14, 1, 486, 2029, 4096, 667, 0, 4096, 28, 632, 496, 205, 0, 2925, 2135, 133, 3, 89, 1643, 152, 0, 73, 1, 3990, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1679, 2, 4019, 133, 424, 2222, 1227, 3783, 1949, 6, 4088, 1320, 1420, 0, 0, 0, 4075, 6, 4, 0, 0, 70, 0, 25, 1172, 273, 709, 55, 11, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 1620, 0, 0, 0, 0, 0, 0, 0, 5, 2635, 0, 0, 0, 0, 6, 252, 0, 1511, 0, 0, 0, 0, 3870, 385, 1025, 0, 949, 0, 0, 0, 0, 0, 1321, 144, 1272, 0, 0, 0, 4046, 25, 0, 3093, 170, 4, 0, 0, 12, 0, 4014, 0, 865, 90, 48, 0, 2, 2305, 3094, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1625, 2640, 1769, 6229, 2737, 7338, 5029, 5401] avg=4096.0000 max=7338 max_rank=5 min=1625 min_rank=0 imbalance_ratio=1.791504 max_load_reduction=25430 max_load_reduction_pct=0.776062 moved_to_echo=20505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4064, 1, 2215, 0, 0, 0, 0, 0, 9, 1620, 0, 3838, 0, 0, 0, 0, 6, 3, 0, 849, 0, 0, 0, 0, 1468, 0, 31, 128, 231, 0, 0, 0, 0, 11, 3846, 0, 1751, 0, 0, 0, 2247, 499, 0, 1517, 253, 4, 0, 0, 2375, 72, 3, 0, 492, 127, 15, 0, 2, 4096, 733, 262, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6280, 5467, 858, 1858, 5608, 4520, 3084, 5093] avg=4096.0000 max=6280 max_rank=0 min=858 min_rank=2 imbalance_ratio=1.533203 max_load_reduction=26488 max_load_reduction_pct=0.808350 moved_to_echo=18305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2651, 8, 2207, 2, 0, 0, 0, 0, 52, 1, 681, 2970, 0, 0, 0, 0, 0, 14, 1, 2260, 0, 0, 0, 0, 3578, 1249, 360, 359, 584, 0, 0, 0, 1189, 0, 355, 6, 1732, 0, 0, 0, 26, 216, 0, 2544, 191, 2, 0, 0, 0, 2736, 0, 0, 1294, 122, 29, 0, 46, 3204, 1905, 194, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4868, 3704, 2275, 6130, 3282, 2979, 4181, 5349] avg=4096.0000 max=6130 max_rank=3 min=2275 min_rank=2 imbalance_ratio=1.496582 max_load_reduction=26638 max_load_reduction_pct=0.812927 moved_to_echo=15791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4084, 1, 79, 1434, 0, 0, 0, 0, 1378, 0, 127, 3062, 0, 0, 0, 0, 0, 0, 860, 2139, 0, 0, 0, 0, 0, 22, 681, 417, 873, 0, 0, 0, 4052, 21, 316, 1, 61, 0, 0, 0, 0, 0, 496, 4087, 197, 4, 0, 0, 106, 2973, 923, 106, 1224, 4, 44, 0, 25, 5, 2845, 121, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5598, 4567, 2999, 1993, 4451, 4784, 5380, 2996] avg=4096.0000 max=5598 max_rank=0 min=1993 min_rank=3 imbalance_ratio=1.366699 max_load_reduction=27170 max_load_reduction_pct=0.829163 moved_to_echo=17611 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1535, 24, 328, 1022, 0, 0, 0, 0, 2711, 87, 0, 33, 0, 0, 0, 0, 4079, 0, 4096, 2016, 0, 0, 0, 0, 449, 2, 2597, 2750, 98, 0, 0, 0, 61, 1, 4095, 5, 256, 0, 0, 0, 1123, 1141, 550, 24, 2, 0, 0, 0, 0, 27, 595, 397, 1153, 18, 5, 0, 2, 1160, 324, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2909, 2831, 10191, 5896, 4418, 2840, 2195, 1488] avg=4096.0000 max=10191 max_rank=2 min=1488 min_rank=7 imbalance_ratio=2.488037 max_load_reduction=22577 max_load_reduction_pct=0.688995 moved_to_echo=10941 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3039, 3, 2225, 0, 0, 0, 0, 0, 52, 0, 1684, 14, 0, 0, 0, 0, 1, 486, 2029, 2605, 0, 0, 0, 0, 667, 0, 4096, 28, 925, 0, 0, 0, 632, 496, 205, 0, 1746, 0, 0, 0, 2925, 2135, 133, 3, 0, 4, 0, 0, 89, 1643, 152, 0, 1491, 123, 47, 0, 73, 1, 3014, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5267, 1750, 5121, 5716, 3079, 5200, 3545, 3090] avg=4096.0000 max=5716 max_rank=3 min=1750 min_rank=1 imbalance_ratio=1.395508 max_load_reduction=27052 max_load_reduction_pct=0.825562 moved_to_echo=14914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1679, 2, 2184, 133, 0, 0, 0, 0, 424, 2222, 1227, 3554, 0, 0, 0, 0, 1949, 6, 4088, 840, 0, 0, 0, 0, 1420, 0, 0, 0, 949, 0, 0, 0, 4075, 6, 4, 0, 1714, 0, 0, 0, 0, 70, 0, 25, 229, 4, 0, 0, 1172, 273, 709, 55, 480, 121, 48, 0, 11, 0, 3095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 7427, 6883, 2369, 5799, 328, 2858, 3106] avg=4096.0000 max=7427 max_rank=1 min=328 min_rank=5 imbalance_ratio=1.813232 max_load_reduction=25341 max_load_reduction_pct=0.773346 moved_to_echo=12091 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 2218, 0, 0, 0, 0, 0, 0, 721, 0, 3661, 0, 0, 0, 0, 100, 0, 11, 2561, 0, 0, 0, 0, 1426, 173, 288, 2, 688, 0, 0, 0, 125, 8, 1512, 7, 1742, 0, 0, 0, 8, 0, 1265, 3267, 236, 3, 0, 0, 1, 806, 4065, 0, 1466, 123, 35, 0, 0, 4000, 2244, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2223, 4382, 2672, 2577, 3394, 4779, 6496, 6245] avg=4096.0000 max=6496 max_rank=6 min=2223 min_rank=0 imbalance_ratio=1.585938 max_load_reduction=26272 max_load_reduction_pct=0.801758 moved_to_echo=20914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[6144, 768, 11136, 14848, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[17152, 128, 13184, 2688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12928, 1920, 9088, 3712, 5376, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10240, 640, 11776, 256, 10368, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 14848, 17280, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3840, 8576, 10496, 640, 8576, 768, 384, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4736, 4736, 3840, 19840, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10496, 4096, 2560, 14592, 1280, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 4092, 1242, 1541, 420, 2790, 4085, 37, 2589, 3635, 322, 0, 833, 0, 0, 34, 354, 427, 21, 0, 4092, 1921, 514, 37, 3781, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[466, 0, 2765, 0, 0, 0, 3418, 138, 0, 126, 2805, 3725, 20, 4056, 4020, 18, 3293, 1, 27, 0, 1504, 54, 809, 35, 0, 22, 0, 3979, 0, 1487, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[20, 5, 0, 115, 0, 2, 2922, 4059, 4096, 0, 3563, 4096, 0, 1299, 0, 21, 0, 10, 0, 0, 0, 0, 14, 423, 3675, 3971, 0, 4051, 2, 1, 0, 423] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[13, 0, 773, 10, 1, 0, 1802, 3338, 745, 0, 3827, 3918, 97, 3315, 3679, 3756, 1, 18, 3, 6, 328, 2, 0, 537, 1, 2, 1, 2448, 894, 1479, 1735, 39] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 14, 1031, 0, 3146, 678, 7, 1152, 9, 3317, 4074, 0, 4011, 3800, 0, 2, 0, 36, 0, 48, 403, 0, 0, 0, 2436, 1808, 3290, 0, 482, 0, 3024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[25, 1, 0, 0, 0, 554, 4090, 0, 5, 74, 1881, 4096, 3294, 2703, 1739, 101, 3891, 0, 0, 0, 0, 1426, 4, 10, 0, 456, 1, 4093, 534, 3752, 0, 38] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[44, 0, 4083, 1, 8, 0, 1627, 762, 0, 3905, 3, 3876, 6, 2667, 3994, 0, 0, 106, 0, 26, 0, 1698, 1386, 0, 212, 3670, 3986, 624, 68, 16, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 2844, 26, 349, 2919, 5, 0, 4087, 145, 4023, 4074, 2, 1629, 4081, 2, 4072, 14, 15, 0, 0, 575, 0, 91, 53, 0, 0, 3761, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 773, 10, 1647, 1148, 0, 0, 1, 0, 1571, 3338, 0, 0, 0, 0, 745, 0, 2837, 0, 0, 0, 0, 0, 97, 3315, 417, 3756, 0, 0, 0, 0, 1, 18, 3, 6, 1615, 1017, 0, 0, 328, 2, 0, 537, 2770, 0, 0, 0, 1, 2, 1, 696, 0, 0, 0, 0, 894, 1479, 1735, 39, 735, 990, 231, 0] rerouted_tokens_per_ep_rank=[3591, 4910, 3582, 7585, 2660, 3637, 700, 6103] avg=4096.0000 max=7585 max_rank=3 min=700 min_rank=6 imbalance_ratio=1.851807 max_load_reduction=25183 max_load_reduction_pct=0.768524 moved_to_echo=13100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[466, 0, 2765, 0, 1800, 1091, 0, 0, 0, 0, 2980, 138, 0, 0, 0, 0, 0, 126, 2080, 0, 0, 0, 0, 0, 20, 4056, 456, 18, 0, 0, 0, 0, 3293, 1, 27, 0, 1764, 1654, 0, 0, 1504, 54, 809, 35, 2634, 0, 0, 0, 0, 22, 0, 1130, 0, 0, 0, 0, 0, 1487, 0, 0, 1195, 725, 438, 0] rerouted_tokens_per_ep_rank=[6122, 3118, 2206, 4550, 6739, 5036, 1152, 3845] avg=4096.0000 max=6739 max_rank=4 min=1152 min_rank=6 imbalance_ratio=1.645264 max_load_reduction=26029 max_load_reduction_pct=0.794342 moved_to_echo=16772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1627, 1196, 0, 0, 1, 0, 3567, 1242, 0, 0, 0, 0, 1541, 420, 2069, 0, 0, 0, 0, 0, 37, 2589, 413, 322, 0, 0, 0, 0, 0, 833, 0, 0, 1595, 213, 0, 0, 34, 354, 427, 21, 2889, 0, 0, 0, 0, 4092, 1921, 147, 0, 0, 0, 0, 37, 3781, 0, 0, 154, 721, 525, 0] rerouted_tokens_per_ep_rank=[2823, 4810, 4030, 3361, 2641, 3725, 6160, 5218] avg=4096.0000 max=6160 max_rank=6 min=2641 min_rank=4 imbalance_ratio=1.503906 max_load_reduction=26608 max_load_reduction_pct=0.812012 moved_to_echo=17744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[44, 0, 4083, 1, 1788, 1135, 0, 0, 8, 0, 1419, 762, 0, 0, 0, 0, 0, 3905, 3, 0, 0, 0, 0, 0, 6, 2667, 453, 0, 0, 0, 0, 0, 0, 106, 0, 26, 1753, 259, 0, 0, 0, 1698, 1386, 0, 2741, 0, 0, 0, 212, 3670, 3986, 178, 0, 0, 0, 0, 68, 16, 0, 0, 187, 0, 208, 0] rerouted_tokens_per_ep_rank=[7051, 2189, 3908, 3126, 2144, 5825, 8046, 479] avg=4096.0000 max=8046 max_rank=6 min=479 min_rank=7 imbalance_ratio=1.964355 max_load_reduction=24722 max_load_reduction_pct=0.754456 moved_to_echo=16494 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 14, 1031, 1701, 1194, 0, 0, 0, 3146, 591, 7, 0, 0, 0, 0, 1152, 9, 2459, 0, 0, 0, 0, 0, 0, 4011, 431, 0, 0, 0, 0, 0, 2, 0, 36, 0, 1668, 1367, 0, 0, 48, 403, 0, 0, 2880, 0, 0, 0, 0, 2436, 1808, 935, 0, 0, 0, 0, 0, 482, 0, 3024, 988, 858, 87, 0] rerouted_tokens_per_ep_rank=[3940, 3744, 3620, 4442, 3073, 3331, 5179, 5439] avg=4096.0000 max=5439 max_rank=7 min=3073 min_rank=4 imbalance_ratio=1.327881 max_load_reduction=27329 max_load_reduction_pct=0.834015 moved_to_echo=17022 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2844, 26, 1827, 1193, 0, 0, 349, 2919, 5, 0, 0, 0, 0, 0, 4087, 145, 2983, 0, 0, 0, 0, 0, 2, 1629, 463, 2, 0, 0, 0, 0, 4072, 14, 15, 0, 1791, 1563, 0, 0, 0, 575, 0, 91, 2881, 0, 0, 0, 53, 0, 0, 1069, 0, 0, 0, 0, 1, 0, 0, 0, 1129, 1040, 0, 0] rerouted_tokens_per_ep_rank=[5890, 3273, 7215, 2096, 7455, 3547, 1122, 2170] avg=4096.0000 max=7455 max_rank=4 min=1122 min_rank=6 imbalance_ratio=1.820068 max_load_reduction=25313 max_load_reduction_pct=0.772491 moved_to_echo=14294 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[20, 5, 0, 115, 0, 1201, 0, 0, 0, 2, 2544, 4059, 0, 0, 0, 0, 4096, 0, 2637, 0, 0, 0, 0, 0, 0, 1299, 0, 21, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1688, 0, 0, 0, 0, 14, 423, 2895, 0, 0, 0, 3675, 3971, 0, 1143, 0, 0, 0, 0, 2, 1, 0, 423, 1220, 926, 378, 0] rerouted_tokens_per_ep_rank=[1341, 6605, 6733, 1320, 1698, 3332, 8789, 2950] avg=4096.0000 max=8789 max_rank=6 min=1320 min_rank=3 imbalance_ratio=2.145752 max_load_reduction=23979 max_load_reduction_pct=0.731781 moved_to_echo=16769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[25, 1, 0, 0, 783, 1201, 0, 0, 0, 554, 3565, 0, 0, 0, 0, 0, 5, 74, 1395, 0, 0, 0, 0, 0, 3294, 2703, 190, 101, 0, 0, 0, 0, 3891, 0, 0, 0, 766, 1701, 0, 0, 0, 1426, 4, 10, 2895, 0, 0, 0, 0, 456, 1, 1163, 0, 0, 0, 0, 534, 3752, 0, 38, 1229, 486, 525, 0] rerouted_tokens_per_ep_rank=[2010, 4119, 1474, 6288, 6358, 4335, 1620, 6564] avg=4096.0000 max=6564 max_rank=7 min=1474 min_rank=2 imbalance_ratio=1.602539 max_load_reduction=26204 max_load_reduction_pct=0.799683 moved_to_echo=18877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3968, 14720, 7808, 6528, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 128, 10496, 1280, 11264, 9472, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11264, 1024, 128, 128, 11008, 9472, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[11648, 4736, 16512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1536, 11008, 1792, 3584, 6912, 5760, 2432, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[3456, 22272, 2944, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[384, 6656, 16256, 9600, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1920, 4608, 2688, 1152, 22656, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2471, 3541, 1146, 3295, 4096, 0, 0, 0, 3938, 341, 126, 0, 133, 2870, 2, 75, 0, 0, 0, 9, 0, 4096, 3213, 230, 0, 2318, 8, 0, 0, 860] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[147, 0, 3910, 1669, 0, 0, 2550, 3667, 3339, 1702, 0, 2312, 8, 0, 4068, 0, 70, 3916, 2, 749, 15, 0, 0, 2314, 200, 0, 339, 788, 0, 114, 719, 170] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 76, 4019, 3700, 1, 2632, 2967, 28, 4093, 279, 0, 0, 722, 2890, 17, 1844, 0, 99, 1000, 0, 0, 84, 0, 4081, 139, 0, 4096, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3996, 4096, 4076, 1, 4026, 4071, 3790, 24, 0, 2546, 1368, 1, 511, 298, 45, 0, 59, 0, 225, 0, 3, 0, 381, 0, 1, 3140, 0, 52, 55, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[2381, 0, 0, 2994, 4096, 0, 2157, 4096, 4005, 84, 3, 794, 1, 0, 3981, 1, 4027, 0, 7, 0, 16, 0, 0, 1245, 19, 112, 520, 0, 2063, 121, 0, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4082, 2974, 0, 3611, 2757, 3880, 4054, 15, 0, 1268, 0, 3765, 0, 4, 3813, 0, 0, 0, 2, 0, 1818, 115, 20, 0, 0, 0, 0, 590, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 7, 3626, 4060, 0, 4005, 1134, 3952, 4040, 1, 115, 4096, 0, 0, 2766, 3062, 227, 0, 0, 60, 8, 26, 202, 751, 118, 0, 508, 0, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[2, 133, 0, 3904, 373, 0, 1494, 2616, 4048, 27, 2294, 4071, 0, 2, 0, 0, 0, 2965, 20, 2, 0, 22, 0, 4071, 1038, 0, 0, 3023, 0, 2447, 0, 216] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3996, 2882, 0, 0, 0, 0, 4076, 1, 0, 1730, 0, 0, 0, 0, 337, 24, 0, 2546, 0, 0, 0, 0, 1368, 1, 511, 298, 568, 0, 0, 0, 45, 0, 59, 0, 1729, 283, 0, 0, 225, 0, 3, 0, 3051, 0, 0, 0, 381, 0, 1, 3140, 646, 612, 0, 0, 0, 52, 55, 3, 4026, 119, 0, 0] rerouted_tokens_per_ep_rank=[6878, 5807, 2907, 2746, 2116, 3279, 4780, 4255] avg=4096.0000 max=6878 max_rank=0 min=2116 min_rank=4 imbalance_ratio=1.679199 max_load_reduction=25890 max_load_reduction_pct=0.790100 moved_to_echo=14430 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2471, 2483, 0, 0, 0, 0, 1146, 3295, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 341, 0, 0, 0, 0, 126, 0, 133, 2870, 495, 0, 0, 0, 2, 75, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 3213, 230, 0, 2318, 563, 0, 0, 0, 8, 0, 0, 860, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4954, 4441, 4279, 3624, 77, 4105, 6324, 4964] avg=4096.0000 max=6324 max_rank=6 min=77 min_rank=4 imbalance_ratio=1.543945 max_load_reduction=26444 max_load_reduction_pct=0.807007 moved_to_echo=15470 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 76, 2828, 0, 0, 0, 0, 3700, 1, 0, 1261, 0, 0, 0, 0, 4, 4093, 279, 0, 0, 0, 0, 0, 0, 722, 2890, 17, 557, 0, 0, 0, 1844, 0, 99, 1000, 1260, 2, 0, 0, 0, 0, 84, 0, 22, 0, 0, 0, 4081, 139, 0, 4096, 634, 446, 0, 0, 0, 0, 0, 1, 2632, 0, 0, 0] rerouted_tokens_per_ep_rank=[2904, 4962, 4376, 4186, 4205, 106, 9396, 2633] avg=4096.0000 max=9396 max_rank=6 min=106 min_rank=5 imbalance_ratio=2.293945 max_load_reduction=23372 max_load_reduction_pct=0.713257 moved_to_echo=16340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[147, 0, 3910, 1175, 0, 0, 0, 0, 0, 0, 0, 1557, 0, 0, 0, 0, 296, 1702, 0, 2312, 0, 0, 0, 0, 8, 0, 4068, 0, 231, 0, 0, 0, 70, 3916, 2, 749, 1558, 250, 0, 0, 15, 0, 0, 2314, 2688, 0, 0, 0, 200, 0, 339, 788, 263, 552, 0, 0, 0, 114, 719, 170, 2550, 105, 0, 0] rerouted_tokens_per_ep_rank=[5232, 1557, 4310, 4307, 6545, 5017, 2142, 3658] avg=4096.0000 max=6545 max_rank=4 min=1557 min_rank=1 imbalance_ratio=1.597900 max_load_reduction=26223 max_load_reduction_pct=0.800262 moved_to_echo=17362 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 133, 0, 2747, 0, 0, 0, 0, 373, 0, 0, 1112, 0, 0, 0, 0, 358, 27, 2294, 4071, 0, 0, 0, 0, 0, 2, 0, 0, 541, 0, 0, 0, 0, 2965, 20, 2, 1111, 303, 0, 0, 0, 22, 0, 4071, 3259, 0, 0, 0, 1038, 0, 0, 3023, 616, 393, 0, 0, 0, 2447, 0, 216, 1494, 128, 0, 0] rerouted_tokens_per_ep_rank=[2882, 1485, 6750, 543, 4401, 7352, 5070, 4285] avg=4096.0000 max=7352 max_rank=5 min=543 min_rank=3 imbalance_ratio=1.794922 max_load_reduction=25416 max_load_reduction_pct=0.775635 moved_to_echo=21108 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2381, 0, 0, 2107, 0, 0, 0, 0, 4096, 0, 0, 1740, 0, 0, 0, 0, 355, 84, 3, 794, 0, 0, 0, 0, 1, 0, 3981, 1, 415, 0, 0, 0, 4027, 0, 7, 0, 1740, 300, 0, 0, 16, 0, 0, 1245, 3224, 0, 0, 0, 19, 112, 520, 0, 472, 616, 0, 0, 2063, 121, 0, 45, 2157, 126, 0, 0] rerouted_tokens_per_ep_rank=[4488, 5836, 1236, 4398, 6074, 4485, 1739, 4512] avg=4096.0000 max=6074 max_rank=4 min=1236 min_rank=2 imbalance_ratio=1.482910 max_load_reduction=26694 max_load_reduction_pct=0.814636 moved_to_echo=16810 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2872, 0, 0, 0, 0, 2974, 0, 0, 1163, 0, 0, 0, 0, 335, 4054, 15, 0, 0, 0, 0, 0, 1268, 0, 3765, 0, 566, 0, 0, 0, 4, 3813, 0, 0, 1175, 292, 0, 0, 0, 2, 0, 1818, 3127, 0, 0, 0, 115, 20, 0, 0, 644, 419, 0, 0, 0, 0, 590, 0, 3611, 126, 0, 0] rerouted_tokens_per_ep_rank=[2872, 4137, 4404, 5599, 5284, 4947, 1198, 4327] avg=4096.0000 max=5599 max_rank=3 min=1198 min_rank=6 imbalance_ratio=1.366943 max_load_reduction=27169 max_load_reduction_pct=0.829132 moved_to_echo=15756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 2551, 0, 0, 0, 0, 4060, 0, 0, 483, 0, 0, 0, 0, 350, 4040, 1, 115, 0, 0, 0, 0, 4096, 0, 0, 2766, 503, 0, 0, 0, 3062, 227, 0, 0, 481, 296, 0, 0, 60, 8, 26, 202, 3181, 0, 0, 0, 751, 118, 0, 508, 572, 170, 0, 0, 0, 0, 0, 4, 4005, 125, 0, 0] rerouted_tokens_per_ep_rank=[2558, 4543, 4506, 7365, 4066, 3477, 2119, 4134] avg=4096.0000 max=7365 max_rank=3 min=2119 min_rank=6 imbalance_ratio=1.798096 max_load_reduction=25403 max_load_reduction_pct=0.775238 moved_to_echo=13796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9856, 640, 896, 13952, 4480, 3328, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[20480, 3328, 0, 9088, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9088, 11008, 256, 1792, 9088, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6912, 768, 15360, 6016, 3968, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2560, 256, 10496, 19712, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 128, 128, 13824, 18560, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2048, 14080, 6656, 10240, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2176, 2816, 1408, 1408, 24576, 768, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[10, 0, 0, 4068, 12, 3598, 4026, 0, 3918, 4095, 165, 0, 3, 0, 7, 0, 0, 5, 0, 250, 1, 70, 4095, 96, 4083, 4, 625, 9, 0, 0, 3601, 27] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 1200, 4096, 17, 3708, 4091, 9, 722, 956, 0, 4083, 2238, 38, 0, 1, 0, 0, 54, 4093, 3709, 3613, 137, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1, 4, 0, 3699, 4096, 0, 1, 0, 3586, 9, 215, 0, 4096, 232, 0, 36, 9, 799, 0, 258, 3552, 0, 4093, 0, 3994, 0, 4084, 3, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 183, 11, 880, 0, 0, 3393, 431, 3969, 0, 0, 0, 155, 4, 4094, 75, 4050, 0, 3524, 4, 1504, 5, 4085, 0, 3772, 3, 0, 139, 2440, 0, 47, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[2337, 0, 1713, 7, 0, 63, 4094, 132, 4086, 166, 824, 0, 2015, 0, 0, 6, 630, 1982, 3797, 8, 245, 3, 4057, 0, 2358, 4019, 191, 0, 0, 0, 0, 35] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[16, 0, 55, 35, 0, 7, 3025, 67, 2340, 1521, 321, 0, 9, 2, 4096, 0, 3784, 1545, 39, 64, 3920, 0, 4094, 0, 3297, 404, 0, 0, 0, 4, 4067, 56] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[336, 0, 1, 0, 0, 76, 2212, 5, 56, 3346, 3, 0, 4019, 0, 3791, 0, 69, 0, 537, 16, 0, 3731, 4052, 0, 2692, 1, 3710, 0, 0, 0, 4090, 25] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1, 13, 0, 3973, 1733, 16, 1744, 93, 0, 4085, 0, 0, 3957, 42, 4093, 2, 4070, 89, 6, 3, 11, 9, 4093, 6, 2188, 201, 0, 0, 2308, 0, 0, 32] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2337, 0, 1713, 7, 1600, 565, 98, 0, 0, 63, 3843, 132, 0, 0, 0, 0, 2486, 166, 824, 0, 0, 0, 0, 0, 2015, 0, 0, 6, 0, 0, 0, 0, 630, 1982, 3797, 8, 0, 251, 57, 0, 245, 3, 1965, 0, 0, 0, 0, 0, 1736, 4019, 191, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1994, 0, 0, 0] rerouted_tokens_per_ep_rank=[6320, 4038, 3476, 2021, 6725, 2213, 5946, 2029] avg=4096.0000 max=6725 max_rank=4 min=2021 min_rank=3 imbalance_ratio=1.641846 max_load_reduction=26043 max_load_reduction_pct=0.794769 moved_to_echo=16913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 1449, 858, 97, 0, 0, 1200, 3848, 17, 0, 0, 0, 0, 2259, 4091, 9, 722, 0, 0, 0, 0, 956, 0, 3022, 2238, 0, 0, 0, 0, 38, 0, 1, 0, 1061, 248, 80, 0, 0, 54, 1987, 3709, 0, 0, 0, 0, 2675, 137, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2009, 0, 0, 0] rerouted_tokens_per_ep_rank=[2406, 5065, 7081, 6216, 1428, 5750, 2812, 2010] avg=4096.0000 max=7081 max_rank=2 min=1428 min_rank=4 imbalance_ratio=1.728760 max_load_reduction=25687 max_load_reduction_pct=0.783905 moved_to_echo=12000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 0, 3699, 1401, 949, 97, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 2185, 9, 215, 0, 0, 0, 0, 0, 4096, 232, 0, 36, 0, 0, 0, 0, 9, 799, 0, 258, 0, 0, 88, 0, 3552, 0, 1987, 0, 0, 0, 0, 0, 2957, 0, 4084, 3, 0, 0, 0, 0, 0, 1, 0, 0, 2009, 0, 0, 0] rerouted_tokens_per_ep_rank=[6151, 4097, 2409, 4364, 1154, 5539, 7044, 2010] avg=4096.0000 max=7044 max_rank=6 min=1154 min_rank=4 imbalance_ratio=1.719727 max_load_reduction=25724 max_load_reduction_pct=0.785034 moved_to_echo=15747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 55, 35, 914, 783, 97, 0, 0, 7, 2842, 67, 0, 0, 0, 0, 1426, 1521, 321, 0, 0, 0, 0, 0, 9, 2, 3031, 0, 0, 0, 0, 0, 3784, 1545, 39, 64, 1065, 183, 73, 0, 3920, 0, 1988, 0, 0, 0, 0, 0, 2441, 404, 0, 0, 0, 0, 0, 0, 0, 4, 4067, 56, 2009, 0, 0, 0] rerouted_tokens_per_ep_rank=[1900, 2916, 3268, 3042, 6753, 5908, 2845, 6136] avg=4096.0000 max=6753 max_rank=4 min=1900 min_rank=0 imbalance_ratio=1.648682 max_load_reduction=26015 max_load_reduction_pct=0.793915 moved_to_echo=21642 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 13, 0, 3973, 0, 520, 97, 0, 1733, 16, 1639, 93, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 3957, 42, 3029, 2, 0, 0, 0, 0, 4070, 89, 6, 3, 1064, 105, 48, 0, 11, 9, 1987, 6, 0, 0, 0, 0, 1620, 201, 0, 0, 0, 0, 0, 0, 2308, 0, 0, 32, 2009, 0, 0, 0] rerouted_tokens_per_ep_rank=[4604, 3481, 4085, 7030, 5385, 2013, 1821, 4349] avg=4096.0000 max=7030 max_rank=3 min=1821 min_rank=6 imbalance_ratio=1.716309 max_load_reduction=25738 max_load_reduction_pct=0.785461 moved_to_echo=13568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 183, 11, 880, 1551, 896, 97, 0, 0, 0, 3188, 431, 0, 0, 0, 0, 2418, 0, 0, 0, 0, 0, 0, 0, 155, 4, 3030, 75, 0, 0, 0, 0, 4050, 0, 3524, 4, 1064, 205, 84, 0, 1504, 5, 1983, 0, 0, 0, 0, 0, 2792, 3, 0, 139, 0, 0, 0, 0, 2440, 0, 47, 0, 2005, 0, 0, 0] rerouted_tokens_per_ep_rank=[3618, 3619, 2418, 3264, 8931, 3492, 2934, 4492] avg=4096.0000 max=8931 max_rank=4 min=2418 min_rank=2 imbalance_ratio=2.180420 max_load_reduction=23837 max_load_reduction_pct=0.727448 moved_to_echo=19849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[336, 0, 1, 0, 21, 639, 96, 0, 0, 76, 2078, 5, 0, 0, 0, 0, 35, 3346, 3, 0, 0, 0, 0, 0, 4019, 0, 2803, 0, 0, 0, 0, 0, 69, 0, 537, 16, 988, 134, 59, 0, 0, 3731, 1967, 0, 0, 0, 0, 0, 1994, 1, 3710, 0, 0, 0, 0, 0, 0, 0, 4090, 25, 1989, 0, 0, 0] rerouted_tokens_per_ep_rank=[1093, 2159, 3384, 6822, 1803, 5698, 5705, 6104] avg=4096.0000 max=6822 max_rank=3 min=1093 min_rank=0 imbalance_ratio=1.665527 max_load_reduction=25946 max_load_reduction_pct=0.791809 moved_to_echo=19310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 4068, 1531, 970, 97, 0, 12, 3598, 3783, 0, 0, 0, 0, 0, 2387, 4095, 165, 0, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 0, 5, 0, 250, 1, 243, 90, 0, 1, 70, 1988, 96, 0, 0, 0, 0, 3023, 4, 625, 9, 0, 0, 0, 0, 0, 0, 3601, 27, 2010, 0, 0, 0] rerouted_tokens_per_ep_rank=[6676, 7393, 6647, 9, 589, 2155, 3661, 5638] avg=4096.0000 max=7393 max_rank=1 min=9 min_rank=3 imbalance_ratio=1.804932 max_load_reduction=25375 max_load_reduction_pct=0.774384 moved_to_echo=12043 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12672, 4480, 7936, 640, 5248, 1408, 640, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2816, 256, 1792, 12672, 8576, 6272, 896, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15232, 384, 14976, 2432, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9344, 3968, 15872, 3840, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4864, 128, 11904, 256, 16128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19328, 4864, 8704, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[5888, 4992, 21248, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13312, 17408, 1664, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4007, 59, 0, 0, 3803, 0, 0, 18, 2109, 1819, 0, 4084, 3750, 0, 19, 405, 0, 28, 0, 3550, 275, 0, 13, 0, 153, 405, 73, 4095, 0, 7, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4, 15, 4055, 14, 0, 3409, 12, 3494, 9, 295, 3825, 0, 52, 2529, 1701, 6, 0, 4, 109, 388, 8, 57, 675, 3893, 0, 125, 420, 3566, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[385, 1, 3, 0, 3650, 8, 0, 1360, 2694, 4095, 3, 3410, 4090, 90, 26, 0, 0, 383, 0, 365, 28, 2630, 0, 47, 0, 3999, 0, 0, 1427, 3699, 0, 375] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4095, 9, 1, 2, 3956, 54, 0, 0, 0, 68, 0, 4090, 2986, 0, 172, 0, 4094, 1068, 0, 3301, 26, 1, 0, 1, 0, 1889, 0, 4096, 2135, 0, 179, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[100, 0, 3968, 21, 4003, 1, 0, 0, 0, 1095, 101, 4036, 4072, 10, 0, 7, 2, 0, 190, 2979, 733, 0, 2504, 2, 1, 3663, 0, 1182, 6, 4092, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 40, 2687, 4095, 201, 0, 346, 67, 4092, 0, 4039, 4093, 3508, 1, 2, 260, 33, 0, 0, 0, 0, 0, 1286, 4, 3628, 165, 4, 11, 4096, 109, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 419, 0, 3723, 0, 0, 0, 373, 3267, 0, 782, 4093, 0, 6, 4096, 0, 57, 0, 4, 0, 0, 1, 3711, 3, 1, 3, 3607, 4083, 0, 443, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[6, 0, 22, 0, 4089, 140, 31, 89, 0, 8, 0, 3080, 4029, 19, 0, 506, 0, 3114, 24, 286, 4040, 0, 0, 1, 400, 2788, 587, 947, 1084, 3943, 13, 3522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[100, 0, 3968, 21, 545, 653, 176, 0, 3458, 1, 0, 0, 0, 0, 0, 0, 0, 1095, 101, 1604, 0, 0, 0, 0, 2807, 10, 0, 7, 0, 0, 0, 0, 2, 0, 190, 2979, 929, 897, 0, 0, 733, 0, 2504, 2, 2432, 336, 0, 0, 1, 3010, 0, 1182, 0, 0, 0, 0, 6, 3019, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5463, 3459, 2800, 2824, 4997, 6007, 4193, 3025] avg=4096.0000 max=6007 max_rank=5 min=2800 min_rank=2 imbalance_ratio=1.466553 max_load_reduction=26761 max_load_reduction_pct=0.816681 moved_to_echo=18222 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4007, 59, 0, 0, 518, 72, 0, 0, 3285, 0, 0, 18, 0, 0, 0, 0, 2109, 1819, 0, 1623, 0, 0, 0, 0, 2585, 0, 19, 405, 0, 0, 0, 0, 0, 28, 0, 3550, 856, 1, 0, 0, 275, 0, 13, 0, 2461, 309, 0, 0, 153, 333, 73, 4095, 0, 0, 0, 0, 0, 6, 1, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4656, 3303, 5551, 3009, 4435, 3058, 4654, 4102] avg=4096.0000 max=5551 max_rank=2 min=3009 min_rank=3 imbalance_ratio=1.355225 max_load_reduction=27217 max_load_reduction_pct=0.830597 moved_to_echo=16249 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[385, 1, 3, 0, 497, 713, 159, 0, 3153, 8, 0, 1360, 0, 0, 0, 0, 2694, 4095, 3, 1355, 0, 0, 0, 0, 2820, 90, 26, 0, 0, 0, 0, 0, 0, 383, 0, 365, 933, 810, 0, 0, 28, 2630, 0, 47, 2055, 337, 0, 0, 0, 3286, 0, 0, 0, 0, 0, 0, 1427, 2730, 0, 375, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1758, 4521, 8147, 2936, 2491, 5097, 3286, 4532] avg=4096.0000 max=8147 max_rank=2 min=1758 min_rank=0 imbalance_ratio=1.989014 max_load_reduction=24621 max_load_reduction_pct=0.751373 moved_to_echo=15406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4, 15, 552, 698, 156, 0, 3503, 14, 0, 3409, 0, 0, 0, 0, 12, 3494, 9, 118, 0, 0, 0, 0, 2637, 0, 52, 2529, 0, 0, 0, 0, 1701, 6, 0, 4, 873, 784, 0, 0, 109, 388, 8, 57, 177, 315, 0, 0, 675, 3195, 0, 125, 0, 0, 0, 0, 420, 2626, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5521, 6926, 3633, 5218, 3368, 1054, 3995, 3053] avg=4096.0000 max=6926 max_rank=1 min=1054 min_rank=5 imbalance_ratio=1.690918 max_load_reduction=25842 max_load_reduction_pct=0.788635 moved_to_echo=11470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 9, 1, 2, 539, 337, 0, 0, 3417, 54, 0, 0, 0, 0, 0, 0, 0, 68, 0, 1625, 0, 0, 0, 0, 2059, 0, 172, 0, 0, 0, 0, 0, 4094, 1068, 0, 3301, 681, 0, 0, 0, 26, 1, 0, 1, 2465, 246, 0, 0, 0, 1552, 0, 4096, 0, 0, 0, 0, 2135, 0, 179, 545, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4983, 3471, 1693, 2231, 9144, 2739, 5648, 2859] avg=4096.0000 max=9144 max_rank=4 min=1693 min_rank=2 imbalance_ratio=2.232422 max_load_reduction=23624 max_load_reduction_pct=0.720947 moved_to_echo=20390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 22, 0, 557, 497, 169, 0, 3532, 140, 31, 89, 0, 0, 0, 0, 0, 8, 0, 1224, 0, 0, 0, 0, 2777, 19, 0, 506, 0, 0, 0, 0, 0, 3114, 24, 286, 920, 864, 0, 0, 4040, 0, 0, 1, 1856, 332, 0, 0, 400, 2291, 587, 947, 0, 0, 0, 0, 1084, 2910, 13, 3522, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1251, 3792, 1232, 3302, 5208, 6229, 4225, 7529] avg=4096.0000 max=7529 max_rank=7 min=1232 min_rank=2 imbalance_ratio=1.838135 max_load_reduction=25239 max_load_reduction_pct=0.770233 moved_to_echo=23191 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 419, 0, 511, 1, 0, 0, 3212, 0, 0, 0, 0, 0, 0, 0, 373, 3267, 0, 308, 0, 0, 0, 0, 2813, 0, 6, 4096, 0, 0, 0, 0, 0, 57, 0, 4, 939, 0, 0, 0, 0, 0, 1, 3711, 474, 341, 0, 0, 3, 0, 3, 3607, 0, 0, 0, 0, 4083, 0, 443, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5027, 3212, 3948, 6915, 1000, 4527, 3613, 4526] avg=4096.0000 max=6915 max_rank=3 min=1000 min_rank=4 imbalance_ratio=1.688232 max_load_reduction=25853 max_load_reduction_pct=0.788971 moved_to_echo=13666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 40, 2687, 558, 647, 176, 0, 3537, 201, 0, 346, 0, 0, 0, 0, 67, 4092, 0, 1605, 0, 0, 0, 0, 2822, 3508, 1, 2, 0, 0, 0, 0, 260, 33, 0, 0, 934, 898, 0, 0, 0, 0, 0, 1286, 2434, 337, 0, 0, 4, 2981, 165, 4, 0, 0, 0, 0, 11, 3022, 109, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4084, 5764, 6333, 2125, 4057, 3154, 3142] avg=4096.0000 max=6333 max_rank=3 min=2125 min_rank=4 imbalance_ratio=1.546143 max_load_reduction=26435 max_load_reduction_pct=0.806732 moved_to_echo=12478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6144, 4736, 256, 10496, 7168, 4352, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21376, 3712, 384, 7552, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[16896, 128, 4480, 2816, 4352, 3712, 896, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5248, 3072, 2560, 5120, 14464, 2560, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5376, 18048, 128, 9472, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[9216, 14336, 768, 8576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[27136, 512, 128, 5248, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 16768, 896, 14080, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 14, 0, 3449, 0, 0, 4096, 97, 9, 4091, 3830, 306, 0, 367, 2, 4030, 465, 0, 0, 0, 4085, 627, 4066, 0, 9, 1, 0, 0, 3223, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 38, 3583, 0, 0, 855, 0, 8, 4095, 59, 0, 0, 0, 0, 36, 37, 0, 17, 165, 4095, 3967, 4095, 0, 4026, 7, 0, 3549, 40, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[12, 0, 33, 0, 0, 95, 8, 83, 3935, 253, 0, 3845, 4059, 0, 11, 1, 10, 3170, 0, 112, 231, 0, 4073, 442, 2915, 1370, 22, 4093, 0, 0, 0, 3995] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3798, 82, 332, 0, 1, 11, 0, 1, 4095, 127, 0, 4021, 36, 0, 1, 0, 0, 3004, 30, 0, 16, 0, 0, 4090, 3053, 1624, 738, 5, 61, 18, 3538, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 16, 14, 12, 1, 2972, 0, 0, 4066, 0, 261, 4094, 3916, 0, 118, 0, 1, 3931, 95, 0, 0, 0, 0, 0, 4095, 0, 92, 0, 1030, 0, 4057, 3997] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[69, 0, 78, 0, 4096, 4072, 0, 6, 3960, 10, 0, 960, 1714, 21, 0, 0, 349, 1693, 178, 0, 12, 1, 3640, 54, 4096, 2079, 291, 4, 0, 0, 1296, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[271, 2443, 1, 229, 39, 1, 0, 0, 3620, 0, 0, 2720, 4096, 0, 0, 0, 3, 4063, 0, 0, 0, 1989, 3973, 9, 3697, 0, 514, 3, 44, 0, 963, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[109, 856, 1, 0, 3, 4045, 0, 0, 287, 39, 55, 1244, 3750, 0, 0, 0, 0, 0, 16, 0, 976, 4066, 4089, 681, 3359, 0, 1496, 39, 0, 34, 3529, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 14, 12, 2951, 310, 0, 0, 1, 2972, 0, 0, 0, 284, 0, 0, 4066, 0, 261, 1143, 0, 0, 0, 0, 3916, 0, 118, 0, 1720, 0, 0, 0, 1, 3931, 95, 0, 1504, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2281, 0, 92, 0, 0, 0, 0, 0, 1030, 0, 4057, 1871, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3303, 3257, 5470, 5754, 5653, 0, 2373, 6958] avg=4096.0000 max=6958 max_rank=7 min=0 min_rank=5 imbalance_ratio=1.698730 max_load_reduction=25810 max_load_reduction_pct=0.787659 moved_to_echo=14984 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[271, 2443, 1, 229, 1961, 280, 0, 0, 39, 1, 0, 0, 1414, 291, 0, 0, 3620, 0, 0, 759, 0, 0, 0, 0, 4096, 0, 0, 0, 1760, 0, 0, 0, 3, 4063, 0, 0, 1358, 125, 0, 0, 0, 1989, 2559, 9, 0, 0, 0, 0, 2059, 0, 514, 3, 0, 0, 0, 0, 44, 0, 963, 1914, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5185, 1745, 4379, 5856, 5549, 4557, 2576, 2921] avg=4096.0000 max=5856 max_rank=3 min=1745 min_rank=1 imbalance_ratio=1.429688 max_load_reduction=26912 max_load_reduction_pct=0.821289 moved_to_echo=15603 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 2955, 313, 0, 0, 38, 3583, 0, 0, 1460, 0, 0, 0, 855, 0, 8, 1140, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 36, 37, 0, 1508, 0, 0, 0, 17, 165, 2635, 3967, 0, 0, 0, 0, 2274, 0, 4026, 7, 0, 0, 0, 0, 0, 3549, 40, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7364, 5081, 2003, 59, 1581, 6784, 6307, 3589] avg=4096.0000 max=7364 max_rank=0 min=59 min_rank=3 imbalance_ratio=1.797852 max_load_reduction=25404 max_load_reduction_pct=0.775269 moved_to_echo=18261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3798, 82, 332, 0, 2899, 231, 0, 0, 1, 11, 0, 1, 0, 290, 0, 0, 4095, 127, 0, 1122, 0, 0, 0, 0, 36, 0, 1, 0, 1759, 0, 0, 0, 0, 3004, 30, 0, 1121, 125, 0, 0, 16, 0, 0, 4090, 0, 0, 0, 0, 1701, 1624, 738, 5, 0, 0, 0, 0, 61, 18, 3538, 1912, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7342, 303, 5344, 1796, 4280, 4106, 4068, 5529] avg=4096.0000 max=7342 max_rank=0 min=303 min_rank=1 imbalance_ratio=1.792480 max_load_reduction=25426 max_load_reduction_pct=0.775940 moved_to_echo=17983 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 33, 0, 2772, 220, 0, 0, 0, 95, 8, 83, 1450, 284, 0, 0, 3935, 253, 0, 1073, 0, 0, 0, 0, 4059, 0, 11, 1, 1719, 0, 0, 0, 10, 3170, 0, 112, 1071, 122, 0, 0, 231, 0, 2623, 442, 0, 0, 0, 0, 1624, 1370, 22, 4093, 0, 0, 0, 0, 0, 0, 0, 1870, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3037, 1920, 5261, 5790, 4485, 3296, 7109, 1870] avg=4096.0000 max=7109 max_rank=6 min=1870 min_rank=7 imbalance_ratio=1.735596 max_load_reduction=25659 max_load_reduction_pct=0.783051 moved_to_echo=16760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 14, 2949, 308, 0, 0, 0, 3449, 0, 0, 1454, 0, 0, 0, 4096, 97, 9, 1142, 0, 0, 0, 0, 3830, 306, 0, 367, 0, 0, 0, 0, 2, 4030, 465, 0, 1494, 0, 0, 0, 0, 0, 2631, 627, 0, 0, 0, 0, 2264, 0, 9, 1, 0, 0, 0, 0, 0, 0, 3223, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3272, 4903, 5344, 4503, 5991, 3258, 2274, 3223] avg=4096.0000 max=5991 max_rank=4 min=2274 min_rank=6 imbalance_ratio=1.462646 max_load_reduction=26777 max_load_reduction_pct=0.817169 moved_to_echo=14746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[69, 0, 78, 0, 692, 310, 0, 0, 4096, 4072, 0, 6, 1296, 291, 0, 0, 3960, 10, 0, 268, 0, 0, 0, 0, 1714, 21, 0, 0, 1760, 0, 0, 0, 349, 1693, 178, 0, 1505, 125, 0, 0, 12, 1, 2344, 54, 0, 0, 0, 0, 2281, 2079, 291, 4, 0, 0, 0, 0, 0, 0, 1296, 1913, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1149, 9761, 4238, 3495, 3850, 2411, 4655, 3209] avg=4096.0000 max=9761 max_rank=1 min=1149 min_rank=0 imbalance_ratio=2.383057 max_load_reduction=23007 max_load_reduction_pct=0.702118 moved_to_echo=14125 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[109, 856, 1, 0, 896, 254, 0, 0, 3, 4045, 0, 0, 1456, 294, 0, 0, 287, 39, 55, 348, 0, 0, 0, 0, 3750, 0, 0, 0, 1765, 0, 0, 0, 0, 0, 16, 0, 1234, 129, 0, 0, 976, 4066, 2633, 681, 0, 0, 0, 0, 1871, 0, 1496, 39, 0, 0, 0, 0, 0, 34, 3529, 1906, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2116, 5798, 729, 5515, 1379, 8356, 3406, 5469] avg=4096.0000 max=8356 max_rank=5 min=729 min_rank=2 imbalance_ratio=2.040039 max_load_reduction=24412 max_load_reduction_pct=0.744995 moved_to_echo=18610 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 3456, 512, 256, 18176, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 18304, 128, 128, 8576, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 3712, 16768, 11392, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 19968, 896, 128, 10880, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21504, 384, 256, 384, 10496, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 6272, 15488, 9984, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[24960, 640, 384, 7040, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16384, 5120, 7296, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 751, 1, 0, 0, 124, 0, 2581, 3899, 3283, 2, 203, 3420, 2930, 58, 0, 7, 342, 142, 3739, 179, 3522, 0, 1285, 0, 2, 0, 18, 1460, 4096, 724] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 3280, 87, 0, 0, 11, 0, 19, 4096, 4077, 0, 0, 0, 4, 0, 9, 1682, 773, 4087, 170, 45, 0, 2967, 740, 2864, 14, 0, 11, 1280, 1998, 4095, 459] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 165, 4046, 1723, 0, 0, 2238, 0, 6, 4096, 470, 3527, 274, 2596, 19, 3811, 2568, 1, 0, 0, 24, 0, 1160, 4095, 1949] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 0, 8, 0, 0, 0, 3913, 3858, 338, 592, 24, 0, 0, 66, 4096, 1967, 0, 0, 269, 4020, 0, 3686, 209, 46, 3514, 158, 10, 1890, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 55, 26, 0, 1193, 0, 4090, 4084, 2529, 0, 4, 1, 316, 0, 44, 0, 883, 23, 135, 3744, 4096, 0, 1261, 1841, 0, 327, 0, 9, 4095, 4011] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[169, 2, 531, 18, 69, 0, 1937, 0, 4054, 3732, 4086, 0, 0, 43, 65, 0, 1993, 4, 3839, 0, 3562, 100, 4080, 0, 0, 0, 6, 362, 0, 0, 77, 4039] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 2699, 0, 423, 0, 0, 0, 1, 0, 0, 4096, 0, 88, 0, 4092, 3, 0, 0, 3971, 3484, 1072, 854, 3776, 2, 0, 15, 0, 0, 15, 0, 4084, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[177, 553, 2, 0, 0, 45, 0, 6, 0, 4088, 3900, 0, 6, 0, 1, 961, 38, 3549, 10, 0, 0, 573, 4045, 1660, 3032, 2174, 31, 0, 13, 0, 4066, 3838] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3337, 311, 0, 0, 0, 0, 0, 0, 4046, 233, 0, 0, 165, 0, 1277, 0, 0, 0, 0, 0, 0, 2238, 0, 6, 2135, 142, 0, 0, 4096, 470, 2239, 274, 0, 0, 0, 0, 2596, 19, 1365, 2568, 0, 0, 0, 0, 1, 0, 0, 24, 446, 1146, 0, 0, 0, 1160, 525, 1949, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3648, 4279, 1442, 4521, 7079, 6548, 1617, 3634] avg=4096.0000 max=7079 max_rank=4 min=1442 min_rank=2 imbalance_ratio=1.728271 max_load_reduction=25689 max_load_reduction_pct=0.783966 moved_to_echo=18878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 55, 3333, 329, 0, 0, 26, 0, 1193, 0, 4084, 230, 0, 0, 4090, 0, 1880, 0, 0, 0, 0, 0, 4, 1, 316, 0, 2292, 35, 0, 0, 44, 0, 563, 23, 0, 0, 0, 0, 135, 3744, 1475, 0, 0, 0, 0, 0, 1261, 1841, 0, 327, 649, 285, 0, 0, 0, 9, 532, 4011, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3718, 5533, 5970, 2648, 630, 5354, 4363, 4552] avg=4096.0000 max=5970 max_rank=2 min=630 min_rank=4 imbalance_ratio=1.457520 max_load_reduction=26798 max_load_reduction_pct=0.817810 moved_to_echo=14899 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 751, 1, 3334, 283, 0, 0, 0, 0, 124, 0, 3899, 230, 0, 0, 2581, 0, 2440, 2, 0, 0, 0, 0, 203, 3420, 2930, 58, 1970, 13, 0, 0, 0, 7, 219, 142, 0, 0, 0, 0, 3739, 179, 1269, 0, 0, 0, 0, 0, 1285, 0, 2, 0, 843, 110, 0, 0, 18, 1460, 532, 724, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4369, 4253, 5023, 8594, 368, 5187, 2240, 2734] avg=4096.0000 max=8594 max_rank=3 min=368 min_rank=4 imbalance_ratio=2.098145 max_load_reduction=24174 max_load_reduction_pct=0.737732 moved_to_echo=10529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[169, 2, 531, 18, 62, 328, 0, 0, 69, 0, 1937, 0, 3732, 4, 0, 0, 4054, 0, 3036, 0, 0, 0, 0, 0, 0, 43, 65, 0, 2283, 152, 0, 0, 1993, 4, 2445, 0, 0, 0, 0, 0, 3562, 100, 1469, 0, 0, 0, 0, 0, 0, 0, 6, 362, 1050, 1242, 0, 0, 0, 0, 11, 4039, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1110, 5742, 7090, 2543, 4442, 5131, 2660, 4050] avg=4096.0000 max=7090 max_rank=2 min=1110 min_rank=0 imbalance_ratio=1.730957 max_load_reduction=25678 max_load_reduction_pct=0.783630 moved_to_echo=16283 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[177, 553, 2, 0, 3309, 325, 0, 0, 0, 45, 0, 6, 4088, 228, 0, 0, 0, 0, 2898, 0, 0, 0, 0, 0, 6, 0, 1, 961, 2263, 0, 0, 0, 38, 3549, 7, 0, 0, 0, 0, 0, 0, 573, 1457, 1660, 0, 0, 0, 0, 3032, 2174, 31, 0, 1002, 3, 0, 0, 13, 0, 529, 3838, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4366, 4367, 2898, 3231, 3594, 3690, 6242, 4380] avg=4096.0000 max=6242 max_rank=6 min=2898 min_rank=2 imbalance_ratio=1.523926 max_load_reduction=26526 max_load_reduction_pct=0.809509 moved_to_echo=17906 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 1538, 323, 0, 0, 0, 8, 0, 0, 3913, 106, 0, 0, 0, 0, 2867, 338, 0, 0, 0, 0, 592, 24, 0, 0, 2249, 78, 0, 0, 66, 4096, 1253, 0, 0, 0, 0, 0, 0, 269, 1448, 0, 0, 0, 0, 0, 3686, 209, 46, 3514, 991, 636, 0, 0, 158, 10, 246, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1869, 4027, 3205, 2943, 5415, 1717, 9082, 4510] avg=4096.0000 max=9082 max_rank=6 min=1717 min_rank=5 imbalance_ratio=2.217285 max_load_reduction=23686 max_load_reduction_pct=0.722839 moved_to_echo=20724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2699, 0, 423, 3324, 304, 0, 0, 0, 0, 0, 1, 0, 229, 0, 0, 0, 0, 3044, 0, 0, 0, 0, 0, 88, 0, 4092, 3, 2113, 157, 0, 0, 0, 0, 2529, 3484, 0, 0, 0, 0, 1072, 854, 1359, 2, 0, 0, 0, 0, 0, 15, 0, 0, 1052, 1285, 0, 0, 15, 0, 531, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6750, 230, 3044, 6453, 6013, 3287, 2352, 4639] avg=4096.0000 max=6750 max_rank=0 min=230 min_rank=1 imbalance_ratio=1.647949 max_load_reduction=26018 max_load_reduction_pct=0.794006 moved_to_echo=16291 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3280, 87, 0, 3333, 238, 0, 0, 0, 11, 0, 19, 4077, 230, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 9, 1660, 162, 0, 0, 1682, 773, 2602, 170, 0, 0, 0, 0, 45, 0, 1069, 740, 0, 0, 0, 0, 2864, 14, 0, 11, 0, 1323, 0, 0, 1280, 1998, 532, 459, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6938, 4337, 4096, 1835, 5227, 1854, 4212, 4269] avg=4096.0000 max=6938 max_rank=0 min=1835 min_rank=3 imbalance_ratio=1.693848 max_load_reduction=25830 max_load_reduction_pct=0.788269 moved_to_echo=15562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11264, 5760, 11008, 4992, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[12160, 4352, 128, 4352, 6144, 6144, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[7936, 8960, 11904, 4096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[896, 5760, 7424, 1152, 17024, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 6656, 1408, 512, 21632, 2560, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15104, 0, 17536, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1536, 4736, 3456, 23296, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 3328, 128, 27904, 1536, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3095, 0, 816, 0, 3703, 0, 510, 0, 0, 407, 204, 3222, 0, 2788, 3590, 0, 556, 4073, 4096, 3653, 0, 0, 56, 1193, 330, 0, 0, 0, 0, 0, 1, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3178, 4091, 2317, 0, 2679, 1304, 0, 3996, 0, 0, 6, 0, 7, 11, 648, 112, 3, 2811, 3722, 4069, 381, 0, 8, 0, 1, 16, 3078, 0, 0, 0, 0, 330] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 77, 0, 60, 3765, 13, 0, 1, 1, 0, 144, 502, 0, 4096, 4083, 0, 0, 16, 2425, 4087, 2569, 29, 3236, 122, 1063, 13, 4012, 493, 3, 5, 1953, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[109, 0, 4086, 3820, 0, 11, 4030, 45, 6, 0, 1, 47, 0, 0, 4055, 1684, 4096, 10, 2327, 21, 0, 0, 0, 0, 3701, 0, 253, 0, 4078, 0, 388, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1415, 0, 3247, 0, 89, 4067, 0, 153, 0, 0, 1911, 0, 4095, 5, 0, 4096, 4093, 3944, 4, 0, 2640, 0, 0, 48, 31, 0, 2901, 0, 0, 29] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[530, 0, 38, 731, 3377, 82, 568, 171, 296, 0, 3639, 3668, 0, 0, 1197, 25, 3, 4096, 79, 3283, 0, 128, 2965, 0, 431, 0, 3416, 0, 4037, 0, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4093, 0, 2583, 0, 0, 0, 0, 1065, 1145, 199, 3817, 0, 13, 3603, 3934, 0, 9, 0, 1, 24, 0, 3687, 4085, 0, 0, 130, 11, 275] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 3424, 83, 4096, 0, 0, 2703, 1, 23, 4017, 915, 534, 183, 3967, 1, 4096, 6, 2343, 2, 0, 0, 0, 190, 1, 0, 0, 1, 5, 1, 28, 2052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2421, 0, 816, 0, 0, 0, 0, 0, 1869, 0, 510, 0, 0, 0, 0, 0, 0, 407, 204, 3222, 1834, 290, 80, 0, 0, 2788, 2831, 0, 0, 0, 0, 0, 556, 4073, 0, 1015, 0, 0, 0, 0, 0, 0, 56, 1193, 3418, 0, 0, 0, 330, 0, 0, 0, 679, 674, 0, 0, 0, 0, 1, 475, 2348, 678, 0, 0] rerouted_tokens_per_ep_rank=[3237, 2379, 6037, 5619, 5644, 4667, 1683, 3502] avg=4096.0000 max=6037 max_rank=2 min=1683 min_rank=6 imbalance_ratio=1.473877 max_load_reduction=26731 max_load_reduction_pct=0.815765 moved_to_echo=15496 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3202, 0, 0, 0, 0, 0, 0, 0, 2066, 0, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 2027, 0, 85, 0, 1145, 199, 3010, 0, 0, 0, 0, 0, 13, 3603, 0, 0, 0, 0, 0, 0, 9, 0, 1, 24, 3283, 0, 0, 0, 0, 3687, 4085, 0, 722, 892, 0, 0, 0, 130, 11, 275, 0, 651, 0, 0] rerouted_tokens_per_ep_rank=[3202, 4649, 3177, 4354, 3616, 3317, 9386, 1067] avg=4096.0000 max=9386 max_rank=6 min=1067 min_rank=7 imbalance_ratio=2.291504 max_load_reduction=23382 max_load_reduction_pct=0.713562 moved_to_echo=17386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2486, 4091, 2317, 0, 0, 0, 0, 0, 1352, 1304, 0, 3996, 0, 0, 0, 0, 0, 0, 6, 0, 1327, 323, 14, 0, 7, 11, 512, 112, 0, 0, 0, 0, 3, 2811, 0, 1131, 0, 0, 0, 0, 381, 0, 8, 0, 3106, 0, 0, 0, 1, 16, 3078, 0, 122, 692, 0, 0, 0, 0, 0, 330, 2615, 616, 0, 0] rerouted_tokens_per_ep_rank=[8894, 6652, 1670, 642, 3945, 3495, 3909, 3561] avg=4096.0000 max=8894 max_rank=0 min=642 min_rank=3 imbalance_ratio=2.171387 max_load_reduction=23874 max_load_reduction_pct=0.728577 moved_to_echo=14910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[82, 0, 4086, 3820, 0, 0, 0, 0, 0, 11, 4030, 45, 0, 0, 0, 0, 6, 0, 1, 47, 0, 5, 94, 0, 0, 0, 3190, 1684, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1943, 0, 0, 0, 3701, 0, 253, 0, 771, 27, 0, 0, 4078, 0, 388, 0, 16, 384, 0, 0] rerouted_tokens_per_ep_rank=[7988, 4086, 153, 4874, 4106, 1943, 4752, 4866] avg=4096.0000 max=7988 max_rank=0 min=153 min_rank=2 imbalance_ratio=1.950195 max_load_reduction=24780 max_load_reduction_pct=0.756226 moved_to_echo=15667 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 77, 0, 60, 0, 0, 0, 0, 1900, 13, 0, 1, 0, 0, 0, 0, 1, 0, 144, 502, 1865, 325, 91, 0, 0, 4096, 3220, 0, 0, 0, 0, 0, 0, 16, 0, 1135, 0, 0, 0, 0, 2569, 29, 3236, 122, 2024, 0, 0, 0, 1063, 13, 4012, 493, 772, 0, 0, 0, 3, 5, 1953, 0, 2627, 401, 0, 0] rerouted_tokens_per_ep_rank=[137, 1914, 2928, 7316, 1151, 7980, 6353, 4989] avg=4096.0000 max=7980 max_rank=5 min=137 min_rank=0 imbalance_ratio=1.948242 max_load_reduction=24788 max_load_reduction_pct=0.756470 moved_to_echo=20473 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1415, 0, 0, 0, 0, 0, 1639, 0, 89, 4067, 0, 0, 0, 0, 0, 153, 0, 0, 1608, 313, 91, 0, 1911, 0, 3230, 5, 0, 0, 0, 0, 0, 4096, 0, 1096, 0, 0, 0, 0, 4, 0, 2640, 0, 3417, 0, 0, 0, 0, 48, 31, 0, 774, 0, 0, 0, 2901, 0, 0, 29, 2535, 676, 0, 0] rerouted_tokens_per_ep_rank=[1415, 5795, 2165, 5146, 5192, 6061, 853, 6141] avg=4096.0000 max=6141 max_rank=7 min=853 min_rank=6 imbalance_ratio=1.499268 max_load_reduction=26627 max_load_reduction_pct=0.812592 moved_to_echo=18247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3204, 0, 3424, 83, 0, 0, 0, 0, 2067, 0, 0, 2703, 0, 0, 0, 0, 1, 23, 4017, 915, 2029, 0, 88, 0, 534, 183, 3129, 1, 0, 0, 0, 0, 4096, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 190, 1956, 0, 0, 0, 1, 0, 0, 1, 750, 892, 0, 0, 5, 1, 28, 2052, 1, 387, 0, 0] rerouted_tokens_per_ep_rank=[6711, 4770, 7073, 3847, 4103, 2146, 1644, 2474] avg=4096.0000 max=7073 max_rank=2 min=1644 min_rank=6 imbalance_ratio=1.726807 max_load_reduction=25695 max_load_reduction_pct=0.784149 moved_to_echo=10367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[415, 0, 38, 731, 0, 0, 0, 0, 1702, 82, 568, 171, 0, 0, 0, 0, 296, 0, 3639, 3668, 1675, 261, 26, 0, 0, 0, 945, 25, 0, 0, 0, 0, 3, 4096, 0, 912, 0, 0, 0, 0, 0, 128, 2965, 0, 66, 0, 0, 0, 431, 0, 3416, 0, 226, 115, 0, 0, 4037, 0, 8, 0, 2110, 13, 0, 0] rerouted_tokens_per_ep_rank=[1184, 2523, 9565, 970, 5011, 3159, 4188, 6168] avg=4096.0000 max=9565 max_rank=2 min=970 min_rank=3 imbalance_ratio=2.335205 max_load_reduction=23203 max_load_reduction_pct=0.708099 moved_to_echo=18526 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11904, 4224, 12160, 4736, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 640, 8064, 9472, 12416, 1536, 640, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5632, 3840, 14976, 512, 4864, 3328, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3712, 7296, 20096, 1920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3072, 256, 8960, 1536, 19328, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8832, 18816, 0, 5376, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11136, 256, 2432, 3200, 12288, 3840, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12672, 1536, 7808, 11008, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4070, 3087, 0, 3949, 0, 2, 0, 0, 3767, 3680, 0, 0, 365, 0, 0, 4095, 0, 331, 4091, 0, 0, 1218, 0, 27, 2, 0, 4057, 0, 0, 0, 27] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 558, 2758, 0, 0, 14, 51, 0, 0, 1196, 4066, 96, 3696, 262, 944, 425, 1236, 26, 0, 3650, 65, 4096, 112, 927, 10, 3022, 5, 0, 4090, 1392, 71, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3830, 0, 0, 0, 0, 0, 0, 315, 4096, 3879, 4058, 4095, 0, 361, 32, 0, 0, 2, 1402, 4096, 0, 1549, 51, 0, 127, 528, 4015, 9, 0, 323] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4058, 0, 0, 0, 44, 3878, 34, 3115, 3708, 30, 953, 0, 0, 1, 4093, 207, 77, 4076, 4, 223, 0, 23, 0, 1, 0, 3776, 4096, 0, 0, 371] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 3523, 4042, 0, 4031, 0, 199, 0, 0, 2249, 3448, 4037, 36, 95, 0, 0, 4089, 38, 4051, 0, 2, 1304, 0, 0, 0, 1, 0, 0, 0, 1622, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 3, 4085, 108, 0, 0, 4081, 0, 179, 771, 244, 80, 5, 416, 3907, 0, 3717, 116, 0, 4087, 0, 7, 0, 0, 3153, 86, 15, 3934, 3125, 0, 649, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4064, 0, 4095, 0, 174, 0, 0, 4016, 4085, 1483, 3923, 54, 0, 0, 4076, 26, 407, 3708, 0, 3, 0, 0, 4, 0, 5, 0, 0, 0, 2645, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 1187, 0, 4, 723, 0, 3543, 12, 3940, 4096, 2678, 630, 150, 0, 0, 3774, 0, 0, 4096, 595, 0, 0, 5, 0, 16, 0, 615, 4095, 2319, 283, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 44, 3878, 1300, 0, 0, 0, 34, 3115, 121, 30, 0, 0, 0, 0, 953, 0, 0, 1, 1367, 0, 0, 0, 619, 207, 77, 4076, 0, 0, 0, 0, 4, 223, 0, 23, 2317, 0, 0, 0, 0, 1, 0, 3776, 1270, 641, 0, 0, 4096, 0, 0, 371, 389, 166, 0, 0] rerouted_tokens_per_ep_rank=[3669, 5222, 3300, 2321, 4979, 2567, 5688, 5022] avg=4096.0000 max=5688 max_rank=6 min=2321 min_rank=3 imbalance_ratio=1.388672 max_load_reduction=27080 max_load_reduction_pct=0.826416 moved_to_echo=18256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3674, 0, 0, 0, 0, 0, 4095, 0, 174, 0, 1295, 0, 0, 0, 0, 4016, 134, 1483, 0, 0, 0, 0, 3923, 54, 0, 0, 1362, 0, 0, 0, 614, 26, 407, 3708, 0, 0, 0, 0, 0, 3, 0, 0, 2552, 0, 0, 0, 4, 0, 5, 0, 1399, 639, 0, 0, 0, 0, 2645, 0, 390, 166, 0, 0] rerouted_tokens_per_ep_rank=[3674, 5564, 5633, 5339, 4755, 2555, 2047, 3201] avg=4096.0000 max=5633 max_rank=2 min=2047 min_rank=6 imbalance_ratio=1.375244 max_load_reduction=27135 max_load_reduction_pct=0.828094 moved_to_echo=12558 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 558, 2493, 0, 0, 0, 0, 0, 0, 14, 51, 0, 392, 0, 0, 0, 0, 1196, 134, 96, 0, 0, 0, 0, 3696, 262, 944, 425, 413, 0, 0, 0, 188, 26, 0, 3650, 0, 0, 0, 0, 65, 4096, 112, 927, 2540, 0, 0, 0, 10, 3022, 5, 0, 1392, 193, 0, 0, 4090, 1392, 71, 0, 265, 50, 0, 0] rerouted_tokens_per_ep_rank=[3051, 457, 1426, 5740, 3864, 7740, 4622, 5868] avg=4096.0000 max=7740 max_rank=5 min=457 min_rank=1 imbalance_ratio=1.889648 max_load_reduction=25028 max_load_reduction_pct=0.763794 moved_to_echo=22094 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 315, 134, 3879, 0, 0, 0, 0, 4058, 4095, 0, 361, 10, 0, 0, 0, 6, 0, 0, 2, 0, 0, 0, 0, 1402, 4096, 0, 1549, 2559, 0, 0, 0, 51, 0, 127, 528, 1403, 5, 0, 0, 4015, 9, 0, 323, 368, 1, 0, 0] rerouted_tokens_per_ep_rank=[3462, 10, 4328, 8524, 8, 9606, 2114, 4716] avg=4096.0000 max=9606 max_rank=5 min=8 min_rank=4 imbalance_ratio=2.345215 max_load_reduction=23162 max_load_reduction_pct=0.706848 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1073, 0, 0, 0, 0, 0, 4, 723, 0, 3543, 1199, 0, 0, 0, 12, 3940, 134, 2678, 0, 0, 0, 0, 630, 150, 0, 0, 1261, 0, 0, 0, 570, 0, 0, 4096, 0, 0, 0, 0, 595, 0, 0, 5, 2559, 0, 0, 0, 0, 16, 0, 615, 1403, 591, 0, 0, 4095, 2319, 283, 7, 114, 153, 0, 0] rerouted_tokens_per_ep_rank=[1073, 5469, 6764, 2041, 4666, 3159, 2625, 6971] avg=4096.0000 max=6971 max_rank=7 min=1073 min_rank=0 imbalance_ratio=1.701904 max_load_reduction=25797 max_load_reduction_pct=0.787262 moved_to_echo=17421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3693, 108, 0, 0, 0, 0, 0, 0, 4081, 0, 1181, 0, 0, 0, 179, 771, 9, 80, 0, 0, 0, 0, 5, 416, 3907, 0, 1242, 0, 0, 0, 561, 116, 0, 4087, 0, 0, 0, 0, 0, 7, 0, 0, 152, 0, 0, 0, 3153, 86, 15, 3934, 83, 582, 0, 0, 3125, 0, 649, 0, 392, 151, 0, 0] rerouted_tokens_per_ep_rank=[3804, 5262, 1039, 5570, 4764, 159, 7853, 4317] avg=4096.0000 max=7853 max_rank=6 min=159 min_rank=5 imbalance_ratio=1.917236 max_load_reduction=24915 max_load_reduction_pct=0.760345 moved_to_echo=17093 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3523, 3651, 0, 0, 0, 0, 0, 4031, 0, 199, 0, 1302, 0, 0, 0, 0, 2249, 104, 4037, 0, 0, 0, 0, 36, 95, 0, 0, 1369, 0, 0, 0, 603, 38, 4051, 0, 0, 0, 0, 0, 2, 1304, 0, 0, 2159, 0, 0, 0, 0, 1, 0, 0, 1185, 645, 0, 0, 0, 1622, 0, 1, 391, 170, 0, 0] rerouted_tokens_per_ep_rank=[7174, 5532, 6390, 1500, 4692, 3465, 1831, 2184] avg=4096.0000 max=7174 max_rank=0 min=1500 min_rank=3 imbalance_ratio=1.751465 max_load_reduction=25594 max_load_reduction_pct=0.781067 moved_to_echo=12172 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4070, 2791, 0, 0, 0, 0, 0, 3949, 0, 2, 0, 1301, 0, 0, 0, 0, 3767, 121, 0, 0, 0, 0, 0, 0, 365, 0, 0, 1368, 0, 0, 0, 618, 0, 331, 4091, 0, 0, 0, 0, 0, 0, 1218, 0, 2299, 0, 0, 0, 27, 2, 0, 4057, 1260, 642, 0, 0, 0, 0, 0, 27, 296, 166, 0, 0] rerouted_tokens_per_ep_rank=[6861, 5252, 3888, 1733, 5040, 3517, 5988, 489] avg=4096.0000 max=6861 max_rank=0 min=489 min_rank=7 imbalance_ratio=1.675049 max_load_reduction=25907 max_load_reduction_pct=0.790619 moved_to_echo=15034 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3328, 3200, 256, 12928, 9472, 3968, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2176, 9856, 1408, 2560, 17152, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13312, 5504, 4864, 896, 8448, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 8192, 24576, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12160, 768, 4608, 7424, 8064, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[19456, 5376, 3712, 768, 2688, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 19456, 896, 12288, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3840, 512, 4992, 23808, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 4051, 0, 0, 0, 432, 3, 39, 0, 4090, 4096, 0, 55, 0, 4013, 4093, 3194, 0, 0, 1427, 4096, 43, 0, 571, 0, 0, 0, 2054, 511, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4094, 4096, 0, 0, 0, 0, 1595, 1, 0, 4096, 4093, 0, 0, 9, 0, 0, 0, 104, 0, 4096, 2496, 0, 52, 0, 0, 72, 2, 4091, 351, 0, 3520, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1, 4077, 289, 1019, 11, 2300, 4062, 0, 107, 0, 0, 3, 3055, 1, 0, 97, 3721, 0, 0, 0, 2795, 0, 322, 0, 0, 2, 15, 347, 3999, 3616, 1495, 1434] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[3843, 3080, 0, 0, 24, 0, 0, 56, 0, 0, 3707, 0, 28, 0, 0, 0, 1281, 4096, 0, 4077, 3, 0, 4028, 452, 12, 0, 0, 4067, 4006, 0, 4, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[2243, 3727, 0, 0, 75, 0, 3967, 0, 275, 4083, 1, 186, 0, 577, 0, 0, 0, 0, 0, 1999, 4009, 92, 30, 4060, 0, 0, 0, 3348, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[3820, 760, 70, 0, 39, 2329, 0, 0, 0, 3425, 0, 2, 3935, 0, 1821, 0, 1, 41, 148, 1478, 4096, 152, 0, 21, 1541, 40, 0, 1003, 20, 0, 4089, 3937] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 3662, 0, 34, 0, 0, 89, 0, 0, 76, 3919, 0, 0, 0, 232, 3829, 4, 323, 0, 4094, 0, 0, 349, 1, 0, 0, 0, 4002, 4025, 0, 0, 4033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[2276, 2674, 745, 0, 0, 0, 0, 101, 0, 4049, 3438, 0, 0, 3192, 954, 0, 473, 0, 0, 3, 13, 205, 0, 4096, 4083, 209, 0, 0, 112, 2074, 4071, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1588, 0, 0, 0, 0, 0, 0, 0, 432, 3, 39, 2463, 0, 0, 0, 0, 2672, 4096, 0, 0, 0, 0, 0, 55, 0, 4013, 4093, 801, 0, 0, 0, 3194, 0, 0, 1427, 455, 0, 0, 0, 4096, 43, 0, 571, 162, 0, 0, 0, 0, 0, 0, 2054, 0, 0, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1588, 2937, 6768, 8962, 5076, 4872, 2054, 511] avg=4096.0000 max=8962 max_rank=3 min=511 min_rank=7 imbalance_ratio=2.187988 max_load_reduction=23806 max_load_reduction_pct=0.726501 moved_to_echo=12513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2276, 1045, 745, 0, 0, 0, 0, 0, 0, 0, 0, 101, 1629, 417, 0, 0, 0, 2636, 3438, 0, 0, 0, 0, 0, 0, 3192, 954, 0, 796, 708, 0, 0, 473, 0, 0, 3, 454, 0, 0, 0, 13, 205, 0, 4096, 163, 0, 0, 0, 4083, 209, 0, 0, 1860, 0, 0, 0, 112, 2074, 1086, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 2147, 6074, 5650, 930, 4477, 6152, 3272] avg=4096.0000 max=6152 max_rank=6 min=930 min_rank=4 imbalance_ratio=1.501953 max_load_reduction=26616 max_load_reduction_pct=0.812256 moved_to_echo=14831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 1435, 0, 34, 0, 0, 0, 0, 0, 0, 89, 0, 2227, 0, 0, 0, 0, 51, 3919, 0, 0, 0, 0, 0, 0, 0, 232, 3829, 14, 0, 0, 0, 4, 323, 0, 4094, 8, 0, 0, 0, 0, 0, 349, 1, 3, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 4025, 0, 0, 4033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5565, 2316, 3970, 4075, 4429, 353, 4002, 8058] avg=4096.0000 max=8058 max_rank=7 min=353 min_rank=5 imbalance_ratio=1.967285 max_load_reduction=24710 max_load_reduction_pct=0.754089 moved_to_echo=16842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4094, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 1595, 1, 2490, 357, 0, 0, 0, 2676, 4093, 0, 0, 0, 0, 0, 0, 9, 0, 0, 802, 609, 0, 0, 0, 104, 0, 4096, 456, 0, 0, 0, 2496, 0, 52, 0, 162, 0, 0, 0, 0, 72, 2, 4091, 1605, 0, 0, 0, 351, 0, 949, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5700, 4443, 6769, 1420, 4656, 2710, 5770, 1300] avg=4096.0000 max=6769 max_rank=2 min=1300 min_rank=7 imbalance_ratio=1.652588 max_load_reduction=25999 max_load_reduction_pct=0.793427 moved_to_echo=14436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3843, 1207, 0, 0, 0, 0, 0, 0, 24, 0, 0, 56, 1873, 0, 0, 0, 0, 0, 3707, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 1281, 4096, 0, 4077, 0, 0, 0, 0, 3, 0, 4028, 452, 0, 0, 0, 0, 12, 0, 0, 4067, 1, 0, 0, 0, 4006, 0, 3, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5050, 1953, 3707, 28, 9454, 4483, 4080, 4013] avg=4096.0000 max=9454 max_rank=4 min=28 min_rank=3 imbalance_ratio=2.308105 max_load_reduction=23314 max_load_reduction_pct=0.711487 moved_to_echo=22030 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 1598, 289, 1019, 0, 0, 0, 0, 11, 2300, 4062, 0, 2479, 151, 0, 0, 107, 0, 0, 3, 0, 0, 0, 0, 3055, 1, 0, 97, 0, 259, 0, 0, 3721, 0, 0, 0, 0, 0, 0, 0, 2795, 0, 322, 0, 0, 0, 0, 0, 0, 2, 15, 347, 681, 0, 0, 0, 3999, 3616, 404, 1434, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2907, 9003, 110, 3412, 3721, 3117, 1045, 9453] avg=4096.0000 max=9453 max_rank=7 min=110 min_rank=2 imbalance_ratio=2.307861 max_load_reduction=23315 max_load_reduction_pct=0.711517 moved_to_echo=17336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3820, 298, 70, 0, 0, 0, 0, 0, 39, 2329, 0, 0, 462, 415, 0, 0, 0, 2238, 0, 2, 0, 0, 0, 0, 3935, 0, 1821, 0, 671, 708, 0, 0, 1, 41, 148, 1478, 381, 0, 0, 0, 4096, 152, 0, 21, 135, 0, 0, 0, 1541, 40, 0, 1003, 1865, 0, 0, 0, 20, 0, 1101, 3937, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4188, 3245, 2240, 7135, 2049, 4404, 4449, 5058] avg=4096.0000 max=7135 max_rank=3 min=2049 min_rank=4 imbalance_ratio=1.741943 max_load_reduction=25633 max_load_reduction_pct=0.782257 moved_to_echo=15960 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2243, 1461, 0, 0, 0, 0, 0, 0, 75, 0, 3967, 0, 2266, 416, 0, 0, 275, 2668, 1, 186, 0, 0, 0, 0, 0, 577, 0, 0, 800, 709, 0, 0, 0, 0, 0, 1999, 454, 0, 0, 0, 4009, 92, 30, 4060, 161, 0, 0, 0, 0, 0, 0, 3348, 1868, 0, 0, 0, 0, 0, 1103, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3704, 6724, 3130, 2086, 2453, 8352, 5216, 1103] avg=4096.0000 max=8352 max_rank=5 min=1103 min_rank=7 imbalance_ratio=2.039062 max_load_reduction=24416 max_load_reduction_pct=0.745117 moved_to_echo=17124 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13056, 5760, 4736, 9472, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7168, 3840, 7040, 8064, 3968, 3072, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 5120, 9728, 256, 16000, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8704, 4608, 256, 17280, 2304, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[20480, 10240, 1152, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5760, 384, 128, 18944, 7936, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 13056, 19328, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17536, 512, 4864, 9216, 896, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[25, 880, 4073, 0, 3815, 4081, 2992, 0, 4096, 471, 0, 251, 0, 27, 0, 3, 4096, 0, 0, 4096, 0, 0, 236, 0, 0, 0, 0, 6, 0, 4, 1, 3615] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4084, 0, 4096, 0, 3564, 195, 1357, 0, 3383, 431, 0, 0, 2141, 4080, 759, 1, 49, 15, 195, 4096, 0, 0, 7, 4077, 0, 0, 0, 195, 36, 0, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 2, 2713, 368, 4034, 0, 3962, 4095, 116, 0, 0, 796, 10, 0, 0, 3415, 76, 0, 1, 3696, 0, 1290, 4096, 0, 0, 0, 0, 0, 1, 1, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 13, 2850, 0, 1, 4075, 491, 684, 4042, 0, 0, 3844, 0, 200, 0, 4039, 2, 0, 500, 3833, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 1, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4075, 0, 4045, 229, 3116, 1851, 533, 4093, 3976, 126, 613, 0, 1143, 33, 0, 4091, 0, 403, 0, 0, 0, 0, 4067, 0, 335, 15, 0, 12, 7, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4055, 0, 522, 2, 4050, 5, 2362, 30, 3639, 0, 384, 0, 4096, 0, 0, 3533, 3574, 0, 0, 4096, 0, 0, 0, 0, 0, 1808, 329, 0, 0, 0, 16, 267] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[353, 4, 0, 473, 33, 0, 106, 1929, 4088, 4079, 0, 0, 258, 4040, 72, 2891, 2417, 71, 0, 4094, 1773, 0, 7, 2011, 10, 1, 1, 0, 0, 80, 0, 3977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[18, 64, 4012, 5, 4092, 3013, 76, 1458, 0, 0, 0, 280, 103, 4096, 0, 3887, 1736, 0, 0, 3666, 0, 5, 4046, 1, 552, 0, 3, 0, 12, 1434, 209, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 2200, 368, 0, 0, 0, 0, 0, 0, 3962, 4095, 0, 0, 0, 0, 103, 0, 0, 796, 0, 0, 0, 0, 10, 0, 0, 1822, 0, 0, 0, 0, 76, 0, 1, 2628, 0, 0, 0, 0, 0, 1290, 4096, 0, 513, 13, 0, 0, 0, 0, 0, 0, 4034, 1066, 0, 0, 1, 1, 0, 4096, 1068, 0, 527, 0] rerouted_tokens_per_ep_rank=[2570, 8057, 899, 1832, 2705, 5912, 5100, 5693] avg=4096.0000 max=8057 max_rank=1 min=899 min_rank=2 imbalance_ratio=1.967041 max_load_reduction=24711 max_load_reduction_pct=0.754120 moved_to_echo=19410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[25, 880, 3308, 0, 0, 0, 0, 0, 0, 2656, 2992, 0, 0, 0, 0, 0, 3772, 471, 0, 251, 0, 0, 0, 0, 0, 27, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 2917, 0, 0, 0, 0, 0, 0, 236, 0, 765, 324, 313, 0, 0, 0, 0, 6, 3815, 0, 0, 0, 0, 4, 1, 3615, 1179, 1112, 0, 0] rerouted_tokens_per_ep_rank=[4213, 5648, 4494, 30, 7013, 1638, 3821, 5911] avg=4096.0000 max=7013 max_rank=4 min=30 min_rank=3 imbalance_ratio=1.712158 max_load_reduction=25755 max_load_reduction_pct=0.785980 moved_to_echo=18383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 3327, 0, 0, 0, 0, 0, 0, 128, 1357, 0, 0, 0, 0, 0, 3116, 431, 0, 0, 0, 0, 0, 0, 2141, 4080, 759, 1, 0, 0, 0, 0, 49, 15, 195, 2917, 0, 0, 0, 0, 0, 0, 7, 4077, 769, 267, 14, 0, 0, 0, 0, 195, 3564, 0, 0, 0, 36, 0, 7, 0, 1179, 53, 0, 0] rerouted_tokens_per_ep_rank=[7411, 1485, 3547, 6981, 3176, 5134, 3759, 1275] avg=4096.0000 max=7411 max_rank=0 min=1275 min_rank=7 imbalance_ratio=1.809326 max_load_reduction=25357 max_load_reduction_pct=0.773834 moved_to_echo=13344 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[18, 64, 3259, 5, 0, 0, 0, 0, 0, 1961, 76, 1458, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 103, 4096, 0, 2083, 0, 0, 0, 0, 1736, 0, 0, 2610, 0, 0, 0, 0, 0, 5, 4046, 1, 753, 0, 231, 0, 552, 0, 3, 0, 4092, 1208, 0, 0, 12, 1434, 209, 0, 1056, 821, 596, 0] rerouted_tokens_per_ep_rank=[3346, 3495, 280, 6282, 4346, 5036, 5855, 4128] avg=4096.0000 max=6282 max_rank=3 min=280 min_rank=2 imbalance_ratio=1.533691 max_load_reduction=26486 max_load_reduction_pct=0.808289 moved_to_echo=19365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4055, 0, 424, 2, 0, 0, 0, 0, 0, 4, 2362, 30, 0, 0, 0, 0, 3352, 0, 384, 0, 0, 0, 0, 0, 4096, 0, 0, 1893, 0, 0, 0, 0, 3574, 0, 0, 2917, 0, 0, 0, 0, 0, 0, 0, 0, 98, 287, 0, 0, 0, 1808, 329, 0, 4050, 1098, 0, 0, 0, 0, 16, 267, 1179, 1, 542, 0] rerouted_tokens_per_ep_rank=[4481, 2396, 3736, 5989, 6491, 385, 7285, 2005] avg=4096.0000 max=7285 max_rank=6 min=385 min_rank=5 imbalance_ratio=1.778564 max_load_reduction=25483 max_load_reduction_pct=0.777679 moved_to_echo=16166 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4075, 0, 3285, 229, 0, 0, 0, 0, 0, 1205, 533, 4093, 0, 0, 0, 0, 3662, 126, 613, 0, 0, 0, 0, 0, 1143, 33, 0, 2193, 0, 0, 0, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 760, 314, 142, 0, 335, 15, 0, 12, 3116, 1271, 0, 0, 7, 5, 0, 0, 0, 504, 627, 0] rerouted_tokens_per_ep_rank=[7589, 5831, 4401, 3369, 403, 5283, 4749, 1143] avg=4096.0000 max=7589 max_rank=0 min=403 min_rank=4 imbalance_ratio=1.852783 max_load_reduction=25179 max_load_reduction_pct=0.768402 moved_to_echo=11578 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 2315, 0, 0, 0, 0, 0, 0, 2646, 491, 684, 0, 0, 0, 0, 3723, 0, 0, 3844, 0, 0, 0, 0, 0, 200, 0, 2165, 0, 0, 0, 0, 2, 0, 500, 2729, 0, 0, 0, 0, 0, 0, 4096, 0, 535, 319, 316, 0, 0, 0, 0, 0, 1, 1255, 0, 0, 1, 1, 0, 4096, 1104, 1113, 619, 0] rerouted_tokens_per_ep_rank=[2328, 3821, 7567, 2365, 3231, 5266, 1256, 6934] avg=4096.0000 max=7567 max_rank=2 min=1256 min_rank=6 imbalance_ratio=1.847412 max_load_reduction=25201 max_load_reduction_pct=0.769073 moved_to_echo=16687 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[353, 4, 0, 473, 0, 0, 0, 0, 0, 0, 106, 1929, 0, 0, 0, 0, 3765, 4079, 0, 0, 0, 0, 0, 0, 258, 4040, 72, 1550, 0, 0, 0, 0, 2417, 71, 0, 2915, 0, 0, 0, 0, 1773, 0, 7, 2011, 0, 323, 0, 0, 10, 1, 1, 0, 33, 898, 0, 0, 0, 80, 0, 3977, 1179, 0, 443, 0] rerouted_tokens_per_ep_rank=[830, 2035, 7844, 5920, 5403, 4114, 943, 5679] avg=4096.0000 max=7844 max_rank=2 min=830 min_rank=0 imbalance_ratio=1.915039 max_load_reduction=24924 max_load_reduction_pct=0.760620 moved_to_echo=16139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12032, 512, 768, 19712, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7808, 12544, 896, 11776, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1536, 256, 16128, 8064, 3712, 3456, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1792, 1408, 16640, 6144, 4224, 1920, 1024, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 8704, 11904, 12416, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[21504, 5120, 1024, 5248, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12672, 1024, 18176, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1024, 1920, 384, 256, 22784, 6912, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 3901, 1274, 0, 4095, 0, 0, 0, 2, 0, 4095, 1, 16, 0, 1, 0, 2676, 4094, 277, 148, 57, 3398, 0, 0, 543, 0, 4094, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 904, 361, 5, 4085, 3160, 0, 0, 0, 515, 158, 1, 0, 1362, 284, 3801, 619, 0, 0, 0, 4018, 50, 192, 0, 1797, 3313, 4094, 0, 4049, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 2757, 4095, 0, 3203, 0, 16, 34, 3992, 0, 0, 0, 37, 0, 602, 0, 426, 0, 0, 325, 4096, 0, 2, 0, 611, 3836, 2, 3980, 0, 219, 439] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[11, 4074, 2, 0, 4096, 0, 0, 92, 0, 4092, 1, 0, 0, 91, 0, 722, 0, 4092, 1, 7, 3953, 3398, 0, 0, 0, 0, 4096, 70, 3970, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 4007, 0, 0, 50, 0, 0, 0, 0, 0, 114, 0, 0, 0, 3259, 8, 4088, 1156, 0, 3480, 3888, 0, 0, 3917, 0, 194, 123, 4082, 0, 214, 95] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 3765, 0, 0, 73, 0, 412, 976, 4072, 0, 0, 0, 3585, 0, 1851, 0, 2165, 0, 2537, 0, 4096, 0, 318, 0, 0, 4069, 9, 710, 0, 31, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 3870, 3, 4, 1543, 0, 0, 1, 0, 2225, 3, 0, 0, 0, 3883, 673, 219, 19, 2366, 0, 45, 4096, 0, 0, 0, 0, 4087, 0, 4096, 38, 1873, 3724] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 3777, 3958, 0, 675, 0, 0, 263, 1320, 2978, 458, 85, 141, 0, 99, 238, 122, 3935, 16, 366, 206, 4077, 22, 0, 3745, 4, 3964, 1, 2311, 0, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 394, 361, 5, 0, 0, 0, 0, 4085, 3160, 0, 0, 985, 130, 0, 0, 0, 515, 158, 1, 8, 426, 0, 0, 0, 1362, 284, 3801, 380, 0, 0, 0, 619, 0, 0, 0, 4, 173, 0, 0, 4018, 38, 192, 0, 0, 0, 0, 0, 1797, 3313, 2683, 0, 0, 0, 0, 0, 3876, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[760, 8360, 1108, 5827, 796, 4248, 7793, 3876] avg=4096.0000 max=8360 max_rank=1 min=760 min_rank=0 imbalance_ratio=2.041016 max_load_reduction=24408 max_load_reduction_pct=0.744873 moved_to_echo=16713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1697, 1274, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 985, 563, 0, 0, 2, 0, 4095, 1, 587, 426, 0, 0, 16, 0, 1, 0, 1641, 0, 0, 0, 2676, 4094, 277, 148, 283, 175, 0, 0, 57, 2528, 0, 0, 0, 0, 0, 0, 543, 0, 2683, 0, 0, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2971, 5643, 5111, 1658, 7653, 2585, 3226, 3921] avg=4096.0000 max=7653 max_rank=4 min=1658 min_rank=3 imbalance_ratio=1.868408 max_load_reduction=25115 max_load_reduction_pct=0.766449 moved_to_echo=17385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1780, 4007, 0, 0, 0, 0, 0, 0, 50, 0, 0, 46, 591, 0, 0, 0, 0, 0, 114, 672, 20, 0, 0, 0, 0, 0, 3259, 1722, 0, 0, 0, 8, 4088, 1156, 0, 324, 174, 0, 0, 3480, 2892, 0, 0, 0, 0, 0, 0, 3917, 0, 128, 123, 0, 0, 0, 0, 3908, 0, 214, 95, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5787, 687, 806, 4981, 5750, 6372, 4168, 4217] avg=4096.0000 max=6372 max_rank=5 min=687 min_rank=1 imbalance_ratio=1.555664 max_load_reduction=26396 max_load_reduction_pct=0.805542 moved_to_echo=20507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[11, 1772, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 92, 985, 588, 0, 0, 0, 4092, 1, 0, 587, 426, 0, 0, 0, 91, 0, 722, 1714, 0, 0, 0, 0, 4092, 1, 7, 283, 170, 0, 0, 3953, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 2685, 70, 0, 0, 0, 0, 3800, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1785, 5761, 5106, 2527, 4553, 6481, 2755, 3800] avg=4096.0000 max=6481 max_rank=5 min=1785 min_rank=0 imbalance_ratio=1.582275 max_load_reduction=26287 max_load_reduction_pct=0.802216 moved_to_echo=17589 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1782, 3765, 0, 0, 0, 0, 0, 0, 73, 0, 412, 979, 591, 0, 0, 976, 4072, 0, 0, 708, 423, 0, 0, 0, 3585, 0, 1851, 1723, 0, 0, 0, 0, 2165, 0, 2537, 341, 30, 0, 0, 0, 3047, 0, 318, 0, 0, 0, 0, 0, 0, 2667, 9, 0, 0, 0, 0, 680, 0, 31, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5547, 2055, 6179, 7159, 5073, 3365, 2676, 714] avg=4096.0000 max=7159 max_rank=3 min=714 min_rank=7 imbalance_ratio=1.747803 max_load_reduction=25609 max_load_reduction_pct=0.781525 moved_to_echo=11828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1683, 3, 4, 0, 0, 0, 0, 1543, 0, 0, 1, 983, 559, 0, 0, 0, 2225, 3, 0, 708, 425, 0, 0, 0, 0, 3883, 673, 1628, 0, 0, 0, 219, 19, 2366, 0, 341, 175, 0, 0, 45, 3047, 0, 0, 0, 0, 0, 0, 0, 0, 2679, 0, 0, 0, 0, 0, 3921, 38, 1873, 3724, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1690, 3086, 3361, 6184, 3120, 3092, 2679, 9556] avg=4096.0000 max=9556 max_rank=7 min=1690 min_rank=0 imbalance_ratio=2.333008 max_load_reduction=23212 max_load_reduction_pct=0.708374 moved_to_echo=18447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1775, 2757, 4095, 0, 0, 0, 0, 0, 3203, 0, 16, 926, 595, 0, 0, 34, 3992, 0, 0, 712, 401, 0, 0, 0, 37, 0, 602, 1726, 0, 0, 0, 0, 426, 0, 0, 346, 174, 0, 0, 325, 3038, 0, 2, 0, 0, 0, 0, 0, 611, 2509, 2, 0, 0, 0, 0, 3806, 0, 219, 439, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8627, 4740, 5139, 2365, 946, 3365, 3122, 4464] avg=4096.0000 max=8627 max_rank=0 min=946 min_rank=4 imbalance_ratio=2.106201 max_load_reduction=24141 max_load_reduction_pct=0.736725 moved_to_echo=11897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1643, 3958, 0, 0, 0, 0, 0, 675, 0, 0, 263, 953, 545, 0, 0, 1320, 2978, 458, 85, 705, 412, 0, 0, 141, 0, 99, 238, 1589, 0, 0, 0, 122, 3935, 16, 366, 340, 98, 0, 0, 206, 3032, 22, 0, 0, 0, 0, 0, 3745, 4, 2599, 1, 0, 0, 0, 0, 2213, 0, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5601, 2436, 5958, 2067, 4877, 3260, 6349, 2220] avg=4096.0000 max=6349 max_rank=6 min=2067 min_rank=3 imbalance_ratio=1.550049 max_load_reduction=26419 max_load_reduction_pct=0.806244 moved_to_echo=16706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14592, 6528, 0, 896, 6912, 4224, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 12544, 16128, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3712, 18944, 3840, 3072, 2304, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 5120, 4352, 11264, 12160, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2432, 17920, 4736, 256, 4736, 3072, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[26240, 128, 2432, 4352, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10112, 3968, 18688, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12160, 20224, 256, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True + [2026-08-30 04:55:21] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 40672.6 | throughput per GPU (TFLOP/s/GPU): 8.2 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102614E+01 | loss scale: 1.0 | grad norm: 44.312 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 23.22 + +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 63624.1650390625 | max allocated: 96252.23828125 | reserved: 99098.0 | max reserved: 99098.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[118, 2822, 825, 259, 2190, 718, 1007, 1493, 1156, 206, 1177, 479, 1045, 842, 197, 531, 187, 599, 1421, 1222, 991, 235, 2478, 1727, 472, 1324, 1210, 1299, 1003, 370, 1397, 1768] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[192, 2881, 501, 433, 1086, 374, 962, 685, 1260, 289, 1692, 398, 1097, 633, 343, 387, 156, 627, 812, 932, 1140, 577, 2962, 2138, 588, 1563, 1474, 925, 1543, 356, 1699, 2063] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[142, 2705, 768, 417, 1709, 536, 964, 645, 989, 148, 1512, 265, 1007, 993, 382, 456, 252, 714, 873, 1145, 1003, 426, 2990, 1792, 457, 955, 1501, 1097, 1218, 294, 1865, 2548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[206, 3389, 800, 318, 1593, 379, 705, 1438, 1107, 316, 2246, 316, 690, 903, 265, 414, 184, 451, 828, 1014, 1094, 215, 2785, 2265, 731, 1407, 1041, 1155, 860, 290, 1667, 1696] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[114, 3077, 525, 306, 1709, 242, 1270, 1263, 1246, 254, 1709, 214, 779, 852, 151, 378, 147, 945, 996, 825, 949, 118, 2884, 1664, 567, 1607, 835, 1227, 1703, 546, 1543, 2123] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[207, 2976, 646, 349, 1655, 686, 1035, 1622, 1320, 180, 1754, 199, 832, 1043, 355, 499, 222, 614, 668, 960, 1321, 242, 2954, 1661, 404, 1141, 1123, 1216, 1015, 309, 1589, 1971] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[104, 3005, 730, 337, 1474, 376, 947, 776, 1103, 274, 1291, 194, 1405, 700, 180, 418, 156, 616, 929, 790, 1466, 149, 3767, 2083, 278, 1419, 1139, 1107, 939, 299, 1767, 2550] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[56, 2739, 361, 191, 1648, 296, 1067, 894, 1160, 189, 2175, 283, 542, 908, 267, 260, 102, 534, 1182, 518, 1268, 221, 2968, 2099, 595, 1065, 1076, 961, 1784, 457, 2566, 2336] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[192, 2881, 501, 433, 28, 6, 0, 0, 1031, 374, 962, 685, 0, 0, 0, 0, 1260, 289, 1692, 398, 613, 49, 0, 0, 1097, 633, 343, 387, 1620, 0, 0, 0, 156, 627, 812, 932, 761, 603, 0, 0, 1140, 577, 739, 2138, 0, 0, 0, 0, 588, 1535, 1474, 925, 0, 0, 0, 0, 1543, 356, 1699, 689, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 3052, 4301, 4080, 3891, 4594, 4522, 4287] avg=4096.0000 max=4594 max_rank=5 min=3052 min_rank=1 imbalance_ratio=1.121582 max_load_reduction=28174 max_load_reduction_pct=0.859802 moved_to_echo=17294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[118, 2822, 825, 259, 24, 13, 0, 0, 2077, 718, 1007, 1493, 0, 0, 0, 0, 1156, 206, 1177, 479, 525, 100, 0, 0, 1045, 842, 197, 531, 1355, 0, 0, 0, 187, 599, 1421, 1222, 652, 505, 0, 0, 991, 235, 618, 1727, 0, 0, 0, 0, 472, 1300, 1210, 1299, 0, 0, 0, 0, 1003, 370, 1397, 591, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 5295, 3643, 3970, 4586, 3571, 4281, 3361] avg=4096.0000 max=5295 max_rank=1 min=3361 min_rank=7 imbalance_ratio=1.292725 max_load_reduction=27473 max_load_reduction_pct=0.838409 moved_to_echo=15799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[142, 2705, 768, 417, 17, 10, 0, 0, 1621, 536, 964, 645, 0, 0, 0, 0, 989, 148, 1512, 265, 757, 78, 0, 0, 1007, 993, 382, 456, 1635, 0, 0, 0, 252, 714, 873, 1145, 940, 609, 0, 0, 1003, 426, 746, 1792, 0, 0, 0, 0, 457, 938, 1501, 1097, 0, 0, 0, 0, 1218, 294, 1865, 851, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 3766, 3749, 4473, 4533, 3967, 3993, 4228] avg=4096.0000 max=4533 max_rank=4 min=3749 min_rank=2 imbalance_ratio=1.106689 max_load_reduction=28235 max_load_reduction_pct=0.861664 moved_to_echo=16721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[206, 3389, 800, 318, 25, 9, 0, 0, 1512, 379, 705, 1438, 0, 0, 0, 0, 1107, 316, 2246, 316, 504, 72, 0, 0, 690, 903, 265, 414, 1523, 0, 0, 0, 184, 451, 828, 1014, 626, 567, 0, 0, 1094, 215, 695, 2265, 0, 0, 0, 0, 731, 1382, 1041, 1155, 0, 0, 0, 0, 860, 290, 1667, 566, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4747, 4034, 4561, 3795, 3670, 4269, 4309, 3383] avg=4096.0000 max=4747 max_rank=0 min=3383 min_rank=7 imbalance_ratio=1.158936 max_load_reduction=28021 max_load_reduction_pct=0.855133 moved_to_echo=15631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[114, 3077, 525, 306, 29, 10, 0, 0, 1621, 242, 1270, 1263, 0, 0, 0, 0, 1246, 254, 1709, 214, 630, 78, 0, 0, 779, 852, 151, 378, 1577, 0, 0, 0, 147, 945, 996, 825, 783, 588, 0, 0, 949, 118, 719, 1664, 0, 0, 0, 0, 567, 1578, 835, 1227, 0, 0, 0, 0, 1703, 546, 1543, 710, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4396, 4131, 3737, 4284, 3450, 4207, 4502] avg=4096.0000 max=4502 max_rank=7 min=3450 min_rank=5 imbalance_ratio=1.099121 max_load_reduction=28266 max_load_reduction_pct=0.862610 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[104, 3005, 730, 337, 29, 12, 0, 0, 1391, 376, 947, 776, 0, 0, 0, 0, 1103, 274, 1291, 194, 761, 71, 0, 0, 1405, 700, 180, 418, 2065, 0, 0, 0, 156, 616, 929, 790, 946, 771, 0, 0, 1466, 149, 931, 2083, 0, 0, 0, 0, 278, 1390, 1139, 1107, 0, 0, 0, 0, 939, 299, 1767, 843, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4217, 3490, 3694, 4768, 4208, 4629, 3914, 3848] avg=4096.0000 max=4768 max_rank=3 min=3490 min_rank=1 imbalance_ratio=1.164062 max_load_reduction=28000 max_load_reduction_pct=0.854492 moved_to_echo=16599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[207, 2976, 646, 349, 20, 9, 0, 0, 1571, 686, 1035, 1622, 0, 0, 0, 0, 1320, 180, 1754, 199, 585, 75, 0, 0, 832, 1043, 355, 499, 1616, 0, 0, 0, 222, 614, 668, 960, 727, 602, 0, 0, 1321, 242, 736, 1661, 0, 0, 0, 0, 404, 1121, 1123, 1216, 0, 0, 0, 0, 1015, 309, 1589, 659, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4207, 4914, 4113, 4345, 3793, 3960, 3864, 3572] avg=4096.0000 max=4914 max_rank=1 min=3572 min_rank=7 imbalance_ratio=1.199707 max_load_reduction=27854 max_load_reduction_pct=0.850037 moved_to_echo=15189 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[56, 2739, 361, 191, 19, 9, 0, 0, 1564, 296, 1067, 894, 0, 0, 0, 0, 1160, 189, 2175, 283, 694, 75, 0, 0, 542, 908, 267, 260, 1623, 0, 0, 0, 102, 534, 1182, 518, 862, 605, 0, 0, 1268, 221, 740, 2099, 0, 0, 0, 0, 595, 1046, 1076, 961, 0, 0, 0, 0, 1784, 457, 2566, 780, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3375, 3821, 4576, 3600, 3803, 4328, 3678, 5587] avg=4096.0000 max=5587 max_rank=7 min=3375 min_rank=0 imbalance_ratio=1.364014 max_load_reduction=27181 max_load_reduction_pct=0.829498 moved_to_echo=17396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7424, 6912, 2176, 3456, 13056, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10112, 2944, 14208, 5760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1408, 5120, 7808, 7424, 6400, 4864, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9344, 2304, 6016, 15488, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 10368, 9472, 9088, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12416, 3712, 8064, 8832, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9344, 1920, 13568, 2432, 5120, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1152, 23680, 5248, 2688, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[72, 14, 3514, 182, 116, 2166, 349, 3439, 85, 14, 260, 3569, 0, 2999, 150, 446, 159, 456, 0, 4095, 260, 398, 6, 300, 740, 12, 669, 4079, 4090, 4, 114, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[723, 0, 1097, 17, 733, 861, 3355, 2229, 115, 28, 1021, 252, 8, 3102, 203, 947, 77, 2542, 125, 3751, 50, 1, 0, 48, 2826, 9, 412, 4094, 4094, 3, 19, 26] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[49, 1, 2214, 725, 1680, 2606, 1091, 3700, 30, 8, 2, 414, 689, 3905, 43, 527, 65, 33, 0, 4053, 1, 395, 2, 30, 1305, 332, 1343, 4058, 3462, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[65, 0, 2507, 80, 141, 3471, 418, 3250, 0, 0, 842, 2116, 0, 2294, 4, 1951, 72, 159, 0, 4095, 100, 0, 1, 287, 314, 262, 2155, 4093, 4090, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1992, 4, 84, 75, 3272, 3902, 2272, 384, 29, 0, 150, 332, 38, 1252, 439, 760, 53, 85, 0, 4082, 0, 3, 0, 13, 3088, 1429, 1203, 3688, 4092, 8, 31, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[236, 0, 3281, 89, 336, 3224, 2391, 3078, 14, 0, 22, 119, 31, 3069, 4, 804, 33, 670, 230, 4089, 4, 11, 1, 37, 605, 40, 2215, 4072, 4038, 0, 22, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[695, 538, 806, 177, 3609, 4031, 3, 3852, 0, 0, 26, 2562, 106, 3403, 0, 623, 1, 6, 184, 4010, 0, 0, 2, 36, 4, 26, 4, 3825, 4084, 0, 145, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[274, 0, 1734, 0, 2483, 3669, 2723, 70, 445, 0, 921, 662, 0, 2931, 99, 333, 88, 695, 0, 4041, 26, 0, 54, 44, 3430, 8, 958, 3485, 3538, 0, 39, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[65, 0, 2507, 80, 1423, 350, 0, 0, 141, 0, 418, 1268, 0, 0, 0, 0, 0, 0, 842, 2116, 1993, 559, 0, 0, 0, 2294, 4, 1951, 204, 0, 0, 0, 72, 159, 0, 3438, 0, 0, 0, 0, 100, 0, 1, 287, 3471, 877, 0, 0, 314, 262, 2155, 1223, 0, 0, 0, 0, 4090, 0, 0, 1, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[4425, 1827, 5510, 4453, 3669, 4736, 3954, 4194] avg=4096.0000 max=5510 max_rank=2 min=1827 min_rank=1 imbalance_ratio=1.345215 max_load_reduction=27258 max_load_reduction_pct=0.831848 moved_to_echo=16553 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1992, 4, 84, 75, 168, 349, 0, 0, 3272, 0, 2272, 150, 0, 0, 0, 0, 29, 0, 150, 332, 1795, 66, 0, 0, 38, 1252, 439, 760, 203, 0, 0, 0, 53, 85, 0, 3427, 0, 0, 0, 0, 0, 3, 0, 13, 3902, 790, 0, 0, 3088, 1429, 1203, 1103, 0, 0, 0, 0, 4092, 8, 31, 8, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[2672, 5694, 2372, 2692, 3565, 4708, 6823, 4242] avg=4096.0000 max=6823 max_rank=6 min=2372 min_rank=2 imbalance_ratio=1.665771 max_load_reduction=25945 max_load_reduction_pct=0.791779 moved_to_echo=19338 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[274, 0, 1734, 0, 30, 346, 0, 0, 2483, 0, 2723, 28, 0, 0, 0, 0, 445, 0, 921, 662, 1696, 12, 0, 0, 0, 2931, 99, 333, 201, 0, 0, 0, 88, 695, 0, 3393, 0, 0, 0, 0, 26, 0, 54, 44, 3669, 746, 0, 0, 3430, 8, 958, 1043, 0, 0, 0, 0, 3538, 0, 39, 18, 101, 0, 0, 0] rerouted_tokens_per_ep_rank=[2384, 5234, 3736, 3564, 4176, 4539, 5439, 3696] avg=4096.0000 max=5439 max_rank=6 min=2384 min_rank=0 imbalance_ratio=1.327881 max_load_reduction=27329 max_load_reduction_pct=0.834015 moved_to_echo=17850 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[72, 14, 3514, 182, 1506, 350, 0, 0, 116, 0, 349, 1342, 0, 0, 0, 0, 85, 14, 260, 3569, 1986, 591, 0, 0, 0, 2999, 150, 446, 204, 0, 0, 0, 159, 456, 0, 3438, 0, 0, 0, 0, 260, 398, 6, 300, 2166, 874, 0, 0, 740, 12, 669, 1219, 0, 0, 0, 0, 4090, 4, 114, 11, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[5638, 1807, 6505, 3799, 4053, 4004, 2640, 4322] avg=4096.0000 max=6505 max_rank=2 min=1807 min_rank=1 imbalance_ratio=1.588135 max_load_reduction=26263 max_load_reduction_pct=0.801483 moved_to_echo=15019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[723, 0, 1097, 17, 976, 321, 0, 0, 733, 0, 3355, 870, 0, 0, 0, 0, 115, 28, 1021, 252, 1993, 383, 0, 0, 8, 3102, 203, 947, 187, 0, 0, 0, 77, 2542, 125, 3149, 0, 0, 0, 0, 50, 1, 0, 48, 861, 877, 0, 0, 2826, 9, 412, 1224, 0, 0, 0, 0, 4094, 3, 19, 26, 94, 0, 0, 0] rerouted_tokens_per_ep_rank=[3134, 4958, 3792, 4447, 5893, 1837, 4471, 4236] avg=4096.0000 max=5893 max_rank=4 min=1837 min_rank=5 imbalance_ratio=1.438721 max_load_reduction=26875 max_load_reduction_pct=0.820160 moved_to_echo=16437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[49, 1, 2214, 725, 1625, 351, 0, 0, 1680, 0, 1091, 1435, 0, 0, 0, 0, 30, 8, 2, 414, 1980, 640, 0, 0, 689, 3905, 43, 527, 205, 0, 0, 0, 65, 33, 0, 3392, 0, 0, 0, 0, 1, 395, 2, 30, 2606, 873, 0, 0, 1305, 332, 1343, 1205, 0, 0, 0, 0, 3462, 0, 5, 0, 105, 0, 0, 0] rerouted_tokens_per_ep_rank=[4965, 4206, 3074, 5369, 3490, 3907, 4185, 3572] avg=4096.0000 max=5369 max_rank=3 min=3074 min_rank=2 imbalance_ratio=1.310791 max_load_reduction=27399 max_load_reduction_pct=0.836151 moved_to_echo=15154 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[236, 0, 3281, 89, 1348, 350, 0, 0, 336, 0, 2391, 1201, 0, 0, 0, 0, 14, 0, 22, 119, 1982, 529, 0, 0, 31, 3069, 4, 804, 204, 0, 0, 0, 33, 670, 230, 3432, 0, 0, 0, 0, 4, 11, 1, 37, 3224, 872, 0, 0, 605, 40, 2215, 1218, 0, 0, 0, 0, 4038, 0, 22, 3, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[5304, 3928, 2666, 4112, 4365, 4149, 4078, 4166] avg=4096.0000 max=5304 max_rank=0 min=2666 min_rank=2 imbalance_ratio=1.294922 max_load_reduction=27464 max_load_reduction_pct=0.838135 moved_to_echo=16758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[695, 538, 806, 177, 1687, 343, 0, 0, 3609, 0, 3, 1502, 0, 0, 0, 0, 0, 0, 26, 2562, 1862, 663, 0, 0, 106, 3403, 0, 623, 200, 0, 0, 0, 1, 6, 184, 3366, 0, 0, 0, 0, 0, 0, 2, 36, 4031, 819, 0, 0, 4, 26, 4, 1144, 0, 0, 0, 0, 4084, 0, 145, 10, 101, 0, 0, 0] rerouted_tokens_per_ep_rank=[4246, 5114, 5113, 4332, 3557, 4888, 1178, 4340] avg=4096.0000 max=5114 max_rank=1 min=1178 min_rank=6 imbalance_ratio=1.248535 max_load_reduction=27654 max_load_reduction_pct=0.843933 moved_to_echo=13963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 640, 15360, 1408, 8832, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[12416, 0, 12672, 7808, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 128, 3328, 10112, 15360, 3456, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 896, 128, 896, 23936, 6784, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12416, 2176, 8960, 9472, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[896, 23040, 1024, 6400, 1664, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 4736, 640, 27136, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31488, 128, 384, 128, 896, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1817, 3661, 12, 4095, 9, 2317, 813, 0, 0, 4082, 323, 0, 13, 68, 4, 0, 0, 4092, 3878, 0, 0, 4090, 8, 23, 3317, 0, 0, 1, 142, 1, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[198, 2883, 1155, 3907, 7, 31, 4095, 0, 5, 3, 3974, 2, 117, 3519, 1, 0, 1, 4095, 821, 0, 1, 235, 11, 3048, 2683, 0, 0, 2, 1590, 0, 330, 54] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[40, 2985, 264, 4005, 33, 0, 4096, 0, 0, 73, 319, 2, 5, 4065, 321, 6, 0, 4093, 135, 0, 96, 57, 159, 4088, 705, 26, 1776, 2, 4042, 0, 142, 1233] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1864, 762, 51, 2741, 0, 0, 4057, 126, 0, 4040, 1014, 1, 1461, 3453, 200, 38, 3, 4079, 1169, 3, 0, 3313, 0, 504, 3603, 6, 2, 2, 126, 1, 148, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1315, 1029, 249, 3135, 419, 2, 4046, 0, 71, 0, 477, 0, 0, 3737, 154, 2658, 30, 4093, 949, 2, 1746, 54, 0, 1982, 2356, 3, 291, 670, 3168, 3, 119, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[103, 3035, 484, 2514, 3, 71, 4016, 0, 0, 3, 2538, 1, 59, 3753, 2323, 504, 1, 4073, 1051, 0, 473, 0, 0, 3795, 168, 1, 0, 0, 3793, 1, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[2529, 1258, 164, 1373, 2, 2665, 3692, 0, 16, 0, 62, 0, 519, 216, 10, 0, 1, 4045, 1720, 0, 0, 2054, 1298, 61, 4029, 0, 19, 1702, 4095, 0, 2, 1236] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4086, 235, 11, 81, 58, 0, 3384, 86, 0, 3532, 3076, 13, 10, 4014, 0, 539, 0, 4096, 449, 3977, 41, 5, 0, 28, 12, 0, 0, 2105, 2804, 125, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1817, 3661, 12, 485, 0, 0, 0, 0, 9, 2317, 778, 0, 0, 0, 0, 0, 0, 4082, 323, 0, 1095, 73, 0, 0, 13, 68, 4, 0, 28, 0, 0, 0, 0, 2329, 3878, 0, 0, 0, 0, 0, 0, 4090, 8, 23, 668, 7, 0, 0, 3317, 0, 0, 1, 1740, 0, 0, 0, 142, 1, 0, 2, 1797, 0, 0, 0] rerouted_tokens_per_ep_rank=[5975, 3104, 5573, 113, 6207, 4796, 5058, 1942] avg=4096.0000 max=6207 max_rank=4 min=113 min_rank=3 imbalance_ratio=1.515381 max_load_reduction=26561 max_load_reduction_pct=0.810577 moved_to_echo=18003 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[198, 2883, 1155, 462, 0, 0, 0, 0, 7, 31, 3913, 0, 0, 0, 0, 0, 5, 3, 3974, 2, 1096, 70, 0, 0, 117, 3519, 1, 0, 145, 0, 0, 0, 1, 2330, 821, 0, 0, 0, 0, 0, 1, 235, 11, 3048, 669, 37, 0, 0, 2683, 0, 0, 2, 1660, 0, 0, 0, 1590, 0, 330, 54, 1715, 0, 0, 0] rerouted_tokens_per_ep_rank=[4698, 3951, 5150, 3782, 3152, 4001, 4345, 3689] avg=4096.0000 max=5150 max_rank=2 min=3152 min_rank=4 imbalance_ratio=1.257324 max_load_reduction=27618 max_load_reduction_pct=0.842834 moved_to_echo=15187 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1864, 762, 51, 325, 0, 0, 0, 0, 0, 0, 3876, 126, 0, 0, 0, 0, 0, 4040, 1014, 1, 1092, 49, 0, 0, 1461, 3453, 200, 38, 144, 0, 0, 0, 3, 2321, 1169, 3, 0, 0, 0, 0, 0, 3313, 0, 504, 666, 37, 0, 0, 3603, 6, 2, 2, 1164, 0, 0, 0, 126, 1, 148, 1, 1203, 0, 0, 0] rerouted_tokens_per_ep_rank=[3002, 4002, 6196, 5296, 3496, 4520, 4777, 1479] avg=4096.0000 max=6196 max_rank=2 min=1479 min_rank=7 imbalance_ratio=1.512695 max_load_reduction=26572 max_load_reduction_pct=0.810913 moved_to_echo=14272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[40, 2985, 264, 474, 0, 0, 0, 0, 33, 0, 3914, 0, 0, 0, 0, 0, 0, 73, 319, 2, 1096, 71, 0, 0, 5, 4065, 321, 6, 145, 0, 0, 0, 0, 2329, 135, 0, 0, 0, 0, 0, 96, 57, 159, 4088, 668, 37, 0, 0, 705, 26, 1776, 2, 1702, 0, 0, 0, 4042, 0, 142, 1233, 1758, 0, 0, 0] rerouted_tokens_per_ep_rank=[3763, 3947, 1561, 4542, 2464, 5105, 4211, 7175] avg=4096.0000 max=7175 max_rank=7 min=1561 min_rank=2 imbalance_ratio=1.751709 max_load_reduction=25593 max_load_reduction_pct=0.781036 moved_to_echo=18955 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1315, 1029, 249, 371, 0, 0, 0, 0, 419, 2, 3866, 0, 0, 0, 0, 0, 71, 0, 477, 0, 1096, 56, 0, 0, 0, 3737, 154, 2658, 143, 0, 0, 0, 30, 2329, 949, 2, 0, 0, 0, 0, 1746, 54, 0, 1982, 668, 37, 0, 0, 2356, 3, 291, 670, 1332, 0, 0, 0, 3168, 3, 119, 10, 1376, 0, 0, 0] rerouted_tokens_per_ep_rank=[2964, 4287, 1700, 6692, 3310, 4487, 4652, 4676] avg=4096.0000 max=6692 max_rank=3 min=1700 min_rank=2 imbalance_ratio=1.633789 max_load_reduction=26076 max_load_reduction_pct=0.795776 moved_to_echo=17125 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[103, 3035, 484, 298, 0, 0, 0, 0, 3, 71, 3838, 0, 0, 0, 0, 0, 0, 3, 2538, 1, 1090, 45, 0, 0, 59, 3753, 2323, 504, 142, 0, 0, 0, 1, 2318, 1051, 0, 0, 0, 0, 0, 473, 0, 0, 3795, 665, 36, 0, 0, 168, 1, 0, 0, 1068, 0, 0, 0, 3793, 1, 0, 5, 1103, 0, 0, 0] rerouted_tokens_per_ep_rank=[3920, 3912, 3677, 6781, 3370, 4969, 1237, 4902] avg=4096.0000 max=6781 max_rank=3 min=1237 min_rank=6 imbalance_ratio=1.655518 max_load_reduction=25987 max_load_reduction_pct=0.793060 moved_to_echo=14478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2529, 1258, 164, 152, 0, 0, 0, 0, 2, 2665, 3521, 0, 0, 0, 0, 0, 16, 0, 62, 0, 1087, 27, 0, 0, 519, 216, 10, 0, 134, 0, 0, 0, 1, 2292, 1720, 0, 0, 0, 0, 0, 0, 2054, 1298, 61, 666, 37, 0, 0, 4029, 0, 19, 1702, 587, 0, 0, 0, 4095, 0, 2, 1236, 607, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 6188, 1192, 879, 4013, 4116, 6337, 5940] avg=4096.0000 max=6337 max_rank=6 min=879 min_rank=3 imbalance_ratio=1.547119 max_load_reduction=26431 max_load_reduction_pct=0.806610 moved_to_echo=20406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4086, 235, 11, 11, 0, 0, 0, 0, 58, 0, 3233, 86, 0, 0, 0, 0, 0, 3532, 3076, 13, 1097, 1, 0, 0, 10, 4014, 0, 539, 120, 0, 0, 0, 0, 2330, 449, 3977, 0, 0, 0, 0, 41, 5, 0, 28, 669, 31, 0, 0, 12, 0, 0, 2105, 34, 0, 0, 0, 2804, 125, 0, 1, 35, 0, 0, 0] rerouted_tokens_per_ep_rank=[4343, 3377, 7719, 4683, 6756, 774, 2151, 2965] avg=4096.0000 max=7719 max_rank=2 min=774 min_rank=5 imbalance_ratio=1.884521 max_load_reduction=25049 max_load_reduction_pct=0.764435 moved_to_echo=12646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 11776, 11904, 128, 8832, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[19840, 256, 768, 2560, 9600, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 18688, 10240, 4096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12032, 15872, 2432, 2688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16896, 128, 2176, 4608, 9344, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2304, 22912, 3072, 3840, 1024, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2432, 9856, 1536, 13568, 5376, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[640, 5120, 27008, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[648, 78, 0, 4081, 2170, 123, 0, 0, 0, 4091, 2, 49, 3959, 0, 1393, 986, 0, 2045, 112, 3483, 3, 0, 0, 4094, 19, 17, 9, 1341, 12, 1, 42, 4010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[93, 135, 0, 3525, 2233, 0, 2905, 0, 0, 14, 106, 86, 3308, 0, 3399, 569, 430, 3, 1565, 0, 0, 0, 0, 4017, 7, 1998, 2, 4087, 0, 5, 235, 4046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[31, 1660, 0, 4096, 3, 0, 1326, 0, 0, 1987, 5, 122, 3698, 3, 41, 19, 0, 4054, 1092, 3983, 2296, 0, 0, 3472, 190, 1, 29, 1271, 1, 0, 164, 3224] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[3302, 4, 3, 606, 76, 338, 3, 0, 3892, 4061, 1330, 16, 4036, 0, 924, 3664, 20, 2694, 4, 3966, 0, 0, 0, 826, 2, 1361, 1188, 64, 237, 23, 1, 127] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[3989, 49, 0, 3562, 3086, 0, 460, 0, 1, 4013, 0, 363, 3836, 1, 29, 258, 0, 129, 11, 1344, 977, 0, 0, 4027, 188, 14, 108, 3323, 10, 0, 374, 2616] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[3917, 53, 0, 3672, 4067, 26, 2747, 0, 0, 1199, 0, 18, 273, 232, 3587, 167, 177, 310, 593, 569, 0, 0, 252, 1898, 638, 706, 0, 3627, 0, 1, 4, 4035] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[2408, 0, 1, 743, 4095, 1142, 0, 0, 11, 2302, 17, 38, 4096, 0, 123, 1, 0, 2830, 8, 3595, 207, 0, 0, 3942, 12, 505, 0, 695, 2375, 0, 1579, 2043] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[773, 956, 3, 3298, 3973, 1496, 32, 16, 405, 347, 40, 2, 359, 317, 3229, 15, 1464, 0, 789, 24, 0, 48, 0, 4077, 4023, 63, 0, 2914, 0, 0, 234, 3871] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[648, 78, 0, 2539, 0, 0, 0, 0, 2170, 123, 0, 0, 503, 0, 0, 0, 0, 4091, 2, 49, 1386, 0, 0, 0, 2321, 0, 1393, 986, 0, 0, 0, 0, 0, 2045, 112, 2965, 0, 0, 0, 0, 3, 0, 0, 4094, 442, 15, 0, 0, 19, 17, 9, 1341, 495, 252, 0, 0, 12, 1, 42, 4010, 605, 0, 0, 0] rerouted_tokens_per_ep_rank=[3265, 2796, 5528, 4700, 5122, 4554, 2133, 4670] avg=4096.0000 max=5528 max_rank=2 min=2133 min_rank=6 imbalance_ratio=1.349609 max_load_reduction=27240 max_load_reduction_pct=0.831299 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3302, 4, 3, 379, 0, 0, 0, 0, 76, 338, 3, 0, 573, 0, 0, 0, 3892, 4061, 1330, 16, 1413, 0, 0, 0, 2366, 0, 924, 3664, 0, 0, 0, 0, 20, 2694, 4, 3376, 0, 0, 0, 0, 0, 0, 0, 826, 65, 17, 0, 0, 2, 1361, 1188, 64, 73, 257, 0, 0, 237, 23, 1, 127, 89, 0, 0, 0] rerouted_tokens_per_ep_rank=[3688, 990, 10712, 6954, 6094, 908, 2945, 477] avg=4096.0000 max=10712 max_rank=2 min=477 min_rank=7 imbalance_ratio=2.615234 max_load_reduction=22056 max_load_reduction_pct=0.673096 moved_to_echo=10424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3989, 49, 0, 2216, 0, 0, 0, 0, 3086, 0, 460, 0, 194, 0, 0, 0, 1, 4013, 0, 363, 1343, 0, 0, 0, 2249, 1, 29, 258, 0, 0, 0, 0, 0, 129, 11, 1145, 0, 0, 0, 0, 977, 0, 0, 4027, 386, 5, 0, 0, 188, 14, 108, 3323, 432, 244, 0, 0, 10, 0, 374, 2616, 528, 0, 0, 0] rerouted_tokens_per_ep_rank=[6254, 3740, 5720, 2537, 1285, 5395, 4309, 3528] avg=4096.0000 max=6254 max_rank=0 min=1285 min_rank=4 imbalance_ratio=1.526855 max_load_reduction=26514 max_load_reduction_pct=0.809143 moved_to_echo=14517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[93, 135, 0, 2194, 0, 0, 0, 0, 2233, 0, 2905, 0, 0, 0, 0, 0, 0, 14, 106, 86, 1158, 0, 0, 0, 1940, 0, 3399, 569, 0, 0, 0, 0, 430, 3, 1565, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 382, 0, 0, 0, 7, 1998, 2, 4087, 427, 210, 0, 0, 0, 5, 235, 4046, 522, 0, 0, 0] rerouted_tokens_per_ep_rank=[2422, 5138, 1364, 5908, 1998, 4399, 6731, 4808] avg=4096.0000 max=6731 max_rank=6 min=1364 min_rank=2 imbalance_ratio=1.643311 max_load_reduction=26037 max_load_reduction_pct=0.794586 moved_to_echo=17936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[31, 1660, 0, 2548, 0, 0, 0, 0, 3, 0, 1326, 0, 575, 0, 0, 0, 0, 1987, 5, 122, 1294, 0, 0, 0, 2169, 3, 41, 19, 0, 0, 0, 0, 0, 4054, 1092, 3391, 0, 0, 0, 0, 2296, 0, 0, 3472, 444, 17, 0, 0, 190, 1, 29, 1271, 497, 235, 0, 0, 1, 0, 164, 3224, 607, 0, 0, 0] rerouted_tokens_per_ep_rank=[4239, 1904, 3408, 2232, 8537, 6229, 2223, 3996] avg=4096.0000 max=8537 max_rank=4 min=1904 min_rank=1 imbalance_ratio=2.084229 max_load_reduction=24231 max_load_reduction_pct=0.739471 moved_to_echo=20985 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3917, 53, 0, 2274, 0, 0, 0, 0, 4067, 26, 2747, 0, 84, 0, 0, 0, 0, 1199, 0, 18, 98, 0, 0, 0, 153, 232, 3587, 167, 0, 0, 0, 0, 177, 310, 593, 479, 0, 0, 0, 0, 0, 0, 252, 1898, 401, 6, 0, 0, 638, 706, 0, 3627, 449, 22, 0, 0, 0, 1, 4, 4035, 548, 0, 0, 0] rerouted_tokens_per_ep_rank=[6244, 6924, 1315, 4139, 1559, 2557, 5442, 4588] avg=4096.0000 max=6924 max_rank=1 min=1315 min_rank=2 imbalance_ratio=1.690430 max_load_reduction=25844 max_load_reduction_pct=0.788696 moved_to_echo=14146 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2408, 0, 1, 463, 0, 0, 0, 0, 4095, 1142, 0, 0, 519, 0, 0, 0, 11, 2302, 17, 38, 1434, 0, 0, 0, 2401, 0, 123, 1, 0, 0, 0, 0, 0, 2830, 8, 3061, 0, 0, 0, 0, 207, 0, 0, 3942, 80, 15, 0, 0, 12, 505, 0, 695, 90, 261, 0, 0, 2375, 0, 1579, 2043, 110, 0, 0, 0] rerouted_tokens_per_ep_rank=[2872, 5756, 3802, 2525, 5899, 4244, 1563, 6107] avg=4096.0000 max=6107 max_rank=7 min=1563 min_rank=6 imbalance_ratio=1.490967 max_load_reduction=26661 max_load_reduction_pct=0.813629 moved_to_echo=17813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[773, 956, 3, 2052, 0, 0, 0, 0, 3973, 1496, 32, 16, 3, 0, 0, 0, 405, 347, 40, 2, 125, 0, 0, 0, 212, 317, 3229, 15, 0, 0, 0, 0, 1464, 0, 789, 21, 0, 0, 0, 0, 0, 48, 0, 4077, 357, 0, 0, 0, 4023, 63, 0, 2914, 400, 22, 0, 0, 0, 0, 234, 3871, 489, 0, 0, 0] rerouted_tokens_per_ep_rank=[3784, 5520, 919, 3773, 2274, 4482, 7422, 4594] avg=4096.0000 max=7422 max_rank=6 min=919 min_rank=2 imbalance_ratio=1.812012 max_load_reduction=25346 max_load_reduction_pct=0.773499 moved_to_echo=18772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2176, 12160, 4224, 14464, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4352, 18048, 1536, 768, 8320, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19712, 3200, 7552, 128, 2560, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15232, 2944, 128, 14720, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5120, 4736, 1408, 17408, 2944, 1536, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13824, 640, 12800, 5760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2688, 128, 2688, 24064, 3584, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3584, 128, 256, 26368, 2560, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1108, 0, 2368, 0, 194, 0, 13, 103, 3847, 0, 4093, 53, 35, 608, 980, 125, 1, 0, 3853, 0, 4092, 0, 2263, 1158, 0, 2808, 32, 0, 0, 2598, 2436, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4073, 0, 4082, 0, 3, 246, 0, 231, 52, 0, 3604, 3942, 2930, 2, 0, 20, 227, 0, 167, 0, 26, 0, 0, 3043, 35, 931, 0, 2055, 0, 3266, 3833, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1003, 0, 4096, 2, 1296, 0, 0, 3996, 765, 2, 336, 4020, 2855, 33, 3, 1, 631, 0, 0, 0, 0, 0, 16, 4020, 0, 613, 1469, 64, 0, 3447, 4096, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[2464, 1, 1449, 304, 0, 3, 1197, 2563, 3933, 1, 1313, 4083, 827, 550, 10, 106, 130, 0, 238, 2, 2092, 242, 0, 3353, 1, 25, 5, 0, 4, 3813, 4040, 19] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4082, 0, 2969, 158, 1499, 2599, 24, 0, 1472, 5, 423, 4057, 3694, 1161, 256, 0, 2166, 0, 18, 1, 0, 38, 0, 3, 1, 2046, 2014, 0, 0, 4, 4078, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[2929, 0, 30, 8, 16, 25, 568, 2000, 3855, 3, 3117, 4082, 1610, 9, 3264, 5, 3, 0, 0, 0, 0, 998, 1516, 35, 0, 1774, 211, 39, 0, 2561, 4093, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1469, 0, 4095, 0, 158, 0, 206, 4095, 4096, 0, 68, 3823, 3687, 14, 0, 968, 0, 0, 603, 0, 199, 0, 2516, 2978, 22, 946, 1912, 11, 0, 40, 803, 59] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3508, 0, 2836, 0, 4096, 74, 27, 4083, 53, 0, 90, 3312, 12, 4039, 559, 0, 1248, 0, 0, 0, 346, 122, 0, 16, 66, 177, 155, 3, 0, 3884, 4062, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1108, 0, 1261, 0, 0, 0, 0, 0, 194, 0, 13, 103, 372, 0, 0, 0, 3847, 0, 4093, 4, 0, 0, 0, 0, 35, 608, 980, 125, 336, 114, 0, 0, 1, 0, 3853, 0, 45, 0, 0, 0, 4092, 0, 2263, 1158, 399, 0, 0, 0, 0, 2808, 32, 0, 1163, 4, 0, 0, 0, 2598, 1159, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2369, 682, 7944, 2198, 3899, 7912, 4007, 3757] avg=4096.0000 max=7944 max_rank=2 min=682 min_rank=1 imbalance_ratio=1.939453 max_load_reduction=24824 max_load_reduction_pct=0.757568 moved_to_echo=19575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1003, 0, 2179, 2, 0, 0, 0, 0, 1296, 0, 0, 3996, 645, 0, 0, 0, 765, 2, 336, 242, 0, 0, 0, 0, 2855, 33, 3, 1, 582, 192, 0, 0, 631, 0, 0, 0, 3448, 0, 0, 0, 0, 0, 16, 4020, 690, 0, 0, 0, 0, 613, 1469, 64, 1955, 330, 0, 0, 0, 3447, 1949, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3184, 5937, 1345, 3666, 4079, 4726, 4431, 5400] avg=4096.0000 max=5937 max_rank=1 min=1345 min_rank=2 imbalance_ratio=1.449463 max_load_reduction=26831 max_load_reduction_pct=0.818817 moved_to_echo=18636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 2172, 0, 0, 0, 0, 0, 3, 246, 0, 231, 642, 0, 0, 0, 52, 0, 3604, 237, 0, 0, 0, 0, 2930, 2, 0, 20, 580, 179, 0, 0, 227, 0, 167, 0, 3381, 0, 0, 0, 26, 0, 0, 3043, 688, 0, 0, 0, 35, 931, 0, 2055, 1829, 324, 0, 0, 0, 3266, 1825, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6245, 1122, 3893, 3711, 3775, 3757, 5174, 5091] avg=4096.0000 max=6245 max_rank=0 min=1122 min_rank=1 imbalance_ratio=1.524658 max_load_reduction=26523 max_load_reduction_pct=0.809418 moved_to_echo=17797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4082, 0, 1580, 158, 0, 0, 0, 0, 1499, 2599, 24, 0, 467, 0, 0, 0, 1472, 5, 423, 244, 0, 0, 0, 0, 3694, 1161, 256, 0, 422, 191, 0, 0, 2166, 0, 18, 1, 3480, 0, 0, 0, 0, 38, 0, 3, 500, 0, 0, 0, 1, 2046, 2014, 0, 1946, 333, 0, 0, 0, 4, 1941, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5820, 4589, 2144, 5724, 5665, 541, 6340, 1945] avg=4096.0000 max=6340 max_rank=6 min=541 min_rank=5 imbalance_ratio=1.547852 max_load_reduction=26428 max_load_reduction_pct=0.806519 moved_to_echo=14491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2464, 1, 772, 304, 0, 0, 0, 0, 0, 3, 1197, 2563, 228, 0, 0, 0, 3933, 1, 1313, 246, 0, 0, 0, 0, 827, 550, 10, 106, 205, 189, 0, 0, 130, 0, 238, 2, 3502, 0, 0, 0, 2092, 242, 0, 3353, 244, 0, 0, 0, 1, 25, 5, 0, 1928, 335, 0, 0, 4, 3813, 1923, 19, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3541, 3991, 5493, 1887, 3872, 5931, 2294, 5759] avg=4096.0000 max=5931 max_rank=5 min=1887 min_rank=3 imbalance_ratio=1.447998 max_load_reduction=26837 max_load_reduction_pct=0.819000 moved_to_echo=17856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1469, 0, 2169, 0, 0, 0, 0, 0, 158, 0, 206, 4095, 649, 0, 0, 0, 4096, 0, 68, 222, 0, 0, 0, 0, 3687, 14, 0, 968, 585, 41, 0, 0, 0, 0, 603, 0, 3282, 0, 0, 0, 199, 0, 2516, 2978, 692, 0, 0, 0, 22, 946, 1912, 11, 387, 319, 0, 0, 0, 40, 375, 59, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3638, 5108, 4386, 5295, 3885, 6385, 3597, 474] avg=4096.0000 max=6385 max_rank=5 min=474 min_rank=7 imbalance_ratio=1.558838 max_load_reduction=26383 max_load_reduction_pct=0.805145 moved_to_echo=14341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2929, 0, 17, 8, 0, 0, 0, 0, 16, 25, 568, 2000, 4, 0, 0, 0, 3855, 3, 3117, 246, 0, 0, 0, 0, 1610, 9, 3264, 5, 4, 192, 0, 0, 3, 0, 0, 0, 3501, 0, 0, 0, 0, 998, 1516, 35, 5, 0, 0, 0, 0, 1774, 211, 39, 1954, 335, 0, 0, 0, 2561, 1947, 17, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2954, 2613, 7221, 5084, 3504, 2554, 4313, 4525] avg=4096.0000 max=7221 max_rank=2 min=2554 min_rank=5 imbalance_ratio=1.762939 max_load_reduction=25547 max_load_reduction_pct=0.779633 moved_to_echo=14896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3508, 0, 1509, 0, 0, 0, 0, 0, 4096, 74, 27, 4083, 446, 0, 0, 0, 53, 0, 90, 199, 0, 0, 0, 0, 12, 4039, 559, 0, 403, 190, 0, 0, 1248, 0, 0, 0, 2841, 0, 0, 0, 346, 122, 0, 16, 478, 0, 0, 0, 66, 177, 155, 3, 1939, 272, 0, 0, 0, 3884, 1933, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5017, 8726, 342, 5203, 4089, 962, 2612, 5817] avg=4096.0000 max=8726 max_rank=1 min=342 min_rank=2 imbalance_ratio=2.130371 max_load_reduction=24042 max_load_reduction_pct=0.733704 moved_to_echo=13480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6784, 1408, 6400, 14720, 3712, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[20736, 128, 11776, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7296, 3072, 2048, 17152, 3456, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[18176, 128, 13056, 1664, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 9344, 5888, 2176, 13184, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 19712, 13056, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15744, 6528, 5120, 1280, 3200, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4480, 0, 4992, 128, 23552, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 1063, 1, 0, 0, 4095, 74, 0, 88, 12, 2859, 0, 4096, 4079, 0, 4094, 14, 0, 0, 0, 12, 4, 0, 0, 4096, 1, 4095, 246, 3838, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 7, 1340, 2131, 0, 84, 7, 3815, 32, 0, 4081, 4096, 0, 4094, 5, 1227, 2902, 0, 0, 0, 139, 0, 126, 0, 929, 164, 9, 4092, 7, 3481, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 491, 49, 0, 47, 2, 101, 57, 3789, 4048, 4089, 0, 3465, 3195, 0, 0, 68, 0, 0, 640, 0, 0, 41, 3935, 4096, 0, 4069, 227, 358, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 54, 326, 26, 0, 0, 0, 9, 2, 0, 0, 4096, 0, 3761, 4096, 0, 4073, 0, 0, 0, 7, 362, 0, 322, 3004, 60, 398, 3912, 281, 4051, 0, 3928] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[452, 159, 1573, 94, 42, 0, 2681, 3565, 9, 0, 3504, 4012, 0, 122, 204, 0, 328, 527, 0, 1252, 0, 864, 183, 2135, 3737, 15, 304, 3793, 0, 3178, 0, 35] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[8, 2782, 1785, 1, 4, 0, 15, 281, 362, 113, 74, 3337, 4, 4090, 4096, 0, 2019, 11, 38, 7, 1260, 1, 0, 0, 562, 4078, 1, 3725, 0, 4029, 18, 67] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3259, 163, 0, 0, 2061, 0, 1416, 83, 0, 2354, 0, 4096, 3886, 0, 10, 6, 0, 1050, 0, 3913, 3, 0, 32, 3990, 611, 2952, 0, 2690, 1, 192] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[384, 90, 1, 1411, 0, 46, 3863, 1356, 3070, 416, 1643, 3502, 97, 4070, 361, 238, 0, 163, 0, 0, 3234, 6, 0, 17, 749, 621, 1410, 2922, 29, 1, 1722, 1346] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1063, 1, 1233, 425, 0, 0, 0, 0, 4095, 74, 1565, 0, 0, 0, 0, 88, 12, 1007, 0, 0, 0, 0, 0, 1663, 4079, 0, 0, 0, 0, 0, 4094, 14, 0, 0, 1394, 619, 0, 0, 0, 12, 4, 0, 2701, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 246, 3838, 0, 1, 443, 8, 0, 0] rerouted_tokens_per_ep_rank=[2722, 5734, 1107, 5742, 6121, 2717, 4089, 4536] avg=4096.0000 max=6121 max_rank=4 min=1107 min_rank=2 imbalance_ratio=1.494385 max_load_reduction=26647 max_load_reduction_pct=0.813202 moved_to_echo=17463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 1340, 2131, 1767, 425, 0, 0, 0, 84, 7, 3815, 1564, 0, 0, 0, 32, 0, 4081, 1442, 0, 0, 0, 0, 0, 1663, 5, 1227, 0, 0, 0, 0, 2902, 0, 0, 0, 1393, 887, 0, 0, 139, 0, 126, 0, 2699, 0, 0, 0, 929, 164, 9, 0, 0, 0, 0, 0, 7, 3481, 0, 0, 442, 0, 0, 0] rerouted_tokens_per_ep_rank=[5670, 5470, 5555, 2895, 5182, 2964, 1102, 3930] avg=4096.0000 max=5670 max_rank=0 min=1102 min_rank=6 imbalance_ratio=1.384277 max_load_reduction=27098 max_load_reduction_pct=0.826965 moved_to_echo=13178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[8, 2782, 1785, 1, 1439, 424, 0, 0, 4, 0, 15, 281, 1562, 0, 0, 0, 362, 113, 74, 1175, 0, 0, 0, 0, 4, 1662, 4096, 0, 0, 0, 0, 0, 2019, 11, 38, 7, 1267, 723, 0, 0, 1260, 1, 0, 0, 2458, 0, 0, 0, 562, 4070, 1, 0, 0, 0, 0, 0, 0, 4029, 18, 67, 442, 8, 0, 0] rerouted_tokens_per_ep_rank=[6439, 1862, 1724, 5762, 4065, 3719, 4633, 4564] avg=4096.0000 max=6439 max_rank=0 min=1724 min_rank=2 imbalance_ratio=1.572021 max_load_reduction=26329 max_load_reduction_pct=0.803497 moved_to_echo=16981 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 491, 49, 1764, 359, 0, 0, 0, 47, 2, 101, 1323, 0, 0, 0, 57, 3789, 4048, 1439, 0, 0, 0, 0, 0, 1409, 3195, 0, 0, 0, 0, 0, 0, 68, 0, 0, 1385, 886, 0, 0, 640, 0, 0, 41, 2684, 0, 0, 0, 3935, 4088, 0, 0, 0, 0, 0, 0, 227, 358, 0, 1, 374, 8, 0, 0] rerouted_tokens_per_ep_rank=[2663, 1473, 9333, 4604, 2339, 3365, 8023, 968] avg=4096.0000 max=9333 max_rank=2 min=968 min_rank=7 imbalance_ratio=2.278564 max_load_reduction=23435 max_load_reduction_pct=0.715179 moved_to_echo=14695 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 54, 326, 26, 1767, 390, 0, 0, 0, 0, 0, 9, 1437, 0, 0, 0, 2, 0, 0, 1442, 0, 0, 0, 0, 0, 1528, 4096, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 1331, 887, 0, 0, 7, 362, 0, 322, 2581, 0, 0, 0, 3004, 60, 398, 0, 0, 0, 0, 0, 281, 4051, 0, 3928, 406, 0, 0, 0] rerouted_tokens_per_ep_rank=[2563, 1446, 1444, 5624, 6291, 3272, 3462, 8666] avg=4096.0000 max=8666 max_rank=7 min=1444 min_rank=2 imbalance_ratio=2.115723 max_load_reduction=24102 max_load_reduction_pct=0.735535 moved_to_echo=21691 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[452, 159, 1573, 94, 1731, 12, 0, 0, 42, 0, 2681, 3565, 46, 0, 0, 0, 9, 0, 3504, 1412, 0, 0, 0, 0, 0, 51, 204, 0, 0, 0, 0, 0, 328, 527, 0, 1252, 1291, 869, 0, 0, 0, 864, 183, 2135, 2502, 0, 0, 0, 3737, 15, 304, 0, 0, 0, 0, 0, 0, 3178, 0, 35, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 6334, 4925, 255, 4267, 5684, 4056, 3226] avg=4096.0000 max=6334 max_rank=1 min=255 min_rank=3 imbalance_ratio=1.546387 max_load_reduction=26434 max_load_reduction_pct=0.806702 moved_to_echo=17233 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3259, 163, 1019, 430, 0, 0, 0, 0, 2061, 0, 1568, 0, 0, 0, 1416, 83, 0, 820, 0, 0, 0, 0, 0, 1651, 3886, 0, 0, 0, 0, 0, 10, 6, 0, 1050, 1005, 515, 0, 0, 0, 3913, 3, 0, 1947, 0, 0, 0, 32, 3981, 611, 0, 0, 0, 0, 0, 0, 2690, 1, 192, 447, 9, 0, 0] rerouted_tokens_per_ep_rank=[4871, 3629, 2319, 5537, 2586, 5863, 4624, 3339] avg=4096.0000 max=5863 max_rank=5 min=2319 min_rank=2 imbalance_ratio=1.431396 max_load_reduction=26905 max_load_reduction_pct=0.821075 moved_to_echo=16412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[384, 90, 1, 1411, 1511, 422, 0, 0, 0, 46, 3863, 1356, 1555, 0, 0, 0, 3070, 416, 1643, 1232, 0, 0, 0, 0, 97, 1653, 361, 238, 0, 0, 0, 0, 0, 163, 0, 0, 995, 759, 0, 0, 3234, 6, 0, 17, 1927, 0, 0, 0, 749, 620, 1410, 0, 0, 0, 0, 0, 29, 1, 1722, 1346, 440, 1, 0, 0] rerouted_tokens_per_ep_rank=[3819, 6820, 6361, 2349, 1917, 5184, 2779, 3539] avg=4096.0000 max=6820 max_rank=1 min=1917 min_rank=4 imbalance_ratio=1.665039 max_load_reduction=25948 max_load_reduction_pct=0.791870 moved_to_echo=13419 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 3200, 9856, 3968, 12288, 2944, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 256, 12800, 9216, 10624, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4992, 4608, 13440, 9984, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[13440, 896, 128, 2432, 10112, 6272, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 21632, 1792, 5632, 3072, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 11392, 19968, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5376, 5248, 384, 2560, 19584, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13056, 17152, 2816, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[48, 0, 2416, 90, 3088, 264, 0, 115, 3653, 284, 3807, 4033, 840, 0, 3922, 0, 4096, 1511, 0, 199, 97, 0, 0, 223, 0, 0, 246, 76, 263, 449, 2112, 936] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[960, 0, 2199, 4068, 4062, 67, 4054, 646, 3937, 2, 0, 2114, 4033, 0, 3930, 35, 57, 626, 890, 5, 292, 0, 0, 0, 0, 389, 0, 11, 2, 0, 0, 389] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[915, 0, 643, 0, 2853, 613, 152, 0, 6, 0, 3972, 1995, 1, 76, 3266, 0, 3576, 0, 242, 427, 0, 0, 44, 495, 3642, 1810, 0, 4096, 3834, 0, 0, 110] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 18, 3616, 3992, 0, 3982, 1146, 419, 0, 4077, 16, 0, 2, 11, 3636, 4079, 0, 2790, 540, 4, 0, 0, 0, 287, 0, 158, 3955, 0, 2, 38, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 160, 0, 4054, 1108, 0, 3923, 17, 3909, 107, 4096, 0, 930, 0, 20, 8, 2113, 67, 3645, 0, 171, 443, 0, 192, 19, 0, 2, 4093, 3, 3673, 0, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[343, 44, 1097, 2762, 3312, 56, 7, 4032, 757, 2655, 721, 551, 3473, 0, 274, 0, 3702, 217, 0, 0, 10, 0, 0, 2, 55, 0, 89, 3149, 271, 1176, 3943, 70] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[53, 0, 3672, 229, 4, 86, 111, 2511, 0, 0, 3449, 0, 0, 2, 3743, 1531, 4024, 0, 316, 198, 1, 484, 691, 781, 366, 0, 182, 2710, 4020, 3561, 38, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[8, 0, 2857, 14, 4096, 105, 538, 5, 6, 1, 1333, 3990, 1, 0, 2785, 439, 3859, 592, 3876, 183, 102, 0, 32, 10, 0, 3784, 0, 3847, 9, 0, 296, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[960, 0, 2199, 4068, 4, 0, 41, 0, 1866, 67, 4054, 646, 0, 0, 0, 0, 3937, 2, 0, 2114, 0, 0, 0, 0, 4033, 0, 3889, 35, 0, 0, 0, 0, 38, 626, 890, 5, 0, 0, 0, 0, 292, 0, 0, 0, 0, 2087, 0, 0, 0, 389, 0, 11, 0, 0, 0, 0, 2, 0, 0, 389, 15, 109, 0, 0] rerouted_tokens_per_ep_rank=[7272, 6633, 6053, 7957, 1559, 2379, 400, 515] avg=4096.0000 max=7957 max_rank=3 min=400 min_rank=6 imbalance_ratio=1.942627 max_load_reduction=24811 max_load_reduction_pct=0.757172 moved_to_echo=4853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 18, 3616, 338, 35, 0, 0, 1834, 0, 3982, 1146, 0, 0, 0, 0, 419, 0, 824, 16, 0, 0, 0, 0, 0, 2, 11, 3636, 0, 0, 0, 0, 2630, 0, 2790, 540, 0, 0, 0, 0, 4, 0, 0, 0, 3253, 2051, 0, 0, 287, 0, 158, 3920, 0, 0, 0, 0, 0, 2, 38, 0, 1111, 107, 0, 0] rerouted_tokens_per_ep_rank=[4007, 6962, 1259, 3649, 5960, 5308, 4365, 1258] avg=4096.0000 max=6962 max_rank=1 min=1258 min_rank=7 imbalance_ratio=1.699707 max_load_reduction=25806 max_load_reduction_pct=0.787537 moved_to_echo=16891 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[48, 0, 2416, 90, 339, 0, 41, 0, 1419, 264, 0, 115, 0, 0, 0, 0, 3653, 284, 769, 4033, 0, 0, 0, 0, 840, 0, 3881, 0, 0, 0, 0, 0, 2642, 1511, 0, 199, 0, 0, 0, 0, 97, 0, 0, 223, 3038, 1587, 0, 0, 0, 0, 246, 76, 0, 0, 0, 0, 263, 449, 2112, 936, 1115, 82, 0, 0] rerouted_tokens_per_ep_rank=[2934, 1798, 8739, 4721, 4352, 4945, 322, 4957] avg=4096.0000 max=8739 max_rank=2 min=322 min_rank=6 imbalance_ratio=2.133545 max_load_reduction=24029 max_load_reduction_pct=0.733307 moved_to_echo=14576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[915, 0, 643, 0, 296, 36, 34, 0, 1311, 613, 152, 0, 0, 0, 0, 0, 6, 0, 803, 1995, 0, 0, 0, 0, 1, 76, 3232, 0, 0, 0, 0, 0, 2306, 0, 242, 427, 0, 0, 0, 0, 0, 0, 44, 495, 3169, 1466, 0, 0, 3642, 1810, 0, 4060, 0, 0, 0, 0, 3834, 0, 0, 110, 974, 76, 0, 0] rerouted_tokens_per_ep_rank=[1924, 2076, 2804, 3309, 2975, 5174, 9512, 4994] avg=4096.0000 max=9512 max_rank=6 min=1924 min_rank=0 imbalance_ratio=2.322266 max_load_reduction=23256 max_load_reduction_pct=0.709717 moved_to_echo=22655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 160, 0, 4054, 175, 36, 0, 0, 510, 0, 3923, 17, 0, 0, 0, 0, 3909, 107, 828, 0, 0, 0, 0, 0, 930, 0, 20, 8, 0, 0, 0, 0, 1363, 67, 3645, 0, 0, 0, 0, 0, 171, 443, 0, 192, 3268, 569, 0, 0, 19, 0, 2, 4057, 0, 0, 0, 0, 3, 3673, 0, 15, 575, 29, 0, 0] rerouted_tokens_per_ep_rank=[4425, 4450, 4844, 958, 5075, 4643, 4078, 4295] avg=4096.0000 max=5075 max_rank=4 min=958 min_rank=3 imbalance_ratio=1.239014 max_load_reduction=27693 max_load_reduction_pct=0.845123 moved_to_echo=18091 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[343, 44, 1097, 2762, 306, 28, 2, 0, 1522, 56, 7, 4032, 0, 0, 0, 0, 757, 2655, 146, 551, 0, 0, 0, 0, 3473, 0, 272, 0, 0, 0, 0, 0, 2388, 217, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 575, 1702, 0, 0, 55, 0, 89, 3121, 0, 0, 0, 0, 271, 1176, 3943, 70, 1008, 88, 0, 0] rerouted_tokens_per_ep_rank=[4582, 5617, 4109, 3745, 2605, 2289, 3265, 6556] avg=4096.0000 max=6556 max_rank=7 min=2289 min_rank=5 imbalance_ratio=1.600586 max_load_reduction=26212 max_load_reduction_pct=0.799927 moved_to_echo=14715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 2857, 14, 323, 39, 33, 0, 1874, 105, 538, 5, 0, 0, 0, 0, 6, 1, 266, 3990, 0, 0, 0, 0, 1, 0, 2752, 439, 0, 0, 0, 0, 2482, 592, 3876, 183, 0, 0, 0, 0, 102, 0, 32, 10, 1067, 2108, 0, 0, 0, 3784, 0, 3808, 0, 0, 0, 0, 9, 0, 296, 0, 1054, 114, 0, 0] rerouted_tokens_per_ep_rank=[3274, 2522, 4263, 3192, 7133, 3319, 7592, 1473] avg=4096.0000 max=7592 max_rank=6 min=1473 min_rank=7 imbalance_ratio=1.853516 max_load_reduction=25176 max_load_reduction_pct=0.768311 moved_to_echo=19517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[53, 0, 3672, 229, 333, 24, 39, 0, 2, 86, 111, 2511, 0, 0, 0, 0, 0, 0, 697, 0, 0, 0, 0, 0, 0, 2, 3704, 1531, 0, 0, 0, 0, 2595, 0, 316, 198, 0, 0, 0, 0, 1, 484, 691, 781, 2752, 2, 0, 0, 366, 0, 182, 2686, 0, 0, 0, 0, 4020, 3561, 38, 5, 1096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4350, 2710, 697, 5237, 3109, 4711, 3234, 8720] avg=4096.0000 max=8720 max_rank=7 min=697 min_rank=2 imbalance_ratio=2.128906 max_load_reduction=24048 max_load_reduction_pct=0.733887 moved_to_echo=19774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8448, 8960, 6528, 1536, 7040, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4480, 6016, 768, 21760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12800, 3072, 4352, 12800, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16512, 3072, 11776, 1664, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10368, 1280, 12800, 8576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 1024, 768, 1792, 17152, 11648, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2432, 256, 12928, 14848, 2176, 256, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9344, 128, 17792, 5760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 357, 0, 568, 2, 81, 4096, 12, 1605, 1590, 0, 34, 3796, 0, 4034, 0, 124, 0, 11, 3, 1454, 2906, 4054, 0, 0, 3457, 0, 1, 3809, 56, 718, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[21, 15, 202, 4086, 1542, 3440, 4034, 3967, 830, 97, 71, 599, 0, 0, 30, 55, 0, 0, 0, 0, 4092, 2871, 4070, 8, 2, 0, 0, 332, 408, 2, 1994, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[726, 0, 2031, 4079, 0, 707, 1320, 1652, 2449, 3992, 0, 0, 0, 28, 0, 0, 15, 0, 2657, 121, 3912, 86, 4065, 3791, 17, 1008, 0, 0, 0, 0, 83, 29] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 746, 0, 4095, 4087, 92, 31, 0, 2778, 1693, 0, 4, 504, 0, 0, 0, 554, 16, 0, 76, 4079, 0, 4061, 1, 3136, 2, 0, 0, 2160, 630, 4023, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1960, 3467, 530, 3930, 0, 5, 4090, 78, 3437, 3281, 0, 0, 545, 0, 0, 0, 1361, 60, 0, 4, 429, 349, 4096, 49, 453, 790, 8, 0, 1, 0, 3845, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 1389, 248, 3964, 3414, 3550, 590, 1378, 3736, 2902, 15, 12, 3257, 0, 39, 0, 0, 493, 1407, 0, 2164, 311, 19, 1590, 135, 0, 589, 0, 1158, 20, 388, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 1528, 0, 709, 12, 4039, 4096, 66, 3262, 9, 24, 0, 0, 0, 0, 0, 2066, 0, 86, 1, 4020, 3130, 2827, 3295, 1888, 129, 0, 0, 467, 26, 1088, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1, 92, 3532, 2, 877, 6, 3893, 3335, 4021, 132, 1437, 229, 114, 98, 2902, 0, 3234, 2208, 80, 0, 83, 0, 15, 0, 3980, 1, 0, 0, 623, 1861, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 357, 0, 423, 0, 0, 0, 0, 2, 81, 80, 12, 0, 0, 0, 0, 1208, 1590, 0, 34, 0, 0, 0, 0, 3796, 0, 4034, 0, 2283, 876, 0, 0, 124, 0, 11, 3, 3178, 0, 0, 0, 1034, 2906, 0, 0, 0, 0, 0, 0, 0, 3457, 0, 1, 1733, 420, 42, 0, 3809, 56, 718, 0, 397, 103, 0, 0] rerouted_tokens_per_ep_rank=[780, 175, 2832, 10989, 3316, 3940, 5653, 5083] avg=4096.0000 max=10989 max_rank=3 min=175 min_rank=1 imbalance_ratio=2.682861 max_load_reduction=21779 max_load_reduction_pct=0.664642 moved_to_echo=17992 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[21, 15, 202, 3037, 0, 0, 0, 0, 1542, 3440, 78, 3967, 0, 0, 0, 0, 625, 97, 71, 599, 0, 0, 0, 0, 0, 0, 30, 55, 2249, 879, 0, 0, 0, 0, 0, 0, 3191, 0, 0, 0, 2909, 2871, 0, 8, 0, 0, 0, 0, 2, 0, 0, 332, 1707, 1183, 307, 0, 408, 2, 1994, 0, 205, 742, 0, 0] rerouted_tokens_per_ep_rank=[3275, 9027, 1392, 3213, 3191, 5788, 3531, 3351] avg=4096.0000 max=9027 max_rank=1 min=1392 min_rank=2 imbalance_ratio=2.203857 max_load_reduction=23741 max_load_reduction_pct=0.724518 moved_to_echo=15861 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1960, 3467, 530, 2921, 0, 0, 0, 0, 0, 5, 80, 78, 0, 0, 0, 0, 2587, 3281, 0, 0, 0, 0, 0, 0, 545, 0, 0, 0, 2280, 885, 0, 0, 1361, 60, 0, 4, 3211, 0, 0, 0, 305, 349, 0, 49, 0, 0, 0, 0, 453, 790, 8, 0, 1730, 124, 295, 0, 1, 0, 3845, 0, 850, 714, 0, 0] rerouted_tokens_per_ep_rank=[8878, 163, 5868, 3710, 4636, 703, 3400, 5410] avg=4096.0000 max=8878 max_rank=0 min=163 min_rank=1 imbalance_ratio=2.167480 max_load_reduction=23890 max_load_reduction_pct=0.729065 moved_to_echo=14149 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[726, 0, 2031, 3031, 0, 0, 0, 0, 0, 707, 27, 1652, 0, 0, 0, 0, 1844, 3992, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 735, 878, 0, 0, 15, 0, 2657, 121, 3187, 0, 0, 0, 2781, 86, 0, 3791, 0, 0, 0, 0, 17, 1008, 0, 0, 558, 1131, 307, 0, 0, 0, 83, 29, 605, 741, 0, 0] rerouted_tokens_per_ep_rank=[5788, 2386, 5836, 1641, 5980, 6658, 3021, 1458] avg=4096.0000 max=6658 max_rank=5 min=1458 min_rank=7 imbalance_ratio=1.625488 max_load_reduction=26110 max_load_reduction_pct=0.796814 moved_to_echo=17117 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 746, 0, 3043, 0, 0, 0, 0, 4087, 92, 1, 0, 0, 0, 0, 0, 2091, 1693, 0, 4, 0, 0, 0, 0, 504, 0, 0, 0, 17, 878, 0, 0, 554, 16, 0, 76, 3183, 0, 0, 0, 2900, 0, 0, 1, 0, 0, 0, 0, 3136, 2, 0, 0, 13, 1179, 308, 0, 2160, 630, 4023, 0, 687, 744, 0, 0] rerouted_tokens_per_ep_rank=[3789, 4180, 3788, 1399, 3829, 2901, 4638, 8244] avg=4096.0000 max=8244 max_rank=7 min=1399 min_rank=3 imbalance_ratio=2.012695 max_load_reduction=24524 max_load_reduction_pct=0.748413 moved_to_echo=19612 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1389, 248, 2946, 0, 0, 0, 0, 3414, 3550, 13, 1378, 0, 0, 0, 0, 2812, 2902, 15, 12, 0, 0, 0, 0, 3257, 0, 39, 0, 328, 4, 0, 0, 0, 493, 1407, 0, 15, 0, 0, 0, 1539, 311, 0, 1590, 0, 0, 0, 0, 135, 0, 589, 0, 249, 625, 298, 0, 1158, 20, 388, 0, 924, 720, 0, 0] rerouted_tokens_per_ep_rank=[4583, 8355, 5741, 3628, 1915, 3440, 1896, 3210] avg=4096.0000 max=8355 max_rank=1 min=1896 min_rank=6 imbalance_ratio=2.039795 max_load_reduction=24413 max_load_reduction_pct=0.745026 moved_to_echo=10461 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1528, 0, 520, 0, 0, 0, 0, 12, 4039, 71, 66, 0, 0, 0, 0, 2453, 9, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2288, 612, 0, 0, 2066, 0, 86, 1, 2215, 0, 0, 0, 2854, 3130, 0, 3295, 0, 0, 0, 0, 1888, 129, 0, 0, 1737, 1166, 57, 0, 467, 26, 1088, 0, 809, 132, 0, 0] rerouted_tokens_per_ep_rank=[2048, 4188, 2486, 2900, 4368, 9279, 4977, 2522] avg=4096.0000 max=9279 max_rank=5 min=2048 min_rank=0 imbalance_ratio=2.265381 max_load_reduction=23489 max_load_reduction_pct=0.716827 moved_to_echo=21146 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 92, 3532, 2, 0, 0, 0, 0, 877, 6, 76, 3335, 0, 0, 0, 0, 3027, 132, 1437, 229, 0, 0, 0, 0, 114, 98, 2902, 0, 2170, 4, 0, 0, 3234, 2208, 80, 0, 11, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 3980, 1, 0, 0, 1647, 24, 0, 0, 623, 1861, 12, 0, 994, 0, 0, 0] rerouted_tokens_per_ep_rank=[3627, 4294, 4825, 5288, 5533, 59, 5652, 3490] avg=4096.0000 max=5652 max_rank=6 min=59 min_rank=5 imbalance_ratio=1.379883 max_load_reduction=27116 max_load_reduction_pct=0.827515 moved_to_echo=14734 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 128, 7040, 128, 12416, 5120, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9984, 12032, 512, 10496, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8704, 2688, 12160, 128, 5504, 3968, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14464, 9728, 0, 8832, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2816, 7680, 6656, 16000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9728, 5504, 640, 384, 9472, 5888, 1664, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16768, 13696, 1664, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7424, 2816, 4352, 256, 18304, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2195, 0, 2813, 236, 4095, 0, 1, 7, 0, 203, 0, 4060, 4088, 84, 0, 30, 5, 4, 0, 0, 9, 2, 0, 3040, 0, 4094, 0, 3523, 0, 3942, 337, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[3106, 1, 0, 0, 4085, 0, 81, 3780, 205, 4096, 2, 4052, 4095, 0, 0, 729, 0, 4, 0, 0, 0, 224, 0, 0, 0, 4083, 0, 503, 0, 3722, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[72, 4, 3433, 0, 3486, 3600, 202, 2126, 121, 8, 2, 1038, 3980, 0, 0, 1135, 0, 140, 0, 3553, 814, 1493, 0, 4093, 0, 2717, 0, 0, 0, 1, 355, 395] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[186, 0, 2312, 0, 1914, 0, 0, 139, 0, 1, 0, 2885, 4095, 1, 5, 9, 3032, 0, 0, 317, 36, 3808, 0, 4095, 1142, 4041, 0, 1315, 10, 3425, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[2706, 0, 3884, 272, 3860, 0, 3899, 0, 0, 88, 0, 4001, 3846, 0, 0, 0, 0, 1928, 42, 0, 382, 1092, 0, 80, 13, 4007, 0, 2233, 0, 152, 280, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3425, 170, 4088, 12, 1, 58, 0, 3281, 0, 4089, 564, 21, 0, 0, 925, 343, 0, 591, 5, 3, 0, 2856, 0, 3617, 677, 3908, 0, 4095, 7, 32] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[3275, 0, 245, 1615, 4096, 0, 43, 62, 835, 4096, 0, 0, 3743, 0, 2, 0, 695, 288, 81, 4022, 1, 0, 0, 0, 0, 2811, 0, 0, 0, 2273, 4096, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[247, 9, 252, 0, 4052, 0, 263, 0, 2884, 743, 4, 3396, 4092, 0, 0, 1037, 0, 1, 0, 3611, 0, 567, 0, 4093, 0, 4065, 0, 0, 1, 3451, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3106, 1, 0, 0, 397, 0, 0, 0, 2546, 0, 81, 3780, 0, 0, 0, 0, 205, 4096, 2, 2793, 0, 0, 0, 0, 4095, 0, 0, 729, 208, 0, 0, 0, 0, 4, 0, 0, 1539, 277, 0, 0, 0, 224, 0, 0, 842, 0, 0, 0, 0, 2701, 0, 503, 0, 0, 0, 0, 0, 3722, 0, 0, 654, 263, 0, 0] rerouted_tokens_per_ep_rank=[3504, 6407, 7096, 5032, 1820, 1066, 3204, 4639] avg=4096.0000 max=7096 max_rank=2 min=1066 min_rank=5 imbalance_ratio=1.732422 max_load_reduction=25672 max_load_reduction_pct=0.783447 moved_to_echo=10729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2195, 0, 2813, 236, 398, 0, 0, 0, 2552, 0, 1, 7, 0, 0, 0, 0, 0, 203, 0, 2798, 0, 0, 0, 0, 4088, 84, 0, 30, 209, 0, 0, 0, 5, 4, 0, 0, 1543, 278, 0, 0, 9, 2, 0, 3040, 844, 0, 0, 0, 0, 2708, 0, 3523, 0, 0, 0, 0, 0, 3942, 337, 0, 655, 264, 0, 0] rerouted_tokens_per_ep_rank=[5642, 2560, 3001, 4411, 1830, 3895, 6231, 5198] avg=4096.0000 max=6231 max_rank=6 min=1830 min_rank=4 imbalance_ratio=1.521240 max_load_reduction=26537 max_load_reduction_pct=0.809845 moved_to_echo=17154 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[186, 0, 2312, 0, 283, 0, 0, 0, 1193, 0, 0, 139, 0, 0, 0, 0, 0, 1, 0, 1988, 0, 0, 0, 0, 4095, 1, 5, 9, 148, 0, 0, 0, 3032, 0, 0, 317, 721, 275, 0, 0, 36, 3808, 0, 4095, 834, 0, 0, 0, 1142, 2671, 0, 1315, 0, 0, 0, 0, 10, 3425, 0, 0, 466, 261, 0, 0] rerouted_tokens_per_ep_rank=[2781, 1332, 1989, 4258, 4345, 8773, 5128, 4162] avg=4096.0000 max=8773 max_rank=5 min=1332 min_rank=1 imbalance_ratio=2.141846 max_load_reduction=23995 max_load_reduction_pct=0.732269 moved_to_echo=22408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2706, 0, 3884, 272, 392, 0, 0, 0, 2406, 0, 3899, 0, 0, 0, 0, 0, 0, 88, 0, 2757, 0, 0, 0, 0, 3846, 0, 0, 0, 206, 0, 0, 0, 0, 1928, 42, 0, 1454, 272, 0, 0, 382, 1092, 0, 80, 826, 0, 0, 0, 13, 2651, 0, 2233, 0, 0, 0, 0, 0, 152, 280, 3, 646, 258, 0, 0] rerouted_tokens_per_ep_rank=[7254, 6305, 2845, 4052, 3696, 2380, 4897, 1339] avg=4096.0000 max=7254 max_rank=0 min=1339 min_rank=7 imbalance_ratio=1.770996 max_load_reduction=25514 max_load_reduction_pct=0.778625 moved_to_echo=12312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[72, 4, 3433, 0, 101, 0, 0, 0, 2173, 3600, 202, 2126, 0, 0, 0, 0, 121, 8, 2, 717, 0, 0, 0, 0, 3980, 0, 0, 1135, 53, 0, 0, 0, 0, 140, 0, 3553, 1313, 184, 0, 0, 814, 1493, 0, 4093, 560, 0, 0, 0, 0, 1798, 0, 0, 0, 0, 0, 0, 0, 1, 355, 395, 167, 175, 0, 0] rerouted_tokens_per_ep_rank=[3610, 8101, 848, 5168, 5190, 6960, 1798, 1093] avg=4096.0000 max=8101 max_rank=1 min=848 min_rank=2 imbalance_ratio=1.977783 max_load_reduction=24667 max_load_reduction_pct=0.752777 moved_to_echo=15041 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3425, 170, 401, 0, 0, 0, 2548, 12, 1, 58, 0, 0, 0, 0, 0, 3281, 0, 2818, 0, 0, 0, 0, 564, 21, 0, 0, 210, 0, 0, 0, 925, 343, 0, 591, 1540, 246, 0, 0, 5, 3, 0, 2856, 746, 0, 0, 0, 0, 2392, 677, 3908, 0, 0, 0, 0, 0, 4095, 7, 32, 660, 233, 0, 0] rerouted_tokens_per_ep_rank=[3996, 2619, 6099, 795, 3645, 3610, 6977, 5027] avg=4096.0000 max=6977 max_rank=6 min=795 min_rank=3 imbalance_ratio=1.703369 max_load_reduction=25791 max_load_reduction_pct=0.787079 moved_to_echo=19259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[247, 9, 252, 0, 338, 0, 0, 0, 2523, 0, 263, 0, 0, 0, 0, 0, 2884, 743, 4, 2328, 0, 0, 0, 0, 4092, 0, 0, 1037, 178, 0, 0, 0, 0, 1, 0, 3611, 1529, 281, 0, 0, 0, 567, 0, 4093, 843, 0, 0, 0, 0, 2674, 0, 0, 0, 0, 0, 0, 1, 3451, 0, 0, 552, 267, 0, 0] rerouted_tokens_per_ep_rank=[846, 2786, 5959, 5307, 5422, 5503, 2674, 4271] avg=4096.0000 max=5959 max_rank=2 min=846 min_rank=0 imbalance_ratio=1.454834 max_load_reduction=26809 max_load_reduction_pct=0.818146 moved_to_echo=17870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3275, 0, 245, 1615, 0, 0, 0, 0, 2553, 0, 43, 62, 0, 0, 0, 0, 835, 4096, 0, 0, 0, 0, 0, 0, 3743, 0, 2, 0, 0, 0, 0, 0, 695, 288, 81, 4022, 1543, 191, 0, 0, 1, 0, 0, 0, 580, 0, 0, 0, 0, 1859, 0, 0, 0, 0, 0, 0, 0, 2273, 4096, 489, 0, 181, 0, 0] rerouted_tokens_per_ep_rank=[5135, 2658, 4931, 3745, 6820, 581, 1859, 7039] avg=4096.0000 max=7039 max_rank=7 min=581 min_rank=5 imbalance_ratio=1.718506 max_load_reduction=25729 max_load_reduction_pct=0.785187 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 7296, 0, 18304, 6144, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4736, 2816, 128, 12160, 11264, 2048, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 12544, 128, 16256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 21120, 5120, 1024, 3840, 1920, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[18560, 3712, 4608, 6272, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11904, 128, 16384, 2304, 2432, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 19456, 768, 11520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[28544, 128, 128, 2944, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1094, 67, 0, 1, 3975, 3864, 2, 0, 1723, 2666, 0, 1, 0, 0, 18, 0, 548, 0, 1, 0, 4095, 11, 4066, 74, 3901, 27, 0, 378, 2240, 0, 0, 4016] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 141, 4, 0, 1986, 2288, 0, 0, 4096, 40, 0, 553, 4070, 0, 0, 0, 0, 2248, 0, 458, 145, 0, 3917, 0, 4092, 1, 223, 0, 318, 1, 4092, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[2, 4019, 736, 0, 351, 3982, 0, 57, 2, 0, 825, 124, 4091, 0, 0, 29, 0, 3961, 1, 3109, 0, 0, 3274, 0, 4096, 2, 0, 0, 13, 0, 6, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[151, 0, 3856, 0, 3685, 0, 0, 4093, 3080, 0, 0, 398, 122, 328, 0, 0, 0, 129, 97, 0, 3981, 5, 3963, 42, 4062, 11, 288, 0, 732, 3481, 261, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 527, 79, 404, 0, 0, 0, 900, 2961, 3924, 0, 2832, 2984, 203, 0, 176, 0, 404, 120, 10, 2603, 0, 2801, 0, 3298, 3367, 494, 470, 147, 0, 0, 4064] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[2026, 3, 0, 0, 0, 3912, 0, 0, 269, 11, 0, 84, 3216, 0, 0, 0, 39, 3, 0, 1258, 4094, 0, 3883, 1197, 3870, 460, 160, 0, 4, 100, 4083, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4094, 2109, 0, 0, 3995, 3689, 0, 1926, 1046, 101, 0, 38, 970, 0, 0, 52, 0, 68, 1, 341, 0, 0, 3958, 978, 4088, 20, 3361, 0, 0, 0, 7, 1926] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3971, 1078, 748, 1, 4096, 0, 85, 3490, 0, 0, 64, 4048, 2298, 3564, 341, 4019, 0, 57, 0, 0, 1981, 51, 0, 6, 477, 1, 88, 0, 1207, 0, 0, 1097] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 4019, 736, 0, 765, 280, 0, 0, 87, 3982, 0, 57, 0, 0, 0, 0, 2, 0, 825, 124, 570, 0, 0, 0, 4091, 0, 0, 29, 996, 85, 0, 0, 0, 3961, 1, 3109, 264, 798, 0, 0, 0, 0, 1711, 0, 0, 0, 0, 0, 3441, 2, 0, 0, 0, 0, 0, 0, 13, 0, 6, 2812, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5802, 4126, 1521, 5201, 8133, 1711, 3443, 2831] avg=4096.0000 max=8133 max_rank=4 min=1521 min_rank=2 imbalance_ratio=1.985596 max_load_reduction=24635 max_load_reduction_pct=0.751801 moved_to_echo=16118 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 141, 4, 0, 916, 281, 0, 0, 492, 2288, 0, 0, 0, 0, 0, 0, 4096, 40, 0, 553, 569, 0, 0, 0, 4070, 0, 0, 0, 998, 85, 0, 0, 0, 2248, 0, 458, 1494, 955, 0, 0, 145, 0, 2046, 0, 0, 0, 0, 0, 3438, 1, 223, 0, 0, 0, 0, 0, 318, 1, 4092, 2816, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1342, 2780, 5258, 5153, 5155, 2191, 3662, 7227] avg=4096.0000 max=7227 max_rank=7 min=1342 min_rank=0 imbalance_ratio=1.764404 max_load_reduction=25541 max_load_reduction_pct=0.779449 moved_to_echo=18235 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1094, 67, 0, 1, 951, 275, 0, 0, 985, 3864, 2, 0, 0, 0, 0, 0, 1723, 2666, 0, 1, 543, 0, 0, 0, 0, 0, 18, 0, 979, 81, 0, 0, 548, 0, 1, 0, 2990, 991, 0, 0, 4095, 11, 2124, 74, 0, 0, 0, 0, 3277, 27, 0, 378, 0, 0, 0, 0, 2240, 0, 0, 2762, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2388, 4851, 4933, 1078, 4530, 6304, 3682, 5002] avg=4096.0000 max=6304 max_rank=5 min=1078 min_rank=3 imbalance_ratio=1.539062 max_load_reduction=26464 max_load_reduction_pct=0.807617 moved_to_echo=19518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 527, 79, 404, 655, 279, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 2961, 3924, 0, 2832, 459, 0, 0, 0, 2984, 203, 0, 176, 990, 69, 0, 0, 0, 404, 120, 10, 0, 683, 0, 0, 2603, 0, 1463, 0, 0, 0, 0, 0, 2770, 3367, 494, 470, 0, 0, 0, 0, 147, 0, 0, 2795, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1944, 900, 10176, 4422, 1217, 4066, 7101, 2942] avg=4096.0000 max=10176 max_rank=2 min=900 min_rank=1 imbalance_ratio=2.484375 max_load_reduction=22592 max_load_reduction_pct=0.689453 moved_to_echo=15326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[151, 0, 3856, 0, 927, 0, 0, 0, 913, 0, 0, 4093, 0, 0, 0, 0, 3080, 0, 0, 398, 565, 0, 0, 0, 122, 328, 0, 0, 0, 85, 0, 0, 0, 129, 97, 0, 2772, 966, 0, 0, 3981, 5, 2070, 42, 0, 0, 0, 0, 3412, 11, 288, 0, 0, 0, 0, 0, 732, 3481, 261, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4934, 5006, 4043, 535, 3964, 6098, 3711, 4477] avg=4096.0000 max=6098 max_rank=5 min=535 min_rank=3 imbalance_ratio=1.488770 max_load_reduction=26670 max_load_reduction_pct=0.813904 moved_to_echo=18250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2026, 3, 0, 0, 908, 281, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 269, 11, 0, 84, 538, 0, 0, 0, 3216, 0, 0, 0, 998, 81, 0, 0, 39, 3, 0, 1258, 0, 946, 0, 0, 4094, 0, 2029, 1197, 0, 0, 0, 0, 3251, 460, 160, 0, 0, 0, 0, 0, 4, 100, 4083, 2817, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3218, 3912, 902, 4295, 2246, 7320, 3871, 7004] avg=4096.0000 max=7320 max_rank=5 min=902 min_rank=2 imbalance_ratio=1.787109 max_load_reduction=25448 max_load_reduction_pct=0.776611 moved_to_echo=20441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4094, 2109, 0, 0, 928, 136, 0, 0, 988, 3689, 0, 1926, 0, 0, 0, 0, 1046, 101, 0, 38, 572, 0, 0, 0, 970, 0, 0, 52, 474, 89, 0, 0, 0, 68, 1, 341, 3007, 968, 0, 0, 0, 0, 2062, 978, 0, 0, 0, 0, 3427, 20, 3361, 0, 0, 0, 0, 0, 0, 0, 7, 1316, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7267, 6603, 1757, 1585, 4385, 3040, 6808, 1323] avg=4096.0000 max=7267 max_rank=0 min=1323 min_rank=7 imbalance_ratio=1.774170 max_load_reduction=25501 max_load_reduction_pct=0.778229 moved_to_echo=15556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3971, 1078, 748, 1, 0, 75, 0, 0, 1015, 0, 85, 3490, 0, 0, 0, 0, 0, 0, 64, 4048, 66, 0, 0, 0, 2298, 3564, 341, 4019, 267, 10, 0, 0, 0, 57, 0, 0, 3081, 0, 0, 0, 1981, 51, 0, 6, 0, 0, 0, 0, 401, 1, 88, 0, 0, 0, 0, 0, 1207, 0, 0, 755, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5873, 4590, 4178, 10499, 3138, 2038, 490, 1962] avg=4096.0000 max=10499 max_rank=3 min=490 min_rank=6 imbalance_ratio=2.563232 max_load_reduction=22269 max_load_reduction_pct=0.679596 moved_to_echo=7628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17792, 4096, 384, 4352, 5760, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17024, 128, 13568, 2304, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 6912, 256, 5248, 13696, 6400, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13184, 6784, 896, 8192, 3968, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4736, 3584, 8576, 16128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11392, 8064, 5504, 512, 6144, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[23424, 3968, 4736, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[4480, 17792, 128, 10496, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[89, 0, 1, 0, 2, 0, 1, 0, 1870, 490, 4090, 0, 13, 0, 2415, 1543, 0, 335, 0, 3985, 3690, 4074, 2360, 87, 1, 16, 40, 60, 5, 1794, 1713, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 212, 0, 1694, 2964, 4069, 0, 0, 4013, 1887, 0, 151, 1, 43, 451, 3727, 1316, 4031, 0, 14, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 396, 0, 3484, 0, 0, 0, 69, 0, 0, 4096, 0, 2, 128, 0, 4047, 4096, 357, 18, 95, 4086, 0, 4040, 4081, 0, 78, 32, 0, 0, 6, 3592, 65] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[2247, 11, 0, 1163, 0, 1, 0, 0, 0, 11, 4079, 2, 1622, 1, 2975, 0, 2, 39, 4093, 0, 0, 3014, 3773, 4, 1543, 0, 0, 0, 0, 0, 4092, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[199, 0, 0, 0, 79, 0, 0, 2, 4091, 3691, 3087, 11, 0, 3689, 783, 0, 213, 12, 1, 1559, 4081, 0, 3221, 0, 0, 0, 0, 0, 0, 73, 4093, 3883] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 1353, 0, 0, 0, 0, 0, 2, 1124, 3987, 2837, 0, 0, 173, 0, 43, 1816, 0, 2836, 869, 2838, 10, 4093, 0, 349, 0, 0, 2033, 0, 384, 3925, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[36, 3078, 0, 356, 0, 0, 0, 0, 1100, 4066, 0, 0, 0, 3910, 3824, 150, 91, 1278, 100, 2347, 4022, 3989, 1, 0, 0, 323, 1, 0, 19, 0, 4057, 20] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[141, 0, 2217, 26, 0, 663, 172, 0, 15, 8, 4056, 0, 9, 0, 4096, 97, 3663, 865, 362, 3513, 3891, 657, 4069, 1, 1, 0, 0, 11, 0, 139, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1353, 0, 0, 1608, 522, 0, 28, 0, 0, 0, 2, 2838, 802, 0, 0, 1124, 3987, 825, 0, 0, 0, 0, 0, 0, 173, 0, 43, 0, 0, 0, 0, 1816, 0, 2836, 841, 0, 0, 0, 0, 0, 10, 2485, 0, 0, 0, 0, 0, 349, 0, 0, 2033, 2216, 1490, 0, 0, 0, 384, 907, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3511, 3642, 5936, 216, 5493, 2495, 6088, 5387] avg=4096.0000 max=6088 max_rank=6 min=216 min_rank=3 imbalance_ratio=1.486328 max_load_reduction=26680 max_load_reduction_pct=0.814209 moved_to_echo=19463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[89, 0, 1, 0, 927, 752, 400, 131, 2, 0, 1, 0, 3690, 350, 0, 0, 1870, 490, 1189, 0, 0, 0, 0, 0, 13, 0, 2015, 1543, 0, 0, 0, 0, 0, 335, 0, 3854, 0, 0, 0, 0, 0, 4074, 1433, 87, 0, 0, 0, 0, 1, 16, 40, 60, 967, 2149, 0, 0, 5, 1794, 396, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2300, 4043, 3549, 3571, 4189, 5594, 3233, 6289] avg=4096.0000 max=6289 max_rank=7 min=2300 min_rank=0 imbalance_ratio=1.535400 max_load_reduction=26479 max_load_reduction_pct=0.808075 moved_to_echo=19305 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1583, 748, 313, 14, 0, 3, 212, 0, 3727, 837, 0, 0, 1694, 2964, 1183, 0, 0, 0, 0, 0, 0, 4013, 1574, 0, 0, 0, 0, 0, 151, 1, 43, 437, 0, 0, 0, 0, 0, 1316, 2448, 0, 0, 0, 0, 0, 14, 0, 0, 0, 2313, 2138, 0, 0, 0, 0, 946, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2658, 4779, 5841, 5587, 632, 3764, 4465, 5042] avg=4096.0000 max=5841 max_rank=2 min=632 min_rank=4 imbalance_ratio=1.426025 max_load_reduction=26927 max_load_reduction_pct=0.821747 moved_to_echo=13903 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[199, 0, 0, 0, 1265, 568, 129, 51, 79, 0, 0, 2, 4081, 836, 0, 0, 4091, 3691, 897, 11, 0, 0, 0, 0, 0, 3689, 654, 0, 0, 0, 0, 0, 213, 12, 1, 1508, 0, 0, 0, 0, 0, 0, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2311, 1622, 0, 0, 0, 73, 946, 3883, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2212, 4998, 8690, 4343, 1734, 1956, 3933, 4902] avg=4096.0000 max=8690 max_rank=2 min=1734 min_rank=4 imbalance_ratio=2.121582 max_load_reduction=24078 max_load_reduction_pct=0.734802 moved_to_echo=12525 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2247, 11, 0, 1163, 1482, 750, 493, 0, 0, 1, 0, 0, 0, 836, 0, 0, 0, 11, 1186, 2, 0, 0, 0, 0, 1622, 1, 2482, 0, 0, 0, 0, 0, 2, 39, 4093, 0, 0, 0, 0, 0, 0, 3014, 2291, 4, 0, 0, 0, 0, 1543, 0, 0, 0, 2310, 2143, 0, 0, 0, 0, 946, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6146, 837, 1199, 4105, 4134, 5309, 5996, 5042] avg=4096.0000 max=6146 max_rank=0 min=837 min_rank=1 imbalance_ratio=1.500488 max_load_reduction=26622 max_load_reduction_pct=0.812439 moved_to_echo=20481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 396, 0, 3484, 1590, 757, 0, 7, 0, 0, 0, 69, 4086, 737, 0, 0, 0, 0, 1185, 0, 0, 0, 0, 0, 2, 128, 0, 4047, 0, 0, 0, 0, 4096, 357, 18, 88, 0, 0, 0, 0, 0, 0, 2450, 4081, 0, 0, 0, 0, 0, 78, 32, 0, 2031, 2154, 0, 0, 0, 6, 824, 65, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6234, 4892, 1185, 4177, 4559, 6531, 4295, 895] avg=4096.0000 max=6531 max_rank=5 min=895 min_rank=7 imbalance_ratio=1.594482 max_load_reduction=26237 max_load_reduction_pct=0.800690 moved_to_echo=16280 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[141, 0, 2217, 26, 1598, 746, 679, 115, 0, 663, 172, 0, 3891, 0, 0, 0, 15, 8, 1179, 0, 0, 0, 0, 0, 9, 0, 3417, 97, 0, 0, 0, 0, 3663, 865, 362, 3398, 0, 0, 0, 0, 0, 657, 2471, 1, 0, 0, 0, 0, 1, 0, 0, 11, 0, 2131, 0, 0, 0, 139, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5522, 4726, 1202, 3523, 8288, 3129, 2143, 4235] avg=4096.0000 max=8288 max_rank=4 min=1202 min_rank=2 imbalance_ratio=2.023438 max_load_reduction=24480 max_load_reduction_pct=0.747070 moved_to_echo=17795 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[36, 3078, 0, 356, 0, 0, 638, 77, 0, 0, 0, 0, 4022, 829, 0, 0, 1100, 4066, 0, 0, 0, 0, 0, 0, 0, 3910, 3186, 150, 0, 0, 0, 0, 91, 1278, 100, 2270, 0, 0, 0, 0, 0, 3989, 1, 0, 0, 0, 0, 0, 0, 323, 1, 0, 2291, 0, 0, 0, 19, 0, 937, 20, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 4851, 5166, 7246, 3739, 3990, 2615, 976] avg=4096.0000 max=7246 max_rank=3 min=976 min_rank=7 imbalance_ratio=1.769043 max_load_reduction=25522 max_load_reduction_pct=0.778870 moved_to_echo=11320 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2816, 4864, 2304, 5120, 10112, 4864, 2688, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1920, 512, 128, 2176, 14464, 13952, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9984, 15232, 7680, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10112, 2944, 7552, 12416, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1664, 12032, 13440, 5888, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 13184, 15616, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 2432, 6016, 24448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 768, 512, 128, 26368, 5248, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[441, 188, 671, 39, 163, 3453, 0, 20, 0, 3401, 1, 2990, 39, 249, 3988, 0, 3186, 0, 3863, 3901, 0, 666, 0, 0, 0, 0, 0, 1133, 4095, 0, 281, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[46, 6, 0, 4, 1322, 3979, 0, 0, 6, 0, 0, 2555, 1181, 39, 4096, 61, 4, 4096, 3693, 0, 0, 1, 1391, 0, 151, 470, 3130, 2, 9, 3001, 2932, 593] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[109, 0, 3549, 0, 4090, 1409, 1, 392, 0, 0, 0, 511, 922, 0, 4063, 38, 1, 4083, 3211, 1, 0, 0, 4069, 0, 11, 10, 57, 0, 0, 0, 3674, 2567] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3581, 624, 12, 104, 455, 3110, 19, 1561, 0, 3279, 1579, 3643, 318, 0, 4095, 13, 125, 1510, 4096, 35, 0, 0, 13, 0, 1, 224, 442, 0, 3893, 0, 0, 36] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4095, 1, 416, 0, 4078, 72, 1, 2920, 0, 1707, 156, 3754, 853, 1, 3391, 1474, 0, 22, 4096, 0, 0, 7, 2733, 12, 0, 16, 1, 0, 0, 29, 2933, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[12, 10, 484, 0, 497, 3135, 3814, 0, 0, 626, 3103, 3795, 0, 0, 2946, 0, 215, 3528, 4096, 2203, 330, 1, 163, 1901, 1658, 0, 76, 0, 149, 0, 26, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[128, 10, 1182, 18, 4010, 4096, 0, 448, 0, 3779, 3556, 3853, 0, 0, 2441, 797, 1, 14, 4096, 30, 0, 2, 11, 0, 2, 92, 30, 6, 52, 0, 4096, 18] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[52, 3026, 3816, 824, 3748, 0, 29, 4058, 0, 0, 4096, 3582, 0, 692, 19, 1, 0, 38, 4096, 1025, 0, 300, 1, 60, 87, 8, 0, 0, 2821, 381, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 1, 416, 0, 1417, 0, 0, 0, 4078, 6, 1, 2920, 0, 0, 0, 0, 0, 1707, 156, 1140, 0, 0, 0, 0, 853, 1, 3391, 1474, 159, 0, 0, 0, 0, 22, 1147, 0, 0, 0, 0, 0, 0, 7, 2733, 12, 57, 860, 0, 0, 0, 16, 1, 0, 2949, 9, 0, 0, 0, 29, 2933, 0, 178, 0, 0, 0] rerouted_tokens_per_ep_rank=[5929, 7005, 3003, 5878, 1169, 3669, 2975, 3140] avg=4096.0000 max=7005 max_rank=1 min=1169 min_rank=4 imbalance_ratio=1.710205 max_load_reduction=25763 max_load_reduction_pct=0.786224 moved_to_echo=10953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[12, 10, 484, 0, 1432, 0, 0, 0, 497, 187, 3814, 0, 0, 0, 0, 0, 0, 626, 3103, 1152, 0, 0, 0, 0, 0, 0, 2946, 0, 161, 0, 0, 0, 215, 3528, 1147, 2203, 0, 0, 0, 0, 330, 1, 163, 1901, 2515, 870, 0, 0, 1658, 0, 76, 0, 2949, 433, 0, 0, 149, 0, 26, 0, 180, 0, 0, 0] rerouted_tokens_per_ep_rank=[1938, 4498, 4881, 3107, 7093, 5780, 5116, 355] avg=4096.0000 max=7093 max_rank=4 min=355 min_rank=7 imbalance_ratio=1.731689 max_load_reduction=25675 max_load_reduction_pct=0.783539 moved_to_echo=18344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3581, 624, 12, 104, 1375, 0, 0, 0, 455, 185, 19, 1561, 0, 0, 0, 0, 0, 3279, 1579, 1105, 0, 0, 0, 0, 318, 0, 4095, 13, 155, 0, 0, 0, 125, 1510, 1147, 35, 0, 0, 0, 0, 0, 0, 13, 0, 2495, 835, 0, 0, 1, 224, 442, 0, 2949, 430, 0, 0, 3893, 0, 0, 36, 173, 0, 0, 0] rerouted_tokens_per_ep_rank=[5696, 2220, 5963, 4581, 2817, 3343, 4046, 4102] avg=4096.0000 max=5963 max_rank=2 min=2220 min_rank=1 imbalance_ratio=1.455811 max_load_reduction=26805 max_load_reduction_pct=0.818024 moved_to_echo=14308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[441, 188, 671, 39, 1128, 0, 0, 0, 163, 206, 0, 20, 0, 0, 0, 0, 0, 3401, 1, 908, 0, 0, 0, 0, 39, 249, 3988, 0, 127, 0, 0, 0, 3186, 0, 1082, 3901, 0, 0, 0, 0, 0, 666, 0, 0, 2770, 685, 0, 0, 0, 0, 0, 1133, 2781, 477, 0, 0, 4095, 0, 281, 0, 142, 0, 0, 0] rerouted_tokens_per_ep_rank=[2467, 389, 4310, 4403, 8169, 4121, 4391, 4518] avg=4096.0000 max=8169 max_rank=4 min=389 min_rank=1 imbalance_ratio=1.994385 max_load_reduction=24599 max_load_reduction_pct=0.750702 moved_to_echo=21199 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[109, 0, 3549, 0, 192, 0, 0, 0, 4090, 85, 1, 392, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 922, 0, 4063, 38, 21, 0, 0, 0, 1, 4083, 900, 1, 0, 0, 0, 0, 0, 0, 4069, 0, 1130, 117, 0, 0, 11, 10, 57, 0, 2311, 194, 0, 0, 0, 0, 3674, 2567, 24, 0, 0, 0] rerouted_tokens_per_ep_rank=[3850, 4568, 157, 5044, 4985, 5316, 2583, 6265] avg=4096.0000 max=6265 max_rank=7 min=157 min_rank=2 imbalance_ratio=1.529541 max_load_reduction=26503 max_load_reduction_pct=0.808807 moved_to_echo=19149 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[46, 6, 0, 4, 964, 0, 0, 0, 1322, 237, 0, 0, 0, 0, 0, 0, 6, 0, 0, 777, 0, 0, 0, 0, 1181, 39, 4096, 61, 108, 0, 0, 0, 4, 4096, 1035, 0, 0, 0, 0, 0, 0, 1, 1391, 0, 3192, 585, 0, 0, 151, 470, 3130, 2, 2658, 550, 0, 0, 9, 3001, 2932, 593, 121, 0, 0, 0] rerouted_tokens_per_ep_rank=[1020, 1559, 783, 5485, 5135, 5169, 6961, 6656] avg=4096.0000 max=6961 max_rank=6 min=783 min_rank=2 imbalance_ratio=1.699463 max_load_reduction=25807 max_load_reduction_pct=0.787567 moved_to_echo=23921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[128, 10, 1182, 18, 1460, 0, 0, 0, 4010, 236, 0, 448, 0, 0, 0, 0, 0, 3779, 3556, 1153, 0, 0, 0, 0, 0, 0, 2441, 797, 168, 0, 0, 0, 1, 14, 1145, 30, 0, 0, 0, 0, 0, 2, 11, 0, 3289, 886, 0, 0, 2, 92, 30, 6, 2951, 571, 0, 0, 52, 0, 4096, 18, 186, 0, 0, 0] rerouted_tokens_per_ep_rank=[2798, 4694, 8488, 3406, 1190, 4188, 3652, 4352] avg=4096.0000 max=8488 max_rank=2 min=1190 min_rank=4 imbalance_ratio=2.072266 max_load_reduction=24280 max_load_reduction_pct=0.740967 moved_to_echo=13382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[52, 3026, 3816, 824, 1352, 0, 0, 0, 3748, 0, 29, 4058, 0, 0, 0, 0, 0, 0, 4096, 1087, 0, 0, 0, 0, 0, 692, 19, 1, 152, 0, 0, 0, 0, 38, 1147, 1025, 0, 0, 0, 0, 0, 300, 1, 60, 0, 821, 0, 0, 87, 8, 0, 0, 2949, 0, 0, 0, 2821, 381, 0, 8, 170, 0, 0, 0] rerouted_tokens_per_ep_rank=[9070, 7835, 5183, 864, 2210, 1182, 3044, 3380] avg=4096.0000 max=9070 max_rank=0 min=864 min_rank=3 imbalance_ratio=2.214355 max_load_reduction=23698 max_load_reduction_pct=0.723206 moved_to_echo=9816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3584, 13312, 8832, 7296, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8576, 3968, 10240, 1024, 9344, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[18432, 1152, 3968, 9472, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 12800, 12544, 7552, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1920, 896, 3840, 1152, 22528, 2688, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 1024, 8448, 2048, 15488, 5760, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11136, 3456, 13952, 3328, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3328, 1024, 25088, 2432, 1152, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[5, 3097, 4096, 1, 0, 0, 0, 0, 2, 4043, 3897, 0, 0, 0, 47, 0, 4055, 1679, 7, 4096, 0, 3888, 3, 0, 0, 0, 0, 3579, 270, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[18, 13, 3941, 0, 2203, 0, 0, 105, 0, 4012, 3202, 2192, 0, 0, 53, 0, 4015, 4096, 50, 3938, 0, 402, 0, 0, 0, 0, 0, 381, 51, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[14, 3, 0, 0, 4069, 0, 0, 3949, 2, 4096, 4039, 4051, 0, 0, 7, 2, 4084, 36, 0, 4093, 110, 72, 2, 0, 0, 0, 1, 41, 4096, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 2313, 3937, 6, 3, 0, 0, 145, 0, 4065, 1509, 272, 0, 23, 0, 786, 3914, 4031, 384, 4096, 0, 0, 2815, 0, 0, 8, 134, 3976, 286, 54, 11, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[13, 153, 3847, 0, 0, 0, 0, 0, 0, 3935, 3655, 486, 0, 0, 0, 0, 4077, 4096, 238, 4096, 3469, 2518, 0, 0, 0, 54, 0, 92, 1856, 9, 0, 174] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[7, 0, 16, 3795, 114, 56, 0, 148, 348, 1645, 0, 829, 1, 567, 2263, 1, 4038, 0, 0, 3750, 44, 0, 0, 16, 0, 0, 2154, 1516, 3010, 4096, 259, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1796, 1609, 2842, 24, 0, 0, 0, 265, 3, 4096, 2014, 77, 0, 0, 288, 0, 3925, 24, 6, 4096, 0, 0, 8, 0, 16, 0, 0, 3507, 4081, 0, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[37, 2882, 4080, 3, 1, 0, 0, 0, 1795, 4092, 0, 5, 150, 0, 3397, 0, 2624, 666, 443, 3927, 20, 8, 75, 10, 0, 3, 0, 4090, 368, 0, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[5, 3097, 3056, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1995, 882, 0, 0, 2, 593, 3897, 0, 0, 0, 0, 0, 0, 0, 47, 0, 3214, 0, 0, 0, 2246, 1679, 7, 0, 0, 0, 0, 0, 0, 3888, 3, 0, 1455, 1123, 0, 0, 0, 0, 0, 3579, 1040, 686, 42, 0, 228, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6159, 2877, 4492, 3261, 3932, 6469, 5347, 231] avg=4096.0000 max=6469 max_rank=5 min=231 min_rank=7 imbalance_ratio=1.579346 max_load_reduction=26299 max_load_reduction_pct=0.802582 moved_to_echo=15979 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[14, 3, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 3949, 2021, 882, 0, 0, 2, 601, 4039, 4051, 0, 0, 0, 0, 0, 0, 7, 2, 3211, 0, 0, 0, 2262, 36, 0, 0, 0, 0, 0, 0, 110, 72, 2, 0, 1474, 1131, 0, 0, 0, 0, 1, 41, 0, 691, 651, 0, 3445, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17, 10921, 8693, 3220, 2298, 2789, 1384, 3446] avg=4096.0000 max=10921 max_rank=1 min=17 min_rank=0 imbalance_ratio=2.666260 max_load_reduction=21847 max_load_reduction_pct=0.666718 moved_to_echo=9917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2313, 2938, 6, 0, 0, 0, 0, 3, 0, 0, 145, 2006, 881, 0, 0, 0, 596, 1509, 272, 0, 0, 0, 0, 0, 23, 0, 786, 3215, 0, 0, 0, 2168, 4031, 384, 0, 0, 0, 0, 0, 0, 0, 2815, 0, 1463, 1084, 0, 0, 0, 8, 134, 3976, 999, 662, 45, 0, 241, 54, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5257, 3035, 2377, 4024, 6583, 5362, 5824, 306] avg=4096.0000 max=6583 max_rank=4 min=306 min_rank=7 imbalance_ratio=1.607178 max_load_reduction=26185 max_load_reduction_pct=0.799103 moved_to_echo=18075 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[18, 13, 2938, 0, 0, 0, 0, 0, 2203, 0, 0, 105, 1985, 848, 0, 0, 0, 580, 3202, 2192, 0, 0, 0, 0, 0, 0, 53, 0, 3090, 0, 0, 0, 2214, 4096, 50, 0, 0, 0, 0, 0, 0, 402, 0, 0, 1447, 1116, 0, 0, 0, 0, 0, 381, 1003, 685, 12, 0, 39, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2969, 5141, 5974, 3143, 6360, 2965, 2081, 4135] avg=4096.0000 max=6360 max_rank=4 min=2081 min_rank=6 imbalance_ratio=1.552734 max_load_reduction=26408 max_load_reduction_pct=0.805908 moved_to_echo=15541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[13, 153, 2870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1942, 882, 0, 0, 0, 577, 3655, 486, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 0, 0, 0, 2258, 4096, 238, 0, 0, 0, 0, 0, 3469, 2518, 0, 0, 1416, 1129, 0, 0, 0, 54, 0, 92, 977, 690, 295, 0, 1561, 9, 0, 174, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3036, 2824, 4718, 3214, 6592, 8532, 2108, 1744] avg=4096.0000 max=8532 max_rank=5 min=1744 min_rank=7 imbalance_ratio=2.083008 max_load_reduction=24236 max_load_reduction_pct=0.739624 moved_to_echo=18976 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1796, 1609, 2121, 24, 0, 0, 0, 0, 0, 0, 0, 265, 2021, 881, 0, 0, 3, 601, 2014, 77, 0, 0, 0, 0, 0, 0, 288, 0, 3215, 0, 0, 0, 2174, 24, 6, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1474, 1087, 0, 0, 16, 0, 0, 3507, 721, 664, 649, 0, 3432, 0, 4091, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5550, 3167, 2695, 3503, 2204, 2569, 5557, 7523] avg=4096.0000 max=7523 max_rank=7 min=2204 min_rank=4 imbalance_ratio=1.836670 max_load_reduction=25245 max_load_reduction_pct=0.770416 moved_to_echo=17853 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[37, 2882, 3044, 3, 0, 0, 0, 0, 1, 0, 0, 0, 2019, 845, 0, 0, 1795, 601, 0, 5, 0, 0, 0, 0, 150, 0, 3397, 0, 3082, 0, 0, 0, 1453, 666, 443, 0, 0, 0, 0, 0, 20, 8, 75, 10, 1472, 727, 0, 0, 0, 3, 0, 4090, 1036, 444, 58, 0, 310, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5966, 2865, 2401, 6629, 2562, 2312, 5631, 4402] avg=4096.0000 max=6629 max_rank=3 min=2312 min_rank=5 imbalance_ratio=1.618408 max_load_reduction=26139 max_load_reduction_pct=0.797699 moved_to_echo=14907 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 12, 3795, 0, 0, 0, 0, 114, 56, 0, 148, 812, 808, 0, 0, 348, 241, 0, 829, 0, 0, 0, 0, 1, 567, 2263, 1, 2942, 0, 0, 0, 2237, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 16, 592, 1118, 0, 0, 0, 0, 2154, 1516, 4, 683, 479, 0, 2531, 4096, 259, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3814, 1938, 1418, 5774, 2237, 1770, 4836, 10981] avg=4096.0000 max=10981 max_rank=7 min=1418 min_rank=2 imbalance_ratio=2.680908 max_load_reduction=21787 max_load_reduction_pct=0.664886 moved_to_echo=19824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6400, 128, 0, 4736, 14848, 6912, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11904, 8320, 4480, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3712, 6912, 2944, 128, 10880, 8576, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2176, 4480, 18432, 7936, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[17024, 14720, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[1920, 10112, 17024, 3840, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 2304, 17280, 5888, 5248, 2304, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 640, 6144, 896, 25216, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[307, 716, 276, 3752, 0, 5, 3415, 0, 698, 0, 554, 0, 147, 4080, 0, 0, 0, 0, 0, 4018, 4038, 0, 1124, 2268, 0, 3922, 0, 3347, 3, 20, 4, 74] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[5, 914, 0, 1, 0, 0, 2026, 0, 0, 4096, 34, 427, 22, 1, 1479, 1, 2214, 946, 0, 1210, 3892, 6, 420, 4091, 0, 3568, 0, 0, 0, 0, 3978, 3437] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4, 4096, 0, 0, 8, 23, 3802, 0, 0, 840, 1359, 0, 16, 4056, 2183, 10, 0, 202, 0, 4063, 1856, 0, 0, 4018, 0, 1983, 0, 4080, 6, 150, 0, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1121, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 125, 7, 37, 0, 2362, 84, 492, 4003, 3518, 39, 1787, 2577, 0, 0, 0, 0, 98, 3159, 1013, 59] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 369, 9, 2045, 0, 0, 51, 0, 0, 4086, 4, 0, 2, 6, 0, 4, 0, 0, 53, 4044, 245, 0, 51, 4093, 16, 1367, 0, 4029, 4096, 0, 9, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 499, 4, 0, 0, 2286, 0, 4039, 0, 4095, 800, 7, 0, 0, 3940, 0, 945, 648, 0, 0, 3967, 0, 0, 3198, 0, 9, 0, 3611, 647, 4069, 1, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[3515, 758, 0, 0, 138, 3, 3595, 0, 66, 167, 775, 2, 2, 0, 3995, 86, 3996, 45, 0, 65, 513, 0, 0, 3629, 0, 3993, 0, 3287, 0, 42, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[367, 4022, 68, 0, 0, 248, 4096, 0, 0, 3448, 4084, 6, 1643, 0, 0, 8, 4096, 1, 1128, 791, 0, 1, 0, 0, 1, 0, 0, 3923, 4084, 700, 0, 53] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 369, 9, 2045, 394, 130, 0, 0, 0, 0, 51, 0, 595, 282, 0, 0, 0, 4086, 4, 0, 455, 194, 0, 0, 2, 6, 0, 4, 1871, 0, 0, 0, 0, 0, 53, 3459, 0, 0, 0, 0, 245, 0, 51, 1940, 0, 0, 0, 0, 16, 1367, 0, 3240, 0, 0, 0, 0, 4096, 0, 9, 3699, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7043, 928, 4739, 1883, 3512, 2236, 4623, 7804] avg=4096.0000 max=7804 max_rank=7 min=928 min_rank=1 imbalance_ratio=1.905273 max_load_reduction=24964 max_load_reduction_pct=0.761841 moved_to_echo=18175 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 4096, 0, 0, 1, 131, 0, 0, 8, 23, 3802, 0, 602, 276, 0, 0, 0, 840, 1359, 0, 457, 197, 0, 0, 16, 4056, 2183, 10, 1837, 0, 0, 0, 0, 202, 0, 3475, 0, 0, 0, 0, 1856, 0, 0, 1905, 0, 0, 0, 0, 0, 1983, 0, 3281, 0, 0, 0, 0, 6, 150, 0, 12, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4232, 4711, 2853, 8102, 3677, 3761, 5264, 168] avg=4096.0000 max=8102 max_rank=3 min=168 min_rank=7 imbalance_ratio=1.978027 max_load_reduction=24666 max_load_reduction_pct=0.752747 moved_to_echo=12870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5, 914, 0, 1, 330, 39, 0, 0, 0, 0, 2026, 0, 0, 281, 0, 0, 0, 4096, 34, 427, 136, 0, 0, 0, 22, 1, 1479, 1, 1870, 0, 0, 0, 2214, 946, 0, 1035, 0, 0, 0, 0, 3892, 6, 420, 1940, 0, 0, 0, 0, 0, 3568, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 3107, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1289, 2307, 4693, 3373, 4195, 6258, 3568, 7085] avg=4096.0000 max=7085 max_rank=7 min=1289 min_rank=0 imbalance_ratio=1.729736 max_load_reduction=25683 max_load_reduction_pct=0.783783 moved_to_echo=21106 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1121, 4096, 0, 0, 5, 129, 0, 0, 0, 0, 4096, 0, 0, 177, 0, 0, 0, 0, 4095, 0, 450, 0, 0, 0, 125, 7, 37, 0, 1178, 0, 0, 0, 2362, 84, 492, 3424, 0, 0, 0, 0, 3518, 39, 1787, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 3159, 1013, 54, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5351, 4273, 4545, 1347, 6362, 6566, 0, 4324] avg=4096.0000 max=6566 max_rank=5 min=0 min_rank=6 imbalance_ratio=1.603027 max_load_reduction=26202 max_load_reduction_pct=0.799622 moved_to_echo=17252 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[307, 716, 276, 3752, 10, 133, 0, 0, 0, 5, 3415, 0, 498, 159, 0, 0, 698, 0, 554, 0, 455, 164, 0, 0, 147, 4080, 0, 0, 1041, 0, 0, 0, 0, 0, 0, 3430, 0, 0, 0, 0, 4038, 0, 1124, 1068, 0, 0, 0, 0, 0, 3922, 0, 2685, 0, 0, 0, 0, 3, 20, 4, 64, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5194, 4077, 1871, 5268, 3430, 6230, 6607, 91] avg=4096.0000 max=6607 max_rank=6 min=91 min_rank=7 imbalance_ratio=1.613037 max_load_reduction=26161 max_load_reduction_pct=0.798370 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[367, 4022, 68, 0, 5, 25, 0, 0, 0, 248, 4096, 0, 579, 0, 0, 0, 0, 3448, 4084, 6, 89, 189, 0, 0, 1643, 0, 0, 8, 0, 0, 0, 0, 4096, 1, 1128, 677, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3155, 0, 0, 0, 0, 4084, 700, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4487, 4923, 7816, 1651, 5902, 1, 3156, 4832] avg=4096.0000 max=7816 max_rank=2 min=1 min_rank=5 imbalance_ratio=1.908203 max_load_reduction=24952 max_load_reduction_pct=0.761475 moved_to_echo=13891 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3515, 758, 0, 0, 394, 2, 0, 0, 138, 3, 3595, 0, 485, 250, 0, 0, 66, 167, 775, 2, 7, 158, 0, 0, 2, 0, 3995, 86, 1659, 0, 0, 0, 3996, 45, 0, 56, 0, 0, 0, 0, 513, 0, 0, 1720, 0, 0, 0, 0, 0, 3993, 0, 2644, 0, 0, 0, 0, 0, 42, 0, 3702, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4669, 4471, 1175, 5742, 4097, 2233, 6637, 3744] avg=4096.0000 max=6637 max_rank=6 min=1175 min_rank=2 imbalance_ratio=1.620361 max_load_reduction=26131 max_load_reduction_pct=0.797455 moved_to_echo=16711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 499, 4, 0, 0, 0, 0, 0, 0, 2286, 0, 4039, 533, 220, 0, 0, 0, 4095, 800, 7, 0, 174, 0, 0, 0, 0, 3940, 0, 1462, 0, 0, 0, 945, 648, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 1516, 0, 0, 0, 0, 0, 9, 0, 2904, 0, 0, 0, 0, 647, 4069, 1, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[503, 7078, 5076, 5402, 1593, 5483, 2913, 4720] avg=4096.0000 max=7078 max_rank=1 min=503 min_rank=0 imbalance_ratio=1.728027 max_load_reduction=25690 max_load_reduction_pct=0.783997 moved_to_echo=14709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8960, 8192, 5120, 10752, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 14848, 0, 17920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2048, 8192, 11648, 128, 11008, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[18048, 128, 3456, 11392, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 16768, 11776, 512, 2176, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9472, 15488, 384, 5888, 1152, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 2688, 21120, 4096, 3328, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13696, 2048, 1792, 15616, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 3901, 2, 1609, 4053, 60, 0, 0, 3544, 3222, 3998, 48, 17, 1, 0, 0, 2011, 0, 0, 0, 0, 544, 985, 10, 0, 0, 67, 556, 4070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1758, 0, 4091, 0, 4085, 2049, 1786, 3980, 0, 154, 0, 3368, 233, 2759, 0, 340, 0, 0, 0, 3981, 3995, 0, 0, 0, 65, 4, 0, 0, 8, 1, 14, 97] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4075, 22, 3851, 3303, 0, 103, 1961, 318, 0, 512, 0, 4053, 0, 2344, 3826, 0, 0, 4095, 654, 0, 3605, 30, 0, 0, 0, 6, 0, 0, 10, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 479, 850, 4096, 3807, 3915, 3678, 3255, 3947, 0, 0, 6, 4096, 3, 340, 18, 2, 0, 4045, 78, 4, 0, 0, 0, 0, 0, 130, 0, 0, 19, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1639, 63, 0, 58, 4066, 1, 2134, 3557, 3335, 5, 0, 3794, 35, 47, 0, 4046, 2665, 7, 0, 2646, 0, 0, 0, 0, 5, 397, 0, 218, 1, 0, 0, 4049] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 3, 867, 4096, 3793, 0, 4096, 264, 80, 0, 3, 0, 4088, 0, 4093, 0, 0, 0, 4059, 4074, 54, 0, 0, 0, 0, 0, 245, 8, 0, 2945, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[2759, 0, 1849, 0, 3419, 3775, 4096, 1, 4088, 11, 302, 18, 0, 0, 0, 22, 0, 0, 0, 4091, 3726, 0, 17, 0, 56, 2, 0, 0, 47, 2, 391, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 354, 933, 2901, 0, 4069, 259, 1503, 168, 3, 0, 994, 367, 0, 2845, 105, 454, 0, 0, 4079, 0, 0, 6, 472, 0, 3938, 25, 4096, 917, 0, 4070, 210] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4075, 22, 753, 347, 0, 0, 0, 3013, 0, 0, 0, 0, 0, 0, 1961, 318, 0, 512, 290, 106, 0, 0, 0, 3130, 0, 2344, 0, 0, 0, 0, 3826, 0, 0, 3642, 0, 0, 0, 0, 654, 0, 3605, 30, 52, 766, 0, 0, 0, 0, 0, 6, 3085, 0, 0, 0, 0, 0, 10, 0, 51, 170, 0, 0] rerouted_tokens_per_ep_rank=[5197, 3013, 3187, 5474, 7468, 5107, 3091, 231] avg=4096.0000 max=7468 max_rank=4 min=231 min_rank=7 imbalance_ratio=1.823242 max_load_reduction=25300 max_load_reduction_pct=0.772095 moved_to_echo=15897 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1639, 63, 0, 58, 8, 224, 0, 0, 0, 1, 2134, 0, 0, 0, 0, 0, 3335, 5, 0, 3794, 0, 68, 0, 0, 35, 38, 0, 4046, 0, 0, 0, 0, 2665, 7, 0, 2354, 0, 0, 0, 0, 0, 0, 0, 0, 1794, 808, 0, 0, 5, 397, 0, 218, 3258, 0, 0, 0, 1, 0, 0, 4049, 1763, 1, 0, 0] rerouted_tokens_per_ep_rank=[1992, 2135, 7202, 4119, 5026, 2602, 3878, 5814] avg=4096.0000 max=7202 max_rank=2 min=1992 min_rank=0 imbalance_ratio=1.758301 max_load_reduction=25566 max_load_reduction_pct=0.780212 moved_to_echo=17320 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1758, 0, 4091, 0, 512, 337, 0, 0, 0, 1869, 1786, 0, 0, 0, 0, 0, 0, 154, 0, 3368, 180, 103, 0, 0, 233, 2132, 0, 340, 0, 0, 0, 0, 0, 0, 0, 3541, 0, 0, 0, 0, 3995, 0, 0, 0, 2009, 813, 0, 0, 65, 4, 0, 0, 3272, 0, 0, 0, 8, 1, 14, 97, 1971, 115, 0, 0] rerouted_tokens_per_ep_rank=[6698, 3655, 3805, 2705, 3541, 6817, 3341, 2206] avg=4096.0000 max=6817 max_rank=5 min=2206 min_rank=7 imbalance_ratio=1.664307 max_load_reduction=25951 max_load_reduction_pct=0.791962 moved_to_echo=15905 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 743, 170, 0, 0, 0, 2, 1609, 0, 0, 0, 0, 0, 60, 0, 0, 3544, 0, 52, 0, 0, 3222, 3088, 48, 17, 0, 0, 0, 0, 1, 0, 0, 1789, 0, 0, 0, 0, 0, 0, 0, 0, 2044, 776, 0, 0, 544, 985, 10, 0, 3125, 0, 0, 0, 0, 67, 556, 4070, 2009, 167, 0, 0] rerouted_tokens_per_ep_rank=[4983, 1611, 3656, 6375, 1790, 2820, 4664, 6869] avg=4096.0000 max=6869 max_rank=7 min=1611 min_rank=1 imbalance_ratio=1.677002 max_load_reduction=25899 max_load_reduction_pct=0.790375 moved_to_echo=16143 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 479, 850, 761, 342, 0, 0, 0, 3473, 3915, 0, 0, 0, 0, 0, 3255, 3947, 0, 0, 334, 105, 0, 0, 6, 3163, 3, 340, 0, 0, 0, 0, 18, 2, 0, 3598, 0, 0, 0, 0, 78, 4, 0, 0, 1856, 815, 0, 0, 0, 0, 0, 130, 3281, 0, 0, 0, 0, 0, 19, 0, 1822, 172, 0, 0] rerouted_tokens_per_ep_rank=[2432, 7388, 7641, 3512, 3618, 2753, 3411, 2013] avg=4096.0000 max=7641 max_rank=2 min=2013 min_rank=7 imbalance_ratio=1.865479 max_load_reduction=25127 max_load_reduction_pct=0.766815 moved_to_echo=11795 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2759, 0, 1849, 0, 0, 346, 0, 0, 0, 3443, 4096, 0, 0, 0, 0, 0, 4088, 11, 302, 18, 332, 106, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 3639, 0, 0, 0, 0, 3726, 0, 17, 0, 0, 679, 0, 0, 56, 2, 0, 0, 2740, 0, 0, 0, 47, 2, 391, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4954, 7539, 4857, 22, 3639, 4422, 2798, 4537] avg=4096.0000 max=7539 max_rank=1 min=22 min_rank=3 imbalance_ratio=1.840576 max_load_reduction=25229 max_load_reduction_pct=0.769928 moved_to_echo=15396 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 867, 762, 347, 0, 0, 0, 3457, 0, 0, 0, 0, 0, 0, 264, 80, 0, 3, 336, 108, 0, 0, 0, 3151, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3604, 0, 0, 0, 0, 4074, 54, 0, 0, 2067, 815, 0, 0, 0, 0, 0, 245, 3281, 0, 0, 0, 8, 0, 2945, 0, 2029, 175, 0, 0] rerouted_tokens_per_ep_rank=[1979, 3457, 791, 7244, 3604, 7010, 3526, 5157] avg=4096.0000 max=7244 max_rank=3 min=791 min_rank=2 imbalance_ratio=1.768555 max_load_reduction=25524 max_load_reduction_pct=0.778931 moved_to_echo=19297 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 354, 933, 2901, 0, 345, 0, 0, 0, 3711, 259, 0, 0, 0, 0, 0, 168, 3, 0, 994, 358, 106, 0, 0, 367, 0, 2845, 105, 0, 0, 0, 0, 454, 0, 0, 3628, 0, 0, 0, 0, 0, 0, 6, 472, 759, 0, 0, 0, 0, 3938, 25, 4096, 0, 0, 0, 0, 917, 0, 4070, 210, 744, 0, 0, 0] rerouted_tokens_per_ep_rank=[4533, 3970, 1629, 3317, 4082, 1237, 8059, 5941] avg=4096.0000 max=8059 max_rank=6 min=1237 min_rank=5 imbalance_ratio=1.967529 max_load_reduction=24709 max_load_reduction_pct=0.754059 moved_to_echo=19319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12544, 128, 3712, 512, 10624, 5504, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10240, 512, 11520, 4736, 3584, 2560, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 128, 8064, 12544, 10496, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3968, 14720, 2944, 11392, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7040, 128, 0, 25856, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13184, 4608, 384, 12288, 1920, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 19072, 13824, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 5376, 128, 4736, 22144, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4026, 497, 91, 40, 94, 3437, 0, 0, 0, 0, 1, 0, 1, 0, 966, 0, 358, 123, 0, 2, 3846, 0, 0, 0, 4095, 4096, 2959, 4096, 0, 4040, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 2210, 957, 102, 2, 0, 0, 0, 0, 0, 3615, 81, 0, 5, 0, 630, 3268, 0, 3291, 3869, 3495, 40, 559, 3486, 0, 0, 0, 2528, 0, 429, 105] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 3563, 4038, 1, 306, 96, 0, 0, 0, 0, 25, 0, 0, 161, 195, 302, 0, 506, 4094, 0, 0, 4083, 35, 0, 9, 3860, 4096, 0, 3468, 0, 3929, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 1170, 45, 1, 23, 0, 0, 0, 0, 0, 45, 532, 0, 2, 0, 0, 3809, 8, 1, 4090, 4078, 4060, 3651, 0, 0, 2710, 147, 4062, 0, 224, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 3829, 3321, 65, 6, 0, 776, 1166, 25, 0, 133, 1, 0, 1, 2839, 0, 1, 4095, 147, 0, 76, 3929, 0, 0, 19, 361, 4096, 0, 3796, 0, 4086, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 3582, 1, 200, 4, 0, 0, 198, 0, 4092, 4, 3247, 2, 196, 0, 3796, 0, 18, 183, 4096, 872, 2, 0, 4023, 0, 30, 3946, 0, 120, 60] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 1077, 0, 47, 0, 0, 409, 1337, 292, 4059, 0, 3597, 69, 0, 14, 3716, 25, 0, 15, 0, 3467, 2360, 0, 0, 190, 0, 4079, 0, 3974, 0, 3919, 122] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 8, 163, 62, 1927, 1665, 0, 4043, 76, 38, 828, 0, 66, 1, 159, 0, 544, 4087, 0, 0, 3995, 4096, 57, 0, 151, 0, 3888, 0, 3222, 0, 0, 3692] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2731, 2210, 957, 0, 0, 0, 0, 102, 2, 0, 0, 1318, 228, 0, 0, 0, 0, 0, 3615, 0, 713, 164, 0, 81, 0, 5, 0, 2355, 0, 0, 0, 630, 3268, 0, 3291, 652, 0, 0, 0, 3869, 912, 40, 559, 0, 0, 0, 0, 3486, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 429, 105, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5898, 1650, 4492, 2441, 7841, 5380, 3486, 1580] avg=4096.0000 max=7841 max_rank=4 min=1580 min_rank=7 imbalance_ratio=1.914307 max_load_reduction=24927 max_load_reduction_pct=0.760712 moved_to_echo=18287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2684, 497, 91, 0, 0, 0, 0, 40, 94, 3437, 0, 2135, 251, 0, 0, 0, 0, 0, 1, 1699, 701, 266, 0, 0, 1, 0, 966, 2592, 0, 0, 0, 0, 358, 123, 0, 641, 0, 0, 0, 2, 1003, 0, 0, 0, 0, 0, 0, 0, 4095, 2397, 2959, 0, 0, 0, 0, 1695, 0, 4040, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3272, 5957, 2667, 3559, 1122, 1005, 9451, 5735] avg=4096.0000 max=9451 max_rank=6 min=1005 min_rank=5 imbalance_ratio=2.307373 max_load_reduction=23317 max_load_reduction_pct=0.711578 moved_to_echo=17313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2731, 1170, 45, 0, 0, 0, 0, 1, 23, 0, 0, 2117, 267, 0, 0, 0, 0, 0, 45, 1124, 713, 264, 0, 532, 0, 2, 0, 2748, 0, 0, 0, 0, 3809, 8, 1, 652, 0, 0, 0, 4090, 1063, 4060, 3651, 0, 0, 0, 0, 0, 0, 1586, 147, 0, 0, 0, 0, 1681, 0, 224, 14, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3946, 2408, 2146, 3282, 4470, 12864, 1733, 1919] avg=4096.0000 max=12864 max_rank=5 min=1733 min_rank=6 imbalance_ratio=3.140625 max_load_reduction=19904 max_load_reduction_pct=0.607422 moved_to_echo=20986 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 719, 0, 47, 0, 0, 0, 0, 0, 0, 409, 1337, 2072, 154, 0, 0, 292, 4059, 0, 3597, 1692, 187, 258, 0, 69, 0, 14, 3716, 1590, 0, 0, 0, 25, 0, 15, 0, 171, 0, 0, 0, 3467, 616, 0, 0, 0, 0, 0, 0, 190, 0, 2387, 0, 0, 0, 0, 0, 1644, 0, 3919, 122, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[766, 3972, 10085, 5389, 211, 4083, 2577, 5685] avg=4096.0000 max=10085 max_rank=2 min=211 min_rank=4 imbalance_ratio=2.462158 max_load_reduction=22683 max_load_reduction_pct=0.692230 moved_to_echo=12556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2376, 4038, 1, 0, 0, 0, 0, 306, 96, 0, 0, 1808, 267, 0, 0, 0, 0, 25, 0, 1699, 620, 226, 0, 0, 161, 195, 302, 2752, 0, 0, 0, 0, 506, 4094, 0, 567, 0, 0, 0, 0, 1064, 35, 0, 0, 0, 0, 0, 9, 3860, 2397, 0, 0, 0, 0, 0, 1434, 0, 3929, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6415, 2477, 2570, 3410, 5167, 1099, 6266, 5364] avg=4096.0000 max=6415 max_rank=0 min=1099 min_rank=5 imbalance_ratio=1.566162 max_load_reduction=26353 max_load_reduction_pct=0.804230 moved_to_echo=17896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2552, 3321, 65, 0, 0, 0, 0, 6, 0, 776, 1166, 1979, 257, 0, 0, 25, 0, 133, 1, 1699, 667, 247, 0, 0, 1, 2839, 0, 2648, 0, 0, 0, 1, 4095, 147, 0, 610, 0, 0, 0, 76, 1024, 0, 0, 0, 0, 0, 0, 19, 361, 2397, 0, 0, 0, 0, 0, 1570, 0, 4086, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5938, 4184, 2772, 5488, 4853, 1100, 2777, 5656] avg=4096.0000 max=5938 max_rank=0 min=1100 min_rank=5 imbalance_ratio=1.449707 max_load_reduction=26830 max_load_reduction_pct=0.818787 moved_to_echo=14386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2720, 0, 3582, 0, 0, 0, 0, 1, 200, 4, 0, 2061, 272, 0, 0, 0, 198, 0, 4092, 0, 718, 262, 0, 4, 3247, 2, 196, 2764, 0, 0, 0, 0, 3796, 0, 18, 658, 0, 0, 0, 183, 1060, 872, 2, 0, 0, 0, 0, 0, 4023, 0, 30, 0, 0, 0, 0, 1623, 0, 120, 60, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6302, 2538, 5270, 6213, 4472, 2117, 4053, 1803] avg=4096.0000 max=6302 max_rank=0 min=1803 min_rank=7 imbalance_ratio=1.538574 max_load_reduction=26466 max_load_reduction_pct=0.807678 moved_to_echo=12445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 163, 62, 0, 0, 0, 0, 1927, 1665, 0, 4043, 1679, 268, 0, 0, 76, 38, 828, 0, 1614, 1, 209, 0, 66, 1, 159, 0, 2760, 0, 0, 0, 544, 4087, 0, 0, 1, 0, 0, 0, 3995, 1068, 57, 0, 0, 0, 0, 0, 151, 0, 2274, 0, 0, 0, 0, 0, 1334, 0, 0, 3692, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[231, 9582, 2766, 2986, 4632, 5120, 2425, 5026] avg=4096.0000 max=9582 max_rank=1 min=231 min_rank=0 imbalance_ratio=2.339355 max_load_reduction=23186 max_load_reduction_pct=0.707581 moved_to_echo=17203 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2432, 2176, 4736, 6656, 15232, 2048, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15744, 7936, 5120, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 4352, 1024, 11392, 9600, 4352, 1920, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 16640, 11520, 4864, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 19968, 4480, 3328, 3968, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[12032, 0, 16768, 4096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3968, 12416, 13440, 3200, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 3456, 3328, 5248, 20224, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[104, 2900, 558, 476, 1581, 417, 782, 1275, 1115, 179, 1756, 369, 837, 791, 221, 656, 325, 479, 994, 1279, 925, 185, 3147, 2371, 284, 1134, 874, 1270, 1331, 527, 1620, 2006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[119, 2861, 421, 345, 1670, 609, 679, 1392, 1095, 233, 1318, 239, 1117, 1278, 205, 383, 96, 687, 820, 797, 1484, 128, 3460, 2221, 458, 1181, 759, 1637, 896, 624, 1557, 1999] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[107, 3092, 508, 522, 1118, 355, 1034, 792, 1023, 122, 2096, 140, 873, 815, 333, 459, 212, 519, 1232, 528, 1046, 218, 3060, 2486, 693, 1577, 822, 1320, 1408, 469, 1845, 1944] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[76, 2849, 439, 329, 1164, 502, 636, 1371, 1218, 259, 1302, 217, 1351, 1344, 230, 429, 147, 487, 1745, 675, 1132, 209, 2537, 2033, 380, 1149, 905, 1319, 1153, 418, 2245, 2518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[220, 2995, 564, 418, 1036, 260, 910, 990, 1305, 189, 1845, 525, 672, 947, 622, 425, 71, 642, 1032, 640, 954, 360, 2930, 2116, 664, 1042, 1004, 1303, 1434, 655, 1967, 2031] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[153, 3374, 406, 372, 1462, 399, 756, 1441, 1192, 206, 1578, 229, 544, 895, 502, 494, 108, 761, 1242, 801, 1054, 228, 2333, 2381, 559, 1174, 694, 1631, 1400, 532, 1845, 2022] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[170, 2844, 1079, 248, 1651, 371, 960, 1336, 1245, 206, 1160, 313, 1047, 567, 198, 394, 158, 627, 902, 1151, 1170, 69, 3223, 1953, 666, 838, 1187, 1270, 1858, 290, 1639, 1978] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[102, 2935, 806, 427, 1114, 338, 800, 1036, 1760, 157, 1336, 494, 1084, 662, 189, 432, 99, 345, 1436, 1143, 1045, 172, 2731, 2254, 588, 791, 779, 935, 2064, 482, 2300, 1932] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[119, 2855, 421, 345, 0, 0, 0, 0, 1670, 609, 679, 1392, 307, 0, 0, 0, 1095, 233, 1318, 239, 772, 0, 0, 0, 1117, 1278, 205, 383, 1739, 0, 0, 0, 96, 687, 820, 797, 1045, 427, 0, 0, 1484, 128, 676, 2221, 0, 0, 0, 0, 458, 1181, 759, 1637, 222, 6, 0, 0, 896, 624, 1557, 271, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3740, 4657, 3657, 4722, 3872, 4509, 4263, 3348] avg=4096.0000 max=4722 max_rank=3 min=3348 min_rank=7 imbalance_ratio=1.152832 max_load_reduction=28046 max_load_reduction_pct=0.855896 moved_to_echo=15992 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[104, 2894, 558, 476, 0, 0, 0, 0, 1581, 417, 782, 1275, 309, 0, 0, 0, 1115, 179, 1756, 369, 774, 0, 0, 0, 837, 791, 221, 656, 1581, 0, 0, 0, 325, 479, 994, 1279, 950, 428, 0, 0, 925, 185, 616, 2371, 0, 0, 0, 0, 284, 1134, 874, 1270, 223, 6, 0, 0, 1331, 527, 1620, 272, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4364, 4193, 4086, 4455, 4097, 3791, 3750] avg=4096.0000 max=4455 max_rank=4 min=3750 min_rank=7 imbalance_ratio=1.087646 max_load_reduction=28313 max_load_reduction_pct=0.864044 moved_to_echo=16093 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[107, 3086, 508, 522, 0, 0, 0, 0, 1118, 355, 1034, 792, 299, 0, 0, 0, 1023, 122, 2096, 140, 750, 0, 0, 0, 873, 815, 333, 459, 1538, 0, 0, 0, 212, 519, 1232, 528, 924, 415, 0, 0, 1046, 218, 598, 2486, 0, 0, 0, 0, 693, 1577, 822, 1320, 216, 6, 0, 0, 1408, 469, 1845, 264, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4223, 3598, 4131, 4018, 3830, 4348, 4634, 3986] avg=4096.0000 max=4634 max_rank=6 min=3598 min_rank=1 imbalance_ratio=1.131348 max_load_reduction=28134 max_load_reduction_pct=0.858582 moved_to_echo=16798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[76, 2843, 439, 329, 0, 0, 0, 0, 1164, 502, 636, 1371, 387, 0, 0, 0, 1218, 259, 1302, 217, 972, 0, 0, 0, 1351, 1344, 230, 429, 1275, 0, 0, 0, 147, 487, 1745, 675, 766, 538, 0, 0, 1132, 209, 496, 2033, 0, 0, 0, 0, 380, 1149, 905, 1319, 280, 6, 0, 0, 1153, 418, 2245, 341, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3687, 4060, 3968, 4629, 4358, 3870, 4039, 4157] avg=4096.0000 max=4629 max_rank=3 min=3687 min_rank=0 imbalance_ratio=1.130127 max_load_reduction=28139 max_load_reduction_pct=0.858734 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[153, 3367, 406, 372, 0, 0, 0, 0, 1462, 399, 756, 1441, 311, 0, 0, 0, 1192, 206, 1578, 229, 781, 0, 0, 0, 544, 895, 502, 494, 1172, 0, 0, 0, 108, 761, 1242, 801, 704, 432, 0, 0, 1054, 228, 457, 2381, 0, 0, 0, 0, 559, 1174, 694, 1631, 225, 7, 0, 0, 1400, 532, 1845, 273, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4298, 4369, 3986, 3607, 4048, 4120, 4290, 4050] avg=4096.0000 max=4369 max_rank=1 min=3607 min_rank=3 imbalance_ratio=1.066650 max_load_reduction=28399 max_load_reduction_pct=0.866669 moved_to_echo=16508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[170, 2836, 1079, 248, 0, 0, 0, 0, 1651, 371, 960, 1336, 309, 0, 0, 0, 1245, 206, 1160, 313, 768, 0, 0, 0, 1047, 567, 198, 394, 1623, 0, 0, 0, 158, 627, 902, 1151, 977, 426, 0, 0, 1170, 69, 623, 1953, 0, 0, 0, 0, 666, 838, 1187, 1270, 223, 8, 0, 0, 1858, 290, 1639, 252, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4333, 4627, 3692, 3829, 4241, 3815, 4192, 4039] avg=4096.0000 max=4627 max_rank=1 min=3692 min_rank=2 imbalance_ratio=1.129639 max_load_reduction=28141 max_load_reduction_pct=0.858795 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[220, 2989, 564, 418, 0, 0, 0, 0, 1036, 260, 910, 990, 312, 0, 0, 0, 1305, 189, 1845, 525, 784, 0, 0, 0, 672, 947, 622, 425, 1472, 0, 0, 0, 71, 642, 1032, 640, 885, 434, 0, 0, 954, 360, 573, 2116, 0, 0, 0, 0, 664, 1042, 1004, 1303, 226, 6, 0, 0, 1434, 655, 1967, 275, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 3508, 4648, 4138, 3704, 4003, 4245, 4331] avg=4096.0000 max=4648 max_rank=2 min=3508 min_rank=1 imbalance_ratio=1.134766 max_load_reduction=28120 max_load_reduction_pct=0.858154 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[102, 2929, 806, 427, 0, 0, 0, 0, 1114, 338, 800, 1036, 297, 0, 0, 0, 1760, 157, 1336, 494, 746, 0, 0, 0, 1084, 662, 189, 432, 1372, 0, 0, 0, 99, 345, 1436, 1143, 824, 413, 0, 0, 1045, 172, 535, 2254, 0, 0, 0, 0, 588, 791, 779, 935, 215, 6, 0, 0, 2064, 482, 2300, 261, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4264, 3585, 4493, 3739, 4260, 4006, 3314, 5107] avg=4096.0000 max=5107 max_rank=7 min=3314 min_rank=6 imbalance_ratio=1.246826 max_load_reduction=27661 max_load_reduction_pct=0.844147 moved_to_echo=16687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10880, 3328, 6656, 9728, 2560, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4352, 8960, 7040, 10752, 1920, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7552, 7424, 2560, 3712, 11776, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11648, 4096, 15104, 2304, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1280, 4608, 9472, 7040, 7168, 3584, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8832, 1664, 4608, 17920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9984, 1664, 12416, 2560, 6400, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 23808, 4864, 3200, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[223, 3, 525, 2, 2270, 3648, 3156, 387, 188, 1, 1029, 1851, 24, 3919, 0, 441, 60, 115, 0, 4065, 3, 14, 642, 143, 446, 169, 1967, 3518, 3854, 1, 103, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[409, 2, 2233, 143, 2328, 3949, 1445, 3227, 210, 2, 500, 1315, 13, 3810, 0, 511, 40, 83, 0, 4036, 19, 11, 0, 84, 135, 129, 178, 3902, 4053, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[14, 0, 2885, 2, 3315, 2275, 54, 3397, 323, 33, 1130, 107, 228, 3864, 14, 1859, 0, 85, 27, 4054, 88, 5, 70, 19, 20, 720, 257, 3866, 4035, 2, 15, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2573, 0, 1663, 78, 952, 2244, 2113, 2245, 21, 0, 369, 1443, 4, 1849, 7, 2572, 65, 57, 449, 4089, 5, 14, 15, 141, 654, 25, 915, 3918, 4088, 14, 88, 98] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[190, 113, 1096, 22, 226, 2581, 1026, 3188, 1, 32, 184, 2693, 206, 2264, 0, 1838, 3, 447, 3, 3848, 34, 0, 51, 0, 100, 1322, 2977, 4089, 3925, 0, 309, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[204, 13, 397, 10, 1357, 2682, 160, 3630, 90, 4, 522, 2552, 178, 1849, 263, 1556, 32, 1816, 16, 4069, 2, 1019, 5, 4, 1032, 158, 796, 3929, 4074, 0, 214, 135] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[24, 0, 3316, 12, 560, 3708, 188, 2509, 4, 1, 137, 1409, 0, 3362, 4, 3447, 7, 1196, 0, 4010, 19, 191, 0, 1, 410, 73, 404, 4091, 3663, 8, 0, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[899, 0, 2328, 61, 3383, 3234, 129, 2922, 294, 0, 1379, 985, 46, 2615, 6, 1677, 23, 147, 4, 3993, 4, 13, 0, 16, 36, 9, 360, 3971, 3959, 0, 130, 145] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[223, 3, 525, 2, 942, 514, 61, 20, 2270, 0, 3156, 183, 0, 0, 0, 0, 188, 1, 1029, 1851, 143, 673, 0, 0, 24, 2977, 0, 441, 0, 0, 0, 0, 60, 115, 0, 3551, 0, 0, 0, 0, 3, 14, 642, 143, 3648, 653, 0, 0, 446, 169, 1967, 2192, 0, 0, 0, 0, 3834, 1, 103, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2290, 5609, 3885, 3442, 3726, 5103, 4774, 3939] avg=4096.0000 max=5609 max_rank=1 min=2290 min_rank=0 imbalance_ratio=1.369385 max_load_reduction=27159 max_load_reduction_pct=0.828827 moved_to_echo=17542 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2573, 0, 1663, 78, 444, 517, 358, 21, 952, 0, 2113, 1056, 0, 0, 0, 0, 21, 0, 369, 1443, 831, 750, 0, 0, 4, 1405, 7, 2572, 0, 0, 0, 0, 65, 57, 449, 3572, 0, 0, 0, 0, 5, 14, 15, 141, 2244, 728, 0, 0, 654, 25, 915, 2440, 0, 0, 0, 0, 4067, 14, 88, 98, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5654, 4121, 3414, 3988, 4143, 3147, 4034, 4267] avg=4096.0000 max=5654 max_rank=0 min=3147 min_rank=5 imbalance_ratio=1.380371 max_load_reduction=27114 max_load_reduction_pct=0.827454 moved_to_echo=15591 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[190, 113, 1096, 22, 544, 487, 508, 20, 226, 0, 1026, 1500, 0, 0, 0, 0, 1, 32, 184, 2693, 1180, 783, 0, 0, 206, 1720, 0, 1838, 0, 0, 0, 0, 3, 447, 3, 3361, 0, 0, 0, 0, 34, 0, 51, 0, 2581, 760, 0, 0, 100, 1322, 2977, 2546, 0, 0, 0, 0, 3905, 0, 309, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2980, 2752, 4873, 3764, 3814, 3426, 6945, 4214] avg=4096.0000 max=6945 max_rank=6 min=2752 min_rank=1 imbalance_ratio=1.695557 max_load_reduction=25823 max_load_reduction_pct=0.788055 moved_to_echo=18399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 2885, 2, 929, 513, 542, 21, 3315, 0, 54, 1597, 0, 0, 0, 0, 323, 33, 1130, 107, 1258, 740, 0, 0, 228, 2935, 14, 1859, 0, 0, 0, 0, 0, 85, 27, 3541, 0, 0, 0, 0, 88, 5, 70, 19, 2275, 718, 0, 0, 20, 720, 257, 2408, 0, 0, 0, 0, 4014, 2, 15, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4906, 4966, 3591, 5036, 3653, 3175, 3405, 4036] avg=4096.0000 max=5036 max_rank=3 min=3175 min_rank=5 imbalance_ratio=1.229492 max_load_reduction=27732 max_load_reduction_pct=0.846313 moved_to_echo=14269 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[409, 2, 2233, 143, 916, 510, 515, 21, 2328, 0, 1445, 1517, 0, 0, 0, 0, 210, 2, 500, 1315, 1195, 747, 0, 0, 13, 2894, 0, 511, 0, 0, 0, 0, 40, 83, 0, 3526, 0, 0, 0, 0, 19, 11, 0, 84, 3949, 725, 0, 0, 135, 129, 178, 2430, 0, 0, 0, 0, 4032, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4749, 5290, 3969, 3418, 3649, 4788, 2872, 4033] avg=4096.0000 max=5290 max_rank=1 min=2872 min_rank=6 imbalance_ratio=1.291504 max_load_reduction=27478 max_load_reduction_pct=0.838562 moved_to_echo=15342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[899, 0, 2328, 61, 628, 505, 466, 20, 3383, 0, 129, 1374, 0, 0, 0, 0, 294, 0, 1379, 985, 1082, 760, 0, 0, 46, 1987, 6, 1677, 0, 0, 0, 0, 23, 147, 4, 3488, 0, 0, 0, 0, 4, 13, 0, 16, 3234, 738, 0, 0, 36, 9, 360, 2473, 0, 0, 0, 0, 3939, 0, 130, 145, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4907, 4886, 4500, 3716, 3662, 4005, 2878, 4214] avg=4096.0000 max=4907 max_rank=0 min=2878 min_rank=6 imbalance_ratio=1.197998 max_load_reduction=27861 max_load_reduction_pct=0.850250 moved_to_echo=14759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[204, 13, 397, 10, 447, 518, 583, 24, 1357, 0, 160, 1699, 0, 0, 0, 0, 90, 4, 522, 2552, 1348, 757, 0, 0, 178, 1402, 263, 1556, 0, 0, 0, 0, 32, 1816, 16, 3551, 0, 0, 0, 0, 2, 1019, 5, 4, 2682, 733, 0, 0, 1032, 158, 796, 2439, 0, 0, 0, 0, 4050, 0, 214, 135, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2196, 3216, 5273, 3399, 5415, 4445, 4425, 4399] avg=4096.0000 max=5415 max_rank=4 min=2196 min_rank=0 imbalance_ratio=1.322021 max_load_reduction=27353 max_load_reduction_pct=0.834747 moved_to_echo=18684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[24, 0, 3316, 12, 808, 507, 400, 19, 560, 0, 188, 1180, 0, 0, 0, 0, 4, 1, 137, 1409, 929, 783, 0, 0, 0, 2554, 4, 3447, 0, 0, 0, 0, 7, 1196, 0, 3503, 0, 0, 0, 0, 19, 191, 0, 1, 3708, 760, 0, 0, 410, 73, 404, 2548, 0, 0, 0, 0, 3644, 8, 0, 14, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5086, 1928, 3263, 6005, 4706, 4679, 3435, 3666] avg=4096.0000 max=6005 max_rank=3 min=1928 min_rank=1 imbalance_ratio=1.466064 max_load_reduction=26763 max_load_reduction_pct=0.816742 moved_to_echo=16486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4608, 256, 14464, 384, 5760, 4096, 3456, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1152, 128, 5376, 12416, 8064, 6016, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2944, 2688, 7936, 19584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 17920, 384, 13952, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 1280, 896, 512, 24448, 5888, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[14464, 0, 8320, 10112, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 3968, 512, 28160, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31488, 128, 896, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4040, 9, 133, 2301, 24, 0, 4032, 0, 18, 3787, 1080, 14, 0, 754, 2, 0, 4, 2368, 3560, 0, 782, 4081, 10, 0, 1587, 8, 1, 51, 4086, 4, 1, 31] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4085, 1505, 131, 4041, 0, 5, 3839, 0, 0, 99, 2308, 77, 13, 4094, 1312, 295, 0, 3986, 1412, 0, 36, 29, 30, 857, 1069, 0, 0, 27, 2148, 89, 828, 453] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4019, 3982, 28, 3615, 0, 26, 4095, 7, 2, 3474, 2446, 1, 2, 2817, 59, 170, 0, 4062, 825, 41, 8, 238, 46, 11, 23, 18, 15, 112, 590, 416, 73, 1547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4067, 932, 980, 1068, 203, 1, 4020, 0, 4, 36, 2762, 25, 2314, 337, 37, 5, 0, 3767, 3924, 3542, 250, 3309, 57, 260, 4, 445, 0, 0, 307, 2, 109, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3873, 3129, 0, 4089, 0, 0, 4095, 0, 33, 0, 2752, 2, 0, 3914, 2, 12, 16, 4089, 31, 1, 629, 507, 0, 648, 1, 173, 1, 17, 50, 4, 2547, 2153] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3847, 4094, 2267, 182, 45, 0, 3196, 0, 0, 3731, 3411, 1, 0, 2352, 188, 68, 0, 4032, 3824, 0, 383, 680, 0, 122, 77, 20, 0, 38, 175, 0, 0, 35] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1453, 1379, 0, 0, 261, 83, 3577, 0, 0, 4056, 4028, 0, 0, 3972, 0, 0, 1, 4095, 3661, 0, 0, 3, 0, 1549, 2821, 107, 17, 38, 711, 8, 0, 948] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[2625, 57, 1991, 3769, 908, 0, 3983, 1, 18, 781, 3387, 0, 96, 972, 306, 197, 76, 4037, 3373, 0, 9, 96, 1091, 11, 2053, 1, 292, 11, 1213, 477, 0, 937] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 133, 1468, 0, 0, 0, 0, 24, 0, 4032, 0, 17, 0, 0, 0, 18, 3787, 810, 14, 0, 0, 0, 0, 0, 754, 2, 0, 253, 395, 0, 0, 4, 660, 3560, 0, 0, 0, 0, 0, 782, 4081, 10, 0, 993, 615, 0, 0, 1587, 8, 1, 51, 3425, 0, 0, 0, 4086, 4, 1, 31, 715, 438, 0, 0] rerouted_tokens_per_ep_rank=[1610, 4073, 4629, 1404, 4224, 6481, 5072, 5275] avg=4096.0000 max=6481 max_rank=5 min=1404 min_rank=3 imbalance_ratio=1.582275 max_load_reduction=26287 max_load_reduction_pct=0.802216 moved_to_echo=21052 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3129, 0, 2607, 0, 0, 0, 0, 0, 0, 4095, 0, 45, 0, 0, 0, 33, 0, 2062, 2, 0, 0, 0, 0, 0, 3914, 2, 12, 645, 703, 0, 0, 16, 1140, 31, 1, 0, 0, 0, 0, 629, 507, 0, 648, 1715, 591, 0, 0, 1, 173, 1, 17, 3282, 0, 0, 0, 50, 4, 2547, 2153, 1234, 779, 0, 0] rerouted_tokens_per_ep_rank=[5736, 4140, 2097, 5276, 1188, 4090, 3474, 6767] avg=4096.0000 max=6767 max_rank=7 min=1188 min_rank=4 imbalance_ratio=1.652100 max_load_reduction=26001 max_load_reduction_pct=0.793488 moved_to_echo=15519 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 2267, 117, 0, 0, 0, 0, 45, 0, 3196, 0, 56, 0, 0, 0, 0, 3731, 2556, 1, 0, 0, 0, 0, 0, 2352, 188, 68, 799, 31, 0, 0, 0, 1124, 3824, 0, 0, 0, 0, 0, 383, 680, 0, 122, 1691, 586, 0, 0, 77, 20, 0, 38, 3261, 0, 0, 0, 175, 0, 0, 35, 1217, 34, 0, 0] rerouted_tokens_per_ep_rank=[6478, 3297, 6288, 3438, 4948, 3462, 3396, 1461] avg=4096.0000 max=6478 max_rank=0 min=1461 min_rank=7 imbalance_ratio=1.581543 max_load_reduction=26290 max_load_reduction_pct=0.802307 moved_to_echo=13267 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1505, 131, 2576, 0, 0, 0, 0, 0, 5, 3839, 0, 38, 0, 0, 0, 0, 99, 1729, 77, 0, 0, 0, 0, 13, 4094, 1312, 295, 541, 695, 0, 0, 0, 1111, 1412, 0, 0, 0, 0, 0, 36, 29, 30, 857, 1672, 623, 0, 0, 1069, 0, 0, 27, 3462, 0, 0, 0, 2148, 89, 828, 453, 1203, 770, 0, 0] rerouted_tokens_per_ep_rank=[4212, 3882, 1905, 6950, 2523, 3247, 4558, 5491] avg=4096.0000 max=6950 max_rank=3 min=1905 min_rank=2 imbalance_ratio=1.696777 max_load_reduction=25818 max_load_reduction_pct=0.787903 moved_to_echo=15819 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3982, 28, 2305, 0, 0, 0, 0, 0, 26, 4095, 7, 40, 0, 0, 0, 2, 3474, 1833, 1, 0, 0, 0, 0, 2, 2817, 59, 170, 573, 621, 0, 0, 0, 1132, 825, 41, 0, 0, 0, 0, 8, 238, 46, 11, 1704, 612, 0, 0, 23, 18, 15, 112, 3407, 0, 0, 0, 590, 416, 73, 1547, 1226, 689, 0, 0] rerouted_tokens_per_ep_rank=[6315, 4168, 5310, 4242, 1998, 2619, 3575, 4541] avg=4096.0000 max=6315 max_rank=0 min=1998 min_rank=4 imbalance_ratio=1.541748 max_load_reduction=26453 max_load_reduction_pct=0.807281 moved_to_echo=12733 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 932, 980, 682, 0, 0, 0, 0, 203, 1, 4020, 0, 45, 0, 0, 0, 4, 36, 2070, 25, 0, 0, 0, 0, 2314, 337, 37, 5, 647, 183, 0, 0, 0, 1050, 3924, 3542, 0, 0, 0, 0, 250, 3309, 57, 260, 1580, 619, 0, 0, 4, 445, 0, 0, 3448, 0, 0, 0, 307, 2, 109, 1, 1137, 203, 0, 0] rerouted_tokens_per_ep_rank=[2594, 4269, 2135, 3523, 8516, 6075, 3897, 1759] avg=4096.0000 max=8516 max_rank=4 min=1759 min_rank=7 imbalance_ratio=2.079102 max_load_reduction=24252 max_load_reduction_pct=0.740112 moved_to_echo=20247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1379, 0, 0, 0, 0, 0, 0, 261, 83, 3577, 0, 70, 0, 0, 0, 0, 4056, 3011, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 947, 0, 0, 0, 1, 1135, 3661, 0, 0, 0, 0, 0, 0, 3, 0, 1549, 1720, 222, 0, 0, 2821, 107, 17, 38, 1231, 0, 0, 0, 711, 8, 0, 948, 1240, 0, 0, 0] rerouted_tokens_per_ep_rank=[1379, 3991, 7067, 4919, 4797, 3494, 4214, 2907] avg=4096.0000 max=7067 max_rank=2 min=1379 min_rank=0 imbalance_ratio=1.725342 max_load_reduction=25701 max_load_reduction_pct=0.784332 moved_to_echo=15412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 57, 1991, 2396, 0, 0, 0, 0, 908, 0, 3983, 1, 56, 0, 0, 0, 18, 781, 2538, 0, 0, 0, 0, 0, 96, 972, 306, 197, 793, 652, 0, 0, 76, 1125, 3373, 0, 0, 0, 0, 0, 9, 96, 1091, 11, 1693, 400, 0, 0, 2053, 1, 292, 11, 2225, 0, 0, 0, 1213, 477, 0, 937, 1219, 721, 0, 0] rerouted_tokens_per_ep_rank=[4444, 4948, 3337, 3016, 4574, 3300, 4582, 4567] avg=4096.0000 max=4948 max_rank=1 min=3016 min_rank=3 imbalance_ratio=1.208008 max_load_reduction=27820 max_load_reduction_pct=0.848999 moved_to_echo=17023 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2176, 8960, 1280, 3584, 12800, 4352, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 15104, 5632, 12160, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9344, 1024, 3584, 6144, 9216, 3712, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 8576, 20736, 3584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 16000, 16640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2432, 19328, 1920, 768, 5248, 3328, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7680, 896, 384, 384, 23808, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1536, 128, 30848, 128, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[51, 0, 0, 4094, 1202, 3081, 7, 1, 0, 3967, 56, 8, 4096, 0, 2221, 0, 0, 390, 1, 19, 21, 14, 0, 4091, 854, 10, 0, 4017, 532, 0, 2, 4033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2509, 3831, 2, 1751, 164, 970, 136, 107, 2214, 4, 373, 465, 4094, 8, 14, 33, 0, 3956, 0, 4093, 1, 1, 0, 1454, 0, 0, 73, 1098, 197, 39, 1254, 3927] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[35, 0, 64, 4096, 1, 0, 0, 10, 0, 4045, 7, 92, 24, 0, 1687, 923, 0, 4065, 302, 1466, 10, 35, 0, 1585, 0, 3507, 16, 3772, 5, 4077, 65, 2879] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[9, 81, 0, 3721, 139, 0, 2480, 0, 0, 213, 116, 846, 3944, 124, 2307, 3004, 0, 4090, 948, 2550, 158, 0, 0, 3747, 18, 239, 548, 314, 0, 1, 10, 3161] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4095, 0, 0, 462, 0, 0, 4, 0, 0, 1263, 0, 0, 2073, 804, 3969, 0, 0, 4022, 51, 122, 1100, 0, 0, 3680, 0, 3385, 360, 4010, 2, 1, 35, 3330] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1811, 1276, 272, 3065, 3494, 2589, 114, 5, 0, 2345, 1, 2351, 605, 5, 5, 0, 0, 4092, 222, 49, 0, 30, 1, 220, 0, 0, 3, 4085, 0, 2007, 48, 4073] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4012, 0, 1, 4088, 3991, 0, 0, 0, 7, 3934, 35, 332, 4096, 0, 0, 35, 0, 2272, 0, 2072, 0, 0, 0, 3733, 0, 2, 163, 3957, 2, 4, 0, 32] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[881, 2, 0, 3536, 3983, 12, 475, 587, 153, 3872, 6, 7, 4062, 308, 213, 4000, 4, 1722, 48, 0, 0, 3, 453, 1045, 3, 1667, 8, 3662, 41, 0, 0, 2015] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[51, 0, 0, 2283, 0, 0, 0, 0, 1202, 3081, 7, 1, 1641, 0, 0, 0, 0, 3967, 56, 8, 60, 323, 46, 0, 2337, 0, 2221, 0, 0, 0, 0, 0, 0, 330, 1, 19, 0, 0, 0, 0, 21, 14, 0, 4091, 1811, 72, 0, 0, 854, 10, 0, 3534, 0, 0, 0, 0, 532, 0, 2, 4033, 160, 0, 0, 0] rerouted_tokens_per_ep_rank=[2334, 5932, 4460, 4558, 350, 6009, 4398, 4727] avg=4096.0000 max=6009 max_rank=5 min=350 min_rank=4 imbalance_ratio=1.467041 max_load_reduction=26759 max_load_reduction_pct=0.816620 moved_to_echo=15484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[35, 0, 64, 2284, 0, 0, 0, 0, 1, 0, 0, 10, 9, 0, 0, 0, 0, 4045, 7, 92, 625, 303, 0, 0, 15, 0, 1687, 923, 0, 0, 0, 0, 0, 3440, 302, 1466, 0, 0, 0, 0, 10, 35, 0, 1585, 1812, 0, 0, 0, 0, 3507, 16, 3319, 0, 0, 0, 0, 5, 4077, 65, 2879, 150, 0, 0, 0] rerouted_tokens_per_ep_rank=[2383, 20, 5072, 2625, 5208, 3442, 6842, 7176] avg=4096.0000 max=7176 max_rank=7 min=20 min_rank=1 imbalance_ratio=1.751953 max_load_reduction=25592 max_load_reduction_pct=0.781006 moved_to_echo=22668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 258, 0, 0, 0, 0, 0, 0, 4, 0, 830, 0, 0, 0, 0, 1263, 0, 0, 619, 323, 23, 0, 1184, 804, 3969, 0, 0, 0, 0, 0, 0, 3403, 51, 122, 0, 0, 0, 0, 1100, 0, 0, 3680, 204, 36, 0, 0, 0, 3385, 360, 3527, 0, 0, 0, 0, 2, 1, 35, 3330, 160, 0, 0, 0] rerouted_tokens_per_ep_rank=[4353, 834, 2228, 5957, 3576, 5020, 7272, 3528] avg=4096.0000 max=7272 max_rank=6 min=834 min_rank=1 imbalance_ratio=1.775391 max_load_reduction=25496 max_load_reduction_pct=0.778076 moved_to_echo=19396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2509, 3831, 2, 977, 0, 0, 0, 0, 164, 970, 136, 107, 1640, 0, 0, 0, 2214, 4, 373, 465, 608, 88, 46, 0, 2336, 8, 14, 33, 0, 0, 0, 0, 0, 3348, 0, 4093, 0, 0, 0, 0, 1, 1, 0, 1454, 774, 72, 0, 0, 0, 0, 73, 967, 0, 0, 0, 0, 197, 39, 1254, 3927, 43, 0, 0, 0] rerouted_tokens_per_ep_rank=[7319, 3017, 3798, 2391, 7441, 2302, 1040, 5460] avg=4096.0000 max=7441 max_rank=4 min=1040 min_rank=6 imbalance_ratio=1.816650 max_load_reduction=25327 max_load_reduction_pct=0.772919 moved_to_echo=16243 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 81, 0, 2075, 0, 0, 0, 0, 139, 0, 2480, 0, 1580, 0, 0, 0, 0, 213, 116, 846, 629, 25, 44, 0, 2250, 124, 2307, 3004, 0, 0, 0, 0, 0, 3461, 948, 2550, 0, 0, 0, 0, 158, 0, 0, 3747, 1646, 70, 0, 0, 18, 239, 548, 277, 0, 0, 0, 0, 0, 1, 10, 3161, 12, 0, 0, 0] rerouted_tokens_per_ep_rank=[2165, 4199, 1873, 7685, 6959, 5621, 1082, 3184] avg=4096.0000 max=7685 max_rank=3 min=1082 min_rank=6 imbalance_ratio=1.876221 max_load_reduction=25083 max_load_reduction_pct=0.765472 moved_to_echo=16846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1811, 1276, 272, 1710, 0, 0, 0, 0, 3494, 2589, 114, 5, 242, 0, 0, 0, 0, 2345, 1, 2351, 629, 329, 6, 0, 347, 5, 5, 0, 0, 0, 0, 0, 0, 3463, 222, 49, 0, 0, 0, 0, 0, 30, 1, 220, 1355, 10, 0, 0, 0, 0, 3, 3593, 0, 0, 0, 0, 0, 2007, 48, 4073, 163, 0, 0, 0] rerouted_tokens_per_ep_rank=[5069, 6444, 5661, 357, 3734, 1616, 3596, 6291] avg=4096.0000 max=6444 max_rank=1 min=357 min_rank=3 imbalance_ratio=1.573242 max_load_reduction=26324 max_load_reduction_pct=0.803345 moved_to_echo=15237 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4012, 0, 1, 2277, 0, 0, 0, 0, 3991, 0, 0, 0, 1646, 0, 0, 0, 7, 3934, 35, 332, 353, 322, 50, 0, 2323, 0, 0, 35, 0, 0, 0, 0, 0, 1919, 0, 2072, 0, 0, 0, 0, 0, 0, 0, 3733, 1811, 77, 0, 0, 0, 2, 163, 3473, 0, 0, 0, 0, 2, 4, 0, 32, 162, 0, 0, 0] rerouted_tokens_per_ep_rank=[6290, 5637, 5033, 2358, 3991, 5621, 3638, 200] avg=4096.0000 max=6290 max_rank=0 min=200 min_rank=7 imbalance_ratio=1.535645 max_load_reduction=26478 max_load_reduction_pct=0.808044 moved_to_echo=13450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[881, 2, 0, 1972, 0, 0, 0, 0, 3983, 12, 475, 587, 1628, 0, 0, 0, 153, 3872, 6, 7, 265, 294, 46, 0, 2316, 308, 213, 4000, 0, 0, 0, 0, 4, 1457, 48, 0, 0, 0, 0, 0, 0, 3, 453, 1045, 1564, 72, 0, 0, 3, 1667, 8, 3222, 0, 0, 0, 0, 41, 0, 0, 2015, 146, 0, 0, 0] rerouted_tokens_per_ep_rank=[2855, 6685, 4643, 6837, 1509, 3137, 4900, 2202] avg=4096.0000 max=6837 max_rank=3 min=1509 min_rank=4 imbalance_ratio=1.669189 max_load_reduction=25931 max_load_reduction_pct=0.791351 moved_to_echo=11748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13184, 1280, 10496, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[896, 8832, 1280, 22016, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 6144, 1536, 23552, 1024, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 20864, 1664, 10496, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13056, 6656, 3328, 768, 9216, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1408, 128, 512, 19584, 11008, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13440, 5248, 384, 13952, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2432, 19712, 640, 4224, 3840, 2048, 384, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3955, 416, 2897, 3, 1258, 4016, 0, 4092, 57, 77, 4096, 4001, 3410, 209, 0, 37, 129, 0, 18, 3, 0, 476, 14, 158, 0, 0, 0, 131, 0, 0, 3313, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4007, 0, 264, 1440, 204, 3804, 3491, 3, 0, 2743, 2030, 0, 12, 209, 515, 0, 1833, 0, 1, 58, 58, 4095, 0, 1119, 5, 0, 1780, 1003, 4092, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[947, 427, 3734, 40, 3563, 11, 191, 3952, 4046, 0, 2, 4040, 2189, 2, 4, 52, 2, 0, 0, 0, 37, 4, 64, 3976, 8, 32, 787, 622, 0, 0, 4033, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4063, 38, 3725, 4, 406, 1, 0, 4089, 2, 1, 2963, 3507, 25, 0, 6, 0, 2352, 0, 3063, 39, 23, 25, 52, 4085, 102, 1919, 37, 0, 0, 0, 2232, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[426, 0, 1855, 787, 102, 5, 0, 3724, 247, 14, 3825, 559, 3660, 3547, 4, 151, 1, 0, 1, 15, 0, 0, 510, 88, 0, 3002, 18, 189, 12, 3669, 4094, 2263] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4077, 0, 4096, 0, 10, 1491, 994, 21, 0, 74, 549, 4089, 2947, 291, 397, 0, 20, 23, 1, 6, 0, 0, 0, 3743, 0, 1157, 568, 43, 60, 4052, 4059, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2212, 0, 3990, 9, 1, 0, 133, 3153, 2940, 2, 0, 4040, 0, 4, 208, 538, 1, 0, 0, 0, 63, 306, 0, 2510, 37, 3530, 4, 1, 3153, 2070, 3863, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 1, 4088, 2047, 4, 0, 0, 4020, 4026, 1, 2855, 2906, 1, 346, 0, 0, 2703, 122, 568, 0, 6, 0, 0, 0, 393, 0, 354, 83, 0, 52, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3955, 416, 941, 3, 0, 0, 0, 0, 1258, 4016, 0, 2846, 0, 0, 0, 0, 57, 77, 4096, 542, 0, 0, 0, 0, 3410, 209, 0, 37, 1213, 161, 0, 0, 129, 0, 18, 3, 3300, 0, 0, 0, 0, 476, 14, 158, 1246, 471, 0, 0, 0, 0, 0, 131, 1795, 159, 0, 0, 0, 0, 1629, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5315, 8120, 4772, 5030, 3450, 2365, 2085, 1631] avg=4096.0000 max=8120 max_rank=1 min=1631 min_rank=7 imbalance_ratio=1.982422 max_load_reduction=24648 max_load_reduction_pct=0.752197 moved_to_echo=9531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[426, 0, 603, 787, 0, 0, 0, 0, 102, 5, 0, 2590, 0, 0, 0, 0, 247, 14, 3825, 76, 0, 0, 0, 0, 3660, 3547, 4, 151, 1499, 103, 0, 0, 1, 0, 1, 15, 461, 0, 0, 0, 0, 0, 510, 88, 1134, 582, 0, 0, 0, 3002, 18, 189, 1149, 22, 0, 0, 12, 3669, 2013, 2263, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1816, 2697, 4162, 8964, 478, 2314, 4380, 7957] avg=4096.0000 max=8964 max_rank=3 min=478 min_rank=4 imbalance_ratio=2.188477 max_load_reduction=23804 max_load_reduction_pct=0.726440 moved_to_echo=15129 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2212, 0, 1296, 9, 0, 0, 0, 0, 1, 0, 133, 2193, 0, 0, 0, 0, 2940, 2, 0, 547, 0, 0, 0, 0, 0, 4, 208, 538, 1414, 222, 0, 0, 1, 0, 0, 0, 3332, 0, 0, 0, 63, 306, 0, 2510, 960, 549, 0, 0, 37, 3530, 4, 1, 2472, 161, 0, 0, 3153, 2070, 1900, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3517, 2327, 3489, 2386, 3333, 4388, 6205, 7123] avg=4096.0000 max=7123 max_rank=7 min=2327 min_rank=1 imbalance_ratio=1.739014 max_load_reduction=25645 max_load_reduction_pct=0.782623 moved_to_echo=21049 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1303, 0, 0, 0, 0, 0, 264, 1440, 204, 2646, 0, 0, 0, 0, 3491, 3, 0, 372, 0, 0, 0, 0, 2030, 0, 12, 209, 1498, 222, 0, 0, 515, 0, 1833, 0, 2262, 0, 0, 0, 1, 58, 58, 4095, 1158, 581, 0, 0, 0, 1119, 5, 0, 2482, 109, 0, 0, 1780, 1003, 2013, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1303, 4554, 3866, 3971, 4610, 5951, 3715, 4798] avg=4096.0000 max=5951 max_rank=5 min=1303 min_rank=0 imbalance_ratio=1.452881 max_load_reduction=26817 max_load_reduction_pct=0.818390 moved_to_echo=19074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[947, 427, 1214, 40, 0, 0, 0, 0, 3563, 11, 191, 2749, 0, 0, 0, 0, 4046, 0, 2, 547, 0, 0, 0, 0, 2189, 2, 4, 52, 1477, 207, 0, 0, 2, 0, 0, 0, 3332, 0, 0, 0, 37, 4, 64, 3976, 1203, 573, 0, 0, 8, 32, 787, 622, 2313, 161, 0, 0, 0, 0, 1983, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2628, 6514, 4595, 3931, 3334, 5857, 3923, 1986] avg=4096.0000 max=6514 max_rank=1 min=1986 min_rank=7 imbalance_ratio=1.590332 max_load_reduction=26254 max_load_reduction_pct=0.801208 moved_to_echo=15100 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4077, 0, 1324, 0, 0, 0, 0, 0, 10, 1491, 994, 10, 0, 0, 0, 0, 0, 74, 549, 545, 0, 0, 0, 0, 2947, 291, 397, 0, 1491, 231, 0, 0, 20, 23, 1, 6, 3377, 0, 0, 0, 0, 0, 0, 3743, 11, 580, 0, 0, 0, 1157, 568, 43, 2541, 167, 0, 0, 60, 4052, 1988, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5401, 2505, 1168, 5357, 3427, 4334, 4476, 6100] avg=4096.0000 max=6100 max_rank=7 min=1168 min_rank=2 imbalance_ratio=1.489258 max_load_reduction=26668 max_load_reduction_pct=0.813843 moved_to_echo=18337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4063, 38, 1210, 4, 0, 0, 0, 0, 406, 1, 0, 2844, 0, 0, 0, 0, 2, 1, 2963, 475, 0, 0, 0, 0, 25, 0, 6, 0, 817, 207, 0, 0, 2352, 0, 3063, 39, 2892, 0, 0, 0, 23, 25, 52, 4085, 1245, 317, 0, 0, 102, 1919, 37, 0, 2308, 140, 0, 0, 0, 0, 1098, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5315, 3251, 3441, 1055, 8346, 5747, 4506, 1107] avg=4096.0000 max=8346 max_rank=4 min=1055 min_rank=3 imbalance_ratio=2.037598 max_load_reduction=24422 max_load_reduction_pct=0.745300 moved_to_echo=19706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 1, 1329, 2047, 0, 0, 0, 0, 4, 0, 0, 2796, 0, 0, 0, 0, 4026, 1, 2855, 393, 0, 0, 0, 0, 1, 346, 0, 0, 1500, 227, 0, 0, 2703, 122, 568, 0, 2397, 0, 0, 0, 6, 0, 0, 0, 1224, 582, 0, 0, 393, 0, 354, 83, 2532, 116, 0, 0, 0, 52, 2014, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7473, 2800, 7275, 2074, 5790, 1812, 3478, 2066] avg=4096.0000 max=7473 max_rank=0 min=1812 min_rank=5 imbalance_ratio=1.824463 max_load_reduction=25295 max_load_reduction_pct=0.771942 moved_to_echo=13146 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14336, 4480, 640, 1024, 11008, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[19840, 896, 9344, 2944, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5120, 10880, 14720, 2304, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 10880, 1792, 1152, 17664, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 896, 768, 18688, 8192, 4352, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5760, 256, 5504, 128, 21376, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[5632, 7040, 1536, 18688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14848, 256, 14336, 3584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4, 45, 1945, 133, 0, 0, 3229, 1017, 2, 3, 0, 4096, 0, 4094, 4091, 3644, 436, 36, 0, 0, 2, 2, 0, 0, 32, 3086, 447, 991, 2034, 3399, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1, 14, 16, 0, 31, 0, 0, 3780, 3661, 254, 7, 3881, 38, 4096, 3126, 23, 2, 6, 7, 33, 933, 4, 162, 0, 0, 2165, 936, 1896, 3812, 0, 1, 3883] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[18, 116, 0, 0, 0, 3861, 46, 41, 3884, 0, 0, 4096, 16, 4034, 4086, 4096, 0, 18, 52, 0, 50, 35, 1015, 85, 127, 3595, 45, 3452, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[6, 4082, 7, 0, 0, 38, 3910, 1, 20, 2, 3199, 4096, 43, 3822, 0, 3277, 863, 0, 1, 64, 0, 687, 23, 0, 431, 4037, 4, 4057, 0, 98, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[15, 0, 0, 0, 10, 0, 2942, 2, 0, 0, 2716, 3870, 1, 3368, 786, 1660, 4093, 3, 0, 0, 9, 639, 1, 0, 634, 3951, 3, 4089, 3314, 662, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 207, 0, 0, 0, 0, 4091, 3360, 11, 0, 101, 3843, 20, 3314, 1, 4, 4044, 1359, 67, 368, 0, 44, 0, 14, 1326, 115, 4088, 3564, 0, 2550, 43, 234] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[188, 3420, 0, 0, 0, 99, 3975, 0, 3842, 168, 239, 4001, 3593, 3734, 2606, 63, 2, 67, 0, 3, 11, 0, 0, 30, 2115, 219, 517, 3698, 0, 156, 2, 20] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[3926, 1, 3909, 126, 0, 1, 689, 902, 1, 0, 1627, 2636, 0, 4096, 3562, 0, 0, 0, 809, 913, 113, 3229, 725, 21, 41, 4059, 0, 1318, 19, 0, 0, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4, 45, 1945, 133, 359, 834, 0, 0, 0, 0, 3229, 1017, 166, 0, 0, 0, 2, 3, 0, 1750, 0, 0, 0, 0, 0, 0, 3651, 3644, 0, 0, 0, 0, 436, 36, 0, 0, 597, 754, 0, 0, 2, 2, 0, 0, 3340, 0, 0, 0, 32, 3086, 447, 35, 0, 0, 0, 0, 2034, 3399, 0, 0, 1512, 274, 0, 0] rerouted_tokens_per_ep_rank=[3320, 4412, 1755, 7295, 1823, 3344, 3600, 7219] avg=4096.0000 max=7295 max_rank=3 min=1755 min_rank=2 imbalance_ratio=1.781006 max_load_reduction=25473 max_load_reduction_pct=0.777374 moved_to_echo=15986 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 14, 16, 0, 688, 790, 0, 0, 31, 0, 0, 3780, 127, 0, 0, 0, 3661, 254, 7, 1658, 0, 0, 0, 0, 38, 0, 2790, 23, 0, 0, 0, 0, 2, 6, 7, 33, 1142, 754, 0, 0, 933, 4, 162, 0, 3342, 0, 0, 0, 0, 2165, 936, 66, 0, 0, 0, 0, 3812, 0, 1, 3883, 1433, 209, 0, 0] rerouted_tokens_per_ep_rank=[1509, 3938, 5580, 2851, 1944, 4441, 3167, 9338] avg=4096.0000 max=9338 max_rank=7 min=1509 min_rank=0 imbalance_ratio=2.279785 max_load_reduction=23430 max_load_reduction_pct=0.715027 moved_to_echo=18890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 207, 0, 0, 1293, 782, 0, 0, 0, 0, 4091, 3360, 0, 0, 0, 0, 11, 0, 101, 1642, 0, 0, 0, 0, 20, 0, 1, 4, 0, 0, 0, 0, 4044, 1359, 67, 368, 2147, 609, 0, 0, 0, 44, 0, 14, 2705, 0, 0, 0, 1326, 115, 4088, 124, 0, 0, 0, 0, 0, 2550, 43, 234, 1419, 0, 0, 0] rerouted_tokens_per_ep_rank=[2282, 7451, 1754, 25, 8594, 2763, 5653, 4246] avg=4096.0000 max=8594 max_rank=4 min=25 min_rank=3 imbalance_ratio=2.098145 max_load_reduction=24174 max_load_reduction_pct=0.737732 moved_to_echo=21256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[18, 116, 0, 0, 1252, 834, 0, 0, 0, 3861, 46, 41, 166, 0, 0, 0, 3884, 0, 0, 1750, 0, 0, 0, 0, 16, 0, 3646, 4096, 0, 0, 0, 0, 0, 18, 52, 0, 2080, 742, 0, 0, 50, 35, 1015, 85, 3292, 0, 0, 0, 127, 3595, 45, 120, 0, 0, 0, 0, 0, 0, 0, 0, 1512, 274, 0, 0] rerouted_tokens_per_ep_rank=[2220, 4114, 5634, 7758, 2892, 4477, 3887, 1786] avg=4096.0000 max=7758 max_rank=3 min=1786 min_rank=7 imbalance_ratio=1.894043 max_load_reduction=25010 max_load_reduction_pct=0.763245 moved_to_echo=13042 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[6, 4082, 7, 0, 1472, 834, 0, 0, 0, 38, 3910, 1, 0, 0, 0, 0, 20, 2, 3199, 1750, 0, 0, 0, 0, 43, 0, 0, 3277, 0, 0, 0, 0, 863, 0, 1, 64, 2444, 704, 0, 0, 0, 687, 23, 0, 3118, 0, 0, 0, 431, 4037, 4, 141, 0, 0, 0, 0, 0, 98, 0, 0, 1512, 0, 0, 0] rerouted_tokens_per_ep_rank=[6401, 3949, 4971, 3320, 4076, 3828, 4613, 1610] avg=4096.0000 max=6401 max_rank=0 min=1610 min_rank=7 imbalance_ratio=1.562744 max_load_reduction=26367 max_load_reduction_pct=0.804657 moved_to_echo=14127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[15, 0, 0, 0, 1488, 792, 0, 0, 10, 0, 2942, 2, 34, 0, 0, 0, 0, 0, 2716, 1645, 0, 0, 0, 0, 1, 0, 696, 1660, 0, 0, 0, 0, 4093, 3, 0, 0, 2466, 620, 0, 0, 9, 639, 1, 0, 2748, 0, 0, 0, 634, 3951, 3, 135, 0, 0, 0, 0, 3314, 662, 0, 0, 1433, 56, 0, 0] rerouted_tokens_per_ep_rank=[2295, 2988, 4361, 2357, 7182, 3397, 4723, 5465] avg=4096.0000 max=7182 max_rank=4 min=2295 min_rank=0 imbalance_ratio=1.753418 max_load_reduction=25586 max_load_reduction_pct=0.780823 moved_to_echo=20767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[188, 3420, 0, 0, 1342, 815, 0, 0, 0, 99, 3975, 0, 106, 0, 0, 0, 3842, 168, 239, 1709, 0, 0, 0, 0, 3593, 0, 2326, 63, 0, 0, 0, 0, 2, 67, 0, 3, 2228, 687, 0, 0, 11, 0, 0, 30, 3047, 0, 0, 0, 2115, 219, 517, 128, 0, 0, 0, 0, 0, 156, 2, 20, 1477, 174, 0, 0] rerouted_tokens_per_ep_rank=[5765, 4180, 5958, 5982, 2987, 3088, 2979, 1829] avg=4096.0000 max=5982 max_rank=3 min=1829 min_rank=7 imbalance_ratio=1.460449 max_load_reduction=26786 max_load_reduction_pct=0.817444 moved_to_echo=10883 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3926, 1, 3909, 126, 478, 536, 0, 0, 0, 1, 689, 902, 144, 0, 0, 0, 1, 0, 1627, 1127, 0, 0, 0, 0, 0, 0, 3180, 0, 0, 0, 0, 0, 0, 0, 809, 913, 794, 754, 0, 0, 113, 3229, 725, 21, 3342, 0, 0, 0, 41, 4059, 0, 46, 0, 0, 0, 0, 19, 0, 0, 45, 973, 238, 0, 0] rerouted_tokens_per_ep_rank=[8976, 1736, 2755, 3180, 3270, 7430, 4146, 1275] avg=4096.0000 max=8976 max_rank=0 min=1275 min_rank=7 imbalance_ratio=2.191406 max_load_reduction=23792 max_load_reduction_pct=0.726074 moved_to_echo=16121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 7936, 5888, 384, 8448, 6272, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9216, 6912, 128, 4224, 11392, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[3712, 0, 16384, 12800, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9472, 1536, 1024, 1408, 13952, 5632, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11520, 512, 7936, 13056, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1152, 4736, 2048, 256, 24960, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 4096, 18944, 9216, 768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4736, 21248, 6144, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[62, 13, 80, 0, 3149, 0, 129, 263, 18, 2108, 4006, 4090, 512, 0, 2292, 0, 4096, 1027, 75, 0, 0, 0, 0, 33, 0, 1, 1, 3937, 3816, 0, 2198, 862] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3770, 0, 0, 64, 3944, 0, 74, 437, 7, 4096, 21, 3621, 1645, 0, 133, 5, 4089, 2442, 153, 0, 5, 0, 0, 72, 4087, 7, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[47, 0, 0, 6, 582, 0, 413, 262, 3864, 3858, 3923, 92, 12, 0, 0, 4064, 3908, 3999, 63, 11, 8, 0, 23, 3720, 0, 0, 0, 3719, 190, 3, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 5, 3897, 3336, 1628, 0, 1301, 151, 531, 1, 0, 4094, 1227, 17, 0, 0, 3101, 1653, 3993, 41, 141, 0, 0, 2, 0, 0, 0, 4070, 3561, 10, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3902, 0, 23, 1142, 4035, 0, 325, 3879, 3810, 4096, 6, 1891, 8, 0, 0, 9, 4088, 0, 0, 0, 1, 0, 0, 1, 1400, 0, 121, 3971, 0, 60, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4, 0, 3996, 4096, 2, 0, 2022, 0, 4087, 46, 2245, 44, 3508, 0, 2399, 0, 3847, 2628, 1121, 0, 48, 0, 10, 2377, 4, 0, 0, 0, 0, 0, 284, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4066, 1796, 3983, 1, 0, 0, 4030, 355, 0, 4063, 17, 0, 289, 5, 0, 4, 3502, 36, 21, 132, 0, 4087, 0, 0, 1, 3829, 977, 0, 0, 1574] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 1, 0, 419, 2783, 698, 4024, 4083, 3701, 0, 4, 1294, 2008, 3070, 1, 1829, 0, 0, 3, 22, 234, 0, 5, 0, 4081, 38, 684, 3784, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[47, 0, 0, 6, 283, 75, 0, 0, 582, 0, 413, 262, 445, 0, 0, 0, 3864, 300, 3923, 0, 0, 0, 0, 0, 12, 0, 0, 4064, 1659, 455, 0, 0, 1521, 3999, 63, 11, 0, 0, 0, 0, 8, 0, 23, 3720, 92, 48, 0, 0, 0, 0, 0, 3644, 0, 0, 0, 0, 190, 3, 1, 0, 3055, 0, 0, 0] rerouted_tokens_per_ep_rank=[411, 1702, 8087, 6190, 5594, 3891, 3644, 3249] avg=4096.0000 max=8087 max_rank=2 min=411 min_rank=0 imbalance_ratio=1.974365 max_load_reduction=24681 max_load_reduction_pct=0.753204 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3902, 0, 23, 1142, 296, 80, 0, 0, 4035, 0, 325, 3879, 465, 0, 0, 0, 3810, 319, 6, 0, 0, 0, 0, 0, 8, 0, 0, 9, 1735, 483, 0, 0, 1592, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1891, 51, 0, 0, 1400, 0, 121, 3891, 0, 0, 0, 0, 0, 60, 0, 0, 3243, 0, 0, 0] rerouted_tokens_per_ep_rank=[5443, 8704, 4135, 2235, 1592, 1944, 5412, 3303] avg=4096.0000 max=8704 max_rank=1 min=1592 min_rank=4 imbalance_ratio=2.125000 max_load_reduction=24064 max_load_reduction_pct=0.734375 moved_to_echo=12251 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[62, 13, 80, 0, 296, 79, 0, 0, 3149, 0, 129, 263, 466, 0, 0, 0, 18, 165, 4006, 0, 0, 0, 0, 0, 512, 0, 2292, 0, 1739, 248, 0, 0, 1595, 1027, 75, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4090, 26, 0, 0, 0, 1, 1, 3858, 0, 0, 0, 0, 3816, 0, 2198, 862, 1669, 0, 0, 0] rerouted_tokens_per_ep_rank=[530, 4007, 4189, 4791, 2697, 4149, 3860, 8545] avg=4096.0000 max=8545 max_rank=7 min=530 min_rank=0 imbalance_ratio=2.086182 max_load_reduction=24223 max_load_reduction_pct=0.739227 moved_to_echo=19251 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4066, 1796, 0, 77, 0, 0, 3983, 1, 0, 0, 0, 0, 0, 0, 4030, 29, 0, 0, 0, 0, 0, 0, 17, 0, 289, 5, 0, 41, 0, 0, 0, 4, 3502, 36, 0, 0, 0, 0, 21, 132, 0, 4087, 4063, 4, 0, 0, 0, 0, 1, 3752, 0, 0, 0, 0, 977, 0, 0, 1574, 281, 0, 0, 0] rerouted_tokens_per_ep_rank=[5939, 3984, 4059, 352, 3542, 8307, 3753, 2832] avg=4096.0000 max=8307 max_rank=5 min=352 min_rank=3 imbalance_ratio=2.028076 max_load_reduction=24461 max_load_reduction_pct=0.746490 moved_to_echo=18434 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3770, 0, 0, 64, 296, 83, 0, 0, 3944, 0, 74, 437, 465, 0, 0, 0, 7, 319, 21, 0, 0, 0, 0, 0, 1645, 0, 133, 5, 1736, 483, 0, 0, 1592, 2442, 153, 0, 0, 0, 0, 0, 5, 0, 0, 72, 3621, 51, 0, 0, 4087, 7, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 3243, 0, 0, 0] rerouted_tokens_per_ep_rank=[4213, 4920, 347, 4002, 4187, 3749, 8107, 3243] avg=4096.0000 max=8107 max_rank=6 min=347 min_rank=2 imbalance_ratio=1.979248 max_load_reduction=24661 max_load_reduction_pct=0.752594 moved_to_echo=19286 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 3897, 3336, 224, 82, 0, 0, 1628, 0, 1301, 151, 353, 0, 0, 0, 531, 1, 0, 0, 0, 0, 0, 0, 1227, 17, 0, 0, 1316, 0, 0, 0, 1208, 1653, 3993, 41, 0, 0, 0, 0, 141, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 3988, 0, 0, 0, 0, 3561, 10, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7544, 3433, 532, 2560, 6895, 4237, 3988, 3579] avg=4096.0000 max=7544 max_rank=0 min=532 min_rank=2 imbalance_ratio=1.841797 max_load_reduction=25224 max_load_reduction_pct=0.769775 moved_to_echo=18699 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 3996, 4096, 282, 0, 0, 0, 2, 0, 2022, 0, 442, 0, 0, 0, 4087, 0, 2245, 0, 0, 0, 0, 0, 3508, 0, 2399, 0, 1637, 10, 0, 0, 1486, 2628, 1121, 0, 0, 0, 0, 0, 48, 0, 10, 2377, 44, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 36, 0, 0, 0] rerouted_tokens_per_ep_rank=[8378, 2466, 6332, 7554, 5235, 2479, 4, 320] avg=4096.0000 max=8378 max_rank=0 min=4 min_rank=6 imbalance_ratio=2.045410 max_load_reduction=24390 max_load_reduction_pct=0.744324 moved_to_echo=8038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 222, 86, 0, 0, 1, 0, 419, 2783, 349, 0, 0, 0, 698, 306, 4083, 0, 0, 0, 0, 0, 0, 4, 1294, 2008, 1303, 475, 0, 0, 1196, 1, 1829, 0, 0, 0, 0, 0, 0, 3, 22, 234, 3701, 52, 0, 0, 0, 5, 0, 3995, 0, 0, 0, 0, 38, 684, 3784, 0, 3191, 0, 0, 0] rerouted_tokens_per_ep_rank=[310, 3552, 5087, 5084, 3026, 4012, 4000, 7697] avg=4096.0000 max=7697 max_rank=7 min=310 min_rank=0 imbalance_ratio=1.879150 max_load_reduction=25071 max_load_reduction_pct=0.765106 moved_to_echo=18735 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7808, 128, 12160, 10496, 1920, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7040, 128, 6528, 6144, 11136, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[10240, 11776, 10752, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17152, 1536, 14336, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8704, 768, 6400, 2560, 14720, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[17408, 128, 4736, 7808, 3072, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5504, 128, 128, 27264, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 256, 128, 10624, 21632, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 49, 601, 4086, 3191, 3626, 20, 4068, 283, 0, 35, 4008, 109, 4019, 35, 0, 134, 0, 1, 591, 0, 3753, 53, 48, 0, 6, 0, 118, 0, 3934, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3776, 4093, 0, 15, 1573, 0, 3071, 2048, 969, 4053, 34, 8, 4080, 0, 1111, 0, 0, 84, 7, 4, 22, 2, 3240, 525, 27, 4024, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 8, 3325, 14, 4089, 3838, 15, 4071, 283, 0, 219, 4080, 15, 0, 50, 4006, 7, 0, 0, 326, 461, 3926, 0, 3559, 85, 8, 3, 362, 7, 0, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3, 379, 257, 800, 3880, 12, 4088, 3592, 0, 324, 3591, 9, 3712, 403, 1, 0, 106, 0, 0, 1, 318, 3687, 1, 4010, 1, 3, 0, 0, 3549, 41] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[10, 79, 3940, 1, 97, 1, 4095, 3, 4004, 220, 0, 3256, 7, 0, 15, 0, 6, 0, 123, 9, 0, 0, 3989, 0, 3846, 4096, 0, 4074, 0, 0, 897, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 16, 886, 4076, 3814, 51, 2, 0, 456, 0, 263, 358, 0, 0, 0, 3240, 3, 4092, 37, 0, 4081, 47, 4092, 0, 0, 0, 3552, 3697] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[116, 0, 0, 594, 0, 13, 2691, 123, 2072, 0, 0, 1706, 4086, 0, 4089, 0, 3, 4, 430, 0, 3932, 0, 4044, 12, 4096, 0, 0, 1, 0, 0, 4089, 667] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[158, 0, 0, 1089, 29, 11, 4058, 11, 0, 2422, 862, 139, 1732, 1, 4078, 0, 2039, 0, 0, 147, 3888, 1268, 3991, 1090, 3402, 549, 0, 83, 1552, 0, 167, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 49, 601, 1375, 871, 310, 0, 4086, 3191, 1704, 20, 0, 0, 0, 0, 2693, 283, 0, 35, 0, 0, 0, 0, 2638, 109, 4019, 35, 0, 0, 0, 0, 0, 134, 0, 1, 1922, 1378, 0, 0, 591, 0, 2065, 53, 0, 0, 0, 0, 30, 0, 6, 0, 0, 0, 0, 0, 118, 0, 3934, 0, 18, 499, 0, 0] rerouted_tokens_per_ep_rank=[3206, 9001, 3011, 6801, 3435, 2709, 36, 4569] avg=4096.0000 max=9001 max_rank=1 min=36 min_rank=6 imbalance_ratio=2.197510 max_load_reduction=23767 max_load_reduction_pct=0.725311 moved_to_echo=10749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 3325, 1376, 887, 324, 0, 14, 4089, 1804, 15, 0, 0, 0, 0, 2695, 283, 0, 219, 0, 0, 0, 0, 2685, 15, 0, 50, 0, 0, 0, 0, 4006, 7, 0, 0, 2034, 1441, 0, 0, 326, 461, 2161, 0, 0, 0, 0, 0, 2182, 85, 8, 3, 0, 0, 0, 0, 362, 7, 0, 11, 1377, 508, 0, 0] rerouted_tokens_per_ep_rank=[5920, 5922, 3197, 2750, 7488, 2948, 2278, 2265] avg=4096.0000 max=7488 max_rank=4 min=2265 min_rank=7 imbalance_ratio=1.828125 max_load_reduction=25280 max_load_reduction_pct=0.771484 moved_to_echo=14979 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3776, 4093, 0, 15, 327, 887, 268, 0, 1573, 0, 1443, 2048, 0, 0, 0, 0, 642, 4053, 34, 8, 0, 0, 0, 0, 2685, 0, 1111, 0, 0, 0, 0, 0, 0, 84, 7, 4, 1628, 1189, 0, 0, 22, 2, 1783, 525, 0, 0, 0, 0, 17, 4024, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 508, 0, 0] rerouted_tokens_per_ep_rank=[9366, 5064, 4737, 3796, 2912, 2332, 4043, 518] avg=4096.0000 max=9366 max_rank=0 min=518 min_rank=7 imbalance_ratio=2.286621 max_load_reduction=23402 max_load_reduction_pct=0.714172 moved_to_echo=9805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 379, 1382, 780, 26, 0, 257, 800, 1823, 12, 0, 0, 0, 0, 2706, 3592, 0, 324, 0, 0, 0, 0, 2364, 9, 3712, 403, 0, 0, 0, 0, 1, 0, 106, 0, 2057, 116, 0, 0, 0, 1, 176, 3687, 0, 0, 0, 0, 1, 4010, 1, 3, 0, 0, 0, 0, 0, 0, 3549, 41, 0, 447, 0, 0] rerouted_tokens_per_ep_rank=[2570, 2892, 6622, 6488, 2280, 3864, 4015, 4037] avg=4096.0000 max=6622 max_rank=2 min=2280 min_rank=4 imbalance_ratio=1.616699 max_load_reduction=26146 max_load_reduction_pct=0.797913 moved_to_echo=14196 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[10, 79, 3940, 1, 1354, 1, 330, 0, 97, 1, 1924, 3, 0, 0, 0, 0, 2650, 220, 0, 3256, 0, 0, 0, 0, 6, 0, 15, 0, 0, 0, 0, 0, 6, 0, 123, 9, 2171, 1464, 0, 0, 0, 0, 2195, 0, 0, 0, 0, 0, 2358, 4096, 0, 4074, 0, 0, 0, 0, 0, 0, 897, 0, 1488, 0, 0, 0] rerouted_tokens_per_ep_rank=[5715, 2025, 6126, 21, 3773, 2195, 10528, 2385] avg=4096.0000 max=10528 max_rank=6 min=21 min_rank=3 imbalance_ratio=2.570312 max_load_reduction=22240 max_load_reduction_pct=0.678711 moved_to_echo=18881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 1289, 99, 3, 0, 0, 16, 417, 4076, 0, 0, 0, 0, 2525, 51, 2, 0, 0, 0, 0, 0, 301, 0, 263, 358, 0, 0, 0, 0, 0, 0, 0, 3240, 469, 13, 0, 0, 3, 4092, 21, 0, 0, 0, 0, 0, 2502, 47, 4092, 0, 0, 0, 0, 0, 0, 0, 3552, 3697, 1579, 56, 0, 0] rerouted_tokens_per_ep_rank=[1396, 4509, 2578, 922, 3722, 4116, 6641, 8884] avg=4096.0000 max=8884 max_rank=7 min=922 min_rank=3 imbalance_ratio=2.168945 max_load_reduction=23884 max_load_reduction_pct=0.728882 moved_to_echo=23363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[116, 0, 0, 594, 704, 891, 337, 0, 0, 13, 1261, 123, 0, 0, 0, 0, 1368, 0, 0, 1706, 0, 0, 0, 0, 2681, 0, 4089, 0, 0, 0, 0, 0, 3, 4, 430, 0, 1430, 1489, 0, 0, 3932, 0, 2218, 12, 0, 0, 0, 0, 2508, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4089, 667, 1588, 514, 0, 0] rerouted_tokens_per_ep_rank=[2642, 1397, 3074, 6770, 3356, 6162, 2509, 6858] avg=4096.0000 max=6858 max_rank=7 min=1397 min_rank=1 imbalance_ratio=1.674316 max_load_reduction=25910 max_load_reduction_pct=0.790710 moved_to_echo=18885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[158, 0, 0, 1089, 0, 376, 330, 0, 29, 11, 1907, 11, 0, 0, 0, 0, 0, 2422, 862, 139, 0, 0, 0, 0, 1141, 1, 4078, 0, 0, 0, 0, 0, 2039, 0, 0, 147, 2151, 1465, 0, 0, 3888, 1268, 2196, 1090, 0, 0, 0, 0, 2086, 549, 0, 83, 0, 0, 0, 0, 1552, 0, 167, 2, 1316, 215, 0, 0] rerouted_tokens_per_ep_rank=[1953, 1958, 3423, 5220, 5802, 8442, 2718, 3252] avg=4096.0000 max=8442 max_rank=5 min=1953 min_rank=0 imbalance_ratio=2.061035 max_load_reduction=24326 max_load_reduction_pct=0.742371 moved_to_echo=20214 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 6016, 7808, 4864, 2048, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6144, 256, 768, 3456, 13952, 8576, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6144, 8192, 12288, 6400, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8832, 5888, 12928, 5376, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2048, 128, 16256, 4480, 7424, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11776, 12928, 4224, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14592, 256, 17408, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[15360, 11008, 1024, 5760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[345, 0, 0, 3, 4096, 0, 1, 4096, 0, 4008, 0, 1498, 4056, 77, 0, 4069, 506, 4, 0, 3429, 77, 1, 16, 4, 0, 191, 0, 2382, 0, 3909, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[3957, 0, 3347, 0, 3100, 0, 0, 244, 5, 0, 24, 4096, 168, 0, 61, 577, 42, 0, 0, 1206, 1365, 4090, 0, 142, 5, 4096, 0, 2236, 69, 3938, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2537, 0, 1503, 1, 4093, 0, 817, 21, 0, 122, 0, 2, 3443, 2, 94, 4089, 3976, 0, 0, 2861, 8, 1971, 2, 22, 14, 33, 5, 290, 1, 3992, 0, 2869] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 3823, 3, 0, 1, 4094, 4087, 0, 4012, 2, 259, 9, 3660, 0, 618, 52, 0, 13, 191, 76, 226, 0, 4096, 11, 0, 0, 3442, 23, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[505, 17, 0, 187, 3799, 0, 1, 1914, 0, 2830, 0, 4069, 4079, 15, 0, 4066, 0, 0, 0, 5, 0, 26, 0, 4080, 0, 7, 7, 4096, 1216, 739, 36, 1074] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[906, 0, 0, 4, 1333, 0, 0, 0, 3753, 3841, 5, 3603, 4053, 289, 0, 85, 0, 39, 0, 3799, 0, 1753, 0, 1202, 0, 3964, 0, 4072, 0, 65, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 177, 0, 4081, 12, 0, 0, 535, 4031, 1, 3140, 4093, 2, 349, 4096, 23, 0, 2, 55, 4090, 0, 945, 0, 134, 2457, 156, 0, 237, 3803, 3, 346] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 4039, 0, 1405, 0, 193, 224, 2, 4096, 3318, 0, 2, 92, 3882, 0, 0, 0, 0, 19, 0, 0, 0, 3933, 0, 4096, 3324, 4087, 0, 50] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[345, 0, 0, 3, 928, 0, 0, 0, 3503, 0, 1, 4096, 0, 0, 0, 0, 0, 4008, 0, 308, 0, 0, 0, 0, 1902, 77, 0, 4069, 0, 0, 0, 0, 506, 4, 0, 3429, 731, 593, 35, 74, 77, 1, 16, 4, 1423, 262, 0, 0, 0, 156, 0, 2382, 0, 0, 0, 0, 0, 3835, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1276, 7600, 4316, 6048, 5372, 1783, 2538, 3835] avg=4096.0000 max=7600 max_rank=1 min=1276 min_rank=0 imbalance_ratio=1.855469 max_load_reduction=25168 max_load_reduction_pct=0.768066 moved_to_echo=13528 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2537, 0, 1503, 1, 1, 0, 0, 0, 3500, 0, 817, 21, 0, 0, 0, 0, 0, 122, 0, 1, 0, 0, 0, 0, 1614, 2, 94, 4089, 0, 0, 0, 0, 3976, 0, 0, 2861, 621, 593, 6, 76, 8, 1971, 2, 22, 1208, 0, 0, 0, 14, 27, 5, 290, 0, 0, 0, 0, 1, 3916, 0, 2869, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4338, 123, 5799, 8133, 3211, 336, 6786] avg=4096.0000 max=8133 max_rank=4 min=123 min_rank=2 imbalance_ratio=1.985596 max_load_reduction=24635 max_load_reduction_pct=0.751801 moved_to_echo=18466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[906, 0, 0, 4, 2234, 0, 0, 0, 1140, 0, 0, 0, 0, 0, 0, 0, 3753, 3841, 5, 738, 0, 0, 0, 0, 1900, 289, 0, 85, 0, 0, 0, 0, 0, 39, 0, 3799, 731, 193, 741, 1, 0, 1753, 0, 1202, 1422, 631, 0, 0, 0, 3223, 0, 4072, 0, 0, 0, 0, 0, 64, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3144, 1140, 8337, 2274, 5504, 5008, 7295, 66] avg=4096.0000 max=8337 max_rank=2 min=66 min_rank=7 imbalance_ratio=2.035400 max_load_reduction=24431 max_load_reduction_pct=0.745575 moved_to_echo=17873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3957, 0, 3347, 0, 2540, 0, 0, 0, 2651, 0, 0, 244, 0, 0, 0, 0, 5, 0, 24, 838, 0, 0, 0, 0, 80, 0, 61, 577, 0, 0, 0, 0, 42, 0, 0, 1206, 30, 449, 766, 75, 1365, 4090, 0, 142, 58, 718, 0, 0, 5, 3330, 0, 2236, 0, 0, 0, 0, 69, 3863, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9844, 2895, 867, 718, 2568, 6373, 5571, 3932] avg=4096.0000 max=9844 max_rank=0 min=718 min_rank=3 imbalance_ratio=2.403320 max_load_reduction=22924 max_load_reduction_pct=0.699585 moved_to_echo=18444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 2487, 0, 0, 0, 3269, 3, 0, 1, 0, 0, 0, 0, 4094, 4087, 0, 822, 0, 0, 0, 0, 2, 259, 9, 3660, 0, 0, 0, 0, 0, 618, 52, 0, 0, 554, 766, 65, 13, 191, 76, 226, 0, 703, 0, 0, 0, 3330, 11, 0, 0, 0, 0, 0, 0, 3377, 23, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6557, 3273, 9003, 3930, 2055, 1209, 3341, 3400] avg=4096.0000 max=9003 max_rank=2 min=1209 min_rank=5 imbalance_ratio=2.197998 max_load_reduction=23765 max_load_reduction_pct=0.725250 moved_to_echo=10005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[505, 17, 0, 187, 2523, 0, 0, 0, 3249, 0, 1, 1914, 0, 0, 0, 0, 0, 2830, 0, 833, 0, 0, 0, 0, 1912, 15, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 5, 735, 550, 1, 14, 0, 26, 0, 4080, 1432, 713, 0, 0, 0, 6, 7, 4096, 0, 0, 0, 0, 1216, 725, 36, 1074, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3232, 5164, 3663, 5993, 1305, 6251, 4109, 3051] avg=4096.0000 max=6251 max_rank=5 min=1305 min_rank=4 imbalance_ratio=1.526123 max_load_reduction=26517 max_load_reduction_pct=0.809235 moved_to_echo=14716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 2543, 0, 0, 0, 3451, 0, 1405, 0, 0, 0, 0, 0, 193, 224, 2, 830, 0, 0, 0, 0, 1546, 0, 2, 92, 0, 0, 0, 0, 3882, 0, 0, 0, 602, 588, 739, 80, 0, 19, 0, 0, 1170, 723, 0, 0, 0, 3194, 0, 4096, 0, 0, 0, 0, 3324, 4007, 0, 50, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2549, 4856, 1249, 1640, 5891, 1912, 7290, 7381] avg=4096.0000 max=7381 max_rank=7 min=1249 min_rank=2 imbalance_ratio=1.802002 max_load_reduction=25387 max_load_reduction_pct=0.774750 moved_to_echo=22474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 177, 0, 1947, 0, 0, 0, 3490, 12, 0, 0, 0, 0, 0, 0, 535, 4031, 1, 643, 0, 0, 0, 0, 1919, 2, 349, 4096, 0, 0, 0, 0, 23, 0, 2, 55, 738, 591, 459, 72, 4090, 0, 945, 0, 1436, 550, 0, 0, 134, 1998, 156, 0, 0, 0, 0, 0, 237, 3731, 3, 346, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2124, 3502, 5210, 6366, 1940, 7021, 2288, 4317] avg=4096.0000 max=7021 max_rank=5 min=1940 min_rank=4 imbalance_ratio=1.714111 max_load_reduction=25747 max_load_reduction_pct=0.785736 moved_to_echo=15566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8704, 19200, 128, 5120, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12416, 128, 5120, 256, 15232, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8448, 768, 128, 11392, 4224, 4224, 3584, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5632, 8064, 1152, 5760, 8192, 4352, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 15360, 256, 17280, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10880, 768, 640, 20736, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[24320, 128, 2304, 6400, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[4864, 23552, 128, 4352, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[72, 0, 79, 0, 3282, 2134, 191, 4026, 4090, 0, 0, 659, 1345, 0, 10, 8, 0, 4091, 0, 10, 1343, 0, 4059, 2, 72, 4, 459, 1, 0, 6, 4091, 2734] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4029, 6, 0, 14, 363, 0, 0, 1735, 1366, 0, 34, 4089, 0, 0, 0, 0, 4067, 0, 0, 909, 0, 4029, 0, 3909, 0, 39, 0, 19, 2, 4062, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[91, 0, 0, 60, 34, 96, 883, 3530, 943, 0, 0, 3990, 0, 6, 1790, 2053, 88, 1508, 78, 1813, 3807, 1, 3867, 0, 4079, 0, 0, 9, 0, 0, 1, 4041] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 104, 0, 3567, 581, 0, 9, 0, 1, 0, 4090, 3794, 361, 21, 4096, 8, 1419, 1, 0, 0, 16, 2954, 0, 3879, 0, 0, 0, 0, 0, 3897, 3969] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 1141, 3, 3977, 417, 0, 3858, 3472, 4095, 29, 15, 0, 2867, 0, 229, 3932, 1, 0, 1, 3631, 0, 0, 0, 0, 0, 0, 1005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[139, 0, 0, 0, 3897, 0, 0, 0, 1835, 2, 0, 4, 4096, 3, 0, 2446, 0, 3021, 189, 17, 0, 3, 3988, 7, 4096, 738, 0, 9, 1, 104, 4079, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[3826, 237, 0, 0, 0, 4034, 1, 3175, 60, 4096, 0, 284, 208, 0, 0, 60, 156, 3592, 0, 88, 1683, 487, 0, 0, 4096, 0, 1281, 1, 0, 108, 1199, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4072, 116, 0, 180, 4091, 3826, 4, 0, 4037, 120, 0, 1112, 55, 1541, 0, 0, 20, 1573, 0, 0, 0, 0, 3829, 4069, 0, 52, 0, 1, 0, 1, 4069, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4029, 6, 0, 2199, 488, 0, 0, 9, 363, 0, 0, 0, 0, 0, 0, 1325, 1366, 0, 34, 0, 0, 0, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 4, 614, 0, 0, 909, 0, 2927, 0, 0, 0, 0, 0, 3909, 0, 39, 0, 410, 1, 197, 0, 19, 2, 4062, 1897, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6722, 372, 2725, 3892, 4685, 3836, 4556, 5980] avg=4096.0000 max=6722 max_rank=0 min=372 min_rank=1 imbalance_ratio=1.641113 max_load_reduction=26046 max_load_reduction_pct=0.794861 moved_to_echo=19057 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 539, 0, 0, 0, 0, 0, 1141, 3, 0, 0, 0, 0, 3037, 417, 0, 3858, 0, 0, 0, 0, 3305, 4095, 29, 15, 0, 0, 0, 0, 0, 2867, 0, 229, 0, 0, 0, 0, 3932, 1, 0, 1, 0, 0, 0, 0, 3631, 0, 0, 0, 940, 0, 167, 0, 0, 0, 0, 466, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4634, 1144, 7312, 7444, 3096, 3934, 4738, 466] avg=4096.0000 max=7444 max_rank=3 min=466 min_rank=7 imbalance_ratio=1.817383 max_load_reduction=25324 max_load_reduction_pct=0.772827 moved_to_echo=12234 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[139, 0, 0, 0, 2198, 483, 0, 0, 2297, 0, 0, 0, 0, 0, 0, 0, 1402, 2, 0, 4, 0, 0, 0, 0, 3898, 3, 0, 2446, 0, 0, 0, 0, 0, 3021, 189, 17, 1170, 607, 0, 0, 0, 3, 2898, 7, 0, 0, 0, 0, 4096, 738, 0, 9, 433, 430, 198, 0, 1, 104, 4079, 1896, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2820, 2297, 1408, 6347, 5004, 2908, 5904, 6080] avg=4096.0000 max=6347 max_rank=3 min=1408 min_rank=2 imbalance_ratio=1.549561 max_load_reduction=26421 max_load_reduction_pct=0.806305 moved_to_echo=19896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[91, 0, 0, 60, 2169, 468, 0, 0, 21, 96, 883, 3530, 0, 0, 0, 0, 721, 0, 0, 3990, 0, 0, 0, 0, 0, 6, 1790, 2053, 0, 0, 0, 0, 88, 1508, 78, 1813, 10, 589, 0, 0, 3807, 1, 2810, 0, 0, 0, 0, 0, 4079, 0, 0, 9, 222, 3, 0, 0, 0, 0, 1, 1872, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2788, 4530, 4711, 3849, 4086, 6618, 4313, 1873] avg=4096.0000 max=6618 max_rank=5 min=1873 min_rank=7 imbalance_ratio=1.615723 max_load_reduction=26150 max_load_reduction_pct=0.798035 moved_to_echo=16890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[72, 0, 79, 0, 1468, 492, 0, 0, 1935, 2134, 191, 4026, 0, 0, 0, 0, 3124, 0, 0, 659, 0, 0, 0, 0, 1280, 0, 10, 8, 0, 0, 0, 0, 0, 4091, 0, 10, 985, 618, 0, 0, 1343, 0, 2949, 2, 0, 0, 0, 0, 72, 4, 459, 1, 966, 362, 65, 0, 0, 6, 4091, 1266, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2111, 8286, 3783, 1298, 5704, 4294, 1929, 5363] avg=4096.0000 max=8286 max_rank=1 min=1298 min_rank=3 imbalance_ratio=2.022949 max_load_reduction=24482 max_load_reduction_pct=0.747131 moved_to_echo=17290 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 104, 0, 2131, 358, 0, 0, 2103, 581, 0, 9, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 3611, 361, 21, 4096, 0, 0, 0, 0, 8, 1419, 1, 0, 1070, 450, 0, 0, 0, 16, 2146, 0, 0, 0, 0, 0, 3879, 0, 0, 0, 0, 394, 183, 0, 0, 0, 3897, 1838, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2594, 2693, 4091, 8089, 2948, 2162, 4456, 5735] avg=4096.0000 max=8089 max_rank=3 min=2162 min_rank=5 imbalance_ratio=1.974854 max_load_reduction=24679 max_load_reduction_pct=0.753143 moved_to_echo=15301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3826, 237, 0, 0, 2202, 0, 0, 0, 0, 4034, 1, 3175, 0, 0, 0, 0, 43, 4096, 0, 284, 0, 0, 0, 0, 195, 0, 0, 60, 0, 0, 0, 0, 156, 3592, 0, 88, 0, 0, 0, 0, 1683, 487, 0, 0, 0, 0, 0, 0, 4096, 0, 1281, 1, 17, 0, 13, 0, 0, 108, 1199, 1894, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6265, 7210, 4423, 255, 3836, 2170, 5408, 3201] avg=4096.0000 max=7210 max_rank=1 min=255 min_rank=3 imbalance_ratio=1.760254 max_load_reduction=25558 max_load_reduction_pct=0.779968 moved_to_echo=14615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4072, 116, 0, 180, 0, 466, 0, 0, 2406, 3826, 4, 0, 0, 0, 0, 0, 3083, 120, 0, 1112, 0, 0, 0, 0, 53, 1541, 0, 0, 0, 0, 0, 0, 20, 1573, 0, 0, 1230, 586, 0, 0, 0, 0, 2777, 4069, 0, 0, 0, 0, 0, 52, 0, 1, 954, 455, 2, 0, 0, 1, 4069, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4834, 6236, 4315, 1594, 3409, 6846, 1464, 4070] avg=4096.0000 max=6846 max_rank=5 min=1464 min_rank=6 imbalance_ratio=1.671387 max_load_reduction=25922 max_load_reduction_pct=0.791077 moved_to_echo=15789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[16256, 6016, 1920, 8704, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8832, 11136, 2304, 10752, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12416, 4480, 256, 256, 12928, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 256, 21504, 11136, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[11776, 512, 16512, 4096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[12800, 6016, 0, 14080, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[23808, 896, 1792, 128, 3968, 1664, 896, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 22144, 384, 2176, 4480, 3584, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 3644, 0, 26, 0, 54, 26, 0, 2, 2954, 4079, 389, 2572, 158, 4064, 2207, 0, 1167, 1169, 0, 248, 3779, 280, 106, 42, 0, 0, 4, 1362, 0, 4047, 389] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 753, 0, 30, 130, 32, 0, 1, 10, 3284, 43, 3990, 4096, 0, 241, 3955, 0, 3940, 3950, 0, 0, 122, 11, 0, 1, 7, 0, 4, 118, 4041, 4009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 26, 0, 13, 4, 0, 927, 3294, 3971, 11, 4, 1121, 1233, 4053, 3931, 3239, 4094, 1, 3599, 976, 1745, 0, 0, 0, 0, 0, 102, 200, 23, 200] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4071, 45, 11, 63, 10, 183, 1, 4093, 0, 0, 1, 4087, 3861, 10, 4096, 2985, 0, 95, 21, 2592, 0, 2, 1568, 0, 0, 0, 900, 4073, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[214, 4090, 1127, 0, 1, 0, 0, 0, 10, 366, 4091, 56, 14, 632, 665, 0, 3, 4094, 8, 517, 0, 3240, 2646, 3383, 0, 0, 5, 0, 0, 0, 4068, 3538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1403, 28, 0, 6, 1, 3207, 1882, 0, 4033, 4096, 174, 0, 0, 1, 106, 4096, 0, 4085, 2825, 4, 3872, 0, 0, 170, 0, 0, 38, 0, 0, 1168, 1573] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1, 57, 0, 85, 0, 210, 0, 500, 4096, 483, 3581, 0, 9, 0, 1, 0, 3979, 4091, 0, 130, 3154, 3130, 3991, 3581, 0, 1460, 0, 0, 12, 217, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 107, 0, 4096, 4096, 4011, 0, 584, 0, 4094, 0, 72, 0, 2506, 0, 3410, 4041, 129, 1169, 0, 0, 0, 0, 360, 0, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3644, 0, 26, 516, 1444, 614, 325, 0, 54, 26, 0, 2668, 807, 0, 0, 2, 2954, 986, 389, 0, 0, 0, 0, 2572, 158, 2620, 2207, 0, 0, 0, 0, 0, 651, 0, 0, 0, 0, 0, 0, 248, 497, 280, 106, 0, 0, 0, 0, 42, 0, 0, 4, 3093, 362, 0, 0, 1362, 0, 3722, 389, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6569, 3555, 4331, 7557, 651, 1131, 3501, 5473] avg=4096.0000 max=7557 max_rank=3 min=651 min_rank=4 imbalance_ratio=1.844971 max_load_reduction=25211 max_load_reduction_pct=0.769379 moved_to_echo=10756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[214, 4090, 1127, 0, 1812, 236, 526, 326, 1, 0, 0, 0, 2287, 5, 0, 0, 10, 366, 989, 56, 0, 0, 0, 0, 14, 632, 429, 0, 0, 0, 0, 0, 3, 2282, 0, 517, 0, 0, 0, 0, 0, 427, 2646, 3383, 0, 0, 0, 0, 0, 0, 5, 0, 3102, 3, 0, 0, 0, 0, 3742, 3538, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8331, 2293, 1421, 1075, 2802, 6456, 3110, 7280] avg=4096.0000 max=8331 max_rank=0 min=1075 min_rank=3 imbalance_ratio=2.033936 max_load_reduction=24437 max_load_reduction_pct=0.745758 moved_to_echo=19648 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4071, 1813, 1452, 3, 327, 45, 11, 63, 10, 14, 2063, 0, 0, 183, 1, 990, 0, 0, 0, 0, 0, 0, 1, 2635, 3861, 0, 0, 0, 0, 10, 2283, 0, 0, 0, 0, 0, 0, 95, 4, 2592, 0, 0, 0, 0, 0, 2, 1568, 0, 0, 3103, 922, 0, 0, 0, 900, 3746, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7666, 2206, 1174, 6497, 2293, 2691, 5595, 4646] avg=4096.0000 max=7666 max_rank=0 min=1174 min_rank=2 imbalance_ratio=1.871582 max_load_reduction=25102 max_load_reduction_pct=0.766052 moved_to_echo=15225 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 753, 0, 0, 0, 0, 324, 30, 130, 32, 0, 0, 2722, 0, 0, 1, 10, 794, 43, 0, 0, 0, 0, 3990, 4096, 0, 241, 0, 0, 0, 0, 3955, 0, 0, 3950, 0, 0, 0, 0, 0, 0, 122, 11, 0, 0, 0, 0, 0, 1, 7, 0, 2490, 1218, 0, 0, 4, 118, 3717, 4009, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1077, 2914, 848, 8327, 7905, 133, 3716, 7848] avg=4096.0000 max=8327 max_rank=3 min=133 min_rank=5 imbalance_ratio=2.032959 max_load_reduction=24441 max_load_reduction_pct=0.745880 moved_to_echo=19602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 26, 1434, 438, 158, 1, 0, 13, 4, 0, 689, 2828, 0, 0, 927, 3294, 960, 11, 0, 0, 0, 0, 4, 1121, 795, 4053, 0, 0, 0, 0, 3931, 1805, 0, 1, 0, 0, 0, 0, 3599, 129, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3011, 1266, 0, 0, 102, 200, 22, 200, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2058, 3534, 5192, 5973, 5737, 5473, 4277, 524] avg=4096.0000 max=5973 max_rank=3 min=524 min_rank=7 imbalance_ratio=1.458252 max_load_reduction=26795 max_load_reduction_pct=0.817719 moved_to_echo=16011 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1403, 28, 0, 0, 0, 629, 93, 6, 1, 3207, 1882, 2733, 2822, 0, 0, 0, 4033, 991, 174, 0, 0, 0, 0, 0, 0, 1, 106, 0, 0, 0, 0, 4096, 0, 0, 2825, 0, 0, 0, 0, 4, 510, 0, 0, 0, 0, 0, 0, 170, 0, 0, 38, 3105, 1263, 0, 0, 0, 0, 1075, 1573, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2153, 10651, 5198, 107, 6921, 514, 4576, 2648] avg=4096.0000 max=10651 max_rank=1 min=107 min_rank=3 imbalance_ratio=2.600342 max_load_reduction=22117 max_load_reduction_pct=0.674957 moved_to_echo=14659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 1456, 661, 332, 0, 0, 107, 0, 2856, 1732, 0, 0, 4096, 4096, 967, 0, 0, 0, 0, 0, 584, 0, 2638, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 3410, 524, 129, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 3044, 774, 0, 0, 360, 0, 3760, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2450, 4695, 9159, 3222, 72, 5232, 3818, 4120] avg=4096.0000 max=9159 max_rank=2 min=72 min_rank=4 imbalance_ratio=2.236084 max_load_reduction=23609 max_load_reduction_pct=0.720490 moved_to_echo=13242 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 57, 0, 85, 1813, 0, 508, 0, 0, 210, 0, 500, 2210, 0, 0, 0, 4096, 483, 866, 0, 0, 0, 0, 0, 9, 0, 1, 0, 0, 0, 0, 0, 3979, 2278, 0, 130, 0, 0, 0, 0, 3154, 412, 3991, 3581, 0, 0, 0, 0, 0, 1460, 0, 0, 2715, 0, 0, 0, 12, 217, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2464, 2920, 5445, 10, 6387, 11138, 4175, 229] avg=4096.0000 max=11138 max_rank=5 min=10 min_rank=3 imbalance_ratio=2.719238 max_load_reduction=21630 max_load_reduction_pct=0.660095 moved_to_echo=21929 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 9216, 1920, 4224, 7424, 5120, 3200, 1792] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10624, 2560, 11520, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7296, 6016, 9216, 10496, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[16128, 9344, 0, 7424, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 3456, 2432, 13568, 13056, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9344, 15360, 7552, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1920, 1536, 19840, 9728, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 3072, 128, 128, 23680, 5888, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 3590, 8, 0, 0, 3913, 0, 0, 68, 0, 34, 4062, 5, 0, 4092, 0, 1, 0, 4096, 4091, 1299, 1208, 193, 25, 0, 0, 109, 0, 2975, 0, 2936, 63] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[2550, 4056, 880, 0, 0, 3466, 0, 24, 0, 2958, 0, 4, 1291, 314, 0, 0, 0, 154, 3907, 4092, 6, 33, 1051, 23, 4040, 344, 3486, 0, 15, 0, 74, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3843, 0, 2991, 22, 4095, 3833, 82, 4037, 0, 0, 3998, 92, 0, 616, 4065, 0, 67, 0, 4083, 0, 203, 0, 0, 0, 1, 6, 590, 0, 4, 83, 0, 57] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 3475, 3139, 84, 0, 452, 3994, 4096, 5, 0, 178, 4096, 0, 0, 4049, 0, 16, 175, 4053, 4028, 44, 0, 0, 0, 874, 0, 1, 0, 0, 8, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[138, 2, 6, 4096, 4069, 0, 0, 0, 0, 0, 3879, 15, 268, 124, 3172, 1, 226, 3932, 113, 1423, 0, 0, 60, 3602, 3, 3738, 13, 0, 3626, 0, 0, 262] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 3999, 2599, 0, 3983, 1, 1288, 0, 0, 204, 0, 3402, 4, 2771, 337, 64, 0, 33, 3991, 858, 0, 0, 95, 0, 1342, 23, 4084, 320, 3349, 18, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3809, 0, 1663, 0, 4093, 13, 11, 35, 0, 8, 3510, 45, 293, 3361, 0, 55, 0, 0, 3519, 0, 9, 4, 5, 0, 17, 0, 145, 0, 4095, 0, 4096, 3982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4092, 4051, 1, 1852, 28, 2594, 15, 4, 2, 0, 1, 4083, 0, 0, 4095, 225, 0, 1, 3954, 250, 3162, 0, 244, 0, 3340, 0, 11, 97, 620, 0, 2, 44] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2550, 211, 880, 0, 0, 0, 0, 0, 0, 3466, 0, 24, 0, 0, 0, 0, 0, 2958, 0, 4, 0, 0, 0, 0, 1291, 314, 0, 0, 0, 0, 0, 0, 0, 154, 1894, 4092, 0, 0, 0, 0, 6, 33, 1051, 23, 3845, 468, 0, 0, 4040, 344, 3486, 0, 1435, 0, 0, 0, 15, 0, 74, 0, 0, 110, 0, 0] rerouted_tokens_per_ep_rank=[3641, 3490, 2962, 1605, 6140, 5426, 9305, 199] avg=4096.0000 max=9305 max_rank=6 min=199 min_rank=7 imbalance_ratio=2.271729 max_load_reduction=23463 max_load_reduction_pct=0.716034 moved_to_echo=21070 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 208, 2599, 0, 0, 0, 0, 0, 1203, 1, 1288, 0, 0, 0, 0, 0, 0, 204, 0, 3402, 520, 0, 0, 0, 4, 2771, 337, 64, 873, 0, 0, 0, 0, 33, 1934, 858, 0, 0, 0, 0, 0, 0, 95, 0, 3791, 478, 0, 0, 1342, 23, 4084, 320, 1466, 0, 0, 0, 3349, 18, 0, 3, 1387, 113, 0, 0] rerouted_tokens_per_ep_rank=[2807, 2492, 4126, 4049, 2825, 4364, 7235, 4870] avg=4096.0000 max=7235 max_rank=6 min=2492 min_rank=1 imbalance_ratio=1.766357 max_load_reduction=25533 max_load_reduction_pct=0.779205 moved_to_echo=19294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 187, 8, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 68, 0, 34, 4062, 0, 0, 0, 0, 5, 0, 4092, 0, 0, 0, 0, 0, 1, 0, 1984, 4091, 0, 0, 0, 0, 1299, 1208, 193, 25, 3403, 491, 0, 0, 0, 0, 109, 0, 1505, 0, 0, 0, 2975, 0, 2936, 63, 0, 116, 0, 0] rerouted_tokens_per_ep_rank=[195, 3913, 4164, 4097, 6076, 6619, 1614, 6090] avg=4096.0000 max=6619 max_rank=5 min=195 min_rank=0 imbalance_ratio=1.615967 max_load_reduction=26149 max_load_reduction_pct=0.798004 moved_to_echo=20399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3843, 0, 2991, 22, 0, 0, 0, 0, 1238, 3833, 82, 4037, 0, 0, 0, 0, 0, 0, 3998, 92, 534, 0, 0, 0, 0, 616, 4065, 0, 897, 0, 0, 0, 67, 0, 1979, 0, 0, 0, 0, 0, 203, 0, 0, 0, 0, 489, 0, 0, 1, 6, 590, 0, 1500, 0, 0, 0, 4, 83, 0, 57, 1426, 115, 0, 0] rerouted_tokens_per_ep_rank=[6856, 9190, 4624, 5578, 2046, 692, 2097, 1685] avg=4096.0000 max=9190 max_rank=1 min=692 min_rank=5 imbalance_ratio=2.243652 max_load_reduction=23578 max_load_reduction_pct=0.719543 moved_to_echo=6520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 181, 3139, 84, 0, 0, 0, 0, 0, 452, 3994, 4096, 0, 0, 0, 0, 5, 0, 178, 4096, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 16, 175, 1963, 4028, 0, 0, 0, 0, 44, 0, 0, 0, 3294, 486, 0, 0, 874, 0, 1, 0, 1489, 0, 0, 0, 0, 8, 1, 0, 0, 115, 0, 0] rerouted_tokens_per_ep_rank=[3404, 8542, 4279, 4049, 6182, 3824, 2364, 124] avg=4096.0000 max=8542 max_rank=1 min=124 min_rank=7 imbalance_ratio=2.085449 max_load_reduction=24226 max_load_reduction_pct=0.739319 moved_to_echo=12494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[138, 1, 6, 4096, 0, 0, 0, 0, 1230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 15, 531, 0, 0, 0, 268, 124, 3172, 1, 891, 0, 0, 0, 226, 3932, 56, 1423, 0, 0, 0, 0, 0, 0, 60, 3602, 1, 13, 0, 0, 3, 3738, 13, 0, 41, 0, 0, 0, 3626, 0, 0, 262, 1417, 3, 0, 0] rerouted_tokens_per_ep_rank=[4241, 1230, 4425, 4456, 5637, 3676, 3795, 5308] avg=4096.0000 max=5637 max_rank=4 min=1230 min_rank=1 imbalance_ratio=1.376221 max_load_reduction=27131 max_load_reduction_pct=0.827972 moved_to_echo=18416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4092, 207, 1, 1852, 0, 0, 0, 0, 3, 2594, 15, 4, 0, 0, 0, 0, 2, 0, 1, 4083, 5, 0, 0, 0, 0, 0, 4095, 225, 8, 0, 0, 0, 0, 1, 1906, 250, 0, 0, 0, 0, 3162, 0, 244, 0, 3844, 477, 0, 0, 3340, 0, 11, 97, 1455, 0, 0, 0, 620, 0, 2, 44, 12, 116, 0, 0] rerouted_tokens_per_ep_rank=[6152, 2616, 4091, 4328, 2157, 7727, 4903, 794] avg=4096.0000 max=7727 max_rank=5 min=794 min_rank=7 imbalance_ratio=1.886475 max_load_reduction=25041 max_load_reduction_pct=0.764191 moved_to_echo=15581 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3809, 0, 1663, 0, 0, 0, 0, 0, 1236, 13, 11, 35, 0, 0, 0, 0, 0, 8, 3510, 45, 534, 0, 0, 0, 293, 3361, 0, 55, 897, 0, 0, 0, 0, 0, 1705, 0, 0, 0, 0, 0, 9, 4, 5, 0, 0, 422, 0, 0, 17, 0, 145, 0, 1293, 0, 0, 0, 4095, 0, 4096, 3982, 1426, 99, 0, 0] rerouted_tokens_per_ep_rank=[5472, 1295, 4097, 4606, 1705, 440, 1455, 13698] avg=4096.0000 max=13698 max_rank=7 min=440 min_rank=5 imbalance_ratio=3.344238 max_load_reduction=19070 max_load_reduction_pct=0.581970 moved_to_echo=17298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14720, 128, 7168, 4480, 5760, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4992, 14336, 5504, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9728, 4224, 8448, 512, 10240, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 4352, 13440, 14848, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14464, 1024, 11392, 6144, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4736, 1280, 1664, 3712, 18304, 3328, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1920, 7296, 19840, 384, 3584, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 3200, 11648, 15872, 2176, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 21, 3584, 0, 0, 0, 0, 4063, 0, 4095, 3318, 3862, 0, 0, 42, 0, 4096, 0, 855, 2115, 1213, 10, 807, 0, 0, 556, 7, 62, 4040, 22, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[228, 2941, 3621, 4090, 1, 1, 5, 0, 0, 4096, 3875, 0, 275, 19, 0, 0, 4096, 4094, 164, 4083, 0, 0, 1, 0, 0, 0, 0, 0, 601, 1, 0, 576] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 1105, 49, 0, 0, 0, 464, 16, 4093, 116, 3912, 0, 32, 2445, 0, 3708, 1514, 3327, 4082, 0, 9, 0, 11, 0, 3, 0, 3802, 2642, 0, 1436, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 3475, 4086, 2, 87, 27, 0, 0, 0, 4091, 4096, 0, 1, 0, 0, 1, 3905, 2, 8, 4092, 224, 3, 0, 0, 0, 252, 0, 3120, 4096, 0, 0, 1200] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 217, 3918, 0, 805, 0, 1, 1795, 1, 4041, 389, 92, 4002, 1708, 3, 0, 3170, 928, 15, 4086, 0, 0, 3, 4087, 86, 0, 0, 3358, 0, 62, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3960, 11, 94, 0, 0, 1010, 7, 4088, 4096, 0, 0, 0, 3507, 0, 4096, 2900, 0, 4096, 0, 0, 133, 338, 0, 835, 0, 3597, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 3175, 0, 3, 0, 0, 2341, 4095, 3585, 216, 3359, 19, 0, 3991, 1429, 0, 0, 0, 3, 0, 0, 43, 1910, 71, 32, 329, 15, 4, 4053] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 82, 0, 0, 0, 0, 0, 2, 4096, 4095, 2911, 0, 0, 2612, 0, 4074, 13, 1, 4095, 0, 0, 0, 0, 0, 22, 0, 3391, 4086, 0, 2965, 323] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 21, 3186, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 2121, 685, 0, 0, 0, 0, 2534, 3862, 0, 0, 0, 0, 0, 0, 42, 0, 1922, 0, 0, 0, 0, 0, 855, 1391, 0, 0, 0, 0, 1213, 10, 807, 0, 3411, 0, 0, 0, 0, 556, 7, 62, 724, 289, 52, 0, 4040, 22, 0, 0, 495, 398, 0, 0] rerouted_tokens_per_ep_rank=[3207, 6869, 6396, 1964, 2246, 5441, 1690, 4955] avg=4096.0000 max=6869 max_rank=1 min=1690 min_rank=6 imbalance_ratio=1.677002 max_load_reduction=25899 max_load_reduction_pct=0.790375 moved_to_echo=14332 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 983, 49, 0, 0, 0, 0, 0, 0, 0, 464, 2120, 621, 0, 0, 16, 0, 89, 3912, 0, 0, 0, 0, 0, 32, 2445, 0, 1922, 0, 0, 0, 0, 1514, 3327, 2684, 0, 0, 0, 0, 0, 9, 0, 11, 3087, 0, 0, 0, 0, 3, 0, 3802, 1398, 10, 51, 0, 2642, 0, 1436, 0, 17, 122, 0, 0] rerouted_tokens_per_ep_rank=[1034, 3205, 4017, 4399, 7525, 3107, 5264, 4217] avg=4096.0000 max=7525 max_rank=4 min=1034 min_rank=0 imbalance_ratio=1.837158 max_load_reduction=25243 max_load_reduction_pct=0.770355 moved_to_echo=20113 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 681, 0, 0, 2, 0, 3127, 2911, 0, 0, 0, 0, 0, 0, 2612, 0, 1924, 0, 0, 0, 0, 13, 1, 2692, 0, 0, 0, 0, 0, 0, 0, 0, 3393, 0, 0, 0, 0, 22, 0, 3391, 1403, 357, 51, 0, 4086, 0, 2965, 323, 611, 9, 0, 0] rerouted_tokens_per_ep_rank=[73, 2802, 6040, 4536, 2706, 3393, 5224, 7994] avg=4096.0000 max=7994 max_rank=7 min=73 min_rank=0 imbalance_ratio=1.951660 max_load_reduction=24774 max_load_reduction_pct=0.756042 moved_to_echo=19317 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 217, 3483, 0, 0, 0, 0, 0, 805, 0, 1, 1795, 2093, 530, 0, 0, 1, 0, 298, 92, 0, 0, 0, 0, 4002, 1708, 3, 0, 1897, 0, 0, 0, 0, 928, 15, 2686, 0, 0, 0, 0, 0, 0, 3, 4087, 2640, 0, 0, 0, 86, 0, 0, 3358, 1400, 33, 51, 0, 0, 62, 0, 1, 58, 435, 0, 0] rerouted_tokens_per_ep_rank=[3700, 5224, 391, 7610, 3629, 6730, 4928, 556] avg=4096.0000 max=7610 max_rank=3 min=391 min_rank=2 imbalance_ratio=1.857910 max_load_reduction=25158 max_load_reduction_pct=0.767761 moved_to_echo=15843 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[228, 2941, 3219, 4090, 0, 0, 0, 0, 1, 1, 5, 0, 2122, 686, 0, 0, 0, 0, 2959, 0, 0, 0, 0, 0, 275, 19, 0, 0, 1923, 0, 0, 0, 0, 4094, 164, 2685, 0, 0, 0, 0, 0, 0, 1, 0, 3410, 0, 0, 0, 0, 0, 0, 0, 1398, 338, 51, 0, 601, 1, 0, 576, 578, 402, 0, 0] rerouted_tokens_per_ep_rank=[10478, 2815, 2959, 2217, 6943, 3411, 1787, 2158] avg=4096.0000 max=10478 max_rank=0 min=1787 min_rank=6 imbalance_ratio=2.558105 max_load_reduction=22290 max_load_reduction_pct=0.680237 moved_to_echo=14299 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3475, 3632, 2, 0, 0, 0, 0, 87, 27, 0, 0, 2119, 653, 0, 0, 0, 0, 3128, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1921, 0, 0, 0, 0, 2, 8, 2690, 0, 0, 0, 0, 224, 3, 0, 0, 3252, 0, 0, 0, 0, 252, 0, 3120, 1402, 357, 51, 0, 4096, 0, 0, 1200, 611, 454, 0, 0] rerouted_tokens_per_ep_rank=[7109, 2886, 3128, 1923, 2700, 3479, 5182, 6361] avg=4096.0000 max=7109 max_rank=0 min=1923 min_rank=3 imbalance_ratio=1.735596 max_load_reduction=25659 max_load_reduction_pct=0.783051 moved_to_echo=17722 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3516, 11, 0, 0, 0, 0, 94, 0, 0, 1010, 2118, 686, 0, 0, 7, 0, 3118, 0, 0, 0, 0, 0, 0, 0, 3507, 0, 1919, 0, 0, 0, 0, 2900, 0, 2690, 0, 0, 0, 0, 0, 0, 133, 338, 3410, 0, 0, 0, 0, 835, 0, 3597, 1406, 362, 51, 0, 0, 0, 0, 0, 616, 444, 0, 0] rerouted_tokens_per_ep_rank=[3527, 3908, 3125, 5426, 5590, 3881, 6251, 1060] avg=4096.0000 max=6251 max_rank=6 min=1060 min_rank=7 imbalance_ratio=1.526123 max_load_reduction=26517 max_load_reduction_pct=0.809235 moved_to_echo=16782 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3640, 0, 0, 0, 0, 0, 3175, 0, 3, 0, 1213, 668, 0, 0, 0, 0, 3127, 3585, 0, 0, 0, 0, 216, 3359, 19, 0, 1099, 0, 0, 0, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3323, 0, 0, 0, 43, 1910, 71, 32, 0, 357, 29, 0, 329, 15, 4, 4053, 611, 455, 0, 0] rerouted_tokens_per_ep_rank=[3640, 5059, 6712, 4693, 1429, 3326, 2442, 5467] avg=4096.0000 max=6712 max_rank=2 min=1429 min_rank=4 imbalance_ratio=1.638672 max_load_reduction=26056 max_load_reduction_pct=0.795166 moved_to_echo=12664 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 6656, 21760, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 128, 128, 7424, 16128, 5248, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4608, 5120, 8704, 128, 14592, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 10880, 4480, 17536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 18432, 14464, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15872, 128, 4480, 6272, 3712, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 3584, 128, 17408, 9216, 2176, 512, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1536, 128, 1024, 4480, 25984, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4095, 1229, 1, 0, 0, 0, 4067, 1, 0, 4094, 3724, 0, 1261, 2358, 18, 0, 0, 0, 63, 1484, 641, 0, 261, 27, 1, 0, 32, 1350, 4082, 0, 0, 3979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[16, 4039, 23, 56, 2, 0, 2472, 8, 0, 4092, 0, 114, 4096, 0, 139, 1531, 1244, 0, 0, 2390, 4019, 0, 5, 4, 11, 8, 0, 3643, 469, 0, 291, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[75, 4090, 690, 0, 17, 0, 0, 517, 70, 110, 2913, 0, 3977, 11, 8, 0, 3377, 0, 0, 3942, 0, 0, 0, 1756, 0, 4096, 0, 1, 3069, 4048, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[13, 4096, 0, 245, 1, 3064, 0, 0, 0, 4095, 1105, 0, 380, 0, 0, 82, 0, 2761, 0, 105, 3669, 0, 0, 326, 0, 0, 13, 2699, 2046, 0, 3972, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4095, 3792, 0, 16, 0, 0, 4033, 0, 0, 3923, 3974, 0, 2, 0, 25, 3, 3, 1, 200, 89, 3977, 50, 475, 0, 0, 9, 0, 131, 4095, 6, 3604, 265] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 3645, 0, 0, 131, 0, 3837, 1, 159, 5, 0, 2523, 0, 0, 4095, 1, 0, 1079, 2257, 0, 3512, 570, 13, 1994, 10, 4096, 0, 1929, 0, 2898, 13, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[261, 3797, 2, 0, 247, 2676, 116, 0, 414, 3751, 3, 0, 2, 0, 2, 0, 0, 1, 0, 3888, 4026, 0, 2872, 3177, 26, 0, 67, 3749, 3669, 22, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[89, 3862, 107, 130, 0, 0, 2554, 3598, 54, 3338, 71, 4, 4096, 36, 0, 0, 3886, 0, 0, 4, 0, 220, 9, 4046, 0, 0, 758, 2, 9, 1803, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[13, 3279, 0, 245, 0, 0, 0, 0, 1, 3064, 0, 0, 532, 305, 137, 0, 0, 3087, 1105, 0, 0, 0, 0, 0, 380, 0, 0, 82, 1249, 0, 0, 0, 0, 2761, 0, 105, 512, 424, 0, 0, 3137, 0, 0, 326, 0, 0, 0, 0, 0, 0, 13, 2699, 797, 584, 0, 0, 0, 0, 3972, 3959, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3537, 4039, 4192, 1711, 3802, 3463, 4093, 7931] avg=4096.0000 max=7931 max_rank=7 min=1711 min_rank=3 imbalance_ratio=1.936279 max_load_reduction=24837 max_load_reduction_pct=0.757965 moved_to_echo=19289 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4095, 985, 1, 0, 0, 0, 0, 0, 0, 0, 4067, 1, 93, 91, 133, 0, 0, 3086, 3724, 0, 0, 0, 0, 0, 1261, 2358, 18, 0, 2491, 0, 0, 0, 0, 0, 63, 1484, 153, 424, 0, 0, 548, 0, 261, 27, 0, 0, 0, 0, 1, 0, 32, 1350, 1591, 584, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5081, 4385, 6810, 6128, 2124, 836, 3558, 3846] avg=4096.0000 max=6810 max_rank=2 min=836 min_rank=5 imbalance_ratio=1.662598 max_load_reduction=25958 max_load_reduction_pct=0.792175 moved_to_echo=10364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[75, 3275, 690, 0, 0, 0, 0, 0, 17, 0, 0, 517, 0, 304, 0, 0, 70, 84, 2913, 0, 0, 0, 0, 0, 3977, 11, 8, 0, 1874, 0, 0, 0, 3377, 0, 0, 3942, 511, 11, 0, 0, 0, 0, 0, 1756, 0, 0, 0, 0, 0, 4096, 0, 1, 1195, 15, 0, 0, 0, 4048, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 838, 3067, 5870, 7841, 1756, 5307, 4049] avg=4096.0000 max=7841 max_rank=4 min=838 min_rank=1 imbalance_ratio=1.914307 max_load_reduction=24927 max_load_reduction_pct=0.760712 moved_to_echo=18953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[16, 3226, 23, 56, 0, 0, 0, 0, 2, 0, 2472, 8, 586, 305, 140, 0, 0, 3076, 0, 114, 0, 0, 0, 0, 4096, 0, 139, 1531, 287, 0, 0, 0, 1244, 0, 0, 2390, 508, 427, 0, 0, 3433, 0, 5, 4, 0, 0, 0, 0, 11, 8, 0, 3643, 182, 589, 0, 0, 0, 0, 291, 3956, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3321, 3513, 3190, 6053, 4569, 3442, 4433, 4247] avg=4096.0000 max=6053 max_rank=3 min=3190 min_rank=2 imbalance_ratio=1.477783 max_load_reduction=26715 max_load_reduction_pct=0.815277 moved_to_echo=16691 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4095, 3036, 0, 16, 0, 0, 0, 0, 0, 0, 4033, 0, 577, 282, 8, 0, 0, 2957, 3974, 0, 0, 0, 0, 0, 2, 0, 25, 3, 2499, 0, 0, 0, 3, 1, 200, 89, 474, 406, 0, 0, 3400, 50, 475, 0, 0, 0, 0, 0, 0, 9, 0, 131, 1596, 560, 0, 0, 0, 6, 3604, 257, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7147, 4900, 6931, 2529, 1173, 3925, 2296, 3867] avg=4096.0000 max=7147 max_rank=0 min=1173 min_rank=4 imbalance_ratio=1.744873 max_load_reduction=25621 max_load_reduction_pct=0.781891 moved_to_echo=11261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 2919, 0, 0, 0, 0, 0, 0, 131, 0, 3837, 1, 509, 271, 0, 0, 159, 5, 0, 2523, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1079, 2257, 0, 455, 0, 0, 0, 3003, 570, 13, 1994, 0, 0, 0, 0, 10, 4096, 0, 1929, 0, 0, 0, 0, 0, 2898, 13, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2919, 4749, 2687, 4096, 3791, 5580, 6035, 2911] avg=4096.0000 max=6035 max_rank=6 min=2687 min_rank=2 imbalance_ratio=1.473389 max_load_reduction=26733 max_load_reduction_pct=0.815826 moved_to_echo=18317 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[89, 3093, 107, 130, 0, 0, 0, 0, 0, 0, 2554, 3598, 0, 287, 0, 0, 54, 2517, 71, 4, 0, 0, 0, 0, 4096, 36, 0, 0, 5, 0, 0, 0, 3886, 0, 0, 4, 482, 345, 0, 0, 0, 220, 9, 4046, 0, 0, 0, 0, 0, 0, 758, 2, 4, 476, 0, 0, 0, 1803, 4092, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3419, 6439, 2646, 4137, 4717, 4275, 1240, 5895] avg=4096.0000 max=6439 max_rank=1 min=1240 min_rank=6 imbalance_ratio=1.572021 max_load_reduction=26329 max_load_reduction_pct=0.803497 moved_to_echo=16127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[261, 3041, 2, 0, 0, 0, 0, 0, 247, 2676, 116, 0, 584, 282, 0, 0, 414, 2828, 3, 0, 0, 0, 0, 0, 2, 0, 2, 0, 2240, 0, 0, 0, 0, 1, 0, 3888, 474, 388, 0, 0, 3442, 0, 2872, 3177, 0, 0, 0, 0, 26, 0, 67, 3749, 1429, 535, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3304, 3905, 3245, 2244, 4751, 9491, 5806, 22] avg=4096.0000 max=9491 max_rank=5 min=22 min_rank=7 imbalance_ratio=2.317139 max_load_reduction=23277 max_load_reduction_pct=0.710358 moved_to_echo=20070 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17024, 896, 3712, 11392, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 5760, 17152, 4224, 2944, 2176, 512, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8576, 3968, 2560, 11904, 3584, 2432, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 8832, 12032, 12032, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 17664, 11904, 2688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13824, 2432, 4352, 1664, 10752, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8704, 22912, 896, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 8320, 896, 13568, 6912, 3456, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[6, 0, 2356, 7, 3219, 3, 2, 0, 3197, 0, 295, 0, 3986, 3795, 0, 0, 0, 0, 6, 4089, 4087, 0, 1517, 208, 1109, 641, 0, 0, 5, 143, 1, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[2, 1, 588, 4013, 3444, 0, 0, 4087, 3954, 0, 22, 0, 253, 1906, 9, 4093, 3927, 100, 8, 3900, 1729, 0, 0, 11, 133, 0, 0, 503, 80, 0, 4, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 30, 13, 0, 3995, 4081, 0, 6, 2850, 0, 0, 70, 0, 3820, 27, 0, 3966, 2, 86, 4095, 111, 0, 4096, 0, 0, 0, 328, 2235, 2, 15, 0, 2940] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[26, 0, 0, 4079, 3766, 4096, 330, 2, 4096, 116, 0, 1839, 829, 1736, 0, 3897, 84, 0, 0, 4021, 0, 0, 0, 412, 0, 4, 63, 0, 0, 0, 17, 3355] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 181, 1, 136, 3951, 3853, 108, 412, 4049, 0, 0, 2, 1, 247, 20, 35, 1581, 0, 1, 4076, 0, 3, 116, 4096, 0, 36, 15, 3986, 0, 0, 3984, 1878] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[922, 0, 2171, 0, 2870, 3971, 3555, 2153, 4050, 22, 28, 0, 0, 4079, 0, 0, 4060, 0, 0, 3996, 457, 0, 0, 0, 0, 12, 0, 0, 238, 31, 1, 152] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1822, 0, 0, 0, 2754, 3440, 588, 4082, 4096, 2494, 0, 1017, 0, 3651, 0, 3330, 0, 0, 37, 3765, 0, 302, 127, 106, 14, 0, 36, 0, 0, 0, 0, 1107] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[51, 1, 6, 69, 3846, 2, 4095, 3257, 75, 0, 18, 617, 3836, 1, 1, 4047, 170, 0, 845, 2358, 0, 101, 0, 315, 49, 0, 0, 4071, 1039, 3876, 0, 22] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[922, 0, 2171, 0, 856, 1051, 0, 0, 0, 2061, 3555, 2153, 0, 0, 0, 0, 4029, 22, 28, 0, 0, 0, 0, 0, 0, 1784, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 2361, 0, 0, 0, 0, 457, 0, 0, 0, 2235, 584, 0, 0, 0, 12, 0, 0, 2014, 0, 0, 0, 238, 31, 1, 152, 1910, 60, 21, 0] rerouted_tokens_per_ep_rank=[5000, 7769, 4079, 1784, 6421, 3276, 2026, 2413] avg=4096.0000 max=7769 max_rank=1 min=1784 min_rank=3 imbalance_ratio=1.896729 max_load_reduction=24999 max_load_reduction_pct=0.762909 moved_to_echo=14136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 2356, 7, 960, 1076, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 3181, 0, 295, 0, 0, 0, 0, 0, 3986, 1659, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2416, 0, 0, 0, 0, 4087, 0, 1517, 208, 2080, 597, 0, 0, 1109, 641, 0, 0, 2259, 0, 0, 0, 5, 143, 1, 4096, 1, 56, 16, 0] rerouted_tokens_per_ep_rank=[4405, 4, 3476, 5645, 2422, 8489, 4009, 4318] avg=4096.0000 max=8489 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.072510 max_load_reduction=24279 max_load_reduction_pct=0.740936 moved_to_echo=19238 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 588, 4013, 1028, 1026, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 3934, 0, 22, 0, 0, 0, 0, 0, 253, 834, 9, 4093, 0, 0, 0, 0, 3927, 100, 8, 2304, 0, 0, 0, 0, 1729, 0, 0, 11, 1044, 570, 0, 0, 133, 0, 0, 503, 2416, 0, 0, 0, 80, 0, 4, 1, 0, 28, 20, 0] rerouted_tokens_per_ep_rank=[6658, 4087, 3956, 5189, 6339, 3354, 3052, 133] avg=4096.0000 max=6658 max_rank=0 min=133 min_rank=7 imbalance_ratio=1.625488 max_load_reduction=26110 max_load_reduction_pct=0.796814 moved_to_echo=12878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 181, 1, 136, 1179, 1072, 0, 0, 0, 2000, 108, 412, 0, 0, 0, 0, 4028, 0, 0, 2, 0, 0, 0, 0, 1, 109, 20, 35, 0, 0, 0, 0, 1581, 0, 1, 2409, 0, 0, 0, 0, 0, 3, 116, 4096, 135, 595, 0, 0, 0, 36, 15, 3986, 2772, 0, 0, 0, 0, 0, 3984, 1878, 1853, 3, 21, 0] rerouted_tokens_per_ep_rank=[2569, 2520, 4030, 165, 3991, 4945, 6809, 7739] avg=4096.0000 max=7739 max_rank=7 min=165 min_rank=3 imbalance_ratio=1.889404 max_load_reduction=25029 max_load_reduction_pct=0.763824 moved_to_echo=23484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 13, 0, 1193, 1082, 0, 0, 0, 2114, 0, 6, 0, 0, 0, 0, 2831, 0, 0, 70, 0, 0, 0, 0, 0, 1661, 27, 0, 0, 0, 0, 0, 3966, 2, 86, 2410, 0, 0, 0, 0, 111, 0, 4096, 0, 2098, 603, 0, 0, 0, 0, 328, 2235, 2802, 0, 0, 0, 2, 15, 0, 2940, 1967, 61, 19, 0] rerouted_tokens_per_ep_rank=[2318, 2120, 2901, 1688, 6464, 6908, 5365, 5004] avg=4096.0000 max=6908 max_rank=5 min=1688 min_rank=3 imbalance_ratio=1.686523 max_load_reduction=25860 max_load_reduction_pct=0.789185 moved_to_echo=23741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[26, 0, 0, 4079, 1124, 1058, 0, 0, 0, 2126, 330, 2, 0, 0, 0, 0, 4075, 116, 0, 1839, 0, 0, 0, 0, 829, 760, 0, 3897, 0, 0, 0, 0, 84, 0, 0, 2376, 0, 0, 0, 0, 0, 0, 0, 412, 951, 587, 0, 0, 0, 4, 63, 0, 2642, 0, 0, 0, 0, 0, 17, 3355, 1970, 25, 21, 0] rerouted_tokens_per_ep_rank=[6287, 2458, 6030, 5486, 2460, 1950, 2709, 5388] avg=4096.0000 max=6287 max_rank=0 min=1950 min_rank=5 imbalance_ratio=1.534912 max_load_reduction=26481 max_load_reduction_pct=0.808136 moved_to_echo=12507 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1822, 0, 0, 0, 823, 990, 0, 0, 0, 1786, 588, 4082, 0, 0, 0, 0, 4075, 2494, 0, 1017, 0, 0, 0, 0, 0, 1596, 0, 3330, 0, 0, 0, 0, 0, 0, 37, 2225, 0, 0, 0, 0, 0, 302, 127, 106, 2001, 550, 0, 0, 14, 0, 36, 0, 1931, 0, 0, 0, 0, 0, 0, 1107, 1654, 54, 21, 0] rerouted_tokens_per_ep_rank=[3635, 6456, 7586, 4926, 2262, 3086, 1981, 2836] avg=4096.0000 max=7586 max_rank=2 min=1981 min_rank=6 imbalance_ratio=1.852051 max_load_reduction=25182 max_load_reduction_pct=0.768494 moved_to_echo=10165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[51, 1, 6, 69, 1149, 620, 0, 0, 0, 2, 4095, 3257, 0, 0, 0, 0, 75, 0, 18, 617, 0, 0, 0, 0, 3836, 1, 1, 4047, 0, 0, 0, 0, 170, 0, 845, 1394, 0, 0, 0, 0, 0, 101, 0, 315, 0, 344, 0, 0, 49, 0, 0, 4071, 2697, 0, 0, 0, 1039, 3876, 0, 22, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1896, 7354, 710, 7885, 2409, 760, 6817, 4937] avg=4096.0000 max=7885 max_rank=3 min=710 min_rank=2 imbalance_ratio=1.925049 max_load_reduction=24883 max_load_reduction_pct=0.759369 moved_to_echo=14923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2944, 256, 5248, 8320, 8320, 8064, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6400, 512, 5888, 5248, 10624, 4480, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8960, 8448, 128, 15488, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 10112, 8704, 14080, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[13824, 128, 1024, 17920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1408, 4096, 4096, 13568, 9472, 384, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1408, 768, 512, 10880, 19584, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[26240, 2688, 384, 3584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4064, 4074, 1, 234, 0, 0, 0, 0, 1, 539, 1, 24, 1, 3494, 1652, 0, 4086, 24, 0, 28, 3733, 295, 0, 0, 0, 4096, 0, 4096, 0, 2325, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 4096, 3, 0, 10, 1, 0, 0, 2, 0, 0, 4091, 14, 10, 473, 0, 0, 4081, 0, 1860, 363, 1127, 3750, 0, 4096, 0, 4096, 604] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[156, 3952, 90, 0, 4096, 0, 0, 0, 4, 0, 2, 217, 1, 1498, 4, 2021, 187, 2979, 0, 2394, 1640, 4096, 0, 0, 1331, 0, 3758, 0, 4087, 0, 255, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 3821, 4002, 115, 0, 11, 10, 2549, 0, 42, 0, 0, 4082, 484, 188, 531, 77, 329, 0, 133, 3264, 3806, 0, 0, 576, 0, 3996, 7, 3460, 289, 5, 991] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[2219, 4086, 217, 0, 0, 2034, 21, 14, 0, 4096, 3, 0, 0, 0, 0, 2084, 0, 3992, 1692, 0, 4093, 0, 2, 0, 51, 0, 4065, 3, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 3964, 3693, 8, 378, 2, 0, 4094, 0, 354, 1026, 0, 4086, 1115, 0, 1980, 434, 2967, 0, 709, 0, 4065, 2, 0, 232, 24, 1309, 4, 2322, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 12, 40, 0, 0, 4062, 27, 0, 1907, 0, 0, 0, 4096, 0, 0, 111, 0, 97, 0, 521, 4004, 5, 0, 12, 1878, 4091, 16, 3896, 1, 3899, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 2473, 3699, 78, 0, 0, 0, 0, 175, 29, 0, 36, 0, 153, 1875, 4020, 43, 4071, 0, 28, 4096, 4096, 0, 0, 0, 0, 3590, 0, 4086, 15, 202, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1912, 4074, 1, 0, 0, 0, 0, 234, 0, 0, 0, 1136, 371, 0, 0, 0, 1, 539, 1, 2152, 1090, 0, 0, 24, 1, 3494, 1177, 0, 0, 0, 0, 0, 4086, 24, 0, 475, 230, 215, 0, 28, 2272, 295, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 0, 0, 0, 2730, 0, 2325, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5987, 1741, 3783, 4696, 5030, 2595, 3881, 5055] avg=4096.0000 max=5987 max_rank=0 min=1741 min_rank=1 imbalance_ratio=1.461670 max_load_reduction=26781 max_load_reduction_pct=0.817291 moved_to_echo=16561 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1924, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 1136, 406, 0, 0, 10, 1, 0, 0, 2167, 1192, 0, 0, 2, 0, 0, 2914, 0, 0, 0, 0, 14, 10, 473, 0, 1177, 230, 197, 0, 0, 2483, 0, 1860, 0, 0, 0, 0, 363, 1127, 3553, 0, 0, 0, 0, 0, 2730, 0, 4096, 604, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1924, 5641, 3370, 2916, 2101, 4343, 5043, 7430] avg=4096.0000 max=7430 max_rank=7 min=1924 min_rank=0 imbalance_ratio=1.813965 max_load_reduction=25338 max_load_reduction_pct=0.773254 moved_to_echo=18917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1797, 4002, 115, 0, 0, 0, 0, 0, 11, 10, 2549, 960, 378, 0, 0, 0, 42, 0, 0, 2024, 1111, 0, 0, 4082, 484, 188, 379, 0, 0, 0, 0, 77, 329, 0, 133, 152, 194, 210, 0, 3264, 2317, 0, 0, 0, 0, 0, 0, 576, 0, 3786, 7, 0, 0, 0, 0, 2306, 289, 5, 991, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5914, 3908, 3177, 5133, 1095, 5581, 4369, 3591] avg=4096.0000 max=5914 max_rank=0 min=1095 min_rank=4 imbalance_ratio=1.443848 max_load_reduction=26854 max_load_reduction_pct=0.819519 moved_to_echo=14636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1925, 12, 40, 0, 0, 0, 0, 0, 0, 4062, 27, 1081, 398, 0, 0, 0, 1907, 0, 0, 2168, 1169, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 111, 0, 97, 0, 0, 218, 215, 0, 521, 2437, 5, 0, 0, 0, 0, 0, 12, 1878, 3876, 16, 0, 0, 0, 0, 2597, 1, 3899, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1977, 5568, 5244, 4096, 641, 2963, 5782, 6497] avg=4096.0000 max=6497 max_rank=7 min=641 min_rank=4 imbalance_ratio=1.586182 max_load_reduction=26271 max_load_reduction_pct=0.801727 moved_to_echo=15883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2219, 1919, 217, 0, 0, 0, 0, 0, 0, 2034, 21, 14, 1139, 0, 0, 0, 0, 4096, 3, 0, 2167, 0, 0, 0, 0, 0, 0, 1480, 0, 0, 0, 0, 0, 3992, 1692, 0, 604, 233, 218, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 51, 0, 3847, 3, 0, 0, 0, 0, 2724, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4355, 3208, 6266, 1480, 6739, 4095, 3901, 2724] avg=4096.0000 max=6739 max_rank=4 min=1480 min_rank=3 imbalance_ratio=1.645264 max_load_reduction=26029 max_load_reduction_pct=0.794342 moved_to_echo=17459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1163, 3699, 78, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 407, 0, 0, 175, 29, 0, 36, 1310, 1196, 0, 0, 0, 153, 1875, 2864, 0, 0, 0, 0, 43, 4071, 0, 28, 1156, 229, 189, 0, 4096, 2493, 0, 0, 0, 0, 0, 0, 0, 0, 3401, 0, 0, 0, 0, 0, 2724, 15, 202, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4940, 1540, 2746, 4892, 5716, 6589, 3401, 2944] avg=4096.0000 max=6589 max_rank=5 min=1540 min_rank=1 imbalance_ratio=1.608643 max_load_reduction=26179 max_load_reduction_pct=0.798920 moved_to_echo=18650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1865, 3693, 8, 0, 0, 0, 0, 378, 2, 0, 4094, 644, 404, 0, 0, 0, 354, 1026, 0, 2099, 1187, 0, 0, 4086, 1115, 0, 1411, 0, 0, 0, 0, 434, 2967, 0, 709, 569, 130, 69, 0, 0, 2474, 2, 0, 0, 0, 0, 0, 232, 24, 1240, 4, 0, 0, 0, 0, 1548, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5566, 5522, 4666, 6612, 4878, 2476, 1500, 1548] avg=4096.0000 max=6612 max_rank=3 min=1500 min_rank=6 imbalance_ratio=1.614258 max_load_reduction=26156 max_load_reduction_pct=0.798218 moved_to_echo=10402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[156, 1859, 90, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1134, 410, 0, 0, 4, 0, 2, 217, 2093, 1200, 0, 0, 1, 1498, 4, 1440, 0, 0, 0, 0, 187, 2979, 0, 2394, 581, 229, 198, 0, 1640, 2486, 0, 0, 0, 0, 0, 0, 1331, 0, 3560, 0, 0, 0, 0, 0, 2724, 0, 255, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2105, 5640, 3516, 2943, 6568, 4126, 4891, 2979] avg=4096.0000 max=6568 max_rank=4 min=2105 min_rank=0 imbalance_ratio=1.603516 max_load_reduction=26200 max_load_reduction_pct=0.799561 moved_to_echo=18564 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 6528, 1664, 256, 16256, 8192, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 18560, 2304, 3328, 4736, 1792, 1536, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4736, 6144, 4096, 6784, 8448, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2432, 14464, 15872, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[20096, 384, 10880, 1664, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13696, 17024, 384, 1920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 7424, 5632, 11776, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2688, 3072, 27264, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True + [2026-08-30 04:55:21] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 809.7 | throughput per GPU (TFLOP/s/GPU): 411.7 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101619E+01 | loss scale: 1.0 | grad norm: 45.750 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[37, 2965, 963, 138, 2318, 114, 1113, 429, 1065, 88, 1740, 622, 940, 1112, 157, 572, 221, 1068, 995, 1737, 1841, 467, 2909, 1131, 88, 1880, 1093, 1318, 826, 398, 966, 1457] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[50, 2807, 1106, 184, 2024, 116, 879, 631, 1320, 115, 2356, 701, 533, 1026, 173, 628, 274, 925, 1377, 2215, 1907, 246, 2661, 993, 24, 1269, 386, 1430, 985, 197, 1811, 1419] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[59, 3136, 1192, 182, 2016, 174, 851, 664, 1135, 134, 1378, 429, 1217, 752, 124, 467, 317, 645, 1574, 1949, 2213, 193, 2776, 1116, 77, 1169, 995, 1412, 1275, 464, 1326, 1357] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[60, 2980, 682, 182, 2546, 92, 644, 459, 1471, 82, 1736, 870, 734, 904, 233, 470, 488, 774, 2082, 2317, 1739, 195, 3163, 1091, 88, 1111, 410, 1114, 865, 329, 1620, 1237] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[186, 2862, 1180, 191, 1825, 60, 918, 494, 1290, 81, 2004, 868, 919, 765, 144, 452, 262, 632, 1505, 1894, 2231, 243, 2673, 1106, 96, 1339, 705, 1478, 978, 430, 1714, 1243] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[69, 3076, 1229, 208, 2302, 74, 888, 500, 1456, 134, 1806, 500, 579, 804, 125, 427, 229, 600, 1077, 2454, 1748, 456, 2539, 1348, 121, 1405, 598, 1152, 1232, 293, 1970, 1369] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[106, 3162, 849, 172, 1776, 48, 977, 545, 1375, 232, 1883, 557, 1224, 1055, 170, 528, 270, 543, 1689, 1477, 2209, 346, 2161, 1341, 111, 1248, 641, 1460, 1111, 470, 1802, 1230] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[168, 2993, 585, 114, 2646, 188, 984, 455, 1295, 71, 1945, 577, 823, 851, 208, 503, 225, 846, 1265, 2151, 2021, 376, 2981, 1337, 32, 1527, 611, 1061, 649, 270, 1653, 1357] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[50, 2678, 1106, 184, 0, 0, 0, 0, 2024, 116, 879, 631, 250, 167, 0, 0, 1320, 115, 2356, 701, 129, 48, 0, 0, 533, 1026, 173, 628, 1600, 0, 0, 0, 274, 925, 1377, 1763, 0, 0, 0, 0, 1907, 246, 594, 993, 0, 0, 0, 0, 24, 1269, 386, 1430, 467, 202, 0, 0, 985, 197, 1596, 1419, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4067, 4669, 3960, 4339, 3740, 3778, 4197] avg=4096.0000 max=4669 max_rank=2 min=3740 min_rank=5 imbalance_ratio=1.139893 max_load_reduction=28099 max_load_reduction_pct=0.857513 moved_to_echo=16054 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[37, 2829, 963, 138, 0, 0, 0, 0, 2318, 114, 1113, 429, 196, 89, 0, 0, 1065, 88, 1740, 622, 136, 26, 0, 0, 940, 1112, 157, 572, 1749, 0, 0, 0, 221, 1068, 995, 1383, 0, 0, 0, 0, 1841, 467, 650, 1131, 0, 0, 0, 0, 88, 1880, 1093, 1318, 510, 158, 0, 0, 826, 398, 851, 1457, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3967, 4259, 3677, 4530, 3667, 4089, 5047, 3532] avg=4096.0000 max=5047 max_rank=6 min=3532 min_rank=7 imbalance_ratio=1.232178 max_load_reduction=27721 max_load_reduction_pct=0.845978 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[60, 2843, 682, 182, 0, 0, 0, 0, 2546, 92, 644, 459, 261, 149, 0, 0, 1471, 82, 1736, 870, 137, 43, 0, 0, 734, 904, 233, 470, 1902, 0, 0, 0, 488, 774, 2082, 1845, 0, 0, 0, 0, 1739, 195, 706, 1091, 0, 0, 0, 0, 88, 1111, 410, 1114, 555, 211, 0, 0, 865, 329, 1428, 1237, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3767, 4151, 4339, 4243, 5189, 3731, 3489, 3859] avg=4096.0000 max=5189 max_rank=4 min=3489 min_rank=6 imbalance_ratio=1.266846 max_load_reduction=27579 max_load_reduction_pct=0.841644 moved_to_echo=16268 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[59, 2992, 1192, 182, 0, 0, 0, 0, 2016, 174, 851, 664, 220, 122, 0, 0, 1135, 134, 1378, 429, 144, 35, 0, 0, 1217, 752, 124, 467, 1669, 0, 0, 0, 317, 645, 1574, 1551, 0, 0, 0, 0, 2213, 193, 620, 1116, 0, 0, 0, 0, 77, 1169, 995, 1412, 487, 178, 0, 0, 1275, 464, 1169, 1357, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4425, 4047, 3255, 4229, 4087, 4142, 4318, 4265] avg=4096.0000 max=4425 max_rank=0 min=3255 min_rank=2 imbalance_ratio=1.080322 max_load_reduction=28343 max_load_reduction_pct=0.864960 moved_to_echo=16812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[69, 2935, 1229, 208, 0, 0, 0, 0, 2302, 74, 888, 500, 277, 181, 0, 0, 1456, 134, 1806, 500, 141, 53, 0, 0, 579, 804, 125, 427, 1527, 0, 0, 0, 229, 600, 1077, 1953, 0, 0, 0, 0, 1748, 456, 567, 1348, 0, 0, 0, 0, 121, 1405, 598, 1152, 445, 224, 0, 0, 1232, 293, 1736, 1369, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4441, 4222, 4090, 3462, 3859, 4119, 3945, 4630] avg=4096.0000 max=4630 max_rank=7 min=3462 min_rank=3 imbalance_ratio=1.130371 max_load_reduction=28138 max_load_reduction_pct=0.858704 moved_to_echo=16553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[186, 2726, 1180, 191, 0, 0, 0, 0, 1825, 60, 918, 494, 217, 162, 0, 0, 1290, 81, 2004, 868, 136, 50, 0, 0, 919, 765, 144, 452, 1611, 0, 0, 0, 262, 632, 1505, 1501, 0, 0, 0, 0, 2231, 243, 588, 1106, 0, 0, 0, 0, 96, 1339, 705, 1478, 474, 176, 0, 0, 978, 430, 1502, 1243, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4283, 3676, 4429, 3891, 3900, 4168, 4268, 4153] avg=4096.0000 max=4429 max_rank=2 min=3676 min_rank=1 imbalance_ratio=1.081299 max_load_reduction=28339 max_load_reduction_pct=0.864838 moved_to_echo=16489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[106, 3017, 849, 172, 0, 0, 0, 0, 1776, 48, 977, 545, 166, 166, 0, 0, 1375, 232, 1883, 557, 145, 48, 0, 0, 1224, 1055, 170, 528, 1299, 0, 0, 0, 270, 543, 1689, 1177, 0, 0, 0, 0, 2209, 346, 483, 1341, 0, 0, 0, 0, 111, 1248, 641, 1460, 379, 134, 0, 0, 1111, 470, 1588, 1230, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 3678, 4240, 4276, 3679, 4379, 3973, 4399] avg=4096.0000 max=4399 max_rank=7 min=3678 min_rank=1 imbalance_ratio=1.073975 max_load_reduction=28369 max_load_reduction_pct=0.865753 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[168, 2856, 585, 114, 0, 0, 0, 0, 2646, 188, 984, 455, 243, 152, 0, 0, 1295, 71, 1945, 577, 137, 44, 0, 0, 823, 851, 208, 503, 1792, 0, 0, 0, 225, 846, 1265, 1712, 0, 0, 0, 0, 2021, 376, 666, 1337, 0, 0, 0, 0, 32, 1527, 611, 1061, 523, 196, 0, 0, 649, 270, 1457, 1357, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3723, 4668, 4069, 4177, 4048, 4400, 3950, 3733] avg=4096.0000 max=4668 max_rank=1 min=3723 min_rank=0 imbalance_ratio=1.139648 max_load_reduction=28100 max_load_reduction_pct=0.857544 moved_to_echo=16131 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7936, 2944, 11392, 10752, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7040, 7296, 1408, 4096, 13184, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[768, 22912, 7808, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16000, 2560, 4992, 9472, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[640, 11008, 5504, 10496, 3840, 1536, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10496, 1024, 14848, 5248, 1152, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2304, 6144, 11648, 12928, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[17536, 896, 7296, 4224, 1920, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[46, 0, 3920, 1, 0, 0, 3836, 153, 54, 1, 0, 0, 0, 1, 100, 11, 135, 11, 6, 0, 3702, 0, 1974, 0, 3914, 2442, 4091, 475, 4089, 0, 178, 3628] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[68, 70, 3908, 76, 0, 0, 3099, 0, 7, 0, 0, 0, 0, 17, 0, 0, 647, 0, 0, 0, 2547, 5, 3726, 11, 4088, 481, 3896, 358, 3998, 0, 3375, 2391] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4, 412, 3377, 74, 0, 0, 2229, 266, 1, 0, 0, 1, 7, 0, 0, 0, 287, 0, 1, 1, 2401, 0, 1834, 48, 4033, 3675, 3771, 750, 4072, 0, 2463, 3061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[37, 8, 3898, 0, 15, 0, 2190, 0, 13, 173, 13, 0, 25, 38, 3, 0, 61, 0, 12, 28, 1843, 4, 1458, 4, 3616, 2554, 4078, 3474, 3882, 0, 1276, 4065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[419, 208, 3901, 4, 0, 0, 1191, 103, 0, 0, 0, 1, 0, 0, 2, 2, 33, 0, 0, 0, 4081, 0, 614, 142, 4089, 3586, 3954, 955, 4087, 0, 2545, 2851] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4, 352, 3810, 84, 1, 10, 2920, 69, 2, 10, 38, 1, 0, 314, 0, 14, 2600, 32, 2, 0, 3046, 14, 194, 11, 3927, 3897, 3747, 1260, 4087, 0, 320, 2002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[12, 423, 4044, 1, 1, 0, 2736, 90, 149, 38, 1, 384, 1, 25, 3, 0, 979, 4, 0, 6, 3967, 0, 3178, 6, 4096, 382, 4067, 1, 3686, 0, 1432, 3056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[18, 2, 4094, 2, 1, 0, 2378, 42, 5, 0, 0, 38, 0, 0, 0, 0, 1332, 3, 0, 0, 2506, 3, 3602, 70, 3992, 1441, 4085, 2317, 4078, 0, 531, 2228] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 412, 3322, 74, 0, 0, 0, 0, 0, 0, 2229, 266, 819, 346, 55, 8, 1, 0, 0, 1, 4004, 36, 0, 0, 7, 0, 0, 0, 4072, 29, 0, 0, 287, 0, 1, 1, 3163, 0, 0, 0, 1582, 0, 1834, 48, 0, 0, 0, 0, 0, 3675, 564, 750, 0, 0, 0, 0, 0, 0, 2463, 2715, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3812, 3723, 4042, 4108, 3452, 3464, 4989, 5178] avg=4096.0000 max=5178 max_rank=7 min=3452 min_rank=4 imbalance_ratio=1.264160 max_load_reduction=27590 max_load_reduction_pct=0.841980 moved_to_echo=17083 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[46, 0, 3856, 1, 0, 0, 0, 0, 0, 0, 3836, 153, 1263, 410, 64, 9, 54, 1, 0, 0, 3886, 39, 0, 0, 0, 1, 100, 11, 4089, 28, 0, 0, 135, 11, 6, 0, 3432, 0, 0, 0, 2439, 0, 1974, 0, 0, 0, 0, 0, 0, 2442, 611, 475, 0, 0, 0, 0, 0, 0, 178, 3218, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3903, 5735, 3980, 4229, 3584, 4413, 3528, 3396] avg=4096.0000 max=5735 max_rank=1 min=3396 min_rank=7 imbalance_ratio=1.400146 max_load_reduction=27033 max_load_reduction_pct=0.824982 moved_to_echo=14921 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[68, 70, 3844, 76, 0, 0, 0, 0, 0, 0, 3099, 0, 869, 270, 64, 8, 7, 0, 0, 0, 4059, 37, 0, 0, 0, 17, 0, 0, 3998, 29, 0, 0, 647, 0, 0, 0, 3268, 0, 0, 0, 1678, 5, 3726, 11, 0, 0, 0, 0, 0, 481, 583, 358, 0, 0, 0, 0, 0, 0, 3375, 2121, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4310, 4103, 4044, 3915, 5420, 1422, 5496] avg=4096.0000 max=5496 max_rank=7 min=1422 min_rank=6 imbalance_ratio=1.341797 max_load_reduction=27272 max_load_reduction_pct=0.832275 moved_to_echo=16253 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[419, 208, 3837, 4, 0, 0, 0, 0, 0, 0, 1191, 103, 1392, 322, 64, 8, 0, 0, 0, 1, 4059, 38, 0, 0, 0, 0, 2, 2, 4087, 30, 0, 0, 33, 0, 0, 0, 3317, 0, 0, 0, 2689, 0, 614, 142, 0, 0, 0, 0, 0, 3586, 591, 955, 0, 0, 0, 0, 0, 0, 2545, 2529, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4468, 3080, 4098, 4121, 3350, 3445, 5132, 5074] avg=4096.0000 max=5132 max_rank=6 min=3080 min_rank=1 imbalance_ratio=1.252930 max_load_reduction=27636 max_load_reduction_pct=0.843384 moved_to_echo=17001 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 352, 3748, 84, 0, 0, 0, 0, 1, 10, 2920, 69, 1039, 226, 62, 8, 2, 10, 38, 1, 3900, 36, 0, 0, 0, 314, 0, 14, 4087, 27, 0, 0, 2600, 32, 2, 0, 3143, 0, 0, 0, 2007, 14, 194, 11, 0, 0, 0, 0, 0, 3897, 560, 1260, 0, 0, 0, 0, 0, 0, 320, 1776, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4188, 4335, 3987, 4442, 5777, 2226, 5717, 2096] avg=4096.0000 max=5777 max_rank=4 min=2096 min_rank=7 imbalance_ratio=1.410400 max_load_reduction=26991 max_load_reduction_pct=0.823700 moved_to_echo=15816 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[12, 423, 3978, 1, 0, 0, 0, 0, 1, 0, 2736, 90, 1353, 345, 66, 9, 149, 38, 1, 384, 4066, 39, 0, 0, 1, 25, 3, 0, 3686, 30, 0, 0, 979, 4, 0, 6, 3412, 0, 0, 0, 2614, 0, 3178, 6, 0, 0, 0, 0, 0, 382, 607, 1, 0, 0, 0, 0, 0, 0, 1432, 2711, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4414, 4600, 4677, 3745, 4401, 5798, 990, 4143] avg=4096.0000 max=5798 max_rank=5 min=990 min_rank=6 imbalance_ratio=1.415527 max_load_reduction=26970 max_load_reduction_pct=0.823059 moved_to_echo=15332 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[37, 8, 3834, 0, 0, 0, 0, 0, 15, 0, 2190, 0, 629, 459, 64, 9, 13, 173, 13, 0, 3591, 39, 0, 0, 25, 38, 3, 0, 3882, 25, 0, 0, 61, 0, 12, 28, 3421, 0, 0, 0, 1214, 4, 1458, 4, 0, 0, 0, 0, 0, 2554, 609, 3474, 0, 0, 0, 0, 0, 0, 1276, 3606, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3879, 3366, 3829, 3973, 3522, 2680, 6637, 4882] avg=4096.0000 max=6637 max_rank=6 min=2680 min_rank=5 imbalance_ratio=1.620361 max_load_reduction=26131 max_load_reduction_pct=0.797455 moved_to_echo=17721 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[18, 2, 4024, 2, 0, 0, 0, 0, 1, 0, 2378, 42, 859, 256, 70, 13, 5, 0, 0, 38, 3964, 45, 0, 0, 0, 0, 0, 0, 4078, 28, 0, 0, 1332, 3, 0, 0, 3432, 0, 0, 0, 1647, 3, 3602, 70, 0, 0, 0, 0, 0, 1441, 595, 2317, 0, 0, 0, 0, 0, 0, 531, 1972, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 3619, 4052, 4106, 4767, 5322, 4353, 2503] avg=4096.0000 max=5322 max_rank=5 min=2503 min_rank=7 imbalance_ratio=1.299316 max_load_reduction=27446 max_load_reduction_pct=0.837585 moved_to_echo=16945 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 18560, 4736, 9600, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15872, 128, 16640, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 12160, 20736, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 20608, 768, 8320, 2688, 512, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6144, 128, 128, 128, 26624, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 128, 128, 32000, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[640, 1536, 30464, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 256, 128, 512, 31616, 384, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[671, 3624, 0, 239, 2, 1843, 0, 3904, 0, 4046, 0, 233, 761, 341, 4094, 0, 0, 0, 2, 4018, 65, 4092, 5, 7, 3935, 0, 696, 12, 52, 0, 0, 126] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[13, 4068, 0, 0, 1, 111, 0, 4094, 0, 4086, 0, 1, 28, 3992, 3874, 6, 0, 1, 0, 4096, 0, 4096, 0, 2, 4072, 1, 96, 0, 0, 0, 0, 130] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[320, 3780, 2, 2, 0, 17, 0, 4091, 0, 4080, 0, 10, 146, 1881, 4096, 0, 0, 0, 1, 3654, 106, 4096, 0, 1225, 4005, 0, 1175, 0, 8, 0, 50, 23] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 3671, 0, 0, 0, 475, 0, 4057, 0, 4096, 5, 3, 1, 3591, 3991, 0, 103, 66, 0, 4004, 11, 4091, 0, 32, 4071, 11, 319, 13, 0, 8, 149, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[521, 4096, 0, 0, 0, 2717, 0, 3855, 0, 4096, 0, 0, 3, 545, 3395, 0, 0, 0, 0, 3753, 275, 4094, 0, 8, 4096, 0, 1266, 0, 0, 0, 1, 47] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[127, 4052, 1, 0, 0, 1, 1, 3132, 0, 3712, 0, 1, 152, 107, 3543, 64, 0, 1, 47, 2373, 600, 3980, 3, 2765, 4090, 0, 3034, 0, 541, 420, 0, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1896, 4062, 0, 0, 21, 1935, 9, 4063, 0, 4087, 0, 0, 73, 123, 3966, 55, 0, 1, 93, 3155, 261, 4069, 0, 306, 3960, 0, 629, 2, 1, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[22, 3917, 0, 1, 0, 29, 0, 4092, 0, 4021, 0, 10, 359, 12, 4094, 3, 1, 1, 15, 3580, 0, 4093, 0, 21, 4096, 108, 3969, 28, 52, 1, 55, 188] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[671, 3357, 0, 239, 0, 0, 0, 0, 2, 1843, 0, 3196, 0, 0, 0, 0, 0, 4046, 0, 233, 32, 0, 0, 0, 761, 341, 2706, 0, 0, 0, 0, 0, 0, 0, 2, 4018, 235, 221, 0, 0, 65, 3399, 5, 7, 0, 0, 0, 0, 2607, 0, 696, 12, 0, 0, 0, 0, 52, 0, 0, 126, 1328, 1388, 708, 472] rerouted_tokens_per_ep_rank=[4267, 5041, 4311, 3808, 4476, 3476, 3315, 4074] avg=4096.0000 max=5041 max_rank=1 min=3315 min_rank=6 imbalance_ratio=1.230713 max_load_reduction=27727 max_load_reduction_pct=0.846161 moved_to_echo=15341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[13, 3768, 0, 0, 0, 0, 0, 0, 1, 111, 0, 3351, 0, 0, 0, 0, 0, 4086, 0, 1, 36, 0, 0, 0, 28, 3992, 2561, 6, 0, 0, 0, 0, 0, 1, 0, 4096, 264, 221, 0, 0, 0, 3402, 0, 2, 0, 0, 0, 0, 2697, 1, 96, 0, 0, 0, 0, 0, 0, 0, 0, 130, 1375, 1313, 743, 473] rerouted_tokens_per_ep_rank=[3781, 3463, 4123, 6587, 4582, 3404, 2794, 4034] avg=4096.0000 max=6587 max_rank=3 min=2794 min_rank=6 imbalance_ratio=1.608154 max_load_reduction=26181 max_load_reduction_pct=0.798981 moved_to_echo=14814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3400, 0, 0, 0, 0, 0, 0, 0, 475, 0, 3321, 0, 0, 0, 0, 0, 4096, 5, 3, 32, 0, 0, 0, 1, 3591, 2638, 0, 0, 0, 0, 0, 103, 66, 0, 4004, 239, 221, 0, 0, 11, 3398, 0, 32, 0, 0, 0, 0, 2697, 11, 319, 13, 0, 0, 0, 0, 0, 8, 149, 0, 1374, 1353, 736, 472] rerouted_tokens_per_ep_rank=[3400, 3796, 4136, 6230, 4633, 3441, 3040, 4092] avg=4096.0000 max=6230 max_rank=3 min=3040 min_rank=6 imbalance_ratio=1.520996 max_load_reduction=26538 max_load_reduction_pct=0.809875 moved_to_echo=15206 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 3794, 0, 0, 0, 0, 0, 0, 0, 2717, 0, 3155, 0, 0, 0, 0, 0, 4096, 0, 0, 36, 0, 0, 0, 3, 545, 2244, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 266, 221, 0, 0, 275, 3400, 0, 8, 0, 0, 0, 0, 2713, 0, 1266, 0, 0, 0, 0, 0, 0, 0, 1, 47, 1383, 1151, 700, 473] rerouted_tokens_per_ep_rank=[4315, 5872, 4132, 2792, 4240, 3683, 3979, 3755] avg=4096.0000 max=5872 max_rank=1 min=2792 min_rank=3 imbalance_ratio=1.433594 max_load_reduction=26896 max_load_reduction_pct=0.820801 moved_to_echo=15657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[320, 3501, 2, 2, 0, 0, 0, 0, 0, 17, 0, 3349, 0, 0, 0, 0, 0, 4080, 0, 10, 33, 0, 0, 0, 146, 1881, 2708, 0, 0, 0, 0, 0, 0, 0, 1, 3654, 246, 221, 0, 0, 106, 3402, 0, 1225, 0, 0, 0, 0, 2653, 0, 1175, 0, 0, 0, 0, 0, 8, 0, 50, 23, 1352, 1388, 742, 473] rerouted_tokens_per_ep_rank=[3825, 3366, 4123, 4735, 4122, 4733, 3828, 4036] avg=4096.0000 max=4735 max_rank=3 min=3366 min_rank=1 imbalance_ratio=1.156006 max_load_reduction=28033 max_load_reduction_pct=0.855499 moved_to_echo=16719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1896, 3762, 0, 0, 0, 0, 0, 0, 21, 1935, 9, 3326, 0, 0, 0, 0, 0, 4087, 0, 0, 36, 0, 0, 0, 73, 123, 2622, 55, 0, 0, 0, 0, 0, 1, 93, 3155, 264, 220, 0, 0, 261, 3379, 0, 306, 0, 0, 0, 0, 2623, 0, 629, 2, 0, 0, 0, 0, 1, 0, 0, 1, 1337, 1344, 737, 470] rerouted_tokens_per_ep_rank=[5658, 5291, 4123, 2873, 3733, 3946, 3254, 3890] avg=4096.0000 max=5658 max_rank=0 min=2873 min_rank=3 imbalance_ratio=1.381348 max_load_reduction=27110 max_load_reduction_pct=0.827332 moved_to_echo=14823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[22, 3618, 0, 1, 0, 0, 0, 0, 0, 29, 0, 3344, 0, 0, 0, 0, 0, 4021, 0, 10, 40, 0, 0, 0, 359, 12, 2704, 3, 0, 0, 0, 0, 1, 1, 15, 3580, 259, 227, 0, 0, 0, 3390, 0, 21, 0, 0, 0, 0, 2710, 108, 3969, 28, 0, 0, 0, 0, 52, 1, 55, 188, 1386, 1390, 748, 476] rerouted_tokens_per_ep_rank=[3641, 3373, 4071, 3078, 4083, 3411, 6815, 4296] avg=4096.0000 max=6815 max_rank=6 min=3078 min_rank=3 imbalance_ratio=1.663818 max_load_reduction=25953 max_load_reduction_pct=0.792023 moved_to_echo=18605 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[127, 3753, 1, 0, 0, 0, 0, 0, 0, 1, 1, 2564, 0, 0, 0, 0, 0, 3712, 0, 1, 36, 0, 0, 0, 152, 107, 2342, 64, 0, 0, 0, 0, 0, 1, 47, 2373, 263, 215, 0, 0, 600, 3306, 3, 2765, 0, 0, 0, 0, 2709, 0, 3034, 0, 0, 0, 0, 0, 541, 420, 0, 21, 1381, 1201, 568, 459] rerouted_tokens_per_ep_rank=[3881, 2566, 3749, 2665, 2899, 6674, 5743, 4591] avg=4096.0000 max=6674 max_rank=5 min=2566 min_rank=1 imbalance_ratio=1.629395 max_load_reduction=26094 max_load_reduction_pct=0.796326 moved_to_echo=19907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[768, 512, 256, 640, 11008, 10624, 5760, 3840] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21504, 128, 11264, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[1536, 10624, 20608, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7168, 128, 25728, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1408, 27136, 128, 4480, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3584, 29056, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 256, 28672, 2048, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32256, 128, 384, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[10, 689, 0, 3958, 0, 0, 541, 2250, 1230, 51, 0, 4078, 1389, 0, 1497, 324, 1, 0, 332, 29, 1383, 42, 1, 0, 3333, 0, 0, 2, 4096, 3789, 0, 3743] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[116, 21, 0, 4096, 7, 0, 1158, 3919, 33, 865, 0, 4096, 3981, 0, 2068, 0, 13, 0, 56, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 4073, 4048, 183, 57] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[112, 346, 0, 4096, 140, 0, 3597, 1515, 90, 900, 0, 3494, 3657, 164, 3131, 2, 0, 38, 0, 1, 0, 0, 0, 0, 3473, 29, 0, 2, 4063, 3913, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[37, 3725, 0, 4096, 0, 2, 242, 2737, 22, 57, 0, 4089, 2701, 26, 3115, 0, 0, 0, 0, 0, 1, 1, 0, 3, 3561, 11, 0, 2, 4069, 4011, 157, 103] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[447, 2, 0, 4083, 1, 0, 0, 2350, 37, 121, 0, 4094, 3775, 0, 3388, 1, 0, 0, 1, 20, 0, 0, 0, 107, 4069, 0, 8, 0, 4049, 3636, 44, 2535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 3563, 0, 4096, 0, 0, 463, 566, 198, 0, 0, 4096, 2657, 0, 969, 0, 377, 0, 29, 7, 0, 1, 0, 6, 4096, 0, 0, 0, 4030, 3683, 120, 3811] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[65, 4023, 6, 4071, 0, 17, 4, 3902, 1083, 621, 73, 4009, 4075, 3, 63, 0, 7, 2, 0, 18, 0, 23, 0, 11, 2225, 3, 0, 46, 4091, 3915, 0, 412] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 992, 0, 4095, 17, 8, 2103, 655, 1204, 2178, 0, 4085, 3778, 24, 187, 0, 7, 0, 219, 0, 0, 26, 1, 1, 3689, 0, 0, 3, 4007, 4067, 64, 1358] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[10, 689, 0, 2261, 0, 0, 0, 0, 0, 0, 541, 2250, 483, 148, 0, 0, 1230, 51, 0, 3056, 0, 0, 0, 0, 952, 0, 1497, 324, 0, 0, 0, 0, 1, 0, 332, 29, 3986, 0, 0, 0, 1383, 42, 1, 0, 1697, 1327, 289, 110, 3333, 0, 0, 2, 539, 0, 0, 0, 0, 2462, 0, 3743, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2960, 3422, 4337, 2773, 4348, 4849, 3874, 6205] avg=4096.0000 max=6205 max_rank=7 min=2773 min_rank=3 imbalance_ratio=1.514893 max_load_reduction=26563 max_load_reduction_pct=0.810638 moved_to_echo=19276 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[37, 3725, 0, 2340, 0, 0, 0, 0, 0, 2, 242, 2737, 484, 288, 0, 0, 22, 57, 0, 3065, 0, 0, 0, 0, 1850, 26, 3115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 0, 1, 1, 0, 3, 1756, 1405, 563, 109, 3561, 11, 0, 2, 540, 0, 0, 0, 0, 2606, 157, 103, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6102, 3753, 3144, 4991, 3960, 3838, 4114, 2866] avg=4096.0000 max=6102 max_rank=0 min=2866 min_rank=7 imbalance_ratio=1.489746 max_load_reduction=26666 max_load_reduction_pct=0.813782 moved_to_echo=14778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[112, 346, 0, 2340, 0, 0, 0, 0, 140, 0, 3597, 1515, 414, 390, 0, 0, 90, 900, 0, 2618, 0, 0, 0, 0, 2504, 164, 3131, 2, 0, 0, 0, 0, 0, 38, 0, 1, 3955, 0, 0, 0, 0, 0, 0, 0, 1756, 1371, 763, 108, 3473, 29, 0, 2, 462, 0, 0, 0, 0, 2542, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2798, 6056, 3608, 5801, 3994, 3998, 3966, 2547] avg=4096.0000 max=6056 max_rank=1 min=2547 min_rank=7 imbalance_ratio=1.478516 max_load_reduction=26712 max_load_reduction_pct=0.815186 moved_to_echo=14505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[116, 21, 0, 2340, 0, 0, 0, 0, 7, 0, 1158, 3919, 485, 424, 0, 0, 33, 865, 0, 3070, 0, 0, 0, 0, 2726, 0, 2068, 0, 0, 0, 0, 0, 13, 0, 56, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 1756, 1418, 831, 108, 3978, 0, 0, 0, 541, 0, 0, 0, 0, 2630, 183, 57, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2477, 5993, 3968, 4794, 4034, 4113, 4519, 2870] avg=4096.0000 max=5993 max_rank=1 min=2477 min_rank=0 imbalance_ratio=1.463135 max_load_reduction=26775 max_load_reduction_pct=0.817108 moved_to_echo=15536 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[447, 2, 0, 2332, 0, 0, 0, 0, 1, 0, 0, 2350, 485, 402, 0, 0, 37, 121, 0, 3068, 0, 0, 0, 0, 2585, 0, 3388, 1, 0, 0, 0, 0, 0, 0, 1, 20, 3940, 0, 0, 0, 0, 0, 0, 107, 1751, 1274, 788, 109, 4069, 0, 8, 0, 541, 0, 0, 0, 0, 2362, 44, 2535, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2781, 3238, 3226, 5974, 3961, 4029, 4618, 4941] avg=4096.0000 max=5974 max_rank=3 min=2781 min_rank=0 imbalance_ratio=1.458496 max_load_reduction=26794 max_load_reduction_pct=0.817688 moved_to_echo=17549 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3563, 0, 2340, 0, 0, 0, 0, 0, 0, 463, 566, 485, 283, 0, 0, 198, 0, 0, 3070, 0, 0, 0, 0, 1820, 0, 969, 0, 0, 0, 0, 0, 377, 0, 29, 7, 3922, 0, 0, 0, 0, 1, 0, 6, 1756, 1290, 554, 108, 4096, 0, 0, 0, 541, 0, 0, 0, 0, 2393, 120, 3811, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5903, 1797, 3268, 2789, 4335, 3715, 4637, 6324] avg=4096.0000 max=6324 max_rank=7 min=1797 min_rank=1 imbalance_ratio=1.543945 max_load_reduction=26444 max_load_reduction_pct=0.807007 moved_to_echo=19011 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 992, 0, 2335, 0, 0, 0, 0, 17, 8, 2103, 655, 487, 407, 0, 0, 1204, 2178, 0, 3054, 0, 0, 0, 0, 2578, 24, 187, 0, 0, 0, 0, 0, 7, 0, 219, 0, 3901, 0, 0, 0, 0, 26, 1, 1, 1760, 1429, 793, 106, 3689, 0, 0, 3, 544, 0, 0, 0, 0, 2638, 64, 1358, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3327, 3677, 6436, 2789, 4127, 4116, 4236, 4060] avg=4096.0000 max=6436 max_rank=2 min=2789 min_rank=3 imbalance_ratio=1.571289 max_load_reduction=26332 max_load_reduction_pct=0.803589 moved_to_echo=16539 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[65, 4023, 6, 2326, 0, 0, 0, 0, 0, 17, 4, 3902, 475, 434, 0, 0, 1083, 621, 73, 3004, 0, 0, 0, 0, 2791, 3, 63, 0, 0, 0, 0, 0, 7, 2, 0, 18, 3982, 0, 0, 0, 0, 23, 0, 11, 1745, 1372, 850, 109, 2225, 3, 0, 46, 530, 0, 0, 0, 0, 2543, 0, 412, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6420, 4832, 4781, 2857, 4009, 4110, 2804, 2955] avg=4096.0000 max=6420 max_rank=0 min=2804 min_rank=6 imbalance_ratio=1.567383 max_load_reduction=26348 max_load_reduction_pct=0.804077 moved_to_echo=13878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1408, 128, 128, 128, 14080, 11008, 5504, 896] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 13440, 128, 18688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 8192, 17920, 3840, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 20224, 640, 12032, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[17920, 256, 14464, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3968, 4864, 128, 24064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[28544, 128, 128, 128, 4352, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 128, 640, 128, 31616, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 0, 2569, 0, 81, 0, 50, 0, 11, 0, 3061, 4087, 4016, 4096, 3, 0, 4, 0, 0, 2219, 0, 3438, 0, 940, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 6, 205, 0, 12, 20, 1, 28, 101, 0, 4053, 4094, 4058, 4096, 0, 0, 3673, 12, 0, 4048, 0, 506, 0, 8, 3766, 0, 0, 0, 4081, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 389, 0, 31, 0, 0, 2, 1, 0, 747, 4072, 4095, 4068, 0, 1, 200, 0, 1, 3613, 29, 3039, 92, 2766, 4096, 0, 0, 0, 4095, 1431, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 92, 3, 0, 0, 0, 85, 2, 26, 3998, 3990, 4064, 4000, 90, 0, 314, 1, 0, 3847, 5, 3703, 28, 422, 4015, 0, 4, 1, 4077, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 138, 0, 0, 3, 1, 0, 2359, 0, 1784, 4038, 4096, 4096, 0, 0, 3093, 0, 0, 812, 0, 490, 0, 3666, 4094, 0, 0, 0, 4096, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 3803, 1, 9, 0, 193, 0, 0, 0, 23, 4073, 4095, 4095, 11, 15, 2829, 16, 1, 2350, 0, 3105, 0, 9, 4036, 3, 0, 4, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1705, 3, 92, 0, 0, 7, 37, 0, 3841, 4005, 4096, 4039, 0, 9, 5, 756, 0, 2871, 0, 3141, 0, 61, 4082, 0, 1, 0, 4016, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 48, 0, 269, 6, 308, 128, 2, 0, 0, 0, 4, 3977, 4016, 4096, 2, 82, 2244, 50, 0, 1532, 0, 4077, 25, 3725, 4043, 38, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 92, 2845, 94, 0, 0, 3, 0, 0, 0, 3906, 0, 0, 0, 85, 2, 26, 3998, 1757, 165, 133, 0, 3990, 41, 0, 90, 0, 0, 0, 0, 0, 314, 1, 0, 1249, 1178, 0, 0, 3847, 5, 1946, 28, 0, 0, 0, 0, 422, 2633, 0, 4, 0, 0, 0, 0, 1, 3912, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3031, 3909, 6166, 4121, 2742, 5826, 3059, 3914] avg=4096.0000 max=6166 max_rank=2 min=2742 min_rank=4 imbalance_ratio=1.505371 max_load_reduction=26602 max_load_reduction_pct=0.811829 moved_to_echo=15541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 389, 2866, 95, 0, 0, 0, 31, 0, 0, 3973, 0, 0, 0, 2, 1, 0, 747, 1442, 165, 136, 0, 4072, 42, 0, 0, 0, 0, 0, 0, 1, 200, 0, 1, 1274, 1187, 0, 0, 3613, 29, 1597, 92, 0, 0, 0, 0, 2766, 2686, 0, 0, 0, 0, 0, 0, 0, 3930, 1431, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3350, 4004, 2493, 4114, 2663, 5331, 5452, 5361] avg=4096.0000 max=5452 max_rank=6 min=2493 min_rank=2 imbalance_ratio=1.331055 max_load_reduction=27316 max_load_reduction_pct=0.833618 moved_to_echo=18807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 205, 2840, 96, 0, 0, 0, 12, 20, 1, 4000, 0, 0, 0, 28, 101, 0, 4053, 240, 165, 125, 0, 4094, 42, 0, 0, 0, 0, 0, 0, 0, 3673, 12, 0, 1172, 1176, 0, 0, 4048, 0, 266, 0, 0, 0, 0, 0, 8, 2469, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3147, 4033, 4712, 4136, 6033, 4314, 2477, 3916] avg=4096.0000 max=6033 max_rank=4 min=2477 min_rank=6 imbalance_ratio=1.472900 max_load_reduction=26735 max_load_reduction_pct=0.815887 moved_to_echo=16740 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 2569, 2811, 96, 0, 0, 0, 81, 0, 50, 4000, 0, 0, 0, 0, 11, 0, 3061, 1631, 165, 136, 0, 4087, 41, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 1274, 1164, 0, 0, 2219, 0, 1807, 0, 0, 0, 0, 0, 940, 2686, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5477, 4131, 5004, 4131, 2442, 4026, 3626, 3931] avg=4096.0000 max=5477 max_rank=0 min=2442 min_rank=4 imbalance_ratio=1.337158 max_load_reduction=27291 max_load_reduction_pct=0.832855 moved_to_echo=14025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 138, 2867, 95, 0, 0, 0, 0, 3, 1, 4001, 0, 0, 0, 0, 2359, 0, 1784, 232, 165, 136, 0, 4038, 42, 0, 0, 0, 0, 0, 0, 0, 3093, 0, 0, 1274, 1187, 0, 0, 812, 0, 258, 0, 0, 0, 0, 0, 3666, 2684, 0, 0, 0, 0, 0, 0, 0, 3931, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3101, 4005, 4676, 4080, 5554, 1070, 6350, 3932] avg=4096.0000 max=6350 max_rank=6 min=1070 min_rank=5 imbalance_ratio=1.550293 max_load_reduction=26418 max_load_reduction_pct=0.806213 moved_to_echo=16906 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1705, 2867, 94, 0, 0, 3, 92, 0, 0, 3945, 0, 0, 0, 7, 37, 0, 3841, 1490, 162, 135, 0, 4005, 42, 0, 0, 0, 0, 0, 0, 9, 5, 756, 0, 1270, 1187, 0, 0, 2871, 0, 1651, 0, 0, 0, 0, 0, 61, 2677, 0, 1, 0, 0, 0, 0, 0, 3854, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4666, 4040, 5672, 4047, 3227, 4522, 2739, 3855] avg=4096.0000 max=5672 max_rank=2 min=2739 min_rank=6 imbalance_ratio=1.384766 max_load_reduction=27096 max_load_reduction_pct=0.826904 moved_to_echo=14343 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 3803, 2871, 96, 0, 0, 1, 9, 0, 193, 3999, 0, 0, 0, 0, 0, 0, 23, 1476, 171, 138, 0, 4073, 35, 0, 11, 0, 0, 0, 0, 15, 2829, 16, 1, 1259, 1189, 0, 0, 2350, 0, 1629, 0, 0, 0, 0, 0, 9, 2639, 3, 0, 0, 0, 0, 0, 4, 3924, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6772, 4202, 1808, 4119, 5309, 3979, 2651, 3928] avg=4096.0000 max=6772 max_rank=0 min=1808 min_rank=2 imbalance_ratio=1.653320 max_load_reduction=25996 max_load_reduction_pct=0.793335 moved_to_echo=15867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 48, 0, 269, 2811, 96, 0, 0, 6, 308, 128, 2, 4000, 0, 0, 0, 0, 0, 0, 4, 1934, 165, 134, 0, 3977, 41, 0, 2, 0, 0, 0, 0, 82, 2244, 50, 0, 1258, 1164, 0, 0, 1532, 0, 2143, 25, 0, 0, 0, 0, 3725, 2651, 38, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3224, 4444, 2237, 4020, 4798, 3700, 6414, 3931] avg=4096.0000 max=6414 max_rank=6 min=2237 min_rank=2 imbalance_ratio=1.565918 max_load_reduction=26354 max_load_reduction_pct=0.804260 moved_to_echo=18843 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 12416, 896, 128, 10112, 9472, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11648, 21248, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[21376, 128, 11392, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 2560, 128, 17536, 10240, 1408, 1152, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 9216, 22784, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 31360, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32384, 384, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 640, 256, 256, 31872, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 190, 0, 499, 0, 600, 0, 3853, 7, 0, 136, 0, 0, 72, 1, 1567, 2474, 3082, 0, 0, 98, 16, 2, 4080, 3791, 24, 8, 4096, 0, 4076, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1976, 0, 3125, 0, 10, 0, 0, 165, 4052, 0, 0, 0, 11, 51, 547, 19, 0, 128, 4041, 3, 0, 362, 13, 2184, 3889, 4048, 0, 0, 4055, 0, 4089, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[2946, 0, 5, 0, 0, 0, 1, 57, 4086, 7, 1, 48, 1, 0, 0, 146, 9, 3889, 3998, 0, 4, 0, 0, 4027, 4096, 3947, 0, 118, 4065, 2, 1315, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[2241, 0, 974, 3, 0, 0, 23, 19, 4096, 0, 0, 0, 0, 0, 449, 0, 144, 1936, 4094, 0, 0, 0, 0, 3782, 4015, 2836, 0, 0, 4096, 0, 4060, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[2866, 0, 347, 1419, 0, 0, 601, 0, 4090, 6, 5, 1, 0, 393, 981, 8, 344, 4090, 4094, 0, 2, 0, 0, 976, 4086, 316, 6, 0, 4095, 0, 4042, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[3664, 0, 2647, 272, 0, 0, 0, 1, 4096, 0, 0, 1, 0, 0, 1, 0, 9, 1539, 4095, 0, 0, 0, 0, 4007, 4096, 156, 0, 0, 4096, 0, 4088, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[274, 0, 1302, 0, 0, 2, 0, 376, 4021, 0, 3, 910, 1, 0, 1, 0, 28, 4035, 3098, 0, 33, 2, 0, 4065, 4048, 1898, 0, 630, 4090, 4, 3928, 19] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 7, 80, 53, 0, 2, 312, 2, 4095, 0, 12, 0, 0, 0, 0, 4, 3435, 938, 4084, 1, 0, 0, 38, 3757, 4095, 3711, 0, 0, 4090, 0, 4044, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2946, 0, 5, 0, 445, 95, 0, 0, 0, 0, 1, 57, 3680, 66, 0, 0, 3991, 7, 1, 48, 0, 0, 0, 0, 1, 0, 0, 146, 2859, 1036, 0, 0, 9, 3889, 1073, 0, 0, 0, 0, 0, 4, 0, 0, 4027, 1188, 0, 0, 0, 1427, 3947, 0, 118, 0, 0, 0, 0, 385, 2, 1315, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3491, 3804, 4047, 4042, 4971, 5219, 5492, 1702] avg=4096.0000 max=5492 max_rank=6 min=1702 min_rank=7 imbalance_ratio=1.340820 max_load_reduction=27276 max_load_reduction_pct=0.832397 moved_to_echo=17384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 190, 0, 443, 90, 0, 0, 499, 0, 600, 0, 3708, 51, 0, 0, 3763, 7, 0, 136, 0, 0, 0, 0, 0, 0, 72, 1, 2204, 1032, 0, 0, 1567, 2474, 827, 0, 0, 0, 0, 0, 0, 98, 16, 2, 1183, 0, 0, 0, 1422, 3791, 24, 8, 0, 0, 0, 0, 388, 0, 4076, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4819, 4858, 3906, 3309, 4868, 1299, 5245, 4464] avg=4096.0000 max=5245 max_rank=6 min=1299 min_rank=5 imbalance_ratio=1.280518 max_load_reduction=27523 max_load_reduction_pct=0.839935 moved_to_echo=15876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1976, 0, 3125, 0, 422, 94, 0, 0, 10, 0, 0, 165, 3671, 67, 0, 0, 3958, 0, 0, 0, 0, 0, 0, 0, 11, 51, 547, 19, 2890, 984, 0, 0, 0, 128, 1084, 3, 0, 0, 0, 0, 0, 362, 13, 2184, 1128, 0, 0, 0, 1355, 4048, 0, 0, 0, 0, 0, 0, 384, 0, 4089, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5617, 3913, 3958, 4502, 1215, 3687, 5403, 4473] avg=4096.0000 max=5617 max_rank=0 min=1215 min_rank=4 imbalance_ratio=1.371338 max_load_reduction=27151 max_load_reduction_pct=0.828583 moved_to_echo=14778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2866, 0, 347, 1419, 444, 95, 0, 0, 0, 0, 601, 0, 3707, 67, 0, 0, 3995, 6, 5, 1, 0, 0, 0, 0, 0, 393, 981, 8, 2928, 1034, 0, 0, 344, 4090, 1099, 0, 0, 0, 0, 0, 2, 0, 0, 976, 1185, 0, 0, 0, 1423, 316, 6, 0, 0, 0, 0, 0, 388, 0, 4042, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5171, 4375, 4007, 5344, 5533, 2163, 1745, 4430] avg=4096.0000 max=5533 max_rank=4 min=1745 min_rank=6 imbalance_ratio=1.350830 max_load_reduction=27235 max_load_reduction_pct=0.831146 moved_to_echo=13871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2241, 0, 974, 3, 436, 95, 0, 0, 0, 0, 23, 19, 3708, 67, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 2928, 1016, 0, 0, 144, 1936, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 3782, 1164, 0, 0, 0, 1399, 2836, 0, 0, 0, 0, 0, 0, 388, 0, 4060, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3749, 3817, 4001, 4393, 3179, 4946, 4235, 4448] avg=4096.0000 max=4946 max_rank=5 min=3179 min_rank=4 imbalance_ratio=1.207520 max_load_reduction=27822 max_load_reduction_pct=0.849060 moved_to_echo=16808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3664, 0, 2647, 272, 445, 95, 0, 0, 0, 0, 0, 1, 3708, 68, 0, 0, 4001, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 2929, 1036, 0, 0, 9, 1539, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 1188, 0, 0, 0, 1427, 156, 0, 0, 0, 0, 0, 0, 388, 0, 4088, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7123, 3777, 4002, 3966, 2646, 5195, 1583, 4476] avg=4096.0000 max=7123 max_rank=0 min=1583 min_rank=6 imbalance_ratio=1.739014 max_load_reduction=25645 max_load_reduction_pct=0.782623 moved_to_echo=13900 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 80, 53, 448, 100, 0, 0, 0, 2, 312, 2, 3706, 71, 0, 0, 3995, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2925, 1041, 0, 0, 3435, 938, 1088, 1, 0, 0, 0, 0, 0, 0, 38, 3757, 1190, 0, 0, 0, 1416, 3711, 0, 0, 0, 0, 0, 0, 384, 0, 4044, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[688, 4093, 4007, 3970, 5462, 4985, 5127, 4436] avg=4096.0000 max=5462 max_rank=4 min=688 min_rank=0 imbalance_ratio=1.333496 max_load_reduction=27306 max_load_reduction_pct=0.833313 moved_to_echo=20010 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[274, 0, 1302, 0, 440, 94, 0, 0, 0, 2, 0, 376, 3702, 51, 0, 0, 3927, 0, 3, 910, 0, 0, 0, 0, 1, 0, 1, 0, 2216, 1024, 0, 0, 28, 4035, 831, 0, 0, 0, 0, 0, 33, 2, 0, 4065, 1174, 0, 0, 0, 1410, 1898, 0, 630, 0, 0, 0, 0, 388, 4, 3928, 19, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2110, 4131, 4840, 3242, 4894, 5274, 3938, 4339] avg=4096.0000 max=5274 max_rank=5 min=2110 min_rank=0 imbalance_ratio=1.287598 max_load_reduction=27494 max_load_reduction_pct=0.839050 moved_to_echo=18445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 512, 2176, 256, 21888, 8320, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11392, 20736, 128, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[18176, 128, 8704, 1792, 3584, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5632, 19072, 8320, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 128, 22912, 9472, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3200, 128, 29696, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[31744, 128, 128, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 128, 1664, 640, 29696, 512, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[588, 30, 1, 1672, 15, 4049, 0, 33, 75, 0, 0, 30, 44, 3994, 2953, 0, 0, 4028, 0, 3905, 0, 2620, 6, 2, 4023, 184, 0, 415, 0, 2, 4086, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1952, 0, 3739, 0, 0, 4006, 0, 13, 0, 92, 12, 782, 11, 1720, 9, 0, 0, 4091, 0, 4079, 7, 4088, 0, 0, 4079, 0, 0, 3, 11, 0, 4074, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1342, 0, 4096, 0, 0, 0, 0, 0, 2174, 1, 4078, 4043, 0, 0, 465, 0, 4096, 5, 148, 0, 0, 4096, 0, 0, 0, 0, 35, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[310, 0, 1855, 1913, 0, 4031, 0, 0, 0, 0, 0, 382, 0, 4091, 3739, 3, 84, 4087, 0, 4094, 50, 157, 0, 0, 4095, 0, 2, 5, 0, 0, 3868, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[391, 0, 585, 619, 0, 4096, 8, 0, 4, 0, 0, 28, 1, 4091, 3298, 0, 0, 3634, 0, 4096, 614, 3067, 0, 0, 4095, 16, 0, 0, 0, 34, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[168, 0, 1906, 1047, 0, 4091, 1210, 0, 0, 0, 74, 5, 0, 3980, 4065, 0, 28, 3729, 0, 4096, 20, 210, 0, 69, 4026, 1, 0, 70, 0, 2, 3959, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4074, 743, 22, 3889, 0, 0, 1, 222, 0, 2, 9, 4072, 4096, 0, 3, 3343, 0, 3948, 0, 127, 0, 0, 4096, 0, 0, 5, 95, 0, 4021, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 14, 1815, 92, 5, 4082, 65, 0, 3, 0, 0, 1317, 0, 4094, 4064, 8, 0, 738, 4, 4083, 0, 4079, 0, 0, 4005, 0, 0, 0, 0, 116, 4078, 106] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[588, 30, 1, 1672, 330, 118, 46, 0, 15, 3931, 0, 33, 0, 0, 0, 0, 75, 0, 0, 30, 2876, 490, 0, 0, 44, 854, 2953, 0, 0, 0, 0, 0, 0, 4028, 0, 1029, 0, 0, 0, 0, 0, 2620, 6, 2, 2320, 0, 0, 0, 3969, 184, 0, 415, 0, 0, 0, 0, 0, 2, 4086, 13, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[2785, 3979, 3471, 3851, 5057, 4948, 4568, 4109] avg=4096.0000 max=5057 max_rank=4 min=2785 min_rank=0 imbalance_ratio=1.234619 max_load_reduction=27711 max_load_reduction_pct=0.845673 moved_to_echo=18682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[310, 0, 1855, 1913, 338, 117, 47, 0, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 3015, 502, 0, 0, 0, 875, 3739, 3, 0, 0, 0, 0, 84, 4087, 0, 1079, 0, 0, 0, 0, 50, 157, 0, 0, 2376, 0, 0, 0, 4040, 0, 2, 5, 0, 0, 0, 0, 0, 0, 3868, 2, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4580, 3914, 3899, 4617, 5250, 2583, 4047, 3878] avg=4096.0000 max=5250 max_rank=4 min=2583 min_rank=5 imbalance_ratio=1.281738 max_load_reduction=27518 max_load_reduction_pct=0.839783 moved_to_echo=15758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4074, 743, 337, 113, 47, 0, 22, 3776, 0, 0, 0, 0, 0, 0, 1, 222, 0, 2, 2908, 500, 0, 0, 9, 870, 4096, 0, 0, 0, 0, 0, 3, 3343, 0, 1040, 0, 0, 0, 0, 0, 127, 0, 0, 2365, 0, 0, 0, 4041, 0, 0, 5, 0, 0, 0, 0, 95, 0, 4021, 0, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[5314, 3798, 3633, 4975, 4386, 2492, 4046, 4124] avg=4096.0000 max=5314 max_rank=0 min=2492 min_rank=5 imbalance_ratio=1.297363 max_load_reduction=27454 max_load_reduction_pct=0.837830 moved_to_echo=15048 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1952, 0, 3739, 0, 142, 116, 47, 0, 0, 3890, 0, 13, 0, 0, 0, 0, 0, 92, 12, 782, 3004, 211, 0, 0, 11, 368, 9, 0, 0, 0, 0, 0, 0, 4091, 0, 1075, 0, 0, 0, 0, 7, 4088, 0, 0, 999, 0, 0, 0, 4024, 0, 0, 3, 0, 0, 0, 0, 11, 0, 4074, 0, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[5996, 3903, 4101, 388, 5166, 5094, 4027, 4093] avg=4096.0000 max=5996 max_rank=0 min=388 min_rank=3 imbalance_ratio=1.463867 max_load_reduction=26772 max_load_reduction_pct=0.817017 moved_to_echo=18380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 1342, 337, 119, 47, 0, 0, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2174, 3017, 501, 0, 0, 1, 871, 4043, 0, 0, 0, 0, 0, 0, 465, 0, 1079, 0, 0, 0, 0, 5, 148, 0, 0, 2369, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4094, 0, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[5940, 3977, 5692, 4915, 1544, 2522, 4041, 4137] avg=4096.0000 max=5940 max_rank=0 min=1544 min_rank=4 imbalance_ratio=1.450195 max_load_reduction=26828 max_load_reduction_pct=0.818726 moved_to_echo=12244 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[391, 0, 585, 619, 338, 119, 47, 0, 0, 3977, 8, 0, 0, 0, 0, 0, 4, 0, 0, 28, 3017, 502, 0, 0, 1, 875, 3298, 0, 0, 0, 0, 0, 0, 3634, 0, 1079, 0, 0, 0, 0, 614, 3067, 0, 0, 2376, 0, 0, 0, 4040, 16, 0, 0, 0, 0, 0, 0, 0, 34, 4091, 0, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[2099, 3985, 3551, 4174, 4713, 6057, 4056, 4133] avg=4096.0000 max=6057 max_rank=5 min=2099 min_rank=0 imbalance_ratio=1.478760 max_load_reduction=26711 max_load_reduction_pct=0.815155 moved_to_echo=18959 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 14, 1815, 92, 344, 122, 52, 0, 5, 3960, 65, 0, 0, 0, 0, 0, 3, 0, 0, 1317, 3009, 507, 0, 0, 0, 861, 4064, 8, 0, 0, 0, 0, 0, 738, 4, 1074, 0, 0, 0, 0, 0, 4079, 0, 0, 2382, 0, 0, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 116, 4078, 106, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[2439, 4030, 4836, 4933, 1816, 6461, 3940, 4313] avg=4096.0000 max=6461 max_rank=5 min=1816 min_rank=4 imbalance_ratio=1.577393 max_load_reduction=26307 max_load_reduction_pct=0.802826 moved_to_echo=16530 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[168, 0, 1906, 1047, 329, 119, 46, 0, 0, 3972, 1210, 0, 0, 0, 0, 0, 0, 0, 74, 5, 3017, 489, 0, 0, 0, 850, 4065, 0, 0, 0, 0, 0, 28, 3729, 0, 1079, 0, 0, 0, 0, 20, 210, 0, 69, 2312, 0, 0, 0, 3972, 1, 0, 70, 0, 0, 0, 0, 0, 2, 3959, 12, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[3615, 5182, 3585, 4915, 4836, 2611, 4043, 3981] avg=4096.0000 max=5182 max_rank=1 min=2611 min_rank=5 imbalance_ratio=1.265137 max_load_reduction=27586 max_load_reduction_pct=0.841858 moved_to_echo=15471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 24192, 128, 8576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 14592, 128, 128, 17536, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3456, 128, 18176, 7552, 2560, 1024, 384, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 384, 128, 4736, 23936, 3712, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 6528, 26368, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32128, 256, 128, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 32384, 256, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31488, 1408, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 12, 0, 0, 4083, 4076, 376, 7, 21, 0, 89, 3, 2897, 0, 4085, 38, 0, 0, 3, 3, 57, 135, 3960, 3993, 8, 77, 2, 0, 0, 58, 593, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4094, 10, 10, 0, 0, 0, 0, 1785, 0, 3321, 0, 7, 0, 0, 0, 101, 0, 3859, 4096, 0, 546, 0, 0, 0, 57, 2599, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 3699, 4076, 0, 0, 0, 0, 0, 0, 382, 0, 4084, 16, 15, 0, 0, 0, 47, 0, 4096, 4093, 0, 0, 0, 4, 0, 3192, 886, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4095, 0, 0, 2, 4093, 4096, 0, 0, 0, 0, 1, 0, 7, 0, 905, 1087, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 1, 0, 0, 4072, 2121, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1126, 0, 0, 0, 3879, 3968, 5, 0, 0, 0, 0, 0, 0, 46, 4086, 0, 2025, 7, 0, 15, 64, 47, 4096, 4089, 0, 0, 0, 74, 1, 3018, 2143, 4079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 2, 4095, 4089, 0, 46, 0, 0, 0, 1, 0, 0, 3237, 0, 22, 4, 0, 0, 515, 0, 3674, 4096, 0, 0, 3, 5, 0, 3941, 1015, 3927] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4071, 0, 0, 0, 4089, 3867, 67, 0, 30, 0, 7, 4, 0, 0, 2162, 0, 16, 21, 0, 9, 0, 659, 4091, 4045, 0, 23, 0, 235, 0, 4057, 1249, 4066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3439, 20, 6, 0, 4082, 4085, 603, 0, 2, 1, 0, 0, 0, 1, 3731, 0, 38, 0, 1, 0, 50, 0, 4073, 4092, 0, 16, 0, 0, 5, 4095, 336, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 12, 0, 0, 317, 105, 0, 0, 4023, 0, 376, 7, 0, 0, 0, 0, 21, 0, 89, 3, 3993, 1, 0, 0, 2897, 0, 4085, 38, 60, 1, 0, 0, 0, 0, 3, 3, 3759, 94, 0, 0, 57, 135, 3855, 0, 0, 0, 0, 0, 8, 77, 2, 0, 4001, 0, 0, 0, 0, 57, 593, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4530, 4406, 4107, 7081, 3859, 4047, 4088, 650] avg=4096.0000 max=7081 max_rank=3 min=650 min_rank=7 imbalance_ratio=1.728760 max_load_reduction=25687 max_load_reduction_pct=0.783905 moved_to_echo=12644 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 318, 109, 34, 0, 3645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 382, 0, 4084, 16, 54, 60, 0, 0, 15, 0, 0, 0, 3758, 94, 0, 0, 47, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3989, 0, 0, 0, 0, 3098, 886, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4555, 3645, 4094, 4596, 3867, 4034, 3993, 3984] avg=4096.0000 max=4596 max_rank=3 min=3645 min_rank=1 imbalance_ratio=1.122070 max_load_reduction=28172 max_load_reduction_pct=0.859741 moved_to_echo=15878 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 2, 318, 97, 42, 0, 4035, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 0, 3237, 0, 60, 74, 0, 0, 22, 4, 0, 0, 3771, 90, 0, 0, 515, 0, 3577, 0, 0, 0, 0, 0, 0, 0, 3, 5, 3836, 0, 0, 0, 0, 3825, 1015, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4555, 4081, 4098, 3371, 3887, 4092, 3844, 4840] avg=4096.0000 max=4840 max_rank=7 min=3371 min_rank=3 imbalance_ratio=1.181641 max_load_reduction=27928 max_load_reduction_pct=0.852295 moved_to_echo=16663 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1126, 0, 0, 0, 308, 109, 32, 0, 3822, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 46, 4086, 0, 57, 57, 0, 0, 2025, 7, 0, 15, 3660, 93, 0, 0, 64, 47, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 74, 3985, 0, 0, 0, 1, 2929, 2143, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1575, 3827, 4090, 4246, 5800, 4098, 4059, 5073] avg=4096.0000 max=5800 max_rank=4 min=1575 min_rank=0 imbalance_ratio=1.416016 max_load_reduction=26968 max_load_reduction_pct=0.822998 moved_to_echo=19030 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 319, 102, 0, 0, 4036, 0, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1785, 0, 3321, 0, 60, 1, 0, 0, 7, 0, 0, 0, 3775, 95, 0, 0, 101, 0, 3757, 0, 0, 0, 0, 0, 0, 546, 0, 0, 3995, 0, 0, 0, 0, 56, 2599, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4517, 4056, 4097, 5167, 3877, 3858, 4541, 2655] avg=4096.0000 max=5167 max_rank=3 min=2655 min_rank=7 imbalance_ratio=1.261475 max_load_reduction=27601 max_load_reduction_pct=0.842316 moved_to_echo=14931 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 2, 318, 109, 44, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 1, 0, 0, 7, 0, 905, 1087, 60, 77, 0, 0, 0, 0, 0, 0, 3778, 94, 0, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4000, 0, 0, 0, 0, 3951, 2121, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4568, 4033, 4098, 2136, 3872, 3987, 4002, 6072] avg=4096.0000 max=6072 max_rank=7 min=2136 min_rank=3 imbalance_ratio=1.482422 max_load_reduction=26696 max_load_reduction_pct=0.814697 moved_to_echo=17933 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3439, 20, 6, 0, 318, 112, 48, 0, 4021, 0, 603, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4092, 2, 0, 0, 0, 1, 3731, 0, 61, 80, 0, 0, 38, 0, 1, 0, 3767, 94, 0, 0, 50, 0, 3961, 0, 0, 0, 0, 0, 0, 16, 0, 0, 3996, 0, 0, 0, 5, 3967, 336, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3943, 4624, 4097, 3873, 3900, 4011, 4012, 4308] avg=4096.0000 max=4624 max_rank=1 min=3873 min_rank=3 imbalance_ratio=1.128906 max_load_reduction=28144 max_load_reduction_pct=0.858887 moved_to_echo=16231 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4071, 0, 0, 0, 301, 109, 44, 0, 4029, 0, 67, 0, 0, 0, 0, 0, 30, 0, 7, 4, 4045, 1, 0, 0, 0, 0, 2162, 0, 60, 76, 0, 0, 16, 21, 0, 9, 3566, 94, 0, 0, 0, 659, 3982, 0, 0, 0, 0, 0, 0, 23, 0, 235, 3971, 0, 0, 0, 0, 3937, 1249, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4525, 4096, 4087, 2298, 3706, 4641, 4229, 5186] avg=4096.0000 max=5186 max_rank=7 min=2298 min_rank=3 imbalance_ratio=1.266113 max_load_reduction=27582 max_load_reduction_pct=0.841736 moved_to_echo=17762 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5120, 128, 25728, 1152, 512, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 21888, 11008, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[29184, 128, 128, 128, 2560, 896, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31744, 0, 1152, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2176, 128, 128, 128, 29952, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 768, 128, 384, 31872, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[896, 896, 31104, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2175, 0, 0, 0, 0, 0, 2, 14, 0, 0, 20, 647, 0, 0, 42, 4090, 3987, 1981, 0, 0, 3427, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4045, 0, 2, 0, 0, 232, 0, 5, 6, 1, 0, 528, 21, 0, 446, 4096, 338, 4096, 0, 53, 0, 3144, 3762, 0, 0, 576, 0, 0, 0, 3831, 3510, 4076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[41, 0, 0, 0, 0, 0, 0, 0, 0, 2, 120, 0, 0, 0, 0, 4096, 3935, 4086, 0, 0, 4083, 4096, 4096, 0, 0, 0, 26, 0, 0, 4090, 1, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[485, 0, 0, 0, 0, 0, 0, 4, 1, 445, 317, 0, 13, 0, 106, 4093, 2454, 4094, 2, 0, 2842, 4092, 4013, 0, 0, 0, 1445, 0, 0, 4096, 170, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 13, 0, 7, 1, 1353, 17, 0, 0, 0, 1, 0, 24, 4096, 4096, 4093, 0, 0, 2633, 4067, 4079, 18, 0, 4, 0, 0, 0, 4095, 82, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 3, 0, 52, 29, 4, 0, 29, 93, 0, 1, 1, 0, 21, 756, 4005, 2728, 4096, 0, 0, 2472, 3993, 4094, 0, 0, 280, 1, 37, 73, 4078, 1875, 4047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[839, 29, 0, 0, 1, 0, 5, 0, 3, 655, 7, 10, 0, 0, 8, 4094, 2415, 4092, 0, 0, 3919, 3986, 4086, 0, 0, 0, 0, 1, 15, 4096, 423, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[503, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 1, 152, 0, 0, 4088, 4090, 4096, 0, 0, 3480, 4049, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4045, 0, 2, 0, 1719, 691, 185, 0, 0, 232, 0, 5, 3874, 0, 0, 0, 6, 1, 0, 528, 2936, 437, 413, 0, 21, 0, 446, 3911, 0, 0, 0, 0, 338, 1160, 0, 53, 0, 0, 0, 0, 0, 988, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 3349, 202, 0, 0, 0, 3140, 3510, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6642, 4111, 4321, 4378, 1551, 988, 4127, 6650] avg=4096.0000 max=6650 max_rank=7 min=988 min_rank=5 imbalance_ratio=1.623535 max_load_reduction=26118 max_load_reduction_pct=0.797058 moved_to_echo=13316 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2175, 0, 0, 0, 2240, 739, 185, 0, 0, 0, 2, 14, 3893, 0, 0, 0, 0, 0, 20, 647, 1420, 570, 449, 0, 0, 0, 42, 3905, 0, 0, 0, 0, 3987, 561, 0, 0, 0, 0, 0, 0, 3427, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3646, 203, 0, 0, 0, 3357, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5339, 3909, 3106, 3947, 4548, 4713, 3849, 3357] avg=4096.0000 max=5339 max_rank=0 min=3106 min_rank=2 imbalance_ratio=1.303467 max_load_reduction=27429 max_load_reduction_pct=0.837067 moved_to_echo=16467 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[41, 0, 0, 0, 2240, 737, 185, 0, 0, 0, 0, 0, 3894, 0, 0, 0, 0, 2, 120, 0, 2929, 570, 449, 0, 0, 0, 0, 3911, 0, 0, 0, 0, 3935, 1157, 0, 0, 0, 0, 0, 0, 4083, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 3647, 202, 0, 0, 0, 3353, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3203, 3894, 4070, 3911, 5092, 5369, 3875, 3354] avg=4096.0000 max=5369 max_rank=5 min=3203 min_rank=0 imbalance_ratio=1.310791 max_load_reduction=27399 max_load_reduction_pct=0.836151 moved_to_echo=17690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[485, 0, 0, 0, 2238, 739, 185, 0, 0, 0, 0, 4, 3893, 0, 0, 0, 1, 445, 317, 0, 2935, 569, 439, 0, 13, 0, 106, 3908, 0, 0, 0, 0, 2454, 1159, 2, 0, 0, 0, 0, 0, 2842, 1285, 0, 0, 0, 0, 0, 0, 0, 0, 1445, 0, 3574, 203, 0, 0, 0, 3357, 170, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3647, 3897, 4706, 4027, 3615, 4127, 5222, 3527] avg=4096.0000 max=5222 max_rank=6 min=3527 min_rank=7 imbalance_ratio=1.274902 max_load_reduction=27546 max_load_reduction_pct=0.840637 moved_to_echo=16491 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 13, 2224, 738, 185, 0, 0, 7, 1, 1353, 3886, 0, 0, 0, 17, 0, 0, 0, 2934, 566, 446, 0, 1, 0, 24, 3911, 0, 0, 0, 0, 4096, 1159, 0, 0, 0, 0, 0, 0, 2633, 1277, 0, 18, 0, 0, 0, 0, 0, 4, 0, 0, 3633, 203, 0, 0, 0, 3357, 82, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3160, 5247, 3963, 3936, 5255, 3928, 3840, 3439] avg=4096.0000 max=5255 max_rank=4 min=3160 min_rank=0 imbalance_ratio=1.282959 max_load_reduction=27513 max_load_reduction_pct=0.839630 moved_to_echo=16462 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[503, 0, 0, 0, 2214, 739, 185, 0, 0, 0, 0, 20, 3893, 0, 0, 0, 0, 0, 0, 1, 2936, 563, 448, 0, 152, 0, 0, 3903, 0, 0, 0, 0, 4090, 1160, 0, 0, 0, 0, 0, 0, 3480, 1272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 203, 0, 0, 0, 3357, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3641, 3913, 3948, 4055, 5250, 4752, 3851, 3358] avg=4096.0000 max=5250 max_rank=4 min=3358 min_rank=7 imbalance_ratio=1.281738 max_load_reduction=27518 max_load_reduction_pct=0.839783 moved_to_echo=17211 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[839, 29, 0, 0, 2183, 742, 189, 0, 1, 0, 5, 0, 3882, 0, 0, 0, 3, 655, 7, 10, 2938, 558, 448, 0, 0, 0, 8, 3905, 0, 0, 0, 0, 2415, 1154, 0, 0, 0, 0, 0, 0, 3919, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3638, 202, 0, 0, 15, 3354, 423, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3982, 3888, 4619, 3913, 3569, 5164, 3841, 3792] avg=4096.0000 max=5164 max_rank=5 min=3569 min_rank=4 imbalance_ratio=1.260742 max_load_reduction=27604 max_load_reduction_pct=0.842407 moved_to_echo=16366 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 52, 2183, 735, 181, 0, 29, 4, 0, 29, 3847, 0, 0, 0, 93, 0, 1, 1, 2936, 555, 449, 0, 0, 21, 756, 3824, 0, 0, 0, 0, 2728, 1160, 0, 0, 0, 0, 0, 0, 2472, 1255, 0, 0, 0, 0, 0, 0, 0, 280, 1, 37, 3645, 200, 0, 0, 73, 3343, 1875, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3154, 3909, 4035, 4601, 3888, 3727, 4163, 5291] avg=4096.0000 max=5291 max_rank=7 min=3154 min_rank=0 imbalance_ratio=1.291748 max_load_reduction=27477 max_load_reduction_pct=0.838531 moved_to_echo=17069 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8192, 128, 128, 128, 17280, 5888, 1536, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[22912, 9984, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 1152, 512, 1280, 22016, 4480, 3584, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[24064, 8704, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 26624, 6144, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 896, 1536, 128, 28800, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 1408, 31232, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 128, 1536, 31104, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[709, 3630, 4065, 358, 11, 0, 4088, 4008, 0, 1, 3251, 26, 278, 0, 0, 0, 38, 13, 4, 0, 4096, 3069, 0, 0, 6, 18, 466, 99, 4041, 0, 493, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4095, 3889, 0, 0, 31, 4095, 4096, 1183, 1, 97, 0, 33, 0, 0, 8, 9, 0, 0, 0, 3558, 4096, 0, 0, 4, 1912, 704, 2, 4093, 0, 862, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1, 4096, 4096, 0, 0, 0, 4091, 4096, 9, 0, 3710, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4009, 3889, 0, 0, 28, 419, 0, 202, 4096, 0, 19, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4081, 4096, 0, 0, 0, 4085, 4096, 0, 7, 1192, 5, 0, 0, 2, 0, 0, 0, 0, 0, 4039, 2843, 8, 0, 0, 79, 6, 4087, 4085, 0, 57, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4091, 4096, 0, 1, 4, 122, 4096, 0, 0, 3577, 0, 59, 0, 2, 0, 2, 0, 0, 0, 4096, 3752, 0, 0, 7, 4084, 0, 683, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 3950, 4073, 0, 2, 0, 0, 4084, 333, 1, 2390, 0, 2, 0, 0, 1, 5, 17, 0, 24, 4091, 4091, 0, 0, 110, 2985, 0, 2521, 4087, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 3215, 4096, 886, 0, 4089, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 6, 2, 1, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4090, 3081, 1, 0, 87, 4007, 4056, 0, 0, 4088, 0, 310, 0, 0, 0, 71, 0, 11, 3, 4096, 6, 0, 0, 1092, 3056, 25, 604, 4084, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[709, 24, 4065, 358, 0, 0, 0, 0, 11, 0, 4088, 1098, 0, 0, 0, 0, 0, 1, 3251, 26, 808, 165, 0, 0, 278, 0, 0, 0, 3128, 928, 0, 0, 38, 13, 4, 0, 3606, 82, 0, 0, 886, 3069, 0, 0, 0, 0, 0, 0, 6, 18, 466, 99, 1174, 0, 0, 0, 3876, 0, 493, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5156, 5197, 4251, 4334, 3743, 3955, 1763, 4369] avg=4096.0000 max=5197 max_rank=1 min=1763 min_rank=6 imbalance_ratio=1.268799 max_load_reduction=27571 max_load_reduction_pct=0.841400 moved_to_echo=13830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 3889, 0, 0, 0, 0, 0, 0, 31, 4095, 1122, 0, 0, 0, 0, 1183, 1, 97, 0, 826, 168, 0, 0, 33, 0, 0, 8, 2717, 948, 0, 0, 9, 0, 0, 0, 4068, 71, 0, 0, 770, 4096, 0, 0, 0, 0, 0, 0, 4, 1912, 704, 2, 1200, 0, 0, 0, 3925, 0, 862, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3916, 5248, 2275, 3706, 4148, 4866, 3822, 4787] avg=4096.0000 max=5248 max_rank=1 min=2275 min_rank=2 imbalance_ratio=1.281250 max_load_reduction=27520 max_load_reduction_pct=0.839844 moved_to_echo=17623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 4096, 0, 0, 0, 0, 0, 1, 4, 122, 1122, 0, 0, 0, 0, 0, 0, 3577, 0, 826, 168, 0, 0, 59, 0, 2, 0, 3128, 948, 0, 0, 2, 0, 0, 0, 4064, 82, 0, 0, 886, 3752, 0, 0, 0, 0, 0, 0, 7, 4084, 0, 683, 1200, 0, 0, 0, 3928, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 1249, 4571, 4137, 4148, 4638, 5974, 3928] avg=4096.0000 max=5974 max_rank=6 min=1249 min_rank=1 imbalance_ratio=1.458496 max_load_reduction=26794 max_load_reduction_pct=0.817688 moved_to_echo=18688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 1122, 0, 0, 0, 0, 0, 7, 1192, 5, 826, 167, 0, 0, 0, 0, 2, 0, 3085, 948, 0, 0, 0, 0, 0, 0, 4054, 81, 0, 0, 873, 2843, 8, 0, 0, 0, 0, 0, 0, 79, 6, 4087, 1200, 0, 0, 0, 3918, 0, 57, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 5207, 2197, 4035, 4135, 3724, 5372, 3975] avg=4096.0000 max=5372 max_rank=6 min=2197 min_rank=2 imbalance_ratio=1.311523 max_load_reduction=27396 max_load_reduction_pct=0.836060 moved_to_echo=17206 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 3215, 1122, 0, 0, 0, 0, 886, 0, 4089, 0, 826, 168, 0, 0, 1, 0, 0, 0, 3128, 948, 0, 0, 0, 0, 0, 0, 4069, 82, 0, 0, 886, 4088, 0, 0, 0, 0, 0, 0, 0, 6, 2, 1, 1200, 0, 0, 0, 3928, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4337, 5969, 4077, 4151, 4974, 1209, 3928] avg=4096.0000 max=5969 max_rank=2 min=1209 min_rank=6 imbalance_ratio=1.457275 max_load_reduction=26799 max_load_reduction_pct=0.817841 moved_to_echo=14262 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 1122, 0, 0, 0, 0, 9, 0, 3710, 0, 826, 168, 0, 0, 7, 0, 0, 0, 3062, 948, 0, 0, 0, 0, 0, 0, 4069, 81, 0, 0, 866, 3889, 0, 0, 0, 0, 0, 0, 28, 419, 0, 202, 1200, 0, 0, 0, 3928, 0, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 5213, 4713, 4017, 4150, 4755, 1849, 3947] avg=4096.0000 max=5213 max_rank=1 min=1849 min_rank=6 imbalance_ratio=1.272705 max_load_reduction=27555 max_load_reduction_pct=0.840912 moved_to_echo=14701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 21, 4073, 0, 0, 0, 0, 0, 2, 0, 0, 1109, 0, 0, 0, 0, 333, 1, 2390, 0, 825, 171, 0, 0, 2, 0, 0, 1, 3130, 952, 0, 0, 5, 17, 0, 24, 3929, 88, 0, 0, 873, 4091, 0, 0, 0, 0, 0, 0, 110, 2985, 0, 2521, 1198, 0, 0, 0, 3916, 0, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 1111, 3720, 4085, 4063, 4964, 6814, 3917] avg=4096.0000 max=6814 max_rank=6 min=1111 min_rank=1 imbalance_ratio=1.663574 max_load_reduction=25954 max_load_reduction_pct=0.792053 moved_to_echo=19758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 3081, 1, 0, 0, 0, 0, 0, 87, 4007, 1112, 0, 0, 0, 0, 0, 0, 4088, 0, 817, 167, 0, 0, 310, 0, 0, 0, 3128, 939, 0, 0, 71, 0, 11, 3, 4063, 82, 0, 0, 886, 6, 0, 0, 0, 0, 0, 0, 1092, 3056, 25, 604, 1188, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3109, 5206, 5072, 4377, 4230, 892, 5965, 3917] avg=4096.0000 max=5965 max_rank=6 min=892 min_rank=5 imbalance_ratio=1.456299 max_load_reduction=26803 max_load_reduction_pct=0.817963 moved_to_echo=15004 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 0, 128, 128, 24576, 7680, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1280, 12672, 1280, 8320, 9600, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 23808, 8960, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7040, 25856, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2432, 128, 22400, 128, 6656, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 256, 31616, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31360, 0, 1536, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32000, 768, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[22, 3762, 566, 0, 55, 0, 0, 3975, 219, 0, 0, 22, 3746, 4086, 36, 7, 1, 212, 5, 4049, 0, 3729, 19, 0, 0, 4060, 65, 9, 73, 10, 4039, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3909, 0, 0, 0, 11, 0, 208, 9, 0, 0, 0, 4088, 4096, 0, 843, 7, 2, 111, 3872, 30, 4095, 0, 0, 0, 3568, 0, 0, 3823, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4096, 4096, 0, 4037, 0, 1, 433, 4096, 0, 4096, 0, 0, 0, 102, 0, 0, 3623, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[11, 4090, 0, 2, 0, 1, 0, 0, 0, 0, 1, 0, 4089, 4052, 32, 3796, 118, 0, 0, 4096, 0, 4095, 46, 0, 0, 176, 0, 3, 4093, 0, 4067, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 77, 0, 4, 0, 0, 4094, 4096, 0, 12, 1, 2, 0, 4095, 0, 4092, 0, 0, 0, 4092, 0, 0, 4010, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4090, 347, 4, 0, 0, 2, 794, 2, 0, 0, 0, 4090, 4093, 0, 3836, 3, 154, 0, 4070, 0, 4093, 0, 0, 2, 8, 0, 2, 3088, 0, 4090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 3485, 0, 0, 3, 0, 0, 510, 37, 4, 2, 1, 4091, 4094, 8, 4096, 0, 14, 0, 4096, 17, 4092, 0, 0, 0, 0, 0, 4, 4002, 0, 4083, 129] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 56, 0, 1, 51, 46, 0, 0, 4068, 4094, 4, 3648, 160, 0, 22, 3590, 0, 4091, 0, 0, 23, 691, 0, 0, 4042, 10, 4075, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[22, 3762, 566, 0, 26, 0, 0, 0, 55, 0, 0, 3975, 3306, 42, 0, 0, 219, 0, 0, 22, 4044, 0, 0, 0, 1438, 0, 36, 7, 0, 0, 0, 0, 1, 212, 5, 3994, 0, 0, 0, 0, 0, 3729, 19, 0, 29, 4, 0, 0, 0, 4060, 65, 9, 2304, 6, 0, 0, 73, 10, 727, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4376, 7378, 4285, 1481, 4212, 3781, 6444, 811] avg=4096.0000 max=7378 max_rank=1 min=811 min_rank=7 imbalance_ratio=1.801270 max_load_reduction=25390 max_load_reduction_pct=0.774841 moved_to_echo=15248 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3909, 0, 0, 25, 0, 0, 0, 0, 11, 0, 208, 3353, 42, 0, 0, 9, 0, 0, 0, 4054, 0, 0, 0, 1569, 0, 0, 843, 0, 0, 0, 0, 7, 2, 111, 3819, 0, 0, 0, 0, 30, 4095, 0, 0, 28, 4, 0, 0, 0, 3568, 0, 0, 2515, 6, 0, 0, 3823, 0, 737, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3934, 3614, 4063, 2412, 3939, 4157, 6089, 4560] avg=4096.0000 max=6089 max_rank=6 min=2412 min_rank=3 imbalance_ratio=1.486572 max_load_reduction=26679 max_load_reduction_pct=0.814178 moved_to_echo=18745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 347, 4, 26, 0, 0, 0, 0, 0, 2, 794, 3348, 42, 0, 0, 2, 0, 0, 0, 4051, 0, 0, 0, 1570, 0, 0, 3836, 0, 0, 0, 0, 3, 154, 0, 4015, 0, 0, 0, 0, 0, 4093, 0, 0, 29, 4, 0, 0, 2, 8, 0, 2, 2516, 6, 0, 0, 3088, 0, 736, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4467, 4186, 4053, 5406, 4172, 4126, 2534, 3824] avg=4096.0000 max=5406 max_rank=3 min=2534 min_rank=6 imbalance_ratio=1.319824 max_load_reduction=27362 max_load_reduction_pct=0.835022 moved_to_echo=14656 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 26, 0, 0, 0, 0, 0, 0, 12, 3353, 42, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 1573, 0, 0, 4037, 0, 0, 0, 0, 0, 1, 433, 4041, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 4, 0, 0, 0, 102, 0, 0, 2519, 6, 0, 0, 3623, 0, 737, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 3407, 4054, 5610, 4475, 4129, 2627, 4360] avg=4096.0000 max=5610 max_rank=3 min=2627 min_rank=6 imbalance_ratio=1.369629 max_load_reduction=27158 max_load_reduction_pct=0.828796 moved_to_echo=15591 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[11, 4090, 0, 2, 26, 0, 0, 0, 0, 1, 0, 0, 3329, 41, 0, 0, 0, 0, 1, 0, 4011, 0, 0, 0, 1570, 0, 32, 3796, 0, 0, 0, 0, 118, 0, 0, 4041, 0, 0, 0, 0, 0, 4095, 46, 0, 29, 4, 0, 0, 0, 176, 0, 3, 2515, 6, 0, 0, 4093, 0, 732, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 3371, 4012, 5398, 4159, 4174, 2700, 4825] avg=4096.0000 max=5398 max_rank=3 min=2700 min_rank=6 imbalance_ratio=1.317871 max_load_reduction=27370 max_load_reduction_pct=0.835266 moved_to_echo=15858 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 26, 0, 0, 0, 0, 1, 0, 77, 3353, 42, 0, 0, 0, 4, 0, 0, 4054, 0, 0, 0, 1572, 0, 0, 12, 0, 0, 0, 0, 1, 2, 0, 4040, 0, 0, 0, 0, 0, 4092, 0, 0, 29, 4, 0, 0, 0, 4092, 0, 0, 2518, 6, 0, 0, 4010, 0, 737, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 3473, 4058, 1584, 4043, 4125, 6616, 4747] avg=4096.0000 max=6616 max_rank=6 min=1584 min_rank=3 imbalance_ratio=1.615234 max_load_reduction=26152 max_load_reduction_pct=0.798096 moved_to_echo=19531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3485, 0, 0, 30, 0, 0, 0, 3, 0, 0, 510, 3347, 43, 0, 0, 37, 4, 2, 1, 4051, 0, 0, 0, 1559, 0, 8, 4096, 0, 0, 0, 0, 0, 14, 0, 4031, 0, 0, 0, 0, 17, 4092, 0, 0, 35, 11, 0, 0, 0, 0, 0, 4, 2521, 11, 0, 0, 4002, 0, 725, 129, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3515, 3903, 4095, 5663, 4045, 4155, 2536, 4856] avg=4096.0000 max=5663 max_rank=3 min=2536 min_rank=6 imbalance_ratio=1.382568 max_load_reduction=27105 max_load_reduction_pct=0.827179 moved_to_echo=15592 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 23, 0, 0, 0, 0, 56, 0, 1, 3336, 43, 0, 0, 51, 46, 0, 0, 4051, 0, 0, 0, 1562, 0, 4, 3648, 0, 0, 0, 0, 160, 0, 22, 3541, 0, 0, 0, 0, 0, 4091, 0, 0, 26, 4, 0, 0, 23, 691, 0, 0, 2502, 6, 0, 0, 4042, 10, 733, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 3436, 4148, 5214, 3723, 4121, 3222, 4785] avg=4096.0000 max=5214 max_rank=3 min=3222 min_rank=6 imbalance_ratio=1.272949 max_load_reduction=27554 max_load_reduction_pct=0.840881 moved_to_echo=15851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 12800, 128, 128, 19968, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[12416, 0, 128, 20352, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 5632, 26752, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[26880, 128, 5888, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31616, 1024, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 512, 640, 31616, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32384, 128, 0, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 128, 128, 32384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[90, 0, 4096, 0, 0, 94, 3835, 4093, 0, 1, 0, 53, 0, 4064, 0, 0, 4093, 0, 4093, 0, 0, 154, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[3915, 2, 3417, 0, 0, 463, 3666, 4056, 0, 15, 2, 0, 0, 4094, 0, 0, 4092, 0, 4096, 0, 0, 17, 0, 0, 819, 0, 0, 1, 17, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4065, 27, 3770, 0, 0, 3, 4096, 4079, 0, 0, 54, 956, 0, 204, 0, 39, 4096, 0, 3093, 146, 36, 10, 0, 0, 4049, 0, 1, 0, 0, 0, 15, 4029] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3690, 31, 456, 0, 0, 0, 3816, 3971, 3, 0, 0, 388, 0, 4094, 3, 0, 4096, 0, 3578, 0, 0, 119, 1, 0, 4093, 0, 0, 98, 227, 8, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[207, 0, 3846, 0, 0, 7, 4096, 4088, 0, 0, 0, 0, 2, 4035, 0, 1, 4095, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 103, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4089, 49, 3928, 170, 0, 771, 4096, 227, 4, 0, 0, 3982, 0, 4039, 0, 0, 2463, 0, 398, 0, 0, 0, 5, 330, 4083, 0, 0, 38, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 4067, 9, 0, 5, 4012, 4087, 0, 87, 0, 29, 0, 4090, 0, 5, 4062, 0, 4073, 0, 0, 0, 1, 0, 4062, 1, 26, 34, 0, 0, 26, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4005, 0, 0, 6, 4084, 3951, 76, 0, 0, 1360, 0, 2800, 0, 0, 4091, 0, 4067, 0, 6, 0, 96, 0, 4088, 0, 0, 0, 43, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4065, 27, 2245, 0, 0, 0, 0, 0, 0, 3, 371, 4079, 0, 0, 0, 0, 0, 0, 54, 956, 3004, 404, 0, 0, 0, 204, 0, 39, 724, 0, 0, 0, 676, 0, 3093, 146, 0, 0, 0, 0, 36, 10, 0, 0, 3725, 416, 0, 0, 4049, 0, 1, 0, 397, 45, 0, 0, 0, 0, 15, 3984, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6337, 4453, 4418, 967, 3915, 4187, 4492, 3999] avg=4096.0000 max=6337 max_rank=0 min=967 min_rank=3 imbalance_ratio=1.547119 max_load_reduction=26431 max_load_reduction_pct=0.806610 moved_to_echo=16593 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3915, 2, 2035, 0, 0, 0, 0, 0, 0, 463, 332, 4056, 0, 0, 0, 0, 0, 15, 2, 0, 3001, 366, 0, 0, 0, 4094, 0, 0, 656, 0, 0, 0, 676, 0, 4096, 0, 0, 0, 0, 0, 0, 17, 0, 0, 3334, 415, 0, 0, 819, 0, 0, 1, 360, 45, 0, 0, 17, 0, 0, 4051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5952, 4851, 3384, 4750, 4772, 3766, 1225, 4068] avg=4096.0000 max=5952 max_rank=0 min=1225 min_rank=6 imbalance_ratio=1.453125 max_load_reduction=26816 max_load_reduction_pct=0.818359 moved_to_echo=13831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4089, 49, 2340, 170, 0, 0, 0, 0, 0, 771, 371, 227, 0, 0, 0, 0, 4, 0, 0, 3982, 1806, 421, 0, 0, 0, 4039, 0, 0, 754, 0, 0, 0, 407, 0, 398, 0, 0, 0, 0, 0, 0, 0, 5, 330, 3725, 250, 0, 0, 4083, 0, 0, 38, 413, 45, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6648, 1369, 6213, 4793, 805, 4310, 4579, 4051] avg=4096.0000 max=6648 max_rank=0 min=805 min_rank=4 imbalance_ratio=1.623047 max_load_reduction=26120 max_load_reduction_pct=0.797119 moved_to_echo=13745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[90, 0, 2440, 0, 0, 0, 0, 0, 0, 94, 347, 4093, 0, 0, 0, 0, 0, 1, 0, 53, 3001, 439, 0, 0, 0, 4064, 0, 0, 786, 0, 0, 0, 677, 0, 4093, 0, 0, 0, 0, 0, 0, 154, 0, 0, 3488, 415, 0, 0, 4006, 0, 0, 0, 431, 45, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2530, 4534, 3494, 4850, 4770, 4057, 4482, 4051] avg=4096.0000 max=4850 max_rank=3 min=2530 min_rank=0 imbalance_ratio=1.184082 max_load_reduction=27918 max_load_reduction_pct=0.851990 moved_to_echo=17360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3690, 31, 273, 0, 0, 0, 0, 0, 0, 0, 346, 3971, 0, 0, 0, 0, 3, 0, 0, 388, 3004, 48, 0, 0, 0, 4094, 3, 0, 87, 0, 0, 0, 676, 0, 3578, 0, 0, 0, 0, 0, 0, 119, 1, 0, 3470, 416, 0, 0, 4093, 0, 0, 98, 48, 45, 0, 0, 227, 8, 0, 4051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 4317, 3443, 4184, 4254, 4006, 4284, 4286] avg=4096.0000 max=4317 max_rank=1 min=3443 min_rank=2 imbalance_ratio=1.053955 max_load_reduction=28451 max_load_reduction_pct=0.868256 moved_to_echo=16830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[207, 0, 2291, 0, 0, 0, 0, 0, 0, 7, 371, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 3003, 412, 0, 0, 2, 4035, 0, 1, 738, 0, 0, 0, 676, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3725, 416, 0, 0, 4096, 0, 0, 0, 405, 45, 0, 0, 0, 103, 0, 4051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2498, 4466, 3415, 4776, 4772, 4141, 4546, 4154] avg=4096.0000 max=4776 max_rank=3 min=2498 min_rank=0 imbalance_ratio=1.166016 max_load_reduction=27992 max_load_reduction_pct=0.854248 moved_to_echo=17613 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2414, 9, 0, 0, 0, 0, 0, 5, 359, 4087, 0, 0, 0, 0, 0, 87, 0, 29, 2982, 438, 0, 0, 0, 4090, 0, 5, 784, 0, 0, 0, 664, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3653, 416, 0, 0, 4062, 1, 26, 34, 431, 51, 0, 0, 0, 0, 26, 4040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2424, 4451, 3536, 4879, 4737, 4070, 4605, 4066] avg=4096.0000 max=4879 max_rank=3 min=2424 min_rank=0 imbalance_ratio=1.191162 max_load_reduction=27889 max_load_reduction_pct=0.851105 moved_to_echo=17478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2385, 0, 0, 0, 0, 0, 0, 6, 370, 3951, 0, 0, 0, 0, 76, 0, 0, 1360, 3000, 429, 0, 0, 0, 2800, 0, 0, 769, 0, 0, 0, 676, 0, 4067, 0, 0, 0, 0, 0, 6, 0, 96, 0, 3714, 415, 0, 0, 4088, 0, 0, 0, 422, 45, 0, 0, 43, 0, 0, 4050, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2385, 4327, 4865, 3569, 4743, 4231, 4555, 4093] avg=4096.0000 max=4865 max_rank=2 min=2385 min_rank=0 imbalance_ratio=1.187744 max_load_reduction=27903 max_load_reduction_pct=0.851532 moved_to_echo=17622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 6784, 22912, 3072, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16128, 128, 16512, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5248, 0, 27520, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 384, 128, 384, 28928, 3200, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 27520, 128, 128, 5376, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[29312, 128, 128, 256, 2944, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 1408, 2944, 28672, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 128, 128, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4092, 13, 0, 0, 3907, 47, 184, 3451, 271, 0, 4081, 52, 0, 2, 6, 0, 0, 10, 0, 336, 3374, 3652, 0, 405, 0, 4046, 733, 0, 51, 9, 0, 4046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3425, 0, 0, 4094, 4096, 0, 429, 790, 0, 0, 1, 1, 0, 3, 0, 3, 46, 1, 0, 2, 0, 4073, 4095, 0, 3519, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3960, 0, 0, 4096, 4096, 0, 473, 0, 118, 7, 916, 0, 0, 0, 0, 3744, 25, 0, 0, 0, 0, 40, 4065, 0, 3067, 0, 0, 4065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 30, 0, 3, 2400, 0, 3, 4094, 4093, 0, 162, 9, 0, 1, 781, 0, 0, 0, 0, 3221, 0, 1243, 0, 0, 4, 4066, 4008, 0, 457, 0, 1, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1290, 0, 0, 4096, 3770, 0, 1283, 621, 4, 0, 1741, 0, 0, 1, 0, 66, 2446, 944, 0, 0, 0, 4095, 4096, 0, 123, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4091, 2, 0, 0, 0, 112, 274, 4074, 3927, 0, 1356, 374, 0, 0, 53, 2, 0, 16, 13, 3694, 860, 474, 0, 15, 0, 4095, 3918, 0, 1271, 60, 2, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[3915, 1, 157, 0, 3, 0, 34, 4096, 4088, 82, 2090, 0, 0, 0, 904, 0, 0, 475, 3, 3923, 0, 0, 1, 11, 17, 1009, 4096, 222, 4010, 0, 0, 3631] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 1464, 0, 0, 4052, 3958, 0, 4028, 801, 0, 0, 0, 0, 4, 38, 0, 48, 95, 4050, 0, 0, 0, 4077, 1333, 5, 666, 0, 0, 4054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4091, 13, 0, 0, 0, 0, 0, 0, 3907, 47, 184, 1688, 0, 0, 0, 0, 155, 0, 4081, 52, 0, 0, 0, 0, 0, 2, 6, 0, 537, 958, 0, 0, 0, 10, 0, 336, 1218, 805, 0, 0, 3374, 3652, 0, 405, 116, 370, 1, 0, 0, 4046, 196, 0, 0, 0, 0, 0, 51, 9, 0, 2458, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 5826, 4288, 1503, 2369, 7918, 4242, 2518] avg=4096.0000 max=7918 max_rank=5 min=1503 min_rank=3 imbalance_ratio=1.933105 max_load_reduction=24850 max_load_reduction_pct=0.758362 moved_to_echo=17047 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 30, 0, 3, 0, 0, 0, 0, 2400, 0, 3, 2002, 0, 0, 0, 0, 2334, 0, 162, 9, 0, 0, 0, 0, 0, 1, 781, 0, 2940, 1137, 0, 0, 0, 0, 0, 3221, 1233, 955, 0, 0, 0, 1243, 0, 0, 1759, 375, 1, 0, 4, 4066, 1068, 0, 0, 0, 0, 0, 457, 0, 1, 2488, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4405, 2505, 4859, 5409, 3378, 5138, 2946] avg=4096.0000 max=5409 max_rank=4 min=2505 min_rank=2 imbalance_ratio=1.320557 max_load_reduction=27359 max_load_reduction_pct=0.834930 moved_to_echo=16871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 2002, 0, 0, 0, 0, 2336, 0, 473, 0, 0, 0, 0, 0, 118, 7, 916, 0, 2981, 1138, 0, 0, 0, 0, 0, 3744, 1224, 956, 0, 0, 25, 0, 0, 0, 1760, 372, 1, 0, 0, 40, 1084, 0, 0, 0, 0, 0, 3067, 0, 0, 2469, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 5962, 2809, 5160, 5924, 2158, 1124, 5536] avg=4096.0000 max=5962 max_rank=1 min=1124 min_rank=6 imbalance_ratio=1.455566 max_load_reduction=26806 max_load_reduction_pct=0.818054 moved_to_echo=14742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 3425, 0, 0, 2002, 0, 0, 0, 0, 2336, 0, 429, 790, 0, 0, 0, 0, 0, 0, 1, 1, 3003, 1137, 0, 0, 0, 3, 0, 3, 1233, 955, 0, 0, 46, 1, 0, 2, 1760, 375, 1, 0, 0, 4073, 1092, 0, 0, 0, 0, 0, 3519, 0, 0, 2486, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 5427, 3555, 4142, 2194, 2185, 5165, 6005] avg=4096.0000 max=6005 max_rank=7 min=2185 min_rank=5 imbalance_ratio=1.466064 max_load_reduction=26763 max_load_reduction_pct=0.816742 moved_to_echo=15549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 1290, 0, 0, 2002, 0, 0, 0, 0, 2150, 0, 1283, 621, 0, 0, 0, 0, 4, 0, 1741, 0, 3004, 1138, 0, 0, 0, 1, 0, 66, 1233, 956, 0, 0, 2446, 944, 0, 0, 1620, 375, 1, 0, 0, 4095, 1092, 0, 0, 0, 0, 0, 123, 0, 0, 2488, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 3292, 4054, 5887, 2256, 5386, 5187, 2611] avg=4096.0000 max=5887 max_rank=3 min=2256 min_rank=4 imbalance_ratio=1.437256 max_load_reduction=26881 max_load_reduction_pct=0.820343 moved_to_echo=15440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4090, 2, 0, 0, 0, 0, 0, 0, 0, 112, 274, 1993, 0, 0, 0, 0, 2240, 0, 1356, 374, 0, 0, 0, 0, 0, 0, 53, 2, 2874, 1131, 0, 0, 0, 16, 13, 3694, 1230, 950, 0, 0, 860, 474, 0, 15, 1687, 374, 1, 0, 0, 4095, 1044, 0, 0, 0, 0, 0, 1271, 60, 2, 2481, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 2379, 3970, 4060, 5903, 3411, 5139, 3814] avg=4096.0000 max=5903 max_rank=4 min=2379 min_rank=1 imbalance_ratio=1.441162 max_load_reduction=26865 max_load_reduction_pct=0.819855 moved_to_echo=18267 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 0, 0, 0, 0, 0, 0, 1464, 0, 0, 1974, 0, 0, 0, 0, 2254, 0, 4028, 801, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1129, 0, 0, 4, 38, 0, 48, 1224, 949, 0, 0, 95, 4050, 0, 0, 1704, 374, 8, 0, 0, 4077, 351, 5, 0, 0, 0, 0, 666, 0, 0, 2456, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 3438, 7083, 2111, 2263, 6231, 4433, 3122] avg=4096.0000 max=7083 max_rank=2 min=2111 min_rank=3 imbalance_ratio=1.729248 max_load_reduction=25685 max_load_reduction_pct=0.783844 moved_to_echo=16049 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3914, 1, 157, 0, 0, 0, 0, 0, 3, 0, 34, 2002, 0, 0, 0, 0, 2332, 82, 2090, 0, 0, 0, 0, 0, 0, 0, 904, 0, 3004, 1138, 0, 0, 0, 475, 3, 3923, 1093, 956, 0, 0, 0, 0, 1, 11, 1756, 332, 1, 0, 17, 1009, 1092, 222, 0, 0, 0, 0, 4010, 0, 0, 2206, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 2039, 4504, 5046, 6450, 2101, 2340, 6216] avg=4096.0000 max=6450 max_rank=4 min=2039 min_rank=1 imbalance_ratio=1.574707 max_load_reduction=26318 max_load_reduction_pct=0.803162 moved_to_echo=17107 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[6912, 10368, 128, 512, 12288, 3072, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16256, 128, 13952, 2688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16512, 256, 512, 15744, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 4480, 128, 19328, 8960, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13184, 128, 128, 19584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 640, 128, 15104, 9728, 7552, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 25600, 7040, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 409, 0, 10, 1, 0, 4084, 3503, 0, 3, 0, 4095, 4096, 4083, 0, 4096, 0, 6, 4095, 361, 0, 0, 0, 2931, 1, 0, 993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[14, 0, 2, 0, 0, 0, 87, 12, 0, 3942, 125, 4089, 0, 0, 25, 0, 4046, 3628, 4085, 0, 3912, 16, 8, 4081, 0, 0, 6, 630, 4009, 1, 48, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 13, 0, 0, 0, 1, 0, 0, 1, 920, 225, 3873, 0, 0, 0, 0, 4095, 4095, 4072, 0, 4040, 0, 91, 4096, 1, 0, 0, 0, 4073, 3172, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 3517, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 4096, 565, 0, 0, 1, 4096, 9, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 75, 0, 0, 0, 0, 4, 1283, 0, 154, 1, 4096, 14, 6, 3, 0, 4096, 4093, 4094, 0, 4031, 0, 0, 4090, 0, 0, 0, 0, 3862, 2866, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[418, 42, 29, 14, 76, 0, 0, 340, 0, 3692, 0, 3948, 205, 27, 0, 1, 4091, 4020, 4046, 0, 3671, 0, 0, 3980, 82, 5, 1, 0, 4075, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 3862, 0, 4096, 31, 0, 0, 1, 4096, 4096, 4096, 0, 4096, 0, 0, 4096, 257, 0, 0, 0, 4041, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[54, 1, 34, 3, 0, 0, 3, 64, 0, 1948, 2, 4089, 0, 11, 0, 0, 4094, 4096, 4079, 0, 3370, 0, 0, 4088, 2514, 0, 4, 11, 3986, 317, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 2, 0, 3967, 0, 0, 0, 0, 0, 87, 12, 3733, 79, 0, 0, 0, 3942, 125, 1815, 0, 0, 0, 0, 0, 0, 25, 0, 3337, 282, 0, 0, 0, 3628, 70, 0, 0, 0, 0, 0, 3912, 16, 8, 168, 0, 0, 0, 0, 0, 0, 6, 630, 2274, 737, 576, 0, 3272, 1, 48, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3983, 3911, 5882, 3644, 3698, 4104, 4223, 3323] avg=4096.0000 max=5882 max_rank=2 min=3323 min_rank=7 imbalance_ratio=1.436035 max_load_reduction=26886 max_load_reduction_pct=0.820496 moved_to_echo=15348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 409, 0, 3731, 80, 0, 0, 10, 1, 0, 1813, 0, 0, 0, 0, 3503, 0, 3, 0, 3348, 282, 0, 0, 0, 4096, 70, 0, 0, 0, 0, 0, 4096, 0, 6, 169, 0, 0, 0, 0, 361, 0, 0, 0, 2271, 539, 578, 0, 2392, 1, 0, 993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4220, 1824, 7136, 4166, 4271, 3749, 3386] avg=4096.0000 max=7136 max_rank=3 min=1824 min_rank=2 imbalance_ratio=1.742188 max_load_reduction=25632 max_load_reduction_pct=0.782227 moved_to_echo=15572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 75, 0, 0, 4015, 0, 0, 0, 0, 0, 4, 1283, 3742, 81, 0, 0, 0, 154, 1, 1819, 0, 0, 0, 0, 14, 6, 3, 0, 3344, 283, 0, 0, 0, 4093, 69, 0, 0, 0, 0, 0, 4031, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 2277, 710, 577, 0, 3152, 2866, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 5110, 1974, 3650, 4162, 4200, 3564, 6018] avg=4096.0000 max=6018 max_rank=7 min=1974 min_rank=2 imbalance_ratio=1.469238 max_load_reduction=26750 max_load_reduction_pct=0.816345 moved_to_echo=17944 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 0, 0, 4015, 0, 0, 0, 0, 1, 0, 0, 3721, 80, 0, 0, 1, 920, 225, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 3349, 281, 0, 0, 0, 4095, 70, 0, 0, 0, 0, 0, 4040, 0, 91, 169, 0, 0, 0, 0, 1, 0, 0, 0, 2153, 749, 578, 0, 3324, 3172, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 3802, 2866, 3630, 4165, 4300, 3481, 6496] avg=4096.0000 max=6496 max_rank=7 min=2866 min_rank=2 imbalance_ratio=1.585938 max_load_reduction=26272 max_load_reduction_pct=0.801758 moved_to_echo=18442 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 4, 3743, 80, 0, 0, 0, 3517, 0, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 3349, 283, 0, 0, 0, 4096, 70, 0, 0, 0, 0, 0, 4096, 0, 0, 169, 0, 0, 0, 0, 565, 0, 0, 1, 2277, 753, 578, 0, 3343, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 3827, 5336, 3632, 4166, 4265, 4174, 3352] avg=4096.0000 max=5336 max_rank=2 min=3352 min_rank=7 imbalance_ratio=1.302734 max_load_reduction=27432 max_load_reduction_pct=0.837158 moved_to_echo=15957 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 3743, 81, 0, 0, 0, 3862, 0, 1819, 0, 0, 0, 0, 31, 0, 0, 1, 3349, 283, 0, 0, 0, 4096, 70, 0, 0, 0, 0, 0, 4096, 0, 0, 169, 0, 0, 0, 0, 257, 0, 0, 0, 2277, 743, 578, 0, 3298, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 3824, 5681, 3664, 4166, 4265, 3855, 3298] avg=4096.0000 max=5681 max_rank=2 min=3298 min_rank=7 imbalance_ratio=1.386963 max_load_reduction=27087 max_load_reduction_pct=0.826630 moved_to_echo=15584 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[54, 1, 34, 3, 4014, 0, 0, 0, 0, 0, 3, 64, 3733, 80, 0, 0, 0, 1948, 2, 1810, 0, 0, 0, 0, 0, 11, 0, 0, 3349, 286, 0, 0, 0, 4096, 60, 0, 0, 0, 0, 0, 3370, 0, 0, 158, 0, 0, 0, 0, 2514, 0, 4, 11, 2279, 738, 581, 0, 3248, 317, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 3880, 3760, 3646, 4156, 3528, 6127, 3565] avg=4096.0000 max=6127 max_rank=6 min=3528 min_rank=5 imbalance_ratio=1.495850 max_load_reduction=26641 max_load_reduction_pct=0.813019 moved_to_echo=17376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[418, 42, 29, 14, 4011, 0, 0, 0, 76, 0, 0, 340, 3698, 80, 0, 0, 0, 3692, 0, 1753, 0, 0, 0, 0, 205, 27, 0, 1, 3254, 279, 0, 0, 0, 4020, 69, 0, 0, 0, 0, 0, 3671, 0, 0, 164, 0, 0, 0, 0, 82, 5, 1, 0, 2195, 750, 562, 0, 3325, 0, 0, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4514, 4194, 5445, 3766, 4089, 3835, 3595, 3330] avg=4096.0000 max=5445 max_rank=2 min=3330 min_rank=7 imbalance_ratio=1.329346 max_load_reduction=27323 max_load_reduction_pct=0.833832 moved_to_echo=14849 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 18048, 384, 14464, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3840, 128, 128, 768, 18048, 5760, 4608, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[25472, 6400, 128, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 512, 1792, 29952, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3840, 128, 128, 128, 26752, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32256, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 256, 128, 128, 32128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31360, 128, 128, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1, 4096, 3830, 4096, 8, 3, 4042, 4, 0, 0, 10, 0, 0, 1, 22, 0, 448, 1311, 0, 333, 0, 4086, 0, 2, 0, 3505, 2831, 0, 0, 4042, 92, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 251, 4071, 0, 0, 4096, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1711, 0, 225, 1611, 4095, 0, 0, 0, 4096, 10, 0, 4086, 4036, 236, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 2328, 3970, 2213, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 195, 0, 0, 123, 0, 4, 864, 4084, 0, 0, 0, 4084, 1929, 0, 3404, 1353, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[11, 4091, 2975, 4000, 576, 0, 4082, 633, 0, 0, 0, 0, 2, 0, 0, 0, 0, 27, 0, 28, 4001, 4096, 0, 0, 1, 4094, 0, 0, 77, 4070, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 4022, 3694, 2, 0, 4096, 42, 0, 11, 0, 0, 0, 18, 56, 0, 1, 39, 0, 3248, 4095, 4043, 0, 0, 11, 4063, 324, 6, 0, 901, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3877, 3333, 1, 0, 4091, 349, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 4054, 4096, 0, 0, 0, 4096, 896, 0, 7, 3861, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4088, 3459, 4090, 0, 0, 3777, 0, 36, 0, 7, 0, 0, 0, 3, 0, 0, 0, 344, 3872, 224, 4096, 102, 83, 3, 4080, 403, 0, 4, 4091, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[7, 4062, 853, 4078, 966, 0, 4048, 3917, 0, 8, 13, 5, 0, 0, 17, 0, 17, 0, 0, 185, 2382, 4091, 8, 0, 0, 4083, 37, 0, 13, 3959, 19, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3830, 1459, 0, 0, 0, 0, 8, 3, 2986, 4, 0, 0, 0, 0, 0, 0, 10, 0, 4091, 0, 0, 0, 0, 1, 22, 0, 2637, 1529, 5, 0, 448, 1311, 0, 333, 1056, 631, 639, 231, 0, 1918, 0, 2, 0, 0, 0, 0, 0, 2874, 2831, 0, 0, 0, 0, 0, 0, 3811, 92, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5290, 3001, 4101, 4194, 4649, 1920, 5705, 3908] avg=4096.0000 max=5705 max_rank=6 min=1920 min_rank=5 imbalance_ratio=1.392822 max_load_reduction=27063 max_load_reduction_pct=0.825897 moved_to_echo=16182 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 2975, 1424, 0, 0, 0, 0, 576, 0, 3016, 633, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 2, 0, 0, 0, 2576, 1532, 6, 0, 0, 27, 0, 28, 1066, 737, 640, 232, 4001, 1924, 0, 0, 0, 0, 0, 0, 1, 3357, 0, 0, 0, 0, 0, 0, 77, 3838, 0, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4410, 4225, 4085, 4116, 2730, 5925, 3358, 3919] avg=4096.0000 max=5925 max_rank=5 min=2730 min_rank=4 imbalance_ratio=1.446533 max_load_reduction=26843 max_load_reduction_pct=0.819183 moved_to_echo=15932 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2328, 1414, 0, 0, 0, 0, 2213, 0, 3026, 25, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 195, 0, 2556, 1528, 5, 0, 0, 123, 0, 4, 1070, 735, 638, 77, 864, 1918, 0, 0, 0, 0, 0, 0, 0, 3349, 1929, 0, 0, 0, 0, 0, 3404, 1276, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3742, 5264, 4091, 4284, 2647, 2782, 5278, 4680] avg=4096.0000 max=5278 max_rank=6 min=2647 min_rank=4 imbalance_ratio=1.288574 max_load_reduction=27490 max_load_reduction_pct=0.838928 moved_to_echo=15387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 251, 1450, 0, 0, 0, 0, 0, 0, 3026, 148, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 2621, 1532, 6, 0, 0, 1711, 0, 225, 1070, 738, 640, 230, 1611, 1923, 0, 0, 0, 0, 0, 0, 0, 3358, 10, 0, 0, 0, 0, 0, 4086, 3806, 236, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1701, 3174, 4090, 4159, 4614, 3534, 3368, 8128] avg=4096.0000 max=8128 max_rank=7 min=1701 min_rank=0 imbalance_ratio=1.984375 max_load_reduction=24640 max_load_reduction_pct=0.751953 moved_to_echo=19644 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3877, 1187, 0, 0, 0, 0, 1, 0, 3023, 349, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 1, 0, 2146, 1532, 5, 0, 0, 1, 0, 1, 1068, 738, 640, 220, 4054, 1924, 0, 0, 0, 0, 0, 0, 0, 3358, 896, 0, 0, 0, 0, 0, 7, 3641, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5064, 3373, 4091, 3684, 2668, 5978, 4254, 3656] avg=4096.0000 max=5978 max_rank=5 min=2668 min_rank=4 imbalance_ratio=1.459473 max_load_reduction=26790 max_load_reduction_pct=0.817566 moved_to_echo=16556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4022, 1316, 0, 0, 0, 0, 2, 0, 3026, 42, 0, 0, 0, 0, 0, 11, 0, 0, 4090, 0, 0, 0, 0, 18, 56, 0, 2378, 1512, 6, 0, 1, 39, 0, 3248, 1070, 732, 632, 51, 4095, 1899, 0, 0, 0, 0, 0, 0, 11, 3331, 324, 6, 0, 0, 0, 0, 0, 850, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5338, 3070, 4101, 3970, 5773, 5994, 3672, 850] avg=4096.0000 max=5994 max_rank=5 min=850 min_rank=7 imbalance_ratio=1.463379 max_load_reduction=26774 max_load_reduction_pct=0.817078 moved_to_echo=16289 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 853, 1447, 0, 0, 0, 0, 966, 0, 2987, 3917, 0, 0, 0, 0, 0, 8, 13, 5, 4057, 0, 0, 0, 0, 0, 17, 0, 2631, 1535, 5, 0, 17, 0, 0, 185, 1061, 738, 644, 231, 2382, 1912, 8, 0, 0, 0, 0, 0, 0, 3345, 37, 0, 0, 0, 0, 0, 13, 3728, 19, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2307, 7870, 4083, 4188, 2876, 4302, 3382, 3760] avg=4096.0000 max=7870 max_rank=1 min=2307 min_rank=0 imbalance_ratio=1.921387 max_load_reduction=24898 max_load_reduction_pct=0.759827 moved_to_echo=14320 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3459, 1457, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 0, 0, 36, 0, 7, 0, 4083, 0, 0, 0, 0, 0, 3, 0, 2633, 1532, 5, 0, 0, 0, 344, 3872, 986, 735, 640, 234, 224, 1924, 102, 83, 0, 0, 0, 0, 3, 3345, 403, 0, 0, 0, 0, 0, 4, 3857, 0, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4916, 2791, 4126, 4173, 6811, 2333, 3751, 3867] avg=4096.0000 max=6811 max_rank=4 min=2333 min_rank=5 imbalance_ratio=1.662842 max_load_reduction=25957 max_load_reduction_pct=0.792145 moved_to_echo=16762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 3328, 384, 7936, 8448, 5888, 5120, 1536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 384, 0, 20224, 12288, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 21632, 11264, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[17280, 15360, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3840, 128, 23936, 5120, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7680, 24832, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 26368, 6528, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3029, 2, 0, 4032, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 935, 0, 0, 0, 4096, 0, 2, 0, 3, 203, 4096, 4082] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[33, 6, 4094, 0, 1641, 0, 4, 4018, 2686, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 1, 2877, 0, 0, 0, 4081, 0, 0, 2, 15, 1045, 4096, 4070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1, 20, 3982, 32, 1126, 0, 0, 3777, 4096, 72, 0, 1, 8, 0, 7, 0, 4091, 0, 0, 0, 1466, 26, 1, 5, 4065, 0, 7, 0, 188, 1668, 4096, 4033] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 52, 4064, 0, 4066, 0, 0, 3979, 4096, 0, 0, 19, 0, 8, 0, 107, 4091, 31, 0, 0, 17, 0, 68, 0, 4096, 0, 33, 0, 2, 0, 4089, 3950] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[337, 0, 4096, 0, 3810, 0, 0, 3401, 4095, 0, 0, 0, 0, 0, 0, 26, 4070, 0, 0, 6, 900, 0, 0, 0, 4091, 0, 0, 0, 8, 3832, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 1618, 4096, 0, 2216, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 269, 4096, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[3, 0, 4091, 53, 962, 365, 0, 4052, 3616, 0, 0, 0, 9, 1, 0, 0, 4028, 23, 6, 0, 61, 32, 11, 0, 4081, 6, 0, 0, 13, 3202, 4069, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[43, 0, 3213, 2, 4075, 0, 0, 4095, 4056, 3, 0, 0, 0, 0, 2, 0, 4090, 0, 0, 67, 1, 247, 0, 76, 4011, 0, 0, 21, 0, 1148, 4033, 3585] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[33, 6, 3945, 0, 0, 0, 0, 0, 1641, 0, 4, 1466, 0, 0, 0, 0, 2686, 0, 0, 3, 142, 106, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 2877, 0, 0, 0, 2552, 874, 9, 0, 4081, 0, 0, 2, 7, 3, 0, 0, 15, 1045, 0, 3090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3984, 3111, 2937, 4087, 4094, 6312, 4093, 4150] avg=4096.0000 max=6312 max_rank=5 min=2937 min_rank=2 imbalance_ratio=1.541016 max_load_reduction=26456 max_load_reduction_pct=0.807373 moved_to_echo=18649 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 20, 3837, 32, 0, 0, 0, 0, 1126, 0, 0, 1378, 0, 0, 0, 0, 4096, 72, 0, 1, 138, 105, 0, 0, 8, 0, 7, 0, 4087, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 1466, 26, 1, 5, 2399, 866, 9, 0, 4065, 0, 7, 0, 7, 3, 0, 0, 188, 1668, 0, 3062, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3890, 2504, 4412, 4102, 4088, 4772, 4082, 4918] avg=4096.0000 max=4918 max_rank=7 min=2504 min_rank=1 imbalance_ratio=1.200684 max_load_reduction=27850 max_load_reduction_pct=0.849915 moved_to_echo=17860 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3947, 0, 0, 0, 0, 0, 3029, 2, 0, 1471, 0, 0, 0, 0, 4096, 0, 0, 0, 142, 106, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 935, 0, 0, 0, 2561, 877, 9, 0, 4096, 0, 2, 0, 7, 3, 0, 0, 3, 203, 0, 3099, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3947, 4502, 4344, 4087, 4093, 4382, 4108, 3305] avg=4096.0000 max=4502 max_rank=1 min=3305 min_rank=7 imbalance_ratio=1.099121 max_load_reduction=28266 max_load_reduction_pct=0.862610 moved_to_echo=15888 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 52, 3916, 0, 0, 0, 0, 0, 4066, 0, 0, 1452, 0, 0, 0, 0, 4096, 0, 0, 19, 141, 103, 0, 0, 0, 8, 0, 107, 4080, 0, 0, 0, 4088, 31, 0, 0, 0, 0, 0, 0, 17, 0, 68, 0, 2527, 848, 9, 0, 4096, 0, 33, 0, 7, 3, 0, 0, 2, 0, 0, 2999, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3968, 5518, 4359, 4195, 4119, 3469, 4139, 3001] avg=4096.0000 max=5518 max_rank=1 min=3001 min_rank=7 imbalance_ratio=1.347168 max_load_reduction=27250 max_load_reduction_pct=0.831604 moved_to_echo=14728 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[337, 0, 3947, 0, 0, 0, 0, 0, 3810, 0, 0, 1241, 0, 0, 0, 0, 4095, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 26, 4087, 0, 0, 0, 4068, 0, 0, 6, 0, 0, 0, 0, 900, 0, 0, 0, 2160, 0, 9, 0, 4091, 0, 0, 0, 7, 2, 0, 0, 8, 3832, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4284, 5051, 4237, 4113, 4074, 3069, 4100, 3840] avg=4096.0000 max=5051 max_rank=1 min=3069 min_rank=5 imbalance_ratio=1.233154 max_load_reduction=27717 max_load_reduction_pct=0.845856 moved_to_echo=15083 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1618, 3947, 0, 0, 0, 0, 0, 2216, 1, 0, 1494, 0, 0, 0, 0, 4096, 0, 0, 0, 142, 106, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2602, 879, 9, 0, 4093, 0, 0, 0, 7, 3, 0, 0, 0, 269, 0, 3106, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5565, 3711, 4344, 4087, 4093, 3490, 4103, 3375] avg=4096.0000 max=5565 max_rank=0 min=3375 min_rank=7 imbalance_ratio=1.358643 max_load_reduction=27203 max_load_reduction_pct=0.830170 moved_to_echo=15061 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3932, 53, 0, 0, 0, 0, 962, 365, 0, 1475, 0, 0, 0, 0, 3616, 0, 0, 0, 147, 109, 0, 0, 9, 1, 0, 0, 4061, 0, 0, 0, 4024, 23, 6, 0, 0, 0, 0, 0, 61, 32, 11, 0, 2577, 881, 8, 0, 4081, 6, 0, 0, 12, 4, 0, 0, 13, 3202, 0, 3094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3988, 2802, 3872, 4071, 4053, 3570, 4103, 6309] avg=4096.0000 max=6309 max_rank=7 min=2802 min_rank=1 imbalance_ratio=1.540283 max_load_reduction=26459 max_load_reduction_pct=0.807465 moved_to_echo=18035 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[43, 0, 3097, 2, 0, 0, 0, 0, 4075, 0, 0, 1494, 0, 0, 0, 0, 4056, 3, 0, 0, 111, 93, 0, 0, 0, 0, 2, 0, 4024, 0, 0, 0, 4087, 0, 0, 67, 0, 0, 0, 0, 1, 247, 0, 76, 2601, 770, 9, 0, 4011, 0, 0, 21, 5, 3, 0, 0, 0, 1148, 0, 2722, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3142, 5569, 4263, 4026, 4154, 3704, 4040, 3870] avg=4096.0000 max=5569 max_rank=1 min=3142 min_rank=0 imbalance_ratio=1.359619 max_load_reduction=27199 max_load_reduction_pct=0.830048 moved_to_echo=15768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6272, 384, 128, 128, 20096, 6016, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[20992, 384, 128, 11520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 11392, 0, 21248, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 1792, 30592, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[30848, 128, 0, 128, 1152, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 256, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[67, 3131, 1468, 41, 2250, 177, 1045, 703, 1014, 125, 1536, 674, 691, 495, 267, 380, 269, 625, 1623, 2592, 1889, 240, 2651, 1337, 113, 1613, 561, 1309, 1004, 308, 1304, 1266] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[49, 2500, 630, 46, 2270, 115, 1075, 298, 1347, 106, 1881, 833, 803, 1210, 47, 428, 202, 1044, 1117, 2212, 1940, 264, 2625, 1538, 34, 1383, 417, 1372, 1018, 355, 1824, 1785] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[94, 2989, 796, 287, 2215, 216, 1231, 648, 1312, 96, 2181, 312, 721, 765, 270, 629, 223, 731, 1639, 2033, 1748, 530, 2039, 1195, 194, 1390, 829, 643, 1133, 317, 1825, 1537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[50, 3155, 965, 80, 1767, 91, 938, 570, 1356, 141, 1925, 367, 617, 865, 108, 357, 233, 559, 2108, 1762, 2061, 418, 2289, 1287, 46, 1671, 559, 1181, 1432, 471, 1942, 1397] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[193, 3413, 1033, 147, 2053, 68, 811, 486, 1463, 102, 2254, 337, 796, 984, 109, 483, 429, 1171, 1097, 2167, 1741, 211, 2506, 1092, 66, 1175, 606, 1572, 1119, 178, 1301, 1605] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[150, 2921, 616, 108, 2520, 165, 895, 854, 1278, 113, 2106, 722, 618, 695, 121, 491, 382, 1054, 1378, 2493, 1968, 309, 3004, 1018, 55, 1233, 633, 896, 843, 270, 1493, 1366] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[45, 3311, 652, 180, 1408, 236, 902, 355, 901, 304, 1598, 843, 754, 930, 138, 665, 505, 741, 1892, 1901, 2481, 127, 2503, 1375, 22, 1904, 520, 1324, 1093, 388, 1183, 1587] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[101, 2865, 781, 115, 2676, 92, 839, 642, 1087, 112, 2251, 515, 919, 909, 221, 327, 256, 578, 1485, 1999, 1710, 398, 2580, 1343, 135, 1144, 503, 1309, 1029, 486, 2110, 1251] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[49, 2479, 630, 46, 0, 0, 0, 0, 2270, 115, 1075, 298, 293, 9, 0, 0, 1347, 106, 1881, 833, 191, 21, 0, 0, 803, 1210, 47, 428, 1943, 0, 0, 0, 202, 1044, 1117, 1474, 0, 0, 0, 0, 1940, 264, 592, 1538, 0, 0, 0, 0, 34, 1383, 417, 1372, 729, 90, 0, 0, 1018, 355, 1340, 1785, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3204, 4060, 4379, 4431, 3837, 4334, 4025, 4498] avg=4096.0000 max=4498 max_rank=7 min=3204 min_rank=0 imbalance_ratio=1.098145 max_load_reduction=28270 max_load_reduction_pct=0.862732 moved_to_echo=16694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[67, 3104, 1468, 41, 0, 0, 0, 0, 2250, 177, 1045, 703, 209, 10, 0, 0, 1014, 125, 1536, 674, 137, 27, 0, 0, 691, 495, 267, 380, 1962, 0, 0, 0, 269, 625, 1623, 1727, 0, 0, 0, 0, 1889, 240, 598, 1337, 0, 0, 0, 0, 113, 1613, 561, 1309, 855, 91, 0, 0, 1004, 308, 958, 1266, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4680, 4394, 3513, 3795, 4244, 4064, 4542, 3536] avg=4096.0000 max=4680 max_rank=0 min=3513 min_rank=2 imbalance_ratio=1.142578 max_load_reduction=28088 max_load_reduction_pct=0.857178 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[150, 2896, 616, 108, 0, 0, 0, 0, 2520, 165, 895, 854, 240, 10, 0, 0, 1278, 113, 2106, 722, 156, 25, 0, 0, 618, 695, 121, 491, 2224, 0, 0, 0, 382, 1054, 1378, 1661, 0, 0, 0, 0, 1968, 309, 677, 1018, 0, 0, 0, 0, 55, 1233, 633, 896, 822, 103, 0, 0, 843, 270, 1097, 1366, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3770, 4684, 4400, 4149, 4475, 3972, 3742, 3576] avg=4096.0000 max=4684 max_rank=1 min=3576 min_rank=7 imbalance_ratio=1.143555 max_load_reduction=28084 max_load_reduction_pct=0.857056 moved_to_echo=15765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[50, 3128, 965, 80, 0, 0, 0, 0, 1767, 91, 938, 570, 312, 7, 0, 0, 1356, 141, 1925, 367, 204, 27, 0, 0, 617, 865, 108, 357, 1694, 0, 0, 0, 233, 559, 2108, 1174, 0, 0, 0, 0, 2061, 418, 517, 1287, 0, 0, 0, 0, 46, 1671, 559, 1181, 581, 78, 0, 0, 1432, 471, 1426, 1397, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4223, 3685, 4020, 3641, 4074, 4283, 4116, 4726] avg=4096.0000 max=4726 max_rank=7 min=3641 min_rank=3 imbalance_ratio=1.153809 max_load_reduction=28042 max_load_reduction_pct=0.855774 moved_to_echo=17199 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[193, 3384, 1033, 147, 0, 0, 0, 0, 2053, 68, 811, 486, 209, 8, 0, 0, 1463, 102, 2254, 337, 136, 29, 0, 0, 796, 984, 109, 483, 1855, 0, 0, 0, 429, 1171, 1097, 1444, 0, 0, 0, 0, 1741, 211, 565, 1092, 0, 0, 0, 0, 66, 1175, 606, 1572, 715, 86, 0, 0, 1119, 178, 956, 1605, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4757, 3635, 4321, 4227, 4141, 3609, 4220, 3858] avg=4096.0000 max=4757 max_rank=0 min=3609 min_rank=5 imbalance_ratio=1.161377 max_load_reduction=28011 max_load_reduction_pct=0.854828 moved_to_echo=15828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[45, 3278, 652, 180, 0, 0, 0, 0, 1408, 236, 902, 355, 192, 10, 0, 0, 901, 304, 1598, 843, 129, 33, 0, 0, 754, 930, 138, 665, 1858, 0, 0, 0, 505, 741, 1892, 1260, 0, 0, 0, 0, 2481, 127, 557, 1375, 0, 0, 0, 0, 22, 1904, 520, 1324, 631, 88, 0, 0, 1093, 388, 862, 1587, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 3103, 3808, 4345, 4398, 4540, 4489, 3930] avg=4096.0000 max=4540 max_rank=5 min=3103 min_rank=1 imbalance_ratio=1.108398 max_load_reduction=28228 max_load_reduction_pct=0.861450 moved_to_echo=17357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[94, 2964, 796, 287, 0, 0, 0, 0, 2215, 216, 1231, 648, 293, 8, 0, 0, 1312, 96, 2181, 312, 191, 25, 0, 0, 721, 765, 270, 629, 1509, 0, 0, 0, 223, 731, 1639, 1355, 0, 0, 0, 0, 1748, 530, 460, 1195, 0, 0, 0, 0, 194, 1390, 829, 643, 670, 70, 0, 0, 1133, 317, 1341, 1537, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4611, 4117, 3894, 3948, 3933, 3796, 4328] avg=4096.0000 max=4611 max_rank=1 min=3796 min_rank=6 imbalance_ratio=1.125732 max_load_reduction=28157 max_load_reduction_pct=0.859283 moved_to_echo=16005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[101, 2841, 781, 115, 0, 0, 0, 0, 2676, 92, 839, 642, 339, 8, 0, 0, 1087, 112, 2251, 515, 221, 24, 0, 0, 919, 909, 221, 327, 1910, 0, 0, 0, 256, 578, 1485, 1332, 0, 0, 0, 0, 1710, 398, 582, 1343, 0, 0, 0, 0, 135, 1144, 503, 1309, 659, 88, 0, 0, 1029, 486, 1550, 1251, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3838, 4596, 4210, 4286, 3651, 4033, 3838, 4316] avg=4096.0000 max=4596 max_rank=1 min=3651 min_rank=4 imbalance_ratio=1.122070 max_load_reduction=28172 max_load_reduction_pct=0.859741 moved_to_echo=15838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6016, 6912, 1408, 3840, 14976, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8704, 2816, 9600, 11904, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15616, 2560, 4608, 10240, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[17280, 1280, 7808, 4608, 2176, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2560, 6528, 12416, 11520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 11520, 4736, 9728, 5760, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 24192, 7040, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9856, 1152, 15744, 4608, 1408, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 8, 4066, 55, 0, 3, 889, 0, 0, 0, 29, 0, 0, 1, 0, 1, 1114, 0, 2, 1, 3860, 0, 2616, 57, 3656, 4058, 3943, 159, 4087, 0, 539, 3624] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[96, 223, 3709, 0, 2, 0, 3571, 1, 0, 0, 8, 1, 6, 0, 2, 8, 171, 4, 30, 0, 3769, 6, 359, 0, 4094, 3221, 4059, 301, 4080, 0, 1760, 3287] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[494, 1589, 3919, 129, 4, 46, 618, 136, 2, 1, 0, 25, 0, 0, 0, 21, 26, 0, 1, 0, 2571, 21, 1665, 2, 3593, 1809, 3945, 450, 4082, 101, 3853, 3665] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4, 2508, 4052, 3, 12, 1, 1759, 70, 3, 1, 11, 26, 3, 20, 1, 0, 798, 0, 11, 3, 1190, 30, 2448, 30, 3506, 666, 4082, 390, 4088, 5, 3223, 3824] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[363, 96, 4067, 21, 283, 0, 2911, 62, 2, 0, 0, 23, 91, 0, 3, 50, 33, 0, 5, 31, 3513, 0, 2359, 17, 3889, 3442, 2711, 410, 3914, 24, 1185, 3263] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[83, 1625, 4078, 569, 0, 0, 3159, 0, 2, 0, 1, 11, 0, 0, 1, 0, 528, 2, 0, 26, 3274, 0, 1030, 50, 4074, 1734, 4056, 208, 4077, 3, 2444, 1733] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 1407, 4052, 5, 0, 0, 553, 579, 0, 0, 7, 0, 0, 21, 16, 6, 458, 3, 3, 6, 502, 0, 3755, 169, 4030, 3351, 4093, 1348, 4035, 1, 2155, 2213] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[12, 993, 3881, 281, 3, 0, 340, 2, 44, 5, 24, 42, 0, 10, 0, 0, 419, 0, 5, 0, 1393, 0, 1198, 84, 4067, 3097, 3701, 2179, 4093, 0, 3016, 3879] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 2837, 55, 0, 0, 0, 0, 0, 3, 889, 0, 1229, 702, 615, 0, 0, 0, 29, 0, 3649, 211, 0, 0, 0, 1, 0, 1, 4087, 7, 0, 0, 1114, 0, 2, 1, 2965, 865, 0, 0, 3245, 0, 2616, 57, 0, 0, 0, 0, 0, 4058, 767, 159, 0, 0, 0, 0, 0, 0, 539, 2057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2900, 3438, 3889, 4096, 4947, 5918, 4984, 2596] avg=4096.0000 max=5918 max_rank=5 min=2596 min_rank=7 imbalance_ratio=1.444824 max_load_reduction=26850 max_load_reduction_pct=0.819397 moved_to_echo=18445 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[96, 223, 2588, 0, 0, 0, 0, 0, 2, 0, 3571, 1, 1121, 637, 600, 0, 0, 0, 8, 1, 4087, 217, 0, 0, 6, 0, 2, 8, 4080, 7, 0, 0, 171, 4, 30, 0, 3052, 784, 0, 0, 3169, 6, 359, 0, 0, 0, 0, 0, 0, 3221, 790, 301, 0, 0, 0, 0, 0, 0, 1760, 1866, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2907, 5932, 4313, 4103, 4041, 3534, 4312, 3626] avg=4096.0000 max=5932 max_rank=1 min=2907 min_rank=0 imbalance_ratio=1.448242 max_load_reduction=26836 max_load_reduction_pct=0.818970 moved_to_echo=15513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[494, 1589, 2735, 129, 0, 0, 0, 0, 4, 46, 618, 136, 1184, 710, 409, 0, 2, 1, 0, 25, 3587, 211, 0, 0, 0, 0, 0, 21, 4082, 6, 0, 0, 26, 0, 1, 0, 2966, 874, 0, 0, 2162, 21, 1665, 2, 0, 0, 0, 0, 0, 1809, 768, 450, 0, 0, 0, 0, 0, 101, 3853, 2081, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4947, 3107, 3826, 4109, 3867, 3850, 3027, 6035] avg=4096.0000 max=6035 max_rank=7 min=3027 min_rank=6 imbalance_ratio=1.473389 max_load_reduction=26733 max_load_reduction_pct=0.815826 moved_to_echo=16779 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 2508, 2828, 3, 0, 0, 0, 0, 12, 1, 1759, 70, 1224, 741, 189, 0, 3, 1, 11, 26, 3500, 219, 0, 0, 3, 20, 1, 0, 4088, 6, 0, 0, 798, 0, 11, 3, 3069, 912, 0, 0, 1001, 30, 2448, 30, 0, 0, 0, 0, 0, 666, 794, 390, 0, 0, 0, 0, 0, 5, 3223, 2171, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5343, 3996, 3760, 4118, 4793, 3509, 1850, 5399] avg=4096.0000 max=5399 max_rank=7 min=1850 min_rank=6 imbalance_ratio=1.318115 max_load_reduction=27369 max_load_reduction_pct=0.835236 moved_to_echo=15551 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1407, 2828, 5, 0, 0, 0, 0, 0, 0, 553, 579, 1224, 429, 80, 0, 0, 0, 7, 0, 4024, 219, 0, 0, 0, 21, 16, 6, 4035, 6, 0, 0, 458, 3, 3, 6, 3077, 528, 0, 0, 422, 0, 3755, 169, 0, 0, 0, 0, 0, 3351, 797, 1348, 0, 0, 0, 0, 0, 1, 2155, 1256, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4240, 2865, 4250, 4084, 4075, 4346, 5496, 3412] avg=4096.0000 max=5496 max_rank=6 min=2865 min_rank=1 imbalance_ratio=1.341797 max_load_reduction=27272 max_load_reduction_pct=0.832275 moved_to_echo=17329 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[363, 96, 2834, 21, 0, 0, 0, 0, 283, 0, 2911, 62, 1233, 637, 564, 0, 2, 0, 0, 23, 3883, 150, 0, 0, 91, 0, 3, 50, 3914, 6, 0, 0, 33, 0, 5, 31, 2041, 783, 0, 0, 2949, 0, 2359, 17, 0, 0, 0, 0, 0, 3442, 520, 410, 0, 0, 0, 0, 0, 24, 1185, 1843, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3314, 5690, 4058, 4064, 2893, 5325, 4372, 3052] avg=4096.0000 max=5690 max_rank=1 min=2893 min_rank=4 imbalance_ratio=1.389160 max_load_reduction=27078 max_load_reduction_pct=0.826355 moved_to_echo=15642 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[12, 993, 2708, 281, 0, 0, 0, 0, 3, 0, 340, 2, 1173, 752, 222, 0, 44, 5, 24, 42, 4060, 198, 0, 0, 0, 10, 0, 0, 4093, 7, 0, 0, 419, 0, 5, 0, 2783, 926, 0, 0, 1171, 0, 1198, 84, 0, 0, 0, 0, 0, 3097, 720, 2179, 0, 0, 0, 0, 0, 0, 3016, 2201, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 2492, 4373, 4110, 4133, 2453, 5996, 5217] avg=4096.0000 max=5996 max_rank=6 min=2453 min_rank=5 imbalance_ratio=1.463867 max_load_reduction=26772 max_load_reduction_pct=0.817017 moved_to_echo=17799 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[83, 1625, 2846, 569, 0, 0, 0, 0, 0, 0, 3159, 0, 1232, 336, 521, 0, 2, 0, 1, 11, 4068, 217, 0, 0, 0, 0, 1, 0, 4077, 6, 0, 0, 528, 2, 0, 26, 3050, 413, 0, 0, 2753, 0, 1030, 50, 0, 0, 0, 0, 0, 1734, 789, 208, 0, 0, 0, 0, 0, 3, 2444, 984, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5123, 5248, 4299, 4084, 4019, 3833, 2731, 3431] avg=4096.0000 max=5248 max_rank=1 min=2731 min_rank=6 imbalance_ratio=1.281250 max_load_reduction=27520 max_load_reduction_pct=0.839844 moved_to_echo=14014 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 13824, 896, 9728, 4992, 3200, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 21504, 6016, 5504, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3584, 128, 128, 128, 23040, 6144, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 256, 18176, 14464, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16896, 128, 15488, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1152, 8576, 22272, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 30976, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[5, 4091, 0, 1, 0, 93, 9, 3707, 0, 3761, 0, 4, 986, 96, 4091, 0, 42, 49, 0, 3766, 526, 4096, 0, 803, 4093, 7, 2487, 0, 53, 1, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 1754, 0, 3, 0, 0, 0, 3751, 6, 4096, 7, 1, 818, 1364, 3863, 0, 0, 35, 34, 4091, 0, 3878, 2, 3868, 4072, 0, 1024, 0, 0, 10, 0, 91] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[18, 3644, 1, 31, 5, 18, 0, 3983, 4, 4000, 0, 0, 5, 198, 4080, 0, 1, 0, 0, 4087, 22, 4081, 2, 20, 4078, 2, 4096, 146, 64, 59, 100, 23] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[48, 3559, 17, 48, 231, 2, 2, 3268, 0, 3978, 0, 23, 81, 0, 4028, 24, 0, 25, 3, 1898, 2499, 4095, 0, 3346, 4094, 2, 69, 606, 5, 817, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[146, 4073, 2, 3, 1, 1, 0, 4092, 2, 4048, 0, 3, 86, 3604, 3969, 0, 0, 30, 96, 4096, 54, 4090, 0, 8, 4094, 1, 224, 0, 5, 39, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[269, 4091, 0, 0, 0, 29, 208, 3675, 15, 4056, 0, 22, 15, 58, 3968, 4, 0, 1, 3, 3914, 237, 4090, 0, 3583, 4093, 6, 229, 4, 0, 142, 50, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[8, 4021, 0, 233, 0, 481, 52, 3957, 3, 4078, 0, 4, 33, 806, 3285, 0, 516, 0, 0, 3593, 4, 4015, 29, 60, 4095, 0, 3332, 0, 0, 7, 0, 156] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[619, 4064, 0, 0, 0, 1, 0, 4021, 0, 4060, 1, 3, 42, 1891, 3813, 0, 0, 20, 2, 3917, 5, 4096, 0, 202, 4094, 0, 1468, 0, 75, 369, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 4091, 0, 1, 265, 0, 0, 0, 0, 93, 9, 3707, 155, 0, 0, 0, 0, 3761, 0, 4, 78, 0, 0, 0, 986, 96, 2982, 0, 0, 0, 0, 0, 42, 49, 0, 3766, 335, 0, 0, 0, 526, 2210, 0, 803, 0, 0, 0, 0, 2386, 7, 2487, 0, 0, 0, 0, 0, 53, 1, 1, 0, 1886, 1707, 276, 0] rerouted_tokens_per_ep_rank=[4362, 3964, 3843, 4064, 4192, 3539, 4880, 3924] avg=4096.0000 max=4880 max_rank=6 min=3539 min_rank=5 imbalance_ratio=1.191406 max_load_reduction=27888 max_load_reduction_pct=0.851074 moved_to_echo=16535 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1754, 0, 3, 250, 0, 0, 0, 0, 0, 0, 3751, 146, 0, 0, 0, 6, 4096, 7, 1, 73, 0, 0, 0, 818, 1364, 2818, 0, 0, 0, 0, 0, 0, 35, 34, 4091, 316, 0, 0, 0, 0, 2092, 2, 3868, 0, 0, 0, 0, 2374, 0, 1024, 0, 0, 0, 0, 0, 0, 10, 0, 91, 1786, 1698, 260, 0] rerouted_tokens_per_ep_rank=[2007, 3897, 4183, 5000, 4476, 5962, 3398, 3845] avg=4096.0000 max=5962 max_rank=5 min=2007 min_rank=0 imbalance_ratio=1.455566 max_load_reduction=26806 max_load_reduction_pct=0.818054 moved_to_echo=17681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[269, 4091, 0, 0, 257, 0, 0, 0, 0, 29, 208, 3675, 150, 0, 0, 0, 15, 4056, 0, 22, 75, 0, 0, 0, 15, 58, 2894, 4, 0, 0, 0, 0, 0, 1, 3, 3914, 325, 0, 0, 0, 237, 2207, 0, 3583, 0, 0, 0, 0, 2386, 6, 229, 4, 0, 0, 0, 0, 0, 142, 50, 6, 1883, 1707, 267, 0] rerouted_tokens_per_ep_rank=[4617, 4062, 4168, 2971, 4243, 6027, 2625, 4055] avg=4096.0000 max=6027 max_rank=5 min=2625 min_rank=6 imbalance_ratio=1.471436 max_load_reduction=26741 max_load_reduction_pct=0.816071 moved_to_echo=16950 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[18, 3644, 1, 31, 264, 0, 0, 0, 5, 18, 0, 3983, 154, 0, 0, 0, 4, 4000, 0, 0, 77, 0, 0, 0, 5, 198, 2976, 0, 0, 0, 0, 0, 1, 0, 0, 4087, 334, 0, 0, 0, 22, 2202, 2, 20, 0, 0, 0, 0, 2377, 2, 4096, 146, 0, 0, 0, 0, 64, 59, 100, 23, 1879, 1701, 275, 0] rerouted_tokens_per_ep_rank=[3958, 4160, 4081, 3179, 4422, 2246, 6621, 4101] avg=4096.0000 max=6621 max_rank=6 min=2246 min_rank=5 imbalance_ratio=1.616455 max_load_reduction=26147 max_load_reduction_pct=0.797943 moved_to_echo=17390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[146, 4073, 2, 3, 257, 0, 0, 0, 1, 1, 0, 4092, 150, 0, 0, 0, 2, 4048, 0, 3, 75, 0, 0, 0, 86, 3604, 2895, 0, 0, 0, 0, 0, 0, 30, 96, 4096, 325, 0, 0, 0, 54, 2207, 0, 8, 0, 0, 0, 0, 2386, 1, 224, 0, 0, 0, 0, 0, 5, 39, 0, 1, 1883, 1708, 267, 0] rerouted_tokens_per_ep_rank=[4481, 4244, 4128, 6585, 4547, 2269, 2611, 3903] avg=4096.0000 max=6585 max_rank=3 min=2269 min_rank=5 imbalance_ratio=1.607666 max_load_reduction=26183 max_load_reduction_pct=0.799042 moved_to_echo=13330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[619, 4064, 0, 0, 247, 0, 0, 0, 0, 1, 0, 4021, 144, 0, 0, 0, 0, 4060, 1, 3, 72, 0, 0, 0, 42, 1891, 2781, 0, 0, 0, 0, 0, 0, 20, 2, 3917, 312, 0, 0, 0, 5, 2210, 0, 202, 0, 0, 0, 0, 2386, 0, 1468, 0, 0, 0, 0, 0, 75, 369, 0, 5, 1886, 1708, 257, 0] rerouted_tokens_per_ep_rank=[4930, 4166, 4136, 4714, 4251, 2417, 3854, 4300] avg=4096.0000 max=4930 max_rank=0 min=2417 min_rank=5 imbalance_ratio=1.203613 max_load_reduction=27838 max_load_reduction_pct=0.849548 moved_to_echo=14822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[48, 3559, 17, 48, 266, 0, 0, 0, 231, 2, 2, 3268, 158, 0, 0, 0, 0, 3978, 0, 23, 81, 0, 0, 0, 81, 0, 2915, 24, 0, 0, 0, 0, 0, 25, 3, 1898, 333, 0, 0, 0, 2499, 2204, 0, 3346, 0, 0, 0, 0, 2383, 2, 69, 606, 0, 0, 0, 0, 5, 817, 0, 0, 1891, 1711, 275, 0] rerouted_tokens_per_ep_rank=[3938, 3661, 4082, 3020, 2259, 8049, 3060, 4699] avg=4096.0000 max=8049 max_rank=5 min=2259 min_rank=4 imbalance_ratio=1.965088 max_load_reduction=24719 max_load_reduction_pct=0.754364 moved_to_echo=18067 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8, 4021, 0, 233, 213, 0, 0, 0, 0, 481, 52, 3957, 124, 0, 0, 0, 3, 4078, 0, 4, 62, 0, 0, 0, 33, 806, 2396, 0, 0, 0, 0, 0, 516, 0, 0, 3593, 269, 0, 0, 0, 4, 2166, 29, 60, 0, 0, 0, 0, 2387, 0, 3332, 0, 0, 0, 0, 0, 0, 7, 0, 156, 1849, 1708, 221, 0] rerouted_tokens_per_ep_rank=[4475, 4614, 4147, 3235, 4378, 2259, 5719, 3941] avg=4096.0000 max=5719 max_rank=6 min=2259 min_rank=5 imbalance_ratio=1.396240 max_load_reduction=27049 max_load_reduction_pct=0.825470 moved_to_echo=16297 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 1536, 256, 384, 14976, 13696, 2176, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3456, 17536, 128, 11904, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2176, 8064, 22784, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[19072, 128, 13056, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 256, 256, 29440, 2560, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 29312, 128, 384, 2048, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32128, 128, 128, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 640, 384, 30464, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 1280, 0, 4096, 85, 0, 2183, 2684, 2291, 9, 0, 4072, 3873, 0, 24, 0, 2, 0, 2, 129, 0, 0, 0, 0, 4095, 0, 0, 0, 4090, 2873, 1, 979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 3961, 42, 4049, 0, 16, 2930, 2620, 46, 0, 31, 4015, 1931, 0, 787, 0, 111, 0, 1, 0, 0, 0, 0, 4, 4088, 0, 0, 0, 4005, 3125, 901, 105] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2, 1, 0, 4075, 28, 14, 4030, 1525, 247, 2114, 45, 4087, 2931, 0, 964, 1, 3, 0, 249, 0, 18, 0, 0, 22, 4052, 1, 1, 0, 4088, 3992, 0, 278] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[5, 34, 8, 4028, 6, 0, 303, 2055, 137, 442, 182, 4088, 3933, 0, 3500, 6, 0, 0, 0, 10, 208, 3, 0, 17, 3944, 0, 2, 0, 4077, 4036, 130, 1614] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 1736, 0, 4096, 1721, 0, 942, 50, 0, 1501, 0, 3987, 799, 0, 4063, 0, 0, 4, 1, 0, 3, 0, 0, 11, 4086, 0, 0, 0, 4078, 3862, 246, 1582] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 738, 0, 4042, 0, 0, 3419, 25, 2323, 44, 251, 4086, 3170, 1, 2782, 0, 1, 0, 4, 8, 0, 2, 1, 0, 3278, 254, 0, 1, 4095, 4089, 0, 154] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 2424, 8, 4096, 2076, 0, 0, 2121, 3, 1, 844, 4079, 4058, 0, 682, 0, 9, 0, 2, 34, 1, 2, 0, 0, 4092, 33, 0, 0, 4091, 4096, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 2960, 44, 3594, 2430, 482, 648, 114, 0, 4, 7, 4087, 3803, 0, 16, 24, 0, 0, 0, 296, 0, 194, 93, 0, 2268, 0, 5, 1, 3804, 4038, 15, 3841] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1736, 0, 2494, 0, 0, 0, 0, 1721, 0, 942, 50, 8, 0, 0, 0, 0, 1501, 0, 2729, 0, 0, 0, 0, 651, 0, 4063, 0, 0, 0, 0, 0, 0, 4, 1, 0, 1602, 1258, 924, 57, 3, 0, 0, 11, 4060, 0, 0, 0, 4086, 0, 0, 0, 83, 0, 0, 0, 18, 2938, 246, 1582, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4230, 2721, 4230, 4714, 3846, 4074, 4169, 4784] avg=4096.0000 max=4784 max_rank=7 min=2721 min_rank=1 imbalance_ratio=1.167969 max_load_reduction=27984 max_load_reduction_pct=0.854004 moved_to_echo=16873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[5, 34, 8, 2452, 0, 0, 0, 0, 6, 0, 303, 2055, 41, 0, 0, 0, 137, 442, 182, 2799, 0, 0, 0, 0, 3199, 0, 3500, 6, 0, 0, 0, 0, 0, 0, 0, 10, 1576, 1289, 965, 281, 208, 3, 0, 17, 4059, 0, 0, 0, 3944, 0, 2, 0, 412, 0, 0, 0, 18, 3071, 130, 1614, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2499, 2405, 3560, 6705, 4121, 4287, 4358, 4833] avg=4096.0000 max=6705 max_rank=3 min=2405 min_rank=1 imbalance_ratio=1.636963 max_load_reduction=26063 max_load_reduction_pct=0.795380 moved_to_echo=17599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1280, 0, 2494, 0, 0, 0, 0, 85, 0, 2183, 2684, 41, 0, 0, 0, 2291, 9, 0, 2788, 0, 0, 0, 0, 3151, 0, 24, 0, 0, 0, 0, 0, 2, 0, 2, 129, 1602, 1284, 687, 276, 0, 0, 0, 0, 4072, 0, 0, 0, 4095, 0, 0, 0, 405, 0, 0, 0, 18, 2186, 1, 979, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3774, 4993, 5088, 3175, 3982, 4072, 4500, 3184] avg=4096.0000 max=5088 max_rank=2 min=3175 min_rank=3 imbalance_ratio=1.242188 max_load_reduction=27680 max_load_reduction_pct=0.844727 moved_to_echo=15738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3961, 42, 2465, 0, 0, 0, 0, 0, 16, 2930, 2620, 20, 0, 0, 0, 46, 0, 31, 2749, 0, 0, 0, 0, 1571, 0, 787, 0, 0, 0, 0, 0, 111, 0, 1, 0, 1584, 1266, 747, 138, 0, 0, 0, 4, 3987, 0, 0, 0, 4088, 0, 0, 0, 202, 0, 0, 0, 18, 2378, 901, 105, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6468, 5586, 2826, 2358, 3847, 3991, 4290, 3402] avg=4096.0000 max=6468 max_rank=0 min=2358 min_rank=3 imbalance_ratio=1.579102 max_load_reduction=26300 max_load_reduction_pct=0.802612 moved_to_echo=15530 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 2481, 0, 0, 0, 0, 28, 14, 4030, 1525, 31, 0, 0, 0, 247, 2114, 45, 2798, 0, 0, 0, 0, 2384, 0, 964, 1, 0, 0, 0, 0, 3, 0, 249, 0, 1594, 1289, 955, 209, 18, 0, 0, 22, 4070, 0, 0, 0, 4052, 1, 1, 0, 307, 0, 0, 0, 18, 3037, 0, 278, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2484, 5628, 5204, 3349, 4299, 4110, 4361, 3333] avg=4096.0000 max=5628 max_rank=1 min=2484 min_rank=0 imbalance_ratio=1.374023 max_load_reduction=27140 max_load_reduction_pct=0.828247 moved_to_echo=16103 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 738, 0, 2461, 0, 0, 0, 0, 0, 0, 3419, 25, 33, 0, 0, 0, 2323, 44, 251, 2797, 0, 0, 0, 0, 2579, 1, 2782, 0, 0, 0, 0, 0, 1, 0, 4, 8, 1581, 1289, 978, 226, 0, 2, 1, 0, 4077, 0, 0, 0, 3278, 254, 0, 1, 332, 0, 0, 0, 18, 3111, 0, 154, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3199, 3477, 5415, 5362, 4087, 4080, 3865, 3283] avg=4096.0000 max=5415 max_rank=2 min=3199 min_rank=0 imbalance_ratio=1.322021 max_load_reduction=27353 max_load_reduction_pct=0.834747 moved_to_echo=15315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2960, 44, 2184, 0, 0, 0, 0, 2430, 482, 648, 114, 44, 0, 0, 0, 0, 4, 7, 2794, 0, 0, 0, 0, 3083, 0, 16, 24, 0, 0, 0, 0, 0, 0, 0, 296, 1410, 1293, 969, 275, 0, 194, 93, 0, 3791, 0, 0, 0, 2268, 0, 5, 1, 401, 0, 0, 0, 13, 3069, 15, 3841, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5188, 3718, 2805, 3123, 4243, 4078, 2675, 6938] avg=4096.0000 max=6938 max_rank=7 min=2675 min_rank=6 imbalance_ratio=1.693848 max_load_reduction=25830 max_load_reduction_pct=0.788269 moved_to_echo=17934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2424, 8, 2494, 0, 0, 0, 0, 2076, 0, 0, 2121, 43, 0, 0, 0, 3, 1, 844, 2792, 0, 0, 0, 0, 3300, 0, 682, 0, 0, 0, 0, 0, 9, 0, 2, 34, 1602, 1287, 980, 290, 1, 2, 0, 0, 4073, 0, 0, 0, 4092, 33, 0, 0, 425, 0, 0, 0, 18, 3116, 16, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4926, 4240, 3640, 3982, 4204, 4076, 4550, 3150] avg=4096.0000 max=4926 max_rank=0 min=3150 min_rank=7 imbalance_ratio=1.202637 max_load_reduction=27842 max_load_reduction_pct=0.849670 moved_to_echo=15980 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32629, 32768, 32768, 32907] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32629 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32629, 32768, 32768, 32907] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32629, 32768, 32768, 32907] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32629, 32768, 32768, 32907] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32629, 32768, 32768, 32907] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32907 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32629, 32768, 32768, 32907] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32629, 32768, 32768, 32907] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32629, 32768, 32768, 32907] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32629 actual_valid_recv=33280 observed_loss=-651 tokens_per_expert=[128, 128, 384, 512, 12672, 10368, 7296, 1792] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 13184, 128, 19584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32907 actual_valid_recv=33152 observed_loss=-245 tokens_per_expert=[256, 22912, 1408, 8576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19968, 128, 12928, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5120, 4224, 1408, 22272, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6400, 512, 14464, 11264, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[29952, 384, 128, 128, 2688, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 256, 128, 128, 32256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 23, 40, 890, 15, 383, 0, 4, 8, 20, 0, 1676, 4044, 4085, 4096, 71, 0, 656, 0, 7, 3391, 0, 967, 262, 4053, 4071, 0, 0, 0, 4005, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 3396, 0, 1218, 1, 0, 18, 0, 0, 60, 4094, 4055, 4096, 0, 0, 494, 12, 0, 1477, 0, 3666, 8, 1471, 4086, 0, 0, 0, 4093, 521, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 381, 106, 0, 0, 655, 1, 2882, 0, 1264, 4023, 4096, 4096, 0, 0, 0, 0, 0, 216, 1, 960, 0, 3751, 3835, 51, 0, 0, 4076, 2350, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 3380, 2, 1451, 0, 775, 4094, 4012, 4096, 3, 0, 136, 0, 0, 4096, 0, 381, 0, 2148, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 43, 0, 0, 14, 0, 82, 3, 3667, 3518, 4014, 4096, 0, 0, 934, 2, 0, 2, 62, 4062, 491, 3575, 4093, 2, 0, 0, 4074, 30, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 328, 0, 0, 137, 421, 12, 427, 0, 936, 3620, 4060, 4093, 22, 0, 15, 59, 6, 2892, 84, 1511, 159, 2711, 4046, 10, 0, 16, 4083, 3117, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 85, 6, 0, 0, 623, 1, 13, 3128, 0, 10, 4096, 4094, 4003, 0, 0, 2921, 0, 0, 224, 0, 3430, 0, 2273, 3762, 0, 0, 0, 4089, 7, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[31, 0, 0, 13, 0, 2224, 283, 8, 152, 1, 2, 1685, 4094, 3516, 3699, 0, 2, 173, 645, 694, 34, 422, 4095, 1, 3835, 2996, 484, 31, 0, 3168, 480, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 3396, 3498, 0, 0, 0, 0, 1218, 1, 0, 1808, 1211, 0, 0, 18, 0, 0, 60, 1173, 706, 24, 0, 4094, 139, 0, 0, 0, 0, 0, 0, 0, 494, 12, 0, 2705, 598, 0, 0, 1477, 0, 3642, 8, 0, 0, 0, 0, 1471, 1105, 0, 0, 0, 0, 0, 0, 0, 3387, 521, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6896, 4238, 1981, 4233, 3809, 5127, 2576, 3908] avg=4096.0000 max=6896 max_rank=0 min=1981 min_rank=2 imbalance_ratio=1.683594 max_load_reduction=25872 max_load_reduction_pct=0.789551 moved_to_echo=15420 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 23, 40, 890, 3499, 0, 0, 0, 15, 383, 0, 4, 1801, 1220, 0, 0, 8, 20, 0, 1676, 1169, 691, 6, 0, 4044, 140, 0, 71, 0, 0, 0, 0, 0, 656, 0, 7, 2725, 597, 0, 0, 3391, 0, 961, 262, 0, 0, 0, 0, 4053, 1101, 0, 0, 0, 0, 0, 0, 0, 3314, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4452, 3423, 3570, 4255, 3985, 4614, 5154, 3315] avg=4096.0000 max=5154 max_rank=6 min=3315 min_rank=7 imbalance_ratio=1.258301 max_load_reduction=27614 max_load_reduction_pct=0.842712 moved_to_echo=17068 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 3498, 0, 0, 0, 43, 0, 0, 14, 1811, 1199, 0, 0, 0, 82, 3, 3667, 1175, 703, 26, 0, 3518, 137, 0, 0, 0, 0, 0, 0, 0, 934, 2, 0, 2678, 598, 0, 0, 2, 62, 4036, 491, 0, 0, 0, 0, 3575, 1107, 2, 0, 0, 0, 0, 0, 0, 3371, 30, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3502, 3067, 5656, 3655, 4212, 4591, 4684, 3401] avg=4096.0000 max=5656 max_rank=2 min=3067 min_rank=1 imbalance_ratio=1.380859 max_load_reduction=27112 max_load_reduction_pct=0.827393 moved_to_echo=16888 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3499, 0, 0, 0, 0, 0, 0, 3380, 1812, 1198, 0, 0, 2, 1451, 0, 775, 1176, 706, 2, 0, 4094, 138, 0, 3, 0, 0, 0, 0, 0, 136, 0, 0, 2676, 597, 0, 0, 4096, 0, 379, 0, 0, 0, 0, 0, 2148, 1108, 0, 0, 0, 0, 0, 0, 0, 3390, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3501, 6390, 4112, 4235, 3409, 4475, 3256, 3390] avg=4096.0000 max=6390 max_rank=1 min=3256 min_rank=6 imbalance_ratio=1.560059 max_load_reduction=26378 max_load_reduction_pct=0.804993 moved_to_echo=14530 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 381, 3498, 0, 0, 0, 106, 0, 0, 655, 1697, 1223, 0, 0, 1, 2882, 0, 1264, 1101, 703, 6, 0, 4023, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2732, 598, 0, 0, 216, 1, 954, 0, 0, 0, 0, 0, 3751, 1037, 51, 0, 0, 0, 0, 0, 0, 3373, 2350, 24, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3879, 3681, 5957, 4164, 3330, 1171, 4839, 5747] avg=4096.0000 max=5957 max_rank=2 min=1171 min_rank=5 imbalance_ratio=1.454346 max_load_reduction=26811 max_load_reduction_pct=0.818207 moved_to_echo=15087 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 328, 3495, 0, 0, 0, 0, 0, 137, 421, 1790, 1212, 0, 0, 12, 427, 0, 936, 1162, 704, 9, 0, 3620, 140, 0, 22, 0, 0, 0, 0, 0, 15, 59, 6, 2708, 598, 0, 0, 2892, 84, 1502, 159, 0, 0, 0, 0, 2711, 1094, 10, 0, 0, 0, 0, 0, 16, 3379, 3117, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3823, 3560, 3250, 3782, 3386, 4637, 3815, 6515] avg=4096.0000 max=6515 max_rank=7 min=3250 min_rank=2 imbalance_ratio=1.590576 max_load_reduction=26253 max_load_reduction_pct=0.801178 moved_to_echo=18353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[31, 0, 0, 13, 3160, 0, 0, 0, 0, 2224, 283, 8, 1330, 1054, 0, 0, 152, 1, 2, 1685, 863, 550, 31, 0, 4094, 113, 0, 0, 0, 0, 0, 0, 2, 173, 645, 694, 2349, 539, 0, 0, 34, 422, 4064, 1, 0, 0, 0, 0, 3835, 803, 484, 31, 0, 0, 0, 0, 0, 2618, 480, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3204, 4899, 3284, 4207, 4402, 4521, 5153, 3098] avg=4096.0000 max=5153 max_rank=6 min=3098 min_rank=7 imbalance_ratio=1.258057 max_load_reduction=27615 max_load_reduction_pct=0.842743 moved_to_echo=17174 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 85, 6, 3420, 0, 0, 0, 0, 0, 623, 1, 1664, 1222, 0, 0, 13, 3128, 0, 10, 1080, 705, 22, 0, 4096, 141, 0, 0, 0, 0, 0, 0, 0, 2921, 0, 0, 2731, 583, 0, 0, 224, 0, 3408, 0, 0, 0, 0, 0, 2273, 1018, 0, 0, 0, 0, 0, 0, 0, 3384, 7, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3511, 3510, 4958, 4237, 6235, 3632, 3291, 3394] avg=4096.0000 max=6235 max_rank=4 min=3291 min_rank=6 imbalance_ratio=1.522217 max_load_reduction=26533 max_load_reduction_pct=0.809723 moved_to_echo=16552 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 3840, 1152, 4608, 13824, 9600, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 5376, 768, 768, 21376, 4736, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 8064, 128, 10112, 8960, 5504, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[23936, 8448, 640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 5120, 27648, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12416, 640, 19072, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 26240, 6528, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[31616, 1152, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[3591, 0, 68, 513, 0, 0, 22, 0, 4087, 5, 0, 0, 1, 0, 0, 0, 4061, 2816, 4096, 0, 0, 0, 4, 1221, 4095, 1, 0, 0, 4086, 1, 4090, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[3664, 10, 1349, 1, 0, 10, 204, 0, 4020, 44, 0, 0, 0, 65, 212, 97, 0, 3998, 4051, 28, 0, 154, 0, 3277, 4030, 96, 2, 0, 4079, 21, 3332, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[15, 4, 2958, 0, 0, 0, 483, 0, 4093, 5, 24, 0, 19, 3, 2, 325, 0, 3758, 4017, 0, 1, 45, 0, 1070, 4093, 3868, 120, 0, 4056, 3, 3761, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1055, 241, 762, 167, 0, 219, 1305, 0, 3906, 1, 0, 0, 0, 6, 53, 160, 0, 3881, 4054, 1, 0, 0, 0, 3513, 3962, 1531, 21, 0, 4020, 0, 3910, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[3613, 0, 5, 47, 4, 0, 1408, 0, 4089, 0, 0, 1, 14, 0, 37, 0, 989, 4066, 4092, 8, 1, 0, 41, 111, 4087, 2040, 3, 0, 4040, 0, 4072, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4095, 121, 1300, 1, 0, 2, 7, 0, 4095, 0, 2, 536, 0, 0, 0, 1, 3330, 2332, 4093, 0, 0, 43, 0, 11, 3191, 1832, 7, 0, 4089, 0, 3680, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[3851, 14, 0, 294, 8, 0, 0, 1, 4066, 0, 43, 1234, 0, 135, 309, 0, 4025, 3831, 1108, 0, 0, 28, 5, 3892, 2904, 2935, 9, 0, 3791, 1, 283, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 138, 10, 0, 0, 413, 0, 4096, 0, 0, 82, 0, 140, 0, 184, 28, 4095, 4074, 0, 0, 0, 0, 4095, 3824, 3472, 0, 0, 4096, 0, 4021, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3591, 0, 68, 513, 315, 209, 0, 0, 0, 0, 22, 0, 3207, 455, 0, 0, 3878, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 179, 0, 0, 4061, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1221, 1356, 514, 0, 0, 2284, 1, 0, 0, 0, 0, 0, 0, 700, 1, 4090, 10, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4696, 3684, 3883, 4276, 6048, 3095, 2285, 4801] avg=4096.0000 max=6048 max_rank=4 min=2285 min_rank=6 imbalance_ratio=1.476562 max_load_reduction=26720 max_load_reduction_pct=0.815430 moved_to_echo=16229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3664, 10, 1349, 1, 447, 205, 0, 0, 0, 10, 204, 0, 3202, 448, 0, 0, 3815, 44, 0, 0, 0, 0, 0, 0, 0, 65, 212, 97, 4051, 179, 0, 0, 0, 2821, 0, 28, 0, 0, 0, 0, 0, 154, 0, 3277, 1334, 730, 0, 0, 2248, 96, 2, 0, 0, 0, 0, 0, 698, 21, 3332, 24, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5676, 3864, 3859, 4604, 2849, 5495, 2346, 4075] avg=4096.0000 max=5676 max_rank=0 min=2346 min_rank=6 imbalance_ratio=1.385742 max_load_reduction=27092 max_load_reduction_pct=0.826782 moved_to_echo=14765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3613, 0, 5, 47, 454, 209, 0, 0, 4, 0, 1408, 0, 3171, 454, 0, 0, 3880, 0, 0, 1, 0, 0, 0, 0, 14, 0, 37, 0, 4092, 177, 0, 0, 989, 2869, 0, 8, 0, 0, 0, 0, 1, 0, 41, 111, 1353, 743, 0, 0, 2280, 2040, 3, 0, 0, 0, 0, 0, 692, 0, 4072, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4328, 5037, 3881, 4320, 3866, 2249, 4323, 4764] avg=4096.0000 max=5037 max_rank=1 min=2249 min_rank=5 imbalance_ratio=1.229736 max_load_reduction=27731 max_load_reduction_pct=0.846283 moved_to_echo=15202 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[15, 4, 2958, 0, 420, 209, 0, 0, 0, 0, 483, 0, 3184, 455, 0, 0, 3884, 5, 24, 0, 0, 0, 0, 0, 19, 3, 2, 325, 4017, 178, 0, 0, 0, 2652, 0, 0, 0, 0, 0, 0, 1, 45, 0, 1070, 1355, 686, 0, 0, 2283, 3868, 120, 0, 0, 0, 0, 0, 694, 3, 3761, 45, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3606, 4122, 3913, 4544, 2652, 3157, 6271, 4503] avg=4096.0000 max=6271 max_rank=6 min=2652 min_rank=4 imbalance_ratio=1.531006 max_load_reduction=26497 max_load_reduction_pct=0.808624 moved_to_echo=16583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1055, 241, 762, 167, 434, 199, 0, 0, 0, 219, 1305, 0, 3155, 440, 0, 0, 3707, 1, 0, 0, 0, 0, 0, 0, 0, 6, 53, 160, 4054, 176, 0, 0, 0, 2738, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3513, 1312, 709, 0, 0, 2210, 1531, 21, 0, 0, 0, 0, 0, 689, 0, 3910, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2858, 5119, 3708, 4449, 2739, 5534, 3762, 4599] avg=4096.0000 max=5534 max_rank=5 min=2739 min_rank=4 imbalance_ratio=1.351074 max_load_reduction=27234 max_load_reduction_pct=0.831116 moved_to_echo=16634 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 138, 10, 458, 209, 0, 0, 0, 0, 413, 0, 3215, 425, 0, 0, 3887, 0, 0, 82, 0, 0, 0, 0, 0, 140, 0, 184, 4074, 180, 0, 0, 28, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1266, 748, 0, 0, 2133, 3472, 0, 0, 0, 0, 0, 0, 701, 0, 4021, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[815, 4053, 3969, 4578, 2917, 6109, 5605, 4722] avg=4096.0000 max=6109 max_rank=5 min=815 min_rank=0 imbalance_ratio=1.491455 max_load_reduction=26659 max_load_reduction_pct=0.813568 moved_to_echo=19353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3851, 14, 0, 294, 432, 212, 0, 0, 8, 0, 0, 1, 2980, 327, 0, 0, 3854, 0, 43, 1234, 0, 0, 0, 0, 0, 135, 309, 0, 1108, 171, 0, 0, 4025, 2696, 0, 0, 0, 0, 0, 0, 0, 28, 5, 3892, 965, 703, 0, 0, 1612, 2935, 9, 0, 0, 0, 0, 0, 640, 1, 283, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4803, 3316, 5131, 1723, 6721, 5593, 4556, 925] avg=4096.0000 max=6721 max_rank=4 min=925 min_rank=7 imbalance_ratio=1.640869 max_load_reduction=26047 max_load_reduction_pct=0.794891 moved_to_echo=17795 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 121, 1300, 1, 260, 209, 0, 0, 0, 2, 7, 0, 3210, 354, 0, 0, 3886, 0, 2, 536, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 180, 0, 0, 3330, 1646, 0, 0, 0, 0, 0, 0, 0, 43, 0, 11, 1056, 426, 0, 0, 1781, 1832, 7, 0, 0, 0, 0, 0, 699, 0, 3680, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5986, 3573, 4424, 4274, 4976, 1536, 3620, 4379] avg=4096.0000 max=5986 max_rank=0 min=1536 min_rank=5 imbalance_ratio=1.461426 max_load_reduction=26782 max_load_reduction_pct=0.817322 moved_to_echo=14511 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 384, 128, 17280, 10112, 5376, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16896, 15872, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[19968, 512, 6656, 1152, 3328, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 3968, 128, 25344, 3456, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12544, 20352, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5632, 128, 27264, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 384, 640, 768, 29696, 1536, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[30848, 128, 128, 1920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[36, 0, 4041, 4, 0, 3994, 3, 0, 0, 0, 0, 400, 24, 4084, 3535, 0, 97, 178, 0, 4082, 1, 4084, 1, 0, 4055, 0, 0, 47, 0, 9, 4093, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 8, 87, 3878, 0, 3855, 254, 0, 0, 0, 0, 3174, 49, 3868, 3366, 13, 1, 1370, 0, 4075, 131, 393, 0, 0, 4095, 0, 0, 0, 16, 97, 4038, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 85, 95, 64, 4083, 843, 0, 0, 0, 0, 0, 0, 4048, 3982, 8, 13, 4094, 51, 4080, 0, 3064, 0, 55, 4047, 5, 7, 0, 0, 3, 4091, 50] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3, 0, 1716, 1775, 2, 3869, 0, 0, 0, 3, 0, 5, 0, 4081, 4042, 0, 254, 4057, 0, 4092, 0, 707, 0, 5, 3905, 0, 51, 0, 33, 106, 4062, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2518, 4, 69, 4096, 250, 0, 1, 0, 0, 130, 0, 4095, 3948, 0, 0, 4091, 0, 4096, 0, 1909, 0, 0, 4088, 7, 0, 0, 0, 0, 3466, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4074, 32, 0, 4096, 8, 1, 502, 504, 215, 3843, 1027, 3951, 0, 14, 23, 34, 0, 3491, 0, 3375, 1, 0, 3660, 0, 0, 2, 5, 0, 3909, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 11, 2219, 0, 19, 4079, 0, 0, 68, 0, 0, 0, 0, 3157, 3038, 0, 0, 3774, 73, 4032, 0, 3999, 0, 0, 3990, 31, 0, 118, 0, 20, 4051, 89] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4093, 18, 0, 3944, 170, 0, 0, 0, 0, 5, 0, 3802, 3764, 0, 0, 4066, 0, 4095, 390, 188, 2, 96, 4004, 2, 31, 0, 7, 0, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[36, 0, 4041, 4, 1028, 0, 0, 0, 0, 3879, 3, 0, 0, 0, 0, 0, 0, 0, 0, 400, 3142, 0, 0, 0, 24, 784, 3535, 0, 0, 0, 0, 0, 97, 178, 0, 1350, 0, 0, 0, 0, 1, 4084, 1, 0, 1676, 158, 0, 0, 4055, 0, 0, 47, 49, 28, 0, 0, 0, 9, 4093, 0, 66, 0, 0, 0] rerouted_tokens_per_ep_rank=[5109, 3882, 3542, 4343, 1625, 5920, 4179, 4168] avg=4096.0000 max=5920 max_rank=5 min=1625 min_rank=4 imbalance_ratio=1.445312 max_load_reduction=26848 max_load_reduction_pct=0.819336 moved_to_echo=15892 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 87, 3878, 1026, 0, 0, 0, 0, 3744, 254, 0, 0, 0, 0, 0, 0, 0, 0, 3174, 2976, 0, 0, 0, 49, 742, 3366, 13, 0, 0, 0, 0, 1, 1370, 0, 1348, 0, 0, 0, 0, 131, 393, 0, 0, 1673, 150, 0, 0, 4095, 0, 0, 0, 48, 28, 0, 0, 16, 97, 4038, 0, 63, 0, 0, 0] rerouted_tokens_per_ep_rank=[4999, 3998, 6150, 4170, 2719, 2347, 4171, 4214] avg=4096.0000 max=6150 max_rank=2 min=2347 min_rank=5 imbalance_ratio=1.501465 max_load_reduction=26618 max_load_reduction_pct=0.812317 moved_to_echo=13451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 85, 95, 1027, 0, 0, 0, 64, 3965, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3114, 0, 0, 0, 0, 777, 3982, 8, 0, 0, 0, 0, 13, 4094, 51, 1350, 0, 0, 0, 0, 0, 3064, 0, 55, 1675, 157, 0, 0, 4047, 5, 7, 0, 51, 28, 0, 0, 0, 3, 4091, 50, 67, 0, 0, 0] rerouted_tokens_per_ep_rank=[1207, 4872, 3114, 4767, 5508, 4951, 4138, 4211] avg=4096.0000 max=5508 max_rank=4 min=1207 min_rank=0 imbalance_ratio=1.344727 max_load_reduction=27260 max_load_reduction_pct=0.831909 moved_to_echo=18808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 1716, 1775, 1030, 0, 0, 0, 2, 3757, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 3139, 0, 0, 0, 0, 784, 4042, 0, 0, 0, 0, 0, 254, 4057, 0, 1354, 0, 0, 0, 0, 0, 707, 0, 5, 1680, 158, 0, 0, 3905, 0, 51, 0, 48, 28, 0, 0, 33, 106, 4062, 0, 64, 0, 0, 0] rerouted_tokens_per_ep_rank=[4524, 3759, 3147, 4826, 5665, 2550, 4032, 4265] avg=4096.0000 max=5665 max_rank=4 min=2550 min_rank=5 imbalance_ratio=1.383057 max_load_reduction=27103 max_load_reduction_pct=0.827118 moved_to_echo=16512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 2219, 0, 1015, 0, 0, 0, 19, 3962, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 2429, 0, 0, 0, 0, 606, 3038, 0, 0, 0, 0, 0, 0, 3774, 73, 1334, 0, 0, 0, 0, 0, 3999, 0, 0, 1655, 122, 0, 0, 3990, 31, 0, 118, 50, 28, 0, 0, 0, 20, 4051, 89, 67, 0, 0, 0] rerouted_tokens_per_ep_rank=[3245, 3981, 2497, 3644, 5181, 5776, 4217, 4227] avg=4096.0000 max=5776 max_rank=5 min=2497 min_rank=2 imbalance_ratio=1.410156 max_load_reduction=26992 max_load_reduction_pct=0.823730 moved_to_echo=19401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2518, 4, 1031, 0, 0, 0, 69, 3978, 250, 0, 0, 0, 0, 0, 1, 0, 0, 130, 3150, 0, 0, 0, 0, 786, 3948, 0, 0, 0, 0, 0, 0, 4091, 0, 1356, 0, 0, 0, 0, 0, 1909, 0, 0, 1681, 159, 0, 0, 4088, 7, 0, 0, 51, 28, 0, 0, 0, 0, 3466, 0, 67, 0, 0, 0] rerouted_tokens_per_ep_rank=[3553, 4297, 3281, 4734, 5447, 3749, 4174, 3533] avg=4096.0000 max=5447 max_rank=4 min=3281 min_rank=2 imbalance_ratio=1.329834 max_load_reduction=27321 max_load_reduction_pct=0.833771 moved_to_echo=16903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4074, 32, 883, 0, 0, 0, 0, 3970, 8, 1, 0, 0, 0, 0, 502, 504, 215, 3843, 3043, 0, 0, 0, 1027, 749, 0, 14, 0, 0, 0, 0, 23, 34, 0, 1145, 0, 0, 0, 0, 0, 3375, 1, 0, 1436, 159, 0, 0, 3660, 0, 0, 2, 54, 27, 0, 0, 5, 0, 3909, 1, 72, 0, 0, 0] rerouted_tokens_per_ep_rank=[4989, 3979, 8107, 1790, 1202, 4971, 3743, 3987] avg=4096.0000 max=8107 max_rank=2 min=1202 min_rank=4 imbalance_ratio=1.979248 max_load_reduction=24661 max_load_reduction_pct=0.752594 moved_to_echo=13903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 18, 1031, 0, 0, 0, 0, 3830, 170, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2925, 0, 0, 0, 0, 730, 3764, 0, 0, 0, 0, 0, 0, 4066, 0, 1355, 0, 0, 0, 0, 390, 188, 2, 96, 1681, 147, 0, 0, 4004, 2, 31, 0, 49, 28, 0, 0, 7, 0, 4091, 0, 65, 0, 0, 0] rerouted_tokens_per_ep_rank=[5142, 4000, 2930, 4494, 5421, 2504, 4114, 4163] avg=4096.0000 max=5421 max_rank=4 min=2504 min_rank=5 imbalance_ratio=1.323486 max_load_reduction=27347 max_load_reduction_pct=0.834564 moved_to_echo=16202 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 18944, 5888, 8192, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[640, 512, 256, 7680, 23936, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[640, 17792, 128, 256, 13184, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 21760, 128, 10624, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 6016, 25728, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 31104, 1536, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 31872, 256, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[31872, 128, 128, 256, 512, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 4094, 0, 0, 39, 0, 0, 1, 0, 0, 3711, 2423, 0, 0, 0, 0, 0, 29, 3062, 4052, 0, 0, 0, 8, 0, 3046, 16, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[3328, 0, 16, 0, 4096, 3354, 0, 0, 0, 0, 2, 5, 1, 0, 4082, 386, 1004, 24, 0, 0, 962, 0, 4087, 4090, 0, 90, 0, 1, 0, 816, 2353, 4071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4082, 34, 0, 0, 4082, 4069, 0, 0, 0, 1, 3, 0, 204, 0, 3173, 0, 2245, 0, 0, 88, 26, 0, 4096, 4031, 18, 0, 0, 0, 0, 722, 1825, 4069] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[3994, 0, 0, 5, 4073, 4085, 324, 7, 0, 4, 0, 0, 2, 0, 163, 0, 4, 0, 0, 14, 0, 55, 4076, 4057, 0, 193, 1, 0, 3, 3930, 4036, 3742] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3861, 35, 0, 88, 4049, 4050, 537, 297, 11, 0, 0, 31, 3269, 4, 1004, 98, 3, 0, 18, 9, 0, 0, 4067, 4071, 50, 0, 1, 0, 0, 9, 3280, 3926] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4094, 0, 1, 0, 4000, 3825, 76, 12, 3, 0, 807, 0, 118, 0, 3934, 3248, 0, 10, 0, 18, 0, 21, 4093, 4089, 0, 76, 12, 0, 0, 1177, 244, 2910] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3500, 0, 0, 0, 4096, 4096, 919, 2, 0, 0, 0, 0, 1749, 0, 1336, 1, 0, 0, 0, 0, 1, 0, 4095, 4096, 0, 800, 0, 0, 0, 3978, 3, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3376, 8, 2, 0, 3628, 3917, 0, 2, 0, 0, 420, 2, 0, 29, 860, 0, 8, 326, 0, 0, 0, 137, 4089, 4095, 0, 2673, 0, 0, 300, 4096, 3146, 1654] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3328, 0, 16, 0, 143, 95, 9, 16, 0, 3259, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 3947, 0, 0, 0, 1, 0, 4082, 386, 425, 0, 0, 0, 1004, 24, 0, 0, 3552, 143, 0, 0, 962, 0, 4071, 0, 0, 0, 0, 0, 0, 90, 0, 1, 3503, 544, 0, 0, 0, 807, 2353, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3607, 3259, 3954, 4894, 4723, 5033, 4138, 3160] avg=4096.0000 max=5033 max_rank=5 min=3160 min_rank=7 imbalance_ratio=1.228760 max_load_reduction=27735 max_load_reduction_pct=0.846405 moved_to_echo=17054 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 144, 116, 36, 12, 0, 3978, 0, 0, 0, 0, 0, 0, 39, 0, 0, 1, 3909, 0, 0, 0, 0, 0, 3711, 2423, 427, 0, 0, 0, 0, 0, 0, 0, 3551, 143, 0, 0, 0, 29, 3050, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3525, 544, 0, 0, 0, 3010, 16, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4404, 3978, 3949, 6561, 3694, 3079, 4077, 3026] avg=4096.0000 max=6561 max_rank=3 min=3026 min_rank=7 imbalance_ratio=1.601807 max_load_reduction=26207 max_load_reduction_pct=0.799774 moved_to_echo=13876 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3861, 35, 0, 88, 138, 115, 0, 16, 0, 3935, 537, 297, 0, 0, 0, 0, 11, 0, 0, 31, 3928, 0, 0, 0, 3269, 4, 1004, 98, 408, 0, 0, 0, 3, 0, 18, 9, 3511, 143, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 50, 0, 1, 0, 3380, 538, 0, 0, 0, 9, 3280, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4253, 4769, 3970, 4783, 3684, 4051, 3969, 3289] avg=4096.0000 max=4783 max_rank=3 min=3289 min_rank=7 imbalance_ratio=1.167725 max_load_reduction=27985 max_load_reduction_pct=0.854034 moved_to_echo=14993 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4082, 34, 0, 0, 145, 116, 8, 16, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 3890, 0, 0, 0, 204, 0, 3173, 0, 423, 0, 0, 0, 2245, 0, 0, 88, 3541, 141, 0, 0, 26, 0, 4080, 0, 0, 0, 0, 0, 18, 0, 0, 0, 3501, 541, 0, 0, 0, 714, 1825, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4401, 3953, 3894, 3800, 6015, 4106, 4060, 2539] avg=4096.0000 max=6015 max_rank=4 min=2539 min_rank=7 imbalance_ratio=1.468506 max_load_reduction=26753 max_load_reduction_pct=0.816437 moved_to_echo=16720 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3500, 0, 0, 0, 145, 116, 47, 16, 0, 3980, 919, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 0, 0, 0, 1749, 0, 1336, 1, 427, 0, 0, 0, 0, 0, 0, 0, 3552, 143, 0, 0, 1, 0, 4079, 0, 0, 0, 0, 0, 0, 800, 0, 0, 3524, 544, 0, 0, 0, 3931, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3824, 4901, 3953, 3513, 3695, 4080, 4868, 3934] avg=4096.0000 max=4901 max_rank=1 min=3513 min_rank=3 imbalance_ratio=1.196533 max_load_reduction=27867 max_load_reduction_pct=0.850433 moved_to_echo=16577 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3994, 0, 0, 5, 132, 116, 46, 16, 0, 3969, 324, 7, 0, 0, 0, 0, 0, 4, 0, 0, 3914, 0, 0, 0, 2, 0, 163, 0, 389, 0, 0, 0, 4, 0, 0, 14, 3532, 143, 0, 0, 0, 55, 4060, 0, 0, 0, 0, 0, 0, 193, 1, 0, 3221, 541, 0, 0, 3, 3884, 4036, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4309, 4300, 3918, 554, 3693, 4115, 3956, 7923] avg=4096.0000 max=7923 max_rank=7 min=554 min_rank=3 imbalance_ratio=1.934326 max_load_reduction=24845 max_load_reduction_pct=0.758209 moved_to_echo=19687 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3376, 8, 2, 0, 59, 115, 52, 20, 0, 3802, 0, 2, 0, 0, 0, 0, 0, 0, 420, 2, 3952, 0, 0, 0, 0, 29, 860, 0, 172, 0, 0, 0, 8, 326, 0, 0, 3147, 143, 0, 0, 0, 137, 4069, 0, 0, 0, 0, 0, 0, 2673, 0, 0, 1423, 481, 0, 0, 300, 4044, 3146, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3632, 3804, 4374, 1061, 3624, 4206, 4577, 7490] avg=4096.0000 max=7490 max_rank=7 min=1061 min_rank=3 imbalance_ratio=1.828613 max_load_reduction=25278 max_load_reduction_pct=0.771423 moved_to_echo=19897 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 1, 0, 104, 109, 14, 16, 0, 3716, 76, 12, 0, 0, 0, 0, 3, 0, 807, 0, 3946, 0, 0, 0, 118, 0, 3934, 3248, 302, 0, 0, 0, 0, 10, 0, 18, 3469, 143, 0, 0, 0, 21, 4077, 0, 0, 0, 0, 0, 0, 76, 12, 0, 2504, 531, 0, 0, 0, 1163, 244, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4338, 3804, 4756, 7602, 3640, 4098, 3123, 1407] avg=4096.0000 max=7602 max_rank=3 min=1407 min_rank=7 imbalance_ratio=1.855957 max_load_reduction=25166 max_load_reduction_pct=0.768005 moved_to_echo=12268 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 17664, 14976, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5376, 128, 18304, 6272, 3072, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 3840, 128, 128, 24704, 4352, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[30336, 128, 128, 128, 1024, 1024, 256, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3328, 384, 128, 256, 27904, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 30592, 1920, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[1024, 256, 31616, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 1280, 128, 31488, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[370, 5, 6, 0, 0, 0, 0, 0, 1, 0, 122, 13, 46, 0, 6, 4092, 3992, 3846, 0, 0, 3950, 4090, 4078, 0, 17, 0, 0, 0, 0, 4033, 30, 4071] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 0, 148, 2737, 0, 0, 0, 0, 0, 10, 0, 3987, 1229, 3878, 0, 25, 3847, 4079, 4012, 0, 16, 182, 44, 0, 234, 4049, 190, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3231, 0, 0, 0, 0, 0, 0, 0, 0, 5, 998, 0, 3, 0, 1, 4096, 3180, 4096, 0, 0, 3739, 645, 4096, 0, 0, 11, 0, 0, 0, 3772, 799, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4004, 0, 0, 6, 0, 55, 0, 66, 0, 0, 0, 0, 45, 3, 3, 4086, 4006, 4072, 80, 0, 1955, 289, 4096, 0, 0, 1468, 35, 0, 1, 4042, 532, 3924] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4043, 32, 6, 0, 4, 0, 0, 0, 3, 116, 2178, 0, 0, 6, 1, 3817, 4092, 4090, 0, 1, 45, 1910, 4090, 14, 0, 36, 6, 0, 1, 4044, 145, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[12, 77, 0, 0, 0, 0, 0, 2555, 6, 0, 26, 0, 189, 0, 0, 3879, 1310, 4096, 13, 113, 4096, 4034, 4063, 0, 0, 0, 0, 0, 141, 4060, 2, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[3233, 0, 361, 222, 20, 11, 40, 3279, 119, 0, 0, 0, 0, 1, 0, 81, 4096, 3990, 0, 0, 4091, 1805, 3913, 79, 3, 666, 0, 30, 0, 437, 3188, 3103] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[34, 1, 0, 0, 3, 0, 2, 2, 0, 36, 0, 0, 0, 0, 0, 4051, 4096, 4057, 1, 12, 4095, 4093, 4078, 0, 2, 35, 0, 0, 0, 4077, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[370, 5, 6, 0, 1641, 764, 0, 0, 0, 0, 0, 0, 2300, 598, 0, 0, 1, 0, 122, 13, 3473, 277, 0, 0, 46, 0, 6, 4092, 513, 142, 0, 0, 3992, 782, 0, 0, 0, 0, 0, 0, 1796, 4090, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 3801, 0, 0, 0, 0, 3891, 30, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2786, 2898, 3886, 4799, 4774, 5886, 3818, 3921] avg=4096.0000 max=5886 max_rank=5 min=2786 min_rank=0 imbalance_ratio=1.437012 max_load_reduction=26882 max_load_reduction_pct=0.820374 moved_to_echo=18399 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 1599, 771, 0, 0, 0, 0, 148, 2737, 2319, 600, 0, 0, 0, 0, 0, 0, 3496, 273, 0, 0, 0, 10, 0, 3987, 500, 143, 0, 0, 1229, 788, 0, 25, 0, 0, 0, 0, 1748, 4079, 0, 0, 0, 0, 0, 0, 16, 182, 44, 0, 3739, 0, 0, 0, 234, 3906, 190, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2375, 5804, 3769, 4640, 2042, 5827, 3981, 4330] avg=4096.0000 max=5827 max_rank=5 min=2042 min_rank=4 imbalance_ratio=1.422607 max_load_reduction=26941 max_load_reduction_pct=0.822174 moved_to_echo=16180 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3231, 0, 0, 0, 1554, 814, 0, 0, 0, 0, 0, 0, 2449, 601, 0, 0, 0, 5, 998, 0, 3495, 280, 0, 0, 3, 0, 1, 4096, 486, 133, 0, 0, 3180, 833, 0, 0, 0, 0, 0, 0, 1699, 645, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3816, 0, 0, 0, 0, 3639, 799, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5599, 3050, 4778, 4719, 4013, 2344, 3827, 4438] avg=4096.0000 max=5599 max_rank=0 min=2344 min_rank=5 imbalance_ratio=1.366943 max_load_reduction=27169 max_load_reduction_pct=0.829132 moved_to_echo=14622 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 77, 0, 0, 1702, 814, 0, 0, 0, 0, 0, 2555, 2449, 600, 0, 0, 6, 0, 26, 0, 3496, 276, 0, 0, 189, 0, 0, 3879, 532, 143, 0, 0, 1310, 833, 13, 113, 0, 0, 0, 0, 1862, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3787, 0, 0, 0, 141, 3917, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2605, 5604, 3804, 4743, 2269, 5896, 3787, 4060] avg=4096.0000 max=5896 max_rank=5 min=2269 min_rank=4 imbalance_ratio=1.439453 max_load_reduction=26872 max_load_reduction_pct=0.820068 moved_to_echo=16012 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4004, 0, 0, 6, 812, 809, 0, 0, 0, 55, 0, 66, 2435, 576, 0, 0, 0, 0, 0, 0, 3348, 280, 0, 0, 45, 3, 3, 4086, 254, 143, 0, 0, 4006, 828, 80, 0, 0, 0, 0, 0, 889, 289, 0, 0, 0, 0, 0, 0, 0, 1468, 35, 0, 3816, 0, 0, 0, 1, 3899, 532, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5631, 3132, 3628, 4534, 4914, 1178, 5319, 4432] avg=4096.0000 max=5631 max_rank=0 min=1178 min_rank=5 imbalance_ratio=1.374756 max_load_reduction=27137 max_load_reduction_pct=0.828156 moved_to_echo=15843 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4043, 32, 6, 0, 18, 813, 0, 0, 4, 0, 0, 0, 2446, 599, 0, 0, 3, 116, 2178, 0, 3489, 278, 0, 0, 0, 6, 1, 3817, 5, 143, 0, 0, 4092, 831, 0, 1, 0, 0, 0, 0, 22, 1910, 0, 14, 0, 0, 0, 0, 0, 36, 6, 0, 3812, 0, 0, 0, 1, 3901, 145, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4912, 3049, 6064, 3972, 4924, 1946, 3854, 4047] avg=4096.0000 max=6064 max_rank=2 min=1946 min_rank=5 imbalance_ratio=1.480469 max_load_reduction=26704 max_load_reduction_pct=0.814941 moved_to_echo=14771 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3233, 0, 361, 222, 1704, 797, 0, 0, 20, 11, 40, 3279, 2391, 456, 0, 0, 119, 0, 0, 0, 2647, 267, 0, 0, 0, 1, 0, 81, 534, 18, 0, 0, 4096, 802, 0, 0, 0, 0, 0, 0, 1853, 1805, 0, 79, 0, 0, 0, 0, 3, 666, 0, 30, 3646, 0, 0, 0, 0, 419, 3188, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6317, 6197, 3033, 634, 4898, 3737, 4345, 3607] avg=4096.0000 max=6317 max_rank=0 min=634 min_rank=3 imbalance_ratio=1.542236 max_load_reduction=26451 max_load_reduction_pct=0.807220 moved_to_echo=16587 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[34, 1, 0, 0, 1702, 806, 0, 0, 3, 0, 2, 2, 2426, 601, 0, 0, 0, 36, 0, 0, 3492, 278, 0, 0, 0, 0, 0, 4051, 532, 144, 0, 0, 4096, 825, 1, 12, 0, 0, 0, 0, 1861, 4093, 0, 0, 0, 0, 0, 0, 2, 35, 0, 0, 3800, 0, 0, 0, 0, 3933, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2543, 3034, 3806, 4727, 4934, 5954, 3837, 3933] avg=4096.0000 max=5954 max_rank=5 min=2543 min_rank=0 imbalance_ratio=1.453613 max_load_reduction=26814 max_load_reduction_pct=0.818298 moved_to_echo=18658 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[14976, 128, 384, 256, 10752, 6400, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[11776, 20992, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 256, 8704, 19328, 4736, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[26112, 6528, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 256, 3328, 128, 27008, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 2432, 128, 128, 30336, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 128, 128, 28160, 3456, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[384, 27520, 4992, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1, 4096, 4040, 0, 1, 0, 1418, 4096, 4056, 0, 2173, 0, 38, 0, 0, 0, 0, 6, 0, 0, 4096, 41, 0, 0, 0, 520, 1, 4091, 4089, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[30, 4016, 4074, 0, 2, 0, 4089, 4096, 0, 2, 3846, 85, 2, 0, 5, 0, 1, 5, 1, 0, 4094, 4083, 0, 47, 0, 222, 0, 1, 4036, 0, 31, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[157, 3924, 4080, 0, 0, 0, 4096, 4094, 3814, 0, 275, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 3665, 0, 2, 367, 7, 0, 0, 4085, 10, 74, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[169, 4096, 4096, 0, 0, 0, 473, 4096, 2, 0, 4095, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 0, 0, 11, 4, 0, 0, 4096, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[337, 4096, 4004, 0, 95, 0, 3806, 4096, 20, 0, 4091, 6, 1837, 16, 0, 5, 0, 1, 0, 16, 4085, 500, 0, 0, 0, 0, 0, 1649, 4096, 0, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 3649, 4085, 0, 54, 334, 2452, 4090, 0, 0, 4060, 80, 3430, 0, 16, 0, 0, 0, 0, 34, 1921, 35, 26, 0, 3794, 293, 2, 317, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[114, 4096, 2934, 24, 0, 0, 1839, 4026, 0, 0, 3951, 0, 3218, 0, 8, 0, 4, 0, 0, 0, 4091, 4050, 0, 104, 199, 15, 0, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[2, 4084, 4092, 0, 0, 9, 4096, 4089, 281, 1, 1304, 2, 1115, 0, 0, 1, 1, 0, 0, 0, 4093, 1536, 0, 1, 0, 1222, 922, 1811, 4095, 11, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 68, 4040, 0, 0, 0, 0, 0, 1, 0, 1418, 1255, 0, 0, 0, 0, 4056, 0, 2173, 0, 75, 8, 0, 0, 38, 0, 0, 0, 2456, 0, 0, 0, 0, 6, 0, 0, 4028, 146, 0, 0, 1956, 41, 0, 0, 0, 0, 0, 0, 0, 520, 1, 4091, 2065, 239, 0, 0, 4081, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 2674, 6312, 2494, 4180, 1997, 6916, 4086] avg=4096.0000 max=6916 max_rank=6 min=1997 min_rank=5 imbalance_ratio=1.688477 max_load_reduction=25852 max_load_reduction_pct=0.788940 moved_to_echo=17179 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[30, 67, 4074, 0, 0, 0, 0, 0, 2, 0, 4089, 1255, 0, 0, 0, 0, 0, 2, 3846, 85, 75, 7, 0, 0, 2, 0, 5, 0, 2456, 0, 0, 0, 1, 5, 1, 0, 3949, 146, 0, 0, 1955, 4083, 0, 47, 0, 0, 0, 0, 0, 222, 0, 1, 2064, 239, 0, 0, 4029, 0, 31, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 5346, 4015, 2463, 4102, 6085, 2526, 4060] avg=4096.0000 max=6085 max_rank=5 min=2463 min_rank=3 imbalance_ratio=1.485596 max_load_reduction=26683 max_load_reduction_pct=0.814301 moved_to_echo=16773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[169, 68, 4096, 0, 0, 0, 0, 0, 0, 0, 473, 1255, 0, 0, 0, 0, 2, 0, 4095, 0, 74, 8, 0, 0, 3457, 0, 0, 0, 2456, 0, 0, 0, 0, 0, 0, 0, 4028, 146, 0, 0, 1947, 4096, 0, 0, 0, 0, 0, 0, 11, 4, 0, 0, 2055, 239, 0, 0, 4088, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4333, 1728, 4179, 5913, 4174, 6043, 2309, 4089] avg=4096.0000 max=6043 max_rank=5 min=1728 min_rank=1 imbalance_ratio=1.475342 max_load_reduction=26725 max_load_reduction_pct=0.815582 moved_to_echo=16615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[157, 65, 4080, 0, 0, 0, 0, 0, 0, 0, 4096, 1255, 0, 0, 0, 0, 3814, 0, 275, 0, 75, 7, 0, 0, 0, 0, 22, 0, 2454, 0, 0, 0, 0, 0, 0, 0, 3859, 146, 0, 0, 1956, 3665, 0, 2, 0, 0, 0, 0, 367, 7, 0, 0, 2065, 239, 0, 0, 4078, 10, 74, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4302, 5351, 4171, 2476, 4005, 5623, 2678, 4162] avg=4096.0000 max=5623 max_rank=5 min=2476 min_rank=3 imbalance_ratio=1.372803 max_load_reduction=27145 max_load_reduction_pct=0.828400 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[337, 68, 4004, 0, 0, 0, 0, 0, 95, 0, 3806, 1255, 0, 0, 0, 0, 20, 0, 4091, 6, 74, 8, 0, 0, 1837, 16, 0, 5, 2456, 0, 0, 0, 0, 1, 0, 16, 4028, 146, 0, 0, 1951, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1649, 2060, 239, 0, 0, 4088, 0, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4409, 5156, 4199, 4314, 4191, 2451, 3948, 4100] avg=4096.0000 max=5156 max_rank=1 min=2451 min_rank=5 imbalance_ratio=1.258789 max_load_reduction=27612 max_load_reduction_pct=0.842651 moved_to_echo=14690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[114, 68, 2934, 24, 0, 0, 0, 0, 0, 0, 1839, 1233, 0, 0, 0, 0, 0, 0, 3951, 0, 74, 8, 0, 0, 3218, 0, 8, 0, 2414, 0, 0, 0, 4, 0, 0, 0, 4028, 144, 0, 0, 1954, 4050, 0, 104, 0, 0, 0, 0, 199, 15, 0, 0, 2063, 235, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3140, 3072, 4033, 5640, 4176, 6108, 2512, 4087] avg=4096.0000 max=6108 max_rank=5 min=2512 min_rank=6 imbalance_ratio=1.491211 max_load_reduction=26660 max_load_reduction_pct=0.813599 moved_to_echo=16883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 57, 4085, 0, 0, 0, 0, 0, 54, 334, 2452, 1242, 0, 0, 0, 0, 0, 0, 4060, 80, 38, 10, 0, 0, 3430, 0, 16, 0, 2455, 0, 0, 0, 0, 0, 0, 34, 3592, 151, 0, 0, 909, 35, 26, 0, 0, 0, 0, 0, 3794, 293, 2, 317, 974, 242, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4082, 4188, 5901, 3777, 970, 5622, 4086] avg=4096.0000 max=5901 max_rank=3 min=970 min_rank=5 imbalance_ratio=1.440674 max_load_reduction=26867 max_load_reduction_pct=0.819916 moved_to_echo=14455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 68, 4092, 0, 0, 0, 0, 0, 0, 9, 4096, 1254, 0, 0, 0, 0, 281, 1, 1304, 2, 75, 8, 0, 0, 1115, 0, 0, 1, 2451, 0, 0, 0, 1, 0, 0, 0, 4016, 146, 0, 0, 1954, 1536, 0, 1, 0, 0, 0, 0, 0, 1222, 922, 1811, 2064, 238, 0, 0, 4087, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 5359, 1671, 3567, 4163, 3491, 6257, 4098] avg=4096.0000 max=6257 max_rank=6 min=1671 min_rank=2 imbalance_ratio=1.527588 max_load_reduction=26511 max_load_reduction_pct=0.809052 moved_to_echo=18009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14592, 18048, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4480, 2304, 1024, 7936, 15488, 1920, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[13184, 128, 128, 128, 19712, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 384, 22272, 10112, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 640, 31488, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 128, 23808, 256, 640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 31616, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 3893, 4096, 0, 4096, 0, 1, 139, 4096, 0, 4095, 0, 0, 0, 3485, 0, 0, 596, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 3511, 316, 0, 2, 0, 0, 664, 12, 0, 0, 0, 4082, 4091, 0, 3983, 0, 0, 0, 4091, 50, 4042, 2, 0, 0, 72, 12, 0, 3742, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[27, 3267, 0, 0, 60, 0, 0, 378, 297, 0, 0, 30, 3595, 4065, 0, 3968, 0, 0, 803, 4019, 0, 4092, 0, 0, 0, 2, 2, 5, 4064, 27, 4067, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4089, 48, 0, 0, 0, 0, 0, 124, 0, 5, 154, 3923, 4087, 0, 1035, 0, 0, 359, 4088, 0, 3856, 0, 3, 121, 3145, 0, 0, 3637, 0, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[20, 3256, 0, 0, 18, 2, 0, 0, 19, 36, 0, 0, 3673, 4061, 0, 4096, 7, 797, 437, 4096, 0, 4078, 0, 10, 0, 30, 0, 0, 4059, 0, 4073, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4078, 9, 0, 0, 6, 12, 3, 226, 78, 0, 0, 3682, 4052, 0, 3901, 0, 195, 0, 4084, 0, 3911, 0, 18, 8, 3978, 43, 0, 282, 0, 4096, 106] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 216, 0, 0, 31, 0, 3854, 0, 0, 1, 51, 4077, 4095, 0, 0, 24, 0, 0, 4096, 9, 4043, 34, 0, 0, 3990, 0, 0, 64, 0, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[18, 4095, 3530, 0, 0, 5, 14, 88, 1, 0, 0, 0, 1499, 4092, 205, 1433, 0, 1714, 1, 3940, 0, 3911, 1, 3, 1030, 0, 108, 0, 2964, 0, 4087, 29] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3293, 316, 0, 0, 0, 0, 0, 2, 0, 0, 664, 2442, 1026, 113, 0, 12, 0, 0, 0, 3978, 0, 0, 0, 1444, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 3561, 0, 0, 0, 0, 50, 4042, 2, 0, 70, 0, 0, 0, 0, 72, 12, 0, 1612, 530, 148, 0, 3742, 0, 1654, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3609, 4247, 3990, 5427, 3561, 4164, 2374, 5396] avg=4096.0000 max=5427 max_rank=3 min=2374 min_rank=6 imbalance_ratio=1.324951 max_load_reduction=27341 max_load_reduction_pct=0.834381 moved_to_echo=15495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 2442, 978, 114, 0, 0, 0, 0, 0, 3982, 0, 0, 0, 1378, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 139, 3565, 0, 0, 0, 0, 0, 4095, 0, 0, 81, 0, 0, 0, 0, 3485, 0, 0, 1537, 531, 173, 0, 596, 0, 1654, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3839, 3616, 3982, 5474, 3705, 4176, 5726, 2250] avg=4096.0000 max=5726 max_rank=6 min=2250 min_rank=7 imbalance_ratio=1.397949 max_load_reduction=27042 max_load_reduction_pct=0.825256 moved_to_echo=15857 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3835, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2441, 986, 113, 0, 124, 0, 5, 154, 3974, 0, 0, 0, 1388, 0, 0, 1035, 0, 0, 0, 0, 0, 0, 359, 3558, 0, 0, 0, 0, 0, 3856, 0, 3, 81, 0, 0, 0, 121, 3145, 0, 0, 1549, 530, 173, 0, 3637, 0, 1653, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3883, 3540, 4257, 2423, 3917, 3940, 5518, 5290] avg=4096.0000 max=5518 max_rank=6 min=2423 min_rank=3 imbalance_ratio=1.347168 max_load_reduction=27250 max_load_reduction_pct=0.831604 moved_to_echo=18665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[27, 3064, 0, 0, 0, 0, 0, 0, 60, 0, 0, 378, 2425, 903, 112, 0, 297, 0, 0, 30, 3953, 0, 0, 0, 1273, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 803, 3498, 0, 0, 0, 0, 0, 4092, 0, 0, 65, 0, 0, 0, 0, 2, 2, 5, 1419, 521, 138, 0, 4064, 27, 1642, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3091, 3878, 4280, 5241, 4301, 4157, 2087, 5733] avg=4096.0000 max=5733 max_rank=7 min=2087 min_rank=6 imbalance_ratio=1.399658 max_load_reduction=27035 max_load_reduction_pct=0.825043 moved_to_echo=16278 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[20, 3054, 0, 0, 0, 0, 0, 0, 18, 2, 0, 0, 2428, 923, 113, 0, 19, 36, 0, 0, 3948, 0, 0, 0, 1300, 0, 0, 4096, 0, 0, 0, 0, 7, 797, 437, 3565, 0, 0, 0, 0, 0, 4078, 0, 10, 65, 0, 0, 0, 0, 30, 0, 0, 1450, 531, 137, 0, 4059, 0, 1645, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3074, 3484, 4003, 5396, 4806, 4153, 2148, 5704] avg=4096.0000 max=5704 max_rank=7 min=2148 min_rank=6 imbalance_ratio=1.392578 max_load_reduction=27064 max_load_reduction_pct=0.825928 moved_to_echo=16811 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3825, 9, 0, 0, 0, 0, 0, 0, 6, 12, 3, 2442, 925, 112, 0, 226, 78, 0, 0, 3940, 0, 0, 0, 1303, 0, 0, 3901, 0, 0, 0, 0, 0, 195, 0, 3554, 0, 0, 0, 0, 0, 3911, 0, 18, 81, 0, 0, 0, 8, 3978, 43, 0, 1454, 530, 172, 0, 282, 0, 1654, 106, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3834, 3500, 4244, 5204, 3749, 4010, 6185, 2042] avg=4096.0000 max=6185 max_rank=6 min=2042 min_rank=7 imbalance_ratio=1.510010 max_load_reduction=26583 max_load_reduction_pct=0.811249 moved_to_echo=15986 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[18, 3832, 3530, 0, 0, 0, 0, 0, 0, 5, 14, 88, 2441, 381, 114, 0, 1, 0, 0, 0, 3978, 0, 0, 0, 522, 0, 205, 1433, 0, 0, 0, 0, 0, 1714, 1, 3425, 0, 0, 0, 0, 0, 3911, 1, 3, 86, 0, 0, 0, 1030, 0, 108, 0, 596, 515, 177, 0, 2964, 0, 1646, 29, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7380, 3043, 3979, 2160, 5140, 4001, 2426, 4639] avg=4096.0000 max=7380 max_rank=0 min=2160 min_rank=3 imbalance_ratio=1.801758 max_load_reduction=25388 max_load_reduction_pct=0.774780 moved_to_echo=16206 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3842, 216, 0, 0, 0, 0, 0, 0, 31, 0, 3854, 2437, 1024, 114, 0, 0, 0, 1, 51, 3981, 0, 0, 0, 1443, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 3565, 0, 0, 0, 0, 9, 4043, 34, 0, 81, 0, 0, 0, 0, 3990, 0, 0, 1610, 531, 173, 0, 64, 0, 1650, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 7460, 4033, 1443, 3589, 4167, 6304, 1714] avg=4096.0000 max=7460 max_rank=1 min=1443 min_rank=3 imbalance_ratio=1.821289 max_load_reduction=25308 max_load_reduction_pct=0.772339 moved_to_echo=15774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 5120, 19584, 7168, 1024, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[10112, 0, 256, 22528, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19456, 128, 13312, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1280, 14720, 256, 128, 11264, 4224, 1408, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 2816, 1792, 28416, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 28672, 4224, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32128, 128, 128, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 128, 128, 256, 31744, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[75, 0, 3693, 0, 0, 37, 4096, 4088, 0, 6, 0, 214, 0, 4094, 0, 0, 4082, 0, 4061, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 144, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[3466, 0, 254, 0, 0, 293, 4010, 106, 0, 37, 0, 3806, 0, 4028, 2, 0, 4058, 0, 4078, 89, 0, 650, 120, 0, 3682, 0, 63, 0, 0, 0, 17, 4009] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[2424, 0, 854, 0, 0, 0, 3952, 4036, 2, 0, 9, 778, 141, 4076, 2, 5, 2804, 6, 4096, 0, 6, 6, 0, 0, 2924, 0, 0, 0, 0, 2557, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3676, 0, 869, 0, 0, 3571, 4081, 4081, 0, 2, 0, 77, 0, 4095, 0, 23, 4063, 0, 3969, 0, 169, 139, 13, 0, 3, 0, 24, 0, 0, 0, 0, 3913] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4042, 222, 2198, 53, 0, 0, 1687, 4036, 0, 0, 30, 4024, 0, 3952, 0, 6, 3025, 3, 2947, 0, 0, 7, 0, 1, 2419, 0, 0, 21, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 65, 4033, 10, 0, 0, 4053, 4096, 467, 0, 1, 458, 219, 3914, 0, 0, 3648, 0, 4084, 95, 0, 1, 0, 4, 3233, 0, 1, 22, 0, 321, 24, 4019] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[58, 2, 3762, 0, 0, 1398, 4096, 3564, 0, 3, 0, 3986, 12, 3477, 26, 0, 3837, 0, 4085, 0, 39, 4, 147, 0, 22, 3, 0, 157, 0, 0, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[498, 0, 185, 127, 0, 3, 4075, 3924, 0, 51, 0, 3636, 0, 616, 533, 0, 4089, 0, 3551, 0, 0, 38, 0, 7, 3924, 3, 0, 393, 0, 3669, 0, 3446] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2424, 0, 854, 0, 223, 67, 0, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 2, 0, 9, 778, 2014, 0, 0, 0, 141, 4076, 2, 5, 456, 0, 0, 0, 2804, 6, 395, 0, 0, 0, 0, 0, 6, 6, 0, 0, 3952, 181, 0, 0, 2924, 0, 0, 0, 1506, 56, 0, 0, 0, 2557, 0, 3355, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3568, 3969, 2803, 4680, 3205, 4145, 4486, 5912] avg=4096.0000 max=5912 max_rank=7 min=2803 min_rank=2 imbalance_ratio=1.443359 max_load_reduction=26856 max_load_reduction_pct=0.819580 moved_to_echo=17748 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3466, 0, 254, 0, 219, 1, 0, 0, 0, 293, 0, 105, 0, 0, 0, 0, 0, 37, 0, 3806, 2005, 0, 0, 0, 0, 4028, 2, 0, 447, 0, 0, 0, 4058, 0, 394, 89, 0, 0, 0, 0, 0, 650, 120, 0, 4010, 180, 0, 0, 3682, 0, 63, 0, 1499, 55, 0, 0, 0, 0, 17, 3288, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3940, 398, 5848, 4477, 4541, 4960, 5299, 3305] avg=4096.0000 max=5848 max_rank=2 min=398 min_rank=1 imbalance_ratio=1.427734 max_load_reduction=26920 max_load_reduction_pct=0.821533 moved_to_echo=18105 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[75, 0, 3693, 0, 224, 68, 0, 0, 0, 37, 0, 4020, 0, 0, 0, 0, 0, 6, 0, 214, 1997, 0, 0, 0, 0, 4094, 0, 0, 457, 0, 0, 0, 4082, 0, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 179, 0, 0, 4082, 0, 0, 0, 1493, 56, 0, 0, 0, 144, 0, 3359, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4057, 2217, 4551, 4474, 4275, 5631, 3503] avg=4096.0000 max=5631 max_rank=6 min=2217 min_rank=2 imbalance_ratio=1.374756 max_load_reduction=27137 max_load_reduction_pct=0.828156 moved_to_echo=17883 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3676, 0, 869, 0, 214, 67, 0, 0, 0, 3571, 0, 4014, 0, 0, 0, 0, 0, 2, 0, 77, 1951, 0, 0, 0, 0, 4095, 0, 23, 437, 0, 0, 0, 4063, 0, 384, 0, 0, 0, 0, 0, 169, 139, 13, 0, 4081, 175, 0, 0, 3, 0, 24, 0, 1459, 53, 0, 0, 0, 0, 0, 3209, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4826, 7585, 2030, 4555, 4447, 4577, 1539, 3209] avg=4096.0000 max=7585 max_rank=1 min=1539 min_rank=6 imbalance_ratio=1.851807 max_load_reduction=25183 max_load_reduction_pct=0.768524 moved_to_echo=13772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 65, 4033, 10, 219, 68, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 467, 0, 1, 458, 2008, 0, 0, 0, 219, 3914, 0, 0, 448, 0, 0, 0, 3648, 0, 394, 95, 0, 0, 0, 0, 0, 1, 0, 4, 4053, 180, 0, 0, 3233, 0, 1, 22, 1502, 55, 0, 0, 0, 321, 24, 3297, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4395, 4028, 2934, 4581, 4137, 4238, 4813, 3642] avg=4096.0000 max=4813 max_rank=6 min=2934 min_rank=2 imbalance_ratio=1.175049 max_load_reduction=27955 max_load_reduction_pct=0.853119 moved_to_echo=16830 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4042, 222, 2198, 53, 223, 67, 0, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 30, 4024, 1449, 0, 0, 0, 0, 3952, 0, 6, 457, 0, 0, 0, 3025, 3, 285, 0, 0, 0, 0, 0, 0, 7, 0, 1, 1687, 130, 0, 0, 2419, 0, 0, 21, 1083, 56, 0, 0, 0, 0, 0, 3359, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6805, 3969, 5503, 4415, 3313, 1825, 3579, 3359] avg=4096.0000 max=6805 max_rank=0 min=1825 min_rank=5 imbalance_ratio=1.661377 max_load_reduction=25963 max_load_reduction_pct=0.792328 moved_to_echo=12076 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[498, 0, 185, 127, 192, 68, 0, 0, 0, 3, 0, 3856, 0, 0, 0, 0, 0, 51, 0, 3636, 1749, 0, 0, 0, 0, 616, 533, 0, 389, 0, 0, 0, 4089, 0, 330, 0, 0, 0, 0, 0, 0, 38, 0, 7, 4075, 162, 0, 0, 3924, 3, 0, 393, 1310, 51, 0, 0, 0, 3669, 0, 2814, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1070, 3859, 5436, 1538, 4419, 4282, 5681, 6483] avg=4096.0000 max=6483 max_rank=7 min=1070 min_rank=0 imbalance_ratio=1.582764 max_load_reduction=26285 max_load_reduction_pct=0.802155 moved_to_echo=20865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[58, 2, 3762, 0, 223, 59, 0, 0, 0, 1398, 0, 3505, 0, 0, 0, 0, 0, 3, 0, 3986, 2008, 0, 0, 0, 12, 3477, 26, 0, 456, 0, 0, 0, 3837, 0, 395, 0, 0, 0, 0, 0, 39, 4, 147, 0, 4096, 180, 0, 0, 22, 3, 0, 157, 1502, 56, 0, 0, 0, 0, 0, 3355, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4903, 5997, 3971, 4232, 4466, 1740, 3355] avg=4096.0000 max=5997 max_rank=2 min=1740 min_rank=6 imbalance_ratio=1.464111 max_load_reduction=26771 max_load_reduction_pct=0.816986 moved_to_echo=13793 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 128, 128, 17024, 15232, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[20352, 128, 128, 640, 11392, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14336, 384, 15872, 256, 1792, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 5376, 0, 27520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[29696, 128, 3072, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 6784, 128, 26112, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 28288, 640, 128, 3584, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 896, 384, 128, 30080, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 4096, 87, 0, 4093, 29, 0, 792, 3280, 4, 0, 0, 50, 0, 0, 0, 2937, 2335, 929, 1, 0, 0, 1954, 3995, 0, 6, 250, 26, 3834] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4083, 14, 0, 0, 10, 0, 1, 4089, 2551, 0, 0, 0, 3, 57, 0, 122, 5, 46, 0, 4043, 224, 1715, 1, 0, 0, 4091, 3793, 0, 3924, 0, 0, 3996] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4081, 49, 0, 0, 317, 5, 2537, 2274, 4087, 0, 3917, 0, 12, 0, 0, 0, 0, 81, 0, 2190, 1036, 1, 0, 0, 0, 4096, 11, 0, 4057, 5, 2, 4010] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4065, 0, 0, 15, 3884, 0, 1, 4085, 3458, 5, 0, 16, 0, 0, 6, 0, 0, 0, 0, 3789, 0, 429, 3, 0, 0, 3996, 4096, 24, 691, 153, 0, 4052] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4094, 0, 1949, 56, 105, 0, 11, 4096, 3588, 34, 9, 1990, 0, 18, 4021, 0, 0, 1659, 0, 28, 840, 135, 0, 0, 6, 1581, 1416, 72, 2964, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[3700, 246, 0, 1, 39, 2, 7, 4086, 3977, 91, 188, 0, 897, 0, 3629, 1434, 8, 0, 1118, 3985, 2871, 16, 0, 0, 0, 2227, 0, 0, 191, 0, 0, 4055] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3360, 0, 834, 4085, 2153, 0, 3955, 137, 0, 298, 0, 0, 0, 68, 0, 0, 2236, 2315, 0, 0, 0, 29, 4096, 0, 1010, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4095, 54, 0, 0, 3804, 2, 0, 4082, 3927, 0, 601, 1097, 0, 0, 0, 0, 0, 248, 0, 2152, 0, 1005, 0, 1430, 4, 1916, 4076, 0, 225, 20, 0, 4030] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3843, 14, 0, 0, 0, 0, 0, 0, 10, 0, 1, 1810, 0, 0, 0, 0, 1788, 0, 0, 0, 0, 0, 0, 0, 3, 57, 0, 122, 2279, 620, 0, 0, 5, 46, 0, 4043, 763, 240, 246, 0, 224, 1715, 1, 0, 985, 1291, 0, 0, 0, 4091, 2256, 0, 0, 0, 0, 0, 3924, 0, 0, 2391, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3857, 1821, 1788, 3081, 5343, 4216, 6347, 6315] avg=4096.0000 max=6347 max_rank=6 min=1788 min_rank=2 imbalance_ratio=1.549561 max_load_reduction=26421 max_load_reduction_pct=0.806305 moved_to_echo=22221 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3854, 0, 1949, 56, 0, 0, 0, 0, 105, 0, 11, 1813, 0, 0, 0, 0, 2515, 34, 9, 1990, 0, 0, 0, 0, 0, 18, 4021, 0, 2283, 635, 0, 0, 0, 1659, 0, 28, 1073, 240, 92, 0, 840, 135, 0, 0, 1009, 482, 0, 0, 6, 1581, 842, 72, 0, 0, 0, 0, 2964, 0, 0, 2452, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5859, 1929, 4548, 6957, 3092, 2466, 2501, 5416] avg=4096.0000 max=6957 max_rank=3 min=1929 min_rank=1 imbalance_ratio=1.698486 max_load_reduction=25811 max_load_reduction_pct=0.787689 moved_to_echo=13475 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3841, 49, 0, 0, 0, 0, 0, 0, 317, 5, 2537, 1007, 0, 0, 0, 0, 2864, 0, 3917, 0, 0, 0, 0, 0, 12, 0, 0, 0, 1267, 622, 0, 0, 0, 81, 0, 2190, 1223, 240, 0, 0, 1036, 1, 0, 0, 988, 3, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 4057, 5, 2, 2400, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3890, 3866, 6781, 1901, 3734, 2028, 4104, 6464] avg=4096.0000 max=6781 max_rank=2 min=1901 min_rank=3 imbalance_ratio=1.655518 max_load_reduction=25987 max_load_reduction_pct=0.793060 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3831, 0, 0, 0, 0, 0, 0, 0, 4096, 87, 0, 1811, 0, 0, 0, 0, 21, 0, 792, 3280, 0, 0, 0, 0, 4, 0, 0, 50, 2282, 595, 0, 0, 0, 0, 0, 2937, 8, 239, 259, 0, 2335, 929, 1, 0, 945, 1360, 0, 0, 0, 1954, 2376, 0, 0, 0, 0, 0, 6, 250, 26, 2294, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3831, 5994, 4093, 2931, 3443, 5570, 4330, 2576] avg=4096.0000 max=5994 max_rank=1 min=2576 min_rank=7 imbalance_ratio=1.463379 max_load_reduction=26774 max_load_reduction_pct=0.817078 moved_to_echo=15919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3826, 0, 0, 15, 0, 0, 0, 0, 3884, 0, 1, 1808, 0, 0, 0, 0, 2424, 5, 0, 16, 0, 0, 0, 0, 0, 0, 6, 0, 2277, 628, 0, 0, 0, 0, 0, 3789, 1034, 239, 266, 0, 0, 429, 3, 0, 999, 1394, 0, 0, 0, 3996, 2436, 24, 0, 0, 0, 0, 691, 153, 0, 2425, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3841, 5693, 2445, 2911, 5328, 2825, 6456, 3269] avg=4096.0000 max=6456 max_rank=6 min=2445 min_rank=2 imbalance_ratio=1.576172 max_load_reduction=26312 max_load_reduction_pct=0.802979 moved_to_echo=17878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3855, 0, 0, 0, 0, 0, 0, 0, 3360, 0, 834, 1808, 0, 0, 0, 0, 1509, 0, 3955, 137, 0, 0, 0, 0, 0, 298, 0, 0, 2277, 635, 0, 0, 0, 68, 0, 0, 644, 241, 266, 0, 2236, 2315, 0, 0, 1009, 1394, 0, 0, 0, 29, 2436, 0, 0, 0, 0, 0, 1010, 0, 0, 2452, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3855, 6002, 5601, 3210, 1219, 6954, 2465, 3462] avg=4096.0000 max=6954 max_rank=5 min=1219 min_rank=4 imbalance_ratio=1.697754 max_load_reduction=25814 max_load_reduction_pct=0.787781 moved_to_echo=14100 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3480, 246, 0, 1, 0, 0, 0, 0, 39, 2, 7, 1803, 0, 0, 0, 0, 2783, 91, 188, 0, 0, 0, 0, 0, 897, 0, 3629, 1434, 2283, 633, 0, 0, 8, 0, 1118, 3985, 1194, 220, 0, 0, 2871, 16, 0, 0, 1004, 0, 0, 0, 0, 2227, 0, 0, 0, 0, 0, 0, 191, 0, 0, 2418, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3727, 1851, 3062, 8876, 6525, 3891, 2227, 2609] avg=4096.0000 max=8876 max_rank=3 min=1851 min_rank=1 imbalance_ratio=2.166992 max_load_reduction=23892 max_load_reduction_pct=0.729126 moved_to_echo=15252 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3854, 54, 0, 0, 0, 0, 0, 0, 3804, 2, 0, 1806, 0, 0, 0, 0, 2752, 0, 601, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 2276, 625, 0, 0, 0, 248, 0, 2152, 1175, 241, 268, 0, 0, 1005, 0, 1430, 993, 1390, 0, 0, 4, 1916, 2418, 0, 0, 0, 0, 0, 225, 20, 0, 2412, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3908, 5612, 4450, 2901, 4084, 4818, 4338, 2657] avg=4096.0000 max=5612 max_rank=1 min=2657 min_rank=7 imbalance_ratio=1.370117 max_load_reduction=27156 max_load_reduction_pct=0.828735 moved_to_echo=15897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9600, 6656, 128, 1536, 7936, 7424, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 384, 7680, 1664, 17280, 5120, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 19968, 12800, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 2176, 1152, 19200, 7168, 1920, 1408, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[15616, 128, 3456, 13696, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13184, 512, 128, 19328, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16768, 256, 9472, 6528, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[30464, 384, 2048, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4093, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 4096, 12, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[92, 45, 25, 0, 25, 0, 1, 284, 0, 1, 54, 3996, 47, 0, 38, 5, 4072, 4053, 4068, 0, 4096, 0, 65, 4022, 0, 0, 0, 616, 4028, 3125, 0, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 69, 21, 0, 9, 0, 0, 64, 11, 0, 0, 3963, 4094, 1, 0, 0, 4080, 4093, 4084, 2, 4033, 0, 36, 4093, 12, 0, 0, 0, 4095, 0, 4, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 6, 0, 7, 0, 156, 0, 4091, 0, 144, 9, 0, 3935, 4083, 3952, 11, 4096, 0, 0, 4078, 9, 0, 0, 0, 4096, 4089, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 56, 0, 0, 1, 1272, 0, 0, 2450, 0, 4067, 0, 7, 10, 244, 4092, 4048, 4096, 0, 3999, 34, 0, 3923, 367, 0, 0, 0, 3913, 8, 27, 154] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[12, 0, 1, 0, 0, 0, 10, 1, 0, 3979, 7, 4085, 119, 0, 0, 0, 4092, 4096, 4091, 0, 4025, 0, 0, 4085, 39, 0, 0, 53, 3976, 0, 0, 97] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 133, 14, 1, 62, 4088, 378, 0, 15, 3850, 4096, 3893, 4096, 0, 4096, 0, 277, 3473, 4096, 0, 0, 15, 184, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[18, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 4065, 9, 0, 0, 319, 4075, 4066, 4092, 0, 4096, 0, 0, 4052, 3769, 0, 3, 71, 4096, 0, 0, 32] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 69, 21, 0, 4065, 0, 0, 0, 9, 0, 0, 64, 3861, 19, 0, 0, 11, 0, 0, 2682, 0, 0, 0, 0, 4094, 1, 0, 0, 1281, 1203, 470, 0, 42, 4093, 0, 2, 0, 0, 0, 0, 68, 0, 36, 4093, 0, 0, 0, 0, 12, 0, 0, 0, 2762, 177, 0, 0, 3625, 0, 4, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 3953, 2693, 7049, 4137, 4197, 2951, 3633] avg=4096.0000 max=7049 max_rank=3 min=2693 min_rank=2 imbalance_ratio=1.720947 max_load_reduction=25719 max_load_reduction_pct=0.784882 moved_to_echo=14918 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 3876, 19, 0, 0, 0, 4080, 0, 2770, 0, 0, 0, 0, 0, 0, 0, 0, 1323, 1222, 470, 0, 42, 4096, 0, 0, 0, 0, 0, 0, 68, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 0, 2806, 178, 0, 0, 3626, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 3895, 6850, 3015, 4138, 4164, 2996, 3626] avg=4096.0000 max=6850 max_rank=2 min=2996 min_rank=6 imbalance_ratio=1.672363 max_load_reduction=25918 max_load_reduction_pct=0.790955 moved_to_echo=14924 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 56, 0, 4076, 0, 0, 0, 0, 1, 1272, 0, 3872, 20, 0, 0, 0, 2450, 0, 2752, 0, 0, 0, 0, 0, 7, 10, 244, 1315, 1193, 449, 0, 43, 4048, 0, 0, 0, 0, 0, 0, 67, 34, 0, 3923, 0, 0, 0, 0, 367, 0, 0, 0, 2739, 177, 0, 0, 3464, 8, 27, 154, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 5165, 5202, 3218, 4091, 4024, 3283, 3653] avg=4096.0000 max=5202 max_rank=2 min=3218 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=27566 max_load_reduction_pct=0.841248 moved_to_echo=15051 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[92, 45, 25, 0, 4049, 0, 0, 0, 25, 0, 1, 284, 3853, 19, 0, 0, 0, 1, 54, 2704, 0, 0, 0, 0, 47, 0, 38, 5, 1292, 1222, 462, 0, 42, 4053, 0, 0, 0, 0, 0, 0, 68, 0, 65, 4022, 0, 0, 0, 0, 0, 0, 0, 616, 2806, 177, 0, 0, 3566, 3125, 0, 10, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4211, 4182, 2759, 3066, 4095, 4155, 3599, 6701] avg=4096.0000 max=6701 max_rank=7 min=2759 min_rank=2 imbalance_ratio=1.635986 max_load_reduction=26067 max_load_reduction_pct=0.795502 moved_to_echo=18550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3934, 0, 0, 0, 0, 6, 0, 7, 3724, 18, 0, 0, 0, 156, 0, 2768, 0, 0, 0, 0, 0, 144, 9, 0, 1323, 1222, 470, 0, 40, 4083, 0, 11, 0, 0, 0, 0, 68, 0, 0, 4078, 0, 0, 0, 0, 9, 0, 0, 0, 2806, 171, 0, 0, 3626, 4089, 0, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3934, 3755, 2924, 3168, 4134, 4146, 2986, 7721] avg=4096.0000 max=7721 max_rank=7 min=2924 min_rank=2 imbalance_ratio=1.885010 max_load_reduction=25047 max_load_reduction_pct=0.764374 moved_to_echo=18987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 1, 0, 4071, 0, 0, 0, 0, 0, 10, 1, 3872, 20, 0, 0, 0, 3979, 7, 2764, 0, 0, 0, 0, 119, 0, 0, 0, 1321, 1201, 456, 0, 43, 4096, 0, 0, 0, 0, 0, 0, 67, 0, 0, 4085, 0, 0, 0, 0, 39, 0, 0, 53, 2757, 177, 0, 0, 3520, 0, 0, 97, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 3903, 6750, 3097, 4139, 4152, 3026, 3617] avg=4096.0000 max=6750 max_rank=2 min=3026 min_rank=6 imbalance_ratio=1.647949 max_load_reduction=26018 max_load_reduction_pct=0.794006 moved_to_echo=14934 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4077, 0, 0, 0, 0, 1, 0, 133, 3882, 19, 0, 0, 14, 1, 62, 2762, 0, 0, 0, 0, 378, 0, 15, 3850, 1326, 1227, 25, 0, 33, 3893, 0, 0, 0, 0, 0, 0, 60, 0, 277, 3473, 0, 0, 0, 0, 4096, 0, 0, 15, 2809, 181, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4035, 2839, 6821, 3926, 3810, 7101, 159] avg=4096.0000 max=7101 max_rank=6 min=159 min_rank=7 imbalance_ratio=1.733643 max_load_reduction=25667 max_load_reduction_pct=0.783295 moved_to_echo=14996 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 0, 0, 4073, 0, 0, 0, 1, 0, 0, 4, 3856, 19, 0, 0, 0, 0, 0, 2751, 0, 0, 0, 0, 9, 0, 0, 319, 1314, 1222, 470, 0, 42, 4066, 0, 0, 0, 0, 0, 0, 68, 0, 0, 4052, 0, 0, 0, 0, 3769, 0, 3, 71, 2806, 177, 0, 0, 3626, 0, 0, 32, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 3880, 2751, 3334, 4108, 4120, 6826, 3658] avg=4096.0000 max=6826 max_rank=6 min=2751 min_rank=2 imbalance_ratio=1.666504 max_load_reduction=25942 max_load_reduction_pct=0.791687 moved_to_echo=18712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4736, 256, 128, 4480, 10496, 9728, 3328, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8320, 0, 128, 768, 22400, 1536, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[25216, 7296, 128, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 10752, 128, 22016, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 32512, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[640, 128, 384, 31872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 128, 0, 32512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 1408, 512, 30848, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 3951, 4, 4093, 0, 1, 3917, 818, 0, 11, 10, 160, 0, 0, 514, 0, 0, 4071, 105, 0, 1, 3280, 93, 0, 0, 3581, 0, 3, 4089, 4000, 66, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1, 4076, 4096, 4096, 16, 0, 4096, 0, 0, 251, 0, 0, 0, 0, 0, 0, 1, 0, 0, 31, 3967, 4043, 0, 143, 4, 4096, 0, 0, 0, 3809, 42, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1, 4084, 1869, 4094, 3997, 0, 4069, 1224, 6, 0, 0, 0, 22, 0, 0, 7, 0, 0, 0, 0, 1, 4081, 0, 0, 0, 4090, 0, 16, 1389, 3797, 17, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4073, 3961, 4085, 3442, 0, 4006, 2, 0, 27, 0, 1, 0, 0, 127, 0, 0, 0, 0, 0, 1597, 4096, 0, 0, 90, 4094, 0, 0, 8, 3092, 0, 67] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 3718, 4096, 8, 0, 4094, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 2331, 4062, 0, 0, 0, 4096, 200, 0, 0, 3367, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[24, 4063, 205, 4060, 0, 5, 4087, 52, 148, 0, 8, 0, 0, 0, 0, 15, 1, 4023, 0, 10, 10, 4082, 0, 0, 0, 4096, 1, 0, 3937, 3941, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[2, 4079, 4094, 0, 3679, 73, 4096, 3887, 0, 2354, 0, 0, 0, 0, 11, 0, 0, 351, 0, 978, 392, 4095, 0, 0, 0, 4062, 202, 4, 1, 408, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4085, 4030, 4084, 15, 0, 4031, 18, 0, 0, 0, 0, 0, 27, 0, 4, 0, 0, 8, 0, 784, 4094, 0, 1, 0, 4092, 4, 0, 4094, 3336, 61, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1869, 1541, 0, 0, 0, 0, 3997, 0, 1622, 1224, 0, 0, 0, 0, 6, 0, 0, 0, 2447, 1408, 58, 0, 22, 0, 0, 7, 4026, 0, 0, 0, 0, 0, 0, 0, 1074, 1145, 991, 0, 1, 3007, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 16, 5, 0, 0, 0, 1389, 2801, 17, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3411, 6843, 3919, 4055, 3210, 3008, 4111, 4211] avg=4096.0000 max=6843 max_rank=1 min=3008 min_rank=5 imbalance_ratio=1.670654 max_load_reduction=25925 max_load_reduction_pct=0.791168 moved_to_echo=14540 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 1541, 0, 0, 0, 0, 0, 1, 1562, 818, 0, 0, 0, 0, 0, 11, 10, 160, 2355, 1407, 56, 0, 0, 0, 514, 0, 3895, 0, 0, 0, 0, 4071, 105, 0, 863, 1145, 1044, 0, 1, 2417, 93, 0, 0, 0, 0, 0, 0, 3581, 0, 3, 5, 0, 0, 0, 4089, 2951, 66, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1545, 2381, 3999, 4409, 7228, 2511, 3589, 7106] avg=4096.0000 max=7228 max_rank=4 min=1545 min_rank=0 imbalance_ratio=1.764648 max_load_reduction=25540 max_load_reduction_pct=0.779419 moved_to_echo=20434 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[24, 0, 205, 1529, 0, 0, 0, 0, 0, 5, 1629, 52, 0, 0, 0, 0, 148, 0, 8, 0, 2458, 1396, 58, 0, 0, 0, 0, 15, 4005, 0, 0, 0, 1, 4023, 0, 10, 1075, 1135, 1029, 0, 10, 3007, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 5, 0, 0, 0, 3937, 2907, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1758, 1686, 4068, 4020, 7273, 3017, 4102, 6844] avg=4096.0000 max=7273 max_rank=4 min=1686 min_rank=1 imbalance_ratio=1.775635 max_load_reduction=25495 max_load_reduction_pct=0.778046 moved_to_echo=21236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3718, 1542, 0, 0, 0, 0, 8, 0, 1632, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 2462, 1409, 59, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 75, 1069, 1145, 879, 0, 2331, 2993, 0, 0, 0, 0, 0, 0, 0, 4096, 200, 0, 4, 0, 0, 0, 0, 2484, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5260, 4260, 3930, 4037, 3168, 5324, 4300, 2489] avg=4096.0000 max=5324 max_rank=5 min=2489 min_rank=7 imbalance_ratio=1.299805 max_load_reduction=27444 max_load_reduction_pct=0.837524 moved_to_echo=15281 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4096, 1542, 0, 0, 0, 0, 16, 0, 1633, 0, 0, 0, 0, 0, 0, 251, 0, 0, 2463, 1409, 58, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 1, 0, 0, 31, 1064, 1145, 994, 0, 3967, 2979, 0, 143, 0, 0, 0, 0, 4, 4096, 0, 0, 5, 0, 0, 0, 0, 2810, 42, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5639, 1649, 4181, 4018, 3235, 7089, 4105, 2852] avg=4096.0000 max=7089 max_rank=5 min=1649 min_rank=1 imbalance_ratio=1.730713 max_load_reduction=25679 max_load_reduction_pct=0.783661 moved_to_echo=17281 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3961, 1538, 0, 0, 0, 0, 3442, 0, 1597, 2, 0, 0, 0, 0, 0, 27, 0, 1, 2409, 1405, 59, 0, 0, 0, 127, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 1078, 1142, 807, 0, 1597, 3018, 0, 0, 0, 0, 0, 0, 90, 4094, 0, 0, 4, 0, 0, 0, 8, 2281, 0, 67, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5499, 5041, 3901, 4141, 3027, 4615, 4188, 2356] avg=4096.0000 max=5499 max_rank=0 min=2356 min_rank=7 imbalance_ratio=1.342529 max_load_reduction=27269 max_load_reduction_pct=0.832184 moved_to_echo=14186 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4094, 0, 0, 0, 0, 0, 3679, 73, 1629, 3887, 0, 0, 0, 0, 0, 2354, 0, 0, 2467, 0, 59, 0, 0, 0, 11, 0, 4020, 0, 0, 0, 0, 351, 0, 978, 1084, 0, 110, 0, 392, 3011, 0, 0, 0, 0, 0, 0, 0, 4062, 202, 4, 5, 0, 0, 0, 1, 293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 9268, 4880, 4031, 2523, 3403, 4273, 294] avg=4096.0000 max=9268 max_rank=1 min=294 min_rank=7 imbalance_ratio=2.262695 max_load_reduction=23500 max_load_reduction_pct=0.717163 moved_to_echo=10493 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4030, 1530, 0, 0, 0, 0, 15, 0, 1607, 18, 0, 0, 0, 0, 0, 0, 0, 0, 2424, 1407, 59, 0, 0, 27, 0, 4, 4026, 0, 0, 0, 0, 0, 8, 0, 1078, 1147, 871, 0, 784, 3016, 0, 1, 0, 0, 0, 0, 0, 4092, 4, 0, 4, 0, 0, 0, 4094, 2461, 61, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5560, 1640, 3890, 4057, 3104, 3801, 4100, 6616] avg=4096.0000 max=6616 max_rank=7 min=1640 min_rank=1 imbalance_ratio=1.615234 max_load_reduction=26152 max_load_reduction_pct=0.798096 moved_to_echo=17621 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 2688, 128, 256, 19584, 9856, 512, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 8448, 128, 1152, 8448, 8064, 6784, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11264, 128, 12928, 8704, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13568, 19072, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 22016, 10880, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9088, 23552, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 768, 128, 32128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32256, 512, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4095, 26, 3489, 0, 2, 4065, 4091, 0, 5, 1, 0, 0, 0, 171, 4096, 0, 0, 28, 4094, 0, 0, 2, 4071, 15, 0, 0, 0, 421, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 278, 4006, 0, 630, 1464, 2, 4093, 4084, 0, 0, 1, 29, 0, 33, 13, 4096, 0, 0, 0, 1700, 0, 0, 88, 4003, 0, 22, 0, 71, 1, 4058, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 3822, 11, 4075, 0, 0, 4083, 4092, 0, 0, 2, 0, 0, 0, 0, 4083, 0, 0, 12, 705, 0, 0, 24, 4014, 0, 81, 0, 215, 20, 4095, 3434] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[69, 1353, 4076, 0, 1185, 97, 118, 4096, 4062, 0, 0, 0, 0, 0, 2, 0, 3945, 0, 0, 0, 4050, 30, 11, 25, 3165, 0, 0, 15, 0, 141, 4079, 2249] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 3406, 4084, 0, 3747, 0, 0, 2480, 4043, 26, 0, 0, 763, 0, 119, 0, 4095, 0, 0, 48, 30, 107, 0, 6, 3487, 0, 332, 0, 0, 0, 3971, 2024] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[14, 1271, 3957, 0, 3406, 129, 0, 4093, 231, 0, 1041, 0, 0, 10, 0, 84, 4085, 2, 0, 81, 123, 0, 33, 0, 3677, 0, 454, 0, 0, 2070, 4048, 3959] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 141, 4096, 0, 4096, 2, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 0, 0, 31, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3992, 139, 1, 3958, 4091, 0, 0, 0, 0, 0, 25, 1, 4064, 11, 5, 0, 12, 69, 0, 0, 4084, 0, 0, 2, 0, 4043, 4096, 79] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3331, 26, 0, 0, 0, 0, 3489, 0, 2, 814, 0, 0, 0, 0, 4091, 0, 5, 1, 368, 0, 0, 0, 0, 0, 0, 171, 3363, 629, 0, 0, 4096, 0, 0, 28, 2, 0, 0, 0, 4094, 0, 0, 2, 2622, 214, 0, 0, 4071, 15, 0, 0, 180, 0, 0, 0, 0, 421, 733, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3357, 4305, 4465, 4163, 4126, 6932, 4266, 1154] avg=4096.0000 max=6932 max_rank=5 min=1154 min_rank=7 imbalance_ratio=1.692383 max_load_reduction=25836 max_load_reduction_pct=0.788452 moved_to_echo=16478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3406, 3322, 0, 0, 0, 0, 0, 3747, 0, 0, 497, 0, 0, 0, 0, 4043, 26, 0, 0, 367, 0, 0, 0, 763, 0, 119, 0, 3260, 384, 0, 0, 4095, 0, 0, 48, 2, 0, 0, 0, 30, 107, 0, 6, 1599, 213, 0, 0, 3487, 0, 332, 0, 180, 0, 0, 0, 0, 0, 711, 2024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6728, 4244, 4436, 4526, 4145, 1955, 3999, 2735] avg=4096.0000 max=6728 max_rank=0 min=1955 min_rank=5 imbalance_ratio=1.642578 max_load_reduction=26040 max_load_reduction_pct=0.794678 moved_to_echo=12834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3109, 11, 0, 0, 0, 0, 4075, 0, 0, 818, 0, 0, 0, 0, 4092, 0, 0, 2, 344, 0, 0, 0, 0, 0, 0, 0, 3362, 632, 0, 0, 4083, 0, 0, 12, 2, 0, 0, 0, 705, 0, 0, 24, 2633, 199, 0, 0, 4014, 0, 81, 0, 168, 0, 0, 0, 215, 20, 733, 3434, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3120, 4893, 4438, 3994, 4097, 3561, 4263, 4402] avg=4096.0000 max=4893 max_rank=1 min=3120 min_rank=0 imbalance_ratio=1.194580 max_load_reduction=27875 max_load_reduction_pct=0.850677 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 278, 3259, 0, 0, 0, 0, 0, 630, 1464, 2, 819, 0, 0, 0, 0, 4084, 0, 0, 1, 360, 0, 0, 0, 29, 0, 33, 13, 3332, 634, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 1700, 0, 0, 88, 2640, 209, 0, 0, 4003, 0, 22, 0, 176, 0, 0, 0, 71, 1, 726, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3537, 2915, 4445, 4041, 4098, 4637, 4201, 4894] avg=4096.0000 max=4894 max_rank=7 min=2915 min_rank=1 imbalance_ratio=1.194824 max_load_reduction=27874 max_load_reduction_pct=0.850647 moved_to_echo=17830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[69, 1353, 3315, 0, 0, 0, 0, 0, 1185, 97, 118, 820, 0, 0, 0, 0, 4062, 0, 0, 0, 367, 0, 0, 0, 0, 0, 2, 0, 3349, 634, 0, 0, 3945, 0, 0, 0, 2, 0, 0, 0, 4050, 30, 11, 25, 2642, 213, 0, 0, 3165, 0, 0, 15, 179, 0, 0, 0, 0, 141, 730, 2249, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4737, 2220, 4429, 3985, 3947, 6971, 3359, 3120] avg=4096.0000 max=6971 max_rank=5 min=2220 min_rank=1 imbalance_ratio=1.701904 max_load_reduction=25797 max_load_reduction_pct=0.787262 moved_to_echo=17397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 141, 3332, 0, 0, 0, 0, 0, 4096, 2, 0, 820, 0, 0, 0, 0, 4095, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 3363, 634, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2642, 214, 0, 0, 3923, 0, 0, 0, 180, 0, 0, 0, 0, 31, 733, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3473, 4918, 4463, 3997, 4098, 2856, 4103, 4860] avg=4096.0000 max=4918 max_rank=1 min=2856 min_rank=5 imbalance_ratio=1.200684 max_load_reduction=27850 max_load_reduction_pct=0.849915 moved_to_echo=15917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[14, 1271, 3199, 0, 0, 0, 0, 0, 3406, 129, 0, 814, 0, 0, 0, 0, 231, 0, 1041, 0, 361, 0, 0, 0, 0, 10, 0, 84, 3329, 637, 0, 0, 4085, 2, 0, 81, 7, 0, 0, 0, 123, 0, 33, 0, 2642, 210, 0, 0, 3677, 0, 454, 0, 180, 0, 0, 0, 0, 2070, 719, 3959, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4484, 4349, 1633, 4060, 4175, 3008, 4311, 6748] avg=4096.0000 max=6748 max_rank=7 min=1633 min_rank=2 imbalance_ratio=1.647461 max_load_reduction=26020 max_load_reduction_pct=0.794067 moved_to_echo=18242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3332, 0, 0, 0, 0, 0, 3992, 139, 1, 792, 0, 0, 0, 0, 4091, 0, 0, 0, 368, 0, 0, 0, 0, 0, 25, 1, 3363, 613, 0, 0, 4064, 11, 5, 0, 2, 0, 0, 0, 12, 69, 0, 0, 2553, 214, 0, 0, 4084, 0, 0, 2, 180, 0, 0, 0, 0, 4043, 733, 79, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3332, 4924, 4459, 4002, 4082, 2848, 4266, 4855] avg=4096.0000 max=4924 max_rank=1 min=2848 min_rank=5 imbalance_ratio=1.202148 max_load_reduction=27844 max_load_reduction_pct=0.849731 moved_to_echo=16051 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10752, 256, 128, 256, 20096, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 6784, 5888, 19968, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[896, 128, 256, 384, 26752, 4864, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 6528, 26240, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[28800, 128, 1152, 128, 2944, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[24704, 1920, 128, 6272, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[30464, 128, 896, 128, 1536, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 256, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-08-30 04:55:22] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 668.1 | throughput per GPU (TFLOP/s/GPU): 499.0 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.032344E+01 | loss scale: 1.0 | grad norm: 204.530 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[11, 3111, 2117, 19, 3219, 11, 990, 34, 1161, 95, 2890, 483, 617, 364, 20, 401, 321, 2099, 1985, 3941, 3045, 155, 909, 142, 0, 1296, 171, 730, 1024, 51, 1191, 165] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[51, 3409, 1603, 31, 2944, 14, 1283, 56, 1726, 40, 3473, 967, 406, 509, 195, 273, 593, 943, 1270, 3585, 3256, 844, 527, 258, 2, 1666, 245, 578, 544, 128, 1132, 217] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[28, 3045, 1901, 92, 2854, 32, 870, 63, 1496, 74, 3322, 838, 402, 486, 115, 289, 376, 1488, 2306, 3530, 3013, 515, 762, 215, 2, 1463, 183, 837, 617, 112, 1320, 122] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[25, 2981, 1380, 38, 2833, 12, 965, 79, 1624, 59, 3343, 971, 461, 575, 69, 294, 474, 1496, 2210, 3859, 3271, 294, 606, 150, 0, 1614, 208, 626, 561, 96, 1430, 164] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[42, 3544, 1560, 22, 2701, 33, 837, 109, 1661, 59, 3168, 676, 520, 686, 26, 359, 630, 1097, 2172, 3662, 2954, 191, 586, 215, 0, 1940, 240, 729, 628, 127, 1446, 148] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[14, 3454, 2253, 69, 2192, 15, 833, 59, 1574, 122, 3350, 554, 459, 518, 97, 256, 524, 1298, 1986, 3621, 3143, 301, 546, 146, 2, 1440, 189, 938, 782, 139, 1693, 201] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[44, 3351, 1827, 23, 2527, 26, 955, 30, 1170, 97, 3362, 725, 308, 698, 46, 283, 417, 1455, 1743, 3836, 3161, 524, 766, 199, 3, 1860, 171, 1034, 566, 251, 1205, 105] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[11, 3646, 1932, 57, 3020, 4, 876, 46, 1238, 71, 3124, 585, 380, 464, 94, 202, 394, 1351, 1775, 3610, 3436, 745, 558, 171, 1, 1583, 208, 840, 722, 240, 1245, 139] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 2066, 2117, 19, 0, 0, 0, 0, 3219, 11, 990, 34, 270, 0, 0, 0, 1161, 95, 1633, 483, 0, 0, 0, 0, 617, 364, 20, 401, 2910, 0, 0, 0, 321, 2099, 1985, 316, 0, 0, 0, 0, 2704, 155, 909, 142, 0, 0, 0, 0, 0, 1296, 171, 730, 1045, 272, 71, 0, 1024, 51, 1191, 165, 985, 715, 0, 0] rerouted_tokens_per_ep_rank=[4213, 4524, 3372, 4312, 4721, 3910, 3585, 4131] avg=4096.0000 max=4721 max_rank=4 min=3372 min_rank=2 imbalance_ratio=1.152588 max_load_reduction=28047 max_load_reduction_pct=0.855927 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[51, 2263, 1603, 31, 0, 0, 0, 0, 2944, 14, 1283, 56, 289, 0, 0, 0, 1726, 40, 1963, 967, 0, 0, 0, 0, 406, 509, 195, 273, 2647, 0, 0, 0, 593, 943, 1270, 288, 0, 0, 0, 0, 2892, 844, 527, 258, 0, 0, 0, 0, 2, 1666, 245, 578, 1146, 327, 75, 0, 544, 128, 1132, 217, 1183, 650, 0, 0] rerouted_tokens_per_ep_rank=[3948, 4586, 4696, 4030, 3094, 4521, 4039, 3854] avg=4096.0000 max=4696 max_rank=2 min=3094 min_rank=4 imbalance_ratio=1.146484 max_load_reduction=28072 max_load_reduction_pct=0.856689 moved_to_echo=15508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[28, 2022, 1901, 92, 0, 0, 0, 0, 2854, 32, 870, 63, 267, 0, 0, 0, 1496, 74, 1877, 838, 0, 0, 0, 0, 402, 486, 115, 289, 2607, 0, 0, 0, 376, 1488, 2306, 282, 0, 0, 0, 0, 2676, 515, 762, 215, 0, 0, 0, 0, 2, 1463, 183, 837, 1023, 313, 70, 0, 617, 112, 1320, 122, 1132, 641, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4086, 4285, 3899, 4452, 4168, 3891, 3944] avg=4096.0000 max=4452 max_rank=4 min=3891 min_rank=6 imbalance_ratio=1.086914 max_load_reduction=28316 max_load_reduction_pct=0.864136 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[25, 1979, 1380, 38, 0, 0, 0, 0, 2833, 12, 965, 79, 290, 0, 0, 0, 1624, 59, 1889, 971, 0, 0, 0, 0, 461, 575, 69, 294, 2850, 0, 0, 0, 474, 1496, 2210, 309, 0, 0, 0, 0, 2905, 294, 606, 150, 0, 0, 0, 0, 0, 1614, 208, 626, 1002, 315, 76, 0, 561, 96, 1430, 164, 1139, 700, 0, 0] rerouted_tokens_per_ep_rank=[3422, 4179, 4543, 4249, 4489, 3955, 3841, 4090] avg=4096.0000 max=4543 max_rank=2 min=3422 min_rank=0 imbalance_ratio=1.109131 max_load_reduction=28225 max_load_reduction_pct=0.861359 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[42, 2353, 1560, 22, 0, 0, 0, 0, 2701, 33, 837, 109, 262, 0, 0, 0, 1661, 59, 1791, 676, 0, 0, 0, 0, 520, 686, 26, 359, 2704, 0, 0, 0, 630, 1097, 2172, 294, 0, 0, 0, 0, 2624, 191, 586, 215, 0, 0, 0, 0, 0, 1940, 240, 729, 1191, 298, 68, 0, 628, 127, 1446, 148, 1079, 664, 0, 0] rerouted_tokens_per_ep_rank=[3977, 3942, 4187, 4295, 4193, 3616, 4466, 4092] avg=4096.0000 max=4466 max_rank=6 min=3616 min_rank=5 imbalance_ratio=1.090332 max_load_reduction=28302 max_load_reduction_pct=0.863708 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[44, 2225, 1827, 23, 0, 0, 0, 0, 2527, 26, 955, 30, 280, 0, 0, 0, 1170, 97, 1899, 725, 0, 0, 0, 0, 308, 698, 46, 283, 2833, 0, 0, 0, 417, 1455, 1743, 307, 0, 0, 0, 0, 2808, 524, 766, 199, 0, 0, 0, 0, 3, 1860, 171, 1034, 1126, 317, 73, 0, 566, 251, 1205, 105, 1146, 696, 0, 0] rerouted_tokens_per_ep_rank=[4119, 3818, 3891, 4168, 3922, 4297, 4584, 3969] avg=4096.0000 max=4584 max_rank=6 min=3818 min_rank=1 imbalance_ratio=1.119141 max_load_reduction=28184 max_load_reduction_pct=0.860107 moved_to_echo=16772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[14, 2289, 2253, 69, 0, 0, 0, 0, 2192, 15, 833, 59, 283, 0, 0, 0, 1574, 122, 1884, 554, 0, 0, 0, 0, 459, 518, 97, 256, 2679, 0, 0, 0, 524, 1298, 1986, 280, 0, 0, 0, 0, 2783, 301, 546, 146, 0, 0, 0, 0, 2, 1440, 189, 938, 1165, 320, 77, 0, 782, 139, 1693, 201, 1146, 662, 0, 0] rerouted_tokens_per_ep_rank=[4625, 3382, 4134, 4009, 4088, 3776, 4131, 4623] avg=4096.0000 max=4625 max_rank=0 min=3382 min_rank=1 imbalance_ratio=1.129150 max_load_reduction=28143 max_load_reduction_pct=0.858856 moved_to_echo=16618 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 2421, 1932, 57, 0, 0, 0, 0, 3020, 4, 876, 46, 305, 0, 0, 0, 1238, 71, 1766, 585, 0, 0, 0, 0, 380, 464, 94, 202, 2666, 0, 0, 0, 394, 1351, 1775, 289, 0, 0, 0, 0, 3051, 745, 558, 171, 0, 0, 0, 0, 1, 1583, 208, 840, 1225, 294, 80, 0, 722, 240, 1245, 139, 1064, 655, 0, 0] rerouted_tokens_per_ep_rank=[4421, 4251, 3660, 3806, 3809, 4525, 4231, 4065] avg=4096.0000 max=4525 max_rank=5 min=3660 min_rank=2 imbalance_ratio=1.104736 max_load_reduction=28243 max_load_reduction_pct=0.861908 moved_to_echo=16630 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 12928, 1664, 6400, 8960, 2560, 640, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11776, 640, 14720, 5888, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3840, 11264, 15488, 2432, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[22528, 3584, 5376, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 17664, 14592, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5504, 1152, 10752, 1280, 8960, 5504, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[22400, 256, 7680, 512, 2304, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3584, 4352, 768, 2432, 22016, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 4094, 1590, 268, 0, 0, 4, 0, 0, 3796, 0, 3, 0, 0, 4026, 1, 0, 0, 6, 0, 3826, 0, 20, 1, 4063, 3882, 4089, 0, 158, 0, 48, 2893] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[10, 4096, 2035, 3, 22, 0, 15, 37, 0, 4035, 0, 2, 0, 0, 4061, 4, 0, 24, 0, 0, 3982, 2, 1, 49, 4021, 2276, 4075, 6, 316, 0, 457, 3239] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 4095, 397, 398, 13, 1, 3, 42, 0, 3579, 1, 0, 2, 15, 3998, 1, 2, 0, 1, 1, 3923, 35, 0, 197, 3992, 3887, 4088, 1, 42, 0, 8, 4046] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[5, 4095, 746, 19, 0, 0, 65, 0, 0, 3739, 1, 0, 0, 0, 3948, 1, 10, 8, 4, 0, 4059, 1, 2, 0, 4035, 3862, 4094, 1, 6, 0, 55, 4012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 4073, 1298, 392, 0, 2, 100, 0, 0, 4043, 2, 0, 0, 87, 2628, 11, 1, 20, 1, 7, 4023, 8, 3, 57, 3801, 4086, 4068, 0, 134, 0, 158, 3765] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 4082, 1706, 23, 4, 0, 1, 1, 6, 4083, 14, 0, 4, 2, 3234, 0, 12, 0, 3, 0, 4045, 0, 16, 126, 4084, 2816, 4093, 0, 371, 0, 28, 4014] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[44, 4090, 610, 0, 60, 0, 1, 35, 9, 4087, 8, 5, 0, 0, 3781, 9, 1, 0, 0, 0, 4088, 8, 22, 0, 4078, 3728, 4079, 0, 271, 0, 3, 3751] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[49, 4096, 273, 1106, 0, 0, 40, 67, 0, 4040, 3, 0, 0, 0, 3818, 1, 0, 0, 0, 0, 2944, 20, 214, 2, 4046, 3783, 4091, 11, 129, 0, 227, 3808] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2731, 397, 398, 0, 0, 0, 0, 13, 1, 3, 42, 3441, 569, 1, 0, 0, 3579, 1, 0, 164, 0, 0, 0, 2, 15, 3998, 1, 392, 0, 0, 0, 2, 0, 1, 1, 4087, 0, 0, 0, 3923, 35, 0, 197, 136, 0, 0, 0, 551, 3887, 0, 1, 0, 0, 0, 0, 42, 0, 8, 4046, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[3526, 4070, 3744, 4408, 4091, 4291, 4439, 4199] avg=4096.0000 max=4439 max_rank=6 min=3526 min_rank=0 imbalance_ratio=1.083740 max_load_reduction=28329 max_load_reduction_pct=0.864532 moved_to_echo=17020 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2730, 1590, 268, 0, 0, 0, 0, 0, 0, 4, 0, 3502, 569, 1, 0, 0, 3796, 0, 3, 164, 0, 0, 0, 0, 0, 4026, 1, 392, 0, 0, 0, 0, 0, 6, 0, 4088, 0, 0, 0, 3826, 0, 20, 1, 136, 0, 0, 0, 561, 3882, 0, 0, 0, 0, 0, 0, 158, 0, 48, 2893, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[4588, 4076, 3963, 4419, 4094, 3983, 4443, 3202] avg=4096.0000 max=4588 max_rank=0 min=3202 min_rank=7 imbalance_ratio=1.120117 max_load_reduction=28180 max_load_reduction_pct=0.859985 moved_to_echo=15722 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[5, 2731, 746, 19, 0, 0, 0, 0, 0, 0, 65, 0, 3478, 569, 1, 0, 0, 3739, 1, 0, 164, 0, 0, 0, 0, 0, 3948, 1, 392, 0, 0, 0, 10, 8, 4, 0, 4093, 0, 0, 0, 4059, 1, 2, 0, 136, 0, 0, 0, 557, 3862, 0, 1, 0, 0, 0, 0, 6, 0, 55, 4012, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[3501, 4113, 3904, 4341, 4115, 4198, 4420, 4176] avg=4096.0000 max=4420 max_rank=6 min=3501 min_rank=0 imbalance_ratio=1.079102 max_load_reduction=28348 max_load_reduction_pct=0.865112 moved_to_echo=16909 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[10, 2731, 2035, 3, 0, 0, 0, 0, 22, 0, 15, 37, 3466, 570, 1, 0, 0, 4035, 0, 2, 164, 0, 0, 0, 0, 0, 4061, 4, 392, 0, 0, 0, 0, 24, 0, 0, 4074, 0, 0, 0, 3982, 2, 1, 49, 136, 0, 0, 0, 555, 2276, 0, 6, 0, 0, 0, 0, 316, 0, 457, 3239, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[4779, 4111, 4201, 4457, 4098, 4170, 2837, 4115] avg=4096.0000 max=4779 max_rank=0 min=2837 min_rank=6 imbalance_ratio=1.166748 max_load_reduction=27989 max_load_reduction_pct=0.854156 moved_to_echo=15220 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2715, 1298, 392, 0, 0, 0, 0, 0, 2, 100, 0, 3276, 566, 1, 0, 0, 4043, 2, 0, 163, 0, 0, 0, 0, 87, 2628, 11, 390, 0, 0, 0, 1, 20, 1, 7, 4067, 0, 0, 0, 4023, 8, 3, 57, 136, 0, 0, 0, 525, 4086, 0, 0, 0, 0, 0, 0, 134, 0, 158, 3765, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[4405, 3945, 4208, 3116, 4096, 4227, 4611, 4160] avg=4096.0000 max=4611 max_rank=6 min=3116 min_rank=3 imbalance_ratio=1.125732 max_load_reduction=28157 max_load_reduction_pct=0.859283 moved_to_echo=17094 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2699, 1706, 23, 0, 0, 0, 0, 4, 0, 1, 1, 3525, 572, 2, 0, 6, 4083, 14, 0, 166, 0, 0, 0, 4, 2, 3234, 0, 395, 0, 0, 0, 12, 0, 3, 0, 4091, 0, 0, 0, 4045, 0, 16, 126, 142, 0, 0, 0, 559, 2816, 0, 0, 0, 0, 0, 0, 371, 0, 28, 4014, 108, 0, 0, 0] rerouted_tokens_per_ep_rank=[4428, 4105, 4269, 3635, 4106, 4329, 3375, 4521] avg=4096.0000 max=4521 max_rank=7 min=3375 min_rank=6 imbalance_ratio=1.103760 max_load_reduction=28247 max_load_reduction_pct=0.862030 moved_to_echo=16331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[44, 2728, 610, 0, 0, 0, 0, 0, 60, 0, 1, 35, 3515, 569, 1, 0, 9, 4087, 8, 5, 163, 0, 0, 0, 0, 0, 3781, 9, 391, 0, 0, 0, 1, 0, 0, 0, 4078, 0, 0, 0, 4088, 8, 22, 0, 136, 0, 0, 0, 563, 3728, 0, 0, 0, 0, 0, 0, 271, 0, 3, 3751, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[3382, 4181, 4272, 4181, 4079, 4254, 4291, 4128] avg=4096.0000 max=4291 max_rank=6 min=3382 min_rank=0 imbalance_ratio=1.047607 max_load_reduction=28477 max_load_reduction_pct=0.869049 moved_to_echo=16752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[49, 2731, 273, 1106, 0, 0, 0, 0, 0, 0, 40, 67, 3488, 570, 2, 0, 0, 4040, 3, 0, 164, 0, 0, 0, 0, 0, 3818, 1, 392, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 2944, 20, 214, 2, 136, 0, 0, 0, 558, 3783, 0, 11, 0, 0, 0, 0, 129, 0, 227, 3808, 103, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4167, 4207, 4211, 4089, 3316, 4352, 4267] avg=4096.0000 max=4352 max_rank=6 min=3316 min_rank=5 imbalance_ratio=1.062500 max_load_reduction=28416 max_load_reduction_pct=0.867188 moved_to_echo=16024 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 21888, 8704, 2304, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4480, 28416, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1536, 0, 1024, 29568, 896, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 256, 256, 27776, 4608, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[30976, 128, 384, 512, 1152, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 29568, 128, 3200, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 31488, 128, 128, 1408, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4, 4072, 0, 4095, 0, 3906, 7, 15, 4089, 4087, 4091, 62, 4084, 0, 0, 22, 0, 130, 0, 5, 2, 4069, 0, 1, 4, 0, 19, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 21, 4067, 14, 4093, 0, 3914, 0, 5, 4093, 4096, 4091, 4, 4090, 4, 0, 4, 0, 186, 1, 1, 35, 3851, 194, 0, 0, 1, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[5, 0, 16, 0, 6, 3612, 0, 4055, 0, 3879, 0, 1666, 4096, 4094, 4093, 36, 4095, 1, 0, 1023, 0, 42, 0, 0, 57, 1303, 661, 0, 21, 0, 1, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[7, 0, 7, 0, 2, 3927, 0, 4096, 3, 2786, 7, 1038, 4091, 4095, 4095, 91, 4088, 0, 0, 44, 0, 20, 1, 2, 0, 3971, 0, 2, 0, 0, 395, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4, 6, 0, 5, 4, 4069, 24, 4087, 0, 4077, 0, 2, 4095, 4081, 4054, 0, 4095, 0, 0, 73, 25, 163, 8, 4, 23, 3650, 212, 1, 0, 6, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 17, 0, 0, 4074, 1, 4094, 15, 4046, 0, 2033, 4095, 4092, 4065, 7, 4082, 0, 0, 110, 0, 48, 0, 0, 0, 1537, 30, 0, 0, 0, 420, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[8, 0, 21, 3, 0, 2102, 1, 4077, 79, 3684, 1, 2363, 4086, 4058, 4092, 0, 4054, 1, 0, 0, 0, 3, 50, 36, 26, 3598, 234, 6, 150, 0, 35, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 34, 0, 0, 0, 4014, 0, 4092, 0, 4016, 0, 1495, 4093, 4096, 4082, 54, 4059, 0, 0, 443, 0, 1, 0, 1, 1, 2191, 0, 0, 80, 0, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4075, 0, 0, 0, 4, 4072, 0, 345, 0, 0, 0, 0, 0, 3099, 7, 15, 0, 0, 0, 0, 0, 0, 4073, 62, 0, 0, 0, 0, 3884, 0, 0, 22, 0, 0, 0, 0, 0, 130, 0, 5, 3992, 14, 0, 0, 2, 4069, 0, 1, 690, 200, 18, 0, 4, 0, 19, 0, 3750, 117, 95, 0] rerouted_tokens_per_ep_rank=[4079, 4421, 3121, 4135, 3906, 4141, 4980, 3985] avg=4096.0000 max=4980 max_rank=6 min=3121 min_rank=2 imbalance_ratio=1.215820 max_load_reduction=27788 max_load_reduction_pct=0.848022 moved_to_echo=17012 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4079, 0, 0, 0, 21, 4067, 14, 345, 0, 0, 0, 0, 0, 3105, 0, 5, 0, 0, 0, 0, 0, 0, 4073, 4, 0, 0, 0, 0, 3890, 4, 0, 4, 0, 0, 0, 0, 0, 186, 1, 1, 3999, 14, 0, 0, 35, 3851, 194, 0, 692, 200, 18, 0, 0, 1, 2, 0, 3748, 117, 97, 0] rerouted_tokens_per_ep_rank=[4080, 4447, 3110, 4077, 3898, 4201, 4990, 3965] avg=4096.0000 max=4990 max_rank=6 min=3110 min_rank=2 imbalance_ratio=1.218262 max_load_reduction=27778 max_load_reduction_pct=0.847717 moved_to_echo=17054 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 16, 0, 4082, 0, 0, 0, 6, 3612, 0, 342, 0, 0, 0, 0, 0, 3077, 0, 1666, 0, 0, 0, 0, 0, 0, 4075, 36, 0, 0, 0, 0, 3894, 1, 0, 1023, 0, 0, 0, 0, 0, 42, 0, 0, 3998, 14, 0, 0, 57, 1303, 661, 0, 686, 201, 18, 0, 21, 0, 1, 6, 3713, 116, 96, 0] rerouted_tokens_per_ep_rank=[4103, 3960, 4743, 4111, 4918, 4054, 2926, 3953] avg=4096.0000 max=4918 max_rank=4 min=2926 min_rank=6 imbalance_ratio=1.200684 max_load_reduction=27850 max_load_reduction_pct=0.849915 moved_to_echo=15851 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 7, 0, 4077, 0, 0, 0, 2, 3927, 0, 346, 0, 0, 0, 0, 3, 2211, 7, 1038, 0, 0, 0, 0, 0, 0, 4077, 91, 0, 0, 0, 0, 3888, 0, 0, 44, 0, 0, 0, 0, 0, 20, 1, 2, 3998, 14, 0, 0, 0, 3971, 0, 2, 492, 200, 18, 0, 0, 0, 395, 0, 3750, 83, 97, 0] rerouted_tokens_per_ep_rank=[4091, 4275, 3259, 4168, 3932, 4035, 4683, 4325] avg=4096.0000 max=4683 max_rank=6 min=3259 min_rank=2 imbalance_ratio=1.143311 max_load_reduction=28085 max_load_reduction_pct=0.857086 moved_to_echo=16975 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 21, 3, 4072, 0, 0, 0, 0, 2102, 1, 344, 0, 0, 0, 0, 79, 2923, 1, 2363, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 3855, 1, 0, 0, 0, 0, 0, 0, 0, 3, 50, 36, 3962, 14, 0, 0, 26, 3598, 234, 6, 651, 199, 18, 0, 150, 0, 35, 0, 3733, 110, 96, 0] rerouted_tokens_per_ep_rank=[4104, 2447, 5366, 4074, 3856, 4065, 4732, 4124] avg=4096.0000 max=5366 max_rank=2 min=2447 min_rank=1 imbalance_ratio=1.310059 max_load_reduction=27402 max_load_reduction_pct=0.836243 moved_to_echo=16777 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 17, 0, 4081, 0, 0, 0, 0, 4074, 1, 345, 0, 0, 0, 0, 15, 3210, 0, 2033, 0, 0, 0, 0, 0, 0, 4047, 7, 0, 0, 0, 0, 3882, 0, 0, 110, 0, 0, 0, 0, 0, 48, 0, 0, 3996, 14, 0, 0, 0, 1537, 30, 0, 715, 200, 18, 0, 0, 0, 420, 0, 3749, 121, 96, 0] rerouted_tokens_per_ep_rank=[4100, 4420, 5258, 4054, 3992, 4058, 2500, 4386] avg=4096.0000 max=5258 max_rank=2 min=2500 min_rank=6 imbalance_ratio=1.283691 max_load_reduction=27510 max_load_reduction_pct=0.839539 moved_to_echo=14936 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 6, 0, 5, 4082, 0, 0, 0, 4, 4069, 24, 342, 0, 0, 0, 0, 0, 3228, 0, 2, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 3890, 0, 0, 73, 0, 0, 0, 0, 25, 163, 8, 4, 3986, 13, 0, 0, 23, 3650, 212, 1, 724, 205, 23, 0, 0, 6, 0, 0, 3745, 125, 95, 0] rerouted_tokens_per_ep_rank=[4097, 4439, 3230, 4031, 3963, 4199, 4838, 3971] avg=4096.0000 max=4838 max_rank=6 min=3230 min_rank=2 imbalance_ratio=1.181152 max_load_reduction=27930 max_load_reduction_pct=0.852356 moved_to_echo=16971 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 34, 0, 0, 4080, 0, 0, 0, 0, 4014, 0, 345, 0, 0, 0, 0, 0, 3186, 0, 1495, 0, 0, 0, 0, 0, 0, 4064, 54, 0, 0, 0, 0, 3860, 0, 0, 443, 0, 0, 0, 0, 0, 1, 0, 1, 4000, 13, 0, 0, 1, 2191, 0, 0, 710, 199, 18, 0, 80, 0, 16, 0, 3747, 120, 96, 0] rerouted_tokens_per_ep_rank=[4114, 4359, 4681, 4118, 4303, 4015, 3119, 4059] avg=4096.0000 max=4681 max_rank=2 min=3119 min_rank=6 imbalance_ratio=1.142822 max_load_reduction=28087 max_load_reduction_pct=0.857147 moved_to_echo=15496 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 24192, 1408, 128, 5376, 1664, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 24064, 128, 8704, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31104, 128, 0, 1792, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32640, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 29952, 128, 2816, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 640, 128, 128, 32000, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 896, 128, 29952, 1024, 768, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4071, 5, 1, 0, 2, 4095, 0, 0, 0, 4095, 0, 4096, 3842, 0, 2, 23, 0, 0, 4085, 3, 4090, 0, 0, 0, 4, 0, 4091, 262, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[40, 0, 95, 4047, 0, 0, 9, 0, 4089, 0, 4, 0, 4096, 1, 4096, 3016, 1, 17, 36, 0, 5, 4056, 5, 3979, 129, 0, 0, 0, 0, 4081, 958, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 2, 0, 4096, 0, 0, 0, 13, 4096, 0, 3, 0, 4096, 0, 4092, 4077, 0, 0, 4, 0, 0, 4096, 0, 4014, 0, 1, 0, 0, 0, 4082, 96, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[25, 5, 0, 4086, 1, 0, 0, 4, 4057, 5, 6, 0, 4085, 18, 4092, 4051, 0, 0, 0, 13, 1, 4085, 0, 3843, 26, 0, 0, 8, 1, 4088, 264, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4083, 2, 1, 30, 0, 4041, 8, 16, 0, 4093, 0, 4095, 4061, 0, 0, 0, 0, 0, 4094, 13, 4081, 0, 0, 2, 49, 1, 4096, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 36, 4079, 0, 0, 0, 0, 4056, 0, 8, 0, 4096, 0, 4088, 3712, 0, 0, 9, 1, 0, 4065, 0, 4091, 37, 67, 5, 11, 7, 4088, 311, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 9, 0, 109, 0, 4048, 12, 153, 0, 4080, 0, 4094, 3862, 1, 0, 5, 2, 0, 4082, 0, 4055, 0, 0, 0, 66, 0, 4088, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[302, 0, 37, 4074, 0, 0, 5, 0, 4096, 0, 49, 0, 4096, 0, 4096, 3772, 1, 0, 96, 0, 0, 4065, 8, 3865, 0, 0, 0, 0, 11, 4096, 99, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4021, 0, 0, 0, 0, 5, 1, 0, 2, 4074, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 295, 0, 0, 3842, 0, 0, 0, 0, 0, 2, 23, 0, 3995, 59, 22, 0, 0, 90, 3, 4090, 0, 0, 0, 0, 0, 0, 0, 4, 3741, 246, 50, 9, 0, 3845, 262, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 4082, 4086, 4137, 4101, 4183, 4050, 4108] avg=4096.0000 max=4183 max_rank=5 min=4021 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16442 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4033, 0, 0, 0, 0, 2, 1, 30, 0, 4074, 0, 0, 0, 4032, 8, 16, 0, 0, 0, 0, 0, 295, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 59, 21, 0, 0, 90, 13, 4081, 0, 0, 0, 0, 0, 0, 2, 49, 3739, 247, 50, 9, 1, 3849, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4107, 4056, 4356, 4084, 4184, 4096, 3852] avg=4096.0000 max=4356 max_rank=3 min=3852 min_rank=7 imbalance_ratio=1.063477 max_load_reduction=28412 max_load_reduction_pct=0.867065 moved_to_echo=16216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 13, 4070, 0, 0, 0, 4087, 0, 3, 0, 0, 0, 0, 0, 295, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 4, 0, 4006, 59, 22, 0, 0, 90, 0, 4014, 0, 0, 0, 0, 0, 1, 0, 0, 3742, 246, 50, 9, 0, 3836, 96, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4083, 4090, 4372, 4091, 4104, 4048, 3932] avg=4096.0000 max=4372 max_rank=3 min=3932 min_rank=7 imbalance_ratio=1.067383 max_load_reduction=28396 max_load_reduction_pct=0.866577 moved_to_echo=16175 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[25, 5, 0, 4036, 0, 0, 0, 0, 1, 0, 0, 4, 4070, 0, 0, 0, 4048, 5, 6, 0, 0, 0, 0, 0, 295, 18, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 13, 3995, 58, 22, 0, 1, 90, 0, 3843, 0, 0, 0, 0, 26, 0, 0, 8, 3732, 246, 50, 9, 1, 3842, 264, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4075, 4059, 4364, 4088, 3934, 4071, 4111] avg=4096.0000 max=4364 max_rank=3 min=3934 min_rank=5 imbalance_ratio=1.065430 max_load_reduction=28404 max_load_reduction_pct=0.866821 moved_to_echo=16204 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[40, 0, 95, 3997, 0, 0, 0, 0, 0, 0, 9, 0, 4075, 0, 0, 0, 4080, 0, 4, 0, 0, 0, 0, 0, 295, 1, 0, 3016, 0, 0, 0, 0, 1, 17, 36, 0, 3967, 59, 21, 0, 5, 89, 5, 3979, 0, 0, 0, 0, 129, 0, 0, 0, 3742, 246, 50, 9, 0, 3835, 958, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4084, 4084, 3312, 4101, 4078, 4176, 4801] avg=4096.0000 max=4801 max_rank=7 min=3312 min_rank=3 imbalance_ratio=1.172119 max_load_reduction=27967 max_load_reduction_pct=0.853485 moved_to_echo=17156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 36, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 4047, 0, 8, 0, 0, 0, 0, 0, 295, 0, 0, 3712, 0, 0, 0, 0, 0, 0, 9, 1, 3975, 59, 22, 0, 0, 90, 0, 4091, 0, 0, 0, 0, 37, 67, 5, 11, 3742, 246, 50, 9, 7, 3842, 311, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4066, 4055, 4007, 4066, 4181, 4167, 4161] avg=4096.0000 max=4181 max_rank=5 min=4007 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16575 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4040, 0, 0, 0, 0, 9, 0, 109, 0, 4073, 0, 0, 0, 4037, 12, 153, 0, 0, 0, 0, 0, 291, 0, 0, 3862, 0, 0, 0, 0, 1, 0, 5, 2, 3996, 60, 21, 0, 0, 86, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 66, 3729, 250, 56, 11, 0, 3838, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4191, 4202, 4153, 4085, 4141, 4112, 3844] avg=4096.0000 max=4202 max_rank=2 min=3844 min_rank=7 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16182 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[302, 0, 37, 4024, 0, 0, 0, 0, 0, 0, 5, 0, 4075, 0, 0, 0, 4087, 0, 49, 0, 0, 0, 0, 0, 295, 0, 0, 3772, 0, 0, 0, 0, 1, 0, 96, 0, 3975, 59, 21, 0, 0, 90, 8, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 3742, 247, 50, 9, 11, 3849, 99, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4363, 4080, 4136, 4067, 4152, 3963, 4048, 3959] avg=4096.0000 max=4363 max_rank=0 min=3959 min_rank=7 imbalance_ratio=1.065186 max_load_reduction=28405 max_load_reduction_pct=0.866852 moved_to_echo=16122 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2432, 128, 0, 30464, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 128, 256, 32256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 30848, 2048, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 128, 256, 29952, 2048, 512, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32512, 128, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 256, 128, 32000, 512, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 768, 128, 32128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 256, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 4013, 0, 1, 4076, 6, 23, 0, 451, 4077, 12, 31, 3662, 1, 0, 4085, 12, 0, 16, 4083, 0, 0, 0, 0, 4070, 62, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 32, 1, 0, 0, 4093, 394, 0, 4096, 4, 15, 0, 403, 4094, 2, 3486, 58, 0, 0, 4009, 0, 1, 0, 4096, 11, 0, 0, 0, 4091, 3, 3879] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 12, 0, 0, 4091, 31, 1, 3346, 36, 704, 3, 8, 4026, 5, 2056, 2398, 1, 0, 4094, 0, 0, 3, 4094, 0, 2, 0, 1, 4008, 0, 3847] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 1, 3909, 3434, 0, 4096, 3, 304, 5, 34, 4096, 0, 506, 3825, 1, 4, 3932, 0, 0, 0, 4052, 0, 0, 0, 1, 4082, 214, 256] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 29, 23, 0, 0, 4096, 1282, 0, 4033, 0, 3, 0, 22, 4092, 0, 679, 3920, 0, 0, 4096, 0, 0, 0, 4090, 1, 0, 0, 0, 4096, 0, 2306] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 190, 0, 0, 4080, 9, 18, 4088, 0, 20, 0, 84, 4095, 3, 13, 4083, 6, 0, 3986, 3, 0, 0, 4096, 0, 0, 0, 0, 4085, 6, 3903] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 3, 0, 9, 0, 4091, 264, 0, 4096, 0, 63, 0, 19, 4096, 0, 974, 3056, 0, 4, 4096, 0, 7, 0, 4071, 1, 0, 0, 1, 4094, 0, 3821] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 125, 0, 0, 4095, 15, 0, 3328, 0, 116, 0, 0, 4096, 0, 1470, 3236, 0, 0, 4094, 0, 0, 0, 4096, 0, 6, 0, 0, 4084, 1, 4006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 3274, 636, 119, 1, 0, 0, 3377, 0, 0, 0, 0, 0, 1, 4076, 6, 23, 44, 0, 0, 0, 0, 451, 3958, 12, 0, 0, 0, 0, 31, 3557, 1, 0, 0, 0, 0, 0, 4085, 12, 0, 16, 50, 0, 0, 0, 4083, 0, 0, 0, 10, 0, 0, 0, 0, 796, 62, 4085, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 3377, 4150, 4421, 3589, 4163, 4093, 4943] avg=4096.0000 max=4943 max_rank=7 min=3377 min_rank=1 imbalance_ratio=1.206787 max_load_reduction=27825 max_load_reduction_pct=0.849152 moved_to_echo=16788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 12, 3224, 649, 117, 0, 0, 0, 3442, 31, 0, 0, 0, 0, 1, 3346, 36, 704, 28, 0, 0, 0, 3, 8, 3909, 5, 0, 0, 0, 0, 2056, 2331, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 3, 33, 0, 0, 0, 4094, 0, 2, 0, 6, 0, 0, 0, 1, 784, 0, 3847, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4003, 3473, 4115, 3925, 4388, 4130, 4102, 4632] avg=4096.0000 max=4632 max_rank=7 min=3473 min_rank=1 imbalance_ratio=1.130859 max_load_reduction=28136 max_load_reduction_pct=0.858643 moved_to_echo=17252 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3283, 620, 119, 1, 13, 1, 3289, 3434, 0, 0, 0, 0, 0, 4096, 3, 304, 46, 0, 0, 0, 5, 34, 3977, 0, 0, 0, 0, 0, 506, 3716, 1, 4, 0, 0, 0, 0, 3932, 0, 0, 0, 52, 0, 0, 0, 4052, 0, 0, 0, 10, 0, 0, 0, 1, 799, 214, 256, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 6737, 4449, 4016, 4227, 3984, 4062, 1270] avg=4096.0000 max=6737 max_rank=1 min=1270 min_rank=7 imbalance_ratio=1.644775 max_load_reduction=26031 max_load_reduction_pct=0.794403 moved_to_echo=13543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 32, 1, 3291, 649, 119, 0, 0, 0, 3444, 394, 0, 0, 0, 0, 0, 4096, 4, 15, 0, 0, 0, 0, 0, 403, 3975, 2, 0, 0, 0, 0, 3486, 58, 0, 0, 0, 0, 0, 0, 4009, 0, 1, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 800, 3, 3879, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 3838, 4115, 4380, 3544, 4010, 4107, 4682] avg=4096.0000 max=4682 max_rank=7 min=3544 min_rank=4 imbalance_ratio=1.143066 max_load_reduction=28086 max_load_reduction_pct=0.857117 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 29, 23, 3295, 649, 119, 1, 0, 0, 3447, 1282, 0, 0, 0, 0, 0, 4033, 0, 3, 47, 0, 0, 0, 0, 22, 3973, 0, 0, 0, 0, 0, 679, 3807, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 54, 0, 0, 0, 4090, 1, 0, 0, 11, 0, 0, 0, 0, 801, 0, 2306, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4729, 4083, 3995, 4486, 4150, 4102, 3107] avg=4096.0000 max=4729 max_rank=1 min=3107 min_rank=7 imbalance_ratio=1.154541 max_load_reduction=28039 max_load_reduction_pct=0.855682 moved_to_echo=15845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 190, 3286, 647, 119, 1, 0, 0, 3433, 9, 0, 0, 0, 0, 18, 4088, 0, 20, 49, 0, 0, 0, 0, 84, 3976, 3, 0, 0, 0, 0, 13, 3966, 6, 0, 0, 0, 0, 0, 3986, 3, 0, 0, 56, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 799, 6, 3903, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4243, 3442, 4175, 4063, 3985, 4045, 4107, 4708] avg=4096.0000 max=4708 max_rank=7 min=3442 min_rank=1 imbalance_ratio=1.149414 max_load_reduction=28060 max_load_reduction_pct=0.856323 moved_to_echo=16845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3, 0, 3296, 653, 124, 3, 9, 0, 3438, 264, 0, 0, 0, 0, 0, 4096, 0, 63, 40, 0, 0, 0, 0, 19, 3972, 0, 0, 0, 0, 0, 974, 2956, 0, 4, 0, 0, 0, 0, 4096, 0, 7, 0, 45, 0, 0, 0, 4071, 1, 0, 0, 12, 0, 0, 0, 1, 798, 0, 3821, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 3711, 4199, 3991, 3934, 4148, 4084, 4620] avg=4096.0000 max=4620 max_rank=7 min=3711 min_rank=1 imbalance_ratio=1.127930 max_load_reduction=28148 max_load_reduction_pct=0.859009 moved_to_echo=16786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 125, 3285, 649, 119, 0, 0, 0, 3446, 15, 0, 0, 0, 0, 0, 3328, 0, 116, 38, 0, 0, 0, 0, 0, 3977, 0, 0, 0, 0, 0, 1470, 3145, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 44, 0, 0, 0, 4096, 0, 6, 0, 9, 0, 0, 0, 0, 799, 1, 4006, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 3461, 3482, 3977, 4615, 4138, 4111, 4806] avg=4096.0000 max=4806 max_rank=7 min=3461 min_rank=1 imbalance_ratio=1.173340 max_load_reduction=27962 max_load_reduction_pct=0.853333 moved_to_echo=17670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 384, 26240, 5248, 1024, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 6400, 384, 26112, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 27392, 5504, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 1024, 31744, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9216, 23552, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31232, 128, 1280, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32512, 128, 128, 128, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 4076, 0, 4, 38, 2, 0, 0, 1, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 4090, 0, 4077, 0, 4094, 2, 1, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4025, 0, 4065, 6, 0, 8, 0, 3, 0, 0, 56, 10, 0, 0, 58, 0, 4090, 0, 0, 4087, 2, 6, 4093, 26, 4085, 0, 4055, 0, 1161, 2932] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4037, 0, 3978, 0, 5, 1, 23, 0, 0, 111, 0, 0, 22, 0, 108, 1, 3939, 0, 0, 4059, 6, 0, 4090, 10, 4077, 18, 4075, 0, 113, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[13, 91, 4074, 0, 4088, 0, 0, 57, 0, 30, 0, 138, 0, 0, 0, 0, 9, 2, 4013, 0, 0, 4093, 0, 44, 4069, 1, 4060, 0, 4091, 0, 326, 3569] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[62, 0, 4096, 1, 4033, 0, 0, 1, 0, 103, 0, 0, 13, 0, 0, 0, 0, 3, 4007, 0, 0, 4094, 0, 41, 4096, 16, 4096, 0, 4093, 0, 0, 4013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4084, 0, 0, 13, 0, 7, 0, 25, 0, 0, 0, 0, 1, 0, 4072, 0, 1, 4096, 0, 1071, 4096, 8, 4096, 0, 4083, 1, 101, 2917] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4090, 14, 4096, 0, 0, 0, 0, 178, 0, 12, 0, 0, 0, 0, 0, 0, 4095, 0, 9, 4091, 0, 3, 4019, 5, 4088, 0, 4085, 0, 0, 3983] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1, 8, 4093, 0, 4024, 13, 0, 0, 0, 112, 0, 1, 40, 0, 0, 0, 0, 0, 3887, 0, 0, 4096, 0, 590, 4096, 0, 4091, 0, 4085, 0, 1181, 2450] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4022, 0, 0, 0, 0, 0, 4065, 6, 0, 8, 19, 3, 0, 0, 0, 3, 0, 0, 3828, 146, 1, 0, 56, 10, 0, 0, 4078, 0, 0, 0, 58, 0, 4090, 0, 48, 0, 0, 0, 0, 3874, 2, 6, 0, 0, 0, 0, 4093, 26, 6, 0, 0, 0, 0, 0, 227, 0, 1161, 2932, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4101, 3978, 4144, 4196, 3882, 4125, 4320] avg=4096.0000 max=4320 max_rank=7 min=3882 min_rank=5 imbalance_ratio=1.054688 max_load_reduction=28448 max_load_reduction_pct=0.868164 moved_to_echo=16523 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 0, 0, 4076, 0, 4, 38, 19, 3, 0, 0, 2, 0, 0, 1, 3864, 147, 1, 0, 5, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 4090, 0, 5, 0, 0, 0, 0, 0, 230, 2, 1, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4140, 4015, 4076, 4144, 3880, 4095, 4329] avg=4096.0000 max=4329 max_rank=7 min=3880 min_rank=5 imbalance_ratio=1.056885 max_load_reduction=28439 max_load_reduction_pct=0.867889 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4034, 0, 0, 0, 0, 0, 3978, 0, 5, 1, 18, 3, 0, 0, 23, 0, 0, 111, 3847, 145, 1, 0, 0, 0, 22, 0, 4071, 0, 0, 0, 108, 1, 3939, 0, 48, 0, 0, 0, 0, 3848, 6, 0, 0, 0, 0, 0, 4090, 10, 5, 18, 0, 0, 0, 0, 228, 0, 113, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4005, 4127, 4093, 4096, 3854, 4123, 4436] avg=4096.0000 max=4436 max_rank=7 min=3854 min_rank=5 imbalance_ratio=1.083008 max_load_reduction=28332 max_load_reduction_pct=0.864624 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 0, 0, 4084, 0, 0, 13, 19, 3, 0, 0, 0, 7, 0, 25, 3854, 147, 1, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 1, 0, 4072, 0, 48, 0, 0, 0, 1, 3882, 0, 1071, 0, 0, 0, 0, 4096, 8, 6, 0, 0, 0, 0, 0, 229, 1, 101, 2917, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4119, 4034, 4089, 4121, 4954, 4110, 3248] avg=4096.0000 max=4954 max_rank=5 min=3248 min_rank=7 imbalance_ratio=1.209473 max_load_reduction=27814 max_load_reduction_pct=0.848816 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[62, 0, 4093, 1, 0, 0, 0, 0, 4033, 0, 0, 1, 19, 3, 0, 0, 0, 103, 0, 0, 3864, 147, 1, 0, 13, 0, 0, 0, 4089, 0, 0, 0, 0, 3, 4007, 0, 48, 0, 0, 0, 0, 3880, 0, 41, 0, 0, 0, 0, 4096, 16, 6, 0, 0, 0, 0, 0, 229, 0, 0, 4013, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4056, 4115, 4102, 4058, 3921, 4118, 4242] avg=4096.0000 max=4242 max_rank=7 min=3921 min_rank=5 imbalance_ratio=1.035645 max_load_reduction=28526 max_load_reduction_pct=0.870544 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[13, 91, 4071, 0, 0, 0, 0, 0, 4088, 0, 0, 57, 19, 3, 0, 0, 0, 30, 0, 138, 3862, 147, 1, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 9, 2, 4013, 0, 48, 0, 0, 0, 0, 3879, 0, 44, 0, 0, 0, 0, 4069, 1, 5, 0, 0, 0, 0, 0, 229, 0, 326, 3569, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4167, 4178, 4054, 4072, 3923, 4075, 4124] avg=4096.0000 max=4178 max_rank=2 min=3923 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16194 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4086, 14, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 4, 0, 0, 0, 178, 0, 12, 3859, 149, 6, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 4095, 0, 51, 0, 0, 0, 9, 3870, 0, 3, 0, 0, 0, 0, 4019, 5, 0, 0, 0, 0, 0, 0, 226, 0, 0, 3983, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4121, 4204, 4082, 4146, 3882, 4024, 4209] avg=4096.0000 max=4209 max_rank=7 min=3882 min_rank=5 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 8, 4090, 0, 0, 0, 0, 0, 4024, 13, 0, 0, 19, 3, 0, 0, 0, 112, 0, 1, 3856, 147, 1, 0, 40, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 3887, 0, 48, 0, 0, 0, 0, 3882, 0, 590, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 229, 0, 1181, 2450, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4059, 4117, 4128, 3935, 4472, 4098, 3860] avg=4096.0000 max=4472 max_rank=5 min=3860 min_rank=7 imbalance_ratio=1.091797 max_load_reduction=28296 max_load_reduction_pct=0.863525 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 32256, 0, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1920, 128, 2944, 28160, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 512, 0, 384, 30848, 1280, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31104, 128, 1792, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 128, 0, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32512, 128, 128, 128, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[11, 0, 1, 0, 0, 4094, 1, 0, 0, 4092, 4, 4, 0, 0, 4094, 0, 0, 3, 0, 1258, 4095, 4096, 4095, 2, 6, 2815, 3, 1, 1, 0, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[44, 0, 0, 0, 0, 4096, 4, 1, 0, 4058, 0, 0, 2, 0, 4090, 1, 0, 79, 13, 3404, 4050, 4064, 1956, 0, 2, 2768, 0, 40, 0, 4, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[25, 1, 0, 0, 1, 4095, 2, 475, 18, 4083, 2, 0, 14, 0, 4077, 0, 0, 0, 4, 2795, 4092, 4096, 4061, 0, 659, 177, 0, 0, 0, 0, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 1, 7, 0, 4081, 12, 426, 50, 4000, 0, 492, 10, 0, 4086, 8, 0, 0, 1, 2812, 4074, 4083, 2380, 0, 51, 2057, 19, 11, 0, 0, 4081, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[5, 3, 0, 0, 0, 4096, 0, 26, 0, 4093, 3, 171, 20, 0, 4072, 41, 0, 0, 0, 2497, 3908, 4071, 1891, 2, 140, 3449, 0, 190, 1, 0, 4089, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1326, 0, 1, 0, 0, 4089, 0, 58, 5, 4091, 0, 0, 5, 3, 4096, 0, 0, 0, 0, 704, 4091, 4094, 2927, 0, 0, 3177, 0, 5, 0, 3, 4093, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[5, 13, 0, 1, 0, 4042, 3, 0, 29, 4096, 0, 0, 0, 0, 4085, 0, 0, 3, 0, 3752, 4096, 4073, 117, 0, 227, 4089, 0, 0, 0, 9, 4074, 54] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 4096, 0, 114, 20, 4096, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 3783, 4080, 4079, 468, 0, 0, 3781, 0, 26, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[25, 1, 0, 0, 3927, 0, 0, 0, 1, 3965, 2, 475, 0, 0, 0, 0, 18, 4003, 2, 0, 0, 0, 0, 0, 14, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 4, 2795, 1296, 169, 0, 0, 1874, 0, 4061, 0, 0, 0, 0, 0, 659, 177, 0, 0, 922, 130, 80, 4, 0, 0, 4087, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3953, 4443, 4023, 4091, 4264, 5935, 1972, 4087] avg=4096.0000 max=5935 max_rank=5 min=1972 min_rank=6 imbalance_ratio=1.448975 max_load_reduction=26833 max_load_reduction_pct=0.818878 moved_to_echo=16258 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 1, 0, 3927, 0, 0, 0, 0, 3964, 1, 0, 0, 0, 0, 0, 0, 4012, 4, 4, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 3, 0, 1258, 1297, 169, 0, 0, 1875, 0, 4095, 2, 0, 0, 0, 0, 6, 2815, 3, 1, 923, 130, 80, 4, 1, 0, 4088, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3939, 3965, 4020, 4094, 2727, 5972, 3962, 4089] avg=4096.0000 max=5972 max_rank=5 min=2727 min_rank=4 imbalance_ratio=1.458008 max_load_reduction=26796 max_load_reduction_pct=0.817749 moved_to_echo=16750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1, 7, 3914, 0, 0, 0, 0, 3951, 12, 426, 0, 0, 0, 0, 50, 3922, 0, 492, 0, 0, 0, 0, 10, 0, 4086, 8, 0, 0, 0, 0, 0, 0, 1, 2812, 1290, 169, 0, 0, 1866, 0, 2380, 0, 0, 0, 0, 0, 51, 2057, 19, 11, 918, 130, 78, 4, 0, 0, 4077, 24, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3924, 4389, 4464, 4104, 4272, 4246, 3268, 4101] avg=4096.0000 max=4464 max_rank=2 min=3268 min_rank=6 imbalance_ratio=1.089844 max_load_reduction=28304 max_load_reduction_pct=0.863770 moved_to_echo=15887 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[44, 0, 0, 0, 3895, 0, 0, 0, 0, 3966, 4, 1, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 2, 0, 4090, 1, 0, 0, 0, 0, 0, 79, 13, 3404, 1283, 169, 0, 0, 1854, 0, 1956, 0, 0, 0, 0, 0, 2, 2768, 0, 40, 913, 130, 79, 4, 0, 4, 4088, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3939, 3971, 3979, 4093, 4948, 3810, 3936, 4092] avg=4096.0000 max=4948 max_rank=4 min=3810 min_rank=5 imbalance_ratio=1.208008 max_load_reduction=27820 max_load_reduction_pct=0.848999 moved_to_echo=16786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 3, 0, 0, 3903, 0, 0, 0, 0, 3966, 0, 26, 0, 0, 0, 0, 0, 4013, 3, 171, 0, 0, 0, 0, 20, 0, 4072, 41, 0, 0, 0, 0, 0, 0, 0, 2497, 1238, 168, 0, 0, 1789, 0, 1891, 2, 0, 0, 0, 0, 140, 3449, 0, 190, 881, 130, 80, 4, 1, 0, 4085, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3911, 3992, 4187, 4133, 3903, 3682, 4874, 4086] avg=4096.0000 max=4874 max_rank=6 min=3682 min_rank=5 imbalance_ratio=1.189941 max_load_reduction=27894 max_load_reduction_pct=0.851257 moved_to_echo=16545 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1326, 0, 1, 0, 3924, 0, 0, 0, 0, 3959, 0, 58, 0, 0, 0, 0, 5, 4011, 0, 0, 0, 0, 0, 0, 5, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 704, 1296, 170, 0, 0, 1873, 0, 2927, 0, 0, 0, 0, 0, 0, 3177, 0, 5, 922, 130, 80, 4, 0, 3, 4089, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5251, 4017, 4016, 4104, 2170, 4800, 4318, 4092] avg=4096.0000 max=5251 max_rank=0 min=2170 min_rank=4 imbalance_ratio=1.281982 max_load_reduction=27517 max_load_reduction_pct=0.839752 moved_to_echo=15380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 13, 0, 1, 3905, 0, 0, 0, 0, 3908, 3, 0, 0, 0, 0, 0, 29, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 3, 0, 3752, 1301, 168, 0, 0, 1869, 0, 117, 0, 0, 0, 0, 0, 227, 4089, 0, 0, 926, 134, 82, 8, 0, 9, 4066, 54, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3924, 3911, 4043, 4085, 5224, 1986, 5466, 4129] avg=4096.0000 max=5466 max_rank=6 min=1986 min_rank=5 imbalance_ratio=1.334473 max_load_reduction=27302 max_load_reduction_pct=0.833191 moved_to_echo=16805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 3911, 0, 0, 0, 0, 3966, 0, 114, 0, 0, 0, 0, 20, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 17, 0, 3783, 1292, 168, 0, 0, 1869, 0, 468, 0, 0, 0, 0, 0, 0, 3781, 0, 26, 919, 130, 80, 4, 0, 0, 4092, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3927, 4080, 4036, 4096, 5260, 2337, 4940, 4092] avg=4096.0000 max=5260 max_rank=4 min=2337 min_rank=5 imbalance_ratio=1.284180 max_load_reduction=27508 max_load_reduction_pct=0.839478 moved_to_echo=16629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32800, 32768, 32768, 32736, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32800, 32768, 32768, 32736, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32800, 32768, 32768, 32736, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32800, 32768, 32768, 32736, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32800, 32768, 32768, 32736, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32800, 32768, 32768, 32736, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32800, 32768, 32768, 32736, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32800, 32768, 32768, 32736, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14976, 0, 17920, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 21120, 10368, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32736 actual_valid_recv=33408 observed_loss=-672 tokens_per_expert=[1152, 22400, 128, 384, 7424, 1152, 640, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1536, 128, 128, 128, 31360, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32800 actual_valid_recv=33152 observed_loss=-352 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32000, 128, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31744, 128, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 4094, 4096, 0, 96, 0, 205, 0, 3, 21, 4078, 0, 4073, 88, 15, 0, 4096, 0, 12, 637, 1, 0, 0, 0, 0, 0, 4086, 3848, 6, 0, 0, 3313] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[2, 4070, 3710, 27, 20, 0, 0, 0, 2163, 42, 4081, 0, 4096, 2, 0, 3, 4096, 2, 0, 2447, 0, 1544, 1, 0, 0, 0, 4096, 2323, 0, 19, 0, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 4095, 3550, 0, 1579, 1, 9, 0, 2149, 0, 4092, 0, 4029, 111, 0, 0, 4092, 1, 3, 436, 0, 0, 0, 4, 2, 0, 4096, 3748, 1, 0, 0, 770] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[94, 4025, 2740, 0, 2383, 2, 0, 0, 76, 0, 4080, 0, 4066, 0, 0, 0, 4053, 30, 13, 1, 0, 59, 16, 11, 13, 0, 4068, 4095, 7, 5, 146, 2785] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 4063, 3108, 0, 3148, 0, 27, 0, 35, 9, 4092, 1, 4055, 0, 0, 0, 4085, 0, 4, 1677, 0, 4, 0, 0, 1, 3, 4084, 4093, 4, 39, 19, 217] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[5, 4075, 3943, 2, 21, 9, 0, 0, 0, 16, 4096, 0, 4059, 3, 1, 0, 4048, 0, 0, 561, 0, 139, 0, 0, 0, 0, 4087, 4096, 0, 52, 22, 3533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 4081, 5, 0, 0, 0, 0, 0, 0, 4094, 20, 4063, 37, 0, 2, 4093, 0, 0, 2966, 1, 28, 0, 0, 0, 8, 4089, 3879, 0, 1, 0, 1305] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1, 4096, 1968, 0, 3, 0, 0, 0, 89, 71, 4096, 0, 4084, 0, 0, 0, 4096, 0, 0, 1884, 0, 650, 0, 0, 0, 0, 4096, 4056, 0, 0, 0, 3578] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 684, 4096, 0, 0, 0, 0, 0, 96, 0, 205, 0, 3171, 0, 0, 0, 3, 21, 3501, 0, 0, 0, 0, 0, 4071, 88, 15, 0, 0, 0, 0, 0, 2771, 0, 12, 637, 0, 0, 0, 0, 1, 0, 0, 0, 3760, 26, 0, 0, 0, 0, 326, 3848, 0, 0, 0, 0, 6, 0, 0, 3313, 1325, 577, 213, 2] rerouted_tokens_per_ep_rank=[4780, 3472, 3525, 4174, 3420, 3787, 4174, 5436] avg=4096.0000 max=5436 max_rank=7 min=3420 min_rank=4 imbalance_ratio=1.327148 max_load_reduction=27332 max_load_reduction_pct=0.834106 moved_to_echo=16817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 684, 3550, 0, 0, 0, 0, 0, 1579, 1, 9, 0, 3172, 0, 0, 0, 2149, 0, 3513, 0, 0, 0, 0, 0, 4027, 111, 0, 0, 0, 0, 0, 0, 2769, 1, 3, 436, 0, 0, 0, 0, 0, 0, 0, 4, 3769, 26, 0, 0, 2, 0, 327, 3748, 0, 0, 0, 0, 1, 0, 0, 770, 1323, 579, 213, 2] rerouted_tokens_per_ep_rank=[4234, 4761, 5662, 4138, 3209, 3799, 4077, 2888] avg=4096.0000 max=5662 max_rank=2 min=2888 min_rank=7 imbalance_ratio=1.382324 max_load_reduction=27106 max_load_reduction_pct=0.827209 moved_to_echo=13973 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[94, 671, 2740, 0, 0, 0, 0, 0, 2383, 2, 0, 0, 3118, 0, 0, 0, 76, 0, 3502, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 2742, 30, 13, 1, 0, 0, 0, 0, 0, 59, 16, 11, 3744, 26, 0, 0, 13, 0, 324, 4095, 0, 0, 0, 0, 7, 5, 146, 2785, 1311, 578, 210, 2] rerouted_tokens_per_ep_rank=[3505, 5503, 3578, 4064, 2786, 3856, 4432, 5044] avg=4096.0000 max=5503 max_rank=1 min=2786 min_rank=4 imbalance_ratio=1.343506 max_load_reduction=27265 max_load_reduction_pct=0.832062 moved_to_echo=16118 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 680, 3710, 27, 0, 0, 0, 0, 20, 0, 0, 0, 3152, 0, 0, 0, 2163, 42, 3503, 0, 0, 0, 0, 0, 4094, 2, 0, 3, 0, 0, 0, 0, 2771, 2, 0, 2447, 0, 0, 0, 0, 0, 1544, 1, 0, 3769, 26, 0, 0, 0, 0, 327, 2323, 0, 0, 0, 0, 0, 19, 0, 24, 1325, 578, 212, 2] rerouted_tokens_per_ep_rank=[4419, 3172, 5708, 4099, 5220, 5340, 2650, 2160] avg=4096.0000 max=5708 max_rank=2 min=2160 min_rank=7 imbalance_ratio=1.393555 max_load_reduction=27060 max_load_reduction_pct=0.825806 moved_to_echo=15370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 684, 4081, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3173, 0, 0, 0, 0, 0, 3514, 20, 0, 0, 0, 0, 4061, 37, 0, 2, 0, 0, 0, 0, 2769, 0, 0, 2966, 0, 0, 0, 0, 1, 28, 0, 0, 3763, 26, 0, 0, 0, 8, 326, 3879, 0, 0, 0, 0, 0, 1, 0, 1305, 1324, 580, 213, 2] rerouted_tokens_per_ep_rank=[4770, 3173, 3534, 4100, 5735, 3818, 4213, 3425] avg=4096.0000 max=5735 max_rank=4 min=3173 min_rank=1 imbalance_ratio=1.400146 max_load_reduction=27033 max_load_reduction_pct=0.824982 moved_to_echo=17191 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 668, 3108, 0, 0, 0, 0, 0, 3148, 0, 27, 0, 3150, 0, 0, 0, 35, 9, 3508, 1, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 2762, 0, 4, 1677, 0, 0, 0, 0, 0, 4, 0, 0, 3764, 29, 0, 0, 1, 3, 320, 4093, 0, 0, 0, 0, 4, 39, 19, 217, 1323, 584, 216, 5] rerouted_tokens_per_ep_rank=[3776, 6325, 3553, 4050, 4443, 3797, 4417, 2407] avg=4096.0000 max=6325 max_rank=1 min=2407 min_rank=7 imbalance_ratio=1.544189 max_load_reduction=26443 max_load_reduction_pct=0.806976 moved_to_echo=15064 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[5, 681, 3943, 2, 0, 0, 0, 0, 21, 9, 0, 0, 3156, 0, 0, 0, 0, 16, 3516, 0, 0, 0, 0, 0, 4057, 3, 1, 0, 0, 0, 0, 0, 2739, 0, 0, 561, 0, 0, 0, 0, 0, 139, 0, 0, 3761, 26, 0, 0, 0, 0, 326, 4096, 0, 0, 0, 0, 0, 52, 22, 3533, 1309, 580, 212, 2] rerouted_tokens_per_ep_rank=[4631, 3186, 3532, 4061, 3300, 3926, 4422, 5710] avg=4096.0000 max=5710 max_rank=7 min=3186 min_rank=1 imbalance_ratio=1.394043 max_load_reduction=27058 max_load_reduction_pct=0.825745 moved_to_echo=17358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 684, 1968, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3173, 0, 0, 0, 89, 71, 3516, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 2771, 0, 0, 1884, 0, 0, 0, 0, 0, 650, 0, 0, 3769, 26, 0, 0, 0, 0, 327, 4056, 0, 0, 0, 0, 0, 0, 0, 3578, 1325, 580, 213, 2] rerouted_tokens_per_ep_rank=[2653, 3176, 3676, 4082, 4655, 4445, 4383, 5698] avg=4096.0000 max=5698 max_rank=7 min=2653 min_rank=0 imbalance_ratio=1.391113 max_load_reduction=27070 max_load_reduction_pct=0.826111 moved_to_echo=19181 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 256, 15616, 10624, 4736, 1792, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[22144, 128, 128, 10624, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7296, 128, 256, 0, 25344, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 5504, 27264, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 2432, 128, 128, 30208, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 2688, 30208, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4608, 256, 28160, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[19, 4038, 4059, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4075, 0, 15, 0, 4084, 4079, 0, 0, 60, 2434, 1639, 28, 3, 0, 0, 9, 34, 0, 4086, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[5, 4080, 4070, 206, 0, 0, 0, 0, 0, 2, 3977, 42, 4089, 0, 0, 0, 4096, 4088, 2, 0, 17, 4065, 19, 0, 1, 0, 0, 0, 13, 0, 3996, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4084, 4090, 194, 0, 14, 3, 2, 8, 3, 4096, 0, 4086, 0, 0, 12, 4015, 4086, 4, 0, 0, 3864, 0, 0, 5, 97, 0, 6, 0, 3, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[7, 4089, 4096, 206, 0, 0, 0, 0, 0, 0, 4056, 0, 4083, 0, 0, 0, 4082, 4094, 0, 0, 14, 2562, 0, 10, 1, 1380, 24, 0, 0, 10, 4054, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4094, 4091, 0, 7, 0, 0, 2, 11, 2, 4093, 12, 4058, 2, 0, 5, 4092, 4094, 0, 3, 0, 3333, 10, 114, 648, 0, 0, 1, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4068, 4096, 106, 0, 0, 0, 0, 0, 0, 4028, 31, 4096, 0, 0, 0, 4093, 4096, 0, 0, 0, 4020, 34, 1, 0, 0, 24, 0, 6, 1, 4068, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[5, 4090, 4070, 0, 0, 0, 25, 0, 13, 9, 4077, 0, 4084, 0, 0, 3, 4072, 4086, 4, 4, 0, 3054, 50, 3, 61, 960, 10, 0, 0, 0, 4088, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4093, 4096, 0, 0, 0, 4039, 0, 57, 6, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4009, 0, 194, 0, 0, 0, 0, 0, 14, 3, 2, 4090, 5, 0, 0, 8, 3, 4096, 0, 14, 0, 0, 0, 4086, 0, 0, 12, 8, 0, 0, 0, 4015, 17, 4, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 376, 40, 0, 0, 5, 97, 0, 6, 3688, 0, 0, 0, 0, 3, 4096, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4203, 4114, 4121, 4106, 4036, 4280, 3796, 4112] avg=4096.0000 max=4280 max_rank=5 min=3796 min_rank=6 imbalance_ratio=1.044922 max_load_reduction=28488 max_load_reduction_pct=0.869385 moved_to_echo=16224 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[19, 3963, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 5, 0, 0, 0, 0, 4095, 0, 14, 0, 0, 0, 4075, 0, 15, 0, 8, 0, 0, 0, 4084, 18, 0, 0, 0, 0, 0, 0, 60, 2434, 1639, 28, 375, 40, 0, 0, 3, 0, 0, 9, 3681, 0, 0, 0, 34, 0, 4086, 10, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[3983, 4064, 4109, 4098, 4102, 4576, 3693, 4143] avg=4096.0000 max=4576 max_rank=5 min=3693 min_rank=6 imbalance_ratio=1.117188 max_load_reduction=28192 max_load_reduction_pct=0.860352 moved_to_echo=16514 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[7, 4014, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 4056, 0, 14, 0, 0, 0, 4083, 0, 0, 0, 8, 0, 0, 0, 4082, 17, 0, 0, 0, 0, 0, 0, 14, 2562, 0, 10, 377, 40, 0, 0, 1, 1380, 24, 0, 3695, 0, 0, 0, 0, 10, 4054, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4227, 4101, 4070, 4091, 4099, 3003, 5100, 4077] avg=4096.0000 max=5100 max_rank=6 min=3003 min_rank=5 imbalance_ratio=1.245117 max_load_reduction=27668 max_load_reduction_pct=0.844360 moved_to_echo=16279 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4019, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 4091, 5, 0, 0, 11, 2, 4093, 12, 14, 0, 0, 0, 4058, 2, 0, 5, 8, 0, 0, 0, 4092, 17, 0, 3, 0, 0, 0, 0, 0, 3333, 10, 114, 377, 40, 0, 0, 648, 0, 0, 1, 3695, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4105, 4132, 4073, 4112, 3874, 4344, 4109] avg=4096.0000 max=4344 max_rank=6 min=3874 min_rank=5 imbalance_ratio=1.060547 max_load_reduction=28424 max_load_reduction_pct=0.867432 moved_to_echo=16439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5, 4005, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 5, 0, 0, 0, 2, 3977, 42, 14, 0, 0, 0, 4089, 0, 0, 0, 8, 0, 0, 0, 4096, 18, 2, 0, 0, 0, 0, 0, 17, 4065, 19, 0, 376, 40, 0, 0, 1, 0, 0, 0, 3689, 0, 0, 0, 13, 0, 3996, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4216, 4075, 4035, 4097, 4116, 4517, 3690, 4022] avg=4096.0000 max=4517 max_rank=5 min=3690 min_rank=6 imbalance_ratio=1.102783 max_load_reduction=28251 max_load_reduction_pct=0.862152 moved_to_echo=16345 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3993, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 4028, 31, 14, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 4093, 17, 0, 0, 0, 0, 0, 0, 0, 4020, 34, 1, 377, 40, 0, 0, 0, 0, 24, 0, 3697, 0, 0, 0, 6, 1, 4068, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4101, 4073, 4104, 4110, 4472, 3721, 4088] avg=4096.0000 max=4472 max_rank=5 min=3721 min_rank=6 imbalance_ratio=1.091797 max_load_reduction=28296 max_load_reduction_pct=0.863525 moved_to_echo=16391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[5, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4070, 12, 0, 0, 13, 9, 4077, 0, 19, 0, 0, 0, 4084, 0, 0, 3, 11, 0, 0, 0, 4072, 3, 4, 4, 0, 0, 0, 0, 0, 3054, 50, 3, 381, 45, 0, 0, 61, 960, 10, 0, 3690, 0, 0, 0, 0, 0, 4088, 0, 20, 0, 0, 0] rerouted_tokens_per_ep_rank=[4000, 4107, 4118, 4098, 4083, 3533, 4721, 4108] avg=4096.0000 max=4721 max_rank=6 min=3533 min_rank=5 imbalance_ratio=1.152588 max_load_reduction=28047 max_load_reduction_pct=0.855927 moved_to_echo=16445 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 0, 4093, 0, 0, 0, 8, 0, 0, 0, 4093, 17, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 57, 377, 40, 0, 0, 6, 0, 0, 0, 3697, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0] rerouted_tokens_per_ep_rank=[4021, 4101, 4110, 4101, 4110, 4513, 3703, 4109] avg=4096.0000 max=4513 max_rank=5 min=3703 min_rank=6 imbalance_ratio=1.101807 max_load_reduction=28255 max_load_reduction_pct=0.862274 moved_to_echo=16435 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 2560, 128, 128, 29568, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32128, 0, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 27392, 1792, 256, 3072, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 3983, 4057, 3827, 30, 63, 17, 3863, 10, 0, 4079, 0, 4033, 0, 3, 0, 0, 113, 3720, 0, 0, 0, 0, 14, 412, 42, 0, 4049, 0, 0, 452, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[12, 3925, 4074, 4064, 0, 0, 0, 3861, 3, 17, 4096, 0, 4086, 8, 0, 0, 1, 83, 4059, 0, 4, 0, 6, 0, 0, 17, 2, 4089, 35, 0, 320, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[5, 3745, 4088, 4078, 3, 0, 3, 0, 0, 1, 4093, 15, 4047, 1, 274, 0, 0, 1, 4091, 45, 0, 0, 22, 7, 0, 0, 13, 4090, 0, 47, 4085, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4095, 4095, 3909, 0, 0, 0, 1698, 0, 0, 4096, 0, 4096, 0, 0, 1, 6, 10, 4056, 40, 0, 0, 0, 0, 0, 1, 1, 4090, 4, 0, 2570, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4049, 4091, 4090, 4, 4, 4, 3817, 0, 1, 4085, 2, 4091, 0, 1, 0, 0, 0, 4062, 2, 99, 0, 0, 0, 0, 0, 0, 4090, 4, 233, 39, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 4095, 0, 1, 3, 3998, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 20, 1, 78, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4076, 4087, 4059, 0, 0, 49, 4057, 1, 0, 4096, 3, 4077, 0, 1, 0, 19, 0, 4049, 41, 31, 1, 2, 0, 0, 14, 0, 4095, 0, 8, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4010, 4063, 4094, 0, 0, 0, 3821, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 20, 4096, 30, 0, 0, 0, 0, 38, 0, 0, 4094, 0, 199, 111, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[12, 3925, 0, 99, 0, 0, 0, 0, 0, 0, 0, 3861, 879, 58, 4, 0, 3, 17, 4094, 0, 0, 0, 0, 0, 4069, 8, 0, 0, 0, 0, 0, 0, 1, 83, 4059, 0, 13, 2, 0, 0, 4, 0, 6, 0, 4065, 0, 0, 0, 0, 17, 2, 4031, 0, 0, 0, 0, 35, 0, 320, 6, 3086, 9, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4802, 4114, 4077, 4158, 4075, 4050, 3456] avg=4096.0000 max=4802 max_rank=1 min=3456 min_rank=7 imbalance_ratio=1.172363 max_load_reduction=27966 max_load_reduction_pct=0.853455 moved_to_echo=15739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[5, 3745, 0, 99, 0, 0, 0, 0, 3, 0, 3, 0, 882, 58, 4, 0, 0, 1, 4091, 15, 0, 0, 0, 0, 4030, 1, 274, 0, 0, 0, 0, 0, 0, 1, 4091, 45, 13, 2, 0, 0, 0, 0, 22, 7, 4080, 0, 0, 0, 0, 0, 13, 4032, 0, 0, 0, 0, 0, 47, 4085, 14, 3097, 8, 0, 0] rerouted_tokens_per_ep_rank=[3849, 950, 4107, 4305, 4152, 4109, 4045, 7251] avg=4096.0000 max=7251 max_rank=7 min=950 min_rank=1 imbalance_ratio=1.770264 max_load_reduction=25517 max_load_reduction_pct=0.778717 moved_to_echo=19557 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3983, 0, 93, 0, 0, 0, 0, 30, 63, 17, 3863, 828, 57, 4, 0, 10, 0, 4077, 0, 0, 0, 0, 0, 4016, 0, 3, 0, 0, 0, 0, 0, 0, 113, 3720, 0, 13, 2, 0, 0, 0, 0, 0, 14, 4048, 0, 0, 0, 412, 42, 0, 3992, 0, 0, 0, 0, 0, 0, 452, 1, 2906, 9, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4862, 4087, 4019, 3848, 4062, 4446, 3368] avg=4096.0000 max=4862 max_rank=1 min=3368 min_rank=7 imbalance_ratio=1.187012 max_load_reduction=27906 max_load_reduction_pct=0.851624 moved_to_echo=15724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 94, 0, 0, 0, 0, 0, 0, 0, 1698, 846, 58, 4, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4079, 0, 0, 1, 0, 0, 0, 0, 6, 10, 4056, 40, 13, 2, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 1, 1, 4032, 0, 0, 0, 0, 4, 0, 2570, 0, 2969, 9, 0, 0] rerouted_tokens_per_ep_rank=[4189, 2606, 4094, 4080, 4127, 4086, 4034, 5552] avg=4096.0000 max=5552 max_rank=7 min=2606 min_rank=1 imbalance_ratio=1.355469 max_load_reduction=27216 max_load_reduction_pct=0.830566 moved_to_echo=17799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4049, 0, 99, 0, 0, 0, 0, 4, 4, 4, 3817, 885, 58, 4, 0, 0, 1, 4083, 2, 0, 0, 0, 0, 4074, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4062, 2, 13, 2, 0, 0, 99, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 4, 233, 39, 0, 3106, 8, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4776, 4086, 4075, 4079, 4182, 4032, 3390] avg=4096.0000 max=4776 max_rank=1 min=3390 min_rank=7 imbalance_ratio=1.166016 max_load_reduction=27992 max_load_reduction_pct=0.854248 moved_to_echo=15683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 99, 0, 0, 0, 0, 0, 1, 3, 3998, 886, 58, 4, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 13, 2, 0, 0, 0, 0, 0, 2, 4084, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 20, 1, 78, 0, 3110, 9, 0, 0] rerouted_tokens_per_ep_rank=[4195, 4950, 4094, 4078, 4110, 4086, 4037, 3218] avg=4096.0000 max=4950 max_rank=1 min=3218 min_rank=7 imbalance_ratio=1.208496 max_load_reduction=27818 max_load_reduction_pct=0.848938 moved_to_echo=15451 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4076, 0, 90, 0, 0, 0, 0, 0, 0, 49, 4057, 881, 59, 7, 0, 1, 0, 4088, 3, 0, 0, 0, 0, 4054, 0, 1, 0, 0, 0, 0, 0, 19, 0, 4049, 41, 16, 8, 0, 0, 31, 1, 2, 0, 4079, 0, 0, 0, 0, 14, 0, 4036, 0, 0, 0, 0, 0, 8, 0, 2, 3088, 8, 0, 0] rerouted_tokens_per_ep_rank=[4166, 5053, 4092, 4055, 4133, 4113, 4050, 3106] avg=4096.0000 max=5053 max_rank=1 min=3106 min_rank=7 imbalance_ratio=1.233643 max_load_reduction=27715 max_load_reduction_pct=0.845795 moved_to_echo=15402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4010, 0, 99, 0, 0, 0, 0, 0, 0, 0, 3821, 886, 58, 4, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 30, 13, 2, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 38, 0, 0, 4036, 0, 0, 0, 0, 0, 199, 111, 0, 3109, 8, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4769, 4094, 4079, 4161, 4055, 4074, 3427] avg=4096.0000 max=4769 max_rank=1 min=3427 min_rank=7 imbalance_ratio=1.164307 max_load_reduction=27999 max_load_reduction_pct=0.854462 moved_to_echo=15717 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 7680, 128, 24576, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 25216, 7040, 512, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32000, 0, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 128, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 32256, 256, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 128, 128, 32256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[20, 0, 0, 10, 4096, 0, 21, 7, 79, 8, 4040, 4065, 0, 2695, 0, 5, 90, 0, 0, 0, 4077, 0, 25, 0, 1245, 0, 2, 4003, 4052, 1, 132, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[24, 1, 0, 0, 4084, 2, 0, 8, 67, 0, 4091, 4057, 0, 1932, 0, 0, 2792, 0, 0, 12, 4096, 0, 0, 0, 3238, 0, 10, 123, 4096, 0, 156, 3979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[5, 0, 3, 0, 4077, 6, 1, 0, 42, 43, 3697, 4015, 3, 0, 0, 0, 3352, 0, 0, 2, 4066, 0, 0, 0, 3094, 1, 0, 1000, 4096, 0, 1170, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 4095, 0, 49, 0, 70, 0, 4069, 4090, 0, 2248, 0, 8, 3894, 1, 0, 0, 4096, 1, 2, 28, 4066, 0, 0, 0, 1951, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[51, 74, 0, 0, 4053, 4, 0, 0, 0, 14, 4021, 4025, 0, 1476, 0, 2, 3096, 3, 4, 0, 4047, 23, 0, 1, 88, 35, 36, 4016, 4093, 0, 22, 3584] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 4096, 0, 6, 2, 0, 100, 4054, 3488, 0, 2202, 44, 0, 785, 0, 9, 0, 4086, 0, 18, 0, 210, 0, 0, 4094, 4079, 0, 1406, 4080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 40, 4096, 4096, 0, 1413, 3, 0, 3533, 0, 3, 3, 4092, 0, 0, 1, 479, 0, 0, 2576, 4048, 0, 223, 4067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4062, 14, 643, 1, 0, 332, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 3078, 4096, 0, 66, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 0, 10, 4095, 60, 0, 0, 4093, 0, 21, 7, 0, 0, 0, 0, 79, 8, 52, 4065, 0, 0, 0, 0, 0, 2695, 0, 5, 2522, 0, 0, 0, 90, 0, 0, 0, 1406, 576, 122, 2, 4077, 0, 25, 0, 1, 0, 0, 0, 1245, 0, 2, 3427, 0, 0, 0, 0, 3930, 1, 132, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 4121, 4204, 5222, 2196, 4103, 4674, 4063] avg=4096.0000 max=5222 max_rank=3 min=2196 min_rank=4 imbalance_ratio=1.274902 max_load_reduction=27546 max_load_reduction_pct=0.840637 moved_to_echo=15036 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[24, 1, 0, 0, 3979, 60, 0, 0, 4081, 2, 0, 8, 0, 0, 0, 0, 67, 0, 54, 4057, 0, 0, 0, 0, 0, 1932, 0, 0, 2553, 0, 0, 0, 2792, 0, 0, 12, 1424, 17, 123, 2, 4096, 0, 0, 0, 1, 0, 0, 0, 3238, 0, 10, 106, 0, 0, 0, 0, 3973, 0, 156, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4091, 4178, 4485, 4370, 4097, 3354, 4129] avg=4096.0000 max=4485 max_rank=3 min=3354 min_rank=6 imbalance_ratio=1.094971 max_load_reduction=28283 max_load_reduction_pct=0.863129 moved_to_echo=15950 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[51, 74, 0, 0, 3584, 59, 0, 0, 4050, 4, 0, 0, 0, 0, 0, 0, 0, 14, 52, 4025, 0, 0, 0, 0, 0, 1476, 0, 2, 2510, 0, 0, 0, 3096, 3, 4, 0, 1400, 578, 123, 2, 4047, 23, 0, 1, 1, 0, 0, 0, 88, 35, 36, 3438, 0, 0, 0, 0, 3970, 0, 22, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3768, 4054, 4091, 3988, 5206, 4072, 3597, 3992] avg=4096.0000 max=5206 max_rank=4 min=3597 min_rank=6 imbalance_ratio=1.270996 max_load_reduction=27562 max_load_reduction_pct=0.841125 moved_to_echo=16867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 3, 0, 4095, 54, 0, 0, 4074, 6, 1, 0, 0, 0, 0, 0, 42, 43, 49, 4015, 0, 0, 0, 0, 3, 0, 0, 0, 2307, 0, 0, 0, 3352, 0, 0, 2, 1287, 144, 123, 2, 4066, 0, 0, 0, 1, 0, 0, 0, 3094, 1, 0, 856, 0, 0, 0, 0, 3973, 0, 1170, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4081, 4149, 2310, 4910, 4067, 3951, 5143] avg=4096.0000 max=5143 max_rank=7 min=2310 min_rank=3 imbalance_ratio=1.255615 max_load_reduction=27625 max_load_reduction_pct=0.843048 moved_to_echo=18071 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 4094, 60, 0, 0, 4092, 0, 49, 0, 0, 0, 0, 0, 70, 0, 53, 4090, 0, 0, 0, 0, 0, 2248, 0, 8, 2540, 0, 0, 0, 3894, 1, 0, 0, 1416, 0, 58, 2, 4096, 1, 2, 28, 1, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 1893, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4141, 4213, 4796, 5371, 4128, 4066, 1893] avg=4096.0000 max=5371 max_rank=4 min=1893 min_rank=7 imbalance_ratio=1.311279 max_load_reduction=27397 max_load_reduction_pct=0.836090 moved_to_echo=15458 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4067, 60, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 40, 53, 4096, 0, 0, 0, 0, 0, 1413, 3, 0, 2557, 0, 0, 0, 3533, 0, 3, 3, 1426, 371, 121, 2, 4092, 0, 0, 1, 1, 0, 0, 0, 479, 0, 0, 2205, 0, 0, 0, 0, 3927, 0, 223, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4092, 4189, 3973, 5459, 4094, 2684, 4150] avg=4096.0000 max=5459 max_rank=4 min=2684 min_rank=6 imbalance_ratio=1.332764 max_load_reduction=27309 max_load_reduction_pct=0.833405 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 4080, 65, 0, 0, 4087, 0, 6, 2, 0, 0, 0, 0, 0, 100, 41, 3488, 0, 0, 0, 0, 0, 2202, 44, 0, 2533, 0, 0, 0, 785, 0, 9, 0, 1415, 592, 126, 3, 4086, 0, 18, 0, 6, 0, 0, 0, 210, 0, 0, 3502, 0, 0, 0, 0, 3953, 0, 1406, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4095, 3629, 4779, 2930, 4110, 3712, 5359] avg=4096.0000 max=5359 max_rank=7 min=2930 min_rank=4 imbalance_ratio=1.308350 max_load_reduction=27409 max_load_reduction_pct=0.836456 moved_to_echo=16111 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 60, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4062, 0, 0, 0, 0, 14, 643, 1, 0, 2557, 0, 0, 0, 332, 0, 0, 0, 1426, 443, 123, 2, 4096, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 2635, 0, 0, 0, 0, 3973, 0, 66, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4093, 4115, 3215, 2326, 4097, 6730, 4039] avg=4096.0000 max=6730 max_rank=6 min=2326 min_rank=4 imbalance_ratio=1.643066 max_load_reduction=26038 max_load_reduction_pct=0.794617 moved_to_echo=17192 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[17920, 128, 128, 128, 11264, 2816, 1024, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 12672, 128, 128, 20096, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16640, 128, 128, 16256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32128, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[29696, 128, 3200, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 256, 512, 32000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 18, 0, 0, 0, 4090, 4096, 4096, 0, 0, 3700, 2, 0, 800, 0, 4095, 26, 4096, 3, 0, 0, 0, 233, 4, 3413, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[5, 4, 20, 2, 101, 5, 19, 0, 4093, 4051, 4096, 0, 0, 3607, 0, 0, 0, 8, 4071, 0, 3986, 0, 0, 0, 0, 214, 571, 3822, 4077, 0, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 5, 0, 3, 0, 4096, 4095, 4044, 1, 0, 3909, 0, 0, 103, 0, 4096, 0, 4071, 1, 0, 0, 0, 131, 0, 4095, 4089, 26, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3, 13, 0, 0, 0, 0, 5, 6, 3779, 2289, 4091, 0, 8, 2186, 2, 3, 0, 0, 4087, 31, 4090, 1, 0, 1, 1, 3629, 0, 4022, 4061, 0, 457, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 13, 0, 14, 10, 0, 0, 0, 4096, 4042, 4076, 0, 7, 4096, 5, 2, 28, 3, 4096, 0, 4082, 0, 0, 0, 0, 0, 0, 4093, 4084, 0, 0, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 144, 8, 0, 8, 0, 1, 0, 4096, 4078, 4031, 0, 0, 3538, 0, 0, 0, 0, 4095, 153, 4096, 0, 1, 0, 0, 0, 332, 4091, 4095, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 0, 0, 0, 4096, 4020, 4086, 0, 0, 0, 0, 0, 0, 0, 4095, 1415, 4060, 2, 23, 0, 0, 3459, 19, 3383, 4096, 12, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4, 6, 0, 0, 592, 0, 25, 0, 4092, 4073, 3993, 6, 0, 2114, 0, 30, 229, 15, 4068, 30, 4082, 0, 39, 0, 9, 1272, 0, 3889, 4079, 83, 8, 30] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[5, 4, 20, 2, 4096, 1, 0, 0, 101, 5, 19, 0, 3836, 193, 0, 0, 256, 4051, 0, 0, 0, 0, 0, 0, 0, 3607, 0, 0, 787, 345, 53, 0, 0, 8, 3726, 0, 0, 0, 0, 0, 3986, 0, 0, 0, 16, 0, 0, 0, 0, 214, 571, 2842, 0, 0, 0, 0, 4008, 0, 16, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4154, 4307, 4792, 3734, 4002, 3627, 4024] avg=4096.0000 max=4792 max_rank=3 min=3627 min_rank=6 imbalance_ratio=1.169922 max_load_reduction=27976 max_load_reduction_pct=0.853760 moved_to_echo=15387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4044, 1, 0, 0, 5, 0, 3, 0, 3839, 207, 0, 0, 256, 4095, 0, 1, 0, 0, 0, 0, 0, 3909, 0, 0, 844, 347, 54, 0, 103, 0, 3749, 0, 0, 0, 0, 0, 4071, 1, 0, 0, 16, 0, 0, 0, 0, 131, 0, 3044, 0, 0, 0, 0, 4019, 26, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4054, 4352, 5154, 3852, 4088, 3175, 4045] avg=4096.0000 max=5154 max_rank=3 min=3175 min_rank=6 imbalance_ratio=1.258301 max_load_reduction=27614 max_load_reduction_pct=0.842712 moved_to_echo=15160 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 0, 0, 18, 0, 0, 0, 3833, 173, 0, 0, 256, 4096, 0, 0, 0, 0, 0, 0, 0, 3700, 2, 0, 703, 347, 54, 0, 800, 0, 3748, 26, 0, 0, 0, 0, 4096, 3, 0, 0, 16, 0, 0, 0, 0, 233, 4, 2537, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4024, 4352, 4806, 4574, 4115, 2774, 4026] avg=4096.0000 max=4806 max_rank=3 min=2774 min_rank=6 imbalance_ratio=1.173340 max_load_reduction=27962 max_load_reduction_pct=0.853333 moved_to_echo=15489 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 13, 0, 0, 4091, 1, 0, 0, 0, 0, 5, 6, 3542, 204, 0, 0, 236, 2289, 0, 0, 0, 0, 0, 0, 8, 2186, 2, 3, 829, 346, 53, 0, 0, 0, 3741, 31, 0, 0, 0, 0, 4090, 1, 0, 1, 16, 0, 0, 0, 1, 3629, 0, 2989, 0, 0, 0, 0, 3992, 0, 457, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 3757, 2525, 3427, 3772, 4108, 6619, 4452] avg=4096.0000 max=6619 max_rank=6 min=2525 min_rank=2 imbalance_ratio=1.615967 max_load_reduction=26149 max_load_reduction_pct=0.798004 moved_to_echo=18951 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4, 6, 0, 0, 3993, 1, 0, 0, 592, 0, 25, 0, 3835, 197, 0, 0, 256, 4073, 0, 6, 0, 0, 0, 0, 0, 2114, 0, 30, 801, 345, 53, 0, 229, 15, 3723, 30, 0, 0, 0, 0, 4082, 0, 39, 0, 16, 0, 0, 0, 9, 1272, 0, 2891, 0, 0, 0, 0, 4010, 83, 8, 30, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4004, 4649, 4335, 3343, 3997, 4137, 4172, 4131] avg=4096.0000 max=4649 max_rank=1 min=3343 min_rank=3 imbalance_ratio=1.135010 max_load_reduction=28119 max_load_reduction_pct=0.858124 moved_to_echo=16437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 144, 8, 0, 4031, 1, 0, 0, 8, 0, 1, 0, 3839, 207, 0, 0, 256, 4078, 0, 0, 0, 0, 0, 0, 0, 3538, 0, 0, 843, 347, 54, 0, 0, 0, 3748, 153, 0, 0, 0, 0, 4096, 0, 1, 0, 16, 0, 0, 0, 0, 0, 332, 3041, 0, 0, 0, 0, 4025, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4184, 4055, 4334, 4782, 3901, 4113, 3373, 4026] avg=4096.0000 max=4782 max_rank=3 min=3373 min_rank=6 imbalance_ratio=1.167480 max_load_reduction=27986 max_load_reduction_pct=0.854065 moved_to_echo=15413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 0, 14, 4076, 8, 0, 0, 10, 0, 0, 0, 3843, 211, 0, 0, 245, 4042, 0, 0, 0, 0, 0, 0, 7, 4096, 5, 2, 847, 352, 57, 0, 28, 3, 3744, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 3035, 0, 0, 0, 0, 4005, 0, 0, 21, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4064, 4287, 5366, 3775, 4104, 3035, 4026] avg=4096.0000 max=5366 max_rank=3 min=3035 min_rank=6 imbalance_ratio=1.310059 max_load_reduction=27402 max_load_reduction_pct=0.836243 moved_to_echo=14940 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4086, 1, 0, 0, 1, 0, 0, 0, 3839, 171, 0, 0, 256, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 347, 54, 0, 0, 0, 3748, 1415, 0, 0, 0, 0, 4060, 2, 23, 0, 16, 0, 0, 0, 0, 3459, 19, 2515, 0, 0, 0, 0, 4026, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4011, 4276, 1098, 5163, 4101, 5993, 4038] avg=4096.0000 max=5993 max_rank=6 min=1098 min_rank=3 imbalance_ratio=1.463135 max_load_reduction=26775 max_load_reduction_pct=0.817108 moved_to_echo=19295 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 8960, 1024, 22912, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 23168, 128, 128, 6400, 2816, 512, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2048, 30848, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 128, 128, 128, 30464, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 128, 29952, 1664, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32128, 128, 512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4030, 0, 4096, 0, 0, 0, 0, 4092, 1, 4061, 0, 0, 204, 0, 0, 0, 4094, 4096, 10, 28, 0, 3867, 17, 0, 0, 4096, 0, 71, 0, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4091, 4, 4094, 0, 1, 0, 30, 4084, 0, 4096, 0, 0, 223, 14, 0, 1, 4092, 4096, 0, 0, 17, 3295, 2, 4, 0, 4031, 0, 576, 0, 0, 17, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4052, 0, 4090, 0, 0, 0, 0, 4095, 0, 4059, 0, 1, 131, 0, 39, 5, 4094, 4081, 0, 25, 0, 3986, 7, 0, 0, 4095, 0, 3, 4, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4040, 0, 3990, 0, 0, 0, 0, 4050, 0, 3963, 17, 31, 0, 25, 0, 0, 4065, 4082, 163, 0, 0, 4070, 8, 0, 4, 4076, 20, 0, 47, 0, 0, 117] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 4087, 0, 0, 2, 0, 4095, 2, 4087, 9, 0, 33, 1, 0, 2, 4094, 3935, 147, 2, 0, 4093, 1, 2, 0, 4063, 2, 0, 16, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[3912, 0, 4096, 0, 0, 0, 0, 4093, 1, 4095, 0, 0, 1, 0, 0, 0, 4082, 4096, 1144, 2, 14, 2421, 0, 0, 0, 4096, 0, 489, 0, 0, 226, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4002, 7, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 454, 0, 0, 0, 4096, 4050, 135, 0, 66, 3665, 0, 0, 0, 3989, 0, 13, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4086, 0, 4066, 0, 67, 2, 1, 4081, 10, 4030, 17, 0, 1, 0, 0, 9, 4096, 4083, 34, 0, 0, 129, 0, 0, 11, 4096, 0, 3949, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4030, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 1, 4061, 0, 0, 21, 2, 0, 0, 204, 0, 0, 0, 3923, 47, 0, 0, 0, 3916, 10, 28, 0, 0, 0, 0, 0, 3867, 17, 0, 618, 159, 115, 0, 0, 3478, 0, 71, 0, 0, 0, 0, 0, 0, 5, 0, 4047, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4090, 4085, 4174, 3954, 4776, 3549, 4052] avg=4096.0000 max=4776 max_rank=5 min=3549 min_rank=6 imbalance_ratio=1.166016 max_load_reduction=27992 max_load_reduction_pct=0.854248 moved_to_echo=16331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 57, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 2, 4087, 9, 0, 20, 2, 0, 0, 33, 1, 0, 2, 3915, 48, 0, 0, 0, 3762, 147, 2, 0, 0, 0, 0, 0, 4093, 1, 2, 613, 153, 115, 0, 0, 3450, 2, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4046, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4095, 4120, 3999, 3911, 4977, 3452, 4062] avg=4096.0000 max=4977 max_rank=5 min=3452 min_rank=6 imbalance_ratio=1.215088 max_load_reduction=27791 max_load_reduction_pct=0.848114 moved_to_echo=16402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4040, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 3963, 17, 31, 21, 2, 0, 0, 0, 25, 0, 0, 3822, 47, 0, 0, 0, 3902, 163, 0, 0, 0, 0, 0, 0, 4070, 8, 0, 615, 159, 112, 0, 4, 3461, 20, 0, 0, 0, 0, 0, 47, 0, 0, 117, 4018, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4048, 4034, 3894, 4065, 4964, 3485, 4182] avg=4096.0000 max=4964 max_rank=5 min=3485 min_rank=6 imbalance_ratio=1.211914 max_load_reduction=27804 max_load_reduction_pct=0.848511 moved_to_echo=16696 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 57, 0, 0, 0, 0, 0, 67, 2, 1, 4079, 0, 0, 0, 0, 10, 4030, 17, 0, 21, 2, 0, 0, 1, 0, 0, 9, 3895, 48, 0, 0, 0, 3903, 34, 0, 0, 0, 0, 0, 0, 129, 0, 0, 618, 159, 114, 0, 11, 3478, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4149, 4080, 3953, 3937, 1020, 7438, 4048] avg=4096.0000 max=7438 max_rank=6 min=1020 min_rank=5 imbalance_ratio=1.815918 max_load_reduction=25330 max_load_reduction_pct=0.773010 moved_to_echo=16443 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4052, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4059, 0, 1, 21, 2, 0, 0, 131, 0, 39, 5, 3918, 47, 0, 0, 0, 3901, 0, 25, 0, 0, 0, 0, 0, 3986, 7, 0, 618, 159, 115, 0, 0, 3477, 0, 3, 0, 0, 0, 0, 4, 0, 0, 1, 4047, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4093, 4083, 4140, 3926, 4885, 3480, 4052] avg=4096.0000 max=4885 max_rank=5 min=3480 min_rank=6 imbalance_ratio=1.192627 max_load_reduction=27883 max_load_reduction_pct=0.850922 moved_to_echo=16343 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4091, 4, 48, 0, 0, 0, 0, 0, 1, 0, 30, 4077, 0, 0, 0, 0, 0, 4096, 0, 0, 25, 7, 0, 0, 223, 14, 0, 1, 3925, 48, 0, 0, 0, 3907, 0, 0, 0, 0, 0, 0, 17, 3295, 2, 4, 610, 164, 121, 0, 0, 3421, 0, 576, 0, 0, 0, 0, 0, 0, 17, 0, 4044, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4108, 4128, 4211, 3907, 4213, 3997, 4061] avg=4096.0000 max=4213 max_rank=5 min=3907 min_rank=4 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3912, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 1, 4095, 0, 0, 21, 2, 0, 0, 1, 0, 0, 0, 3923, 48, 0, 0, 0, 3916, 1144, 2, 0, 0, 0, 0, 14, 2421, 0, 0, 618, 159, 115, 0, 0, 3478, 0, 489, 0, 0, 0, 0, 0, 0, 226, 0, 4034, 0, 0, 0] rerouted_tokens_per_ep_rank=[3970, 4091, 4119, 3972, 5062, 3327, 3967, 4260] avg=4096.0000 max=5062 max_rank=4 min=3327 min_rank=5 imbalance_ratio=1.235840 max_load_reduction=27706 max_load_reduction_pct=0.845520 moved_to_echo=16616 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4002, 7, 58, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 21, 2, 0, 0, 454, 0, 0, 0, 3923, 48, 0, 0, 0, 3871, 135, 0, 0, 0, 0, 0, 66, 3665, 0, 0, 602, 158, 115, 0, 0, 3387, 0, 13, 0, 0, 0, 0, 0, 0, 3, 0, 4048, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4094, 4119, 4425, 4006, 4606, 3400, 4051] avg=4096.0000 max=4606 max_rank=5 min=3400 min_rank=6 imbalance_ratio=1.124512 max_load_reduction=28162 max_load_reduction_pct=0.859436 moved_to_echo=16063 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 25600, 128, 128, 4992, 1280, 1024, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 31104, 1664, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32384, 128, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 27648, 128, 5120, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 256, 128, 32384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1152, 128, 128, 128, 31360, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32512, 128, 128, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 2105, 0, 4094, 24, 4092, 0, 2, 1, 3748, 23, 4008, 0, 1852, 0, 3, 0, 4, 4096, 0, 418, 0, 0, 0, 1, 4, 54, 4063, 84, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 7, 3571, 17, 4021, 116, 4057, 1, 0, 0, 4090, 0, 4084, 10, 219, 0, 10, 2, 0, 3669, 0, 0, 0, 159, 0, 4, 0, 24, 4067, 561, 4079, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 145, 0, 4096, 5, 3994, 0, 0, 0, 4091, 0, 4089, 0, 4096, 0, 0, 0, 0, 3932, 14, 79, 37, 0, 0, 0, 0, 0, 4093, 0, 4096, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4038, 0, 4096, 0, 4088, 0, 6, 0, 4063, 2, 4090, 0, 1125, 2, 0, 0, 0, 3050, 3, 0, 7, 0, 0, 0, 5, 0, 4096, 1, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 3969, 2, 4081, 0, 3986, 17, 84, 14, 4025, 0, 4081, 0, 4096, 69, 0, 7, 0, 89, 0, 23, 0, 34, 13, 24, 0, 0, 4081, 0, 4072, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 3298, 0, 4047, 1, 4087, 0, 0, 0, 4095, 0, 4096, 3, 1068, 0, 0, 2, 1, 3780, 0, 0, 2, 86, 0, 0, 0, 1, 4094, 0, 4090, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[22, 8, 3916, 0, 4081, 0, 4095, 0, 0, 0, 4096, 25, 4052, 0, 1214, 0, 13, 0, 0, 3061, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4085, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 40, 0, 4096, 0, 3964, 0, 0, 0, 3994, 0, 4095, 0, 3932, 0, 0, 0, 0, 4096, 0, 368, 0, 0, 0, 0, 0, 0, 4050, 37, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 3571, 17, 1343, 111, 0, 0, 31, 116, 4057, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 50, 0, 0, 0, 1891, 10, 219, 0, 0, 0, 0, 0, 10, 2, 0, 3669, 850, 20, 0, 0, 0, 0, 0, 159, 3879, 8, 0, 0, 0, 4, 0, 24, 4071, 0, 0, 0, 3997, 561, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5049, 4205, 4140, 2120, 4551, 4046, 4099, 4558] avg=4096.0000 max=5049 max_rank=0 min=2120 min_rank=3 imbalance_ratio=1.232666 max_load_reduction=27719 max_load_reduction_pct=0.845917 moved_to_echo=17254 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2105, 0, 1318, 113, 0, 0, 31, 24, 4092, 0, 0, 0, 0, 0, 2, 1, 3748, 23, 50, 0, 0, 0, 1855, 0, 1852, 0, 0, 0, 0, 0, 3, 0, 4, 4096, 835, 20, 0, 0, 0, 418, 0, 0, 3950, 9, 0, 0, 0, 1, 4, 54, 4083, 0, 0, 0, 3993, 84, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3536, 4147, 3824, 3707, 4958, 4377, 4142, 4077] avg=4096.0000 max=4958 max_rank=4 min=3536 min_rank=0 imbalance_ratio=1.210449 max_load_reduction=27810 max_load_reduction_pct=0.848694 moved_to_echo=17554 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3969, 2, 1342, 113, 0, 0, 31, 0, 3986, 17, 0, 0, 0, 0, 84, 14, 4025, 0, 51, 0, 0, 0, 1889, 0, 4096, 69, 0, 0, 0, 0, 0, 7, 0, 89, 850, 20, 0, 0, 0, 23, 0, 34, 3937, 8, 0, 0, 13, 24, 0, 0, 4064, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5427, 4034, 4174, 6054, 966, 4002, 4101, 4010] avg=4096.0000 max=6054 max_rank=3 min=966 min_rank=4 imbalance_ratio=1.478027 max_load_reduction=26714 max_load_reduction_pct=0.815247 moved_to_echo=13079 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4038, 0, 1345, 113, 0, 0, 31, 0, 4088, 0, 0, 0, 0, 0, 6, 0, 4063, 2, 51, 0, 0, 0, 1893, 0, 1125, 2, 0, 0, 0, 0, 0, 0, 0, 3050, 852, 20, 0, 0, 3, 0, 7, 0, 3952, 9, 0, 0, 0, 0, 5, 0, 4087, 0, 0, 0, 4025, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5496, 4119, 4122, 3020, 3922, 3971, 4092, 4026] avg=4096.0000 max=5496 max_rank=0 min=3020 min_rank=3 imbalance_ratio=1.341797 max_load_reduction=27272 max_load_reduction_pct=0.832275 moved_to_echo=16011 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 145, 0, 1345, 113, 0, 0, 31, 5, 3994, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 51, 0, 0, 0, 1893, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 851, 20, 0, 0, 14, 79, 37, 0, 3952, 10, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4022, 0, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1603, 4030, 4142, 5989, 4803, 4092, 4086, 4023] avg=4096.0000 max=5989 max_rank=3 min=1603 min_rank=0 imbalance_ratio=1.462158 max_load_reduction=26779 max_load_reduction_pct=0.817230 moved_to_echo=17004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3298, 0, 1347, 112, 0, 0, 31, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 51, 0, 0, 0, 1896, 3, 1068, 0, 0, 0, 0, 0, 0, 2, 1, 3780, 853, 20, 0, 0, 0, 0, 2, 86, 3904, 9, 0, 0, 0, 0, 0, 1, 4081, 0, 0, 0, 4023, 0, 0, 15, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4759, 4119, 4146, 2967, 4656, 4001, 4082, 4038] avg=4096.0000 max=4759 max_rank=0 min=2967 min_rank=3 imbalance_ratio=1.161865 max_load_reduction=28009 max_load_reduction_pct=0.854767 moved_to_echo=16777 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[22, 8, 3916, 0, 1335, 117, 0, 0, 24, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 55, 0, 0, 0, 1870, 0, 1214, 0, 0, 0, 0, 0, 13, 0, 0, 3061, 847, 22, 0, 0, 0, 0, 0, 0, 3940, 9, 0, 0, 0, 0, 0, 4, 4076, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5398, 4119, 4176, 3084, 3943, 3949, 4080, 4019] avg=4096.0000 max=5398 max_rank=0 min=3084 min_rank=3 imbalance_ratio=1.317871 max_load_reduction=27370 max_load_reduction_pct=0.835266 moved_to_echo=15991 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 40, 0, 1347, 113, 0, 0, 31, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 50, 0, 0, 0, 1895, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 853, 20, 0, 0, 0, 368, 0, 0, 3952, 10, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 3980, 37, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1500, 3995, 4044, 5827, 4969, 4330, 4086, 4017] avg=4096.0000 max=5827 max_rank=3 min=1500 min_rank=0 imbalance_ratio=1.422607 max_load_reduction=26941 max_load_reduction_pct=0.822174 moved_to_echo=17402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 21120, 128, 10752, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15104, 128, 17664, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 25856, 6912, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32128, 768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 896, 128, 384, 31488, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 256, 32384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32256, 128, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[3994, 0, 2, 0, 0, 4075, 4082, 14, 2, 4084, 4094, 0, 11, 3876, 88, 6, 0, 225, 4086, 0, 1, 0, 18, 0, 10, 0, 0, 19, 4044, 0, 28, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4, 4096, 4096, 0, 0, 4094, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 7, 0, 4096, 4096, 0, 0, 4087, 4095, 0, 0, 228, 0, 0, 0, 0, 4096, 0, 0, 0, 3812, 0, 54, 0, 1, 0, 4096, 4, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4096, 6, 0, 0, 0, 4083, 4096, 0, 0, 4063, 4089, 0, 0, 4087, 0, 0, 0, 0, 4094, 0, 1, 0, 1, 0, 33, 2, 0, 0, 4092, 7, 18, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4091, 0, 0, 6, 2, 4091, 3713, 2, 2, 4093, 4093, 0, 0, 2759, 673, 2, 1, 0, 4094, 3, 0, 0, 1598, 0, 83, 0, 0, 11, 3440, 4, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4057, 0, 0, 120, 0, 4088, 4084, 31, 6, 4075, 3977, 0, 11, 4065, 0, 0, 7, 19, 4082, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 33, 38, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 76, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4012, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4074, 0, 0, 0, 0, 3996, 4087, 0, 8, 4096, 4018, 17, 0, 3987, 0, 5, 0, 200, 4074, 0, 93, 0, 0, 0, 0, 1, 0, 16, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3994, 0, 2, 0, 3, 0, 0, 0, 0, 38, 4082, 14, 0, 0, 0, 0, 2, 22, 4094, 0, 0, 0, 0, 0, 11, 3876, 88, 6, 597, 0, 0, 0, 0, 225, 4035, 0, 0, 0, 0, 0, 1, 0, 18, 0, 3401, 0, 0, 0, 10, 0, 0, 19, 4062, 3, 0, 0, 4044, 0, 28, 9, 48, 36, 0, 0] rerouted_tokens_per_ep_rank=[3999, 4134, 4118, 4578, 4260, 3420, 4094, 4165] avg=4096.0000 max=4578 max_rank=3 min=3420 min_rank=5 imbalance_ratio=1.117676 max_load_reduction=28190 max_load_reduction_pct=0.860291 moved_to_echo=15939 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4057, 0, 0, 120, 3, 0, 0, 0, 0, 38, 4084, 31, 0, 0, 0, 0, 6, 22, 3977, 0, 0, 0, 0, 0, 11, 4065, 0, 0, 599, 0, 0, 0, 7, 19, 4031, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 4053, 3, 0, 0, 4065, 33, 38, 0, 48, 36, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4153, 4005, 4675, 4067, 3412, 4056, 4220] avg=4096.0000 max=4675 max_rank=3 min=3412 min_rank=5 imbalance_ratio=1.141357 max_load_reduction=28093 max_load_reduction_pct=0.857330 moved_to_echo=15755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4096, 6, 0, 0, 3, 0, 0, 0, 0, 38, 4096, 0, 0, 0, 0, 0, 0, 22, 4089, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 598, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3408, 0, 0, 0, 33, 2, 0, 0, 4041, 3, 0, 0, 4092, 7, 18, 0, 48, 36, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4134, 4111, 4685, 4043, 3410, 4079, 4201] avg=4096.0000 max=4685 max_rank=3 min=3410 min_rank=5 imbalance_ratio=1.143799 max_load_reduction=28083 max_load_reduction_pct=0.857025 moved_to_echo=15733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 6, 3, 0, 0, 0, 2, 38, 3713, 2, 0, 0, 0, 0, 2, 22, 4093, 0, 0, 0, 0, 0, 0, 2759, 673, 2, 599, 0, 0, 0, 1, 0, 4043, 3, 0, 0, 0, 0, 0, 0, 1598, 0, 3415, 0, 0, 0, 83, 0, 0, 11, 4071, 3, 0, 0, 3440, 4, 7, 0, 48, 36, 0, 0] rerouted_tokens_per_ep_rank=[4100, 3755, 4117, 4033, 4047, 5013, 4168, 3535] avg=4096.0000 max=5013 max_rank=5 min=3535 min_rank=7 imbalance_ratio=1.223877 max_load_reduction=27755 max_load_reduction_pct=0.847015 moved_to_echo=16763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 7, 3, 0, 0, 0, 0, 38, 4096, 0, 0, 0, 0, 0, 0, 22, 4095, 0, 0, 0, 0, 0, 0, 228, 0, 0, 600, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 3812, 0, 3419, 0, 0, 0, 54, 0, 1, 0, 4065, 3, 0, 0, 4096, 4, 0, 0, 48, 36, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4134, 4117, 828, 4045, 7231, 4123, 4184] avg=4096.0000 max=7231 max_rank=5 min=828 min_rank=3 imbalance_ratio=1.765381 max_load_reduction=25537 max_load_reduction_pct=0.779327 moved_to_echo=19583 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 4, 38, 4096, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 600, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 0, 0, 0, 0, 0, 0, 0, 4072, 3, 0, 0, 4094, 0, 0, 0, 48, 36, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4138, 4118, 4696, 4045, 3419, 4075, 4178] avg=4096.0000 max=4696 max_rank=3 min=3419 min_rank=5 imbalance_ratio=1.146484 max_load_reduction=28072 max_load_reduction_pct=0.856689 moved_to_echo=15717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4074, 0, 0, 0, 6, 0, 0, 0, 0, 23, 4087, 0, 0, 0, 0, 0, 8, 21, 4018, 17, 0, 0, 0, 0, 0, 3987, 0, 5, 588, 0, 0, 0, 0, 200, 4016, 0, 0, 0, 0, 0, 93, 0, 0, 0, 3340, 0, 0, 0, 0, 1, 0, 16, 4075, 4, 0, 0, 4096, 0, 0, 0, 52, 41, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4110, 4064, 4580, 4216, 3433, 4096, 4189] avg=4096.0000 max=4580 max_rank=3 min=3433 min_rank=5 imbalance_ratio=1.118164 max_load_reduction=28188 max_load_reduction_pct=0.860229 moved_to_echo=15934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 76, 38, 4096, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 600, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3419, 0, 0, 0, 0, 0, 0, 0, 4074, 3, 0, 0, 4012, 0, 0, 0, 48, 36, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4210, 4118, 4693, 4045, 3430, 4077, 4096] avg=4096.0000 max=4693 max_rank=3 min=3430 min_rank=5 imbalance_ratio=1.145752 max_load_reduction=28075 max_load_reduction_pct=0.856781 moved_to_echo=15648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 5504, 0, 27264, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 27264, 768, 128, 4864, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32000, 128, 128, 128, 512, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 32384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 256, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 384, 32384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[31, 3244, 30, 0, 4080, 8, 0, 0, 4086, 4094, 207, 0, 4068, 0, 3803, 0, 9, 25, 9, 22, 18, 1, 4077, 0, 1, 2, 0, 4072, 863, 2, 0, 16] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 1681, 0, 0, 4093, 0, 0, 0, 4095, 4094, 135, 0, 4084, 38, 3926, 0, 22, 0, 0, 0, 5, 0, 4091, 3, 0, 1, 0, 4094, 2404, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[5, 4010, 8, 0, 3983, 0, 0, 0, 1115, 4088, 0, 0, 4092, 2981, 4069, 2, 4, 1, 11, 0, 0, 0, 4089, 0, 1, 177, 37, 4084, 0, 0, 11, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 1368, 4, 20, 4076, 0, 0, 23, 4092, 4072, 1, 0, 4096, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 4078, 2728, 0, 0, 70] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 3728, 1, 0, 4093, 0, 0, 0, 4085, 4096, 0, 0, 4095, 120, 4016, 0, 1, 0, 6, 0, 0, 228, 4089, 111, 0, 0, 0, 4096, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[3, 70, 0, 0, 3919, 8, 0, 0, 4071, 4008, 0, 0, 4095, 111, 4096, 0, 0, 0, 12, 3, 0, 189, 4089, 0, 0, 186, 194, 4082, 3618, 1, 2, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 2718, 0, 0, 4093, 0, 0, 0, 4096, 4094, 0, 1, 4096, 0, 4028, 0, 0, 0, 7, 0, 0, 2, 4093, 0, 3, 0, 4, 4095, 1438, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 3876, 0, 0, 4096, 0, 0, 0, 4096, 4088, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 212, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[31, 3244, 30, 0, 1057, 302, 95, 28, 4080, 8, 0, 0, 36, 0, 0, 0, 4086, 339, 207, 0, 0, 0, 0, 0, 0, 0, 3501, 0, 0, 0, 0, 0, 9, 25, 9, 22, 4057, 0, 0, 0, 18, 1, 4013, 0, 0, 0, 0, 0, 1, 2, 0, 3977, 0, 0, 0, 0, 863, 2, 0, 16, 2698, 11, 0, 0] rerouted_tokens_per_ep_rank=[4787, 4124, 4632, 3501, 4122, 4032, 3980, 3590] avg=4096.0000 max=4787 max_rank=0 min=3501 min_rank=3 imbalance_ratio=1.168701 max_load_reduction=27981 max_load_reduction_pct=0.853912 moved_to_echo=15724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 4010, 8, 0, 1056, 323, 95, 28, 3983, 0, 0, 0, 36, 0, 0, 0, 1115, 338, 0, 0, 0, 0, 0, 0, 0, 2981, 3746, 2, 0, 0, 0, 0, 4, 1, 11, 0, 4081, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 1, 177, 37, 3989, 0, 0, 0, 0, 0, 0, 11, 0, 2694, 11, 0, 0] rerouted_tokens_per_ep_rank=[5525, 4019, 1453, 6729, 4097, 4025, 4204, 2716] avg=4096.0000 max=6729 max_rank=3 min=1453 min_rank=2 imbalance_ratio=1.642822 max_load_reduction=26039 max_load_reduction_pct=0.794647 moved_to_echo=15042 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 70, 0, 0, 1035, 325, 95, 28, 3919, 8, 0, 0, 36, 0, 0, 0, 4071, 331, 0, 0, 0, 0, 0, 0, 0, 111, 3771, 0, 0, 0, 0, 0, 0, 0, 12, 3, 4084, 0, 0, 0, 0, 189, 4025, 0, 0, 0, 0, 0, 0, 186, 194, 3987, 0, 0, 0, 0, 3618, 1, 2, 11, 2642, 11, 0, 0] rerouted_tokens_per_ep_rank=[1556, 3963, 4402, 3882, 4099, 4214, 4367, 6285] avg=4096.0000 max=6285 max_rank=7 min=1556 min_rank=0 imbalance_ratio=1.534424 max_load_reduction=26483 max_load_reduction_pct=0.808197 moved_to_echo=18965 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1681, 0, 0, 1057, 311, 95, 28, 4093, 0, 0, 0, 37, 0, 0, 0, 4095, 339, 135, 0, 0, 0, 0, 0, 0, 38, 3615, 0, 0, 0, 0, 0, 22, 0, 0, 0, 4073, 0, 0, 0, 5, 0, 4026, 3, 0, 0, 0, 0, 0, 1, 0, 3999, 0, 0, 0, 0, 2404, 0, 2, 0, 2698, 11, 0, 0] rerouted_tokens_per_ep_rank=[3172, 4130, 4569, 3653, 4095, 4034, 4000, 5115] avg=4096.0000 max=5115 max_rank=7 min=3172 min_rank=0 imbalance_ratio=1.248779 max_load_reduction=27653 max_load_reduction_pct=0.843903 moved_to_echo=17244 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3728, 1, 0, 1058, 318, 95, 28, 4093, 0, 0, 0, 36, 0, 0, 0, 4085, 338, 0, 0, 0, 0, 0, 0, 0, 120, 3698, 0, 0, 0, 0, 0, 1, 0, 6, 0, 4084, 0, 0, 0, 0, 228, 4025, 111, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 3, 0, 0, 0, 2700, 11, 0, 0] rerouted_tokens_per_ep_rank=[5228, 4129, 4423, 3818, 4091, 4364, 4001, 2714] avg=4096.0000 max=5228 max_rank=0 min=2714 min_rank=7 imbalance_ratio=1.276367 max_load_reduction=27540 max_load_reduction_pct=0.840454 moved_to_echo=15170 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2718, 0, 0, 1057, 319, 95, 28, 4093, 0, 0, 0, 37, 0, 0, 0, 4096, 339, 0, 1, 0, 0, 0, 0, 0, 0, 3709, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4085, 0, 0, 0, 0, 2, 4028, 0, 0, 0, 0, 0, 3, 0, 4, 4000, 0, 0, 0, 0, 1438, 0, 0, 0, 2698, 11, 0, 0] rerouted_tokens_per_ep_rank=[4217, 4130, 4436, 3709, 4092, 4030, 4007, 4147] avg=4096.0000 max=4436 max_rank=2 min=3709 min_rank=3 imbalance_ratio=1.083008 max_load_reduction=28332 max_load_reduction_pct=0.864624 moved_to_echo=16276 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1368, 4, 20, 1057, 323, 100, 31, 4076, 0, 0, 23, 41, 0, 0, 0, 4092, 326, 1, 0, 0, 0, 0, 0, 0, 0, 3703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 18, 0, 0, 3978, 0, 0, 0, 0, 2728, 0, 0, 70, 2689, 10, 0, 0] rerouted_tokens_per_ep_rank=[2903, 4140, 4419, 3703, 4086, 4024, 3996, 5497] avg=4096.0000 max=5497 max_rank=7 min=2903 min_rank=0 imbalance_ratio=1.342041 max_load_reduction=27271 max_load_reduction_pct=0.832245 moved_to_echo=17603 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3876, 0, 0, 1056, 325, 95, 28, 4096, 0, 0, 0, 37, 0, 0, 0, 4096, 338, 0, 0, 0, 0, 0, 0, 0, 2, 3771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 14, 4031, 0, 0, 0, 0, 0, 0, 0, 212, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 10, 0, 0] rerouted_tokens_per_ep_rank=[5380, 4133, 4434, 3773, 4086, 4045, 4213, 2704] avg=4096.0000 max=5380 max_rank=0 min=2704 min_rank=7 imbalance_ratio=1.313477 max_load_reduction=27388 max_load_reduction_pct=0.835815 moved_to_echo=15048 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 20736, 128, 128, 8448, 2560, 768, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11136, 128, 128, 128, 21632, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[29824, 2688, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 3328, 29568, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 384, 512, 32000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 128, 0, 128, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 512, 32256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[15, 3546, 1580, 67, 2906, 2, 948, 43, 1710, 53, 3375, 912, 907, 392, 46, 292, 595, 889, 1676, 3694, 3156, 843, 586, 205, 2, 1765, 198, 512, 440, 76, 1297, 40] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[16, 3441, 1807, 45, 2989, 1, 1320, 19, 1746, 115, 3378, 682, 311, 551, 66, 183, 355, 1539, 1208, 3547, 3384, 628, 674, 161, 9, 1930, 173, 709, 240, 64, 1341, 136] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[16, 3578, 2027, 16, 2505, 6, 895, 16, 1461, 56, 3336, 666, 407, 646, 51, 208, 278, 1250, 1610, 3753, 3364, 390, 788, 170, 1, 2202, 195, 914, 527, 157, 1127, 152] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[19, 3261, 2066, 58, 2919, 16, 907, 63, 1135, 125, 3140, 1051, 539, 480, 109, 265, 495, 1357, 1637, 3732, 3142, 468, 629, 142, 14, 1658, 356, 770, 539, 221, 1034, 421] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[9, 3175, 1682, 11, 2742, 7, 1037, 36, 1073, 129, 3224, 1140, 569, 426, 143, 246, 591, 1033, 1629, 3850, 3299, 775, 569, 180, 0, 1726, 240, 650, 775, 113, 1535, 154] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[25, 3257, 1823, 30, 2648, 30, 1074, 40, 1109, 167, 3279, 926, 406, 502, 158, 227, 408, 1216, 1871, 3653, 3165, 553, 586, 290, 3, 1371, 209, 772, 885, 159, 1339, 587] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[12, 3120, 1712, 25, 2564, 3, 1030, 27, 2077, 132, 3528, 670, 480, 762, 52, 221, 387, 750, 2075, 3419, 3287, 888, 512, 303, 0, 1465, 211, 720, 560, 99, 1638, 39] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[10, 3393, 1691, 105, 2977, 6, 1184, 30, 1723, 34, 3142, 635, 454, 465, 165, 241, 583, 1323, 1477, 3601, 3359, 675, 636, 379, 2, 1741, 253, 618, 187, 74, 1399, 206] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[15, 2372, 1580, 67, 0, 0, 0, 0, 2906, 2, 948, 43, 214, 0, 0, 0, 1710, 53, 1693, 912, 0, 0, 0, 0, 907, 392, 46, 292, 2752, 0, 0, 0, 595, 889, 1676, 827, 0, 0, 0, 0, 2503, 843, 586, 205, 0, 0, 0, 0, 2, 1765, 198, 512, 1025, 439, 0, 0, 440, 76, 1297, 40, 1682, 149, 115, 0] rerouted_tokens_per_ep_rank=[4034, 4113, 4368, 4389, 3987, 4137, 3941, 3799] avg=4096.0000 max=4389 max_rank=3 min=3799 min_rank=7 imbalance_ratio=1.071533 max_load_reduction=28379 max_load_reduction_pct=0.866058 moved_to_echo=15864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 2302, 1807, 45, 0, 0, 0, 0, 2989, 1, 1320, 19, 230, 0, 0, 0, 1746, 115, 1695, 682, 0, 0, 0, 0, 311, 551, 66, 183, 2643, 0, 0, 0, 355, 1539, 1208, 794, 0, 0, 0, 0, 2684, 628, 674, 161, 0, 0, 0, 0, 9, 1930, 173, 709, 994, 470, 0, 0, 240, 64, 1341, 136, 1683, 145, 110, 0] rerouted_tokens_per_ep_rank=[4170, 4559, 4238, 3754, 3896, 4147, 4285, 3719] avg=4096.0000 max=4559 max_rank=1 min=3719 min_rank=7 imbalance_ratio=1.113037 max_load_reduction=28209 max_load_reduction_pct=0.860870 moved_to_echo=16047 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[9, 2124, 1682, 11, 0, 0, 0, 0, 2742, 7, 1037, 36, 224, 0, 0, 0, 1073, 129, 1617, 1140, 0, 0, 0, 0, 569, 426, 143, 246, 2869, 0, 0, 0, 591, 1033, 1629, 861, 0, 0, 0, 0, 2616, 775, 569, 180, 0, 0, 0, 0, 0, 1726, 240, 650, 917, 459, 0, 0, 775, 113, 1535, 154, 1607, 134, 120, 0] rerouted_tokens_per_ep_rank=[3826, 4046, 3959, 4253, 4114, 4140, 3992, 4438] avg=4096.0000 max=4438 max_rank=7 min=3826 min_rank=0 imbalance_ratio=1.083496 max_load_reduction=28330 max_load_reduction_pct=0.864563 moved_to_echo=16684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 2393, 2027, 16, 0, 0, 0, 0, 2505, 6, 895, 16, 228, 0, 0, 0, 1461, 56, 1674, 666, 0, 0, 0, 0, 407, 646, 51, 208, 2796, 0, 0, 0, 278, 1250, 1610, 840, 0, 0, 0, 0, 2668, 390, 788, 170, 0, 0, 0, 0, 1, 2202, 195, 914, 1034, 468, 0, 0, 527, 157, 1127, 152, 1662, 151, 117, 0] rerouted_tokens_per_ep_rank=[4452, 3650, 3857, 4108, 3978, 4016, 4814, 3893] avg=4096.0000 max=4814 max_rank=6 min=3650 min_rank=1 imbalance_ratio=1.175293 max_load_reduction=27954 max_load_reduction_pct=0.853088 moved_to_echo=16701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[12, 2088, 1712, 25, 0, 0, 0, 0, 2564, 3, 1030, 27, 223, 0, 0, 0, 2077, 132, 1770, 670, 0, 0, 0, 0, 480, 762, 52, 221, 2548, 0, 0, 0, 387, 750, 2075, 765, 0, 0, 0, 0, 2607, 888, 512, 303, 0, 0, 0, 0, 0, 1465, 211, 720, 901, 457, 0, 0, 560, 99, 1638, 39, 1758, 131, 106, 0] rerouted_tokens_per_ep_rank=[3837, 3847, 4649, 4063, 3977, 4310, 3754, 4331] avg=4096.0000 max=4649 max_rank=2 min=3754 min_rank=6 imbalance_ratio=1.135010 max_load_reduction=28119 max_load_reduction_pct=0.858124 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[25, 2170, 1823, 30, 0, 0, 0, 0, 2648, 30, 1074, 40, 218, 0, 0, 0, 1109, 167, 1641, 926, 0, 0, 0, 0, 406, 502, 158, 227, 2726, 0, 0, 0, 408, 1216, 1871, 808, 0, 0, 0, 0, 2504, 553, 586, 290, 0, 0, 0, 0, 3, 1371, 209, 772, 946, 443, 0, 0, 885, 159, 1339, 587, 1638, 141, 119, 0] rerouted_tokens_per_ep_rank=[4048, 4010, 3843, 4019, 4303, 3933, 3744, 4868] avg=4096.0000 max=4868 max_rank=7 min=3744 min_rank=6 imbalance_ratio=1.188477 max_load_reduction=27900 max_load_reduction_pct=0.851440 moved_to_echo=16848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 2270, 1691, 105, 0, 0, 0, 0, 2977, 6, 1184, 30, 228, 0, 0, 0, 1723, 34, 1576, 635, 0, 0, 0, 0, 454, 465, 165, 241, 2683, 0, 0, 0, 583, 1323, 1477, 806, 0, 0, 0, 0, 2664, 675, 636, 379, 0, 0, 0, 0, 2, 1741, 253, 618, 980, 467, 0, 0, 187, 74, 1399, 206, 1566, 143, 112, 0] rerouted_tokens_per_ep_rank=[4076, 4425, 3968, 4008, 4189, 4354, 4061, 3687] avg=4096.0000 max=4425 max_rank=1 min=3687 min_rank=7 imbalance_ratio=1.080322 max_load_reduction=28343 max_load_reduction_pct=0.864960 moved_to_echo=16291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[19, 2182, 2066, 58, 0, 0, 0, 0, 2919, 16, 907, 63, 213, 0, 0, 0, 1135, 125, 1575, 1051, 0, 0, 0, 0, 539, 480, 109, 265, 2781, 0, 0, 0, 495, 1357, 1637, 835, 0, 0, 0, 0, 2492, 468, 629, 142, 0, 0, 0, 0, 14, 1658, 356, 770, 942, 437, 0, 0, 539, 221, 1034, 421, 1565, 137, 116, 0] rerouted_tokens_per_ep_rank=[4325, 4118, 3886, 4174, 4324, 3731, 4177, 4033] avg=4096.0000 max=4325 max_rank=0 min=3731 min_rank=5 imbalance_ratio=1.055908 max_load_reduction=28443 max_load_reduction_pct=0.868011 moved_to_echo=16265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 13952, 1920, 5760, 7808, 3712, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 1024, 10752, 1792, 13184, 1152, 1024, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[3712, 9472, 13184, 6656, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 17920, 14464, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[22272, 128, 8448, 384, 1792, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12160, 896, 13312, 6784, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4224, 896, 1920, 21888, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[20864, 5248, 4992, 1920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[6, 4080, 650, 52, 0, 0, 39, 0, 1, 3892, 0, 0, 0, 0, 3755, 0, 0, 0, 0, 0, 4054, 0, 50, 31, 4096, 3988, 4096, 0, 7, 0, 1, 3970] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1, 4069, 1709, 4, 0, 0, 0, 179, 11, 4077, 1, 2, 1, 0, 4030, 0, 17, 0, 0, 0, 4012, 3, 30, 17, 4096, 2370, 4092, 0, 87, 0, 0, 3960] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[7, 4093, 1431, 332, 0, 0, 189, 36, 0, 3765, 18, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 4062, 0, 1, 74, 4027, 3921, 4096, 0, 3, 0, 40, 2652] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[7, 4092, 537, 80, 0, 0, 87, 9, 0, 3916, 3, 2, 1, 0, 4012, 0, 0, 0, 2, 0, 4036, 2, 2, 35, 4077, 4060, 4093, 3, 117, 1, 31, 3563] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 4096, 694, 13, 3, 0, 0, 210, 2, 3756, 0, 3, 0, 0, 3829, 1, 0, 0, 0, 0, 4092, 1, 29, 7, 4085, 3886, 4096, 0, 45, 1, 0, 3919] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 4094, 1271, 53, 0, 0, 2, 21, 0, 4090, 71, 0, 0, 0, 3720, 0, 0, 0, 0, 0, 3963, 0, 0, 2, 4089, 3187, 4096, 2, 4, 0, 29, 4074] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2, 4054, 2504, 31, 0, 1, 15, 9, 0, 3842, 89, 1, 1, 0, 3562, 0, 0, 0, 0, 0, 4086, 24, 10, 0, 4055, 3604, 4089, 0, 909, 19, 40, 1821] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 4061, 1679, 35, 0, 4, 38, 1, 1, 3451, 0, 0, 6, 8, 4043, 0, 0, 7, 1, 1, 3902, 0, 63, 5, 4022, 2942, 4091, 21, 527, 2, 2, 3855] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 2704, 1709, 4, 0, 0, 0, 0, 0, 0, 0, 179, 3493, 517, 1, 0, 11, 4077, 1, 2, 221, 0, 0, 0, 1, 0, 4030, 0, 221, 0, 0, 0, 17, 0, 0, 0, 4091, 0, 0, 0, 4012, 3, 30, 17, 21, 0, 0, 0, 603, 2370, 0, 0, 0, 0, 0, 0, 87, 0, 0, 3960, 385, 0, 0, 0] rerouted_tokens_per_ep_rank=[4418, 4190, 4312, 4252, 4108, 4083, 2973, 4432] avg=4096.0000 max=4432 max_rank=7 min=2973 min_rank=6 imbalance_ratio=1.082031 max_load_reduction=28336 max_load_reduction_pct=0.864746 moved_to_echo=15596 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[7, 2720, 537, 80, 0, 0, 0, 0, 0, 0, 87, 9, 3477, 520, 1, 0, 0, 3916, 3, 2, 222, 0, 0, 0, 1, 0, 4012, 0, 222, 0, 0, 0, 0, 0, 2, 0, 4092, 0, 0, 0, 4036, 2, 2, 35, 21, 0, 0, 0, 600, 4060, 0, 3, 0, 0, 0, 0, 117, 1, 31, 3563, 387, 0, 0, 0] rerouted_tokens_per_ep_rank=[3344, 4094, 4143, 4235, 4094, 4096, 4663, 4099] avg=4096.0000 max=4663 max_rank=6 min=3344 min_rank=0 imbalance_ratio=1.138428 max_load_reduction=28105 max_load_reduction_pct=0.857697 moved_to_echo=16952 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 2711, 650, 52, 0, 0, 0, 0, 0, 0, 39, 0, 3493, 519, 1, 0, 1, 3892, 0, 0, 221, 0, 0, 0, 0, 0, 3755, 0, 222, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4054, 0, 50, 31, 21, 0, 0, 0, 603, 3988, 0, 0, 0, 0, 0, 0, 7, 0, 1, 3970, 386, 0, 0, 0] rerouted_tokens_per_ep_rank=[3419, 4052, 4114, 3977, 4095, 4156, 4591, 4364] avg=4096.0000 max=4591 max_rank=6 min=3419 min_rank=0 imbalance_ratio=1.120850 max_load_reduction=28177 max_load_reduction_pct=0.859894 moved_to_echo=17206 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[7, 2721, 1431, 332, 0, 0, 0, 0, 0, 0, 189, 36, 3435, 520, 1, 0, 0, 3765, 18, 0, 222, 0, 0, 0, 0, 0, 4021, 0, 222, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4062, 0, 1, 74, 21, 0, 0, 0, 592, 3921, 0, 0, 0, 0, 0, 0, 3, 0, 40, 2652, 387, 0, 0, 0] rerouted_tokens_per_ep_rank=[4491, 4181, 4005, 4243, 4095, 4158, 4513, 3082] avg=4096.0000 max=4513 max_rank=6 min=3082 min_rank=7 imbalance_ratio=1.101807 max_load_reduction=28255 max_load_reduction_pct=0.862274 moved_to_echo=15848 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2721, 1271, 53, 0, 0, 0, 0, 0, 0, 2, 21, 3487, 520, 1, 0, 0, 4090, 71, 0, 222, 0, 0, 0, 0, 0, 3720, 0, 223, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3963, 0, 0, 2, 21, 0, 0, 0, 602, 3187, 0, 2, 0, 0, 0, 0, 4, 0, 29, 4074, 387, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4031, 4383, 3943, 4095, 3986, 3791, 4494] avg=4096.0000 max=4494 max_rank=7 min=3791 min_rank=6 imbalance_ratio=1.097168 max_load_reduction=28274 max_load_reduction_pct=0.862854 moved_to_echo=16366 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2722, 694, 13, 0, 0, 0, 0, 3, 0, 0, 210, 3484, 521, 1, 0, 2, 3756, 0, 3, 222, 0, 0, 0, 0, 0, 3829, 1, 223, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4092, 1, 29, 7, 21, 0, 0, 0, 601, 3886, 0, 0, 0, 0, 0, 0, 45, 1, 0, 3919, 387, 0, 0, 0] rerouted_tokens_per_ep_rank=[3429, 4219, 3983, 4053, 4095, 4150, 4487, 4352] avg=4096.0000 max=4487 max_rank=6 min=3429 min_rank=0 imbalance_ratio=1.095459 max_load_reduction=28281 max_load_reduction_pct=0.863068 moved_to_echo=17084 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2676, 1679, 35, 0, 0, 0, 0, 0, 4, 38, 1, 3436, 521, 2, 0, 1, 3451, 0, 0, 224, 0, 0, 0, 6, 8, 4043, 0, 225, 0, 0, 0, 0, 7, 1, 1, 4089, 0, 0, 0, 3902, 0, 63, 5, 28, 0, 0, 0, 586, 2942, 0, 21, 0, 0, 0, 0, 527, 2, 2, 3855, 387, 0, 0, 0] rerouted_tokens_per_ep_rank=[4390, 4002, 3676, 4282, 4098, 3998, 3549, 4773] avg=4096.0000 max=4773 max_rank=7 min=3549 min_rank=6 imbalance_ratio=1.165283 max_load_reduction=27995 max_load_reduction_pct=0.854340 moved_to_echo=16418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 2695, 2504, 31, 0, 0, 0, 0, 0, 1, 15, 9, 3458, 515, 1, 0, 0, 3842, 89, 1, 220, 0, 0, 0, 1, 0, 3562, 0, 220, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4086, 24, 10, 0, 21, 0, 0, 0, 597, 3604, 0, 0, 0, 0, 0, 0, 909, 19, 40, 1821, 383, 0, 0, 0] rerouted_tokens_per_ep_rank=[5232, 3999, 4152, 3783, 4088, 4141, 4201, 3172] avg=4096.0000 max=5232 max_rank=0 min=3172 min_rank=7 imbalance_ratio=1.277344 max_load_reduction=27536 max_load_reduction_pct=0.840332 moved_to_echo=15602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 21760, 10496, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 384, 512, 27776, 4224, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1792, 128, 256, 27904, 3200, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4864, 28032, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 30976, 128, 1792, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32256, 128, 256, 256, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 30848, 256, 128, 1792, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 3930, 0, 4096, 0, 3207, 0, 1081, 4096, 4096, 4096, 38, 4096, 0, 0, 43, 0, 3, 0, 0, 0, 3776, 188, 0, 8, 0, 13, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4016, 0, 4096, 0, 3774, 0, 25, 4096, 4096, 4096, 0, 4096, 0, 0, 439, 0, 248, 0, 0, 0, 3726, 30, 0, 28, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 45, 0, 0, 0, 4065, 0, 4095, 2, 4026, 0, 12, 4084, 4094, 4081, 14, 4095, 0, 0, 407, 0, 328, 0, 1, 1, 3391, 26, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 4070, 0, 4096, 0, 4011, 0, 70, 4094, 4093, 4093, 0, 4083, 0, 9, 40, 0, 0, 0, 1, 0, 4014, 42, 0, 48, 1, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 3, 0, 0, 4095, 1, 4090, 2, 3619, 1, 34, 4076, 4093, 4096, 0, 4096, 0, 0, 0, 0, 496, 0, 1, 32, 4008, 23, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 3979, 0, 4096, 0, 2435, 0, 10, 4095, 4091, 4094, 0, 4090, 0, 0, 1681, 0, 126, 0, 4, 46, 3992, 22, 0, 6, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[59, 0, 2, 0, 70, 3908, 0, 4074, 0, 3796, 88, 6, 4058, 4059, 4024, 86, 4076, 23, 15, 192, 57, 0, 3, 1, 84, 3654, 373, 2, 11, 0, 47, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 13, 4081, 0, 4093, 0, 3033, 0, 887, 4037, 4096, 4070, 27, 4084, 0, 20, 530, 0, 128, 1, 1, 0, 3574, 90, 0, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4088, 0, 0, 0, 0, 3930, 0, 69, 0, 0, 0, 0, 0, 3207, 0, 1081, 213, 117, 0, 0, 4092, 0, 0, 38, 0, 0, 0, 0, 3678, 0, 0, 43, 0, 0, 0, 0, 0, 3, 0, 0, 3910, 14, 0, 0, 0, 3776, 188, 0, 205, 4, 0, 0, 8, 0, 13, 0, 4082, 8, 0, 0] rerouted_tokens_per_ep_rank=[4089, 3999, 4618, 4130, 3721, 3927, 4173, 4111] avg=4096.0000 max=4618 max_rank=2 min=3721 min_rank=4 imbalance_ratio=1.127441 max_load_reduction=28150 max_load_reduction_pct=0.859070 moved_to_echo=15932 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4084, 0, 0, 0, 0, 4070, 0, 69, 0, 0, 0, 0, 0, 4011, 0, 70, 212, 117, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 3667, 0, 9, 40, 0, 0, 0, 0, 0, 0, 0, 1, 3910, 14, 0, 0, 0, 4014, 42, 0, 204, 4, 0, 0, 48, 1, 0, 1, 4079, 9, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4139, 4410, 4090, 3716, 3925, 4264, 4138] avg=4096.0000 max=4410 max_rank=2 min=3716 min_rank=4 imbalance_ratio=1.076660 max_load_reduction=28358 max_load_reduction_pct=0.865417 moved_to_echo=16043 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 45, 0, 0, 4085, 0, 0, 0, 0, 4065, 0, 69, 0, 0, 0, 0, 2, 4026, 0, 12, 213, 117, 0, 0, 4080, 0, 0, 14, 0, 0, 0, 0, 3677, 0, 0, 407, 0, 0, 0, 0, 0, 328, 0, 1, 3909, 14, 0, 0, 1, 3391, 26, 0, 205, 4, 0, 0, 0, 0, 0, 0, 4067, 9, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4134, 4370, 4094, 4084, 4252, 3627, 4076] avg=4096.0000 max=4370 max_rank=2 min=3627 min_rank=6 imbalance_ratio=1.066895 max_load_reduction=28398 max_load_reduction_pct=0.866638 moved_to_echo=16039 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3, 0, 4085, 0, 0, 0, 0, 4095, 1, 69, 0, 0, 0, 0, 2, 3619, 1, 34, 213, 117, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 1, 3904, 15, 0, 0, 32, 4008, 23, 0, 205, 4, 0, 0, 1, 0, 0, 0, 4081, 8, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4165, 3986, 4072, 3678, 4416, 4272, 4090] avg=4096.0000 max=4416 max_rank=5 min=3678 min_rank=4 imbalance_ratio=1.078125 max_load_reduction=28352 max_load_reduction_pct=0.865234 moved_to_echo=16456 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 0, 0, 0, 0, 4016, 0, 69, 0, 0, 0, 0, 0, 3774, 0, 25, 213, 117, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 3678, 0, 0, 439, 0, 0, 0, 0, 0, 248, 0, 0, 3910, 14, 0, 0, 0, 3726, 30, 0, 205, 4, 0, 0, 28, 0, 2, 0, 4082, 9, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4085, 4129, 4092, 4117, 4172, 3965, 4121] avg=4096.0000 max=4172 max_rank=5 min=3965 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4083, 0, 0, 0, 0, 3979, 0, 69, 0, 0, 0, 0, 0, 2435, 0, 10, 213, 117, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 3672, 0, 0, 1681, 0, 0, 0, 0, 0, 126, 0, 4, 3910, 15, 0, 0, 46, 3992, 22, 0, 205, 4, 0, 0, 6, 0, 0, 0, 4079, 8, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4048, 2775, 4091, 5353, 4055, 4269, 4093] avg=4096.0000 max=5353 max_rank=4 min=2775 min_rank=2 imbalance_ratio=1.306885 max_load_reduction=27415 max_load_reduction_pct=0.836639 moved_to_echo=17770 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[59, 0, 2, 0, 4051, 0, 0, 0, 70, 3908, 0, 57, 0, 0, 0, 0, 0, 3796, 88, 6, 217, 124, 0, 0, 4053, 0, 0, 86, 0, 0, 0, 0, 3651, 23, 15, 192, 0, 0, 0, 0, 57, 0, 3, 1, 3893, 15, 0, 0, 84, 3654, 373, 2, 208, 5, 0, 0, 11, 0, 47, 0, 4009, 8, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4035, 4231, 4139, 3881, 3969, 4326, 4075] avg=4096.0000 max=4326 max_rank=6 min=3881 min_rank=4 imbalance_ratio=1.056152 max_load_reduction=28442 max_load_reduction_pct=0.867981 moved_to_echo=16251 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4088, 0, 0, 0, 13, 4081, 0, 69, 0, 0, 0, 0, 0, 3033, 0, 887, 212, 117, 0, 0, 4033, 0, 0, 27, 0, 0, 0, 0, 3668, 0, 20, 530, 0, 0, 0, 0, 0, 128, 1, 1, 3907, 15, 0, 0, 0, 3574, 90, 0, 204, 4, 0, 0, 1, 0, 0, 0, 4055, 8, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4163, 4249, 4060, 4218, 4052, 3872, 4064] avg=4096.0000 max=4249 max_rank=2 min=3872 min_rank=6 imbalance_ratio=1.037354 max_load_reduction=28519 max_load_reduction_pct=0.870331 moved_to_echo=16206 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[29440, 128, 128, 3456, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1408, 128, 128, 31360, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 30208, 896, 128, 1664, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32640, 0, 0, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 27904, 128, 2176, 1792, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32256, 128, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 4096, 2988, 0, 0, 1090, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 15, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 3, 20, 4090, 0, 0, 1, 0, 4090, 0, 0, 1, 4081, 0, 4093, 3046, 0, 0, 1046, 0, 0, 4094, 8, 4091, 0, 4, 0, 0, 5, 4092, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[31, 0, 32, 4024, 1, 2, 492, 1, 4079, 9, 34, 0, 4052, 7, 4063, 4083, 1, 0, 199, 6, 0, 4052, 4, 3454, 47, 0, 3, 6, 0, 4083, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[36, 0, 27, 4093, 0, 0, 144, 0, 4095, 15, 6, 1, 4095, 0, 4094, 3788, 0, 1, 6, 0, 0, 4096, 0, 4096, 0, 1, 0, 74, 0, 4096, 3, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[47, 0, 511, 4094, 0, 0, 659, 0, 4096, 0, 76, 0, 4096, 0, 4096, 3199, 0, 0, 6, 0, 20, 4091, 9, 3644, 0, 0, 0, 0, 0, 4096, 28, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 111, 4083, 0, 0, 336, 0, 4096, 0, 0, 0, 4091, 6, 4069, 3488, 15, 4, 195, 0, 1, 4092, 0, 4081, 0, 0, 0, 3, 4, 4091, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 169, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 4096, 3390, 8, 0, 542, 0, 0, 4096, 0, 4062, 0, 0, 0, 0, 0, 4096, 23, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[24, 0, 0, 4087, 0, 0, 4, 0, 4084, 0, 0, 4, 4093, 9, 4096, 4095, 0, 1, 0, 0, 0, 4095, 0, 4095, 0, 1, 0, 1, 4, 4071, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 20, 3977, 0, 0, 0, 0, 0, 0, 1, 0, 3886, 9, 0, 0, 4077, 0, 0, 1, 0, 0, 0, 0, 583, 0, 0, 3046, 0, 0, 0, 0, 0, 0, 1046, 0, 3498, 113, 69, 13, 0, 139, 8, 4091, 0, 0, 0, 0, 0, 4, 0, 0, 4084, 0, 0, 0, 5, 4092, 0, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4000, 3896, 4078, 3629, 4739, 4238, 4088, 4100] avg=4096.0000 max=4739 max_rank=4 min=3629 min_rank=3 imbalance_ratio=1.156982 max_load_reduction=28029 max_load_reduction_pct=0.855377 moved_to_echo=17165 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 3888, 9, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 585, 0, 0, 2988, 0, 0, 0, 0, 0, 0, 1090, 0, 3510, 114, 69, 13, 0, 139, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3986, 3897, 4083, 3573, 4796, 4235, 4087, 4111] avg=4096.0000 max=4796 max_rank=4 min=3573 min_rank=3 imbalance_ratio=1.170898 max_load_reduction=27972 max_load_reduction_pct=0.853638 moved_to_echo=17229 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 111, 3970, 0, 0, 0, 0, 0, 0, 336, 0, 3884, 9, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 585, 6, 0, 3488, 0, 0, 0, 0, 15, 4, 195, 0, 3506, 113, 69, 13, 1, 139, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 3, 4060, 0, 0, 0, 4, 4091, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4229, 4083, 4079, 3915, 4221, 4063, 4097] avg=4096.0000 max=4229 max_rank=1 min=3915 min_rank=4 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[36, 0, 27, 3979, 0, 0, 0, 0, 0, 0, 144, 0, 3888, 9, 0, 0, 4082, 15, 6, 1, 0, 0, 0, 0, 585, 0, 0, 3788, 0, 0, 0, 0, 0, 1, 6, 0, 3510, 114, 69, 13, 0, 139, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 74, 4085, 0, 0, 0, 0, 4096, 3, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4041, 4104, 4373, 3713, 4235, 4160, 4100] avg=4096.0000 max=4373 max_rank=3 min=3713 min_rank=4 imbalance_ratio=1.067627 max_load_reduction=28395 max_load_reduction_pct=0.866547 moved_to_echo=16208 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[47, 0, 511, 3980, 0, 0, 0, 0, 0, 0, 659, 0, 3883, 10, 0, 0, 4083, 0, 76, 0, 0, 0, 0, 0, 585, 0, 0, 3199, 0, 0, 0, 0, 0, 0, 6, 0, 3511, 114, 69, 13, 20, 139, 9, 3644, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 28, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4538, 4552, 4159, 3784, 3713, 3812, 4086, 4124] avg=4096.0000 max=4552 max_rank=1 min=3713 min_rank=4 imbalance_ratio=1.111328 max_load_reduction=28216 max_load_reduction_pct=0.861084 moved_to_echo=15735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[31, 0, 32, 3909, 0, 0, 0, 0, 1, 2, 492, 1, 3849, 10, 0, 0, 4060, 9, 34, 0, 0, 0, 0, 0, 576, 7, 0, 4083, 0, 0, 0, 0, 1, 0, 199, 6, 3476, 115, 69, 19, 0, 134, 4, 3454, 0, 0, 0, 0, 47, 0, 3, 6, 4053, 0, 0, 0, 0, 4083, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3972, 4355, 4103, 4666, 3885, 3592, 4109, 4086] avg=4096.0000 max=4666 max_rank=3 min=3592 min_rank=5 imbalance_ratio=1.139160 max_load_reduction=28102 max_load_reduction_pct=0.857605 moved_to_echo=15672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 169, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 3888, 9, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 585, 0, 0, 3390, 0, 0, 0, 0, 8, 0, 542, 0, 3506, 114, 69, 13, 0, 139, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 23, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 3897, 4083, 3975, 4252, 4201, 4087, 4122] avg=4096.0000 max=4252 max_rank=4 min=3897 min_rank=1 imbalance_ratio=1.038086 max_load_reduction=28516 max_load_reduction_pct=0.870239 moved_to_echo=16662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[24, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 4, 0, 3887, 10, 0, 0, 4071, 0, 0, 4, 0, 0, 0, 0, 585, 9, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 3508, 113, 69, 13, 0, 139, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 4086, 0, 0, 0, 4, 4071, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 3901, 4075, 4689, 3704, 4234, 4088, 4079] avg=4096.0000 max=4689 max_rank=3 min=3704 min_rank=4 imbalance_ratio=1.144775 max_load_reduction=28079 max_load_reduction_pct=0.856903 moved_to_echo=16105 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32717, 32768, 32768, 32819] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32717, 32768, 32768, 32819] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32717, 32768, 32768, 32819] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32717, 32768, 32768, 32819] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32717, 32768, 32768, 32819] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32717, 32768, 32768, 32819] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32717, 32768, 32768, 32819] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32717, 32768, 32768, 32819] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32717 actual_valid_recv=33408 observed_loss=-691 tokens_per_expert=[128, 128, 3200, 128, 28032, 1024, 640, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4736, 128, 0, 28160, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 896, 31872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 1152, 128, 31744, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 1664, 128, 31104, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32819 actual_valid_recv=33152 observed_loss=-333 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4065, 314, 0, 4036, 0, 1402, 0, 0, 4096, 0, 841, 2070, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3656] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 0, 4094, 16, 22, 4085, 0, 21, 0, 2, 4027, 0, 837, 3792, 0, 0, 4095, 0, 0, 0, 4096, 3, 0, 3, 0, 4091, 50, 3524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4060, 21, 0, 4095, 0, 648, 0, 28, 4096, 0, 1094, 2832, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 24, 3581] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 4095, 1304, 0, 4092, 0, 158, 0, 0, 4096, 0, 839, 2373, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 3, 0, 4092, 0, 3517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4087, 591, 0, 4091, 0, 46, 0, 0, 4079, 0, 275, 3267, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 16, 0, 0, 4, 3968, 2230, 0, 4073, 0, 9, 0, 14, 4085, 1, 684, 2705, 0, 0, 4090, 0, 0, 5, 4094, 0, 2, 0, 0, 4094, 3, 2689] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 26, 0, 4069, 3237, 0, 4005, 0, 13, 0, 66, 4086, 15, 246, 3936, 49, 10, 4059, 9, 11, 25, 4053, 0, 1, 6, 33, 4063, 186, 561] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[7, 0, 8, 85, 1, 2, 4095, 1498, 0, 4088, 12, 0, 25, 1, 4096, 0, 939, 4008, 1, 1, 4083, 0, 0, 0, 4091, 0, 7, 2, 0, 4030, 0, 1688] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 2936, 1133, 22, 0, 0, 0, 2961, 16, 0, 0, 0, 0, 22, 3819, 0, 21, 0, 0, 0, 0, 0, 2, 4024, 0, 0, 0, 0, 0, 837, 3792, 0, 0, 244, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 3, 2, 0, 0, 0, 0, 1155, 50, 3524, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 2977, 3862, 4026, 4874, 4095, 4104, 4729] avg=4096.0000 max=4874 max_rank=4 min=2977 min_rank=1 imbalance_ratio=1.189941 max_load_reduction=27894 max_load_reduction_pct=0.851257 moved_to_echo=17802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2940, 1125, 21, 0, 0, 0, 2940, 314, 0, 0, 0, 0, 0, 3774, 0, 1402, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 841, 2070, 0, 0, 241, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 1156, 0, 3656, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 3254, 5176, 4092, 3154, 4096, 4098, 4812] avg=4096.0000 max=5176 max_rank=2 min=3154 min_rank=4 imbalance_ratio=1.263672 max_load_reduction=27592 max_load_reduction_pct=0.842041 moved_to_echo=16160 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 2937, 1133, 22, 0, 0, 0, 2962, 1304, 0, 0, 0, 0, 0, 3825, 0, 158, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 839, 2373, 0, 0, 245, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 3, 2, 0, 0, 0, 0, 1155, 0, 3517, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4266, 3983, 4092, 3459, 4096, 4102, 4672] avg=4096.0000 max=4672 max_rank=7 min=3459 min_rank=4 imbalance_ratio=1.140625 max_load_reduction=28096 max_load_reduction_pct=0.857422 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2939, 1124, 22, 0, 0, 0, 2936, 21, 0, 0, 0, 0, 0, 3828, 0, 648, 0, 0, 0, 0, 0, 28, 4092, 0, 0, 0, 0, 0, 1094, 2832, 2, 0, 245, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 1156, 24, 3581, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 2957, 4476, 4120, 4175, 4096, 4098, 4761] avg=4096.0000 max=4761 max_rank=7 min=2957 min_rank=1 imbalance_ratio=1.162354 max_load_reduction=28007 max_load_reduction_pct=0.854706 moved_to_echo=17130 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 16, 0, 2939, 1098, 22, 0, 0, 4, 2870, 2230, 0, 0, 0, 0, 0, 3807, 0, 9, 0, 0, 0, 0, 0, 14, 4081, 1, 0, 0, 0, 0, 684, 2705, 0, 0, 244, 2, 0, 0, 4090, 0, 0, 5, 0, 0, 0, 0, 4094, 0, 2, 0, 2, 0, 0, 0, 0, 1155, 3, 2689, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 5104, 3816, 4096, 3635, 4095, 4098, 3847] avg=4096.0000 max=5104 max_rank=1 min=3635 min_rank=4 imbalance_ratio=1.246094 max_load_reduction=27664 max_load_reduction_pct=0.844238 moved_to_echo=15675 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2940, 1131, 22, 0, 0, 0, 2956, 591, 0, 0, 0, 0, 0, 3824, 0, 46, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 275, 3267, 0, 0, 245, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 1156, 0, 4044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 3547, 3870, 4076, 3788, 4096, 4098, 5200] avg=4096.0000 max=5200 max_rank=7 min=3547 min_rank=1 imbalance_ratio=1.269531 max_load_reduction=27568 max_load_reduction_pct=0.841309 moved_to_echo=17182 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 2921, 1132, 24, 0, 26, 0, 2937, 3237, 0, 0, 0, 0, 0, 3738, 0, 13, 0, 0, 0, 0, 0, 66, 4067, 15, 0, 0, 0, 0, 246, 3936, 49, 10, 243, 4, 0, 0, 4059, 9, 11, 25, 7, 0, 0, 0, 4053, 0, 1, 6, 8, 0, 0, 0, 33, 1142, 186, 561, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 6200, 3751, 4148, 4488, 4111, 4068, 1922] avg=4096.0000 max=6200 max_rank=1 min=1922 min_rank=7 imbalance_ratio=1.513672 max_load_reduction=26568 max_load_reduction_pct=0.810791 moved_to_echo=14589 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 8, 85, 2893, 1133, 22, 0, 1, 2, 2962, 1498, 0, 0, 0, 0, 0, 3822, 12, 0, 0, 0, 0, 0, 25, 1, 4092, 0, 0, 0, 0, 0, 939, 4008, 1, 1, 244, 2, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 7, 2, 2, 0, 0, 0, 0, 1137, 0, 1688, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4463, 3834, 4118, 5195, 4083, 4102, 2825] avg=4096.0000 max=5195 max_rank=4 min=2825 min_rank=7 imbalance_ratio=1.268311 max_load_reduction=27573 max_load_reduction_pct=0.841461 moved_to_echo=16205 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 23552, 9088, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 9216, 384, 23296, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 23552, 9216, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5760, 25088, 128, 128, 2048, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 30464, 128, 2304, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 13, 4070] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[7, 2, 4095, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 58, 5, 4096, 0, 0, 4096, 0, 26, 4086, 6, 4096, 0, 4094, 0, 90, 3917] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[3, 0, 4096, 0, 4095, 0, 0, 0, 0, 479, 0, 0, 0, 4, 0, 4, 106, 1, 4095, 0, 2, 4096, 0, 16, 4080, 0, 4089, 1, 4096, 0, 0, 3505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 8, 4096, 0, 4089, 0, 4096, 0, 101, 3982] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[14, 0, 4087, 0, 4096, 0, 0, 1, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 4037, 0, 4096, 0, 4093, 0, 49, 4044] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[3, 0, 4075, 7, 4095, 0, 0, 1, 1, 11, 1, 3, 0, 0, 0, 0, 0, 5, 4092, 1, 10, 4094, 0, 166, 4093, 0, 4093, 0, 4096, 0, 431, 3490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4018, 0, 4086, 3, 0, 31, 73, 5, 1, 23, 0, 8, 3, 4, 0, 0, 4062, 0, 19, 4059, 35, 1, 4094, 18, 4070, 10, 4076, 0, 0, 4068] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[7, 67, 4086, 10, 4019, 0, 0, 0, 1, 192, 0, 2, 14, 54, 0, 0, 0, 4, 4093, 0, 1, 4096, 1, 1800, 4027, 0, 4093, 0, 3968, 0, 114, 2119] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 8, 0, 0, 3738, 256, 11, 0, 0, 0, 0, 0, 4080, 8, 0, 0, 0, 4, 4079, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 350, 0, 13, 4070, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4102, 4013, 4088, 4083, 3840, 4112, 4433] avg=4096.0000 max=4433 max_rank=7 min=3840 min_rank=5 imbalance_ratio=1.082275 max_load_reduction=28335 max_load_reduction_pct=0.864716 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 6, 0, 0, 0, 0, 479, 0, 0, 3738, 256, 11, 0, 0, 4, 0, 4, 4073, 8, 0, 0, 106, 1, 4078, 0, 0, 0, 0, 0, 2, 3840, 0, 16, 0, 0, 0, 0, 4080, 0, 16, 1, 0, 0, 0, 0, 350, 0, 0, 3505, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4101, 4484, 4089, 4185, 3858, 4097, 3855] avg=4096.0000 max=4484 max_rank=2 min=3855 min_rank=7 imbalance_ratio=1.094727 max_load_reduction=28284 max_load_reduction_pct=0.863159 moved_to_echo=15995 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4075, 7, 0, 0, 0, 0, 4095, 0, 0, 1, 6, 0, 0, 0, 1, 11, 1, 3, 3738, 256, 11, 0, 0, 0, 0, 0, 4077, 8, 0, 0, 0, 5, 4075, 1, 0, 0, 0, 0, 10, 3838, 0, 166, 0, 0, 0, 0, 4093, 0, 16, 0, 0, 0, 0, 0, 350, 0, 431, 3490, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4102, 4021, 4085, 4081, 4014, 4109, 4271] avg=4096.0000 max=4271 max_rank=7 min=4014 min_rank=5 imbalance_ratio=1.042725 max_load_reduction=28497 max_load_reduction_pct=0.869659 moved_to_echo=16475 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[7, 2, 4095, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 3736, 256, 11, 0, 0, 0, 0, 0, 4080, 8, 0, 0, 58, 5, 4079, 0, 0, 0, 0, 0, 0, 3840, 0, 26, 0, 0, 0, 0, 4086, 6, 16, 0, 0, 0, 0, 0, 350, 0, 90, 3917, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4100, 4003, 4088, 4142, 3866, 4108, 4357] avg=4096.0000 max=4357 max_rank=7 min=3866 min_rank=5 imbalance_ratio=1.063721 max_load_reduction=28411 max_load_reduction_pct=0.867035 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 6, 0, 0, 0, 0, 58, 0, 0, 3735, 256, 11, 0, 0, 0, 0, 0, 4080, 8, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 3840, 0, 1, 0, 0, 0, 0, 4037, 0, 16, 0, 0, 0, 0, 0, 350, 0, 49, 4044, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4103, 4060, 4088, 4079, 3841, 4053, 4443] avg=4096.0000 max=4443 max_rank=7 min=3841 min_rank=5 imbalance_ratio=1.084717 max_load_reduction=28325 max_load_reduction_pct=0.864410 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3738, 256, 11, 0, 8, 0, 0, 0, 4073, 8, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 3840, 0, 8, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 350, 0, 101, 3982, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4102, 4005, 4089, 4079, 3848, 4112, 4433] avg=4096.0000 max=4433 max_rank=7 min=3848 min_rank=5 imbalance_ratio=1.082275 max_load_reduction=28335 max_load_reduction_pct=0.864716 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4011, 0, 0, 0, 0, 0, 4086, 3, 0, 31, 6, 7, 0, 0, 73, 5, 1, 23, 3725, 255, 17, 0, 0, 8, 3, 4, 4058, 15, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 19, 3804, 35, 1, 0, 0, 0, 0, 4094, 18, 12, 10, 0, 0, 0, 0, 336, 0, 0, 4068, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4133, 4099, 4088, 4039, 3859, 4134, 4404] avg=4096.0000 max=4404 max_rank=7 min=3859 min_rank=5 imbalance_ratio=1.075195 max_load_reduction=28364 max_load_reduction_pct=0.865601 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[7, 67, 4086, 10, 0, 0, 0, 0, 4019, 0, 0, 0, 6, 0, 0, 0, 1, 192, 0, 2, 3621, 256, 11, 0, 14, 54, 0, 0, 4077, 8, 0, 0, 0, 4, 4076, 0, 0, 0, 0, 0, 1, 3840, 1, 1800, 0, 0, 0, 0, 4027, 0, 16, 0, 0, 0, 0, 0, 339, 0, 114, 2119, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4025, 4083, 4153, 4080, 5642, 4043, 2572] avg=4096.0000 max=5642 max_rank=5 min=2572 min_rank=7 imbalance_ratio=1.377441 max_load_reduction=27126 max_load_reduction_pct=0.827820 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 768, 128, 128, 29824, 2048, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 30720, 128, 2048, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2816, 0, 896, 29312, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[529, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 86, 1, 793, 4093, 4091, 4092, 0, 0, 2697, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[156, 0, 0, 0, 0, 4096, 0, 18, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2984, 4096, 4096, 1186, 0, 1, 3847, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 1, 0, 4076, 0, 26, 0, 0, 4096, 0, 0, 0, 0, 2774, 4096, 4096, 3924, 0, 2, 1467, 0, 18, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[856, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 2141, 4096, 4096, 3821, 0, 0, 1374, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[3892, 25, 0, 14, 16, 4078, 25, 41, 0, 4033, 0, 3, 2, 34, 4047, 48, 0, 52, 2, 193, 4004, 4058, 1107, 0, 1069, 1902, 12, 0, 30, 0, 4080, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[220, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3104, 4096, 4096, 3997, 0, 2, 867, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1187, 0, 0, 0, 0, 4094, 0, 67, 0, 4095, 0, 8, 0, 0, 4096, 0, 2, 1, 0, 3364, 4096, 4095, 188, 1, 12, 3365, 0, 6, 0, 0, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1795, 0, 0, 6, 0, 4092, 1, 3, 0, 4090, 0, 50, 0, 0, 4094, 5, 0, 0, 6, 2280, 4093, 4021, 145, 0, 35, 3940, 0, 17, 0, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[156, 0, 0, 0, 3020, 0, 0, 0, 0, 4078, 0, 18, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2984, 1076, 803, 0, 0, 0, 1795, 1186, 0, 0, 0, 0, 0, 1, 3847, 0, 0, 1498, 18, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3176, 4096, 4096, 4092, 4863, 2981, 5368, 4096] avg=4096.0000 max=5368 max_rank=6 min=2981 min_rank=5 imbalance_ratio=1.310547 max_load_reduction=27400 max_load_reduction_pct=0.836182 moved_to_echo=17308 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[856, 0, 0, 0, 3020, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 2, 0, 2141, 1076, 803, 0, 0, 0, 1795, 3821, 0, 0, 0, 0, 0, 0, 1374, 0, 0, 1498, 18, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3876, 4078, 4097, 4089, 4022, 5616, 2894, 4096] avg=4096.0000 max=5616 max_rank=5 min=2894 min_rank=6 imbalance_ratio=1.371094 max_load_reduction=27152 max_load_reduction_pct=0.828613 moved_to_echo=16628 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[529, 0, 0, 0, 3017, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 86, 1, 793, 1076, 802, 0, 0, 0, 1793, 4092, 0, 0, 0, 0, 0, 0, 2697, 0, 0, 1496, 18, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3546, 4078, 4096, 4094, 2758, 5885, 4215, 4096] avg=4096.0000 max=5885 max_rank=5 min=2758 min_rank=4 imbalance_ratio=1.436768 max_load_reduction=26883 max_load_reduction_pct=0.820404 moved_to_echo=16954 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1187, 0, 0, 0, 3020, 0, 0, 0, 0, 4076, 0, 67, 0, 0, 0, 0, 0, 4095, 0, 8, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 2, 1, 0, 3364, 1076, 802, 0, 0, 0, 1796, 188, 1, 0, 0, 0, 0, 12, 3365, 0, 6, 1497, 18, 4, 0, 0, 0, 4091, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4207, 4143, 4103, 4092, 5245, 1985, 4902, 4091] avg=4096.0000 max=5245 max_rank=4 min=1985 min_rank=5 imbalance_ratio=1.280518 max_load_reduction=27523 max_load_reduction_pct=0.839935 moved_to_echo=16223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3020, 0, 0, 0, 0, 4078, 0, 1, 0, 0, 0, 0, 0, 4076, 0, 26, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2774, 1076, 803, 0, 0, 0, 1795, 3924, 0, 0, 0, 0, 0, 2, 1467, 0, 18, 1498, 18, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3020, 4079, 4102, 4092, 4653, 5719, 3007, 4096] avg=4096.0000 max=5719 max_rank=5 min=3007 min_rank=6 imbalance_ratio=1.396240 max_load_reduction=27049 max_load_reduction_pct=0.825470 moved_to_echo=17475 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[220, 0, 0, 0, 3020, 0, 0, 0, 0, 4078, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3104, 1076, 803, 0, 0, 0, 1795, 3997, 0, 0, 0, 0, 0, 2, 867, 0, 0, 1498, 18, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3240, 4080, 4096, 4092, 4983, 5792, 2389, 4096] avg=4096.0000 max=5792 max_rank=5 min=2389 min_rank=6 imbalance_ratio=1.414062 max_load_reduction=26976 max_load_reduction_pct=0.823242 moved_to_echo=17260 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3892, 25, 0, 14, 2953, 0, 0, 0, 16, 4054, 25, 41, 0, 0, 0, 0, 0, 4033, 0, 3, 2, 0, 0, 0, 2, 34, 4038, 48, 0, 0, 0, 0, 0, 52, 2, 193, 1051, 797, 0, 0, 0, 1775, 1107, 0, 0, 0, 0, 0, 1069, 1902, 12, 0, 1486, 24, 9, 7, 30, 0, 4071, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6884, 4136, 4038, 4122, 2095, 2882, 4509, 4102] avg=4096.0000 max=6884 max_rank=0 min=2095 min_rank=4 imbalance_ratio=1.680664 max_load_reduction=25884 max_load_reduction_pct=0.789917 moved_to_echo=13588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1795, 0, 0, 6, 3018, 0, 0, 0, 0, 4074, 1, 3, 0, 0, 0, 0, 0, 4090, 0, 50, 0, 0, 0, 0, 0, 0, 4090, 5, 0, 0, 0, 0, 0, 0, 6, 2280, 1075, 788, 0, 0, 0, 1763, 145, 0, 0, 0, 0, 0, 35, 3940, 0, 17, 1470, 18, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4819, 4078, 4140, 4095, 4149, 1908, 5484, 4095] avg=4096.0000 max=5484 max_rank=6 min=1908 min_rank=5 imbalance_ratio=1.338867 max_load_reduction=27284 max_load_reduction_pct=0.832642 moved_to_echo=15636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 14336, 18560, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8704, 128, 0, 128, 24192, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 128, 17664, 8704, 6528, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[1152, 19584, 128, 128, 12032, 256, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 4095, 3148, 0, 1072, 0, 0, 0, 5, 0, 4096, 0, 4094, 0, 0, 1, 4096, 0, 0, 3591, 0, 231, 0, 0, 0, 0, 4096, 4096, 0, 31, 0, 116] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 3256, 0, 1098, 0, 0, 0, 74, 0, 4096, 3, 4096, 5, 1, 10, 4096, 0, 0, 3810, 0, 2, 0, 0, 0, 4, 4096, 4001, 0, 0, 0, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 4066, 2043, 0, 2296, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 4094, 0, 0, 431, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 31, 0, 3421] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 4071, 0, 294, 0, 0, 0, 112, 0, 4096, 0, 4096, 1, 0, 0, 4096, 0, 0, 1658, 0, 0, 0, 0, 0, 0, 4096, 3560, 0, 0, 0, 2592] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 3991, 4096, 9, 981, 44, 7, 0, 393, 31, 4051, 0, 4088, 0, 0, 28, 4068, 0, 1, 421, 21, 385, 4, 0, 6, 0, 4039, 4029, 27, 11, 53, 1984] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 3642, 0, 106, 0, 0, 0, 193, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 2503, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 57, 0, 1698] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 4093, 3888, 0, 199, 0, 1, 0, 0, 0, 4096, 11, 4089, 34, 10, 5, 4095, 0, 0, 3949, 0, 18, 0, 1, 0, 0, 4096, 4079, 0, 13, 0, 91] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 4057, 3808, 8, 0, 0, 20, 0, 171, 3, 4065, 16, 4093, 314, 0, 0, 4094, 3, 0, 1493, 0, 145, 0, 5, 0, 0, 4096, 4079, 1, 48, 0, 2249] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 604, 3148, 0, 0, 0, 0, 0, 1072, 0, 0, 0, 3345, 3, 0, 0, 5, 0, 3979, 0, 0, 0, 0, 0, 4046, 0, 0, 1, 0, 0, 0, 0, 1866, 0, 0, 3591, 0, 0, 0, 0, 0, 231, 0, 0, 4001, 0, 0, 0, 0, 0, 92, 4096, 0, 0, 0, 0, 0, 31, 0, 116, 2230, 146, 117, 48] rerouted_tokens_per_ep_rank=[3752, 4420, 3984, 4047, 5457, 4232, 4188, 2688] avg=4096.0000 max=5457 max_rank=4 min=2688 min_rank=7 imbalance_ratio=1.332275 max_load_reduction=27311 max_load_reduction_pct=0.833466 moved_to_echo=16565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 604, 4071, 0, 0, 0, 0, 0, 294, 0, 0, 0, 3346, 3, 0, 0, 112, 0, 3979, 0, 0, 0, 0, 0, 4048, 1, 0, 0, 0, 0, 0, 0, 1866, 0, 0, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 92, 3560, 0, 0, 0, 0, 0, 0, 0, 2592, 2230, 146, 117, 48] rerouted_tokens_per_ep_rank=[4675, 3643, 4091, 4049, 3524, 4001, 3652, 5133] avg=4096.0000 max=5133 max_rank=7 min=3524 min_rank=4 imbalance_ratio=1.253174 max_load_reduction=27635 max_load_reduction_pct=0.843353 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 600, 2043, 0, 0, 0, 0, 0, 2296, 0, 0, 0, 3321, 3, 0, 0, 0, 0, 3979, 2, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 1865, 0, 0, 431, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 92, 4096, 0, 0, 0, 0, 0, 31, 0, 3421, 2229, 145, 117, 48] rerouted_tokens_per_ep_rank=[2643, 5620, 3981, 4048, 2296, 4001, 4188, 5991] avg=4096.0000 max=5991 max_rank=7 min=2296 min_rank=4 imbalance_ratio=1.462646 max_load_reduction=26777 max_load_reduction_pct=0.817169 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 604, 3256, 0, 0, 0, 0, 0, 1098, 0, 0, 0, 3346, 3, 0, 0, 74, 0, 3979, 3, 0, 0, 0, 0, 4048, 5, 1, 10, 0, 0, 0, 0, 1866, 0, 0, 3810, 0, 0, 0, 0, 0, 2, 0, 0, 4001, 0, 0, 0, 0, 4, 92, 4001, 0, 0, 0, 0, 0, 0, 0, 24, 2230, 146, 117, 48] rerouted_tokens_per_ep_rank=[3860, 4447, 4056, 4064, 5676, 4003, 4097, 2565] avg=4096.0000 max=5676 max_rank=4 min=2565 min_rank=7 imbalance_ratio=1.385742 max_load_reduction=27092 max_load_reduction_pct=0.826782 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 604, 3888, 0, 0, 0, 0, 0, 199, 0, 1, 0, 3343, 3, 0, 0, 0, 0, 3979, 11, 0, 0, 0, 0, 4041, 34, 10, 5, 0, 0, 0, 0, 1865, 0, 0, 3949, 0, 0, 0, 0, 0, 18, 0, 1, 4001, 0, 0, 0, 0, 0, 92, 4079, 0, 0, 0, 0, 0, 13, 0, 91, 2230, 146, 117, 48] rerouted_tokens_per_ep_rank=[4492, 3546, 3990, 4090, 5814, 4020, 4171, 2645] avg=4096.0000 max=5814 max_rank=4 min=2645 min_rank=7 imbalance_ratio=1.419434 max_load_reduction=26954 max_load_reduction_pct=0.822571 moved_to_echo=16650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 604, 3642, 0, 0, 0, 0, 0, 106, 0, 0, 0, 3346, 3, 0, 0, 193, 0, 3979, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 1866, 0, 0, 2503, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 92, 4089, 0, 0, 0, 0, 0, 57, 0, 1698, 2230, 146, 117, 48] rerouted_tokens_per_ep_rank=[4246, 3455, 4172, 4048, 4369, 4001, 4181, 4296] avg=4096.0000 max=4369 max_rank=4 min=3455 min_rank=1 imbalance_ratio=1.066650 max_load_reduction=28399 max_load_reduction_pct=0.866669 moved_to_echo=16847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 581, 4096, 9, 0, 0, 0, 0, 981, 44, 7, 0, 3263, 5, 0, 0, 393, 31, 3931, 0, 0, 0, 0, 0, 4035, 0, 0, 28, 0, 0, 0, 0, 1849, 0, 1, 421, 0, 0, 0, 0, 21, 385, 4, 0, 3949, 0, 0, 0, 6, 0, 85, 4029, 0, 0, 0, 0, 27, 11, 53, 1984, 2219, 147, 120, 53] rerouted_tokens_per_ep_rank=[4686, 4300, 4355, 4063, 2271, 4359, 4120, 4614] avg=4096.0000 max=4686 max_rank=0 min=2271 min_rank=4 imbalance_ratio=1.144043 max_load_reduction=28082 max_load_reduction_pct=0.856995 moved_to_echo=15364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 598, 3808, 8, 0, 0, 0, 0, 0, 0, 20, 0, 3314, 3, 0, 0, 171, 3, 3949, 16, 0, 0, 0, 0, 4045, 314, 0, 0, 0, 0, 0, 0, 1865, 3, 0, 1493, 0, 0, 0, 0, 0, 145, 0, 5, 4001, 0, 0, 0, 0, 0, 92, 4079, 0, 0, 0, 0, 1, 48, 0, 2249, 2229, 145, 116, 48] rerouted_tokens_per_ep_rank=[4414, 3337, 4139, 4359, 3361, 4151, 4171, 4836] avg=4096.0000 max=4836 max_rank=7 min=3337 min_rank=1 imbalance_ratio=1.180664 max_load_reduction=27932 max_load_reduction_pct=0.852417 moved_to_echo=16519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14976, 128, 128, 17920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 256, 128, 12288, 17920, 1280, 1024, 512] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6144, 128, 128, 0, 26624, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 4864, 28032, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 896, 128, 128, 32000, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32384, 384, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 768, 32128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 128, 31872, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4092, 3, 0, 1, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 4090, 0, 4082, 0, 0, 0, 4096, 4096, 0, 0, 0, 541, 5, 0, 3554, 12, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 4088, 0, 4048, 0, 0, 0, 4096, 4094, 0, 0, 0, 3561, 494, 1, 99, 0, 5, 0, 0, 0, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4089, 4094, 210, 1, 1, 0, 0, 0, 0, 4094, 203, 4096, 2, 0, 0, 4092, 4094, 0, 2, 0, 3684, 6, 0, 0, 0, 4, 0, 0, 0, 4093, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 36, 0, 0, 0, 0, 0, 0, 4096, 407, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 3367, 0, 0, 285, 0, 1, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4085, 4096, 0, 0, 3, 0, 0, 0, 0, 4075, 0, 4096, 0, 0, 0, 4092, 4096, 0, 0, 0, 4064, 40, 20, 10, 4, 10, 0, 0, 0, 4077, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4, 4014, 4060, 898, 18, 0, 0, 0, 7, 54, 3956, 0, 4072, 33, 20, 84, 4050, 4061, 0, 1, 33, 1387, 511, 1192, 337, 0, 2, 30, 0, 33, 3892, 19] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 4091, 63, 21, 0, 0, 6, 0, 5, 4095, 296, 4042, 0, 0, 0, 4096, 4091, 0, 5, 3, 3964, 0, 169, 74, 0, 0, 1, 0, 0, 3648, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 543, 140, 20, 0, 1, 0, 0, 0, 3546, 1, 0, 0, 0, 0, 4096, 0, 78, 0, 0, 0] rerouted_tokens_per_ep_rank=[3956, 4095, 3998, 4096, 4103, 4798, 3548, 4174] avg=4096.0000 max=4798 max_rank=5 min=3548 min_rank=6 imbalance_ratio=1.171387 max_load_reduction=27970 max_load_reduction_pct=0.853577 moved_to_echo=16623 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3950, 0, 210, 0, 0, 0, 0, 1, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 3996, 203, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4092, 7, 0, 2, 0, 0, 0, 0, 0, 3684, 6, 0, 543, 139, 20, 0, 0, 0, 4, 0, 3544, 1, 0, 0, 0, 0, 4093, 3, 78, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4095, 4199, 4098, 4101, 4392, 3549, 4174] avg=4096.0000 max=4392 max_rank=5 min=3549 min_rank=6 imbalance_ratio=1.072266 max_load_reduction=28376 max_load_reduction_pct=0.865967 moved_to_echo=16216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3956, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3992, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 541, 5, 0, 543, 140, 20, 0, 3554, 12, 0, 0, 3546, 1, 0, 0, 0, 0, 4096, 0, 78, 0, 0, 0] rerouted_tokens_per_ep_rank=[3960, 4095, 3992, 4082, 4103, 1249, 7113, 4174] avg=4096.0000 max=7113 max_rank=6 min=1249 min_rank=5 imbalance_ratio=1.736572 max_load_reduction=25655 max_load_reduction_pct=0.782928 moved_to_echo=16639 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3946, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4095, 0, 0, 0, 0, 0, 3977, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 7, 0, 0, 0, 0, 0, 0, 0, 4064, 40, 20, 543, 139, 20, 0, 10, 4, 10, 0, 3546, 1, 0, 0, 0, 0, 4077, 0, 78, 0, 0, 0] rerouted_tokens_per_ep_rank=[3946, 4098, 3977, 4096, 4099, 4826, 3571, 4155] avg=4096.0000 max=4826 max_rank=5 min=3571 min_rank=6 imbalance_ratio=1.178223 max_load_reduction=27942 max_load_reduction_pct=0.852722 moved_to_echo=16651 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3956, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3990, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 3561, 494, 1, 543, 140, 20, 0, 99, 0, 5, 0, 3544, 1, 0, 0, 0, 0, 4087, 0, 78, 0, 0, 0] rerouted_tokens_per_ep_rank=[3959, 4095, 3990, 4048, 4103, 4759, 3649, 4165] avg=4096.0000 max=4759 max_rank=5 min=3649 min_rank=6 imbalance_ratio=1.161865 max_load_reduction=28009 max_load_reduction_pct=0.854767 moved_to_echo=16676 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3956, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3998, 407, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 3367, 0, 0, 543, 140, 20, 0, 285, 0, 1, 0, 3546, 1, 0, 0, 0, 0, 4096, 0, 78, 0, 0, 0] rerouted_tokens_per_ep_rank=[3992, 4095, 4405, 4096, 4103, 4070, 3833, 4174] avg=4096.0000 max=4405 max_rank=2 min=3833 min_rank=6 imbalance_ratio=1.075439 max_load_reduction=28363 max_load_reduction_pct=0.865570 moved_to_echo=16180 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 3874, 0, 898, 0, 0, 0, 0, 18, 0, 0, 0, 4060, 0, 0, 0, 7, 54, 3848, 0, 0, 0, 0, 0, 4072, 33, 20, 84, 1, 0, 0, 0, 4050, 0, 0, 1, 0, 0, 0, 0, 33, 1387, 511, 1192, 545, 140, 27, 0, 337, 0, 2, 30, 3516, 0, 0, 0, 0, 33, 3892, 19, 80, 0, 0, 0] rerouted_tokens_per_ep_rank=[4776, 4078, 3909, 4210, 4051, 3835, 3885, 4024] avg=4096.0000 max=4776 max_rank=0 min=3835 min_rank=5 imbalance_ratio=1.166016 max_load_reduction=27992 max_load_reduction_pct=0.854248 moved_to_echo=15795 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3956, 0, 63, 0, 0, 0, 0, 21, 0, 0, 6, 4090, 0, 0, 0, 0, 5, 3997, 296, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 5, 0, 0, 0, 0, 3, 3964, 0, 169, 543, 140, 20, 0, 74, 0, 0, 1, 3544, 1, 0, 0, 0, 0, 3648, 2, 78, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4117, 4298, 4042, 4105, 4839, 3620, 3728] avg=4096.0000 max=4839 max_rank=5 min=3620 min_rank=6 imbalance_ratio=1.181396 max_load_reduction=27929 max_load_reduction_pct=0.852325 moved_to_echo=16292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4480, 128, 128, 128, 28416, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 24704, 1152, 1408, 4352, 1152, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 31616, 0, 1280, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 31872, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32128, 128, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4088, 4096, 4095, 0, 0, 0, 1876, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 2234, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4092, 4036, 0, 0, 7, 4065, 0, 0, 4085, 0, 4096, 3, 1, 0, 0, 7, 4093, 53, 0, 1, 0, 1, 36, 0, 0, 4095, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4037, 4096, 4096, 0, 25, 0, 540, 0, 1, 4084, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 3583, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[45, 4061, 4051, 3781, 38, 0, 131, 4085, 12, 0, 4063, 1, 3994, 0, 0, 3, 0, 1, 3824, 293, 0, 0, 0, 30, 200, 141, 9, 3929, 46, 0, 30, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 0, 176, 0, 1, 4078, 0, 4096, 0, 0, 0, 0, 1, 4096, 72, 0, 0, 0, 0, 37, 0, 0, 4096, 0, 906, 2912, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4092, 0, 0, 0, 2099, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 2001, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 0, 2882, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 51, 1163, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4093, 4089, 4092, 0, 0, 9, 3323, 0, 6, 3948, 3, 4079, 0, 0, 0, 1, 15, 4092, 0, 11, 0, 0, 0, 0, 10, 1, 4085, 0, 107, 772, 32] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 44, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1876, 1678, 12, 0, 0, 0, 0, 4096, 0, 19, 5, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 2234, 0, 2365, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 3566, 4120, 4109, 4072, 4097, 4066, 4599] avg=4096.0000 max=4599 max_rank=7 min=3566 min_rank=1 imbalance_ratio=1.122803 max_load_reduction=28169 max_load_reduction_pct=0.859650 moved_to_echo=16834 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 44, 0, 4036, 0, 0, 0, 0, 0, 0, 7, 4065, 1681, 12, 0, 0, 0, 0, 4085, 0, 19, 5, 0, 0, 4096, 3, 1, 0, 13, 0, 0, 0, 0, 7, 4075, 53, 0, 0, 0, 0, 0, 1, 0, 1, 4092, 1, 0, 0, 36, 0, 0, 4064, 0, 0, 0, 0, 0, 1, 0, 0, 2370, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 5765, 4109, 4113, 4135, 4095, 4100, 2371] avg=4096.0000 max=5765 max_rank=1 min=2371 min_rank=7 imbalance_ratio=1.407471 max_load_reduction=27003 max_load_reduction_pct=0.824066 moved_to_echo=14701 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 0, 4096, 0, 0, 0, 0, 0, 25, 0, 540, 1657, 12, 0, 0, 0, 1, 4084, 0, 19, 5, 0, 0, 4096, 0, 1, 0, 13, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 17, 3583, 0, 2336, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 2234, 4109, 4110, 4078, 4097, 4065, 5936] avg=4096.0000 max=5936 max_rank=7 min=2234 min_rank=1 imbalance_ratio=1.449219 max_load_reduction=26832 max_load_reduction_pct=0.818848 moved_to_echo=18176 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 44, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 176, 1681, 12, 0, 0, 0, 1, 4078, 0, 19, 5, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 1, 4078, 72, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 37, 0, 0, 4065, 0, 0, 0, 0, 0, 906, 2912, 9, 2370, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 1869, 4103, 4109, 4151, 4097, 4102, 6197] avg=4096.0000 max=6197 max_rank=7 min=1869 min_rank=1 imbalance_ratio=1.512939 max_load_reduction=26571 max_load_reduction_pct=0.810883 moved_to_echo=18547 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 44, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2882, 1681, 12, 0, 0, 0, 0, 4093, 0, 19, 5, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 3, 51, 1163, 0, 2370, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4575, 4117, 4109, 4078, 4097, 4065, 3587] avg=4096.0000 max=4575 max_rank=1 min=3587 min_rank=7 imbalance_ratio=1.116943 max_load_reduction=28193 max_load_reduction_pct=0.860382 moved_to_echo=15827 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 44, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 2099, 1681, 12, 0, 0, 0, 0, 4096, 0, 19, 5, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 2001, 0, 2370, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 3792, 4120, 4109, 4077, 4097, 4066, 4371] avg=4096.0000 max=4371 max_rank=7 min=3792 min_rank=1 imbalance_ratio=1.067139 max_load_reduction=28397 max_load_reduction_pct=0.866608 moved_to_echo=16611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[45, 32, 0, 3781, 0, 0, 0, 0, 38, 0, 131, 4085, 1671, 18, 0, 0, 12, 0, 4063, 1, 22, 11, 0, 0, 3994, 0, 0, 3, 20, 0, 0, 0, 0, 1, 3793, 293, 0, 0, 0, 0, 0, 0, 0, 30, 4051, 6, 0, 0, 200, 141, 9, 3889, 0, 0, 0, 0, 46, 0, 30, 0, 2352, 0, 0, 0] rerouted_tokens_per_ep_rank=[3858, 5943, 4109, 4017, 4087, 4087, 4239, 2428] avg=4096.0000 max=5943 max_rank=1 min=2428 min_rank=7 imbalance_ratio=1.450928 max_load_reduction=26825 max_load_reduction_pct=0.818634 moved_to_echo=14841 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 44, 0, 4092, 0, 0, 0, 0, 0, 0, 9, 3323, 1680, 12, 0, 0, 0, 6, 3948, 3, 19, 5, 0, 0, 4079, 0, 0, 0, 13, 0, 0, 0, 1, 15, 4074, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4089, 1, 0, 0, 0, 10, 1, 4054, 0, 0, 0, 0, 0, 107, 772, 32, 2368, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 5024, 3981, 4092, 4090, 4101, 4065, 3279] avg=4096.0000 max=5024 max_rank=1 min=3279 min_rank=7 imbalance_ratio=1.226562 max_load_reduction=27744 max_load_reduction_pct=0.846680 moved_to_echo=15535 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 1152, 12800, 128, 18944, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 256, 19072, 13440, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32384, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 256, 128, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 384, 0, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32640, 128, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4096, 4095, 0, 299, 0, 0, 1336, 0, 0, 0, 4096, 0, 0, 0, 2462, 0, 0, 4089, 4093, 0, 7, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 4090, 5, 2761, 0, 0, 3236, 0, 0, 0, 4096, 0, 0, 0, 845, 0, 0, 1346, 4094, 0, 8, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4031, 0, 1773, 0, 4, 762, 0, 0, 5, 4096, 0, 0, 42, 3547, 0, 0, 2074, 4096, 0, 50, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 2, 546, 3, 0, 1357, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 2390, 4093, 18, 28, 3853] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 4076, 4096, 42, 3, 0, 0, 3430, 0, 0, 0, 4095, 0, 0, 0, 3906, 0, 0, 707, 4094, 0, 156, 4059] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4044, 0, 3, 2, 1, 2, 4056, 4080, 47, 549, 0, 0, 1662, 0, 2, 3, 4096, 2, 0, 0, 1941, 0, 0, 3276, 4020, 0, 1225, 3757] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[54, 68, 0, 0, 4020, 30, 1, 21, 57, 0, 3913, 4026, 83, 9, 0, 5, 3420, 29, 1, 0, 4036, 36, 0, 0, 167, 35, 4, 3814, 4055, 0, 912, 3972] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[2, 0, 0, 13, 4087, 0, 0, 0, 1, 16, 4093, 3988, 0, 90, 0, 2, 767, 1, 0, 0, 4095, 2, 8, 2, 3339, 0, 0, 4078, 4072, 0, 191, 3921] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 6, 3, 0, 0, 0, 3, 25, 4095, 0, 0, 0, 0, 0, 299, 0, 0, 3343, 0, 0, 0, 1336, 0, 0, 0, 1755, 728, 208, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2462, 0, 0, 2334, 0, 0, 0, 0, 0, 0, 7, 3882, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4105, 4123, 3642, 4027, 4093, 4796, 3889] avg=4096.0000 max=4796 max_rank=6 min=3642 min_rank=3 imbalance_ratio=1.170898 max_load_reduction=27972 max_load_reduction_pct=0.853638 moved_to_echo=16805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 1, 6, 3, 0, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 2, 546, 3, 0, 3343, 0, 0, 0, 1357, 0, 0, 0, 1026, 728, 196, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1364, 0, 0, 0, 0, 0, 18, 28, 3651, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4106, 4121, 3894, 3307, 4090, 5460, 3697] avg=4096.0000 max=5460 max_rank=6 min=3307 min_rank=4 imbalance_ratio=1.333008 max_load_reduction=27308 max_load_reduction_pct=0.833374 moved_to_echo=16554 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 6, 3, 0, 0, 0, 1, 25, 4090, 0, 0, 0, 0, 5, 2761, 0, 0, 3343, 0, 0, 0, 3236, 0, 0, 0, 577, 728, 208, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 845, 0, 0, 769, 0, 0, 0, 0, 0, 0, 8, 3880, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4105, 4116, 6109, 4749, 4093, 1614, 3888] avg=4096.0000 max=6109 max_rank=3 min=1614 min_rank=6 imbalance_ratio=1.491455 max_load_reduction=26659 max_load_reduction_pct=0.813568 moved_to_echo=14344 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4020, 0, 0, 0, 4044, 0, 3, 2, 6, 3, 0, 0, 1, 2, 25, 4080, 0, 0, 0, 0, 47, 549, 0, 0, 3310, 0, 0, 0, 1662, 0, 2, 3, 1406, 721, 191, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 1941, 0, 0, 1870, 0, 0, 0, 0, 0, 0, 1225, 3560, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4058, 4108, 3906, 3985, 4095, 3811, 4785] avg=4096.0000 max=4785 max_rank=7 min=3811 min_rank=6 imbalance_ratio=1.168213 max_load_reduction=27983 max_load_reduction_pct=0.853973 moved_to_echo=16676 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 6, 3, 0, 0, 0, 0, 25, 4031, 0, 0, 0, 0, 0, 1773, 0, 4, 3343, 0, 0, 0, 762, 0, 0, 5, 890, 728, 208, 0, 4093, 0, 0, 42, 0, 0, 0, 0, 3547, 0, 0, 1184, 0, 0, 0, 0, 0, 0, 50, 3882, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4105, 4056, 5120, 2593, 4135, 4731, 3932] avg=4096.0000 max=5120 max_rank=3 min=2593 min_rank=4 imbalance_ratio=1.250000 max_load_reduction=27648 max_load_reduction_pct=0.843750 moved_to_echo=15391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 6, 3, 0, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 42, 3, 0, 0, 3326, 0, 0, 0, 3430, 0, 0, 0, 303, 725, 206, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 3906, 0, 0, 404, 0, 0, 0, 0, 0, 0, 156, 3847, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4105, 4121, 3371, 4664, 4092, 4310, 4003] avg=4096.0000 max=4664 max_rank=4 min=3371 min_rank=3 imbalance_ratio=1.138672 max_load_reduction=28104 max_load_reduction_pct=0.857666 moved_to_echo=17069 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[54, 68, 0, 0, 4055, 6, 0, 0, 4020, 30, 1, 21, 10, 6, 0, 0, 57, 0, 10, 4026, 0, 0, 0, 0, 83, 9, 0, 5, 3197, 0, 0, 0, 3420, 29, 1, 0, 1641, 700, 207, 0, 4030, 36, 0, 0, 0, 0, 0, 0, 167, 35, 4, 2173, 0, 0, 0, 0, 0, 0, 912, 3755, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4183, 4088, 4093, 3294, 5998, 4066, 2379, 4667] avg=4096.0000 max=5998 max_rank=4 min=2379 min_rank=6 imbalance_ratio=1.464355 max_load_reduction=26770 max_load_reduction_pct=0.816956 moved_to_echo=17110 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 13, 4072, 0, 0, 0, 4087, 0, 0, 0, 6, 3, 0, 0, 1, 16, 25, 3988, 0, 0, 0, 0, 0, 90, 0, 2, 3340, 0, 0, 0, 767, 1, 0, 0, 1750, 728, 199, 0, 4092, 2, 8, 2, 0, 0, 0, 0, 3339, 0, 0, 2328, 0, 0, 0, 0, 0, 0, 191, 3716, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4096, 4030, 3432, 3445, 4104, 5667, 3907] avg=4096.0000 max=5667 max_rank=6 min=3432 min_rank=3 imbalance_ratio=1.383545 max_load_reduction=27101 max_load_reduction_pct=0.827057 moved_to_echo=17123 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[16000, 128, 128, 128, 9472, 5888, 1664, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 6144, 128, 128, 26624, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[20352, 128, 128, 12544, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 256, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 2688, 30208, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 3994, 0, 0, 3182, 0, 0, 3, 0, 4096, 27, 4096, 0, 0, 0, 0, 614, 94, 4096, 4096, 0, 275, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4095, 4096, 4094, 0, 0, 3039, 0, 0, 0, 0, 4096, 261, 4096, 0, 0, 0, 0, 4037, 66, 719, 4096, 34, 38, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 2, 0, 0, 0, 4096, 4083, 4093, 0, 0, 106, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4055, 0, 4041, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 15, 0, 0, 0, 4091, 4067, 4088, 0, 0, 161, 0, 0, 0, 0, 4089, 0, 4092, 0, 0, 0, 0, 3965, 3, 4094, 4096, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 19, 0, 0, 0, 1, 2, 4096, 4090, 4053, 0, 0, 3754, 1, 0, 3, 0, 4092, 68, 4096, 0, 0, 0, 0, 252, 15, 4094, 4091, 5, 33, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[49, 35, 0, 14, 0, 12, 45, 34, 4044, 3728, 3940, 8, 52, 4066, 0, 2, 3, 0, 4066, 145, 4045, 0, 0, 15, 43, 9, 0, 4058, 4027, 313, 0, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 1258, 0, 0, 0, 0, 4096, 6, 4096, 0, 0, 0, 0, 2831, 6, 4091, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 7, 1, 0, 9, 4093, 4091, 4095, 0, 0, 3626, 0, 1, 9, 0, 4089, 66, 4095, 0, 0, 0, 0, 508, 0, 3933, 4091, 50, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 3943, 74, 8, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 51, 0, 0, 0, 0, 0, 0, 3182, 0, 0, 1733, 86, 62, 0, 3, 0, 4029, 27, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 614, 94, 2289, 0, 0, 0, 0, 4010, 0, 275, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4088, 4147, 5063, 4059, 4101, 2997, 4285] avg=4096.0000 max=5063 max_rank=3 min=2997 min_rank=6 imbalance_ratio=1.236084 max_load_reduction=27705 max_load_reduction_pct=0.845490 moved_to_echo=15442 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 4035, 74, 9, 0, 15, 0, 0, 0, 4082, 0, 0, 0, 0, 4067, 53, 0, 0, 0, 0, 0, 0, 161, 0, 0, 1732, 86, 62, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 5, 0, 0, 0, 0, 3965, 3, 2288, 0, 0, 0, 0, 4010, 0, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4097, 4120, 2041, 4022, 4097, 6256, 4012] avg=4096.0000 max=6256 max_rank=6 min=2041 min_rank=3 imbalance_ratio=1.527344 max_load_reduction=26512 max_load_reduction_pct=0.809082 moved_to_echo=18387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4040, 73, 8, 0, 2, 0, 0, 0, 4088, 0, 0, 0, 0, 4083, 53, 0, 0, 0, 0, 0, 0, 106, 0, 0, 1710, 86, 62, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4055, 0, 2258, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4090, 4136, 1964, 4029, 4101, 6313, 4010] avg=4096.0000 max=6313 max_rank=6 min=1964 min_rank=3 imbalance_ratio=1.541260 max_load_reduction=26455 max_load_reduction_pct=0.807343 moved_to_echo=18453 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4041, 13, 9, 0, 1, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 53, 0, 0, 0, 0, 0, 0, 3039, 0, 0, 304, 86, 62, 0, 0, 0, 4029, 261, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4037, 66, 402, 0, 0, 0, 0, 4010, 34, 38, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4087, 4149, 3491, 4290, 4101, 4505, 4082] avg=4096.0000 max=4505 max_rank=6 min=3491 min_rank=3 imbalance_ratio=1.099854 max_load_reduction=28263 max_load_reduction_pct=0.862518 moved_to_echo=16978 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 19, 0, 4001, 74, 8, 0, 0, 0, 1, 2, 4088, 0, 0, 0, 0, 4090, 52, 0, 0, 0, 0, 0, 0, 3754, 1, 0, 1732, 86, 62, 0, 3, 0, 4025, 68, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 252, 15, 2288, 0, 0, 0, 0, 4005, 5, 33, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4091, 4142, 5635, 4096, 4101, 2555, 4045] avg=4096.0000 max=5635 max_rank=3 min=2555 min_rank=6 imbalance_ratio=1.375732 max_load_reduction=27133 max_load_reduction_pct=0.828033 moved_to_echo=14797 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4041, 74, 8, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 53, 0, 0, 0, 0, 0, 0, 1258, 0, 0, 1731, 86, 62, 0, 0, 0, 4029, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 2831, 6, 2286, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4088, 4149, 3137, 4035, 4101, 5123, 4010] avg=4096.0000 max=5123 max_rank=6 min=3137 min_rank=3 imbalance_ratio=1.250732 max_load_reduction=27645 max_load_reduction_pct=0.843658 moved_to_echo=17269 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[49, 35, 0, 14, 3895, 75, 9, 0, 0, 12, 45, 34, 4035, 0, 0, 0, 0, 3728, 45, 8, 0, 0, 0, 0, 52, 4066, 0, 2, 1722, 88, 68, 0, 3, 0, 3992, 145, 0, 0, 0, 0, 4045, 0, 0, 15, 6, 0, 0, 0, 43, 9, 0, 2261, 0, 0, 0, 0, 3939, 313, 0, 15, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4126, 3781, 5998, 4140, 4066, 2313, 4267] avg=4096.0000 max=5998 max_rank=3 min=2313 min_rank=6 imbalance_ratio=1.464355 max_load_reduction=26770 max_load_reduction_pct=0.816956 moved_to_echo=14786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4042, 71, 9, 0, 7, 1, 0, 9, 4084, 0, 0, 0, 0, 4091, 53, 0, 0, 0, 0, 0, 0, 3626, 0, 1, 1664, 86, 62, 0, 9, 0, 4022, 66, 0, 0, 0, 0, 4095, 0, 0, 0, 5, 0, 0, 0, 0, 508, 0, 2198, 0, 0, 0, 0, 4005, 50, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4101, 4144, 5439, 4097, 4100, 2706, 4057] avg=4096.0000 max=5439 max_rank=3 min=2706 min_rank=6 imbalance_ratio=1.327881 max_load_reduction=27329 max_load_reduction_pct=0.834015 moved_to_echo=14960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 19200, 128, 128, 12416, 768, 512, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 16384, 256, 16384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32128, 640, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32000, 512, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32256, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32384, 512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4016, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 275, 0, 0, 0, 4096, 4063, 2, 22, 0, 4041, 40, 0, 0, 3925, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 23, 0, 0, 0, 4096, 4093, 0, 0, 0, 4072, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4078, 1, 4090, 3, 4, 0, 1, 4096, 0, 4096, 0, 0, 9, 0, 3, 0, 4096, 4096, 0, 11, 0, 4087, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4093, 0, 4072, 0, 1, 0, 0, 4095, 0, 4096, 0, 0, 190, 0, 0, 1, 4096, 4095, 0, 18, 0, 3906, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4093, 0, 4096, 0, 0, 0, 5, 4092, 0, 4054, 0, 0, 27, 0, 8, 0, 4088, 4096, 0, 1, 0, 4070, 0, 0, 3, 4066, 0, 67, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4095, 1, 4027, 1, 100, 2, 63, 4052, 6, 3989, 1, 7, 137, 38, 0, 0, 4063, 4091, 159, 0, 0, 3988, 0, 0, 3, 3782, 36, 1, 85, 1, 0, 40] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4093, 6, 4087, 4, 26, 2, 0, 4095, 0, 4090, 0, 0, 0, 0, 0, 0, 4093, 4095, 0, 135, 0, 3777, 0, 0, 1, 4074, 0, 176, 10, 3, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4007, 12, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 76, 10, 0, 0, 0, 4096, 0, 0, 27, 1, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4078, 4113, 4019, 4059, 4182, 4124, 4084] avg=4096.0000 max=4182 max_rank=5 min=4019 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16449 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 275, 0, 0, 0, 3929, 13, 0, 0, 4059, 0, 2, 22, 0, 0, 0, 0, 0, 4041, 40, 0, 74, 10, 0, 0, 0, 3925, 0, 1, 27, 1, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4078, 4113, 4217, 4083, 4165, 3954, 4050] avg=4096.0000 max=4217 max_rank=3 min=3954 min_rank=6 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[13, 1, 4090, 3, 0, 0, 0, 0, 4, 0, 1, 4078, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 9, 0, 3, 0, 3990, 12, 0, 0, 4059, 0, 0, 11, 0, 0, 0, 0, 0, 4087, 0, 0, 75, 10, 0, 0, 0, 4096, 0, 1, 27, 1, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4083, 4113, 4014, 4070, 4172, 4125, 4084] avg=4096.0000 max=4172 max_rank=5 min=4014 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16451 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 23, 0, 0, 0, 4007, 12, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 76, 10, 0, 0, 0, 4096, 0, 5, 27, 1, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4078, 4113, 4042, 4059, 4158, 4129, 4081] avg=4096.0000 max=4158 max_rank=5 min=4042 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16427 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 4074, 0, 0, 0, 0, 0, 4054, 0, 0, 17, 0, 0, 0, 27, 0, 8, 0, 4005, 12, 0, 0, 4051, 0, 0, 1, 0, 0, 0, 0, 0, 4070, 0, 0, 76, 10, 0, 0, 3, 4066, 0, 67, 27, 1, 0, 0, 0, 2, 0, 0, 4084, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4079, 4071, 4052, 4052, 4156, 4164, 4086] avg=4096.0000 max=4164 max_rank=6 min=4052 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16458 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4072, 0, 0, 0, 0, 0, 1, 0, 0, 4077, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 190, 0, 0, 1, 4005, 12, 0, 0, 4059, 0, 0, 18, 0, 0, 0, 0, 0, 3906, 0, 0, 76, 10, 0, 0, 0, 4096, 0, 9, 27, 1, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4078, 4113, 4208, 4077, 3992, 4133, 4083] avg=4096.0000 max=4208 max_rank=3 min=3992 min_rank=5 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16285 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[6, 1, 4027, 1, 0, 0, 0, 0, 100, 2, 63, 4024, 0, 0, 0, 0, 6, 3989, 1, 7, 22, 0, 0, 0, 137, 38, 0, 0, 4010, 13, 0, 0, 4018, 0, 159, 0, 0, 0, 0, 0, 0, 3988, 0, 0, 79, 13, 0, 0, 3, 3782, 36, 1, 32, 6, 0, 0, 85, 1, 0, 40, 4078, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4189, 4025, 4198, 4177, 4080, 3860, 4204] avg=4096.0000 max=4204 max_rank=7 min=3860 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 6, 4087, 4, 0, 0, 0, 0, 26, 2, 0, 4077, 0, 0, 0, 0, 0, 4090, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4005, 13, 0, 0, 4056, 0, 0, 135, 0, 0, 0, 0, 0, 3777, 0, 0, 76, 10, 0, 0, 1, 4074, 0, 176, 27, 1, 0, 0, 10, 3, 1, 0, 4082, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4105, 4107, 4018, 4191, 3863, 4279, 4096] avg=4096.0000 max=4279 max_rank=6 min=3863 min_rank=5 imbalance_ratio=1.044678 max_load_reduction=28489 max_load_reduction_pct=0.869415 moved_to_echo=16429 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[768, 128, 128, 128, 32000, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32128, 128, 0, 640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32256, 128, 384, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 0, 256, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32640, 128, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 444, 0, 4096, 0, 4090, 0, 0, 0, 4092, 0, 4096, 0, 4056, 0, 0, 0, 0, 3687, 0, 3, 0, 12, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 3676, 0, 4095, 8, 4051, 0, 0, 0, 4096, 6, 4092, 0, 550, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 2894, 0, 4095, 0, 4094, 3, 0, 3, 4096, 158, 4096, 0, 1301, 0, 0, 0, 0, 3765, 0, 0, 0, 74, 0, 0, 0, 0, 4096, 0, 4093, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 3605, 8, 4096, 0, 4092, 0, 0, 0, 4093, 0, 4092, 0, 603, 0, 0, 0, 0, 3969, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 368, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 4096, 0, 3731, 0, 0, 0, 0, 4086, 0, 1, 0, 5, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 688, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 4096, 0, 3102, 0, 0, 0, 0, 4089, 0, 302, 0, 5, 0, 0, 0, 0, 4096, 5, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[41, 16, 1389, 0, 4033, 0, 3939, 0, 76, 25, 4078, 144, 4055, 0, 3845, 7, 50, 0, 31, 2828, 51, 0, 0, 117, 1, 0, 0, 0, 4061, 0, 3977, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 5, 4060, 0, 4096, 0, 4095, 4, 0, 1, 4093, 0, 4091, 0, 985, 0, 0, 9, 0, 3034, 0, 7, 0, 0, 0, 6, 0, 0, 4087, 105, 4090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3605, 8, 1913, 34, 0, 0, 27, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 1826, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 353, 48, 0, 0, 0, 18, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5560, 4119, 4045, 2429, 4370, 4047, 4100, 4098] avg=4096.0000 max=5560 max_rank=0 min=2429 min_rank=3 imbalance_ratio=1.357422 max_load_reduction=27208 max_load_reduction_pct=0.830322 moved_to_echo=16615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2894, 0, 1915, 34, 0, 0, 27, 0, 4094, 3, 0, 0, 0, 0, 0, 3, 4048, 158, 0, 0, 0, 0, 1827, 0, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 3765, 354, 48, 0, 0, 0, 0, 0, 74, 4028, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4843, 4124, 4209, 3128, 4167, 4102, 4100, 4095] avg=4096.0000 max=4843 max_rank=0 min=3128 min_rank=3 imbalance_ratio=1.182373 max_load_reduction=27925 max_load_reduction_pct=0.852203 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 368, 0, 1915, 34, 0, 0, 27, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4048, 1, 0, 0, 0, 0, 1827, 0, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 354, 48, 0, 0, 0, 1, 0, 5, 4029, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2317, 4123, 4049, 5558, 4488, 4035, 4100, 4098] avg=4096.0000 max=5558 max_rank=3 min=2317 min_rank=0 imbalance_ratio=1.356934 max_load_reduction=27210 max_load_reduction_pct=0.830383 moved_to_echo=16721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3676, 0, 1913, 34, 0, 0, 27, 8, 4051, 0, 0, 0, 0, 0, 0, 0, 4048, 6, 0, 0, 0, 0, 1826, 0, 550, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 353, 48, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 4091, 6, 0, 0, 2, 2, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5623, 4086, 4054, 2376, 4404, 4028, 4097, 4100] avg=4096.0000 max=5623 max_rank=0 min=2376 min_rank=3 imbalance_ratio=1.372803 max_load_reduction=27145 max_load_reduction_pct=0.828400 moved_to_echo=16629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 444, 0, 1915, 34, 0, 0, 27, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 1827, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 3687, 354, 48, 0, 0, 0, 3, 0, 12, 4029, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2393, 4117, 4044, 5883, 4089, 4044, 4100, 4098] avg=4096.0000 max=5883 max_rank=3 min=2393 min_rank=0 imbalance_ratio=1.436279 max_load_reduction=26885 max_load_reduction_pct=0.820465 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 688, 0, 1915, 34, 0, 0, 27, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4048, 1, 0, 0, 0, 0, 1827, 0, 3102, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 354, 48, 0, 0, 0, 302, 0, 5, 4029, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 2, 5, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2637, 4123, 4049, 4929, 4491, 4336, 4100, 4103] avg=4096.0000 max=4929 max_rank=3 min=2637 min_rank=0 imbalance_ratio=1.203369 max_load_reduction=27839 max_load_reduction_pct=0.849579 moved_to_echo=17030 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[41, 16, 1389, 0, 1898, 40, 0, 0, 11, 0, 3939, 0, 0, 0, 0, 0, 76, 25, 4027, 144, 0, 0, 0, 0, 1802, 0, 3845, 7, 0, 0, 0, 0, 50, 0, 31, 2828, 355, 51, 0, 0, 51, 0, 0, 117, 3972, 0, 0, 0, 1, 0, 0, 0, 4061, 10, 0, 0, 0, 0, 3977, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3384, 3950, 4272, 5654, 3315, 4140, 4072, 3981] avg=4096.0000 max=5654 max_rank=3 min=3315 min_rank=4 imbalance_ratio=1.380371 max_load_reduction=27114 max_load_reduction_pct=0.827454 moved_to_echo=15508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4060, 0, 1912, 34, 0, 0, 27, 0, 4095, 4, 0, 0, 0, 0, 0, 1, 4045, 0, 0, 0, 0, 0, 1826, 0, 985, 0, 0, 0, 0, 0, 0, 9, 0, 3034, 353, 48, 0, 0, 0, 7, 0, 0, 4029, 0, 0, 0, 0, 6, 0, 0, 4087, 6, 0, 0, 0, 105, 4090, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6011, 4126, 4046, 2811, 3444, 4036, 4099, 4195] avg=4096.0000 max=6011 max_rank=0 min=2811 min_rank=3 imbalance_ratio=1.467529 max_load_reduction=26757 max_load_reduction_pct=0.816559 moved_to_echo=15774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[14592, 0, 18176, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 17152, 128, 15360, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 29568, 2944, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 384, 0, 256, 32256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32384, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4092, 0, 3, 19, 0, 4094, 4083, 0, 0, 4090, 4096, 6, 2, 4052, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 38, 0, 0, 3, 4094, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 5, 4096, 4079, 0, 0, 4096, 4096, 4, 0, 4092, 0, 0, 1, 0, 4085, 0, 2, 0, 0, 6, 15, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 29, 0, 0, 2, 4096, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4063, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 3884, 261, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 4091, 0, 0, 4003, 0, 0, 0, 0, 4096, 0, 0, 0, 100, 0, 0, 0, 0, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[3982, 26, 0, 107, 0, 4022, 4082, 0, 18, 4034, 3931, 0, 58, 3104, 0, 10, 0, 992, 4074, 19, 149, 2, 0, 0, 0, 14, 0, 0, 4070, 74, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 4096, 4077, 7, 1, 4094, 4094, 0, 2, 3968, 0, 0, 0, 29, 4095, 3, 18, 0, 0, 0, 0, 0, 96, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 3, 19, 0, 0, 0, 0, 0, 4094, 9, 0, 0, 0, 0, 0, 0, 18, 4096, 6, 0, 0, 0, 0, 2, 4052, 0, 0, 126, 13, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4, 0, 0, 38, 0, 0, 3, 4070, 0, 0, 0, 4094, 0, 0, 0, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4103, 4120, 4193, 3969, 4063, 4111, 4099] avg=4096.0000 max=4193 max_rank=3 min=3969 min_rank=4 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 4096, 29, 0, 126, 13, 0, 0, 0, 2, 3969, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4065, 4, 0, 0, 1, 0, 0, 0, 4083, 0, 0, 0, 4063, 0, 0, 0, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4105, 4114, 4264, 3971, 4070, 4084, 4068] avg=4096.0000 max=4264 max_rank=3 min=3971 min_rank=4 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 9, 0, 0, 0, 0, 0, 0, 18, 4096, 4, 0, 0, 0, 0, 0, 4092, 0, 0, 125, 13, 0, 0, 1, 0, 3959, 0, 0, 0, 0, 0, 2, 0, 0, 6, 4065, 4, 0, 0, 15, 0, 0, 0, 4066, 0, 0, 0, 4096, 0, 0, 0, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4110, 4118, 4230, 3960, 4077, 4081, 4101] avg=4096.0000 max=4230 max_rank=3 min=3960 min_rank=4 imbalance_ratio=1.032715 max_load_reduction=28538 max_load_reduction_pct=0.870911 moved_to_echo=16219 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 3884, 261, 0, 126, 13, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4047, 0, 0, 0, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4105, 4114, 4284, 3969, 4069, 4083, 4052] avg=4096.0000 max=4284 max_rank=3 min=3969 min_rank=4 imbalance_ratio=1.045898 max_load_reduction=28484 max_load_reduction_pct=0.869263 moved_to_echo=16173 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 18, 4091, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 126, 13, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 100, 0, 4065, 4, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4095, 0, 0, 0, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4105, 4109, 4142, 3969, 4169, 4082, 4100] avg=4096.0000 max=4169 max_rank=5 min=3969 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 126, 13, 0, 0, 0, 5, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4092, 0, 0, 0, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4105, 4114, 4234, 3974, 4069, 4083, 4097] avg=4096.0000 max=4234 max_rank=3 min=3974 min_rank=4 imbalance_ratio=1.033691 max_load_reduction=28534 max_load_reduction_pct=0.870789 moved_to_echo=16223 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3978, 26, 0, 107, 0, 0, 0, 0, 0, 4022, 1, 0, 0, 0, 0, 0, 18, 17, 3931, 0, 0, 0, 0, 0, 58, 3104, 0, 10, 127, 13, 0, 0, 0, 992, 3945, 19, 0, 0, 0, 0, 149, 2, 0, 0, 4004, 11, 0, 0, 0, 14, 0, 0, 4070, 0, 0, 0, 4070, 74, 0, 0, 4, 2, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4023, 3966, 3312, 4956, 4166, 4084, 4150] avg=4096.0000 max=4956 max_rank=4 min=3312 min_rank=3 imbalance_ratio=1.209961 max_load_reduction=27812 max_load_reduction_pct=0.848755 moved_to_echo=17356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 7, 0, 0, 0, 0, 1, 18, 4094, 0, 0, 0, 0, 0, 2, 3968, 0, 0, 126, 13, 0, 0, 0, 29, 3968, 3, 0, 0, 0, 0, 18, 0, 0, 0, 4063, 4, 0, 0, 0, 0, 96, 0, 4064, 0, 0, 0, 4096, 0, 0, 0, 4, 1, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4112, 4113, 4109, 4000, 4085, 4160, 4101] avg=4096.0000 max=4160 max_rank=6 min=4000 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 128, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31360, 384, 128, 1024, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 1152, 31744, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 3941, 0, 19, 4081, 2, 0, 0, 4076, 4088, 42, 10, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 13, 0, 0, 0, 4096, 115, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4020, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 34, 0, 4096, 42, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 3562, 0, 0, 4095, 0, 0, 0, 4096, 4096, 31, 0, 4096, 62, 4079, 0, 0, 0, 0, 0, 0, 13, 4094, 0, 0, 0, 0, 4096, 448, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4083, 1, 0, 0, 4095, 4096, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 8, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 1532, 14, 0, 4035, 12, 7, 81, 4002, 3922, 1, 0, 4040, 0, 3718, 0, 28, 0, 0, 15, 73, 151, 4068, 0, 1, 156, 0, 3977, 2757, 11, 139, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[7, 3617, 0, 0, 4096, 0, 0, 2, 4088, 4096, 5, 0, 4096, 0, 4095, 0, 0, 2, 0, 0, 5, 261, 4090, 1, 1, 0, 0, 4092, 214, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4082, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 10, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4084, 0, 0, 0, 3998, 4096, 54, 0, 4094, 0, 4089, 0, 0, 2, 4, 1, 0, 17, 4094, 0, 0, 31, 12, 4094, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3941, 0, 19, 396, 62, 15, 0, 4081, 2, 0, 0, 0, 0, 0, 0, 4076, 10, 42, 10, 0, 0, 0, 0, 44, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 25, 0, 0, 0, 0, 4031, 13, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 115, 0, 0, 0, 3631, 0, 0, 0] rerouted_tokens_per_ep_rank=[4433, 4083, 4138, 4140, 4103, 4044, 4081, 3746] avg=4096.0000 max=4433 max_rank=0 min=3746 min_rank=7 imbalance_ratio=1.082275 max_load_reduction=28335 max_load_reduction_pct=0.864716 moved_to_echo=15974 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4020, 0, 0, 396, 62, 15, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 44, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 25, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 34, 0, 4081, 0, 0, 0, 0, 42, 0, 0, 0, 3631, 0, 0, 0] rerouted_tokens_per_ep_rank=[4493, 4096, 4106, 4140, 4111, 4034, 4115, 3673] avg=4096.0000 max=4493 max_rank=0 min=3673 min_rank=7 imbalance_ratio=1.096924 max_load_reduction=28275 max_load_reduction_pct=0.862885 moved_to_echo=15933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[7, 3617, 0, 0, 396, 61, 15, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4088, 10, 5, 0, 0, 0, 0, 0, 44, 0, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4086, 25, 0, 0, 5, 261, 4029, 1, 0, 0, 0, 0, 1, 0, 0, 4077, 0, 0, 0, 0, 214, 0, 0, 0, 3631, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4098, 4103, 4139, 4113, 4296, 4078, 3845] avg=4096.0000 max=4296 max_rank=5 min=3845 min_rank=7 imbalance_ratio=1.048828 max_load_reduction=28472 max_load_reduction_pct=0.868896 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3562, 0, 0, 396, 62, 15, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 31, 0, 0, 0, 0, 0, 44, 62, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 25, 0, 0, 0, 13, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 448, 0, 0, 0, 3631, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4095, 4137, 4185, 4111, 4045, 4081, 4079] avg=4096.0000 max=4185 max_rank=3 min=4035 min_rank=0 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 396, 62, 15, 0, 4083, 1, 0, 0, 0, 0, 0, 0, 4095, 10, 0, 2, 0, 0, 0, 0, 44, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 25, 0, 0, 0, 3, 4034, 0, 0, 0, 0, 0, 0, 0, 8, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 3631, 0, 0, 0] rerouted_tokens_per_ep_rank=[4569, 4084, 4107, 4140, 4111, 4037, 4089, 3631] avg=4096.0000 max=4569 max_rank=0 min=3631 min_rank=7 imbalance_ratio=1.115479 max_load_reduction=28199 max_load_reduction_pct=0.860565 moved_to_echo=15868 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 0, 0, 396, 62, 15, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 44, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 25, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 2, 0, 4081, 0, 0, 0, 0, 10, 0, 0, 0, 3631, 0, 0, 0] rerouted_tokens_per_ep_rank=[4555, 4096, 4106, 4142, 4111, 4034, 4083, 3641] avg=4096.0000 max=4555 max_rank=0 min=3641 min_rank=7 imbalance_ratio=1.112061 max_load_reduction=28213 max_load_reduction_pct=0.860992 moved_to_echo=15869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1532, 14, 0, 392, 63, 15, 3, 4032, 12, 7, 81, 0, 0, 0, 0, 4002, 6, 1, 0, 0, 0, 0, 0, 30, 0, 3718, 0, 0, 0, 0, 0, 28, 0, 0, 15, 3916, 31, 0, 0, 73, 151, 4005, 0, 0, 0, 0, 0, 1, 156, 0, 3962, 0, 0, 0, 0, 2757, 11, 139, 28, 3587, 0, 0, 0] rerouted_tokens_per_ep_rank=[2019, 4132, 4009, 3748, 3990, 4229, 4119, 6522] avg=4096.0000 max=6522 max_rank=7 min=2019 min_rank=0 imbalance_ratio=1.592285 max_load_reduction=26246 max_load_reduction_pct=0.800964 moved_to_echo=18860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 395, 62, 15, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 3998, 10, 54, 0, 0, 0, 0, 0, 45, 0, 4089, 0, 0, 0, 0, 0, 0, 2, 4, 1, 4086, 25, 0, 0, 0, 17, 4032, 0, 0, 0, 0, 0, 0, 31, 12, 4079, 0, 0, 0, 0, 0, 2, 0, 0, 3629, 0, 0, 0] rerouted_tokens_per_ep_rank=[4568, 4084, 4062, 4134, 4118, 4049, 4122, 3631] avg=4096.0000 max=4568 max_rank=0 min=3631 min_rank=7 imbalance_ratio=1.115234 max_load_reduction=28200 max_load_reduction_pct=0.860596 moved_to_echo=15920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 29056, 128, 128, 3200, 512, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 512, 32256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3712, 128, 256, 128, 29056, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[384, 128, 32384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32512, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-08-30 04:55:23] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 673.5 | throughput per GPU (TFLOP/s/GPU): 495.0 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.318632E+01 | loss scale: 1.0 | grad norm: 34.255 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[5, 4045, 2063, 9, 2181, 0, 739, 8, 2406, 188, 812, 2034, 1287, 510, 11, 433, 1024, 1093, 831, 4096, 1084, 1364, 761, 45, 1, 719, 214, 846, 963, 34, 2962, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[8, 3991, 1868, 7, 1589, 1, 780, 12, 2847, 115, 589, 2291, 1667, 159, 41, 428, 1019, 782, 955, 4086, 1666, 1478, 796, 84, 0, 632, 322, 710, 1032, 77, 2716, 20] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[11, 4030, 1475, 15, 1437, 2, 752, 21, 2815, 239, 1140, 1585, 956, 508, 35, 554, 949, 674, 946, 4094, 1682, 1418, 536, 51, 0, 645, 205, 505, 2088, 103, 3295, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[7, 4080, 1842, 0, 1528, 0, 1056, 3, 3049, 115, 1090, 1967, 617, 576, 12, 548, 1110, 1086, 697, 4096, 1650, 1156, 846, 11, 0, 720, 234, 512, 1134, 35, 2988, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[15, 4040, 1596, 6, 1535, 4, 800, 12, 3136, 172, 975, 1938, 1011, 498, 7, 440, 1144, 980, 874, 4093, 1593, 1325, 567, 46, 0, 776, 211, 486, 1276, 34, 3177, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[9, 4027, 1689, 12, 1317, 0, 676, 9, 2801, 307, 1148, 1814, 1219, 734, 13, 506, 682, 1021, 1069, 4089, 1459, 1514, 633, 51, 1, 431, 209, 737, 1528, 36, 3021, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[16, 4067, 2117, 8, 1902, 10, 1115, 25, 2909, 106, 627, 2034, 1194, 513, 6, 554, 553, 1402, 557, 4086, 1473, 1259, 735, 21, 2, 686, 184, 635, 1099, 31, 2839, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[9, 4034, 1536, 23, 1900, 2, 717, 15, 2874, 160, 1087, 2163, 1186, 406, 26, 520, 853, 578, 871, 4093, 1598, 1355, 724, 49, 0, 662, 206, 1058, 1058, 29, 2974, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 2307, 2063, 9, 0, 0, 0, 0, 2181, 0, 739, 8, 1496, 70, 0, 0, 851, 188, 812, 2034, 0, 0, 0, 0, 1287, 510, 11, 433, 1485, 187, 0, 0, 1024, 1093, 831, 1379, 0, 0, 0, 0, 1084, 1364, 761, 45, 242, 222, 0, 0, 1, 719, 214, 846, 2530, 0, 0, 0, 963, 34, 2740, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4384, 4494, 3885, 3913, 4327, 3718, 4310, 3737] avg=4096.0000 max=4494 max_rank=1 min=3718 min_rank=5 imbalance_ratio=1.097168 max_load_reduction=28274 max_load_reduction_pct=0.862854 moved_to_echo=16092 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[8, 2276, 1868, 7, 0, 0, 0, 0, 1589, 1, 780, 12, 1476, 83, 0, 0, 1007, 115, 589, 2291, 0, 0, 0, 0, 1667, 159, 41, 428, 1757, 186, 0, 0, 1019, 782, 955, 1377, 0, 0, 0, 0, 1666, 1478, 796, 84, 239, 203, 0, 0, 0, 632, 322, 710, 2523, 0, 0, 0, 1032, 77, 2513, 20, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 3941, 4002, 4238, 4133, 4466, 4187, 3642] avg=4096.0000 max=4466 max_rank=5 min=3642 min_rank=7 imbalance_ratio=1.090332 max_load_reduction=28302 max_load_reduction_pct=0.863708 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[15, 2304, 1596, 6, 0, 0, 0, 0, 1535, 4, 800, 12, 1494, 91, 0, 0, 1110, 172, 975, 1938, 0, 0, 0, 0, 1011, 498, 7, 440, 1935, 187, 0, 0, 1144, 980, 874, 1378, 0, 0, 0, 0, 1593, 1325, 567, 46, 242, 238, 0, 0, 0, 776, 211, 486, 2528, 0, 0, 0, 1276, 34, 2939, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3921, 3936, 4195, 4078, 4376, 4011, 4001, 4250] avg=4096.0000 max=4376 max_rank=4 min=3921 min_rank=0 imbalance_ratio=1.068359 max_load_reduction=28392 max_load_reduction_pct=0.866455 moved_to_echo=16638 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 2319, 2117, 8, 0, 0, 0, 0, 1902, 10, 1115, 25, 1504, 85, 0, 0, 1029, 106, 627, 2034, 0, 0, 0, 0, 1194, 513, 6, 554, 1795, 186, 0, 0, 553, 1402, 557, 1377, 0, 0, 0, 0, 1473, 1259, 735, 21, 244, 212, 0, 0, 2, 686, 184, 635, 2523, 0, 0, 0, 1099, 31, 2627, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4460, 4641, 3796, 4248, 3889, 3944, 4030, 3760] avg=4096.0000 max=4641 max_rank=1 min=3760 min_rank=7 imbalance_ratio=1.133057 max_load_reduction=28127 max_load_reduction_pct=0.858368 moved_to_echo=15623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 2327, 1842, 0, 0, 0, 0, 0, 1528, 0, 1056, 3, 1509, 89, 0, 0, 1078, 115, 1090, 1967, 0, 0, 0, 0, 617, 576, 12, 548, 1882, 187, 0, 0, 1110, 1086, 697, 1379, 0, 0, 0, 0, 1650, 1156, 846, 11, 244, 224, 0, 0, 0, 720, 234, 512, 2530, 0, 0, 0, 1134, 35, 2764, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4176, 4185, 4250, 3822, 4272, 4131, 3996, 3936] avg=4096.0000 max=4272 max_rank=4 min=3822 min_rank=3 imbalance_ratio=1.042969 max_load_reduction=28496 max_load_reduction_pct=0.869629 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 2299, 1475, 15, 0, 0, 0, 0, 1437, 2, 752, 21, 1490, 82, 0, 0, 996, 239, 1140, 1585, 0, 0, 0, 0, 956, 508, 35, 554, 1737, 187, 0, 0, 949, 674, 946, 1379, 0, 0, 0, 0, 1682, 1418, 536, 51, 241, 247, 0, 0, 0, 645, 205, 505, 2528, 0, 0, 0, 2088, 103, 3048, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3800, 3784, 3960, 3977, 3948, 4175, 3883, 5241] avg=4096.0000 max=5241 max_rank=7 min=3784 min_rank=1 imbalance_ratio=1.279541 max_load_reduction=27527 max_load_reduction_pct=0.840057 moved_to_echo=17247 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[9, 2290, 1689, 12, 0, 0, 0, 0, 1317, 0, 676, 9, 1491, 84, 0, 0, 985, 307, 1148, 1814, 0, 0, 0, 0, 1219, 734, 13, 506, 1732, 189, 0, 0, 682, 1021, 1069, 1371, 0, 0, 0, 0, 1459, 1514, 633, 51, 246, 229, 0, 0, 1, 431, 209, 737, 2529, 0, 0, 0, 1528, 36, 2792, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4000, 3577, 4254, 4393, 4143, 4132, 3907, 4362] avg=4096.0000 max=4393 max_rank=3 min=3577 min_rank=1 imbalance_ratio=1.072510 max_load_reduction=28375 max_load_reduction_pct=0.865936 moved_to_echo=16544 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[9, 2300, 1536, 23, 0, 0, 0, 0, 1900, 2, 717, 15, 1492, 84, 0, 0, 1016, 160, 1087, 2163, 0, 0, 0, 0, 1186, 406, 26, 520, 1774, 187, 0, 0, 853, 578, 871, 1378, 0, 0, 0, 0, 1598, 1355, 724, 49, 242, 223, 0, 0, 0, 662, 206, 1058, 2528, 0, 0, 0, 1058, 29, 2751, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3868, 4210, 4426, 4099, 3680, 4191, 4454, 3840] avg=4096.0000 max=4454 max_rank=6 min=3680 min_rank=4 imbalance_ratio=1.087402 max_load_reduction=28314 max_load_reduction_pct=0.864075 moved_to_echo=16165 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12288, 10880, 5632, 384, 2048, 1920, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7424, 7680, 6912, 11136, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 1408, 7552, 15872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 18432, 14208, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13440, 128, 6656, 128, 12032, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9216, 3968, 256, 4096, 14208, 1536, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10240, 384, 22272, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 5376, 1792, 5504, 20224, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 740, 4004, 0, 3, 4046, 4019, 3, 78, 0, 2, 5, 3, 4095, 0, 29, 0, 3999, 1, 5, 0, 4082, 1089, 1, 10, 4085, 22, 2433, 0, 9, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 181, 4083, 4, 0, 4051, 4096, 9, 57, 0, 0, 0, 0, 4094, 0, 0, 0, 4075, 0, 0, 0, 4088, 693, 0, 0, 4096, 3, 3236, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 172, 4073, 0, 0, 4090, 4096, 0, 2, 0, 0, 0, 0, 4031, 0, 0, 0, 4096, 0, 0, 0, 4093, 370, 0, 5, 4093, 0, 3647, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[9, 0, 135, 4089, 4, 0, 4086, 4092, 1, 0, 1, 0, 2, 0, 4092, 0, 0, 0, 4086, 0, 2, 0, 3992, 62, 1, 5, 4094, 0, 4011, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3, 1, 868, 4065, 0, 2, 4093, 3968, 2, 0, 0, 0, 0, 9, 4096, 0, 1, 0, 4075, 0, 11, 0, 4087, 129, 0, 3, 4093, 0, 3251, 0, 11, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 182, 4086, 0, 0, 4095, 4094, 1, 2, 2, 0, 0, 0, 4092, 2, 2, 3, 4096, 0, 0, 0, 4089, 1537, 0, 0, 4094, 0, 2388, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 162, 4066, 0, 0, 4093, 4079, 0, 28, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 3929, 218, 0, 0, 4096, 0, 3910, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[6, 0, 599, 4080, 0, 0, 4095, 4082, 5, 3, 0, 0, 0, 0, 4096, 1, 0, 0, 4087, 0, 9, 0, 4046, 660, 0, 0, 4096, 1, 2902, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 740, 3657, 0, 0, 0, 0, 0, 3, 29, 4019, 0, 0, 0, 0, 3, 78, 0, 2, 4017, 19, 0, 0, 5, 3, 4095, 0, 3, 2, 0, 0, 29, 0, 3999, 1, 15, 0, 0, 0, 5, 0, 3526, 1089, 0, 0, 0, 0, 1, 10, 4082, 22, 0, 0, 0, 0, 2433, 0, 9, 3, 537, 330, 0, 0] rerouted_tokens_per_ep_rank=[4399, 4051, 4119, 4108, 4044, 4620, 4115, 3312] avg=4096.0000 max=4620 max_rank=5 min=3312 min_rank=7 imbalance_ratio=1.127930 max_load_reduction=28148 max_load_reduction_pct=0.859009 moved_to_echo=16091 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 181, 3727, 0, 0, 0, 0, 4, 0, 29, 4096, 0, 0, 0, 0, 9, 57, 0, 0, 4022, 19, 0, 0, 0, 0, 4094, 0, 3, 3, 0, 0, 0, 0, 4075, 0, 16, 0, 0, 0, 0, 0, 3531, 693, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 3236, 0, 2, 0, 538, 337, 0, 0] rerouted_tokens_per_ep_rank=[3908, 4129, 4107, 4100, 4091, 4224, 4096, 4113] avg=4096.0000 max=4224 max_rank=5 min=3908 min_rank=0 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16524 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 868, 3713, 0, 0, 0, 0, 0, 2, 29, 3968, 0, 0, 0, 0, 2, 0, 0, 0, 4064, 19, 0, 0, 0, 9, 4096, 0, 3, 2, 0, 0, 1, 0, 4075, 0, 15, 0, 0, 0, 11, 0, 3530, 129, 0, 0, 0, 0, 0, 3, 4090, 0, 0, 0, 0, 0, 3251, 0, 11, 0, 538, 335, 0, 0] rerouted_tokens_per_ep_rank=[4585, 3999, 4085, 4110, 4091, 3670, 4093, 4135] avg=4096.0000 max=4585 max_rank=0 min=3670 min_rank=5 imbalance_ratio=1.119385 max_load_reduction=28183 max_load_reduction_pct=0.860077 moved_to_echo=15989 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 135, 3733, 0, 0, 0, 0, 4, 0, 29, 4092, 0, 0, 0, 0, 1, 0, 1, 0, 4057, 18, 0, 0, 2, 0, 4092, 0, 3, 3, 0, 0, 0, 0, 4086, 0, 16, 0, 0, 0, 2, 0, 3448, 62, 0, 0, 0, 0, 1, 5, 4091, 0, 0, 0, 0, 0, 4011, 0, 4, 0, 526, 337, 0, 0] rerouted_tokens_per_ep_rank=[3877, 4125, 4077, 4100, 4102, 3512, 4097, 4878] avg=4096.0000 max=4878 max_rank=7 min=3512 min_rank=5 imbalance_ratio=1.190918 max_load_reduction=27890 max_load_reduction_pct=0.851135 moved_to_echo=16589 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 599, 3725, 0, 0, 0, 0, 0, 0, 29, 4082, 0, 0, 0, 0, 5, 3, 0, 0, 4066, 18, 0, 0, 0, 0, 4096, 1, 3, 3, 0, 0, 0, 0, 4087, 0, 16, 0, 0, 0, 9, 0, 3495, 660, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 2902, 0, 0, 0, 533, 336, 0, 0] rerouted_tokens_per_ep_rank=[4330, 4111, 4092, 4103, 4103, 4164, 4094, 3771] avg=4096.0000 max=4330 max_rank=0 min=3771 min_rank=7 imbalance_ratio=1.057129 max_load_reduction=28438 max_load_reduction_pct=0.867859 moved_to_echo=16132 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 162, 3714, 0, 0, 0, 0, 0, 0, 29, 4079, 0, 0, 0, 0, 0, 28, 0, 0, 4064, 18, 0, 0, 0, 0, 4096, 0, 3, 2, 0, 0, 0, 0, 4090, 0, 15, 0, 0, 0, 0, 0, 3394, 218, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3910, 0, 0, 0, 517, 335, 0, 0] rerouted_tokens_per_ep_rank=[3877, 4108, 4110, 4101, 4105, 3612, 4093, 4762] avg=4096.0000 max=4762 max_rank=7 min=3612 min_rank=5 imbalance_ratio=1.162598 max_load_reduction=28006 max_load_reduction_pct=0.854675 moved_to_echo=16572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 172, 3718, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 4061, 19, 0, 0, 0, 0, 4031, 0, 3, 3, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 3535, 370, 0, 0, 0, 0, 0, 5, 4090, 0, 0, 0, 0, 0, 3647, 0, 0, 0, 539, 336, 0, 0] rerouted_tokens_per_ep_rank=[3890, 4125, 4082, 4037, 4112, 3905, 4095, 4522] avg=4096.0000 max=4522 max_rank=7 min=3890 min_rank=0 imbalance_ratio=1.104004 max_load_reduction=28246 max_load_reduction_pct=0.862000 moved_to_echo=16634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 182, 3719, 0, 0, 0, 0, 0, 0, 26, 4094, 0, 0, 0, 0, 1, 2, 2, 0, 4069, 22, 0, 0, 0, 0, 4092, 2, 9, 6, 0, 0, 2, 3, 4096, 0, 19, 0, 0, 0, 0, 0, 3524, 1537, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 2388, 0, 0, 2, 543, 342, 0, 0] rerouted_tokens_per_ep_rank=[3902, 4120, 4096, 4109, 4120, 5061, 4085, 3275] avg=4096.0000 max=5061 max_rank=5 min=3275 min_rank=7 imbalance_ratio=1.235596 max_load_reduction=27707 max_load_reduction_pct=0.845551 moved_to_echo=16541 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[25856, 0, 128, 128, 4352, 2688, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 3072, 29824, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 28032, 4864, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 256, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 256, 128, 128, 32512, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 4035, 0, 1964, 4096, 0, 0, 0, 0, 4095, 5, 1865, 0, 0, 3885, 0, 4073, 0, 4096, 1, 550, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 1975, 4096, 1, 0, 0, 0, 4044, 24, 2067, 0, 0, 3536, 0, 4024, 0, 4096, 0, 718, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 0, 3, 4095, 0, 2, 0, 4087, 4, 4093, 4096, 5, 5, 0, 0, 4095, 0, 269, 0, 0, 3838, 1, 3880, 0, 4095, 0, 197, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[41, 0, 1, 4, 0, 16, 14, 4082, 0, 0, 14, 3971, 1, 4068, 4088, 0, 14, 3, 0, 3879, 157, 221, 0, 15, 4079, 2, 3666, 3, 4094, 1, 334, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 4, 4096, 4, 4, 0, 4060, 0, 3030, 4094, 13, 0, 0, 0, 4092, 0, 1880, 2, 0, 3546, 0, 3453, 2, 4079, 0, 407, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 3994, 0, 4033, 4086, 0, 0, 0, 0, 4091, 50, 305, 0, 0, 3734, 0, 4028, 0, 4096, 0, 249, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3369, 0, 3421, 4096, 11, 0, 0, 0, 4089, 2, 2140, 0, 0, 2149, 0, 3897, 0, 4095, 0, 1403, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 6, 0, 4, 4090, 9, 0, 0, 4045, 0, 3428, 4090, 0, 0, 2, 0, 4051, 8, 2378, 2, 5, 2537, 1, 3686, 0, 4089, 0, 333, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3255, 887, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 6, 0, 0, 4035, 134, 0, 0, 0, 0, 1964, 841, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 35, 3, 0, 0, 5, 1865, 0, 0, 2462, 351, 0, 0, 3885, 0, 724, 0, 0, 0, 0, 0, 3576, 1, 550, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4093, 4175, 2805, 4133, 4683, 4609, 4128] avg=4096.0000 max=4683 max_rank=5 min=2805 min_rank=3 imbalance_ratio=1.143311 max_load_reduction=28085 max_load_reduction_pct=0.857086 moved_to_echo=17553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3255, 876, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4091, 134, 0, 0, 0, 0, 1975, 841, 1, 0, 0, 0, 0, 0, 0, 0, 4044, 35, 3, 0, 0, 24, 2067, 0, 0, 2432, 351, 0, 0, 3536, 0, 716, 0, 0, 0, 0, 0, 3576, 0, 718, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4093, 4225, 2817, 4082, 4874, 4252, 4294] avg=4096.0000 max=4874 max_rank=5 min=2817 min_rank=3 imbalance_ratio=1.189941 max_load_reduction=27894 max_load_reduction_pct=0.851257 moved_to_echo=17502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 3255, 845, 0, 0, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 2, 0, 4087, 134, 0, 0, 0, 4, 4093, 841, 5, 0, 0, 0, 0, 5, 0, 0, 4095, 35, 3, 0, 0, 0, 269, 0, 0, 2345, 351, 0, 0, 3838, 1, 690, 0, 0, 0, 0, 0, 3575, 0, 197, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4095, 4223, 4943, 4138, 2965, 4529, 3773] avg=4096.0000 max=4943 max_rank=3 min=2965 min_rank=5 imbalance_ratio=1.206787 max_load_reduction=27825 max_load_reduction_pct=0.849152 moved_to_echo=15405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3253, 752, 0, 0, 0, 0, 4, 4093, 0, 0, 0, 0, 4, 4, 0, 4060, 133, 0, 0, 0, 0, 3030, 841, 13, 0, 0, 0, 0, 0, 0, 0, 4092, 35, 3, 0, 0, 0, 1880, 2, 0, 2087, 350, 0, 0, 3546, 0, 614, 2, 0, 0, 0, 0, 3561, 0, 407, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4097, 4201, 3884, 4130, 4319, 4162, 3968] avg=4096.0000 max=4319 max_rank=5 min=3884 min_rank=3 imbalance_ratio=1.054443 max_load_reduction=28449 max_load_reduction_pct=0.868195 moved_to_echo=16579 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[41, 0, 1, 4, 3248, 798, 0, 0, 0, 16, 14, 4079, 0, 0, 0, 0, 0, 0, 14, 3971, 134, 0, 0, 0, 1, 4068, 840, 0, 0, 0, 0, 0, 14, 3, 0, 3879, 35, 3, 0, 0, 157, 221, 0, 15, 2216, 351, 0, 0, 4079, 2, 652, 3, 0, 0, 0, 0, 3574, 1, 334, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4109, 4119, 4909, 3934, 2960, 4736, 3909] avg=4096.0000 max=4909 max_rank=3 min=2960 min_rank=5 imbalance_ratio=1.198486 max_load_reduction=27859 max_load_reduction_pct=0.850189 moved_to_echo=15539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 3247, 877, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2, 0, 3994, 134, 0, 0, 0, 0, 4033, 839, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 35, 3, 0, 0, 50, 305, 0, 0, 2435, 351, 0, 0, 3734, 0, 716, 0, 0, 0, 0, 0, 3576, 0, 249, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4093, 4130, 4872, 4129, 3141, 4450, 3825] avg=4096.0000 max=4872 max_rank=3 min=3141 min_rank=5 imbalance_ratio=1.189453 max_load_reduction=27896 max_load_reduction_pct=0.851318 moved_to_echo=15545 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 3253, 806, 0, 0, 6, 0, 4, 4085, 0, 0, 0, 0, 9, 0, 0, 4045, 138, 0, 0, 0, 0, 3428, 837, 0, 0, 0, 0, 0, 0, 2, 0, 4051, 37, 5, 0, 0, 8, 2378, 2, 5, 2231, 354, 0, 0, 2537, 1, 649, 0, 0, 0, 0, 0, 3560, 0, 333, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4095, 4192, 4265, 4095, 4978, 3187, 3893] avg=4096.0000 max=4978 max_rank=5 min=3187 min_rank=6 imbalance_ratio=1.215332 max_load_reduction=27790 max_load_reduction_pct=0.848083 moved_to_echo=16153 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3255, 848, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3369, 134, 0, 0, 0, 0, 3421, 841, 11, 0, 0, 0, 0, 0, 0, 0, 4089, 35, 3, 0, 0, 2, 2140, 0, 0, 2355, 351, 0, 0, 2149, 0, 694, 0, 0, 0, 0, 0, 3575, 0, 1403, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4093, 3503, 4273, 4127, 4848, 2843, 4978] avg=4096.0000 max=4978 max_rank=7 min=2843 min_rank=6 imbalance_ratio=1.215332 max_load_reduction=27790 max_load_reduction_pct=0.848083 moved_to_echo=16796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 11136, 128, 128, 18688, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[27392, 128, 5504, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 26112, 6784, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 26112, 6784, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[28672, 128, 4224, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 31744, 1152, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32512, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4088, 11, 0, 0, 13, 4, 0, 4090, 2226, 1, 7, 22, 4076, 0, 18, 4049, 1, 4089, 4092, 19, 1866, 0, 8, 0, 9, 0, 4, 0, 0, 4075, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 1, 4096, 1548, 0, 0, 349, 4096, 0, 0, 4032, 1, 4095, 4095, 0, 2259, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 0, 0, 4, 4094, 2845, 0, 0, 0, 4096, 0, 0, 3987, 3, 4093, 4096, 2, 1276, 0, 0, 0, 0, 73, 0, 0, 1, 4096, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 1548, 0, 0, 4, 4096, 0, 0, 3530, 0, 4096, 4096, 0, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 2802, 0, 0, 11, 4096, 0, 0, 4078, 0, 4094, 4096, 0, 1296, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4090, 0, 13, 0, 0, 0, 0, 4094, 1587, 0, 0, 585, 4092, 0, 0, 3971, 2, 4084, 4096, 0, 2049, 2, 0, 0, 0, 6, 1, 0, 0, 4094, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4085, 0, 0, 6, 1, 2, 0, 4089, 2436, 0, 2, 24, 4090, 0, 0, 3839, 0, 4093, 4093, 3, 1883, 0, 6, 4, 4, 12, 3, 4, 0, 4089, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4043, 0, 0, 0, 0, 0, 3, 4096, 734, 0, 0, 2, 4096, 0, 0, 4087, 0, 4096, 4096, 0, 3419, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 1548, 0, 0, 349, 1991, 15, 0, 0, 149, 0, 0, 4032, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 2259, 0, 0, 0, 1950, 0, 0, 0, 0, 4, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4096, 3903, 4181, 4096, 4209, 4084, 4097] avg=4096.0000 max=4209 max_rank=5 min=3903 min_rank=2 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4088, 11, 0, 0, 6, 0, 0, 0, 13, 4, 0, 4089, 0, 0, 0, 0, 2226, 1, 7, 22, 1981, 15, 0, 0, 148, 0, 18, 4049, 0, 0, 0, 0, 1, 4089, 0, 19, 0, 0, 0, 0, 1866, 0, 8, 0, 1941, 0, 0, 0, 9, 0, 4, 0, 4077, 0, 0, 0, 0, 4075, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4106, 4252, 4215, 4109, 3815, 4090, 4076] avg=4096.0000 max=4252 max_rank=2 min=3815 min_rank=5 imbalance_ratio=1.038086 max_load_reduction=28516 max_load_reduction_pct=0.870239 moved_to_echo=16090 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1548, 0, 0, 4, 1991, 14, 0, 0, 149, 0, 0, 3530, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3110, 0, 0, 0, 1950, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4095, 3557, 3679, 4096, 5060, 4082, 4097] avg=4096.0000 max=5060 max_rank=5 min=3557 min_rank=2 imbalance_ratio=1.235352 max_load_reduction=27708 max_load_reduction_pct=0.845581 moved_to_echo=17335 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 13, 0, 6, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 1587, 0, 0, 585, 1989, 15, 0, 0, 149, 0, 0, 3971, 0, 0, 0, 0, 2, 4084, 0, 0, 0, 0, 0, 0, 2049, 2, 0, 0, 1948, 0, 0, 0, 0, 6, 1, 0, 4081, 0, 0, 0, 0, 4094, 0, 2, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4093, 4176, 4120, 4086, 3999, 4088, 4097] avg=4096.0000 max=4176 max_rank=2 min=3999 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16270 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4, 4093, 0, 0, 0, 0, 2845, 0, 0, 0, 1991, 15, 0, 0, 149, 0, 0, 3987, 0, 0, 0, 0, 3, 4093, 0, 2, 0, 0, 0, 0, 1276, 0, 0, 0, 1950, 0, 0, 0, 0, 73, 0, 0, 4081, 0, 0, 0, 1, 4096, 12, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4851, 4136, 4098, 3226, 4154, 4110] avg=4096.0000 max=4851 max_rank=2 min=3226 min_rank=5 imbalance_ratio=1.184326 max_load_reduction=27917 max_load_reduction_pct=0.851959 moved_to_echo=15588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2802, 0, 0, 11, 1991, 15, 0, 0, 149, 0, 0, 4078, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1296, 0, 0, 0, 1950, 0, 0, 0, 0, 7, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4095, 4819, 4227, 4094, 3246, 4088, 4097] avg=4096.0000 max=4819 max_rank=2 min=3246 min_rank=5 imbalance_ratio=1.176514 max_load_reduction=27949 max_load_reduction_pct=0.852936 moved_to_echo=15525 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 0, 6, 12, 0, 0, 0, 1, 2, 0, 4085, 0, 0, 0, 0, 2436, 0, 2, 24, 1991, 15, 0, 0, 135, 0, 0, 3839, 0, 0, 0, 0, 0, 4090, 0, 3, 0, 0, 0, 0, 1883, 0, 6, 4, 1951, 0, 0, 0, 4, 12, 3, 4, 4078, 0, 0, 0, 0, 4089, 0, 0, 4, 3, 1, 0] rerouted_tokens_per_ep_rank=[4103, 4088, 4468, 3974, 4093, 3844, 4101, 4097] avg=4096.0000 max=4468 max_rank=2 min=3844 min_rank=5 imbalance_ratio=1.090820 max_load_reduction=28300 max_load_reduction_pct=0.863647 moved_to_echo=16135 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4043, 0, 0, 0, 6, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 734, 0, 0, 2, 1991, 15, 0, 0, 149, 0, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3419, 0, 0, 0, 1950, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4098, 2742, 4236, 4096, 5369, 4081, 4097] avg=4096.0000 max=5369 max_rank=5 min=2742 min_rank=2 imbalance_ratio=1.310791 max_load_reduction=27399 max_load_reduction_pct=0.836151 moved_to_echo=17643 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[15744, 128, 128, 1024, 16000, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[17280, 128, 128, 128, 15616, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1280, 0, 128, 31616, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4087, 4096, 0, 0, 0, 4096, 0, 3711, 0, 0, 0, 0, 4095, 0, 0, 0, 352, 0, 4096, 0, 4046, 1, 31, 1, 0, 0, 0, 4096, 60, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4089, 4095, 0, 0, 0, 4096, 0, 4075, 1, 0, 0, 0, 4096, 0, 0, 0, 17, 0, 4095, 0, 3896, 7, 4, 0, 2, 2, 1, 4094, 198, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 4091, 4089, 6, 0, 0, 3891, 0, 3713, 0, 2, 3, 6, 4090, 0, 5, 0, 14, 3, 4092, 11, 4075, 0, 125, 116, 0, 0, 2, 4093, 339, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4089, 4096, 0, 0, 0, 4096, 1, 3000, 0, 0, 0, 0, 4093, 0, 0, 0, 199, 0, 4096, 0, 3798, 9, 125, 96, 0, 0, 0, 4096, 974, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[6, 26, 4038, 4056, 51, 0, 0, 4028, 0, 4060, 0, 26, 12, 3, 4045, 0, 5, 6, 167, 11, 4074, 11, 3998, 0, 0, 6, 5, 0, 20, 4085, 29, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4090, 4096, 1, 0, 0, 4096, 0, 3640, 0, 0, 0, 0, 4096, 0, 0, 0, 102, 0, 4096, 0, 3807, 2, 6, 366, 0, 0, 0, 4096, 274, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4020, 4096, 0, 0, 0, 4095, 0, 3996, 0, 0, 0, 0, 4096, 0, 0, 0, 137, 0, 4096, 0, 4094, 4, 0, 2, 0, 0, 0, 4096, 36, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4043, 4096, 0, 0, 1, 4096, 1, 3814, 0, 0, 0, 0, 4096, 0, 0, 3, 64, 0, 4096, 0, 4096, 9, 0, 223, 0, 0, 0, 4096, 34, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4087, 24, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 3711, 0, 0, 212, 128, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 0, 0, 0, 352, 0, 3966, 0, 0, 0, 114, 0, 4046, 1, 0, 0, 0, 0, 31, 1, 0, 0, 3854, 106, 0, 0, 0, 3854, 60, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4122, 4051, 4099, 4318, 4161, 3992, 3914] avg=4096.0000 max=4318 max_rank=4 min=3914 min_rank=7 imbalance_ratio=1.054199 max_load_reduction=28450 max_load_reduction_pct=0.868225 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 24, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 4075, 1, 0, 212, 128, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 17, 0, 3965, 0, 0, 0, 114, 0, 3896, 7, 0, 0, 0, 0, 4, 0, 2, 2, 3853, 106, 0, 0, 1, 3852, 198, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4122, 4416, 4100, 3982, 4017, 3967, 4051] avg=4096.0000 max=4416 max_rank=2 min=3967 min_rank=6 imbalance_ratio=1.078125 max_load_reduction=28352 max_load_reduction_pct=0.865234 moved_to_echo=16017 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 26, 4038, 24, 0, 0, 0, 0, 51, 0, 0, 4028, 26, 0, 0, 0, 0, 4060, 0, 26, 211, 128, 0, 0, 12, 3, 4045, 0, 4, 0, 0, 0, 5, 6, 167, 11, 3927, 0, 0, 0, 113, 11, 3998, 0, 0, 0, 0, 0, 0, 6, 5, 0, 3833, 105, 0, 0, 20, 3844, 29, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4105, 4425, 4064, 4116, 4122, 3949, 3893] avg=4096.0000 max=4425 max_rank=2 min=3893 min_rank=7 imbalance_ratio=1.080322 max_load_reduction=28343 max_load_reduction_pct=0.864960 moved_to_echo=16080 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 24, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 1, 3000, 0, 0, 212, 128, 0, 0, 0, 0, 4093, 0, 4, 0, 0, 0, 0, 0, 199, 0, 3966, 0, 0, 0, 114, 0, 3798, 9, 0, 0, 0, 0, 125, 96, 0, 0, 3854, 106, 0, 0, 0, 3854, 974, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4122, 3341, 4097, 4165, 3921, 4181, 4828] avg=4096.0000 max=4828 max_rank=7 min=3341 min_rank=2 imbalance_ratio=1.178711 max_load_reduction=27940 max_load_reduction_pct=0.852661 moved_to_echo=17095 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 24, 0, 0, 0, 0, 1, 0, 0, 4096, 26, 0, 0, 0, 0, 3640, 0, 0, 212, 128, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 102, 0, 3966, 0, 0, 0, 114, 0, 3807, 2, 0, 0, 0, 0, 6, 366, 0, 0, 3854, 106, 0, 0, 0, 3854, 274, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4123, 3980, 4100, 4068, 3923, 4332, 4128] avg=4096.0000 max=4332 max_rank=6 min=3923 min_rank=5 imbalance_ratio=1.057617 max_load_reduction=28436 max_load_reduction_pct=0.867798 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4091, 19, 0, 0, 0, 0, 6, 0, 0, 3891, 33, 0, 0, 0, 0, 3713, 0, 2, 216, 133, 0, 0, 3, 6, 4090, 0, 9, 0, 0, 0, 5, 0, 14, 3, 3961, 0, 0, 0, 107, 11, 4075, 0, 0, 0, 0, 0, 125, 116, 0, 0, 3852, 109, 0, 0, 2, 3835, 339, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 3930, 4064, 4108, 3983, 4193, 4202, 4176] avg=4096.0000 max=4202 max_rank=6 min=3930 min_rank=1 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4043, 24, 0, 0, 0, 0, 0, 0, 1, 4096, 26, 0, 0, 0, 1, 3814, 0, 0, 212, 128, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 3, 64, 0, 3966, 0, 0, 0, 114, 0, 4096, 9, 0, 0, 0, 0, 0, 223, 0, 0, 3854, 106, 0, 0, 0, 3854, 34, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4123, 4155, 4100, 4033, 4219, 4183, 3888] avg=4096.0000 max=4219 max_rank=5 min=3888 min_rank=7 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4020, 24, 0, 0, 0, 0, 0, 0, 0, 4095, 26, 0, 0, 0, 0, 3996, 0, 0, 212, 128, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 137, 0, 3966, 0, 0, 0, 114, 0, 4094, 4, 0, 0, 0, 0, 0, 2, 0, 0, 3854, 106, 0, 0, 0, 3854, 36, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4121, 4336, 4100, 4103, 4212, 3962, 3890] avg=4096.0000 max=4336 max_rank=2 min=3890 min_rank=7 imbalance_ratio=1.058594 max_load_reduction=28432 max_load_reduction_pct=0.867676 moved_to_echo=16167 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 30848, 2048, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 1152, 128, 31744, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 128, 31872, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 896, 128, 128, 30848, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 30080, 128, 128, 1792, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32512, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4094, 6, 0, 6, 0, 0, 1, 2, 0, 0, 0, 4093, 2, 0, 26, 0, 4067, 4096, 4086, 2, 1, 0, 0, 2, 4095, 0, 0, 0, 4094, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4083, 4089, 4093, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 4061, 4089, 0, 0, 0, 0, 36, 4096, 0, 7, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4073, 0, 2, 244, 8, 20, 0, 0, 23, 60, 0, 4075, 0, 2, 3, 0, 3943, 3753, 4070, 5, 28, 2, 6, 211, 4043, 15, 0, 15, 4075, 17, 4066, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 4049, 4041, 4096, 0, 0, 0, 0, 58, 4096, 0, 34, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 62, 0, 4053, 4025, 4096, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4021, 4096, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4091, 0, 8, 82, 3, 7, 3, 3, 2, 3, 0, 4079, 0, 0, 11, 0, 4020, 4013, 4093, 1, 2, 20, 3, 34, 4090, 0, 7, 5, 4092, 3, 4088, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4042, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 2, 0, 4082, 2, 0, 0, 75, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 54, 11, 5, 4096, 0, 19, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4093, 4091, 4086, 4164, 4078, 4115, 4097] avg=4096.0000 max=4164 max_rank=4 min=4044 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4040, 6, 0, 6, 0, 0, 0, 0, 0, 0, 1, 2, 4091, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 2, 0, 26, 0, 4075, 2, 0, 0, 74, 4096, 0, 2, 0, 0, 0, 0, 1, 0, 0, 2, 3993, 54, 11, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4094, 4088, 4105, 4172, 4066, 4095, 4096] avg=4096.0000 max=4172 max_rank=4 min=4052 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4020, 0, 2, 244, 0, 0, 0, 0, 8, 20, 0, 0, 4072, 0, 0, 0, 23, 60, 0, 4070, 0, 0, 0, 0, 0, 2, 3, 0, 4059, 2, 0, 0, 72, 3753, 0, 5, 0, 0, 0, 0, 28, 2, 6, 211, 3871, 53, 11, 5, 4043, 15, 0, 15, 0, 0, 0, 0, 1, 17, 4066, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4266, 4100, 4153, 4066, 3830, 4187, 4073, 4093] avg=4096.0000 max=4266 max_rank=0 min=3830 min_rank=4 imbalance_ratio=1.041504 max_load_reduction=28502 max_load_reduction_pct=0.869812 moved_to_echo=16183 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4042, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3, 0, 0, 75, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4017, 54, 11, 5, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4095, 4091, 4081, 4136, 4123, 4103, 4096] avg=4096.0000 max=4136 max_rank=4 min=4043 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4042, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 2, 0, 0, 73, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3948, 54, 11, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4093, 4091, 4087, 4169, 4019, 4096, 4097] avg=4096.0000 max=4169 max_rank=4 min=4019 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 10, 0, 4085, 2, 0, 0, 74, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 3975, 54, 11, 5, 4096, 0, 34, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4093, 4091, 4097, 4115, 4103, 4130, 4097] avg=4096.0000 max=4130 max_rank=6 min=4042 min_rank=0 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4036, 0, 8, 82, 0, 0, 0, 0, 3, 7, 3, 3, 4090, 0, 0, 0, 2, 3, 0, 4067, 0, 0, 0, 0, 0, 0, 11, 0, 4083, 2, 0, 0, 69, 4013, 0, 1, 0, 0, 0, 0, 2, 20, 3, 34, 3951, 55, 10, 12, 4090, 0, 7, 5, 0, 0, 0, 0, 0, 3, 4088, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4106, 4072, 4096, 4083, 4087, 4102, 4096] avg=4096.0000 max=4126 max_rank=0 min=4072 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4042, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 62, 0, 4086, 2, 0, 0, 74, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3979, 54, 10, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4094, 4091, 4150, 4099, 4063, 4096, 4096] avg=4096.0000 max=4150 max_rank=3 min=4063 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32726, 32810, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32726, 32810, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32726, 32810, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32726, 32810, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32726, 32810, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32726, 32810, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32726, 32810, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32726, 32810, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32384, 128, 128, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 0, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32726 actual_valid_recv=33280 observed_loss=-554 tokens_per_expert=[128, 128, 128, 384, 31744, 512, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[640, 32256, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32810 actual_valid_recv=33152 observed_loss=-342 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4063, 0, 11, 0, 0, 0, 10, 4074, 2, 119, 13, 3971, 0, 0, 4072, 0, 13, 0, 0, 4071, 23, 19, 16, 0, 4084, 3, 18, 4069, 0, 21, 4068, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 2, 4093, 0, 0, 4, 2, 4096, 0, 0, 0, 0, 4094, 2, 0, 4094, 0, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 1, 5, 0, 4094, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 4096, 410, 23, 0, 4087, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 3670, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 4096, 0, 0, 4095, 0, 0, 22, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4067, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4091, 2, 0, 1, 0, 0, 2, 4092, 13, 0, 0, 4091, 0, 0, 4088, 0, 5, 0, 0, 4082, 5, 13, 5, 0, 4093, 1, 6, 4091, 0, 4, 4081, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 2, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 1, 5, 0, 4038, 0, 0, 0, 0, 1, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 49, 7, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4097, 4044, 4101, 4095, 4088, 4097, 4147] avg=4096.0000 max=4147 max_rank=7 min=4044 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 49, 7, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4097, 4044, 4099, 4095, 4088, 4097, 4149] avg=4096.0000 max=4149 max_rank=7 min=4044 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 2, 4093, 0, 4, 0, 0, 0, 0, 4, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 49, 8, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4095, 4040, 4099, 4101, 4086, 4096, 4153] avg=4096.0000 max=4153 max_rank=7 min=4040 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4063, 0, 11, 0, 3, 0, 0, 0, 0, 0, 10, 4074, 0, 0, 0, 0, 2, 119, 13, 3916, 0, 0, 0, 0, 0, 0, 4072, 0, 4, 0, 0, 0, 13, 0, 0, 4071, 0, 0, 0, 0, 23, 19, 16, 0, 4076, 0, 0, 0, 1, 3, 18, 4069, 0, 0, 0, 0, 0, 21, 4068, 28, 48, 7, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4084, 4050, 4076, 4084, 4134, 4091, 4172] avg=4096.0000 max=4172 max_rank=7 min=4050 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 8, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 0, 0, 22, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4067, 0, 49, 7, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4097, 4048, 4099, 4117, 4088, 4097, 4123] avg=4096.0000 max=4123 max_rank=7 min=4048 min_rank=2 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 4096, 1, 0, 0, 0, 410, 23, 0, 4031, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3670, 0, 49, 8, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4100, 4464, 4100, 4095, 4087, 4096, 3727] avg=4096.0000 max=4464 max_rank=2 min=3727 min_rank=7 imbalance_ratio=1.089844 max_load_reduction=28304 max_load_reduction_pct=0.863770 moved_to_echo=16005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4091, 2, 0, 1, 4, 0, 0, 0, 0, 0, 2, 4092, 2, 3, 0, 0, 13, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 4088, 0, 6, 0, 0, 0, 5, 0, 0, 4080, 0, 0, 0, 0, 5, 13, 5, 0, 4086, 0, 0, 0, 0, 1, 6, 4091, 0, 0, 0, 0, 0, 4, 4081, 2, 54, 7, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4099, 4037, 4094, 4085, 4109, 4098, 4148] avg=4096.0000 max=4148 max_rank=7 min=4037 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 4096, 1, 0, 0, 0, 1, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 49, 8, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4099, 4041, 4100, 4096, 4088, 4096, 4149] avg=4096.0000 max=4149 max_rank=7 min=4041 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 32384, 128, 512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 256, 128, 32256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4073, 4, 26, 3, 62, 0, 0, 25, 0, 5, 27, 33, 4070, 2, 6, 0, 0, 4058, 20, 0, 0, 4057, 4068, 4027, 16, 0, 1111, 18, 0, 4069, 0, 2988] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 4, 4096, 0, 0, 5, 4096, 4096, 4071, 0, 0, 1769, 0, 0, 4096, 0, 2320] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 4096, 1, 0, 0, 1, 4094, 0, 2, 2, 4094, 4094, 4077, 0, 0, 794, 0, 1, 4093, 0, 3322] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0, 1728, 0, 0, 4096, 0, 2368] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 4, 0, 4096, 4096, 3872, 0, 0, 323, 0, 0, 4096, 3, 3989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 4096, 3815, 0, 0, 873, 0, 0, 4096, 0, 3500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 4096, 0, 0, 0, 4096, 4096, 3953, 0, 0, 203, 0, 0, 4096, 0, 4027] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4086, 4, 9, 0, 7, 0, 0, 3, 0, 0, 5, 0, 4093, 2, 6, 0, 0, 4087, 4, 0, 1, 4091, 4090, 4063, 2, 0, 851, 5, 0, 4092, 0, 3267] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3995, 86, 5, 0, 2, 0, 0, 0, 4089, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 2, 0, 0, 0, 0, 2, 99, 0, 4077, 0, 0, 0, 0, 0, 0, 794, 0, 3133, 1, 0, 0, 1, 874, 0, 3322, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4089, 4091, 4096, 4097, 4178, 3928, 4197] avg=4096.0000 max=4197 max_rank=7 min=3928 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3997, 86, 5, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 99, 0, 3953, 0, 0, 0, 0, 0, 0, 203, 0, 3135, 1, 0, 0, 0, 875, 0, 4027, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4089, 4091, 4095, 4104, 4052, 3339, 4902] avg=4096.0000 max=4902 max_rank=7 min=3339 min_rank=6 imbalance_ratio=1.196777 max_load_reduction=27866 max_load_reduction_pct=0.850403 moved_to_echo=16397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4073, 4, 26, 3, 0, 0, 0, 0, 62, 0, 0, 25, 3959, 85, 4, 0, 0, 5, 27, 33, 4064, 0, 0, 0, 4069, 2, 6, 0, 0, 0, 0, 0, 0, 4058, 20, 0, 0, 0, 0, 0, 0, 98, 0, 4027, 0, 0, 0, 0, 16, 0, 1111, 18, 3115, 1, 0, 0, 0, 869, 0, 2988, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4135, 4129, 4077, 4078, 4125, 4261, 3857] avg=4096.0000 max=4261 max_rank=6 min=3857 min_rank=7 imbalance_ratio=1.040283 max_load_reduction=28507 max_load_reduction_pct=0.869965 moved_to_echo=16321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3997, 86, 5, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 99, 0, 3815, 0, 0, 0, 0, 0, 0, 873, 0, 3135, 1, 0, 0, 0, 875, 0, 3500, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4091, 4091, 4095, 4097, 3914, 4009, 4375] avg=4096.0000 max=4375 max_rank=7 min=3914 min_rank=5 imbalance_ratio=1.068115 max_load_reduction=28393 max_load_reduction_pct=0.866486 moved_to_echo=16395 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 86, 5, 0, 0, 4, 0, 0, 4091, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 99, 0, 3872, 0, 0, 0, 0, 0, 0, 323, 0, 3135, 1, 0, 0, 0, 875, 3, 3989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4088, 4095, 4092, 4100, 3971, 3459, 4867] avg=4096.0000 max=4867 max_rank=7 min=3459 min_rank=6 imbalance_ratio=1.188232 max_load_reduction=27901 max_load_reduction_pct=0.851471 moved_to_echo=16397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 86, 5, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 99, 0, 4094, 0, 0, 0, 0, 0, 0, 1728, 0, 3135, 1, 0, 0, 0, 875, 0, 2368, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4088, 4091, 4097, 4096, 4193, 4864, 3243] avg=4096.0000 max=4864 max_rank=6 min=3243 min_rank=7 imbalance_ratio=1.187500 max_load_reduction=27904 max_load_reduction_pct=0.851562 moved_to_echo=16396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4077, 4, 9, 0, 0, 0, 0, 0, 7, 0, 0, 3, 3995, 89, 5, 0, 0, 0, 5, 0, 4085, 0, 0, 0, 4090, 2, 6, 0, 0, 0, 0, 0, 0, 4087, 4, 0, 5, 0, 0, 0, 1, 96, 0, 4063, 0, 0, 0, 0, 2, 0, 851, 5, 3138, 3, 4, 0, 0, 865, 0, 3267, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4099, 4090, 4098, 4096, 4160, 4003, 4132] avg=4096.0000 max=4160 max_rank=5 min=4003 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 86, 6, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4095, 0, 0, 23, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 5, 99, 0, 4071, 0, 0, 0, 0, 0, 0, 1769, 0, 3135, 1, 0, 0, 0, 875, 0, 2320, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4089, 4090, 4118, 4100, 4175, 4905, 3195] avg=4096.0000 max=4905 max_rank=6 min=3195 min_rank=7 imbalance_ratio=1.197510 max_load_reduction=27863 max_load_reduction_pct=0.850311 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 7680, 128, 25088, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7040, 128, 25856, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 896, 0, 32000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32000, 768, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 214, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3883, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4072, 45, 25, 21, 18, 0, 14, 0, 4062, 140, 23, 0, 3987, 6, 4075, 0, 34, 39, 4009, 4029, 17, 0, 4062, 0, 23, 4067, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 20, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4079, 0, 4096, 0, 0, 0, 4093, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 129, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3967, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 260, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3847, 0, 4094, 1, 0, 0, 4096, 4096, 0, 0, 4082, 5, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 290, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3806, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 786, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3310, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 117, 4090, 1, 4, 5, 6, 0, 3, 0, 4089, 166, 5, 0, 3831, 0, 4089, 0, 10, 0, 4090, 4066, 2, 0, 4091, 0, 3, 4091, 0, 7, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 214, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3883, 215, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4092, 4099, 4098, 4094, 4101, 4097, 4099] avg=4096.0000 max=4101 max_rank=5 min=4088 min_rank=0 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3852, 0, 0, 0, 0, 45, 25, 21, 18, 4004, 1, 0, 0, 0, 14, 0, 4062, 2, 0, 0, 0, 140, 23, 0, 3987, 214, 0, 0, 0, 6, 4073, 0, 34, 0, 0, 0, 0, 39, 5, 4029, 17, 0, 0, 0, 0, 0, 4062, 0, 23, 2, 0, 0, 0, 4067, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[3852, 4114, 4078, 4364, 4113, 4090, 4087, 4070] avg=4096.0000 max=4364 max_rank=3 min=3852 min_rank=0 imbalance_ratio=1.065430 max_load_reduction=28404 max_load_reduction_pct=0.866821 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 20, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4079, 215, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[3894, 4089, 4099, 4294, 4094, 4101, 4098, 4099] avg=4096.0000 max=4294 max_rank=3 min=3894 min_rank=0 imbalance_ratio=1.048340 max_load_reduction=28474 max_load_reduction_pct=0.868958 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 260, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3847, 215, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4082, 5, 0, 2, 0, 0, 0, 4095, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4092, 4099, 4062, 4093, 4101, 4089, 4098] avg=4096.0000 max=4134 max_rank=0 min=4062 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 129, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3967, 215, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4003, 4092, 4099, 4182, 4094, 4101, 4098, 4099] avg=4096.0000 max=4182 max_rank=3 min=4003 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 290, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3806, 215, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4092, 4099, 4021, 4094, 4101, 4098, 4099] avg=4096.0000 max=4164 max_rank=0 min=4021 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 117, 3856, 0, 0, 0, 0, 1, 4, 5, 6, 4086, 3, 0, 0, 0, 3, 0, 4089, 4, 0, 0, 0, 166, 5, 0, 3831, 220, 0, 0, 0, 0, 4082, 0, 10, 0, 0, 0, 0, 0, 4, 4066, 2, 0, 0, 0, 0, 0, 4091, 0, 3, 7, 2, 0, 0, 4091, 0, 7, 2, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[3973, 4105, 4096, 4222, 4092, 4072, 4103, 4105] avg=4096.0000 max=4222 max_rank=3 min=3973 min_rank=0 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 786, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3310, 215, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4660, 4092, 4099, 3525, 4094, 4101, 4098, 4099] avg=4096.0000 max=4660 max_rank=0 min=3525 min_rank=3 imbalance_ratio=1.137695 max_load_reduction=28108 max_load_reduction_pct=0.857788 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 1920, 30976, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 128, 0, 30720, 1792, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[14, 4055, 0, 21, 0, 3988, 104, 4070, 0, 42, 0, 25, 4056, 7, 23, 11, 40, 4054, 6, 0, 41, 0, 0, 10, 4038, 9, 8, 0, 4069, 0, 4077, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[3, 4091, 3, 3, 0, 4052, 42, 4090, 0, 8, 0, 5, 4088, 3, 3, 0, 21, 4067, 1, 0, 8, 0, 0, 4, 4087, 0, 4, 0, 4090, 0, 4092, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 6, 0, 2, 4094, 2, 4094, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4091, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 7, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4100, 4089, 4096, 4096, 4090, 4100, 4101] avg=4096.0000 max=4101 max_rank=7 min=4089 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 6, 0, 0, 0, 0, 0, 2, 4094, 2, 4, 0, 0, 0, 0, 3, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 4094, 0, 5, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4102, 4090, 4096, 4093, 4086, 4100, 4100] avg=4096.0000 max=4102 max_rank=1 min=4086 min_rank=5 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4100, 4089, 4096, 4096, 4091, 4100, 4101] avg=4096.0000 max=4101 max_rank=7 min=4089 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[14, 4054, 0, 21, 0, 0, 0, 0, 0, 3988, 104, 4, 0, 0, 0, 0, 0, 42, 0, 25, 4063, 0, 0, 0, 4056, 7, 23, 11, 0, 0, 0, 0, 40, 4054, 6, 0, 0, 0, 0, 0, 41, 0, 0, 10, 4072, 0, 0, 0, 4038, 9, 8, 0, 3, 1, 0, 0, 4069, 0, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4096, 4130, 4097, 4100, 4123, 4059, 4074] avg=4096.0000 max=4130 max_rank=2 min=4059 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 2, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4100, 4091, 4096, 4094, 4091, 4100, 4101] avg=4096.0000 max=4101 max_rank=7 min=4091 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4090, 4089, 4096, 4106, 4091, 4100, 4101] avg=4096.0000 max=4106 max_rank=4 min=4089 min_rank=2 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 4086, 3, 3, 0, 0, 0, 0, 0, 4052, 42, 0, 0, 0, 0, 0, 0, 8, 0, 5, 4085, 3, 0, 0, 4088, 3, 3, 0, 1, 0, 0, 0, 21, 4060, 1, 0, 0, 0, 0, 0, 8, 0, 0, 4, 4089, 0, 0, 0, 4087, 0, 4, 0, 5, 5, 6, 0, 4090, 0, 0, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4094, 4101, 4095, 4082, 4101, 4107, 4093] avg=4096.0000 max=4107 max_rank=6 min=4082 min_rank=4 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 7, 0, 0, 0, 0, 0, 0, 4084, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 5, 1, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4086, 4089, 4096, 4101, 4095, 4102, 4097] avg=4096.0000 max=4102 max_rank=0 min=4086 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4093, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 4036, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[34, 4048, 0, 17, 26, 0, 0, 0, 4068, 4064, 4021, 20, 4054, 4070, 0, 0, 0, 36, 7, 0, 0, 17, 40, 0, 75, 33, 0, 5, 4029, 22, 4054, 28] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4075, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4083, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 4086, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 3935, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4076, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 4091, 4095, 4090, 0, 4096, 4096, 0, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 333, 3768, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4071, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[11, 4086, 0, 5, 5, 0, 1, 0, 4091, 4090, 4072, 6, 4089, 4090, 1, 1, 0, 8, 0, 0, 1, 1, 1, 0, 16, 10, 1, 2, 4082, 1, 4088, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 0, 4079, 17, 0, 0, 0, 0, 0, 63, 4002, 1, 0, 0, 4036, 0, 77, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4107, 4103, 4092, 4096, 4066, 4113] avg=4096.0000 max=4113 max_rank=7 min=4066 min_rank=6 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[34, 4047, 0, 17, 0, 0, 0, 0, 26, 0, 0, 0, 4064, 0, 0, 0, 11, 0, 4021, 20, 0, 0, 0, 0, 4054, 7, 0, 0, 0, 0, 0, 0, 0, 36, 7, 0, 4063, 3, 0, 0, 0, 17, 40, 0, 4054, 17, 0, 0, 75, 33, 0, 5, 3961, 1, 0, 0, 4029, 22, 76, 28, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4090, 4052, 4061, 4109, 4128, 4075, 4155] avg=4096.0000 max=4155 max_rank=7 min=4052 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16467 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 0, 4061, 17, 0, 0, 0, 0, 0, 34, 4002, 1, 0, 0, 4083, 0, 77, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4107, 4103, 4092, 4078, 4037, 4160] avg=4096.0000 max=4160 max_rank=7 min=4037 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 11, 0, 4090, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 4089, 3, 0, 0, 0, 0, 0, 0, 4077, 17, 0, 0, 0, 0, 0, 333, 4002, 1, 0, 0, 3768, 0, 77, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4101, 4103, 4098, 4094, 4336, 3845] avg=4096.0000 max=4336 max_rank=6 min=3845 min_rank=7 imbalance_ratio=1.058594 max_load_reduction=28432 max_load_reduction_pct=0.867676 moved_to_echo=16373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 0, 4082, 17, 0, 0, 0, 0, 0, 20, 4002, 1, 0, 0, 4076, 0, 77, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4107, 4103, 4092, 4099, 4023, 4153] avg=4096.0000 max=4153 max_rank=7 min=4023 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 0, 4072, 17, 0, 0, 0, 0, 0, 170, 4002, 1, 0, 0, 3935, 0, 77, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4097, 4107, 4103, 4092, 4089, 4173, 4012] avg=4096.0000 max=4173 max_rank=6 min=4012 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16366 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[11, 4084, 0, 5, 0, 0, 0, 0, 5, 0, 1, 0, 4090, 5, 0, 0, 2, 0, 4072, 6, 0, 0, 0, 0, 4089, 0, 1, 1, 0, 0, 0, 0, 0, 8, 0, 0, 4085, 6, 0, 0, 1, 1, 1, 0, 4083, 24, 0, 0, 16, 10, 1, 2, 3999, 2, 0, 0, 4082, 1, 65, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4101, 4080, 4091, 4099, 4110, 4030, 4157] avg=4096.0000 max=4157 max_rank=7 min=4030 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 4057, 17, 0, 0, 0, 0, 0, 25, 4002, 1, 0, 0, 4096, 0, 77, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4107, 4101, 4094, 4074, 4028, 4173] avg=4096.0000 max=4173 max_rank=7 min=4028 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32128, 128, 640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 768, 32000, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[52, 0, 4058, 9, 0, 0, 71, 2, 0, 0, 4068, 38, 0, 4061, 4, 4067, 200, 25, 4044, 4081, 0, 1, 21, 24, 0, 31, 26, 10, 4037, 3795, 0, 43] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 4096, 2, 0, 4096, 0, 4096, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 28, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[9, 25, 4088, 0, 0, 0, 6, 0, 3, 0, 4083, 0, 0, 4089, 1, 4090, 0, 4, 4086, 4091, 0, 9, 0, 5, 0, 3, 6, 7, 4085, 4063, 0, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 9, 0, 2, 4089, 0, 0, 4096, 0, 4096, 13, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4060, 21, 5, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 36, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4091, 4096, 4102, 4071, 4096, 4089, 4132] avg=4096.0000 max=4132 max_rank=7 min=4071 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 21, 5, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 1, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 36, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4087, 4096, 4102, 4076, 4096, 4089, 4131] avg=4096.0000 max=4131 max_rank=7 min=4076 min_rank=4 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[52, 0, 4053, 9, 0, 0, 0, 0, 0, 0, 71, 2, 4001, 21, 5, 1, 0, 0, 4068, 38, 0, 0, 0, 0, 0, 4061, 4, 5, 0, 0, 0, 0, 200, 25, 0, 4060, 0, 0, 0, 0, 0, 1, 21, 24, 4044, 0, 0, 0, 0, 31, 26, 10, 4061, 0, 0, 0, 36, 3795, 0, 43, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4101, 4106, 4070, 4285, 4090, 4128, 3874] avg=4096.0000 max=4285 max_rank=4 min=3874 min_rank=7 imbalance_ratio=1.046143 max_load_reduction=28483 max_load_reduction_pct=0.869232 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4060, 21, 5, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 36, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4115, 4096, 4102, 4047, 4096, 4089, 4132] avg=4096.0000 max=4132 max_rank=7 min=4047 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4060, 21, 5, 1, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 13, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 36, 4088, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4096, 4091, 4102, 4079, 4096, 4089, 4124] avg=4096.0000 max=4124 max_rank=7 min=4079 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4060, 21, 5, 1, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 36, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4097, 4098, 4102, 4063, 4096, 4089, 4132] avg=4096.0000 max=4132 max_rank=7 min=4063 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[9, 25, 4074, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4052, 23, 11, 2, 3, 0, 4083, 0, 3, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 4, 0, 4068, 0, 0, 0, 0, 0, 9, 0, 5, 4086, 2, 0, 0, 0, 3, 6, 7, 4086, 0, 0, 0, 33, 4063, 0, 15, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4094, 4089, 4090, 4072, 4102, 4102, 4111] avg=4096.0000 max=4111 max_rank=7 min=4072 min_rank=4 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 21, 5, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 36, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4087, 4096, 4098, 4075, 4096, 4093, 4132] avg=4096.0000 max=4132 max_rank=7 min=4075 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 0, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 32512, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 32512, 256, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[61, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4029, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[14, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4081, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 1, 4095, 4095, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 18, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4068, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[330, 37, 0, 23, 54, 0, 4058, 61, 30, 0, 0, 35, 3988, 24, 4005, 18, 0, 0, 14, 4013, 0, 28, 3, 0, 5, 4060, 1, 4066, 3853, 10, 5, 4047] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4093, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[20, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 4095, 4074, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[54, 11, 0, 5, 9, 1, 4055, 24, 7, 0, 0, 7, 4068, 4, 4079, 5, 0, 0, 0, 4085, 1, 5, 4, 0, 3, 4089, 0, 4089, 4075, 0, 0, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 0, 0, 4030, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 17, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4081, 0, 0, 49, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4089, 4098, 4108, 4103, 4094, 4101, 4130] avg=4096.0000 max=4130 max_rank=7 min=4045 min_rank=0 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[61, 0, 0, 0, 4030, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 17, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4029, 0, 0, 49, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4089, 4098, 4108, 4108, 4094, 4101, 4078] avg=4096.0000 max=4108 max_rank=3 min=4078 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4030, 1, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4081, 17, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 1, 4096, 1, 5, 0, 0, 0, 0, 4095, 0, 0, 49, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4087, 4100, 4108, 4102, 4093, 4103, 4144] avg=4096.0000 max=4144 max_rank=7 min=4031 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[330, 37, 0, 23, 3982, 1, 0, 0, 54, 0, 4051, 61, 0, 0, 0, 0, 30, 0, 0, 35, 3991, 17, 0, 0, 3988, 24, 11, 18, 0, 0, 0, 0, 0, 0, 14, 4013, 6, 0, 0, 0, 0, 28, 3, 0, 4062, 3, 0, 0, 5, 4060, 1, 4, 0, 0, 0, 0, 3853, 10, 5, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4373, 4166, 4073, 4041, 4033, 4096, 4070, 3916] avg=4096.0000 max=4373 max_rank=0 min=3916 min_rank=7 imbalance_ratio=1.067627 max_load_reduction=28395 max_load_reduction_pct=0.866547 moved_to_echo=16115 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 4030, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 17, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4068, 0, 0, 49, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4089, 4098, 4108, 4120, 4094, 4101, 4117] avg=4096.0000 max=4120 max_rank=4 min=4041 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 0, 0, 4030, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4081, 17, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 1, 4090, 3, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4074, 0, 0, 49, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4089, 4100, 4108, 4102, 4094, 4101, 4123] avg=4096.0000 max=4123 max_rank=7 min=4051 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[54, 11, 0, 5, 4025, 6, 0, 0, 9, 1, 4036, 24, 0, 0, 0, 0, 7, 0, 0, 7, 4069, 20, 0, 0, 4068, 4, 2, 5, 0, 0, 0, 0, 0, 0, 0, 4085, 13, 0, 0, 0, 1, 5, 4, 0, 4089, 8, 0, 0, 3, 4089, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 43, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4070, 4103, 4079, 4098, 4107, 4092, 4118] avg=4096.0000 max=4118 max_rank=7 min=4070 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4030, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 17, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 49, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4089, 4098, 4108, 4102, 4096, 4099, 4142] avg=4096.0000 max=4142 max_rank=7 min=4034 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32384, 128, 128, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 128, 0, 128, 32256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[37, 0, 0, 31, 36, 0, 4069, 46, 12, 38, 0, 95, 4054, 30, 0, 4072, 0, 0, 15, 43, 3969, 0, 0, 4065, 0, 9, 25, 4011, 4047, 4060, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 2, 0, 0, 0, 4092, 0, 0, 0, 0, 4, 4095, 0, 9, 4087, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 2, 4096, 0, 20, 4030, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4095, 4096, 4096, 0, 41] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 45, 4051, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 4065, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 31] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 4094, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[20, 0, 0, 7, 7, 0, 4089, 8, 4, 9, 0, 11, 4091, 6, 0, 4091, 0, 0, 0, 7, 4067, 0, 1, 4089, 0, 4, 4, 4082, 4083, 4088, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 2, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 2, 0, 0, 10, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 2, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4089, 4077, 4102, 4094, 4117, 4103, 4098] avg=4096.0000 max=4117 max_rank=5 min=4077 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 0, 4085, 2, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4075, 2, 0, 0, 10, 0, 9, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4085, 4081, 4106, 4092, 4117, 4103, 4094] avg=4096.0000 max=4117 max_rank=5 min=4081 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[37, 0, 0, 31, 4044, 2, 0, 0, 36, 0, 4063, 46, 0, 0, 0, 0, 12, 38, 0, 95, 4045, 2, 0, 0, 10, 30, 0, 4072, 0, 0, 0, 0, 0, 0, 15, 43, 4058, 0, 0, 0, 3969, 0, 0, 20, 0, 0, 0, 0, 0, 9, 25, 4011, 4, 2, 0, 0, 4045, 0, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4145, 4192, 4112, 4116, 3989, 4051, 4049] avg=4096.0000 max=4192 max_rank=2 min=3989 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16205 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 2, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4075, 2, 0, 0, 10, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 2, 0, 0, 4094, 0, 0, 31, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4089, 4078, 4075, 4094, 4116, 4103, 4125] avg=4096.0000 max=4125 max_rank=7 min=4075 min_rank=3 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16438 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4086, 3, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 3, 2, 4075, 2, 0, 0, 10, 0, 20, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 2, 0, 0, 4094, 0, 0, 41, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4089, 4082, 4060, 4093, 4117, 4102, 4135] avg=4096.0000 max=4135 max_rank=7 min=4060 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16447 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 2, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 2, 0, 0, 10, 0, 45, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4089, 4077, 4106, 4094, 4117, 4103, 4094] avg=4096.0000 max=4117 max_rank=5 min=4077 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16408 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 0, 7, 4087, 3, 4, 0, 7, 0, 4078, 8, 0, 0, 0, 0, 4, 9, 0, 11, 4075, 5, 0, 0, 4, 6, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 7, 4085, 0, 0, 0, 4067, 0, 1, 10, 0, 0, 0, 0, 0, 4, 4, 4082, 6, 4, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4093, 4104, 4101, 4092, 4078, 4100, 4079] avg=4096.0000 max=4121 max_rank=0 min=4078 min_rank=5 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16349 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 3, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 2, 0, 0, 10, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4089, 4077, 4106, 4093, 4117, 4103, 4094] avg=4096.0000 max=4117 max_rank=5 min=4077 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16407 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 0, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 218, 405, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4069, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3503, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 1180, 361, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2555, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 102, 1902, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4095, 4096, 4095, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 2092, 4, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 308, 2462, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3884, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1537, 1, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[2, 0, 159, 3418, 0, 6, 4058, 29, 15, 90, 38, 36, 0, 4058, 6, 23, 3893, 4061, 4074, 35, 3990, 0, 0, 0, 0, 0, 33, 19, 649, 0, 4067, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 311, 2432, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1353, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 461, 295, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4014, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3420, 2, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[17, 0, 561, 900, 0, 2, 4086, 5, 4, 12, 9, 10, 0, 4090, 0, 6, 4033, 4089, 4091, 7, 4079, 0, 0, 0, 2, 0, 3, 11, 2664, 0, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 218, 405, 2161, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 41, 5, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 4069, 69, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 3503, 0, 1877, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2784, 4096, 4073, 4095, 4138, 4111, 4091, 5380] avg=4096.0000 max=5380 max_rank=7 min=2784 min_rank=0 imbalance_ratio=1.313477 max_load_reduction=27388 max_load_reduction_pct=0.835815 moved_to_echo=17720 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 102, 1902, 2161, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 41, 5, 0, 0, 4092, 0, 0, 2, 0, 0, 0, 4095, 69, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 2092, 4, 1877, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4096, 4073, 4094, 4164, 4112, 4090, 3973] avg=4096.0000 max=4166 max_rank=0 min=3973 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16339 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1180, 361, 2161, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 41, 5, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 69, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 2555, 0, 1877, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3702, 4096, 4073, 4098, 4165, 4111, 4091, 4432] avg=4096.0000 max=4432 max_rank=7 min=3702 min_rank=0 imbalance_ratio=1.082031 max_load_reduction=28336 max_load_reduction_pct=0.864746 moved_to_echo=16799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 308, 2462, 2161, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 41, 5, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 3884, 69, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 1537, 1, 1877, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4931, 4096, 4073, 4098, 3953, 4111, 4091, 3415] avg=4096.0000 max=4931 max_rank=0 min=3415 min_rank=7 imbalance_ratio=1.203857 max_load_reduction=27837 max_load_reduction_pct=0.849518 moved_to_echo=15570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 159, 3418, 2146, 0, 0, 0, 0, 6, 4058, 29, 0, 0, 0, 0, 15, 90, 38, 36, 3993, 41, 4, 0, 0, 4058, 6, 23, 1, 0, 0, 0, 3893, 68, 0, 35, 0, 0, 0, 0, 3990, 0, 0, 0, 15, 0, 0, 0, 0, 0, 33, 19, 4070, 0, 0, 0, 649, 0, 1864, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5725, 4093, 4217, 4088, 3996, 4005, 4122, 2522] avg=4096.0000 max=5725 max_rank=0 min=2522 min_rank=7 imbalance_ratio=1.397705 max_load_reduction=27043 max_load_reduction_pct=0.825287 moved_to_echo=14645 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 311, 2432, 2161, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 41, 5, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 69, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 1353, 0, 1877, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4904, 4096, 4073, 4098, 4165, 4111, 4091, 3230] avg=4096.0000 max=4904 max_rank=0 min=3230 min_rank=7 imbalance_ratio=1.197266 max_load_reduction=27864 max_load_reduction_pct=0.850342 moved_to_echo=15597 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 561, 900, 2161, 0, 0, 0, 0, 2, 4086, 5, 6, 0, 0, 0, 4, 12, 9, 10, 4025, 47, 5, 0, 0, 4090, 0, 6, 3, 0, 0, 0, 4033, 64, 0, 7, 0, 0, 0, 0, 4079, 0, 0, 0, 17, 0, 0, 0, 2, 0, 3, 11, 4086, 0, 0, 0, 2664, 0, 1853, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3639, 4099, 4112, 4099, 4104, 4096, 4102, 4517] avg=4096.0000 max=4517 max_rank=7 min=3639 min_rank=0 imbalance_ratio=1.102783 max_load_reduction=28251 max_load_reduction_pct=0.862152 moved_to_echo=16819 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 461, 295, 2161, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 41, 6, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4014, 69, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 3420, 2, 1877, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2917, 4096, 4074, 4098, 4083, 4111, 4090, 5299] avg=4096.0000 max=5299 max_rank=7 min=2917 min_rank=0 imbalance_ratio=1.293701 max_load_reduction=27469 max_load_reduction_pct=0.838287 moved_to_echo=17583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 3328, 12288, 17280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[17792, 128, 15104, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32256, 640, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32256, 384, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 4094, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[23, 65, 4059, 14, 4065, 47, 30, 4056, 0, 4040, 0, 0, 30, 6, 37, 44, 4046, 28, 0, 0, 3784, 2, 0, 0, 0, 83, 215, 1, 3930, 36, 4060, 67] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 4089, 10, 9, 4088, 0, 4083, 0, 0, 8, 0, 7, 9, 4086, 5, 1, 0, 4066, 8, 16, 0, 0, 4, 27, 4, 4065, 7, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 25, 0, 0, 4071, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 38, 1, 0, 0, 0, 0, 0, 0, 4043, 13, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4091, 4104, 4083, 4099, 4135, 4056, 4111] avg=4096.0000 max=4135 max_rank=5 min=4056 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 1, 0, 6, 2, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 2, 0, 38, 1, 0, 0, 0, 0, 0, 0, 4043, 13, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4091, 4105, 4081, 4099, 4137, 4056, 4111] avg=4096.0000 max=4137 max_rank=5 min=4056 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4094, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 38, 1, 0, 0, 0, 2, 0, 0, 4043, 12, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4091, 4102, 4084, 4099, 4135, 4057, 4111] avg=4096.0000 max=4135 max_rank=5 min=4057 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4095, 0, 0, 0, 38, 1, 0, 0, 0, 1, 0, 0, 4043, 13, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4091, 4104, 4083, 4099, 4134, 4057, 4111] avg=4096.0000 max=4134 max_rank=5 min=4057 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 38, 1, 0, 0, 0, 0, 0, 0, 4043, 13, 0, 0, 4095, 0, 15, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4091, 4104, 4083, 4099, 4135, 4056, 4111] avg=4096.0000 max=4135 max_rank=5 min=4056 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[23, 65, 4053, 14, 0, 0, 0, 0, 0, 47, 30, 4051, 0, 0, 0, 0, 0, 4040, 0, 0, 5, 2, 0, 0, 30, 6, 37, 44, 4052, 0, 0, 0, 4046, 28, 0, 0, 3, 0, 0, 0, 3784, 2, 0, 0, 37, 1, 0, 0, 0, 83, 215, 1, 4008, 13, 0, 0, 3930, 36, 15, 67, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4128, 4047, 4169, 4077, 3824, 4320, 4048] avg=4096.0000 max=4320 max_rank=6 min=3824 min_rank=5 imbalance_ratio=1.054688 max_load_reduction=28448 max_load_reduction_pct=0.868164 moved_to_echo=16269 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4080, 0, 0, 0, 0, 0, 0, 10, 9, 4075, 0, 0, 0, 0, 0, 4083, 0, 0, 7, 8, 0, 0, 8, 0, 7, 9, 4077, 0, 0, 0, 4086, 5, 1, 0, 5, 0, 0, 0, 4066, 8, 16, 0, 41, 2, 0, 0, 0, 4, 27, 4, 4039, 12, 0, 0, 4065, 7, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4094, 4098, 4101, 4097, 4133, 4086, 4079] avg=4096.0000 max=4133 max_rank=5 min=4079 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16395 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 38, 1, 0, 0, 0, 25, 0, 0, 4043, 12, 0, 0, 4071, 0, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4091, 4104, 4084, 4099, 4135, 4080, 4086] avg=4096.0000 max=4135 max_rank=5 min=4080 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 256, 128, 32384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32512, 128, 128, 0, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1, 4065, 1612, 3, 1754, 2, 1021, 0, 3054, 182, 863, 2204, 873, 442, 13, 430, 728, 1519, 635, 4095, 1263, 1529, 470, 45, 0, 526, 205, 496, 1344, 25, 3363, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[6, 4082, 2287, 17, 1023, 0, 949, 5, 2820, 328, 1098, 1934, 1421, 493, 11, 552, 700, 1099, 903, 4096, 1759, 1373, 474, 18, 0, 769, 178, 664, 781, 44, 2882, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2, 4066, 1993, 19, 1784, 1, 887, 3, 3118, 116, 909, 1936, 1318, 331, 1, 783, 1019, 1186, 766, 4091, 1140, 1034, 646, 82, 0, 734, 214, 1074, 676, 23, 2814, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1, 4038, 2186, 14, 1266, 0, 987, 8, 3327, 118, 1188, 2524, 1247, 434, 6, 522, 719, 1180, 484, 4089, 1656, 1006, 715, 30, 0, 619, 183, 511, 902, 28, 2773, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[10, 3825, 1310, 131, 1549, 0, 734, 32, 2833, 616, 1017, 1284, 868, 580, 37, 739, 937, 898, 711, 4044, 1954, 2026, 723, 82, 5, 751, 221, 1145, 846, 26, 2822, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3, 4078, 2171, 3, 1167, 2, 557, 32, 3044, 251, 898, 2028, 1087, 309, 8, 531, 916, 744, 1451, 4089, 1140, 1101, 710, 39, 0, 611, 222, 922, 1474, 46, 3122, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[4, 4078, 1948, 13, 1217, 1, 765, 0, 3153, 131, 831, 2032, 1269, 490, 4, 714, 925, 942, 967, 4096, 1118, 1204, 819, 22, 0, 756, 309, 733, 1234, 52, 2935, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1, 4079, 1438, 10, 1454, 0, 1019, 3, 3435, 86, 1334, 2595, 1444, 572, 7, 583, 713, 1000, 479, 4095, 1537, 1543, 442, 17, 1, 776, 153, 452, 502, 15, 2982, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 2214, 1612, 3, 0, 0, 0, 0, 1754, 2, 1021, 0, 1090, 724, 0, 0, 773, 182, 863, 2204, 0, 0, 0, 0, 873, 442, 13, 430, 1557, 251, 0, 0, 728, 1519, 635, 1397, 0, 0, 0, 0, 1263, 1529, 470, 45, 635, 0, 0, 0, 0, 526, 205, 496, 2447, 0, 0, 0, 1344, 25, 3363, 6, 126, 0, 0, 0] rerouted_tokens_per_ep_rank=[3830, 4591, 4022, 3566, 4279, 3942, 3674, 4864] avg=4096.0000 max=4864 max_rank=7 min=3566 min_rank=3 imbalance_ratio=1.187500 max_load_reduction=27904 max_load_reduction_pct=0.851562 moved_to_echo=16759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[6, 2223, 2287, 17, 0, 0, 0, 0, 1023, 0, 949, 5, 1094, 669, 0, 0, 713, 328, 1098, 1934, 0, 0, 0, 0, 1421, 493, 11, 552, 1438, 251, 0, 0, 700, 1099, 903, 1398, 0, 0, 0, 0, 1759, 1373, 474, 18, 638, 0, 0, 0, 0, 769, 178, 664, 2447, 0, 0, 0, 781, 44, 2882, 2, 127, 0, 0, 0] rerouted_tokens_per_ep_rank=[4533, 3740, 4073, 4166, 4100, 4262, 4058, 3836] avg=4096.0000 max=4533 max_rank=0 min=3740 min_rank=1 imbalance_ratio=1.106689 max_load_reduction=28235 max_load_reduction_pct=0.861664 moved_to_echo=16256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 2221, 2171, 3, 0, 0, 0, 0, 1167, 2, 557, 32, 1093, 722, 0, 0, 770, 251, 898, 2028, 0, 0, 0, 0, 1087, 309, 8, 531, 1552, 251, 0, 0, 916, 744, 1451, 1395, 0, 0, 0, 0, 1140, 1101, 710, 39, 637, 0, 0, 0, 0, 611, 222, 922, 2443, 0, 0, 0, 1474, 46, 3122, 12, 127, 0, 0, 0] rerouted_tokens_per_ep_rank=[4398, 3573, 3947, 3738, 4506, 3627, 4198, 4781] avg=4096.0000 max=4781 max_rank=7 min=3573 min_rank=1 imbalance_ratio=1.167236 max_load_reduction=27987 max_load_reduction_pct=0.854095 moved_to_echo=17112 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 2215, 1993, 19, 0, 0, 0, 0, 1784, 1, 887, 3, 1090, 739, 0, 0, 789, 116, 909, 1936, 0, 0, 0, 0, 1318, 331, 1, 783, 1590, 251, 0, 0, 1019, 1186, 766, 1396, 0, 0, 0, 0, 1140, 1034, 646, 82, 635, 0, 0, 0, 0, 734, 214, 1074, 2444, 0, 0, 0, 676, 23, 2814, 2, 126, 0, 0, 0] rerouted_tokens_per_ep_rank=[4229, 4504, 3750, 4274, 4367, 3537, 4466, 3641] avg=4096.0000 max=4504 max_rank=1 min=3537 min_rank=5 imbalance_ratio=1.099609 max_load_reduction=28264 max_load_reduction_pct=0.862549 moved_to_echo=16011 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 2198, 2186, 14, 0, 0, 0, 0, 1266, 0, 987, 8, 1083, 789, 0, 0, 842, 118, 1188, 2524, 0, 0, 0, 0, 1247, 434, 6, 522, 1696, 251, 0, 0, 719, 1180, 484, 1395, 0, 0, 0, 0, 1656, 1006, 715, 30, 631, 0, 0, 0, 0, 619, 183, 511, 2443, 0, 0, 0, 902, 28, 2773, 7, 126, 0, 0, 0] rerouted_tokens_per_ep_rank=[4399, 4133, 4672, 4156, 3778, 4038, 3756, 3836] avg=4096.0000 max=4672 max_rank=2 min=3756 min_rank=6 imbalance_ratio=1.140625 max_load_reduction=28096 max_load_reduction_pct=0.857422 moved_to_echo=15408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 2209, 1948, 13, 0, 0, 0, 0, 1217, 1, 765, 0, 1097, 751, 0, 0, 791, 131, 831, 2032, 0, 0, 0, 0, 1269, 490, 4, 714, 1611, 256, 0, 0, 925, 942, 967, 1389, 0, 0, 0, 0, 1118, 1204, 819, 22, 641, 0, 0, 0, 0, 756, 309, 733, 2451, 0, 0, 0, 1234, 52, 2935, 6, 131, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 3831, 3785, 4344, 4223, 3804, 4249, 4358] avg=4096.0000 max=4358 max_rank=7 min=3785 min_rank=2 imbalance_ratio=1.063965 max_load_reduction=28410 max_load_reduction_pct=0.867004 moved_to_echo=16634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 2221, 1438, 10, 0, 0, 0, 0, 1454, 0, 1019, 3, 1094, 814, 0, 0, 870, 86, 1334, 2595, 0, 0, 0, 0, 1444, 572, 7, 583, 1751, 251, 0, 0, 713, 1000, 479, 1397, 0, 0, 0, 0, 1537, 1543, 442, 17, 637, 0, 0, 0, 1, 776, 153, 452, 2447, 0, 0, 0, 502, 15, 2982, 1, 127, 0, 0, 0] rerouted_tokens_per_ep_rank=[3670, 4384, 4885, 4608, 3589, 4176, 3829, 3627] avg=4096.0000 max=4885 max_rank=2 min=3589 min_rank=4 imbalance_ratio=1.192627 max_load_reduction=27883 max_load_reduction_pct=0.850922 moved_to_echo=15221 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 2084, 1310, 131, 0, 0, 0, 0, 1549, 0, 734, 32, 1025, 672, 0, 0, 717, 616, 1017, 1284, 0, 0, 0, 0, 868, 580, 37, 739, 1444, 248, 0, 0, 937, 898, 711, 1380, 0, 0, 0, 0, 1954, 2026, 723, 82, 597, 0, 0, 0, 5, 751, 221, 1145, 2416, 0, 0, 0, 846, 26, 2822, 12, 119, 0, 0, 0] rerouted_tokens_per_ep_rank=[3535, 4012, 3634, 3916, 3926, 5382, 4538, 3825] avg=4096.0000 max=5382 max_rank=5 min=3535 min_rank=0 imbalance_ratio=1.313965 max_load_reduction=27386 max_load_reduction_pct=0.835754 moved_to_echo=17671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11264, 128, 7040, 128, 8704, 5888, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6784, 8576, 6400, 11264, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9600, 3712, 128, 4864, 12672, 2048, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11648, 10880, 5120, 384, 5120, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 5632, 1792, 6016, 19584, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 17664, 14976, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6272, 1920, 8192, 16640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7808, 384, 23808, 128, 1024, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[7, 1, 164, 4096, 2, 0, 4082, 4092, 0, 8, 0, 0, 1, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 4096, 754, 0, 0, 4089, 0, 3180, 0, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 62, 4085, 0, 0, 4080, 4096, 10, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4082, 0, 0, 0, 4067, 481, 6, 0, 4095, 2, 3605, 0, 4, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[21, 1, 895, 4043, 3, 0, 4083, 4063, 0, 4, 1, 0, 0, 37, 4090, 1, 0, 0, 4083, 11, 0, 0, 4072, 633, 0, 20, 4095, 0, 2610, 1, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1072, 4096, 6, 3, 4090, 4096, 3, 0, 1, 0, 1, 0, 4095, 0, 1, 1, 4094, 0, 0, 0, 4087, 1061, 0, 15, 4096, 0, 1950, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[6, 0, 116, 4092, 0, 0, 4091, 4095, 0, 24, 4, 1, 0, 8, 4085, 0, 0, 0, 4049, 0, 0, 1, 4089, 1505, 0, 0, 4096, 1, 2502, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[6, 1, 396, 4089, 0, 0, 4085, 4071, 0, 6, 0, 0, 0, 0, 4093, 1, 1, 2, 4095, 0, 0, 1, 4026, 552, 0, 3, 4095, 2, 3243, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[15, 0, 231, 4089, 9, 0, 4053, 4092, 0, 135, 0, 3, 0, 0, 4094, 0, 39, 6, 4082, 0, 2, 0, 4071, 189, 0, 0, 4094, 3, 3547, 0, 14, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 86, 4096, 0, 0, 4094, 4094, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 4089, 182, 0, 1, 4096, 0, 3838, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 62, 3711, 0, 0, 0, 0, 0, 0, 4080, 12, 0, 0, 0, 0, 10, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 2, 1, 0, 0, 0, 0, 3420, 481, 0, 0, 0, 0, 6, 0, 4091, 2, 0, 0, 0, 0, 3605, 0, 4, 4, 647, 374, 5, 3] rerouted_tokens_per_ep_rank=[3773, 4092, 4089, 4087, 4085, 3901, 4099, 4642] avg=4096.0000 max=4642 max_rank=7 min=3773 min_rank=0 imbalance_ratio=1.133301 max_load_reduction=28126 max_load_reduction_pct=0.858337 moved_to_echo=16727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[7, 1, 164, 3721, 0, 0, 0, 0, 2, 0, 4082, 12, 0, 0, 0, 0, 0, 8, 0, 0, 4075, 0, 0, 0, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 3444, 754, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 3180, 0, 6, 0, 652, 375, 5, 3] rerouted_tokens_per_ep_rank=[3893, 4096, 4083, 4093, 4099, 4198, 4085, 4221] avg=4096.0000 max=4221 max_rank=7 min=3893 min_rank=0 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16603 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[21, 1, 895, 3673, 0, 0, 0, 0, 3, 0, 4083, 12, 0, 0, 0, 0, 0, 4, 1, 0, 4046, 0, 0, 0, 0, 37, 4088, 1, 0, 0, 0, 0, 0, 0, 4083, 11, 2, 1, 0, 0, 0, 0, 3424, 633, 0, 0, 0, 0, 0, 20, 4091, 0, 0, 0, 0, 0, 2610, 1, 1, 0, 648, 370, 5, 3] rerouted_tokens_per_ep_rank=[4590, 4098, 4051, 4126, 4097, 4057, 4111, 3638] avg=4096.0000 max=4590 max_rank=0 min=3638 min_rank=7 imbalance_ratio=1.120605 max_load_reduction=28178 max_load_reduction_pct=0.859924 moved_to_echo=15903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 116, 3717, 0, 0, 0, 0, 0, 0, 4091, 12, 0, 0, 0, 0, 0, 24, 4, 1, 4078, 0, 0, 0, 0, 8, 4083, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 2, 1, 0, 0, 0, 1, 3438, 1505, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 2502, 0, 3, 0, 651, 375, 5, 3] rerouted_tokens_per_ep_rank=[3839, 4103, 4107, 4091, 4052, 4944, 4093, 3539] avg=4096.0000 max=4944 max_rank=5 min=3539 min_rank=7 imbalance_ratio=1.207031 max_load_reduction=27824 max_load_reduction_pct=0.849121 moved_to_echo=16628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1072, 3721, 0, 0, 0, 0, 6, 3, 4090, 12, 0, 0, 0, 0, 3, 0, 1, 0, 4079, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 1, 1, 4094, 0, 2, 1, 0, 0, 0, 0, 3437, 1061, 0, 0, 0, 0, 0, 15, 4092, 0, 0, 0, 0, 0, 1950, 0, 0, 0, 650, 375, 5, 3] rerouted_tokens_per_ep_rank=[4793, 4111, 4083, 4094, 4099, 4498, 4107, 2983] avg=4096.0000 max=4793 max_rank=0 min=2983 min_rank=7 imbalance_ratio=1.170166 max_load_reduction=27975 max_load_reduction_pct=0.853729 moved_to_echo=15687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 1, 396, 3714, 0, 0, 0, 0, 0, 0, 4085, 12, 0, 0, 0, 0, 0, 6, 0, 0, 4054, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 1, 2, 4095, 0, 2, 1, 0, 0, 0, 1, 3385, 552, 0, 0, 0, 0, 0, 3, 4091, 2, 0, 0, 0, 0, 3243, 0, 0, 0, 641, 375, 5, 3] rerouted_tokens_per_ep_rank=[4117, 4097, 4060, 4092, 4101, 3938, 4096, 4267] avg=4096.0000 max=4267 max_rank=7 min=3938 min_rank=5 imbalance_ratio=1.041748 max_load_reduction=28501 max_load_reduction_pct=0.869781 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 86, 3716, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 1, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 8, 0, 0, 0, 0, 3436, 182, 0, 0, 0, 0, 0, 1, 4083, 0, 0, 0, 0, 0, 3838, 0, 0, 0, 653, 380, 10, 5] rerouted_tokens_per_ep_rank=[3803, 4097, 4082, 4093, 4105, 3618, 4084, 4886] avg=4096.0000 max=4886 max_rank=7 min=3618 min_rank=5 imbalance_ratio=1.192871 max_load_reduction=27882 max_load_reduction_pct=0.850891 moved_to_echo=16693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[15, 0, 231, 3714, 0, 0, 0, 0, 9, 0, 4053, 12, 0, 0, 0, 0, 0, 135, 0, 3, 4075, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 39, 6, 4082, 0, 2, 1, 0, 0, 2, 0, 3423, 189, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 0, 0, 3547, 0, 14, 0, 648, 375, 5, 3] rerouted_tokens_per_ep_rank=[3960, 4074, 4213, 4092, 4130, 3614, 4093, 4592] avg=4096.0000 max=4592 max_rank=7 min=3614 min_rank=5 imbalance_ratio=1.121094 max_load_reduction=28176 max_load_reduction_pct=0.859863 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[24576, 128, 128, 128, 5248, 3072, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 27520, 5376, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 3072, 29696, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 1, 0, 0, 3, 0, 4096, 0, 0, 0, 4055, 0, 2775, 4093, 5, 1, 0, 0, 4043, 0, 916, 0, 0, 3953, 0, 4024, 0, 4095, 0, 702, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 8, 0, 1, 1, 1, 13, 4094, 0, 0, 17, 4055, 9, 2073, 4096, 0, 0, 3, 0, 4049, 66, 2548, 0, 0, 3910, 0, 3322, 0, 4070, 0, 426, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4, 4032, 0, 3370, 4090, 1, 0, 0, 0, 4075, 0, 497, 0, 0, 3709, 0, 3960, 0, 4096, 0, 830, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 2, 0, 1, 4096, 0, 3, 2, 4079, 0, 3789, 4094, 10, 0, 0, 0, 4082, 0, 2168, 0, 0, 1636, 0, 3411, 0, 4094, 0, 1299, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4057, 0, 2606, 4094, 0, 0, 0, 1, 4094, 23, 1705, 0, 0, 3757, 1, 3509, 0, 4093, 6, 718, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4086, 1, 4075, 4096, 0, 1, 0, 1, 4095, 1, 403, 0, 0, 3932, 0, 3205, 0, 4094, 2, 677, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[41, 0, 1, 0, 69, 0, 0, 4095, 0, 1, 0, 4051, 0, 3368, 4090, 260, 48, 0, 13, 3698, 64, 655, 1, 1, 2257, 0, 4087, 1, 4081, 1, 1885, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4059, 0, 2346, 4096, 0, 0, 0, 0, 4095, 16, 2198, 0, 0, 3644, 0, 3131, 0, 4096, 0, 991, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 3080, 1110, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 6, 4, 4032, 21, 11, 0, 0, 0, 3370, 1010, 1, 0, 0, 0, 0, 0, 0, 0, 4075, 58, 0, 0, 0, 0, 497, 0, 0, 2024, 858, 0, 0, 3709, 0, 826, 0, 0, 0, 0, 0, 3159, 0, 830, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4085, 4074, 4381, 4133, 3379, 4535, 3989] avg=4096.0000 max=4535 max_rank=6 min=3379 min_rank=5 imbalance_ratio=1.107178 max_load_reduction=28233 max_load_reduction_pct=0.861603 moved_to_echo=16036 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 1, 0, 3083, 1128, 0, 0, 0, 3, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4055, 21, 11, 0, 0, 0, 2775, 1010, 5, 0, 0, 0, 0, 1, 0, 0, 4043, 58, 0, 0, 0, 0, 916, 0, 0, 2057, 858, 0, 0, 3953, 0, 839, 0, 0, 0, 0, 0, 3158, 0, 702, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4218, 4088, 4087, 3790, 4102, 3831, 4792, 3860] avg=4096.0000 max=4792 max_rank=6 min=3790 min_rank=3 imbalance_ratio=1.169922 max_load_reduction=27976 max_load_reduction_pct=0.853760 moved_to_echo=16585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 3083, 956, 0, 0, 2, 0, 1, 4085, 0, 0, 0, 0, 0, 3, 2, 4079, 21, 11, 0, 0, 0, 3789, 1011, 10, 0, 0, 0, 0, 0, 0, 0, 4082, 58, 0, 0, 0, 0, 2168, 0, 0, 1743, 858, 0, 0, 1636, 0, 712, 0, 0, 0, 0, 0, 3157, 0, 1299, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4088, 4116, 4810, 4140, 4769, 2348, 4456] avg=4096.0000 max=4810 max_rank=3 min=2348 min_rank=6 imbalance_ratio=1.174316 max_load_reduction=27958 max_load_reduction_pct=0.853210 moved_to_echo=15713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3085, 898, 0, 0, 0, 0, 1, 4085, 0, 0, 0, 0, 0, 0, 0, 4086, 21, 11, 0, 0, 1, 4075, 1011, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 58, 0, 0, 0, 1, 403, 0, 0, 1638, 858, 0, 0, 3932, 0, 669, 0, 0, 0, 0, 0, 3157, 2, 677, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3983, 4086, 4118, 5087, 4155, 2900, 4601, 3838] avg=4096.0000 max=5087 max_rank=3 min=2900 min_rank=5 imbalance_ratio=1.241943 max_load_reduction=27681 max_load_reduction_pct=0.844757 moved_to_echo=15494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 1, 3085, 931, 0, 0, 1, 1, 13, 4084, 0, 0, 0, 0, 0, 0, 17, 4055, 21, 10, 0, 0, 9, 2073, 1011, 0, 0, 0, 0, 0, 0, 3, 0, 4049, 58, 0, 0, 0, 66, 2548, 0, 0, 1698, 853, 0, 0, 3910, 0, 693, 0, 0, 0, 0, 0, 3138, 0, 426, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4099, 4103, 3093, 4110, 5165, 4603, 3570] avg=4096.0000 max=5165 max_rank=5 min=3093 min_rank=3 imbalance_ratio=1.260986 max_load_reduction=27603 max_load_reduction_pct=0.842377 moved_to_echo=17448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 0, 0, 3083, 984, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4, 4057, 21, 11, 0, 0, 0, 2606, 1011, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 58, 0, 0, 0, 23, 1705, 0, 0, 1794, 858, 0, 0, 3757, 1, 731, 0, 0, 0, 0, 0, 3156, 6, 718, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4085, 4093, 3617, 4153, 4380, 4489, 3881] avg=4096.0000 max=4489 max_rank=6 min=3617 min_rank=3 imbalance_ratio=1.095947 max_load_reduction=28279 max_load_reduction_pct=0.863007 moved_to_echo=16903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3090, 881, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 4059, 22, 13, 0, 0, 0, 2346, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 63, 0, 0, 0, 16, 2198, 0, 0, 1604, 861, 0, 0, 3644, 0, 646, 0, 0, 0, 0, 0, 3150, 0, 991, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3971, 4083, 4094, 3352, 4158, 4679, 4290, 4141] avg=4096.0000 max=4679 max_rank=5 min=3352 min_rank=3 imbalance_ratio=1.142334 max_load_reduction=28089 max_load_reduction_pct=0.857208 moved_to_echo=17268 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[41, 0, 1, 0, 3080, 1146, 0, 0, 69, 0, 0, 4085, 0, 0, 0, 0, 0, 1, 0, 4051, 21, 10, 0, 0, 0, 3368, 1010, 260, 0, 0, 0, 0, 48, 0, 13, 3698, 58, 0, 0, 0, 64, 655, 1, 1, 2089, 855, 0, 0, 2257, 0, 852, 1, 0, 0, 0, 0, 3147, 1, 1885, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4268, 4154, 4083, 4638, 3817, 3665, 3110, 5033] avg=4096.0000 max=5033 max_rank=7 min=3110 min_rank=6 imbalance_ratio=1.228760 max_load_reduction=27735 max_load_reduction_pct=0.846405 moved_to_echo=15625 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 24704, 8064, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 11136, 128, 128, 14720, 6912, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 24448, 8192, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[25344, 128, 7552, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[26880, 128, 6016, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32256, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 32512, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 4096, 635, 0, 0, 637, 4096, 0, 0, 4095, 0, 4090, 4095, 0, 2831, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4064, 0, 0, 0, 0, 0, 1, 4095, 297, 0, 0, 213, 4096, 0, 0, 3962, 1, 4088, 4096, 0, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 0, 17, 4096, 3499, 0, 0, 1, 4096, 0, 0, 3970, 0, 4092, 4096, 0, 686, 0, 0, 8, 0, 18, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4081, 3, 3, 0, 1, 0, 9, 4091, 3370, 1, 0, 28, 4094, 0, 0, 4013, 0, 4094, 4095, 0, 782, 0, 0, 0, 1, 6, 1, 0, 0, 4094, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4046, 0, 50, 6, 0, 0, 0, 4089, 176, 0, 0, 8, 4092, 0, 0, 4039, 3, 4083, 4070, 4, 3959, 5, 0, 1, 25, 0, 0, 0, 0, 4096, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 1, 0, 1, 1, 5, 4094, 2228, 0, 1, 2, 4096, 0, 1, 3570, 0, 4096, 4092, 0, 2386, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 1374, 0, 0, 132, 4096, 0, 0, 4091, 0, 4096, 4096, 0, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4054, 0, 1, 0, 1, 0, 39, 4094, 1343, 0, 0, 86, 4029, 1, 0, 4004, 0, 4096, 4096, 0, 2766, 0, 0, 63, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 635, 0, 0, 637, 2347, 0, 0, 0, 129, 0, 0, 4095, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 0, 0, 2831, 0, 0, 0, 1620, 1, 0, 0, 0, 3, 0, 0, 4091, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4089, 3619, 4224, 4094, 4452, 4094, 4092] avg=4096.0000 max=4452 max_rank=5 min=3619 min_rank=2 imbalance_ratio=1.086914 max_load_reduction=28316 max_load_reduction_pct=0.864136 moved_to_echo=16732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 6, 4, 0, 0, 0, 0, 17, 4089, 0, 0, 0, 0, 3499, 0, 0, 1, 2347, 0, 0, 0, 129, 0, 0, 3970, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 686, 0, 0, 8, 1620, 1, 0, 0, 0, 18, 0, 0, 4092, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4106, 5847, 4099, 4096, 2315, 4110, 4092] avg=4096.0000 max=5847 max_rank=2 min=2315 min_rank=5 imbalance_ratio=1.427490 max_load_reduction=26921 max_load_reduction_pct=0.821564 moved_to_echo=14613 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4064, 0, 0, 0, 6, 4, 0, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 297, 0, 0, 213, 2347, 0, 0, 0, 129, 0, 0, 3962, 0, 0, 0, 0, 1, 4088, 4, 0, 0, 0, 0, 0, 3744, 0, 0, 0, 1620, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4092, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4089, 2857, 4091, 4093, 5365, 4092, 4107] avg=4096.0000 max=5365 max_rank=5 min=2857 min_rank=2 imbalance_ratio=1.309814 max_load_reduction=27403 max_load_reduction_pct=0.836273 moved_to_echo=17657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4081, 3, 3, 0, 6, 3, 0, 0, 1, 0, 9, 4085, 0, 0, 0, 0, 3370, 1, 0, 28, 2346, 0, 0, 0, 129, 0, 0, 4013, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 0, 0, 782, 0, 0, 0, 1619, 0, 0, 0, 1, 6, 1, 0, 4091, 0, 0, 0, 0, 4091, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 5745, 4142, 4098, 2401, 4099, 4092] avg=4096.0000 max=5745 max_rank=2 min=2401 min_rank=5 imbalance_ratio=1.402588 max_load_reduction=27023 max_load_reduction_pct=0.824677 moved_to_echo=14690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4046, 0, 50, 6, 6, 4, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 176, 0, 0, 8, 2345, 0, 0, 0, 129, 0, 0, 4039, 0, 0, 0, 0, 3, 4083, 4, 4, 0, 0, 0, 0, 3959, 5, 0, 1, 1618, 0, 0, 0, 25, 0, 0, 0, 4066, 0, 0, 0, 0, 4092, 16, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4083, 2529, 4168, 4094, 5583, 4091, 4108] avg=4096.0000 max=5583 max_rank=5 min=2529 min_rank=2 imbalance_ratio=1.363037 max_load_reduction=27185 max_load_reduction_pct=0.829620 moved_to_echo=17876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 1374, 0, 0, 132, 2350, 0, 0, 0, 120, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2594, 0, 0, 0, 1623, 3, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4085, 3856, 4211, 4096, 4220, 4099, 4091] avg=4096.0000 max=4220 max_rank=5 min=3856 min_rank=2 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4054, 0, 1, 0, 6, 3, 0, 0, 1, 0, 39, 4087, 0, 0, 0, 0, 1343, 0, 0, 86, 2308, 0, 0, 0, 128, 1, 0, 4004, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 2766, 0, 0, 63, 1593, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4127, 3737, 4133, 4097, 4423, 4095, 4092] avg=4096.0000 max=4423 max_rank=5 min=3737 min_rank=2 imbalance_ratio=1.079834 max_load_reduction=28345 max_load_reduction_pct=0.865021 moved_to_echo=16707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 1, 0, 6, 4, 0, 0, 1, 1, 5, 4087, 0, 0, 0, 0, 2228, 0, 1, 2, 2347, 0, 0, 0, 129, 0, 1, 3570, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 2386, 0, 2, 0, 1620, 1, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4094, 4578, 3700, 4100, 4009, 4088, 4094] avg=4096.0000 max=4578 max_rank=2 min=3700 min_rank=3 imbalance_ratio=1.117676 max_load_reduction=28190 max_load_reduction_pct=0.860291 moved_to_echo=16291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[19840, 128, 128, 128, 13056, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12928, 128, 128, 1152, 18816, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1024, 128, 128, 31744, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4037, 4096, 0, 0, 0, 4096, 0, 3958, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 4071, 6, 2, 44, 0, 0, 0, 4096, 169, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3995, 4096, 0, 0, 0, 4055, 3, 2420, 3, 0, 0, 14, 4095, 0, 0, 24, 662, 0, 4076, 0, 4096, 47, 16, 48, 0, 1, 0, 4096, 1021, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4005, 4095, 0, 0, 1, 4093, 3, 2008, 0, 0, 0, 0, 4045, 0, 0, 0, 1802, 0, 4096, 0, 4092, 4, 1, 94, 0, 0, 0, 4096, 333, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4069, 4095, 0, 0, 0, 4096, 1, 3930, 0, 0, 0, 0, 4085, 0, 0, 1, 202, 0, 4096, 0, 4077, 11, 0, 9, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4096, 0, 0, 0, 4095, 0, 2433, 0, 0, 0, 0, 4096, 0, 0, 0, 371, 0, 4096, 0, 4093, 0, 0, 1, 0, 0, 0, 4096, 1297, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 4092, 4094, 4, 0, 2, 4095, 0, 3570, 0, 0, 0, 1, 4096, 0, 0, 0, 95, 0, 4093, 2, 4046, 0, 43, 266, 1, 0, 0, 4095, 172, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3920, 4096, 0, 0, 23, 4047, 155, 3615, 1, 0, 0, 0, 4096, 0, 0, 2, 8, 0, 4062, 0, 4094, 70, 0, 470, 0, 0, 0, 4095, 14, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4088, 4096, 1, 0, 0, 4096, 0, 3947, 0, 0, 0, 0, 4095, 0, 0, 1, 183, 0, 4096, 0, 3765, 2, 3, 1, 0, 0, 0, 4096, 296, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4037, 59, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 3958, 0, 0, 441, 398, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 1, 0, 3596, 81, 0, 0, 38, 0, 4071, 6, 0, 0, 0, 0, 2, 44, 0, 0, 3977, 0, 0, 0, 0, 3684, 169, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4104, 4797, 4102, 3678, 4115, 4023, 3853] avg=4096.0000 max=4797 max_rank=2 min=3678 min_rank=4 imbalance_ratio=1.171143 max_load_reduction=27971 max_load_reduction_pct=0.853607 moved_to_echo=15669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4005, 59, 0, 0, 0, 0, 0, 0, 1, 4093, 8, 0, 0, 0, 3, 2008, 0, 0, 441, 398, 0, 0, 0, 0, 4045, 0, 6, 0, 0, 0, 0, 0, 1802, 0, 3595, 81, 0, 0, 38, 0, 4092, 4, 0, 0, 0, 0, 1, 94, 0, 0, 3977, 0, 0, 0, 0, 3684, 333, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4102, 2850, 4051, 5478, 4134, 4072, 4017] avg=4096.0000 max=5478 max_rank=4 min=2850 min_rank=2 imbalance_ratio=1.337402 max_load_reduction=27290 max_load_reduction_pct=0.832825 moved_to_echo=17701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3995, 59, 0, 0, 0, 0, 0, 0, 0, 4055, 8, 0, 0, 0, 3, 2420, 3, 0, 441, 398, 0, 0, 0, 14, 4095, 0, 6, 0, 0, 0, 0, 24, 662, 0, 3596, 81, 0, 0, 37, 0, 4096, 47, 0, 0, 0, 0, 16, 48, 0, 1, 3958, 0, 0, 0, 0, 3684, 1021, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4063, 3265, 4115, 4363, 4180, 4023, 4705] avg=4096.0000 max=4705 max_rank=7 min=3265 min_rank=2 imbalance_ratio=1.148682 max_load_reduction=28063 max_load_reduction_pct=0.856415 moved_to_echo=17271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4069, 59, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 1, 3930, 0, 0, 441, 398, 0, 0, 0, 0, 4085, 0, 6, 0, 0, 0, 0, 1, 202, 0, 3595, 81, 0, 0, 38, 0, 4077, 11, 0, 0, 0, 0, 0, 9, 0, 0, 3977, 0, 0, 0, 0, 3684, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4104, 4770, 4091, 3879, 4126, 3986, 3684] avg=4096.0000 max=4770 max_rank=2 min=3684 min_rank=7 imbalance_ratio=1.164551 max_load_reduction=27998 max_load_reduction_pct=0.854431 moved_to_echo=15675 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4088, 59, 0, 0, 0, 0, 1, 0, 0, 4096, 8, 0, 0, 0, 0, 3947, 0, 0, 441, 398, 0, 0, 0, 0, 4095, 0, 6, 0, 0, 0, 0, 1, 183, 0, 3596, 81, 0, 0, 38, 0, 3765, 2, 0, 0, 0, 0, 3, 1, 0, 0, 3977, 0, 0, 0, 0, 3684, 296, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4105, 4786, 4101, 3861, 3805, 3981, 3982] avg=4096.0000 max=4786 max_rank=2 min=3805 min_rank=5 imbalance_ratio=1.168457 max_load_reduction=27982 max_load_reduction_pct=0.853943 moved_to_echo=15629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4092, 59, 0, 0, 0, 0, 4, 0, 2, 4095, 7, 0, 0, 0, 0, 3570, 0, 0, 441, 398, 0, 0, 0, 1, 4096, 0, 6, 0, 0, 0, 0, 0, 95, 0, 3594, 81, 0, 0, 37, 2, 4046, 0, 0, 0, 0, 0, 43, 266, 1, 0, 3975, 0, 0, 0, 0, 3684, 172, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4108, 4409, 4103, 3770, 4085, 4285, 3856] avg=4096.0000 max=4409 max_rank=2 min=3770 min_rank=4 imbalance_ratio=1.076416 max_load_reduction=28359 max_load_reduction_pct=0.865448 moved_to_echo=15996 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 54, 0, 0, 0, 0, 0, 0, 0, 4095, 10, 0, 0, 0, 0, 2433, 0, 0, 445, 403, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 371, 0, 3597, 85, 0, 0, 28, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3983, 0, 0, 0, 0, 3676, 1297, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4105, 3281, 4103, 4053, 4121, 3984, 4973] avg=4096.0000 max=4973 max_rank=7 min=3281 min_rank=2 imbalance_ratio=1.214111 max_load_reduction=27795 max_load_reduction_pct=0.848236 moved_to_echo=17131 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3920, 59, 0, 0, 0, 0, 0, 0, 23, 4047, 7, 0, 0, 0, 155, 3615, 1, 0, 441, 398, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 2, 8, 0, 3596, 80, 0, 0, 38, 0, 4094, 70, 0, 0, 0, 0, 0, 470, 0, 0, 3944, 0, 0, 0, 0, 3684, 14, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3979, 4077, 4610, 4102, 3686, 4202, 4414, 3698] avg=4096.0000 max=4610 max_rank=2 min=3686 min_rank=4 imbalance_ratio=1.125488 max_load_reduction=28158 max_load_reduction_pct=0.859314 moved_to_echo=16000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 25984, 128, 0, 3584, 3200, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 3328, 0, 28800, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 29568, 3328, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 32384, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32384, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 1024, 128, 128, 31872, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 27, 0, 4089, 4063, 4084, 0, 0, 0, 0, 1, 4096, 0, 23, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 5, 0, 4084, 4048, 4082, 0, 0, 6, 0, 56, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 32, 0, 4079, 4053, 4095, 0, 0, 2, 0, 28, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4, 4095, 4096, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 1, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4, 98, 0, 4096, 4092, 4000, 0, 0, 0, 0, 1, 4092, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 4, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1, 4094, 4072, 4095, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 4075, 4096, 0, 0, 0, 0, 2, 4096, 0, 22, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 54, 1, 478, 0, 0, 0, 0, 0, 0, 1, 4096, 10, 0, 323, 0, 4046, 3279, 4026, 0, 0, 2, 0, 34, 4085, 0, 45, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4027, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 27, 0, 3959, 69, 10, 4, 0, 4063, 125, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 0, 0, 0, 4086, 0, 23, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4096, 4096, 4069, 4188, 4086, 4109, 4096] avg=4096.0000 max=4188 max_rank=4 min=4028 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 32, 0, 3970, 69, 10, 4, 0, 4053, 125, 0, 0, 0, 0, 0, 0, 2, 0, 28, 4075, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4096, 4095, 4085, 4178, 4105, 4086, 4096] avg=4096.0000 max=4178 max_rank=4 min=4027 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4027, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 5, 0, 3957, 69, 10, 4, 0, 4048, 125, 0, 0, 0, 0, 0, 0, 6, 0, 56, 4080, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4096, 4096, 4051, 4173, 4142, 4086, 4096] avg=4096.0000 max=4173 max_rank=4 min=4028 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4027, 4, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 1, 3970, 69, 10, 4, 0, 4072, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 4086, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4095, 4096, 4055, 4197, 4091, 4097, 4096] avg=4096.0000 max=4197 max_rank=4 min=4041 min_rank=0 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4026, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4, 3967, 69, 10, 4, 0, 4096, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4097, 4096, 4055, 4221, 4091, 4086, 4096] avg=4096.0000 max=4221 max_rank=4 min=4026 min_rank=0 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4, 98, 0, 3878, 69, 10, 4, 0, 4092, 122, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4096, 4096, 4064, 4214, 4093, 4082, 4096] avg=4096.0000 max=4214 max_rank=4 min=4027 min_rank=0 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4024, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 72, 16, 5, 0, 4075, 120, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4081, 0, 0, 0, 4080, 0, 22, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4096, 4096, 4069, 4195, 4083, 4102, 4096] avg=4096.0000 max=4195 max_rank=4 min=4031 min_rank=0 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4027, 54, 1, 478, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 10, 0, 323, 0, 3903, 69, 10, 5, 0, 3279, 123, 0, 0, 0, 0, 0, 0, 2, 0, 34, 4041, 0, 0, 0, 4075, 0, 45, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4560, 4096, 4097, 4320, 3402, 4077, 4120, 4096] avg=4096.0000 max=4560 max_rank=0 min=3402 min_rank=4 imbalance_ratio=1.113281 max_load_reduction=28208 max_load_reduction_pct=0.860840 moved_to_echo=15695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32256, 128, 128, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 31872, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 512, 128, 31616, 640, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 125, 0, 0, 4096, 0, 0, 4094, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3969, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 3, 0, 0, 4096, 22, 87, 0, 4084, 12, 0, 4096, 0, 0, 4, 0, 4096, 0, 0, 0, 3, 4092, 0, 0, 4096, 0, 0, 3985, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4, 0, 0, 4095, 5, 1, 0, 4087, 0, 1, 4096, 0, 0, 0, 1, 4095, 1, 0, 0, 0, 4095, 0, 1, 4095, 0, 1, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 4096, 3, 69, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4024, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 15, 1, 0, 4096, 0, 0, 0, 4096, 2, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4086, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 68, 0, 0, 4096, 0, 0, 0, 4053, 0, 0, 4074, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4095, 4, 3, 1, 4090, 2, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 1, 0, 4094, 0, 0, 4096, 0, 0, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 2, 2, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 125, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3969, 0, 31, 10, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4085, 4190, 4095, 4100, 4095, 4097, 4010] avg=4096.0000 max=4190 max_rank=2 min=4010 min_rank=7 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4085, 0, 0, 0, 0, 3, 69, 0, 4062, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4024, 0, 31, 10, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4088, 4134, 4097, 4096, 4095, 4097, 4065] avg=4096.0000 max=4134 max_rank=2 min=4065 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4084, 0, 0, 0, 0, 5, 1, 0, 4056, 0, 0, 0, 0, 0, 1, 4096, 0, 1, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 1, 4096, 0, 31, 10, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4088, 4062, 4098, 4096, 4095, 4097, 4138] avg=4096.0000 max=4138 max_rank=7 min=4062 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 15, 1, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 2, 0, 4088, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4086, 0, 31, 10, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4065, 4091, 4096, 4095, 4097, 4127] avg=4096.0000 max=4127 max_rank=7 min=4065 min_rank=2 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4, 3, 1, 4059, 0, 0, 0, 0, 2, 0, 4096, 2, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 31, 10, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4084, 4067, 4101, 4096, 4096, 4095, 4133] avg=4096.0000 max=4133 max_rank=7 min=4067 min_rank=2 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4085, 0, 0, 0, 0, 22, 87, 0, 4053, 0, 0, 0, 0, 12, 0, 4096, 0, 1, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3985, 0, 31, 10, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4088, 4162, 4109, 4100, 4098, 4093, 4026] avg=4096.0000 max=4162 max_rank=2 min=4026 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 2, 2, 0, 4061, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 2, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 33, 11, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4081, 4065, 4102, 4088, 4098, 4096, 4136] avg=4096.0000 max=4136 max_rank=7 min=4065 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 4074, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 30, 10, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4153, 4023, 4075, 4096, 4096, 4096, 4133] avg=4096.0000 max=4153 max_rank=1 min=4023 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 256, 128, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32512, 0, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 10, 4096, 4096, 4095, 0, 0, 244, 0, 0, 4096, 0, 3842] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4095, 1, 1, 0, 0, 0, 2, 1, 0, 4, 1, 0, 4094, 0, 0, 5, 9, 4091, 1, 1, 0, 4094, 4095, 4083, 0, 0, 1388, 0, 0, 4095, 0, 2707] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4096, 0, 9, 15, 4091, 4096, 4088, 0, 0, 1214, 0, 0, 4096, 0, 2867] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 200, 4096, 0, 0, 10, 4094, 4096, 4072, 0, 0, 393, 0, 0, 4096, 0, 3519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 35, 4096, 0, 12, 6, 4096, 4096, 3271, 0, 0, 1099, 0, 0, 4096, 0, 3769] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 0, 9, 171, 4096, 0, 41, 0, 4090, 4096, 3985, 0, 0, 806, 0, 0, 4096, 0, 3186] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 3, 5, 4096, 4096, 4090, 0, 0, 164, 0, 1, 4096, 0, 3930] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 4096, 4091, 0, 0, 125, 0, 0, 4096, 0, 3971] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3977, 119, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 10, 119, 0, 4095, 0, 0, 0, 0, 0, 0, 244, 0, 3354, 60, 2, 0, 0, 623, 0, 3842, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4095, 4036, 4224, 3660, 4465] avg=4096.0000 max=4465 max_rank=7 min=3660 min_rank=6 imbalance_ratio=1.090088 max_load_reduction=28303 max_load_reduction_pct=0.863739 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4094, 0, 0, 0, 0, 4, 1, 0, 3975, 119, 1, 0, 4093, 0, 0, 5, 0, 0, 0, 0, 9, 4031, 1, 1, 0, 0, 0, 0, 0, 119, 0, 4083, 0, 0, 0, 0, 0, 0, 1388, 0, 3353, 60, 1, 0, 0, 623, 0, 2707, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4097, 4100, 4098, 4042, 4202, 4802, 3330] avg=4096.0000 max=4802 max_rank=6 min=3330 min_rank=7 imbalance_ratio=1.172363 max_load_reduction=27966 max_load_reduction_pct=0.853455 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3972, 119, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 0, 0, 4036, 0, 9, 0, 0, 0, 0, 15, 119, 0, 4088, 0, 0, 0, 0, 0, 0, 1214, 0, 3354, 60, 2, 0, 0, 623, 0, 2867, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4091, 4098, 4045, 4222, 4630, 3490] avg=4096.0000 max=4630 max_rank=6 min=3490 min_rank=7 imbalance_ratio=1.130371 max_load_reduction=28138 max_load_reduction_pct=0.858704 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3975, 119, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 200, 4036, 0, 0, 0, 0, 0, 0, 10, 119, 0, 4072, 0, 0, 0, 0, 0, 0, 393, 0, 3354, 60, 2, 0, 0, 623, 0, 3519, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4094, 4094, 4236, 4201, 3809, 4142] avg=4096.0000 max=4236 max_rank=4 min=3809 min_rank=6 imbalance_ratio=1.034180 max_load_reduction=28532 max_load_reduction_pct=0.870728 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3977, 119, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 3, 0, 0, 0, 0, 5, 119, 0, 4090, 0, 0, 0, 0, 0, 0, 164, 0, 3354, 60, 1, 0, 1, 623, 0, 3930, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4097, 4094, 4039, 4214, 3579, 4554] avg=4096.0000 max=4554 max_rank=7 min=3579 min_rank=6 imbalance_ratio=1.111816 max_load_reduction=28214 max_load_reduction_pct=0.861023 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 3971, 119, 0, 0, 4094, 1, 0, 9, 0, 0, 0, 0, 171, 4036, 0, 41, 0, 0, 0, 0, 0, 119, 0, 3985, 0, 0, 0, 0, 0, 0, 806, 0, 3354, 60, 1, 0, 0, 623, 0, 3186, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4091, 4104, 4248, 4104, 4221, 3809] avg=4096.0000 max=4248 max_rank=4 min=3809 min_rank=7 imbalance_ratio=1.037109 max_load_reduction=28520 max_load_reduction_pct=0.870361 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3982, 121, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 5, 4034, 0, 0, 0, 0, 0, 0, 0, 114, 0, 4091, 0, 0, 0, 0, 0, 0, 125, 0, 3360, 62, 5, 0, 0, 615, 0, 3971, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4103, 4091, 4039, 4205, 3552, 4586] avg=4096.0000 max=4586 max_rank=7 min=3552 min_rank=6 imbalance_ratio=1.119629 max_load_reduction=28182 max_load_reduction_pct=0.860046 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3977, 119, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 35, 4036, 0, 12, 0, 0, 0, 0, 6, 119, 0, 3271, 0, 0, 0, 0, 0, 0, 1099, 0, 3354, 60, 2, 0, 0, 623, 0, 3769, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4094, 4083, 3396, 4515, 4392] avg=4096.0000 max=4515 max_rank=6 min=3396 min_rank=5 imbalance_ratio=1.102295 max_load_reduction=28253 max_load_reduction_pct=0.862213 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 4992, 0, 27904, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 5504, 0, 26880, 512, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 1024, 0, 31872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 32384, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 31872, 1024, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 184, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3914, 0, 4096, 0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 403, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 3694, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 298, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 3798, 0, 4094, 0, 0, 0, 4096, 4095, 0, 0, 4096, 2, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1, 2, 818, 4096, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 3293, 4, 4096, 0, 0, 0, 4075, 4096, 0, 0, 4096, 1, 0, 4096, 1, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1130, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 2961, 5, 4096, 0, 0, 0, 4096, 4096, 0, 1, 4088, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 828, 4095, 0, 1, 1, 1, 0, 0, 0, 4094, 6, 1, 0, 3258, 7, 4095, 0, 1, 1, 4089, 4093, 0, 0, 4095, 2, 1, 4095, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3124, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 980, 5, 4096, 0, 0, 0, 4080, 4087, 0, 12, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 24, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4072, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 184, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3914, 844, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3430, 4094, 4096, 4760, 4094, 4102, 4096, 4096] avg=4096.0000 max=4760 max_rank=3 min=3430 min_rank=0 imbalance_ratio=1.162109 max_load_reduction=28008 max_load_reduction_pct=0.854736 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 298, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3798, 844, 2, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3544, 4095, 4097, 4644, 4092, 4102, 4098, 4096] avg=4096.0000 max=4644 max_rank=3 min=3544 min_rank=0 imbalance_ratio=1.133789 max_load_reduction=28124 max_load_reduction_pct=0.858276 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 403, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 3694, 844, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3649, 4096, 4096, 4542, 4094, 4102, 4093, 4096] avg=4096.0000 max=4542 max_rank=3 min=3649 min_rank=0 imbalance_ratio=1.108887 max_load_reduction=28226 max_load_reduction_pct=0.861389 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 818, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 3293, 844, 2, 0, 0, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 4075, 6, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4096, 4092, 4139, 4098, 4081, 4097, 4098] avg=4096.0000 max=4139 max_rank=3 min=4067 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1130, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 2961, 844, 2, 0, 0, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4376, 4096, 4096, 3814, 4099, 4102, 4089, 4096] avg=4096.0000 max=4376 max_rank=0 min=3814 min_rank=3 imbalance_ratio=1.068359 max_load_reduction=28392 max_load_reduction_pct=0.866455 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 828, 3245, 0, 0, 0, 0, 0, 1, 1, 1, 4087, 6, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 6, 1, 0, 3258, 844, 2, 0, 0, 7, 4093, 0, 1, 0, 0, 0, 0, 1, 4089, 6, 0, 0, 0, 0, 0, 0, 4095, 2, 1, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4096, 4094, 4111, 4101, 4096, 4098, 4097] avg=4096.0000 max=4111 max_rank=3 min=4075 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 24, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 13, 0, 0, 0, 0, 0, 4096, 3, 2, 0, 0, 0, 0, 0, 4072, 850, 5, 5, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3257, 4108, 4101, 4932, 4091, 4097, 4089, 4093] avg=4096.0000 max=4932 max_rank=3 min=3257 min_rank=0 imbalance_ratio=1.204102 max_load_reduction=27836 max_load_reduction_pct=0.849487 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3124, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 980, 844, 2, 0, 0, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 4080, 6, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6370, 4087, 4096, 1826, 4099, 4086, 4108, 4096] avg=4096.0000 max=6370 max_rank=0 min=1826 min_rank=3 imbalance_ratio=1.555176 max_load_reduction=26398 max_load_reduction_pct=0.805603 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 6912, 25984, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 25984, 6784, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 58, 0, 0, 4039, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 4, 0, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 10, 0, 0, 4094, 2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4094, 41, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4055, 0, 15, 0, 2, 0, 0, 4093, 0, 0, 0, 4096, 4, 4080, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1, 4095, 20, 1, 0, 4075, 0, 4095, 0, 0, 0, 1, 4095, 1, 1, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 4095, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 15, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 3, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 1, 4093, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 666, 0, 0, 3445, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4061, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3996, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 3995, 99, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4184, 4095, 4096, 4103, 4094, 4096, 4104] avg=4096.0000 max=4184 max_rank=1 min=3996 min_rank=0 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3996, 58, 0, 0, 0, 0, 0, 0, 4039, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4, 1, 0, 0, 0, 0, 2, 0, 0, 3995, 99, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4091, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4133, 4095, 4096, 4099, 4096, 4096, 4099] avg=4096.0000 max=4133 max_rank=1 min=4054 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3996, 10, 0, 0, 0, 0, 0, 0, 4094, 2, 94, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3995, 99, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 8, 0, 4087, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4190, 4095, 4096, 4094, 4094, 4098, 4095] avg=4096.0000 max=4190 max_rank=1 min=4006 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3994, 41, 0, 0, 0, 0, 0, 0, 4096, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 15, 1, 0, 0, 0, 0, 2, 0, 0, 3995, 99, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 8, 4, 4080, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4190, 4095, 4096, 4071, 4096, 4093, 4092] avg=4096.0000 max=4190 max_rank=1 min=4035 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3996, 15, 0, 0, 0, 0, 0, 0, 4094, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 3995, 99, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 8, 1, 4093, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4188, 4095, 4096, 4087, 4094, 4095, 4102] avg=4096.0000 max=4188 max_rank=1 min=4011 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 3995, 20, 1, 0, 0, 0, 0, 0, 4075, 0, 93, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 7, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 3995, 99, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4095, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4168, 4095, 4097, 4097, 4095, 4095, 4104] avg=4096.0000 max=4168 max_rank=1 min=4017 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3987, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 14, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4002, 103, 0, 0, 4096, 0, 0, 0, 2, 1, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3987, 4176, 4103, 4095, 4100, 4105, 4099, 4103] avg=4096.0000 max=4176 max_rank=1 min=3987 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3996, 666, 0, 0, 0, 0, 0, 0, 3445, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 20, 1, 0, 0, 0, 0, 0, 0, 0, 3995, 99, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4061, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4662, 3539, 4095, 4096, 4117, 4094, 4096, 4069] avg=4096.0000 max=4662 max_rank=0 min=3539 min_rank=1 imbalance_ratio=1.138184 max_load_reduction=28106 max_load_reduction_pct=0.857727 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32000, 896, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32000, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32128, 128, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4059, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4045, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4092, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 3768, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4065, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4095, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 1, 0, 0, 0, 4095, 4095, 4094, 1, 4095, 4095, 0, 2, 0, 1, 0, 0, 0, 0, 1, 2, 0, 0, 0, 33, 4062, 1, 4095, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 4084, 4095, 4096, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4083, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4048, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 188, 3955, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 3962, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 332, 3992, 0, 0, 0, 3768, 0, 88, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4103, 4096, 4101, 4096, 4324, 3856] avg=4096.0000 max=4324 max_rank=6 min=3856 min_rank=7 imbalance_ratio=1.055664 max_load_reduction=28444 max_load_reduction_pct=0.868042 moved_to_echo=16377 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4059, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 37, 3992, 0, 0, 0, 4096, 0, 88, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4070, 4096, 4101, 4096, 4029, 4184] avg=4096.0000 max=4184 max_rank=7 min=4029 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16410 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4065, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 32, 3992, 0, 0, 0, 4095, 0, 88, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4076, 4096, 4101, 4096, 4024, 4183] avg=4096.0000 max=4183 max_rank=7 min=4024 min_rank=6 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16404 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 52, 3992, 0, 0, 0, 4045, 0, 88, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4107, 4096, 4100, 4096, 4044, 4133] avg=4096.0000 max=4133 max_rank=7 min=4044 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 4084, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4085, 16, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 22, 3992, 0, 0, 0, 4083, 0, 88, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4095, 4096, 4104, 4096, 4014, 4171] avg=4096.0000 max=4171 max_rank=7 min=4014 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16385 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 11, 1, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 4083, 15, 0, 0, 0, 0, 1, 2, 4095, 0, 0, 0, 0, 0, 0, 33, 3991, 0, 0, 0, 4062, 1, 89, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4107, 4097, 4099, 4098, 4024, 4153] avg=4096.0000 max=4153 max_rank=7 min=4024 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16374 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 17, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 134, 3995, 0, 0, 0, 3962, 0, 83, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4103, 4095, 4107, 4095, 4129, 4045] avg=4096.0000 max=4129 max_rank=6 min=4045 min_rank=7 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16376 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4037, 16, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 188, 3992, 0, 0, 0, 3955, 0, 88, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4107, 4096, 4055, 4095, 4180, 4043] avg=4096.0000 max=4180 max_rank=6 min=4043 min_rank=7 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32128, 128, 768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 896, 32000, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32640, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 1, 1, 0, 0, 4095, 0, 1, 4095, 0, 4095, 16, 1, 4095, 4090, 0, 0, 2, 1, 5, 1, 1, 0, 4095, 4077, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 33, 0, 0, 4096, 0, 0, 4096, 0, 4089, 0, 0, 4096, 4065, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 54, 0, 4096, 0, 4096, 0, 0, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 4096, 0, 4096, 20, 0, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 1, 4095, 2, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4, 4096, 0, 0, 0, 0, 31, 0, 0, 4096, 16, 0, 4096, 0, 4093, 0, 0, 4096, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 8, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 17, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4086, 4090, 4098, 4111, 4095, 4093, 4099] avg=4096.0000 max=4111 max_rank=4 min=4086 min_rank=1 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4078, 8, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 2, 0, 4089, 0, 0, 0, 0, 0, 0, 17, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 5, 0, 0, 0, 4092, 1, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4119, 4088, 4091, 4082, 4095, 4098, 4099] avg=4096.0000 max=4119 max_rank=1 min=4082 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4078, 8, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 2, 1, 4095, 0, 0, 0, 0, 2, 0, 17, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4087, 4088, 4098, 4112, 4095, 4093, 4099] avg=4096.0000 max=4112 max_rank=4 min=4087 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 8, 0, 0, 0, 0, 4088, 54, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 17, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4086, 4142, 4096, 4059, 4097, 4094, 4098] avg=4096.0000 max=4142 max_rank=2 min=4059 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4077, 8, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 1, 2, 0, 4095, 0, 0, 0, 0, 16, 1, 17, 4090, 0, 0, 0, 0, 0, 0, 2, 1, 4093, 0, 0, 0, 5, 1, 1, 0, 4091, 1, 0, 0, 4, 4077, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4087, 4087, 4098, 4124, 4096, 4099, 4082] avg=4096.0000 max=4124 max_rank=4 min=4082 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4078, 8, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 20, 0, 17, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4087, 4087, 4098, 4113, 4095, 4093, 4099] avg=4096.0000 max=4113 max_rank=4 min=4087 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 14, 3, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4094, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4099, 4082, 4096, 4102, 4098, 4102, 4096] avg=4096.0000 max=4102 max_rank=4 min=4082 min_rank=2 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4078, 8, 0, 0, 0, 0, 4088, 16, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 17, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4117, 4104, 4093, 4065, 4097, 4096, 4096] avg=4096.0000 max=4117 max_rank=1 min=4065 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32640, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 4096, 0, 4096, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4078, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[923, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3162, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[685, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 41, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3371, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[28, 2, 0, 0, 0, 0, 4095, 4, 1, 0, 0, 1, 4094, 0, 4093, 1, 0, 0, 0, 4094, 2, 0, 0, 0, 3, 4092, 0, 4095, 4068, 1, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[19, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 0, 4096, 0, 4096, 0, 0, 133, 0, 4096, 0, 0, 0, 0, 0, 4080, 0, 4096, 3944, 0, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[598, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 4096, 0, 0, 157, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3336, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[307, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3789, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 3730, 43, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4078, 0, 0, 366, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3776, 4096, 4100, 4096, 4061, 4096, 4099, 4444] avg=4096.0000 max=4444 max_rank=7 min=3776 min_rank=0 imbalance_ratio=1.084961 max_load_reduction=28324 max_load_reduction_pct=0.864380 moved_to_echo=16700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[923, 0, 0, 0, 3730, 43, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 11, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 3162, 0, 0, 366, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4698, 4096, 4091, 4096, 4064, 4096, 4099, 3528] avg=4096.0000 max=4698 max_rank=0 min=3528 min_rank=7 imbalance_ratio=1.146973 max_load_reduction=28070 max_load_reduction_pct=0.856628 moved_to_echo=15787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[685, 0, 0, 0, 3729, 43, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 41, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 3371, 0, 0, 366, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4459, 4096, 4091, 4096, 4094, 4096, 4099, 3737] avg=4096.0000 max=4459 max_rank=0 min=3737 min_rank=7 imbalance_ratio=1.088623 max_load_reduction=28309 max_load_reduction_pct=0.863922 moved_to_echo=16026 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[28, 2, 0, 0, 3728, 43, 2, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 1, 0, 0, 1, 4090, 0, 0, 0, 1, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 2, 0, 0, 0, 4093, 0, 0, 0, 3, 4092, 0, 3, 0, 0, 0, 0, 4068, 1, 0, 366, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3803, 4099, 4092, 4095, 4051, 4095, 4098, 4435] avg=4096.0000 max=4435 max_rank=7 min=3803 min_rank=0 imbalance_ratio=1.082764 max_load_reduction=28333 max_load_reduction_pct=0.864655 moved_to_echo=16679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3730, 43, 2, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 366, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3775, 4097, 4090, 4097, 4053, 4095, 4099, 4462] avg=4096.0000 max=4462 max_rank=7 min=3775 min_rank=0 imbalance_ratio=1.089355 max_load_reduction=28306 max_load_reduction_pct=0.863831 moved_to_echo=16709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 0, 0, 3725, 43, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 133, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4080, 0, 2, 0, 0, 0, 0, 3944, 0, 0, 365, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3789, 4096, 4113, 4096, 4186, 4097, 4082, 4309] avg=4096.0000 max=4309 max_rank=7 min=3789 min_rank=0 imbalance_ratio=1.052002 max_load_reduction=28459 max_load_reduction_pct=0.868500 moved_to_echo=16674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[307, 0, 0, 0, 3733, 48, 3, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3789, 0, 0, 362, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4092, 4093, 4096, 4049, 4096, 4096, 4151] avg=4096.0000 max=4151 max_rank=7 min=4049 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[598, 0, 0, 0, 3730, 43, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 157, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3336, 0, 0, 366, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4373, 4096, 4098, 4096, 4210, 4097, 4096, 3702] avg=4096.0000 max=4373 max_rank=0 min=3702 min_rank=7 imbalance_ratio=1.067627 max_load_reduction=28395 max_load_reduction_pct=0.866547 moved_to_echo=16105 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2688, 128, 0, 0, 29952, 384, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[29952, 128, 0, 2944, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 384, 0, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 10, 4085, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 4074, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 24] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 87, 4008, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4079, 0, 18, 3988, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 107] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 0, 4095, 1, 0, 1, 0, 3, 4092, 0, 0, 4095, 1, 1, 2, 1, 4094, 0, 0, 4095, 0, 1, 1, 4094, 4094, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4024, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 72] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4075, 0, 89, 4008, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 20] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4072, 28, 0, 0, 24, 0, 10, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4101, 4119, 4096, 4096, 4096, 4068] avg=4096.0000 max=4119 max_rank=3 min=4068 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16356 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 28, 0, 0, 24, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4068, 0, 0, 24, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4098, 4098, 4095, 4097, 4096, 4092] avg=4096.0000 max=4098 max_rank=2 min=4092 min_rank=7 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16380 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 28, 0, 0, 24, 0, 87, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4068, 0, 0, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4095, 4119, 4096, 4096, 4096, 4074] avg=4096.0000 max=4119 max_rank=3 min=4074 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16362 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 28, 0, 0, 24, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4068, 0, 0, 72, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4100, 4048, 4096, 4096, 4096, 4140] avg=4096.0000 max=4140 max_rank=7 min=4048 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16428 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 3, 4068, 28, 0, 0, 24, 0, 0, 4095, 0, 0, 0, 0, 1, 1, 2, 1, 4094, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4100, 4119, 4099, 4095, 4096, 4066] avg=4096.0000 max=4119 max_rank=3 min=4066 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16356 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 28, 0, 0, 24, 0, 18, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4068, 0, 0, 107, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4083, 4030, 4096, 4096, 4096, 4175] avg=4096.0000 max=4175 max_rank=7 min=4030 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16463 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 31, 1, 1, 18, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4111, 4114, 4095, 4096, 4096, 4065] avg=4096.0000 max=4114 max_rank=3 min=4065 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16352 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 28, 1, 0, 24, 0, 89, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4068, 0, 0, 20, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4080, 4121, 4095, 4096, 4096, 4088] avg=4096.0000 max=4121 max_rank=3 min=4080 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16375 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 0, 256, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 32640, 256, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32640, 0, 0, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 188, 2630, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3922, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1454, 22, 4075, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 625, 159, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3312, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 159, 2973, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3968, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1092, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 238, 1999, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1860, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 198, 1172, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3970, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2852, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 160, 2212, 1, 1, 4092, 0, 0, 4, 1, 1, 0, 4094, 0, 0, 4092, 4095, 4095, 2, 4094, 0, 0, 0, 1, 0, 2, 1, 1726, 0, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 321, 738, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3948, 4096, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 3184, 1, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 2047, 1815, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4093, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 238, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 159, 2973, 1893, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 71, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3968, 0, 73, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1092, 0, 2132, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5025, 4096, 4094, 4096, 4041, 4096, 4096, 3224] avg=4096.0000 max=5025 max_rank=0 min=3224 min_rank=7 imbalance_ratio=1.226807 max_load_reduction=27743 max_load_reduction_pct=0.846649 moved_to_echo=15457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 188, 2630, 1883, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 71, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3922, 0, 73, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1454, 22, 2121, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4701, 4096, 4095, 4093, 3995, 4096, 4095, 3597] avg=4096.0000 max=4701 max_rank=0 min=3597 min_rank=7 imbalance_ratio=1.147705 max_load_reduction=28067 max_load_reduction_pct=0.856537 moved_to_echo=15783 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 625, 159, 1893, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 71, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 73, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3312, 0, 2132, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2677, 4096, 4094, 4096, 4169, 4096, 4096, 5444] avg=4096.0000 max=5444 max_rank=7 min=2677 min_rank=0 imbalance_ratio=1.329102 max_load_reduction=27324 max_load_reduction_pct=0.833862 moved_to_echo=17805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 198, 1172, 1893, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 71, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3970, 0, 73, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2852, 0, 2132, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3263, 4096, 4094, 4096, 4043, 4096, 4096, 4984] avg=4096.0000 max=4984 max_rank=7 min=3263 min_rank=0 imbalance_ratio=1.216797 max_load_reduction=27784 max_load_reduction_pct=0.847900 moved_to_echo=17219 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 321, 738, 1892, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 71, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3948, 0, 73, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3184, 1, 2132, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2951, 4096, 4094, 4096, 4021, 4097, 4096, 5317] avg=4096.0000 max=5317 max_rank=7 min=2951 min_rank=0 imbalance_ratio=1.298096 max_load_reduction=27451 max_load_reduction_pct=0.837738 moved_to_echo=17531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 160, 2212, 1892, 0, 0, 0, 1, 1, 4092, 0, 0, 0, 0, 0, 0, 4, 1, 1, 4022, 71, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4092, 0, 73, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 4095, 0, 0, 0, 1726, 0, 2131, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4264, 4094, 4099, 4094, 4167, 4094, 4099, 3857] avg=4096.0000 max=4264 max_rank=0 min=3857 min_rank=7 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16217 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 238, 1999, 1895, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4025, 77, 1, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 4095, 0, 71, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1860, 0, 2115, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4098, 4103, 4102, 4166, 4097, 4095, 3975] avg=4096.0000 max=4166 max_rank=4 min=3975 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2047, 1815, 1893, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 71, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4093, 0, 73, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 238, 0, 2132, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5755, 4096, 4095, 4095, 4166, 4096, 4095, 2370] avg=4096.0000 max=5755 max_rank=0 min=2370 min_rank=7 imbalance_ratio=1.405029 max_load_reduction=27013 max_load_reduction_pct=0.824371 moved_to_echo=14727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 3968, 13824, 15232, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15744, 128, 17152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32256, 0, 640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32256, 640, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4092, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4094, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1, 4095, 1, 1, 4095, 0, 4092, 0, 0, 1, 0, 1, 1, 4095, 0, 0, 1, 4095, 0, 0, 0, 0, 9, 1, 0, 4079, 1, 4094, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 4096, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 4, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4095, 4096, 4096, 4096, 4094, 4099] avg=4096.0000 max=4099 max_rank=7 min=4094 min_rank=6 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4094, 4098] avg=4096.0000 max=4098 max_rank=7 min=4094 min_rank=6 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4097, 4096, 4096, 4097, 4094, 4096] avg=4096.0000 max=4097 max_rank=2 min=4094 min_rank=6 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4094, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 4092, 1, 0, 0, 4079, 1, 2, 10, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4092, 4097, 4096, 4095, 4103, 4092] avg=4096.0000 max=4103 max_rank=6 min=4092 min_rank=2 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 1, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4097, 4095, 4096, 4096, 4096, 4098] avg=4096.0000 max=4098 max_rank=7 min=4094 min_rank=0 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4095, 4096, 4096, 4097, 4095, 4095, 4096] avg=4096.0000 max=4098 max_rank=0 min=4095 min_rank=1 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4099, 4096, 4095, 4097, 4096, 4093] avg=4096.0000 max=4099 max_rank=2 min=4093 min_rank=7 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True + [2026-08-30 04:55:23] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 669.8 | throughput per GPU (TFLOP/s/GPU): 497.7 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.039575E+01 | loss scale: 1.0 | grad norm: 16.806 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[9, 4007, 1303, 1, 809, 0, 700, 0, 2024, 813, 16, 2657, 1728, 664, 5, 2145, 2041, 1368, 175, 249, 50, 2649, 706, 4, 1, 353, 299, 1618, 2518, 24, 3831, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[0, 3867, 1635, 1, 1036, 4, 542, 7, 2073, 339, 3, 2250, 2032, 963, 1, 2671, 2000, 1777, 405, 508, 19, 2566, 932, 9, 10, 564, 302, 1163, 1495, 27, 3567, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 4053, 1496, 1, 1057, 0, 634, 0, 1386, 808, 5, 2443, 1493, 1281, 0, 2904, 2377, 1165, 86, 231, 15, 3165, 899, 3, 0, 234, 179, 1394, 1598, 14, 3847, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 3910, 1579, 3, 709, 0, 486, 0, 1836, 952, 7, 2534, 1696, 924, 0, 2933, 2096, 1802, 193, 302, 66, 3002, 801, 0, 0, 209, 205, 1173, 1802, 20, 3520, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3, 3949, 1851, 0, 1000, 2, 565, 0, 2140, 601, 11, 2302, 1958, 767, 5, 2555, 2249, 1120, 446, 232, 38, 2106, 851, 2, 0, 325, 191, 1505, 2392, 32, 3570, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 4042, 1369, 0, 888, 0, 718, 0, 2095, 914, 5, 2191, 2478, 1001, 0, 2820, 1568, 1094, 207, 360, 43, 3006, 729, 5, 0, 191, 217, 1133, 1815, 15, 3864, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[6, 4015, 1953, 0, 828, 0, 810, 3, 1458, 907, 14, 2609, 2171, 947, 2, 2086, 1520, 1972, 314, 251, 40, 2081, 865, 2, 5, 234, 195, 1680, 2139, 24, 3637, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[5, 4044, 1545, 0, 924, 0, 526, 2, 2389, 653, 6, 2726, 2591, 458, 8, 2517, 1860, 855, 214, 360, 61, 3218, 851, 0, 1, 369, 242, 1025, 1396, 18, 3892, 12] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[9, 2515, 1303, 1, 0, 0, 0, 0, 809, 0, 700, 0, 1586, 1031, 0, 0, 2024, 813, 16, 1526, 0, 0, 0, 0, 1728, 664, 5, 998, 0, 0, 0, 0, 2041, 1368, 175, 249, 184, 0, 0, 0, 50, 2649, 706, 4, 537, 0, 0, 0, 1, 353, 299, 1618, 1147, 461, 410, 0, 2518, 24, 2245, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3828, 4126, 4379, 3395, 4017, 3946, 4289, 4788] avg=4096.0000 max=4788 max_rank=7 min=3395 min_rank=3 imbalance_ratio=1.168945 max_load_reduction=27980 max_load_reduction_pct=0.853882 moved_to_echo=17040 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2544, 1496, 1, 0, 0, 0, 0, 1057, 0, 634, 0, 1592, 1043, 0, 0, 1386, 808, 5, 1404, 0, 0, 0, 0, 1493, 1281, 0, 1351, 0, 0, 0, 0, 2377, 1165, 86, 231, 169, 0, 0, 0, 15, 3165, 899, 3, 493, 0, 0, 0, 0, 234, 179, 1394, 1553, 466, 377, 0, 1598, 14, 2255, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4326, 3603, 4125, 4028, 4575, 4203, 3867] avg=4096.0000 max=4575 max_rank=5 min=3603 min_rank=2 imbalance_ratio=1.116943 max_load_reduction=28193 max_load_reduction_pct=0.860382 moved_to_echo=16673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2427, 1635, 1, 0, 0, 0, 0, 1036, 4, 542, 7, 1477, 995, 0, 0, 2073, 339, 3, 1293, 0, 0, 0, 0, 2032, 963, 1, 1243, 0, 0, 0, 0, 2000, 1777, 405, 508, 156, 0, 0, 0, 19, 2566, 932, 9, 454, 0, 0, 0, 10, 564, 302, 1163, 1428, 445, 347, 0, 1495, 27, 2090, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4061, 3708, 4239, 4846, 3980, 4259, 3612] avg=4096.0000 max=4846 max_rank=4 min=3612 min_rank=7 imbalance_ratio=1.183105 max_load_reduction=27922 max_load_reduction_pct=0.852112 moved_to_echo=16697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[6, 2520, 1953, 0, 0, 0, 0, 0, 828, 0, 810, 3, 1506, 1033, 0, 0, 1458, 907, 14, 1499, 0, 0, 0, 0, 2171, 947, 2, 971, 0, 0, 0, 0, 1520, 1972, 314, 251, 181, 0, 0, 0, 40, 2081, 865, 2, 527, 0, 0, 0, 5, 234, 195, 1680, 1115, 462, 402, 0, 2139, 24, 2131, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4479, 4180, 3878, 4091, 4238, 3515, 4093, 4294] avg=4096.0000 max=4479 max_rank=0 min=3515 min_rank=5 imbalance_ratio=1.093506 max_load_reduction=28289 max_load_reduction_pct=0.863312 moved_to_echo=16140 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2537, 1369, 0, 0, 0, 0, 0, 888, 0, 718, 0, 1600, 1040, 0, 0, 2095, 914, 5, 1259, 0, 0, 0, 0, 2478, 1001, 0, 1312, 0, 0, 0, 0, 1568, 1094, 207, 360, 152, 0, 0, 0, 43, 3006, 729, 5, 442, 0, 0, 0, 0, 191, 217, 1133, 1508, 465, 338, 0, 1815, 15, 2264, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3906, 4246, 4273, 4791, 3381, 4225, 3852, 4094] avg=4096.0000 max=4791 max_rank=3 min=3381 min_rank=4 imbalance_ratio=1.169678 max_load_reduction=27977 max_load_reduction_pct=0.853790 moved_to_echo=15552 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2447, 1579, 3, 0, 0, 0, 0, 709, 0, 486, 0, 1460, 1010, 0, 0, 1836, 952, 7, 1443, 0, 0, 0, 0, 1696, 924, 0, 1360, 0, 0, 0, 0, 2096, 1802, 193, 302, 180, 0, 0, 0, 66, 3002, 801, 0, 516, 0, 0, 0, 0, 209, 205, 1173, 1573, 453, 395, 0, 1802, 20, 2060, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 3665, 4238, 3980, 4573, 4385, 4008, 3890] avg=4096.0000 max=4573 max_rank=4 min=3665 min_rank=1 imbalance_ratio=1.116455 max_load_reduction=28195 max_load_reduction_pct=0.860443 moved_to_echo=16856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 2539, 1545, 0, 0, 0, 0, 0, 924, 0, 526, 2, 1611, 1040, 0, 0, 2389, 653, 6, 1567, 0, 0, 0, 0, 2591, 458, 8, 1171, 0, 0, 0, 0, 1860, 855, 214, 360, 189, 0, 0, 0, 61, 3218, 851, 0, 550, 0, 0, 0, 1, 369, 242, 1025, 1346, 465, 420, 0, 1396, 18, 2281, 12, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4103, 4615, 4228, 3478, 4680, 3868, 3707] avg=4096.0000 max=4680 max_rank=5 min=3478 min_rank=4 imbalance_ratio=1.142578 max_load_reduction=28088 max_load_reduction_pct=0.857178 moved_to_echo=15733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 2479, 1851, 0, 0, 0, 0, 0, 1000, 2, 565, 0, 1478, 1016, 0, 0, 2140, 601, 11, 1322, 0, 0, 0, 0, 1958, 767, 5, 1189, 0, 0, 0, 0, 2249, 1120, 446, 232, 160, 0, 0, 0, 38, 2106, 851, 2, 465, 0, 0, 0, 0, 325, 191, 1505, 1366, 454, 355, 0, 2392, 32, 2092, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4333, 4061, 4074, 3919, 4207, 3462, 4196, 4516] avg=4096.0000 max=4516 max_rank=7 min=3462 min_rank=5 imbalance_ratio=1.102539 max_load_reduction=28252 max_load_reduction_pct=0.862183 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7296, 128, 4992, 128, 12416, 8320, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 2560, 1920, 10752, 11136, 3712, 3072, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15744, 11264, 2048, 2560, 1408, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15488, 6016, 128, 11392, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 20096, 12800, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16256, 7040, 128, 9600, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[15232, 256, 17536, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 21888, 6656, 128, 4096, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[9, 4, 4089, 3892, 12, 0, 0, 4096, 348, 3694, 0, 0, 3, 1, 1745, 1, 0, 0, 4083, 2, 0, 0, 9, 4093, 0, 11, 3, 17, 2, 0, 2571, 4083] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[12, 0, 4089, 4025, 1, 0, 0, 4094, 262, 3227, 0, 0, 0, 0, 2596, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 1, 0, 31, 1, 0, 2147, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[53, 0, 4085, 3860, 1, 0, 8, 4087, 71, 3678, 0, 0, 0, 7, 1213, 0, 13, 0, 4073, 0, 27, 0, 0, 4095, 0, 0, 0, 7, 5, 0, 3401, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[28, 0, 4085, 4055, 0, 0, 0, 4096, 876, 3119, 0, 0, 0, 0, 1884, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 24, 17, 0, 2296, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[33, 2, 4089, 3678, 0, 3, 0, 4073, 22, 2674, 0, 0, 7, 0, 2420, 0, 0, 1, 4093, 3, 6, 0, 1, 4090, 0, 23, 3, 46, 3, 0, 3402, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[30, 0, 4095, 3709, 0, 0, 2, 4083, 137, 2830, 0, 0, 0, 5, 1499, 0, 33, 0, 4074, 1, 12, 1, 2, 4090, 0, 3, 0, 115, 123, 7, 3826, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[11, 0, 4095, 3560, 0, 0, 0, 4088, 397, 3685, 0, 0, 0, 0, 1043, 5, 6, 0, 4096, 0, 1, 0, 0, 4096, 0, 1, 0, 4, 22, 0, 3577, 4081] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[20, 1, 4055, 4049, 0, 0, 1, 4095, 214, 2801, 0, 0, 0, 0, 2096, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 0, 0, 3, 35, 45, 0, 3067, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[9, 4, 218, 3892, 0, 0, 0, 0, 12, 0, 0, 4096, 3, 0, 0, 0, 348, 3694, 0, 0, 589, 0, 0, 0, 3, 1, 1745, 1, 2276, 0, 0, 0, 0, 0, 4083, 2, 0, 0, 0, 0, 0, 0, 9, 4090, 0, 0, 0, 0, 0, 11, 3, 17, 3871, 186, 0, 0, 2, 0, 2571, 1032, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4111, 4631, 4026, 4085, 4099, 4088, 3605] avg=4096.0000 max=4631 max_rank=2 min=3605 min_rank=7 imbalance_ratio=1.130615 max_load_reduction=28137 max_load_reduction_pct=0.858673 moved_to_echo=15877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 218, 4025, 0, 0, 0, 0, 1, 0, 0, 4094, 3, 0, 0, 0, 262, 3227, 0, 0, 591, 1, 0, 0, 0, 0, 2596, 0, 2283, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 1, 0, 31, 3871, 187, 0, 0, 1, 0, 2147, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4255, 4098, 4081, 4879, 4091, 4092, 4090, 3182] avg=4096.0000 max=4879 max_rank=3 min=3182 min_rank=7 imbalance_ratio=1.191162 max_load_reduction=27889 max_load_reduction_pct=0.851105 moved_to_echo=15455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[30, 0, 218, 3709, 0, 0, 0, 0, 0, 0, 2, 4083, 3, 0, 0, 0, 137, 2830, 0, 0, 590, 0, 0, 0, 0, 5, 1499, 0, 2280, 0, 0, 0, 33, 0, 4074, 1, 0, 0, 0, 0, 12, 1, 2, 4087, 0, 0, 0, 0, 0, 3, 0, 115, 3877, 187, 0, 0, 123, 7, 3826, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3957, 4088, 3557, 3784, 4108, 4102, 4182, 4990] avg=4096.0000 max=4990 max_rank=7 min=3557 min_rank=2 imbalance_ratio=1.218262 max_load_reduction=27778 max_load_reduction_pct=0.847717 moved_to_echo=17382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[20, 1, 216, 4049, 0, 0, 0, 0, 0, 0, 1, 4095, 3, 0, 0, 0, 214, 2801, 0, 0, 591, 1, 0, 0, 0, 0, 2096, 0, 2282, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 3, 35, 3839, 187, 0, 0, 45, 0, 3067, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4286, 4099, 3607, 4378, 4097, 4091, 4064, 4146] avg=4096.0000 max=4378 max_rank=3 min=3607 min_rank=2 imbalance_ratio=1.068848 max_load_reduction=28390 max_load_reduction_pct=0.866394 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 218, 3560, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 397, 3685, 0, 0, 589, 1, 0, 0, 0, 0, 1043, 5, 2275, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 1, 0, 4, 3877, 186, 0, 0, 22, 0, 3577, 1031, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3789, 4091, 4672, 3323, 4102, 4093, 4068, 4630] avg=4096.0000 max=4672 max_rank=2 min=3323 min_rank=3 imbalance_ratio=1.140625 max_load_reduction=28096 max_load_reduction_pct=0.857422 moved_to_echo=16893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 218, 4055, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 876, 3119, 0, 0, 591, 1, 0, 0, 0, 0, 1884, 1, 2283, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 24, 3867, 187, 0, 0, 17, 0, 2296, 1034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4301, 4099, 4587, 4168, 4096, 4092, 4078, 3347] avg=4096.0000 max=4587 max_rank=2 min=3347 min_rank=7 imbalance_ratio=1.119873 max_load_reduction=28181 max_load_reduction_pct=0.860016 moved_to_echo=15613 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[53, 0, 218, 3860, 0, 0, 0, 0, 1, 0, 8, 4087, 3, 0, 0, 0, 71, 3678, 0, 0, 589, 1, 0, 0, 0, 7, 1213, 0, 2276, 0, 0, 0, 13, 0, 4073, 0, 0, 0, 0, 0, 27, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 7, 3867, 186, 0, 0, 5, 0, 3401, 1033, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4099, 4339, 3496, 4086, 4118, 4060, 4439] avg=4096.0000 max=4439 max_rank=7 min=3496 min_rank=3 imbalance_ratio=1.083740 max_load_reduction=28329 max_load_reduction_pct=0.864532 moved_to_echo=16703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[33, 2, 213, 3678, 0, 0, 0, 0, 0, 3, 0, 4073, 7, 0, 0, 0, 22, 2674, 0, 0, 595, 3, 0, 0, 7, 0, 2420, 0, 2287, 0, 0, 0, 0, 1, 4093, 3, 6, 0, 0, 0, 6, 0, 1, 4074, 0, 0, 0, 0, 0, 23, 3, 46, 3876, 190, 0, 0, 3, 0, 3402, 1024, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3926, 4083, 3294, 4714, 4103, 4081, 4138, 4429] avg=4096.0000 max=4714 max_rank=3 min=3294 min_rank=2 imbalance_ratio=1.150879 max_load_reduction=28054 max_load_reduction_pct=0.856140 moved_to_echo=16751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 14592, 128, 18304, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 24320, 8320, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2432, 25728, 0, 0, 4736, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 1792, 30848, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 384, 30976, 1536, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4094, 9, 3, 1, 0, 1, 16, 4095, 0, 0, 171, 4095, 0, 0, 0, 1008, 0, 1, 1, 0, 4080, 3014, 2, 0, 8, 1, 3976, 0, 4095, 1, 4092, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4065, 2, 0, 1, 13, 0, 0, 4095, 1, 0, 20, 4093, 0, 0, 0, 950, 0, 0, 0, 0, 4094, 3263, 0, 0, 2, 3, 3977, 2, 4095, 0, 4091, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 46, 4096, 0, 0, 0, 2849, 0, 0, 0, 0, 4095, 1234, 0, 0, 1, 0, 4063, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4093, 1, 0, 1, 20, 0, 0, 4091, 0, 0, 127, 4081, 0, 0, 2, 945, 0, 4, 3, 2, 4071, 3082, 4, 0, 7, 1, 4041, 0, 4092, 0, 4096, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4091, 9, 0, 0, 0, 0, 1, 4096, 0, 4, 48, 4096, 0, 0, 0, 2041, 0, 0, 0, 0, 4076, 2096, 0, 0, 0, 0, 4018, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 135, 4096, 0, 0, 1, 2451, 0, 0, 0, 0, 4094, 1656, 0, 0, 5, 0, 3957, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 238, 4096, 0, 0, 0, 1490, 0, 0, 0, 0, 4091, 2430, 0, 0, 10, 0, 4029, 0, 4096, 9, 4088, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 17, 1, 2, 4095, 0, 0, 192, 4085, 2, 0, 0, 274, 3, 0, 2, 5, 4081, 3656, 0, 0, 4, 0, 4062, 5, 4093, 1, 4095, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4065, 2, 0, 1, 3, 0, 0, 0, 13, 0, 0, 4088, 0, 0, 0, 0, 1, 0, 20, 3974, 0, 0, 0, 0, 0, 0, 0, 950, 2549, 49, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1545, 3263, 0, 0, 0, 0, 0, 0, 2, 3, 3977, 2, 70, 4, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4101, 3995, 3549, 4094, 4808, 4058, 4092] avg=4096.0000 max=4808 max_rank=5 min=3549 min_rank=3 imbalance_ratio=1.173828 max_load_reduction=27960 max_load_reduction_pct=0.853271 moved_to_echo=17052 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4091, 9, 0, 0, 3, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 4, 48, 3977, 0, 0, 0, 0, 0, 0, 0, 2041, 2538, 49, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1538, 2096, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 70, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4090, 4029, 4630, 4094, 3634, 4092, 4096] avg=4096.0000 max=4630 max_rank=3 min=3634 min_rank=5 imbalance_ratio=1.130371 max_load_reduction=28138 max_load_reduction_pct=0.858704 moved_to_echo=15916 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4094, 9, 3, 1, 3, 0, 0, 0, 0, 1, 16, 4088, 0, 0, 0, 0, 0, 0, 171, 3976, 0, 0, 0, 0, 0, 0, 0, 1008, 2540, 49, 2, 0, 0, 1, 1, 0, 4093, 0, 0, 0, 1540, 3014, 2, 0, 0, 0, 0, 0, 8, 1, 3976, 0, 70, 4, 0, 0, 0, 1, 4092, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4105, 4147, 3599, 4095, 4556, 4059, 4097] avg=4096.0000 max=4556 max_rank=5 min=3599 min_rank=3 imbalance_ratio=1.112305 max_load_reduction=28212 max_load_reduction_pct=0.860962 moved_to_echo=16807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 46, 3977, 0, 0, 0, 0, 0, 0, 0, 2849, 2549, 49, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1546, 1234, 0, 0, 0, 0, 0, 0, 1, 0, 4063, 0, 70, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4089, 4023, 5448, 4095, 2780, 4138, 4096] avg=4096.0000 max=5448 max_rank=3 min=2780 min_rank=5 imbalance_ratio=1.330078 max_load_reduction=27320 max_load_reduction_pct=0.833740 moved_to_echo=15109 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 135, 3977, 0, 0, 0, 0, 0, 0, 1, 2451, 2549, 49, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1545, 1656, 0, 0, 0, 0, 0, 0, 5, 0, 3957, 0, 70, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4089, 4112, 5051, 4095, 3201, 4036, 4096] avg=4096.0000 max=5051 max_rank=3 min=3201 min_rank=5 imbalance_ratio=1.233154 max_load_reduction=27717 max_load_reduction_pct=0.845856 moved_to_echo=15428 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4093, 1, 0, 1, 7, 3, 0, 0, 20, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 127, 3960, 0, 0, 0, 0, 0, 0, 2, 945, 2538, 50, 2, 0, 0, 4, 3, 2, 4090, 0, 0, 0, 1533, 3082, 4, 0, 0, 0, 0, 0, 7, 1, 4041, 0, 71, 7, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4097, 4087, 3537, 4099, 4619, 4127, 4097] avg=4096.0000 max=4619 max_rank=5 min=3537 min_rank=3 imbalance_ratio=1.127686 max_load_reduction=28149 max_load_reduction_pct=0.859039 moved_to_echo=16942 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 3, 0, 0, 0, 17, 1, 2, 4088, 0, 0, 0, 0, 0, 0, 192, 3967, 0, 0, 0, 0, 2, 0, 0, 274, 2541, 48, 1, 0, 3, 0, 2, 5, 4092, 0, 0, 0, 1540, 3656, 0, 0, 0, 0, 0, 0, 4, 0, 4062, 5, 70, 4, 0, 0, 0, 1, 4095, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4108, 4159, 2866, 4102, 5196, 4145, 4097] avg=4096.0000 max=5196 max_rank=5 min=2866 min_rank=3 imbalance_ratio=1.268555 max_load_reduction=27572 max_load_reduction_pct=0.841431 moved_to_echo=17540 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 1, 238, 3977, 0, 0, 0, 0, 0, 0, 0, 1490, 2547, 49, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1544, 2430, 0, 0, 0, 0, 0, 0, 10, 0, 4029, 0, 70, 4, 0, 0, 0, 9, 4088, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4089, 4216, 4088, 4094, 3974, 4113, 4097] avg=4096.0000 max=4216 max_rank=2 min=3974 min_rank=5 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16278 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 12032, 20352, 512, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12416, 20480, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32128, 128, 640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 1024, 31872, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 1, 4094, 4095, 1, 0, 0, 4095, 0, 1, 0, 2, 0, 4095, 2105, 1, 4095, 0, 0, 0, 0, 1667, 1, 1, 1, 4093, 1, 324] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4087, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1845, 0, 4096, 0, 0, 0, 0, 2215, 0, 0, 0, 4096, 0, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4088, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2558, 0, 4096, 0, 0, 0, 0, 1383, 0, 0, 0, 4096, 0, 163] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 2, 4090, 0, 0, 0, 4096, 4096, 5, 0, 0, 4092, 0, 0, 1, 0, 0, 4095, 1520, 0, 4096, 0, 0, 1, 6, 2182, 0, 0, 0, 4091, 0, 395] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2979, 0, 4096, 0, 0, 0, 0, 957, 0, 0, 0, 4096, 0, 160] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[2, 2, 4093, 1, 1, 1, 4061, 4095, 0, 1, 1, 4094, 0, 0, 1, 0, 1, 4092, 2328, 4, 4093, 0, 0, 1, 0, 1753, 0, 0, 3, 4095, 0, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1016, 0, 4096, 0, 0, 0, 0, 2939, 0, 0, 0, 4096, 0, 141] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 7, 4092, 0, 3, 4, 4085, 4089, 0, 0, 4, 4091, 2, 4, 6, 1, 0, 4091, 2915, 13, 4089, 4, 0, 0, 0, 1084, 0, 5, 0, 4091, 0, 88] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 4094, 7, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 2, 4088, 5, 0, 0, 0, 1937, 2105, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1667, 1, 1, 2153, 168, 0, 0, 1, 3925, 1, 324, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4102, 4096, 4096, 4043, 4095, 3990, 4251] avg=4096.0000 max=4251 max_rank=7 min=3990 min_rank=6 imbalance_ratio=1.037842 max_load_reduction=28517 max_load_reduction_pct=0.870270 moved_to_echo=16379 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 1937, 2558, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1383, 0, 0, 2154, 168, 0, 0, 0, 3928, 0, 163, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4096, 4094, 4495, 4096, 3705, 4091] avg=4096.0000 max=4495 max_rank=4 min=3705 min_rank=6 imbalance_ratio=1.097412 max_load_reduction=28273 max_load_reduction_pct=0.862823 moved_to_echo=16387 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 5, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 5, 0, 0, 0, 1937, 1520, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 6, 2182, 0, 0, 2153, 168, 0, 0, 0, 3923, 0, 395, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4103, 4097, 4095, 3457, 4097, 4509, 4318] avg=4096.0000 max=4509 max_rank=6 min=3457 min_rank=4 imbalance_ratio=1.100830 max_load_reduction=28259 max_load_reduction_pct=0.862396 moved_to_echo=16381 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 1937, 1845, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2215, 0, 0, 2154, 168, 0, 0, 0, 3928, 0, 45, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 4096, 4094, 3782, 4096, 4537, 3973] avg=4096.0000 max=4537 max_rank=6 min=3782 min_rank=4 imbalance_ratio=1.107666 max_load_reduction=28231 max_load_reduction_pct=0.861542 moved_to_echo=16388 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 1937, 2979, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 957, 0, 0, 2154, 168, 0, 0, 0, 3928, 0, 160, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4103, 4096, 4094, 4916, 4096, 3279, 4088] avg=4096.0000 max=4916 max_rank=4 min=3279 min_rank=6 imbalance_ratio=1.200195 max_load_reduction=27852 max_load_reduction_pct=0.849976 moved_to_echo=16379 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 4093, 1, 0, 0, 0, 0, 1, 1, 4061, 7, 0, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 5, 0, 0, 1, 1935, 2328, 4, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 1753, 0, 0, 2152, 168, 0, 0, 3, 3927, 0, 45, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4070, 4096, 4094, 4268, 4094, 4073, 3975] avg=4096.0000 max=4268 max_rank=4 min=3975 min_rank=7 imbalance_ratio=1.041992 max_load_reduction=28500 max_load_reduction_pct=0.869751 moved_to_echo=16410 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 4092, 0, 0, 0, 0, 0, 3, 4, 4085, 6, 0, 0, 0, 0, 0, 0, 4, 4091, 0, 0, 0, 0, 2, 4, 6, 1, 4083, 11, 0, 0, 0, 1926, 2915, 13, 0, 0, 0, 0, 4089, 4, 0, 0, 5, 0, 0, 0, 0, 1084, 0, 5, 2154, 171, 0, 0, 0, 3915, 0, 88, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4098, 4095, 4107, 4854, 4098, 3414, 4003] avg=4096.0000 max=4854 max_rank=4 min=3414 min_rank=6 imbalance_ratio=1.185059 max_load_reduction=27914 max_load_reduction_pct=0.851868 moved_to_echo=16369 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 1937, 1016, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2939, 0, 0, 2154, 168, 0, 0, 0, 3928, 0, 141, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4103, 4096, 4094, 2953, 4096, 5261, 4069] avg=4096.0000 max=5261 max_rank=6 min=2953 min_rank=4 imbalance_ratio=1.284424 max_load_reduction=27507 max_load_reduction_pct=0.839447 moved_to_echo=16379 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 15488, 17280, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 14208, 128, 128, 17280, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31488, 128, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4090, 4096, 2, 0, 0, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 0, 2911, 4096, 4096, 0, 0, 0, 0, 2394, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1, 2, 4018, 4092, 0, 4, 4, 2863, 13, 0, 0, 2, 2, 2, 5, 0, 0, 0, 4092, 1, 5, 4088, 0, 1921, 4094, 4091, 0, 1, 0, 5, 3462, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3968, 4096, 0, 0, 0, 2950, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 2278, 4096, 4096, 0, 0, 0, 13, 3090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 4027, 4095, 6, 0, 0, 3760, 0, 0, 0, 0, 1, 1, 0, 0, 2, 0, 4095, 1, 0, 4094, 0, 2531, 4095, 4094, 0, 1, 0, 0, 1963, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4092, 4096, 0, 0, 3, 3773, 19, 0, 1, 0, 0, 0, 0, 9, 0, 12, 4094, 0, 0, 4078, 5, 2908, 4094, 4095, 1, 0, 1, 0, 1487, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4096, 0, 0, 0, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 2252, 4096, 4096, 0, 0, 0, 1, 3754, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4078, 4096, 0, 0, 0, 2382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4, 3969, 1, 2302, 4096, 4096, 0, 0, 0, 0, 3644, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[5, 3, 4095, 4085, 19, 0, 19, 3200, 8, 0, 4, 7, 4, 7, 0, 0, 3, 0, 4091, 3, 0, 4093, 0, 1439, 4093, 4090, 1, 3, 0, 0, 3496, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4027, 37, 0, 0, 0, 0, 6, 0, 0, 3760, 1090, 1, 0, 0, 0, 0, 0, 0, 4058, 29, 2, 0, 1, 1, 0, 0, 4092, 0, 0, 0, 2, 0, 4094, 1, 0, 0, 0, 0, 0, 1787, 0, 2531, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 1963, 0, 1188, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4857, 4089, 4094, 4097, 4318, 4096, 3151] avg=4096.0000 max=4857 max_rank=1 min=3151 min_rank=7 imbalance_ratio=1.185791 max_load_reduction=27911 max_load_reduction_pct=0.851776 moved_to_echo=15662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3968, 37, 0, 0, 0, 0, 0, 0, 0, 2950, 1087, 1, 0, 0, 0, 1, 0, 0, 4059, 29, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1783, 0, 2278, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 13, 3090, 0, 1185, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 4038, 4091, 4093, 4095, 4061, 4097, 4288] avg=4096.0000 max=4288 max_rank=7 min=4005 min_rank=0 imbalance_ratio=1.046875 max_load_reduction=28480 max_load_reduction_pct=0.869141 moved_to_echo=16541 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 4018, 37, 0, 0, 0, 0, 0, 4, 4, 2863, 1089, 1, 0, 0, 13, 0, 0, 2, 4055, 29, 1, 0, 2, 2, 5, 0, 4092, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 5, 1784, 0, 1921, 0, 0, 0, 0, 1, 4091, 0, 1, 0, 0, 0, 0, 0, 5, 3462, 0, 1186, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 3961, 4100, 4101, 4092, 3710, 4093, 4653] avg=4096.0000 max=4653 max_rank=7 min=3710 min_rank=5 imbalance_ratio=1.135986 max_load_reduction=28115 max_load_reduction_pct=0.858002 moved_to_echo=16548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 37, 0, 0, 0, 0, 2, 0, 0, 2898, 1089, 1, 0, 0, 0, 0, 0, 0, 4059, 29, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1785, 0, 2911, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2394, 0, 1186, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 3990, 4090, 4093, 4095, 4696, 4097, 3580] avg=4096.0000 max=4696 max_rank=5 min=3580 min_rank=7 imbalance_ratio=1.146484 max_load_reduction=28072 max_load_reduction_pct=0.856689 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4092, 37, 0, 0, 0, 0, 0, 0, 3, 3773, 1086, 1, 0, 0, 19, 0, 1, 0, 4059, 29, 1, 0, 0, 0, 0, 9, 4091, 0, 0, 0, 0, 12, 4093, 0, 0, 0, 0, 0, 0, 1780, 5, 2908, 0, 0, 0, 0, 2, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 1487, 0, 1183, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4863, 4109, 4100, 4105, 4693, 4098, 2671] avg=4096.0000 max=4863 max_rank=1 min=2671 min_rank=7 imbalance_ratio=1.187256 max_load_reduction=27905 max_load_reduction_pct=0.851593 moved_to_echo=15567 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 37, 0, 0, 0, 0, 0, 0, 0, 2190, 1090, 1, 0, 0, 0, 0, 0, 0, 4059, 29, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1786, 0, 2252, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 3754, 0, 1187, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 3281, 4090, 4093, 4095, 4038, 4097, 4942] avg=4096.0000 max=4942 max_rank=7 min=3281 min_rank=1 imbalance_ratio=1.206543 max_load_reduction=27826 max_load_reduction_pct=0.849182 moved_to_echo=17172 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 3, 4095, 33, 0, 0, 0, 0, 19, 0, 19, 3200, 1093, 7, 0, 0, 8, 0, 4, 7, 4052, 35, 3, 0, 4, 7, 0, 0, 4090, 0, 0, 0, 3, 0, 4084, 3, 0, 0, 0, 0, 0, 1773, 0, 1439, 0, 0, 0, 0, 0, 4090, 1, 3, 0, 0, 0, 0, 0, 0, 3496, 0, 1192, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4338, 4109, 4101, 4090, 3212, 4094, 4688] avg=4096.0000 max=4688 max_rank=7 min=3212 min_rank=5 imbalance_ratio=1.144531 max_load_reduction=28080 max_load_reduction_pct=0.856934 moved_to_echo=16084 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4078, 37, 0, 0, 0, 0, 0, 0, 0, 2382, 1057, 1, 0, 0, 0, 0, 0, 0, 4059, 28, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 4, 1733, 1, 2302, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3644, 0, 1151, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 3440, 4090, 4093, 4099, 4040, 4096, 4795] avg=4096.0000 max=4795 max_rank=7 min=3440 min_rank=1 imbalance_ratio=1.170654 max_load_reduction=27973 max_load_reduction_pct=0.853668 moved_to_echo=17030 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 23296, 0, 9472, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 24064, 8704, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 14336, 128, 18560, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32512, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32512, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4095, 322, 1494, 4095, 0, 0, 0, 1, 1, 2, 4095, 0, 0, 2373, 0, 0, 0, 0, 3, 1, 0, 3991, 0, 21, 4081, 1, 1, 0, 4095, 1, 4094, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 361, 2224, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1496, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 15, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 285, 2446, 4090, 0, 0, 1, 0, 0, 0, 4096, 2, 0, 1268, 42, 0, 0, 0, 0, 0, 0, 4068, 0, 93, 4089, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 13, 441, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3643, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4089, 189, 2200, 4094, 0, 1, 0, 1, 2, 7, 4086, 0, 0, 1948, 5, 1, 0, 2, 0, 0, 0, 3810, 1, 75, 4075, 3, 0, 2, 4093, 2, 4079, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4092, 142, 1903, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2133, 4, 0, 0, 0, 0, 0, 0, 4036, 0, 2, 4072, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4091, 89, 2836, 4092, 1, 0, 0, 4, 6, 10, 4077, 0, 0, 1242, 0, 0, 0, 0, 6, 3, 0, 4003, 4, 32, 4089, 0, 1, 3, 4090, 6, 4078, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4093, 92, 1030, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3005, 2, 0, 0, 0, 0, 0, 0, 4080, 0, 1, 4081, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2090, 322, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1, 2, 4095, 0, 0, 0, 0, 0, 0, 2373, 0, 0, 1951, 0, 0, 0, 0, 0, 3, 1, 4092, 2, 0, 0, 0, 3991, 0, 21, 43, 0, 0, 0, 4081, 1, 1, 0, 9, 0, 0, 0, 3, 1, 4094, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3906, 4096, 4098, 4324, 4098, 4055, 4092, 4099] avg=4096.0000 max=4324 max_rank=3 min=3906 min_rank=0 imbalance_ratio=1.055664 max_load_reduction=28444 max_load_reduction_pct=0.868042 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2091, 285, 2446, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 1268, 42, 0, 1951, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 4068, 0, 93, 43, 0, 0, 0, 4089, 0, 0, 0, 9, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4822, 4091, 4098, 3261, 4095, 4204, 4098, 4099] avg=4096.0000 max=4822 max_rank=0 min=3261 min_rank=3 imbalance_ratio=1.177246 max_load_reduction=27946 max_load_reduction_pct=0.852844 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2091, 361, 2224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1496, 3, 0, 1951, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 4093, 0, 15, 43, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4676, 4096, 4096, 3450, 4095, 4151, 4105, 4099] avg=4096.0000 max=4676 max_rank=0 min=3450 min_rank=3 imbalance_ratio=1.141602 max_load_reduction=28092 max_load_reduction_pct=0.857300 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2091, 13, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3643, 0, 0, 1951, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 4095, 0, 0, 43, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2545, 4096, 4096, 5594, 4095, 4138, 4105, 4099] avg=4096.0000 max=5594 max_rank=3 min=2545 min_rank=0 imbalance_ratio=1.365723 max_load_reduction=27174 max_load_reduction_pct=0.829285 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2089, 92, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3005, 2, 0, 1950, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 4080, 0, 1, 43, 0, 0, 0, 4081, 0, 0, 0, 9, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3211, 4096, 4096, 4957, 4095, 4124, 4090, 4099] avg=4096.0000 max=4957 max_rank=3 min=3211 min_rank=0 imbalance_ratio=1.210205 max_load_reduction=27811 max_load_reduction_pct=0.848724 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2087, 189, 2200, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4094, 0, 0, 0, 2, 7, 4086, 0, 0, 0, 0, 0, 0, 1948, 5, 1, 1948, 0, 0, 0, 0, 2, 0, 0, 4090, 2, 0, 0, 0, 3810, 1, 75, 43, 0, 0, 0, 4075, 3, 0, 2, 9, 0, 0, 0, 3, 2, 4079, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4476, 4096, 4095, 3902, 4094, 3929, 4089, 4087] avg=4096.0000 max=4476 max_rank=0 min=3902 min_rank=3 imbalance_ratio=1.092773 max_load_reduction=28292 max_load_reduction_pct=0.863403 moved_to_echo=16199 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2073, 89, 2836, 0, 0, 0, 0, 0, 1, 0, 0, 4, 4092, 3, 0, 0, 6, 10, 4077, 0, 0, 0, 0, 0, 0, 1242, 0, 0, 1952, 0, 0, 0, 0, 0, 6, 3, 4087, 4, 0, 0, 0, 4003, 4, 32, 47, 0, 0, 0, 4089, 0, 1, 3, 15, 0, 0, 0, 0, 6, 4078, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4998, 4100, 4093, 3194, 4100, 4086, 4108, 4089] avg=4096.0000 max=4998 max_rank=0 min=3194 min_rank=3 imbalance_ratio=1.220215 max_load_reduction=27770 max_load_reduction_pct=0.847473 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2089, 142, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2133, 4, 0, 1949, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 4036, 0, 2, 43, 0, 0, 0, 4072, 0, 0, 0, 9, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4097, 4096, 4086, 4096, 4081, 4081, 4097] avg=4096.0000 max=4134 max_rank=0 min=4081 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[16768, 1536, 14592, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 17152, 128, 128, 15616, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32256, 128, 256, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4093, 0, 0, 1, 4094, 1, 1, 4095, 4095, 4095, 0, 0, 0, 0, 0, 7, 1, 4009, 0, 4095, 1, 4089, 0, 0, 1, 0, 1, 87, 0, 1, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3985, 26, 4096, 0, 4080, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4091, 2, 0, 0, 4087, 0, 1, 4092, 4092, 4093, 0, 0, 0, 0, 5, 12, 1, 4093, 0, 4083, 2, 4079, 3, 5, 3, 4, 3, 14, 0, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 12, 0, 4090, 17, 4096, 0, 4063, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4029, 2, 4096, 0, 4094, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4088, 4, 0, 5, 4086, 2, 3, 4089, 4089, 4084, 0, 3, 0, 0, 0, 14, 5, 4087, 0, 4069, 6, 4035, 6, 0, 8, 1, 7, 65, 0, 7, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 4096, 0, 0, 4095, 4096, 4092, 0, 0, 4, 0, 4, 18, 0, 4051, 76, 4096, 0, 4029, 0, 1, 0, 0, 0, 18, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4091, 4096, 4096, 0, 0, 0, 0, 0, 7, 0, 4090, 9, 4096, 0, 4089, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 1, 1, 0, 0, 0, 4094, 1, 1, 2, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4086, 0, 0, 0, 1, 4009, 0, 26, 0, 0, 0, 0, 1, 4089, 0, 0, 23, 0, 0, 0, 1, 0, 1, 87, 4045, 4, 0, 0, 0, 1, 0, 1, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4098, 4099, 4093, 4036, 4113, 4138, 4096] avg=4096.0000 max=4138 max_rank=6 min=4036 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4087, 0, 0, 0, 0, 4090, 17, 26, 0, 0, 0, 0, 0, 4063, 0, 0, 23, 0, 0, 0, 0, 0, 0, 13, 4046, 4, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4098, 4097, 4099, 4133, 4086, 4063, 4095] avg=4096.0000 max=4133 max_rank=4 min=4063 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 3985, 26, 26, 0, 0, 0, 0, 0, 4080, 0, 0, 23, 0, 0, 0, 0, 0, 0, 101, 4046, 4, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4098, 4100, 4087, 4037, 4103, 4151, 4095] avg=4096.0000 max=4151 max_rank=6 min=4037 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4091, 2, 0, 0, 1, 0, 0, 0, 4087, 0, 1, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 5, 12, 4083, 0, 0, 0, 1, 4093, 0, 25, 0, 0, 0, 0, 2, 4079, 3, 5, 23, 0, 0, 0, 3, 4, 3, 14, 4034, 4, 0, 0, 0, 3, 0, 0, 4090, 5, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4090, 4093, 4100, 4119, 4112, 4062, 4098] avg=4096.0000 max=4119 max_rank=4 min=4062 min_rank=6 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4, 4092, 0, 0, 0, 0, 0, 0, 4, 0, 4, 18, 4087, 0, 0, 0, 0, 4051, 76, 26, 0, 0, 0, 0, 0, 4029, 0, 1, 23, 0, 0, 0, 0, 0, 0, 18, 4046, 4, 0, 0, 0, 0, 1, 0, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4098, 4096, 4113, 4153, 4053, 4068, 4095] avg=4096.0000 max=4153 max_rank=4 min=4053 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4029, 2, 26, 0, 0, 0, 0, 0, 4094, 0, 0, 23, 0, 0, 0, 0, 0, 0, 67, 4046, 4, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4098, 4100, 4087, 4057, 4117, 4117, 4095] avg=4096.0000 max=4117 max_rank=5 min=4057 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4088, 4, 0, 5, 2, 0, 0, 0, 4086, 2, 3, 0, 0, 0, 0, 0, 0, 4084, 0, 3, 0, 0, 0, 0, 0, 0, 0, 14, 4081, 0, 0, 0, 5, 4087, 0, 16, 0, 0, 0, 0, 6, 4035, 6, 0, 25, 0, 0, 0, 8, 1, 7, 65, 4026, 7, 0, 0, 0, 7, 0, 5, 4089, 1, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4091, 4087, 4095, 4108, 4072, 4114, 4102] avg=4096.0000 max=4114 max_rank=6 min=4072 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4087, 0, 0, 0, 0, 4090, 9, 26, 0, 0, 0, 0, 0, 4089, 0, 0, 23, 0, 0, 0, 0, 0, 0, 2, 4046, 7, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4097, 4096, 4094, 4125, 4112, 4055, 4092] avg=4096.0000 max=4125 max_rank=4 min=4055 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 384, 32384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32640, 128, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32512, 256, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 4095, 1, 1, 1, 0, 0, 1, 4092, 0, 0, 0, 4095, 4095, 1, 1, 0, 0, 4095, 4095, 0, 0, 4095, 4095, 1, 0, 1, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[3, 2, 5, 0, 4094, 32, 2, 3, 0, 3, 3, 4061, 0, 0, 1, 4091, 4092, 0, 3, 0, 0, 4093, 4093, 0, 0, 4088, 4093, 3, 0, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 23, 0, 0, 0, 12, 0, 4077, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 4095, 4096, 0, 0, 4080, 4093, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4081, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 4095, 0, 0, 0, 7, 4096, 4096, 0, 0, 4087, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3, 7, 10, 0, 4090, 23, 8, 6, 0, 12, 8, 4073, 0, 0, 1, 4063, 4086, 0, 5, 0, 0, 4088, 4089, 0, 0, 4077, 4088, 8, 0, 15, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 4087, 3, 1, 0, 4083, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 4092, 5, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 8, 1, 0, 0, 0, 0, 0, 1, 0, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4086, 4098, 4099, 4097, 4095, 4104, 4096] avg=4096.0000 max=4104 max_rank=6 min=4086 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 3, 2, 0, 4084, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4081, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4099, 4101, 4100, 4096, 4096, 4089, 4094] avg=4096.0000 max=4101 max_rank=2 min=4089 min_rank=6 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 2, 5, 0, 4085, 3, 3, 0, 4082, 32, 2, 3, 0, 0, 0, 0, 0, 3, 3, 4061, 5, 0, 0, 0, 0, 0, 1, 4091, 4, 0, 0, 0, 4092, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4088, 8, 3, 0, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4119, 4072, 4096, 4095, 4093, 4099, 4093] avg=4096.0000 max=4119 max_rank=1 min=4072 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 3, 1, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4084, 4101, 4100, 4096, 4098, 4104, 4093] avg=4096.0000 max=4104 max_rank=6 min=4084 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 3, 1, 0, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4094, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4086, 4101, 4100, 4096, 4097, 4102, 4094] avg=4096.0000 max=4102 max_rank=6 min=4086 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 7, 10, 0, 4086, 5, 3, 0, 4060, 23, 8, 6, 0, 0, 0, 0, 0, 12, 8, 4073, 10, 0, 0, 0, 0, 0, 1, 4063, 9, 0, 0, 0, 4086, 0, 5, 0, 6, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4077, 2, 8, 0, 0, 0, 0, 0, 15, 0, 8, 4086, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4097, 4103, 4073, 4097, 4088, 4087, 4109] avg=4096.0000 max=4114 max_rank=0 min=4073 min_rank=3 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 3, 3, 0, 4084, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4090, 4098, 4100, 4095, 4103, 4095, 4093] avg=4096.0000 max=4103 max_rank=5 min=4090 min_rank=1 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4085, 3, 1, 0, 4084, 23, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4077, 5, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 2, 0, 0, 0, 0, 0, 0, 4080, 8, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4107, 4094, 4100, 4096, 4098, 4088, 4096] avg=4096.0000 max=4107 max_rank=1 min=4088 min_rank=6 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 38, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 4096, 4068, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 439, 4089, 1, 4, 7, 7, 2, 47, 0, 4088, 9, 8, 0, 0, 0, 4088, 0, 13, 3, 0, 0, 5, 4071, 4091, 0, 5, 4089, 3624, 0, 4077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 716, 4091, 1, 1, 1, 1, 0, 0, 3, 4094, 0, 1, 0, 0, 0, 4095, 0, 1, 2, 0, 0, 1, 4089, 4095, 0, 1, 4095, 3384, 1, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 47, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 4096, 4062, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 44, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4052, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 243, 4090, 0, 0, 2, 3, 0, 0, 3, 4091, 0, 3, 2, 0, 0, 4093, 0, 5, 4, 0, 6, 2, 4078, 4095, 0, 3, 4093, 3857, 3, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 204, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 4096, 3896, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4, 0, 348, 4096, 0, 0, 0, 0, 0, 1, 0, 4069, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 4092, 0, 0, 4096, 3783, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 716, 3833, 0, 0, 0, 0, 1, 1, 1, 1, 3832, 258, 1, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 1, 0, 0, 4085, 6, 1, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 2, 0, 0, 1, 4094, 0, 0, 0, 4089, 10, 0, 1, 0, 0, 0, 0, 0, 3384, 1, 256, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4550, 4095, 4096, 4093, 4096, 4097, 4100, 3641] avg=4096.0000 max=4550 max_rank=0 min=3641 min_rank=7 imbalance_ratio=1.110840 max_load_reduction=28218 max_load_reduction_pct=0.861145 moved_to_echo=15934 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 38, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 258, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4086, 10, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 256, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3876, 4093, 4095, 4093, 4096, 4095, 4096, 4324] avg=4096.0000 max=4324 max_rank=7 min=3876 min_rank=0 imbalance_ratio=1.055664 max_load_reduction=28444 max_load_reduction_pct=0.868042 moved_to_echo=16611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 44, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 258, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 256, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3882, 4093, 4095, 4094, 4096, 4094, 4106, 4308] avg=4096.0000 max=4308 max_rank=7 min=3882 min_rank=0 imbalance_ratio=1.051758 max_load_reduction=28460 max_load_reduction_pct=0.868530 moved_to_echo=16604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 439, 3827, 0, 0, 0, 0, 1, 4, 7, 7, 3820, 262, 7, 7, 2, 47, 0, 4081, 0, 0, 0, 0, 9, 8, 0, 0, 4086, 11, 2, 0, 0, 4081, 0, 13, 0, 0, 0, 0, 3, 0, 0, 5, 4087, 0, 0, 0, 4071, 5, 0, 5, 0, 0, 0, 0, 0, 3624, 0, 246, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4267, 4115, 4130, 4116, 4094, 4095, 4081, 3870] avg=4096.0000 max=4267 max_rank=0 min=3870 min_rank=7 imbalance_ratio=1.041748 max_load_reduction=28501 max_load_reduction_pct=0.869781 moved_to_echo=16140 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 47, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 258, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4083, 10, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 256, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3885, 4093, 4095, 4093, 4096, 4095, 4093, 4318] avg=4096.0000 max=4318 max_rank=7 min=3885 min_rank=0 imbalance_ratio=1.054199 max_load_reduction=28450 max_load_reduction_pct=0.868225 moved_to_echo=16602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 243, 3832, 0, 0, 0, 0, 0, 0, 2, 3, 3829, 258, 1, 0, 0, 0, 3, 4090, 0, 0, 0, 0, 0, 3, 2, 0, 4085, 6, 2, 0, 0, 4093, 0, 5, 0, 0, 0, 0, 4, 0, 6, 2, 4091, 0, 0, 0, 4078, 10, 0, 3, 0, 0, 0, 0, 0, 3857, 3, 256, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4093, 4093, 4098, 4098, 4103, 4091, 4116] avg=4096.0000 max=4116 max_rank=7 min=4076 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16408 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 348, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 258, 1, 0, 0, 1, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 6, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4087, 10, 0, 0, 0, 0, 0, 0, 0, 3783, 0, 256, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4093, 4069, 4089, 4096, 4095, 4097, 4039] avg=4096.0000 max=4190 max_rank=0 min=4039 min_rank=7 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16327 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 204, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 258, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 6, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 10, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 256, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4093, 4095, 4092, 4096, 4094, 4104, 4152] avg=4096.0000 max=4152 max_rank=7 min=4042 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 30720, 2176, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 30848, 128, 2048, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 2176, 30720, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 7, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[3, 4094, 28, 1, 4094, 0, 4095, 4060, 0, 2, 0, 1, 4095, 1, 1, 0, 0, 4084, 1, 4, 0, 0, 0, 1, 1, 0, 4091, 2, 4095, 0, 14, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 44, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 5, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1, 4092, 5, 3, 4091, 0, 4092, 4078, 4, 5, 0, 3, 4092, 3, 2, 5, 3, 4056, 2, 0, 7, 0, 0, 1, 0, 0, 4089, 15, 4088, 0, 31, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3744, 0, 37, 0, 0, 3, 0, 0, 0, 4096, 316, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[6, 4086, 0, 10, 4088, 0, 4088, 4050, 0, 14, 0, 6, 4085, 4, 7, 1, 7, 4059, 7, 0, 0, 0, 0, 7, 0, 0, 4082, 30, 4086, 0, 43, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 6, 0, 4096, 0, 4096, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4069, 0, 18, 0, 0, 0, 0, 0, 0, 4095, 9, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 4088, 28, 1, 0, 0, 0, 0, 14, 0, 0, 4060, 0, 0, 0, 0, 0, 2, 0, 1, 4080, 10, 1, 0, 4094, 1, 1, 0, 0, 0, 0, 0, 0, 4084, 1, 4, 39, 8, 6, 1, 0, 0, 0, 1, 4094, 0, 0, 0, 1, 0, 4042, 2, 0, 0, 0, 0, 4087, 0, 14, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4074, 4094, 4096, 4143, 4095, 4045, 4101] avg=4096.0000 max=4143 max_rank=4 min=4045 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 10, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 10, 39, 8, 6, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4047, 5, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4110, 4093, 4095, 4145, 4095, 4052, 4088] avg=4096.0000 max=4145 max_rank=4 min=4052 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 1, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 10, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 2, 39, 8, 6, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4047, 7, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4110, 4092, 4095, 4142, 4096, 4054, 4088] avg=4096.0000 max=4142 max_rank=4 min=4054 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 10, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 39, 8, 6, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4047, 44, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4110, 4093, 4095, 4106, 4095, 4091, 4088] avg=4096.0000 max=4110 max_rank=1 min=4088 min_rank=7 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 10, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 0, 37, 39, 8, 6, 1, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 4047, 316, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4106, 4092, 4095, 3835, 4099, 4363, 4088] avg=4096.0000 max=4363 max_rank=6 min=3835 min_rank=4 imbalance_ratio=1.065186 max_load_reduction=28405 max_load_reduction_pct=0.866852 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 4087, 5, 3, 0, 0, 0, 0, 14, 0, 0, 4078, 0, 0, 0, 0, 4, 5, 0, 3, 4077, 10, 1, 0, 4092, 3, 2, 5, 0, 0, 0, 0, 3, 4056, 2, 0, 39, 8, 5, 0, 7, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 4040, 15, 0, 0, 0, 0, 4080, 0, 31, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4092, 4100, 4102, 4113, 4099, 4055, 4111] avg=4096.0000 max=4113 max_rank=4 min=4055 min_rank=6 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[6, 4079, 0, 10, 0, 0, 0, 0, 11, 0, 0, 4050, 0, 0, 0, 0, 0, 14, 0, 6, 4077, 13, 1, 0, 4083, 4, 7, 1, 0, 0, 0, 0, 7, 4059, 7, 0, 46, 15, 7, 2, 0, 0, 0, 7, 4087, 0, 0, 0, 0, 0, 4023, 30, 0, 0, 0, 0, 4071, 0, 43, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4061, 4111, 4095, 4143, 4094, 4053, 4116] avg=4096.0000 max=4143 max_rank=4 min=4053 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 6, 0, 0, 0, 0, 0, 14, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 10, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 18, 39, 8, 6, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4046, 9, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4105, 4093, 4095, 4141, 4095, 4055, 4088] avg=4096.0000 max=4141 max_rank=4 min=4055 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 32256, 128, 128, 384, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32384, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 1, 0, 28, 0, 4095, 0, 2, 1, 0, 1184, 1, 4095, 4093, 1, 0, 0, 217, 0, 1, 4095, 4095, 2, 0, 4094, 2904, 0, 3857, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 3, 0, 0, 0, 4092, 2, 0, 3, 11, 444, 4, 4092, 4087, 4, 2, 2, 304, 0, 1, 4090, 4093, 6, 4, 4081, 3455, 0, 3980, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 21, 1099, 0, 4096, 4096, 0, 0, 0, 536, 0, 0, 4092, 4096, 0, 0, 4096, 2744, 0, 3792, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 593, 0, 4096, 4096, 0, 0, 0, 405, 0, 0, 4096, 4096, 0, 0, 4096, 3391, 0, 3801, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1673, 0, 4096, 4096, 0, 0, 0, 266, 0, 0, 4096, 4096, 0, 0, 4096, 2383, 0, 3870, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 19, 711, 0, 4096, 4096, 0, 0, 0, 812, 0, 0, 4092, 4096, 0, 0, 4096, 3493, 0, 3157, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 24, 656, 0, 4096, 4096, 0, 0, 0, 254, 0, 0, 4086, 4096, 0, 0, 4096, 3372, 0, 3887, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 5, 0, 0, 0, 4088, 0, 1, 9, 14, 202, 8, 4087, 4070, 11, 4, 0, 147, 0, 8, 4086, 4089, 10, 4, 4026, 3793, 1, 4082, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4087, 7, 0, 0, 1, 0, 28, 0, 2935, 831, 363, 0, 4095, 0, 2, 1, 0, 0, 0, 0, 0, 1184, 1, 3264, 0, 0, 0, 0, 4093, 1, 0, 0, 1, 0, 0, 0, 217, 0, 1, 3731, 0, 0, 0, 0, 8, 2, 0, 4094, 0, 0, 0, 0, 2904, 0, 915, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4158, 4098, 4449, 4095, 3949, 4104, 3820] avg=4096.0000 max=4449 max_rank=3 min=3820 min_rank=7 imbalance_ratio=1.086182 max_load_reduction=28319 max_load_reduction_pct=0.864227 moved_to_echo=15968 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 7, 0, 0, 0, 0, 4, 0, 2885, 831, 363, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 21, 1099, 0, 3265, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 536, 0, 0, 3728, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 2744, 0, 900, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4083, 4096, 4385, 4097, 4264, 4104, 3644] avg=4096.0000 max=4385 max_rank=3 min=3644 min_rank=7 imbalance_ratio=1.070557 max_load_reduction=28383 max_load_reduction_pct=0.866180 moved_to_echo=16109 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 4085, 7, 0, 0, 3, 0, 0, 0, 3028, 831, 363, 0, 4092, 2, 0, 3, 0, 0, 0, 0, 11, 444, 4, 3261, 0, 0, 0, 0, 4087, 4, 2, 2, 1, 0, 0, 0, 304, 0, 1, 3726, 0, 0, 0, 0, 8, 6, 4, 4081, 0, 0, 0, 0, 3455, 0, 945, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4225, 4097, 3720, 4096, 4031, 4099, 4404] avg=4096.0000 max=4404 max_rank=7 min=3720 min_rank=3 imbalance_ratio=1.075195 max_load_reduction=28364 max_load_reduction_pct=0.865601 moved_to_echo=16630 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 5, 0, 0, 0, 0, 4, 0, 2402, 831, 363, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 19, 711, 0, 3265, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 812, 0, 0, 3728, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 3493, 0, 750, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 3600, 4096, 3995, 4097, 4540, 4104, 4243] avg=4096.0000 max=4540 max_rank=5 min=3600 min_rank=1 imbalance_ratio=1.108398 max_load_reduction=28228 max_load_reduction_pct=0.861450 moved_to_echo=16984 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 7, 0, 0, 0, 0, 0, 0, 2892, 831, 363, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 593, 0, 3265, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 405, 0, 0, 3732, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 3391, 0, 902, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4086, 4096, 3860, 4097, 4137, 4104, 4293] avg=4096.0000 max=4293 max_rank=7 min=3860 min_rank=3 imbalance_ratio=1.048096 max_load_reduction=28475 max_load_reduction_pct=0.868988 moved_to_echo=16631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 7, 0, 0, 0, 0, 0, 0, 2945, 831, 363, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1673, 0, 3265, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 266, 0, 0, 3732, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 2383, 0, 918, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4139, 4096, 4938, 4097, 3998, 4104, 3301] avg=4096.0000 max=4938 max_rank=3 min=3301 min_rank=7 imbalance_ratio=1.205566 max_load_reduction=27830 max_load_reduction_pct=0.849304 moved_to_echo=15500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 4084, 10, 0, 0, 5, 0, 0, 0, 3110, 835, 367, 0, 4083, 0, 1, 9, 0, 0, 0, 0, 14, 202, 8, 3252, 0, 0, 0, 0, 4070, 11, 4, 0, 2, 5, 0, 0, 147, 0, 8, 3717, 0, 0, 0, 0, 5, 10, 4, 4026, 0, 0, 0, 0, 3793, 1, 962, 13, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4317, 4093, 3476, 4092, 3872, 4045, 4769] avg=4096.0000 max=4769 max_rank=7 min=3476 min_rank=3 imbalance_ratio=1.164307 max_load_reduction=27999 max_load_reduction_pct=0.854462 moved_to_echo=16778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 7, 0, 0, 0, 0, 9, 0, 2958, 831, 362, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 24, 656, 0, 3265, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 254, 0, 0, 3723, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 3372, 0, 922, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4160, 4096, 3945, 4097, 3977, 4104, 4294] avg=4096.0000 max=4294 max_rank=7 min=3945 min_rank=3 imbalance_ratio=1.048340 max_load_reduction=28474 max_load_reduction_pct=0.868958 moved_to_echo=16472 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 0, 23168, 6656, 2944, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[25600, 128, 7296, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 6656, 128, 26112, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[2944, 0, 128, 29824, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4095, 4093, 1, 0, 0, 2, 4071, 0, 0, 0, 4095, 0, 0, 0, 23, 1, 0, 0, 3, 0, 1, 4095, 1, 4094, 1, 1, 2, 4095, 0, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4090, 4082, 3, 0, 0, 7, 4061, 3, 1, 4, 4092, 0, 1, 0, 33, 2, 1, 0, 5, 0, 4, 4092, 4, 4089, 3, 0, 4, 4091, 0, 4087, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4091, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4086, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[2, 4083, 4087, 6, 0, 0, 17, 3937, 8, 0, 0, 4086, 0, 8, 0, 153, 4, 3, 0, 42, 0, 8, 4087, 8, 4079, 8, 1, 9, 4078, 0, 4049, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4093, 1, 0, 0, 0, 0, 0, 0, 2, 4071, 22, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 23, 4066, 0, 0, 0, 1, 0, 0, 3, 4089, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 0, 0, 4094, 1, 1, 2, 0, 0, 0, 0, 7, 0, 4094, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4096, 4095, 4089, 4093, 4096, 4098, 4101] avg=4096.0000 max=4101 max_rank=7 min=4089 min_rank=3 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 4067, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4119, 4096, 4074, 4090, 4095, 4096, 4103] avg=4096.0000 max=4119 max_rank=1 min=4074 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4082, 3, 0, 0, 0, 0, 0, 0, 7, 4061, 22, 1, 0, 0, 3, 1, 4, 4092, 0, 0, 0, 0, 0, 1, 0, 33, 4062, 0, 0, 0, 2, 1, 0, 5, 4086, 0, 0, 0, 0, 4, 4091, 4, 0, 0, 0, 0, 4089, 3, 0, 4, 0, 0, 0, 0, 7, 0, 4087, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4091, 4100, 4096, 4094, 4099, 4096, 4103] avg=4096.0000 max=4103 max_rank=7 min=4089 min_rank=0 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4119, 4096, 4067, 4090, 4095, 4096, 4103] avg=4096.0000 max=4119 max_rank=1 min=4067 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 4067, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4119, 4096, 4070, 4090, 4095, 4096, 4103] avg=4096.0000 max=4119 max_rank=1 min=4070 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 4067, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4091, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4119, 4096, 4072, 4090, 4095, 4096, 4098] avg=4096.0000 max=4119 max_rank=1 min=4072 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4087, 6, 0, 0, 0, 0, 0, 0, 17, 3937, 23, 5, 3, 1, 8, 0, 0, 4085, 0, 0, 0, 0, 0, 8, 0, 153, 4055, 6, 0, 0, 4, 3, 0, 42, 4077, 0, 0, 0, 0, 8, 4082, 8, 0, 0, 0, 0, 4076, 8, 1, 9, 0, 0, 0, 0, 0, 0, 4049, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 3986, 4093, 4222, 4126, 4098, 4094, 4054] avg=4096.0000 max=4222 max_rank=3 min=3986 min_rank=1 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 4067, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4119, 4096, 4078, 4095, 4095, 4096, 4103] avg=4096.0000 max=4119 max_rank=1 min=4078 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 128, 32640, 256, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[27, 0, 1, 1, 0, 4069, 4095, 4095, 1, 1, 0, 1, 1, 1, 4093, 0, 1, 4095, 0, 4093, 0, 0, 0, 0, 0, 2, 0, 4095, 1, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 4020, 4096, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 73, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4070, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 26, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[68, 3, 2, 3, 6, 4004, 4090, 4092, 4, 2, 15, 6, 2, 3, 4086, 0, 2, 4091, 0, 4090, 0, 0, 0, 9, 0, 0, 1, 4092, 4, 0, 3, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3936, 4096, 4092, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 154, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3959, 4096, 4095, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 130, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4038, 4096, 4096, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 52, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[70, 1, 16, 8, 7, 3983, 4082, 4089, 8, 8, 13, 0, 8, 6, 4068, 0, 2, 4081, 1, 4078, 0, 0, 0, 38, 0, 0, 4, 4088, 12, 0, 13, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[27, 0, 1, 1, 4063, 8, 0, 0, 0, 4069, 86, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4087, 0, 0, 0, 1, 1, 4093, 0, 1, 1, 0, 0, 1, 4, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 28, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4155, 4090, 4097, 4098, 4036, 4097, 4095] avg=4096.0000 max=4155 max_rank=1 min=4036 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16326 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4064, 8, 0, 0, 0, 3936, 86, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4084, 0, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 154, 4009, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4022, 4094, 4098, 4100, 4191, 4096, 4095] avg=4096.0000 max=4191 max_rank=5 min=4022 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4064, 8, 0, 0, 0, 4020, 86, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 73, 4009, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4106, 4090, 4098, 4100, 4110, 4096, 4095] avg=4096.0000 max=4110 max_rank=5 min=4073 min_rank=0 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4064, 9, 0, 0, 0, 3959, 86, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4086, 0, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 130, 4009, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4045, 4094, 4098, 4100, 4167, 4096, 4095] avg=4096.0000 max=4167 max_rank=5 min=4045 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16458 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[68, 3, 2, 3, 4060, 9, 0, 0, 6, 4004, 85, 0, 0, 0, 0, 0, 4, 2, 15, 6, 4083, 0, 0, 0, 2, 3, 4086, 0, 1, 0, 0, 0, 2, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 9, 4004, 30, 0, 0, 0, 0, 1, 4092, 0, 0, 0, 0, 4, 0, 3, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4095, 4110, 4092, 4093, 4043, 4093, 4097] avg=4096.0000 max=4145 max_rank=0 min=4043 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16326 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4064, 8, 0, 0, 0, 4070, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 26, 4009, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4156, 4088, 4098, 4100, 4063, 4096, 4095] avg=4096.0000 max=4156 max_rank=1 min=4063 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[70, 1, 16, 8, 4053, 9, 0, 0, 7, 3983, 75, 0, 0, 0, 0, 0, 8, 8, 13, 0, 4080, 0, 0, 0, 8, 6, 4068, 0, 5, 2, 0, 0, 2, 0, 1, 4078, 0, 0, 0, 0, 0, 0, 0, 38, 4002, 28, 0, 0, 0, 0, 4, 4088, 6, 0, 0, 0, 12, 0, 13, 4076, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4065, 4109, 4089, 4081, 4068, 4098, 4101] avg=4096.0000 max=4157 max_rank=0 min=4065 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4067, 9, 0, 0, 0, 4038, 86, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 52, 4009, 29, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4124, 4093, 4098, 4096, 4090, 4096, 4095] avg=4096.0000 max=4124 max_rank=1 min=4076 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 128, 128, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 512, 32128, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32128, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3563, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 531, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 11, 1, 1, 0, 4095, 1, 1, 1, 3436, 4095, 0, 1, 4093, 1, 0, 0, 0, 1, 0, 0, 0, 4095, 4085, 0, 1, 0, 659, 4095, 1, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3242, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 4094, 0, 0, 0, 850, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 3144, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 4095, 4096, 0, 0, 0, 947, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4, 0, 0, 4, 4, 0, 4092, 4, 0, 5, 3292, 4091, 4, 3, 4090, 0, 0, 1, 0, 5, 0, 5, 0, 4092, 4090, 1, 3, 0, 800, 4092, 1, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3633, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 462, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[3, 0, 0, 9, 9, 0, 4088, 10, 14, 13, 3618, 4085, 7, 4, 4082, 1, 0, 1, 0, 11, 0, 0, 0, 4087, 4079, 4, 7, 0, 467, 4087, 4, 4078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 2915, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 32, 0, 4096, 4090, 0, 0, 0, 1150, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 1, 3358, 730, 4, 1, 1, 0, 4094, 1, 0, 0, 0, 0, 1, 1, 3436, 737, 0, 0, 0, 0, 0, 1, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4085, 0, 1, 0, 1, 1, 0, 0, 659, 0, 1, 3364, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4096, 4175, 4095, 4095, 4090, 4088, 4024] avg=4096.0000 max=4175 max_rank=2 min=4024 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16297 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3353, 730, 4, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3242, 736, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 13, 0, 4091, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 1, 0, 0, 850, 0, 0, 3366, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4095, 3978, 4096, 4095, 4104, 4096, 4216] avg=4096.0000 max=4216 max_rank=7 min=3978 min_rank=2 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16511 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 3354, 730, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3144, 737, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 1, 0, 0, 947, 0, 0, 3366, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4095, 3881, 4096, 4096, 4099, 4098, 4313] avg=4096.0000 max=4313 max_rank=7 min=3881 min_rank=2 imbalance_ratio=1.052979 max_load_reduction=28455 max_load_reduction_pct=0.868378 moved_to_echo=16606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3358, 730, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3563, 738, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 1, 0, 0, 531, 0, 0, 3366, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4095, 4301, 4096, 4096, 4093, 4098, 3897] avg=4096.0000 max=4301 max_rank=2 min=3897 min_rank=7 imbalance_ratio=1.050049 max_load_reduction=28467 max_load_reduction_pct=0.868744 moved_to_echo=16184 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 4, 3354, 728, 4, 1, 4, 0, 4091, 4, 0, 0, 0, 0, 0, 5, 3292, 737, 0, 0, 0, 0, 4, 3, 4090, 0, 0, 0, 0, 0, 0, 1, 0, 5, 4091, 0, 0, 0, 0, 5, 0, 4087, 0, 0, 0, 0, 4090, 1, 3, 0, 1, 1, 0, 0, 800, 0, 1, 3357, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4099, 4034, 4097, 4097, 4092, 4096, 4158] avg=4096.0000 max=4158 max_rank=7 min=4034 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16443 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3358, 730, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3633, 738, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 1, 0, 0, 462, 0, 0, 3366, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4095, 4371, 4096, 4096, 4091, 4099, 3828] avg=4096.0000 max=4371 max_rank=2 min=3828 min_rank=7 imbalance_ratio=1.067139 max_load_reduction=28397 max_load_reduction_pct=0.866608 moved_to_echo=16114 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 9, 3355, 733, 7, 1, 9, 0, 4079, 10, 0, 0, 0, 0, 14, 13, 3618, 730, 0, 0, 0, 0, 7, 4, 4082, 1, 2, 0, 0, 0, 0, 1, 0, 11, 4086, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4079, 4, 7, 0, 7, 4, 0, 0, 467, 0, 4, 3345, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4098, 4375, 4096, 4098, 4076, 4101, 3816] avg=4096.0000 max=4375 max_rank=2 min=3816 min_rank=7 imbalance_ratio=1.068115 max_load_reduction=28393 max_load_reduction_pct=0.866486 moved_to_echo=16091 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 3358, 730, 4, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2915, 738, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 32, 0, 4091, 0, 0, 0, 0, 4090, 0, 0, 0, 1, 1, 0, 0, 1150, 0, 0, 3366, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4095, 3653, 4096, 4095, 4123, 4092, 4516] avg=4096.0000 max=4516 max_rank=7 min=3653 min_rank=2 imbalance_ratio=1.102539 max_load_reduction=28252 max_load_reduction_pct=0.862183 moved_to_echo=16826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 26880, 5888, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 26880, 6016, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5888, 0, 128, 27008, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4039, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 0, 59, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 3, 0, 4094, 0, 1, 0, 0, 1, 4095, 1, 1, 4092, 0, 0, 0, 0, 4095, 4095, 1, 1, 4095, 0, 1, 0, 4093, 1, 4095, 0, 1, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3895, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 207, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4050, 0, 0, 0, 4, 0, 4096, 0, 0, 4063, 0, 0, 0, 0, 4096, 4092, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 79, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 0, 4090, 0, 4, 0, 0, 4, 4092, 5, 4, 4087, 8, 4, 0, 9, 4090, 4092, 5, 2, 4091, 0, 0, 0, 4082, 4, 4092, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4049, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 47, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[8, 0, 0, 4081, 0, 8, 0, 0, 9, 4085, 11, 11, 4085, 21, 6, 0, 0, 4085, 4087, 11, 1, 4083, 0, 0, 0, 4069, 10, 4084, 0, 5, 4, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3868, 0, 0, 0, 0, 0, 4096, 0, 0, 4072, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 0, 254, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 4094, 73, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 1, 4092, 1, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 3, 2, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 1, 4095, 0, 1, 1, 0, 0, 0, 0, 4093, 1, 6, 0, 0, 0, 0, 0, 1, 1, 1, 4014, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4096, 4095, 4097, 4095, 4098, 4100, 4017] avg=4096.0000 max=4170 max_rank=0 min=4017 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16310 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3895, 73, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 3, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 207, 0, 0, 4015, 0, 0, 0] rerouted_tokens_per_ep_rank=[3968, 4096, 4093, 4095, 4095, 4097, 4102, 4222] avg=4096.0000 max=4222 max_rank=7 min=3968 min_rank=0 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16516 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4039, 73, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 6, 0, 0, 0, 0, 0, 59, 0, 0, 4015, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4096, 4093, 4101, 4095, 4097, 4100, 4074] avg=4096.0000 max=4112 max_rank=0 min=4074 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16366 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4050, 73, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 3, 2, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 79, 0, 0, 4015, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4100, 4093, 4068, 4092, 4096, 4102, 4094] avg=4096.0000 max=4123 max_rank=0 min=4068 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4090, 73, 0, 0, 0, 0, 4, 0, 0, 4090, 0, 0, 0, 4, 4089, 5, 4, 0, 0, 0, 0, 4087, 8, 4, 0, 3, 4, 0, 0, 9, 0, 4092, 5, 0, 0, 0, 0, 2, 4091, 0, 0, 0, 0, 0, 0, 0, 4082, 4, 3, 0, 0, 0, 0, 0, 2, 0, 0, 4012, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4094, 4102, 4106, 4106, 4093, 4089, 4014] avg=4096.0000 max=4164 max_rank=0 min=4014 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16302 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4049, 73, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 47, 0, 0, 4015, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4096, 4093, 4101, 4095, 4097, 4102, 4062] avg=4096.0000 max=4122 max_rank=0 min=4062 min_rank=7 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16356 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 4081, 79, 0, 0, 0, 0, 8, 0, 0, 4085, 0, 0, 0, 9, 4075, 11, 11, 0, 0, 0, 0, 4085, 21, 6, 0, 4, 2, 0, 0, 0, 0, 4084, 11, 0, 0, 0, 0, 1, 4083, 0, 0, 3, 6, 0, 0, 0, 4069, 10, 0, 0, 0, 0, 0, 0, 5, 4, 4, 4003, 0, 0, 0] rerouted_tokens_per_ep_rank=[4168, 4093, 4106, 4118, 4095, 4093, 4079, 4016] avg=4096.0000 max=4168 max_rank=0 min=4016 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16283 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3868, 73, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 3, 7, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 4015, 0, 0, 0] rerouted_tokens_per_ep_rank=[3941, 4097, 4093, 4082, 4095, 4097, 4094, 4269] avg=4096.0000 max=4269 max_rank=7 min=3941 min_rank=0 imbalance_ratio=1.042236 max_load_reduction=28499 max_load_reduction_pct=0.869720 moved_to_echo=16555 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 768, 128, 128, 32128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32256, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4091, 4092, 0, 11, 7, 0, 4092, 0, 0, 0, 0, 3, 0, 4, 6, 4082, 4, 4, 0, 0, 5, 0, 1, 0, 4088, 4092, 4, 0, 0, 4092, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1, 4094, 4095, 0, 3, 1, 0, 4095, 0, 0, 0, 0, 1, 0, 1, 2, 4091, 1, 1, 0, 1, 2, 0, 0, 0, 4094, 4095, 1, 0, 0, 4095, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4085, 4083, 0, 33, 14, 0, 4087, 0, 0, 0, 0, 7, 0, 10, 16, 4078, 10, 11, 0, 0, 13, 0, 0, 0, 4056, 4086, 9, 0, 2, 4086, 4082] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 3, 4095, 0, 0, 0, 0, 0, 3, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 1, 0, 1, 2, 4085, 5, 0, 0, 4091, 1, 1, 0, 1, 0, 0, 0, 1, 2, 0, 0, 4087, 5, 0, 0, 0, 4094, 5, 1, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4093, 4096, 4094, 4094, 4095, 4100, 4097] avg=4096.0000 max=4100 max_rank=6 min=4093 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 4086, 5, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4090, 4097, 4091, 4097, 4094, 4101, 4099] avg=4096.0000 max=4101 max_rank=6 min=4090 min_rank=1 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 4086, 5, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4090, 4097, 4091, 4097, 4094, 4101, 4099] avg=4096.0000 max=4101 max_rank=6 min=4090 min_rank=1 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 4086, 5, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4090, 4097, 4091, 4097, 4094, 4101, 4099] avg=4096.0000 max=4101 max_rank=6 min=4090 min_rank=1 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4092, 0, 0, 0, 0, 0, 11, 7, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 3, 0, 4, 6, 4082, 5, 0, 0, 4082, 4, 4, 0, 1, 0, 0, 0, 0, 5, 0, 1, 4085, 5, 0, 0, 0, 4088, 5, 4, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4104, 4093, 4100, 4091, 4096, 4097, 4093] avg=4096.0000 max=4104 max_rank=1 min=4091 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 4086, 5, 0, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 4089, 5, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4089, 4098, 4091, 4093, 4098, 4101, 4099] avg=4096.0000 max=4101 max_rank=6 min=4089 min_rank=1 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4083, 0, 0, 0, 0, 0, 33, 14, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 0, 0, 7, 0, 10, 16, 4077, 9, 0, 0, 4078, 10, 11, 0, 3, 0, 0, 0, 0, 13, 0, 0, 4079, 8, 0, 0, 0, 4056, 1, 9, 0, 0, 0, 0, 0, 2, 0, 4082, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4122, 4092, 4119, 4102, 4100, 4066, 4084] avg=4096.0000 max=4122 max_rank=1 min=4066 min_rank=6 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 0, 0, 0, 4086, 5, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4090, 4098, 4091, 4097, 4097, 4101, 4098] avg=4096.0000 max=4101 max_rank=6 min=4090 min_rank=1 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 4061, 2079, 2, 957, 0, 677, 1, 1765, 539, 5, 2334, 1609, 893, 0, 2272, 2102, 1988, 134, 173, 20, 2401, 983, 0, 0, 365, 171, 1109, 2357, 19, 3751, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1, 3940, 1813, 0, 935, 0, 479, 0, 1360, 967, 7, 2752, 1874, 710, 0, 2378, 2577, 1360, 377, 373, 37, 2844, 915, 10, 0, 356, 241, 1286, 1760, 24, 3392, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 3938, 1760, 4, 974, 1, 647, 0, 1821, 556, 3, 2261, 1604, 743, 0, 2909, 2336, 1079, 539, 338, 6, 2407, 1233, 5, 0, 235, 190, 1463, 1923, 15, 3778, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1, 3955, 1746, 0, 1008, 0, 578, 1, 2154, 918, 17, 2201, 2185, 788, 1, 2623, 1945, 1351, 103, 418, 45, 2412, 1035, 10, 0, 373, 305, 1404, 1452, 30, 3708, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2, 3900, 1803, 12, 1006, 4, 456, 11, 1666, 558, 0, 1962, 1667, 778, 5, 2647, 2480, 1714, 395, 201, 51, 2808, 928, 7, 0, 109, 208, 1230, 2340, 35, 3785, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 3981, 1753, 2, 821, 0, 756, 0, 1752, 968, 3, 2353, 2257, 659, 0, 2507, 1969, 1709, 174, 287, 21, 2326, 623, 1, 0, 402, 183, 1023, 2290, 13, 3932, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3, 3594, 2142, 6, 844, 0, 729, 2, 2020, 577, 14, 2793, 2049, 658, 2, 2524, 2036, 1154, 260, 369, 75, 2432, 762, 6, 1, 427, 182, 1109, 2197, 34, 3761, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1, 4011, 1859, 3, 901, 1, 523, 19, 1979, 760, 5, 2627, 1800, 1011, 4, 2440, 2363, 1131, 224, 398, 31, 2563, 640, 1, 3, 150, 267, 1195, 2119, 23, 3714, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2301, 2079, 2, 0, 0, 0, 0, 957, 0, 677, 1, 1722, 872, 0, 0, 1765, 539, 5, 1497, 0, 0, 0, 0, 1609, 893, 0, 1284, 0, 0, 0, 0, 1950, 1988, 134, 173, 0, 0, 0, 0, 20, 2401, 983, 0, 464, 152, 0, 0, 0, 365, 171, 1109, 988, 888, 373, 0, 2357, 19, 2029, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4382, 4229, 3806, 3786, 4245, 4020, 3894, 4406] avg=4096.0000 max=4406 max_rank=7 min=3786 min_rank=3 imbalance_ratio=1.075684 max_load_reduction=28362 max_load_reduction_pct=0.865540 moved_to_echo=16565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2232, 1760, 4, 0, 0, 0, 0, 974, 1, 647, 0, 1735, 845, 0, 0, 1821, 556, 3, 1449, 0, 0, 0, 0, 1604, 743, 0, 1644, 0, 0, 0, 0, 2167, 1079, 539, 338, 0, 0, 0, 0, 6, 2407, 1233, 5, 450, 169, 0, 0, 0, 235, 190, 1463, 1265, 861, 362, 0, 1923, 15, 2043, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4202, 3829, 3991, 4123, 4270, 4376, 3981] avg=4096.0000 max=4376 max_rank=6 min=3829 min_rank=2 imbalance_ratio=1.068359 max_load_reduction=28392 max_load_reduction_pct=0.866455 moved_to_echo=16750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 2233, 1813, 0, 0, 0, 0, 0, 935, 0, 479, 0, 1557, 846, 0, 0, 1360, 967, 7, 1764, 0, 0, 0, 0, 1874, 710, 0, 1344, 0, 0, 0, 0, 2391, 1360, 377, 373, 0, 0, 0, 0, 37, 2844, 915, 10, 548, 186, 0, 0, 0, 356, 241, 1286, 1034, 861, 440, 0, 1760, 24, 1835, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 3817, 4098, 3928, 4501, 4540, 4218, 3619] avg=4096.0000 max=4540 max_rank=5 min=3619 min_rank=7 imbalance_ratio=1.108398 max_load_reduction=28228 max_load_reduction_pct=0.861450 moved_to_echo=16878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 2210, 1803, 12, 0, 0, 0, 0, 1006, 4, 456, 11, 1738, 837, 0, 0, 1666, 558, 0, 1258, 0, 0, 0, 0, 1667, 778, 5, 1496, 0, 0, 0, 0, 2301, 1714, 395, 201, 0, 0, 0, 0, 51, 2808, 928, 7, 390, 179, 0, 0, 0, 109, 208, 1230, 1151, 853, 314, 0, 2340, 35, 2047, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4052, 3482, 3946, 4611, 4363, 3865, 4422] avg=4096.0000 max=4611 max_rank=4 min=3482 min_rank=2 imbalance_ratio=1.125732 max_load_reduction=28157 max_load_reduction_pct=0.859283 moved_to_echo=17261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 2241, 1746, 0, 0, 0, 0, 0, 1008, 0, 578, 1, 1703, 849, 0, 0, 2154, 918, 17, 1411, 0, 0, 0, 0, 2185, 788, 1, 1483, 0, 0, 0, 0, 1805, 1351, 103, 418, 0, 0, 0, 0, 45, 2412, 1035, 10, 438, 140, 0, 0, 0, 373, 305, 1404, 1140, 865, 352, 0, 1452, 30, 2005, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3988, 4139, 4500, 4457, 3677, 4080, 4439, 3488] avg=4096.0000 max=4500 max_rank=2 min=3488 min_rank=7 imbalance_ratio=1.098633 max_load_reduction=28268 max_load_reduction_pct=0.862671 moved_to_echo=15684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2249, 1753, 2, 0, 0, 0, 0, 821, 0, 756, 0, 1810, 859, 0, 0, 1752, 968, 3, 1501, 0, 0, 0, 0, 2257, 659, 0, 1414, 0, 0, 0, 0, 1824, 1709, 174, 287, 0, 0, 0, 0, 21, 2326, 623, 1, 472, 145, 0, 0, 0, 402, 183, 1023, 1093, 873, 380, 0, 2290, 13, 2122, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4004, 4246, 4224, 4330, 3994, 3588, 3954, 4428] avg=4096.0000 max=4428 max_rank=7 min=3588 min_rank=5 imbalance_ratio=1.081055 max_load_reduction=28340 max_load_reduction_pct=0.864868 moved_to_echo=15964 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 2037, 2142, 6, 0, 0, 0, 0, 844, 0, 729, 2, 1727, 771, 0, 0, 2020, 577, 14, 1790, 0, 0, 0, 0, 2049, 658, 2, 1427, 0, 0, 0, 0, 1889, 1154, 260, 369, 0, 0, 0, 0, 75, 2432, 762, 6, 556, 147, 0, 0, 1, 427, 182, 1109, 1097, 786, 447, 0, 2197, 34, 2034, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4188, 4073, 4401, 4136, 3672, 3978, 4049, 4271] avg=4096.0000 max=4401 max_rank=2 min=3672 min_rank=4 imbalance_ratio=1.074463 max_load_reduction=28367 max_load_reduction_pct=0.865692 moved_to_echo=15970 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 2273, 1859, 3, 0, 0, 0, 0, 901, 1, 523, 19, 1705, 861, 0, 0, 1979, 760, 5, 1684, 0, 0, 0, 0, 1800, 1011, 4, 1379, 0, 0, 0, 0, 2192, 1131, 224, 398, 0, 0, 0, 0, 31, 2563, 640, 1, 523, 171, 0, 0, 3, 150, 267, 1195, 1061, 877, 420, 0, 2119, 23, 2009, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4010, 4428, 4194, 3945, 3929, 3973, 4153] avg=4096.0000 max=4428 max_rank=2 min=3929 min_rank=5 imbalance_ratio=1.081055 max_load_reduction=28340 max_load_reduction_pct=0.864868 moved_to_echo=16000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 2432, 1792, 9856, 8832, 6912, 3200, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7552, 128, 4864, 128, 13824, 6784, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 17792, 14976, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16640, 11520, 2304, 2560, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16512, 256, 16128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15104, 6272, 128, 11520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 20224, 7168, 128, 3968, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14592, 5888, 128, 12416, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[17, 0, 4081, 3326, 0, 1, 0, 4078, 217, 2962, 0, 0, 0, 0, 2727, 2, 7, 0, 4096, 0, 6, 3, 0, 4093, 1, 0, 0, 1, 0, 0, 3055, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[86, 0, 4095, 4069, 0, 0, 0, 4096, 88, 2733, 0, 0, 0, 1, 1309, 4, 1, 0, 4095, 0, 0, 0, 0, 4088, 1, 1, 12, 191, 82, 0, 3720, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[21, 0, 4089, 3951, 1, 0, 0, 4096, 111, 2933, 0, 0, 0, 0, 1367, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 179, 2, 0, 3730, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4054, 4024, 0, 0, 0, 4096, 642, 2361, 0, 0, 0, 1, 3096, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 1, 0, 2182, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[46, 0, 4074, 3942, 1, 0, 0, 4096, 297, 2156, 0, 0, 0, 0, 2598, 0, 11, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 27, 3, 0, 3232, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[9, 1, 4045, 3693, 0, 0, 1, 4093, 396, 1899, 0, 0, 3, 13, 2524, 3, 0, 0, 4089, 1, 0, 3, 14, 4083, 0, 3, 0, 79, 52, 5, 3670, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 4, 4078, 3896, 0, 5, 0, 4087, 297, 1906, 0, 2, 0, 7, 3065, 0, 0, 0, 4073, 0, 29, 0, 11, 4091, 0, 3, 1, 40, 11, 0, 3071, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4, 0, 4088, 3821, 0, 0, 0, 4096, 136, 3137, 0, 0, 0, 0, 1459, 0, 2, 0, 4094, 0, 0, 0, 2, 4096, 0, 0, 2, 114, 41, 0, 3584, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 233, 3326, 0, 0, 0, 0, 0, 1, 0, 4078, 2, 0, 0, 0, 217, 2962, 0, 0, 1310, 1, 0, 0, 0, 0, 2727, 2, 1824, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 6, 3, 0, 4090, 0, 0, 0, 0, 1, 0, 0, 1, 3848, 168, 0, 0, 0, 0, 3055, 793, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3576, 4081, 4490, 4553, 4103, 4099, 4018, 3848] avg=4096.0000 max=4553 max_rank=3 min=3576 min_rank=0 imbalance_ratio=1.111572 max_load_reduction=28215 max_load_reduction_pct=0.861053 moved_to_echo=16068 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[86, 0, 234, 4069, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 88, 2733, 0, 0, 1311, 1, 0, 0, 0, 1, 1309, 4, 1825, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 1, 1, 12, 191, 3861, 168, 0, 0, 82, 0, 3720, 792, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4389, 4098, 4133, 3139, 4096, 4085, 4234, 4594] avg=4096.0000 max=4594 max_rank=7 min=3139 min_rank=3 imbalance_ratio=1.121582 max_load_reduction=28174 max_load_reduction_pct=0.859802 moved_to_echo=17009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 234, 3951, 0, 0, 0, 0, 1, 0, 0, 4096, 2, 0, 0, 0, 111, 2933, 0, 0, 1310, 1, 0, 0, 0, 0, 1367, 0, 1824, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 179, 3855, 168, 0, 0, 2, 0, 3730, 793, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4206, 4099, 4355, 3191, 4097, 4093, 4202, 4525] avg=4096.0000 max=4525 max_rank=7 min=3191 min_rank=3 imbalance_ratio=1.104736 max_load_reduction=28243 max_load_reduction_pct=0.861908 moved_to_echo=16917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[9, 1, 231, 3693, 0, 0, 0, 0, 0, 0, 1, 4093, 2, 0, 0, 0, 396, 1899, 0, 0, 1309, 1, 0, 0, 3, 13, 2524, 3, 1821, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 3, 14, 4080, 0, 0, 0, 0, 0, 3, 0, 79, 3814, 167, 0, 0, 52, 5, 3670, 792, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3934, 4096, 3605, 4364, 4090, 4097, 4063, 4519] avg=4096.0000 max=4519 max_rank=7 min=3605 min_rank=2 imbalance_ratio=1.103271 max_load_reduction=28249 max_load_reduction_pct=0.862091 moved_to_echo=16769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[46, 0, 233, 3942, 0, 0, 0, 0, 1, 0, 0, 4096, 2, 0, 0, 0, 297, 2156, 0, 0, 1310, 1, 0, 0, 0, 0, 2598, 0, 1823, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 27, 3841, 168, 0, 0, 3, 0, 3232, 792, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4221, 4099, 3764, 4421, 4107, 4093, 4036, 4027] avg=4096.0000 max=4421 max_rank=3 min=3764 min_rank=2 imbalance_ratio=1.079346 max_load_reduction=28347 max_load_reduction_pct=0.865082 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 227, 4024, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 642, 2361, 0, 0, 1314, 6, 0, 0, 0, 1, 3096, 0, 1827, 0, 0, 0, 5, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 22, 3827, 170, 0, 0, 1, 0, 2182, 781, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4251, 4103, 4323, 4924, 4106, 4078, 4019, 2964] avg=4096.0000 max=4924 max_rank=3 min=2964 min_rank=7 imbalance_ratio=1.202148 max_load_reduction=27844 max_load_reduction_pct=0.849731 moved_to_echo=15167 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 233, 3821, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 136, 3137, 0, 0, 1309, 1, 0, 0, 0, 0, 1459, 0, 1823, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 2, 114, 3855, 168, 0, 0, 41, 0, 3584, 792, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4098, 4583, 3282, 4096, 4095, 4139, 4417] avg=4096.0000 max=4583 max_rank=2 min=3282 min_rank=3 imbalance_ratio=1.118896 max_load_reduction=28185 max_load_reduction_pct=0.860138 moved_to_echo=16747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 233, 3896, 0, 0, 0, 0, 0, 5, 0, 4087, 2, 0, 0, 0, 297, 1906, 0, 2, 1309, 1, 0, 0, 0, 7, 3065, 0, 1822, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 29, 0, 11, 4088, 0, 0, 0, 0, 0, 3, 1, 40, 3845, 168, 0, 0, 11, 0, 3071, 792, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4094, 3515, 4894, 4073, 4128, 4057, 3874] avg=4096.0000 max=4894 max_rank=3 min=3515 min_rank=2 imbalance_ratio=1.194824 max_load_reduction=27874 max_load_reduction_pct=0.850647 moved_to_echo=16132 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 18176, 128, 14592, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2304, 20096, 0, 128, 10496, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 26368, 6400, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 1920, 30848, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 768, 30848, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 0, 0, 6, 4096, 0, 0, 83, 4094, 0, 1, 0, 1212, 0, 0, 0, 0, 4095, 2810, 0, 2, 0, 1, 4084, 0, 4092, 5, 4090, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 16, 4095, 0, 0, 2, 602, 0, 0, 0, 0, 4093, 3484, 0, 0, 0, 0, 4094, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4090, 6, 0, 0, 0, 0, 0, 4095, 0, 1, 71, 4093, 0, 0, 0, 2299, 0, 0, 0, 0, 4095, 1763, 0, 0, 1, 0, 4062, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 4095, 0, 0, 0, 359, 0, 0, 0, 0, 4022, 3804, 0, 0, 0, 0, 4094, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 56, 4096, 0, 0, 1, 1988, 0, 0, 0, 0, 4091, 2120, 0, 0, 0, 0, 4036, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4096, 2, 0, 0, 0, 0, 0, 4096, 0, 2, 13, 4094, 0, 0, 2, 345, 0, 0, 0, 0, 4095, 3739, 0, 0, 0, 0, 4092, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4086, 0, 0, 1, 2, 3, 0, 4089, 7, 11, 76, 4092, 6, 1, 0, 607, 0, 2, 0, 0, 4088, 3399, 0, 0, 11, 5, 4080, 23, 4087, 0, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4096, 2, 1, 0, 0, 0, 0, 4095, 0, 17, 3, 4093, 1, 0, 4, 870, 0, 0, 0, 0, 4072, 3227, 0, 0, 0, 0, 4095, 0, 4094, 0, 4096, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 1, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 83, 4051, 0, 0, 0, 0, 0, 1, 0, 1212, 3036, 31, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 1057, 2810, 0, 2, 0, 0, 0, 0, 0, 1, 4084, 0, 11, 0, 0, 0, 4092, 5, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4102, 4134, 4280, 4091, 3869, 4096, 4098] avg=4096.0000 max=4280 max_rank=3 min=3869 min_rank=5 imbalance_ratio=1.044922 max_load_reduction=28488 max_load_reduction_pct=0.869385 moved_to_echo=16154 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4090, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 71, 4050, 0, 0, 0, 0, 0, 0, 0, 2299, 3036, 31, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 1057, 1763, 0, 0, 0, 0, 0, 0, 1, 0, 4062, 0, 11, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4095, 4122, 5366, 4096, 2820, 4074, 4098] avg=4096.0000 max=5366 max_rank=3 min=2820 min_rank=5 imbalance_ratio=1.310059 max_load_reduction=27402 max_load_reduction_pct=0.836243 moved_to_echo=15088 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 16, 4052, 0, 0, 0, 0, 0, 0, 2, 602, 3035, 31, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 1056, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 11, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4069, 3670, 4096, 4540, 4105, 4098] avg=4096.0000 max=4540 max_rank=5 min=3670 min_rank=3 imbalance_ratio=1.108398 max_load_reduction=28228 max_load_reduction_pct=0.861450 moved_to_echo=16839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 4052, 0, 0, 0, 0, 0, 0, 0, 359, 2982, 31, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 1039, 3804, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 11, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4062, 3372, 4095, 4843, 4105, 4098] avg=4096.0000 max=4843 max_rank=5 min=3372 min_rank=3 imbalance_ratio=1.182373 max_load_reduction=27925 max_load_reduction_pct=0.852203 moved_to_echo=17141 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 0, 1, 1, 0, 0, 0, 2, 3, 0, 4089, 0, 0, 0, 0, 7, 11, 76, 4049, 0, 0, 0, 0, 6, 1, 0, 607, 3031, 31, 0, 0, 0, 2, 0, 0, 4091, 2, 0, 0, 1055, 3399, 0, 0, 0, 0, 0, 0, 11, 5, 4080, 23, 11, 0, 0, 0, 4087, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4094, 4143, 3676, 4095, 4454, 4130, 4088] avg=4096.0000 max=4454 max_rank=5 min=3676 min_rank=3 imbalance_ratio=1.087402 max_load_reduction=28314 max_load_reduction_pct=0.864075 moved_to_echo=16767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 13, 4051, 0, 0, 0, 0, 0, 0, 2, 345, 3036, 31, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 1057, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 4066, 3414, 4098, 4796, 4103, 4096] avg=4096.0000 max=4796 max_rank=5 min=3414 min_rank=3 imbalance_ratio=1.170898 max_load_reduction=27972 max_load_reduction_pct=0.853638 moved_to_echo=17093 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 56, 4045, 0, 0, 0, 0, 0, 0, 1, 1988, 3040, 36, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 1047, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4094, 4102, 5065, 4100, 3167, 4049, 4096] avg=4096.0000 max=5065 max_rank=3 min=3167 min_rank=5 imbalance_ratio=1.236572 max_load_reduction=27703 max_load_reduction_pct=0.845428 moved_to_echo=15412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 17, 3, 4050, 0, 0, 0, 0, 1, 0, 4, 870, 3019, 31, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1052, 3227, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 11, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4095, 4070, 3925, 4097, 4279, 4106, 4096] avg=4096.0000 max=4279 max_rank=5 min=3925 min_rank=3 imbalance_ratio=1.044678 max_load_reduction=28489 max_load_reduction_pct=0.869415 moved_to_echo=16578 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 8320, 24320, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8448, 24448, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 384, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 4047, 4093, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 4094, 1996, 2, 4096, 0, 0, 7, 2, 2041, 0, 0, 4, 4092, 0, 102] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4093, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1288, 0, 4096, 0, 0, 0, 0, 2732, 0, 0, 0, 4095, 0, 80] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 4090, 1, 4, 6, 4091, 4090, 0, 0, 5, 4092, 0, 2, 5, 9, 0, 4094, 3099, 6, 4085, 3, 0, 0, 0, 539, 0, 2, 2, 4091, 0, 451] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4087, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2724, 0, 4096, 0, 0, 2, 0, 923, 0, 0, 0, 4092, 0, 460] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4087, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1726, 0, 4096, 0, 0, 0, 0, 2286, 0, 0, 0, 4096, 0, 93] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1546, 0, 4095, 0, 0, 0, 0, 2550, 0, 0, 0, 4096, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2719, 0, 4096, 0, 0, 0, 0, 938, 0, 0, 0, 4096, 0, 439] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4063, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1623, 1, 4096, 0, 0, 7, 0, 2463, 0, 0, 0, 4089, 0, 42] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4047, 13, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 11, 0, 0, 0, 2005, 1996, 2, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 2, 2041, 0, 0, 2078, 206, 0, 0, 4, 3886, 0, 102, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4060, 4099, 4091, 4003, 4103, 4327, 3992] avg=4096.0000 max=4327 max_rank=6 min=3992 min_rank=7 imbalance_ratio=1.056396 max_load_reduction=28441 max_load_reduction_pct=0.867950 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4090, 1, 0, 0, 0, 0, 4, 6, 4091, 13, 0, 0, 0, 0, 0, 0, 5, 4092, 0, 0, 0, 0, 0, 2, 5, 9, 4077, 11, 0, 0, 0, 2005, 3099, 6, 0, 0, 0, 0, 4085, 3, 0, 0, 0, 0, 0, 0, 0, 539, 0, 2, 2078, 206, 0, 0, 2, 3885, 0, 451, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4114, 4097, 4104, 5110, 4088, 2825, 4338] avg=4096.0000 max=5110 max_rank=4 min=2825 min_rank=6 imbalance_ratio=1.247559 max_load_reduction=27658 max_load_reduction_pct=0.844055 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 11, 0, 0, 0, 2006, 1288, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2732, 0, 0, 2079, 206, 0, 0, 0, 3889, 0, 80, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4106, 4096, 4094, 3294, 4096, 5017, 3969] avg=4096.0000 max=5017 max_rank=6 min=3294 min_rank=4 imbalance_ratio=1.224854 max_load_reduction=27751 max_load_reduction_pct=0.846893 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 11, 0, 0, 0, 2006, 2724, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 923, 0, 0, 2079, 206, 0, 0, 0, 3886, 0, 460, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4100, 4096, 4094, 4730, 4098, 3208, 4346] avg=4096.0000 max=4730 max_rank=4 min=3208 min_rank=6 imbalance_ratio=1.154785 max_load_reduction=28038 max_load_reduction_pct=0.855652 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 11, 0, 0, 0, 2006, 1546, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2550, 0, 0, 2079, 206, 0, 0, 0, 3890, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4109, 4096, 4094, 3552, 4095, 4835, 3891] avg=4096.0000 max=4835 max_rank=6 min=3552 min_rank=4 imbalance_ratio=1.180420 max_load_reduction=27933 max_load_reduction_pct=0.852448 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 11, 0, 0, 0, 2006, 1726, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2286, 0, 0, 2079, 206, 0, 0, 0, 3890, 0, 93, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4100, 4096, 4094, 3732, 4096, 4571, 3983] avg=4096.0000 max=4571 max_rank=6 min=3732 min_rank=4 imbalance_ratio=1.115967 max_load_reduction=28197 max_load_reduction_pct=0.860504 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4, 3, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 14, 0, 0, 0, 1998, 2719, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 938, 0, 0, 2084, 211, 4, 0, 0, 3885, 0, 439, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4103, 4092, 4103, 4717, 4089, 3237, 4324] avg=4096.0000 max=4717 max_rank=4 min=3237 min_rank=6 imbalance_ratio=1.151611 max_load_reduction=28051 max_load_reduction_pct=0.856049 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4063, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 11, 0, 0, 0, 2006, 1623, 1, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 2463, 0, 0, 2079, 206, 0, 0, 0, 3883, 0, 42, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4076, 4096, 4094, 3630, 4103, 4748, 3925] avg=4096.0000 max=4748 max_rank=6 min=3630 min_rank=4 imbalance_ratio=1.159180 max_load_reduction=28020 max_load_reduction_pct=0.855103 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 14592, 0, 128, 16640, 1664, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 16128, 16768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 31104, 0, 1792, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4079, 4096, 7, 0, 0, 2714, 28, 6, 1, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 3, 2071, 4096, 4096, 0, 0, 0, 0, 3380, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4096, 1, 0, 0, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2489, 4096, 4095, 0, 0, 0, 2, 3422, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4062, 4096, 0, 0, 0, 3520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 6, 2766, 4096, 4096, 0, 0, 0, 1, 1943, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4086, 4096, 3, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 877, 4096, 4096, 0, 0, 0, 0, 3335, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4088, 4096, 1, 0, 0, 1908, 12, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 2727, 4096, 4096, 0, 0, 0, 0, 3558, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[5, 7, 4082, 4089, 0, 1, 15, 1719, 37, 0, 0, 7, 5, 7, 0, 0, 5, 0, 4089, 5, 0, 4092, 0, 3046, 4091, 4081, 6, 1, 2, 0, 3376, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4093, 4096, 1, 0, 0, 3744, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 2058, 4096, 4096, 0, 0, 0, 5, 2394, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4012, 4096, 0, 0, 0, 3606, 3, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 4079, 6, 986, 4096, 4096, 0, 0, 0, 0, 3673, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4079, 21, 0, 0, 0, 0, 7, 0, 0, 2714, 1157, 0, 0, 0, 28, 6, 1, 2, 4075, 6, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1970, 3, 2071, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3380, 0, 960, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 3878, 4118, 4094, 4096, 4044, 4098, 4340] avg=4096.0000 max=4340 max_rank=7 min=3878 min_rank=1 imbalance_ratio=1.059570 max_load_reduction=28428 max_load_reduction_pct=0.867554 moved_to_echo=16578 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4062, 21, 0, 0, 0, 0, 0, 0, 0, 3520, 1154, 0, 0, 0, 1, 0, 0, 0, 4075, 6, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1967, 6, 2766, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 1943, 0, 958, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4674, 4082, 4095, 4096, 4739, 4097, 2902] avg=4096.0000 max=4739 max_rank=5 min=2902 min_rank=7 imbalance_ratio=1.156982 max_load_reduction=28029 max_load_reduction_pct=0.855377 moved_to_echo=15834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 7, 4082, 21, 0, 0, 0, 0, 0, 1, 15, 1719, 1156, 0, 0, 0, 37, 0, 0, 7, 4068, 6, 0, 0, 5, 7, 0, 0, 4089, 0, 0, 0, 5, 0, 4089, 5, 0, 0, 0, 0, 0, 1970, 0, 3046, 0, 0, 0, 0, 2, 4081, 6, 1, 0, 0, 0, 0, 2, 0, 3376, 0, 960, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 2891, 4118, 4101, 4099, 5016, 4090, 4338] avg=4096.0000 max=5016 max_rank=5 min=2891 min_rank=1 imbalance_ratio=1.224609 max_load_reduction=27752 max_load_reduction_pct=0.846924 moved_to_echo=17543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 21, 0, 0, 0, 0, 1, 0, 0, 2281, 1157, 0, 0, 0, 0, 0, 0, 0, 4075, 6, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1972, 0, 2489, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 3422, 0, 961, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 3439, 4081, 4094, 4096, 4461, 4097, 4385] avg=4096.0000 max=4461 max_rank=5 min=3439 min_rank=1 imbalance_ratio=1.089111 max_load_reduction=28307 max_load_reduction_pct=0.863861 moved_to_echo=17039 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4088, 21, 0, 0, 0, 0, 1, 0, 0, 1908, 1155, 0, 0, 0, 12, 0, 2, 0, 4075, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1968, 0, 2727, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 0, 959, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 3064, 4095, 4096, 4096, 4695, 4096, 4517] avg=4096.0000 max=4695 max_rank=5 min=3064 min_rank=1 imbalance_ratio=1.146240 max_load_reduction=28073 max_load_reduction_pct=0.856720 moved_to_echo=17404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4086, 21, 0, 0, 0, 0, 3, 0, 0, 4003, 1153, 0, 0, 0, 0, 0, 0, 0, 4075, 6, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1964, 0, 877, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3335, 0, 957, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 5159, 4081, 4094, 4096, 2841, 4098, 4292] avg=4096.0000 max=5159 max_rank=1 min=2841 min_rank=5 imbalance_ratio=1.259521 max_load_reduction=27609 max_load_reduction_pct=0.842560 moved_to_echo=15327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 13, 0, 0, 0, 0, 1, 0, 0, 3744, 1159, 0, 0, 0, 1, 0, 0, 0, 4081, 8, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1958, 0, 2058, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 2394, 0, 963, 2, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4904, 4090, 4098, 4094, 4016, 4096, 3364] avg=4096.0000 max=4904 max_rank=1 min=3364 min_rank=7 imbalance_ratio=1.197266 max_load_reduction=27864 max_load_reduction_pct=0.850342 moved_to_echo=15570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4012, 21, 0, 0, 0, 0, 0, 0, 0, 3606, 1153, 0, 0, 0, 3, 0, 19, 0, 4075, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 1963, 6, 986, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3673, 0, 957, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4759, 4103, 4096, 4095, 2956, 4096, 4630] avg=4096.0000 max=4759 max_rank=1 min=2956 min_rank=5 imbalance_ratio=1.161865 max_load_reduction=28009 max_load_reduction_pct=0.854767 moved_to_echo=15777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 15744, 128, 17024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 23552, 9344, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 25088, 0, 7680, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4089, 109, 390, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3645, 7, 0, 0, 0, 0, 0, 0, 4059, 0, 3, 4082, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 45, 2773, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1325, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 6, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4090, 390, 2965, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 636, 6, 0, 0, 3, 0, 0, 0, 4085, 0, 127, 4082, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4075, 120, 2767, 4091, 0, 11, 0, 3, 6, 8, 4084, 0, 0, 1264, 52, 0, 0, 0, 8, 4, 0, 4074, 7, 0, 4037, 1, 0, 7, 4088, 5, 4052, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 113, 2394, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1622, 1, 0, 0, 0, 0, 0, 0, 4089, 0, 2, 4067, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4063, 470, 1850, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1851, 3, 0, 0, 0, 0, 0, 0, 4047, 0, 16, 4084, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4095, 108, 1898, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2117, 7, 0, 0, 0, 0, 0, 0, 4068, 0, 1, 4090, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4076, 724, 1737, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1726, 14, 0, 0, 3, 0, 0, 0, 4023, 0, 17, 4064, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1743, 390, 2965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 636, 6, 0, 2314, 0, 0, 0, 0, 3, 0, 0, 4091, 2, 1, 0, 0, 4085, 0, 127, 12, 0, 0, 0, 4082, 0, 0, 0, 19, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5098, 4095, 4096, 2956, 4097, 4224, 4101, 4101] avg=4096.0000 max=5098 max_rank=0 min=2956 min_rank=3 imbalance_ratio=1.244629 max_load_reduction=27670 max_load_reduction_pct=0.844421 moved_to_echo=16523 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1738, 120, 2767, 0, 0, 0, 0, 0, 0, 11, 0, 3, 4089, 0, 0, 0, 6, 8, 4084, 0, 0, 0, 0, 0, 0, 1264, 52, 0, 2305, 0, 0, 0, 0, 0, 8, 4, 4083, 1, 2, 0, 0, 4074, 7, 0, 12, 0, 0, 0, 4037, 1, 0, 7, 19, 0, 0, 0, 5, 5, 4052, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4625, 4103, 4098, 3621, 4098, 4093, 4064, 4066] avg=4096.0000 max=4625 max_rank=0 min=3621 min_rank=3 imbalance_ratio=1.129150 max_load_reduction=28143 max_load_reduction_pct=0.858856 moved_to_echo=16321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1743, 109, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3645, 7, 0, 2313, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 1, 0, 0, 4059, 0, 3, 12, 0, 0, 0, 4082, 0, 0, 0, 19, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2242, 4095, 4096, 5965, 4094, 4074, 4101, 4101] avg=4096.0000 max=5965 max_rank=3 min=2242 min_rank=0 imbalance_ratio=1.456299 max_load_reduction=26803 max_load_reduction_pct=0.817963 moved_to_echo=16370 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1746, 45, 2773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1325, 0, 0, 2317, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 1, 0, 0, 4043, 0, 6, 12, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4564, 4095, 4096, 3642, 4094, 4061, 4115, 4101] avg=4096.0000 max=4564 max_rank=0 min=3642 min_rank=3 imbalance_ratio=1.114258 max_load_reduction=28204 max_load_reduction_pct=0.860718 moved_to_echo=16371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1746, 113, 2394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1622, 1, 0, 2317, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 1, 0, 0, 4089, 0, 2, 12, 0, 0, 0, 4067, 0, 0, 0, 19, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4253, 4095, 4096, 3940, 4094, 4103, 4086, 4101] avg=4096.0000 max=4253 max_rank=0 min=3940 min_rank=3 imbalance_ratio=1.038330 max_load_reduction=28515 max_load_reduction_pct=0.870209 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1732, 470, 1850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1851, 3, 0, 2299, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 1, 0, 0, 4047, 0, 16, 12, 0, 0, 0, 4084, 0, 0, 0, 19, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4095, 4096, 4153, 4093, 4075, 4103, 4101] avg=4096.0000 max=4153 max_rank=3 min=4052 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1729, 108, 1898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2117, 7, 0, 2321, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 1, 0, 0, 4068, 0, 1, 15, 2, 0, 0, 4090, 0, 0, 0, 25, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3735, 4095, 4094, 4445, 4102, 4086, 4115, 4096] avg=4096.0000 max=4445 max_rank=3 min=3735 min_rank=0 imbalance_ratio=1.085205 max_load_reduction=28323 max_load_reduction_pct=0.864349 moved_to_echo=16399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1738, 724, 1737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1726, 14, 0, 2306, 0, 0, 0, 0, 3, 0, 0, 4091, 1, 1, 0, 0, 4023, 0, 17, 12, 0, 0, 0, 4064, 0, 0, 0, 19, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4199, 4095, 4096, 4046, 4096, 4052, 4083, 4101] avg=4096.0000 max=4199 max_rank=0 min=4046 min_rank=3 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16332 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13952, 2176, 16896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 14208, 128, 0, 18560, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 0, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32512, 128, 256, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 4096, 0, 0, 4088, 4096, 4094, 0, 0, 1, 0, 0, 8, 0, 4045, 84, 4096, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 10, 0, 4096, 2, 4096, 0, 4079, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4068, 3, 4087, 0, 4088, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4049, 6, 4082, 0, 4096, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 5, 0, 4096, 15, 4096, 0, 4075, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 13, 0, 4070, 29, 4096, 0, 4074, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4087, 6, 0, 8, 4064, 1, 4, 4090, 4088, 4089, 0, 0, 0, 0, 25, 0, 1, 4087, 0, 3972, 8, 4094, 7, 5, 8, 0, 5, 112, 2, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 4080, 0, 0, 0, 0, 4045, 84, 4, 0, 0, 0, 0, 0, 4063, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4101, 4096, 4089, 4133, 4073, 4084, 4095] avg=4096.0000 max=4133 max_rank=4 min=4073 min_rank=5 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4088, 0, 0, 0, 0, 4096, 2, 4, 0, 0, 0, 0, 0, 4079, 0, 0, 10, 0, 0, 0, 0, 0, 0, 6, 4082, 2, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4097, 4098, 4102, 4089, 4090, 4095] avg=4096.0000 max=4102 max_rank=4 min=4089 min_rank=5 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4068, 3, 1, 0, 0, 0, 0, 0, 4088, 0, 0, 10, 0, 0, 0, 0, 0, 0, 42, 4076, 2, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4098, 4098, 4088, 4072, 4098, 4120, 4098] avg=4096.0000 max=4120 max_rank=6 min=4072 min_rank=4 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4049, 6, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 55, 4068, 2, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4098, 4088, 4059, 4106, 4125, 4095] avg=4096.0000 max=4125 max_rank=6 min=4059 min_rank=4 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4088, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 4087, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4098, 4097, 4100, 4097, 4084, 4095] avg=4096.0000 max=4101 max_rank=1 min=4084 min_rank=6 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4093, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 14, 6, 0, 0, 0, 0, 0, 1, 4082, 2, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4096, 4096, 4098, 4111, 4095, 4085, 4097] avg=4096.0000 max=4111 max_rank=4 min=4085 min_rank=6 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4087, 6, 0, 8, 0, 0, 0, 0, 4064, 1, 4, 5, 0, 0, 0, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4082, 0, 0, 0, 1, 4087, 0, 4, 0, 0, 0, 0, 8, 4094, 7, 5, 10, 0, 0, 0, 8, 0, 5, 112, 3958, 2, 0, 0, 2, 5, 0, 0, 4086, 1, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4074, 4091, 4107, 4092, 4124, 4085, 4094] avg=4096.0000 max=4124 max_rank=5 min=4074 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4090, 0, 0, 0, 0, 4070, 29, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 12, 0, 0, 0, 0, 0, 0, 8, 4084, 3, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4094, 4103, 4099, 4086, 4095, 4099] avg=4096.0000 max=4103 max_rank=3 min=4086 min_rank=5 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 20, 0, 0, 0, 1, 0, 4076, 0, 0, 0, 4096, 4093, 5, 0, 0, 0, 4096, 4096, 0, 0, 4095, 4095, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 4094, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 4096, 4096, 0, 0, 4087, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4090, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4082, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 8, 6, 4, 4089, 3, 4, 7, 0, 0, 9, 4087, 0, 0, 6, 4079, 4086, 0, 6, 0, 0, 4087, 4088, 8, 0, 4078, 4089, 10, 0, 7, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 4096, 0, 0, 0, 15, 4096, 4096, 0, 0, 4081, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 4, 1, 0, 4086, 20, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4076, 2, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4106, 4079, 4097, 4098, 4095, 4102, 4098] avg=4096.0000 max=4106 max_rank=1 min=4079 min_rank=2 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 4, 1, 0, 4090, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 4087, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4098, 4098, 4097, 4096, 4096, 4094, 4095] avg=4096.0000 max=4098 max_rank=1 min=4094 min_rank=0 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 4, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4090, 4098, 4097, 4096, 4095, 4103, 4095] avg=4096.0000 max=4103 max_rank=6 min=4090 min_rank=1 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 4, 1, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4092, 4097, 4097, 4097, 4095, 4101, 4095] avg=4096.0000 max=4101 max_rank=6 min=4092 min_rank=1 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 4, 1, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4090, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4098, 4097, 4096, 4095, 4097, 4095] avg=4096.0000 max=4098 max_rank=2 min=4094 min_rank=0 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 6, 4, 4082, 4, 1, 0, 4083, 3, 4, 7, 0, 0, 0, 0, 0, 0, 9, 4087, 2, 0, 0, 0, 0, 0, 6, 4079, 1, 0, 0, 0, 4086, 0, 6, 0, 0, 0, 0, 0, 0, 4086, 0, 8, 0, 0, 0, 0, 0, 4078, 7, 10, 0, 0, 0, 0, 0, 7, 0, 7, 4087, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4097, 4098, 4086, 4092, 4094, 4095, 4101] avg=4096.0000 max=4105 max_rank=0 min=4086 min_rank=3 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 8, 2, 0, 4079, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 3, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4082, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4093, 4108, 4100, 4097, 4088, 4088, 4094] avg=4096.0000 max=4108 max_rank=2 min=4088 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 4, 1, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 15, 4095, 0, 0, 0, 0, 0, 0, 0, 4081, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4092, 4097, 4096, 4110, 4088, 4095] avg=4096.0000 max=4110 max_rank=5 min=4088 min_rank=6 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 224, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 4096, 3882, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 213, 4092, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4003, 4091, 0, 0, 4096, 3985, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 770, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 3327, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1494, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 4096, 2620, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 523, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 4096, 3576, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 82, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 4096, 4022, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 390, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 4091, 0, 0, 4096, 3707, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4, 1006, 4088, 0, 4, 7, 8, 3, 19, 0, 4088, 8, 8, 0, 0, 0, 4089, 0, 16, 4, 0, 3, 6, 4079, 4090, 0, 6, 4088, 3064, 0, 4080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 213, 3507, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 585, 2, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 15, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4003, 17, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 575, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3720, 4093, 4097, 4090, 4093, 4095, 4020, 4560] avg=4096.0000 max=4560 max_rank=7 min=3720 min_rank=0 imbalance_ratio=1.113281 max_load_reduction=28208 max_load_reduction_pct=0.860840 moved_to_echo=16768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 82, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 585, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 15, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4087, 17, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 575, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3594, 4093, 4094, 4095, 4096, 4095, 4104, 4597] avg=4096.0000 max=4597 max_rank=7 min=3594 min_rank=0 imbalance_ratio=1.122314 max_load_reduction=28171 max_load_reduction_pct=0.859711 moved_to_echo=16892 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1494, 3499, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 583, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 15, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 17, 0, 0, 0, 0, 0, 0, 0, 2620, 0, 575, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4993, 4091, 4094, 4094, 4096, 4096, 4109, 3195] avg=4096.0000 max=4993 max_rank=0 min=3195 min_rank=7 imbalance_ratio=1.218994 max_load_reduction=27775 max_load_reduction_pct=0.847626 moved_to_echo=15496 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 770, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 585, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 15, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 17, 0, 0, 0, 0, 0, 0, 0, 3327, 0, 575, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4281, 4093, 4094, 4095, 4096, 4095, 4112, 3902] avg=4096.0000 max=4281 max_rank=0 min=3902 min_rank=7 imbalance_ratio=1.045166 max_load_reduction=28487 max_load_reduction_pct=0.869354 moved_to_echo=16205 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 523, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 585, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 15, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 17, 0, 0, 0, 0, 0, 0, 0, 3576, 0, 575, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4093, 4094, 4094, 4096, 4096, 4111, 4151] avg=4096.0000 max=4151 max_rank=7 min=4033 min_rank=0 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16454 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 224, 3507, 0, 0, 0, 0, 0, 0, 0, 0, 3512, 589, 7, 6, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 21, 1, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4086, 12, 0, 0, 0, 0, 0, 0, 0, 3882, 0, 563, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3731, 4114, 4089, 4106, 4090, 4095, 4098, 4445] avg=4096.0000 max=4445 max_rank=7 min=3731 min_rank=0 imbalance_ratio=1.085205 max_load_reduction=28323 max_load_reduction_pct=0.864349 moved_to_echo=16728 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 1006, 3504, 0, 0, 0, 0, 0, 4, 7, 8, 3493, 584, 2, 0, 3, 19, 0, 4086, 0, 0, 0, 0, 8, 8, 0, 0, 4073, 15, 0, 0, 0, 4089, 0, 16, 0, 0, 0, 0, 4, 0, 3, 6, 4088, 0, 0, 0, 4079, 17, 0, 6, 0, 0, 0, 0, 0, 3064, 0, 572, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4514, 4098, 4108, 4104, 4105, 4101, 4102, 3636] avg=4096.0000 max=4514 max_rank=0 min=3636 min_rank=7 imbalance_ratio=1.102051 max_load_reduction=28254 max_load_reduction_pct=0.862244 moved_to_echo=15944 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 390, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 585, 2, 0, 0, 0, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 15, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 17, 0, 0, 0, 0, 0, 0, 0, 3707, 0, 575, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3902, 4093, 4098, 4090, 4096, 4095, 4112, 4282] avg=4096.0000 max=4282 max_rank=7 min=3902 min_rank=0 imbalance_ratio=1.045410 max_load_reduction=28486 max_load_reduction_pct=0.869324 moved_to_echo=16585 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 28160, 4736, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 4736, 28160, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 28288, 0, 4608, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 256, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32640, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 8, 0, 4096, 0, 4096, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4069, 0, 21, 0, 0, 0, 0, 0, 0, 4096, 5, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 57, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 22, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4030, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 42, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3903, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 182, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 4096, 0, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4031, 0, 40, 0, 1, 0, 0, 0, 0, 4096, 25, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4085, 1, 11, 4084, 0, 4087, 4080, 0, 16, 0, 9, 4086, 5, 8, 0, 6, 3848, 7, 0, 8, 0, 0, 7, 2, 2, 4078, 168, 4087, 0, 83, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 21, 0, 4096, 0, 4096, 4075, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3711, 0, 104, 0, 0, 0, 0, 0, 0, 4096, 281, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4052, 0, 19, 0, 0, 0, 0, 0, 0, 4096, 25, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 8, 0, 0, 0, 0, 0, 7, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 21, 92, 9, 6, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4001, 5, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4093, 4096, 4197, 4095, 4006, 4087] avg=4096.0000 max=4197 max_rank=4 min=4006 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 92, 9, 6, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4001, 57, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4103, 4093, 4096, 4146, 4095, 4058, 4087] avg=4096.0000 max=4146 max_rank=4 min=4058 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 2, 0, 0, 0, 0, 0, 7, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 40, 92, 9, 6, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 4001, 25, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4100, 4093, 4096, 4178, 4096, 4026, 4087] avg=4096.0000 max=4178 max_rank=4 min=4026 min_rank=6 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4079, 1, 11, 0, 0, 0, 0, 6, 0, 0, 4080, 0, 0, 0, 0, 0, 16, 0, 9, 4078, 3, 0, 0, 4086, 5, 8, 0, 0, 0, 0, 0, 6, 3848, 7, 0, 92, 9, 6, 0, 8, 0, 0, 7, 4087, 0, 0, 0, 2, 2, 3983, 168, 0, 0, 0, 0, 4078, 0, 83, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4086, 4106, 4099, 3968, 4102, 4155, 4161] avg=4096.0000 max=4161 max_rank=7 min=3968 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 0, 12, 92, 9, 6, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4001, 182, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4102, 4093, 4096, 4022, 4095, 4183, 4087] avg=4096.0000 max=4183 max_rank=6 min=4022 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 22, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 3, 92, 9, 6, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4000, 42, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4103, 4093, 4096, 4140, 4095, 4042, 4087] avg=4096.0000 max=4140 max_rank=4 min=4042 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 19, 98, 11, 12, 3, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3991, 25, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4096, 4104, 4093, 4195, 4095, 4016, 4085] avg=4096.0000 max=4195 max_rank=4 min=4016 min_rank=6 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 21, 0, 0, 0, 0, 0, 7, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 0, 104, 92, 9, 6, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4001, 281, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4082, 4093, 4096, 3922, 4095, 4282, 4087] avg=4096.0000 max=4282 max_rank=6 min=3922 min_rank=4 imbalance_ratio=1.045410 max_load_reduction=28486 max_load_reduction_pct=0.869324 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 31744, 128, 256, 768, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32000, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 39, 0, 4096, 4096, 0, 0, 0, 1094, 0, 0, 4083, 4096, 0, 0, 4096, 3939, 0, 3123, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[11, 0, 0, 0, 8, 0, 2, 0, 4086, 0, 0, 10, 0, 334, 9, 4085, 3992, 12, 4, 10, 34, 0, 15, 4084, 4088, 9, 0, 4073, 3803, 0, 4084, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 1869, 0, 4096, 4096, 0, 0, 0, 40, 0, 0, 4088, 4096, 0, 0, 4096, 2516, 0, 3768, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 39, 0, 4096, 0, 0, 0, 1, 805, 0, 4096, 4096, 0, 0, 0, 788, 0, 0, 4087, 4096, 0, 0, 4096, 3622, 0, 2946, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 1026, 0, 4096, 4096, 0, 0, 0, 355, 0, 0, 4087, 4096, 0, 0, 4096, 3587, 0, 3224, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 80, 0, 4096, 4051, 0, 0, 0, 737, 0, 0, 4096, 4096, 0, 0, 4096, 3859, 0, 3561, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 28, 0, 4096, 0, 0, 0, 17, 609, 0, 4096, 4096, 0, 0, 0, 684, 0, 0, 4080, 4096, 0, 0, 4096, 3324, 0, 3546, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 21, 86, 0, 4094, 4096, 0, 0, 0, 344, 0, 0, 4096, 4096, 0, 1, 4096, 3750, 0, 3991, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 7, 0, 2115, 611, 487, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1869, 0, 3485, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 40, 0, 0, 3586, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 401, 0, 3768, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 3220, 4096, 5354, 4111, 3626, 4098, 4169] avg=4096.0000 max=5354 max_rank=3 min=3220 min_rank=1 imbalance_ratio=1.307129 max_load_reduction=27414 max_load_reduction_pct=0.836609 moved_to_echo=16004 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 0, 0, 4086, 1, 0, 0, 8, 0, 2, 0, 3197, 610, 487, 0, 4086, 0, 0, 10, 0, 0, 0, 0, 0, 334, 9, 3475, 0, 0, 0, 0, 3992, 12, 4, 10, 15, 0, 0, 0, 34, 0, 15, 3582, 0, 0, 0, 0, 2, 9, 0, 4073, 0, 0, 0, 0, 605, 0, 4084, 15, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4304, 4096, 3818, 4033, 3631, 4084, 4704] avg=4096.0000 max=4704 max_rank=7 min=3631 min_rank=5 imbalance_ratio=1.148438 max_load_reduction=28064 max_load_reduction_pct=0.856445 moved_to_echo=16452 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 3312, 611, 487, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 39, 0, 3485, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 1094, 0, 0, 3581, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 626, 0, 3123, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4410, 4096, 3534, 4111, 4675, 4098, 3749] avg=4096.0000 max=4675 max_rank=5 min=3534 min_rank=3 imbalance_ratio=1.141357 max_load_reduction=28093 max_load_reduction_pct=0.857330 moved_to_echo=16633 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 39, 0, 3045, 611, 487, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 805, 0, 3485, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 788, 0, 0, 3585, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 576, 0, 2946, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4182, 4096, 4291, 4111, 4373, 4098, 3522] avg=4096.0000 max=4373 max_rank=5 min=3522 min_rank=7 imbalance_ratio=1.067627 max_load_reduction=28395 max_load_reduction_pct=0.866547 moved_to_echo=16104 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 3244, 611, 488, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 3485, 0, 0, 0, 0, 4051, 0, 0, 0, 15, 0, 0, 0, 737, 0, 0, 3593, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 614, 0, 3561, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4343, 4096, 3565, 4066, 4330, 4098, 4175] avg=4096.0000 max=4343 max_rank=1 min=3565 min_rank=3 imbalance_ratio=1.060303 max_load_reduction=28425 max_load_reduction_pct=0.867462 moved_to_echo=16669 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 9, 0, 3020, 617, 491, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 0, 3479, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 355, 0, 0, 3578, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 563, 0, 3224, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4137, 4096, 4505, 4114, 3933, 4096, 3787] avg=4096.0000 max=4505 max_rank=3 min=3787 min_rank=7 imbalance_ratio=1.099854 max_load_reduction=28263 max_load_reduction_pct=0.862518 moved_to_echo=15930 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 1, 0, 3153, 611, 488, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 21, 86, 0, 3483, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 344, 0, 0, 3593, 0, 0, 0, 0, 2, 0, 1, 4096, 0, 0, 0, 0, 596, 0, 3991, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4253, 4096, 3590, 4111, 3937, 4099, 4587] avg=4096.0000 max=4587 max_rank=7 min=3590 min_rank=3 imbalance_ratio=1.119873 max_load_reduction=28181 max_load_reduction_pct=0.860016 moved_to_echo=16734 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 28, 0, 2794, 611, 486, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 609, 0, 3485, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 684, 0, 0, 3579, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 529, 0, 3546, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3919, 4096, 4111, 4111, 4263, 4097, 4075] avg=4096.0000 max=4263 max_rank=5 min=3919 min_rank=1 imbalance_ratio=1.040771 max_load_reduction=28505 max_load_reduction_pct=0.869904 moved_to_echo=16546 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 23936, 4992, 3968, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4608, 0, 28288, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 0, 128, 28800, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 4864, 128, 27904, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[2, 4083, 4063, 5, 0, 0, 16, 3986, 10, 1, 0, 4085, 0, 11, 0, 100, 5, 4, 0, 42, 0, 10, 4084, 9, 4079, 7, 4, 11, 4076, 0, 4065, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4079, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 8, 4063, 5, 0, 0, 0, 0, 0, 0, 16, 3986, 11, 0, 0, 0, 10, 1, 0, 4085, 0, 0, 0, 0, 0, 11, 0, 100, 4064, 1, 0, 0, 5, 4, 0, 42, 4072, 0, 0, 0, 0, 10, 4084, 9, 0, 0, 0, 0, 4079, 7, 4, 11, 0, 0, 0, 0, 3, 0, 4065, 10, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4013, 4096, 4176, 4123, 4103, 4101, 4078] avg=4096.0000 max=4176 max_rank=3 min=4013 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16405 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4077, 2, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4101, 4096, 4085, 4092, 4096, 4096, 4098] avg=4096.0000 max=4104 max_rank=0 min=4085 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 4077, 2, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4107, 4096, 4082, 4092, 4096, 4096, 4098] avg=4096.0000 max=4107 max_rank=1 min=4082 min_rank=3 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 4077, 2, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4106, 4096, 4083, 4092, 4096, 4096, 4098] avg=4096.0000 max=4106 max_rank=1 min=4083 min_rank=3 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 4077, 2, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4107, 4096, 4083, 4092, 4096, 4096, 4098] avg=4096.0000 max=4107 max_rank=1 min=4083 min_rank=3 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 2, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4107, 4096, 4079, 4092, 4096, 4096, 4104] avg=4096.0000 max=4107 max_rank=1 min=4079 min_rank=3 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 7, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4120, 4096, 4084, 4093, 4089, 4091, 4096] avg=4096.0000 max=4120 max_rank=1 min=4084 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16369 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 4077, 2, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4107, 4096, 4096, 4092, 4096, 4096, 4098] avg=4096.0000 max=4107 max_rank=1 min=4087 min_rank=0 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3987, 4096, 4093, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 109, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4016, 4096, 4092, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 2, 80, 0, 0, 0, 4094, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4082, 4096, 4096, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[219, 0, 10, 9, 21, 3818, 4074, 4087, 12, 9, 0, 15, 7, 9, 4068, 0, 0, 4082, 0, 4078, 0, 2, 0, 52, 0, 11, 3, 4087, 14, 0, 1, 4080] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[13, 0, 0, 0, 0, 4071, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4000, 4096, 4089, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 87, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 4077, 4096, 4096, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4084, 4096, 4096, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4058, 7, 0, 0, 0, 3987, 0, 77, 0, 0, 0, 0, 0, 0, 3, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 109, 4015, 36, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4064, 4092, 4097, 4098, 4160, 4096, 4096] avg=4096.0000 max=4160 max_rank=5 min=4064 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16450 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4058, 7, 0, 0, 0, 4016, 0, 77, 0, 0, 0, 0, 0, 0, 4, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 80, 4014, 34, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4093, 4093, 4097, 4100, 4130, 4094, 4096] avg=4096.0000 max=4130 max_rank=5 min=4065 min_rank=0 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16420 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[219, 0, 10, 9, 4044, 7, 0, 0, 21, 3818, 0, 77, 0, 0, 0, 0, 12, 9, 0, 15, 4067, 0, 0, 0, 7, 9, 4068, 0, 1, 0, 0, 0, 0, 4, 0, 4078, 0, 0, 0, 0, 0, 2, 0, 52, 4009, 34, 0, 0, 0, 11, 3, 4087, 0, 0, 0, 0, 14, 0, 1, 4080, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4289, 3916, 4103, 4085, 4082, 4097, 4101, 4095] avg=4096.0000 max=4289 max_rank=0 min=3916 min_rank=1 imbalance_ratio=1.047119 max_load_reduction=28479 max_load_reduction_pct=0.869110 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4058, 8, 0, 0, 0, 4082, 0, 77, 0, 0, 0, 0, 0, 0, 6, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 4018, 38, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4159, 4094, 4097, 4096, 4064, 4096, 4096] avg=4096.0000 max=4159 max_rank=1 min=4064 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16352 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 0, 4058, 7, 0, 0, 0, 4071, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4018, 34, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4148, 4089, 4097, 4100, 4064, 4096, 4096] avg=4096.0000 max=4148 max_rank=1 min=4064 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16356 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4058, 7, 0, 0, 0, 4077, 0, 77, 0, 0, 0, 0, 0, 0, 7, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 4018, 34, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4154, 4096, 4097, 4100, 4063, 4096, 4096] avg=4096.0000 max=4154 max_rank=1 min=4063 min_rank=5 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16355 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4062, 8, 0, 0, 0, 4084, 0, 73, 0, 0, 0, 0, 0, 0, 9, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 3, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 4020, 34, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4070, 4157, 4097, 4101, 4096, 4057, 4093, 4097] avg=4096.0000 max=4157 max_rank=1 min=4057 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16343 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4061, 8, 0, 0, 0, 4000, 0, 77, 0, 0, 0, 0, 0, 0, 16, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 87, 4011, 35, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4077, 4104, 4097, 4096, 4133, 4096, 4096] avg=4096.0000 max=4133 max_rank=5 min=4069 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16421 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 0, 128, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 384, 32128, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32256, 0, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 3493, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4095, 0, 0, 0, 599, 4095, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3873, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 223, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3097, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 4094, 0, 0, 0, 995, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4044, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 4096, 0, 0, 0, 49, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3175, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 921, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4019, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 77, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[3, 0, 0, 11, 11, 0, 4087, 10, 10, 13, 3182, 4085, 2, 6, 4080, 5, 0, 0, 0, 13, 0, 9, 0, 4086, 4074, 4, 8, 11, 891, 4086, 15, 4066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3343, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 4096, 0, 0, 0, 753, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 3530, 562, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3493, 566, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 3534, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4059, 4096, 4095, 4096, 4095, 4133] avg=4096.0000 max=4133 max_rank=7 min=4059 min_rank=2 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3528, 562, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4044, 565, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 3534, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4096, 4609, 4096, 4096, 4100, 4096, 3583] avg=4096.0000 max=4609 max_rank=2 min=3583 min_rank=7 imbalance_ratio=1.125244 max_load_reduction=28159 max_load_reduction_pct=0.859344 moved_to_echo=15875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3530, 562, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3097, 566, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 995, 0, 0, 3534, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 3663, 4096, 4096, 4100, 4094, 4529] avg=4096.0000 max=4529 max_rank=7 min=3663 min_rank=2 imbalance_ratio=1.105713 max_load_reduction=28239 max_load_reduction_pct=0.861786 moved_to_echo=16819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3530, 562, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3175, 566, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 921, 0, 0, 3534, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 3741, 4096, 4096, 4094, 4096, 4455] avg=4096.0000 max=4455 max_rank=7 min=3741 min_rank=2 imbalance_ratio=1.087646 max_load_reduction=28313 max_load_reduction_pct=0.864044 moved_to_echo=16741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 11, 3521, 558, 2, 0, 11, 0, 4087, 10, 0, 0, 0, 0, 10, 13, 3182, 564, 0, 0, 0, 0, 2, 6, 4080, 5, 0, 0, 0, 0, 0, 0, 0, 13, 4085, 0, 0, 0, 0, 9, 0, 4084, 0, 0, 0, 0, 4074, 4, 8, 11, 0, 0, 0, 0, 891, 1, 15, 3508, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4108, 3769, 4093, 4098, 4093, 4097, 4415] avg=4096.0000 max=4415 max_rank=7 min=3769 min_rank=2 imbalance_ratio=1.077881 max_load_reduction=28353 max_load_reduction_pct=0.865265 moved_to_echo=16703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3530, 562, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4019, 566, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 77, 1, 0, 3534, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4585, 4096, 4095, 4094, 4096, 3612] avg=4096.0000 max=4585 max_rank=2 min=3612 min_rank=7 imbalance_ratio=1.119385 max_load_reduction=28183 max_load_reduction_pct=0.860077 moved_to_echo=15897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3535, 563, 5, 7, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 3873, 561, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 223, 0, 0, 3533, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4084, 4434, 4099, 4096, 4091, 4098, 3756] avg=4096.0000 max=4434 max_rank=2 min=3756 min_rank=7 imbalance_ratio=1.082520 max_load_reduction=28334 max_load_reduction_pct=0.864685 moved_to_echo=16041 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3525, 562, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3343, 565, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 753, 0, 0, 3534, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4096, 3908, 4096, 4096, 4100, 4096, 4287] avg=4096.0000 max=4287 max_rank=7 min=3908 min_rank=2 imbalance_ratio=1.046631 max_load_reduction=28481 max_load_reduction_pct=0.869171 moved_to_echo=16579 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 32768, 32768, 32768, 32768, 32768, 32770] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32766 actual_valid_recv=33536 observed_loss=-770 tokens_per_expert=[128, 0, 128, 128, 28288, 4608, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 28288, 4608, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32770 actual_valid_recv=33152 observed_loss=-382 tokens_per_expert=[4608, 128, 128, 28288, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4060, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 0, 0, 4096, 4093, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 47, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4049, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 47, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3850, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 246, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 8, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3632, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4091, 0, 4096, 0, 483, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[10, 0, 0, 4084, 0, 9, 0, 0, 9, 4086, 13, 12, 4084, 14, 8, 0, 7, 4077, 4088, 12, 0, 4080, 0, 6, 0, 4067, 10, 4086, 0, 0, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3809, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 297, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4085, 0, 0, 0, 0, 0, 4096, 0, 0, 4073, 0, 0, 0, 1, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 34, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4060, 136, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 47, 0, 0, 3951, 0, 0, 0] rerouted_tokens_per_ep_rank=[4196, 4093, 4094, 4093, 4096, 4097, 4101, 3998] avg=4096.0000 max=4196 max_rank=0 min=3998 min_rank=7 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4049, 136, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 47, 0, 0, 3951, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 4096, 4094, 4101, 4096, 4097, 4101, 3998] avg=4096.0000 max=4185 max_rank=0 min=3998 min_rank=7 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3632, 136, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4091, 0, 4, 0, 0, 0, 0, 0, 483, 0, 0, 3952, 0, 0, 0] rerouted_tokens_per_ep_rank=[3768, 4096, 4094, 4087, 4096, 4097, 4095, 4435] avg=4096.0000 max=4435 max_rank=7 min=3768 min_rank=0 imbalance_ratio=1.082764 max_load_reduction=28333 max_load_reduction_pct=0.864655 moved_to_echo=16723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3850, 136, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 246, 0, 0, 3951, 0, 0, 0] rerouted_tokens_per_ep_rank=[3986, 4096, 4094, 4101, 4096, 4097, 4101, 4197] avg=4096.0000 max=4197 max_rank=7 min=3986 min_rank=0 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16491 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 4084, 136, 0, 0, 0, 0, 9, 0, 0, 4088, 0, 0, 0, 9, 4084, 13, 12, 0, 0, 0, 0, 4084, 14, 8, 0, 4, 1, 0, 0, 7, 4077, 0, 12, 0, 0, 0, 0, 0, 4080, 0, 6, 1, 0, 0, 0, 0, 4067, 10, 5, 0, 0, 0, 0, 0, 0, 0, 6, 3941, 0, 0, 0] rerouted_tokens_per_ep_rank=[4230, 4097, 4118, 4111, 4096, 4087, 4082, 3947] avg=4096.0000 max=4230 max_rank=0 min=3947 min_rank=7 imbalance_ratio=1.032715 max_load_reduction=28538 max_load_reduction_pct=0.870911 moved_to_echo=16212 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 136, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 8, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3952, 0, 0, 0] rerouted_tokens_per_ep_rank=[4230, 4096, 4094, 4099, 4096, 4097, 4096, 3960] avg=4096.0000 max=4230 max_rank=0 min=3960 min_rank=7 imbalance_ratio=1.032715 max_load_reduction=28538 max_load_reduction_pct=0.870911 moved_to_echo=16249 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3809, 141, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4, 8, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 3951, 0, 0, 0] rerouted_tokens_per_ep_rank=[3950, 4096, 4086, 4098, 4095, 4099, 4096, 4248] avg=4096.0000 max=4248 max_rank=7 min=3950 min_rank=0 imbalance_ratio=1.037109 max_load_reduction=28520 max_load_reduction_pct=0.870361 moved_to_echo=16538 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4085, 138, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4, 1, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 3951, 0, 0, 0] rerouted_tokens_per_ep_rank=[4223, 4098, 4094, 4078, 4097, 4097, 4096, 3985] avg=4096.0000 max=4223 max_rank=0 min=3985 min_rank=7 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16275 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 31744, 1152, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 1280, 0, 128, 31616, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4085, 4085, 0, 45, 17, 1, 4082, 0, 0, 0, 0, 11, 0, 9, 13, 4075, 9, 12, 0, 0, 16, 0, 0, 0, 4044, 4086, 10, 0, 5, 4084, 4079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4091, 3, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4091, 4097, 4092, 4096, 4097, 4102, 4098] avg=4096.0000 max=4102 max_rank=6 min=4091 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4091, 4097, 4092, 4096, 4094, 4102, 4098] avg=4096.0000 max=4102 max_rank=6 min=4091 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4091, 4097, 4092, 4096, 4094, 4102, 4098] avg=4096.0000 max=4102 max_rank=6 min=4091 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4091, 4097, 4092, 4096, 4094, 4102, 4098] avg=4096.0000 max=4102 max_rank=6 min=4091 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4091, 4097, 4092, 4096, 4094, 4102, 4098] avg=4096.0000 max=4102 max_rank=6 min=4091 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4085, 0, 0, 0, 0, 0, 45, 17, 1, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 11, 0, 9, 13, 4077, 5, 0, 0, 4075, 9, 12, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4080, 3, 0, 0, 0, 4044, 6, 10, 0, 0, 0, 0, 0, 5, 2, 4079, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4140, 4084, 4115, 4096, 4099, 4060, 4086] avg=4096.0000 max=4140 max_rank=1 min=4060 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4092, 9, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4082, 4100, 4101, 4101, 4096, 4096, 4096] avg=4096.0000 max=4101 max_rank=3 min=4082 min_rank=1 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4091, 3, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4091, 4099, 4092, 4091, 4100, 4102, 4096] avg=4096.0000 max=4102 max_rank=6 min=4091 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-08-30 04:55:24] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 672.0 | throughput per GPU (TFLOP/s/GPU): 496.1 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.039238E+01 | loss scale: 1.0 | grad norm: 31.942 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 1651, 10, 707, 6, 599, 0, 859, 3280, 2, 3087, 1081, 1768, 1, 3628, 2417, 3133, 147, 0, 7, 1848, 2253, 50, 17, 25, 423, 1890, 1271, 19, 2553, 36] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 1903, 2, 803, 0, 588, 2, 864, 3534, 0, 3271, 815, 1775, 1, 3611, 2764, 2993, 106, 0, 2, 2265, 1744, 0, 2, 17, 638, 1635, 735, 14, 2681, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[0, 2, 1826, 0, 704, 1, 540, 2, 853, 3619, 2, 3267, 778, 1730, 4, 3697, 3156, 2661, 99, 0, 0, 1721, 2288, 4, 3, 64, 485, 1946, 655, 3, 2656, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[5, 0, 2113, 8, 647, 4, 560, 1, 608, 3255, 7, 3058, 1288, 1960, 3, 3557, 2771, 2488, 160, 1, 1, 1489, 2323, 0, 0, 49, 619, 2161, 943, 12, 2667, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[12, 1, 2249, 0, 1295, 0, 558, 5, 867, 3040, 0, 2330, 1035, 1604, 0, 3528, 2980, 3009, 467, 0, 1, 1436, 2256, 0, 0, 124, 563, 1495, 949, 16, 2943, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2005, 0, 647, 0, 558, 0, 662, 3681, 0, 2998, 609, 1374, 1, 3618, 3305, 3100, 118, 0, 4, 1878, 2433, 1, 0, 40, 524, 1545, 752, 3, 2911, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2069, 0, 769, 0, 386, 1, 394, 3496, 0, 3017, 1006, 1975, 0, 3615, 3329, 2600, 109, 0, 0, 1732, 2202, 1, 1, 25, 552, 1707, 1103, 15, 2664, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2, 1, 2539, 0, 420, 0, 606, 3, 813, 3708, 0, 3433, 889, 2186, 0, 3283, 1956, 2652, 116, 3, 3, 1856, 1953, 4, 1, 28, 540, 1555, 1185, 6, 3027, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1651, 10, 1714, 342, 0, 0, 707, 6, 599, 0, 2655, 0, 0, 0, 859, 283, 2, 3087, 0, 0, 0, 0, 1081, 1768, 1, 1385, 0, 0, 0, 0, 940, 3133, 147, 0, 0, 0, 0, 0, 7, 1848, 2253, 50, 107, 0, 0, 0, 17, 25, 423, 1890, 1058, 529, 0, 0, 1271, 19, 2553, 36, 312, 0, 0, 0] rerouted_tokens_per_ep_rank=[3717, 3967, 4231, 4235, 4220, 4265, 3942, 4191] avg=4096.0000 max=4265 max_rank=5 min=3717 min_rank=0 imbalance_ratio=1.041260 max_load_reduction=28503 max_load_reduction_pct=0.869843 moved_to_echo=16618 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1826, 0, 1747, 377, 0, 0, 704, 1, 540, 2, 2930, 0, 0, 0, 853, 312, 2, 3267, 0, 0, 0, 0, 778, 1730, 4, 1411, 0, 0, 0, 0, 1228, 2661, 99, 0, 0, 0, 0, 0, 0, 1721, 2288, 4, 140, 0, 0, 0, 3, 64, 485, 1946, 1381, 539, 0, 0, 655, 3, 2656, 2, 407, 0, 0, 0] rerouted_tokens_per_ep_rank=[3952, 4177, 4434, 3923, 3988, 4153, 4418, 3723] avg=4096.0000 max=4434 max_rank=2 min=3723 min_rank=7 imbalance_ratio=1.082520 max_load_reduction=28334 max_load_reduction_pct=0.864685 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 1903, 2, 1706, 368, 0, 0, 803, 0, 588, 2, 2861, 0, 0, 0, 864, 305, 0, 3271, 0, 0, 0, 0, 815, 1775, 1, 1378, 0, 0, 0, 0, 1074, 2993, 106, 0, 0, 0, 0, 0, 2, 2265, 1744, 0, 123, 0, 0, 0, 2, 17, 638, 1635, 1210, 527, 0, 0, 735, 14, 2681, 0, 357, 0, 0, 0] rerouted_tokens_per_ep_rank=[3982, 4254, 4440, 3969, 4173, 4134, 4029, 3787] avg=4096.0000 max=4440 max_rank=2 min=3787 min_rank=7 imbalance_ratio=1.083984 max_load_reduction=28328 max_load_reduction_pct=0.864502 moved_to_echo=16123 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 2113, 8, 1680, 339, 0, 0, 647, 4, 560, 1, 2635, 0, 0, 0, 608, 281, 7, 3058, 0, 0, 0, 0, 1288, 1960, 3, 1358, 0, 0, 0, 0, 1077, 2488, 160, 1, 0, 0, 0, 0, 1, 1489, 2323, 0, 123, 0, 0, 0, 0, 49, 619, 2161, 1213, 519, 0, 0, 943, 12, 2667, 10, 358, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 3847, 3954, 4609, 3726, 3936, 4561, 3990] avg=4096.0000 max=4609 max_rank=3 min=3726 min_rank=4 imbalance_ratio=1.125244 max_load_reduction=28159 max_load_reduction_pct=0.859344 moved_to_echo=16213 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2005, 0, 1709, 384, 0, 0, 647, 0, 558, 0, 2980, 0, 0, 0, 662, 317, 0, 2998, 0, 0, 0, 0, 609, 1374, 1, 1381, 0, 0, 0, 0, 1285, 3100, 118, 0, 0, 0, 0, 0, 4, 1878, 2433, 1, 147, 0, 0, 0, 0, 40, 524, 1545, 1446, 528, 0, 0, 752, 3, 2911, 1, 427, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4185, 3977, 3365, 4503, 4463, 4083, 4094] avg=4096.0000 max=4503 max_rank=4 min=3365 min_rank=3 imbalance_ratio=1.099365 max_load_reduction=28265 max_load_reduction_pct=0.862579 moved_to_echo=17143 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[12, 1, 2249, 0, 1671, 321, 0, 0, 1295, 0, 558, 5, 2463, 0, 0, 0, 867, 256, 0, 2330, 0, 0, 0, 0, 1035, 1604, 0, 1339, 0, 0, 0, 0, 1147, 3009, 467, 0, 0, 0, 0, 0, 1, 1436, 2256, 0, 138, 0, 0, 0, 0, 124, 563, 1495, 1307, 518, 0, 0, 949, 16, 2943, 5, 388, 0, 0, 0] rerouted_tokens_per_ep_rank=[4254, 4321, 3453, 3978, 4623, 3831, 4007, 4301] avg=4096.0000 max=4623 max_rank=4 min=3453 min_rank=2 imbalance_ratio=1.128662 max_load_reduction=28145 max_load_reduction_pct=0.858917 moved_to_echo=16762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 2539, 0, 1551, 386, 0, 0, 420, 0, 606, 3, 3002, 0, 0, 0, 813, 320, 0, 3433, 0, 0, 0, 0, 889, 2186, 0, 1253, 0, 0, 0, 0, 761, 2652, 116, 3, 0, 0, 0, 0, 3, 1856, 1953, 4, 87, 0, 0, 0, 1, 28, 540, 1555, 856, 479, 0, 0, 1185, 6, 3027, 0, 252, 0, 0, 0] rerouted_tokens_per_ep_rank=[4479, 4031, 4566, 4328, 3532, 3903, 3459, 4470] avg=4096.0000 max=4566 max_rank=2 min=3459 min_rank=6 imbalance_ratio=1.114746 max_load_reduction=28202 max_load_reduction_pct=0.860657 moved_to_echo=15364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2069, 0, 1708, 364, 0, 0, 769, 0, 386, 1, 2830, 0, 0, 0, 394, 302, 0, 3017, 0, 0, 0, 0, 1006, 1975, 0, 1380, 0, 0, 0, 0, 1294, 2600, 109, 0, 0, 0, 0, 0, 0, 1732, 2202, 1, 148, 0, 0, 0, 1, 25, 552, 1707, 1457, 527, 0, 0, 1103, 15, 2664, 0, 430, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 3986, 3713, 4361, 4003, 4083, 4269, 4212] avg=4096.0000 max=4361 max_rank=3 min=3713 min_rank=2 imbalance_ratio=1.064697 max_load_reduction=28407 max_load_reduction_pct=0.866913 moved_to_echo=16567 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 384, 4352, 13952, 9984, 4224, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7552, 14464, 128, 11008, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8832, 22656, 1408, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 14336, 17536, 128, 1024, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 16384, 128, 13568, 2944, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6016, 128, 4480, 128, 22400, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[6016, 2432, 128, 24576, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7680, 128, 22144, 128, 2944, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4094, 0, 2430, 2, 9, 0, 0, 0, 4094, 1, 0, 1, 0, 4094, 0, 0, 1682, 1, 1, 1, 11, 1, 0, 4095, 0, 0, 1, 4095, 0, 0, 4060, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 3069, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4094, 4, 0, 1164, 0, 0, 0, 58, 0, 0, 4094, 0, 0, 0, 4085, 3, 6, 3906, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4091, 0, 2882, 9, 1, 0, 0, 0, 4095, 15, 1, 0, 2, 4096, 1, 0, 1403, 6, 0, 3, 125, 0, 0, 4091, 0, 0, 0, 4007, 0, 4, 3846, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4094, 1, 3322, 0, 10, 0, 0, 0, 4093, 3, 0, 0, 3, 4096, 4, 0, 851, 1, 1, 2, 2, 0, 0, 4092, 0, 0, 0, 4087, 1, 2, 4010, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4094, 1, 1662, 1, 9, 0, 0, 0, 4092, 0, 0, 4, 1, 4090, 1, 0, 2508, 0, 3, 0, 12, 1, 0, 4095, 0, 3, 3, 4028, 1, 1, 4065, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4094, 0, 1965, 2, 18, 0, 0, 0, 4092, 0, 6, 3, 4, 4095, 5, 0, 2209, 3, 0, 0, 1, 1, 0, 4087, 0, 3, 0, 4081, 0, 0, 4007, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 2602, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 4096, 0, 2, 1589, 0, 0, 1, 24, 0, 0, 4096, 0, 7, 0, 4093, 0, 1, 3969, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4094, 0, 2926, 0, 0, 0, 0, 0, 4096, 29, 4, 0, 0, 4087, 0, 0, 1143, 0, 0, 0, 142, 0, 0, 4096, 0, 5, 22, 3994, 0, 0, 4034, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1488, 0, 2430, 2, 0, 0, 0, 0, 9, 0, 0, 0, 3988, 102, 0, 0, 4088, 1, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1682, 1, 1, 1, 2504, 19, 0, 0, 11, 1, 0, 4052, 0, 0, 0, 0, 0, 0, 1, 4095, 24, 6, 0, 0, 0, 0, 4060, 107, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3920, 4099, 4090, 4094, 4208, 4064, 4126, 4167] avg=4096.0000 max=4208 max_rank=4 min=3920 min_rank=0 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1487, 0, 2882, 9, 0, 0, 0, 0, 1, 0, 0, 0, 3983, 102, 0, 0, 4089, 15, 1, 0, 0, 0, 0, 0, 2, 4096, 1, 0, 0, 0, 0, 0, 1403, 6, 0, 3, 2502, 19, 0, 0, 125, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 4007, 24, 6, 0, 0, 0, 4, 3846, 107, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4378, 4086, 4105, 4099, 3933, 4173, 4037, 3957] avg=4096.0000 max=4378 max_rank=0 min=3933 min_rank=4 imbalance_ratio=1.068848 max_load_reduction=28390 max_load_reduction_pct=0.866394 moved_to_echo=16100 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1488, 1, 1662, 1, 0, 0, 0, 0, 9, 0, 0, 0, 3986, 102, 0, 0, 4086, 0, 0, 4, 0, 0, 0, 0, 1, 4090, 1, 0, 0, 0, 0, 0, 2508, 0, 3, 0, 2504, 19, 0, 0, 12, 1, 0, 4052, 0, 0, 0, 0, 0, 3, 3, 4028, 24, 6, 0, 0, 1, 1, 4065, 107, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3152, 4097, 4090, 4092, 5034, 4065, 4064, 4174] avg=4096.0000 max=5034 max_rank=4 min=3152 min_rank=0 imbalance_ratio=1.229004 max_load_reduction=27734 max_load_reduction_pct=0.846375 moved_to_echo=17337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1488, 1, 3322, 0, 0, 0, 0, 0, 10, 0, 0, 0, 3986, 102, 0, 0, 4087, 3, 0, 0, 0, 0, 0, 0, 3, 4096, 4, 0, 0, 0, 0, 0, 851, 1, 1, 2, 2504, 19, 0, 0, 2, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 4087, 24, 6, 0, 0, 1, 2, 4010, 107, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4811, 4098, 4090, 4103, 3378, 4051, 4117, 4120] avg=4096.0000 max=4811 max_rank=0 min=3378 min_rank=4 imbalance_ratio=1.174561 max_load_reduction=27957 max_load_reduction_pct=0.853180 moved_to_echo=15666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1488, 0, 1965, 2, 0, 0, 0, 0, 18, 0, 0, 0, 3985, 102, 0, 0, 4086, 0, 6, 3, 0, 0, 0, 0, 4, 4095, 5, 0, 0, 0, 0, 0, 2209, 3, 0, 0, 2504, 19, 0, 0, 1, 1, 0, 4044, 0, 0, 0, 0, 0, 3, 0, 4081, 24, 6, 0, 0, 0, 0, 4007, 107, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3455, 4105, 4095, 4104, 4735, 4046, 4114, 4114] avg=4096.0000 max=4735 max_rank=4 min=3455 min_rank=0 imbalance_ratio=1.156006 max_load_reduction=28033 max_load_reduction_pct=0.855499 moved_to_echo=17009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1488, 0, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 102, 0, 0, 4090, 29, 4, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 1143, 0, 0, 0, 2504, 19, 0, 0, 142, 0, 0, 4053, 0, 0, 0, 0, 0, 5, 22, 3994, 24, 6, 0, 0, 0, 0, 4034, 107, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4414, 4091, 4123, 4087, 3666, 4195, 4051, 4141] avg=4096.0000 max=4414 max_rank=0 min=3666 min_rank=4 imbalance_ratio=1.077637 max_load_reduction=28354 max_load_reduction_pct=0.865295 moved_to_echo=16053 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1489, 0, 2602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 102, 0, 0, 4089, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 1589, 0, 0, 1, 2505, 19, 0, 0, 24, 0, 0, 4053, 0, 0, 0, 0, 0, 7, 0, 4093, 24, 6, 0, 0, 0, 1, 3969, 107, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4091, 4090, 4098, 4114, 4077, 4130, 4077] avg=4096.0000 max=4130 max_rank=6 min=4077 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1477, 0, 3069, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 108, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4, 0, 0, 0, 0, 0, 1164, 0, 0, 0, 2511, 25, 0, 0, 58, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 4085, 30, 7, 7, 0, 3, 6, 3906, 103, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4547, 4101, 4085, 4091, 3700, 4097, 4129, 4018] avg=4096.0000 max=4547 max_rank=0 min=3700 min_rank=4 imbalance_ratio=1.110107 max_load_reduction=28221 max_load_reduction_pct=0.861237 moved_to_echo=15944 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[12672, 128, 128, 128, 20096, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11904, 128, 20864, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32000, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 128, 0, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32512, 256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32000, 896, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4080, 0, 801, 227, 0, 3112, 4067, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 6, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4095, 4094, 0, 1, 4095, 0, 282, 307, 2, 3555, 4094, 0, 0, 0, 0, 4094, 1, 1, 2, 0, 4095, 0, 0, 0, 3, 0, 0, 4039, 7, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4093, 4095, 1, 0, 4096, 0, 981, 257, 0, 2857, 4092, 10, 0, 0, 0, 4094, 0, 0, 3, 0, 4091, 0, 8, 5, 0, 0, 0, 4084, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4094, 4093, 0, 2, 4094, 0, 1334, 385, 0, 2394, 4096, 2, 4, 1, 0, 4095, 1, 1, 0, 1, 4094, 0, 2, 0, 0, 0, 1, 4074, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4092, 4093, 3, 2, 4090, 0, 691, 555, 0, 2846, 4095, 4, 0, 4, 0, 4092, 2, 4, 2, 0, 4087, 0, 1, 9, 0, 5, 0, 4091, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4095, 0, 780, 823, 0, 2518, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3, 4071, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4093, 4092, 3, 1, 4096, 0, 1420, 335, 0, 2344, 4093, 0, 0, 0, 0, 4095, 1, 0, 2, 0, 4091, 0, 0, 3, 0, 0, 0, 4092, 0, 6, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4096, 4095, 1, 0, 4096, 0, 1215, 310, 0, 2584, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4082, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2751, 0, 801, 227, 0, 0, 0, 0, 0, 3112, 1310, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 15, 0, 0, 0, 6, 0, 0, 0, 2757, 1314, 2, 0] rerouted_tokens_per_ep_rank=[4096, 3779, 4422, 4096, 4094, 4096, 4106, 4079] avg=4096.0000 max=4422 max_rank=2 min=3779 min_rank=1 imbalance_ratio=1.079590 max_load_reduction=28346 max_load_reduction_pct=0.865051 moved_to_echo=16375 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 3, 2, 0, 0, 0, 0, 2758, 0, 691, 555, 0, 0, 0, 0, 0, 2846, 1319, 4, 0, 0, 0, 0, 0, 4, 0, 4092, 0, 0, 0, 0, 2, 4, 2, 0, 4092, 0, 0, 0, 4087, 0, 1, 9, 0, 0, 0, 0, 0, 5, 0, 4091, 15, 0, 0, 0, 0, 0, 0, 0, 2776, 1317, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4004, 4169, 4096, 4100, 4097, 4111, 4093] avg=4096.0000 max=4169 max_rank=2 min=4004 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16401 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 1, 0, 0, 0, 0, 2760, 0, 282, 307, 0, 0, 0, 0, 2, 3555, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 1, 2, 0, 4094, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4039, 16, 0, 0, 0, 7, 1, 0, 0, 2775, 1319, 1, 0] rerouted_tokens_per_ep_rank=[4095, 3349, 4876, 4094, 4098, 4095, 4058, 4103] avg=4096.0000 max=4876 max_rank=2 min=3349 min_rank=1 imbalance_ratio=1.190430 max_load_reduction=27892 max_load_reduction_pct=0.851196 moved_to_echo=16354 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 0, 0, 0, 2761, 0, 981, 257, 0, 0, 0, 0, 0, 2857, 1318, 10, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 0, 0, 0, 4091, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 4084, 16, 0, 0, 0, 0, 0, 0, 1, 2774, 1319, 1, 0] rerouted_tokens_per_ep_rank=[4096, 3999, 4185, 4094, 4095, 4104, 4100, 4095] avg=4096.0000 max=4185 max_rank=2 min=3999 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16394 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 3, 1, 0, 0, 0, 0, 2761, 0, 1420, 335, 0, 0, 0, 0, 0, 2344, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 2, 0, 4092, 0, 0, 0, 4091, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4092, 16, 0, 0, 0, 0, 6, 1, 0, 2774, 1319, 1, 0] rerouted_tokens_per_ep_rank=[4096, 4516, 3663, 4095, 4095, 4094, 4108, 4101] avg=4096.0000 max=4516 max_rank=1 min=3663 min_rank=2 imbalance_ratio=1.102539 max_load_reduction=28252 max_load_reduction_pct=0.862183 moved_to_echo=16398 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 2, 0, 0, 0, 0, 2759, 0, 1334, 385, 0, 0, 0, 0, 0, 2394, 1320, 2, 0, 0, 0, 0, 4, 1, 0, 4095, 0, 0, 0, 0, 1, 1, 0, 1, 4093, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 4074, 16, 0, 0, 0, 0, 0, 0, 0, 2776, 1319, 1, 0] rerouted_tokens_per_ep_rank=[4095, 4478, 3716, 4100, 4096, 4096, 4091, 4096] avg=4096.0000 max=4478 max_rank=1 min=3716 min_rank=2 imbalance_ratio=1.093262 max_load_reduction=28290 max_load_reduction_pct=0.863342 moved_to_echo=16379 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2754, 0, 780, 823, 0, 0, 0, 0, 0, 2518, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4071, 18, 4, 0, 0, 0, 0, 0, 0, 2780, 1323, 1, 0] rerouted_tokens_per_ep_rank=[4096, 4357, 3834, 4096, 4095, 4090, 4096, 4104] avg=4096.0000 max=4357 max_rank=1 min=3834 min_rank=2 imbalance_ratio=1.063721 max_load_reduction=28411 max_load_reduction_pct=0.867035 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 0, 0, 0, 2761, 0, 1215, 310, 0, 0, 0, 0, 0, 2584, 1319, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4082, 16, 0, 0, 0, 0, 0, 1, 0, 2776, 1319, 1, 0] rerouted_tokens_per_ep_rank=[4096, 4286, 3903, 4097, 4095, 4096, 4098, 4097] avg=4096.0000 max=4286 max_rank=1 min=3903 min_rank=2 imbalance_ratio=1.046387 max_load_reduction=28482 max_load_reduction_pct=0.869202 moved_to_echo=16386 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 22272, 10624, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 22272, 10624, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[22144, 0, 7552, 3200, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32640, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 4095, 4095, 1, 1, 0, 4095, 1019, 0, 0, 1, 4095, 0, 2, 1, 0, 1290, 0, 0, 4094, 0, 0, 3, 1, 20, 4094, 0, 1, 2, 1762, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4, 4092, 4091, 4, 1, 0, 4089, 1985, 0, 2, 4, 4091, 0, 2, 5, 2, 874, 0, 1, 4091, 2, 1, 0, 0, 107, 4089, 5, 7, 0, 1143, 0, 4076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 4095, 4096, 0, 0, 0, 4084, 1483, 0, 0, 0, 4096, 0, 0, 0, 0, 1347, 9, 0, 4091, 8, 2, 0, 10, 10, 4093, 0, 0, 6, 1254, 0, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 4091, 4096, 0, 0, 0, 4092, 1358, 0, 0, 0, 4096, 0, 0, 0, 0, 1188, 0, 0, 4093, 13, 0, 1, 4, 24, 4096, 0, 0, 3, 1517, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 4087, 4096, 0, 0, 0, 4095, 1235, 0, 0, 0, 4096, 0, 0, 0, 0, 950, 0, 0, 4091, 112, 0, 0, 7, 21, 4096, 0, 0, 0, 1786, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1, 4095, 4095, 1, 1, 0, 4094, 1152, 0, 0, 1, 4095, 0, 0, 2, 0, 1733, 0, 2, 4095, 1, 0, 8, 0, 18, 4094, 0, 1, 0, 1189, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 4092, 4096, 0, 0, 0, 4096, 2587, 0, 0, 0, 4096, 0, 0, 0, 0, 997, 0, 0, 4096, 9, 0, 0, 4, 2, 4096, 0, 0, 0, 501, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 4095, 4096, 0, 0, 0, 4095, 626, 0, 0, 0, 4096, 0, 0, 0, 0, 1267, 1, 0, 4095, 11, 0, 1, 1, 51, 4096, 0, 0, 0, 2141, 2, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 4092, 2, 4, 0, 0, 0, 0, 1, 0, 2664, 1985, 0, 0, 0, 0, 0, 2, 4, 4091, 0, 0, 0, 0, 0, 2, 5, 2, 4089, 0, 0, 0, 874, 0, 1, 2888, 0, 0, 0, 0, 2, 1, 0, 0, 1425, 1403, 1203, 31, 107, 4058, 5, 7, 0, 0, 0, 0, 0, 1143, 0, 2673, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4650, 4097, 4098, 3763, 4065, 4177, 3816] avg=4096.0000 max=4650 max_rank=1 min=3763 min_rank=4 imbalance_ratio=1.135254 max_load_reduction=28118 max_load_reduction_pct=0.858093 moved_to_echo=15821 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 4095, 2, 1, 0, 0, 0, 0, 1, 0, 2668, 1019, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 2, 1, 0, 4093, 0, 0, 0, 1290, 0, 0, 2890, 0, 0, 0, 0, 0, 0, 3, 1, 1427, 1409, 1204, 31, 20, 4063, 0, 1, 0, 0, 0, 0, 2, 1762, 0, 2686, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 3688, 4096, 4096, 4180, 4075, 4084, 4450] avg=4096.0000 max=4450 max_rank=7 min=3688 min_rank=1 imbalance_ratio=1.086426 max_load_reduction=28318 max_load_reduction_pct=0.864197 moved_to_echo=16789 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 2668, 1235, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 950, 0, 0, 2888, 0, 0, 0, 0, 112, 0, 0, 7, 1427, 1409, 1203, 31, 21, 4065, 0, 0, 0, 0, 0, 0, 0, 1786, 0, 2687, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 3903, 4096, 4094, 3838, 4189, 4086, 4473] avg=4096.0000 max=4473 max_rank=7 min=3838 min_rank=4 imbalance_ratio=1.092041 max_load_reduction=28295 max_load_reduction_pct=0.863495 moved_to_echo=16586 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 4095, 2, 1, 0, 0, 0, 0, 1, 0, 2667, 1152, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 1733, 0, 2, 2890, 0, 0, 0, 0, 1, 0, 8, 0, 1427, 1407, 1205, 31, 18, 4063, 0, 1, 0, 0, 0, 0, 0, 1189, 0, 2683, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 3820, 4096, 4095, 4625, 4079, 4082, 3872] avg=4096.0000 max=4625 max_rank=4 min=3820 min_rank=1 imbalance_ratio=1.129150 max_load_reduction=28143 max_load_reduction_pct=0.858856 moved_to_echo=16658 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 2666, 1358, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1188, 0, 0, 2889, 0, 0, 0, 0, 13, 0, 1, 4, 1426, 1409, 1204, 31, 24, 4065, 0, 0, 0, 0, 0, 0, 3, 1517, 0, 2687, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4024, 4096, 4094, 4077, 4088, 4089, 4207] avg=4096.0000 max=4207 max_rank=7 min=4024 min_rank=1 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16461 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 2661, 1483, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1347, 9, 0, 2888, 0, 0, 0, 0, 8, 2, 0, 10, 1423, 1405, 1203, 31, 10, 4062, 0, 0, 0, 0, 0, 0, 6, 1254, 0, 2679, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4144, 4096, 4094, 4244, 4082, 4072, 3939] avg=4096.0000 max=4244 max_rank=4 min=3939 min_rank=7 imbalance_ratio=1.036133 max_load_reduction=28524 max_load_reduction_pct=0.870483 moved_to_echo=16337 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 2587, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 997, 0, 0, 2886, 0, 0, 0, 0, 9, 0, 0, 4, 1431, 1414, 1210, 36, 2, 4060, 0, 0, 0, 0, 0, 0, 0, 501, 0, 2682, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 5245, 4096, 4103, 3883, 4104, 4062, 3183] avg=4096.0000 max=5245 max_rank=1 min=3183 min_rank=7 imbalance_ratio=1.280518 max_load_reduction=27523 max_load_reduction_pct=0.839935 moved_to_echo=15232 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 2668, 626, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1267, 1, 0, 2890, 0, 0, 0, 0, 11, 0, 1, 1, 1427, 1409, 1205, 31, 51, 4065, 0, 0, 0, 0, 0, 0, 0, 2141, 2, 2685, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3294, 4096, 4094, 4158, 4085, 4116, 4828] avg=4096.0000 max=4828 max_rank=7 min=3294 min_rank=1 imbalance_ratio=1.178711 max_load_reduction=27940 max_load_reduction_pct=0.852661 moved_to_echo=17187 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32769, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32769, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32769, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32769, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32769, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32769, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32769, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32769, 32768, 32768, 32767, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32767 actual_valid_recv=33536 observed_loss=-769 tokens_per_expert=[256, 128, 128, 128, 11520, 11392, 9728, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 11392, 128, 21504, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9728, 128, 128, 23168, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 21376, 11520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 32512, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32769 actual_valid_recv=33024 observed_loss=-255 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 4096, 0, 4096, 0, 3848, 0, 4076, 0, 0, 0, 0, 21, 3, 4093, 4096, 0, 0, 4077, 0, 1, 4094, 249, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 22, 4096, 0, 4090, 0, 4081, 3, 4092, 0, 0, 0, 0, 7, 0, 4092, 4085, 0, 0, 4088, 3, 7, 4086, 11, 0, 0, 0, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 2, 4093, 1, 4095, 0, 4059, 0, 4087, 1, 1, 1, 0, 3, 5, 4089, 4093, 1, 1, 4095, 0, 0, 4092, 40, 0, 2, 0, 0, 2, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[5, 4, 5, 0, 4089, 0, 4091, 0, 4091, 0, 4086, 10, 5, 7, 0, 3, 7, 4090, 4086, 0, 3, 4093, 0, 0, 4085, 0, 0, 1, 0, 7, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 17, 4064, 0, 4095, 0, 3925, 9, 4093, 0, 0, 0, 0, 1, 0, 4089, 4082, 0, 0, 4083, 7, 18, 4088, 192, 0, 0, 0, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2, 1, 2, 0, 4092, 0, 4095, 1, 4069, 0, 4094, 2, 2, 1, 0, 0, 0, 4092, 4094, 0, 1, 4095, 0, 0, 4094, 14, 0, 5, 0, 0, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 4096, 0, 4089, 0, 3989, 0, 4081, 0, 0, 0, 0, 3, 4, 4085, 4095, 0, 0, 4092, 0, 2, 4096, 124, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4094, 0, 4096, 0, 4054, 1, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4095, 0, 0, 4095, 40, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 3993, 79, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 3848, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4086, 4, 0, 0, 3, 4093, 3, 0, 0, 0, 0, 0, 0, 4077, 0, 1, 0, 0, 0, 0, 4015, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4103, 3927, 4111, 4099, 4078, 4264, 4096] avg=4096.0000 max=4264 max_rank=6 min=3927 min_rank=2 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 4, 5, 0, 4002, 79, 0, 0, 4089, 0, 7, 0, 0, 0, 0, 0, 4091, 0, 80, 10, 0, 0, 0, 0, 5, 7, 0, 3, 4081, 4, 0, 0, 7, 4090, 3, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 1, 0, 0, 0, 0, 0, 7, 0, 0, 4083, 3, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4181, 4100, 4100, 4096, 4007, 4093] avg=4096.0000 max=4181 max_rank=2 min=4007 min_rank=6 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 22, 4008, 79, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 4081, 3, 80, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4080, 4, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 4088, 3, 7, 0, 0, 0, 0, 4007, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4082, 3, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4103, 4164, 4091, 4095, 4098, 4018, 4090] avg=4096.0000 max=4164 max_rank=2 min=4018 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 2, 0, 4010, 79, 0, 0, 4092, 0, 7, 1, 0, 0, 0, 0, 4069, 0, 80, 2, 0, 0, 0, 0, 2, 1, 0, 0, 4085, 4, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 4015, 14, 0, 5, 0, 0, 0, 0, 0, 0, 12, 0, 4091, 3, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4100, 4151, 4092, 4095, 4096, 4034, 4106] avg=4096.0000 max=4151 max_rank=2 min=4034 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 2, 4003, 79, 0, 0, 4093, 1, 7, 0, 0, 0, 0, 0, 4059, 0, 80, 1, 0, 0, 0, 0, 1, 1, 0, 3, 4085, 4, 0, 0, 5, 4089, 3, 1, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 4013, 40, 0, 2, 0, 0, 0, 0, 0, 0, 2, 2, 4090, 3, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4101, 4140, 4094, 4098, 4096, 4055, 4097] avg=4096.0000 max=4140 max_rank=2 min=4055 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 17, 4009, 79, 0, 0, 4064, 0, 7, 0, 0, 0, 0, 0, 3925, 9, 80, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 4, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 0, 4083, 7, 18, 0, 0, 0, 0, 4009, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4079, 3, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4071, 4014, 4090, 4092, 4108, 4201, 4087] avg=4096.0000 max=4201 max_rank=6 min=4014 min_rank=2 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 4003, 80, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4085, 11, 0, 0, 4, 4085, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 7, 0, 0, 0, 4009, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4056, 4099, 4090, 4101, 4133, 4098] avg=4096.0000 max=4133 max_rank=6 min=4056 min_rank=2 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4012, 79, 0, 0, 4094, 0, 4, 0, 0, 0, 0, 0, 4054, 1, 80, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 4, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4016, 40, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 6, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4098, 4135, 4091, 4099, 4095, 4056, 4101] avg=4096.0000 max=4135 max_rank=2 min=4056 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32128, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32128, 768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32128, 128, 640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4091, 0, 0, 5, 0, 0, 2348, 4096, 0, 0, 4096, 1748, 0, 0, 1, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 4092, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 857, 4096, 0, 0, 4096, 3239, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[8, 4087, 12, 4092, 0, 7, 0, 4, 6, 1813, 4086, 0, 0, 4079, 2278, 0, 0, 0, 5, 4, 0, 4088, 2, 4090, 4, 5, 1, 5, 4085, 5, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4094, 3, 4095, 0, 1, 0, 1, 1, 1439, 4092, 0, 0, 4094, 2657, 1, 0, 0, 2, 1, 0, 4095, 0, 4095, 0, 1, 0, 0, 4094, 1, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4090, 0, 0, 5, 0, 0, 1138, 4096, 0, 0, 4096, 2958, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1625, 4096, 0, 0, 4096, 2471, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[2, 4092, 4, 4095, 0, 3, 0, 1, 2, 2485, 4094, 1, 0, 4092, 1609, 0, 0, 0, 1, 1, 0, 4094, 1, 4094, 0, 2, 0, 0, 4093, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1267, 4096, 0, 0, 4096, 2829, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2472, 1621, 1, 0, 0, 857, 2475, 0, 0, 0, 0, 0, 0, 1624, 3239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4094, 3332, 4863, 4096, 4098, 4094, 4096] avg=4096.0000 max=4863 max_rank=3 min=3332 min_rank=2 imbalance_ratio=1.187256 max_load_reduction=27905 max_load_reduction_pct=0.851593 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 5, 0, 2472, 1621, 0, 0, 0, 2348, 2475, 0, 0, 0, 0, 0, 0, 1624, 1748, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4092, 0, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4098, 4823, 3372, 4097, 4097, 4094, 4096] avg=4096.0000 max=4823 max_rank=2 min=3372 min_rank=3 imbalance_ratio=1.177490 max_load_reduction=27945 max_load_reduction_pct=0.852814 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 12, 4092, 0, 0, 0, 0, 0, 7, 0, 4, 2462, 1617, 0, 0, 6, 1813, 2469, 0, 0, 0, 0, 0, 0, 1617, 2278, 0, 0, 0, 0, 0, 0, 0, 5, 4, 4087, 0, 0, 0, 0, 4088, 2, 1, 0, 0, 0, 0, 4, 5, 1, 5, 4089, 0, 0, 0, 4085, 5, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4090, 4288, 3895, 4096, 4091, 4104, 4092] avg=4096.0000 max=4288 max_rank=2 min=3895 min_rank=3 imbalance_ratio=1.046875 max_load_reduction=28480 max_load_reduction_pct=0.869141 moved_to_echo=16383 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4, 4094, 0, 0, 0, 0, 0, 3, 0, 1, 2470, 1620, 1, 0, 2, 2485, 2474, 1, 0, 0, 0, 0, 0, 1622, 1609, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4092, 0, 0, 0, 0, 4094, 1, 2, 0, 0, 0, 0, 0, 2, 0, 0, 4092, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4095, 4962, 3231, 4094, 4097, 4094, 4095] avg=4096.0000 max=4962 max_rank=2 min=3231 min_rank=3 imbalance_ratio=1.211426 max_load_reduction=27806 max_load_reduction_pct=0.848572 moved_to_echo=16380 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 4094, 0, 0, 0, 0, 0, 1, 0, 1, 2471, 1619, 1, 0, 1, 1439, 2473, 0, 0, 0, 0, 0, 0, 1623, 2657, 1, 0, 0, 0, 0, 0, 0, 2, 1, 4094, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 4094, 1, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4093, 3913, 4281, 4097, 4095, 4096, 4096] avg=4096.0000 max=4281 max_rank=3 min=3913 min_rank=2 imbalance_ratio=1.045166 max_load_reduction=28487 max_load_reduction_pct=0.869354 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 5, 0, 2472, 1621, 0, 0, 0, 1138, 2475, 0, 0, 0, 0, 0, 0, 1624, 2958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4098, 3613, 4582, 4096, 4098, 4094, 4097] avg=4096.0000 max=4582 max_rank=3 min=3613 min_rank=2 imbalance_ratio=1.118652 max_load_reduction=28186 max_load_reduction_pct=0.860168 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 2476, 1626, 4, 0, 0, 1267, 2470, 0, 0, 0, 0, 0, 0, 1620, 2829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4106, 3737, 4449, 4096, 4096, 4096, 4100] avg=4096.0000 max=4449 max_rank=3 min=3737 min_rank=2 imbalance_ratio=1.086182 max_load_reduction=28319 max_load_reduction_pct=0.864227 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2472, 1621, 1, 0, 0, 1625, 2475, 0, 0, 0, 0, 0, 0, 1624, 2471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4094, 4100, 4095, 4096, 4096, 4096, 4096] avg=4096.0000 max=4100 max_rank=2 min=4094 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 19840, 13056, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 13056, 19840, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 13056, 19840, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 3089, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 1005, 0, 0, 0, 0, 4094, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 3, 0, 3, 0, 2, 4, 12, 3092, 4089, 0, 2, 0, 0, 5, 4080, 0, 4088, 4083, 0, 7, 4091, 1001, 0, 8, 0, 8, 4089, 0, 6, 5, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 2, 2, 0, 3886, 4095, 0, 1, 0, 0, 1, 4095, 0, 4094, 4094, 0, 1, 4095, 210, 0, 2, 0, 1, 4093, 0, 1, 1, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 0, 0, 0, 3412, 4093, 0, 0, 2, 4, 0, 4096, 0, 4095, 4096, 0, 0, 4085, 682, 0, 0, 9, 0, 4086, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1, 2, 0, 0, 0, 1, 0, 3, 3635, 4094, 0, 1, 0, 0, 1, 4091, 1, 4093, 4093, 0, 2, 4094, 460, 1, 2, 0, 0, 4095, 1, 1, 2, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 0, 3899, 4095, 0, 0, 4, 5, 0, 4096, 0, 4096, 4096, 0, 0, 4094, 236, 0, 0, 2, 0, 4047, 0, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 0, 3448, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 671, 0, 0, 0, 0, 4068, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 3528, 4096, 0, 0, 0, 1, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 567, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 2, 0, 0, 4, 0, 0, 0, 3496, 595, 0, 0, 3089, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 3493, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 3687, 4096, 4099, 4498, 4102, 4096] avg=4096.0000 max=4498 max_rank=5 min=3687 min_rank=2 imbalance_ratio=1.098145 max_load_reduction=28270 max_load_reduction_pct=0.862732 moved_to_echo=16795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 3, 4085, 2, 0, 0, 0, 2, 4, 12, 3490, 594, 0, 0, 3092, 597, 0, 2, 0, 0, 0, 0, 0, 0, 5, 4080, 0, 0, 0, 0, 0, 3, 4083, 0, 0, 0, 0, 0, 7, 3490, 1001, 0, 0, 0, 0, 0, 8, 0, 8, 4089, 7, 0, 0, 0, 0, 6, 5, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4102, 3691, 4085, 4086, 4498, 4112, 4101] avg=4096.0000 max=4498 max_rank=5 min=3691 min_rank=2 imbalance_ratio=1.098145 max_load_reduction=28270 max_load_reduction_pct=0.862732 moved_to_echo=16797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 0, 0, 4090, 2, 0, 0, 0, 1, 0, 3, 3494, 594, 0, 0, 3635, 598, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 1, 3, 4093, 0, 0, 0, 0, 0, 2, 3492, 460, 1, 0, 0, 0, 0, 2, 0, 0, 4095, 8, 0, 0, 0, 1, 1, 2, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4092, 4234, 4092, 4097, 3955, 4105, 4098] avg=4096.0000 max=4234 max_rank=2 min=3955 min_rank=5 imbalance_ratio=1.033691 max_load_reduction=28534 max_load_reduction_pct=0.870789 moved_to_echo=16255 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 2, 0, 0, 1, 2, 2, 0, 3495, 594, 0, 0, 3886, 598, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 1, 3493, 210, 0, 0, 0, 0, 0, 2, 0, 1, 4093, 8, 0, 0, 0, 0, 1, 1, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4094, 4485, 4096, 4096, 3704, 4104, 4095] avg=4096.0000 max=4485 max_rank=2 min=3704 min_rank=5 imbalance_ratio=1.094971 max_load_reduction=28283 max_load_reduction_pct=0.863129 moved_to_echo=15999 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 2, 0, 0, 12, 0, 0, 0, 3493, 593, 0, 0, 3412, 598, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 3485, 682, 0, 0, 0, 0, 0, 0, 9, 0, 4086, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4098, 4010, 4102, 4099, 4167, 4102, 4096] avg=4096.0000 max=4167 max_rank=5 min=4010 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 2, 0, 0, 5, 0, 0, 0, 3495, 594, 0, 0, 3899, 598, 0, 0, 0, 0, 0, 0, 4, 5, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 3492, 236, 0, 0, 0, 0, 0, 0, 2, 0, 4047, 8, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4094, 4497, 4105, 4099, 3728, 4057, 4093] avg=4096.0000 max=4497 max_rank=2 min=3728 min_rank=5 imbalance_ratio=1.097900 max_load_reduction=28271 max_load_reduction_pct=0.862762 moved_to_echo=15977 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 8, 0, 0, 5, 0, 0, 0, 3498, 599, 0, 0, 3448, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3487, 671, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 10, 3, 1, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4102, 4038, 4095, 4096, 4158, 4082, 4093] avg=4096.0000 max=4158 max_rank=5 min=4038 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 0, 0, 1, 0, 0, 0, 3496, 594, 0, 0, 3528, 598, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3493, 567, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4091, 4126, 4097, 4096, 4060, 4104, 4096] avg=4096.0000 max=4126 max_rank=2 min=4060 min_rank=5 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[28032, 4864, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 28032, 4864, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 28032, 4864, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 1, 1, 4095, 4095, 0, 1, 0, 1346, 1, 0, 1, 0, 4091, 4094, 4095, 0, 1, 0, 0, 4095, 4095, 1, 0, 2751, 0, 1, 1, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 11, 3360, 0, 0, 0, 0, 4051, 4096, 4096, 0, 0, 4, 35, 4096, 4091, 0, 0, 736, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 7, 9, 0, 4090, 4086, 4, 6, 0, 2148, 6, 0, 0, 0, 4090, 4083, 4086, 0, 6, 0, 0, 4089, 4090, 7, 0, 1940, 3, 10, 0, 1, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2321, 0, 0, 0, 0, 4077, 4096, 4096, 0, 0, 8, 19, 4092, 4089, 0, 0, 1774, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 2, 1, 0, 4094, 4093, 0, 2, 0, 2574, 2, 0, 0, 0, 4094, 4090, 4094, 0, 2, 0, 0, 4094, 4094, 2, 0, 1524, 0, 2, 1, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3111, 0, 0, 0, 0, 4087, 4096, 4096, 0, 0, 8, 6, 4096, 4091, 0, 0, 981, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 2, 3585, 0, 0, 0, 0, 4088, 4096, 4096, 0, 0, 0, 13, 4096, 4096, 0, 0, 504, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1483, 0, 0, 0, 0, 4092, 4096, 4096, 0, 0, 1, 4, 4096, 4095, 0, 0, 2613, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 11, 3360, 0, 0, 1593, 7, 2, 0, 0, 0, 4051, 13, 0, 0, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 0, 35, 0, 4084, 0, 0, 0, 0, 0, 0, 736, 0, 0, 2490, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4973, 4064, 4098, 4119, 3229, 4095] avg=4096.0000 max=4973 max_rank=2 min=3229 min_rank=6 imbalance_ratio=1.214111 max_load_reduction=27795 max_load_reduction_pct=0.848236 moved_to_echo=15541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 1, 4093, 2, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 1346, 1, 0, 1592, 7, 2, 0, 1, 0, 4091, 13, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 2751, 0, 1, 2489, 2, 0, 0, 1, 0, 0, 1, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 2948, 4105, 4094, 4089, 5243, 4095] avg=4096.0000 max=5243 max_rank=6 min=2948 min_rank=2 imbalance_ratio=1.280029 max_load_reduction=27525 max_load_reduction_pct=0.839996 moved_to_echo=17521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 9, 0, 4087, 1, 0, 0, 0, 4086, 4, 6, 0, 0, 0, 0, 0, 2148, 6, 0, 1588, 7, 2, 0, 0, 0, 4090, 13, 0, 0, 0, 0, 4084, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4083, 7, 0, 0, 0, 0, 0, 1940, 3, 10, 2482, 2, 0, 0, 0, 1, 0, 7, 4089, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4096, 3751, 4103, 4090, 4090, 4437, 4097] avg=4096.0000 max=4437 max_rank=6 min=3751 min_rank=2 imbalance_ratio=1.083252 max_load_reduction=28331 max_load_reduction_pct=0.864594 moved_to_echo=16714 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 1, 0, 4091, 1, 0, 0, 1, 4093, 0, 2, 0, 0, 0, 0, 0, 2574, 2, 0, 1591, 7, 2, 0, 0, 0, 4094, 13, 0, 0, 0, 0, 4092, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 0, 0, 0, 0, 0, 1524, 0, 2, 2486, 3, 0, 0, 1, 0, 0, 2, 4092, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4176, 4107, 4094, 4089, 4015, 4095] avg=4096.0000 max=4176 max_rank=2 min=4015 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16293 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3111, 0, 0, 1593, 7, 2, 0, 0, 0, 4087, 13, 0, 0, 0, 0, 4094, 0, 0, 8, 0, 0, 0, 0, 6, 0, 4084, 0, 0, 0, 0, 0, 0, 981, 0, 0, 2490, 3, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4713, 4100, 4102, 4090, 3474, 4099] avg=4096.0000 max=4713 max_rank=2 min=3474 min_rank=6 imbalance_ratio=1.150635 max_load_reduction=28055 max_load_reduction_pct=0.856171 moved_to_echo=15765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2321, 0, 0, 1593, 7, 2, 0, 0, 0, 4077, 13, 0, 0, 0, 0, 4094, 0, 0, 8, 0, 0, 0, 0, 19, 0, 4082, 0, 0, 0, 0, 0, 0, 1774, 0, 0, 2490, 3, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4096, 3923, 4090, 4102, 4101, 4267, 4099] avg=4096.0000 max=4267 max_rank=6 min=3923 min_rank=2 imbalance_ratio=1.041748 max_load_reduction=28501 max_load_reduction_pct=0.869781 moved_to_echo=16569 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 3585, 0, 0, 1598, 11, 3, 0, 0, 0, 4088, 6, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4085, 0, 0, 0, 0, 0, 0, 504, 0, 0, 2492, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 5199, 4094, 4093, 4098, 2998, 4094] avg=4096.0000 max=5199 max_rank=2 min=2998 min_rank=6 imbalance_ratio=1.269287 max_load_reduction=27569 max_load_reduction_pct=0.841339 moved_to_echo=15283 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1483, 0, 0, 1593, 7, 2, 0, 0, 0, 4092, 13, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 4, 0, 4088, 0, 0, 0, 0, 0, 0, 2613, 0, 0, 2490, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 3085, 4105, 4095, 4092, 5105, 4094] avg=4096.0000 max=5105 max_rank=6 min=3085 min_rank=2 imbalance_ratio=1.246338 max_load_reduction=27663 max_load_reduction_pct=0.844208 moved_to_echo=17386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 12928, 128, 128, 19968, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 19968, 128, 0, 12800, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 91, 0, 0, 0, 4072, 4048, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 4006, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4094, 0, 4093, 0, 0, 0, 2, 2, 4094, 4076, 4091, 2, 2, 2, 0, 2, 0, 2, 0, 3, 21, 0, 3, 0, 0, 4094, 0, 0, 4094, 4091, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4094, 0, 4094, 0, 6, 0, 1, 2, 4093, 4088, 4094, 2, 1, 1, 0, 2, 1, 0, 0, 3, 3, 0, 2, 0, 1, 4094, 0, 0, 4095, 4091, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 24, 0, 0, 0, 4076, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4086, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 24, 0, 0, 0, 4072, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4091, 0, 0, 4096, 4096, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4087, 0, 4084, 0, 0, 0, 6, 7, 4090, 4055, 4073, 8, 6, 9, 0, 5, 0, 15, 0, 15, 45, 0, 9, 0, 0, 4090, 0, 0, 4088, 4076, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 81, 0, 0, 0, 4018, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 4071, 0, 0, 4096, 4096, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 6, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4090, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4096, 0, 0, 0, 0, 0, 91, 0, 0, 0, 4011, 19, 0, 0, 4072, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 71, 0, 0, 0, 4073, 1, 0, 0, 0, 4006, 0, 0, 17, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4121, 4092, 4093, 4095, 4145, 4023, 4100] avg=4096.0000 max=4145 max_rank=5 min=4023 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4094, 0, 0, 0, 0, 0, 6, 0, 1, 2, 4051, 19, 0, 0, 4093, 19, 0, 2, 0, 0, 0, 0, 1, 1, 0, 2, 4092, 0, 0, 0, 1, 0, 0, 3, 4093, 0, 0, 0, 3, 0, 2, 0, 4073, 1, 0, 0, 1, 4094, 0, 0, 18, 0, 0, 0, 3, 4091, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4079, 4114, 4096, 4097, 4079, 4113, 4094] avg=4096.0000 max=4114 max_rank=2 min=4079 min_rank=1 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 6, 7, 4018, 19, 0, 0, 4090, 19, 0, 8, 0, 0, 0, 0, 6, 9, 0, 5, 4084, 0, 0, 0, 0, 15, 0, 15, 4072, 0, 0, 0, 45, 0, 9, 0, 4065, 1, 0, 0, 0, 4090, 0, 0, 18, 0, 0, 0, 4, 4076, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4050, 4117, 4104, 4102, 4120, 4108, 4080] avg=4096.0000 max=4120 max_rank=5 min=4050 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4063, 23, 0, 0, 4076, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 6, 0, 0, 0, 4073, 1, 0, 0, 0, 4086, 0, 0, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4110, 4089, 4096, 4095, 4080, 4106, 4096] avg=4096.0000 max=4110 max_rank=1 min=4080 min_rank=5 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4096, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4059, 19, 0, 0, 4072, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 4073, 1, 0, 0, 0, 4091, 0, 0, 18, 0, 0, 0, 4, 4096, 0, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4102, 4091, 4093, 4095, 4076, 4109, 4103] avg=4096.0000 max=4109 max_rank=6 min=4076 min_rank=5 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4096, 0, 0, 0, 0, 0, 81, 0, 0, 0, 4059, 19, 0, 0, 4018, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 18, 0, 0, 0, 4073, 1, 0, 0, 0, 4071, 0, 0, 18, 0, 0, 0, 4, 4096, 0, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4159, 4037, 4093, 4095, 4092, 4089, 4104] avg=4096.0000 max=4159 max_rank=1 min=4037 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4059, 20, 0, 0, 4094, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 4076, 1, 0, 0, 0, 4090, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4085, 4113, 4096, 4095, 4079, 4108, 4096] avg=4096.0000 max=4113 max_rank=2 min=4079 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4039, 19, 0, 0, 4094, 19, 0, 2, 0, 0, 0, 0, 2, 2, 0, 2, 4091, 0, 0, 0, 0, 2, 0, 3, 4089, 0, 0, 0, 21, 0, 3, 0, 4071, 2, 0, 0, 0, 4094, 0, 0, 18, 0, 0, 0, 4, 4091, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4062, 4115, 4097, 4094, 4097, 4112, 4095] avg=4096.0000 max=4115 max_rank=2 min=4062 min_rank=1 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 0, 128, 128, 32384, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 256, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 0, 128, 0, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32640, 0, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4090, 4096, 0, 0, 4096, 0, 4096, 0, 5, 4089, 0, 0, 4096, 0, 0, 24, 0, 0, 0, 4, 0, 7, 4072, 0, 0, 0, 4092, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 2, 0, 4096, 0, 4096, 0, 0, 4056, 0, 0, 4091, 0, 0, 21, 0, 0, 0, 0, 0, 83, 4036, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4095, 4095, 7, 2, 4095, 1, 4094, 2, 2, 4094, 0, 1, 4018, 1, 1, 451, 0, 0, 1, 0, 0, 1, 3712, 2, 0, 0, 4092, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4087, 4083, 0, 9, 4083, 3, 4087, 0, 0, 4090, 0, 8, 4001, 5, 7, 693, 5, 0, 6, 0, 44, 0, 3439, 6, 9, 0, 4082, 0, 5, 0, 16, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4094, 4096, 0, 0, 4096, 0, 4096, 0, 8, 4077, 0, 0, 4086, 0, 0, 210, 0, 0, 0, 3, 0, 18, 3896, 0, 0, 0, 4088, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4092, 0, 0, 4093, 0, 0, 168, 0, 0, 0, 2, 0, 17, 3916, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4095, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4094, 0, 0, 4091, 0, 0, 339, 0, 0, 0, 1, 0, 2, 3762, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4093, 4093, 1, 3, 4093, 0, 4094, 2, 0, 4094, 0, 2, 4077, 2, 2, 423, 0, 1, 2, 0, 14, 0, 3672, 1, 3, 0, 4092, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 7, 0, 0, 0, 3824, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 83, 4036, 0, 258, 1, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4098, 4063, 3845, 4095, 4378, 4097, 4094] avg=4096.0000 max=4378 max_rank=5 min=3845 min_rank=3 imbalance_ratio=1.068848 max_load_reduction=28390 max_load_reduction_pct=0.866394 moved_to_echo=16664 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 5, 4089, 0, 0, 7, 0, 0, 0, 3829, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 7, 4072, 0, 258, 1, 0, 0, 0, 0, 4092, 1, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4098, 4101, 3853, 4099, 4338, 4095, 4094] avg=4096.0000 max=4338 max_rank=5 min=3853 min_rank=3 imbalance_ratio=1.059082 max_load_reduction=28430 max_load_reduction_pct=0.867615 moved_to_echo=16626 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 0, 9, 0, 0, 0, 0, 4083, 3, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 8, 6, 0, 0, 0, 3741, 5, 7, 693, 0, 0, 0, 0, 5, 0, 6, 0, 4083, 0, 0, 0, 44, 0, 3439, 6, 252, 1, 0, 0, 9, 0, 4082, 0, 2, 0, 0, 0, 5, 0, 16, 0, 4085, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4087, 4104, 4446, 4094, 3742, 4093, 4106] avg=4096.0000 max=4446 max_rank=3 min=3742 min_rank=5 imbalance_ratio=1.085449 max_load_reduction=28322 max_load_reduction_pct=0.864319 moved_to_echo=16035 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 8, 4077, 0, 0, 7, 0, 0, 0, 3819, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 18, 3896, 0, 258, 1, 0, 0, 0, 0, 4088, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4098, 4092, 4029, 4097, 4173, 4090, 4095] avg=4096.0000 max=4173 max_rank=5 min=4029 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16455 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 7, 2, 0, 0, 0, 0, 4095, 1, 0, 2, 0, 0, 0, 0, 2, 4094, 0, 1, 6, 0, 0, 0, 3757, 1, 1, 451, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 1, 3712, 2, 253, 1, 0, 0, 0, 0, 4092, 0, 2, 0, 0, 0, 0, 0, 1, 0, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4098, 4103, 4210, 4095, 3969, 4094, 4095] avg=4096.0000 max=4210 max_rank=3 min=3969 min_rank=5 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 9, 0, 0, 0, 3816, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 17, 3916, 0, 262, 3, 0, 0, 0, 0, 4096, 0, 6, 2, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4101, 3984, 4097, 4198, 4104, 4094] avg=4096.0000 max=4198 max_rank=5 min=3984 min_rank=3 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16493 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 1, 3, 0, 0, 0, 0, 4093, 0, 2, 2, 0, 0, 0, 0, 0, 4094, 0, 2, 7, 0, 0, 0, 3811, 2, 2, 423, 0, 0, 0, 0, 0, 1, 2, 0, 4092, 0, 0, 0, 14, 0, 3672, 1, 257, 1, 0, 0, 3, 0, 4092, 0, 2, 0, 0, 0, 0, 0, 4, 0, 4091, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4097, 4103, 4238, 4095, 3945, 4097, 4096] avg=4096.0000 max=4238 max_rank=3 min=3945 min_rank=5 imbalance_ratio=1.034668 max_load_reduction=28530 max_load_reduction_pct=0.870667 moved_to_echo=16233 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 7, 0, 0, 0, 3824, 0, 0, 339, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 2, 3762, 0, 258, 3, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4101, 4163, 4096, 4025, 4098, 4094] avg=4096.0000 max=4163 max_rank=3 min=4025 min_rank=5 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16313 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[30464, 128, 128, 2432, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 30592, 128, 2176, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[11, 6, 4084, 0, 5, 0, 0, 0, 4086, 0, 1, 8, 0, 11, 9, 4086, 0, 10, 5, 4087, 4081, 3759, 6, 4075, 4089, 334, 0, 0, 0, 0, 0, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 26, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 4014, 4096, 62, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[2, 1, 4094, 0, 1, 0, 0, 0, 4094, 0, 0, 2, 0, 1, 1, 4094, 0, 2, 1, 4094, 4091, 4089, 2, 4070, 4095, 26, 4, 0, 0, 0, 2, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 4096, 4096, 4089, 0, 4068, 4096, 0, 6, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 4096, 8, 0, 0, 4096, 4096, 4094, 0, 4087, 4096, 0, 3, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4096, 4096, 4070, 0, 4087, 4096, 26, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[3, 1, 4093, 0, 1, 0, 0, 0, 4093, 1, 0, 2, 0, 4, 2, 4094, 0, 3, 2, 4094, 4090, 4021, 1, 4080, 4094, 83, 2, 0, 0, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 4096, 4094, 0, 4094, 4096, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 26, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4089, 0, 57, 0, 0, 0, 0, 4029, 62, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 67, 6, 5] rerouted_tokens_per_ep_rank=[4116, 4098, 4096, 4096, 4091, 4146, 4092, 4033] avg=4096.0000 max=4146 max_rank=5 min=4033 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[11, 6, 4078, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4081, 2, 0, 0, 4086, 0, 1, 8, 0, 0, 0, 0, 0, 11, 9, 4086, 0, 0, 0, 0, 0, 10, 5, 4082, 0, 0, 0, 0, 0, 3759, 6, 58, 0, 0, 0, 0, 4022, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4015, 67, 6, 5] rerouted_tokens_per_ep_rank=[4095, 4088, 4095, 4106, 4097, 3823, 4356, 4108] avg=4096.0000 max=4356 max_rank=6 min=3823 min_rank=5 imbalance_ratio=1.063477 max_load_reduction=28412 max_load_reduction_pct=0.867065 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 4087, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4090, 2, 0, 0, 4093, 1, 0, 2, 0, 0, 0, 0, 0, 4, 2, 4094, 0, 0, 0, 0, 0, 3, 2, 4089, 0, 0, 0, 0, 0, 4021, 1, 58, 0, 0, 0, 0, 4027, 83, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4020, 67, 6, 5] rerouted_tokens_per_ep_rank=[4091, 4093, 4096, 4100, 4094, 4080, 4112, 4102] avg=4096.0000 max=4112 max_rank=6 min=4080 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 4088, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4091, 2, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 2, 1, 4089, 0, 0, 0, 0, 0, 4089, 2, 58, 0, 0, 0, 0, 4028, 26, 4, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4010, 67, 6, 5] rerouted_tokens_per_ep_rank=[4091, 4094, 4096, 4096, 4092, 4149, 4058, 4092] avg=4096.0000 max=4149 max_rank=5 min=4058 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 17, 0, 0, 4091, 0, 0, 0, 0, 0, 4089, 0, 58, 0, 0, 0, 0, 4029, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 67, 6, 5] rerouted_tokens_per_ep_rank=[4102, 4098, 4096, 4096, 4108, 4147, 4035, 4086] avg=4096.0000 max=4147 max_rank=5 min=4035 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 4091, 0, 0, 0, 0, 0, 4094, 0, 58, 0, 0, 0, 0, 4029, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 67, 6, 5] rerouted_tokens_per_ep_rank=[4090, 4098, 4095, 4097, 4099, 4152, 4032, 4105] avg=4096.0000 max=4152 max_rank=5 min=4032 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 5, 0, 0, 4090, 0, 0, 0, 0, 0, 4070, 0, 49, 0, 0, 0, 0, 4026, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 70, 8, 6] rerouted_tokens_per_ep_rank=[4092, 4101, 4098, 4094, 4095, 4119, 4052, 4117] avg=4096.0000 max=4119 max_rank=5 min=4052 min_rank=6 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 4094, 0, 58, 0, 0, 0, 0, 4029, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 67, 6, 5] rerouted_tokens_per_ep_rank=[4091, 4098, 4096, 4096, 4092, 4152, 4031, 4112] avg=4096.0000 max=4152 max_rank=5 min=4031 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32781, 32768, 32768, 32768, 32755] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32781, 32768, 32768, 32768, 32755] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32781, 32768, 32768, 32768, 32755] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32781, 32768, 32768, 32768, 32755] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32781, 32768, 32768, 32768, 32755] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32781, 32768, 32768, 32768, 32755] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32781, 32768, 32768, 32768, 32755] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32781, 32768, 32768, 32768, 32755] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32384, 128, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32755 actual_valid_recv=33280 observed_loss=-525 tokens_per_expert=[0, 0, 128, 128, 32128, 640, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32256, 640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32781 actual_valid_recv=33152 observed_loss=-371 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 84, 4012, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4095, 4093, 0, 1, 0, 3, 4095, 0, 0, 1, 0, 4094, 10, 2, 0, 1, 0, 0, 1, 2, 0, 1, 4093, 2, 4094, 1, 0, 0, 4082, 2, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 21, 4074, 3, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4093, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 86, 4002, 32, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4087, 4083, 3, 4, 0, 0, 4089, 0, 7, 5, 0, 4084, 0, 12, 0, 14, 2, 0, 6, 51, 0, 8, 4085, 8, 4075, 7, 5, 1, 4044, 0, 0, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4093, 4092, 2, 1, 0, 0, 4094, 0, 2, 2, 0, 4093, 0, 4, 0, 4, 2, 0, 2, 16, 0, 1, 4092, 2, 4090, 2, 0, 0, 4080, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 45, 4049, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 6, 4088, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 0, 0, 0, 84, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4082, 5, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4093, 4096, 4093, 4089, 4096, 4097, 4107] avg=4096.0000 max=4107 max_rank=7 min=4089 min_rank=4 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16389 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4083, 3, 4, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 7, 5, 0, 4084, 0, 0, 0, 0, 0, 12, 0, 14, 4082, 0, 0, 0, 2, 0, 6, 51, 4077, 5, 2, 0, 0, 8, 4085, 8, 0, 0, 0, 0, 4075, 7, 5, 1, 1, 0, 0, 0, 4044, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4086, 4096, 4108, 4143, 4101, 4089, 4055] avg=4096.0000 max=4143 max_rank=4 min=4055 min_rank=7 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 4096, 0, 0, 0, 0, 0, 0, 0, 86, 3999, 32, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4061, 3, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4117, 4096, 4090, 4066, 4096, 4097, 4107] avg=4096.0000 max=4117 max_rank=1 min=4066 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16366 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 4092, 2, 1, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 2, 2, 0, 4093, 0, 0, 0, 0, 0, 4, 0, 4, 4088, 0, 0, 0, 2, 0, 2, 16, 4083, 3, 2, 0, 0, 1, 4092, 2, 0, 0, 0, 0, 4090, 2, 0, 0, 1, 0, 0, 0, 4080, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4091, 4097, 4096, 4108, 4095, 4093, 4091] avg=4096.0000 max=4108 max_rank=4 min=4091 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 1, 0, 0, 0, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 10, 2, 0, 1, 4089, 0, 0, 0, 0, 0, 1, 2, 4084, 6, 2, 0, 0, 1, 4093, 2, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 0, 0, 0, 4082, 2, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4095, 4095, 4102, 4095, 4096, 4096, 4095] avg=4096.0000 max=4102 max_rank=3 min=4094 min_rank=0 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 4096, 0, 0, 0, 0, 0, 0, 0, 45, 4046, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4079, 3, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4099, 4096, 4090, 4084, 4096, 4097, 4107] avg=4096.0000 max=4107 max_rank=7 min=4084 min_rank=4 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 21, 4068, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 4, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 4, 0, 0, 4093, 0, 0, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4092, 4096, 4096, 4095, 4092, 4102, 4099] avg=4096.0000 max=4102 max_rank=6 min=4092 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4085, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4083, 3, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4096, 4093, 4088, 4096, 4097, 4107] avg=4096.0000 max=4107 max_rank=7 min=4088 min_rank=4 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 256, 32512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32640, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4028, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 70, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4046, 0, 0, 0, 4094, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0, 0, 50, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4089, 0, 18, 8, 4086, 4079, 4073, 4087, 2, 11, 0, 0, 11, 8, 0, 0, 10, 4084, 0, 3404, 0, 0, 0, 4078, 31, 9, 4, 663, 13, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4075, 0, 0, 0, 4092, 4092, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4096, 0, 4096, 0, 0, 21, 4096, 3, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 4096, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 11, 4096, 7, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4094, 2, 3, 1, 4094, 4090, 4092, 4094, 1, 1, 2, 1, 2, 1, 0, 0, 1, 4093, 0, 3821, 0, 0, 0, 4093, 201, 2, 1, 76, 1, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4094, 0, 6, 2, 4094, 4090, 4089, 4093, 1, 3, 0, 0, 3, 2, 0, 0, 4, 4092, 0, 3893, 0, 0, 0, 4091, 52, 2, 0, 152, 4, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4071, 0, 0, 0, 4096, 4091, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 25, 4096, 5, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4028, 0, 0, 0, 18, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 4091, 3, 2, 0, 0, 147, 0, 4094, 0, 0, 0, 0, 0, 0, 70, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4101, 4095, 4096, 4241, 4148, 3946, 4095] avg=4096.0000 max=4241 max_rank=4 min=3946 min_rank=6 imbalance_ratio=1.035400 max_load_reduction=28527 max_load_reduction_pct=0.870575 moved_to_echo=16430 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 18, 8, 18, 0, 0, 0, 0, 4079, 5, 0, 0, 0, 0, 0, 2, 11, 0, 0, 4086, 1, 0, 0, 11, 8, 0, 0, 4068, 3, 0, 0, 10, 146, 0, 3404, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 31, 9, 4, 663, 3935, 0, 0, 0, 13, 0, 0, 0, 4086, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4084, 4100, 4090, 3560, 4060, 4642, 4099] avg=4096.0000 max=4642 max_rank=6 min=3560 min_rank=4 imbalance_ratio=1.133301 max_load_reduction=28126 max_load_reduction_pct=0.858337 moved_to_echo=16361 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4046, 0, 0, 0, 18, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2, 0, 4091, 3, 1, 0, 0, 147, 0, 4096, 0, 0, 0, 0, 0, 0, 50, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4101, 4093, 4097, 4243, 4128, 3946, 4096] avg=4096.0000 max=4243 max_rank=4 min=3946 min_rank=6 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16413 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 6, 2, 18, 0, 0, 0, 0, 4090, 5, 0, 0, 0, 0, 0, 1, 3, 0, 0, 4093, 0, 0, 0, 3, 2, 0, 0, 4084, 3, 1, 0, 4, 147, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 52, 2, 0, 152, 3942, 0, 0, 0, 4, 0, 1, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4095, 4097, 4093, 4044, 4073, 4148, 4098] avg=4096.0000 max=4148 max_rank=6 min=4044 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16363 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4075, 0, 0, 0, 18, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 1, 4, 4091, 3, 1, 0, 0, 147, 0, 4096, 0, 0, 0, 0, 0, 0, 21, 4078, 0, 0, 0, 0, 3, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4097, 4092, 4100, 4243, 4099, 3949, 4095] avg=4096.0000 max=4243 max_rank=4 min=3949 min_rank=6 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16386 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4094, 2, 3, 1, 18, 0, 0, 0, 0, 4090, 5, 0, 0, 0, 0, 0, 1, 1, 2, 1, 4093, 1, 0, 0, 2, 1, 0, 0, 4087, 3, 1, 0, 1, 147, 0, 3821, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 201, 2, 1, 76, 3943, 0, 0, 0, 1, 0, 1, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4095, 4099, 4094, 3969, 4075, 4223, 4095] avg=4096.0000 max=4223 max_rank=6 min=3969 min_rank=4 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16362 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4071, 0, 0, 0, 20, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4092, 10, 1, 0, 0, 135, 0, 4096, 0, 0, 0, 0, 0, 0, 25, 4071, 0, 0, 0, 0, 5, 0, 0, 0, 3951, 5, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4095, 4096, 4103, 4231, 4096, 3961, 4095] avg=4096.0000 max=4231 max_rank=4 min=3961 min_rank=6 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16383 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 18, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4091, 3, 1, 0, 0, 147, 0, 4090, 0, 0, 0, 0, 0, 0, 11, 4078, 0, 0, 0, 0, 7, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4100, 4096, 4095, 4237, 4089, 3953, 4095] avg=4096.0000 max=4237 max_rank=4 min=3953 min_rank=6 imbalance_ratio=1.034424 max_load_reduction=28531 max_load_reduction_pct=0.870697 moved_to_echo=16374 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 1280, 0, 31616, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[384, 128, 128, 896, 31616, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 256, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4007, 89, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 0, 2, 2, 0, 4095, 2, 1, 1, 4093, 4094, 1, 0, 4093, 1, 0, 0, 0, 2, 1, 4093, 0, 4094, 0, 1, 1, 1, 0, 4094, 4094, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 0, 4088, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4041, 52, 0, 0, 0, 0, 4096, 4095, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[3, 0, 0, 12, 9, 0, 4089, 10, 30, 0, 4082, 4085, 4, 0, 4060, 11, 0, 0, 0, 10, 6, 4083, 0, 4090, 0, 4, 7, 0, 0, 4086, 4087, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 0, 3, 3, 0, 4094, 3, 7, 0, 4093, 4093, 1, 0, 4087, 3, 0, 0, 0, 2, 2, 4092, 0, 4094, 0, 1, 2, 0, 0, 4093, 4093, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4096, 0, 3984, 112, 0, 0, 0, 0, 4096, 4091, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4072, 24, 0, 0, 0, 0, 4096, 4096, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 24, 0, 0, 1, 0, 4096, 4095, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 39, 0, 4007, 0, 0, 0, 0, 89, 0, 0, 0, 4056, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4094, 4099, 4096, 4046, 4147, 4096] avg=4096.0000 max=4147 max_rank=6 min=4046 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16385 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 3, 4091, 0, 0, 0, 3, 0, 4094, 3, 0, 0, 0, 0, 7, 0, 4091, 0, 0, 0, 0, 0, 1, 0, 4087, 3, 2, 1, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 2, 39, 0, 4094, 0, 0, 0, 0, 0, 1, 2, 0, 4052, 2, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4100, 4098, 4094, 4095, 4135, 4057, 4093] avg=4096.0000 max=4135 max_rank=5 min=4057 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16380 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 12, 4084, 1, 0, 0, 9, 0, 4089, 10, 0, 0, 0, 0, 30, 0, 4080, 0, 0, 0, 0, 0, 4, 0, 4060, 11, 2, 1, 0, 0, 0, 0, 0, 10, 4085, 0, 0, 0, 6, 39, 0, 4090, 0, 0, 0, 0, 0, 4, 7, 0, 4043, 1, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4108, 4110, 4078, 4095, 4135, 4055, 4087] avg=4096.0000 max=4135 max_rank=5 min=4055 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4094, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 39, 0, 4041, 0, 0, 0, 0, 52, 0, 0, 0, 4056, 2, 0, 0, 0, 0, 4095, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4088, 4094, 4099, 4096, 4080, 4110, 4103] avg=4096.0000 max=4110 max_rank=6 min=4080 min_rank=5 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16389 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 2, 4092, 1, 0, 0, 2, 0, 4095, 2, 0, 0, 0, 0, 1, 1, 4091, 0, 0, 0, 0, 0, 1, 0, 4093, 1, 2, 1, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 1, 39, 0, 4094, 0, 0, 0, 0, 0, 1, 1, 1, 4053, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4099, 4093, 4098, 4095, 4134, 4058, 4094] avg=4096.0000 max=4134 max_rank=5 min=4058 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16381 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 39, 0, 3984, 0, 0, 0, 0, 112, 0, 0, 0, 4056, 3, 0, 0, 0, 0, 4091, 14, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4082, 4094, 4099, 4101, 4023, 4171, 4105] avg=4096.0000 max=4171 max_rank=6 min=4023 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16400 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 4093, 0, 6, 1, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 33, 0, 4072, 0, 0, 0, 0, 24, 0, 0, 0, 4061, 2, 0, 0, 0, 0, 4090, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4100, 4091, 4102, 4095, 4105, 4087, 4093] avg=4096.0000 max=4105 max_rank=5 min=4087 min_rank=6 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16380 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 39, 0, 4071, 0, 0, 0, 0, 24, 0, 0, 1, 4056, 2, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4094, 4099, 4095, 4110, 4083, 4097] avg=4096.0000 max=4110 max_rank=5 min=4083 min_rank=6 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16385 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 384, 0, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 128, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 0, 0, 0, 4096, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 4096, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[3, 0, 2, 57, 0, 2, 0, 4092, 2, 4051, 2, 2, 0, 2, 1, 0, 1, 5, 0, 2, 2, 4094, 0, 4093, 4092, 4094, 0, 4094, 0, 4075, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 37, 0, 0, 0, 4096, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[11, 0, 0, 128, 0, 8, 0, 4070, 9, 4087, 10, 10, 4, 12, 8, 0, 0, 5, 54, 11, 9, 4084, 0, 4085, 4041, 4082, 6, 4073, 0, 3958, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[3, 0, 2, 38, 0, 2, 0, 4089, 2, 4094, 3, 3, 2, 5, 2, 0, 0, 0, 16, 3, 2, 4093, 0, 4093, 4079, 4092, 0, 4083, 0, 4062, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 27, 0, 0, 0, 4096, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4055, 2, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 3973, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 120, 0, 0, 4018, 41, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4099, 4015, 4094, 4179, 4096, 4106, 4119] avg=4096.0000 max=4179 max_rank=4 min=4015 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16500 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 4055, 2, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4025, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 57, 0, 0, 4018, 40, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4099, 4067, 4094, 4115, 4097, 4106, 4119] avg=4096.0000 max=4119 max_rank=7 min=4067 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 0, 128, 4044, 1, 0, 0, 0, 8, 0, 4070, 3, 0, 0, 0, 9, 4087, 10, 10, 42, 0, 0, 0, 4, 12, 8, 0, 4081, 0, 0, 0, 0, 5, 54, 11, 3995, 40, 0, 0, 9, 0, 0, 4085, 0, 0, 0, 0, 4041, 0, 6, 10, 0, 0, 0, 0, 0, 3958, 3, 0, 23, 0, 0, 0] rerouted_tokens_per_ep_rank=[4184, 4081, 4158, 4105, 4105, 4094, 4057, 3984] avg=4096.0000 max=4184 max_rank=0 min=3984 min_rank=7 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16240 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 2, 38, 4052, 1, 0, 0, 0, 2, 0, 4089, 3, 0, 0, 0, 2, 4094, 3, 3, 42, 0, 0, 0, 2, 5, 2, 0, 4091, 0, 0, 0, 0, 0, 16, 3, 4005, 39, 0, 0, 2, 2, 0, 4093, 0, 0, 0, 0, 4079, 0, 0, 10, 0, 0, 0, 0, 0, 4062, 0, 0, 23, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 4144, 4100, 4063, 4097, 4089, 4085] avg=4096.0000 max=4144 max_rank=2 min=4063 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16334 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 37, 4055, 2, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4024, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 35, 0, 0, 4018, 40, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4099, 4066, 4094, 4093, 4097, 4106, 4119] avg=4096.0000 max=4119 max_rank=7 min=4066 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 2, 57, 4054, 2, 0, 0, 0, 2, 0, 4092, 3, 0, 0, 0, 2, 4051, 2, 2, 42, 0, 0, 0, 0, 2, 1, 0, 4092, 0, 0, 0, 1, 5, 0, 2, 4016, 40, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 4092, 0, 0, 10, 0, 0, 0, 0, 0, 4075, 0, 0, 23, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4097, 4099, 4095, 4064, 4095, 4102, 4098] avg=4096.0000 max=4118 max_rank=0 min=4064 min_rank=4 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16359 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 27, 4056, 3, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4067, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 2, 0, 0, 4019, 40, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4100, 4111, 4093, 4061, 4096, 4096, 4125] avg=4096.0000 max=4125 max_rank=7 min=4061 min_rank=4 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4056, 3, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4066, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 30, 0, 0, 4018, 40, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4099, 4108, 4093, 4088, 4096, 4106, 4119] avg=4096.0000 max=4119 max_rank=7 min=4059 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16409 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 384, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 256, 128, 128, 32128, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32640, 128, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32512, 128, 128, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 2039, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2057, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4094, 8, 2042, 3, 3, 0, 30, 1, 0, 4091, 0, 2, 0, 0, 1, 2027, 2, 2, 0, 4082, 3, 0, 0, 4093, 0, 4094, 1, 0, 2, 4093, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[15, 4086, 31, 2985, 19, 14, 0, 114, 4, 0, 4050, 0, 10, 0, 0, 6, 1065, 7, 10, 0, 3926, 15, 0, 0, 4075, 0, 4086, 9, 0, 78, 4081, 4082] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 2361, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1735, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 0, 1214, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2885, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4094, 0, 3175, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 923, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 19, 3495, 6, 4, 0, 44, 0, 0, 4087, 0, 3, 0, 2, 3, 656, 1, 3, 0, 3974, 4, 0, 0, 4090, 0, 4093, 2, 0, 5, 4092, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 2410, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1686, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[15, 1620, 31, 2985, 0, 0, 0, 0, 19, 14, 0, 114, 4057, 1, 0, 0, 4, 0, 4050, 0, 6, 0, 0, 0, 10, 0, 0, 6, 4081, 0, 0, 0, 1065, 7, 10, 0, 2432, 26, 0, 0, 3926, 15, 0, 0, 34, 0, 0, 0, 4075, 0, 3, 9, 0, 0, 0, 0, 0, 78, 4075, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4651, 4205, 4060, 4097, 3540, 3975, 4087, 4153] avg=4096.0000 max=4651 max_rank=0 min=3540 min_rank=4 imbalance_ratio=1.135498 max_load_reduction=28117 max_load_reduction_pct=0.858063 moved_to_echo=15755 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1624, 0, 2039, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 1, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2057, 0, 0, 0, 2438, 26, 0, 0, 4096, 0, 0, 0, 34, 1, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3663, 4068, 4102, 4095, 4521, 4131, 4099, 4089] avg=4096.0000 max=4521 max_rank=4 min=3663 min_rank=0 imbalance_ratio=1.103760 max_load_reduction=28247 max_load_reduction_pct=0.862030 moved_to_echo=16840 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1623, 8, 2042, 0, 0, 0, 0, 3, 3, 0, 30, 4065, 1, 0, 0, 1, 0, 4091, 0, 6, 0, 0, 0, 2, 0, 0, 1, 4093, 0, 0, 0, 2027, 2, 2, 0, 2437, 26, 0, 0, 4082, 3, 0, 0, 34, 0, 0, 0, 4093, 0, 3, 1, 0, 0, 0, 0, 0, 2, 4087, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3673, 4102, 4098, 4096, 4494, 4119, 4097, 4089] avg=4096.0000 max=4494 max_rank=4 min=3673 min_rank=0 imbalance_ratio=1.097168 max_load_reduction=28274 max_load_reduction_pct=0.862854 moved_to_echo=16799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1623, 0, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 1, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2885, 0, 0, 0, 2436, 26, 0, 0, 4096, 0, 0, 0, 34, 1, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2837, 4068, 4102, 4095, 5347, 4131, 4099, 4089] avg=4096.0000 max=5347 max_rank=4 min=2837 min_rank=0 imbalance_ratio=1.305420 max_load_reduction=27421 max_load_reduction_pct=0.836823 moved_to_echo=17666 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1623, 19, 3495, 0, 0, 0, 0, 6, 4, 0, 44, 4064, 0, 0, 0, 0, 0, 4087, 0, 6, 0, 0, 0, 3, 0, 2, 3, 4092, 0, 0, 0, 656, 1, 3, 0, 2436, 26, 0, 0, 3974, 4, 0, 0, 34, 0, 0, 0, 4090, 0, 3, 2, 0, 0, 0, 0, 0, 5, 4086, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5137, 4118, 4093, 4100, 3122, 4012, 4095, 4091] avg=4096.0000 max=5137 max_rank=0 min=3122 min_rank=4 imbalance_ratio=1.254150 max_load_reduction=27631 max_load_reduction_pct=0.843231 moved_to_echo=15320 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1623, 0, 3175, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 1, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 923, 0, 0, 0, 2437, 26, 0, 0, 4096, 0, 0, 0, 34, 1, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4798, 4068, 4102, 4095, 3386, 4131, 4099, 4089] avg=4096.0000 max=4798 max_rank=0 min=3386 min_rank=4 imbalance_ratio=1.171387 max_load_reduction=27970 max_load_reduction_pct=0.853577 moved_to_echo=15705 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1614, 0, 2361, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 1, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1735, 0, 0, 0, 2444, 26, 0, 0, 4096, 0, 0, 0, 38, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3975, 4071, 4109, 4095, 4205, 4138, 4096, 4079] avg=4096.0000 max=4205 max_rank=4 min=3975 min_rank=0 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1624, 0, 2410, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 1, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1686, 0, 0, 0, 2438, 29, 0, 0, 4096, 0, 0, 0, 34, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4068, 4102, 4095, 4153, 4131, 4096, 4089] avg=4096.0000 max=4153 max_rank=4 min=4034 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16469 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13056, 128, 128, 0, 19584, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 13056, 128, 19840, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 128, 0, 0, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 2070, 0, 708, 0, 563, 1, 536, 3554, 0, 3037, 971, 2486, 0, 3250, 2389, 2521, 73, 0, 0, 1664, 2045, 3, 0, 87, 471, 2075, 1265, 1, 2996, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2287, 4, 990, 0, 385, 0, 716, 3415, 0, 2977, 1122, 1271, 0, 3680, 3389, 2658, 156, 0, 0, 1569, 2209, 5, 0, 43, 680, 1946, 808, 4, 2454, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2, 1, 1794, 4, 504, 2, 520, 0, 696, 3411, 0, 2948, 653, 2002, 0, 3844, 3283, 2963, 85, 0, 0, 1828, 2629, 0, 0, 46, 579, 1644, 530, 22, 2778, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 2099, 1, 865, 0, 564, 2, 718, 3540, 0, 2971, 466, 1664, 0, 3735, 3521, 2988, 69, 0, 15, 1749, 2526, 2, 5, 75, 418, 1592, 634, 5, 2541, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 1895, 0, 606, 0, 624, 2, 501, 3451, 0, 2983, 647, 1444, 0, 3557, 3345, 3002, 26, 0, 0, 2316, 2394, 0, 4, 63, 732, 1638, 833, 8, 2696, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[13, 1, 1977, 1, 759, 1, 639, 0, 1233, 3472, 2, 3130, 1011, 2225, 0, 3704, 2882, 2853, 110, 0, 8, 1756, 1637, 2, 1, 51, 358, 1482, 551, 11, 2894, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2, 3, 2209, 2, 422, 0, 480, 0, 735, 3316, 1, 2985, 704, 1797, 0, 3228, 2562, 2959, 336, 3, 6, 2215, 1652, 1, 2, 57, 324, 1679, 1837, 8, 3243, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[0, 0, 2330, 0, 587, 0, 372, 0, 831, 3454, 0, 3120, 861, 2037, 0, 3452, 2905, 2672, 359, 0, 0, 1616, 1920, 0, 0, 24, 643, 2043, 649, 20, 2873, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2070, 0, 1630, 230, 0, 0, 708, 0, 563, 1, 2982, 0, 0, 0, 536, 342, 0, 3037, 0, 0, 0, 0, 971, 2486, 0, 1304, 0, 0, 0, 0, 880, 2521, 73, 0, 0, 0, 0, 0, 0, 1664, 2045, 3, 98, 0, 0, 0, 0, 87, 471, 2075, 1106, 316, 0, 0, 1265, 1, 2996, 1, 305, 0, 0, 0] rerouted_tokens_per_ep_rank=[3931, 4254, 3915, 4761, 3474, 3810, 4055, 4568] avg=4096.0000 max=4761 max_rank=3 min=3474 min_rank=4 imbalance_ratio=1.162354 max_load_reduction=28007 max_load_reduction_pct=0.854706 moved_to_echo=15907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1895, 0, 1784, 224, 0, 0, 606, 0, 624, 2, 2895, 0, 0, 0, 501, 332, 0, 2983, 0, 0, 0, 0, 647, 1444, 0, 1428, 0, 0, 0, 0, 1233, 3002, 26, 0, 0, 0, 0, 0, 0, 2316, 2394, 0, 137, 0, 0, 0, 4, 63, 732, 1638, 1548, 345, 0, 0, 833, 8, 2696, 1, 427, 0, 0, 0] rerouted_tokens_per_ep_rank=[3903, 4127, 3816, 3519, 4261, 4847, 4330, 3965] avg=4096.0000 max=4847 max_rank=5 min=3519 min_rank=3 imbalance_ratio=1.183350 max_load_reduction=27921 max_load_reduction_pct=0.852081 moved_to_echo=17403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2287, 4, 1846, 221, 0, 0, 990, 0, 385, 0, 2865, 0, 0, 0, 716, 329, 0, 2977, 0, 0, 0, 0, 1122, 1271, 0, 1477, 0, 0, 0, 0, 1250, 2658, 156, 0, 0, 0, 0, 0, 0, 1569, 2209, 5, 139, 0, 0, 0, 0, 43, 680, 1946, 1568, 357, 0, 0, 808, 4, 2454, 0, 432, 0, 0, 0] rerouted_tokens_per_ep_rank=[4358, 4240, 4022, 3870, 4064, 3922, 4594, 3698] avg=4096.0000 max=4594 max_rank=6 min=3698 min_rank=7 imbalance_ratio=1.121582 max_load_reduction=28174 max_load_reduction_pct=0.859802 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 2099, 1, 1874, 229, 0, 0, 865, 0, 564, 2, 2970, 0, 0, 0, 718, 341, 0, 2971, 0, 0, 0, 0, 466, 1664, 0, 1498, 0, 0, 0, 0, 1297, 2988, 69, 0, 0, 0, 0, 0, 15, 1749, 2526, 2, 145, 0, 0, 0, 5, 75, 418, 1592, 1630, 363, 0, 0, 634, 5, 2541, 0, 449, 0, 0, 0] rerouted_tokens_per_ep_rank=[4206, 4401, 4030, 3628, 4354, 4437, 4083, 3629] avg=4096.0000 max=4437 max_rank=5 min=3628 min_rank=3 imbalance_ratio=1.083252 max_load_reduction=28331 max_load_reduction_pct=0.864594 moved_to_echo=16503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 1794, 4, 1928, 221, 0, 0, 504, 2, 520, 0, 2862, 0, 0, 0, 696, 328, 0, 2948, 0, 0, 0, 0, 653, 2002, 0, 1543, 0, 0, 0, 0, 1210, 2963, 85, 0, 0, 0, 0, 0, 0, 1828, 2629, 0, 135, 0, 0, 0, 0, 46, 579, 1644, 1519, 373, 0, 0, 530, 22, 2778, 0, 419, 0, 0, 0] rerouted_tokens_per_ep_rank=[3950, 3888, 3972, 4198, 4258, 4592, 4161, 3749] avg=4096.0000 max=4592 max_rank=5 min=3749 min_rank=7 imbalance_ratio=1.121094 max_load_reduction=28176 max_load_reduction_pct=0.859863 moved_to_echo=16760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 2209, 2, 1624, 219, 0, 0, 422, 0, 480, 0, 2787, 0, 0, 0, 735, 310, 1, 2985, 0, 0, 0, 0, 704, 1797, 0, 1286, 0, 0, 0, 0, 932, 2959, 336, 3, 0, 0, 0, 0, 6, 2215, 1652, 1, 110, 0, 0, 0, 2, 57, 324, 1679, 1190, 318, 0, 0, 1837, 8, 3243, 0, 330, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 3689, 4031, 3787, 4230, 3984, 3570, 5418] avg=4096.0000 max=5418 max_rank=7 min=3570 min_rank=6 imbalance_ratio=1.322754 max_load_reduction=27350 max_load_reduction_pct=0.834656 moved_to_echo=17202 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[13, 1, 1977, 1, 1858, 225, 0, 0, 759, 1, 639, 0, 2913, 0, 0, 0, 1233, 334, 2, 3130, 0, 0, 0, 0, 1011, 2225, 0, 1486, 0, 0, 0, 0, 1062, 2853, 110, 0, 0, 0, 0, 0, 8, 1756, 1637, 2, 118, 0, 0, 0, 1, 51, 358, 1482, 1334, 360, 0, 0, 551, 11, 2894, 4, 368, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4312, 4699, 4722, 4025, 3521, 3586, 3828] avg=4096.0000 max=4722 max_rank=3 min=3521 min_rank=5 imbalance_ratio=1.152832 max_load_reduction=28046 max_load_reduction_pct=0.855896 moved_to_echo=14960 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2330, 0, 1732, 224, 0, 0, 587, 0, 372, 0, 2898, 0, 0, 0, 831, 332, 0, 3120, 0, 0, 0, 0, 861, 2037, 0, 1385, 0, 0, 0, 0, 1071, 2672, 359, 0, 0, 0, 0, 0, 0, 1616, 1920, 0, 119, 0, 0, 0, 0, 24, 643, 2043, 1344, 335, 0, 0, 649, 20, 2873, 0, 371, 0, 0, 0] rerouted_tokens_per_ep_rank=[4286, 3857, 4283, 4283, 4102, 3655, 4389, 3913] avg=4096.0000 max=4389 max_rank=6 min=3655 min_rank=5 imbalance_ratio=1.071533 max_load_reduction=28379 max_load_reduction_pct=0.866058 moved_to_echo=16059 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6528, 14976, 0, 11520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 16768, 128, 14336, 1920, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 512, 4224, 14208, 11264, 2816, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8960, 22656, 1280, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5504, 128, 4224, 128, 23296, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7168, 128, 22528, 128, 3200, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6016, 2688, 128, 24192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 14720, 17024, 128, 1024, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4094, 2, 3566, 0, 0, 1, 0, 0, 4093, 1, 1, 0, 0, 4092, 4, 2, 681, 0, 0, 2, 13, 0, 3, 4093, 0, 9, 3, 4087, 0, 0, 3927, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 2561, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 1725, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 3893, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 3124, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1015, 0, 0, 0, 15, 0, 0, 4096, 3, 0, 0, 4073, 0, 0, 4057, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 2706, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 1, 1493, 0, 0, 0, 28, 0, 0, 4096, 0, 0, 0, 4057, 0, 0, 4001, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4091, 0, 2470, 0, 0, 3, 0, 3, 4093, 0, 4, 0, 9, 4092, 1, 12, 1665, 0, 1, 0, 18, 0, 0, 4093, 0, 0, 3, 4081, 0, 0, 4034, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4095, 1, 2834, 1, 11, 1, 0, 0, 4095, 0, 1, 0, 0, 4094, 0, 0, 1303, 1, 0, 1, 0, 1, 0, 4095, 0, 2, 1, 4089, 2, 0, 4045, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 3343, 1, 1, 0, 0, 0, 4094, 0, 1, 0, 0, 4095, 1, 2, 813, 0, 0, 0, 7, 0, 0, 4094, 0, 0, 0, 4089, 0, 0, 4036, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 2612, 0, 14, 0, 0, 0, 4095, 1, 0, 0, 0, 4096, 0, 0, 1736, 0, 0, 0, 19, 0, 1, 4096, 0, 1, 0, 4076, 0, 0, 3830, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1195, 0, 3124, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3978, 113, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1015, 0, 0, 0, 2788, 3, 0, 0, 15, 0, 0, 4084, 0, 0, 0, 0, 3, 0, 0, 4073, 9, 2, 0, 0, 0, 0, 4057, 118, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4319, 4092, 4096, 4094, 3806, 4099, 4087, 4175] avg=4096.0000 max=4319 max_rank=0 min=3806 min_rank=4 imbalance_ratio=1.054443 max_load_reduction=28449 max_load_reduction_pct=0.868195 moved_to_echo=16167 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1194, 2, 3566, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3976, 113, 0, 0, 4093, 1, 1, 0, 0, 0, 0, 0, 0, 4090, 4, 2, 0, 0, 0, 0, 681, 0, 0, 2, 2787, 3, 0, 0, 13, 0, 3, 4081, 0, 0, 0, 0, 0, 9, 3, 4087, 9, 2, 0, 0, 0, 0, 3927, 118, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4762, 4090, 4095, 4096, 3473, 4097, 4110, 4045] avg=4096.0000 max=4762 max_rank=0 min=3473 min_rank=4 imbalance_ratio=1.162598 max_load_reduction=28006 max_load_reduction_pct=0.854675 moved_to_echo=15725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1195, 0, 2561, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3978, 113, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 1725, 0, 0, 0, 2788, 3, 0, 0, 14, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4093, 9, 2, 0, 0, 0, 0, 3893, 118, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3756, 4092, 4096, 4095, 4516, 4098, 4104, 4011] avg=4096.0000 max=4516 max_rank=4 min=3756 min_rank=0 imbalance_ratio=1.102539 max_load_reduction=28252 max_load_reduction_pct=0.862183 moved_to_echo=16729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1194, 0, 2470, 0, 0, 0, 0, 0, 0, 3, 0, 3, 3977, 113, 0, 0, 4093, 0, 4, 0, 0, 0, 0, 0, 9, 4090, 1, 12, 0, 0, 0, 0, 1665, 0, 1, 0, 2784, 3, 0, 0, 18, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 3, 4081, 9, 2, 0, 0, 0, 0, 4034, 118, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3664, 4096, 4097, 4112, 4453, 4099, 4095, 4152] avg=4096.0000 max=4453 max_rank=4 min=3664 min_rank=0 imbalance_ratio=1.087158 max_load_reduction=28315 max_load_reduction_pct=0.864105 moved_to_echo=16799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1195, 0, 2706, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3978, 113, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 0, 1493, 0, 0, 0, 2788, 3, 0, 0, 28, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4057, 9, 2, 0, 0, 0, 0, 4001, 118, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3901, 4092, 4096, 4096, 4284, 4112, 4068, 4119] avg=4096.0000 max=4284 max_rank=4 min=3901 min_rank=0 imbalance_ratio=1.045898 max_load_reduction=28484 max_load_reduction_pct=0.869263 moved_to_echo=16583 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1195, 1, 2834, 1, 0, 0, 0, 0, 11, 1, 0, 0, 3977, 113, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 1303, 1, 0, 1, 2787, 3, 0, 0, 0, 1, 0, 4083, 0, 0, 0, 0, 0, 2, 1, 4089, 9, 2, 0, 0, 2, 0, 4045, 118, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4102, 4096, 4092, 4095, 4084, 4103, 4165] avg=4096.0000 max=4165 max_rank=7 min=4031 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1184, 0, 3343, 1, 0, 0, 0, 0, 1, 0, 0, 0, 3979, 120, 0, 0, 4094, 0, 1, 0, 1, 0, 0, 0, 0, 4086, 1, 2, 0, 0, 0, 0, 813, 0, 0, 0, 2791, 10, 0, 0, 7, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 4089, 16, 8, 0, 0, 0, 0, 4036, 117, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4528, 4100, 4096, 4089, 3614, 4075, 4113, 4153] avg=4096.0000 max=4528 max_rank=0 min=3614 min_rank=4 imbalance_ratio=1.105469 max_load_reduction=28240 max_load_reduction_pct=0.861816 moved_to_echo=15955 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1195, 0, 2612, 0, 0, 0, 0, 0, 14, 0, 0, 0, 3977, 113, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1736, 0, 0, 0, 2788, 3, 0, 0, 19, 0, 1, 4084, 0, 0, 0, 0, 0, 1, 0, 4076, 9, 2, 0, 0, 0, 0, 3830, 118, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3807, 4104, 4096, 4094, 4527, 4104, 4088, 3948] avg=4096.0000 max=4527 max_rank=4 min=3807 min_rank=0 imbalance_ratio=1.105225 max_load_reduction=28241 max_load_reduction_pct=0.861847 moved_to_echo=16667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9600, 128, 23296, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[10496, 128, 128, 128, 22400, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 31872, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 31872, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 315, 460, 0, 3344, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4093, 4092, 0, 2, 4094, 1, 937, 884, 0, 2274, 4094, 0, 2, 2, 0, 4094, 2, 2, 0, 2, 4095, 0, 0, 0, 3, 0, 0, 4094, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4094, 4092, 0, 1, 4094, 1, 1888, 387, 1, 1821, 4095, 1, 2, 1, 0, 4093, 1, 1, 0, 2, 4095, 0, 0, 0, 5, 1, 0, 4088, 3, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 462, 921, 0, 2741, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4090, 0, 472, 709, 0, 2951, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 818, 573, 0, 2716, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 576, 191, 0, 3354, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2, 4073, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 1411, 251, 0, 2454, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 0, 0, 2691, 0, 315, 460, 0, 0, 0, 0, 0, 3344, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 1390, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 15, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3466, 4735, 4096, 4095, 4096, 4088, 4095] avg=4096.0000 max=4735 max_rank=2 min=3466 min_rank=1 imbalance_ratio=1.156006 max_load_reduction=28033 max_load_reduction_pct=0.855499 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 0, 0, 2691, 0, 462, 921, 0, 0, 0, 0, 0, 2741, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2697, 1390, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 15, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4074, 4128, 4096, 4087, 4095, 4096, 4095] avg=4096.0000 max=4128 max_rank=2 min=4074 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4092, 0, 1, 0, 0, 0, 0, 2690, 1, 1888, 387, 0, 0, 0, 0, 1, 1821, 1390, 1, 0, 0, 0, 0, 2, 1, 0, 4093, 0, 0, 0, 0, 1, 1, 0, 2, 2705, 1389, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 4088, 15, 0, 0, 0, 3, 0, 0, 1, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4966, 3213, 4096, 4098, 4095, 4109, 4097] avg=4096.0000 max=4966 max_rank=1 min=3213 min_rank=2 imbalance_ratio=1.212402 max_load_reduction=27802 max_load_reduction_pct=0.848450 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 0, 0, 2691, 0, 818, 573, 0, 0, 0, 0, 0, 2716, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 1390, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 15, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4082, 4107, 4096, 4095, 4096, 4100, 4095] avg=4096.0000 max=4107 max_rank=2 min=4082 min_rank=1 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2687, 0, 472, 709, 0, 0, 0, 0, 0, 2951, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 1388, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3868, 4342, 4096, 4093, 4096, 4081, 4096] avg=4096.0000 max=4342 max_rank=2 min=3868 min_rank=1 imbalance_ratio=1.060059 max_load_reduction=28426 max_load_reduction_pct=0.867493 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2680, 0, 576, 191, 0, 0, 0, 0, 0, 3354, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2710, 1396, 0, 0, 4093, 0, 0, 0, 4, 2, 0, 0, 0, 0, 2, 4073, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3447, 4734, 4094, 4106, 4099, 4091, 4101] avg=4096.0000 max=4734 max_rank=2 min=3447 min_rank=1 imbalance_ratio=1.155762 max_load_reduction=28034 max_load_reduction_pct=0.855530 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4092, 0, 2, 0, 0, 0, 0, 2690, 1, 937, 884, 0, 0, 0, 0, 0, 2274, 1390, 0, 0, 0, 0, 0, 2, 2, 0, 4094, 0, 0, 0, 0, 2, 2, 0, 2, 2704, 1389, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4094, 15, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4512, 3664, 4098, 4099, 4095, 4112, 4093] avg=4096.0000 max=4512 max_rank=1 min=3664 min_rank=2 imbalance_ratio=1.101562 max_load_reduction=28256 max_load_reduction_pct=0.862305 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2691, 0, 1411, 251, 0, 0, 0, 0, 0, 2454, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 1390, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4353, 3845, 4096, 4095, 4096, 4091, 4096] avg=4096.0000 max=4353 max_rank=1 min=3845 min_rank=2 imbalance_ratio=1.062744 max_load_reduction=28415 max_load_reduction_pct=0.867157 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 21760, 11136, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 21760, 11136, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[21632, 128, 6912, 4480, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32640, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 4096, 970, 0, 0, 0, 4096, 0, 0, 0, 0, 1672, 0, 0, 4096, 29, 0, 2, 0, 0, 4096, 0, 0, 0, 1423, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 4090, 4096, 0, 0, 0, 4094, 848, 0, 0, 0, 4096, 0, 0, 0, 0, 1790, 0, 0, 4088, 235, 0, 0, 7, 8, 4096, 0, 0, 0, 1222, 10, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4, 4094, 4094, 1, 2, 1, 4094, 1085, 0, 0, 2, 4094, 0, 1, 2, 2, 1369, 0, 2, 4094, 0, 1, 32, 0, 51, 4093, 0, 0, 0, 1559, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 4096, 1573, 0, 0, 0, 4096, 0, 0, 0, 0, 878, 0, 0, 4096, 16, 0, 2, 0, 1, 4096, 0, 0, 0, 1626, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[2, 4094, 4094, 2, 3, 1, 4093, 679, 0, 0, 2, 4093, 0, 0, 2, 2, 1295, 7, 2, 4094, 0, 0, 7, 0, 43, 4092, 0, 3, 0, 2072, 0, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 4096, 1005, 0, 0, 0, 4096, 0, 0, 0, 0, 2055, 0, 0, 4096, 87, 0, 2, 0, 4, 4096, 0, 0, 0, 943, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 4096, 1495, 0, 0, 0, 4096, 0, 0, 0, 0, 1373, 0, 0, 4095, 1, 0, 19, 1, 30, 4096, 0, 0, 0, 1178, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 4095, 4096, 0, 0, 0, 4094, 1707, 0, 0, 0, 4096, 0, 0, 0, 0, 1992, 0, 0, 4095, 11, 0, 5, 2, 27, 4096, 0, 0, 0, 356, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 2925, 848, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1790, 0, 0, 2538, 0, 0, 0, 0, 235, 0, 0, 7, 1550, 1294, 1169, 19, 8, 4077, 0, 0, 0, 0, 0, 0, 0, 1222, 10, 2794, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 3773, 4096, 4095, 4328, 4274, 4085, 4026] avg=4096.0000 max=4328 max_rank=4 min=3773 min_rank=1 imbalance_ratio=1.056641 max_load_reduction=28440 max_load_reduction_pct=0.867920 moved_to_echo=16713 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 2926, 1573, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 878, 0, 0, 2543, 0, 0, 0, 0, 16, 0, 2, 0, 1553, 1296, 1170, 19, 1, 4077, 0, 0, 0, 0, 0, 0, 0, 1626, 0, 2800, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4499, 4096, 4095, 3421, 4056, 4078, 4426] avg=4096.0000 max=4499 max_rank=1 min=3421 min_rank=4 imbalance_ratio=1.098389 max_load_reduction=28269 max_load_reduction_pct=0.862701 moved_to_echo=15981 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 4094, 1, 2, 0, 0, 0, 0, 3, 1, 2924, 679, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 2, 2, 4093, 0, 0, 0, 1295, 7, 2, 2542, 0, 0, 0, 0, 0, 0, 7, 0, 1552, 1293, 1169, 19, 43, 4073, 0, 3, 0, 0, 0, 0, 0, 2072, 0, 2793, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 3607, 4095, 4097, 3846, 4040, 4119, 4865] avg=4096.0000 max=4865 max_rank=7 min=3607 min_rank=1 imbalance_ratio=1.187744 max_load_reduction=27903 max_load_reduction_pct=0.851532 moved_to_echo=16870 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2926, 970, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1672, 0, 0, 2543, 0, 0, 0, 0, 29, 0, 2, 0, 1553, 1296, 1170, 19, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 1423, 0, 2800, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3896, 4096, 4096, 4215, 4069, 4077, 4223] avg=4096.0000 max=4223 max_rank=7 min=3896 min_rank=1 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16584 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 4094, 1, 1, 0, 0, 0, 0, 2, 1, 2925, 1085, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 1, 2, 2, 4093, 0, 0, 0, 1369, 0, 2, 2542, 0, 0, 0, 0, 0, 1, 32, 0, 1552, 1295, 1169, 19, 51, 4074, 0, 0, 0, 0, 0, 0, 0, 1559, 0, 2796, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4013, 4096, 4098, 3913, 4068, 4125, 4355] avg=4096.0000 max=4355 max_rank=7 min=3913 min_rank=4 imbalance_ratio=1.063232 max_load_reduction=28413 max_load_reduction_pct=0.867096 moved_to_echo=16461 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 2926, 1005, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2055, 0, 0, 2543, 0, 0, 0, 0, 87, 0, 2, 0, 1553, 1296, 1170, 19, 4, 4077, 0, 0, 0, 0, 0, 0, 0, 943, 0, 2800, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3931, 4096, 4095, 4598, 4127, 4081, 3743] avg=4096.0000 max=4598 max_rank=4 min=3743 min_rank=7 imbalance_ratio=1.122559 max_load_reduction=28170 max_load_reduction_pct=0.859680 moved_to_echo=16549 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2921, 1707, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1992, 0, 0, 2539, 0, 0, 0, 0, 11, 0, 5, 2, 1556, 1300, 1173, 26, 27, 4069, 0, 0, 0, 0, 0, 0, 0, 356, 0, 2796, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4628, 4097, 4096, 4531, 4073, 4096, 3152] avg=4096.0000 max=4628 max_rank=1 min=3152 min_rank=7 imbalance_ratio=1.129883 max_load_reduction=28140 max_load_reduction_pct=0.858765 moved_to_echo=15852 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2926, 1495, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1373, 0, 0, 2543, 0, 0, 0, 0, 1, 0, 19, 1, 1552, 1296, 1170, 19, 30, 4077, 0, 0, 0, 0, 0, 0, 0, 1178, 0, 2800, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4421, 4096, 4096, 3916, 4058, 4107, 3978] avg=4096.0000 max=4421 max_rank=1 min=3916 min_rank=4 imbalance_ratio=1.079346 max_load_reduction=28347 max_load_reduction_pct=0.865082 moved_to_echo=16059 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32771, 32768, 32768, 32768, 32768, 32765, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32771, 32768, 32768, 32768, 32768, 32765, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32771, 32768, 32768, 32768, 32768, 32765, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32771, 32768, 32768, 32768, 32768, 32765, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32771, 32768, 32768, 32768, 32768, 32765, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32771, 32768, 32768, 32768, 32768, 32765, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32771, 32768, 32768, 32768, 32768, 32765, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32771, 32768, 32768, 32768, 32768, 32765, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32765 actual_valid_recv=33408 observed_loss=-643 tokens_per_expert=[384, 128, 128, 128, 12544, 10368, 9472, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 10496, 128, 22400, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12544, 128, 128, 20352, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 23424, 9472, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32771 actual_valid_recv=33152 observed_loss=-381 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 32640, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4096, 0, 4088, 0, 4078, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4096, 0, 0, 4096, 32, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4092, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4096, 0, 4090, 0, 4047, 0, 4095, 0, 0, 0, 0, 0, 0, 4043, 4096, 0, 0, 4095, 0, 0, 4096, 107, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 3, 7, 4094, 1, 4094, 0, 4051, 0, 4093, 2, 2, 2, 0, 0, 2, 4084, 4092, 1, 2, 4095, 0, 0, 4093, 15, 0, 0, 0, 2, 31, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 0, 4084, 0, 4093, 0, 0, 0, 0, 1, 0, 4093, 4096, 0, 0, 4096, 0, 6, 4096, 9, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4096, 0, 4096, 0, 4039, 0, 4093, 0, 0, 0, 0, 2, 2, 4092, 4096, 0, 0, 4094, 0, 0, 4096, 57, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2, 2, 3, 0, 4094, 0, 4094, 0, 4060, 0, 4092, 2, 3, 3, 0, 0, 2, 4088, 4093, 2, 2, 4094, 0, 0, 4093, 33, 0, 0, 0, 2, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 30, 4096, 0, 4093, 0, 3959, 1, 4076, 0, 0, 0, 0, 0, 8, 4085, 4096, 0, 0, 4087, 0, 0, 4096, 141, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4043, 49, 0, 0, 4, 0, 4088, 0, 0, 0, 0, 0, 4078, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 4089, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4047, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 7, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4092, 4124, 4093, 4101, 4096, 4079, 4090] avg=4096.0000 max=4124 max_rank=2 min=4079 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4043, 49, 0, 0, 4, 0, 4092, 0, 0, 0, 0, 0, 4089, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4047, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 7, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4096, 4135, 4093, 4108, 4096, 4058, 4090] avg=4096.0000 max=4135 max_rank=2 min=4058 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 3, 0, 4039, 49, 0, 0, 3, 0, 4094, 0, 0, 0, 0, 0, 4060, 0, 46, 2, 0, 0, 0, 0, 3, 3, 0, 0, 4091, 1, 0, 0, 2, 4088, 11, 2, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 4044, 33, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4081, 7, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4097, 4108, 4098, 4103, 4096, 4077, 4094] avg=4096.0000 max=4108 max_rank=2 min=4077 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4042, 49, 0, 0, 4, 0, 4090, 0, 0, 0, 0, 0, 4047, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 4043, 12, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4047, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 7, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4094, 4093, 4093, 4055, 4095, 4154, 4090] avg=4096.0000 max=4154 max_rank=6 min=4055 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 3, 7, 4040, 49, 0, 0, 3, 1, 4094, 0, 0, 0, 0, 0, 4051, 0, 46, 2, 0, 0, 0, 0, 2, 2, 0, 0, 4091, 1, 0, 0, 2, 4084, 11, 1, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 4044, 15, 0, 0, 0, 0, 0, 0, 0, 2, 31, 0, 4080, 7, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4098, 4099, 4096, 4098, 4097, 4059, 4120] avg=4096.0000 max=4120 max_rank=7 min=4059 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4046, 51, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4039, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 8, 0, 0, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 5, 0, 0, 0, 4040, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 13, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4096, 4073, 4106, 4094, 4099, 4097, 4101] avg=4096.0000 max=4106 max_rank=3 min=4073 min_rank=2 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4040, 49, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 0, 4093, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 4047, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4084, 7, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4098, 4130, 4096, 4104, 4102, 4056, 4093] avg=4096.0000 max=4130 max_rank=2 min=4056 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 30, 4023, 49, 0, 0, 4, 0, 4093, 0, 0, 0, 0, 0, 3959, 1, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 8, 4085, 12, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4047, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 7, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4097, 4006, 4093, 4105, 4087, 4188, 4090] avg=4096.0000 max=4188 max_rank=6 min=4006 min_rank=2 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 128, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32384, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32384, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1294, 4096, 0, 0, 4096, 2802, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4091, 9, 4092, 0, 3, 0, 2, 3, 1612, 4092, 0, 0, 4085, 2482, 0, 0, 0, 6, 2, 0, 4093, 4, 4093, 0, 3, 0, 1, 4090, 3, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1529, 4096, 0, 0, 4096, 2567, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2455, 4096, 0, 0, 4096, 1641, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4092, 7, 4092, 0, 4, 0, 2, 3, 2652, 4093, 0, 0, 4088, 1442, 0, 3, 0, 2, 2, 0, 4093, 1, 4094, 0, 2, 1, 1, 4091, 2, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2277, 4096, 0, 0, 4096, 1819, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2104, 4096, 0, 0, 4096, 1992, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1754, 4096, 0, 0, 4096, 2342, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1294, 2137, 0, 0, 0, 0, 0, 0, 1962, 2802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2134, 1959, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 3431, 4764, 4094, 4096, 4096, 4096] avg=4096.0000 max=4764 max_rank=3 min=3431 min_rank=2 imbalance_ratio=1.163086 max_load_reduction=28004 max_load_reduction_pct=0.854614 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 9, 0, 0, 0, 0, 0, 0, 3, 0, 2, 4091, 0, 0, 0, 3, 1612, 2134, 0, 0, 0, 0, 0, 0, 1956, 2482, 0, 0, 0, 0, 0, 0, 0, 6, 2, 2129, 1958, 1, 0, 0, 4093, 4, 1, 0, 0, 0, 0, 0, 3, 0, 1, 4092, 0, 0, 0, 4090, 3, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4096, 3749, 4438, 4096, 4098, 4096, 4095] avg=4096.0000 max=4438 max_rank=3 min=3749 min_rank=2 imbalance_ratio=1.083496 max_load_reduction=28330 max_load_reduction_pct=0.864563 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 7, 0, 0, 0, 0, 0, 0, 4, 0, 2, 4091, 0, 0, 0, 3, 2652, 2135, 0, 0, 0, 0, 0, 0, 1958, 1442, 0, 0, 0, 0, 0, 3, 0, 2, 2, 2130, 1958, 1, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 2, 1, 1, 4094, 0, 0, 0, 4091, 2, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4097, 4790, 3400, 4096, 4094, 4098, 4094] avg=4096.0000 max=4790 max_rank=2 min=3400 min_rank=3 imbalance_ratio=1.169434 max_load_reduction=27978 max_load_reduction_pct=0.853821 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1529, 2137, 0, 0, 0, 0, 0, 0, 1962, 2567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2134, 1959, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 3666, 4529, 4093, 4096, 4096, 4096] avg=4096.0000 max=4529 max_rank=3 min=3666 min_rank=2 imbalance_ratio=1.105713 max_load_reduction=28239 max_load_reduction_pct=0.861786 moved_to_echo=16381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1754, 2137, 0, 0, 0, 0, 0, 0, 1962, 2342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2134, 1959, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 3891, 4304, 4094, 4096, 4096, 4096] avg=4096.0000 max=4304 max_rank=3 min=3891 min_rank=2 imbalance_ratio=1.050781 max_load_reduction=28464 max_load_reduction_pct=0.868652 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2277, 2137, 0, 0, 0, 0, 0, 0, 1962, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2134, 1959, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4414, 3781, 4094, 4096, 4096, 4096] avg=4096.0000 max=4414 max_rank=2 min=3781 min_rank=3 imbalance_ratio=1.077637 max_load_reduction=28354 max_load_reduction_pct=0.865295 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 2455, 2131, 0, 0, 0, 0, 0, 0, 1957, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2139, 1965, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4097, 4586, 3598, 4108, 4096, 4095, 4099] avg=4096.0000 max=4586 max_rank=2 min=3598 min_rank=3 imbalance_ratio=1.119629 max_load_reduction=28182 max_load_reduction_pct=0.860046 moved_to_echo=16398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 2104, 2137, 0, 0, 0, 0, 0, 0, 1962, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2134, 1959, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4241, 3954, 4093, 4096, 4095, 4096] avg=4096.0000 max=4241 max_rank=2 min=3954 min_rank=3 imbalance_ratio=1.035400 max_load_reduction=28527 max_load_reduction_pct=0.870575 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 17152, 15744, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 15744, 17152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 15744, 17152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3632, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 465, 0, 0, 0, 0, 4095, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3695, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 401, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 3982, 4093, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 114, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3051, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 1045, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3527, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 570, 0, 0, 0, 0, 4095, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 2, 1, 5, 3854, 4094, 0, 0, 0, 0, 3, 4088, 1, 4092, 4092, 0, 3, 4094, 243, 0, 4, 0, 0, 4093, 1, 1, 2, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 2197, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 1899, 0, 0, 0, 0, 4095, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 3, 2, 8, 3823, 4092, 0, 0, 0, 0, 2, 4087, 0, 4092, 4091, 0, 3, 4093, 267, 3, 5, 0, 1, 4093, 0, 3, 3, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 3468, 625, 0, 0, 3632, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3471, 465, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4093, 4259, 4096, 4097, 3936, 4095, 4096] avg=4096.0000 max=4259 max_rank=2 min=3936 min_rank=5 imbalance_ratio=1.039795 max_load_reduction=28509 max_load_reduction_pct=0.870026 moved_to_echo=16224 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 1, 0, 0, 4, 0, 0, 0, 3466, 624, 0, 0, 3982, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3471, 114, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 4608, 4096, 4097, 3585, 4096, 4096] avg=4096.0000 max=4608 max_rank=2 min=3585 min_rank=5 imbalance_ratio=1.125000 max_load_reduction=28160 max_load_reduction_pct=0.859375 moved_to_echo=15874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 3468, 625, 0, 0, 3695, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3471, 401, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4093, 4322, 4096, 4097, 3872, 4096, 4096] avg=4096.0000 max=4322 max_rank=2 min=3872 min_rank=5 imbalance_ratio=1.055176 max_load_reduction=28446 max_load_reduction_pct=0.868103 moved_to_echo=16161 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 3468, 625, 0, 0, 3051, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3471, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4093, 3678, 4096, 4097, 4516, 4096, 4096] avg=4096.0000 max=4516 max_rank=5 min=3678 min_rank=2 imbalance_ratio=1.102539 max_load_reduction=28252 max_load_reduction_pct=0.862183 moved_to_echo=16805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4091, 1, 0, 0, 0, 2, 1, 5, 3467, 624, 0, 0, 3854, 626, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4088, 0, 0, 0, 0, 1, 1, 4092, 0, 0, 0, 0, 0, 3, 3470, 243, 0, 0, 0, 0, 0, 4, 0, 0, 4093, 0, 0, 0, 0, 1, 1, 2, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4099, 4480, 4091, 4094, 3716, 4097, 4097] avg=4096.0000 max=4480 max_rank=2 min=3716 min_rank=5 imbalance_ratio=1.093750 max_load_reduction=28288 max_load_reduction_pct=0.863281 moved_to_echo=16004 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 3472, 628, 0, 0, 3527, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 4, 1, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3461, 570, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4100, 4148, 4108, 4097, 4031, 4094, 4092] avg=4096.0000 max=4148 max_rank=2 min=4031 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 4091, 1, 0, 0, 0, 3, 2, 8, 3465, 624, 0, 0, 3823, 626, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4087, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 3, 3469, 267, 3, 0, 0, 0, 0, 5, 0, 1, 4093, 0, 0, 0, 0, 0, 3, 3, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4102, 4449, 4089, 4092, 3742, 4099, 4099] avg=4096.0000 max=4449 max_rank=2 min=3742 min_rank=5 imbalance_ratio=1.086182 max_load_reduction=28319 max_load_reduction_pct=0.864227 moved_to_echo=16032 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 1, 0, 0, 1, 0, 0, 0, 3468, 625, 0, 0, 2197, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3471, 1899, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 2824, 4096, 4097, 5370, 4095, 4096] avg=4096.0000 max=5370 max_rank=5 min=2824 min_rank=2 imbalance_ratio=1.311035 max_load_reduction=27398 max_load_reduction_pct=0.836121 moved_to_echo=17658 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 27776, 5120, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 27776, 5120, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[27776, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3, 4, 5, 0, 4093, 4089, 0, 3, 0, 2109, 4, 0, 0, 0, 4093, 4087, 4091, 0, 3, 0, 0, 4092, 4092, 4, 0, 1988, 0, 5, 0, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 3, 4, 1, 4093, 4092, 1, 2, 0, 2976, 3, 0, 0, 0, 4093, 4086, 4093, 0, 3, 0, 0, 4092, 4093, 3, 0, 1122, 1, 4, 0, 1, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 1, 2427, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 1669, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3262, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 9, 4096, 4096, 0, 0, 826, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 12, 1698, 0, 0, 0, 0, 4084, 4096, 4096, 0, 0, 0, 23, 4096, 4094, 0, 0, 2375, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2330, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 1766, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2814, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 2, 4096, 4096, 0, 0, 1281, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2194, 0, 0, 0, 0, 4093, 4096, 4096, 0, 0, 0, 7, 4096, 4096, 0, 0, 1898, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 1, 2427, 0, 0, 1613, 4, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 1669, 0, 0, 2479, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4097, 4045, 4099, 4096, 4092, 4149, 4096] avg=4096.0000 max=4149 max_rank=6 min=4045 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 4, 5, 0, 4091, 0, 0, 0, 1, 4089, 0, 3, 0, 0, 0, 0, 0, 2109, 4, 0, 1610, 4, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 4091, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 0, 0, 1988, 0, 5, 2473, 1, 0, 0, 0, 0, 0, 3, 4092, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4093, 3727, 4097, 4094, 4092, 4467, 4095] avg=4096.0000 max=4467 max_rank=6 min=3727 min_rank=2 imbalance_ratio=1.090576 max_load_reduction=28301 max_load_reduction_pct=0.863678 moved_to_echo=16748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4, 1, 4091, 0, 0, 0, 1, 4092, 1, 2, 0, 0, 0, 0, 0, 2976, 3, 0, 1609, 4, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 1122, 1, 4, 2473, 1, 0, 0, 0, 1, 0, 2, 4092, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 4592, 4097, 4096, 4092, 3601, 4095] avg=4096.0000 max=4592 max_rank=2 min=3601 min_rank=6 imbalance_ratio=1.121094 max_load_reduction=28176 max_load_reduction_pct=0.859863 moved_to_echo=15884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 12, 1698, 0, 0, 1613, 4, 0, 0, 0, 0, 4084, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4090, 0, 0, 0, 0, 0, 0, 2375, 0, 0, 2479, 1, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4097, 3327, 4088, 4096, 4113, 4855, 4098] avg=4096.0000 max=4855 max_rank=6 min=3327 min_rank=2 imbalance_ratio=1.185303 max_load_reduction=27913 max_load_reduction_pct=0.851837 moved_to_echo=17162 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 2330, 0, 0, 1613, 4, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 1766, 0, 0, 2479, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4097, 3947, 4100, 4096, 4092, 4246, 4096] avg=4096.0000 max=4246 max_rank=6 min=3947 min_rank=2 imbalance_ratio=1.036621 max_load_reduction=28522 max_load_reduction_pct=0.870422 moved_to_echo=16530 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3262, 0, 0, 1613, 4, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4092, 0, 0, 0, 0, 0, 0, 826, 0, 0, 2479, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4879, 4099, 4096, 4101, 3307, 4096] avg=4096.0000 max=4879 max_rank=2 min=3307 min_rank=6 imbalance_ratio=1.191162 max_load_reduction=27889 max_load_reduction_pct=0.851105 moved_to_echo=15600 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2194, 0, 0, 1617, 9, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 7, 0, 4085, 0, 0, 0, 0, 0, 0, 1898, 0, 0, 2479, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 3820, 4093, 4098, 4092, 4378, 4096] avg=4096.0000 max=4378 max_rank=6 min=3820 min_rank=2 imbalance_ratio=1.068848 max_load_reduction=28390 max_load_reduction_pct=0.866394 moved_to_echo=16664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2814, 0, 0, 1613, 4, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 1281, 0, 0, 2483, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4431, 4095, 4096, 4094, 3765, 4096] avg=4096.0000 max=4431 max_rank=2 min=3765 min_rank=6 imbalance_ratio=1.081787 max_load_reduction=28337 max_load_reduction_pct=0.864777 moved_to_echo=16051 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 19840, 128, 0, 12928, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 12928, 128, 128, 19840, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 5, 0, 0, 0, 4090, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4094, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 41, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4056, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4093, 0, 4092, 1, 0, 0, 3, 3, 4092, 4091, 4088, 2, 3, 3, 0, 2, 0, 3, 0, 4, 15, 0, 3, 1, 0, 4094, 0, 1, 4093, 4081, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4092, 0, 4089, 0, 0, 0, 3, 4, 4092, 4091, 4087, 3, 4, 4, 0, 2, 0, 6, 0, 6, 19, 0, 5, 0, 0, 4094, 0, 1, 4092, 4074, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 9, 0, 0, 0, 4091, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 34, 0, 0, 0, 4077, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4072, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 20, 0, 0, 0, 4085, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4080, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4079, 3, 0, 0, 4090, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 3, 0, 0, 0, 4087, 1, 0, 0, 0, 4094, 0, 0, 10, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4087, 4097, 4095, 4094, 4091, 4104, 4102] avg=4096.0000 max=4104 max_rank=6 min=4087 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4070, 3, 0, 0, 4092, 0, 7, 3, 0, 0, 0, 0, 4, 4, 0, 2, 4090, 0, 0, 0, 0, 6, 0, 6, 4089, 1, 0, 0, 19, 0, 5, 0, 4083, 1, 0, 0, 0, 4094, 0, 1, 10, 0, 0, 0, 6, 4074, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4080, 4102, 4100, 4102, 4108, 4105, 4080] avg=4096.0000 max=4108 max_rank=5 min=4080 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16395 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 41, 0, 0, 0, 4079, 3, 0, 0, 4094, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 0, 0, 0, 4087, 2, 0, 0, 0, 4056, 0, 0, 10, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4123, 4101, 4095, 4095, 4090, 4066, 4102] avg=4096.0000 max=4123 max_rank=1 min=4066 min_rank=6 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 3, 3, 4071, 3, 0, 0, 4092, 0, 7, 2, 0, 0, 0, 0, 3, 3, 0, 2, 4090, 0, 0, 0, 0, 3, 0, 4, 4090, 1, 0, 0, 15, 0, 3, 1, 4084, 1, 0, 0, 0, 4094, 0, 1, 10, 0, 0, 0, 6, 4081, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4080, 4101, 4098, 4098, 4104, 4105, 4087] avg=4096.0000 max=4105 max_rank=6 min=4080 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 3, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 1, 0, 0, 0, 4087, 1, 0, 0, 0, 4095, 0, 0, 10, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4082, 4103, 4095, 4094, 4089, 4105, 4102] avg=4096.0000 max=4105 max_rank=6 min=4082 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16390 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4096, 0, 0, 0, 0, 0, 34, 0, 0, 0, 4079, 3, 0, 0, 4077, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 9, 0, 0, 0, 4087, 1, 0, 0, 0, 4072, 0, 0, 10, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4116, 4084, 4095, 4094, 4097, 4082, 4102] avg=4096.0000 max=4116 max_rank=1 min=4082 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4084, 8, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 7, 0, 0, 0, 4088, 1, 0, 0, 0, 4080, 0, 0, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4112, 4085, 4096, 4095, 4096, 4092, 4096] avg=4096.0000 max=4112 max_rank=1 min=4085 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4076, 3, 0, 0, 4091, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 4096, 0, 0, 13, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4088, 4095, 4094, 4096, 4093, 4109, 4097] avg=4096.0000 max=4109 max_rank=6 min=4088 min_rank=1 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16395 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4080, 0, 0, 1309, 0, 0, 0, 0, 0, 2, 2801, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 1, 0, 4096, 0, 4096, 0, 0, 4086, 0, 0, 4079, 0, 0, 59, 0, 0, 0, 1, 0, 26, 4037, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4089, 4090, 0, 5, 4090, 2, 4092, 2, 0, 4093, 0, 4, 4017, 2, 3, 633, 4, 0, 3, 0, 22, 0, 3509, 3, 4, 0, 4091, 0, 4, 0, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 0, 282, 0, 0, 0, 0, 0, 3, 3813, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4094, 0, 0, 4079, 0, 0, 86, 0, 0, 0, 0, 0, 11, 4018, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4093, 4092, 0, 3, 4093, 1, 4093, 2, 0, 4094, 0, 3, 4050, 2, 3, 688, 2, 0, 2, 0, 17, 0, 3430, 2, 4, 0, 4089, 0, 0, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 2, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 4094, 0, 0, 76, 0, 0, 0, 0, 0, 3, 4019, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 4092, 0, 0, 384, 0, 0, 0, 0, 0, 2, 3715, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3663, 0, 0, 1309, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2, 2801, 0, 416, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4097, 4972, 4095, 3220, 4096, 4096] avg=4096.0000 max=4972 max_rank=3 min=3220 min_rank=5 imbalance_ratio=1.213867 max_load_reduction=27796 max_load_reduction_pct=0.848267 moved_to_echo=15507 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 1, 0, 0, 0, 3662, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 26, 4037, 0, 416, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4087, 3721, 4096, 4480, 4095, 4096] avg=4096.0000 max=4480 max_rank=5 min=3721 min_rank=3 imbalance_ratio=1.093750 max_load_reduction=28288 max_load_reduction_pct=0.863281 moved_to_echo=16767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3675, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3, 3813, 0, 418, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4097, 3957, 4095, 4235, 4096, 4095] avg=4096.0000 max=4235 max_rank=5 min=3957 min_rank=3 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 5, 0, 0, 0, 0, 4090, 2, 1, 2, 0, 0, 0, 0, 0, 4093, 0, 4, 1, 0, 0, 0, 3606, 2, 3, 633, 0, 0, 0, 0, 4, 0, 3, 0, 4091, 0, 0, 0, 22, 0, 3509, 3, 410, 0, 0, 0, 4, 0, 4091, 0, 0, 0, 0, 0, 4, 0, 6, 0, 4089, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4098, 4244, 4098, 3944, 4095, 4099] avg=4096.0000 max=4244 max_rank=3 min=3944 min_rank=5 imbalance_ratio=1.036133 max_load_reduction=28524 max_load_reduction_pct=0.870483 moved_to_echo=16236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 3, 0, 0, 0, 0, 4093, 1, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 3, 1, 0, 0, 0, 3636, 2, 3, 688, 0, 0, 0, 0, 2, 0, 2, 0, 4092, 0, 0, 0, 17, 0, 3430, 2, 413, 0, 0, 0, 4, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4098, 4329, 4096, 3862, 4093, 4099] avg=4096.0000 max=4329 max_rank=3 min=3862 min_rank=5 imbalance_ratio=1.056885 max_load_reduction=28439 max_load_reduction_pct=0.867889 moved_to_echo=16150 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 3675, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4019, 0, 418, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4096, 3751, 4096, 4441, 4095, 4095] avg=4096.0000 max=4441 max_rank=5 min=3751 min_rank=3 imbalance_ratio=1.084229 max_load_reduction=28327 max_load_reduction_pct=0.864471 moved_to_echo=16727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 5, 0, 0, 0, 3650, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 4018, 0, 419, 2, 0, 0, 0, 0, 4096, 0, 5, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4099, 3736, 4096, 4450, 4102, 4094] avg=4096.0000 max=4450 max_rank=5 min=3736 min_rank=3 imbalance_ratio=1.086426 max_load_reduction=28318 max_load_reduction_pct=0.864197 moved_to_echo=16742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 3674, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 3715, 0, 417, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4058, 4096, 4136, 4096, 4094] avg=4096.0000 max=4136 max_rank=5 min=4058 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[29312, 128, 128, 3584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 29440, 128, 3328, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 32768, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 4096, 4092, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4, 2, 4092, 0, 2, 1, 0, 0, 4092, 1, 2, 3, 0, 4, 3, 4092, 0, 4, 2, 4092, 4089, 4070, 2, 4076, 4093, 38, 0, 0, 0, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 13, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4064, 0, 4080, 4096, 33, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 4096, 4096, 0, 4094, 4096, 1, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[6, 2, 4090, 0, 2, 1, 0, 0, 4092, 0, 3, 4, 0, 0, 4, 4091, 0, 6, 3, 4091, 4087, 4035, 2, 4071, 4092, 76, 3, 0, 0, 0, 1, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 6, 0, 0, 4096, 4096, 4096, 0, 4078, 4096, 8, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4070, 0, 4095, 4096, 26, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4096, 4096, 4096, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 4074, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 22, 2, 2] rerouted_tokens_per_ep_rank=[4094, 4097, 4096, 4096, 4095, 4115, 4075, 4100] avg=4096.0000 max=4115 max_rank=5 min=4075 min_rank=6 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 2, 4088, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4087, 1, 0, 0, 4092, 0, 3, 4, 0, 0, 0, 0, 0, 0, 4, 4091, 0, 0, 0, 0, 0, 6, 3, 4089, 0, 0, 0, 0, 0, 4035, 2, 19, 0, 0, 0, 0, 4070, 76, 3, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4051, 22, 2, 2] rerouted_tokens_per_ep_rank=[4096, 4091, 4099, 4095, 4098, 4056, 4149, 4084] avg=4096.0000 max=4149 max_rank=6 min=4056 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4064, 0, 19, 0, 0, 0, 0, 4074, 33, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 22, 2, 2] rerouted_tokens_per_ep_rank=[4107, 4097, 4096, 4096, 4094, 4083, 4109, 4086] avg=4096.0000 max=4109 max_rank=6 min=4083 min_rank=5 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16372 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 4092, 0, 19, 0, 0, 0, 0, 4074, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 22, 2, 2] rerouted_tokens_per_ep_rank=[4094, 4097, 4096, 4096, 4095, 4111, 4077, 4102] avg=4096.0000 max=4111 max_rank=5 min=4077 min_rank=6 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 2, 4090, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4089, 1, 0, 0, 4092, 1, 2, 3, 0, 0, 0, 0, 0, 4, 3, 4092, 0, 0, 0, 0, 0, 4, 2, 4090, 0, 0, 0, 0, 0, 4070, 2, 19, 0, 0, 0, 0, 4071, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4056, 22, 2, 2] rerouted_tokens_per_ep_rank=[4096, 4093, 4098, 4099, 4096, 4091, 4109, 4086] avg=4096.0000 max=4109 max_rank=6 min=4086 min_rank=7 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 4074, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 22, 2, 2] rerouted_tokens_per_ep_rank=[4097, 4097, 4096, 4097, 4100, 4115, 4082, 4084] avg=4096.0000 max=4115 max_rank=5 min=4082 min_rank=6 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 29, 9, 4] rerouted_tokens_per_ep_rank=[4089, 4099, 4096, 4096, 4096, 4108, 4067, 4117] avg=4096.0000 max=4117 max_rank=7 min=4067 min_rank=6 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4070, 0, 19, 0, 0, 0, 0, 4074, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 22, 2, 2] rerouted_tokens_per_ep_rank=[4095, 4097, 4096, 4096, 4094, 4089, 4100, 4101] avg=4096.0000 max=4101 max_rank=7 min=4089 min_rank=5 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32771, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32771, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32771, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32771, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32771, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32771, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32771, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32773, 32771, 32768, 32768, 32768, 32760] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32771 actual_valid_recv=33024 observed_loss=-253 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32640, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32760 actual_valid_recv=33408 observed_loss=-648 tokens_per_expert=[0, 0, 128, 128, 32640, 256, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32773 actual_valid_recv=33152 observed_loss=-379 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4092, 4089, 0, 2, 0, 0, 4093, 0, 3, 2, 0, 4091, 0, 5, 0, 5, 4, 1, 3, 19, 0, 3, 4091, 3, 4090, 3, 1, 0, 4075, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 45, 4051, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4, 4069, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4092, 4089, 1, 2, 0, 0, 4090, 0, 5, 2, 0, 4090, 0, 6, 0, 8, 4, 0, 3, 23, 1, 4, 4090, 4, 4085, 4, 1, 0, 4072, 0, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 10, 4074, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 6, 4093, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4092, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4094, 4096, 4095, 4089, 4096, 4097, 4103] avg=4096.0000 max=4103 max_rank=7 min=4089 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4097, 4096, 4093, 4089, 4096, 4097, 4102] avg=4096.0000 max=4102 max_rank=7 min=4089 min_rank=4 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 0, 0, 0, 45, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4096, 4093, 4090, 4096, 4097, 4103] avg=4096.0000 max=4103 max_rank=7 min=4090 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 4089, 0, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3, 2, 0, 4091, 0, 0, 0, 0, 0, 5, 0, 5, 4089, 0, 0, 0, 4, 1, 3, 19, 4085, 1, 0, 0, 0, 3, 4091, 3, 0, 0, 0, 0, 4090, 3, 1, 0, 0, 0, 0, 0, 4075, 0, 0, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4093, 4096, 4099, 4113, 4097, 4094, 4083] avg=4096.0000 max=4113 max_rank=4 min=4083 min_rank=7 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 4089, 1, 2, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 5, 2, 0, 4090, 0, 0, 0, 0, 0, 6, 0, 8, 4089, 0, 0, 0, 4, 0, 3, 23, 4084, 1, 0, 0, 1, 4, 4090, 4, 0, 0, 0, 0, 4085, 4, 1, 0, 0, 0, 0, 0, 4072, 0, 0, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4090, 4097, 4103, 4115, 4099, 4090, 4080] avg=4096.0000 max=4115 max_rank=4 min=4080 min_rank=7 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 4069, 23, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4094, 4090, 4096, 4097, 4103] avg=4096.0000 max=4103 max_rank=7 min=4090 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 4074, 12, 4, 2, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4103, 4095, 4095, 4093, 4092, 4098, 4096] avg=4096.0000 max=4103 max_rank=1 min=4092 min_rank=5 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 4092, 0, 0, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4099, 4096, 4096, 4089, 4096, 4098, 4098] avg=4096.0000 max=4099 max_rank=1 min=4089 min_rank=4 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 2, 4096, 8, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4093, 0, 8, 3, 4092, 4090, 4085, 4092, 0, 5, 0, 0, 4, 4, 0, 0, 1, 4091, 0, 3675, 0, 0, 0, 4088, 227, 2, 2, 198, 5, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4060, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4070, 0, 0, 15, 4096, 47, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4063, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 33, 4096, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4093, 0, 9, 5, 4091, 4087, 4083, 4091, 1, 6, 0, 2, 5, 3, 0, 0, 0, 4089, 0, 3792, 0, 0, 0, 4088, 49, 8, 2, 254, 6, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 4, 4096, 5, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 0, 0, 23, 4096, 8, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 7, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 100, 0, 4088, 0, 0, 0, 0, 0, 0, 2, 4089, 0, 0, 0, 0, 8, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 1, 0] rerouted_tokens_per_ep_rank=[4101, 4100, 4095, 4095, 4188, 4091, 4003, 4095] avg=4096.0000 max=4188 max_rank=4 min=4003 min_rank=6 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16377 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 8, 3, 7, 0, 0, 0, 0, 3, 4085, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4091, 0, 0, 0, 4, 4, 0, 0, 4090, 1, 0, 0, 1, 100, 0, 3675, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 227, 2, 2, 198, 3990, 0, 0, 0, 5, 0, 3, 0, 4087, 2, 1, 0] rerouted_tokens_per_ep_rank=[4111, 4088, 4096, 4099, 3776, 4081, 4419, 4098] avg=4096.0000 max=4419 max_rank=6 min=3776 min_rank=4 imbalance_ratio=1.078857 max_load_reduction=28349 max_load_reduction_pct=0.865143 moved_to_echo=16374 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 7, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 100, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 1, 0] rerouted_tokens_per_ep_rank=[4103, 4100, 4095, 4095, 4192, 4089, 3999, 4095] avg=4096.0000 max=4192 max_rank=4 min=3999 min_rank=6 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 9, 5, 7, 0, 0, 0, 0, 3, 4083, 0, 0, 0, 0, 0, 1, 6, 0, 2, 4090, 0, 0, 0, 5, 3, 0, 0, 4089, 1, 0, 0, 0, 100, 0, 3792, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 49, 8, 2, 254, 3988, 0, 0, 0, 6, 0, 4, 0, 4084, 2, 1, 0] rerouted_tokens_per_ep_rank=[4114, 4086, 4099, 4098, 3892, 4081, 4301, 4097] avg=4096.0000 max=4301 max_rank=6 min=3892 min_rank=4 imbalance_ratio=1.050049 max_load_reduction=28467 max_load_reduction_pct=0.868744 moved_to_echo=16371 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4063, 0, 0, 0, 7, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 100, 0, 4094, 0, 0, 0, 0, 0, 0, 33, 4089, 0, 0, 0, 0, 2, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 0] rerouted_tokens_per_ep_rank=[4070, 4100, 4095, 4096, 4194, 4122, 3997, 4094] avg=4096.0000 max=4194 max_rank=4 min=3997 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16407 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4060, 0, 0, 0, 7, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 100, 0, 4070, 0, 0, 0, 0, 0, 0, 15, 4089, 0, 0, 0, 0, 47, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 0] rerouted_tokens_per_ep_rank=[4067, 4100, 4096, 4095, 4170, 4104, 4042, 4094] avg=4096.0000 max=4170 max_rank=4 min=4042 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16410 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 12, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 92, 0, 4071, 0, 0, 0, 0, 0, 0, 23, 4084, 0, 0, 0, 0, 8, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 2, 0] rerouted_tokens_per_ep_rank=[4105, 4096, 4096, 4095, 4163, 4107, 4007, 4099] avg=4096.0000 max=4163 max_rank=4 min=4007 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16376 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 7, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 100, 0, 4093, 0, 0, 0, 0, 0, 0, 4, 4089, 0, 0, 0, 0, 5, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0] rerouted_tokens_per_ep_rank=[4097, 4098, 4096, 4095, 4193, 4093, 4000, 4096] avg=4096.0000 max=4193 max_rank=4 min=4000 min_rank=6 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16382 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 128, 512, 32000, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 896, 0, 32000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[3, 0, 0, 4, 4, 0, 4093, 4, 10, 2, 4090, 4092, 0, 0, 4087, 3, 0, 0, 0, 4, 2, 4088, 0, 4093, 0, 2, 3, 0, 0, 4092, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 1, 0, 0, 0, 0, 4096, 4096, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 1, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4, 0, 0, 6, 5, 0, 4093, 5, 14, 0, 4090, 4091, 0, 0, 4088, 4, 0, 0, 0, 4, 3, 4080, 0, 4092, 0, 2, 4, 0, 0, 4091, 4092, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 3, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 4, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 14, 0, 0, 0, 0, 4096, 4095, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4090, 0, 0, 0, 0, 1, 0, 0, 0, 4092, 1, 1, 0, 0, 0, 4096, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4095, 4097, 4096, 4094, 4095, 4101] avg=4096.0000 max=4101 max_rank=7 min=4094 min_rank=5 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4095, 4097, 4096, 4100, 4094, 4096] avg=4096.0000 max=4100 max_rank=5 min=4094 min_rank=6 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 6, 4090, 0, 0, 0, 5, 0, 4092, 5, 0, 0, 0, 0, 14, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 1, 0, 0, 0, 0, 0, 0, 4, 4091, 0, 0, 0, 3, 4, 0, 4092, 0, 0, 0, 0, 0, 2, 4, 0, 4076, 1, 0, 0, 0, 1, 4092, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4102, 4103, 4093, 4095, 4099, 4083, 4093] avg=4096.0000 max=4103 max_rank=2 min=4083 min_rank=6 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16370 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4, 4091, 0, 0, 0, 4, 0, 4092, 4, 0, 0, 0, 0, 10, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 4087, 3, 1, 0, 0, 0, 0, 0, 0, 4, 4092, 0, 0, 0, 2, 4, 0, 4093, 0, 0, 0, 0, 0, 2, 3, 0, 4084, 1, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4100, 4101, 4091, 4096, 4099, 4091, 4092] avg=4096.0000 max=4101 max_rank=2 min=4091 min_rank=3 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 4092, 1, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4095, 4097, 4096, 4099, 4095, 4096] avg=4096.0000 max=4099 max_rank=5 min=4095 min_rank=0 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4093, 0, 0, 0, 0, 3, 0, 0, 0, 4092, 1, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4095, 4097, 4096, 4097, 4096, 4097] avg=4096.0000 max=4097 max_rank=3 min=4095 min_rank=0 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3, 0, 4081, 0, 0, 0, 0, 14, 0, 0, 0, 4093, 7, 0, 2, 0, 0, 4095, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4091, 4089, 4099, 4097, 4084, 4116, 4097] avg=4096.0000 max=4116 max_rank=6 min=4084 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4092, 0, 0, 0, 0, 4, 0, 0, 0, 4092, 1, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4095, 4097, 4096, 4096, 4098, 4096] avg=4096.0000 max=4098 max_rank=6 min=4095 min_rank=0 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 128, 128] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[6, 0, 1, 46, 0, 3, 0, 4085, 3, 4093, 4, 4, 1, 8, 3, 1, 0, 0, 24, 5, 4, 4090, 1, 4091, 4069, 4091, 0, 4086, 0, 4047, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 56, 0, 0, 0, 4096, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 4096, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4096, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[8, 0, 1, 76, 0, 4, 0, 4082, 5, 4091, 5, 6, 2, 10, 3, 2, 0, 0, 30, 5, 4, 4089, 0, 4090, 4063, 4089, 0, 4075, 0, 4026, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 4096, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4070, 2, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4061, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 34, 0, 0, 4021, 25, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4098, 4111, 4094, 4080, 4097, 4105, 4110] avg=4096.0000 max=4111 max_rank=2 min=4073 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 56, 4070, 2, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4014, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 26, 0, 0, 4021, 26, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4098, 4064, 4094, 4073, 4096, 4105, 4110] avg=4096.0000 max=4128 max_rank=0 min=4064 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 1, 46, 4065, 2, 0, 0, 0, 3, 0, 4085, 2, 0, 0, 0, 3, 4093, 4, 4, 49, 0, 0, 0, 1, 8, 3, 1, 4089, 0, 0, 0, 0, 0, 24, 5, 4011, 25, 0, 0, 4, 4090, 1, 1, 0, 0, 0, 0, 4069, 0, 0, 10, 0, 0, 0, 0, 0, 4047, 2, 0, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4090, 4153, 4102, 4065, 4096, 4079, 4063] avg=4096.0000 max=4153 max_rank=2 min=4063 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16303 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 1, 76, 4064, 2, 0, 0, 0, 4, 0, 4082, 2, 0, 0, 0, 5, 4091, 5, 6, 49, 0, 0, 0, 2, 10, 3, 2, 4087, 0, 0, 0, 0, 0, 30, 5, 4001, 25, 0, 0, 4, 4089, 0, 1, 0, 0, 0, 0, 4063, 0, 0, 9, 0, 0, 0, 0, 0, 4026, 2, 0, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4088, 4156, 4104, 4061, 4094, 4072, 4042] avg=4096.0000 max=4156 max_rank=2 min=4042 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16269 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4070, 2, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4066, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 30, 0, 0, 4021, 25, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4098, 4116, 4094, 4076, 4097, 4105, 4110] avg=4096.0000 max=4116 max_rank=2 min=4072 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4071, 3, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4053, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 40, 0, 0, 4026, 25, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4100, 4105, 4093, 4091, 4096, 4096, 4110] avg=4096.0000 max=4110 max_rank=7 min=4077 min_rank=0 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4070, 2, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 3978, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 116, 0, 0, 4021, 26, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4098, 4028, 4094, 4163, 4096, 4105, 4110] avg=4096.0000 max=4163 max_rank=4 min=4028 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4070, 3, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 3985, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 111, 0, 0, 4022, 26, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4098, 4035, 4093, 4159, 4096, 4101, 4113] avg=4096.0000 max=4159 max_rank=4 min=4035 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16469 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32384, 128, 128, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 384, 128, 128, 32256, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1239, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2857, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[6, 4091, 17, 3441, 9, 8, 0, 56, 2, 0, 4076, 0, 5, 0, 0, 5, 614, 5, 6, 0, 4043, 8, 0, 0, 4086, 0, 4091, 4, 0, 17, 4089, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1848, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2248, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 2536, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1560, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[5, 4092, 24, 3385, 9, 6, 0, 55, 1, 0, 4082, 0, 4, 0, 0, 4, 753, 3, 4, 0, 3965, 6, 0, 0, 4088, 0, 4092, 3, 0, 6, 4091, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 2673, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1423, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 2680, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1450, 0, 0, 0, 4062, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1687, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2409, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1655, 0, 1239, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2857, 0, 0, 0, 2414, 16, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2894, 4079, 4099, 4095, 5287, 4121, 4098, 4095] avg=4096.0000 max=5287 max_rank=4 min=2894 min_rank=0 imbalance_ratio=1.290771 max_load_reduction=27481 max_load_reduction_pct=0.838654 moved_to_echo=17601 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[6, 1653, 17, 3441, 0, 0, 0, 0, 9, 8, 0, 56, 4073, 0, 0, 0, 2, 0, 4076, 0, 2, 1, 0, 0, 5, 0, 0, 5, 4089, 0, 0, 0, 614, 5, 6, 0, 2411, 16, 0, 0, 4043, 8, 0, 0, 25, 0, 0, 0, 4086, 0, 2, 4, 0, 0, 0, 0, 0, 17, 0, 4088, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5117, 4146, 4081, 4099, 3052, 4076, 4092, 4105] avg=4096.0000 max=5117 max_rank=0 min=3052 min_rank=4 imbalance_ratio=1.249268 max_load_reduction=27651 max_load_reduction_pct=0.843842 moved_to_echo=15325 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1655, 0, 1848, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2248, 0, 0, 0, 2414, 16, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3503, 4079, 4099, 4095, 4678, 4121, 4098, 4095] avg=4096.0000 max=4678 max_rank=4 min=3503 min_rank=0 imbalance_ratio=1.142090 max_load_reduction=28090 max_load_reduction_pct=0.857239 moved_to_echo=16992 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 1654, 24, 3385, 0, 0, 0, 0, 9, 6, 0, 55, 4074, 1, 0, 0, 1, 0, 4082, 0, 2, 1, 0, 0, 4, 0, 0, 4, 4090, 0, 0, 0, 753, 3, 4, 0, 2411, 16, 0, 0, 3965, 6, 0, 0, 25, 0, 0, 0, 4088, 0, 2, 3, 0, 0, 0, 0, 0, 6, 0, 4089, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5068, 4145, 4086, 4098, 3187, 3996, 4093, 4095] avg=4096.0000 max=5068 max_rank=0 min=3187 min_rank=4 imbalance_ratio=1.237305 max_load_reduction=27700 max_load_reduction_pct=0.845337 moved_to_echo=15371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1655, 0, 2536, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1560, 0, 0, 0, 2414, 16, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 4078, 4099, 4096, 3990, 4121, 4098, 4095] avg=4096.0000 max=4191 max_rank=0 min=3990 min_rank=4 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1655, 0, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1423, 0, 0, 0, 2414, 17, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4328, 4079, 4099, 4095, 3854, 4121, 4097, 4095] avg=4096.0000 max=4328 max_rank=0 min=3854 min_rank=4 imbalance_ratio=1.056641 max_load_reduction=28440 max_load_reduction_pct=0.867920 moved_to_echo=16167 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1645, 0, 1687, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 1, 0, 0, 0, 0, 4096, 0, 7, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2409, 0, 0, 0, 2415, 16, 0, 0, 4096, 0, 0, 0, 29, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3332, 4081, 4106, 4095, 4840, 4125, 4096, 4093] avg=4096.0000 max=4840 max_rank=4 min=3332 min_rank=0 imbalance_ratio=1.181641 max_load_reduction=27928 max_load_reduction_pct=0.852295 moved_to_echo=17154 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1655, 0, 2680, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 1, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1450, 0, 0, 0, 2414, 16, 0, 0, 4062, 0, 0, 0, 25, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4335, 4081, 4099, 4095, 3880, 4087, 4096, 4095] avg=4096.0000 max=4335 max_rank=0 min=3880 min_rank=4 imbalance_ratio=1.058350 max_load_reduction=28433 max_load_reduction_pct=0.867706 moved_to_echo=16158 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[13440, 128, 128, 0, 19328, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 13312, 128, 19584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 128, 0, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 32768, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True + [2026-08-30 04:55:25] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 684.9 | throughput per GPU (TFLOP/s/GPU): 486.8 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 1.009821E+01 | loss scale: 1.0 | grad norm: 7.214 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[22, 0, 849, 10, 2676, 1, 3188, 0, 1991, 1170, 0, 34, 2589, 3918, 0, 455, 18, 1057, 2863, 0, 0, 222, 755, 5, 0, 769, 3733, 2979, 3054, 216, 194, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3, 0, 482, 1, 3687, 0, 2944, 3, 2973, 458, 0, 54, 2756, 3453, 1, 684, 33, 763, 2869, 0, 0, 178, 834, 2, 0, 922, 3876, 2787, 2821, 96, 87, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[4, 0, 643, 18, 3160, 0, 2856, 3, 2379, 861, 0, 38, 2082, 3871, 0, 593, 9, 1457, 2887, 0, 0, 116, 711, 6, 0, 967, 3790, 3129, 2969, 101, 118, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[7, 0, 633, 0, 3119, 8, 3116, 4, 3035, 721, 0, 71, 2565, 3718, 0, 631, 39, 1353, 2773, 1, 0, 204, 609, 8, 1, 801, 3859, 2525, 2777, 82, 108, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[32, 0, 733, 3, 3150, 0, 2938, 0, 1867, 856, 0, 65, 2420, 3933, 0, 669, 52, 1057, 3054, 0, 0, 331, 595, 5, 0, 667, 3966, 3047, 3011, 268, 49, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[18, 1, 927, 3, 3418, 0, 2562, 0, 2682, 637, 2, 65, 2590, 3781, 1, 427, 58, 816, 3150, 0, 1, 200, 869, 18, 3, 453, 3784, 3010, 3124, 85, 83, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2, 0, 965, 0, 3098, 0, 2899, 2, 2202, 680, 0, 36, 2649, 3867, 0, 335, 17, 698, 3206, 0, 0, 170, 610, 3, 0, 701, 3942, 2942, 3465, 132, 147, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[11, 0, 878, 19, 3245, 0, 2230, 5, 2824, 660, 0, 48, 2428, 3950, 0, 529, 65, 1015, 3106, 0, 0, 431, 516, 0, 3, 551, 3776, 3178, 3063, 113, 124, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 643, 18, 3269, 0, 0, 0, 1240, 0, 2856, 3, 0, 0, 0, 0, 2379, 861, 0, 38, 786, 0, 0, 0, 2082, 1062, 0, 593, 0, 0, 0, 0, 9, 1457, 2887, 0, 43, 0, 0, 0, 0, 116, 711, 6, 2809, 290, 109, 0, 0, 967, 412, 3129, 0, 0, 0, 0, 2969, 101, 118, 0, 801, 0, 0, 0] rerouted_tokens_per_ep_rank=[3934, 4099, 4064, 3737, 4396, 4041, 4508, 3989] avg=4096.0000 max=4508 max_rank=6 min=3737 min_rank=3 imbalance_ratio=1.100586 max_load_reduction=28260 max_load_reduction_pct=0.862427 moved_to_echo=16934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 849, 10, 3220, 0, 0, 0, 1051, 1, 3188, 0, 0, 0, 0, 0, 1991, 1170, 0, 34, 665, 0, 0, 0, 2589, 1075, 0, 455, 0, 0, 0, 0, 18, 1057, 2863, 0, 36, 0, 0, 0, 0, 222, 755, 5, 2843, 246, 108, 0, 0, 769, 405, 2979, 0, 0, 0, 0, 3054, 216, 194, 0, 678, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4240, 3860, 4119, 3974, 4179, 4153, 4142] avg=4096.0000 max=4240 max_rank=1 min=3860 min_rank=2 imbalance_ratio=1.035156 max_load_reduction=28528 max_load_reduction_pct=0.870605 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 482, 1, 3343, 0, 0, 0, 1447, 0, 2944, 3, 0, 0, 0, 0, 2973, 458, 0, 54, 917, 0, 0, 0, 2756, 948, 1, 684, 0, 0, 0, 0, 33, 763, 2869, 0, 50, 0, 0, 0, 0, 178, 834, 2, 2505, 339, 112, 0, 0, 922, 421, 2787, 0, 0, 0, 0, 2821, 96, 87, 1, 934, 0, 0, 0] rerouted_tokens_per_ep_rank=[3829, 4394, 4402, 4389, 3715, 3970, 4130, 3939] avg=4096.0000 max=4402 max_rank=2 min=3715 min_rank=4 imbalance_ratio=1.074707 max_load_reduction=28366 max_load_reduction_pct=0.865662 moved_to_echo=15754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[18, 1, 927, 3, 3264, 0, 0, 0, 1341, 0, 2562, 0, 0, 0, 0, 0, 2682, 637, 2, 65, 850, 0, 0, 0, 2590, 1038, 1, 427, 0, 0, 0, 0, 58, 816, 3150, 0, 47, 0, 0, 0, 1, 200, 869, 18, 2743, 314, 109, 0, 3, 453, 411, 3010, 0, 0, 0, 0, 3124, 85, 83, 0, 866, 0, 0, 0] rerouted_tokens_per_ep_rank=[4213, 3903, 4236, 4056, 4071, 4254, 3877, 4158] avg=4096.0000 max=4254 max_rank=5 min=3877 min_rank=6 imbalance_ratio=1.038574 max_load_reduction=28514 max_load_reduction_pct=0.870178 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 633, 0, 3328, 0, 0, 0, 1224, 8, 3116, 4, 0, 0, 0, 0, 3035, 721, 0, 71, 776, 0, 0, 0, 2565, 1020, 0, 631, 0, 0, 0, 0, 39, 1353, 2773, 1, 42, 0, 0, 0, 0, 204, 609, 8, 2698, 287, 111, 0, 1, 801, 420, 2525, 0, 0, 0, 0, 2777, 82, 108, 0, 790, 0, 0, 0] rerouted_tokens_per_ep_rank=[3968, 4352, 4603, 4216, 4208, 3917, 3747, 3757] avg=4096.0000 max=4603 max_rank=2 min=3747 min_rank=6 imbalance_ratio=1.123779 max_load_reduction=28165 max_load_reduction_pct=0.859528 moved_to_echo=15629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 733, 3, 3423, 0, 0, 0, 1216, 0, 2938, 0, 0, 0, 0, 0, 1867, 856, 0, 65, 788, 0, 0, 0, 2420, 1076, 0, 669, 0, 0, 0, 0, 52, 1057, 3054, 0, 48, 0, 0, 0, 0, 331, 595, 5, 2857, 294, 117, 0, 0, 667, 426, 3047, 0, 0, 0, 0, 3011, 268, 49, 0, 804, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 4154, 3576, 4165, 4211, 4199, 4140, 4132] avg=4096.0000 max=4211 max_rank=4 min=3576 min_rank=2 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 965, 0, 3400, 0, 0, 0, 1216, 0, 2899, 2, 0, 0, 0, 0, 2202, 680, 0, 36, 770, 0, 0, 0, 2649, 1061, 0, 335, 0, 0, 0, 0, 17, 698, 3206, 0, 42, 0, 0, 0, 0, 170, 610, 3, 2806, 285, 114, 0, 0, 701, 428, 2942, 0, 0, 0, 0, 3465, 132, 147, 0, 785, 0, 0, 0] rerouted_tokens_per_ep_rank=[4367, 4117, 3688, 4045, 3963, 3988, 4071, 4529] avg=4096.0000 max=4529 max_rank=7 min=3688 min_rank=2 imbalance_ratio=1.105713 max_load_reduction=28239 max_load_reduction_pct=0.861786 moved_to_echo=16551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 878, 19, 3257, 0, 0, 0, 1274, 0, 2230, 5, 0, 0, 0, 0, 2824, 660, 0, 48, 807, 0, 0, 0, 2428, 1084, 0, 529, 0, 0, 0, 0, 65, 1015, 3106, 0, 44, 0, 0, 0, 0, 431, 516, 0, 2866, 298, 109, 0, 3, 551, 410, 3178, 0, 0, 0, 0, 3063, 113, 124, 0, 822, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 3509, 4339, 4041, 4230, 4220, 4142, 4122] avg=4096.0000 max=4339 max_rank=2 min=3509 min_rank=1 imbalance_ratio=1.059326 max_load_reduction=28429 max_load_reduction_pct=0.867584 moved_to_echo=16714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 8320, 23936, 128, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 1920, 5504, 128, 22144, 2432, 896, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[20096, 8448, 128, 4352, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19968, 6144, 128, 512, 6400, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10112, 128, 22784, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 6144, 128, 26624, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 5888, 3456, 23680, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[24320, 1152, 1024, 128, 6528, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 4095, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 4095, 0, 0, 0, 1, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 3860, 0, 0, 0, 1, 4036, 0, 0, 0, 0, 5, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 298, 4093, 0, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4093, 1, 0, 3, 4034, 0, 0, 0, 1, 4086, 0, 1, 3, 0, 16, 4093, 4094, 3, 2, 1, 4085, 0, 0, 0, 64, 4090, 0, 0, 3, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 4092, 0, 0, 0, 6, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 4093, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 8, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 36, 4058, 0, 0, 0, 2, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 12, 4096, 4096, 0, 0, 0, 4093, 0, 0, 0, 2, 4087, 0, 0, 0, 1, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4096, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 12, 4094, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 4083, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 1, 0, 1, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 13, 4081, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 37, 0, 0, 0, 1, 4036, 0, 0, 5, 3, 0, 0, 0, 0, 5, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 298, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 3897, 4045, 4095, 4096, 4096, 4351, 4094] avg=4096.0000 max=4351 max_rank=6 min=3897 min_rank=1 imbalance_ratio=1.062256 max_load_reduction=28417 max_load_reduction_pct=0.867218 moved_to_echo=16637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 37, 0, 0, 0, 2, 4094, 0, 0, 5, 3, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4132, 4104, 4090, 4096, 4098, 4055, 4096] avg=4096.0000 max=4132 max_rank=1 min=4055 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4093, 1, 0, 3, 0, 0, 0, 0, 4034, 0, 0, 0, 37, 0, 0, 0, 1, 4086, 0, 1, 5, 3, 0, 0, 3, 0, 16, 4087, 0, 0, 0, 0, 4094, 3, 2, 1, 0, 0, 0, 0, 4085, 0, 0, 0, 1, 0, 0, 0, 64, 4050, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4071, 4096, 4106, 4100, 4086, 4114, 4098] avg=4096.0000 max=4114 max_rank=6 min=4071 min_rank=1 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 37, 0, 0, 0, 0, 4091, 0, 0, 5, 3, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 3, 4052, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4133, 4099, 4090, 4096, 4097, 4055, 4102] avg=4096.0000 max=4133 max_rank=1 min=4055 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 4, 3, 0, 0, 4093, 0, 0, 0, 43, 0, 0, 0, 1, 4096, 0, 0, 8, 8, 0, 0, 0, 0, 8, 4085, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3, 5, 0, 0, 36, 4007, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4088, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4136, 4113, 4093, 4092, 4103, 4043, 4090] avg=4096.0000 max=4136 max_rank=1 min=4043 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 37, 0, 0, 0, 1, 4095, 0, 0, 5, 3, 0, 0, 0, 0, 12, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4133, 4104, 4100, 4096, 4097, 4047, 4096] avg=4096.0000 max=4133 max_rank=1 min=4047 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 37, 0, 0, 0, 1, 4096, 0, 0, 5, 3, 0, 0, 1, 0, 1, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 13, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4133, 4105, 4092, 4096, 4097, 4054, 4096] avg=4096.0000 max=4133 max_rank=1 min=4054 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 37, 0, 0, 0, 1, 4093, 0, 0, 5, 3, 0, 0, 0, 0, 12, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 2, 4047, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4133, 4102, 4102, 4096, 4094, 4049, 4096] avg=4096.0000 max=4133 max_rank=1 min=4049 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[512, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32512, 0, 0, 0, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 4096, 1, 4094, 4096, 0, 0, 4096, 0, 0, 51, 0, 0, 0, 0, 0, 0, 589, 3458, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 4096, 0, 4092, 4094, 0, 0, 4096, 0, 1, 26, 0, 0, 0, 0, 0, 0, 268, 3811, 0, 0, 0, 0, 0, 4091, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 4092, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 20, 0, 0, 0, 0, 0, 0, 1564, 2520, 0, 0, 0, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4, 4085, 4095, 0, 0, 0, 4094, 1, 4091, 4096, 0, 0, 4093, 1, 0, 56, 1, 0, 0, 0, 0, 0, 126, 3927, 3, 0, 2, 0, 1, 4092, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 180, 0, 0, 0, 0, 0, 0, 234, 3682, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 4096, 0, 4082, 4096, 0, 0, 4094, 0, 0, 39, 0, 0, 0, 0, 0, 0, 364, 3709, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 4083, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 4096, 0, 0, 22, 0, 0, 0, 0, 0, 0, 104, 3986, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 65, 0, 0, 0, 0, 0, 0, 202, 3832, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4094, 5, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 589, 3458, 49, 0, 0, 0, 0, 0, 0, 0, 4088, 6, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4097, 4099, 4091, 4096, 4096, 4094, 4097] avg=4096.0000 max=4099 max_rank=2 min=4091 min_rank=3 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16383 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 1564, 2520, 49, 0, 0, 0, 0, 0, 0, 0, 4088, 6, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4101, 4060, 4096, 4133, 4094, 4093] avg=4096.0000 max=4133 max_rank=5 min=4060 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16416 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 4040, 0, 1, 26, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 268, 3811, 49, 0, 0, 0, 0, 0, 0, 0, 4087, 6, 0, 0, 0, 4091, 0, 1, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 4096, 4067, 4096, 4128, 4093, 4093] avg=4096.0000 max=4128 max_rank=5 min=4067 min_rank=3 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16410 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 4038, 1, 0, 56, 0, 0, 0, 0, 1, 0, 0, 0, 4082, 3, 0, 0, 0, 0, 126, 3927, 48, 0, 0, 0, 3, 0, 2, 0, 4088, 6, 0, 0, 1, 4092, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4095, 4096, 4095, 4086, 4101, 4099, 4094] avg=4096.0000 max=4102 max_rank=0 min=4086 min_rank=4 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16380 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4082, 4, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 364, 3709, 48, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 4086, 4078, 4096, 4121, 4095, 4097] avg=4096.0000 max=4121 max_rank=5 min=4078 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16409 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4083, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 104, 3986, 49, 0, 0, 0, 0, 0, 0, 0, 4088, 6, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4096, 4098, 4062, 4096, 4139, 4094, 4097] avg=4096.0000 max=4139 max_rank=5 min=4062 min_rank=3 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16426 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 234, 3682, 51, 0, 0, 0, 0, 0, 0, 0, 4089, 11, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4210, 4103, 3967, 4100, 4100] avg=4096.0000 max=4210 max_rank=3 min=3967 min_rank=5 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16270 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 202, 3832, 49, 0, 0, 0, 0, 0, 0, 0, 4093, 6, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 4096, 4105, 4099, 4083, 4099, 4097] avg=4096.0000 max=4105 max_rank=3 min=4083 min_rank=5 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16378 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 3456, 28928, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32384, 128, 128, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 4081, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 11, 0, 4096, 3, 4096, 4088, 0, 0, 0, 0, 4095, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 4094, 4, 4096, 3994, 0, 0, 0, 0, 4096, 105] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 24, 0, 4095, 13, 4096, 4079, 0, 0, 0, 0, 4096, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 4080, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 4094, 12, 0, 4096, 4, 4095, 4031, 0, 0, 0, 0, 4096, 68] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 4096, 1, 4096, 4087, 0, 0, 0, 0, 4096, 16] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 4082, 4, 0, 0, 0, 0, 1, 0, 2, 0, 4093, 0, 0, 0, 0, 4092, 0, 0, 4092, 13, 0, 4096, 8, 4095, 4020, 0, 0, 1, 0, 4096, 73] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4028, 0, 0, 0, 0, 4096, 68] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 4078, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4082, 0, 0, 0, 0, 4096, 32] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4081, 3, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4049, 46, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 4085, 3, 0, 0, 0, 0, 46, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4095, 4096, 4097, 4099, 4134, 4057] avg=4096.0000 max=4134 max_rank=6 min=4057 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4049, 46, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4083, 4, 0, 0, 0, 0, 46, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 105, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4095, 4096, 4096, 4097, 4089, 4040, 4155] avg=4096.0000 max=4155 max_rank=7 min=4040 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4071, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4049, 46, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 24, 0, 4084, 13, 0, 0, 0, 0, 46, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4095, 4095, 4096, 4097, 4121, 4125, 4057] avg=4096.0000 max=4125 max_rank=6 min=4057 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4049, 46, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 4085, 1, 0, 0, 0, 0, 46, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 16, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4096, 4096, 4097, 4089, 4133, 4066] avg=4096.0000 max=4133 max_rank=6 min=4066 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 2, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4048, 46, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 12, 0, 4085, 4, 0, 0, 0, 0, 46, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 68, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4094, 4094, 4096, 4095, 4101, 4077, 4118] avg=4096.0000 max=4118 max_rank=7 min=4077 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 4, 0, 11, 0, 0, 0, 0, 0, 0, 1, 4048, 46, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 13, 0, 4085, 8, 0, 0, 0, 0, 46, 4020, 0, 0, 0, 0, 0, 0, 1, 0, 4050, 73, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4095, 4095, 4092, 4093, 4106, 4066, 4124] avg=4096.0000 max=4124 max_rank=7 min=4066 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 12, 7, 0, 0, 0, 0, 0, 0, 4056, 48, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 37, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 68, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4104, 4102, 4099, 4096, 4078, 4065, 4109] avg=4096.0000 max=4115 max_rank=0 min=4065 min_rank=6 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16348 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4078, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4049, 46, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 46, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 32, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4095, 4095, 4097, 4096, 4085, 4128, 4082] avg=4096.0000 max=4128 max_rank=6 min=4082 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[384, 32512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32512, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32512, 384, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 50, 0, 0, 0, 0, 4046, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 144, 0, 0, 0, 0, 3952, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 24, 0, 0, 0, 0, 4069, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 4095, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 2, 4094, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 43, 0, 0, 0, 0, 4052, 0, 4096, 0, 4096, 0, 4090, 4092, 0, 4095, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 55, 0, 0, 0, 0, 4041, 0, 4096, 0, 4096, 0, 4091, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 67, 0, 0, 0, 0, 4029, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 4066, 0, 4096, 0, 4095, 0, 4095, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4095, 0, 4096, 0, 4096, 0, 4096, 4095, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 50, 0, 0, 0, 4043, 0, 0, 0, 0, 4046, 0, 53, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4095, 4096, 4093, 4099, 4098, 4096] avg=4096.0000 max=4099 max_rank=5 min=4093 min_rank=4 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16386 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 144, 0, 0, 0, 4043, 0, 0, 0, 0, 3952, 0, 53, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4094, 4096, 4187, 4005, 4098, 4097] avg=4096.0000 max=4187 max_rank=4 min=4005 min_rank=5 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 24, 0, 0, 0, 4043, 0, 0, 0, 0, 4069, 0, 53, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4096, 4095, 4067, 4122, 4097, 4098] avg=4096.0000 max=4122 max_rank=5 min=4067 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 4043, 0, 0, 0, 0, 4095, 0, 53, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4095, 4096, 4045, 4148, 4098, 4095] avg=4096.0000 max=4148 max_rank=5 min=4045 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16386 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4094, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 43, 0, 0, 0, 4043, 0, 0, 0, 0, 4052, 0, 53, 0, 0, 0, 0, 0, 2, 0, 4090, 0, 0, 0, 0, 4092, 0, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 4100, 4096, 4086, 4105, 4092, 4094] avg=4096.0000 max=4105 max_rank=5 min=4086 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16377 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 55, 0, 0, 0, 4043, 0, 0, 0, 0, 4041, 0, 53, 0, 0, 0, 0, 0, 2, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4100, 4096, 4098, 4094, 4093, 4096] avg=4096.0000 max=4100 max_rank=2 min=4093 min_rank=6 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 67, 0, 0, 0, 4049, 2, 0, 0, 0, 4029, 0, 47, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4094, 4097, 4118, 4076, 4096, 4096] avg=4096.0000 max=4118 max_rank=4 min=4076 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16386 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 30, 0, 0, 0, 4043, 1, 0, 0, 0, 4066, 0, 53, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4094, 4096, 4074, 4119, 4096, 4096] avg=4096.0000 max=4119 max_rank=5 min=4074 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 0, 0, 0, 32384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32384, 0, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4082, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 14, 5, 1925, 0, 0, 0, 0, 4096, 6, 0, 4096, 0, 0, 2166, 4090, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1573, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 2523, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4085, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1838, 0, 0, 0, 0, 4096, 130, 0, 4096, 0, 0, 2140, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4089, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 8, 4, 1071, 0, 3, 0, 0, 4096, 81, 0, 4096, 0, 0, 2943, 4090, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4079, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 1453, 0, 0, 0, 0, 4096, 27, 0, 4096, 0, 0, 2631, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4090, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 1374, 0, 0, 0, 0, 4096, 61, 0, 4096, 0, 0, 2665, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2882, 0, 0, 0, 0, 4096, 37, 0, 4096, 0, 0, 1184, 4093, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4080, 0, 5, 0, 4096, 0, 0, 4088, 0, 0, 0, 13, 6, 1980, 0, 2, 0, 0, 4095, 38, 0, 4096, 0, 0, 2083, 4090, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1573, 2281, 47, 0, 0, 0, 0, 0, 0, 4086, 8, 0, 0, 4049, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4096, 4096, 3901, 4094, 4050, 4329, 4096] avg=4096.0000 max=4329 max_rank=6 min=3901 min_rank=3 imbalance_ratio=1.056885 max_load_reduction=28439 max_load_reduction_pct=0.867889 moved_to_echo=16569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1374, 2280, 47, 0, 0, 0, 0, 0, 0, 4086, 8, 0, 0, 4049, 61, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4096, 4096, 3705, 4094, 4110, 4471, 4096] avg=4096.0000 max=4471 max_rank=6 min=3705 min_rank=3 imbalance_ratio=1.091553 max_load_reduction=28297 max_load_reduction_pct=0.863556 moved_to_echo=16771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 5, 1925, 2278, 47, 0, 0, 0, 0, 0, 0, 4086, 8, 0, 0, 4049, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4096, 4096, 4269, 4094, 4055, 3970, 4096] avg=4096.0000 max=4269 max_rank=3 min=3970 min_rank=6 imbalance_ratio=1.042236 max_load_reduction=28499 max_load_reduction_pct=0.869720 moved_to_echo=16215 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 8, 4, 1071, 2278, 47, 0, 0, 0, 3, 0, 0, 4086, 8, 0, 0, 4049, 81, 0, 0, 0, 0, 0, 0, 0, 0, 2943, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 4095, 3408, 4097, 4130, 4747, 4096] avg=4096.0000 max=4747 max_rank=6 min=3408 min_rank=3 imbalance_ratio=1.158936 max_load_reduction=28021 max_load_reduction_pct=0.855133 moved_to_echo=17070 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1453, 2281, 47, 0, 0, 0, 0, 0, 0, 4086, 8, 0, 0, 4049, 27, 0, 0, 0, 0, 0, 0, 0, 0, 2631, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4096, 4096, 3784, 4094, 4076, 4437, 4096] avg=4096.0000 max=4437 max_rank=6 min=3784 min_rank=3 imbalance_ratio=1.083252 max_load_reduction=28331 max_load_reduction_pct=0.864594 moved_to_echo=16703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4085, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 4, 0, 0, 0, 0, 0, 1838, 2283, 48, 0, 0, 0, 0, 0, 0, 4091, 14, 0, 0, 4044, 130, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 1798, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4091, 4105, 4169, 4105, 4174, 3938, 4096] avg=4096.0000 max=4174 max_rank=5 min=3938 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4080, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 13, 6, 1980, 2278, 46, 0, 0, 0, 2, 0, 0, 4086, 8, 0, 0, 4049, 38, 0, 0, 0, 0, 0, 0, 0, 0, 2083, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4101, 4088, 4323, 4096, 4087, 3887, 4096] avg=4096.0000 max=4323 max_rank=3 min=3887 min_rank=6 imbalance_ratio=1.055420 max_load_reduction=28445 max_load_reduction_pct=0.868073 moved_to_echo=16166 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2882, 2280, 47, 0, 0, 0, 0, 0, 0, 4086, 8, 0, 0, 4049, 37, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4096, 4096, 5209, 4094, 4086, 2989, 4096] avg=4096.0000 max=5209 max_rank=3 min=2989 min_rank=6 imbalance_ratio=1.271729 max_load_reduction=27559 max_load_reduction_pct=0.841034 moved_to_echo=15265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 18432, 14464, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 14208, 18304, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4087, 0, 0, 3859, 0, 0, 4089, 0, 4096, 0, 253, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4084, 0, 0, 3422, 0, 0, 4090, 0, 4096, 0, 690, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4073, 0, 3, 2801, 0, 0, 4088, 0, 4096, 0, 1319, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 3992, 0, 2, 3360, 0, 0, 4085, 0, 4096, 0, 849, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 4092, 0, 4096, 3, 0, 0, 4013, 0, 22, 2724, 0, 4, 3989, 0, 4096, 1, 1533, 4096, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4075, 0, 0, 3045, 0, 2, 4063, 0, 4096, 0, 1103, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4091, 0, 1, 3816, 0, 0, 3863, 0, 4096, 0, 515, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4059, 1, 2, 2771, 0, 0, 4063, 0, 4096, 0, 1392, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4087, 36, 0, 0, 0, 0, 0, 3859, 0, 867, 0, 0, 0, 0, 4089, 0, 55, 0, 0, 0, 0, 0, 253, 3140, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4041, 53, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4123, 4726, 4144, 3393, 4096, 4095, 4094] avg=4096.0000 max=4726 max_rank=2 min=3393 min_rank=4 imbalance_ratio=1.153809 max_load_reduction=28042 max_load_reduction_pct=0.855774 moved_to_echo=15678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 36, 0, 0, 0, 0, 3, 2801, 0, 867, 0, 0, 0, 0, 4088, 0, 55, 0, 0, 0, 0, 0, 1319, 3140, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4041, 53, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4109, 3671, 4143, 4459, 4096, 4096, 4098] avg=4096.0000 max=4459 max_rank=4 min=3671 min_rank=2 imbalance_ratio=1.088623 max_load_reduction=28309 max_load_reduction_pct=0.863922 moved_to_echo=16749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4091, 36, 0, 0, 0, 0, 1, 3816, 0, 867, 0, 0, 0, 0, 3863, 0, 55, 0, 0, 0, 0, 0, 515, 3140, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 2, 4041, 53, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4127, 4684, 3918, 3655, 4096, 4095, 4096] avg=4096.0000 max=4684 max_rank=2 min=3655 min_rank=4 imbalance_ratio=1.143555 max_load_reduction=28084 max_load_reduction_pct=0.857056 moved_to_echo=15942 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 1, 0, 0, 0, 0, 3, 0, 0, 4013, 36, 0, 0, 0, 0, 22, 2724, 0, 867, 0, 0, 0, 4, 3989, 0, 55, 0, 0, 0, 0, 1, 1533, 3140, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3, 4041, 53, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4052, 3613, 4048, 4674, 4095, 4096, 4097] avg=4096.0000 max=4674 max_rank=4 min=3613 min_rank=2 imbalance_ratio=1.141113 max_load_reduction=28094 max_load_reduction_pct=0.857361 moved_to_echo=16962 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4075, 36, 0, 0, 0, 0, 0, 3045, 0, 867, 0, 0, 0, 2, 4063, 0, 55, 0, 0, 0, 0, 0, 1103, 3140, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4041, 53, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4111, 3912, 4120, 4243, 4096, 4095, 4094] avg=4096.0000 max=4243 max_rank=4 min=3912 min_rank=2 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16528 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 36, 0, 0, 0, 0, 0, 3422, 0, 867, 0, 0, 0, 0, 4090, 0, 55, 0, 0, 0, 0, 0, 690, 3140, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4041, 53, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4120, 4289, 4145, 3830, 4096, 4096, 4096] avg=4096.0000 max=4289 max_rank=2 min=3830 min_rank=4 imbalance_ratio=1.047119 max_load_reduction=28479 max_load_reduction_pct=0.869110 moved_to_echo=16118 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 39, 0, 0, 0, 0, 2, 3360, 0, 870, 0, 0, 0, 0, 4085, 0, 47, 0, 0, 0, 0, 0, 849, 3133, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4046, 53, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4031, 4232, 4132, 3982, 4097, 4099, 4099] avg=4096.0000 max=4232 max_rank=2 min=3982 min_rank=4 imbalance_ratio=1.033203 max_load_reduction=28536 max_load_reduction_pct=0.870850 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 36, 0, 0, 0, 1, 2, 2771, 0, 867, 0, 0, 0, 0, 4063, 0, 55, 0, 0, 0, 0, 0, 1392, 3140, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4041, 53, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 3641, 4118, 4532, 4096, 4096, 4094] avg=4096.0000 max=4532 max_rank=4 min=3641 min_rank=2 imbalance_ratio=1.106445 max_load_reduction=28236 max_load_reduction_pct=0.861694 moved_to_echo=16818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 7680, 25216, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 25856, 0, 7040, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 32512, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32384, 0, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32384, 512, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1984, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 2112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 2406, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 4096, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 1457, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 4096, 2636, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2332, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 4096, 1762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2228, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3374, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2010, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 2086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2491, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 1984, 1811, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2285, 1809, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2287, 2112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3795, 4096, 4094, 4096, 4399, 4096, 4096] avg=4096.0000 max=4399 max_rank=5 min=3795 min_rank=1 imbalance_ratio=1.073975 max_load_reduction=28369 max_load_reduction_pct=0.865753 moved_to_echo=16687 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 2406, 1811, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2285, 1809, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2287, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4217, 4096, 4096, 4096, 3975, 4096, 4097] avg=4096.0000 max=4217 max_rank=1 min=3975 min_rank=5 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16264 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 0, 0, 1457, 1811, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2285, 1809, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2287, 2636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 3268, 4096, 4097, 4096, 4923, 4096, 4099] avg=4096.0000 max=4923 max_rank=5 min=3268 min_rank=1 imbalance_ratio=1.201904 max_load_reduction=27845 max_load_reduction_pct=0.849762 moved_to_echo=17214 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2332, 1811, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2285, 1809, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2287, 1762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4143, 4096, 4097, 4096, 4049, 4096, 4095] avg=4096.0000 max=4143 max_rank=1 min=4049 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16336 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3374, 1811, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2285, 1809, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2287, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 5185, 4096, 4094, 4096, 3009, 4096, 4096] avg=4096.0000 max=5185 max_rank=1 min=3009 min_rank=5 imbalance_ratio=1.265869 max_load_reduction=27583 max_load_reduction_pct=0.841766 moved_to_echo=15297 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4, 0, 0, 0, 2228, 1809, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2287, 1812, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2280, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4037, 4096, 4100, 4096, 4148, 4096, 4095] avg=4096.0000 max=4148 max_rank=5 min=4037 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16435 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2491, 1811, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2285, 1809, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2287, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4302, 4096, 4095, 4096, 3892, 4096, 4095] avg=4096.0000 max=4302 max_rank=1 min=3892 min_rank=5 imbalance_ratio=1.050293 max_load_reduction=28466 max_load_reduction_pct=0.868713 moved_to_echo=16179 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2010, 1811, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2285, 1809, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2287, 2086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3821, 4096, 4095, 4096, 4373, 4096, 4095] avg=4096.0000 max=4373 max_rank=5 min=3821 min_rank=1 imbalance_ratio=1.067627 max_load_reduction=28395 max_load_reduction_pct=0.866547 moved_to_echo=16660 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 18304, 14592, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[18304, 14592, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[18304, 14592, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 1, 4096, 0, 4096, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 4, 4096, 0, 4096, 0, 0, 0, 4092, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4095, 3, 4096, 0, 4096, 0, 0, 0, 4089, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4098, 4096, 4096, 4096, 4096, 4094] avg=4096.0000 max=4098 max_rank=2 min=4094 min_rank=7 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4097, 4096, 4096, 4096, 4096, 4094] avg=4096.0000 max=4097 max_rank=0 min=4094 min_rank=7 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4096, 4094, 4096, 4096, 4096, 4096, 4094] avg=4096.0000 max=4100 max_rank=0 min=4094 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4098, 4096, 4095, 4096, 4096, 4095] avg=4096.0000 max=4098 max_rank=2 min=4095 min_rank=4 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4098, 4096, 4096, 4096, 4096, 4094] avg=4096.0000 max=4098 max_rank=2 min=4094 min_rank=7 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 5, 4092, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4091, 4096, 4096, 4096, 4098, 4097] avg=4096.0000 max=4098 max_rank=0 min=4091 min_rank=2 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 1] rerouted_tokens_per_ep_rank=[4089, 4096, 4096, 4096, 4096, 4096, 4095, 4104] avg=4096.0000 max=4104 max_rank=7 min=4089 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16391 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4097, 4096, 4095, 4096] avg=4096.0000 max=4097 max_rank=4 min=4095 min_rank=6 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 128, 128] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3796, 0, 4096, 0, 4096, 4074, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 4095, 42, 1, 0, 0, 0, 282, 4093, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3539, 0, 4096, 0, 4096, 4058, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 43, 0, 0, 0, 0, 552, 4094, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3740, 0, 4096, 0, 4096, 4061, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 368, 4091, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3503, 0, 4096, 0, 4096, 4015, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 7, 4094, 109, 5, 0, 0, 0, 563, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3779, 0, 4096, 0, 4096, 4079, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 317, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3449, 0, 4096, 0, 4096, 4058, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 41, 0, 0, 0, 0, 644, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 2701, 0, 4096, 0, 4096, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 36, 0, 0, 0, 0, 1369, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3987, 0, 4096, 0, 4096, 4082, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 114, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3740, 0, 535, 0, 0, 0, 0, 0, 32, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 3530, 524, 39, 0, 4056, 29, 0, 0, 0, 0, 0, 0, 0, 0, 368, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 31, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4275, 4093, 4096, 4093, 4085, 3935, 4095] avg=4096.0000 max=4275 max_rank=1 min=3935 min_rank=6 imbalance_ratio=1.043701 max_load_reduction=28493 max_load_reduction_pct=0.869537 moved_to_echo=16208 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3796, 0, 535, 0, 0, 0, 0, 0, 32, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 2, 3530, 524, 39, 0, 4055, 42, 1, 0, 0, 0, 0, 0, 0, 0, 282, 3569, 0, 0, 0, 0, 0, 1, 0, 0, 4064, 31, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4331, 4106, 4095, 4095, 4098, 3851, 4096] avg=4096.0000 max=4331 max_rank=1 min=3851 min_rank=6 imbalance_ratio=1.057373 max_load_reduction=28437 max_load_reduction_pct=0.867828 moved_to_echo=16140 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3539, 0, 535, 0, 0, 0, 0, 0, 32, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3530, 524, 39, 0, 4056, 43, 0, 0, 0, 0, 0, 0, 0, 0, 552, 3570, 0, 0, 0, 0, 0, 0, 2, 0, 4064, 31, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4074, 4090, 4097, 4093, 4099, 4122, 4097] avg=4096.0000 max=4122 max_rank=6 min=4074 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16411 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3503, 0, 535, 0, 0, 0, 0, 0, 32, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 7, 3530, 524, 39, 0, 4054, 109, 5, 0, 0, 0, 0, 0, 0, 0, 563, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 31, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4038, 4047, 4089, 4100, 4168, 4135, 4095] avg=4096.0000 max=4168 max_rank=5 min=4038 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16498 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3779, 0, 535, 0, 0, 0, 0, 0, 32, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3530, 524, 39, 0, 4056, 17, 0, 0, 0, 0, 0, 0, 0, 0, 317, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 31, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4314, 4111, 4097, 4093, 4073, 3889, 4095] avg=4096.0000 max=4314 max_rank=1 min=3889 min_rank=6 imbalance_ratio=1.053223 max_load_reduction=28454 max_load_reduction_pct=0.868347 moved_to_echo=16150 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 2701, 0, 535, 0, 0, 0, 0, 0, 32, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3530, 524, 39, 0, 4056, 36, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 3571, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 31, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3236, 4119, 4097, 4093, 4092, 4940, 4095] avg=4096.0000 max=4940 max_rank=6 min=3236 min_rank=1 imbalance_ratio=1.206055 max_load_reduction=27828 max_load_reduction_pct=0.849243 moved_to_echo=17220 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3449, 0, 529, 0, 0, 0, 0, 0, 30, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 3533, 530, 45, 0, 4047, 41, 0, 0, 0, 0, 0, 0, 0, 0, 644, 3566, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 34, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3978, 4088, 4100, 4108, 4088, 4210, 4100] avg=4096.0000 max=4210 max_rank=6 min=3978 min_rank=1 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16506 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3987, 0, 535, 0, 0, 0, 0, 0, 32, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3530, 524, 39, 0, 4056, 9, 0, 0, 0, 0, 0, 0, 0, 0, 114, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 31, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4522, 4114, 4097, 4093, 4065, 3686, 4095] avg=4096.0000 max=4522 max_rank=1 min=3686 min_rank=6 imbalance_ratio=1.104004 max_load_reduction=28246 max_load_reduction_pct=0.862000 moved_to_echo=15939 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 28288, 4224, 384, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 4224, 28672, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[28544, 0, 4352, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4095, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 4096, 0, 0, 4096, 4092, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4097, 4096, 4095, 4096, 4096] avg=4096.0000 max=4097 max_rank=3 min=4095 min_rank=5 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16383 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4095, 4097, 4096] avg=4096.0000 max=4097 max_rank=6 min=4095 min_rank=5 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4095, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4097 max_rank=0 min=4095 min_rank=2 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4097, 4091, 4096, 4102, 4094, 4096] avg=4096.0000 max=4102 max_rank=5 min=4091 min_rank=3 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4096, 4100, 4096, 4092, 4097, 4096] avg=4096.0000 max=4100 max_rank=3 min=4092 min_rank=5 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=3 active_echo_slots_local=3 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4090, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 6, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 3899, 4096, 4096, 0, 0, 197, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4082, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4073, 4096, 4096, 0, 0, 23, 0, 14, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 0, 0, 4095, 1, 4096, 0, 0, 3, 0, 4087, 4096, 4096, 0, 0, 9, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 28, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4099, 4096, 4096, 4125, 4068, 4092] avg=4096.0000 max=4125 max_rank=5 min=4068 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 3899, 29, 0, 0, 0, 0, 4068, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 28, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4098, 4096, 4096, 3929, 4265, 4092] avg=4096.0000 max=4265 max_rank=6 min=3929 min_rank=5 imbalance_ratio=1.041260 max_load_reduction=28503 max_load_reduction_pct=0.869843 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4064, 28, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4093, 4096, 4096, 4125, 4068, 4098] avg=4096.0000 max=4125 max_rank=5 min=4068 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4064, 28, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4099, 4095, 4096, 4125, 4068, 4093] avg=4096.0000 max=4125 max_rank=5 min=4068 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 29, 0, 0, 0, 0, 4068, 0, 0, 23, 0, 0, 0, 0, 0, 14, 0, 0, 4064, 28, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4085, 4096, 4096, 4102, 4091, 4106] avg=4096.0000 max=4106 max_rank=7 min=4085 min_rank=2 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16395 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 32, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4099, 4098, 4095, 4118, 4063, 4103] avg=4096.0000 max=4118 max_rank=5 min=4063 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16379 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 28, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4099, 4096, 4096, 4125, 4068, 4092] avg=4096.0000 max=4125 max_rank=5 min=4068 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 4087, 29, 0, 0, 0, 0, 4068, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 28, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4095, 4097, 4119, 4077, 4092] avg=4096.0000 max=4119 max_rank=5 min=4077 min_rank=6 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32640, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32640, 0, 0, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4097, 4096, 4096, 4095, 4096, 4096] avg=4096.0000 max=4097 max_rank=2 min=4095 min_rank=5 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16383 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4097, 4096, 4096, 4095, 4096, 4096] avg=4096.0000 max=4097 max_rank=2 min=4095 min_rank=5 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16383 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4094, 4096, 4096, 4098, 4096, 4096] avg=4096.0000 max=4098 max_rank=5 min=4094 min_rank=2 imbalance_ratio=1.000488 max_load_reduction=28670 max_load_reduction_pct=0.874939 moved_to_echo=16386 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4096 max_rank=0 min=4096 min_rank=0 imbalance_ratio=1.000000 max_load_reduction=28672 max_load_reduction_pct=0.875000 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 36, 0, 4092, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4064, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 0, 4096, 0, 0, 0, 0, 4096, 4095, 4096, 0, 4075, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 4084, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4093, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 153, 0, 4086, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3948, 0, 8, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 72, 0, 4091, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4028, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 36, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4060, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 58, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4038, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 147, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3949, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 65, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 62, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4066, 4096, 4129, 4093] avg=4096.0000 max=4129 max_rank=6 min=4066 min_rank=4 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 65, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 63, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4055, 4096, 4140, 4093] avg=4096.0000 max=4140 max_rank=6 min=4055 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 153, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 65, 0, 0, 3948, 0, 0, 0, 0, 0, 8, 0, 0, 4031, 62, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4093, 4177, 4096, 4013, 4101] avg=4096.0000 max=4177 max_rank=4 min=4013 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16387 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 72, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 65, 0, 0, 4028, 0, 0, 0, 0, 0, 2, 0, 0, 4031, 62, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4095, 4101, 4096, 4093, 4095] avg=4096.0000 max=4101 max_rank=4 min=4093 min_rank=6 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16385 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 65, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 62, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4037, 4096, 4158, 4093] avg=4096.0000 max=4158 max_rank=6 min=4037 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 147, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 65, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 63, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4180, 4096, 4014, 4094] avg=4096.0000 max=4180 max_rank=4 min=4014 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 36, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 58, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 67, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4100, 4061, 4096, 4118, 4105] avg=4096.0000 max=4118 max_rank=6 min=4061 min_rank=4 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16380 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 58, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 65, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 63, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4096, 4091, 4096, 4103, 4094] avg=4096.0000 max=4103 max_rank=6 min=4091 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16384 active_clone_links=4 active_echo_slots_local=4 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[640, 0, 0, 32256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[640, 0, 32256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 32256, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4063, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4089, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4069, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4094, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4066, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4094, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4054, 4095, 4096, 0, 0, 12, 1, 0, 0, 0, 35, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 4089, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4080, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4069, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 27, 4076, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4094, 4104, 4096, 4096, 4095, 4096, 4098] avg=4096.0000 max=4104 max_rank=2 min=4089 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 7, 4076, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4093, 4084, 4096, 4096, 4096, 4096, 4098] avg=4096.0000 max=4109 max_rank=0 min=4084 min_rank=2 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4063, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 33, 4076, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4096, 4110, 4096, 4096, 4095, 4096, 4096] avg=4096.0000 max=4110 max_rank=2 min=4083 min_rank=0 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 2, 4076, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4094, 4080, 4096, 4096, 4095, 4096, 4097] avg=4096.0000 max=4114 max_rank=0 min=4080 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4066, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 30, 4076, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4093, 4107, 4096, 4096, 4098, 4096, 4096] avg=4096.0000 max=4107 max_rank=2 min=4086 min_rank=0 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4054, 20, 0, 0, 0, 0, 0, 0, 0, 12, 1, 4093, 0, 0, 0, 0, 0, 0, 35, 4075, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4106, 4111, 4095, 4096, 4099, 4096, 4091] avg=4096.0000 max=4111 max_rank=2 min=4074 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 16, 4078, 1, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4095, 4097, 4096, 4095, 4096, 4096] avg=4096.0000 max=4097 max_rank=0 min=4095 min_rank=2 imbalance_ratio=1.000244 max_load_reduction=28671 max_load_reduction_pct=0.874969 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4076, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4096, 4077, 4096, 4096, 4095, 4096, 4096] avg=4096.0000 max=4116 max_rank=0 min=4077 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32640, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32768 slack_slots=32768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[8, 1, 535, 3, 2985, 0, 3423, 0, 2727, 717, 3, 84, 2624, 3795, 1, 475, 15, 1188, 2678, 0, 0, 171, 529, 2, 0, 801, 3775, 2627, 3299, 91, 211, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1, 0, 697, 0, 2754, 0, 3034, 0, 2589, 1129, 0, 31, 2995, 3933, 0, 399, 5, 1286, 2173, 0, 0, 109, 544, 0, 0, 439, 3865, 3080, 3422, 162, 121, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[6, 0, 394, 0, 2683, 1, 3247, 1, 2601, 1454, 0, 26, 2762, 3875, 0, 483, 9, 692, 2761, 0, 0, 83, 510, 1, 0, 505, 3859, 2715, 3555, 425, 120, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[7, 3, 748, 0, 2487, 1, 3422, 0, 2073, 1107, 0, 29, 2475, 3883, 0, 460, 35, 1210, 2481, 1, 0, 93, 689, 2, 0, 924, 3898, 2881, 3402, 234, 222, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[11, 0, 773, 0, 3148, 0, 2789, 1, 2360, 766, 0, 63, 2856, 3838, 0, 468, 52, 1313, 2876, 0, 0, 108, 382, 0, 0, 1020, 3933, 2867, 2984, 106, 54, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[13, 3, 568, 3, 3217, 4, 3206, 3, 2621, 589, 0, 45, 2535, 3867, 5, 545, 44, 1204, 2689, 1, 0, 242, 863, 14, 2, 863, 3829, 2719, 2821, 95, 153, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2, 5, 999, 2, 3344, 6, 3154, 2, 2854, 644, 3, 36, 2176, 3793, 17, 827, 13, 1220, 3189, 0, 0, 129, 696, 12, 3, 588, 3663, 2619, 2616, 103, 51, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[35, 0, 1097, 1, 2092, 0, 3217, 9, 2369, 765, 0, 49, 2962, 3755, 0, 662, 10, 1048, 2693, 0, 0, 378, 816, 9, 0, 610, 3898, 2770, 3232, 155, 136, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[8, 1, 535, 3, 3193, 107, 0, 0, 2985, 0, 1349, 0, 0, 0, 0, 0, 2727, 717, 3, 84, 675, 0, 0, 0, 2624, 870, 1, 475, 0, 0, 0, 0, 15, 1188, 2678, 0, 252, 0, 0, 0, 0, 171, 529, 2, 2818, 477, 0, 0, 0, 801, 582, 2627, 0, 0, 0, 0, 3299, 91, 211, 0, 670, 0, 0, 0] rerouted_tokens_per_ep_rank=[3847, 4334, 4206, 3970, 4133, 3997, 4010, 4271] avg=4096.0000 max=4334 max_rank=1 min=3847 min_rank=0 imbalance_ratio=1.058105 max_load_reduction=28434 max_load_reduction_pct=0.867737 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 697, 0, 3269, 111, 0, 0, 2754, 0, 1196, 0, 0, 0, 0, 0, 2589, 1129, 0, 31, 599, 0, 0, 0, 2995, 901, 0, 399, 0, 0, 0, 0, 5, 1286, 2173, 0, 223, 0, 0, 0, 0, 109, 544, 0, 2921, 423, 0, 0, 0, 439, 596, 3080, 0, 0, 0, 0, 3422, 162, 121, 0, 593, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 3950, 4348, 4295, 3687, 3997, 4115, 4298] avg=4096.0000 max=4348 max_rank=2 min=3687 min_rank=4 imbalance_ratio=1.061523 max_load_reduction=28420 max_load_reduction_pct=0.867310 moved_to_echo=16097 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 394, 0, 3264, 109, 0, 0, 2683, 1, 1280, 1, 0, 0, 0, 0, 2601, 1454, 0, 26, 641, 0, 0, 0, 2762, 889, 0, 483, 0, 0, 0, 0, 9, 692, 2761, 0, 239, 0, 0, 0, 0, 83, 510, 1, 2877, 452, 0, 0, 0, 505, 595, 2715, 0, 0, 0, 0, 3555, 425, 120, 0, 635, 0, 0, 0] rerouted_tokens_per_ep_rank=[3773, 3965, 4722, 4134, 3701, 3923, 3815, 4735] avg=4096.0000 max=4735 max_rank=7 min=3701 min_rank=4 imbalance_ratio=1.156006 max_load_reduction=28033 max_load_reduction_pct=0.855499 moved_to_echo=16174 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 3, 748, 0, 3297, 109, 0, 0, 2487, 1, 1349, 0, 0, 0, 0, 0, 2073, 1107, 0, 29, 675, 0, 0, 0, 2475, 891, 0, 460, 0, 0, 0, 0, 35, 1210, 2481, 1, 252, 0, 0, 0, 0, 93, 689, 2, 2883, 477, 0, 0, 0, 924, 601, 2881, 0, 0, 0, 0, 3402, 234, 222, 1, 669, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 3837, 3884, 3826, 3979, 4144, 4406, 4528] avg=4096.0000 max=4528 max_rank=7 min=3826 min_rank=3 imbalance_ratio=1.105469 max_load_reduction=28240 max_load_reduction_pct=0.861816 moved_to_echo=17057 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[13, 3, 568, 3, 3238, 109, 0, 0, 3217, 4, 1263, 3, 0, 0, 0, 0, 2621, 589, 0, 45, 633, 0, 0, 0, 2535, 886, 5, 545, 0, 0, 0, 0, 44, 1204, 2689, 1, 236, 0, 0, 0, 0, 242, 863, 14, 2872, 447, 0, 0, 2, 863, 591, 2719, 0, 0, 0, 0, 2821, 95, 153, 5, 627, 0, 0, 0] rerouted_tokens_per_ep_rank=[3934, 4487, 3888, 3971, 4174, 4438, 4175, 3701] avg=4096.0000 max=4487 max_rank=1 min=3701 min_rank=7 imbalance_ratio=1.095459 max_load_reduction=28281 max_load_reduction_pct=0.863068 moved_to_echo=16488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 773, 0, 3326, 108, 0, 0, 3148, 0, 1100, 1, 0, 0, 0, 0, 2360, 766, 0, 63, 550, 0, 0, 0, 2856, 880, 0, 468, 0, 0, 0, 0, 52, 1313, 2876, 0, 205, 0, 0, 0, 0, 108, 382, 0, 2850, 389, 0, 0, 0, 1020, 607, 2867, 0, 0, 0, 0, 2984, 106, 54, 0, 545, 0, 0, 0] rerouted_tokens_per_ep_rank=[4218, 4249, 3739, 4204, 4446, 3729, 4494, 3689] avg=4096.0000 max=4494 max_rank=6 min=3689 min_rank=7 imbalance_ratio=1.097168 max_load_reduction=28274 max_load_reduction_pct=0.862854 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[35, 0, 1097, 1, 3300, 108, 0, 0, 2092, 0, 1249, 9, 0, 0, 0, 0, 2369, 765, 0, 49, 639, 0, 0, 0, 2962, 855, 0, 662, 0, 0, 0, 0, 10, 1048, 2693, 0, 243, 0, 0, 0, 0, 378, 816, 9, 2792, 452, 0, 0, 0, 610, 598, 2770, 0, 0, 0, 0, 3232, 155, 136, 0, 634, 0, 0, 0] rerouted_tokens_per_ep_rank=[4541, 3350, 3822, 4479, 3994, 4447, 3978, 4157] avg=4096.0000 max=4541 max_rank=0 min=3350 min_rank=1 imbalance_ratio=1.108643 max_load_reduction=28227 max_load_reduction_pct=0.861420 moved_to_echo=16576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 5, 999, 2, 3098, 107, 0, 0, 3344, 6, 1244, 2, 0, 0, 0, 0, 2854, 644, 3, 36, 622, 0, 0, 0, 2176, 869, 17, 827, 0, 0, 0, 0, 13, 1220, 3189, 0, 232, 0, 0, 0, 0, 129, 696, 12, 2817, 439, 0, 0, 3, 588, 565, 2619, 0, 0, 0, 0, 2616, 103, 51, 2, 617, 0, 0, 0] rerouted_tokens_per_ep_rank=[4213, 4596, 4159, 3889, 4654, 4093, 3775, 3389] avg=4096.0000 max=4654 max_rank=4 min=3389 min_rank=7 imbalance_ratio=1.136230 max_load_reduction=28114 max_load_reduction_pct=0.857971 moved_to_echo=15911 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 9216, 21632, 128, 1920, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 1408, 5120, 128, 22912, 3584, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[21504, 7168, 128, 4352, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 5760, 4736, 22400, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[20224, 7296, 128, 384, 5120, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[25344, 1408, 1152, 128, 4992, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 5888, 128, 26112, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[22784, 128, 10112, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 4096, 0, 0, 0, 0, 4083, 0, 0, 4, 0, 30, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 1, 4074, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4095, 1, 1, 0, 4095, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 4095, 0, 1, 0, 4093, 0, 0, 0, 6, 4095, 0, 0, 1, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4072, 2, 2, 0, 4079, 3, 1, 0, 0, 4085, 0, 0, 1, 8, 11, 4088, 4084, 3, 0, 1, 4093, 1, 5, 0, 30, 4095, 8, 2, 1, 8, 0, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 4096, 0, 0, 0, 0, 4075, 1, 0, 0, 0, 23, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 8, 4085, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4094, 0, 1, 0, 0, 4092, 1, 0, 2, 0, 1, 4096, 4095, 0, 0, 0, 4094, 1, 0, 0, 7, 4093, 0, 0, 0, 0, 1, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4093, 0, 0, 2, 4096, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 13, 4085, 4092, 1, 3, 0, 4096, 0, 0, 5, 1, 4095, 1, 0, 0, 1, 0, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4087, 0, 2, 0, 4092, 0, 0, 1, 0, 4087, 0, 0, 0, 4, 8, 4087, 4096, 0, 0, 0, 4095, 0, 0, 4, 16, 4095, 0, 0, 0, 3, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 2, 1, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4083, 0, 0, 7, 0, 0, 0, 4, 0, 30, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4097, 4090, 4121, 4096, 4095, 4072, 4097] avg=4096.0000 max=4121 max_rank=3 min=4072 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4072, 2, 2, 0, 2, 1, 0, 0, 4079, 3, 1, 0, 1, 0, 0, 0, 0, 4085, 0, 0, 7, 0, 0, 0, 1, 8, 11, 4079, 0, 0, 0, 0, 4084, 3, 0, 1, 0, 0, 0, 0, 4093, 1, 5, 0, 0, 0, 0, 0, 30, 4092, 8, 2, 0, 0, 0, 0, 1, 8, 0, 4085, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4084, 4092, 4099, 4088, 4099, 4132, 4095] avg=4096.0000 max=4132 max_rank=6 min=4079 min_rank=0 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 2, 1, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4075, 1, 0, 7, 0, 0, 0, 0, 0, 23, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 8, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4097, 4083, 4110, 4096, 4095, 4090, 4097] avg=4096.0000 max=4110 max_rank=3 min=4083 min_rank=2 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4095, 1, 1, 0, 2, 1, 0, 0, 4095, 0, 1, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 6, 4092, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4097, 4101, 4087, 4096, 4093, 4098, 4096] avg=4096.0000 max=4101 max_rank=2 min=4087 min_rank=3 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 2, 1, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 1, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4094, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4097, 4104, 4088, 4096, 4093, 4093, 4098] avg=4096.0000 max=4104 max_rank=2 min=4088 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 2, 1, 0, 0, 4094, 0, 1, 0, 1, 0, 0, 0, 0, 4092, 1, 0, 7, 0, 0, 0, 2, 0, 1, 4087, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 7, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4100, 4090, 4095, 4095, 4097, 4098] avg=4096.0000 max=4100 max_rank=2 min=4090 min_rank=3 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 2, 4, 2, 0, 0, 4096, 0, 0, 2, 8, 0, 0, 0, 0, 4093, 0, 0, 11, 0, 0, 0, 0, 0, 13, 4070, 0, 0, 0, 0, 4092, 1, 3, 0, 7, 2, 0, 0, 4094, 0, 0, 5, 0, 0, 0, 0, 1, 4076, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4106, 4104, 4083, 4105, 4099, 4078, 4092] avg=4096.0000 max=4106 max_rank=1 min=4078 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 2, 0, 2, 1, 0, 0, 4092, 0, 0, 1, 1, 0, 0, 0, 0, 4087, 0, 0, 7, 0, 0, 0, 0, 4, 8, 4078, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4, 0, 0, 0, 0, 16, 4092, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4091, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4094, 4094, 4090, 4096, 4099, 4108, 4095] avg=4096.0000 max=4108 max_rank=6 min=4090 min_rank=3 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 4094, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 70, 0, 0, 0, 0, 0, 0, 89, 3939, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 4093, 4076, 1, 1, 0, 4095, 1, 4093, 4094, 14, 1, 4095, 1, 0, 40, 1, 1, 0, 1, 0, 0, 117, 3945, 0, 0, 0, 0, 0, 4095, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 4095, 4087, 0, 1, 1, 4096, 3, 4090, 4095, 0, 0, 4095, 0, 0, 481, 0, 0, 1, 0, 0, 0, 364, 3261, 3, 0, 0, 1, 0, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 20, 0, 0, 0, 0, 0, 0, 64, 4012, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4096, 4087, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 64, 0, 0, 0, 0, 0, 0, 260, 3781, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 4093, 4042, 3, 0, 1, 4091, 1, 4082, 4090, 0, 1, 4094, 2, 4, 71, 4, 4, 0, 2, 0, 1, 1210, 2851, 8, 0, 5, 0, 1, 4091, 1, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[6, 4095, 4095, 0, 5, 0, 4096, 0, 4087, 4096, 0, 1, 4089, 0, 0, 111, 0, 0, 0, 0, 0, 0, 182, 3822, 0, 0, 2, 0, 1, 4080, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1, 4096, 4086, 0, 0, 0, 4096, 7, 4081, 4096, 0, 0, 4089, 0, 0, 65, 0, 0, 0, 0, 0, 0, 684, 3371, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 89, 3939, 100, 1, 0, 0, 0, 0, 0, 0, 4079, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4095, 4100, 4053, 4095, 4129, 4092, 4096] avg=4096.0000 max=4129 max_rank=5 min=4053 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 4087, 0, 0, 0, 0, 0, 1, 1, 4095, 3, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 3982, 0, 0, 481, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 3, 0, 0, 0, 0, 364, 3261, 100, 1, 0, 0, 3, 0, 0, 1, 4080, 13, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4100, 4094, 4463, 4095, 3726, 4097, 4094] avg=4096.0000 max=4463 max_rank=3 min=3726 min_rank=5 imbalance_ratio=1.089600 max_load_reduction=28305 max_load_reduction_pct=0.863800 moved_to_echo=16012 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 64, 4012, 100, 1, 0, 0, 0, 0, 0, 0, 4081, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4095, 4100, 4003, 4095, 4177, 4094, 4096] avg=4096.0000 max=4177 max_rank=5 min=4003 min_rank=3 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 4042, 3, 0, 0, 0, 0, 0, 1, 4090, 1, 0, 0, 0, 0, 4082, 4, 0, 1, 0, 0, 0, 0, 3981, 2, 4, 71, 0, 0, 0, 0, 4, 4, 0, 2, 4086, 3, 0, 0, 0, 1, 1210, 2851, 100, 1, 0, 0, 8, 0, 5, 0, 4078, 13, 0, 0, 1, 4091, 1, 15, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4092, 4087, 4058, 4099, 4163, 4104, 4108] avg=4096.0000 max=4163 max_rank=5 min=4057 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 4076, 1, 0, 0, 0, 0, 1, 0, 4094, 1, 0, 0, 0, 0, 4093, 4, 14, 1, 0, 0, 0, 0, 3982, 1, 0, 40, 0, 0, 0, 0, 1, 1, 0, 1, 4090, 3, 0, 0, 0, 0, 117, 3945, 100, 1, 0, 0, 0, 0, 0, 0, 4078, 13, 0, 0, 0, 4095, 0, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4096, 4112, 4023, 4096, 4163, 4091, 4098] avg=4096.0000 max=4163 max_rank=5 min=4023 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 4087, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 260, 3781, 100, 1, 0, 0, 0, 0, 0, 0, 4081, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4095, 4100, 4047, 4095, 4142, 4094, 4096] avg=4096.0000 max=4142 max_rank=5 min=4047 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[6, 8, 4095, 0, 0, 0, 0, 0, 5, 0, 4088, 0, 0, 0, 0, 0, 4087, 2, 0, 1, 0, 0, 0, 0, 3969, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 0, 182, 3822, 103, 5, 0, 0, 0, 0, 2, 0, 4083, 17, 0, 0, 1, 4080, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4093, 4090, 4080, 4098, 4112, 4102, 4084] avg=4096.0000 max=4112 max_rank=5 min=4080 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 12, 4086, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 4081, 4, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 684, 3371, 100, 1, 0, 0, 0, 0, 0, 0, 4081, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4102, 4085, 4041, 4095, 4156, 4094, 4096] avg=4096.0000 max=4156 max_rank=5 min=4041 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[31872, 128, 128, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 3072, 29056, 896, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 4050, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 4096, 3, 4096, 3911, 0, 0, 0, 0, 4096, 225] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[2, 4095, 3, 1, 1, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 1, 1, 4095, 0, 1, 4095, 0, 1, 4092, 0, 4095, 4088, 1, 0, 0, 0, 4093, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1, 4096, 3965, 0, 0, 0, 0, 4096, 133] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[6, 4068, 15, 5, 2, 0, 5, 0, 0, 0, 4, 4085, 4, 0, 5, 0, 4091, 0, 6, 4091, 19, 2, 4084, 2, 4090, 4022, 0, 5, 0, 1, 4084, 72] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 4070, 0, 0, 0, 0, 0, 1, 0, 2, 0, 4094, 0, 0, 0, 0, 4095, 0, 0, 4095, 22, 0, 4094, 2, 4096, 3798, 0, 0, 0, 0, 4096, 303] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 4067, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 4096, 0, 4096, 3913, 0, 0, 0, 0, 4096, 209] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 3988, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4095, 0, 0, 4089, 12, 0, 4088, 32, 4096, 4071, 0, 0, 0, 0, 4096, 102] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 4057, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 4095, 20, 0, 4091, 14, 4096, 4072, 0, 0, 0, 0, 4096, 41] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4050, 1, 0, 19, 10, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 3979, 115, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4084, 3, 0, 0, 0, 0, 117, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 225, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4095, 4098, 4094, 4097, 4089, 4028, 4187] avg=4096.0000 max=4187 max_rank=7 min=4028 min_rank=6 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[6, 4068, 15, 5, 18, 10, 0, 0, 2, 0, 5, 0, 4090, 0, 0, 0, 0, 0, 4, 4085, 2, 0, 0, 0, 4, 0, 5, 0, 3974, 115, 0, 0, 1, 0, 6, 4091, 0, 0, 0, 0, 19, 2, 4072, 2, 0, 0, 0, 0, 116, 4022, 0, 5, 0, 0, 0, 0, 0, 1, 3951, 72, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4097, 4091, 4098, 4098, 4095, 4143, 4024] avg=4096.0000 max=4143 max_rank=6 min=4024 min_rank=7 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 4095, 3, 1, 18, 10, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 1, 4095, 2, 0, 0, 0, 0, 0, 1, 1, 3978, 115, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 0, 1, 4080, 0, 0, 0, 0, 0, 117, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 3960, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4095, 4099, 4095, 4097, 4081, 4206, 3966] avg=4096.0000 max=4206 max_rank=6 min=3966 min_rank=7 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4067, 1, 0, 19, 10, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 3979, 115, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4084, 0, 0, 0, 0, 0, 117, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 209, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4095, 4098, 4094, 4097, 4086, 4030, 4171] avg=4096.0000 max=4171 max_rank=7 min=4030 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 1, 0, 19, 10, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 3979, 115, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4084, 1, 0, 0, 0, 0, 117, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 133, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4095, 4097, 4094, 4097, 4085, 4082, 4095] avg=4096.0000 max=4123 max_rank=0 min=4082 min_rank=6 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4070, 0, 0, 19, 10, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 2, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 3979, 115, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 22, 0, 4082, 2, 0, 0, 0, 0, 117, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 303, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4095, 4098, 4094, 4096, 4106, 3915, 4265] avg=4096.0000 max=4265 max_rank=7 min=3915 min_rank=6 imbalance_ratio=1.041260 max_load_reduction=28503 max_load_reduction_pct=0.869843 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3988, 7, 0, 21, 15, 0, 0, 0, 0, 0, 0, 4094, 7, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 3985, 120, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 0, 12, 0, 4069, 32, 0, 0, 0, 0, 104, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 102, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4101, 4096, 4105, 4090, 4113, 4175, 4057] avg=4096.0000 max=4175 max_rank=6 min=4031 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4057, 1, 0, 19, 10, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 3979, 115, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 20, 0, 4079, 14, 0, 0, 0, 0, 117, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 41, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4095, 4091, 4094, 4096, 4113, 4189, 4003] avg=4096.0000 max=4189 max_rank=6 min=4003 min_rank=7 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 31744, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 31872, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32512, 128, 128, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1024, 31872, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 176, 0, 0, 0, 0, 3920, 0, 4096, 0, 4096, 0, 4094, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 0, 0, 0, 4095, 0, 1, 0, 0, 6, 1, 1, 0, 4095, 100, 0, 0, 0, 1, 4003, 0, 4095, 0, 4083, 0, 4095, 4093, 1, 4095, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 469, 0, 0, 0, 0, 3627, 0, 4096, 0, 4096, 0, 4095, 4095, 0, 4096, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 94, 0, 0, 0, 0, 4002, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 7, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 96, 0, 0, 0, 0, 3997, 0, 4096, 0, 4096, 0, 4085, 4095, 0, 4090, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 217, 0, 0, 0, 0, 3879, 0, 4096, 0, 4095, 0, 4094, 4096, 0, 4096, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 113, 0, 0, 0, 0, 3983, 0, 4096, 0, 4096, 1, 4090, 4095, 0, 4094, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[6, 6, 7, 0, 0, 4, 4091, 0, 4, 0, 3, 5, 6, 7, 0, 4089, 15, 2, 0, 0, 4, 4090, 0, 4076, 0, 4086, 0, 4090, 4084, 4, 4089, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 176, 0, 0, 0, 3936, 0, 0, 0, 0, 3920, 0, 159, 0, 0, 0, 0, 0, 4, 0, 4094, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4094, 4096, 4112, 4079, 4098, 4097] avg=4096.0000 max=4112 max_rank=4 min=4079 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 94, 0, 0, 0, 3936, 0, 0, 0, 0, 4002, 0, 159, 0, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4094, 4096, 4030, 4161, 4099, 4097] avg=4096.0000 max=4161 max_rank=5 min=4030 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 0, 4093, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 6, 4079, 1, 1, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 100, 0, 0, 0, 3935, 0, 0, 0, 1, 4003, 0, 159, 0, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 4093, 1, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4088, 4097, 4035, 4163, 4099, 4095] avg=4096.0000 max=4163 max_rank=5 min=4035 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 1, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 469, 0, 0, 0, 3936, 0, 0, 0, 0, 3627, 0, 159, 0, 0, 0, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4095, 4096, 4405, 3786, 4099, 4097] avg=4096.0000 max=4405 max_rank=4 min=3786 min_rank=5 imbalance_ratio=1.075439 max_load_reduction=28363 max_load_reduction_pct=0.865570 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4091, 1, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 217, 0, 0, 0, 3936, 0, 0, 0, 0, 3879, 0, 159, 0, 0, 0, 0, 0, 4, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4097, 4096, 4096, 4153, 4038, 4098, 4096] avg=4096.0000 max=4153 max_rank=4 min=4038 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4090, 0, 0, 0, 0, 7, 4091, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 96, 0, 0, 0, 3942, 7, 5, 0, 0, 3997, 0, 150, 0, 0, 0, 0, 0, 1, 0, 4085, 0, 0, 0, 0, 4095, 0, 0, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4098, 4107, 4089, 4050, 4147, 4086, 4098] avg=4096.0000 max=4147 max_rank=5 min=4050 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 6, 7, 0, 4087, 0, 0, 0, 0, 4, 4091, 0, 0, 0, 0, 0, 4, 0, 3, 5, 4082, 1, 1, 0, 6, 7, 0, 4089, 0, 0, 0, 0, 15, 2, 0, 0, 3917, 0, 0, 0, 4, 4090, 0, 158, 0, 0, 0, 0, 0, 4, 0, 4090, 0, 0, 0, 0, 4084, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4095, 4096, 4102, 3934, 4252, 4094, 4089] avg=4096.0000 max=4252 max_rank=5 min=3934 min_rank=4 imbalance_ratio=1.038086 max_load_reduction=28516 max_load_reduction_pct=0.870239 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4092, 1, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 113, 0, 0, 0, 3936, 0, 0, 0, 0, 3983, 0, 159, 0, 0, 0, 0, 0, 4, 1, 4090, 0, 0, 0, 0, 4095, 0, 0, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 4098, 4096, 4049, 4142, 4095, 4099] avg=4096.0000 max=4142 max_rank=5 min=4049 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1280, 128, 0, 0, 31488, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 31616, 0, 1280, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4080, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 12, 0, 1033, 0, 0, 0, 0, 4096, 124, 0, 4096, 0, 0, 2946, 4093, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4087, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 840, 0, 0, 0, 0, 4096, 96, 0, 4096, 0, 0, 3166, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4063, 4, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 11, 13, 3012, 0, 0, 0, 0, 4095, 13, 0, 4096, 0, 0, 1091, 4084, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 2, 0, 4096, 0, 0, 4094, 0, 0, 0, 1, 4, 438, 0, 1, 0, 0, 4096, 102, 0, 4096, 0, 0, 3564, 4083, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 4095, 1, 0, 2, 4095, 1, 1, 4095, 0, 1, 0, 0, 0, 533, 0, 0, 1, 1, 4095, 74, 1, 4095, 0, 1, 3487, 4093, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1397, 0, 0, 0, 0, 4096, 41, 0, 4096, 0, 0, 2659, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4069, 0, 11, 0, 4096, 0, 0, 4088, 0, 0, 0, 26, 8, 282, 0, 0, 0, 0, 4096, 342, 0, 4096, 0, 0, 3473, 4085, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4088, 4091, 6, 0, 7, 4087, 5, 7, 4090, 0, 0, 0, 0, 20, 561, 2, 0, 7, 6, 4089, 29, 5, 4089, 0, 7, 3495, 4069, 0, 7, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 4080, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 12, 0, 1033, 2972, 102, 1, 0, 0, 0, 0, 0, 4085, 8, 0, 0, 3994, 124, 0, 0, 0, 0, 0, 0, 0, 0, 2946, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4094, 4097, 4120, 4093, 4118, 4059, 4096] avg=4096.0000 max=4120 max_rank=3 min=4059 min_rank=6 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16366 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 4087, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4, 0, 840, 2974, 102, 1, 0, 0, 0, 0, 0, 4085, 8, 0, 0, 3994, 96, 0, 0, 0, 0, 0, 0, 0, 0, 3166, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4094, 4097, 3921, 4093, 4090, 4279, 4096] avg=4096.0000 max=4279 max_rank=6 min=3921 min_rank=3 imbalance_ratio=1.044678 max_load_reduction=28489 max_load_reduction_pct=0.869415 moved_to_echo=16558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 4095, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 1, 4, 438, 2965, 102, 1, 0, 0, 1, 0, 0, 4085, 8, 0, 0, 3994, 102, 0, 0, 0, 0, 0, 0, 0, 0, 3564, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4096, 4095, 3511, 4094, 4096, 4674, 4096] avg=4096.0000 max=4674 max_rank=6 min=3511 min_rank=3 imbalance_ratio=1.141113 max_load_reduction=28094 max_load_reduction_pct=0.857361 moved_to_echo=16960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 4095, 1, 0, 0, 0, 0, 0, 2, 4093, 1, 0, 0, 0, 0, 1, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 0, 533, 2972, 102, 1, 0, 0, 0, 1, 1, 4084, 8, 0, 0, 3993, 74, 1, 0, 0, 0, 0, 0, 0, 1, 3487, 1113, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4096, 4098, 3608, 4094, 4068, 4601, 4096] avg=4096.0000 max=4601 max_rank=6 min=3608 min_rank=3 imbalance_ratio=1.123291 max_load_reduction=28167 max_load_reduction_pct=0.859589 moved_to_echo=16859 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1397, 2974, 102, 1, 0, 0, 0, 0, 0, 4085, 8, 0, 0, 3994, 41, 0, 0, 0, 0, 0, 0, 0, 0, 2659, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4094, 4097, 4474, 4093, 4035, 3772, 4096] avg=4096.0000 max=4474 max_rank=3 min=3772 min_rank=6 imbalance_ratio=1.092285 max_load_reduction=28294 max_load_reduction_pct=0.863464 moved_to_echo=15996 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 4091, 6, 0, 0, 0, 0, 0, 7, 4086, 5, 0, 0, 0, 0, 7, 4090, 0, 0, 1, 0, 0, 0, 0, 0, 20, 561, 2955, 102, 0, 0, 2, 0, 7, 6, 4077, 8, 0, 0, 3987, 29, 5, 0, 0, 0, 0, 0, 0, 7, 3495, 1106, 0, 0, 0, 0, 0, 7, 1, 0, 4089, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4098, 4098, 3638, 4100, 4021, 4608, 4097] avg=4096.0000 max=4608 max_rank=6 min=3638 min_rank=3 imbalance_ratio=1.125000 max_load_reduction=28160 max_load_reduction_pct=0.859375 moved_to_echo=16826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4069, 0, 0, 0, 0, 0, 11, 0, 4091, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 3, 0, 0, 0, 0, 26, 8, 282, 2969, 104, 2, 0, 0, 0, 0, 0, 4092, 15, 1, 0, 3992, 342, 0, 0, 0, 0, 0, 0, 0, 0, 3473, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4102, 4091, 3391, 4108, 4334, 4574, 4095] avg=4096.0000 max=4574 max_rank=6 min=3391 min_rank=3 imbalance_ratio=1.116699 max_load_reduction=28194 max_load_reduction_pct=0.860413 moved_to_echo=17111 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 4063, 4, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 11, 13, 3012, 2966, 102, 1, 0, 0, 0, 0, 0, 4085, 8, 0, 0, 3993, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4094, 4095, 6105, 4093, 4006, 2201, 4096] avg=4096.0000 max=6105 max_rank=3 min=2201 min_rank=6 imbalance_ratio=1.490479 max_load_reduction=26663 max_load_reduction_pct=0.813690 moved_to_echo=14396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 8192, 23808, 896, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 23936, 8960, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32000, 896, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4026, 0, 1, 3529, 0, 0, 4070, 0, 4096, 0, 662, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[6, 4089, 0, 4088, 0, 4, 0, 4084, 3, 0, 2473, 5, 0, 3665, 7, 4088, 3, 2020, 4083, 0, 7, 8, 4088, 10, 9, 0, 1, 19, 0, 2, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4090, 1, 2, 3621, 0, 0, 4093, 0, 4096, 0, 481, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 7, 0, 0, 4083, 0, 0, 3783, 0, 7, 4080, 0, 4096, 0, 335, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 4088, 0, 1, 3586, 0, 0, 4067, 0, 4096, 0, 546, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 2, 0, 0, 4048, 0, 0, 4005, 0, 1, 4036, 0, 4096, 0, 196, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1, 4095, 0, 4095, 0, 1, 0, 4075, 3, 7, 3876, 1, 0, 3835, 1, 4094, 0, 489, 4095, 0, 1, 1, 4095, 0, 1, 0, 0, 1, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4096, 6, 0, 0, 4066, 0, 0, 1983, 0, 5, 4085, 0, 4096, 0, 2144, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4026, 21, 2, 0, 0, 0, 1, 3529, 0, 736, 0, 0, 0, 0, 4070, 0, 103, 0, 0, 0, 0, 0, 662, 3239, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 100, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4049, 4266, 4173, 3901, 4094, 4092, 4095] avg=4096.0000 max=4266 max_rank=2 min=3901 min_rank=4 imbalance_ratio=1.041504 max_load_reduction=28502 max_load_reduction_pct=0.869812 moved_to_echo=16182 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 4089, 0, 1, 0, 0, 0, 0, 0, 4, 0, 4084, 21, 2, 0, 0, 3, 0, 2473, 5, 733, 0, 0, 0, 0, 3665, 7, 102, 0, 0, 0, 0, 3, 2020, 3229, 0, 0, 0, 0, 0, 7, 8, 4086, 10, 0, 0, 0, 0, 9, 0, 1, 19, 3985, 100, 0, 0, 0, 2, 6, 0, 4087, 1, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4111, 3214, 3774, 5252, 4111, 4114, 4096] avg=4096.0000 max=5252 max_rank=4 min=3214 min_rank=2 imbalance_ratio=1.282227 max_load_reduction=27516 max_load_reduction_pct=0.839722 moved_to_echo=17573 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4088, 21, 2, 0, 0, 0, 1, 3586, 0, 736, 0, 0, 0, 0, 4067, 0, 103, 0, 0, 0, 0, 0, 546, 3239, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 100, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4111, 4323, 4170, 3785, 4094, 4092, 4095] avg=4096.0000 max=4323 max_rank=2 min=3785 min_rank=4 imbalance_ratio=1.055420 max_load_reduction=28445 max_load_reduction_pct=0.868073 moved_to_echo=16066 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 2, 0, 0, 0, 0, 2, 0, 0, 4048, 21, 2, 0, 0, 0, 0, 4005, 0, 736, 0, 0, 0, 1, 4036, 0, 103, 0, 0, 0, 0, 0, 196, 3239, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 100, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4073, 4741, 4140, 3435, 4094, 4092, 4095] avg=4096.0000 max=4741 max_rank=2 min=3435 min_rank=4 imbalance_ratio=1.157471 max_load_reduction=28027 max_load_reduction_pct=0.855316 moved_to_echo=15716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 21, 2, 0, 0, 1, 2, 3621, 0, 736, 0, 0, 0, 0, 4093, 0, 103, 0, 0, 0, 0, 0, 481, 3239, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 100, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4113, 4360, 4196, 3720, 4094, 4092, 4097] avg=4096.0000 max=4360 max_rank=2 min=3720 min_rank=4 imbalance_ratio=1.064453 max_load_reduction=28408 max_load_reduction_pct=0.866943 moved_to_echo=16003 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 4095, 0, 2, 0, 0, 0, 0, 0, 1, 0, 4075, 21, 2, 0, 0, 3, 7, 3876, 1, 736, 0, 0, 0, 0, 3835, 1, 103, 0, 0, 0, 0, 0, 489, 3238, 0, 0, 0, 0, 0, 1, 1, 4093, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3990, 100, 0, 0, 0, 0, 1, 0, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4099, 4623, 3939, 3727, 4095, 4092, 4095] avg=4096.0000 max=4623 max_rank=2 min=3727 min_rank=4 imbalance_ratio=1.128662 max_load_reduction=28145 max_load_reduction_pct=0.858917 moved_to_echo=16009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4083, 23, 4, 0, 0, 0, 0, 3783, 0, 739, 0, 0, 0, 7, 4080, 0, 96, 0, 0, 0, 0, 0, 335, 3230, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 104, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4117, 4522, 4183, 3565, 4092, 4102, 4098] avg=4096.0000 max=4522 max_rank=2 min=3565 min_rank=4 imbalance_ratio=1.104004 max_load_reduction=28246 max_load_reduction_pct=0.862000 moved_to_echo=15857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4066, 21, 2, 0, 0, 0, 0, 1983, 0, 736, 0, 0, 0, 5, 4085, 0, 103, 0, 0, 0, 0, 0, 2144, 3239, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 100, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 2719, 4193, 5383, 4094, 4092, 4097] avg=4096.0000 max=5383 max_rank=4 min=2719 min_rank=2 imbalance_ratio=1.314209 max_load_reduction=27385 max_load_reduction_pct=0.835724 moved_to_echo=17666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 6912, 25984, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 26880, 128, 5888, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32000, 128, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32000, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32640, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 2822, 4095, 0, 1, 4094, 4095, 1, 0, 0, 0, 1, 0, 4095, 0, 1, 4095, 4094, 1273, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[11, 8, 3, 4088, 2397, 4085, 0, 7, 4088, 4088, 8, 3, 0, 0, 0, 0, 4085, 0, 7, 4087, 4085, 1693, 0, 8, 0, 0, 1, 8, 0, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2849, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2321, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 2427, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 4096, 4096, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2642, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 4096, 1453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 3411, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 4096, 683, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1190, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4096, 4096, 2899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 2321, 1589, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2507, 1585, 1, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 2511, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 3910, 4096, 4093, 4097, 4286, 4096, 4095] avg=4096.0000 max=4286 max_rank=5 min=3910 min_rank=1 imbalance_ratio=1.046387 max_load_reduction=28482 max_load_reduction_pct=0.869202 moved_to_echo=16574 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 2849, 1589, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2507, 1585, 1, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 2511, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4438, 4096, 4093, 4097, 3758, 4095, 4096] avg=4096.0000 max=4438 max_rank=1 min=3758 min_rank=5 imbalance_ratio=1.083496 max_load_reduction=28330 max_load_reduction_pct=0.864563 moved_to_echo=16046 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4094, 0, 0, 0, 0, 2822, 1588, 0, 1, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2507, 1584, 1, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 2510, 1273, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4411, 4095, 4093, 4097, 3784, 4096, 4095] avg=4096.0000 max=4411 max_rank=1 min=3784 min_rank=5 imbalance_ratio=1.076904 max_load_reduction=28357 max_load_reduction_pct=0.865387 moved_to_echo=16072 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[11, 8, 3, 4087, 0, 0, 0, 0, 2397, 1584, 0, 7, 0, 0, 0, 0, 4088, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2501, 1581, 1, 0, 4085, 0, 7, 0, 0, 0, 0, 0, 2504, 1693, 0, 8, 0, 0, 0, 0, 0, 0, 1, 8, 4088, 0, 0, 0, 0, 0, 0, 8, 4087, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 3988, 4099, 4083, 4092, 4205, 4097, 4095] avg=4096.0000 max=4205 max_rank=5 min=3988 min_rank=1 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 3411, 1589, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2507, 1585, 1, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 2511, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 5000, 4096, 4095, 4097, 3194, 4096, 4096] avg=4096.0000 max=5000 max_rank=1 min=3194 min_rank=5 imbalance_ratio=1.220703 max_load_reduction=27768 max_load_reduction_pct=0.847412 moved_to_echo=15483 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 2642, 1589, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2507, 1585, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2511, 1453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4231, 4096, 4094, 4096, 3964, 4096, 4096] avg=4096.0000 max=4231 max_rank=1 min=3964 min_rank=5 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16252 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4088, 0, 0, 0, 0, 2427, 1584, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2512, 1590, 5, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 2506, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4011, 4094, 4117, 4096, 4167, 4096, 4099] avg=4096.0000 max=4167 max_rank=5 min=4011 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 1190, 1589, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2507, 1585, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2511, 2899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 2779, 4096, 4100, 4096, 5410, 4096, 4096] avg=4096.0000 max=5410 max_rank=5 min=2779 min_rank=1 imbalance_ratio=1.320801 max_load_reduction=27358 max_load_reduction_pct=0.834900 moved_to_echo=17698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 20096, 12800, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[20096, 12800, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[20096, 12800, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4078, 0, 4084, 8, 4086, 0, 7, 7, 4089, 4059, 4085, 9, 8, 9, 0, 0, 0, 0, 5, 24, 4088, 5, 8, 3, 4059, 44, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 4096, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 6, 4096, 0, 4096, 0, 0, 0, 4090, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4094, 0, 4084, 0, 4095, 0, 1, 1, 4095, 4095, 4094, 1, 1, 1, 0, 0, 0, 1, 0, 4, 4095, 1, 1, 1, 4088, 15, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 9, 4096, 0, 4096, 0, 0, 0, 4087, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 6, 4096, 0, 4096, 0, 0, 0, 4089, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4091, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4100, 4096, 4093, 4095, 4095, 4097] avg=4096.0000 max=4100 max_rank=2 min=4093 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 4091, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4100, 4095, 4093, 4095, 4095, 4098] avg=4096.0000 max=4100 max_rank=2 min=4093 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4084, 8, 0, 0, 0, 0, 4086, 0, 7, 7, 0, 0, 0, 0, 4, 4059, 0, 9, 0, 0, 0, 0, 8, 9, 0, 0, 4078, 1, 0, 0, 0, 0, 5, 24, 4084, 1, 1, 0, 4087, 5, 8, 3, 0, 0, 0, 0, 4058, 44, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4085, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4100, 4072, 4096, 4115, 4103, 4102, 4088] avg=4096.0000 max=4115 max_rank=4 min=4072 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16408 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4084, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 4, 4095, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 4093, 1, 0, 0, 0, 1, 0, 4, 4090, 1, 1, 0, 4094, 1, 1, 1, 0, 0, 0, 0, 4087, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4097, 4100, 4096, 4097, 4097, 4102, 4095] avg=4096.0000 max=4102 max_rank=6 min=4084 min_rank=0 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4085, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4096, 4099, 4096, 4087, 4095, 4095, 4098] avg=4096.0000 max=4102 max_rank=0 min=4087 min_rank=4 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 1, 0, 0, 4090, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4100, 4096, 4093, 4095, 4095, 4097] avg=4096.0000 max=4100 max_rank=2 min=4093 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 4084, 7, 3, 5, 4093, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4091, 4097, 4097, 4099, 4093, 4089, 4097] avg=4096.0000 max=4105 max_rank=0 min=4089 min_rank=6 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4084, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4096, 4100, 4096, 4086, 4095, 4095, 4098] avg=4096.0000 max=4102 max_rank=0 min=4086 min_rank=4 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32773, 32768, 32768, 32768, 32763, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32773, 32768, 32768, 32768, 32763, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32773, 32768, 32768, 32768, 32763, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32773, 32768, 32768, 32768, 32763, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32773, 32768, 32768, 32768, 32763, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32773, 32768, 32768, 32768, 32763, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32773, 32768, 32768, 32768, 32763, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32773, 32768, 32768, 32768, 32763, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32763 actual_valid_recv=33536 observed_loss=-773 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32773 actual_valid_recv=33024 observed_loss=-251 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3887, 0, 4096, 0, 4096, 4038, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 65, 0, 0, 0, 0, 202, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1, 3511, 1, 4095, 1, 4064, 4095, 0, 1, 0, 1, 1, 4095, 0, 0, 1, 0, 4095, 8, 0, 1, 0, 0, 628, 3992, 0, 0, 82, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3861, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 44, 0, 0, 0, 0, 195, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3612, 0, 4096, 0, 4096, 4064, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4096, 39, 1, 0, 0, 0, 496, 4075, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3584, 0, 4096, 0, 4096, 3923, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 6, 4090, 143, 6, 0, 0, 0, 567, 4088, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3785, 0, 4096, 0, 4096, 4086, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 293, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[5, 2, 4087, 11, 3578, 3, 4087, 8, 3970, 4088, 0, 8, 0, 5, 9, 4085, 9, 0, 6, 0, 4087, 0, 0, 4, 6, 0, 484, 3963, 2, 0, 261, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3790, 0, 4096, 0, 4096, 4072, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 6, 4096, 112, 3, 0, 0, 0, 218, 4091, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 3887, 0, 394, 0, 0, 0, 0, 0, 38, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 0, 0, 0, 0, 0, 0, 4058, 53, 0, 0, 4042, 65, 0, 0, 0, 0, 0, 0, 0, 0, 202, 3742, 0, 0, 0, 0, 0, 0, 0, 0, 3649, 354, 52, 0] rerouted_tokens_per_ep_rank=[4095, 4281, 4076, 4099, 4111, 4107, 3944, 4055] avg=4096.0000 max=4281 max_rank=1 min=3944 min_rank=6 imbalance_ratio=1.045166 max_load_reduction=28487 max_load_reduction_pct=0.869354 moved_to_echo=16217 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 1, 0, 0, 0, 0, 3511, 1, 394, 1, 0, 0, 0, 0, 38, 4095, 0, 1, 0, 0, 0, 0, 0, 1, 1, 4095, 2, 1, 0, 0, 0, 0, 1, 0, 4026, 53, 0, 0, 4041, 8, 0, 1, 0, 0, 0, 0, 0, 0, 628, 3647, 0, 0, 0, 0, 0, 0, 82, 0, 3648, 345, 52, 0] rerouted_tokens_per_ep_rank=[4095, 3907, 4134, 4100, 4080, 4050, 4275, 4127] avg=4096.0000 max=4275 max_rank=6 min=3907 min_rank=1 imbalance_ratio=1.043701 max_load_reduction=28493 max_load_reduction_pct=0.869537 moved_to_echo=16532 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 3612, 0, 394, 0, 0, 0, 0, 0, 38, 4064, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 1, 0, 0, 0, 0, 0, 1, 4058, 53, 0, 0, 4042, 39, 1, 0, 0, 0, 0, 0, 0, 0, 496, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 3649, 352, 52, 0] rerouted_tokens_per_ep_rank=[4095, 4006, 4103, 4098, 4112, 4082, 4219, 4053] avg=4096.0000 max=4219 max_rank=6 min=4006 min_rank=1 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[5, 2, 4086, 11, 0, 0, 0, 0, 3578, 3, 393, 8, 0, 0, 0, 0, 37, 4088, 0, 8, 0, 0, 0, 0, 0, 5, 9, 4085, 2, 1, 0, 0, 9, 0, 6, 0, 3933, 53, 0, 0, 4033, 0, 0, 4, 0, 0, 0, 0, 6, 0, 484, 3620, 0, 0, 0, 0, 2, 0, 261, 0, 3641, 343, 52, 0] rerouted_tokens_per_ep_rank=[4104, 3982, 4133, 4102, 4001, 4037, 4110, 4299] avg=4096.0000 max=4299 max_rank=7 min=3982 min_rank=1 imbalance_ratio=1.049561 max_load_reduction=28469 max_load_reduction_pct=0.868805 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 3861, 0, 394, 0, 0, 0, 0, 0, 38, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 0, 0, 0, 0, 0, 0, 4058, 53, 0, 0, 4042, 44, 0, 0, 0, 0, 0, 0, 0, 0, 195, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 3649, 354, 52, 0] rerouted_tokens_per_ep_rank=[4095, 4255, 4132, 4099, 4111, 4086, 3935, 4055] avg=4096.0000 max=4255 max_rank=1 min=3935 min_rank=6 imbalance_ratio=1.038818 max_load_reduction=28513 max_load_reduction_pct=0.870148 moved_to_echo=16187 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 3785, 0, 394, 0, 0, 0, 0, 0, 38, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 0, 0, 0, 0, 0, 0, 4058, 53, 0, 0, 4042, 30, 0, 0, 0, 0, 0, 0, 0, 0, 293, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 3649, 354, 52, 0] rerouted_tokens_per_ep_rank=[4095, 4179, 4124, 4099, 4111, 4072, 4033, 4055] avg=4096.0000 max=4179 max_rank=1 min=4033 min_rank=6 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 0, 0, 0, 3584, 0, 390, 0, 0, 0, 0, 0, 33, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 3, 2, 0, 0, 0, 0, 0, 6, 4063, 56, 0, 0, 4028, 143, 6, 0, 0, 0, 0, 0, 0, 0, 567, 3730, 0, 0, 0, 0, 0, 0, 0, 0, 3650, 358, 59, 0] rerouted_tokens_per_ep_rank=[4094, 3974, 3956, 4078, 4125, 4177, 4297, 4067] avg=4096.0000 max=4297 max_rank=6 min=3956 min_rank=2 imbalance_ratio=1.049072 max_load_reduction=28471 max_load_reduction_pct=0.868866 moved_to_echo=16666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 3790, 0, 394, 0, 0, 0, 0, 0, 38, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 1, 0, 0, 0, 0, 0, 6, 4058, 53, 0, 0, 4042, 112, 3, 0, 0, 0, 0, 0, 0, 0, 218, 3737, 0, 0, 0, 0, 0, 0, 2, 0, 3649, 354, 52, 0] rerouted_tokens_per_ep_rank=[4095, 4184, 4110, 4093, 4117, 4157, 3955, 4057] avg=4096.0000 max=4184 max_rank=1 min=3955 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 384, 0, 29184, 2816, 512, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[29696, 128, 3200, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 3200, 29696, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 32512, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32384, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32384, 512, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 4096, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[2, 1, 4094, 4084, 1, 0, 0, 0, 0, 0, 0, 1, 0, 15, 1, 4095, 4094, 1, 1, 4095, 4067, 0, 1, 0, 4095, 23, 4094, 0, 0, 0, 1, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[9, 8, 4076, 4001, 5, 0, 0, 0, 0, 5, 1, 9, 0, 62, 11, 4085, 4074, 12, 3, 4087, 4012, 0, 18, 1, 4088, 96, 4082, 0, 0, 0, 10, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 4096, 4094, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 10, 2, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 11, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4096, 4095, 4090, 4097, 4104, 4083, 4095] avg=4096.0000 max=4108 max_rank=0 min=4083 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 10, 2, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 11, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4096, 4094, 4087, 4098, 4107, 4083, 4095] avg=4096.0000 max=4108 max_rank=0 min=4083 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[9, 8, 12, 4001, 0, 0, 0, 0, 5, 0, 0, 0, 4088, 0, 0, 0, 0, 5, 1, 9, 4087, 0, 0, 0, 0, 62, 11, 4075, 0, 0, 0, 0, 4074, 12, 3, 0, 0, 0, 0, 0, 4012, 0, 18, 1, 10, 1, 0, 0, 0, 96, 4071, 0, 0, 0, 0, 0, 0, 0, 10, 13, 4064, 10, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4093, 4102, 4148, 4089, 4042, 4167, 4097] avg=4096.0000 max=4167 max_rank=6 min=4030 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16395 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 12, 4084, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 15, 1, 4085, 0, 0, 0, 0, 4094, 1, 1, 2, 0, 0, 0, 0, 4067, 0, 1, 0, 10, 2, 0, 0, 0, 23, 4081, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4082, 11, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 4094, 4101, 4098, 4080, 4104, 4096] avg=4096.0000 max=4104 max_rank=6 min=4080 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 2, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 11, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4096, 4094, 4086, 4098, 4108, 4083, 4095] avg=4096.0000 max=4108 max_rank=0 min=4083 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 10, 2, 0, 0, 0, 1, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 11, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4096, 4096, 4088, 4096, 4106, 4084, 4095] avg=4096.0000 max=4107 max_rank=0 min=4084 min_rank=6 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 3, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 12, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4098, 4098, 4082, 4096, 4113, 4081, 4100] avg=4096.0000 max=4113 max_rank=5 min=4081 min_rank=6 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16390 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 2, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 11, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4097, 4095, 4086, 4096, 4108, 4083, 4095] avg=4096.0000 max=4108 max_rank=0 min=4083 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4086, 1, 0, 0, 4093, 6, 4089, 0, 1, 4095, 4094, 4095, 2, 0, 4095, 1, 0, 1, 9, 1, 0, 4094, 1, 0, 1, 1, 0, 0, 0, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4073, 10, 0, 0, 4053, 31, 4036, 0, 5, 4086, 4080, 4084, 13, 0, 4087, 9, 1, 9, 56, 4, 1, 4082, 9, 0, 10, 11, 0, 0, 1, 3, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[3, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 2, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4100, 4099, 4096, 4090, 4096, 4094, 4095] avg=4096.0000 max=4100 max_rank=1 min=4090 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 1, 0, 2, 0, 0, 0, 0, 4, 6, 4089, 0, 0, 0, 0, 0, 1, 3, 4094, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 1, 0, 1, 9, 4087, 0, 0, 0, 1, 0, 4094, 1, 0, 0, 0, 0, 0, 1, 1, 0, 4092, 1, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4099, 4098, 4097, 4098, 4096, 4095, 4096] avg=4096.0000 max=4099 max_rank=1 min=4089 min_rank=0 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4100, 4099, 4096, 4090, 4096, 4093, 4096] avg=4096.0000 max=4100 max_rank=1 min=4090 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4073, 10, 0, 2, 0, 0, 0, 0, 4, 31, 4036, 0, 0, 0, 0, 0, 5, 0, 4080, 0, 0, 0, 0, 0, 13, 0, 4087, 0, 0, 0, 0, 9, 1, 9, 56, 4047, 0, 0, 0, 4, 1, 4082, 9, 0, 0, 0, 0, 0, 10, 11, 0, 4086, 1, 0, 0, 0, 1, 3, 14, 4083, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4071, 4085, 4100, 4122, 4096, 4108, 4101] avg=4096.0000 max=4122 max_rank=4 min=4071 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4100, 4096, 4096, 4091, 4096, 4096, 4095] avg=4096.0000 max=4100 max_rank=1 min=4091 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4100, 4099, 4096, 4090, 4096, 4094, 4095] avg=4096.0000 max=4100 max_rank=1 min=4090 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 4096, 0, 0, 3, 2, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4098, 4096, 4091, 4094, 4096, 4094, 4095] avg=4096.0000 max=4104 max_rank=0 min=4091 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4100, 4096, 4096, 4093, 4096, 4094, 4095] avg=4096.0000 max=4100 max_rank=1 min=4093 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=28668 max_load_reduction_pct=0.874878 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 4094, 4096, 4096, 0, 0, 2, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[8, 1, 3, 1, 0, 0, 0, 4095, 0, 2, 4095, 4093, 1, 0, 0, 3957, 112, 4093, 0, 0, 0, 0, 3933, 4088, 4094, 1, 1, 188, 1, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[52, 7, 16, 10, 22, 0, 0, 4085, 0, 12, 4086, 4080, 11, 0, 0, 3538, 483, 4064, 0, 0, 0, 0, 2915, 4042, 4082, 5, 2, 1236, 10, 0, 10, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4088, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 8, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 4095, 4096, 4096, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4076, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 20, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4090, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4087, 4096, 4096, 0, 0, 9, 0, 6, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3926, 164, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 6, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 70, 14, 1, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 170, 0, 0, 0, 0, 3918, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 4090, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4095, 4099, 4181, 4026, 4265, 3920, 4092] avg=4096.0000 max=4265 max_rank=5 min=3920 min_rank=6 imbalance_ratio=1.041260 max_load_reduction=28503 max_load_reduction_pct=0.869843 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[52, 7, 16, 10, 3874, 163, 0, 0, 22, 0, 0, 4084, 0, 0, 0, 0, 0, 12, 6, 4080, 0, 0, 0, 0, 11, 0, 0, 3538, 69, 14, 1, 0, 483, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 2915, 168, 0, 0, 0, 0, 3905, 5, 2, 1236, 0, 0, 0, 0, 10, 0, 10, 0, 4080, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4106, 4098, 3633, 4478, 3083, 5148, 4100] avg=4096.0000 max=5148 max_rank=6 min=3083 min_rank=5 imbalance_ratio=1.256836 max_load_reduction=27620 max_load_reduction_pct=0.842896 moved_to_echo=16809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[8, 1, 3, 1, 3919, 164, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 6, 4093, 0, 0, 0, 0, 1, 0, 0, 3957, 70, 14, 1, 0, 112, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 169, 0, 0, 0, 0, 3916, 1, 1, 188, 0, 0, 0, 0, 1, 0, 1, 0, 4089, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 4101, 4043, 4135, 4102, 4106, 4091] avg=4096.0000 max=4135 max_rank=4 min=4043 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3926, 164, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 70, 14, 1, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 170, 0, 0, 0, 0, 3918, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4095, 4100, 4181, 4026, 4267, 3919, 4090] avg=4096.0000 max=4267 max_rank=5 min=3919 min_rank=6 imbalance_ratio=1.041748 max_load_reduction=28501 max_load_reduction_pct=0.869781 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3926, 164, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 6, 4088, 0, 0, 0, 0, 0, 0, 0, 4096, 70, 14, 1, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 170, 0, 0, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4090, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4095, 4094, 4181, 4026, 4266, 3918, 4098] avg=4096.0000 max=4266 max_rank=5 min=3918 min_rank=6 imbalance_ratio=1.041504 max_load_reduction=28502 max_load_reduction_pct=0.869812 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3926, 164, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 6, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 70, 14, 1, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 170, 0, 0, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4095, 4101, 4181, 4026, 4267, 3918, 4090] avg=4096.0000 max=4267 max_rank=5 min=3918 min_rank=6 imbalance_ratio=1.041748 max_load_reduction=28501 max_load_reduction_pct=0.869781 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3931, 167, 2, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 4076, 0, 0, 0, 0, 0, 0, 0, 4096, 71, 17, 3, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 165, 0, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4091, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4093, 4079, 4187, 4025, 4261, 3912, 4111] avg=4096.0000 max=4261 max_rank=5 min=3912 min_rank=6 imbalance_ratio=1.040283 max_load_reduction=28507 max_load_reduction_pct=0.869965 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3926, 164, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 70, 14, 1, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 170, 0, 0, 0, 0, 3918, 0, 0, 9, 0, 0, 0, 0, 0, 6, 0, 0, 4090, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4095, 4096, 4181, 4026, 4257, 3927, 4096] avg=4096.0000 max=4257 max_rank=5 min=3927 min_rank=6 imbalance_ratio=1.039307 max_load_reduction=28511 max_load_reduction_pct=0.870087 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 31360, 1408, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 32128, 640, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[31360, 128, 128, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 31488, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[640, 32256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[9, 0, 0, 9, 10, 0, 73, 11, 1665, 8, 4013, 4084, 0, 0, 0, 11, 4069, 0, 4076, 12, 7, 2431, 0, 21, 4068, 2, 8, 4078, 0, 19, 133, 3951] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 1, 0, 8, 1, 479, 1, 4077, 4095, 0, 0, 0, 1, 4093, 0, 4094, 1, 1, 3616, 0, 1, 4092, 1, 1, 4094, 0, 3, 20, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4089, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 258, 5, 2, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4085, 3838, 4096, 4099, 4361, 4100, 4094] avg=4096.0000 max=4361 max_rank=5 min=3838 min_rank=2 imbalance_ratio=1.064697 max_load_reduction=28407 max_load_reduction_pct=0.866913 moved_to_echo=16654 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 9, 4073, 2, 0, 0, 10, 0, 73, 11, 4067, 0, 0, 0, 1665, 8, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4084, 0, 0, 0, 4069, 0, 3, 12, 0, 0, 0, 0, 7, 2431, 0, 21, 252, 5, 2, 0, 4068, 2, 8, 4, 0, 0, 0, 0, 0, 19, 133, 3949, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4161, 5434, 4095, 4084, 2718, 4082, 4101] avg=4096.0000 max=5434 max_rank=2 min=2718 min_rank=5 imbalance_ratio=1.326660 max_load_reduction=27334 max_load_reduction_pct=0.834167 moved_to_echo=14985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 1, 4091, 2, 0, 0, 1, 0, 8, 1, 4083, 0, 0, 0, 479, 1, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 4093, 0, 3, 1, 0, 0, 0, 0, 1, 3616, 0, 1, 256, 5, 2, 0, 4092, 1, 1, 4, 0, 0, 0, 0, 0, 3, 20, 4083, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4093, 4301, 4096, 4097, 3881, 4098, 4106] avg=4096.0000 max=4301 max_rank=2 min=3881 min_rank=5 imbalance_ratio=1.050049 max_load_reduction=28467 max_load_reduction_pct=0.868744 moved_to_echo=16182 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 258, 5, 2, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4085, 3838, 4096, 4099, 4361, 4100, 4094] avg=4096.0000 max=4361 max_rank=5 min=3838 min_rank=2 imbalance_ratio=1.064697 max_load_reduction=28407 max_load_reduction_pct=0.866913 moved_to_echo=16654 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 258, 9, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4085, 3838, 4097, 4098, 4365, 4096, 4094] avg=4096.0000 max=4365 max_rank=5 min=3838 min_rank=2 imbalance_ratio=1.065674 max_load_reduction=28403 max_load_reduction_pct=0.866791 moved_to_echo=16653 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 258, 5, 2, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4085, 3838, 4096, 4099, 4361, 4100, 4094] avg=4096.0000 max=4361 max_rank=5 min=3838 min_rank=2 imbalance_ratio=1.064697 max_load_reduction=28407 max_load_reduction_pct=0.866913 moved_to_echo=16654 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 3836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 260, 5, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4086, 3836, 4096, 4096, 4366, 4096, 4091] avg=4096.0000 max=4366 max_rank=5 min=3836 min_rank=2 imbalance_ratio=1.065918 max_load_reduction=28402 max_load_reduction_pct=0.866760 moved_to_echo=16649 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 7, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 258, 6, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4088, 3845, 4096, 4096, 4355, 4096, 4094] avg=4096.0000 max=4355 max_rank=5 min=3845 min_rank=2 imbalance_ratio=1.063232 max_load_reduction=28413 max_load_reduction_pct=0.867096 moved_to_echo=16641 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2176, 128, 30720, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 256, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 30720, 0, 128, 2176, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 61, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4035, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 0, 4085, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4091, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4083, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 13, 0, 4087, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4090, 0, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 59, 0, 4074, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4059, 0, 5, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[16, 98, 11, 4086, 18, 11, 0, 4070, 9, 14, 11, 10, 4085, 25, 4, 6, 440, 0, 4087, 0, 12, 6, 0, 4082, 3915, 4044, 0, 3703, 4, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[2, 20, 1, 4095, 0, 1, 0, 4093, 1, 2, 1, 1, 4095, 3, 1, 1, 129, 0, 4095, 1, 1, 4, 1, 4094, 4073, 4089, 0, 3964, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 0, 4081, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4090, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 85, 17, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 83, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4096, 4112, 4094, 4016, 4096, 4174, 4101] avg=4096.0000 max=4174 max_rank=6 min=4016 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 85, 17, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 61, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 83, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4096, 4112, 4094, 4072, 4096, 4118, 4101] avg=4096.0000 max=4118 max_rank=6 min=4072 min_rank=4 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 85, 17, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 83, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4096, 4112, 4094, 4024, 4096, 4166, 4101] avg=4096.0000 max=4166 max_rank=6 min=4024 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[16, 98, 11, 4069, 0, 0, 0, 0, 18, 11, 0, 4070, 0, 0, 0, 0, 9, 14, 11, 10, 3830, 85, 17, 2, 4083, 25, 4, 6, 0, 0, 0, 0, 440, 0, 4002, 0, 0, 0, 0, 0, 12, 6, 0, 4082, 0, 0, 0, 0, 80, 0, 0, 3703, 0, 0, 0, 0, 4, 0, 0, 1, 4044, 5, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4099, 3978, 4118, 4442, 4100, 3783, 4054] avg=4096.0000 max=4442 max_rank=4 min=3783 min_rank=6 imbalance_ratio=1.084473 max_load_reduction=28326 max_load_reduction_pct=0.864441 moved_to_echo=16379 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 85, 17, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 83, 0, 0, 4090, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 5, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4096, 4112, 4093, 4015, 4096, 4173, 4104] avg=4096.0000 max=4173 max_rank=6 min=4015 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 20, 1, 4078, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 1, 2, 1, 1, 3985, 85, 17, 2, 4093, 3, 1, 1, 0, 0, 0, 0, 129, 0, 4010, 1, 0, 0, 0, 0, 1, 4, 1, 4094, 0, 0, 0, 0, 83, 0, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4094, 4094, 4098, 4140, 4100, 4047, 4094] avg=4096.0000 max=4140 max_rank=4 min=4047 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 91, 18, 8, 4083, 0, 0, 0, 0, 0, 0, 0, 59, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 75, 0, 0, 4059, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 11, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4096, 4127, 4083, 4042, 4096, 4134, 4112] avg=4096.0000 max=4134 max_rank=6 min=4042 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 85, 17, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 21, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 83, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4096, 4112, 4094, 4017, 4096, 4173, 4101] avg=4096.0000 max=4173 max_rank=6 min=4017 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32759, 32768, 32768, 32776, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32759, 32768, 32768, 32776, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32759, 32768, 32768, 32776, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32759, 32768, 32768, 32776, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32759, 32768, 32768, 32776, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32759, 32768, 32768, 32776, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32759, 32768, 32768, 32776, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32769, 32759, 32768, 32768, 32776, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 0, 32128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[768, 0, 0, 32128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32769 actual_valid_recv=33024 observed_loss=-255 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32759 actual_valid_recv=33536 observed_loss=-777 tokens_per_expert=[128, 128, 128, 128, 31872, 768, 256, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32776 actual_valid_recv=33152 observed_loss=-376 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[14, 4095, 4095, 4016, 2, 2, 93, 29, 1, 0, 0, 8, 1, 4042, 0, 1, 0, 1, 1, 0, 0, 2, 0, 0, 0, 0, 4095, 1, 0, 4094, 4094, 4081] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4087, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[76, 4086, 4083, 3595, 17, 17, 512, 171, 7, 0, 0, 4, 11, 3902, 0, 7, 0, 8, 13, 0, 0, 15, 0, 0, 0, 0, 4085, 10, 0, 4081, 4079, 3989] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4056, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4029, 4095, 4096, 0, 0, 1, 0, 0, 0, 0, 67, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 4088, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4060, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4090, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4078, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 18, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4095, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 62, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 7, 0, 0, 0, 0, 0, 40, 4073, 3, 0, 0, 0, 4096, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 3973, 4116, 4124, 4095, 4094, 4096, 4112] avg=4096.0000 max=4158 max_rank=0 min=3973 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16397 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[76, 63, 0, 3595, 0, 0, 0, 0, 17, 17, 512, 171, 3995, 7, 0, 0, 7, 0, 0, 4, 4058, 3, 0, 0, 11, 3902, 0, 7, 28, 0, 0, 0, 0, 8, 13, 0, 4083, 0, 0, 0, 0, 15, 0, 0, 4076, 0, 0, 0, 0, 0, 4085, 10, 0, 0, 0, 0, 0, 16, 0, 3989, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3734, 4719, 4072, 3948, 4104, 4091, 4095, 4005] avg=4096.0000 max=4719 max_rank=1 min=3734 min_rank=0 imbalance_ratio=1.152100 max_load_reduction=28049 max_load_reduction_pct=0.855988 moved_to_echo=16295 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 63, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 7, 0, 0, 0, 0, 0, 9, 4073, 2, 0, 0, 0, 4096, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4003, 4084, 4124, 4094, 4096, 4096, 4112] avg=4096.0000 max=4159 max_rank=0 min=4003 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16398 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 63, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 7, 0, 0, 0, 0, 0, 0, 4073, 3, 0, 0, 0, 4096, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4012, 4076, 4124, 4095, 4094, 4096, 4112] avg=4096.0000 max=4159 max_rank=0 min=4012 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16397 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[14, 63, 0, 4016, 0, 0, 0, 0, 2, 2, 93, 29, 4004, 7, 0, 0, 1, 0, 0, 8, 4071, 2, 0, 0, 1, 4042, 0, 1, 28, 0, 0, 0, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 2, 0, 0, 4092, 1, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 16, 0, 4081, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4137, 4082, 4072, 4096, 4095, 4096, 4097] avg=4096.0000 max=4137 max_rank=1 min=4072 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16384 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 52, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 3943, 9, 0, 0, 0, 0, 0, 67, 4069, 3, 0, 0, 0, 4096, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 8, 0, 4093, 1, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 3953, 4139, 4129, 4094, 4102, 4097, 4106] avg=4096.0000 max=4148 max_rank=0 min=3953 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16399 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 62, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 7, 0, 0, 0, 0, 0, 36, 4067, 3, 0, 0, 0, 4096, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 6, 0, 4093, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 3977, 4106, 4124, 4095, 4100, 4096, 4112] avg=4096.0000 max=4158 max_rank=0 min=3977 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16403 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 63, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 7, 0, 0, 0, 0, 1, 18, 4072, 2, 0, 0, 0, 4095, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 4094, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 3994, 4093, 4123, 4095, 4096, 4096, 4112] avg=4096.0000 max=4159 max_rank=0 min=3994 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16399 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 512, 0, 32256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 640, 256, 31872, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 0, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True + [2026-08-30 04:55:25] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 670.2 | throughput per GPU (TFLOP/s/GPU): 497.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.535870E+00 | loss scale: 1.0 | grad norm: 5.809 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[312, 0, 1170, 17, 1584, 0, 2907, 18, 1654, 1839, 2, 64, 2027, 3318, 11, 534, 22, 941, 2030, 0, 0, 467, 801, 24, 2, 3219, 3448, 1568, 3132, 1504, 152, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[381, 11, 2067, 26, 1058, 0, 2470, 52, 1687, 1513, 0, 21, 2016, 3143, 14, 379, 6, 1316, 1819, 0, 4, 697, 689, 116, 3, 2783, 3677, 1746, 3139, 1767, 166, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[238, 0, 2029, 5, 736, 0, 2613, 8, 1234, 1695, 0, 20, 1858, 3698, 2, 544, 4, 1267, 2003, 0, 0, 620, 1096, 15, 2, 2658, 3554, 1744, 3250, 1803, 72, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[355, 0, 1683, 11, 1505, 1, 2131, 49, 1673, 1120, 0, 23, 1350, 3523, 11, 695, 13, 1556, 2531, 0, 0, 572, 1299, 79, 0, 2851, 3829, 1693, 2709, 1364, 142, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[195, 0, 1840, 14, 1261, 0, 2353, 32, 1109, 1133, 0, 27, 1816, 3201, 8, 757, 30, 1192, 2373, 0, 1, 527, 1523, 71, 8, 2992, 3796, 1898, 2918, 1622, 70, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[230, 1, 1850, 12, 1336, 1, 2519, 10, 1407, 1365, 0, 21, 1189, 3637, 5, 595, 24, 1576, 2258, 0, 0, 594, 914, 14, 0, 2941, 3766, 1916, 3100, 1440, 47, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[184, 0, 1547, 8, 1392, 0, 2760, 5, 1504, 934, 0, 37, 1793, 3236, 9, 579, 11, 1115, 2560, 0, 0, 883, 957, 38, 0, 3031, 3773, 1795, 2872, 1685, 60, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[230, 0, 1511, 23, 1447, 1, 2657, 6, 1294, 1477, 0, 27, 1125, 3464, 5, 790, 2, 1603, 2017, 0, 0, 766, 1127, 32, 0, 3087, 3947, 2243, 2401, 1457, 26, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[312, 0, 1170, 17, 1267, 715, 0, 0, 1584, 0, 2907, 18, 247, 0, 0, 0, 1654, 1839, 2, 64, 815, 436, 0, 0, 2027, 1788, 11, 534, 0, 0, 0, 0, 22, 941, 2030, 0, 303, 303, 0, 0, 0, 467, 801, 24, 2181, 0, 0, 0, 2, 2480, 0, 1568, 0, 0, 0, 0, 2582, 1504, 152, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3481, 4756, 4810, 4360, 3599, 3473, 4050, 4239] avg=4096.0000 max=4810 max_rank=2 min=3473 min_rank=5 imbalance_ratio=1.174316 max_load_reduction=27958 max_load_reduction_pct=0.853210 moved_to_echo=15361 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[381, 11, 2067, 26, 1351, 677, 0, 0, 1058, 0, 2470, 52, 247, 0, 0, 0, 1687, 1513, 0, 21, 772, 377, 0, 0, 2016, 1694, 14, 379, 0, 0, 0, 0, 6, 1316, 1819, 0, 304, 261, 0, 0, 4, 697, 689, 116, 2326, 0, 0, 0, 3, 2145, 0, 1746, 0, 0, 0, 0, 2588, 1767, 166, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4513, 3827, 4370, 4103, 3706, 3832, 3894, 4523] avg=4096.0000 max=4523 max_rank=7 min=3706 min_rank=4 imbalance_ratio=1.104248 max_load_reduction=28245 max_load_reduction_pct=0.861969 moved_to_echo=15955 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[238, 0, 2029, 5, 1306, 797, 0, 0, 736, 0, 2613, 8, 256, 0, 0, 0, 1234, 1695, 0, 20, 909, 360, 0, 0, 1858, 1992, 2, 544, 0, 0, 0, 0, 4, 1267, 2003, 0, 315, 250, 0, 0, 0, 620, 1096, 15, 2248, 0, 0, 0, 2, 2048, 0, 1744, 0, 0, 0, 0, 2679, 1803, 72, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4375, 3613, 4218, 4396, 3839, 3979, 3794, 4554] avg=4096.0000 max=4554 max_rank=7 min=3613 min_rank=1 imbalance_ratio=1.111816 max_load_reduction=28214 max_load_reduction_pct=0.861023 moved_to_echo=16166 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[184, 0, 1547, 8, 1386, 697, 0, 0, 1392, 0, 2760, 5, 226, 0, 0, 0, 1504, 934, 0, 37, 795, 410, 0, 0, 1793, 1744, 9, 579, 0, 0, 0, 0, 11, 1115, 2560, 0, 278, 285, 0, 0, 0, 883, 957, 38, 2387, 0, 0, 0, 0, 2336, 0, 1795, 0, 0, 0, 0, 2368, 1685, 60, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3822, 4383, 3680, 4125, 4249, 4265, 4131, 4113] avg=4096.0000 max=4383 max_rank=1 min=3680 min_rank=2 imbalance_ratio=1.070068 max_load_reduction=28385 max_load_reduction_pct=0.866241 moved_to_echo=16758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[230, 1, 1850, 12, 1384, 784, 0, 0, 1336, 1, 2519, 10, 244, 0, 0, 0, 1407, 1365, 0, 21, 894, 398, 0, 0, 1189, 1959, 5, 595, 0, 0, 0, 0, 24, 1576, 2258, 0, 300, 276, 0, 0, 0, 594, 914, 14, 2382, 0, 0, 0, 0, 2267, 0, 1916, 0, 0, 0, 0, 2556, 1440, 47, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4261, 4110, 4085, 3748, 4434, 3904, 4183, 4043] avg=4096.0000 max=4434 max_rank=4 min=3748 min_rank=3 imbalance_ratio=1.082520 max_load_reduction=28334 max_load_reduction_pct=0.864685 moved_to_echo=16564 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[355, 0, 1683, 11, 1407, 759, 0, 0, 1505, 1, 2131, 49, 213, 0, 0, 0, 1673, 1120, 0, 23, 866, 386, 0, 0, 1350, 1898, 11, 695, 0, 0, 0, 0, 13, 1556, 2531, 0, 262, 268, 0, 0, 0, 572, 1299, 79, 2422, 0, 0, 0, 0, 2197, 0, 1693, 0, 0, 0, 0, 2234, 1364, 142, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4215, 3899, 4068, 3954, 4630, 4372, 3890, 3740] avg=4096.0000 max=4630 max_rank=4 min=3740 min_rank=7 imbalance_ratio=1.130371 max_load_reduction=28138 max_load_reduction_pct=0.858704 moved_to_echo=16632 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[195, 0, 1840, 14, 1396, 694, 0, 0, 1261, 0, 2353, 32, 234, 0, 0, 0, 1109, 1133, 0, 27, 792, 409, 0, 0, 1816, 1715, 8, 757, 0, 0, 0, 0, 30, 1192, 2373, 0, 287, 285, 0, 0, 1, 527, 1523, 71, 2400, 0, 0, 0, 8, 2298, 0, 1898, 0, 0, 0, 0, 2397, 1622, 70, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 3880, 3470, 4296, 4167, 4522, 4204, 4090] avg=4096.0000 max=4522 max_rank=5 min=3470 min_rank=2 imbalance_ratio=1.104004 max_load_reduction=28246 max_load_reduction_pct=0.862000 moved_to_echo=16983 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[230, 0, 1511, 23, 1451, 747, 0, 0, 1447, 1, 2657, 6, 189, 0, 0, 0, 1294, 1477, 0, 27, 851, 418, 0, 0, 1125, 1866, 5, 790, 0, 0, 0, 0, 2, 1603, 2017, 0, 232, 290, 0, 0, 0, 766, 1127, 32, 2496, 0, 0, 0, 0, 2379, 0, 2243, 0, 0, 0, 0, 1980, 1457, 26, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3962, 4300, 4067, 3786, 4144, 4421, 4622, 3466] avg=4096.0000 max=4622 max_rank=6 min=3466 min_rank=7 imbalance_ratio=1.128418 max_load_reduction=28146 max_load_reduction_pct=0.858948 moved_to_echo=16653 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2176, 128, 13824, 128, 11008, 5888, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 5248, 8448, 512, 18944, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 18176, 0, 14720, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 10624, 17664, 0, 2304, 2304, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10368, 128, 20480, 256, 1920, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13184, 14720, 128, 4992, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[19456, 12672, 768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11648, 11136, 128, 256, 6784, 3200, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1312, 0, 1, 0, 1747, 0, 0, 0, 0, 2, 3390, 0, 0, 1, 4095, 4096, 4094, 0, 0, 0, 2975, 0, 0, 0, 4096, 2863, 0, 0, 0, 4089, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[191, 0, 0, 0, 1446, 0, 0, 0, 0, 45, 2561, 0, 45, 0, 4096, 4096, 4096, 0, 0, 0, 3974, 0, 0, 0, 4094, 4028, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[93, 1, 0, 0, 659, 0, 0, 1, 0, 29, 3370, 0, 1, 0, 4093, 4096, 4095, 0, 1, 0, 4049, 0, 1, 0, 4090, 4093, 0, 0, 0, 4095, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[415, 0, 0, 0, 652, 0, 0, 0, 0, 4, 3260, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 3908, 0, 0, 0, 4093, 4053, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[513, 0, 0, 0, 1379, 0, 0, 0, 0, 42, 2664, 3, 0, 1, 4096, 4096, 4093, 0, 0, 0, 3640, 0, 0, 0, 4094, 4065, 0, 0, 0, 4082, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[951, 0, 0, 0, 2062, 0, 0, 0, 0, 2, 1741, 0, 3, 0, 4096, 4096, 4096, 0, 0, 0, 3550, 0, 0, 0, 4096, 3979, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[438, 0, 0, 2, 774, 0, 0, 3, 0, 5, 2970, 4, 15, 0, 4094, 4096, 4092, 0, 0, 0, 4013, 0, 0, 0, 4089, 4076, 0, 0, 1, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[746, 0, 0, 1, 1668, 0, 0, 0, 2, 9, 2268, 0, 17, 4, 4096, 4087, 4096, 1, 0, 0, 3676, 0, 0, 0, 4095, 3891, 0, 0, 0, 4096, 1, 14] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[191, 0, 0, 0, 3513, 0, 0, 0, 1446, 0, 0, 0, 2215, 583, 0, 0, 0, 45, 2561, 0, 1299, 0, 0, 0, 45, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 363, 9, 0, 0, 217, 4028, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3704, 4244, 3905, 4132, 4096, 4346, 4245, 4096] avg=4096.0000 max=4346 max_rank=5 min=3704 min_rank=0 imbalance_ratio=1.061035 max_load_reduction=28422 max_load_reduction_pct=0.867371 moved_to_echo=16783 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[513, 0, 0, 0, 3514, 0, 0, 0, 1379, 0, 0, 0, 2215, 582, 0, 0, 0, 42, 2664, 3, 1299, 0, 0, 0, 0, 1, 0, 4087, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3640, 0, 0, 0, 363, 9, 0, 0, 217, 4065, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4176, 4008, 4088, 4093, 4012, 4282, 4082] avg=4096.0000 max=4282 max_rank=6 min=4008 min_rank=2 imbalance_ratio=1.045410 max_load_reduction=28486 max_load_reduction_pct=0.869324 moved_to_echo=16469 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[438, 0, 0, 2, 3513, 0, 0, 0, 774, 0, 0, 3, 2212, 581, 0, 0, 0, 5, 2970, 4, 1298, 0, 0, 0, 15, 0, 0, 4087, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 363, 9, 0, 0, 216, 4076, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3953, 3570, 4277, 4102, 4092, 4385, 4292, 4097] avg=4096.0000 max=4385 max_rank=5 min=3570 min_rank=1 imbalance_ratio=1.070557 max_load_reduction=28383 max_load_reduction_pct=0.866180 moved_to_echo=16866 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1312, 0, 1, 0, 3512, 0, 0, 0, 1747, 0, 0, 0, 2216, 583, 0, 0, 0, 2, 3390, 0, 1300, 0, 0, 0, 0, 1, 0, 4087, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2975, 0, 0, 0, 363, 9, 0, 0, 217, 2863, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4825, 4546, 4692, 4088, 4094, 3347, 3080, 4096] avg=4096.0000 max=4825 max_rank=0 min=3080 min_rank=6 imbalance_ratio=1.177979 max_load_reduction=27943 max_load_reduction_pct=0.852753 moved_to_echo=14617 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[415, 0, 0, 0, 3512, 0, 0, 0, 652, 0, 0, 0, 2214, 583, 0, 0, 0, 4, 3260, 0, 1299, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3908, 0, 0, 0, 363, 9, 0, 0, 217, 4053, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3927, 3449, 4563, 4087, 4096, 4280, 4270, 4096] avg=4096.0000 max=4563 max_rank=2 min=3449 min_rank=1 imbalance_ratio=1.114014 max_load_reduction=28205 max_load_reduction_pct=0.860748 moved_to_echo=16742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[93, 1, 0, 0, 3512, 0, 0, 0, 659, 0, 0, 1, 2213, 581, 0, 0, 0, 29, 3370, 0, 1298, 0, 0, 0, 1, 0, 0, 4087, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 4049, 0, 1, 0, 363, 9, 0, 0, 216, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3606, 3454, 4697, 4088, 4096, 4422, 4309, 4096] avg=4096.0000 max=4697 max_rank=2 min=3454 min_rank=1 imbalance_ratio=1.146729 max_load_reduction=28071 max_load_reduction_pct=0.856659 moved_to_echo=16923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[746, 0, 0, 1, 3514, 0, 0, 0, 1668, 0, 0, 0, 2219, 582, 0, 0, 2, 9, 2268, 0, 1304, 0, 0, 0, 17, 4, 0, 4072, 0, 0, 0, 0, 4096, 1, 0, 0, 6, 2, 0, 0, 3676, 0, 0, 0, 369, 9, 0, 0, 203, 3891, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 14, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4261, 4469, 3583, 4093, 4105, 4054, 4094, 4109] avg=4096.0000 max=4469 max_rank=1 min=3583 min_rank=2 imbalance_ratio=1.091064 max_load_reduction=28299 max_load_reduction_pct=0.863617 moved_to_echo=16362 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[951, 0, 0, 0, 3514, 0, 0, 0, 2062, 0, 0, 0, 2216, 582, 0, 0, 0, 2, 1741, 0, 1300, 0, 0, 0, 3, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3550, 0, 0, 0, 363, 9, 0, 0, 217, 3979, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4465, 4860, 3043, 4090, 4096, 3922, 4196, 4096] avg=4096.0000 max=4860 max_rank=1 min=3043 min_rank=2 imbalance_ratio=1.186523 max_load_reduction=27908 max_load_reduction_pct=0.851685 moved_to_echo=16310 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10496, 0, 0, 128, 17792, 4736, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 22272, 128, 10496, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4736, 128, 128, 128, 28160, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[1792, 31104, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[29824, 0, 128, 0, 2944, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[6, 26, 4088, 0, 0, 0, 4096, 0, 4086, 4096, 0, 0, 4094, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 3, 3509, 94, 0, 0, 0, 1249, 0, 3320] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4, 5, 4085, 0, 4, 0, 4096, 0, 4060, 4096, 0, 0, 4093, 0, 0, 10, 0, 0, 0, 0, 0, 17, 4096, 7, 3941, 9, 2, 0, 0, 165, 0, 4078] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 33, 4064, 0, 0, 0, 4096, 1, 3975, 4096, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 2, 3910, 58, 0, 0, 0, 258, 0, 4082] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3943, 0, 0, 0, 4096, 0, 4096, 4096, 1, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 12, 4096, 4, 3741, 6, 0, 0, 0, 519, 0, 4056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3954, 1, 2, 0, 4095, 0, 4090, 4093, 2, 1, 4095, 0, 0, 1, 1, 1, 0, 1, 0, 4, 4095, 0, 3878, 81, 0, 1, 0, 283, 0, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 9, 4076, 0, 0, 0, 4096, 0, 4057, 4096, 0, 0, 4093, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 3880, 9, 3, 0, 0, 370, 0, 3979] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 11, 4078, 0, 0, 0, 4096, 0, 4071, 4096, 0, 0, 4092, 0, 0, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 3279, 20, 0, 0, 0, 865, 0, 4045] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 14, 3938, 0, 0, 0, 4096, 0, 4022, 4096, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 8, 3743, 16, 0, 0, 0, 699, 0, 3942] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 5, 4085, 0, 43, 7, 4, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 4060, 39, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 39, 0, 0, 0, 17, 4089, 7, 0, 0, 0, 0, 3941, 9, 2, 0, 334, 0, 0, 0, 0, 165, 0, 3662, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4100, 4099, 4099, 4096, 4113, 4286, 3827] avg=4096.0000 max=4286 max_rank=6 min=3827 min_rank=7 imbalance_ratio=1.046387 max_load_reduction=28482 max_load_reduction_pct=0.869202 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[6, 26, 4088, 0, 35, 7, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4086, 39, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 32, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 3509, 94, 0, 0, 271, 0, 0, 0, 0, 1249, 0, 2982, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4096, 4125, 4095, 4089, 4092, 3874, 4231] avg=4096.0000 max=4231 max_rank=7 min=3874 min_rank=6 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 4078, 0, 43, 7, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4071, 39, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 39, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 3279, 20, 0, 0, 331, 0, 0, 0, 0, 865, 0, 3632, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4096, 4110, 4107, 4096, 4089, 3630, 4497] avg=4096.0000 max=4497 max_rank=7 min=3630 min_rank=6 imbalance_ratio=1.097900 max_load_reduction=28271 max_load_reduction_pct=0.862762 moved_to_echo=16312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 33, 4064, 0, 43, 7, 4, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 3975, 39, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 39, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 3910, 58, 0, 0, 334, 0, 0, 0, 0, 258, 0, 3666, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4097, 4014, 4093, 4096, 4091, 4302, 3924] avg=4096.0000 max=4302 max_rank=6 min=3924 min_rank=7 imbalance_ratio=1.050293 max_load_reduction=28466 max_load_reduction_pct=0.868713 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3943, 0, 43, 7, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 39, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 39, 0, 0, 0, 12, 4089, 4, 0, 0, 0, 0, 3741, 6, 0, 0, 332, 0, 0, 0, 0, 519, 0, 3642, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 4096, 4136, 4097, 4096, 4105, 4079, 4161] avg=4096.0000 max=4161 max_rank=7 min=3998 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3954, 1, 43, 6, 4, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 4090, 39, 2, 1, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 4054, 39, 0, 0, 0, 4, 4089, 0, 0, 0, 0, 0, 3878, 81, 0, 1, 334, 0, 0, 0, 0, 283, 0, 3673, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4008, 4097, 4132, 4092, 4096, 4093, 4294, 3956] avg=4096.0000 max=4294 max_rank=6 min=3956 min_rank=7 imbalance_ratio=1.048340 max_load_reduction=28474 max_load_reduction_pct=0.868958 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 4076, 0, 45, 8, 6, 6, 0, 0, 4090, 0, 0, 0, 0, 0, 4057, 34, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 42, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 3880, 9, 3, 0, 330, 0, 0, 0, 0, 370, 0, 3562, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4090, 4091, 4091, 4104, 4088, 4222, 3932] avg=4096.0000 max=4222 max_rank=6 min=3932 min_rank=7 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 14, 3938, 0, 41, 7, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4022, 39, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 38, 0, 0, 0, 0, 4089, 8, 0, 0, 0, 0, 3743, 16, 0, 0, 322, 0, 0, 0, 0, 699, 0, 3541, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4004, 4096, 4061, 4094, 4095, 4097, 4081, 4240] avg=4096.0000 max=4240 max_rank=7 min=4004 min_rank=0 imbalance_ratio=1.035156 max_load_reduction=28528 max_load_reduction_pct=0.870605 moved_to_echo=16513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 4480, 0, 28416, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32256, 128, 384, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32512, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 384, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[29952, 384, 128, 128, 2688, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 4096, 0, 4074, 4096, 307, 3, 0, 0, 0, 0, 3813, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 4096, 0, 4081, 4096, 175, 1, 0, 0, 0, 0, 3942, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 4096, 0, 4081, 4096, 240, 4, 0, 0, 0, 0, 3877, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 4094, 0, 4095, 4096, 209, 0, 1, 0, 0, 0, 3899, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 1, 0, 4095, 1, 1, 4095, 4092, 0, 4093, 4092, 321, 1, 3, 5, 0, 1, 3774, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 4096, 0, 4086, 4096, 235, 0, 0, 0, 0, 0, 3900, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 4082, 4096, 313, 8, 0, 0, 0, 0, 3791, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 4096, 0, 4094, 4096, 436, 0, 0, 0, 0, 0, 3672, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4078, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 8, 0, 0, 4090, 0, 0, 0, 0, 11, 0, 4074, 0, 0, 0, 0, 0, 307, 3, 0, 0, 3813, 0, 0, 0, 0, 0, 3813, 264, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4097, 4095, 4098, 4085, 4123, 4077] avg=4096.0000 max=4123 max_rank=6 min=4077 min_rank=7 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4078, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 8, 0, 0, 4086, 0, 0, 0, 0, 11, 0, 4081, 0, 0, 0, 0, 0, 240, 4, 0, 0, 3813, 0, 0, 0, 0, 0, 3877, 264, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4097, 4095, 4094, 4092, 4057, 4141] avg=4096.0000 max=4141 max_rank=7 min=4057 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4078, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 8, 0, 0, 4081, 0, 0, 0, 0, 11, 0, 4086, 0, 0, 0, 0, 0, 235, 0, 0, 0, 3813, 0, 0, 0, 0, 0, 3900, 264, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4083, 4095, 4089, 4097, 4048, 4164] avg=4096.0000 max=4164 max_rank=7 min=4048 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4078, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 8, 0, 0, 4089, 0, 0, 0, 0, 11, 0, 4081, 0, 0, 0, 0, 0, 175, 1, 0, 0, 3813, 0, 0, 0, 0, 0, 3942, 264, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4097, 4095, 4097, 4092, 3989, 4206] avg=4096.0000 max=4206 max_rank=7 min=3989 min_rank=6 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4076, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 8, 0, 0, 4087, 0, 0, 0, 0, 11, 0, 4095, 0, 0, 0, 0, 0, 209, 0, 1, 0, 3812, 0, 0, 0, 0, 0, 3899, 264, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4097, 4095, 4095, 4106, 4022, 4163] avg=4096.0000 max=4163 max_rank=7 min=4022 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4074, 18, 0, 0, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 4087, 7, 0, 0, 8, 1, 1, 4095, 0, 0, 0, 0, 11, 0, 4093, 0, 0, 0, 0, 0, 321, 1, 3, 5, 3811, 0, 0, 0, 0, 1, 3774, 264, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4093, 4097, 4096, 4105, 4104, 4141, 4039] avg=4096.0000 max=4141 max_rank=6 min=4039 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4080, 20, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4091, 11, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 5, 0, 4082, 0, 0, 0, 0, 0, 313, 8, 0, 0, 3818, 0, 0, 0, 0, 0, 3791, 251, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4099, 4103, 4102, 4096, 4087, 4139, 4042] avg=4096.0000 max=4139 max_rank=6 min=4042 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4078, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 8, 0, 0, 4086, 0, 0, 0, 0, 11, 0, 4094, 0, 0, 0, 0, 0, 436, 0, 0, 0, 3813, 0, 0, 0, 0, 0, 3672, 264, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4097, 4095, 4094, 4105, 4249, 3936] avg=4096.0000 max=4249 max_rank=6 min=3936 min_rank=7 imbalance_ratio=1.037354 max_load_reduction=28519 max_load_reduction_pct=0.870331 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 30720, 2176, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2304, 128, 128, 128, 30592, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1563, 0, 4096, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 26, 4090, 0, 4090, 0, 2526, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 1339, 0, 4093, 4095, 1, 1, 0, 0, 4095, 1, 0, 0, 4095, 4091, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4066, 0, 4094, 1, 2792, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1542, 1, 4065, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4032, 0, 4094, 0, 2649, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1342, 0, 4012, 4096, 0, 0, 2, 0, 4092, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4002, 0, 4086, 0, 2932, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 769, 0, 4093, 4096, 0, 0, 5, 0, 4090, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 19, 0, 12, 4077, 0, 4087, 0, 3320, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1483, 0, 4041, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 12, 0, 10, 4069, 0, 4090, 0, 2684, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1515, 0, 4089, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4053, 0, 4089, 0, 2630, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1627, 0, 4045, 4096, 0, 0, 0, 0, 4072, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4053, 0, 4085, 0, 2564, 9] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1563, 2698, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 4066, 24, 0, 0, 0, 26, 4090, 0, 28, 0, 0, 0, 1333, 0, 2526, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4261, 4126, 4093, 4096, 4097, 4092, 4144, 3859] avg=4096.0000 max=4261 max_rank=0 min=3859 min_rank=7 imbalance_ratio=1.040283 max_load_reduction=28507 max_load_reduction_pct=0.869965 moved_to_echo=16192 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1342, 2695, 0, 0, 0, 0, 4012, 30, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 24, 0, 0, 0, 8, 4002, 0, 28, 0, 0, 0, 1333, 0, 2932, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4042, 4100, 4096, 4095, 4090, 4038, 4270] avg=4096.0000 max=4270 max_rank=7 min=4037 min_rank=0 imbalance_ratio=1.042480 max_load_reduction=28498 max_load_reduction_pct=0.869690 moved_to_echo=16493 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 1339, 2701, 0, 0, 0, 0, 4093, 29, 1, 0, 0, 0, 0, 1, 0, 0, 4095, 6, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 4091, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 4066, 24, 0, 0, 0, 0, 4066, 0, 28, 0, 0, 0, 1334, 1, 2792, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4123, 4102, 4096, 4092, 4091, 4094, 4127] avg=4096.0000 max=4127 max_rank=7 min=4043 min_rank=0 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16404 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1483, 2698, 0, 0, 0, 0, 4041, 30, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 4066, 24, 0, 0, 0, 10, 4069, 0, 28, 0, 0, 0, 1333, 0, 2684, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4071, 4094, 4096, 4097, 4102, 4107, 4020] avg=4096.0000 max=4181 max_rank=0 min=4020 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16326 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1542, 2701, 0, 0, 0, 1, 4065, 30, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 4066, 24, 0, 0, 0, 0, 4032, 0, 28, 0, 0, 0, 1334, 0, 2649, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4243, 4096, 4102, 4096, 4096, 4092, 4060, 3983] avg=4096.0000 max=4243 max_rank=0 min=3983 min_rank=7 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16231 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1627, 2699, 0, 0, 0, 0, 4045, 23, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4073, 29, 0, 0, 0, 26, 4053, 0, 32, 0, 0, 0, 1309, 0, 2564, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4326, 4068, 4084, 4096, 4099, 4102, 4111, 3882] avg=4096.0000 max=4326 max_rank=0 min=3882 min_rank=7 imbalance_ratio=1.056152 max_load_reduction=28442 max_load_reduction_pct=0.867981 moved_to_echo=16194 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1515, 2697, 0, 0, 0, 0, 4089, 30, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 24, 0, 0, 0, 16, 4053, 0, 28, 0, 0, 0, 1334, 0, 2630, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4212, 4119, 4092, 4096, 4096, 4090, 4097, 3966] avg=4096.0000 max=4212 max_rank=0 min=3966 min_rank=7 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16249 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 769, 2696, 0, 0, 0, 0, 4093, 30, 0, 0, 0, 0, 0, 0, 5, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4066, 24, 0, 0, 0, 12, 4077, 0, 28, 0, 0, 0, 1333, 0, 3320, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3465, 4123, 4101, 4096, 4096, 4109, 4117, 4661] avg=4096.0000 max=4661 max_rank=7 min=3465 min_rank=0 imbalance_ratio=1.137939 max_load_reduction=28107 max_load_reduction_pct=0.857758 moved_to_echo=16983 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 11264, 21632, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10752, 128, 22144, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32512, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3333, 0, 4094, 0, 0, 3862, 0, 0, 0, 5, 15, 4095, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 981, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2693, 0, 4096, 0, 0, 3704, 0, 0, 0, 34, 39, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1722, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 2, 0, 528, 9, 4093, 1, 1, 3950, 0, 0, 0, 0, 0, 4087, 1, 4092, 1, 1, 4095, 1, 0, 4095, 1, 1, 4, 3709, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3757, 0, 4096, 0, 0, 3762, 0, 0, 0, 8, 15, 4096, 0, 4096, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 644, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2533, 6, 4095, 0, 0, 3474, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 2181, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2697, 0, 4096, 0, 0, 3218, 0, 0, 0, 30, 43, 4095, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 2204, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3310, 0, 4096, 0, 0, 3299, 0, 0, 0, 18, 5, 4096, 0, 4096, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 1557, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2761, 0, 4096, 0, 0, 3111, 0, 0, 0, 5, 25, 4095, 0, 4090, 0, 0, 4093, 6, 0, 4096, 0, 0, 0, 2294, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3333, 0, 1394, 0, 0, 0, 0, 0, 0, 3862, 0, 0, 518, 28, 0, 0, 0, 5, 15, 4067, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 981, 2182, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4727, 4408, 4087, 4096, 4095, 3163, 4096] avg=4096.0000 max=4727 max_rank=1 min=3163 min_rank=6 imbalance_ratio=1.154053 max_load_reduction=28041 max_load_reduction_pct=0.855743 moved_to_echo=15450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2693, 0, 1393, 0, 0, 0, 0, 0, 0, 3704, 0, 0, 519, 28, 0, 0, 0, 34, 39, 4068, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1722, 2184, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4086, 4251, 4141, 4097, 4095, 3906, 4096] avg=4096.0000 max=4251 max_rank=2 min=3906 min_rank=6 imbalance_ratio=1.037842 max_load_reduction=28517 max_load_reduction_pct=0.870270 moved_to_echo=16194 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3757, 0, 1393, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 519, 28, 0, 0, 0, 8, 15, 4068, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 644, 2184, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 5150, 4309, 4091, 4097, 4101, 2828, 4096] avg=4096.0000 max=5150 max_rank=1 min=2828 min_rank=6 imbalance_ratio=1.257324 max_load_reduction=27618 max_load_reduction_pct=0.842834 moved_to_echo=15122 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 2, 0, 0, 0, 0, 0, 528, 9, 1393, 1, 0, 0, 0, 0, 1, 3950, 0, 0, 518, 28, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 1, 4092, 1, 1, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 1, 1, 4, 3709, 2182, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 1931, 4497, 4059, 4095, 4096, 5897, 4096] avg=4096.0000 max=5897 max_rank=6 min=1931 min_rank=1 imbalance_ratio=1.439697 max_load_reduction=26871 max_load_reduction_pct=0.820038 moved_to_echo=18184 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2533, 6, 1393, 0, 0, 0, 0, 0, 0, 3474, 0, 0, 519, 28, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2181, 2183, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3932, 4021, 4068, 4095, 4095, 4365, 4096] avg=4096.0000 max=4365 max_rank=6 min=3932 min_rank=1 imbalance_ratio=1.065674 max_load_reduction=28403 max_load_reduction_pct=0.866791 moved_to_echo=16651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 2697, 0, 1388, 0, 0, 0, 0, 0, 0, 3218, 0, 0, 522, 34, 4, 0, 0, 30, 43, 4061, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 1, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2204, 2186, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4085, 3778, 4134, 4101, 4089, 4390, 4096] avg=4096.0000 max=4390 max_rank=6 min=3778 min_rank=2 imbalance_ratio=1.071777 max_load_reduction=28378 max_load_reduction_pct=0.866028 moved_to_echo=16676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2761, 0, 1393, 0, 0, 0, 0, 0, 0, 3111, 0, 0, 519, 28, 0, 0, 0, 5, 25, 4067, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4093, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2294, 2184, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4154, 3658, 4097, 4090, 4099, 4478, 4096] avg=4096.0000 max=4478 max_rank=6 min=3658 min_rank=2 imbalance_ratio=1.093262 max_load_reduction=28290 max_load_reduction_pct=0.863342 moved_to_echo=16763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3310, 0, 1393, 0, 0, 0, 0, 0, 0, 3299, 0, 0, 519, 28, 0, 0, 0, 18, 5, 4068, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1557, 2184, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4703, 3846, 4091, 4097, 4098, 3741, 4096] avg=4096.0000 max=4703 max_rank=1 min=3741 min_rank=6 imbalance_ratio=1.148193 max_load_reduction=28065 max_load_reduction_pct=0.856476 moved_to_echo=16032 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 15360, 17536, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[21632, 128, 11264, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 28416, 0, 0, 4224, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 256, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4086, 0, 4096, 4092, 0, 0, 0, 0, 4, 10, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 4083, 0, 4096, 4029, 0, 0, 0, 0, 67, 10, 0, 5, 1, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 4063, 0, 4096, 4074, 0, 0, 0, 0, 39, 1, 0, 15, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 4093, 0, 4096, 4052, 0, 0, 0, 1, 46, 0, 0, 1, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 4091, 0, 4095, 4068, 1, 0, 0, 4, 26, 0, 1, 0, 0, 1, 4095, 4095, 1, 4094, 0, 1, 1, 4095, 0, 2, 0, 0, 1, 0, 4094, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 4082, 0, 4096, 3948, 0, 0, 0, 0, 148, 7, 0, 7, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 4073, 0, 4096, 4077, 0, 0, 0, 0, 20, 0, 0, 22, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 4081, 0, 4096, 4089, 0, 0, 0, 0, 7, 10, 0, 10, 4, 0, 4096, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 0, 16, 0, 0, 0, 0, 4092, 0, 0, 0, 35, 7, 0, 0, 0, 4, 10, 0, 4045, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4134, 4059, 4089, 4097, 4096, 4095, 4096] avg=4096.0000 max=4134 max_rank=1 min=4059 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4083, 0, 16, 0, 0, 0, 0, 4029, 0, 0, 0, 35, 7, 0, 0, 0, 67, 10, 0, 4045, 0, 0, 0, 5, 1, 0, 4089, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4071, 4122, 4095, 4097, 4096, 4095, 4093] avg=4096.0000 max=4122 max_rank=2 min=4071 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 16, 0, 0, 0, 0, 4052, 0, 0, 0, 35, 7, 0, 0, 1, 46, 0, 0, 4045, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4094, 4092, 4090, 4096, 4096, 4095, 4096] avg=4096.0000 max=4109 max_rank=0 min=4090 min_rank=3 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4063, 0, 16, 0, 0, 0, 0, 4074, 0, 0, 0, 35, 7, 0, 0, 0, 39, 1, 0, 4045, 0, 0, 0, 15, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4116, 4085, 4104, 4097, 4096, 4095, 4096] avg=4096.0000 max=4116 max_rank=1 min=4079 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4081, 0, 16, 0, 0, 0, 0, 4089, 0, 0, 0, 35, 7, 0, 0, 0, 7, 10, 0, 4045, 0, 0, 0, 10, 4, 0, 4089, 0, 0, 0, 0, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4131, 4062, 4103, 4092, 4096, 4095, 4092] avg=4096.0000 max=4131 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 4091, 0, 16, 0, 0, 0, 0, 4068, 1, 0, 0, 34, 7, 0, 0, 4, 26, 0, 1, 4045, 0, 0, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 2, 0, 0, 1, 4093, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4110, 4076, 4089, 4097, 4097, 4096, 4095] avg=4096.0000 max=4110 max_rank=1 min=4076 min_rank=2 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4073, 0, 3, 0, 0, 0, 0, 4077, 0, 0, 0, 36, 9, 0, 0, 0, 20, 0, 0, 4052, 0, 0, 0, 22, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 4096, 0, 0, 7, 1, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4122, 4072, 4102, 4096, 4095, 4101, 4104] avg=4096.0000 max=4122 max_rank=1 min=4072 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 0, 16, 0, 0, 0, 0, 3948, 0, 0, 0, 35, 7, 0, 0, 0, 148, 7, 0, 4045, 0, 0, 0, 7, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 3990, 4200, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4200 max_rank=2 min=3990 min_rank=1 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 384, 128, 128, 32384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32512, 128, 0, 0, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 4096, 0, 0, 4096, 4096, 0, 167, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3928, 0, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 4096, 0, 0, 4096, 4096, 0, 247, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3851, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 107, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3989, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 4096, 0, 0, 4096, 4096, 0, 304, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3793, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 51, 4095, 2, 4082, 0, 1, 4092, 4095, 1, 182, 0, 0, 6, 0, 4095, 0, 1, 4095, 4094, 0, 0, 1, 3870, 1, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 14, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 8, 4096, 0, 4095, 0, 0, 4096, 4096, 0, 89, 0, 0, 1, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 261, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3835, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3925, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 0, 171, 5, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4097, 4172, 4095, 4096, 4096, 4028, 4096] avg=4096.0000 max=4172 max_rank=2 min=4028 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16316 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3925, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3928, 0, 0, 0, 171, 5, 1, 0, 0, 0, 4, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4097, 4092, 4096, 4096, 4096, 4105, 4099] avg=4096.0000 max=4105 max_rank=6 min=4087 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3925, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 171, 5, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4097, 4032, 4095, 4096, 4096, 4166, 4096] avg=4096.0000 max=4166 max_rank=6 min=4032 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16454 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3925, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 171, 5, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4097, 3939, 4094, 4096, 4096, 4260, 4096] avg=4096.0000 max=4260 max_rank=6 min=3939 min_rank=2 imbalance_ratio=1.040039 max_load_reduction=28508 max_load_reduction_pct=0.869995 moved_to_echo=16548 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 51, 4089, 0, 0, 0, 0, 2, 4082, 0, 1, 1, 0, 0, 0, 3922, 0, 1, 182, 0, 0, 0, 0, 0, 0, 6, 0, 4094, 1, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 3870, 1, 0, 1, 170, 5, 1, 0, 0, 0, 0, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4086, 4105, 4101, 4096, 4095, 4048, 4095] avg=4096.0000 max=4142 max_rank=0 min=4048 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16334 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4083, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 3924, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 3793, 0, 0, 0, 172, 6, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4101, 4228, 4096, 4096, 4097, 3972, 4095] avg=4096.0000 max=4228 max_rank=2 min=3972 min_rank=6 imbalance_ratio=1.032227 max_load_reduction=28540 max_load_reduction_pct=0.870972 moved_to_echo=16260 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 4090, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 3925, 0, 0, 89, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 171, 5, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4014, 4095, 4096, 4096, 4177, 4096] avg=4096.0000 max=4177 max_rank=6 min=4014 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3925, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3835, 0, 0, 0, 171, 5, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4097, 4186, 4096, 4096, 4096, 4012, 4095] avg=4096.0000 max=4186 max_rank=2 min=4012 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[31360, 128, 0, 128, 1408, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31488, 0, 128, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4081, 1273, 0, 4096, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 2835, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 2194, 0, 4096, 0, 0, 0, 4054, 4096, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 1897, 4096, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 868, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 3221, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4095, 262, 0, 4095, 0, 1, 1, 4095, 4092, 4095, 1, 1, 1, 0, 38, 0, 0, 1, 3802, 4095, 1, 1, 0, 0, 16, 0, 0, 0, 0, 0, 4075] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 2266, 0, 4096, 0, 0, 0, 4089, 4092, 4096, 0, 0, 0, 0, 25, 0, 0, 2, 1805, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4091, 1923, 0, 4096, 0, 0, 0, 4093, 4096, 4096, 0, 0, 0, 0, 17, 0, 0, 0, 2161, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4092, 953, 0, 4096, 0, 0, 0, 4080, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 3146, 4096, 0, 0, 0, 14, 0, 0, 0, 0, 4, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2669, 1273, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4066, 13, 0, 0, 0, 0, 0, 2835, 1411, 18, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3942, 4096, 4104, 4080, 4264, 4096, 4089, 4097] avg=4096.0000 max=4264 max_rank=4 min=3942 min_rank=0 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16546 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2678, 2194, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4054, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4066, 13, 0, 0, 0, 0, 0, 1897, 1416, 18, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4872, 4096, 4066, 4082, 3331, 4096, 4128, 4097] avg=4096.0000 max=4872 max_rank=0 min=3331 min_rank=4 imbalance_ratio=1.189453 max_load_reduction=27896 max_load_reduction_pct=0.851318 moved_to_echo=15652 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2678, 2266, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4063, 13, 0, 0, 0, 0, 2, 1805, 1416, 18, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4083, 0, 0, 0, 0, 2, 0, 4096, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4944, 4096, 4100, 4101, 3241, 4096, 4090, 4100] avg=4096.0000 max=4944 max_rank=0 min=3241 min_rank=4 imbalance_ratio=1.207031 max_load_reduction=27824 max_load_reduction_pct=0.849121 moved_to_echo=15527 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2678, 868, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4066, 13, 0, 0, 0, 0, 0, 3221, 1416, 18, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4083, 0, 0, 0, 0, 1, 0, 4095, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[3546, 4096, 4108, 4083, 4655, 4096, 4086, 4098] avg=4096.0000 max=4655 max_rank=4 min=3546 min_rank=0 imbalance_ratio=1.136475 max_load_reduction=28113 max_load_reduction_pct=0.857941 moved_to_echo=16935 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2677, 262, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 4095, 11, 0, 1, 0, 0, 0, 0, 1, 1, 0, 38, 4063, 14, 0, 0, 0, 0, 1, 3802, 1416, 18, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 4075, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[2939, 4097, 4107, 4117, 5237, 4097, 4097, 4077] avg=4096.0000 max=5237 max_rank=4 min=2939 min_rank=0 imbalance_ratio=1.278564 max_load_reduction=27531 max_load_reduction_pct=0.840179 moved_to_echo=17508 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2668, 953, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4080, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4072, 14, 0, 0, 0, 0, 0, 3146, 1420, 20, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4082, 0, 0, 0, 0, 4, 0, 4094, 4, 1, 1, 0] rerouted_tokens_per_ep_rank=[3621, 4095, 4084, 4087, 4586, 4095, 4096, 4104] avg=4096.0000 max=4586 max_rank=4 min=3621 min_rank=0 imbalance_ratio=1.119629 max_load_reduction=28182 max_load_reduction_pct=0.860046 moved_to_echo=16881 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2675, 1923, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4066, 14, 0, 0, 0, 0, 0, 2161, 1415, 18, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4598, 4096, 4105, 4097, 3594, 4096, 4085, 4097] avg=4096.0000 max=4598 max_rank=0 min=3594 min_rank=4 imbalance_ratio=1.122559 max_load_reduction=28170 max_load_reduction_pct=0.859680 moved_to_echo=15872 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 1628, 0, 4096, 0, 0, 0, 4082, 4096, 4096, 0, 0, 0, 0, 42, 0, 0, 0, 2426, 4096, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2678, 1628, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4066, 13, 0, 0, 0, 0, 0, 2426, 1416, 18, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4306, 4096, 4094, 4121, 3860, 4096, 4097, 4098] avg=4096.0000 max=4306 max_rank=0 min=3860 min_rank=4 imbalance_ratio=1.051270 max_load_reduction=28462 max_load_reduction_pct=0.868591 moved_to_echo=16151 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 21504, 11392, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 21376, 11392, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 4081, 0, 0, 19, 0, 4096, 1, 0, 0, 0, 4090, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 4078, 0, 0, 30, 0, 4096, 2, 0, 0, 0, 4090, 0, 0, 3, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 13, 0, 4095, 1, 0, 0, 0, 0, 0, 4096, 4070, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 13, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 4089, 0, 173, 1, 3994, 34, 0, 1, 1, 1, 1, 4095, 3987, 0, 1, 0, 6, 4093, 0, 0, 1, 0, 4095, 3, 0, 0, 4095, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4065, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 33, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 4090, 0, 0, 17, 0, 4096, 4, 0, 0, 0, 4096, 4, 0, 2, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4082, 0, 2, 0, 4089, 0, 3, 0, 0, 0, 0, 4090, 4080, 0, 0, 31, 0, 4096, 7, 0, 0, 0, 4096, 0, 0, 1, 4096, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 4077, 0, 0, 20, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4095, 0, 0, 0, 4081, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 4081, 0, 0, 19, 7, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4083, 4105, 4094, 4107, 4095, 4090, 4096] avg=4096.0000 max=4107 max_rank=4 min=4083 min_rank=1 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4081, 0, 13, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 8, 1, 0, 0, 0, 0, 0, 4096, 2, 1, 0, 0, 4070, 0, 0, 0, 7, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4094, 4105, 4099, 4077, 4094, 4108, 4096] avg=4096.0000 max=4108 max_rank=6 min=4077 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 4095, 0, 0, 0, 4074, 0, 173, 1, 0, 0, 0, 0, 3994, 34, 0, 1, 8, 0, 0, 0, 1, 1, 1, 4095, 2, 1, 0, 0, 3987, 0, 1, 0, 7, 0, 0, 0, 6, 4091, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4248, 4037, 4101, 3995, 4097, 4098, 4095] avg=4096.0000 max=4248 max_rank=1 min=3995 min_rank=4 imbalance_ratio=1.037109 max_load_reduction=28520 max_load_reduction_pct=0.870361 moved_to_echo=16285 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4065, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4089, 3, 2, 4, 0, 4078, 0, 0, 30, 12, 1, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4069, 4107, 4098, 4121, 4094, 4088, 4095] avg=4096.0000 max=4121 max_rank=4 min=4069 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4081, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4091, 2, 1, 0, 0, 4077, 0, 0, 20, 7, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4085, 4104, 4094, 4104, 4094, 4095, 4096] avg=4096.0000 max=4104 max_rank=2 min=4085 min_rank=1 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4050, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 4096, 2, 1, 0, 0, 4092, 0, 0, 33, 7, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4051, 4105, 4099, 4132, 4095, 4095, 4096] avg=4096.0000 max=4132 max_rank=4 min=4051 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4067, 0, 2, 0, 0, 0, 0, 0, 4089, 0, 3, 0, 8, 0, 0, 0, 0, 0, 0, 4090, 2, 1, 0, 0, 4080, 0, 0, 31, 7, 0, 0, 0, 0, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4069, 4100, 4093, 4118, 4101, 4095, 4096] avg=4096.0000 max=4118 max_rank=4 min=4069 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 4087, 2, 1, 0, 0, 4090, 0, 0, 17, 7, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4069, 4105, 4090, 4114, 4098, 4099, 4098] avg=4096.0000 max=4114 max_rank=4 min=4069 min_rank=1 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16409 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 0, 128, 256, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 0, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 4096, 4093, 4096, 0, 0, 0, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4, 0, 4096, 4091, 4096, 0, 0, 0, 0, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1, 1, 4091, 26, 1, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 0, 4094, 1, 0, 4095, 0, 0, 30, 1, 4095, 4095, 4043, 0, 0, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 9, 0, 4096, 4072, 4096, 0, 0, 0, 0, 4087] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 7, 0, 4096, 4086, 4095, 0, 0, 0, 0, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4069, 4081, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4, 0, 4096, 4089, 4096, 0, 0, 0, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4081, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4086, 11, 0, 0, 0, 4093, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4096, 4096, 4096, 4096, 4100, 4103, 4095] avg=4096.0000 max=4103 max_rank=6 min=4086 min_rank=0 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16394 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4081, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4086, 11, 0, 0, 0, 4091, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4096, 4096, 4096, 4096, 4101, 4101, 4093] avg=4096.0000 max=4101 max_rank=5 min=4089 min_rank=0 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16391 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4081, 42, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 11, 0, 0, 0, 4069, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4096, 4095, 4096, 4097, 4082, 4079, 4100] avg=4096.0000 max=4123 max_rank=0 min=4079 min_rank=6 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16358 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4077, 26, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1, 4033, 10, 0, 0, 0, 4095, 10, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4096, 4097, 4096, 4095, 4074, 4105, 4100] avg=4096.0000 max=4105 max_rank=0 min=4074 min_rank=5 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16374 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4081, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4085, 11, 0, 0, 0, 4086, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4096, 4096, 4096, 4103, 4096, 4088] avg=4096.0000 max=4103 max_rank=5 min=4088 min_rank=7 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16383 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4074, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4089, 12, 0, 0, 0, 4072, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 10, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4096, 4096, 4096, 4110, 4079, 4097] avg=4096.0000 max=4110 max_rank=5 min=4079 min_rank=6 imbalance_ratio=1.003418 max_load_reduction=28658 max_load_reduction_pct=0.874573 moved_to_echo=16382 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4081, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4086, 11, 0, 0, 0, 4089, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4096, 4096, 4096, 4096, 4101, 4099, 4095] avg=4096.0000 max=4101 max_rank=5 min=4089 min_rank=0 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16391 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 11, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4096, 4096, 4096, 4096, 4097, 4106, 4100] avg=4096.0000 max=4106 max_rank=6 min=4081 min_rank=0 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16399 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4093, 4096, 0, 0, 4096, 0, 4096, 0, 12, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4093, 4096, 0, 0, 4096, 2, 4096, 0, 6, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4044, 4093, 1, 0, 0, 4092, 4095, 4063, 0, 1, 4095, 0, 4095, 2, 0, 4095, 1, 0, 1, 79, 1, 0, 0, 1, 0, 1, 1, 5, 0, 0, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4094, 4096, 0, 0, 4096, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4095, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4095, 4096, 0, 0, 0, 4096, 4090, 4096, 0, 0, 4096, 0, 4096, 0, 9, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4088, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4101, 4096, 4095, 4085, 4096, 4095, 4096] avg=4096.0000 max=4104 max_rank=0 min=4085 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4101, 4096, 4095, 4085, 4096, 4095, 4096] avg=4096.0000 max=4104 max_rank=0 min=4085 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4101, 4098, 4096, 4085, 4096, 4092, 4096] avg=4096.0000 max=4104 max_rank=0 min=4085 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16369 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4044, 8, 1, 0, 0, 0, 0, 0, 0, 0, 5, 4063, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 1, 0, 1, 79, 4081, 1, 0, 0, 1, 0, 0, 1, 4092, 0, 0, 0, 0, 1, 1, 5, 4090, 4, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4068, 4096, 4096, 4163, 4094, 4101, 4097] avg=4096.0000 max=4163 max_rank=4 min=4053 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16455 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4101, 4096, 4097, 4085, 4096, 4093, 4096] avg=4096.0000 max=4104 max_rank=0 min=4085 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16370 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 8, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4084, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4101, 4096, 4095, 4086, 4096, 4095, 4096] avg=4096.0000 max=4103 max_rank=0 min=4086 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 9, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 2, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 4089, 8, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4094, 4099, 4094, 4098, 4097, 4095] avg=4096.0000 max=4099 max_rank=3 min=4094 min_rank=2 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16384 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4099, 4096, 4095, 4085, 4096, 4100, 4096] avg=4096.0000 max=4101 max_rank=0 min=4085 min_rank=4 imbalance_ratio=1.001221 max_load_reduction=28667 max_load_reduction_pct=0.874847 moved_to_echo=16377 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 252, 0, 0, 3853, 0, 0, 4096, 4, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4083, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 284, 0, 0, 3819, 0, 0, 4096, 0, 0, 0, 4096, 0, 6, 4096, 4096, 0, 0, 4083, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[11, 1, 2, 1, 0, 0, 0, 4095, 0, 6, 180, 0, 1, 3950, 0, 0, 4095, 0, 0, 0, 4094, 0, 0, 4055, 4094, 1, 0, 4095, 1, 4085, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 403, 0, 0, 3706, 0, 0, 4096, 0, 0, 0, 4096, 0, 15, 4096, 4096, 0, 0, 4068, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 317, 0, 0, 3856, 0, 0, 4096, 2, 0, 0, 4096, 0, 11, 4096, 4096, 0, 0, 4006, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 277, 0, 0, 3862, 0, 1, 4095, 0, 0, 0, 4096, 0, 7, 4096, 4096, 0, 0, 4046, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 755, 0, 0, 3357, 0, 0, 4096, 0, 0, 0, 4096, 0, 10, 4096, 4096, 0, 0, 4070, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 618, 0, 0, 3515, 0, 15, 4094, 1, 0, 0, 4096, 0, 11, 4096, 4096, 0, 0, 4034, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 284, 0, 3708, 2, 0, 0, 0, 3819, 0, 0, 352, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4095, 0, 0, 0, 0, 36, 0, 0, 4083, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 3994, 4172, 4096, 4101, 4119, 4096] avg=4096.0000 max=4172 max_rank=3 min=3994 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 403, 0, 3708, 2, 0, 0, 0, 3706, 0, 0, 352, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4095, 0, 0, 0, 0, 36, 0, 0, 4068, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4113, 4059, 4096, 4110, 4104, 4096] avg=4096.0000 max=4113 max_rank=2 min=4059 min_rank=3 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 252, 0, 3708, 2, 0, 0, 0, 3853, 0, 0, 352, 1, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 36, 0, 0, 4083, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 3962, 4206, 4100, 4095, 4119, 4096] avg=4096.0000 max=4206 max_rank=3 min=3962 min_rank=2 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 277, 0, 3708, 2, 0, 0, 0, 3862, 0, 1, 352, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4095, 0, 0, 0, 0, 36, 0, 0, 4046, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 3987, 4216, 4095, 4102, 4082, 4096] avg=4096.0000 max=4216 max_rank=3 min=3987 min_rank=2 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[11, 1, 2, 1, 4093, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 6, 180, 0, 3706, 1, 0, 0, 1, 3950, 0, 0, 352, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 36, 1, 0, 4095, 0, 0, 0, 0, 1, 4085, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4095, 3893, 4304, 4095, 4054, 4132, 4087] avg=4096.0000 max=4304 max_rank=3 min=3893 min_rank=2 imbalance_ratio=1.050781 max_load_reduction=28464 max_load_reduction_pct=0.868652 moved_to_echo=16368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 317, 0, 3708, 2, 0, 0, 0, 3856, 0, 0, 352, 1, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4095, 0, 0, 0, 0, 36, 0, 0, 4006, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4027, 4209, 4098, 4106, 4042, 4096] avg=4096.0000 max=4209 max_rank=3 min=4027 min_rank=2 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 618, 0, 3709, 1, 0, 0, 0, 3515, 0, 15, 357, 5, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4084, 0, 0, 0, 0, 30, 0, 0, 4034, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 2, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4097, 4328, 3892, 4092, 4095, 4064, 4105] avg=4096.0000 max=4328 max_rank=2 min=3892 min_rank=3 imbalance_ratio=1.056641 max_load_reduction=28440 max_load_reduction_pct=0.867920 moved_to_echo=16356 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 755, 0, 3708, 1, 0, 0, 0, 3357, 0, 0, 352, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4095, 0, 0, 0, 0, 36, 0, 0, 4070, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4464, 3710, 4096, 4105, 4106, 4096] avg=4096.0000 max=4464 max_rank=2 min=3710 min_rank=3 imbalance_ratio=1.089844 max_load_reduction=28304 max_load_reduction_pct=0.863770 moved_to_echo=16403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 3200, 0, 29696, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 29952, 0, 128, 2944, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 128, 0, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 5, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 1, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[303, 0, 0, 1, 1, 0, 10, 1, 4095, 1, 3762, 9, 0, 0, 0, 1, 4094, 4094, 4092, 1, 1, 0, 0, 2, 4081, 0, 1, 4095, 0, 14, 55, 4054] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4087, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 17, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 8, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4070, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 33, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4090, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 10, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 5, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4043, 8, 3, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 48, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4085, 2, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4094, 4143, 4096, 4096, 4092, 4099, 4093] avg=4096.0000 max=4143 max_rank=2 min=4055 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4048, 8, 3, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 48, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4094, 4135, 4096, 4098, 4093, 4099, 4093] avg=4096.0000 max=4135 max_rank=2 min=4060 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4048, 8, 3, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 48, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4094, 4143, 4096, 4097, 4086, 4099, 4093] avg=4096.0000 max=4143 max_rank=2 min=4060 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[303, 0, 0, 1, 4047, 8, 3, 0, 1, 0, 10, 1, 4094, 0, 0, 0, 48, 1, 3762, 9, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 1, 0, 0, 2, 4087, 0, 0, 0, 4081, 0, 1, 0, 0, 0, 0, 0, 0, 14, 55, 4051, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4362, 4106, 3820, 4095, 4093, 4090, 4082, 4120] avg=4096.0000 max=4362 max_rank=0 min=3820 min_rank=2 imbalance_ratio=1.064941 max_load_reduction=28406 max_load_reduction_pct=0.866882 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4041, 8, 3, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 48, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 33, 0, 0, 4086, 1, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4094, 4118, 4097, 4096, 4120, 4098, 4093] avg=4096.0000 max=4120 max_rank=5 min=4052 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16407 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4039, 8, 3, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 48, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4085, 2, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4094, 4135, 4096, 4096, 4104, 4099, 4093] avg=4096.0000 max=4135 max_rank=2 min=4051 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 4041, 11, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 47, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4096, 4137, 4096, 4096, 4095, 4096, 4090] avg=4096.0000 max=4137 max_rank=2 min=4062 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 4048, 10, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 48, 0, 4084, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4096, 4137, 4096, 4096, 4088, 4096, 4093] avg=4096.0000 max=4137 max_rank=2 min=4066 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 128, 32384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 0, 0, 128, 32384, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4, 0, 4073, 3826, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 266, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 23, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4066, 3499, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 562, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 64, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 3804, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 266, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 31, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4030, 3617, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 391, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 154, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4061, 3395, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 700, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 32, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1, 3, 1, 4089, 3730, 1, 0, 4094, 1, 1, 1, 1, 4095, 1, 1, 1, 4090, 0, 14, 416, 1, 0, 0, 4095, 4036, 0, 0, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 2, 0, 4072, 2769, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1174, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 175, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4058, 3722, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 341, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 70, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4066, 27, 0, 0, 0, 3499, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 515, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 6, 0, 0, 0, 0, 64, 0, 4029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4050, 4094, 4096, 4143, 4096, 4103, 4093] avg=4096.0000 max=4143 max_rank=4 min=4050 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16435 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4061, 27, 0, 0, 0, 3395, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 515, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 0, 700, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 6, 0, 0, 0, 0, 32, 0, 4029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 3946, 4094, 4096, 4281, 4096, 4106, 4061] avg=4096.0000 max=4281 max_rank=4 min=3946 min_rank=1 imbalance_ratio=1.045166 max_load_reduction=28487 max_load_reduction_pct=0.869354 moved_to_echo=16544 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4030, 27, 0, 0, 0, 3617, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 515, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 0, 391, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 154, 0, 4029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4168, 4094, 4096, 3972, 4096, 4102, 4183] avg=4096.0000 max=4183 max_rank=7 min=3972 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16353 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 27, 0, 0, 0, 3804, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 515, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 31, 0, 4029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4355, 4094, 4096, 3847, 4096, 4102, 4060] avg=4096.0000 max=4355 max_rank=1 min=3847 min_rank=4 imbalance_ratio=1.063232 max_load_reduction=28413 max_load_reduction_pct=0.867096 moved_to_echo=16105 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 4073, 27, 0, 0, 0, 3826, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 515, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 23, 0, 4029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4377, 4094, 4096, 3847, 4096, 4102, 4052] avg=4096.0000 max=4377 max_rank=1 min=3847 min_rank=4 imbalance_ratio=1.068604 max_load_reduction=28391 max_load_reduction_pct=0.866425 moved_to_echo=16097 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 3, 1, 4089, 27, 0, 0, 0, 3730, 1, 0, 551, 0, 0, 0, 0, 1, 1, 1, 1, 3543, 514, 34, 0, 4095, 1, 1, 1, 0, 0, 0, 0, 3576, 0, 14, 416, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 4036, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4282, 4095, 4098, 4006, 4096, 4042, 4028] avg=4096.0000 max=4282 max_rank=1 min=4006 min_rank=4 imbalance_ratio=1.045410 max_load_reduction=28486 max_load_reduction_pct=0.869324 moved_to_echo=16172 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4058, 28, 0, 0, 0, 3722, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 0, 3548, 520, 41, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 3576, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 10, 2, 0, 0, 0, 70, 0, 4017, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4270, 4109, 4094, 3917, 4096, 4109, 4087] avg=4096.0000 max=4270 max_rank=1 min=3917 min_rank=4 imbalance_ratio=1.042480 max_load_reduction=28498 max_load_reduction_pct=0.869690 moved_to_echo=16209 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4072, 27, 0, 0, 0, 2769, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 515, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 0, 1174, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 175, 0, 4029, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 3320, 4094, 4096, 4755, 4096, 4102, 4204] avg=4096.0000 max=4755 max_rank=4 min=3320 min_rank=1 imbalance_ratio=1.160889 max_load_reduction=28013 max_load_reduction_pct=0.854889 moved_to_echo=17157 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[28416, 128, 0, 4480, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[28672, 0, 128, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 28416, 4224, 384, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32640, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 640, 0, 32256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4096, 4068, 105, 4078, 0, 0, 4096, 46, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 0, 0, 0, 4096, 0, 0, 0, 2664, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4091, 4082, 134, 4094, 0, 0, 4096, 35, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 655, 0, 0, 0, 4096, 0, 0, 0, 3295, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4095, 4095, 161, 3926, 2, 2, 4094, 166, 1, 0, 0, 4093, 1, 4067, 0, 0, 0, 1, 1, 0, 0, 1, 949, 0, 0, 0, 4095, 1, 0, 1, 3016, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4084, 4080, 93, 4095, 0, 0, 4096, 24, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 0, 0, 4096, 0, 0, 0, 3705, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 36, 4095, 0, 0, 4096, 14, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 4096, 0, 0, 0, 3863, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4090, 4073, 129, 4084, 0, 0, 4096, 34, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 0, 4096, 0, 0, 0, 3515, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4091, 4070, 78, 4096, 0, 0, 4095, 53, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 0, 0, 4096, 0, 0, 0, 3579, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4082, 4060, 25, 4067, 0, 0, 4096, 76, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, 0, 4096, 0, 0, 0, 3511, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 134, 4028, 0, 0, 0, 0, 0, 0, 4041, 35, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 655, 0, 3492, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3295, 0, 585, 66, 51, 0] rerouted_tokens_per_ep_rank=[4162, 4076, 4096, 4098, 4096, 4147, 4096, 3997] avg=4096.0000 max=4162 max_rank=0 min=3997 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16336 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 105, 4012, 0, 0, 0, 0, 0, 0, 4041, 46, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 1327, 0, 3479, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2664, 0, 584, 66, 51, 0] rerouted_tokens_per_ep_rank=[4117, 4087, 4096, 4100, 4101, 4806, 4096, 3365] avg=4096.0000 max=4806 max_rank=5 min=3365 min_rank=7 imbalance_ratio=1.173340 max_load_reduction=27962 max_load_reduction_pct=0.853333 moved_to_echo=16368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 129, 4018, 0, 0, 0, 0, 0, 0, 4041, 34, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4090, 5, 0, 0, 0, 0, 459, 0, 3485, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3515, 0, 583, 66, 51, 0] rerouted_tokens_per_ep_rank=[4147, 4075, 4096, 4100, 4095, 3944, 4096, 4215] avg=4096.0000 max=4215 max_rank=7 min=3944 min_rank=5 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16350 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 93, 4029, 0, 0, 0, 0, 0, 0, 4041, 24, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4084, 5, 0, 0, 0, 0, 304, 0, 3490, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3705, 0, 585, 66, 51, 0] rerouted_tokens_per_ep_rank=[4122, 4065, 4096, 4099, 4089, 3794, 4096, 4407] avg=4096.0000 max=4407 max_rank=7 min=3794 min_rank=5 imbalance_ratio=1.075928 max_load_reduction=28361 max_load_reduction_pct=0.865509 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 36, 4029, 0, 0, 0, 0, 0, 0, 4041, 14, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 185, 0, 3503, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3863, 0, 588, 66, 51, 0] rerouted_tokens_per_ep_rank=[4065, 4055, 4095, 4100, 4101, 3688, 4096, 4568] avg=4096.0000 max=4568 max_rank=7 min=3688 min_rank=5 imbalance_ratio=1.115234 max_load_reduction=28200 max_load_reduction_pct=0.860596 moved_to_echo=16453 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 161, 3863, 0, 0, 0, 0, 2, 2, 4040, 166, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 1, 4067, 0, 0, 3, 0, 0, 0, 0, 1, 1, 0, 4095, 5, 0, 0, 0, 1, 949, 0, 3503, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 3016, 0, 587, 63, 51, 0] rerouted_tokens_per_ep_rank=[4024, 4210, 4094, 4071, 4102, 4453, 4096, 3718] avg=4096.0000 max=4453 max_rank=5 min=3718 min_rank=7 imbalance_ratio=1.087158 max_load_reduction=28315 max_load_reduction_pct=0.864105 moved_to_echo=16369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 25, 3998, 0, 0, 0, 0, 0, 0, 4030, 76, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4095, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4082, 6, 0, 0, 0, 0, 564, 0, 3472, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3511, 0, 582, 69, 57, 0] rerouted_tokens_per_ep_rank=[4023, 4106, 4099, 4101, 4088, 4036, 4096, 4219] avg=4096.0000 max=4219 max_rank=7 min=4023 min_rank=0 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 78, 4030, 0, 0, 0, 0, 0, 0, 4041, 53, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 418, 0, 3482, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3579, 0, 583, 66, 51, 0] rerouted_tokens_per_ep_rank=[4108, 4094, 4096, 4099, 4096, 3900, 4096, 4279] avg=4096.0000 max=4279 max_rank=7 min=3900 min_rank=5 imbalance_ratio=1.044678 max_load_reduction=28489 max_load_reduction_pct=0.869415 moved_to_echo=16371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 27264, 0, 4736, 640, 512, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 4864, 0, 28032, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 768, 32128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32384, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[298, 0, 2248, 4, 1095, 1, 2223, 33, 1155, 1377, 0, 29, 2013, 3392, 2, 676, 2, 782, 2964, 0, 2, 321, 1095, 36, 0, 3138, 3660, 1896, 3230, 1047, 48, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[337, 0, 1658, 19, 1564, 1, 2824, 8, 1419, 1231, 3, 25, 1701, 3552, 1, 526, 12, 1207, 1997, 0, 5, 556, 1162, 40, 0, 2578, 3736, 2040, 2965, 1507, 94, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[214, 3, 1413, 3, 1186, 0, 2903, 23, 1805, 1319, 0, 14, 1693, 3553, 10, 692, 2, 1147, 2173, 2, 4, 710, 799, 37, 1, 3127, 3913, 1668, 3073, 1229, 52, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[130, 3, 1802, 14, 1384, 26, 2205, 34, 1608, 1618, 0, 23, 1426, 2849, 6, 661, 15, 2069, 1353, 1, 5, 503, 1446, 68, 3, 3046, 3670, 1865, 3425, 1408, 96, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[180, 1, 1192, 5, 1102, 0, 2833, 33, 1339, 1521, 1, 33, 2098, 3490, 2, 669, 6, 1374, 1963, 0, 1, 505, 1127, 29, 0, 3366, 3766, 1687, 2807, 1576, 62, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[312, 0, 2053, 25, 1604, 0, 2392, 9, 1525, 1330, 0, 32, 1008, 3672, 14, 852, 36, 1586, 2244, 0, 0, 626, 1299, 56, 0, 2887, 3881, 1867, 2178, 1250, 30, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[113, 3, 1748, 65, 1641, 4, 2171, 3, 1649, 1123, 0, 26, 2067, 3505, 10, 601, 6, 1305, 2107, 0, 0, 565, 1351, 156, 0, 2510, 3791, 2065, 2925, 1150, 108, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[196, 0, 1351, 9, 1590, 0, 2818, 6, 1571, 1301, 0, 25, 1352, 3372, 5, 667, 2, 2038, 2622, 0, 0, 843, 776, 49, 0, 2751, 3496, 1451, 3096, 1321, 60, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[298, 0, 2248, 4, 1384, 749, 0, 0, 1095, 1, 2223, 33, 143, 0, 0, 0, 1155, 1377, 0, 29, 906, 314, 0, 0, 2013, 1737, 2, 676, 0, 0, 0, 0, 2, 782, 2964, 0, 379, 125, 0, 0, 2, 321, 1095, 36, 2276, 0, 0, 0, 0, 2445, 0, 1896, 0, 0, 0, 0, 2962, 1047, 48, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4683, 3495, 3781, 4428, 4252, 3730, 4341, 4058] avg=4096.0000 max=4683 max_rank=0 min=3495 min_rank=1 imbalance_ratio=1.143311 max_load_reduction=28085 max_load_reduction_pct=0.857086 moved_to_echo=16381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[337, 0, 1658, 19, 1414, 784, 0, 0, 1564, 1, 2824, 8, 131, 0, 0, 0, 1419, 1231, 3, 25, 949, 258, 0, 0, 1701, 1819, 1, 526, 0, 0, 0, 0, 12, 1207, 1997, 0, 311, 115, 0, 0, 5, 556, 1162, 40, 2322, 0, 0, 0, 0, 2009, 0, 2040, 0, 0, 0, 0, 2719, 1507, 94, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4212, 4528, 3885, 4047, 3642, 4085, 4049, 4320] avg=4096.0000 max=4528 max_rank=1 min=3642 min_rank=4 imbalance_ratio=1.105469 max_load_reduction=28240 max_load_reduction_pct=0.861816 moved_to_echo=16096 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[312, 0, 2053, 25, 1468, 811, 0, 0, 1604, 0, 2392, 9, 96, 0, 0, 0, 1525, 1330, 0, 32, 981, 289, 0, 0, 1008, 1880, 14, 852, 0, 0, 0, 0, 36, 1586, 2244, 0, 348, 84, 0, 0, 0, 626, 1299, 56, 2413, 0, 0, 0, 0, 2250, 0, 1867, 0, 0, 0, 0, 1998, 1250, 30, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4669, 4101, 4157, 3754, 4298, 4394, 4117, 3278] avg=4096.0000 max=4669 max_rank=0 min=3278 min_rank=7 imbalance_ratio=1.139893 max_load_reduction=28099 max_load_reduction_pct=0.857513 moved_to_echo=16087 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[130, 3, 1802, 14, 1388, 629, 0, 0, 1384, 26, 2205, 34, 152, 0, 0, 0, 1608, 1618, 0, 23, 761, 305, 0, 0, 1426, 1459, 6, 661, 0, 0, 0, 0, 15, 2069, 1353, 1, 368, 133, 0, 0, 5, 503, 1446, 68, 2282, 0, 0, 0, 3, 2373, 0, 1865, 0, 0, 0, 0, 3140, 1408, 96, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3966, 3801, 4315, 3552, 3939, 4304, 4241, 4650] avg=4096.0000 max=4650 max_rank=7 min=3552 min_rank=3 imbalance_ratio=1.135254 max_load_reduction=28118 max_load_reduction_pct=0.858093 moved_to_echo=17134 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[214, 3, 1413, 3, 1481, 785, 0, 0, 1186, 0, 2903, 23, 136, 0, 0, 0, 1805, 1319, 0, 14, 949, 313, 0, 0, 1693, 1819, 10, 692, 0, 0, 0, 0, 2, 1147, 2173, 2, 377, 119, 0, 0, 4, 710, 799, 37, 2432, 0, 0, 0, 1, 2437, 0, 1668, 0, 0, 0, 0, 2818, 1229, 52, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3899, 4248, 4400, 4214, 3820, 3982, 4106, 4099] avg=4096.0000 max=4400 max_rank=2 min=3820 min_rank=4 imbalance_ratio=1.074219 max_load_reduction=28368 max_load_reduction_pct=0.865723 moved_to_echo=16007 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[180, 1, 1192, 5, 1425, 775, 0, 0, 1102, 0, 2833, 33, 128, 0, 0, 0, 1339, 1521, 1, 33, 935, 343, 0, 0, 2098, 1780, 2, 669, 0, 0, 0, 0, 6, 1374, 1963, 0, 411, 114, 0, 0, 1, 505, 1127, 29, 2341, 0, 0, 0, 0, 2612, 0, 1687, 0, 0, 0, 0, 2565, 1576, 62, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3578, 4096, 4172, 4549, 3868, 4003, 4299, 4203] avg=4096.0000 max=4549 max_rank=3 min=3578 min_rank=0 imbalance_ratio=1.110596 max_load_reduction=28219 max_load_reduction_pct=0.861176 moved_to_echo=16373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[113, 3, 1748, 65, 1434, 774, 0, 0, 1641, 4, 2171, 3, 129, 0, 0, 0, 1649, 1123, 0, 26, 936, 251, 0, 0, 2067, 1795, 10, 601, 0, 0, 0, 0, 6, 1305, 2107, 0, 303, 114, 0, 0, 0, 565, 1351, 156, 2357, 0, 0, 0, 0, 1956, 0, 2065, 0, 0, 0, 0, 2682, 1150, 108, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 3948, 3985, 4473, 3835, 4429, 4021, 3940] avg=4096.0000 max=4473 max_rank=3 min=3835 min_rank=4 imbalance_ratio=1.092041 max_load_reduction=28295 max_load_reduction_pct=0.863495 moved_to_echo=16225 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[196, 0, 1351, 9, 1323, 745, 0, 0, 1590, 0, 2818, 6, 137, 0, 0, 0, 1571, 1301, 0, 25, 900, 276, 0, 0, 1352, 1727, 5, 667, 0, 0, 0, 0, 2, 2038, 2622, 0, 332, 120, 0, 0, 0, 843, 776, 49, 2173, 0, 0, 0, 0, 2143, 0, 1451, 0, 0, 0, 0, 2839, 1321, 60, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3624, 4551, 4073, 3751, 5114, 3841, 3594, 4220] avg=4096.0000 max=5114 max_rank=4 min=3594 min_rank=6 imbalance_ratio=1.248535 max_load_reduction=27654 max_load_reduction_pct=0.843933 moved_to_echo=16769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1792, 128, 13568, 256, 11392, 6144, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[12160, 10880, 128, 256, 7424, 2432, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[13440, 14080, 128, 5376, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 18304, 0, 14592, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 11520, 17536, 128, 2944, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21760, 10496, 640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[11264, 128, 20480, 256, 1152, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 4736, 9088, 512, 18688, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[456, 0, 0, 0, 1292, 0, 0, 0, 0, 18, 3138, 0, 3, 0, 4095, 4093, 4096, 0, 0, 0, 3771, 0, 0, 0, 4095, 3616, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[345, 0, 0, 0, 438, 0, 0, 0, 0, 1, 3512, 0, 3, 0, 4096, 4096, 4096, 0, 0, 0, 4004, 0, 0, 0, 4093, 3988, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[398, 0, 0, 0, 1107, 0, 0, 0, 0, 0, 3156, 0, 12, 0, 4096, 4096, 4096, 0, 0, 0, 3949, 0, 0, 0, 4096, 3681, 0, 0, 0, 4081, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1375, 0, 0, 0, 1197, 0, 0, 0, 1, 6, 3204, 0, 1, 0, 4096, 4091, 4096, 0, 0, 0, 3017, 0, 1, 1, 4094, 3492, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[229, 0, 0, 0, 1152, 0, 0, 0, 0, 7, 2786, 0, 18, 0, 4096, 4095, 4096, 0, 0, 0, 4070, 0, 0, 0, 4094, 4023, 0, 0, 0, 4096, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[350, 3, 0, 0, 549, 0, 14, 0, 18, 56, 3195, 0, 0, 0, 4082, 4072, 4079, 7, 1, 0, 4028, 0, 3, 0, 4083, 4094, 16, 14, 7, 4076, 8, 13] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1371, 1, 0, 3, 1416, 0, 0, 0, 0, 5, 2528, 0, 4, 0, 4096, 4087, 4096, 0, 0, 0, 3248, 0, 0, 0, 4091, 3727, 0, 0, 0, 4093, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[504, 0, 0, 0, 832, 0, 0, 0, 0, 4, 3161, 0, 6, 0, 4096, 4096, 4096, 0, 0, 0, 3821, 0, 0, 0, 4093, 3963, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[456, 0, 0, 0, 3467, 0, 0, 0, 1292, 0, 0, 0, 2470, 628, 0, 0, 0, 18, 3138, 0, 997, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3771, 0, 0, 0, 357, 0, 0, 0, 271, 3616, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3923, 4390, 4153, 4096, 4096, 4128, 3887, 4095] avg=4096.0000 max=4390 max_rank=1 min=3887 min_rank=6 imbalance_ratio=1.071777 max_load_reduction=28378 max_load_reduction_pct=0.866028 moved_to_echo=16206 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[345, 0, 0, 0, 3468, 0, 0, 0, 438, 0, 0, 0, 2469, 628, 0, 0, 0, 1, 3512, 0, 996, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 0, 0, 356, 0, 0, 0, 272, 3988, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3813, 3535, 4509, 4099, 4096, 4360, 4260, 4096] avg=4096.0000 max=4509 max_rank=2 min=3535 min_rank=1 imbalance_ratio=1.100830 max_load_reduction=28259 max_load_reduction_pct=0.862396 moved_to_echo=16812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[398, 0, 0, 0, 3468, 0, 0, 0, 1107, 0, 0, 0, 2471, 628, 0, 0, 0, 0, 3156, 0, 997, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 357, 0, 0, 0, 271, 3681, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3866, 4206, 4153, 4108, 4096, 4306, 3952, 4081] avg=4096.0000 max=4306 max_rank=5 min=3866 min_rank=0 imbalance_ratio=1.051270 max_load_reduction=28462 max_load_reduction_pct=0.868591 moved_to_echo=16435 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[229, 0, 0, 0, 3468, 0, 0, 0, 1152, 0, 0, 0, 2469, 628, 0, 0, 0, 7, 2786, 0, 996, 0, 0, 0, 18, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 357, 0, 0, 0, 272, 4023, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3697, 4249, 3789, 4113, 4096, 4427, 4295, 4102] avg=4096.0000 max=4427 max_rank=5 min=3697 min_rank=0 imbalance_ratio=1.080811 max_load_reduction=28341 max_load_reduction_pct=0.864899 moved_to_echo=16920 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1371, 1, 0, 3, 3468, 0, 0, 0, 1416, 0, 0, 0, 2468, 628, 0, 0, 0, 5, 2528, 0, 996, 0, 0, 0, 4, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3248, 0, 0, 0, 356, 0, 0, 0, 271, 3727, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4843, 4512, 3529, 4091, 4096, 3604, 3998, 4095] avg=4096.0000 max=4843 max_rank=0 min=3529 min_rank=2 imbalance_ratio=1.182373 max_load_reduction=27925 max_load_reduction_pct=0.852203 moved_to_echo=15793 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[350, 3, 0, 0, 3456, 0, 0, 0, 549, 0, 14, 0, 2463, 626, 0, 0, 18, 56, 3195, 0, 994, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 4079, 7, 1, 0, 0, 0, 0, 0, 4028, 0, 3, 0, 356, 0, 0, 0, 270, 4094, 16, 14, 0, 0, 0, 0, 7, 4076, 8, 13, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3809, 3652, 4263, 4072, 4087, 4387, 4394, 4104] avg=4096.0000 max=4394 max_rank=6 min=3652 min_rank=1 imbalance_ratio=1.072754 max_load_reduction=28374 max_load_reduction_pct=0.865906 moved_to_echo=16972 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1375, 0, 0, 0, 3469, 0, 0, 0, 1197, 0, 0, 0, 2472, 627, 0, 0, 1, 6, 3204, 0, 1000, 0, 0, 0, 1, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 2, 0, 0, 3017, 0, 1, 1, 360, 0, 0, 0, 255, 3492, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4844, 4296, 4211, 4087, 4105, 3379, 3747, 4099] avg=4096.0000 max=4844 max_rank=0 min=3379 min_rank=5 imbalance_ratio=1.182617 max_load_reduction=27924 max_load_reduction_pct=0.852173 moved_to_echo=15330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[504, 0, 0, 0, 3469, 0, 0, 0, 832, 0, 0, 0, 2469, 627, 0, 0, 0, 4, 3161, 0, 996, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3821, 0, 0, 0, 356, 0, 0, 0, 272, 3963, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3973, 3928, 4161, 4102, 4096, 4177, 4235, 4096] avg=4096.0000 max=4235 max_rank=6 min=3928 min_rank=1 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5120, 128, 0, 128, 27776, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 24704, 0, 8064, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8064, 0, 128, 0, 19840, 5120, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2176, 30592, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[29952, 0, 128, 128, 2944, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3, 7, 4027, 0, 2, 0, 4096, 0, 4089, 4096, 0, 1, 4092, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 3666, 7, 0, 0, 0, 808, 0, 3775] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3957, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 8, 3788, 38, 0, 0, 0, 469, 0, 4026] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 4, 4086, 0, 0, 0, 4096, 3, 4087, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 1, 4096, 0, 3950, 59, 0, 0, 0, 115, 0, 4076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 11, 3984, 0, 0, 0, 4096, 4, 4008, 4096, 0, 0, 4090, 0, 0, 21, 0, 0, 0, 0, 0, 0, 4096, 3, 3982, 54, 0, 0, 0, 257, 0, 4066] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[8, 7, 4082, 4, 0, 0, 4082, 0, 4053, 4092, 0, 10, 4093, 2, 1, 0, 10, 4, 0, 4, 0, 0, 4086, 0, 4061, 28, 7, 1, 26, 35, 0, 4072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 4096, 0, 4081, 4096, 3, 0, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 1, 4096, 0, 3836, 32, 0, 0, 0, 332, 0, 3998] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 4071, 0, 0, 0, 4096, 0, 4096, 4096, 1, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 2, 4096, 3, 3565, 49, 0, 0, 0, 505, 0, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 4042, 0, 1, 0, 4096, 0, 4094, 4096, 2, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 7, 4096, 1, 3622, 56, 0, 0, 1, 679, 0, 3876] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4086, 0, 42, 3, 1, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4087, 19, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 16, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 3950, 59, 0, 0, 250, 0, 0, 0, 0, 115, 0, 3768, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4099, 4106, 4096, 4093, 4096, 4259, 3883] avg=4096.0000 max=4259 max_rank=6 min=3883 min_rank=7 imbalance_ratio=1.039795 max_load_reduction=28509 max_load_reduction_pct=0.870026 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3957, 0, 42, 3, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 16, 0, 0, 0, 1, 4095, 8, 0, 0, 0, 0, 3788, 38, 0, 0, 247, 0, 0, 0, 0, 469, 0, 3721, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4004, 4096, 4115, 4093, 4093, 4104, 4073, 4190] avg=4096.0000 max=4190 max_rank=7 min=4004 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 42, 3, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4081, 19, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 16, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 3836, 32, 0, 0, 245, 0, 0, 0, 0, 332, 0, 3695, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4096, 4103, 4104, 4093, 4096, 4113, 4027] avg=4096.0000 max=4136 max_rank=0 min=4027 min_rank=7 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 7, 4027, 0, 39, 3, 1, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4089, 19, 0, 1, 0, 0, 0, 0, 4089, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 15, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3666, 7, 0, 0, 232, 0, 0, 0, 0, 808, 0, 3489, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4098, 4109, 4092, 4092, 4095, 3905, 4297] avg=4096.0000 max=4297 max_rank=7 min=3905 min_rank=6 imbalance_ratio=1.049072 max_load_reduction=28471 max_load_reduction_pct=0.868866 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 3984, 0, 42, 3, 1, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4008, 19, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 16, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 3982, 54, 0, 0, 250, 0, 0, 0, 0, 257, 0, 3758, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4100, 4027, 4108, 4093, 4098, 4286, 4015] avg=4096.0000 max=4286 max_rank=6 min=4015 min_rank=7 imbalance_ratio=1.046387 max_load_reduction=28482 max_load_reduction_pct=0.869202 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[8, 7, 4082, 4, 42, 3, 1, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 4053, 19, 0, 10, 0, 0, 0, 0, 4090, 2, 1, 0, 0, 0, 0, 0, 10, 4, 0, 4, 4073, 16, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 4061, 28, 7, 1, 250, 0, 0, 0, 26, 35, 0, 3764, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4082, 4082, 4093, 4107, 4085, 4347, 3825] avg=4096.0000 max=4347 max_rank=6 min=3825 min_rank=7 imbalance_ratio=1.061279 max_load_reduction=28421 max_load_reduction_pct=0.867340 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4042, 0, 45, 10, 6, 0, 1, 0, 4096, 0, 4, 0, 0, 0, 4094, 14, 2, 0, 0, 0, 0, 0, 4085, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 21, 0, 0, 0, 7, 4086, 1, 0, 0, 0, 0, 3622, 56, 0, 0, 242, 0, 0, 0, 1, 679, 0, 3568, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4101, 4110, 4087, 4103, 4094, 3920, 4248] avg=4096.0000 max=4248 max_rank=7 min=3920 min_rank=6 imbalance_ratio=1.037109 max_load_reduction=28520 max_load_reduction_pct=0.870361 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4071, 0, 43, 3, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 19, 1, 0, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 17, 0, 0, 0, 2, 4095, 3, 0, 0, 0, 0, 3565, 49, 0, 0, 251, 0, 0, 0, 0, 505, 0, 3778, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4096, 4116, 4095, 4094, 4100, 3865, 4283] avg=4096.0000 max=4283 max_rank=7 min=3865 min_rank=6 imbalance_ratio=1.045654 max_load_reduction=28485 max_load_reduction_pct=0.869293 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[30592, 384, 128, 128, 2048, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 32384, 128, 384, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 3200, 0, 29568, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4087, 4096, 126, 1, 0, 0, 0, 0, 3978, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 4076, 4096, 0, 4072, 4096, 146, 3, 2, 0, 0, 0, 3989, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[5, 0, 0, 4, 4, 1, 5, 0, 0, 4089, 4, 0, 4, 0, 4, 3, 4092, 2, 5, 4092, 4089, 1, 4015, 4090, 227, 18, 0, 0, 0, 5, 3917, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 4096, 0, 4081, 4096, 187, 7, 0, 0, 0, 0, 3930, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4072, 4096, 0, 4096, 4096, 241, 1, 0, 0, 0, 0, 3878, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4095, 4096, 218, 0, 0, 0, 0, 0, 3884, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 4096, 0, 4096, 4096, 141, 1, 0, 0, 0, 0, 3965, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 4096, 4096, 326, 0, 1, 0, 0, 0, 3772, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 22, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 17, 0, 4087, 0, 0, 0, 0, 0, 126, 1, 0, 0, 3891, 0, 0, 0, 0, 0, 3978, 183, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 4096, 4095, 4104, 4104, 4018, 4161] avg=4096.0000 max=4161 max_rank=7 min=4018 min_rank=6 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4088, 6, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 21, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 17, 0, 4095, 0, 0, 0, 0, 0, 218, 0, 0, 0, 3888, 0, 0, 0, 0, 0, 3884, 184, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4094, 4094, 4104, 4112, 4106, 4068] avg=4096.0000 max=4112 max_rank=5 min=4068 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16390 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4088, 6, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 22, 0, 0, 8, 0, 0, 4083, 0, 0, 0, 0, 17, 0, 4081, 0, 0, 0, 0, 0, 187, 7, 0, 0, 3891, 0, 0, 0, 0, 0, 3930, 183, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4096, 4095, 4091, 4098, 4085, 4113] avg=4096.0000 max=4113 max_rank=7 min=4085 min_rank=6 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 4, 4089, 0, 0, 0, 4, 1, 5, 0, 4084, 6, 1, 0, 0, 4089, 4, 0, 0, 0, 0, 0, 4, 0, 4, 3, 4066, 21, 0, 0, 8, 2, 5, 4092, 0, 0, 0, 0, 17, 1, 4015, 0, 0, 0, 0, 0, 227, 18, 0, 0, 3887, 0, 0, 0, 0, 5, 3917, 184, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4101, 4093, 4098, 4107, 4033, 4132, 4106] avg=4096.0000 max=4132 max_rank=6 min=4033 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 22, 0, 0, 8, 0, 0, 4072, 0, 0, 0, 0, 17, 0, 4096, 0, 0, 0, 0, 0, 241, 1, 0, 0, 3891, 0, 0, 0, 0, 0, 3878, 183, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 4096, 4095, 4080, 4113, 4133, 4061] avg=4096.0000 max=4133 max_rank=6 min=4061 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 6, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 22, 0, 0, 8, 3, 0, 4076, 0, 0, 0, 0, 17, 0, 4072, 0, 0, 0, 0, 0, 146, 3, 2, 0, 3891, 0, 0, 0, 0, 0, 3989, 183, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4093, 4094, 4095, 4087, 4089, 4042, 4172] avg=4096.0000 max=4172 max_rank=7 min=4042 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16390 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 8, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4078, 25, 0, 0, 2, 0, 0, 4089, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 141, 1, 0, 0, 3894, 0, 0, 0, 0, 0, 3965, 166, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4102, 4103, 4103, 4091, 4106, 4036, 4131] avg=4096.0000 max=4131 max_rank=7 min=4036 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 21, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 17, 0, 4096, 0, 0, 0, 0, 0, 326, 0, 1, 0, 3889, 0, 0, 0, 0, 0, 3772, 184, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 4096, 4093, 4104, 4113, 4216, 3956] avg=4096.0000 max=4216 max_rank=6 min=3956 min_rank=7 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 31360, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1664, 128, 128, 0, 31232, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 32640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1088, 0, 4074, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 9, 4076, 0, 4087, 0, 3049, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1251, 0, 4045, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 1, 0, 14, 4087, 0, 4085, 0, 2902, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 953, 0, 4088, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 6, 4089, 0, 4093, 0, 3157, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1397, 0, 4049, 4096, 0, 0, 3, 0, 4088, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4085, 0, 4072, 0, 2745, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1175, 0, 4075, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4088, 0, 4094, 0, 2950, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4, 4, 5, 358, 0, 4081, 4091, 0, 2, 0, 0, 4087, 5, 4, 0, 4091, 4091, 0, 0, 0, 6, 6, 0, 0, 0, 1, 4055, 0, 4092, 3, 3782, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 636, 0, 4068, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 1, 4060, 0, 4094, 0, 3520, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 852, 1, 4094, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4005, 0, 4095, 0, 3336, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1088, 3129, 0, 0, 0, 0, 4074, 25, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4071, 21, 0, 0, 0, 9, 4076, 0, 19, 0, 0, 0, 915, 0, 3049, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4217, 4099, 4099, 4096, 4096, 4093, 4104, 3964] avg=4096.0000 max=4217 max_rank=0 min=3964 min_rank=7 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 953, 3133, 0, 0, 0, 0, 4088, 25, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4071, 21, 0, 0, 0, 6, 4089, 0, 19, 0, 0, 0, 917, 0, 3157, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4113, 4093, 4096, 4096, 4096, 4114, 4074] avg=4096.0000 max=4114 max_rank=6 min=4074 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1175, 3134, 0, 0, 0, 0, 4075, 25, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4071, 21, 0, 0, 0, 0, 4088, 0, 19, 0, 0, 0, 917, 0, 2950, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4309, 4100, 4099, 4096, 4095, 4095, 4107, 3867] avg=4096.0000 max=4309 max_rank=0 min=3867 min_rank=7 imbalance_ratio=1.052002 max_load_reduction=28459 max_load_reduction_pct=0.868500 moved_to_echo=16164 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1251, 3127, 0, 0, 0, 0, 4045, 25, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4071, 21, 0, 0, 0, 14, 4087, 0, 19, 0, 0, 0, 915, 0, 2902, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4378, 4070, 4090, 4096, 4096, 4094, 4120, 3824] avg=4096.0000 max=4378 max_rank=0 min=3824 min_rank=7 imbalance_ratio=1.068848 max_load_reduction=28390 max_load_reduction_pct=0.866394 moved_to_echo=16134 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1397, 3117, 0, 0, 0, 0, 4049, 25, 0, 0, 0, 0, 0, 0, 3, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 21, 0, 0, 0, 35, 4085, 0, 19, 0, 0, 0, 912, 0, 2745, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4514, 4074, 4094, 4095, 4095, 4092, 4139, 3665] avg=4096.0000 max=4514 max_rank=0 min=3665 min_rank=7 imbalance_ratio=1.102051 max_load_reduction=28254 max_load_reduction_pct=0.862244 moved_to_echo=15991 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4, 4, 5, 358, 3132, 0, 0, 0, 0, 4081, 25, 0, 0, 0, 0, 0, 2, 0, 0, 4087, 3, 0, 0, 0, 5, 4, 0, 4091, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 4066, 21, 0, 0, 0, 1, 4055, 0, 19, 0, 0, 0, 917, 3, 3782, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3503, 4106, 4092, 4100, 4091, 4099, 4075, 4702] avg=4096.0000 max=4702 max_rank=7 min=3503 min_rank=0 imbalance_ratio=1.147949 max_load_reduction=28066 max_load_reduction_pct=0.856506 moved_to_echo=16967 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 636, 3138, 0, 0, 0, 0, 4068, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 4, 4078, 24, 0, 0, 0, 1, 4060, 0, 24, 0, 0, 0, 897, 0, 3520, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3774, 4086, 4102, 4093, 4103, 4106, 4085, 4419] avg=4096.0000 max=4419 max_rank=7 min=3774 min_rank=0 imbalance_ratio=1.078857 max_load_reduction=28349 max_load_reduction_pct=0.865143 moved_to_echo=16713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 852, 3135, 0, 0, 0, 1, 4094, 25, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4071, 21, 0, 0, 0, 0, 4005, 0, 19, 0, 0, 0, 917, 0, 3336, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3987, 4120, 4099, 4096, 4096, 4093, 4024, 4253] avg=4096.0000 max=4253 max_rank=7 min=3987 min_rank=0 imbalance_ratio=1.038330 max_load_reduction=28515 max_load_reduction_pct=0.870209 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 7808, 25088, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7424, 128, 25472, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32640, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32640, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32640, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2016, 0, 4096, 0, 0, 3129, 0, 0, 0, 1, 21, 4095, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 3025, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4090, 3, 1, 646, 1, 4089, 4, 6, 4022, 0, 6, 0, 1, 98, 4067, 3, 4084, 7, 6, 4090, 22, 0, 4091, 0, 7, 0, 3418, 0, 6, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 5, 1454, 0, 4092, 0, 0, 3517, 0, 0, 0, 47, 19, 4094, 0, 4093, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 3165, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3459, 0, 4094, 0, 0, 3673, 0, 0, 0, 8, 20, 4093, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1037, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1706, 3, 4096, 0, 0, 3509, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 2, 2974, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3478, 0, 4096, 0, 0, 3335, 0, 0, 0, 4, 1, 4096, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 1375, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2802, 1, 4095, 0, 0, 3632, 0, 0, 0, 1, 0, 4096, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 2, 1756, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3112, 0, 4096, 0, 0, 3487, 0, 0, 0, 22, 22, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 1548, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2016, 0, 1763, 0, 0, 0, 0, 0, 0, 3129, 0, 0, 526, 28, 1, 0, 0, 1, 21, 4067, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3025, 1807, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3779, 3684, 4089, 4096, 4095, 4833, 4096] avg=4096.0000 max=4833 max_rank=6 min=3684 min_rank=2 imbalance_ratio=1.179932 max_load_reduction=27935 max_load_reduction_pct=0.852509 moved_to_echo=17120 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 5, 0, 0, 0, 0, 1454, 0, 1761, 0, 0, 0, 0, 0, 0, 3517, 0, 0, 526, 28, 1, 0, 0, 47, 19, 4066, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3165, 1805, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 3215, 4072, 4132, 4093, 4089, 4970, 4096] avg=4096.0000 max=4970 max_rank=6 min=3215 min_rank=1 imbalance_ratio=1.213379 max_load_reduction=27798 max_load_reduction_pct=0.848328 moved_to_echo=17248 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 1706, 3, 1763, 0, 0, 0, 0, 0, 0, 3509, 0, 0, 526, 28, 1, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2974, 1807, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3472, 4064, 4068, 4094, 4095, 4783, 4096] avg=4096.0000 max=4783 max_rank=6 min=3472 min_rank=1 imbalance_ratio=1.167725 max_load_reduction=27985 max_load_reduction_pct=0.854034 moved_to_echo=17068 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 3, 1, 0, 0, 0, 0, 646, 1, 1760, 4, 0, 0, 0, 0, 6, 4022, 0, 6, 525, 28, 1, 0, 0, 1, 98, 4039, 0, 0, 0, 0, 3, 4084, 7, 6, 0, 0, 0, 0, 4089, 22, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3418, 1804, 0, 0, 0, 0, 6, 0, 0, 4091, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 2411, 4588, 4138, 4100, 4111, 5229, 4097] avg=4096.0000 max=5229 max_rank=6 min=2411 min_rank=1 imbalance_ratio=1.276611 max_load_reduction=27539 max_load_reduction_pct=0.840424 moved_to_echo=17537 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3459, 0, 1762, 0, 0, 0, 0, 0, 0, 3673, 0, 0, 526, 28, 1, 0, 0, 8, 20, 4065, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1806, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 5221, 4228, 4093, 4096, 4095, 2843, 4096] avg=4096.0000 max=5221 max_rank=1 min=2843 min_rank=6 imbalance_ratio=1.274658 max_load_reduction=27547 max_load_reduction_pct=0.840668 moved_to_echo=15130 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 0, 3478, 0, 1752, 0, 0, 0, 0, 0, 0, 3335, 0, 0, 532, 34, 2, 0, 0, 4, 1, 4062, 0, 0, 0, 0, 0, 4094, 0, 0, 3, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1375, 1812, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 5230, 3903, 4067, 4098, 4093, 3189, 4095] avg=4096.0000 max=5230 max_rank=1 min=3189 min_rank=6 imbalance_ratio=1.276855 max_load_reduction=27538 max_load_reduction_pct=0.840393 moved_to_echo=15475 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2802, 1, 1762, 0, 0, 0, 0, 0, 0, 3632, 0, 0, 526, 28, 1, 0, 0, 1, 0, 4068, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1756, 1807, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4565, 4187, 4069, 4095, 4095, 3565, 4096] avg=4096.0000 max=4565 max_rank=1 min=3565 min_rank=6 imbalance_ratio=1.114502 max_load_reduction=28203 max_load_reduction_pct=0.860687 moved_to_echo=15851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3112, 0, 1763, 0, 0, 0, 0, 0, 0, 3487, 0, 0, 526, 28, 1, 0, 0, 22, 22, 4068, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1548, 1807, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4875, 4042, 4112, 4096, 4095, 3356, 4096] avg=4096.0000 max=4875 max_rank=1 min=3356 min_rank=6 imbalance_ratio=1.190186 max_load_reduction=27893 max_load_reduction_pct=0.851227 moved_to_echo=15643 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 18304, 14464, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[18688, 128, 14208, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 28416, 0, 128, 4224, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 256, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 4093, 0, 4096, 4074, 0, 0, 0, 0, 23, 0, 0, 2, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4, 4063, 0, 4089, 3934, 7, 4, 3, 0, 159, 0, 6, 8, 0, 7, 4089, 4090, 2, 4081, 0, 6, 7, 4089, 4, 9, 0, 0, 16, 0, 4087, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 4089, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 4061, 0, 4096, 4077, 0, 0, 0, 0, 19, 24, 0, 17, 7, 0, 4096, 4092, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4077, 0, 4096, 3988, 0, 0, 0, 0, 115, 3, 0, 9, 4, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 4093, 0, 4096, 4041, 0, 0, 0, 1, 56, 1, 0, 1, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 4086, 0, 4096, 3806, 0, 0, 0, 0, 297, 0, 0, 3, 1, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 4064, 0, 4096, 4054, 0, 0, 0, 0, 44, 31, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 17, 0, 0, 0, 0, 4074, 0, 0, 0, 81, 5, 0, 0, 0, 23, 0, 0, 3998, 0, 0, 0, 2, 0, 0, 4090, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4160, 4021, 4092, 4098, 4095, 4094, 4098] avg=4096.0000 max=4160 max_rank=1 min=4021 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 17, 0, 0, 0, 0, 4089, 0, 0, 0, 81, 5, 0, 0, 0, 7, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4175, 4005, 4090, 4098, 4095, 4094, 4098] avg=4096.0000 max=4175 max_rank=1 min=4005 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 4063, 0, 18, 0, 0, 0, 0, 3934, 7, 4, 3, 80, 5, 0, 0, 0, 159, 0, 6, 3991, 0, 0, 0, 8, 0, 7, 4083, 0, 0, 0, 0, 2, 2, 4081, 0, 0, 0, 0, 0, 6, 7, 4088, 4, 0, 0, 0, 0, 9, 0, 0, 16, 4088, 0, 0, 0, 0, 4087, 4, 0, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4033, 4156, 4098, 4085, 4105, 4113, 4093] avg=4096.0000 max=4156 max_rank=2 min=4033 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16396 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4061, 0, 17, 0, 0, 0, 0, 4077, 0, 0, 0, 81, 5, 0, 0, 0, 19, 24, 0, 3998, 0, 0, 0, 17, 7, 0, 4090, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4087, 0, 0, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4163, 4041, 4114, 4098, 4095, 4090, 4089] avg=4096.0000 max=4163 max_rank=1 min=4041 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16372 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4064, 0, 17, 0, 0, 0, 0, 4054, 0, 0, 0, 81, 5, 0, 0, 0, 44, 31, 0, 3998, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4140, 4073, 4090, 4098, 4095, 4093, 4098] avg=4096.0000 max=4140 max_rank=1 min=4073 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 0, 11, 0, 0, 0, 0, 3806, 0, 0, 0, 82, 8, 0, 0, 0, 297, 0, 0, 4003, 0, 0, 0, 3, 1, 0, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 3, 2, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3896, 4300, 4090, 4096, 4093, 4096, 4100] avg=4096.0000 max=4300 max_rank=2 min=3896 min_rank=1 imbalance_ratio=1.049805 max_load_reduction=28468 max_load_reduction_pct=0.868774 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4077, 0, 17, 0, 0, 0, 0, 3988, 0, 0, 0, 81, 5, 0, 0, 0, 115, 3, 0, 3998, 0, 0, 0, 9, 4, 0, 4090, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4092, 0, 0, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4074, 4116, 4103, 4098, 4095, 4094, 4094] avg=4096.0000 max=4116 max_rank=2 min=4074 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 17, 0, 0, 0, 0, 4041, 0, 0, 0, 81, 5, 0, 0, 1, 56, 1, 0, 3998, 0, 0, 0, 1, 0, 0, 4090, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4127, 4056, 4091, 4097, 4095, 4094, 4098] avg=4096.0000 max=4127 max_rank=1 min=4056 min_rank=2 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32640, 0, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 768, 128, 128, 32000, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32128, 128, 128, 128, 768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 89, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 4096, 0, 0, 4096, 4096, 0, 199, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3897, 0, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 4096, 0, 0, 4096, 4096, 0, 58, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 57, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[8, 7, 44, 4087, 2, 4088, 0, 6, 4089, 4090, 6, 83, 0, 0, 0, 0, 4088, 0, 6, 4088, 4083, 0, 0, 6, 3973, 0, 0, 7, 0, 0, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 123, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 4096, 0, 0, 4096, 4096, 0, 159, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3, 4096, 0, 4095, 0, 0, 4096, 4096, 0, 300, 0, 0, 1, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 3793, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 133, 4, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4096, 4052, 4096, 4098, 4096, 4144, 4094] avg=4096.0000 max=4144 max_rank=6 min=4052 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16432 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 133, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4096, 4020, 4096, 4096, 4096, 4176, 4096] avg=4096.0000 max=4176 max_rank=6 min=4020 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 0, 0, 133, 4, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4096, 4086, 4096, 4096, 4096, 4111, 4095] avg=4096.0000 max=4111 max_rank=6 min=4086 min_rank=2 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 133, 4, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4096, 4021, 4096, 4096, 4096, 4171, 4101] avg=4096.0000 max=4171 max_rank=6 min=4021 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3897, 0, 0, 0, 133, 4, 1, 0, 0, 0, 4, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4096, 4162, 4097, 4096, 4096, 4035, 4098] avg=4096.0000 max=4162 max_rank=2 min=4035 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16325 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[8, 7, 44, 4083, 0, 0, 0, 0, 2, 4088, 0, 6, 0, 0, 0, 0, 3956, 0, 6, 83, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 6, 4088, 0, 0, 0, 0, 4083, 0, 0, 6, 0, 0, 0, 0, 3973, 0, 0, 7, 133, 4, 0, 0, 0, 0, 0, 7, 4088, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4096, 4045, 4090, 4094, 4089, 4117, 4095] avg=4096.0000 max=4142 max_rank=0 min=4045 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16395 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3960, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 3937, 0, 0, 0, 136, 11, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4097, 4119, 4098, 4096, 4103, 4084, 4095] avg=4096.0000 max=4119 max_rank=2 min=4076 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 4092, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 300, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3793, 0, 0, 0, 133, 4, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4263, 4099, 4096, 4096, 3930, 4094] avg=4096.0000 max=4263 max_rank=2 min=3930 min_rank=6 imbalance_ratio=1.040771 max_load_reduction=28505 max_load_reduction_pct=0.869904 moved_to_echo=16216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31744, 0, 128, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[31744, 0, 0, 128, 1152, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 1177, 0, 4096, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 0, 47, 0, 0, 0, 2873, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 4094, 1230, 0, 4096, 0, 0, 0, 4081, 4096, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 2839, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 7, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 1985, 0, 4096, 0, 0, 0, 4069, 4096, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 2118, 4096, 0, 0, 0, 34, 0, 0, 0, 0, 1, 0, 4072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4053, 58, 4, 4088, 0, 6, 6, 4091, 4076, 4088, 8, 7, 4, 0, 38, 0, 0, 10, 3998, 4090, 5, 6, 0, 27, 31, 0, 0, 5, 0, 0, 4069] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 563, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 3525, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 1085, 0, 4096, 0, 0, 0, 4084, 4095, 4096, 0, 0, 0, 1, 3, 0, 0, 0, 3008, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 1626, 0, 4096, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 2456, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 2, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 2099, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 1980, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2873, 1177, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4064, 14, 0, 0, 0, 0, 0, 2873, 1222, 24, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4096, 4100, 4125, 4119, 4096, 4086, 4096] avg=4096.0000 max=4125 max_rank=3 min=4050 min_rank=0 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2872, 1230, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4081, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4064, 14, 0, 0, 0, 0, 0, 2839, 1221, 24, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4082, 0, 0, 0, 0, 7, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4096, 4089, 4100, 4084, 4096, 4097, 4104] avg=4096.0000 max=4104 max_rank=7 min=4084 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2873, 563, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4064, 14, 0, 0, 0, 0, 0, 3525, 1222, 24, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4, 0, 4094, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3436, 4096, 4104, 4084, 4771, 4096, 4082, 4099] avg=4096.0000 max=4771 max_rank=4 min=3436 min_rank=0 imbalance_ratio=1.164795 max_load_reduction=27997 max_load_reduction_pct=0.854401 moved_to_echo=17048 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2843, 58, 4, 0, 0, 0, 0, 4088, 0, 6, 6, 0, 0, 0, 0, 4091, 8, 0, 8, 0, 0, 0, 0, 7, 4, 0, 38, 4044, 14, 0, 0, 0, 0, 10, 3998, 1209, 24, 0, 0, 4090, 5, 6, 0, 0, 0, 0, 0, 27, 31, 0, 0, 4074, 0, 0, 0, 5, 0, 0, 4069, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[2905, 4100, 4107, 4107, 5241, 4101, 4132, 4075] avg=4096.0000 max=5241 max_rank=4 min=2905 min_rank=0 imbalance_ratio=1.279541 max_load_reduction=27527 max_load_reduction_pct=0.840057 moved_to_echo=17549 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2873, 1985, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4069, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4064, 14, 0, 0, 0, 0, 0, 2118, 1222, 24, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 4082, 0, 0, 0, 0, 1, 0, 4072, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4858, 4096, 4077, 4087, 3364, 4096, 4116, 4074] avg=4096.0000 max=4858 max_rank=0 min=3364 min_rank=4 imbalance_ratio=1.186035 max_load_reduction=27910 max_load_reduction_pct=0.851746 moved_to_echo=15650 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2873, 1085, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4084, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4063, 14, 0, 0, 0, 0, 0, 3008, 1222, 24, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 12, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3958, 4096, 4092, 4081, 4254, 4096, 4082, 4109] avg=4096.0000 max=4254 max_rank=4 min=3958 min_rank=0 imbalance_ratio=1.038574 max_load_reduction=28514 max_load_reduction_pct=0.870178 moved_to_echo=16541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2861, 1626, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4069, 15, 0, 0, 0, 0, 0, 2456, 1227, 26, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4081, 0, 0, 0, 0, 2, 0, 4096, 8, 5, 4, 0] rerouted_tokens_per_ep_rank=[4487, 4092, 4095, 4092, 3709, 4091, 4087, 4115] avg=4096.0000 max=4487 max_rank=0 min=3709 min_rank=4 imbalance_ratio=1.095459 max_load_reduction=28281 max_load_reduction_pct=0.863068 moved_to_echo=16002 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2873, 2099, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4064, 14, 0, 0, 0, 0, 0, 1980, 1222, 24, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4972, 4096, 4104, 4092, 3226, 4096, 4086, 4096] avg=4096.0000 max=4972 max_rank=0 min=3226 min_rank=4 imbalance_ratio=1.213867 max_load_reduction=27796 max_load_reduction_pct=0.848267 moved_to_echo=15504 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 23040, 9856, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 22912, 9856, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 256, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 12, 0, 4096, 6, 0, 0, 0, 4096, 2, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 10, 0, 4096, 0, 1, 0, 0, 0, 0, 4094, 4075, 0, 0, 4, 0, 4096, 1, 0, 0, 0, 4095, 9, 0, 0, 4096, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 4080, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 2, 0, 4094, 1, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4091, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 4012, 0, 0, 80, 0, 4096, 8, 0, 0, 0, 4095, 0, 0, 5, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 16, 0, 4095, 1, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 5, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4078, 0, 1, 0, 4096, 0, 1, 0, 0, 0, 0, 4089, 4064, 0, 0, 15, 0, 4096, 7, 0, 0, 0, 4096, 27, 0, 7, 4094, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4, 0, 48, 10, 4047, 5, 206, 6, 4087, 7, 0, 7, 4, 5, 7, 4084, 3852, 0, 6, 0, 39, 4083, 0, 0, 0, 0, 4078, 4, 0, 0, 4091, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 0, 4075, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4080, 0, 0, 9, 21, 7, 6, 3, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4083, 4096, 4095, 4126, 4090, 4093, 4096] avg=4096.0000 max=4126 max_rank=4 min=4083 min_rank=1 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16405 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 12, 21, 7, 6, 3, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4060, 4096, 4096, 4140, 4096, 4095, 4096] avg=4096.0000 max=4140 max_rank=4 min=4060 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16427 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 0, 0, 0, 4074, 0, 2, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 21, 8, 6, 3, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4076, 4095, 4096, 4131, 4090, 4096, 4096] avg=4096.0000 max=4131 max_rank=4 min=4076 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4089, 0, 0, 0, 4070, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4012, 0, 0, 80, 21, 7, 6, 3, 0, 4090, 8, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4074, 4096, 4088, 4129, 4098, 4092, 4101] avg=4096.0000 max=4129 max_rank=4 min=4074 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16420 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 0, 4073, 0, 10, 0, 0, 0, 0, 0, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 6, 2, 0, 4075, 0, 0, 4, 23, 7, 7, 4, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 4085, 9, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4083, 4091, 4108, 4120, 4090, 4094, 4093] avg=4096.0000 max=4120 max_rank=4 min=4083 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4087, 0, 0, 0, 4057, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4064, 0, 0, 15, 21, 7, 6, 3, 0, 4090, 7, 0, 0, 0, 0, 0, 0, 0, 4093, 27, 0, 0, 0, 0, 0, 7, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4058, 4097, 4089, 4116, 4097, 4120, 4102] avg=4096.0000 max=4120 max_rank=6 min=4058 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16435 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 48, 10, 4083, 0, 0, 0, 4026, 5, 206, 6, 0, 0, 0, 0, 4087, 7, 0, 7, 0, 0, 0, 0, 4, 5, 7, 4084, 0, 0, 0, 0, 3852, 0, 6, 0, 21, 8, 5, 2, 39, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4243, 4101, 4100, 3894, 4117, 4080, 4088] avg=4096.0000 max=4243 max_rank=1 min=3894 min_rank=4 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 0, 4075, 0, 16, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4075, 0, 0, 0, 21, 7, 6, 3, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4091, 4096, 4096, 4112, 4090, 4098, 4096] avg=4096.0000 max=4112 max_rank=4 min=4089 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32384, 0, 128, 128, 256, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 128, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4, 0, 4096, 4089, 4096, 0, 0, 0, 0, 4087] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4091, 4095, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 10, 0, 4096, 4059, 4096, 0, 0, 0, 0, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4085, 4096, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[13, 7, 4062, 1, 5, 0, 0, 0, 0, 0, 0, 7, 0, 4089, 10, 11, 4081, 9, 0, 4087, 0, 0, 142, 4, 4089, 4088, 3962, 0, 2, 0, 8, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 6, 0, 0, 0, 18, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4095, 4097, 4078, 4113, 4097] avg=4096.0000 max=4113 max_rank=6 min=4078 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 6, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4096, 4096, 4095, 4097, 4083, 4114, 4097] avg=4096.0000 max=4114 max_rank=6 min=4083 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 6, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4096, 4096, 4095, 4097, 4072, 4114, 4096] avg=4096.0000 max=4114 max_rank=6 min=4072 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 10, 0, 4041, 6, 0, 0, 0, 17, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4096, 4096, 4095, 4097, 4057, 4113, 4087] avg=4096.0000 max=4127 max_rank=0 min=4057 min_rank=5 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4, 0, 4070, 6, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4096, 4096, 4096, 4097, 4080, 4114, 4088] avg=4096.0000 max=4114 max_rank=6 min=4080 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4083, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4082, 11, 0, 0, 0, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 8, 2, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4097, 4098, 4088, 4096, 4094, 4106, 4104] avg=4096.0000 max=4106 max_rank=6 min=4085 min_rank=0 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[13, 7, 4056, 1, 0, 0, 0, 0, 5, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 7, 4086, 1, 0, 0, 0, 4088, 10, 11, 0, 0, 0, 0, 4081, 9, 0, 1, 0, 0, 0, 0, 0, 0, 142, 4, 4069, 6, 0, 0, 0, 18, 3962, 0, 0, 0, 0, 0, 2, 0, 8, 4091, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4094, 4094, 4109, 4091, 4221, 3980, 4102] avg=4096.0000 max=4221 max_rank=5 min=3980 min_rank=6 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 6, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4097, 4096, 4095, 4096, 4083, 4114, 4097] avg=4096.0000 max=4114 max_rank=6 min=4083 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16390 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 256, 32640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 256, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4088, 4096, 0, 0, 4096, 7, 4096, 0, 11, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4095, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4094, 4096, 0, 0, 4096, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4092, 4096, 0, 0, 4096, 3, 4096, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3991, 4079, 9, 0, 0, 4072, 4087, 3896, 1, 4, 4088, 0, 4087, 12, 0, 4089, 9, 0, 7, 273, 4, 0, 2, 7, 0, 8, 5, 15, 0, 2, 4, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4095, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 4096, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4083, 10, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4122, 4096, 4095, 4061, 4095, 4093, 4097] avg=4096.0000 max=4122 max_rank=1 min=4061 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16346 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 13, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4060, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4083, 10, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4122, 4096, 4095, 4062, 4095, 4093, 4097] avg=4096.0000 max=4122 max_rank=1 min=4062 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16347 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4083, 10, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4122, 4096, 4097, 4059, 4095, 4094, 4096] avg=4096.0000 max=4122 max_rank=1 min=4059 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16344 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 11, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4083, 10, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4122, 4103, 4096, 4053, 4095, 4093, 4097] avg=4096.0000 max=4122 max_rank=1 min=4053 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16338 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 6, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 1, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4083, 10, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4122, 4099, 4095, 4057, 4097, 4094, 4095] avg=4096.0000 max=4122 max_rank=1 min=4057 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16343 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3991, 13, 9, 0, 0, 0, 0, 0, 0, 0, 26, 3896, 0, 0, 0, 0, 1, 4, 4088, 0, 0, 0, 0, 0, 0, 12, 0, 4088, 0, 0, 0, 0, 9, 0, 7, 273, 4051, 1, 0, 0, 4, 0, 2, 7, 4086, 0, 0, 0, 0, 8, 5, 15, 4066, 10, 0, 0, 0, 2, 4, 17, 4072, 1, 0, 0] rerouted_tokens_per_ep_rank=[4013, 3922, 4093, 4100, 4341, 4099, 4104, 4096] avg=4096.0000 max=4341 max_rank=4 min=3922 min_rank=1 imbalance_ratio=1.059814 max_load_reduction=28427 max_load_reduction_pct=0.867523 moved_to_echo=16640 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 2, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 13, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4114, 4089, 4095, 4073, 4096, 4103, 4095] avg=4096.0000 max=4114 max_rank=1 min=4073 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16367 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 13, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4060, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4083, 10, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4122, 4096, 4095, 4062, 4096, 4094, 4095] avg=4096.0000 max=4122 max_rank=1 min=4062 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16347 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 256, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 384, 32512, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 251, 0, 0, 3906, 0, 4, 4092, 4, 0, 0, 4096, 0, 9, 4096, 4096, 0, 0, 4023, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 119, 0, 0, 3980, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4091, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 154, 0, 0, 3978, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 4096, 4096, 0, 0, 4056, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 314, 0, 0, 3895, 0, 2, 4095, 0, 0, 0, 4096, 0, 18, 4096, 4096, 0, 0, 3964, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 216, 0, 0, 4017, 0, 0, 4096, 1, 0, 0, 4096, 0, 8, 4096, 4096, 0, 0, 3950, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[72, 10, 10, 8, 26, 0, 0, 4086, 0, 8, 789, 7, 9, 3351, 0, 0, 4088, 0, 0, 0, 4077, 0, 0, 3969, 4083, 5, 0, 4091, 7, 4065, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 225, 0, 0, 3904, 0, 1, 4096, 4, 0, 0, 4096, 0, 1, 4096, 4096, 0, 0, 4058, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 81, 0, 0, 4021, 0, 0, 4096, 3, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4087, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 154, 0, 3827, 0, 0, 0, 0, 3978, 0, 0, 212, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4, 4096, 0, 0, 0, 0, 57, 0, 0, 4056, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4095, 3981, 4190, 4096, 4111, 4113, 4097] avg=4096.0000 max=4190 max_rank=3 min=3981 min_rank=2 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 119, 0, 3827, 0, 0, 0, 0, 3980, 0, 0, 212, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 57, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4095, 3946, 4192, 4098, 4107, 4148, 4097] avg=4096.0000 max=4192 max_rank=3 min=3946 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 216, 0, 3827, 0, 0, 0, 0, 4017, 0, 0, 212, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 11, 0, 8, 4096, 0, 0, 0, 0, 57, 0, 0, 3950, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4095, 4043, 4229, 4097, 4115, 4007, 4097] avg=4096.0000 max=4229 max_rank=3 min=4007 min_rank=6 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 251, 0, 3827, 0, 0, 0, 0, 3906, 0, 4, 212, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 11, 0, 9, 4096, 0, 0, 0, 0, 57, 0, 0, 4023, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4094, 4078, 4122, 4096, 4116, 4080, 4097] avg=4096.0000 max=4122 max_rank=3 min=4078 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 314, 0, 3827, 0, 0, 0, 0, 3895, 0, 2, 212, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 11, 0, 18, 4096, 0, 0, 0, 0, 57, 0, 0, 3964, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4095, 4141, 4109, 4095, 4125, 4021, 4097] avg=4096.0000 max=4141 max_rank=2 min=4021 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[72, 10, 10, 8, 4066, 0, 0, 0, 26, 0, 0, 4085, 0, 0, 0, 0, 0, 8, 789, 7, 3814, 0, 0, 0, 9, 3351, 0, 0, 211, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3969, 0, 0, 0, 0, 58, 5, 0, 4091, 0, 0, 0, 0, 7, 4065, 7, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4111, 4618, 3571, 4088, 3980, 4154, 4080] avg=4096.0000 max=4618 max_rank=2 min=3571 min_rank=3 imbalance_ratio=1.127441 max_load_reduction=28150 max_load_reduction_pct=0.859070 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 6, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 225, 0, 3828, 0, 0, 0, 0, 3904, 0, 1, 217, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 10, 0, 1, 4096, 0, 0, 0, 0, 43, 0, 0, 4058, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 2, 1, 0] rerouted_tokens_per_ep_rank=[4092, 4088, 4053, 4122, 4099, 4107, 4101, 4106] avg=4096.0000 max=4122 max_rank=3 min=4053 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 81, 0, 3827, 0, 0, 0, 0, 4021, 0, 0, 212, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 57, 0, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4095, 3908, 4233, 4099, 4107, 4144, 4097] avg=4096.0000 max=4233 max_rank=3 min=3908 min_rank=2 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 128, 0, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31104, 0, 128, 1792, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 2176, 128, 30720, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 1, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[417, 0, 0, 8, 8, 0, 54, 9, 4088, 16, 3565, 1, 0, 0, 0, 8, 4073, 4080, 4071, 10, 7, 4, 0, 15, 4037, 0, 5, 4086, 0, 104, 180, 3922] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 4, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 1, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[6, 0, 2, 0, 0, 0, 0, 0, 4085, 0, 4090, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 10, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4030, 13, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 66, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4088, 4160, 4097, 4099, 4094, 4104, 4083] avg=4096.0000 max=4160 max_rank=2 min=4043 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4030, 13, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 66, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4088, 4160, 4097, 4099, 4094, 4104, 4083] avg=4096.0000 max=4160 max_rank=2 min=4043 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4030, 13, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 66, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4088, 4162, 4097, 4099, 4092, 4104, 4083] avg=4096.0000 max=4162 max_rank=2 min=4043 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[417, 0, 0, 8, 4022, 13, 0, 0, 8, 0, 54, 9, 4078, 0, 0, 0, 66, 16, 3565, 1, 0, 0, 0, 0, 0, 0, 0, 8, 4080, 1, 0, 0, 2, 0, 4071, 10, 0, 0, 0, 0, 7, 4, 0, 15, 4070, 0, 0, 0, 4037, 0, 5, 8, 0, 0, 0, 0, 0, 104, 180, 3909, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4460, 4149, 3648, 4089, 4083, 4096, 4050, 4193] avg=4096.0000 max=4460 max_rank=0 min=3648 min_rank=2 imbalance_ratio=1.088867 max_load_reduction=28308 max_load_reduction_pct=0.863892 moved_to_echo=16422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4031, 19, 3, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 64, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4091, 4160, 4098, 4096, 4097, 4096, 4077] avg=4096.0000 max=4160 max_rank=2 min=4053 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16366 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4, 0, 4025, 13, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 66, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4088, 4160, 4097, 4097, 4096, 4104, 4083] avg=4096.0000 max=4160 max_rank=2 min=4043 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 2, 0, 4019, 13, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 66, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4088, 4156, 4096, 4099, 4102, 4104, 4083] avg=4096.0000 max=4156 max_rank=2 min=4040 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4030, 13, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 66, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4088, 4162, 4097, 4096, 4097, 4102, 4083] avg=4096.0000 max=4162 max_rank=2 min=4043 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[512, 0, 128, 128, 32256, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 256, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 128, 32256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 3, 0, 4081, 3774, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 265, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 69, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4084, 3827, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 241, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 40, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4083, 3821, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 203, 0, 0, 0, 4096, 4096, 0, 0, 7, 0, 78, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 5, 0, 3925, 2884, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1328, 0, 0, 0, 4096, 4096, 0, 0, 3, 0, 47, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 0, 4074, 3701, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 405, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 11, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[9, 55, 8, 4062, 3095, 8, 0, 4077, 7, 3, 9, 8, 4086, 12, 1, 4, 4084, 0, 11, 1233, 10, 0, 0, 4082, 3796, 0, 0, 17, 5, 0, 0, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 2, 0, 4087, 3633, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 414, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 55, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 3, 0, 4080, 4050, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 35, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 24, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 4081, 25, 0, 0, 0, 3774, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 3598, 496, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 14, 0, 0, 0, 69, 0, 4057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4272, 4094, 4096, 3846, 4096, 4129, 4126] avg=4096.0000 max=4272 max_rank=1 min=3846 min_rank=4 imbalance_ratio=1.042969 max_load_reduction=28496 max_load_reduction_pct=0.869629 moved_to_echo=16197 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4084, 25, 0, 0, 0, 3827, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 3598, 496, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 14, 0, 0, 0, 40, 0, 4057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4325, 4094, 4096, 3822, 4096, 4129, 4097] avg=4096.0000 max=4325 max_rank=1 min=3822 min_rank=4 imbalance_ratio=1.055908 max_load_reduction=28443 max_load_reduction_pct=0.868011 moved_to_echo=16144 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4074, 25, 0, 0, 0, 3701, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 3598, 496, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 0, 405, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 14, 0, 0, 0, 11, 0, 4057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4199, 4094, 4096, 3986, 4096, 4129, 4068] avg=4096.0000 max=4199 max_rank=1 min=3986 min_rank=4 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4083, 25, 0, 0, 0, 3821, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 3598, 496, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 0, 203, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 7, 19, 14, 0, 0, 0, 78, 0, 4057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4319, 4094, 4096, 3784, 4096, 4136, 4135] avg=4096.0000 max=4319 max_rank=1 min=3784 min_rank=4 imbalance_ratio=1.054443 max_load_reduction=28449 max_load_reduction_pct=0.868195 moved_to_echo=16151 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[9, 55, 8, 4062, 25, 0, 0, 0, 3095, 8, 0, 495, 0, 0, 0, 0, 7, 3, 9, 8, 3582, 494, 0, 0, 4086, 12, 1, 4, 0, 0, 0, 0, 3571, 0, 11, 1233, 0, 0, 0, 0, 10, 0, 0, 4082, 0, 0, 0, 0, 3796, 0, 0, 17, 19, 14, 0, 0, 5, 0, 0, 4047, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 3598, 4103, 4103, 4815, 4092, 3846, 4052] avg=4096.0000 max=4815 max_rank=4 min=3598 min_rank=1 imbalance_ratio=1.175537 max_load_reduction=27953 max_load_reduction_pct=0.853058 moved_to_echo=16805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 3925, 25, 0, 0, 0, 2884, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 3598, 496, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 0, 1328, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 3, 19, 14, 0, 0, 0, 47, 0, 4057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3955, 3382, 4094, 4096, 4909, 4096, 4132, 4104] avg=4096.0000 max=4909 max_rank=4 min=3382 min_rank=1 imbalance_ratio=1.198486 max_load_reduction=27859 max_load_reduction_pct=0.850189 moved_to_echo=17241 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4087, 28, 3, 0, 0, 3633, 0, 0, 492, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 497, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 3576, 0, 0, 414, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 1, 23, 18, 0, 0, 0, 55, 0, 4050, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4125, 4101, 4089, 3990, 4100, 4138, 4105] avg=4096.0000 max=4138 max_rank=6 min=3990 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 4080, 25, 0, 0, 0, 4050, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 3598, 496, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 14, 0, 0, 0, 24, 0, 4057, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4548, 4094, 4096, 3616, 4096, 4129, 4081] avg=4096.0000 max=4548 max_rank=1 min=3616 min_rank=4 imbalance_ratio=1.110352 max_load_reduction=28220 max_load_reduction_pct=0.861206 moved_to_echo=15922 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[28672, 0, 128, 4224, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 28800, 3968, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[28800, 128, 0, 4096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32512, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 384, 0, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 0, 0, 128, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4083, 4050, 217, 4079, 0, 0, 4096, 112, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 0, 0, 0, 4096, 0, 0, 0, 2831, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 4070, 61, 4077, 0, 0, 4096, 23, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 0, 4096, 0, 0, 0, 3428, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4094, 4090, 54, 4096, 0, 0, 4096, 33, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 0, 0, 4096, 0, 0, 0, 3759, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4088, 4084, 108, 4096, 0, 0, 4096, 22, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 0, 4096, 0, 0, 0, 3653, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4080, 4089, 149, 3360, 11, 14, 4082, 825, 6, 0, 0, 4084, 9, 4016, 0, 10, 0, 7, 12, 0, 0, 12, 1935, 0, 0, 0, 4087, 8, 0, 17, 1955, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 35, 4088, 0, 0, 4096, 8, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 0, 0, 4096, 0, 0, 0, 3752, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4095, 4092, 12, 4096, 0, 0, 4096, 39, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 434, 0, 0, 0, 4096, 0, 0, 0, 3616, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 39, 4094, 0, 0, 4096, 29, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 0, 4096, 0, 0, 0, 3347, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 54, 4096, 0, 0, 0, 0, 0, 0, 3960, 33, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 258, 0, 3400, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3759, 0, 673, 129, 0, 0] rerouted_tokens_per_ep_rank=[4165, 3993, 4096, 4103, 4096, 3658, 4096, 4561] avg=4096.0000 max=4561 max_rank=7 min=3658 min_rank=5 imbalance_ratio=1.113525 max_load_reduction=28207 max_load_reduction_pct=0.860809 moved_to_echo=16411 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 14, 61, 4077, 0, 0, 0, 0, 0, 0, 3960, 23, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 630, 0, 3384, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3428, 0, 670, 129, 0, 0] rerouted_tokens_per_ep_rank=[4152, 3983, 4096, 4102, 4098, 4014, 4096, 4227] avg=4096.0000 max=4227 max_rank=7 min=3983 min_rank=1 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16435 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 217, 4079, 0, 0, 0, 0, 0, 0, 3960, 112, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4083, 2, 0, 0, 0, 0, 1012, 0, 3367, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 666, 129, 0, 0] rerouted_tokens_per_ep_rank=[4311, 4072, 4096, 4103, 4085, 4379, 4096, 3626] avg=4096.0000 max=4379 max_rank=5 min=3626 min_rank=7 imbalance_ratio=1.069092 max_load_reduction=28389 max_load_reduction_pct=0.866364 moved_to_echo=16186 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 149, 3360, 0, 0, 0, 0, 11, 14, 3947, 825, 0, 0, 0, 0, 6, 0, 0, 4084, 0, 0, 0, 0, 9, 4016, 0, 10, 7, 0, 0, 0, 0, 7, 12, 0, 4080, 2, 0, 0, 0, 12, 1935, 0, 3399, 0, 0, 0, 0, 0, 4087, 8, 0, 0, 0, 0, 0, 17, 1955, 0, 673, 128, 0, 0] rerouted_tokens_per_ep_rank=[3524, 4797, 4090, 4042, 4101, 5346, 4095, 2773] avg=4096.0000 max=5346 max_rank=5 min=2773 min_rank=7 imbalance_ratio=1.305176 max_load_reduction=27422 max_load_reduction_pct=0.836853 moved_to_echo=16315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 35, 4088, 0, 0, 0, 0, 0, 0, 3960, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 309, 0, 3405, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3752, 0, 674, 129, 0, 0] rerouted_tokens_per_ep_rank=[4138, 3968, 4096, 4103, 4098, 3714, 4096, 4555] avg=4096.0000 max=4555 max_rank=7 min=3714 min_rank=5 imbalance_ratio=1.112061 max_load_reduction=28213 max_load_reduction_pct=0.860992 moved_to_echo=16463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 108, 4096, 0, 0, 0, 0, 0, 0, 3960, 22, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 333, 0, 3395, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3653, 0, 672, 129, 0, 0] rerouted_tokens_per_ep_rank=[4219, 3982, 4096, 4103, 4090, 3728, 4096, 4454] avg=4096.0000 max=4454 max_rank=7 min=3728 min_rank=5 imbalance_ratio=1.087402 max_load_reduction=28314 max_load_reduction_pct=0.864075 moved_to_echo=16368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 12, 4096, 0, 0, 0, 0, 0, 0, 3945, 39, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 434, 0, 3406, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 3616, 0, 676, 130, 0, 0] rerouted_tokens_per_ep_rank=[4111, 3984, 4103, 4109, 4102, 3840, 4097, 4422] avg=4096.0000 max=4422 max_rank=7 min=3840 min_rank=5 imbalance_ratio=1.079590 max_load_reduction=28346 max_load_reduction_pct=0.865051 moved_to_echo=16461 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 39, 4094, 0, 0, 0, 0, 0, 0, 3960, 29, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 684, 0, 3405, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3347, 0, 674, 129, 0, 0] rerouted_tokens_per_ep_rank=[4148, 3989, 4095, 4103, 4098, 4089, 4096, 4150] avg=4096.0000 max=4150 max_rank=7 min=3989 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16433 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 5632, 0, 27264, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 26368, 0, 5504, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 768, 32000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 31744, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True + [2026-08-30 04:55:26] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 680.6 | throughput per GPU (TFLOP/s/GPU): 489.9 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.256692E+00 | loss scale: 1.0 | grad norm: 5.715 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1003, 0, 2051, 13, 1050, 0, 2773, 54, 616, 2303, 0, 25, 1513, 2781, 7, 709, 19, 1331, 1915, 0, 0, 914, 1663, 207, 0, 2802, 3421, 1342, 2471, 1682, 103, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[748, 0, 1473, 31, 1529, 0, 2674, 24, 1191, 1706, 0, 40, 1073, 2784, 17, 1053, 12, 1420, 2091, 0, 0, 976, 1520, 136, 0, 2873, 2971, 1523, 2698, 2044, 161, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[590, 0, 1900, 94, 1347, 0, 1740, 77, 1199, 1446, 0, 79, 1615, 2935, 10, 704, 17, 1246, 2204, 0, 0, 1001, 1686, 431, 0, 2939, 3130, 1500, 2921, 1809, 148, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[832, 0, 2000, 39, 853, 0, 2526, 78, 1226, 1378, 0, 22, 1140, 2542, 24, 612, 5, 1773, 2565, 0, 2, 789, 1290, 122, 1, 3151, 3099, 1523, 2928, 1999, 248, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1184, 0, 1393, 106, 1659, 1, 2228, 51, 1713, 1338, 0, 36, 947, 2887, 7, 756, 12, 2155, 1795, 1, 0, 767, 1474, 255, 1, 2791, 3177, 1717, 2011, 2251, 55, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1112, 0, 1697, 102, 1467, 0, 2234, 69, 1573, 1796, 1, 68, 1645, 2344, 21, 887, 80, 1639, 1913, 1, 1, 814, 1582, 358, 0, 2908, 3025, 1475, 2182, 1719, 55, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[478, 0, 1512, 63, 1154, 0, 2438, 30, 1732, 1695, 0, 32, 1088, 3318, 7, 836, 36, 1646, 1642, 0, 0, 766, 1281, 163, 0, 3231, 3054, 1646, 2179, 2603, 138, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1123, 0, 1429, 63, 970, 0, 2314, 109, 1414, 2036, 1, 53, 1401, 3131, 14, 882, 25, 1490, 1676, 0, 2, 1507, 1210, 122, 0, 2814, 3376, 1478, 1845, 2198, 85, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[590, 0, 1900, 94, 1454, 0, 0, 0, 1347, 0, 1740, 77, 368, 56, 0, 0, 1199, 1446, 0, 79, 808, 222, 0, 0, 1615, 2042, 10, 704, 0, 0, 0, 0, 17, 1246, 2204, 0, 515, 85, 0, 0, 0, 1001, 1686, 431, 1454, 0, 0, 0, 0, 2571, 0, 1500, 0, 0, 0, 0, 2350, 1809, 148, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4038, 3588, 3754, 4371, 4067, 4572, 4071, 4307] avg=4096.0000 max=4572 max_rank=5 min=3588 min_rank=1 imbalance_ratio=1.116211 max_load_reduction=28196 max_load_reduction_pct=0.860474 moved_to_echo=17017 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1003, 0, 2051, 13, 1589, 0, 0, 0, 1050, 0, 2773, 54, 351, 47, 0, 0, 616, 2303, 0, 25, 765, 244, 0, 0, 1513, 1935, 7, 709, 0, 0, 0, 0, 19, 1331, 1915, 0, 436, 81, 0, 0, 0, 914, 1663, 207, 1588, 0, 0, 0, 0, 2451, 0, 1342, 0, 0, 0, 0, 1988, 1682, 103, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4656, 4275, 3953, 4164, 3782, 4372, 3793, 3773] avg=4096.0000 max=4656 max_rank=0 min=3773 min_rank=7 imbalance_ratio=1.136719 max_load_reduction=28112 max_load_reduction_pct=0.857910 moved_to_echo=15720 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[478, 0, 1512, 63, 1418, 0, 0, 0, 1154, 0, 2438, 30, 405, 42, 0, 0, 1732, 1695, 0, 32, 913, 218, 0, 0, 1088, 2308, 7, 836, 0, 0, 0, 0, 36, 1646, 1642, 0, 384, 97, 0, 0, 0, 766, 1281, 163, 1418, 0, 0, 0, 0, 2826, 0, 1646, 0, 0, 0, 0, 1753, 2603, 138, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3471, 4069, 4590, 4239, 3805, 3628, 4472, 4494] avg=4096.0000 max=4590 max_rank=2 min=3471 min_rank=0 imbalance_ratio=1.120605 max_load_reduction=28178 max_load_reduction_pct=0.859924 moved_to_echo=16399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[748, 0, 1473, 31, 1380, 0, 0, 0, 1529, 0, 2674, 24, 360, 52, 0, 0, 1191, 1706, 0, 40, 766, 210, 0, 0, 1073, 1937, 17, 1053, 0, 0, 0, 0, 12, 1420, 2091, 0, 476, 81, 0, 0, 0, 976, 1520, 136, 1381, 0, 0, 0, 0, 2513, 0, 1523, 0, 0, 0, 0, 2170, 2044, 161, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3632, 4639, 3913, 4080, 4080, 4013, 4036, 4375] avg=4096.0000 max=4639 max_rank=1 min=3632 min_rank=0 imbalance_ratio=1.132568 max_load_reduction=28129 max_load_reduction_pct=0.858429 moved_to_echo=16504 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1184, 0, 1393, 106, 1476, 0, 0, 0, 1659, 1, 2228, 51, 349, 38, 0, 0, 1713, 1338, 0, 36, 794, 226, 0, 0, 947, 2009, 7, 756, 0, 0, 0, 0, 12, 2155, 1795, 1, 354, 84, 0, 0, 0, 767, 1474, 255, 1475, 0, 0, 0, 1, 2442, 0, 1717, 0, 0, 0, 0, 1619, 2251, 55, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4326, 4107, 3719, 4401, 3971, 4160, 3925] avg=4096.0000 max=4401 max_rank=4 min=3719 min_rank=3 imbalance_ratio=1.074463 max_load_reduction=28367 max_load_reduction_pct=0.865692 moved_to_echo=16457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[832, 0, 2000, 39, 1439, 0, 0, 0, 853, 0, 2526, 78, 394, 56, 0, 0, 1226, 1378, 0, 22, 699, 219, 0, 0, 1140, 1769, 24, 612, 0, 0, 0, 0, 5, 1773, 2565, 0, 516, 74, 0, 0, 2, 789, 1290, 122, 1441, 0, 0, 0, 1, 2757, 0, 1523, 0, 0, 0, 0, 2356, 1999, 248, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4310, 3907, 3544, 3545, 4933, 3644, 4281, 4604] avg=4096.0000 max=4933 max_rank=4 min=3544 min_rank=2 imbalance_ratio=1.204346 max_load_reduction=27835 max_load_reduction_pct=0.849457 moved_to_echo=17462 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1112, 0, 1697, 102, 1407, 0, 0, 0, 1467, 0, 2234, 69, 368, 46, 0, 0, 1573, 1796, 1, 68, 650, 214, 0, 0, 1645, 1622, 21, 887, 0, 0, 0, 0, 80, 1639, 1913, 1, 388, 72, 0, 0, 1, 814, 1582, 358, 1404, 0, 0, 0, 0, 2540, 0, 1475, 0, 0, 0, 0, 1748, 1719, 55, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4318, 4184, 4302, 4175, 4093, 4159, 4015, 3522] avg=4096.0000 max=4318 max_rank=0 min=3522 min_rank=7 imbalance_ratio=1.054199 max_load_reduction=28450 max_load_reduction_pct=0.868225 moved_to_echo=15789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1123, 0, 1429, 63, 1569, 0, 0, 0, 970, 0, 2314, 109, 352, 35, 0, 0, 1414, 2036, 1, 53, 862, 239, 0, 0, 1401, 2178, 14, 882, 0, 0, 0, 0, 25, 1490, 1676, 0, 325, 91, 0, 0, 2, 1507, 1210, 122, 1568, 0, 0, 0, 0, 2462, 0, 1478, 0, 0, 0, 0, 1485, 2198, 85, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4184, 3780, 4605, 4475, 3607, 4409, 3940, 3768] avg=4096.0000 max=4605 max_rank=2 min=3607 min_rank=4 imbalance_ratio=1.124268 max_load_reduction=28163 max_load_reduction_pct=0.859467 moved_to_echo=15724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7168, 0, 13568, 512, 11776, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[256, 12800, 15872, 128, 3456, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10752, 13824, 128, 384, 6272, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7552, 11776, 1920, 11776, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10496, 15872, 128, 6528, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10112, 128, 18944, 512, 3072, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 20608, 0, 12288, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15488, 16384, 1024, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[411, 0, 0, 0, 1385, 0, 0, 0, 0, 305, 4079, 0, 1640, 0, 4095, 4095, 4096, 0, 0, 0, 1275, 0, 0, 0, 4095, 3196, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[136, 0, 2, 0, 453, 0, 0, 0, 0, 172, 4041, 0, 2002, 4, 4096, 4091, 4094, 0, 1, 0, 2710, 0, 0, 0, 4095, 2777, 0, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[129, 0, 0, 0, 290, 0, 0, 0, 0, 65, 4023, 0, 2393, 1, 4096, 4078, 4095, 0, 0, 0, 3487, 0, 0, 0, 4095, 1920, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[346, 2, 0, 2, 1275, 3, 3, 0, 1, 104, 4038, 0, 2566, 4, 4095, 4092, 4093, 1, 1, 1, 2406, 0, 0, 0, 4092, 1546, 0, 0, 0, 4094, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[332, 0, 5, 0, 792, 0, 0, 0, 0, 93, 4081, 0, 2711, 2, 4095, 4094, 4095, 1, 0, 0, 2518, 0, 0, 0, 4092, 1759, 1, 0, 1, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[64, 0, 0, 0, 134, 0, 0, 0, 0, 77, 4089, 8, 1646, 0, 4096, 4093, 4094, 0, 0, 0, 3253, 0, 0, 0, 4096, 3022, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[570, 0, 0, 4, 1359, 0, 0, 0, 0, 134, 4090, 9, 1990, 0, 4096, 4086, 4092, 0, 0, 0, 2135, 0, 0, 0, 4095, 2013, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[456, 0, 0, 0, 2233, 0, 0, 0, 0, 119, 4079, 0, 2067, 0, 4096, 4096, 4096, 0, 0, 0, 1289, 0, 0, 0, 4092, 2049, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[129, 0, 0, 0, 3788, 0, 0, 0, 290, 0, 0, 0, 2283, 513, 308, 0, 0, 65, 3921, 0, 0, 0, 0, 0, 2393, 1, 0, 1962, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3487, 0, 0, 0, 1603, 102, 0, 0, 1812, 1920, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3917, 3394, 3986, 4356, 4095, 5192, 3732, 4096] avg=4096.0000 max=5192 max_rank=5 min=3394 min_rank=1 imbalance_ratio=1.267578 max_load_reduction=27576 max_load_reduction_pct=0.841553 moved_to_echo=17115 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[411, 0, 0, 0, 3788, 0, 0, 0, 1385, 0, 0, 0, 2283, 515, 307, 0, 0, 305, 3974, 0, 0, 0, 0, 0, 1640, 0, 0, 1971, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 1275, 0, 0, 0, 1609, 104, 0, 0, 1812, 3196, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4199, 4490, 4279, 3611, 4097, 2988, 5008, 4096] avg=4096.0000 max=5008 max_rank=6 min=2988 min_rank=5 imbalance_ratio=1.222656 max_load_reduction=27760 max_load_reduction_pct=0.847168 moved_to_echo=16189 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[136, 0, 2, 0, 3789, 0, 0, 0, 453, 0, 0, 0, 2283, 514, 307, 0, 0, 172, 3938, 0, 0, 0, 0, 0, 2002, 4, 0, 1969, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 2710, 0, 0, 0, 1608, 103, 0, 0, 1812, 2777, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3927, 3557, 4110, 3975, 4095, 4421, 4589, 4094] avg=4096.0000 max=4589 max_rank=6 min=3557 min_rank=1 imbalance_ratio=1.120361 max_load_reduction=28179 max_load_reduction_pct=0.859955 moved_to_echo=17199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[346, 2, 0, 2, 3789, 0, 0, 0, 1275, 3, 3, 0, 2282, 514, 306, 0, 1, 104, 3935, 0, 0, 0, 0, 0, 2566, 4, 0, 1970, 0, 0, 0, 0, 4093, 1, 1, 1, 0, 0, 0, 0, 2406, 0, 0, 0, 1608, 103, 0, 0, 1810, 1546, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4383, 4040, 4540, 4096, 4117, 3356, 4097] avg=4096.0000 max=4540 max_rank=3 min=3356 min_rank=6 imbalance_ratio=1.108398 max_load_reduction=28228 max_load_reduction_pct=0.861450 moved_to_echo=15666 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[332, 0, 5, 0, 3788, 0, 0, 0, 792, 0, 0, 0, 2282, 515, 307, 0, 0, 93, 3976, 0, 0, 0, 0, 0, 2711, 2, 0, 1970, 0, 0, 0, 0, 4095, 1, 0, 0, 1, 0, 0, 0, 2518, 0, 0, 0, 1609, 104, 0, 0, 1810, 1759, 1, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 3896, 4069, 4683, 4097, 4231, 3570, 4097] avg=4096.0000 max=4683 max_rank=3 min=3570 min_rank=6 imbalance_ratio=1.143311 max_load_reduction=28085 max_load_reduction_pct=0.857086 moved_to_echo=15995 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[456, 0, 0, 0, 3789, 0, 0, 0, 2233, 0, 0, 0, 2282, 515, 307, 0, 0, 119, 3974, 0, 0, 0, 0, 0, 2067, 0, 0, 1971, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 1289, 0, 0, 0, 1610, 104, 0, 0, 1810, 2049, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4245, 5337, 4093, 4038, 4097, 3003, 3859, 4096] avg=4096.0000 max=5337 max_rank=1 min=3003 min_rank=5 imbalance_ratio=1.302979 max_load_reduction=27431 max_load_reduction_pct=0.837128 moved_to_echo=15055 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[570, 0, 0, 4, 3789, 0, 0, 0, 1359, 0, 0, 0, 2283, 514, 307, 0, 0, 134, 3985, 9, 0, 0, 0, 0, 1990, 0, 0, 1966, 0, 0, 0, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 2135, 0, 0, 0, 1606, 104, 0, 0, 1812, 2013, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4363, 4463, 4128, 3956, 4093, 3845, 3825, 4095] avg=4096.0000 max=4463 max_rank=1 min=3825 min_rank=6 imbalance_ratio=1.089600 max_load_reduction=28305 max_load_reduction_pct=0.863800 moved_to_echo=15858 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[64, 0, 0, 0, 3789, 0, 0, 0, 134, 0, 0, 0, 2289, 518, 307, 0, 0, 77, 3978, 8, 0, 0, 0, 0, 1646, 0, 0, 1963, 0, 0, 0, 0, 4094, 0, 0, 0, 4, 0, 0, 0, 3253, 0, 0, 0, 1612, 106, 0, 0, 1807, 3022, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3853, 3248, 4063, 3609, 4098, 4971, 4829, 4097] avg=4096.0000 max=4971 max_rank=5 min=3248 min_rank=1 imbalance_ratio=1.213623 max_load_reduction=27797 max_load_reduction_pct=0.848297 moved_to_echo=17995 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[19200, 0, 0, 0, 12928, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[17024, 128, 0, 15744, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14592, 18304, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[2560, 128, 128, 128, 30336, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 1152, 31744, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7936, 128, 128, 0, 18304, 4224, 2560, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3504, 2, 0, 0, 4096, 4, 4070, 4093, 2070, 3, 4093, 2, 0, 0, 0, 2, 0, 0, 0, 946, 4095, 1, 31, 4073, 1, 3, 1, 477, 0, 1200] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 2863, 0, 1, 0, 4096, 0, 3946, 4096, 856, 0, 4094, 0, 0, 11, 0, 0, 0, 0, 0, 1464, 4096, 0, 71, 4095, 0, 0, 0, 396, 0, 2680] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3039, 0, 1, 1, 4096, 1, 3929, 4096, 1672, 0, 4095, 0, 3, 0, 0, 0, 1, 0, 0, 930, 4096, 2, 8, 4084, 1, 0, 0, 799, 0, 1914] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 1682, 0, 0, 0, 4096, 0, 3961, 4096, 1754, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1608, 4096, 7, 2, 4082, 0, 0, 0, 1007, 0, 2279] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1294, 0, 0, 0, 4096, 0, 3912, 4096, 1493, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2310, 4096, 0, 73, 4092, 0, 0, 0, 978, 0, 2232] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 5, 1625, 0, 0, 0, 4096, 0, 3792, 4096, 778, 0, 4095, 0, 0, 3, 0, 0, 0, 0, 0, 2073, 4096, 2, 3, 4091, 0, 0, 0, 1000, 0, 3013] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1, 6, 3167, 0, 3, 1, 4096, 2, 3869, 4096, 1689, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 1101, 4096, 7, 4, 4083, 1, 0, 0, 1152, 0, 1286] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3, 6, 2548, 0, 1, 0, 4096, 0, 3692, 4096, 1306, 0, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 1474, 4096, 3, 48, 4088, 0, 0, 0, 945, 0, 2256] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3504, 2, 1490, 142, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 2762, 0, 2070, 3, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 946, 2605, 1, 0, 0, 0, 0, 31, 4053, 1, 3, 0, 0, 0, 0, 1, 477, 0, 1200, 1166, 20, 4, 1] rerouted_tokens_per_ep_rank=[5139, 4099, 4835, 4091, 4095, 3552, 4088, 2869] avg=4096.0000 max=5139 max_rank=0 min=2869 min_rank=7 imbalance_ratio=1.254639 max_load_reduction=27629 max_load_reduction_pct=0.843170 moved_to_echo=14604 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1682, 0, 1490, 138, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 2688, 0, 1754, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1608, 2606, 7, 0, 0, 0, 0, 2, 4062, 0, 0, 0, 0, 0, 0, 0, 1007, 0, 2279, 1135, 20, 4, 1] rerouted_tokens_per_ep_rank=[3312, 4095, 4442, 4092, 4096, 4221, 4064, 4446] avg=4096.0000 max=4446 max_rank=7 min=3312 min_rank=0 imbalance_ratio=1.085449 max_load_reduction=28322 max_load_reduction_pct=0.864319 moved_to_echo=16827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 2863, 0, 1490, 137, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 2679, 0, 856, 0, 0, 0, 0, 0, 4090, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1464, 2606, 0, 0, 0, 0, 0, 71, 4075, 0, 0, 0, 0, 0, 0, 0, 396, 0, 2680, 1130, 20, 4, 1] rerouted_tokens_per_ep_rank=[4493, 4096, 3535, 4101, 4096, 4070, 4146, 4231] avg=4096.0000 max=4493 max_rank=0 min=3535 min_rank=2 imbalance_ratio=1.096924 max_load_reduction=28275 max_load_reduction_pct=0.862885 moved_to_echo=16543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3039, 0, 1490, 137, 0, 0, 1, 1, 4095, 1, 0, 0, 0, 0, 2666, 0, 1672, 0, 0, 0, 0, 0, 4091, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 930, 2606, 2, 0, 0, 0, 0, 8, 4064, 1, 0, 0, 0, 0, 0, 0, 799, 0, 1914, 1126, 20, 4, 1] rerouted_tokens_per_ep_rank=[4666, 4098, 4338, 4094, 4097, 3538, 4073, 3864] avg=4096.0000 max=4666 max_rank=0 min=3538 min_rank=5 imbalance_ratio=1.139160 max_load_reduction=28102 max_load_reduction_pct=0.857605 moved_to_echo=15572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1294, 0, 1490, 136, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 2655, 0, 1493, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2310, 2606, 0, 0, 0, 0, 0, 73, 4072, 0, 0, 0, 0, 0, 0, 0, 978, 0, 2232, 1121, 20, 4, 1] rerouted_tokens_per_ep_rank=[2920, 4095, 4148, 4092, 4096, 4916, 4145, 4356] avg=4096.0000 max=4916 max_rank=5 min=2920 min_rank=0 imbalance_ratio=1.200195 max_load_reduction=27852 max_load_reduction_pct=0.849976 moved_to_echo=17513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 1625, 0, 1490, 132, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 2574, 0, 778, 0, 0, 0, 0, 0, 4091, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2073, 2606, 2, 0, 0, 0, 0, 3, 4071, 0, 0, 0, 0, 0, 0, 0, 1000, 0, 3013, 1086, 20, 4, 1] rerouted_tokens_per_ep_rank=[3252, 4095, 3352, 4094, 4096, 4681, 4074, 5124] avg=4096.0000 max=5124 max_rank=7 min=3252 min_rank=0 imbalance_ratio=1.250977 max_load_reduction=27644 max_load_reduction_pct=0.843628 moved_to_echo=17975 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 6, 2548, 0, 1497, 133, 0, 0, 1, 0, 4088, 0, 0, 0, 0, 0, 2497, 0, 1306, 0, 0, 0, 0, 0, 4086, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1474, 2599, 3, 0, 0, 0, 0, 48, 4062, 0, 0, 0, 0, 0, 0, 0, 945, 0, 2256, 1062, 26, 10, 8] rerouted_tokens_per_ep_rank=[4187, 4089, 3803, 4100, 4096, 4076, 4110, 4307] avg=4096.0000 max=4307 max_rank=7 min=3803 min_rank=2 imbalance_ratio=1.051514 max_load_reduction=28461 max_load_reduction_pct=0.868561 moved_to_echo=16589 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 6, 3167, 0, 1490, 135, 0, 0, 3, 1, 4095, 2, 0, 0, 0, 0, 2626, 0, 1689, 0, 0, 0, 0, 0, 4092, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1101, 2606, 7, 0, 0, 0, 0, 4, 4063, 1, 0, 0, 0, 0, 0, 0, 1152, 0, 1286, 1108, 20, 4, 1] rerouted_tokens_per_ep_rank=[4799, 4101, 4315, 4104, 4096, 3714, 4068, 3571] avg=4096.0000 max=4799 max_rank=0 min=3571 min_rank=7 imbalance_ratio=1.171631 max_load_reduction=27969 max_load_reduction_pct=0.853546 moved_to_echo=15449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 19840, 128, 12032, 1152, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 6784, 0, 16896, 8960, 256, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 12032, 20864, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[21248, 0, 11648, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 32640, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4005, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 74, 0, 4094, 4089, 0, 2664, 4096, 54, 0, 346, 13, 0, 0, 1046, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4059, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 58, 0, 4094, 4096, 0, 2147, 4096, 220, 10, 452, 10, 0, 0, 1238, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 24, 0, 4096, 4096, 0, 2991, 4096, 41, 2, 182, 12, 0, 0, 848, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4014, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 31, 0, 4094, 4086, 0, 2003, 4096, 85, 4, 533, 69, 0, 0, 1469, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 0, 2, 2, 4020, 4, 0, 0, 4094, 2, 0, 0, 0, 1, 0, 4094, 21, 2, 4094, 4093, 0, 1079, 4092, 181, 11, 1005, 147, 0, 2, 1725, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 117, 0, 4094, 4096, 0, 2842, 4096, 68, 4, 391, 20, 0, 0, 665, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4009, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 27, 0, 4083, 4096, 0, 2159, 4096, 92, 5, 269, 4, 0, 0, 1640, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4062, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 26, 0, 4094, 4096, 0, 2089, 4096, 297, 0, 375, 14, 0, 0, 1331, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4005, 0, 0, 43, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 74, 0, 4051, 0, 0, 0, 0, 1848, 0, 2664, 0, 0, 0, 0, 0, 54, 0, 346, 13, 2241, 1237, 0, 0, 0, 0, 1046, 2851, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4055, 4096, 4096, 4125, 4512, 3891, 3897] avg=4096.0000 max=4512 max_rank=5 min=3891 min_rank=6 imbalance_ratio=1.101562 max_load_reduction=28256 max_load_reduction_pct=0.862305 moved_to_echo=16425 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 4059, 0, 0, 43, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 58, 0, 4051, 0, 0, 0, 0, 1851, 0, 2147, 0, 0, 0, 0, 0, 220, 10, 452, 10, 2245, 1237, 0, 0, 0, 0, 1238, 2852, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4109, 4096, 4097, 4109, 3998, 4174, 4090] avg=4096.0000 max=4174 max_rank=6 min=3998 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16371 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 2, 4092, 0, 0, 0, 2, 4020, 4, 0, 43, 7, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 21, 2, 4051, 0, 0, 0, 0, 1850, 0, 1079, 0, 0, 0, 0, 0, 181, 11, 1005, 147, 2243, 1237, 0, 0, 0, 2, 1725, 2851, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4076, 4096, 4095, 4074, 2929, 4824, 4578] avg=4096.0000 max=4824 max_rank=6 min=2929 min_rank=5 imbalance_ratio=1.177734 max_load_reduction=27944 max_load_reduction_pct=0.852783 moved_to_echo=16405 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 43, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 24, 0, 4053, 0, 0, 0, 0, 1851, 0, 2991, 0, 0, 0, 0, 0, 41, 2, 182, 12, 2245, 1237, 0, 0, 0, 0, 848, 2851, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4143, 4096, 4096, 4077, 4842, 3719, 3699] avg=4096.0000 max=4842 max_rank=5 min=3699 min_rank=7 imbalance_ratio=1.182129 max_load_reduction=27926 max_load_reduction_pct=0.852234 moved_to_echo=16337 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4014, 0, 0, 43, 7, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 31, 0, 4051, 0, 0, 0, 0, 1846, 0, 2003, 0, 0, 0, 0, 0, 85, 4, 533, 69, 2240, 1237, 0, 0, 0, 0, 1469, 2852, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4064, 4092, 4096, 4082, 3849, 4168, 4321] avg=4096.0000 max=4321 max_rank=7 min=3849 min_rank=5 imbalance_ratio=1.054932 max_load_reduction=28447 max_load_reduction_pct=0.868134 moved_to_echo=16420 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 43, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 117, 0, 4051, 0, 0, 0, 0, 1851, 0, 2842, 0, 0, 0, 0, 0, 68, 4, 391, 20, 2245, 1237, 0, 0, 0, 0, 665, 2852, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4137, 4096, 4096, 4168, 4693, 3965, 3517] avg=4096.0000 max=4693 max_rank=5 min=3517 min_rank=7 imbalance_ratio=1.145752 max_load_reduction=28075 max_load_reduction_pct=0.856781 moved_to_echo=16343 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 0, 0, 0, 4062, 0, 0, 50, 13, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 26, 0, 4040, 0, 0, 0, 0, 1846, 0, 2089, 0, 0, 0, 0, 0, 297, 0, 375, 14, 2249, 1240, 0, 0, 0, 0, 1331, 2843, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4125, 4100, 4096, 4066, 3935, 4175, 4174] avg=4096.0000 max=4175 max_rank=6 min=3935 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16350 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4009, 0, 0, 43, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 27, 0, 4040, 0, 0, 0, 0, 1851, 0, 2159, 0, 0, 0, 0, 0, 92, 5, 269, 4, 2245, 1237, 0, 0, 0, 0, 1640, 2852, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4059, 4096, 4096, 4067, 4010, 3852, 4492] avg=4096.0000 max=4492 max_rank=7 min=3852 min_rank=6 imbalance_ratio=1.096680 max_load_reduction=28276 max_load_reduction_pct=0.862915 moved_to_echo=16421 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1152, 128, 3584, 384, 18048, 9984, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[14848, 0, 18048, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 9984, 22912, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 384, 128, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32384, 128, 0, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4084, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4049, 0, 0, 0, 0, 0, 0, 59, 0, 0, 4092, 0, 4092, 0, 4, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 2, 0, 4092, 0, 0, 0, 4096, 3704, 0, 0, 0, 0, 0, 0, 392, 0, 5, 4094, 0, 4089, 0, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3898, 0, 0, 0, 0, 0, 0, 199, 0, 0, 4095, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 4062, 0, 0, 0, 0, 0, 0, 37, 0, 4, 4096, 0, 4085, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[2, 2, 2, 4094, 0, 4089, 4094, 2, 0, 1, 0, 4093, 2, 0, 0, 4094, 3971, 0, 1, 0, 1, 8, 0, 127, 2, 1, 4093, 0, 4088, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 42, 0, 9, 4092, 0, 4087, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 5, 0, 4095, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 37, 0, 2, 4096, 0, 4092, 0, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2, 4096, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 3431, 0, 0, 0, 0, 0, 0, 663, 0, 9, 4096, 0, 4083, 0, 0, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4049, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 59, 3900, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4092, 0, 4, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4087, 4097, 4096, 4242, 3959, 4091, 4100] avg=4096.0000 max=4242 max_rank=4 min=3959 min_rank=5 imbalance_ratio=1.035645 max_load_reduction=28526 max_load_reduction_pct=0.870544 moved_to_echo=16392 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3704, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 392, 3900, 0, 0, 0, 0, 5, 4093, 0, 0, 0, 0, 0, 4089, 0, 0, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4099, 4095, 4096, 3897, 4292, 4098, 4095] avg=4096.0000 max=4292 max_rank=5 min=3897 min_rank=4 imbalance_ratio=1.047852 max_load_reduction=28476 max_load_reduction_pct=0.869019 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 2, 4094, 0, 0, 0, 0, 0, 4089, 4, 2, 0, 0, 0, 0, 0, 1, 0, 4093, 1, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 3971, 0, 1, 0, 192, 0, 0, 0, 1, 8, 0, 127, 3898, 0, 0, 0, 2, 1, 4092, 0, 0, 0, 0, 0, 4088, 0, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4095, 4095, 4096, 4164, 4034, 4095, 4089] avg=4096.0000 max=4164 max_rank=4 min=4034 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3898, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 199, 3900, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4099, 4097, 4096, 4091, 4099, 4094, 4096] avg=4096.0000 max=4099 max_rank=1 min=4091 min_rank=4 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16380 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4062, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 37, 3900, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 4085, 0, 0, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4098, 4094, 4096, 4255, 3937, 4099, 4093] avg=4096.0000 max=4255 max_rank=4 min=3937 min_rank=5 imbalance_ratio=1.038818 max_load_reduction=28513 max_load_reduction_pct=0.870148 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4054, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 37, 3903, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 4092, 0, 0, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4101, 4096, 4247, 3940, 4097, 4097] avg=4096.0000 max=4247 max_rank=4 min=3940 min_rank=5 imbalance_ratio=1.036865 max_load_reduction=28521 max_load_reduction_pct=0.870392 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3431, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 663, 3900, 0, 0, 0, 0, 9, 4085, 0, 0, 0, 0, 0, 4083, 0, 0, 11, 7, 2, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4098, 4096, 4096, 3624, 4563, 4094, 4103] avg=4096.0000 max=4563 max_rank=5 min=3624 min_rank=4 imbalance_ratio=1.114014 max_load_reduction=28205 max_load_reduction_pct=0.860748 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4054, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 42, 3902, 0, 0, 0, 0, 9, 4091, 0, 0, 0, 0, 0, 4087, 0, 0, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4093, 4096, 4248, 3944, 4100, 4095] avg=4096.0000 max=4248 max_rank=4 min=3944 min_rank=5 imbalance_ratio=1.037109 max_load_reduction=28520 max_load_reduction_pct=0.870361 moved_to_echo=16387 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31232, 0, 128, 0, 1664, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 1664, 31232, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4092, 4096, 4019, 0, 0, 44, 0, 0, 0, 1, 6, 4096, 0, 4096, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 27, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4095, 4096, 3917, 0, 0, 115, 0, 0, 0, 0, 12, 4096, 0, 4092, 4, 0, 4096, 6, 0, 4096, 0, 0, 3, 44, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 4084, 4070, 3988, 0, 0, 91, 0, 0, 0, 24, 9, 4096, 0, 4093, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4088, 4091, 3970, 0, 0, 80, 0, 0, 0, 5, 7, 4095, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 11, 36, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4065, 4029, 3780, 0, 0, 290, 0, 0, 0, 49, 8, 4093, 0, 4096, 3, 0, 4096, 7, 0, 4096, 0, 0, 0, 60, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4077, 4087, 3977, 0, 0, 45, 0, 0, 0, 3, 14, 4096, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 87, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4095, 4086, 3964, 0, 0, 101, 0, 0, 0, 12, 26, 4096, 0, 4088, 1, 0, 4096, 1, 0, 4096, 0, 0, 2, 8, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 4081, 4093, 3959, 2, 2, 94, 2, 0, 2, 0, 75, 4046, 0, 4094, 8, 2, 4094, 0, 0, 4094, 2, 0, 0, 20, 0, 2, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 150, 3917, 0, 0, 0, 0, 0, 0, 115, 0, 0, 3946, 29, 24, 1, 0, 0, 12, 4072, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 44, 4066, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4067, 4115, 4084, 4096, 4101, 4113, 4096] avg=4096.0000 max=4115 max_rank=2 min=4067 min_rank=1 imbalance_ratio=1.004639 max_load_reduction=28653 max_load_reduction_pct=0.874420 moved_to_echo=16406 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 0, 149, 3988, 0, 0, 0, 0, 0, 0, 91, 0, 0, 3921, 29, 24, 1, 0, 24, 9, 4072, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4055, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4137, 4066, 4105, 4093, 4094, 4080, 4096] avg=4096.0000 max=4137 max_rank=1 min=4066 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16363 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 150, 4019, 0, 0, 0, 0, 0, 0, 44, 0, 0, 3946, 29, 24, 1, 0, 1, 6, 4072, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4063, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4169, 4044, 4079, 4096, 4098, 4090, 4096] avg=4096.0000 max=4169 max_rank=1 min=4044 min_rank=2 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16380 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 150, 3970, 0, 0, 0, 0, 0, 0, 80, 0, 0, 3941, 29, 24, 1, 0, 5, 7, 4071, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 11, 36, 4059, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4120, 4075, 4083, 4096, 4096, 4106, 4096] avg=4096.0000 max=4120 max_rank=1 min=4075 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16394 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 150, 3977, 0, 0, 0, 0, 0, 0, 45, 0, 0, 3937, 30, 24, 1, 0, 3, 14, 4072, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 4047, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4127, 4037, 4089, 4094, 4095, 4134, 4096] avg=4096.0000 max=4134 max_rank=6 min=4037 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16419 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 0, 0, 0, 0, 143, 3780, 0, 0, 0, 0, 0, 0, 290, 0, 0, 3886, 30, 31, 6, 0, 49, 8, 4062, 0, 0, 0, 0, 0, 4096, 3, 0, 1, 0, 0, 0, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 4035, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3923, 4243, 4119, 4100, 4095, 4096, 4095] avg=4096.0000 max=4243 max_rank=2 min=3923 min_rank=1 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16386 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 0, 150, 3959, 2, 0, 0, 0, 0, 2, 94, 2, 0, 3943, 30, 24, 1, 2, 0, 75, 4022, 0, 0, 0, 0, 0, 4094, 8, 2, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 20, 4051, 0, 0, 0, 0, 2, 0, 2, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4111, 4096, 4099, 4104, 4093, 4073, 4098] avg=4096.0000 max=4111 max_rank=1 min=4073 min_rank=6 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 150, 3964, 0, 0, 0, 0, 0, 0, 101, 0, 0, 3936, 30, 24, 1, 0, 12, 26, 4072, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 4065, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4114, 4092, 4110, 4089, 4096, 4076, 4095] avg=4096.0000 max=4114 max_rank=1 min=4076 min_rank=6 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16356 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 896, 128, 0, 31488, 256, 256, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 1280, 31616, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 384, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 256, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 6, 0, 4096, 3719, 0, 0, 0, 0, 4096, 367, 0, 4, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4040, 0, 0, 0, 0, 4066, 76, 0, 12, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 2, 0, 4096, 4059, 0, 1, 0, 0, 4089, 26, 0, 16, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4076, 0, 0, 0, 0, 4083, 20, 0, 13, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 4096, 4038, 0, 0, 0, 0, 4096, 54, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4037, 0, 0, 0, 0, 4079, 59, 0, 17, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4090, 0, 0, 0, 0, 4075, 12, 0, 15, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[2, 2, 0, 4093, 3620, 2, 0, 0, 42, 4094, 407, 2, 1, 2, 2, 4093, 4094, 10, 4094, 1, 2, 1, 4094, 14, 0, 1, 0, 0, 0, 4093, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 4095, 0, 0, 0, 0, 3719, 0, 0, 0, 122, 9, 2, 1, 0, 3974, 367, 0, 0, 0, 0, 0, 4, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 3853, 4341, 4091, 4096, 4094, 4096, 4096] avg=4096.0000 max=4341 max_rank=2 min=3853 min_rank=1 imbalance_ratio=1.059814 max_load_reduction=28427 max_load_reduction_pct=0.867523 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4040, 0, 0, 0, 121, 9, 2, 1, 0, 3945, 76, 0, 0, 0, 0, 0, 12, 0, 0, 4087, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4173, 4021, 4099, 4095, 4094, 4097, 4094] avg=4096.0000 max=4173 max_rank=1 min=4021 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16380 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4076, 0, 0, 0, 121, 9, 2, 1, 0, 3962, 20, 0, 0, 0, 0, 0, 13, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4209, 3982, 4100, 4096, 4094, 4096, 4096] avg=4096.0000 max=4209 max_rank=1 min=3982 min_rank=2 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4095, 0, 0, 0, 0, 4059, 0, 1, 0, 122, 9, 2, 1, 0, 3967, 26, 0, 0, 0, 0, 0, 16, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4194, 3993, 4103, 4096, 4094, 4096, 4095] avg=4096.0000 max=4194 max_rank=1 min=3993 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16381 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 0, 4092, 0, 0, 0, 0, 3620, 2, 0, 0, 122, 9, 2, 1, 42, 3972, 407, 2, 0, 0, 0, 0, 1, 2, 2, 4084, 0, 0, 0, 0, 4094, 10, 0, 1, 0, 0, 0, 0, 2, 1, 4092, 14, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 4093, 1, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3756, 4423, 4089, 4105, 4109, 4095, 4095] avg=4096.0000 max=4423 max_rank=2 min=3756 min_rank=1 imbalance_ratio=1.079834 max_load_reduction=28345 max_load_reduction_pct=0.865021 moved_to_echo=16404 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4095, 0, 0, 0, 0, 4038, 0, 0, 0, 122, 9, 2, 1, 0, 3974, 54, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4172, 4028, 4087, 4096, 4097, 4096, 4096] avg=4096.0000 max=4172 max_rank=1 min=4028 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 0, 0, 4090, 0, 0, 0, 124, 16, 6, 3, 0, 3951, 12, 0, 0, 0, 0, 0, 15, 0, 0, 4080, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4239, 3963, 4095, 4091, 4091, 4096, 4100] avg=4096.0000 max=4239 max_rank=1 min=3963 min_rank=2 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16378 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4037, 0, 0, 0, 121, 9, 2, 1, 0, 3958, 59, 0, 0, 0, 0, 0, 17, 0, 0, 4087, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4170, 4017, 4104, 4095, 4095, 4096, 4096] avg=4096.0000 max=4170 max_rank=1 min=4017 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16382 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32766, 32768, 32768, 32770, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 31744, 1024, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32770 actual_valid_recv=33024 observed_loss=-254 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32766 actual_valid_recv=33408 observed_loss=-642 tokens_per_expert=[31744, 128, 128, 0, 1024, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 2, 3906, 4091, 0, 256, 0, 2, 4094, 4094, 2, 4028, 2, 1, 0, 0, 4094, 0, 2, 4094, 4091, 0, 0, 2, 0, 3, 0, 1, 0, 0, 1, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3957, 4096, 0, 194, 0, 0, 4096, 4096, 0, 4041, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3939, 4095, 0, 189, 0, 0, 4096, 4096, 0, 4064, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3685, 4096, 0, 452, 0, 0, 4096, 4096, 0, 4054, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4006, 4096, 0, 96, 0, 0, 4096, 4096, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3966, 4094, 0, 141, 0, 0, 4096, 4096, 0, 4085, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3931, 4096, 0, 204, 0, 0, 4096, 4096, 0, 4057, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3912, 4096, 0, 331, 0, 0, 4096, 4096, 0, 3949, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3957, 185, 0, 0, 0, 0, 0, 194, 0, 0, 3863, 0, 0, 0, 0, 50, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4046, 48, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4057, 4091, 4096, 4096, 4096, 4096, 4094] avg=4096.0000 max=4142 max_rank=0 min=4057 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 3906, 184, 0, 0, 0, 0, 0, 256, 0, 2, 3859, 0, 0, 0, 0, 50, 2, 4028, 0, 0, 0, 0, 2, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 4091, 0, 0, 2, 0, 0, 0, 0, 0, 3, 0, 1, 4094, 0, 0, 0, 0, 0, 1, 2, 4044, 48, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4117, 4080, 4097, 4096, 4093, 4098, 4095] avg=4096.0000 max=4117 max_rank=1 min=4080 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4006, 185, 0, 0, 0, 0, 0, 96, 0, 0, 3863, 0, 0, 0, 0, 50, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4046, 48, 1, 0] rerouted_tokens_per_ep_rank=[4191, 3959, 4140, 4096, 4096, 4096, 4095, 4095] avg=4096.0000 max=4191 max_rank=0 min=3959 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3939, 185, 0, 0, 0, 0, 0, 189, 0, 0, 3862, 0, 0, 0, 0, 50, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4046, 48, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4051, 4114, 4096, 4096, 4096, 4096, 4095] avg=4096.0000 max=4124 max_rank=0 min=4051 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3966, 185, 0, 0, 0, 0, 0, 141, 0, 0, 3861, 0, 0, 0, 0, 50, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 2, 0, 4046, 48, 1, 0] rerouted_tokens_per_ep_rank=[4151, 4002, 4135, 4096, 4096, 4096, 4095, 4097] avg=4096.0000 max=4151 max_rank=0 min=4002 min_rank=1 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3685, 185, 0, 0, 0, 0, 0, 452, 0, 0, 3863, 0, 0, 0, 0, 50, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 1, 0, 4046, 48, 1, 0] rerouted_tokens_per_ep_rank=[3870, 4315, 4104, 4095, 4096, 4096, 4096, 4096] avg=4096.0000 max=4315 max_rank=1 min=3870 min_rank=0 imbalance_ratio=1.053467 max_load_reduction=28453 max_load_reduction_pct=0.868317 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3912, 170, 0, 0, 0, 0, 0, 331, 0, 0, 3869, 0, 0, 0, 0, 48, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4048, 54, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4200, 3997, 4096, 4096, 4099, 4096, 4102] avg=4096.0000 max=4200 max_rank=1 min=3997 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3931, 185, 0, 0, 0, 0, 0, 204, 0, 0, 3863, 0, 0, 0, 0, 50, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4046, 48, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4067, 4107, 4096, 4096, 4096, 4096, 4094] avg=4096.0000 max=4116 max_rank=0 min=4067 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 31360, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 1920, 0, 128, 30976, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 32384, 512, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 512, 128, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 2509, 2, 0, 4096, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 1061, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 4087] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 2190, 9, 0, 4096, 0, 0, 0, 4087, 4096, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 1171, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 754, 0, 4072] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 2708, 10, 0, 4096, 0, 0, 0, 4090, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 1481, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 44, 0, 3948] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 2447, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1858, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 3827] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 532, 2, 0, 4094, 0, 2, 2, 4092, 4092, 4093, 1, 1, 1, 0, 4096, 0, 0, 2, 3603, 4093, 2, 2, 0, 0, 22, 2, 2, 0, 73, 0, 3959] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 2268, 3, 0, 4096, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 2, 2067, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 92, 0, 3755] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 2668, 28, 0, 4096, 0, 0, 0, 4087, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 1182, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 260, 0, 4061] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 2334, 18, 0, 4096, 0, 0, 0, 4080, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 1861, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 53, 0, 3942] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2509, 2, 0, 1787, 96, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 1061, 2302, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 534, 0, 3990, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4394, 4096, 4103, 4097, 3370, 4096, 4088, 4524] avg=4096.0000 max=4524 max_rank=7 min=3370 min_rank=4 imbalance_ratio=1.104492 max_load_reduction=28244 max_load_reduction_pct=0.861938 moved_to_echo=16078 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2268, 3, 0, 1787, 88, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 2067, 2302, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4089, 0, 0, 0, 0, 92, 0, 3667, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4096, 4099, 4094, 4378, 4096, 4100, 3759] avg=4096.0000 max=4378 max_rank=4 min=3759 min_rank=7 imbalance_ratio=1.068848 max_load_reduction=28390 max_load_reduction_pct=0.866394 moved_to_echo=16333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 532, 2, 0, 1785, 93, 0, 0, 4094, 0, 2, 2, 0, 0, 0, 0, 4092, 7, 0, 1, 0, 0, 0, 0, 1, 1, 0, 4096, 1, 0, 0, 0, 0, 0, 2, 3603, 2300, 7, 0, 0, 4093, 2, 2, 0, 0, 0, 0, 0, 0, 22, 2, 2, 4086, 0, 0, 0, 0, 73, 0, 3865, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2412, 4098, 4100, 4099, 5912, 4097, 4112, 3938] avg=4096.0000 max=5912 max_rank=4 min=2412 min_rank=0 imbalance_ratio=1.443359 max_load_reduction=26856 max_load_reduction_pct=0.819580 moved_to_echo=18059 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2447, 0, 0, 1787, 90, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1858, 2302, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 60, 0, 3737, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4324, 4096, 4103, 4096, 4167, 4096, 4089, 3797] avg=4096.0000 max=4324 max_rank=0 min=3797 min_rank=7 imbalance_ratio=1.055664 max_load_reduction=28444 max_load_reduction_pct=0.868042 moved_to_echo=16149 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2708, 10, 0, 1786, 92, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 1481, 2302, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4089, 0, 0, 0, 0, 44, 0, 3856, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4596, 4096, 4097, 4095, 3791, 4096, 4097, 3900] avg=4096.0000 max=4596 max_rank=0 min=3791 min_rank=4 imbalance_ratio=1.122070 max_load_reduction=28172 max_load_reduction_pct=0.859741 moved_to_echo=15884 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2668, 28, 0, 1787, 95, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 1182, 2302, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4089, 0, 0, 0, 0, 260, 0, 3965, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4578, 4096, 4094, 4096, 3491, 4096, 4092, 4225] avg=4096.0000 max=4578 max_rank=0 min=3491 min_rank=4 imbalance_ratio=1.117676 max_load_reduction=28190 max_load_reduction_pct=0.860291 moved_to_echo=15904 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2334, 18, 0, 1789, 96, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 2, 1, 0, 0, 0, 0, 1861, 2307, 8, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4088, 0, 0, 0, 0, 53, 0, 3842, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4237, 4094, 4080, 4101, 4176, 4095, 4090, 3895] avg=4096.0000 max=4237 max_rank=0 min=3895 min_rank=7 imbalance_ratio=1.034424 max_load_reduction=28531 max_load_reduction_pct=0.870697 moved_to_echo=16256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2190, 9, 0, 1787, 95, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 1171, 2304, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4088, 0, 0, 0, 0, 754, 0, 3976, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4096, 4092, 4090, 3483, 4096, 4100, 4730] avg=4096.0000 max=4730 max_rank=7 min=3483 min_rank=4 imbalance_ratio=1.154785 max_load_reduction=28038 max_load_reduction_pct=0.855652 moved_to_echo=16409 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 14336, 18432, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 17664, 128, 0, 14336, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 1920, 0, 30976, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 250, 0, 0, 0, 0, 0, 0, 4096, 3063, 0, 0, 3560, 0, 4096, 0, 0, 0, 0, 4096, 1317, 0, 3, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 516, 0, 2, 0, 0, 0, 0, 4089, 3085, 0, 0, 3931, 0, 4096, 8, 0, 0, 0, 4093, 647, 0, 16, 4094, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 4096, 3443, 0, 0, 3824, 0, 4096, 0, 0, 0, 0, 4096, 835, 0, 0, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 3, 0, 4095, 0, 0, 1, 231, 46, 1, 2, 0, 2, 2, 4093, 2965, 0, 2, 3210, 0, 4084, 9, 1, 0, 0, 4092, 1736, 2, 2, 4093, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 4094, 3752, 0, 0, 3945, 0, 4096, 6, 0, 0, 0, 4095, 398, 0, 8, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 4087, 3943, 0, 0, 3256, 0, 4096, 2, 0, 0, 0, 4094, 902, 0, 21, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 130, 0, 1, 0, 0, 0, 0, 4089, 2973, 0, 0, 3536, 0, 4096, 8, 0, 0, 0, 4090, 1517, 0, 41, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 270, 0, 1, 0, 0, 0, 0, 4089, 3346, 0, 0, 3964, 0, 4096, 8, 0, 0, 0, 4096, 606, 0, 5, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 3, 0, 4094, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 231, 46, 1, 2, 2529, 992, 10, 0, 0, 2, 2, 4093, 3, 0, 0, 0, 2965, 0, 2, 681, 0, 0, 0, 0, 0, 4081, 9, 1, 0, 0, 0, 0, 0, 0, 3100, 1736, 0, 0, 0, 0, 2, 2, 4083, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4096, 3811, 4100, 3648, 4091, 4836, 4087] avg=4096.0000 max=4836 max_rank=6 min=3648 min_rank=4 imbalance_ratio=1.180664 max_load_reduction=27932 max_load_reduction_pct=0.852417 moved_to_echo=16662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 3108, 992, 10, 1, 0, 0, 0, 4094, 3, 0, 0, 0, 3752, 0, 0, 837, 0, 0, 0, 0, 0, 4093, 6, 0, 0, 0, 0, 0, 0, 0, 3103, 398, 0, 0, 0, 0, 0, 8, 4086, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4197, 4097, 4589, 4099, 3501, 4094] avg=4096.0000 max=4589 max_rank=4 min=3501 min_rank=6 imbalance_ratio=1.120361 max_load_reduction=28179 max_load_reduction_pct=0.859955 moved_to_echo=16283 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 3013, 993, 10, 1, 0, 0, 0, 4096, 3, 0, 0, 0, 3443, 0, 0, 811, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3103, 835, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4107, 4099, 4254, 4093, 3938, 4086] avg=4096.0000 max=4254 max_rank=4 min=3938 min_rank=6 imbalance_ratio=1.038574 max_load_reduction=28514 max_load_reduction_pct=0.870178 moved_to_echo=16371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 516, 0, 2, 0, 3097, 992, 10, 1, 0, 0, 0, 4089, 3, 0, 0, 0, 3085, 0, 0, 834, 0, 0, 0, 0, 0, 4093, 8, 0, 0, 0, 0, 0, 0, 0, 3101, 647, 0, 0, 0, 0, 0, 16, 4084, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4618, 4092, 3919, 4101, 3748, 4100] avg=4096.0000 max=4618 max_rank=2 min=3748 min_rank=6 imbalance_ratio=1.127441 max_load_reduction=28150 max_load_reduction_pct=0.859070 moved_to_echo=15868 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 2565, 992, 10, 0, 0, 0, 0, 4087, 3, 0, 0, 0, 3943, 0, 0, 691, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 3102, 902, 0, 0, 0, 0, 0, 21, 4086, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 3647, 4090, 4634, 4095, 4004, 4107] avg=4096.0000 max=4634 max_rank=4 min=3647 min_rank=2 imbalance_ratio=1.131348 max_load_reduction=28134 max_load_reduction_pct=0.858582 moved_to_echo=16840 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 130, 0, 1, 0, 2786, 991, 10, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 2973, 0, 0, 750, 0, 0, 0, 0, 0, 4093, 8, 0, 0, 0, 0, 0, 0, 0, 3099, 1517, 0, 0, 0, 0, 0, 41, 4086, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 3918, 4092, 3723, 4101, 4616, 4127] avg=4096.0000 max=4616 max_rank=6 min=3723 min_rank=4 imbalance_ratio=1.126953 max_load_reduction=28152 max_load_reduction_pct=0.859131 moved_to_echo=16567 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 250, 0, 0, 0, 2805, 993, 10, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 3063, 0, 0, 755, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3103, 1317, 0, 0, 0, 0, 0, 3, 4086, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4058, 4099, 3818, 4093, 4420, 4089] avg=4096.0000 max=4420 max_rank=6 min=3818 min_rank=4 imbalance_ratio=1.079102 max_load_reduction=28348 max_load_reduction_pct=0.865112 moved_to_echo=16420 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 5, 0, 0, 0, 270, 0, 1, 0, 3127, 997, 17, 0, 0, 0, 0, 4089, 4, 6, 0, 0, 3346, 0, 0, 837, 0, 0, 0, 0, 0, 4087, 8, 0, 0, 0, 0, 0, 0, 0, 3099, 606, 0, 0, 0, 0, 0, 5, 4073, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4100, 4412, 4099, 4183, 4095, 3705, 4078] avg=4096.0000 max=4412 max_rank=2 min=3705 min_rank=6 imbalance_ratio=1.077148 max_load_reduction=28356 max_load_reduction_pct=0.865356 moved_to_echo=16061 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1664, 128, 128, 128, 23040, 8064, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[26624, 0, 128, 6272, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 24832, 8064, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 4096, 3853, 2547, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 50, 0, 4096, 3610, 3157, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4095, 2116, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 28, 0, 4096, 3380, 2670, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 75, 0, 4096, 3535, 3327, 0, 0, 0, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 5, 0, 4096, 3564, 3277, 0, 0, 0, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 4085, 2266, 2, 2, 0, 5, 0, 1, 0, 2, 1, 4093, 0, 5, 4094, 1, 1, 4094, 0, 1, 324, 0, 4094, 1879, 3719, 2, 2, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 21, 0, 4096, 3944, 1961, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 100, 0, 4096, 3183, 3323, 0, 0, 0, 0, 4089] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2346, 1789, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 2596, 1749, 1, 0, 0, 1257, 2547, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4095, 4096, 4096, 4096, 4349, 3804, 4097] avg=4096.0000 max=4349 max_rank=5 min=3804 min_rank=6 imbalance_ratio=1.061768 max_load_reduction=28419 max_load_reduction_pct=0.867279 moved_to_echo=16346 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2345, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 75, 0, 2382, 1749, 1, 0, 0, 1153, 3327, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3604, 4095, 4096, 4097, 4096, 4207, 4480, 4093] avg=4096.0000 max=4480 max_rank=6 min=3604 min_rank=0 imbalance_ratio=1.093750 max_load_reduction=28288 max_load_reduction_pct=0.863281 moved_to_echo=16876 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2346, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 50, 0, 2433, 1749, 1, 0, 0, 1177, 3157, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3721, 4095, 4096, 4096, 4096, 4233, 4334, 4097] avg=4096.0000 max=4334 max_rank=6 min=3721 min_rank=0 imbalance_ratio=1.058105 max_load_reduction=28434 max_load_reduction_pct=0.867737 moved_to_echo=16760 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 2340, 2266, 0, 0, 0, 0, 2, 2, 0, 5, 4093, 0, 0, 0, 0, 1, 0, 2, 4094, 0, 0, 0, 1, 4093, 0, 5, 0, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 1, 324, 0, 1266, 1744, 1, 0, 0, 613, 3719, 2, 0, 0, 0, 0, 2, 0, 0, 4093, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4608, 4102, 4097, 4099, 4096, 3336, 4334, 4096] avg=4096.0000 max=4608 max_rank=0 min=3336 min_rank=5 imbalance_ratio=1.125000 max_load_reduction=28160 max_load_reduction_pct=0.859375 moved_to_echo=15862 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2345, 2116, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 28, 0, 2278, 1749, 1, 0, 0, 1102, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4461, 4095, 4096, 4097, 4096, 4056, 3772, 4095] avg=4096.0000 max=4461 max_rank=0 min=3772 min_rank=6 imbalance_ratio=1.089111 max_load_reduction=28307 max_load_reduction_pct=0.863861 moved_to_echo=16019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2346, 2268, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 21, 0, 2658, 1749, 1, 0, 0, 1286, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4614, 4095, 4096, 4096, 4096, 4429, 3247, 4095] avg=4096.0000 max=4614 max_rank=0 min=3247 min_rank=6 imbalance_ratio=1.126465 max_load_reduction=28154 max_load_reduction_pct=0.859192 moved_to_echo=15867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2335, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 100, 0, 2148, 1752, 1, 0, 0, 1035, 3323, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 8, 5, 0, 0] rerouted_tokens_per_ep_rank=[3929, 4096, 4095, 4091, 4096, 4001, 4358, 4102] avg=4096.0000 max=4358 max_rank=6 min=3929 min_rank=0 imbalance_ratio=1.063965 max_load_reduction=28410 max_load_reduction_pct=0.867004 moved_to_echo=16557 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2346, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 2402, 1749, 1, 0, 0, 1162, 3277, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3696, 4095, 4096, 4096, 4096, 4157, 4439, 4093] avg=4096.0000 max=4439 max_rank=6 min=3696 min_rank=0 imbalance_ratio=1.083740 max_load_reduction=28329 max_load_reduction_pct=0.864532 moved_to_echo=16785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 640, 0, 18176, 14080, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 18816, 14080, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 8832, 24064, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[210, 4096, 0, 0, 0, 4096, 4096, 1085, 0, 0, 4095, 0, 4096, 0, 2990, 4035, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[532, 4096, 0, 0, 0, 4096, 4096, 1007, 0, 0, 4096, 0, 4096, 0, 3328, 3474, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[84, 4096, 0, 0, 0, 4096, 4096, 1130, 0, 0, 4096, 0, 4096, 0, 3389, 3695, 0, 0, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[93, 4096, 0, 0, 0, 4096, 4096, 2019, 0, 0, 4096, 0, 4096, 0, 2625, 3486, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1434, 4089, 0, 0, 0, 4089, 4094, 434, 0, 2, 4095, 0, 4094, 2, 2840, 3645, 6, 2, 1, 3905, 1, 2, 0, 2, 0, 1, 0, 24, 0, 1, 0, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[166, 4096, 0, 0, 0, 4096, 4096, 1031, 0, 0, 4096, 0, 4096, 0, 3713, 3445, 0, 0, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[905, 4096, 0, 0, 0, 4096, 4096, 1238, 0, 0, 4096, 0, 4096, 0, 3200, 3182, 0, 0, 0, 3763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[582, 4096, 0, 0, 0, 4096, 4096, 204, 0, 0, 4096, 0, 4096, 0, 3686, 3893, 0, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[210, 3597, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 1085, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2990, 979, 0, 0, 0, 0, 0, 0, 0, 3969, 158, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3056, 1017, 341, 1, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[3807, 4164, 4095, 3969, 4127, 4096, 4415, 4095] avg=4096.0000 max=4415 max_rank=6 min=3807 min_rank=0 imbalance_ratio=1.077881 max_load_reduction=28353 max_load_reduction_pct=0.865265 moved_to_echo=16733 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[84, 3597, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 1130, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3389, 896, 0, 0, 0, 0, 0, 0, 0, 3990, 158, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2799, 1017, 341, 1, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[3681, 4209, 4096, 4285, 4148, 4095, 4158, 4096] avg=4096.0000 max=4285 max_rank=3 min=3681 min_rank=0 imbalance_ratio=1.046143 max_load_reduction=28483 max_load_reduction_pct=0.869232 moved_to_echo=16497 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1434, 3591, 0, 0, 0, 0, 0, 0, 0, 3073, 0, 434, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 2, 2840, 884, 0, 0, 0, 0, 6, 2, 1, 3905, 157, 0, 0, 0, 1, 2, 0, 2, 4094, 0, 0, 0, 0, 1, 0, 24, 2761, 1016, 341, 1, 0, 1, 0, 5, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[5025, 3507, 4097, 3726, 4071, 4099, 4144, 4099] avg=4096.0000 max=5025 max_rank=0 min=3507 min_rank=1 imbalance_ratio=1.226807 max_load_reduction=27743 max_load_reduction_pct=0.846649 moved_to_echo=16413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[532, 3597, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 1007, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3328, 843, 0, 0, 0, 0, 0, 0, 0, 3947, 158, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2631, 1017, 341, 1, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4086, 4096, 4171, 4105, 4095, 3990, 4096] avg=4096.0000 max=4171 max_rank=3 min=3990 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16286 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[166, 3597, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 1031, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3713, 836, 0, 0, 0, 0, 0, 0, 0, 3933, 158, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2609, 1017, 341, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[3763, 4110, 4096, 4549, 4091, 4095, 3967, 4097] avg=4096.0000 max=4549 max_rank=3 min=3763 min_rank=0 imbalance_ratio=1.110596 max_load_reduction=28219 max_load_reduction_pct=0.861176 moved_to_echo=16250 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[93, 3597, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 2019, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2625, 846, 0, 0, 0, 0, 0, 0, 0, 4065, 158, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2640, 1017, 341, 1, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[3690, 5098, 4096, 3471, 4223, 4096, 3999, 4095] avg=4096.0000 max=5098 max_rank=1 min=3471 min_rank=3 imbalance_ratio=1.244629 max_load_reduction=27670 max_load_reduction_pct=0.844421 moved_to_echo=16413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[905, 3589, 0, 0, 0, 0, 0, 0, 0, 3073, 0, 1238, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3200, 767, 0, 0, 0, 0, 0, 0, 0, 3763, 159, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2415, 1023, 348, 1, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4494, 4311, 4096, 3967, 3922, 4096, 3787, 4095] avg=4096.0000 max=4494 max_rank=0 min=3787 min_rank=6 imbalance_ratio=1.097168 max_load_reduction=28274 max_load_reduction_pct=0.862854 moved_to_echo=15900 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[582, 3597, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 204, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3686, 944, 0, 0, 0, 0, 0, 0, 0, 3923, 158, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2949, 1017, 341, 1, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 3283, 4096, 4630, 4081, 4096, 4308, 4095] avg=4096.0000 max=4630 max_rank=3 min=3283 min_rank=1 imbalance_ratio=1.130371 max_load_reduction=28138 max_load_reduction_pct=0.858704 moved_to_echo=16580 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 24704, 0, 8192, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 25856, 7040, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[4096, 28800, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 31616, 1280, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 21888, 8192, 2816, 128] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[20, 0, 1, 2, 0, 2, 0, 4093, 1, 10, 4094, 0, 0, 4075, 3, 3, 4093, 0, 0, 0, 4088, 0, 1, 3939, 4094, 7, 2, 157, 0, 4083, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 4093, 0, 2, 4096, 0, 0, 0, 4096, 0, 3, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 4096, 0, 6, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 4096, 0, 6, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4096, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4079, 14, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 4096, 4096, 4096, 4117, 4077, 4097] avg=4096.0000 max=4117 max_rank=5 min=4077 min_rank=6 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4079, 14, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 4096, 4100, 4096, 4113, 4077, 4097] avg=4096.0000 max=4113 max_rank=5 min=4077 min_rank=6 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 1, 2, 4071, 14, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 1, 10, 4094, 0, 0, 0, 0, 0, 0, 4075, 3, 3, 4, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1, 3939, 0, 0, 0, 0, 4075, 7, 2, 157, 0, 0, 0, 0, 0, 4083, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4095, 4105, 4085, 4093, 3957, 4241, 4084] avg=4096.0000 max=4241 max_rank=6 min=3957 min_rank=5 imbalance_ratio=1.035400 max_load_reduction=28527 max_load_reduction_pct=0.870575 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4079, 14, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 0, 3, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4094, 4096, 4099, 4096, 4116, 4077, 4097] avg=4096.0000 max=4116 max_rank=5 min=4077 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4079, 14, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 4096, 4100, 4096, 4113, 4077, 4097] avg=4096.0000 max=4113 max_rank=5 min=4077 min_rank=6 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4082, 20, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 14, 0, 6, 4096, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 3, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4099, 4087, 4098, 4099, 4116, 4065, 4102] avg=4096.0000 max=4116 max_rank=5 min=4065 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4079, 14, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 4096, 4096, 4096, 4117, 4077, 4097] avg=4096.0000 max=4117 max_rank=5 min=4077 min_rank=6 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4079, 14, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 0, 6, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 4096, 4094, 4096, 4119, 4077, 4097] avg=4096.0000 max=4119 max_rank=5 min=4077 min_rank=6 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 0, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 683, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 5, 0, 0, 3408, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 5, 0, 0, 0, 0, 0, 4094, 1053, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 7, 0, 0, 3034, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 771, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 0, 0, 3323, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 0, 0, 0, 4093, 962, 7, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 19, 0, 0, 3111, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4093, 1, 1, 1, 1, 1, 3, 1, 4093, 2014, 1, 0, 2, 2, 1, 1, 4094, 4089, 4092, 0, 0, 0, 0, 0, 2015, 2, 1, 4094, 2, 81, 76, 4006] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 3, 0, 0, 0, 0, 0, 4092, 517, 4, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 23, 0, 0, 3555, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4091, 0, 4, 0, 0, 0, 0, 0, 4091, 636, 7, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 26, 0, 0, 3433, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4091, 497, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 14, 0, 0, 3590, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3205, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3183, 891, 8, 0, 3408, 0, 0, 913, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 3888, 4096, 4097, 4087, 4321, 4088] avg=4096.0000 max=4321 max_rank=6 min=3888 min_rank=2 imbalance_ratio=1.054932 max_load_reduction=28447 max_load_reduction_pct=0.868134 moved_to_echo=16593 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3202, 962, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 19, 0, 0, 3183, 891, 8, 0, 3111, 0, 0, 913, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4171, 4096, 4097, 4101, 4024, 4088] avg=4096.0000 max=4171 max_rank=2 min=4024 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16310 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3203, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3183, 891, 8, 0, 3034, 0, 0, 913, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4096, 4256, 4095, 4097, 4089, 3947, 4088] avg=4096.0000 max=4256 max_rank=2 min=3947 min_rank=6 imbalance_ratio=1.039062 max_load_reduction=28512 max_load_reduction_pct=0.870117 moved_to_echo=16221 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4093, 1, 1, 1, 0, 0, 0, 0, 1, 1, 3, 1, 4094, 0, 0, 0, 3202, 2014, 1, 0, 0, 0, 0, 0, 2, 2, 1, 1, 4091, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3182, 891, 8, 0, 2015, 2, 1, 912, 0, 0, 0, 0, 2, 81, 76, 3998, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4100, 5217, 4097, 4090, 4081, 2930, 4157] avg=4096.0000 max=5217 max_rank=2 min=2930 min_rank=6 imbalance_ratio=1.273682 max_load_reduction=27551 max_load_reduction_pct=0.840790 moved_to_echo=15258 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3205, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3183, 891, 8, 0, 3323, 0, 0, 913, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 3976, 4096, 4097, 4084, 4236, 4088] avg=4096.0000 max=4236 max_rank=6 min=3976 min_rank=2 imbalance_ratio=1.034180 max_load_reduction=28532 max_load_reduction_pct=0.870728 moved_to_echo=16505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3202, 517, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 23, 0, 0, 3183, 890, 8, 0, 3555, 0, 0, 913, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4095, 3723, 4096, 4097, 4104, 4468, 4088] avg=4096.0000 max=4468 max_rank=6 min=3723 min_rank=2 imbalance_ratio=1.090820 max_load_reduction=28300 max_load_reduction_pct=0.863647 moved_to_echo=16757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3196, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 3189, 895, 13, 4, 3590, 0, 0, 906, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4096, 3693, 4097, 4096, 4115, 4496, 4083] avg=4096.0000 max=4496 max_rank=6 min=3693 min_rank=2 imbalance_ratio=1.097656 max_load_reduction=28272 max_load_reduction_pct=0.862793 moved_to_echo=16790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3201, 636, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3183, 890, 8, 0, 3433, 0, 0, 913, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 3844, 4095, 4097, 4107, 4346, 4088] avg=4096.0000 max=4346 max_rank=6 min=3844 min_rank=2 imbalance_ratio=1.061035 max_load_reduction=28422 max_load_reduction_pct=0.867371 moved_to_echo=16638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 25472, 7168, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[25472, 128, 128, 7296, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[25728, 7168, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4, 660, 0, 3, 18, 2, 1, 4096, 1, 0, 1, 2, 4090, 2, 0, 0, 4088, 0, 86, 4092, 1, 0, 1, 4093, 3944, 1, 0, 4, 0, 3485, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 4092, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 4094, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 4090, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 4086, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 9, 7, 1, 0, 0, 4092, 0, 80, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4095, 4096, 4096, 4087, 4096, 4108, 4172] avg=4096.0000 max=4172 max_rank=7 min=4018 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4, 660, 0, 3, 4006, 0, 0, 0, 18, 2, 1, 4095, 0, 0, 0, 0, 1, 0, 1, 2, 4092, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 4079, 0, 86, 0, 0, 0, 0, 0, 1, 0, 1, 4093, 0, 0, 0, 0, 3944, 1, 0, 4, 9, 7, 1, 0, 0, 3485, 0, 80, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4673, 4116, 4096, 4092, 4165, 4095, 3966, 3565] avg=4096.0000 max=4673 max_rank=0 min=3565 min_rank=7 imbalance_ratio=1.140869 max_load_reduction=28095 max_load_reduction_pct=0.857391 moved_to_echo=15791 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 9, 7, 1, 0, 0, 4096, 0, 80, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4009, 4095, 4095, 4096, 4087, 4096, 4114, 4176] avg=4096.0000 max=4176 max_rank=7 min=4009 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16473 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 9, 7, 1, 0, 0, 4090, 0, 80, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4095, 4096, 4096, 4087, 4096, 4111, 4170] avg=4096.0000 max=4170 max_rank=7 min=4017 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4009, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 1, 9, 7, 1, 0, 0, 4096, 0, 80, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4096, 4095, 4095, 4087, 4096, 4112, 4176] avg=4096.0000 max=4176 max_rank=7 min=4011 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16471 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 9, 7, 1, 0, 0, 4086, 0, 80, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4095, 4096, 4095, 4087, 4096, 4114, 4166] avg=4096.0000 max=4166 max_rank=7 min=4019 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 15, 11, 8, 0, 0, 4096, 0, 71, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 4081, 4098, 4102, 4081, 4097, 4130, 4167] avg=4096.0000 max=4167 max_rank=7 min=4012 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16475 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 7, 1, 0, 0, 4096, 0, 80, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4009, 4095, 4096, 4096, 4087, 4096, 4113, 4176] avg=4096.0000 max=4176 max_rank=7 min=4009 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16472 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 768, 0, 128, 32128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32640, 128, 0, 128, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32256, 0, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 353, 3937, 0, 0, 2, 4096, 0, 0, 0, 4090, 0, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4090, 4090, 141, 4088, 0, 0, 14, 4096, 0, 0, 1, 4095, 0, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4090, 4091, 330, 3846, 0, 0, 6, 4096, 0, 0, 0, 4096, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4094, 4093, 175, 4031, 0, 0, 40, 4096, 0, 0, 0, 4096, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4088, 4094, 78, 4083, 0, 0, 3, 4096, 0, 0, 0, 4096, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4093, 4091, 303, 3621, 1, 2, 397, 4094, 1, 2, 3, 4094, 1, 3863, 0, 2, 1, 1, 0, 2, 2, 0, 4095, 0, 1, 0, 4094, 0, 2, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 246, 3996, 0, 0, 1, 4096, 0, 0, 0, 4071, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4088, 4096, 168, 4038, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 353, 3855, 0, 0, 0, 0, 0, 0, 2, 4038, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 0, 0, 3906, 0, 0, 77, 55, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0] rerouted_tokens_per_ep_rank=[4208, 4040, 4093, 4038, 4099, 4096, 4096, 4098] avg=4096.0000 max=4208 max_rank=0 min=4038 min_rank=3 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 78, 3998, 0, 0, 0, 0, 0, 0, 3, 4038, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4038, 0, 0, 80, 55, 0, 0, 0, 0, 0, 0, 4085, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4041, 4099, 4173, 4090, 4096, 4096, 4097] avg=4096.0000 max=4173 max_rank=3 min=4041 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 246, 3913, 0, 0, 0, 0, 0, 0, 1, 4038, 0, 0, 0, 0, 0, 0, 0, 4071, 3, 0, 0, 0, 0, 3974, 0, 0, 78, 55, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4039, 4074, 4107, 4099, 4096, 4096, 4098] avg=4096.0000 max=4159 max_rank=0 min=4039 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 303, 3546, 0, 0, 0, 0, 1, 2, 397, 4036, 0, 0, 0, 0, 1, 2, 3, 4094, 3, 0, 0, 0, 1, 3863, 0, 2, 70, 55, 0, 0, 1, 1, 0, 2, 4091, 5, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 2, 0, 4091, 2, 0, 0] rerouted_tokens_per_ep_rank=[3849, 4436, 4103, 3991, 4100, 4097, 4095, 4097] avg=4096.0000 max=4436 max_rank=1 min=3849 min_rank=0 imbalance_ratio=1.083008 max_load_reduction=28332 max_load_reduction_pct=0.864624 moved_to_echo=16389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 175, 3948, 0, 0, 0, 0, 0, 0, 40, 4038, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 3951, 0, 0, 78, 55, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4078, 4099, 4084, 4097, 4096, 4096, 4095] avg=4096.0000 max=4123 max_rank=0 min=4078 min_rank=1 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 141, 4003, 0, 0, 0, 0, 0, 0, 14, 4038, 0, 0, 0, 0, 0, 0, 1, 4095, 3, 0, 0, 0, 0, 3961, 0, 0, 80, 55, 0, 0, 0, 0, 0, 0, 4088, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4052, 4099, 4096, 4093, 4096, 4096, 4092] avg=4096.0000 max=4144 max_rank=0 min=4052 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 168, 3945, 0, 0, 0, 0, 0, 0, 4, 4034, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 3990, 0, 0, 82, 56, 0, 0, 0, 0, 0, 0, 4086, 11, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4038, 4102, 4128, 4097, 4095, 4097, 4098] avg=4096.0000 max=4128 max_rank=3 min=4038 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 330, 3766, 0, 0, 0, 0, 0, 0, 6, 4038, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4021, 0, 0, 75, 55, 0, 0, 0, 0, 0, 0, 4088, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4044, 4099, 4151, 4093, 4096, 4096, 4093] avg=4096.0000 max=4151 max_rank=3 min=4044 min_rank=1 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 1920, 30976, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31744, 0, 128, 640, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 512, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[677, 0, 1387, 60, 1713, 5, 2084, 122, 1311, 1501, 0, 33, 1179, 3018, 26, 856, 32, 1336, 1872, 0, 0, 1875, 1485, 253, 0, 3402, 3390, 1325, 2021, 1725, 80, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[498, 0, 1979, 70, 737, 0, 2264, 17, 1484, 1320, 1, 36, 2070, 3264, 9, 761, 7, 1683, 1710, 0, 0, 913, 1593, 269, 1, 2902, 3199, 1750, 1952, 2152, 127, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[834, 2, 1929, 81, 1154, 1, 2360, 62, 1530, 2085, 0, 59, 1338, 2876, 6, 1040, 57, 1309, 1542, 0, 2, 977, 1233, 132, 0, 2998, 3221, 1665, 2338, 1864, 73, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[686, 0, 2075, 59, 1481, 2, 1708, 26, 1453, 1014, 5, 38, 1716, 3056, 21, 893, 20, 1313, 2609, 2, 0, 1293, 1333, 325, 2, 2911, 3408, 1277, 2315, 1572, 151, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[958, 0, 2069, 52, 966, 0, 2309, 90, 1381, 1585, 0, 22, 1730, 2184, 30, 558, 10, 1570, 1774, 0, 0, 710, 1213, 167, 0, 3330, 3478, 1670, 3058, 1682, 172, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[433, 0, 1965, 96, 1443, 2, 2057, 171, 1483, 944, 0, 39, 1610, 2737, 38, 774, 41, 1607, 2025, 0, 0, 1209, 1966, 201, 0, 2790, 2878, 1338, 2565, 2201, 155, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[523, 0, 1713, 101, 856, 0, 1905, 19, 1127, 2133, 0, 27, 1148, 3519, 12, 1119, 40, 2223, 1955, 0, 0, 1120, 880, 171, 0, 2787, 2667, 1349, 2746, 2544, 84, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[577, 1, 1525, 102, 993, 0, 2645, 23, 1109, 2161, 1, 38, 1201, 3268, 36, 821, 18, 1468, 1782, 0, 0, 968, 1170, 189, 0, 2841, 3336, 1568, 2856, 1967, 104, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[677, 0, 1387, 60, 1632, 0, 0, 0, 1713, 5, 2084, 122, 433, 215, 0, 0, 1311, 1501, 0, 33, 846, 285, 0, 0, 1179, 1739, 26, 856, 0, 0, 0, 0, 32, 1336, 1872, 0, 445, 165, 0, 0, 0, 1875, 1485, 253, 1473, 0, 0, 0, 0, 2957, 0, 1325, 0, 0, 0, 0, 1641, 1725, 80, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3756, 4572, 3976, 3800, 3850, 5086, 4282, 3446] avg=4096.0000 max=5086 max_rank=5 min=3446 min_rank=7 imbalance_ratio=1.241699 max_load_reduction=27682 max_load_reduction_pct=0.844788 moved_to_echo=16664 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[834, 2, 1929, 81, 1550, 0, 0, 0, 1154, 1, 2360, 62, 413, 249, 0, 0, 1530, 2085, 0, 59, 806, 269, 0, 0, 1338, 1657, 6, 1040, 0, 0, 0, 0, 57, 1309, 1542, 0, 392, 191, 0, 0, 2, 977, 1233, 132, 1402, 0, 0, 0, 0, 2606, 0, 1665, 0, 0, 0, 0, 1898, 1864, 73, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4396, 4239, 4749, 4041, 3491, 3746, 4271, 3835] avg=4096.0000 max=4749 max_rank=2 min=3491 min_rank=4 imbalance_ratio=1.159424 max_load_reduction=28019 max_load_reduction_pct=0.855072 moved_to_echo=15343 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[498, 0, 1979, 70, 1540, 0, 0, 0, 737, 0, 2264, 17, 469, 207, 0, 0, 1484, 1320, 1, 36, 915, 267, 0, 0, 2070, 1880, 9, 761, 0, 0, 0, 0, 7, 1683, 1710, 0, 380, 159, 0, 0, 0, 913, 1593, 269, 1392, 0, 0, 0, 1, 2522, 0, 1750, 0, 0, 0, 0, 1586, 2152, 127, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 3694, 4023, 4720, 3939, 4167, 4273, 3865] avg=4096.0000 max=4720 max_rank=3 min=3694 min_rank=1 imbalance_ratio=1.152344 max_load_reduction=28048 max_load_reduction_pct=0.855957 moved_to_echo=16244 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[958, 0, 2069, 52, 1675, 0, 0, 0, 966, 0, 2309, 90, 313, 325, 0, 0, 1381, 1585, 0, 22, 612, 291, 0, 0, 1730, 1259, 30, 558, 0, 0, 0, 0, 10, 1570, 1774, 0, 436, 250, 0, 0, 0, 710, 1213, 167, 1512, 0, 0, 0, 0, 2894, 0, 1670, 0, 0, 0, 0, 2483, 1682, 172, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4754, 4003, 3891, 3577, 4040, 3602, 4564, 4337] avg=4096.0000 max=4754 max_rank=0 min=3577 min_rank=3 imbalance_ratio=1.160645 max_load_reduction=28014 max_load_reduction_pct=0.854919 moved_to_echo=16543 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[686, 0, 2075, 59, 1642, 0, 0, 0, 1481, 2, 1708, 26, 443, 250, 0, 0, 1453, 1014, 5, 38, 860, 285, 0, 0, 1716, 1753, 21, 893, 0, 0, 0, 0, 20, 1313, 2609, 2, 384, 194, 0, 0, 0, 1293, 1333, 325, 1481, 0, 0, 0, 2, 2527, 0, 1277, 0, 0, 0, 0, 1871, 1572, 151, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4462, 3910, 3655, 4383, 4522, 4432, 3806, 3598] avg=4096.0000 max=4522 max_rank=4 min=3598 min_rank=7 imbalance_ratio=1.104004 max_load_reduction=28246 max_load_reduction_pct=0.862000 moved_to_echo=16358 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[433, 0, 1965, 96, 1386, 0, 0, 0, 1443, 2, 2057, 171, 393, 273, 0, 0, 1483, 944, 0, 39, 767, 241, 0, 0, 1610, 1577, 38, 774, 0, 0, 0, 0, 41, 1607, 2025, 0, 365, 209, 0, 0, 0, 1209, 1966, 201, 1251, 0, 0, 0, 0, 2425, 0, 1338, 0, 0, 0, 0, 2083, 2201, 155, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3880, 4339, 3474, 3999, 4247, 4627, 3763, 4439] avg=4096.0000 max=4627 max_rank=5 min=3474 min_rank=2 imbalance_ratio=1.129639 max_load_reduction=28141 max_load_reduction_pct=0.858795 moved_to_echo=17076 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[523, 0, 1713, 101, 1284, 0, 0, 0, 856, 0, 1905, 19, 505, 292, 0, 0, 1127, 2133, 0, 27, 986, 223, 0, 0, 1148, 2028, 12, 1119, 0, 0, 0, 0, 40, 2223, 1955, 0, 364, 224, 0, 0, 0, 1120, 880, 171, 1160, 0, 0, 0, 0, 2423, 0, 1349, 0, 0, 0, 0, 2230, 2544, 84, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3621, 3577, 4496, 4307, 4806, 3331, 3772, 4858] avg=4096.0000 max=4858 max_rank=7 min=3331 min_rank=5 imbalance_ratio=1.186035 max_load_reduction=27910 max_load_reduction_pct=0.851746 moved_to_echo=16767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[577, 1, 1525, 102, 1607, 0, 0, 0, 993, 0, 2645, 23, 469, 304, 0, 0, 1109, 2161, 1, 38, 916, 279, 0, 0, 1201, 1883, 36, 821, 0, 0, 0, 0, 18, 1468, 1782, 0, 372, 233, 0, 0, 0, 968, 1170, 189, 1450, 0, 0, 0, 0, 2469, 0, 1568, 0, 0, 0, 0, 2319, 1967, 104, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3812, 4434, 4504, 3941, 3873, 3777, 4037, 4390] avg=4096.0000 max=4504 max_rank=2 min=3777 min_rank=5 imbalance_ratio=1.099609 max_load_reduction=28264 max_load_reduction_pct=0.862549 moved_to_echo=16077 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10880, 12800, 128, 384, 6784, 2176, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 12544, 15360, 128, 3200, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[5248, 128, 14720, 640, 12416, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 9088, 10880, 1792, 11136, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12032, 13824, 256, 6912, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9344, 128, 17408, 640, 3456, 2176, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 20864, 0, 12032, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[16128, 15744, 1024, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[59, 0, 0, 5, 89, 0, 0, 0, 0, 137, 4065, 4, 1227, 2, 4096, 4083, 4096, 0, 0, 0, 3184, 0, 0, 0, 4090, 3537, 0, 0, 0, 4093, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[535, 0, 0, 0, 1267, 0, 0, 0, 0, 74, 4066, 0, 2100, 0, 4096, 4092, 4096, 0, 0, 0, 2349, 0, 0, 0, 4095, 1902, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[728, 0, 0, 0, 3063, 0, 0, 0, 0, 84, 4049, 0, 1953, 0, 4087, 4095, 4096, 0, 0, 0, 1124, 0, 0, 0, 4095, 1299, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1247, 0, 0, 0, 2487, 0, 0, 0, 0, 219, 4095, 0, 2580, 0, 4096, 4070, 4096, 0, 0, 0, 1059, 0, 0, 0, 4096, 628, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[150, 2, 0, 0, 825, 0, 5, 0, 4, 128, 3979, 1, 1172, 4, 4089, 4077, 4091, 5, 4, 2, 2672, 0, 5, 0, 4091, 3365, 4, 3, 0, 4088, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[228, 0, 0, 0, 827, 0, 1, 0, 0, 69, 4093, 0, 1513, 0, 4089, 4095, 4095, 0, 1, 0, 2237, 1, 0, 0, 4095, 3337, 0, 0, 0, 4086, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1347, 0, 2, 0, 1754, 0, 0, 0, 0, 63, 4095, 0, 1822, 0, 4096, 4087, 4094, 0, 0, 0, 1870, 0, 0, 0, 4096, 1348, 0, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1540, 0, 2, 0, 2591, 0, 0, 0, 0, 67, 4079, 2, 1953, 7, 4093, 4096, 4093, 0, 0, 0, 1078, 0, 0, 0, 4095, 972, 0, 3, 0, 4095, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[59, 0, 0, 5, 3368, 0, 0, 0, 89, 0, 0, 0, 1753, 728, 0, 0, 0, 137, 3990, 4, 0, 0, 0, 0, 1227, 2, 0, 2303, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3184, 0, 0, 0, 1780, 292, 73, 0, 2045, 3537, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[3432, 2570, 4131, 3532, 4096, 5329, 5582, 4096] avg=4096.0000 max=5582 max_rank=6 min=2570 min_rank=1 imbalance_ratio=1.362793 max_load_reduction=27186 max_load_reduction_pct=0.829651 moved_to_echo=19103 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[728, 0, 0, 0, 3360, 0, 0, 0, 3063, 0, 0, 0, 1755, 727, 0, 0, 0, 84, 3975, 0, 0, 0, 0, 0, 1953, 0, 0, 2309, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1124, 0, 0, 0, 1786, 292, 72, 0, 2048, 1299, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 5545, 4059, 4262, 4096, 3274, 3347, 4097] avg=4096.0000 max=5545 max_rank=1 min=3274 min_rank=5 imbalance_ratio=1.353760 max_load_reduction=27223 max_load_reduction_pct=0.830780 moved_to_echo=14814 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[228, 0, 0, 0, 3362, 0, 0, 0, 827, 0, 1, 0, 1755, 727, 0, 0, 0, 69, 4018, 0, 0, 0, 0, 0, 1513, 0, 0, 2309, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 2237, 1, 0, 0, 1786, 292, 73, 0, 2048, 3337, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 1, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[3590, 3310, 4087, 3822, 4096, 4389, 5385, 4089] avg=4096.0000 max=5385 max_rank=6 min=3310 min_rank=1 imbalance_ratio=1.314697 max_load_reduction=27383 max_load_reduction_pct=0.835663 moved_to_echo=17959 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1247, 0, 0, 0, 3368, 0, 0, 0, 2487, 0, 0, 0, 1755, 728, 0, 0, 0, 219, 4020, 0, 0, 0, 0, 0, 2580, 0, 0, 2295, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1059, 0, 0, 0, 1775, 293, 73, 0, 2048, 628, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4615, 4970, 4239, 4875, 4096, 3200, 2676, 4097] avg=4096.0000 max=4970 max_rank=1 min=2676 min_rank=6 imbalance_ratio=1.213379 max_load_reduction=27798 max_load_reduction_pct=0.848328 moved_to_echo=14069 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[535, 0, 0, 0, 3368, 0, 0, 0, 1267, 0, 0, 0, 1755, 728, 0, 0, 0, 74, 3991, 0, 0, 0, 0, 0, 2100, 0, 0, 2308, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2349, 0, 0, 0, 1784, 292, 73, 0, 2048, 1902, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[3903, 3750, 4065, 4408, 4096, 4498, 3950, 4098] avg=4096.0000 max=4498 max_rank=5 min=3750 min_rank=1 imbalance_ratio=1.098145 max_load_reduction=28270 max_load_reduction_pct=0.862732 moved_to_echo=16642 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1347, 0, 2, 0, 3368, 0, 0, 0, 1754, 0, 0, 0, 1755, 728, 0, 0, 0, 63, 4020, 0, 0, 0, 0, 0, 1822, 0, 0, 2305, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1870, 0, 0, 0, 1782, 293, 73, 0, 2048, 1348, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4717, 4237, 4083, 4127, 4094, 4018, 3396, 4096] avg=4096.0000 max=4717 max_rank=0 min=3396 min_rank=6 imbalance_ratio=1.151611 max_load_reduction=28051 max_load_reduction_pct=0.856049 moved_to_echo=15604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[150, 2, 0, 0, 3363, 0, 0, 0, 825, 0, 5, 0, 1757, 726, 0, 0, 4, 128, 3898, 1, 0, 0, 0, 0, 1172, 4, 0, 2296, 0, 0, 0, 0, 4090, 5, 4, 2, 0, 0, 0, 0, 2672, 0, 5, 0, 1781, 297, 76, 0, 2037, 3365, 4, 3, 0, 0, 0, 0, 0, 4088, 2, 0, 5, 1, 0, 0] rerouted_tokens_per_ep_rank=[3515, 3313, 4031, 3472, 4101, 4831, 5409, 4096] avg=4096.0000 max=5409 max_rank=6 min=3313 min_rank=1 imbalance_ratio=1.320557 max_load_reduction=27359 max_load_reduction_pct=0.834930 moved_to_echo=18437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1540, 0, 2, 0, 3366, 0, 0, 0, 2591, 0, 0, 0, 1755, 727, 0, 0, 0, 67, 4004, 2, 0, 0, 0, 0, 1953, 7, 0, 2310, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1078, 0, 0, 0, 1786, 292, 73, 0, 2048, 972, 0, 3, 0, 0, 0, 0, 0, 4095, 0, 2, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4908, 5073, 4073, 4270, 4093, 3229, 3023, 4099] avg=4096.0000 max=5073 max_rank=1 min=3023 min_rank=6 imbalance_ratio=1.238525 max_load_reduction=27695 max_load_reduction_pct=0.845184 moved_to_echo=14444 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[12928, 0, 128, 0, 14080, 5888, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[14336, 128, 0, 18560, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16384, 16512, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[5888, 128, 128, 128, 27008, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[15616, 128, 128, 0, 14336, 2432, 640, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 896, 32000, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 1511, 0, 0, 1, 4095, 0, 4009, 4096, 1253, 1, 4095, 0, 3, 6, 1, 0, 0, 0, 0, 2414, 4095, 0, 18, 4092, 2, 0, 0, 542, 0, 2533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2107, 0, 4, 0, 4096, 1, 3763, 4096, 855, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 4096, 20, 111, 4065, 0, 0, 0, 473, 0, 3552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 2695, 0, 0, 0, 4096, 0, 3966, 4096, 2265, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 797, 4096, 10, 41, 4089, 0, 0, 0, 573, 0, 1947] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3374, 0, 0, 0, 4096, 0, 4035, 4096, 1705, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 1418, 4096, 0, 2, 4077, 0, 0, 0, 814, 0, 957] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 2110, 0, 0, 0, 4096, 0, 4078, 4096, 1537, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 2304, 4096, 11, 60, 4088, 0, 0, 0, 890, 0, 1300] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[2, 5, 2682, 0, 3, 0, 4096, 0, 4003, 4096, 2026, 0, 4095, 0, 0, 12, 0, 0, 0, 0, 0, 2251, 4096, 0, 10, 4079, 0, 0, 0, 766, 0, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1349, 5, 0, 0, 4090, 0, 3999, 4083, 1077, 2, 4092, 4, 17, 0, 5, 5, 0, 7, 5, 1756, 4091, 0, 21, 4074, 0, 0, 6, 788, 0, 3292] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 7, 3485, 0, 25, 0, 4095, 1, 3993, 4096, 2596, 0, 4095, 0, 0, 4, 0, 0, 0, 0, 0, 950, 4096, 0, 1, 4080, 0, 0, 0, 753, 0, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2107, 0, 1670, 7, 1, 0, 4, 0, 4093, 1, 0, 0, 0, 0, 2299, 0, 855, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1433, 2426, 20, 0, 0, 0, 0, 111, 4048, 0, 0, 0, 0, 0, 0, 0, 473, 0, 3552, 1457, 17, 5, 3] rerouted_tokens_per_ep_rank=[3785, 4098, 3154, 4091, 4095, 3879, 4159, 5507] avg=4096.0000 max=5507 max_rank=7 min=3154 min_rank=2 imbalance_ratio=1.344482 max_load_reduction=27261 max_load_reduction_pct=0.831940 moved_to_echo=17640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1511, 0, 1670, 7, 1, 0, 0, 1, 4092, 0, 0, 0, 0, 0, 2450, 0, 1253, 1, 0, 0, 0, 0, 4090, 0, 3, 6, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 2414, 2425, 0, 0, 0, 0, 0, 18, 4075, 2, 0, 0, 0, 0, 0, 0, 542, 0, 2533, 1552, 17, 5, 3] rerouted_tokens_per_ep_rank=[3190, 4093, 3704, 4099, 4096, 4839, 4095, 4652] avg=4096.0000 max=4839 max_rank=5 min=3190 min_rank=0 imbalance_ratio=1.181396 max_load_reduction=27929 max_load_reduction_pct=0.852325 moved_to_echo=17682 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 5, 2682, 0, 1670, 7, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 2446, 0, 2026, 0, 0, 0, 0, 0, 4090, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2251, 2426, 0, 0, 0, 0, 0, 10, 4062, 0, 0, 0, 0, 0, 0, 0, 766, 0, 546, 1550, 17, 5, 3] rerouted_tokens_per_ep_rank=[4366, 4096, 4472, 4102, 4096, 4677, 4072, 2887] avg=4096.0000 max=4677 max_rank=5 min=2887 min_rank=7 imbalance_ratio=1.141846 max_load_reduction=28091 max_load_reduction_pct=0.857269 moved_to_echo=15732 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3374, 0, 1670, 7, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2466, 0, 1705, 0, 0, 0, 0, 0, 4091, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1418, 2426, 0, 0, 0, 0, 0, 2, 4060, 0, 0, 0, 0, 0, 0, 0, 814, 0, 957, 1562, 17, 5, 3] rerouted_tokens_per_ep_rank=[5051, 4093, 4171, 4093, 4096, 3844, 4062, 3358] avg=4096.0000 max=5051 max_rank=0 min=3358 min_rank=7 imbalance_ratio=1.233154 max_load_reduction=27717 max_load_reduction_pct=0.845856 moved_to_echo=15360 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2695, 0, 1670, 7, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2424, 0, 2265, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 797, 2426, 10, 0, 0, 0, 0, 41, 4072, 0, 0, 0, 0, 0, 0, 0, 573, 0, 1947, 1535, 17, 5, 3] rerouted_tokens_per_ep_rank=[4374, 4093, 4689, 4091, 4095, 3233, 4113, 4080] avg=4096.0000 max=4689 max_rank=2 min=3233 min_rank=5 imbalance_ratio=1.144775 max_load_reduction=28079 max_load_reduction_pct=0.856903 moved_to_echo=15521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 2110, 0, 1670, 8, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2491, 0, 1537, 0, 0, 0, 0, 0, 4091, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2304, 2426, 11, 0, 0, 0, 0, 60, 4071, 0, 0, 0, 0, 0, 0, 0, 890, 0, 1300, 1579, 17, 5, 3] rerouted_tokens_per_ep_rank=[3791, 4093, 4028, 4094, 4096, 4741, 4131, 3794] avg=4096.0000 max=4741 max_rank=5 min=3791 min_rank=0 imbalance_ratio=1.157471 max_load_reduction=28027 max_load_reduction_pct=0.855316 moved_to_echo=16762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1349, 5, 1673, 13, 1, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 2435, 0, 1077, 2, 0, 0, 0, 0, 4083, 4, 17, 0, 0, 0, 0, 0, 5, 5, 0, 7, 4082, 0, 0, 0, 5, 1756, 2418, 0, 0, 0, 0, 0, 21, 4051, 0, 0, 0, 0, 0, 0, 6, 788, 0, 3292, 1551, 23, 9, 6] rerouted_tokens_per_ep_rank=[3041, 4084, 3514, 4104, 4099, 4179, 4072, 5675] avg=4096.0000 max=5675 max_rank=7 min=3041 min_rank=0 imbalance_ratio=1.385498 max_load_reduction=27093 max_load_reduction_pct=0.826813 moved_to_echo=18025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 3485, 0, 1670, 7, 1, 0, 25, 0, 4092, 1, 0, 0, 0, 0, 2440, 0, 2596, 0, 0, 0, 0, 0, 4090, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 950, 2426, 0, 0, 0, 0, 0, 1, 4063, 0, 0, 0, 0, 0, 0, 0, 753, 0, 491, 1546, 17, 5, 3] rerouted_tokens_per_ep_rank=[5170, 4118, 5036, 4094, 4095, 3376, 4064, 2815] avg=4096.0000 max=5170 max_rank=0 min=2815 min_rank=7 imbalance_ratio=1.262207 max_load_reduction=27598 max_load_reduction_pct=0.842224 moved_to_echo=14350 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 5632, 0, 14720, 12416, 256, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 19328, 128, 13440, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[19456, 0, 13440, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 13440, 19456, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 32512, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4060, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 4093, 4096, 0, 3170, 4096, 88, 3, 183, 6, 0, 0, 666, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4043, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 4094, 4095, 0, 1561, 4096, 102, 2, 429, 32, 1, 0, 2008, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 4013, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 4096, 4096, 0, 2441, 4096, 147, 0, 362, 30, 0, 0, 1193, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 63, 0, 4093, 4096, 0, 2271, 4096, 43, 7, 274, 11, 2, 0, 1443, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4057, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 52, 0, 4092, 4095, 0, 1204, 4096, 271, 0, 461, 17, 0, 0, 2136, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4063, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 4080, 4096, 0, 2075, 4096, 113, 1, 571, 14, 0, 0, 1350, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4042, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 23, 0, 4096, 4096, 0, 1488, 4096, 80, 14, 558, 54, 0, 0, 1933, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[12, 0, 0, 5, 2, 4083, 0, 3, 0, 4091, 5, 0, 5, 0, 6, 0, 4090, 22, 6, 4091, 4072, 3, 2808, 4083, 9, 0, 546, 92, 0, 5, 645, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 1, 0, 0, 0, 4060, 0, 0, 24, 15, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 19, 0, 4069, 0, 0, 0, 0, 1970, 0, 3170, 0, 0, 0, 0, 0, 88, 3, 183, 6, 2126, 1405, 1, 0, 0, 0, 666, 2676, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4099, 4096, 4095, 4088, 5140, 3812, 3342] avg=4096.0000 max=5140 max_rank=5 min=3342 min_rank=7 imbalance_ratio=1.254883 max_load_reduction=27628 max_load_reduction_pct=0.843140 moved_to_echo=16382 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 1, 0, 0, 0, 4013, 0, 0, 24, 15, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 5, 0, 4072, 0, 0, 0, 0, 1970, 0, 2441, 0, 0, 0, 0, 0, 147, 0, 362, 30, 2126, 1405, 1, 0, 0, 0, 1193, 2676, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4052, 4096, 4095, 4077, 4411, 4071, 3869] avg=4096.0000 max=4411 max_rank=5 min=3869 min_rank=7 imbalance_ratio=1.076904 max_load_reduction=28357 max_load_reduction_pct=0.865387 moved_to_echo=16428 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 4081, 0, 0, 24, 15, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 63, 0, 4069, 0, 0, 0, 0, 1970, 0, 2271, 0, 0, 0, 0, 0, 43, 7, 274, 11, 2126, 1405, 2, 0, 2, 0, 1443, 2676, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4120, 4096, 4096, 4132, 4241, 3868, 4121] avg=4096.0000 max=4241 max_rank=5 min=3868 min_rank=6 imbalance_ratio=1.035400 max_load_reduction=28527 max_load_reduction_pct=0.870575 moved_to_echo=16362 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 4043, 0, 0, 24, 15, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 18, 0, 4070, 0, 0, 0, 0, 1970, 0, 1561, 0, 0, 0, 0, 0, 102, 2, 429, 32, 2125, 1405, 2, 0, 1, 0, 2008, 2676, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4082, 4095, 4096, 4088, 3531, 4097, 4685] avg=4096.0000 max=4685 max_rank=7 min=3531 min_rank=5 imbalance_ratio=1.143799 max_load_reduction=28083 max_load_reduction_pct=0.857025 moved_to_echo=16401 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 4063, 0, 0, 24, 15, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 0, 4056, 0, 0, 0, 0, 1970, 0, 2075, 0, 0, 0, 0, 0, 113, 1, 571, 14, 2126, 1405, 1, 0, 0, 0, 1350, 2676, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4102, 4096, 4096, 4077, 4045, 4231, 4026] avg=4096.0000 max=4231 max_rank=6 min=4026 min_rank=7 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16379 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 1, 0, 0, 0, 4057, 0, 0, 24, 15, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 52, 0, 4068, 0, 0, 0, 0, 1970, 0, 1204, 0, 0, 0, 0, 0, 271, 0, 461, 17, 2125, 1405, 1, 0, 0, 0, 2136, 2675, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4096, 4095, 4120, 3174, 4280, 4811] avg=4096.0000 max=4811 max_rank=7 min=3174 min_rank=5 imbalance_ratio=1.174561 max_load_reduction=27957 max_load_reduction_pct=0.853180 moved_to_echo=16385 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 5, 4082, 1, 0, 0, 2, 4083, 0, 3, 27, 21, 0, 0, 0, 4091, 5, 0, 1, 0, 0, 0, 5, 0, 6, 0, 4089, 0, 0, 0, 0, 22, 6, 4063, 0, 0, 0, 0, 1957, 3, 2808, 0, 0, 0, 0, 0, 9, 0, 546, 92, 2115, 1408, 1, 0, 0, 5, 645, 2655, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4136, 4097, 4100, 4091, 4768, 4171, 3305] avg=4096.0000 max=4768 max_rank=5 min=3305 min_rank=7 imbalance_ratio=1.164062 max_load_reduction=28000 max_load_reduction_pct=0.854492 moved_to_echo=16335 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 1, 0, 0, 0, 4042, 0, 0, 24, 15, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 23, 0, 4072, 0, 0, 0, 0, 1970, 0, 1488, 0, 0, 0, 0, 0, 80, 14, 558, 54, 2126, 1405, 1, 0, 0, 0, 1933, 2676, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4081, 4096, 4095, 4095, 3458, 4238, 4609] avg=4096.0000 max=4609 max_rank=7 min=3458 min_rank=5 imbalance_ratio=1.125244 max_load_reduction=28159 max_load_reduction_pct=0.859344 moved_to_echo=16400 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 128, 3456, 256, 17024, 11264, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15872, 128, 17024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 11392, 21504, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 256, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32512, 0, 128, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4091, 0, 0, 0, 4096, 3890, 0, 0, 0, 0, 0, 0, 206, 0, 6, 4094, 0, 4085, 0, 0, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 1, 0, 4094, 0, 0, 0, 4096, 3110, 0, 0, 0, 0, 0, 0, 990, 0, 1, 4091, 0, 4096, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4071, 0, 0, 0, 0, 0, 0, 27, 0, 0, 4095, 0, 4092, 0, 1, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 4090, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 4031, 0, 0, 0, 0, 0, 0, 65, 0, 5, 4096, 0, 4091, 0, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4053, 0, 0, 0, 0, 0, 0, 51, 0, 2, 4095, 0, 4089, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[5, 5, 7, 4089, 0, 4087, 4090, 0, 4, 0, 0, 4091, 5, 2, 3, 4090, 3655, 1, 0, 0, 5, 20, 0, 420, 1, 0, 4090, 2, 4081, 6, 9, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 4047, 0, 0, 0, 0, 0, 0, 46, 0, 6, 4096, 0, 4092, 0, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3890, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 206, 3867, 0, 0, 0, 0, 6, 4094, 0, 0, 0, 0, 0, 4085, 0, 0, 11, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4098, 4093, 4096, 4117, 4073, 4100, 4096] avg=4096.0000 max=4117 max_rank=4 min=4073 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16386 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3110, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 990, 3868, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4096, 4096, 3337, 4858, 4092, 4098] avg=4096.0000 max=4858 max_rank=5 min=3337 min_rank=4 imbalance_ratio=1.186035 max_load_reduction=27910 max_load_reduction_pct=0.851746 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4031, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 65, 3866, 0, 0, 0, 0, 5, 4095, 0, 0, 0, 0, 0, 4091, 0, 0, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4099, 4094, 4096, 4258, 3931, 4100, 4095] avg=4096.0000 max=4258 max_rank=4 min=3931 min_rank=5 imbalance_ratio=1.039551 max_load_reduction=28510 max_load_reduction_pct=0.870056 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 10, 3866, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4090, 0, 0, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4099, 4098, 4096, 4315, 3876, 4095, 4094] avg=4096.0000 max=4315 max_rank=4 min=3876 min_rank=5 imbalance_ratio=1.053467 max_load_reduction=28453 max_load_reduction_pct=0.868317 moved_to_echo=16380 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4071, 0, 0, 0, 228, 0, 0, 0, 0, 0, 0, 27, 3868, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4092, 0, 1, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4098, 4096, 4299, 3895, 4094, 4096] avg=4096.0000 max=4299 max_rank=4 min=3895 min_rank=5 imbalance_ratio=1.049561 max_load_reduction=28469 max_load_reduction_pct=0.868805 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4053, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 51, 3866, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 4089, 0, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4098, 4096, 4096, 4280, 3917, 4096, 4091] avg=4096.0000 max=4280 max_rank=4 min=3917 min_rank=5 imbalance_ratio=1.044922 max_load_reduction=28488 max_load_reduction_pct=0.869385 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 5, 7, 4087, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4091, 2, 3, 1, 0, 5, 2, 3, 4086, 0, 0, 0, 0, 3655, 1, 0, 0, 229, 0, 0, 0, 5, 20, 0, 420, 3861, 0, 0, 0, 1, 0, 4087, 2, 0, 0, 0, 0, 4081, 6, 9, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4087, 4101, 4096, 3885, 4306, 4090, 4099] avg=4096.0000 max=4306 max_rank=5 min=3885 min_rank=4 imbalance_ratio=1.051270 max_load_reduction=28462 max_load_reduction_pct=0.868591 moved_to_echo=16380 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4047, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 46, 3866, 0, 0, 0, 0, 6, 4095, 0, 0, 0, 0, 0, 4092, 0, 0, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4092, 4096, 4277, 3912, 4101, 4099] avg=4096.0000 max=4277 max_rank=4 min=3912 min_rank=5 imbalance_ratio=1.044189 max_load_reduction=28491 max_load_reduction_pct=0.869476 moved_to_echo=16389 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 1920, 30976, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[30976, 128, 0, 0, 1920, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3943, 4084, 3944, 0, 0, 45, 0, 0, 0, 8, 26, 4096, 0, 4082, 22, 0, 4096, 0, 0, 4096, 0, 0, 0, 230, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4091, 4084, 3890, 0, 0, 158, 0, 0, 0, 11, 20, 4096, 0, 4091, 2, 0, 4094, 3, 0, 4096, 0, 0, 0, 36, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4086, 4095, 3915, 0, 0, 167, 0, 0, 0, 1, 9, 4095, 0, 4095, 0, 0, 4096, 1, 0, 4096, 0, 0, 1, 15, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4091, 4092, 4004, 0, 0, 52, 0, 0, 0, 6, 4, 4094, 0, 4094, 17, 0, 4096, 0, 0, 4096, 0, 0, 0, 26, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 4095, 4094, 4022, 0, 0, 55, 0, 0, 0, 3, 12, 4096, 0, 4094, 0, 0, 4095, 2, 0, 4096, 0, 0, 1, 6, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4080, 4058, 0, 0, 32, 0, 0, 0, 16, 4, 4096, 0, 4094, 0, 0, 4096, 2, 0, 4096, 0, 0, 1, 1, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4085, 4067, 4053, 0, 0, 24, 0, 0, 0, 18, 34, 4094, 0, 4096, 0, 0, 4095, 8, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[2, 4085, 4, 0, 4039, 4091, 3474, 5, 6, 437, 3, 5, 0, 0, 39, 4067, 3, 4064, 142, 5, 4089, 0, 2, 4090, 3, 5, 12, 90, 0, 6, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 171, 0, 3944, 0, 0, 0, 0, 0, 0, 45, 0, 0, 3772, 43, 22, 16, 0, 8, 26, 4074, 0, 0, 0, 0, 0, 4066, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 4041, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4115, 3898, 4108, 4088, 4096, 4271, 4096] avg=4096.0000 max=4271 max_rank=6 min=3898 min_rank=2 imbalance_ratio=1.042725 max_load_reduction=28497 max_load_reduction_pct=0.869659 moved_to_echo=16551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 177, 0, 3890, 0, 0, 0, 0, 0, 0, 158, 0, 0, 3914, 43, 22, 16, 0, 11, 20, 4074, 0, 0, 0, 0, 0, 4075, 2, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4041, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4067, 4153, 4105, 4077, 4097, 4077, 4096] avg=4096.0000 max=4153 max_rank=2 min=4067 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 177, 0, 4004, 0, 0, 0, 0, 0, 0, 52, 0, 0, 3914, 44, 22, 16, 0, 6, 4, 4072, 0, 0, 0, 0, 0, 4078, 17, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4048, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4181, 4048, 4082, 4095, 4096, 4074, 4096] avg=4096.0000 max=4181 max_rank=1 min=4048 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 177, 0, 4058, 0, 0, 0, 0, 0, 0, 32, 0, 0, 3919, 43, 22, 16, 0, 16, 4, 4074, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4037, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4235, 4032, 4094, 4078, 4098, 4039, 4096] avg=4096.0000 max=4235 max_rank=1 min=4032 min_rank=2 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 177, 0, 4022, 0, 0, 0, 0, 0, 0, 55, 0, 0, 3918, 43, 22, 16, 0, 3, 12, 4074, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4051, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4199, 4054, 4089, 4078, 4097, 4058, 4096] avg=4096.0000 max=4199 max_rank=1 min=4054 min_rank=2 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 177, 0, 3915, 0, 0, 0, 0, 0, 0, 167, 0, 0, 3909, 44, 22, 16, 0, 1, 9, 4073, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 4051, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4092, 4158, 4083, 4079, 4097, 4067, 4096] avg=4096.0000 max=4158 max_rank=2 min=4067 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 4085, 4, 0, 4, 0, 0, 0, 170, 0, 3474, 5, 0, 0, 0, 0, 6, 437, 3, 5, 3869, 44, 23, 21, 0, 0, 39, 4044, 0, 0, 0, 0, 3, 4043, 142, 5, 0, 0, 0, 0, 4085, 0, 2, 0, 0, 0, 0, 0, 3, 5, 12, 90, 4047, 0, 0, 0, 0, 6, 0, 0, 4090, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 3649, 4408, 4083, 4193, 4087, 4157, 4096] avg=4096.0000 max=4408 max_rank=2 min=3649 min_rank=1 imbalance_ratio=1.076172 max_load_reduction=28360 max_load_reduction_pct=0.865479 moved_to_echo=16533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 177, 0, 4053, 0, 0, 0, 0, 0, 0, 24, 0, 0, 3908, 44, 22, 16, 0, 18, 34, 4072, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 4095, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4023, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4230, 4014, 4124, 4080, 4103, 4025, 4096] avg=4096.0000 max=4230 max_rank=1 min=4014 min_rank=2 imbalance_ratio=1.032715 max_load_reduction=28538 max_load_reduction_pct=0.870911 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32765, 32768, 32768, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32765, 32768, 32768, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32765, 32768, 32768, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32765, 32768, 32768, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32765, 32768, 32768, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32765, 32768, 32768, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32765, 32768, 32768, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32765, 32768, 32768, 32771, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32771 actual_valid_recv=33024 observed_loss=-253 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32765 actual_valid_recv=33536 observed_loss=-771 tokens_per_expert=[128, 1024, 128, 128, 31232, 384, 256, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 512, 32384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[1408, 0, 31360, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 256, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 13, 0, 4096, 4012, 0, 0, 0, 0, 4081, 78, 0, 13, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 7, 0, 4096, 4018, 0, 0, 0, 0, 4096, 66, 0, 4, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3908, 0, 0, 0, 0, 4095, 187, 0, 5, 0, 0, 4096, 4092, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3863, 0, 0, 0, 0, 4089, 242, 0, 2, 0, 0, 4096, 4092, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4038, 0, 0, 0, 0, 4077, 69, 0, 7, 1, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4075, 0, 0, 0, 0, 4085, 21, 0, 11, 1, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[5, 6, 0, 4089, 3591, 6, 0, 6, 104, 4085, 166, 5, 0, 0, 6, 4082, 4087, 152, 4083, 0, 6, 7, 4090, 84, 9, 0, 3, 4, 0, 4087, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 4096, 3889, 0, 0, 0, 0, 4078, 207, 0, 17, 3, 0, 4096, 4093, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 0, 4095, 0, 0, 0, 0, 4012, 0, 0, 0, 132, 16, 11, 7, 0, 3949, 78, 0, 0, 0, 0, 0, 13, 0, 0, 4089, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4178, 4027, 4102, 4080, 4085, 4096, 4092] avg=4096.0000 max=4178 max_rank=1 min=4027 min_rank=2 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16353 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4075, 0, 0, 0, 132, 16, 11, 7, 0, 3953, 21, 0, 0, 0, 0, 0, 11, 1, 0, 4089, 0, 0, 0, 0, 4079, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4241, 3974, 4101, 4080, 4085, 4095, 4097] avg=4096.0000 max=4241 max_rank=1 min=3974 min_rank=2 imbalance_ratio=1.035400 max_load_reduction=28527 max_load_reduction_pct=0.870575 moved_to_echo=16357 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4038, 0, 0, 0, 132, 16, 11, 7, 0, 3945, 69, 0, 0, 0, 0, 0, 7, 1, 0, 4089, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4204, 4014, 4097, 4080, 4086, 4096, 4096] avg=4096.0000 max=4204 max_rank=1 min=4014 min_rank=2 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16358 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 4095, 0, 0, 0, 0, 4018, 0, 0, 0, 133, 16, 11, 7, 0, 3963, 66, 0, 0, 0, 0, 0, 4, 0, 0, 4089, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4185, 4029, 4093, 4080, 4086, 4096, 4097] avg=4096.0000 max=4185 max_rank=1 min=4029 min_rank=2 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16359 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 3908, 0, 0, 0, 133, 16, 11, 7, 0, 3962, 187, 0, 0, 0, 0, 0, 5, 0, 0, 4089, 0, 0, 0, 0, 4076, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4075, 4149, 4094, 4078, 4085, 4095, 4097] avg=4096.0000 max=4149 max_rank=2 min=4075 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16355 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 3863, 0, 0, 0, 132, 16, 11, 7, 0, 3957, 242, 0, 0, 0, 0, 0, 2, 0, 0, 4089, 0, 0, 0, 0, 4076, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4029, 4199, 4091, 4077, 4085, 4095, 4097] avg=4096.0000 max=4199 max_rank=2 min=4029 min_rank=1 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16354 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 6, 0, 4086, 0, 0, 0, 0, 3591, 6, 0, 6, 137, 20, 16, 7, 104, 3948, 166, 5, 0, 0, 0, 0, 0, 0, 6, 4075, 0, 0, 0, 0, 4067, 152, 0, 0, 0, 0, 0, 0, 6, 7, 4074, 84, 0, 0, 0, 0, 9, 0, 3, 4, 4083, 0, 0, 0, 0, 4087, 5, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 3783, 4223, 4081, 4219, 4171, 4099, 4095] avg=4096.0000 max=4223 max_rank=2 min=3783 min_rank=1 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16584 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4095, 0, 0, 0, 0, 3889, 0, 0, 0, 132, 16, 11, 7, 0, 3946, 207, 0, 0, 0, 0, 0, 17, 3, 0, 4089, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4055, 4153, 4109, 4077, 4085, 4096, 4097] avg=4096.0000 max=4153 max_rank=2 min=4055 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16355 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32783, 32750, 32768, 32768, 32771, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32783, 32750, 32768, 32768, 32771, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32783, 32750, 32768, 32768, 32771, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32783, 32750, 32768, 32768, 32771, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32783, 32750, 32768, 32768, 32771, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32783, 32750, 32768, 32768, 32771, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32783, 32750, 32768, 32768, 32771, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32783, 32750, 32768, 32768, 32771, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32783 actual_valid_recv=33024 observed_loss=-241 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32750 actual_valid_recv=33408 observed_loss=-658 tokens_per_expert=[31488, 128, 0, 128, 1152, 256, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32771 actual_valid_recv=33024 observed_loss=-253 tokens_per_expert=[32640, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 31744, 1152, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3834, 4096, 0, 301, 0, 0, 4096, 4096, 0, 4057, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3959, 4096, 0, 154, 0, 0, 4096, 4096, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3964, 4096, 0, 164, 0, 0, 4096, 4096, 0, 4064, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3937, 4095, 0, 196, 0, 0, 4096, 4096, 0, 4059, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3996, 4096, 0, 106, 0, 0, 4096, 4096, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3808, 4096, 0, 409, 0, 0, 4096, 4096, 0, 3974, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[5, 7, 3975, 4087, 3, 261, 0, 6, 4075, 4090, 6, 3944, 4, 0, 16, 0, 4088, 0, 6, 4089, 4066, 1, 1, 6, 0, 22, 0, 4, 0, 0, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 3812, 4093, 0, 325, 0, 0, 4096, 4096, 0, 4055, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3834, 186, 0, 0, 0, 0, 0, 301, 0, 0, 3856, 0, 0, 0, 56, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4040, 52, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4157, 4113, 4093, 4096, 4098, 4095, 4096] avg=4096.0000 max=4157 max_rank=1 min=4020 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3964, 186, 0, 0, 0, 0, 0, 164, 0, 0, 3856, 0, 0, 0, 56, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4040, 52, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4020, 4120, 4093, 4098, 4098, 4093, 4096] avg=4096.0000 max=4150 max_rank=0 min=4020 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3937, 186, 0, 0, 0, 0, 0, 196, 0, 0, 3855, 0, 0, 0, 56, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4040, 52, 2, 0, 0, 0, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4051, 4115, 4095, 4096, 4098, 4094, 4096] avg=4096.0000 max=4123 max_rank=0 min=4051 min_rank=1 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3959, 186, 0, 0, 0, 0, 0, 154, 0, 0, 3856, 0, 0, 0, 56, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4040, 52, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4010, 4135, 4093, 4096, 4098, 4095, 4096] avg=4096.0000 max=4145 max_rank=0 min=4010 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3996, 186, 0, 0, 0, 0, 0, 106, 0, 0, 3856, 0, 0, 0, 56, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4040, 52, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 3962, 4146, 4096, 4096, 4098, 4093, 4095] avg=4096.0000 max=4182 max_rank=0 min=3962 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3808, 186, 0, 0, 0, 0, 0, 409, 0, 0, 3856, 0, 0, 0, 56, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 4040, 52, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 4265, 4030, 4093, 4096, 4098, 4096, 4096] avg=4096.0000 max=4265 max_rank=1 min=3994 min_rank=0 imbalance_ratio=1.041260 max_load_reduction=28503 max_load_reduction_pct=0.869843 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[5, 7, 3975, 170, 0, 0, 0, 0, 3, 261, 0, 6, 3854, 0, 0, 0, 48, 0, 6, 3944, 0, 0, 0, 0, 4, 0, 16, 0, 4088, 1, 0, 0, 4088, 0, 6, 0, 0, 0, 0, 0, 4066, 1, 1, 6, 8, 0, 0, 0, 0, 22, 0, 4, 4027, 55, 1, 0, 0, 0, 0, 6, 4089, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4124, 3998, 4109, 4094, 4082, 4109, 4095] avg=4096.0000 max=4157 max_rank=0 min=3998 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3812, 185, 0, 0, 0, 0, 0, 325, 0, 0, 3854, 0, 0, 0, 56, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4040, 52, 1, 0, 0, 0, 3, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[3997, 4179, 4111, 4096, 4096, 4098, 4093, 4098] avg=4096.0000 max=4179 max_rank=1 min=3997 min_rank=0 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 31360, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 1920, 0, 128, 30848, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 32384, 512, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 0, 128, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 2504, 4, 0, 4096, 0, 0, 0, 4088, 4096, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 1918, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 3768] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 2969, 13, 0, 4096, 0, 0, 0, 4089, 4096, 4096, 0, 0, 0, 0, 4092, 0, 0, 1, 1272, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 26, 0, 3922] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 3003, 1, 0, 4096, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 591, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 522, 0, 4076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 2320, 1, 0, 4096, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 1687, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 4056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 2860, 9, 0, 4096, 0, 0, 0, 4090, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 1073, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 191, 0, 4065] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 2957, 7, 0, 4096, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 4093, 0, 0, 1, 1204, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 81, 0, 3945] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 2817, 8, 0, 4096, 0, 0, 0, 4085, 4096, 4096, 0, 0, 0, 1, 4093, 0, 0, 0, 915, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 370, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 920, 0, 2, 4089, 0, 6, 6, 4090, 4082, 4079, 7, 7, 5, 0, 4086, 0, 9, 0, 3537, 4089, 5, 9, 5, 4, 78, 0, 0, 0, 28, 0, 3625] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2504, 4, 0, 1530, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 1918, 2560, 10, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4086, 0, 0, 0, 0, 2, 0, 3753, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4096, 4094, 4094, 4488, 4095, 4093, 3755] avg=4096.0000 max=4488 max_rank=4 min=3755 min_rank=7 imbalance_ratio=1.095703 max_load_reduction=28280 max_load_reduction_pct=0.863037 moved_to_echo=16431 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2969, 13, 0, 1530, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 1, 1272, 2560, 11, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4085, 0, 0, 0, 0, 26, 0, 3906, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4528, 4096, 4095, 4093, 3844, 4094, 4086, 3932] avg=4096.0000 max=4528 max_rank=0 min=3844 min_rank=4 imbalance_ratio=1.105469 max_load_reduction=28240 max_load_reduction_pct=0.861816 moved_to_echo=15956 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2860, 9, 0, 1530, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 1073, 2560, 10, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4086, 0, 0, 0, 0, 191, 0, 4049, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4415, 4096, 4096, 4095, 3643, 4095, 4088, 4240] avg=4096.0000 max=4415 max_rank=0 min=3643 min_rank=4 imbalance_ratio=1.077881 max_load_reduction=28353 max_load_reduction_pct=0.865265 moved_to_echo=16066 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3003, 1, 0, 1530, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 591, 2560, 10, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4086, 0, 0, 0, 0, 522, 0, 4060, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4550, 4096, 4101, 4096, 3161, 4095, 4087, 4582] avg=4096.0000 max=4582 max_rank=7 min=3161 min_rank=4 imbalance_ratio=1.118652 max_load_reduction=28186 max_load_reduction_pct=0.860168 moved_to_echo=15925 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2320, 1, 0, 1530, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 1687, 2561, 11, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 129, 0, 4040, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3867, 4096, 4100, 4097, 4259, 4095, 4085, 4169] avg=4096.0000 max=4259 max_rank=4 min=3867 min_rank=0 imbalance_ratio=1.039795 max_load_reduction=28509 max_load_reduction_pct=0.870026 moved_to_echo=16608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2957, 7, 0, 1530, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 1, 1204, 2560, 10, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4086, 0, 0, 0, 0, 81, 0, 3929, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4510, 4096, 4100, 4094, 3775, 4095, 4088, 4010] avg=4096.0000 max=4510 max_rank=0 min=3775 min_rank=4 imbalance_ratio=1.101074 max_load_reduction=28258 max_load_reduction_pct=0.862366 moved_to_echo=15968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 920, 0, 2, 1529, 19, 3, 0, 4084, 0, 6, 6, 0, 0, 0, 0, 4090, 0, 0, 7, 0, 0, 0, 0, 7, 5, 0, 4086, 1, 5, 0, 0, 0, 9, 0, 3537, 2550, 11, 0, 0, 4085, 5, 9, 5, 0, 0, 0, 0, 4, 78, 0, 0, 4071, 0, 0, 0, 0, 28, 0, 3606, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2473, 4096, 4097, 4104, 6107, 4104, 4153, 3634] avg=4096.0000 max=6107 max_rank=4 min=2473 min_rank=0 imbalance_ratio=1.490967 max_load_reduction=26661 max_load_reduction_pct=0.813629 moved_to_echo=17998 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2817, 8, 0, 1531, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 1, 0, 0, 0, 0, 0, 0, 915, 2565, 11, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4085, 0, 0, 0, 0, 370, 0, 4076, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4372, 4096, 4085, 4095, 3491, 4095, 4088, 4446] avg=4096.0000 max=4446 max_rank=7 min=3491 min_rank=4 imbalance_ratio=1.085449 max_load_reduction=28322 max_load_reduction_pct=0.864319 moved_to_echo=16120 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 20352, 128, 128, 12288, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 12288, 20480, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 1408, 0, 31488, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 97, 0, 2, 0, 0, 0, 0, 4090, 3186, 0, 0, 3836, 0, 4096, 6, 0, 0, 0, 4094, 1060, 0, 14, 4095, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 67, 0, 3, 0, 0, 0, 0, 4088, 3890, 0, 0, 4065, 0, 4096, 6, 0, 0, 0, 4090, 154, 0, 23, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 243, 0, 1, 0, 0, 0, 0, 4096, 2742, 0, 0, 3876, 0, 4096, 10, 0, 0, 0, 4096, 1321, 0, 2, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 186, 0, 1, 0, 0, 0, 0, 4091, 3214, 0, 0, 3781, 0, 4096, 6, 0, 0, 0, 4094, 1006, 0, 6, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 402, 0, 1, 0, 0, 0, 0, 4092, 2781, 0, 0, 3400, 0, 4096, 4, 0, 0, 0, 4095, 1598, 0, 13, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 0, 947, 0, 0, 0, 0, 0, 0, 4096, 2687, 0, 0, 3979, 0, 4096, 0, 0, 0, 0, 4096, 584, 0, 1, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[2, 6, 0, 7, 4070, 0, 15, 6, 63, 125, 0, 7, 5, 5, 6, 4089, 3866, 4, 6, 2769, 41, 4044, 0, 1, 7, 0, 4088, 1359, 0, 0, 4090, 4087] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 300, 0, 4, 0, 0, 0, 0, 4090, 2027, 0, 0, 3858, 0, 4096, 33, 0, 0, 0, 4092, 1971, 0, 9, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 67, 0, 3, 0, 2914, 1128, 7, 2, 0, 0, 0, 4088, 2, 0, 0, 0, 3890, 0, 0, 1151, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 2962, 154, 0, 0, 0, 0, 0, 23, 2, 4089, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4121, 4090, 5041, 4096, 3116, 4114] avg=4096.0000 max=5041 max_rank=4 min=3116 min_rank=6 imbalance_ratio=1.230713 max_load_reduction=27727 max_load_reduction_pct=0.846161 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 97, 0, 2, 0, 2750, 1129, 7, 2, 0, 0, 0, 4090, 2, 0, 0, 0, 3186, 0, 0, 1086, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 2965, 1060, 0, 0, 0, 0, 0, 14, 0, 4089, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4098, 3987, 4092, 4272, 4096, 4025, 4103] avg=4096.0000 max=4272 max_rank=4 min=3987 min_rank=2 imbalance_ratio=1.042969 max_load_reduction=28496 max_load_reduction_pct=0.869629 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 186, 0, 1, 0, 2711, 1129, 7, 2, 0, 0, 0, 4091, 2, 0, 0, 0, 3214, 0, 0, 1070, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 2965, 1006, 0, 0, 0, 0, 0, 6, 2, 4089, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4097, 4036, 4093, 4284, 4096, 3971, 4097] avg=4096.0000 max=4284 max_rank=4 min=3971 min_rank=6 imbalance_ratio=1.045898 max_load_reduction=28484 max_load_reduction_pct=0.869263 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 402, 0, 1, 0, 2437, 1130, 7, 2, 0, 0, 0, 4092, 2, 0, 0, 0, 2781, 0, 0, 963, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 0, 2965, 1598, 0, 0, 0, 0, 0, 13, 2, 4089, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 3979, 4094, 3744, 4094, 4563, 4104] avg=4096.0000 max=4563 max_rank=6 min=3744 min_rank=4 imbalance_ratio=1.114014 max_load_reduction=28205 max_load_reduction_pct=0.860748 moved_to_echo=16505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 4093, 0, 0, 0, 2, 0, 0, 0, 243, 0, 1, 0, 2779, 1130, 7, 2, 0, 0, 0, 4096, 2, 0, 0, 0, 2742, 0, 0, 1097, 0, 0, 0, 0, 0, 4090, 10, 0, 0, 0, 0, 0, 0, 0, 2966, 1321, 0, 0, 0, 0, 0, 2, 2, 4089, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4095, 4162, 4098, 3839, 4100, 4287, 4093] avg=4096.0000 max=4287 max_rank=6 min=3839 min_rank=4 imbalance_ratio=1.046631 max_load_reduction=28481 max_load_reduction_pct=0.869171 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 2, 0, 0, 0, 947, 0, 0, 0, 2853, 1130, 7, 2, 0, 0, 0, 4096, 2, 0, 0, 0, 2687, 0, 0, 1126, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2966, 584, 0, 0, 0, 0, 0, 1, 0, 4089, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4092, 4939, 4098, 3813, 4090, 3550, 4090] avg=4096.0000 max=4939 max_rank=2 min=3550 min_rank=6 imbalance_ratio=1.205811 max_load_reduction=27829 max_load_reduction_pct=0.849274 moved_to_echo=15543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 6, 0, 7, 4090, 0, 0, 0, 4070, 0, 15, 6, 5, 0, 0, 0, 63, 125, 0, 7, 1989, 1132, 10, 2, 5, 5, 6, 4089, 6, 0, 0, 0, 3866, 4, 6, 780, 0, 0, 0, 0, 41, 4031, 0, 1, 0, 0, 0, 0, 7, 0, 2956, 1359, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4096, 3328, 4111, 4656, 4073, 4322, 4077] avg=4096.0000 max=4656 max_rank=4 min=3328 min_rank=2 imbalance_ratio=1.136719 max_load_reduction=28112 max_load_reduction_pct=0.857910 moved_to_echo=17128 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 300, 0, 4, 0, 2766, 1129, 7, 2, 0, 0, 0, 4090, 2, 0, 0, 0, 2027, 0, 0, 1092, 0, 0, 0, 0, 0, 4090, 33, 0, 0, 0, 0, 0, 0, 0, 2963, 1971, 0, 0, 0, 0, 0, 9, 0, 4089, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4098, 4208, 4092, 3119, 4123, 4934, 4098] avg=4096.0000 max=4934 max_rank=6 min=3119 min_rank=4 imbalance_ratio=1.204590 max_load_reduction=27834 max_load_reduction_pct=0.849426 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32760, 32768, 32768, 32768, 32768, 32776] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32760, 32768, 32768, 32768, 32768, 32776] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32760, 32768, 32768, 32768, 32768, 32776] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32760, 32768, 32768, 32768, 32768, 32776] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32760, 32768, 32768, 32768, 32768, 32776] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32760, 32768, 32768, 32768, 32768, 32776] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32760, 32768, 32768, 32768, 32768, 32776] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32760, 32768, 32768, 32768, 32768, 32776] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[24448, 128, 128, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32760 actual_valid_recv=33408 observed_loss=-648 tokens_per_expert=[2432, 128, 128, 128, 21248, 9088, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 23808, 9088, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32776 actual_valid_recv=33024 observed_loss=-248 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 26, 0, 4096, 3461, 3164, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[6, 6, 4073, 2503, 5, 0, 0, 0, 0, 0, 17, 7, 0, 4089, 7, 6, 4080, 8, 4, 4088, 0, 0, 452, 8, 4089, 2605, 2617, 0, 1, 0, 7, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 6, 0, 4096, 3834, 3351, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 18, 0, 4096, 3633, 3266, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 12, 0, 4096, 3383, 3076, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 97, 0, 4096, 3667, 2026, 0, 0, 0, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 14, 0, 4096, 3441, 3803, 0, 0, 0, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4, 0, 4096, 3918, 3509, 0, 0, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2579, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 26, 0, 2475, 1517, 1, 0, 0, 986, 3164, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4096, 4093, 4096, 4098, 4019, 4150, 4094] avg=4096.0000 max=4150 max_rank=6 min=4019 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16361 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2579, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2742, 1517, 1, 0, 0, 1092, 3351, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3580, 4096, 4095, 4097, 4096, 4266, 4443, 4095] avg=4096.0000 max=4443 max_rank=6 min=3580 min_rank=0 imbalance_ratio=1.084717 max_load_reduction=28325 max_load_reduction_pct=0.864410 moved_to_echo=16900 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2579, 2407, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 2622, 1517, 1, 0, 0, 1045, 2026, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4986, 4096, 4095, 4096, 4096, 4237, 3071, 4091] avg=4096.0000 max=4986 max_rank=0 min=3071 min_rank=6 imbalance_ratio=1.217285 max_load_reduction=27782 max_load_reduction_pct=0.847839 moved_to_echo=15495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2579, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 2598, 1517, 3, 0, 0, 1035, 3266, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3854, 4096, 4093, 4096, 4096, 4136, 4301, 4096] avg=4096.0000 max=4301 max_rank=6 min=3854 min_rank=0 imbalance_ratio=1.050049 max_load_reduction=28467 max_load_reduction_pct=0.868744 moved_to_echo=16629 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2579, 1721, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2419, 1517, 1, 0, 0, 964, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4300, 4096, 4095, 4096, 4096, 3949, 4040, 4096] avg=4096.0000 max=4300 max_rank=0 min=3949 min_rank=5 imbalance_ratio=1.049805 max_load_reduction=28468 max_load_reduction_pct=0.868774 moved_to_echo=16181 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 6, 2551, 2503, 0, 0, 0, 0, 5, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 17, 7, 4086, 0, 0, 0, 0, 4082, 7, 6, 0, 0, 0, 0, 4080, 8, 4, 0, 0, 0, 0, 0, 0, 0, 452, 8, 1867, 1516, 1, 0, 0, 738, 2617, 0, 0, 0, 0, 0, 1, 0, 7, 4090, 7, 6, 0, 0] rerouted_tokens_per_ep_rank=[5066, 4095, 4110, 4095, 4092, 3844, 3355, 4111] avg=4096.0000 max=5066 max_rank=0 min=3355 min_rank=6 imbalance_ratio=1.236816 max_load_reduction=27702 max_load_reduction_pct=0.845398 moved_to_echo=15402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2579, 764, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 2802, 1517, 1, 0, 0, 1116, 3509, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3343, 4096, 4093, 4096, 4098, 4324, 4625, 4093] avg=4096.0000 max=4625 max_rank=6 min=3343 min_rank=0 imbalance_ratio=1.129150 max_load_reduction=28143 max_load_reduction_pct=0.858856 moved_to_echo=17140 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2579, 938, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 2461, 1517, 1, 0, 0, 980, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3517, 4097, 4094, 4096, 4096, 3993, 4783, 4092] avg=4096.0000 max=4783 max_rank=6 min=3517 min_rank=0 imbalance_ratio=1.167725 max_load_reduction=27985 max_load_reduction_pct=0.854034 moved_to_echo=16964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 640, 128, 20096, 12160, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 20608, 12160, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 8064, 24832, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[128, 4096, 0, 0, 0, 4096, 4096, 1802, 0, 0, 4096, 0, 4096, 0, 3682, 2889, 0, 0, 0, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[290, 4096, 0, 0, 0, 4096, 4096, 673, 0, 0, 4096, 0, 4096, 0, 3392, 3996, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[223, 4096, 0, 0, 0, 4096, 4096, 315, 0, 0, 4096, 0, 4096, 0, 3942, 3906, 0, 0, 0, 3902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[394, 4096, 0, 0, 0, 4096, 4096, 675, 0, 0, 4096, 0, 4096, 0, 3378, 3831, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[102, 4096, 0, 0, 0, 4096, 4096, 817, 0, 0, 4096, 0, 4096, 0, 3374, 3928, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[150, 4096, 0, 0, 1, 4096, 4096, 784, 0, 0, 4096, 0, 4096, 0, 3542, 3898, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1546, 4079, 10, 0, 0, 4063, 4088, 178, 4, 5, 4082, 0, 4087, 9, 2338, 4081, 25, 4, 6, 4088, 6, 1, 0, 6, 0, 7, 12, 27, 0, 1, 5, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1169, 4096, 0, 0, 0, 4096, 4096, 456, 0, 0, 4096, 0, 4096, 0, 2905, 3868, 0, 0, 0, 3890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[128, 3598, 0, 0, 0, 0, 0, 0, 0, 3387, 0, 1802, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3682, 590, 0, 0, 0, 0, 0, 0, 0, 3787, 142, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2299, 709, 356, 1, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[3726, 5189, 4096, 4272, 3929, 4095, 3365, 4096] avg=4096.0000 max=5189 max_rank=1 min=3365 min_rank=6 imbalance_ratio=1.266846 max_load_reduction=27579 max_load_reduction_pct=0.841644 moved_to_echo=15485 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[150, 3598, 0, 0, 0, 0, 0, 0, 1, 3387, 0, 784, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3542, 796, 0, 0, 0, 0, 0, 0, 0, 3913, 142, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3102, 709, 356, 2, 0, 0, 0, 0, 4094, 1, 0, 0] rerouted_tokens_per_ep_rank=[3748, 4172, 4096, 4338, 4055, 4095, 4169, 4095] avg=4096.0000 max=4338 max_rank=3 min=3748 min_rank=0 imbalance_ratio=1.059082 max_load_reduction=28430 max_load_reduction_pct=0.867615 moved_to_echo=16414 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[223, 3598, 0, 0, 0, 0, 0, 0, 0, 3387, 0, 315, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3942, 798, 0, 0, 0, 0, 0, 0, 0, 3902, 142, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3108, 709, 356, 1, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[3821, 3702, 4096, 4740, 4044, 4095, 4174, 4096] avg=4096.0000 max=4740 max_rank=3 min=3702 min_rank=1 imbalance_ratio=1.157227 max_load_reduction=28028 max_load_reduction_pct=0.855347 moved_to_echo=16409 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[290, 3598, 0, 0, 0, 0, 0, 0, 0, 3387, 0, 673, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3392, 816, 0, 0, 0, 0, 0, 0, 0, 3937, 142, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3180, 709, 356, 1, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[3888, 4060, 4096, 4208, 4079, 4095, 4246, 4096] avg=4096.0000 max=4246 max_rank=6 min=3888 min_rank=0 imbalance_ratio=1.036621 max_load_reduction=28522 max_load_reduction_pct=0.870422 moved_to_echo=16516 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[102, 3598, 0, 0, 0, 0, 0, 0, 0, 3387, 0, 817, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3374, 802, 0, 0, 0, 0, 0, 0, 0, 4067, 142, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3126, 709, 356, 1, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[3700, 4204, 4096, 4176, 4209, 4095, 4192, 4096] avg=4096.0000 max=4209 max_rank=4 min=3700 min_rank=0 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16592 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[394, 3598, 0, 0, 0, 0, 0, 0, 0, 3387, 0, 675, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3378, 783, 0, 0, 0, 0, 0, 0, 0, 4010, 142, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3048, 709, 356, 2, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[3992, 4062, 4096, 4161, 4152, 4096, 4115, 4094] avg=4096.0000 max=4161 max_rank=3 min=3992 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16457 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1546, 3570, 10, 0, 0, 0, 0, 0, 0, 3358, 0, 178, 0, 0, 0, 0, 4, 5, 4082, 0, 5, 0, 0, 0, 0, 9, 2338, 831, 0, 0, 0, 0, 25, 4, 6, 4088, 145, 0, 0, 0, 6, 1, 0, 6, 4088, 0, 0, 0, 0, 7, 12, 27, 3250, 705, 359, 2, 0, 1, 5, 10, 4085, 0, 0, 0] rerouted_tokens_per_ep_rank=[5126, 3536, 4096, 3178, 4268, 4101, 4362, 4101] avg=4096.0000 max=5126 max_rank=0 min=3178 min_rank=3 imbalance_ratio=1.251465 max_load_reduction=27642 max_load_reduction_pct=0.843567 moved_to_echo=16832 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1169, 3598, 0, 0, 0, 0, 0, 0, 0, 3387, 0, 456, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2905, 790, 0, 0, 0, 0, 0, 0, 0, 3890, 142, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3078, 709, 356, 2, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4767, 3843, 4096, 3695, 4032, 4096, 4145, 4094] avg=4096.0000 max=4767 max_rank=0 min=3695 min_rank=3 imbalance_ratio=1.163818 max_load_reduction=28001 max_load_reduction_pct=0.854523 moved_to_echo=16367 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 26624, 6272, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 28800, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 24192, 5760, 2944, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 27136, 0, 5760, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 31616, 1152, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 4096, 0, 0, 4092, 0, 4, 4095, 0, 0, 0, 4096, 0, 1, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 4096, 0, 7, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[126, 7, 22, 7, 4, 0, 0, 4089, 2, 24, 4077, 0, 11, 3966, 0, 0, 4089, 0, 0, 5, 4073, 0, 0, 3717, 4081, 3, 0, 392, 8, 4058, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 4096, 0, 0, 4092, 0, 1, 4095, 0, 0, 0, 4096, 0, 5, 4096, 4096, 3, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4050, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 46, 0, 7, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4096, 4096, 4105, 4096, 4149, 4050, 4098] avg=4096.0000 max=4149 max_rank=5 min=4050 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4050, 28, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 4, 16, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 46, 0, 1, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4095, 4097, 4112, 4095, 4143, 4050, 4098] avg=4096.0000 max=4143 max_rank=5 min=4050 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4050, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 46, 0, 2, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4096, 4096, 4110, 4096, 4144, 4050, 4098] avg=4096.0000 max=4144 max_rank=5 min=4050 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4050, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4096, 4096, 4112, 4096, 4142, 4050, 4098] avg=4096.0000 max=4142 max_rank=5 min=4050 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4050, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 46, 0, 1, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4096, 4096, 4111, 4096, 4143, 4050, 4098] avg=4096.0000 max=4143 max_rank=5 min=4050 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4050, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4096, 4097, 4111, 4096, 4142, 4050, 4098] avg=4096.0000 max=4142 max_rank=5 min=4050 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[126, 7, 22, 7, 4032, 28, 0, 0, 4, 0, 0, 4089, 5, 3, 0, 0, 2, 24, 4068, 0, 0, 0, 0, 0, 11, 3966, 0, 0, 21, 0, 0, 0, 4089, 0, 0, 5, 4, 0, 0, 0, 41, 0, 0, 3717, 0, 0, 0, 0, 4020, 3, 0, 392, 0, 0, 0, 0, 8, 4058, 7, 0, 9, 0, 0, 0] rerouted_tokens_per_ep_rank=[4222, 4101, 4094, 3998, 4098, 3758, 4415, 4082] avg=4096.0000 max=4415 max_rank=6 min=3758 min_rank=5 imbalance_ratio=1.077881 max_load_reduction=28353 max_load_reduction_pct=0.865265 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4050, 28, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 16, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 46, 0, 5, 4096, 0, 0, 0, 0, 4050, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4092, 4096, 4109, 4095, 4147, 4053, 4098] avg=4096.0000 max=4147 max_rank=5 min=4053 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32384, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 0, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32384, 128, 0, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[384, 0, 128, 32512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 0, 0, 0, 4094, 823, 4, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 3270, 0, 0, 4096, 1, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 5, 0, 0, 0, 0, 0, 4093, 972, 3, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 19, 0, 0, 3103, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 6, 0, 0, 0, 0, 0, 4091, 1231, 2, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 24, 0, 0, 2838, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 933, 4, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 3159, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 0, 0, 0, 4096, 313, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 9, 0, 0, 3774, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1168, 1, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 6, 0, 0, 2922, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4088, 0, 0, 9, 8, 0, 57, 7, 4090, 2410, 2, 0, 0, 0, 0, 8, 4081, 4068, 4058, 8, 6, 0, 1, 15, 1585, 0, 6, 4087, 0, 317, 68, 3789] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 4, 0, 0, 0, 0, 0, 4091, 402, 18, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 15, 0, 0, 3663, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 3059, 1231, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 24, 0, 0, 3039, 1032, 9, 1, 2838, 0, 0, 1057, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4090, 4292, 4096, 4102, 4105, 3895, 4087] avg=4096.0000 max=4292 max_rank=2 min=3895 min_rank=6 imbalance_ratio=1.047852 max_load_reduction=28476 max_load_reduction_pct=0.869019 moved_to_echo=16189 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 3061, 972, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 19, 0, 0, 3039, 1032, 9, 1, 3103, 0, 0, 1057, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4090, 4036, 4096, 4102, 4100, 4160, 4087] avg=4096.0000 max=4160 max_rank=6 min=4036 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 3061, 823, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3039, 1033, 9, 1, 3270, 0, 0, 1057, 0, 0, 0, 0, 1, 0, 0, 4087, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4090, 3888, 4096, 4102, 4082, 4327, 4088] avg=4096.0000 max=4327 max_rank=6 min=3888 min_rank=2 imbalance_ratio=1.056396 max_load_reduction=28441 max_load_reduction_pct=0.867950 moved_to_echo=16599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 3063, 933, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3039, 1033, 9, 1, 3159, 0, 0, 1057, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4090, 4000, 4096, 4102, 4082, 4216, 4087] avg=4096.0000 max=4216 max_rank=6 min=4000 min_rank=2 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 3063, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3039, 1033, 9, 1, 3774, 0, 0, 1057, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4090, 3376, 4096, 4102, 4091, 4831, 4087] avg=4096.0000 max=4831 max_rank=6 min=3376 min_rank=2 imbalance_ratio=1.179443 max_load_reduction=27937 max_load_reduction_pct=0.852570 moved_to_echo=17111 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 3062, 1168, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3039, 1033, 9, 1, 2922, 0, 0, 1057, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4090, 4231, 4096, 4102, 4088, 3979, 4087] avg=4096.0000 max=4231 max_rank=2 min=3979 min_rank=6 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 0, 9, 0, 0, 0, 0, 8, 0, 57, 7, 4066, 0, 0, 0, 3054, 2410, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4081, 2, 0, 0, 0, 0, 4058, 8, 0, 0, 0, 0, 6, 0, 1, 15, 3038, 1036, 16, 5, 1585, 0, 6, 1049, 0, 0, 0, 0, 0, 317, 68, 3773, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4138, 5466, 4091, 4066, 4117, 2640, 4158] avg=4096.0000 max=5466 max_rank=2 min=2640 min_rank=6 imbalance_ratio=1.334473 max_load_reduction=27302 max_load_reduction_pct=0.833191 moved_to_echo=14981 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 3059, 402, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3039, 1032, 9, 1, 3663, 0, 0, 1057, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4090, 3479, 4101, 4097, 4096, 4720, 4087] avg=4096.0000 max=4720 max_rank=6 min=3479 min_rank=2 imbalance_ratio=1.152344 max_load_reduction=28048 max_load_reduction_pct=0.855957 moved_to_echo=17000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32775, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32775, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32775, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32775, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32775, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32775, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32775, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32775, 32761, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32761 actual_valid_recv=33408 observed_loss=-647 tokens_per_expert=[128, 128, 128, 128, 24320, 8320, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[24576, 8320, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[24320, 0, 128, 8448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 384, 128, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32775 actual_valid_recv=33024 observed_loss=-249 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 4092, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[8, 1285, 7, 25, 222, 8, 0, 4076, 6, 2, 8, 8, 4088, 12, 2, 5, 4071, 0, 98, 4088, 8, 0, 4, 4086, 3834, 0, 0, 0, 4, 2728, 0, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 4093, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 3924, 2, 1, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 23, 9, 1, 0, 0, 4094, 0, 172, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3930, 4071, 4095, 4094, 4087, 4096, 4129, 4266] avg=4096.0000 max=4266 max_rank=7 min=3930 min_rank=0 imbalance_ratio=1.041504 max_load_reduction=28502 max_load_reduction_pct=0.869812 moved_to_echo=16578 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 3924, 2, 2, 0, 1, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 23, 9, 1, 0, 0, 4093, 0, 172, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3933, 4072, 4094, 4094, 4087, 4096, 4127, 4265] avg=4096.0000 max=4265 max_rank=7 min=3933 min_rank=0 imbalance_ratio=1.041260 max_load_reduction=28503 max_load_reduction_pct=0.869843 moved_to_echo=16575 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3924, 2, 1, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 23, 9, 1, 0, 0, 4096, 0, 172, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3928, 4071, 4094, 4095, 4087, 4096, 4129, 4268] avg=4096.0000 max=4268 max_rank=7 min=3928 min_rank=0 imbalance_ratio=1.041992 max_load_reduction=28500 max_load_reduction_pct=0.869751 moved_to_echo=16580 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 3924, 2, 1, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 23, 9, 1, 0, 0, 4092, 0, 172, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3934, 4071, 4095, 4095, 4087, 4096, 4126, 4264] avg=4096.0000 max=4264 max_rank=7 min=3934 min_rank=0 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16573 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 3924, 2, 2, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 23, 9, 1, 0, 0, 4096, 0, 172, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3937, 4071, 4094, 4095, 4087, 4096, 4120, 4268] avg=4096.0000 max=4268 max_rank=7 min=3937 min_rank=0 imbalance_ratio=1.041992 max_load_reduction=28500 max_load_reduction_pct=0.869751 moved_to_echo=16571 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3924, 2, 1, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 23, 9, 1, 0, 0, 4095, 0, 172, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3928, 4071, 4095, 4095, 4087, 4096, 4129, 4267] avg=4096.0000 max=4267 max_rank=7 min=3928 min_rank=0 imbalance_ratio=1.041748 max_load_reduction=28501 max_load_reduction_pct=0.869781 moved_to_echo=16579 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 1285, 7, 25, 3915, 7, 2, 0, 222, 8, 0, 4040, 0, 0, 0, 0, 6, 2, 8, 8, 4086, 0, 0, 0, 4086, 12, 2, 5, 0, 0, 0, 0, 4061, 0, 98, 0, 0, 0, 0, 0, 8, 0, 4, 4084, 0, 0, 0, 0, 3834, 0, 0, 0, 29, 10, 2, 2, 4, 2728, 0, 170, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5249, 4270, 4110, 4105, 4159, 4096, 3877, 2902] avg=4096.0000 max=5249 max_rank=0 min=2902 min_rank=7 imbalance_ratio=1.281494 max_load_reduction=27519 max_load_reduction_pct=0.839813 moved_to_echo=15034 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3924, 2, 2, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2, 23, 9, 1, 0, 0, 4096, 0, 172, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3929, 4071, 4091, 4095, 4087, 4096, 4131, 4268] avg=4096.0000 max=4268 max_rank=7 min=3929 min_rank=0 imbalance_ratio=1.041992 max_load_reduction=28500 max_load_reduction_pct=0.869751 moved_to_echo=16582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[128, 1408, 128, 128, 31488, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 31488, 0, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 128, 0, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32512, 0, 0, 128, 256, 128, 128, 128] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4088, 4092, 134, 4046, 0, 0, 22, 4096, 0, 0, 0, 4096, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4095, 4094, 73, 4053, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 183, 4046, 0, 0, 12, 4096, 0, 0, 0, 4092, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4093, 4091, 84, 4052, 0, 0, 3, 4096, 0, 0, 0, 4096, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4093, 4092, 100, 4086, 0, 0, 5, 4096, 0, 0, 1, 4096, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4095, 4096, 171, 3973, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4090, 4090, 557, 1635, 13, 16, 2070, 4087, 6, 0, 0, 4080, 9, 3948, 0, 2, 0, 6, 10, 0, 0, 12, 4004, 0, 0, 0, 4088, 7, 0, 3, 35, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4084, 4090, 309, 3965, 0, 0, 12, 4096, 0, 0, 0, 4096, 0, 3924, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[163, 0, 134, 4046, 0, 0, 0, 0, 0, 0, 22, 3830, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4002, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 163, 0, 0] rerouted_tokens_per_ep_rank=[4343, 3852, 4097, 4094, 4093, 4106, 4096, 4087] avg=4096.0000 max=4343 max_rank=0 min=3852 min_rank=1 imbalance_ratio=1.060303 max_load_reduction=28425 max_load_reduction_pct=0.867462 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[164, 0, 84, 4052, 0, 0, 0, 0, 0, 0, 3, 3830, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4061, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3928, 163, 0, 0] rerouted_tokens_per_ep_rank=[4300, 3833, 4097, 4153, 4092, 4106, 4096, 4091] avg=4096.0000 max=4300 max_rank=0 min=3833 min_rank=1 imbalance_ratio=1.049805 max_load_reduction=28468 max_load_reduction_pct=0.868774 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[164, 0, 100, 4086, 0, 0, 0, 0, 0, 0, 5, 3830, 0, 0, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 0, 4007, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3928, 163, 0, 0] rerouted_tokens_per_ep_rank=[4350, 3835, 4098, 4099, 4093, 4106, 4096, 4091] avg=4096.0000 max=4350 max_rank=0 min=3835 min_rank=1 imbalance_ratio=1.062012 max_load_reduction=28418 max_load_reduction_pct=0.867249 moved_to_echo=16386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[164, 0, 73, 4053, 0, 0, 0, 0, 0, 0, 1, 3830, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4068, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 163, 0, 0] rerouted_tokens_per_ep_rank=[4290, 3831, 4097, 4160, 4095, 4106, 4096, 4093] avg=4096.0000 max=4290 max_rank=0 min=3831 min_rank=1 imbalance_ratio=1.047363 max_load_reduction=28478 max_load_reduction_pct=0.869080 moved_to_echo=16390 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[164, 0, 183, 4046, 0, 0, 0, 0, 0, 0, 12, 3830, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 3955, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 163, 0, 0] rerouted_tokens_per_ep_rank=[4393, 3842, 4093, 4047, 4097, 4106, 4096, 4094] avg=4096.0000 max=4393 max_rank=0 min=3842 min_rank=1 imbalance_ratio=1.072510 max_load_reduction=28375 max_load_reduction_pct=0.865936 moved_to_echo=16393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[164, 0, 171, 3973, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4050, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 163, 0, 0] rerouted_tokens_per_ep_rank=[4308, 3830, 4096, 4142, 4097, 4106, 4096, 4093] avg=4096.0000 max=4308 max_rank=0 min=3830 min_rank=1 imbalance_ratio=1.051758 max_load_reduction=28460 max_load_reduction_pct=0.868530 moved_to_echo=16392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[155, 0, 557, 1635, 0, 0, 0, 0, 13, 16, 2070, 3804, 0, 0, 0, 0, 6, 0, 0, 4080, 7, 0, 0, 0, 9, 3948, 0, 2, 98, 0, 0, 0, 0, 6, 10, 0, 4090, 4, 0, 0, 0, 12, 4004, 0, 10, 0, 0, 0, 0, 0, 4088, 7, 1, 0, 0, 0, 0, 3, 35, 0, 3931, 167, 0, 0] rerouted_tokens_per_ep_rank=[2347, 5903, 4093, 4057, 4110, 4026, 4096, 4136] avg=4096.0000 max=5903 max_rank=1 min=2347 min_rank=0 imbalance_ratio=1.441162 max_load_reduction=26865 max_load_reduction_pct=0.819855 moved_to_echo=16368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[163, 0, 309, 3965, 0, 0, 0, 0, 0, 0, 12, 3830, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 3924, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 163, 0, 0] rerouted_tokens_per_ep_rank=[4437, 3842, 4097, 4016, 4091, 4106, 4096, 4083] avg=4096.0000 max=4437 max_rank=0 min=3842 min_rank=1 imbalance_ratio=1.083252 max_load_reduction=28331 max_load_reduction_pct=0.864594 moved_to_echo=16376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1408, 0, 1664, 29952, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 2176, 30720, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 31488, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32128, 0, 128, 768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True + [2026-08-30 04:55:27] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 669.2 | throughput per GPU (TFLOP/s/GPU): 498.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.090190E+00 | loss scale: 1.0 | grad norm: 5.516 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1404, 0, 2004, 246, 792, 0, 1890, 138, 1134, 2606, 0, 55, 1663, 2093, 235, 1310, 38, 1379, 1020, 0, 0, 1050, 2364, 1037, 2, 1424, 2863, 1549, 1704, 2544, 224, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1739, 0, 2215, 231, 909, 0, 2025, 243, 1247, 2497, 0, 28, 1379, 1951, 183, 1535, 40, 1823, 963, 1, 0, 1402, 1743, 379, 2, 1478, 2683, 1229, 2235, 2248, 360, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1948, 1, 1283, 423, 965, 0, 1514, 338, 1137, 1720, 1, 52, 1128, 2411, 187, 1624, 65, 1656, 1511, 0, 0, 1558, 2169, 707, 0, 1992, 2663, 1655, 1121, 2691, 245, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1480, 0, 1278, 168, 766, 0, 1801, 223, 1888, 2603, 0, 40, 970, 2425, 64, 1528, 100, 2192, 587, 0, 0, 1939, 2339, 1171, 0, 1811, 2306, 1144, 1159, 2569, 217, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2006, 0, 1837, 275, 948, 0, 2173, 138, 1838, 1856, 0, 44, 1159, 2113, 201, 1755, 65, 1575, 1347, 0, 0, 1954, 1353, 892, 0, 1948, 2264, 1236, 849, 2660, 282, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1445, 0, 1724, 408, 903, 7, 1621, 217, 1419, 1721, 1, 28, 1582, 1636, 152, 1952, 59, 1552, 1596, 0, 1, 1442, 2143, 1059, 0, 1932, 2459, 1334, 1473, 2609, 293, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1767, 0, 1497, 151, 1257, 1, 2303, 260, 1153, 1584, 0, 99, 1383, 1511, 232, 1050, 40, 1732, 1726, 1, 1, 1204, 2342, 833, 1, 2152, 2791, 1647, 1522, 2218, 309, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1523, 0, 1499, 226, 702, 0, 1480, 282, 654, 1985, 0, 117, 1821, 1940, 177, 1677, 60, 1589, 1951, 0, 0, 984, 2224, 785, 0, 2175, 2315, 1380, 2532, 2434, 254, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1404, 0, 2004, 246, 283, 253, 0, 0, 792, 0, 1890, 138, 1248, 0, 0, 0, 1134, 2606, 0, 55, 644, 43, 0, 0, 1663, 1019, 235, 1310, 0, 0, 0, 0, 38, 1379, 1020, 0, 674, 430, 0, 0, 0, 1050, 2038, 1037, 0, 0, 0, 0, 2, 1424, 941, 1549, 0, 0, 0, 0, 1704, 2291, 224, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4068, 4482, 4227, 3541, 4125, 3916, 4219] avg=4096.0000 max=4482 max_rank=2 min=3541 min_rank=4 imbalance_ratio=1.094238 max_load_reduction=28286 max_load_reduction_pct=0.863220 moved_to_echo=15801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1948, 1, 1283, 423, 260, 268, 0, 0, 965, 0, 1514, 338, 1161, 0, 0, 0, 1137, 1720, 1, 52, 742, 39, 0, 0, 1128, 1174, 187, 1624, 0, 0, 0, 0, 65, 1656, 1511, 0, 627, 495, 0, 0, 0, 1558, 1870, 707, 0, 0, 0, 0, 0, 1992, 875, 1655, 0, 0, 0, 0, 1121, 2423, 245, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4183, 3978, 3691, 4113, 4354, 4135, 4522, 3792] avg=4096.0000 max=4522 max_rank=6 min=3691 min_rank=2 imbalance_ratio=1.104004 max_load_reduction=28246 max_load_reduction_pct=0.862000 moved_to_echo=16803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1739, 0, 2215, 231, 209, 223, 0, 0, 909, 0, 2025, 243, 1170, 0, 0, 0, 1247, 2497, 0, 28, 601, 32, 0, 0, 1379, 950, 183, 1535, 0, 0, 0, 0, 40, 1823, 963, 1, 632, 400, 0, 0, 0, 1402, 1502, 379, 0, 0, 0, 0, 2, 1478, 881, 1229, 0, 0, 0, 0, 2235, 2025, 360, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4617, 4347, 4405, 4047, 3859, 3283, 3590, 4620] avg=4096.0000 max=4620 max_rank=7 min=3283 min_rank=5 imbalance_ratio=1.127930 max_load_reduction=28148 max_load_reduction_pct=0.859009 moved_to_echo=15352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1480, 0, 1278, 168, 280, 255, 0, 0, 766, 0, 1801, 223, 1005, 0, 0, 0, 1888, 2603, 0, 40, 747, 43, 0, 0, 970, 1180, 64, 1528, 0, 0, 0, 0, 100, 2192, 587, 0, 543, 498, 0, 0, 0, 1939, 2016, 1171, 0, 0, 0, 0, 0, 1811, 758, 1144, 0, 0, 0, 0, 1159, 2314, 217, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3461, 3795, 5321, 3742, 3920, 5126, 3713, 3690] avg=4096.0000 max=5321 max_rank=2 min=3461 min_rank=0 imbalance_ratio=1.299072 max_load_reduction=27447 max_load_reduction_pct=0.837616 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2006, 0, 1837, 275, 162, 265, 0, 0, 948, 0, 2173, 138, 987, 0, 0, 0, 1838, 1856, 0, 44, 650, 24, 0, 0, 1159, 1029, 201, 1755, 0, 0, 0, 0, 65, 1575, 1347, 0, 533, 434, 0, 0, 0, 1954, 1167, 892, 0, 0, 0, 0, 0, 1948, 744, 1236, 0, 0, 0, 0, 849, 2395, 282, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4545, 4246, 4412, 4144, 3954, 4013, 3928, 3526] avg=4096.0000 max=4545 max_rank=0 min=3526 min_rank=7 imbalance_ratio=1.109619 max_load_reduction=28223 max_load_reduction_pct=0.861298 moved_to_echo=15421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1445, 0, 1724, 408, 260, 264, 0, 0, 903, 7, 1621, 217, 1075, 0, 0, 0, 1419, 1721, 1, 28, 508, 43, 0, 0, 1582, 788, 152, 1952, 0, 0, 0, 0, 59, 1552, 1596, 0, 584, 340, 0, 0, 1, 1442, 1840, 1059, 0, 0, 0, 0, 0, 1932, 800, 1334, 0, 0, 0, 0, 1473, 2345, 293, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 3823, 3720, 4474, 4131, 4342, 4066, 4111] avg=4096.0000 max=4474 max_rank=3 min=3720 min_rank=2 imbalance_ratio=1.092285 max_load_reduction=28294 max_load_reduction_pct=0.863464 moved_to_echo=16650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1523, 0, 1499, 226, 266, 242, 0, 0, 702, 0, 1480, 282, 1009, 0, 0, 0, 654, 1985, 0, 117, 597, 40, 0, 0, 1821, 945, 177, 1677, 0, 0, 0, 0, 60, 1589, 1951, 0, 545, 398, 0, 0, 0, 984, 1918, 785, 0, 0, 0, 0, 0, 2175, 761, 1380, 0, 0, 0, 0, 2532, 2192, 254, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3756, 3473, 3393, 4620, 4543, 3687, 4316, 4980] avg=4096.0000 max=4980 max_rank=7 min=3393 min_rank=2 imbalance_ratio=1.215820 max_load_reduction=27788 max_load_reduction_pct=0.848022 moved_to_echo=17526 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1767, 0, 1497, 151, 280, 220, 0, 0, 1257, 1, 2303, 260, 1217, 0, 0, 0, 1153, 1584, 0, 99, 465, 43, 0, 0, 1383, 736, 232, 1050, 0, 0, 0, 0, 40, 1732, 1726, 1, 657, 310, 0, 0, 1, 1204, 2019, 833, 0, 0, 0, 0, 1, 2152, 917, 1647, 0, 0, 0, 0, 1522, 1998, 309, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3915, 5038, 3344, 3401, 4466, 4057, 4717, 3830] avg=4096.0000 max=5038 max_rank=1 min=3344 min_rank=2 imbalance_ratio=1.229980 max_load_reduction=27730 max_load_reduction_pct=0.846252 moved_to_echo=17070 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7296, 128, 14848, 1920, 8960, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 14976, 6784, 11264, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[12672, 18048, 2304, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13312, 128, 13440, 2176, 2048, 2048, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 11648, 14464, 6912, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10496, 16640, 128, 512, 4992, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11136, 7936, 1536, 12544, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 13568, 10752, 128, 4864, 3328, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[77, 1, 1, 0, 1361, 1, 0, 0, 1, 4093, 4075, 3835, 3994, 0, 23, 2660, 4095, 0, 0, 1, 491, 1, 1, 0, 4094, 4, 0, 0, 0, 3958, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[207, 1, 0, 0, 1169, 2, 0, 0, 1, 4015, 4070, 4045, 4028, 0, 156, 2606, 4094, 2, 0, 1, 409, 0, 2, 0, 4093, 18, 1, 1, 0, 3845, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 270, 0, 0, 0, 0, 4075, 4056, 4067, 4046, 1, 31, 3534, 4096, 0, 0, 0, 493, 0, 0, 0, 4095, 82, 0, 0, 0, 3914, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[346, 0, 0, 0, 2913, 0, 0, 0, 0, 3961, 4093, 4070, 4018, 0, 142, 917, 4093, 0, 0, 0, 148, 0, 0, 0, 4095, 3, 0, 1, 0, 3968, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[273, 0, 0, 0, 1559, 0, 0, 0, 0, 3969, 4007, 3997, 3738, 0, 26, 2795, 4096, 0, 0, 0, 167, 0, 0, 0, 4094, 4, 0, 0, 0, 4043, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[47, 1, 0, 0, 597, 1, 0, 0, 2, 4052, 4012, 3916, 4063, 9, 127, 2641, 4094, 0, 1, 1, 1103, 0, 0, 0, 4091, 43, 0, 0, 0, 3965, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 494, 0, 0, 0, 0, 3995, 4082, 4086, 4036, 4, 68, 3446, 4094, 0, 0, 0, 339, 0, 0, 7, 4095, 4, 0, 0, 0, 3993, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[243, 0, 0, 0, 1662, 0, 0, 0, 0, 3975, 4085, 4071, 4096, 0, 267, 1728, 4096, 0, 0, 0, 455, 0, 0, 0, 4095, 0, 0, 0, 0, 3995, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[77, 1, 1, 0, 3957, 0, 0, 0, 1361, 1, 0, 0, 2431, 413, 0, 0, 1, 87, 0, 3835, 0, 0, 0, 0, 1446, 0, 23, 2660, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 491, 1, 1, 0, 3593, 118, 0, 0, 4077, 4, 0, 0, 0, 0, 0, 0, 0, 3958, 1, 0, 117, 17, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4206, 3923, 4129, 4096, 4204, 4081, 4093] avg=4096.0000 max=4206 max_rank=1 min=3923 min_rank=2 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[346, 0, 0, 0, 3975, 0, 0, 0, 2913, 0, 0, 0, 2445, 400, 0, 0, 0, 84, 0, 4070, 0, 0, 0, 0, 1455, 0, 142, 917, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0, 0, 3477, 118, 0, 0, 4078, 3, 0, 1, 0, 0, 0, 0, 0, 3968, 0, 0, 118, 17, 0, 0] rerouted_tokens_per_ep_rank=[4321, 5758, 4154, 2514, 4093, 3743, 4082, 4103] avg=4096.0000 max=5758 max_rank=1 min=2514 min_rank=3 imbalance_ratio=1.405762 max_load_reduction=27010 max_load_reduction_pct=0.824280 moved_to_echo=16021 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[207, 1, 0, 0, 3951, 0, 0, 0, 1169, 2, 0, 0, 2451, 405, 0, 0, 1, 86, 0, 4045, 0, 0, 0, 0, 1459, 0, 156, 2606, 0, 0, 0, 0, 4094, 2, 0, 1, 0, 0, 0, 0, 409, 0, 2, 0, 3524, 119, 0, 0, 4076, 18, 1, 1, 0, 0, 0, 0, 0, 3845, 2, 0, 118, 17, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4027, 4132, 4221, 4097, 4054, 4096, 3982] avg=4096.0000 max=4221 max_rank=3 min=3982 min_rank=7 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 3939, 0, 0, 0, 270, 0, 0, 0, 2462, 411, 0, 0, 0, 87, 0, 4067, 0, 0, 0, 0, 1466, 1, 31, 3534, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 493, 0, 0, 0, 3577, 117, 0, 0, 4078, 82, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 118, 17, 0, 0] rerouted_tokens_per_ep_rank=[3947, 3143, 4154, 5032, 4096, 4187, 4160, 4049] avg=4096.0000 max=5032 max_rank=3 min=3143 min_rank=1 imbalance_ratio=1.228516 max_load_reduction=27736 max_load_reduction_pct=0.846436 moved_to_echo=16492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[273, 0, 0, 0, 3890, 0, 0, 0, 1559, 0, 0, 0, 2275, 401, 0, 0, 0, 84, 0, 3997, 0, 0, 0, 0, 1354, 0, 26, 2795, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 3484, 117, 0, 0, 4077, 4, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 109, 17, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4235, 4081, 4175, 4096, 3768, 4081, 4169] avg=4096.0000 max=4235 max_rank=1 min=3768 min_rank=5 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16114 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[243, 0, 0, 0, 3966, 0, 0, 0, 1662, 0, 0, 0, 2493, 401, 0, 0, 0, 85, 0, 4071, 0, 0, 0, 0, 1483, 0, 267, 1728, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 0, 3489, 119, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 120, 17, 0, 0] rerouted_tokens_per_ep_rank=[4209, 4556, 4156, 3478, 4096, 4063, 4078, 4132] avg=4096.0000 max=4556 max_rank=1 min=3478 min_rank=3 imbalance_ratio=1.112305 max_load_reduction=28212 max_load_reduction_pct=0.860962 moved_to_echo=16369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[47, 1, 0, 0, 3896, 0, 0, 0, 597, 1, 0, 0, 2473, 409, 0, 0, 2, 86, 0, 3916, 0, 0, 0, 0, 1471, 9, 127, 2641, 0, 0, 0, 0, 4094, 0, 1, 1, 0, 0, 0, 0, 1103, 0, 0, 0, 3557, 116, 0, 0, 4074, 43, 0, 0, 0, 0, 0, 0, 0, 3965, 1, 1, 119, 17, 0, 0] rerouted_tokens_per_ep_rank=[3944, 3480, 4004, 4248, 4096, 4776, 4117, 4103] avg=4096.0000 max=4776 max_rank=5 min=3480 min_rank=1 imbalance_ratio=1.166016 max_load_reduction=27992 max_load_reduction_pct=0.854248 moved_to_echo=17092 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 3964, 0, 0, 0, 494, 0, 0, 0, 2461, 408, 0, 0, 0, 78, 0, 4086, 0, 0, 0, 0, 1453, 4, 68, 3446, 0, 0, 0, 0, 4094, 0, 0, 0, 4, 0, 0, 0, 339, 0, 0, 7, 3509, 118, 0, 0, 4069, 4, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 122, 22, 0, 0] rerouted_tokens_per_ep_rank=[3989, 3363, 4164, 4971, 4098, 3973, 4073, 4137] avg=4096.0000 max=4971 max_rank=3 min=3363 min_rank=1 imbalance_ratio=1.213623 max_load_reduction=27797 max_load_reduction_pct=0.848297 moved_to_echo=16281 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10112, 128, 0, 0, 19584, 3328, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11648, 128, 896, 20352, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3712, 128, 128, 128, 28288, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1280, 128, 128, 0, 31616, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 31744, 128, 128, 1024, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 768, 0, 32128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 100, 0, 2905, 0, 4096, 15, 1556, 4030, 4045, 0, 4095, 0, 1301, 13, 0, 0, 0, 0, 0, 118, 4096, 0, 0, 4086, 0, 0, 0, 2310, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 22, 0, 3765, 0, 4096, 0, 1744, 3936, 4027, 0, 4096, 0, 1516, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 4062, 0, 0, 0, 1393, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 13, 1, 3486, 0, 4095, 1, 902, 4003, 3994, 0, 4095, 1, 1646, 2, 1, 1, 0, 2, 1, 47, 4095, 0, 7, 4041, 0, 1, 0, 2313, 1, 19] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 47, 1, 3278, 0, 4094, 0, 1755, 4041, 4026, 0, 4094, 1, 1899, 0, 1, 2, 0, 2, 2, 22, 4094, 0, 10, 4074, 0, 7, 3, 1270, 0, 44] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 161, 0, 3201, 0, 4096, 0, 629, 4005, 3480, 0, 4096, 0, 2182, 0, 0, 0, 0, 0, 0, 53, 4096, 1, 0, 4088, 0, 0, 0, 2639, 0, 41] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[9, 9, 63, 0, 3234, 0, 4096, 0, 599, 4072, 3884, 0, 4088, 0, 2591, 7, 0, 0, 0, 0, 0, 84, 4096, 2, 0, 4059, 0, 0, 0, 1865, 0, 10] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 32, 1, 3115, 0, 4095, 0, 1098, 3985, 3930, 0, 4095, 1, 2284, 0, 1, 1, 0, 2, 1, 47, 4095, 22, 2, 4066, 0, 2, 0, 1891, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 174, 0, 2695, 0, 4096, 0, 1449, 4072, 4044, 0, 4096, 0, 1942, 0, 0, 0, 0, 0, 0, 119, 4096, 1, 0, 4091, 0, 0, 0, 1893, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 100, 0, 3135, 881, 0, 0, 2905, 0, 885, 15, 0, 0, 0, 0, 1556, 0, 2965, 0, 0, 0, 0, 0, 2174, 0, 1301, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 76, 0, 0, 0, 118, 4031, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 21, 0, 0, 0, 0, 2310, 0, 1, 1080, 1040, 44, 0] rerouted_tokens_per_ep_rank=[4117, 3805, 4521, 3488, 4106, 4149, 4107, 4475] avg=4096.0000 max=4521 max_rank=2 min=3488 min_rank=3 imbalance_ratio=1.103760 max_load_reduction=28247 max_load_reduction_pct=0.862030 moved_to_echo=16837 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 1, 3134, 881, 0, 0, 3486, 0, 885, 1, 0, 0, 0, 0, 902, 0, 2928, 0, 0, 0, 0, 0, 2174, 1, 1646, 2, 0, 0, 0, 0, 1, 1, 0, 2, 4003, 76, 0, 0, 1, 47, 4030, 0, 0, 0, 0, 0, 7, 4041, 0, 1, 21, 0, 0, 0, 0, 2313, 1, 19, 1066, 1040, 44, 0] rerouted_tokens_per_ep_rank=[4029, 4372, 3830, 3823, 4083, 4078, 4070, 4483] avg=4096.0000 max=4483 max_rank=7 min=3823 min_rank=3 imbalance_ratio=1.094482 max_load_reduction=28285 max_load_reduction_pct=0.863190 moved_to_echo=16714 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 47, 1, 3133, 881, 0, 0, 3278, 0, 885, 0, 0, 0, 0, 0, 1755, 0, 2951, 0, 0, 0, 0, 0, 2174, 1, 1899, 0, 0, 0, 0, 0, 1, 2, 0, 2, 4041, 76, 0, 0, 2, 22, 4029, 0, 0, 0, 0, 0, 10, 4074, 0, 7, 21, 0, 0, 0, 3, 1270, 0, 44, 1075, 1039, 44, 0] rerouted_tokens_per_ep_rank=[4063, 4163, 4706, 4074, 4122, 4053, 4112, 3475] avg=4096.0000 max=4706 max_rank=2 min=3475 min_rank=7 imbalance_ratio=1.148926 max_load_reduction=28062 max_load_reduction_pct=0.856384 moved_to_echo=15762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 22, 0, 3135, 881, 0, 0, 3765, 0, 885, 0, 0, 0, 0, 0, 1744, 0, 2952, 0, 0, 0, 0, 0, 2175, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 76, 0, 0, 0, 15, 4031, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 21, 0, 0, 0, 0, 1393, 0, 0, 1075, 1040, 44, 0] rerouted_tokens_per_ep_rank=[4038, 4650, 4696, 3691, 4012, 4046, 4083, 3552] avg=4096.0000 max=4696 max_rank=2 min=3552 min_rank=7 imbalance_ratio=1.146484 max_load_reduction=28072 max_load_reduction_pct=0.856689 moved_to_echo=15693 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 161, 0, 3135, 881, 0, 0, 3201, 0, 885, 0, 0, 0, 0, 0, 629, 0, 2551, 0, 0, 0, 0, 0, 2175, 0, 2182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 76, 0, 0, 0, 53, 4031, 1, 0, 0, 0, 0, 0, 4088, 0, 0, 21, 0, 0, 0, 0, 2639, 0, 41, 929, 1040, 44, 0] rerouted_tokens_per_ep_rank=[4177, 4086, 3180, 4357, 4081, 4085, 4109, 4693] avg=4096.0000 max=4693 max_rank=7 min=3180 min_rank=2 imbalance_ratio=1.145752 max_load_reduction=28075 max_load_reduction_pct=0.856781 moved_to_echo=16968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[9, 9, 63, 0, 3139, 886, 0, 0, 3234, 0, 878, 0, 0, 0, 0, 0, 599, 0, 2844, 0, 0, 0, 0, 0, 2161, 0, 2591, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 79, 0, 0, 0, 84, 4020, 2, 0, 0, 0, 0, 0, 4059, 0, 0, 25, 0, 0, 0, 0, 1865, 0, 10, 1040, 1041, 51, 0] rerouted_tokens_per_ep_rank=[4106, 4112, 3443, 4759, 4151, 4106, 4084, 4007] avg=4096.0000 max=4759 max_rank=3 min=3443 min_rank=2 imbalance_ratio=1.161865 max_load_reduction=28009 max_load_reduction_pct=0.854767 moved_to_echo=16348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 174, 0, 3135, 881, 0, 0, 2695, 0, 885, 0, 0, 0, 0, 0, 1449, 0, 2964, 0, 0, 0, 0, 0, 2175, 0, 1942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 76, 0, 0, 0, 119, 4031, 1, 0, 0, 0, 0, 0, 4091, 0, 0, 21, 0, 0, 0, 0, 1893, 0, 0, 1080, 1040, 44, 0] rerouted_tokens_per_ep_rank=[4190, 3580, 4413, 4117, 4148, 4151, 4112, 4057] avg=4096.0000 max=4413 max_rank=2 min=3580 min_rank=1 imbalance_ratio=1.077393 max_load_reduction=28355 max_load_reduction_pct=0.865326 moved_to_echo=16468 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 32, 1, 3134, 881, 0, 0, 3115, 0, 885, 0, 0, 0, 0, 0, 1098, 0, 2881, 0, 0, 0, 0, 0, 2174, 1, 2284, 0, 0, 0, 0, 0, 1, 1, 0, 2, 3985, 76, 0, 0, 1, 47, 4030, 22, 0, 0, 0, 0, 2, 4066, 0, 2, 21, 0, 0, 0, 0, 1891, 1, 1, 1049, 1040, 44, 0] rerouted_tokens_per_ep_rank=[4048, 4000, 3979, 4459, 4065, 4100, 4091, 4026] avg=4096.0000 max=4459 max_rank=3 min=3979 min_rank=2 imbalance_ratio=1.088623 max_load_reduction=28309 max_load_reduction_pct=0.863922 moved_to_echo=16282 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[17408, 128, 15488, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 15616, 128, 128, 8448, 8320, 384, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[25728, 0, 7168, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[9856, 0, 23040, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 640, 128, 25088, 7168, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32256, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 512, 32256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 0, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[3, 5, 0, 2, 2, 3305, 4, 750, 0, 4092, 1, 0, 2, 0, 1, 0, 4090, 2074, 2, 4094, 0, 0, 0, 2547, 6, 0, 3820, 3874, 0, 2, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2917, 0, 1102, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1436, 0, 4028, 38, 0, 1, 3381, 16, 22, 3849, 3677, 0, 0, 13, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 2200, 0, 1644, 0, 4078, 1, 0, 1, 0, 1, 0, 4094, 2310, 1, 4080, 0, 1, 0, 3044, 21, 5, 3796, 3396, 0, 1, 1, 4090] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3073, 0, 270, 0, 4085, 0, 0, 0, 0, 0, 0, 4096, 1836, 0, 4092, 0, 0, 0, 3573, 5, 3, 3884, 3754, 2, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2584, 0, 262, 0, 4078, 0, 0, 0, 0, 0, 0, 4096, 2026, 0, 4091, 11, 0, 0, 3769, 1, 12, 3896, 3838, 7, 0, 1, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3923, 0, 913, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 322, 0, 4094, 2, 0, 0, 3746, 0, 0, 3626, 3853, 2, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[5, 0, 0, 1, 0, 3034, 1, 1012, 0, 3968, 1, 0, 1, 0, 1, 0, 4094, 1722, 1, 4030, 0, 1, 0, 3109, 70, 0, 3886, 3729, 0, 1, 8, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3284, 0, 914, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 1211, 0, 4081, 0, 0, 0, 3702, 3, 2, 3426, 3871, 1, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3558, 650, 0, 0, 0, 2584, 0, 262, 197, 0, 0, 0, 0, 4078, 0, 0, 20, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2026, 0, 2491, 0, 0, 0, 0, 11, 0, 0, 3769, 733, 0, 0, 0, 1, 12, 338, 3838, 0, 0, 0, 0, 7, 0, 1, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4208, 3043, 4101, 4096, 4517, 4513, 4189, 4101] avg=4096.0000 max=4517 max_rank=4 min=3043 min_rank=1 imbalance_ratio=1.102783 max_load_reduction=28251 max_load_reduction_pct=0.862152 moved_to_echo=17320 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 5, 0, 2, 3489, 650, 0, 0, 2, 3305, 4, 750, 197, 0, 0, 0, 0, 4092, 1, 0, 20, 2, 0, 0, 2, 0, 1, 0, 4090, 0, 0, 0, 0, 2074, 2, 2493, 0, 0, 0, 0, 0, 0, 0, 2547, 734, 0, 0, 0, 6, 0, 331, 3874, 0, 0, 0, 0, 0, 2, 0, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4258, 4115, 4093, 4569, 3281, 4211, 4092] avg=4096.0000 max=4569 max_rank=4 min=3281 min_rank=5 imbalance_ratio=1.115479 max_load_reduction=28199 max_load_reduction_pct=0.860565 moved_to_echo=16153 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3547, 650, 0, 0, 0, 3073, 0, 270, 197, 0, 0, 0, 0, 4085, 0, 0, 20, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1836, 0, 2492, 0, 0, 0, 0, 0, 0, 0, 3573, 733, 0, 0, 0, 5, 3, 337, 3754, 0, 0, 0, 0, 2, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4197, 3540, 4108, 4096, 4328, 4306, 4099, 4094] avg=4096.0000 max=4328 max_rank=4 min=3540 min_rank=1 imbalance_ratio=1.056641 max_load_reduction=28440 max_load_reduction_pct=0.867920 moved_to_echo=16827 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3519, 643, 0, 0, 0, 2917, 0, 1102, 198, 0, 0, 0, 0, 4096, 0, 0, 27, 6, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 1436, 0, 2435, 0, 0, 0, 0, 38, 0, 1, 3381, 725, 0, 0, 0, 16, 22, 327, 3677, 0, 0, 0, 0, 0, 0, 13, 4090, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 4217, 4129, 4099, 3871, 4145, 4042, 4103] avg=4096.0000 max=4217 max_rank=1 min=3871 min_rank=4 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16161 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3312, 650, 0, 0, 0, 3923, 0, 913, 197, 0, 0, 0, 0, 4095, 0, 0, 20, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 322, 0, 2493, 0, 0, 0, 0, 2, 0, 0, 3746, 734, 0, 0, 0, 0, 0, 314, 3853, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3962, 5033, 4118, 4096, 2815, 4482, 4167, 4095] avg=4096.0000 max=5033 max_rank=1 min=2815 min_rank=4 imbalance_ratio=1.228760 max_load_reduction=27735 max_load_reduction_pct=0.846405 moved_to_echo=15559 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3129, 648, 0, 0, 0, 3284, 0, 914, 197, 0, 0, 0, 0, 4082, 0, 0, 20, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1211, 0, 2485, 0, 0, 0, 0, 0, 0, 0, 3702, 731, 0, 0, 0, 3, 2, 297, 3871, 0, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3777, 4395, 4105, 4096, 3696, 4433, 4173, 4093] avg=4096.0000 max=4433 max_rank=5 min=3696 min_rank=4 imbalance_ratio=1.082275 max_load_reduction=28335 max_load_reduction_pct=0.864716 moved_to_echo=16395 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 1, 3549, 640, 0, 0, 0, 3034, 1, 1012, 194, 0, 0, 0, 0, 3968, 1, 0, 20, 2, 0, 0, 1, 0, 1, 0, 4094, 0, 0, 0, 0, 1722, 1, 2454, 0, 0, 0, 0, 0, 1, 0, 3109, 722, 0, 0, 0, 70, 0, 337, 3729, 0, 0, 0, 0, 0, 1, 8, 4091, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4195, 4241, 3991, 4096, 4177, 3832, 4136, 4100] avg=4096.0000 max=4241 max_rank=1 min=3832 min_rank=5 imbalance_ratio=1.035400 max_load_reduction=28527 max_load_reduction_pct=0.870575 moved_to_echo=16245 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 3467, 648, 0, 0, 0, 2200, 0, 1644, 197, 0, 0, 0, 0, 4078, 1, 0, 20, 2, 0, 0, 1, 0, 1, 0, 4094, 0, 0, 0, 0, 2310, 1, 2484, 0, 0, 0, 0, 0, 1, 0, 3044, 731, 0, 0, 0, 21, 5, 329, 3396, 0, 0, 0, 0, 0, 1, 1, 4088, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4041, 4101, 4096, 4795, 3776, 3751, 4090] avg=4096.0000 max=4795 max_rank=4 min=3751 min_rank=6 imbalance_ratio=1.170654 max_load_reduction=27973 max_load_reduction_pct=0.853668 moved_to_echo=16412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 24320, 128, 6912, 1664, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 27648, 5248, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 26880, 5888, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 13056, 128, 19840, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 2688, 30080, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32640, 128, 0, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 2245, 1903, 4096, 0, 0, 8, 0, 4095, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3983, 0, 1, 4088, 0, 4067, 0, 0, 88] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2246, 1871, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4079, 0, 4087, 0, 0, 15] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 809, 3378, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 4092, 0, 4081, 0, 0, 52] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1407, 2754, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 4049, 0, 4068, 0, 0, 30] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[2, 2, 2, 4094, 800, 3434, 4094, 0, 2, 1, 0, 4094, 2, 1, 0, 4094, 1, 0, 0, 0, 2, 15, 0, 3942, 0, 0, 4076, 1, 4088, 2, 0, 19] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 488, 3591, 4096, 0, 0, 26, 0, 4075, 0, 0, 0, 4095, 11, 0, 0, 0, 0, 0, 0, 4065, 0, 22, 4088, 0, 3978, 0, 0, 137] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 708, 3423, 4095, 0, 1, 3, 0, 4095, 1, 0, 1, 4095, 0, 0, 0, 0, 1, 1, 0, 4062, 0, 0, 4091, 0, 4030, 1, 0, 62] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 931, 3238, 4095, 1, 1, 1, 0, 4095, 1, 0, 0, 4095, 7, 0, 0, 0, 1, 2, 0, 4033, 0, 0, 4088, 1, 4034, 1, 0, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 2245, 1866, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3983, 37, 6, 4, 0, 0, 1, 4088, 0, 8, 2, 0, 0, 4053, 0, 0, 88, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4111, 4101, 4096, 4095, 4030, 4099, 4141] avg=4096.0000 max=4141 max_rank=7 min=4030 min_rank=5 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 809, 3313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3966, 65, 6, 4, 0, 0, 0, 4092, 0, 8, 2, 0, 0, 4067, 0, 0, 52, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4122, 4094, 4096, 4098, 4041, 4102, 4119] avg=4096.0000 max=4122 max_rank=1 min=4041 min_rank=5 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16360 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 2, 4094, 0, 0, 0, 0, 800, 3368, 0, 0, 0, 0, 0, 0, 2, 1, 0, 4092, 0, 0, 0, 0, 2, 1, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 4090, 0, 0, 0, 2, 15, 0, 3942, 66, 6, 4, 0, 0, 0, 4076, 1, 8, 2, 0, 0, 4074, 2, 0, 19, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4168, 4095, 4097, 4091, 4035, 4087, 4095] avg=4096.0000 max=4168 max_rank=1 min=4035 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16308 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2246, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4080, 36, 6, 4, 0, 0, 0, 4079, 0, 8, 2, 0, 0, 4073, 0, 0, 15, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4081, 4094, 4096, 4098, 4126, 4089, 4088] avg=4096.0000 max=4126 max_rank=5 min=4081 min_rank=1 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 1407, 2701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4077, 53, 6, 4, 0, 0, 0, 4049, 0, 8, 2, 0, 0, 4054, 0, 0, 30, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4108, 4094, 4096, 4092, 4140, 4059, 4084] avg=4096.0000 max=4140 max_rank=5 min=4059 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 0, 0, 0, 488, 3517, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 11, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4065, 74, 7, 3, 0, 0, 22, 4088, 0, 12, 3, 6, 1, 3959, 0, 0, 137, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4005, 4098, 4094, 4104, 4149, 4132, 4096] avg=4096.0000 max=4149 max_rank=5 min=4005 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 0, 0, 0, 0, 931, 3176, 0, 1, 0, 0, 0, 0, 1, 1, 0, 4093, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 7, 0, 0, 0, 4091, 0, 0, 0, 1, 2, 0, 4033, 62, 6, 4, 0, 0, 0, 4088, 1, 8, 2, 0, 0, 4020, 1, 0, 45, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4108, 4095, 4096, 4098, 4108, 4099, 4066] avg=4096.0000 max=4108 max_rank=1 min=4066 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=28660 max_load_reduction_pct=0.874634 moved_to_echo=16371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 0, 0, 0, 0, 708, 3357, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4093, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 1, 1, 0, 4062, 66, 6, 3, 0, 0, 0, 4091, 0, 8, 2, 0, 0, 4016, 1, 0, 62, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4065, 4097, 4097, 4092, 4139, 4101, 4079] avg=4096.0000 max=4139 max_rank=5 min=4065 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16411 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9728, 23168, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32384, 128, 0, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32256, 512, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 3723, 0, 0, 593, 4095, 0, 1, 1, 1517, 0, 1, 0, 1303, 2718, 3607, 1, 753, 4092, 1, 4095, 2101, 1, 4095, 0, 0, 4, 64, 0, 1, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 3862, 0, 0, 95, 4096, 0, 0, 0, 857, 0, 0, 0, 1428, 3834, 4029, 0, 1228, 4085, 0, 4096, 888, 0, 4096, 0, 0, 0, 174, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 3669, 1, 0, 778, 4094, 1, 2, 2, 602, 0, 2, 0, 828, 3726, 3423, 0, 656, 4096, 2, 4094, 2595, 2, 4094, 2, 0, 16, 81, 0, 2, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 3593, 0, 1, 704, 4078, 0, 0, 0, 677, 0, 0, 0, 1727, 3112, 3499, 0, 936, 4084, 0, 4096, 2154, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 3565, 0, 0, 189, 4079, 0, 0, 0, 1076, 0, 0, 0, 1339, 3763, 3811, 0, 502, 4071, 0, 4096, 2146, 0, 4096, 0, 0, 1, 34, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 3742, 0, 0, 581, 4096, 0, 0, 0, 1721, 0, 0, 0, 1192, 1940, 4049, 0, 1061, 4081, 0, 4096, 2025, 0, 4096, 0, 0, 6, 82, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 3934, 0, 0, 641, 4095, 0, 1, 1, 605, 1, 1, 0, 1024, 3490, 3641, 1, 1188, 4085, 1, 4095, 1775, 0, 4095, 0, 0, 14, 78, 0, 1, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 3904, 0, 3, 563, 4056, 2, 0, 0, 730, 0, 0, 0, 1398, 3805, 4030, 0, 875, 4022, 0, 4096, 1151, 0, 4090, 0, 0, 0, 43, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3593, 0, 1, 326, 0, 0, 0, 704, 3571, 0, 0, 0, 0, 0, 0, 0, 677, 0, 0, 1595, 883, 507, 128, 0, 1727, 2658, 0, 0, 0, 0, 0, 0, 936, 3201, 0, 0, 0, 0, 0, 0, 2154, 0, 2243, 0, 0, 0, 0, 0, 0, 0, 11, 3499, 258, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3920, 4275, 3790, 4385, 4137, 4397, 3768, 4096] avg=4096.0000 max=4397 max_rank=5 min=3768 min_rank=6 imbalance_ratio=1.073486 max_load_reduction=28371 max_load_reduction_pct=0.865814 moved_to_echo=16398 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3723, 0, 0, 285, 0, 0, 0, 593, 3585, 0, 1, 0, 0, 0, 0, 1, 1517, 0, 1, 1595, 884, 510, 112, 0, 1303, 2321, 0, 0, 0, 0, 0, 1, 753, 3208, 1, 0, 0, 0, 0, 0, 2101, 1, 2242, 0, 0, 0, 0, 0, 0, 4, 64, 3607, 258, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4008, 4179, 4620, 3624, 3963, 4344, 3933, 4097] avg=4096.0000 max=4620 max_rank=2 min=3624 min_rank=3 imbalance_ratio=1.127930 max_load_reduction=28148 max_load_reduction_pct=0.859009 moved_to_echo=16337 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3669, 1, 0, 391, 0, 0, 0, 778, 3585, 1, 2, 0, 0, 0, 0, 2, 602, 0, 2, 1594, 885, 509, 153, 0, 828, 3182, 0, 0, 0, 0, 0, 0, 656, 3211, 2, 0, 0, 0, 0, 0, 2595, 2, 2242, 0, 0, 0, 0, 2, 0, 16, 81, 3423, 258, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4366, 3747, 4010, 3869, 4839, 3780, 4096] avg=4096.0000 max=4839 max_rank=5 min=3747 min_rank=2 imbalance_ratio=1.181396 max_load_reduction=27929 max_load_reduction_pct=0.852325 moved_to_echo=16584 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3862, 0, 0, 402, 0, 0, 0, 95, 3586, 0, 0, 0, 0, 0, 0, 0, 857, 0, 0, 1595, 883, 510, 158, 0, 1428, 3274, 0, 0, 0, 0, 0, 0, 1228, 3202, 0, 0, 0, 0, 0, 0, 888, 0, 2243, 0, 0, 0, 0, 0, 0, 0, 174, 4029, 258, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4264, 3681, 4003, 4702, 4430, 3131, 4461, 4096] avg=4096.0000 max=4702 max_rank=3 min=3131 min_rank=5 imbalance_ratio=1.147949 max_load_reduction=28066 max_load_reduction_pct=0.856506 moved_to_echo=16118 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3565, 0, 0, 395, 0, 0, 0, 189, 3571, 0, 0, 0, 0, 0, 0, 0, 1076, 0, 0, 1595, 880, 508, 155, 0, 1339, 3213, 0, 0, 0, 0, 0, 0, 502, 3191, 0, 0, 0, 0, 0, 0, 2146, 0, 2243, 0, 0, 0, 0, 0, 0, 1, 34, 3811, 258, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3960, 3760, 4214, 4552, 3693, 4389, 4104, 4096] avg=4096.0000 max=4552 max_rank=3 min=3693 min_rank=4 imbalance_ratio=1.111328 max_load_reduction=28216 max_load_reduction_pct=0.861084 moved_to_echo=16282 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3742, 0, 0, 203, 0, 0, 0, 581, 3586, 0, 0, 0, 0, 0, 0, 0, 1721, 0, 0, 1595, 882, 510, 79, 0, 1192, 1658, 0, 0, 0, 0, 0, 0, 1061, 3199, 0, 0, 0, 0, 0, 0, 2025, 0, 2243, 0, 0, 0, 0, 0, 0, 6, 82, 4049, 258, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3945, 4167, 4787, 2850, 4260, 4268, 4395, 4096] avg=4096.0000 max=4787 max_rank=2 min=2850 min_rank=3 imbalance_ratio=1.168701 max_load_reduction=27981 max_load_reduction_pct=0.853912 moved_to_echo=17019 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3904, 0, 3, 403, 0, 0, 0, 563, 3548, 2, 0, 0, 0, 0, 0, 0, 730, 0, 0, 1597, 873, 508, 160, 0, 1398, 3242, 0, 0, 0, 0, 0, 0, 875, 3149, 0, 0, 0, 0, 0, 0, 1151, 0, 2232, 0, 0, 0, 0, 0, 0, 0, 43, 4030, 261, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4310, 4113, 3868, 4640, 4024, 3383, 4335, 4095] avg=4096.0000 max=4640 max_rank=3 min=3383 min_rank=5 imbalance_ratio=1.132812 max_load_reduction=28128 max_load_reduction_pct=0.858398 moved_to_echo=15837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3934, 0, 0, 366, 0, 0, 0, 641, 3585, 0, 1, 0, 0, 0, 0, 1, 605, 1, 1, 1595, 883, 510, 143, 0, 1024, 2981, 0, 0, 0, 0, 0, 1, 1188, 3202, 1, 0, 0, 0, 0, 0, 1775, 0, 2242, 0, 0, 0, 0, 0, 0, 14, 78, 3641, 258, 1, 0, 0, 1, 1, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4300, 4227, 3739, 4005, 4392, 4017, 3992, 4096] avg=4096.0000 max=4392 max_rank=4 min=3739 min_rank=2 imbalance_ratio=1.072266 max_load_reduction=28376 max_load_reduction_pct=0.865967 moved_to_echo=16497 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 7808, 128, 128, 12800, 7168, 4096, 1152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 14848, 128, 18048, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7296, 25600, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 10240, 22656, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 30080, 128, 128, 2816, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 128, 640, 30208, 2176, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 28672, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 0, 0, 78, 1932, 0, 3379, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 2798, 0, 0, 5, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 10, 96, 1501, 0, 3710, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 2864, 0, 0, 11, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4, 18, 1261, 0, 3953, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 2764, 0, 0, 192, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[2, 1, 0, 4094, 0, 2, 0, 3, 82, 106, 1329, 2, 3434, 0, 2, 4092, 4094, 4, 4089, 0, 2, 2, 4094, 3223, 0, 0, 16, 1, 0, 4093, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 1, 0, 0, 61, 78, 1308, 1, 3308, 0, 1, 4093, 4095, 1, 4093, 0, 1, 1, 4095, 3426, 2, 0, 12, 0, 1, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 1, 0, 0, 3, 128, 2010, 1, 2699, 0, 1, 4093, 4095, 1, 4094, 0, 1, 1, 4095, 3316, 2, 0, 35, 0, 1, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 11, 0, 0, 0, 0, 20, 2151, 0, 3665, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 2355, 0, 0, 0, 0, 0, 4086, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 114, 1722, 0, 3223, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 3133, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 78, 1932, 0, 2344, 0, 0, 0, 3379, 0, 0, 675, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1112, 2798, 0, 0, 0, 0, 0, 0, 5, 0, 3421, 639, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 4354, 4054, 4098, 3910, 4065, 4097] avg=4096.0000 max=4354 max_rank=2 min=3910 min_rank=5 imbalance_ratio=1.062988 max_load_reduction=28414 max_load_reduction_pct=0.867126 moved_to_echo=16170 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 3, 4092, 0, 0, 0, 82, 106, 1329, 2, 2343, 0, 0, 0, 3434, 0, 2, 674, 0, 0, 0, 0, 2, 4, 4089, 0, 0, 0, 0, 0, 2, 2, 1111, 3223, 0, 0, 0, 0, 0, 0, 16, 1, 3418, 639, 0, 0, 0, 4093, 1, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4097, 3862, 4110, 4095, 4338, 4074, 4095] avg=4096.0000 max=4338 max_rank=5 min=3862 min_rank=2 imbalance_ratio=1.059082 max_load_reduction=28430 max_load_reduction_pct=0.867615 moved_to_echo=16602 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4, 18, 1261, 0, 2344, 0, 0, 0, 3953, 0, 0, 675, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1112, 2764, 0, 0, 0, 0, 0, 0, 192, 0, 3421, 639, 2, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 3627, 4628, 4096, 3876, 4254, 4097] avg=4096.0000 max=4628 max_rank=3 min=3627 min_rank=2 imbalance_ratio=1.129883 max_load_reduction=28140 max_load_reduction_pct=0.858765 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 61, 78, 1308, 1, 2343, 0, 0, 0, 3308, 0, 1, 674, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 1, 1, 1112, 3426, 0, 0, 0, 0, 2, 0, 12, 0, 3419, 639, 2, 0, 1, 4094, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 3791, 3983, 4094, 4540, 4074, 4096] avg=4096.0000 max=4540 max_rank=5 min=3791 min_rank=2 imbalance_ratio=1.108398 max_load_reduction=28228 max_load_reduction_pct=0.861450 moved_to_echo=16804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 10, 96, 1501, 0, 2344, 0, 0, 0, 3710, 0, 0, 675, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1112, 2864, 0, 0, 0, 0, 0, 0, 11, 0, 3421, 639, 1, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 3951, 4385, 4097, 3976, 4072, 4097] avg=4096.0000 max=4385 max_rank=3 min=3951 min_rank=2 imbalance_ratio=1.070557 max_load_reduction=28383 max_load_reduction_pct=0.866180 moved_to_echo=16242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 114, 1722, 0, 2344, 0, 0, 0, 3223, 0, 0, 675, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1112, 3133, 0, 0, 0, 0, 0, 0, 0, 0, 3421, 639, 1, 0, 0, 4096, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 4180, 3898, 4097, 4245, 4061, 4097] avg=4096.0000 max=4245 max_rank=5 min=3898 min_rank=3 imbalance_ratio=1.036377 max_load_reduction=28523 max_load_reduction_pct=0.870453 moved_to_echo=16500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 11, 0, 0, 0, 4095, 0, 0, 0, 0, 20, 2151, 0, 2347, 0, 0, 0, 3665, 0, 0, 671, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1098, 2355, 0, 0, 0, 0, 0, 0, 0, 0, 3425, 646, 1, 0, 0, 4086, 0, 0, 5, 1, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4106, 4518, 4336, 4096, 3453, 4072, 4092] avg=4096.0000 max=4518 max_rank=2 min=3453 min_rank=5 imbalance_ratio=1.103027 max_load_reduction=28250 max_load_reduction_pct=0.862122 moved_to_echo=15713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 3, 128, 2010, 1, 2343, 0, 0, 0, 2699, 0, 1, 674, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 1, 1, 1112, 3316, 0, 0, 0, 0, 2, 0, 35, 0, 3419, 639, 1, 0, 1, 4095, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4485, 3374, 4095, 4430, 4096, 4097] avg=4096.0000 max=4485 max_rank=2 min=3374 min_rank=3 imbalance_ratio=1.094971 max_load_reduction=28283 max_load_reduction_pct=0.863129 moved_to_echo=16718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 640, 13312, 128, 18816, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 8960, 23936, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 384, 128, 27392, 5120, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[27392, 0, 128, 5504, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 4095, 4095, 0, 4056, 0, 1, 4093, 4095, 1, 2, 1, 0, 4, 0, 4095, 0, 1, 4095, 4091, 0, 0, 1, 1, 38, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4080, 4093, 0, 4093, 0, 0, 4096, 4096, 0, 21, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4078, 0, 4096, 0, 0, 4096, 4096, 0, 18, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4092, 4095, 0, 4096, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[2, 2, 4094, 4093, 1, 4046, 0, 2, 4091, 4094, 2, 6, 1, 0, 8, 0, 4094, 0, 2, 4094, 4070, 0, 0, 2, 0, 61, 0, 1, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4096, 0, 4095, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 4095, 4095, 0, 4039, 0, 1, 4094, 4095, 1, 1, 1, 0, 7, 0, 4095, 0, 1, 4095, 4088, 0, 0, 1, 0, 56, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4048, 4085, 0, 4095, 0, 0, 4096, 4096, 0, 47, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 11, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4080, 9, 0, 0, 0, 0, 0, 4093, 0, 0, 11, 7, 0, 0, 4085, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4073, 4, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4111, 4106, 4093, 4096, 4100, 4077, 4096] avg=4096.0000 max=4111 max_rank=1 min=4077 min_rank=6 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4095, 9, 0, 0, 0, 0, 0, 4056, 0, 1, 11, 7, 0, 0, 4082, 0, 1, 2, 0, 0, 0, 0, 1, 0, 4, 0, 4091, 2, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4091, 0, 0, 1, 4, 0, 0, 0, 1, 38, 0, 0, 4075, 4, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4075, 4085, 4098, 4096, 4096, 4118, 4094] avg=4096.0000 max=4118 max_rank=6 min=4075 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4092, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 7, 0, 0, 4085, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 2, 2, 0, 0, 4075, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4114, 4086, 4093, 4096, 4100, 4083, 4095] avg=4096.0000 max=4114 max_rank=1 min=4083 min_rank=6 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 7, 0, 0, 4085, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 0, 4058, 4, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4114, 4103, 4094, 4096, 4100, 4064, 4094] avg=4096.0000 max=4114 max_rank=1 min=4064 min_rank=6 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 4094, 9, 0, 0, 0, 0, 1, 4046, 0, 2, 11, 7, 0, 0, 4080, 0, 2, 6, 0, 0, 0, 0, 1, 0, 8, 0, 4090, 1, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 4070, 0, 0, 2, 4, 0, 0, 0, 0, 61, 0, 1, 4073, 4, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4067, 4088, 4100, 4096, 4076, 4139, 4095] avg=4096.0000 max=4139 max_rank=6 min=4067 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 9, 0, 0, 0, 0, 0, 4095, 0, 0, 11, 7, 0, 0, 4085, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4076, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4113, 4087, 4093, 4096, 4100, 4080, 4095] avg=4096.0000 max=4113 max_rank=1 min=4080 min_rank=6 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4048, 4, 0, 0, 0, 0, 0, 4095, 0, 0, 13, 8, 0, 0, 4081, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 2, 0, 0, 0, 4068, 3, 0, 0, 0, 0, 11, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4116, 4128, 4095, 4096, 4103, 4073, 4105] avg=4096.0000 max=4128 max_rank=2 min=4052 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4095, 9, 0, 0, 0, 0, 0, 4039, 0, 1, 11, 7, 0, 0, 4083, 0, 1, 1, 0, 0, 0, 0, 1, 0, 7, 0, 4092, 2, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4088, 0, 0, 1, 4, 0, 0, 0, 0, 56, 0, 0, 4075, 3, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4058, 4085, 4102, 4096, 4093, 4134, 4094] avg=4096.0000 max=4134 max_rank=6 min=4058 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16417 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 256, 0, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4096, 0, 0, 0, 4094, 4095, 4094, 0, 0, 0, 0, 4096, 0, 0, 1, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4093, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 4094, 0, 2, 2, 4094, 4091, 3935, 2, 2, 2, 0, 4091, 0, 1, 10, 373, 4094, 2, 3, 1, 0, 34, 0, 0, 0, 3884, 0, 4048] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 1, 1, 4095, 4089, 4076, 1, 1, 1, 0, 4095, 0, 0, 7, 194, 4095, 1, 1, 1, 0, 15, 0, 0, 0, 3917, 0, 4081] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 1, 1, 4095, 4088, 3845, 1, 1, 1, 0, 4095, 0, 1, 8, 295, 4095, 1, 1, 1, 1, 15, 0, 0, 0, 4076, 0, 4050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 25, 0, 4096, 0, 0, 0, 4070, 4090, 4096, 0, 0, 0, 0, 4082, 0, 0, 12, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 55, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 1, 3, 3995, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 47, 1, 0, 0, 4096, 0, 53, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4149, 4097, 3999, 4095, 4087, 4149] avg=4096.0000 max=4149 max_rank=2 min=3999 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 55, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 1, 3996, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4041, 47, 1, 0, 0, 4096, 0, 53, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4148, 4097, 3997, 4095, 4091, 4149] avg=4096.0000 max=4149 max_rank=7 min=3997 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16332 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4088, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 55, 0, 4076, 1, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 7, 194, 3981, 0, 0, 0, 4095, 1, 1, 1, 0, 0, 0, 0, 0, 15, 0, 0, 4040, 47, 1, 0, 0, 3917, 0, 53, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4097, 4132, 4097, 4182, 4098, 4103, 3970] avg=4096.0000 max=4182 max_rank=4 min=3970 min_rank=7 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 4090, 0, 0, 0, 4094, 0, 2, 2, 0, 0, 0, 0, 55, 0, 3935, 2, 0, 0, 0, 0, 2, 2, 0, 4091, 0, 0, 0, 0, 0, 1, 10, 373, 3949, 0, 0, 0, 4094, 2, 3, 1, 0, 0, 0, 0, 0, 34, 0, 0, 4039, 47, 1, 0, 0, 3884, 0, 52, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4098, 3992, 4095, 4333, 4100, 4121, 3936] avg=4096.0000 max=4333 max_rank=4 min=3936 min_rank=7 imbalance_ratio=1.057861 max_load_reduction=28435 max_load_reduction_pct=0.867767 moved_to_echo=16490 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 55, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 1, 3996, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4041, 47, 0, 0, 0, 4093, 0, 53, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4148, 4097, 3997, 4095, 4093, 4146] avg=4096.0000 max=4148 max_rank=2 min=3997 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 55, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4041, 47, 1, 0, 0, 4096, 0, 53, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4149, 4097, 3996, 4095, 4091, 4149] avg=4096.0000 max=4149 max_rank=2 min=3996 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 25, 0, 4090, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 53, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4, 4, 1, 0, 0, 0, 12, 0, 3997, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4017, 53, 0, 0, 0, 4096, 0, 45, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4092, 4149, 4091, 4009, 4092, 4079, 4141] avg=4096.0000 max=4149 max_rank=2 min=4009 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4088, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 55, 0, 3845, 1, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 1, 8, 295, 3951, 0, 0, 0, 4095, 1, 1, 1, 0, 0, 0, 0, 1, 15, 0, 0, 4040, 47, 0, 0, 0, 4076, 0, 52, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4097, 3901, 4097, 4255, 4098, 4103, 4128] avg=4096.0000 max=4255 max_rank=4 min=3901 min_rank=2 imbalance_ratio=1.038818 max_load_reduction=28513 max_load_reduction_pct=0.870148 moved_to_echo=16584 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 896, 31872, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 32384, 384, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[512, 0, 32384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32384, 0, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4082, 0, 0, 0, 10, 0, 3085, 1513, 0, 0, 0, 906, 2691, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 1, 1, 4093, 0, 0, 1, 5, 10, 3532, 1, 1, 1, 1, 4095, 9, 2, 1, 32, 5, 1726, 2609, 0, 1, 0, 1971, 2459, 0, 4021, 4095, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 0, 2, 4091, 0, 3, 2, 29, 19, 3474, 2, 2, 2, 2, 4094, 6, 5, 2, 3, 3, 2892, 2826, 0, 2, 0, 802, 2413, 0, 3902, 4094, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4089, 3, 0, 0, 10, 0, 2714, 2311, 0, 0, 0, 324, 2836, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 0, 5, 0, 2474, 1583, 0, 0, 0, 694, 3434, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 4065, 1, 0, 0, 67, 0, 2550, 3036, 0, 0, 0, 200, 2373, 0, 4095, 4096, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4094, 2, 0, 0, 1, 0, 3258, 3114, 0, 0, 0, 258, 1566, 0, 4089, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 4093, 0, 0, 1, 7, 20, 3718, 1, 1, 1, 1, 4095, 33, 4, 1, 11, 1, 2356, 2438, 0, 0, 0, 1309, 2493, 0, 3991, 4095, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 214, 0, 0, 0, 0, 0, 0, 4082, 6, 0, 0, 0, 0, 0, 0, 10, 4048, 25, 1, 0, 0, 1956, 1513, 0, 0, 0, 0, 0, 0, 0, 906, 2691, 884, 0, 0, 0, 0, 4096, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4311, 4088, 4084, 3469, 4481, 4144] avg=4096.0000 max=4481 max_rank=6 min=3469 min_rank=5 imbalance_ratio=1.093994 max_load_reduction=28287 max_load_reduction_pct=0.863251 moved_to_echo=16178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 1, 4094, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 5, 10, 3532, 1, 120, 0, 0, 0, 1, 1, 1, 4095, 3, 0, 0, 0, 9, 2, 1, 32, 4046, 14, 1, 0, 5, 1095, 2609, 0, 0, 0, 0, 0, 1, 0, 1971, 2459, 494, 0, 0, 0, 0, 4021, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4094, 3668, 4101, 4105, 3709, 4925, 4069] avg=4096.0000 max=4925 max_rank=6 min=3668 min_rank=2 imbalance_ratio=1.202393 max_load_reduction=27843 max_load_reduction_pct=0.849701 moved_to_echo=16808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 2, 4093, 0, 0, 0, 4091, 0, 3, 2, 0, 0, 0, 0, 29, 19, 3474, 2, 201, 0, 0, 0, 2, 2, 2, 4094, 6, 0, 0, 0, 6, 5, 2, 3, 4046, 23, 1, 0, 3, 1833, 2826, 0, 0, 0, 0, 0, 2, 0, 802, 2413, 829, 0, 0, 0, 0, 3902, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 3725, 4106, 4086, 4662, 4046, 3950] avg=4096.0000 max=4662 max_rank=5 min=3725 min_rank=2 imbalance_ratio=1.138184 max_load_reduction=28106 max_load_reduction_pct=0.857727 moved_to_echo=16744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 189, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 3, 0, 0, 10, 4048, 22, 1, 0, 0, 1720, 2311, 0, 0, 0, 0, 0, 0, 0, 324, 2836, 778, 0, 0, 0, 0, 4096, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4286, 4094, 4084, 4031, 3938, 4144] avg=4096.0000 max=4286 max_rank=2 min=3938 min_rank=6 imbalance_ratio=1.046387 max_load_reduction=28482 max_load_reduction_pct=0.869202 moved_to_echo=16197 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 172, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 2, 0, 0, 5, 4048, 20, 1, 0, 0, 1568, 1583, 0, 0, 0, 0, 0, 0, 0, 694, 3434, 709, 0, 0, 0, 0, 4096, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4268, 4101, 4076, 3151, 4837, 4144] avg=4096.0000 max=4837 max_rank=6 min=3151 min_rank=5 imbalance_ratio=1.180908 max_load_reduction=27931 max_load_reduction_pct=0.852386 moved_to_echo=16208 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 5, 0, 0, 0, 2, 0, 4093, 0, 180, 0, 0, 0, 0, 0, 0, 4065, 9, 0, 0, 0, 1, 0, 0, 67, 4049, 23, 0, 0, 0, 1599, 3036, 0, 0, 0, 0, 0, 0, 0, 200, 2373, 734, 0, 0, 0, 0, 4095, 0, 46, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4100, 4275, 4074, 4140, 4635, 3307, 4141] avg=4096.0000 max=4635 max_rank=5 min=3307 min_rank=6 imbalance_ratio=1.131592 max_load_reduction=28133 max_load_reduction_pct=0.858551 moved_to_echo=16223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 227, 0, 0, 0, 0, 0, 0, 4094, 7, 0, 0, 0, 2, 0, 0, 1, 4048, 26, 1, 0, 0, 2065, 3114, 0, 0, 0, 0, 0, 0, 0, 258, 1566, 933, 0, 0, 0, 0, 4089, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4325, 4101, 4078, 5179, 2757, 4137] avg=4096.0000 max=5179 max_rank=5 min=2757 min_rank=6 imbalance_ratio=1.264404 max_load_reduction=27589 max_load_reduction_pct=0.841949 moved_to_echo=16151 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 4095, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 7, 20, 3718, 1, 164, 0, 0, 0, 1, 1, 1, 4095, 5, 0, 0, 0, 33, 4, 1, 11, 4047, 19, 0, 0, 1, 1493, 2438, 0, 0, 0, 0, 0, 0, 0, 1309, 2493, 675, 0, 0, 0, 0, 3991, 0, 48, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4094, 3910, 4103, 4115, 3932, 4477, 4039] avg=4096.0000 max=4477 max_rank=6 min=3910 min_rank=2 imbalance_ratio=1.093018 max_load_reduction=28291 max_load_reduction_pct=0.863373 moved_to_echo=16563 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 6528, 20352, 6144, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 13440, 19456, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 256, 32384, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 31232, 128, 1536, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32512, 0, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 4071, 194, 1, 0, 0, 3922, 0, 0, 4, 1, 0, 4095, 1, 1, 4094, 1, 1, 4095, 0, 0, 4079, 1, 4095, 15, 0, 0, 0, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 58, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4091, 47, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 4096, 0, 10, 0, 0, 0, 0, 4077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4094, 102, 0, 0, 0, 4004, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4086, 0, 4096, 0, 2, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 16, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 4088, 117, 1, 0, 0, 3971, 0, 0, 9, 1, 0, 4095, 1, 0, 4095, 1, 1, 4095, 0, 0, 4094, 1, 4095, 5, 1, 0, 0, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 484, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3791, 0, 4096, 0, 4, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 2, 4073, 251, 2, 0, 0, 3859, 0, 0, 11, 2, 0, 4094, 2, 0, 4091, 3, 2, 4094, 0, 0, 4077, 3, 4094, 11, 0, 0, 0, 0, 2, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3943, 102, 0, 0, 0, 0, 0, 0, 0, 4004, 109, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 42, 3, 0, 0, 4091, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4116, 4093, 4097, 4096, 4131, 4093, 4097] avg=4096.0000 max=4131 max_rank=5 min=4045 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16417 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3922, 194, 0, 0, 0, 0, 1, 0, 0, 3922, 108, 2, 0, 0, 0, 0, 4, 1, 4093, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4079, 1, 41, 3, 0, 0, 4090, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4033, 4098, 4097, 4096, 4124, 4105, 4098] avg=4096.0000 max=4124 max_rank=5 min=4033 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16423 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3933, 47, 0, 0, 0, 0, 0, 0, 0, 4065, 113, 4, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 45, 6, 0, 0, 4087, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3, 2, 1, 0] rerouted_tokens_per_ep_rank=[3980, 4182, 4092, 4094, 4095, 4145, 4097, 4083] avg=4096.0000 max=4182 max_rank=1 min=3980 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16420 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3945, 16, 0, 0, 0, 0, 0, 0, 0, 4091, 109, 4, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 42, 3, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[3961, 4204, 4092, 4096, 4096, 4130, 4091, 4098] avg=4096.0000 max=4204 max_rank=1 min=3961 min_rank=0 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3945, 58, 0, 0, 0, 0, 0, 0, 0, 4041, 109, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 42, 3, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4003, 4153, 4093, 4096, 4096, 4138, 4091, 4098] avg=4096.0000 max=4153 max_rank=1 min=4003 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16423 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3945, 484, 0, 0, 0, 0, 0, 0, 0, 3913, 109, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 42, 3, 0, 0, 4091, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4429, 4025, 4093, 4096, 4096, 3836, 4095, 4098] avg=4096.0000 max=4429 max_rank=0 min=3836 min_rank=5 imbalance_ratio=1.081299 max_load_reduction=28339 max_load_reduction_pct=0.864838 moved_to_echo=16125 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3937, 117, 0, 0, 0, 0, 1, 0, 0, 3971, 109, 3, 0, 0, 0, 0, 9, 1, 4092, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 42, 3, 0, 0, 4090, 5, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4084, 4102, 4096, 4097, 4140, 4096, 4098] avg=4096.0000 max=4140 max_rank=5 min=4055 min_rank=0 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16431 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 3924, 251, 0, 0, 0, 0, 2, 0, 0, 3859, 108, 2, 0, 0, 0, 0, 11, 2, 4092, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 4091, 3, 2, 0, 0, 0, 0, 0, 0, 0, 4077, 3, 41, 3, 0, 0, 4089, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 2, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 3971, 4105, 4096, 4096, 4124, 4100, 4098] avg=4096.0000 max=4178 max_rank=0 min=3971 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 31616, 1280, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 31872, 896, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 32512, 128, 384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3275, 4096, 0, 0, 0, 4096, 4096, 1, 0, 0, 4096, 0, 4096, 0, 4095, 955, 0, 0, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[2936, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 1223, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[3274, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 895, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[2893, 4093, 0, 0, 0, 4090, 4095, 0, 0, 1, 4086, 0, 4095, 1, 3906, 1396, 4, 1, 1, 4064, 1, 0, 0, 1, 0, 1, 1, 34, 0, 1, 1, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[2599, 4090, 2, 0, 0, 4079, 4094, 0, 1, 2, 4092, 0, 4093, 4, 3909, 1711, 15, 2, 2, 4024, 2, 0, 0, 2, 0, 2, 3, 36, 0, 0, 2, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3019, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 1123, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[3170, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4095, 0, 4096, 0, 4096, 1041, 0, 0, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[2694, 4093, 0, 0, 0, 4091, 4095, 0, 0, 1, 4093, 0, 4095, 1, 3972, 1575, 4, 1, 1, 4006, 1, 0, 0, 1, 0, 1, 2, 37, 0, 1, 1, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3275, 1114, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 2901, 955, 0, 0, 0, 0, 0, 0, 0, 3960, 79, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 2980, 1114, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0] rerouted_tokens_per_ep_rank=[4389, 4101, 4097, 3856, 4039, 4096, 4096, 4094] avg=4096.0000 max=4389 max_rank=0 min=3856 min_rank=3 imbalance_ratio=1.071533 max_load_reduction=28379 max_load_reduction_pct=0.866058 moved_to_echo=16325 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3274, 1114, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 2901, 895, 0, 0, 0, 0, 0, 0, 0, 4023, 79, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2980, 1115, 0, 0, 0, 0, 0, 0, 4092, 2, 1, 0] rerouted_tokens_per_ep_rank=[4388, 4100, 4097, 3796, 4102, 4095, 4095, 4095] avg=4096.0000 max=4388 max_rank=0 min=3796 min_rank=3 imbalance_ratio=1.071289 max_load_reduction=28380 max_load_reduction_pct=0.866089 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2893, 1114, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 1, 4086, 0, 1, 0, 0, 0, 0, 1, 2767, 1396, 0, 0, 0, 0, 4, 1, 1, 4064, 75, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 1, 34, 2978, 1063, 0, 0, 0, 1, 1, 2, 4091, 1, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4094, 4088, 4164, 4145, 4097, 4077, 4096] avg=4096.0000 max=4164 max_rank=3 min=4007 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2936, 1114, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 2901, 1223, 0, 0, 0, 0, 0, 0, 0, 4031, 79, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 2980, 1115, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4100, 4097, 4124, 4110, 4096, 4097, 4094] avg=4096.0000 max=4124 max_rank=3 min=4050 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3019, 1114, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 2901, 1123, 0, 0, 0, 0, 0, 0, 0, 4050, 79, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2980, 1115, 0, 0, 0, 0, 0, 0, 4092, 2, 1, 0] rerouted_tokens_per_ep_rank=[4133, 4100, 4097, 4024, 4129, 4095, 4095, 4095] avg=4096.0000 max=4133 max_rank=0 min=4024 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16414 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2599, 1114, 2, 0, 0, 0, 0, 0, 0, 4079, 4, 0, 0, 0, 0, 0, 1, 2, 4092, 0, 1, 0, 0, 0, 0, 4, 2769, 1711, 0, 0, 0, 0, 15, 2, 2, 4024, 75, 0, 0, 0, 2, 0, 0, 2, 4092, 0, 0, 0, 0, 2, 3, 36, 2975, 1064, 0, 0, 0, 0, 2, 2, 4090, 1, 1, 0] rerouted_tokens_per_ep_rank=[3715, 4083, 4096, 4484, 4118, 4096, 4080, 4096] avg=4096.0000 max=4484 max_rank=3 min=3715 min_rank=0 imbalance_ratio=1.094727 max_load_reduction=28284 max_load_reduction_pct=0.863159 moved_to_echo=16390 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3170, 1108, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 0, 0, 0, 0, 0, 2890, 1041, 0, 0, 0, 0, 0, 0, 0, 3953, 83, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 29, 2983, 1117, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0] rerouted_tokens_per_ep_rank=[4278, 4096, 4101, 3931, 4036, 4096, 4129, 4101] avg=4096.0000 max=4278 max_rank=0 min=3931 min_rank=3 imbalance_ratio=1.044434 max_load_reduction=28490 max_load_reduction_pct=0.869446 moved_to_echo=16362 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2694, 1114, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 1, 0, 0, 0, 0, 1, 2813, 1575, 0, 0, 0, 0, 4, 1, 1, 4006, 77, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 2, 37, 2978, 1081, 0, 0, 0, 1, 1, 2, 4092, 1, 0, 0] rerouted_tokens_per_ep_rank=[3808, 4094, 4095, 4389, 4089, 4097, 4099, 4097] avg=4096.0000 max=4389 max_rank=3 min=3808 min_rank=0 imbalance_ratio=1.071533 max_load_reduction=28379 max_load_reduction_pct=0.866058 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 256, 23936, 8832, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[23936, 8960, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 22912, 9984, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32128, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4028, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 4095, 4096, 84, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[49, 1, 2, 1, 0, 1, 0, 76, 4081, 4, 4084, 0, 2, 4049, 0, 0, 4095, 0, 0, 1, 4083, 0, 0, 3932, 4094, 23, 0, 104, 1, 4084, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[76, 2, 8, 2, 0, 0, 0, 95, 4061, 10, 4086, 0, 3, 4023, 0, 0, 4094, 0, 0, 2, 4084, 0, 0, 3866, 4090, 21, 0, 164, 2, 4077, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4020, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 76, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 3863, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 232, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4033, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 63, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2, 0, 4079, 0, 3984, 0, 0, 4083, 0, 0, 4094, 0, 0, 0, 4096, 0, 11, 4096, 4096, 131, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[24, 1, 3, 1, 0, 0, 0, 83, 4088, 6, 3988, 0, 2, 4070, 0, 0, 4095, 0, 0, 1, 4089, 0, 0, 3921, 4093, 131, 1, 86, 1, 4083, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4026, 52, 0, 0, 0, 0, 0, 0, 3955, 113, 0, 0, 84, 0, 4028, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 20, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 70, 0, 1, 4095, 0, 0, 0, 0, 3959, 84, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4068, 4112, 4105, 4096, 4166, 4043, 4100] avg=4096.0000 max=4166 max_rank=5 min=4043 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[49, 1, 2, 1, 4013, 52, 0, 0, 0, 1, 0, 76, 3945, 112, 0, 0, 84, 4, 4084, 0, 0, 0, 0, 0, 2, 4049, 0, 0, 20, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 70, 0, 0, 3932, 0, 0, 0, 0, 3958, 23, 0, 104, 0, 0, 0, 0, 1, 4084, 1, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4134, 4172, 4071, 4096, 4002, 4085, 4090] avg=4096.0000 max=4172 max_rank=2 min=4002 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4026, 52, 0, 0, 0, 0, 0, 0, 3960, 113, 0, 0, 84, 0, 4033, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4096, 0, 0, 0, 0, 3959, 63, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4073, 4117, 4116, 4096, 4166, 4022, 4100] avg=4096.0000 max=4166 max_rank=5 min=4022 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[76, 2, 8, 2, 4014, 52, 0, 0, 0, 0, 0, 95, 3926, 112, 0, 0, 83, 10, 4086, 0, 0, 0, 0, 0, 3, 4023, 0, 0, 20, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 70, 0, 0, 3866, 0, 0, 0, 0, 3954, 21, 0, 164, 0, 0, 0, 0, 2, 4077, 2, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4133, 4179, 4046, 4096, 3936, 4139, 4085] avg=4096.0000 max=4179 max_rank=2 min=3936 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4026, 52, 0, 0, 0, 0, 0, 0, 3960, 113, 0, 0, 84, 0, 4020, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4096, 0, 0, 0, 0, 3959, 76, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4073, 4104, 4116, 4096, 4166, 4035, 4100] avg=4096.0000 max=4166 max_rank=5 min=4035 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4026, 52, 0, 0, 0, 0, 0, 1, 3960, 113, 0, 0, 84, 0, 3863, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4096, 0, 0, 0, 0, 3959, 232, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4074, 3947, 4116, 4096, 4166, 4191, 4100] avg=4096.0000 max=4191 max_rank=6 min=3947 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4028, 56, 0, 0, 0, 0, 2, 0, 3948, 116, 0, 0, 75, 0, 3984, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 23, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 68, 0, 11, 4096, 0, 0, 0, 0, 3947, 131, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4066, 4059, 4106, 4096, 4175, 4078, 4104] avg=4096.0000 max=4175 max_rank=5 min=4059 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[24, 1, 3, 1, 4019, 52, 0, 0, 0, 0, 0, 83, 3952, 112, 0, 0, 84, 6, 3988, 0, 0, 0, 0, 0, 2, 4070, 0, 0, 20, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 70, 0, 0, 3921, 0, 0, 0, 0, 3957, 131, 1, 86, 0, 0, 0, 0, 1, 4083, 1, 0, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4147, 4078, 4092, 4096, 3991, 4175, 4089] avg=4096.0000 max=4175 max_rank=6 min=3991 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[768, 128, 32128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[256, 128, 128, 128, 32256, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[640, 0, 128, 32256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31744, 768, 128, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32640, 0, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 256, 31616, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 3028, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 1064, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 2971, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 1118, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 3966, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 128, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4092, 0, 3301, 0, 0, 0, 0, 0, 4079, 4096, 1, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 33, 0, 0, 0, 0, 0, 4096, 0, 771, 0, 11] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 2623, 1, 1, 0, 9, 1, 4095, 4088, 0, 0, 0, 0, 0, 1, 4093, 4094, 4088, 1, 1, 12, 13, 2, 0, 0, 1, 4095, 0, 1447, 7, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 1399, 3, 2, 0, 18, 2, 4094, 4094, 0, 0, 0, 0, 0, 3, 4092, 4087, 4078, 2, 2, 17, 37, 4, 0, 0, 2, 4092, 0, 2638, 9, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4092, 0, 2941, 1, 1, 0, 8, 1, 4095, 4092, 0, 0, 0, 0, 0, 1, 4094, 4091, 4088, 1, 1, 10, 2, 2, 0, 0, 1, 4095, 0, 1142, 9, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 3604, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 489, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1117, 0, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1064, 0, 0, 2979, 13, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4092, 4100, 4096, 4101, 4082, 4096, 4056] avg=4096.0000 max=4145 max_rank=0 min=4056 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16335 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1117, 0, 2623, 1, 0, 0, 0, 0, 1, 0, 9, 1, 4084, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 5, 4088, 1, 0, 0, 0, 0, 1, 12, 13, 2, 4076, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 1447, 7, 0, 2978, 13, 0, 0] rerouted_tokens_per_ep_rank=[3741, 4095, 4099, 4094, 4094, 4104, 4096, 4445] avg=4096.0000 max=4445 max_rank=7 min=3741 min_rank=0 imbalance_ratio=1.085205 max_load_reduction=28323 max_load_reduction_pct=0.864349 moved_to_echo=16739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1117, 0, 2971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1118, 0, 1, 2979, 13, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4092, 4100, 4096, 4101, 4084, 4096, 4111] avg=4096.0000 max=4111 max_rank=7 min=4084 min_rank=5 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16392 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1117, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4078, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 128, 0, 0, 2979, 13, 0, 0] rerouted_tokens_per_ep_rank=[5083, 4092, 4098, 4096, 4101, 4082, 4096, 3120] avg=4096.0000 max=5083 max_rank=0 min=3120 min_rank=7 imbalance_ratio=1.240967 max_load_reduction=27685 max_load_reduction_pct=0.844879 moved_to_echo=15399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1117, 0, 1399, 3, 0, 0, 0, 0, 2, 0, 18, 2, 4090, 0, 0, 0, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 0, 0, 0, 0, 5, 4078, 2, 0, 0, 0, 0, 2, 17, 37, 4, 4069, 0, 0, 0, 0, 0, 2, 4092, 0, 0, 0, 0, 0, 2638, 9, 0, 2976, 13, 0, 0] rerouted_tokens_per_ep_rank=[2519, 4112, 4098, 4095, 4085, 4129, 4094, 5636] avg=4096.0000 max=5636 max_rank=7 min=2519 min_rank=0 imbalance_ratio=1.375977 max_load_reduction=27132 max_load_reduction_pct=0.828003 moved_to_echo=17944 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1112, 0, 3301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4079, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 33, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 771, 0, 11, 2978, 14, 0, 0] rerouted_tokens_per_ep_rank=[4413, 4095, 4080, 4097, 4096, 4115, 4098, 3774] avg=4096.0000 max=4413 max_rank=0 min=3774 min_rank=7 imbalance_ratio=1.077393 max_load_reduction=28355 max_load_reduction_pct=0.865326 moved_to_echo=16083 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1117, 0, 3604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 489, 0, 1, 2978, 13, 0, 0] rerouted_tokens_per_ep_rank=[4721, 4092, 4098, 4096, 4101, 4083, 4096, 3481] avg=4096.0000 max=4721 max_rank=0 min=3481 min_rank=7 imbalance_ratio=1.152588 max_load_reduction=28047 max_load_reduction_pct=0.855927 moved_to_echo=15761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1116, 0, 2941, 1, 0, 0, 0, 0, 1, 0, 8, 1, 4088, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 3, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 1, 10, 2, 2, 4073, 1, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 1142, 9, 0, 2976, 18, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4098, 4095, 4098, 4089, 4089, 4096, 4145] avg=4096.0000 max=4145 max_rank=7 min=4058 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16419 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8960, 0, 23936, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 8832, 128, 128, 23936, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[3, 4092, 2, 0, 162, 2, 0, 4092, 2, 1, 2, 2, 4094, 4, 0, 1, 4090, 0, 55, 4094, 2, 0, 2, 4093, 1, 0, 0, 0, 2, 3876, 1, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4071, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4040, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 56, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4081, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4023, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 80, 0, 0, 8, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4092, 1, 0, 105, 1, 0, 4094, 1, 0, 1, 1, 4094, 2, 1, 0, 4093, 0, 17, 4095, 1, 0, 0, 4095, 18, 0, 0, 1, 1, 3960, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4024, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 72, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 129, 1, 1, 4094, 1, 0, 1, 1, 4095, 2, 0, 0, 4092, 0, 11, 4095, 1, 0, 1, 4095, 0, 0, 0, 0, 1, 3957, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4040, 0, 0, 0, 21, 6, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 56, 0, 0, 0, 4034, 28, 2, 0, 0, 4096, 0, 62, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4047, 4094, 4067, 4088, 4096, 4120, 4158] avg=4096.0000 max=4158 max_rank=7 min=4047 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16462 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3, 4092, 2, 0, 1, 0, 0, 0, 162, 2, 0, 4044, 0, 0, 0, 0, 2, 1, 2, 2, 4093, 0, 0, 0, 4094, 4, 0, 1, 20, 6, 0, 0, 4083, 0, 55, 0, 0, 0, 0, 0, 2, 0, 2, 4093, 0, 0, 0, 0, 1, 0, 0, 0, 4031, 28, 1, 0, 2, 3876, 1, 62, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4208, 4100, 4125, 4138, 4097, 4061, 3941] avg=4096.0000 max=4208 max_rank=1 min=3941 min_rank=7 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16237 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 21, 6, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4034, 28, 1, 0, 0, 4096, 0, 62, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4047, 4095, 4122, 4088, 4096, 4064, 4158] avg=4096.0000 max=4158 max_rank=7 min=4047 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 1, 0, 1, 0, 0, 0, 105, 1, 0, 4046, 0, 0, 0, 0, 1, 0, 1, 1, 4094, 0, 0, 0, 4094, 2, 1, 0, 20, 6, 0, 0, 4086, 0, 17, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 18, 0, 0, 1, 4032, 28, 1, 0, 1, 3960, 0, 62, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4152, 4097, 4123, 4103, 4096, 4080, 4023] avg=4096.0000 max=4152 max_rank=1 min=4023 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16302 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4071, 0, 0, 0, 21, 6, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 25, 0, 0, 0, 4034, 28, 1, 0, 0, 4096, 0, 62, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4047, 4095, 4098, 4088, 4096, 4088, 4158] avg=4096.0000 max=4158 max_rank=7 min=4047 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16430 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4024, 0, 0, 0, 21, 6, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 72, 0, 0, 1, 4034, 28, 1, 0, 0, 4096, 0, 62, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4047, 4095, 4051, 4088, 4096, 4136, 4158] avg=4096.0000 max=4158 max_rank=7 min=4047 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16478 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4081, 0, 0, 5, 2, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4023, 0, 0, 0, 24, 12, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 80, 0, 0, 8, 4040, 29, 1, 0, 0, 4096, 0, 56, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4043, 4095, 4059, 4079, 4094, 4158, 4152] avg=4096.0000 max=4158 max_rank=6 min=4043 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16483 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 1, 0, 0, 0, 129, 1, 1, 4046, 0, 0, 0, 0, 1, 0, 1, 1, 4094, 0, 0, 0, 4095, 2, 0, 0, 20, 6, 0, 0, 4085, 0, 11, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 28, 1, 0, 1, 3957, 0, 62, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4177, 4097, 4123, 4096, 4097, 4061, 4020] avg=4096.0000 max=4177 max_rank=1 min=4020 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16274 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 0, 0, 128, 32384, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32384, 128, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 128, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[512, 128, 128, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4095, 4095, 4095, 3946, 2, 2, 15, 4095, 1, 0, 0, 150, 1, 4078, 0, 0, 0, 1, 0, 0, 0, 1, 4094, 0, 0, 0, 4095, 1, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4094, 4094, 4093, 3925, 3, 5, 46, 4094, 2, 1, 0, 160, 2, 4063, 0, 2, 0, 2, 2, 0, 0, 3, 4080, 0, 0, 0, 4094, 2, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4092, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4085, 4096, 4096, 4093, 0, 0, 0, 4096, 0, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4061, 4094, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 35, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4095, 4095, 4095, 3977, 2, 2, 6, 4095, 1, 0, 0, 117, 1, 4089, 0, 0, 0, 1, 0, 0, 0, 1, 4094, 0, 0, 0, 4095, 1, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[56, 0, 0, 3946, 0, 0, 0, 0, 2, 2, 15, 4087, 0, 0, 0, 0, 1, 0, 0, 150, 4039, 1, 0, 0, 1, 4078, 0, 0, 6, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4002, 4106, 4191, 4085, 4096, 4096, 4096, 4096] avg=4096.0000 max=4191 max_rank=2 min=4002 min_rank=0 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[56, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 14, 4029, 1, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4088, 4044, 4102, 4096, 4097, 4096, 4096] avg=4096.0000 max=4149 max_rank=0 min=4044 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[56, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 1, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4088, 4041, 4102, 4095, 4097, 4096, 4097] avg=4096.0000 max=4152 max_rank=0 min=4041 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[56, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4, 4040, 1, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4088, 4045, 4102, 4095, 4097, 4096, 4097] avg=4096.0000 max=4148 max_rank=0 min=4045 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[56, 0, 0, 3925, 0, 0, 0, 0, 3, 5, 46, 4086, 0, 0, 0, 0, 2, 1, 0, 160, 4038, 1, 0, 0, 2, 4063, 0, 2, 6, 0, 0, 0, 0, 2, 2, 0, 4094, 0, 0, 0, 0, 3, 4080, 0, 1, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[3981, 4140, 4202, 4073, 4098, 4084, 4096, 4094] avg=4096.0000 max=4202 max_rank=2 min=3981 min_rank=0 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[56, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 1, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4088, 4041, 4102, 4096, 4097, 4096, 4096] avg=4096.0000 max=4152 max_rank=0 min=4041 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[55, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 35, 4006, 5, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4073, 4046, 4106, 4096, 4104, 4096, 4096] avg=4096.0000 max=4151 max_rank=0 min=4046 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[56, 0, 0, 3977, 0, 0, 0, 0, 2, 2, 6, 4087, 0, 0, 0, 0, 1, 0, 0, 117, 4039, 1, 0, 0, 1, 4089, 0, 0, 6, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4097, 4158, 4096, 4096, 4096, 4096, 4096] avg=4096.0000 max=4158 max_rank=2 min=4033 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[512, 0, 0, 32384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 512, 32384, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2095, 0, 1365, 292, 793, 0, 2006, 138, 1689, 2485, 1, 100, 1044, 2161, 202, 1901, 44, 1427, 1671, 0, 1, 1498, 1742, 419, 1, 1878, 2310, 1186, 1401, 2794, 124, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1609, 0, 1915, 214, 883, 0, 2591, 86, 1090, 2785, 0, 49, 832, 2222, 80, 881, 32, 2153, 879, 0, 0, 2266, 1883, 450, 0, 1943, 2684, 1205, 997, 2720, 318, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1706, 0, 1727, 121, 887, 0, 1598, 320, 1147, 2220, 0, 59, 1185, 2338, 57, 1370, 28, 1837, 2009, 0, 2, 1121, 1426, 361, 0, 2354, 1786, 1258, 2390, 3271, 190, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2227, 0, 1880, 175, 917, 0, 1717, 159, 1217, 1925, 0, 45, 1007, 1820, 91, 1278, 39, 1645, 1478, 0, 1, 1567, 1735, 490, 0, 2204, 2178, 1867, 2131, 2829, 145, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2021, 1, 2128, 296, 772, 0, 1809, 342, 1648, 2297, 0, 75, 1214, 1568, 76, 961, 67, 1651, 1264, 0, 0, 1480, 1951, 474, 0, 2125, 2795, 1213, 2243, 2035, 262, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1957, 2, 1856, 255, 1025, 5, 2105, 200, 1430, 2285, 0, 110, 1056, 1753, 187, 1291, 40, 1715, 941, 0, 6, 1678, 1999, 421, 0, 1373, 2503, 1246, 2338, 2772, 219, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2049, 1, 2218, 206, 904, 2, 1374, 262, 1168, 1608, 0, 59, 1516, 1188, 107, 796, 30, 1803, 1903, 1, 5, 1391, 2300, 1210, 0, 2145, 2287, 1758, 2079, 2073, 318, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1741, 0, 2015, 178, 1085, 1, 1607, 342, 1312, 2012, 0, 45, 1259, 1733, 52, 1094, 78, 1605, 1981, 0, 0, 1283, 1773, 679, 0, 2373, 2571, 1538, 1878, 2272, 261, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2095, 0, 1365, 292, 75, 0, 0, 0, 793, 0, 2006, 138, 1068, 0, 0, 0, 1689, 2485, 1, 100, 525, 0, 0, 0, 1044, 1935, 202, 1901, 0, 0, 0, 0, 44, 1427, 1671, 0, 625, 171, 0, 0, 1, 1498, 1742, 419, 151, 16, 0, 0, 1, 1878, 617, 1186, 0, 0, 0, 0, 1401, 2082, 124, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3827, 4005, 4800, 5082, 3938, 3827, 3682, 3607] avg=4096.0000 max=5082 max_rank=3 min=3607 min_rank=7 imbalance_ratio=1.240723 max_load_reduction=27686 max_load_reduction_pct=0.844910 moved_to_echo=15054 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1609, 0, 1915, 214, 77, 0, 0, 0, 883, 0, 2591, 86, 1241, 0, 0, 0, 1090, 2785, 0, 49, 511, 0, 0, 0, 832, 1990, 80, 881, 0, 0, 0, 0, 32, 2153, 879, 0, 726, 166, 0, 0, 0, 2266, 1883, 450, 155, 15, 0, 0, 0, 1943, 717, 1205, 0, 0, 0, 0, 997, 2028, 318, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3815, 4801, 4435, 3783, 3956, 4769, 3865, 3344] avg=4096.0000 max=4801 max_rank=1 min=3344 min_rank=7 imbalance_ratio=1.172119 max_load_reduction=27967 max_load_reduction_pct=0.853485 moved_to_echo=15934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2227, 0, 1880, 175, 63, 0, 0, 0, 917, 0, 1717, 159, 1007, 0, 0, 0, 1217, 1925, 0, 45, 532, 0, 0, 0, 1007, 1630, 91, 1278, 0, 0, 0, 0, 39, 1645, 1478, 0, 589, 173, 0, 0, 1, 1567, 1735, 490, 127, 16, 0, 0, 0, 2204, 582, 1867, 0, 0, 0, 0, 2131, 2108, 145, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4345, 3800, 3719, 4006, 3924, 3936, 4653, 4385] avg=4096.0000 max=4653 max_rank=6 min=3719 min_rank=2 imbalance_ratio=1.135986 max_load_reduction=28115 max_load_reduction_pct=0.858002 moved_to_echo=16898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1741, 0, 2015, 178, 60, 0, 0, 0, 1085, 1, 1607, 342, 1188, 0, 0, 0, 1312, 2012, 0, 45, 427, 0, 0, 0, 1259, 1552, 52, 1094, 0, 0, 0, 0, 78, 1605, 1981, 0, 696, 139, 0, 0, 0, 1283, 1773, 679, 121, 13, 0, 0, 0, 2373, 687, 1538, 0, 0, 0, 0, 1878, 1693, 261, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 4223, 3796, 3957, 4499, 3869, 4598, 3832] avg=4096.0000 max=4598 max_rank=6 min=3796 min_rank=2 imbalance_ratio=1.122559 max_load_reduction=28170 max_load_reduction_pct=0.859680 moved_to_echo=16798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1706, 0, 1727, 121, 81, 0, 0, 0, 887, 0, 1598, 320, 825, 0, 0, 0, 1147, 2220, 0, 59, 615, 0, 0, 0, 1185, 2094, 57, 1370, 0, 0, 0, 0, 28, 1837, 2009, 0, 483, 200, 0, 0, 2, 1121, 1426, 361, 163, 19, 0, 0, 0, 2354, 478, 1258, 0, 0, 0, 0, 2390, 2437, 190, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3635, 3630, 4041, 4706, 4557, 3092, 4090, 5017] avg=4096.0000 max=5017 max_rank=7 min=3092 min_rank=5 imbalance_ratio=1.224854 max_load_reduction=27751 max_load_reduction_pct=0.846893 moved_to_echo=16756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2021, 1, 2128, 296, 54, 0, 0, 0, 772, 0, 1809, 342, 1292, 0, 0, 0, 1648, 2297, 0, 75, 382, 0, 0, 0, 1214, 1405, 76, 961, 0, 0, 0, 0, 67, 1651, 1264, 0, 756, 124, 0, 0, 0, 1480, 1951, 474, 109, 11, 0, 0, 0, 2125, 747, 1213, 0, 0, 0, 0, 2243, 1518, 262, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4500, 4215, 4402, 3656, 3862, 4025, 4085, 4023] avg=4096.0000 max=4500 max_rank=0 min=3656 min_rank=3 imbalance_ratio=1.098633 max_load_reduction=28268 max_load_reduction_pct=0.862671 moved_to_echo=15995 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2049, 1, 2218, 206, 47, 0, 0, 0, 904, 2, 1374, 262, 1060, 0, 0, 0, 1168, 1608, 0, 59, 394, 0, 0, 0, 1516, 1055, 107, 796, 0, 0, 0, 0, 30, 1803, 1903, 1, 623, 130, 0, 0, 5, 1391, 2300, 1210, 86, 16, 0, 0, 0, 2145, 604, 1758, 0, 0, 0, 0, 2079, 1533, 318, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4521, 3602, 3229, 3474, 4490, 5008, 4507, 3937] avg=4096.0000 max=5008 max_rank=5 min=3229 min_rank=2 imbalance_ratio=1.222656 max_load_reduction=27760 max_load_reduction_pct=0.847168 moved_to_echo=17942 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1957, 2, 1856, 255, 61, 0, 0, 0, 1025, 5, 2105, 200, 1157, 0, 0, 0, 1430, 2285, 0, 110, 521, 0, 0, 0, 1056, 1570, 187, 1291, 0, 0, 0, 0, 40, 1715, 941, 0, 677, 169, 0, 0, 6, 1678, 1999, 421, 122, 16, 0, 0, 0, 1373, 669, 1246, 0, 0, 0, 0, 2338, 2066, 219, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4492, 4346, 4104, 3542, 4242, 3288, 4623] avg=4096.0000 max=4623 max_rank=7 min=3288 min_rank=6 imbalance_ratio=1.128662 max_load_reduction=28145 max_load_reduction_pct=0.858917 moved_to_echo=15695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[15488, 128, 15104, 1792, 640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10752, 17664, 128, 640, 3968, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[15488, 15488, 1920, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7296, 128, 14848, 1920, 8960, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[9216, 13312, 896, 9600, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 13952, 12160, 128, 5248, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 16512, 5120, 11392, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 12288, 14848, 4608, 1152, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[140, 0, 0, 0, 2510, 0, 0, 0, 2, 4076, 3967, 4034, 4030, 0, 214, 1359, 4096, 0, 0, 0, 346, 0, 0, 0, 4095, 18, 0, 0, 0, 3881, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[134, 1, 0, 0, 2467, 0, 1, 0, 3, 3932, 3689, 4055, 4062, 2, 101, 1596, 4091, 0, 0, 1, 515, 0, 1, 0, 4091, 9, 2, 0, 0, 4014, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[325, 0, 0, 0, 1914, 0, 0, 0, 1, 4064, 4072, 4071, 3997, 0, 39, 1536, 4096, 0, 0, 0, 469, 0, 0, 0, 4095, 7, 0, 0, 0, 4082, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[94, 0, 0, 0, 2822, 0, 0, 0, 0, 4058, 4055, 3981, 4059, 0, 14, 1021, 4096, 0, 0, 0, 459, 0, 0, 0, 4095, 11, 0, 0, 0, 4003, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[270, 0, 1, 1, 2811, 0, 0, 0, 0, 3946, 4056, 4058, 4038, 3, 68, 1135, 4096, 0, 0, 0, 191, 0, 0, 0, 4094, 13, 0, 0, 0, 3986, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[57, 0, 1, 2, 806, 0, 0, 0, 0, 4072, 4094, 4080, 4039, 0, 43, 3236, 4096, 0, 0, 0, 109, 0, 0, 1, 4095, 12, 0, 0, 0, 4025, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[271, 0, 0, 0, 998, 0, 0, 0, 0, 4064, 4064, 4077, 4033, 0, 192, 2234, 4096, 0, 0, 0, 542, 0, 0, 0, 4096, 16, 0, 0, 0, 4085, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[598, 0, 0, 0, 2211, 0, 0, 0, 0, 3892, 4086, 3902, 3869, 0, 78, 1703, 4096, 0, 0, 0, 491, 0, 0, 0, 4086, 2, 0, 0, 0, 3754, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[140, 0, 0, 0, 3860, 0, 0, 0, 2510, 0, 0, 0, 1638, 401, 0, 0, 2, 88, 3967, 0, 0, 0, 0, 0, 2283, 0, 214, 1359, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 346, 0, 0, 0, 3587, 174, 0, 0, 4087, 18, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 109, 8, 0, 0] rerouted_tokens_per_ep_rank=[4000, 4549, 4057, 3856, 4096, 4107, 4105, 3998] avg=4096.0000 max=4549 max_rank=1 min=3856 min_rank=3 imbalance_ratio=1.110596 max_load_reduction=28219 max_load_reduction_pct=0.861176 moved_to_echo=16306 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[325, 0, 0, 0, 3897, 0, 0, 0, 1914, 0, 0, 0, 1625, 400, 0, 0, 1, 87, 4072, 0, 0, 0, 0, 0, 2264, 0, 39, 1536, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 469, 0, 0, 0, 3577, 174, 0, 0, 4087, 7, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 108, 8, 0, 0] rerouted_tokens_per_ep_rank=[4222, 3939, 4160, 3839, 4096, 4220, 4094, 4198] avg=4096.0000 max=4222 max_rank=0 min=3839 min_rank=3 imbalance_ratio=1.030762 max_load_reduction=28546 max_load_reduction_pct=0.871155 moved_to_echo=16608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[270, 0, 1, 1, 3883, 0, 0, 0, 2811, 0, 0, 0, 1641, 389, 0, 0, 0, 84, 4056, 0, 0, 0, 0, 0, 2288, 3, 68, 1135, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 0, 3473, 175, 0, 0, 4086, 13, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 1, 109, 8, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4841, 4140, 3494, 4096, 3839, 4099, 4104] avg=4096.0000 max=4841 max_rank=1 min=3494 min_rank=3 imbalance_ratio=1.181885 max_load_reduction=27927 max_load_reduction_pct=0.852264 moved_to_echo=16138 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[134, 1, 0, 0, 3881, 0, 0, 0, 2467, 0, 1, 0, 1654, 392, 0, 0, 3, 75, 3689, 0, 0, 0, 0, 0, 2295, 2, 101, 1596, 0, 0, 0, 0, 4091, 0, 0, 1, 4, 0, 0, 0, 515, 0, 1, 0, 3465, 174, 0, 0, 4078, 9, 2, 0, 0, 0, 0, 0, 0, 4014, 1, 0, 113, 9, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4514, 3767, 3994, 4096, 4155, 4089, 4137] avg=4096.0000 max=4514 max_rank=1 min=3767 min_rank=2 imbalance_ratio=1.102051 max_load_reduction=28254 max_load_reduction_pct=0.862244 moved_to_echo=16477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[94, 0, 0, 0, 3810, 0, 0, 0, 2822, 0, 0, 0, 1650, 400, 0, 0, 0, 87, 4055, 0, 0, 0, 0, 0, 2299, 0, 14, 1021, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 0, 3571, 171, 0, 0, 4087, 11, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 110, 8, 0, 0] rerouted_tokens_per_ep_rank=[3904, 4872, 4142, 3334, 4096, 4201, 4098, 4121] avg=4096.0000 max=4872 max_rank=1 min=3334 min_rank=3 imbalance_ratio=1.189453 max_load_reduction=27896 max_load_reduction_pct=0.851318 moved_to_echo=16516 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[271, 0, 0, 0, 3902, 0, 0, 0, 998, 0, 0, 0, 1639, 400, 0, 0, 0, 87, 4064, 0, 0, 0, 0, 0, 2285, 0, 192, 2234, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 542, 0, 0, 0, 3577, 175, 0, 0, 4088, 16, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 109, 8, 0, 0] rerouted_tokens_per_ep_rank=[4173, 3037, 4151, 4711, 4096, 4294, 4104, 4202] avg=4096.0000 max=4711 max_rank=3 min=3037 min_rank=1 imbalance_ratio=1.150146 max_load_reduction=28057 max_load_reduction_pct=0.856232 moved_to_echo=16696 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[598, 0, 0, 0, 3735, 0, 0, 0, 2211, 0, 0, 0, 1573, 383, 0, 0, 0, 84, 4086, 0, 0, 0, 0, 0, 2192, 0, 78, 1703, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 491, 0, 0, 0, 3425, 167, 0, 0, 4078, 2, 0, 0, 0, 0, 0, 0, 0, 3754, 0, 0, 104, 8, 0, 0] rerouted_tokens_per_ep_rank=[4333, 4167, 4170, 3973, 4096, 4083, 4080, 3866] avg=4096.0000 max=4333 max_rank=0 min=3866 min_rank=7 imbalance_ratio=1.057861 max_load_reduction=28435 max_load_reduction_pct=0.867767 moved_to_echo=16125 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[57, 0, 1, 2, 3905, 0, 0, 0, 806, 0, 0, 0, 1642, 401, 0, 0, 0, 87, 4094, 0, 0, 0, 0, 0, 2288, 0, 43, 3236, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 1, 3584, 175, 0, 0, 4087, 12, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 109, 8, 0, 0] rerouted_tokens_per_ep_rank=[3965, 2849, 4181, 5567, 4096, 3869, 4099, 4142] avg=4096.0000 max=5567 max_rank=3 min=2849 min_rank=1 imbalance_ratio=1.359131 max_load_reduction=27201 max_load_reduction_pct=0.830109 moved_to_echo=16206 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16640, 0, 128, 0, 13184, 3200, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[18304, 128, 768, 13824, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3200, 0, 128, 128, 28288, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1920, 128, 128, 128, 30976, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 31872, 128, 128, 896, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 768, 32128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 17, 0, 3816, 0, 4096, 0, 1629, 3952, 3966, 0, 4096, 0, 1061, 0, 0, 0, 0, 0, 0, 57, 4096, 5, 0, 4026, 0, 0, 0, 1946, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 63, 0, 1955, 0, 4096, 1, 1321, 4053, 3953, 0, 4096, 0, 2753, 6, 0, 0, 0, 0, 0, 173, 4096, 13, 0, 4090, 0, 0, 0, 2093, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 24, 0, 3561, 0, 4096, 4, 1031, 4034, 4053, 0, 4096, 0, 1657, 1, 0, 0, 0, 0, 0, 82, 4096, 0, 0, 4077, 0, 0, 0, 1954, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 24, 0, 2680, 0, 4096, 0, 1612, 4040, 4016, 0, 4096, 0, 983, 7, 0, 0, 0, 0, 0, 61, 4096, 0, 0, 4088, 0, 0, 0, 2968, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 2, 24, 0, 3122, 0, 4096, 0, 1752, 3902, 4057, 0, 4095, 0, 1487, 5, 0, 0, 0, 0, 0, 43, 4095, 23, 0, 4069, 1, 0, 1, 1992, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 56, 0, 3595, 0, 4096, 0, 1631, 3787, 4080, 0, 4095, 0, 1283, 9, 0, 0, 0, 0, 0, 102, 4096, 9, 0, 4053, 0, 0, 0, 1875, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1, 3, 17, 0, 3783, 2, 4096, 0, 1362, 4036, 3800, 0, 4096, 0, 1805, 16, 0, 0, 0, 0, 0, 44, 4096, 0, 0, 4090, 0, 0, 0, 1521, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 14, 1, 1928, 0, 4095, 0, 1523, 4060, 4013, 0, 4095, 1, 1960, 0, 1, 1, 0, 1, 1, 244, 4095, 97, 0, 4048, 0, 0, 0, 2582, 0, 6] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 17, 0, 2952, 1112, 0, 0, 3816, 0, 1041, 0, 0, 0, 0, 0, 1629, 2594, 0, 0, 0, 0, 0, 0, 2468, 0, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 103, 0, 0, 0, 57, 3978, 5, 0, 0, 0, 0, 0, 4026, 0, 0, 28, 0, 0, 0, 0, 1946, 0, 4, 1358, 516, 90, 0] rerouted_tokens_per_ep_rank=[4082, 4857, 4223, 3529, 4069, 4040, 4054, 3914] avg=4096.0000 max=4857 max_rank=1 min=3529 min_rank=3 imbalance_ratio=1.185791 max_load_reduction=27911 max_load_reduction_pct=0.851776 moved_to_echo=16077 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 63, 0, 2952, 1112, 0, 0, 1955, 0, 1041, 1, 0, 0, 0, 0, 1321, 2660, 0, 0, 0, 0, 0, 0, 2468, 0, 2753, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 103, 0, 0, 0, 173, 3978, 13, 0, 0, 0, 0, 0, 4090, 0, 0, 28, 0, 0, 0, 0, 2093, 0, 6, 1393, 516, 90, 0] rerouted_tokens_per_ep_rank=[4127, 2997, 3981, 5227, 4056, 4164, 4118, 4098] avg=4096.0000 max=5227 max_rank=3 min=2997 min_rank=1 imbalance_ratio=1.276123 max_load_reduction=27541 max_load_reduction_pct=0.840485 moved_to_echo=16436 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 56, 0, 2952, 1111, 0, 0, 3595, 0, 1041, 0, 0, 0, 0, 0, 1631, 2485, 0, 0, 0, 0, 0, 0, 2468, 0, 1283, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 103, 0, 0, 0, 102, 3978, 9, 0, 0, 0, 0, 0, 4053, 0, 0, 28, 0, 0, 0, 0, 1875, 0, 0, 1302, 516, 90, 0] rerouted_tokens_per_ep_rank=[4120, 4636, 4116, 3760, 4183, 4089, 4081, 3783] avg=4096.0000 max=4636 max_rank=1 min=3760 min_rank=3 imbalance_ratio=1.131836 max_load_reduction=28132 max_load_reduction_pct=0.858521 moved_to_echo=16136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 24, 0, 2952, 1112, 0, 0, 3561, 0, 1041, 4, 0, 0, 0, 0, 1031, 2647, 0, 0, 0, 0, 0, 0, 2468, 0, 1657, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 103, 0, 0, 0, 82, 3978, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 28, 0, 0, 0, 0, 1954, 0, 2, 1387, 516, 90, 0] rerouted_tokens_per_ep_rank=[4088, 4606, 3678, 4126, 4156, 4060, 4105, 3949] avg=4096.0000 max=4606 max_rank=1 min=3678 min_rank=2 imbalance_ratio=1.124512 max_load_reduction=28162 max_load_reduction_pct=0.859436 moved_to_echo=16270 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 24, 0, 2952, 1111, 0, 0, 3122, 0, 1041, 0, 0, 0, 0, 0, 1752, 2561, 0, 0, 0, 0, 0, 0, 2468, 0, 1487, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 103, 0, 0, 0, 43, 3977, 23, 0, 0, 0, 0, 0, 4069, 1, 0, 28, 0, 0, 0, 1, 1992, 0, 1, 1341, 516, 90, 0] rerouted_tokens_per_ep_rank=[4090, 4163, 4313, 3960, 4160, 4043, 4098, 3941] avg=4096.0000 max=4313 max_rank=2 min=3941 min_rank=7 imbalance_ratio=1.052979 max_load_reduction=28455 max_load_reduction_pct=0.868378 moved_to_echo=16242 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 24, 0, 2952, 1112, 0, 0, 2680, 0, 1041, 0, 0, 0, 0, 0, 1612, 2651, 0, 0, 0, 0, 0, 0, 2468, 0, 983, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 103, 0, 0, 0, 61, 3978, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 28, 0, 0, 0, 0, 2968, 0, 0, 1389, 516, 90, 0] rerouted_tokens_per_ep_rank=[4089, 3721, 4263, 3458, 4119, 4039, 4116, 4963] avg=4096.0000 max=4963 max_rank=7 min=3458 min_rank=3 imbalance_ratio=1.211670 max_load_reduction=27805 max_load_reduction_pct=0.848541 moved_to_echo=17237 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 14, 1, 2955, 1115, 0, 0, 1928, 0, 1034, 0, 0, 0, 0, 0, 1523, 2660, 0, 0, 0, 0, 0, 0, 2458, 1, 1960, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4013, 106, 0, 0, 1, 244, 3969, 97, 0, 0, 0, 0, 0, 4048, 0, 0, 30, 0, 0, 0, 0, 2582, 0, 6, 1400, 522, 96, 0] rerouted_tokens_per_ep_rank=[4087, 2962, 4183, 4419, 4122, 4311, 4078, 4606] avg=4096.0000 max=4606 max_rank=7 min=2962 min_rank=1 imbalance_ratio=1.124512 max_load_reduction=28162 max_load_reduction_pct=0.859436 moved_to_echo=17117 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 3, 17, 0, 2952, 1112, 0, 0, 3783, 2, 1041, 0, 0, 0, 0, 0, 1362, 2649, 0, 0, 0, 0, 0, 0, 2468, 0, 1805, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 103, 0, 0, 0, 44, 3978, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 28, 0, 0, 0, 0, 1521, 0, 0, 1387, 516, 90, 0] rerouted_tokens_per_ep_rank=[4085, 4826, 4011, 4289, 3903, 4022, 4118, 3514] avg=4096.0000 max=4826 max_rank=1 min=3514 min_rank=7 imbalance_ratio=1.178223 max_load_reduction=27942 max_load_reduction_pct=0.852722 moved_to_echo=15557 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 17024, 0, 128, 11008, 4224, 768, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 256, 128, 23680, 8960, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[11904, 20992, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[24448, 128, 8448, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[19840, 128, 13056, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32000, 896, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 896, 31872, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32640, 128, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2294, 0, 1422, 0, 3954, 0, 0, 0, 0, 0, 0, 4096, 1887, 0, 4051, 1, 0, 0, 3372, 2, 6, 3885, 3692, 1, 0, 9, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2560, 0, 1417, 0, 4064, 0, 0, 0, 0, 0, 0, 4096, 1264, 0, 4003, 3, 0, 0, 3678, 4, 5, 3615, 3960, 0, 0, 3, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2689, 0, 1337, 0, 4084, 0, 0, 0, 0, 0, 0, 4096, 1807, 0, 4095, 2, 0, 0, 3428, 9, 5, 3536, 3583, 1, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2786, 0, 804, 0, 3950, 0, 0, 0, 0, 0, 0, 4096, 2373, 0, 4087, 0, 0, 0, 3339, 0, 2, 3522, 3712, 2, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2796, 0, 1278, 0, 4033, 0, 0, 0, 0, 0, 0, 4096, 1912, 0, 4078, 2, 0, 0, 2638, 1, 3, 3947, 3884, 0, 0, 5, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3330, 0, 830, 0, 4077, 0, 0, 0, 0, 0, 0, 4096, 1637, 0, 4090, 0, 0, 0, 2532, 9, 0, 4068, 3997, 0, 0, 8, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2656, 0, 934, 0, 4079, 0, 0, 0, 0, 0, 0, 4096, 1896, 0, 4083, 9, 0, 0, 3526, 7, 17, 3793, 3576, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 1, 3008, 0, 874, 0, 4082, 1, 0, 1, 0, 1, 0, 4095, 2134, 1, 4081, 0, 0, 0, 3134, 33, 1, 3509, 3714, 0, 1, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 2689, 0, 1337, 220, 0, 0, 0, 0, 4084, 0, 0, 52, 3, 0, 0, 0, 0, 0, 0, 3251, 683, 1, 0, 0, 1807, 0, 2247, 0, 0, 0, 0, 2, 0, 0, 3428, 893, 0, 0, 0, 9, 5, 3536, 332, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4246, 4139, 3935, 4054, 4323, 3882, 4094] avg=4096.0000 max=4323 max_rank=5 min=3882 min_rank=6 imbalance_ratio=1.055420 max_load_reduction=28445 max_load_reduction_pct=0.868073 moved_to_echo=16353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 2560, 0, 1417, 215, 0, 0, 0, 0, 4064, 0, 0, 51, 3, 0, 0, 0, 0, 0, 0, 3593, 668, 0, 0, 0, 1264, 0, 2196, 0, 0, 0, 0, 3, 0, 0, 3678, 873, 0, 0, 0, 4, 5, 3615, 367, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4192, 4118, 4261, 3460, 4554, 3991, 4096] avg=4096.0000 max=4554 max_rank=5 min=3460 min_rank=4 imbalance_ratio=1.111816 max_load_reduction=28214 max_load_reduction_pct=0.861023 moved_to_echo=16101 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 2786, 0, 804, 219, 0, 0, 0, 0, 3950, 0, 0, 52, 3, 0, 0, 0, 0, 0, 0, 3368, 682, 0, 0, 0, 2373, 0, 2243, 0, 0, 0, 0, 0, 0, 0, 3339, 891, 0, 0, 0, 0, 2, 3522, 344, 0, 0, 0, 0, 2, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3809, 4005, 4050, 4616, 4230, 3868, 4094] avg=4096.0000 max=4616 max_rank=4 min=3809 min_rank=1 imbalance_ratio=1.126953 max_load_reduction=28152 max_load_reduction_pct=0.859131 moved_to_echo=16808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 2796, 0, 1278, 219, 0, 0, 0, 0, 4033, 0, 0, 52, 3, 0, 0, 0, 0, 0, 0, 3524, 680, 0, 0, 0, 1912, 0, 2238, 0, 0, 0, 0, 2, 0, 0, 2638, 889, 0, 0, 0, 1, 3, 3947, 360, 0, 0, 0, 0, 0, 0, 5, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4293, 4088, 4204, 4150, 3529, 4311, 4097] avg=4096.0000 max=4311 max_rank=6 min=3529 min_rank=5 imbalance_ratio=1.052490 max_load_reduction=28457 max_load_reduction_pct=0.868439 moved_to_echo=16087 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 2294, 0, 1422, 217, 0, 0, 0, 0, 3954, 0, 0, 52, 3, 0, 0, 0, 0, 0, 0, 3350, 676, 0, 0, 0, 1887, 0, 2223, 0, 0, 0, 0, 1, 0, 0, 3372, 883, 0, 0, 0, 2, 6, 3885, 342, 0, 0, 0, 0, 1, 0, 9, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3933, 4009, 4026, 4110, 4256, 4235, 4103] avg=4096.0000 max=4256 max_rank=5 min=3933 min_rank=1 imbalance_ratio=1.039062 max_load_reduction=28512 max_load_reduction_pct=0.870117 moved_to_echo=16704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 3330, 0, 830, 220, 0, 0, 0, 0, 4077, 0, 0, 52, 3, 0, 0, 0, 0, 0, 0, 3626, 682, 0, 0, 0, 1637, 0, 2244, 0, 0, 0, 0, 0, 0, 0, 2532, 892, 0, 0, 0, 9, 0, 4068, 371, 0, 0, 0, 0, 0, 0, 8, 4091, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4380, 4132, 4308, 3881, 3424, 4448, 4099] avg=4096.0000 max=4448 max_rank=6 min=3424 min_rank=5 imbalance_ratio=1.085938 max_load_reduction=28320 max_load_reduction_pct=0.864258 moved_to_echo=15852 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 0, 0, 0, 1, 3008, 0, 874, 223, 0, 0, 0, 0, 4082, 1, 0, 56, 4, 0, 0, 1, 0, 1, 0, 3373, 684, 0, 0, 0, 2134, 1, 2225, 0, 0, 0, 0, 0, 0, 0, 3134, 893, 0, 0, 0, 33, 1, 3509, 341, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4106, 4143, 4059, 4360, 4027, 3884, 4092] avg=4096.0000 max=4360 max_rank=4 min=3884 min_rank=6 imbalance_ratio=1.064453 max_load_reduction=28408 max_load_reduction_pct=0.866943 moved_to_echo=16363 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 2656, 0, 934, 219, 0, 0, 0, 0, 4079, 0, 0, 52, 3, 0, 0, 0, 0, 0, 0, 3244, 681, 0, 0, 0, 1896, 0, 2241, 0, 0, 0, 0, 9, 0, 0, 3526, 890, 0, 0, 0, 7, 17, 3793, 332, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3809, 4134, 3925, 4137, 4425, 4149, 4093] avg=4096.0000 max=4425 max_rank=5 min=3809 min_rank=1 imbalance_ratio=1.080322 max_load_reduction=28343 max_load_reduction_pct=0.864960 moved_to_echo=16804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 14976, 128, 17920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 25728, 7168, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 22144, 0, 8960, 1792, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 27392, 5504, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 29952, 2816, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 32384, 128, 0, 512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2038, 2067, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4064, 0, 4049, 0, 1, 76] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 699, 3493, 4096, 0, 0, 2, 0, 4095, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 4027, 0, 1, 4046, 0, 4002, 0, 0, 105] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2087, 2039, 4096, 0, 0, 2, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 0, 4085, 0, 4071, 0, 0, 99] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 815, 3320, 4096, 0, 0, 11, 0, 4095, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4065, 0, 1, 4058, 0, 4050, 0, 0, 62] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 900, 3303, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4066, 0, 1, 4031, 0, 4061, 0, 0, 22] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 508, 3615, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 4090, 0, 4086, 0, 0, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 572, 3552, 4096, 0, 0, 5, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 5, 4089, 0, 4064, 0, 0, 45] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 983, 3055, 4095, 0, 1, 0, 0, 4095, 1, 0, 1, 4095, 7, 0, 0, 0, 1, 0, 0, 4072, 0, 0, 4095, 0, 3953, 1, 0, 215] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2038, 2044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4086, 23, 3, 2, 0, 0, 0, 4064, 0, 24, 0, 0, 0, 4023, 0, 1, 76, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4082, 4096, 4096, 4096, 4114, 4088, 4100] avg=4096.0000 max=4114 max_rank=5 min=4082 min_rank=1 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2087, 2016, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4004, 23, 3, 2, 0, 0, 0, 4085, 0, 24, 0, 0, 0, 4045, 0, 0, 99, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4103, 4095, 4096, 4093, 4032, 4109, 4144] avg=4096.0000 max=4144 max_rank=7 min=4032 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16378 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 815, 3283, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4065, 37, 3, 2, 0, 0, 1, 4058, 0, 24, 0, 0, 0, 4024, 0, 0, 62, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4098, 4106, 4096, 4096, 4107, 4083, 4086] avg=4096.0000 max=4107 max_rank=5 min=4083 min_rank=6 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 699, 3454, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4027, 39, 3, 2, 0, 0, 1, 4046, 0, 23, 0, 0, 0, 3977, 0, 0, 105, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4153, 4097, 4096, 4103, 4071, 4070, 4082] avg=4096.0000 max=4153 max_rank=1 min=4070 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16326 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 900, 3266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4066, 37, 3, 2, 0, 0, 1, 4031, 0, 24, 0, 0, 0, 4035, 0, 0, 22, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4166, 4095, 4096, 4094, 4108, 4056, 4057] avg=4096.0000 max=4166 max_rank=1 min=4056 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 508, 3574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4067, 41, 3, 2, 0, 0, 0, 4090, 0, 24, 0, 0, 0, 4060, 0, 0, 17, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4082, 4096, 4096, 4094, 4113, 4114, 4077] avg=4096.0000 max=4114 max_rank=6 min=4077 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4093, 0, 0, 0, 0, 983, 3017, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4088, 0, 0, 0, 0, 1, 0, 1, 4094, 0, 0, 0, 0, 7, 0, 0, 0, 4092, 0, 0, 0, 1, 0, 0, 4072, 38, 3, 5, 0, 0, 0, 4095, 0, 25, 7, 2, 1, 3923, 1, 0, 215, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4000, 4089, 4096, 4099, 4119, 4130, 4139] avg=4096.0000 max=4139 max_rank=7 min=4000 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 572, 3512, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4059, 40, 3, 2, 0, 0, 5, 4089, 0, 24, 0, 0, 0, 4038, 0, 0, 45, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4084, 4094, 4096, 4093, 4104, 4118, 4083] avg=4096.0000 max=4118 max_rank=6 min=4083 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32128, 128, 128, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[8704, 24192, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 32640, 0, 256, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 0, 32512, 384, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4016, 0, 0, 1014, 4090, 0, 0, 0, 937, 0, 0, 0, 1005, 1669, 3920, 0, 1183, 4088, 0, 4096, 2638, 0, 4096, 0, 0, 3, 13, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 3227, 0, 0, 871, 4090, 1, 0, 0, 524, 0, 0, 1, 1345, 3197, 3873, 0, 677, 4060, 0, 4096, 2700, 0, 4095, 0, 0, 1, 10, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 3478, 0, 0, 1487, 4094, 0, 0, 0, 545, 0, 0, 0, 1449, 2933, 3847, 0, 976, 4081, 0, 4096, 1603, 0, 4095, 0, 0, 1, 83, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 3491, 0, 0, 1775, 4094, 2, 0, 0, 575, 0, 0, 0, 1861, 2480, 3834, 0, 541, 4029, 0, 4095, 1846, 0, 4096, 0, 0, 15, 34, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 3093, 0, 0, 832, 4095, 0, 0, 0, 1683, 0, 0, 0, 550, 3244, 3699, 0, 278, 4082, 0, 4096, 3008, 0, 4096, 0, 0, 1, 11, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 3356, 0, 0, 325, 4095, 0, 0, 0, 477, 0, 0, 0, 1967, 3347, 3767, 0, 1589, 4081, 0, 4096, 1554, 0, 4096, 0, 0, 1, 17, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 3786, 0, 1, 274, 4095, 0, 1, 1, 716, 0, 1, 0, 1223, 3234, 3942, 1, 559, 4072, 1, 4094, 2638, 1, 4095, 1, 0, 26, 5, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 3859, 0, 1, 638, 4082, 1, 0, 0, 1653, 0, 0, 0, 917, 2863, 3888, 0, 561, 4053, 0, 4096, 1983, 0, 4096, 0, 0, 0, 77, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4016, 0, 0, 495, 37, 0, 0, 1014, 3192, 0, 0, 0, 0, 0, 0, 0, 937, 0, 0, 2033, 898, 276, 0, 0, 1005, 1632, 0, 0, 0, 0, 0, 0, 1183, 3317, 0, 0, 0, 0, 0, 0, 2638, 0, 1851, 0, 0, 0, 0, 0, 0, 3, 13, 3920, 212, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4548, 4206, 4144, 2637, 4500, 4489, 4148, 4096] avg=4096.0000 max=4548 max_rank=0 min=2637 min_rank=3 imbalance_ratio=1.110352 max_load_reduction=28220 max_load_reduction_pct=0.861206 moved_to_echo=17233 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3478, 0, 0, 494, 66, 0, 0, 1487, 3196, 0, 0, 0, 0, 0, 0, 0, 545, 0, 0, 2033, 898, 275, 0, 0, 1449, 2867, 0, 0, 0, 0, 0, 0, 976, 3312, 0, 0, 0, 0, 0, 0, 1603, 0, 1850, 0, 0, 0, 0, 0, 0, 1, 83, 3846, 212, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[4038, 4683, 3751, 4316, 4288, 3453, 4142, 4097] avg=4096.0000 max=4683 max_rank=1 min=3453 min_rank=5 imbalance_ratio=1.143311 max_load_reduction=28085 max_load_reduction_pct=0.857086 moved_to_echo=15980 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3227, 0, 0, 491, 71, 0, 0, 871, 3192, 1, 0, 0, 0, 0, 0, 0, 524, 0, 0, 2033, 898, 274, 0, 1, 1345, 3126, 0, 0, 0, 0, 0, 0, 677, 3295, 0, 0, 0, 0, 0, 0, 2700, 0, 1850, 0, 0, 0, 0, 0, 0, 1, 10, 3873, 212, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3789, 4064, 3729, 4472, 3972, 4550, 4096, 4096] avg=4096.0000 max=4550 max_rank=5 min=3729 min_rank=2 imbalance_ratio=1.110840 max_load_reduction=28218 max_load_reduction_pct=0.861145 moved_to_echo=16714 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3491, 0, 0, 488, 55, 0, 0, 1775, 3196, 2, 0, 0, 0, 0, 0, 0, 575, 0, 0, 2033, 898, 272, 0, 0, 1861, 2425, 0, 0, 0, 0, 0, 0, 541, 3269, 0, 0, 0, 0, 0, 0, 1846, 0, 1851, 0, 0, 0, 0, 0, 0, 15, 34, 3834, 212, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4973, 3778, 4286, 3810, 3697, 4095, 4095] avg=4096.0000 max=4973 max_rank=1 min=3697 min_rank=5 imbalance_ratio=1.214111 max_load_reduction=27795 max_load_reduction_pct=0.848236 moved_to_echo=15697 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3356, 0, 0, 494, 75, 0, 0, 325, 3196, 0, 0, 0, 0, 0, 0, 0, 477, 0, 0, 2033, 899, 275, 0, 0, 1967, 3272, 0, 0, 0, 0, 0, 0, 1589, 3312, 0, 0, 0, 0, 0, 0, 1554, 0, 1851, 0, 0, 0, 0, 0, 0, 1, 17, 3766, 212, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0] rerouted_tokens_per_ep_rank=[3925, 3521, 3684, 5239, 4901, 3405, 3996, 4097] avg=4096.0000 max=5239 max_rank=3 min=3405 min_rank=5 imbalance_ratio=1.279053 max_load_reduction=27529 max_load_reduction_pct=0.840118 moved_to_echo=16399 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3093, 0, 0, 494, 73, 0, 0, 832, 3196, 0, 0, 0, 0, 0, 0, 0, 1683, 0, 0, 2033, 899, 276, 0, 0, 550, 3171, 0, 0, 0, 0, 0, 0, 278, 3312, 0, 0, 0, 0, 0, 0, 3008, 0, 1851, 0, 0, 0, 0, 0, 0, 1, 11, 3699, 212, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3660, 4028, 4891, 3721, 3590, 4859, 3923, 4096] avg=4096.0000 max=4891 max_rank=2 min=3590 min_rank=4 imbalance_ratio=1.194092 max_load_reduction=27877 max_load_reduction_pct=0.850739 moved_to_echo=16468 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3786, 0, 1, 496, 76, 0, 0, 274, 3193, 0, 1, 0, 0, 0, 0, 1, 716, 0, 1, 2036, 902, 279, 0, 0, 1223, 3158, 0, 0, 0, 0, 0, 1, 559, 3297, 1, 0, 0, 0, 0, 0, 2638, 1, 1842, 0, 0, 0, 0, 1, 0, 26, 5, 3942, 217, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4359, 3468, 3935, 4381, 3858, 4481, 4191, 4095] avg=4096.0000 max=4481 max_rank=5 min=3468 min_rank=1 imbalance_ratio=1.093994 max_load_reduction=28287 max_load_reduction_pct=0.863251 moved_to_echo=16625 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3859, 0, 1, 491, 64, 0, 0, 638, 3186, 1, 0, 0, 0, 0, 0, 0, 1653, 0, 0, 2033, 896, 274, 0, 0, 917, 2799, 0, 0, 0, 0, 0, 0, 561, 3288, 0, 0, 0, 0, 0, 0, 1983, 0, 1851, 0, 0, 0, 0, 0, 0, 0, 77, 3888, 212, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4415, 3825, 4856, 3716, 3849, 3834, 4177, 4096] avg=4096.0000 max=4856 max_rank=2 min=3716 min_rank=3 imbalance_ratio=1.185547 max_load_reduction=27912 max_load_reduction_pct=0.851807 moved_to_echo=15956 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 7168, 0, 128, 16384, 7296, 2304, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 6400, 26496, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 18048, 128, 14848, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 10368, 22528, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 28416, 0, 128, 3968, 640, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7296, 25600, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 256, 30848, 1792, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 1, 125, 1763, 0, 3648, 0, 0, 4096, 4096, 1, 4096, 0, 0, 0, 4096, 2654, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 5, 0, 1, 0, 0, 186, 1762, 0, 3056, 1, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 3159, 0, 0, 24, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2, 0, 1, 0, 0, 334, 1963, 0, 3318, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 4096, 2559, 0, 0, 17, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 146, 1725, 0, 3505, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 2799, 0, 0, 17, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 0, 1, 172, 2456, 0, 2886, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 2659, 0, 0, 17, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 2, 0, 0, 242, 1710, 0, 3422, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 2815, 0, 0, 1, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 1, 0, 0, 0, 151, 2177, 1, 3062, 0, 1, 4095, 4095, 0, 4094, 0, 1, 1, 4095, 2798, 1, 0, 2, 0, 1, 4095, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3, 0, 0, 0, 0, 149, 1250, 0, 3483, 0, 0, 4096, 4093, 0, 4096, 0, 0, 0, 4096, 3307, 0, 0, 3, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 125, 1763, 0, 2056, 0, 0, 0, 3648, 0, 0, 799, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1253, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 3297, 787, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 3945, 4447, 4098, 3907, 4085, 4096] avg=4096.0000 max=4447 max_rank=3 min=3907 min_rank=5 imbalance_ratio=1.085693 max_load_reduction=28321 max_load_reduction_pct=0.864288 moved_to_echo=16186 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 146, 1725, 0, 2056, 0, 0, 0, 3505, 0, 0, 799, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1253, 2799, 0, 0, 0, 0, 0, 0, 17, 0, 3297, 787, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 3927, 4304, 4097, 4052, 4102, 4096] avg=4096.0000 max=4304 max_rank=3 min=3927 min_rank=2 imbalance_ratio=1.050781 max_load_reduction=28464 max_load_reduction_pct=0.868652 moved_to_echo=16347 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 1, 0, 4094, 0, 0, 0, 0, 334, 1963, 0, 2056, 0, 0, 0, 3318, 0, 0, 799, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1253, 2559, 0, 0, 0, 0, 0, 0, 17, 0, 3297, 787, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4353, 4117, 4095, 3812, 4102, 4096] avg=4096.0000 max=4353 max_rank=2 min=3812 min_rank=5 imbalance_ratio=1.062744 max_load_reduction=28415 max_load_reduction_pct=0.867157 moved_to_echo=16105 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 1, 0, 4094, 0, 0, 0, 0, 186, 1762, 0, 2056, 0, 0, 0, 3056, 1, 0, 799, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1253, 3159, 0, 0, 0, 0, 0, 0, 24, 0, 3297, 787, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4100, 4004, 3856, 4096, 4412, 4109, 4095] avg=4096.0000 max=4412 max_rank=5 min=3856 min_rank=3 imbalance_ratio=1.077148 max_load_reduction=28356 max_load_reduction_pct=0.865356 moved_to_echo=16712 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 1, 172, 2456, 0, 2056, 0, 0, 0, 2886, 0, 0, 799, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1253, 2659, 0, 0, 0, 0, 0, 0, 17, 0, 3297, 787, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4685, 3685, 4097, 3912, 4102, 4096] avg=4096.0000 max=4685 max_rank=2 min=3685 min_rank=3 imbalance_ratio=1.143799 max_load_reduction=28083 max_load_reduction_pct=0.857025 moved_to_echo=16207 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 242, 1710, 0, 2056, 0, 0, 0, 3422, 0, 0, 799, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1253, 2815, 0, 0, 0, 0, 0, 0, 1, 0, 3297, 787, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4008, 4221, 4097, 4068, 4086, 4096] avg=4096.0000 max=4221 max_rank=3 min=4008 min_rank=2 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16347 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 151, 2177, 1, 2062, 0, 0, 0, 3062, 0, 1, 793, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1245, 2798, 0, 0, 0, 0, 1, 0, 2, 0, 3302, 788, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4095, 4391, 3856, 4095, 4045, 4093, 4097] avg=4096.0000 max=4391 max_rank=2 min=3856 min_rank=3 imbalance_ratio=1.072021 max_load_reduction=28377 max_load_reduction_pct=0.865997 moved_to_echo=16330 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 4094, 0, 0, 0, 0, 149, 1250, 0, 2056, 0, 0, 0, 3483, 0, 0, 799, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 3307, 0, 0, 0, 0, 0, 0, 3, 0, 3297, 787, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 3455, 4282, 4093, 4560, 4089, 4096] avg=4096.0000 max=4560 max_rank=5 min=3455 min_rank=2 imbalance_ratio=1.113281 max_load_reduction=28208 max_load_reduction_pct=0.860840 moved_to_echo=16838 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 1536, 14848, 128, 16512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 26496, 6400, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 10112, 22784, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[26496, 128, 128, 6400, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 4094, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4091, 4095, 0, 4091, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4090, 4096, 0, 4081, 0, 0, 4096, 4096, 0, 21, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4091, 4095, 0, 4091, 0, 0, 4096, 4096, 0, 7, 0, 0, 1, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4094, 0, 4093, 0, 0, 4096, 4096, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4073, 4092, 0, 4093, 0, 0, 4096, 4096, 0, 28, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 4095, 4092, 0, 4095, 0, 1, 4094, 4095, 1, 3, 1, 0, 1, 0, 4095, 0, 1, 4095, 4092, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4075, 4093, 0, 4035, 0, 0, 4096, 4096, 0, 82, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 8, 0, 0, 0, 0, 0, 4081, 0, 0, 11, 0, 0, 0, 4078, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4092, 4099, 4096, 4096, 4096, 4096, 4095] avg=4096.0000 max=4099 max_rank=2 min=4092 min_rank=1 imbalance_ratio=1.000732 max_load_reduction=28669 max_load_reduction_pct=0.874908 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 8, 0, 0, 0, 0, 0, 4091, 0, 0, 11, 0, 0, 0, 4078, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 7, 1, 0, 0, 0, 1, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4102, 4087, 4096, 4096, 4096, 4095, 4097] avg=4096.0000 max=4102 max_rank=1 min=4087 min_rank=2 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 8, 0, 0, 0, 0, 0, 4094, 0, 0, 11, 0, 0, 0, 4078, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4105, 4080, 4096, 4096, 4096, 4096, 4095] avg=4096.0000 max=4105 max_rank=1 min=4080 min_rank=2 imbalance_ratio=1.002197 max_load_reduction=28663 max_load_reduction_pct=0.874725 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 8, 0, 0, 0, 0, 0, 4091, 0, 0, 11, 0, 0, 0, 4078, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4087, 7, 1, 0, 0, 0, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4102, 4085, 4097, 4096, 4096, 4097, 4096] avg=4096.0000 max=4102 max_rank=1 min=4085 min_rank=2 imbalance_ratio=1.001465 max_load_reduction=28666 max_load_reduction_pct=0.874817 moved_to_echo=16385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4073, 8, 0, 0, 0, 0, 0, 4093, 0, 0, 11, 0, 0, 0, 4078, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4084, 7, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4104, 4106, 4095, 4096, 4096, 4094, 4096] avg=4096.0000 max=4106 max_rank=2 min=4081 min_rank=0 imbalance_ratio=1.002441 max_load_reduction=28662 max_load_reduction_pct=0.874695 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4095, 5, 0, 0, 0, 0, 0, 4095, 0, 1, 17, 0, 0, 0, 4064, 0, 1, 3, 0, 0, 0, 0, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4092, 0, 0, 1, 3, 0, 0, 0, 0, 3, 0, 0, 4087, 10, 1, 0, 0, 0, 0, 1, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4113, 4068, 4097, 4096, 4096, 4101, 4095] avg=4096.0000 max=4113 max_rank=1 min=4068 min_rank=2 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 8, 0, 0, 0, 0, 0, 4093, 0, 0, 11, 0, 0, 0, 4078, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4086, 7, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4104, 4083, 4095, 4096, 4096, 4096, 4096] avg=4096.0000 max=4104 max_rank=1 min=4083 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=28664 max_load_reduction_pct=0.874756 moved_to_echo=16384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4075, 8, 0, 0, 0, 0, 0, 4035, 0, 0, 11, 0, 0, 0, 4078, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 7, 1, 0, 0, 0, 3, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4046, 4160, 4096, 4096, 4096, 4093, 4098] avg=4096.0000 max=4160 max_rank=2 min=4046 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 0, 128, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4096, 0, 0, 0, 4096, 4096, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4096, 0, 0, 0, 4093, 4095, 4094, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 3, 0, 4095, 0, 1, 1, 4095, 4089, 3868, 1, 1, 1, 0, 4086, 0, 0, 12, 268, 4095, 1, 1, 1, 3, 11, 0, 0, 0, 4078, 0, 4056] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4096, 0, 0, 0, 4094, 4095, 4093, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4096, 0, 0, 0, 4091, 4095, 4092, 0, 0, 0, 0, 4096, 0, 0, 4, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 0, 0, 4095, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4095, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4060, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 1, 0, 0, 6, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4121, 4096, 4066, 4096, 4094, 4102] avg=4096.0000 max=4121 max_rank=2 min=4066 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16358 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4090, 3, 1, 0, 0, 6, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4129, 4096, 4059, 4096, 4095, 4102] avg=4096.0000 max=4129 max_rank=2 min=4059 min_rank=4 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16352 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 1, 4057, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4090, 3, 1, 0, 0, 6, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4127, 4095, 4059, 4096, 4096, 4102] avg=4096.0000 max=4127 max_rank=2 min=4059 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 1, 4058, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4090, 3, 1, 0, 0, 6, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4126, 4095, 4060, 4096, 4096, 4102] avg=4096.0000 max=4126 max_rank=2 min=4060 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 1, 0, 0, 6, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4128, 4096, 4060, 4096, 4094, 4102] avg=4096.0000 max=4128 max_rank=2 min=4060 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16352 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3, 0, 4089, 0, 0, 0, 4094, 0, 1, 1, 0, 0, 0, 0, 17, 0, 3868, 1, 0, 0, 0, 0, 1, 1, 0, 4086, 7, 2, 1, 0, 0, 0, 12, 268, 4062, 0, 0, 0, 4093, 1, 1, 1, 0, 0, 0, 0, 3, 11, 0, 0, 4078, 9, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 3886, 4098, 4342, 4096, 4101, 4056] avg=4096.0000 max=4342 max_rank=4 min=3886 min_rank=2 imbalance_ratio=1.060059 max_load_reduction=28426 max_load_reduction_pct=0.867493 moved_to_echo=16595 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4, 4055, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4125, 4096, 4063, 4096, 4095, 4100] avg=4096.0000 max=4125 max_rank=2 min=4063 min_rank=4 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4089, 3, 1, 0, 0, 6, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4126, 4096, 4059, 4096, 4097, 4102] avg=4096.0000 max=4126 max_rank=2 min=4059 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 384, 32512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[256, 0, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 2, 0, 0, 2, 0, 1731, 2722, 0, 0, 0, 732, 3006, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4089, 2, 0, 0, 35, 0, 3253, 2159, 0, 0, 0, 994, 1754, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 1, 0, 0, 5, 0, 2346, 2597, 0, 0, 0, 592, 2658, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 3, 1, 4093, 0, 0, 1, 5, 0, 3731, 1, 1, 1, 1, 4095, 35, 5, 1, 5, 2, 2795, 2278, 0, 0, 0, 842, 2622, 0, 4059, 4095, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 1, 0, 0, 10, 0, 1971, 2452, 0, 0, 0, 537, 3228, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 2368, 2238, 0, 0, 0, 499, 3091, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 13, 0, 2201, 2383, 0, 0, 0, 482, 3124, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4093, 2, 0, 0, 3, 0, 1705, 3198, 0, 0, 0, 591, 2695, 0, 4096, 4096, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 0, 2, 0, 0, 2, 4080, 1, 0, 0, 0, 1731, 1958, 0, 0, 0, 0, 0, 0, 0, 732, 3006, 711, 11, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4095, 4144, 4099, 4085, 3689, 4460, 4101] avg=4096.0000 max=4460 max_rank=6 min=3689 min_rank=5 imbalance_ratio=1.088867 max_load_reduction=28308 max_load_reduction_pct=0.863892 moved_to_echo=16335 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 46, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 1, 0, 0, 5, 4080, 0, 0, 0, 0, 2346, 1868, 0, 0, 0, 0, 0, 0, 0, 592, 2658, 678, 11, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4142, 4094, 4086, 4214, 3939, 4101] avg=4096.0000 max=4214 max_rank=5 min=3939 min_rank=6 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 38, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 0, 2, 0, 0, 35, 4080, 1, 0, 0, 0, 3253, 1553, 0, 0, 0, 0, 0, 0, 0, 994, 1754, 564, 11, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4136, 4093, 4118, 4806, 3323, 4101] avg=4096.0000 max=4806 max_rank=5 min=3323 min_rank=6 imbalance_ratio=1.173340 max_load_reduction=27962 max_load_reduction_pct=0.853333 moved_to_echo=16348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 43, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 0, 1, 0, 0, 10, 4080, 1, 0, 0, 0, 1971, 1765, 0, 0, 0, 0, 0, 0, 0, 537, 3228, 640, 11, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4139, 4093, 4092, 3736, 4416, 4101] avg=4096.0000 max=4416 max_rank=6 min=3736 min_rank=5 imbalance_ratio=1.078125 max_load_reduction=28352 max_load_reduction_pct=0.865234 moved_to_echo=16345 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 39, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 1, 0, 0, 0, 4080, 0, 0, 0, 0, 2368, 1611, 0, 0, 0, 0, 0, 0, 0, 499, 3091, 584, 11, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4134, 4096, 4081, 3979, 4185, 4101] avg=4096.0000 max=4185 max_rank=6 min=3979 min_rank=5 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16346 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3, 1, 4095, 0, 0, 0, 4093, 0, 0, 1, 5, 0, 0, 0, 5, 0, 3731, 1, 43, 0, 0, 0, 1, 1, 1, 4095, 7, 0, 0, 0, 35, 5, 1, 5, 4081, 0, 0, 0, 2, 2795, 1625, 0, 0, 0, 0, 0, 0, 0, 842, 2622, 598, 12, 0, 0, 0, 4059, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4099, 3780, 4105, 4127, 4422, 4074, 4061] avg=4096.0000 max=4422 max_rank=5 min=3780 min_rank=2 imbalance_ratio=1.079590 max_load_reduction=28346 max_load_reduction_pct=0.865051 moved_to_echo=16684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 42, 0, 0, 0, 0, 0, 0, 4085, 4, 0, 0, 0, 0, 0, 0, 13, 4080, 0, 0, 0, 0, 2201, 1715, 0, 0, 0, 0, 0, 0, 0, 482, 3124, 622, 11, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4094, 4140, 4089, 4093, 3916, 4239, 4101] avg=4096.0000 max=4239 max_rank=6 min=3916 min_rank=5 imbalance_ratio=1.034912 max_load_reduction=28529 max_load_reduction_pct=0.870636 moved_to_echo=16349 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 56, 0, 0, 0, 0, 0, 0, 4093, 6, 0, 0, 0, 2, 0, 0, 3, 4080, 1, 0, 0, 0, 1705, 2301, 0, 0, 0, 0, 0, 0, 0, 591, 2695, 835, 11, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4153, 4099, 4086, 4006, 4132, 4101] avg=4096.0000 max=4153 max_rank=2 min=4006 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16325 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 5376, 22272, 5248, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 18432, 14464, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 32512, 128, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4095, 113, 0, 0, 0, 4026, 0, 0, 0, 0, 1, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 4052, 0, 4096, 0, 4, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4094, 200, 0, 0, 0, 4025, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3940, 0, 4096, 0, 29, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 644, 0, 0, 0, 3692, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3853, 0, 4096, 0, 3, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 212, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4015, 0, 4096, 0, 2, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 4073, 93, 1, 0, 0, 4030, 0, 0, 4, 1, 0, 4095, 1, 0, 4094, 1, 1, 4095, 0, 0, 4066, 1, 4095, 1, 19, 0, 0, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 279, 0, 0, 0, 3848, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4061, 0, 4096, 0, 4, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4094, 117, 0, 0, 0, 4003, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4072, 0, 4096, 0, 4, 0, 0, 0, 0, 4092] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 78, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3883, 644, 0, 0, 0, 0, 0, 0, 0, 3692, 144, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 69, 7, 0, 0, 4089, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4527, 3836, 4096, 4096, 4096, 3929, 4092, 4096] avg=4096.0000 max=4527 max_rank=0 min=3836 min_rank=1 imbalance_ratio=1.105225 max_load_reduction=28241 max_load_reduction_pct=0.861847 moved_to_echo=16213 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3881, 200, 0, 0, 0, 0, 0, 0, 0, 4025, 144, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 69, 7, 0, 0, 4089, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4170, 4095, 4097, 4096, 4016, 4118, 4095] avg=4096.0000 max=4170 max_rank=1 min=4016 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3882, 113, 0, 0, 0, 0, 0, 0, 0, 4026, 144, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 69, 7, 0, 0, 4089, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3995, 4170, 4096, 4096, 4096, 4128, 4093, 4094] avg=4096.0000 max=4170 max_rank=1 min=3995 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3883, 212, 0, 0, 0, 0, 0, 0, 0, 3963, 144, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 69, 7, 0, 0, 4089, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4107, 4096, 4096, 4096, 4091, 4091, 4096] avg=4096.0000 max=4107 max_rank=1 min=4091 min_rank=5 imbalance_ratio=1.002686 max_load_reduction=28661 max_load_reduction_pct=0.874664 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3855, 93, 0, 0, 0, 0, 1, 0, 0, 4030, 145, 1, 0, 0, 0, 0, 4, 1, 4094, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4066, 1, 73, 12, 0, 0, 4079, 1, 19, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4, 3, 0, 0] rerouted_tokens_per_ep_rank=[3949, 4177, 4099, 4093, 4096, 4152, 4099, 4103] avg=4096.0000 max=4177 max_rank=1 min=3949 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3883, 279, 0, 0, 0, 0, 0, 0, 0, 3848, 144, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 69, 7, 0, 0, 4089, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 3993, 4095, 4096, 4096, 4137, 4093, 4096] avg=4096.0000 max=4162 max_rank=0 min=3993 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3883, 78, 0, 0, 0, 0, 0, 0, 0, 4023, 144, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 69, 7, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3961, 4167, 4096, 4096, 4096, 4167, 4089, 4096] avg=4096.0000 max=4167 max_rank=1 min=3961 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3881, 117, 0, 0, 0, 0, 0, 0, 0, 4003, 144, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 69, 7, 0, 0, 4089, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 4148, 4095, 4098, 4096, 4148, 4093, 4092] avg=4096.0000 max=4148 max_rank=1 min=3998 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 31104, 1792, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 0, 32256, 128, 640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 31616, 1280, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3191, 4096, 0, 0, 1, 4096, 4096, 0, 0, 0, 4094, 0, 4096, 0, 4096, 1090, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[3000, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4095, 0, 4096, 0, 4093, 1190, 0, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3530, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 636, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3402, 4096, 0, 0, 0, 4096, 4096, 1, 0, 0, 4092, 0, 4096, 0, 4096, 780, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[3770, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 382, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[2951, 4091, 0, 0, 0, 4091, 4095, 0, 0, 1, 4090, 0, 4095, 1, 4011, 1356, 4, 1, 1, 3916, 1, 0, 0, 1, 0, 1, 0, 55, 0, 1, 1, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3473, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 674, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3477, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 801, 0, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3191, 747, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 3243, 1090, 0, 0, 0, 0, 0, 0, 0, 3912, 115, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3349, 737, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[3938, 4097, 4095, 4333, 4027, 4096, 4086, 4096] avg=4096.0000 max=4333 max_rank=3 min=3938 min_rank=0 imbalance_ratio=1.057861 max_load_reduction=28435 max_load_reduction_pct=0.867767 moved_to_echo=16305 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3000, 747, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 3240, 1190, 0, 0, 0, 0, 0, 0, 0, 4004, 115, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 2, 3349, 737, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0] rerouted_tokens_per_ep_rank=[3747, 4097, 4096, 4430, 4119, 4095, 4088, 4096] avg=4096.0000 max=4430 max_rank=3 min=3747 min_rank=0 imbalance_ratio=1.081543 max_load_reduction=28338 max_load_reduction_pct=0.864807 moved_to_echo=16398 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3402, 747, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 3243, 780, 0, 0, 0, 0, 0, 0, 0, 4007, 115, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 3349, 737, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4097, 4093, 4023, 4122, 4096, 4092, 4096] avg=4096.0000 max=4149 max_rank=0 min=4023 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3530, 747, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 3243, 636, 0, 0, 0, 0, 0, 0, 0, 4016, 115, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 3349, 737, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4277, 4097, 4097, 3879, 4131, 4096, 4096, 4095] avg=4096.0000 max=4277 max_rank=0 min=3879 min_rank=3 imbalance_ratio=1.044189 max_load_reduction=28491 max_load_reduction_pct=0.869476 moved_to_echo=16418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3770, 747, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 3243, 382, 0, 0, 0, 0, 0, 0, 0, 4040, 115, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3349, 737, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4517, 4096, 4097, 3625, 4155, 4096, 4087, 4095] avg=4096.0000 max=4517 max_rank=0 min=3625 min_rank=3 imbalance_ratio=1.102783 max_load_reduction=28251 max_load_reduction_pct=0.862152 moved_to_echo=16433 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2951, 745, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 5, 0, 0, 0, 0, 1, 3160, 1356, 0, 0, 0, 0, 4, 1, 1, 3916, 120, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 55, 3346, 726, 1, 0, 0, 1, 1, 4, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[3696, 4091, 4096, 4517, 4042, 4097, 4129, 4100] avg=4096.0000 max=4517 max_rank=3 min=3696 min_rank=0 imbalance_ratio=1.102783 max_load_reduction=28251 max_load_reduction_pct=0.862152 moved_to_echo=16368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3477, 747, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 3243, 801, 0, 0, 0, 0, 0, 0, 0, 3912, 115, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 3349, 737, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4224, 4097, 4097, 4044, 4027, 4096, 4088, 4095] avg=4096.0000 max=4224 max_rank=0 min=4027 min_rank=4 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16306 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3473, 747, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 3243, 674, 0, 0, 0, 0, 0, 0, 0, 4030, 115, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 3349, 737, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4220, 4096, 4097, 3917, 4145, 4096, 4102, 4095] avg=4096.0000 max=4220 max_rank=0 min=3917 min_rank=3 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16438 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[26880, 6016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 26880, 5888, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 25984, 6912, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 31872, 1024, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4059, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 38, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 4019, 0, 0, 4091, 0, 1, 4095, 0, 0, 0, 4096, 0, 1, 4096, 4096, 78, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 3929, 0, 0, 4087, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 4096, 4096, 180, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 3913, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 182, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3949, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 4096, 4096, 150, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[9, 1, 2, 1, 0, 0, 0, 16, 4086, 2, 4000, 0, 1, 4083, 0, 0, 4095, 0, 0, 1, 4094, 0, 0, 3983, 4093, 104, 0, 108, 1, 4087, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3861, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 235, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 3904, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 193, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 3941, 153, 0, 0, 142, 0, 4059, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 3939, 38, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4094, 4201, 4100, 4096, 4110, 3977, 4096] avg=4096.0000 max=4201 max_rank=2 min=3977 min_rank=6 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 1, 3942, 153, 0, 0, 142, 0, 3913, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 3939, 182, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4055, 4100, 4096, 4110, 4121, 4096] avg=4096.0000 max=4121 max_rank=6 min=4055 min_rank=2 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 4, 3941, 153, 0, 0, 142, 0, 4019, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 14, 0, 1, 4096, 0, 0, 0, 0, 3939, 78, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4098, 4161, 4096, 4095, 4111, 4017, 4096] avg=4096.0000 max=4161 max_rank=2 min=4017 min_rank=6 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 3942, 153, 0, 0, 142, 0, 3949, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 0, 2, 4096, 0, 0, 0, 0, 3939, 150, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4095, 4091, 4095, 4096, 4112, 4089, 4096] avg=4096.0000 max=4112 max_rank=5 min=4089 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 3942, 153, 0, 0, 142, 0, 3861, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 3939, 235, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4095, 4003, 4100, 4096, 4110, 4174, 4096] avg=4096.0000 max=4174 max_rank=6 min=4003 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[9, 1, 2, 1, 4084, 13, 0, 0, 0, 0, 0, 16, 3935, 153, 0, 0, 138, 2, 4000, 0, 0, 0, 0, 0, 1, 4083, 0, 0, 5, 0, 0, 0, 4095, 0, 0, 1, 1, 0, 0, 0, 10, 0, 0, 3983, 0, 0, 0, 0, 3927, 104, 0, 108, 0, 0, 0, 0, 1, 4087, 1, 0, 7, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4104, 4140, 4089, 4097, 3993, 4139, 4096] avg=4096.0000 max=4140 max_rank=2 min=3993 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 1, 3935, 153, 0, 0, 142, 0, 3929, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 0, 2, 4096, 0, 0, 0, 0, 3939, 180, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4089, 4071, 4091, 4096, 4112, 4119, 4096] avg=4096.0000 max=4119 max_rank=6 min=4071 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=28649 max_load_reduction_pct=0.874298 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 2, 3942, 153, 0, 0, 142, 0, 3904, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 3939, 193, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4097, 4046, 4097, 4096, 4110, 4132, 4096] avg=4096.0000 max=4132 max_rank=6 min=4046 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1152, 128, 31744, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31616, 1280, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 31616, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 3518, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 1, 579, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 3539, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 558, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 2914, 1, 1, 0, 6, 1, 4095, 4095, 0, 0, 0, 0, 0, 1, 4094, 4095, 4087, 1, 1, 32, 9, 2, 0, 0, 1, 4091, 0, 1142, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 3206, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 887, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 3343, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 749, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 3328, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 761, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 3236, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 860, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 3159, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 932, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[816, 0, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 749, 0, 1, 3279, 6, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4095, 4099, 4096, 4097, 4091, 4096, 4035] avg=4096.0000 max=4159 max_rank=0 min=4035 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[816, 0, 3539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 558, 0, 2, 3279, 6, 0, 0] rerouted_tokens_per_ep_rank=[4355, 4095, 4096, 4096, 4097, 4088, 4096, 3845] avg=4096.0000 max=4355 max_rank=0 min=3845 min_rank=7 imbalance_ratio=1.063232 max_load_reduction=28413 max_load_reduction_pct=0.867096 moved_to_echo=16126 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[816, 0, 3518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 579, 0, 2, 3278, 6, 0, 0] rerouted_tokens_per_ep_rank=[4334, 4095, 4097, 4096, 4097, 4087, 4096, 3866] avg=4096.0000 max=4334 max_rank=0 min=3866 min_rank=7 imbalance_ratio=1.058105 max_load_reduction=28434 max_load_reduction_pct=0.867737 moved_to_echo=16146 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[817, 0, 3206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 887, 0, 0, 3279, 7, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4095, 4097, 4096, 4097, 4091, 4096, 4173] avg=4096.0000 max=4173 max_rank=7 min=4023 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[817, 0, 3159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 932, 0, 0, 3279, 6, 0, 0] rerouted_tokens_per_ep_rank=[3976, 4095, 4097, 4096, 4097, 4094, 4096, 4217] avg=4096.0000 max=4217 max_rank=7 min=3976 min_rank=0 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[809, 0, 2914, 1, 0, 0, 0, 0, 1, 0, 6, 1, 4093, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1, 0, 4087, 1, 0, 0, 0, 0, 1, 32, 9, 2, 4087, 0, 0, 0, 0, 0, 1, 4091, 4, 0, 0, 0, 0, 1142, 5, 0, 3281, 6, 0, 0] rerouted_tokens_per_ep_rank=[3724, 4103, 4095, 4096, 4089, 4131, 4096, 4434] avg=4096.0000 max=4434 max_rank=7 min=3724 min_rank=0 imbalance_ratio=1.082520 max_load_reduction=28334 max_load_reduction_pct=0.864685 moved_to_echo=16750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[817, 0, 3236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 860, 0, 1, 3279, 6, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4095, 4098, 4096, 4097, 4087, 4096, 4146] avg=4096.0000 max=4146 max_rank=7 min=4053 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[816, 0, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 761, 0, 7, 3278, 6, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4095, 4089, 4096, 4097, 4099, 4096, 4052] avg=4096.0000 max=4144 max_rank=0 min=4052 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6528, 0, 26368, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 6528, 128, 128, 26240, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 19, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 46, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 98, 1, 0, 4094, 1, 0, 1, 1, 4095, 2, 1, 0, 4094, 0, 2, 4095, 1, 0, 0, 4095, 17, 0, 0, 4, 1, 3974, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4054, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 44, 0, 0, 3, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 11, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4085, 4096, 4113, 4096, 4096, 4074, 4112] avg=4096.0000 max=4113 max_rank=3 min=4074 min_rank=6 imbalance_ratio=1.004150 max_load_reduction=28655 max_load_reduction_pct=0.874481 moved_to_echo=16378 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4050, 0, 0, 0, 11, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 46, 0, 0, 0, 4074, 1, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4085, 4095, 4067, 4096, 4096, 4121, 4112] avg=4096.0000 max=4121 max_rank=6 min=4067 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16425 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4077, 0, 0, 0, 11, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 19, 0, 0, 1, 4074, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4085, 4096, 4094, 4096, 4096, 4094, 4112] avg=4096.0000 max=4112 max_rank=7 min=4085 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16398 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 11, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 4074, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4085, 4096, 4106, 4096, 4096, 4081, 4112] avg=4096.0000 max=4112 max_rank=7 min=4081 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16385 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 6, 0, 0, 0, 98, 1, 0, 4074, 0, 0, 0, 0, 1, 0, 1, 1, 4095, 0, 0, 0, 4095, 2, 1, 0, 14, 8, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 17, 0, 0, 4, 4078, 0, 0, 0, 1, 3974, 0, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4173, 4098, 4120, 4096, 4096, 4099, 3984] avg=4096.0000 max=4173 max_rank=1 min=3984 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16275 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 0, 11, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 0, 4074, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4085, 4096, 4103, 4096, 4096, 4084, 4112] avg=4096.0000 max=4112 max_rank=7 min=4084 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16388 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4077, 0, 0, 0, 11, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 19, 0, 0, 0, 4074, 1, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4085, 4095, 4094, 4096, 4096, 4094, 4112] avg=4096.0000 max=4112 max_rank=7 min=4085 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=28656 max_load_reduction_pct=0.874512 moved_to_echo=16398 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4054, 0, 0, 0, 11, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 44, 0, 0, 3, 4074, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4085, 4096, 4071, 4096, 4096, 4121, 4112] avg=4096.0000 max=4121 max_rank=6 min=4071 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=28647 max_load_reduction_pct=0.874237 moved_to_echo=16425 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32640, 128, 128, 0, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 0, 0, 128, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4088, 4096, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4091, 4095, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4094, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4088, 4096, 4096, 4082, 0, 0, 0, 4096, 0, 0, 0, 22, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4095, 4095, 4095, 3953, 1, 2, 25, 4095, 1, 0, 0, 133, 1, 4079, 0, 0, 0, 1, 0, 0, 0, 1, 4094, 0, 0, 0, 4095, 1, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4095, 4096, 4096, 4093, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 1, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4093, 4076, 4098, 4096, 4096, 4096, 4096] avg=4096.0000 max=4117 max_rank=0 min=4076 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 9, 4067, 1, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4093, 4077, 4098, 4096, 4096, 4096, 4096] avg=4096.0000 max=4116 max_rank=0 min=4077 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 5, 4070, 1, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4093, 4076, 4098, 4096, 4096, 4096, 4096] avg=4096.0000 max=4117 max_rank=0 min=4076 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2, 4073, 1, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4093, 4076, 4098, 4096, 4096, 4096, 4096] avg=4096.0000 max=4117 max_rank=0 min=4076 min_rank=2 imbalance_ratio=1.005127 max_load_reduction=28651 max_load_reduction_pct=0.874359 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 4075, 1, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4093, 4077, 4098, 4096, 4096, 4096, 4096] avg=4096.0000 max=4116 max_rank=0 min=4077 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[15, 0, 0, 3953, 0, 0, 0, 0, 1, 2, 25, 4089, 0, 0, 0, 0, 1, 0, 0, 133, 4080, 3, 0, 0, 1, 4079, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[3968, 4117, 4217, 4082, 4096, 4096, 4096, 4096] avg=4096.0000 max=4217 max_rank=2 min=3968 min_rank=0 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4, 4074, 1, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4093, 4079, 4098, 4096, 4096, 4096, 4096] avg=4096.0000 max=4114 max_rank=0 min=4079 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 22, 4067, 1, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4093, 4090, 4098, 4096, 4096, 4096, 4096] avg=4096.0000 max=4103 max_rank=0 min=4090 min_rank=2 imbalance_ratio=1.001709 max_load_reduction=28665 max_load_reduction_pct=0.874786 moved_to_echo=16384 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[256, 0, 0, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 256, 32640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True + [2026-08-30 04:55:27] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 680.5 | throughput per GPU (TFLOP/s/GPU): 489.9 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.955948E+00 | loss scale: 1.0 | grad norm: 5.283 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2754, 0, 1205, 788, 877, 9, 893, 1487, 1119, 2010, 0, 208, 1301, 660, 865, 2592, 107, 1496, 1109, 0, 4, 1649, 2672, 2498, 0, 821, 1108, 870, 1229, 1965, 468, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2525, 0, 1420, 664, 744, 4, 1447, 757, 1062, 2833, 0, 131, 1162, 1092, 787, 2138, 60, 1437, 608, 0, 1, 2075, 1887, 1983, 0, 955, 1061, 1296, 1451, 2609, 578, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3248, 1, 1320, 811, 844, 0, 1396, 1008, 1316, 2524, 0, 103, 802, 1049, 608, 2066, 113, 1862, 679, 0, 2, 2290, 2294, 1829, 0, 966, 1231, 1014, 820, 2249, 323, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2469, 0, 1513, 779, 916, 0, 1764, 821, 1433, 1961, 0, 110, 1309, 806, 455, 2678, 76, 1933, 950, 0, 0, 2056, 2053, 1919, 2, 934, 1045, 1080, 1016, 1879, 811, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2704, 0, 1411, 839, 1340, 1, 1024, 672, 1272, 2132, 0, 96, 1177, 546, 680, 2460, 72, 1617, 1012, 0, 0, 1994, 2224, 2497, 0, 1160, 1269, 867, 1023, 2127, 552, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2846, 5, 1269, 751, 1139, 0, 973, 808, 1016, 2492, 0, 256, 1245, 1129, 1317, 2210, 67, 1897, 475, 0, 0, 1966, 2267, 2173, 0, 655, 1254, 828, 660, 2539, 523, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[3005, 0, 1678, 749, 567, 0, 1132, 691, 998, 2441, 1, 62, 715, 1321, 448, 2434, 167, 1887, 676, 1, 2, 2305, 2670, 1609, 10, 928, 1370, 1067, 719, 2764, 351, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2727, 2, 909, 764, 1272, 0, 1296, 963, 1485, 1535, 2, 107, 862, 451, 484, 2791, 82, 1961, 616, 2, 0, 1936, 3212, 2275, 1, 868, 1471, 1355, 1091, 1864, 384, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1966, 0, 1205, 788, 0, 0, 0, 0, 877, 9, 893, 1487, 1035, 60, 0, 0, 1119, 2010, 0, 208, 484, 18, 0, 0, 1301, 660, 865, 1540, 0, 0, 0, 0, 107, 1496, 1109, 0, 1637, 0, 0, 0, 4, 1649, 0, 2455, 0, 0, 0, 0, 0, 821, 1108, 870, 974, 0, 0, 0, 1229, 1965, 468, 4, 304, 43, 0, 0] rerouted_tokens_per_ep_rank=[3959, 4361, 3839, 4366, 4349, 4108, 3773, 4013] avg=4096.0000 max=4366 max_rank=3 min=3773 min_rank=6 imbalance_ratio=1.065918 max_load_reduction=28402 max_load_reduction_pct=0.866760 moved_to_echo=16243 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1802, 0, 1420, 664, 0, 0, 0, 0, 744, 4, 1447, 757, 731, 49, 0, 0, 1062, 2833, 0, 131, 444, 15, 0, 0, 1162, 1092, 787, 1271, 0, 0, 0, 0, 60, 1437, 608, 0, 1156, 0, 0, 0, 1, 2075, 0, 1949, 0, 0, 0, 0, 0, 955, 1061, 1296, 803, 0, 0, 0, 1451, 2609, 578, 1, 279, 34, 0, 0] rerouted_tokens_per_ep_rank=[3886, 3732, 4485, 4312, 3261, 4025, 4115, 4952] avg=4096.0000 max=4952 max_rank=7 min=3261 min_rank=4 imbalance_ratio=1.208984 max_load_reduction=27816 max_load_reduction_pct=0.848877 moved_to_echo=16353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2318, 1, 1320, 811, 0, 0, 0, 0, 844, 0, 1396, 1008, 890, 47, 0, 0, 1316, 2524, 0, 103, 571, 14, 0, 0, 802, 1049, 608, 1229, 0, 0, 0, 0, 113, 1862, 679, 0, 1404, 0, 0, 0, 2, 2290, 0, 1797, 0, 0, 0, 0, 0, 966, 1231, 1014, 776, 0, 0, 0, 820, 2249, 323, 0, 359, 32, 0, 0] rerouted_tokens_per_ep_rank=[4450, 4185, 4528, 3688, 4058, 4089, 3987, 3783] avg=4096.0000 max=4528 max_rank=2 min=3688 min_rank=3 imbalance_ratio=1.105469 max_load_reduction=28240 max_load_reduction_pct=0.861816 moved_to_echo=15917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1929, 0, 1411, 839, 0, 0, 0, 0, 1340, 1, 1024, 672, 862, 57, 0, 0, 1272, 2132, 0, 96, 476, 17, 0, 0, 1177, 546, 680, 1462, 0, 0, 0, 0, 72, 1617, 1012, 0, 1362, 0, 0, 0, 0, 1994, 0, 2454, 0, 0, 0, 0, 0, 1160, 1269, 867, 924, 0, 0, 0, 1023, 2127, 552, 0, 299, 43, 0, 0] rerouted_tokens_per_ep_rank=[4179, 3956, 3993, 3865, 4063, 4448, 4220, 4044] avg=4096.0000 max=4448 max_rank=5 min=3865 min_rank=3 imbalance_ratio=1.085938 max_load_reduction=28320 max_load_reduction_pct=0.864258 moved_to_echo=16775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2031, 5, 1269, 751, 0, 0, 0, 0, 1139, 0, 973, 808, 879, 51, 0, 0, 1016, 2492, 0, 256, 501, 15, 0, 0, 1245, 1129, 1317, 1314, 0, 0, 0, 0, 67, 1897, 475, 0, 1388, 0, 0, 0, 0, 1966, 0, 2135, 0, 0, 0, 0, 0, 655, 1254, 828, 830, 0, 0, 0, 660, 2539, 523, 8, 314, 38, 0, 0] rerouted_tokens_per_ep_rank=[4056, 3850, 4280, 5005, 3827, 4101, 3567, 4082] avg=4096.0000 max=5005 max_rank=3 min=3567 min_rank=6 imbalance_ratio=1.221924 max_load_reduction=27763 max_load_reduction_pct=0.847260 moved_to_echo=15577 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1762, 0, 1513, 779, 0, 0, 0, 0, 916, 0, 1764, 821, 795, 62, 0, 0, 1433, 1961, 0, 110, 434, 19, 0, 0, 1309, 806, 455, 1591, 0, 0, 0, 0, 76, 1933, 950, 0, 1258, 0, 0, 0, 0, 2056, 0, 1886, 0, 0, 0, 0, 2, 934, 1045, 1080, 1006, 0, 0, 0, 1016, 1879, 811, 0, 273, 33, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4358, 3957, 4161, 4217, 3942, 4067, 4012] avg=4096.0000 max=4358 max_rank=1 min=3942 min_rank=5 imbalance_ratio=1.063965 max_load_reduction=28410 max_load_reduction_pct=0.867004 moved_to_echo=16238 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2136, 0, 1678, 749, 0, 0, 0, 0, 567, 0, 1132, 691, 1035, 60, 0, 0, 998, 2441, 1, 62, 533, 22, 0, 0, 715, 1321, 448, 1432, 0, 0, 0, 0, 167, 1887, 676, 1, 1635, 0, 0, 0, 2, 2305, 0, 1576, 0, 0, 0, 0, 10, 928, 1370, 1067, 920, 0, 0, 0, 719, 2764, 351, 0, 336, 33, 0, 0] rerouted_tokens_per_ep_rank=[4563, 3485, 4057, 3916, 4366, 3883, 4295, 4203] avg=4096.0000 max=4563 max_rank=0 min=3485 min_rank=1 imbalance_ratio=1.114014 max_load_reduction=28205 max_load_reduction_pct=0.860748 moved_to_echo=16747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1946, 2, 909, 764, 0, 0, 0, 0, 1272, 0, 1296, 963, 1246, 64, 0, 0, 1485, 1535, 2, 107, 480, 20, 0, 0, 862, 451, 484, 1658, 0, 0, 0, 0, 82, 1961, 616, 2, 1966, 0, 0, 0, 0, 1936, 0, 2236, 0, 0, 0, 0, 1, 868, 1471, 1355, 1049, 0, 0, 0, 1091, 1864, 384, 0, 301, 39, 0, 0] rerouted_tokens_per_ep_rank=[3621, 4841, 3629, 3455, 4627, 4172, 4744, 3679] avg=4096.0000 max=4841 max_rank=1 min=3455 min_rank=3 imbalance_ratio=1.181885 max_load_reduction=27927 max_load_reduction_pct=0.852264 moved_to_echo=17222 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16000, 128, 10752, 6272, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 7296, 9856, 8448, 7296, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8576, 7168, 5760, 11520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[7808, 128, 9984, 7296, 7552, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[8064, 18048, 4096, 128, 2560, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 16384, 0, 16512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[768, 14208, 6144, 128, 11904, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9728, 18048, 128, 1152, 3968, 256, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1174, 0, 274, 0, 4035, 0, 0, 1, 0, 4095, 3831, 4093, 4089, 8, 2, 157, 4096, 0, 0, 0, 2685, 1, 0, 0, 4081, 43, 0, 0, 0, 102, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[3738, 0, 34, 13, 4096, 0, 0, 0, 0, 4093, 4064, 4096, 4096, 1, 3, 0, 4091, 0, 0, 0, 359, 1, 0, 0, 4032, 4, 0, 0, 0, 47, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1373, 0, 709, 1, 3866, 0, 0, 0, 1, 4091, 4035, 4096, 4088, 1, 0, 33, 4096, 0, 0, 0, 2143, 0, 0, 0, 4063, 34, 0, 0, 0, 138, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[775, 0, 495, 0, 4035, 0, 0, 0, 2, 4081, 3971, 4096, 4093, 1, 1, 25, 4096, 0, 0, 0, 2916, 0, 0, 0, 4094, 6, 0, 0, 0, 81, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[3097, 0, 99, 2, 4000, 0, 0, 0, 3, 4080, 4072, 4093, 4015, 9, 2, 176, 4095, 0, 0, 0, 829, 0, 0, 0, 4093, 14, 0, 0, 0, 89, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[615, 6, 1403, 0, 3034, 0, 5, 0, 1, 4059, 3793, 4086, 4053, 16, 0, 65, 4090, 2, 3, 1, 3304, 0, 0, 0, 4096, 17, 10, 7, 0, 93, 2, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3403, 0, 82, 4, 4076, 3, 3, 1, 1, 4090, 4062, 4092, 4093, 0, 5, 70, 4093, 0, 0, 2, 564, 0, 0, 1, 4081, 6, 2, 0, 0, 31, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[828, 2, 962, 0, 3814, 2, 2, 0, 0, 4088, 3473, 4093, 4086, 31, 0, 711, 4094, 0, 2, 2, 2270, 3, 0, 1, 4077, 120, 0, 2, 0, 102, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[775, 0, 495, 0, 1708, 0, 0, 0, 4035, 0, 0, 0, 145, 65, 13, 0, 2, 184, 3971, 0, 0, 0, 0, 0, 3948, 1, 1, 25, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2916, 0, 0, 0, 2124, 84, 0, 0, 4081, 6, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 4012, 0, 0, 0] rerouted_tokens_per_ep_rank=[2978, 4258, 4157, 3975, 4096, 5124, 4087, 4093] avg=4096.0000 max=5124 max_rank=5 min=2978 min_rank=0 imbalance_ratio=1.250977 max_load_reduction=27644 max_load_reduction_pct=0.843628 moved_to_echo=17400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3738, 0, 34, 13, 1713, 0, 0, 0, 4096, 0, 0, 0, 145, 65, 13, 0, 0, 184, 4064, 0, 0, 0, 0, 0, 3951, 1, 3, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 359, 1, 0, 0, 2131, 84, 0, 0, 4019, 4, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 4012, 0, 0, 0] rerouted_tokens_per_ep_rank=[5498, 4319, 4248, 3955, 4091, 2575, 4023, 4059] avg=4096.0000 max=5498 max_rank=0 min=2575 min_rank=5 imbalance_ratio=1.342285 max_load_reduction=27270 max_load_reduction_pct=0.832214 moved_to_echo=14748 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1174, 0, 274, 0, 1714, 0, 0, 0, 4035, 0, 0, 1, 145, 65, 13, 0, 0, 184, 3831, 0, 0, 0, 0, 0, 3944, 8, 2, 157, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2685, 1, 0, 0, 2132, 84, 0, 0, 4068, 43, 0, 0, 0, 0, 0, 0, 0, 102, 0, 1, 4009, 0, 0, 0] rerouted_tokens_per_ep_rank=[3162, 4259, 4015, 4111, 4096, 4902, 4111, 4112] avg=4096.0000 max=4902 max_rank=5 min=3162 min_rank=0 imbalance_ratio=1.196777 max_load_reduction=27866 max_load_reduction_pct=0.850403 moved_to_echo=17221 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3403, 0, 82, 4, 1712, 0, 0, 0, 4076, 3, 3, 1, 145, 65, 13, 0, 1, 184, 4062, 0, 0, 0, 0, 0, 3948, 0, 5, 70, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 564, 0, 0, 1, 2129, 85, 0, 0, 4068, 6, 2, 0, 0, 0, 0, 0, 0, 31, 3, 0, 4007, 0, 0, 0] rerouted_tokens_per_ep_rank=[5201, 4306, 4247, 4023, 4095, 2779, 4076, 4041] avg=4096.0000 max=5201 max_rank=0 min=2779 min_rank=5 imbalance_ratio=1.269775 max_load_reduction=27567 max_load_reduction_pct=0.841278 moved_to_echo=14991 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1373, 0, 709, 1, 1712, 0, 0, 0, 3866, 0, 0, 0, 145, 65, 13, 0, 1, 184, 4035, 0, 0, 0, 0, 0, 3943, 1, 0, 33, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2143, 0, 0, 0, 2130, 84, 0, 0, 4050, 34, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 4012, 0, 0, 0] rerouted_tokens_per_ep_rank=[3795, 4089, 4220, 3977, 4096, 4357, 4084, 4150] avg=4096.0000 max=4357 max_rank=5 min=3795 min_rank=0 imbalance_ratio=1.063721 max_load_reduction=28411 max_load_reduction_pct=0.867035 moved_to_echo=16687 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3097, 0, 99, 2, 1710, 0, 0, 0, 4000, 0, 0, 0, 148, 69, 18, 0, 3, 173, 4072, 0, 0, 0, 0, 0, 3867, 9, 2, 176, 0, 0, 0, 0, 4095, 0, 0, 0, 5, 0, 0, 0, 829, 0, 0, 0, 2128, 85, 0, 0, 4070, 14, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 4008, 0, 0, 0] rerouted_tokens_per_ep_rank=[4908, 4235, 4248, 4054, 4100, 3042, 4084, 4097] avg=4096.0000 max=4908 max_rank=0 min=3042 min_rank=5 imbalance_ratio=1.198242 max_load_reduction=27860 max_load_reduction_pct=0.850220 moved_to_echo=15323 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[615, 6, 1403, 0, 1699, 0, 0, 0, 3034, 0, 5, 0, 144, 65, 13, 0, 1, 182, 3793, 0, 0, 0, 0, 0, 3909, 16, 0, 65, 0, 0, 0, 0, 4090, 2, 3, 1, 0, 0, 0, 0, 3304, 0, 0, 0, 2113, 85, 0, 0, 4083, 17, 10, 7, 0, 0, 0, 0, 0, 93, 2, 7, 4001, 0, 0, 0] rerouted_tokens_per_ep_rank=[3723, 3261, 3976, 3990, 4096, 5502, 4117, 4103] avg=4096.0000 max=5502 max_rank=5 min=3261 min_rank=1 imbalance_ratio=1.343262 max_load_reduction=27266 max_load_reduction_pct=0.832092 moved_to_echo=17818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[828, 2, 962, 0, 1711, 0, 0, 0, 3814, 2, 2, 0, 145, 65, 13, 0, 0, 184, 3473, 0, 0, 0, 0, 0, 3941, 31, 0, 711, 0, 0, 0, 0, 4094, 0, 2, 2, 0, 0, 0, 0, 2270, 3, 0, 1, 2128, 85, 0, 0, 4064, 120, 0, 2, 0, 0, 0, 0, 0, 102, 3, 0, 4008, 0, 0, 0] rerouted_tokens_per_ep_rank=[3503, 4041, 3657, 4683, 4098, 4487, 4186, 4113] avg=4096.0000 max=4683 max_rank=3 min=3503 min_rank=0 imbalance_ratio=1.143311 max_load_reduction=28085 max_load_reduction_pct=0.857086 moved_to_echo=16884 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[15104, 128, 0, 128, 17024, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[15104, 128, 4096, 128, 13696, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 768, 128, 128, 32128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[30976, 128, 128, 128, 1280, 640, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 1536, 31360, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31488, 128, 128, 1280, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[10, 0, 4, 0, 4075, 6, 4094, 1, 6, 3, 4035, 0, 4095, 0, 4083, 19, 0, 0, 0, 0, 0, 1, 4094, 218, 3, 416, 40, 4021, 0, 3541, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 4072, 3, 4096, 0, 46, 0, 3774, 0, 4096, 0, 4090, 172, 0, 0, 0, 0, 0, 2, 4096, 163, 0, 877, 183, 3750, 1, 3342, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 48, 2, 4095, 45, 4075, 1, 29, 0, 4076, 1, 4092, 0, 4096, 202, 0, 0, 4, 0, 0, 17, 4092, 121, 0, 612, 10, 3795, 7, 3347, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 9, 2, 4064, 1, 4085, 1, 24, 0, 3823, 0, 4088, 2, 4096, 8, 9, 7, 0, 10, 0, 2, 4086, 27, 17, 548, 224, 3935, 45, 3649, 0, 4] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 4, 97, 0, 4093, 9, 4095, 7, 68, 15, 4050, 0, 4089, 0, 4064, 316, 0, 0, 0, 0, 0, 1, 4096, 357, 0, 753, 25, 3369, 0, 3260, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4077, 19, 4083, 6, 97, 3, 3366, 1, 4094, 2, 4094, 16, 2, 2, 0, 10, 2, 1, 4094, 329, 5, 1341, 425, 4021, 0, 2651, 0, 25] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 2, 58, 0, 4077, 2, 4096, 0, 22, 0, 4023, 0, 4095, 0, 4092, 75, 0, 0, 0, 0, 0, 4, 4095, 130, 0, 993, 213, 3904, 0, 2886, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 25, 0, 4094, 7, 4090, 0, 64, 4, 3925, 0, 4091, 0, 4086, 73, 0, 0, 0, 0, 0, 2, 4094, 288, 0, 1019, 121, 3455, 0, 3328, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 4, 0, 4064, 2, 0, 0, 4075, 6, 3, 1, 0, 0, 0, 0, 6, 3, 4035, 0, 102, 61, 0, 0, 0, 0, 3981, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 1, 3890, 218, 0, 0, 0, 0, 3, 416, 40, 3316, 0, 0, 0, 0, 0, 3541, 0, 3, 705, 143, 27, 0] rerouted_tokens_per_ep_rank=[4080, 4085, 4207, 4000, 4093, 4109, 3775, 4419] avg=4096.0000 max=4419 max_rank=7 min=3775 min_rank=6 imbalance_ratio=1.078857 max_load_reduction=28349 max_load_reduction_pct=0.865143 moved_to_echo=16396 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 48, 2, 4045, 2, 0, 0, 4095, 45, 1, 1, 0, 0, 0, 0, 29, 0, 4076, 1, 102, 61, 0, 0, 0, 0, 3994, 202, 0, 0, 0, 0, 0, 0, 4, 0, 4090, 0, 0, 0, 0, 17, 3888, 121, 0, 0, 0, 0, 0, 612, 10, 3130, 0, 0, 0, 0, 7, 3347, 0, 1, 665, 143, 29, 0] rerouted_tokens_per_ep_rank=[4097, 4142, 4269, 4196, 4094, 4026, 3752, 4192] avg=4096.0000 max=4269 max_rank=2 min=3752 min_rank=6 imbalance_ratio=1.042236 max_load_reduction=28499 max_load_reduction_pct=0.869720 moved_to_echo=16064 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 4066, 2, 0, 0, 4072, 3, 3, 0, 0, 0, 0, 0, 46, 0, 3774, 0, 102, 61, 0, 0, 0, 0, 3988, 172, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 3892, 163, 0, 0, 0, 0, 0, 877, 183, 3093, 0, 0, 0, 0, 1, 3342, 0, 0, 657, 143, 27, 0] rerouted_tokens_per_ep_rank=[4073, 4078, 3983, 4160, 4094, 4057, 4153, 4170] avg=4096.0000 max=4170 max_rank=7 min=3983 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 9, 2, 4055, 2, 0, 0, 4064, 1, 0, 1, 0, 0, 0, 0, 24, 0, 3823, 0, 102, 61, 0, 0, 0, 2, 3994, 8, 0, 0, 0, 0, 9, 7, 0, 10, 4086, 0, 0, 0, 0, 2, 3882, 27, 0, 0, 0, 0, 17, 548, 224, 3245, 0, 0, 0, 0, 45, 3649, 0, 4, 690, 143, 30, 0] rerouted_tokens_per_ep_rank=[4070, 4066, 4010, 4004, 4112, 3911, 4034, 4561] avg=4096.0000 max=4561 max_rank=7 min=3911 min_rank=5 imbalance_ratio=1.113525 max_load_reduction=28207 max_load_reduction_pct=0.860809 moved_to_echo=16618 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 97, 0, 4065, 1, 0, 0, 4093, 9, 3, 7, 0, 0, 0, 0, 68, 15, 4050, 0, 101, 61, 0, 0, 0, 0, 3963, 316, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 1, 3892, 357, 0, 0, 0, 0, 0, 753, 25, 2778, 0, 0, 0, 0, 0, 3260, 0, 0, 591, 143, 27, 0] rerouted_tokens_per_ep_rank=[4167, 4112, 4295, 4279, 4088, 4250, 3556, 4021] avg=4096.0000 max=4295 max_rank=2 min=3556 min_rank=6 imbalance_ratio=1.048584 max_load_reduction=28473 max_load_reduction_pct=0.868927 moved_to_echo=15915 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 58, 0, 4066, 2, 0, 0, 4077, 2, 3, 0, 0, 0, 0, 0, 22, 0, 4023, 0, 102, 61, 0, 0, 0, 0, 3990, 75, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4, 3891, 130, 0, 0, 0, 0, 0, 993, 213, 3220, 0, 0, 0, 0, 0, 2886, 0, 0, 684, 143, 27, 0] rerouted_tokens_per_ep_rank=[4129, 4082, 4208, 4065, 4093, 4025, 4426, 3740] avg=4096.0000 max=4426 max_rank=6 min=3740 min_rank=7 imbalance_ratio=1.080566 max_load_reduction=28342 max_load_reduction_pct=0.864929 moved_to_echo=16284 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 25, 0, 4063, 2, 0, 0, 4094, 7, 0, 0, 0, 0, 0, 0, 64, 4, 3925, 0, 105, 68, 0, 0, 0, 0, 3981, 73, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 2, 3878, 288, 0, 0, 0, 0, 0, 1019, 121, 2845, 0, 0, 0, 0, 0, 3328, 0, 0, 610, 148, 27, 0] rerouted_tokens_per_ep_rank=[4092, 4101, 4166, 4054, 4089, 4168, 3985, 4113] avg=4096.0000 max=4168 max_rank=5 min=3985 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16355 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4055, 3, 0, 0, 4077, 19, 0, 6, 0, 0, 0, 0, 97, 3, 3366, 1, 102, 61, 0, 0, 0, 2, 3992, 16, 0, 0, 0, 0, 2, 2, 0, 10, 4091, 0, 0, 0, 2, 1, 3890, 329, 0, 0, 0, 0, 5, 1341, 425, 3316, 0, 0, 0, 0, 0, 2651, 0, 25, 705, 143, 28, 0] rerouted_tokens_per_ep_rank=[4060, 4102, 3630, 4010, 4105, 4222, 5087, 3552] avg=4096.0000 max=5087 max_rank=6 min=3552 min_rank=7 imbalance_ratio=1.241943 max_load_reduction=27681 max_load_reduction_pct=0.844757 moved_to_echo=16966 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 26112, 0, 128, 5376, 1152, 256, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 6656, 1280, 24960, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 256, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 128, 31104, 128, 896, 512, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 31104, 1664, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32000, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 292, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3573, 0, 4095, 2, 0, 0, 350, 0, 646, 3413, 4031, 4095, 0, 0, 4079] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 113, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3635, 0, 4096, 0, 0, 0, 50, 0, 700, 3790, 4089, 4090, 0, 1, 4012] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[95, 8, 0, 7, 0, 82, 13, 4083, 0, 2, 0, 7, 7, 0, 6, 0, 4087, 3582, 8, 4090, 0, 3, 0, 14, 1, 691, 3881, 4075, 3992, 0, 0, 4034] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 115, 2, 4086, 0, 6, 0, 0, 0, 0, 0, 0, 4092, 3177, 0, 4082, 2, 0, 0, 88, 5, 1623, 3322, 4069, 4093, 0, 4, 4002] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 168, 0, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 3720, 0, 4096, 3, 0, 0, 253, 2, 555, 3633, 4027, 4067, 0, 2, 4050] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 103, 3, 4072, 0, 8, 0, 0, 0, 0, 0, 0, 4078, 3612, 0, 4051, 4, 1, 0, 138, 4, 811, 3759, 4069, 4081, 0, 12, 3962] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[28, 0, 1, 2, 0, 236, 15, 4089, 0, 0, 2, 0, 2, 0, 2, 0, 4094, 3324, 2, 4093, 0, 2, 0, 19, 5, 1298, 3445, 4070, 4051, 2, 0, 3986] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 69, 0, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 3134, 0, 4094, 2, 0, 0, 186, 16, 1075, 3813, 4033, 4088, 0, 3, 4067] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3988, 114, 0, 0, 0, 292, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 36, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 3573, 0, 627, 0, 0, 0, 0, 2, 0, 0, 350, 3354, 436, 144, 0, 0, 646, 2977, 0, 0, 0, 0, 0, 71, 0, 0, 4079, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4244, 4065, 4098, 4200, 4286, 3623, 4150] avg=4096.0000 max=4286 max_rank=5 min=3623 min_rank=6 imbalance_ratio=1.046387 max_load_reduction=28482 max_load_reduction_pct=0.869202 moved_to_echo=16259 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3961, 114, 0, 0, 0, 168, 0, 3950, 0, 0, 0, 0, 0, 3, 0, 0, 4024, 36, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 3720, 0, 628, 0, 0, 0, 0, 3, 0, 0, 253, 3354, 465, 143, 0, 2, 555, 3168, 0, 0, 0, 0, 0, 70, 0, 2, 4050, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4118, 4063, 4099, 4348, 4218, 3725, 4122] avg=4096.0000 max=4348 max_rank=4 min=3725 min_rank=6 imbalance_ratio=1.061523 max_load_reduction=28420 max_load_reduction_pct=0.867310 moved_to_echo=16413 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3983, 114, 0, 0, 0, 113, 1, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 36, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 3635, 0, 628, 0, 0, 0, 0, 0, 0, 0, 50, 3354, 485, 144, 0, 0, 700, 3305, 0, 0, 0, 0, 0, 71, 0, 1, 4012, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4066, 4123, 4097, 4263, 4033, 4005, 4084] avg=4096.0000 max=4263 max_rank=4 min=4005 min_rank=6 imbalance_ratio=1.040771 max_load_reduction=28505 max_load_reduction_pct=0.869904 moved_to_echo=16385 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3986, 114, 0, 0, 0, 115, 2, 3943, 0, 0, 0, 0, 0, 6, 0, 0, 4067, 36, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 3177, 0, 625, 0, 0, 0, 0, 2, 0, 0, 88, 3343, 425, 143, 0, 5, 1623, 2897, 0, 0, 0, 0, 0, 71, 0, 4, 4002, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4060, 4109, 4094, 3802, 4001, 4525, 4077] avg=4096.0000 max=4525 max_rank=6 min=3802 min_rank=4 imbalance_ratio=1.104736 max_load_reduction=28243 max_load_reduction_pct=0.861908 moved_to_echo=16405 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[95, 8, 0, 7, 3888, 114, 0, 0, 0, 82, 13, 3940, 0, 0, 0, 0, 0, 2, 0, 7, 4073, 35, 0, 0, 7, 0, 6, 0, 4087, 2, 0, 0, 0, 3582, 8, 627, 0, 0, 0, 0, 0, 3, 0, 14, 3349, 496, 143, 0, 1, 691, 3385, 0, 0, 0, 0, 0, 69, 0, 0, 4034, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4035, 4117, 4102, 4217, 4005, 4077, 4103] avg=4096.0000 max=4217 max_rank=4 min=4005 min_rank=5 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16402 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3975, 113, 0, 0, 0, 103, 3, 3929, 0, 0, 0, 0, 0, 8, 0, 0, 4067, 36, 0, 0, 0, 0, 0, 0, 4078, 2, 0, 0, 0, 3612, 0, 620, 0, 0, 0, 0, 4, 1, 0, 138, 3318, 481, 143, 0, 4, 811, 3278, 0, 0, 0, 0, 0, 70, 0, 12, 3962, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4035, 4111, 4080, 4232, 4085, 4093, 4044] avg=4096.0000 max=4232 max_rank=4 min=4035 min_rank=1 imbalance_ratio=1.033203 max_load_reduction=28536 max_load_reduction_pct=0.870850 moved_to_echo=16454 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3986, 118, 0, 0, 0, 69, 0, 3944, 0, 0, 0, 0, 0, 1, 0, 0, 4031, 42, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 3134, 0, 619, 0, 0, 0, 0, 2, 0, 0, 186, 3357, 491, 147, 0, 16, 1075, 3322, 0, 0, 0, 0, 0, 60, 0, 3, 4067, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4013, 4074, 4098, 3753, 4183, 4413, 4130] avg=4096.0000 max=4413 max_rank=6 min=3753 min_rank=4 imbalance_ratio=1.077393 max_load_reduction=28355 max_load_reduction_pct=0.865326 moved_to_echo=16479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 1, 2, 3945, 114, 0, 0, 0, 236, 15, 3946, 0, 0, 0, 0, 0, 0, 2, 0, 4068, 36, 0, 0, 2, 0, 2, 0, 4094, 2, 0, 0, 0, 3324, 2, 627, 0, 0, 0, 0, 0, 2, 0, 19, 3352, 441, 143, 0, 5, 1298, 3004, 0, 0, 0, 0, 0, 70, 2, 0, 3986, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4197, 4106, 4100, 3953, 3957, 4307, 4058] avg=4096.0000 max=4307 max_rank=6 min=3953 min_rank=4 imbalance_ratio=1.051514 max_load_reduction=28461 max_load_reduction_pct=0.868561 moved_to_echo=16275 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 0, 1152, 26880, 3840, 1152, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 7424, 25344, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 27776, 128, 5120, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 31744, 1024, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[640, 128, 128, 32256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 1280, 128, 31616, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32512, 384, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 41, 4096, 0, 0, 2315, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 77, 600, 7, 4079, 0, 1088, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 31, 4096, 0, 0, 2375, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 39, 1482, 0, 3972, 0, 291, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 4096, 24, 4096, 0, 0, 2247, 0, 4095, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 22, 1690, 2, 3832, 0, 377, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 4094, 25, 4096, 0, 0, 1664, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 16, 2345, 6, 3916, 0, 230, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[7, 6, 8, 4087, 4072, 60, 4088, 4, 5, 1993, 0, 4089, 8, 12, 0, 4087, 2, 4, 0, 0, 7, 197, 0, 86, 753, 0, 4069, 8, 1031, 6, 2, 4077] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4090, 9, 4096, 0, 0, 1066, 0, 4092, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 329, 2250, 4, 4057, 0, 486, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4092, 46, 4096, 0, 0, 1524, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 77, 2156, 10, 3980, 0, 413, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[2, 2, 2, 4094, 4082, 12, 4093, 2, 1, 2766, 1, 4094, 2, 2, 1, 4094, 0, 1, 0, 0, 2, 0, 0, 48, 594, 0, 3908, 18, 852, 2, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4071, 41, 0, 0, 0, 0, 0, 0, 0, 2315, 0, 2097, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 77, 1984, 1406, 594, 25, 600, 7, 2673, 0, 0, 0, 0, 0, 1088, 0, 0, 3502, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4112, 4412, 4096, 4096, 4086, 3280, 4590] avg=4096.0000 max=4590 max_rank=7 min=3280 min_rank=6 imbalance_ratio=1.120605 max_load_reduction=28178 max_load_reduction_pct=0.859924 moved_to_echo=16052 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 4069, 25, 1, 0, 0, 0, 0, 0, 0, 1664, 0, 2102, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 16, 1988, 1350, 594, 25, 2345, 6, 2566, 0, 0, 0, 0, 0, 230, 0, 0, 3502, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4095, 3766, 4096, 4095, 3973, 4917, 3732] avg=4096.0000 max=4917 max_rank=6 min=3732 min_rank=7 imbalance_ratio=1.200439 max_load_reduction=27851 max_load_reduction_pct=0.849945 moved_to_echo=16717 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4065, 9, 0, 0, 0, 0, 0, 0, 0, 1066, 0, 2103, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 329, 1989, 1398, 594, 25, 2250, 4, 2659, 0, 0, 0, 0, 0, 486, 0, 0, 3502, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4074, 3169, 4095, 4098, 4335, 4913, 3988] avg=4096.0000 max=4913 max_rank=6 min=3169 min_rank=2 imbalance_ratio=1.199463 max_load_reduction=27855 max_load_reduction_pct=0.850067 moved_to_echo=17334 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 4071, 24, 0, 0, 0, 0, 0, 0, 0, 2247, 0, 2104, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 22, 1991, 1321, 594, 25, 1690, 2, 2511, 0, 0, 0, 0, 0, 377, 0, 0, 3502, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4095, 4351, 4095, 4097, 3954, 4203, 3879] avg=4096.0000 max=4351 max_rank=2 min=3879 min_rank=7 imbalance_ratio=1.062256 max_load_reduction=28417 max_load_reduction_pct=0.867218 moved_to_echo=16133 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[7, 6, 8, 4087, 0, 0, 0, 0, 4047, 60, 0, 4, 0, 0, 0, 0, 5, 1993, 0, 2101, 0, 0, 0, 0, 8, 12, 0, 4087, 0, 0, 0, 0, 2, 4, 0, 0, 4088, 0, 0, 0, 7, 197, 0, 86, 1988, 1403, 592, 25, 753, 0, 2666, 8, 0, 0, 0, 0, 1031, 6, 2, 3485, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4111, 4099, 4107, 4094, 4298, 3427, 4524] avg=4096.0000 max=4524 max_rank=7 min=3427 min_rank=6 imbalance_ratio=1.104492 max_load_reduction=28244 max_load_reduction_pct=0.861938 moved_to_echo=16343 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4071, 31, 0, 0, 0, 0, 0, 0, 0, 2375, 0, 2105, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 39, 1991, 1369, 594, 25, 1482, 0, 2603, 0, 0, 0, 0, 0, 291, 0, 0, 3502, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4102, 4480, 4096, 4098, 4018, 4085, 3793] avg=4096.0000 max=4480 max_rank=2 min=3793 min_rank=7 imbalance_ratio=1.093750 max_load_reduction=28288 max_load_reduction_pct=0.863281 moved_to_echo=15994 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4063, 46, 0, 0, 0, 0, 0, 0, 0, 1524, 0, 2095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 77, 1991, 1376, 600, 29, 2156, 10, 2604, 0, 0, 0, 0, 0, 413, 0, 0, 3496, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4109, 3619, 4096, 4096, 4073, 4770, 3909] avg=4096.0000 max=4770 max_rank=6 min=3619 min_rank=2 imbalance_ratio=1.164551 max_load_reduction=27998 max_load_reduction_pct=0.854431 moved_to_echo=16848 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 2, 4094, 0, 0, 0, 0, 4057, 12, 0, 2, 0, 0, 0, 0, 1, 2766, 1, 2104, 0, 0, 0, 0, 2, 2, 1, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 2, 0, 0, 48, 1990, 1347, 594, 25, 594, 0, 2561, 18, 0, 0, 0, 0, 852, 2, 0, 3499, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4071, 4872, 4099, 4094, 4006, 3173, 4353] avg=4096.0000 max=4872 max_rank=2 min=3173 min_rank=6 imbalance_ratio=1.189453 max_load_reduction=27896 max_load_reduction_pct=0.851318 moved_to_echo=15626 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32780, 32769, 32768, 32780, 32768, 32743, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32780, 32769, 32768, 32780, 32768, 32743, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32780, 32769, 32768, 32780, 32768, 32743, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32780, 32769, 32768, 32780, 32768, 32743, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32780, 32769, 32768, 32780, 32768, 32743, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32780, 32769, 32768, 32780, 32768, 32743, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32780, 32769, 32768, 32780, 32768, 32743, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32780, 32769, 32768, 32780, 32768, 32743, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32743 actual_valid_recv=33408 observed_loss=-665 tokens_per_expert=[128, 256, 128, 768, 16000, 11008, 4864, 256] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 16000, 128, 16896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[11904, 128, 20864, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4864, 128, 128, 28032, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32780 actual_valid_recv=33152 observed_loss=-372 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32769 actual_valid_recv=33152 observed_loss=-383 tokens_per_expert=[32640, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32780 actual_valid_recv=33152 observed_loss=-372 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 12, 0, 1, 565, 4095, 0, 0, 0, 4049, 0, 0, 0, 4096, 3928, 10, 0, 3, 4084, 0, 4096, 3728, 0, 0, 0, 0, 412, 3688, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 12, 3, 0, 257, 4090, 0, 7, 8, 4059, 6, 7, 0, 4009, 3599, 12, 0, 5, 4096, 7, 4088, 3780, 7, 7, 0, 2, 630, 4069, 0, 8, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 48, 0, 0, 913, 4086, 3, 0, 0, 2653, 0, 0, 0, 4096, 3722, 24, 0, 1, 4025, 0, 4093, 3695, 0, 0, 0, 0, 1848, 3561, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 362, 4096, 0, 0, 0, 3804, 0, 0, 0, 4095, 3688, 3, 0, 3, 4080, 0, 4096, 3857, 0, 1, 0, 0, 880, 3797, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 46, 0, 0, 449, 4091, 0, 0, 0, 3940, 0, 0, 0, 4096, 3909, 32, 0, 3, 4075, 0, 4096, 3949, 0, 2, 0, 0, 340, 3740, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 51, 0, 1, 1111, 4091, 3, 0, 0, 3780, 0, 0, 3, 4096, 2850, 6, 0, 7, 4062, 0, 4096, 3314, 0, 7, 0, 0, 2080, 3210, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 62, 0, 0, 96, 4094, 0, 2, 2, 3952, 2, 2, 0, 4004, 3974, 5, 2, 10, 4091, 2, 4093, 3446, 0, 6, 2, 1, 964, 3953, 0, 3, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 69, 0, 2, 218, 4085, 0, 0, 0, 3813, 0, 0, 2, 4096, 3682, 13, 0, 3, 4069, 0, 4096, 3596, 0, 0, 0, 0, 1283, 3741, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 48, 0, 0, 3256, 650, 139, 0, 913, 3595, 3, 0, 0, 0, 0, 0, 0, 2653, 0, 0, 335, 0, 0, 0, 0, 416, 3722, 24, 0, 0, 0, 0, 0, 1, 4025, 0, 17, 0, 0, 0, 422, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 1848, 2911, 0, 0, 0, 0, 0, 0, 0, 0, 3671, 424, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4511, 2988, 4162, 4043, 4117, 4759, 4095] avg=4096.0000 max=4759 max_rank=6 min=2988 min_rank=2 imbalance_ratio=1.161865 max_load_reduction=28009 max_load_reduction_pct=0.854767 moved_to_echo=17014 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 46, 0, 0, 3256, 682, 139, 0, 449, 3599, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 336, 0, 0, 0, 0, 416, 3909, 32, 0, 0, 0, 0, 0, 3, 4075, 0, 17, 0, 0, 0, 422, 3949, 0, 2, 0, 0, 0, 0, 0, 0, 340, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 3674, 424, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4048, 4276, 4357, 4095, 4373, 3398, 4098] avg=4096.0000 max=4373 max_rank=5 min=3398 min_rank=6 imbalance_ratio=1.067627 max_load_reduction=28395 max_load_reduction_pct=0.866547 moved_to_echo=15964 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 0, 1, 3256, 673, 139, 0, 565, 3603, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 336, 0, 0, 0, 0, 416, 3928, 10, 0, 0, 0, 0, 0, 3, 4084, 0, 17, 0, 0, 0, 422, 3728, 0, 0, 0, 0, 0, 0, 0, 0, 412, 3015, 0, 0, 0, 0, 1, 0, 0, 0, 3674, 424, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4168, 4385, 4354, 4104, 4150, 3427, 4099] avg=4096.0000 max=4385 max_rank=2 min=3427 min_rank=6 imbalance_ratio=1.070557 max_load_reduction=28383 max_load_reduction_pct=0.866180 moved_to_echo=15780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 0, 1, 3256, 586, 139, 0, 1111, 3599, 3, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 336, 0, 0, 0, 3, 416, 2850, 6, 0, 0, 0, 0, 0, 7, 4062, 0, 17, 0, 0, 0, 422, 3314, 0, 7, 0, 0, 0, 0, 0, 0, 2080, 2624, 0, 0, 0, 0, 0, 0, 0, 0, 3674, 424, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4713, 4116, 3275, 4086, 3743, 4704, 4098] avg=4096.0000 max=4713 max_rank=1 min=3275 min_rank=3 imbalance_ratio=1.150635 max_load_reduction=28055 max_load_reduction_pct=0.856171 moved_to_echo=16631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 3, 0, 3187, 742, 139, 0, 257, 3598, 0, 7, 0, 0, 0, 0, 8, 4059, 6, 7, 336, 0, 0, 0, 0, 407, 3599, 12, 0, 0, 0, 0, 0, 5, 4096, 7, 17, 0, 0, 0, 421, 3780, 7, 7, 0, 0, 0, 0, 0, 2, 630, 3327, 0, 0, 0, 0, 0, 8, 0, 0, 3667, 415, 0, 0] rerouted_tokens_per_ep_rank=[4083, 3862, 4416, 4018, 4125, 4215, 3959, 4090] avg=4096.0000 max=4416 max_rank=2 min=3862 min_rank=1 imbalance_ratio=1.078125 max_load_reduction=28352 max_load_reduction_pct=0.865234 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 3255, 693, 139, 0, 362, 3604, 0, 0, 0, 0, 0, 0, 0, 3804, 0, 0, 336, 0, 0, 0, 0, 416, 3688, 3, 0, 0, 0, 0, 0, 3, 4080, 0, 17, 0, 0, 0, 422, 3857, 0, 1, 0, 0, 0, 0, 0, 0, 880, 3104, 0, 0, 0, 0, 0, 0, 0, 0, 3674, 424, 0, 0] rerouted_tokens_per_ep_rank=[4093, 3966, 4140, 4107, 4100, 4280, 3984, 4098] avg=4096.0000 max=4280 max_rank=5 min=3966 min_rank=1 imbalance_ratio=1.044922 max_load_reduction=28488 max_load_reduction_pct=0.869385 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 69, 0, 2, 3258, 686, 142, 0, 218, 3582, 0, 0, 0, 0, 0, 0, 0, 3813, 0, 0, 340, 0, 0, 0, 2, 412, 3682, 13, 0, 0, 0, 0, 0, 3, 4069, 0, 21, 0, 0, 0, 420, 3596, 0, 0, 0, 0, 0, 0, 0, 0, 1283, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 3676, 426, 0, 0] rerouted_tokens_per_ep_rank=[4157, 3800, 4153, 4109, 4093, 4016, 4338, 4102] avg=4096.0000 max=4338 max_rank=6 min=3800 min_rank=1 imbalance_ratio=1.059082 max_load_reduction=28430 max_load_reduction_pct=0.867615 moved_to_echo=16549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 62, 0, 0, 3183, 721, 139, 0, 96, 3602, 0, 2, 0, 0, 0, 0, 2, 3952, 2, 2, 336, 0, 0, 0, 0, 407, 3974, 5, 0, 0, 0, 0, 2, 10, 4091, 2, 17, 0, 0, 0, 422, 3446, 0, 6, 0, 0, 0, 0, 2, 1, 964, 3232, 0, 0, 0, 0, 0, 3, 0, 0, 3671, 414, 0, 0] rerouted_tokens_per_ep_rank=[4105, 3700, 4294, 4386, 4122, 3874, 4199, 4088] avg=4096.0000 max=4386 max_rank=3 min=3700 min_rank=1 imbalance_ratio=1.070801 max_load_reduction=28382 max_load_reduction_pct=0.866150 moved_to_echo=16283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 384, 128, 128, 25984, 5504, 1152, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 8448, 24448, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 28800, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 3328, 29440, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3456, 29440, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 29440, 3456, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 30080, 128, 128, 2816, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 32640, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 2, 0, 3990, 0, 1133, 0, 1014, 0, 3210, 1, 0, 2237, 4092, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 706, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4049, 0, 280, 0, 370, 0, 3101, 0, 0, 3533, 4094, 3, 4096, 0, 0, 0, 4096, 0, 0, 0, 954, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[7, 7, 0, 4085, 0, 9, 3883, 11, 1847, 48, 1463, 7, 1724, 0, 8, 2058, 4085, 19, 4062, 0, 8, 9, 4087, 94, 4, 0, 1115, 29, 0, 4087, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 3951, 0, 1632, 5, 1083, 0, 2534, 0, 0, 2551, 4089, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 539, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 3195, 0, 1093, 0, 1921, 0, 3327, 0, 0, 2431, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 322, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 6, 0, 3431, 0, 1366, 9, 955, 0, 3725, 3, 0, 2647, 4079, 0, 4096, 0, 0, 0, 4096, 0, 0, 3, 166, 0, 0, 4093, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[2, 1, 0, 4093, 0, 2, 3953, 1, 2007, 2, 435, 2, 2207, 0, 3, 3032, 4094, 3, 4085, 0, 2, 2, 4093, 67, 2, 0, 581, 1, 2, 4090, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2, 0, 3773, 0, 1368, 1, 698, 0, 3574, 2, 0, 2683, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 191, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4049, 0, 310, 21, 0, 0, 280, 0, 370, 0, 1256, 569, 0, 0, 1535, 0, 0, 3533, 0, 0, 0, 0, 4094, 3, 3, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 954, 0, 3524, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4380, 2475, 5068, 4100, 4075, 4478, 4096] avg=4096.0000 max=5068 max_rank=3 min=2475 min_rank=2 imbalance_ratio=1.237305 max_load_reduction=27700 max_load_reduction_pct=0.845337 moved_to_echo=16749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 2, 0, 3990, 0, 321, 21, 0, 0, 1133, 0, 1014, 0, 1300, 569, 0, 0, 1589, 1, 0, 2237, 0, 0, 0, 0, 4092, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 1, 706, 0, 3524, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4334, 4016, 3827, 4095, 4075, 4231, 4095] avg=4096.0000 max=4334 max_rank=1 min=3827 min_rank=3 imbalance_ratio=1.058105 max_load_reduction=28434 max_load_reduction_pct=0.867737 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 3195, 0, 332, 21, 0, 0, 1093, 0, 1921, 0, 1348, 569, 0, 0, 1647, 0, 0, 2431, 0, 0, 0, 0, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 322, 0, 3524, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3549, 4931, 4078, 4097, 4075, 3846, 4096] avg=4096.0000 max=4931 max_rank=2 min=3549 min_rank=1 imbalance_ratio=1.203857 max_load_reduction=27837 max_load_reduction_pct=0.849518 moved_to_echo=16114 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 0, 0, 6, 0, 3431, 0, 372, 21, 0, 0, 1366, 9, 955, 0, 1509, 569, 0, 0, 1844, 3, 0, 2647, 0, 0, 0, 0, 4079, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 3, 166, 0, 3524, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 3830, 4408, 4494, 4082, 4075, 3693, 4093] avg=4096.0000 max=4494 max_rank=3 min=3693 min_rank=6 imbalance_ratio=1.097168 max_load_reduction=28274 max_load_reduction_pct=0.862854 moved_to_echo=15943 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[7, 7, 0, 4085, 0, 0, 0, 0, 0, 9, 3883, 11, 172, 20, 0, 0, 1847, 48, 1463, 7, 698, 564, 0, 0, 854, 0, 8, 2058, 0, 0, 0, 0, 4085, 19, 4, 0, 0, 0, 0, 0, 8, 9, 4067, 94, 0, 0, 0, 0, 4, 0, 1115, 29, 3494, 0, 0, 0, 0, 4087, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4095, 4627, 2920, 4108, 4178, 4642, 4099] avg=4096.0000 max=4642 max_rank=6 min=2920 min_rank=3 imbalance_ratio=1.133301 max_load_reduction=28126 max_load_reduction_pct=0.858337 moved_to_echo=17027 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3951, 0, 253, 21, 0, 0, 1632, 5, 1083, 0, 1026, 569, 0, 0, 1255, 0, 0, 2551, 0, 0, 0, 0, 4089, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 539, 0, 3524, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4225, 4315, 3806, 4092, 4075, 4063, 4096] avg=4096.0000 max=4315 max_rank=2 min=3806 min_rank=3 imbalance_ratio=1.053467 max_load_reduction=28453 max_load_reduction_pct=0.868317 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 0, 2, 0, 3773, 0, 361, 23, 0, 0, 1368, 1, 698, 0, 1451, 571, 0, 0, 1762, 2, 0, 2683, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 191, 0, 3525, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4159, 4089, 4447, 4094, 4071, 3716, 4095] avg=4096.0000 max=4447 max_rank=3 min=3716 min_rank=6 imbalance_ratio=1.085693 max_load_reduction=28321 max_load_reduction_pct=0.864288 moved_to_echo=15976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 4093, 0, 0, 0, 0, 0, 2, 3953, 1, 220, 20, 0, 0, 2007, 2, 435, 2, 894, 567, 0, 0, 1093, 0, 3, 3032, 0, 0, 0, 0, 4094, 3, 3, 0, 0, 0, 0, 0, 2, 2, 4073, 67, 0, 0, 0, 0, 2, 0, 581, 1, 3515, 0, 0, 0, 2, 4090, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4196, 3907, 4128, 4100, 4144, 4099, 4098] avg=4096.0000 max=4196 max_rank=1 min=3907 min_rank=2 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[11648, 128, 128, 21248, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[10752, 128, 8064, 128, 9600, 4608, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 4608, 128, 28160, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 30336, 128, 2432, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4070, 207, 0, 4084, 0, 0, 4096, 4096, 0, 2, 0, 0, 3535, 0, 4096, 0, 0, 4096, 68, 0, 0, 0, 777, 3637, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4071, 225, 0, 4084, 0, 0, 4095, 4096, 0, 3, 0, 0, 4045, 0, 4096, 0, 0, 4096, 96, 0, 0, 0, 98, 3757, 0, 0, 0, 0, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4088, 189, 0, 4092, 0, 0, 4094, 4096, 0, 0, 0, 0, 3918, 0, 4096, 0, 0, 4096, 21, 0, 0, 0, 381, 3695, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[9, 10, 4088, 214, 12, 3836, 0, 8, 4024, 4087, 9, 24, 7, 0, 3279, 0, 4081, 0, 9, 4085, 0, 3, 3, 8, 1427, 3535, 0, 2, 0, 0, 0, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4075, 128, 0, 4094, 0, 0, 4089, 4096, 0, 0, 0, 0, 4077, 3, 4096, 0, 0, 4096, 18, 0, 0, 0, 69, 3922, 0, 0, 0, 0, 5, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4096, 368, 0, 4060, 0, 0, 4090, 4096, 0, 0, 0, 0, 4024, 0, 4096, 0, 0, 4096, 35, 0, 0, 0, 1048, 2759, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4070, 370, 0, 4088, 0, 0, 4090, 4096, 0, 0, 0, 0, 3980, 0, 4096, 0, 0, 4096, 99, 0, 0, 0, 386, 3387, 0, 0, 0, 0, 10, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[2, 2, 4094, 195, 2, 3979, 0, 2, 4075, 4093, 3, 14, 3, 0, 3755, 0, 4086, 0, 3, 4093, 1, 0, 0, 3, 590, 3770, 0, 1, 0, 0, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3846, 207, 0, 0, 0, 0, 0, 4084, 0, 0, 54, 0, 0, 0, 4096, 9, 0, 2, 0, 0, 0, 0, 0, 0, 3535, 0, 224, 36, 5, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 68, 0, 0, 0, 4051, 1, 0, 0, 777, 3578, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4138, 4107, 3800, 4098, 4120, 4355, 4097] avg=4096.0000 max=4355 max_rank=6 min=3800 min_rank=3 imbalance_ratio=1.063232 max_load_reduction=28413 max_load_reduction_pct=0.867096 moved_to_echo=16670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3847, 225, 0, 0, 0, 0, 0, 4084, 0, 0, 56, 0, 0, 0, 4095, 9, 0, 3, 0, 0, 0, 0, 0, 0, 4045, 0, 224, 36, 6, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 96, 0, 0, 0, 4051, 1, 0, 0, 98, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4140, 4107, 4311, 4098, 4148, 3793, 4099] avg=4096.0000 max=4311 max_rank=3 min=3793 min_rank=6 imbalance_ratio=1.052490 max_load_reduction=28457 max_load_reduction_pct=0.868439 moved_to_echo=16138 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3850, 128, 0, 0, 0, 0, 0, 4094, 0, 0, 58, 0, 0, 0, 4089, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3, 225, 36, 6, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 18, 0, 0, 0, 4051, 1, 0, 0, 69, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[3978, 4152, 4098, 4347, 4098, 4070, 3927, 4098] avg=4096.0000 max=4347 max_rank=3 min=3927 min_rank=6 imbalance_ratio=1.061279 max_load_reduction=28421 max_load_reduction_pct=0.867340 moved_to_echo=16193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3863, 189, 0, 0, 0, 0, 0, 4092, 0, 0, 55, 0, 0, 0, 4094, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 225, 36, 5, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 21, 0, 0, 0, 4051, 1, 0, 0, 381, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4147, 4103, 4184, 4098, 4073, 4016, 4095] avg=4096.0000 max=4184 max_rank=3 min=4016 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[9, 10, 3863, 214, 0, 0, 0, 0, 12, 3836, 0, 8, 53, 0, 0, 0, 4024, 9, 9, 24, 0, 0, 0, 0, 7, 0, 3279, 0, 225, 36, 5, 0, 4081, 0, 9, 2, 0, 0, 0, 0, 0, 3, 3, 8, 4042, 1, 0, 0, 1427, 3477, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8, 4082, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3909, 4066, 3552, 4092, 4057, 4906, 4090] avg=4096.0000 max=4906 max_rank=6 min=3552 min_rank=3 imbalance_ratio=1.197754 max_load_reduction=27862 max_load_reduction_pct=0.850281 moved_to_echo=17145 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3870, 368, 0, 0, 0, 0, 0, 4060, 0, 0, 41, 0, 0, 0, 4090, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 226, 36, 4, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 35, 0, 0, 0, 4051, 1, 0, 0, 1048, 2714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4238, 4101, 4099, 4290, 4098, 4087, 3762, 4093] avg=4096.0000 max=4290 max_rank=3 min=3762 min_rank=6 imbalance_ratio=1.047363 max_load_reduction=28478 max_load_reduction_pct=0.869080 moved_to_echo=16040 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3842, 370, 0, 0, 0, 0, 0, 4088, 0, 0, 54, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 228, 41, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 4055, 3, 0, 0, 386, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4212, 4142, 4090, 4258, 4096, 4157, 3710, 4103] avg=4096.0000 max=4258 max_rank=3 min=3710 min_rank=6 imbalance_ratio=1.039551 max_load_reduction=28510 max_load_reduction_pct=0.870056 moved_to_echo=16066 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 3868, 195, 0, 0, 0, 0, 2, 3979, 0, 2, 56, 0, 0, 0, 4075, 6, 3, 14, 0, 0, 0, 0, 3, 0, 3755, 0, 226, 36, 6, 0, 4086, 0, 3, 1, 0, 0, 0, 0, 1, 0, 0, 3, 4051, 1, 0, 0, 590, 3708, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 4091, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4039, 4098, 4026, 4090, 4056, 4299, 4093] avg=4096.0000 max=4299 max_rank=6 min=4026 min_rank=3 imbalance_ratio=1.049561 max_load_reduction=28469 max_load_reduction_pct=0.868805 moved_to_echo=16538 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 30976, 1920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 0, 30720, 128, 1920, 384, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32384, 0, 128, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[384, 128, 128, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4864, 28032, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 101, 4096, 0, 0, 0, 400, 0, 0, 0, 0, 4096, 0, 3595] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 4096, 0, 0, 0, 4092, 4096, 9, 0, 0, 0, 2, 4087, 0, 0, 4096, 9, 4094, 0, 0, 0, 83, 0, 0, 0, 0, 4094, 0, 4005] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 4096, 0, 0, 0, 4090, 4095, 3, 0, 0, 0, 3, 4093, 0, 0, 4089, 18, 4087, 0, 0, 0, 106, 0, 4, 1, 0, 4096, 0, 3972] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4096, 0, 0, 0, 4095, 4092, 4, 0, 0, 0, 0, 4095, 0, 0, 4096, 5, 4088, 0, 0, 0, 125, 0, 0, 0, 0, 4095, 0, 3973] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4084, 0, 8, 8, 4088, 4073, 0, 10, 9, 7, 0, 4083, 0, 10, 4081, 1979, 4087, 5, 12, 0, 49, 149, 0, 0, 0, 3906, 0, 2115] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4096, 0, 0, 0, 4093, 4095, 3, 0, 0, 0, 1, 4095, 0, 0, 4096, 535, 4094, 0, 0, 0, 241, 0, 1, 0, 0, 4096, 0, 3320] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3, 1, 4093, 0, 2, 3, 4093, 4085, 2, 3, 3, 2, 0, 4085, 0, 0, 4093, 1344, 4093, 2, 4, 2, 36, 45, 0, 0, 1, 3884, 0, 2889] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 81, 4096, 0, 0, 0, 138, 0, 0, 0, 0, 4096, 0, 3877] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 3590, 101, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 3460, 505, 2, 0, 0, 636, 0, 3595, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4096, 4097, 3691, 4096, 4367, 4231] avg=4096.0000 max=4367 max_rank=6 min=3691 min_rank=4 imbalance_ratio=1.066162 max_load_reduction=28401 max_load_reduction_pct=0.866730 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 1, 0, 0, 0, 0, 0, 3584, 18, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 106, 0, 4, 1, 3460, 504, 2, 0, 0, 636, 0, 3972, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4096, 4098, 4097, 3602, 4087, 4077, 4608] avg=4096.0000 max=4608 max_rank=7 min=3602 min_rank=4 imbalance_ratio=1.125000 max_load_reduction=28160 max_load_reduction_pct=0.859375 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 3590, 5, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 3459, 505, 0, 0, 0, 636, 0, 3973, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4098, 4096, 3595, 4088, 4089, 4609] avg=4096.0000 max=4609 max_rank=7 min=3595 min_rank=4 imbalance_ratio=1.125244 max_load_reduction=28159 max_load_reduction_pct=0.859344 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 4092, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 2, 4087, 4, 0, 0, 0, 0, 0, 3579, 9, 0, 0, 0, 0, 4094, 0, 0, 0, 6, 2, 0, 0, 83, 0, 0, 0, 3463, 511, 0, 0, 0, 631, 0, 4005, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4090, 4105, 4093, 3588, 4102, 4057, 4636] avg=4096.0000 max=4636 max_rank=7 min=3588 min_rank=4 imbalance_ratio=1.131836 max_load_reduction=28132 max_load_reduction_pct=0.858521 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 3590, 535, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 241, 0, 1, 0, 3460, 505, 0, 0, 0, 636, 0, 3320, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4096, 4100, 4097, 4125, 4094, 4207, 3956] avg=4096.0000 max=4207 max_rank=6 min=3956 min_rank=7 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4087, 0, 0, 0, 4084, 0, 8, 8, 0, 0, 0, 0, 0, 4073, 0, 10, 0, 0, 0, 0, 9, 7, 0, 4083, 1, 0, 0, 0, 0, 10, 3577, 1979, 0, 0, 0, 0, 4087, 5, 12, 0, 0, 0, 0, 0, 49, 149, 0, 0, 3299, 503, 1, 0, 0, 607, 0, 2115, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4100, 4083, 4100, 5566, 4104, 4001, 2722] avg=4096.0000 max=5566 max_rank=4 min=2722 min_rank=7 imbalance_ratio=1.358887 max_load_reduction=27202 max_load_reduction_pct=0.830139 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 3589, 81, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 3460, 505, 0, 0, 0, 636, 0, 3877, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4098, 4097, 3670, 4096, 4103, 4513] avg=4096.0000 max=4513 max_rank=7 min=3670 min_rank=4 imbalance_ratio=1.101807 max_load_reduction=28255 max_load_reduction_pct=0.862274 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 1, 4093, 0, 0, 0, 4093, 0, 2, 3, 0, 0, 0, 0, 0, 4085, 2, 3, 0, 0, 0, 0, 3, 2, 0, 4085, 1, 0, 0, 0, 0, 0, 3587, 1344, 0, 0, 0, 0, 4093, 2, 4, 2, 0, 0, 0, 0, 36, 45, 0, 0, 3281, 505, 0, 0, 1, 603, 0, 2889, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4098, 4090, 4091, 4931, 4101, 3867, 3493] avg=4096.0000 max=4931 max_rank=4 min=3493 min_rank=7 imbalance_ratio=1.203857 max_load_reduction=27837 max_load_reduction_pct=0.849518 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[1280, 256, 128, 128, 27392, 4096, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 5120, 0, 27776, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 28800, 4096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4067, 0, 4096, 0, 0, 0, 0, 4076, 0, 0, 0, 28, 0, 5, 4096, 0, 0, 0, 2, 2022, 0, 2133, 4051, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4065, 0, 4095, 0, 0, 0, 0, 4064, 0, 0, 0, 27, 0, 1, 4096, 0, 0, 0, 24, 2515, 0, 1887, 3802, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1, 8, 6, 10, 4072, 0, 4, 8, 3869, 42, 2918, 9, 7, 7, 9, 4084, 13, 178, 8, 4, 0, 0, 4076, 0, 11, 0, 601, 3685, 0, 967, 4085, 4086] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4096, 0, 0, 0, 3951, 0, 4096, 0, 0, 0, 0, 4070, 0, 0, 0, 14, 0, 7, 4096, 0, 0, 0, 14, 2697, 0, 1808, 3823, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3678, 0, 4096, 0, 0, 0, 0, 4063, 0, 1, 0, 12, 0, 3, 4096, 0, 0, 0, 2, 2713, 0, 2377, 3535, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[2, 3, 1, 3, 4091, 0, 0, 2, 3999, 7, 3300, 3, 2, 2, 3, 4092, 4, 116, 2, 7, 1, 0, 4089, 0, 3, 0, 261, 2632, 0, 2101, 3949, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3970, 0, 4096, 0, 0, 0, 0, 4043, 0, 0, 0, 72, 0, 8, 4096, 0, 0, 0, 0, 2323, 0, 2150, 3818, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3935, 0, 4096, 0, 0, 0, 0, 4084, 0, 2, 0, 2, 0, 0, 4096, 0, 0, 0, 4, 2546, 0, 2821, 2990, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 247, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 20, 0, 0, 0, 0, 0, 0, 27, 3818, 333, 3, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 24, 2515, 1355, 0, 0, 0, 0, 1887, 2093, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4097, 4342, 4084, 4181, 4097, 3894, 3980] avg=4096.0000 max=4342 max_rank=2 min=3894 min_rank=6 imbalance_ratio=1.060059 max_load_reduction=28426 max_load_reduction_pct=0.867493 moved_to_echo=16152 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 247, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 21, 0, 0, 0, 0, 0, 0, 28, 3820, 355, 3, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 2022, 1444, 0, 0, 0, 0, 2133, 2229, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4098, 4343, 4097, 4206, 4101, 3468, 4362] avg=4096.0000 max=4362 max_rank=7 min=3468 min_rank=6 imbalance_ratio=1.064941 max_load_reduction=28406 max_load_reduction_pct=0.866882 moved_to_echo=16137 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 240, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 20, 0, 0, 0, 0, 0, 0, 14, 3711, 335, 3, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 2697, 1362, 0, 0, 0, 0, 1808, 2105, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4097, 4336, 4090, 4063, 4103, 4073, 3913] avg=4096.0000 max=4336 max_rank=2 min=3913 min_rank=7 imbalance_ratio=1.058594 max_load_reduction=28432 max_load_reduction_pct=0.867676 moved_to_echo=16152 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 8, 6, 10, 4084, 0, 0, 0, 4072, 0, 4, 8, 2, 0, 0, 0, 235, 42, 2918, 9, 0, 0, 0, 0, 7, 7, 9, 4084, 22, 0, 0, 0, 13, 178, 8, 4, 3634, 358, 2, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 11, 0, 601, 3685, 1456, 0, 0, 0, 0, 967, 2247, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4086, 3204, 4129, 4197, 4076, 5753, 3214] avg=4096.0000 max=5753 max_rank=6 min=3204 min_rank=2 imbalance_ratio=1.404541 max_load_reduction=27015 max_load_reduction_pct=0.824432 moved_to_echo=17240 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 224, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 19, 0, 0, 0, 0, 1, 0, 12, 3454, 310, 3, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 2713, 1260, 0, 0, 0, 0, 2377, 1945, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4097, 4320, 4082, 3780, 4099, 3975, 4322] avg=4096.0000 max=4322 max_rank=7 min=3780 min_rank=4 imbalance_ratio=1.055176 max_load_reduction=28446 max_load_reduction_pct=0.868103 moved_to_echo=16176 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 239, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 23, 0, 0, 0, 0, 0, 0, 72, 3731, 338, 2, 0, 0, 8, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 2323, 1364, 0, 0, 0, 0, 2150, 2085, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4102, 4335, 4066, 4143, 4106, 3687, 4235] avg=4096.0000 max=4335 max_rank=2 min=3687 min_rank=6 imbalance_ratio=1.058350 max_load_reduction=28433 max_load_reduction_pct=0.867706 moved_to_echo=16171 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 239, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 16, 0, 0, 0, 0, 2, 0, 2, 3696, 262, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 2546, 1065, 0, 0, 0, 0, 2821, 1646, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4097, 4335, 4100, 3965, 4096, 3615, 4467] avg=4096.0000 max=4467 max_rank=7 min=3615 min_rank=6 imbalance_ratio=1.090576 max_load_reduction=28301 max_load_reduction_pct=0.863678 moved_to_echo=16143 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 1, 3, 4091, 0, 0, 0, 4091, 0, 0, 2, 1, 0, 0, 0, 243, 7, 3300, 3, 0, 0, 0, 0, 2, 2, 3, 4092, 21, 0, 0, 0, 4, 116, 2, 7, 3756, 346, 2, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 3, 0, 261, 2632, 1407, 0, 0, 0, 0, 2101, 2174, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4094, 3553, 4120, 4233, 4090, 4303, 4275] avg=4096.0000 max=4303 max_rank=6 min=3553 min_rank=2 imbalance_ratio=1.050537 max_load_reduction=28465 max_load_reduction_pct=0.868683 moved_to_echo=16901 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 1024, 21248, 10752, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 16256, 16640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 384, 128, 256, 29696, 2688, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[1920, 128, 30848, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32640, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4089, 4096, 0, 0, 0, 34, 0, 0, 3997, 0, 2, 4068, 0, 0, 4096, 0, 0, 4096, 0, 0, 90, 0, 4096, 0, 8, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 36, 0, 0, 4029, 0, 31, 4091, 0, 0, 4096, 0, 0, 4095, 0, 1, 5, 0, 4096, 0, 11, 0, 0, 0, 0, 4085] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 29, 0, 0, 3994, 0, 9, 4078, 0, 0, 4096, 0, 0, 4096, 0, 0, 77, 0, 4096, 0, 6, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4093, 4096, 0, 0, 0, 165, 0, 0, 3884, 0, 5, 4087, 0, 0, 4096, 0, 0, 4096, 0, 0, 54, 0, 4096, 0, 5, 0, 0, 0, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[9, 9, 3018, 3882, 7, 0, 0, 814, 0, 0, 3595, 10, 0, 3820, 11, 7, 4071, 13, 0, 4085, 0, 0, 1203, 8, 4086, 15, 0, 0, 7, 0, 10, 4088] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[2, 3, 3650, 4017, 2, 0, 2, 493, 0, 0, 3914, 3, 0, 3823, 3, 7, 4087, 3, 2, 4093, 0, 0, 472, 3, 4093, 0, 0, 0, 0, 0, 3, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4082, 4096, 0, 0, 0, 77, 0, 0, 3911, 0, 6, 4087, 0, 0, 4096, 0, 0, 4096, 0, 0, 123, 0, 4096, 0, 2, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4085, 4096, 0, 0, 0, 27, 0, 0, 3958, 0, 8, 4070, 0, 0, 4096, 0, 0, 4096, 0, 0, 128, 0, 4096, 0, 15, 0, 0, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 195, 0, 0, 0, 0, 0, 0, 0, 36, 3886, 0, 0, 0, 0, 0, 4029, 0, 185, 0, 0, 0, 31, 4091, 0, 0, 65, 4, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 1, 5, 0, 3651, 206, 0, 0, 4091, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4291, 3922, 4214, 4191, 4099, 3863, 4102, 4086] avg=4096.0000 max=4291 max_rank=0 min=3863 min_rank=5 imbalance_ratio=1.047607 max_load_reduction=28477 max_load_reduction_pct=0.869049 moved_to_echo=16150 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 195, 0, 0, 0, 0, 0, 0, 0, 34, 3887, 0, 0, 0, 0, 0, 3997, 0, 185, 0, 0, 0, 2, 4068, 0, 0, 65, 4, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 90, 0, 3651, 206, 0, 0, 4091, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4284, 3921, 4182, 4139, 4099, 3947, 4099, 4097] avg=4096.0000 max=4284 max_rank=0 min=3921 min_rank=1 imbalance_ratio=1.045898 max_load_reduction=28484 max_load_reduction_pct=0.869263 moved_to_echo=16242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 195, 0, 0, 0, 0, 0, 0, 0, 165, 3887, 0, 0, 0, 0, 0, 3884, 0, 185, 0, 0, 0, 5, 4087, 0, 0, 65, 4, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 54, 0, 3651, 206, 0, 0, 4091, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4288, 4052, 4069, 4161, 4099, 3911, 4096, 4092] avg=4096.0000 max=4288 max_rank=0 min=3911 min_rank=5 imbalance_ratio=1.046875 max_load_reduction=28480 max_load_reduction_pct=0.869141 moved_to_echo=16198 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 195, 0, 0, 0, 0, 0, 0, 0, 29, 3887, 0, 0, 0, 0, 0, 3994, 0, 185, 0, 0, 0, 9, 4078, 0, 0, 65, 4, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 77, 0, 3651, 206, 0, 0, 4091, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4291, 3916, 4179, 4156, 4099, 3934, 4097, 4096] avg=4096.0000 max=4291 max_rank=0 min=3916 min_rank=1 imbalance_ratio=1.047607 max_load_reduction=28477 max_load_reduction_pct=0.869049 moved_to_echo=16226 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[9, 9, 3018, 185, 0, 0, 0, 0, 7, 0, 0, 814, 3876, 0, 0, 0, 0, 0, 3595, 10, 176, 0, 0, 0, 0, 3820, 11, 7, 61, 4, 0, 0, 4071, 13, 0, 1, 0, 0, 0, 0, 0, 0, 1203, 8, 3460, 208, 0, 0, 4081, 15, 0, 0, 0, 0, 0, 0, 7, 0, 10, 4088, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3221, 4697, 3781, 3903, 4085, 4879, 4096, 4106] avg=4096.0000 max=4879 max_rank=5 min=3221 min_rank=0 imbalance_ratio=1.191162 max_load_reduction=27889 max_load_reduction_pct=0.851105 moved_to_echo=17166 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4082, 195, 0, 0, 0, 0, 0, 0, 0, 77, 3887, 0, 0, 0, 0, 0, 3911, 0, 185, 0, 0, 0, 6, 4087, 0, 0, 65, 4, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 123, 0, 3651, 206, 0, 0, 4091, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4277, 3964, 4096, 4162, 4099, 3980, 4093, 4097] avg=4096.0000 max=4277 max_rank=0 min=3964 min_rank=1 imbalance_ratio=1.044189 max_load_reduction=28491 max_load_reduction_pct=0.869476 moved_to_echo=16269 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4085, 184, 0, 0, 0, 0, 0, 0, 0, 27, 3888, 0, 0, 0, 0, 0, 3958, 0, 190, 0, 0, 0, 8, 4070, 0, 0, 68, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 3654, 208, 0, 0, 4082, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0] rerouted_tokens_per_ep_rank=[4269, 3915, 4148, 4156, 4096, 3990, 4097, 4097] avg=4096.0000 max=4269 max_rank=0 min=3915 min_rank=1 imbalance_ratio=1.042236 max_load_reduction=28499 max_load_reduction_pct=0.869720 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 3650, 192, 0, 0, 0, 0, 2, 0, 2, 493, 3884, 0, 0, 0, 0, 0, 3914, 3, 182, 0, 0, 0, 0, 3823, 3, 7, 63, 4, 0, 0, 4087, 3, 2, 0, 0, 0, 0, 0, 0, 0, 472, 3, 3580, 209, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3847, 4381, 4099, 3900, 4092, 4264, 4088, 4097] avg=4096.0000 max=4381 max_rank=1 min=3847 min_rank=0 imbalance_ratio=1.069580 max_load_reduction=28387 max_load_reduction_pct=0.866302 moved_to_echo=16541 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 31232, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 1792, 31104, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 2176, 128, 29056, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 31360, 128, 1536, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32128, 128, 128, 640, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[3688, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 3963, 0, 4096, 0, 4096, 52, 0, 0, 0, 1179, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3632, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 3886, 0, 4096, 0, 4096, 676, 0, 0, 0, 966, 0, 0, 0, 0, 0, 0, 0, 3128, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3324, 4096, 0, 2, 1, 4096, 4096, 4, 0, 0, 3875, 0, 4096, 0, 4095, 246, 0, 0, 0, 1393, 0, 0, 0, 0, 0, 0, 0, 3444, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[3253, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 3736, 0, 4096, 0, 4096, 672, 0, 0, 0, 1004, 0, 0, 0, 0, 0, 0, 0, 3623, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[3591, 4074, 9, 0, 0, 4002, 4085, 0, 6, 7, 3828, 0, 4083, 28, 3382, 1351, 147, 6, 9, 3495, 8, 0, 0, 9, 0, 11, 19, 574, 0, 0, 5, 39] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3312, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 3669, 0, 4096, 0, 4096, 268, 0, 0, 0, 1931, 0, 0, 0, 0, 0, 0, 0, 3108, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3090, 4088, 0, 0, 0, 4077, 4092, 0, 1, 2, 3830, 0, 4093, 8, 3868, 1260, 63, 2, 3, 2778, 3, 0, 0, 3, 0, 3, 6, 1486, 0, 2, 2, 8] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[2850, 4096, 0, 0, 1, 4096, 4096, 0, 0, 0, 4021, 0, 4096, 0, 4093, 724, 0, 0, 0, 1382, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3632, 754, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 250, 0, 0, 0, 0, 0, 3538, 676, 0, 0, 0, 0, 0, 0, 0, 966, 2303, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3128, 1032, 308, 0, 0, 0, 0, 0, 0, 4082, 7, 1, 0] rerouted_tokens_per_ep_rank=[4386, 4110, 4136, 4214, 3269, 4095, 4468, 4090] avg=4096.0000 max=4468 max_rank=6 min=3269 min_rank=4 imbalance_ratio=1.090820 max_load_reduction=28300 max_load_reduction_pct=0.863647 moved_to_echo=15922 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3688, 754, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 250, 0, 0, 0, 0, 0, 3538, 52, 0, 0, 0, 0, 0, 0, 0, 1179, 2303, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3406, 1032, 308, 0, 0, 0, 0, 0, 0, 4082, 7, 1, 0] rerouted_tokens_per_ep_rank=[4442, 4110, 4213, 3590, 3482, 4095, 4746, 4090] avg=4096.0000 max=4746 max_rank=6 min=3482 min_rank=4 imbalance_ratio=1.158691 max_load_reduction=28022 max_load_reduction_pct=0.855164 moved_to_echo=16413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3324, 754, 0, 2, 0, 0, 0, 0, 1, 4096, 14, 4, 0, 0, 0, 0, 0, 0, 3875, 0, 250, 0, 0, 0, 0, 0, 3537, 246, 0, 0, 0, 0, 0, 0, 0, 1393, 2303, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3444, 1032, 308, 0, 0, 0, 0, 0, 0, 4082, 7, 1, 0] rerouted_tokens_per_ep_rank=[4080, 4115, 4125, 3783, 3696, 4095, 4784, 4090] avg=4096.0000 max=4784 max_rank=6 min=3696 min_rank=4 imbalance_ratio=1.167969 max_load_reduction=27984 max_load_reduction_pct=0.854004 moved_to_echo=16665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3312, 754, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 3669, 0, 250, 0, 0, 0, 0, 0, 3538, 268, 0, 0, 0, 0, 0, 0, 0, 1931, 2303, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3108, 1032, 308, 0, 0, 0, 0, 0, 0, 4082, 7, 1, 0] rerouted_tokens_per_ep_rank=[4066, 4110, 3919, 3806, 4234, 4095, 4448, 4090] avg=4096.0000 max=4448 max_rank=6 min=3806 min_rank=3 imbalance_ratio=1.085938 max_load_reduction=28320 max_load_reduction_pct=0.864258 moved_to_echo=16867 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3591, 751, 9, 0, 0, 0, 0, 0, 0, 4002, 14, 0, 0, 0, 0, 0, 6, 7, 3828, 0, 206, 0, 0, 0, 0, 28, 2922, 1351, 0, 0, 0, 0, 147, 6, 9, 3495, 2290, 0, 0, 0, 8, 0, 0, 9, 4082, 0, 0, 0, 0, 11, 19, 574, 1026, 254, 0, 0, 0, 0, 5, 39, 4071, 7, 1, 0] rerouted_tokens_per_ep_rank=[4351, 4016, 4047, 4301, 5947, 4099, 1884, 4123] avg=4096.0000 max=5947 max_rank=4 min=1884 min_rank=6 imbalance_ratio=1.451904 max_load_reduction=26821 max_load_reduction_pct=0.818512 moved_to_echo=16053 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3253, 743, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 3736, 0, 252, 0, 0, 0, 0, 0, 3534, 672, 0, 0, 0, 0, 0, 0, 0, 1004, 2307, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3623, 1036, 310, 0, 0, 0, 0, 0, 0, 4087, 10, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4105, 3988, 4206, 3311, 4096, 4969, 4097] avg=4096.0000 max=4969 max_rank=6 min=3311 min_rank=4 imbalance_ratio=1.213135 max_load_reduction=27799 max_load_reduction_pct=0.848358 moved_to_echo=16473 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2850, 754, 0, 0, 0, 0, 0, 0, 1, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 250, 0, 0, 0, 0, 0, 3535, 724, 0, 0, 0, 0, 0, 0, 0, 1382, 2303, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3313, 1032, 308, 0, 0, 0, 0, 0, 0, 4082, 7, 1, 0] rerouted_tokens_per_ep_rank=[3604, 4111, 4271, 4259, 3685, 4095, 4653, 4090] avg=4096.0000 max=4653 max_rank=6 min=3604 min_rank=0 imbalance_ratio=1.135986 max_load_reduction=28115 max_load_reduction_pct=0.858002 moved_to_echo=16523 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3090, 753, 0, 0, 0, 0, 0, 0, 0, 4077, 14, 0, 0, 0, 0, 0, 1, 2, 3830, 0, 236, 0, 0, 0, 0, 8, 3341, 1260, 0, 0, 0, 0, 63, 2, 3, 2778, 2298, 0, 0, 0, 3, 0, 0, 3, 4092, 0, 0, 0, 0, 3, 6, 1486, 1030, 291, 0, 0, 0, 2, 2, 8, 4078, 7, 1, 0] rerouted_tokens_per_ep_rank=[3843, 4091, 4069, 4609, 5144, 4098, 2816, 4098] avg=4096.0000 max=5144 max_rank=4 min=2816 min_rank=6 imbalance_ratio=1.255859 max_load_reduction=27624 max_load_reduction_pct=0.843018 moved_to_echo=16156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[256, 128, 128, 14208, 18432, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[26752, 6144, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 22144, 8320, 2432, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 27520, 5376, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 30848, 0, 2048, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[204, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 81, 0, 0, 4095, 0, 1, 4095, 0, 0, 0, 4080, 0, 1, 3888, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[102, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 58, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 4010, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[2399, 8, 34, 10, 7, 0, 0, 69, 3925, 69, 465, 0, 14, 3995, 0, 0, 4086, 0, 0, 6, 3658, 0, 0, 1696, 4062, 4078, 0, 200, 10, 3966, 11, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[137, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 100, 0, 0, 4081, 0, 5, 4091, 0, 0, 0, 4093, 0, 4, 3879, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[278, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 55, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 3776, 4096, 4095, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[113, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 117, 0, 0, 4078, 0, 2, 4094, 0, 0, 0, 4092, 0, 2, 3936, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1641, 2, 9, 3, 3, 0, 0, 16, 3983, 40, 396, 0, 4, 4012, 0, 0, 4093, 0, 0, 2, 3878, 0, 0, 2360, 4087, 4092, 0, 87, 3, 4054, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[102, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 82, 0, 0, 4091, 0, 2, 4094, 0, 0, 0, 4083, 0, 3, 3958, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[102, 0, 0, 0, 3423, 105, 9, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 3906, 0, 58, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 14, 10, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 671, 0, 0, 4010, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[3639, 4087, 3964, 4119, 4097, 4681, 4067, 4114] avg=4096.0000 max=4681 max_rank=5 min=3639 min_rank=0 imbalance_ratio=1.142822 max_load_reduction=28087 max_load_reduction_pct=0.857147 moved_to_echo=16959 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[204, 0, 0, 0, 3411, 105, 9, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 3916, 0, 81, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 14, 10, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 669, 0, 1, 3888, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[3729, 4087, 3997, 4120, 4096, 4558, 4067, 4114] avg=4096.0000 max=4558 max_rank=5 min=3729 min_rank=0 imbalance_ratio=1.112793 max_load_reduction=28210 max_load_reduction_pct=0.860901 moved_to_echo=16835 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[137, 0, 0, 0, 3422, 107, 9, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 3969, 0, 100, 0, 0, 0, 0, 0, 0, 4081, 0, 5, 14, 10, 0, 0, 4091, 0, 0, 0, 1, 0, 0, 0, 671, 0, 4, 3879, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[3675, 4087, 4069, 4110, 4092, 4554, 4067, 4114] avg=4096.0000 max=4554 max_rank=5 min=3675 min_rank=0 imbalance_ratio=1.111816 max_load_reduction=28214 max_load_reduction_pct=0.861023 moved_to_echo=16827 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2399, 8, 34, 10, 3058, 102, 8, 0, 7, 0, 0, 69, 4070, 0, 0, 0, 3810, 69, 465, 0, 0, 0, 0, 0, 14, 3995, 0, 0, 13, 10, 0, 0, 4086, 0, 0, 6, 1, 0, 0, 0, 600, 0, 0, 1696, 0, 0, 0, 0, 4034, 0, 0, 200, 0, 0, 0, 0, 10, 3966, 11, 0, 17, 0, 0, 0] rerouted_tokens_per_ep_rank=[5619, 4146, 4344, 4032, 4093, 2296, 4234, 4004] avg=4096.0000 max=5619 max_rank=0 min=2296 min_rank=5 imbalance_ratio=1.371826 max_load_reduction=27149 max_load_reduction_pct=0.828522 moved_to_echo=14627 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[102, 0, 0, 0, 3413, 106, 9, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 3945, 0, 82, 0, 0, 0, 0, 0, 0, 4091, 0, 2, 14, 10, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 670, 0, 3, 3958, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[3630, 4087, 4027, 4117, 4095, 4631, 4067, 4114] avg=4096.0000 max=4631 max_rank=5 min=3630 min_rank=0 imbalance_ratio=1.130615 max_load_reduction=28137 max_load_reduction_pct=0.858673 moved_to_echo=16907 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[278, 0, 0, 0, 3416, 107, 9, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 3973, 1, 55, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 14, 10, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 670, 0, 0, 3776, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[3810, 4086, 4029, 4119, 4097, 4446, 4067, 4114] avg=4096.0000 max=4446 max_rank=5 min=3810 min_rank=0 imbalance_ratio=1.085449 max_load_reduction=28322 max_load_reduction_pct=0.864319 moved_to_echo=16724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[113, 0, 0, 0, 3424, 109, 10, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 3920, 0, 117, 0, 0, 0, 0, 0, 0, 4078, 0, 2, 17, 14, 0, 0, 4094, 0, 0, 0, 8, 0, 0, 0, 668, 0, 2, 3936, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 0, 0] rerouted_tokens_per_ep_rank=[3656, 4086, 4037, 4111, 4102, 4606, 4054, 4116] avg=4096.0000 max=4606 max_rank=5 min=3656 min_rank=0 imbalance_ratio=1.124512 max_load_reduction=28162 max_load_reduction_pct=0.859436 moved_to_echo=16878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1641, 2, 9, 3, 3242, 104, 9, 0, 3, 0, 0, 16, 4083, 0, 0, 0, 3865, 40, 396, 0, 0, 0, 0, 0, 4, 4012, 0, 0, 14, 10, 0, 0, 4093, 0, 0, 2, 1, 0, 0, 0, 636, 0, 0, 2360, 0, 0, 0, 0, 4058, 0, 0, 87, 0, 0, 0, 0, 3, 4054, 3, 0, 18, 0, 0, 0] rerouted_tokens_per_ep_rank=[5010, 4102, 4301, 4040, 4096, 2996, 4145, 4078] avg=4096.0000 max=5010 max_rank=0 min=2996 min_rank=5 imbalance_ratio=1.223145 max_load_reduction=27758 max_load_reduction_pct=0.847107 moved_to_echo=15315 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4992, 128, 128, 128, 26880, 896, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5376, 0, 128, 27520, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 128, 0, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[31360, 128, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32512, 0, 0, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 26, 0, 0, 1, 0, 0, 3884, 4096, 1, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 184, 0, 0, 0, 0, 4096, 5, 4092, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 10, 0, 0, 11, 0, 0, 3955, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 3, 118, 0, 0, 0, 0, 4096, 1, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 63, 0, 0, 14, 0, 0, 3875, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4, 143, 0, 0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 72, 0, 0, 25, 0, 0, 3851, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 148, 0, 0, 0, 0, 4096, 5, 4092, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4092, 0, 45, 0, 0, 33, 0, 0, 3947, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 11, 60, 0, 0, 0, 0, 4096, 12, 4088, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4081, 0, 0, 11, 10, 0, 112, 9, 4076, 4086, 0, 0, 0, 0, 0, 11, 4077, 4071, 3968, 12, 9, 0, 20, 19, 0, 0, 7, 4084, 0, 4085, 20, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 25, 0, 0, 4046, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 26, 0, 0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4090, 0, 0, 3, 3, 0, 30, 3, 4060, 4093, 0, 0, 0, 0, 0, 3, 4091, 4084, 4064, 3, 3, 0, 33, 5, 0, 0, 2, 4093, 0, 4093, 12, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4069, 0, 63, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4066, 0, 0, 0, 3875, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 4, 143, 0, 3961, 24, 10, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4080, 4010, 4097, 4115, 4144, 4096, 4094] avg=4096.0000 max=4144 max_rank=5 min=4010 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16449 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4069, 0, 26, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4066, 0, 0, 0, 3884, 135, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 0, 184, 0, 3961, 24, 10, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4090, 0, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4067, 4020, 4097, 4115, 4181, 4096, 4097] avg=4096.0000 max=4181 max_rank=5 min=4020 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16489 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4069, 0, 72, 0, 0, 0, 0, 0, 0, 25, 0, 0, 4066, 0, 0, 0, 3851, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 2, 148, 0, 3961, 24, 10, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4090, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4091, 3986, 4097, 4115, 4147, 4096, 4095] avg=4096.0000 max=4147 max_rank=5 min=3986 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16453 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4071, 0, 10, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4066, 0, 0, 0, 3955, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 3, 118, 0, 3961, 24, 10, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4077, 4090, 4097, 4115, 4118, 4096, 4094] avg=4096.0000 max=4118 max_rank=5 min=4077 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=28650 max_load_reduction_pct=0.874329 moved_to_echo=16423 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4057, 0, 0, 11, 0, 0, 0, 0, 10, 0, 112, 9, 4041, 0, 0, 0, 4076, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4077, 1, 0, 0, 0, 19, 3968, 12, 0, 0, 0, 0, 9, 0, 20, 19, 3952, 24, 10, 2, 0, 0, 7, 4084, 0, 0, 0, 0, 0, 4083, 20, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4172, 4210, 4089, 3999, 4036, 4091, 4103] avg=4096.0000 max=4210 max_rank=2 min=3999 min_rank=4 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4066, 0, 45, 0, 0, 0, 0, 0, 0, 33, 0, 0, 4067, 0, 0, 0, 3947, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 11, 60, 0, 3967, 26, 14, 6, 0, 0, 0, 4096, 6, 0, 0, 0, 12, 4076, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4100, 4076, 4099, 4108, 4084, 4102, 4088] avg=4096.0000 max=4111 max_rank=0 min=4076 min_rank=2 imbalance_ratio=1.003662 max_load_reduction=28657 max_load_reduction_pct=0.874542 moved_to_echo=16382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4071, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 4066, 0, 0, 0, 4046, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 0, 26, 0, 3961, 24, 10, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4091, 4181, 4097, 4115, 4023, 4096, 4094] avg=4096.0000 max=4181 max_rank=2 min=4023 min_rank=5 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16328 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4066, 0, 0, 3, 0, 0, 0, 0, 3, 0, 30, 3, 4054, 0, 0, 0, 4060, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 1, 0, 0, 0, 19, 4064, 3, 0, 0, 0, 0, 3, 0, 33, 5, 3958, 24, 10, 2, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 4091, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4090, 4195, 4095, 4086, 4035, 4095, 4103] avg=4096.0000 max=4195 max_rank=2 min=4035 min_rank=5 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16319 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 768, 128, 31744, 256, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31744, 1152, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 256, 32640, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32640, 0, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 256, 128, 32512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 3821, 0, 0, 1213, 0, 0, 4096, 0, 0, 0, 0, 47, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3053, 0, 0, 58, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 3904, 0, 0, 588, 0, 0, 4096, 0, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3570, 0, 0, 102, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 3854, 0, 0, 414, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3867, 0, 0, 51, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[15, 3318, 10, 0, 2177, 11, 0, 4075, 7, 3, 11, 11, 1988, 12, 0, 0, 4060, 0, 70, 4085, 12, 0, 4, 4083, 1170, 0, 0, 40, 8, 3514, 0, 4084] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4017, 0, 0, 2368, 0, 0, 4096, 0, 0, 0, 0, 33, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1736, 0, 0, 39, 0, 4095, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 3959, 0, 0, 202, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3796, 0, 0, 235, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[2, 3573, 3, 0, 1608, 3, 2, 4089, 2, 0, 3, 3, 811, 4, 0, 0, 4090, 0, 34, 4093, 3, 0, 1, 4092, 2397, 0, 0, 61, 2, 3799, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 3923, 0, 0, 624, 0, 0, 4096, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3575, 0, 0, 57, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3904, 0, 0, 288, 7, 0, 0, 588, 0, 0, 2948, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 28, 0, 0, 0, 3726, 3, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3570, 0, 0, 102, 860, 260, 0, 0, 0, 4096, 0, 110, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4199, 3536, 4093, 3757, 4089, 4096, 4792, 4206] avg=4096.0000 max=4792 max_rank=6 min=3536 min_rank=1 imbalance_ratio=1.169922 max_load_reduction=27976 max_load_reduction_pct=0.853760 moved_to_echo=17183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3854, 0, 0, 288, 7, 0, 0, 414, 0, 0, 2948, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 6, 0, 0, 0, 3726, 3, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3867, 0, 0, 51, 860, 260, 0, 0, 0, 4096, 0, 110, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 3362, 4093, 3735, 4089, 4096, 5038, 4206] avg=4096.0000 max=5038 max_rank=6 min=3362 min_rank=1 imbalance_ratio=1.229980 max_load_reduction=27730 max_load_reduction_pct=0.846252 moved_to_echo=17429 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3821, 0, 0, 288, 7, 0, 0, 1213, 0, 0, 2948, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 47, 0, 0, 0, 3726, 3, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3053, 0, 0, 58, 860, 260, 0, 0, 0, 4096, 0, 110, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4161, 4093, 3776, 4089, 4096, 4231, 4206] avg=4096.0000 max=4231 max_rank=6 min=3776 min_rank=3 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3959, 0, 0, 288, 7, 0, 0, 202, 0, 0, 2948, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 3726, 4, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3796, 0, 0, 235, 860, 260, 0, 0, 0, 4096, 0, 110, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4254, 3150, 4092, 3730, 4089, 4096, 5151, 4206] avg=4096.0000 max=5151 max_rank=6 min=3150 min_rank=1 imbalance_ratio=1.257568 max_load_reduction=27617 max_load_reduction_pct=0.842804 moved_to_echo=17542 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[15, 3318, 10, 0, 286, 7, 0, 0, 2177, 11, 0, 2933, 0, 0, 0, 0, 7, 3, 11, 11, 4082, 0, 0, 0, 1988, 12, 0, 0, 3715, 3, 0, 0, 4053, 0, 70, 0, 0, 0, 0, 0, 12, 0, 4, 4083, 0, 0, 0, 0, 1170, 0, 0, 40, 856, 259, 0, 0, 8, 3514, 0, 110, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3636, 5121, 4114, 5718, 4123, 4099, 2325, 3632] avg=4096.0000 max=5718 max_rank=3 min=2325 min_rank=6 imbalance_ratio=1.395996 max_load_reduction=27050 max_load_reduction_pct=0.825500 moved_to_echo=14179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4017, 0, 0, 288, 7, 0, 0, 2368, 0, 0, 2948, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 33, 0, 0, 0, 3726, 4, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1736, 0, 0, 39, 860, 260, 0, 0, 0, 4095, 0, 110, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4312, 5316, 4092, 3763, 4089, 4096, 2895, 4205] avg=4096.0000 max=5316 max_rank=1 min=2895 min_rank=6 imbalance_ratio=1.297852 max_load_reduction=27452 max_load_reduction_pct=0.837769 moved_to_echo=15285 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3923, 0, 0, 291, 13, 3, 0, 624, 0, 0, 2942, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 13, 0, 0, 0, 3732, 3, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3575, 0, 0, 57, 863, 264, 0, 0, 0, 4096, 0, 100, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4230, 3566, 4093, 3748, 4083, 4093, 4759, 4196] avg=4096.0000 max=4759 max_rank=6 min=3566 min_rank=1 imbalance_ratio=1.161865 max_load_reduction=28009 max_load_reduction_pct=0.854767 moved_to_echo=17131 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 3573, 3, 0, 287, 7, 0, 0, 1608, 3, 2, 2943, 0, 0, 0, 0, 2, 0, 3, 3, 4090, 0, 0, 0, 811, 4, 0, 0, 3723, 3, 0, 0, 4083, 0, 34, 0, 0, 0, 0, 0, 3, 0, 1, 4092, 0, 0, 0, 0, 2397, 0, 0, 61, 859, 260, 0, 0, 2, 3799, 0, 110, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3872, 4556, 4098, 4541, 4117, 4096, 3577, 3911] avg=4096.0000 max=4556 max_rank=1 min=3577 min_rank=6 imbalance_ratio=1.112305 max_load_reduction=28212 max_load_reduction_pct=0.860962 moved_to_echo=15701 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[23168, 0, 0, 768, 6912, 2176, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[9216, 128, 128, 23680, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[2944, 128, 0, 0, 29824, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 30464, 128, 0, 2304, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32000, 0, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[2648, 4093, 4096, 4096, 0, 0, 0, 4096, 0, 0, 9, 3, 0, 4096, 0, 0, 804, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 635, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[2649, 4087, 4084, 3612, 15, 19, 65, 4080, 7, 3, 0, 3, 12, 4077, 0, 2, 277, 9, 10, 0, 0, 14, 4016, 0, 0, 0, 4086, 9, 0, 1625, 0, 7] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[2326, 4095, 4096, 4096, 0, 0, 0, 4095, 0, 0, 1, 1, 0, 4095, 0, 0, 600, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1171, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1137, 4096, 4096, 4095, 0, 0, 0, 4086, 0, 0, 0, 1, 0, 4096, 0, 0, 987, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1982, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1284, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 4096, 0, 0, 2363, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 445, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1999, 4096, 4096, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 1279, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 821, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[988, 4093, 4092, 3933, 4, 6, 55, 4092, 2, 0, 0, 32, 3, 4081, 0, 8, 437, 3, 2, 0, 0, 4, 4061, 0, 0, 0, 4093, 3, 0, 2773, 0, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1574, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4085, 0, 0, 1378, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1144, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2648, 0, 0, 2317, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 9, 3, 4085, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 804, 0, 0, 0, 3071, 8, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 635, 0, 0, 1779, 1025, 1, 0] rerouted_tokens_per_ep_rank=[4965, 4081, 4097, 4098, 3883, 4108, 4096, 3440] avg=4096.0000 max=4965 max_rank=0 min=3440 min_rank=7 imbalance_ratio=1.212158 max_load_reduction=27803 max_load_reduction_pct=0.848480 moved_to_echo=15527 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1137, 0, 0, 2316, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 987, 0, 0, 0, 3071, 8, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1982, 0, 0, 1779, 1025, 1, 0] rerouted_tokens_per_ep_rank=[3453, 4071, 4089, 4098, 4066, 4108, 4096, 4787] avg=4096.0000 max=4787 max_rank=7 min=3453 min_rank=0 imbalance_ratio=1.168701 max_load_reduction=27981 max_load_reduction_pct=0.853912 moved_to_echo=17057 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1284, 0, 0, 2317, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4, 4088, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 2363, 0, 0, 0, 3071, 8, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 445, 0, 0, 1779, 1025, 1, 0] rerouted_tokens_per_ep_rank=[3601, 4081, 4092, 4098, 5442, 4108, 4096, 3250] avg=4096.0000 max=5442 max_rank=4 min=3250 min_rank=7 imbalance_ratio=1.328613 max_load_reduction=27326 max_load_reduction_pct=0.833923 moved_to_echo=16896 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2326, 0, 0, 2317, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 1, 1, 4087, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 600, 0, 0, 0, 3071, 8, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1171, 0, 0, 1779, 1025, 1, 0] rerouted_tokens_per_ep_rank=[4643, 4080, 4089, 4097, 3679, 4108, 4096, 3976] avg=4096.0000 max=4643 max_rank=0 min=3679 min_rank=4 imbalance_ratio=1.133545 max_load_reduction=28125 max_load_reduction_pct=0.858307 moved_to_echo=15859 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2649, 0, 0, 2043, 0, 0, 0, 0, 15, 19, 65, 4065, 0, 0, 0, 0, 7, 3, 0, 3, 4080, 0, 0, 0, 12, 4077, 0, 2, 2, 0, 0, 0, 277, 9, 10, 0, 3062, 7, 0, 0, 0, 14, 4016, 0, 12, 0, 0, 0, 0, 0, 4086, 9, 0, 0, 0, 0, 0, 1625, 0, 7, 1569, 1022, 1, 0] rerouted_tokens_per_ep_rank=[4692, 4164, 4093, 4093, 3365, 4042, 4095, 4224] avg=4096.0000 max=4692 max_rank=0 min=3365 min_rank=4 imbalance_ratio=1.145508 max_load_reduction=28076 max_load_reduction_pct=0.856812 moved_to_echo=15726 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1999, 0, 0, 2317, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 1279, 0, 0, 0, 3071, 8, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 821, 0, 0, 1779, 1025, 1, 0] rerouted_tokens_per_ep_rank=[4316, 4078, 4088, 4098, 4358, 4108, 4096, 3626] avg=4096.0000 max=4358 max_rank=4 min=3626 min_rank=7 imbalance_ratio=1.063965 max_load_reduction=28410 max_load_reduction_pct=0.867004 moved_to_echo=16188 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1574, 0, 0, 2311, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 11, 4089, 0, 0, 0, 0, 4085, 0, 0, 7, 0, 0, 0, 1378, 0, 0, 0, 3072, 7, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 1144, 0, 0, 1785, 1024, 4, 0] rerouted_tokens_per_ep_rank=[3885, 4071, 4100, 4092, 4457, 4109, 4097, 3957] avg=4096.0000 max=4457 max_rank=4 min=3885 min_rank=0 imbalance_ratio=1.088135 max_load_reduction=28311 max_load_reduction_pct=0.863983 moved_to_echo=16620 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[988, 0, 0, 2225, 0, 0, 0, 0, 4, 6, 55, 4077, 0, 0, 0, 0, 2, 0, 0, 32, 4086, 0, 0, 0, 3, 4081, 0, 8, 2, 0, 0, 0, 437, 3, 2, 0, 3069, 7, 0, 0, 0, 4, 4061, 0, 12, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 2773, 0, 3, 1708, 1023, 1, 0] rerouted_tokens_per_ep_rank=[3213, 4142, 4120, 4094, 3518, 4077, 4096, 5508] avg=4096.0000 max=5508 max_rank=7 min=3213 min_rank=0 imbalance_ratio=1.344727 max_load_reduction=27260 max_load_reduction_pct=0.831909 moved_to_echo=17199 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[14720, 0, 0, 18176, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 10624, 0, 128, 14080, 8320, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[8192, 128, 128, 0, 24576, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 128, 32640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2239, 0, 1757, 769, 762, 3, 1510, 809, 1780, 2307, 0, 72, 939, 1115, 547, 2893, 50, 1938, 947, 0, 0, 2532, 2022, 1957, 0, 663, 1250, 685, 572, 2009, 641, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2744, 0, 1226, 587, 908, 2, 1158, 1144, 1522, 2563, 0, 198, 1204, 1014, 695, 2114, 115, 1651, 887, 0, 2, 1609, 2608, 1473, 0, 883, 1606, 925, 938, 2425, 565, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2911, 1, 1096, 1021, 932, 0, 1749, 611, 1003, 2186, 0, 156, 1036, 1107, 775, 2525, 97, 1899, 597, 0, 0, 1615, 2830, 1841, 11, 1174, 1281, 970, 754, 1990, 599, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2396, 0, 1484, 760, 713, 0, 1529, 723, 979, 2644, 1, 112, 1157, 949, 613, 2022, 85, 1655, 896, 0, 0, 1710, 2927, 1799, 0, 749, 1210, 1382, 1118, 2845, 307, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2830, 0, 1333, 893, 817, 0, 1269, 967, 977, 2119, 1, 155, 903, 923, 610, 2174, 93, 1782, 899, 1, 4, 2095, 3098, 1888, 2, 861, 1483, 838, 874, 2367, 512, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2534, 0, 1584, 1252, 697, 0, 1533, 803, 1071, 2531, 0, 118, 1280, 846, 667, 2285, 82, 1622, 666, 1, 0, 1857, 2331, 1992, 0, 1077, 1098, 1095, 1071, 2170, 505, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2160, 0, 1929, 1311, 1360, 7, 572, 981, 1125, 1106, 0, 210, 995, 902, 447, 2214, 192, 1334, 1874, 0, 0, 1905, 3052, 2863, 1, 496, 1625, 1854, 209, 1669, 375, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2986, 0, 984, 563, 726, 0, 1037, 1257, 810, 2497, 1, 59, 639, 1522, 809, 2498, 90, 1526, 1310, 0, 1, 2327, 2449, 1391, 0, 1008, 870, 670, 1474, 2234, 1029, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1532, 0, 1757, 769, 0, 0, 0, 0, 762, 3, 1510, 809, 776, 0, 0, 0, 1780, 2307, 0, 72, 330, 215, 0, 0, 939, 1115, 547, 1711, 0, 0, 0, 0, 50, 1938, 947, 0, 993, 0, 0, 0, 0, 2532, 183, 1957, 0, 0, 0, 0, 0, 663, 1250, 685, 967, 70, 0, 0, 572, 2009, 641, 0, 377, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 3860, 4704, 4312, 3928, 4672, 3635, 3599] avg=4096.0000 max=4704 max_rank=2 min=3599 min_rank=7 imbalance_ratio=1.148438 max_load_reduction=28064 max_load_reduction_pct=0.856445 moved_to_echo=15834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1990, 1, 1096, 1021, 0, 0, 0, 0, 932, 0, 1749, 611, 1087, 0, 0, 0, 1003, 2186, 0, 156, 430, 187, 0, 0, 1036, 1107, 775, 1494, 0, 0, 0, 0, 97, 1899, 597, 0, 1391, 0, 0, 0, 0, 1615, 254, 1841, 0, 0, 0, 0, 11, 1174, 1281, 970, 844, 98, 0, 0, 754, 1990, 599, 1, 491, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4379, 3962, 4412, 3984, 3710, 4378, 3835] avg=4096.0000 max=4412 max_rank=3 min=3710 min_rank=5 imbalance_ratio=1.077148 max_load_reduction=28356 max_load_reduction_pct=0.865356 moved_to_echo=15907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1877, 0, 1226, 587, 0, 0, 0, 0, 908, 2, 1158, 1144, 1001, 0, 0, 0, 1522, 2563, 0, 198, 405, 157, 0, 0, 1204, 1014, 695, 1251, 0, 0, 0, 0, 115, 1651, 887, 0, 1282, 0, 0, 0, 2, 1609, 235, 1473, 0, 0, 0, 0, 0, 883, 1606, 925, 706, 90, 0, 0, 938, 2425, 565, 2, 462, 0, 0, 0] rerouted_tokens_per_ep_rank=[3690, 4213, 4845, 4164, 3935, 3319, 4210, 4392] avg=4096.0000 max=4845 max_rank=2 min=3319 min_rank=5 imbalance_ratio=1.182861 max_load_reduction=27923 max_load_reduction_pct=0.852142 moved_to_echo=15856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1639, 0, 1484, 760, 0, 0, 0, 0, 713, 0, 1529, 723, 1124, 0, 0, 0, 979, 2644, 1, 112, 353, 150, 0, 0, 1157, 949, 613, 1197, 0, 0, 0, 0, 85, 1655, 896, 0, 1438, 0, 0, 0, 0, 1710, 264, 1799, 0, 0, 0, 0, 0, 749, 1210, 1382, 675, 101, 0, 0, 1118, 2845, 307, 3, 404, 0, 0, 0] rerouted_tokens_per_ep_rank=[3883, 4089, 4239, 3916, 4074, 3773, 4117, 4677] avg=4096.0000 max=4677 max_rank=7 min=3773 min_rank=5 imbalance_ratio=1.141846 max_load_reduction=28091 max_load_reduction_pct=0.857269 moved_to_echo=16641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1935, 0, 1333, 893, 0, 0, 0, 0, 817, 0, 1269, 967, 1190, 0, 0, 0, 977, 2119, 1, 155, 418, 161, 0, 0, 903, 923, 610, 1287, 0, 0, 0, 0, 93, 1782, 899, 1, 1522, 0, 0, 0, 4, 2095, 279, 1888, 0, 0, 0, 0, 2, 861, 1483, 838, 726, 107, 0, 0, 874, 2367, 512, 0, 477, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4243, 3831, 3723, 4297, 4266, 4017, 4230] avg=4096.0000 max=4297 max_rank=4 min=3723 min_rank=3 imbalance_ratio=1.049072 max_load_reduction=28471 max_load_reduction_pct=0.868866 moved_to_echo=16810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1470, 0, 1929, 1311, 0, 0, 0, 0, 1360, 7, 572, 981, 1176, 0, 0, 0, 1125, 1106, 0, 210, 322, 168, 0, 0, 995, 902, 447, 1302, 0, 0, 0, 0, 192, 1334, 1874, 0, 1505, 0, 0, 0, 0, 1905, 261, 2863, 0, 0, 0, 0, 1, 496, 1625, 1854, 744, 110, 0, 0, 209, 1669, 375, 0, 368, 0, 0, 0] rerouted_tokens_per_ep_rank=[4710, 4096, 2931, 3646, 4905, 5029, 4830, 2621] avg=4096.0000 max=5029 max_rank=5 min=2621 min_rank=7 imbalance_ratio=1.227783 max_load_reduction=27739 max_load_reduction_pct=0.846527 moved_to_echo=17385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1733, 0, 1584, 1252, 0, 0, 0, 0, 697, 0, 1533, 803, 895, 0, 0, 0, 1071, 2531, 0, 118, 374, 169, 0, 0, 1280, 846, 667, 1353, 0, 0, 0, 0, 82, 1622, 666, 1, 1145, 0, 0, 0, 0, 1857, 210, 1992, 0, 0, 0, 0, 0, 1077, 1098, 1095, 763, 81, 0, 0, 1071, 2170, 505, 0, 427, 0, 0, 0] rerouted_tokens_per_ep_rank=[4569, 3928, 4263, 4146, 3516, 4059, 4114, 4173] avg=4096.0000 max=4569 max_rank=0 min=3516 min_rank=4 imbalance_ratio=1.115479 max_load_reduction=28199 max_load_reduction_pct=0.860565 moved_to_echo=15862 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2042, 0, 984, 563, 0, 0, 0, 0, 726, 0, 1037, 1257, 940, 0, 0, 0, 810, 2497, 1, 59, 441, 185, 0, 0, 639, 1522, 809, 1479, 0, 0, 0, 0, 90, 1526, 1310, 0, 1203, 0, 0, 0, 1, 2327, 221, 1391, 0, 0, 0, 0, 0, 1008, 870, 670, 834, 85, 0, 0, 1474, 2234, 1029, 1, 503, 0, 0, 0] rerouted_tokens_per_ep_rank=[3589, 3960, 3993, 4449, 4129, 3940, 3467, 5241] avg=4096.0000 max=5241 max_rank=7 min=3467 min_rank=6 imbalance_ratio=1.279541 max_load_reduction=27527 max_load_reduction_pct=0.840057 moved_to_echo=16777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 6912, 10496, 8448, 6272, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[7040, 128, 10368, 7296, 8192, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[14336, 128, 11520, 7168, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[896, 13440, 8192, 128, 10496, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[9344, 18048, 128, 1152, 3200, 1408, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[8192, 8448, 5248, 11136, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 15744, 1920, 15232, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[7040, 17792, 4608, 128, 3584, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[3131, 0, 258, 0, 4075, 0, 1, 0, 0, 4061, 4054, 4071, 4083, 3, 6, 73, 4095, 0, 0, 0, 670, 0, 0, 0, 4094, 27, 1, 0, 1, 63, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1929, 0, 494, 0, 4004, 2, 1, 1, 0, 4057, 4025, 4093, 4083, 14, 5, 11, 4096, 0, 0, 0, 1787, 0, 1, 0, 4076, 19, 0, 1, 0, 69, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1473, 0, 437, 1, 3955, 0, 0, 1, 1, 4095, 4030, 4096, 4086, 5, 1, 18, 4094, 0, 0, 0, 2294, 0, 0, 0, 4090, 24, 0, 0, 0, 66, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1690, 1, 777, 1, 3692, 0, 0, 0, 0, 4095, 4033, 4094, 4064, 8, 2, 15, 4095, 0, 1, 1, 1907, 1, 0, 0, 4078, 107, 0, 0, 1, 104, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[953, 1, 1198, 0, 3325, 1, 1, 1, 0, 4096, 4008, 4094, 4054, 25, 1, 63, 4095, 0, 1, 0, 2629, 0, 0, 0, 4094, 96, 0, 0, 1, 31, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[405, 0, 1136, 0, 3871, 0, 0, 1, 0, 4095, 2791, 4094, 4094, 1, 4, 543, 4096, 0, 0, 0, 3473, 0, 0, 0, 4094, 42, 0, 0, 1, 25, 1, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2584, 1, 271, 7, 4083, 0, 1, 0, 2, 4051, 4066, 4095, 4090, 5, 4, 8, 4095, 0, 0, 1, 1298, 1, 0, 0, 4091, 0, 0, 1, 0, 12, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1518, 1, 764, 1, 3811, 0, 0, 0, 0, 4095, 3972, 4094, 4095, 7, 2, 33, 4095, 1, 0, 1, 2066, 0, 1, 0, 4089, 100, 0, 0, 0, 22, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3131, 0, 258, 0, 1708, 0, 0, 0, 4075, 0, 1, 0, 105, 92, 44, 0, 0, 224, 4054, 0, 0, 0, 0, 0, 3991, 3, 6, 73, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 670, 0, 0, 0, 2024, 46, 0, 0, 4050, 27, 1, 0, 0, 0, 0, 0, 1, 63, 0, 1, 4025, 0, 0, 0] rerouted_tokens_per_ep_rank=[5097, 4317, 4278, 4073, 4095, 2740, 4078, 4090] avg=4096.0000 max=5097 max_rank=0 min=2740 min_rank=5 imbalance_ratio=1.244385 max_load_reduction=27671 max_load_reduction_pct=0.844452 moved_to_echo=15003 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[953, 1, 1198, 0, 1723, 0, 0, 0, 3325, 1, 1, 1, 106, 91, 44, 0, 0, 225, 4008, 0, 0, 0, 0, 0, 3963, 25, 1, 63, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 2629, 0, 0, 0, 2042, 45, 0, 0, 4050, 96, 0, 0, 0, 0, 0, 0, 1, 31, 0, 0, 4049, 0, 0, 0] rerouted_tokens_per_ep_rank=[3875, 3569, 4233, 4052, 4096, 4716, 4146, 4081] avg=4096.0000 max=4716 max_rank=5 min=3569 min_rank=1 imbalance_ratio=1.151367 max_load_reduction=28052 max_load_reduction_pct=0.856079 moved_to_echo=17039 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1473, 0, 437, 1, 1723, 0, 0, 0, 3955, 0, 0, 1, 106, 92, 44, 0, 1, 225, 4030, 0, 0, 0, 0, 0, 3994, 5, 1, 18, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 2294, 0, 0, 0, 2041, 46, 0, 0, 4046, 24, 0, 0, 0, 0, 0, 0, 0, 66, 1, 0, 4050, 0, 0, 0] rerouted_tokens_per_ep_rank=[3634, 4198, 4256, 4018, 4094, 4381, 4070, 4117] avg=4096.0000 max=4381 max_rank=5 min=3634 min_rank=0 imbalance_ratio=1.069580 max_load_reduction=28387 max_load_reduction_pct=0.866302 moved_to_echo=16662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1929, 0, 494, 0, 1707, 0, 0, 0, 4004, 2, 1, 1, 105, 92, 44, 0, 0, 223, 4025, 0, 0, 0, 0, 0, 3991, 14, 5, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1787, 0, 1, 0, 2022, 45, 0, 0, 4032, 19, 0, 1, 0, 0, 0, 0, 0, 69, 0, 0, 4048, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4249, 4248, 4021, 4096, 3855, 4052, 4117] avg=4096.0000 max=4249 max_rank=1 min=3855 min_rank=5 imbalance_ratio=1.037354 max_load_reduction=28519 max_load_reduction_pct=0.870331 moved_to_echo=16120 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1690, 1, 777, 1, 1723, 0, 0, 0, 3692, 0, 0, 0, 106, 91, 44, 0, 0, 225, 4033, 0, 0, 0, 0, 0, 3973, 8, 2, 15, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 1907, 1, 0, 0, 2041, 45, 0, 0, 4034, 107, 0, 0, 0, 0, 0, 0, 1, 104, 1, 0, 4049, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 3933, 4258, 3998, 4097, 3994, 4141, 4155] avg=4096.0000 max=4258 max_rank=2 min=3933 min_rank=1 imbalance_ratio=1.039551 max_load_reduction=28510 max_load_reduction_pct=0.870056 moved_to_echo=16387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1518, 1, 764, 1, 1723, 0, 0, 0, 3811, 0, 0, 0, 106, 92, 44, 0, 0, 225, 3972, 0, 0, 0, 0, 0, 4003, 7, 2, 33, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 0, 2066, 0, 1, 0, 2041, 46, 0, 0, 4045, 100, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 4048, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4053, 4197, 4045, 4097, 4154, 4145, 4070] avg=4096.0000 max=4197 max_rank=2 min=4007 min_rank=0 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16466 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[405, 0, 1136, 0, 1725, 0, 0, 0, 3871, 0, 0, 1, 109, 96, 47, 0, 0, 216, 2791, 0, 0, 0, 0, 0, 3998, 1, 4, 543, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 3473, 0, 0, 0, 2045, 46, 0, 0, 4046, 42, 0, 0, 0, 0, 0, 0, 1, 25, 1, 1, 4048, 0, 0, 0] rerouted_tokens_per_ep_rank=[3266, 4124, 3007, 4546, 4097, 5564, 4088, 4076] avg=4096.0000 max=5564 max_rank=5 min=3007 min_rank=2 imbalance_ratio=1.358398 max_load_reduction=27204 max_load_reduction_pct=0.830200 moved_to_echo=17825 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2584, 1, 271, 7, 1704, 0, 0, 0, 4083, 0, 1, 0, 105, 92, 44, 0, 2, 223, 4066, 0, 0, 0, 0, 0, 3998, 5, 4, 8, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 1298, 1, 0, 0, 2019, 46, 0, 0, 4047, 0, 0, 1, 0, 0, 0, 0, 0, 12, 0, 1, 4049, 0, 0, 0] rerouted_tokens_per_ep_rank=[4567, 4325, 4291, 4015, 4096, 3364, 4048, 4062] avg=4096.0000 max=4567 max_rank=0 min=3364 min_rank=5 imbalance_ratio=1.114990 max_load_reduction=28201 max_load_reduction_pct=0.860626 moved_to_echo=15570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[13696, 128, 5376, 128, 13824, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[16128, 128, 128, 0, 16384, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[30848, 128, 128, 128, 896, 768, 384, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 1792, 31104, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32000, 128, 128, 768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32384, 512, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 512, 128, 128, 32384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4, 1, 44, 0, 4091, 36, 4093, 0, 148, 0, 3888, 0, 4093, 0, 4095, 145, 1, 1, 0, 0, 0, 15, 4096, 697, 0, 916, 26, 3453, 1, 2924, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 20, 0, 4089, 3, 4095, 10, 65, 2, 3755, 0, 4093, 0, 4093, 10, 0, 0, 1, 0, 0, 45, 4095, 145, 0, 1655, 203, 3782, 7, 2598, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 17, 1, 4023, 11, 4095, 4, 4, 0, 3683, 0, 4079, 1, 4034, 62, 0, 1, 0, 1, 1, 1, 4095, 17, 0, 623, 804, 4009, 1, 3176, 2, 21] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[2, 3, 5, 0, 4087, 0, 4090, 1, 58, 2, 3910, 0, 4093, 0, 4058, 76, 0, 0, 0, 0, 0, 5, 4096, 65, 0, 896, 251, 4021, 0, 3049, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 9, 1, 4093, 1, 4095, 2, 129, 0, 3530, 1, 4095, 1, 4094, 9, 1, 1, 0, 1, 1, 0, 4095, 107, 2, 997, 247, 3943, 0, 3312, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[5, 0, 270, 1, 4090, 12, 4090, 7, 30, 0, 4012, 0, 4096, 1, 4087, 129, 1, 1, 0, 1, 0, 5, 4095, 248, 0, 1071, 6, 3609, 0, 2899, 0, 2] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 8, 0, 3899, 2, 4091, 0, 34, 1, 3677, 0, 4095, 0, 4091, 1, 1, 0, 1, 0, 0, 115, 4096, 138, 0, 910, 117, 4020, 1, 3469, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 58, 0, 4079, 124, 4087, 3, 23, 3, 3764, 0, 4095, 0, 4088, 18, 0, 0, 1, 0, 0, 7, 4095, 102, 0, 399, 337, 4080, 2, 3384, 0, 17] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 1, 44, 0, 4039, 0, 0, 0, 4091, 36, 15, 0, 0, 0, 0, 0, 148, 0, 3888, 0, 213, 40, 1, 0, 0, 0, 4055, 145, 0, 0, 0, 0, 1, 1, 0, 0, 4093, 1, 0, 0, 0, 15, 3883, 697, 0, 0, 0, 0, 0, 916, 26, 2604, 0, 0, 0, 0, 1, 2924, 0, 0, 848, 38, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4142, 4290, 4200, 4096, 4595, 3546, 3811] avg=4096.0000 max=4595 max_rank=5 min=3546 min_rank=6 imbalance_ratio=1.121826 max_load_reduction=28173 max_load_reduction_pct=0.859772 moved_to_echo=16048 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 17, 1, 4041, 0, 0, 0, 4023, 11, 15, 4, 0, 0, 0, 0, 4, 0, 3683, 0, 213, 40, 1, 0, 0, 1, 3994, 62, 0, 0, 0, 0, 0, 1, 0, 1, 4079, 1, 0, 0, 1, 1, 3882, 17, 0, 0, 0, 0, 0, 623, 804, 3024, 0, 0, 0, 0, 1, 3176, 2, 21, 984, 38, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4053, 3941, 4057, 4082, 3901, 4451, 4222] avg=4096.0000 max=4451 max_rank=6 min=3901 min_rank=5 imbalance_ratio=1.086670 max_load_reduction=28317 max_load_reduction_pct=0.864166 moved_to_echo=16656 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 5, 0, 4036, 0, 0, 0, 4087, 0, 15, 1, 0, 0, 0, 0, 58, 2, 3910, 0, 213, 40, 1, 0, 0, 0, 4018, 76, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 5, 3883, 65, 0, 0, 0, 0, 0, 896, 251, 3033, 0, 0, 0, 0, 0, 3049, 0, 0, 987, 38, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4103, 4224, 4094, 4094, 3953, 4180, 4074] avg=4096.0000 max=4224 max_rank=2 min=3953 min_rank=5 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16301 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 20, 0, 4041, 0, 0, 0, 4089, 3, 15, 10, 0, 0, 0, 0, 65, 2, 3755, 0, 213, 40, 1, 0, 0, 0, 4053, 10, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 1, 0, 0, 0, 45, 3882, 145, 0, 0, 0, 0, 0, 1655, 203, 2853, 0, 0, 0, 0, 7, 2598, 0, 0, 928, 38, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4117, 4076, 4063, 4095, 4072, 4711, 3571] avg=4096.0000 max=4711 max_rank=6 min=3571 min_rank=7 imbalance_ratio=1.150146 max_load_reduction=28057 max_load_reduction_pct=0.856232 moved_to_echo=16449 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 9, 1, 4041, 0, 0, 0, 4093, 1, 15, 2, 0, 0, 0, 0, 129, 0, 3530, 1, 213, 40, 1, 0, 0, 1, 4054, 9, 0, 0, 0, 0, 1, 1, 0, 1, 4095, 1, 0, 0, 1, 0, 3882, 107, 0, 0, 0, 0, 2, 997, 247, 2974, 0, 0, 0, 0, 0, 3312, 0, 0, 968, 38, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4111, 3914, 4064, 4099, 3990, 4220, 4318] avg=4096.0000 max=4318 max_rank=7 min=3914 min_rank=2 imbalance_ratio=1.054199 max_load_reduction=28450 max_load_reduction_pct=0.868225 moved_to_echo=16627 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 8, 0, 4044, 0, 0, 0, 3899, 2, 0, 0, 0, 0, 0, 0, 34, 1, 3677, 0, 218, 45, 8, 0, 0, 0, 4046, 1, 0, 0, 0, 0, 1, 0, 1, 0, 4095, 8, 0, 0, 0, 115, 3878, 138, 0, 0, 0, 0, 0, 910, 117, 3021, 0, 0, 0, 0, 1, 3469, 0, 0, 991, 39, 0, 0] rerouted_tokens_per_ep_rank=[4053, 3901, 3983, 4047, 4105, 4131, 4048, 4500] avg=4096.0000 max=4500 max_rank=7 min=3901 min_rank=1 imbalance_ratio=1.098633 max_load_reduction=28268 max_load_reduction_pct=0.862671 moved_to_echo=16784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 58, 0, 4033, 0, 0, 0, 4079, 124, 15, 3, 0, 0, 0, 0, 23, 3, 3764, 0, 213, 40, 1, 0, 0, 0, 4048, 18, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 0, 0, 7, 3882, 102, 0, 0, 0, 0, 0, 399, 337, 3077, 0, 0, 0, 0, 2, 3384, 0, 17, 1002, 38, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4221, 4044, 4066, 4097, 3991, 3813, 4443] avg=4096.0000 max=4443 max_rank=7 min=3813 min_rank=6 imbalance_ratio=1.084717 max_load_reduction=28325 max_load_reduction_pct=0.864410 moved_to_echo=16344 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 270, 1, 4036, 0, 0, 0, 4090, 12, 11, 7, 0, 0, 0, 0, 30, 0, 4012, 0, 213, 40, 1, 0, 0, 1, 4047, 129, 0, 0, 0, 0, 1, 1, 0, 1, 4096, 1, 0, 0, 0, 5, 3882, 248, 0, 0, 0, 0, 0, 1071, 6, 2722, 0, 0, 0, 0, 0, 2899, 0, 2, 886, 42, 0, 0] rerouted_tokens_per_ep_rank=[4312, 4120, 4296, 4177, 4100, 4135, 3799, 3829] avg=4096.0000 max=4312 max_rank=0 min=3799 min_rank=6 imbalance_ratio=1.052734 max_load_reduction=28456 max_load_reduction_pct=0.868408 moved_to_echo=15863 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 24832, 128, 128, 7680, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 7552, 2048, 23424, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 512, 128, 32384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32512, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 256, 31104, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[512, 128, 30336, 128, 1792, 384, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32384, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 87, 0, 4084, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 3497, 0, 4091, 2, 0, 0, 111, 10, 1315, 3520, 3994, 4086, 0, 7, 3866] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 79, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3536, 0, 4074, 2, 0, 0, 128, 15, 1058, 3657, 4063, 4080, 0, 10, 3879] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[9, 0, 0, 1, 1, 90, 0, 4086, 0, 0, 1, 0, 1, 0, 1, 0, 4094, 3453, 1, 4094, 0, 1, 0, 21, 6, 1095, 3782, 3999, 4068, 0, 0, 3964] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 250, 1, 4095, 0, 33, 1, 1, 1, 0, 0, 0, 4095, 2756, 1, 4090, 0, 0, 0, 209, 0, 1380, 3852, 3929, 4089, 1, 5, 3977] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 235, 0, 4092, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 3762, 0, 4094, 3, 0, 0, 71, 0, 593, 3650, 4063, 4086, 0, 0, 4020] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 91, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 3471, 0, 4085, 1, 0, 1, 42, 5, 883, 3864, 4045, 4096, 0, 1, 3993] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[5, 0, 0, 1, 0, 66, 0, 4085, 0, 5, 1, 1, 1, 0, 1, 0, 4057, 3515, 1, 4080, 0, 1, 0, 117, 15, 1484, 3503, 4020, 4067, 0, 0, 3742] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 133, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 1, 4095, 3345, 0, 4094, 1, 0, 0, 219, 5, 817, 3855, 3940, 4085, 0, 5, 4076] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 3924, 92, 0, 0, 0, 250, 1, 3972, 0, 0, 0, 0, 0, 33, 1, 1, 3843, 252, 0, 0, 1, 0, 0, 0, 4095, 5, 0, 0, 0, 2756, 1, 680, 0, 0, 0, 0, 0, 0, 0, 209, 3158, 726, 123, 0, 0, 1380, 3126, 0, 0, 0, 0, 0, 154, 1, 5, 3977, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4223, 4130, 4101, 3437, 4216, 4506, 4137] avg=4096.0000 max=4506 max_rank=6 min=3437 min_rank=4 imbalance_ratio=1.100098 max_load_reduction=28262 max_load_reduction_pct=0.862488 moved_to_echo=16296 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4057, 92, 0, 0, 0, 235, 0, 3969, 0, 0, 0, 0, 0, 3, 0, 0, 3840, 252, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 3762, 0, 681, 0, 0, 0, 0, 3, 0, 0, 71, 3161, 688, 123, 0, 0, 593, 2962, 0, 0, 0, 0, 0, 154, 0, 0, 4020, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4204, 4095, 4102, 4443, 4046, 3555, 4174] avg=4096.0000 max=4443 max_rank=4 min=3555 min_rank=6 imbalance_ratio=1.084717 max_load_reduction=28325 max_load_reduction_pct=0.864410 moved_to_echo=16218 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3990, 92, 0, 0, 0, 87, 0, 3961, 0, 0, 0, 0, 0, 3, 0, 0, 3840, 252, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 3497, 0, 680, 0, 0, 0, 0, 2, 0, 0, 111, 3159, 663, 123, 0, 10, 1315, 2857, 0, 0, 0, 0, 0, 154, 0, 7, 3866, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4048, 4095, 4099, 4177, 4058, 4182, 4027] avg=4096.0000 max=4182 max_rank=6 min=4027 min_rank=7 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16444 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4057, 92, 0, 0, 0, 79, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 3835, 251, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 3536, 0, 677, 0, 0, 0, 0, 2, 0, 0, 128, 3146, 689, 123, 0, 15, 1058, 2968, 0, 0, 0, 0, 0, 153, 0, 10, 3879, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4047, 4086, 4102, 4213, 4088, 4041, 4042] avg=4096.0000 max=4213 max_rank=4 min=4041 min_rank=6 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16384 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 1, 3994, 91, 0, 0, 1, 90, 0, 3963, 0, 0, 0, 0, 0, 0, 1, 0, 3823, 252, 0, 0, 1, 0, 1, 0, 4094, 5, 0, 0, 0, 3453, 1, 681, 0, 0, 0, 0, 0, 1, 0, 21, 3161, 713, 123, 0, 6, 1095, 3069, 0, 0, 0, 0, 0, 154, 0, 0, 3964, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4054, 4076, 4101, 4135, 4019, 4170, 4118] avg=4096.0000 max=4170 max_rank=6 min=4019 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16442 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4040, 95, 0, 0, 0, 91, 0, 3968, 0, 0, 0, 0, 0, 1, 0, 0, 3855, 256, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 0, 3471, 0, 671, 0, 0, 0, 0, 1, 0, 1, 42, 3158, 731, 127, 0, 5, 883, 3133, 0, 0, 0, 0, 0, 146, 0, 1, 3993, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4059, 4112, 4099, 4142, 4060, 4021, 4140] avg=4096.0000 max=4142 max_rank=4 min=4021 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16363 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3936, 92, 0, 0, 0, 133, 0, 3972, 0, 0, 0, 0, 0, 2, 0, 0, 3839, 252, 0, 0, 0, 0, 0, 1, 4095, 4, 0, 0, 0, 3345, 0, 681, 0, 0, 0, 0, 1, 0, 0, 219, 3161, 726, 123, 0, 5, 817, 3129, 0, 0, 0, 0, 0, 154, 0, 5, 4076, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4105, 4093, 4100, 4026, 4230, 3951, 4235] avg=4096.0000 max=4235 max_rank=7 min=3951 min_rank=6 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16442 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 1, 4015, 91, 0, 0, 0, 66, 0, 3962, 0, 0, 0, 0, 0, 5, 1, 1, 3823, 251, 0, 0, 1, 0, 1, 0, 4057, 5, 0, 0, 0, 3515, 1, 679, 0, 0, 0, 0, 0, 1, 0, 117, 3150, 660, 123, 0, 15, 1484, 2843, 0, 0, 0, 0, 0, 153, 0, 0, 3742, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4028, 4081, 4064, 4195, 4051, 4342, 3895] avg=4096.0000 max=4342 max_rank=6 min=3895 min_rank=7 imbalance_ratio=1.060059 max_load_reduction=28426 max_load_reduction_pct=0.867493 moved_to_echo=16483 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 8704, 24192, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 1024, 25344, 5632, 1024, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 27392, 128, 5504, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 30720, 2048, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[1280, 128, 128, 31616, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 32128, 768, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 1152, 128, 31744, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4093, 66, 4095, 0, 0, 1181, 1, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 96, 2517, 1, 4030, 0, 305, 0, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 4091, 182, 4095, 0, 1, 1840, 0, 4095, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 2, 0, 110, 1066, 1, 3949, 0, 1045, 1, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4096, 100, 4096, 0, 0, 2385, 0, 4094, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 28, 1422, 3, 3949, 0, 305, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4089, 49, 4096, 0, 0, 2522, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 118, 863, 6, 3908, 0, 737, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 4091, 18, 4095, 0, 1, 3455, 0, 4095, 1, 0, 1, 4095, 1, 0, 0, 0, 1, 5, 0, 13, 618, 0, 3839, 0, 245, 1, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 4092, 42, 4095, 0, 1, 1447, 0, 4095, 1, 0, 0, 4095, 4, 0, 0, 0, 1, 11, 0, 37, 1605, 0, 3968, 0, 1080, 1, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4096, 126, 4096, 0, 0, 1343, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 570, 744, 2, 3953, 0, 1457, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 12, 4096, 0, 0, 2640, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 1217, 3, 3933, 0, 385, 0, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4022, 66, 1, 0, 0, 0, 0, 0, 0, 1181, 1, 1995, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 96, 2100, 1128, 694, 71, 2517, 1, 2902, 0, 0, 0, 0, 0, 305, 0, 0, 3401, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4089, 3177, 4096, 4094, 4090, 5420, 3706] avg=4096.0000 max=5420 max_rank=6 min=3177 min_rank=2 imbalance_ratio=1.323242 max_load_reduction=27348 max_load_reduction_pct=0.834595 moved_to_echo=17310 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 0, 0, 0, 0, 4020, 182, 1, 0, 0, 0, 0, 0, 1, 1840, 0, 1995, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 2, 0, 110, 2100, 1105, 694, 71, 1066, 1, 2844, 0, 0, 0, 0, 0, 1045, 1, 0, 3401, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4203, 3836, 4096, 4094, 4083, 3911, 4447] avg=4096.0000 max=4447 max_rank=7 min=3836 min_rank=2 imbalance_ratio=1.085693 max_load_reduction=28321 max_load_reduction_pct=0.864288 moved_to_echo=16535 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4025, 100, 0, 0, 0, 0, 0, 0, 0, 2385, 0, 1994, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 28, 2100, 1105, 694, 71, 1422, 3, 2844, 0, 0, 0, 0, 0, 305, 0, 0, 3402, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4125, 4379, 4096, 4099, 3998, 4269, 3707] avg=4096.0000 max=4379 max_rank=2 min=3707 min_rank=7 imbalance_ratio=1.069092 max_load_reduction=28389 max_load_reduction_pct=0.866364 moved_to_echo=16073 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 0, 0, 0, 0, 4020, 18, 1, 0, 0, 0, 0, 0, 1, 3455, 0, 1995, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 1, 5, 0, 13, 2100, 1075, 694, 71, 618, 0, 2764, 0, 0, 0, 0, 0, 245, 1, 0, 3401, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4039, 5451, 4097, 4095, 3959, 3382, 3647] avg=4096.0000 max=5451 max_rank=2 min=3382 min_rank=6 imbalance_ratio=1.330811 max_load_reduction=27317 max_load_reduction_pct=0.833649 moved_to_echo=15083 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4022, 126, 0, 0, 0, 0, 0, 0, 0, 1343, 0, 1990, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 570, 2104, 1110, 700, 74, 744, 2, 2843, 0, 0, 0, 0, 0, 1457, 0, 0, 3396, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4148, 3333, 4096, 4096, 4558, 3589, 4853] avg=4096.0000 max=4853 max_rank=7 min=3333 min_rank=2 imbalance_ratio=1.184814 max_load_reduction=27915 max_load_reduction_pct=0.851898 moved_to_echo=17096 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4025, 12, 0, 0, 0, 0, 0, 0, 0, 2640, 0, 1994, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 2099, 1101, 694, 71, 1217, 3, 2832, 0, 0, 0, 0, 0, 385, 0, 0, 3402, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4037, 4634, 4096, 4096, 3970, 4052, 3787] avg=4096.0000 max=4634 max_rank=2 min=3787 min_rank=7 imbalance_ratio=1.131348 max_load_reduction=28134 max_load_reduction_pct=0.858582 moved_to_echo=15905 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4095, 0, 0, 0, 0, 4021, 42, 0, 0, 0, 0, 0, 0, 1, 1447, 0, 1995, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 4095, 0, 0, 0, 1, 11, 0, 37, 2100, 1111, 694, 71, 1605, 0, 2857, 0, 0, 0, 0, 0, 1080, 1, 0, 3401, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4063, 3443, 4096, 4099, 4025, 4462, 4482] avg=4096.0000 max=4482 max_rank=7 min=3443 min_rank=2 imbalance_ratio=1.094238 max_load_reduction=28286 max_load_reduction_pct=0.863220 moved_to_echo=17068 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4018, 49, 1, 0, 0, 0, 0, 0, 0, 2522, 0, 1993, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 118, 2099, 1094, 694, 71, 863, 6, 2814, 0, 0, 0, 0, 0, 737, 0, 0, 3402, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4068, 4515, 4096, 4095, 4076, 3683, 4139] avg=4096.0000 max=4515 max_rank=2 min=3683 min_rank=6 imbalance_ratio=1.102295 max_load_reduction=28253 max_load_reduction_pct=0.862213 moved_to_echo=15993 active_clone_links=5 active_echo_slots_local=5 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32772, 32772, 32768, 32769, 32768, 32759, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32772, 32772, 32768, 32769, 32768, 32759, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 32772, 32768, 32769, 32768, 32759, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32772, 32772, 32768, 32769, 32768, 32759, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32772, 32772, 32768, 32769, 32768, 32759, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 32772, 32768, 32769, 32768, 32759, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32772, 32772, 32768, 32769, 32768, 32759, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32772, 32772, 32768, 32769, 32768, 32759, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32759 actual_valid_recv=33408 observed_loss=-649 tokens_per_expert=[128, 128, 128, 1024, 16896, 8832, 5632, 640] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[10112, 128, 22784, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[5632, 128, 0, 27264, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 16896, 128, 16000, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32772 actual_valid_recv=33152 observed_loss=-380 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32772 actual_valid_recv=33024 observed_loss=-252 tokens_per_expert=[32256, 640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32769 actual_valid_recv=33024 observed_loss=-255 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 18, 0, 0, 643, 4075, 0, 0, 0, 3398, 0, 0, 0, 4096, 3546, 1, 0, 1, 4091, 0, 4096, 3483, 0, 0, 0, 0, 1659, 3660, 1, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 35, 0, 2, 875, 4092, 3, 0, 0, 3804, 0, 0, 2, 4096, 2795, 20, 0, 0, 4062, 0, 4094, 3892, 0, 0, 0, 0, 1333, 3663, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 28, 0, 2, 745, 4093, 0, 0, 0, 3847, 0, 0, 0, 4096, 3502, 5, 0, 2, 4084, 0, 4096, 3906, 0, 0, 0, 0, 518, 3844, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 104, 1, 2, 110, 4082, 0, 1, 1, 3919, 0, 1, 0, 4095, 3907, 38, 0, 6, 4082, 1, 4093, 3743, 0, 3, 1, 0, 522, 4055, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 1, 0, 384, 4095, 0, 1, 1, 3826, 1, 1, 0, 4089, 3458, 2, 1, 1, 4089, 1, 4095, 3768, 0, 0, 0, 0, 1418, 3529, 0, 1, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 32, 0, 1, 67, 4094, 1, 0, 0, 4084, 0, 0, 0, 4096, 3931, 106, 0, 1, 4074, 0, 4096, 3886, 0, 0, 0, 0, 297, 4002, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 39, 0, 1, 221, 4090, 0, 0, 0, 3601, 0, 0, 3, 4096, 3377, 26, 0, 30, 4087, 0, 4096, 3967, 0, 0, 0, 0, 1162, 3972, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 1636, 4093, 0, 1, 1, 3557, 1, 0, 0, 4092, 3054, 6, 1, 6, 4055, 1, 4086, 3616, 1, 2, 0, 1, 912, 3629, 0, 1, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 0, 0, 3158, 652, 226, 0, 643, 3499, 0, 0, 0, 0, 0, 0, 0, 3398, 0, 0, 339, 0, 0, 0, 0, 624, 3546, 1, 0, 0, 0, 0, 0, 1, 4091, 0, 11, 0, 0, 0, 313, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 1659, 3008, 0, 0, 0, 0, 1, 0, 0, 0, 3783, 314, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4142, 3737, 4171, 4103, 3796, 4667, 4098] avg=4096.0000 max=4667 max_rank=6 min=3737 min_rank=2 imbalance_ratio=1.139404 max_load_reduction=28101 max_load_reduction_pct=0.857574 moved_to_echo=16664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 104, 1, 2, 3157, 722, 226, 0, 110, 3505, 0, 1, 0, 0, 0, 0, 1, 3919, 0, 1, 340, 0, 0, 0, 0, 624, 3907, 38, 0, 0, 0, 0, 0, 6, 4082, 1, 11, 0, 0, 0, 313, 3743, 0, 3, 0, 0, 0, 0, 1, 0, 522, 3333, 0, 0, 0, 0, 0, 1, 0, 0, 3780, 314, 0, 0] rerouted_tokens_per_ep_rank=[4212, 3616, 4261, 4569, 4100, 4059, 3856, 4095] avg=4096.0000 max=4569 max_rank=3 min=3616 min_rank=1 imbalance_ratio=1.115479 max_load_reduction=28199 max_load_reduction_pct=0.860565 moved_to_echo=16110 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 35, 0, 2, 3158, 652, 227, 0, 875, 3514, 3, 0, 0, 0, 0, 0, 0, 3804, 0, 0, 340, 0, 0, 0, 2, 624, 2795, 20, 0, 0, 0, 0, 0, 0, 4062, 0, 11, 0, 0, 0, 313, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 3011, 0, 0, 0, 0, 0, 0, 0, 0, 3781, 314, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4392, 4144, 3441, 4073, 4205, 4344, 4095] avg=4096.0000 max=4392 max_rank=1 min=3441 min_rank=3 imbalance_ratio=1.072266 max_load_reduction=28376 max_load_reduction_pct=0.865967 moved_to_echo=16717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 0, 2, 3158, 684, 227, 0, 745, 3515, 0, 0, 0, 0, 0, 0, 0, 3847, 0, 0, 340, 0, 0, 0, 0, 624, 3502, 5, 0, 0, 0, 0, 0, 2, 4084, 0, 11, 0, 0, 0, 313, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 518, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 3783, 314, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4260, 4187, 4131, 4097, 4219, 3678, 4097] avg=4096.0000 max=4260 max_rank=1 min=3678 min_rank=6 imbalance_ratio=1.040039 max_load_reduction=28508 max_load_reduction_pct=0.869995 moved_to_echo=16091 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 32, 0, 1, 3160, 718, 230, 0, 67, 3504, 1, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 345, 0, 0, 0, 0, 620, 3931, 106, 0, 0, 0, 0, 0, 1, 4074, 0, 15, 0, 0, 0, 310, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 297, 3284, 0, 0, 0, 0, 0, 0, 0, 0, 3786, 316, 0, 0] rerouted_tokens_per_ep_rank=[4141, 3572, 4429, 4657, 4090, 4196, 3581, 4102] avg=4096.0000 max=4657 max_rank=3 min=3572 min_rank=1 imbalance_ratio=1.136963 max_load_reduction=28111 max_load_reduction_pct=0.857880 moved_to_echo=15969 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 39, 0, 1, 3158, 707, 227, 0, 221, 3512, 0, 0, 0, 0, 0, 0, 0, 3601, 0, 0, 340, 0, 0, 0, 3, 624, 3377, 26, 0, 0, 0, 0, 0, 30, 4087, 0, 11, 0, 0, 0, 313, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 1162, 3265, 0, 0, 0, 0, 0, 0, 0, 0, 3783, 314, 0, 0] rerouted_tokens_per_ep_rank=[4132, 3733, 3941, 4030, 4128, 4280, 4427, 4097] avg=4096.0000 max=4427 max_rank=6 min=3733 min_rank=1 imbalance_ratio=1.080811 max_load_reduction=28341 max_load_reduction_pct=0.864899 moved_to_echo=16932 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 1, 0, 3152, 628, 227, 0, 384, 3516, 0, 1, 0, 0, 0, 0, 1, 3826, 1, 1, 341, 0, 0, 0, 0, 624, 3458, 2, 0, 0, 0, 0, 1, 1, 4089, 1, 11, 0, 0, 0, 313, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 1418, 2901, 0, 0, 0, 0, 0, 1, 1, 0, 3782, 313, 0, 0] rerouted_tokens_per_ep_rank=[4013, 3901, 4170, 4084, 4103, 4081, 4319, 4097] avg=4096.0000 max=4319 max_rank=6 min=3901 min_rank=1 imbalance_ratio=1.054443 max_load_reduction=28449 max_load_reduction_pct=0.868195 moved_to_echo=16600 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 3154, 646, 227, 0, 1636, 3515, 0, 1, 0, 0, 0, 0, 1, 3557, 1, 0, 340, 0, 0, 0, 0, 625, 3054, 6, 0, 0, 0, 0, 1, 6, 4055, 1, 11, 0, 0, 0, 313, 3616, 1, 2, 0, 0, 0, 0, 0, 1, 912, 2983, 0, 0, 0, 0, 0, 1, 0, 0, 3773, 313, 0, 0] rerouted_tokens_per_ep_rank=[4043, 5152, 3899, 3685, 4074, 3932, 3896, 4087] avg=4096.0000 max=5152 max_rank=1 min=3685 min_rank=3 imbalance_ratio=1.257812 max_load_reduction=27616 max_load_reduction_pct=0.842773 moved_to_echo=15989 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 384, 128, 128, 25344, 5504, 1920, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 7936, 24960, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 4992, 27648, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4736, 28160, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 30080, 128, 128, 2816, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 30336, 2560, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[2560, 30336, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 3790, 0, 1745, 0, 832, 0, 3288, 0, 0, 2384, 4096, 0, 4096, 1, 0, 0, 4096, 0, 0, 1, 248, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3, 0, 4057, 0, 1123, 0, 368, 0, 3361, 1, 0, 2503, 4083, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 887, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 0, 3961, 0, 1938, 1, 633, 1, 2817, 0, 1, 2398, 4095, 3, 4090, 0, 1, 1, 4095, 0, 3, 0, 538, 0, 0, 4095, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2, 0, 3797, 0, 969, 9, 1375, 0, 2552, 4, 0, 2821, 4091, 3, 4096, 0, 0, 0, 4096, 0, 0, 0, 762, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1, 1, 0, 4095, 0, 1, 3897, 0, 1750, 25, 578, 1, 2946, 0, 1, 2580, 4094, 6, 4091, 0, 1, 1, 4095, 6, 1, 0, 501, 0, 1, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 3953, 0, 1701, 0, 482, 0, 1940, 1, 0, 3799, 4089, 5, 4096, 0, 0, 0, 4096, 3, 0, 1, 411, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 1, 0, 4094, 0, 1, 3579, 0, 1040, 18, 948, 1, 3487, 0, 1, 2289, 4081, 9, 4091, 0, 1, 1, 4095, 2, 1, 0, 937, 0, 0, 4090, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 3, 0, 3932, 0, 1255, 0, 1201, 0, 2120, 1, 0, 3274, 4090, 1, 4096, 1, 0, 0, 4096, 0, 0, 0, 508, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 3790, 0, 261, 0, 0, 0, 1745, 0, 832, 0, 1462, 595, 0, 0, 1565, 0, 0, 2384, 0, 0, 0, 0, 4096, 0, 4, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 248, 0, 3497, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4052, 4634, 3949, 4101, 4095, 3746, 4096] avg=4096.0000 max=4634 max_rank=2 min=3746 min_rank=6 imbalance_ratio=1.131348 max_load_reduction=28134 max_load_reduction_pct=0.858582 moved_to_echo=16038 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3961, 0, 223, 0, 0, 0, 1938, 1, 633, 1, 1252, 594, 0, 0, 1342, 0, 1, 2398, 0, 0, 0, 0, 4095, 3, 4, 0, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 3, 0, 538, 0, 3492, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4184, 4419, 3741, 4102, 4096, 4033, 4097] avg=4096.0000 max=4419 max_rank=2 min=3741 min_rank=3 imbalance_ratio=1.078857 max_load_reduction=28349 max_load_reduction_pct=0.865143 moved_to_echo=16328 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 4057, 0, 267, 0, 0, 0, 1123, 0, 368, 0, 1494, 595, 0, 0, 1600, 1, 0, 2503, 0, 0, 0, 0, 4083, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 887, 0, 3497, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4327, 3580, 4104, 4087, 4095, 4384, 4095] avg=4096.0000 max=4384 max_rank=6 min=3580 min_rank=2 imbalance_ratio=1.070312 max_load_reduction=28384 max_load_reduction_pct=0.866211 moved_to_echo=16661 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 3797, 0, 202, 0, 0, 0, 969, 9, 1375, 0, 1134, 595, 0, 0, 1216, 4, 0, 2821, 0, 0, 0, 0, 4091, 3, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 762, 0, 3497, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4001, 4082, 4041, 4098, 4095, 4259, 4096] avg=4096.0000 max=4259 max_rank=6 min=4001 min_rank=1 imbalance_ratio=1.039795 max_load_reduction=28509 max_load_reduction_pct=0.870026 moved_to_echo=16548 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 4095, 0, 0, 0, 0, 0, 1, 3897, 0, 234, 0, 0, 0, 1750, 25, 578, 1, 1310, 594, 0, 0, 1402, 0, 1, 2580, 0, 0, 0, 0, 4094, 6, 4, 0, 0, 0, 0, 0, 1, 1, 4094, 6, 0, 0, 0, 0, 1, 0, 501, 0, 3493, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4132, 4258, 3983, 4104, 4102, 3995, 4097] avg=4096.0000 max=4258 max_rank=2 min=3983 min_rank=3 imbalance_ratio=1.039551 max_load_reduction=28510 max_load_reduction_pct=0.870056 moved_to_echo=16298 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 2, 0, 0, 0, 1, 0, 3953, 0, 158, 0, 0, 0, 1701, 0, 482, 0, 867, 599, 0, 0, 915, 1, 0, 3799, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 1, 411, 0, 3497, 0, 0, 0, 0, 4095, 0, 0, 5, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4112, 3649, 4715, 4094, 4092, 3909, 4100] avg=4096.0000 max=4715 max_rank=3 min=3649 min_rank=2 imbalance_ratio=1.151123 max_load_reduction=28053 max_load_reduction_pct=0.856110 moved_to_echo=16195 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 3, 0, 3932, 0, 168, 0, 0, 0, 1255, 0, 1201, 0, 942, 595, 0, 0, 1010, 1, 0, 3274, 0, 0, 0, 0, 4090, 1, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 508, 0, 3501, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4103, 3993, 4285, 4092, 4095, 4009, 4096] avg=4096.0000 max=4285 max_rank=3 min=3993 min_rank=2 imbalance_ratio=1.046143 max_load_reduction=28483 max_load_reduction_pct=0.869232 moved_to_echo=16292 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 4094, 0, 0, 0, 0, 0, 1, 3579, 0, 277, 0, 0, 0, 1040, 18, 948, 1, 1550, 596, 0, 0, 1660, 0, 1, 2289, 0, 0, 0, 0, 4081, 9, 0, 0, 0, 0, 0, 0, 1, 1, 4094, 2, 0, 0, 0, 0, 1, 0, 937, 0, 3495, 0, 0, 0, 0, 4090, 0, 0, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 3857, 4153, 3950, 4090, 4098, 4433, 4091] avg=4096.0000 max=4433 max_rank=6 min=3857 min_rank=1 imbalance_ratio=1.082275 max_load_reduction=28335 max_load_reduction_pct=0.864716 moved_to_echo=16712 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10752, 128, 128, 22144, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[11648, 128, 6528, 128, 10112, 4864, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 4864, 0, 28032, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 128, 30976, 0, 1792, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 4095, 158, 1, 4079, 0, 1, 4095, 4095, 1, 3, 0, 0, 3970, 0, 4092, 0, 1, 4095, 13, 0, 0, 1, 271, 3793, 0, 1, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4095, 241, 0, 4091, 0, 0, 4092, 4096, 0, 2, 0, 0, 3883, 1, 4096, 0, 0, 4096, 89, 0, 0, 0, 295, 3690, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4066, 115, 0, 4060, 0, 0, 4087, 4096, 0, 0, 0, 0, 4022, 1, 4096, 0, 0, 4096, 33, 0, 0, 0, 214, 3878, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4081, 448, 0, 4094, 0, 0, 4095, 4096, 0, 2, 0, 0, 3928, 0, 4096, 0, 0, 4096, 43, 0, 0, 0, 423, 3363, 0, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 4095, 340, 2, 4036, 0, 1, 4085, 4095, 1, 1, 1, 0, 3762, 0, 4095, 0, 1, 4095, 2, 0, 0, 1, 595, 3557, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4093, 164, 0, 4091, 0, 0, 4096, 4096, 0, 0, 0, 0, 3694, 1, 4096, 0, 0, 4096, 38, 0, 0, 0, 806, 3495, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4082, 163, 0, 4096, 0, 0, 4096, 4096, 0, 8, 0, 0, 3990, 1, 4096, 0, 0, 4096, 43, 0, 0, 0, 878, 3120, 0, 0, 0, 0, 3, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 4095, 154, 3, 4043, 0, 1, 4057, 4095, 1, 1, 1, 0, 4001, 0, 4095, 0, 1, 4095, 34, 0, 0, 1, 247, 3840, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 3881, 158, 0, 0, 0, 0, 1, 4079, 0, 1, 21, 0, 0, 0, 4095, 7, 1, 3, 0, 0, 0, 0, 0, 0, 3970, 0, 156, 32, 0, 0, 4092, 0, 1, 1, 0, 0, 0, 0, 13, 0, 0, 1, 4056, 1, 0, 0, 271, 3793, 0, 1, 37, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4102, 4106, 4158, 4094, 4071, 4102, 4094] avg=4096.0000 max=4158 max_rank=3 min=4041 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3881, 241, 0, 0, 0, 0, 0, 4091, 0, 0, 21, 0, 0, 0, 4092, 7, 0, 2, 0, 0, 0, 0, 0, 0, 3883, 1, 156, 32, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 4057, 1, 0, 0, 295, 3690, 0, 0, 37, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4112, 4101, 4072, 4096, 4147, 4022, 4096] avg=4096.0000 max=4147 max_rank=5 min=4022 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3867, 448, 0, 0, 0, 0, 0, 4094, 0, 0, 21, 0, 0, 0, 4095, 7, 0, 2, 0, 0, 0, 0, 0, 0, 3928, 0, 156, 32, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 4057, 1, 0, 0, 423, 3363, 0, 0, 37, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4315, 4115, 4104, 4116, 4096, 4101, 3823, 4098] avg=4096.0000 max=4315 max_rank=0 min=3823 min_rank=6 imbalance_ratio=1.053467 max_load_reduction=28453 max_load_reduction_pct=0.868317 moved_to_echo=16118 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3853, 115, 0, 0, 0, 0, 0, 4060, 0, 0, 21, 0, 0, 0, 4087, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 1, 155, 32, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 33, 0, 0, 0, 4057, 1, 0, 0, 214, 3878, 0, 0, 37, 0, 0, 0, 0, 0, 4, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[3968, 4081, 4094, 4210, 4097, 4091, 4129, 4098] avg=4096.0000 max=4210 max_rank=3 min=3968 min_rank=0 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 3881, 340, 0, 0, 0, 0, 2, 4036, 0, 1, 21, 0, 0, 0, 4085, 7, 1, 1, 0, 0, 0, 0, 1, 0, 3762, 0, 156, 32, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 2, 0, 0, 1, 4056, 1, 0, 0, 595, 3557, 0, 0, 37, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4223, 4060, 4094, 3951, 4097, 4060, 4189, 4094] avg=4096.0000 max=4223 max_rank=0 min=3951 min_rank=3 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3867, 164, 0, 0, 0, 0, 0, 4091, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3694, 1, 161, 36, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4060, 2, 0, 0, 806, 3495, 0, 0, 43, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4113, 4096, 3892, 4096, 4100, 4344, 4096] avg=4096.0000 max=4344 max_rank=6 min=3892 min_rank=3 imbalance_ratio=1.060547 max_load_reduction=28424 max_load_reduction_pct=0.867432 moved_to_echo=16636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3868, 163, 0, 0, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 4096, 7, 0, 8, 0, 0, 0, 0, 0, 0, 3990, 1, 156, 32, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 4057, 1, 0, 0, 878, 3120, 0, 0, 37, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4117, 4111, 4179, 4096, 4101, 4035, 4098] avg=4096.0000 max=4179 max_rank=3 min=4031 min_rank=0 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 3881, 154, 0, 0, 0, 0, 3, 4043, 0, 1, 21, 0, 0, 0, 4057, 3, 1, 1, 0, 0, 0, 0, 1, 0, 4001, 0, 156, 32, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 34, 0, 0, 1, 4060, 2, 0, 0, 247, 3840, 0, 0, 37, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4068, 4062, 4190, 4096, 4097, 4124, 4094] avg=4096.0000 max=4190 max_rank=3 min=4037 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 31104, 1792, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 0, 31360, 128, 1280, 384, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3840, 28800, 0, 128, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[384, 0, 0, 128, 32512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 0, 128, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4096, 0, 0, 0, 4093, 4096, 1, 0, 0, 0, 0, 4095, 0, 0, 4095, 7, 4092, 0, 0, 0, 149, 0, 1, 1, 0, 4096, 0, 3943] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 2, 1, 4095, 0, 1, 1, 4095, 4084, 18, 1, 1, 1, 0, 4093, 0, 1, 4093, 658, 4095, 0, 1, 0, 220, 4, 0, 0, 0, 3916, 0, 3386] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 4096, 0, 0, 0, 4093, 4095, 3, 0, 0, 0, 2, 4094, 0, 0, 4096, 137, 4091, 0, 0, 0, 256, 0, 0, 0, 0, 4096, 0, 3703] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 4096, 0, 0, 0, 4093, 4095, 3, 0, 0, 0, 2, 4094, 0, 0, 4095, 285, 4089, 0, 0, 0, 327, 0, 0, 0, 0, 4096, 0, 3486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4096, 0, 0, 0, 4093, 4095, 1, 0, 0, 0, 0, 4095, 0, 0, 4095, 738, 4095, 0, 0, 0, 631, 0, 1, 0, 0, 4096, 0, 2729] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4095, 0, 1, 1, 4095, 4086, 0, 1, 1, 1, 0, 4095, 0, 0, 4094, 419, 4095, 1, 1, 1, 18, 27, 0, 0, 0, 4041, 0, 3693] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4095, 0, 1, 1, 4095, 4085, 5, 1, 1, 1, 0, 4087, 0, 0, 4095, 330, 4095, 1, 1, 1, 134, 10, 0, 0, 0, 4081, 0, 3646] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 4096, 1, 0, 0, 4092, 4094, 2, 0, 0, 0, 2, 4095, 0, 0, 4096, 33, 4091, 0, 0, 0, 268, 0, 0, 0, 0, 4078, 0, 3814] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3771, 7, 0, 0, 0, 0, 4092, 0, 0, 0, 2, 0, 0, 0, 149, 0, 1, 1, 3545, 322, 2, 0, 0, 551, 0, 3943, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4097, 4095, 3778, 4094, 4020, 4494] avg=4096.0000 max=4494 max_rank=7 min=3778 min_rank=4 imbalance_ratio=1.097168 max_load_reduction=28274 max_load_reduction_pct=0.862854 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2, 1, 4093, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 1, 4084, 18, 1, 0, 0, 0, 0, 1, 1, 0, 4093, 0, 0, 0, 0, 0, 1, 3770, 658, 0, 0, 0, 0, 4095, 0, 1, 0, 2, 0, 0, 0, 220, 4, 0, 0, 3389, 321, 1, 0, 0, 527, 0, 3386, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4097, 4104, 4095, 4429, 4098, 3935, 3913] avg=4096.0000 max=4429 max_rank=4 min=3913 min_rank=7 imbalance_ratio=1.081299 max_load_reduction=28339 max_load_reduction_pct=0.864838 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 3772, 137, 0, 0, 0, 0, 4091, 0, 0, 0, 2, 0, 0, 0, 256, 0, 0, 0, 3545, 322, 2, 0, 0, 551, 0, 3703, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4096, 4098, 4096, 3909, 4093, 4125, 4254] avg=4096.0000 max=4254 max_rank=7 min=3909 min_rank=4 imbalance_ratio=1.038574 max_load_reduction=28514 max_load_reduction_pct=0.870178 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 3771, 285, 0, 0, 0, 0, 4089, 0, 0, 0, 2, 0, 0, 0, 327, 0, 0, 0, 3545, 322, 2, 0, 0, 551, 0, 3486, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4096, 4098, 4096, 4056, 4091, 4196, 4037] avg=4096.0000 max=4196 max_rank=6 min=4037 min_rank=7 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4092, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 4, 0, 0, 0, 0, 3763, 738, 0, 0, 0, 0, 4095, 0, 0, 0, 4, 0, 0, 0, 631, 0, 1, 0, 3550, 324, 1, 0, 0, 546, 0, 2729, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4092, 4096, 4103, 4501, 4099, 4507, 3275] avg=4096.0000 max=4507 max_rank=6 min=3275 min_rank=7 imbalance_ratio=1.100342 max_load_reduction=28261 max_load_reduction_pct=0.862457 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4093, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 1, 4086, 0, 1, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 3770, 419, 0, 0, 0, 0, 4095, 1, 1, 1, 2, 0, 0, 0, 18, 27, 0, 0, 3497, 322, 1, 0, 0, 544, 0, 3693, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4097, 4088, 4097, 4189, 4100, 3865, 4237] avg=4096.0000 max=4237 max_rank=7 min=3865 min_rank=6 imbalance_ratio=1.034424 max_load_reduction=28531 max_load_reduction_pct=0.870697 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 4090, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 3772, 33, 0, 0, 0, 0, 4091, 0, 0, 0, 2, 0, 0, 0, 268, 0, 0, 0, 3529, 322, 2, 0, 0, 549, 0, 3814, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4096, 4097, 3805, 4093, 4121, 4363] avg=4096.0000 max=4363 max_rank=7 min=3805 min_rank=4 imbalance_ratio=1.065186 max_load_reduction=28405 max_load_reduction_pct=0.866852 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4094, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 4085, 5, 1, 0, 0, 0, 0, 1, 1, 0, 4087, 0, 0, 0, 0, 0, 0, 3771, 330, 0, 0, 0, 0, 4095, 1, 1, 1, 2, 0, 0, 0, 134, 10, 0, 0, 3532, 322, 1, 0, 0, 549, 0, 3646, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4091, 4089, 4101, 4100, 3999, 4195] avg=4096.0000 max=4195 max_rank=7 min=3999 min_rank=6 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 4480, 0, 28416, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 30208, 2688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[2048, 128, 128, 128, 28160, 2688, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 2, 1, 4094, 0, 0, 1, 3887, 0, 3717, 1, 1, 1, 1, 4095, 1, 39, 1, 57, 0, 0, 4095, 0, 0, 0, 209, 2950, 0, 1688, 3831, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4005, 0, 4096, 0, 0, 0, 0, 4084, 3, 0, 0, 7, 0, 5, 4096, 0, 0, 0, 13, 2250, 0, 2508, 3510, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4082, 0, 4095, 0, 0, 0, 0, 4057, 0, 0, 0, 41, 0, 6, 4096, 0, 0, 0, 8, 2033, 0, 2287, 3871, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3955, 0, 4096, 0, 0, 0, 0, 4063, 0, 1, 0, 30, 0, 7, 4096, 0, 0, 0, 4, 2721, 0, 2036, 3567, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 1, 1, 4093, 0, 0, 0, 3854, 2, 3727, 1, 1, 1, 1, 4095, 0, 26, 1, 7, 0, 0, 4094, 0, 2, 0, 263, 2190, 1, 2437, 3874, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3909, 0, 4094, 0, 0, 0, 0, 4074, 3, 0, 0, 68, 0, 3, 4096, 0, 0, 0, 2, 1939, 0, 2614, 3775, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 1, 1, 4092, 0, 0, 1, 3997, 1, 3905, 1, 1, 1, 1, 4095, 1, 42, 1, 2, 0, 0, 4095, 0, 1, 0, 126, 2649, 0, 1666, 3993, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4017, 0, 4096, 0, 0, 0, 0, 4060, 1, 0, 0, 55, 0, 4, 4096, 0, 0, 0, 32, 1652, 0, 2662, 3902, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 14, 1, 0, 0, 3, 0, 0, 7, 3960, 186, 0, 0, 0, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 13, 2250, 1588, 0, 0, 0, 0, 2508, 1722, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4141, 4099, 4156, 4100, 3851, 4230] avg=4096.0000 max=4230 max_rank=7 min=3851 min_rank=6 imbalance_ratio=1.032715 max_load_reduction=28538 max_load_reduction_pct=0.870911 moved_to_echo=16337 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2, 1, 4094, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 3887, 0, 124, 1, 0, 0, 0, 0, 1, 1, 1, 4095, 15, 1, 0, 0, 1, 39, 1, 57, 3593, 203, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 209, 2950, 1733, 0, 0, 0, 0, 1688, 1880, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4095, 4012, 4114, 3895, 4094, 4892, 3568] avg=4096.0000 max=4892 max_rank=6 min=3568 min_rank=7 imbalance_ratio=1.194336 max_load_reduction=27876 max_load_reduction_pct=0.850708 moved_to_echo=16449 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 14, 1, 0, 0, 0, 1, 0, 30, 3960, 189, 0, 0, 0, 7, 4095, 0, 0, 0, 0, 0, 0, 0, 4, 2721, 1614, 0, 0, 0, 0, 2036, 1750, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4096, 4091, 4078, 4180, 4102, 4339, 3786] avg=4096.0000 max=4339 max_rank=6 min=3786 min_rank=7 imbalance_ratio=1.059326 max_load_reduction=28429 max_load_reduction_pct=0.867584 moved_to_echo=16407 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 15, 1, 0, 0, 0, 0, 0, 41, 3959, 205, 1, 0, 0, 6, 4095, 0, 0, 0, 0, 0, 0, 0, 8, 2033, 1751, 0, 0, 0, 0, 2287, 1900, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4218, 4073, 4206, 4101, 3792, 4187] avg=4096.0000 max=4218 max_rank=2 min=3792 min_rank=6 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16286 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 3909, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 18, 7, 0, 0, 3, 0, 0, 68, 3960, 203, 0, 0, 0, 3, 4082, 0, 0, 0, 0, 0, 0, 0, 2, 1939, 1712, 0, 0, 0, 0, 2614, 1842, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4103, 4043, 4099, 4234, 4085, 3653, 4456] avg=4096.0000 max=4456 max_rank=7 min=3653 min_rank=6 imbalance_ratio=1.087891 max_load_reduction=28312 max_load_reduction_pct=0.864014 moved_to_echo=16428 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 1, 4094, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 3854, 2, 124, 1, 0, 0, 0, 0, 1, 1, 1, 4095, 15, 1, 0, 0, 0, 26, 1, 7, 3603, 205, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2, 0, 263, 2190, 1753, 0, 0, 0, 1, 2437, 1901, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4093, 3981, 4114, 3843, 4093, 4208, 4339] avg=4096.0000 max=4339 max_rank=7 min=3843 min_rank=4 imbalance_ratio=1.059326 max_load_reduction=28429 max_load_reduction_pct=0.867584 moved_to_echo=16483 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 15, 1, 0, 0, 1, 0, 0, 55, 3960, 206, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 32, 1652, 1766, 0, 0, 0, 0, 2662, 1915, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4096, 4153, 4076, 4222, 4099, 3450, 4577] avg=4096.0000 max=4577 max_rank=7 min=3450 min_rank=6 imbalance_ratio=1.117432 max_load_reduction=28191 max_load_reduction_pct=0.860321 moved_to_echo=16348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 1, 4094, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 3997, 1, 130, 1, 0, 0, 0, 0, 1, 1, 1, 4095, 16, 1, 0, 0, 1, 42, 1, 2, 3775, 211, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 0, 126, 2649, 1807, 0, 0, 0, 0, 1666, 1959, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4093, 4129, 4115, 4032, 4094, 4583, 3625] avg=4096.0000 max=4583 max_rank=6 min=3625 min_rank=7 imbalance_ratio=1.118896 max_load_reduction=28185 max_load_reduction_pct=0.860138 moved_to_echo=16334 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 17920, 14976, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 768, 18432, 13824, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 128, 384, 30848, 1664, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[31744, 128, 1152, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32640, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4093, 4096, 0, 1, 0, 143, 0, 0, 3846, 0, 17, 4086, 0, 0, 4096, 0, 0, 4095, 0, 1, 102, 0, 4096, 0, 3, 0, 0, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 3813, 4085, 1, 0, 2, 205, 0, 0, 3848, 1, 0, 3905, 1, 7, 4093, 1, 0, 4095, 0, 0, 517, 1, 4095, 0, 0, 0, 1, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 396, 0, 0, 3662, 0, 8, 4047, 0, 0, 4096, 0, 0, 4096, 0, 0, 79, 0, 4096, 0, 2, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4096, 0, 0, 0, 496, 0, 0, 3442, 0, 8, 4013, 0, 0, 4096, 0, 0, 4096, 0, 0, 235, 0, 4096, 0, 3, 0, 0, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 3795, 4087, 1, 0, 1, 72, 0, 0, 4029, 1, 0, 4073, 1, 6, 4094, 1, 0, 4095, 0, 0, 318, 1, 4095, 0, 0, 0, 1, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 3740, 4082, 1, 0, 0, 202, 0, 0, 3749, 1, 0, 3941, 1, 8, 4095, 1, 1, 4095, 0, 0, 658, 0, 4095, 0, 0, 0, 1, 0, 1, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4092, 4096, 0, 0, 0, 11, 0, 0, 4050, 0, 12, 4090, 0, 0, 4096, 0, 0, 4096, 0, 0, 26, 0, 4096, 0, 12, 0, 0, 0, 0, 4091] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4096, 0, 0, 0, 40, 0, 0, 3707, 0, 8, 4048, 0, 0, 4096, 0, 0, 4096, 0, 0, 388, 0, 4096, 0, 5, 0, 0, 0, 0, 4093] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 120, 0, 0, 0, 0, 0, 1, 0, 143, 3899, 0, 0, 0, 0, 0, 3846, 0, 304, 0, 0, 0, 17, 4086, 0, 0, 59, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 102, 0, 3613, 196, 0, 0, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4213, 4043, 4150, 4163, 4096, 3912, 4097, 4094] avg=4096.0000 max=4213 max_rank=0 min=3912 min_rank=5 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16199 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3813, 120, 0, 0, 0, 0, 1, 0, 2, 205, 3899, 0, 0, 0, 0, 0, 3848, 1, 303, 0, 0, 0, 0, 3905, 1, 7, 59, 1, 0, 0, 4093, 1, 0, 1, 0, 0, 0, 0, 0, 0, 517, 1, 3603, 195, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3934, 4107, 4152, 3973, 4095, 4316, 4093, 4098] avg=4096.0000 max=4316 max_rank=5 min=3934 min_rank=0 imbalance_ratio=1.053711 max_load_reduction=28452 max_load_reduction_pct=0.868286 moved_to_echo=16602 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 120, 0, 0, 0, 0, 0, 0, 0, 40, 3899, 0, 0, 0, 0, 0, 3707, 0, 304, 0, 0, 0, 8, 4048, 0, 0, 59, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 0, 3613, 197, 0, 0, 4094, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4215, 3939, 4011, 4116, 4096, 4198, 4099, 4094] avg=4096.0000 max=4215 max_rank=0 min=3939 min_rank=1 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 107, 0, 0, 0, 0, 0, 0, 0, 396, 3901, 0, 0, 0, 0, 0, 3662, 0, 307, 0, 0, 0, 8, 4047, 0, 0, 66, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 3616, 195, 0, 0, 4088, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[4203, 4297, 3969, 4123, 4096, 3890, 4090, 4100] avg=4096.0000 max=4297 max_rank=1 min=3890 min_rank=5 imbalance_ratio=1.049072 max_load_reduction=28471 max_load_reduction_pct=0.868866 moved_to_echo=16176 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4092, 120, 0, 0, 0, 0, 0, 0, 0, 11, 3899, 0, 0, 0, 0, 0, 4050, 0, 304, 0, 0, 0, 12, 4090, 0, 0, 59, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 3613, 197, 0, 0, 4094, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4212, 3910, 4354, 4162, 4096, 3836, 4106, 4092] avg=4096.0000 max=4354 max_rank=2 min=3836 min_rank=5 imbalance_ratio=1.062988 max_load_reduction=28414 max_load_reduction_pct=0.867126 moved_to_echo=16130 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3740, 120, 0, 0, 0, 0, 1, 0, 0, 202, 3899, 0, 0, 0, 0, 0, 3749, 1, 303, 0, 0, 0, 0, 3941, 1, 8, 59, 1, 0, 0, 4095, 1, 1, 1, 0, 0, 0, 0, 0, 0, 658, 0, 3600, 195, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3861, 4102, 4053, 4010, 4098, 4453, 4093, 4098] avg=4096.0000 max=4453 max_rank=5 min=3861 min_rank=0 imbalance_ratio=1.087158 max_load_reduction=28315 max_load_reduction_pct=0.864105 moved_to_echo=16742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 120, 0, 0, 0, 0, 0, 0, 0, 496, 3900, 0, 0, 0, 0, 0, 3442, 0, 304, 0, 0, 0, 8, 4013, 0, 0, 59, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 3613, 196, 0, 0, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[4214, 4396, 3746, 4081, 4096, 4044, 4097, 4094] avg=4096.0000 max=4396 max_rank=1 min=3746 min_rank=2 imbalance_ratio=1.073242 max_load_reduction=28372 max_load_reduction_pct=0.865845 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3795, 120, 0, 0, 0, 0, 1, 0, 1, 72, 3900, 0, 0, 0, 0, 0, 4029, 1, 303, 0, 0, 0, 0, 4073, 1, 6, 59, 1, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 318, 1, 3605, 195, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 0] rerouted_tokens_per_ep_rank=[3916, 3974, 4333, 4140, 4095, 4119, 4093, 4098] avg=4096.0000 max=4333 max_rank=2 min=3916 min_rank=0 imbalance_ratio=1.057861 max_load_reduction=28435 max_load_reduction_pct=0.867767 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 30336, 128, 2432, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 1664, 31232, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 31872, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 32256, 128, 128, 512, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[0, 128, 2432, 128, 28928, 1664, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[2715, 4096, 0, 1, 0, 4096, 4096, 1, 0, 0, 3588, 0, 4096, 0, 4096, 641, 0, 0, 0, 1642, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[2613, 4092, 0, 0, 0, 4085, 4095, 0, 1, 1, 4008, 0, 4095, 2, 3947, 528, 12, 1, 1, 3151, 1, 0, 0, 1, 0, 1, 2, 2130, 0, 0, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3251, 4096, 0, 0, 1, 4096, 4096, 1, 0, 0, 3813, 1, 4096, 0, 4096, 456, 0, 0, 0, 1610, 0, 0, 0, 0, 0, 0, 0, 3155, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3231, 4096, 0, 0, 1, 4096, 4096, 0, 0, 0, 3873, 0, 4096, 0, 4096, 380, 0, 0, 0, 1799, 0, 0, 0, 0, 0, 0, 0, 3004, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3508, 4092, 0, 0, 0, 4086, 4095, 0, 0, 1, 3952, 0, 4095, 3, 3915, 690, 28, 1, 1, 1835, 1, 0, 0, 1, 0, 1, 1, 2457, 0, 1, 1, 3] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3420, 4091, 0, 0, 0, 4085, 4095, 0, 1, 1, 3761, 0, 4095, 0, 3942, 747, 11, 1, 1, 2767, 1, 0, 0, 1, 0, 1, 1, 1740, 0, 0, 1, 5] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[3454, 4096, 0, 0, 0, 4096, 4096, 1, 0, 0, 3915, 0, 4096, 0, 4096, 814, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 3551, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1747, 4096, 0, 0, 0, 4096, 4096, 1, 0, 0, 3787, 0, 4096, 0, 4096, 154, 0, 0, 0, 3245, 0, 0, 0, 0, 0, 0, 0, 3354, 0, 0, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2613, 1105, 0, 0, 0, 0, 0, 0, 0, 4085, 4, 0, 0, 0, 0, 0, 1, 1, 4008, 0, 252, 0, 0, 0, 0, 2, 3467, 528, 0, 0, 0, 0, 12, 1, 1, 3151, 2012, 0, 0, 0, 1, 0, 0, 1, 4094, 0, 0, 0, 0, 1, 2, 2130, 974, 228, 0, 0, 0, 0, 0, 1, 4091, 1, 1, 0] rerouted_tokens_per_ep_rank=[3718, 4089, 4262, 3997, 5177, 4096, 3335, 4094] avg=4096.0000 max=5177 max_rank=4 min=3335 min_rank=6 imbalance_ratio=1.263916 max_load_reduction=27591 max_load_reduction_pct=0.842010 moved_to_echo=16702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2715, 1106, 0, 1, 0, 0, 0, 0, 0, 4096, 4, 1, 0, 0, 0, 0, 0, 0, 3588, 0, 261, 0, 0, 0, 0, 0, 3599, 641, 0, 0, 0, 0, 0, 0, 0, 1642, 2014, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3700, 975, 236, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0] rerouted_tokens_per_ep_rank=[3822, 4101, 3849, 4240, 3656, 4096, 4911, 4093] avg=4096.0000 max=4911 max_rank=6 min=3656 min_rank=4 imbalance_ratio=1.198975 max_load_reduction=27857 max_load_reduction_pct=0.850128 moved_to_echo=16756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3231, 1106, 0, 0, 0, 0, 0, 0, 1, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 3873, 0, 261, 0, 0, 0, 0, 0, 3599, 380, 0, 0, 0, 0, 0, 0, 0, 1799, 2014, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3004, 975, 236, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0] rerouted_tokens_per_ep_rank=[4337, 4101, 4134, 3979, 3813, 4096, 4215, 4093] avg=4096.0000 max=4337 max_rank=0 min=3813 min_rank=4 imbalance_ratio=1.058838 max_load_reduction=28431 max_load_reduction_pct=0.867645 moved_to_echo=16217 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3251, 1106, 0, 0, 0, 0, 0, 0, 1, 4096, 4, 1, 0, 0, 0, 0, 0, 0, 3813, 1, 261, 0, 0, 0, 0, 0, 3599, 456, 0, 0, 0, 0, 0, 0, 0, 1610, 2014, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3155, 975, 236, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 0] rerouted_tokens_per_ep_rank=[4357, 4102, 4075, 4055, 3624, 4095, 4366, 4094] avg=4096.0000 max=4366 max_rank=6 min=3624 min_rank=4 imbalance_ratio=1.065918 max_load_reduction=28402 max_load_reduction_pct=0.866760 moved_to_echo=16179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3508, 1105, 0, 0, 0, 0, 0, 0, 0, 4086, 4, 0, 0, 0, 0, 0, 0, 1, 3952, 0, 250, 0, 0, 0, 0, 3, 3439, 690, 0, 0, 0, 0, 28, 1, 1, 1835, 2012, 0, 0, 0, 1, 0, 0, 1, 4094, 0, 0, 0, 0, 1, 1, 2457, 974, 226, 0, 0, 0, 1, 1, 3, 4091, 1, 1, 0] rerouted_tokens_per_ep_rank=[4613, 4090, 4203, 4132, 3877, 4096, 3659, 4098] avg=4096.0000 max=4613 max_rank=0 min=3659 min_rank=6 imbalance_ratio=1.126221 max_load_reduction=28155 max_load_reduction_pct=0.859222 moved_to_echo=15730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3454, 1106, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 1, 0, 0, 0, 0, 0, 0, 3915, 0, 261, 0, 0, 0, 0, 0, 3599, 814, 0, 0, 0, 0, 0, 0, 0, 553, 2014, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3551, 975, 236, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0] rerouted_tokens_per_ep_rank=[4560, 4101, 4176, 4413, 2567, 4096, 4762, 4093] avg=4096.0000 max=4762 max_rank=6 min=2567 min_rank=4 imbalance_ratio=1.162598 max_load_reduction=28006 max_load_reduction_pct=0.854675 moved_to_echo=15518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1747, 1089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 3787, 0, 267, 0, 0, 0, 0, 0, 3588, 154, 0, 0, 0, 0, 0, 0, 0, 3245, 2018, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3354, 982, 241, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0] rerouted_tokens_per_ep_rank=[2836, 4097, 4054, 3742, 5263, 4096, 4577, 4103] avg=4096.0000 max=5263 max_rank=4 min=2836 min_rank=0 imbalance_ratio=1.284912 max_load_reduction=27505 max_load_reduction_pct=0.839386 moved_to_echo=18039 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3420, 1105, 0, 0, 0, 0, 0, 0, 0, 4085, 2, 0, 0, 0, 0, 0, 1, 1, 3761, 0, 252, 0, 0, 0, 0, 0, 3463, 747, 0, 0, 0, 0, 11, 1, 1, 2767, 2011, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 1, 1740, 974, 227, 0, 0, 0, 0, 1, 5, 4093, 1, 0, 0] rerouted_tokens_per_ep_rank=[4525, 4087, 4015, 4210, 4791, 4097, 2943, 4100] avg=4096.0000 max=4791 max_rank=4 min=2943 min_rank=6 imbalance_ratio=1.169678 max_load_reduction=27977 max_load_reduction_pct=0.853790 moved_to_echo=15931 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 16640, 16128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[24064, 8832, 0, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[0, 128, 128, 23168, 7808, 1920, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 28416, 4480, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 30720, 128, 2176, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[292, 0, 0, 0, 0, 0, 2, 0, 4056, 0, 50, 0, 0, 4092, 0, 0, 4094, 0, 0, 0, 3964, 0, 2, 3928, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[266, 0, 0, 0, 0, 0, 1, 0, 4087, 0, 69, 0, 0, 4092, 0, 1, 4094, 0, 0, 0, 4092, 0, 2, 3777, 4096, 4095, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1119, 1, 3, 1, 0, 0, 0, 8, 4076, 13, 153, 0, 2, 4065, 0, 0, 4095, 1, 0, 1, 3959, 0, 0, 2967, 4093, 4094, 0, 30, 1, 4085, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1098, 1, 2, 1, 0, 0, 0, 6, 3939, 9, 379, 0, 1, 4031, 0, 0, 4095, 0, 0, 1, 3995, 0, 0, 2899, 4094, 4095, 0, 31, 1, 4089, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[146, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 112, 0, 0, 4088, 0, 2, 4094, 0, 0, 0, 4096, 0, 0, 3894, 4096, 4096, 1, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[166, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 54, 0, 0, 4091, 0, 1, 4095, 0, 0, 0, 4088, 0, 1, 3917, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[123, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 81, 0, 0, 4084, 0, 1, 4095, 0, 0, 0, 4092, 0, 1, 4019, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1406, 1, 2, 1, 0, 0, 0, 13, 4073, 7, 150, 0, 1, 4039, 0, 0, 4095, 0, 0, 1, 3989, 0, 0, 2687, 4094, 4095, 0, 31, 1, 4081, 1, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1098, 1, 2, 1, 3416, 62, 3, 0, 0, 0, 0, 6, 4092, 0, 0, 0, 3862, 9, 379, 0, 0, 0, 0, 0, 1, 4031, 0, 0, 15, 6, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 579, 0, 0, 2899, 0, 0, 0, 0, 4085, 0, 0, 31, 0, 0, 0, 0, 1, 4089, 1, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4583, 4098, 4250, 4053, 4096, 3478, 4116, 4094] avg=4096.0000 max=4583 max_rank=0 min=3478 min_rank=5 imbalance_ratio=1.118896 max_load_reduction=28185 max_load_reduction_pct=0.860138 moved_to_echo=15784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[292, 0, 0, 0, 3389, 64, 3, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 3977, 0, 50, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 15, 6, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 575, 0, 2, 3928, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[3748, 4095, 4027, 4113, 4094, 4505, 4087, 4099] avg=4096.0000 max=4505 max_rank=5 min=3748 min_rank=0 imbalance_ratio=1.099854 max_load_reduction=28263 max_load_reduction_pct=0.862518 moved_to_echo=16785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[146, 0, 0, 0, 3502, 64, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3968, 0, 112, 0, 0, 0, 0, 0, 0, 4088, 0, 2, 15, 6, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 594, 0, 0, 3894, 0, 0, 0, 0, 4087, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[3715, 4093, 4080, 4111, 4094, 4488, 4088, 4099] avg=4096.0000 max=4488 max_rank=5 min=3715 min_rank=0 imbalance_ratio=1.095703 max_load_reduction=28280 max_load_reduction_pct=0.863037 moved_to_echo=16769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[266, 0, 0, 0, 3499, 64, 3, 0, 0, 0, 1, 0, 4092, 0, 0, 0, 4008, 0, 69, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 15, 6, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 593, 0, 2, 3777, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[3832, 4093, 4077, 4114, 4094, 4372, 4087, 4099] avg=4096.0000 max=4372 max_rank=5 min=3832 min_rank=0 imbalance_ratio=1.067383 max_load_reduction=28396 max_load_reduction_pct=0.866577 moved_to_echo=16652 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1119, 1, 3, 1, 3385, 64, 3, 0, 0, 0, 0, 8, 4091, 0, 0, 0, 3997, 13, 153, 0, 0, 0, 0, 0, 2, 4065, 0, 0, 15, 6, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 0, 574, 0, 0, 2967, 0, 0, 0, 0, 4084, 0, 0, 30, 0, 0, 0, 0, 1, 4085, 1, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4576, 4099, 4163, 4088, 4097, 3541, 4114, 4090] avg=4096.0000 max=4576 max_rank=0 min=3541 min_rank=5 imbalance_ratio=1.117188 max_load_reduction=28192 max_load_reduction_pct=0.860352 moved_to_echo=15842 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[166, 0, 0, 0, 3495, 64, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3988, 0, 54, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 15, 6, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 593, 0, 1, 3917, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[3728, 4093, 4042, 4113, 4095, 4511, 4087, 4099] avg=4096.0000 max=4511 max_rank=5 min=3728 min_rank=0 imbalance_ratio=1.101318 max_load_reduction=28257 max_load_reduction_pct=0.862335 moved_to_echo=16792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[123, 0, 0, 0, 3504, 67, 4, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3897, 0, 81, 0, 0, 0, 0, 0, 0, 4084, 0, 1, 20, 10, 0, 0, 4095, 0, 0, 0, 7, 0, 0, 0, 588, 0, 1, 4019, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0] rerouted_tokens_per_ep_rank=[3698, 4092, 3978, 4115, 4102, 4608, 4073, 4102] avg=4096.0000 max=4608 max_rank=5 min=3698 min_rank=0 imbalance_ratio=1.125000 max_load_reduction=28160 max_load_reduction_pct=0.859375 moved_to_echo=16885 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1406, 1, 2, 1, 3411, 64, 3, 0, 0, 0, 0, 13, 4092, 0, 0, 0, 3994, 7, 150, 0, 0, 0, 0, 0, 1, 4039, 0, 0, 15, 6, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 578, 0, 0, 2687, 0, 0, 0, 0, 4085, 0, 0, 31, 0, 0, 0, 0, 1, 4081, 1, 0, 3, 0, 0, 0] rerouted_tokens_per_ep_rank=[4888, 4105, 4151, 4061, 4096, 3265, 4116, 4086] avg=4096.0000 max=4888 max_rank=0 min=3265 min_rank=5 imbalance_ratio=1.193359 max_load_reduction=27880 max_load_reduction_pct=0.850830 moved_to_echo=15563 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4736, 0, 128, 28160, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4736, 128, 128, 128, 27648, 640, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31744, 128, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32640, 0, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 1, 10, 5, 1, 4075, 4095, 0, 0, 0, 0, 0, 1, 4094, 4092, 4080, 1, 1, 14, 7, 1, 0, 0, 1, 4095, 0, 4095, 2, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4087, 0, 11, 0, 0, 18, 0, 0, 3958, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 20, 96, 0, 0, 0, 0, 4096, 7, 4091, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4087, 0, 0, 1, 1, 3, 16, 1, 3953, 4095, 0, 0, 0, 0, 0, 1, 4094, 4095, 4079, 1, 1, 0, 139, 2, 0, 0, 1, 4095, 0, 4095, 8, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4094, 1, 12, 0, 0, 20, 0, 0, 3920, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 132, 0, 0, 0, 0, 4096, 13, 4095, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4090, 0, 35, 0, 0, 6, 0, 0, 3949, 4095, 1, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 6, 99, 0, 0, 0, 0, 4096, 9, 4094, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 24, 0, 0, 27, 0, 0, 3988, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 57, 0, 0, 0, 0, 4096, 2, 4094, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4089, 0, 0, 1, 1, 8, 10, 1, 3997, 4095, 0, 0, 0, 0, 0, 1, 4094, 4095, 4085, 1, 1, 0, 91, 1, 0, 0, 1, 4095, 0, 4095, 6, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4095, 1, 37, 0, 0, 24, 0, 0, 3947, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 84, 0, 0, 0, 0, 4096, 9, 4090, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 1, 1, 0, 0, 0, 0, 1, 10, 5, 1, 4073, 0, 0, 0, 4075, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 7, 4080, 1, 0, 0, 0, 0, 1, 14, 7, 1, 3972, 12, 11, 5, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4090, 4198, 4095, 4088, 4023, 4096, 4092] avg=4096.0000 max=4198 max_rank=2 min=4023 min_rank=5 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4083, 1, 12, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4077, 0, 0, 0, 3920, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 132, 0, 3973, 12, 11, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 13, 4090, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4097, 4043, 4096, 4103, 4133, 4096, 4104] avg=4096.0000 max=4133 max_rank=5 min=4043 min_rank=2 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4076, 0, 11, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4077, 0, 0, 0, 3958, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 20, 96, 0, 3973, 12, 11, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 4087, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4095, 4081, 4096, 4103, 4116, 4096, 4094] avg=4096.0000 max=4116 max_rank=5 min=4081 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=28652 max_load_reduction_pct=0.874390 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4079, 0, 35, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4077, 0, 0, 0, 3949, 123, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 6, 99, 0, 3972, 12, 11, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 4089, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4083, 4073, 4096, 4103, 4105, 4096, 4098] avg=4096.0000 max=4114 max_rank=0 min=4073 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=28654 max_load_reduction_pct=0.874451 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4076, 0, 0, 1, 0, 0, 0, 0, 1, 3, 16, 1, 4076, 0, 0, 0, 3953, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 7, 4079, 1, 0, 0, 0, 0, 1, 0, 139, 2, 3972, 12, 11, 5, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 4090, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4097, 4076, 4095, 4087, 4142, 4096, 4098] avg=4096.0000 max=4142 max_rank=5 min=4076 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4084, 1, 37, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4077, 0, 0, 0, 3947, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 1, 84, 0, 3972, 13, 11, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 4086, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4101, 4070, 4096, 4102, 4085, 4096, 4096] avg=4096.0000 max=4122 max_rank=0 min=4070 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 24, 0, 0, 0, 0, 0, 0, 27, 0, 0, 4082, 0, 0, 0, 3988, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 57, 0, 3977, 12, 12, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4087, 0, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4109, 4107, 4098, 4096, 4065, 4096, 4090] avg=4096.0000 max=4109 max_rank=1 min=4065 min_rank=5 imbalance_ratio=1.003174 max_load_reduction=28659 max_load_reduction_pct=0.874603 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4078, 0, 0, 1, 0, 0, 0, 0, 1, 8, 10, 1, 4076, 0, 0, 0, 3997, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 0, 0, 4085, 1, 0, 0, 0, 0, 1, 0, 91, 1, 3972, 18, 11, 5, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4096, 4120, 4096, 4086, 4099, 4096, 4096] avg=4096.0000 max=4120 max_rank=2 min=4079 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=28648 max_load_reduction_pct=0.874268 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[31872, 1024, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 128, 768, 128, 31872, 128, 128, 128] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 128, 128, 128, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 3608, 0, 0, 1105, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3033, 0, 0, 437, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1, 3378, 1, 0, 1985, 1, 0, 4094, 1, 0, 1, 1, 562, 2, 0, 0, 4095, 0, 2, 4095, 1, 0, 0, 4095, 2265, 0, 0, 136, 1, 3956, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 3995, 0, 0, 631, 0, 0, 4096, 0, 0, 0, 0, 31, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3503, 0, 0, 32, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 3649, 1, 0, 1045, 1, 0, 4094, 1, 0, 1, 1, 367, 3, 0, 1, 4094, 0, 1, 4095, 1, 0, 1, 4095, 3126, 0, 0, 133, 1, 3962, 0, 4095] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 3797, 0, 0, 1009, 0, 0, 4096, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3250, 0, 0, 117, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 3567, 1, 0, 1550, 1, 1, 4094, 1, 1, 1, 1, 580, 2, 0, 0, 4092, 0, 6, 4095, 1, 0, 1, 4095, 2509, 0, 0, 110, 1, 3964, 0, 4094] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 3943, 0, 0, 961, 0, 0, 4096, 0, 0, 0, 0, 29, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3171, 0, 0, 88, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 3585, 0, 0, 2048, 0, 0, 4096, 0, 0, 0, 0, 47, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2463, 0, 0, 49, 0, 4096, 0, 4096] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3608, 0, 0, 404, 0, 0, 0, 1105, 0, 0, 2806, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 9, 0, 0, 0, 3889, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3033, 0, 0, 437, 886, 156, 0, 0, 0, 4096, 0, 51, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4012, 3911, 4095, 3899, 4096, 4096, 4512, 4147] avg=4096.0000 max=4512 max_rank=6 min=3899 min_rank=3 imbalance_ratio=1.101562 max_load_reduction=28256 max_load_reduction_pct=0.862305 moved_to_echo=16851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 3378, 1, 0, 404, 0, 0, 0, 1985, 1, 0, 2804, 0, 0, 0, 0, 1, 0, 1, 1, 4094, 0, 0, 0, 562, 2, 0, 0, 3888, 1, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 2265, 0, 0, 136, 886, 156, 0, 0, 1, 3956, 0, 51, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3784, 4790, 4097, 4453, 4097, 4096, 3443, 4008] avg=4096.0000 max=4790 max_rank=1 min=3443 min_rank=6 imbalance_ratio=1.169434 max_load_reduction=27978 max_load_reduction_pct=0.853821 moved_to_echo=15644 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3797, 0, 0, 404, 0, 0, 0, 1009, 0, 0, 2806, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 19, 0, 0, 0, 3889, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3250, 0, 0, 117, 886, 156, 0, 0, 0, 4096, 0, 51, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4201, 3815, 4095, 3909, 4096, 4096, 4409, 4147] avg=4096.0000 max=4409 max_rank=6 min=3815 min_rank=1 imbalance_ratio=1.076416 max_load_reduction=28359 max_load_reduction_pct=0.865448 moved_to_echo=16748 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3995, 0, 0, 404, 0, 0, 0, 631, 0, 0, 2806, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 31, 0, 0, 0, 3889, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3503, 0, 0, 32, 886, 156, 0, 0, 0, 4096, 0, 51, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4399, 3437, 4095, 3921, 4096, 4096, 4577, 4147] avg=4096.0000 max=4577 max_rank=6 min=3437 min_rank=1 imbalance_ratio=1.117432 max_load_reduction=28191 max_load_reduction_pct=0.860321 moved_to_echo=16916 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3567, 1, 0, 404, 0, 0, 0, 1550, 1, 1, 2804, 0, 0, 0, 0, 1, 1, 1, 1, 4094, 0, 0, 0, 580, 2, 0, 0, 3887, 1, 0, 0, 4092, 0, 6, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 2509, 0, 0, 110, 886, 156, 0, 0, 1, 3964, 0, 51, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3972, 4356, 4098, 4470, 4098, 4097, 3661, 4016] avg=4096.0000 max=4470 max_rank=3 min=3661 min_rank=6 imbalance_ratio=1.091309 max_load_reduction=28298 max_load_reduction_pct=0.863586 moved_to_echo=15872 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3943, 0, 0, 404, 0, 0, 0, 961, 0, 0, 2806, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 29, 0, 0, 0, 3889, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3171, 0, 0, 88, 886, 156, 0, 0, 0, 4096, 0, 51, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4347, 3767, 4095, 3919, 4096, 4096, 4301, 4147] avg=4096.0000 max=4347 max_rank=0 min=3767 min_rank=1 imbalance_ratio=1.061279 max_load_reduction=28421 max_load_reduction_pct=0.867340 moved_to_echo=16640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3585, 0, 0, 410, 2, 2, 0, 2048, 0, 0, 2794, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 47, 0, 0, 0, 3890, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2463, 0, 0, 49, 892, 160, 0, 0, 0, 4096, 0, 46, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3999, 4842, 4096, 3937, 4094, 4094, 3564, 4142] avg=4096.0000 max=4842 max_rank=1 min=3564 min_rank=6 imbalance_ratio=1.182129 max_load_reduction=27926 max_load_reduction_pct=0.852234 moved_to_echo=15894 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3649, 1, 0, 404, 0, 0, 0, 1045, 1, 0, 2804, 0, 0, 0, 0, 1, 0, 1, 1, 4094, 0, 0, 0, 367, 3, 0, 1, 3888, 1, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 3126, 0, 0, 133, 886, 156, 0, 0, 1, 3962, 0, 51, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 3850, 4097, 4260, 4095, 4097, 4301, 4014] avg=4096.0000 max=4301 max_rank=6 min=3850 min_rank=1 imbalance_ratio=1.050049 max_load_reduction=28467 max_load_reduction_pct=0.868744 moved_to_echo=16507 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[23424, 0, 0, 1152, 7168, 1280, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[10368, 128, 128, 22528, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[128, 29568, 128, 0, 3328, 128, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[1664, 128, 0, 128, 31232, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 32384, 0, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1448, 4095, 4096, 4096, 0, 0, 2, 4094, 0, 0, 3, 1, 0, 4093, 0, 0, 1348, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1300, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[2086, 4095, 4096, 4096, 0, 0, 0, 4095, 0, 0, 2, 3, 0, 4093, 0, 0, 322, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1688, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[2156, 4096, 4096, 4096, 0, 0, 0, 4092, 0, 0, 0, 2, 0, 4094, 0, 0, 380, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1564, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1833, 4095, 4095, 4031, 1, 2, 0, 4095, 1, 0, 0, 50, 1, 4092, 0, 3, 818, 1, 0, 0, 0, 1, 4094, 0, 0, 0, 4095, 1, 0, 1458, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1994, 4095, 4095, 4038, 2, 2, 2, 4089, 1, 0, 0, 44, 1, 4092, 0, 2, 668, 1, 0, 0, 0, 1, 4093, 0, 0, 0, 4095, 1, 0, 1451, 0, 1] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1847, 4095, 4096, 4096, 0, 0, 0, 4094, 0, 0, 1, 8, 0, 4092, 0, 0, 1462, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 785, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1812, 4095, 4095, 4042, 1, 2, 1, 4095, 1, 0, 0, 40, 1, 4093, 0, 3, 966, 1, 0, 0, 0, 1, 4095, 0, 0, 0, 4095, 1, 1, 1327, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1739, 4095, 4096, 4096, 0, 0, 0, 4082, 0, 0, 0, 1, 0, 4096, 0, 0, 423, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1948, 0, 0] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1448, 0, 0, 2245, 0, 0, 0, 0, 0, 0, 2, 4094, 2, 0, 0, 0, 0, 0, 3, 1, 4077, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 1348, 0, 0, 0, 3278, 19, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1300, 0, 0, 1848, 817, 0, 0] rerouted_tokens_per_ep_rank=[3693, 4098, 4081, 4094, 4645, 4096, 4096, 3965] avg=4096.0000 max=4645 max_rank=4 min=3693 min_rank=0 imbalance_ratio=1.134033 max_load_reduction=28123 max_load_reduction_pct=0.858246 moved_to_echo=16802 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2156, 0, 0, 2245, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 2, 4077, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 380, 0, 0, 0, 3279, 19, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1564, 0, 0, 1848, 817, 0, 0] rerouted_tokens_per_ep_rank=[4401, 4094, 4079, 4095, 3678, 4096, 4096, 4229] avg=4096.0000 max=4401 max_rank=0 min=3678 min_rank=4 imbalance_ratio=1.074463 max_load_reduction=28367 max_load_reduction_pct=0.865692 moved_to_echo=16099 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1833, 0, 0, 2210, 0, 0, 0, 0, 1, 2, 0, 4095, 2, 0, 0, 0, 1, 0, 0, 50, 4075, 0, 0, 0, 1, 4092, 0, 3, 1, 0, 0, 0, 818, 1, 0, 0, 3277, 20, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1458, 0, 1, 1818, 818, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4100, 4126, 4097, 4116, 4095, 4096, 4095] avg=4096.0000 max=4126 max_rank=2 min=4043 min_rank=0 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2086, 0, 0, 2245, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 2, 3, 4076, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 322, 0, 0, 0, 3278, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1688, 0, 0, 1848, 817, 0, 0] rerouted_tokens_per_ep_rank=[4331, 4097, 4081, 4094, 3620, 4096, 4096, 4353] avg=4096.0000 max=4353 max_rank=7 min=3620 min_rank=4 imbalance_ratio=1.062744 max_load_reduction=28415 max_load_reduction_pct=0.867157 moved_to_echo=16165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1847, 0, 0, 2245, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 1, 8, 4077, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 1462, 0, 0, 0, 3278, 19, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 785, 0, 0, 1848, 817, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4096, 4086, 4093, 4759, 4096, 4096, 3450] avg=4096.0000 max=4759 max_rank=4 min=3450 min_rank=7 imbalance_ratio=1.161865 max_load_reduction=28009 max_load_reduction_pct=0.854767 moved_to_echo=16401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1994, 0, 0, 2213, 0, 0, 0, 0, 2, 2, 2, 4089, 2, 0, 0, 0, 1, 0, 0, 44, 4075, 0, 0, 0, 1, 4092, 0, 2, 1, 0, 0, 0, 668, 1, 0, 0, 3277, 20, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 1451, 0, 1, 1822, 818, 0, 0] rerouted_tokens_per_ep_rank=[4207, 4097, 4120, 4096, 3966, 4094, 4096, 4092] avg=4096.0000 max=4207 max_rank=0 min=3966 min_rank=4 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16248 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1739, 0, 0, 2234, 0, 0, 0, 0, 0, 0, 0, 4082, 3, 0, 0, 0, 0, 0, 0, 1, 4077, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 423, 0, 0, 0, 3278, 19, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1948, 0, 0, 1851, 817, 0, 0] rerouted_tokens_per_ep_rank=[3973, 4085, 4078, 4101, 3720, 4099, 4096, 4616] avg=4096.0000 max=4616 max_rank=7 min=3720 min_rank=4 imbalance_ratio=1.126953 max_load_reduction=28152 max_load_reduction_pct=0.859131 moved_to_echo=16531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1812, 0, 0, 2216, 0, 0, 0, 0, 1, 2, 1, 4095, 2, 0, 0, 0, 1, 0, 0, 40, 4076, 0, 0, 0, 1, 4093, 0, 3, 1, 0, 0, 0, 966, 1, 0, 0, 3278, 19, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 1, 1327, 0, 0, 1823, 817, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4101, 4117, 4098, 4264, 4096, 4096, 3968] avg=4096.0000 max=4264 max_rank=4 min=3968 min_rank=7 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[14976, 0, 0, 17920, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 11648, 0, 128, 14720, 6656, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[6400, 128, 0, 0, 26240, 256, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[128, 0, 128, 256, 32640, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=32896 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=32896 slack_slots=32640 device_initiated_grouped_gemm=True + [2026-08-30 04:55:28] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 673.0 | throughput per GPU (TFLOP/s/GPU): 495.3 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.818356E+00 | loss scale: 1.0 | grad norm: 5.223 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 04:55:28 +[rank0]:[W830 04:55:29.985849151 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..7071230b5fba050afdb5e564e9d6b310f74f6fac --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1/mem_samples.csv @@ -0,0 +1,256 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 2 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 629 +3, 629 +4, 629 +5, 629 +6, 629 +7, 629 +0, 861 +1, 869 +2, 873 +3, 871 +4, 873 +5, 861 +6, 881 +7, 863 +0, 1895 +1, 1895 +2, 1895 +3, 1895 +4, 1895 +5, 1895 +6, 1895 +7, 1895 +0, 38795 +1, 38785 +2, 38811 +3, 38787 +4, 38785 +5, 38815 +6, 38819 +7, 38839 +0, 39143 +1, 39167 +2, 39141 +3, 39145 +4, 39137 +5, 39153 +6, 39133 +7, 39137 +0, 41799 +1, 41783 +2, 41783 +3, 41783 +4, 41783 +5, 41783 +6, 41783 +7, 41783 +0, 42023 +1, 42007 +2, 42007 +3, 42007 +4, 42007 +5, 42007 +6, 42007 +7, 42007 +0, 42165 +1, 42137 +2, 42137 +3, 42153 +4, 42171 +5, 42159 +6, 42173 +7, 42165 +0, 42951 +1, 42935 +2, 42935 +3, 42935 +4, 42935 +5, 42935 +6, 42935 +7, 42935 +0, 43501 +1, 43485 +2, 43485 +3, 43485 +4, 43485 +5, 43485 +6, 43485 +7, 43485 +0, 44549 +1, 44533 +2, 44533 +3, 44533 +4, 44533 +5, 44533 +6, 44533 +7, 44533 +0, 45355 +1, 45339 +2, 45339 +3, 45339 +4, 45339 +5, 45339 +6, 45339 +7, 45339 +0, 78813 +1, 78891 +2, 78891 +3, 78891 +4, 78929 +5, 78891 +6, 78891 +7, 78891 +0, 78877 +1, 78955 +2, 78955 +3, 78955 +4, 78993 +5, 78955 +6, 78955 +7, 78955 +0, 78879 +1, 78957 +2, 78957 +3, 78957 +4, 78995 +5, 78957 +6, 78957 +7, 78957 +0, 78879 +1, 78957 +2, 78957 +3, 78957 +4, 78995 +5, 78957 +6, 78957 +7, 78957 +0, 78991 +1, 79069 +2, 79069 +3, 79069 +4, 79107 +5, 79069 +6, 79069 +7, 79069 +0, 97507 +1, 97595 +2, 97575 +3, 97603 +4, 97673 +5, 97595 +6, 97579 +7, 97607 +0, 98357 +1, 98397 +2, 98409 +3, 98451 +4, 98491 +5, 98437 +6, 98453 +7, 98453 +0, 99023 +1, 99101 +2, 99101 +3, 99101 +4, 99139 +5, 99101 +6, 99101 +7, 99101 +0, 99621 +1, 99693 +2, 99715 +3, 99715 +4, 99747 +5, 99711 +6, 99745 +7, 99703 +0, 122685 +1, 122853 +2, 122853 +3, 122829 +4, 122827 +5, 122829 +6, 122853 +7, 122829 +0, 122685 +1, 122853 +2, 122853 +3, 122829 +4, 122827 +5, 122829 +6, 122853 +7, 122829 +0, 122685 +1, 122853 +2, 122853 +3, 122829 +4, 122827 +5, 122829 +6, 122853 +7, 122829 +0, 122685 +1, 122853 +2, 122853 +3, 122829 +4, 122827 +5, 122829 +6, 122853 +7, 122829 +0, 121759 +1, 3176 +2, 1896 +3, 1896 +4, 1896 +5, 2152 +6, 121927 +7, 3432 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1/run.env b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..f31a7f9b43fcef2201cdfe80eb585c9c87d3919b --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1/run.env @@ -0,0 +1,70 @@ +backend=echo +config=G3 +family=tierA +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=1792 +num_experts=64 +topk=16 +experts_per_rank=8 +expert_params=22020096 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=65536 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.072265625 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1/train.log b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0dd6d8ad4393e021b220679d1e45dea46cee216e --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1/train.log @@ -0,0 +1,16605 @@ +[04:59:41] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G3 / echo (slots/rank=8 policy=matched) + E=64 K=16 H'=1792 m=8 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.072265625 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0830 04:59:43.330000 459339 torch/distributed/run.py:766] +W0830 04:59:43.330000 459339 torch/distributed/run.py:766] ***************************************** +W0830 04:59:43.330000 459339 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 04:59:43.330000 459339 torch/distributed/run.py:766] ***************************************** +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1792 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.072265625 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 16 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 246 is connected to is connected to is connected to 777 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777 + + +[Gloo] Rank 0 is connected to [Gloo] Rank 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 54 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to [Gloo] Rank is connected to 000 peer ranks. is connected to peer ranks. Expected number of connected peer ranks is : 0[Gloo] Rank Expected number of connected peer ranks is : 0 peer ranks. 00 +Expected number of connected peer ranks is : is connected to +[Gloo] Rank 000 + peer ranks. is connected to Expected number of connected peer ranks is : 00 peer ranks. +Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W830 05:00:01.470627633 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 05:00:01.470665202 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 05:00:01.470684868 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 05:00:01.470713378 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 05:00:01.470752610 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 05:00:01.470765205 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 05:00:01.470805849 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:00:01.622973096 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.722 seconds +time to initialize megatron (seconds): 7.138 +[after megatron is initialized] datetime: 2026-08-30 05:00:07 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3906080768 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42217472 elements, 42217472 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight +Params for bucket 3 (42213376 elements, 42213376 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_qkv.weight +Params for bucket 4 (42213376 elements, 42213376 padded size): + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 5 (42213376 elements, 42213376 padded size): + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.weight +Params for bucket 6 (42213376 elements, 42213376 padded size): + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.mlp.router.weight +Params for bucket 7 (42213376 elements, 42213376 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight +Params for bucket 8 (42213376 elements, 42213376 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight +Params for bucket 9 (42213376 elements, 42213376 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight +Params for bucket 10 (42213376 elements, 42213376 padded size): + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight +Params for bucket 11 (42213376 elements, 42213376 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight +Params for bucket 12 (42213376 elements, 42213376 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 13 (42213376 elements, 42213376 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight +Params for bucket 14 (42213376 elements, 42213376 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 15 (42213376 elements, 42213376 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 16 (42213376 elements, 42213376 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 17 (42213376 elements, 42213376 padded size): + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 64 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 +Params for bucket 6 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 +Params for bucket 10 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 +Params for bucket 17 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 +Params for bucket 18 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 +Params for bucket 24 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 28 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 +Params for bucket 30 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 +Params for bucket 31 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 34 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 +Params for bucket 35 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 +Params for bucket 42 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 43 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 +Params for bucket 47 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 48 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 49 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 55 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 56 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 61 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 +Params for bucket 62 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 +Params for bucket 63 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:00:07 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:00:07 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (464.17, 475.27) + train/valid/test-data-iterators-setup ..........: (66.85, 75.16) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 05:00:07 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[162, 3092, 875, 350, 1294, 649, 738, 1218, 788, 307, 1074, 482, 1187, 1028, 159, 416, 87, 555, 1541, 892, 908, 188, 2714, 1775, 843, 996, 1111, 1443, 1531, 337, 1836, 2115, 2817, 442, 333, 607, 276, 2134, 2028, 475, 2743, 288, 1602, 748, 1393, 1391, 935, 1328, 375, 441, 242, 1770, 1239, 598, 589, 105, 630, 306, 911, 1521, 1460, 705, 1200, 1213] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[54, 3033, 551, 196, 1764, 593, 1155, 790, 1025, 198, 1617, 332, 994, 751, 226, 330, 78, 916, 610, 624, 1034, 266, 3312, 2027, 580, 1280, 925, 1225, 928, 339, 1562, 1948, 3153, 816, 287, 246, 96, 2396, 1717, 518, 2397, 606, 1591, 1216, 1468, 1787, 1365, 1544, 195, 624, 275, 1992, 725, 491, 432, 73, 925, 290, 1011, 1922, 1581, 523, 645, 1366] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[77, 3019, 747, 263, 1640, 686, 1102, 1055, 1108, 184, 1157, 214, 709, 611, 264, 572, 169, 764, 680, 1116, 944, 178, 3032, 1833, 523, 772, 1057, 808, 1383, 464, 1764, 2125, 3104, 403, 252, 495, 142, 2099, 2287, 663, 3129, 863, 1630, 495, 849, 1291, 1269, 1441, 258, 469, 369, 2170, 1015, 456, 543, 169, 422, 703, 799, 1770, 1923, 651, 1183, 1204] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[186, 2772, 1185, 281, 1241, 539, 1129, 1110, 1329, 200, 1003, 187, 1078, 657, 204, 480, 317, 403, 1303, 955, 1052, 166, 3023, 1644, 524, 886, 1327, 648, 1559, 648, 1597, 1693, 3163, 512, 181, 583, 213, 2202, 2457, 603, 3297, 347, 1183, 1200, 1570, 1392, 1291, 741, 257, 297, 316, 2146, 1230, 459, 741, 93, 672, 515, 1237, 1256, 1311, 784, 1153, 808] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[90, 3010, 627, 240, 1266, 487, 833, 1247, 1199, 333, 1154, 185, 1014, 883, 327, 419, 130, 465, 352, 1330, 1101, 134, 2828, 1796, 326, 1157, 1184, 1121, 1515, 496, 1556, 2300, 3245, 522, 289, 1034, 182, 2576, 1882, 823, 2999, 432, 1242, 630, 1176, 1370, 1334, 1060, 190, 255, 248, 2332, 1494, 336, 337, 86, 675, 531, 1013, 1710, 1557, 424, 1529, 918] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[196, 3067, 638, 638, 1552, 621, 1102, 629, 1215, 145, 1974, 135, 1196, 1105, 216, 487, 138, 419, 1360, 802, 820, 502, 2978, 2015, 532, 880, 994, 1018, 1340, 290, 1644, 1491, 2993, 540, 268, 240, 251, 2830, 2070, 570, 2647, 229, 1356, 891, 1290, 1450, 1181, 1228, 349, 214, 450, 1723, 856, 946, 740, 71, 795, 572, 1322, 1690, 1600, 648, 360, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[59, 2861, 979, 480, 1431, 556, 811, 937, 977, 167, 1101, 642, 1240, 612, 368, 516, 147, 481, 1240, 989, 1110, 126, 3048, 2161, 497, 1020, 856, 1326, 1337, 407, 2132, 2454, 2843, 851, 263, 524, 212, 2085, 1649, 339, 2467, 910, 1104, 1118, 1348, 1677, 1133, 1301, 277, 227, 624, 2183, 834, 394, 596, 98, 641, 191, 1132, 1641, 1450, 377, 797, 1182] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[133, 2389, 807, 497, 1358, 248, 1064, 879, 1318, 157, 1554, 721, 830, 707, 166, 471, 126, 756, 579, 1176, 1117, 178, 2861, 1928, 565, 889, 938, 820, 1189, 148, 1843, 1950, 2981, 489, 424, 493, 547, 2577, 1704, 738, 2678, 368, 1360, 1003, 1637, 1241, 1553, 851, 305, 612, 330, 2604, 926, 426, 615, 96, 669, 658, 1368, 1609, 2088, 326, 629, 1269] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[90, 3010, 627, 240, 1266, 487, 833, 1247, 30, 0, 0, 0, 0, 0, 0, 0, 1199, 333, 1154, 185, 1014, 883, 327, 419, 1296, 1066, 471, 0, 0, 0, 0, 0, 130, 465, 352, 1330, 1101, 134, 2264, 1796, 0, 0, 0, 0, 0, 0, 0, 0, 326, 1157, 1184, 1121, 1515, 496, 1556, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 1751, 522, 289, 1034, 182, 2576, 1882, 823, 0, 0, 0, 0, 0, 0, 0, 0, 167, 432, 1242, 630, 1176, 1370, 1334, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 190, 255, 248, 2332, 1494, 336, 337, 86, 2832, 198, 0, 0, 0, 0, 0, 0, 675, 531, 1013, 1710, 1557, 424, 1529, 918, 63, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7830, 8347, 7572, 8589, 9059, 7411, 8308, 8420] avg=8192.0000 max=9059 max_rank=4 min=7411 min_rank=5 imbalance_ratio=1.105835 max_load_reduction=56477 max_load_reduction_pct=0.861771 moved_to_echo=33198 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[162, 3092, 875, 350, 1294, 649, 738, 1218, 29, 0, 0, 0, 0, 0, 0, 0, 788, 307, 1074, 482, 1187, 1028, 159, 416, 1125, 980, 452, 0, 0, 0, 0, 0, 87, 555, 1541, 892, 908, 188, 2172, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 843, 996, 1111, 1443, 1531, 337, 1836, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 1520, 442, 333, 607, 276, 2134, 2028, 475, 0, 0, 0, 0, 0, 0, 0, 0, 153, 288, 1602, 748, 1393, 1391, 935, 1328, 0, 0, 0, 0, 0, 0, 0, 0, 375, 441, 242, 1770, 1239, 598, 589, 105, 2590, 172, 0, 0, 0, 0, 0, 0, 630, 306, 911, 1521, 1460, 705, 1200, 1213, 61, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8407, 7998, 8118, 9232, 7815, 7838, 8121, 8007] avg=8192.0000 max=9232 max_rank=3 min=7815 min_rank=4 imbalance_ratio=1.126953 max_load_reduction=56304 max_load_reduction_pct=0.859131 moved_to_echo=31781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[77, 3019, 747, 263, 1640, 686, 1102, 1055, 32, 0, 0, 0, 0, 0, 0, 0, 1108, 184, 1157, 214, 709, 611, 264, 572, 1240, 985, 505, 0, 0, 0, 0, 0, 169, 764, 680, 1116, 944, 178, 2427, 1833, 0, 0, 0, 0, 0, 0, 0, 0, 523, 772, 1057, 808, 1383, 464, 1764, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 403, 252, 495, 142, 2099, 2287, 663, 0, 0, 0, 0, 0, 0, 0, 0, 174, 863, 1630, 495, 849, 1291, 1269, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 258, 469, 369, 2170, 1015, 456, 543, 169, 2955, 190, 0, 0, 0, 0, 0, 0, 422, 703, 799, 1770, 1923, 651, 1183, 1204, 68, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8621, 7549, 8111, 7911, 8015, 8012, 8594, 8723] avg=8192.0000 max=8723 max_rank=7 min=7549 min_rank=1 imbalance_ratio=1.064819 max_load_reduction=56813 max_load_reduction_pct=0.866898 moved_to_echo=33344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[54, 3033, 551, 196, 1764, 593, 1155, 790, 35, 0, 0, 0, 0, 0, 0, 0, 1025, 198, 1617, 332, 994, 751, 226, 330, 1259, 903, 552, 0, 0, 0, 0, 0, 78, 916, 610, 624, 1034, 266, 2651, 2027, 0, 0, 0, 0, 0, 0, 0, 0, 580, 1280, 925, 1225, 928, 339, 1562, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1701, 816, 287, 246, 96, 2396, 1717, 518, 0, 0, 0, 0, 0, 0, 0, 0, 133, 606, 1591, 1216, 1468, 1787, 1365, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 195, 624, 275, 1992, 725, 491, 432, 73, 2264, 193, 0, 0, 0, 0, 0, 0, 925, 290, 1011, 1922, 1581, 523, 645, 1366, 74, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8187, 8206, 7884, 7777, 9710, 7264, 8337] avg=8192.0000 max=9710 max_rank=5 min=7264 min_rank=6 imbalance_ratio=1.185303 max_load_reduction=55826 max_load_reduction_pct=0.851837 moved_to_echo=33088 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[196, 3067, 638, 638, 1552, 621, 1102, 629, 32, 0, 0, 0, 0, 0, 0, 0, 1215, 145, 1974, 135, 1196, 1105, 216, 487, 1195, 691, 496, 0, 0, 0, 0, 0, 138, 419, 1360, 802, 820, 502, 2383, 2015, 0, 0, 0, 0, 0, 0, 0, 0, 532, 880, 994, 1018, 1340, 290, 1644, 800, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 540, 268, 240, 251, 2830, 2070, 570, 0, 0, 0, 0, 0, 0, 0, 0, 147, 229, 1356, 891, 1290, 1450, 1181, 1228, 0, 0, 0, 0, 0, 0, 0, 0, 349, 214, 450, 1723, 856, 946, 740, 71, 2500, 183, 0, 0, 0, 0, 0, 0, 795, 572, 1322, 1690, 1600, 648, 360, 1027, 67, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8475, 8855, 8439, 7498, 8384, 7772, 8032, 8081] avg=8192.0000 max=8855 max_rank=1 min=7498 min_rank=3 imbalance_ratio=1.080933 max_load_reduction=56681 max_load_reduction_pct=0.864883 moved_to_echo=32269 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[186, 2772, 1185, 281, 1241, 539, 1129, 1110, 36, 0, 0, 0, 0, 0, 0, 0, 1329, 200, 1003, 187, 1078, 657, 204, 480, 1268, 789, 507, 0, 0, 0, 0, 0, 317, 403, 1303, 955, 1052, 166, 2407, 1644, 0, 0, 0, 0, 0, 0, 0, 0, 524, 886, 1327, 648, 1559, 648, 1597, 904, 0, 0, 0, 0, 0, 0, 0, 0, 1697, 512, 181, 583, 213, 2202, 2457, 603, 0, 0, 0, 0, 0, 0, 0, 0, 179, 347, 1183, 1200, 1570, 1392, 1291, 741, 0, 0, 0, 0, 0, 0, 0, 0, 257, 297, 316, 2146, 1230, 459, 741, 93, 3118, 198, 0, 0, 0, 0, 0, 0, 672, 515, 1237, 1256, 1311, 784, 1153, 808, 73, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8479, 7702, 8247, 8093, 8448, 7903, 8855, 7809] avg=8192.0000 max=8855 max_rank=6 min=7702 min_rank=1 imbalance_ratio=1.080933 max_load_reduction=56681 max_load_reduction_pct=0.864883 moved_to_echo=33015 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[59, 2861, 979, 480, 1431, 556, 811, 937, 33, 0, 0, 0, 0, 0, 0, 0, 977, 167, 1101, 642, 1240, 612, 368, 516, 1135, 1138, 508, 0, 0, 0, 0, 0, 147, 481, 1240, 989, 1110, 126, 2439, 2161, 0, 0, 0, 0, 0, 0, 0, 0, 497, 1020, 856, 1326, 1337, 407, 2132, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 1534, 851, 263, 524, 212, 2085, 1649, 339, 0, 0, 0, 0, 0, 0, 0, 0, 137, 910, 1104, 1118, 1348, 1677, 1133, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 277, 227, 624, 2183, 834, 394, 596, 98, 2330, 174, 0, 0, 0, 0, 0, 0, 641, 191, 1132, 1641, 1450, 377, 797, 1182, 68, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8404, 8693, 8891, 7457, 8728, 7737, 7479] avg=8192.0000 max=8891 max_rank=3 min=7457 min_rank=4 imbalance_ratio=1.085327 max_load_reduction=56645 max_load_reduction_pct=0.864334 moved_to_echo=31401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[133, 2389, 807, 497, 1358, 248, 1064, 879, 31, 0, 0, 0, 0, 0, 0, 0, 1318, 157, 1554, 721, 830, 707, 166, 471, 1190, 904, 476, 0, 0, 0, 0, 0, 126, 756, 579, 1176, 1117, 178, 2290, 1928, 0, 0, 0, 0, 0, 0, 0, 0, 565, 889, 938, 820, 1189, 148, 1843, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1609, 489, 424, 493, 547, 2577, 1704, 738, 0, 0, 0, 0, 0, 0, 0, 0, 149, 368, 1360, 1003, 1637, 1241, 1553, 851, 0, 0, 0, 0, 0, 0, 0, 0, 305, 612, 330, 2604, 926, 426, 615, 96, 2529, 182, 0, 0, 0, 0, 0, 0, 669, 658, 1368, 1609, 2088, 326, 629, 1269, 64, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7406, 8494, 8150, 7438, 8581, 8162, 8625, 8680] avg=8192.0000 max=8680 max_rank=7 min=7406 min_rank=0 imbalance_ratio=1.059570 max_load_reduction=56856 max_load_reduction_pct=0.867554 moved_to_echo=34048 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8960, 1792, 10752, 2944, 8320, 6400, 2048, 3712, 9728, 7552, 3968, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1024, 23296, 6528, 3072, 11648, 4480, 7936, 7936, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2304, 3200, 2944, 17024, 8320, 4224, 4608, 896, 21120, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5504, 3840, 8832, 13184, 13056, 4480, 7552, 9088, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1280, 4096, 11136, 7424, 10752, 11648, 10112, 9600, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4480, 7936, 8448, 8448, 10880, 3200, 13952, 8704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1280, 4864, 7680, 7936, 8192, 1792, 19072, 15232, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[13184, 4608, 2304, 4224, 1920, 18944, 15872, 4736, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[3963, 92, 298, 3564, 35, 95, 4074, 0, 0, 3158, 0, 16, 3, 5, 81, 0, 1, 4054, 2494, 909, 148, 1194, 0, 1791, 3126, 77, 0, 3159, 4034, 305, 0, 51, 2170, 453, 8, 3601, 2584, 2479, 0, 87, 623, 68, 26, 2679, 19, 37, 2234, 3, 3219, 518, 1540, 0, 12, 0, 1, 1, 25, 38, 519, 204, 0, 1518, 4092, 51] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3979, 36, 21, 2627, 396, 503, 3498, 7, 0, 2578, 0, 0, 0, 0, 0, 19, 2, 4032, 2743, 943, 307, 29, 0, 1292, 3088, 556, 2, 3417, 4076, 319, 0, 190, 2783, 228, 1, 4009, 2655, 103, 0, 5, 111, 1, 283, 3223, 160, 0, 178, 832, 3863, 3407, 4021, 9, 108, 0, 103, 0, 2, 0, 212, 171, 9, 293, 4086, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[3054, 1, 151, 3816, 5, 922, 4096, 0, 0, 3811, 0, 10, 0, 0, 8, 140, 0, 4026, 457, 409, 155, 2137, 3, 1004, 3127, 1, 596, 3275, 1646, 1, 0, 1018, 3324, 940, 38, 4089, 3131, 4043, 4, 3, 112, 16, 24, 3378, 97, 5, 31, 1081, 3530, 930, 2710, 26, 0, 0, 240, 0, 0, 8, 6, 183, 14, 178, 3491, 35] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1725, 42, 13, 1599, 2286, 228, 4093, 0, 0, 2708, 0, 90, 1, 1, 22, 17, 7, 3793, 2338, 6, 66, 1685, 0, 1318, 2131, 36, 1, 4064, 4063, 0, 0, 2, 375, 5, 0, 3095, 3099, 1393, 234, 234, 242, 0, 2506, 3257, 0, 2286, 8, 7, 3141, 23, 3753, 0, 104, 1, 0, 0, 0, 0, 1755, 0, 0, 2794, 1353, 3536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4075, 209, 306, 3404, 22, 437, 3773, 5, 78, 3665, 0, 222, 10, 540, 129, 0, 251, 4039, 506, 209, 437, 251, 0, 3307, 4017, 33, 0, 3224, 3030, 2650, 0, 0, 586, 1166, 0, 1752, 178, 1477, 0, 0, 245, 7, 58, 2481, 154, 103, 1955, 125, 3050, 1473, 3719, 95, 455, 0, 50, 0, 0, 268, 319, 197, 103, 871, 3817, 2003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[3889, 4, 190, 2315, 849, 222, 4038, 9, 57, 2904, 2, 0, 3, 2, 81, 193, 9, 4077, 1475, 1083, 262, 193, 0, 732, 3611, 178, 0, 4024, 4016, 0, 0, 675, 3599, 56, 22, 3934, 2565, 680, 0, 16, 86, 0, 603, 3901, 5, 73, 67, 0, 2566, 1057, 4059, 4, 1493, 2, 0, 50, 0, 10, 437, 22, 1, 1025, 4090, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[3813, 187, 280, 3366, 239, 353, 4062, 1, 10, 3817, 0, 0, 1, 1, 0, 502, 29, 4023, 866, 940, 1837, 29, 12, 201, 3269, 42, 9, 3736, 3797, 41, 5, 13, 1420, 1818, 1, 4091, 1241, 3074, 0, 195, 25, 240, 186, 1476, 65, 102, 148, 148, 3647, 620, 1995, 40, 2, 0, 0, 11, 0, 0, 329, 274, 56, 3879, 3901, 1071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3380, 5, 5, 3675, 1, 234, 3820, 1, 4, 2719, 137, 41, 0, 241, 5, 24, 107, 3940, 391, 1574, 1, 1785, 0, 411, 3517, 334, 16, 1011, 4088, 1884, 0, 14, 3587, 1941, 1, 1855, 4059, 2806, 0, 508, 2177, 429, 117, 161, 0, 10, 97, 8, 2629, 995, 3729, 470, 183, 0, 0, 3, 6, 38, 0, 3, 0, 764, 3547, 2048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3963, 92, 298, 3564, 35, 95, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3158, 0, 16, 3, 5, 81, 0, 3477, 1491, 0, 0, 0, 0, 0, 0, 1, 3448, 2494, 909, 148, 1194, 0, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 3126, 77, 0, 3159, 659, 305, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 2170, 453, 8, 568, 2584, 2479, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 623, 68, 26, 2679, 19, 37, 2234, 3, 1884, 1815, 0, 0, 0, 0, 0, 0, 3219, 518, 1540, 0, 12, 0, 1, 1, 238, 0, 0, 0, 0, 0, 0, 0, 25, 38, 519, 204, 0, 1518, 4092, 51, 1218, 368, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8644, 8231, 9985, 7377, 8349, 9388, 5529, 8033] avg=8192.0000 max=9985 max_rank=2 min=5529 min_rank=6 imbalance_ratio=1.218872 max_load_reduction=55551 max_load_reduction_pct=0.847641 moved_to_echo=31299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4075, 209, 306, 3404, 22, 437, 553, 5, 0, 0, 0, 0, 0, 0, 0, 0, 78, 3665, 0, 222, 10, 540, 129, 0, 3220, 1120, 0, 0, 0, 0, 0, 0, 251, 3436, 506, 209, 437, 251, 0, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 33, 0, 3224, 495, 2650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 1166, 0, 277, 178, 1477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 7, 58, 2481, 154, 103, 1955, 125, 1415, 883, 0, 0, 0, 0, 0, 0, 3050, 1473, 3719, 95, 455, 0, 50, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 268, 319, 197, 103, 871, 3817, 2003, 592, 366, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9011, 8984, 8397, 10419, 3684, 7426, 9079, 8536] avg=8192.0000 max=10419 max_rank=3 min=3684 min_rank=4 imbalance_ratio=1.271851 max_load_reduction=55117 max_load_reduction_pct=0.841019 moved_to_echo=28725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3054, 1, 151, 3816, 5, 922, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3811, 0, 10, 0, 0, 8, 140, 3496, 608, 0, 0, 0, 0, 0, 0, 0, 3425, 457, 409, 155, 2137, 3, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 3127, 1, 596, 3275, 270, 1, 0, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 3324, 940, 38, 645, 3131, 4043, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 112, 16, 24, 3378, 97, 5, 31, 1081, 768, 2061, 0, 0, 0, 0, 0, 0, 3530, 930, 2710, 26, 0, 0, 240, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 183, 14, 178, 3491, 35, 1383, 365, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8549, 8073, 7590, 8288, 12128, 7573, 7672, 5663] avg=8192.0000 max=12128 max_rank=4 min=5663 min_rank=7 imbalance_ratio=1.480469 max_load_reduction=53408 max_load_reduction_pct=0.814941 moved_to_echo=33036 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3889, 4, 190, 2315, 849, 222, 591, 9, 0, 0, 0, 0, 0, 0, 0, 0, 57, 2904, 2, 0, 3, 2, 81, 193, 3447, 1484, 0, 0, 0, 0, 0, 0, 9, 3468, 1475, 1083, 262, 193, 0, 732, 0, 0, 0, 0, 0, 0, 0, 0, 3611, 178, 0, 4024, 656, 0, 0, 675, 0, 0, 0, 0, 0, 0, 0, 0, 3599, 56, 22, 621, 2565, 680, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 603, 3901, 5, 73, 67, 0, 1876, 1983, 0, 0, 0, 0, 0, 0, 2566, 1057, 4059, 4, 1493, 2, 0, 50, 239, 0, 0, 0, 0, 0, 0, 0, 0, 10, 437, 22, 1, 1025, 4090, 20, 1330, 370, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8069, 8173, 7222, 9144, 7559, 8594, 9470, 7305] avg=8192.0000 max=9470 max_rank=6 min=7222 min_rank=2 imbalance_ratio=1.156006 max_load_reduction=56066 max_load_reduction_pct=0.855499 moved_to_echo=32928 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1725, 42, 13, 1599, 2286, 228, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2708, 0, 90, 1, 1, 22, 17, 3498, 1505, 0, 0, 0, 0, 0, 0, 7, 3219, 2338, 6, 66, 1685, 0, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 2131, 36, 1, 4064, 656, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 375, 5, 0, 479, 3099, 1393, 234, 234, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 2506, 3257, 0, 2286, 8, 7, 1902, 1566, 0, 0, 0, 0, 0, 0, 3141, 23, 3753, 0, 104, 1, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1755, 0, 0, 2794, 1353, 3536, 1050, 348, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6488, 7842, 8639, 6890, 5819, 11774, 7248, 10836] avg=8192.0000 max=11774 max_rank=5 min=5819 min_rank=4 imbalance_ratio=1.437256 max_load_reduction=53762 max_load_reduction_pct=0.820343 moved_to_echo=35677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3380, 5, 5, 3675, 1, 234, 559, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2719, 137, 41, 0, 241, 5, 24, 3261, 1511, 0, 0, 0, 0, 0, 0, 107, 3352, 391, 1574, 1, 1785, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 3517, 334, 16, 1011, 668, 1884, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3587, 1941, 1, 293, 4059, 2806, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 2177, 429, 117, 161, 0, 10, 97, 8, 1909, 935, 0, 0, 0, 0, 0, 0, 2629, 995, 3729, 470, 183, 0, 0, 3, 231, 0, 0, 0, 0, 0, 0, 0, 6, 38, 0, 3, 0, 764, 3547, 2048, 627, 357, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7860, 7943, 7621, 7444, 13195, 5843, 8240, 7390] avg=8192.0000 max=13195 max_rank=4 min=5843 min_rank=5 imbalance_ratio=1.610718 max_load_reduction=52341 max_load_reduction_pct=0.798660 moved_to_echo=34668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3813, 187, 280, 3366, 239, 353, 595, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3817, 0, 0, 1, 1, 0, 502, 3467, 1403, 0, 0, 0, 0, 0, 0, 29, 3422, 866, 940, 1837, 29, 12, 201, 0, 0, 0, 0, 0, 0, 0, 0, 3269, 42, 9, 3736, 621, 41, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 1818, 1, 646, 1241, 3074, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 25, 240, 186, 1476, 65, 102, 148, 148, 1773, 2062, 0, 0, 0, 0, 0, 0, 3647, 620, 1995, 40, 2, 0, 0, 11, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 274, 56, 3879, 3901, 1071, 1383, 365, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8834, 9201, 7336, 7736, 8395, 6225, 6551, 11258] avg=8192.0000 max=11258 max_rank=7 min=6225 min_rank=5 imbalance_ratio=1.374268 max_load_reduction=54278 max_load_reduction_pct=0.828217 moved_to_echo=32429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3979, 36, 21, 2627, 396, 503, 512, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2578, 0, 0, 0, 0, 0, 19, 2986, 1506, 0, 0, 0, 0, 0, 0, 2, 3430, 2743, 943, 307, 29, 0, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 3088, 556, 2, 3417, 666, 319, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 2783, 228, 1, 632, 2655, 103, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 111, 1, 283, 3223, 160, 0, 178, 832, 1904, 2021, 0, 0, 0, 0, 0, 0, 3863, 3407, 4021, 9, 108, 0, 103, 0, 236, 0, 0, 0, 0, 0, 0, 0, 2, 0, 212, 171, 9, 293, 4086, 20, 1356, 366, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8081, 7089, 8746, 8238, 6407, 8713, 11747, 6515] avg=8192.0000 max=11747 max_rank=6 min=6407 min_rank=4 imbalance_ratio=1.433960 max_load_reduction=53789 max_load_reduction_pct=0.820755 moved_to_echo=33382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[17920, 6656, 128, 4224, 19584, 16128, 256, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[27904, 640, 1280, 24448, 3840, 3072, 4608, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[512, 27264, 11392, 6144, 3328, 7424, 128, 10112, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3712, 768, 3840, 20608, 512, 2688, 4736, 2304, 13440, 13440, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[25728, 9088, 25600, 768, 2432, 128, 512, 128, 1920, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 25472, 256, 384, 128, 896, 384, 896, 26880, 10752, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25984, 1280, 640, 25984, 4736, 5248, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 384, 3584, 1152, 256, 11392, 28416, 8832, 8960, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1286, 7, 4088, 235, 99, 17, 4001, 4090, 3325, 5, 176, 0, 1, 0, 293, 0, 56, 4, 19, 8, 6, 249, 0, 4092, 0, 0, 0, 8, 770, 3305, 4089, 0, 1, 77, 4064, 704, 13, 4077, 1, 0, 2681, 3957, 57, 3935, 1, 1, 3982, 57, 2, 2, 60, 277, 2158, 23, 90, 0, 1, 3680, 367, 1022, 4017, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4089, 0, 3726, 1, 100, 2, 0, 3468, 3171, 2, 19, 1, 8, 1, 0, 431, 3134, 194, 0, 603, 1, 4069, 0, 4074, 36, 0, 35, 0, 0, 16, 4088, 1, 155, 4, 4064, 1, 0, 3800, 0, 74, 4094, 367, 0, 4096, 0, 0, 4007, 2725, 109, 147, 231, 101, 1173, 7, 351, 33, 0, 2109, 0, 3273, 3345, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[42, 7, 543, 163, 834, 1, 3679, 4095, 3984, 6, 885, 20, 0, 0, 108, 0, 16, 5, 0, 3407, 0, 396, 0, 3041, 524, 1194, 0, 470, 1013, 20, 4056, 80, 3998, 0, 3755, 3, 158, 1508, 0, 0, 0, 2667, 25, 3920, 0, 0, 3381, 1337, 0, 0, 6, 4058, 120, 3273, 0, 511, 0, 4080, 39, 897, 3159, 0, 52, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4074, 0, 4096, 204, 136, 39, 180, 3449, 14, 48, 3079, 0, 0, 615, 0, 70, 99, 0, 258, 2719, 6, 4069, 0, 4075, 0, 51, 1, 0, 1445, 3870, 4076, 332, 357, 5, 4065, 3, 2, 2931, 0, 44, 2397, 973, 52, 4076, 1, 0, 0, 283, 14, 0, 0, 49, 712, 3, 1, 0, 0, 4030, 0, 3165, 3619, 13, 50, 1686] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3745, 0, 2498, 2, 28, 7, 2950, 4094, 62, 0, 3, 0, 5, 1, 22, 5, 1, 0, 792, 136, 2, 3728, 6, 3961, 126, 13, 3960, 77, 0, 1161, 1637, 2387, 0, 303, 4009, 3, 1, 83, 0, 2, 380, 4056, 713, 3641, 3, 0, 3352, 3270, 0, 0, 9, 50, 3653, 2, 22, 31, 0, 4093, 61, 2229, 4079, 51, 31, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3378, 0, 3687, 37, 3149, 0, 3166, 4062, 3964, 5, 220, 999, 0, 2, 583, 85, 5, 0, 1935, 4, 8, 925, 8, 787, 0, 1, 375, 2340, 1192, 0, 3808, 41, 765, 0, 3996, 350, 0, 0, 0, 3, 1, 3342, 10, 4004, 74, 1, 4096, 2396, 0, 19, 25, 199, 788, 147, 0, 0, 0, 3110, 1067, 2472, 3522, 25, 54, 304] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3979, 1630, 2519, 103, 0, 0, 546, 3241, 343, 54, 1890, 0, 9, 60, 0, 49, 2, 709, 851, 606, 3, 58, 1, 1161, 0, 0, 0, 3586, 16, 136, 3707, 1162, 0, 3745, 3529, 16, 6, 524, 4, 129, 3983, 30, 0, 3440, 0, 0, 3118, 3995, 69, 0, 0, 25, 4089, 0, 50, 72, 0, 4087, 0, 4085, 4096, 8, 15, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4002, 2814, 4096, 2, 18, 0, 37, 4096, 4096, 0, 0, 15, 0, 628, 0, 3961, 0, 11, 111, 1337, 85, 4051, 630, 1554, 9, 0, 1, 1, 1792, 361, 515, 0, 159, 849, 3025, 43, 285, 301, 0, 0, 3634, 0, 0, 3727, 0, 1, 2178, 1195, 0, 0, 0, 705, 3889, 0, 369, 0, 0, 4021, 0, 2, 4050, 0, 0, 2880] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 2472, 1, 100, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3171, 2, 19, 1, 8, 1, 0, 431, 2678, 961, 0, 0, 0, 0, 0, 0, 3134, 194, 0, 603, 1, 4069, 0, 4074, 1219, 19, 0, 0, 0, 0, 0, 0, 36, 0, 35, 0, 0, 16, 4088, 1, 1129, 0, 0, 0, 0, 0, 0, 0, 155, 4, 4064, 1, 0, 3800, 0, 74, 987, 106, 0, 0, 0, 0, 0, 0, 4094, 367, 0, 0, 0, 0, 2788, 2725, 0, 0, 0, 0, 0, 0, 0, 0, 109, 147, 231, 101, 1173, 7, 351, 33, 4096, 790, 0, 0, 0, 0, 0, 0, 0, 2109, 0, 3273, 1397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6664, 7272, 13313, 5305, 9191, 9974, 7038, 6779] avg=8192.0000 max=13313 max_rank=2 min=5305 min_rank=3 imbalance_ratio=1.625122 max_load_reduction=52223 max_load_reduction_pct=0.796860 moved_to_echo=32982 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[42, 7, 362, 163, 834, 1, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 6, 885, 20, 0, 0, 108, 0, 3161, 908, 0, 0, 0, 0, 0, 0, 16, 5, 0, 3407, 0, 396, 0, 3041, 1028, 2, 0, 0, 0, 0, 0, 0, 524, 1194, 0, 470, 1013, 20, 4056, 80, 164, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 3755, 3, 158, 1508, 0, 0, 932, 15, 0, 0, 0, 0, 0, 0, 0, 2667, 25, 0, 0, 0, 2353, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4058, 120, 3273, 0, 511, 3920, 934, 0, 0, 0, 0, 0, 0, 0, 4080, 39, 897, 1319, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5088, 9072, 7895, 7521, 10369, 6382, 12822, 6387] avg=8192.0000 max=12822 max_rank=6 min=5088 min_rank=0 imbalance_ratio=1.565186 max_load_reduction=52714 max_load_reduction_pct=0.804352 moved_to_echo=35960 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1286, 7, 2712, 235, 99, 17, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3325, 5, 176, 0, 1, 0, 293, 0, 3157, 1154, 0, 0, 0, 0, 0, 0, 56, 4, 19, 8, 6, 249, 0, 4092, 1211, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 770, 3305, 4089, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 1, 77, 4064, 704, 13, 4077, 1, 0, 1185, 117, 0, 0, 0, 0, 0, 0, 2681, 3957, 57, 0, 1, 1, 2771, 57, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 60, 277, 2158, 23, 90, 0, 3935, 933, 0, 0, 0, 0, 0, 0, 1, 3680, 367, 1022, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8357, 8111, 5666, 9410, 10239, 9525, 7480, 6748] avg=8192.0000 max=10239 max_rank=4 min=5666 min_rank=2 imbalance_ratio=1.249878 max_load_reduction=55297 max_load_reduction_pct=0.843765 moved_to_echo=33992 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3378, 0, 2446, 37, 3149, 0, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 5, 220, 999, 0, 2, 583, 85, 3136, 1012, 0, 0, 0, 0, 0, 0, 5, 0, 1935, 4, 8, 925, 8, 787, 1246, 19, 0, 0, 0, 0, 0, 0, 0, 1, 375, 2340, 1192, 0, 3808, 41, 1117, 0, 0, 0, 0, 0, 0, 0, 765, 0, 3996, 350, 0, 0, 0, 3, 1039, 105, 0, 0, 0, 0, 0, 0, 1, 3342, 10, 0, 74, 1, 2850, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 25, 199, 788, 147, 0, 0, 4004, 926, 0, 0, 0, 0, 0, 0, 0, 3110, 1067, 2472, 1471, 25, 54, 304, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12176, 10006, 4937, 8874, 6258, 8674, 6108, 8503] avg=8192.0000 max=12176 max_rank=0 min=4937 min_rank=2 imbalance_ratio=1.486328 max_load_reduction=53360 max_load_reduction_pct=0.814209 moved_to_echo=29543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3745, 0, 1659, 2, 28, 7, 2950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 3, 0, 5, 1, 22, 5, 3161, 1172, 0, 0, 0, 0, 0, 0, 1, 0, 792, 136, 2, 3728, 6, 3961, 1019, 12, 0, 0, 0, 0, 0, 0, 126, 13, 3960, 77, 0, 1161, 1637, 2387, 756, 0, 0, 0, 0, 0, 0, 0, 0, 303, 4009, 3, 1, 83, 0, 2, 1203, 71, 0, 0, 0, 0, 0, 0, 380, 4056, 713, 0, 3, 0, 2333, 3270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 50, 3653, 2, 22, 31, 3641, 933, 0, 0, 0, 0, 0, 0, 0, 4093, 61, 2229, 1704, 51, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8391, 4431, 9657, 10117, 5675, 10755, 8341, 8169] avg=8192.0000 max=10755 max_rank=5 min=4431 min_rank=1 imbalance_ratio=1.312866 max_load_reduction=54781 max_load_reduction_pct=0.835892 moved_to_echo=32940 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4074, 0, 2717, 204, 136, 39, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 48, 3079, 0, 0, 615, 0, 70, 2663, 1040, 0, 0, 0, 0, 0, 0, 99, 0, 258, 2719, 6, 4069, 0, 4075, 0, 21, 0, 0, 0, 0, 0, 0, 0, 51, 1, 0, 1445, 3870, 4076, 332, 1241, 0, 0, 0, 0, 0, 0, 0, 357, 5, 4065, 3, 2, 2931, 0, 44, 1067, 117, 0, 0, 0, 0, 0, 0, 2397, 973, 52, 0, 1, 0, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 49, 712, 3, 1, 0, 4076, 786, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 3165, 1512, 13, 50, 1686, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7350, 7529, 11247, 11016, 8591, 3706, 5641, 10456] avg=8192.0000 max=11247 max_rank=2 min=3706 min_rank=5 imbalance_ratio=1.372925 max_load_reduction=54289 max_load_reduction_pct=0.828384 moved_to_echo=28394 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3979, 1630, 1662, 103, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 54, 1890, 0, 9, 60, 0, 49, 2503, 1180, 0, 0, 0, 0, 0, 0, 2, 709, 851, 606, 3, 58, 1, 1161, 952, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3586, 16, 136, 3707, 1162, 766, 0, 0, 0, 0, 0, 0, 0, 0, 3745, 3529, 16, 6, 524, 4, 129, 1211, 75, 0, 0, 0, 0, 0, 0, 3983, 30, 0, 0, 0, 0, 2166, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 25, 4089, 0, 50, 72, 3440, 738, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4085, 1705, 8, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7920, 6088, 4359, 9373, 9239, 10174, 8483, 9900] avg=8192.0000 max=10174 max_rank=5 min=4359 min_rank=2 imbalance_ratio=1.241943 max_load_reduction=55362 max_load_reduction_pct=0.844757 moved_to_echo=37796 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4002, 2814, 2717, 2, 18, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 628, 0, 3961, 3163, 1164, 0, 0, 0, 0, 0, 0, 0, 11, 111, 1337, 85, 4051, 630, 1554, 662, 21, 0, 0, 0, 0, 0, 0, 9, 0, 1, 1, 1792, 361, 515, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 159, 849, 3025, 43, 285, 301, 0, 0, 1195, 117, 0, 0, 0, 0, 0, 0, 3634, 0, 0, 0, 0, 1, 1516, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 705, 3889, 0, 369, 0, 3727, 933, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 2, 1691, 0, 0, 2880, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9590, 13027, 8462, 3920, 5974, 6346, 9623, 8594] avg=8192.0000 max=13027 max_rank=1 min=3920 min_rank=3 imbalance_ratio=1.590210 max_load_reduction=52509 max_load_reduction_pct=0.801224 moved_to_echo=30537 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[17280, 15488, 896, 0, 128, 128, 16896, 15360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 256, 384, 5504, 16640, 3456, 896, 768, 30848, 7040, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3328, 1024, 3968, 8832, 128, 17664, 768, 22784, 7424, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[19072, 128, 6272, 1152, 128, 1408, 1024, 4608, 23680, 8704, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[768, 1280, 4480, 6528, 6272, 8960, 25984, 4096, 7680, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[24704, 4480, 16768, 768, 4480, 128, 14592, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 29312, 1536, 17152, 12544, 128, 256, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5504, 4992, 30592, 1152, 512, 13312, 128, 256, 8832, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 3690, 3896, 1, 1, 4, 3890, 4, 225, 2, 48, 2, 0, 46, 3645, 26, 4078, 12, 289, 320, 1, 437, 3539, 3, 0, 0, 4094, 0, 1, 3, 0, 27, 1264, 0, 1426, 117, 0, 4049, 560, 361, 953, 0, 4093, 4086, 0, 4066, 0, 607, 241, 1, 1190, 3, 1765, 3875, 423, 3897, 7, 0, 0, 404, 0, 3863, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[2315, 0, 3582, 3093, 10, 0, 18, 4049, 1097, 3477, 4000, 71, 3, 0, 275, 13, 13, 3764, 2, 0, 0, 0, 15, 125, 2, 0, 51, 3973, 0, 0, 0, 31, 16, 0, 0, 3897, 288, 0, 2542, 109, 3963, 0, 887, 4095, 3467, 0, 2032, 0, 4093, 27, 245, 12, 18, 48, 1248, 288, 4095, 0, 0, 0, 0, 56, 4086, 45] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4045, 21, 1378, 333, 0, 900, 4049, 0, 75, 47, 0, 1, 21, 2638, 31, 3527, 5, 0, 89, 0, 24, 34, 0, 0, 78, 4092, 22, 41, 0, 194, 1, 7, 17, 4016, 0, 0, 3998, 1, 4094, 0, 3283, 4089, 4096, 5, 4036, 51, 3958, 3, 0, 7, 20, 3, 4088, 204, 4092, 0, 2, 0, 0, 1, 3819, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[62, 0, 4096, 28, 0, 1, 8, 4071, 3, 10, 2945, 973, 15, 68, 128, 4085, 88, 4086, 9, 3505, 0, 0, 3605, 2434, 1, 0, 0, 3286, 48, 0, 1, 0, 3549, 0, 2323, 0, 0, 0, 1257, 12, 4084, 0, 0, 727, 0, 8, 1, 8, 2646, 31, 3455, 4033, 1938, 133, 648, 4, 3729, 329, 44, 425, 116, 0, 134, 2346] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[725, 118, 3747, 5, 15, 47, 388, 0, 0, 3005, 57, 0, 262, 0, 6, 0, 17, 4062, 29, 0, 12, 2, 0, 3551, 0, 113, 1637, 4095, 1, 0, 0, 3827, 683, 3836, 94, 1145, 1, 0, 3835, 0, 1016, 267, 0, 3474, 3810, 0, 1308, 85, 0, 0, 0, 2, 47, 17, 2877, 3932, 3991, 4067, 0, 1082, 1, 0, 4060, 185] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[11, 0, 4037, 5, 11, 0, 0, 3770, 0, 3421, 1013, 0, 157, 0, 0, 4058, 8, 2937, 0, 214, 0, 622, 0, 2248, 0, 178, 1, 4050, 0, 1, 6, 2135, 20, 27, 0, 4096, 0, 0, 280, 0, 3283, 330, 135, 1181, 36, 11, 3357, 0, 4083, 54, 0, 680, 0, 0, 2777, 6, 4096, 3518, 137, 0, 1857, 0, 3453, 3236] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1745, 0, 11, 351, 336, 3625, 0, 0, 306, 13, 0, 318, 46, 1365, 3837, 662, 208, 0, 0, 5, 704, 0, 0, 80, 3827, 3, 0, 5, 4, 163, 3904, 0, 3969, 215, 2, 4088, 0, 3711, 0, 3928, 3912, 5, 0, 25, 106, 3556, 0, 527, 3, 0, 18, 4055, 0, 4087, 3146, 0, 5, 51, 443, 4070, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1952, 1, 8, 29, 0, 0, 649, 0, 26, 1468, 249, 0, 1, 3408, 3, 4060, 338, 0, 244, 3, 174, 0, 1136, 12, 0, 4096, 0, 0, 0, 15, 0, 0, 23, 3961, 11, 185, 4022, 134, 4092, 13, 4046, 4089, 3, 0, 2868, 219, 0, 0, 0, 1197, 3953, 0, 4095, 0, 4096, 3743, 2484, 0, 932, 0, 3445, 53] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[62, 0, 4096, 28, 0, 1, 8, 4071, 730, 6, 0, 0, 0, 0, 0, 0, 3, 10, 2945, 973, 15, 68, 128, 4085, 1652, 35, 0, 0, 0, 0, 0, 0, 88, 4086, 9, 3505, 0, 0, 3605, 2434, 927, 364, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3286, 48, 0, 1, 0, 692, 0, 0, 0, 0, 0, 0, 0, 3549, 0, 2323, 0, 0, 0, 1257, 12, 37, 0, 0, 0, 0, 0, 0, 0, 2990, 0, 0, 0, 0, 8, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2646, 31, 3455, 4033, 1938, 133, 605, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 329, 44, 425, 116, 0, 134, 2346, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9002, 9914, 15018, 4028, 7178, 3007, 12845, 4544] avg=8192.0000 max=15018 max_rank=2 min=3007 min_rank=5 imbalance_ratio=1.833252 max_load_reduction=50518 max_load_reduction_pct=0.770844 moved_to_echo=27574 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4045, 21, 1378, 333, 0, 900, 732, 40, 0, 0, 0, 0, 0, 0, 4049, 0, 75, 47, 0, 1, 21, 2638, 1813, 201, 0, 0, 0, 0, 0, 0, 31, 3527, 5, 0, 89, 0, 24, 34, 1017, 365, 0, 0, 0, 0, 0, 0, 0, 0, 78, 4092, 22, 41, 0, 194, 3888, 0, 0, 0, 0, 0, 0, 0, 1, 7, 17, 4016, 0, 0, 3998, 1, 235, 0, 0, 0, 0, 0, 0, 0, 2997, 0, 3283, 0, 4096, 5, 4036, 51, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 3, 0, 7, 20, 3, 3813, 204, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 0, 2, 0, 0, 1, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7449, 8845, 5092, 8315, 8275, 14468, 8008, 5084] avg=8192.0000 max=14468 max_rank=5 min=5084 min_rank=7 imbalance_ratio=1.766113 max_load_reduction=51068 max_load_reduction_pct=0.779236 moved_to_echo=35835 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[725, 118, 3747, 5, 15, 47, 388, 0, 181, 28, 0, 0, 0, 0, 0, 0, 0, 3005, 57, 0, 262, 0, 6, 0, 1768, 172, 0, 0, 0, 0, 0, 0, 17, 4062, 29, 0, 12, 2, 0, 3551, 992, 90, 0, 0, 0, 0, 0, 0, 0, 113, 1637, 4095, 1, 0, 0, 3827, 3302, 0, 0, 0, 0, 0, 0, 0, 683, 3836, 94, 1145, 1, 0, 3835, 0, 166, 0, 0, 0, 0, 0, 0, 0, 745, 267, 0, 0, 3810, 0, 1308, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 47, 17, 2683, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 4067, 0, 1082, 1, 0, 4060, 185, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5254, 5270, 8755, 12975, 9760, 6215, 6681, 10626] avg=8192.0000 max=12975 max_rank=3 min=5254 min_rank=0 imbalance_ratio=1.583862 max_load_reduction=52561 max_load_reduction_pct=0.802017 moved_to_echo=33282 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1952, 1, 8, 29, 0, 0, 732, 40, 0, 0, 0, 0, 0, 0, 649, 0, 26, 1468, 249, 0, 1, 3408, 1814, 201, 0, 0, 0, 0, 0, 0, 3, 4060, 338, 0, 244, 3, 174, 0, 1018, 365, 0, 0, 0, 0, 0, 0, 1136, 12, 0, 4096, 0, 0, 0, 15, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3961, 11, 185, 4022, 134, 236, 0, 0, 0, 0, 0, 0, 0, 2995, 13, 4046, 0, 3, 0, 2868, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 3953, 0, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 3743, 2484, 0, 932, 0, 3445, 53, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2762, 7816, 6205, 9147, 8572, 10144, 8969, 11921] avg=8192.0000 max=11921 max_rank=7 min=2762 min_rank=0 imbalance_ratio=1.455200 max_load_reduction=53615 max_load_reduction_pct=0.818100 moved_to_echo=39606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2315, 0, 3582, 3093, 10, 0, 18, 4049, 709, 12, 0, 0, 0, 0, 0, 0, 1097, 3477, 4000, 71, 3, 0, 275, 13, 1814, 203, 0, 0, 0, 0, 0, 0, 13, 3764, 2, 0, 0, 0, 15, 125, 1018, 353, 0, 0, 0, 0, 0, 0, 2, 0, 51, 3973, 0, 0, 0, 31, 3892, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 3897, 288, 0, 2542, 109, 72, 0, 0, 0, 0, 0, 0, 0, 2901, 0, 887, 0, 3467, 0, 2032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 27, 245, 12, 18, 48, 1164, 288, 0, 0, 0, 0, 0, 0, 0, 0, 1263, 0, 0, 0, 0, 56, 4086, 45, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13788, 10953, 5290, 7949, 6924, 9287, 5895, 5450] avg=8192.0000 max=13788 max_rank=0 min=5290 min_rank=2 imbalance_ratio=1.683105 max_load_reduction=51748 max_load_reduction_pct=0.789612 moved_to_echo=27556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3690, 3896, 1, 1, 4, 3890, 64, 37, 0, 0, 0, 0, 0, 0, 4, 225, 2, 48, 2, 0, 46, 3645, 1726, 203, 0, 0, 0, 0, 0, 0, 26, 4078, 12, 289, 320, 1, 437, 3539, 968, 32, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4094, 0, 1, 3, 0, 3890, 0, 0, 0, 0, 0, 0, 0, 27, 1264, 0, 1426, 117, 0, 4049, 560, 223, 0, 0, 0, 0, 0, 0, 0, 265, 953, 0, 0, 4086, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 241, 1, 1190, 3, 1765, 3615, 423, 0, 0, 0, 0, 0, 0, 0, 0, 1203, 7, 0, 0, 404, 0, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11584, 5901, 9702, 7991, 7666, 9370, 7845, 5477] avg=8192.0000 max=11584 max_rank=0 min=5477 min_rank=7 imbalance_ratio=1.414062 max_load_reduction=53952 max_load_reduction_pct=0.823242 moved_to_echo=30358 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 4037, 5, 11, 0, 0, 3770, 591, 30, 0, 0, 0, 0, 0, 0, 0, 3421, 1013, 0, 157, 0, 0, 4058, 1818, 59, 0, 0, 0, 0, 0, 0, 8, 2937, 0, 214, 0, 622, 0, 2248, 1021, 296, 0, 0, 0, 0, 0, 0, 0, 178, 1, 4050, 0, 1, 6, 2135, 1122, 0, 0, 0, 0, 0, 0, 0, 20, 27, 0, 4096, 0, 0, 280, 0, 163, 0, 0, 0, 0, 0, 0, 0, 2396, 330, 135, 0, 36, 11, 3357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 54, 0, 680, 0, 0, 2584, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 3518, 137, 0, 1857, 0, 3453, 3236, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8455, 10526, 7346, 7493, 4586, 6265, 7407, 13458] avg=8192.0000 max=13458 max_rank=7 min=4586 min_rank=4 imbalance_ratio=1.642822 max_load_reduction=52078 max_load_reduction_pct=0.794647 moved_to_echo=31716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 1745, 0, 11, 351, 336, 664, 39, 0, 0, 0, 0, 0, 0, 3625, 0, 0, 306, 13, 0, 318, 46, 1810, 193, 0, 0, 0, 0, 0, 0, 1365, 3837, 662, 208, 0, 0, 5, 704, 1016, 331, 0, 0, 0, 0, 0, 0, 0, 0, 80, 3827, 3, 0, 5, 4, 3719, 0, 0, 0, 0, 0, 0, 0, 163, 3904, 0, 3969, 215, 2, 4088, 0, 234, 0, 0, 0, 0, 0, 0, 0, 2716, 0, 3928, 0, 5, 0, 25, 106, 0, 0, 0, 0, 0, 0, 0, 0, 3556, 0, 527, 3, 0, 18, 3782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 3146, 0, 5, 51, 443, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7242, 6311, 8128, 7638, 12575, 6780, 7886, 8976] avg=8192.0000 max=12575 max_rank=4 min=6311 min_rank=1 imbalance_ratio=1.535034 max_load_reduction=52961 max_load_reduction_pct=0.808121 moved_to_echo=36217 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[9984, 14848, 2688, 1536, 3456, 512, 27008, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[18048, 1664, 12288, 0, 15616, 128, 17792, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3200, 128, 29312, 8832, 1536, 512, 896, 17024, 4480, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[18944, 384, 4352, 7168, 6016, 2048, 22144, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1664, 30464, 1152, 4224, 768, 640, 4352, 12672, 8064, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4480, 9088, 2560, 22528, 640, 256, 24192, 896, 1408, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1152, 384, 1920, 31616, 128, 128, 128, 6272, 24448, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9472, 10240, 8192, 2944, 768, 128, 896, 17920, 14336, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[227, 3, 0, 9, 4029, 6, 0, 3882, 0, 4017, 21, 3553, 369, 0, 19, 1658, 2914, 0, 0, 3719, 335, 0, 0, 0, 82, 3925, 0, 7, 0, 31, 2, 1368, 4048, 873, 25, 0, 0, 2572, 0, 24, 0, 4024, 3755, 0, 104, 19, 5, 3887, 4, 3699, 243, 3048, 0, 3941, 0, 0, 301, 61, 0, 0, 166, 455, 10, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4094, 0, 3700, 0, 4039, 1511, 0, 4094, 0, 4010, 2492, 3238, 0, 126, 197, 0, 2, 0, 0, 628, 53, 0, 6, 129, 200, 4084, 0, 37, 10, 4096, 18, 3956, 3838, 30, 0, 5, 1, 0, 4021, 0, 0, 0, 10, 0, 4066, 0, 0, 79, 0, 3313, 0, 0, 0, 0, 236, 0, 3993, 4, 3, 54, 3, 1107, 18, 4035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4025, 0, 2552, 87, 3840, 332, 0, 4095, 0, 3784, 1, 4089, 11, 0, 0, 0, 3932, 0, 0, 0, 9, 867, 0, 0, 3173, 4049, 0, 0, 0, 16, 2, 4, 1321, 1, 382, 0, 0, 39, 1, 0, 2523, 0, 0, 4091, 119, 3955, 218, 0, 4020, 3594, 0, 0, 21, 0, 0, 0, 2816, 141, 4054, 59, 1, 0, 12, 3300] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[676, 0, 110, 0, 4076, 994, 0, 3210, 0, 3793, 57, 2951, 0, 0, 0, 1, 985, 0, 0, 0, 321, 0, 398, 0, 525, 4075, 0, 63, 3460, 3945, 51, 65, 14, 3, 5, 0, 52, 3, 3780, 0, 4086, 1537, 1, 3084, 1138, 1, 13, 3996, 253, 3816, 0, 7, 8, 4056, 3894, 30, 3025, 0, 9, 1057, 98, 25, 6, 1783] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 3495, 3393, 64, 4095, 0, 0, 3809, 458, 3880, 0, 2992, 53, 11, 0, 737, 4088, 0, 0, 0, 50, 786, 84, 2, 0, 3900, 0, 1, 73, 2800, 3, 221, 4096, 33, 0, 0, 6, 0, 125, 0, 4083, 79, 0, 1758, 3568, 492, 10, 3667, 7, 2987, 0, 0, 2, 7, 4, 0, 2150, 0, 0, 4001, 0, 2, 1967, 1497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[2666, 0, 3905, 0, 4009, 0, 10, 3104, 337, 4082, 74, 3735, 22, 0, 40, 0, 2996, 0, 0, 0, 0, 1406, 68, 218, 2197, 3547, 0, 49, 471, 3487, 2, 143, 3474, 0, 83, 0, 0, 2843, 652, 0, 1059, 0, 0, 3442, 4077, 67, 0, 124, 0, 4028, 1, 0, 0, 556, 3684, 0, 992, 0, 0, 29, 0, 132, 0, 3725] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[34, 0, 4081, 164, 3420, 747, 1, 4077, 0, 3739, 0, 4096, 11, 16, 35, 0, 3837, 3, 0, 293, 0, 0, 0, 31, 1102, 3970, 8, 134, 2, 4057, 0, 7, 2752, 0, 1569, 974, 0, 4095, 537, 0, 1076, 35, 13, 2064, 3, 0, 32, 0, 0, 4090, 0, 0, 0, 0, 4078, 0, 3587, 0, 27, 0, 0, 2951, 480, 3308] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4094, 0, 2184, 103, 4082, 0, 0, 4021, 0, 4068, 86, 3965, 0, 2, 0, 0, 0, 68, 1, 222, 3772, 0, 0, 0, 2987, 2310, 177, 9, 3875, 25, 0, 315, 4082, 60, 73, 4, 16, 3996, 913, 13, 3445, 3769, 14, 1577, 48, 0, 0, 2961, 26, 252, 0, 17, 106, 1, 5, 12, 4096, 0, 0, 0, 0, 127, 0, 3557] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[227, 3, 0, 9, 0, 6, 0, 2855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3852, 21, 3553, 369, 0, 19, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 2914, 0, 0, 3719, 335, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 82, 2930, 0, 7, 0, 31, 2, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 873, 25, 0, 0, 2572, 0, 24, 0, 408, 0, 0, 0, 0, 0, 0, 0, 4024, 3755, 0, 104, 19, 5, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3699, 243, 3048, 0, 3941, 0, 0, 874, 619, 0, 0, 0, 0, 0, 0, 301, 61, 0, 0, 166, 455, 10, 4096, 165, 121, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3100, 9472, 10997, 4420, 7950, 11794, 12428, 5375] avg=8192.0000 max=12428 max_rank=6 min=3100 min_rank=0 imbalance_ratio=1.517090 max_load_reduction=53108 max_load_reduction_pct=0.810364 moved_to_echo=37547 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[676, 0, 110, 0, 0, 994, 0, 2361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3638, 57, 2951, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 321, 0, 398, 0, 4076, 432, 0, 0, 0, 0, 0, 0, 525, 3041, 0, 63, 3460, 3945, 51, 65, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3, 5, 0, 52, 3, 3780, 0, 2740, 337, 0, 0, 0, 0, 0, 0, 914, 1537, 1, 3084, 1138, 1, 13, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 253, 3816, 0, 7, 8, 4056, 3894, 30, 908, 512, 0, 0, 0, 0, 0, 0, 3025, 0, 9, 1057, 98, 25, 6, 1783, 155, 126, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 6647, 6212, 11150, 6934, 10684, 13484, 6284] avg=8192.0000 max=13484 max_rank=6 min=4141 min_rank=0 imbalance_ratio=1.645996 max_load_reduction=52052 max_load_reduction_pct=0.794250 moved_to_echo=37386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4025, 0, 2552, 87, 0, 332, 0, 3012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 1, 4089, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 0, 9, 867, 0, 0, 3840, 267, 0, 0, 0, 0, 0, 0, 3173, 3022, 0, 0, 0, 16, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1321, 1, 382, 0, 0, 39, 1, 0, 1692, 430, 0, 0, 0, 0, 0, 0, 564, 0, 0, 4091, 119, 3955, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 3594, 0, 0, 21, 0, 0, 0, 902, 653, 0, 0, 0, 0, 0, 0, 2816, 141, 4054, 59, 1, 0, 12, 3300, 155, 125, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10008, 7730, 8915, 6217, 3866, 8947, 9190, 10663] avg=8192.0000 max=10663 max_rank=7 min=3866 min_rank=4 imbalance_ratio=1.301636 max_load_reduction=54873 max_load_reduction_pct=0.837296 moved_to_echo=32666 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 3700, 0, 0, 1511, 0, 3011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 2492, 3238, 0, 126, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 628, 53, 0, 6, 129, 4039, 0, 0, 0, 0, 0, 0, 0, 200, 3048, 0, 37, 10, 4096, 18, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 30, 0, 5, 1, 0, 4021, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4066, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 0, 0, 0, 236, 0, 910, 653, 0, 0, 0, 0, 0, 0, 3993, 4, 3, 54, 3, 1107, 18, 4035, 164, 126, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12316, 9899, 4857, 11365, 8325, 4155, 5112, 9507] avg=8192.0000 max=12316 max_rank=0 min=4155 min_rank=5 imbalance_ratio=1.503418 max_load_reduction=53220 max_load_reduction_pct=0.812073 moved_to_echo=27099 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3495, 3393, 64, 0, 0, 0, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 458, 3721, 0, 2992, 53, 11, 0, 737, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 50, 786, 84, 2, 4095, 432, 0, 0, 0, 0, 0, 0, 0, 2911, 0, 1, 73, 2800, 3, 221, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 33, 0, 0, 6, 0, 125, 0, 2738, 400, 0, 0, 0, 0, 0, 0, 913, 79, 0, 1758, 3568, 492, 10, 3667, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2987, 0, 0, 2, 7, 4, 0, 869, 608, 0, 0, 0, 0, 0, 0, 2150, 0, 0, 4001, 0, 2, 1967, 1497, 159, 120, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9753, 7972, 9537, 6009, 7398, 10487, 4484, 9896] avg=8192.0000 max=10487 max_rank=5 min=4484 min_rank=6 imbalance_ratio=1.280151 max_load_reduction=55049 max_load_reduction_pct=0.839981 moved_to_echo=32265 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2666, 0, 3905, 0, 0, 0, 10, 2283, 0, 0, 0, 0, 0, 0, 0, 0, 337, 3915, 74, 3735, 22, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2996, 0, 0, 0, 0, 1406, 68, 218, 4009, 112, 0, 0, 0, 0, 0, 0, 2197, 2648, 0, 49, 471, 3487, 2, 143, 0, 0, 0, 0, 0, 0, 0, 0, 3474, 0, 83, 0, 0, 2843, 652, 0, 710, 326, 0, 0, 0, 0, 0, 0, 237, 0, 0, 3442, 4077, 67, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 1, 0, 0, 556, 3684, 0, 790, 495, 0, 0, 0, 0, 0, 0, 992, 0, 0, 29, 0, 132, 0, 3725, 167, 109, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8864, 8123, 8809, 8997, 8088, 7947, 9554, 5154] avg=8192.0000 max=9554 max_rank=6 min=5154 min_rank=7 imbalance_ratio=1.166260 max_load_reduction=55982 max_load_reduction_pct=0.854218 moved_to_echo=30743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 2184, 103, 0, 0, 0, 2957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 86, 3965, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 1, 222, 3772, 0, 0, 0, 4082, 364, 0, 0, 0, 0, 0, 0, 2987, 1725, 177, 9, 3875, 25, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 60, 73, 4, 16, 3996, 913, 13, 2310, 423, 0, 0, 0, 0, 0, 0, 771, 3769, 14, 1577, 48, 0, 0, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 26, 252, 0, 17, 106, 1, 5, 12, 514, 641, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 127, 0, 3557, 167, 71, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9338, 7954, 8509, 9113, 11890, 9140, 1574, 8018] avg=8192.0000 max=11890 max_rank=4 min=1574 min_rank=6 imbalance_ratio=1.451416 max_load_reduction=53646 max_load_reduction_pct=0.818573 moved_to_echo=30622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[34, 0, 4081, 164, 0, 747, 1, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 4096, 11, 16, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3837, 3, 0, 293, 0, 0, 0, 31, 3420, 116, 0, 0, 0, 0, 0, 0, 1102, 2955, 8, 134, 2, 4057, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2752, 0, 1569, 974, 0, 4095, 537, 0, 725, 433, 0, 0, 0, 0, 0, 0, 235, 35, 13, 2064, 3, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4078, 0, 887, 655, 0, 0, 0, 0, 0, 0, 3587, 0, 27, 0, 0, 2951, 480, 3308, 158, 128, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8016, 7739, 7700, 8265, 11085, 2382, 9710, 10639] avg=8192.0000 max=11085 max_rank=4 min=2382 min_rank=5 imbalance_ratio=1.353149 max_load_reduction=54451 max_load_reduction_pct=0.830856 moved_to_echo=33816 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[15872, 3584, 19968, 512, 0, 3712, 128, 22272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3712, 9472, 3840, 16128, 13184, 4608, 384, 14720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4352, 25856, 256, 3072, 256, 8576, 11904, 128, 6656, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[896, 30208, 2816, 28672, 512, 256, 384, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[10368, 22400, 256, 384, 7936, 18560, 128, 6144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[20992, 256, 4096, 5248, 384, 4864, 2560, 25344, 1408, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[18816, 128, 128, 4864, 4608, 3072, 640, 384, 31616, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23680, 1024, 2176, 1024, 128, 13568, 10112, 128, 11008, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[88, 0, 0, 49, 0, 3659, 0, 3830, 195, 2716, 4023, 0, 0, 700, 1, 86, 0, 4096, 4046, 0, 1, 4094, 999, 3870, 22, 4088, 131, 12, 0, 0, 4078, 3468, 442, 0, 0, 1, 3846, 0, 14, 1, 0, 110, 441, 0, 8, 35, 3911, 0, 846, 0, 4086, 0, 3204, 0, 5, 0, 37, 222, 4, 0, 2506, 5, 1560, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 27, 0, 0, 1, 4092, 0, 4, 4094, 11, 3880, 4, 0, 8, 0, 0, 0, 4058, 3671, 64, 0, 215, 14, 4092, 2496, 3047, 55, 0, 0, 20, 910, 254, 1, 3236, 726, 4040, 72, 0, 1, 0, 51, 5, 22, 3854, 0, 170, 3556, 68, 8, 21, 4015, 3759, 4011, 2317, 0, 100, 1, 7, 0, 0, 3563, 1, 0, 914] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[19, 0, 2490, 0, 2, 3668, 0, 550, 197, 0, 4046, 0, 0, 116, 0, 4, 375, 3860, 4095, 702, 0, 741, 1, 3770, 29, 3607, 764, 131, 8, 0, 3383, 4096, 1800, 0, 0, 77, 0, 1118, 2662, 0, 3729, 69, 0, 0, 361, 1, 1298, 0, 4096, 3292, 3691, 2692, 8, 443, 0, 0, 0, 0, 0, 0, 0, 345, 3084, 116] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4094, 0, 396, 526, 4, 45, 0, 958, 2974, 49, 4096, 0, 32, 1, 1015, 0, 26, 327, 4021, 0, 0, 2430, 0, 3984, 2, 2950, 0, 1176, 2, 0, 4080, 3396, 1, 0, 991, 0, 4096, 21, 0, 1222, 68, 25, 3449, 879, 1, 0, 0, 0, 3785, 0, 3098, 154, 3583, 1, 0, 0, 22, 4094, 55, 0, 3400, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 1, 0, 12, 231, 0, 103, 0, 1792, 0, 4096, 0, 0, 0, 104, 0, 1457, 4015, 4095, 12, 0, 2498, 4068, 4096, 2481, 0, 0, 97, 0, 5, 0, 3425, 4039, 0, 1217, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 767, 2907, 0, 4096, 3478, 4077, 2635, 459, 826, 41, 4, 0, 0, 300, 0, 19, 0, 3960, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[2689, 0, 233, 0, 0, 1784, 0, 60, 0, 872, 4060, 0, 0, 543, 154, 0, 0, 338, 4091, 4083, 16, 903, 4088, 2176, 1, 4074, 0, 0, 6, 7, 4077, 4096, 2263, 0, 2, 0, 44, 0, 0, 1, 0, 4068, 0, 3851, 5, 54, 0, 30, 3481, 250, 4096, 0, 4088, 3429, 0, 0, 1, 2, 0, 0, 64, 0, 1456, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[2, 21, 3749, 0, 63, 3721, 0, 300, 21, 172, 3557, 1113, 0, 4093, 12, 0, 0, 3875, 4052, 0, 583, 4092, 985, 3929, 0, 4093, 103, 112, 2, 0, 3698, 1835, 8, 0, 2946, 0, 224, 0, 0, 0, 0, 0, 3469, 3471, 0, 79, 0, 0, 1855, 0, 4096, 0, 9, 0, 0, 724, 457, 5, 51, 0, 0, 0, 26, 3933] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4028, 0, 0, 1, 0, 2922, 0, 279, 2650, 1, 4090, 0, 0, 1118, 3, 0, 1, 4096, 4078, 0, 3540, 4085, 4027, 4083, 1589, 19, 0, 52, 0, 610, 58, 128, 3956, 184, 0, 318, 5, 1762, 0, 12, 2, 0, 3768, 4022, 0, 0, 1039, 0, 0, 0, 3628, 4, 4096, 34, 21, 0, 0, 0, 8, 1, 1044, 4, 157, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[88, 0, 0, 49, 0, 3659, 0, 3830, 1707, 499, 0, 0, 0, 0, 0, 0, 195, 2716, 4023, 0, 0, 700, 1, 86, 1047, 536, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 1, 4094, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 2737, 131, 12, 0, 0, 4078, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 442, 0, 0, 1, 3846, 0, 14, 1, 3120, 0, 0, 0, 0, 0, 0, 0, 0, 110, 441, 0, 8, 35, 3911, 0, 1889, 304, 0, 0, 0, 0, 0, 0, 846, 0, 490, 0, 3204, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 222, 4, 0, 2506, 5, 1560, 0, 4046, 251, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9832, 9304, 8654, 10448, 7424, 6698, 4545, 8631] avg=8192.0000 max=10448 max_rank=3 min=4545 min_rank=6 imbalance_ratio=1.275391 max_load_reduction=55088 max_load_reduction_pct=0.840576 moved_to_echo=27298 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 2490, 0, 2, 3668, 0, 550, 1542, 486, 0, 0, 0, 0, 0, 0, 197, 0, 4046, 0, 0, 116, 0, 4, 924, 505, 0, 0, 0, 0, 0, 0, 375, 3355, 0, 702, 0, 741, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2415, 764, 131, 8, 0, 3383, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1800, 0, 0, 77, 0, 1118, 2662, 0, 3040, 0, 0, 0, 0, 0, 0, 0, 3729, 69, 0, 0, 361, 1, 1298, 0, 1706, 268, 0, 0, 0, 0, 0, 0, 4096, 3292, 443, 2692, 8, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 3084, 116, 4095, 244, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8757, 5792, 5174, 10826, 8697, 7432, 10974, 7884] avg=8192.0000 max=10974 max_rank=6 min=5174 min_rank=2 imbalance_ratio=1.339600 max_load_reduction=54562 max_load_reduction_pct=0.832550 moved_to_echo=34987 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 21, 3749, 0, 63, 3721, 0, 300, 1712, 506, 0, 0, 0, 0, 0, 0, 21, 172, 3557, 1113, 0, 4093, 12, 0, 1049, 507, 0, 0, 0, 0, 0, 0, 0, 3368, 0, 0, 583, 4092, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2740, 103, 112, 2, 0, 3698, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2946, 0, 224, 0, 0, 0, 3167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3469, 3471, 0, 79, 0, 0, 1894, 304, 0, 0, 0, 0, 0, 0, 1855, 0, 490, 0, 9, 0, 0, 724, 0, 0, 0, 0, 0, 0, 0, 0, 457, 5, 51, 0, 0, 0, 26, 3933, 4052, 256, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10074, 10524, 9028, 8490, 6345, 9217, 3078, 8780] avg=8192.0000 max=10524 max_rank=1 min=3078 min_rank=6 imbalance_ratio=1.284668 max_load_reduction=55012 max_load_reduction_pct=0.839417 moved_to_echo=27420 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 0, 0, 1, 4092, 0, 4, 1678, 527, 0, 0, 0, 0, 0, 0, 4094, 11, 3880, 4, 0, 8, 0, 0, 781, 531, 0, 0, 0, 0, 0, 0, 0, 3527, 0, 64, 0, 215, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2496, 2040, 55, 0, 0, 20, 910, 254, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3236, 726, 4040, 72, 0, 1, 0, 3300, 0, 0, 0, 0, 0, 0, 0, 51, 5, 22, 3854, 0, 170, 3556, 68, 1856, 226, 0, 0, 0, 0, 0, 0, 8, 21, 481, 3759, 4011, 2317, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 3563, 1, 0, 914, 3671, 265, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6329, 9309, 3820, 5775, 11376, 9808, 10697, 8422] avg=8192.0000 max=11376 max_rank=4 min=3820 min_rank=2 imbalance_ratio=1.388672 max_load_reduction=54160 max_load_reduction_pct=0.826416 moved_to_echo=40303 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 396, 526, 4, 45, 0, 958, 1294, 514, 0, 0, 0, 0, 0, 0, 2974, 49, 4096, 0, 32, 1, 1015, 0, 756, 42, 0, 0, 0, 0, 0, 0, 26, 285, 0, 0, 0, 2430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1975, 0, 1176, 2, 0, 4080, 3396, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 991, 0, 4096, 21, 0, 1222, 3212, 0, 0, 0, 0, 0, 0, 0, 68, 25, 3449, 879, 1, 0, 0, 0, 1432, 219, 0, 0, 0, 0, 0, 0, 3785, 0, 372, 154, 3583, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4094, 55, 0, 3400, 0, 0, 7, 4021, 258, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7831, 8965, 2741, 10631, 9543, 6073, 7895, 11857] avg=8192.0000 max=11857 max_rank=7 min=2741 min_rank=2 imbalance_ratio=1.447388 max_load_reduction=53679 max_load_reduction_pct=0.819077 moved_to_echo=35368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 1, 0, 12, 231, 0, 103, 0, 1704, 529, 0, 0, 0, 0, 0, 0, 1792, 0, 4096, 0, 0, 0, 104, 0, 0, 525, 0, 0, 0, 0, 0, 0, 1457, 3490, 0, 12, 0, 2498, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 97, 0, 5, 0, 3425, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 1217, 0, 0, 0, 0, 0, 3302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 767, 2907, 0, 1885, 0, 0, 0, 0, 0, 0, 0, 4096, 3478, 488, 2635, 459, 826, 41, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 19, 0, 3960, 0, 4095, 265, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6676, 6517, 11525, 6008, 8558, 5586, 12027, 8639] avg=8192.0000 max=12027 max_rank=6 min=5586 min_rank=5 imbalance_ratio=1.468140 max_load_reduction=53509 max_load_reduction_pct=0.816483 moved_to_echo=34810 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2689, 0, 233, 0, 0, 1784, 0, 60, 1716, 282, 0, 0, 0, 0, 0, 0, 0, 872, 4060, 0, 0, 543, 154, 0, 1047, 47, 0, 0, 0, 0, 0, 0, 0, 291, 0, 4083, 16, 903, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2720, 0, 0, 6, 7, 4077, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2263, 0, 2, 0, 44, 0, 0, 1, 1754, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 3851, 5, 54, 0, 30, 1898, 307, 0, 0, 0, 0, 0, 0, 3481, 250, 482, 0, 4088, 3429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 64, 0, 1456, 0, 4091, 140, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6764, 6723, 9381, 10907, 4064, 10213, 11730, 5754] avg=8192.0000 max=11730 max_rank=6 min=4064 min_rank=4 imbalance_ratio=1.431885 max_load_reduction=53806 max_load_reduction_pct=0.821014 moved_to_echo=31761 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4028, 0, 0, 1, 0, 2922, 0, 279, 1516, 527, 0, 0, 0, 0, 0, 0, 2650, 1, 4090, 0, 0, 1118, 3, 0, 4, 536, 0, 0, 0, 0, 0, 0, 1, 3560, 0, 0, 3540, 4085, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1589, 14, 0, 52, 0, 610, 58, 128, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 184, 0, 318, 5, 1762, 0, 12, 3292, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3768, 4022, 0, 0, 1039, 0, 1677, 1, 0, 0, 0, 0, 0, 0, 0, 0, 435, 4, 4096, 34, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 1044, 4, 157, 13, 4078, 264, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9273, 8402, 15213, 2451, 9529, 10509, 4590, 5569] avg=8192.0000 max=15213 max_rank=2 min=2451 min_rank=3 imbalance_ratio=1.857056 max_load_reduction=50323 max_load_reduction_pct=0.767868 moved_to_echo=30197 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[1920, 21504, 0, 4864, 4224, 19072, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6656, 14720, 1152, 1664, 128, 768, 20352, 20736, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[12544, 3456, 5888, 4480, 8320, 2944, 2688, 1280, 24192, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[18176, 7168, 3712, 9344, 19584, 7168, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3968, 4352, 11264, 16128, 384, 1152, 12800, 128, 14336, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12032, 3840, 31872, 1152, 128, 6656, 1408, 128, 5632, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[15104, 128, 6912, 640, 384, 19968, 128, 6016, 12928, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 4352, 512, 128, 10624, 384, 10368, 4992, 32256, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 3689, 0, 0, 0, 0, 0, 0, 4073, 3739, 0, 4056, 3512, 3735, 0, 1451, 0, 142, 4035, 210, 0, 77, 46, 0, 0, 34, 671, 1104, 0, 125, 732, 0, 4044, 0, 11, 0, 1, 310, 1, 665, 961, 3844, 0, 2523, 1, 4, 0, 4017, 0, 0, 0, 4096, 1619, 3, 3054, 238, 4, 0, 258, 3848, 1, 506, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 297, 0, 57, 4, 4060, 326, 0, 23, 0, 664, 25, 0, 0, 4044, 8, 1989, 4088, 1371, 1, 1288, 0, 0, 0, 0, 780, 77, 4084, 0, 3868, 3377, 0, 14, 1, 20, 4092, 573, 9, 0, 1, 34, 2721, 4078, 35, 0, 160, 1476, 97, 660, 3057, 8, 134, 0, 73, 1, 3216, 3820, 3280, 3946, 0, 3597] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 33, 7, 2, 2560, 4096, 4042, 0, 763, 229, 0, 0, 3694, 51, 0, 4083, 588, 9, 7, 0, 76, 0, 0, 0, 4078, 0, 3942, 41, 7, 4092, 6, 11, 3, 191, 0, 0, 283, 237, 3164, 0, 6, 4073, 29, 0, 3953, 0, 0, 0, 4068, 0, 0, 2798, 0, 4096, 0, 2827, 3751, 0, 57, 3583] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3, 22, 91, 617, 1, 0, 0, 1074, 1277, 4075, 3485, 1095, 1032, 0, 0, 0, 3557, 0, 0, 3173, 1149, 4, 422, 0, 1, 2810, 9, 10, 3481, 2914, 0, 4073, 274, 1684, 0, 350, 0, 645, 287, 0, 1417, 1403, 2391, 4, 4096, 57, 45, 7, 0, 0, 0, 0, 902, 0, 0, 3750, 0, 3443, 0, 3019, 3952, 0, 3435, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2146, 4, 0, 0, 84, 0, 4044, 2290, 3645, 268, 0, 0, 0, 0, 3132, 0, 5, 3797, 0, 259, 20, 110, 0, 477, 3, 0, 3487, 0, 103, 0, 3970, 4096, 4084, 0, 0, 0, 2854, 0, 0, 5, 3837, 0, 4096, 1465, 1, 0, 2722, 364, 45, 22, 203, 0, 213, 4086, 3298, 0, 616, 4010, 1512, 2, 0, 161] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[33, 0, 161, 6, 0, 0, 67, 4085, 0, 3476, 3054, 4051, 3463, 0, 2, 27, 3327, 3781, 427, 4075, 93, 0, 0, 0, 6, 0, 102, 1, 3622, 1001, 2618, 30, 340, 647, 0, 1466, 0, 0, 413, 0, 17, 3292, 2499, 0, 581, 22, 448, 0, 29, 156, 9, 0, 33, 0, 0, 4077, 0, 4065, 1338, 3994, 850, 20, 3732, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 910, 0, 0, 0, 4096, 3385, 4023, 4096, 0, 4047, 141, 45, 4079, 0, 0, 4096, 0, 0, 2, 124, 0, 1457, 0, 2, 2999, 18, 96, 1216, 615, 4010, 3012, 14, 28, 0, 505, 0, 0, 2021, 0, 0, 4084, 3876, 0, 0, 2285, 330, 0, 985, 3050, 0, 143, 1725, 1, 0, 0, 3803, 209, 0, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4095, 68, 0, 2, 0, 0, 0, 4069, 4091, 2021, 3823, 261, 1811, 2, 4094, 1793, 0, 1725, 3718, 0, 283, 3, 270, 36, 24, 2, 3491, 0, 4095, 0, 10, 4061, 3713, 0, 27, 125, 2441, 32, 0, 0, 336, 3, 3, 3121, 0, 12, 22, 0, 0, 1352, 0, 0, 90, 2020, 4066, 25, 0, 3753, 0, 546, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 297, 0, 57, 4, 0, 3282, 0, 0, 0, 0, 0, 0, 4060, 132, 0, 23, 0, 664, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 8, 1334, 4088, 1371, 1, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 780, 77, 4084, 0, 97, 538, 0, 0, 0, 0, 0, 0, 3868, 3377, 0, 14, 1, 20, 4092, 573, 383, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1, 34, 2721, 4078, 35, 0, 272, 6, 0, 0, 0, 0, 0, 0, 160, 1476, 97, 660, 3057, 8, 134, 0, 97, 0, 0, 0, 0, 0, 0, 0, 73, 1, 3216, 0, 3274, 3946, 0, 3597, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3642, 4904, 12134, 5576, 12328, 7156, 5689, 14107] avg=8192.0000 max=14107 max_rank=7 min=3642 min_rank=0 imbalance_ratio=1.722046 max_load_reduction=51429 max_load_reduction_pct=0.784744 moved_to_echo=39280 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3689, 0, 0, 0, 0, 0, 3739, 222, 0, 0, 0, 0, 0, 0, 0, 1630, 0, 0, 4056, 3512, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 0, 142, 2705, 210, 0, 77, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 671, 1104, 0, 125, 732, 1223, 36, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 11, 0, 1, 310, 1, 778, 1, 0, 0, 0, 0, 0, 0, 665, 961, 3844, 0, 2523, 1, 4, 0, 552, 11, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 0, 4096, 1619, 3, 3054, 1219, 0, 0, 0, 0, 0, 0, 0, 238, 4, 0, 0, 3837, 1, 506, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7651, 12933, 4631, 3925, 5146, 8561, 14008, 8681] avg=8192.0000 max=14008 max_rank=6 min=3925 min_rank=3 imbalance_ratio=1.709961 max_load_reduction=51528 max_load_reduction_pct=0.786255 moved_to_echo=36396 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 33, 7, 2, 4042, 2429, 0, 0, 0, 0, 0, 0, 2560, 1639, 0, 0, 763, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3694, 51, 0, 2737, 588, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 4078, 0, 3942, 41, 1230, 398, 0, 0, 0, 0, 0, 0, 7, 4092, 6, 11, 3, 191, 0, 0, 787, 1, 0, 0, 0, 0, 0, 0, 283, 237, 3164, 0, 6, 4073, 29, 0, 559, 7, 0, 0, 0, 0, 0, 0, 3953, 0, 0, 0, 4068, 0, 0, 2798, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3744, 0, 57, 3583, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6513, 5191, 7086, 9765, 5098, 8358, 12045, 11480] avg=8192.0000 max=12045 max_rank=6 min=5098 min_rank=4 imbalance_ratio=1.470337 max_load_reduction=53491 max_load_reduction_pct=0.816208 moved_to_echo=36981 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2146, 4, 0, 0, 84, 0, 3645, 3445, 0, 0, 0, 0, 0, 0, 4044, 917, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3132, 0, 5, 2546, 0, 259, 20, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 3, 0, 3487, 0, 103, 0, 688, 565, 0, 0, 0, 0, 0, 0, 3970, 4096, 4084, 0, 0, 0, 2854, 0, 732, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3837, 0, 4096, 1465, 1, 0, 519, 3, 0, 0, 0, 0, 0, 0, 2722, 364, 45, 22, 203, 0, 213, 4086, 685, 0, 0, 0, 0, 0, 0, 0, 3298, 0, 616, 0, 1509, 2, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9324, 5229, 6072, 5323, 15736, 9926, 8340, 5586] avg=8192.0000 max=15736 max_rank=4 min=5229 min_rank=1 imbalance_ratio=1.920898 max_load_reduction=49800 max_load_reduction_pct=0.759888 moved_to_echo=39588 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 22, 91, 617, 1, 0, 0, 1074, 3485, 2594, 0, 0, 0, 0, 0, 0, 1277, 1630, 0, 1095, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3557, 0, 0, 2128, 1149, 4, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2810, 9, 10, 3481, 2914, 0, 4073, 1224, 425, 0, 0, 0, 0, 0, 0, 274, 1684, 0, 350, 0, 645, 287, 0, 611, 1, 0, 0, 0, 0, 0, 0, 1417, 1403, 2391, 4, 4096, 57, 45, 7, 434, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 902, 0, 0, 3750, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 3443, 0, 0, 3944, 0, 3435, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7887, 5034, 7260, 14947, 3852, 9862, 5872, 10822] avg=8192.0000 max=14947 max_rank=3 min=3852 min_rank=4 imbalance_ratio=1.824585 max_load_reduction=50589 max_load_reduction_pct=0.771927 moved_to_echo=30408 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 910, 0, 0, 0, 4023, 3267, 0, 0, 0, 0, 0, 0, 4096, 1354, 0, 4096, 0, 4047, 141, 45, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 2747, 0, 0, 2, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1457, 0, 2, 2999, 18, 96, 1216, 1017, 536, 0, 0, 0, 0, 0, 0, 615, 4010, 3012, 14, 28, 0, 505, 0, 789, 1, 0, 0, 0, 0, 0, 0, 0, 2021, 0, 0, 4084, 3876, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 2285, 330, 0, 985, 3050, 0, 143, 1725, 1013, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 209, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 13779, 6952, 7341, 8974, 10541, 9531, 218] avg=8192.0000 max=13779 max_rank=1 min=218 min_rank=7 imbalance_ratio=1.682007 max_load_reduction=51757 max_load_reduction_pct=0.789749 moved_to_echo=29264 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[33, 0, 161, 6, 0, 0, 67, 4085, 3054, 3432, 0, 0, 0, 0, 0, 0, 0, 1391, 0, 4051, 3463, 0, 2, 27, 0, 0, 0, 0, 0, 0, 0, 0, 3327, 3781, 427, 2733, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 102, 1, 3622, 1001, 2618, 30, 1044, 562, 0, 0, 0, 0, 0, 0, 340, 647, 0, 1466, 0, 0, 413, 0, 785, 1, 0, 0, 0, 0, 0, 0, 17, 3292, 2499, 0, 581, 22, 448, 0, 557, 1, 0, 0, 0, 0, 0, 0, 29, 156, 9, 0, 33, 0, 0, 4077, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 1338, 0, 849, 20, 3732, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10838, 8934, 10361, 8986, 3652, 7417, 5344, 10004] avg=8192.0000 max=10838 max_rank=0 min=3652 min_rank=4 imbalance_ratio=1.322998 max_load_reduction=54698 max_load_reduction_pct=0.834625 moved_to_echo=26417 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 68, 0, 2, 0, 0, 4091, 3225, 0, 0, 0, 0, 0, 0, 0, 1614, 0, 2021, 3823, 261, 1811, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1793, 0, 1149, 3718, 0, 283, 3, 0, 0, 0, 0, 0, 0, 0, 0, 270, 36, 24, 2, 3491, 0, 4095, 0, 1227, 528, 0, 0, 0, 0, 0, 0, 10, 4061, 3713, 0, 27, 125, 2441, 32, 336, 5, 0, 0, 0, 0, 0, 0, 0, 0, 336, 3, 3, 3121, 0, 12, 240, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1352, 0, 0, 90, 2020, 1223, 0, 0, 0, 0, 0, 0, 0, 4066, 25, 0, 0, 0, 546, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11481, 9532, 11040, 9673, 10750, 3715, 4707, 4638] avg=8192.0000 max=11481 max_rank=0 min=3715 min_rank=5 imbalance_ratio=1.401489 max_load_reduction=54055 max_load_reduction_pct=0.824814 moved_to_echo=23810 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7680, 11648, 5248, 0, 17408, 4608, 7808, 11520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9088, 26112, 10880, 1920, 128, 1024, 11008, 640, 5248, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 4864, 256, 768, 23168, 4096, 15104, 6144, 7808, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[16128, 10368, 0, 11648, 13184, 8832, 5760, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23424, 9728, 640, 18176, 9856, 1664, 896, 1664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[13312, 2432, 256, 3072, 15488, 1664, 640, 21632, 7808, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2432, 7936, 16128, 128, 18176, 16768, 640, 128, 3712, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 10240, 768, 1280, 128, 256, 5248, 26112, 22016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 68, 0, 0, 0, 1961, 0, 3417, 3911, 0, 0, 6, 0, 4081, 7, 0, 1, 10, 0, 4096, 3300, 0, 3974, 4049, 0, 299, 9, 279, 0, 3308, 0, 3075, 0, 0, 4095, 0, 9, 4063, 0, 1116, 38, 0, 10, 8, 381, 3230, 17, 1291, 0, 825, 0, 0, 4096, 1683, 0, 63, 0, 1197, 3501, 259, 4, 0, 3799, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 1959, 0, 4078, 3879, 0, 232, 143, 318, 3583, 4019, 7, 0, 0, 0, 239, 886, 650, 0, 0, 3649, 1, 0, 15, 90, 14, 4060, 795, 0, 0, 1509, 0, 0, 0, 4059, 11, 0, 4090, 0, 3, 0, 0, 0, 0, 4096, 0, 3004, 1143, 0, 4089, 3984, 0, 0, 15, 0, 2335, 1012, 3705, 59, 3770, 19] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4094, 0, 2044, 1, 93, 0, 0, 1998, 0, 123, 29, 0, 4058, 4096, 1141, 3803, 7, 2013, 0, 4095, 246, 0, 56, 4096, 0, 0, 1372, 0, 0, 978, 0, 0, 0, 4096, 16, 0, 0, 4014, 0, 0, 75, 0, 84, 4081, 2, 1319, 2323, 3077, 5, 0, 0, 0, 4070, 45, 17, 0, 0, 0, 4090, 0, 3358, 47, 474, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 3591, 0, 0, 3943, 198, 2, 3960, 9, 2108, 0, 7, 0, 4053, 927, 4096, 0, 5, 0, 803, 0, 0, 0, 4096, 4096, 17, 53, 4087, 0, 0, 3682, 4096, 0, 1, 0, 52, 25, 4095, 0, 511, 0, 16, 0, 1, 0, 0, 90, 0, 258, 1712, 2265, 0, 3853, 30, 0, 0, 0, 668, 4068, 0, 0, 0, 4033, 29] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3980, 0, 0, 0, 0, 0, 2723, 4083, 3982, 58, 2990, 153, 0, 4096, 258, 2, 0, 2, 0, 1471, 0, 0, 1234, 4089, 3695, 0, 67, 254, 0, 4010, 7, 4066, 170, 104, 29, 0, 0, 4055, 0, 29, 0, 20, 1, 652, 0, 0, 0, 3337, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 6, 4095, 0, 4096, 0, 3624, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3945, 11, 3445, 0, 4086, 0, 142, 1, 0, 1488, 32, 285, 1, 4088, 0, 4079, 3982, 0, 85, 3855, 4096, 0, 29, 3786, 0, 0, 0, 0, 80, 490, 2, 0, 0, 0, 2, 4085, 234, 342, 87, 6, 0, 0, 4093, 3834, 0, 4, 0, 825, 296, 40, 4096, 163, 1112, 0, 0, 0, 0, 0, 4096, 0, 4035, 0, 178, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4041, 0, 0, 0, 1768, 1, 16, 126, 19, 0, 4, 36, 356, 3922, 9, 7, 0, 0, 10, 3772, 17, 1, 227, 4073, 517, 114, 11, 0, 3, 4056, 0, 4092, 0, 56, 3418, 0, 2804, 3838, 0, 4096, 1018, 1006, 3, 3745, 0, 0, 216, 4025, 0, 0, 14, 0, 4087, 43, 0, 0, 2, 0, 4096, 11, 3735, 0, 2125, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 68, 0, 0, 0, 1961, 0, 2702, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 0, 6, 0, 2985, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 4096, 3300, 0, 3974, 2749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 9, 279, 0, 3308, 0, 3075, 715, 88, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 9, 4063, 0, 1116, 832, 0, 0, 0, 0, 0, 0, 0, 38, 0, 10, 8, 381, 3230, 17, 1291, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 825, 0, 0, 4096, 1683, 0, 63, 1300, 176, 45, 0, 0, 0, 0, 0, 0, 1197, 1997, 259, 4, 0, 3799, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4731, 6909, 14130, 7773, 10115, 6434, 8188, 7256] avg=8192.0000 max=14130 max_rank=2 min=4731 min_rank=0 imbalance_ratio=1.724854 max_load_reduction=51406 max_load_reduction_pct=0.784393 moved_to_echo=31993 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 1959, 0, 4078, 3068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 143, 318, 3583, 2939, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 886, 650, 0, 0, 2477, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 90, 14, 4060, 795, 0, 811, 87, 0, 0, 0, 0, 0, 0, 0, 1509, 0, 0, 0, 4059, 11, 0, 820, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3, 0, 0, 0, 0, 4096, 973, 0, 0, 0, 0, 0, 0, 0, 0, 3004, 1143, 0, 4089, 3984, 0, 0, 1172, 173, 30, 0, 0, 0, 0, 0, 15, 0, 1332, 1012, 3705, 59, 3770, 19, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9121, 7222, 4252, 5873, 6399, 9162, 13595, 9912] avg=8192.0000 max=13595 max_rank=6 min=4252 min_rank=2 imbalance_ratio=1.659546 max_load_reduction=51941 max_load_reduction_pct=0.792557 moved_to_echo=39068 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3591, 0, 0, 3943, 198, 2, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2108, 0, 7, 0, 2964, 927, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 803, 0, 0, 0, 2781, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 53, 4087, 0, 0, 3682, 4096, 828, 88, 0, 0, 0, 0, 0, 0, 0, 1, 0, 52, 25, 4095, 0, 511, 827, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 1, 0, 0, 90, 0, 1696, 0, 0, 0, 0, 0, 0, 0, 258, 1712, 2265, 0, 3853, 30, 0, 0, 1315, 174, 52, 0, 0, 0, 0, 0, 0, 668, 2320, 0, 0, 0, 4033, 29, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10866, 10111, 3589, 16947, 5511, 1803, 9659, 7050] avg=8192.0000 max=16947 max_rank=3 min=1803 min_rank=5 imbalance_ratio=2.068726 max_load_reduction=48589 max_load_reduction_pct=0.741409 moved_to_echo=24023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8, 9, 0, 1, 2203, 75, 4079, 2, 0, 17, 484, 4, 2819, 0, 3974, 0, 0, 0, 751, 0, 0, 4089, 3887, 57, 180, 31, 4096, 141, 0, 0, 4055, 0, 3877, 2, 2186, 72, 467, 330, 1212, 0, 1531, 5, 4027, 318, 0, 4, 0, 0, 0, 0, 0, 3812, 4023, 0, 682, 0, 3933, 4030, 2, 4059, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 2044, 1, 93, 0, 0, 1577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 29, 0, 4058, 2996, 1141, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2013, 0, 4095, 246, 0, 56, 2781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1372, 0, 0, 978, 0, 0, 421, 88, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 4014, 0, 0, 836, 0, 0, 0, 0, 0, 0, 0, 75, 0, 84, 4081, 2, 1319, 2323, 3077, 1705, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4070, 45, 17, 0, 1315, 176, 52, 0, 0, 0, 0, 0, 0, 0, 2333, 0, 3358, 47, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7809, 12150, 9198, 2859, 8962, 12666, 5680, 6212] avg=8192.0000 max=12666 max_rank=5 min=2859 min_rank=3 imbalance_ratio=1.546143 max_load_reduction=52870 max_load_reduction_pct=0.806732 moved_to_echo=33520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3980, 0, 0, 0, 0, 0, 2723, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 58, 2990, 153, 0, 2996, 258, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1471, 0, 0, 1234, 2776, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 0, 67, 254, 0, 4010, 7, 4066, 854, 88, 0, 0, 0, 0, 0, 0, 170, 104, 29, 0, 0, 4055, 0, 29, 836, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1, 652, 0, 0, 0, 3337, 1707, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 1313, 176, 52, 0, 0, 0, 0, 0, 0, 6, 2336, 0, 4096, 0, 3624, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9932, 10439, 5483, 13041, 5223, 5717, 5639, 10062] avg=8192.0000 max=13041 max_rank=3 min=5223 min_rank=4 imbalance_ratio=1.591919 max_load_reduction=52495 max_load_reduction_pct=0.801010 moved_to_echo=26641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4041, 0, 0, 0, 1768, 1, 16, 100, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4, 36, 356, 2868, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3772, 17, 1, 227, 2765, 0, 0, 0, 0, 0, 0, 0, 0, 517, 114, 11, 0, 3, 4056, 0, 4092, 26, 85, 0, 0, 0, 0, 0, 0, 0, 56, 3418, 0, 2804, 3838, 0, 4096, 800, 0, 0, 0, 0, 0, 0, 0, 1018, 1006, 3, 3745, 0, 0, 216, 4025, 1708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4087, 43, 0, 0, 1308, 169, 52, 0, 0, 0, 0, 0, 2, 0, 2336, 11, 3735, 0, 2125, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5926, 3299, 6792, 8904, 15012, 11721, 5673, 8209] avg=8192.0000 max=15012 max_rank=4 min=3299 min_rank=1 imbalance_ratio=1.832520 max_load_reduction=50524 max_load_reduction_pct=0.770935 moved_to_echo=40615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3945, 11, 3445, 0, 4086, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1488, 32, 285, 1, 2978, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 85, 3855, 4096, 0, 29, 2566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 490, 2, 0, 1, 92, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4085, 234, 342, 87, 6, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3834, 0, 4, 0, 825, 1712, 0, 0, 0, 0, 0, 0, 0, 296, 40, 4096, 163, 1112, 0, 0, 0, 1220, 181, 57, 0, 0, 0, 0, 0, 0, 0, 2327, 0, 4035, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11629, 8863, 14613, 665, 5593, 10468, 7165, 6540] avg=8192.0000 max=14613 max_rank=2 min=665 min_rank=3 imbalance_ratio=1.783813 max_load_reduction=50923 max_load_reduction_pct=0.777023 moved_to_echo=29766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 9, 0, 1, 2203, 75, 3226, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 17, 484, 4, 2062, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 751, 0, 0, 4089, 2639, 0, 0, 0, 0, 0, 0, 0, 0, 57, 180, 31, 4096, 141, 0, 0, 4055, 853, 61, 0, 0, 0, 0, 0, 0, 0, 3877, 2, 2186, 72, 467, 330, 1212, 575, 0, 0, 0, 0, 0, 0, 0, 0, 1531, 5, 4027, 318, 0, 4, 0, 1680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3812, 4023, 0, 682, 1248, 121, 51, 0, 0, 0, 0, 0, 0, 3933, 2299, 2, 4059, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5522, 6543, 7479, 9474, 8721, 7565, 9937, 10295] avg=8192.0000 max=10295 max_rank=7 min=5522 min_rank=0 imbalance_ratio=1.256714 max_load_reduction=55241 max_load_reduction_pct=0.842911 moved_to_echo=36518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8448, 640, 1664, 8832, 256, 17024, 4608, 19456, 4608, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5248, 2688, 4224, 16384, 768, 4608, 2688, 16768, 12672, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 2048, 384, 19840, 8320, 128, 9728, 21632, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 5888, 17280, 1408, 23040, 128, 18048, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7936, 4096, 3328, 1408, 8064, 22912, 2432, 16000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[16128, 3712, 5504, 128, 11904, 4480, 7040, 17152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 9728, 7680, 6400, 3200, 24960, 512, 7040, 6400, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 5632, 7552, 256, 29312, 9856, 128, 768, 10240, 1408, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2545, 0, 4090, 917, 0, 0, 23, 0, 595, 78, 3497, 56, 0, 28, 725, 0, 0, 0, 0, 2, 1798, 0, 16, 4096, 0, 4055, 4093, 0, 0, 4027, 0, 2, 0, 4072, 4, 64, 0, 1, 4091, 3852, 0, 2908, 1055, 12, 0, 3234, 400, 2574, 0, 0, 1579, 0, 3332, 0, 0, 3700, 0, 0, 0, 6, 9, 66, 3934, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[7, 315, 152, 3, 361, 2, 0, 0, 31, 0, 163, 0, 91, 2237, 0, 227, 0, 0, 0, 63, 12, 294, 4056, 3707, 384, 4096, 3890, 903, 57, 1, 529, 0, 6, 3925, 1625, 0, 1648, 3672, 2737, 3564, 4093, 1465, 0, 0, 0, 228, 3849, 22, 0, 3181, 2816, 307, 4096, 0, 121, 3685, 1380, 8, 0, 0, 57, 18, 1452, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3955, 252, 4096, 0, 11, 646, 29, 1, 0, 451, 3340, 0, 0, 1028, 2, 11, 2795, 24, 1, 0, 4089, 2, 45, 4058, 1, 4070, 3083, 20, 155, 0, 5, 1, 0, 262, 2277, 1107, 0, 17, 4092, 4094, 0, 4066, 0, 52, 0, 0, 3590, 2844, 53, 0, 0, 0, 0, 0, 0, 3161, 0, 0, 68, 0, 3873, 1807, 1550, 452] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[7, 4088, 3835, 0, 0, 0, 37, 0, 0, 0, 1, 4078, 0, 4071, 3, 3579, 2, 1, 0, 0, 139, 0, 3702, 4031, 11, 4096, 929, 0, 1524, 10, 0, 0, 0, 194, 0, 0, 3, 366, 4091, 997, 0, 3924, 2673, 1556, 56, 19, 0, 3533, 50, 3484, 3, 47, 2440, 2, 0, 3469, 49, 0, 340, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 3244, 7, 0, 0, 0, 0, 0, 1, 0, 144, 0, 0, 0, 0, 0, 0, 0, 3363, 2898, 0, 2, 4096, 1, 4095, 2125, 1691, 2412, 0, 10, 0, 0, 0, 412, 608, 0, 0, 3886, 2, 3963, 4088, 1256, 146, 5, 2, 4096, 3335, 0, 4025, 0, 0, 3954, 1153, 5, 2356, 0, 0, 0, 34, 71, 148, 3816, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[3717, 0, 3920, 11, 0, 55, 429, 0, 6, 5, 6, 4002, 0, 3470, 1, 174, 0, 448, 1525, 4062, 18, 0, 843, 4084, 0, 2500, 2144, 716, 0, 0, 4096, 11, 0, 0, 393, 149, 1797, 980, 13, 0, 3, 3391, 3965, 0, 28, 0, 3409, 865, 4, 3831, 4036, 0, 4096, 0, 0, 0, 0, 31, 1608, 0, 107, 0, 71, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[3504, 0, 4095, 0, 0, 0, 4063, 4, 0, 0, 3234, 2796, 0, 2, 0, 558, 0, 0, 885, 0, 3688, 0, 0, 4081, 0, 13, 10, 0, 78, 4032, 902, 1, 4, 3166, 3153, 461, 522, 0, 1975, 1231, 1040, 3668, 3927, 38, 35, 132, 1549, 0, 0, 1253, 9, 0, 4040, 0, 0, 2501, 400, 0, 0, 0, 384, 23, 0, 4079] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3863, 64, 0, 2, 0, 0, 9, 0, 0, 56, 0, 1411, 1, 4065, 273, 0, 0, 202, 387, 9, 2446, 3140, 0, 4096, 1921, 0, 0, 4, 124, 0, 0, 0, 0, 0, 149, 0, 3942, 4033, 0, 3934, 4096, 4, 0, 0, 4096, 4096, 0, 0, 3021, 4096, 4095, 372, 10, 9, 0, 0, 0, 0, 0, 3718, 3792, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2545, 0, 4090, 917, 0, 0, 23, 0, 82, 503, 594, 208, 0, 0, 0, 0, 595, 78, 3497, 56, 0, 28, 725, 0, 1644, 740, 420, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1798, 0, 16, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 4093, 0, 0, 4027, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4, 64, 0, 1, 2757, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 12, 0, 3234, 318, 2574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1579, 0, 1688, 0, 0, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 9, 66, 3934, 0, 2488, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8962, 7783, 5409, 11969, 10750, 7193, 6967, 6503] avg=8192.0000 max=11969 max_rank=3 min=5409 min_rank=2 imbalance_ratio=1.461060 max_load_reduction=53567 max_load_reduction_pct=0.817368 moved_to_echo=31413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[7, 315, 152, 3, 361, 2, 0, 0, 795, 455, 397, 210, 0, 0, 0, 0, 31, 0, 163, 0, 91, 2237, 0, 227, 2021, 495, 211, 0, 0, 0, 0, 0, 0, 0, 0, 63, 12, 294, 4056, 3252, 0, 0, 0, 0, 0, 0, 0, 0, 384, 3886, 3890, 903, 57, 1, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3925, 1625, 0, 1648, 3672, 1845, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 228, 3054, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3181, 2816, 307, 2075, 0, 121, 3685, 0, 0, 0, 0, 0, 0, 0, 0, 1380, 8, 0, 0, 57, 18, 1452, 0, 1254, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2697, 5476, 7677, 9650, 16285, 7397, 12185, 4169] avg=8192.0000 max=16285 max_rank=4 min=2697 min_rank=0 imbalance_ratio=1.987915 max_load_reduction=49251 max_load_reduction_pct=0.751511 moved_to_echo=40036 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[7, 4088, 3835, 0, 0, 0, 37, 0, 0, 495, 594, 210, 0, 0, 0, 0, 0, 0, 1, 4078, 0, 4071, 3, 3579, 1204, 740, 566, 0, 0, 0, 0, 0, 2, 1, 0, 0, 139, 0, 3702, 3536, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3886, 929, 0, 1524, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 3, 366, 2757, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2673, 1556, 56, 19, 0, 3533, 0, 0, 0, 0, 0, 0, 0, 0, 50, 3484, 3, 47, 1236, 2, 0, 3469, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 340, 0, 4096, 0, 0, 0, 3358, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9266, 14242, 7380, 6360, 4317, 7837, 8291, 7843] avg=8192.0000 max=14242 max_rank=1 min=4317 min_rank=4 imbalance_ratio=1.738525 max_load_reduction=51294 max_load_reduction_pct=0.782684 moved_to_echo=28288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3717, 0, 3920, 11, 0, 55, 429, 0, 704, 502, 1, 128, 0, 0, 0, 0, 6, 5, 6, 4002, 0, 3470, 1, 174, 2021, 2, 488, 0, 0, 0, 0, 0, 0, 448, 1525, 4062, 18, 0, 843, 3582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2372, 2144, 716, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 149, 1797, 980, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3965, 0, 28, 0, 2705, 865, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3831, 4036, 0, 2075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 1608, 0, 107, 0, 71, 516, 2903, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9467, 10175, 10478, 9339, 3329, 7566, 9946, 5236] avg=8192.0000 max=10478 max_rank=2 min=3329 min_rank=4 imbalance_ratio=1.279053 max_load_reduction=55058 max_load_reduction_pct=0.840118 moved_to_echo=26077 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3955, 252, 4096, 0, 11, 646, 29, 1, 742, 498, 595, 209, 0, 0, 0, 0, 0, 451, 3340, 0, 0, 1028, 2, 11, 0, 740, 586, 0, 0, 0, 0, 0, 2795, 24, 1, 0, 4089, 2, 45, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3861, 3083, 20, 155, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 2277, 1107, 0, 17, 2757, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 2848, 2844, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 3873, 1807, 1550, 452, 3480, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11034, 6158, 10516, 7126, 10514, 5744, 3214, 11230] avg=8192.0000 max=11230 max_rank=7 min=3214 min_rank=6 imbalance_ratio=1.370850 max_load_reduction=54306 max_load_reduction_pct=0.828644 moved_to_echo=30702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3244, 7, 0, 0, 0, 0, 846, 503, 565, 210, 0, 0, 0, 0, 0, 1, 0, 144, 0, 0, 0, 0, 1951, 702, 589, 0, 0, 0, 0, 0, 0, 0, 0, 3363, 2898, 0, 2, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3885, 2125, 1691, 2412, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 608, 0, 0, 2619, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 1256, 146, 5, 2, 3250, 3335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 2003, 1153, 5, 2356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 71, 148, 3816, 4085, 3499, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5376, 3387, 9856, 10124, 3641, 11957, 9542, 11653] avg=8192.0000 max=11957 max_rank=5 min=3387 min_rank=1 imbalance_ratio=1.459595 max_load_reduction=53579 max_load_reduction_pct=0.817551 moved_to_echo=36793 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3504, 0, 4095, 0, 0, 0, 4063, 4, 320, 501, 287, 0, 0, 0, 0, 0, 0, 0, 3234, 2796, 0, 2, 0, 558, 1993, 357, 528, 0, 0, 0, 0, 0, 0, 0, 885, 0, 3688, 0, 0, 3580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 10, 0, 78, 4032, 902, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3166, 3153, 461, 522, 0, 1331, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 0, 3927, 38, 35, 132, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 9, 0, 2047, 0, 0, 2501, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 384, 23, 0, 4079, 3140, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12774, 9468, 8153, 5036, 9868, 6401, 5810, 8026] avg=8192.0000 max=12774 max_rank=0 min=5036 min_rank=3 imbalance_ratio=1.559326 max_load_reduction=52762 max_load_reduction_pct=0.805084 moved_to_echo=30105 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3863, 64, 0, 2, 0, 0, 851, 390, 577, 213, 0, 0, 0, 0, 9, 0, 0, 56, 0, 1411, 1, 4065, 2022, 715, 568, 0, 0, 0, 0, 0, 273, 0, 0, 202, 387, 9, 2446, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 1921, 0, 0, 4, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 2650, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 3245, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3021, 4096, 2073, 372, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3718, 3792, 0, 3366, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5960, 8847, 6067, 5932, 6832, 11441, 9581, 10876] avg=8192.0000 max=11441 max_rank=5 min=5932 min_rank=3 imbalance_ratio=1.396606 max_load_reduction=54095 max_load_reduction_pct=0.825424 moved_to_echo=38730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 11648, 7936, 2432, 4224, 5120, 16768, 17792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[13824, 4736, 27392, 1024, 384, 768, 4608, 128, 4352, 3968, 3712, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 640, 10368, 11136, 128, 12288, 768, 8704, 12928, 4608, 3968, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 15872, 11520, 4480, 13312, 1536, 256, 18944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[9216, 0, 17024, 1920, 128, 3712, 16768, 17280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3072, 512, 2432, 7808, 13056, 384, 11136, 27520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 25728, 18304, 3456, 4352, 8192, 5760, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1920, 128, 2048, 128, 8704, 5888, 14720, 9216, 23552, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3, 4096, 78, 0, 0, 0, 4018, 1, 4, 0, 148, 1253, 0, 2045, 784, 0, 1330, 0, 674, 5, 1567, 1973, 4096, 3938, 1503, 3896, 1869, 2841, 1957, 0, 4094, 1, 3468, 0, 0, 0, 3262, 0, 2706, 0, 0, 198, 2493, 87, 0, 768, 0, 0, 1487, 1116, 0, 0, 1, 0, 3917, 0, 95, 3457, 0, 0, 241, 66, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 3400, 8, 1, 0, 0, 2, 0, 0, 0, 3984, 4096, 0, 1258, 0, 51, 0, 2, 238, 0, 0, 0, 1522, 1829, 4039, 21, 2, 4018, 0, 15, 4092, 4072, 0, 0, 1878, 0, 2787, 959, 4091, 38, 269, 0, 0, 4, 0, 10, 0, 427, 1903, 4095, 0, 0, 12, 0, 4073, 0, 0, 4096, 0, 4068, 0, 8, 267, 3901] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[207, 1, 3586, 17, 3, 6, 0, 41, 0, 0, 0, 3356, 0, 621, 0, 0, 0, 0, 0, 751, 0, 0, 0, 1865, 2002, 0, 0, 17, 9, 3620, 4090, 366, 2166, 1296, 165, 3964, 3587, 2383, 4021, 0, 0, 991, 1, 4, 0, 2071, 3668, 2627, 4096, 3562, 1192, 40, 492, 0, 3439, 1, 813, 2551, 0, 0, 1115, 703, 25, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 3351, 56, 3985, 1, 0, 0, 0, 4096, 0, 177, 2702, 149, 0, 3823, 0, 4052, 0, 0, 924, 3763, 3, 305, 0, 0, 4088, 0, 4096, 0, 0, 81, 0, 0, 0, 0, 4091, 4096, 12, 4082, 0, 8, 611, 0, 1, 0, 4020, 19, 0, 3250, 0, 0, 40, 309, 57, 4095, 0, 0, 0, 47, 721, 4076, 0, 349, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 71, 0, 0, 3147, 2, 1369, 0, 42, 593, 1598, 0, 311, 379, 2790, 0, 0, 442, 4085, 0, 1890, 0, 5, 84, 3316, 8, 4096, 2187, 1, 1516, 0, 0, 221, 15, 0, 4096, 3929, 3848, 2, 0, 1378, 5, 2189, 0, 1, 47, 0, 418, 4089, 0, 0, 2820, 0, 4096, 2, 1092, 2807, 1, 4096, 2181, 0, 142, 129] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 17, 0, 1, 0, 0, 104, 0, 4096, 0, 0, 3123, 99, 1, 0, 0, 112, 0, 781, 93, 2924, 0, 4, 0, 0, 79, 0, 2793, 163, 288, 4091, 3914, 0, 0, 16, 3915, 3944, 0, 4085, 78, 0, 8, 0, 0, 3, 431, 0, 0, 2956, 4075, 201, 0, 9, 20, 3993, 123, 1764, 4094, 1220, 3811, 4017, 0, 0, 4090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 85, 0, 3819, 9, 0, 0, 27, 60, 0, 541, 4096, 0, 7, 439, 252, 3830, 1323, 0, 3898, 3, 0, 1, 0, 0, 3900, 15, 0, 0, 0, 5, 2765, 337, 0, 4090, 4091, 4062, 0, 3492, 23, 327, 0, 0, 0, 0, 3042, 202, 4096, 4055, 0, 980, 0, 0, 32, 4096, 0, 0, 4096, 0, 10, 3430, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[22, 0, 696, 73, 2, 0, 102, 4096, 2438, 14, 0, 713, 0, 3, 0, 78, 3366, 0, 0, 1, 1, 1, 13, 1, 65, 4016, 0, 0, 0, 127, 4094, 0, 0, 80, 80, 4086, 4093, 2896, 3918, 10, 0, 2, 0, 1559, 1, 4073, 0, 3964, 34, 178, 229, 0, 17, 0, 3841, 51, 3299, 0, 1887, 0, 4090, 354, 2776, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3400, 8, 1, 0, 0, 2, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 4096, 0, 1258, 0, 51, 2308, 318, 0, 0, 0, 0, 0, 0, 0, 2, 238, 0, 0, 0, 1522, 1829, 1027, 489, 0, 0, 0, 0, 0, 0, 4039, 21, 2, 4018, 0, 15, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1878, 0, 1980, 959, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 269, 0, 0, 4, 0, 10, 0, 427, 4092, 515, 164, 0, 0, 0, 0, 0, 1903, 4095, 0, 0, 12, 0, 3046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1273, 0, 4068, 0, 8, 267, 3901, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7338, 12015, 5107, 12167, 4855, 5481, 9056, 9517] avg=8192.0000 max=12167 max_rank=3 min=4855 min_rank=4 imbalance_ratio=1.485229 max_load_reduction=53369 max_load_reduction_pct=0.814346 moved_to_echo=28909 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 4096, 78, 0, 0, 0, 4018, 1, 2597, 0, 0, 0, 0, 0, 0, 0, 4, 0, 148, 1253, 0, 2045, 784, 0, 1948, 373, 0, 0, 0, 0, 0, 0, 1330, 0, 674, 5, 1567, 1973, 4096, 3938, 987, 572, 143, 0, 0, 0, 0, 0, 1503, 3753, 1869, 2841, 1957, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 0, 0, 2317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 2493, 87, 0, 768, 0, 0, 4094, 435, 109, 0, 0, 0, 0, 0, 1487, 1116, 0, 0, 1, 0, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 1074, 0, 0, 241, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10793, 6555, 15285, 11924, 5785, 8184, 5534, 1476] avg=8192.0000 max=15285 max_rank=2 min=1476 min_rank=7 imbalance_ratio=1.865845 max_load_reduction=50251 max_load_reduction_pct=0.766769 moved_to_echo=20979 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3351, 56, 3985, 1, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 177, 2702, 149, 0, 3823, 0, 0, 468, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 924, 3763, 3, 305, 0, 1032, 719, 150, 0, 0, 0, 0, 0, 0, 3938, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2909, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 611, 0, 1, 0, 4020, 19, 0, 81, 0, 163, 0, 0, 0, 0, 0, 3250, 0, 0, 40, 309, 57, 3063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 721, 4076, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11312, 11415, 10948, 8034, 7012, 4903, 6719, 5193] avg=8192.0000 max=11415 max_rank=1 min=4903 min_rank=5 imbalance_ratio=1.393433 max_load_reduction=54121 max_load_reduction_pct=0.825821 moved_to_echo=23827 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 0, 1, 0, 0, 104, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3123, 99, 1, 0, 0, 2307, 451, 0, 0, 0, 0, 0, 0, 112, 0, 781, 93, 2924, 0, 4, 0, 1006, 692, 2, 0, 0, 0, 0, 0, 0, 77, 0, 2793, 163, 288, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3915, 2801, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3, 431, 0, 0, 4091, 515, 164, 0, 0, 0, 0, 0, 2956, 4075, 201, 0, 9, 20, 2987, 123, 0, 0, 0, 0, 0, 0, 0, 0, 1764, 1272, 1220, 3811, 4017, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 10077, 5614, 7235, 6810, 5212, 10371, 16174] avg=8192.0000 max=16174 max_rank=7 min=4043 min_rank=0 imbalance_ratio=1.974365 max_load_reduction=49362 max_load_reduction_pct=0.753204 moved_to_echo=38567 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 71, 0, 0, 3147, 2, 1369, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 42, 593, 1598, 0, 311, 379, 2790, 1581, 468, 0, 0, 0, 0, 0, 0, 0, 0, 442, 4085, 0, 1890, 0, 5, 1032, 719, 122, 0, 0, 0, 0, 0, 84, 3194, 8, 4096, 2187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 15, 0, 2909, 3929, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1378, 5, 2189, 0, 1, 47, 0, 1516, 353, 153, 0, 0, 0, 0, 0, 418, 4089, 0, 0, 2820, 0, 3064, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 873, 1, 4096, 2181, 0, 142, 129, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 7762, 8295, 9570, 7076, 5642, 10393, 8514] avg=8192.0000 max=10393 max_rank=6 min=5642 min_rank=5 imbalance_ratio=1.268677 max_load_reduction=55143 max_load_reduction_pct=0.841415 moved_to_echo=31625 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 696, 73, 2, 0, 102, 4096, 3761, 0, 0, 0, 0, 0, 0, 0, 2438, 14, 0, 713, 0, 3, 0, 78, 0, 468, 0, 0, 0, 0, 0, 0, 3366, 0, 0, 1, 1, 1, 13, 1, 968, 718, 148, 0, 0, 0, 0, 0, 65, 3868, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 4086, 2907, 2896, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1559, 1, 4073, 0, 3964, 4094, 0, 157, 0, 0, 0, 0, 0, 34, 178, 229, 0, 17, 0, 2873, 51, 0, 0, 0, 0, 0, 0, 0, 0, 3299, 0, 1887, 0, 4090, 354, 2776, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8752, 3714, 5217, 4060, 10059, 13850, 3382, 16502] avg=8192.0000 max=16502 max_rank=7 min=3382 min_rank=6 imbalance_ratio=2.014404 max_load_reduction=49034 max_load_reduction_pct=0.748199 moved_to_echo=43793 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[207, 1, 3586, 17, 3, 6, 0, 41, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 621, 0, 0, 1437, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 751, 0, 0, 0, 1865, 867, 629, 0, 0, 0, 0, 0, 0, 2002, 0, 0, 17, 9, 3620, 0, 366, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 1296, 165, 3964, 2548, 2383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1, 4, 0, 2071, 3668, 2627, 4090, 321, 161, 0, 0, 0, 0, 0, 4096, 3562, 1192, 40, 492, 0, 2572, 1, 0, 0, 0, 0, 0, 0, 0, 0, 813, 793, 0, 0, 1115, 703, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7721, 5824, 4112, 6014, 12522, 13934, 11955, 3454] avg=8192.0000 max=13934 max_rank=5 min=3454 min_rank=7 imbalance_ratio=1.700928 max_load_reduction=51602 max_load_reduction_pct=0.787384 moved_to_echo=41865 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 85, 0, 3819, 9, 0, 0, 27, 3353, 0, 0, 0, 0, 0, 0, 0, 60, 0, 541, 4096, 0, 7, 439, 252, 2311, 468, 0, 0, 0, 0, 0, 0, 3830, 1323, 0, 3898, 3, 0, 1, 0, 1037, 718, 148, 0, 0, 0, 0, 0, 0, 3752, 15, 0, 0, 0, 0, 2765, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 4090, 4091, 2876, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 0, 0, 3042, 202, 4096, 5, 519, 139, 0, 0, 0, 0, 0, 4055, 0, 980, 0, 0, 32, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1266, 0, 10, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7293, 8174, 10958, 6532, 11417, 8330, 8126, 4706] avg=8192.0000 max=11417 max_rank=4 min=4706 min_rank=7 imbalance_ratio=1.393677 max_load_reduction=54119 max_load_reduction_pct=0.825790 moved_to_echo=32579 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7168, 6656, 3200, 12800, 19200, 1152, 3584, 12288, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[12800, 1408, 2176, 9856, 8320, 3968, 6016, 7680, 8064, 5376, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[6016, 1664, 6272, 20224, 21248, 10240, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[18304, 17152, 2688, 128, 3712, 128, 23680, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[10752, 128, 5504, 20992, 256, 4352, 5504, 3200, 11904, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 11008, 4608, 7936, 128, 3200, 4352, 5632, 29056, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7808, 18688, 1920, 17920, 4352, 4096, 0, 11136, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 3200, 2560, 3968, 128, 14464, 3968, 11136, 22144, 2688, 1280, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[986, 0, 0, 0, 0, 145, 3956, 3827, 154, 4043, 1, 0, 6, 0, 21, 4, 10, 0, 0, 0, 170, 3976, 1, 4093, 5, 0, 0, 4093, 346, 0, 4096, 0, 24, 33, 2949, 0, 73, 0, 0, 4080, 0, 22, 1073, 4096, 2, 4069, 0, 720, 0, 0, 0, 91, 2584, 3637, 81, 1818, 2964, 4017, 3, 0, 0, 0, 3267, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1819, 1041, 0, 3939, 163, 3375, 0, 1693, 0, 0, 4086, 0, 0, 4095, 4055, 20, 332, 4092, 0, 0, 195, 111, 0, 0, 1, 4, 0, 4041, 4094, 4, 3, 3505, 4091, 1298, 0, 0, 16, 4063, 0, 0, 0, 4095, 3, 5, 0, 4096, 5, 19, 4, 0, 911, 2771, 3446, 5, 0, 0, 3, 37, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[785, 367, 21, 0, 0, 0, 0, 3640, 89, 0, 29, 1898, 3059, 0, 0, 4043, 0, 13, 1, 4041, 295, 1643, 0, 3987, 1, 9, 2122, 19, 0, 3646, 139, 0, 3935, 0, 0, 0, 6, 0, 548, 596, 3821, 445, 148, 1967, 0, 0, 0, 4071, 133, 0, 1247, 119, 43, 4096, 3693, 0, 0, 14, 3507, 2486, 3663, 1116, 0, 35] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4057, 0, 45, 1, 4096, 4081, 820, 3762, 0, 0, 0, 114, 2, 552, 27, 7, 1962, 2, 0, 3994, 0, 4096, 0, 377, 0, 299, 3814, 0, 3078, 216, 3, 0, 0, 0, 90, 0, 1988, 3946, 299, 323, 0, 3705, 0, 3148, 0, 823, 0, 6, 0, 68, 0, 4096, 0, 0, 73, 3, 1, 0, 0, 3569, 4009, 3984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 3050, 4083, 11, 0, 4094, 3996, 4096, 0, 3028, 65, 0, 0, 0, 9, 324, 0, 3337, 0, 3246, 3, 0, 0, 4082, 0, 19, 4086, 0, 3916, 0, 0, 0, 90, 4092, 1715, 0, 0, 0, 3145, 0, 0, 941, 0, 181, 896, 711, 15, 0, 0, 4, 0, 28, 0, 956, 50, 0, 60, 0, 3761, 1, 4086, 1, 0, 3358] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1866, 0, 0, 4096, 126, 3261, 370, 4091, 6, 0, 1337, 3, 0, 0, 0, 0, 232, 4091, 349, 4, 591, 4051, 0, 169, 3184, 0, 514, 0, 3864, 0, 0, 4096, 0, 1, 0, 0, 0, 3871, 25, 0, 3657, 3933, 7, 259, 0, 4096, 0, 0, 0, 0, 0, 2257, 3789, 600, 0, 456, 1, 0, 3038, 1, 3202, 42] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 156, 0, 2297, 0, 52, 3703, 4091, 0, 370, 2968, 1, 1417, 0, 0, 0, 0, 4096, 0, 0, 0, 3177, 0, 4080, 535, 3737, 0, 0, 0, 0, 4096, 0, 0, 4096, 3912, 82, 281, 31, 3605, 3271, 0, 0, 0, 229, 0, 4070, 0, 3859, 208, 610, 0, 0, 0, 1766, 0, 0, 1, 0, 20, 1481, 2360, 0, 0, 878] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[26, 1, 3, 1939, 0, 4083, 4065, 3682, 20, 4096, 2486, 0, 0, 1, 0, 101, 1, 4092, 60, 3768, 421, 3652, 0, 4062, 0, 795, 12, 520, 4052, 0, 3270, 10, 0, 0, 19, 4, 46, 0, 4062, 2330, 1, 0, 0, 4096, 0, 1425, 0, 0, 0, 0, 21, 0, 0, 4067, 0, 178, 0, 0, 0, 14, 0, 0, 4038, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[986, 0, 0, 0, 0, 145, 3956, 825, 0, 0, 0, 0, 0, 0, 0, 0, 154, 4043, 1, 0, 6, 0, 21, 4, 2137, 185, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 170, 3976, 1, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4093, 346, 0, 4096, 0, 433, 0, 0, 0, 0, 0, 0, 0, 24, 33, 2949, 0, 73, 0, 0, 3260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 1073, 4096, 2, 4069, 0, 720, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 2584, 3637, 81, 1818, 2817, 0, 0, 0, 0, 0, 0, 0, 2964, 4017, 3, 0, 0, 0, 3267, 0, 582, 350, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5912, 6551, 5531, 8973, 6339, 10019, 11028, 11183] avg=8192.0000 max=11183 max_rank=7 min=5531 min_rank=2 imbalance_ratio=1.365112 max_load_reduction=54353 max_load_reduction_pct=0.829361 moved_to_echo=38569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3050, 4083, 11, 0, 4094, 3996, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3028, 65, 0, 0, 0, 9, 324, 2131, 198, 0, 0, 0, 0, 0, 0, 0, 3337, 0, 3246, 3, 0, 0, 1370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4086, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 4092, 1715, 0, 0, 0, 3145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 0, 181, 896, 711, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 28, 0, 956, 50, 0, 3016, 0, 0, 0, 0, 0, 0, 0, 60, 0, 3761, 1, 4086, 1, 0, 3358, 581, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16116, 5755, 7956, 8021, 9042, 2744, 4054, 11848] avg=8192.0000 max=16116 max_rank=0 min=2744 min_rank=5 imbalance_ratio=1.967285 max_load_reduction=49420 max_load_reduction_pct=0.754089 moved_to_echo=27688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[785, 367, 21, 0, 0, 0, 0, 784, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 29, 1898, 3059, 0, 0, 4043, 2082, 176, 0, 0, 0, 0, 0, 0, 0, 13, 1, 4041, 295, 1643, 0, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 2122, 19, 0, 3646, 139, 0, 63, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 6, 0, 548, 477, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 445, 148, 1967, 0, 0, 0, 4071, 5, 0, 0, 0, 0, 0, 0, 0, 133, 0, 1247, 119, 43, 4096, 3693, 0, 2680, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3507, 2486, 3663, 1116, 0, 35, 567, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1957, 11376, 7331, 5999, 4966, 10457, 12011, 11439] avg=8192.0000 max=12011 max_rank=6 min=1957 min_rank=0 imbalance_ratio=1.466187 max_load_reduction=53525 max_load_reduction_pct=0.816727 moved_to_echo=38873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1819, 1041, 0, 3939, 163, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 0, 0, 4086, 0, 0, 4095, 57, 163, 0, 0, 0, 0, 0, 0, 4055, 20, 332, 4092, 0, 0, 195, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4041, 4094, 4, 431, 0, 0, 0, 0, 0, 0, 0, 3, 3505, 4091, 1298, 0, 0, 16, 3247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 5, 0, 4096, 37, 0, 0, 0, 0, 0, 0, 0, 5, 19, 4, 0, 911, 2771, 3446, 5, 2485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 37, 0, 0, 0, 0, 15, 348, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7689, 10094, 8733, 8575, 12160, 8236, 9646, 403] avg=8192.0000 max=12160 max_rank=4 min=403 min_rank=7 imbalance_ratio=1.484375 max_load_reduction=53376 max_load_reduction_pct=0.814453 moved_to_echo=30445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4057, 0, 45, 1, 4096, 880, 0, 0, 0, 0, 0, 0, 0, 0, 820, 3762, 0, 0, 0, 114, 2, 552, 2139, 197, 0, 0, 0, 0, 0, 0, 27, 7, 1962, 2, 0, 3994, 0, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 299, 3814, 0, 3078, 216, 419, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 90, 0, 1988, 3153, 0, 0, 0, 0, 0, 0, 0, 0, 299, 323, 0, 3705, 0, 3148, 0, 823, 36, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 68, 0, 4096, 0, 0, 3004, 0, 0, 0, 0, 0, 0, 0, 73, 3, 1, 0, 0, 3569, 4009, 3984, 583, 338, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9079, 7586, 7366, 8203, 5234, 8334, 7174, 12560] avg=8192.0000 max=12560 max_rank=7 min=5234 min_rank=4 imbalance_ratio=1.533203 max_load_reduction=52976 max_load_reduction_pct=0.808350 moved_to_echo=33302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1866, 0, 0, 4096, 126, 703, 0, 0, 0, 0, 0, 0, 0, 0, 370, 4091, 6, 0, 1337, 3, 0, 0, 2115, 157, 0, 0, 0, 0, 0, 0, 0, 0, 232, 4091, 349, 4, 591, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 3184, 0, 514, 0, 3864, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 3657, 3933, 7, 259, 0, 4096, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2257, 3789, 600, 2401, 0, 0, 0, 0, 0, 0, 0, 0, 456, 1, 0, 3038, 1, 3202, 42, 576, 332, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6791, 8079, 6627, 8142, 7190, 12012, 9047, 7648] avg=8192.0000 max=12012 max_rank=5 min=6627 min_rank=2 imbalance_ratio=1.466309 max_load_reduction=53524 max_load_reduction_pct=0.816711 moved_to_echo=35897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 156, 0, 2297, 0, 52, 3703, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 2968, 1, 1417, 0, 0, 0, 2134, 202, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3177, 0, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 535, 3737, 0, 0, 0, 0, 4096, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3912, 82, 281, 31, 3605, 2605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 4070, 0, 3859, 33, 0, 0, 0, 0, 0, 0, 0, 208, 610, 0, 0, 0, 1766, 0, 0, 3015, 0, 0, 0, 0, 0, 0, 0, 1, 0, 20, 1481, 2360, 0, 0, 878, 584, 284, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7082, 7092, 8635, 8717, 14612, 8191, 5599, 5608] avg=8192.0000 max=14612 max_rank=4 min=5599 min_rank=6 imbalance_ratio=1.783691 max_load_reduction=50924 max_load_reduction_pct=0.777039 moved_to_echo=34010 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[26, 1, 3, 1939, 0, 4083, 4065, 793, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 2486, 0, 0, 1, 0, 101, 2121, 178, 0, 0, 0, 0, 0, 0, 1, 4092, 60, 3768, 421, 3652, 0, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 12, 520, 4052, 0, 3270, 10, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4, 46, 0, 4062, 1862, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 1425, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 4067, 0, 178, 2711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4038, 17, 578, 200, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10910, 9003, 13357, 8906, 5993, 5543, 6977, 4847] avg=8192.0000 max=13357 max_rank=2 min=4847 min_rank=7 imbalance_ratio=1.630493 max_load_reduction=52179 max_load_reduction_pct=0.796188 moved_to_echo=23360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3200, 4608, 7296, 4096, 13184, 4736, 14592, 8320, 4096, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1920, 3584, 11904, 5376, 128, 16512, 20224, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 1792, 4992, 22400, 1024, 13696, 128, 17792, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4096, 15872, 12800, 1408, 512, 128, 13440, 17792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 5120, 9472, 4992, 12672, 7808, 22656, 256, 2432, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4096, 11648, 2688, 19328, 1280, 16512, 896, 9600, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1536, 21120, 5632, 1920, 9984, 128, 128, 9216, 14976, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[384, 640, 1280, 384, 3584, 23680, 11136, 2688, 22144, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 2256, 0, 0, 96, 4096, 185, 0, 0, 0, 3850, 3015, 0, 35, 1, 0, 2909, 145, 0, 4095, 4093, 3, 716, 81, 143, 0, 3966, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1375, 3527, 646, 3828, 0, 835, 2974, 0, 0, 4056, 2154, 4044, 905, 0, 0, 0, 0, 0, 0, 43, 25, 3300, 0, 0, 4014, 26] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[2072, 793, 2241, 101, 5, 2715, 0, 0, 0, 4, 3921, 3689, 2, 4010, 363, 314, 131, 0, 1710, 1530, 3193, 1198, 0, 0, 2, 0, 4088, 7, 4007, 2223, 4096, 0, 0, 0, 1034, 0, 3151, 636, 4092, 24, 0, 4078, 4, 2, 72, 2874, 0, 0, 2397, 0, 0, 3331, 20, 0, 216, 1173, 0, 0, 2, 0, 15, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[30, 0, 14, 0, 2849, 4060, 2, 0, 305, 0, 2, 3646, 15, 0, 5, 6, 3063, 0, 0, 3248, 2739, 0, 200, 4096, 3, 0, 4095, 2466, 0, 4067, 553, 3961, 0, 0, 0, 0, 432, 768, 3656, 1552, 3994, 3, 135, 4058, 0, 2, 898, 2200, 1381, 1, 0, 1370, 0, 0, 3993, 0, 2, 178, 0, 33, 0, 1454, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[779, 0, 3700, 40, 4096, 0, 3987, 0, 0, 238, 0, 3084, 0, 3955, 117, 84, 2, 15, 2939, 0, 4026, 0, 3, 2, 0, 3813, 4093, 0, 4096, 3786, 451, 181, 3, 0, 0, 0, 3035, 0, 1251, 0, 0, 0, 83, 16, 3820, 0, 488, 0, 0, 52, 0, 4096, 0, 0, 1658, 0, 0, 0, 3995, 0, 2, 23, 3527, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3, 150, 3180, 0, 0, 4084, 0, 4071, 0, 9, 1869, 7, 273, 4096, 0, 193, 20, 12, 0, 0, 884, 4095, 4, 0, 4067, 103, 0, 0, 45, 4096, 23, 0, 0, 1879, 0, 2267, 4041, 0, 4096, 4040, 3285, 3919, 5, 0, 0, 2, 0, 2094, 4006, 234, 0, 398, 0, 0, 3400, 0, 0, 0, 469, 0, 0, 15, 102, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[2517, 0, 4095, 3369, 0, 2876, 0, 1508, 0, 0, 2, 0, 712, 3768, 1, 11, 1, 0, 231, 0, 3943, 1135, 0, 0, 2302, 4096, 4075, 0, 94, 124, 4091, 30, 23, 1287, 2, 3935, 2, 0, 2, 91, 0, 4096, 2, 4096, 0, 2658, 0, 124, 2, 1, 2, 19, 0, 42, 4076, 1, 0, 0, 2165, 0, 0, 0, 3929, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[17, 4071, 0, 2, 0, 3687, 0, 28, 1, 1368, 4096, 892, 0, 4022, 2384, 0, 2, 0, 4092, 0, 4076, 453, 11, 0, 7, 19, 3, 0, 474, 2358, 4094, 0, 427, 0, 0, 1, 4096, 0, 4062, 0, 0, 2467, 22, 0, 0, 4016, 2276, 3889, 5, 21, 176, 0, 0, 260, 3853, 15, 0, 0, 144, 3553, 0, 83, 12, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1837, 1087, 2136, 723, 18, 1495, 22, 72, 0, 0, 4, 0, 0, 178, 0, 3959, 0, 0, 4096, 4075, 4096, 1, 0, 2953, 0, 3882, 4061, 0, 3030, 0, 0, 0, 0, 246, 0, 150, 0, 0, 0, 2716, 0, 132, 339, 4096, 164, 3597, 89, 5, 4092, 36, 2, 3945, 0, 0, 2940, 0, 0, 7, 3965, 1289, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[30, 0, 14, 0, 2849, 2362, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 2, 3646, 15, 0, 5, 6, 657, 94, 0, 0, 0, 0, 0, 0, 3063, 0, 0, 3248, 1851, 0, 200, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2466, 0, 3760, 553, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 768, 3656, 1552, 231, 307, 0, 0, 0, 0, 0, 0, 3994, 2, 135, 4058, 0, 2, 898, 2200, 0, 0, 0, 0, 0, 0, 0, 0, 1381, 1, 0, 1370, 0, 0, 3993, 0, 1604, 0, 0, 0, 0, 0, 0, 0, 2, 178, 0, 33, 0, 1454, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5257, 4730, 12458, 10743, 6946, 11289, 8349, 5764] avg=8192.0000 max=12458 max_rank=2 min=4730 min_rank=1 imbalance_ratio=1.520752 max_load_reduction=53078 max_load_reduction_pct=0.809906 moved_to_echo=32348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[779, 0, 3700, 40, 4096, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 3084, 0, 3955, 117, 84, 966, 0, 0, 0, 0, 0, 0, 0, 2, 15, 2939, 0, 2721, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3813, 0, 0, 4096, 3500, 451, 181, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3035, 0, 1251, 0, 339, 286, 0, 0, 0, 0, 0, 0, 0, 0, 83, 16, 3820, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4096, 0, 0, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 2, 23, 3527, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12602, 8444, 5682, 12041, 4914, 4407, 5806, 11640] avg=8192.0000 max=12602 max_rank=0 min=4407 min_rank=5 imbalance_ratio=1.538330 max_load_reduction=52934 max_load_reduction_pct=0.807709 moved_to_echo=26767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[17, 4071, 0, 2, 0, 2145, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1368, 4096, 892, 0, 4022, 2384, 0, 978, 85, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 0, 2754, 453, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 19, 0, 0, 474, 2180, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 1, 4096, 0, 4062, 0, 344, 178, 0, 0, 0, 0, 0, 0, 0, 1016, 22, 0, 0, 4016, 2276, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 5, 21, 176, 0, 0, 260, 3853, 15, 1457, 282, 0, 0, 0, 0, 0, 0, 0, 0, 144, 3553, 0, 83, 12, 1, 3, 1169, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6263, 13826, 7312, 6774, 9108, 11219, 6069, 4965] avg=8192.0000 max=13826 max_rank=1 min=4965 min_rank=7 imbalance_ratio=1.687744 max_load_reduction=51710 max_load_reduction_pct=0.789032 moved_to_echo=31361 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2256, 0, 0, 96, 2384, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 3015, 0, 35, 1, 0, 983, 94, 0, 0, 0, 0, 0, 0, 2909, 145, 0, 4095, 2765, 3, 716, 81, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1375, 3527, 345, 0, 0, 0, 0, 0, 0, 0, 646, 1576, 0, 835, 2974, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 2154, 4044, 905, 0, 0, 0, 0, 0, 1618, 438, 0, 0, 0, 0, 0, 0, 0, 43, 25, 3300, 0, 0, 4014, 26, 3966, 1814, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4921, 7978, 10714, 149, 9340, 10087, 9159, 13188] avg=8192.0000 max=13188 max_rank=7 min=149 min_rank=3 imbalance_ratio=1.609863 max_load_reduction=52348 max_load_reduction_pct=0.798767 moved_to_echo=41774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2072, 793, 2241, 101, 5, 1580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3921, 3689, 2, 4010, 363, 314, 766, 62, 0, 0, 0, 0, 0, 0, 131, 0, 1710, 1530, 2158, 1198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 4007, 2055, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 0, 3151, 636, 4092, 24, 269, 168, 0, 0, 0, 0, 0, 0, 0, 1679, 4, 2, 72, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2397, 0, 0, 3331, 20, 0, 216, 1173, 1073, 467, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 15, 0, 0, 0, 4088, 1932, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6792, 13131, 6727, 10167, 9374, 4631, 8677, 6037] avg=8192.0000 max=13131 max_rank=1 min=4631 min_rank=5 imbalance_ratio=1.602905 max_load_reduction=52405 max_load_reduction_pct=0.799637 moved_to_echo=28719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 150, 3180, 0, 0, 2376, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1869, 7, 273, 4096, 0, 193, 212, 94, 0, 0, 0, 0, 0, 0, 20, 12, 0, 0, 598, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 103, 0, 0, 45, 3787, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1879, 0, 2267, 4041, 0, 4096, 4040, 74, 309, 0, 0, 0, 0, 0, 0, 3285, 1613, 5, 0, 0, 2, 0, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 234, 0, 398, 0, 0, 3400, 0, 1614, 449, 0, 0, 0, 0, 0, 0, 0, 0, 469, 0, 0, 15, 102, 0, 0, 1857, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9780, 6753, 4729, 8025, 16706, 6999, 10101, 2443] avg=8192.0000 max=16706 max_rank=4 min=2443 min_rank=7 imbalance_ratio=2.039307 max_load_reduction=48830 max_load_reduction_pct=0.745087 moved_to_echo=36249 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1837, 1087, 2136, 723, 18, 871, 22, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 178, 0, 3959, 983, 34, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 2768, 1, 0, 2953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3882, 0, 0, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 150, 0, 0, 0, 2716, 345, 0, 0, 0, 0, 0, 0, 0, 0, 55, 339, 4096, 164, 3597, 89, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 36, 2, 3945, 0, 0, 2940, 0, 590, 15, 0, 0, 0, 0, 0, 0, 0, 7, 3965, 1289, 0, 0, 0, 1, 4061, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6766, 5158, 13893, 6912, 3457, 8345, 11620, 9385] avg=8192.0000 max=13893 max_rank=2 min=3457 min_rank=4 imbalance_ratio=1.695923 max_load_reduction=51643 max_load_reduction_pct=0.788010 moved_to_echo=32807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2517, 0, 4095, 3369, 0, 1666, 0, 1508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 712, 3768, 1, 11, 952, 70, 0, 0, 0, 0, 0, 0, 1, 0, 231, 0, 2654, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2302, 4096, 0, 0, 94, 112, 4091, 30, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1287, 2, 3935, 2, 0, 2, 91, 337, 12, 0, 0, 0, 0, 0, 0, 0, 1679, 2, 4096, 0, 2658, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 19, 0, 42, 4076, 1, 1140, 472, 0, 0, 0, 0, 0, 0, 0, 0, 2165, 0, 0, 0, 3929, 0, 4075, 1945, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13155, 5516, 4021, 10725, 5691, 8559, 5755, 12114] avg=8192.0000 max=13155 max_rank=0 min=4021 min_rank=2 imbalance_ratio=1.605835 max_load_reduction=52381 max_load_reduction_pct=0.799271 moved_to_echo=32119 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[6528, 12032, 0, 2560, 11776, 15488, 13312, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7936, 7680, 640, 13184, 7040, 13184, 3840, 12416, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6144, 256, 13184, 13056, 18304, 6912, 1024, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 1664, 13824, 14336, 1024, 20096, 2944, 4608, 6528, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 256, 10880, 8192, 128, 1664, 11648, 128, 24448, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7296, 8448, 15488, 4352, 7168, 13440, 4224, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 3456, 1152, 6400, 18944, 1408, 18560, 12032, 2304, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[14080, 4480, 1152, 13184, 128, 384, 20224, 1280, 9216, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[524, 0, 0, 4081, 1109, 0, 0, 1, 2563, 1090, 0, 4048, 527, 5, 550, 3382, 3890, 2915, 0, 0, 0, 31, 4096, 70, 0, 0, 3, 4096, 0, 2, 0, 4034, 0, 12, 131, 4090, 4096, 0, 40, 521, 0, 0, 0, 0, 0, 3991, 0, 13, 3229, 3719, 3225, 3044, 0, 0, 58, 2, 0, 0, 0, 74, 9, 1480, 0, 785] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[3019, 0, 4094, 1, 3927, 124, 16, 1, 4050, 1, 0, 4091, 2, 0, 112, 0, 4048, 4039, 4060, 0, 1648, 0, 175, 22, 0, 1192, 3950, 4069, 2744, 3987, 1, 0, 0, 5, 1, 393, 1997, 203, 44, 4096, 0, 0, 0, 0, 0, 0, 838, 0, 4092, 10, 0, 497, 227, 0, 0, 0, 85, 10, 2686, 0, 31, 948, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4085, 0, 362, 30, 4060, 4096, 69, 0, 0, 0, 2426, 3668, 4, 4080, 256, 2, 0, 4087, 0, 0, 21, 0, 137, 2981, 3845, 3618, 0, 0, 4096, 1, 4095, 1575, 0, 467, 0, 71, 0, 1, 0, 0, 0, 0, 4088, 0, 42, 937, 0, 84, 4053, 0, 0, 0, 680, 0, 801, 64, 3906, 42, 2693, 0, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1314, 0, 4077, 2729, 0, 3, 0, 4095, 1, 0, 0, 2232, 205, 2621, 0, 4095, 4096, 5, 3337, 2, 328, 0, 0, 30, 0, 0, 30, 4082, 0, 1344, 107, 2721, 0, 0, 0, 0, 4093, 3373, 0, 0, 180, 0, 0, 1, 39, 0, 0, 50, 3636, 346, 365, 4096, 0, 14, 2030, 4095, 0, 1, 20, 4095, 1567, 7, 72, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[3390, 0, 0, 3576, 0, 671, 147, 0, 3974, 0, 0, 0, 0, 4076, 0, 4096, 4094, 50, 3556, 0, 15, 0, 0, 0, 0, 52, 0, 2651, 0, 0, 0, 4046, 0, 6, 0, 1940, 1362, 2949, 0, 0, 3932, 4095, 631, 4, 0, 0, 2, 17, 3221, 0, 1, 4, 147, 0, 5, 101, 4076, 0, 469, 4088, 0, 4090, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[3891, 1, 0, 0, 0, 3444, 11, 0, 0, 127, 0, 0, 0, 2974, 424, 3621, 1790, 4096, 651, 270, 0, 8, 601, 0, 0, 77, 3732, 4096, 0, 0, 4, 2768, 0, 0, 0, 0, 4096, 1, 3969, 35, 4066, 1692, 0, 0, 0, 4058, 2171, 288, 77, 0, 0, 3835, 635, 7, 1, 0, 12, 1236, 158, 2454, 0, 3728, 73, 358] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3781, 4047, 8, 4, 435, 4090, 3903, 0, 0, 1, 404, 3034, 633, 4096, 3706, 6, 1788, 0, 130, 4, 0, 0, 0, 0, 3318, 4090, 0, 0, 2, 4090, 0, 9, 29, 0, 84, 3329, 0, 1058, 0, 0, 108, 0, 0, 1, 0, 1, 2254, 9, 0, 3968, 0, 298, 37, 3231, 41, 0, 4093, 3984, 62, 1370, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[3978, 0, 643, 4019, 4028, 0, 0, 63, 0, 0, 1216, 2877, 0, 2478, 174, 3293, 586, 0, 0, 0, 327, 149, 0, 294, 949, 0, 0, 2737, 52, 0, 23, 3569, 1446, 0, 1373, 84, 0, 4091, 1, 3824, 109, 991, 1, 0, 0, 3429, 0, 0, 4096, 0, 0, 3168, 716, 2947, 4, 0, 17, 45, 3700, 26, 0, 3965, 48, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3019, 0, 4094, 1, 3927, 124, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 1, 0, 4091, 2, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 4039, 4060, 0, 1648, 0, 175, 22, 36, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 3950, 1634, 2744, 3987, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 393, 1997, 203, 44, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 0, 2435, 0, 0, 0, 0, 0, 0, 0, 4092, 10, 0, 461, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 10, 2686, 0, 31, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11182, 8256, 14028, 13508, 6739, 3273, 4790, 3760] avg=8192.0000 max=14028 max_rank=2 min=3273 min_rank=5 imbalance_ratio=1.712402 max_load_reduction=51508 max_load_reduction_pct=0.785950 moved_to_echo=18562 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1850, 0, 362, 30, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 0, 0, 0, 2426, 3668, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 256, 2, 0, 4087, 0, 0, 21, 297, 92, 0, 0, 0, 0, 0, 0, 0, 137, 2981, 1544, 3618, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1575, 0, 467, 0, 71, 0, 1186, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4088, 0, 42, 2301, 2, 0, 0, 0, 0, 0, 0, 937, 0, 84, 3756, 0, 0, 0, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 64, 3906, 42, 2693, 0, 13, 957, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6302, 10261, 8835, 12376, 7395, 6434, 5457, 8476] avg=8192.0000 max=12376 max_rank=3 min=5457 min_rank=6 imbalance_ratio=1.510742 max_load_reduction=53160 max_load_reduction_pct=0.811157 moved_to_echo=27762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[524, 0, 0, 1848, 1109, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2563, 1090, 0, 4048, 527, 5, 550, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 2915, 0, 0, 0, 31, 4096, 70, 223, 92, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1645, 0, 2, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 131, 4090, 4096, 0, 40, 521, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 13, 2451, 1937, 0, 0, 0, 0, 0, 0, 3229, 3719, 3225, 2821, 0, 0, 58, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 9, 1480, 0, 785, 956, 414, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3482, 9814, 11317, 5684, 10075, 8392, 13054, 3718] avg=8192.0000 max=13054 max_rank=6 min=3482 min_rank=0 imbalance_ratio=1.593506 max_load_reduction=52482 max_load_reduction_pct=0.800812 moved_to_echo=35239 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1314, 0, 4077, 1225, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2232, 205, 2621, 0, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 3337, 2, 328, 0, 0, 30, 305, 65, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1633, 0, 1344, 107, 2721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3373, 0, 0, 795, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 1, 39, 0, 0, 50, 2449, 2347, 0, 0, 0, 0, 0, 0, 3636, 346, 365, 3791, 0, 14, 2030, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 4095, 1567, 7, 72, 2, 644, 503, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10714, 6304, 8168, 5835, 8261, 5066, 14277, 6911] avg=8192.0000 max=14277 max_rank=6 min=5066 min_rank=5 imbalance_ratio=1.742798 max_load_reduction=51259 max_load_reduction_pct=0.782150 moved_to_echo=34515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3891, 1, 0, 0, 0, 3444, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 2974, 424, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 1790, 4096, 651, 270, 0, 8, 601, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 77, 3732, 1645, 0, 0, 4, 2768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3969, 35, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 1692, 0, 0, 0, 4058, 2171, 288, 2451, 2074, 0, 0, 0, 0, 0, 0, 77, 0, 0, 3554, 635, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1236, 158, 2454, 0, 3728, 73, 358, 0, 443, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7347, 4629, 7697, 8226, 8101, 16800, 4274, 8462] avg=8192.0000 max=16800 max_rank=5 min=4274 min_rank=6 imbalance_ratio=2.050781 max_load_reduction=48736 max_load_reduction_pct=0.743652 moved_to_echo=37637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3781, 1833, 8, 4, 435, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 0, 0, 1, 404, 3034, 633, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 3706, 6, 1788, 0, 130, 4, 0, 0, 291, 91, 0, 0, 0, 0, 0, 0, 0, 0, 3318, 1642, 0, 0, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 29, 0, 84, 3329, 0, 1058, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 1, 0, 1, 2448, 2346, 0, 0, 0, 0, 0, 0, 2254, 9, 0, 3677, 0, 298, 37, 3231, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 4093, 3984, 62, 1370, 0, 0, 948, 501, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10151, 9224, 6016, 9052, 5684, 4904, 9506, 10999] avg=8192.0000 max=10999 max_rank=7 min=4904 min_rank=5 imbalance_ratio=1.342651 max_load_reduction=54537 max_load_reduction_pct=0.832169 moved_to_echo=31093 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3390, 0, 0, 1619, 0, 671, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 4076, 0, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 50, 3556, 0, 15, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 52, 0, 1065, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1940, 1362, 2949, 0, 0, 1038, 0, 0, 0, 0, 0, 0, 0, 3932, 4095, 631, 4, 0, 0, 2, 17, 1586, 2346, 0, 0, 0, 0, 0, 0, 3221, 0, 1, 4, 147, 0, 5, 101, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 469, 4088, 0, 4090, 2, 0, 838, 501, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5827, 9299, 7796, 5163, 7295, 12613, 3479, 14064] avg=8192.0000 max=14064 max_rank=7 min=3479 min_rank=6 imbalance_ratio=1.716797 max_load_reduction=51472 max_load_reduction_pct=0.785400 moved_to_echo=37451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3978, 0, 643, 1819, 4028, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 2877, 0, 2478, 174, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 0, 327, 149, 0, 294, 232, 91, 0, 0, 0, 0, 0, 0, 949, 0, 0, 1099, 52, 0, 23, 3569, 0, 0, 0, 0, 0, 0, 0, 0, 1446, 0, 1373, 84, 0, 4091, 1, 3824, 1167, 0, 0, 0, 0, 0, 0, 0, 109, 991, 1, 0, 0, 3429, 0, 0, 1638, 1886, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2936, 716, 2947, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 45, 3700, 26, 0, 3965, 48, 0, 942, 403, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10531, 7749, 1679, 5692, 11986, 8054, 10699, 9146] avg=8192.0000 max=11986 max_rank=4 min=1679 min_rank=2 imbalance_ratio=1.463135 max_load_reduction=53550 max_load_reduction_pct=0.817108 moved_to_echo=39885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 6784, 768, 128, 128, 15616, 3072, 512, 17792, 13056, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4352, 2176, 11264, 18688, 1792, 18304, 256, 1280, 5376, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[16128, 128, 12672, 10240, 9088, 4608, 640, 12416, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[21632, 4096, 3712, 21120, 1792, 3328, 2176, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[18688, 1408, 1280, 13312, 1152, 17664, 5632, 6912, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[26368, 11392, 13440, 384, 6656, 256, 4992, 512, 1792, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1536, 4224, 3200, 6528, 16256, 13952, 4224, 9600, 6656, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 1536, 14080, 12032, 6528, 5376, 256, 25344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4066, 0, 15, 3517, 0, 0, 303, 87, 0, 6, 0, 3353, 0, 28, 0, 132, 4087, 0, 0, 1, 3721, 3, 4050, 3, 0, 3, 0, 3064, 40, 362, 4079, 376, 3361, 2032, 38, 4095, 4, 4080, 0, 2557, 1118, 126, 0, 79, 7, 0, 0, 12, 0, 0, 12, 0, 4094, 1161, 0, 3944, 0, 180, 0, 4016, 523, 2801, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4043, 0, 0, 1986, 3, 0, 0, 0, 155, 0, 0, 41, 18, 0, 0, 12, 2473, 8, 0, 0, 4075, 2537, 4060, 3925, 0, 2195, 305, 0, 4047, 0, 0, 0, 1313, 4094, 4, 1, 2, 4096, 0, 22, 0, 3873, 15, 0, 4094, 52, 1236, 0, 1, 143, 4001, 0, 0, 0, 0, 0, 0, 69, 4088, 1, 4073, 379, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[2574, 0, 4088, 0, 0, 1, 0, 0, 942, 0, 4, 2, 0, 0, 0, 281, 87, 0, 5, 10, 62, 2944, 0, 3789, 3821, 4092, 12, 0, 138, 0, 496, 8, 23, 4094, 0, 3, 110, 0, 3009, 3, 0, 4074, 193, 2324, 0, 0, 0, 0, 0, 1980, 4094, 1577, 4096, 0, 3616, 760, 0, 0, 4094, 0, 4071, 0, 4036, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 2, 4064, 19, 0, 2, 0, 0, 0, 0, 0, 0, 3654, 1935, 0, 6, 0, 3692, 0, 43, 0, 3904, 0, 19, 4046, 4089, 4095, 0, 0, 0, 1533, 1, 3716, 0, 4095, 0, 0, 3755, 55, 0, 443, 3469, 0, 558, 323, 281, 4096, 4068, 5, 167, 0, 0, 0, 152, 447, 112, 2400, 2191, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[773, 0, 4092, 0, 4037, 772, 0, 0, 1, 23, 12, 3950, 0, 75, 0, 0, 4096, 0, 1, 1166, 90, 58, 0, 4062, 6, 0, 0, 0, 3598, 0, 0, 0, 4045, 0, 0, 0, 0, 3946, 3328, 3794, 0, 1155, 0, 0, 5, 4096, 71, 0, 0, 3971, 1066, 413, 79, 29, 4096, 507, 3, 0, 0, 27, 2, 3995, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[3638, 0, 141, 2846, 356, 535, 74, 0, 246, 0, 3291, 4085, 0, 0, 0, 4005, 0, 0, 0, 24, 3781, 3489, 406, 1595, 3867, 0, 95, 1172, 0, 23, 0, 4089, 5, 0, 8, 16, 0, 4095, 6, 0, 13, 3087, 0, 1, 4096, 0, 0, 0, 1, 3177, 0, 0, 1036, 0, 4069, 3762, 0, 0, 0, 3, 0, 196, 4052, 155] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[99, 0, 0, 2588, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 1, 3995, 105, 0, 0, 4096, 4060, 1290, 4096, 4096, 0, 0, 3975, 3416, 0, 244, 3262, 0, 2534, 2, 46, 928, 881, 1607, 0, 1, 0, 0, 6, 0, 3225, 2446, 1, 70, 2337, 3962, 0, 6, 0, 0, 0, 0, 0, 3847, 0, 0, 4096, 0, 127] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 1470, 2921, 0, 0, 17, 4095, 0, 0, 0, 4033, 0, 0, 0, 2160, 4096, 9, 0, 48, 4051, 3271, 4086, 484, 18, 773, 0, 3, 3972, 2510, 0, 4045, 3838, 1933, 1058, 0, 0, 0, 8, 0, 605, 4096, 121, 0, 0, 0, 0, 0, 260, 4096, 3079, 14, 165, 0, 0, 0, 0, 2, 0, 43, 18, 21, 10, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2574, 0, 4088, 0, 0, 1, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 942, 0, 4, 2, 0, 0, 0, 281, 62, 52, 0, 0, 0, 0, 0, 0, 67, 0, 5, 10, 0, 2944, 0, 3789, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 4092, 12, 0, 48, 0, 496, 8, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4094, 0, 3, 110, 0, 3009, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 193, 2324, 0, 0, 0, 0, 0, 38, 489, 0, 0, 0, 0, 0, 0, 1980, 2327, 1577, 4096, 0, 3616, 760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4071, 0, 4036, 23, 1278, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6680, 1343, 6815, 8477, 7242, 7118, 14356, 13505] avg=8192.0000 max=14356 max_rank=6 min=1343 min_rank=1 imbalance_ratio=1.752441 max_load_reduction=51180 max_load_reduction_pct=0.780945 moved_to_echo=42221 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4066, 0, 15, 3517, 0, 0, 303, 87, 811, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3353, 0, 28, 0, 132, 3721, 15, 0, 0, 0, 0, 0, 0, 3130, 0, 0, 1, 0, 3, 4050, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3064, 14, 362, 4079, 376, 0, 0, 0, 0, 0, 0, 0, 0, 3361, 2032, 38, 4095, 4, 2377, 0, 2557, 0, 0, 0, 0, 0, 0, 0, 0, 1118, 126, 0, 79, 7, 0, 0, 12, 1703, 11, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4094, 1161, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 4016, 523, 2801, 0, 0, 3, 146, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8799, 7255, 7187, 7898, 14464, 3057, 9207, 7669] avg=8192.0000 max=14464 max_rank=4 min=3057 min_rank=5 imbalance_ratio=1.765625 max_load_reduction=51072 max_load_reduction_pct=0.779297 moved_to_echo=34397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 2, 4064, 19, 0, 384, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3654, 3692, 1560, 0, 0, 0, 0, 0, 0, 1482, 0, 6, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 19, 4046, 1374, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1533, 1, 3716, 0, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 55, 0, 443, 3469, 0, 558, 323, 1705, 1155, 485, 0, 0, 0, 0, 0, 281, 4096, 2313, 5, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 447, 112, 2400, 2191, 6, 0, 0, 1270, 69, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8562, 8908, 1531, 13438, 7640, 11948, 6862, 6647] avg=8192.0000 max=13438 max_rank=3 min=1531 min_rank=2 imbalance_ratio=1.640381 max_load_reduction=52098 max_load_reduction_pct=0.794952 moved_to_echo=33097 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4043, 0, 0, 1986, 3, 0, 0, 0, 491, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 41, 18, 0, 0, 12, 4075, 1544, 0, 0, 0, 0, 0, 0, 1894, 8, 0, 0, 0, 2537, 4060, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2195, 305, 0, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1313, 4094, 4, 1, 2, 2390, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 15, 0, 4094, 52, 1236, 0, 1706, 1143, 477, 0, 0, 0, 0, 0, 1, 143, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 4088, 1, 4073, 379, 4096, 0, 1249, 88, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6523, 5845, 12424, 3860, 7826, 12596, 2419, 14043] avg=8192.0000 max=14043 max_rank=7 min=2419 min_rank=6 imbalance_ratio=1.714233 max_load_reduction=51493 max_load_reduction_pct=0.785721 moved_to_echo=36884 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3638, 0, 141, 2846, 356, 535, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 3291, 4085, 0, 0, 0, 4005, 3781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 3489, 406, 1595, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 0, 95, 1172, 0, 23, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8, 16, 0, 2390, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3087, 0, 1, 4096, 0, 0, 0, 1705, 0, 0, 0, 0, 0, 0, 0, 1, 3177, 0, 0, 1036, 0, 4069, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 196, 4052, 155, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7590, 15408, 5514, 9246, 2425, 8902, 12045, 4406] avg=8192.0000 max=15408 max_rank=1 min=2425 min_rank=4 imbalance_ratio=1.880859 max_load_reduction=50128 max_load_reduction_pct=0.764893 moved_to_echo=27778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[773, 0, 4092, 0, 4037, 772, 0, 0, 813, 0, 0, 0, 0, 0, 0, 0, 1, 23, 12, 3950, 0, 75, 0, 0, 90, 1373, 0, 0, 0, 0, 0, 0, 3137, 0, 1, 1166, 0, 58, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 2303, 3328, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 0, 5, 4096, 71, 0, 1643, 1016, 127, 0, 0, 0, 0, 0, 0, 3971, 607, 413, 79, 29, 4096, 507, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 27, 2, 3995, 4096, 0, 332, 146, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10487, 5524, 8424, 1215, 13470, 8113, 9702, 8601] avg=8192.0000 max=13470 max_rank=4 min=1215 min_rank=3 imbalance_ratio=1.644287 max_load_reduction=52066 max_load_reduction_pct=0.794464 moved_to_echo=39886 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[99, 0, 0, 2588, 0, 0, 0, 0, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 1, 4096, 1303, 0, 0, 0, 0, 0, 0, 3060, 105, 0, 0, 0, 4060, 1290, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3975, 1148, 0, 244, 3262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2534, 2, 46, 928, 515, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 3225, 2446, 1, 366, 965, 473, 0, 0, 0, 0, 0, 70, 2337, 2253, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 0, 0, 4096, 0, 127, 1236, 142, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3480, 9491, 12611, 12725, 5632, 7483, 4666, 9448] avg=8192.0000 max=12725 max_rank=3 min=3480 min_rank=0 imbalance_ratio=1.553345 max_load_reduction=52811 max_load_reduction_pct=0.805832 moved_to_echo=27229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1470, 2921, 0, 0, 17, 4095, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 2160, 4051, 1518, 0, 0, 0, 0, 0, 0, 3132, 9, 0, 48, 0, 3271, 4086, 484, 0, 0, 0, 0, 0, 0, 0, 0, 18, 773, 0, 3, 1328, 2510, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 1933, 1058, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, 4096, 121, 0, 0, 0, 0, 0, 0, 1126, 371, 0, 0, 0, 0, 0, 260, 4096, 1744, 14, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 43, 18, 21, 10, 11, 964, 148, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13415, 11762, 11030, 8677, 6837, 6319, 6279, 1217] avg=8192.0000 max=13415 max_rank=0 min=1217 min_rank=7 imbalance_ratio=1.637573 max_load_reduction=52121 max_load_reduction_pct=0.795303 moved_to_echo=20652 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 19840, 11648, 2048, 9728, 1280, 11904, 9088, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[15744, 7168, 512, 12288, 6528, 7040, 4864, 11904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 768, 12160, 6528, 10880, 11520, 16384, 384, 6400, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[16000, 128, 128, 1280, 0, 16384, 13952, 18048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12672, 16256, 1152, 7936, 1152, 12416, 8064, 6400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1408, 128, 7424, 15488, 128, 128, 0, 10368, 23680, 7424, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5504, 16512, 384, 2944, 11776, 7424, 4352, 384, 8832, 5504, 2432, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[19328, 0, 9856, 18048, 4480, 5376, 512, 4224, 4224, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 74, 0, 0, 0, 4079, 2843, 4091, 220, 28, 0, 2, 0, 3941, 583, 4093, 1719, 0, 0, 72, 0, 0, 3820, 4096, 0, 4096, 4088, 0, 0, 13, 0, 0, 1211, 0, 3469, 4094, 0, 1066, 4074, 2674, 26, 0, 2042, 0, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 4096, 4014, 0, 0, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 145, 0, 21, 0, 3854, 7, 0, 3994, 5, 1, 304, 0, 0, 2763, 0, 877, 3254, 1007, 26, 4052, 0, 3471, 17, 0, 0, 0, 0, 0, 12, 194, 0, 6, 4055, 4, 0, 4018, 0, 0, 34, 11, 3939, 0, 258, 3521, 3794, 3288, 1914, 387, 0, 87, 4085, 0, 0, 0, 33, 8, 3808, 4096, 395, 1736, 2055, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 2010, 225, 0, 1062, 0, 3232, 2822, 0, 0, 4096, 0, 4082, 0, 0, 106, 4090, 17, 3534, 0, 0, 0, 0, 0, 0, 2818, 0, 0, 0, 3951, 0, 0, 115, 0, 2083, 344, 0, 433, 29, 161, 3810, 4096, 0, 0, 306, 4096, 4092, 3930, 0, 0, 0, 2065, 0, 2, 3, 55, 0, 4011, 3840, 0, 9, 11, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 3522, 0, 0, 567, 296, 0, 3078, 0, 0, 3, 17, 70, 0, 564, 1655, 0, 21, 1, 30, 0, 4096, 4050, 2, 3241, 0, 292, 0, 1, 4078, 2, 0, 571, 562, 41, 0, 3598, 3725, 329, 4096, 4096, 4068, 3851, 0, 3773, 3635, 3062, 49, 7, 50, 0, 0, 0, 0, 0, 0, 68, 3941, 425, 0, 0, 1, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 232, 0, 3540, 1, 0, 0, 0, 0, 115, 255, 4092, 4070, 0, 11, 3727, 0, 96, 0, 3769, 37, 3959, 0, 872, 374, 4012, 0, 201, 207, 0, 4090, 0, 0, 3909, 0, 12, 3032, 1, 10, 0, 4073, 3967, 1572, 3886, 0, 190, 0, 797, 3, 0, 1214, 0, 38, 0, 4094, 0, 3727, 731, 603, 0, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[13, 0, 1, 0, 0, 1, 0, 4096, 44, 4054, 0, 0, 1702, 113, 319, 87, 35, 0, 0, 0, 2643, 1535, 1653, 0, 4001, 3, 1326, 587, 632, 4070, 1, 223, 279, 0, 3979, 445, 3536, 0, 3325, 1, 4095, 4059, 19, 3922, 0, 3920, 4096, 163, 0, 2401, 0, 3990, 0, 0, 24, 0, 0, 0, 0, 0, 65, 68, 10, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 8, 3234, 14, 3124, 0, 1313, 0, 490, 0, 0, 0, 192, 3541, 0, 2875, 0, 0, 0, 3928, 4009, 98, 1775, 2096, 0, 0, 9, 3907, 0, 4045, 3845, 4096, 277, 1601, 0, 3832, 0, 28, 4096, 2, 4089, 3933, 10, 0, 149, 3, 0, 0, 4, 0, 0, 0, 66, 0, 0, 0, 199, 0, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4001, 4096, 4092, 0, 2416, 4096, 0, 0, 161, 0, 1180, 0, 4084, 395, 4096, 311, 2862, 0, 3612, 4033, 0, 0, 0, 4096, 0, 0, 1, 3, 31, 99, 2, 0, 13, 0, 4096, 3211, 8, 0, 63, 4051, 4094, 19, 0, 0, 0, 17, 0, 1, 0, 0, 0, 2178, 4, 0, 0, 7, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 74, 0, 0, 0, 4079, 1315, 821, 0, 0, 0, 0, 0, 0, 2843, 4091, 220, 28, 0, 2, 0, 3941, 563, 245, 0, 0, 0, 0, 0, 0, 583, 4093, 1474, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2248, 4096, 0, 4096, 4088, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1211, 0, 2085, 4094, 0, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 2674, 26, 0, 2042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 0, 0, 0, 0, 0, 0, 0, 110, 4096, 4014, 0, 0, 0, 5, 0, 2812, 0, 257, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6289, 11933, 6222, 14541, 8456, 6004, 797, 11294] avg=8192.0000 max=14541 max_rank=3 min=797 min_rank=6 imbalance_ratio=1.775024 max_load_reduction=50995 max_load_reduction_pct=0.778122 moved_to_echo=26551 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 145, 0, 21, 0, 3854, 7, 0, 0, 953, 0, 0, 0, 0, 0, 0, 3994, 5, 1, 304, 0, 0, 2763, 0, 655, 143, 0, 0, 0, 0, 0, 0, 877, 3254, 864, 26, 4052, 0, 3471, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4055, 4, 0, 2410, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3939, 0, 258, 3521, 0, 0, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 87, 4085, 0, 0, 0, 33, 3288, 3421, 0, 0, 0, 0, 0, 0, 8, 3808, 4096, 395, 1736, 2055, 0, 0, 7, 373, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4980, 7865, 12561, 206, 6509, 9636, 11301, 12478] avg=8192.0000 max=12561 max_rank=2 min=206 min_rank=3 imbalance_ratio=1.533325 max_load_reduction=52975 max_load_reduction_pct=0.808334 moved_to_echo=39924 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 232, 0, 3540, 1, 1362, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 255, 4092, 4070, 635, 532, 0, 0, 0, 0, 0, 0, 0, 11, 3195, 0, 96, 0, 3769, 37, 0, 0, 0, 0, 0, 0, 0, 0, 2330, 0, 872, 374, 4012, 0, 201, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 2349, 0, 12, 3032, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 4073, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 797, 3, 0, 1214, 0, 38, 3886, 1417, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3727, 731, 603, 0, 17, 0, 155, 267, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6060, 9699, 7108, 7996, 9483, 8051, 7545, 9594] avg=8192.0000 max=9699 max_rank=1 min=6060 min_rank=0 imbalance_ratio=1.183960 max_load_reduction=55837 max_load_reduction_pct=0.852005 moved_to_echo=34673 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2010, 225, 0, 1062, 0, 3232, 2822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 106, 0, 505, 0, 0, 0, 0, 0, 0, 4090, 17, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2818, 0, 0, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 2083, 344, 0, 433, 29, 161, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 4096, 0, 0, 306, 0, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 0, 2, 3, 55, 4092, 3694, 0, 0, 0, 0, 0, 0, 0, 4011, 3840, 0, 9, 11, 0, 0, 2629, 402, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9351, 8789, 7136, 6769, 3165, 9513, 9911, 10902] avg=8192.0000 max=10902 max_rank=7 min=3165 min_rank=4 imbalance_ratio=1.330811 max_load_reduction=54634 max_load_reduction_pct=0.833649 moved_to_echo=33491 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3522, 0, 0, 567, 296, 0, 3078, 1115, 852, 0, 0, 0, 0, 0, 0, 0, 0, 3, 17, 70, 0, 564, 1655, 584, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 30, 0, 4096, 4050, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1908, 0, 292, 0, 1, 4078, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 562, 41, 0, 2162, 3725, 329, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 4068, 3851, 0, 3773, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 7, 50, 0, 0, 0, 0, 0, 0, 3062, 3277, 0, 0, 0, 0, 0, 0, 68, 3941, 425, 0, 0, 1, 2, 0, 2827, 358, 218, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9430, 2893, 8200, 6281, 11486, 13010, 6396, 7840] avg=8192.0000 max=13010 max_rank=5 min=2893 min_rank=1 imbalance_ratio=1.588135 max_load_reduction=52526 max_load_reduction_pct=0.801483 moved_to_echo=38732 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 1, 0, 0, 1, 0, 4096, 1377, 837, 0, 0, 0, 0, 0, 0, 44, 4054, 0, 0, 1702, 113, 319, 87, 574, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 2643, 1535, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2354, 3, 1326, 587, 632, 4070, 1, 223, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 3979, 445, 2125, 0, 3325, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 4059, 19, 3922, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2401, 0, 3990, 0, 0, 24, 0, 4096, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 68, 10, 0, 2826, 385, 270, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6325, 6893, 5866, 9196, 10154, 9432, 14046, 3624] avg=8192.0000 max=14046 max_rank=6 min=3624 min_rank=7 imbalance_ratio=1.714600 max_load_reduction=51490 max_load_reduction_pct=0.785675 moved_to_echo=37256 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 8, 3234, 1352, 970, 0, 0, 0, 0, 0, 0, 14, 3124, 0, 1313, 0, 490, 0, 0, 665, 506, 0, 0, 0, 0, 0, 0, 0, 192, 3035, 0, 2875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2311, 4009, 98, 1775, 2096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 0, 4045, 3845, 2461, 277, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 0, 28, 4096, 2, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 3, 0, 0, 4, 0, 0, 3933, 3686, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 199, 0, 552, 2644, 403, 265, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9660, 6112, 6102, 10298, 16136, 5324, 7775, 4129] avg=8192.0000 max=16136 max_rank=4 min=4129 min_rank=7 imbalance_ratio=1.969727 max_load_reduction=49400 max_load_reduction_pct=0.753784 moved_to_echo=33364 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4001, 4096, 1246, 2, 0, 0, 0, 0, 0, 0, 4092, 0, 2416, 4096, 0, 0, 161, 0, 0, 587, 0, 0, 0, 0, 0, 0, 1180, 0, 3497, 395, 4096, 311, 2862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2120, 4033, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 31, 99, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1265, 3211, 8, 0, 63, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1, 0, 0, 4094, 3653, 0, 0, 0, 0, 0, 0, 0, 2178, 4, 0, 0, 7, 11, 0, 2831, 398, 246, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13441, 11352, 12341, 10249, 147, 4566, 7765, 5675] avg=8192.0000 max=13441 max_rank=0 min=147 min_rank=4 imbalance_ratio=1.640747 max_load_reduction=52095 max_load_reduction_pct=0.794907 moved_to_echo=18153 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4992, 8832, 11520, 4736, 13696, 8576, 5376, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[13312, 14976, 2688, 6912, 10880, 16256, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[6784, 7680, 15104, 512, 13824, 6016, 15872, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 2688, 896, 10240, 0, 1280, 128, 128, 27264, 22784, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 22272, 12416, 4224, 2560, 2944, 128, 640, 16640, 2560, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11008, 11392, 6784, 5760, 6016, 896, 7936, 9984, 3712, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7552, 22144, 3968, 12416, 13696, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 9856, 256, 4224, 1920, 4224, 10880, 21504, 7808, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[3671, 0, 0, 36, 865, 4076, 0, 316, 0, 4096, 0, 289, 0, 0, 1006, 0, 0, 0, 3648, 648, 0, 8, 1003, 0, 4065, 2049, 85, 0, 0, 0, 0, 0, 11, 0, 0, 6, 98, 0, 0, 4085, 1, 0, 4063, 1, 13, 1125, 4069, 3892, 4094, 2549, 0, 0, 4096, 0, 3589, 0, 5, 4096, 222, 0, 3660, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4088, 0, 0, 0, 1260, 3472, 0, 401, 0, 4096, 0, 0, 4096, 1, 1, 210, 3539, 11, 0, 48, 0, 0, 3351, 1462, 1, 3112, 84, 0, 358, 502, 0, 0, 3128, 0, 0, 4073, 0, 2, 0, 4096, 3658, 0, 4084, 0, 9, 0, 4009, 0, 3506, 879, 568, 454, 2381, 0, 126, 639, 0, 0, 0, 109, 3722, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[78, 4, 1509, 2768, 0, 120, 2133, 0, 3, 1600, 4096, 436, 167, 4017, 0, 14, 0, 0, 4094, 24, 1721, 3829, 1, 2590, 0, 3547, 19, 5, 0, 0, 0, 0, 0, 4, 0, 4050, 0, 0, 413, 3940, 2459, 0, 3649, 27, 4096, 3762, 0, 20, 1, 0, 4, 687, 4086, 0, 2861, 0, 0, 176, 0, 438, 2086, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 3625, 74, 0, 0, 201, 1404, 3635, 1005, 4055, 0, 538, 0, 2923, 12, 0, 82, 0, 151, 0, 3065, 1769, 0, 0, 11, 1, 0, 79, 0, 0, 0, 0, 45, 0, 0, 0, 2, 0, 4065, 2406, 0, 0, 2959, 2903, 0, 4088, 0, 4029, 0, 0, 2332, 16, 4095, 0, 3905, 0, 1423, 0, 0, 1, 3973, 13, 2555, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[94, 4004, 41, 0, 0, 4096, 0, 0, 0, 2314, 2903, 0, 0, 4094, 0, 0, 15, 49, 465, 0, 3533, 0, 3145, 0, 0, 4096, 3, 0, 0, 0, 47, 2, 0, 0, 0, 3878, 0, 2, 0, 4086, 2806, 0, 720, 0, 159, 3635, 0, 20, 1769, 0, 3934, 0, 4079, 1, 1036, 0, 0, 2363, 4049, 2, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 3, 3674, 36, 2, 0, 0, 0, 4093, 0, 4096, 4096, 0, 4090, 2651, 755, 0, 0, 4096, 0, 3459, 0, 3, 31, 0, 3, 0, 108, 607, 0, 0, 1263, 0, 0, 0, 0, 0, 3175, 1879, 3968, 2956, 0, 3140, 149, 300, 0, 743, 240, 258, 0, 3786, 15, 4052, 0, 31, 3, 0, 6, 0, 3688, 4081, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[2, 1486, 0, 0, 23, 4085, 0, 0, 1, 2, 7, 3943, 109, 0, 2255, 0, 0, 3991, 3769, 4091, 3737, 145, 4095, 0, 5, 22, 3, 0, 0, 0, 0, 2663, 12, 3, 0, 1, 0, 0, 1877, 4096, 3811, 0, 524, 0, 0, 299, 4096, 1, 0, 657, 0, 0, 0, 4096, 0, 0, 0, 4074, 0, 0, 4080, 2962, 0, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[3195, 0, 28, 0, 2124, 4081, 17, 0, 0, 2, 0, 7, 1604, 200, 3890, 4092, 0, 64, 0, 0, 0, 0, 0, 0, 0, 3868, 4033, 2, 8, 2082, 841, 3984, 541, 0, 1958, 3954, 0, 0, 186, 4036, 4096, 0, 4091, 5, 0, 0, 526, 1819, 1042, 0, 2562, 86, 4096, 263, 1831, 0, 19, 0, 0, 0, 303, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3674, 36, 2, 0, 0, 0, 77, 49, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 4096, 0, 4090, 2651, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 3459, 0, 3, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 108, 607, 0, 0, 1263, 2908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3175, 1879, 3968, 220, 0, 0, 0, 0, 0, 0, 0, 2956, 0, 232, 149, 300, 0, 743, 240, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 3786, 15, 2708, 0, 31, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3688, 4081, 0, 0, 0, 1047, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3841, 19781, 7540, 4889, 9242, 4620, 6801, 8822] avg=8192.0000 max=19781 max_rank=1 min=3841 min_rank=0 imbalance_ratio=2.414673 max_load_reduction=45755 max_load_reduction_pct=0.698166 moved_to_echo=29485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3671, 0, 0, 36, 865, 4076, 0, 316, 78, 44, 0, 0, 0, 0, 0, 0, 0, 973, 0, 289, 0, 0, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 648, 0, 8, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 2049, 85, 0, 0, 0, 0, 0, 3763, 1637, 0, 0, 0, 0, 0, 0, 11, 0, 0, 6, 98, 0, 0, 4085, 222, 0, 0, 0, 0, 0, 0, 0, 1, 0, 300, 1, 13, 1125, 4069, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2549, 0, 0, 2737, 0, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 222, 0, 3660, 0, 0, 0, 1059, 1486, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9086, 2268, 5263, 11599, 4422, 9401, 12969, 10528] avg=8192.0000 max=12969 max_rank=6 min=2268 min_rank=1 imbalance_ratio=1.583130 max_load_reduction=52567 max_load_reduction_pct=0.802109 moved_to_echo=37320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 0, 0, 1260, 3472, 0, 401, 45, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 4096, 1, 1, 210, 0, 0, 0, 0, 0, 0, 0, 0, 3539, 11, 0, 48, 0, 0, 3351, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3112, 84, 0, 358, 502, 0, 0, 3782, 1637, 0, 0, 0, 0, 0, 0, 3128, 0, 0, 4073, 0, 2, 0, 4096, 129, 0, 0, 0, 0, 0, 0, 0, 3658, 0, 302, 0, 9, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 879, 568, 454, 1592, 0, 126, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 3722, 0, 0, 0, 615, 1486, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9266, 5281, 8411, 9476, 11428, 7978, 7764, 5932] avg=8192.0000 max=11428 max_rank=4 min=5281 min_rank=1 imbalance_ratio=1.395020 max_load_reduction=54108 max_load_reduction_pct=0.825623 moved_to_echo=33102 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 3625, 74, 0, 0, 201, 1404, 3635, 78, 1, 0, 0, 0, 0, 0, 0, 1005, 964, 0, 538, 0, 2923, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 150, 0, 3065, 1769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 79, 0, 0, 0, 0, 2740, 1620, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 2, 0, 4065, 2406, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 2903, 0, 4088, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2332, 16, 2737, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1423, 0, 0, 1, 3973, 13, 2555, 0, 1058, 1471, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13114, 5442, 5066, 4451, 6740, 11239, 8990, 10494] avg=8192.0000 max=13114 max_rank=0 min=4451 min_rank=3 imbalance_ratio=1.600830 max_load_reduction=52422 max_load_reduction_pct=0.799896 moved_to_echo=37463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[78, 4, 1509, 2768, 0, 120, 2133, 0, 83, 49, 0, 0, 0, 0, 0, 0, 3, 374, 4096, 436, 167, 4017, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 24, 1721, 3829, 1, 2590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 19, 5, 0, 0, 0, 0, 3379, 641, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4050, 0, 0, 413, 3940, 224, 0, 0, 0, 0, 0, 0, 0, 2459, 0, 270, 27, 4096, 3762, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 687, 2719, 0, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 438, 2086, 0, 0, 2, 1060, 585, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6744, 9107, 12210, 7591, 8631, 10634, 6272, 4347] avg=8192.0000 max=12210 max_rank=2 min=4347 min_rank=7 imbalance_ratio=1.490479 max_load_reduction=53326 max_load_reduction_pct=0.813690 moved_to_echo=29884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[94, 4004, 41, 0, 0, 4096, 0, 0, 78, 5, 0, 0, 0, 0, 0, 0, 0, 550, 2903, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 49, 460, 0, 3533, 0, 3145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 47, 2, 666, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3878, 0, 2, 0, 4086, 221, 0, 0, 0, 0, 0, 0, 0, 2806, 0, 54, 0, 159, 3635, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1769, 0, 3934, 0, 2726, 1, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2363, 4049, 2, 4096, 0, 0, 0, 1054, 839, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8318, 7547, 7202, 5739, 8187, 6674, 9466, 12403] avg=8192.0000 max=12403 max_rank=7 min=5739 min_rank=3 imbalance_ratio=1.514038 max_load_reduction=53133 max_load_reduction_pct=0.810745 moved_to_echo=36730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 1486, 0, 0, 23, 4085, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 1, 0, 7, 3943, 109, 0, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 3721, 4091, 3737, 145, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 22, 3, 0, 0, 0, 0, 2663, 489, 2, 0, 0, 0, 0, 0, 0, 12, 3, 0, 1, 0, 0, 1877, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3811, 0, 35, 0, 0, 299, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4080, 2962, 0, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5644, 6315, 19780, 3184, 5989, 8242, 4753, 11629] avg=8192.0000 max=19780 max_rank=2 min=3184 min_rank=3 imbalance_ratio=2.414551 max_load_reduction=45756 max_load_reduction_pct=0.698181 moved_to_echo=30613 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3195, 0, 28, 0, 2124, 4081, 17, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 1604, 200, 3890, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 4033, 2, 8, 2082, 841, 3984, 3789, 0, 0, 0, 0, 0, 0, 0, 541, 0, 1958, 3954, 0, 0, 186, 4036, 222, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 302, 5, 0, 0, 526, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 0, 2562, 86, 2737, 263, 1831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 303, 0, 0, 0, 1059, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9523, 9795, 64, 18607, 10897, 6748, 8521, 1381] avg=8192.0000 max=18607 max_rank=3 min=64 min_rank=2 imbalance_ratio=2.271362 max_load_reduction=46929 max_load_reduction_pct=0.716080 moved_to_echo=27547 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3712, 4224, 16128, 4864, 15616, 5760, 11648, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1536, 10752, 4352, 4352, 26112, 3072, 2560, 640, 7040, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[15232, 9216, 5376, 2944, 4352, 20224, 3584, 4352, 640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[19840, 0, 1792, 3200, 4608, 12928, 13568, 10112, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[10752, 4096, 13312, 1280, 18048, 4480, 13440, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 16768, 4352, 256, 1024, 2688, 896, 7936, 21632, 6528, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[5120, 3840, 11136, 9344, 6016, 15360, 9856, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3840, 128, 2048, 16000, 128, 3200, 8448, 30720, 1536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[253, 2976, 761, 401, 1203, 466, 779, 1046, 1504, 175, 1522, 396, 1144, 687, 125, 371, 177, 732, 1560, 681, 924, 197, 2760, 1732, 319, 894, 1127, 862, 1501, 536, 1774, 1775, 2824, 621, 369, 534, 133, 2272, 2092, 531, 2842, 398, 1505, 999, 1290, 1625, 1269, 1207, 317, 346, 277, 2288, 977, 595, 497, 126, 485, 855, 899, 1658, 1414, 791, 859, 1281] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[40, 2704, 516, 396, 1931, 407, 732, 756, 1063, 97, 1916, 249, 1057, 650, 178, 497, 273, 940, 560, 891, 708, 262, 2743, 2239, 525, 1469, 1027, 1129, 988, 332, 1382, 2318, 3316, 722, 472, 433, 190, 2255, 2071, 722, 2655, 372, 1341, 669, 1695, 1966, 1025, 1431, 327, 371, 430, 2166, 808, 696, 618, 83, 1182, 371, 959, 1913, 1344, 627, 421, 910] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[59, 2778, 1006, 380, 1835, 537, 1145, 922, 867, 143, 1763, 658, 1262, 526, 243, 423, 196, 418, 931, 839, 1014, 55, 3510, 2091, 655, 1068, 982, 820, 1069, 258, 1103, 1588, 3083, 599, 462, 653, 300, 1629, 1650, 709, 2886, 210, 1097, 769, 1215, 1752, 1601, 1036, 308, 468, 342, 1811, 1012, 425, 871, 121, 941, 859, 1616, 1884, 1753, 603, 717, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[63, 3149, 541, 265, 1592, 338, 877, 1015, 1534, 84, 1658, 144, 1121, 1196, 557, 402, 61, 569, 800, 550, 1155, 279, 2817, 2128, 887, 945, 948, 909, 1221, 435, 1963, 1965, 2895, 606, 434, 324, 77, 2590, 2036, 608, 3140, 249, 1583, 597, 1357, 1829, 1150, 1040, 307, 379, 266, 1744, 767, 519, 414, 92, 764, 390, 1166, 2071, 1538, 386, 735, 1315] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[140, 2870, 879, 266, 1568, 430, 828, 1154, 853, 73, 1262, 285, 1416, 716, 213, 464, 275, 493, 1414, 1540, 789, 92, 2806, 1489, 478, 731, 951, 1073, 1665, 537, 1232, 1970, 3429, 775, 380, 442, 510, 2087, 2209, 455, 2653, 387, 1381, 988, 1574, 1448, 941, 1491, 415, 198, 412, 2051, 1141, 486, 850, 265, 672, 219, 852, 1783, 1440, 788, 746, 1116] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[75, 2425, 818, 207, 1245, 286, 932, 726, 1465, 189, 1627, 275, 478, 1020, 332, 437, 122, 677, 1734, 869, 956, 396, 2901, 2108, 591, 1229, 908, 1099, 1422, 374, 2146, 1718, 2832, 478, 685, 445, 153, 2473, 1777, 1188, 2438, 286, 1974, 1060, 1629, 1380, 847, 954, 296, 415, 237, 1973, 740, 566, 644, 102, 548, 378, 874, 1887, 1847, 433, 1080, 1130] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[55, 2485, 515, 250, 1518, 292, 1109, 1017, 1047, 141, 1224, 304, 1108, 778, 343, 305, 45, 402, 578, 885, 1244, 83, 3523, 1906, 677, 968, 1009, 1088, 1333, 221, 1460, 2231, 3526, 897, 265, 860, 329, 2027, 2712, 806, 2792, 196, 1200, 627, 1367, 1442, 1222, 682, 227, 450, 301, 2177, 836, 518, 722, 120, 742, 478, 617, 2476, 1967, 460, 1139, 1212] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[409, 2761, 378, 523, 1680, 306, 744, 1018, 1366, 146, 1489, 190, 1152, 935, 194, 420, 90, 653, 1025, 711, 758, 270, 3154, 2226, 649, 883, 850, 983, 1009, 524, 1857, 1611, 2719, 1136, 378, 810, 80, 1821, 1775, 751, 2524, 760, 1860, 879, 1766, 1469, 856, 1009, 333, 389, 540, 2354, 1129, 484, 344, 88, 649, 373, 1523, 1785, 1900, 454, 450, 1184] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[253, 2976, 761, 401, 1203, 466, 779, 1046, 277, 148, 0, 0, 0, 0, 0, 0, 1504, 175, 1522, 396, 1144, 687, 125, 371, 1292, 623, 273, 0, 0, 0, 0, 0, 177, 732, 1560, 681, 924, 197, 2137, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 319, 894, 1127, 862, 1501, 536, 1774, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 1232, 621, 369, 534, 133, 2272, 2092, 531, 0, 0, 0, 0, 0, 0, 0, 0, 206, 398, 1505, 999, 1290, 1625, 1269, 1207, 0, 0, 0, 0, 0, 0, 0, 0, 317, 346, 277, 2288, 977, 595, 497, 126, 2636, 300, 0, 0, 0, 0, 0, 0, 485, 855, 899, 1510, 1414, 791, 859, 1281, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8112, 8140, 8238, 7784, 8499, 8359, 8094] avg=8192.0000 max=8499 max_rank=5 min=7784 min_rank=4 imbalance_ratio=1.037476 max_load_reduction=57037 max_load_reduction_pct=0.870316 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[59, 2778, 1006, 380, 1835, 537, 1145, 922, 248, 168, 0, 0, 0, 0, 0, 0, 867, 143, 1763, 658, 1262, 526, 243, 423, 1410, 792, 244, 0, 0, 0, 0, 0, 196, 418, 931, 839, 1014, 55, 2718, 2091, 0, 0, 0, 0, 0, 0, 0, 0, 655, 1068, 982, 820, 1069, 258, 1103, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 1346, 599, 462, 653, 300, 1629, 1650, 709, 0, 0, 0, 0, 0, 0, 0, 0, 209, 210, 1097, 769, 1215, 1752, 1601, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 308, 468, 342, 1811, 1012, 425, 871, 121, 2677, 327, 0, 0, 0, 0, 0, 0, 941, 859, 1616, 1716, 1753, 603, 717, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9078, 8331, 8262, 7051, 7348, 7889, 8362, 9215] avg=8192.0000 max=9215 max_rank=7 min=7051 min_rank=3 imbalance_ratio=1.124878 max_load_reduction=56321 max_load_reduction_pct=0.859390 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[40, 2704, 516, 396, 1931, 407, 732, 756, 362, 171, 0, 0, 0, 0, 0, 0, 1063, 97, 1916, 249, 1057, 650, 178, 497, 1517, 619, 357, 0, 0, 0, 0, 0, 273, 940, 560, 891, 708, 262, 2124, 2239, 0, 0, 0, 0, 0, 0, 0, 0, 525, 1469, 1027, 1129, 988, 332, 1382, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 1447, 722, 472, 433, 190, 2255, 2071, 722, 0, 0, 0, 0, 0, 0, 0, 0, 192, 372, 1341, 669, 1695, 1966, 1025, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 327, 371, 430, 2166, 808, 696, 618, 83, 2463, 352, 0, 0, 0, 0, 0, 0, 1182, 371, 959, 1742, 1344, 627, 421, 910, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8015, 8200, 7997, 8451, 8312, 8691, 8314, 7556] avg=8192.0000 max=8691 max_rank=5 min=7556 min_rank=7 imbalance_ratio=1.060913 max_load_reduction=56845 max_load_reduction_pct=0.867386 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[140, 2870, 879, 266, 1568, 430, 828, 1154, 308, 159, 0, 0, 0, 0, 0, 0, 853, 73, 1262, 285, 1416, 716, 213, 464, 1568, 633, 303, 0, 0, 0, 0, 0, 275, 493, 1414, 1540, 789, 92, 2173, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 478, 731, 951, 1073, 1665, 537, 1232, 1359, 0, 0, 0, 0, 0, 0, 0, 0, 1497, 775, 380, 442, 510, 2087, 2209, 455, 0, 0, 0, 0, 0, 0, 0, 0, 192, 387, 1381, 988, 1574, 1448, 941, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 415, 198, 412, 2051, 1141, 486, 850, 265, 2461, 364, 0, 0, 0, 0, 0, 0, 672, 219, 852, 1624, 1440, 788, 746, 1116, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8602, 7786, 8265, 8026, 8355, 8402, 8643, 7457] avg=8192.0000 max=8643 max_rank=6 min=7457 min_rank=7 imbalance_ratio=1.055054 max_load_reduction=56893 max_load_reduction_pct=0.868118 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[63, 3149, 541, 265, 1592, 338, 877, 1015, 307, 185, 0, 0, 0, 0, 0, 0, 1534, 84, 1658, 144, 1121, 1196, 557, 402, 1324, 636, 302, 0, 0, 0, 0, 0, 61, 569, 800, 550, 1155, 279, 2181, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 887, 945, 948, 909, 1221, 435, 1963, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 606, 434, 324, 77, 2590, 2036, 608, 0, 0, 0, 0, 0, 0, 0, 0, 227, 249, 1583, 597, 1357, 1829, 1150, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 307, 379, 266, 1744, 767, 519, 414, 92, 2913, 307, 0, 0, 0, 0, 0, 0, 764, 390, 1166, 1886, 1538, 386, 735, 1315, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8332, 8958, 7723, 8664, 7939, 8032, 7708, 8180] avg=8192.0000 max=8958 max_rank=1 min=7708 min_rank=6 imbalance_ratio=1.093506 max_load_reduction=56578 max_load_reduction_pct=0.863312 moved_to_echo=31859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[75, 2425, 818, 207, 1245, 286, 932, 726, 268, 168, 0, 0, 0, 0, 0, 0, 1465, 189, 1627, 275, 478, 1020, 332, 437, 1295, 655, 264, 0, 0, 0, 0, 0, 122, 677, 1734, 869, 956, 396, 2246, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 591, 1229, 908, 1099, 1422, 374, 2146, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 478, 685, 445, 153, 2473, 1777, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 177, 286, 1974, 1060, 1629, 1380, 847, 954, 0, 0, 0, 0, 0, 0, 0, 0, 296, 415, 237, 1973, 740, 566, 644, 102, 2261, 301, 0, 0, 0, 0, 0, 0, 548, 378, 874, 1719, 1847, 433, 1080, 1130, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7150, 8037, 9108, 8955, 8435, 8307, 7535, 8009] avg=8192.0000 max=9108 max_rank=2 min=7150 min_rank=0 imbalance_ratio=1.111816 max_load_reduction=56428 max_load_reduction_pct=0.861023 moved_to_echo=32286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[55, 2485, 515, 250, 1518, 292, 1109, 1017, 349, 221, 0, 0, 0, 0, 0, 0, 1047, 141, 1224, 304, 1108, 778, 343, 305, 1613, 795, 343, 0, 0, 0, 0, 0, 45, 402, 578, 885, 1244, 83, 2728, 1906, 0, 0, 0, 0, 0, 0, 0, 0, 677, 968, 1009, 1088, 1333, 221, 1460, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 1538, 897, 265, 860, 329, 2027, 2712, 806, 0, 0, 0, 0, 0, 0, 0, 0, 202, 196, 1200, 627, 1367, 1442, 1222, 682, 0, 0, 0, 0, 0, 0, 0, 0, 227, 450, 301, 2177, 836, 518, 722, 120, 2590, 375, 0, 0, 0, 0, 0, 0, 742, 478, 617, 2255, 1967, 460, 1139, 1212, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7811, 8001, 7871, 8295, 9434, 6938, 8316, 8870] avg=8192.0000 max=9434 max_rank=4 min=6938 min_rank=5 imbalance_ratio=1.151611 max_load_reduction=56102 max_load_reduction_pct=0.856049 moved_to_echo=33558 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[409, 2761, 378, 523, 1680, 306, 744, 1018, 256, 163, 0, 0, 0, 0, 0, 0, 1366, 146, 1489, 190, 1152, 935, 194, 420, 1248, 717, 254, 0, 0, 0, 0, 0, 90, 653, 1025, 711, 758, 270, 2437, 2226, 0, 0, 0, 0, 0, 0, 0, 0, 649, 883, 850, 983, 1009, 524, 1857, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 1178, 1136, 378, 810, 80, 1821, 1775, 751, 0, 0, 0, 0, 0, 0, 0, 0, 179, 760, 1860, 879, 1766, 1469, 856, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 333, 389, 540, 2354, 1129, 484, 344, 88, 2345, 293, 0, 0, 0, 0, 0, 0, 649, 373, 1523, 1622, 1900, 454, 450, 1184, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8111, 8170, 7856, 7929, 8778, 8299, 8155] avg=8192.0000 max=8778 max_rank=5 min=7856 min_rank=3 imbalance_ratio=1.071533 max_load_reduction=56758 max_load_reduction_pct=0.866058 moved_to_echo=33161 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[9728, 1152, 12544, 2560, 8832, 6528, 2304, 3328, 11392, 5504, 2432, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1152, 22272, 5504, 2688, 12672, 3072, 7168, 7680, 2432, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1664, 2944, 12032, 6656, 11904, 12928, 8960, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4864, 8192, 7808, 8064, 10240, 3328, 12928, 10496, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6016, 3968, 8576, 14080, 13312, 4608, 6272, 9216, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2560, 3072, 2816, 16640, 7424, 4352, 4992, 1024, 20352, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1280, 4992, 8704, 7040, 7552, 1664, 18816, 16000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[10752, 5888, 3456, 4608, 1792, 17280, 16384, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2509, 249, 1103, 3590, 29, 485, 4096, 0, 0, 3866, 0, 34, 38, 12, 33, 12, 13, 4096, 3895, 139, 89, 2593, 0, 3263, 4096, 0, 146, 3195, 3742, 442, 0, 1, 460, 1386, 0, 406, 3700, 3993, 0, 8, 34, 0, 21, 336, 0, 1, 250, 92, 3404, 255, 1759, 115, 8, 4, 9, 8, 0, 0, 22, 374, 119, 1575, 4007, 1424] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[3194, 10, 183, 4087, 23, 3084, 3998, 1, 13, 3926, 0, 2, 14, 0, 30, 0, 15, 4057, 1024, 305, 4, 974, 0, 1297, 3088, 0, 4, 4088, 3967, 5, 0, 0, 1322, 276, 0, 2691, 274, 3731, 0, 31, 699, 3, 26, 3779, 29, 7, 35, 1, 3993, 270, 3906, 0, 1178, 0, 0, 46, 0, 0, 19, 15, 3, 372, 3861, 1576] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1224, 8, 0, 3299, 209, 398, 4068, 0, 0, 3144, 1, 0, 5, 69, 11, 63, 1, 4082, 1669, 1587, 0, 596, 1, 10, 2151, 0, 50, 2664, 3884, 429, 0, 3227, 2188, 2121, 12, 3997, 2561, 4086, 15, 7, 630, 0, 9, 1433, 4, 246, 6, 0, 2450, 2635, 1422, 84, 84, 0, 26, 133, 0, 1, 9, 39, 0, 670, 3983, 3835] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[3241, 308, 74, 2784, 4, 161, 4096, 164, 0, 1226, 2, 1, 6, 0, 35, 1, 2, 4055, 276, 26, 129, 1016, 0, 3677, 3982, 21, 19, 3783, 3950, 256, 0, 12, 406, 18, 2, 1487, 4062, 833, 1, 1205, 282, 0, 231, 4060, 14, 1827, 589, 45, 3258, 3045, 3852, 0, 1508, 124, 111, 0, 0, 35, 153, 0, 0, 1202, 3786, 93] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4053, 0, 2356, 3602, 398, 131, 4046, 0, 30, 687, 9, 1, 0, 0, 484, 0, 31, 3642, 1924, 0, 57, 2002, 0, 43, 2248, 5, 0, 4055, 3891, 30, 47, 64, 878, 191, 0, 2974, 2898, 3421, 0, 2, 829, 19, 5, 2516, 9, 174, 143, 9, 2791, 36, 4032, 0, 2790, 0, 20, 288, 1, 28, 570, 1993, 3, 99, 4044, 937] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2358, 31, 1, 1114, 194, 2000, 4096, 2, 12, 4050, 0, 7, 0, 0, 7, 27, 3, 4004, 1730, 1113, 73, 1041, 15, 1856, 2429, 0, 17, 3609, 4001, 6, 7, 291, 1797, 121, 8, 4093, 2014, 3431, 1, 11, 173, 2, 32, 3864, 20, 1071, 600, 2, 3720, 1324, 3742, 15, 16, 0, 0, 217, 58, 3, 388, 149, 0, 392, 4087, 91] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3259, 0, 112, 4061, 2163, 677, 3538, 42, 65, 4006, 0, 0, 0, 24, 89, 121, 21, 4051, 827, 1066, 0, 1018, 1, 86, 3885, 0, 0, 3766, 3955, 12, 9, 35, 4, 120, 1, 3950, 1410, 4066, 2, 104, 588, 0, 0, 1457, 209, 135, 1, 7, 2913, 615, 3026, 0, 585, 2, 8, 31, 1, 0, 9, 56, 0, 2219, 4022, 3106] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4031, 10, 280, 2885, 659, 448, 4095, 0, 1, 1846, 0, 0, 0, 14, 0, 424, 0, 4077, 1, 2939, 59, 878, 0, 2814, 1370, 0, 1, 2094, 3415, 1, 13, 153, 231, 783, 0, 4087, 4006, 2887, 0, 747, 1440, 0, 81, 743, 0, 321, 2843, 37, 2881, 4062, 3294, 27, 2, 32, 0, 0, 0, 0, 4, 10, 0, 1446, 3064, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3241, 308, 74, 2784, 4, 161, 32, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1226, 2, 1, 6, 0, 35, 1, 4064, 1192, 0, 0, 0, 0, 0, 0, 2, 2952, 276, 26, 129, 1016, 0, 3677, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 21, 19, 3783, 1249, 256, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 406, 18, 2, 1487, 4062, 204, 1, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 231, 4060, 14, 1827, 589, 45, 629, 1509, 223, 0, 0, 0, 0, 0, 2666, 3045, 3852, 0, 1508, 124, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 153, 0, 0, 1202, 3786, 93, 880, 592, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6768, 6527, 8078, 9322, 7385, 9409, 11306, 6741] avg=8192.0000 max=11306 max_rank=6 min=6527 min_rank=1 imbalance_ratio=1.380127 max_load_reduction=54230 max_load_reduction_pct=0.827484 moved_to_echo=34841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4053, 0, 2356, 3602, 398, 131, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 687, 9, 1, 0, 0, 484, 0, 4014, 1175, 0, 0, 0, 0, 0, 0, 31, 2652, 1924, 0, 57, 2002, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 2248, 5, 0, 4055, 1230, 30, 47, 64, 0, 0, 0, 0, 0, 0, 0, 0, 878, 191, 0, 2974, 2898, 837, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 829, 19, 5, 2516, 9, 174, 143, 9, 2584, 1486, 200, 0, 0, 0, 0, 0, 2284, 36, 4032, 0, 2790, 0, 20, 288, 0, 0, 0, 0, 0, 0, 0, 0, 1, 28, 570, 1993, 3, 99, 4044, 937, 790, 507, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10572, 6400, 6709, 7679, 7780, 7974, 9450, 8972] avg=8192.0000 max=10572 max_rank=0 min=6400 min_rank=1 imbalance_ratio=1.290527 max_load_reduction=54964 max_load_reduction_pct=0.838684 moved_to_echo=34176 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2358, 31, 1, 1114, 194, 2000, 32, 2, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4050, 0, 7, 0, 0, 7, 27, 4064, 1208, 0, 0, 0, 0, 0, 0, 3, 2915, 1730, 1113, 73, 1041, 15, 1856, 0, 0, 0, 0, 0, 0, 0, 0, 2429, 0, 17, 3609, 1265, 6, 7, 291, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 121, 8, 4093, 2014, 839, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 173, 2, 32, 3864, 20, 1071, 600, 2, 2592, 1528, 220, 0, 0, 0, 0, 0, 3044, 1324, 3742, 15, 16, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 58, 3, 388, 149, 0, 392, 4087, 91, 869, 676, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5732, 9375, 8746, 7624, 8884, 10104, 8358, 6713] avg=8192.0000 max=10104 max_rank=5 min=5732 min_rank=0 imbalance_ratio=1.233398 max_load_reduction=55432 max_load_reduction_pct=0.845825 moved_to_echo=34059 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3259, 0, 112, 4061, 2163, 677, 28, 42, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4006, 0, 0, 0, 24, 89, 121, 3510, 1194, 0, 0, 0, 0, 0, 0, 21, 2950, 827, 1066, 0, 1018, 1, 86, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 0, 3766, 1250, 12, 9, 35, 0, 0, 0, 0, 0, 0, 0, 0, 4, 120, 1, 3950, 1410, 995, 2, 104, 0, 0, 0, 0, 0, 0, 0, 0, 588, 0, 0, 1457, 209, 135, 1, 7, 3071, 1511, 222, 0, 0, 0, 0, 0, 2384, 615, 3026, 0, 585, 2, 8, 31, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 56, 0, 2219, 4022, 3106, 879, 529, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10342, 9009, 5969, 8957, 6586, 7201, 6651, 10821] avg=8192.0000 max=10821 max_rank=7 min=5969 min_rank=2 imbalance_ratio=1.320923 max_load_reduction=54715 max_load_reduction_pct=0.834885 moved_to_echo=31259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1224, 8, 0, 3299, 209, 398, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3144, 1, 0, 5, 69, 11, 63, 4036, 1172, 0, 0, 0, 0, 0, 0, 1, 2972, 1669, 1587, 0, 596, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 0, 50, 2664, 1228, 429, 0, 3227, 0, 0, 0, 0, 0, 0, 0, 0, 2188, 2121, 12, 3997, 2561, 1000, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 9, 1433, 4, 246, 6, 0, 3086, 1484, 224, 0, 0, 0, 0, 0, 2005, 2635, 1422, 84, 84, 0, 26, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 39, 0, 670, 3983, 3835, 886, 445, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5170, 8501, 6836, 9749, 11901, 7122, 6389, 9868] avg=8192.0000 max=11901 max_rank=4 min=5170 min_rank=0 imbalance_ratio=1.452759 max_load_reduction=53635 max_load_reduction_pct=0.818405 moved_to_echo=35280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2509, 249, 1103, 3590, 29, 485, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 0, 34, 38, 12, 33, 12, 4064, 1130, 0, 0, 0, 0, 0, 0, 13, 2982, 3895, 139, 89, 2593, 0, 3263, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 146, 3195, 1183, 442, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 460, 1386, 0, 406, 3700, 977, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 21, 336, 0, 1, 250, 92, 3016, 1429, 225, 0, 0, 0, 0, 0, 2786, 255, 1759, 115, 8, 4, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 374, 119, 1575, 4007, 1424, 889, 618, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7997, 9189, 12974, 9063, 6937, 5404, 4944, 9028] avg=8192.0000 max=12974 max_rank=2 min=4944 min_rank=6 imbalance_ratio=1.583740 max_load_reduction=52562 max_load_reduction_pct=0.802032 moved_to_echo=26313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4031, 10, 280, 2885, 659, 448, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1846, 0, 0, 0, 14, 0, 424, 4066, 1034, 0, 0, 0, 0, 0, 0, 0, 2961, 1, 2939, 59, 878, 0, 2814, 0, 0, 0, 0, 0, 0, 0, 0, 1370, 0, 1, 2094, 1071, 1, 13, 153, 0, 0, 0, 0, 0, 0, 0, 0, 231, 783, 0, 4087, 4006, 702, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 1440, 0, 81, 743, 0, 321, 2843, 37, 2185, 1310, 227, 0, 0, 0, 0, 0, 2354, 4062, 3294, 27, 2, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 10, 0, 1446, 3064, 0, 889, 527, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8342, 7385, 9652, 4703, 10556, 9187, 9771, 5940] avg=8192.0000 max=10556 max_rank=4 min=4703 min_rank=3 imbalance_ratio=1.288574 max_load_reduction=54980 max_load_reduction_pct=0.838928 moved_to_echo=35454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3194, 10, 183, 4087, 23, 3084, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3926, 0, 2, 14, 0, 30, 0, 3967, 1198, 0, 0, 0, 0, 0, 0, 15, 2953, 1024, 305, 4, 974, 0, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 3088, 0, 4, 4088, 1254, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 276, 0, 2691, 274, 913, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 699, 3, 26, 3779, 29, 7, 35, 1, 2818, 1515, 223, 0, 0, 0, 0, 0, 3267, 270, 3906, 0, 1178, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 15, 3, 372, 3861, 1576, 881, 726, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10613, 9150, 6572, 8439, 5507, 9135, 8667, 7453] avg=8192.0000 max=10613 max_rank=0 min=5507 min_rank=4 imbalance_ratio=1.295532 max_load_reduction=54923 max_load_reduction_pct=0.838058 moved_to_echo=30762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[20864, 12288, 25088, 256, 6272, 256, 256, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7296, 5120, 128, 23808, 20992, 6528, 128, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[23936, 640, 4224, 25472, 3712, 7424, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4736, 128, 512, 18304, 384, 3840, 4480, 256, 20096, 11776, 1792, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23296, 128, 256, 27264, 9856, 1280, 128, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 23424, 11392, 7296, 512, 10240, 128, 13056, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 1280, 2688, 128, 8064, 30976, 11136, 7040, 4736, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 22784, 128, 128, 128, 128, 768, 768, 31872, 9344, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4065, 915, 3922, 336, 0, 0, 232, 3481, 2858, 566, 0, 1, 4, 85, 524, 0, 551, 0, 17, 120, 19, 3144, 0, 4096, 32, 72, 1424, 1, 1, 1186, 3770, 75, 3, 85, 3926, 0, 2827, 1093, 3, 317, 3402, 194, 23, 2578, 0, 0, 4091, 2, 0, 0, 6, 21, 4093, 79, 6, 59, 0, 4082, 18, 3046, 4055, 6, 2, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4049, 0, 3963, 119, 4029, 0, 3695, 4089, 3049, 4, 2776, 0, 0, 1, 984, 8, 13, 0, 0, 4, 0, 86, 16, 4083, 2, 124, 3, 0, 80, 0, 4096, 15, 68, 0, 4025, 38, 6, 3033, 6, 0, 623, 2604, 0, 3281, 324, 0, 4081, 942, 0, 0, 16, 921, 1817, 279, 0, 20, 3, 4047, 1, 14, 3896, 0, 133, 70] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3998, 0, 4052, 10, 26, 5, 13, 4043, 0, 315, 77, 0, 0, 32, 278, 10, 52, 0, 11, 0, 85, 2810, 0, 4021, 0, 40, 0, 0, 223, 4095, 3822, 87, 0, 48, 4053, 794, 0, 1295, 22, 735, 2277, 1472, 35, 3034, 0, 0, 4096, 2945, 0, 3, 0, 8, 4028, 65, 1012, 2, 0, 3912, 50, 3573, 3972, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3406, 3, 4093, 0, 302, 0, 3873, 3820, 4, 164, 4018, 2, 2, 0, 3852, 454, 0, 0, 18, 0, 90, 2, 0, 3726, 0, 211, 69, 0, 1748, 143, 4090, 1, 2, 283, 4014, 0, 48, 4095, 0, 0, 2, 4080, 116, 4074, 2, 0, 3543, 297, 44, 1, 0, 98, 19, 504, 225, 1, 7, 4089, 0, 4084, 1242, 570, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3758, 246, 249, 0, 91, 2, 3665, 3750, 2566, 0, 4094, 2, 0, 12, 0, 0, 2, 0, 3247, 48, 0, 1134, 276, 3, 0, 0, 0, 932, 247, 61, 3817, 40, 424, 0, 4060, 0, 5, 1922, 3, 42, 21, 4052, 0, 2006, 27, 5, 4058, 1, 1, 189, 1, 3917, 3365, 14, 692, 22, 198, 4055, 1512, 2640, 4059, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4094, 59, 4093, 337, 3410, 0, 0, 3923, 14, 32, 314, 0, 0, 16, 41, 1124, 1222, 3, 4, 166, 41, 2776, 21, 4082, 3, 0, 45, 46, 3175, 500, 2562, 103, 0, 1, 4096, 13, 0, 3712, 1, 9, 3737, 3698, 0, 3854, 4, 0, 2292, 3539, 3, 0, 65, 157, 3711, 136, 0, 0, 0, 4057, 145, 0, 1, 99, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3237, 0, 3901, 1, 0, 0, 2517, 3940, 1706, 0, 534, 168, 75, 75, 545, 1151, 113, 0, 187, 199, 0, 3789, 128, 3720, 0, 248, 513, 0, 0, 4034, 3793, 5, 9, 1585, 4070, 0, 13, 4025, 0, 215, 647, 38, 0, 2440, 25, 10, 1696, 117, 0, 8, 1, 41, 4036, 1306, 5, 167, 0, 2928, 37, 3763, 3761, 7, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[141, 292, 4074, 0, 17, 0, 3705, 4090, 3368, 39, 0, 0, 0, 518, 16, 224, 0, 0, 214, 0, 113, 4079, 0, 4026, 0, 0, 143, 0, 3, 4090, 3949, 870, 186, 0, 4092, 11, 0, 1626, 0, 81, 4080, 555, 0, 3977, 0, 0, 959, 3912, 0, 31, 0, 47, 1124, 0, 1372, 2, 0, 3535, 0, 0, 3989, 0, 0, 1986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4065, 915, 1508, 336, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2858, 566, 0, 1, 4, 85, 524, 0, 2510, 588, 0, 0, 0, 0, 0, 0, 551, 0, 17, 120, 19, 3144, 0, 4096, 1795, 0, 0, 0, 0, 0, 0, 0, 32, 72, 1424, 1, 1, 1186, 3770, 75, 1424, 31, 0, 0, 0, 0, 0, 0, 3, 85, 3926, 0, 2827, 1093, 3, 317, 509, 189, 0, 0, 0, 0, 0, 0, 3402, 194, 23, 0, 0, 0, 3582, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 21, 4093, 79, 6, 59, 3481, 68, 0, 0, 0, 0, 0, 0, 0, 2469, 18, 3046, 4055, 6, 2, 22, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7056, 7136, 9742, 8016, 8952, 7203, 7813, 9618] avg=8192.0000 max=9742 max_rank=2 min=7056 min_rank=0 imbalance_ratio=1.189209 max_load_reduction=55794 max_load_reduction_pct=0.851349 moved_to_echo=33586 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3998, 0, 1558, 10, 26, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 77, 0, 0, 32, 278, 10, 2954, 607, 0, 0, 0, 0, 0, 0, 52, 0, 11, 0, 85, 2810, 0, 4021, 1855, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 223, 4095, 3822, 87, 1365, 32, 0, 0, 0, 0, 0, 0, 0, 48, 4053, 794, 0, 1295, 22, 735, 509, 181, 0, 0, 0, 0, 0, 0, 2277, 1472, 35, 0, 0, 0, 3587, 2945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 4028, 65, 1012, 2, 4043, 80, 0, 0, 0, 0, 0, 0, 0, 2366, 50, 3573, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5610, 4273, 8834, 9664, 7637, 10316, 9241, 9961] avg=8192.0000 max=10316 max_rank=5 min=4273 min_rank=1 imbalance_ratio=1.259277 max_load_reduction=55220 max_load_reduction_pct=0.842590 moved_to_echo=37155 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4049, 0, 1523, 119, 4029, 0, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3049, 4, 2776, 0, 0, 1, 984, 8, 3195, 594, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4, 0, 86, 16, 4083, 1814, 0, 0, 0, 0, 0, 0, 0, 2, 124, 3, 0, 80, 0, 4096, 15, 1412, 32, 0, 0, 0, 0, 0, 0, 68, 0, 4025, 38, 6, 3033, 6, 0, 507, 188, 0, 0, 0, 0, 0, 0, 623, 2604, 0, 0, 324, 0, 3574, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 921, 1817, 279, 0, 20, 4089, 86, 0, 0, 0, 0, 0, 0, 3, 2447, 1, 14, 3896, 0, 133, 70, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13415, 10611, 6016, 5764, 7871, 8067, 7228, 6564] avg=8192.0000 max=13415 max_rank=0 min=5764 min_rank=3 imbalance_ratio=1.637573 max_load_reduction=52121 max_load_reduction_pct=0.795303 moved_to_echo=29730 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3758, 246, 96, 0, 91, 2, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 4094, 2, 0, 12, 0, 0, 1953, 37, 0, 0, 0, 0, 0, 0, 2, 0, 3247, 48, 0, 1134, 276, 3, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 247, 61, 3817, 40, 1415, 2, 0, 0, 0, 0, 0, 0, 424, 0, 4060, 0, 5, 1922, 3, 42, 504, 188, 0, 0, 0, 0, 0, 0, 21, 4052, 0, 0, 27, 5, 3554, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 189, 1, 3917, 3365, 14, 692, 22, 3750, 53, 0, 0, 0, 0, 0, 0, 198, 2452, 1512, 2640, 4059, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7858, 8664, 4824, 6514, 7148, 7660, 12004, 10864] avg=8192.0000 max=12004 max_rank=6 min=4824 min_rank=2 imbalance_ratio=1.465332 max_load_reduction=53532 max_load_reduction_pct=0.816833 moved_to_echo=37676 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3406, 3, 1573, 0, 302, 0, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 164, 4018, 2, 2, 0, 3852, 454, 3968, 613, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 90, 2, 0, 3726, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 211, 69, 0, 1748, 143, 4090, 1, 1427, 33, 0, 0, 0, 0, 0, 0, 2, 283, 4014, 0, 48, 4095, 0, 0, 440, 190, 0, 0, 0, 0, 0, 0, 2, 4080, 116, 0, 2, 0, 3103, 297, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1, 0, 98, 19, 504, 225, 1, 3820, 106, 0, 0, 0, 0, 0, 0, 7, 2472, 0, 4084, 1242, 570, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9157, 13077, 5710, 7722, 9072, 7600, 4818, 8380] avg=8192.0000 max=13077 max_rank=1 min=4818 min_rank=6 imbalance_ratio=1.596313 max_load_reduction=52459 max_load_reduction_pct=0.800461 moved_to_echo=29870 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4094, 59, 1573, 337, 3410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 32, 314, 0, 0, 16, 41, 1124, 3754, 613, 0, 0, 0, 0, 0, 0, 1222, 3, 4, 166, 41, 2776, 21, 4082, 1874, 0, 0, 0, 0, 0, 0, 0, 3, 0, 45, 46, 3175, 500, 2562, 103, 1416, 33, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 13, 0, 3712, 1, 9, 285, 188, 0, 0, 0, 0, 0, 0, 3737, 3698, 0, 0, 4, 0, 2007, 3539, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 65, 157, 3711, 136, 0, 0, 3923, 100, 0, 0, 0, 0, 0, 0, 0, 2453, 145, 0, 1, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9473, 5908, 10189, 7883, 8305, 12985, 8095, 2698] avg=8192.0000 max=12985 max_rank=5 min=2698 min_rank=7 imbalance_ratio=1.585083 max_load_reduction=52551 max_load_reduction_pct=0.801865 moved_to_echo=32083 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3237, 0, 1500, 1, 0, 0, 2517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1706, 0, 534, 168, 75, 75, 545, 1151, 2376, 584, 0, 0, 0, 0, 0, 0, 113, 0, 187, 199, 0, 3789, 128, 3720, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 248, 513, 0, 0, 4034, 3793, 5, 1021, 31, 0, 0, 0, 0, 0, 0, 9, 1585, 4070, 0, 13, 4025, 0, 215, 211, 136, 0, 0, 0, 0, 0, 0, 647, 38, 0, 0, 25, 10, 1485, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 41, 4036, 1306, 5, 167, 3940, 64, 0, 0, 0, 0, 0, 0, 0, 1771, 37, 3763, 3761, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7255, 7214, 9922, 9645, 10264, 2322, 9568, 9346] avg=8192.0000 max=10264 max_rank=4 min=2322 min_rank=5 imbalance_ratio=1.252930 max_load_reduction=55272 max_load_reduction_pct=0.843384 moved_to_echo=31500 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[141, 292, 1557, 0, 17, 0, 3705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3368, 39, 0, 0, 0, 518, 16, 224, 3874, 614, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 113, 4079, 0, 4026, 1867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 3, 4090, 3949, 870, 1237, 36, 0, 0, 0, 0, 0, 0, 186, 0, 4092, 11, 0, 1626, 0, 81, 123, 168, 0, 0, 0, 0, 0, 0, 4080, 555, 0, 0, 0, 0, 836, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 47, 1124, 0, 1372, 2, 4090, 103, 0, 0, 0, 0, 0, 0, 0, 2130, 0, 0, 3989, 0, 0, 1986, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5712, 8653, 10299, 10328, 6287, 9383, 6769, 8105] avg=8192.0000 max=10328 max_rank=3 min=5712 min_rank=0 imbalance_ratio=1.260742 max_load_reduction=55208 max_load_reduction_pct=0.842407 moved_to_echo=30544 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[14848, 16768, 256, 0, 384, 128, 21760, 11776, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 18560, 1792, 17152, 25088, 768, 256, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26752, 1536, 11008, 896, 7936, 128, 17792, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[13568, 1152, 11904, 256, 128, 768, 6272, 3072, 24704, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 256, 128, 5248, 22272, 2432, 3328, 384, 31232, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2048, 128, 3712, 640, 384, 17920, 512, 27776, 13056, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 2048, 32384, 896, 2944, 20864, 128, 1408, 3200, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 768, 2304, 1024, 5504, 14208, 29952, 1280, 10752, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4041, 1, 0, 4, 0, 1812, 164, 1366, 93, 374, 41, 0, 0, 3759, 0, 4089, 79, 4096, 301, 0, 0, 2, 2, 0, 1079, 4009, 0, 0, 75, 108, 3, 3, 0, 4093, 0, 37, 354, 35, 4091, 1884, 2075, 3934, 490, 6, 2926, 0, 4096, 16, 8, 3727, 1, 0, 4095, 0, 4093, 0, 0, 0, 0, 1, 4073, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[8, 0, 4062, 0, 4067, 1, 0, 143, 3673, 1, 0, 1, 416, 0, 0, 4086, 7, 803, 0, 0, 3, 0, 4, 3757, 191, 151, 1, 4096, 0, 0, 0, 3, 0, 769, 0, 4088, 35, 1, 3973, 8, 4091, 3761, 6, 4094, 1292, 0, 3143, 442, 189, 14, 1, 3273, 0, 0, 4093, 0, 4096, 0, 67, 0, 39, 0, 2587, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 3556, 1, 275, 0, 2, 3319, 0, 0, 4, 308, 1, 0, 0, 4005, 0, 4096, 0, 61, 17, 85, 783, 58, 2657, 696, 12, 4039, 0, 0, 0, 3, 0, 869, 74, 4093, 676, 0, 1193, 0, 3793, 4, 4003, 3007, 454, 14, 4094, 121, 4035, 2, 36, 553, 0, 12, 4055, 0, 3823, 135, 0, 9, 2062, 0, 3846, 594] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[28, 0, 4063, 2323, 30, 13, 91, 802, 0, 1402, 7, 6, 117, 12, 1, 3034, 0, 161, 3040, 0, 0, 0, 2558, 80, 3285, 203, 0, 4096, 0, 1, 2, 0, 4, 3, 14, 4095, 4, 0, 1175, 0, 3756, 0, 385, 3917, 4005, 0, 4087, 10, 3942, 163, 3, 6, 3552, 0, 4080, 13, 4007, 253, 2, 0, 183, 21, 2476, 25] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[416, 139, 3988, 1076, 20, 1, 181, 4027, 3843, 0, 634, 51, 65, 13, 0, 3286, 0, 2117, 73, 2, 56, 7, 0, 3112, 70, 154, 0, 3884, 0, 0, 0, 0, 60, 0, 0, 4047, 3, 0, 4022, 0, 3073, 701, 25, 4091, 3383, 0, 260, 9, 1552, 568, 25, 32, 9, 24, 4059, 3880, 4079, 310, 0, 13, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4078, 259, 6, 2, 2, 24, 2532, 8, 1, 0, 498, 0, 66, 3213, 0, 3989, 0, 6, 10, 0, 0, 1397, 465, 0, 0, 3647, 0, 1, 0, 25, 97, 0, 410, 4038, 285, 44, 229, 164, 2883, 4064, 512, 4058, 4094, 0, 4092, 0, 2345, 0, 0, 100, 433, 5, 4091, 4, 4091, 1728, 2581, 0, 839, 25, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 35, 4094, 1898, 1161, 315, 51, 38, 26, 0, 4041, 0, 84, 0, 0, 2675, 0, 4086, 20, 1400, 0, 0, 8, 4096, 9, 163, 0, 3913, 0, 1, 0, 127, 779, 53, 1, 3803, 4, 641, 2070, 0, 3859, 3, 4096, 3257, 1843, 0, 4, 0, 3927, 0, 0, 722, 0, 0, 4075, 987, 4078, 469, 0, 0, 57, 2, 2553, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1214, 0, 3956, 3, 1944, 0, 168, 238, 339, 3761, 3518, 0, 0, 0, 94, 2389, 0, 2581, 1, 674, 1, 150, 0, 3169, 7, 30, 3285, 139, 12, 121, 0, 1, 1, 31, 0, 2685, 4050, 2, 2753, 0, 320, 0, 5, 3433, 4096, 0, 731, 10, 3821, 5, 0, 124, 0, 0, 3804, 0, 3869, 3771, 0, 71, 1, 1, 4094, 63] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4041, 1, 0, 4, 0, 1812, 962, 0, 0, 0, 0, 0, 0, 0, 164, 1366, 93, 374, 41, 0, 0, 3759, 1823, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 79, 4096, 301, 0, 0, 2, 1499, 662, 0, 0, 0, 0, 0, 0, 2, 0, 1079, 4009, 0, 0, 75, 108, 3272, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4093, 0, 37, 354, 35, 1141, 71, 24, 0, 0, 0, 0, 0, 698, 1884, 2075, 0, 490, 6, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 8, 3727, 1, 0, 2954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3107, 0, 0, 0, 0, 1, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6820, 7620, 10728, 8545, 5761, 8079, 10802, 7181] avg=8192.0000 max=10802 max_rank=6 min=5761 min_rank=4 imbalance_ratio=1.318604 max_load_reduction=54734 max_load_reduction_pct=0.835175 moved_to_echo=31823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3556, 1, 275, 0, 2, 3319, 899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 308, 1, 0, 0, 4005, 1690, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 61, 17, 85, 783, 58, 1390, 506, 0, 0, 0, 0, 0, 0, 2657, 696, 12, 4039, 0, 0, 0, 3, 2501, 0, 0, 0, 0, 0, 0, 0, 0, 869, 74, 4093, 676, 0, 1193, 0, 1130, 65, 23, 0, 0, 0, 0, 0, 648, 4, 4003, 0, 454, 14, 4094, 121, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 2, 36, 553, 0, 12, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2901, 135, 0, 9, 2062, 0, 3846, 594, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8053, 6008, 6996, 9908, 8123, 9338, 7563, 9547] avg=8192.0000 max=9908 max_rank=3 min=6008 min_rank=1 imbalance_ratio=1.209473 max_load_reduction=55628 max_load_reduction_pct=0.848816 moved_to_echo=34571 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4062, 0, 4067, 1, 0, 143, 963, 0, 0, 0, 0, 0, 0, 0, 3673, 1, 0, 1, 416, 0, 0, 4086, 1823, 0, 0, 0, 0, 0, 0, 0, 7, 803, 0, 0, 3, 0, 4, 3757, 1499, 689, 0, 0, 0, 0, 0, 0, 191, 151, 1, 4096, 0, 0, 0, 3, 3405, 0, 0, 0, 0, 0, 0, 0, 0, 769, 0, 4088, 35, 1, 3973, 8, 1141, 71, 24, 0, 0, 0, 0, 0, 698, 3761, 6, 0, 1292, 0, 3143, 442, 0, 0, 0, 0, 0, 0, 0, 0, 189, 14, 1, 3273, 0, 0, 2952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 0, 67, 0, 39, 0, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9244, 10000, 6762, 7847, 10110, 9342, 6429, 5802] avg=8192.0000 max=10110 max_rank=4 min=5802 min_rank=7 imbalance_ratio=1.234131 max_load_reduction=55426 max_load_reduction_pct=0.845734 moved_to_echo=31683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[416, 139, 3988, 1076, 20, 1, 181, 4027, 959, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 634, 51, 65, 13, 0, 3286, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 2117, 73, 2, 56, 7, 0, 3112, 1126, 689, 0, 0, 0, 0, 0, 0, 70, 154, 0, 3884, 0, 0, 0, 0, 3402, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 4047, 3, 0, 4022, 0, 1131, 53, 24, 0, 0, 0, 0, 0, 525, 701, 25, 0, 3383, 0, 260, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1552, 568, 25, 32, 9, 24, 2928, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 3096, 310, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10807, 9261, 7182, 7510, 9340, 4903, 9018, 7515] avg=8192.0000 max=10807 max_rank=0 min=4903 min_rank=5 imbalance_ratio=1.319214 max_load_reduction=54729 max_load_reduction_pct=0.835098 moved_to_echo=30776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 4063, 2323, 30, 13, 91, 802, 942, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 7, 6, 117, 12, 1, 3034, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 161, 3040, 0, 0, 0, 2558, 80, 1377, 660, 0, 0, 0, 0, 0, 0, 3285, 203, 0, 4096, 0, 1, 2, 0, 3257, 0, 0, 0, 0, 0, 0, 0, 4, 3, 14, 4095, 4, 0, 1175, 0, 1137, 65, 24, 0, 0, 0, 0, 0, 641, 0, 385, 0, 4005, 0, 4087, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 163, 3, 6, 3552, 0, 2943, 13, 0, 0, 0, 0, 0, 0, 0, 0, 3041, 253, 2, 0, 183, 21, 2476, 25, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8292, 6252, 7876, 10844, 6521, 9128, 10622, 6001] avg=8192.0000 max=10844 max_rank=3 min=6001 min_rank=7 imbalance_ratio=1.323730 max_load_reduction=54692 max_load_reduction_pct=0.834534 moved_to_echo=32272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1214, 0, 3956, 3, 1944, 0, 168, 238, 910, 0, 0, 0, 0, 0, 0, 0, 339, 3761, 3518, 0, 0, 0, 94, 2389, 142, 0, 0, 0, 0, 0, 0, 0, 0, 2581, 1, 674, 1, 150, 0, 3169, 117, 579, 0, 0, 0, 0, 0, 0, 7, 30, 3285, 139, 12, 121, 0, 1, 2854, 0, 0, 0, 0, 0, 0, 0, 1, 31, 0, 2685, 4050, 2, 2753, 0, 1060, 5, 23, 0, 0, 0, 0, 0, 56, 0, 5, 0, 4096, 0, 731, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 5, 0, 124, 0, 0, 2744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2936, 3771, 0, 71, 1, 1, 4094, 63, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8433, 10243, 7272, 6449, 10610, 4898, 6694, 10937] avg=8192.0000 max=10937 max_rank=7 min=4898 min_rank=5 imbalance_ratio=1.335083 max_load_reduction=54599 max_load_reduction_pct=0.833115 moved_to_echo=33139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4078, 259, 6, 2, 2, 24, 962, 0, 0, 0, 0, 0, 0, 0, 2532, 8, 1, 0, 498, 0, 66, 3213, 1284, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 6, 10, 0, 0, 1397, 1056, 684, 0, 0, 0, 0, 0, 0, 465, 0, 0, 3647, 0, 1, 0, 25, 3374, 0, 0, 0, 0, 0, 0, 0, 97, 0, 410, 4038, 285, 44, 229, 164, 1140, 50, 24, 0, 0, 0, 0, 0, 493, 4064, 512, 0, 4094, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2345, 0, 0, 100, 433, 5, 2951, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3105, 1728, 2581, 0, 839, 25, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5333, 7602, 7142, 7512, 6481, 13255, 5838, 12373] avg=8192.0000 max=13255 max_rank=5 min=5333 min_rank=0 imbalance_ratio=1.618042 max_load_reduction=52281 max_load_reduction_pct=0.797745 moved_to_echo=37947 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 35, 4094, 1898, 1161, 315, 51, 38, 962, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4041, 0, 84, 0, 0, 2675, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 20, 1400, 0, 0, 8, 4096, 1419, 549, 0, 0, 0, 0, 0, 0, 9, 163, 0, 3913, 0, 1, 0, 127, 2708, 0, 0, 0, 0, 0, 0, 0, 779, 53, 1, 3803, 4, 641, 2070, 0, 1141, 69, 29, 0, 0, 0, 0, 0, 647, 3, 4096, 0, 1843, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 722, 0, 0, 2934, 987, 0, 0, 0, 0, 0, 0, 0, 0, 3087, 469, 0, 0, 57, 2, 2553, 12, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8554, 8550, 11578, 6921, 8590, 6593, 8570, 6180] avg=8192.0000 max=11578 max_rank=2 min=6180 min_rank=7 imbalance_ratio=1.413330 max_load_reduction=53958 max_load_reduction_pct=0.823334 moved_to_echo=29933 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[24448, 6784, 2688, 128, 3200, 128, 27904, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23936, 768, 128, 8576, 4096, 128, 23424, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4480, 10496, 11136, 0, 19712, 128, 19456, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6784, 1408, 4480, 27904, 128, 128, 128, 384, 24832, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[10624, 6656, 8320, 768, 1280, 128, 256, 26496, 11648, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1792, 256, 31872, 5632, 7552, 384, 512, 10496, 7680, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 1792, 512, 30976, 5120, 768, 15872, 256, 9088, 512, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 22016, 3328, 6272, 512, 256, 3456, 15744, 9600, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[3802, 0, 350, 0, 3989, 252, 9, 4080, 0, 4006, 36, 1095, 3937, 13, 0, 0, 3, 1077, 0, 0, 0, 1, 0, 3724, 487, 1445, 0, 0, 3948, 3945, 1812, 43, 2772, 3911, 151, 189, 0, 10, 0, 1, 68, 36, 25, 3623, 6, 0, 0, 0, 37, 4083, 1, 0, 0, 0, 440, 4096, 3375, 0, 71, 1, 445, 68, 0, 4073] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4096, 4014, 0, 1313, 3451, 0, 0, 4081, 0, 2023, 0, 3965, 16, 0, 0, 14, 3699, 3, 0, 2000, 544, 0, 0, 8, 122, 4083, 99, 61, 5, 4096, 82, 3106, 563, 4009, 1522, 54, 0, 87, 4, 0, 4036, 42, 3, 4076, 54, 245, 0, 839, 41, 3561, 205, 0, 0, 0, 2, 56, 4009, 0, 18, 0, 72, 0, 0, 1157] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[3373, 0, 4095, 3, 3837, 1317, 43, 3407, 13, 4095, 14, 3980, 1508, 35, 0, 0, 3, 0, 17, 245, 1569, 0, 0, 0, 3, 3885, 0, 50, 3441, 3491, 2, 0, 1851, 3, 6, 2821, 50, 38, 1238, 0, 4080, 115, 0, 35, 4089, 0, 0, 11, 0, 3633, 0, 0, 0, 1000, 1, 0, 3472, 232, 71, 65, 62, 162, 0, 4075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4096, 408, 8, 2194, 3254, 7, 0, 4064, 0, 3237, 0, 3265, 3387, 0, 6, 2, 0, 16, 0, 0, 313, 1, 10, 0, 2389, 3909, 107, 923, 3926, 4035, 118, 2018, 2043, 0, 0, 0, 0, 4053, 0, 922, 737, 9, 0, 4036, 0, 0, 31, 13, 275, 124, 3, 28, 0, 0, 4, 370, 4079, 1008, 0, 1376, 634, 5, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4048, 1, 68, 0, 4083, 6, 0, 3820, 0, 4089, 0, 3991, 615, 0, 41, 2, 2282, 27, 0, 1, 3624, 0, 0, 0, 391, 3004, 0, 2, 25, 248, 213, 11, 0, 59, 405, 2, 1, 3553, 0, 16, 3956, 0, 4005, 4094, 885, 0, 137, 0, 1, 3672, 170, 1593, 0, 1202, 0, 0, 4050, 0, 15, 0, 1516, 1549, 1, 4062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4094, 93, 239, 0, 3657, 197, 96, 3894, 0, 3634, 1, 4095, 0, 425, 39, 0, 4075, 155, 0, 0, 4096, 54, 0, 0, 2776, 3951, 9, 179, 3488, 0, 49, 2225, 3608, 0, 1, 0, 23, 4079, 0, 58, 3638, 0, 0, 0, 0, 7, 0, 0, 5, 3961, 0, 0, 1, 0, 96, 21, 4094, 7, 203, 123, 3, 0, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 334, 32, 0, 632, 0, 4065, 0, 2755, 48, 0, 1, 1, 3966, 70, 0, 15, 1, 14, 0, 2678, 0, 4053, 0, 4, 4090, 141, 305, 0, 4096, 1, 10, 2, 4, 3645, 2, 798, 4063, 654, 339, 327, 0, 542, 1226, 10, 0, 4092, 0, 1, 3, 229, 3819, 1753, 2747, 0, 4074, 4037, 0, 74, 812, 874] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 3653, 0, 0, 3230, 0, 3901, 4020, 3928, 0, 0, 17, 2, 135, 6, 0, 3002, 49, 560, 52, 1, 3925, 4079, 0, 17, 3707, 82, 0, 238, 84, 224, 1168, 0, 19, 3904, 0, 1, 26, 0, 496, 1056, 587, 13, 0, 2383, 0, 3465, 5, 0, 4, 0, 4095, 330, 4024, 0, 278, 579, 0, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 350, 0, 3989, 252, 9, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3347, 36, 1095, 3937, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1077, 0, 0, 0, 1, 0, 3724, 3291, 0, 0, 0, 0, 0, 0, 0, 487, 261, 0, 0, 3948, 3945, 1812, 43, 0, 0, 0, 0, 0, 0, 0, 0, 2772, 3911, 151, 189, 0, 10, 0, 1, 432, 561, 0, 0, 0, 0, 0, 0, 68, 36, 25, 3623, 6, 0, 0, 0, 659, 387, 401, 0, 0, 0, 0, 0, 37, 4083, 1, 0, 0, 0, 440, 4096, 752, 511, 0, 0, 0, 0, 0, 0, 2427, 0, 71, 1, 445, 68, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8428, 8096, 10496, 8027, 5205, 9920, 7085] avg=8192.0000 max=10496 max_rank=3 min=5205 min_rank=5 imbalance_ratio=1.281250 max_load_reduction=55040 max_load_reduction_pct=0.839844 moved_to_echo=30237 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 3, 3837, 1317, 43, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3422, 14, 3980, 1508, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 17, 245, 1569, 0, 0, 0, 2919, 0, 0, 0, 0, 0, 0, 0, 3, 699, 0, 50, 3441, 3491, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1851, 3, 6, 2821, 50, 38, 1238, 0, 1163, 577, 0, 0, 0, 0, 0, 0, 4080, 115, 0, 35, 4089, 0, 0, 11, 673, 398, 335, 0, 0, 0, 0, 0, 0, 3633, 0, 0, 0, 1000, 1, 0, 2023, 454, 0, 0, 0, 0, 0, 0, 2497, 232, 71, 65, 62, 162, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12367, 8972, 4753, 7686, 7747, 9736, 7111, 7164] avg=8192.0000 max=12367 max_rank=0 min=4753 min_rank=2 imbalance_ratio=1.509644 max_load_reduction=53169 max_load_reduction_pct=0.811295 moved_to_echo=31758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 408, 8, 2194, 3254, 7, 0, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 0, 3265, 3387, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 313, 1, 10, 0, 3545, 0, 0, 0, 0, 0, 0, 0, 2389, 703, 107, 923, 3926, 4035, 118, 2018, 0, 0, 0, 0, 0, 0, 0, 0, 2043, 0, 0, 0, 0, 4053, 0, 922, 1170, 678, 0, 0, 0, 0, 0, 0, 737, 9, 0, 4036, 0, 0, 31, 13, 532, 468, 399, 0, 0, 0, 0, 0, 275, 124, 3, 28, 0, 0, 4, 370, 2036, 551, 0, 0, 0, 0, 0, 0, 2933, 1008, 0, 1376, 634, 5, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9536, 9365, 3885, 14219, 8866, 6225, 3391, 10049] avg=8192.0000 max=14219 max_rank=3 min=3391 min_rank=6 imbalance_ratio=1.735718 max_load_reduction=51317 max_load_reduction_pct=0.783035 moved_to_echo=28531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 93, 239, 0, 3657, 197, 96, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3036, 1, 4095, 0, 425, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 155, 0, 0, 4096, 54, 0, 0, 3543, 0, 0, 0, 0, 0, 0, 0, 2776, 711, 9, 179, 3488, 0, 49, 2225, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 0, 1, 0, 23, 4079, 0, 58, 1183, 680, 0, 0, 0, 0, 0, 0, 3638, 0, 0, 0, 0, 7, 0, 0, 598, 469, 383, 0, 0, 0, 0, 0, 5, 3961, 0, 0, 1, 0, 96, 21, 2057, 551, 0, 0, 0, 0, 0, 0, 2945, 7, 203, 123, 3, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7793, 7596, 11923, 9437, 9632, 5095, 6692, 7368] avg=8192.0000 max=11923 max_rank=2 min=5095 min_rank=5 imbalance_ratio=1.455444 max_load_reduction=53613 max_load_reduction_pct=0.818069 moved_to_echo=28787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4014, 0, 1313, 3451, 0, 0, 3680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1691, 0, 3965, 16, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3699, 3, 0, 2000, 544, 0, 0, 8, 3546, 0, 0, 0, 0, 0, 0, 0, 122, 734, 99, 61, 5, 4096, 82, 3106, 0, 0, 0, 0, 0, 0, 0, 0, 563, 4009, 1522, 54, 0, 87, 4, 0, 1223, 666, 0, 0, 0, 0, 0, 0, 4036, 42, 3, 4076, 54, 245, 0, 839, 332, 459, 401, 0, 0, 0, 0, 0, 41, 3561, 205, 0, 0, 0, 2, 56, 2126, 550, 0, 0, 0, 0, 0, 0, 2884, 0, 18, 0, 72, 0, 0, 1157, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12458, 5686, 9800, 8305, 8128, 10487, 6541, 4131] avg=8192.0000 max=12458 max_rank=0 min=4131 min_rank=7 imbalance_ratio=1.520752 max_load_reduction=53078 max_load_reduction_pct=0.809906 moved_to_echo=29287 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 334, 32, 0, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3392, 0, 2755, 48, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 70, 0, 15, 1, 14, 0, 2678, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 719, 0, 4, 4090, 141, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 10, 2, 4, 3645, 2, 798, 1219, 461, 0, 0, 0, 0, 0, 0, 4063, 654, 339, 327, 0, 542, 1226, 10, 673, 319, 66, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 3, 229, 3819, 1753, 2115, 550, 0, 0, 0, 0, 0, 0, 1967, 0, 4074, 4037, 0, 74, 812, 874, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[933, 6197, 10290, 5259, 10238, 8219, 12562, 11838] avg=8192.0000 max=12562 max_rank=6 min=933 min_rank=0 imbalance_ratio=1.533447 max_load_reduction=52974 max_load_reduction_pct=0.808319 moved_to_echo=42857 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3653, 0, 0, 2913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3259, 4020, 3928, 0, 0, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 135, 6, 0, 3002, 49, 560, 52, 1, 3546, 0, 0, 0, 0, 0, 0, 0, 3925, 734, 0, 17, 3707, 82, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 84, 224, 1168, 0, 19, 3904, 0, 1, 1221, 669, 0, 0, 0, 0, 0, 0, 26, 0, 496, 1056, 587, 13, 0, 2383, 642, 461, 317, 0, 0, 0, 0, 0, 0, 3465, 5, 0, 4, 0, 4095, 330, 2124, 550, 0, 0, 0, 0, 0, 0, 2894, 0, 278, 579, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6567, 11226, 7351, 8703, 7290, 5981, 10573, 7845] avg=8192.0000 max=11226 max_rank=1 min=5981 min_rank=5 imbalance_ratio=1.370361 max_load_reduction=54310 max_load_reduction_pct=0.828705 moved_to_echo=31689 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 68, 0, 4083, 6, 0, 3445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3417, 0, 3991, 615, 0, 41, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2282, 27, 0, 1, 3624, 0, 0, 0, 3504, 0, 0, 0, 0, 0, 0, 0, 391, 541, 0, 2, 25, 248, 213, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 405, 2, 1, 3553, 0, 16, 899, 673, 0, 0, 0, 0, 0, 0, 3956, 0, 4005, 4094, 885, 0, 137, 0, 672, 464, 375, 0, 0, 0, 0, 0, 1, 3672, 170, 1593, 0, 1202, 0, 0, 1564, 544, 0, 0, 0, 0, 0, 0, 2913, 0, 15, 0, 1516, 1549, 1, 4062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7603, 8066, 9438, 1431, 5608, 14588, 8746, 10056] avg=8192.0000 max=14588 max_rank=5 min=1431 min_rank=3 imbalance_ratio=1.780762 max_load_reduction=50948 max_load_reduction_pct=0.777405 moved_to_echo=38998 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 4608, 4864, 3584, 26368, 1920, 256, 24576, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[10112, 5120, 256, 1280, 22656, 16128, 2688, 7680, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[21504, 1280, 4736, 6272, 2816, 1920, 896, 26624, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 24320, 4096, 27136, 9600, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[14208, 1408, 128, 5376, 10240, 640, 128, 6528, 27520, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 26624, 384, 1664, 128, 2432, 8576, 6656, 14848, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[15104, 8320, 3328, 3072, 128, 19456, 1280, 1920, 8576, 4992, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[20608, 896, 4992, 17280, 5632, 896, 1408, 3328, 4864, 3456, 2688, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4039, 0, 13, 0, 5, 4092, 0, 5, 25, 0, 4031, 76, 0, 8, 0, 150, 0, 1284, 4073, 100, 0, 2118, 4060, 3980, 3117, 2952, 0, 0, 0, 0, 4, 4095, 1918, 4, 43, 104, 4093, 0, 37, 5, 0, 0, 144, 2874, 0, 6, 60, 0, 97, 0, 4089, 2, 5, 4093, 8, 0, 0, 4094, 0, 0, 3231, 4, 2337, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2036, 57, 864, 4, 259, 0, 7, 0, 3916, 0, 2198, 2, 1, 16, 0, 0, 4, 0, 3779, 8, 1763, 4095, 3997, 4039, 8, 3504, 0, 530, 15, 0, 4047, 3813, 83, 45, 1, 9, 0, 117, 0, 42, 0, 667, 0, 23, 0, 684, 2443, 1, 1437, 0, 4096, 4009, 4079, 350, 0, 0, 177, 2924, 1353, 2, 0, 0, 4031, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[3552, 19, 8, 0, 0, 1, 0, 8, 947, 3504, 3853, 3, 1, 3508, 0, 9, 130, 4096, 4052, 0, 0, 0, 753, 4088, 3959, 4061, 210, 0, 0, 0, 4070, 3596, 413, 35, 370, 0, 0, 238, 0, 0, 17, 0, 47, 873, 0, 0, 19, 0, 3878, 0, 4065, 4044, 4047, 0, 37, 0, 1413, 0, 1061, 0, 421, 0, 0, 130] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[259, 0, 946, 1, 0, 2760, 39, 3241, 0, 72, 4004, 0, 64, 4091, 53, 96, 1, 4070, 4096, 0, 239, 3934, 1161, 1371, 4010, 1676, 2, 0, 201, 0, 4094, 2384, 3254, 0, 3292, 0, 22, 0, 0, 2, 9, 0, 4087, 0, 163, 11, 2910, 0, 1748, 0, 3617, 197, 59, 2, 1, 0, 0, 1658, 840, 0, 272, 0, 3, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[2224, 0, 10, 0, 3323, 11, 0, 51, 270, 297, 4034, 1, 0, 3, 0, 27, 5, 4080, 112, 3693, 1539, 0, 3953, 4096, 944, 3, 0, 0, 0, 86, 4094, 4090, 0, 0, 4083, 3279, 0, 0, 0, 0, 0, 0, 229, 571, 0, 0, 4033, 0, 1028, 3243, 4069, 50, 1, 481, 0, 0, 3112, 3, 206, 0, 4005, 1, 196, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[3949, 16, 0, 388, 7, 1, 0, 187, 3890, 5, 4096, 8, 0, 9, 10, 0, 0, 3829, 3329, 1840, 42, 3965, 15, 3, 3948, 82, 0, 0, 0, 0, 4073, 4094, 0, 2, 1002, 438, 110, 253, 0, 147, 0, 2224, 0, 0, 79, 375, 1444, 0, 4089, 4096, 3437, 2252, 1024, 56, 0, 0, 0, 8, 0, 0, 2742, 0, 3914, 58] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[69, 0, 1, 0, 9, 3073, 0, 7, 565, 2500, 2722, 0, 0, 2512, 1, 0, 0, 2432, 3832, 57, 340, 4096, 4096, 3710, 383, 4092, 0, 476, 0, 0, 4096, 4057, 1269, 0, 75, 0, 0, 0, 0, 0, 0, 3791, 2, 0, 0, 4, 3396, 0, 4089, 35, 4096, 71, 3839, 1260, 0, 20, 108, 0, 0, 0, 0, 0, 298, 57] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[3693, 0, 47, 0, 302, 1, 0, 1847, 3485, 3348, 2468, 80, 0, 2531, 0, 2, 130, 4023, 4043, 0, 0, 3865, 3870, 4096, 0, 2961, 0, 0, 0, 0, 4095, 3321, 0, 0, 1361, 3, 298, 0, 0, 2317, 0, 0, 3, 3950, 0, 0, 0, 0, 1, 1, 3001, 242, 4096, 1, 113, 0, 0, 2, 79, 0, 1026, 2, 832, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4039, 0, 13, 0, 5, 4092, 0, 5, 1844, 558, 0, 0, 0, 0, 0, 0, 25, 0, 4031, 76, 0, 8, 0, 150, 96, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 100, 0, 2118, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3117, 2952, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1918, 4, 43, 104, 4093, 0, 37, 5, 4095, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 144, 2874, 0, 6, 60, 0, 2982, 1631, 0, 0, 0, 0, 0, 0, 97, 0, 988, 2, 5, 4093, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3231, 4, 2337, 61, 2349, 1257, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10556, 4386, 6908, 6073, 11390, 7697, 5193, 13333] avg=8192.0000 max=13333 max_rank=7 min=4386 min_rank=1 imbalance_ratio=1.627563 max_load_reduction=52203 max_load_reduction_pct=0.796555 moved_to_echo=37613 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2036, 57, 864, 4, 259, 0, 7, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 2198, 2, 1, 16, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8, 1763, 4095, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3504, 0, 530, 15, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 45, 1, 9, 0, 117, 0, 42, 3813, 1012, 0, 0, 0, 0, 0, 0, 0, 667, 0, 23, 0, 684, 2443, 1, 2767, 1655, 0, 0, 0, 0, 0, 0, 1437, 0, 990, 4009, 4079, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 2924, 1353, 2, 0, 0, 4031, 1, 2384, 1259, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5074, 6169, 9867, 8068, 5122, 8240, 10865, 12131] avg=8192.0000 max=12131 max_rank=7 min=5074 min_rank=0 imbalance_ratio=1.480835 max_load_reduction=53405 max_load_reduction_pct=0.814896 moved_to_echo=36358 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3552, 19, 8, 0, 0, 1, 0, 8, 1833, 1781, 0, 0, 0, 0, 0, 0, 947, 3504, 3853, 3, 1, 3508, 0, 9, 307, 36, 0, 0, 0, 0, 0, 0, 130, 2008, 0, 0, 0, 0, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 4061, 210, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 35, 370, 0, 0, 238, 0, 0, 3596, 1086, 0, 0, 0, 0, 0, 0, 17, 0, 47, 873, 0, 0, 19, 0, 2966, 1675, 0, 0, 0, 0, 0, 0, 3878, 0, 982, 4044, 4047, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1413, 0, 1061, 0, 421, 0, 0, 130, 2413, 1250, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7202, 12168, 2891, 12264, 5738, 5597, 12988, 6688] avg=8192.0000 max=12988 max_rank=6 min=2891 min_rank=2 imbalance_ratio=1.585449 max_load_reduction=52548 max_load_reduction_pct=0.801819 moved_to_echo=31011 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3949, 16, 0, 388, 7, 1, 0, 187, 1550, 1665, 0, 0, 0, 0, 0, 0, 3890, 5, 4096, 8, 0, 9, 10, 0, 287, 36, 0, 0, 0, 0, 0, 0, 0, 1877, 0, 1840, 42, 3965, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 82, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1002, 438, 110, 253, 0, 147, 4094, 893, 0, 0, 0, 0, 0, 0, 0, 2224, 0, 0, 79, 375, 1444, 0, 2436, 2, 0, 0, 0, 0, 0, 0, 4089, 4096, 831, 2252, 1024, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2742, 0, 3914, 58, 1, 1056, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7763, 8341, 7739, 8067, 6939, 6560, 12348, 7779] avg=8192.0000 max=12348 max_rank=6 min=6560 min_rank=5 imbalance_ratio=1.507324 max_load_reduction=53188 max_load_reduction_pct=0.811584 moved_to_echo=33626 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[259, 0, 946, 1, 0, 2760, 39, 3241, 1631, 1770, 0, 0, 0, 0, 0, 0, 0, 72, 4004, 0, 64, 4091, 53, 96, 305, 36, 0, 0, 0, 0, 0, 0, 1, 1995, 0, 0, 239, 3934, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 1676, 2, 0, 201, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3254, 0, 3292, 0, 22, 0, 0, 2, 2384, 1098, 0, 0, 0, 0, 0, 0, 9, 0, 4087, 0, 163, 11, 2910, 0, 2998, 562, 0, 0, 0, 0, 0, 0, 1748, 0, 874, 197, 59, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1658, 840, 0, 272, 0, 3, 524, 809, 1112, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10647, 8721, 7330, 9947, 10052, 10740, 2881, 5218] avg=8192.0000 max=10740 max_rank=5 min=2881 min_rank=6 imbalance_ratio=1.311035 max_load_reduction=54796 max_load_reduction_pct=0.836121 moved_to_echo=28891 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[69, 0, 1, 0, 9, 3073, 0, 7, 1847, 1058, 0, 0, 0, 0, 0, 0, 565, 2500, 2722, 0, 0, 2512, 1, 0, 182, 36, 0, 0, 0, 0, 0, 0, 0, 1192, 0, 57, 340, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 4092, 0, 476, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 0, 75, 0, 0, 0, 0, 0, 4057, 1027, 0, 0, 0, 0, 0, 0, 0, 3791, 2, 0, 0, 4, 3396, 0, 2805, 1520, 0, 0, 0, 0, 0, 0, 4089, 35, 990, 71, 3839, 1260, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 298, 57, 2190, 1259, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6064, 8518, 9781, 9011, 6428, 11518, 10304, 3912] avg=8192.0000 max=11518 max_rank=5 min=3912 min_rank=7 imbalance_ratio=1.406006 max_load_reduction=54018 max_load_reduction_pct=0.824249 moved_to_echo=32162 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2224, 0, 10, 0, 3323, 11, 0, 51, 1840, 1778, 0, 0, 0, 0, 0, 0, 270, 297, 4034, 1, 0, 3, 0, 27, 309, 41, 0, 0, 0, 0, 0, 0, 5, 1993, 0, 3693, 1539, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 3, 0, 0, 0, 86, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3279, 0, 0, 0, 0, 4090, 31, 0, 0, 0, 0, 0, 0, 0, 0, 229, 571, 0, 0, 4033, 0, 81, 1679, 0, 0, 0, 0, 0, 0, 1028, 3243, 974, 50, 1, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3112, 3, 206, 0, 4005, 1, 196, 0, 2417, 1255, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9237, 4982, 11183, 5086, 11483, 6593, 5777, 11195] avg=8192.0000 max=11483 max_rank=4 min=4982 min_rank=1 imbalance_ratio=1.401733 max_load_reduction=54053 max_load_reduction_pct=0.824783 moved_to_echo=35048 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3693, 0, 47, 0, 302, 1, 0, 1847, 1353, 1750, 0, 0, 0, 0, 0, 0, 3485, 3348, 2468, 80, 0, 2531, 0, 2, 301, 36, 0, 0, 0, 0, 0, 0, 130, 1972, 0, 0, 0, 3865, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2961, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1361, 3, 298, 0, 0, 2317, 3321, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3950, 0, 0, 0, 0, 2959, 1679, 0, 0, 0, 0, 0, 0, 1, 1, 726, 242, 4096, 1, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 79, 0, 1026, 2, 832, 0, 2417, 922, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8993, 12251, 9837, 7020, 8384, 8591, 5180, 5280] avg=8192.0000 max=12251 max_rank=1 min=5180 min_rank=6 imbalance_ratio=1.495483 max_load_reduction=53285 max_load_reduction_pct=0.813065 moved_to_echo=27435 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 11776, 0, 5760, 3968, 22144, 22016, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4864, 8704, 3584, 128, 11776, 128, 11648, 896, 15104, 9472, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[13184, 9728, 27520, 256, 128, 12800, 128, 384, 1792, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[16384, 7424, 7424, 10880, 17152, 6272, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 6784, 4608, 8320, 256, 1152, 14336, 128, 20096, 10496, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[19840, 128, 1920, 512, 3968, 9984, 128, 5376, 13824, 10368, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[16384, 19456, 256, 1024, 256, 128, 28416, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7040, 128, 10240, 3840, 4608, 640, 128, 2560, 29568, 7424, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4007, 0, 0, 391, 0, 2, 0, 2, 3683, 3992, 4096, 1805, 1814, 0, 74, 0, 0, 3938, 95, 4096, 0, 6, 4, 1, 0, 0, 4091, 0, 3765, 306, 57, 382, 0, 0, 0, 0, 4006, 103, 0, 1810, 4096, 0, 4096, 33, 1, 2448, 4096, 0, 0, 124, 4, 144, 724, 3294, 2616, 0, 12, 1319, 0, 2, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4049, 0, 3, 303, 0, 8, 2975, 2365, 4096, 3356, 4080, 2399, 1152, 26, 3946, 0, 0, 4096, 3, 0, 0, 5, 0, 67, 0, 91, 4094, 0, 60, 0, 0, 3827, 4041, 4, 2, 0, 4093, 2, 0, 2759, 0, 0, 3909, 214, 0, 0, 1686, 302, 0, 1028, 3, 0, 1423, 6, 0, 0, 0, 4053, 1, 0, 1005, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 27, 20, 219, 0, 20, 2161, 0, 14, 4096, 3617, 6, 4019, 164, 0, 0, 3634, 106, 0, 4095, 0, 87, 0, 0, 0, 45, 153, 2, 3981, 0, 0, 3375, 0, 4028, 53, 12, 0, 4093, 0, 120, 913, 6, 1541, 182, 468, 0, 0, 33, 3805, 0, 0, 1, 4096, 127, 102, 3999, 91, 482, 71, 4080, 258, 3141, 3992, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 863, 1, 3127, 0, 25, 58, 3942, 3904, 4085, 1229, 3217, 32, 0, 4, 4086, 372, 163, 3979, 1047, 0, 0, 0, 76, 0, 9, 0, 3681, 0, 415, 2, 3269, 4033, 4001, 0, 1, 1, 0, 0, 0, 3368, 3643, 0, 3431, 0, 124, 0, 1698, 0, 0, 24, 445, 6, 0, 65, 0, 118, 0, 0, 3, 1170, 4082, 1736] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 1443, 0, 0, 0, 1654, 31, 4001, 3286, 3380, 0, 3702, 0, 5, 1, 2884, 0, 11, 4086, 63, 1135, 0, 115, 0, 408, 113, 3183, 331, 0, 0, 666, 759, 3915, 0, 23, 0, 0, 4075, 0, 1, 27, 1, 134, 4084, 3822, 0, 0, 895, 4096, 7, 7, 1, 2, 0, 4090, 490, 0, 1, 4093, 0, 506, 4009, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3308, 0, 2, 0, 2, 2282, 3, 850, 4089, 3539, 4060, 823, 0, 35, 4096, 3615, 10, 0, 373, 0, 3230, 0, 0, 0, 878, 0, 69, 3554, 12, 54, 3721, 0, 45, 0, 0, 0, 0, 347, 0, 0, 1577, 0, 0, 80, 1, 0, 4004, 3881, 0, 23, 118, 12, 0, 4088, 4074, 4024, 0, 0, 696, 3961, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3237, 0, 4, 22, 0, 0, 1301, 3308, 3974, 0, 379, 4025, 0, 0, 4096, 137, 108, 4094, 3885, 1750, 709, 0, 0, 1, 0, 158, 4066, 0, 487, 7, 0, 2225, 3, 0, 0, 0, 3903, 0, 0, 805, 3585, 2, 27, 16, 52, 0, 2056, 0, 0, 551, 4093, 187, 2272, 4095, 13, 52, 0, 35, 1544, 175, 4085, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 175, 3393, 0, 0, 13, 1392, 0, 0, 0, 3928, 1100, 4068, 2724, 0, 355, 0, 19, 0, 3718, 102, 3115, 1, 0, 52, 727, 8, 0, 4010, 4007, 476, 0, 3320, 379, 4001, 0, 138, 0, 1349, 18, 23, 0, 783, 0, 4093, 1, 2, 0, 3695, 4094, 42, 0, 7, 0, 4067, 4093, 1471, 0, 88, 360, 0, 0, 28, 101] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4007, 0, 0, 391, 0, 2, 3683, 361, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3992, 1026, 1805, 1814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 2584, 95, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 4091, 0, 3765, 306, 2676, 0, 0, 0, 0, 0, 0, 0, 57, 382, 0, 0, 0, 0, 4006, 103, 166, 0, 0, 0, 0, 0, 0, 0, 0, 1810, 4096, 0, 4096, 33, 1, 2448, 1685, 137, 33, 0, 0, 0, 0, 0, 4096, 0, 0, 124, 4, 144, 724, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 2616, 0, 12, 1319, 0, 2, 1, 0, 1051, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8444, 8639, 6855, 10843, 4714, 14339, 6701, 5001] avg=8192.0000 max=14339 max_rank=5 min=4714 min_rank=4 imbalance_ratio=1.750366 max_load_reduction=51197 max_load_reduction_pct=0.781204 moved_to_echo=30755 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 20, 219, 0, 20, 2161, 0, 3617, 355, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 6, 1006, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 106, 0, 2687, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 153, 2, 3981, 0, 0, 3375, 2626, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 53, 12, 0, 4093, 0, 120, 173, 0, 0, 0, 0, 0, 0, 0, 913, 6, 1541, 182, 468, 0, 0, 33, 2046, 142, 32, 0, 0, 0, 0, 0, 3805, 0, 0, 1, 4096, 127, 102, 1953, 0, 0, 0, 0, 0, 0, 0, 0, 91, 482, 71, 4080, 258, 3141, 3992, 1, 1093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6419, 5286, 6514, 10182, 8479, 5363, 10084, 13209] avg=8192.0000 max=13209 max_rank=7 min=5286 min_rank=1 imbalance_ratio=1.612427 max_load_reduction=52327 max_load_reduction_pct=0.798447 moved_to_echo=37135 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1443, 0, 0, 0, 1654, 31, 3380, 327, 0, 0, 0, 0, 0, 0, 4001, 3286, 0, 0, 926, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2884, 0, 11, 2680, 63, 1135, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 113, 3183, 331, 0, 0, 666, 2419, 0, 0, 0, 0, 0, 0, 0, 759, 3915, 0, 23, 0, 0, 4075, 0, 173, 0, 0, 0, 0, 0, 0, 0, 1, 27, 1, 134, 4084, 3822, 0, 0, 2092, 142, 30, 0, 0, 0, 0, 0, 895, 4096, 7, 7, 1, 2, 0, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 1, 4093, 0, 506, 4009, 0, 1091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6835, 8219, 6888, 7120, 8945, 10333, 7006, 10190] avg=8192.0000 max=10333 max_rank=5 min=6835 min_rank=0 imbalance_ratio=1.261353 max_load_reduction=55203 max_load_reduction_pct=0.842331 moved_to_echo=36474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 863, 1, 3127, 0, 25, 58, 4085, 284, 0, 0, 0, 0, 0, 0, 3942, 3904, 0, 1229, 805, 32, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 372, 163, 2598, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 9, 0, 3681, 0, 415, 2, 2102, 0, 0, 0, 0, 0, 0, 0, 3269, 4033, 4001, 0, 1, 1, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 3368, 3643, 0, 3431, 0, 124, 0, 33, 143, 26, 0, 0, 0, 0, 0, 1698, 0, 0, 24, 445, 6, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 3, 1170, 4082, 1736, 1066, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8444, 9916, 8266, 6285, 11477, 10768, 2205, 8175] avg=8192.0000 max=11477 max_rank=4 min=2205 min_rank=6 imbalance_ratio=1.401001 max_load_reduction=54059 max_load_reduction_pct=0.824875 moved_to_echo=32625 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3237, 0, 4, 22, 0, 0, 3974, 33, 0, 0, 0, 0, 0, 0, 1301, 3308, 0, 0, 96, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 137, 108, 2686, 3885, 1750, 709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 158, 4066, 0, 487, 7, 247, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 3, 0, 0, 0, 3903, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 805, 3585, 2, 27, 16, 52, 0, 2095, 142, 3, 0, 0, 0, 0, 0, 2056, 0, 0, 551, 4093, 187, 2272, 2000, 0, 0, 0, 0, 0, 0, 0, 0, 13, 52, 0, 35, 1544, 175, 4085, 12, 1093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7270, 8730, 13371, 4966, 6304, 6727, 11159, 7009] avg=8192.0000 max=13371 max_rank=2 min=4966 min_rank=3 imbalance_ratio=1.632202 max_load_reduction=52165 max_load_reduction_pct=0.795975 moved_to_echo=31199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 175, 3393, 0, 0, 13, 1392, 0, 3928, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 1018, 2724, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2439, 102, 3115, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 727, 8, 0, 4010, 4007, 476, 0, 2658, 0, 0, 0, 0, 0, 0, 0, 3320, 379, 4001, 0, 138, 0, 1349, 18, 157, 0, 0, 0, 0, 0, 0, 0, 23, 0, 783, 0, 4093, 1, 2, 0, 2094, 129, 33, 0, 0, 0, 0, 0, 3695, 4094, 42, 0, 7, 0, 4067, 1999, 0, 0, 0, 0, 0, 0, 0, 0, 1471, 0, 88, 360, 0, 0, 28, 101, 993, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9260, 5197, 5676, 11938, 9362, 7158, 13904, 3041] avg=8192.0000 max=13904 max_rank=6 min=3041 min_rank=7 imbalance_ratio=1.697266 max_load_reduction=51632 max_load_reduction_pct=0.787842 moved_to_echo=33465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3308, 0, 2, 0, 2, 2282, 4089, 362, 0, 0, 0, 0, 0, 0, 3, 850, 0, 3539, 1005, 823, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3615, 10, 0, 373, 0, 3230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 0, 69, 3554, 12, 54, 2657, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 45, 0, 0, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 80, 1, 0, 2094, 0, 36, 0, 0, 0, 0, 0, 4004, 3881, 0, 23, 118, 12, 0, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4024, 0, 0, 696, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10045, 6255, 11324, 7224, 4113, 3788, 10032, 12755] avg=8192.0000 max=12755 max_rank=7 min=3788 min_rank=5 imbalance_ratio=1.557007 max_load_reduction=52781 max_load_reduction_pct=0.805374 moved_to_echo=30688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4049, 0, 3, 303, 0, 8, 4096, 360, 0, 0, 0, 0, 0, 0, 2975, 2365, 0, 3356, 1021, 2399, 1152, 26, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 2688, 3, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 91, 4094, 0, 60, 0, 2666, 0, 0, 0, 0, 0, 0, 0, 0, 3827, 4041, 4, 2, 0, 4093, 2, 173, 0, 0, 0, 0, 0, 0, 0, 0, 2759, 0, 0, 3909, 214, 0, 0, 3, 142, 33, 0, 0, 0, 0, 0, 1686, 302, 0, 1028, 3, 0, 1423, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 1, 0, 1005, 4, 1093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8819, 13294, 6642, 6978, 12142, 7060, 4445, 6156] avg=8192.0000 max=13294 max_rank=1 min=4445 min_rank=6 imbalance_ratio=1.622803 max_load_reduction=52242 max_load_reduction_pct=0.797150 moved_to_echo=29803 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[22016, 12416, 128, 1792, 8832, 512, 8704, 11648, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 8832, 15232, 384, 20224, 4224, 256, 2560, 12160, 1024, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 1280, 1280, 3456, 24448, 7680, 5248, 4480, 18176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8832, 4736, 256, 13952, 2560, 8960, 17280, 1920, 7552, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[22912, 4352, 384, 18432, 5632, 10240, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 256, 20352, 256, 3200, 768, 5248, 2432, 30976, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12288, 17920, 0, 13312, 6912, 12032, 3072, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11136, 18816, 12160, 128, 256, 4096, 17536, 640, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[2360, 1, 4088, 659, 3263, 0, 4042, 3138, 258, 89, 1491, 0, 4038, 3917, 2288, 16, 46, 3132, 1, 261, 0, 0, 2658, 213, 698, 32, 1, 17, 0, 1, 2, 3971, 3, 17, 62, 4085, 10, 0, 0, 62, 1, 0, 8, 163, 0, 0, 3044, 0, 134, 82, 4080, 0, 1480, 3409, 0, 11, 40, 0, 4055, 0, 4094, 14, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[111, 3906, 47, 7, 738, 0, 6, 0, 0, 208, 3061, 0, 3515, 155, 0, 1594, 0, 18, 1, 4096, 3161, 89, 0, 4028, 0, 3366, 171, 0, 1468, 0, 20, 0, 0, 1, 3972, 3719, 0, 4093, 0, 0, 0, 18, 3940, 981, 0, 0, 57, 1296, 0, 0, 3793, 1, 3952, 1401, 10, 46, 95, 3, 4095, 5, 4096, 0, 196, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[3889, 0, 0, 2228, 2378, 279, 0, 4026, 3623, 1, 238, 80, 524, 3952, 0, 0, 0, 0, 0, 3950, 3889, 0, 2665, 4095, 1873, 0, 12, 0, 0, 32, 7, 4061, 8, 4042, 3458, 1, 154, 0, 0, 46, 0, 1, 43, 0, 0, 15, 0, 0, 0, 3434, 58, 21, 4091, 165, 0, 0, 0, 0, 4089, 14, 4093, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3994, 0, 733, 0, 0, 3877, 68, 1136, 88, 0, 0, 1254, 2091, 3374, 0, 125, 2, 34, 460, 4038, 3452, 0, 3, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3510, 4003, 14, 2303, 98, 0, 69, 0, 0, 3961, 138, 1, 0, 0, 2057, 1, 3971, 2, 0, 4075, 4096, 0, 0, 4, 0, 4083, 220, 4092, 3, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4087, 0, 3, 4070, 0, 1235, 8, 4086, 206, 3381, 0, 0, 0, 0, 0, 3977, 4026, 3, 2478, 2571, 37, 0, 70, 0, 38, 369, 0, 0, 0, 0, 3766, 47, 0, 163, 0, 4089, 945, 0, 4, 716, 5, 0, 0, 3889, 15, 3559, 7, 0, 4039, 2729, 4, 11, 2, 1023, 3988, 5, 1637, 2, 4065, 176] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 2251, 1430, 0, 1707, 4046, 3630, 545, 0, 1349, 0, 3819, 0, 3920, 0, 3086, 28, 3933, 3948, 0, 879, 3649, 0, 0, 0, 0, 0, 800, 0, 4096, 0, 465, 4075, 1, 45, 3676, 0, 302, 0, 25, 2, 484, 0, 284, 0, 872, 0, 0, 2, 0, 4070, 15, 0, 95, 5, 14, 4094, 14, 3880, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4078, 291, 290, 0, 3894, 1102, 1124, 150, 4096, 252, 28, 932, 0, 2417, 0, 3, 4, 4, 0, 4096, 4092, 0, 72, 3830, 0, 20, 0, 1, 0, 0, 1, 643, 0, 0, 3847, 0, 3969, 1631, 0, 1443, 0, 1, 2780, 708, 171, 44, 0, 0, 0, 3371, 5, 4, 4035, 1, 4, 136, 0, 0, 4096, 1, 4096, 0, 3773, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1328, 1293, 3956, 423, 4038, 0, 4096, 4090, 0, 0, 0, 4080, 0, 21, 0, 3, 7, 549, 0, 4078, 0, 186, 2100, 1398, 21, 0, 0, 604, 0, 141, 0, 4096, 0, 55, 318, 4096, 9, 4075, 0, 4096, 8, 7, 0, 3928, 0, 0, 9, 476, 0, 0, 6, 0, 849, 3508, 0, 5, 396, 3083, 4095, 1, 8, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[111, 3906, 47, 7, 495, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 3061, 0, 3515, 123, 0, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 84, 3161, 89, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3366, 171, 0, 1468, 0, 20, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2559, 3719, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3940, 981, 0, 0, 57, 1296, 0, 1413, 243, 781, 27, 0, 0, 0, 0, 0, 3793, 1, 3952, 1401, 10, 46, 5, 0, 0, 0, 0, 0, 0, 0, 95, 3, 3314, 5, 4096, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4572, 8501, 7381, 9037, 10372, 8756, 9208, 7709] avg=8192.0000 max=10372 max_rank=4 min=4572 min_rank=0 imbalance_ratio=1.266113 max_load_reduction=55164 max_load_reduction_pct=0.841736 moved_to_echo=36045 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2360, 1, 4088, 659, 2188, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 89, 1491, 0, 4038, 3088, 2288, 16, 0, 0, 0, 0, 0, 0, 0, 0, 46, 3132, 1, 6, 0, 0, 2658, 213, 0, 0, 0, 0, 0, 0, 0, 0, 698, 32, 1, 17, 0, 1, 2, 3971, 255, 1674, 0, 0, 0, 0, 0, 0, 3, 17, 40, 4085, 10, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 163, 0, 0, 3044, 0, 1464, 22, 1075, 773, 689, 0, 0, 0, 134, 82, 4080, 0, 1480, 3409, 0, 11, 140, 0, 0, 0, 0, 0, 0, 0, 40, 0, 3282, 0, 4094, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13338, 11268, 6056, 6651, 4217, 7239, 9336, 7431] avg=8192.0000 max=13338 max_rank=0 min=4217 min_rank=4 imbalance_ratio=1.628174 max_load_reduction=52198 max_load_reduction_pct=0.796478 moved_to_echo=28223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3994, 0, 733, 0, 0, 3877, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 1254, 2091, 2661, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 2, 34, 460, 83, 3452, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3955, 606, 0, 0, 0, 0, 0, 0, 0, 3510, 2579, 14, 2303, 98, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 138, 1, 0, 0, 2057, 530, 1424, 0, 779, 593, 0, 0, 0, 1, 3971, 2, 0, 4075, 4096, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3304, 220, 4092, 3, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8672, 6219, 8130, 4562, 8573, 9483, 12265, 7632] avg=8192.0000 max=12265 max_rank=6 min=4562 min_rank=3 imbalance_ratio=1.497192 max_load_reduction=53271 max_load_reduction_pct=0.812851 moved_to_echo=37953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3889, 0, 0, 2228, 1595, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3623, 1, 238, 80, 524, 3116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 3889, 0, 2665, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1873, 0, 12, 0, 0, 32, 7, 4061, 3868, 2147, 0, 0, 0, 0, 0, 0, 8, 4042, 2228, 1, 154, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 43, 0, 0, 15, 0, 0, 1879, 1230, 783, 780, 695, 0, 0, 0, 0, 3434, 58, 21, 4091, 165, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3309, 14, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7991, 7582, 10731, 12000, 6479, 5426, 7910, 7417] avg=8192.0000 max=12000 max_rank=3 min=5426 min_rank=5 imbalance_ratio=1.464844 max_load_reduction=53536 max_load_reduction_pct=0.816895 moved_to_echo=27232 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 2741, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1235, 8, 4086, 206, 2665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 4026, 3, 2478, 2571, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 70, 0, 38, 369, 0, 0, 3895, 2171, 0, 0, 0, 0, 0, 0, 0, 0, 2427, 47, 0, 163, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 945, 0, 4, 716, 5, 0, 0, 3889, 1899, 1339, 1346, 761, 595, 0, 0, 0, 15, 3559, 7, 0, 4039, 2729, 4, 11, 121, 0, 0, 0, 0, 0, 0, 0, 2, 1023, 3227, 5, 1637, 2, 4065, 176, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2749, 8200, 9160, 6580, 6726, 11499, 10485, 10137] avg=8192.0000 max=11499 max_rank=5 min=2749 min_rank=0 imbalance_ratio=1.403687 max_load_reduction=54037 max_load_reduction_pct=0.824539 moved_to_echo=38847 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2251, 959, 0, 1707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3630, 545, 0, 1349, 0, 3011, 0, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3086, 28, 81, 3948, 0, 879, 3649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 800, 0, 4096, 3852, 2159, 0, 0, 0, 0, 0, 0, 0, 465, 2626, 1, 45, 3676, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2, 484, 0, 284, 0, 872, 1887, 1449, 471, 781, 672, 0, 0, 0, 0, 0, 2, 0, 4070, 15, 0, 95, 136, 0, 0, 0, 0, 0, 0, 0, 5, 14, 3313, 14, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4917, 12455, 11671, 10907, 7115, 6927, 4318, 7226] avg=8192.0000 max=12455 max_rank=1 min=4318 min_rank=6 imbalance_ratio=1.520386 max_load_reduction=53081 max_load_reduction_pct=0.809952 moved_to_echo=25586 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1328, 1293, 3956, 423, 2705, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 11, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7, 549, 0, 81, 0, 186, 2100, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 604, 0, 141, 0, 4096, 3997, 2183, 0, 0, 0, 0, 0, 0, 0, 55, 201, 4096, 9, 4075, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7, 0, 3928, 0, 0, 9, 476, 1907, 117, 1333, 784, 6, 0, 0, 0, 0, 0, 6, 0, 849, 3508, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 396, 3083, 3311, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13801, 4094, 4321, 11042, 12532, 8575, 4372, 6799] avg=8192.0000 max=13801 max_rank=0 min=4094 min_rank=1 imbalance_ratio=1.684692 max_load_reduction=51735 max_load_reduction_pct=0.789413 moved_to_echo=32278 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4078, 291, 290, 0, 2611, 1102, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 252, 28, 932, 0, 1906, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 84, 4092, 0, 72, 3830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 1, 0, 0, 1, 643, 4012, 80, 0, 0, 0, 0, 0, 0, 0, 0, 2479, 0, 3969, 1631, 0, 1443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2780, 708, 171, 44, 0, 0, 70, 1368, 1283, 781, 425, 0, 0, 0, 0, 3371, 5, 4, 4035, 1, 4, 136, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3315, 1, 4096, 0, 3773, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9496, 7217, 8086, 4757, 9522, 7631, 7642, 11185] avg=8192.0000 max=11185 max_rank=7 min=4757 min_rank=3 imbalance_ratio=1.365356 max_load_reduction=54351 max_load_reduction_pct=0.829330 moved_to_echo=35980 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[15872, 5504, 9216, 5632, 13312, 5376, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 8192, 15232, 12032, 6528, 13824, 0, 10112, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 4224, 26496, 384, 26112, 128, 8064, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[1024, 128, 10752, 7168, 256, 384, 3200, 8704, 9728, 8448, 6656, 6272, 3712, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 14464, 8064, 128, 26624, 15360, 128, 384, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 6912, 512, 640, 22656, 384, 10880, 23936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2688, 3456, 256, 640, 1536, 1408, 128, 16896, 27904, 11136, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[11776, 2432, 4864, 11904, 10496, 16640, 2304, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[2705, 0, 3842, 21, 126, 0, 273, 0, 0, 0, 0, 4055, 0, 743, 0, 6, 0, 0, 0, 3518, 0, 0, 0, 4095, 0, 3766, 244, 1, 0, 6, 3425, 0, 0, 0, 0, 1199, 102, 0, 4096, 0, 0, 3176, 4082, 128, 4086, 0, 4096, 590, 0, 4096, 3, 3930, 4070, 0, 43, 3911, 0, 2, 27, 1, 1, 0, 0, 1071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2574, 1371, 0, 695, 0, 0, 12, 460, 0, 2708, 0, 0, 484, 0, 0, 21, 0, 0, 65, 82, 0, 3722, 0, 4090, 4023, 2066, 653, 200, 0, 2585, 8, 528, 2902, 0, 1376, 0, 3980, 4029, 0, 4096, 613, 9, 1063, 0, 0, 2363, 90, 185, 84, 0, 3970, 30, 21, 3502, 0, 1, 8, 0, 2101, 624, 4046, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1231, 0, 4082, 0, 0, 1175, 39, 0, 0, 0, 2086, 447, 0, 2630, 70, 116, 96, 111, 36, 3700, 4096, 49, 3, 4096, 9, 3638, 25, 0, 1417, 234, 10, 0, 0, 0, 3995, 0, 15, 0, 4003, 4, 196, 4077, 4096, 716, 0, 206, 582, 0, 0, 4086, 0, 1276, 3808, 29, 0, 3923, 0, 39, 0, 21, 3916, 4, 1148, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3513, 0, 4073, 122, 2, 38, 4093, 0, 1046, 0, 0, 0, 0, 3552, 0, 0, 244, 0, 73, 3915, 86, 0, 0, 4096, 0, 3141, 1506, 3290, 492, 16, 3981, 2, 0, 0, 951, 897, 10, 0, 4094, 13, 7, 2520, 4046, 31, 25, 0, 4091, 3800, 0, 344, 150, 558, 0, 0, 0, 39, 0, 0, 0, 30, 4069, 0, 0, 2580] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4, 1050, 4016, 0, 60, 0, 4054, 3, 0, 0, 0, 3237, 18, 0, 0, 1, 0, 0, 55, 4095, 1, 905, 44, 3936, 0, 0, 3, 3232, 3920, 0, 2693, 39, 0, 1012, 0, 6, 0, 0, 2523, 0, 1261, 520, 4010, 0, 204, 490, 4096, 3787, 61, 0, 0, 3566, 3986, 0, 0, 3006, 91, 1225, 227, 5, 1, 0, 4093, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[725, 0, 4054, 14, 5, 0, 4083, 0, 334, 125, 0, 2242, 0, 3780, 795, 3, 0, 107, 0, 4081, 41, 0, 1, 3515, 0, 3427, 0, 2, 0, 0, 4096, 4, 0, 193, 722, 77, 0, 41, 3983, 0, 95, 4053, 4030, 0, 19, 0, 3784, 3070, 19, 1247, 3, 3983, 4088, 0, 0, 4050, 398, 1, 0, 0, 0, 0, 186, 60] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2267, 1374, 1979, 0, 796, 0, 2753, 0, 4001, 313, 0, 4092, 0, 428, 0, 40, 147, 0, 10, 0, 3619, 0, 3976, 2793, 1552, 3994, 0, 552, 3, 0, 0, 219, 0, 2149, 2328, 16, 0, 0, 3739, 1711, 184, 4095, 0, 1, 0, 0, 0, 3808, 110, 0, 5, 3, 3490, 76, 130, 2018, 0, 0, 0, 0, 3028, 24, 0, 3713] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1883, 665, 353, 0, 1, 255, 1, 0, 0, 3923, 3280, 15, 0, 1264, 0, 0, 759, 0, 15, 0, 80, 0, 4039, 3959, 1, 1756, 4055, 0, 287, 3306, 2209, 7, 0, 4088, 1696, 720, 0, 370, 3877, 1, 0, 4096, 31, 0, 0, 0, 4095, 0, 4, 0, 53, 233, 3938, 3075, 43, 581, 0, 0, 0, 0, 3549, 199, 0, 2774] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2705, 0, 3842, 21, 126, 0, 273, 0, 671, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 743, 0, 6, 1544, 531, 694, 0, 0, 0, 0, 0, 0, 0, 0, 3518, 0, 0, 0, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2401, 244, 1, 0, 6, 3425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 102, 0, 4096, 0, 440, 247, 0, 0, 0, 0, 0, 0, 0, 0, 3642, 128, 4086, 0, 4096, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 3930, 2526, 0, 43, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 27, 1, 1, 0, 0, 1071, 2645, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7762, 7573, 7242, 6077, 6084, 12542, 14509, 3747] avg=8192.0000 max=14509 max_rank=6 min=3747 min_rank=7 imbalance_ratio=1.771118 max_load_reduction=51027 max_load_reduction_pct=0.778610 moved_to_echo=36882 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2574, 1371, 0, 695, 0, 0, 729, 113, 0, 0, 0, 0, 0, 0, 12, 460, 0, 2708, 0, 0, 484, 0, 1506, 685, 754, 0, 0, 0, 0, 0, 0, 21, 0, 0, 65, 82, 0, 3384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 4023, 2066, 653, 200, 0, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 8, 528, 2902, 0, 1376, 0, 3980, 4029, 66, 225, 0, 0, 0, 0, 0, 0, 0, 0, 547, 9, 1063, 0, 0, 2363, 0, 0, 0, 0, 0, 0, 0, 0, 90, 185, 84, 0, 2464, 30, 21, 3502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 2101, 624, 4046, 4096, 3411, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5482, 6609, 3552, 12134, 13114, 3982, 6376, 14287] avg=8192.0000 max=14287 max_rank=7 min=3552 min_rank=2 imbalance_ratio=1.744019 max_load_reduction=51249 max_load_reduction_pct=0.781998 moved_to_echo=37759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1231, 0, 4082, 0, 0, 1175, 39, 0, 648, 124, 0, 0, 0, 0, 0, 0, 0, 0, 2086, 447, 0, 2630, 70, 116, 1445, 682, 671, 0, 0, 0, 0, 0, 96, 111, 36, 3700, 4096, 49, 3, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2319, 25, 0, 1417, 234, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 15, 0, 4003, 4, 441, 247, 0, 0, 0, 0, 0, 0, 196, 0, 3655, 716, 0, 206, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 1276, 2363, 29, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 21, 3916, 4, 1148, 0, 3395, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7299, 8147, 11816, 4014, 8705, 5355, 11677, 8523] avg=8192.0000 max=11816 max_rank=2 min=4014 min_rank=3 imbalance_ratio=1.442383 max_load_reduction=53720 max_load_reduction_pct=0.819702 moved_to_echo=34260 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3513, 0, 4073, 122, 2, 38, 4093, 0, 560, 124, 0, 0, 0, 0, 0, 0, 1046, 0, 0, 0, 0, 3552, 0, 0, 0, 421, 579, 0, 0, 0, 0, 0, 244, 0, 73, 3915, 86, 0, 0, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2002, 1506, 3290, 492, 16, 3981, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 897, 10, 0, 4094, 13, 436, 247, 0, 0, 0, 0, 0, 0, 7, 0, 3610, 31, 25, 0, 4091, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 150, 558, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4069, 0, 0, 2580, 2099, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12525, 5598, 8043, 11289, 6648, 11564, 1091, 8778] avg=8192.0000 max=12525 max_rank=0 min=1091 min_rank=6 imbalance_ratio=1.528931 max_load_reduction=53011 max_load_reduction_pct=0.808884 moved_to_echo=28081 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 1050, 4016, 0, 60, 0, 4054, 3, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 18, 0, 0, 1, 1512, 87, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4095, 1, 905, 44, 3579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3232, 3920, 0, 2693, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 0, 6, 0, 0, 2523, 0, 432, 238, 0, 0, 0, 0, 0, 0, 1261, 0, 3578, 0, 204, 490, 4096, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 3566, 2474, 0, 0, 3006, 0, 0, 0, 0, 0, 0, 0, 0, 91, 1225, 227, 5, 1, 0, 4093, 0, 433, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9306, 4855, 8679, 9887, 4211, 13416, 9107, 6075] avg=8192.0000 max=13416 max_rank=5 min=4211 min_rank=4 imbalance_ratio=1.637695 max_load_reduction=52120 max_load_reduction_pct=0.795288 moved_to_echo=32809 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2267, 1374, 1979, 0, 796, 0, 2753, 0, 712, 84, 0, 0, 0, 0, 0, 0, 4001, 313, 0, 4092, 0, 428, 0, 40, 1324, 686, 737, 0, 0, 0, 0, 0, 147, 0, 10, 0, 3619, 0, 3976, 2541, 0, 0, 0, 0, 0, 0, 0, 0, 1552, 2545, 0, 552, 3, 0, 0, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2149, 2328, 16, 0, 0, 3739, 1711, 0, 168, 0, 0, 0, 0, 0, 0, 184, 0, 0, 1, 0, 0, 0, 3808, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 5, 3, 2166, 76, 130, 2018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3028, 24, 0, 3713, 3409, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9965, 11621, 10293, 4871, 10111, 3993, 4508, 10174] avg=8192.0000 max=11621 max_rank=1 min=3993 min_rank=5 imbalance_ratio=1.418579 max_load_reduction=53915 max_load_reduction_pct=0.822678 moved_to_echo=28786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[725, 0, 4054, 14, 5, 0, 4083, 0, 611, 106, 0, 0, 0, 0, 0, 0, 334, 125, 0, 2242, 0, 3780, 795, 3, 1551, 678, 632, 0, 0, 0, 0, 0, 0, 107, 0, 4081, 41, 0, 1, 3197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2184, 0, 2, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 722, 77, 0, 41, 3983, 0, 434, 212, 0, 0, 0, 0, 0, 0, 95, 0, 3596, 0, 19, 0, 3784, 3070, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1247, 3, 3983, 2537, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 398, 1, 0, 0, 0, 0, 186, 60, 3375, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9598, 10140, 7427, 6286, 5662, 10564, 11839, 4020] avg=8192.0000 max=11839 max_rank=6 min=4020 min_rank=7 imbalance_ratio=1.445190 max_load_reduction=53697 max_load_reduction_pct=0.819351 moved_to_echo=32085 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1883, 665, 353, 0, 1, 255, 1, 0, 316, 125, 0, 0, 0, 0, 0, 0, 0, 3923, 3280, 15, 0, 1264, 0, 0, 1498, 686, 327, 0, 0, 0, 0, 0, 759, 0, 15, 0, 80, 0, 4039, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1113, 4055, 0, 287, 3306, 2209, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1696, 720, 0, 370, 3877, 1, 6, 243, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 53, 233, 2440, 3075, 43, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3549, 199, 0, 2774, 3410, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3599, 10993, 8484, 10978, 11001, 4120, 6429, 9932] avg=8192.0000 max=11001 max_rank=4 min=3599 min_rank=0 imbalance_ratio=1.342896 max_load_reduction=54535 max_load_reduction_pct=0.832138 moved_to_echo=31482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 9984, 384, 13568, 17024, 3328, 256, 21120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[12416, 3200, 25088, 1536, 1024, 2176, 15360, 128, 4352, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[5504, 4864, 5376, 16896, 128, 12416, 1408, 256, 10496, 4480, 4480, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 1280, 384, 128, 16768, 896, 9600, 14336, 22272, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1280, 256, 256, 19328, 8064, 1152, 8064, 27520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1664, 15232, 9856, 9216, 6784, 3840, 16512, 2944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1792, 0, 18688, 896, 5504, 768, 20864, 17536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 8064, 12672, 2944, 1536, 512, 30336, 5760, 2304, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[7, 0, 59, 1024, 0, 0, 352, 0, 3848, 0, 4086, 3079, 0, 0, 0, 7, 0, 3940, 0, 0, 2102, 10, 0, 0, 0, 4030, 0, 8, 0, 59, 0, 3882, 150, 0, 2, 4036, 4065, 0, 4086, 4096, 32, 12, 1, 55, 2636, 0, 990, 1353, 3670, 4055, 0, 2, 0, 34, 4086, 2, 0, 4096, 0, 146, 1437, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[507, 0, 0, 0, 106, 802, 314, 2, 3706, 6, 0, 0, 0, 69, 0, 3333, 3980, 0, 0, 3301, 502, 0, 0, 0, 0, 1714, 124, 3, 2375, 0, 4068, 0, 0, 0, 4088, 4096, 4095, 4, 739, 0, 0, 83, 22, 0, 0, 3964, 0, 2411, 3159, 0, 30, 0, 4061, 73, 4049, 0, 1380, 20, 194, 4094, 4025, 0, 0, 37] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4086, 19, 506, 0, 0, 0, 0, 0, 1668, 3678, 4073, 0, 879, 45, 4096, 3836, 0, 15, 0, 0, 0, 20, 0, 0, 3961, 47, 1109, 2709, 0, 2848, 5, 0, 0, 0, 4096, 3415, 0, 1952, 0, 0, 0, 0, 0, 0, 3866, 404, 0, 44, 2193, 0, 0, 0, 0, 4096, 5, 265, 4092, 2874, 4055, 407, 62, 2, 108] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[165, 0, 3, 0, 29, 0, 9, 0, 4046, 0, 0, 3569, 38, 0, 1, 4093, 3, 1612, 0, 4029, 524, 0, 0, 0, 3, 0, 0, 16, 0, 0, 3712, 1673, 0, 0, 4, 4074, 4096, 56, 2509, 0, 28, 3872, 0, 1, 0, 4090, 3, 0, 4063, 198, 4086, 2, 0, 25, 4096, 0, 21, 3883, 370, 0, 3926, 7, 22, 2579] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 16, 14, 3208, 0, 1415, 12, 4055, 2, 0, 0, 4002, 32, 0, 0, 221, 0, 0, 0, 0, 0, 4075, 0, 0, 508, 0, 2548, 2, 95, 29, 0, 1262, 3949, 0, 4091, 4072, 383, 4096, 9, 0, 0, 0, 0, 4068, 4094, 1946, 2663, 0, 25, 0, 3132, 0, 39, 4096, 4089, 2886, 398, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4069, 0, 3, 31, 0, 0, 0, 0, 165, 4003, 4096, 0, 67, 0, 4078, 1698, 0, 4, 0, 5, 0, 0, 17, 0, 4077, 448, 74, 2436, 0, 3956, 1820, 22, 7, 0, 3957, 4096, 0, 4065, 1839, 2, 0, 0, 2, 3906, 46, 0, 0, 4085, 966, 0, 0, 10, 57, 4041, 0, 4, 4096, 0, 86, 3200, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[10, 4096, 0, 177, 0, 212, 0, 0, 4085, 155, 11, 2168, 4, 0, 0, 83, 1048, 64, 0, 1057, 0, 0, 0, 0, 477, 2634, 0, 1, 0, 0, 4012, 4090, 22, 0, 0, 4083, 3913, 1181, 4095, 0, 9, 4096, 9, 0, 0, 17, 0, 0, 4093, 8, 3686, 0, 1, 0, 4070, 0, 18, 4096, 0, 3659, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 0, 0, 9, 96, 0, 4002, 0, 6, 10, 666, 48, 3801, 0, 2023, 0, 0, 3967, 0, 0, 0, 0, 37, 3790, 4, 419, 2, 1293, 0, 1462, 288, 1, 108, 3510, 0, 4080, 0, 0, 3, 17, 3184, 8, 2818, 16, 4096, 3934, 1466, 0, 10, 73, 0, 4095, 0, 1652, 4075, 1074, 3698, 2087, 9, 52, 3523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 59, 1024, 0, 0, 352, 0, 4065, 2078, 0, 0, 0, 0, 0, 0, 3848, 0, 4086, 3079, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 2102, 10, 0, 0, 2445, 2, 728, 0, 0, 0, 0, 0, 0, 4030, 0, 8, 0, 59, 0, 3882, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 2, 3532, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 32, 12, 1, 55, 2636, 0, 990, 1353, 2, 0, 504, 0, 0, 0, 0, 0, 3670, 4055, 0, 2, 0, 34, 1641, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1290, 0, 146, 1437, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7585, 11016, 9227, 7979, 11866, 5585, 9404, 2874] avg=8192.0000 max=11866 max_rank=4 min=2874 min_rank=7 imbalance_ratio=1.448486 max_load_reduction=53670 max_load_reduction_pct=0.818939 moved_to_echo=29729 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[507, 0, 0, 0, 106, 802, 314, 2, 4095, 10, 0, 0, 0, 0, 0, 0, 3706, 6, 0, 0, 0, 69, 0, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 3301, 502, 0, 0, 0, 2422, 1066, 3, 0, 0, 0, 0, 0, 0, 1714, 124, 3, 2375, 0, 3432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3585, 0, 4, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 22, 0, 0, 3964, 0, 2411, 1163, 636, 511, 0, 0, 0, 0, 0, 3159, 0, 30, 0, 4061, 73, 1627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1380, 7, 194, 4094, 4025, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5836, 4885, 11274, 7648, 8416, 8790, 8950, 9737] avg=8192.0000 max=11274 max_rank=2 min=4885 min_rank=1 imbalance_ratio=1.376221 max_load_reduction=54262 max_load_reduction_pct=0.827972 moved_to_echo=35893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 16, 14, 3208, 3949, 2078, 0, 0, 0, 0, 0, 0, 0, 1415, 12, 4055, 2, 0, 0, 1326, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 221, 0, 0, 0, 0, 1874, 1279, 728, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 508, 0, 2150, 2, 0, 0, 0, 0, 0, 0, 0, 0, 95, 29, 0, 1105, 0, 0, 4091, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 383, 4096, 9, 0, 0, 0, 0, 4068, 1397, 398, 157, 0, 0, 0, 0, 0, 4094, 1946, 2663, 0, 25, 0, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1290, 4089, 2886, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9269, 6810, 4134, 6735, 9392, 10508, 9986, 8702] avg=8192.0000 max=10508 max_rank=5 min=4134 min_rank=2 imbalance_ratio=1.282715 max_load_reduction=55028 max_load_reduction_pct=0.839661 moved_to_echo=38588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 19, 506, 0, 0, 0, 0, 3415, 2076, 0, 0, 0, 0, 0, 0, 0, 1668, 3678, 4073, 0, 879, 45, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 15, 0, 0, 0, 20, 0, 2451, 1310, 728, 0, 0, 0, 0, 0, 0, 3961, 47, 1109, 2709, 0, 2403, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3585, 0, 0, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 404, 0, 1430, 445, 511, 0, 0, 0, 0, 0, 44, 2193, 0, 0, 0, 0, 1645, 5, 0, 0, 0, 0, 0, 0, 0, 0, 265, 1288, 2874, 4055, 407, 62, 2, 108, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10102, 11699, 8360, 10234, 5537, 6656, 3887, 9061] avg=8192.0000 max=11699 max_rank=1 min=3887 min_rank=6 imbalance_ratio=1.428101 max_load_reduction=53837 max_load_reduction_pct=0.821487 moved_to_echo=25141 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[165, 0, 3, 0, 29, 0, 9, 0, 4096, 1970, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 3569, 38, 0, 1, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1612, 0, 4029, 524, 0, 0, 0, 2451, 1309, 691, 0, 0, 0, 0, 0, 3, 0, 0, 16, 0, 0, 3131, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3566, 0, 56, 2509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3872, 0, 1, 0, 4090, 3, 0, 1429, 581, 508, 0, 0, 0, 0, 0, 4063, 198, 4086, 2, 0, 25, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1222, 370, 0, 3926, 7, 22, 2579, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6272, 9009, 10619, 4823, 6135, 10512, 10019, 8147] avg=8192.0000 max=10619 max_rank=2 min=4823 min_rank=3 imbalance_ratio=1.296265 max_load_reduction=54917 max_load_reduction_pct=0.837967 moved_to_echo=34813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 0, 0, 9, 96, 3510, 2067, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 6, 10, 666, 48, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2023, 0, 0, 3967, 0, 0, 0, 2450, 1215, 725, 0, 0, 0, 0, 0, 0, 37, 3790, 4, 419, 2, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1462, 288, 1, 95, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 17, 3184, 8, 2818, 16, 4096, 1327, 202, 13, 0, 0, 0, 0, 0, 3934, 1466, 0, 10, 73, 0, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1652, 1283, 1074, 3698, 2087, 9, 52, 3523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5706, 5991, 10380, 5343, 5926, 11684, 7128, 13378] avg=8192.0000 max=13378 max_rank=7 min=5343 min_rank=3 imbalance_ratio=1.633057 max_load_reduction=52158 max_load_reduction_pct=0.795868 moved_to_echo=38116 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4069, 0, 3, 31, 0, 0, 0, 4096, 2078, 0, 0, 0, 0, 0, 0, 0, 165, 4003, 4096, 0, 67, 0, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 1698, 0, 4, 0, 5, 0, 0, 17, 2418, 1304, 728, 0, 0, 0, 0, 0, 0, 4077, 448, 74, 2436, 0, 3337, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 22, 7, 0, 3463, 0, 0, 4065, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 3906, 46, 0, 0, 1424, 619, 494, 0, 0, 0, 0, 0, 4085, 966, 0, 0, 10, 57, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1290, 0, 86, 3200, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10277, 9681, 6174, 12192, 9396, 6493, 6741, 4582] avg=8192.0000 max=12192 max_rank=3 min=4582 min_rank=7 imbalance_ratio=1.488281 max_load_reduction=53344 max_load_reduction_pct=0.813965 moved_to_echo=27212 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[10, 4096, 0, 177, 0, 212, 0, 0, 3913, 2081, 0, 0, 0, 0, 0, 0, 4085, 155, 11, 2168, 4, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 64, 0, 1057, 0, 0, 0, 0, 2437, 29, 733, 0, 0, 0, 0, 0, 477, 2634, 0, 1, 0, 0, 3380, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 3570, 0, 1181, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 9, 0, 0, 17, 0, 0, 32, 632, 513, 0, 0, 0, 0, 0, 4093, 8, 3686, 0, 1, 0, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1282, 0, 3659, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10489, 6445, 5368, 10582, 8868, 5308, 9421, 9055] avg=8192.0000 max=10582 max_rank=3 min=5308 min_rank=5 imbalance_ratio=1.291748 max_load_reduction=54954 max_load_reduction_pct=0.838531 moved_to_echo=32652 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3456, 8960, 8704, 18688, 19584, 128, 128, 6272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[15744, 7424, 11904, 21120, 128, 1792, 128, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1792, 384, 4096, 22528, 0, 1280, 25728, 10112, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[27264, 10880, 10496, 128, 4224, 256, 12800, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[512, 12288, 128, 3328, 6656, 14848, 1536, 12032, 8320, 3584, 3328, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[10624, 7680, 128, 8704, 7168, 128, 128, 128, 19072, 7552, 5120, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[512, 20608, 4480, 1280, 8448, 128, 18944, 11520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 12288, 128, 1792, 256, 1152, 768, 3328, 31232, 14464, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[22, 0, 683, 4072, 1, 3100, 4096, 3441, 74, 3239, 4091, 6, 0, 0, 0, 637, 14, 0, 0, 1, 49, 4096, 0, 4092, 4, 15, 408, 3365, 8, 0, 4096, 0, 36, 1078, 5, 0, 0, 15, 24, 4051, 6, 0, 4036, 3995, 0, 1238, 0, 0, 0, 0, 0, 40, 5, 3873, 1, 0, 451, 1, 0, 3407, 2, 74, 3588, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 98, 1273, 0, 4043, 3315, 2472, 6, 3977, 4081, 1, 7, 0, 0, 0, 1112, 3914, 0, 11, 0, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 29, 816, 0, 4088, 0, 222, 0, 0, 3011, 569, 2566, 0, 2, 1918, 0, 4096, 0, 3108, 34, 0, 0, 3833, 0, 4096, 24, 61, 671, 11, 1441, 0, 2456, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[3, 14, 4088, 4093, 0, 3326, 31, 4026, 261, 3666, 0, 23, 3866, 0, 4, 0, 0, 1489, 0, 2282, 0, 0, 0, 1889, 0, 0, 0, 4089, 0, 279, 244, 0, 7, 4041, 2911, 2284, 0, 470, 33, 2297, 110, 0, 110, 3475, 689, 1016, 0, 1854, 0, 1, 0, 4, 0, 3768, 598, 10, 2, 0, 0, 2, 4084, 2, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[611, 53, 63, 0, 0, 4086, 1, 3847, 0, 0, 509, 0, 4075, 0, 0, 0, 0, 1, 314, 0, 0, 4091, 3674, 0, 10, 0, 2, 2946, 0, 4, 4095, 0, 4, 2158, 0, 0, 5, 29, 2042, 3700, 0, 0, 3642, 2496, 0, 0, 0, 3389, 0, 1, 11, 278, 0, 4051, 0, 3994, 8, 963, 0, 0, 3879, 2, 3802, 2700] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[128, 0, 3428, 0, 0, 0, 3986, 4094, 0, 4000, 0, 3934, 21, 3, 0, 0, 0, 3951, 11, 48, 44, 0, 0, 76, 9, 500, 4012, 3239, 0, 0, 4096, 289, 4077, 0, 21, 5, 0, 0, 4078, 0, 100, 0, 0, 4018, 0, 4094, 51, 1901, 15, 74, 0, 1, 692, 3786, 66, 91, 0, 0, 750, 28, 3549, 0, 2270, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 1115, 12, 522, 0, 3314, 961, 3951, 0, 3807, 349, 5, 0, 8, 0, 0, 1, 1, 1180, 0, 0, 4038, 0, 664, 18, 3617, 2, 3580, 3199, 0, 213, 2, 185, 2762, 2207, 2011, 0, 44, 4096, 871, 3, 10, 3233, 4096, 1050, 403, 16, 2621, 0, 60, 0, 12, 1502, 4094, 0, 1373, 0, 0, 4096, 4, 17, 208, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[17, 0, 1, 2271, 0, 1, 3698, 4096, 565, 3994, 0, 1569, 0, 0, 0, 0, 3109, 3053, 0, 2, 0, 2228, 0, 4089, 0, 4000, 0, 688, 1, 1487, 768, 0, 879, 1, 1612, 0, 2840, 0, 2310, 0, 20, 0, 53, 1, 0, 4096, 0, 5, 0, 6, 0, 164, 0, 4058, 332, 25, 4066, 0, 0, 4, 4036, 3, 4095, 1293] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 394, 4094, 325, 4, 4064, 3809, 4096, 0, 3966, 4096, 0, 3787, 17, 2, 0, 1221, 3628, 0, 0, 0, 4085, 0, 4015, 0, 6, 743, 0, 13, 0, 4069, 98, 21, 464, 69, 25, 0, 0, 1, 3139, 0, 0, 0, 4095, 260, 2, 24, 0, 0, 6, 0, 626, 0, 1666, 0, 0, 3, 0, 4027, 4029, 537, 1, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 98, 1273, 0, 3365, 3315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3977, 4081, 1, 7, 0, 0, 0, 678, 443, 60, 0, 0, 0, 0, 0, 1112, 3914, 0, 11, 0, 3172, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 816, 643, 481, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 222, 0, 0, 3011, 569, 58, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 2, 1732, 0, 4096, 0, 3108, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 3833, 0, 4096, 24, 61, 1829, 0, 0, 0, 0, 0, 0, 0, 671, 11, 1441, 0, 2456, 2, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8052, 9253, 12284, 1969, 7948, 11504, 9877, 4649] avg=8192.0000 max=12284 max_rank=2 min=1969 min_rank=3 imbalance_ratio=1.499512 max_load_reduction=53252 max_load_reduction_pct=0.812561 moved_to_echo=33978 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 683, 4072, 1, 2580, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 3239, 4091, 6, 0, 0, 0, 637, 520, 443, 125, 0, 0, 0, 0, 0, 14, 0, 0, 1, 49, 3172, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4, 15, 408, 3365, 8, 0, 4096, 0, 896, 481, 0, 0, 0, 0, 0, 0, 36, 1078, 5, 0, 0, 15, 24, 4051, 122, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4036, 3605, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 5, 3873, 1, 0, 2545, 0, 0, 0, 0, 0, 0, 0, 451, 1, 0, 3407, 2, 74, 3588, 0, 143, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11454, 9135, 7328, 9273, 5331, 8885, 6464, 7666] avg=8192.0000 max=11454 max_rank=0 min=5331 min_rank=4 imbalance_ratio=1.398193 max_load_reduction=54082 max_load_reduction_pct=0.825226 moved_to_echo=28346 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[611, 53, 63, 0, 0, 3401, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 0, 4075, 0, 0, 0, 685, 442, 78, 0, 0, 0, 0, 0, 0, 1, 314, 0, 0, 3169, 3674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2, 2946, 0, 4, 4095, 0, 1002, 480, 0, 0, 0, 0, 0, 0, 4, 2158, 0, 0, 5, 29, 2042, 3700, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3642, 2253, 0, 0, 0, 3389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 278, 0, 4051, 0, 3994, 2845, 0, 0, 0, 0, 0, 0, 0, 8, 963, 0, 0, 3879, 2, 3802, 2700, 89, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 5789, 7158, 8539, 8014, 9284, 11180, 11443] avg=8192.0000 max=11443 max_rank=7 min=4129 min_rank=0 imbalance_ratio=1.396851 max_load_reduction=54093 max_load_reduction_pct=0.825394 moved_to_echo=39921 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[128, 0, 3428, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 3934, 21, 3, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 3951, 11, 48, 44, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 9, 500, 4012, 3239, 0, 0, 4096, 289, 1066, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 21, 5, 0, 0, 4078, 0, 123, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 3625, 0, 4094, 51, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 15, 74, 0, 1, 692, 3786, 66, 91, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 750, 28, 3549, 0, 2270, 0, 144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7542, 8084, 4130, 13211, 8304, 9771, 7753, 6741] avg=8192.0000 max=13211 max_rank=3 min=4130 min_rank=2 imbalance_ratio=1.612671 max_load_reduction=52325 max_load_reduction_pct=0.798416 moved_to_echo=32569 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 14, 4088, 4093, 0, 2767, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 3666, 0, 23, 3866, 0, 4, 0, 559, 0, 112, 0, 0, 0, 0, 0, 0, 1489, 0, 2282, 0, 0, 0, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 279, 244, 0, 1049, 0, 0, 0, 0, 0, 0, 0, 7, 4041, 2911, 2284, 0, 470, 33, 2297, 109, 0, 0, 0, 0, 0, 0, 0, 110, 0, 110, 3127, 689, 1016, 0, 1854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 3768, 598, 10, 2977, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 4084, 2, 4095, 0, 127, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10996, 8491, 5660, 5661, 12152, 6906, 7358, 8312] avg=8192.0000 max=12152 max_rank=4 min=5660 min_rank=2 imbalance_ratio=1.483398 max_load_reduction=53384 max_load_reduction_pct=0.814575 moved_to_echo=34728 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1115, 12, 522, 0, 2758, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3807, 349, 5, 0, 8, 0, 0, 556, 437, 129, 0, 0, 0, 0, 0, 1, 1, 1180, 0, 0, 3127, 0, 664, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3617, 2, 3580, 3199, 0, 213, 2, 1029, 474, 0, 0, 0, 0, 0, 0, 185, 2762, 2207, 2011, 0, 44, 4096, 871, 125, 0, 0, 0, 0, 0, 0, 0, 3, 10, 3233, 3695, 1050, 403, 16, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 12, 1502, 4094, 0, 1373, 2922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 17, 208, 0, 3, 147, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5368, 5291, 4973, 12134, 12301, 11031, 9963, 4475] avg=8192.0000 max=12301 max_rank=4 min=4475 min_rank=7 imbalance_ratio=1.501587 max_load_reduction=53235 max_load_reduction_pct=0.812302 moved_to_echo=37770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 394, 4094, 325, 4, 3382, 3809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 4096, 0, 3787, 17, 2, 0, 682, 442, 128, 0, 0, 0, 0, 0, 1221, 3628, 0, 0, 0, 3164, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 743, 0, 13, 0, 4069, 98, 1067, 479, 0, 0, 0, 0, 0, 0, 21, 464, 69, 25, 0, 0, 1, 3139, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 260, 2, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 626, 0, 1666, 0, 0, 3029, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4027, 4029, 537, 1, 0, 9, 147, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12008, 13120, 12028, 6475, 3844, 3981, 5327, 8753] avg=8192.0000 max=13120 max_rank=1 min=3844 min_rank=4 imbalance_ratio=1.601562 max_load_reduction=52416 max_load_reduction_pct=0.799805 moved_to_echo=21905 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 1, 2271, 0, 0, 3698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 3994, 0, 1569, 0, 0, 0, 0, 1, 243, 1, 0, 0, 0, 0, 0, 3109, 3053, 0, 2, 0, 1722, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 688, 1, 1487, 768, 0, 1067, 263, 0, 0, 0, 0, 0, 0, 879, 1, 1612, 0, 2840, 0, 2310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 53, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 164, 0, 4058, 332, 25, 3029, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4, 4036, 3, 4095, 1293, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5987, 6373, 11975, 8274, 7642, 4174, 7614, 13497] avg=8192.0000 max=13497 max_rank=7 min=4174 min_rank=5 imbalance_ratio=1.647583 max_load_reduction=52039 max_load_reduction_pct=0.794052 moved_to_echo=32927 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 8192, 5248, 17920, 3328, 1792, 17664, 1280, 7936, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2816, 128, 11136, 21760, 2048, 14976, 128, 12928, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5248, 14592, 6912, 4608, 2944, 640, 15616, 14720, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[896, 1664, 12544, 12672, 128, 18304, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5248, 1024, 10368, 7552, 18560, 384, 17920, 4096, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 256, 128, 4992, 2304, 29440, 1024, 5632, 22272, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[5504, 16128, 1536, 2432, 128, 17536, 3712, 18944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1024, 26752, 13184, 5632, 11776, 128, 128, 640, 3712, 2560, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3165, 0, 0, 0, 3885, 4089, 0, 11, 187, 0, 503, 2527, 0, 929, 57, 3, 3, 0, 0, 0, 336, 0, 4024, 4090, 3279, 2, 4052, 41, 245, 4069, 0, 0, 0, 338, 0, 0, 3543, 410, 795, 3960, 0, 0, 72, 3227, 1365, 0, 3157, 0, 1239, 1344, 0, 869, 4066, 1, 322, 0, 3422, 0, 0, 0, 1906, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4095, 0, 722, 0, 3185, 221, 0, 3, 4016, 0, 0, 0, 89, 3523, 286, 4096, 12, 4090, 0, 0, 4032, 4055, 2, 3132, 0, 4096, 0, 0, 33, 0, 4096, 1739, 0, 4096, 0, 19, 3116, 0, 17, 0, 66, 0, 1207, 560, 3, 1860, 0, 33, 561, 4000, 0, 3, 0, 0, 0, 46, 0, 326, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 5, 4094, 0, 1, 3860, 0, 3, 0, 0, 2206, 0, 3589, 290, 0, 2, 297, 15, 596, 1, 4089, 4005, 0, 0, 3, 3763, 687, 0, 3904, 2, 0, 1485, 0, 0, 0, 0, 4096, 15, 329, 3931, 0, 4060, 0, 0, 0, 4096, 149, 1, 4094, 0, 98, 4096, 1182, 3, 4069, 14, 3, 0, 0, 0, 5, 0, 2398, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4095, 8, 3769, 13, 484, 0, 32, 28, 24, 1446, 4070, 0, 0, 4043, 0, 4083, 0, 29, 3142, 57, 0, 4090, 0, 55, 0, 0, 3462, 944, 4009, 0, 39, 0, 0, 30, 0, 32, 2615, 3585, 621, 6, 3303, 4064, 1214, 0, 4095, 250, 0, 0, 2283, 1391, 0, 47, 3, 0, 4055, 0, 2, 0, 18, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3993, 0, 0, 0, 316, 8, 58, 4093, 3914, 0, 0, 3983, 0, 0, 234, 2, 3995, 6, 3756, 3217, 75, 45, 708, 4094, 4086, 158, 992, 3397, 0, 0, 16, 0, 5, 0, 1371, 644, 1215, 6, 1, 1221, 2313, 4046, 627, 4061, 0, 1470, 4, 352, 355, 0, 0, 0, 3936, 12, 0, 0, 2735, 0, 4, 0, 12, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 5, 4063, 0, 1, 774, 0, 266, 0, 0, 4089, 0, 1, 1232, 0, 10, 0, 89, 4083, 464, 4096, 1465, 0, 0, 0, 0, 3094, 0, 47, 3935, 0, 10, 0, 938, 0, 602, 4096, 2, 0, 0, 318, 3944, 0, 0, 0, 3841, 3773, 3483, 4094, 727, 2699, 4096, 0, 44, 3718, 11, 153, 0, 0, 0, 327, 0, 946, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4095, 135, 607, 0, 45, 0, 0, 32, 0, 12, 3652, 359, 1, 3994, 0, 45, 1451, 0, 3939, 0, 3495, 0, 3872, 0, 0, 3796, 723, 0, 674, 3977, 131, 1723, 0, 0, 1, 459, 4091, 0, 4048, 152, 0, 3212, 3474, 0, 0, 747, 1, 0, 4096, 0, 0, 322, 4085, 0, 4043, 0, 0, 0, 0, 4, 5, 0, 1, 37] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3871, 0, 4079, 2791, 0, 0, 0, 1397, 0, 354, 4095, 3, 0, 3826, 2, 11, 0, 0, 3968, 3, 4096, 4077, 8, 0, 9, 4012, 0, 0, 2500, 2541, 2711, 3284, 0, 1, 0, 84, 2768, 0, 9, 0, 51, 3951, 65, 0, 0, 4086, 119, 1999, 7, 148, 128, 0, 0, 0, 4094, 0, 31, 0, 349, 0, 0, 7, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3165, 0, 0, 0, 3885, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 187, 0, 503, 2527, 0, 929, 0, 0, 0, 0, 0, 0, 0, 0, 57, 3, 3, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 1470, 2, 4052, 41, 245, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 3543, 410, 1121, 0, 0, 0, 0, 0, 0, 0, 795, 1610, 0, 0, 72, 3227, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3157, 0, 1239, 1344, 0, 869, 2737, 1, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 3422, 0, 0, 0, 1906, 3, 4090, 2350, 1809, 208, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11139, 4157, 399, 13903, 5412, 7069, 9347, 14110] avg=8192.0000 max=14110 max_rank=7 min=399 min_rank=2 imbalance_ratio=1.722412 max_load_reduction=51426 max_load_reduction_pct=0.784698 moved_to_echo=35938 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4095, 0, 722, 453, 0, 0, 0, 0, 0, 0, 0, 0, 3185, 221, 0, 3, 4016, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 89, 3523, 286, 2404, 12, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1818, 2, 3132, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4096, 1739, 0, 4096, 0, 1103, 1159, 0, 0, 0, 0, 0, 0, 19, 1267, 0, 17, 0, 66, 0, 1207, 0, 0, 0, 0, 0, 0, 0, 0, 560, 3, 1860, 0, 33, 561, 2692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 46, 0, 326, 4, 4032, 1849, 2237, 205, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9366, 7505, 10404, 9048, 12226, 2576, 5709, 8702] avg=8192.0000 max=12226 max_rank=4 min=2576 min_rank=5 imbalance_ratio=1.492432 max_load_reduction=53310 max_load_reduction_pct=0.813446 moved_to_echo=29213 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 8, 3769, 13, 484, 0, 32, 28, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1446, 4070, 0, 0, 4043, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 3142, 57, 0, 4090, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1552, 944, 4009, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 32, 2615, 3585, 621, 6, 1118, 0, 0, 0, 0, 0, 0, 0, 3303, 1652, 1214, 0, 4095, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2283, 1391, 0, 47, 3, 0, 2730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 18, 0, 0, 0, 0, 0, 0, 2412, 1910, 207, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8429, 13666, 7373, 6544, 8007, 10514, 6454, 4549] avg=8192.0000 max=13666 max_rank=1 min=4549 min_rank=7 imbalance_ratio=1.668213 max_load_reduction=51870 max_load_reduction_pct=0.791473 moved_to_echo=29524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 135, 607, 0, 45, 0, 0, 32, 386, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3652, 359, 1, 3994, 0, 45, 69, 0, 0, 0, 0, 0, 0, 0, 1451, 0, 3939, 0, 2051, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 674, 3977, 131, 1723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 459, 4091, 0, 4048, 152, 1115, 989, 0, 0, 0, 0, 0, 0, 0, 1306, 3474, 0, 0, 747, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 322, 4085, 0, 2721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 1, 37, 3796, 1906, 398, 207, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5300, 8132, 11313, 6830, 10855, 5528, 11224, 6354] avg=8192.0000 max=11313 max_rank=2 min=5300 min_rank=0 imbalance_ratio=1.380981 max_load_reduction=54223 max_load_reduction_pct=0.827377 moved_to_echo=33961 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3993, 0, 0, 0, 316, 8, 415, 0, 0, 0, 0, 0, 0, 0, 58, 4093, 3914, 0, 0, 3983, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 234, 2, 3995, 6, 2204, 3217, 75, 45, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 1832, 158, 992, 3397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 5, 0, 1371, 644, 1215, 6, 1085, 1063, 0, 0, 0, 0, 0, 0, 1, 497, 2313, 4046, 627, 4061, 0, 1470, 0, 0, 0, 0, 0, 0, 0, 0, 4, 352, 355, 0, 0, 0, 2650, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 0, 4, 0, 12, 0, 4094, 724, 2254, 201, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4732, 12122, 9778, 7087, 5405, 13015, 3373, 10024] avg=8192.0000 max=13015 max_rank=5 min=3373 min_rank=6 imbalance_ratio=1.588745 max_load_reduction=52521 max_load_reduction_pct=0.801407 moved_to_echo=31817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4094, 0, 1, 3860, 0, 3, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2206, 0, 3589, 290, 0, 2, 84, 0, 0, 0, 0, 0, 0, 0, 297, 15, 596, 1, 2390, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 306, 0, 3904, 2, 0, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 329, 3931, 1127, 1161, 0, 0, 0, 0, 0, 0, 0, 1644, 0, 0, 0, 4096, 149, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 98, 4096, 1182, 3, 2729, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 5, 0, 2398, 0, 3763, 2416, 381, 213, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8417, 6171, 7304, 5700, 10659, 5890, 12216, 9179] avg=8192.0000 max=12216 max_rank=6 min=5700 min_rank=3 imbalance_ratio=1.491211 max_load_reduction=53320 max_load_reduction_pct=0.813599 moved_to_echo=37944 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4063, 0, 1, 774, 0, 266, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1, 1232, 0, 10, 80, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4083, 464, 2404, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 47, 3935, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 938, 0, 602, 4096, 2, 0, 0, 1025, 1159, 0, 0, 0, 0, 0, 0, 318, 1603, 0, 0, 0, 3841, 3773, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 727, 2699, 4096, 0, 44, 2503, 11, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 327, 0, 946, 0, 0, 2341, 1707, 190, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5562, 5412, 8505, 5379, 7822, 13018, 14174, 5664] avg=8192.0000 max=14174 max_rank=6 min=5379 min_rank=3 imbalance_ratio=1.730225 max_load_reduction=51362 max_load_reduction_pct=0.783722 moved_to_echo=40678 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3871, 0, 4079, 2791, 0, 0, 0, 1397, 453, 0, 0, 0, 0, 0, 0, 0, 0, 354, 4095, 3, 0, 3826, 2, 11, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 3, 2404, 4077, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2500, 2541, 2711, 3284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 84, 2768, 0, 9, 0, 1129, 1159, 0, 0, 0, 0, 0, 0, 51, 1606, 65, 0, 0, 4086, 119, 1999, 0, 0, 0, 0, 0, 0, 0, 0, 7, 148, 128, 0, 0, 0, 2756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 349, 0, 0, 7, 0, 1, 4012, 2345, 0, 209, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12591, 8371, 10460, 11045, 5150, 7926, 3039, 6954] avg=8192.0000 max=12591 max_rank=0 min=3039 min_rank=6 imbalance_ratio=1.536987 max_load_reduction=52945 max_load_reduction_pct=0.807877 moved_to_echo=23069 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2048, 256, 23296, 896, 13952, 16896, 8448, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4608, 11264, 7168, 4096, 4864, 20480, 5504, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 0, 6528, 128, 512, 128, 5632, 128, 23808, 16384, 10752, 1664, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[18304, 2688, 6400, 9984, 5376, 1536, 21632, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 1024, 128, 5632, 20864, 4352, 13952, 4608, 8832, 6784, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 9216, 22528, 384, 4224, 23936, 128, 5120, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[12160, 256, 27904, 2816, 640, 8832, 4352, 6656, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4864, 0, 8704, 1152, 19328, 13952, 7296, 10624, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[15, 358, 0, 3097, 52, 3985, 7, 0, 3849, 0, 0, 126, 0, 618, 0, 4091, 2423, 0, 0, 1074, 2828, 0, 1227, 392, 0, 0, 4051, 4027, 204, 57, 0, 4085, 447, 0, 397, 2392, 4079, 2, 8, 4096, 57, 1248, 300, 0, 439, 0, 0, 0, 1518, 0, 3638, 3446, 0, 269, 54, 0, 0, 172, 3681, 2724, 2, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4042, 0, 3636, 1173, 33, 0, 0, 12, 3, 0, 487, 3994, 0, 0, 0, 4095, 3889, 2, 29, 0, 5, 503, 3707, 1222, 241, 230, 0, 3985, 14, 0, 1, 4061, 1182, 0, 0, 1682, 2782, 4096, 30, 0, 3762, 530, 4085, 62, 3, 4096, 5, 0, 3345, 0, 0, 3928, 20, 263, 0, 300, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4024, 30, 145, 3567, 16, 130, 4, 0, 3687, 0, 0, 104, 4066, 0, 0, 3964, 4080, 4096, 2, 0, 0, 0, 239, 1861, 0, 0, 0, 4096, 1, 4085, 0, 4096, 0, 1, 1, 4093, 831, 1173, 0, 0, 3908, 0, 0, 11, 0, 0, 0, 0, 4079, 274, 0, 704, 0, 148, 0, 0, 456, 4, 10, 4087, 8, 525, 0, 2930] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 30, 0, 1248, 0, 2787, 2, 3719, 4096, 21, 0, 0, 949, 0, 4082, 3630, 4041, 0, 0, 1, 4039, 0, 0, 302, 0, 44, 2, 3597, 122, 0, 0, 4094, 0, 4048, 440, 0, 2446, 54, 7, 0, 14, 0, 555, 222, 0, 0, 36, 2, 3799, 0, 3410, 4096, 0, 0, 0, 4096, 0, 0, 1952, 3544, 0, 9, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 2530, 1946, 0, 79, 0, 0, 4091, 0, 0, 22, 46, 49, 516, 3095, 4096, 4092, 1, 266, 520, 0, 263, 475, 0, 0, 4052, 3992, 0, 0, 0, 4096, 9, 0, 0, 0, 2644, 4075, 762, 11, 534, 0, 0, 7, 0, 0, 1466, 0, 2761, 43, 0, 3969, 0, 0, 0, 3638, 106, 0, 4028, 4096, 0, 1633, 106, 1419] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 3590, 26, 2165, 0, 3236, 4095, 0, 0, 0, 200, 4039, 0, 4070, 4096, 21, 0, 752, 0, 0, 19, 0, 0, 0, 1813, 4011, 0, 1, 0, 4059, 0, 956, 170, 7, 0, 4096, 100, 0, 0, 31, 3843, 0, 8, 0, 0, 0, 3926, 365, 0, 3728, 0, 0, 0, 0, 4, 0, 4096, 4094, 368, 3480, 65, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[788, 0, 20, 3485, 0, 0, 0, 0, 3911, 12, 0, 4089, 174, 0, 0, 3987, 3956, 599, 73, 0, 0, 0, 1862, 1, 0, 18, 3483, 4096, 0, 4094, 0, 4096, 0, 0, 0, 112, 0, 772, 3, 0, 1856, 0, 0, 0, 0, 4067, 6, 0, 2724, 0, 0, 4096, 0, 1, 170, 4096, 3745, 0, 1038, 4096, 10, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[3970, 55, 0, 4095, 6, 1195, 0, 2, 920, 3, 0, 2306, 81, 1446, 0, 4093, 3884, 888, 30, 0, 0, 9, 1, 37, 0, 1, 4096, 3368, 0, 20, 0, 4096, 403, 0, 0, 0, 4069, 9, 6, 1, 4084, 0, 1, 0, 0, 0, 12, 0, 4084, 0, 0, 3977, 1775, 1, 19, 4096, 2, 284, 4079, 4023, 4, 5, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4024, 30, 145, 3567, 16, 130, 4, 0, 157, 0, 0, 0, 0, 0, 0, 0, 3687, 0, 0, 104, 4066, 0, 0, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 2, 0, 0, 0, 239, 1861, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4085, 0, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4093, 831, 1173, 0, 0, 1563, 20, 0, 0, 0, 0, 0, 0, 3908, 0, 0, 11, 0, 0, 0, 0, 3126, 681, 0, 0, 0, 0, 0, 0, 4079, 274, 0, 320, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 4, 10, 4087, 8, 525, 0, 2930, 116, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8073, 9577, 10369, 9152, 7682, 7726, 4821, 8136] avg=8192.0000 max=10369 max_rank=2 min=4821 min_rank=6 imbalance_ratio=1.265747 max_load_reduction=55167 max_load_reduction_pct=0.841782 moved_to_echo=28365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4042, 0, 3636, 1173, 33, 0, 0, 12, 876, 0, 0, 0, 0, 0, 0, 0, 3, 0, 487, 3994, 0, 0, 0, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 2, 29, 0, 5, 503, 3707, 1222, 510, 0, 0, 0, 0, 0, 0, 0, 241, 230, 0, 3985, 14, 0, 1, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1182, 0, 0, 1682, 2782, 4096, 30, 0, 1614, 113, 0, 0, 0, 0, 0, 0, 3762, 530, 4085, 62, 3, 4096, 5, 0, 3100, 704, 0, 0, 0, 0, 0, 0, 3345, 0, 0, 1777, 20, 263, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9772, 6261, 9867, 5432, 11499, 16347, 5705, 653] avg=8192.0000 max=16347 max_rank=5 min=653 min_rank=7 imbalance_ratio=1.995483 max_load_reduction=49189 max_load_reduction_pct=0.750565 moved_to_echo=34204 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 2530, 1946, 0, 79, 0, 0, 885, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 22, 46, 49, 516, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 1, 266, 520, 0, 263, 475, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 3992, 0, 0, 0, 970, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2644, 4075, 762, 11, 1220, 115, 0, 0, 0, 0, 0, 0, 534, 0, 0, 7, 0, 0, 1466, 0, 3126, 532, 0, 0, 0, 0, 0, 0, 2761, 43, 0, 1794, 0, 0, 0, 3638, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 4028, 4096, 0, 1633, 106, 1419, 659, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5442, 6067, 10229, 9014, 8836, 5665, 8236, 12047] avg=8192.0000 max=12047 max_rank=7 min=5442 min_rank=0 imbalance_ratio=1.470581 max_load_reduction=53489 max_load_reduction_pct=0.816177 moved_to_echo=34784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[15, 358, 0, 3097, 52, 3985, 7, 0, 768, 0, 0, 0, 0, 0, 0, 0, 3849, 0, 0, 126, 0, 618, 0, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 2423, 0, 0, 1074, 2828, 0, 1227, 392, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4027, 204, 57, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 397, 2392, 4079, 2, 8, 4096, 1613, 99, 0, 0, 0, 0, 0, 0, 57, 1248, 300, 0, 439, 0, 0, 0, 3118, 703, 0, 0, 0, 0, 0, 0, 1518, 0, 3638, 1559, 0, 269, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 3681, 2724, 2, 1, 0, 0, 572, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 6368, 8392, 9306, 13133, 5865, 7038, 7152] avg=8192.0000 max=13133 max_rank=4 min=5865 min_rank=5 imbalance_ratio=1.603149 max_load_reduction=52403 max_load_reduction_pct=0.799606 moved_to_echo=33188 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 0, 1248, 0, 2787, 2, 3719, 913, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 949, 0, 4082, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 1, 4039, 0, 0, 302, 532, 0, 0, 0, 0, 0, 0, 0, 0, 44, 2, 3597, 122, 0, 0, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 440, 0, 2446, 54, 7, 0, 1431, 118, 0, 0, 0, 0, 0, 0, 14, 0, 555, 222, 0, 0, 36, 2, 3125, 624, 0, 0, 0, 0, 0, 0, 3799, 0, 3410, 1853, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1952, 3544, 0, 9, 0, 0, 680, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8699, 10723, 8915, 4734, 8544, 4578, 13158, 6185] avg=8192.0000 max=13158 max_rank=6 min=4578 min_rank=5 imbalance_ratio=1.606201 max_load_reduction=52378 max_load_reduction_pct=0.799225 moved_to_echo=32465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3590, 26, 2165, 0, 3236, 831, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 200, 4039, 0, 1766, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 752, 0, 0, 19, 0, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1813, 4011, 0, 1, 0, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 956, 170, 7, 0, 4096, 100, 0, 1604, 108, 0, 0, 0, 0, 0, 0, 0, 31, 3843, 0, 8, 0, 0, 0, 3098, 700, 0, 0, 0, 0, 0, 0, 3926, 365, 0, 1686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 4094, 368, 3480, 65, 6, 619, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9848, 10100, 5372, 6786, 7041, 7680, 5977, 12732] avg=8192.0000 max=12732 max_rank=7 min=5372 min_rank=2 imbalance_ratio=1.554199 max_load_reduction=52804 max_load_reduction_pct=0.805725 moved_to_echo=33430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[788, 0, 20, 3485, 0, 0, 0, 0, 913, 0, 0, 0, 0, 0, 0, 0, 3911, 12, 0, 4089, 174, 0, 0, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 599, 73, 0, 0, 0, 1862, 1, 532, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3483, 4096, 0, 4094, 0, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 772, 3, 0, 1572, 118, 0, 0, 0, 0, 0, 0, 1856, 0, 0, 0, 0, 4067, 6, 0, 3126, 685, 0, 0, 0, 0, 0, 0, 2724, 0, 0, 1853, 0, 1, 170, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3745, 0, 1038, 4096, 10, 0, 0, 0, 680, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5206, 9916, 7023, 12661, 2577, 9740, 8844, 9569] avg=8192.0000 max=12661 max_rank=3 min=2577 min_rank=4 imbalance_ratio=1.545532 max_load_reduction=52875 max_load_reduction_pct=0.806808 moved_to_echo=30730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3970, 55, 0, 4095, 6, 1195, 0, 2, 891, 0, 0, 0, 0, 0, 0, 0, 920, 3, 0, 2306, 81, 1446, 0, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 888, 30, 0, 0, 9, 1, 37, 520, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3368, 0, 20, 0, 966, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 4069, 9, 6, 1, 1617, 119, 0, 0, 0, 0, 0, 0, 4084, 0, 1, 0, 0, 0, 12, 0, 3130, 708, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 1782, 1775, 1, 19, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 284, 4079, 4023, 4, 5, 0, 0, 665, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10214, 6524, 5369, 8451, 6224, 7935, 11757, 9062] avg=8192.0000 max=11757 max_rank=6 min=5369 min_rank=2 imbalance_ratio=1.435181 max_load_reduction=53779 max_load_reduction_pct=0.820602 moved_to_echo=34978 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4352, 512, 18944, 26752, 512, 5760, 256, 4480, 4736, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[14336, 1920, 8832, 384, 512, 8192, 1536, 128, 24960, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2048, 5120, 1024, 8320, 16896, 14336, 1024, 4224, 12288, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26240, 768, 7168, 12672, 1920, 768, 256, 16256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[30592, 9728, 256, 2176, 7424, 512, 7424, 4352, 3712, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[24704, 128, 512, 10752, 5632, 6272, 4608, 13568, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12928, 512, 6400, 22272, 256, 10368, 128, 7040, 6272, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 384, 17536, 31232, 384, 8320, 128, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[3902, 0, 709, 1764, 142, 0, 0, 0, 0, 426, 0, 3966, 0, 10, 0, 4096, 0, 0, 74, 1090, 21, 0, 13, 4077, 3953, 0, 199, 3856, 3203, 3664, 0, 0, 97, 4083, 3157, 2627, 10, 95, 0, 0, 0, 989, 2, 0, 4092, 58, 7, 1542, 0, 4068, 2772, 412, 454, 0, 3443, 0, 0, 12, 0, 0, 97, 0, 562, 1792] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4073, 0, 0, 2540, 0, 0, 2, 2, 1, 0, 97, 4071, 0, 0, 2542, 169, 1, 0, 2952, 14, 40, 0, 4061, 1311, 0, 0, 0, 727, 38, 0, 1339, 4086, 1, 3939, 0, 0, 0, 4096, 0, 2099, 50, 3316, 0, 3723, 1, 3398, 2, 0, 0, 1397, 0, 10, 585, 3, 0, 437, 0, 465, 3477, 2, 3775, 2605, 4089, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[2266, 3, 310, 4094, 713, 3630, 31, 47, 250, 0, 2209, 0, 26, 0, 0, 3870, 34, 0, 0, 0, 0, 16, 0, 2498, 12, 0, 0, 4088, 0, 153, 306, 27, 1390, 2045, 55, 3691, 0, 4072, 0, 4078, 1213, 4096, 0, 194, 372, 2, 0, 37, 2880, 4054, 3903, 7, 4040, 0, 2, 4044, 0, 0, 2, 0, 2, 769, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4080, 6, 0, 4096, 107, 0, 3576, 9, 0, 746, 0, 3156, 0, 2, 0, 235, 0, 0, 3988, 0, 4061, 149, 21, 1337, 0, 0, 87, 20, 3644, 4092, 0, 1351, 64, 4079, 452, 83, 0, 3976, 3949, 176, 2127, 0, 1343, 5, 0, 3821, 0, 0, 14, 0, 937, 29, 3932, 0, 35, 0, 0, 0, 1660, 0, 4, 4085, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4077, 0, 1, 256, 0, 773, 0, 0, 349, 3236, 0, 4096, 0, 0, 23, 0, 19, 0, 0, 263, 1532, 4089, 3742, 0, 0, 0, 0, 1, 531, 0, 2126, 0, 439, 3999, 3, 978, 0, 3334, 3849, 0, 38, 4091, 0, 0, 0, 4, 0, 1, 0, 3628, 953, 0, 3700, 0, 4095, 1484, 0, 317, 596, 4094, 3195, 136, 1446, 42] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4064, 0, 364, 0, 0, 74, 13, 38, 12, 0, 3981, 0, 6, 108, 0, 0, 117, 4045, 35, 0, 7, 4091, 65, 4096, 1341, 4, 3, 0, 4096, 0, 0, 4077, 2, 0, 0, 0, 138, 4094, 3865, 0, 3436, 0, 1, 644, 3600, 0, 17, 0, 316, 282, 4, 0, 4096, 2255, 0, 0, 51, 0, 3948, 12, 0, 4026, 73, 4039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3, 0, 0, 176, 0, 0, 90, 11, 2711, 6, 0, 0, 6, 14, 0, 0, 0, 2696, 0, 4086, 4096, 2686, 0, 0, 1233, 0, 2876, 52, 4096, 0, 4095, 20, 10, 4, 4025, 4050, 0, 0, 0, 0, 250, 0, 80, 14, 0, 1779, 3523, 4096, 0, 107, 0, 2380, 2779, 8, 0, 3576, 0, 1127, 4056, 0, 623] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[2601, 4, 4096, 4094, 83, 0, 441, 0, 0, 0, 0, 2815, 94, 0, 0, 0, 7, 0, 0, 0, 7, 1101, 0, 3928, 3460, 0, 0, 8, 3585, 3112, 0, 0, 4, 921, 1, 0, 0, 0, 290, 1671, 0, 205, 0, 204, 157, 3698, 0, 0, 4088, 4094, 4017, 3878, 836, 0, 4084, 81, 0, 0, 4096, 0, 3613, 0, 162, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 0, 2540, 0, 0, 2, 2, 137, 0, 0, 0, 0, 0, 0, 0, 1, 0, 97, 4071, 0, 0, 2542, 169, 1397, 185, 0, 0, 0, 0, 0, 0, 1, 0, 2952, 14, 40, 0, 4061, 1311, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 32, 0, 1339, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3939, 0, 0, 0, 887, 0, 2099, 0, 0, 0, 0, 0, 0, 0, 0, 50, 3316, 0, 3723, 1, 3398, 2, 0, 3024, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 342, 3, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 3477, 2, 3775, 2605, 4089, 0, 67, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6754, 8462, 8383, 6184, 6926, 13553, 792, 14482] avg=8192.0000 max=14482 max_rank=7 min=792 min_rank=6 imbalance_ratio=1.767822 max_load_reduction=51054 max_load_reduction_pct=0.779022 moved_to_echo=35753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4077, 0, 1, 256, 0, 773, 0, 0, 871, 0, 0, 0, 0, 0, 0, 0, 349, 3236, 0, 4096, 0, 0, 23, 0, 3628, 151, 0, 0, 0, 0, 0, 0, 19, 0, 0, 263, 1532, 4089, 3742, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 440, 0, 2126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 3999, 3, 978, 0, 721, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4091, 0, 0, 0, 4, 0, 1, 2462, 252, 0, 0, 0, 0, 0, 0, 0, 0, 953, 0, 2152, 0, 4095, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 596, 4094, 3195, 136, 1446, 42, 425, 30, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5978, 11483, 9706, 2567, 9989, 6848, 8684, 10281] avg=8192.0000 max=11483 max_rank=1 min=2567 min_rank=3 imbalance_ratio=1.401733 max_load_reduction=54053 max_load_reduction_pct=0.824783 moved_to_echo=35802 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4080, 6, 0, 4096, 107, 0, 3576, 9, 925, 0, 0, 0, 0, 0, 0, 0, 0, 746, 0, 3156, 0, 2, 0, 235, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 4061, 149, 21, 1337, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 20, 3018, 4092, 0, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 64, 4079, 452, 83, 0, 860, 3949, 176, 0, 0, 0, 0, 0, 0, 0, 0, 2127, 0, 1343, 5, 0, 3821, 0, 0, 2936, 268, 0, 0, 0, 0, 0, 0, 14, 0, 937, 29, 2287, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 0, 4, 4085, 2, 0, 452, 206, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12799, 4319, 9976, 8568, 9663, 10500, 3302, 6409] avg=8192.0000 max=12799 max_rank=0 min=3302 min_rank=6 imbalance_ratio=1.562378 max_load_reduction=52737 max_load_reduction_pct=0.804703 moved_to_echo=29874 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3902, 0, 709, 1764, 142, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 3966, 0, 10, 0, 4096, 4068, 4, 0, 0, 0, 0, 0, 0, 0, 0, 74, 1090, 21, 0, 13, 4077, 369, 0, 0, 0, 0, 0, 0, 0, 3953, 0, 199, 3856, 2653, 3664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 4083, 3157, 2627, 10, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 989, 2, 0, 4092, 58, 7, 1542, 70, 30, 0, 0, 0, 0, 0, 0, 0, 0, 2772, 412, 266, 0, 3443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 97, 0, 562, 1792, 52, 181, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6623, 12570, 5644, 14325, 9995, 6790, 6893, 2696] avg=8192.0000 max=14325 max_rank=3 min=2696 min_rank=7 imbalance_ratio=1.748657 max_load_reduction=51211 max_load_reduction_pct=0.781418 moved_to_echo=26374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2266, 3, 310, 4094, 713, 3630, 31, 47, 951, 0, 0, 0, 0, 0, 0, 0, 250, 0, 2209, 0, 26, 0, 0, 3870, 4054, 187, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 16, 0, 2498, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4088, 0, 153, 306, 27, 0, 0, 0, 0, 0, 0, 0, 0, 1390, 2045, 55, 3691, 0, 876, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 1213, 4096, 0, 194, 372, 2, 0, 37, 3009, 275, 0, 0, 0, 0, 0, 0, 2880, 0, 3903, 7, 2350, 0, 2, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 769, 5, 0, 464, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12045, 10596, 2548, 4586, 12135, 9198, 13186, 1242] avg=8192.0000 max=13186 max_rank=6 min=1242 min_rank=7 imbalance_ratio=1.609619 max_load_reduction=52350 max_load_reduction_pct=0.798798 moved_to_echo=35761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4064, 0, 364, 0, 0, 74, 13, 38, 964, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3981, 0, 6, 108, 0, 0, 282, 185, 0, 0, 0, 0, 0, 0, 117, 4045, 35, 0, 7, 4091, 65, 4096, 472, 0, 0, 0, 0, 0, 0, 0, 1341, 4, 3, 0, 3393, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 138, 886, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3436, 0, 1, 644, 3600, 0, 17, 0, 3023, 279, 0, 0, 0, 0, 0, 0, 316, 0, 4, 0, 2383, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 3948, 12, 0, 4026, 73, 4039, 470, 231, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5517, 4574, 12928, 8818, 4891, 11000, 4958, 12850] avg=8192.0000 max=12928 max_rank=2 min=4574 min_rank=1 imbalance_ratio=1.578125 max_load_reduction=52608 max_load_reduction_pct=0.802734 moved_to_echo=33699 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 3, 0, 0, 176, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 90, 11, 2711, 6, 0, 0, 6, 3523, 182, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 2696, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2686, 0, 0, 1233, 0, 2876, 52, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 20, 10, 4, 871, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, 80, 14, 0, 2972, 7, 0, 0, 0, 0, 0, 0, 1779, 0, 4096, 0, 63, 0, 2380, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3576, 0, 1127, 4056, 0, 623, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4300, 6529, 10892, 10943, 9050, 3323, 11097, 9402] avg=8192.0000 max=11097 max_rank=6 min=3323 min_rank=5 imbalance_ratio=1.354614 max_load_reduction=54439 max_load_reduction_pct=0.830673 moved_to_echo=32872 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2601, 4, 4096, 4094, 83, 0, 441, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2815, 94, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 1101, 0, 3928, 416, 0, 0, 0, 0, 0, 0, 0, 3460, 0, 0, 8, 2965, 3112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 921, 1, 0, 0, 0, 290, 1671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 204, 157, 3698, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 4088, 0, 4017, 3878, 476, 0, 4084, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3613, 0, 162, 0, 99, 204, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11520, 7003, 5459, 9545, 2887, 4324, 16624, 8174] avg=8192.0000 max=16624 max_rank=6 min=2887 min_rank=4 imbalance_ratio=2.029297 max_load_reduction=48912 max_load_reduction_pct=0.746338 moved_to_echo=32009 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[9088, 0, 16768, 4352, 10368, 2304, 14080, 8832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 896, 17408, 4224, 11904, 15744, 6400, 6528, 2048, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2048, 23168, 3712, 7424, 256, 5248, 16128, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[29184, 128, 5504, 16896, 1152, 4480, 4352, 128, 4224, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11520, 128, 384, 9984, 12544, 13952, 3840, 13696, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 4096, 7168, 1408, 8448, 9472, 12032, 21376, 1792, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 4608, 6400, 20864, 256, 128, 2688, 8448, 21120, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[6912, 12800, 1408, 5120, 8320, 11136, 128, 1664, 17536, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4000, 508, 9, 120, 0, 34, 4055, 0, 643, 4091, 0, 0, 533, 718, 0, 2178, 0, 89, 0, 276, 0, 3, 0, 0, 0, 0, 1616, 15, 4096, 5, 14, 3753, 0, 988, 24, 4007, 0, 0, 3134, 4081, 3581, 40, 554, 3990, 4068, 4090, 4089, 101, 0, 0, 1898, 4, 0, 4010, 0, 0, 0, 0, 85, 0, 2, 33, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 339, 1, 0, 8, 3206, 9, 0, 165, 3719, 1, 0, 57, 0, 0, 845, 7, 15, 4075, 0, 0, 3, 0, 1685, 4096, 1, 4074, 3710, 3779, 3513, 4095, 0, 118, 65, 3875, 4093, 4053, 3044, 0, 0, 4096, 221, 0, 0, 270, 0, 0, 0, 89, 0, 0, 2, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[29, 2560, 0, 6, 0, 17, 676, 4066, 1239, 1402, 4089, 9, 0, 2, 23, 0, 3955, 1551, 99, 0, 3654, 0, 3959, 0, 0, 4096, 0, 51, 10, 3786, 0, 324, 0, 0, 0, 0, 45, 49, 0, 4096, 4070, 3079, 3448, 0, 1042, 0, 4059, 0, 257, 0, 0, 71, 0, 0, 0, 0, 1738, 4096, 3880, 0, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[3743, 0, 0, 91, 1, 191, 0, 4092, 0, 208, 0, 147, 0, 0, 3281, 15, 0, 0, 4089, 2797, 0, 70, 0, 118, 3571, 2175, 15, 897, 0, 1896, 2, 0, 0, 0, 0, 0, 1499, 2, 206, 4010, 2597, 49, 3972, 4091, 4078, 3994, 3888, 0, 1, 28, 250, 0, 0, 0, 88, 619, 0, 4093, 3489, 0, 11, 0, 1170, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 6, 14, 20, 0, 4074, 766, 0, 0, 3, 4096, 0, 34, 20, 2, 0, 4083, 3767, 16, 58, 0, 3988, 3348, 1, 0, 4095, 0, 99, 371, 4035, 0, 0, 4096, 174, 3935, 0, 3111, 0, 0, 3885, 4020, 4080, 0, 0, 32, 3944, 0, 0, 0, 1, 6, 62, 0, 0, 144, 0, 0, 4096, 1028, 0, 0, 0, 26, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 8, 150, 7, 0, 4088, 1637, 0, 0, 427, 506, 0, 13, 1220, 0, 1407, 48, 36, 0, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 4095, 0, 3, 0, 0, 3574, 1, 4091, 15, 4094, 76, 32, 3860, 53, 3984, 2337, 4059, 3632, 218, 211, 1328, 0, 3955, 35, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4061, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[3513, 113, 148, 0, 4096, 4092, 0, 5, 0, 0, 0, 2870, 10, 619, 0, 3, 0, 4096, 2609, 3, 4071, 3579, 0, 1, 0, 2898, 0, 3, 0, 1514, 0, 29, 0, 0, 0, 0, 3903, 517, 189, 233, 4037, 4094, 3755, 2, 0, 4084, 2392, 24, 39, 0, 15, 0, 31, 15, 0, 0, 2469, 3982, 0, 196, 0, 206, 1077, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 2018, 29, 129, 3672, 0, 0, 0, 2978, 0, 1470, 0, 2, 0, 1585, 0, 0, 3307, 0, 0, 0, 0, 3967, 0, 4014, 3965, 1, 4096, 0, 4095, 0, 0, 0, 0, 2458, 373, 2294, 0, 4057, 0, 586, 2549, 0, 3, 632, 1406, 4094, 817, 0, 0, 0, 0, 1275, 0, 0, 0, 49, 4092, 4096, 113, 22, 0, 1292, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4096, 0, 1483, 461, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 339, 1, 0, 8, 3325, 1253, 0, 0, 0, 0, 0, 0, 3206, 9, 0, 165, 2089, 1, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 845, 7, 15, 4075, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1685, 4096, 1, 4074, 3710, 888, 0, 0, 0, 0, 0, 0, 0, 0, 3513, 195, 0, 118, 65, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 3044, 0, 0, 4096, 221, 0, 0, 270, 4093, 2622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 2, 14, 2430, 575, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6041, 9022, 5527, 5089, 14457, 7944, 14346, 3110] avg=8192.0000 max=14457 max_rank=4 min=3110 min_rank=7 imbalance_ratio=1.764771 max_load_reduction=51079 max_load_reduction_pct=0.779404 moved_to_echo=39857 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4000, 508, 9, 120, 0, 34, 4055, 110, 382, 0, 0, 0, 0, 0, 0, 0, 643, 4091, 0, 0, 533, 718, 0, 2908, 1315, 0, 0, 0, 0, 0, 0, 2178, 0, 89, 0, 156, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1616, 15, 4096, 5, 14, 10, 0, 0, 0, 0, 0, 0, 0, 3753, 0, 988, 24, 4007, 0, 0, 737, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 170, 40, 554, 3990, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 1898, 4, 0, 4010, 0, 4090, 2753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 2, 33, 1, 2015, 503, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9218, 10208, 2426, 5756, 9509, 12924, 12856, 2639] avg=8192.0000 max=12924 max_rank=5 min=2426 min_rank=2 imbalance_ratio=1.577637 max_load_reduction=52612 max_load_reduction_pct=0.802795 moved_to_echo=37928 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 14, 20, 0, 4074, 766, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 34, 20, 2, 0, 3313, 1274, 0, 0, 0, 0, 0, 0, 4083, 3767, 16, 58, 0, 3988, 3348, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 99, 371, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 174, 3935, 0, 3111, 0, 0, 914, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 194, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 62, 0, 0, 144, 0, 0, 2670, 0, 0, 0, 0, 0, 0, 0, 4096, 1028, 0, 0, 0, 26, 0, 2498, 573, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5353, 8742, 15261, 8600, 12230, 4246, 2883, 8221] avg=8192.0000 max=15261 max_rank=2 min=2883 min_rank=6 imbalance_ratio=1.862915 max_load_reduction=50275 max_load_reduction_pct=0.767136 moved_to_echo=27580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[29, 2560, 0, 6, 0, 17, 676, 4066, 1458, 499, 0, 0, 0, 0, 0, 0, 1239, 1402, 4089, 9, 0, 2, 23, 0, 2500, 0, 0, 0, 0, 0, 0, 0, 3955, 1551, 99, 0, 2051, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 51, 10, 3786, 0, 324, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 49, 0, 963, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 147, 3448, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 71, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 1738, 4096, 3880, 0, 0, 0, 3, 0, 2634, 432, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9311, 9264, 11615, 8412, 1057, 8707, 4387, 12783] avg=8192.0000 max=12783 max_rank=7 min=1057 min_rank=4 imbalance_ratio=1.560425 max_load_reduction=52753 max_load_reduction_pct=0.804947 moved_to_echo=26934 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3513, 113, 148, 0, 4096, 4092, 0, 5, 1624, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 10, 619, 0, 3, 3324, 1319, 0, 0, 0, 0, 0, 0, 0, 4096, 2609, 3, 2286, 3579, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2898, 0, 3, 0, 1514, 0, 29, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 517, 189, 56, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 195, 3755, 2, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 15, 0, 31, 15, 0, 0, 2392, 2765, 0, 0, 0, 0, 0, 0, 2469, 3982, 0, 196, 0, 206, 1077, 4, 149, 575, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13619, 8145, 12574, 4605, 4665, 8013, 5257, 8658] avg=8192.0000 max=13619 max_rank=0 min=4605 min_rank=3 imbalance_ratio=1.662476 max_load_reduction=51917 max_load_reduction_pct=0.792191 moved_to_echo=26593 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3743, 0, 0, 91, 1, 191, 0, 4092, 0, 489, 0, 0, 0, 0, 0, 0, 0, 208, 0, 147, 0, 0, 3281, 15, 39, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2797, 0, 70, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 2175, 15, 897, 0, 1896, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1499, 2, 206, 942, 0, 0, 0, 0, 0, 0, 0, 0, 2597, 4, 3972, 4091, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 28, 250, 0, 0, 0, 88, 619, 3888, 2704, 0, 0, 0, 0, 0, 0, 0, 4093, 3489, 0, 11, 0, 1170, 2, 2579, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8607, 4980, 7074, 8556, 2649, 14742, 7578, 11350] avg=8192.0000 max=14742 max_rank=5 min=2649 min_rank=4 imbalance_ratio=1.799561 max_load_reduction=50794 max_load_reduction_pct=0.775055 moved_to_echo=36319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2018, 29, 129, 3672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2978, 0, 1470, 0, 2, 0, 1585, 0, 2070, 454, 0, 0, 0, 0, 0, 0, 0, 3307, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 3965, 1, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2458, 373, 2294, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 121, 0, 3, 632, 0, 0, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1275, 0, 0, 0, 4094, 952, 0, 0, 0, 0, 0, 0, 49, 4092, 4096, 113, 22, 0, 1292, 0, 0, 358, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5848, 8559, 7274, 16171, 9182, 2159, 6321, 10022] avg=8192.0000 max=16171 max_rank=3 min=2159 min_rank=5 imbalance_ratio=1.973999 max_load_reduction=49365 max_load_reduction_pct=0.753250 moved_to_echo=27684 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 8, 150, 7, 0, 4088, 1637, 1636, 12, 0, 0, 0, 0, 0, 0, 0, 0, 427, 506, 0, 13, 1220, 0, 3138, 1312, 0, 0, 0, 0, 0, 0, 1407, 48, 36, 0, 2294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4095, 0, 3, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3574, 1, 4091, 15, 4094, 12, 0, 0, 0, 0, 0, 0, 0, 0, 32, 177, 53, 3984, 2337, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 211, 1328, 0, 3955, 35, 0, 0, 0, 3632, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4061, 4093, 52, 545, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7539, 6616, 3785, 8347, 11787, 6801, 11908, 8753] avg=8192.0000 max=11908 max_rank=6 min=3785 min_rank=2 imbalance_ratio=1.453613 max_load_reduction=53628 max_load_reduction_pct=0.818298 moved_to_echo=39249 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7936, 256, 12672, 4608, 19072, 4736, 12288, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4352, 20480, 12544, 512, 128, 256, 7680, 4224, 12416, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[14848, 12800, 7040, 3072, 8960, 7680, 11392, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7296, 8704, 768, 512, 7936, 8448, 9728, 13952, 6400, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23040, 1280, 11392, 8832, 12288, 0, 0, 9216, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3712, 1408, 384, 10112, 1664, 128, 4352, 896, 26368, 17280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 2304, 18304, 3584, 512, 1280, 6912, 128, 20736, 8320, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11776, 17280, 896, 6784, 512, 27648, 128, 384, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 1550, 0, 1, 0, 0, 1401, 0, 0, 0, 0, 0, 3, 3853, 1, 421, 0, 730, 4017, 2885, 4095, 212, 0, 0, 4094, 515, 0, 185, 0, 0, 64, 0, 6, 2609, 0, 4052, 0, 0, 354, 4096, 3717, 2, 0, 0, 3142, 4077, 0, 12, 0, 4059, 617, 654, 3726, 0, 0, 0, 2063, 3964, 1942, 0, 1359, 128, 0, 930] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3114, 0, 7, 0, 2007, 4094, 0, 0, 0, 2974, 0, 0, 308, 16, 4095, 0, 0, 4, 0, 4096, 2, 263, 694, 0, 3947, 10, 0, 0, 864, 94, 0, 675, 1727, 9, 0, 4090, 1946, 0, 0, 4092, 0, 0, 4, 1, 0, 4095, 3920, 3697, 334, 4053, 1, 2, 3266, 0, 0, 311, 0, 0, 0, 12, 3055, 0, 0, 3657] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4085, 2820, 0, 0, 1221, 4096, 0, 7, 149, 4096, 910, 601, 4065, 8, 3994, 0, 0, 155, 0, 4079, 0, 546, 0, 0, 0, 265, 4, 0, 34, 77, 0, 17, 1745, 0, 0, 4052, 0, 0, 0, 4096, 0, 0, 3402, 0, 0, 0, 4092, 0, 4, 4096, 1309, 6, 3083, 0, 4090, 4, 0, 2613, 0, 4, 1000, 0, 0, 711] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3938, 113, 0, 47, 0, 0, 16, 4096, 2478, 27, 0, 533, 3576, 4096, 3991, 0, 0, 0, 0, 30, 3837, 0, 4086, 2, 0, 31, 0, 100, 369, 1783, 0, 0, 0, 202, 0, 0, 4, 2, 191, 3467, 4024, 7, 3720, 101, 346, 0, 3721, 3948, 4089, 3860, 0, 20, 4060, 0, 0, 0, 0, 4, 621, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[481, 0, 0, 0, 0, 1593, 0, 4096, 3847, 0, 0, 0, 4092, 0, 0, 2, 2, 1615, 459, 164, 0, 3953, 3967, 0, 0, 659, 4095, 134, 13, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 4041, 0, 0, 0, 3794, 3116, 4095, 4096, 2777, 95, 4096, 0, 0, 0, 0, 64, 0, 404, 3873, 0, 38, 1695] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 3837, 19, 847, 0, 3569, 0, 0, 500, 4096, 0, 0, 0, 0, 3679, 0, 0, 0, 4096, 36, 599, 146, 2231, 1773, 4015, 0, 171, 0, 1868, 0, 0, 1, 0, 4094, 245, 4049, 0, 160, 28, 4092, 3269, 0, 117, 0, 593, 2103, 0, 0, 19, 4000, 3294, 0, 3361, 53, 415, 8, 3, 0, 0, 0, 4056, 0, 0, 94] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[8, 4014, 507, 0, 0, 0, 0, 0, 0, 3773, 636, 0, 3991, 4096, 7, 3626, 0, 0, 3268, 4083, 4094, 0, 0, 60, 3, 1, 1316, 0, 0, 0, 2, 0, 1695, 2217, 0, 2019, 0, 0, 483, 3943, 3584, 7, 4096, 338, 33, 0, 4096, 823, 134, 0, 0, 0, 2586, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 1874, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 0, 3823, 0, 0, 0, 4091, 3981, 0, 1942, 4096, 5, 350, 48, 0, 0, 2714, 1311, 0, 836, 235, 16, 808, 290, 0, 0, 0, 51, 2111, 1, 0, 610, 4096, 0, 0, 0, 3875, 12, 52, 4096, 30, 3, 3058, 8, 4032, 3303, 0, 3731, 0, 0, 0, 3708, 0, 0, 0, 0, 20, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1550, 0, 1, 0, 0, 1401, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3853, 1, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 4017, 2885, 4095, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 515, 0, 185, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2609, 0, 4052, 0, 0, 354, 3196, 0, 0, 0, 0, 0, 0, 0, 0, 3717, 2, 0, 0, 3142, 4077, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 617, 654, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 3964, 1942, 0, 1359, 128, 0, 930, 2447, 699, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3153, 4278, 11939, 4858, 10217, 10950, 6609, 13532] avg=8192.0000 max=13532 max_rank=7 min=3153 min_rank=0 imbalance_ratio=1.651855 max_load_reduction=52004 max_load_reduction_pct=0.793518 moved_to_echo=41308 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3114, 0, 7, 0, 2007, 4094, 0, 0, 41, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 16, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 2, 263, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 10, 0, 0, 864, 94, 0, 675, 3920, 2687, 0, 0, 0, 0, 0, 0, 1727, 9, 0, 4090, 1946, 0, 0, 3193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 4095, 0, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 334, 4053, 1, 2, 1122, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3055, 0, 0, 3657, 2144, 698, 287, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9464, 4378, 5059, 12197, 10965, 7797, 5823, 9853] avg=8192.0000 max=12197 max_rank=3 min=4378 min_rank=1 imbalance_ratio=1.488892 max_load_reduction=53339 max_load_reduction_pct=0.813889 moved_to_echo=34438 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3938, 113, 0, 47, 0, 0, 16, 4096, 479, 393, 170, 0, 0, 0, 0, 0, 2478, 0, 0, 533, 3097, 4096, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3837, 0, 4086, 2, 21, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 100, 369, 1783, 0, 0, 3721, 25, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 4, 2, 191, 2706, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 7, 3306, 101, 346, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3860, 0, 20, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 621, 0, 0, 0, 0, 0, 2666, 591, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9252, 14195, 7976, 6029, 3105, 11732, 9363, 3884] avg=8192.0000 max=14195 max_rank=1 min=3105 min_rank=4 imbalance_ratio=1.732788 max_load_reduction=51341 max_load_reduction_pct=0.783401 moved_to_echo=28084 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3837, 19, 847, 0, 3569, 0, 0, 0, 12, 201, 0, 0, 0, 0, 0, 500, 0, 0, 0, 0, 0, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 36, 599, 146, 2231, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 171, 0, 1868, 0, 0, 1, 0, 3701, 0, 0, 0, 0, 0, 0, 0, 4094, 245, 4049, 0, 160, 28, 3193, 0, 0, 0, 0, 0, 0, 0, 0, 3269, 0, 105, 0, 593, 2103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4000, 3294, 0, 1154, 53, 415, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4056, 0, 0, 94, 2207, 698, 395, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8485, 4179, 8881, 9756, 11769, 6070, 8943, 7453] avg=8192.0000 max=11769 max_rank=4 min=4179 min_rank=1 imbalance_ratio=1.436646 max_load_reduction=53767 max_load_reduction_pct=0.820419 moved_to_echo=34235 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4085, 2820, 0, 0, 1221, 4096, 0, 7, 545, 359, 201, 0, 0, 0, 0, 0, 149, 0, 910, 601, 3520, 8, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 4079, 0, 546, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 265, 4, 0, 34, 77, 0, 17, 4092, 3702, 0, 0, 0, 0, 0, 0, 1745, 0, 0, 4052, 0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 1309, 6, 1059, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 0, 4, 1000, 0, 0, 711, 2024, 699, 394, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13334, 9182, 4799, 8191, 8993, 3024, 10568, 7445] avg=8192.0000 max=13334 max_rank=0 min=3024 min_rank=5 imbalance_ratio=1.627686 max_load_reduction=52202 max_load_reduction_pct=0.796539 moved_to_echo=30030 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[481, 0, 0, 0, 0, 1593, 0, 4096, 549, 427, 201, 0, 0, 0, 0, 0, 3847, 0, 0, 0, 3543, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1615, 459, 164, 0, 3953, 3967, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 659, 4095, 134, 13, 0, 0, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3591, 0, 0, 0, 0, 3116, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 2777, 95, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 404, 3873, 0, 38, 1695, 2690, 699, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7347, 7392, 10183, 8695, 3269, 6718, 12469, 9463] avg=8192.0000 max=12469 max_rank=6 min=3269 min_rank=4 imbalance_ratio=1.522095 max_load_reduction=53067 max_load_reduction_pct=0.809738 moved_to_echo=31919 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 0, 3823, 0, 0, 260, 433, 190, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 1682, 4096, 5, 350, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 2714, 1311, 0, 836, 235, 23, 0, 0, 0, 0, 0, 0, 0, 16, 808, 290, 0, 0, 0, 51, 2111, 8, 3698, 0, 0, 0, 0, 0, 0, 1, 0, 610, 4096, 0, 0, 0, 3024, 0, 0, 0, 0, 0, 0, 0, 0, 12, 52, 3640, 30, 3, 3058, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 0, 3731, 0, 0, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 0, 0, 0, 0, 661, 393, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8803, 10114, 5167, 6982, 7731, 10827, 10742, 5170] avg=8192.0000 max=10827 max_rank=5 min=5167 min_rank=2 imbalance_ratio=1.321655 max_load_reduction=54709 max_load_reduction_pct=0.834793 moved_to_echo=34470 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[8, 4014, 507, 0, 0, 0, 0, 0, 538, 436, 195, 0, 0, 0, 0, 0, 0, 0, 636, 0, 3453, 4096, 7, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3268, 4083, 4094, 0, 0, 60, 27, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1316, 0, 0, 0, 2, 0, 4096, 3410, 0, 0, 0, 0, 0, 0, 1695, 2217, 0, 2019, 0, 0, 483, 3073, 0, 0, 0, 0, 0, 0, 0, 0, 3584, 7, 3633, 338, 33, 0, 0, 823, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 1874, 0, 1701, 675, 363, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5698, 11818, 11532, 8828, 9487, 8418, 1019, 8736] avg=8192.0000 max=11818 max_rank=1 min=1019 min_rank=6 imbalance_ratio=1.442627 max_load_reduction=53718 max_load_reduction_pct=0.819672 moved_to_echo=27660 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12032, 24192, 11776, 896, 8320, 128, 8320, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7040, 0, 5632, 1152, 15616, 16256, 15872, 4480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5248, 9216, 896, 22400, 2048, 256, 1152, 24832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2176, 6656, 2688, 512, 21632, 128, 1920, 7168, 16000, 5504, 1920, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[14720, 128, 17408, 512, 4224, 13440, 0, 15744, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[15744, 12416, 640, 896, 3328, 17280, 1536, 8320, 2432, 2176, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12160, 2304, 5888, 512, 3200, 2048, 128, 2816, 19712, 17280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 2560, 11904, 18176, 13952, 5120, 11904, 2176, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True + [2026-08-30 05:00:48] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 40471.0 | throughput per GPU (TFLOP/s/GPU): 8.2 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102348E+01 | loss scale: 1.0 | grad norm: 44.162 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 63756.6884765625 | max allocated: 88305.85302734375 | reserved: 91544.0 | max reserved: 91544.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[98, 2728, 762, 233, 2151, 681, 982, 1414, 1055, 167, 1177, 453, 1029, 783, 183, 451, 159, 559, 1336, 1233, 932, 228, 2389, 1626, 427, 1331, 1173, 1142, 942, 327, 1334, 1730, 3402, 601, 275, 793, 129, 2395, 2452, 608, 2171, 240, 1780, 871, 1318, 1182, 1297, 1559, 432, 222, 377, 2471, 853, 195, 704, 157, 723, 192, 1051, 1670, 1608, 649, 663, 1281] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[46, 2711, 349, 203, 1607, 290, 1038, 881, 1128, 164, 2110, 263, 522, 877, 241, 247, 86, 488, 1287, 489, 1222, 255, 3015, 2041, 529, 1102, 1053, 850, 1736, 418, 2547, 2291, 3109, 459, 591, 373, 79, 2499, 2175, 358, 2755, 570, 1512, 751, 1540, 1917, 828, 543, 216, 296, 229, 1724, 1041, 234, 353, 55, 508, 344, 1095, 2208, 1703, 365, 1795, 1225] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[92, 2934, 669, 292, 1315, 338, 829, 700, 1045, 242, 1223, 132, 1306, 624, 142, 368, 130, 490, 709, 738, 1370, 160, 3691, 1927, 235, 1372, 1072, 864, 831, 227, 1698, 2434, 3237, 1076, 318, 559, 123, 2726, 2162, 923, 2889, 131, 1087, 708, 1925, 1676, 1649, 1093, 299, 299, 279, 2491, 565, 337, 352, 86, 803, 461, 989, 1819, 1466, 669, 631, 1509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[192, 3567, 792, 290, 1657, 376, 678, 1470, 1074, 310, 2243, 293, 731, 867, 237, 363, 172, 420, 834, 1151, 1065, 286, 2767, 2217, 672, 1449, 998, 1070, 858, 263, 1628, 1672, 2489, 460, 597, 639, 95, 2019, 1846, 314, 2589, 594, 1729, 1077, 1219, 1642, 930, 681, 306, 543, 497, 2182, 980, 726, 573, 113, 586, 337, 1051, 1848, 1255, 719, 661, 1577] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[114, 2567, 662, 333, 1497, 474, 856, 625, 843, 120, 1458, 214, 913, 862, 285, 377, 199, 599, 749, 1018, 897, 424, 2901, 1641, 322, 892, 1342, 884, 1087, 215, 1744, 2432, 3315, 639, 176, 452, 588, 2583, 2744, 420, 2608, 713, 1304, 929, 1228, 1641, 1084, 1799, 266, 144, 365, 2004, 801, 419, 715, 176, 1066, 397, 1399, 2364, 1463, 528, 592, 1068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[161, 2852, 586, 279, 1608, 631, 974, 1570, 1303, 130, 1693, 169, 787, 890, 302, 464, 192, 548, 582, 822, 1233, 339, 2869, 1544, 346, 1151, 1059, 1031, 957, 263, 1539, 1882, 2898, 606, 322, 550, 230, 2612, 2391, 629, 2608, 367, 1197, 570, 1391, 1596, 976, 896, 219, 316, 321, 2652, 979, 645, 763, 82, 798, 382, 1697, 1786, 1256, 662, 811, 1572] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[170, 2787, 479, 407, 998, 359, 916, 701, 1211, 245, 1622, 453, 1072, 594, 300, 350, 132, 613, 751, 1014, 1099, 664, 2947, 2039, 484, 1539, 1448, 794, 1458, 309, 1661, 1989, 3065, 1055, 446, 517, 208, 2165, 1931, 802, 2751, 540, 1367, 790, 1431, 1375, 1180, 842, 323, 638, 445, 2000, 695, 479, 589, 171, 464, 649, 1000, 1904, 1714, 470, 827, 1098] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[94, 2960, 494, 287, 1582, 186, 1189, 1193, 1175, 201, 1612, 172, 777, 754, 120, 347, 105, 907, 926, 790, 880, 104, 2764, 1560, 511, 1553, 859, 1076, 1614, 455, 1532, 2035, 2843, 444, 748, 558, 78, 2832, 2236, 810, 2557, 335, 1316, 826, 1575, 1841, 1073, 1027, 315, 567, 229, 2431, 1266, 549, 627, 92, 364, 195, 1051, 2228, 1293, 541, 792, 1083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[98, 2728, 762, 233, 2151, 681, 982, 1414, 151, 116, 0, 0, 0, 0, 0, 0, 1055, 167, 1177, 453, 1029, 783, 183, 451, 1651, 756, 270, 0, 0, 0, 0, 0, 159, 559, 1336, 1233, 932, 228, 2003, 1626, 0, 0, 0, 0, 0, 0, 0, 0, 427, 1331, 1173, 1142, 942, 327, 1334, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1135, 601, 275, 793, 129, 2395, 2452, 608, 0, 0, 0, 0, 0, 0, 0, 0, 237, 240, 1780, 871, 1318, 1182, 1297, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 432, 222, 377, 2471, 853, 195, 704, 157, 1934, 616, 0, 0, 0, 0, 0, 0, 723, 192, 1051, 1519, 1608, 649, 663, 1281, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9316, 7975, 8076, 7650, 8388, 8484, 7961, 7686] avg=8192.0000 max=9316 max_rank=0 min=7650 min_rank=3 imbalance_ratio=1.137207 max_load_reduction=56220 max_load_reduction_pct=0.857849 moved_to_echo=32519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[46, 2711, 349, 203, 1607, 290, 1038, 881, 200, 147, 0, 0, 0, 0, 0, 0, 1128, 164, 2110, 263, 522, 877, 241, 247, 1508, 1002, 341, 0, 0, 0, 0, 0, 86, 488, 1287, 489, 1222, 255, 2527, 2041, 0, 0, 0, 0, 0, 0, 0, 0, 529, 1102, 1053, 850, 1736, 418, 2547, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 459, 591, 373, 79, 2499, 2175, 358, 0, 0, 0, 0, 0, 0, 0, 0, 300, 570, 1512, 751, 1540, 1917, 828, 543, 0, 0, 0, 0, 0, 0, 0, 0, 216, 296, 229, 1724, 1041, 234, 353, 55, 2455, 563, 0, 0, 0, 0, 0, 0, 508, 344, 1095, 2008, 1703, 365, 1795, 1225, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7472, 8403, 8395, 9524, 7572, 7961, 7166, 9043] avg=8192.0000 max=9524 max_rank=3 min=7166 min_rank=6 imbalance_ratio=1.162598 max_load_reduction=56012 max_load_reduction_pct=0.854675 moved_to_echo=31742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[192, 3567, 792, 290, 1657, 376, 678, 1470, 168, 135, 0, 0, 0, 0, 0, 0, 1074, 310, 2243, 293, 731, 867, 237, 363, 1208, 731, 313, 0, 0, 0, 0, 0, 172, 420, 834, 1151, 1065, 286, 2319, 2217, 0, 0, 0, 0, 0, 0, 0, 0, 672, 1449, 998, 1070, 858, 263, 1628, 941, 0, 0, 0, 0, 0, 0, 0, 0, 830, 460, 597, 639, 95, 2019, 1846, 314, 0, 0, 0, 0, 0, 0, 0, 0, 282, 594, 1729, 1077, 1219, 1642, 930, 681, 0, 0, 0, 0, 0, 0, 0, 0, 306, 543, 497, 2182, 980, 726, 573, 113, 2307, 451, 0, 0, 0, 0, 0, 0, 586, 337, 1051, 1680, 1255, 719, 661, 1577, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9325, 8370, 8464, 7879, 6800, 8154, 8678, 7866] avg=8192.0000 max=9325 max_rank=0 min=6800 min_rank=4 imbalance_ratio=1.138306 max_load_reduction=56211 max_load_reduction_pct=0.857712 moved_to_echo=31498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[114, 2567, 662, 333, 1497, 474, 856, 625, 214, 141, 0, 0, 0, 0, 0, 0, 843, 120, 1458, 214, 913, 862, 285, 377, 1608, 1063, 328, 0, 0, 0, 0, 0, 199, 599, 749, 1018, 897, 424, 2432, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 322, 892, 1342, 884, 1087, 215, 1744, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 639, 176, 452, 588, 2583, 2744, 420, 0, 0, 0, 0, 0, 0, 0, 0, 284, 713, 1304, 929, 1228, 1641, 1084, 1799, 0, 0, 0, 0, 0, 0, 0, 0, 266, 144, 365, 2004, 801, 419, 715, 176, 2324, 600, 0, 0, 0, 0, 0, 0, 1066, 397, 1399, 2150, 1463, 528, 592, 1068, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7483, 8071, 7959, 7855, 8709, 8982, 7814, 8663] avg=8192.0000 max=8982 max_rank=5 min=7483 min_rank=0 imbalance_ratio=1.096436 max_load_reduction=56554 max_load_reduction_pct=0.862946 moved_to_echo=34168 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[92, 2934, 669, 292, 1315, 338, 829, 700, 169, 183, 0, 0, 0, 0, 0, 0, 1045, 242, 1223, 132, 1306, 624, 142, 368, 1575, 1067, 423, 0, 0, 0, 0, 0, 130, 490, 709, 738, 1370, 160, 3085, 1927, 0, 0, 0, 0, 0, 0, 0, 0, 235, 1372, 1072, 864, 831, 227, 1698, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1076, 318, 559, 123, 2726, 2162, 923, 0, 0, 0, 0, 0, 0, 0, 0, 312, 131, 1087, 708, 1925, 1676, 1649, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 299, 299, 279, 2491, 565, 337, 352, 86, 2577, 591, 0, 0, 0, 0, 0, 0, 803, 461, 989, 1650, 1466, 669, 631, 1509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7521, 8147, 8609, 7666, 8958, 8581, 7876, 8178] avg=8192.0000 max=8958 max_rank=4 min=7521 min_rank=0 imbalance_ratio=1.093506 max_load_reduction=56578 max_load_reduction_pct=0.863312 moved_to_echo=33593 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[170, 2787, 479, 407, 998, 359, 916, 701, 173, 144, 0, 0, 0, 0, 0, 0, 1211, 245, 1622, 453, 1072, 594, 300, 350, 1487, 870, 334, 0, 0, 0, 0, 0, 132, 613, 751, 1014, 1099, 664, 2469, 2039, 0, 0, 0, 0, 0, 0, 0, 0, 484, 1539, 1448, 794, 1458, 309, 1661, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1055, 446, 517, 208, 2165, 1931, 802, 0, 0, 0, 0, 0, 0, 0, 0, 300, 540, 1367, 790, 1431, 1375, 1180, 842, 0, 0, 0, 0, 0, 0, 0, 0, 323, 638, 445, 2000, 695, 479, 589, 171, 2451, 555, 0, 0, 0, 0, 0, 0, 464, 649, 1000, 1731, 1714, 470, 827, 1098, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7134, 8538, 8781, 8812, 8147, 7825, 8346, 7953] avg=8192.0000 max=8812 max_rank=3 min=7134 min_rank=0 imbalance_ratio=1.075684 max_load_reduction=56724 max_load_reduction_pct=0.865540 moved_to_echo=32271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[161, 2852, 586, 279, 1608, 631, 974, 1570, 162, 140, 0, 0, 0, 0, 0, 0, 1303, 130, 1693, 169, 787, 890, 302, 464, 1406, 823, 325, 0, 0, 0, 0, 0, 192, 548, 582, 822, 1233, 339, 2404, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 346, 1151, 1059, 1031, 957, 263, 1539, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 967, 606, 322, 550, 230, 2612, 2391, 629, 0, 0, 0, 0, 0, 0, 0, 0, 284, 367, 1197, 570, 1391, 1596, 976, 896, 0, 0, 0, 0, 0, 0, 0, 0, 219, 316, 321, 2652, 979, 645, 763, 82, 2324, 525, 0, 0, 0, 0, 0, 0, 798, 382, 1697, 1624, 1256, 662, 811, 1572, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8963, 8292, 7664, 7405, 8307, 7277, 8826, 8802] avg=8192.0000 max=8963 max_rank=0 min=7277 min_rank=5 imbalance_ratio=1.094116 max_load_reduction=56573 max_load_reduction_pct=0.863235 moved_to_echo=33212 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[94, 2960, 494, 287, 1582, 186, 1189, 1193, 202, 135, 0, 0, 0, 0, 0, 0, 1175, 201, 1612, 172, 777, 754, 120, 347, 1379, 890, 313, 0, 0, 0, 0, 0, 105, 907, 926, 790, 880, 104, 2316, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 511, 1553, 859, 1076, 1614, 455, 1532, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 949, 444, 748, 558, 78, 2832, 2236, 810, 0, 0, 0, 0, 0, 0, 0, 0, 279, 335, 1316, 826, 1575, 1841, 1073, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 315, 567, 229, 2431, 1266, 549, 627, 92, 2278, 515, 0, 0, 0, 0, 0, 0, 364, 195, 1051, 2026, 1293, 541, 792, 1083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8322, 7740, 7588, 8745, 8655, 8272, 8869, 7345] avg=8192.0000 max=8869 max_rank=6 min=7345 min_rank=7 imbalance_ratio=1.082642 max_load_reduction=56667 max_load_reduction_pct=0.864670 moved_to_echo=33141 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8960, 1664, 13184, 2176, 7168, 6272, 1920, 3072, 11904, 7296, 2688, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[5376, 3072, 9344, 14464, 11776, 4608, 6784, 10496, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1280, 4736, 7296, 7296, 8704, 2560, 19584, 14720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 23168, 4864, 2432, 12416, 3456, 7552, 8576, 1536, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 5376, 3584, 4480, 1536, 19840, 18048, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2304, 3584, 11392, 6528, 11648, 12928, 9088, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3584, 10496, 9088, 7808, 9600, 2560, 13696, 9344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2432, 3072, 2816, 18048, 7296, 3584, 4736, 1024, 18688, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[3676, 199, 701, 3104, 523, 1414, 3886, 0, 130, 657, 80, 0, 0, 4, 710, 12, 1, 4035, 286, 2381, 1053, 1330, 1, 1732, 3089, 14, 394, 4067, 4086, 0, 138, 199, 8, 972, 0, 4092, 544, 1561, 0, 55, 552, 0, 212, 3604, 0, 60, 126, 0, 3812, 1037, 3101, 18, 89, 0, 0, 18, 300, 7, 162, 1255, 0, 1586, 4068, 395] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[3381, 16, 20, 2003, 72, 42, 3175, 18, 477, 4054, 102, 19, 0, 579, 2, 184, 90, 4091, 1263, 3453, 145, 3, 0, 372, 627, 0, 0, 3570, 3572, 485, 0, 529, 2, 2393, 57, 4061, 32, 4066, 46, 30, 2975, 52, 0, 184, 360, 3173, 210, 0, 4048, 2835, 2628, 6, 4, 0, 0, 420, 36, 0, 5, 50, 0, 1317, 3897, 305] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[3890, 0, 0, 4089, 889, 1654, 3895, 0, 0, 4073, 0, 0, 0, 2, 5, 1, 15, 3810, 3884, 1404, 0, 2399, 0, 1678, 4062, 0, 0, 3307, 3981, 122, 0, 13, 56, 4, 0, 1309, 3830, 3817, 0, 35, 39, 0, 94, 1901, 0, 84, 71, 87, 3642, 260, 1558, 0, 11, 0, 0, 0, 0, 0, 115, 110, 0, 951, 4051, 338] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3894, 0, 291, 2524, 65, 82, 4096, 2, 3, 3091, 0, 1, 2, 11, 404, 4, 0, 4053, 11, 37, 5, 2687, 13, 294, 2933, 134, 245, 2340, 4084, 19, 0, 201, 1372, 446, 0, 4020, 224, 3872, 18, 83, 2699, 3, 90, 2188, 84, 1798, 1124, 218, 3326, 3563, 4095, 8, 291, 0, 0, 25, 141, 2, 228, 3, 0, 29, 4042, 18] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[3916, 4, 62, 3049, 6, 88, 3892, 0, 12, 3342, 0, 0, 111, 329, 129, 1, 3, 3741, 1845, 50, 0, 2116, 0, 75, 4032, 228, 4, 4061, 4057, 0, 0, 73, 2243, 265, 16, 3501, 158, 3685, 0, 87, 495, 0, 60, 738, 1, 723, 32, 0, 2689, 3634, 2490, 28, 474, 0, 2, 114, 0, 0, 1116, 227, 0, 3594, 3797, 141] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[3402, 74, 1340, 2784, 14, 816, 3473, 0, 0, 2792, 0, 1, 0, 114, 167, 1, 1, 4060, 457, 2174, 215, 799, 0, 921, 3801, 1, 4, 2704, 3988, 31, 37, 77, 4057, 692, 1, 1107, 3936, 1344, 0, 27, 388, 0, 404, 3832, 833, 104, 65, 6, 3244, 678, 4051, 8, 2050, 55, 0, 113, 3, 0, 53, 0, 41, 5, 3881, 310] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4068, 2, 1224, 3860, 2, 18, 4012, 4, 0, 3625, 3, 10, 18, 68, 421, 4, 3, 4061, 1371, 22, 0, 2931, 0, 338, 4093, 14, 104, 4045, 4043, 4, 0, 228, 1829, 467, 0, 4064, 667, 1457, 0, 14, 0, 0, 0, 1409, 230, 5, 269, 4, 2455, 305, 4068, 0, 1718, 19, 82, 37, 0, 6, 17, 42, 4, 17, 4096, 3659] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4029, 16, 15, 3331, 6, 2935, 4086, 0, 3, 2118, 0, 1, 0, 0, 485, 14, 1, 4080, 1536, 213, 23, 171, 0, 1096, 3524, 39, 1, 3636, 4094, 2, 1, 40, 17, 51, 0, 4095, 2719, 4067, 0, 19, 802, 45, 2, 963, 0, 98, 1869, 10, 1727, 3998, 3415, 0, 1093, 0, 0, 31, 2, 0, 16, 20, 0, 713, 4096, 172] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3424, 199, 701, 3104, 523, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 657, 80, 0, 0, 4, 710, 12, 3886, 851, 0, 0, 0, 0, 0, 0, 1, 3114, 286, 2381, 1053, 1330, 1, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 3089, 14, 394, 4067, 1059, 0, 138, 199, 0, 0, 0, 0, 0, 0, 0, 0, 8, 972, 0, 2213, 544, 1561, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 552, 0, 212, 3604, 0, 60, 126, 0, 2176, 1879, 135, 0, 0, 0, 0, 0, 3812, 1037, 2146, 18, 89, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 300, 7, 162, 1255, 0, 1586, 4068, 395, 921, 820, 252, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9365, 6330, 9898, 8960, 5353, 8744, 7120, 9766] avg=8192.0000 max=9898 max_rank=2 min=5353 min_rank=4 imbalance_ratio=1.208252 max_load_reduction=55638 max_load_reduction_pct=0.848969 moved_to_echo=30983 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3149, 16, 20, 2003, 72, 42, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 477, 4054, 102, 19, 0, 579, 2, 184, 3175, 743, 0, 0, 0, 0, 0, 0, 90, 3157, 1263, 3453, 145, 3, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 3570, 927, 485, 0, 529, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2393, 57, 2197, 32, 4066, 46, 30, 0, 0, 0, 0, 0, 0, 0, 0, 2975, 52, 0, 184, 360, 3173, 210, 0, 1902, 1864, 114, 0, 0, 0, 0, 0, 4048, 2835, 1819, 6, 4, 0, 0, 420, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 5, 50, 0, 1317, 3897, 305, 934, 695, 232, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5320, 9335, 8483, 6138, 8823, 10834, 9132, 7471] avg=8192.0000 max=10834 max_rank=5 min=5320 min_rank=0 imbalance_ratio=1.322510 max_load_reduction=54702 max_load_reduction_pct=0.834686 moved_to_echo=36260 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3789, 2, 1224, 3860, 2, 18, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3625, 3, 10, 18, 68, 421, 4, 4012, 842, 0, 0, 0, 0, 0, 0, 3, 3134, 1371, 22, 0, 2931, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 14, 104, 4045, 1048, 4, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 467, 0, 2198, 667, 1457, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 230, 5, 269, 4, 2153, 1866, 177, 0, 0, 0, 0, 0, 2455, 305, 2815, 0, 1718, 19, 82, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 17, 42, 4, 17, 4096, 3659, 927, 1076, 279, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8899, 9003, 7799, 9536, 6632, 6113, 7431, 10123] avg=8192.0000 max=10123 max_rank=7 min=6113 min_rank=5 imbalance_ratio=1.235718 max_load_reduction=55413 max_load_reduction_pct=0.845535 moved_to_echo=30299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3169, 74, 1340, 2784, 14, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2792, 0, 1, 0, 114, 167, 1, 3473, 830, 0, 0, 0, 0, 0, 0, 1, 3133, 457, 2174, 215, 799, 0, 921, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 1, 4, 2704, 1034, 31, 37, 77, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 692, 1, 599, 3936, 1344, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 388, 0, 404, 3832, 833, 104, 65, 6, 2124, 508, 177, 0, 0, 0, 0, 0, 3244, 678, 2802, 8, 2050, 55, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 53, 0, 41, 5, 3881, 310, 927, 1072, 233, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 7378, 7700, 7689, 10656, 8441, 8950, 6525] avg=8192.0000 max=10656 max_rank=4 min=6525 min_rank=7 imbalance_ratio=1.300781 max_load_reduction=54880 max_load_reduction_pct=0.837402 moved_to_echo=34572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3623, 0, 0, 4089, 889, 1654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 2, 5, 1, 3895, 829, 0, 0, 0, 0, 0, 0, 15, 2940, 3884, 1404, 0, 2399, 0, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 3307, 1032, 122, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4, 0, 708, 3830, 3817, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 94, 1901, 0, 84, 71, 87, 2120, 601, 68, 0, 0, 0, 0, 0, 3642, 260, 1078, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 110, 0, 951, 4051, 338, 870, 412, 267, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10255, 8805, 12320, 8536, 8450, 5065, 4991, 7114] avg=8192.0000 max=12320 max_rank=2 min=4991 min_rank=6 imbalance_ratio=1.503906 max_load_reduction=53216 max_load_reduction_pct=0.812012 moved_to_echo=25620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3627, 0, 291, 2524, 65, 82, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3091, 0, 1, 2, 11, 404, 4, 4096, 850, 0, 0, 0, 0, 0, 0, 0, 3128, 11, 37, 5, 2687, 13, 294, 0, 0, 0, 0, 0, 0, 0, 0, 2933, 134, 245, 2340, 1059, 19, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 1372, 446, 0, 2174, 224, 3872, 18, 83, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 3, 90, 2188, 84, 1798, 1124, 218, 2175, 1846, 179, 0, 0, 0, 0, 0, 3326, 3563, 2832, 8, 291, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 141, 2, 228, 3, 0, 29, 4042, 18, 925, 1084, 267, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6591, 8462, 6175, 6931, 8189, 12404, 10045, 6739] avg=8192.0000 max=12404 max_rank=5 min=6175 min_rank=2 imbalance_ratio=1.514160 max_load_reduction=53132 max_load_reduction_pct=0.810730 moved_to_echo=37377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3753, 16, 15, 3331, 6, 2935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2118, 0, 1, 0, 0, 485, 14, 4086, 852, 0, 0, 0, 0, 0, 0, 1, 3149, 1536, 213, 23, 171, 0, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 39, 1, 3636, 1062, 2, 1, 40, 0, 0, 0, 0, 0, 0, 0, 0, 17, 51, 0, 2215, 2719, 4067, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 802, 45, 2, 963, 0, 98, 1869, 10, 2180, 1880, 149, 0, 0, 0, 0, 0, 1727, 3998, 2362, 0, 1093, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 16, 20, 0, 713, 4096, 172, 931, 904, 276, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10056, 7559, 6189, 8305, 9088, 7998, 9211, 7130] avg=8192.0000 max=10056 max_rank=0 min=6189 min_rank=2 imbalance_ratio=1.227539 max_load_reduction=55480 max_load_reduction_pct=0.846558 moved_to_echo=33427 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3644, 4, 62, 3049, 6, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3342, 0, 0, 111, 329, 129, 1, 3892, 848, 0, 0, 0, 0, 0, 0, 3, 2883, 1845, 50, 0, 2116, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 228, 4, 4061, 1043, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 2243, 265, 16, 1891, 158, 3685, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 495, 0, 60, 738, 1, 723, 32, 0, 2166, 1610, 112, 0, 0, 0, 0, 0, 2689, 3634, 1715, 28, 474, 0, 2, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 227, 0, 3594, 3797, 141, 858, 663, 272, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6853, 8664, 6972, 9441, 8345, 5937, 8656, 10668] avg=8192.0000 max=10668 max_rank=7 min=5937 min_rank=5 imbalance_ratio=1.302246 max_load_reduction=54868 max_load_reduction_pct=0.837219 moved_to_echo=33606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 24704, 10752, 9856, 1536, 12544, 128, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[9600, 5376, 128, 14208, 12160, 23936, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[28288, 384, 3712, 24832, 1664, 7168, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 128, 896, 14848, 1536, 6144, 3840, 384, 17024, 12160, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[24960, 16384, 17664, 128, 5760, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26240, 512, 768, 27776, 8320, 768, 256, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 23808, 256, 128, 256, 1152, 2432, 256, 30592, 6656, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[512, 128, 1792, 1792, 128, 8320, 32000, 5376, 7296, 6784, 2176, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4096, 601, 4084, 0, 73, 0, 1480, 4094, 18, 0, 328, 2, 8, 0, 14, 10, 55, 0, 0, 9, 1, 1109, 0, 3445, 18, 102, 0, 383, 31, 4063, 3985, 8, 2, 1, 4084, 1317, 420, 3118, 0, 312, 4096, 3989, 10, 1569, 3, 0, 698, 271, 595, 0, 30, 2723, 194, 511, 4052, 0, 0, 3476, 0, 2349, 3616, 0, 0, 83] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4081, 0, 3929, 37, 1, 18, 2790, 3981, 1, 3, 3603, 1, 0, 0, 16, 2, 2434, 4, 13, 21, 0, 1165, 13, 3426, 10, 570, 514, 1601, 6, 376, 3999, 0, 47, 8, 4092, 113, 0, 3703, 0, 37, 596, 4088, 49, 159, 0, 1, 4096, 53, 13, 0, 0, 317, 3810, 0, 0, 0, 0, 4088, 0, 3641, 3966, 0, 0, 44] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4095, 2027, 4096, 11, 2, 0, 7, 4075, 31, 2, 754, 13, 0, 5, 0, 8, 847, 1, 1, 0, 0, 2943, 4, 4003, 0, 0, 0, 89, 148, 1049, 4067, 8, 0, 2, 4096, 292, 0, 626, 2, 17, 4092, 1723, 0, 4036, 0, 0, 2855, 1516, 0, 1, 0, 3984, 21, 0, 4049, 3, 0, 3976, 0, 1863, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3261, 1, 4081, 0, 48, 31, 1, 2010, 2281, 5, 0, 6, 66, 196, 2619, 2716, 157, 7, 0, 548, 10, 3445, 0, 2382, 20, 12, 1224, 10, 47, 4002, 3916, 1215, 85, 0, 4047, 3776, 0, 4067, 0, 34, 3900, 40, 12, 1633, 161, 605, 167, 4057, 0, 0, 0, 22, 4064, 36, 0, 2, 0, 4096, 15, 10, 211, 0, 175, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3406, 5, 4045, 447, 0, 2, 2465, 3956, 25, 0, 83, 0, 0, 52, 1, 4, 202, 0, 0, 13, 0, 4095, 0, 4061, 0, 71, 0, 6, 315, 4011, 4006, 3, 8, 0, 4090, 2, 0, 201, 113, 94, 1313, 3402, 12, 3914, 0, 2, 3987, 47, 0, 135, 0, 1178, 3638, 0, 127, 15, 96, 4096, 0, 3750, 4041, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3474, 1263, 1668, 0, 2950, 0, 3859, 4089, 4067, 0, 4088, 0, 0, 0, 247, 3, 9, 0, 4020, 0, 65, 0, 1145, 992, 0, 0, 2613, 33, 520, 421, 1398, 2044, 0, 0, 3703, 0, 0, 1109, 0, 1311, 51, 3782, 51, 4, 0, 0, 4089, 0, 0, 31, 1, 418, 1633, 0, 2, 0, 0, 4095, 29, 1573, 4046, 608, 31, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3311, 732, 4096, 388, 0, 27, 4064, 3882, 7, 2, 1748, 52, 0, 79, 267, 0, 26, 7, 136, 117, 2, 3910, 0, 2938, 203, 129, 1, 158, 118, 57, 4015, 6, 3, 19, 4052, 31, 0, 13, 4, 13, 3889, 1894, 0, 4014, 0, 0, 4078, 908, 35, 0, 43, 623, 4061, 2, 3, 133, 9, 4092, 199, 2304, 4083, 273, 276, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3895, 342, 4095, 0, 277, 0, 1970, 4096, 4095, 0, 3901, 0, 556, 11, 0, 20, 2, 0, 0, 0, 1, 17, 31, 3529, 76, 0, 1, 1115, 1475, 1331, 3977, 18, 748, 23, 3719, 192, 0, 602, 0, 0, 3400, 1936, 0, 2388, 0, 0, 4052, 262, 0, 2, 0, 4020, 1, 0, 1436, 0, 0, 3717, 8, 62, 4095, 0, 0, 42] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 601, 1359, 0, 73, 0, 1480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 328, 2, 8, 0, 14, 10, 4094, 97, 0, 0, 0, 0, 0, 0, 55, 0, 0, 9, 1, 1109, 0, 3445, 1923, 0, 0, 0, 0, 0, 0, 0, 18, 102, 0, 383, 31, 4063, 3985, 8, 390, 459, 0, 0, 0, 0, 0, 0, 2, 1, 4084, 1317, 420, 3118, 0, 312, 895, 411, 0, 0, 0, 0, 0, 0, 3637, 3989, 10, 1569, 3, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 30, 2723, 194, 511, 4052, 0, 601, 391, 0, 0, 0, 0, 0, 0, 0, 2191, 0, 2349, 3616, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7609, 4571, 6542, 9439, 10560, 9479, 9097, 8239] avg=8192.0000 max=10560 max_rank=4 min=4571 min_rank=1 imbalance_ratio=1.289062 max_load_reduction=54976 max_load_reduction_pct=0.838867 moved_to_echo=37375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4081, 0, 1307, 37, 1, 18, 2790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3603, 1, 0, 0, 16, 2, 3981, 568, 0, 0, 0, 0, 0, 0, 2434, 4, 13, 21, 0, 1165, 13, 3426, 1850, 0, 0, 0, 0, 0, 0, 0, 10, 570, 514, 1601, 6, 376, 3999, 0, 458, 66, 0, 0, 0, 0, 0, 0, 47, 8, 4092, 113, 0, 3703, 0, 37, 1053, 396, 0, 0, 0, 0, 0, 0, 530, 4088, 49, 159, 0, 1, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 317, 3810, 0, 0, 0, 3528, 376, 0, 0, 0, 0, 0, 0, 0, 2577, 0, 3641, 3966, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8175, 8926, 7600, 9449, 4880, 8044, 10228] avg=8192.0000 max=10228 max_rank=7 min=4880 min_rank=5 imbalance_ratio=1.248535 max_load_reduction=55308 max_load_reduction_pct=0.843933 moved_to_echo=32601 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3406, 5, 1345, 447, 0, 2, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 83, 0, 0, 52, 1, 4, 3956, 553, 0, 0, 0, 0, 0, 0, 202, 0, 0, 13, 0, 4095, 0, 4061, 1905, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 6, 315, 4011, 4006, 3, 459, 147, 0, 0, 0, 0, 0, 0, 8, 0, 4090, 2, 0, 201, 113, 94, 1055, 407, 0, 0, 0, 0, 0, 0, 1166, 3402, 12, 3914, 0, 2, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 1178, 3638, 0, 127, 15, 3434, 388, 0, 0, 0, 0, 0, 0, 96, 2582, 0, 3750, 4041, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7670, 4674, 10276, 9018, 5970, 8543, 8915, 10470] avg=8192.0000 max=10470 max_rank=7 min=4674 min_rank=1 imbalance_ratio=1.278076 max_load_reduction=55066 max_load_reduction_pct=0.840240 moved_to_echo=33898 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3261, 1, 1357, 0, 48, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2281, 5, 0, 6, 66, 196, 2619, 2716, 2010, 24, 0, 0, 0, 0, 0, 0, 157, 7, 0, 548, 10, 3445, 0, 2382, 1922, 0, 0, 0, 0, 0, 0, 0, 20, 12, 1224, 10, 47, 4002, 3916, 1215, 459, 437, 0, 0, 0, 0, 0, 0, 85, 0, 4047, 3776, 0, 4067, 0, 34, 1055, 411, 0, 0, 0, 0, 0, 0, 3463, 40, 12, 1633, 161, 605, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4064, 36, 0, 2, 143, 391, 0, 0, 0, 0, 0, 0, 0, 2582, 15, 10, 211, 0, 175, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4699, 9923, 8471, 11342, 13475, 9971, 4658, 2997] avg=8192.0000 max=13475 max_rank=4 min=2997 min_rank=7 imbalance_ratio=1.644897 max_load_reduction=52061 max_load_reduction_pct=0.794388 moved_to_echo=31101 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3311, 732, 1363, 388, 0, 27, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 1748, 52, 0, 79, 267, 0, 3882, 566, 0, 0, 0, 0, 0, 0, 26, 7, 136, 117, 2, 3910, 0, 2938, 1929, 0, 0, 0, 0, 0, 0, 0, 203, 129, 1, 158, 118, 57, 4015, 6, 459, 435, 0, 0, 0, 0, 0, 0, 3, 19, 4052, 31, 0, 13, 4, 13, 1054, 412, 0, 0, 0, 0, 0, 0, 3454, 1894, 0, 4014, 0, 0, 0, 908, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 43, 623, 4061, 2, 3, 133, 3512, 392, 0, 0, 0, 0, 0, 0, 9, 2579, 199, 2304, 4083, 273, 276, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9885, 6603, 9065, 5581, 5601, 10270, 8804, 9727] avg=8192.0000 max=10270 max_rank=5 min=5581 min_rank=3 imbalance_ratio=1.253662 max_load_reduction=55266 max_load_reduction_pct=0.843292 moved_to_echo=34402 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4095, 2027, 1363, 11, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 2, 754, 13, 0, 5, 0, 8, 4075, 397, 0, 0, 0, 0, 0, 0, 847, 1, 1, 0, 0, 2943, 4, 4003, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 148, 1049, 4067, 8, 446, 458, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 292, 0, 626, 2, 17, 1024, 412, 0, 0, 0, 0, 0, 0, 3634, 1723, 0, 4036, 0, 0, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3984, 21, 0, 4049, 3, 2458, 392, 0, 0, 0, 0, 0, 0, 0, 2506, 0, 1863, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7505, 5285, 9728, 6265, 6471, 10909, 10908, 8465] avg=8192.0000 max=10909 max_rank=5 min=5285 min_rank=1 imbalance_ratio=1.331665 max_load_reduction=54627 max_load_reduction_pct=0.833542 moved_to_echo=36753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3474, 1263, 555, 0, 2950, 0, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 4088, 0, 0, 0, 247, 3, 4089, 569, 0, 0, 0, 0, 0, 0, 9, 0, 4020, 0, 65, 0, 1145, 992, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 33, 520, 421, 1398, 2044, 459, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3703, 0, 0, 1109, 0, 1311, 1055, 168, 0, 0, 0, 0, 0, 0, 46, 3782, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 1, 418, 1633, 0, 2, 0, 3520, 160, 0, 0, 0, 0, 0, 0, 0, 2581, 29, 1573, 4046, 608, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12101, 13063, 7016, 7493, 7346, 3883, 5765, 8869] avg=8192.0000 max=13063 max_rank=1 min=3883 min_rank=5 imbalance_ratio=1.594604 max_load_reduction=52473 max_load_reduction_pct=0.800674 moved_to_echo=25863 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3895, 342, 1349, 0, 277, 0, 1970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3901, 0, 556, 11, 0, 20, 4096, 563, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 17, 31, 3529, 1932, 0, 0, 0, 0, 0, 0, 0, 76, 0, 1, 1115, 1475, 1331, 3977, 18, 420, 385, 0, 0, 0, 0, 0, 0, 748, 23, 3719, 192, 0, 602, 0, 0, 963, 417, 0, 0, 0, 0, 0, 0, 3015, 1936, 0, 2388, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4020, 1, 0, 1436, 0, 3489, 397, 0, 0, 0, 0, 0, 0, 0, 2334, 8, 62, 4095, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7833, 13242, 5512, 8798, 6664, 7601, 9345, 6541] avg=8192.0000 max=13242 max_rank=1 min=5512 min_rank=2 imbalance_ratio=1.616455 max_load_reduction=52294 max_load_reduction_pct=0.797943 moved_to_echo=30151 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[19072, 20864, 256, 17792, 256, 640, 0, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[896, 128, 32000, 5760, 512, 13440, 128, 1920, 8192, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3840, 128, 4224, 768, 128, 16768, 1280, 24832, 14208, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 19968, 256, 15616, 28160, 896, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 256, 128, 13312, 17536, 640, 9728, 256, 20736, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 896, 4480, 3456, 2688, 15360, 29440, 3328, 3584, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[10624, 128, 14592, 128, 640, 384, 3200, 2816, 30208, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[29696, 4992, 10112, 896, 3456, 128, 16640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[168, 0, 4087, 7, 1339, 95, 0, 3563, 5, 213, 3420, 0, 33, 18, 0, 3062, 71, 0, 40, 86, 0, 0, 370, 13, 338, 453, 3510, 4080, 0, 0, 1, 53, 0, 12, 0, 4096, 4096, 35, 1434, 10, 209, 5, 54, 3889, 4095, 0, 3959, 0, 1543, 8, 0, 3037, 179, 0, 3991, 1791, 3981, 0, 0, 0, 0, 52, 4024, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[325, 3, 3971, 4018, 0, 76, 1310, 893, 0, 61, 4079, 0, 891, 0, 21, 2285, 1, 3989, 3593, 431, 22, 171, 132, 3, 0, 172, 856, 4094, 5, 19, 0, 1, 1077, 0, 0, 1285, 0, 0, 1652, 1, 3694, 5, 1408, 3756, 155, 0, 2273, 24, 2303, 0, 452, 1, 213, 6, 4032, 4037, 3557, 0, 0, 19, 0, 0, 4095, 69] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[353, 0, 3865, 983, 1937, 0, 0, 2688, 248, 682, 3570, 30, 6, 7, 0, 4075, 3, 950, 497, 24, 2, 0, 0, 0, 860, 0, 97, 4092, 0, 0, 61, 2, 0, 0, 0, 4058, 105, 0, 3943, 8, 4091, 0, 997, 4096, 2835, 0, 1235, 0, 3949, 0, 0, 0, 3248, 0, 3844, 58, 4047, 13, 0, 586, 239, 3, 3114, 35] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[29, 0, 4010, 2, 2427, 3, 5, 24, 1788, 0, 4046, 0, 0, 26, 915, 2815, 4000, 0, 4039, 22, 1, 41, 0, 24, 0, 9, 0, 3987, 207, 1, 0, 0, 11, 4, 0, 3128, 272, 183, 3979, 0, 4084, 8, 70, 4092, 4096, 45, 0, 0, 1072, 10, 14, 31, 2048, 1723, 314, 0, 4087, 0, 0, 0, 1303, 3959, 2504, 78] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[2979, 0, 4033, 3, 0, 8, 263, 3748, 0, 5, 465, 0, 2681, 0, 0, 2708, 0, 4080, 0, 54, 0, 22, 0, 14, 0, 90, 7, 4041, 0, 2, 0, 0, 0, 3787, 26, 4096, 0, 0, 137, 0, 3861, 224, 10, 4093, 205, 0, 3111, 13, 735, 1008, 54, 1466, 3732, 37, 2108, 3023, 4096, 0, 3, 204, 208, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1192, 0, 4092, 48, 0, 0, 0, 121, 2377, 3, 114, 0, 12, 0, 0, 523, 4054, 4066, 0, 2988, 0, 182, 0, 25, 0, 1, 27, 4093, 0, 1, 0, 121, 1735, 2198, 1, 3910, 0, 0, 4091, 55, 3738, 905, 79, 3620, 365, 0, 1515, 0, 2214, 0, 0, 603, 0, 0, 3799, 0, 4096, 3934, 7, 0, 0, 0, 4016, 615] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 4096, 0, 28, 1, 38, 1, 3586, 0, 0, 0, 0, 0, 0, 2145, 18, 4049, 144, 2, 0, 0, 986, 7, 617, 424, 0, 3520, 0, 0, 1, 3353, 0, 555, 0, 4096, 7, 0, 482, 2, 4093, 1413, 5, 4096, 2929, 0, 3996, 0, 0, 2, 0, 4055, 3, 0, 3942, 2, 4026, 571, 0, 0, 4090, 42, 3706, 403] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 487, 0, 2, 0, 3828, 861, 0, 172, 0, 14, 0, 0, 0, 11, 3923, 680, 136, 0, 2, 229, 4, 40, 13, 0, 4091, 0, 0, 77, 8, 0, 3843, 1, 4094, 3686, 2, 4095, 102, 3821, 41, 217, 4092, 4018, 0, 75, 0, 3, 0, 0, 1872, 72, 84, 3405, 340, 3998, 1634, 0, 91, 1456, 233, 3724, 1863] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[168, 0, 4087, 7, 1339, 95, 0, 3563, 2, 0, 0, 0, 0, 0, 0, 0, 5, 213, 3420, 0, 33, 18, 0, 3062, 704, 773, 691, 16, 0, 0, 0, 0, 71, 0, 40, 86, 0, 0, 370, 13, 1462, 1180, 0, 0, 0, 0, 0, 0, 338, 453, 3510, 4080, 0, 0, 1, 53, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3405, 4096, 35, 1434, 10, 0, 0, 0, 0, 0, 0, 0, 0, 191, 5, 54, 0, 4095, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1543, 8, 0, 3037, 179, 0, 3218, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 1815, 0, 0, 0, 0, 52, 4024, 11, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9261, 8935, 3222, 11144, 8992, 8304, 9776, 5902] avg=8192.0000 max=11144 max_rank=3 min=3222 min_rank=2 imbalance_ratio=1.360352 max_load_reduction=54392 max_load_reduction_pct=0.829956 moved_to_echo=32974 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[353, 0, 3865, 983, 1937, 0, 0, 2688, 42, 0, 0, 0, 0, 0, 0, 0, 248, 682, 3570, 30, 6, 7, 0, 4075, 715, 744, 684, 319, 0, 0, 0, 0, 3, 950, 497, 24, 2, 0, 0, 0, 1486, 1242, 0, 0, 0, 0, 0, 0, 860, 0, 97, 4092, 0, 0, 61, 2, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3374, 105, 0, 3943, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3730, 0, 997, 0, 2835, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 3248, 0, 3100, 58, 0, 0, 0, 0, 0, 0, 0, 0, 1846, 13, 0, 586, 239, 3, 3114, 35, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9868, 11080, 4204, 7966, 7430, 8797, 10355, 5836] avg=8192.0000 max=11080 max_rank=1 min=4204 min_rank=2 imbalance_ratio=1.352539 max_load_reduction=54456 max_load_reduction_pct=0.830933 moved_to_echo=32418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[325, 3, 3971, 4018, 0, 76, 1310, 893, 38, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4079, 0, 891, 0, 21, 2285, 629, 781, 216, 288, 0, 0, 0, 0, 1, 3989, 3593, 431, 22, 171, 132, 3, 1306, 1138, 0, 0, 0, 0, 0, 0, 0, 172, 856, 4094, 5, 19, 0, 1, 2618, 0, 0, 0, 0, 0, 0, 0, 1077, 0, 0, 1069, 0, 0, 1652, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3368, 5, 1408, 0, 155, 0, 2273, 24, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 0, 452, 1, 213, 6, 3251, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 1622, 0, 0, 19, 0, 0, 4095, 69, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10634, 9251, 10786, 7765, 3799, 7233, 10263, 5805] avg=8192.0000 max=10786 max_rank=2 min=3799 min_rank=4 imbalance_ratio=1.316650 max_load_reduction=54750 max_load_reduction_pct=0.835419 moved_to_echo=27100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2979, 0, 4033, 3, 0, 8, 263, 3748, 40, 0, 0, 0, 0, 0, 0, 0, 0, 5, 465, 0, 2681, 0, 0, 2708, 724, 408, 691, 301, 0, 0, 0, 0, 0, 4080, 0, 54, 0, 22, 0, 14, 1504, 1241, 0, 0, 0, 0, 0, 0, 0, 90, 7, 4041, 0, 2, 0, 0, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 3787, 26, 3405, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3520, 224, 10, 0, 205, 0, 3111, 13, 0, 0, 0, 0, 0, 0, 0, 0, 735, 1008, 54, 1466, 3732, 37, 1700, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 1868, 0, 3, 204, 208, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11074, 7983, 6915, 6992, 7355, 7083, 11755, 6379] avg=8192.0000 max=11755 max_rank=6 min=6379 min_rank=7 imbalance_ratio=1.434937 max_load_reduction=53781 max_load_reduction_pct=0.820633 moved_to_echo=32572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 4010, 2, 2427, 3, 5, 24, 42, 0, 0, 0, 0, 0, 0, 0, 1788, 0, 4046, 0, 0, 26, 915, 2815, 722, 60, 527, 318, 0, 0, 0, 0, 4000, 0, 4039, 22, 1, 41, 0, 24, 1501, 1241, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3987, 207, 1, 0, 0, 2851, 0, 0, 0, 0, 0, 0, 0, 11, 4, 0, 2601, 272, 183, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3724, 8, 70, 0, 4096, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 10, 14, 31, 2048, 1723, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1864, 0, 0, 0, 1303, 3959, 2504, 78, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6542, 11217, 10869, 7055, 7050, 7943, 5152, 9708] avg=8192.0000 max=11217 max_rank=1 min=5152 min_rank=6 imbalance_ratio=1.369263 max_load_reduction=54319 max_load_reduction_pct=0.828842 moved_to_echo=29853 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1192, 0, 4092, 48, 0, 0, 0, 121, 38, 0, 0, 0, 0, 0, 0, 0, 2377, 3, 114, 0, 12, 0, 0, 523, 724, 735, 659, 291, 0, 0, 0, 0, 4054, 4066, 0, 2988, 0, 182, 0, 25, 1504, 1098, 0, 0, 0, 0, 0, 0, 0, 1, 27, 4093, 0, 1, 0, 121, 2522, 0, 0, 0, 0, 0, 0, 0, 1735, 2198, 1, 3251, 0, 0, 4091, 55, 0, 0, 0, 0, 0, 0, 0, 0, 3409, 905, 79, 0, 365, 0, 1515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 0, 0, 603, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1868, 3934, 7, 0, 0, 0, 4016, 615, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5491, 5438, 13917, 6765, 11331, 6273, 5881, 10440] avg=8192.0000 max=13917 max_rank=2 min=5438 min_rank=1 imbalance_ratio=1.698853 max_load_reduction=51619 max_load_reduction_pct=0.787643 moved_to_echo=33925 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 0, 28, 1, 38, 1, 42, 0, 0, 0, 0, 0, 0, 0, 3586, 0, 0, 0, 0, 0, 0, 2145, 712, 763, 691, 319, 0, 0, 0, 0, 18, 4049, 144, 2, 0, 0, 986, 7, 1478, 1242, 0, 0, 0, 0, 0, 0, 617, 424, 0, 3520, 0, 0, 1, 3353, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 555, 0, 3405, 7, 0, 482, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3732, 1413, 5, 0, 2929, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4055, 3, 0, 3179, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 571, 0, 0, 4090, 42, 3706, 403, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4210, 8216, 7926, 10769, 4451, 12075, 7241, 10648] avg=8192.0000 max=12075 max_rank=5 min=4210 min_rank=0 imbalance_ratio=1.473999 max_load_reduction=53461 max_load_reduction_pct=0.815750 moved_to_echo=34415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 487, 0, 2, 0, 3828, 43, 0, 0, 0, 0, 0, 0, 0, 861, 0, 172, 0, 14, 0, 0, 0, 710, 663, 695, 301, 0, 0, 0, 0, 11, 3923, 680, 136, 0, 2, 229, 4, 1471, 1241, 0, 0, 0, 0, 0, 0, 40, 13, 0, 4091, 0, 0, 77, 8, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 1, 3399, 3686, 2, 4095, 102, 0, 0, 0, 0, 0, 0, 0, 0, 3477, 41, 217, 0, 4018, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1872, 72, 84, 2742, 340, 0, 0, 0, 0, 0, 0, 0, 0, 1817, 1634, 0, 91, 1456, 233, 3724, 1863, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8456, 3416, 7697, 7080, 15128, 7828, 5113, 10818] avg=8192.0000 max=15128 max_rank=4 min=3416 min_rank=1 imbalance_ratio=1.846680 max_load_reduction=50408 max_load_reduction_pct=0.769165 moved_to_echo=38887 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8960, 1024, 15872, 128, 3712, 128, 1024, 17664, 5760, 4992, 4864, 2176, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11904, 1152, 640, 11136, 9600, 1920, 20608, 9344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2944, 10496, 128, 23936, 8192, 256, 19840, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 21120, 9088, 3840, 128, 512, 1792, 128, 11776, 9728, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[25216, 2688, 2944, 0, 18816, 128, 16256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[14592, 6272, 128, 1024, 7296, 4352, 29312, 3200, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5120, 128, 32256, 5632, 5760, 256, 1664, 14976, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1920, 1280, 4608, 32000, 256, 128, 256, 3584, 22144, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4090, 1, 4096, 0, 4024, 0, 0, 2735, 11, 4094, 0, 3919, 200, 21, 0, 570, 3221, 0, 0, 133, 48, 0, 0, 132, 0, 4078, 192, 0, 3914, 4094, 0, 0, 3908, 1352, 403, 2, 0, 1718, 0, 352, 1165, 3393, 65, 51, 28, 0, 4, 52, 0, 4013, 94, 0, 73, 0, 367, 538, 3804, 0, 614, 1, 2, 0, 0, 3964] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[2914, 1, 3296, 0, 4031, 0, 0, 2656, 1, 1142, 0, 4094, 2939, 849, 0, 20, 1063, 0, 0, 3, 1918, 559, 0, 4028, 0, 3947, 0, 0, 3, 4069, 813, 0, 4096, 18, 2465, 0, 0, 0, 0, 2775, 3610, 0, 0, 3418, 0, 157, 207, 3, 0, 3971, 1, 0, 731, 3, 0, 92, 2994, 2, 552, 2026, 2, 0, 67, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3895, 3009, 74, 2, 536, 0, 34, 3982, 15, 4032, 0, 1990, 3983, 0, 21, 0, 4096, 0, 47, 4087, 930, 0, 0, 0, 827, 2110, 0, 0, 0, 1649, 54, 679, 3447, 2468, 25, 0, 1823, 10, 0, 156, 3046, 0, 2106, 5, 7, 23, 0, 4031, 0, 3074, 0, 0, 3149, 6, 1, 0, 4096, 350, 399, 0, 1184, 0, 0, 78] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[39, 2, 3, 0, 4014, 0, 0, 4081, 1, 4095, 0, 3812, 264, 0, 166, 26, 1403, 0, 0, 4, 0, 27, 0, 650, 0, 4096, 0, 0, 61, 4067, 72, 0, 4096, 235, 0, 449, 0, 3822, 0, 2594, 3569, 28, 0, 2002, 1806, 54, 0, 4096, 0, 4053, 0, 951, 1, 36, 2677, 0, 3359, 1, 0, 6, 66, 0, 959, 3793] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[523, 1, 785, 435, 3551, 0, 0, 96, 0, 4066, 0, 4094, 95, 0, 0, 7, 1366, 62, 1, 0, 4096, 88, 0, 0, 642, 4032, 61, 25, 3925, 2223, 1, 0, 3873, 2, 1, 0, 255, 3485, 0, 0, 1, 3084, 33, 2280, 9, 0, 201, 0, 3070, 4056, 0, 5, 11, 0, 802, 206, 3583, 3, 2464, 3695, 2003, 0, 0, 2239] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3931, 0, 0, 0, 279, 0, 0, 4092, 1, 4095, 0, 3928, 4091, 0, 0, 28, 3, 1, 0, 0, 0, 9, 0, 0, 2075, 4096, 0, 0, 4090, 3655, 0, 0, 89, 1042, 446, 0, 0, 4001, 0, 178, 5, 2123, 378, 3364, 0, 0, 133, 401, 0, 4090, 675, 443, 3, 1552, 4053, 0, 3959, 0, 0, 2, 11, 0, 120, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3717, 0, 3712, 1, 4096, 0, 2, 3847, 0, 162, 294, 4096, 0, 9, 0, 0, 930, 5, 0, 0, 1380, 0, 0, 0, 6, 4095, 0, 1, 1911, 4092, 3165, 20, 4076, 338, 0, 0, 0, 4092, 4, 0, 441, 79, 0, 2357, 1, 0, 48, 2345, 238, 4095, 0, 0, 0, 3, 2405, 4, 4090, 0, 308, 27, 710, 5, 233, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3917, 0, 4095, 101, 4096, 150, 1, 4095, 0, 3318, 479, 4096, 0, 55, 0, 0, 1319, 1, 0, 0, 42, 3, 0, 0, 5, 4004, 6, 1, 296, 4095, 4, 0, 4005, 3, 0, 0, 0, 4060, 3, 0, 0, 6, 6, 142, 768, 0, 6, 1045, 1022, 4029, 79, 339, 0, 3, 3883, 3494, 4007, 302, 2, 0, 4, 53, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2914, 1, 3296, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1142, 0, 3598, 2939, 849, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 0, 0, 3, 1918, 559, 0, 4028, 2656, 1075, 0, 0, 0, 0, 0, 0, 0, 1911, 0, 0, 3, 4069, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 18, 2465, 0, 0, 0, 0, 2775, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 0, 0, 3418, 0, 157, 207, 3, 961, 496, 229, 0, 0, 0, 0, 0, 0, 3971, 1, 0, 731, 3, 0, 92, 313, 93, 59, 0, 0, 0, 0, 0, 2706, 2, 552, 2026, 2, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9929, 8549, 11302, 6796, 9261, 9081, 5263, 5355] avg=8192.0000 max=11302 max_rank=2 min=5263 min_rank=6 imbalance_ratio=1.379639 max_load_reduction=54234 max_load_reduction_pct=0.827545 moved_to_echo=28960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4090, 1, 4096, 0, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4094, 0, 3444, 200, 21, 0, 570, 0, 0, 0, 0, 0, 0, 0, 0, 3221, 0, 0, 133, 48, 0, 0, 132, 2735, 1110, 0, 0, 0, 0, 0, 0, 0, 1975, 192, 0, 3914, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3819, 1352, 403, 2, 0, 1718, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 3393, 65, 51, 28, 0, 4, 52, 993, 475, 291, 0, 0, 0, 0, 0, 0, 4013, 94, 0, 73, 0, 367, 538, 313, 89, 75, 0, 0, 0, 0, 0, 3438, 0, 614, 1, 2, 0, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11898, 8340, 7379, 10175, 7646, 6517, 5562, 8019] avg=8192.0000 max=11898 max_rank=0 min=5562 min_rank=6 imbalance_ratio=1.452393 max_load_reduction=53638 max_load_reduction_pct=0.818451 moved_to_echo=27744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[523, 1, 785, 435, 3275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 3598, 95, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1366, 62, 1, 0, 4096, 88, 0, 0, 96, 1098, 0, 0, 0, 0, 0, 0, 642, 1953, 61, 25, 3925, 2223, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 2, 1, 0, 255, 3485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3084, 33, 2280, 9, 0, 201, 0, 981, 496, 274, 0, 0, 0, 0, 0, 3070, 4056, 0, 5, 11, 0, 802, 206, 276, 88, 71, 0, 0, 0, 0, 0, 3238, 3, 2464, 3695, 2003, 0, 0, 2239, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5019, 7766, 6807, 8830, 7528, 7359, 8585, 13642] avg=8192.0000 max=13642 max_rank=7 min=5019 min_rank=0 imbalance_ratio=1.665283 max_load_reduction=51894 max_load_reduction_pct=0.791840 moved_to_echo=37114 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3895, 3009, 74, 2, 495, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4032, 0, 1749, 3983, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 47, 4087, 930, 0, 0, 0, 3982, 574, 0, 0, 0, 0, 0, 0, 827, 1023, 0, 0, 0, 1649, 54, 679, 0, 0, 0, 0, 0, 0, 0, 0, 3369, 2468, 25, 0, 1823, 10, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 3046, 0, 2106, 5, 7, 23, 0, 4031, 513, 241, 314, 0, 0, 0, 0, 0, 0, 3074, 0, 0, 3149, 6, 1, 0, 41, 78, 81, 0, 0, 0, 0, 0, 3701, 350, 399, 0, 1184, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7509, 9800, 13716, 4232, 7851, 10286, 6430, 5712] avg=8192.0000 max=13716 max_rank=2 min=4232 min_rank=3 imbalance_ratio=1.674316 max_load_reduction=51820 max_load_reduction_pct=0.790710 moved_to_echo=30279 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3717, 0, 3712, 1, 3777, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 294, 3600, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 5, 0, 0, 1380, 0, 0, 0, 3847, 1115, 0, 0, 0, 0, 0, 0, 6, 1983, 0, 1, 1911, 4092, 3165, 20, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 338, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 441, 79, 0, 2357, 1, 0, 48, 2345, 997, 496, 313, 0, 0, 0, 0, 0, 238, 4095, 0, 0, 0, 3, 2405, 4, 319, 93, 81, 0, 0, 0, 0, 0, 3696, 0, 308, 27, 710, 5, 233, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11209, 4065, 7277, 11178, 8417, 7077, 7238, 9075] avg=8192.0000 max=11209 max_rank=0 min=4065 min_rank=1 imbalance_ratio=1.368286 max_load_reduction=54327 max_load_reduction_pct=0.828964 moved_to_echo=31807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3917, 0, 4095, 101, 3777, 150, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3318, 479, 3600, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1319, 1, 0, 0, 42, 3, 0, 0, 4095, 1090, 0, 0, 0, 0, 0, 0, 5, 1939, 6, 1, 296, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 3, 0, 0, 0, 4060, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 142, 768, 0, 6, 1045, 975, 496, 307, 0, 0, 0, 0, 0, 1022, 4029, 79, 339, 0, 3, 3883, 3494, 319, 91, 79, 0, 0, 0, 0, 0, 3621, 302, 2, 0, 4, 53, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12041, 7452, 6550, 6346, 7980, 3751, 13338, 8078] avg=8192.0000 max=13338 max_rank=6 min=3751 min_rank=5 imbalance_ratio=1.628174 max_load_reduction=52198 max_load_reduction_pct=0.796478 moved_to_echo=33147 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[39, 2, 3, 0, 3698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 3345, 264, 0, 166, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1403, 0, 0, 4, 0, 27, 0, 650, 4081, 1120, 0, 0, 0, 0, 0, 0, 0, 1975, 0, 0, 61, 4067, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 235, 0, 449, 0, 3822, 0, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 28, 0, 2002, 1806, 54, 0, 4096, 1001, 467, 262, 0, 0, 0, 0, 0, 0, 4053, 0, 951, 1, 36, 2677, 0, 316, 97, 72, 0, 0, 0, 0, 0, 3025, 1, 0, 6, 66, 0, 959, 3793, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3742, 7897, 7285, 6175, 11099, 13285, 8203, 7850] avg=8192.0000 max=13285 max_rank=5 min=3742 min_rank=0 imbalance_ratio=1.621704 max_load_reduction=52251 max_load_reduction_pct=0.797287 moved_to_echo=40437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3931, 0, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 3452, 4091, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 9, 0, 0, 4092, 1115, 0, 0, 0, 0, 0, 0, 2075, 1984, 0, 0, 4090, 3655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1042, 446, 0, 0, 4001, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2123, 378, 3364, 0, 0, 133, 401, 997, 476, 303, 0, 0, 0, 0, 0, 0, 4090, 675, 443, 3, 1552, 4053, 0, 21, 2, 78, 0, 0, 0, 0, 0, 3578, 0, 0, 2, 11, 0, 120, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4189, 11667, 5220, 11804, 5754, 8180, 10917, 7805] avg=8192.0000 max=11804 max_rank=3 min=4189 min_rank=0 imbalance_ratio=1.440918 max_load_reduction=53732 max_load_reduction_pct=0.819885 moved_to_echo=32656 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23040, 3072, 16128, 640, 22784, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[11904, 8832, 2688, 13696, 2688, 256, 640, 12032, 7424, 3712, 2304, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4352, 31488, 896, 1792, 3968, 1664, 14208, 4352, 1920, 640, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3584, 14848, 384, 128, 14208, 28032, 4224, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[27008, 5504, 3456, 512, 2176, 21248, 128, 6144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 25088, 896, 26496, 11648, 1024, 256, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[13440, 128, 128, 4352, 8448, 768, 0, 4864, 25600, 8320, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[27008, 768, 4352, 5760, 4096, 128, 1408, 22400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[690, 0, 260, 0, 8, 4030, 0, 4, 0, 1, 4084, 0, 8, 0, 3923, 238, 0, 4063, 4071, 12, 2, 11, 11, 4095, 1409, 2710, 14, 1, 0, 9, 205, 4026, 4027, 15, 23, 0, 2, 641, 0, 8, 0, 0, 24, 3683, 0, 1006, 4088, 4, 80, 41, 4081, 0, 3974, 27, 1, 0, 3478, 1994, 0, 1, 7, 0, 3936, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[29, 4096, 0, 0, 0, 1316, 0, 537, 1752, 14, 4029, 161, 0, 3878, 3993, 961, 13, 3947, 174, 0, 27, 4049, 15, 2461, 4094, 6, 1, 0, 86, 0, 4095, 3700, 45, 0, 701, 10, 0, 0, 0, 247, 8, 0, 3778, 0, 0, 435, 2781, 1, 2981, 1520, 4086, 4075, 0, 1, 2, 140, 33, 131, 0, 0, 0, 145, 946, 36] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4042, 0, 28, 0, 24, 48, 0, 0, 10, 0, 4087, 0, 15, 157, 35, 0, 0, 4031, 41, 2602, 1679, 4094, 0, 3207, 0, 3874, 0, 0, 5, 9, 3981, 4094, 2480, 1115, 0, 0, 3981, 102, 0, 50, 1, 1220, 139, 1456, 44, 0, 1206, 2, 3871, 19, 4085, 46, 1607, 3120, 329, 0, 218, 1, 1380, 13, 0, 6, 2982, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4093, 0, 0, 4019, 0, 30, 0, 1064, 99, 3, 4091, 0, 0, 1842, 14, 0, 0, 3, 4095, 5, 5, 4038, 3699, 2277, 806, 32, 0, 0, 0, 351, 4074, 4096, 35, 0, 118, 0, 3845, 3905, 0, 4, 0, 71, 0, 0, 39, 945, 1959, 5, 4094, 0, 4074, 0, 3011, 417, 3, 0, 8, 129, 73, 0, 0, 0, 4043, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 148, 0, 0, 9, 36, 46, 3781, 0, 0, 205, 3662, 0, 114, 3881, 0, 4096, 4077, 2056, 2782, 3769, 2231, 885, 0, 0, 0, 0, 4046, 4096, 0, 0, 1854, 0, 361, 1100, 0, 3, 0, 221, 4032, 1, 0, 1, 3315, 0, 2603, 0, 3886, 1, 2, 42, 0, 0, 0, 0, 1, 0, 4090, 0, 17, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[346, 117, 0, 0, 252, 2874, 0, 3586, 1, 955, 1110, 0, 1, 4090, 0, 1, 1, 4072, 483, 20, 3936, 3679, 4093, 9, 0, 4090, 0, 0, 0, 0, 4041, 4096, 0, 3, 14, 0, 0, 0, 0, 0, 0, 161, 4089, 3600, 0, 2, 77, 0, 1726, 0, 4096, 24, 4094, 2, 2290, 3, 0, 0, 2992, 0, 0, 0, 6, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[3150, 0, 0, 0, 0, 4, 58, 33, 2603, 54, 4088, 0, 0, 3457, 109, 0, 9, 3999, 20, 1227, 1, 469, 4062, 3943, 352, 0, 0, 0, 0, 54, 2804, 4096, 14, 0, 467, 0, 12, 0, 0, 3, 0, 4096, 109, 3555, 2, 1, 4034, 17, 4095, 0, 4096, 16, 4095, 584, 0, 0, 2, 35, 1775, 0, 2300, 0, 1636, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[2454, 341, 0, 0, 128, 4065, 0, 10, 1, 2, 4096, 0, 100, 674, 79, 1698, 905, 3871, 44, 4088, 4020, 0, 1229, 4095, 0, 4094, 8, 0, 0, 0, 0, 3839, 0, 57, 20, 0, 1118, 1227, 0, 0, 0, 4, 437, 4025, 0, 1, 3041, 3, 24, 0, 3811, 5, 2771, 2475, 0, 0, 2, 0, 40, 4, 0, 0, 2542, 4088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4042, 0, 28, 0, 24, 48, 0, 0, 1313, 1176, 0, 0, 0, 0, 0, 0, 10, 0, 4087, 0, 15, 157, 35, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 2602, 1679, 3090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 0, 0, 5, 9, 3981, 2195, 0, 0, 0, 0, 0, 0, 0, 0, 2480, 1115, 0, 0, 3981, 102, 0, 50, 4031, 1352, 0, 0, 0, 0, 0, 0, 1, 1220, 139, 1456, 44, 0, 1206, 2, 586, 717, 0, 0, 0, 0, 0, 0, 3871, 19, 1437, 46, 1607, 3120, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 1, 1380, 13, 0, 6, 2982, 0, 1855, 1472, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6631, 4591, 7412, 10064, 13111, 5371, 10429, 7927] avg=8192.0000 max=13111 max_rank=4 min=4591 min_rank=1 imbalance_ratio=1.600464 max_load_reduction=52425 max_load_reduction_pct=0.799942 moved_to_echo=36838 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 148, 0, 0, 9, 1314, 1118, 0, 0, 0, 0, 0, 0, 36, 46, 3781, 0, 0, 205, 3662, 0, 144, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 4096, 4077, 1552, 2782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2231, 885, 0, 0, 0, 0, 4046, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1854, 0, 361, 1100, 0, 3, 3881, 1590, 0, 0, 0, 0, 0, 0, 0, 221, 4032, 1, 0, 1, 3315, 0, 586, 360, 0, 0, 0, 0, 0, 0, 2603, 0, 1367, 1, 2, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 17, 1, 2179, 1401, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6674, 7874, 12621, 9358, 8789, 8516, 4015, 7689] avg=8192.0000 max=12621 max_rank=2 min=4015 min_rank=6 imbalance_ratio=1.540649 max_load_reduction=52915 max_load_reduction_pct=0.807419 moved_to_echo=29009 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[690, 0, 260, 0, 8, 4030, 0, 4, 1291, 1174, 0, 0, 0, 0, 0, 0, 0, 1, 4084, 0, 8, 0, 3923, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 12, 2, 10, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 2710, 14, 1, 0, 9, 205, 2159, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 15, 23, 0, 2, 641, 0, 8, 4063, 1727, 0, 0, 0, 0, 0, 0, 0, 0, 24, 3683, 0, 1006, 4088, 4, 576, 1, 0, 0, 0, 0, 0, 0, 80, 41, 1436, 0, 3974, 27, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3478, 1994, 0, 1, 7, 0, 3936, 510, 2368, 1471, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7457, 8254, 4106, 6507, 10506, 9382, 5559, 13765] avg=8192.0000 max=13765 max_rank=7 min=4106 min_rank=2 imbalance_ratio=1.680298 max_load_reduction=51771 max_load_reduction_pct=0.789963 moved_to_echo=39212 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[29, 4096, 0, 0, 0, 1316, 0, 537, 1187, 1176, 0, 0, 0, 0, 0, 0, 1752, 14, 4029, 161, 0, 3878, 3993, 961, 288, 0, 0, 0, 0, 0, 0, 0, 13, 0, 174, 0, 27, 3048, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 1, 0, 86, 0, 4095, 1983, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 701, 10, 0, 0, 0, 247, 3947, 1039, 0, 0, 0, 0, 0, 0, 8, 0, 3778, 0, 0, 435, 2781, 1, 530, 713, 0, 0, 0, 0, 0, 0, 2981, 1520, 1437, 4075, 0, 1, 2, 140, 0, 0, 0, 0, 0, 0, 0, 0, 33, 131, 0, 0, 0, 145, 946, 36, 1422, 1473, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8341, 15076, 3277, 10265, 5989, 8246, 10156, 4186] avg=8192.0000 max=15076 max_rank=1 min=3277 min_rank=2 imbalance_ratio=1.840332 max_load_reduction=50460 max_load_reduction_pct=0.769958 moved_to_echo=28577 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 4019, 0, 30, 0, 1064, 1314, 1172, 0, 0, 0, 0, 0, 0, 99, 3, 4091, 0, 0, 1842, 14, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 5, 3047, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 806, 32, 0, 0, 0, 351, 4074, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 118, 0, 3845, 3905, 0, 4, 3, 960, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 39, 945, 1959, 5, 586, 708, 0, 0, 0, 0, 0, 0, 4094, 0, 1434, 0, 3011, 417, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 129, 73, 0, 0, 0, 4043, 22, 1317, 1468, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11692, 6332, 10851, 7459, 8870, 4313, 8959, 7060] avg=8192.0000 max=11692 max_rank=0 min=4313 min_rank=5 imbalance_ratio=1.427246 max_load_reduction=53844 max_load_reduction_pct=0.821594 moved_to_echo=29202 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[346, 117, 0, 0, 252, 2874, 0, 3586, 1314, 1179, 0, 0, 0, 0, 0, 0, 1, 955, 1110, 0, 1, 4090, 0, 1, 258, 0, 0, 0, 0, 0, 0, 0, 1, 0, 483, 20, 3936, 2776, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4041, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 14, 0, 0, 0, 0, 0, 4072, 3, 0, 0, 0, 0, 0, 0, 0, 161, 4089, 3600, 0, 2, 77, 0, 586, 645, 0, 0, 0, 0, 0, 0, 1726, 0, 1441, 24, 4094, 2, 2290, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2992, 0, 0, 0, 6, 504, 6, 1476, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9668, 6416, 11309, 10327, 4092, 9160, 9580, 4984] avg=8192.0000 max=11309 max_rank=2 min=4092 min_rank=4 imbalance_ratio=1.380493 max_load_reduction=54227 max_load_reduction_pct=0.827438 moved_to_echo=27816 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3150, 0, 0, 0, 0, 4, 58, 33, 1314, 1179, 0, 0, 0, 0, 0, 0, 2603, 54, 4088, 0, 0, 3457, 109, 0, 32, 0, 0, 0, 0, 0, 0, 0, 9, 0, 20, 1227, 1, 355, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 0, 0, 0, 54, 2804, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 467, 0, 12, 0, 0, 3, 3999, 1663, 0, 0, 0, 0, 0, 0, 0, 4096, 109, 3555, 2, 1, 4034, 17, 586, 82, 0, 0, 0, 0, 0, 0, 4095, 0, 1441, 16, 4095, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 35, 1775, 0, 2300, 0, 1636, 0, 2280, 1476, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5738, 10343, 5674, 5406, 6158, 12482, 10231, 9504] avg=8192.0000 max=12482 max_rank=5 min=5406 min_rank=3 imbalance_ratio=1.523682 max_load_reduction=53054 max_load_reduction_pct=0.809540 moved_to_echo=38375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2454, 341, 0, 0, 128, 4065, 0, 10, 1236, 1101, 0, 0, 0, 0, 0, 0, 1, 2, 4096, 0, 100, 674, 79, 1698, 0, 0, 0, 0, 0, 0, 0, 0, 905, 0, 44, 4088, 4020, 0, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 8, 0, 0, 0, 0, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 20, 0, 1118, 1227, 0, 0, 3871, 1728, 0, 0, 0, 0, 0, 0, 0, 4, 437, 4025, 0, 1, 3041, 3, 555, 0, 0, 0, 0, 0, 0, 0, 24, 0, 1332, 5, 2771, 2475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 40, 4, 0, 0, 2542, 4088, 2367, 1378, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9335, 6650, 10286, 6150, 8021, 8066, 6607, 10421] avg=8192.0000 max=10421 max_rank=7 min=6150 min_rank=3 imbalance_ratio=1.272095 max_load_reduction=55115 max_load_reduction_pct=0.840988 moved_to_echo=33115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3840, 2304, 6272, 128, 6400, 256, 16128, 5248, 13824, 11648, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 5888, 12672, 16384, 128, 2432, 20608, 128, 4608, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4608, 1152, 29440, 256, 128, 14336, 11904, 2944, 1408, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8960, 15744, 128, 128, 128, 512, 23296, 17280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1152, 0, 8960, 12160, 13824, 13952, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[18944, 4608, 384, 4096, 640, 12416, 128, 5248, 10368, 9344, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[19584, 1664, 11392, 4224, 19584, 6784, 2688, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6656, 1280, 3200, 128, 9344, 7040, 0, 384, 27904, 10112, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2847, 0, 0, 0, 0, 3334, 3850, 4096, 3842, 1, 973, 0, 0, 0, 4081, 3933, 8, 4095, 7, 0, 0, 0, 0, 264, 2, 0, 2, 5, 0, 112, 680, 3812, 20, 0, 0, 1, 1211, 1, 0, 0, 4050, 7, 4087, 20, 0, 0, 3404, 4095, 0, 0, 685, 179, 3706, 3672, 0, 0, 0, 4086, 5, 12, 351, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[5, 0, 4091, 0, 719, 0, 0, 3548, 2859, 4078, 4046, 3635, 661, 37, 0, 0, 4083, 4095, 0, 3178, 2, 0, 0, 0, 0, 0, 1, 1491, 108, 61, 2, 12, 543, 159, 115, 2661, 643, 1, 54, 63, 414, 0, 4029, 0, 1174, 4096, 1, 0, 1574, 6, 0, 0, 4027, 0, 3768, 4096, 219, 0, 0, 827, 317, 37, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 49, 0, 776, 0, 0, 3403, 3189, 3797, 4065, 3591, 112, 2113, 5, 0, 4060, 61, 0, 4087, 4061, 0, 0, 0, 0, 2037, 67, 0, 4092, 1, 0, 4, 165, 189, 4059, 5, 0, 2, 15, 11, 1, 0, 4, 0, 0, 395, 0, 0, 4001, 0, 0, 28, 0, 0, 3291, 2519, 1446, 672, 0, 2661, 2421, 0, 5, 4074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 27, 1275, 0, 6, 3020, 1610, 1941, 3565, 2302, 33, 4, 3084, 0, 4, 4083, 18, 0, 4034, 1177, 0, 0, 0, 1, 466, 0, 1851, 117, 343, 1, 3301, 0, 4088, 1042, 0, 0, 3209, 0, 698, 1, 125, 0, 0, 297, 0, 719, 0, 4018, 2589, 0, 0, 2208, 0, 10, 3151, 3614, 54, 0, 473, 4096, 2867, 6, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 353, 4048, 8, 5, 0, 3685, 0, 113, 4094, 3922, 1, 291, 0, 0, 5, 7, 37, 0, 3330, 0, 37, 73, 132, 0, 327, 0, 121, 1, 0, 3573, 329, 139, 0, 0, 0, 0, 12, 3744, 0, 0, 4096, 1465, 62, 4018, 4088, 0, 0, 4096, 4094, 0, 23, 3039, 0, 1096, 4096, 0, 2899, 3134, 70, 0, 0, 873, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[6, 0, 0, 23, 43, 23, 0, 1782, 27, 4086, 3923, 5, 3313, 728, 0, 5, 1716, 2565, 2, 4096, 0, 1800, 21, 0, 0, 0, 4091, 1494, 4096, 1, 4065, 25, 105, 4088, 325, 0, 0, 240, 4095, 0, 0, 38, 2093, 0, 4088, 25, 6, 12, 4096, 51, 0, 0, 0, 8, 34, 3986, 66, 0, 63, 4096, 0, 0, 0, 85] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 0, 2, 3669, 0, 3887, 4096, 3994, 19, 0, 0, 2, 3426, 1350, 0, 4091, 0, 0, 73, 0, 0, 137, 161, 4, 1605, 0, 155, 0, 332, 4088, 0, 0, 0, 99, 2576, 0, 0, 81, 4060, 389, 4096, 3880, 0, 8, 2692, 871, 0, 22, 0, 0, 0, 4095, 0, 3205, 0, 4096, 0, 0, 184, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4094, 2, 0, 0, 0, 0, 0, 4087, 4096, 399, 4093, 0, 0, 2, 4068, 0, 0, 4096, 454, 0, 315, 0, 4, 0, 164, 1, 4095, 0, 4096, 12, 3800, 2075, 123, 14, 0, 5, 7, 0, 0, 9, 2930, 1, 4, 0, 0, 21, 4021, 241, 0, 2, 4087, 0, 1246, 4095, 0, 3388, 25, 4026, 0, 0, 79, 1259] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2847, 0, 0, 0, 0, 3334, 1229, 1170, 0, 0, 0, 0, 0, 0, 3850, 0, 3355, 1, 973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3933, 8, 2614, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 2, 0, 2, 5, 0, 112, 2926, 0, 0, 0, 0, 0, 0, 0, 680, 3812, 20, 0, 0, 1, 1211, 1, 2005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 7, 4087, 20, 0, 0, 937, 438, 0, 0, 0, 0, 0, 0, 3188, 4095, 0, 0, 685, 179, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 5, 12, 351, 0, 544, 487, 216, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8580, 8179, 10643, 3311, 7730, 9539, 11853, 5701] avg=8192.0000 max=11853 max_rank=6 min=3311 min_rank=3 imbalance_ratio=1.446899 max_load_reduction=53683 max_load_reduction_pct=0.819138 moved_to_echo=34823 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 353, 4048, 8, 5, 0, 3685, 0, 1370, 1169, 0, 0, 0, 0, 0, 0, 113, 0, 3425, 1, 291, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 7, 37, 0, 2126, 0, 37, 73, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 121, 1, 0, 3573, 329, 2925, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 12, 3744, 0, 2236, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1465, 62, 4018, 4088, 0, 0, 762, 490, 0, 0, 0, 0, 0, 0, 3836, 4094, 0, 23, 3039, 0, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2899, 3134, 70, 0, 0, 873, 0, 442, 497, 260, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10638, 3835, 2412, 7276, 6131, 14981, 12088, 8175] avg=8192.0000 max=14981 max_rank=5 min=2412 min_rank=2 imbalance_ratio=1.828735 max_load_reduction=50555 max_load_reduction_pct=0.771408 moved_to_echo=41375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 23, 43, 23, 0, 1782, 1333, 1167, 0, 0, 0, 0, 0, 0, 27, 0, 3426, 5, 3313, 728, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1716, 2565, 2, 2615, 0, 1800, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1494, 4096, 1, 4065, 25, 2919, 0, 0, 0, 0, 0, 0, 0, 105, 4088, 325, 0, 0, 240, 4095, 0, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 38, 2093, 0, 4088, 25, 6, 12, 937, 477, 0, 0, 0, 0, 0, 0, 3836, 51, 0, 0, 0, 8, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 63, 4096, 0, 0, 0, 85, 544, 497, 260, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4377, 7504, 8719, 16691, 11029, 7676, 3929, 5611] avg=8192.0000 max=16691 max_rank=3 min=3929 min_rank=6 imbalance_ratio=2.037476 max_load_reduction=48845 max_load_reduction_pct=0.745316 moved_to_echo=28245 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 27, 1275, 0, 6, 3020, 1610, 1055, 1019, 0, 0, 0, 0, 0, 0, 1941, 0, 2010, 33, 4, 3084, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 18, 0, 2575, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 466, 0, 1851, 117, 343, 1, 3301, 2546, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1042, 0, 0, 3209, 0, 698, 1720, 0, 0, 0, 0, 0, 0, 0, 1, 125, 0, 0, 297, 0, 719, 0, 923, 376, 0, 0, 0, 0, 0, 0, 3763, 2589, 0, 0, 2208, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3614, 54, 0, 473, 4096, 2867, 6, 8, 536, 292, 255, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8012, 7076, 7853, 8626, 10757, 2441, 8570, 12201] avg=8192.0000 max=12201 max_rank=7 min=2441 min_rank=5 imbalance_ratio=1.489380 max_load_reduction=53335 max_load_reduction_pct=0.813828 moved_to_echo=33969 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 49, 0, 776, 0, 0, 3403, 842, 1084, 0, 0, 0, 0, 0, 0, 3189, 0, 3550, 3591, 112, 2113, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 61, 0, 2609, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2037, 67, 0, 4092, 1, 0, 4, 2713, 0, 0, 0, 0, 0, 0, 0, 165, 189, 4059, 5, 0, 2, 15, 11, 1377, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 395, 0, 0, 935, 300, 0, 0, 0, 0, 0, 0, 3747, 0, 0, 28, 0, 0, 3291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1446, 672, 0, 2661, 2421, 0, 5, 4074, 543, 515, 254, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6156, 12560, 10791, 8914, 5823, 1635, 7066, 12591] avg=8192.0000 max=12591 max_rank=7 min=1635 min_rank=5 imbalance_ratio=1.536987 max_load_reduction=52945 max_load_reduction_pct=0.807877 moved_to_echo=27115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 2, 0, 0, 0, 0, 1369, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 3577, 399, 4093, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 2615, 454, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 164, 1, 4095, 0, 4096, 12, 2920, 0, 0, 0, 0, 0, 0, 0, 3800, 2075, 123, 14, 0, 5, 7, 0, 2235, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2930, 1, 4, 0, 0, 21, 937, 491, 0, 0, 0, 0, 0, 0, 3766, 241, 0, 2, 4087, 0, 1246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3388, 25, 4026, 0, 0, 79, 1259, 544, 519, 255, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6632, 8071, 7452, 11292, 8259, 4393, 9342, 10095] avg=8192.0000 max=11292 max_rank=3 min=4393 min_rank=5 imbalance_ratio=1.378418 max_load_reduction=54244 max_load_reduction_pct=0.827698 moved_to_echo=32089 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 4091, 0, 719, 0, 0, 3548, 1371, 1165, 0, 0, 0, 0, 0, 0, 2859, 0, 3527, 3635, 661, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4095, 0, 2025, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1491, 108, 61, 2, 12, 2913, 0, 0, 0, 0, 0, 0, 0, 543, 159, 115, 2661, 643, 1, 54, 63, 2236, 0, 0, 0, 0, 0, 0, 0, 414, 0, 4029, 0, 1174, 4096, 1, 0, 729, 489, 0, 0, 0, 0, 0, 0, 1472, 6, 0, 0, 4027, 0, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 827, 317, 37, 0, 0, 424, 519, 102, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10899, 10719, 10205, 4588, 6475, 10932, 9273, 2445] avg=8192.0000 max=10932 max_rank=5 min=2445 min_rank=7 imbalance_ratio=1.334473 max_load_reduction=54604 max_load_reduction_pct=0.833191 moved_to_echo=29125 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 0, 2, 3669, 1369, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 3577, 3994, 19, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3426, 1350, 0, 2612, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 161, 4, 1605, 0, 155, 0, 2776, 0, 0, 0, 0, 0, 0, 0, 332, 4088, 0, 0, 0, 99, 2576, 0, 2237, 0, 0, 0, 0, 0, 0, 0, 0, 81, 4060, 389, 4096, 3880, 0, 8, 936, 489, 0, 0, 0, 0, 0, 0, 2522, 871, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3205, 0, 4096, 0, 0, 184, 0, 543, 519, 170, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10242, 7592, 7461, 4838, 9332, 13939, 3415, 8717] avg=8192.0000 max=13939 max_rank=5 min=3415 min_rank=6 imbalance_ratio=1.701538 max_load_reduction=51597 max_load_reduction_pct=0.787308 moved_to_echo=35403 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 4352, 18688, 512, 17792, 12544, 768, 128, 7168, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 384, 19328, 1408, 1664, 128, 6784, 17408, 9984, 9088, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[26240, 12032, 0, 128, 14080, 256, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[25600, 12160, 128, 19840, 5760, 1920, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12032, 0, 26496, 11776, 9472, 6016, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5376, 10240, 3328, 20352, 6912, 2944, 1536, 5504, 4224, 3968, 1792, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 3328, 4608, 4992, 14208, 512, 11904, 3840, 22656, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5888, 18560, 5760, 2688, 768, 3584, 11776, 896, 16256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 563, 0, 0, 4072, 220, 3682, 350, 4075, 0, 4, 1, 0, 583, 0, 67, 0, 1, 84, 3499, 3133, 0, 78, 1559, 727, 194, 0, 0, 33, 80, 0, 4095, 0, 0, 761, 0, 0, 0, 341, 66, 193, 2289, 0, 4018, 3, 0, 1, 2852, 341, 2704, 156, 0, 3822, 1935, 2, 0, 3336, 0, 4088, 12, 3237, 0, 4096, 87] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4093, 449, 0, 7, 3847, 0, 0, 3434, 49, 0, 4, 488, 1339, 3303, 34, 6, 0, 0, 0, 201, 0, 0, 0, 2486, 646, 0, 307, 0, 106, 115, 538, 3950, 0, 0, 2695, 4083, 4090, 4030, 0, 2, 6, 0, 2858, 243, 0, 133, 56, 149, 0, 0, 4096, 0, 3743, 0, 20, 0, 12, 3629, 4096, 0, 4085, 0, 2108, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[3761, 2, 24, 108, 49, 4091, 3956, 4096, 59, 363, 0, 3937, 1, 3059, 0, 0, 283, 2135, 0, 4079, 4095, 82, 0, 0, 0, 1, 0, 241, 0, 1, 151, 4002, 0, 4045, 3942, 2776, 2, 6, 0, 404, 0, 0, 0, 0, 18, 0, 2, 0, 0, 326, 0, 6, 3204, 0, 116, 0, 0, 23, 3943, 134, 4008, 0, 4001, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[18, 229, 3768, 0, 0, 3718, 964, 3171, 4096, 68, 0, 6, 0, 3357, 0, 227, 5, 552, 0, 3660, 2859, 0, 902, 3723, 0, 36, 3750, 4, 37, 0, 606, 2854, 0, 2, 3215, 0, 117, 0, 45, 4015, 46, 273, 454, 0, 2405, 0, 25, 8, 0, 115, 0, 4, 4017, 4057, 0, 0, 85, 31, 4096, 0, 3879, 0, 0, 37] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[7, 3, 219, 6, 89, 178, 0, 137, 4096, 4051, 0, 4021, 0, 3648, 0, 90, 3881, 0, 96, 4092, 4096, 0, 0, 4054, 0, 1, 0, 21, 0, 0, 22, 109, 0, 0, 4030, 3966, 3, 340, 1, 0, 0, 0, 0, 3799, 114, 0, 12, 0, 0, 4079, 11, 2956, 3955, 2338, 0, 0, 0, 1, 3938, 0, 2466, 0, 564, 46] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4090, 110, 3985, 0, 4048, 0, 1255, 25, 0, 2, 27, 3972, 45, 4087, 0, 21, 0, 34, 0, 4027, 4090, 0, 0, 3903, 0, 0, 0, 0, 3, 140, 0, 1578, 0, 76, 77, 4087, 165, 2, 26, 24, 0, 0, 463, 3620, 0, 3, 0, 0, 0, 55, 1709, 68, 3965, 2748, 0, 0, 0, 2382, 3874, 0, 4096, 0, 2654, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[2, 1, 1185, 181, 4096, 0, 0, 45, 27, 219, 0, 1, 0, 4034, 0, 87, 1, 0, 0, 4066, 4096, 101, 0, 4027, 0, 1, 0, 4081, 0, 0, 92, 1059, 0, 3876, 0, 4094, 39, 13, 74, 45, 0, 0, 0, 3609, 0, 0, 3, 44, 0, 4021, 2696, 135, 3296, 4086, 0, 0, 0, 4094, 3741, 0, 4045, 0, 2, 221] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 3688, 221, 0, 3479, 3842, 4085, 1806, 4060, 3, 754, 3029, 1, 3069, 0, 111, 0, 0, 0, 3754, 4081, 0, 2, 3359, 71, 0, 0, 0, 5, 4, 1, 4017, 0, 0, 2, 174, 6, 0, 9, 0, 3228, 0, 415, 39, 0, 2, 0, 0, 206, 346, 458, 868, 4094, 4038, 0, 0, 0, 0, 4093, 0, 23, 0, 4093, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2992, 563, 0, 0, 0, 220, 3682, 350, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4, 1, 0, 583, 0, 67, 243, 0, 0, 0, 0, 0, 0, 0, 0, 1, 84, 989, 3133, 0, 78, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 727, 194, 0, 0, 33, 80, 0, 4095, 2510, 1934, 363, 0, 0, 0, 0, 0, 0, 0, 761, 0, 0, 0, 341, 66, 811, 861, 0, 0, 0, 0, 0, 0, 193, 2289, 0, 4018, 3, 0, 1, 2852, 3056, 2138, 0, 0, 0, 0, 0, 0, 341, 2704, 156, 0, 2648, 1935, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3336, 0, 1032, 12, 3237, 0, 4096, 87, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7807, 4973, 5844, 9936, 2840, 14550, 7786, 11800] avg=8192.0000 max=14550 max_rank=5 min=2840 min_rank=4 imbalance_ratio=1.776123 max_load_reduction=50986 max_load_reduction_pct=0.777985 moved_to_echo=36976 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2747, 2, 24, 108, 0, 4091, 3956, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 59, 363, 0, 3937, 1, 3059, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 283, 2135, 0, 1153, 4095, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 241, 0, 1, 151, 4002, 2926, 24, 305, 0, 0, 0, 0, 0, 0, 4045, 3942, 2776, 2, 6, 0, 404, 680, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 2, 0, 2947, 25, 0, 0, 0, 0, 0, 0, 0, 326, 0, 6, 2219, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 996, 134, 4008, 0, 4001, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15024, 7642, 7748, 7651, 12646, 2992, 2667, 9166] avg=8192.0000 max=15024 max_rank=0 min=2667 min_rank=6 imbalance_ratio=1.833984 max_load_reduction=50512 max_load_reduction_pct=0.770752 moved_to_echo=27471 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2989, 449, 0, 7, 0, 0, 0, 3434, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 4, 488, 1339, 3303, 34, 6, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 646, 0, 307, 0, 106, 115, 538, 3950, 144, 1827, 356, 0, 0, 0, 0, 0, 0, 0, 2695, 4083, 4090, 4030, 0, 2, 794, 861, 0, 0, 0, 0, 0, 0, 6, 0, 2858, 243, 0, 133, 56, 149, 3062, 2020, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2593, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3629, 1034, 0, 4085, 0, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6879, 5466, 2543, 7989, 16555, 8527, 6709, 10868] avg=8192.0000 max=16555 max_rank=4 min=2543 min_rank=2 imbalance_ratio=2.020874 max_load_reduction=48981 max_load_reduction_pct=0.747391 moved_to_echo=42659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[11, 229, 3768, 0, 0, 3718, 964, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 68, 0, 6, 0, 3357, 0, 227, 1, 0, 0, 0, 0, 0, 0, 0, 5, 552, 0, 1035, 2859, 0, 902, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3750, 4, 37, 0, 606, 2854, 2625, 0, 382, 0, 0, 0, 0, 0, 0, 2, 3215, 0, 117, 0, 45, 4015, 852, 6, 0, 0, 0, 0, 0, 0, 46, 273, 454, 0, 2405, 0, 25, 8, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 4, 2783, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 31, 1034, 0, 3879, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11861, 7755, 9076, 10294, 8252, 6273, 6959, 5066] avg=8192.0000 max=11861 max_rank=0 min=5066 min_rank=7 imbalance_ratio=1.447876 max_load_reduction=53675 max_load_reduction_pct=0.819016 moved_to_echo=26550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3688, 221, 0, 0, 3842, 4085, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 3, 754, 3029, 1, 3069, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 4081, 0, 2, 3359, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 5, 4, 1, 4017, 2692, 1653, 389, 0, 0, 0, 0, 0, 0, 0, 2, 174, 6, 0, 9, 0, 869, 0, 0, 0, 0, 0, 0, 0, 3228, 0, 415, 39, 0, 2, 0, 0, 3059, 1826, 0, 0, 0, 0, 0, 0, 206, 346, 458, 868, 2836, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 0, 23, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13642, 11027, 8504, 8832, 1060, 8569, 8752, 5150] avg=8192.0000 max=13642 max_rank=0 min=1060 min_rank=4 imbalance_ratio=1.665283 max_load_reduction=51894 max_load_reduction_pct=0.791840 moved_to_echo=23531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[6, 3, 219, 6, 0, 178, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4051, 0, 4021, 0, 3648, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 96, 1157, 4096, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 21, 0, 0, 22, 109, 2935, 42, 376, 0, 0, 0, 0, 0, 0, 0, 4030, 3966, 3, 340, 1, 0, 839, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3799, 114, 0, 12, 0, 2943, 47, 0, 0, 0, 0, 0, 0, 0, 4079, 11, 2956, 2740, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 995, 0, 2466, 0, 564, 46, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[549, 15906, 13284, 3506, 9180, 6915, 12124, 4072] avg=8192.0000 max=15906 max_rank=1 min=549 min_rank=0 imbalance_ratio=1.941650 max_load_reduction=49630 max_load_reduction_pct=0.757294 moved_to_echo=32291 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2987, 110, 3985, 0, 0, 0, 1255, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 27, 3972, 45, 4087, 0, 21, 243, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 1139, 4090, 0, 0, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 140, 0, 1578, 2888, 1924, 377, 0, 0, 0, 0, 0, 0, 76, 77, 4087, 165, 2, 26, 24, 841, 860, 0, 0, 0, 0, 0, 0, 0, 0, 463, 3620, 0, 3, 0, 0, 2896, 2124, 0, 0, 0, 0, 0, 0, 0, 55, 1709, 68, 2747, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2382, 978, 0, 4096, 0, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8362, 8397, 9166, 6910, 6158, 9106, 7327, 10110] avg=8192.0000 max=10110 max_rank=7 min=6158 min_rank=4 imbalance_ratio=1.234131 max_load_reduction=55426 max_load_reduction_pct=0.845734 moved_to_echo=32701 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1185, 181, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 27, 219, 0, 1, 0, 4034, 0, 87, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1146, 4096, 101, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4081, 0, 0, 92, 1059, 2920, 1947, 318, 0, 0, 0, 0, 0, 0, 3876, 0, 4094, 39, 13, 74, 45, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3609, 0, 0, 3, 44, 2799, 2149, 0, 0, 0, 0, 0, 0, 0, 4021, 2696, 135, 2274, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 942, 0, 4045, 0, 2, 221, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1412, 4370, 9371, 10418, 8845, 8604, 13212, 9304] avg=8192.0000 max=13212 max_rank=6 min=1412 min_rank=0 imbalance_ratio=1.612793 max_load_reduction=52324 max_load_reduction_pct=0.798401 moved_to_echo=39965 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11776, 5120, 9472, 384, 0, 12160, 13952, 13184, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 8064, 14848, 19200, 4480, 4480, 512, 4608, 6400, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3584, 2688, 4224, 15360, 2560, 256, 128, 3072, 23936, 10368, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3456, 10240, 8064, 256, 25856, 0, 17536, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 11648, 9216, 4096, 20864, 19328, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4224, 2816, 256, 7808, 26496, 256, 1024, 23168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[16512, 4736, 896, 15488, 1408, 25216, 128, 640, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1536, 256, 4096, 4352, 256, 384, 1536, 21760, 19712, 9472, 2944, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[101, 0, 3974, 165, 51, 0, 567, 30, 4045, 471, 0, 0, 0, 3531, 67, 0, 1, 0, 73, 8, 4096, 0, 11, 2864, 0, 3910, 4057, 0, 34, 0, 0, 0, 902, 1196, 0, 334, 310, 1, 1910, 0, 1337, 4096, 2979, 1, 81, 4096, 1302, 0, 0, 0, 4018, 0, 3760, 2903, 9, 1, 93, 0, 0, 0, 4092, 0, 0, 4059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[914, 0, 6, 4083, 0, 716, 94, 13, 0, 5, 6, 0, 36, 23, 0, 68, 0, 0, 0, 3982, 1633, 0, 2, 3344, 12, 0, 4030, 0, 532, 3959, 2856, 0, 1, 4093, 9, 0, 0, 0, 2831, 0, 0, 4096, 3578, 0, 31, 4038, 4096, 4055, 0, 4092, 0, 0, 3, 7, 313, 3850, 0, 0, 0, 0, 20, 13, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 734, 0, 0, 0, 1, 0, 4, 4065, 44, 3860, 0, 8, 339, 2070, 2711, 0, 4039, 4083, 0, 4055, 2466, 2781, 0, 0, 20, 0, 0, 0, 3142, 17, 852, 0, 3806, 280, 491, 4095, 444, 42, 20, 2857, 4087, 4070, 0, 0, 4, 81, 3590, 0, 154, 51, 204, 139, 0, 8, 1616, 108, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3235, 397, 0, 0, 3916, 0, 3327, 0, 3, 0, 3, 3840, 0, 0, 0, 0, 20, 74, 4096, 0, 4092, 3651, 3, 389, 13, 0, 127, 388, 1262, 0, 0, 3887, 394, 3165, 0, 0, 4095, 0, 18, 4096, 152, 0, 0, 42, 4096, 18, 0, 0, 0, 2787, 4093, 743, 33, 0, 0, 0, 0, 0, 4090, 2, 3836, 1153] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3895, 0, 3560, 77, 0, 0, 0, 0, 2, 0, 0, 4090, 0, 4079, 34, 2, 0, 0, 0, 2571, 0, 0, 1399, 4096, 0, 4096, 4095, 0, 3500, 199, 2122, 0, 0, 0, 6, 0, 6, 0, 4096, 0, 0, 4075, 4096, 3, 104, 0, 4096, 3991, 0, 0, 0, 7, 72, 49, 26, 4096, 0, 0, 0, 0, 0, 1, 2097, 898] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[363, 0, 4096, 3487, 0, 6, 0, 0, 0, 402, 46, 0, 0, 3929, 0, 13, 0, 0, 2, 3771, 1645, 0, 3771, 4094, 0, 3921, 0, 137, 304, 0, 360, 0, 0, 11, 3985, 55, 0, 0, 1089, 2, 34, 1960, 4049, 48, 230, 0, 1923, 3994, 0, 0, 3684, 3850, 2970, 2383, 0, 2839, 0, 150, 0, 1, 143, 2, 0, 1787] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4077, 0, 3739, 79, 0, 0, 3, 0, 1, 0, 3520, 3948, 0, 4096, 0, 303, 0, 0, 38, 259, 4015, 0, 4092, 4096, 0, 4096, 3879, 0, 0, 106, 0, 0, 0, 3849, 627, 61, 51, 0, 4096, 338, 0, 4045, 182, 11, 0, 0, 1, 4087, 0, 0, 0, 5, 4084, 0, 1, 12, 0, 0, 0, 0, 0, 8, 3731, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[3924, 0, 5, 0, 0, 2, 2264, 0, 1, 78, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 3877, 0, 4093, 3354, 907, 0, 3988, 0, 491, 89, 610, 0, 3, 4089, 2600, 0, 0, 0, 2316, 0, 182, 4096, 0, 0, 82, 0, 4096, 353, 0, 4033, 6, 0, 4093, 1469, 2814, 0, 0, 0, 0, 0, 173, 10, 3487, 66] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[914, 0, 6, 4083, 0, 716, 94, 13, 1382, 109, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 36, 23, 0, 68, 1412, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 1633, 0, 2, 550, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4030, 0, 532, 3959, 2856, 0, 0, 50, 0, 0, 0, 0, 0, 0, 1, 4093, 9, 0, 0, 0, 2831, 0, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 31, 4038, 2722, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3, 7, 313, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 13, 0, 4096, 3947, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7317, 1699, 6167, 11439, 8149, 14424, 8265, 8076] avg=8192.0000 max=14424 max_rank=5 min=1699 min_rank=1 imbalance_ratio=1.760742 max_load_reduction=51112 max_load_reduction_pct=0.779907 moved_to_echo=38914 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 734, 0, 1688, 109, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 4065, 44, 3860, 1724, 148, 0, 0, 0, 0, 0, 0, 0, 8, 339, 2070, 2711, 0, 4039, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 2466, 2781, 0, 0, 20, 0, 229, 50, 0, 0, 0, 0, 0, 0, 0, 0, 3142, 17, 852, 0, 3806, 280, 1212, 0, 0, 0, 0, 0, 0, 0, 491, 0, 444, 42, 20, 2857, 2716, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 81, 3361, 0, 154, 51, 0, 0, 0, 0, 0, 0, 0, 0, 204, 139, 0, 8, 1616, 108, 2, 0, 3947, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6627, 9846, 9838, 9601, 9309, 10640, 3651, 6024] avg=8192.0000 max=10640 max_rank=5 min=3651 min_rank=6 imbalance_ratio=1.298828 max_load_reduction=54896 max_load_reduction_pct=0.837646 moved_to_echo=29624 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[101, 0, 3974, 165, 51, 0, 567, 30, 1184, 34, 0, 0, 0, 0, 0, 0, 4045, 471, 0, 0, 0, 3531, 67, 0, 1209, 149, 0, 0, 0, 0, 0, 0, 1, 0, 73, 8, 4096, 0, 11, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 4057, 0, 34, 0, 0, 0, 240, 16, 0, 0, 0, 0, 0, 0, 902, 1196, 0, 334, 310, 1, 1910, 0, 386, 0, 0, 0, 0, 0, 0, 0, 1337, 0, 2979, 1, 81, 4096, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 3520, 2903, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 4092, 0, 0, 4059, 3947, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 9472, 4660, 8257, 5039, 9360, 10451, 12191] avg=8192.0000 max=12191 max_rank=7 min=4660 min_rank=2 imbalance_ratio=1.488159 max_load_reduction=53345 max_load_reduction_pct=0.813980 moved_to_echo=37041 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[363, 0, 4096, 3487, 0, 6, 0, 0, 1693, 51, 0, 0, 0, 0, 0, 0, 0, 402, 46, 0, 0, 3929, 0, 13, 1728, 71, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3771, 1645, 0, 3771, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 137, 304, 0, 360, 0, 189, 23, 0, 0, 0, 0, 0, 0, 0, 11, 3985, 55, 0, 0, 1089, 2, 570, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4049, 48, 230, 0, 1279, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3684, 3850, 2781, 2383, 0, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 1, 143, 2, 0, 1787, 1889, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9696, 6189, 9862, 4934, 5712, 9634, 15537, 3972] avg=8192.0000 max=15537 max_rank=6 min=3972 min_rank=7 imbalance_ratio=1.896606 max_load_reduction=49999 max_load_reduction_pct=0.762924 moved_to_echo=34855 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3235, 397, 0, 0, 3916, 0, 1509, 109, 0, 0, 0, 0, 0, 0, 3327, 0, 3, 0, 3, 3840, 0, 0, 1541, 148, 0, 0, 0, 0, 0, 0, 0, 0, 20, 74, 4096, 0, 4092, 601, 0, 0, 0, 0, 0, 0, 0, 0, 3, 389, 13, 0, 127, 388, 1262, 0, 261, 50, 0, 0, 0, 0, 0, 0, 0, 3887, 394, 3165, 0, 0, 4095, 0, 1215, 0, 0, 0, 0, 0, 0, 0, 18, 0, 152, 0, 0, 42, 2722, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2787, 3832, 743, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 3836, 1153, 3948, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9166, 8862, 8883, 2493, 12756, 2952, 7395, 13029] avg=8192.0000 max=13029 max_rank=7 min=2493 min_rank=3 imbalance_ratio=1.590454 max_load_reduction=52507 max_load_reduction_pct=0.801193 moved_to_echo=36132 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4077, 0, 3739, 79, 0, 0, 3, 0, 1693, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3520, 3948, 0, 4096, 0, 303, 1729, 146, 0, 0, 0, 0, 0, 0, 0, 0, 38, 259, 4015, 0, 4092, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3879, 0, 0, 106, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 627, 61, 51, 0, 4096, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 11, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3823, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3731, 0, 3899, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9591, 13743, 9078, 8342, 9022, 4281, 3841, 7638] avg=8192.0000 max=13743 max_rank=1 min=3841 min_rank=6 imbalance_ratio=1.677612 max_load_reduction=51793 max_load_reduction_pct=0.790298 moved_to_echo=24782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3895, 0, 3560, 77, 0, 0, 0, 0, 1693, 109, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4090, 0, 4079, 34, 2, 1729, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2571, 0, 0, 1399, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 3500, 199, 2122, 0, 4, 50, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 4096, 0, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 104, 0, 2722, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 68, 49, 26, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2097, 898, 3927, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9334, 10084, 4644, 14066, 5323, 10916, 4246, 6923] avg=8192.0000 max=14066 max_rank=3 min=4246 min_rank=6 imbalance_ratio=1.717041 max_load_reduction=51470 max_load_reduction_pct=0.785370 moved_to_echo=27408 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3924, 0, 5, 0, 0, 2, 2264, 0, 1392, 112, 0, 0, 0, 0, 0, 0, 1, 78, 3993, 0, 0, 0, 0, 0, 1420, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 3877, 0, 4093, 542, 0, 0, 0, 0, 0, 0, 0, 0, 907, 0, 3988, 0, 491, 89, 610, 0, 265, 54, 0, 0, 0, 0, 0, 0, 3, 4089, 2600, 0, 0, 0, 2316, 0, 1218, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 82, 0, 2712, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 6, 0, 3828, 1469, 2814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 10, 3487, 66, 3947, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7699, 5641, 12404, 6404, 10226, 3329, 12150, 7683] avg=8192.0000 max=12404 max_rank=2 min=3329 min_rank=5 imbalance_ratio=1.514160 max_load_reduction=53132 max_load_reduction_pct=0.810730 moved_to_echo=33388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2176, 0, 15488, 128, 640, 11136, 15744, 20608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 8192, 7808, 6784, 21248, 7680, 3456, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1024, 20480, 22528, 2944, 4992, 4864, 7296, 0, 1536, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[7424, 1024, 7680, 8064, 128, 23680, 256, 4352, 12544, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 512, 16640, 22144, 0, 21504, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 17152, 10880, 3712, 1280, 128, 24320, 640, 7040, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 384, 0, 128, 10240, 256, 13184, 12160, 29568, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[13312, 0, 22784, 8320, 128, 768, 7680, 128, 12288, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 1443, 580, 780, 1771, 0, 5, 0, 0, 0, 1, 3969, 11, 0, 0, 4083, 0, 0, 0, 0, 0, 205, 0, 0, 4034, 4, 0, 3876, 99, 580, 3295, 0, 0, 142, 223, 1647, 4093, 4096, 3979, 2, 4093, 5, 2, 0, 21, 4094, 0, 4, 1, 4090, 4056, 12, 1055, 0, 2863, 0, 106, 3007, 1, 0, 3203, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 30, 4081, 3750, 0, 3, 0, 0, 0, 1, 3278, 13, 1456, 0, 2614, 0, 3939, 1, 0, 715, 0, 901, 0, 45, 4003, 0, 0, 0, 0, 962, 4096, 0, 0, 0, 4031, 64, 0, 3217, 17, 10, 858, 0, 0, 0, 0, 2611, 1200, 4019, 19, 1, 0, 15, 693, 4056, 0, 2, 4094, 53, 2765, 3462, 0, 396, 4065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4094, 305, 0, 3, 0, 0, 0, 838, 10, 4, 1718, 0, 2, 293, 1803, 97, 4070, 0, 43, 5, 0, 3953, 0, 0, 1479, 98, 3983, 111, 298, 122, 744, 10, 0, 0, 4096, 3391, 0, 4095, 62, 1, 0, 0, 0, 0, 3733, 3717, 22, 4093, 71, 0, 0, 0, 29, 4096, 0, 2470, 4096, 185, 36, 3168, 0, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 59, 106, 0, 0, 1, 0, 0, 4091, 4064, 4071, 4096, 0, 3477, 17, 4095, 38, 3633, 4, 4, 0, 0, 0, 514, 160, 0, 0, 542, 0, 7, 3843, 4012, 28, 0, 0, 185, 4092, 2, 4096, 0, 0, 6, 0, 0, 0, 687, 207, 2433, 566, 3859, 0, 0, 17, 0, 4077, 0, 0, 3855, 0, 4096, 1, 0, 0, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[2940, 4096, 829, 0, 0, 6, 0, 0, 4096, 0, 0, 3544, 0, 0, 277, 4096, 24, 0, 40, 0, 853, 0, 0, 2013, 483, 3439, 0, 2, 0, 1693, 259, 556, 0, 0, 0, 4096, 3667, 9, 4031, 20, 0, 0, 0, 0, 18, 4033, 2753, 246, 3221, 179, 15, 2, 78, 0, 2713, 0, 0, 4019, 0, 2, 3149, 0, 0, 4039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 579, 1304, 0, 8, 0, 4, 0, 131, 0, 3652, 9, 6, 5, 527, 3156, 3649, 152, 561, 0, 4090, 46, 1, 2630, 577, 468, 0, 3469, 0, 38, 1707, 4059, 0, 13, 0, 4086, 2604, 3515, 691, 68, 311, 0, 0, 17, 0, 159, 4096, 57, 3432, 45, 0, 0, 167, 0, 3693, 0, 0, 3928, 184, 0, 3548, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[24, 3849, 1722, 0, 17, 2, 0, 0, 150, 0, 0, 497, 0, 0, 4084, 165, 0, 3897, 474, 4094, 0, 0, 857, 4087, 0, 3578, 0, 3989, 0, 3, 2, 187, 2291, 0, 0, 4096, 4076, 65, 4096, 0, 0, 1193, 368, 0, 0, 28, 85, 0, 3337, 0, 825, 435, 5, 0, 3412, 0, 0, 3628, 1777, 60, 4081, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 0, 4043, 70, 0, 0, 207, 659, 0, 1996, 3772, 5, 82, 0, 423, 3973, 4050, 0, 643, 9, 0, 0, 0, 0, 4084, 377, 0, 3701, 0, 4070, 28, 1, 0, 0, 160, 4091, 0, 4084, 4066, 2, 0, 0, 0, 0, 0, 288, 9, 1636, 3740, 0, 0, 0, 76, 4096, 0, 0, 3931, 18, 4064, 19, 0, 63, 2998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 30, 4081, 3750, 0, 3, 0, 2940, 397, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1741, 13, 1456, 0, 2614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 1, 0, 715, 0, 901, 0, 1789, 455, 60, 0, 0, 0, 0, 0, 45, 2214, 0, 0, 0, 0, 962, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 64, 0, 277, 17, 0, 0, 0, 0, 0, 0, 0, 0, 10, 858, 0, 0, 0, 0, 2611, 1200, 2512, 1477, 0, 0, 0, 0, 0, 0, 4019, 19, 1, 0, 15, 693, 3601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1185, 53, 2765, 3462, 0, 396, 4065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11201, 5825, 7860, 7317, 4389, 8668, 8348, 11928] avg=8192.0000 max=11928 max_rank=7 min=4389 min_rank=4 imbalance_ratio=1.456055 max_load_reduction=53608 max_load_reduction_pct=0.817993 moved_to_echo=33333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2940, 4096, 829, 0, 0, 6, 0, 0, 3684, 389, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1882, 0, 0, 277, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 40, 0, 853, 0, 0, 2013, 1537, 304, 65, 0, 0, 0, 0, 0, 483, 1902, 0, 2, 0, 1693, 259, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3667, 9, 347, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4033, 2753, 246, 2466, 1597, 0, 0, 0, 0, 0, 0, 3221, 179, 15, 2, 78, 0, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1164, 0, 2, 3149, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11944, 10351, 4836, 4895, 8139, 11113, 5904, 8354] avg=8192.0000 max=11944 max_rank=0 min=4836 min_rank=2 imbalance_ratio=1.458008 max_load_reduction=53592 max_load_reduction_pct=0.817749 moved_to_echo=33510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1443, 580, 780, 1771, 0, 5, 0, 3637, 291, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2107, 11, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 1, 321, 73, 0, 0, 0, 0, 0, 4034, 3, 0, 3876, 99, 580, 3295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 223, 1647, 4093, 4096, 342, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 2, 0, 21, 4094, 0, 4, 1845, 1789, 0, 0, 0, 0, 0, 0, 1, 4090, 4056, 12, 1055, 0, 2542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 871, 1, 0, 3203, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8507, 6202, 600, 11887, 10545, 11853, 11756, 4186] avg=8192.0000 max=11887 max_rank=3 min=600 min_rank=2 imbalance_ratio=1.451050 max_load_reduction=53649 max_load_reduction_pct=0.818619 moved_to_echo=38340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 579, 1304, 0, 8, 0, 4, 0, 631, 381, 0, 0, 0, 0, 0, 0, 131, 0, 3652, 5, 6, 5, 527, 3156, 0, 0, 0, 0, 0, 0, 0, 0, 3649, 152, 561, 0, 4090, 46, 1, 2630, 209, 414, 0, 0, 0, 0, 0, 0, 577, 259, 0, 3469, 0, 38, 1707, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4086, 2604, 3515, 60, 68, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 17, 0, 159, 4096, 57, 2410, 4, 0, 0, 0, 0, 0, 0, 3432, 45, 0, 0, 167, 0, 3279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 184, 0, 3548, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2907, 7482, 11752, 10109, 10346, 7054, 6923, 8963] avg=8192.0000 max=11752 max_rank=2 min=2907 min_rank=0 imbalance_ratio=1.434570 max_load_reduction=53784 max_load_reduction_pct=0.820679 moved_to_echo=33286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 59, 106, 0, 0, 1, 0, 0, 3744, 374, 0, 0, 0, 0, 0, 0, 4091, 4064, 4071, 2175, 0, 3477, 17, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3633, 4, 4, 0, 0, 0, 514, 0, 458, 75, 0, 0, 0, 0, 0, 160, 0, 0, 542, 0, 7, 3843, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 185, 4092, 2, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 687, 207, 2433, 2365, 1846, 0, 0, 0, 0, 0, 0, 566, 3859, 0, 0, 17, 0, 3619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 0, 4096, 1, 0, 0, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4284, 21990, 4726, 8564, 4659, 7544, 8061, 5708] avg=8192.0000 max=21990 max_rank=1 min=4284 min_rank=0 imbalance_ratio=2.684326 max_load_reduction=43546 max_load_reduction_pct=0.664459 moved_to_echo=25972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 305, 0, 3, 0, 0, 0, 3743, 397, 0, 0, 0, 0, 0, 0, 838, 10, 4, 913, 0, 2, 293, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 97, 4070, 0, 43, 5, 0, 3953, 0, 661, 460, 31, 0, 0, 0, 0, 0, 0, 818, 98, 3983, 111, 298, 122, 744, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 3391, 0, 352, 62, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3733, 3717, 22, 2513, 774, 0, 0, 0, 0, 0, 0, 4093, 71, 0, 0, 0, 29, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2470, 1186, 185, 36, 3168, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8542, 3863, 9320, 6174, 7911, 10760, 7829, 11137] avg=8192.0000 max=11137 max_rank=7 min=3863 min_rank=1 imbalance_ratio=1.359497 max_load_reduction=54399 max_load_reduction_pct=0.830063 moved_to_echo=37637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4043, 70, 0, 0, 207, 3733, 381, 0, 0, 0, 0, 0, 0, 659, 0, 1996, 2003, 5, 82, 0, 423, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 4050, 0, 643, 9, 0, 0, 0, 1826, 460, 69, 0, 0, 0, 0, 0, 0, 2258, 377, 0, 3701, 0, 4070, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 160, 4091, 0, 351, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 288, 9, 2412, 1700, 0, 0, 0, 0, 0, 0, 1636, 3740, 0, 0, 0, 76, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1138, 18, 4064, 19, 0, 63, 2998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8436, 5168, 11030, 10434, 8669, 4411, 9088, 8300] avg=8192.0000 max=11030 max_rank=2 min=4411 min_rank=5 imbalance_ratio=1.346436 max_load_reduction=54506 max_load_reduction_pct=0.831696 moved_to_echo=30468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[24, 3849, 1722, 0, 17, 2, 0, 0, 3746, 355, 0, 0, 0, 0, 0, 0, 150, 0, 0, 256, 0, 0, 4084, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 474, 4094, 0, 0, 857, 4087, 1603, 387, 13, 0, 0, 0, 0, 0, 0, 1975, 0, 3989, 0, 3, 2, 187, 0, 0, 0, 0, 0, 0, 0, 0, 2291, 0, 0, 4096, 4076, 65, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 368, 0, 0, 28, 85, 0, 2231, 228, 0, 0, 0, 0, 0, 0, 3337, 0, 825, 435, 5, 0, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1777, 60, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9715, 4655, 15412, 6156, 10878, 4133, 7627, 6960] avg=8192.0000 max=15412 max_rank=2 min=4133 min_rank=5 imbalance_ratio=1.881348 max_load_reduction=50124 max_load_reduction_pct=0.764832 moved_to_echo=29598 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[7808, 19840, 1152, 4864, 5760, 256, 5760, 9344, 7680, 3328, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4480, 2176, 384, 128, 128, 12800, 13824, 4096, 18816, 9472, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[3072, 14208, 4992, 8960, 5632, 128, 128, 256, 25984, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[20352, 12032, 4992, 512, 1408, 896, 25856, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2688, 8960, 2304, 11136, 20736, 0, 512, 19840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2432, 256, 256, 22400, 26112, 7808, 2432, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[9984, 4096, 9728, 11136, 128, 5120, 5248, 20480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[5376, 9472, 512, 15872, 3968, 2688, 14336, 13696, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 1, 3545, 2119, 4078, 45, 4096, 0, 127, 10, 0, 0, 3867, 275, 0, 523, 66, 107, 3419, 544, 4089, 0, 1, 3052, 1010, 3892, 3, 2556, 1829, 1, 0, 4, 0, 630, 0, 3895, 2867, 0, 14, 4091, 1764, 0, 2455, 0, 1, 0, 1, 0, 42, 0, 171, 0, 29, 1802, 0, 0, 0, 1229, 0, 2592, 598] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 8, 0, 67, 0, 1011, 3910, 4086, 0, 4079, 575, 0, 488, 0, 0, 61, 60, 140, 3483, 0, 0, 4037, 0, 155, 232, 2787, 0, 1909, 0, 0, 4096, 0, 0, 12, 0, 10, 0, 1, 4072, 4075, 0, 0, 4069, 3709, 1, 207, 0, 3657, 3, 0, 0, 45, 0, 4066, 0, 0, 0, 0, 4094, 2221, 20, 0, 4090, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 231, 0, 1, 0, 4096, 0, 2196, 0, 2822, 16, 0, 34, 0, 0, 0, 0, 3392, 994, 0, 0, 2726, 98, 2871, 0, 566, 0, 3934, 789, 0, 4096, 0, 0, 2315, 39, 23, 0, 4045, 1, 4081, 0, 0, 4025, 4096, 811, 1662, 0, 3723, 2, 0, 0, 339, 1197, 269, 3898, 0, 79, 1, 2971, 0, 69, 318, 2710, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4054, 24, 1823, 0, 4096, 7, 4096, 37, 4080, 1, 0, 0, 212, 156, 0, 1, 8, 1590, 3352, 0, 18, 360, 124, 576, 318, 4, 2831, 3978, 247, 8, 0, 4088, 0, 0, 0, 3521, 1, 1, 48, 4094, 3669, 0, 3368, 0, 2443, 5, 0, 0, 1, 87, 4016, 0, 118, 0, 0, 2226, 4049, 0, 0, 1800, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[25, 1, 5, 0, 0, 3951, 0, 3145, 15, 941, 2801, 516, 4050, 0, 0, 428, 86, 3852, 0, 0, 3, 4063, 0, 2140, 0, 0, 52, 1, 2362, 0, 4059, 334, 4015, 3935, 0, 0, 3, 837, 2859, 4094, 0, 207, 741, 5, 24, 2904, 0, 7, 0, 14, 0, 0, 9, 3431, 0, 95, 0, 0, 4019, 0, 1486, 0, 4021, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1193, 2, 0, 0, 0, 2732, 4030, 152, 3072, 0, 10, 0, 0, 0, 0, 0, 9, 1, 0, 2325, 0, 4049, 0, 4011, 0, 3666, 873, 2, 3975, 36, 4095, 189, 0, 4084, 103, 0, 24, 244, 0, 4082, 0, 7, 3941, 3851, 0, 0, 0, 4062, 0, 0, 0, 45, 0, 3924, 0, 14, 94, 73, 48, 207, 0, 2216, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 3797, 2252, 417, 3595, 4054, 0, 4096, 4, 4096, 4080, 14, 2, 0, 0, 107, 75, 3961, 0, 2, 0, 598, 9, 4096, 0, 3306, 8, 34, 376, 4096, 29, 0, 14, 2134, 435, 0, 0, 0, 3942, 4093, 436, 0, 212, 4082, 0, 29, 0, 0, 0, 0, 0, 81, 41, 0, 172, 0, 0, 0, 2924, 2, 3823, 0, 12, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 28, 0, 0, 4095, 8, 4096, 464, 3580, 1, 0, 0, 0, 8, 2184, 0, 2, 3587, 4033, 0, 4096, 0, 2924, 3, 13, 4096, 11, 1862, 0, 0, 0, 4006, 0, 199, 0, 4096, 4090, 0, 0, 3835, 0, 0, 4093, 0, 0, 0, 0, 0, 83, 70, 4087, 41, 0, 0, 0, 27, 0, 608, 0, 3996, 1211] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4054, 24, 1823, 0, 2284, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 37, 4080, 1, 0, 0, 212, 1388, 504, 0, 0, 0, 0, 0, 0, 156, 0, 1, 8, 1590, 2108, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 360, 124, 576, 318, 4, 2831, 2364, 247, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4088, 0, 0, 0, 3521, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 48, 2325, 3669, 0, 3368, 0, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 87, 4016, 0, 118, 0, 1812, 1769, 226, 0, 0, 0, 0, 0, 0, 2226, 4049, 0, 0, 1800, 0, 740, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 10325, 3881, 6824, 7618, 11854, 8034, 8815] avg=8192.0000 max=11854 max_rank=5 min=3881 min_rank=2 imbalance_ratio=1.447021 max_load_reduction=53682 max_load_reduction_pct=0.819122 moved_to_echo=36321 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[25, 1, 5, 0, 0, 3951, 0, 1754, 0, 0, 0, 0, 0, 0, 0, 0, 15, 941, 2801, 516, 4050, 0, 0, 428, 1416, 611, 0, 0, 0, 0, 0, 0, 86, 3852, 0, 0, 3, 2555, 0, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1, 2362, 0, 2412, 334, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 3935, 0, 0, 3, 837, 2859, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 421, 5, 24, 2904, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 9, 3431, 0, 95, 2308, 1391, 320, 231, 0, 0, 0, 0, 0, 0, 4019, 0, 1486, 0, 4021, 0, 897, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5736, 10778, 8636, 5161, 13435, 3568, 7799, 10423] avg=8192.0000 max=13435 max_rank=4 min=3568 min_rank=5 imbalance_ratio=1.640015 max_load_reduction=52101 max_load_reduction_pct=0.794998 moved_to_echo=35225 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 1, 3545, 2119, 2274, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4096, 0, 127, 10, 0, 0, 3867, 892, 514, 0, 0, 0, 0, 0, 0, 275, 0, 523, 66, 107, 2150, 544, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3052, 1010, 3892, 3, 1519, 1829, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 630, 0, 3895, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2323, 1764, 0, 2455, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 42, 0, 171, 0, 29, 1616, 1804, 1768, 145, 0, 0, 0, 0, 1802, 0, 0, 0, 1229, 0, 2592, 598, 755, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12035, 9551, 7754, 11306, 5781, 6557, 5576, 6976] avg=8192.0000 max=12035 max_rank=0 min=5576 min_rank=6 imbalance_ratio=1.469116 max_load_reduction=53501 max_load_reduction_pct=0.816360 moved_to_echo=24890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 231, 0, 1, 0, 4096, 0, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2822, 16, 0, 34, 0, 0, 0, 1429, 410, 0, 0, 0, 0, 0, 0, 0, 3392, 994, 0, 0, 1714, 98, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 0, 3934, 789, 0, 2434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2315, 39, 23, 0, 4045, 1, 1780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2286, 4096, 811, 1662, 0, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 339, 1197, 269, 3898, 0, 2301, 971, 1739, 233, 0, 0, 0, 0, 79, 1, 2971, 0, 69, 318, 2710, 0, 602, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5553, 4711, 9069, 7723, 8203, 12578, 10949, 6750] avg=8192.0000 max=12578 max_rank=5 min=4711 min_rank=1 imbalance_ratio=1.535400 max_load_reduction=52958 max_load_reduction_pct=0.808075 moved_to_echo=38480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 67, 0, 1011, 3910, 2279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 575, 0, 488, 0, 0, 61, 1429, 607, 0, 0, 0, 0, 0, 0, 60, 140, 3483, 0, 0, 2539, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 232, 2787, 0, 1909, 0, 0, 2434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 10, 0, 1, 4072, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2311, 3709, 1, 207, 0, 3657, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 45, 0, 4066, 0, 0, 2298, 1807, 1758, 233, 0, 0, 0, 0, 0, 0, 4094, 2221, 20, 0, 4090, 0, 891, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7275, 7239, 6377, 7362, 5872, 9885, 10210, 11316] avg=8192.0000 max=11316 max_rank=7 min=5872 min_rank=4 imbalance_ratio=1.381348 max_load_reduction=54220 max_load_reduction_pct=0.827332 moved_to_echo=37283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1193, 2, 0, 0, 0, 2732, 4030, 85, 0, 0, 0, 0, 0, 0, 0, 0, 3072, 0, 10, 0, 0, 0, 0, 0, 1429, 609, 0, 0, 0, 0, 0, 0, 9, 1, 0, 2325, 0, 2546, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3666, 873, 2, 3975, 36, 2433, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 103, 0, 24, 244, 0, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2238, 3851, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 3924, 0, 14, 2301, 67, 1703, 233, 0, 0, 0, 0, 94, 73, 48, 207, 0, 2216, 4095, 0, 894, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8042, 5120, 8892, 11174, 6236, 10158, 8287, 7627] avg=8192.0000 max=11174 max_rank=3 min=5120 min_rank=1 imbalance_ratio=1.364014 max_load_reduction=54362 max_load_reduction_pct=0.829498 moved_to_echo=32308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3797, 2252, 417, 3595, 4054, 0, 2284, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4080, 14, 2, 0, 0, 107, 10, 89, 0, 0, 0, 0, 0, 0, 75, 3961, 0, 2, 0, 377, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3306, 8, 34, 376, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2134, 435, 0, 0, 0, 3942, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 436, 0, 121, 4082, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 41, 0, 172, 0, 2308, 1812, 91, 1, 0, 0, 0, 0, 0, 0, 2924, 2, 3823, 0, 12, 0, 132, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 8402, 8520, 7838, 8310, 4668, 4506, 6893] avg=8192.0000 max=16399 max_rank=0 min=4506 min_rank=6 imbalance_ratio=2.001831 max_load_reduction=49137 max_load_reduction_pct=0.749771 moved_to_echo=24377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 28, 0, 0, 2280, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 464, 3580, 1, 0, 0, 0, 652, 609, 0, 0, 0, 0, 0, 0, 8, 2184, 0, 2, 3587, 2530, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2924, 3, 13, 4096, 11, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 199, 0, 4096, 1780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2175, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 70, 4087, 41, 0, 2310, 1815, 1660, 109, 0, 0, 0, 0, 0, 0, 27, 0, 608, 0, 3996, 1211, 894, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2311, 9410, 12407, 8148, 10081, 6268, 10175, 6736] avg=8192.0000 max=12407 max_rank=2 min=2311 min_rank=0 imbalance_ratio=1.514526 max_load_reduction=53129 max_load_reduction_pct=0.810684 moved_to_echo=33260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1280, 4096, 6400, 4608, 3712, 21248, 10112, 14592, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 13568, 5120, 2432, 5376, 16640, 768, 21504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2048, 128, 16384, 6528, 7296, 2560, 23424, 1920, 5888, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 640, 1408, 19968, 4224, 256, 15488, 11520, 10880, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[512, 384, 14208, 21248, 896, 14720, 0, 13952, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 13440, 4608, 7296, 15616, 7040, 14720, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3200, 24320, 8064, 8320, 4608, 0, 0, 4736, 8704, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4096, 12544, 8704, 128, 896, 5248, 22400, 12032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[10, 3, 4093, 0, 63, 0, 1538, 32, 1834, 1, 1836, 0, 4096, 0, 478, 0, 0, 0, 1848, 0, 4014, 19, 23, 549, 38, 3806, 4096, 3567, 10, 0, 0, 969, 1843, 0, 6, 0, 2750, 62, 46, 4089, 0, 3719, 4005, 4096, 2048, 0, 935, 0, 65, 0, 172, 4070, 7, 0, 41, 0, 0, 0, 3865, 0, 0, 14, 727, 53] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 113, 1001, 0, 7, 4096, 4096, 3247, 296, 24, 491, 1252, 32, 4093, 94, 2723, 192, 0, 341, 0, 3805, 4095, 0, 4082, 0, 2689, 0, 475, 0, 19, 823, 38, 235, 3492, 30, 149, 3529, 29, 2313, 0, 0, 0, 0, 2599, 25, 0, 3862, 1318, 0, 4081, 1746, 24, 2, 3971, 0, 1, 0, 3, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1652, 6, 0, 0, 0, 0, 0, 0, 2080, 88, 0, 2289, 1191, 0, 30, 0, 4067, 0, 101, 4035, 4027, 0, 766, 4096, 0, 0, 4076, 0, 2, 1809, 68, 0, 354, 330, 1735, 266, 0, 2147, 0, 4060, 50, 2010, 0, 132, 44, 20, 3117, 4095, 653, 4095, 0, 0, 4096, 0, 0, 0, 3940, 0, 0, 0, 4009, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[16, 3991, 38, 3997, 4084, 0, 0, 0, 0, 1968, 4043, 0, 45, 82, 211, 0, 13, 373, 3153, 0, 2945, 2818, 0, 0, 144, 0, 3820, 7, 3607, 0, 0, 3819, 0, 77, 0, 0, 4096, 0, 3755, 0, 4063, 3906, 82, 0, 0, 0, 89, 0, 1, 30, 0, 0, 0, 4001, 0, 0, 0, 349, 26, 3952, 0, 888, 0, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3477, 0, 3912, 0, 4095, 912, 0, 0, 0, 3, 94, 170, 1, 4096, 545, 2570, 0, 30, 3504, 4095, 38, 690, 0, 1139, 0, 4058, 0, 145, 186, 0, 107, 4095, 0, 0, 0, 4063, 0, 14, 0, 139, 1, 1, 0, 3772, 6, 0, 0, 0, 3800, 2486, 672, 0, 2982, 242, 3732, 3, 0, 1, 3914, 1, 0, 0, 1745, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3796, 0, 4095, 1, 373, 0, 0, 0, 0, 47, 1934, 1, 738, 1225, 959, 0, 629, 0, 10, 2, 3476, 4061, 4032, 0, 1, 4096, 0, 0, 304, 0, 4096, 80, 8, 0, 86, 37, 4080, 0, 470, 0, 0, 4096, 1035, 3052, 396, 40, 0, 1, 4039, 1, 61, 3888, 3076, 2, 4096, 0, 0, 0, 3022, 0, 0, 0, 94, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 4095, 4080, 3444, 0, 1, 0, 0, 0, 4082, 0, 4092, 2135, 2690, 0, 3952, 2556, 20, 97, 96, 4068, 3824, 11, 0, 0, 0, 0, 4096, 435, 993, 0, 0, 4096, 34, 227, 12, 3981, 4062, 0, 4096, 0, 36, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 5, 22, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[3874, 170, 4081, 0, 0, 0, 0, 0, 0, 31, 3086, 0, 4008, 0, 1, 0, 2, 0, 1450, 0, 4095, 227, 19, 0, 0, 493, 4096, 0, 8, 0, 0, 4, 0, 0, 0, 0, 4096, 4059, 966, 2467, 0, 4096, 12, 4096, 6, 0, 0, 0, 4052, 0, 4095, 4087, 526, 0, 4094, 2679, 0, 0, 0, 0, 12, 0, 548, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[10, 3, 4093, 0, 63, 0, 1538, 32, 1838, 419, 0, 0, 0, 0, 0, 0, 1834, 1, 1836, 0, 4096, 0, 478, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 875, 0, 4014, 19, 23, 549, 0, 0, 0, 0, 0, 0, 0, 0, 38, 458, 4096, 3567, 10, 0, 0, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1843, 0, 6, 0, 2750, 62, 46, 4089, 554, 15, 133, 0, 0, 0, 0, 0, 0, 3346, 4005, 4096, 2048, 0, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 172, 4070, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 14, 727, 53, 41, 1510, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7996, 8485, 5480, 9138, 9498, 14430, 4299, 6210] avg=8192.0000 max=14430 max_rank=5 min=4299 min_rank=6 imbalance_ratio=1.761475 max_load_reduction=51106 max_load_reduction_pct=0.779816 moved_to_echo=34437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 113, 1001, 1838, 929, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4096, 3247, 296, 24, 491, 228, 0, 0, 0, 0, 0, 0, 0, 1252, 32, 1937, 94, 2723, 192, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 4095, 0, 4082, 0, 2689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 19, 823, 38, 235, 3492, 30, 1227, 614, 126, 0, 0, 0, 0, 0, 149, 3175, 29, 2313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1985, 25, 0, 3862, 1318, 0, 0, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 24, 2, 3971, 0, 1, 0, 3, 3, 4081, 1509, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3881, 12485, 6571, 11324, 7079, 5666, 8936, 9594] avg=8192.0000 max=12485 max_rank=1 min=3881 min_rank=0 imbalance_ratio=1.524048 max_load_reduction=53051 max_load_reduction_pct=0.809494 moved_to_echo=31275 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3477, 0, 3912, 0, 4095, 912, 0, 0, 1960, 795, 0, 0, 0, 0, 0, 0, 0, 3, 94, 170, 1, 4096, 545, 2570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1659, 4095, 38, 690, 0, 1139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 0, 145, 186, 0, 107, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 14, 0, 139, 1050, 897, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3772, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2903, 2486, 672, 0, 2982, 242, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3914, 1, 0, 0, 1745, 0, 3732, 1610, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15151, 7479, 7651, 5021, 6163, 3780, 9288, 11003] avg=8192.0000 max=15151 max_rank=0 min=3780 min_rank=5 imbalance_ratio=1.849487 max_load_reduction=50385 max_load_reduction_pct=0.768814 moved_to_echo=30234 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1652, 6, 0, 0, 0, 0, 1978, 923, 0, 0, 0, 0, 0, 0, 0, 0, 2080, 88, 0, 2289, 1191, 0, 262, 0, 0, 0, 0, 0, 0, 0, 30, 0, 1925, 0, 101, 4035, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 766, 493, 0, 0, 4076, 0, 2, 1809, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 354, 330, 1735, 266, 0, 2147, 1219, 736, 145, 0, 0, 0, 0, 0, 0, 3653, 50, 2010, 0, 132, 44, 20, 0, 0, 0, 0, 0, 0, 0, 0, 2381, 4095, 653, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 0, 4009, 0, 4096, 1625, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4559, 5910, 10118, 7146, 7000, 5909, 11224, 13670] avg=8192.0000 max=13670 max_rank=7 min=4559 min_rank=0 imbalance_ratio=1.668701 max_load_reduction=51866 max_load_reduction_pct=0.791412 moved_to_echo=37803 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[16, 3991, 38, 3997, 4084, 0, 0, 0, 0, 716, 0, 0, 0, 0, 0, 0, 0, 1968, 4043, 0, 45, 82, 211, 0, 252, 0, 0, 0, 0, 0, 0, 0, 13, 373, 1492, 0, 2945, 2818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 3820, 7, 3607, 0, 0, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 4096, 0, 3755, 0, 945, 0, 140, 0, 0, 0, 0, 0, 4063, 3514, 82, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 30, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 26, 3952, 0, 888, 0, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12842, 6601, 7641, 11397, 9013, 7748, 4032, 6262] avg=8192.0000 max=12842 max_rank=0 min=4032 min_rank=6 imbalance_ratio=1.567627 max_load_reduction=52694 max_load_reduction_pct=0.804047 moved_to_echo=27055 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3796, 0, 4095, 1, 373, 0, 0, 0, 1978, 2, 0, 0, 0, 0, 0, 0, 0, 47, 1934, 1, 738, 1225, 959, 0, 264, 0, 0, 0, 0, 0, 0, 0, 629, 0, 6, 2, 3476, 4061, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 493, 0, 0, 304, 0, 4096, 80, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 86, 37, 4080, 0, 470, 0, 2, 954, 147, 0, 0, 0, 0, 0, 0, 3685, 1035, 3052, 396, 40, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3085, 1, 61, 3888, 3076, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3022, 0, 0, 0, 94, 0, 4096, 1625, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10245, 5168, 12206, 4974, 5784, 8209, 10113, 8837] avg=8192.0000 max=12206 max_rank=2 min=4974 min_rank=3 imbalance_ratio=1.489990 max_load_reduction=53330 max_load_reduction_pct=0.813751 moved_to_echo=32943 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3874, 170, 4081, 0, 0, 0, 0, 0, 243, 333, 0, 0, 0, 0, 0, 0, 0, 31, 3086, 0, 4008, 0, 1, 0, 267, 0, 0, 0, 0, 0, 0, 0, 2, 0, 678, 0, 4095, 227, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4096, 0, 8, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4059, 966, 2467, 439, 960, 150, 0, 0, 0, 0, 0, 0, 3679, 12, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3092, 0, 4095, 4087, 526, 0, 0, 2679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 548, 0, 4094, 198, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8701, 7393, 5021, 4160, 13137, 7793, 14479, 4852] avg=8192.0000 max=14479 max_rank=6 min=4160 min_rank=3 imbalance_ratio=1.767456 max_load_reduction=51057 max_load_reduction_pct=0.779068 moved_to_echo=40261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1234, 927, 0, 0, 0, 0, 0, 0, 131, 0, 4095, 4080, 3444, 0, 1, 0, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1931, 0, 4092, 2135, 2690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 308, 20, 97, 96, 4068, 3824, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 435, 993, 0, 1224, 967, 147, 0, 0, 0, 0, 0, 0, 3685, 34, 227, 12, 3981, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3129, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 22, 0, 4067, 1014, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2161, 12015, 10848, 12376, 7862, 12001, 3165, 5108] avg=8192.0000 max=12376 max_rank=3 min=2161 min_rank=0 imbalance_ratio=1.510742 max_load_reduction=53160 max_load_reduction_pct=0.811157 moved_to_echo=28136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11264, 4224, 17920, 4096, 8704, 1024, 1664, 1152, 11136, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2048, 2176, 21376, 8448, 15616, 8064, 3456, 3072, 1792, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[16640, 6656, 5760, 20096, 7936, 4352, 0, 4480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4224, 24832, 5248, 19584, 2560, 4224, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2048, 512, 10624, 4224, 21504, 14208, 10880, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4992, 2816, 16128, 3840, 12416, 4096, 10752, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2432, 128, 512, 5376, 20992, 5120, 9728, 8960, 6784, 5248, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 384, 18816, 3968, 128, 1024, 7168, 1152, 24320, 9216, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 1643, 3275, 84, 0, 551, 1533, 4096, 0, 32, 1, 33, 4093, 0, 3966, 4096, 0, 215, 0, 0, 0, 2, 4078, 1, 0, 768, 4065, 0, 0, 0, 2731, 0, 3841, 1036, 0, 3865, 1266, 0, 447, 178, 0, 3576, 0, 28, 26, 0, 110, 1, 0, 1262, 4003, 0, 4036, 0, 2320, 1, 7, 4046, 188, 36, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[3335, 0, 4094, 469, 0, 3145, 3466, 0, 0, 48, 143, 72, 0, 4075, 0, 11, 3005, 26, 0, 0, 2343, 0, 301, 658, 0, 0, 0, 1776, 0, 0, 364, 3968, 8, 4094, 0, 7, 4005, 0, 32, 4012, 4094, 0, 4093, 0, 0, 4, 0, 0, 4081, 134, 0, 1216, 3, 0, 126, 604, 0, 0, 173, 3749, 0, 3778, 24, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[3026, 0, 4096, 0, 13, 2599, 0, 10, 3701, 0, 157, 0, 279, 3725, 0, 2622, 4067, 0, 0, 45, 0, 0, 0, 252, 15, 29, 619, 3674, 29, 0, 3, 4041, 0, 0, 0, 4086, 4066, 3497, 0, 8, 18, 4096, 988, 8, 4096, 13, 0, 0, 4096, 2, 7, 4, 24, 2280, 0, 0, 333, 0, 1448, 1402, 449, 268, 1241, 104] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[12, 351, 261, 4096, 0, 2168, 60, 0, 4060, 0, 0, 93, 0, 121, 1, 3989, 3694, 977, 0, 0, 27, 673, 9, 0, 0, 294, 3430, 1664, 0, 0, 0, 0, 69, 0, 2696, 0, 3883, 4093, 0, 1, 4096, 841, 4085, 0, 0, 0, 0, 0, 40, 4061, 0, 3563, 39, 121, 2, 0, 1200, 445, 3955, 4095, 309, 1962, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1, 168, 4066, 4096, 0, 1201, 0, 0, 4042, 0, 0, 2317, 0, 3815, 0, 4053, 3523, 8, 308, 0, 2795, 10, 89, 0, 1, 0, 965, 2776, 0, 0, 0, 3333, 48, 0, 0, 0, 3961, 21, 0, 2, 46, 16, 34, 0, 18, 3079, 952, 0, 2765, 0, 0, 4084, 35, 596, 1, 4084, 0, 0, 3841, 367, 0, 4019, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4078, 214, 118, 4096, 0, 4096, 0, 0, 1540, 0, 0, 26, 0, 4096, 127, 3886, 3601, 2694, 0, 1055, 0, 875, 110, 0, 3, 316, 4096, 4049, 0, 0, 0, 3508, 60, 0, 0, 1282, 4015, 32, 1185, 0, 12, 152, 12, 0, 5, 0, 528, 0, 49, 0, 0, 3389, 397, 4004, 4, 0, 0, 0, 3726, 1, 0, 4083, 16, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1764, 46, 0, 3926, 0, 770, 0, 44, 2, 0, 0, 3, 0, 3782, 1174, 4024, 4096, 3929, 0, 132, 122, 0, 0, 69, 0, 0, 4060, 4074, 0, 0, 489, 2838, 0, 64, 2, 0, 4084, 40, 0, 259, 0, 182, 2090, 49, 0, 0, 0, 0, 3947, 3609, 10, 4096, 3, 41, 0, 1482, 0, 9, 3463, 4037, 131, 2593, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 0, 3816, 0, 55, 0, 374, 4054, 0, 0, 928, 0, 2303, 0, 4091, 4096, 6, 3, 0, 4096, 0, 0, 4095, 0, 0, 1, 3967, 4071, 1757, 149, 3595, 0, 0, 19, 0, 2956, 6, 0, 0, 825, 119, 239, 5, 0, 0, 0, 0, 4092, 0, 2112, 3807, 0, 3, 0, 1, 545, 1, 4038, 4090, 11, 1128, 19, 62] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 351, 261, 3119, 0, 2168, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 93, 0, 121, 1, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 3694, 977, 0, 0, 27, 673, 9, 0, 525, 147, 0, 0, 0, 0, 0, 0, 0, 294, 3430, 1283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 2696, 0, 3883, 4093, 0, 1, 428, 0, 0, 0, 0, 0, 0, 0, 4096, 841, 4085, 0, 0, 0, 0, 0, 2102, 1341, 234, 0, 0, 0, 0, 0, 40, 4061, 0, 2222, 39, 121, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 445, 3955, 4095, 309, 1962, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5971, 6162, 6052, 5007, 11170, 12699, 6485, 11990] avg=8192.0000 max=12699 max_rank=5 min=5007 min_rank=3 imbalance_ratio=1.550171 max_load_reduction=52837 max_load_reduction_pct=0.806229 moved_to_echo=42344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1643, 2494, 84, 0, 551, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 32, 1, 33, 4093, 0, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 215, 0, 0, 0, 2, 4078, 420, 359, 0, 0, 0, 0, 0, 0, 1, 0, 768, 3132, 0, 0, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 1036, 0, 3865, 1266, 0, 447, 342, 0, 0, 0, 0, 0, 0, 0, 178, 0, 3576, 0, 28, 26, 0, 110, 2090, 1507, 574, 0, 0, 0, 0, 0, 1, 0, 1262, 2496, 0, 4036, 0, 2320, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 4046, 188, 36, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6305, 10131, 9170, 6632, 10797, 8089, 10115, 4297] avg=8192.0000 max=10797 max_rank=4 min=4297 min_rank=7 imbalance_ratio=1.317993 max_load_reduction=54739 max_load_reduction_pct=0.835251 moved_to_echo=33298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3026, 0, 4096, 0, 13, 2599, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3701, 0, 157, 0, 279, 3725, 0, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 45, 0, 0, 0, 252, 0, 325, 0, 0, 0, 0, 0, 0, 15, 29, 619, 2831, 29, 0, 3, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4066, 3497, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 988, 8, 4096, 13, 0, 0, 1382, 1, 518, 0, 0, 0, 0, 0, 4096, 2, 7, 3, 24, 2280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 1448, 1402, 449, 268, 1241, 104, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9744, 9102, 4689, 7567, 11657, 11120, 6412, 5245] avg=8192.0000 max=11657 max_rank=4 min=4689 min_rank=2 imbalance_ratio=1.422974 max_load_reduction=53879 max_load_reduction_pct=0.822128 moved_to_echo=34434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3335, 0, 4094, 358, 0, 3145, 3466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 143, 72, 0, 4075, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3005, 26, 0, 0, 2343, 0, 301, 658, 60, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 0, 0, 364, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4094, 0, 7, 4005, 0, 32, 4012, 49, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4093, 0, 0, 4, 0, 0, 5, 457, 250, 0, 0, 0, 0, 0, 4081, 134, 0, 759, 3, 0, 126, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 3749, 0, 3778, 24, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14398, 4344, 6550, 5701, 12207, 8903, 5707, 7726] avg=8192.0000 max=14398 max_rank=0 min=4344 min_rank=1 imbalance_ratio=1.757568 max_load_reduction=51138 max_load_reduction_pct=0.780304 moved_to_echo=34543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 168, 4066, 3119, 0, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 2317, 0, 3815, 0, 1917, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 8, 308, 0, 2795, 10, 89, 0, 525, 245, 0, 0, 0, 0, 0, 0, 1, 0, 965, 2139, 0, 0, 0, 3333, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 3961, 21, 0, 2, 428, 0, 0, 0, 0, 0, 0, 0, 46, 16, 34, 0, 18, 3079, 952, 0, 2136, 1538, 392, 0, 0, 0, 0, 0, 2765, 0, 0, 2546, 35, 596, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 367, 0, 4019, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8555, 12091, 7503, 6438, 4460, 8211, 10027, 8251] avg=8192.0000 max=12091 max_rank=1 min=4460 min_rank=4 imbalance_ratio=1.475952 max_load_reduction=53445 max_load_reduction_pct=0.815506 moved_to_echo=30949 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 2906, 0, 55, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 928, 0, 2303, 0, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 3, 0, 4096, 0, 0, 4095, 489, 351, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3056, 4071, 1757, 149, 3595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2956, 6, 0, 0, 399, 0, 0, 0, 0, 0, 0, 0, 825, 119, 239, 5, 0, 0, 0, 0, 2156, 1433, 560, 0, 0, 0, 0, 0, 4092, 0, 2112, 2374, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 545, 1, 4038, 4090, 11, 1128, 19, 62, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3336, 9220, 13136, 12629, 3380, 5337, 8582, 9916] avg=8192.0000 max=13136 max_rank=2 min=3336 min_rank=0 imbalance_ratio=1.603516 max_load_reduction=52400 max_load_reduction_pct=0.799561 moved_to_echo=27215 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1764, 46, 0, 2990, 0, 770, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 3782, 1174, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3929, 0, 132, 122, 0, 0, 69, 503, 360, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 3139, 0, 0, 489, 2838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 2, 0, 4084, 40, 0, 259, 410, 0, 0, 0, 0, 0, 0, 0, 0, 182, 2090, 49, 0, 0, 0, 0, 2121, 1542, 575, 0, 0, 0, 0, 0, 3947, 3609, 10, 2554, 3, 41, 0, 1482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3463, 4037, 131, 2593, 0, 1, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5614, 6864, 9211, 10526, 4859, 6559, 11646, 10257] avg=8192.0000 max=11646 max_rank=6 min=4859 min_rank=4 imbalance_ratio=1.421631 max_load_reduction=53890 max_load_reduction_pct=0.822296 moved_to_echo=33321 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4078, 214, 118, 3107, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1540, 0, 0, 26, 0, 4096, 127, 1833, 0, 0, 0, 0, 0, 0, 0, 0, 3601, 2694, 0, 1055, 0, 875, 110, 0, 529, 361, 0, 0, 0, 0, 0, 0, 3, 316, 4096, 3113, 0, 0, 0, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 1282, 4015, 32, 1185, 0, 432, 0, 0, 0, 0, 0, 0, 0, 12, 152, 12, 0, 5, 0, 528, 0, 2053, 1281, 575, 0, 0, 0, 0, 0, 49, 0, 0, 2108, 397, 4004, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 1, 0, 4083, 16, 0, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11613, 7622, 9225, 11036, 7006, 4618, 6562, 7854] avg=8192.0000 max=11613 max_rank=0 min=4618 min_rank=5 imbalance_ratio=1.417603 max_load_reduction=53923 max_load_reduction_pct=0.822800 moved_to_echo=26040 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[9344, 5504, 15232, 128, 4224, 3200, 1536, 128, 14080, 9216, 3712, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12288, 896, 14336, 18176, 128, 14080, 4096, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 8064, 3840, 5376, 30848, 8960, 1280, 4736, 2560, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 640, 13952, 20096, 4224, 1792, 1024, 24064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[21504, 128, 384, 3456, 384, 26112, 1408, 12672, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[19072, 7808, 3456, 15104, 512, 11136, 256, 8576, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2176, 512, 24704, 18048, 1024, 17920, 1408, 256, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[30208, 7680, 640, 1280, 9472, 1664, 512, 9216, 3072, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[3809, 0, 10, 38, 122, 0, 4095, 0, 1052, 0, 0, 2887, 0, 0, 0, 160, 0, 81, 584, 0, 4063, 2037, 0, 4094, 113, 2, 0, 30, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 2528, 1903, 1513, 69, 0, 359, 4063, 0, 219, 336, 78, 8, 2368, 2499, 3391, 0, 4094, 1, 0, 0, 3519, 6, 3745, 0, 3388, 69] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 0, 3249, 0, 0, 114, 0, 112, 2, 0, 3443, 0, 228, 0, 4093, 38, 0, 84, 0, 4001, 1476, 1317, 42, 268, 0, 3135, 0, 3845, 0, 0, 0, 0, 4041, 0, 1176, 8, 1478, 1209, 0, 7, 4051, 0, 67, 1185, 1326, 0, 179, 0, 705, 4094, 2, 3885, 0, 2491, 0, 22, 17, 2166, 3816, 0, 0, 4032, 39] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 180, 0, 0, 4, 0, 4089, 0, 4, 0, 0, 3981, 3952, 7, 186, 0, 0, 0, 58, 0, 1456, 4085, 4018, 4090, 4096, 0, 235, 64, 507, 1, 3009, 2031, 0, 3810, 0, 0, 9, 0, 3791, 0, 0, 0, 0, 34, 3842, 4, 0, 0, 4091, 2353, 115, 2342, 0, 930, 1239, 233, 0, 1, 2979, 0, 378, 0, 3079, 253] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3318, 0, 0, 0, 0, 0, 109, 4092, 339, 0, 0, 0, 926, 2, 40, 0, 0, 0, 4086, 0, 4046, 4059, 0, 1116, 643, 4096, 0, 4, 0, 0, 1143, 36, 0, 5, 3636, 4020, 12, 2, 4096, 4, 186, 780, 1, 1917, 2070, 4078, 4093, 44, 0, 1, 0, 3750, 71, 0, 7, 443, 0, 0, 0, 659, 3510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 4077, 1982, 0, 0, 339, 44, 0, 0, 0, 3411, 0, 0, 18, 0, 23, 26, 0, 0, 95, 3151, 0, 4096, 0, 4014, 10, 12, 4096, 431, 0, 455, 0, 3980, 0, 0, 0, 3641, 1281, 0, 0, 3272, 148, 158, 0, 0, 0, 216, 820, 4096, 2375, 3, 4073, 0, 4070, 0, 0, 0, 4069, 2776, 150, 0, 32, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 105, 0, 0, 0, 0, 1995, 0, 0, 0, 4096, 0, 1, 849, 0, 181, 3, 554, 246, 1515, 3870, 2127, 2176, 379, 4095, 3965, 1, 2720, 0, 3672, 4093, 17, 131, 0, 0, 0, 4074, 3805, 230, 0, 4096, 15, 206, 772, 0, 0, 0, 0, 3597, 621, 1, 2565, 2641, 1392, 0, 0, 10, 274, 2, 51, 286, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 14, 4094, 2454, 0, 795, 15, 3, 0, 0, 0, 0, 0, 29, 0, 1712, 2, 0, 0, 4083, 0, 2126, 4090, 4093, 1512, 0, 1938, 562, 4047, 0, 16, 0, 4096, 0, 21, 0, 0, 1812, 0, 0, 0, 0, 0, 0, 2678, 0, 0, 271, 4096, 4095, 141, 3328, 0, 5, 3, 0, 0, 0, 3, 197, 4065, 1486, 3560] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1992, 0, 4096, 4094, 2029, 0, 42, 0, 6, 0, 0, 3435, 0, 0, 0, 0, 4063, 2615, 11, 62, 8, 0, 0, 3912, 216, 3328, 0, 0, 4096, 804, 0, 0, 0, 4096, 0, 0, 0, 0, 3193, 3824, 0, 0, 0, 410, 0, 0, 197, 0, 754, 4096, 3077, 101, 2739, 0, 3945, 1, 0, 0, 0, 267, 4009, 2, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3500, 0, 10, 38, 122, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 0, 0, 2887, 0, 0, 0, 160, 8, 782, 0, 0, 0, 0, 0, 0, 0, 81, 584, 0, 4063, 2037, 0, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 113, 2, 0, 30, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 0, 0, 11, 2528, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 1513, 69, 0, 359, 4063, 0, 219, 336, 2450, 1517, 556, 0, 0, 0, 0, 0, 78, 0, 2368, 2499, 3391, 0, 1894, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3519, 6, 3745, 0, 3388, 69, 1644, 309, 254, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7765, 4889, 8560, 1791, 8284, 11082, 10231, 12934] avg=8192.0000 max=12934 max_rank=7 min=1791 min_rank=3 imbalance_ratio=1.578857 max_load_reduction=52602 max_load_reduction_pct=0.802643 moved_to_echo=42531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3764, 0, 0, 3318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 4092, 339, 0, 0, 0, 926, 4093, 772, 0, 0, 0, 0, 0, 0, 2, 40, 0, 0, 0, 4086, 0, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 1116, 643, 1646, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 36, 0, 5, 3636, 4020, 12, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4, 186, 780, 1, 1917, 2070, 2450, 1499, 509, 0, 0, 0, 0, 0, 4078, 0, 44, 0, 1, 0, 1735, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 443, 0, 0, 0, 659, 3510, 1506, 332, 70, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7082, 10331, 5903, 7468, 8782, 13514, 5929, 6527] avg=8192.0000 max=13514 max_rank=5 min=5903 min_rank=2 imbalance_ratio=1.649658 max_load_reduction=52022 max_load_reduction_pct=0.793793 moved_to_echo=34752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3761, 0, 0, 3249, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 2, 0, 3443, 0, 228, 0, 4093, 705, 8, 0, 0, 0, 0, 0, 0, 38, 0, 84, 0, 4001, 1476, 1317, 19, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 3135, 0, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 1176, 8, 1478, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4051, 0, 67, 1185, 1326, 0, 179, 2300, 15, 338, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 3885, 0, 1153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 17, 2166, 3816, 0, 0, 4032, 39, 1000, 332, 250, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7124, 8591, 6935, 4948, 7662, 9468, 9134, 11674] avg=8192.0000 max=11674 max_rank=7 min=4948 min_rank=3 imbalance_ratio=1.425049 max_load_reduction=53862 max_load_reduction_pct=0.821869 moved_to_echo=37938 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 180, 0, 0, 4, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3981, 3952, 7, 186, 0, 2353, 781, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 1456, 4085, 4018, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 235, 64, 204, 1, 3009, 2031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3574, 0, 0, 9, 0, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 3842, 4, 0, 0, 303, 1515, 168, 0, 0, 0, 0, 0, 4091, 0, 115, 2342, 0, 930, 574, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2979, 0, 378, 0, 3079, 253, 497, 0, 236, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4273, 11264, 11411, 9640, 7374, 5866, 8285, 7423] avg=8192.0000 max=11411 max_rank=2 min=4273 min_rank=0 imbalance_ratio=1.392944 max_load_reduction=54125 max_load_reduction_pct=0.825882 moved_to_echo=28948 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3762, 0, 14, 4094, 2454, 0, 795, 15, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 29, 0, 4096, 781, 0, 0, 0, 0, 0, 0, 1712, 2, 0, 0, 4083, 0, 2126, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1512, 0, 1938, 226, 4047, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 21, 0, 0, 1812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2678, 0, 0, 336, 1515, 0, 0, 0, 0, 0, 0, 271, 0, 4095, 141, 3328, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 197, 4065, 1486, 3560, 2, 332, 254, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11134, 4909, 9717, 11832, 5675, 4529, 7841, 9899] avg=8192.0000 max=11832 max_rank=3 min=4529 min_rank=5 imbalance_ratio=1.444336 max_load_reduction=53704 max_load_reduction_pct=0.819458 moved_to_echo=27944 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3764, 0, 4077, 1982, 0, 0, 339, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3411, 0, 0, 18, 0, 4096, 782, 0, 0, 0, 0, 0, 0, 23, 26, 0, 0, 95, 3151, 0, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 10, 12, 1646, 431, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3733, 0, 0, 0, 3641, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3272, 148, 158, 0, 0, 0, 216, 2450, 1517, 552, 0, 0, 0, 0, 0, 820, 0, 2375, 3, 4073, 0, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 2776, 150, 0, 32, 0, 1635, 332, 247, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10206, 8307, 5092, 6568, 8655, 8313, 9154, 9241] avg=8192.0000 max=10206 max_rank=0 min=5092 min_rank=2 imbalance_ratio=1.245850 max_load_reduction=55330 max_load_reduction_pct=0.844269 moved_to_echo=35363 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3764, 0, 105, 0, 0, 0, 0, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 849, 0, 3597, 415, 0, 0, 0, 0, 0, 0, 181, 3, 554, 246, 1515, 3870, 2127, 955, 0, 0, 0, 0, 0, 0, 0, 0, 379, 4095, 3965, 1, 1093, 0, 3672, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 17, 123, 0, 0, 0, 4074, 3805, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 206, 772, 0, 0, 0, 1627, 806, 189, 0, 0, 0, 0, 0, 0, 0, 621, 1, 2565, 2641, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 274, 2, 51, 286, 11, 0, 559, 332, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5864, 8958, 9451, 17298, 8249, 7711, 6472, 1533] avg=8192.0000 max=17298 max_rank=3 min=1533 min_rank=7 imbalance_ratio=2.111572 max_load_reduction=48238 max_load_reduction_pct=0.736053 moved_to_echo=23965 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1827, 0, 4096, 4094, 2029, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3435, 0, 0, 0, 0, 4096, 750, 0, 0, 0, 0, 0, 0, 4063, 2615, 11, 62, 8, 0, 0, 1708, 0, 0, 0, 0, 0, 0, 0, 0, 216, 3328, 0, 0, 1643, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 0, 0, 0, 0, 3193, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 0, 197, 0, 2453, 1454, 539, 0, 0, 0, 0, 0, 754, 0, 3077, 101, 2739, 0, 1818, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 4009, 2, 0, 16, 1588, 165, 258, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12088, 8287, 8467, 5991, 10855, 5053, 8490, 6305] avg=8192.0000 max=12088 max_rank=0 min=5053 min_rank=5 imbalance_ratio=1.475586 max_load_reduction=53448 max_load_reduction_pct=0.815552 moved_to_echo=30703 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 13568, 6912, 8576, 4480, 12800, 7552, 8448, 2176, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[13312, 13056, 8576, 2688, 9728, 5376, 6784, 6656, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1536, 15616, 256, 1536, 10752, 4096, 2432, 2816, 14464, 9856, 2944, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[10112, 0, 16896, 5120, 20096, 3584, 9728, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6144, 2816, 1408, 384, 15232, 18816, 9600, 11648, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[24192, 256, 8320, 16896, 4736, 0, 9600, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 23936, 128, 1280, 128, 12928, 21760, 6016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1280, 128, 4096, 21632, 3968, 256, 1152, 5248, 23168, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4038, 2974, 2743, 0, 4072, 26, 0, 0, 10, 1, 724, 3588, 109, 1423, 2714, 18, 0, 0, 36, 4068, 0, 0, 18, 4086, 833, 0, 3, 178, 3565, 3, 2047, 4040, 0, 2, 50, 4076, 1, 3469, 33, 3802, 4096, 4070, 317, 0, 0, 4, 12, 0, 0, 0, 4, 22, 165, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 7, 639, 4, 733, 2888, 1, 8, 3993, 5, 0, 0, 14, 0, 4089, 0, 0, 0, 4094, 17, 2008, 0, 812, 73, 0, 0, 0, 4094, 0, 0, 1, 1, 1844, 1, 4086, 0, 3075, 324, 98, 4089, 2, 3624, 4096, 3134, 3443, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2938, 4, 4, 0, 139, 999, 2024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[90, 0, 0, 3931, 0, 257, 11, 3674, 0, 5, 4096, 1161, 0, 0, 0, 633, 36, 379, 1586, 3976, 46, 0, 2095, 0, 0, 3812, 0, 0, 0, 1266, 0, 0, 0, 0, 0, 0, 4096, 314, 65, 1962, 0, 47, 4047, 2565, 3461, 3425, 3577, 4051, 0, 0, 8, 2310, 0, 0, 4032, 58, 1, 7, 0, 0, 0, 0, 4041, 415] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[50, 0, 7, 0, 0, 4095, 320, 0, 550, 2, 0, 0, 0, 0, 3755, 0, 24, 1, 0, 2167, 4059, 11, 1717, 0, 2149, 34, 2257, 3551, 3084, 3874, 0, 0, 44, 0, 28, 0, 4094, 4081, 4009, 0, 4016, 0, 4076, 0, 573, 4096, 2406, 4096, 1028, 104, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 1171, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[231, 0, 0, 315, 0, 80, 2098, 2894, 0, 0, 4058, 2246, 0, 3686, 0, 4049, 4096, 34, 4051, 0, 0, 0, 8, 0, 0, 4060, 0, 4091, 0, 3411, 0, 0, 19, 0, 69, 0, 878, 122, 19, 2563, 767, 3378, 81, 50, 4096, 0, 4016, 4092, 0, 0, 0, 1420, 0, 0, 0, 468, 0, 4074, 0, 0, 0, 0, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4, 258, 11, 748, 0, 719, 1512, 0, 0, 0, 1096, 30, 0, 0, 1988, 0, 87, 3992, 260, 2975, 3865, 0, 2031, 4043, 51, 0, 0, 29, 0, 52, 53, 0, 4096, 0, 4073, 0, 45, 0, 2303, 4052, 0, 3588, 3982, 4094, 1766, 7, 0, 275, 1, 227, 0, 1947, 301, 0, 0, 4039, 3701, 76, 1, 0, 3158, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[28, 0, 0, 32, 0, 24, 93, 2930, 0, 1734, 15, 2, 0, 1764, 163, 1141, 281, 4095, 0, 0, 3770, 1173, 0, 0, 952, 3881, 0, 3369, 0, 1, 1507, 0, 3224, 0, 4032, 0, 4096, 4095, 423, 3626, 0, 4096, 170, 55, 4078, 1020, 3295, 9, 0, 0, 0, 345, 0, 1, 3934, 0, 0, 57, 0, 256, 0, 0, 241, 1528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 75, 0, 0, 905, 3835, 3858, 2109, 0, 147, 0, 0, 2468, 0, 0, 11, 0, 2534, 445, 1208, 4089, 0, 582, 133, 17, 623, 4061, 11, 0, 4, 0, 0, 3603, 0, 4087, 66, 4081, 360, 4063, 3395, 1301, 602, 3772, 2880, 3144, 0, 44, 0, 86, 4039, 0, 0, 0, 0, 0, 2140, 693, 10, 0, 55, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[90, 0, 0, 3931, 0, 257, 11, 3674, 1610, 1151, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 1161, 0, 0, 0, 633, 24, 710, 269, 0, 0, 0, 0, 0, 36, 379, 1586, 3976, 22, 0, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3543, 0, 0, 0, 1266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1717, 314, 65, 1962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 2565, 0, 1564, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2310, 0, 0, 4032, 58, 3577, 2706, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 4041, 415, 47, 755, 769, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10724, 6898, 8094, 4809, 4058, 12227, 12691, 6035] avg=8192.0000 max=12691 max_rank=6 min=4058 min_rank=4 imbalance_ratio=1.549194 max_load_reduction=52845 max_load_reduction_pct=0.806351 moved_to_echo=35011 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4038, 2974, 805, 1278, 0, 0, 0, 0, 0, 0, 2743, 0, 4072, 26, 0, 0, 10, 1, 1465, 788, 287, 0, 0, 0, 0, 0, 724, 3588, 109, 1423, 1249, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3781, 0, 0, 18, 4086, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 178, 3565, 3, 858, 4040, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 1, 3469, 0, 1736, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 4, 12, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 4, 22, 165, 0, 0, 0, 0, 0, 4076, 8, 384, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13191, 9392, 7111, 8754, 8649, 9326, 4454, 4659] avg=8192.0000 max=13191 max_rank=0 min=4454 min_rank=6 imbalance_ratio=1.610229 max_load_reduction=52345 max_load_reduction_pct=0.798721 moved_to_echo=27088 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 258, 11, 748, 0, 719, 1601, 1376, 0, 0, 0, 0, 0, 0, 1512, 0, 0, 0, 1096, 30, 0, 0, 140, 849, 285, 0, 0, 0, 0, 0, 1988, 0, 87, 3992, 120, 2975, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2031, 3758, 51, 0, 0, 29, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 4096, 0, 1707, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 0, 0, 3588, 0, 1869, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 1, 227, 0, 1947, 301, 0, 1766, 3112, 0, 0, 0, 0, 0, 0, 0, 4039, 3701, 76, 1, 0, 3158, 0, 4052, 870, 765, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4717, 3912, 13027, 5921, 5901, 7767, 7629, 16662] avg=8192.0000 max=16662 max_rank=7 min=3912 min_rank=1 imbalance_ratio=2.033936 max_load_reduction=48874 max_load_reduction_pct=0.745758 moved_to_echo=37959 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[50, 0, 7, 0, 0, 4095, 320, 0, 1610, 1376, 0, 0, 0, 0, 0, 0, 550, 2, 0, 0, 0, 0, 3755, 0, 2191, 849, 2, 0, 0, 0, 0, 0, 24, 1, 0, 2167, 1868, 11, 1717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2149, 32, 2257, 3551, 3084, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 28, 0, 1715, 4081, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 4076, 0, 0, 1871, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 104, 0, 3, 0, 0, 0, 0, 2406, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1171, 0, 2, 0, 0, 125, 769, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7458, 7349, 5788, 14947, 9877, 14059, 3989, 2069] avg=8192.0000 max=14947 max_rank=3 min=2069 min_rank=7 imbalance_ratio=1.824585 max_load_reduction=50589 max_load_reduction_pct=0.771927 moved_to_echo=29994 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 7, 639, 4, 733, 2888, 1607, 1053, 0, 0, 0, 0, 0, 0, 1, 8, 3993, 5, 0, 0, 14, 0, 2210, 649, 5, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 1884, 17, 2008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 812, 68, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1844, 1, 1712, 0, 3075, 324, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 2, 3624, 0, 1432, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3443, 3201, 0, 0, 0, 0, 0, 0, 0, 2938, 4, 4, 0, 139, 999, 2024, 4089, 895, 767, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11027, 6885, 7998, 4974, 6958, 9191, 6644, 11859] avg=8192.0000 max=11859 max_rank=7 min=4974 min_rank=3 imbalance_ratio=1.447632 max_load_reduction=53677 max_load_reduction_pct=0.819046 moved_to_echo=34652 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[231, 0, 0, 315, 0, 80, 2098, 2894, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 2246, 0, 3686, 0, 4049, 0, 0, 286, 0, 0, 0, 0, 0, 4096, 34, 4051, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 0, 4091, 0, 3411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 69, 0, 369, 122, 19, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 767, 0, 81, 50, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 0, 0, 0, 468, 4016, 3201, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 16, 3378, 895, 164, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5963, 14325, 8189, 11276, 3161, 4990, 9105, 8527] avg=8192.0000 max=14325 max_rank=1 min=3161 min_rank=4 imbalance_ratio=1.748657 max_load_reduction=51211 max_load_reduction_pct=0.781418 moved_to_echo=25783 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 75, 0, 0, 905, 3835, 1607, 968, 0, 0, 0, 0, 0, 0, 3858, 2109, 0, 147, 0, 0, 2468, 0, 240, 597, 9, 0, 0, 0, 0, 0, 0, 11, 0, 2534, 205, 1208, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 124, 17, 623, 4061, 11, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 0, 1713, 66, 4081, 360, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 1301, 602, 0, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 86, 4039, 0, 0, 0, 0, 3144, 2948, 0, 0, 0, 0, 0, 0, 0, 2140, 693, 10, 0, 55, 0, 0, 3395, 824, 767, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7390, 9428, 8047, 5422, 9823, 7281, 10261, 7884] avg=8192.0000 max=10261 max_rank=6 min=5422 min_rank=3 imbalance_ratio=1.252563 max_load_reduction=55275 max_load_reduction_pct=0.843430 moved_to_echo=35249 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 0, 32, 0, 24, 93, 2930, 1614, 345, 0, 0, 0, 0, 0, 0, 0, 1734, 15, 2, 0, 1764, 163, 1141, 2037, 214, 277, 0, 0, 0, 0, 0, 281, 4095, 0, 0, 1733, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 952, 3604, 0, 3369, 0, 1, 1507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 0, 4032, 0, 1709, 4095, 423, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 55, 0, 461, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 0, 1, 3934, 0, 3295, 3188, 0, 0, 0, 0, 0, 0, 0, 57, 0, 256, 0, 0, 241, 1528, 4096, 890, 773, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5066, 7347, 7282, 9433, 17109, 695, 10763, 7841] avg=8192.0000 max=17109 max_rank=4 min=695 min_rank=5 imbalance_ratio=2.088501 max_load_reduction=48427 max_load_reduction_pct=0.738937 moved_to_echo=36408 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11392, 0, 9728, 14080, 0, 10368, 0, 20480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3456, 256, 17280, 128, 11520, 12800, 11776, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[512, 8192, 128, 4736, 768, 5248, 8320, 19968, 10880, 7552, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 13312, 4608, 384, 1280, 256, 8448, 4096, 23168, 5376, 5248, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1408, 384, 128, 8448, 128, 2048, 8320, 640, 25856, 18944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8704, 3968, 16256, 3712, 1152, 5504, 6528, 5888, 8320, 4736, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6656, 18688, 2432, 11648, 7168, 16896, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11264, 8192, 5888, 14208, 7168, 5504, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[249, 3845, 11, 0, 0, 4084, 0, 194, 191, 0, 12, 3483, 1198, 15, 3, 40, 0, 49, 4088, 4081, 1467, 1416, 4074, 13, 0, 0, 7, 301, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 3992, 1, 0, 3754, 0, 0, 4094, 0, 4095, 4096, 0, 4094, 65, 3309, 0, 0, 0, 0, 3439, 0, 0, 1352, 0, 3, 323] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3399, 4083, 1, 2, 2001, 0, 0, 179, 0, 1332, 0, 0, 1049, 0, 1035, 0, 0, 93, 3962, 0, 331, 4083, 7, 3570, 1, 32, 3870, 0, 2, 5, 0, 0, 0, 0, 131, 28, 21, 0, 0, 4063, 0, 0, 0, 0, 0, 3849, 0, 0, 4095, 4078, 0, 3740, 4096, 0, 0, 231, 0, 4001, 2163, 314, 4096, 18, 0, 1575] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[2259, 3195, 0, 0, 268, 4072, 0, 0, 0, 0, 0, 0, 0, 49, 341, 18, 0, 0, 3977, 1081, 4096, 133, 4096, 0, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 339, 1, 160, 0, 126, 94, 4096, 3673, 49, 4011, 93, 0, 3750, 2611, 3891, 0, 132, 2835, 0, 3563, 213, 0, 0, 1, 4094, 0, 0, 3799, 0, 0, 323] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 184, 3406, 31, 0, 0, 1, 1264, 0, 0, 12, 3876, 66, 0, 115, 0, 510, 0, 0, 0, 2193, 9, 98, 3183, 0, 59, 1291, 0, 3572, 5, 4096, 0, 0, 771, 0, 0, 3820, 0, 0, 0, 0, 874, 2903, 1665, 0, 0, 3779, 3323, 434, 3894, 6, 23, 18, 0, 3850, 4008, 2, 4050, 0, 0, 4048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1, 4096, 0, 1412, 0, 4096, 0, 0, 0, 2166, 0, 0, 1, 2275, 3950, 0, 0, 0, 3564, 194, 4095, 2145, 4048, 368, 0, 0, 0, 0, 149, 0, 0, 2458, 35, 4096, 0, 1567, 2, 4, 137, 4095, 3935, 0, 0, 0, 0, 3722, 0, 165, 0, 0, 0, 3, 2861, 0, 0, 36, 0, 1671, 0, 2, 4096, 0, 1, 4090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[3478, 2, 91, 0, 0, 1, 0, 0, 0, 4019, 244, 28, 6, 224, 3653, 6, 0, 0, 139, 4087, 255, 3916, 4084, 9, 0, 1828, 29, 4060, 0, 0, 0, 0, 0, 1017, 3615, 47, 6, 15, 0, 3985, 18, 0, 4092, 0, 0, 4094, 3492, 3947, 3986, 0, 0, 33, 4096, 0, 0, 0, 0, 2918, 0, 0, 0, 0, 2, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[3578, 1, 3880, 0, 0, 47, 0, 224, 0, 4027, 48, 41, 4072, 4075, 4, 12, 4, 0, 1, 3373, 10, 0, 4064, 3, 0, 4039, 1, 0, 0, 0, 0, 0, 3328, 18, 0, 26, 25, 0, 0, 4056, 1, 0, 4095, 733, 0, 3634, 4051, 4095, 11, 3197, 0, 54, 1782, 0, 719, 0, 0, 2044, 0, 0, 1685, 0, 5, 473] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[8, 2774, 0, 2828, 4076, 0, 0, 0, 289, 35, 33, 4014, 4077, 4078, 2505, 2, 0, 0, 8, 44, 251, 3902, 4096, 1318, 1, 61, 4039, 0, 0, 0, 0, 0, 45, 21, 0, 4096, 0, 2, 59, 4096, 1259, 0, 3822, 1568, 0, 0, 358, 2, 0, 39, 0, 0, 1807, 47, 0, 57, 0, 4094, 0, 1728, 3995, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[249, 3738, 11, 0, 0, 4084, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 12, 3483, 1198, 15, 3, 40, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4088, 4081, 1467, 1416, 432, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 301, 0, 0, 0, 0, 3642, 1645, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 3992, 428, 339, 0, 0, 0, 0, 0, 0, 1, 0, 3754, 0, 0, 816, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 65, 3309, 0, 0, 0, 107, 21, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 0, 1352, 0, 3, 323, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 6147, 11546, 5595, 8857, 8666, 11692, 4757] avg=8192.0000 max=11692 max_rank=6 min=4757 min_rank=7 imbalance_ratio=1.427246 max_load_reduction=53844 max_load_reduction_pct=0.821594 moved_to_echo=33972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3399, 3969, 1, 2, 2001, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1332, 0, 0, 1049, 0, 1035, 0, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 93, 3962, 0, 331, 4083, 1, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 3870, 0, 2, 5, 0, 0, 6, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 131, 28, 21, 0, 0, 4063, 402, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4078, 0, 3740, 4096, 0, 0, 231, 114, 25, 0, 0, 0, 0, 0, 0, 0, 3582, 2163, 314, 4096, 18, 0, 1575, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9551, 4549, 12040, 5462, 5039, 768, 16379, 11748] avg=8192.0000 max=16379 max_rank=6 min=768 min_rank=5 imbalance_ratio=1.999390 max_load_reduction=49157 max_load_reduction_pct=0.750076 moved_to_echo=33934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 3982, 0, 1412, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 1, 2275, 3950, 0, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3564, 194, 4095, 2145, 429, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 2458, 3619, 1495, 0, 0, 0, 0, 0, 0, 35, 4096, 0, 1567, 2, 4, 137, 4095, 389, 164, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 743, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2861, 0, 0, 36, 114, 10, 0, 0, 0, 0, 0, 0, 0, 1497, 0, 2, 4096, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9491, 9487, 10795, 7721, 10489, 4843, 3024, 9686] avg=8192.0000 max=10795 max_rank=2 min=3024 min_rank=6 imbalance_ratio=1.317749 max_load_reduction=54741 max_load_reduction_pct=0.835281 moved_to_echo=28042 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 3982, 0, 0, 184, 3406, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1264, 0, 0, 12, 3876, 66, 854, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 510, 0, 0, 0, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 9, 98, 3183, 0, 59, 1291, 0, 3572, 0, 1166, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 771, 0, 0, 3820, 303, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 580, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 3323, 434, 3894, 6, 23, 18, 114, 24, 0, 0, 0, 0, 0, 0, 0, 3447, 4008, 2, 4050, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7604, 6073, 2818, 9378, 9374, 3119, 11615, 15555] avg=8192.0000 max=15555 max_rank=7 min=2818 min_rank=2 imbalance_ratio=1.898804 max_load_reduction=49981 max_load_reduction_pct=0.762650 moved_to_echo=39663 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2259, 3106, 0, 0, 268, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 341, 18, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 1081, 4096, 133, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4094, 0, 0, 0, 0, 0, 3662, 1506, 0, 0, 0, 0, 0, 0, 0, 339, 1, 160, 0, 126, 94, 4096, 392, 403, 0, 0, 0, 0, 0, 0, 3673, 49, 4011, 93, 0, 749, 2611, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 2835, 0, 3563, 213, 0, 0, 89, 26, 0, 0, 0, 0, 0, 0, 1, 3665, 0, 0, 3799, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9705, 1511, 9721, 9265, 5611, 15077, 6858, 7788] avg=8192.0000 max=15077 max_rank=5 min=1511 min_rank=1 imbalance_ratio=1.840454 max_load_reduction=50459 max_load_reduction_pct=0.769943 moved_to_echo=35334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[8, 2697, 0, 2828, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 35, 33, 4014, 4077, 4078, 2505, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 44, 251, 3902, 434, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 1, 61, 4039, 0, 0, 0, 0, 0, 3662, 0, 0, 0, 0, 0, 0, 0, 45, 21, 0, 4096, 0, 2, 59, 4096, 0, 403, 0, 0, 0, 0, 0, 0, 1259, 0, 3822, 1568, 0, 0, 358, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 1807, 47, 0, 57, 77, 26, 0, 0, 0, 0, 0, 0, 0, 3665, 0, 1728, 3995, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9609, 15033, 5957, 7763, 8722, 7009, 2053, 9390] avg=8192.0000 max=15033 max_rank=1 min=2053 min_rank=6 imbalance_ratio=1.835083 max_load_reduction=50503 max_load_reduction_pct=0.770615 moved_to_echo=27174 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3578, 1, 3880, 0, 0, 47, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 48, 41, 4072, 4075, 4, 12, 1069, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 3373, 10, 0, 431, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 1, 0, 0, 0, 0, 0, 3633, 1460, 0, 0, 0, 0, 0, 0, 3328, 18, 0, 26, 25, 0, 0, 4056, 380, 201, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 733, 0, 725, 4051, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3197, 0, 54, 1782, 0, 719, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1830, 0, 0, 1685, 0, 5, 473, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7730, 13348, 3822, 9133, 8034, 13700, 5776, 3993] avg=8192.0000 max=13700 max_rank=5 min=3822 min_rank=2 imbalance_ratio=1.672363 max_load_reduction=51836 max_load_reduction_pct=0.790955 moved_to_echo=31503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3478, 0, 91, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 244, 28, 6, 224, 3653, 6, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 4087, 255, 3916, 431, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1828, 29, 4060, 0, 0, 0, 0, 3653, 1649, 0, 0, 0, 0, 0, 0, 0, 1017, 3615, 47, 6, 15, 0, 3985, 432, 293, 0, 0, 0, 0, 0, 0, 18, 0, 4092, 0, 0, 805, 3492, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 0, 33, 4096, 0, 0, 0, 2, 22, 0, 0, 0, 0, 0, 0, 0, 2603, 0, 0, 0, 0, 2, 14, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3570, 9388, 8837, 11219, 9410, 12354, 8139, 2619] avg=8192.0000 max=12354 max_rank=5 min=2619 min_rank=7 imbalance_ratio=1.508057 max_load_reduction=53182 max_load_reduction_pct=0.811493 moved_to_echo=32522 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8960, 128, 19840, 2432, 896, 5248, 12288, 16256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12288, 11264, 10368, 4352, 25472, 384, 768, 384, 640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 23424, 6272, 2048, 23168, 128, 128, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 11648, 1664, 7680, 10496, 10752, 15488, 256, 7680, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 384, 15744, 13440, 10624, 15616, 2688, 7552, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[13056, 21504, 4096, 4352, 6656, 15744, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 6144, 15232, 4480, 256, 1408, 0, 6144, 21888, 10496, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3456, 13696, 3840, 6016, 896, 256, 384, 32256, 2816, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[95, 2843, 823, 402, 1062, 358, 760, 999, 1800, 147, 1305, 480, 1068, 628, 165, 414, 88, 385, 1389, 1140, 1024, 181, 2725, 2147, 572, 877, 763, 795, 2007, 433, 2311, 1887, 2838, 377, 627, 937, 91, 2003, 1556, 655, 2843, 749, 1558, 958, 1433, 1889, 828, 732, 257, 500, 250, 2268, 1092, 478, 622, 66, 573, 504, 1045, 1801, 1655, 373, 955, 950] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[96, 2728, 371, 324, 1537, 517, 622, 1288, 953, 183, 1252, 174, 1043, 1092, 137, 339, 67, 592, 662, 676, 1376, 107, 3356, 2079, 366, 1073, 717, 1485, 796, 528, 1465, 1885, 3212, 472, 192, 447, 223, 2732, 2901, 323, 2480, 316, 1572, 824, 1391, 1929, 1348, 1603, 386, 359, 283, 2636, 1009, 312, 423, 104, 998, 205, 1035, 2209, 1081, 657, 1211, 777] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[101, 3081, 489, 469, 1104, 371, 1026, 827, 984, 97, 2055, 113, 909, 781, 296, 429, 172, 483, 1213, 440, 1007, 388, 3183, 2427, 627, 1581, 845, 1231, 1400, 420, 1803, 1914, 3252, 757, 587, 602, 26, 2431, 1622, 596, 2545, 426, 1654, 980, 1345, 1613, 1076, 1213, 369, 308, 390, 1928, 791, 658, 318, 86, 548, 158, 1201, 2252, 1224, 612, 492, 1210] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[194, 2873, 533, 416, 974, 205, 846, 968, 1227, 165, 1759, 606, 691, 833, 512, 405, 56, 600, 935, 596, 837, 354, 2864, 2019, 571, 1069, 932, 1204, 1358, 583, 1954, 1964, 2840, 559, 838, 540, 179, 2100, 1876, 916, 2543, 701, 1801, 952, 1180, 1960, 804, 1025, 220, 447, 333, 2166, 811, 687, 361, 76, 852, 555, 1197, 1848, 1595, 366, 1121, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[85, 2816, 507, 404, 1510, 407, 765, 1223, 1139, 154, 1631, 334, 830, 723, 179, 610, 298, 432, 994, 1280, 851, 214, 3157, 2241, 261, 1158, 921, 1135, 1281, 469, 1580, 1934, 3045, 424, 297, 497, 127, 1980, 2355, 662, 2810, 474, 1484, 1028, 1672, 1457, 1176, 723, 298, 316, 348, 3103, 974, 393, 708, 47, 438, 234, 895, 1688, 1466, 509, 1301, 1084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[72, 2792, 422, 300, 1097, 488, 611, 1356, 1257, 242, 1270, 181, 1309, 1289, 219, 382, 126, 451, 1807, 718, 1021, 298, 2489, 1927, 375, 1192, 901, 1206, 1111, 378, 2245, 2465, 2630, 708, 549, 422, 223, 2538, 1997, 610, 2420, 251, 1681, 630, 1541, 1494, 1263, 1620, 295, 341, 469, 1312, 1100, 625, 604, 65, 790, 440, 776, 1833, 1933, 796, 558, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[131, 3302, 353, 287, 1377, 369, 692, 1349, 1133, 158, 1458, 179, 471, 770, 384, 432, 78, 660, 1097, 704, 948, 244, 2178, 2199, 465, 1108, 708, 1504, 1326, 455, 1777, 1948, 3103, 466, 657, 611, 47, 2455, 2411, 273, 3015, 715, 1364, 866, 1638, 1772, 800, 1140, 260, 519, 394, 2302, 1277, 541, 550, 63, 387, 406, 849, 2067, 1283, 478, 1173, 1410] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[148, 2766, 989, 219, 1506, 318, 921, 1286, 1191, 175, 1118, 320, 1027, 525, 156, 355, 133, 566, 879, 1207, 1120, 53, 3210, 1876, 587, 879, 1122, 1152, 1806, 236, 1581, 1907, 2886, 713, 324, 607, 178, 2234, 2283, 474, 2878, 441, 1493, 1075, 1467, 1461, 1121, 765, 414, 496, 382, 2893, 802, 475, 458, 152, 565, 225, 913, 1742, 1245, 451, 1315, 1274] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[95, 2843, 823, 402, 1062, 358, 760, 999, 550, 81, 0, 0, 0, 0, 0, 0, 1800, 147, 1305, 480, 1068, 628, 165, 414, 1493, 992, 0, 0, 0, 0, 0, 0, 88, 385, 1389, 1140, 1024, 181, 2081, 2147, 0, 0, 0, 0, 0, 0, 0, 0, 572, 877, 763, 795, 2007, 433, 2311, 814, 0, 0, 0, 0, 0, 0, 0, 0, 1342, 377, 627, 937, 91, 2003, 1556, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 749, 1558, 958, 1433, 1759, 828, 732, 0, 0, 0, 0, 0, 0, 0, 0, 257, 500, 250, 2268, 1092, 478, 622, 66, 2843, 3, 0, 0, 0, 0, 0, 0, 573, 504, 1045, 1801, 1655, 373, 955, 950, 130, 94, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7973, 8492, 8435, 8572, 7588, 8017, 8379, 8080] avg=8192.0000 max=8572 max_rank=3 min=7588 min_rank=4 imbalance_ratio=1.046387 max_load_reduction=56964 max_load_reduction_pct=0.869202 moved_to_echo=32064 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[96, 2728, 371, 324, 1537, 517, 622, 1288, 678, 81, 0, 0, 0, 0, 0, 0, 953, 183, 1252, 174, 1043, 1092, 137, 339, 1689, 991, 0, 0, 0, 0, 0, 0, 67, 592, 662, 676, 1376, 107, 2562, 2079, 0, 0, 0, 0, 0, 0, 0, 0, 366, 1073, 717, 1485, 796, 528, 1465, 813, 0, 0, 0, 0, 0, 0, 0, 0, 1519, 472, 192, 447, 223, 2732, 2901, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 1572, 824, 1391, 1796, 1348, 1603, 0, 0, 0, 0, 0, 0, 0, 0, 386, 359, 283, 2636, 1009, 312, 423, 104, 2480, 4, 0, 0, 0, 0, 0, 0, 998, 205, 1035, 2209, 1081, 657, 1211, 777, 133, 116, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 7853, 8121, 7243, 8809, 8850, 7996, 8422] avg=8192.0000 max=8850 max_rank=5 min=7243 min_rank=3 imbalance_ratio=1.080322 max_load_reduction=56686 max_load_reduction_pct=0.864960 moved_to_echo=34077 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[194, 2873, 533, 416, 974, 205, 846, 968, 578, 84, 0, 0, 0, 0, 0, 0, 1227, 165, 1759, 606, 691, 833, 512, 405, 1494, 1033, 0, 0, 0, 0, 0, 0, 56, 600, 935, 596, 837, 354, 2187, 2019, 0, 0, 0, 0, 0, 0, 0, 0, 571, 1069, 932, 1204, 1358, 583, 1954, 847, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 559, 838, 540, 179, 2100, 1876, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 1801, 952, 1180, 1825, 804, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 220, 447, 333, 2166, 811, 687, 361, 76, 2543, 3, 0, 0, 0, 0, 0, 0, 852, 555, 1197, 1848, 1595, 366, 1121, 984, 135, 99, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7671, 8725, 7584, 8518, 8351, 8288, 7647, 8752] avg=8192.0000 max=8752 max_rank=7 min=7584 min_rank=2 imbalance_ratio=1.068359 max_load_reduction=56784 max_load_reduction_pct=0.866455 moved_to_echo=33038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[101, 3081, 489, 469, 1104, 371, 1026, 827, 643, 82, 0, 0, 0, 0, 0, 0, 984, 97, 2055, 113, 909, 781, 296, 429, 1710, 1007, 0, 0, 0, 0, 0, 0, 172, 483, 1213, 440, 1007, 388, 2430, 2427, 0, 0, 0, 0, 0, 0, 0, 0, 627, 1581, 845, 1231, 1400, 420, 1803, 825, 0, 0, 0, 0, 0, 0, 0, 0, 1538, 757, 587, 602, 26, 2431, 1622, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 1654, 980, 1345, 1502, 1076, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 369, 308, 390, 1928, 791, 658, 318, 86, 2545, 4, 0, 0, 0, 0, 0, 0, 548, 158, 1201, 2252, 1224, 612, 492, 1210, 111, 110, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8381, 8560, 8732, 8159, 8196, 7397, 7918] avg=8192.0000 max=8732 max_rank=3 min=7397 min_rank=6 imbalance_ratio=1.065918 max_load_reduction=56804 max_load_reduction_pct=0.866760 moved_to_echo=31670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[85, 2816, 507, 404, 1510, 407, 765, 1223, 638, 83, 0, 0, 0, 0, 0, 0, 1139, 154, 1631, 334, 830, 723, 179, 610, 1602, 1017, 0, 0, 0, 0, 0, 0, 298, 432, 994, 1280, 851, 214, 2410, 2241, 0, 0, 0, 0, 0, 0, 0, 0, 261, 1158, 921, 1135, 1281, 469, 1580, 834, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 424, 297, 497, 127, 1980, 2355, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 1484, 1028, 1672, 1357, 1176, 723, 0, 0, 0, 0, 0, 0, 0, 0, 298, 316, 348, 3103, 974, 393, 708, 47, 2810, 4, 0, 0, 0, 0, 0, 0, 438, 234, 895, 1688, 1466, 509, 1301, 1084, 100, 109, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8438, 8219, 8720, 7639, 7781, 7914, 9001, 7824] avg=8192.0000 max=9001 max_rank=6 min=7639 min_rank=3 imbalance_ratio=1.098755 max_load_reduction=56535 max_load_reduction_pct=0.862656 moved_to_echo=32520 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[72, 2792, 422, 300, 1097, 488, 611, 1356, 503, 106, 0, 0, 0, 0, 0, 0, 1257, 242, 1270, 181, 1309, 1289, 219, 382, 1383, 1296, 0, 0, 0, 0, 0, 0, 126, 451, 1807, 718, 1021, 298, 1900, 1927, 0, 0, 0, 0, 0, 0, 0, 0, 375, 1192, 901, 1206, 1111, 378, 2245, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1244, 708, 549, 422, 223, 2538, 1997, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 1681, 630, 1541, 1391, 1263, 1620, 0, 0, 0, 0, 0, 0, 0, 0, 295, 341, 469, 1312, 1100, 625, 604, 65, 2420, 3, 0, 0, 0, 0, 0, 0, 790, 440, 776, 1833, 1933, 796, 558, 1025, 103, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7747, 8828, 8248, 8471, 8291, 8377, 7234, 8340] avg=8192.0000 max=8828 max_rank=1 min=7234 min_rank=6 imbalance_ratio=1.077637 max_load_reduction=56708 max_load_reduction_pct=0.865295 moved_to_echo=32242 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[131, 3302, 353, 287, 1377, 369, 692, 1349, 440, 83, 0, 0, 0, 0, 0, 0, 1133, 158, 1458, 179, 471, 770, 384, 432, 1632, 1024, 0, 0, 0, 0, 0, 0, 78, 660, 1097, 704, 948, 244, 1663, 2199, 0, 0, 0, 0, 0, 0, 0, 0, 465, 1108, 708, 1504, 1326, 455, 1777, 841, 0, 0, 0, 0, 0, 0, 0, 0, 1467, 466, 657, 611, 47, 2455, 2411, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 715, 1364, 866, 1638, 1650, 800, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 260, 519, 394, 2302, 1277, 541, 550, 63, 3015, 4, 0, 0, 0, 0, 0, 0, 387, 406, 849, 2067, 1283, 478, 1173, 1410, 122, 75, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8383, 7641, 7593, 8184, 8387, 8173, 8925, 8250] avg=8192.0000 max=8925 max_rank=6 min=7593 min_rank=2 imbalance_ratio=1.089478 max_load_reduction=56611 max_load_reduction_pct=0.863815 moved_to_echo=33735 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[148, 2766, 989, 219, 1506, 318, 921, 1286, 651, 85, 0, 0, 0, 0, 0, 0, 1191, 175, 1118, 320, 1027, 525, 156, 355, 1522, 1008, 0, 0, 0, 0, 0, 0, 133, 566, 879, 1207, 1120, 53, 2441, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 587, 879, 1122, 1152, 1806, 236, 1581, 814, 0, 0, 0, 0, 0, 0, 0, 0, 1357, 713, 324, 607, 178, 2234, 2283, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 441, 1493, 1075, 1467, 1359, 1121, 765, 0, 0, 0, 0, 0, 0, 0, 0, 414, 496, 382, 2893, 802, 475, 458, 152, 2878, 7, 0, 0, 0, 0, 0, 0, 565, 225, 913, 1742, 1245, 451, 1315, 1274, 102, 118, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8889, 7397, 8275, 8177, 8170, 7721, 8957, 7950] avg=8192.0000 max=8957 max_rank=6 min=7397 min_rank=1 imbalance_ratio=1.093384 max_load_reduction=56579 max_load_reduction_pct=0.863327 moved_to_echo=32798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 4096, 12672, 7424, 11776, 12672, 8448, 8832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[3840, 8960, 6912, 9728, 11136, 3584, 14720, 6912, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[9728, 1408, 11904, 2432, 7424, 6656, 2048, 3456, 12544, 8448, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1024, 23296, 4608, 2944, 10240, 3072, 6272, 9344, 4736, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[5248, 2816, 7936, 15488, 11520, 4352, 8192, 8832, 1024, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[11264, 4480, 4096, 4736, 1152, 18560, 17024, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2560, 3328, 2944, 18688, 7936, 4224, 4096, 768, 21632, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1024, 4224, 9088, 6784, 8192, 1920, 17792, 17024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2395, 2, 1425, 3479, 49, 24, 4011, 14, 31, 2398, 0, 0, 2, 46, 618, 40, 37, 4006, 2830, 1713, 35, 1198, 0, 481, 1158, 15, 108, 2143, 2393, 299, 38, 866, 894, 738, 0, 3572, 3874, 2847, 0, 64, 2806, 0, 3, 1145, 71, 107, 1443, 125, 4014, 2548, 3916, 0, 24, 0, 91, 0, 7, 18, 0, 153, 2, 883, 3825, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2317, 5, 31, 2613, 79, 961, 4069, 0, 1, 3824, 6, 2, 31, 1, 71, 268, 0, 3989, 623, 3962, 20, 157, 0, 2934, 1476, 1, 0, 3339, 3964, 2, 12, 332, 13, 3064, 0, 4070, 172, 2757, 4, 22, 132, 0, 194, 1862, 10, 638, 1018, 39, 2906, 2990, 3968, 0, 172, 0, 60, 6, 161, 0, 364, 48, 0, 191, 4095, 1490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4075, 10, 123, 3570, 15, 388, 3785, 11, 2, 3631, 12, 9, 13, 249, 65, 26, 24, 4095, 696, 370, 51, 479, 23, 2750, 3145, 2, 185, 3060, 4081, 1115, 16, 891, 614, 1317, 4, 4013, 397, 2812, 0, 1786, 2781, 0, 43, 672, 47, 122, 52, 246, 2973, 3244, 3242, 240, 26, 0, 4, 8, 13, 4, 1, 26, 0, 238, 3386, 258] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4012, 0, 391, 3931, 41, 89, 4080, 0, 9, 2415, 1, 2, 0, 61, 0, 0, 30, 2992, 3300, 66, 77, 3527, 12, 2329, 1430, 225, 0, 4077, 4075, 285, 0, 17, 15, 2194, 0, 4083, 764, 3749, 0, 40, 0, 0, 53, 333, 11, 0, 2796, 2, 683, 3987, 4024, 34, 8, 8, 208, 60, 11, 0, 14, 165, 2, 699, 4058, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4032, 2, 408, 3963, 6, 28, 4093, 0, 26, 1375, 0, 1, 0, 13, 1, 48, 0, 3972, 161, 1950, 130, 838, 0, 1091, 1264, 492, 5, 3948, 3987, 10, 4, 217, 3386, 1407, 1, 4058, 119, 2914, 1, 32, 1508, 34, 624, 2573, 729, 540, 955, 113, 2986, 3337, 3711, 3, 22, 6, 0, 7, 1, 9, 492, 66, 219, 15, 3219, 384] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2974, 35, 1099, 3740, 61, 1673, 4090, 5, 0, 3987, 0, 31, 0, 134, 23, 69, 1, 4060, 1392, 1911, 6, 851, 0, 0, 3186, 11, 3, 4051, 4053, 23, 0, 125, 1108, 392, 18, 3995, 2138, 2421, 77, 373, 36, 0, 9, 2541, 33, 59, 4, 0, 4037, 564, 3301, 43, 9, 80, 4, 13, 0, 0, 622, 191, 4, 498, 3979, 1393] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[3031, 224, 29, 3790, 189, 504, 1018, 0, 0, 4050, 0, 19, 1, 36, 6, 9, 1, 4069, 1215, 1, 399, 972, 2, 443, 3993, 21, 1, 4007, 3831, 2, 5, 117, 16, 751, 0, 3668, 1006, 3853, 4, 1, 2962, 0, 151, 3497, 351, 1362, 83, 215, 3686, 217, 2539, 0, 346, 1, 16, 210, 5, 1, 218, 102, 40, 1232, 3951, 3067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[3651, 29, 41, 924, 262, 22, 4024, 0, 0, 3287, 0, 0, 1, 2, 0, 2, 12, 4063, 1338, 1415, 29, 682, 0, 378, 4089, 34, 8, 3791, 3961, 24, 0, 16, 138, 372, 7, 3540, 2974, 3983, 0, 248, 1575, 0, 20, 2498, 379, 1214, 62, 18, 2315, 3342, 790, 547, 474, 0, 0, 0, 227, 0, 1397, 19, 1, 2704, 2032, 2575] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4075, 10, 123, 3570, 15, 388, 619, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3631, 12, 9, 13, 249, 65, 26, 3166, 1835, 0, 0, 0, 0, 0, 0, 24, 2962, 696, 370, 51, 479, 23, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 3145, 2, 185, 3060, 1595, 1115, 16, 891, 0, 0, 0, 0, 0, 0, 0, 0, 614, 1317, 4, 1250, 397, 2812, 0, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 2781, 0, 43, 672, 47, 122, 52, 246, 2349, 928, 0, 0, 0, 0, 0, 0, 2973, 3244, 2414, 240, 26, 0, 4, 8, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4, 1, 26, 0, 238, 3386, 258, 1133, 828, 137, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8811, 9008, 7355, 10009, 8180, 7240, 8909, 6024] avg=8192.0000 max=10009 max_rank=3 min=6024 min_rank=7 imbalance_ratio=1.221802 max_load_reduction=55527 max_load_reduction_pct=0.847275 moved_to_echo=30353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4012, 0, 391, 3931, 41, 89, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2415, 1, 2, 0, 61, 0, 0, 3413, 1867, 0, 0, 0, 0, 0, 0, 30, 2164, 3300, 66, 77, 3527, 12, 2329, 0, 0, 0, 0, 0, 0, 0, 0, 1430, 225, 0, 4077, 1592, 285, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2194, 0, 1272, 764, 3749, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 333, 11, 0, 2796, 2, 2346, 944, 0, 0, 0, 0, 0, 0, 683, 3987, 2996, 34, 8, 8, 208, 60, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 14, 165, 2, 699, 4058, 61, 828, 1028, 137, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9131, 7768, 11505, 7626, 8034, 6485, 7984, 7003] avg=8192.0000 max=11505 max_rank=2 min=6485 min_rank=5 imbalance_ratio=1.404419 max_load_reduction=54031 max_load_reduction_pct=0.824448 moved_to_echo=29506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2395, 2, 1425, 3479, 49, 24, 656, 14, 0, 0, 0, 0, 0, 0, 0, 0, 31, 2398, 0, 0, 2, 46, 618, 40, 3355, 1633, 0, 0, 0, 0, 0, 0, 37, 2897, 2830, 1713, 35, 1198, 0, 481, 0, 0, 0, 0, 0, 0, 0, 0, 1158, 15, 108, 2143, 936, 299, 38, 866, 0, 0, 0, 0, 0, 0, 0, 0, 894, 738, 0, 1113, 3874, 2847, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 2806, 0, 3, 1145, 71, 107, 1443, 125, 1377, 826, 0, 0, 0, 0, 0, 0, 4014, 2548, 2916, 0, 24, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 18, 0, 153, 2, 883, 3825, 512, 1109, 1000, 80, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8044, 8123, 9191, 5563, 9530, 7903, 9593, 7589] avg=8192.0000 max=9593 max_rank=6 min=5563 min_rank=3 imbalance_ratio=1.171021 max_load_reduction=55943 max_load_reduction_pct=0.853622 moved_to_echo=34615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2317, 5, 31, 2613, 79, 961, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3824, 6, 2, 31, 1, 71, 268, 3404, 1861, 0, 0, 0, 0, 0, 0, 0, 2885, 623, 3962, 20, 157, 0, 2934, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 1, 0, 3339, 1549, 2, 12, 332, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3064, 0, 1268, 172, 2757, 4, 22, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 194, 1862, 10, 638, 1018, 39, 2282, 941, 0, 0, 0, 0, 0, 0, 2906, 2990, 2955, 0, 172, 0, 60, 6, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 364, 48, 0, 191, 4095, 1490, 1104, 1013, 133, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6671, 9469, 10581, 6711, 7300, 7116, 9089, 8599] avg=8192.0000 max=10581 max_rank=2 min=6671 min_rank=0 imbalance_ratio=1.291626 max_load_reduction=54955 max_load_reduction_pct=0.838547 moved_to_echo=32104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2974, 35, 1099, 3740, 61, 1673, 668, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 31, 0, 134, 23, 69, 3422, 1826, 0, 0, 0, 0, 0, 0, 1, 2936, 1392, 1911, 6, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3186, 11, 3, 4051, 1584, 23, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 392, 18, 1246, 2138, 2421, 77, 373, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 9, 2541, 33, 59, 4, 0, 2333, 923, 0, 0, 0, 0, 0, 0, 4037, 564, 2458, 43, 9, 80, 4, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 191, 4, 498, 3979, 1393, 1124, 843, 136, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10255, 9492, 7097, 8983, 7773, 5938, 7208, 8790] avg=8192.0000 max=10255 max_rank=0 min=5938 min_rank=5 imbalance_ratio=1.251831 max_load_reduction=55281 max_load_reduction_pct=0.843521 moved_to_echo=29709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4032, 2, 408, 3963, 6, 28, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 1375, 0, 1, 0, 13, 1, 48, 3424, 1855, 0, 0, 0, 0, 0, 0, 0, 2873, 161, 1950, 130, 838, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 492, 5, 3948, 1558, 10, 4, 217, 0, 0, 0, 0, 0, 0, 0, 0, 3386, 1407, 1, 1265, 119, 2914, 1, 32, 0, 0, 0, 0, 0, 0, 0, 0, 1508, 34, 624, 2573, 729, 540, 955, 113, 2295, 938, 0, 0, 0, 0, 0, 0, 2986, 3337, 2763, 3, 22, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 492, 66, 219, 15, 3219, 384, 1099, 948, 134, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9108, 6743, 7043, 7498, 9125, 10309, 9124, 6586] avg=8192.0000 max=10309 max_rank=5 min=6586 min_rank=7 imbalance_ratio=1.258423 max_load_reduction=55227 max_load_reduction_pct=0.842697 moved_to_echo=35144 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3031, 224, 29, 3790, 189, 504, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 19, 1, 36, 6, 9, 851, 1677, 0, 0, 0, 0, 0, 0, 1, 2943, 1215, 1, 399, 972, 2, 443, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 21, 1, 4007, 1497, 2, 5, 117, 0, 0, 0, 0, 0, 0, 0, 0, 16, 751, 0, 1143, 1006, 3853, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2962, 0, 151, 3497, 351, 1362, 83, 215, 2205, 848, 0, 0, 0, 0, 0, 0, 3686, 217, 1891, 0, 346, 1, 16, 210, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 218, 102, 40, 1232, 3951, 3067, 1126, 648, 129, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7934, 6649, 5976, 9643, 6774, 11674, 6367, 10519] avg=8192.0000 max=11674 max_rank=5 min=5976 min_rank=2 imbalance_ratio=1.425049 max_load_reduction=53862 max_load_reduction_pct=0.821869 moved_to_echo=35334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3651, 29, 41, 924, 262, 22, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3287, 0, 0, 1, 2, 0, 2, 3371, 1621, 0, 0, 0, 0, 0, 0, 12, 2934, 1338, 1415, 29, 682, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 34, 8, 3791, 1541, 24, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 138, 372, 7, 1098, 2974, 3983, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 1575, 0, 20, 2498, 379, 1214, 62, 18, 2284, 821, 0, 0, 0, 0, 0, 0, 2315, 3342, 584, 547, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 1397, 19, 1, 2704, 2032, 2575, 1129, 206, 136, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5582, 8284, 6788, 9503, 8820, 8871, 7262, 10426] avg=8192.0000 max=10426 max_rank=7 min=5582 min_rank=0 imbalance_ratio=1.272705 max_load_reduction=55110 max_load_reduction_pct=0.840912 moved_to_echo=35379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[19840, 896, 384, 28416, 11904, 1792, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 128, 3200, 896, 384, 6528, 28672, 9856, 8704, 6528, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11904, 128, 1152, 15232, 1664, 4096, 6528, 768, 17536, 7296, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 22656, 11648, 11392, 768, 8704, 128, 10496, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26496, 384, 3584, 26112, 768, 3712, 4864, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6272, 10240, 128, 9728, 11520, 25344, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 25088, 128, 128, 128, 640, 896, 512, 24448, 14208, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23680, 20352, 19072, 896, 1152, 128, 384, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4001, 0, 3997, 13, 0, 3, 4008, 4067, 3700, 76, 270, 102, 0, 12, 30, 443, 0, 1, 388, 38, 528, 64, 0, 1058, 41, 1006, 39, 2901, 4080, 52, 3806, 0, 63, 46, 4026, 0, 0, 860, 20, 8, 60, 2565, 932, 3070, 1, 0, 3697, 170, 9, 0, 38, 4096, 2827, 0, 2, 0, 0, 3037, 28, 1818, 3328, 19, 87, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3348, 667, 4086, 113, 0, 0, 4, 3811, 3892, 111, 2220, 0, 1, 909, 0, 0, 0, 0, 316, 758, 28, 3921, 177, 1, 0, 0, 0, 436, 107, 3094, 4087, 37, 409, 201, 3828, 9, 1236, 389, 0, 1, 16, 3014, 6, 4000, 0, 0, 3465, 168, 0, 0, 4, 205, 225, 78, 3897, 20, 0, 3749, 0, 4000, 3729, 3, 0, 760] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3748, 0, 3742, 0, 16, 1, 3992, 3883, 4066, 0, 0, 15, 4, 0, 1, 12, 129, 56, 4, 0, 4, 267, 0, 3931, 6, 494, 158, 403, 1082, 0, 4063, 181, 10, 2124, 4092, 0, 92, 3961, 100, 8, 475, 1217, 9, 4096, 2, 117, 936, 611, 1, 39, 1, 4077, 3280, 433, 85, 3, 38, 4084, 1, 1804, 3575, 7, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[620, 1, 4043, 10, 484, 0, 0, 3911, 3590, 9, 1, 0, 2, 9, 4017, 1151, 0, 0, 0, 184, 0, 2131, 339, 205, 0, 0, 173, 0, 0, 3309, 4045, 1763, 1, 50, 3938, 3671, 0, 3591, 17, 638, 280, 537, 0, 333, 0, 334, 4085, 3217, 39, 0, 0, 1317, 3687, 10, 37, 43, 9, 4092, 983, 63, 4056, 511, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4079, 3332, 4084, 0, 1922, 305, 0, 4088, 29, 0, 3699, 1, 0, 12, 0, 0, 0, 0, 22, 3, 107, 4093, 1, 3977, 0, 10, 0, 384, 1013, 15, 4094, 7, 211, 174, 3914, 0, 0, 0, 26, 111, 4016, 146, 0, 3849, 608, 0, 3832, 3713, 20, 5, 0, 2278, 3268, 120, 83, 2, 0, 3828, 19, 12, 21, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1277, 0, 4023, 1, 2917, 22, 3257, 4096, 694, 4, 60, 10, 648, 4, 357, 0, 360, 0, 304, 0, 0, 2793, 10, 3473, 0, 0, 0, 0, 625, 1152, 4091, 4, 0, 375, 4083, 0, 8, 4018, 0, 531, 2320, 579, 0, 3727, 4, 0, 4095, 3511, 0, 2, 105, 3043, 549, 8, 125, 0, 0, 4080, 14, 1051, 3126, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3994, 0, 3992, 12, 3565, 12, 283, 4095, 148, 5, 3426, 7, 0, 0, 1439, 0, 14, 0, 44, 296, 56, 2611, 35, 4087, 0, 1742, 2885, 0, 23, 0, 4052, 13, 105, 26, 3986, 5, 0, 3274, 0, 0, 3062, 142, 1, 3948, 942, 143, 3548, 2796, 2, 0, 0, 86, 170, 151, 0, 3, 31, 4088, 926, 35, 1115, 0, 115, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4051, 0, 4096, 167, 10, 11, 3, 4095, 2970, 20, 3505, 27, 0, 0, 0, 1, 0, 31, 0, 369, 248, 304, 0, 3974, 0, 3661, 37, 2, 245, 4053, 3880, 0, 344, 0, 4082, 21, 7, 715, 8, 18, 206, 3343, 4, 3323, 0, 0, 3369, 1180, 66, 1, 0, 0, 2901, 39, 21, 0, 207, 4096, 0, 1141, 4086, 589, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4001, 0, 0, 13, 0, 3, 4008, 1941, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 76, 270, 102, 0, 12, 30, 443, 2708, 502, 0, 0, 0, 0, 0, 0, 0, 1, 388, 38, 528, 64, 0, 1058, 2102, 989, 0, 0, 0, 0, 0, 0, 41, 1006, 39, 2901, 4080, 52, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 46, 4026, 0, 0, 860, 20, 8, 277, 215, 146, 24, 0, 0, 0, 0, 60, 2565, 932, 2924, 1, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 38, 4096, 2827, 0, 2, 0, 3495, 0, 0, 0, 0, 0, 0, 0, 0, 2760, 28, 1818, 3328, 19, 87, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9966, 7843, 5168, 11710, 5685, 6652, 10467, 8045] avg=8192.0000 max=11710 max_rank=3 min=5168 min_rank=2 imbalance_ratio=1.429443 max_load_reduction=53826 max_load_reduction_pct=0.821320 moved_to_echo=30849 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3748, 0, 0, 0, 16, 1, 3992, 1853, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 15, 4, 0, 1, 12, 686, 471, 0, 0, 0, 0, 0, 0, 129, 56, 4, 0, 4, 267, 0, 3931, 2007, 250, 0, 0, 0, 0, 0, 0, 6, 494, 158, 403, 1082, 0, 3834, 181, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2124, 4092, 0, 92, 3961, 100, 8, 373, 229, 195, 23, 0, 0, 0, 0, 475, 1217, 9, 3901, 2, 117, 0, 611, 0, 0, 0, 0, 0, 0, 0, 0, 1, 39, 1, 4077, 3280, 433, 85, 3, 3271, 0, 0, 0, 0, 0, 0, 0, 38, 3711, 1, 1804, 3575, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9610, 5255, 6648, 6158, 11207, 6332, 11190, 9136] avg=8192.0000 max=11207 max_rank=4 min=5255 min_rank=1 imbalance_ratio=1.368042 max_load_reduction=54329 max_load_reduction_pct=0.828995 moved_to_echo=37865 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3348, 667, 0, 113, 0, 0, 4, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 111, 2220, 0, 1, 909, 0, 0, 2539, 514, 0, 0, 0, 0, 0, 0, 0, 0, 316, 758, 28, 3921, 177, 1, 1970, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 107, 3094, 3857, 37, 0, 0, 0, 0, 0, 0, 0, 0, 409, 201, 3828, 9, 1236, 389, 0, 1, 343, 230, 190, 22, 0, 0, 0, 0, 16, 3014, 6, 3810, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 205, 225, 78, 3897, 20, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 4000, 3729, 3, 0, 760, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5951, 10186, 8097, 7531, 6858, 7014, 8001, 11898] avg=8192.0000 max=11898 max_rank=7 min=5951 min_rank=0 imbalance_ratio=1.452393 max_load_reduction=53638 max_load_reduction_pct=0.818451 moved_to_echo=33771 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1277, 0, 0, 1, 2917, 22, 3257, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 694, 4, 60, 10, 648, 4, 357, 0, 2999, 505, 0, 0, 0, 0, 0, 0, 360, 0, 304, 0, 0, 2793, 10, 3473, 2117, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 625, 1152, 3860, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 4083, 0, 8, 4018, 0, 531, 373, 231, 177, 24, 0, 0, 0, 0, 2320, 579, 0, 3550, 4, 0, 0, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 105, 3043, 549, 8, 125, 0, 3518, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 14, 1051, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9429, 5281, 10153, 5641, 9820, 9964, 7350, 7898] avg=8192.0000 max=10153 max_rank=2 min=5281 min_rank=1 imbalance_ratio=1.239380 max_load_reduction=55383 max_load_reduction_pct=0.845078 moved_to_echo=35032 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[620, 1, 0, 10, 484, 0, 0, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 9, 1, 0, 2, 9, 4017, 1151, 2992, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 2131, 339, 205, 2022, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 3309, 3817, 1763, 0, 0, 0, 0, 0, 0, 0, 0, 1, 50, 3938, 3671, 0, 3591, 17, 638, 374, 228, 15, 23, 0, 0, 0, 0, 280, 537, 0, 318, 0, 334, 0, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 1317, 3687, 10, 37, 43, 3536, 0, 0, 0, 0, 0, 0, 0, 9, 3718, 983, 63, 4056, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2981, 12278, 5974, 9062, 12546, 4686, 8669, 9340] avg=8192.0000 max=12546 max_rank=4 min=2981 min_rank=0 imbalance_ratio=1.531494 max_load_reduction=52990 max_load_reduction_pct=0.808563 moved_to_echo=35241 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3994, 0, 0, 12, 3565, 12, 283, 1954, 0, 0, 0, 0, 0, 0, 0, 0, 148, 5, 3426, 7, 0, 0, 1439, 0, 2599, 501, 0, 0, 0, 0, 0, 0, 14, 0, 44, 296, 56, 2611, 35, 4087, 2117, 949, 0, 0, 0, 0, 0, 0, 0, 1742, 2885, 0, 23, 0, 3824, 13, 0, 0, 0, 0, 0, 0, 0, 0, 105, 26, 3986, 5, 0, 3274, 0, 0, 374, 228, 188, 24, 0, 0, 0, 0, 3062, 142, 1, 3760, 942, 143, 0, 2796, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 86, 170, 151, 0, 3, 3491, 0, 0, 0, 0, 0, 0, 0, 31, 3714, 926, 35, 1115, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9820, 8125, 10209, 8487, 8210, 10846, 3903, 5936] avg=8192.0000 max=10846 max_rank=5 min=3903 min_rank=6 imbalance_ratio=1.323975 max_load_reduction=54690 max_load_reduction_pct=0.834503 moved_to_echo=28895 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4079, 3332, 0, 0, 1922, 305, 0, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 3699, 1, 0, 12, 0, 0, 2807, 513, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3, 107, 4093, 1, 3977, 2113, 1025, 0, 0, 0, 0, 0, 0, 0, 10, 0, 384, 1013, 15, 3863, 7, 0, 0, 0, 0, 0, 0, 0, 0, 211, 174, 3914, 0, 0, 0, 26, 111, 350, 231, 183, 24, 0, 0, 0, 0, 4016, 146, 0, 3666, 608, 0, 0, 3713, 0, 0, 0, 0, 0, 0, 0, 0, 20, 5, 0, 2278, 3268, 120, 83, 2, 3571, 0, 0, 0, 0, 0, 0, 0, 0, 3478, 19, 12, 21, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11589, 7061, 11341, 5292, 5224, 12149, 9347, 3533] avg=8192.0000 max=12149 max_rank=5 min=3533 min_rank=7 imbalance_ratio=1.483032 max_load_reduction=53387 max_load_reduction_pct=0.814621 moved_to_echo=30253 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4051, 0, 0, 167, 10, 11, 3, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 2970, 20, 3505, 27, 0, 0, 0, 1, 2469, 515, 0, 0, 0, 0, 0, 0, 0, 31, 0, 369, 248, 304, 0, 3974, 2120, 900, 0, 0, 0, 0, 0, 0, 0, 3661, 37, 2, 245, 4053, 3657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 4082, 21, 7, 715, 8, 18, 378, 223, 163, 27, 0, 0, 0, 0, 206, 3343, 4, 3160, 0, 0, 0, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1, 0, 0, 2901, 39, 21, 0, 3581, 0, 0, 0, 0, 0, 0, 0, 207, 3718, 0, 1141, 4086, 589, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6190, 9507, 7946, 11655, 5986, 7893, 6609, 9750] avg=8192.0000 max=11655 max_rank=3 min=5986 min_rank=4 imbalance_ratio=1.422729 max_load_reduction=53881 max_load_reduction_pct=0.822159 moved_to_echo=30238 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25216, 4096, 0, 384, 8960, 384, 11648, 15360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 28288, 2048, 9984, 23040, 1152, 256, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[19200, 256, 13184, 256, 768, 1024, 5888, 1664, 19840, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[10496, 11648, 1024, 25216, 1664, 640, 0, 15488, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 128, 1152, 1664, 1024, 16256, 640, 20736, 16640, 7296, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 128, 256, 15104, 17024, 896, 4352, 128, 28160, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 7040, 3328, 4224, 7296, 11776, 30336, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1152, 3072, 32000, 3712, 1408, 16896, 256, 1408, 2944, 1920, 1280, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 3196, 0, 0, 0, 25, 136, 0, 2373, 94, 29, 65, 0, 0, 4084, 10, 3340, 54, 0, 1, 0, 0, 1, 696, 417, 3, 4056, 0, 0, 1, 58, 0, 3064, 1, 158, 2690, 0, 3856, 9, 4092, 2988, 679, 2673, 3560, 13, 270, 0, 4003, 182, 195, 3379, 2, 84, 3125, 1127, 4096, 2858, 43, 0, 2, 0, 3748, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4093, 5, 838, 1, 0, 0, 347, 1, 0, 0, 13, 0, 0, 0, 0, 2531, 0, 37, 127, 59, 0, 1182, 1171, 1063, 0, 4087, 190, 0, 0, 3073, 0, 548, 1, 4082, 2648, 0, 0, 6, 3401, 55, 4090, 4094, 4058, 0, 1528, 0, 498, 0, 265, 450, 782, 1, 759, 3955, 4064, 3814, 0, 0, 19, 8, 3547, 4045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 43, 3870, 0, 0, 0, 0, 4055, 3146, 1441, 889, 1526, 313, 0, 63, 486, 25, 2822, 2039, 3, 0, 33, 8, 56, 0, 2, 42, 4085, 2, 0, 0, 0, 3, 0, 0, 4080, 68, 0, 3555, 48, 20, 0, 4096, 3792, 3284, 0, 4023, 2, 1323, 0, 0, 684, 244, 0, 4031, 101, 4096, 73, 4, 8, 3165, 0, 3880, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[126, 0, 4093, 7, 1, 0, 80, 9, 146, 344, 2, 0, 19, 1, 0, 4091, 0, 304, 76, 2, 0, 0, 2353, 5, 795, 0, 114, 3911, 0, 18, 0, 0, 0, 3471, 0, 3091, 1030, 1988, 4084, 0, 3595, 854, 0, 3559, 1051, 13, 4021, 0, 4075, 0, 0, 4047, 2854, 0, 2816, 29, 4095, 0, 7, 0, 0, 0, 4096, 263] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[73, 28, 3993, 0, 248, 7, 0, 310, 5, 17, 2896, 41, 1108, 38, 0, 4052, 283, 2674, 0, 327, 0, 0, 24, 6, 605, 0, 0, 4073, 5, 0, 0, 1727, 14, 31, 0, 3433, 788, 1, 2689, 1, 1793, 1408, 3, 4047, 2245, 0, 961, 163, 2976, 0, 0, 2115, 3935, 0, 3927, 1962, 4095, 275, 13, 19, 83, 0, 3070, 2949] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[446, 0, 4048, 52, 8, 0, 17, 3966, 3125, 926, 3344, 82, 29, 0, 0, 3651, 0, 4094, 5, 406, 9, 0, 4095, 376, 90, 0, 868, 4095, 0, 0, 0, 2, 0, 0, 38, 1721, 20, 5, 280, 0, 4081, 96, 112, 4088, 3637, 10, 0, 1101, 4068, 1, 3, 0, 0, 158, 2144, 0, 3857, 54, 1, 0, 6, 4, 3947, 2370] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 17, 4082, 0, 149, 0, 99, 169, 1, 0, 0, 4, 799, 0, 0, 3163, 3241, 4074, 20, 11, 30, 0, 4040, 981, 20, 993, 0, 4071, 13, 0, 4, 3794, 3, 197, 0, 2907, 1219, 1, 3441, 0, 3727, 1947, 2, 3647, 257, 0, 1115, 0, 74, 225, 140, 3213, 0, 4, 3999, 37, 4075, 1634, 0, 2, 5, 3, 3887, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 3815, 4085, 146, 0, 4, 976, 3927, 141, 3695, 0, 84, 0, 315, 429, 614, 3381, 552, 0, 0, 0, 0, 5, 2, 0, 1391, 3724, 0, 0, 0, 467, 5, 0, 0, 2347, 2458, 0, 0, 30, 2038, 0, 0, 4094, 3999, 0, 4086, 0, 2717, 0, 436, 0, 2356, 0, 4096, 163, 4095, 0, 29, 4, 4, 1, 4090, 735] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 5, 838, 1, 0, 0, 1182, 1197, 0, 0, 0, 0, 0, 0, 347, 1, 0, 0, 13, 0, 0, 0, 1105, 207, 0, 0, 0, 0, 0, 0, 0, 2531, 0, 37, 127, 59, 0, 1182, 2897, 0, 0, 0, 0, 0, 0, 0, 1171, 1063, 0, 4087, 190, 0, 0, 3073, 1284, 829, 0, 0, 0, 0, 0, 0, 0, 548, 1, 4082, 2648, 0, 0, 6, 165, 0, 0, 0, 0, 0, 0, 0, 1467, 55, 4090, 0, 4058, 0, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 0, 265, 450, 782, 1, 387, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 1598, 3814, 0, 0, 19, 8, 3547, 4045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7316, 1673, 6833, 11697, 7450, 11198, 6338, 13031] avg=8192.0000 max=13031 max_rank=7 min=1673 min_rank=1 imbalance_ratio=1.590698 max_load_reduction=52505 max_load_reduction_pct=0.801163 moved_to_echo=38017 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3196, 0, 0, 0, 25, 136, 1191, 781, 0, 0, 0, 0, 0, 0, 0, 2373, 94, 29, 65, 0, 0, 4084, 1329, 853, 0, 0, 0, 0, 0, 0, 10, 3340, 54, 0, 1, 0, 0, 1, 1892, 0, 0, 0, 0, 0, 0, 0, 696, 417, 3, 4056, 0, 0, 1, 58, 1294, 997, 0, 0, 0, 0, 0, 0, 0, 3064, 1, 158, 2690, 0, 3856, 9, 681, 0, 0, 0, 0, 0, 0, 0, 1766, 2988, 679, 0, 3560, 13, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 182, 195, 3379, 2, 84, 1591, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 2858, 43, 0, 2, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5329, 8827, 5298, 7522, 10459, 9276, 10563, 8262] avg=8192.0000 max=10563 max_rank=6 min=5298 min_rank=2 imbalance_ratio=1.289429 max_load_reduction=54973 max_load_reduction_pct=0.838821 moved_to_echo=38560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[73, 28, 3993, 0, 248, 7, 0, 310, 1191, 1184, 0, 0, 0, 0, 0, 0, 5, 17, 2896, 41, 1108, 38, 0, 4052, 582, 1072, 0, 0, 0, 0, 0, 0, 283, 2674, 0, 327, 0, 0, 24, 6, 2863, 0, 0, 0, 0, 0, 0, 0, 605, 0, 0, 4073, 5, 0, 0, 1727, 1294, 437, 0, 0, 0, 0, 0, 0, 14, 31, 0, 3433, 788, 1, 2689, 1, 855, 0, 0, 0, 0, 0, 0, 0, 774, 1408, 3, 0, 2245, 0, 961, 163, 0, 0, 0, 0, 0, 0, 0, 0, 2976, 0, 0, 2115, 3935, 0, 2000, 1962, 0, 0, 0, 0, 0, 0, 0, 0, 1610, 275, 13, 19, 83, 0, 3070, 2949, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7034, 9811, 6177, 8141, 7812, 5554, 12988, 8019] avg=8192.0000 max=12988 max_rank=6 min=5554 min_rank=5 imbalance_ratio=1.585449 max_load_reduction=52548 max_load_reduction_pct=0.801819 moved_to_echo=34373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 3870, 0, 0, 0, 0, 4055, 1191, 1110, 0, 0, 0, 0, 0, 0, 3146, 1441, 889, 1526, 313, 0, 63, 486, 6, 1100, 0, 0, 0, 0, 0, 0, 25, 2822, 2039, 3, 0, 33, 8, 56, 2682, 0, 0, 0, 0, 0, 0, 0, 0, 2, 42, 4085, 2, 0, 0, 0, 1294, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4080, 68, 0, 3555, 48, 878, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 3284, 0, 4023, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1323, 0, 0, 684, 244, 0, 2053, 101, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 73, 4, 8, 3165, 0, 3880, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10269, 8970, 7668, 5429, 8632, 11415, 4405, 8748] avg=8192.0000 max=11415 max_rank=5 min=4405 min_rank=6 imbalance_ratio=1.393433 max_load_reduction=54121 max_load_reduction_pct=0.825821 moved_to_echo=33200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 4082, 0, 149, 0, 99, 169, 1185, 1066, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 799, 0, 0, 3163, 1211, 1091, 0, 0, 0, 0, 0, 0, 3241, 4074, 20, 11, 30, 0, 4040, 981, 2581, 0, 0, 0, 0, 0, 0, 0, 20, 993, 0, 4071, 13, 0, 4, 3794, 1287, 908, 0, 0, 0, 0, 0, 0, 3, 197, 0, 2907, 1219, 1, 3441, 0, 871, 0, 0, 0, 0, 0, 0, 0, 1608, 1947, 2, 0, 257, 0, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 225, 140, 3213, 0, 4, 2037, 37, 0, 0, 0, 0, 0, 0, 0, 0, 1603, 1634, 0, 2, 5, 3, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6767, 6269, 14978, 11090, 8639, 4929, 5730, 7134] avg=8192.0000 max=14978 max_rank=2 min=4929 min_rank=5 imbalance_ratio=1.828369 max_load_reduction=50558 max_load_reduction_pct=0.771454 moved_to_echo=26432 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[126, 0, 4093, 7, 1, 0, 80, 9, 1191, 1042, 0, 0, 0, 0, 0, 0, 146, 344, 2, 0, 19, 1, 0, 4091, 1168, 768, 0, 0, 0, 0, 0, 0, 0, 304, 76, 2, 0, 0, 2353, 5, 2517, 0, 0, 0, 0, 0, 0, 0, 795, 0, 114, 3911, 0, 18, 0, 0, 1294, 876, 0, 0, 0, 0, 0, 0, 0, 3471, 0, 3091, 1030, 1988, 4084, 0, 613, 0, 0, 0, 0, 0, 0, 0, 1551, 854, 0, 0, 1051, 13, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4047, 2854, 0, 1435, 29, 0, 0, 0, 0, 0, 0, 0, 0, 1610, 0, 7, 0, 0, 0, 4096, 263, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6549, 6539, 5257, 7008, 14277, 7490, 12440, 5976] avg=8192.0000 max=14277 max_rank=4 min=5257 min_rank=2 imbalance_ratio=1.742798 max_load_reduction=51259 max_load_reduction_pct=0.782150 moved_to_echo=40183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[446, 0, 4048, 52, 8, 0, 17, 3966, 1121, 1196, 0, 0, 0, 0, 0, 0, 3125, 926, 3344, 82, 29, 0, 0, 3651, 1326, 585, 0, 0, 0, 0, 0, 0, 0, 4094, 5, 406, 9, 0, 4095, 376, 2892, 0, 0, 0, 0, 0, 0, 0, 90, 0, 868, 4095, 0, 0, 0, 2, 1218, 994, 0, 0, 0, 0, 0, 0, 0, 0, 38, 1721, 20, 5, 280, 0, 467, 0, 0, 0, 0, 0, 0, 0, 1761, 96, 112, 0, 3637, 10, 0, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 1, 3, 0, 0, 158, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 54, 1, 0, 6, 4, 3947, 2370, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10854, 13068, 11877, 7267, 2531, 6717, 5322, 7900] avg=8192.0000 max=13068 max_rank=1 min=2531 min_rank=4 imbalance_ratio=1.595215 max_load_reduction=52468 max_load_reduction_pct=0.800598 moved_to_echo=22470 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3815, 4085, 146, 0, 4, 976, 1194, 1198, 0, 0, 0, 0, 0, 0, 3927, 141, 3695, 0, 84, 0, 315, 429, 666, 1122, 0, 0, 0, 0, 0, 0, 614, 3381, 552, 0, 0, 0, 0, 5, 2896, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1391, 3724, 0, 0, 0, 467, 1298, 500, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2347, 2458, 0, 0, 30, 896, 0, 0, 0, 0, 0, 0, 0, 872, 0, 0, 0, 3999, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2717, 0, 436, 0, 2356, 0, 2078, 163, 0, 0, 0, 0, 0, 0, 0, 0, 1603, 0, 29, 4, 4, 1, 4090, 735, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11418, 10379, 7448, 7382, 5736, 8957, 7750, 6466] avg=8192.0000 max=11418 max_rank=0 min=5736 min_rank=4 imbalance_ratio=1.393799 max_load_reduction=54118 max_load_reduction_pct=0.825775 moved_to_echo=28909 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 7424, 128, 21888, 11008, 2048, 17920, 128, 5504, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[19840, 512, 1152, 13952, 10240, 256, 12800, 7424, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[10752, 5248, 11008, 1792, 2432, 128, 384, 19968, 7424, 6912, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[9856, 7424, 9088, 0, 22144, 128, 16128, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4224, 23296, 2816, 896, 256, 128, 10624, 2688, 21248, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 128, 31232, 4224, 1408, 128, 256, 9728, 9472, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12800, 8832, 128, 128, 3328, 128, 30336, 10496, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3456, 2560, 2432, 32128, 256, 128, 128, 9216, 10368, 5632, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4068, 18, 339, 731, 4096, 62, 0, 4096, 11, 3163, 7, 4071, 2, 67, 10, 0, 1742, 0, 0, 9, 0, 621, 0, 86, 2035, 221, 0, 0, 4070, 3688, 0, 0, 4095, 0, 0, 0, 529, 4080, 1730, 26, 3451, 0, 1, 2096, 0, 0, 0, 3439, 4, 4089, 0, 0, 0, 199, 0, 4013, 3285, 2, 33, 1, 0, 0, 1, 1249] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 866, 10, 3963, 29, 0, 4063, 24, 4060, 0, 3171, 2524, 6, 71, 2413, 3398, 2, 5, 7, 4061, 172, 7, 1, 0, 2891, 0, 2, 53, 4061, 29, 3, 4033, 26, 27, 1, 302, 1575, 15, 0, 4019, 4081, 0, 1449, 1481, 1707, 0, 1969, 54, 2986, 0, 0, 21, 0, 48, 10, 1734, 4, 2, 5, 108, 0, 3949, 38] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 126, 0, 0, 3858, 25, 4031, 219, 4095, 2827, 279, 0, 0, 2297, 54, 34, 45, 69, 1826, 0, 1555, 0, 4066, 0, 0, 4087, 3298, 0, 254, 3922, 0, 0, 11, 42, 0, 1, 0, 3791, 0, 139, 1594, 23, 5, 12, 2552, 0, 4020, 0, 6, 29, 0, 3987, 5, 4054, 80, 0, 1, 0, 84, 0, 4036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[3725, 24, 0, 312, 3986, 157, 1, 4096, 0, 4041, 301, 4084, 31, 15, 1, 1618, 0, 3, 0, 0, 71, 3679, 0, 876, 0, 1508, 6, 66, 4086, 4096, 277, 0, 3919, 4, 225, 0, 0, 467, 0, 69, 273, 2, 8, 35, 0, 40, 0, 3911, 16, 4074, 112, 0, 0, 72, 44, 3333, 4096, 0, 7, 814, 0, 0, 2859, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3462, 2350, 2, 0, 4096, 0, 1, 4077, 0, 4095, 0, 4096, 2367, 52, 820, 111, 11, 17, 0, 1129, 2791, 0, 2, 5, 0, 757, 0, 2, 1, 3889, 0, 1488, 297, 2, 0, 0, 0, 3909, 0, 0, 12, 7, 4063, 3970, 1317, 0, 0, 0, 0, 4096, 0, 11, 0, 100, 3100, 7, 4067, 18, 16, 0, 0, 827, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[2005, 0, 0, 1006, 3687, 349, 3, 2696, 0, 3999, 0, 3228, 0, 89, 0, 0, 3148, 146, 88, 3, 8, 23, 252, 0, 3624, 4096, 0, 0, 80, 3963, 0, 0, 3691, 0, 3, 105, 159, 1503, 2527, 0, 4094, 0, 0, 2197, 12, 53, 2425, 4087, 0, 3682, 178, 0, 0, 0, 193, 0, 4010, 0, 0, 6, 15, 0, 19, 4084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4093, 0, 3163, 0, 4093, 0, 0, 1112, 0, 4067, 0, 3798, 0, 0, 0, 814, 4092, 0, 1, 0, 2, 1, 0, 0, 244, 4043, 2, 8, 0, 3959, 0, 557, 4096, 134, 0, 0, 0, 1366, 356, 0, 1103, 46, 0, 2861, 346, 1278, 13, 2, 3928, 3703, 0, 203, 134, 152, 20, 0, 4093, 0, 74, 27, 30, 3438, 1, 4083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[24, 0, 327, 0, 636, 0, 308, 1510, 0, 3796, 341, 4096, 0, 0, 6, 0, 4083, 0, 0, 1037, 301, 177, 2, 0, 3553, 3799, 0, 0, 4086, 4081, 1212, 128, 3149, 108, 497, 0, 0, 3465, 344, 160, 90, 3965, 1, 3570, 57, 0, 0, 255, 12, 3996, 1, 0, 0, 4006, 0, 0, 4096, 0, 101, 3, 0, 64, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4068, 18, 339, 731, 4096, 62, 0, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2009, 7, 4071, 2, 67, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 0, 0, 9, 0, 621, 0, 86, 2000, 1728, 0, 0, 0, 0, 0, 0, 2035, 221, 0, 0, 4070, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 529, 4080, 1730, 26, 925, 870, 0, 0, 0, 0, 0, 0, 3451, 0, 1, 1346, 0, 0, 0, 3439, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4089, 0, 0, 0, 199, 0, 4013, 750, 252, 229, 0, 0, 0, 0, 0, 3033, 2, 33, 1, 0, 0, 1, 1249, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10812, 6177, 6186, 8014, 12255, 8237, 9536, 4319] avg=8192.0000 max=12255 max_rank=4 min=4319 min_rank=7 imbalance_ratio=1.495972 max_load_reduction=53281 max_load_reduction_pct=0.813004 moved_to_echo=34347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 126, 0, 0, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2561, 219, 4095, 2827, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2297, 54, 34, 45, 69, 1826, 0, 1555, 1788, 1628, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4087, 1510, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 0, 0, 11, 42, 0, 1, 0, 1179, 820, 0, 0, 0, 0, 0, 0, 3791, 0, 139, 1024, 23, 5, 12, 2552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 6, 29, 0, 3987, 5, 570, 311, 291, 0, 0, 0, 0, 0, 3743, 80, 0, 1, 0, 84, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5633, 10006, 9296, 9917, 5975, 7546, 9219, 7944] avg=8192.0000 max=10006 max_rank=1 min=5633 min_rank=0 imbalance_ratio=1.221436 max_load_reduction=55530 max_load_reduction_pct=0.847321 moved_to_echo=30684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3725, 24, 0, 312, 3986, 157, 1, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2567, 301, 4084, 31, 15, 1, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 71, 3679, 0, 876, 2221, 1728, 0, 0, 0, 0, 0, 0, 0, 1508, 6, 66, 4086, 1875, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 4, 225, 0, 0, 467, 0, 69, 1182, 870, 0, 0, 0, 0, 0, 0, 273, 2, 8, 23, 0, 40, 0, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4074, 112, 0, 0, 72, 44, 3333, 12, 314, 292, 0, 0, 0, 0, 0, 3782, 0, 7, 814, 0, 0, 2859, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9703, 8617, 8578, 7818, 6736, 4257, 8269, 11558] avg=8192.0000 max=11558 max_rank=7 min=4257 min_rank=5 imbalance_ratio=1.410889 max_load_reduction=53978 max_load_reduction_pct=0.823639 moved_to_echo=30820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 866, 10, 3963, 29, 0, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 24, 2580, 0, 3171, 2524, 6, 71, 2413, 0, 0, 0, 0, 0, 0, 0, 0, 3398, 2, 5, 7, 4061, 172, 7, 1, 2202, 1714, 0, 0, 0, 0, 0, 0, 0, 2891, 0, 2, 53, 1859, 29, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 26, 27, 1, 302, 1575, 15, 0, 1187, 863, 0, 0, 0, 0, 0, 0, 4019, 4081, 0, 930, 1481, 1707, 0, 1969, 0, 0, 0, 0, 0, 0, 0, 0, 54, 2986, 0, 0, 21, 0, 48, 10, 519, 133, 293, 0, 0, 0, 0, 0, 1601, 4, 2, 5, 108, 0, 3949, 38, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6354, 10789, 11569, 4837, 8029, 14187, 4064, 5707] avg=8192.0000 max=14187 max_rank=5 min=4064 min_rank=6 imbalance_ratio=1.731812 max_load_reduction=51349 max_load_reduction_pct=0.783524 moved_to_echo=31987 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2005, 0, 0, 1006, 3687, 349, 3, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2541, 0, 3228, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3148, 146, 88, 3, 8, 23, 252, 0, 2149, 1137, 0, 0, 0, 0, 0, 0, 3624, 4096, 0, 0, 80, 1814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3691, 0, 3, 105, 159, 1503, 2527, 0, 1169, 573, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1411, 12, 53, 2425, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3682, 178, 0, 0, 0, 193, 0, 786, 308, 289, 0, 0, 0, 0, 0, 3702, 0, 0, 6, 15, 0, 19, 4084, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8036, 5858, 6954, 9614, 9730, 12082, 5436, 7826] avg=8192.0000 max=12082 max_rank=5 min=5436 min_rank=6 imbalance_ratio=1.474854 max_load_reduction=53454 max_load_reduction_pct=0.815643 moved_to_echo=35074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3462, 2350, 2, 0, 4096, 0, 1, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2601, 0, 4096, 2367, 52, 820, 111, 0, 0, 0, 0, 0, 0, 0, 0, 11, 17, 0, 1129, 2791, 0, 2, 5, 2109, 1720, 0, 0, 0, 0, 0, 0, 0, 757, 0, 2, 1, 1780, 0, 1488, 0, 0, 0, 0, 0, 0, 0, 0, 297, 2, 0, 0, 0, 3909, 0, 0, 1198, 866, 0, 0, 0, 0, 0, 0, 12, 7, 4063, 2548, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 100, 3100, 7, 1422, 312, 296, 0, 0, 0, 0, 0, 3755, 18, 16, 0, 0, 827, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11402, 10047, 7784, 4028, 6272, 7947, 9344, 8712] avg=8192.0000 max=11402 max_rank=0 min=4028 min_rank=3 imbalance_ratio=1.391846 max_load_reduction=54134 max_load_reduction_pct=0.826019 moved_to_echo=32275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 3163, 0, 4093, 0, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2584, 0, 3798, 0, 0, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 2, 1, 0, 0, 2147, 469, 0, 0, 0, 0, 0, 0, 244, 4043, 2, 8, 0, 1812, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 134, 0, 0, 0, 1366, 356, 0, 1189, 236, 0, 0, 0, 0, 0, 0, 1103, 46, 0, 1837, 346, 1278, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3928, 3703, 0, 203, 134, 152, 20, 0, 1024, 314, 294, 0, 0, 0, 0, 0, 3779, 0, 74, 27, 30, 3438, 1, 4083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11756, 7196, 6712, 6666, 7377, 4625, 9772, 11432] avg=8192.0000 max=11756 max_rank=0 min=4625 min_rank=5 imbalance_ratio=1.435059 max_load_reduction=53780 max_load_reduction_pct=0.820618 moved_to_echo=33206 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[24, 0, 327, 0, 636, 0, 308, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2403, 341, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 1037, 301, 177, 2, 0, 2217, 640, 0, 0, 0, 0, 0, 0, 3553, 3799, 0, 0, 4086, 1864, 1212, 128, 0, 0, 0, 0, 0, 0, 0, 0, 3149, 108, 497, 0, 0, 3465, 344, 160, 1114, 325, 0, 0, 0, 0, 0, 0, 90, 3965, 1, 2287, 57, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3996, 1, 0, 0, 4006, 0, 0, 1283, 319, 279, 0, 0, 0, 0, 0, 3777, 0, 101, 3, 0, 64, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1840, 6846, 8457, 14642, 9162, 6655, 9896, 8038] avg=8192.0000 max=14642 max_rank=3 min=1840 min_rank=0 imbalance_ratio=1.787354 max_load_reduction=50894 max_load_reduction_pct=0.776581 moved_to_echo=33751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[21504, 2432, 4736, 2176, 24704, 640, 384, 9344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[27264, 128, 256, 896, 256, 4480, 6912, 25856, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[16896, 8192, 4224, 11520, 3328, 3200, 2560, 16256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[9472, 21504, 128, 128, 16512, 14208, 1536, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[18816, 256, 128, 2304, 7424, 6528, 384, 2560, 16896, 10880, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[27264, 384, 768, 128, 1152, 16384, 4992, 256, 9216, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 19968, 896, 30720, 7808, 512, 1024, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 30720, 384, 256, 256, 4608, 7424, 7424, 6400, 2304, 2304, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[2496, 8, 0, 87, 1, 4036, 0, 180, 1154, 0, 3968, 0, 2, 0, 0, 0, 1300, 4095, 3364, 3072, 5, 1065, 2882, 4096, 102, 1508, 0, 37, 0, 3, 1081, 3929, 175, 0, 1076, 54, 3802, 18, 41, 0, 0, 0, 3934, 3675, 38, 144, 0, 0, 0, 0, 3486, 0, 0, 2231, 151, 7, 0, 7, 0, 0, 4095, 3, 173, 3955] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4073, 0, 4, 10, 0, 5, 3, 0, 8, 0, 3485, 0, 4, 363, 139, 385, 36, 3045, 4096, 0, 2, 3838, 57, 3944, 0, 674, 0, 1, 57, 248, 4095, 4070, 99, 2, 0, 1, 4007, 4, 0, 755, 0, 1627, 4087, 663, 0, 36, 3739, 2, 2705, 606, 3807, 4045, 3543, 450, 75, 0, 2, 657, 44, 0, 1938, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[3243, 0, 0, 0, 0, 2812, 19, 0, 13, 24, 3895, 0, 0, 167, 1, 0, 1040, 0, 4096, 7, 0, 3806, 227, 4084, 0, 3956, 2, 4, 97, 33, 4088, 4059, 3311, 0, 313, 0, 0, 3040, 0, 0, 12, 0, 0, 2753, 0, 0, 4053, 0, 0, 0, 4093, 214, 4035, 2, 329, 0, 2659, 1059, 10, 0, 0, 0, 3980, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1086, 0, 3412, 0, 0, 0, 0, 3, 0, 1781, 4038, 0, 0, 155, 8, 3, 0, 4096, 626, 0, 0, 420, 2242, 4096, 0, 0, 36, 0, 0, 0, 2988, 4069, 1, 0, 325, 0, 0, 2262, 0, 122, 1, 45, 4049, 3938, 46, 0, 3003, 0, 2976, 0, 3442, 1343, 3918, 208, 14, 10, 3567, 163, 3823, 1, 1, 42, 3177, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[2623, 0, 115, 51, 0, 2188, 0, 0, 2894, 3, 4069, 0, 0, 3271, 0, 11, 0, 1970, 3957, 2, 609, 3884, 550, 4096, 4016, 256, 0, 0, 0, 20, 4094, 4096, 4, 0, 0, 0, 85, 0, 0, 78, 0, 1043, 3471, 87, 2, 0, 3413, 75, 1072, 0, 4096, 3995, 4077, 48, 1, 0, 4, 454, 480, 0, 0, 0, 276, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4083, 0, 0, 4064, 360, 2458, 238, 4, 4010, 0, 4092, 4, 0, 3304, 78, 31, 17, 4096, 3727, 0, 0, 17, 1, 3871, 336, 4089, 0, 0, 105, 66, 4076, 2483, 11, 0, 0, 1, 3801, 470, 1, 0, 0, 108, 3509, 2, 0, 0, 4000, 782, 425, 0, 1378, 0, 3943, 0, 1, 0, 0, 1376, 36, 0, 25, 5, 0, 52] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[2099, 3984, 0, 0, 139, 3945, 4, 3832, 0, 997, 4096, 0, 485, 1, 0, 0, 2695, 2087, 4096, 0, 0, 4096, 3966, 561, 1160, 696, 0, 29, 0, 0, 0, 1334, 1655, 0, 3718, 9, 1, 0, 627, 55, 0, 0, 151, 2409, 0, 379, 3726, 3, 0, 3708, 3985, 3647, 1047, 0, 0, 6, 3, 0, 5, 0, 0, 4, 96, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4019, 4, 0, 0, 390, 2861, 3835, 0, 0, 2895, 554, 0, 9, 4030, 31, 0, 1371, 3218, 3255, 4096, 3352, 274, 2, 0, 0, 0, 4096, 4067, 1175, 5, 0, 0, 177, 592, 0, 32, 1, 0, 42, 4082, 0, 0, 1451, 0, 0, 0, 4091, 4092, 4096, 0, 0, 0, 2413, 0, 1, 0, 924, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 4, 10, 0, 5, 3, 0, 1213, 116, 0, 0, 0, 0, 0, 0, 8, 0, 3485, 0, 4, 363, 139, 385, 1078, 80, 0, 0, 0, 0, 0, 0, 36, 2841, 0, 0, 2, 3838, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 674, 0, 1, 57, 248, 4095, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 99, 2, 0, 1, 4007, 4, 0, 755, 3944, 817, 0, 0, 0, 0, 0, 0, 0, 1627, 4087, 663, 0, 36, 3739, 2, 124, 0, 0, 0, 0, 0, 0, 0, 2705, 606, 1143, 4045, 3543, 450, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 657, 44, 0, 1938, 0, 0, 0, 3279, 1451, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5424, 5542, 6774, 7951, 9629, 10278, 12567, 7371] avg=8192.0000 max=12567 max_rank=6 min=5424 min_rank=0 imbalance_ratio=1.534058 max_load_reduction=52969 max_load_reduction_pct=0.808243 moved_to_echo=39845 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3243, 0, 0, 0, 0, 2812, 19, 0, 1304, 115, 0, 0, 0, 0, 0, 0, 13, 24, 3895, 0, 0, 167, 1, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 1040, 0, 0, 7, 0, 3806, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 2, 4, 97, 33, 4088, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 3311, 0, 313, 0, 0, 3040, 0, 0, 4084, 818, 0, 0, 0, 0, 0, 0, 12, 0, 0, 2753, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1229, 214, 4035, 2, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2659, 1059, 10, 0, 0, 0, 3980, 0, 3278, 1560, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7493, 5175, 5080, 11049, 11566, 6818, 5809, 12546] avg=8192.0000 max=12546 max_rank=7 min=5080 min_rank=2 imbalance_ratio=1.531494 max_load_reduction=52990 max_load_reduction_pct=0.808563 moved_to_echo=36739 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1086, 0, 3412, 0, 0, 0, 0, 3, 1096, 116, 0, 0, 0, 0, 0, 0, 0, 1781, 4038, 0, 0, 155, 8, 3, 1077, 107, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 420, 2242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 2988, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 325, 0, 0, 2262, 0, 122, 4096, 124, 0, 0, 0, 0, 0, 0, 1, 45, 4049, 3938, 46, 0, 3003, 0, 167, 0, 0, 0, 0, 0, 0, 0, 2976, 0, 1034, 1343, 3918, 208, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3567, 163, 3823, 1, 1, 42, 3177, 0, 502, 1312, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5713, 7169, 6484, 5900, 6930, 11249, 9503, 12588] avg=8192.0000 max=12588 max_rank=7 min=5713 min_rank=0 imbalance_ratio=1.536621 max_load_reduction=52948 max_load_reduction_pct=0.807922 moved_to_echo=40270 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2496, 8, 0, 87, 1, 4036, 0, 180, 1110, 112, 0, 0, 0, 0, 0, 0, 1154, 0, 3968, 0, 2, 0, 0, 0, 1040, 107, 0, 0, 0, 0, 0, 0, 1300, 3822, 0, 3072, 5, 1065, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 1508, 0, 37, 0, 3, 1081, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 1076, 54, 3802, 18, 41, 0, 4096, 672, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 3675, 38, 144, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 0, 0, 2231, 151, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4095, 3, 173, 3955, 2692, 1329, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8030, 6271, 12146, 5508, 9934, 7957, 3436, 12254] avg=8192.0000 max=12254 max_rank=7 min=3436 min_rank=6 imbalance_ratio=1.495850 max_load_reduction=53282 max_load_reduction_pct=0.813019 moved_to_echo=33581 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2623, 0, 115, 51, 0, 2188, 0, 0, 1305, 117, 0, 0, 0, 0, 0, 0, 2894, 3, 4069, 0, 0, 3271, 0, 11, 1084, 51, 0, 0, 0, 0, 0, 0, 0, 1839, 0, 2, 609, 3884, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 256, 0, 0, 0, 20, 4094, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 85, 0, 0, 78, 4096, 789, 0, 0, 0, 0, 0, 0, 0, 1043, 3471, 87, 2, 0, 3413, 75, 80, 0, 0, 0, 0, 0, 0, 0, 1072, 0, 1230, 3995, 4077, 48, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 454, 480, 0, 0, 0, 276, 0, 3168, 1561, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6399, 11383, 6884, 11281, 5052, 8171, 10423, 5943] avg=8192.0000 max=11383 max_rank=1 min=5052 min_rank=4 imbalance_ratio=1.389526 max_load_reduction=54153 max_load_reduction_pct=0.826309 moved_to_echo=29589 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2099, 3984, 0, 0, 139, 3945, 4, 3832, 1269, 38, 0, 0, 0, 0, 0, 0, 0, 997, 4096, 0, 485, 1, 0, 0, 353, 54, 0, 0, 0, 0, 0, 0, 2695, 1948, 0, 0, 0, 4096, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 696, 0, 29, 0, 0, 0, 943, 0, 0, 0, 0, 0, 0, 0, 0, 1655, 0, 3718, 9, 1, 0, 627, 55, 561, 817, 0, 0, 0, 0, 0, 0, 0, 0, 151, 2409, 0, 379, 3726, 3, 85, 0, 0, 0, 0, 0, 0, 0, 0, 3708, 1197, 3647, 1047, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 0, 0, 4, 96, 0, 3279, 1519, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15310, 5986, 12705, 2828, 7443, 6753, 9605, 4906] avg=8192.0000 max=15310 max_rank=0 min=2828 min_rank=3 imbalance_ratio=1.868896 max_load_reduction=50226 max_load_reduction_pct=0.766388 moved_to_echo=28707 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 0, 4064, 360, 2458, 238, 4, 439, 70, 0, 0, 0, 0, 0, 0, 4010, 0, 4092, 4, 0, 3304, 78, 31, 657, 107, 0, 0, 0, 0, 0, 0, 17, 3822, 0, 0, 0, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 4089, 0, 0, 105, 66, 4076, 1756, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1, 3801, 470, 1, 0, 3871, 743, 0, 0, 0, 0, 0, 0, 0, 108, 3509, 2, 0, 0, 4000, 782, 167, 0, 0, 0, 0, 0, 0, 0, 425, 0, 414, 0, 3943, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1376, 36, 0, 25, 5, 0, 52, 2984, 525, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11716, 12283, 3857, 10428, 8898, 8568, 4783, 5003] avg=8192.0000 max=12283 max_rank=1 min=3857 min_rank=2 imbalance_ratio=1.499390 max_load_reduction=53253 max_load_reduction_pct=0.812576 moved_to_echo=27252 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4019, 4, 0, 0, 1307, 119, 0, 0, 0, 0, 0, 0, 390, 2861, 3835, 0, 0, 2895, 554, 0, 1081, 111, 0, 0, 0, 0, 0, 0, 9, 3753, 0, 0, 1371, 3218, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3352, 274, 2, 0, 0, 0, 4096, 2867, 0, 0, 0, 0, 0, 0, 0, 0, 1175, 5, 0, 0, 177, 592, 0, 32, 4096, 7, 0, 0, 0, 0, 0, 0, 1, 0, 42, 4082, 0, 0, 1451, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1222, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2413, 0, 1, 0, 924, 0, 0, 1, 24, 1562, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5451, 11727, 11606, 10591, 6084, 5742, 9410, 4925] avg=8192.0000 max=11727 max_rank=1 min=4925 min_rank=7 imbalance_ratio=1.431519 max_load_reduction=53809 max_load_reduction_pct=0.821060 moved_to_echo=26161 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9088, 11520, 128, 128, 384, 384, 24576, 19968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7296, 4352, 8576, 17408, 24704, 2944, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[6528, 128, 5504, 128, 11904, 6400, 768, 1152, 28928, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[19712, 4096, 3584, 4224, 4608, 15488, 384, 4096, 9088, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[5120, 21888, 0, 3200, 2048, 20352, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8576, 5760, 31488, 128, 512, 10240, 896, 512, 7552, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8704, 3840, 4480, 128, 7040, 128, 7808, 4096, 19328, 10880, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 2944, 19328, 17664, 128, 640, 23424, 896, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 465, 60, 2, 0, 9, 0, 0, 4054, 3870, 4096, 519, 4095, 1004, 2786, 1, 4018, 0, 0, 4096, 4013, 0, 0, 0, 0, 634, 8, 4, 4096, 0, 3361, 491, 0, 2276, 3, 38, 0, 5, 559, 0, 0, 634, 151, 2, 3763, 0, 0, 41, 3665, 318, 0, 15, 3214, 0, 154, 4094, 23, 0, 18, 13, 806, 0, 0, 4062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[6, 0, 1698, 0, 0, 0, 0, 4027, 3982, 4096, 1020, 2972, 4079, 10, 1, 30, 4026, 1, 0, 3620, 0, 9, 0, 0, 0, 504, 63, 0, 37, 291, 0, 0, 4085, 0, 2976, 0, 0, 16, 3513, 1, 0, 4094, 4080, 0, 4010, 0, 76, 0, 3674, 0, 0, 1012, 5, 0, 2048, 1536, 468, 0, 28, 641, 130, 0, 0, 2671] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 2, 4091, 0, 0, 21, 0, 0, 347, 3853, 4096, 0, 3993, 3900, 2600, 0, 4049, 0, 194, 4096, 358, 7, 0, 47, 0, 0, 1854, 0, 4096, 0, 0, 0, 3873, 602, 746, 0, 0, 0, 3576, 0, 0, 0, 0, 0, 3748, 0, 0, 6, 1423, 460, 1, 0, 3479, 45, 1266, 3754, 23, 2, 0, 713, 3358, 781, 6, 70] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[18, 0, 1549, 902, 1, 0, 3962, 2, 104, 3791, 3675, 149, 2810, 18, 0, 0, 2929, 408, 1, 4081, 67, 1622, 0, 17, 0, 3, 0, 2644, 4060, 0, 3972, 0, 147, 4089, 261, 1, 0, 367, 0, 0, 133, 8, 58, 7, 4077, 4095, 6, 0, 3848, 0, 0, 0, 138, 0, 0, 4086, 0, 1785, 0, 3978, 2, 1486, 0, 179] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3542, 0, 0, 0, 2, 5, 4062, 3663, 4094, 4028, 31, 3792, 0, 0, 3764, 3192, 0, 1083, 3, 4, 935, 0, 0, 3700, 0, 0, 3932, 6, 103, 4085, 4095, 387, 2, 166, 38, 2, 7, 0, 799, 4087, 3026, 0, 181, 4, 0, 186, 0, 0, 0, 0, 3935, 5, 0, 4090, 5, 0, 0, 0, 76, 7, 1, 411] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[38, 183, 2341, 0, 0, 5, 3, 589, 0, 3605, 4009, 2722, 4080, 4085, 261, 0, 4095, 3918, 51, 4096, 0, 160, 12, 0, 4, 0, 1, 0, 0, 0, 776, 0, 0, 3442, 3373, 0, 81, 0, 0, 1, 0, 0, 33, 214, 262, 0, 9, 331, 3670, 0, 933, 0, 1442, 1975, 2560, 3849, 0, 4096, 23, 115, 0, 4069, 24, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 216, 1714, 0, 0, 1, 125, 182, 4039, 2474, 162, 4082, 1942, 0, 22, 114, 2623, 0, 4095, 4091, 4076, 16, 1006, 115, 0, 87, 5, 0, 5, 0, 19, 0, 3857, 486, 174, 0, 1608, 974, 3534, 0, 0, 17, 3310, 0, 0, 0, 1, 3428, 0, 0, 1, 22, 0, 4093, 3699, 0, 248, 61, 3974, 2168, 2089, 581, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4032, 9, 6, 0, 5, 2606, 4096, 4096, 4095, 0, 1377, 3, 0, 0, 2599, 4045, 0, 4096, 3782, 0, 87, 19, 0, 1111, 0, 0, 29, 0, 0, 52, 49, 4096, 3, 9, 0, 0, 574, 0, 0, 462, 0, 0, 4095, 1, 4, 0, 3637, 0, 0, 0, 4031, 0, 1292, 2813, 0, 2, 0, 4005, 0, 0, 222, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 1698, 0, 0, 0, 0, 4027, 4096, 84, 0, 0, 0, 0, 0, 0, 3982, 0, 0, 2972, 2942, 10, 1, 30, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 1, 0, 642, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 63, 0, 37, 291, 0, 0, 936, 14, 0, 0, 0, 0, 0, 0, 4085, 0, 2976, 0, 0, 16, 3513, 1, 1137, 253, 0, 0, 0, 0, 0, 0, 0, 4094, 4080, 0, 4010, 0, 76, 0, 2415, 0, 0, 0, 0, 0, 0, 0, 3674, 0, 0, 1012, 5, 0, 2048, 537, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 28, 641, 130, 0, 0, 2671, 746, 549, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9911, 9937, 4678, 1845, 11981, 14675, 7276, 5233] avg=8192.0000 max=14675 max_rank=5 min=1845 min_rank=3 imbalance_ratio=1.791382 max_load_reduction=50861 max_load_reduction_pct=0.776077 moved_to_echo=39165 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[38, 183, 2341, 0, 0, 5, 3, 589, 3605, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2722, 2943, 4085, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3918, 51, 726, 0, 160, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 776, 0, 3676, 16, 0, 0, 0, 0, 0, 0, 0, 3442, 3373, 0, 81, 0, 0, 1, 1137, 633, 0, 0, 0, 0, 0, 0, 0, 0, 33, 214, 262, 0, 9, 331, 2732, 0, 0, 0, 0, 0, 0, 0, 3670, 0, 933, 0, 1442, 1975, 2560, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 23, 115, 0, 4069, 24, 0, 1871, 622, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7097, 10011, 8962, 4473, 8667, 3581, 11925, 10820] avg=8192.0000 max=11925 max_rank=6 min=3581 min_rank=5 imbalance_ratio=1.455688 max_load_reduction=53611 max_load_reduction_pct=0.818039 moved_to_echo=34993 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 465, 60, 2, 0, 9, 0, 0, 3870, 340, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 519, 2954, 1004, 2786, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 726, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 8, 4, 4096, 0, 3361, 491, 3756, 16, 0, 0, 0, 0, 0, 0, 0, 2276, 3, 38, 0, 5, 559, 0, 1141, 674, 0, 0, 0, 0, 0, 0, 0, 634, 151, 2, 3763, 0, 0, 41, 2732, 0, 0, 0, 0, 0, 0, 0, 3665, 318, 0, 15, 3214, 0, 154, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 18, 13, 806, 0, 0, 4062, 1990, 622, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4746, 11318, 8757, 12366, 4696, 7323, 8796, 7534] avg=8192.0000 max=12366 max_rank=3 min=4696 min_rank=4 imbalance_ratio=1.509521 max_load_reduction=53170 max_load_reduction_pct=0.811310 moved_to_echo=28349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4091, 0, 0, 21, 0, 0, 3853, 339, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 2880, 3900, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 194, 726, 358, 7, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1854, 0, 4096, 0, 0, 0, 3757, 16, 0, 0, 0, 0, 0, 0, 3873, 602, 746, 0, 0, 0, 3576, 0, 1113, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 0, 0, 6, 2732, 0, 0, 0, 0, 0, 0, 0, 1423, 460, 1, 0, 3479, 45, 1266, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 0, 713, 3358, 781, 6, 70, 1825, 622, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 9727, 5381, 9723, 10528, 6486, 7985, 7400] avg=8192.0000 max=10528 max_rank=4 min=5381 min_rank=2 imbalance_ratio=1.285156 max_load_reduction=55008 max_load_reduction_pct=0.839355 moved_to_echo=32399 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3542, 0, 0, 0, 2, 5, 3663, 340, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 4028, 23, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3764, 3192, 0, 193, 3, 4, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 3932, 6, 103, 4085, 3754, 4, 0, 0, 0, 0, 0, 0, 4095, 387, 2, 166, 38, 2, 7, 0, 8, 673, 0, 0, 0, 0, 0, 0, 799, 4087, 3026, 0, 181, 4, 0, 186, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 5, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 76, 7, 1, 411, 1988, 164, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7552, 11905, 8091, 15584, 5378, 9005, 5369, 2652] avg=8192.0000 max=15584 max_rank=3 min=2652 min_rank=7 imbalance_ratio=1.902344 max_load_reduction=49952 max_load_reduction_pct=0.762207 moved_to_echo=22404 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 1549, 902, 1, 0, 3962, 2, 3791, 305, 0, 0, 0, 0, 0, 0, 104, 0, 0, 149, 2027, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2929, 408, 1, 724, 67, 1622, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2644, 4060, 0, 3972, 0, 3370, 16, 0, 0, 0, 0, 0, 0, 147, 4089, 261, 1, 0, 367, 0, 0, 783, 673, 0, 0, 0, 0, 0, 0, 133, 8, 58, 7, 4077, 4095, 6, 0, 2722, 0, 0, 0, 0, 0, 0, 0, 3848, 0, 0, 0, 138, 0, 0, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 0, 3978, 2, 1486, 0, 179, 1986, 619, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10530, 2298, 5768, 14065, 6321, 11106, 5413, 10035] avg=8192.0000 max=14065 max_rank=3 min=2298 min_rank=1 imbalance_ratio=1.716919 max_load_reduction=51471 max_load_reduction_pct=0.785385 moved_to_echo=32875 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 216, 1714, 0, 0, 1, 125, 4039, 205, 0, 0, 0, 0, 0, 0, 182, 0, 0, 162, 2944, 1942, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 114, 2623, 0, 725, 4091, 4076, 16, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 87, 5, 0, 5, 0, 19, 2269, 16, 0, 0, 0, 0, 0, 0, 0, 3857, 486, 174, 0, 1608, 974, 3534, 1138, 609, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3310, 0, 0, 0, 1, 2732, 0, 0, 0, 0, 0, 0, 0, 3428, 0, 0, 1, 22, 0, 4093, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 61, 3974, 2168, 2089, 581, 0, 1798, 622, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6300, 5252, 12651, 2516, 12380, 6060, 8836, 11541] avg=8192.0000 max=12651 max_rank=2 min=2516 min_rank=3 imbalance_ratio=1.544312 max_load_reduction=52885 max_load_reduction_pct=0.806961 moved_to_echo=38817 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4032, 9, 6, 0, 5, 2606, 4096, 340, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 989, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2599, 4045, 0, 716, 3782, 0, 87, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 0, 0, 29, 0, 0, 52, 3755, 17, 0, 0, 0, 0, 0, 0, 49, 4096, 3, 9, 0, 0, 574, 0, 388, 466, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 4095, 1, 4, 0, 2738, 0, 0, 0, 0, 0, 0, 0, 3637, 0, 0, 0, 4031, 0, 1292, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4005, 0, 0, 222, 4096, 1371, 625, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11094, 5088, 11248, 4964, 5585, 7300, 9936, 10321] avg=8192.0000 max=11248 max_rank=2 min=4964 min_rank=3 imbalance_ratio=1.373047 max_load_reduction=54288 max_load_reduction_pct=0.828369 moved_to_echo=33142 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 6016, 2048, 2688, 16256, 384, 8320, 4736, 25344, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 6144, 256, 13440, 6656, 8448, 896, 11520, 13696, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[16896, 0, 0, 10624, 17792, 14848, 5760, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12288, 18816, 7936, 512, 128, 2048, 9216, 3584, 6912, 4608, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 9344, 7424, 3584, 20224, 4224, 128, 640, 19584, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 768, 17536, 2688, 128, 128, 4096, 7424, 31104, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[23424, 896, 1024, 1152, 16384, 2048, 11520, 9856, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[25600, 14208, 256, 5248, 12416, 5888, 1152, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[3064, 0, 0, 0, 3097, 4095, 800, 4096, 2222, 0, 0, 0, 0, 4073, 5, 0, 0, 0, 0, 3689, 0, 230, 1744, 4013, 0, 3, 8, 0, 0, 2828, 236, 4095, 0, 45, 190, 3101, 51, 57, 0, 4061, 15, 651, 0, 0, 4, 59, 488, 189, 80, 0, 3986, 0, 2, 358, 6, 0, 553, 3110, 3728, 0, 2495, 20, 3989, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2694, 238, 3367, 0, 0, 2614, 1654, 0, 0, 0, 1761, 3811, 6, 3710, 0, 3911, 209, 3178, 4049, 0, 0, 4004, 0, 1, 0, 36, 86, 2, 0, 3824, 0, 1, 3650, 50, 695, 13, 12, 623, 3, 0, 0, 3640, 4050, 0, 264, 1, 0, 0, 359, 2, 1630, 0, 0, 0, 47, 4012, 4087, 5, 55, 130, 3052, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[14, 1, 486, 4016, 4096, 4095, 4, 3303, 4076, 0, 151, 0, 1, 4092, 0, 136, 1814, 0, 0, 3943, 4090, 0, 0, 7, 0, 1, 0, 0, 156, 1, 0, 0, 0, 1300, 3591, 270, 85, 1205, 0, 815, 0, 0, 4070, 5, 0, 3, 393, 26, 0, 4012, 92, 0, 3998, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 2995, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 5, 74, 2532, 8, 580, 2753, 0, 0, 0, 121, 0, 3028, 0, 3926, 465, 0, 0, 4096, 0, 86, 2278, 3952, 0, 629, 1, 4096, 0, 347, 0, 3, 115, 2531, 188, 2895, 2075, 4011, 0, 3900, 0, 0, 1, 4083, 27, 0, 0, 1, 51, 0, 0, 0, 3727, 4093, 0, 0, 82, 0, 3229, 0, 4090, 0, 1457, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3783, 0, 10, 0, 6, 0, 409, 0, 4091, 0, 2063, 0, 340, 4007, 510, 1, 0, 204, 5, 4095, 4089, 0, 1229, 4091, 1, 0, 0, 113, 38, 4035, 0, 4082, 35, 0, 2686, 143, 131, 135, 4, 0, 5, 12, 1120, 5, 0, 0, 1, 3952, 2, 28, 2771, 0, 4066, 20, 0, 1, 0, 179, 4095, 626, 4082, 0, 4018, 217] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3889, 0, 0, 3983, 3888, 4095, 0, 4095, 3981, 63, 1, 4096, 4024, 2878, 417, 1028, 6, 0, 0, 4088, 648, 0, 0, 2847, 3, 3, 0, 0, 0, 0, 14, 4095, 0, 0, 1, 0, 0, 0, 0, 4096, 294, 17, 436, 1, 2029, 0, 0, 3797, 0, 365, 0, 0, 926, 112, 13, 0, 34, 65, 4096, 0, 1, 0, 1111, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[805, 116, 0, 0, 110, 74, 0, 3838, 3967, 427, 6, 2, 0, 4094, 4091, 1111, 0, 0, 6, 3923, 1564, 228, 534, 4096, 0, 0, 0, 1045, 0, 0, 2867, 4088, 0, 0, 816, 3495, 0, 4024, 0, 0, 0, 444, 0, 76, 0, 70, 4096, 0, 50, 229, 0, 221, 3814, 1854, 0, 0, 0, 181, 3739, 0, 1665, 2, 3765, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3349, 647, 1786, 2071, 674, 0, 0, 4096, 0, 4096, 80, 0, 71, 1, 218, 3864, 4019, 0, 12, 3664, 0, 403, 0, 0, 0, 19, 119, 4085, 3, 2495, 3167, 1, 732, 3347, 0, 10, 0, 0, 4052, 385, 0, 0, 0, 2264, 0, 3311, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 207, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3064, 0, 0, 0, 3097, 4095, 800, 958, 0, 0, 0, 0, 0, 0, 0, 0, 2222, 0, 0, 0, 0, 1720, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 855, 0, 230, 1744, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 0, 2828, 236, 4095, 2998, 1016, 0, 0, 0, 0, 0, 0, 0, 45, 190, 3101, 51, 57, 0, 4061, 560, 0, 0, 0, 0, 0, 0, 0, 15, 651, 0, 0, 4, 59, 488, 189, 2834, 140, 0, 0, 0, 0, 0, 0, 80, 0, 3986, 0, 2, 358, 6, 0, 1361, 1337, 0, 0, 0, 0, 0, 0, 553, 3110, 1807, 0, 2495, 20, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12014, 3947, 6842, 11184, 8065, 4380, 7130, 11974] avg=8192.0000 max=12014 max_rank=0 min=3947 min_rank=1 imbalance_ratio=1.466553 max_load_reduction=53522 max_load_reduction_pct=0.816681 moved_to_echo=31549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2694, 238, 3367, 0, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 1761, 1609, 6, 3710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 209, 737, 4049, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 36, 86, 2, 0, 3824, 1913, 951, 0, 0, 0, 0, 0, 0, 0, 1, 3650, 50, 695, 13, 12, 623, 614, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3640, 4050, 0, 264, 1, 2441, 89, 0, 0, 0, 0, 0, 0, 0, 0, 359, 2, 1630, 0, 0, 0, 1492, 1251, 0, 0, 0, 0, 0, 0, 47, 4012, 1981, 5, 55, 130, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6911, 8740, 12910, 6813, 5658, 10488, 4734, 9282] avg=8192.0000 max=12910 max_rank=2 min=4734 min_rank=6 imbalance_ratio=1.575928 max_load_reduction=52626 max_load_reduction_pct=0.803009 moved_to_echo=30162 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3889, 0, 0, 3983, 3888, 4095, 0, 958, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 63, 1, 4096, 4024, 1216, 417, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 947, 648, 0, 0, 2847, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 14, 4095, 2997, 718, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 615, 0, 0, 0, 0, 0, 0, 0, 294, 17, 436, 1, 2029, 0, 0, 3797, 3141, 140, 0, 0, 0, 0, 0, 0, 0, 365, 0, 0, 926, 112, 13, 0, 1496, 944, 0, 0, 0, 0, 0, 0, 34, 65, 1985, 0, 1, 0, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16813, 14826, 4448, 7830, 4712, 9855, 3856, 3196] avg=8192.0000 max=16813 max_rank=0 min=3196 min_rank=7 imbalance_ratio=2.052368 max_load_reduction=48723 max_load_reduction_pct=0.743454 moved_to_echo=21619 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[14, 1, 486, 4016, 4096, 4095, 4, 773, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 151, 0, 1, 1728, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 0, 0, 914, 4090, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 156, 1, 0, 0, 2417, 1021, 0, 0, 0, 0, 0, 0, 0, 1300, 3591, 270, 85, 1205, 0, 815, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 5, 0, 3, 393, 26, 3029, 113, 0, 0, 0, 0, 0, 0, 0, 4012, 92, 0, 3998, 0, 0, 1, 1496, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 1985, 0, 4096, 0, 2995, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13485, 6092, 6825, 3596, 7881, 7639, 10942, 9076] avg=8192.0000 max=13485 max_rank=0 min=3596 min_rank=3 imbalance_ratio=1.646118 max_load_reduction=52051 max_load_reduction_pct=0.794235 moved_to_echo=35538 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 74, 2532, 8, 580, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 1279, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 949, 0, 86, 2278, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 1, 4096, 0, 347, 0, 3, 2015, 755, 0, 0, 0, 0, 0, 0, 115, 2531, 188, 2895, 2075, 4011, 0, 3900, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4083, 27, 0, 0, 1, 3147, 94, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 3727, 4093, 0, 0, 1179, 994, 0, 0, 0, 0, 0, 0, 82, 0, 1565, 0, 4090, 0, 1457, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3843, 5326, 7730, 7846, 16200, 7353, 10044, 7194] avg=8192.0000 max=16200 max_rank=4 min=3843 min_rank=0 imbalance_ratio=1.977539 max_load_reduction=49336 max_load_reduction_pct=0.752808 moved_to_echo=40791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[805, 116, 0, 0, 110, 74, 0, 898, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 427, 6, 2, 0, 1729, 4091, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 909, 1564, 228, 534, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 0, 0, 2867, 4088, 2809, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 816, 3495, 0, 4024, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 76, 0, 70, 4096, 0, 3014, 131, 0, 0, 0, 0, 0, 0, 50, 229, 0, 221, 3814, 1854, 0, 0, 1365, 1344, 0, 0, 0, 0, 0, 0, 0, 181, 1812, 0, 1665, 2, 3765, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2003, 11333, 7337, 11830, 8897, 7831, 8877, 7428] avg=8192.0000 max=11830 max_rank=3 min=2003 min_rank=0 imbalance_ratio=1.444092 max_load_reduction=53706 max_load_reduction_pct=0.819489 moved_to_echo=33033 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3783, 0, 10, 0, 6, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2063, 0, 340, 1693, 510, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 5, 949, 4089, 0, 1229, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 113, 38, 4035, 0, 4082, 0, 999, 0, 0, 0, 0, 0, 0, 35, 0, 2686, 143, 131, 135, 4, 0, 615, 0, 0, 0, 0, 0, 0, 0, 5, 12, 1120, 5, 0, 0, 1, 3952, 3146, 0, 0, 0, 0, 0, 0, 0, 2, 28, 2771, 0, 4066, 20, 0, 1, 1495, 1315, 0, 0, 0, 0, 0, 0, 0, 179, 1985, 626, 4082, 0, 4018, 217, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4208, 8698, 10567, 9268, 3749, 8241, 9698, 11107] avg=8192.0000 max=11107 max_rank=7 min=3749 min_rank=4 imbalance_ratio=1.355835 max_load_reduction=54429 max_load_reduction_pct=0.830521 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3349, 647, 1786, 477, 0, 0, 0, 0, 0, 0, 0, 0, 674, 0, 0, 4096, 0, 1724, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 1, 218, 892, 4019, 0, 12, 3664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 19, 119, 4085, 1518, 1025, 0, 0, 0, 0, 0, 0, 3, 2495, 3167, 1, 732, 3347, 0, 10, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 385, 0, 0, 0, 2264, 2972, 76, 0, 0, 0, 0, 0, 0, 0, 3311, 0, 0, 4096, 0, 0, 0, 1501, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 1976, 0, 4096, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6259, 6574, 8877, 7169, 10374, 9749, 10255, 6279] avg=8192.0000 max=10374 max_rank=4 min=6259 min_rank=0 imbalance_ratio=1.266357 max_load_reduction=55162 max_load_reduction_pct=0.841705 moved_to_echo=36657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[11648, 128, 3200, 8320, 20480, 13056, 3584, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[768, 7552, 15104, 640, 20608, 256, 20608, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2432, 4224, 512, 7168, 18560, 640, 5888, 26752, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 6400, 14336, 9984, 3840, 12800, 128, 13568, 4736, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 1152, 9728, 8320, 6144, 256, 5248, 10240, 23808, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[20736, 512, 2304, 8320, 6144, 12800, 5120, 9984, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 8064, 7296, 256, 22272, 6528, 128, 128, 11392, 9984, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 1152, 128, 5376, 384, 7296, 3328, 24320, 16768, 7552, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[36, 0, 41, 65, 0, 0, 4013, 0, 16, 39, 3678, 496, 0, 4080, 12, 3104, 0, 0, 11, 2, 74, 24, 3710, 3962, 55, 3235, 4032, 18, 0, 3880, 105, 0, 0, 3976, 4056, 0, 149, 0, 3878, 13, 132, 3942, 291, 19, 0, 6, 1344, 4088, 0, 38, 3, 626, 4079, 0, 67, 22, 0, 0, 0, 0, 0, 23, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 2626, 4095, 3, 0, 598, 0, 0, 0, 2634, 1341, 0, 3984, 0, 0, 34, 7, 58, 0, 4066, 0, 0, 4075, 9, 3287, 182, 182, 42, 96, 115, 0, 3, 1231, 0, 15, 480, 0, 4096, 0, 0, 3774, 1639, 0, 2363, 0, 4038, 4034, 0, 3923, 5, 0, 4095, 151, 0, 4, 0, 35, 20, 16, 4082, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3015, 0, 3853, 0, 0, 19, 0, 3, 7, 0, 3, 3943, 3, 2392, 0, 2366, 0, 0, 0, 3846, 252, 0, 1170, 3052, 0, 4096, 4096, 0, 2300, 2537, 0, 37, 0, 0, 4096, 410, 0, 1182, 19, 1505, 4094, 4072, 26, 264, 0, 130, 0, 2115, 0, 4, 0, 1223, 5, 0, 157, 3533, 0, 51, 0, 0, 928, 0, 3610, 1122] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 4074, 0, 0, 109, 4091, 160, 0, 0, 3473, 1, 0, 0, 0, 248, 11, 2, 1, 1215, 4091, 0, 3283, 3864, 0, 22, 0, 0, 4087, 3870, 4095, 10, 0, 221, 2358, 4069, 0, 0, 4083, 44, 66, 3865, 0, 0, 1, 0, 4096, 3914, 0, 0, 0, 518, 651, 613, 0, 50, 15, 153, 0, 25, 4087, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3313, 22, 13, 0, 0, 0, 0, 0, 3949, 3779, 0, 3867, 1305, 4095, 0, 0, 0, 23, 2, 1, 3863, 582, 0, 3017, 4, 256, 0, 0, 0, 0, 0, 180, 0, 17, 44, 93, 4096, 3787, 0, 4061, 4096, 48, 2481, 0, 1694, 3103, 0, 4072, 0, 428, 3971, 0, 18, 3859, 0, 0, 0, 0, 1301, 1, 73, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 42, 4008, 0, 0, 0, 4022, 11, 0, 9, 3883, 288, 0, 259, 65, 0, 0, 0, 4, 500, 294, 34, 4096, 0, 0, 4096, 4096, 15, 55, 2850, 0, 549, 0, 9, 4050, 6, 0, 1752, 4096, 300, 49, 2531, 4095, 169, 0, 95, 3640, 0, 70, 3544, 0, 3797, 3648, 0, 0, 0, 613, 5, 0, 0, 0, 0, 3891, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[3171, 0, 2112, 0, 1, 445, 110, 12, 610, 0, 52, 0, 0, 0, 4082, 4070, 145, 5, 227, 0, 4016, 0, 0, 4096, 253, 0, 3912, 0, 0, 4092, 2778, 0, 8, 4096, 0, 2136, 210, 0, 2154, 8, 1351, 3731, 0, 37, 23, 173, 1312, 154, 0, 3467, 0, 0, 4055, 6, 357, 1, 127, 0, 5, 2, 3858, 0, 2, 4074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4084, 0, 2675, 0, 132, 3227, 4096, 1, 0, 0, 5, 3338, 0, 4085, 0, 15, 25, 10, 0, 0, 2303, 1, 4033, 4096, 0, 3759, 0, 0, 831, 0, 34, 8, 0, 0, 3689, 3405, 0, 2, 4081, 843, 4, 2928, 15, 0, 0, 0, 4091, 525, 57, 0, 0, 0, 294, 0, 4017, 4055, 0, 0, 0, 0, 0, 2, 770, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[36, 0, 41, 65, 0, 0, 4013, 0, 609, 462, 0, 0, 0, 0, 0, 0, 16, 39, 3678, 496, 0, 3198, 12, 3104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 74, 24, 3710, 3962, 80, 0, 0, 0, 0, 0, 0, 0, 55, 2415, 4032, 18, 0, 3880, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 4056, 0, 149, 0, 2502, 13, 0, 0, 0, 0, 0, 0, 0, 0, 132, 781, 291, 19, 0, 6, 1344, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3, 626, 4079, 0, 67, 22, 802, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4096, 0, 3161, 767, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5226, 10543, 7863, 10505, 10696, 6661, 5995, 8047] avg=8192.0000 max=10696 max_rank=4 min=5226 min_rank=0 imbalance_ratio=1.305664 max_load_reduction=54840 max_load_reduction_pct=0.836792 moved_to_echo=31399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4074, 0, 0, 109, 4091, 160, 641, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3473, 1, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 1, 1215, 4091, 0, 3283, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4087, 3870, 4095, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 2358, 4069, 0, 0, 2634, 44, 0, 0, 0, 0, 0, 0, 0, 0, 66, 766, 0, 0, 1, 0, 4096, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 651, 613, 0, 50, 0, 2, 0, 0, 0, 0, 0, 0, 15, 153, 0, 25, 4087, 0, 0, 0, 3099, 808, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9078, 3722, 12467, 12079, 9326, 8843, 1834, 8187] avg=8192.0000 max=12467 max_rank=2 min=1834 min_rank=6 imbalance_ratio=1.521851 max_load_reduction=53069 max_load_reduction_pct=0.809769 moved_to_echo=28190 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2626, 4095, 3, 0, 598, 0, 643, 469, 0, 0, 0, 0, 0, 0, 0, 0, 2634, 1341, 0, 3123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 7, 58, 0, 4066, 0, 0, 4075, 78, 0, 0, 0, 0, 0, 0, 0, 9, 2454, 182, 182, 42, 96, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1231, 0, 15, 480, 0, 2643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 1639, 0, 2363, 0, 4038, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 5, 0, 4095, 151, 0, 4, 783, 364, 0, 0, 0, 0, 0, 0, 0, 35, 20, 16, 4082, 1, 0, 4096, 3027, 810, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8435, 7098, 8318, 3080, 4372, 12821, 9325, 12087] avg=8192.0000 max=12821 max_rank=5 min=3080 min_rank=3 imbalance_ratio=1.565063 max_load_reduction=52715 max_load_reduction_pct=0.804367 moved_to_echo=38605 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 42, 4008, 0, 0, 0, 4022, 11, 643, 585, 0, 0, 0, 0, 0, 0, 0, 9, 3883, 288, 0, 204, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 500, 294, 34, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3058, 4096, 15, 55, 2850, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4050, 6, 0, 1752, 2643, 300, 0, 0, 0, 0, 0, 0, 0, 0, 49, 501, 4095, 169, 0, 95, 3640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 3544, 0, 3797, 3648, 0, 0, 0, 50, 453, 0, 0, 0, 0, 0, 0, 613, 5, 0, 0, 0, 0, 3891, 0, 2030, 810, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9311, 4449, 4933, 10623, 8760, 8549, 11562, 7349] avg=8192.0000 max=11562 max_rank=6 min=4449 min_rank=1 imbalance_ratio=1.411377 max_load_reduction=53974 max_load_reduction_pct=0.823578 moved_to_echo=36220 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3015, 0, 3853, 0, 0, 19, 0, 3, 2, 585, 0, 0, 0, 0, 0, 0, 7, 0, 3, 3943, 3, 1875, 0, 2366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 252, 0, 1170, 3052, 47, 0, 0, 0, 0, 0, 0, 0, 0, 3058, 4096, 0, 2300, 2537, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 410, 0, 1182, 14, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 806, 26, 264, 0, 130, 0, 2115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1223, 5, 0, 157, 3533, 470, 453, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 928, 0, 3610, 1122, 3266, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7477, 8197, 8367, 12028, 7207, 7435, 5845, 8980] avg=8192.0000 max=12028 max_rank=3 min=5845 min_rank=6 imbalance_ratio=1.468262 max_load_reduction=53508 max_load_reduction_pct=0.816467 moved_to_echo=29467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3171, 0, 2112, 0, 1, 445, 110, 12, 338, 0, 0, 0, 0, 0, 0, 0, 610, 0, 52, 0, 0, 0, 4082, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 145, 5, 227, 0, 4016, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 3912, 0, 0, 4092, 2778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 2136, 210, 0, 1390, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 739, 0, 37, 23, 173, 1312, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3467, 0, 0, 4055, 6, 357, 1, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 5, 2, 3858, 0, 2, 4074, 2992, 426, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6189, 8814, 8489, 11035, 7848, 3789, 7886, 11486] avg=8192.0000 max=11486 max_rank=7 min=3789 min_rank=5 imbalance_ratio=1.402100 max_load_reduction=54050 max_load_reduction_pct=0.824738 moved_to_echo=31009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3313, 22, 13, 0, 0, 0, 643, 431, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 3779, 0, 3031, 1305, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 1, 3863, 582, 76, 0, 0, 0, 0, 0, 0, 0, 0, 2252, 4, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 17, 44, 93, 2643, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 4096, 48, 2481, 0, 1694, 3103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 428, 3971, 0, 18, 3859, 760, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 1, 73, 22, 3257, 810, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4422, 16159, 4547, 2512, 6764, 12226, 13442, 5464] avg=8192.0000 max=16159 max_rank=1 min=2512 min_rank=3 imbalance_ratio=1.972534 max_load_reduction=49377 max_load_reduction_pct=0.753433 moved_to_echo=37896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 2675, 0, 132, 3227, 4096, 1, 644, 539, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3338, 0, 3196, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 25, 10, 0, 0, 2303, 1, 4033, 4096, 84, 0, 0, 0, 0, 0, 0, 0, 0, 2801, 0, 0, 831, 0, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 3405, 0, 2, 2624, 843, 0, 0, 0, 0, 0, 0, 0, 0, 4, 577, 15, 0, 0, 0, 4091, 525, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 294, 0, 4017, 4055, 805, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 770, 0, 2351, 813, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15398, 6554, 10552, 3674, 10563, 5212, 9647, 3936] avg=8192.0000 max=15398 max_rank=0 min=3674 min_rank=3 imbalance_ratio=1.879639 max_load_reduction=50138 max_load_reduction_pct=0.765045 moved_to_echo=29358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 16128, 16384, 512, 7424, 17408, 7168, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 15104, 128, 6656, 20864, 896, 4736, 11648, 3712, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5760, 5760, 10240, 640, 4992, 512, 20224, 18048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 9728, 18304, 10112, 896, 3072, 17152, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[10368, 128, 22784, 4224, 256, 3840, 17024, 256, 4224, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 256, 128, 128, 14336, 128, 12544, 9344, 23296, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 128, 384, 5632, 15104, 128, 20224, 23808, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 128, 17792, 13312, 128, 14720, 5504, 13952, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 1978, 0, 122, 4, 0, 4078, 3869, 0, 4090, 0, 0, 1, 3912, 3895, 0, 7, 0, 551, 2, 0, 0, 0, 123, 815, 2922, 0, 79, 1267, 775, 0, 30, 3076, 4070, 3486, 4, 4091, 2, 0, 3932, 0, 0, 19, 887, 2247, 154, 4053, 1252, 0, 0, 8, 0, 3361, 0, 0, 4094, 0, 365, 1915, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 74, 0, 62, 0, 7, 12, 13, 3986, 0, 374, 4096, 0, 0, 0, 0, 117, 0, 0, 3144, 632, 567, 3825, 91, 0, 3984, 0, 0, 0, 0, 3115, 102, 0, 894, 182, 3925, 3604, 66, 4096, 5, 0, 151, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4, 2, 396, 3917, 0, 17, 4095, 3920, 4042, 3794, 0, 0, 42] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 17, 0, 0, 32, 0, 0, 0, 0, 269, 0, 1984, 4096, 0, 0, 11, 0, 4059, 73, 0, 0, 17, 5, 592, 0, 1181, 0, 3593, 4018, 0, 3643, 762, 1, 4096, 4018, 94, 3471, 0, 6, 14, 39, 0, 41, 1493, 3398, 4057, 3454, 0, 778, 513, 127, 0, 4095, 0, 3080, 213, 0, 4089, 7, 68, 37, 3995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[50, 10, 0, 0, 0, 0, 12, 0, 3139, 0, 72, 0, 2, 1135, 1823, 0, 171, 0, 38, 9, 1684, 0, 282, 404, 0, 1372, 150, 4092, 4060, 0, 4095, 18, 0, 0, 0, 3664, 19, 0, 3880, 0, 0, 5, 0, 0, 0, 3971, 4090, 1884, 4077, 18, 8, 195, 0, 0, 4095, 0, 66, 3438, 2355, 3909, 2885, 263, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 651, 3282, 0, 1, 11, 220, 8, 0, 1998, 3988, 1, 0, 120, 2331, 4082, 0, 485, 3724, 2, 2805, 202, 0, 3, 699, 0, 67, 2, 40, 3833, 0, 0, 9, 9, 3483, 4082, 3965, 4092, 0, 1, 4087, 0, 2, 275, 4096, 123, 0, 1612, 114, 0, 0, 0, 0, 3979, 0, 0, 4036, 136, 2657, 92, 108, 0, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 164, 0, 958, 3649, 0, 391, 0, 15, 0, 30, 4095, 0, 2, 0, 2960, 0, 3949, 0, 3054, 9, 0, 0, 129, 677, 2417, 60, 3194, 0, 0, 0, 1075, 15, 0, 3384, 3804, 4088, 0, 3941, 57, 1, 0, 0, 0, 0, 4015, 1, 21, 4033, 836, 0, 0, 0, 43, 4096, 0, 0, 4093, 0, 1201, 29, 830, 156, 4064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 84, 3, 1542, 2, 0, 0, 0, 0, 762, 0, 3943, 0, 0, 0, 1823, 0, 4025, 0, 3479, 0, 0, 274, 0, 29, 5, 30, 4058, 0, 0, 152, 4096, 0, 0, 29, 3837, 3856, 250, 4091, 0, 84, 4089, 0, 0, 0, 551, 260, 0, 3222, 0, 2, 0, 1801, 0, 4096, 0, 30, 2937, 117, 3823, 16, 4003, 39, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[7, 0, 3817, 124, 0, 0, 254, 16, 312, 0, 0, 157, 0, 122, 0, 3872, 0, 4070, 0, 1418, 0, 131, 12, 2, 3441, 3716, 0, 2, 7, 0, 2913, 0, 50, 3, 1, 4094, 4096, 0, 3717, 1321, 29, 1404, 0, 2, 58, 0, 232, 6, 4063, 0, 933, 0, 0, 0, 4022, 0, 2, 4075, 4096, 363, 2315, 2165, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1978, 0, 122, 4, 0, 4091, 2517, 0, 0, 0, 0, 0, 0, 4078, 3869, 0, 4090, 0, 0, 1, 3912, 87, 77, 0, 0, 0, 0, 0, 0, 3895, 0, 7, 0, 551, 2, 0, 0, 140, 685, 108, 0, 0, 0, 0, 0, 0, 123, 815, 2922, 0, 79, 1180, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3076, 2686, 3486, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 19, 887, 2247, 154, 1577, 699, 0, 0, 0, 0, 0, 0, 4053, 1252, 0, 0, 8, 0, 3176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8712, 16114, 5388, 5894, 9284, 9515, 8489, 2140] avg=8192.0000 max=16114 max_rank=1 min=2140 min_rank=7 imbalance_ratio=1.967041 max_load_reduction=49422 max_load_reduction_pct=0.754120 moved_to_echo=29428 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 74, 0, 62, 0, 7, 12, 13, 4096, 2518, 0, 0, 0, 0, 0, 0, 3986, 0, 374, 4096, 0, 0, 0, 0, 214, 90, 0, 0, 0, 0, 0, 0, 117, 0, 0, 3144, 632, 567, 3825, 91, 1554, 660, 126, 0, 0, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 2901, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 894, 182, 2591, 3604, 66, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 4096, 1577, 674, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4, 2, 396, 3701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 3920, 2488, 3794, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6782, 8760, 10716, 6987, 7342, 6498, 8190, 10261] avg=8192.0000 max=10716 max_rank=2 min=6498 min_rank=5 imbalance_ratio=1.308105 max_load_reduction=54820 max_load_reduction_pct=0.836487 moved_to_echo=32291 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 84, 3, 1542, 2, 0, 0, 0, 4091, 1805, 0, 0, 0, 0, 0, 0, 0, 762, 0, 3943, 0, 0, 0, 1823, 10, 94, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 3479, 0, 0, 274, 0, 1470, 646, 132, 0, 0, 0, 0, 0, 29, 5, 30, 4058, 0, 0, 142, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2532, 3856, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 4089, 0, 0, 0, 551, 260, 0, 1132, 659, 0, 0, 0, 0, 0, 0, 3222, 0, 2, 0, 1801, 0, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 117, 2353, 16, 4003, 39, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7527, 6632, 10026, 8360, 6667, 6775, 8895, 10654] avg=8192.0000 max=10654 max_rank=7 min=6632 min_rank=1 imbalance_ratio=1.300537 max_load_reduction=54882 max_load_reduction_pct=0.837433 moved_to_echo=32991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 17, 0, 0, 32, 0, 3471, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 0, 1984, 4096, 0, 277, 94, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4059, 73, 0, 0, 17, 1572, 689, 132, 0, 0, 0, 0, 0, 5, 592, 0, 1181, 0, 3593, 3741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3643, 762, 1, 2704, 4018, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 14, 39, 0, 41, 1493, 3398, 4057, 82, 703, 0, 0, 0, 0, 0, 0, 3454, 0, 778, 513, 127, 0, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 0, 0, 2517, 7, 68, 37, 3995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3651, 6720, 6553, 9112, 11222, 9833, 8741, 9704] avg=8192.0000 max=11222 max_rank=4 min=3651 min_rank=0 imbalance_ratio=1.369873 max_load_reduction=54314 max_load_reduction_pct=0.828766 moved_to_echo=39500 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 651, 3282, 0, 1, 11, 220, 4092, 2481, 0, 0, 0, 0, 0, 0, 8, 0, 1998, 3988, 1, 0, 120, 2331, 264, 92, 0, 0, 0, 0, 0, 0, 4082, 0, 485, 3724, 2, 2805, 202, 0, 1022, 586, 128, 0, 0, 0, 0, 0, 3, 699, 0, 67, 2, 40, 3569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 2299, 4082, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 2, 275, 4096, 123, 0, 1555, 598, 0, 0, 0, 0, 0, 0, 1612, 114, 0, 0, 0, 0, 3759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 1635, 92, 108, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10738, 8802, 13036, 4380, 10364, 10737, 5485, 1994] avg=8192.0000 max=13036 max_rank=2 min=1994 min_rank=7 imbalance_ratio=1.591309 max_load_reduction=52500 max_load_reduction_pct=0.801086 moved_to_echo=28580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[50, 10, 0, 0, 0, 0, 12, 0, 3880, 2114, 0, 0, 0, 0, 0, 0, 3139, 0, 72, 0, 2, 1135, 1823, 0, 282, 94, 0, 0, 0, 0, 0, 0, 171, 0, 38, 9, 1684, 0, 282, 404, 1503, 617, 132, 0, 0, 0, 0, 0, 0, 1372, 150, 4092, 4060, 0, 3813, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2418, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3971, 4090, 1884, 1324, 629, 0, 0, 0, 0, 0, 0, 4077, 18, 8, 195, 0, 0, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 2355, 2406, 2885, 263, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6066, 6547, 4840, 13505, 2437, 11903, 8167, 12071] avg=8192.0000 max=13505 max_rank=3 min=2437 min_rank=4 imbalance_ratio=1.648560 max_load_reduction=52031 max_load_reduction_pct=0.793930 moved_to_echo=34578 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 164, 0, 958, 3649, 0, 391, 0, 3941, 2516, 0, 0, 0, 0, 0, 0, 15, 0, 30, 4095, 0, 2, 0, 2960, 0, 94, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 3054, 9, 0, 0, 129, 461, 640, 132, 0, 0, 0, 0, 0, 677, 2417, 60, 3194, 0, 0, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3384, 2511, 4088, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4015, 1, 21, 1577, 653, 0, 0, 0, 0, 0, 0, 4033, 836, 0, 0, 0, 43, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 740, 29, 830, 156, 4064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11619, 7196, 8374, 7423, 10055, 6268, 8782, 5819] avg=8192.0000 max=11619 max_rank=0 min=5819 min_rank=7 imbalance_ratio=1.418335 max_load_reduction=53917 max_load_reduction_pct=0.822708 moved_to_echo=30924 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 3817, 124, 0, 0, 254, 16, 3717, 2506, 0, 0, 0, 0, 0, 0, 312, 0, 0, 157, 0, 122, 0, 3872, 204, 98, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 1418, 0, 131, 12, 2, 144, 693, 133, 0, 0, 0, 0, 0, 3441, 3716, 0, 2, 7, 0, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 3, 1, 2694, 4096, 0, 0, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 29, 1404, 0, 2, 58, 0, 232, 6, 1569, 707, 0, 0, 0, 0, 0, 0, 4063, 0, 933, 0, 0, 0, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 219, 2315, 2165, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10441, 4765, 6603, 9875, 8165, 4007, 8787, 12893] avg=8192.0000 max=12893 max_rank=7 min=4007 min_rank=5 imbalance_ratio=1.573853 max_load_reduction=52643 max_load_reduction_pct=0.803268 moved_to_echo=33852 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3200, 0, 10624, 12672, 11136, 7552, 256, 20480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 12160, 640, 18944, 3072, 3584, 4608, 768, 7936, 5248, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3712, 1792, 6784, 20480, 27264, 4480, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[11648, 4736, 2560, 20736, 128, 3328, 6144, 14976, 1408, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4224, 12928, 1152, 15616, 4096, 3712, 18176, 6144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 13696, 128, 128, 512, 15104, 10368, 10240, 10496, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 384, 4480, 8064, 3712, 256, 768, 256, 31488, 16640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[28672, 2304, 1792, 768, 2048, 512, 29952, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4, 26, 3995, 4044, 31, 1, 338, 4081, 7, 0, 0, 1, 4096, 7, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3921, 30, 0, 2728, 4061, 0, 0, 3818, 4096, 117, 781, 12, 4082, 3310, 0, 207, 310, 0, 962, 0, 0, 100, 67, 34, 0, 0, 8, 4093, 4095, 2, 0, 0, 0, 1, 4092, 0, 1171, 2708] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1892, 699, 0, 0, 1318, 1417, 0, 4081, 0, 51, 160, 10, 3, 4, 0, 0, 19, 4095, 6, 4074, 0, 4096, 0, 2548, 1, 0, 1960, 3875, 0, 1681, 0, 0, 0, 32, 0, 0, 0, 3927, 0, 146, 4018, 3190, 0, 4070, 0, 2439, 0, 0, 0, 0, 0, 4091, 1, 79, 0, 0, 3172, 191, 4095, 0, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[5, 0, 16, 4066, 606, 3910, 0, 4096, 1, 4086, 1631, 359, 0, 0, 0, 3971, 10, 6, 0, 84, 0, 8, 1, 4081, 0, 3575, 0, 0, 4031, 2351, 3570, 0, 34, 0, 952, 0, 0, 77, 3362, 218, 0, 0, 0, 24, 0, 919, 0, 3544, 5, 0, 6, 0, 97, 4091, 963, 2, 259, 0, 0, 0, 3345, 219, 3730, 3225] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 275, 0, 1888, 3695, 3507, 1233, 3333, 3551, 0, 0, 6, 3, 0, 0, 486, 0, 1, 2356, 0, 0, 1, 0, 0, 18, 4084, 17, 4091, 1276, 0, 0, 509, 4080, 692, 2062, 0, 0, 4095, 0, 0, 3656, 3, 2990, 3944, 1, 1742, 0, 49, 4, 0, 4096, 1659, 4096, 0, 6, 2031, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 165, 401, 3882, 0, 0, 4096, 4087, 0, 392, 0, 2352, 64, 0, 0, 0, 0, 3596, 0, 3544, 1, 4096, 0, 29, 58, 1809, 0, 0, 114, 806, 3097, 734, 89, 2253, 4094, 94, 1, 38, 4095, 0, 0, 0, 1479, 4096, 0, 3999, 0, 0, 3428, 0, 0, 0, 3916, 4096, 0, 85, 0, 7, 60, 0, 224, 29, 129, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[84, 22, 19, 3310, 0, 4085, 3868, 4091, 0, 4026, 1085, 4074, 1327, 3, 0, 0, 3644, 4096, 445, 571, 0, 2546, 1, 43, 0, 3995, 0, 0, 0, 1513, 3327, 0, 0, 111, 0, 149, 0, 1, 3867, 3601, 0, 0, 14, 4064, 0, 0, 0, 294, 3, 0, 0, 10, 0, 2558, 0, 0, 0, 0, 329, 260, 4096, 0, 1, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[95, 1, 28, 0, 3, 4096, 0, 85, 0, 342, 131, 0, 0, 0, 0, 0, 42, 3463, 4096, 2908, 18, 0, 0, 181, 0, 201, 0, 0, 0, 0, 4092, 0, 302, 1, 59, 0, 1, 564, 3546, 2373, 0, 0, 3884, 3927, 3927, 4065, 0, 4096, 0, 0, 0, 0, 16, 4042, 3358, 4050, 0, 0, 5, 0, 3851, 2384, 1303, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 181, 4096, 2749, 0, 4042, 747, 3513, 0, 4096, 2, 0, 4093, 40, 0, 0, 0, 169, 5, 309, 33, 4073, 3942, 4041, 1089, 0, 14, 1297, 6, 0, 128, 1, 0, 407, 3283, 4096, 0, 0, 1814, 4093, 7, 0, 3870, 0, 3774, 1, 0, 19, 0, 0, 0, 0, 0, 3, 35, 0, 0, 0, 2, 314, 840, 117, 4096, 99] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 26, 3995, 4044, 31, 1, 338, 453, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 4096, 7, 0, 3, 2774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 30, 0, 2728, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 4096, 117, 781, 12, 4082, 1833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 163, 0, 962, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 67, 34, 0, 0, 8, 4093, 4095, 2, 144, 818, 116, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 1171, 2708, 1477, 854, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8892, 6888, 3278, 10624, 14739, 1432, 9377, 10306] avg=8192.0000 max=14739 max_rank=4 min=1432 min_rank=5 imbalance_ratio=1.799194 max_load_reduction=50797 max_load_reduction_pct=0.775101 moved_to_echo=35854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1892, 699, 0, 0, 1318, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 51, 160, 10, 3, 4, 963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4095, 6, 3260, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2548, 1, 0, 1960, 3875, 0, 1681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 2174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 2096, 3190, 0, 4070, 0, 2439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 79, 1879, 814, 0, 0, 0, 0, 0, 0, 0, 0, 3172, 191, 4095, 0, 4095, 0, 1753, 296, 43, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 5272, 11476, 10065, 2206, 11941, 6864, 13645] avg=8192.0000 max=13645 max_rank=7 min=2206 min_rank=4 imbalance_ratio=1.665649 max_load_reduction=51891 max_load_reduction_pct=0.791794 moved_to_echo=34656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 165, 401, 3882, 0, 0, 4096, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 2352, 64, 0, 0, 0, 2778, 0, 0, 0, 0, 0, 0, 0, 0, 3596, 0, 3544, 1, 3278, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 58, 1809, 0, 0, 114, 806, 3008, 734, 0, 0, 0, 0, 0, 0, 0, 0, 89, 2253, 4094, 94, 1, 38, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 773, 4096, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3428, 0, 0, 0, 3916, 4096, 0, 85, 691, 818, 89, 0, 0, 0, 0, 0, 0, 7, 60, 0, 224, 29, 129, 0, 0, 855, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8999, 5586, 10448, 6529, 10664, 8868, 13123, 1319] avg=8192.0000 max=13123 max_rank=6 min=1319 min_rank=7 imbalance_ratio=1.601929 max_load_reduction=52413 max_load_reduction_pct=0.799759 moved_to_echo=33974 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 16, 4066, 606, 3910, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 1631, 359, 0, 0, 0, 3971, 2784, 0, 0, 0, 0, 0, 0, 0, 10, 6, 0, 84, 0, 7, 1, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3575, 0, 0, 4031, 2351, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 952, 0, 0, 77, 3362, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 919, 0, 3544, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 97, 4091, 963, 2, 0, 1, 102, 0, 0, 0, 0, 0, 259, 0, 0, 0, 3345, 219, 3730, 3225, 97, 857, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9058, 12832, 4189, 13425, 4546, 4487, 5267, 11732] avg=8192.0000 max=13425 max_rank=3 min=4189 min_rank=2 imbalance_ratio=1.638794 max_load_reduction=52111 max_load_reduction_pct=0.795151 moved_to_echo=26032 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 275, 0, 1888, 3695, 389, 0, 0, 0, 0, 0, 0, 0, 0, 1233, 3333, 3551, 0, 0, 6, 3, 0, 2384, 0, 0, 0, 0, 0, 0, 0, 0, 486, 0, 1, 2356, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4084, 17, 4091, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 4080, 692, 2062, 0, 0, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1908, 3, 2990, 3944, 1, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4, 0, 4096, 1659, 4096, 0, 1709, 0, 36, 0, 0, 0, 0, 0, 6, 2031, 0, 0, 0, 0, 0, 0, 1828, 734, 39, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6247, 10510, 2844, 9450, 9610, 10588, 11649, 4638] avg=8192.0000 max=11649 max_rank=6 min=2844 min_rank=2 imbalance_ratio=1.421997 max_load_reduction=53887 max_load_reduction_pct=0.822250 moved_to_echo=36485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[84, 22, 19, 3310, 0, 4085, 3868, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 1085, 4074, 1327, 3, 0, 0, 2781, 0, 0, 0, 0, 0, 0, 0, 3644, 4096, 445, 571, 0, 2038, 1, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 0, 1513, 3232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 149, 0, 1, 3867, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4064, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 10, 0, 2558, 0, 0, 6, 508, 95, 0, 0, 0, 0, 0, 0, 0, 329, 260, 4096, 0, 1, 3, 1607, 856, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11842, 13296, 10838, 8740, 6122, 4366, 3180, 7152] avg=8192.0000 max=13296 max_rank=1 min=3180 min_rank=6 imbalance_ratio=1.623047 max_load_reduction=52240 max_load_reduction_pct=0.797119 moved_to_echo=20820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[95, 1, 28, 0, 3, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 131, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 42, 3463, 4096, 2908, 18, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 1, 59, 0, 1, 564, 3546, 1314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2027, 3927, 3927, 4065, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4042, 3358, 4050, 1816, 0, 117, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3851, 2384, 1303, 0, 1059, 17, 41, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4234, 530, 10708, 4176, 5787, 18042, 13399, 8660] avg=8192.0000 max=18042 max_rank=5 min=530 min_rank=1 imbalance_ratio=2.202393 max_load_reduction=47494 max_load_reduction_pct=0.724701 moved_to_echo=45888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 181, 4096, 2749, 0, 4042, 747, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4093, 40, 0, 0, 2391, 0, 0, 0, 0, 0, 0, 0, 0, 169, 5, 309, 33, 3256, 3942, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 0, 14, 1297, 6, 0, 120, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 3283, 4096, 0, 0, 1814, 2262, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2011, 0, 3774, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 35, 0, 1814, 817, 8, 0, 0, 0, 0, 0, 0, 0, 2, 314, 840, 117, 4096, 99, 1831, 740, 45, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12197, 10622, 11755, 2527, 11862, 5812, 2677, 8084] avg=8192.0000 max=12197 max_rank=0 min=2527 min_rank=3 imbalance_ratio=1.488892 max_load_reduction=53339 max_load_reduction_pct=0.813889 moved_to_echo=28435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1280, 20480, 6400, 6912, 9856, 128, 128, 4096, 17024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3712, 11904, 4608, 11520, 2432, 15232, 3968, 12544, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 7168, 16640, 5248, 2944, 768, 20864, 12032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3584, 128, 128, 128, 8192, 24704, 12672, 4224, 8064, 3840, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1152, 12160, 3968, 5504, 6144, 15488, 19072, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 2048, 3584, 768, 20608, 2816, 14592, 6144, 9728, 5248, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 512, 10496, 19072, 640, 18048, 14080, 2816, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 384, 9088, 15360, 11776, 17024, 128, 12288, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1373, 638, 3, 0, 1, 4091, 16, 2, 0, 0, 3450, 0, 0, 0, 0, 0, 3, 126, 0, 0, 4083, 1288, 0, 42, 1103, 236, 4096, 0, 0, 0, 0, 0, 0, 0, 570, 7, 4091, 4077, 4096, 4094, 3, 2387, 3507, 927, 0, 453, 0, 4091, 1, 3843, 692, 2468, 363, 0, 4078, 2, 0, 0, 10, 1103, 0, 25, 4092, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 1106, 198, 6, 0, 112, 0, 0, 0, 339, 3647, 4075, 7, 3855, 1577, 0, 9, 0, 4096, 0, 1721, 4094, 0, 0, 1, 4095, 113, 0, 3102, 0, 4096, 0, 46, 1, 1, 352, 3317, 2053, 4085, 16, 0, 3861, 1, 253, 428, 11, 1, 3800, 0, 9, 479, 0, 0, 0, 4071, 0, 2, 0, 2466, 0, 4034, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[511, 585, 3904, 0, 0, 0, 0, 62, 1, 0, 3644, 3466, 12, 4095, 0, 0, 0, 0, 4096, 0, 1000, 4086, 0, 729, 0, 4075, 0, 3564, 3989, 750, 0, 0, 58, 0, 22, 0, 4089, 4093, 0, 4091, 1, 4046, 0, 0, 754, 2, 0, 0, 4055, 612, 81, 8, 3932, 0, 0, 103, 0, 0, 131, 0, 0, 0, 771, 118] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[20, 3899, 4094, 0, 1219, 0, 0, 0, 59, 589, 4063, 51, 4062, 3553, 0, 0, 0, 0, 0, 47, 4036, 0, 6, 3727, 0, 0, 3409, 0, 0, 1, 4089, 80, 210, 186, 0, 689, 0, 0, 658, 20, 277, 3981, 0, 3, 33, 0, 1, 0, 2419, 5, 0, 4088, 1, 4088, 3586, 0, 0, 0, 761, 0, 3690, 0, 3836, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[541, 0, 3940, 0, 0, 9, 69, 0, 1, 1277, 4041, 335, 0, 4095, 0, 0, 0, 0, 8, 1, 4095, 2859, 0, 140, 0, 4095, 1325, 40, 0, 1, 0, 0, 1, 0, 88, 57, 4094, 3655, 3092, 4072, 100, 4065, 3056, 67, 4096, 3901, 12, 32, 4096, 0, 0, 0, 3647, 392, 3, 0, 1, 0, 0, 0, 137, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[232, 0, 271, 0, 0, 3987, 4089, 0, 0, 0, 0, 3994, 0, 4093, 0, 3805, 691, 0, 3954, 0, 0, 5, 10, 0, 0, 4094, 4061, 3972, 2575, 39, 3359, 1, 0, 0, 204, 0, 27, 827, 1540, 4083, 589, 2, 350, 759, 0, 3947, 12, 1876, 30, 186, 0, 19, 17, 0, 0, 0, 4, 0, 4066, 120, 227, 4, 3415, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[12, 1488, 2528, 3022, 3139, 0, 84, 0, 0, 0, 2197, 878, 0, 1714, 0, 0, 40, 0, 1351, 30, 0, 354, 3, 7, 0, 1, 4084, 47, 3655, 4096, 0, 2128, 21, 0, 73, 0, 3231, 22, 2024, 1, 18, 3790, 4096, 0, 3917, 4080, 39, 328, 2981, 0, 2095, 4030, 0, 0, 3866, 56, 0, 0, 0, 0, 0, 2, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 56, 0, 277, 214, 176, 0, 3560, 0, 0, 4033, 3481, 0, 0, 0, 4078, 87, 0, 4096, 1, 4093, 3914, 0, 552, 0, 0, 3, 0, 0, 0, 0, 0, 1084, 155, 0, 2779, 2680, 0, 462, 0, 0, 4096, 0, 813, 13, 3032, 0, 1055, 0, 972, 4078, 13, 1828, 0, 3816, 0, 0, 1, 3614, 2306, 0, 0, 0, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[511, 585, 3904, 0, 0, 0, 0, 62, 0, 674, 0, 0, 0, 0, 0, 0, 1, 0, 1751, 3466, 12, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1000, 4086, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 3564, 3989, 750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 22, 0, 2255, 4093, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2284, 0, 0, 754, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 612, 81, 8, 3932, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 771, 118, 1893, 1762, 1160, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5736, 9325, 9911, 12378, 10519, 3041, 8791, 5835] avg=8192.0000 max=12378 max_rank=3 min=3041 min_rank=5 imbalance_ratio=1.510986 max_load_reduction=53158 max_load_reduction_pct=0.811127 moved_to_echo=28186 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[20, 3899, 4094, 0, 1219, 0, 0, 0, 1666, 0, 0, 0, 0, 0, 0, 0, 59, 589, 1952, 51, 4062, 3553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4036, 0, 6, 3727, 1029, 96, 0, 0, 0, 0, 0, 0, 0, 0, 1647, 0, 0, 1, 4089, 80, 0, 0, 0, 0, 0, 0, 0, 0, 210, 186, 0, 689, 0, 0, 658, 20, 0, 0, 0, 0, 0, 0, 0, 0, 277, 2247, 0, 3, 33, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2419, 5, 0, 4088, 1, 4088, 2557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 0, 3690, 0, 3836, 0, 2111, 1734, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10898, 10266, 8941, 5817, 1763, 2561, 13158, 12132] avg=8192.0000 max=13158 max_rank=6 min=1763 min_rank=4 imbalance_ratio=1.606201 max_load_reduction=52378 max_load_reduction_pct=0.799225 moved_to_echo=29614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[541, 0, 3940, 0, 0, 9, 69, 0, 647, 675, 0, 0, 0, 0, 0, 0, 1, 1277, 1941, 335, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 4095, 2859, 0, 140, 0, 37, 0, 0, 0, 0, 0, 0, 0, 4095, 641, 40, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 88, 57, 2258, 3655, 3092, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 100, 2295, 3056, 67, 4096, 3901, 12, 32, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3647, 392, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 137, 0, 0, 0, 2100, 1770, 1161, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5881, 7649, 7140, 4777, 13223, 13559, 8138, 5169] avg=8192.0000 max=13559 max_rank=5 min=4777 min_rank=3 imbalance_ratio=1.655151 max_load_reduction=51977 max_load_reduction_pct=0.793106 moved_to_echo=40089 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1373, 638, 3, 0, 1, 4091, 16, 2, 2004, 674, 0, 0, 0, 0, 0, 0, 0, 0, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 126, 0, 0, 4083, 1288, 0, 42, 1170, 118, 0, 0, 0, 0, 0, 0, 1103, 236, 1974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 7, 2257, 4077, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1348, 3507, 927, 0, 453, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3843, 692, 2468, 363, 0, 2908, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1103, 0, 25, 4092, 5, 1792, 1039, 1160, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8802, 1658, 6830, 3313, 15101, 10329, 10277, 9226] avg=8192.0000 max=15101 max_rank=4 min=1658 min_rank=1 imbalance_ratio=1.843384 max_load_reduction=50435 max_load_reduction_pct=0.769577 moved_to_echo=44933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1106, 198, 6, 0, 112, 0, 0, 55, 546, 0, 0, 0, 0, 0, 0, 0, 339, 1752, 4075, 7, 3855, 1577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 1721, 4094, 0, 0, 1168, 3, 0, 0, 0, 0, 0, 0, 1, 4095, 55, 0, 3102, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 1, 352, 1830, 2053, 4085, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2179, 1, 253, 428, 11, 1, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 479, 0, 0, 0, 2903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2466, 0, 4034, 0, 0, 0, 1895, 1682, 941, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2023, 11605, 11091, 11349, 8384, 6673, 3391, 11020] avg=8192.0000 max=11605 max_rank=1 min=2023 min_rank=0 imbalance_ratio=1.416626 max_load_reduction=53931 max_load_reduction_pct=0.822922 moved_to_echo=29468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[232, 0, 271, 0, 0, 3987, 4089, 0, 1985, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 4093, 0, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, 3954, 0, 0, 5, 10, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 4094, 1962, 3972, 2575, 39, 3359, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 16, 827, 1540, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 589, 2, 350, 759, 0, 3947, 12, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 30, 186, 0, 19, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4066, 120, 227, 4, 3415, 0, 0, 0, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10568, 11892, 4774, 16002, 6670, 7535, 252, 7843] avg=8192.0000 max=16002 max_rank=3 min=252 min_rank=6 imbalance_ratio=1.953369 max_load_reduction=49534 max_load_reduction_pct=0.755829 moved_to_echo=22300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[12, 1488, 2528, 3022, 3139, 0, 84, 0, 1996, 532, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 878, 0, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 1351, 30, 0, 354, 3, 7, 1109, 115, 0, 0, 0, 0, 0, 0, 0, 1, 1973, 47, 3655, 4096, 0, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 73, 0, 1783, 22, 2024, 1, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2139, 4096, 0, 3917, 4080, 39, 328, 0, 0, 0, 0, 0, 0, 0, 0, 2981, 0, 2095, 4030, 0, 0, 2757, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 1141, 1651, 916, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12801, 3648, 3009, 11900, 3924, 14617, 11919, 3718] avg=8192.0000 max=14617 max_rank=5 min=3009 min_rank=2 imbalance_ratio=1.784302 max_load_reduction=50919 max_load_reduction_pct=0.776962 moved_to_echo=34178 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 56, 0, 277, 214, 176, 0, 3560, 3, 445, 0, 0, 0, 0, 0, 0, 0, 0, 1934, 3481, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 4096, 1, 4093, 3914, 0, 552, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 155, 0, 2779, 1472, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2308, 0, 813, 13, 3032, 0, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 972, 4078, 13, 1828, 0, 2719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3614, 2306, 0, 0, 0, 22, 2099, 1788, 763, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8827, 9493, 13840, 0, 5952, 7221, 9610, 10593] avg=8192.0000 max=13840 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.689453 max_load_reduction=51696 max_load_reduction_pct=0.788818 moved_to_echo=33376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1152, 16640, 8320, 7680, 13440, 4992, 11648, 2304, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 11136, 3584, 8192, 128, 12160, 256, 13056, 11520, 6144, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[13696, 5632, 7552, 10752, 9856, 4480, 13952, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1536, 384, 1024, 3968, 11904, 14848, 16000, 16384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 14848, 11136, 2944, 9344, 15488, 128, 11264, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6912, 7808, 14976, 3328, 4608, 8448, 4352, 3712, 8448, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 2304, 12160, 16384, 4096, 21504, 1664, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[896, 128, 17664, 128, 19072, 16640, 128, 5248, 5632, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 3996, 4091, 0, 0, 67, 611, 3697, 4, 0, 4085, 342, 0, 26, 4058, 3569, 0, 68, 1199, 0, 0, 13, 121, 5, 0, 3008, 2037, 31, 367, 0, 3490, 0, 1132, 0, 2174, 3940, 1291, 0, 0, 0, 3748, 51, 0, 0, 3990, 17, 32, 0, 4074, 0, 970, 0, 600, 0, 113, 445, 262, 3871, 3939, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1, 23, 0, 3855, 71, 1835, 0, 34, 3775, 0, 0, 4096, 0, 2024, 260, 3943, 1545, 2, 0, 0, 0, 6, 309, 6, 0, 0, 3165, 3971, 0, 0, 0, 4063, 220, 0, 0, 3573, 0, 418, 4, 0, 3235, 0, 0, 1, 1318, 0, 729, 0, 4096, 0, 4076, 3035, 10, 1858, 2, 98, 0, 0, 4096, 4096, 21, 84, 1474, 108] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 2435, 3124, 0, 1, 0, 527, 4096, 8, 489, 3953, 1033, 3479, 0, 4025, 4050, 6, 0, 721, 0, 0, 1607, 4090, 0, 0, 4096, 2368, 0, 0, 1, 4095, 0, 1409, 0, 0, 0, 0, 53, 0, 0, 0, 1255, 0, 540, 70, 80, 0, 4096, 0, 0, 4094, 0, 2, 0, 28, 1, 12, 3716, 1887, 0, 4089, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4031, 0, 4094, 2784, 4096, 0, 0, 14, 3954, 0, 3, 0, 0, 7, 0, 4096, 3981, 0, 69, 0, 2272, 0, 543, 0, 0, 0, 0, 1899, 0, 106, 0, 3928, 0, 79, 0, 0, 3602, 0, 0, 0, 2752, 246, 4096, 0, 0, 2686, 471, 23, 3859, 967, 0, 214, 913, 132, 0, 0, 0, 227, 1687, 3470, 0, 3543, 660, 32] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[2644, 1, 0, 3438, 204, 4074, 0, 1596, 4096, 0, 0, 0, 0, 4096, 0, 3670, 4087, 4096, 0, 77, 0, 24, 0, 36, 0, 6, 482, 4083, 307, 4096, 0, 0, 0, 2876, 0, 6, 4070, 0, 0, 205, 0, 3863, 0, 0, 0, 8, 0, 22, 4096, 0, 0, 12, 13, 3883, 48, 0, 7, 0, 1368, 0, 0, 3849, 0, 97] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[3968, 0, 4092, 0, 4, 2035, 777, 4063, 0, 0, 0, 0, 0, 204, 0, 3559, 3962, 3990, 0, 0, 1273, 1428, 545, 109, 8, 26, 3987, 4096, 0, 2, 0, 752, 6, 0, 0, 0, 2145, 0, 2644, 64, 0, 0, 60, 257, 49, 4096, 7, 0, 9, 0, 0, 1408, 3389, 0, 178, 4092, 0, 5, 4093, 105, 18, 4031, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[3930, 3, 4048, 3783, 0, 108, 0, 51, 1677, 0, 0, 0, 0, 0, 71, 2, 4096, 4072, 0, 0, 212, 0, 2331, 8, 0, 0, 3313, 1928, 0, 395, 0, 4096, 8, 4065, 0, 59, 4086, 1, 0, 2200, 1, 0, 90, 8, 192, 0, 375, 0, 3983, 0, 8, 4093, 0, 242, 0, 4094, 2, 4, 0, 3847, 13, 4040, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[8, 0, 100, 3587, 0, 502, 0, 0, 2, 0, 0, 0, 0, 3994, 104, 4096, 4095, 4028, 0, 0, 1189, 0, 0, 0, 0, 12, 3509, 4095, 0, 0, 0, 4088, 0, 2033, 0, 467, 26, 4096, 0, 0, 879, 3, 1, 10, 0, 0, 0, 4093, 2862, 0, 14, 3893, 0, 0, 1, 4096, 1170, 24, 2, 4093, 0, 4067, 297, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 23, 0, 1797, 71, 1835, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 4096, 0, 2024, 260, 2293, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 2, 0, 0, 0, 6, 309, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3165, 3971, 0, 0, 0, 2344, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 3573, 0, 418, 4, 0, 1467, 1445, 5, 0, 0, 0, 0, 0, 3235, 0, 0, 1, 1318, 0, 729, 0, 2058, 1650, 252, 0, 0, 0, 0, 0, 2651, 0, 4076, 3035, 10, 1858, 2, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 21, 73, 1474, 108, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3761, 12448, 1874, 9480, 7132, 9243, 11730, 9868] avg=8192.0000 max=12448 max_rank=1 min=1874 min_rank=2 imbalance_ratio=1.519531 max_load_reduction=53088 max_load_reduction_pct=0.810059 moved_to_echo=37973 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3996, 1906, 0, 0, 67, 611, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 4, 0, 4085, 342, 0, 26, 2360, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 0, 68, 1199, 0, 0, 13, 121, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3008, 2037, 31, 367, 0, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 0, 2174, 3940, 1291, 0, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 51, 0, 0, 3990, 17, 32, 2185, 1698, 216, 0, 0, 0, 0, 0, 0, 4074, 0, 970, 0, 600, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 445, 262, 3871, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6582, 10514, 4970, 7462, 9797, 11937, 5757, 8517] avg=8192.0000 max=11937 max_rank=5 min=4970 min_rank=2 imbalance_ratio=1.457153 max_load_reduction=53599 max_load_reduction_pct=0.817856 moved_to_echo=36008 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2435, 1456, 0, 1, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 489, 3953, 1033, 3479, 0, 2341, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 6, 0, 721, 0, 0, 1607, 4090, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2368, 0, 0, 1, 2363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 0, 0, 0, 0, 53, 0, 1478, 1445, 278, 0, 0, 0, 0, 0, 0, 0, 1255, 0, 540, 70, 80, 0, 1668, 1684, 254, 0, 0, 0, 0, 0, 2651, 0, 0, 4094, 0, 2, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 3716, 1887, 0, 3518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4419, 15399, 10767, 8828, 4663, 5551, 6775, 9134] avg=8192.0000 max=15399 max_rank=1 min=4419 min_rank=0 imbalance_ratio=1.879761 max_load_reduction=50137 max_load_reduction_pct=0.765030 moved_to_echo=26123 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4031, 0, 4094, 1298, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 3, 0, 0, 7, 0, 2382, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 69, 0, 2272, 0, 543, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1899, 0, 106, 0, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 3602, 0, 0, 0, 1418, 1361, 241, 0, 0, 0, 0, 0, 2752, 246, 4096, 0, 0, 2686, 471, 23, 1486, 1714, 243, 0, 0, 0, 0, 0, 2498, 967, 0, 214, 913, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 1687, 3470, 0, 3048, 660, 32, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13533, 6346, 7119, 4272, 6701, 13717, 4724, 9124] avg=8192.0000 max=13717 max_rank=5 min=4272 min_rank=3 imbalance_ratio=1.674438 max_load_reduction=51819 max_load_reduction_pct=0.790695 moved_to_echo=34266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3968, 0, 4092, 0, 4, 2035, 777, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 3990, 0, 0, 1273, 1428, 545, 109, 289, 0, 0, 0, 0, 0, 0, 0, 8, 26, 3987, 4096, 0, 2, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2145, 0, 2644, 64, 271, 3, 274, 0, 0, 0, 0, 0, 0, 0, 60, 257, 49, 4096, 7, 0, 0, 1489, 46, 0, 0, 0, 0, 0, 6, 0, 0, 1408, 3389, 0, 178, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4093, 105, 18, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14939, 2274, 11596, 8554, 5407, 6004, 9073, 7689] avg=8192.0000 max=14939 max_rank=0 min=2274 min_rank=1 imbalance_ratio=1.823608 max_load_reduction=50597 max_load_reduction_pct=0.772049 moved_to_echo=28173 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3930, 3, 4048, 1763, 0, 108, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 1677, 0, 0, 0, 0, 0, 71, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 212, 0, 2331, 8, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 1928, 0, 395, 0, 2363, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4065, 0, 59, 4086, 1, 0, 2200, 1479, 1405, 275, 0, 0, 0, 0, 0, 1, 0, 90, 8, 192, 0, 375, 0, 2020, 0, 254, 0, 0, 0, 0, 0, 2578, 0, 8, 4093, 0, 242, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 3847, 13, 3476, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9903, 1750, 11008, 7999, 13578, 2940, 11015, 7343] avg=8192.0000 max=13578 max_rank=4 min=1750 min_rank=1 imbalance_ratio=1.657471 max_load_reduction=51958 max_load_reduction_pct=0.792816 moved_to_echo=34876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2644, 1, 0, 1602, 204, 4074, 0, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2135, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 77, 0, 24, 0, 36, 276, 0, 0, 0, 0, 0, 0, 0, 0, 6, 482, 4083, 307, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2876, 0, 6, 4070, 0, 0, 205, 0, 1445, 262, 0, 0, 0, 0, 0, 0, 3863, 0, 0, 0, 8, 0, 22, 1836, 1535, 0, 0, 0, 0, 0, 0, 2651, 0, 0, 12, 13, 3883, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1368, 0, 0, 3311, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10121, 10327, 8596, 8974, 8864, 7264, 6607, 4783] avg=8192.0000 max=10327 max_rank=1 min=4783 min_rank=7 imbalance_ratio=1.260620 max_load_reduction=55209 max_load_reduction_pct=0.842422 moved_to_echo=27518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 100, 1668, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3994, 104, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4028, 0, 0, 1189, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3509, 4095, 0, 0, 0, 2351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2033, 0, 467, 26, 4096, 0, 0, 1480, 1011, 281, 0, 0, 0, 0, 0, 879, 3, 1, 10, 0, 0, 0, 4093, 1919, 1718, 257, 0, 0, 0, 0, 0, 1851, 0, 14, 3893, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 24, 2, 4093, 0, 3492, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2278, 6478, 9606, 9967, 9394, 8880, 9855, 9078] avg=8192.0000 max=9967 max_rank=3 min=2278 min_rank=0 imbalance_ratio=1.216675 max_load_reduction=55569 max_load_reduction_pct=0.847916 moved_to_echo=37207 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[21376, 128, 512, 12160, 1408, 13824, 512, 16000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[14592, 128, 18816, 11520, 4480, 8576, 896, 6912, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 21632, 24576, 384, 4992, 128, 14208, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 11648, 0, 6400, 17920, 5888, 2816, 2560, 8960, 8192, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[29440, 16256, 256, 2048, 4992, 1536, 5376, 4480, 1792, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1664, 640, 18944, 21504, 128, 20480, 2432, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6912, 7936, 5632, 384, 2176, 10880, 1792, 4224, 13184, 11520, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[14976, 5120, 4224, 17792, 4352, 6784, 256, 12544, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4004, 0, 1748, 85, 62, 0, 0, 46, 3958, 464, 10, 0, 0, 2, 0, 3933, 0, 6, 0, 0, 0, 0, 712, 4088, 3629, 0, 398, 3826, 2191, 189, 0, 32, 0, 4070, 0, 9, 3481, 1666, 146, 2, 1843, 0, 11, 23, 474, 4073, 0, 3137, 84, 3403, 4073, 5, 4094, 0, 46, 1, 0, 0, 0, 3412, 1938, 0, 5, 157] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[3697, 0, 0, 2050, 84, 2003, 0, 0, 0, 53, 1, 3985, 151, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 24, 234, 4088, 0, 0, 3753, 0, 0, 0, 147, 3366, 1, 0, 28, 10, 1953, 2070, 3301, 4038, 349, 371, 0, 4035, 4, 122, 0, 3763, 0, 0, 758, 0, 0, 4096, 0, 0, 712, 0, 4066, 0, 4038, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[3559, 0, 3091, 848, 53, 0, 0, 7, 0, 0, 0, 2697, 0, 1, 27, 0, 6, 0, 0, 0, 4094, 4043, 3, 4051, 557, 249, 0, 0, 1818, 0, 17, 175, 1, 4096, 0, 0, 1313, 3799, 4012, 62, 4043, 8, 0, 0, 0, 14, 0, 0, 3507, 243, 0, 1760, 4075, 0, 0, 4082, 1, 590, 4096, 0, 142, 368, 4028, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 8, 4005, 17, 0, 0, 0, 2844, 36, 0, 0, 4096, 1962, 1, 893, 2004, 4067, 0, 1, 2258, 89, 0, 3, 4064, 1598, 41, 0, 0, 4094, 1181, 13, 34, 4050, 116, 430, 10, 0, 4032, 0, 4096, 0, 0, 0, 0, 3610, 3382, 0, 0, 3, 4096, 4076, 13, 0, 6, 1496, 11, 0, 0, 0, 98, 85, 2613, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4026, 23, 0, 1, 0, 3554, 0, 0, 70, 3887, 9, 3, 1568, 0, 4071, 1, 3521, 2909, 2960, 68, 351, 607, 1368, 0, 0, 4066, 371, 2956, 849, 3932, 0, 0, 3844, 0, 0, 428, 50, 0, 0, 0, 0, 0, 0, 4096, 4036, 1, 2588, 0, 98, 220, 0, 0, 0, 22, 201, 4083, 52, 550] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4025, 0, 0, 1977, 186, 938, 305, 0, 105, 0, 4096, 4087, 0, 0, 0, 3717, 0, 64, 0, 120, 0, 0, 2383, 3769, 0, 0, 3, 1367, 0, 0, 0, 110, 1, 2125, 48, 4025, 0, 157, 3665, 2837, 14, 4096, 0, 1, 4096, 0, 0, 0, 457, 4093, 0, 0, 527, 4085, 10, 0, 0, 0, 6, 0, 574, 4082, 132, 3253] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 329, 3951, 0, 0, 0, 0, 2, 0, 0, 3728, 163, 0, 0, 1, 0, 223, 4, 0, 4019, 4093, 1614, 3906, 18, 0, 1857, 825, 4086, 164, 19, 4067, 0, 0, 30, 3, 2, 38, 748, 0, 0, 0, 37, 0, 3884, 0, 0, 2460, 0, 4096, 209, 0, 0, 0, 3734, 0, 0, 0, 3512, 0, 3612, 3902, 2104, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 2, 273, 0, 0, 4089, 3912, 0, 0, 84, 0, 2669, 194, 75, 0, 8, 264, 12, 0, 0, 0, 268, 0, 4007, 4096, 22, 0, 122, 3995, 686, 0, 3822, 0, 3662, 6, 28, 0, 1709, 140, 0, 66, 0, 1, 0, 3863, 4073, 4086, 0, 1238, 387, 21, 0, 4096, 0, 0, 4, 0, 0, 3741, 0, 3373, 2350, 4092, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4004, 0, 1748, 85, 62, 0, 0, 46, 894, 93, 0, 0, 0, 0, 0, 0, 3958, 464, 10, 0, 0, 2, 0, 3933, 1510, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 712, 3648, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 0, 398, 3826, 2191, 189, 0, 32, 347, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 0, 9, 3481, 1666, 146, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1843, 0, 11, 23, 474, 4073, 0, 3137, 823, 199, 0, 0, 0, 0, 0, 0, 84, 1694, 4073, 5, 4094, 0, 46, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 1938, 0, 5, 157, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6932, 9877, 4366, 10612, 7657, 10583, 9997, 5512] avg=8192.0000 max=10612 max_rank=3 min=4366 min_rank=2 imbalance_ratio=1.295410 max_load_reduction=54924 max_load_reduction_pct=0.838074 moved_to_echo=33749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3559, 0, 3091, 848, 53, 0, 0, 7, 900, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2697, 0, 1, 27, 0, 107, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4094, 4043, 3, 3615, 0, 0, 0, 0, 0, 0, 0, 0, 557, 249, 0, 0, 1818, 0, 17, 175, 344, 9, 0, 0, 0, 0, 0, 0, 1, 2368, 0, 0, 1313, 3799, 4012, 62, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 8, 0, 0, 0, 14, 0, 0, 828, 14, 0, 0, 0, 0, 0, 0, 3507, 122, 0, 1760, 4075, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 1, 590, 4096, 0, 142, 359, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8550, 2832, 11761, 3169, 11555, 4907, 13546, 9216] avg=8192.0000 max=13546 max_rank=6 min=2832 min_rank=1 imbalance_ratio=1.653564 max_load_reduction=51990 max_load_reduction_pct=0.793304 moved_to_echo=39224 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3697, 0, 0, 2050, 84, 2003, 0, 0, 739, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1, 3985, 151, 0, 0, 4089, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 234, 4088, 0, 0, 3753, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 147, 1947, 1, 0, 28, 10, 1953, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 3301, 4038, 349, 371, 0, 4035, 4, 122, 680, 220, 0, 0, 0, 0, 0, 0, 0, 1873, 0, 0, 758, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 0, 4066, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8573, 9949, 4118, 8077, 6156, 13120, 6727, 8816] avg=8192.0000 max=13120 max_rank=5 min=4118 min_rank=2 imbalance_ratio=1.601562 max_load_reduction=52416 max_load_reduction_pct=0.799805 moved_to_echo=34819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4005, 17, 0, 0, 0, 2844, 25, 93, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4096, 1962, 1, 893, 2004, 1818, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 1, 2258, 89, 0, 3, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 1598, 41, 0, 0, 4094, 1181, 13, 34, 345, 68, 0, 0, 0, 0, 0, 0, 4050, 68, 430, 10, 0, 4032, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 3382, 0, 0, 23, 240, 0, 0, 0, 0, 0, 0, 3, 2038, 4076, 13, 0, 6, 1496, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 85, 2545, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6992, 10810, 10044, 7374, 12686, 7255, 7643, 2732] avg=8192.0000 max=12686 max_rank=4 min=2732 min_rank=7 imbalance_ratio=1.548584 max_load_reduction=52850 max_load_reduction_pct=0.806427 moved_to_echo=30316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4026, 23, 649, 66, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3554, 0, 0, 70, 3887, 1818, 0, 0, 0, 0, 0, 0, 0, 9, 3, 1568, 0, 4071, 1, 3521, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 2960, 68, 351, 607, 1368, 0, 0, 4066, 247, 107, 0, 0, 0, 0, 0, 0, 371, 1710, 849, 3932, 0, 0, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 50, 0, 0, 0, 0, 0, 597, 240, 0, 0, 0, 0, 0, 0, 0, 2038, 4036, 1, 2588, 0, 98, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 201, 3976, 52, 550, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8860, 9330, 11769, 9774, 10706, 1315, 8981, 4801] avg=8192.0000 max=11769 max_rank=2 min=1315 min_rank=5 imbalance_ratio=1.436646 max_load_reduction=53767 max_load_reduction_pct=0.820419 moved_to_echo=25803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 273, 0, 0, 4089, 3912, 0, 804, 91, 0, 0, 0, 0, 0, 0, 0, 84, 0, 2669, 194, 75, 0, 8, 171, 0, 0, 0, 0, 0, 0, 0, 264, 12, 0, 0, 0, 268, 0, 3576, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 0, 122, 3995, 686, 0, 3822, 340, 61, 0, 0, 0, 0, 0, 0, 0, 2118, 6, 28, 0, 1709, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 1, 0, 3863, 4073, 4086, 0, 740, 22, 0, 0, 0, 0, 0, 0, 1238, 194, 21, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3741, 0, 3373, 2289, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9171, 3201, 4120, 13144, 4001, 12851, 5553, 13495] avg=8192.0000 max=13495 max_rank=7 min=3201 min_rank=1 imbalance_ratio=1.647339 max_load_reduction=52041 max_load_reduction_pct=0.794083 moved_to_echo=35900 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 329, 3951, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3728, 163, 0, 0, 1, 1818, 0, 0, 0, 0, 0, 0, 0, 0, 223, 4, 0, 4019, 4093, 1614, 3485, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1857, 825, 4086, 164, 19, 4067, 332, 102, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3, 2, 38, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 3884, 0, 0, 2460, 0, 240, 0, 0, 0, 0, 0, 0, 0, 2038, 209, 0, 0, 0, 3734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3512, 0, 3612, 3800, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8465, 5712, 13438, 11470, 821, 6621, 5981, 13028] avg=8192.0000 max=13438 max_rank=2 min=821 min_rank=4 imbalance_ratio=1.640381 max_load_reduction=52098 max_load_reduction_pct=0.794952 moved_to_echo=26451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4025, 0, 0, 1977, 186, 938, 305, 0, 471, 91, 0, 0, 0, 0, 0, 0, 105, 0, 4096, 4087, 0, 0, 0, 3717, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 120, 0, 0, 2383, 3353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1367, 0, 0, 0, 110, 325, 111, 0, 0, 0, 0, 0, 0, 1, 1221, 48, 4025, 0, 157, 3665, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 1, 4096, 0, 0, 0, 433, 244, 0, 0, 0, 0, 0, 0, 457, 2029, 0, 0, 527, 4085, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 574, 3971, 132, 3253, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7993, 13825, 5920, 1916, 11954, 8884, 7108, 7936] avg=8192.0000 max=13825 max_rank=1 min=1916 min_rank=3 imbalance_ratio=1.687622 max_load_reduction=51711 max_load_reduction_pct=0.789047 moved_to_echo=35882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9344, 8576, 512, 512, 16000, 15616, 4096, 5760, 4224, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[13184, 4480, 2688, 6784, 21376, 2304, 128, 12416, 2304, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4224, 640, 4224, 24832, 2560, 128, 1024, 17664, 10752, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4608, 11904, 1408, 8064, 4864, 11520, 14592, 9088, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[5376, 12032, 12416, 1792, 16256, 4096, 5504, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 640, 12160, 3584, 14080, 17024, 14464, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[23552, 128, 9472, 8960, 512, 7040, 8320, 2944, 4608, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4352, 384, 5760, 2432, 12288, 8448, 8320, 23936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 224, 160, 3929, 1002, 0, 190, 0, 0, 0, 1, 10, 3804, 0, 0, 0, 1, 0, 4096, 0, 8, 4084, 870, 3109, 0, 4096, 0, 0, 2, 1, 4084, 0, 4045, 105, 4083, 10, 4074, 3953, 2790, 175, 3687, 20, 4096, 0, 14, 0, 0, 8, 0, 3, 4095, 3, 0, 4090, 2, 0, 0, 0, 612, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4, 4096, 818, 4003, 30, 0, 26, 4024, 641, 288, 4, 0, 0, 0, 271, 68, 0, 38, 0, 1944, 3281, 0, 3737, 0, 3261, 0, 0, 1, 172, 0, 14, 0, 0, 18, 0, 4056, 892, 0, 3736, 2546, 4096, 4057, 0, 4071, 4045, 3429, 3894, 0, 382, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 382, 0, 0, 3092, 31] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 289, 9, 3855, 0, 3, 4096, 372, 32, 0, 0, 0, 3917, 7, 4006, 2, 301, 128, 4080, 901, 0, 3, 0, 0, 4091, 0, 298, 0, 1058, 4083, 9, 0, 0, 3875, 0, 4096, 4073, 0, 0, 0, 2575, 2, 4096, 4095, 4058, 3395, 3698, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 53, 0, 0, 0, 0, 0, 2125, 4080, 14, 3817, 0, 1, 0, 2280, 0, 31, 4089, 279, 0, 0, 3, 0, 4096, 3559, 11, 1659, 0, 4056, 511, 0, 0, 0, 4084, 4044, 3470, 3821, 653, 1, 266, 4061, 0, 1497, 0, 4094, 3981, 0, 0, 0, 12, 10, 17, 3, 0, 0, 4, 0, 4093, 0, 16, 0, 638, 107] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 41, 0, 2706, 37, 2488, 0, 4016, 4, 0, 92, 18, 0, 0, 3704, 0, 0, 4093, 482, 2331, 0, 203, 3938, 0, 0, 0, 4096, 0, 0, 608, 429, 3732, 189, 3814, 2933, 0, 0, 2432, 2731, 3892, 0, 15, 4068, 3913, 4060, 0, 0, 0, 0, 33, 0, 0, 0, 0, 4059, 14, 0, 0, 119, 66, 172] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[3650, 0, 1, 0, 0, 1, 2621, 4096, 1219, 3933, 290, 0, 187, 0, 870, 426, 15, 0, 1848, 0, 0, 3777, 0, 0, 0, 3881, 0, 442, 2410, 4045, 5, 0, 3559, 161, 1772, 1224, 0, 0, 0, 3792, 4096, 2753, 687, 0, 2040, 425, 3104, 4071, 0, 0, 0, 0, 0, 0, 4046, 0, 35, 0, 0, 0, 0, 53, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 101, 0, 11, 0, 4094, 1, 3841, 0, 0, 0, 4096, 2091, 0, 1036, 30, 15, 67, 0, 0, 4096, 2737, 0, 0, 2043, 1962, 0, 243, 133, 4096, 0, 0, 20, 3396, 0, 4041, 0, 4027, 268, 0, 4054, 0, 1600, 3581, 4095, 3985, 30, 0, 709, 0, 0, 6, 1, 4045, 0, 25, 0, 0, 7, 1, 949, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[3296, 0, 0, 0, 0, 0, 0, 4096, 673, 410, 4096, 6, 0, 0, 0, 280, 0, 0, 898, 4096, 0, 4094, 3459, 242, 0, 4085, 0, 147, 0, 3435, 0, 0, 23, 0, 2, 0, 3918, 0, 3055, 313, 0, 3975, 0, 0, 443, 4030, 3930, 614, 0, 9, 0, 3597, 0, 0, 0, 83, 0, 4096, 4096, 4, 22, 0, 13, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4, 4096, 818, 4003, 30, 0, 26, 4024, 2152, 0, 0, 0, 0, 0, 0, 0, 641, 288, 4, 0, 0, 0, 271, 68, 0, 312, 0, 0, 0, 0, 0, 0, 0, 38, 0, 1944, 3281, 0, 3737, 0, 350, 0, 0, 0, 0, 0, 0, 0, 3261, 0, 0, 1, 172, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4056, 0, 0, 3736, 2546, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4071, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 0, 0, 0, 0, 88, 3894, 2818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 3092, 31, 892, 1239, 965, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15153, 1584, 9350, 3448, 10356, 11862, 7182, 6601] avg=8192.0000 max=15153 max_rank=0 min=1584 min_rank=1 imbalance_ratio=1.849731 max_load_reduction=50383 max_load_reduction_pct=0.768784 moved_to_echo=36001 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 224, 160, 3929, 12, 0, 0, 0, 0, 0, 0, 0, 1002, 0, 190, 0, 0, 0, 1, 10, 1483, 1, 453, 0, 0, 0, 0, 0, 3804, 0, 0, 0, 1, 0, 4096, 0, 319, 307, 0, 0, 0, 0, 0, 0, 8, 4084, 870, 3109, 0, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3324, 0, 0, 105, 4083, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 2790, 175, 3368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8, 0, 3, 4095, 3, 4096, 2744, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 612, 0, 4045, 1209, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4325, 3140, 8527, 10684, 7525, 10407, 10963, 9965] avg=8192.0000 max=10963 max_rank=6 min=3140 min_rank=1 imbalance_ratio=1.338257 max_load_reduction=54573 max_load_reduction_pct=0.832718 moved_to_echo=38860 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 289, 9, 3855, 0, 3, 4096, 372, 2130, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 3917, 7, 4006, 2, 1479, 308, 0, 0, 0, 0, 0, 0, 301, 128, 4080, 901, 0, 3, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 298, 0, 1058, 2604, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2575, 0, 4096, 4095, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 3698, 1, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 1, 0, 4073, 1, 957, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10754, 9751, 5765, 8060, 7971, 14472, 3703, 5060] avg=8192.0000 max=14472 max_rank=5 min=3703 min_rank=6 imbalance_ratio=1.766602 max_load_reduction=51064 max_load_reduction_pct=0.779175 moved_to_echo=31206 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 41, 0, 2706, 37, 2553, 0, 0, 0, 0, 0, 0, 0, 2488, 0, 4016, 4, 0, 92, 18, 0, 1483, 370, 414, 0, 0, 0, 0, 0, 0, 3704, 0, 0, 4093, 482, 2331, 0, 1, 281, 0, 0, 0, 0, 0, 0, 203, 3938, 0, 0, 0, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 608, 429, 3037, 189, 0, 2933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2432, 0, 3892, 0, 14, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 3913, 1896, 0, 0, 0, 0, 0, 0, 0, 4059, 14, 0, 0, 119, 66, 172, 3814, 835, 1145, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5345, 8885, 10892, 6754, 7196, 10398, 5842, 10224] avg=8192.0000 max=10892 max_rank=2 min=5345 min_rank=0 imbalance_ratio=1.329590 max_load_reduction=54644 max_load_reduction_pct=0.833801 moved_to_echo=33660 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3296, 0, 0, 0, 0, 0, 0, 4096, 2529, 0, 0, 0, 0, 0, 0, 0, 673, 410, 4096, 6, 0, 0, 0, 280, 1244, 366, 0, 0, 0, 0, 0, 0, 0, 0, 898, 4096, 0, 4094, 3459, 242, 38, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 147, 0, 2191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 2, 0, 0, 0, 3055, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 0, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3597, 0, 0, 0, 83, 3930, 2760, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 22, 0, 13, 0, 3918, 1215, 1135, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9921, 7075, 12827, 6423, 3393, 1019, 10379, 14499] avg=8192.0000 max=14499 max_rank=7 min=1019 min_rank=5 imbalance_ratio=1.769897 max_load_reduction=51037 max_load_reduction_pct=0.778763 moved_to_echo=29290 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 53, 0, 0, 0, 0, 0, 2571, 0, 0, 0, 0, 0, 0, 0, 2125, 4080, 14, 3817, 0, 1, 0, 2280, 1469, 371, 453, 0, 0, 0, 0, 0, 0, 31, 4089, 279, 0, 0, 3, 0, 0, 307, 0, 0, 0, 0, 0, 0, 4096, 3559, 11, 1659, 0, 2587, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3324, 4044, 0, 3821, 653, 1, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 10, 17, 3, 0, 0, 3981, 2820, 0, 0, 0, 0, 0, 0, 4, 0, 4093, 0, 16, 0, 638, 107, 3470, 1241, 1152, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2624, 14610, 4709, 12423, 11843, 1763, 6843, 10721] avg=8192.0000 max=14610 max_rank=1 min=1763 min_rank=5 imbalance_ratio=1.783447 max_load_reduction=50926 max_load_reduction_pct=0.777069 moved_to_echo=31170 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 101, 0, 11, 0, 4094, 1, 2570, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 0, 0, 4096, 2091, 0, 1036, 48, 372, 377, 0, 0, 0, 0, 0, 30, 15, 67, 0, 0, 4096, 2737, 0, 310, 255, 0, 0, 0, 0, 0, 0, 0, 2043, 1962, 0, 243, 85, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2764, 0, 0, 0, 4027, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1600, 3271, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 0, 0, 6, 1, 4045, 0, 3985, 2816, 0, 0, 0, 0, 0, 0, 25, 0, 0, 7, 1, 949, 3, 0, 4041, 1238, 1153, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6777, 11861, 7510, 8429, 7079, 4901, 11562, 7417] avg=8192.0000 max=11861 max_rank=1 min=4901 min_rank=5 imbalance_ratio=1.447876 max_load_reduction=53675 max_load_reduction_pct=0.819016 moved_to_echo=30959 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3650, 0, 1, 0, 0, 1, 2621, 4096, 268, 0, 0, 0, 0, 0, 0, 0, 1219, 3933, 290, 0, 187, 0, 870, 426, 1468, 38, 199, 0, 0, 0, 0, 0, 15, 0, 1848, 0, 0, 3777, 0, 0, 180, 136, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 442, 2410, 2577, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3559, 161, 1437, 1224, 0, 0, 0, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 687, 0, 1860, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 3104, 1912, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 53, 1, 0, 0, 841, 119, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10637, 8630, 5956, 9315, 10173, 10714, 9062, 1049] avg=8192.0000 max=10714 max_rank=5 min=1049 min_rank=7 imbalance_ratio=1.307861 max_load_reduction=54822 max_load_reduction_pct=0.836517 moved_to_echo=30998 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[17664, 0, 11520, 11520, 16384, 0, 0, 8832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[12032, 8832, 8704, 3840, 8320, 2304, 5248, 4224, 8704, 2176, 1920, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 12288, 8320, 512, 128, 1152, 4480, 384, 24320, 7936, 6656, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7040, 4480, 1024, 7936, 128, 256, 13824, 16640, 14848, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4224, 4608, 13952, 13696, 0, 6912, 15616, 7040, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 768, 128, 3712, 128, 128, 12288, 256, 30720, 17792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11776, 21632, 3200, 5376, 3968, 15360, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4224, 3968, 11008, 7296, 7424, 12544, 16384, 256, 1664, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 128, 4081, 0, 4084, 712, 2, 0, 0, 627, 84, 1401, 3778, 0, 0, 2069, 2323, 0, 4074, 2973, 0, 4, 1, 0, 0, 2, 0, 5, 1687, 0, 0, 0, 3886, 0, 0, 0, 4095, 3930, 0, 3576, 1, 0, 4012, 1023, 1, 2746, 0, 1768, 0, 4068, 0, 30, 0, 0, 4076, 0, 865, 3405, 19, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1894, 2328, 0, 246, 0, 0, 49, 0, 0, 4096, 99, 17, 0, 175, 0, 0, 4061, 0, 4085, 9, 3845, 695, 0, 0, 4078, 3845, 1799, 0, 0, 0, 0, 0, 36, 192, 0, 0, 0, 1, 1598, 4079, 17, 0, 4090, 35, 0, 0, 2, 0, 4023, 3169, 0, 0, 4093, 15, 6, 0, 167, 4066, 3931, 0, 2587, 0, 2108, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4075, 0, 0, 135, 81, 0, 3, 0, 1934, 0, 0, 1879, 1, 1, 3, 0, 208, 27, 4095, 0, 3980, 3945, 3810, 0, 0, 0, 3810, 0, 0, 0, 284, 13, 9, 0, 0, 2386, 3134, 0, 0, 4030, 396, 0, 4045, 1015, 0, 3745, 119, 1920, 3979, 0, 16, 0, 3194, 6, 1, 0, 0, 4096, 0, 799, 4096, 0, 266, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 3019, 4096, 0, 0, 0, 986, 4, 0, 0, 4020, 0, 2930, 0, 53, 0, 409, 0, 4096, 1923, 51, 4080, 2717, 1043, 0, 0, 88, 0, 0, 129, 1, 0, 0, 0, 0, 4017, 0, 0, 3, 4038, 34, 0, 3123, 113, 0, 3681, 1204, 3938, 99, 0, 3078, 111, 1342, 21, 70, 0, 0, 2923, 4037, 0, 4006, 0, 14, 39] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[3470, 0, 0, 0, 2709, 4063, 0, 0, 28, 1195, 3939, 0, 3334, 0, 4043, 4018, 0, 0, 0, 4096, 4050, 2, 4096, 0, 64, 317, 14, 0, 0, 0, 0, 130, 35, 29, 0, 2091, 0, 1, 33, 4096, 1116, 0, 4031, 15, 0, 1250, 126, 24, 0, 4095, 0, 0, 3179, 1690, 0, 0, 0, 4094, 0, 46, 13, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1, 46, 1057, 0, 77, 4095, 0, 4, 0, 3950, 4096, 0, 248, 4096, 0, 0, 0, 0, 0, 2834, 3937, 0, 3764, 0, 0, 0, 84, 0, 0, 79, 0, 1864, 0, 0, 0, 4063, 0, 0, 22, 2295, 3916, 0, 4092, 1, 2889, 35, 0, 0, 3863, 0, 171, 440, 4095, 42, 4041, 0, 0, 3195, 0, 0, 2144, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1421, 0, 3, 0, 0, 4073, 335, 0, 10, 3963, 3966, 68, 0, 0, 0, 3949, 0, 83, 436, 3340, 4096, 1903, 0, 0, 0, 4093, 4003, 0, 0, 0, 6, 0, 128, 1, 0, 4084, 0, 0, 0, 4078, 4096, 530, 4039, 0, 0, 57, 4045, 39, 0, 0, 19, 0, 4094, 160, 0, 0, 2, 0, 28, 3672, 716, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[322, 1869, 3777, 819, 993, 4096, 0, 0, 0, 3853, 4095, 0, 1, 718, 0, 0, 72, 0, 0, 0, 0, 0, 4096, 1402, 4058, 255, 33, 0, 0, 0, 0, 1465, 3009, 0, 0, 3840, 0, 1, 0, 3953, 1, 0, 3991, 0, 3738, 0, 1899, 4091, 11, 366, 313, 0, 3313, 5, 68, 0, 0, 0, 0, 0, 4096, 917, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 128, 4081, 0, 4084, 765, 0, 0, 0, 0, 0, 0, 0, 712, 2, 0, 0, 627, 84, 1401, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2069, 2323, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 2, 0, 5, 1687, 2973, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 1442, 1, 0, 4012, 1023, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2746, 0, 1768, 0, 4068, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 865, 3405, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9058, 6604, 8466, 6041, 7981, 10409, 8612, 8365] avg=8192.0000 max=10409 max_rank=5 min=6041 min_rank=3 imbalance_ratio=1.270630 max_load_reduction=55127 max_load_reduction_pct=0.841171 moved_to_echo=35367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 46, 1057, 0, 77, 4095, 0, 4, 876, 93, 0, 0, 0, 0, 0, 0, 0, 3950, 2913, 0, 248, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2834, 2807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 79, 0, 1864, 3764, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 22, 2295, 448, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 1649, 1, 2889, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 0, 171, 440, 4095, 42, 4041, 0, 682, 55, 0, 0, 0, 0, 0, 0, 0, 3195, 0, 0, 2144, 0, 0, 0, 1035, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6249, 11207, 5641, 7358, 6828, 8490, 13389, 6374] avg=8192.0000 max=13389 max_rank=6 min=5641 min_rank=2 imbalance_ratio=1.634399 max_load_reduction=52147 max_load_reduction_pct=0.795700 moved_to_echo=35081 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4075, 0, 0, 135, 81, 0, 3, 0, 866, 0, 0, 0, 0, 0, 0, 0, 1934, 0, 0, 1879, 1, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 27, 4095, 0, 2838, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 0, 0, 0, 284, 13, 3810, 1549, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2386, 3134, 0, 0, 4030, 453, 0, 0, 0, 0, 0, 0, 0, 396, 0, 1630, 1015, 0, 3745, 119, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 16, 0, 3194, 6, 1, 0, 689, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 799, 4096, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5160, 3818, 11113, 9466, 10012, 8825, 7885, 9257] avg=8192.0000 max=11113 max_rank=2 min=3818 min_rank=1 imbalance_ratio=1.356567 max_load_reduction=54423 max_load_reduction_pct=0.830429 moved_to_echo=35979 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3019, 4096, 0, 0, 0, 986, 4, 668, 91, 0, 0, 0, 0, 0, 0, 0, 0, 2860, 0, 2930, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 4096, 1923, 38, 4080, 0, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 129, 1, 0, 2717, 1196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 3, 4038, 5, 0, 0, 0, 0, 0, 0, 0, 34, 0, 1259, 113, 0, 3681, 1204, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 3078, 111, 1342, 21, 70, 0, 8, 54, 0, 0, 0, 0, 0, 0, 0, 2923, 4037, 0, 4006, 0, 14, 39, 1015, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8864, 5843, 11589, 4131, 8063, 10229, 4783, 12034] avg=8192.0000 max=12034 max_rank=7 min=4131 min_rank=3 imbalance_ratio=1.468994 max_load_reduction=53502 max_load_reduction_pct=0.816376 moved_to_echo=35109 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1894, 2328, 0, 246, 0, 0, 49, 0, 875, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 71, 17, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4085, 9, 2742, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3845, 1799, 0, 0, 0, 0, 0, 0, 1566, 0, 0, 0, 0, 0, 0, 36, 192, 0, 0, 0, 1, 1598, 4079, 437, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1649, 35, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 3169, 0, 0, 4093, 15, 6, 0, 666, 1, 0, 0, 0, 0, 0, 0, 167, 4066, 3931, 0, 2587, 0, 2108, 0, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5394, 4359, 11592, 11288, 6343, 1703, 11973, 12884] avg=8192.0000 max=12884 max_rank=7 min=1703 min_rank=5 imbalance_ratio=1.572754 max_load_reduction=52652 max_load_reduction_pct=0.803406 moved_to_echo=32903 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3470, 0, 0, 0, 2709, 4063, 0, 0, 863, 89, 0, 0, 0, 0, 0, 0, 28, 1195, 2802, 0, 3334, 0, 4043, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2887, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 317, 14, 0, 0, 0, 0, 130, 4096, 1544, 0, 0, 0, 0, 0, 0, 35, 29, 0, 2091, 0, 1, 33, 4096, 461, 0, 0, 0, 0, 0, 0, 0, 1116, 0, 1624, 15, 0, 1250, 126, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3179, 1690, 0, 0, 702, 53, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 46, 13, 0, 0, 4, 995, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11194, 15420, 6985, 6165, 6746, 4155, 9719, 5152] avg=8192.0000 max=15420 max_rank=1 min=4155 min_rank=5 imbalance_ratio=1.882324 max_load_reduction=50116 max_load_reduction_pct=0.764709 moved_to_echo=25772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1421, 0, 3, 0, 0, 4073, 335, 0, 864, 90, 0, 0, 0, 0, 0, 0, 10, 3963, 2820, 68, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 436, 3340, 2914, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4003, 0, 0, 0, 6, 0, 0, 1547, 0, 0, 0, 0, 0, 0, 128, 1, 0, 4084, 0, 0, 0, 4078, 469, 0, 0, 0, 0, 0, 0, 0, 4096, 530, 1628, 0, 0, 57, 4045, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4094, 160, 0, 0, 713, 54, 0, 0, 0, 0, 0, 0, 2, 0, 28, 3672, 716, 0, 0, 0, 1002, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6786, 10810, 8676, 9649, 8760, 10395, 5040, 5420] avg=8192.0000 max=10810 max_rank=1 min=5040 min_rank=6 imbalance_ratio=1.319580 max_load_reduction=54726 max_load_reduction_pct=0.835052 moved_to_echo=29615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[322, 1869, 3777, 819, 993, 4096, 0, 0, 859, 96, 0, 0, 0, 0, 0, 0, 0, 3853, 2903, 0, 1, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 255, 33, 0, 0, 0, 0, 1465, 4096, 1531, 0, 0, 0, 0, 0, 0, 3009, 0, 0, 3840, 0, 1, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1601, 0, 3738, 0, 1899, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 11, 366, 313, 0, 3313, 5, 68, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 917, 0, 0, 1037, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12831, 7475, 1474, 11438, 10803, 11330, 4135, 6050] avg=8192.0000 max=12831 max_rank=0 min=1474 min_rank=2 imbalance_ratio=1.566284 max_load_reduction=52705 max_load_reduction_pct=0.804214 moved_to_echo=32318 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[13568, 640, 12544, 1280, 6656, 12800, 8448, 10112, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4864, 128, 14848, 14592, 14336, 14720, 0, 2560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2688, 17152, 14464, 2048, 7168, 5120, 5504, 11776, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3328, 256, 0, 24448, 3200, 128, 1664, 30720, 2304, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[14848, 7680, 5376, 640, 27392, 2048, 4224, 0, 3584, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8576, 9856, 0, 128, 256, 384, 5248, 21504, 11904, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 22528, 8064, 5504, 21120, 1024, 2432, 128, 5120, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11264, 7296, 8960, 1280, 4096, 20480, 1408, 4096, 6656, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True + [2026-08-30 05:00:49] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 893.1 | throughput per GPU (TFLOP/s/GPU): 373.3 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101572E+01 | loss scale: 1.0 | grad norm: 45.912 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[65, 2208, 1404, 194, 1716, 57, 809, 493, 1399, 111, 1549, 662, 569, 774, 97, 685, 187, 1201, 1349, 1183, 1828, 516, 2497, 1586, 54, 1565, 293, 1301, 952, 163, 1800, 1956, 3505, 418, 607, 545, 440, 1282, 1754, 711, 2520, 1487, 1700, 824, 719, 2168, 993, 754, 246, 566, 366, 1256, 432, 319, 2085, 156, 292, 790, 2119, 2333, 805, 459, 1079, 583] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[54, 2704, 1470, 183, 1753, 82, 788, 539, 1109, 121, 640, 367, 1307, 557, 49, 514, 223, 795, 1603, 1047, 2218, 364, 2724, 1566, 85, 1532, 904, 1308, 1280, 416, 1326, 1916, 3498, 408, 204, 408, 132, 635, 2182, 1148, 3348, 1701, 1222, 395, 780, 2139, 1064, 728, 270, 610, 279, 1306, 553, 597, 2164, 133, 126, 860, 2105, 1782, 1000, 376, 1106, 733] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[82, 2502, 1495, 248, 2052, 33, 823, 402, 1628, 129, 1075, 432, 708, 607, 76, 469, 157, 777, 1070, 1481, 1715, 700, 2477, 1881, 132, 1790, 600, 1028, 1256, 265, 1979, 1929, 3247, 257, 768, 509, 197, 1207, 1958, 704, 2364, 1963, 1962, 668, 432, 1952, 960, 753, 220, 529, 485, 1311, 266, 537, 1283, 93, 84, 1122, 2468, 1935, 955, 353, 1348, 648] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[42, 2251, 1258, 138, 1933, 39, 1022, 294, 926, 80, 867, 605, 1004, 832, 64, 592, 132, 1289, 978, 625, 1743, 658, 2740, 1730, 95, 2093, 934, 1137, 777, 338, 1020, 2017, 3528, 504, 335, 707, 245, 1373, 1947, 1247, 2816, 1162, 1233, 359, 823, 2310, 1628, 1063, 305, 340, 307, 1097, 351, 380, 1915, 96, 436, 737, 2059, 2554, 1031, 634, 984, 777] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[200, 2230, 1557, 188, 1655, 30, 874, 504, 1397, 81, 1305, 821, 1069, 632, 88, 504, 199, 898, 1701, 972, 2272, 505, 2583, 1620, 113, 1748, 645, 1445, 994, 423, 1758, 1823, 3246, 426, 538, 445, 194, 1046, 2041, 656, 2642, 1173, 1423, 855, 579, 2208, 1019, 598, 377, 523, 369, 1186, 285, 273, 1427, 99, 95, 1330, 2043, 2108, 1224, 685, 1010, 579] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[63, 2361, 858, 200, 2236, 29, 608, 320, 1442, 83, 864, 831, 791, 723, 152, 549, 364, 992, 2060, 1393, 1732, 368, 3073, 1651, 95, 1575, 345, 1056, 869, 370, 1546, 1837, 3175, 455, 555, 204, 473, 1423, 1706, 1176, 2005, 1388, 2095, 729, 950, 1632, 858, 1395, 206, 532, 334, 999, 206, 306, 2014, 225, 217, 1398, 1663, 2571, 1006, 631, 887, 686] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[115, 2666, 1108, 166, 1477, 19, 897, 390, 1417, 212, 1111, 536, 1276, 776, 89, 558, 188, 727, 1697, 637, 2278, 567, 2069, 1828, 115, 1599, 631, 1332, 1116, 413, 1769, 1766, 3277, 495, 588, 673, 294, 1486, 1739, 618, 2421, 1393, 1471, 371, 528, 2014, 1107, 666, 205, 351, 359, 1244, 645, 444, 1965, 115, 72, 1545, 2165, 1956, 1333, 423, 1171, 857] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[164, 2355, 686, 110, 2202, 91, 870, 313, 1140, 62, 1178, 482, 854, 646, 84, 525, 152, 985, 1133, 1154, 1877, 559, 2813, 1785, 47, 1726, 466, 839, 646, 232, 1594, 1844, 3680, 525, 174, 605, 208, 1310, 2257, 940, 2396, 1533, 1381, 390, 591, 2385, 1376, 1092, 250, 360, 1018, 1059, 457, 497, 1774, 286, 427, 1117, 2648, 2589, 1097, 490, 461, 549] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[54, 2704, 1470, 183, 1753, 82, 788, 539, 577, 204, 188, 0, 0, 0, 0, 0, 1109, 121, 640, 367, 1307, 557, 49, 514, 1890, 992, 39, 0, 0, 0, 0, 0, 223, 795, 1603, 1047, 2218, 364, 343, 1566, 0, 0, 0, 0, 0, 0, 0, 0, 85, 1532, 904, 1308, 1280, 416, 1326, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 2506, 408, 204, 408, 132, 635, 2182, 1148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1701, 1222, 395, 780, 1935, 1064, 728, 0, 0, 0, 0, 0, 0, 0, 0, 270, 610, 279, 1306, 553, 597, 2164, 133, 3348, 491, 0, 0, 0, 0, 0, 0, 126, 860, 2105, 1166, 1000, 376, 1106, 733, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8542, 7585, 8159, 8579, 7623, 7825, 9751, 7472] avg=8192.0000 max=9751 max_rank=6 min=7472 min_rank=7 imbalance_ratio=1.190308 max_load_reduction=55785 max_load_reduction_pct=0.851212 moved_to_echo=32671 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[65, 2208, 1404, 194, 1716, 57, 809, 493, 755, 207, 192, 0, 0, 0, 0, 0, 1399, 111, 1549, 662, 569, 774, 97, 685, 1732, 994, 51, 0, 0, 0, 0, 0, 187, 1201, 1349, 1183, 1828, 516, 315, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 54, 1565, 293, 1301, 952, 163, 1800, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 2511, 418, 607, 545, 440, 1282, 1754, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 1700, 824, 719, 1961, 993, 754, 0, 0, 0, 0, 0, 0, 0, 0, 246, 566, 366, 1256, 432, 319, 2085, 156, 2520, 450, 0, 0, 0, 0, 0, 0, 292, 790, 2119, 1527, 805, 459, 1079, 583, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8623, 8165, 7892, 8268, 8438, 8396, 7654] avg=8192.0000 max=8623 max_rank=1 min=7654 min_rank=7 imbalance_ratio=1.052612 max_load_reduction=56913 max_load_reduction_pct=0.868423 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[42, 2251, 1258, 138, 1933, 39, 1022, 294, 827, 220, 198, 0, 0, 0, 0, 0, 926, 80, 867, 605, 1004, 832, 64, 592, 1901, 1000, 56, 0, 0, 0, 0, 0, 132, 1289, 978, 625, 1743, 658, 346, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 95, 2093, 934, 1137, 777, 338, 1020, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 2528, 504, 335, 707, 245, 1373, 1947, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1162, 1233, 359, 823, 2090, 1628, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 305, 340, 307, 1097, 351, 380, 1915, 96, 2816, 493, 0, 0, 0, 0, 0, 0, 436, 737, 2059, 1671, 1031, 634, 984, 777, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 7927, 7501, 8213, 8886, 8358, 8100, 8329] avg=8192.0000 max=8886 max_rank=4 min=7501 min_rank=2 imbalance_ratio=1.084717 max_load_reduction=56650 max_load_reduction_pct=0.864410 moved_to_echo=33673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[82, 2502, 1495, 248, 2052, 33, 823, 402, 626, 186, 189, 0, 0, 0, 0, 0, 1628, 129, 1075, 432, 708, 607, 76, 469, 1718, 920, 43, 0, 0, 0, 0, 0, 157, 777, 1070, 1481, 1715, 700, 313, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 132, 1790, 600, 1028, 1256, 265, 1979, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 2327, 257, 768, 509, 197, 1207, 1958, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1963, 1962, 668, 432, 1766, 960, 753, 0, 0, 0, 0, 0, 0, 0, 0, 220, 529, 485, 1311, 266, 537, 1283, 93, 2364, 446, 0, 0, 0, 0, 0, 0, 84, 1122, 2468, 1266, 955, 353, 1348, 648, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8638, 7805, 8094, 8790, 7927, 8504, 7534, 8244] avg=8192.0000 max=8790 max_rank=3 min=7534 min_rank=6 imbalance_ratio=1.072998 max_load_reduction=56746 max_load_reduction_pct=0.865875 moved_to_echo=32209 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[63, 2361, 858, 200, 2236, 29, 608, 320, 832, 155, 180, 0, 0, 0, 0, 0, 1442, 83, 864, 831, 791, 723, 152, 549, 2132, 900, 57, 0, 0, 0, 0, 0, 364, 992, 2060, 1393, 1732, 368, 388, 1651, 0, 0, 0, 0, 0, 0, 0, 0, 95, 1575, 345, 1056, 869, 370, 1546, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 2275, 455, 555, 204, 473, 1423, 1706, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1388, 2095, 729, 950, 1477, 858, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 206, 532, 334, 999, 206, 306, 2014, 225, 2005, 553, 0, 0, 0, 0, 0, 0, 217, 1398, 1663, 1682, 1006, 631, 887, 686, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7842, 8524, 8948, 7513, 8267, 8892, 7380, 8170] avg=8192.0000 max=8948 max_rank=2 min=7380 min_rank=6 imbalance_ratio=1.092285 max_load_reduction=56588 max_load_reduction_pct=0.863464 moved_to_echo=32709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[115, 2666, 1108, 166, 1477, 19, 897, 390, 633, 192, 173, 0, 0, 0, 0, 0, 1417, 212, 1111, 536, 1276, 776, 89, 558, 1435, 929, 43, 0, 0, 0, 0, 0, 188, 727, 1697, 637, 2278, 567, 262, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 115, 1599, 631, 1332, 1116, 413, 1769, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 2348, 495, 588, 673, 294, 1486, 1739, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1393, 1471, 371, 528, 1822, 1107, 666, 0, 0, 0, 0, 0, 0, 0, 0, 205, 351, 359, 1244, 645, 444, 1965, 115, 2421, 372, 0, 0, 0, 0, 0, 0, 72, 1545, 2165, 1280, 1333, 423, 1171, 857, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7836, 8382, 8184, 8568, 8241, 7358, 8121, 8846] avg=8192.0000 max=8846 max_rank=7 min=7358 min_rank=5 imbalance_ratio=1.079834 max_load_reduction=56690 max_load_reduction_pct=0.865021 moved_to_echo=32566 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[200, 2230, 1557, 188, 1655, 30, 874, 504, 685, 215, 181, 0, 0, 0, 0, 0, 1397, 81, 1305, 821, 1069, 632, 88, 504, 1796, 924, 51, 0, 0, 0, 0, 0, 199, 898, 1701, 972, 2272, 505, 318, 1620, 0, 0, 0, 0, 0, 0, 0, 0, 113, 1748, 645, 1445, 994, 423, 1758, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 2322, 426, 538, 445, 194, 1046, 2041, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1173, 1423, 855, 579, 1993, 1019, 598, 0, 0, 0, 0, 0, 0, 0, 0, 377, 523, 369, 1186, 285, 273, 1427, 99, 2642, 469, 0, 0, 0, 0, 0, 0, 95, 1330, 2043, 1372, 1224, 685, 1010, 579, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8319, 8668, 8485, 8768, 7668, 7640, 7650, 8338] avg=8192.0000 max=8768 max_rank=3 min=7640 min_rank=5 imbalance_ratio=1.070312 max_load_reduction=56768 max_load_reduction_pct=0.866211 moved_to_echo=31296 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[164, 2355, 686, 110, 2202, 91, 870, 313, 838, 227, 181, 0, 0, 0, 0, 0, 1140, 62, 1178, 482, 854, 646, 84, 525, 1951, 1043, 57, 0, 0, 0, 0, 0, 152, 985, 1133, 1154, 1877, 559, 355, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 47, 1726, 466, 839, 646, 232, 1594, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 525, 174, 605, 208, 1310, 2257, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1533, 1381, 390, 591, 2158, 1376, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 250, 360, 1018, 1059, 457, 497, 1774, 286, 2396, 507, 0, 0, 0, 0, 0, 0, 427, 1117, 2648, 1694, 1097, 490, 461, 549, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8037, 8022, 8000, 7213, 8656, 8521, 8604, 8483] avg=8192.0000 max=8656 max_rank=4 min=7213 min_rank=3 imbalance_ratio=1.056641 max_load_reduction=56880 max_load_reduction_pct=0.867920 moved_to_echo=34264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[10496, 896, 8704, 4736, 7680, 5632, 768, 4480, 14592, 7808, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[896, 19328, 9856, 1536, 15104, 384, 6784, 3328, 5888, 1664, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2176, 3840, 3584, 9472, 3200, 3456, 14720, 1280, 20608, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 11904, 12544, 4608, 5504, 15232, 9088, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1792, 8960, 17280, 11776, 8576, 4096, 8064, 5504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[19456, 3584, 3840, 4096, 2304, 9856, 15616, 7296, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1664, 7680, 11648, 8576, 15744, 4352, 2688, 13696, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[768, 13696, 4864, 9472, 7936, 2688, 12800, 13696, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4088, 2, 64, 19, 17, 182, 76, 0, 0, 4078, 3, 0, 0, 1, 25, 10, 0, 0, 3217, 3982, 3929, 4094, 0, 4024, 3909, 0, 0, 1205, 211, 1, 10, 11, 1524, 3665, 0, 4083, 0, 14, 1, 0, 1, 41, 3064, 4094, 271, 4, 144, 2, 0, 755, 2, 3177, 1, 3638, 0, 0, 24, 0, 36, 0, 4089, 3691, 0, 57] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4084, 0, 1, 1974, 0, 269, 34, 18, 0, 4062, 632, 6, 40, 6, 1052, 41, 0, 0, 2839, 4043, 4075, 4069, 4, 4088, 3887, 0, 0, 30, 1384, 0, 3, 64, 3011, 3787, 0, 4094, 0, 55, 34, 1, 370, 0, 1230, 3792, 0, 0, 596, 5, 14, 1880, 29, 1333, 50, 1070, 11, 2, 0, 0, 0, 122, 4086, 3114, 9, 136] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4088, 0, 65, 338, 0, 9, 0, 3, 31, 4059, 85, 5, 10, 1, 12, 8, 0, 0, 3843, 4071, 4048, 3443, 0, 4081, 3941, 1, 0, 1, 582, 0, 79, 53, 4071, 3865, 0, 4096, 0, 0, 0, 0, 56, 42, 3559, 3441, 81, 0, 65, 0, 0, 1038, 9, 2111, 0, 2471, 0, 0, 0, 28, 0, 0, 3815, 3910, 0, 21] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4069, 6, 0, 285, 1, 487, 0, 0, 15, 4080, 369, 0, 0, 0, 110, 164, 0, 0, 1387, 4056, 3948, 4037, 0, 4096, 4076, 1, 1, 6, 141, 0, 224, 2, 3970, 4074, 0, 4095, 0, 18, 0, 0, 978, 0, 295, 4053, 0, 0, 199, 1, 0, 2546, 31, 3113, 0, 2066, 0, 8, 12, 0, 1, 3, 3694, 3781, 3, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3785, 0, 0, 1041, 32, 0, 5, 16, 0, 4091, 9, 74, 0, 122, 7, 0, 15, 52, 2737, 4081, 3898, 4089, 0, 3979, 4089, 9, 2, 106, 133, 3, 5, 0, 3527, 3996, 20, 4056, 17, 6, 0, 0, 0, 551, 1286, 2026, 1, 2, 1120, 58, 0, 3942, 134, 2929, 3, 1011, 0, 0, 51, 0, 8, 176, 4080, 4067, 0, 89] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4093, 92, 3, 68, 37, 2, 1, 0, 0, 3899, 1, 1, 82, 0, 1, 50, 0, 2, 1731, 4038, 4095, 3944, 0, 4070, 4088, 0, 4, 3, 2663, 24, 0, 0, 2462, 4079, 4, 4095, 16, 12, 1, 10, 256, 1, 3936, 3354, 18, 0, 743, 4, 6, 2469, 0, 321, 0, 2246, 43, 3, 79, 0, 1, 107, 3772, 3819, 1, 686] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4096, 3, 0, 469, 1, 2, 3, 15, 5, 4084, 786, 0, 2, 368, 154, 225, 0, 0, 3645, 4095, 4077, 3718, 0, 4068, 3956, 0, 0, 58, 2360, 0, 1, 0, 1963, 4096, 0, 4096, 0, 308, 11, 0, 15, 2, 1283, 3484, 6, 30, 56, 0, 0, 1772, 0, 1542, 1, 2452, 0, 0, 25, 1, 7, 14, 4093, 4028, 13, 47] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4094, 0, 0, 44, 0, 9, 95, 0, 0, 3977, 123, 0, 1, 1, 237, 20, 0, 0, 3215, 4068, 4021, 3337, 0, 4080, 4051, 18, 0, 112, 1736, 0, 158, 0, 2235, 3878, 3, 4095, 0, 1069, 0, 5, 23, 3, 1251, 3913, 5, 3, 233, 1, 4, 2340, 2, 2363, 0, 2548, 0, 0, 0, 0, 0, 87, 4012, 4023, 3, 40] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 65, 338, 0, 9, 0, 3, 2909, 525, 0, 0, 0, 0, 0, 0, 31, 4059, 85, 5, 10, 1, 12, 8, 3556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 1530, 3443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 1, 0, 1, 582, 0, 79, 53, 1612, 1162, 0, 0, 0, 0, 0, 0, 4071, 3865, 0, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 42, 3559, 3441, 81, 0, 65, 0, 1265, 678, 0, 0, 0, 0, 0, 0, 0, 1038, 9, 2111, 0, 2471, 0, 0, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 3815, 3910, 0, 21, 68, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7937, 7767, 8816, 7431, 9155, 9187, 7401, 7842] avg=8192.0000 max=9187 max_rank=5 min=7401 min_rank=6 imbalance_ratio=1.121460 max_load_reduction=56349 max_load_reduction_pct=0.859818 moved_to_echo=33585 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4088, 2, 64, 19, 17, 182, 76, 0, 2845, 517, 0, 0, 0, 0, 0, 0, 0, 4078, 3, 0, 0, 1, 25, 10, 3507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 1485, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 0, 0, 1205, 211, 1, 10, 11, 1607, 1137, 0, 0, 0, 0, 0, 0, 1524, 3665, 0, 1215, 0, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 41, 3064, 4094, 271, 4, 144, 2, 1261, 658, 0, 0, 0, 0, 0, 0, 0, 755, 2, 3177, 1, 3638, 0, 0, 1720, 0, 0, 0, 0, 0, 0, 0, 24, 0, 36, 0, 4089, 3691, 0, 57, 66, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7810, 7624, 8796, 8091, 6419, 9540, 9293, 7963] avg=8192.0000 max=9540 max_rank=5 min=6419 min_rank=4 imbalance_ratio=1.164551 max_load_reduction=55996 max_load_reduction_pct=0.854431 moved_to_echo=33215 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 3, 0, 469, 1, 2, 3, 15, 2926, 522, 0, 0, 0, 0, 0, 0, 5, 4084, 786, 0, 2, 368, 154, 225, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3645, 0, 1542, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 58, 2360, 0, 1, 0, 1612, 1169, 0, 0, 0, 0, 0, 0, 1963, 4096, 0, 1219, 0, 308, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2, 1283, 3484, 6, 30, 56, 0, 1265, 683, 0, 0, 0, 0, 0, 0, 0, 1772, 0, 1542, 1, 2452, 0, 0, 1784, 0, 0, 0, 0, 0, 0, 0, 25, 1, 7, 14, 4093, 4028, 13, 47, 68, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8037, 9170, 8905, 9156, 7597, 6824, 7551, 8296] avg=8192.0000 max=9170 max_rank=1 min=6824 min_rank=5 imbalance_ratio=1.119385 max_load_reduction=56366 max_load_reduction_pct=0.860077 moved_to_echo=30268 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4069, 6, 0, 285, 1, 487, 0, 0, 2898, 526, 0, 0, 0, 0, 0, 0, 15, 4080, 369, 0, 0, 0, 110, 164, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 1492, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 1, 1, 6, 141, 0, 224, 2, 1611, 1158, 0, 0, 0, 0, 0, 0, 3970, 4074, 0, 1219, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 0, 295, 4053, 0, 0, 199, 1, 1265, 662, 0, 0, 0, 0, 0, 0, 0, 2546, 31, 3113, 0, 2066, 0, 8, 1728, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 3, 3694, 3781, 3, 1034, 66, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8308, 6916, 7220, 9281, 7453, 9492, 8594] avg=8192.0000 max=9492 max_rank=6 min=6916 min_rank=2 imbalance_ratio=1.158691 max_load_reduction=56044 max_load_reduction_pct=0.855164 moved_to_echo=34820 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 1, 1974, 0, 269, 34, 18, 2888, 525, 0, 0, 0, 0, 0, 0, 0, 4062, 632, 6, 40, 6, 1052, 41, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2839, 0, 1540, 4069, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 0, 30, 1384, 0, 3, 64, 1611, 1155, 0, 0, 0, 0, 0, 0, 3011, 3787, 0, 1219, 0, 55, 34, 1, 0, 0, 0, 0, 0, 0, 0, 0, 370, 0, 1230, 3792, 0, 0, 596, 5, 1264, 683, 0, 0, 0, 0, 0, 0, 14, 1880, 29, 1333, 50, 1070, 11, 2, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 4086, 3114, 9, 136, 68, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9793, 9402, 8452, 8134, 8107, 7940, 6173, 7535] avg=8192.0000 max=9793 max_rank=0 min=6173 min_rank=6 imbalance_ratio=1.195435 max_load_reduction=55743 max_load_reduction_pct=0.850571 moved_to_echo=29755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4093, 92, 3, 68, 37, 2, 1, 0, 2886, 524, 0, 0, 0, 0, 0, 0, 0, 3899, 1, 1, 82, 0, 1, 50, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1731, 0, 1548, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4, 3, 2663, 24, 0, 0, 1611, 1152, 0, 0, 0, 0, 0, 0, 2462, 4079, 4, 1219, 16, 12, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 256, 1, 3936, 3354, 18, 0, 743, 4, 1265, 686, 0, 0, 0, 0, 0, 0, 6, 2469, 0, 321, 0, 2246, 43, 3, 1792, 0, 0, 0, 0, 0, 0, 0, 79, 0, 1, 107, 3772, 3819, 1, 686, 69, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7706, 7580, 7225, 9545, 7803, 10263, 6880, 8534] avg=8192.0000 max=10263 max_rank=5 min=6880 min_rank=6 imbalance_ratio=1.252808 max_load_reduction=55273 max_load_reduction_pct=0.843399 moved_to_echo=33480 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3785, 0, 0, 1041, 32, 0, 5, 16, 2917, 511, 0, 0, 0, 0, 0, 0, 0, 4091, 9, 74, 0, 122, 7, 0, 3468, 0, 0, 0, 0, 0, 0, 0, 15, 52, 2737, 0, 1474, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 9, 2, 106, 133, 3, 5, 0, 1596, 1164, 0, 0, 0, 0, 0, 0, 3527, 3996, 20, 1207, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 1286, 2026, 1, 2, 1120, 58, 1253, 653, 0, 0, 0, 0, 0, 0, 0, 3942, 134, 2929, 3, 1011, 0, 0, 1706, 0, 0, 0, 0, 0, 0, 0, 51, 0, 8, 176, 4080, 4067, 0, 89, 65, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 7771, 8367, 7107, 8773, 6950, 9725, 8536] avg=8192.0000 max=9725 max_rank=6 min=6950 min_rank=5 imbalance_ratio=1.187134 max_load_reduction=55811 max_load_reduction_pct=0.851608 moved_to_echo=33984 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 44, 0, 9, 95, 0, 2907, 525, 0, 0, 0, 0, 0, 0, 0, 3977, 123, 0, 1, 1, 237, 20, 3555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3215, 0, 1507, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 18, 0, 112, 1736, 0, 158, 0, 1616, 1161, 0, 0, 0, 0, 0, 0, 2235, 3878, 3, 1211, 0, 1069, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3, 1251, 3913, 5, 3, 233, 1, 1268, 679, 0, 0, 0, 0, 0, 0, 4, 2340, 2, 2363, 0, 2548, 0, 0, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 4012, 4023, 3, 40, 71, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7674, 7914, 8059, 8852, 8401, 7379, 9021, 8236] avg=8192.0000 max=9021 max_rank=6 min=7379 min_rank=5 imbalance_ratio=1.101196 max_load_reduction=56515 max_load_reduction_pct=0.862350 moved_to_echo=33037 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 16768, 256, 16896, 128, 17536, 128, 128, 14080, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[22784, 31488, 128, 9728, 128, 1536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32384, 2048, 128, 256, 512, 1664, 640, 28416, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 128, 128, 512, 31744, 30464, 128, 2176, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32000, 128, 128, 1536, 9216, 128, 512, 256, 12928, 9344, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1792, 640, 16000, 28160, 384, 128, 3200, 128, 10112, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 22656, 0, 12160, 30848, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32512, 128, 256, 4352, 128, 1024, 256, 128, 23296, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 3286, 2108, 0, 3884, 0, 0, 138, 472, 0, 0, 2301, 4095, 3859, 4071, 6, 0, 25, 0, 0, 4064, 344, 0, 0, 9, 0, 3427, 0, 0, 4085, 0, 11, 4013, 2440, 651, 7, 0, 4096, 0, 0, 4096, 2, 9, 0, 0, 0, 46, 86, 65, 1, 4069, 17, 0, 0, 0, 0, 3993, 641, 0, 5, 4065, 0, 0, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 77, 953, 0, 3582, 134, 0, 0, 140, 2, 1, 3993, 4089, 3127, 3859, 0, 0, 6, 0, 0, 4078, 3022, 17, 0, 0, 0, 480, 0, 0, 4033, 4065, 0, 4094, 827, 50, 0, 65, 3508, 0, 77, 4034, 22, 0, 0, 17, 6, 1564, 439, 0, 0, 4011, 10, 0, 105, 13, 1, 3955, 2910, 0, 0, 4040, 0, 96, 33] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[20, 4085, 375, 0, 4016, 0, 6, 2, 0, 0, 1, 3125, 4021, 3782, 3603, 31, 227, 3675, 0, 25, 3888, 4, 10, 288, 0, 191, 3440, 2, 1, 4095, 31, 0, 4087, 744, 2, 0, 11, 3758, 0, 8, 4036, 0, 98, 0, 7, 0, 0, 7, 0, 991, 4088, 0, 2, 90, 0, 19, 4076, 39, 0, 40, 4093, 9, 1, 386] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1, 178, 3028, 0, 3329, 6, 0, 60, 0, 0, 0, 3327, 4094, 4096, 3826, 0, 34, 41, 0, 2, 4015, 165, 0, 0, 205, 31, 905, 5, 0, 4095, 4033, 0, 4088, 245, 6, 2520, 0, 4048, 0, 0, 4092, 27, 0, 0, 0, 290, 690, 1523, 0, 5, 4035, 0, 3, 0, 3, 3, 4010, 303, 4, 0, 4096, 0, 0, 69] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 2996, 459, 39, 3942, 0, 0, 0, 7, 0, 3, 3592, 4067, 4071, 3915, 82, 38, 783, 2, 0, 4072, 8, 0, 0, 297, 0, 3225, 0, 1, 4006, 3631, 0, 4039, 22, 102, 0, 0, 4011, 1, 0, 4066, 74, 37, 252, 5, 0, 501, 47, 109, 187, 4093, 33, 0, 5, 0, 27, 4096, 327, 0, 190, 4062, 0, 1, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 61, 3810, 4, 2447, 7, 0, 26, 46, 6, 29, 3791, 4091, 3142, 3988, 8, 18, 3320, 0, 4, 4095, 694, 0, 3, 0, 80, 1504, 0, 10, 3981, 142, 0, 4067, 1332, 0, 7, 4, 4034, 98, 0, 2405, 2327, 0, 0, 0, 0, 894, 7, 1, 0, 3242, 0, 0, 9, 47, 0, 3905, 1724, 0, 0, 3885, 0, 339, 1902] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 4044, 4084, 1, 2641, 0, 0, 1, 52, 0, 0, 3062, 4093, 4031, 4060, 12, 0, 2530, 0, 0, 2532, 0, 0, 0, 50, 0, 2213, 0, 0, 4096, 967, 0, 4094, 2, 78, 531, 47, 4096, 2, 1, 4095, 0, 0, 0, 0, 0, 6, 200, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 108, 0, 1, 4096, 0, 0, 1516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 3043, 3758, 0, 4078, 0, 0, 109, 405, 0, 0, 483, 4095, 4095, 4093, 0, 19, 371, 0, 1, 4088, 1, 0, 0, 0, 0, 3449, 0, 9, 4090, 50, 0, 1627, 143, 0, 176, 0, 4096, 7, 0, 4096, 0, 0, 2, 0, 10, 0, 2822, 146, 9, 4059, 0, 0, 0, 6, 8, 4094, 9, 0, 0, 4090, 0, 0, 3899] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3286, 2108, 0, 3884, 0, 0, 138, 92, 0, 0, 0, 0, 0, 0, 0, 472, 0, 0, 2301, 0, 3859, 1343, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 4064, 344, 0, 0, 1743, 662, 0, 0, 0, 0, 0, 0, 9, 0, 3427, 0, 0, 4085, 0, 11, 76, 0, 0, 0, 0, 0, 0, 0, 4013, 2440, 651, 7, 0, 3266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 9, 0, 0, 0, 46, 86, 2728, 102, 90, 0, 0, 0, 0, 0, 65, 1, 4069, 17, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 3993, 641, 0, 5, 2220, 0, 0, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9508, 7981, 6838, 7608, 10377, 7159, 8157, 7908] avg=8192.0000 max=10377 max_rank=4 min=6838 min_rank=2 imbalance_ratio=1.266724 max_load_reduction=55159 max_load_reduction_pct=0.841660 moved_to_echo=33601 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 178, 3028, 0, 3329, 6, 0, 60, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3327, 0, 4096, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 41, 0, 2, 4015, 165, 0, 0, 1756, 654, 0, 0, 0, 0, 0, 0, 205, 31, 905, 5, 0, 4095, 4033, 0, 75, 0, 0, 0, 0, 0, 0, 0, 4088, 245, 6, 2520, 0, 3228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 27, 0, 0, 0, 290, 690, 1523, 2564, 103, 90, 0, 0, 0, 0, 0, 0, 5, 4035, 0, 3, 0, 3, 3, 4004, 0, 0, 0, 0, 0, 0, 0, 4010, 303, 4, 0, 2237, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6693, 8685, 6667, 9349, 10087, 9379, 8053, 6623] avg=8192.0000 max=10087 max_rank=4 min=6623 min_rank=7 imbalance_ratio=1.231323 max_load_reduction=55449 max_load_reduction_pct=0.846085 moved_to_echo=34142 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 77, 953, 0, 3582, 134, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 140, 2, 1, 3993, 0, 3127, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4078, 3022, 17, 0, 1732, 567, 0, 0, 0, 0, 0, 0, 0, 0, 480, 0, 0, 4033, 4065, 0, 65, 0, 0, 0, 0, 0, 0, 0, 4094, 827, 50, 0, 65, 2797, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 22, 0, 0, 17, 6, 1564, 439, 2586, 102, 90, 0, 0, 0, 0, 0, 0, 0, 4011, 10, 0, 105, 13, 1, 3999, 0, 0, 0, 0, 0, 0, 0, 3955, 2910, 0, 0, 2206, 0, 96, 33, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4826, 8536, 9422, 8643, 7910, 8860, 8139, 9200] avg=8192.0000 max=9422 max_rank=2 min=4826 min_rank=0 imbalance_ratio=1.150146 max_load_reduction=56114 max_load_reduction_pct=0.856232 moved_to_echo=34109 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2996, 459, 39, 3942, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 3592, 0, 4071, 1291, 82, 0, 0, 0, 0, 0, 0, 0, 0, 38, 783, 2, 0, 4072, 8, 0, 0, 1742, 648, 0, 0, 0, 0, 0, 0, 297, 0, 3225, 0, 1, 4006, 3631, 0, 74, 0, 0, 0, 0, 0, 0, 0, 4039, 22, 102, 0, 0, 3199, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 74, 37, 252, 5, 0, 501, 47, 2624, 102, 89, 0, 0, 0, 0, 0, 109, 187, 4093, 33, 0, 5, 0, 27, 3978, 0, 0, 0, 0, 0, 0, 0, 4096, 327, 0, 190, 2218, 0, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7526, 9046, 7293, 11234, 7363, 7797, 8432, 6845] avg=8192.0000 max=11234 max_rank=3 min=6845 min_rank=7 imbalance_ratio=1.371338 max_load_reduction=54302 max_load_reduction_pct=0.828583 moved_to_echo=30437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4044, 4084, 1, 2641, 0, 0, 1, 92, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 3062, 0, 4031, 1339, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 0, 0, 2532, 0, 0, 0, 1756, 662, 0, 0, 0, 0, 0, 0, 50, 0, 2213, 0, 0, 4096, 967, 0, 76, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 78, 531, 47, 3266, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 6, 200, 2721, 103, 90, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 4096, 108, 0, 1, 2237, 0, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10863, 8496, 7480, 7402, 8021, 7215, 8101, 7958] avg=8192.0000 max=10863 max_rank=0 min=7215 min_rank=5 imbalance_ratio=1.326050 max_load_reduction=54673 max_load_reduction_pct=0.834244 moved_to_echo=31295 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[20, 4085, 375, 0, 4016, 0, 6, 2, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3125, 0, 3782, 1188, 31, 0, 0, 0, 0, 0, 0, 0, 0, 227, 3675, 0, 25, 3888, 4, 10, 288, 1755, 607, 0, 0, 0, 0, 0, 0, 0, 191, 3440, 2, 1, 4095, 31, 0, 70, 0, 0, 0, 0, 0, 0, 0, 4087, 744, 2, 0, 11, 2996, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 98, 0, 7, 0, 0, 7, 2415, 103, 89, 0, 0, 0, 0, 0, 0, 991, 4088, 0, 2, 90, 0, 19, 3932, 0, 0, 0, 0, 0, 0, 0, 4076, 39, 0, 40, 2235, 9, 1, 386, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8589, 8127, 10479, 7830, 7848, 6755, 9122, 6786] avg=8192.0000 max=10479 max_rank=2 min=6755 min_rank=5 imbalance_ratio=1.279175 max_load_reduction=55057 max_load_reduction_pct=0.840103 moved_to_echo=30511 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 61, 3810, 4, 2447, 7, 0, 26, 91, 0, 0, 0, 0, 0, 0, 0, 46, 6, 29, 3791, 0, 3142, 1315, 8, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3320, 0, 4, 4095, 694, 0, 3, 1666, 652, 0, 0, 0, 0, 0, 0, 0, 80, 1504, 0, 10, 3981, 142, 0, 75, 0, 0, 0, 0, 0, 0, 0, 4067, 1332, 0, 7, 4, 3216, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2405, 2327, 0, 0, 0, 0, 894, 7, 2673, 98, 90, 0, 0, 0, 0, 0, 1, 0, 3242, 0, 0, 9, 47, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 3905, 1724, 0, 0, 2121, 0, 339, 1902, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6446, 8337, 10452, 5792, 8724, 8494, 7300, 9991] avg=8192.0000 max=10452 max_rank=2 min=5792 min_rank=3 imbalance_ratio=1.275879 max_load_reduction=55084 max_load_reduction_pct=0.840515 moved_to_echo=34509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3043, 3758, 0, 4078, 0, 0, 109, 97, 0, 0, 0, 0, 0, 0, 0, 405, 0, 0, 483, 0, 4095, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 371, 0, 1, 4088, 1, 0, 0, 1759, 666, 0, 0, 0, 0, 0, 0, 0, 0, 3449, 0, 9, 4090, 50, 0, 80, 0, 0, 0, 0, 0, 0, 0, 1627, 143, 0, 176, 0, 3253, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 10, 0, 2822, 2748, 108, 91, 0, 0, 0, 0, 0, 146, 9, 4059, 0, 0, 0, 6, 8, 4004, 0, 0, 0, 0, 0, 0, 0, 4094, 9, 0, 0, 2223, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11085, 6328, 6905, 7678, 5206, 9877, 8232, 10225] avg=8192.0000 max=11085 max_rank=0 min=5206 min_rank=4 imbalance_ratio=1.353149 max_load_reduction=54451 max_load_reduction_pct=0.830856 moved_to_echo=33540 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32256, 6144, 128, 256, 17792, 128, 512, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[30976, 2560, 256, 256, 128, 384, 3712, 5248, 21120, 896, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1152, 128, 128, 23680, 0, 30208, 10368, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[30208, 5760, 896, 3328, 128, 25344, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 17792, 18688, 128, 28032, 256, 128, 384, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 10752, 128, 128, 30848, 4352, 128, 384, 13952, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 1280, 31744, 128, 128, 256, 128, 128, 32000, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 384, 18688, 128, 128, 32512, 12928, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 2976, 4087, 0, 4022, 12, 0, 1141, 0, 0, 0, 0, 2274, 0, 0, 0, 3663, 0, 4096, 96, 3152, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 4096, 163, 0, 2658, 4096, 0, 0, 4096, 0, 0, 1023, 0, 667, 0, 583, 0, 4050, 0, 0, 0, 0, 1, 1491, 4096, 0, 22, 0, 0, 0, 692, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[802, 0, 1543, 3956, 4, 4008, 2171, 0, 53, 1, 26, 45, 47, 4095, 120, 0, 386, 2645, 0, 4043, 1, 177, 0, 175, 4060, 12, 0, 0, 21, 2, 4038, 50, 4082, 25, 0, 1991, 4074, 0, 0, 4000, 7, 0, 6, 1, 20, 0, 581, 0, 3820, 0, 2, 0, 0, 27, 427, 3991, 1782, 21, 0, 36, 0, 4068, 10, 4084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[11, 0, 4021, 4086, 0, 1453, 41, 0, 1731, 0, 0, 4, 0, 2268, 0, 0, 1774, 3982, 0, 4095, 22, 201, 0, 0, 4095, 0, 5, 11, 0, 0, 3880, 2, 3899, 1, 0, 3071, 4096, 20, 0, 4096, 0, 0, 2450, 2, 6, 0, 51, 0, 3124, 0, 0, 0, 2, 0, 529, 4096, 564, 0, 0, 7, 0, 3756, 0, 4084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 3729, 3966, 0, 3688, 0, 40, 2399, 18, 0, 0, 1, 0, 113, 0, 0, 4096, 4, 4078, 63, 4039, 6, 1, 4087, 0, 0, 0, 0, 0, 3993, 0, 1906, 2, 0, 3925, 4053, 0, 8, 4090, 0, 0, 37, 395, 0, 14, 205, 0, 4043, 26, 0, 4, 394, 35, 3777, 4053, 11, 0, 0, 57, 0, 13, 97, 4069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4021, 4096, 0, 3699, 0, 0, 0, 576, 0, 0, 9, 1147, 369, 0, 12, 3389, 0, 3932, 0, 3161, 0, 0, 4096, 0, 0, 0, 0, 0, 4064, 1, 4082, 0, 0, 3434, 3144, 0, 0, 4096, 0, 0, 263, 590, 63, 0, 1192, 0, 4096, 0, 0, 0, 132, 12, 1460, 4010, 1543, 0, 1, 0, 1, 430, 322, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[70, 141, 2549, 4033, 33, 3985, 1, 0, 845, 1, 0, 1, 6, 59, 0, 6, 0, 4091, 4, 4031, 0, 389, 11, 22, 4069, 4, 1, 89, 10, 0, 4079, 13, 3988, 24, 0, 1641, 4089, 0, 0, 4028, 11, 0, 1546, 3, 73, 0, 3198, 0, 3789, 1, 7, 12, 100, 31, 1529, 4091, 626, 0, 1, 0, 51, 4094, 0, 4060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 3539, 4096, 0, 2566, 2, 0, 2224, 5, 0, 0, 6, 4, 1, 0, 1148, 3722, 0, 4096, 0, 2247, 0, 11, 4096, 0, 0, 0, 0, 3, 4049, 0, 4089, 0, 0, 3432, 4094, 12, 0, 4093, 0, 0, 301, 1, 0, 0, 234, 0, 4022, 0, 0, 0, 0, 0, 834, 4090, 265, 0, 1, 0, 0, 3826, 330, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[9, 0, 3275, 4096, 0, 4025, 134, 0, 129, 0, 0, 0, 0, 23, 49, 1, 0, 3377, 32, 4096, 0, 4094, 0, 0, 4044, 0, 1, 0, 0, 1, 4090, 49, 569, 191, 2, 3762, 4095, 0, 41, 3907, 1, 3, 21, 0, 1, 0, 1159, 0, 4064, 14, 3, 0, 3, 0, 4025, 4074, 0, 0, 1, 1, 1, 3973, 5, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2976, 1131, 0, 4022, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1141, 0, 0, 0, 0, 2274, 0, 0, 2234, 2160, 789, 634, 0, 0, 0, 0, 0, 3663, 0, 1936, 96, 3152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 41, 0, 0, 0, 0, 0, 0, 0, 4096, 163, 0, 2658, 1862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 0, 667, 0, 583, 0, 4096, 2322, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 1, 1491, 2478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 692, 0, 4094, 788, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 9232, 8847, 8230, 8779, 8691, 8020, 5596] avg=8192.0000 max=9232 max_rank=1 min=5596 min_rank=7 imbalance_ratio=1.126953 max_load_reduction=56304 max_load_reduction_pct=0.859131 moved_to_echo=31086 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 4021, 1130, 0, 1453, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1731, 0, 0, 4, 0, 2268, 0, 0, 2234, 2159, 789, 634, 0, 0, 0, 0, 1774, 3982, 0, 1936, 22, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 5, 11, 0, 0, 3880, 2, 41, 0, 0, 0, 0, 0, 0, 0, 3899, 1, 0, 3071, 1862, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2450, 2, 6, 0, 51, 0, 4096, 2322, 0, 0, 0, 0, 0, 0, 3124, 0, 0, 0, 2, 0, 529, 2478, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, 7, 0, 3756, 0, 4084, 788, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6656, 9819, 7915, 8034, 8853, 8927, 6133, 9199] avg=8192.0000 max=9819 max_rank=1 min=6133 min_rank=6 imbalance_ratio=1.198608 max_load_reduction=55717 max_load_reduction_pct=0.850174 moved_to_echo=33112 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3729, 1098, 0, 3688, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 2399, 18, 0, 0, 1, 0, 113, 0, 2211, 2150, 781, 615, 0, 0, 0, 0, 0, 4096, 4, 1928, 63, 4039, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 3993, 0, 41, 0, 0, 0, 0, 0, 0, 0, 1906, 2, 0, 3925, 1842, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 395, 0, 14, 205, 0, 4090, 2253, 0, 0, 0, 0, 0, 0, 4043, 26, 0, 4, 394, 35, 3777, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 57, 0, 13, 97, 4069, 779, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8556, 8288, 10137, 8121, 7683, 6994, 10731, 5026] avg=8192.0000 max=10731 max_rank=6 min=5026 min_rank=7 imbalance_ratio=1.309937 max_load_reduction=54805 max_load_reduction_pct=0.836258 moved_to_echo=30434 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3539, 1134, 0, 2566, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2224, 5, 0, 0, 6, 4, 1, 0, 2233, 2160, 788, 635, 0, 0, 0, 0, 1148, 3722, 0, 1936, 0, 2247, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4049, 0, 41, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 3432, 1861, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 1, 0, 0, 234, 0, 4093, 2327, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 834, 2474, 0, 0, 0, 0, 0, 0, 0, 0, 265, 0, 1, 0, 0, 3826, 330, 4095, 787, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7243, 8056, 9064, 8189, 9394, 6956, 7330, 9304] avg=8192.0000 max=9394 max_rank=4 min=6956 min_rank=5 imbalance_ratio=1.146729 max_load_reduction=56142 max_load_reduction_pct=0.856659 moved_to_echo=32984 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[802, 0, 1543, 1095, 4, 4008, 2171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1, 26, 45, 47, 4095, 120, 0, 2222, 2132, 769, 613, 0, 0, 0, 0, 386, 2645, 0, 1911, 1, 177, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 12, 0, 0, 21, 2, 4038, 50, 40, 0, 0, 0, 0, 0, 0, 0, 4082, 25, 0, 1991, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 6, 1, 20, 0, 581, 0, 4000, 2248, 0, 0, 0, 0, 0, 0, 3820, 0, 2, 0, 0, 27, 427, 2415, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 21, 0, 36, 0, 4068, 10, 4084, 767, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9623, 10123, 5295, 8223, 7950, 6863, 6691, 10768] avg=8192.0000 max=10768 max_rank=7 min=5295 min_rank=2 imbalance_ratio=1.314453 max_load_reduction=54768 max_load_reduction_pct=0.835693 moved_to_echo=32272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[70, 141, 2549, 1117, 33, 3985, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 845, 1, 0, 1, 6, 59, 0, 6, 2231, 2126, 788, 625, 0, 0, 0, 0, 0, 4091, 4, 1905, 0, 389, 11, 22, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4, 1, 89, 10, 0, 4079, 13, 41, 0, 0, 0, 0, 0, 0, 0, 3988, 24, 0, 1641, 1858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1546, 3, 73, 0, 3198, 0, 4028, 2291, 0, 0, 0, 0, 0, 0, 3789, 1, 7, 12, 100, 31, 1529, 2475, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 1, 0, 51, 4094, 0, 4060, 787, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7896, 6688, 6422, 8306, 7511, 11150, 7944, 9619] avg=8192.0000 max=11150 max_rank=5 min=6422 min_rank=2 imbalance_ratio=1.361084 max_load_reduction=54386 max_load_reduction_pct=0.829865 moved_to_echo=36224 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4021, 1134, 0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 0, 9, 1147, 369, 0, 1715, 2073, 772, 635, 0, 0, 0, 0, 12, 3389, 0, 1859, 0, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4064, 1, 40, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 3434, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 590, 63, 0, 1192, 0, 4096, 2327, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 132, 12, 1460, 2427, 0, 0, 0, 0, 0, 0, 0, 0, 1543, 0, 1, 0, 1, 430, 322, 4093, 771, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8854, 7296, 8421, 8201, 8945, 8531, 8127, 7161] avg=8192.0000 max=8945 max_rank=4 min=7161 min_rank=7 imbalance_ratio=1.091919 max_load_reduction=56591 max_load_reduction_pct=0.863510 moved_to_echo=32764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 3275, 1124, 0, 4025, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 23, 49, 1, 2239, 2164, 789, 640, 0, 0, 0, 0, 0, 3377, 32, 1932, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 1, 0, 0, 1, 4090, 49, 47, 0, 0, 0, 0, 0, 0, 0, 569, 191, 2, 3762, 1856, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 21, 0, 1, 0, 1159, 0, 3907, 2332, 0, 0, 0, 0, 0, 0, 4064, 14, 3, 0, 3, 0, 4025, 2451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 3973, 5, 4095, 787, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8567, 6034, 9435, 8232, 6421, 7424, 10560, 8863] avg=8192.0000 max=10560 max_rank=6 min=6034 min_rank=1 imbalance_ratio=1.289062 max_load_reduction=54976 max_load_reduction_pct=0.838867 moved_to_echo=33268 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8576, 640, 128, 128, 128, 9984, 768, 128, 17408, 17152, 6272, 5120, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3328, 29056, 128, 15360, 256, 17536, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26752, 512, 128, 23936, 14464, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[896, 256, 25728, 9088, 128, 27520, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32384, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4864, 128, 128, 128, 128, 20864, 768, 32768, 6272, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31104, 128, 128, 128, 640, 128, 14080, 19712, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 5760, 1024, 896, 128, 7296, 0, 32512, 18432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 1, 0, 3161, 1, 737, 270, 4062, 2118, 0, 12, 4093, 0, 65, 0, 0, 54, 3917, 3206, 0, 18, 107, 0, 0, 0, 4096, 135, 14, 3982, 4063, 3062, 4, 0, 4, 4090, 4093, 0, 0, 0, 0, 4095, 2, 10, 0, 77, 3455, 0, 0, 4096, 3930, 0, 0, 110, 142, 0, 66, 4094, 0, 90, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[209, 0, 78, 0, 0, 130, 0, 220, 6, 0, 2, 4094, 1864, 0, 699, 4096, 0, 1319, 0, 276, 0, 3639, 2705, 0, 0, 333, 0, 0, 1, 3958, 1466, 304, 4067, 3216, 3681, 2292, 2, 0, 4039, 4040, 14, 1, 14, 5, 162, 0, 0, 11, 0, 3914, 0, 0, 3093, 2822, 1, 0, 177, 0, 102, 1010, 4063, 0, 3410, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3, 0, 0, 1557, 0, 0, 3, 22, 0, 0, 635, 4096, 3583, 0, 214, 4032, 832, 0, 0, 0, 4, 4095, 2611, 0, 0, 0, 0, 0, 0, 4095, 12, 600, 3575, 4069, 2725, 1137, 0, 6, 4096, 4096, 5, 0, 0, 0, 4093, 0, 0, 0, 86, 0, 0, 0, 2994, 4089, 0, 0, 7, 83, 3716, 277, 4085, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 50, 0, 121, 17, 17, 0, 2971, 122, 0, 0, 3861, 350, 41, 375, 4087, 13, 3164, 0, 1, 29, 3461, 4030, 1, 0, 90, 1, 34, 0, 4092, 1193, 3489, 3655, 3503, 127, 1, 0, 50, 4090, 4019, 0, 0, 3, 0, 1532, 0, 0, 2, 7, 3968, 24, 0, 4096, 3896, 0, 0, 277, 0, 4, 579, 4090, 1, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 448, 0, 0, 0, 246, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 2052, 0, 0, 2096, 4064, 2690, 0, 0, 0, 0, 0, 0, 4096, 1, 401, 4093, 4094, 254, 3003, 0, 0, 4095, 4096, 1, 1, 0, 0, 4068, 0, 0, 0, 0, 1035, 0, 0, 4095, 4096, 0, 0, 0, 1, 60, 18, 4096, 0, 48, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 42, 10, 0, 0, 9, 1, 1198, 0, 13, 1, 3976, 3604, 0, 0, 3979, 363, 3912, 4, 0, 206, 4022, 4040, 80, 199, 138, 0, 1, 2, 4091, 117, 0, 3709, 3222, 3912, 1, 0, 0, 4096, 4096, 3, 1, 48, 13, 4090, 0, 0, 0, 2, 32, 0, 0, 4089, 4096, 19, 0, 0, 3, 0, 26, 4070, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1058, 0, 0, 115, 4091, 3267, 0, 0, 4096, 0, 3129, 0, 0, 2382, 4096, 4093, 0, 0, 1, 0, 0, 0, 4096, 0, 296, 4096, 4096, 2098, 5, 0, 0, 4089, 4096, 8, 0, 2, 0, 3242, 0, 0, 0, 0, 1, 0, 0, 3896, 4096, 0, 0, 171, 116, 0, 683, 4096, 0, 25, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2, 0, 4, 1, 1, 151, 138, 4037, 3657, 0, 1, 4061, 23, 4093, 0, 0, 3885, 4092, 4092, 0, 0, 0, 1, 1, 0, 4096, 0, 6, 4036, 3876, 3992, 151, 1, 1, 4072, 4083, 406, 0, 0, 0, 131, 0, 0, 11, 0, 10, 0, 30, 4093, 4050, 39, 0, 60, 53, 1, 0, 4095, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 1, 0, 3161, 4090, 2670, 0, 0, 0, 0, 0, 0, 1, 737, 270, 4062, 2118, 0, 12, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 54, 1260, 3206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 107, 0, 0, 0, 4096, 135, 14, 1975, 1056, 0, 0, 0, 0, 0, 0, 1135, 4063, 3062, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 10, 0, 3193, 1423, 872, 0, 0, 0, 0, 0, 77, 3455, 0, 0, 4096, 2831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 142, 0, 66, 4094, 0, 90, 2, 1601, 1099, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9924, 8100, 4585, 7401, 8268, 9595, 10459, 7204] avg=8192.0000 max=10459 max_rank=6 min=4585 min_rank=2 imbalance_ratio=1.276733 max_load_reduction=55077 max_load_reduction_pct=0.840408 moved_to_echo=35526 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1058, 4089, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 115, 4091, 3267, 0, 0, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3129, 0, 0, 2382, 1317, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 296, 2032, 1105, 0, 0, 0, 0, 0, 0, 1167, 4096, 2098, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 3242, 0, 0, 0, 3195, 1423, 897, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3896, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 116, 0, 683, 4096, 0, 25, 0, 1674, 1145, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7820, 8374, 10921, 7530, 7366, 8767, 6848, 7910] avg=8192.0000 max=10921 max_rank=2 min=6848 min_rank=6 imbalance_ratio=1.333130 max_load_reduction=54615 max_load_reduction_pct=0.833359 moved_to_echo=30891 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 448, 0, 0, 0, 246, 4095, 2672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2052, 0, 0, 2096, 1307, 2690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 401, 2030, 1096, 0, 0, 0, 0, 0, 0, 1167, 4094, 254, 3003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4068, 0, 0, 0, 3195, 1424, 896, 0, 0, 0, 0, 0, 0, 1035, 0, 0, 4095, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 60, 18, 4096, 0, 48, 0, 1661, 1145, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7461, 9093, 8145, 7624, 8518, 9585, 8081, 7029] avg=8192.0000 max=9585 max_rank=5 min=7029 min_rank=7 imbalance_ratio=1.170044 max_load_reduction=55951 max_load_reduction_pct=0.853745 moved_to_echo=33213 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 42, 10, 0, 0, 9, 1, 1198, 4096, 2673, 0, 0, 0, 0, 0, 0, 0, 13, 1, 3976, 3604, 0, 0, 875, 0, 0, 0, 0, 0, 0, 0, 0, 363, 3912, 4, 0, 206, 1293, 4040, 80, 0, 0, 0, 0, 0, 0, 0, 0, 199, 138, 0, 1, 2, 4091, 117, 0, 1840, 1085, 0, 0, 0, 0, 0, 0, 1057, 3222, 3912, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 48, 13, 4090, 0, 0, 0, 3104, 1423, 812, 0, 0, 0, 0, 0, 2, 32, 0, 0, 4089, 2951, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 26, 4070, 0, 0, 0, 1644, 1145, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8029, 8469, 9898, 7473, 8192, 9494, 7093, 6888] avg=8192.0000 max=9898 max_rank=2 min=6888 min_rank=7 imbalance_ratio=1.208252 max_load_reduction=55638 max_load_reduction_pct=0.848969 moved_to_echo=31667 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[209, 0, 78, 0, 0, 130, 0, 220, 4039, 2637, 0, 0, 0, 0, 0, 0, 6, 0, 2, 4094, 1864, 0, 699, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1319, 0, 276, 0, 1171, 2705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 1, 3958, 1466, 304, 2017, 981, 0, 0, 0, 0, 0, 0, 1159, 3216, 3681, 2292, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 14, 5, 162, 0, 0, 11, 3195, 1403, 891, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 3093, 2033, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 102, 1010, 4063, 0, 3410, 1, 1487, 789, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7313, 7566, 5471, 9060, 10350, 5696, 9041, 11039] avg=8192.0000 max=11039 max_rank=7 min=5471 min_rank=2 imbalance_ratio=1.347534 max_load_reduction=54497 max_load_reduction_pct=0.831558 moved_to_echo=36126 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 50, 0, 121, 17, 17, 0, 2971, 4090, 2623, 0, 0, 0, 0, 0, 0, 122, 0, 0, 3861, 350, 41, 375, 899, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3164, 0, 1, 29, 1114, 4030, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 1, 34, 0, 4092, 1193, 3489, 1813, 933, 0, 0, 0, 0, 0, 0, 1042, 3503, 127, 1, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1532, 0, 0, 2, 3188, 1396, 800, 0, 0, 0, 0, 0, 7, 3968, 24, 0, 4096, 2807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 4, 579, 4090, 1, 2, 0, 1414, 1089, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9889, 5648, 8352, 11645, 4723, 6921, 10902, 7456] avg=8192.0000 max=11645 max_rank=3 min=4723 min_rank=4 imbalance_ratio=1.421509 max_load_reduction=53891 max_load_reduction_pct=0.822311 moved_to_echo=30002 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 1557, 0, 0, 3, 22, 4096, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 635, 4096, 3583, 0, 214, 886, 0, 0, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 4, 1317, 2611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 12, 600, 1773, 1104, 0, 0, 0, 0, 0, 0, 1019, 4069, 2725, 1137, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4093, 0, 0, 0, 3146, 1423, 783, 0, 0, 0, 0, 0, 86, 0, 0, 0, 2994, 2946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 83, 3716, 277, 4085, 0, 3, 0, 1674, 1143, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8354, 9414, 4764, 7584, 8956, 9450, 6026, 10988] avg=8192.0000 max=10988 max_rank=7 min=4764 min_rank=2 imbalance_ratio=1.341309 max_load_reduction=54548 max_load_reduction_pct=0.832336 moved_to_echo=35420 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2, 0, 4, 1, 4072, 2665, 0, 0, 0, 0, 0, 0, 1, 151, 138, 4037, 3657, 0, 1, 887, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4093, 0, 0, 3885, 1307, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4096, 0, 6, 2006, 1109, 0, 0, 0, 0, 0, 0, 1142, 3876, 3992, 151, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 131, 0, 0, 11, 3174, 1418, 888, 0, 0, 0, 0, 0, 0, 10, 0, 30, 4093, 2914, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 53, 1, 0, 4095, 0, 1, 0, 1676, 1136, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6746, 8872, 13400, 7219, 9163, 6028, 7086, 7022] avg=8192.0000 max=13400 max_rank=2 min=6028 min_rank=5 imbalance_ratio=1.635742 max_load_reduction=52136 max_load_reduction_pct=0.795532 moved_to_echo=29299 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1280, 17792, 128, 384, 8704, 10112, 27520, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 12416, 128, 128, 30464, 22400, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8960, 30208, 19968, 6656, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 128, 128, 2176, 128, 256, 128, 8960, 32768, 21376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[512, 128, 128, 128, 21504, 128, 128, 128, 25472, 11392, 6912, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[896, 512, 3968, 2688, 32768, 128, 3584, 128, 12928, 8704, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 1024, 1280, 32384, 22656, 128, 1408, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 768, 128, 128, 128, 32640, 2944, 5120, 15488, 8576, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 3, 69, 22, 0, 0, 4096, 4096, 0, 4096, 0, 2, 806, 4096, 0, 4095, 4063, 0, 0, 0, 0, 0, 3670, 0, 4096, 0, 1128, 0, 2778, 0, 0, 1705, 4, 0, 0, 1, 3562, 13, 0, 4001, 0, 0, 4092, 0, 140, 47, 0, 0, 0, 0, 4091, 0, 0, 3, 2575, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4, 3963, 199, 0, 17, 0, 0, 0, 3988, 18, 0, 0, 776, 4076, 3, 4063, 13, 212, 3, 4088, 0, 3950, 4095, 0, 0, 1, 86, 25, 4029, 0, 3978, 1, 4091, 0, 4096, 0, 0, 0, 37, 1, 0, 94, 3842, 8, 5, 3121, 0, 37, 665, 0, 1, 39, 58, 34, 0, 0, 4096, 0, 113, 0, 3799, 8, 0, 3803] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 4003, 2, 4, 3, 3847, 4093, 0, 4064, 0, 0, 453, 3856, 0, 4091, 3423, 0, 17, 0, 0, 14, 4096, 73, 4085, 0, 506, 1, 1716, 0, 27, 0, 3230, 0, 162, 0, 81, 666, 6, 3209, 1, 0, 3453, 0, 36, 0, 29, 0, 1, 0, 4020, 13, 0, 0, 4062, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 30, 1152, 36, 0, 0, 2752, 4096, 0, 4095, 0, 0, 7, 4096, 0, 4061, 3954, 2, 0, 0, 0, 0, 4018, 2, 4096, 0, 3721, 0, 1107, 0, 1, 2, 8, 17, 0, 2, 3325, 0, 0, 3657, 0, 4, 979, 0, 3965, 0, 0, 0, 0, 0, 4096, 0, 0, 30, 4092, 1, 0, 4037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[250, 4078, 0, 0, 0, 2, 1, 0, 1878, 34, 0, 0, 2641, 4092, 12, 4023, 3, 2, 0, 4096, 120, 4080, 4096, 0, 76, 0, 1, 1, 4092, 0, 4092, 0, 174, 0, 0, 0, 0, 3995, 0, 70, 0, 0, 13, 0, 0, 4020, 15, 0, 4030, 0, 3245, 3, 0, 36, 0, 0, 4070, 0, 0, 2, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4022, 0, 0, 0, 0, 0, 130, 69, 9, 0, 0, 4049, 4096, 1, 4045, 0, 39, 644, 3710, 0, 4088, 4088, 0, 5, 1, 0, 0, 4096, 243, 4096, 0, 4095, 7, 1885, 0, 1, 2928, 2041, 8, 4, 0, 0, 1123, 0, 3729, 2, 0, 0, 0, 0, 8, 3, 0, 0, 4, 4094, 0, 0, 3, 4096, 0, 0, 4074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4089, 9, 0, 0, 0, 5, 0, 2260, 0, 8, 1, 3276, 4095, 0, 4094, 0, 1515, 0, 4096, 0, 4089, 4095, 0, 141, 1, 0, 2, 3920, 1, 4088, 0, 42, 0, 3070, 0, 0, 3446, 0, 0, 0, 16, 63, 1, 0, 4096, 0, 6, 527, 0, 2092, 482, 0, 0, 0, 0, 4065, 0, 0, 0, 4096, 0, 0, 3749] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4020, 0, 0, 0, 0, 0, 216, 18, 236, 2, 0, 2485, 4092, 1, 4096, 0, 2, 0, 4096, 18, 4096, 4055, 0, 0, 0, 0, 11, 3884, 0, 4086, 33, 0, 64, 2599, 23, 0, 1, 2374, 30, 0, 272, 2310, 3, 1, 4095, 0, 5, 1908, 0, 3995, 171, 0, 9, 0, 0, 4068, 0, 23, 0, 4006, 0, 39, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 3, 3750, 302, 0, 0, 0, 0, 0, 0, 69, 22, 0, 0, 4096, 0, 0, 3494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 806, 3803, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3670, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 1128, 0, 2778, 0, 0, 1705, 4, 0, 977, 602, 245, 0, 0, 0, 0, 0, 0, 1, 3562, 13, 0, 4001, 0, 0, 2505, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 140, 47, 0, 0, 0, 0, 4096, 345, 0, 0, 0, 0, 0, 0, 307, 0, 0, 3, 2575, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 7681, 8674, 7814, 7439, 10082, 8720, 6981] avg=8192.0000 max=10082 max_rank=5 min=6981 min_rank=7 imbalance_ratio=1.230713 max_load_reduction=55454 max_load_reduction_pct=0.846161 moved_to_echo=33222 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 9, 0, 0, 0, 5, 0, 3743, 301, 0, 0, 0, 0, 0, 0, 2260, 0, 8, 1, 3276, 0, 0, 3492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1515, 0, 3803, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 1, 0, 2, 3920, 1, 4088, 0, 48, 0, 0, 0, 0, 0, 0, 0, 42, 0, 3070, 0, 0, 3446, 0, 0, 970, 602, 245, 0, 0, 0, 0, 0, 0, 16, 63, 1, 0, 4096, 0, 6, 2489, 0, 0, 0, 0, 0, 0, 0, 527, 0, 2092, 482, 0, 0, 0, 0, 4095, 346, 0, 0, 0, 0, 0, 0, 305, 0, 0, 0, 4096, 0, 0, 3749, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 9037, 9413, 8201, 8375, 6671, 7542, 8150] avg=8192.0000 max=9413 max_rank=2 min=6671 min_rank=5 imbalance_ratio=1.149048 max_load_reduction=56123 max_load_reduction_pct=0.856369 moved_to_echo=30738 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[250, 4078, 0, 0, 0, 2, 1, 0, 3735, 301, 0, 0, 0, 0, 0, 0, 1878, 34, 0, 0, 2641, 0, 12, 3432, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 3803, 120, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 1, 1, 4092, 0, 4092, 0, 48, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 3995, 0, 70, 972, 591, 245, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4020, 15, 0, 2492, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 3245, 3, 0, 36, 0, 0, 4092, 345, 0, 0, 0, 0, 0, 0, 305, 0, 0, 2, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8367, 7997, 8024, 8310, 6047, 6540, 11751, 8500] avg=8192.0000 max=11751 max_rank=6 min=6047 min_rank=4 imbalance_ratio=1.434448 max_load_reduction=53785 max_load_reduction_pct=0.820694 moved_to_echo=32838 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4, 3963, 199, 0, 17, 0, 0, 0, 3616, 303, 0, 0, 0, 0, 0, 0, 3988, 18, 0, 0, 776, 0, 3, 3466, 0, 0, 0, 0, 0, 0, 0, 0, 13, 212, 3, 3795, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 86, 25, 4029, 0, 3978, 1, 48, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 37, 1, 978, 597, 245, 0, 0, 0, 0, 0, 0, 94, 3842, 8, 5, 3121, 0, 37, 2508, 0, 0, 0, 0, 0, 0, 0, 665, 0, 1, 39, 58, 34, 0, 0, 4076, 334, 0, 0, 0, 0, 0, 0, 307, 0, 113, 0, 3799, 8, 0, 3803, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8251, 8118, 8168, 10045, 9615, 5207, 8030] avg=8192.0000 max=10045 max_rank=4 min=5207 min_rank=6 imbalance_ratio=1.226196 max_load_reduction=55491 max_load_reduction_pct=0.846725 moved_to_echo=32897 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 3746, 297, 0, 0, 0, 0, 0, 0, 4003, 2, 4, 3, 3847, 0, 0, 3467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 3580, 0, 0, 3423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 14, 4096, 73, 4085, 0, 45, 0, 0, 0, 0, 0, 0, 0, 506, 1, 1716, 0, 27, 0, 3230, 0, 960, 597, 231, 0, 0, 0, 0, 0, 162, 0, 81, 666, 6, 3209, 1, 0, 2461, 0, 0, 0, 0, 0, 0, 0, 3453, 0, 36, 0, 29, 0, 1, 0, 4093, 345, 0, 0, 0, 0, 0, 0, 302, 13, 0, 0, 4062, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 11326, 7456, 8330, 7268, 6586, 7957, 8473] avg=8192.0000 max=11326 max_rank=1 min=6586 min_rank=5 imbalance_ratio=1.382568 max_load_reduction=54210 max_load_reduction_pct=0.827179 moved_to_echo=30284 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 30, 3718, 303, 0, 0, 0, 0, 0, 0, 1152, 36, 0, 0, 2752, 0, 0, 3493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3803, 0, 0, 3954, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 2, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 1107, 0, 1, 2, 8, 17, 978, 602, 245, 0, 0, 0, 0, 0, 0, 2, 3325, 0, 0, 3657, 0, 4, 2508, 0, 0, 0, 0, 0, 0, 0, 979, 0, 3965, 0, 0, 0, 0, 0, 4096, 343, 0, 0, 0, 0, 0, 0, 307, 0, 0, 30, 4092, 1, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 7433, 7766, 8164, 6681, 9496, 9383, 8467] avg=8192.0000 max=9496 max_rank=5 min=6681 min_rank=4 imbalance_ratio=1.159180 max_load_reduction=56040 max_load_reduction_pct=0.855103 moved_to_echo=34027 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4022, 0, 0, 0, 0, 0, 130, 3743, 303, 0, 0, 0, 0, 0, 0, 69, 9, 0, 0, 4049, 0, 1, 3451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 644, 3445, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 4096, 243, 4096, 0, 43, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 1885, 0, 1, 2928, 2041, 8, 977, 594, 222, 0, 0, 0, 0, 0, 4, 0, 0, 1123, 0, 3729, 2, 0, 2507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 4, 4096, 345, 0, 0, 0, 0, 0, 0, 307, 0, 0, 3, 4096, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 7579, 8216, 8484, 12758, 7365, 4456, 8480] avg=8192.0000 max=12758 max_rank=4 min=4456 min_rank=6 imbalance_ratio=1.557373 max_load_reduction=52778 max_load_reduction_pct=0.805328 moved_to_echo=33059 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4020, 0, 0, 0, 0, 0, 216, 3751, 304, 0, 0, 0, 0, 0, 0, 18, 236, 2, 0, 2485, 0, 1, 3490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3794, 18, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3884, 0, 4086, 33, 51, 0, 0, 0, 0, 0, 0, 0, 0, 64, 2599, 23, 0, 1, 2374, 30, 975, 606, 251, 0, 0, 0, 0, 0, 0, 272, 2310, 3, 1, 4095, 0, 5, 2495, 0, 0, 0, 0, 0, 0, 0, 1908, 0, 3995, 171, 0, 9, 0, 0, 4092, 345, 0, 0, 0, 0, 0, 0, 294, 0, 23, 0, 4006, 0, 39, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 6232, 7869, 8065, 6923, 9181, 10520, 8455] avg=8192.0000 max=10520 max_rank=6 min=6232 min_rank=1 imbalance_ratio=1.284180 max_load_reduction=55016 max_load_reduction_pct=0.839478 moved_to_echo=35079 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[13824, 128, 17280, 128, 128, 12160, 7808, 128, 7808, 4864, 2048, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 512, 13312, 1920, 128, 29952, 128, 128, 19968, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[13440, 384, 128, 128, 23936, 0, 128, 27904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 1792, 1920, 29952, 256, 0, 31872, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 32512, 256, 0, 128, 128, 128, 384, 29824, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[15744, 0, 13568, 768, 128, 128, 128, 128, 32768, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 128, 128, 128, 31872, 384, 32640, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2560, 128, 256, 128, 30848, 128, 128, 32128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 3, 0, 4095, 0, 1997, 4096, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 469, 4096, 894, 3316, 0, 0, 34, 4, 0, 0, 0, 0, 0, 388, 4096, 0, 46, 4085, 4072, 0, 0, 0, 3588, 0, 4096, 4096, 0, 0, 0, 0, 4092, 4094, 0, 0, 4083, 1099, 548, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 4, 4090, 4088, 33, 0, 0, 0, 27, 0, 0, 0, 0, 0, 1, 7, 0, 0, 1, 0, 0, 351, 4074, 4018, 4022, 0, 0, 1357, 0, 60, 0, 0, 1, 0, 486, 4096, 0, 4, 2017, 4083, 3, 25, 0, 74, 0, 4094, 4091, 2, 0, 0, 0, 4078, 4059, 0, 0, 4058, 535, 3487, 19, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3548, 1, 0, 0, 3616, 0, 4080, 4072, 367, 85, 0, 0, 0, 0, 16, 0, 20, 19, 0, 452, 0, 186, 200, 18, 0, 1, 4074, 22, 3621, 0, 0, 305, 168, 0, 73, 51, 125, 0, 3656, 4096, 0, 0, 317, 3827, 171, 56, 4, 2246, 659, 4094, 4054, 0, 0, 0, 0, 4027, 4063, 1, 1, 2136, 3795, 2877, 297, 39] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4081, 0, 887, 0, 4058, 0, 4095, 3864, 0, 0, 0, 9, 121, 11, 13, 0, 340, 1932, 0, 21, 113, 0, 0, 2, 0, 3302, 4096, 3301, 3805, 0, 0, 0, 33, 1, 11, 0, 0, 1, 1053, 2433, 0, 21, 2352, 4093, 0, 0, 0, 474, 0, 4018, 4076, 0, 0, 161, 7, 4095, 3825, 42, 0, 4087, 6, 0, 694, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4086, 11, 0, 0, 3322, 27, 4095, 4051, 0, 0, 2, 0, 3, 0, 0, 0, 2, 10, 4, 0, 59, 0, 0, 3, 2, 608, 4006, 127, 3477, 71, 2, 500, 0, 92, 0, 0, 0, 5, 1339, 4095, 2, 433, 4063, 3787, 0, 0, 0, 3431, 0, 4096, 4044, 0, 657, 4, 103, 4081, 4004, 0, 0, 3842, 2, 2983, 2, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4092, 0, 0, 5, 4096, 0, 3394, 4096, 256, 0, 30, 0, 2276, 4, 0, 0, 0, 48, 0, 100, 0, 11, 0, 0, 0, 0, 4079, 0, 2547, 0, 0, 1, 19, 858, 0, 0, 0, 0, 4057, 3603, 0, 0, 1, 4039, 0, 1, 1, 3631, 0, 4096, 4090, 0, 2130, 0, 27, 4078, 3404, 0, 0, 3974, 40, 1877, 24, 551] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 712, 0, 4093, 0, 3011, 4082, 0, 0, 39, 0, 13, 0, 0, 1, 0, 2, 0, 6, 0, 0, 0, 0, 0, 3057, 4096, 5, 4091, 0, 0, 0, 1639, 2, 0, 0, 0, 1, 292, 4068, 0, 5, 4083, 1617, 0, 0, 0, 4032, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4091, 0, 0, 4096, 30, 1989, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 26, 0, 4096, 1, 93, 4018, 2, 0, 15, 3, 0, 1, 0, 0, 7, 630, 0, 0, 0, 313, 1, 20, 0, 4069, 4090, 1132, 4095, 9, 0, 0, 339, 0, 17, 3, 0, 0, 384, 3909, 0, 9, 4058, 3895, 0, 11, 0, 1811, 73, 4095, 4095, 0, 0, 1, 0, 4034, 4081, 0, 36, 4095, 5, 3831, 37, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[959, 0, 3, 0, 4095, 0, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 3137, 2758, 1023, 513, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 3583, 0, 0, 0, 0, 0, 0, 0, 469, 1624, 894, 3316, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 388, 4096, 1449, 682, 440, 0, 0, 0, 0, 0, 0, 46, 4085, 3390, 0, 0, 0, 3588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1336, 0, 0, 4083, 1099, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7054, 11477, 7686, 6337, 7059, 11109, 7748, 7066] avg=8192.0000 max=11477 max_rank=1 min=6337 min_rank=3 imbalance_ratio=1.401001 max_load_reduction=54059 max_load_reduction_pct=0.824875 moved_to_echo=32982 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[959, 0, 0, 0, 4095, 4, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 27, 0, 0, 0, 3137, 2735, 1017, 511, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 1, 0, 4094, 3577, 0, 0, 0, 0, 0, 0, 0, 351, 1616, 4018, 4022, 0, 0, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 1, 0, 486, 4096, 1441, 684, 440, 0, 0, 0, 0, 0, 0, 4, 2017, 3399, 3, 25, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3651, 2, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 1324, 0, 0, 4058, 535, 3487, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9148, 7460, 7680, 11364, 7208, 5522, 7731, 9423] avg=8192.0000 max=11364 max_rank=3 min=5522 min_rank=5 imbalance_ratio=1.387207 max_load_reduction=54172 max_load_reduction_pct=0.826599 moved_to_echo=29884 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[957, 11, 0, 0, 3322, 27, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 3129, 2698, 1000, 507, 0, 0, 0, 0, 2, 10, 4, 0, 59, 0, 0, 3, 4096, 3544, 0, 0, 0, 0, 0, 0, 2, 608, 1589, 127, 3477, 71, 2, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 5, 1339, 4095, 1417, 635, 434, 0, 0, 0, 0, 0, 2, 433, 4063, 3152, 0, 0, 0, 3431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 0, 657, 4, 103, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 1306, 0, 0, 3842, 2, 2983, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8412, 7339, 7718, 6376, 8017, 11081, 8455, 8138] avg=8192.0000 max=11081 max_rank=5 min=6376 min_rank=3 imbalance_ratio=1.352661 max_load_reduction=54455 max_load_reduction_pct=0.830917 moved_to_echo=35691 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[956, 0, 887, 0, 4058, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 121, 11, 13, 0, 3125, 2577, 1023, 484, 0, 0, 0, 0, 340, 1932, 0, 21, 113, 0, 0, 2, 4018, 3380, 0, 0, 0, 0, 0, 0, 0, 3302, 1624, 3301, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 1, 11, 0, 0, 1, 1053, 2433, 1449, 686, 438, 0, 0, 0, 0, 0, 0, 21, 2352, 3407, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3638, 0, 0, 161, 7, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1248, 42, 0, 4087, 6, 0, 694, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9996, 7363, 9806, 12032, 6105, 6254, 7901, 6079] avg=8192.0000 max=12032 max_rank=3 min=6079 min_rank=7 imbalance_ratio=1.468750 max_load_reduction=53504 max_load_reduction_pct=0.816406 moved_to_echo=26339 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[959, 0, 712, 0, 4093, 0, 3011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 13, 0, 0, 1, 3136, 2756, 1023, 510, 0, 0, 0, 0, 0, 2, 0, 6, 0, 0, 0, 0, 4096, 3572, 0, 0, 0, 0, 0, 0, 0, 3057, 1624, 5, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1639, 2, 0, 0, 0, 1, 292, 4068, 1449, 271, 440, 0, 0, 0, 0, 0, 0, 5, 4083, 1346, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1335, 0, 0, 4096, 30, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8775, 7478, 7676, 8777, 8162, 9466, 7752, 7450] avg=8192.0000 max=9466 max_rank=5 min=7450 min_rank=7 imbalance_ratio=1.155518 max_load_reduction=56070 max_load_reduction_pct=0.855560 moved_to_echo=32830 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[831, 1, 0, 0, 3616, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 85, 0, 0, 0, 0, 16, 0, 2717, 2738, 1017, 510, 0, 0, 0, 0, 20, 19, 0, 452, 0, 186, 200, 18, 4094, 3562, 0, 0, 0, 0, 0, 0, 0, 1, 1616, 22, 3621, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 73, 51, 125, 0, 3656, 4096, 1441, 641, 436, 0, 0, 0, 0, 0, 0, 0, 317, 3186, 171, 56, 4, 2246, 0, 0, 0, 0, 0, 0, 0, 0, 659, 0, 3618, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 1, 1, 2136, 3795, 2877, 297, 39, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8528, 7450, 8551, 5565, 10687, 5980, 8304, 10471] avg=8192.0000 max=10687 max_rank=4 min=5565 min_rank=3 imbalance_ratio=1.304565 max_load_reduction=54849 max_load_reduction_pct=0.836929 moved_to_echo=35442 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[958, 0, 0, 5, 4096, 0, 3394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 30, 0, 2276, 4, 0, 0, 3134, 2293, 1019, 513, 0, 0, 0, 0, 0, 48, 0, 100, 0, 11, 0, 0, 4096, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 1617, 0, 2547, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 19, 858, 0, 0, 0, 0, 4057, 3603, 1443, 677, 439, 0, 0, 0, 0, 0, 0, 0, 1, 3362, 0, 1, 1, 3631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3651, 0, 2130, 0, 27, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 0, 0, 3974, 40, 1877, 24, 551, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8453, 9525, 7838, 4165, 11096, 6996, 9886, 7577] avg=8192.0000 max=11096 max_rank=4 min=4165 min_rank=3 imbalance_ratio=1.354492 max_load_reduction=54440 max_load_reduction_pct=0.830688 moved_to_echo=35555 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[954, 0, 26, 0, 4096, 1, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 15, 3, 0, 1, 0, 0, 3141, 2754, 1025, 503, 0, 0, 0, 0, 7, 630, 0, 0, 0, 313, 1, 20, 4095, 3515, 0, 0, 0, 0, 0, 0, 0, 4069, 1615, 1132, 4095, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 17, 3, 0, 0, 384, 3909, 1450, 656, 444, 0, 0, 0, 0, 0, 0, 9, 4058, 3239, 0, 11, 0, 1811, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 3651, 0, 0, 1, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 0, 36, 4095, 5, 3831, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5170, 7444, 8581, 10920, 7202, 9128, 7759, 9332] avg=8192.0000 max=10920 max_rank=3 min=5170 min_rank=0 imbalance_ratio=1.333008 max_load_reduction=54616 max_load_reduction_pct=0.833374 moved_to_echo=33421 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[768, 128, 128, 128, 6528, 128, 128, 128, 24704, 21376, 8192, 4096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 640, 20992, 24576, 256, 128, 128, 19328, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 11904, 12928, 9600, 29056, 128, 128, 2304, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[10368, 128, 128, 30464, 5632, 17664, 1152, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7552, 128, 1664, 128, 31488, 128, 24960, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[768, 0, 29184, 128, 2816, 256, 256, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 2688, 128, 640, 256, 512, 256, 128, 32768, 28416, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2304, 1024, 128, 128, 128, 128, 11776, 30464, 11648, 4992, 3584, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 1406, 18, 850, 0, 4089, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 276, 5, 3767, 0, 3980, 0, 0, 0, 3909, 2, 3032, 0, 0, 4096, 0, 0, 4095, 4093, 9, 88, 3120, 0, 4096, 15, 4096, 0, 0, 0, 0, 0, 2228, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1903, 4096, 4056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 4092, 0, 3790, 146, 14, 3848, 4, 0, 0, 15, 0, 2, 11, 0, 0, 0, 4080, 14, 4010, 1, 4023, 0, 0, 0, 3259, 331, 909, 0, 0, 4096, 85, 0, 4095, 3971, 0, 0, 497, 0, 4017, 2, 4096, 4, 4, 0, 0, 2, 3490, 860, 0, 2, 29, 2, 4, 0, 3, 3862, 0, 0, 0, 66, 492, 4096, 3212] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[28, 4095, 381, 0, 1319, 0, 4086, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 3299, 4088, 57, 14, 1123, 0, 0, 2, 4084, 0, 517, 12, 0, 3584, 9, 0, 4050, 4064, 0, 2482, 1, 0, 4096, 0, 3949, 11, 0, 0, 0, 4, 3343, 464, 0, 0, 0, 21, 0, 0, 0, 4087, 0, 0, 0, 13, 4068, 4090, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[18, 4096, 2, 1135, 1, 0, 3998, 27, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1225, 4034, 1851, 0, 444, 0, 0, 0, 4067, 7, 2018, 0, 0, 2288, 0, 0, 3996, 4036, 8, 2850, 1863, 8, 4089, 35, 3861, 0, 15, 0, 35, 30, 3588, 3841, 256, 0, 0, 0, 3, 69, 3, 4096, 0, 19, 0, 18, 512, 4036, 3055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 3796, 19, 0, 3242, 0, 3701, 0, 308, 0, 4, 13, 0, 0, 22, 43, 0, 916, 4089, 3477, 61, 0, 40, 268, 0, 3320, 803, 0, 10, 713, 3331, 31, 36, 3413, 4024, 16, 4083, 0, 1, 4073, 122, 3859, 0, 0, 0, 0, 0, 425, 4094, 64, 0, 60, 0, 67, 0, 116, 4028, 1, 0, 2, 0, 1038, 4048, 3759] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 4094, 0, 551, 1538, 0, 4096, 0, 0, 0, 0, 3, 0, 31, 3, 0, 0, 924, 70, 0, 44, 3716, 0, 0, 0, 4040, 0, 0, 0, 0, 678, 0, 2, 3774, 4096, 0, 4023, 3725, 0, 4096, 1, 4096, 441, 0, 0, 0, 0, 4090, 2899, 56, 0, 0, 337, 0, 0, 1, 4094, 0, 1, 0, 115, 3934, 4096, 1871] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 29, 1418, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 3834, 2595, 1, 629, 0, 0, 0, 4096, 0, 0, 3, 0, 4096, 0, 0, 3961, 4096, 125, 865, 4064, 0, 4096, 121, 4096, 0, 0, 0, 0, 0, 2679, 46, 79, 0, 0, 0, 0, 0, 0, 4096, 0, 105, 0, 0, 90, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[5, 4067, 0, 359, 3497, 0, 4016, 0, 0, 6, 4, 2, 5, 0, 56, 0, 2, 728, 781, 3508, 7, 4073, 5, 21, 0, 4091, 0, 526, 0, 6, 4094, 3, 19, 4090, 4041, 25, 3926, 79, 1, 4062, 5, 4089, 0, 0, 0, 0, 0, 940, 0, 431, 1, 0, 0, 0, 0, 0, 4092, 2, 0, 3, 0, 1721, 4051, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1745, 1406, 18, 850, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 276, 5, 2892, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3832, 2, 3032, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 271, 9, 88, 3120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 2228, 842, 875, 77, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 3822, 2351, 324, 30, 0, 0, 0, 0, 2930, 0, 0, 0, 0, 1903, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8166, 7153, 10962, 7583, 8133, 6542, 8889] avg=8192.0000 max=10962 max_rank=3 min=6542 min_rank=6 imbalance_ratio=1.338135 max_load_reduction=54574 max_load_reduction_pct=0.832733 moved_to_echo=31147 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[28, 1744, 381, 0, 1319, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 0, 0, 0, 0, 0, 3299, 4088, 44, 14, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4004, 0, 517, 12, 0, 3584, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 270, 0, 2482, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 11, 0, 0, 0, 4, 3343, 840, 13, 80, 0, 0, 0, 0, 0, 464, 0, 0, 0, 21, 0, 0, 0, 3794, 2351, 324, 31, 0, 0, 0, 0, 2923, 0, 0, 0, 13, 4068, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7558, 8163, 8568, 8128, 6803, 8240, 6985, 11091] avg=8192.0000 max=11091 max_rank=7 min=6803 min_rank=4 imbalance_ratio=1.353882 max_load_reduction=54445 max_load_reduction_pct=0.830765 moved_to_echo=33119 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[18, 1745, 2, 1135, 1, 0, 3998, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 4089, 4006, 0, 0, 0, 0, 0, 0, 0, 1225, 4034, 1421, 0, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 7, 2018, 0, 0, 2288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 268, 8, 2850, 1863, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 3861, 0, 15, 0, 35, 30, 3588, 842, 430, 80, 0, 0, 0, 0, 0, 3841, 256, 0, 0, 0, 3, 69, 3, 3768, 2351, 324, 30, 0, 0, 0, 0, 2930, 0, 19, 0, 18, 512, 0, 3055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6926, 8098, 7124, 8300, 8993, 8916, 10645, 6534] avg=8192.0000 max=10645 max_rank=6 min=6534 min_rank=7 imbalance_ratio=1.299438 max_load_reduction=54891 max_load_reduction_pct=0.837570 moved_to_echo=35088 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1745, 29, 1418, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 3932, 3834, 1993, 1, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 272, 125, 865, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 4096, 0, 0, 0, 0, 0, 2679, 842, 602, 81, 0, 0, 0, 0, 0, 46, 79, 0, 0, 0, 0, 0, 0, 3824, 2351, 324, 31, 0, 0, 0, 0, 2930, 0, 105, 0, 0, 90, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7288, 8161, 10389, 8114, 9287, 8421, 6655, 7221] avg=8192.0000 max=10389 max_rank=2 min=6655 min_rank=6 imbalance_ratio=1.268188 max_load_reduction=55147 max_load_reduction_pct=0.841476 moved_to_echo=31584 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1743, 0, 3790, 146, 14, 3848, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 2, 11, 0, 0, 4017, 4065, 0, 0, 0, 0, 0, 0, 0, 4080, 14, 3079, 1, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3195, 331, 909, 0, 0, 4096, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 263, 0, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4, 4, 0, 0, 2, 3490, 794, 931, 64, 0, 0, 0, 0, 0, 860, 0, 2, 29, 2, 4, 0, 3, 3708, 2349, 306, 31, 0, 0, 0, 0, 2762, 0, 0, 0, 66, 492, 0, 3212, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9545, 8110, 11197, 8616, 4855, 9387, 7294, 6532] avg=8192.0000 max=11197 max_rank=2 min=4855 min_rank=4 imbalance_ratio=1.366821 max_load_reduction=54339 max_load_reduction_pct=0.829147 moved_to_echo=28068 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1617, 19, 0, 3242, 0, 3701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 0, 4, 13, 0, 0, 22, 43, 4073, 4019, 0, 0, 0, 0, 0, 0, 0, 916, 4089, 2670, 61, 0, 40, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3255, 803, 0, 10, 713, 3331, 31, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3413, 267, 16, 4083, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 3859, 0, 0, 0, 0, 0, 425, 828, 807, 65, 0, 0, 0, 0, 0, 4094, 64, 0, 60, 0, 67, 0, 116, 3757, 2179, 319, 29, 0, 0, 0, 0, 2881, 1, 0, 2, 0, 1038, 0, 3759, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8579, 8482, 8044, 8143, 7816, 6106, 10685, 7681] avg=8192.0000 max=10685 max_rank=6 min=6106 min_rank=5 imbalance_ratio=1.304321 max_load_reduction=54851 max_load_reduction_pct=0.836960 moved_to_echo=32288 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1744, 0, 551, 1538, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 31, 3, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 924, 70, 0, 44, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 0, 0, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3774, 272, 0, 4023, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 441, 0, 0, 0, 0, 4090, 842, 0, 80, 0, 0, 0, 0, 0, 2899, 56, 0, 0, 337, 0, 0, 1, 3824, 2350, 324, 30, 0, 0, 0, 0, 2928, 0, 1, 0, 115, 3934, 0, 1871, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7929, 8199, 4754, 4638, 11796, 9550, 9821, 8849] avg=8192.0000 max=11796 max_rank=4 min=4638 min_rank=3 imbalance_ratio=1.439941 max_load_reduction=53740 max_load_reduction_pct=0.820007 moved_to_echo=40016 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[5, 1726, 0, 359, 3497, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 2, 5, 0, 56, 0, 4062, 4022, 0, 0, 0, 0, 0, 0, 2, 728, 781, 2690, 7, 4073, 5, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 526, 0, 6, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4090, 263, 25, 3926, 79, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4089, 0, 0, 0, 0, 0, 940, 846, 818, 85, 0, 0, 0, 0, 0, 0, 431, 1, 0, 0, 0, 0, 0, 3778, 2341, 329, 29, 0, 0, 0, 0, 2917, 2, 0, 3, 0, 1721, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9603, 8157, 8307, 8635, 8403, 6783, 6909, 8739] avg=8192.0000 max=9603 max_rank=0 min=6783 min_rank=5 imbalance_ratio=1.172241 max_load_reduction=55933 max_load_reduction_pct=0.853470 moved_to_echo=30834 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12288, 896, 128, 128, 384, 128, 128, 128, 30336, 18688, 2688, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23296, 128, 128, 128, 256, 13824, 0, 28288, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 13824, 1920, 7296, 10624, 128, 32000, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 15488, 17024, 14848, 128, 18048, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 128, 128, 128, 128, 128, 128, 128, 32640, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 31488, 2176, 256, 18432, 13440, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 32256, 512, 128, 0, 128, 128, 20864, 6784, 4480, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 30336, 1152, 7040, 128, 768, 26368, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 3820, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4080, 86, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 2130, 0, 4093, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3346, 0, 2888, 0, 4079, 0, 0, 0, 4095, 0, 4096, 1, 0, 0, 0, 0, 0, 31, 4096, 8, 0, 0, 0, 0, 17, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[16, 4063, 4096, 4062, 1, 0, 0, 5, 0, 1, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 3010, 0, 4096, 2995, 0, 4072, 8, 0, 0, 4093, 0, 4090, 0, 0, 4093, 0, 0, 3806, 0, 1, 0, 4082, 0, 0, 0, 4096, 0, 4091, 3, 0, 0, 0, 0, 73, 1, 4094, 1686, 0, 4, 0, 0, 792, 0, 3, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 4065, 3635, 0, 0, 0, 77, 0, 27, 0, 4090, 27, 10, 0, 0, 0, 0, 0, 1165, 16, 4007, 2565, 0, 3963, 131, 1, 0, 4050, 0, 4096, 0, 0, 3916, 0, 224, 4079, 0, 4095, 24, 4076, 0, 1, 29, 4071, 30, 3927, 246, 1, 0, 2, 3, 4, 10, 4058, 125, 570, 0, 10, 0, 4087, 0, 0, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 2852, 4096, 4045, 0, 87, 8, 35, 18, 0, 0, 3795, 0, 0, 195, 0, 0, 0, 58, 3951, 0, 4096, 3736, 0, 3344, 0, 0, 0, 4094, 0, 3598, 0, 2, 4081, 0, 0, 1457, 0, 3672, 59, 4070, 0, 6, 1, 4016, 0, 4089, 4, 5, 0, 193, 0, 365, 0, 4027, 221, 71, 0, 0, 0, 1189, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3, 4093, 4096, 4091, 0, 19, 43, 1, 0, 0, 0, 4037, 0, 0, 0, 10, 0, 0, 0, 2071, 0, 4096, 2793, 44, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 3861, 0, 1, 4042, 0, 3022, 0, 4092, 0, 0, 0, 4090, 0, 3697, 377, 2, 0, 0, 0, 0, 0, 4096, 293, 0, 0, 279, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 148, 4096, 3606, 1, 4, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4004, 2, 4060, 4043, 14, 4063, 39, 36, 0, 4096, 0, 4084, 7, 0, 4096, 0, 1, 3812, 29, 4085, 34, 3962, 0, 521, 0, 3864, 9, 4096, 21, 0, 0, 0, 6, 0, 0, 4084, 168, 0, 0, 0, 203, 110, 31, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 4080, 0, 4096, 440, 0, 4093, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4094, 0, 3, 0, 4091, 0, 4096, 0, 0, 0, 0, 1, 6, 0, 4096, 121, 0, 0, 0, 0, 3548, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 3590, 4093, 4075, 1, 0, 0, 45, 0, 0, 5, 3768, 941, 0, 1, 0, 0, 0, 0, 4088, 13, 3997, 149, 22, 4095, 0, 0, 0, 4096, 0, 4041, 1, 0, 4093, 0, 0, 3068, 3, 128, 0, 3874, 5, 0, 3, 4043, 425, 4082, 0, 0, 8, 0, 2, 0, 454, 4095, 3896, 92, 0, 94, 0, 129, 0, 21, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3820, 1867, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 86, 0, 0, 0, 2229, 1192, 608, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2524, 2130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 164, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1960, 0, 0, 3346, 0, 2888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 8, 1572, 1528, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4095, 2740, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9783, 8195, 8748, 8353, 8194, 8094, 7317, 6852] avg=8192.0000 max=9783 max_rank=0 min=6852 min_rank=7 imbalance_ratio=1.194214 max_load_reduction=55753 max_load_reduction_pct=0.850723 moved_to_echo=30457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[16, 4063, 1867, 4062, 1, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 2, 0, 0, 2229, 1191, 608, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 0, 2524, 2995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 8, 0, 0, 164, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1959, 0, 0, 3806, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 1, 4094, 1686, 1572, 1526, 82, 0, 0, 0, 0, 0, 0, 4, 0, 0, 792, 0, 3, 5, 4096, 2738, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10014, 8127, 8529, 8334, 5766, 8094, 9034, 7638] avg=8192.0000 max=10014 max_rank=0 min=5766 min_rank=4 imbalance_ratio=1.222412 max_load_reduction=55522 max_load_reduction_pct=0.847198 moved_to_echo=30532 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2852, 1867, 4045, 0, 87, 8, 35, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 3795, 0, 0, 195, 0, 2229, 1192, 606, 0, 0, 0, 0, 0, 0, 0, 58, 3951, 0, 2524, 3736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3344, 0, 0, 0, 163, 0, 3598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1953, 0, 0, 1457, 0, 3672, 59, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 6, 1, 0, 0, 4089, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 193, 0, 365, 0, 4027, 221, 1572, 1522, 82, 0, 0, 0, 0, 0, 71, 0, 0, 0, 1189, 0, 0, 0, 4016, 2739, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8894, 8035, 10269, 7105, 7143, 8088, 7987, 8015] avg=8192.0000 max=10269 max_rank=2 min=7105 min_rank=3 imbalance_ratio=1.253540 max_load_reduction=55267 max_load_reduction_pct=0.843307 moved_to_echo=31233 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1853, 3635, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4090, 27, 10, 0, 0, 2212, 1179, 581, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 16, 2470, 2565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 131, 1, 0, 162, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1875, 0, 224, 4079, 0, 4095, 24, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 1, 29, 0, 30, 3927, 246, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 3, 4, 10, 4058, 125, 1537, 1460, 82, 0, 0, 0, 0, 0, 570, 0, 10, 0, 4087, 0, 0, 23, 4071, 2709, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5565, 8126, 6216, 8353, 10297, 8227, 7282, 11470] avg=8192.0000 max=11470 max_rank=7 min=5565 min_rank=0 imbalance_ratio=1.400146 max_load_reduction=54066 max_load_reduction_pct=0.824982 moved_to_echo=37276 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 1867, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 2229, 1192, 608, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 2524, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 164, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1960, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 4096, 121, 1572, 1528, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3548, 0, 0, 0, 4091, 2740, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5967, 8124, 7044, 8353, 10152, 8111, 7406, 10379] avg=8192.0000 max=10379 max_rank=7 min=5967 min_rank=0 imbalance_ratio=1.266968 max_load_reduction=55157 max_load_reduction_pct=0.841629 moved_to_echo=36048 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 148, 1867, 3606, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 2229, 1192, 608, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 2, 2502, 4043, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 39, 36, 0, 164, 0, 4084, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1960, 0, 1, 3812, 29, 4085, 34, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 0, 521, 0, 0, 9, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4084, 168, 1558, 1528, 79, 0, 0, 0, 0, 0, 0, 0, 0, 203, 110, 31, 0, 0, 3864, 2740, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5626, 8130, 10565, 8393, 9921, 8530, 7423, 6948] avg=8192.0000 max=10565 max_rank=2 min=5626 min_rank=0 imbalance_ratio=1.289673 max_load_reduction=54971 max_load_reduction_pct=0.838791 moved_to_echo=32822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 4093, 1867, 4091, 0, 19, 43, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 10, 2229, 1192, 573, 0, 0, 0, 0, 0, 0, 0, 0, 2071, 0, 2524, 2793, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 164, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1848, 0, 1, 4042, 0, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 3697, 377, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 293, 1572, 1440, 82, 0, 0, 0, 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 4090, 2740, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10117, 8041, 7432, 8355, 8913, 8084, 7485, 7109] avg=8192.0000 max=10117 max_rank=0 min=7109 min_rank=7 imbalance_ratio=1.234985 max_load_reduction=55419 max_load_reduction_pct=0.845627 moved_to_echo=31591 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3590, 1859, 4075, 1, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3768, 941, 0, 1, 0, 2234, 1197, 612, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 13, 2461, 149, 22, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 153, 0, 4041, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1951, 0, 0, 3068, 3, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3793, 5, 0, 3, 0, 425, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 454, 4095, 3896, 1536, 1530, 81, 0, 0, 0, 0, 0, 92, 0, 94, 0, 129, 0, 21, 0, 4043, 2746, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9570, 8758, 6733, 8290, 5150, 8308, 11602, 7125] avg=8192.0000 max=11602 max_rank=6 min=5150 min_rank=4 imbalance_ratio=1.416260 max_load_reduction=53934 max_load_reduction_pct=0.822968 moved_to_echo=32185 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 18688, 14976, 31744, 128, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 256, 128, 512, 512, 32768, 6528, 12544, 12160, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31872, 256, 128, 0, 1408, 0, 32256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 26496, 128, 20096, 18944, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 15488, 0, 256, 27776, 128, 22016, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32128, 1152, 128, 256, 128, 17920, 9600, 4864, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 128, 384, 256, 9984, 128, 128, 128, 32384, 22016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31744, 128, 640, 128, 0, 512, 32256, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[48, 4096, 0, 0, 87, 346, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 40, 0, 0, 4095, 0, 0, 3641, 0, 0, 4096, 0, 0, 4096, 4096, 4000, 0, 0, 4096, 0, 0, 242, 4096, 4096, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 0, 0, 61, 4096, 4092, 4096, 0, 3810, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4091, 0, 134, 0, 85, 1, 777, 3945, 20, 13, 26, 0, 229, 0, 24, 4078, 0, 1, 1, 7, 4031, 0, 0, 3078, 0, 0, 4063, 0, 0, 2, 4096, 4069, 0, 19, 4095, 0, 0, 259, 4051, 4096, 0, 0, 36, 1, 0, 1, 1, 84, 0, 0, 2, 0, 0, 4096, 4007, 3837, 4096, 0, 4076, 0, 8, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3728, 4088, 42, 0, 140, 0, 10, 559, 3920, 0, 0, 0, 0, 0, 5, 0, 2232, 0, 26, 0, 272, 4096, 0, 2, 3460, 0, 0, 4054, 0, 0, 3842, 4085, 4096, 0, 2, 4096, 0, 0, 2292, 4096, 4093, 0, 0, 0, 0, 0, 10, 0, 0, 30, 0, 0, 0, 0, 0, 4096, 4065, 4089, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4029, 0, 0, 5, 26, 0, 3979, 2990, 0, 0, 1, 78, 3, 0, 0, 3861, 0, 705, 0, 0, 4064, 53, 0, 0, 339, 0, 3568, 0, 9, 3876, 4080, 3709, 3, 0, 4057, 179, 0, 163, 3902, 3967, 0, 0, 0, 18, 29, 0, 107, 343, 28, 0, 25, 0, 0, 4096, 4016, 4084, 4047, 76, 951, 0, 69, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[85, 4091, 0, 0, 1107, 0, 7, 515, 3775, 37, 0, 0, 0, 289, 0, 0, 2972, 4, 0, 0, 21, 4087, 5, 0, 1674, 366, 0, 4078, 0, 79, 4012, 3933, 3817, 0, 1, 4080, 0, 0, 3974, 4096, 4072, 0, 0, 0, 0, 40, 0, 2, 27, 1, 0, 15, 27, 1, 1694, 4096, 3971, 4077, 0, 391, 17, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[22, 4096, 0, 0, 2, 917, 0, 1490, 4095, 0, 1, 0, 0, 5, 0, 0, 4092, 0, 0, 0, 100, 4096, 0, 0, 3159, 0, 0, 4018, 0, 0, 4072, 4096, 4096, 0, 0, 4096, 0, 0, 2021, 4096, 4096, 0, 0, 0, 0, 0, 1, 1, 8, 0, 0, 0, 0, 0, 38, 4096, 4096, 4096, 0, 534, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1079, 4096, 0, 1, 89, 0, 2, 372, 3751, 0, 0, 0, 0, 1382, 0, 0, 3421, 0, 0, 0, 0, 4096, 0, 2, 4092, 0, 0, 4091, 0, 0, 4053, 4096, 3585, 0, 0, 4096, 13, 0, 2422, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 412, 0, 0, 0, 0, 0, 21, 4096, 43, 4096, 0, 3896, 0, 41, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[44, 4082, 0, 0, 172, 277, 0, 0, 3790, 0, 0, 1, 0, 288, 21, 12, 4057, 0, 0, 1, 0, 4060, 1, 20, 3639, 0, 0, 4094, 5, 8, 3911, 4095, 4034, 0, 2, 4095, 7, 0, 0, 4079, 4057, 0, 14, 2, 2, 11, 281, 0, 0, 0, 0, 0, 68, 0, 1984, 4016, 2414, 4083, 0, 3805, 0, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[48, 4096, 0, 0, 87, 346, 0, 0, 1200, 899, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 40, 0, 0, 4095, 0, 0, 358, 0, 0, 0, 0, 0, 0, 0, 3641, 0, 0, 1805, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 242, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 16, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 61, 4096, 2291, 232, 20, 0, 0, 0, 0, 0, 4092, 2664, 0, 3810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6676, 8197, 8588, 9542, 7081, 8185, 6701, 10566] avg=8192.0000 max=10566 max_rank=7 min=6676 min_rank=0 imbalance_ratio=1.289795 max_load_reduction=54970 max_load_reduction_pct=0.838776 moved_to_echo=32533 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3728, 4088, 42, 0, 140, 0, 10, 559, 1198, 899, 0, 0, 0, 0, 0, 0, 3920, 0, 0, 0, 0, 0, 5, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 2232, 0, 26, 0, 272, 4096, 0, 2, 358, 0, 0, 0, 0, 0, 0, 0, 3460, 0, 0, 1786, 0, 0, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 2292, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 10, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 4096, 2268, 232, 21, 0, 0, 0, 0, 0, 4065, 2659, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10664, 8024, 6986, 9088, 9229, 8164, 6647, 6734] avg=8192.0000 max=10664 max_rank=0 min=6647 min_rank=6 imbalance_ratio=1.301758 max_load_reduction=54872 max_load_reduction_pct=0.837280 moved_to_echo=30774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[85, 4091, 0, 0, 1107, 0, 7, 515, 1195, 899, 0, 0, 0, 0, 0, 0, 3775, 37, 0, 0, 0, 289, 0, 0, 4080, 3, 0, 0, 0, 0, 0, 0, 2972, 4, 0, 0, 21, 4087, 5, 0, 358, 0, 0, 0, 0, 0, 0, 0, 1674, 366, 0, 1797, 0, 79, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 0, 1, 0, 0, 0, 3974, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 40, 0, 2, 3911, 0, 0, 0, 0, 0, 0, 0, 27, 1, 0, 15, 27, 1, 1694, 4096, 2281, 231, 19, 0, 0, 0, 0, 0, 3971, 2651, 0, 391, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7899, 8184, 7447, 7928, 10631, 8025, 8392, 7030] avg=8192.0000 max=10631 max_rank=4 min=7030 min_rank=7 imbalance_ratio=1.297729 max_load_reduction=54905 max_load_reduction_pct=0.837784 moved_to_echo=34078 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 0, 134, 0, 85, 1, 777, 1200, 889, 0, 0, 0, 0, 0, 0, 3945, 20, 13, 26, 0, 229, 0, 24, 4095, 3, 0, 0, 0, 0, 0, 0, 4078, 0, 1, 1, 7, 4031, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 3078, 0, 0, 1790, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 19, 0, 0, 0, 259, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 36, 1, 0, 1, 1, 4073, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 2, 0, 0, 4096, 4007, 2273, 232, 20, 0, 0, 0, 0, 0, 3837, 2664, 0, 4076, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7177, 8355, 8472, 4870, 7155, 8208, 10714, 10585] avg=8192.0000 max=10714 max_rank=6 min=4870 min_rank=3 imbalance_ratio=1.307861 max_load_reduction=54822 max_load_reduction_pct=0.836517 moved_to_echo=36662 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[22, 4096, 0, 0, 2, 917, 0, 1490, 1200, 899, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 5, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 100, 4096, 0, 0, 358, 0, 0, 0, 0, 0, 0, 0, 3159, 0, 0, 1770, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2021, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 1, 4072, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 38, 4096, 2248, 232, 21, 0, 0, 0, 0, 0, 4096, 2664, 0, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8626, 8200, 8646, 9001, 8956, 8170, 6643, 7294] avg=8192.0000 max=9001 max_rank=3 min=6643 min_rank=6 imbalance_ratio=1.098755 max_load_reduction=56535 max_load_reduction_pct=0.862656 moved_to_echo=31063 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4029, 0, 0, 5, 26, 0, 3979, 1186, 856, 0, 0, 0, 0, 0, 0, 2990, 0, 0, 1, 78, 3, 0, 0, 4057, 4, 0, 0, 0, 0, 0, 0, 3861, 0, 705, 0, 0, 4064, 53, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 1572, 0, 9, 3876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 3, 0, 0, 179, 0, 163, 2705, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 18, 29, 0, 107, 4056, 0, 0, 0, 0, 0, 0, 0, 343, 28, 0, 25, 0, 0, 4096, 4016, 1996, 229, 20, 0, 0, 0, 0, 0, 4084, 2632, 76, 951, 0, 69, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10081, 7133, 9024, 5796, 6759, 8177, 10753, 7813] avg=8192.0000 max=10753 max_rank=6 min=5796 min_rank=3 imbalance_ratio=1.312622 max_load_reduction=54783 max_load_reduction_pct=0.835922 moved_to_echo=33502 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1079, 4096, 0, 1, 89, 0, 2, 372, 1200, 899, 0, 0, 0, 0, 0, 0, 3751, 0, 0, 0, 0, 1382, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3421, 0, 0, 0, 0, 4096, 0, 2, 358, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1803, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3585, 0, 0, 0, 13, 0, 2422, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 412, 0, 0, 0, 0, 0, 21, 4096, 2288, 232, 20, 0, 0, 0, 0, 0, 43, 2664, 0, 3896, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7738, 9232, 7877, 9948, 8859, 8169, 7069, 6644] avg=8192.0000 max=9948 max_rank=3 min=6644 min_rank=7 imbalance_ratio=1.214355 max_load_reduction=55588 max_load_reduction_pct=0.848206 moved_to_echo=30741 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[44, 4082, 0, 0, 172, 277, 0, 0, 1201, 899, 0, 0, 0, 0, 0, 0, 3790, 0, 0, 1, 0, 288, 21, 12, 4095, 4, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 1, 0, 4060, 1, 20, 357, 0, 0, 0, 0, 0, 0, 0, 3639, 0, 0, 1800, 5, 8, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 2, 0, 7, 0, 0, 2823, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 14, 2, 2, 11, 281, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 1984, 4016, 2294, 235, 20, 0, 0, 0, 0, 0, 2414, 2647, 0, 3805, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6675, 8211, 8496, 9363, 6866, 8438, 8617, 8870] avg=8192.0000 max=9363 max_rank=3 min=6675 min_rank=0 imbalance_ratio=1.142944 max_load_reduction=56173 max_load_reduction_pct=0.857132 moved_to_echo=32791 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[22784, 768, 0, 14208, 128, 128, 27904, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[5120, 32768, 128, 256, 1664, 1664, 128, 7808, 9600, 7168, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[28928, 128, 896, 128, 512, 32640, 128, 128, 2944, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31488, 128, 128, 0, 256, 0, 11392, 22656, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 0, 128, 128, 128, 128, 384, 128, 32512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26624, 21248, 128, 17536, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[30464, 128, 128, 128, 128, 2304, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[896, 128, 128, 128, 128, 128, 12032, 32640, 18048, 1920, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 405, 0, 14, 0, 16, 4025, 0, 0, 0, 0, 0, 0, 0, 730, 1370, 0, 232, 5, 0, 4093, 167, 4076, 0, 1, 0, 4096, 1, 164, 453, 105, 2112, 40, 242, 0, 4, 4018, 3998, 8, 0, 2663, 3900, 0, 54, 5, 1813, 40, 0, 0, 83, 3822, 2917, 3624, 48, 4096, 4033, 10, 1, 3403, 0, 68, 0, 485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 3401, 0, 0, 0, 0, 3671, 3774, 0, 0, 2022, 0, 24, 0, 0, 258, 145, 0, 4080, 0, 0, 3270, 0, 4096, 0, 0, 0, 4095, 3, 169, 0, 0, 27, 0, 0, 0, 3186, 4090, 4096, 0, 0, 0, 4093, 0, 0, 0, 85, 0, 0, 0, 0, 4096, 3955, 4018, 0, 4096, 4096, 0, 0, 641, 0, 0, 49, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 3916, 0, 50, 77, 0, 3197, 4083, 2, 1, 4012, 26, 0, 15, 6, 0, 0, 0, 3984, 0, 0, 168, 0, 4096, 0, 1, 0, 179, 59, 596, 0, 0, 4096, 0, 37, 0, 735, 4061, 3888, 0, 8, 85, 3437, 0, 0, 0, 0, 167, 0, 5, 0, 4095, 4087, 4079, 0, 4096, 4096, 1, 0, 4032, 10, 2, 0, 51] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4, 3436, 0, 0, 0, 0, 1058, 3971, 0, 158, 375, 0, 4, 0, 0, 0, 582, 0, 3697, 0, 0, 3066, 1036, 4001, 0, 0, 0, 4096, 0, 0, 1, 0, 3657, 0, 0, 0, 3785, 4089, 4096, 0, 0, 7, 4029, 6, 0, 0, 0, 0, 0, 3, 12, 4093, 4096, 3918, 0, 4095, 4077, 0, 0, 15, 0, 6, 67, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3445, 33, 0, 0, 0, 9, 1649, 0, 40, 4094, 0, 2330, 0, 485, 0, 0, 0, 4096, 0, 0, 4095, 118, 3973, 0, 0, 0, 4096, 0, 0, 0, 0, 174, 0, 0, 0, 4059, 4087, 4089, 0, 0, 0, 4096, 0, 5, 0, 57, 16, 0, 0, 0, 3415, 4096, 4083, 0, 4096, 4066, 0, 0, 718, 0, 1, 0, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 3920, 2527, 0, 1690, 4067, 0, 73, 0, 0, 0, 8, 3, 54, 0, 0, 4096, 61, 4096, 0, 0, 0, 139, 0, 25, 32, 0, 4094, 0, 0, 3, 1079, 4096, 4096, 0, 0, 1206, 4093, 0, 0, 0, 149, 0, 3, 0, 0, 4096, 3673, 4096, 0, 4096, 4096, 0, 0, 1773, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[92, 3385, 58, 0, 0, 17, 4096, 1042, 0, 0, 3465, 44, 376, 0, 0, 0, 3663, 0, 3345, 0, 0, 3051, 501, 4046, 0, 9, 0, 3918, 53, 497, 2153, 0, 0, 0, 0, 0, 3264, 2621, 3695, 0, 0, 0, 4084, 0, 0, 12, 0, 0, 0, 0, 40, 1512, 4062, 4090, 0, 4025, 4088, 0, 1, 224, 0, 0, 5, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 3645, 0, 0, 0, 0, 2162, 4096, 19, 4096, 4093, 0, 0, 11, 0, 1, 0, 0, 3482, 0, 0, 3425, 129, 4093, 0, 22, 0, 2789, 76, 108, 0, 8, 920, 3, 0, 0, 0, 4096, 4089, 0, 1, 0, 881, 0, 0, 0, 0, 0, 16, 1, 0, 4091, 4096, 3994, 1, 4092, 4069, 6, 24, 2752, 0, 0, 145, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2485, 0, 0, 0, 0, 3671, 3774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2022, 0, 24, 0, 0, 258, 3105, 1098, 0, 0, 0, 0, 0, 0, 145, 0, 4080, 0, 0, 3270, 0, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 169, 0, 0, 4096, 626, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 3186, 4090, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 85, 0, 2273, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3955, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 641, 0, 0, 49, 0, 1320, 916, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9930, 6507, 8696, 8989, 7668, 8026, 8698, 7022] avg=8192.0000 max=9930 max_rank=0 min=6507 min_rank=1 imbalance_ratio=1.212158 max_load_reduction=55606 max_load_reduction_pct=0.848480 moved_to_echo=31414 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 2510, 0, 0, 0, 0, 1058, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 375, 0, 4, 0, 0, 0, 3105, 1071, 0, 0, 0, 0, 0, 0, 582, 0, 3697, 0, 0, 3066, 1036, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4095, 626, 0, 0, 0, 0, 0, 0, 3657, 0, 0, 0, 3785, 4089, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4029, 6, 0, 0, 0, 0, 2217, 1538, 0, 0, 0, 0, 0, 0, 0, 3, 12, 4093, 4096, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 15, 0, 6, 67, 0, 1290, 926, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7543, 4713, 9554, 8818, 11896, 7797, 8834, 6381] avg=8192.0000 max=11896 max_rank=4 min=4713 min_rank=1 imbalance_ratio=1.452148 max_load_reduction=53640 max_load_reduction_pct=0.818481 moved_to_echo=34908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2861, 0, 50, 77, 0, 3197, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4012, 26, 0, 15, 6, 0, 2947, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 0, 168, 0, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 179, 59, 596, 0, 0, 4096, 594, 0, 0, 0, 0, 0, 0, 4096, 0, 37, 0, 735, 4061, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 85, 3437, 0, 0, 0, 0, 167, 2308, 1575, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4095, 4087, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4032, 10, 2, 0, 51, 1320, 1055, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10268, 8124, 5353, 5525, 9276, 7580, 8843, 10567] avg=8192.0000 max=10567 max_rank=7 min=5353 min_rank=2 imbalance_ratio=1.289917 max_load_reduction=54969 max_load_reduction_pct=0.838760 moved_to_echo=36266 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[92, 2473, 58, 0, 0, 17, 4096, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3465, 44, 376, 0, 0, 0, 2801, 1118, 0, 0, 0, 0, 0, 0, 3663, 0, 3345, 0, 0, 3051, 501, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3918, 53, 497, 2153, 0, 4025, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 2621, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 12, 0, 0, 2314, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 40, 1512, 4062, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 1, 224, 0, 0, 5, 2, 1304, 912, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7778, 7804, 11747, 11219, 6215, 7965, 6272, 6536] avg=8192.0000 max=11747 max_rank=2 min=6215 min_rank=4 imbalance_ratio=1.433960 max_load_reduction=53789 max_load_reduction_pct=0.820755 moved_to_echo=26988 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2992, 405, 0, 14, 0, 16, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 3030, 991, 0, 0, 0, 0, 0, 0, 1370, 0, 232, 5, 0, 4093, 167, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 1, 164, 453, 105, 4096, 611, 0, 0, 0, 0, 0, 0, 2112, 40, 242, 0, 4, 4018, 357, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 3900, 0, 54, 5, 1813, 40, 2051, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 83, 3822, 2917, 582, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 10, 1, 3403, 0, 68, 0, 485, 1314, 1104, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7452, 4751, 7062, 9527, 6781, 12093, 7452, 10418] avg=8192.0000 max=12093 max_rank=5 min=4751 min_rank=1 imbalance_ratio=1.476196 max_load_reduction=53443 max_load_reduction_pct=0.815475 moved_to_echo=36744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2992, 0, 0, 0, 0, 3920, 2527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1690, 4067, 0, 73, 0, 0, 0, 3105, 1120, 0, 0, 0, 0, 0, 0, 8, 3, 54, 0, 0, 4096, 61, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 25, 32, 0, 4096, 626, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3, 1079, 4096, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1206, 4093, 0, 0, 0, 149, 0, 2318, 1575, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 3673, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1773, 0, 0, 0, 0, 1320, 1104, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9439, 10055, 5423, 4918, 9637, 9341, 8430, 8293] avg=8192.0000 max=10055 max_rank=1 min=4918 min_rank=3 imbalance_ratio=1.227417 max_load_reduction=55481 max_load_reduction_pct=0.846573 moved_to_echo=35701 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2517, 33, 0, 0, 0, 9, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4094, 0, 2330, 0, 485, 0, 3099, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 118, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 625, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 4059, 4087, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 57, 16, 2310, 1527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3415, 4096, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 718, 0, 1, 0, 15, 1281, 928, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4208, 11164, 9474, 8817, 8685, 8011, 8168, 7009] avg=8192.0000 max=11164 max_rank=1 min=4208 min_rank=0 imbalance_ratio=1.362793 max_load_reduction=54372 max_load_reduction_pct=0.829651 moved_to_echo=31873 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2660, 0, 0, 0, 0, 2162, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 4093, 0, 0, 11, 0, 1, 3102, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 3482, 0, 0, 3425, 129, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 2789, 76, 108, 0, 8, 4092, 628, 0, 0, 0, 0, 0, 0, 920, 3, 0, 0, 0, 4096, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 881, 0, 0, 0, 0, 0, 2264, 1577, 0, 0, 0, 0, 0, 0, 16, 1, 0, 4091, 4096, 634, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 6, 24, 2752, 0, 0, 145, 4, 1325, 985, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8918, 12418, 8227, 7723, 5378, 4723, 8839, 9310] avg=8192.0000 max=12418 max_rank=1 min=4723 min_rank=5 imbalance_ratio=1.515869 max_load_reduction=53118 max_load_reduction_pct=0.810516 moved_to_echo=28250 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 21504, 512, 128, 128, 128, 18176, 25216, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 6016, 22144, 128, 2816, 128, 512, 1024, 24320, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 256, 29312, 31104, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 23424, 256, 1664, 2688, 128, 32768, 4992, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5888, 128, 23040, 128, 0, 25344, 2048, 9600, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[15104, 128, 384, 128, 16128, 31232, 2944, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 13568, 128, 128, 384, 640, 10496, 7936, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 3968, 28672, 128, 128, 128, 2176, 256, 18176, 12544, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[40, 0, 4096, 1, 0, 0, 4096, 0, 0, 58, 0, 0, 0, 0, 1, 0, 4037, 4096, 0, 0, 82, 3454, 4096, 63, 4032, 0, 1416, 0, 0, 4096, 204, 4096, 0, 0, 0, 4023, 0, 4096, 0, 0, 0, 5, 0, 27, 19, 47, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 3317, 3457, 307, 0, 0, 4094, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1510, 0, 4088, 19, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 1, 1, 3816, 4090, 0, 0, 0, 363, 4094, 0, 3204, 1, 1770, 0, 0, 4070, 1, 3983, 3883, 0, 0, 2613, 0, 4096, 0, 3, 0, 136, 0, 7, 6, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 30, 0, 3995, 3424, 0, 10, 4096, 0, 4039, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3726, 0, 4039, 233, 43, 0, 4096, 0, 0, 2, 0, 0, 70, 0, 0, 0, 13, 3963, 0, 0, 1780, 4087, 4095, 121, 4045, 1737, 313, 0, 0, 4074, 0, 2762, 4062, 5, 0, 3797, 15, 2954, 17, 0, 0, 122, 25, 0, 117, 0, 0, 4076, 1, 0, 10, 0, 0, 8, 0, 2337, 6, 558, 0, 0, 4096, 125, 4006, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1077, 245, 3796, 3104, 0, 1, 4038, 61, 0, 3745, 0, 1, 0, 0, 0, 14, 4096, 4096, 0, 302, 0, 3368, 4088, 0, 4092, 0, 5, 0, 0, 3595, 19, 4096, 0, 0, 0, 2621, 28, 3844, 71, 5, 0, 14, 0, 410, 0, 630, 0, 4071, 0, 143, 0, 48, 7, 0, 44, 0, 2170, 1, 0, 1, 3672, 1, 3916, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[722, 0, 4096, 2557, 4, 3, 4091, 0, 0, 0, 11, 0, 0, 0, 0, 0, 277, 4082, 734, 0, 1034, 0, 4075, 4094, 3487, 1, 3700, 0, 0, 4049, 0, 3158, 0, 0, 0, 4095, 0, 3823, 0, 0, 0, 0, 0, 3120, 113, 543, 0, 4096, 0, 28, 0, 0, 0, 1, 0, 545, 79, 613, 116, 0, 4095, 0, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[450, 0, 4094, 21, 247, 2, 4059, 13, 0, 2679, 3, 4, 0, 406, 0, 0, 690, 4096, 3158, 0, 202, 35, 2807, 3397, 3191, 73, 0, 0, 0, 4072, 0, 4091, 0, 16, 0, 930, 28, 4049, 0, 0, 0, 29, 0, 0, 0, 4095, 0, 3413, 1, 4, 0, 0, 45, 0, 0, 4096, 2455, 2, 0, 483, 4007, 3, 4087, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 95, 0, 0, 4095, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1366, 4096, 1, 0, 8, 3856, 4095, 84, 4096, 16, 0, 0, 0, 4096, 7, 4096, 4095, 0, 0, 4096, 0, 4092, 0, 0, 0, 2, 0, 193, 0, 276, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2741, 235, 3393, 0, 22, 4094, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[683, 1, 4083, 0, 0, 0, 4073, 0, 0, 478, 0, 28, 233, 0, 752, 9, 2123, 4087, 0, 0, 3, 54, 2611, 7, 3432, 0, 978, 0, 12, 4025, 1, 4085, 0, 0, 6, 3865, 5, 4095, 0, 0, 0, 0, 0, 3198, 0, 320, 5, 4067, 19, 0, 0, 2, 0, 0, 18, 3437, 2945, 3608, 0, 2, 4088, 23, 4075, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[40, 0, 4096, 1, 0, 0, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 1, 0, 4096, 3120, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 82, 3454, 2575, 63, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 1416, 0, 0, 0, 204, 3467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 3596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 27, 19, 47, 0, 4084, 1521, 629, 500, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3317, 3432, 1738, 976, 0, 0, 0, 0, 0, 3457, 307, 0, 0, 4094, 0, 664, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6433, 7275, 10211, 9119, 7619, 6894, 9463, 8522] avg=8192.0000 max=10211 max_rank=2 min=6433 min_rank=0 imbalance_ratio=1.246460 max_load_reduction=55325 max_load_reduction_pct=0.844193 moved_to_echo=32498 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1510, 0, 4088, 19, 0, 0, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 1, 4090, 3099, 0, 0, 0, 0, 0, 0, 3816, 0, 0, 0, 0, 363, 2574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3204, 1, 1770, 0, 0, 0, 1, 3372, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 0, 0, 2613, 0, 3596, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 7, 6, 0, 0, 4086, 1520, 611, 500, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 3384, 1737, 971, 0, 0, 0, 0, 0, 3995, 3424, 0, 10, 4096, 0, 655, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7913, 7193, 6753, 8348, 10095, 6928, 6122, 12184] avg=8192.0000 max=12184 max_rank=7 min=6122 min_rank=6 imbalance_ratio=1.487305 max_load_reduction=53352 max_load_reduction_pct=0.814087 moved_to_echo=35329 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3726, 0, 4039, 233, 43, 0, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 70, 0, 0, 0, 3963, 3102, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 1780, 4087, 2574, 121, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 1737, 313, 0, 0, 0, 0, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 5, 0, 3797, 15, 2594, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 25, 0, 117, 0, 0, 4076, 1521, 424, 360, 62, 0, 0, 0, 0, 1, 0, 10, 0, 0, 8, 0, 2337, 3356, 1738, 972, 0, 0, 0, 0, 0, 6, 558, 0, 0, 4096, 125, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10337, 7137, 8575, 8433, 10490, 6707, 8422, 5435] avg=8192.0000 max=10490 max_rank=4 min=5435 min_rank=7 imbalance_ratio=1.280518 max_load_reduction=55046 max_load_reduction_pct=0.839935 moved_to_echo=31054 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 95, 0, 0, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 4096, 3119, 0, 0, 0, 0, 0, 0, 1366, 0, 1, 0, 8, 3856, 2574, 84, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 7, 3467, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 193, 0, 276, 0, 4096, 1521, 629, 499, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2741, 3432, 1737, 977, 0, 0, 0, 0, 0, 235, 3393, 0, 22, 4094, 0, 664, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6487, 7217, 7889, 7586, 11784, 7278, 8887, 8408] avg=8192.0000 max=11784 max_rank=4 min=6487 min_rank=0 imbalance_ratio=1.438477 max_load_reduction=53752 max_load_reduction_pct=0.820190 moved_to_echo=36357 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1077, 245, 3796, 3104, 0, 1, 2263, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3745, 0, 1, 0, 0, 0, 14, 4096, 2738, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 302, 0, 3368, 2570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 5, 0, 0, 0, 19, 3467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2621, 28, 3375, 71, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 410, 0, 630, 0, 4071, 1518, 629, 469, 62, 0, 0, 0, 0, 0, 143, 0, 48, 7, 0, 44, 0, 3281, 1713, 857, 0, 0, 0, 0, 0, 2170, 1, 0, 1, 3672, 1, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10547, 10594, 10336, 7583, 6100, 7803, 6093, 6480] avg=8192.0000 max=10594 max_rank=1 min=6093 min_rank=6 imbalance_ratio=1.293213 max_load_reduction=54942 max_load_reduction_pct=0.838348 moved_to_echo=26476 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[450, 0, 4094, 21, 247, 2, 2275, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2679, 3, 4, 0, 406, 0, 0, 4096, 3102, 0, 0, 0, 0, 0, 0, 690, 0, 3158, 0, 202, 35, 1765, 3397, 0, 0, 0, 0, 0, 0, 0, 0, 3191, 73, 0, 0, 0, 0, 0, 3463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 930, 28, 3555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 4095, 0, 3413, 1042, 628, 494, 62, 0, 0, 0, 0, 1, 4, 0, 0, 45, 0, 0, 4096, 3424, 1722, 970, 0, 0, 0, 0, 0, 2455, 2, 0, 483, 4007, 3, 663, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7102, 10290, 9247, 6727, 4529, 9763, 10262, 7616] avg=8192.0000 max=10290 max_rank=1 min=4529 min_rank=4 imbalance_ratio=1.256104 max_load_reduction=55246 max_load_reduction_pct=0.842987 moved_to_echo=32170 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[722, 0, 4096, 2557, 4, 3, 2294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4082, 3084, 0, 0, 0, 0, 0, 0, 277, 0, 734, 0, 1034, 0, 2562, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3487, 1, 3700, 0, 0, 0, 0, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3120, 113, 543, 0, 4096, 1513, 485, 466, 62, 0, 0, 0, 0, 0, 28, 0, 0, 0, 1, 0, 545, 3430, 1735, 965, 0, 0, 0, 0, 0, 79, 613, 116, 0, 4095, 0, 664, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9676, 7177, 8701, 9861, 7452, 10398, 6704, 5567] avg=8192.0000 max=10398 max_rank=5 min=5567 min_rank=7 imbalance_ratio=1.269287 max_load_reduction=55138 max_load_reduction_pct=0.841339 moved_to_echo=30121 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[683, 1, 4083, 0, 0, 0, 2274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 28, 233, 0, 752, 9, 4087, 3066, 0, 0, 0, 0, 0, 0, 2123, 0, 0, 0, 3, 54, 1637, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3432, 0, 978, 0, 12, 0, 1, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3865, 5, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3198, 0, 320, 5, 4067, 974, 629, 504, 68, 0, 0, 0, 0, 19, 0, 0, 2, 0, 0, 18, 3437, 3417, 1731, 959, 0, 0, 0, 0, 0, 2945, 3608, 0, 2, 4088, 23, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7041, 8653, 3824, 7879, 7467, 9765, 9583, 11324] avg=8192.0000 max=11324 max_rank=7 min=3824 min_rank=2 imbalance_ratio=1.382324 max_load_reduction=54212 max_load_reduction_pct=0.827209 moved_to_echo=38139 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 256, 32512, 6144, 384, 128, 18304, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[16512, 0, 3968, 384, 3200, 15232, 18944, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 7040, 128, 128, 384, 512, 768, 128, 32640, 24448, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12160, 128, 128, 26112, 128, 27264, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[15360, 12032, 128, 640, 32256, 256, 5376, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[29696, 1920, 8192, 0, 128, 0, 256, 25728, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 256, 128, 128, 128, 128, 128, 16512, 27264, 13952, 7680, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 384, 128, 7040, 256, 6016, 128, 32000, 11136, 4736, 3840, 512, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 51, 0, 0, 2165, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 6, 0, 0, 0, 2, 4093, 1876, 3783, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4056, 0, 4096, 0, 0, 54, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4078, 7, 0, 30, 4084, 0, 3, 0, 4096, 0, 4026, 4061, 3197, 54, 1110, 21, 21, 16, 0, 8, 0, 60, 0, 0, 0, 0, 0, 27, 0, 0, 34, 3654, 99, 4096, 57, 4082, 0, 0, 0, 1766, 1186, 1, 184, 0, 0, 1, 1079, 3999, 3, 0, 1362, 3994, 0, 3164, 0, 4060, 83, 765, 2824, 0, 4047, 97, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 23, 0, 3039, 0, 0, 6, 4096, 0, 4081, 4094, 4092, 241, 14, 0, 0, 0, 0, 0, 0, 271, 7, 1, 0, 0, 0, 2, 0, 0, 0, 4095, 117, 4096, 0, 1467, 0, 0, 0, 1, 4088, 289, 3806, 0, 0, 0, 0, 3363, 1, 0, 4094, 4096, 0, 3779, 0, 4091, 0, 3, 5, 1, 4079, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 3264, 0, 0, 337, 0, 7, 0, 272, 0, 3883, 4060, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 1957, 15, 35, 57, 59, 0, 0, 0, 0, 4086, 0, 4051, 68, 38, 0, 96, 0, 0, 2605, 40, 4096, 1, 0, 0, 19, 4046, 0, 0, 3236, 4096, 21, 3795, 432, 4016, 0, 40, 432, 2, 4084, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 3639, 2, 0, 0, 21, 0, 0, 580, 4096, 0, 4095, 4095, 4093, 0, 141, 2011, 0, 0, 0, 13, 0, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1305, 1, 4096, 0, 0, 0, 0, 0, 3796, 4015, 2, 4096, 0, 0, 0, 216, 3778, 0, 0, 4096, 3262, 0, 4001, 0, 4096, 0, 4, 64, 0, 4095, 582, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4073, 2509, 0, 0, 3487, 0, 0, 0, 3730, 0, 3871, 4081, 4061, 0, 29, 0, 2, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4049, 134, 3927, 0, 885, 0, 0, 0, 3802, 2545, 716, 2299, 0, 0, 0, 69, 3348, 0, 16, 3956, 4071, 0, 1467, 0, 4095, 0, 4, 18, 0, 4087, 48, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 77, 0, 0, 1232, 0, 0, 0, 0, 0, 4072, 4096, 4096, 4, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 15, 0, 0, 0, 0, 0, 4096, 3059, 4096, 0, 0, 0, 0, 0, 4091, 3374, 59, 4096, 0, 0, 0, 0, 4092, 0, 0, 4096, 4096, 0, 4090, 0, 4096, 4, 6, 0, 0, 4096, 395, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[6, 4083, 381, 0, 5, 4096, 0, 0, 1, 3, 1, 4067, 4051, 4086, 0, 0, 0, 1, 0, 9, 925, 0, 84, 0, 97, 0, 938, 0, 3, 0, 44, 13, 4095, 8, 3943, 0, 0, 1, 453, 1, 2942, 4095, 0, 3070, 8, 6, 0, 0, 4091, 0, 0, 3538, 4087, 1, 4095, 0, 4081, 0, 23, 0, 14, 4090, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 51, 0, 0, 2165, 0, 0, 656, 505, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1386, 0, 4096, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3013, 2710, 872, 545, 37, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 0, 0, 0, 0, 4096, 0, 3220, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3043, 1876, 3783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 0, 0, 4096, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 54, 0, 3440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7473, 9840, 7177, 8195, 7322, 8704, 9235, 7590] avg=8192.0000 max=9840 max_rank=1 min=7177 min_rank=2 imbalance_ratio=1.201172 max_load_reduction=55696 max_load_reduction_pct=0.849854 moved_to_echo=32851 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 23, 0, 3039, 0, 0, 654, 505, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 1381, 0, 4092, 241, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 7, 2474, 2700, 873, 544, 36, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 4094, 4060, 0, 0, 0, 0, 0, 0, 4095, 117, 3223, 0, 1467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3039, 289, 3806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 889, 1, 0, 4094, 0, 0, 3779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3, 5, 1, 3425, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 9830, 6905, 8157, 8902, 7135, 8763, 7527] avg=8192.0000 max=9830 max_rank=1 min=6905 min_rank=2 imbalance_ratio=1.199951 max_load_reduction=55706 max_load_reduction_pct=0.850006 moved_to_echo=32327 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4073, 2509, 0, 0, 3487, 0, 0, 655, 314, 0, 0, 0, 0, 0, 0, 0, 3730, 0, 1310, 0, 4061, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 147, 0, 2463, 2561, 837, 339, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4081, 4034, 0, 0, 0, 0, 0, 0, 4049, 134, 3090, 0, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3802, 1892, 716, 2299, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 885, 0, 16, 3956, 0, 0, 1467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 18, 0, 3432, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11038, 9130, 6386, 8125, 8158, 8778, 6324, 7597] avg=8192.0000 max=11038 max_rank=0 min=6324 min_rank=6 imbalance_ratio=1.347412 max_load_reduction=54498 max_load_reduction_pct=0.831573 moved_to_echo=30857 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3264, 0, 0, 337, 0, 7, 654, 321, 0, 0, 0, 0, 0, 0, 0, 272, 0, 1314, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 1957, 2977, 2569, 863, 347, 37, 0, 0, 0, 15, 35, 57, 59, 0, 0, 0, 0, 4060, 4059, 0, 0, 0, 0, 0, 0, 4086, 0, 3188, 68, 38, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1937, 40, 4096, 1, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 0, 0, 3236, 0, 21, 3795, 432, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 40, 432, 2, 3430, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8679, 5682, 12847, 8285, 7476, 6093, 8553, 7921] avg=8192.0000 max=12847 max_rank=2 min=5682 min_rank=1 imbalance_ratio=1.568237 max_load_reduction=52689 max_load_reduction_pct=0.803970 moved_to_echo=30043 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4078, 7, 0, 30, 4084, 0, 3, 649, 146, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1362, 0, 3197, 54, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 21, 21, 16, 0, 8, 0, 60, 0, 2942, 2664, 873, 158, 36, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 34, 4061, 3958, 0, 0, 0, 0, 0, 0, 3654, 99, 3223, 57, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1766, 882, 1, 184, 0, 0, 1, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 3, 0, 1362, 0, 0, 3164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 83, 765, 2824, 0, 3398, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8997, 9819, 6799, 8080, 11115, 3913, 5586, 11227] avg=8192.0000 max=11227 max_rank=7 min=3913 min_rank=5 imbalance_ratio=1.370483 max_load_reduction=54309 max_load_reduction_pct=0.828690 moved_to_echo=31841 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 77, 0, 0, 1232, 0, 0, 656, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1378, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 3010, 2694, 873, 449, 36, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 4096, 4060, 0, 0, 0, 0, 0, 0, 4096, 3059, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2509, 59, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 6, 0, 0, 3440, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6477, 5478, 7068, 8171, 10378, 10755, 9268, 7941] avg=8192.0000 max=10755 max_rank=5 min=5478 min_rank=1 imbalance_ratio=1.312866 max_load_reduction=54781 max_load_reduction_pct=0.835892 moved_to_echo=38342 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3639, 2, 0, 0, 21, 0, 0, 656, 496, 0, 0, 0, 0, 0, 0, 580, 4096, 0, 1385, 0, 4093, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 2011, 0, 0, 0, 13, 0, 1245, 0, 2779, 2710, 873, 534, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3232, 0, 0, 0, 0, 0, 0, 1305, 1, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 2985, 2, 4096, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 999, 0, 0, 4096, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 64, 0, 3439, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4814, 10295, 10195, 7327, 4529, 11095, 9096, 8185] avg=8192.0000 max=11095 max_rank=5 min=4529 min_rank=4 imbalance_ratio=1.354370 max_load_reduction=54441 max_load_reduction_pct=0.830704 moved_to_echo=32905 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[6, 4083, 381, 0, 5, 4096, 0, 0, 660, 510, 0, 0, 0, 0, 0, 0, 1, 3, 1, 1371, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 925, 0, 84, 0, 3014, 2696, 845, 548, 37, 0, 0, 0, 97, 0, 938, 0, 3, 0, 44, 13, 4051, 4050, 0, 0, 0, 0, 0, 0, 4095, 8, 3098, 0, 0, 1, 453, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2942, 3037, 0, 3070, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 0, 0, 3538, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 23, 0, 14, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9741, 5462, 8159, 9196, 7656, 9063, 8711, 7548] avg=8192.0000 max=9741 max_rank=0 min=5462 min_rank=1 imbalance_ratio=1.189087 max_load_reduction=55795 max_load_reduction_pct=0.851364 moved_to_echo=32978 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[2048, 128, 128, 128, 1024, 0, 6016, 2048, 22784, 21376, 6912, 3584, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[16512, 19328, 3072, 25472, 128, 128, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 20480, 128, 11008, 0, 31872, 640, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[29568, 3456, 25600, 128, 6528, 128, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 128, 896, 3456, 128, 27520, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32384, 6400, 128, 128, 18560, 0, 128, 5248, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 128, 128, 28544, 0, 128, 28544, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 1152, 128, 128, 0, 128, 128, 32640, 31616, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[2475, 0, 0, 0, 4095, 1353, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 50, 4083, 0, 789, 0, 4026, 4093, 0, 151, 1253, 0, 3, 2, 0, 0, 0, 0, 4096, 0, 4069, 1, 0, 0, 0, 26, 522, 0, 0, 0, 0, 0, 0, 0, 2766, 2, 4093, 0, 4074, 0, 3984, 13, 4095, 10, 0, 31, 3113, 0, 4074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[935, 0, 0, 0, 1984, 4093, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 11, 0, 1877, 4096, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 3116, 4096, 0, 1, 42, 0, 0, 0, 1, 454, 0, 4048, 0, 4096, 1889, 3227, 0, 3882, 3727, 0, 0, 3396, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1981, 0, 0, 0, 4094, 1090, 4096, 156, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 2, 0, 4057, 4093, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 130, 3291, 0, 0, 0, 0, 0, 0, 62, 76, 0, 4096, 1, 4096, 2394, 4096, 0, 3922, 3390, 0, 4, 2, 0, 3929] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[3985, 0, 168, 0, 3957, 4080, 3701, 0, 4059, 0, 109, 0, 0, 29, 0, 0, 65, 3836, 0, 348, 0, 4062, 4096, 0, 0, 1, 20, 0, 310, 1, 1, 37, 0, 3691, 37, 3985, 0, 11, 0, 0, 3518, 0, 0, 635, 0, 6, 0, 0, 17, 3067, 143, 467, 16, 4096, 707, 4090, 0, 4096, 5, 0, 9, 0, 0, 4075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[3962, 0, 0, 0, 4092, 2699, 4096, 126, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 424, 0, 4081, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 21, 8, 4057, 0, 4096, 0, 4096, 0, 4096, 964, 0, 0, 66, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[3813, 3, 3, 29, 3887, 343, 4037, 3868, 4056, 6, 0, 0, 31, 0, 33, 43, 90, 4094, 0, 3122, 0, 4015, 3912, 0, 369, 735, 2, 8, 72, 86, 185, 1, 0, 4051, 14, 4057, 0, 0, 3, 0, 149, 145, 29, 235, 0, 0, 3, 0, 0, 8, 0, 4096, 163, 3699, 0, 4089, 0, 3940, 327, 0, 265, 29, 0, 3391] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[3751, 0, 0, 0, 4096, 2493, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 389, 1, 1248, 0, 0, 32, 4074, 3941, 0, 886, 0, 1, 0, 0, 0, 1, 2, 0, 4065, 0, 3364, 0, 0, 2, 0, 0, 1103, 26, 5, 14, 0, 0, 1, 487, 187, 1, 4090, 0, 4093, 3997, 3671, 0, 4076, 5, 0, 153, 3006, 0, 4083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4086, 0, 0, 1, 3884, 0, 4058, 413, 4089, 0, 0, 0, 0, 2, 0, 321, 2, 3744, 0, 0, 0, 3152, 4096, 0, 0, 498, 9, 21, 0, 4, 0, 0, 0, 4073, 5, 4095, 1, 0, 2, 0, 0, 654, 32, 0, 6, 1, 0, 0, 491, 1255, 0, 3120, 0, 4094, 3006, 4092, 0, 4073, 0, 4, 3, 4058, 0, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2475, 0, 0, 0, 2672, 1353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 1918, 1500, 681, 0, 0, 0, 0, 0, 50, 4083, 0, 789, 0, 4026, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 1253, 0, 3, 2, 0, 0, 0, 4074, 3593, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4069, 1, 0, 0, 0, 742, 0, 0, 0, 0, 0, 0, 0, 26, 522, 0, 0, 0, 0, 0, 0, 3841, 1179, 503, 0, 0, 0, 0, 0, 0, 2766, 2, 4093, 0, 0, 0, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1416, 10, 0, 31, 3113, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6500, 8197, 9200, 9076, 8908, 6071, 8927, 8657] avg=8192.0000 max=9200 max_rank=2 min=6071 min_rank=5 imbalance_ratio=1.123047 max_load_reduction=56336 max_load_reduction_pct=0.859619 moved_to_echo=32563 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1981, 0, 0, 0, 2672, 1090, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1972, 1436, 680, 0, 0, 0, 0, 0, 0, 4080, 0, 2, 0, 4057, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 4096, 3594, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 742, 0, 0, 0, 0, 0, 0, 0, 130, 3291, 0, 0, 0, 0, 0, 0, 3841, 1129, 502, 0, 0, 0, 0, 0, 62, 76, 0, 4096, 1, 0, 2394, 2124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1357, 3390, 0, 4, 2, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5899, 8184, 8391, 7799, 8935, 8893, 8753, 8682] avg=8192.0000 max=8935 max_rank=4 min=5899 min_rank=0 imbalance_ratio=1.090698 max_load_reduction=56601 max_load_reduction_pct=0.863663 moved_to_echo=35263 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3751, 0, 0, 0, 2673, 2493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 389, 1767, 1493, 681, 0, 0, 0, 0, 0, 1, 1248, 0, 0, 32, 4074, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 0, 1, 0, 0, 0, 1, 2, 4093, 3594, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 3364, 0, 0, 2, 0, 742, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 26, 5, 14, 0, 0, 1, 3698, 1174, 502, 0, 0, 0, 0, 0, 487, 187, 1, 4090, 0, 0, 3997, 1904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 5, 0, 153, 3006, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8917, 8426, 5598, 8577, 8173, 6523, 10666, 8656] avg=8192.0000 max=10666 max_rank=6 min=5598 min_rank=2 imbalance_ratio=1.302002 max_load_reduction=54870 max_load_reduction_pct=0.837250 moved_to_echo=34018 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3962, 0, 0, 0, 2671, 2699, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1972, 1500, 680, 0, 0, 0, 0, 0, 0, 4096, 0, 424, 0, 4081, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 3593, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 4095, 0, 0, 0, 0, 741, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3843, 1179, 503, 0, 0, 0, 0, 0, 0, 21, 8, 4057, 0, 0, 0, 2124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 964, 0, 0, 66, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9458, 8248, 8854, 7699, 8903, 9621, 6210, 6543] avg=8192.0000 max=9621 max_rank=5 min=6210 min_rank=6 imbalance_ratio=1.174438 max_load_reduction=55915 max_load_reduction_pct=0.853195 moved_to_echo=31277 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3985, 0, 168, 0, 2582, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 109, 0, 0, 29, 0, 0, 1969, 1500, 658, 0, 0, 0, 0, 0, 65, 3836, 0, 348, 0, 4062, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 0, 310, 1, 1, 37, 4096, 3248, 0, 0, 0, 0, 0, 0, 0, 3691, 37, 3985, 0, 11, 0, 0, 717, 0, 0, 0, 0, 0, 0, 0, 3518, 0, 0, 635, 0, 6, 0, 0, 3843, 1179, 453, 0, 0, 0, 0, 0, 17, 3067, 143, 467, 16, 0, 707, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 5, 0, 9, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10815, 8324, 8564, 7714, 8441, 9634, 6538, 5506] avg=8192.0000 max=10815 max_rank=0 min=5506 min_rank=7 imbalance_ratio=1.320190 max_load_reduction=54721 max_load_reduction_pct=0.834976 moved_to_echo=30119 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[935, 0, 0, 0, 1295, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1554, 1422, 330, 0, 0, 0, 0, 0, 0, 4080, 0, 11, 0, 1877, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 4096, 3594, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 3116, 4096, 0, 1, 42, 0, 0, 0, 3843, 1118, 502, 0, 0, 0, 0, 0, 1, 454, 0, 4048, 0, 0, 1889, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1342, 3727, 0, 0, 3396, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6323, 7400, 6221, 7784, 4464, 12718, 8065, 12561] avg=8192.0000 max=12718 max_rank=5 min=4464 min_rank=4 imbalance_ratio=1.552490 max_load_reduction=52818 max_load_reduction_pct=0.805939 moved_to_echo=37808 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3813, 3, 3, 29, 2537, 343, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 6, 0, 0, 31, 0, 33, 43, 1969, 1443, 646, 0, 0, 0, 0, 0, 90, 4094, 0, 3122, 0, 4015, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 735, 2, 8, 72, 86, 185, 1, 3699, 3543, 0, 0, 0, 0, 0, 0, 0, 4051, 14, 4057, 0, 0, 3, 0, 704, 0, 0, 0, 0, 0, 0, 0, 149, 145, 29, 235, 0, 0, 3, 0, 3671, 1134, 494, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 163, 0, 0, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1363, 327, 0, 265, 29, 0, 3391, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10596, 8227, 11562, 8700, 8829, 5860, 6387, 5375] avg=8192.0000 max=11562 max_rank=2 min=5375 min_rank=7 imbalance_ratio=1.411377 max_load_reduction=53974 max_load_reduction_pct=0.823578 moved_to_echo=26451 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 0, 1, 2528, 0, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 2, 0, 321, 1974, 1495, 649, 0, 0, 0, 0, 0, 2, 3744, 0, 0, 0, 3152, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 9, 21, 0, 4, 0, 0, 4094, 3561, 0, 0, 0, 0, 0, 0, 0, 4073, 5, 4095, 1, 0, 2, 0, 707, 0, 0, 0, 0, 0, 0, 0, 0, 654, 32, 0, 6, 1, 0, 0, 3848, 1178, 497, 0, 0, 0, 0, 0, 491, 1255, 0, 3120, 0, 0, 3006, 2118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1400, 0, 4, 3, 4058, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7028, 8530, 7146, 8187, 8883, 6216, 9990, 9556] avg=8192.0000 max=9990 max_rank=6 min=6216 min_rank=5 imbalance_ratio=1.219482 max_load_reduction=55546 max_load_reduction_pct=0.847565 moved_to_echo=34645 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[25088, 128, 256, 128, 19712, 16256, 0, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1152, 7936, 256, 28160, 256, 0, 12032, 16256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 11136, 8448, 128, 512, 13696, 0, 31872, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 28160, 128, 31872, 128, 128, 128, 128, 5504, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 29312, 0, 4736, 128, 29440, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1536, 2560, 128, 128, 384, 128, 384, 128, 32384, 28416, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[7040, 13952, 128, 896, 128, 128, 128, 128, 30464, 9344, 3968, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 128, 768, 15104, 11904, 5120, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3996, 17, 2858, 4073, 3433, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 872, 4073, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 4, 0, 0, 119, 0, 3971, 22, 801, 4096, 0, 0, 4096, 4090, 0, 3021, 1342, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 75, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 4058, 0, 30, 3929, 3989, 0, 4085, 0, 1, 0, 0, 179, 90, 0, 0, 685, 4094, 204, 0, 3961, 0, 0, 0, 42, 0, 0, 0, 4096, 17, 4095, 1, 0, 0, 0, 0, 54, 0, 4095, 0, 0, 4094, 0, 0, 4077, 4096, 0, 3487, 3874, 0, 4096, 0, 0, 4089, 0, 0, 1, 1, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 16, 74, 0, 822, 0, 1631, 3881, 4094, 0, 3737, 0, 0, 4, 153, 209, 4076, 39, 0, 108, 3930, 260, 0, 3902, 0, 34, 4, 0, 0, 0, 0, 4052, 77, 4095, 0, 0, 0, 0, 1, 3796, 11, 1802, 0, 325, 4092, 0, 0, 3993, 4073, 201, 3, 3933, 6, 4035, 1, 1, 4036, 0, 0, 6, 0, 0, 0, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4075, 46, 4096, 4052, 2500, 356, 4065, 0, 0, 0, 0, 1798, 66, 20, 28, 4096, 3457, 0, 0, 4039, 0, 0, 0, 1, 36, 0, 66, 4060, 0, 3968, 0, 144, 0, 0, 0, 0, 2, 3925, 0, 1635, 4084, 0, 0, 2626, 4090, 46, 32, 0, 1, 4004, 4, 0, 4096, 0, 0, 2, 0, 20, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3134, 0, 4096, 4096, 0, 427, 3893, 0, 0, 0, 1, 0, 0, 0, 0, 4075, 4057, 2, 0, 4093, 0, 0, 0, 599, 0, 117, 0, 4096, 0, 4094, 0, 0, 0, 6, 0, 0, 0, 4096, 24, 0, 4084, 2, 0, 4059, 4096, 0, 70, 3626, 0, 4094, 0, 0, 3996, 224, 0, 58, 0, 301, 19, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 3909, 5, 4096, 4096, 349, 1, 4093, 0, 0, 0, 2, 104, 252, 0, 0, 3475, 4096, 75, 0, 4095, 0, 0, 0, 0, 0, 8, 18, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 464, 4096, 0, 0, 4095, 4096, 0, 152, 3457, 0, 4096, 0, 0, 4096, 1, 1, 0, 0, 10, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1146, 2, 5, 4096, 3789, 540, 116, 0, 1, 0, 3, 2991, 3, 0, 0, 3843, 4063, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 15, 4087, 0, 4096, 0, 41, 0, 0, 0, 9, 16, 3941, 0, 0, 4096, 0, 1, 4087, 4042, 0, 3962, 4071, 0, 4096, 37, 0, 4096, 0, 0, 0, 0, 143, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 184, 60, 4056, 3716, 7, 5, 4096, 14, 0, 217, 1, 0, 2788, 0, 0, 39, 4093, 169, 0, 3869, 10, 0, 0, 0, 24, 6, 0, 4087, 0, 4075, 4, 0, 216, 0, 13, 876, 0, 4096, 0, 27, 4084, 0, 0, 4053, 3978, 57, 1408, 4096, 0, 4071, 0, 0, 4094, 0, 0, 0, 4, 2834, 0, 107] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3996, 17, 2858, 2947, 0, 0, 0, 0, 0, 0, 0, 0, 3433, 0, 4096, 0, 0, 0, 0, 0, 1055, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 872, 4073, 0, 0, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 119, 3024, 405, 71, 0, 0, 0, 0, 0, 0, 3971, 22, 801, 138, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3599, 0, 3021, 1342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 75, 0, 0, 3553, 89, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9818, 9075, 6017, 8104, 7719, 9028, 7962, 7813] avg=8192.0000 max=9818 max_rank=0 min=6017 min_rank=2 imbalance_ratio=1.198486 max_load_reduction=55718 max_load_reduction_pct=0.850189 moved_to_echo=32522 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 4058, 0, 30, 2843, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 4085, 0, 1, 0, 0, 179, 1017, 492, 0, 0, 0, 0, 0, 0, 90, 0, 0, 685, 4094, 204, 0, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 4096, 4006, 0, 0, 0, 0, 0, 0, 0, 17, 4095, 1, 0, 0, 0, 0, 54, 2925, 404, 69, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 139, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 0, 3487, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 1, 6, 0, 0, 3551, 90, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6941, 9763, 6109, 8144, 7565, 8311, 10965, 7738] avg=8192.0000 max=10965 max_rank=6 min=6109 min_rank=2 imbalance_ratio=1.338501 max_load_reduction=54571 max_load_reduction_pct=0.832687 moved_to_echo=34579 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3134, 0, 4096, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 3893, 0, 0, 0, 1, 0, 1061, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4057, 2, 0, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 117, 0, 4096, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 6, 0, 0, 3022, 403, 72, 0, 0, 0, 0, 0, 0, 4096, 24, 0, 138, 2, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 0, 70, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 224, 0, 58, 0, 301, 19, 1, 3543, 89, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10193, 5874, 9205, 8817, 7597, 8319, 7300, 8231] avg=8192.0000 max=10193 max_rank=0 min=5874 min_rank=1 imbalance_ratio=1.244263 max_load_reduction=55343 max_load_reduction_pct=0.844467 moved_to_echo=31447 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 3909, 5, 4096, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 349, 1, 4093, 0, 0, 0, 2, 104, 1061, 492, 0, 0, 0, 0, 0, 0, 252, 0, 0, 3475, 4096, 75, 0, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 18, 4096, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3024, 405, 72, 0, 0, 0, 0, 0, 0, 4096, 0, 464, 138, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 0, 152, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 10, 0, 0, 3553, 90, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10983, 6102, 8969, 8128, 7597, 8793, 7213, 7751] avg=8192.0000 max=10983 max_rank=0 min=6102 min_rank=1 imbalance_ratio=1.340698 max_load_reduction=54553 max_load_reduction_pct=0.832413 moved_to_echo=31354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 74, 0, 822, 0, 1631, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3737, 0, 0, 4, 153, 209, 1005, 489, 0, 0, 0, 0, 0, 0, 4076, 39, 0, 108, 3930, 260, 0, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4, 0, 0, 0, 0, 4052, 3948, 0, 0, 0, 0, 0, 0, 0, 77, 4095, 0, 0, 0, 0, 1, 3796, 2881, 404, 68, 0, 0, 0, 0, 0, 11, 1802, 0, 325, 139, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 3584, 201, 3, 3933, 6, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 6, 0, 0, 0, 23, 3549, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5351, 9691, 9434, 8038, 11322, 6270, 7729, 7701] avg=8192.0000 max=11322 max_rank=4 min=5351 min_rank=0 imbalance_ratio=1.382080 max_load_reduction=54214 max_load_reduction_pct=0.827240 moved_to_echo=33022 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4075, 46, 4096, 2932, 0, 0, 0, 0, 0, 0, 0, 0, 2500, 356, 4065, 0, 0, 0, 0, 1798, 1049, 491, 0, 0, 0, 0, 0, 0, 66, 20, 28, 4096, 3457, 0, 0, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 36, 0, 66, 4060, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 144, 0, 0, 0, 0, 2982, 403, 71, 0, 0, 0, 0, 0, 2, 3925, 0, 1635, 138, 0, 0, 2626, 0, 0, 0, 0, 0, 0, 0, 0, 3599, 46, 32, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 20, 0, 0, 3543, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11149, 10259, 8724, 8080, 7568, 8326, 3682, 7748] avg=8192.0000 max=11149 max_rank=0 min=3682 min_rank=6 imbalance_ratio=1.360962 max_load_reduction=54387 max_load_reduction_pct=0.829880 moved_to_echo=27324 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1146, 2, 5, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 540, 116, 0, 1, 0, 3, 2991, 1061, 485, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3843, 4063, 0, 0, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 15, 4087, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 41, 0, 0, 0, 9, 3024, 405, 72, 0, 0, 0, 0, 0, 16, 3941, 0, 0, 138, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 3557, 0, 3962, 4071, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 143, 0, 0, 3553, 89, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 8986, 8981, 8115, 7647, 8183, 11627, 7881] avg=8192.0000 max=11627 max_rank=6 min=4116 min_rank=0 imbalance_ratio=1.419312 max_load_reduction=53909 max_load_reduction_pct=0.822586 moved_to_echo=35338 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 184, 60, 4056, 2683, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 4096, 14, 0, 217, 1, 0, 965, 481, 0, 0, 0, 0, 0, 0, 2788, 0, 0, 39, 4093, 169, 0, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 24, 6, 0, 4087, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4, 0, 216, 0, 13, 876, 2861, 408, 68, 0, 0, 0, 0, 0, 0, 4096, 0, 27, 130, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 3497, 57, 1408, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4, 2834, 0, 107, 3546, 88, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6985, 5786, 8097, 8110, 8521, 8306, 9058, 10673] avg=8192.0000 max=10673 max_rank=7 min=5786 min_rank=1 imbalance_ratio=1.302856 max_load_reduction=54863 max_load_reduction_pct=0.837143 moved_to_echo=36558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 21376, 256, 20992, 23168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[28672, 384, 12160, 24448, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7296, 128, 128, 17280, 31872, 768, 0, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 256, 128, 128, 128, 3456, 128, 256, 28416, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[18176, 1408, 28288, 128, 128, 256, 256, 5376, 8320, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 768, 128, 256, 128, 32768, 32000, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 30080, 128, 3328, 1152, 128, 128, 31104, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32640, 128, 256, 256, 128, 128, 4864, 23808, 3328, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[2, 0, 0, 21, 0, 4096, 1066, 0, 0, 0, 0, 2872, 0, 0, 0, 4061, 0, 3917, 1, 4095, 3792, 13, 4096, 0, 0, 0, 0, 2502, 4096, 0, 0, 2794, 0, 312, 0, 0, 0, 4096, 116, 224, 0, 2742, 0, 0, 0, 0, 4, 0, 0, 3725, 0, 0, 3988, 0, 0, 4094, 3, 4094, 4095, 433, 0, 184, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 524, 13, 0, 0, 0, 463, 4027, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 2560, 4095, 0, 0, 3938, 0, 169, 0, 0, 0, 3933, 61, 4087, 0, 577, 0, 0, 0, 0, 4075, 0, 0, 0, 18, 2, 4096, 0, 2, 4096, 0, 4096, 4096, 24, 0, 100, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4094, 74, 0, 0, 0, 1, 1443, 1, 0, 0, 4086, 0, 4096, 2, 4096, 3991, 0, 4079, 0, 0, 0, 0, 6, 4096, 0, 0, 4089, 0, 163, 0, 0, 20, 4096, 0, 4043, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 94, 3069, 0, 89, 3936, 0, 3851, 4036, 3718, 0, 4029, 14, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3681, 0, 0, 0, 0, 80, 4011, 0, 0, 0, 4096, 0, 4095, 0, 4096, 4096, 1, 4096, 0, 10, 0, 0, 0, 735, 0, 0, 3981, 0, 698, 0, 0, 0, 4096, 175, 1354, 0, 3612, 0, 0, 0, 0, 2297, 0, 0, 4092, 0, 764, 3180, 0, 0, 4096, 0, 4094, 4096, 0, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[5, 0, 102, 4, 0, 4077, 3455, 0, 177, 0, 210, 1811, 388, 4, 0, 3924, 24, 4074, 59, 3574, 4096, 0, 4032, 26, 1599, 0, 68, 0, 3965, 0, 0, 0, 1, 0, 0, 0, 0, 4031, 8, 4077, 4, 14, 0, 48, 8, 0, 515, 0, 0, 4, 0, 4075, 3737, 0, 0, 4095, 7, 1304, 4034, 0, 124, 3776, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1064, 1729, 52, 0, 0, 0, 3936, 1, 0, 0, 4076, 0, 4096, 0, 4093, 3579, 0, 4096, 0, 33, 89, 0, 0, 4094, 0, 0, 4095, 0, 0, 0, 0, 0, 3779, 4045, 18, 0, 1388, 0, 0, 1, 0, 0, 0, 0, 4084, 490, 9, 4020, 0, 155, 0, 0, 4081, 4096, 279, 7, 4050, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[5, 0, 0, 1, 0, 3873, 24, 0, 113, 10, 1531, 4009, 51, 0, 0, 4092, 0, 4095, 0, 4027, 4096, 0, 4063, 15, 0, 0, 0, 2615, 3803, 5, 30, 376, 21, 323, 0, 0, 0, 4055, 0, 3993, 0, 3821, 0, 0, 73, 0, 6, 1, 37, 38, 240, 3610, 44, 0, 248, 3918, 7, 4096, 4095, 0, 5, 0, 71, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1, 20, 6, 4, 3, 4009, 0, 87, 0, 3, 1576, 4092, 0, 0, 0, 4095, 0, 3735, 0, 4088, 4084, 0, 4095, 8, 0, 155, 28, 17, 824, 0, 0, 4049, 0, 288, 0, 0, 7, 3133, 340, 0, 5, 4088, 0, 0, 2, 0, 686, 1, 55, 4074, 21, 2626, 4094, 1, 0, 3615, 10, 3416, 4094, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 21, 0, 4096, 1066, 0, 2690, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2872, 0, 0, 0, 4061, 275, 0, 0, 0, 0, 0, 0, 0, 0, 170, 1, 4095, 3792, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2502, 4096, 0, 0, 2794, 851, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 0, 0, 4096, 116, 224, 848, 502, 0, 0, 0, 0, 0, 0, 0, 2742, 0, 0, 0, 0, 4, 0, 4096, 1057, 0, 0, 0, 0, 0, 0, 0, 3725, 0, 0, 3988, 0, 0, 1582, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 2467, 433, 0, 184, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9539, 7208, 8071, 10243, 6098, 7899, 9295, 7183] avg=8192.0000 max=10243 max_rank=3 min=6098 min_rank=4 imbalance_ratio=1.250366 max_load_reduction=55293 max_load_reduction_pct=0.843704 moved_to_echo=30475 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 524, 13, 0, 2813, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 463, 4027, 0, 0, 0, 4096, 276, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2560, 4095, 0, 0, 3938, 851, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 3933, 61, 4087, 848, 502, 0, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 4075, 0, 4096, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2, 4096, 0, 2, 1583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2467, 24, 0, 100, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5015, 8862, 8370, 11444, 9600, 9853, 5701, 6691] avg=8192.0000 max=11444 max_rank=3 min=5015 min_rank=0 imbalance_ratio=1.396973 max_load_reduction=54092 max_load_reduction_pct=0.825378 moved_to_echo=31845 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4094, 74, 0, 2813, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1443, 1, 0, 0, 4086, 272, 0, 0, 0, 0, 0, 0, 0, 0, 178, 2, 4096, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 4089, 839, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 20, 4096, 0, 4043, 815, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 4079, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 3069, 0, 89, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 2431, 3718, 0, 4029, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8581, 5803, 8267, 9030, 9631, 5407, 4773, 14044] avg=8192.0000 max=14044 max_rank=7 min=4773 min_rank=6 imbalance_ratio=1.714355 max_load_reduction=51492 max_load_reduction_pct=0.785706 moved_to_echo=33855 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3681, 0, 0, 2812, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 80, 4011, 0, 0, 0, 4096, 276, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 735, 0, 0, 3981, 851, 0, 0, 0, 0, 0, 0, 0, 0, 698, 0, 0, 0, 4096, 175, 1354, 848, 502, 0, 0, 0, 0, 0, 0, 0, 3612, 0, 0, 0, 0, 2297, 0, 4096, 1105, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 764, 3180, 0, 0, 1583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2467, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8463, 8371, 5577, 7673, 11110, 9619, 6565] avg=8192.0000 max=11110 max_rank=5 min=5577 min_rank=3 imbalance_ratio=1.356201 max_load_reduction=54426 max_load_reduction_pct=0.830475 moved_to_echo=34967 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 102, 4, 0, 4077, 3455, 0, 2798, 1664, 0, 0, 0, 0, 0, 0, 177, 0, 210, 1811, 388, 4, 0, 3924, 271, 0, 0, 0, 0, 0, 0, 0, 24, 177, 59, 3574, 4096, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1599, 0, 68, 0, 3965, 0, 0, 0, 839, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4031, 8, 4077, 848, 494, 0, 0, 0, 0, 0, 0, 4, 14, 0, 48, 8, 0, 515, 0, 4032, 1099, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4075, 3737, 0, 0, 1583, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1304, 2430, 0, 124, 3776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12105, 6785, 7956, 6471, 9459, 5720, 9399, 7641] avg=8192.0000 max=12105 max_rank=0 min=5720 min_rank=5 imbalance_ratio=1.477661 max_load_reduction=53431 max_load_reduction_pct=0.815292 moved_to_echo=32219 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1064, 1729, 52, 2813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 1, 0, 0, 4076, 276, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 4093, 3579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 89, 0, 0, 4094, 0, 0, 4095, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 4045, 18, 0, 502, 0, 0, 0, 0, 0, 0, 0, 1388, 0, 0, 1, 0, 0, 0, 4096, 1105, 0, 0, 0, 0, 0, 0, 0, 4084, 490, 9, 4020, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 2467, 279, 7, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5659, 8289, 7850, 9162, 8344, 6590, 8758, 10884] avg=8192.0000 max=10884 max_rank=7 min=5659 min_rank=0 imbalance_ratio=1.328613 max_load_reduction=54652 max_load_reduction_pct=0.833923 moved_to_echo=34576 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 1, 0, 3873, 24, 0, 2812, 1592, 0, 0, 0, 0, 0, 0, 113, 10, 1531, 4009, 51, 0, 0, 4092, 275, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 4027, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2615, 3803, 5, 30, 376, 851, 0, 0, 0, 0, 0, 0, 0, 21, 323, 0, 0, 0, 4055, 0, 3993, 812, 502, 0, 0, 0, 0, 0, 0, 0, 3821, 0, 0, 73, 0, 6, 1, 4063, 1105, 0, 0, 0, 0, 0, 0, 37, 38, 240, 3610, 44, 0, 248, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 2467, 0, 5, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 10081, 8316, 7680, 9706, 9069, 5731, 6646] avg=8192.0000 max=10081 max_rank=1 min=5731 min_rank=6 imbalance_ratio=1.230591 max_load_reduction=55455 max_load_reduction_pct=0.846176 moved_to_echo=31152 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 20, 6, 4, 3, 4009, 0, 87, 2569, 1473, 0, 0, 0, 0, 0, 0, 0, 3, 1576, 4092, 0, 0, 0, 4095, 279, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 4088, 4084, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 28, 17, 824, 0, 0, 4049, 856, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 7, 3133, 340, 0, 753, 504, 0, 0, 0, 0, 0, 0, 5, 4088, 0, 0, 2, 0, 686, 1, 4095, 1011, 0, 0, 0, 0, 0, 0, 55, 4074, 21, 2626, 4094, 1, 0, 1389, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3416, 2455, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 10045, 8335, 5929, 5025, 9888, 12260, 5882] avg=8192.0000 max=12260 max_rank=6 min=5025 min_rank=4 imbalance_ratio=1.496582 max_load_reduction=53276 max_load_reduction_pct=0.812927 moved_to_echo=33055 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 16256, 0, 128, 128, 0, 7808, 128, 32768, 8704, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 29056, 19712, 4480, 256, 12160, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 16128, 896, 11264, 26240, 128, 512, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1664, 256, 128, 7808, 25728, 128, 128, 23424, 6912, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 2048, 0, 0, 128, 31232, 4864, 17920, 5888, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 25472, 6400, 256, 22144, 11392, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 1408, 128, 32256, 31872, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 128, 3968, 26240, 512, 128, 0, 32640, 2304, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[199, 2815, 1328, 128, 1744, 30, 720, 292, 1552, 91, 1416, 279, 875, 766, 57, 512, 297, 1529, 1072, 1212, 1722, 456, 2303, 1547, 72, 1541, 538, 1422, 1100, 163, 1265, 2115, 3323, 402, 358, 289, 102, 1380, 2219, 1204, 2289, 1449, 1503, 648, 619, 2200, 1286, 682, 287, 711, 373, 1244, 237, 530, 1671, 168, 298, 595, 2653, 2420, 1230, 428, 1025, 555] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[61, 1800, 942, 60, 2068, 55, 1055, 245, 1443, 100, 1149, 877, 980, 1060, 28, 500, 160, 1434, 1281, 1428, 1988, 514, 2510, 2197, 55, 1789, 348, 1349, 1069, 349, 1859, 2356, 3546, 442, 312, 635, 132, 1000, 1268, 715, 2290, 1397, 1410, 362, 615, 2339, 1096, 752, 184, 423, 511, 883, 422, 307, 1849, 103, 440, 1109, 2113, 2654, 1066, 453, 869, 730] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[102, 2491, 971, 276, 1988, 112, 1124, 537, 1311, 95, 1448, 274, 790, 599, 172, 640, 161, 861, 1721, 1277, 1693, 638, 2006, 1669, 228, 1612, 822, 593, 1080, 323, 1799, 1974, 2778, 271, 534, 332, 212, 1005, 2213, 557, 2859, 1562, 1750, 902, 493, 2020, 1124, 661, 365, 460, 934, 1677, 657, 532, 1614, 164, 128, 1100, 2539, 1687, 946, 688, 622, 763] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[122, 2251, 1013, 129, 2398, 50, 760, 531, 1077, 111, 1553, 585, 1017, 729, 145, 352, 179, 781, 1388, 1153, 1754, 672, 2479, 1881, 151, 1576, 466, 1238, 1073, 475, 2063, 1718, 2883, 272, 395, 608, 278, 1225, 1912, 419, 2532, 1739, 1540, 828, 863, 1829, 555, 1502, 428, 253, 369, 1243, 860, 321, 1521, 103, 79, 1160, 2410, 2241, 1109, 287, 1420, 482] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[41, 2844, 788, 183, 1182, 132, 793, 213, 802, 267, 865, 778, 779, 713, 67, 659, 374, 839, 1757, 983, 2421, 286, 2424, 1711, 44, 2130, 473, 1061, 1034, 313, 1104, 1988, 3322, 850, 255, 851, 298, 1355, 2012, 693, 2440, 915, 1361, 551, 548, 2129, 1622, 995, 402, 332, 459, 1311, 758, 1068, 1621, 115, 796, 774, 2603, 1511, 857, 770, 1055, 859] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[75, 2649, 1713, 41, 1889, 80, 936, 591, 907, 96, 717, 635, 791, 364, 140, 422, 181, 842, 1551, 1662, 1826, 444, 2513, 1755, 132, 1865, 502, 1188, 1100, 278, 1341, 1776, 3596, 424, 333, 863, 307, 896, 1946, 767, 2808, 1530, 1441, 690, 567, 2279, 1176, 679, 344, 438, 415, 1285, 462, 363, 1779, 103, 197, 760, 1956, 2578, 1134, 398, 1177, 843] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[62, 2699, 1197, 92, 1607, 45, 851, 439, 1556, 123, 1168, 400, 694, 700, 55, 404, 154, 716, 2106, 884, 2126, 657, 2198, 1859, 69, 2029, 466, 1109, 1449, 423, 1887, 2005, 3055, 290, 807, 959, 173, 1259, 2004, 705, 2778, 1348, 1203, 552, 424, 1789, 1128, 703, 352, 367, 517, 1663, 655, 418, 1159, 102, 235, 862, 2184, 1699, 1347, 482, 1531, 557] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[154, 2107, 823, 118, 2173, 93, 817, 748, 1189, 99, 1286, 629, 638, 527, 48, 508, 276, 1288, 1376, 1524, 1910, 517, 2867, 1557, 65, 1551, 543, 731, 870, 303, 1417, 1860, 3707, 352, 430, 241, 586, 1000, 2562, 616, 2544, 1165, 1536, 616, 655, 2170, 1280, 913, 483, 442, 465, 1280, 370, 214, 2307, 352, 296, 716, 2284, 2329, 860, 751, 984, 418] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[199, 2815, 1328, 128, 1744, 30, 720, 292, 659, 288, 105, 79, 20, 0, 0, 0, 1552, 91, 1416, 279, 875, 766, 57, 512, 2289, 404, 0, 0, 0, 0, 0, 0, 297, 1529, 1072, 1212, 1643, 456, 0, 1547, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1541, 538, 1422, 1100, 163, 1265, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 2664, 402, 358, 289, 102, 1380, 2219, 1204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1449, 1503, 648, 619, 2180, 1286, 682, 0, 0, 0, 0, 0, 0, 0, 0, 287, 711, 373, 1244, 237, 530, 1671, 168, 1899, 823, 0, 0, 0, 0, 0, 0, 298, 595, 1725, 2420, 1230, 428, 1025, 555, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8407, 8241, 7756, 7928, 8618, 8367, 7943, 8276] avg=8192.0000 max=8618 max_rank=4 min=7756 min_rank=2 imbalance_ratio=1.052002 max_load_reduction=56918 max_load_reduction_pct=0.868500 moved_to_echo=33204 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[154, 2107, 823, 118, 2173, 93, 817, 748, 735, 253, 91, 87, 20, 0, 0, 0, 1189, 99, 1286, 629, 638, 527, 48, 508, 2544, 503, 0, 0, 0, 0, 0, 0, 276, 1288, 1376, 1524, 1823, 517, 0, 1557, 0, 0, 0, 0, 0, 0, 0, 0, 65, 1551, 543, 731, 870, 303, 1417, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 2972, 352, 430, 241, 586, 1000, 2562, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 1536, 616, 655, 2150, 1280, 913, 0, 0, 0, 0, 0, 0, 0, 0, 483, 442, 465, 1280, 370, 214, 2307, 352, 2364, 709, 0, 0, 0, 0, 0, 0, 296, 716, 1484, 2329, 860, 751, 984, 418, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 7971, 8361, 7087, 8759, 8315, 8986, 7838] avg=8192.0000 max=8986 max_rank=6 min=7087 min_rank=3 imbalance_ratio=1.096924 max_load_reduction=56550 max_load_reduction_pct=0.862885 moved_to_echo=33898 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[75, 2649, 1713, 41, 1889, 80, 936, 591, 713, 241, 78, 84, 21, 0, 0, 0, 907, 96, 717, 635, 791, 364, 140, 422, 2808, 441, 0, 0, 0, 0, 0, 0, 181, 842, 1551, 1662, 1742, 444, 0, 1755, 0, 0, 0, 0, 0, 0, 0, 0, 132, 1865, 502, 1188, 1100, 278, 1341, 1535, 0, 0, 0, 0, 0, 0, 0, 0, 2883, 424, 333, 863, 307, 896, 1946, 767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1530, 1441, 690, 567, 2258, 1176, 679, 0, 0, 0, 0, 0, 0, 0, 0, 344, 438, 415, 1285, 462, 363, 1779, 103, 2072, 607, 0, 0, 0, 0, 0, 0, 197, 760, 1271, 2578, 1134, 398, 1177, 843, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9111, 7321, 8177, 7941, 8419, 8341, 7868, 8358] avg=8192.0000 max=9111 max_rank=0 min=7321 min_rank=1 imbalance_ratio=1.112183 max_load_reduction=56425 max_load_reduction_pct=0.860977 moved_to_echo=32986 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[61, 1800, 942, 60, 2068, 55, 1055, 245, 703, 320, 84, 91, 21, 0, 0, 0, 1443, 100, 1149, 877, 980, 1060, 28, 500, 2290, 440, 0, 0, 0, 0, 0, 0, 160, 1434, 1281, 1428, 1897, 514, 0, 2197, 0, 0, 0, 0, 0, 0, 0, 0, 55, 1789, 348, 1349, 1069, 349, 1859, 2036, 0, 0, 0, 0, 0, 0, 0, 0, 2843, 442, 312, 635, 132, 1000, 1268, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 1410, 362, 615, 2318, 1096, 752, 0, 0, 0, 0, 0, 0, 0, 0, 184, 423, 511, 883, 422, 307, 1849, 103, 2070, 655, 0, 0, 0, 0, 0, 0, 440, 1109, 1374, 2654, 1066, 453, 869, 730, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7505, 8867, 8911, 8854, 7347, 7950, 7407, 8695] avg=8192.0000 max=8911 max_rank=2 min=7347 min_rank=4 imbalance_ratio=1.087769 max_load_reduction=56625 max_load_reduction_pct=0.864029 moved_to_echo=31399 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[62, 2699, 1197, 92, 1607, 45, 851, 439, 605, 273, 87, 97, 16, 0, 0, 0, 1556, 123, 1168, 400, 694, 700, 55, 404, 2778, 385, 0, 0, 0, 0, 0, 0, 154, 716, 2106, 884, 2029, 657, 0, 1859, 0, 0, 0, 0, 0, 0, 0, 0, 69, 2029, 466, 1109, 1449, 423, 1887, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 2450, 290, 807, 959, 173, 1259, 2004, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1348, 1203, 552, 424, 1773, 1128, 703, 0, 0, 0, 0, 0, 0, 0, 0, 352, 367, 517, 1663, 655, 418, 1159, 102, 1813, 677, 0, 0, 0, 0, 0, 0, 235, 862, 1420, 1699, 1347, 482, 1531, 557, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 8263, 8405, 9164, 8647, 7131, 7723, 8133] avg=8192.0000 max=9164 max_rank=3 min=7131 min_rank=5 imbalance_ratio=1.118652 max_load_reduction=56372 max_load_reduction_pct=0.860168 moved_to_echo=31634 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[102, 2491, 971, 276, 1988, 112, 1124, 537, 551, 268, 101, 77, 18, 0, 0, 0, 1311, 95, 1448, 274, 790, 599, 172, 640, 2859, 353, 0, 0, 0, 0, 0, 0, 161, 861, 1721, 1277, 1616, 638, 0, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 228, 1612, 822, 593, 1080, 323, 1799, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 2227, 271, 534, 332, 212, 1005, 2213, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1562, 1750, 902, 493, 2002, 1124, 661, 0, 0, 0, 0, 0, 0, 0, 0, 365, 460, 934, 1677, 657, 532, 1614, 164, 1653, 788, 0, 0, 0, 0, 0, 0, 128, 1100, 1650, 1687, 946, 688, 622, 763, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8616, 8541, 7943, 8163, 7351, 8494, 8844, 7584] avg=8192.0000 max=8844 max_rank=6 min=7351 min_rank=4 imbalance_ratio=1.079590 max_load_reduction=56692 max_load_reduction_pct=0.865051 moved_to_echo=32273 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[122, 2251, 1013, 129, 2398, 50, 760, 531, 571, 234, 96, 80, 16, 0, 0, 0, 1077, 111, 1553, 585, 1017, 729, 145, 352, 2532, 435, 0, 0, 0, 0, 0, 0, 179, 781, 1388, 1153, 1674, 672, 0, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 151, 1576, 466, 1238, 1073, 475, 2063, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 2312, 272, 395, 608, 278, 1225, 1912, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1739, 1540, 828, 863, 1813, 555, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 428, 253, 369, 1243, 860, 321, 1521, 103, 2044, 748, 0, 0, 0, 0, 0, 0, 79, 1160, 1566, 2241, 1109, 287, 1420, 482, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8536, 7728, 8526, 7421, 8840, 7890, 8344] avg=8192.0000 max=8840 max_rank=5 min=7421 min_rank=4 imbalance_ratio=1.079102 max_load_reduction=56696 max_load_reduction_pct=0.865112 moved_to_echo=32495 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[41, 2844, 788, 183, 1182, 132, 793, 213, 662, 274, 106, 116, 23, 0, 0, 0, 802, 267, 865, 778, 779, 713, 67, 659, 2440, 426, 0, 0, 0, 0, 0, 0, 374, 839, 1757, 983, 2305, 286, 0, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 44, 2130, 473, 1061, 1034, 313, 1104, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 2660, 850, 255, 851, 298, 1355, 2012, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 915, 1361, 551, 548, 2106, 1622, 995, 0, 0, 0, 0, 0, 0, 0, 0, 402, 332, 459, 1311, 758, 1068, 1621, 115, 1998, 811, 0, 0, 0, 0, 0, 0, 796, 774, 1686, 1511, 857, 770, 1055, 859, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7357, 7796, 8255, 7873, 8974, 8098, 8875, 8308] avg=8192.0000 max=8974 max_rank=4 min=7357 min_rank=0 imbalance_ratio=1.095459 max_load_reduction=56562 max_load_reduction_pct=0.863068 moved_to_echo=34255 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[896, 19712, 8832, 1152, 15104, 640, 7168, 3712, 5248, 2176, 768, 768, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 11136, 11776, 5248, 4864, 16640, 9344, 6912, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2560, 7168, 12288, 17152, 8576, 4352, 8704, 5248, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2944, 3456, 4096, 10624, 4480, 3840, 13568, 1280, 16000, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[896, 14208, 4224, 8704, 8832, 2688, 12800, 13696, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9856, 1024, 9728, 4480, 6656, 5504, 768, 4096, 20608, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1792, 8320, 12288, 10240, 14848, 4224, 0, 14208, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[21120, 3328, 3456, 4864, 2176, 9216, 16256, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4077, 0, 0, 450, 0, 27, 0, 2, 2, 3980, 99, 0, 3, 0, 52, 0, 1, 0, 1685, 4084, 4019, 3894, 0, 4028, 4088, 0, 7, 480, 1846, 1, 0, 8, 4088, 2634, 0, 4076, 114, 2010, 6, 0, 16, 6, 2409, 3464, 42, 1, 820, 101, 1, 503, 0, 3426, 16, 687, 1, 6, 0, 1, 2, 71, 4096, 4092, 0, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4027, 0, 0, 657, 2, 54, 101, 3, 35, 3993, 8, 0, 6, 2, 284, 0, 41, 0, 724, 3550, 4068, 4002, 1, 3961, 4091, 1, 182, 461, 4, 0, 98, 75, 4039, 2756, 0, 4094, 561, 0, 2, 0, 912, 1, 4085, 4084, 0, 0, 1, 5, 7, 2302, 27, 3581, 1, 119, 2, 1, 10, 24, 0, 542, 3864, 4072, 0, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4031, 8, 1, 171, 0, 28, 2, 64, 3, 4086, 5, 0, 0, 4, 12, 9, 0, 11, 2283, 4096, 3993, 4035, 0, 4086, 4094, 1, 1, 2, 2325, 10, 4, 1, 3007, 4086, 0, 4079, 15, 94, 0, 0, 255, 28, 2458, 4030, 4, 2, 44, 0, 2, 786, 11, 928, 29, 3839, 1, 1, 2, 61, 0, 6, 3973, 3808, 35, 586] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4074, 25, 0, 4, 0, 65, 42, 2, 10, 3560, 125, 0, 5, 0, 297, 5, 0, 1, 652, 4065, 4082, 4094, 0, 4079, 4095, 1, 70, 705, 41, 1, 825, 1, 4030, 2263, 4, 4085, 0, 2, 31, 1, 18, 6, 2757, 3978, 0, 0, 21, 18, 0, 2413, 9, 3469, 1, 3750, 17, 0, 219, 0, 0, 0, 4031, 3472, 15, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3399, 0, 0, 63, 0, 6, 0, 78, 67, 4096, 87, 70, 438, 109, 652, 41, 25, 32, 2425, 4002, 4010, 3663, 10, 4090, 4059, 0, 28, 1, 25, 0, 2, 75, 2652, 4094, 65, 4039, 0, 777, 1, 2, 2130, 1, 3307, 2934, 0, 7, 1, 0, 19, 252, 4, 2640, 0, 2804, 0, 1, 0, 0, 0, 32, 3985, 4017, 0, 219] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 13, 1, 5, 0, 3, 0, 48, 1, 3529, 69, 2, 99, 1, 976, 2, 0, 0, 2799, 4094, 3857, 4076, 0, 4093, 4086, 17, 3, 321, 921, 3, 279, 18, 1628, 3098, 0, 4096, 5, 9, 0, 2, 6, 5, 3503, 3966, 222, 0, 423, 53, 0, 3434, 28, 510, 2, 1659, 1, 1, 378, 1, 0, 4, 4073, 4068, 0, 949] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3787, 0, 0, 5, 0, 29, 3, 4, 0, 4037, 308, 4, 0, 4, 7, 654, 0, 0, 3571, 4038, 4090, 3757, 0, 4095, 3934, 0, 0, 1, 46, 160, 0, 9, 768, 4079, 0, 4096, 0, 206, 38, 146, 3264, 2, 259, 2264, 2955, 0, 109, 0, 0, 710, 1, 3552, 0, 2318, 0, 0, 2, 0, 0, 3, 3861, 3826, 0, 534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[3885, 0, 273, 49, 0, 63, 6, 253, 1, 3392, 15, 0, 3, 0, 1799, 12, 0, 7, 1099, 2839, 3914, 4005, 0, 4060, 4094, 0, 41, 1866, 532, 337, 920, 0, 644, 3541, 9, 3930, 1, 15, 291, 3, 950, 112, 4085, 4083, 182, 10, 89, 6, 218, 2294, 0, 3084, 0, 319, 0, 283, 379, 0, 89, 0, 3702, 3683, 67, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4077, 0, 0, 450, 0, 27, 0, 2, 3912, 0, 0, 0, 0, 0, 0, 0, 2, 3980, 99, 0, 3, 0, 52, 0, 3455, 116, 0, 0, 0, 0, 0, 0, 1, 0, 1685, 4084, 0, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 7, 480, 1846, 1, 0, 8, 1974, 564, 0, 0, 0, 0, 0, 0, 4088, 2634, 0, 1722, 114, 2010, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 6, 2409, 3464, 42, 1, 820, 101, 135, 0, 0, 0, 0, 0, 0, 0, 1, 503, 0, 3426, 16, 687, 1, 6, 1497, 380, 39, 0, 0, 0, 0, 0, 0, 1, 2, 71, 3922, 4092, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8468, 7707, 8167, 8968, 10574, 6994, 6556, 8102] avg=8192.0000 max=10574 max_rank=4 min=6556 min_rank=6 imbalance_ratio=1.290771 max_load_reduction=54962 max_load_reduction_pct=0.838654 moved_to_echo=32226 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4074, 25, 0, 4, 0, 65, 42, 2, 3960, 0, 0, 0, 0, 0, 0, 0, 10, 3560, 125, 0, 5, 0, 297, 5, 3508, 119, 0, 0, 0, 0, 0, 0, 0, 1, 652, 4065, 0, 2520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 70, 705, 41, 1, 825, 1, 1978, 574, 0, 0, 0, 0, 0, 0, 4030, 2263, 4, 1726, 0, 2, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 18, 6, 2757, 3978, 0, 0, 21, 18, 132, 0, 0, 0, 0, 0, 0, 0, 0, 2413, 9, 3469, 1, 3750, 17, 0, 1574, 381, 38, 0, 0, 0, 0, 0, 219, 0, 0, 0, 3861, 3472, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 7629, 7238, 8291, 8057, 6930, 11652, 7567] avg=8192.0000 max=11652 max_rank=6 min=6930 min_rank=5 imbalance_ratio=1.422363 max_load_reduction=53884 max_load_reduction_pct=0.822205 moved_to_echo=34206 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4031, 8, 1, 171, 0, 28, 2, 64, 3967, 0, 0, 0, 0, 0, 0, 0, 3, 4086, 5, 0, 0, 4, 12, 9, 3432, 119, 0, 0, 0, 0, 0, 0, 0, 11, 2283, 4096, 0, 2484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 2, 2325, 10, 4, 1, 1975, 561, 0, 0, 0, 0, 0, 0, 3007, 4086, 0, 1723, 15, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 28, 2458, 4030, 4, 2, 44, 0, 131, 0, 0, 0, 0, 0, 0, 0, 2, 786, 11, 928, 29, 3839, 1, 1, 1551, 381, 38, 0, 0, 0, 0, 0, 2, 61, 0, 6, 3804, 3808, 35, 586, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 7670, 8874, 8974, 8925, 6952, 7567, 8302] avg=8192.0000 max=8974 max_rank=3 min=6952 min_rank=5 imbalance_ratio=1.095459 max_load_reduction=56562 max_load_reduction_pct=0.863068 moved_to_echo=31746 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3399, 0, 0, 63, 0, 6, 0, 78, 3971, 0, 0, 0, 0, 0, 0, 0, 67, 4096, 87, 70, 438, 109, 652, 41, 3446, 119, 0, 0, 0, 0, 0, 0, 25, 32, 2425, 4002, 0, 2255, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 28, 1, 25, 0, 2, 75, 1956, 564, 0, 0, 0, 0, 0, 0, 2652, 4094, 65, 1706, 0, 777, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2130, 1, 3307, 2934, 0, 7, 1, 0, 131, 0, 0, 0, 0, 0, 0, 0, 19, 252, 4, 2640, 0, 2804, 0, 1, 1408, 377, 38, 0, 0, 0, 0, 0, 0, 0, 0, 32, 3816, 4017, 0, 219, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7517, 9125, 8749, 6710, 9297, 8511, 7543, 8084] avg=8192.0000 max=9297 max_rank=4 min=6710 min_rank=3 imbalance_ratio=1.134888 max_load_reduction=56239 max_load_reduction_pct=0.858139 moved_to_echo=33435 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4027, 0, 0, 657, 2, 54, 101, 3, 3847, 0, 0, 0, 0, 0, 0, 0, 35, 3993, 8, 0, 6, 2, 284, 0, 3497, 114, 0, 0, 0, 0, 0, 0, 41, 0, 724, 3550, 0, 2463, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 182, 461, 4, 0, 98, 75, 1983, 571, 0, 0, 0, 0, 0, 0, 4039, 2756, 0, 1729, 561, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 1, 4085, 4084, 0, 0, 1, 5, 127, 0, 0, 0, 0, 0, 0, 0, 7, 2302, 27, 3581, 1, 119, 2, 1, 1539, 382, 37, 0, 0, 0, 0, 0, 10, 24, 0, 542, 3700, 4072, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8691, 7939, 6779, 7466, 9087, 9215, 7998, 8361] avg=8192.0000 max=9215 max_rank=5 min=6779 min_rank=2 imbalance_ratio=1.124878 max_load_reduction=56321 max_load_reduction_pct=0.859390 moved_to_echo=34661 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 13, 1, 5, 0, 3, 0, 48, 3974, 0, 0, 0, 0, 0, 0, 0, 1, 3529, 69, 2, 99, 1, 976, 2, 3315, 119, 0, 0, 0, 0, 0, 0, 0, 0, 2799, 4094, 0, 2509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 17, 3, 321, 921, 3, 279, 18, 1984, 542, 0, 0, 0, 0, 0, 0, 1628, 3098, 0, 1730, 5, 9, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 3503, 3966, 222, 0, 423, 53, 134, 0, 0, 0, 0, 0, 0, 0, 0, 3434, 28, 510, 2, 1659, 1, 1, 1567, 382, 39, 0, 0, 0, 0, 0, 378, 1, 0, 4, 3900, 4068, 0, 949, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8113, 9402, 8174, 6472, 8312, 7623, 9300] avg=8192.0000 max=9402 max_rank=2 min=6472 min_rank=4 imbalance_ratio=1.147705 max_load_reduction=56134 max_load_reduction_pct=0.856537 moved_to_echo=31707 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3787, 0, 0, 5, 0, 29, 3, 4, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 308, 4, 0, 4, 7, 654, 3516, 119, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 4038, 0, 2313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 1, 46, 160, 0, 9, 1984, 574, 0, 0, 0, 0, 0, 0, 768, 4079, 0, 1730, 0, 206, 38, 146, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 2, 259, 2264, 2955, 0, 109, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 710, 1, 3552, 0, 2318, 0, 0, 1444, 382, 37, 0, 0, 0, 0, 0, 2, 0, 0, 3, 3697, 3826, 0, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7804, 8649, 9922, 6708, 6967, 8980, 8444, 8062] avg=8192.0000 max=9922 max_rank=2 min=6708 min_rank=3 imbalance_ratio=1.211182 max_load_reduction=55614 max_load_reduction_pct=0.848602 moved_to_echo=32453 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3885, 0, 273, 49, 0, 63, 6, 253, 3943, 0, 0, 0, 0, 0, 0, 0, 1, 3392, 15, 0, 3, 0, 1799, 12, 3365, 117, 0, 0, 0, 0, 0, 0, 0, 7, 1099, 2839, 0, 2460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 41, 1866, 532, 337, 920, 0, 1906, 549, 0, 0, 0, 0, 0, 0, 644, 3541, 9, 1653, 1, 15, 291, 3, 0, 0, 0, 0, 0, 0, 0, 0, 950, 112, 4085, 4083, 182, 10, 89, 6, 125, 0, 0, 0, 0, 0, 0, 0, 218, 2294, 0, 3084, 0, 319, 0, 283, 1545, 371, 39, 0, 0, 0, 0, 0, 379, 0, 89, 0, 3538, 3683, 67, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8472, 8704, 6405, 10245, 6157, 9642, 8153, 7758] avg=8192.0000 max=10245 max_rank=3 min=6157 min_rank=4 imbalance_ratio=1.250610 max_load_reduction=55291 max_load_reduction_pct=0.843674 moved_to_echo=31710 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 15360, 30848, 0, 19456, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 12800, 128, 21248, 128, 15616, 128, 384, 12160, 3072, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[20864, 26624, 128, 13824, 768, 3200, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 128, 384, 3840, 5760, 512, 2176, 256, 15744, 4608, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[31488, 128, 384, 1408, 128, 384, 256, 512, 31616, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 30720, 768, 128, 640, 128, 4096, 768, 27648, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7552, 256, 22912, 28928, 3456, 128, 1536, 256, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1024, 128, 128, 768, 30336, 31104, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 336, 4087, 0, 172, 0, 0, 64, 356, 13, 5, 3947, 4091, 3048, 93, 253, 1, 3012, 0, 0, 4061, 262, 1, 0, 374, 0, 3235, 1, 0, 4086, 3738, 2, 4096, 534, 172, 225, 27, 4096, 11, 0, 4085, 4, 1, 66, 0, 0, 0, 6, 210, 63, 4090, 0, 0, 164, 0, 0, 3761, 696, 7, 0, 4071, 0, 0, 3913] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 3102, 4062, 0, 4092, 0, 0, 1107, 22, 0, 0, 4094, 4086, 1120, 2002, 28, 0, 46, 0, 7, 3834, 192, 1, 165, 143, 16, 3641, 0, 1, 4094, 3, 0, 3962, 940, 35, 33, 6, 4086, 1, 1, 4090, 72, 2, 4, 14, 0, 582, 98, 18, 274, 4010, 0, 2, 0, 29, 0, 4068, 2930, 0, 0, 4093, 0, 71, 257] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 3752, 3391, 0, 2585, 0, 3, 0, 23, 0, 0, 3005, 4093, 2884, 4085, 12, 54, 497, 4, 0, 3906, 0, 49, 0, 0, 2, 3771, 0, 0, 4055, 3301, 0, 4088, 3, 5, 645, 2, 4074, 0, 4, 4091, 1, 9, 0, 1, 0, 257, 1, 28, 0, 3995, 0, 1, 0, 8, 0, 4096, 0, 8, 1, 4034, 1, 0, 711] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[5, 615, 400, 0, 4094, 0, 25, 8, 3, 0, 4, 3939, 4075, 4009, 4036, 0, 0, 3999, 6, 0, 3372, 484, 8, 0, 94, 621, 3411, 64, 0, 4074, 247, 0, 4096, 38, 448, 1, 2, 3743, 1, 0, 4053, 10, 7, 0, 2, 292, 35, 0, 545, 0, 4075, 69, 0, 27, 4, 0, 4038, 2310, 5, 0, 4090, 4, 5, 43] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 144, 3945, 0, 2641, 0, 0, 75, 890, 0, 0, 853, 4055, 4096, 4096, 232, 0, 3, 0, 0, 1725, 1, 0, 0, 77, 0, 4089, 0, 1, 4091, 232, 37, 3994, 3894, 1855, 1586, 0, 4096, 796, 0, 3769, 0, 0, 0, 0, 63, 210, 7, 0, 0, 3837, 7, 82, 0, 0, 0, 3795, 2310, 28, 0, 3915, 4, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1641, 971, 5, 4077, 0, 10, 187, 46, 1, 0, 1653, 4096, 3870, 4096, 38, 0, 3, 0, 2, 2638, 2940, 0, 2, 274, 0, 3982, 0, 33, 4076, 4075, 24, 3435, 96, 64, 1150, 1, 4014, 0, 0, 4093, 3, 78, 1, 4, 42, 0, 1078, 0, 0, 4049, 0, 0, 16, 0, 0, 4083, 238, 0, 0, 4059, 0, 1, 291] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 4020, 2758, 0, 4074, 0, 0, 200, 2275, 6, 0, 3721, 4096, 2416, 4096, 41, 1, 1282, 0, 0, 4095, 1, 0, 0, 5, 0, 4005, 0, 0, 4096, 308, 0, 178, 1078, 81, 20, 0, 4096, 0, 0, 4096, 1, 0, 44, 0, 378, 0, 204, 6, 3, 4087, 0, 0, 0, 0, 0, 3876, 26, 0, 0, 4096, 0, 0, 1770] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[9, 642, 3520, 0, 2293, 211, 13, 0, 693, 0, 0, 4095, 4095, 320, 2402, 0, 27, 402, 2, 179, 1657, 1541, 125, 28, 0, 202, 3727, 0, 0, 3608, 971, 115, 3390, 1424, 0, 904, 54, 4007, 0, 1, 4094, 3, 0, 189, 2, 721, 822, 175, 2927, 0, 4065, 1, 3, 6, 37, 7, 3921, 4042, 0, 1, 3095, 82, 684, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3102, 4062, 0, 4092, 0, 0, 1107, 279, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 2253, 0, 1120, 2002, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 7, 3834, 192, 1, 165, 1841, 996, 723, 0, 0, 0, 0, 0, 143, 16, 3641, 0, 1, 2636, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 940, 35, 33, 6, 2811, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 72, 2, 4, 14, 0, 582, 98, 2314, 735, 488, 0, 0, 0, 0, 0, 18, 274, 4010, 0, 2, 0, 29, 0, 3598, 0, 0, 0, 0, 0, 0, 0, 1754, 2930, 0, 0, 4093, 0, 71, 257, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12642, 5425, 7805, 6440, 7789, 8399, 7931, 9105] avg=8192.0000 max=12642 max_rank=0 min=5425 min_rank=1 imbalance_ratio=1.543213 max_load_reduction=52894 max_load_reduction_pct=0.807098 moved_to_echo=33224 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 615, 400, 0, 4094, 0, 25, 8, 255, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 2168, 0, 4009, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 6, 0, 3372, 484, 8, 0, 1771, 912, 719, 0, 0, 0, 0, 0, 94, 621, 3411, 64, 0, 2624, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 38, 448, 1, 2, 2576, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 10, 7, 0, 2, 292, 35, 0, 2297, 731, 485, 0, 0, 0, 0, 0, 545, 0, 4075, 69, 0, 27, 4, 0, 3590, 0, 0, 0, 0, 0, 0, 0, 1741, 2310, 5, 0, 4090, 4, 5, 43, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5402, 10220, 11271, 7061, 7162, 7912, 8310, 8198] avg=8192.0000 max=11271 max_rank=2 min=5402 min_rank=0 imbalance_ratio=1.375854 max_load_reduction=54265 max_load_reduction_pct=0.828018 moved_to_echo=31582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 144, 3945, 0, 2641, 0, 0, 75, 279, 0, 0, 0, 0, 0, 0, 0, 890, 0, 0, 470, 0, 4096, 4096, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1725, 1, 0, 0, 383, 998, 722, 0, 0, 0, 0, 0, 77, 0, 4089, 0, 1, 2635, 232, 37, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 3894, 1855, 1586, 0, 2819, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3769, 0, 0, 0, 0, 63, 210, 7, 2159, 734, 483, 0, 0, 0, 0, 0, 0, 0, 3837, 7, 82, 0, 0, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 1636, 2310, 28, 0, 3915, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7084, 9784, 3832, 7071, 14944, 7425, 7498, 7898] avg=8192.0000 max=14944 max_rank=4 min=3832 min_rank=2 imbalance_ratio=1.824219 max_load_reduction=50592 max_load_reduction_pct=0.771973 moved_to_echo=37765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 336, 4087, 0, 172, 0, 0, 64, 279, 0, 0, 0, 0, 0, 0, 0, 356, 13, 5, 2172, 0, 3048, 93, 253, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3012, 0, 0, 4061, 262, 1, 0, 1775, 998, 722, 0, 0, 0, 0, 0, 374, 0, 3235, 1, 0, 2631, 3738, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 534, 172, 225, 27, 2819, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4, 1, 66, 0, 0, 0, 6, 2139, 733, 488, 0, 0, 0, 0, 0, 210, 63, 4090, 0, 0, 164, 0, 0, 3603, 0, 0, 0, 0, 0, 0, 0, 1622, 696, 7, 0, 4071, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4938, 5940, 10832, 9981, 7884, 7522, 8130, 10309] avg=8192.0000 max=10832 max_rank=2 min=4938 min_rank=0 imbalance_ratio=1.322266 max_load_reduction=54704 max_load_reduction_pct=0.834717 moved_to_echo=33845 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3752, 3391, 0, 2585, 0, 3, 0, 278, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 1654, 0, 2884, 4085, 12, 0, 0, 0, 0, 0, 0, 0, 0, 54, 497, 4, 0, 3906, 0, 49, 0, 1351, 993, 716, 0, 0, 0, 0, 0, 0, 2, 3771, 0, 0, 2611, 3301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 5, 645, 2, 2803, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 9, 0, 1, 0, 257, 1, 2330, 728, 487, 0, 0, 0, 0, 0, 28, 0, 3995, 0, 1, 0, 8, 0, 3606, 0, 0, 0, 0, 0, 0, 0, 1766, 0, 8, 1, 4034, 1, 0, 711, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10009, 8658, 7570, 9685, 7550, 7905, 7638, 6521] avg=8192.0000 max=10009 max_rank=0 min=6521 min_rank=7 imbalance_ratio=1.221802 max_load_reduction=55527 max_load_reduction_pct=0.847275 moved_to_echo=29614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1641, 971, 5, 4077, 0, 10, 187, 274, 0, 0, 0, 0, 0, 0, 0, 46, 1, 0, 910, 0, 3870, 4096, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 2638, 2940, 0, 2, 743, 978, 720, 0, 0, 0, 0, 0, 274, 0, 3982, 0, 33, 2625, 4075, 24, 0, 0, 0, 0, 0, 0, 0, 0, 3435, 96, 64, 1150, 1, 2762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 78, 1, 4, 42, 0, 1078, 2323, 731, 488, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 16, 0, 0, 3608, 0, 0, 0, 0, 0, 0, 0, 1760, 238, 0, 0, 4059, 0, 1, 291, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7165, 8961, 8026, 11013, 7508, 8841, 7673, 6349] avg=8192.0000 max=11013 max_rank=3 min=6349 min_rank=7 imbalance_ratio=1.344360 max_load_reduction=54523 max_load_reduction_pct=0.831955 moved_to_echo=30371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4020, 2758, 0, 4074, 0, 0, 200, 279, 0, 0, 0, 0, 0, 0, 0, 2275, 6, 0, 2048, 0, 2416, 4096, 41, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1282, 0, 0, 4095, 1, 0, 0, 1673, 998, 723, 0, 0, 0, 0, 0, 5, 0, 4005, 0, 0, 2638, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 1078, 81, 20, 0, 2819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 44, 0, 378, 0, 204, 2205, 735, 488, 0, 0, 0, 0, 0, 6, 3, 4087, 0, 0, 0, 0, 0, 3608, 0, 0, 0, 0, 0, 0, 0, 1671, 26, 0, 0, 4096, 0, 0, 1770, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11331, 10882, 8773, 6956, 4176, 8151, 7704, 7563] avg=8192.0000 max=11331 max_rank=0 min=4176 min_rank=4 imbalance_ratio=1.383179 max_load_reduction=54205 max_load_reduction_pct=0.827103 moved_to_echo=27594 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[9, 642, 3520, 0, 2293, 211, 13, 0, 277, 0, 0, 0, 0, 0, 0, 0, 693, 0, 0, 2251, 0, 320, 2402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 402, 2, 179, 1657, 1541, 125, 28, 1844, 980, 642, 0, 0, 0, 0, 0, 0, 202, 3727, 0, 0, 2314, 971, 115, 0, 0, 0, 0, 0, 0, 0, 0, 3390, 1424, 0, 904, 54, 2750, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 189, 2, 721, 822, 175, 2234, 652, 489, 0, 0, 0, 0, 0, 2927, 0, 4065, 1, 3, 6, 37, 7, 3606, 0, 0, 0, 0, 0, 0, 0, 1687, 4042, 0, 1, 3095, 82, 684, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6965, 5666, 7427, 7329, 8523, 9381, 10652, 9593] avg=8192.0000 max=10652 max_rank=6 min=5666 min_rank=1 imbalance_ratio=1.300293 max_load_reduction=54884 max_load_reduction_pct=0.837463 moved_to_echo=38149 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 9344, 128, 256, 25344, 5504, 256, 256, 11392, 7936, 5760, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4352, 128, 128, 13952, 0, 21888, 24960, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32384, 128, 128, 384, 128, 1536, 1920, 1664, 18048, 5888, 3968, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3840, 384, 32256, 128, 128, 256, 128, 128, 28800, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[27264, 8064, 2688, 4608, 128, 22272, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 14336, 23168, 128, 24064, 256, 128, 1664, 2304, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1024, 896, 29952, 128, 128, 20736, 12928, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[13696, 12672, 128, 128, 31488, 128, 768, 7040, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 70, 902, 4042, 0, 497, 0, 0, 1556, 0, 0, 0, 0, 415, 41, 0, 2955, 3114, 197, 4054, 0, 1647, 0, 0, 4019, 0, 0, 0, 63, 556, 4038, 0, 4022, 0, 0, 131, 3596, 115, 0, 3795, 56, 0, 1, 7, 1, 2, 3995, 0, 3399, 0, 0, 0, 26, 1, 3861, 4015, 2039, 0, 0, 1, 115, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[453, 0, 4094, 3912, 2, 2676, 1, 3, 77, 0, 0, 0, 34, 3331, 59, 0, 3910, 4025, 0, 4080, 0, 4090, 1, 0, 4094, 0, 0, 9, 0, 7, 4094, 0, 42, 0, 20, 1931, 4076, 0, 0, 3497, 0, 0, 2397, 3, 97, 0, 48, 1, 4038, 9, 0, 0, 125, 0, 2016, 4074, 0, 32, 0, 0, 2, 83, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 3944, 4084, 4, 3215, 240, 0, 2, 904, 0, 304, 0, 1703, 38, 10, 4, 4049, 0, 4083, 2, 200, 0, 0, 4091, 0, 0, 0, 0, 1, 4095, 0, 3572, 115, 0, 3456, 4044, 0, 0, 4096, 0, 0, 0, 1, 45, 0, 3982, 0, 4087, 0, 0, 19, 227, 46, 1754, 3896, 10, 0, 2, 0, 0, 1054, 87, 4070] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 2, 4058, 4046, 43, 4041, 0, 0, 42, 1, 1, 0, 0, 1898, 6, 0, 8, 3929, 0, 4096, 10, 1926, 1, 197, 4010, 0, 1, 71, 0, 0, 4044, 36, 4004, 88, 0, 4000, 4026, 7, 2, 4061, 47, 0, 11, 85, 22, 50, 0, 15, 4080, 0, 13, 0, 0, 163, 3974, 4085, 34, 0, 2, 0, 4, 2, 321, 3973] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[7, 1, 1014, 4095, 9, 393, 0, 9, 3764, 3, 0, 0, 1, 0, 0, 0, 1050, 4081, 0, 4095, 1, 898, 4, 289, 4083, 1, 89, 0, 8, 115, 4093, 0, 4096, 0, 0, 3950, 3327, 0, 1, 3747, 27, 0, 4059, 0, 8, 0, 3028, 0, 3860, 1, 0, 0, 0, 14, 3103, 4064, 161, 2, 0, 0, 5, 99, 77, 3804] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 8, 1893, 4063, 81, 1932, 333, 0, 4041, 0, 0, 0, 0, 3192, 0, 0, 286, 4096, 47, 4065, 2, 1394, 48, 4, 4083, 10, 14, 0, 0, 0, 4010, 0, 4076, 14, 9, 786, 3601, 1676, 16, 4048, 0, 0, 0, 0, 0, 0, 834, 0, 3931, 0, 0, 28, 42, 168, 19, 4052, 417, 12, 0, 0, 9, 3951, 185, 4060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1765, 3435, 32, 4092, 59, 0, 3810, 0, 0, 0, 0, 206, 0, 0, 93, 4094, 0, 4096, 0, 2, 0, 9, 4096, 0, 0, 0, 0, 0, 3841, 0, 2876, 11, 0, 450, 4096, 0, 0, 3984, 0, 0, 8, 0, 3800, 0, 183, 0, 4076, 0, 0, 0, 0, 170, 4027, 3931, 162, 0, 0, 0, 0, 4036, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4072, 1695, 0, 4019, 6, 2, 48, 865, 360, 3335, 621, 4031, 0, 7, 6, 3323, 488, 3671, 32, 647, 0, 282, 3898, 0, 596, 0, 2, 0, 4048, 53, 1869, 1673, 0, 932, 3810, 986, 0, 3336, 0, 0, 829, 0, 1516, 5, 32, 0, 4096, 3, 0, 0, 122, 551, 0, 4095, 2, 0, 0, 0, 30, 2476, 52, 3014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 70, 902, 2968, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1556, 0, 0, 0, 0, 415, 41, 0, 2281, 1116, 462, 0, 0, 0, 0, 0, 2955, 3114, 197, 1773, 0, 1647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 0, 0, 63, 556, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 131, 0, 115, 0, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 1, 7, 1, 2, 3995, 0, 3596, 1217, 0, 0, 0, 0, 0, 0, 3399, 0, 0, 0, 26, 1, 3861, 1682, 0, 0, 0, 0, 0, 0, 0, 0, 2039, 0, 0, 1, 115, 4096, 0, 4096, 1074, 773, 90, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4437, 5871, 9686, 8586, 6828, 8875, 8969, 12284] avg=8192.0000 max=12284 max_rank=7 min=4437 min_rank=0 imbalance_ratio=1.499512 max_load_reduction=53252 max_load_reduction_pct=0.812561 moved_to_echo=36956 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4058, 2971, 43, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1, 1, 0, 0, 1898, 6, 0, 2305, 1136, 494, 0, 0, 0, 0, 0, 8, 3929, 0, 1791, 10, 1926, 1, 197, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 0, 1, 71, 0, 0, 4044, 36, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 88, 0, 4000, 0, 7, 2, 2740, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 11, 85, 22, 50, 0, 15, 4026, 1238, 0, 0, 0, 0, 0, 0, 4080, 0, 13, 0, 0, 163, 3974, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 2, 0, 4, 2, 321, 3973, 1075, 827, 90, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11115, 5883, 7862, 8072, 10841, 5494, 9941, 6328] avg=8192.0000 max=11115 max_rank=0 min=5494 min_rank=5 imbalance_ratio=1.356812 max_load_reduction=54421 max_load_reduction_pct=0.830399 moved_to_echo=32604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3944, 2999, 4, 3215, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 904, 0, 304, 0, 1703, 38, 10, 2297, 1083, 499, 0, 0, 0, 0, 0, 4, 4049, 0, 1786, 2, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3572, 115, 0, 3456, 0, 0, 0, 2763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 45, 0, 3982, 0, 4044, 1181, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 19, 227, 46, 1754, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2, 0, 0, 1054, 87, 4070, 1085, 834, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10402, 6840, 6041, 8095, 9906, 9253, 7765, 7234] avg=8192.0000 max=10402 max_rank=0 min=6041 min_rank=2 imbalance_ratio=1.269775 max_load_reduction=55134 max_load_reduction_pct=0.841278 moved_to_echo=34158 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[453, 0, 4094, 2873, 2, 2676, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 34, 3331, 59, 0, 2296, 1133, 426, 0, 0, 0, 0, 0, 3910, 4025, 0, 1784, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 9, 0, 7, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 20, 1931, 0, 0, 0, 2359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2397, 3, 97, 0, 48, 1, 4076, 1235, 0, 0, 0, 0, 0, 0, 4038, 9, 0, 0, 125, 0, 2016, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 2, 83, 0, 4093, 1039, 712, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10102, 7356, 13810, 8112, 4352, 7857, 7894, 6053] avg=8192.0000 max=13810 max_rank=2 min=4352 min_rank=4 imbalance_ratio=1.685791 max_load_reduction=51726 max_load_reduction_pct=0.789276 moved_to_echo=26156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 1893, 2983, 81, 1932, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 3192, 0, 0, 2287, 1127, 493, 0, 0, 0, 0, 0, 286, 4096, 47, 1778, 2, 1394, 48, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 10, 14, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 14, 9, 786, 0, 1676, 16, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 834, 0, 3601, 1228, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 28, 42, 168, 19, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 417, 12, 0, 0, 9, 3951, 185, 4060, 1080, 824, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7230, 11140, 7655, 8025, 9308, 5663, 5885, 10630] avg=8192.0000 max=11140 max_rank=1 min=5663 min_rank=5 imbalance_ratio=1.359863 max_load_reduction=54396 max_load_reduction_pct=0.830017 moved_to_echo=31486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[7, 1, 1014, 3007, 9, 393, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3764, 3, 0, 0, 1, 0, 0, 0, 2304, 1130, 456, 0, 0, 0, 0, 0, 1050, 4081, 0, 1791, 1, 898, 4, 289, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 1, 89, 0, 8, 115, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3950, 0, 0, 1, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4059, 0, 8, 0, 3028, 0, 3327, 1231, 0, 0, 0, 0, 0, 0, 3860, 1, 0, 0, 0, 14, 3103, 1703, 0, 0, 0, 0, 0, 0, 0, 0, 161, 2, 0, 0, 5, 99, 77, 3804, 1088, 763, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4440, 7658, 8114, 8297, 10575, 11680, 8681, 6091] avg=8192.0000 max=11680 max_rank=5 min=4440 min_rank=0 imbalance_ratio=1.425781 max_load_reduction=53856 max_load_reduction_pct=0.821777 moved_to_echo=37027 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1765, 2522, 32, 4092, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 0, 0, 0, 0, 206, 0, 0, 2305, 1093, 485, 0, 0, 0, 0, 0, 93, 4094, 0, 1791, 0, 2, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 0, 0, 0, 0, 3841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2876, 11, 0, 450, 0, 0, 0, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3800, 0, 183, 0, 4096, 1191, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 170, 4027, 1647, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 4036, 0, 4096, 913, 811, 92, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8470, 7899, 5989, 7845, 6025, 9278, 9920, 10110] avg=8192.0000 max=10110 max_rank=7 min=5989 min_rank=2 imbalance_ratio=1.234131 max_load_reduction=55426 max_load_reduction_pct=0.845734 moved_to_echo=35333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4072, 1241, 0, 4019, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 48, 865, 360, 3335, 621, 4031, 0, 7, 2070, 1142, 410, 0, 0, 0, 0, 0, 6, 3323, 488, 1601, 32, 647, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 3805, 0, 596, 0, 2, 0, 4048, 53, 0, 0, 0, 0, 0, 0, 0, 0, 1869, 1673, 0, 932, 0, 986, 0, 2241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 829, 0, 1516, 5, 32, 0, 3810, 1244, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 122, 551, 0, 1709, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 30, 2476, 52, 3014, 454, 685, 93, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9340, 12889, 6379, 8504, 7701, 7436, 6481, 6806] avg=8192.0000 max=12889 max_rank=1 min=6379 min_rank=2 imbalance_ratio=1.573364 max_load_reduction=52647 max_load_reduction_pct=0.803329 moved_to_echo=28424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[24576, 1920, 128, 15744, 0, 2816, 128, 20736, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 128, 21760, 21632, 256, 20992, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[13440, 1792, 384, 3712, 768, 14848, 256, 128, 18176, 8960, 3840, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31744, 128, 768, 128, 128, 768, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 30720, 768, 14208, 128, 10880, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31616, 128, 128, 128, 640, 1152, 18816, 13568, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2944, 128, 128, 128, 256, 15872, 768, 31232, 7808, 6272, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 7424, 128, 5504, 128, 12160, 128, 30592, 9856, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[353, 0, 0, 2, 0, 0, 1, 2938, 0, 0, 409, 3899, 3284, 8, 11, 4086, 0, 2, 37, 122, 1, 3928, 4079, 0, 0, 0, 24, 0, 2, 4077, 66, 0, 4081, 3972, 4088, 247, 0, 0, 4096, 4086, 478, 0, 9, 5, 3949, 0, 102, 0, 1, 0, 10, 2, 4039, 3497, 0, 10, 262, 661, 1, 519, 4088, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 132, 301, 1211, 0, 0, 0, 2, 0, 0, 70, 4095, 4095, 0, 9, 4092, 0, 0, 0, 0, 5, 4086, 3883, 0, 11, 0, 0, 0, 0, 4017, 0, 4, 4052, 4017, 4042, 0, 25, 1, 4094, 4096, 2508, 0, 3, 0, 3233, 0, 66, 0, 0, 164, 0, 0, 4096, 3612, 0, 0, 0, 67, 1, 1303, 4096, 2, 45, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 433, 0, 0, 8, 0, 0, 4096, 0, 298, 0, 3568, 3859, 0, 0, 3760, 0, 119, 0, 281, 3652, 4029, 3908, 0, 0, 0, 0, 0, 0, 4095, 6, 172, 3844, 4096, 3071, 216, 274, 0, 4013, 3819, 0, 123, 708, 1, 4043, 0, 0, 8, 0, 111, 0, 0, 4081, 488, 0, 22, 0, 1, 0, 0, 4096, 0, 237, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 2, 0, 10, 0, 21, 4011, 8, 0, 142, 3401, 1367, 2, 0, 4046, 308, 265, 4, 11, 179, 3874, 4082, 0, 0, 445, 37, 0, 0, 4007, 0, 2116, 4080, 4096, 0, 310, 0, 0, 4075, 4048, 16, 0, 3895, 0, 4093, 0, 0, 0, 7, 233, 0, 11, 4095, 4085, 0, 25, 0, 15, 10, 0, 4090, 0, 0, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 17, 0, 15, 3, 3854, 45, 118, 3, 3848, 4051, 0, 0, 4088, 0, 121, 14, 3, 2, 4093, 4058, 0, 14, 10, 3, 0, 0, 3953, 60, 8, 3483, 4060, 3541, 0, 1, 0, 4087, 4096, 0, 0, 318, 33, 4057, 0, 0, 1, 23, 2, 1, 0, 4036, 4095, 0, 34, 1, 197, 562, 0, 4037, 0, 490, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 3, 28, 24, 0, 0, 1, 478, 0, 183, 0, 4016, 3790, 0, 0, 4083, 23, 4, 1, 9, 3793, 1579, 3445, 0, 0, 40, 69, 0, 0, 3249, 0, 3652, 4096, 4068, 317, 75, 0, 0, 4096, 4081, 1408, 8, 0, 25, 4095, 0, 1, 4, 0, 0, 0, 2, 4029, 4053, 0, 4, 0, 0, 212, 2386, 4095, 0, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3982, 0, 0, 0, 4059, 4032, 0, 0, 4096, 0, 355, 0, 0, 2557, 3236, 2897, 0, 0, 128, 0, 0, 0, 3744, 0, 360, 4096, 4084, 11, 779, 0, 0, 3959, 4077, 2, 0, 948, 0, 3519, 0, 0, 0, 0, 3859, 0, 0, 4085, 2457, 0, 0, 8, 2, 0, 43, 4096, 0, 65, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[17, 999, 22, 3, 15, 66, 0, 3841, 47, 0, 0, 3263, 134, 0, 0, 2005, 725, 2648, 1, 0, 4093, 312, 3888, 14, 32, 5, 0, 284, 14, 2250, 1619, 1, 77, 1582, 4064, 2, 0, 557, 3742, 4081, 0, 0, 3560, 0, 3962, 0, 1, 19, 0, 283, 0, 222, 3825, 3378, 0, 0, 905, 159, 1391, 32, 4084, 0, 3312, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[353, 0, 0, 2, 0, 0, 1, 2938, 4086, 798, 0, 0, 0, 0, 0, 0, 0, 0, 409, 3899, 3284, 8, 11, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 37, 122, 1, 3928, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 2, 4077, 66, 0, 3298, 143, 0, 0, 0, 0, 0, 0, 4081, 2135, 4088, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 9, 5, 3949, 0, 102, 0, 2670, 74, 0, 0, 0, 0, 0, 0, 1, 0, 10, 2, 4039, 3497, 0, 10, 345, 0, 0, 0, 0, 0, 0, 0, 262, 661, 1, 519, 4088, 4, 0, 0, 1763, 894, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8884, 6930, 7610, 10551, 7287, 7904, 8192] avg=8192.0000 max=10551 max_rank=4 min=6930 min_rank=2 imbalance_ratio=1.287964 max_load_reduction=54985 max_load_reduction_pct=0.839005 moved_to_echo=33934 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 433, 0, 0, 8, 0, 0, 4096, 3819, 781, 0, 0, 0, 0, 0, 0, 0, 298, 0, 3568, 3859, 0, 0, 1172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 281, 3652, 4029, 2722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 172, 3232, 131, 0, 0, 0, 0, 0, 0, 3844, 2202, 3071, 216, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 708, 1, 4043, 0, 0, 8, 2457, 76, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 4081, 488, 0, 22, 330, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 237, 0, 1818, 856, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9137, 8897, 10803, 7636, 9607, 7416, 5032, 7008] avg=8192.0000 max=10803 max_rank=2 min=5032 min_rank=6 imbalance_ratio=1.318726 max_load_reduction=54733 max_load_reduction_pct=0.835159 moved_to_echo=29063 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 17, 0, 15, 3, 3854, 4096, 795, 0, 0, 0, 0, 0, 0, 45, 118, 3, 3848, 4051, 0, 0, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 14, 3, 2, 4093, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 10, 3, 0, 0, 3953, 60, 8, 3292, 143, 0, 0, 0, 0, 0, 0, 3483, 2182, 3541, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 33, 4057, 0, 0, 1, 2671, 76, 0, 0, 0, 0, 0, 0, 23, 2, 1, 0, 4036, 4095, 0, 34, 343, 0, 0, 0, 0, 0, 0, 0, 1, 197, 562, 0, 4037, 0, 490, 0, 1802, 889, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8780, 9339, 7059, 7483, 9207, 7156, 8534, 7978] avg=8192.0000 max=9339 max_rank=1 min=7059 min_rank=2 imbalance_ratio=1.140015 max_load_reduction=56197 max_load_reduction_pct=0.857498 moved_to_echo=32875 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 2, 0, 10, 0, 21, 4011, 4048, 793, 0, 0, 0, 0, 0, 0, 8, 0, 142, 3401, 1367, 2, 0, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 308, 265, 4, 11, 179, 3874, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 37, 0, 0, 4007, 0, 2116, 3282, 141, 0, 0, 0, 0, 0, 0, 4080, 2202, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3895, 0, 4093, 0, 0, 0, 2644, 76, 0, 0, 0, 0, 0, 0, 7, 233, 0, 11, 4095, 4085, 0, 25, 345, 0, 0, 0, 0, 0, 0, 0, 0, 15, 10, 0, 4090, 0, 0, 12, 1818, 895, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8887, 6181, 7483, 10028, 6592, 10724, 8801, 6840] avg=8192.0000 max=10724 max_rank=5 min=6181 min_rank=1 imbalance_ratio=1.309082 max_load_reduction=54812 max_load_reduction_pct=0.836365 moved_to_echo=32957 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 132, 301, 1211, 0, 0, 0, 2, 4096, 796, 0, 0, 0, 0, 0, 0, 0, 0, 70, 4095, 4095, 0, 9, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4086, 2704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4017, 0, 4, 3298, 143, 0, 0, 0, 0, 0, 0, 4052, 2159, 4042, 0, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2508, 0, 3, 0, 3233, 0, 66, 0, 2674, 75, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 4096, 3612, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 67, 1, 1303, 4096, 2, 45, 0, 1783, 851, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6538, 9544, 6795, 7473, 10279, 8559, 8200, 8148] avg=8192.0000 max=10279 max_rank=4 min=6538 min_rank=0 imbalance_ratio=1.254761 max_load_reduction=55257 max_load_reduction_pct=0.843155 moved_to_echo=35186 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 28, 24, 0, 0, 1, 478, 4081, 798, 0, 0, 0, 0, 0, 0, 0, 183, 0, 4016, 3790, 0, 0, 1272, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4, 1, 9, 3793, 1579, 2399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 69, 0, 0, 3249, 0, 3652, 3298, 143, 0, 0, 0, 0, 0, 0, 4096, 2187, 317, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1408, 8, 0, 25, 4095, 0, 1, 4, 2668, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4029, 4053, 0, 4, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 2386, 4095, 0, 11, 0, 1805, 755, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5413, 9261, 7808, 10451, 6675, 8285, 8379, 9264] avg=8192.0000 max=10451 max_rank=3 min=5413 min_rank=0 imbalance_ratio=1.275757 max_load_reduction=55085 max_load_reduction_pct=0.840530 moved_to_echo=32603 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3982, 4077, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4032, 0, 0, 1277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 0, 2557, 3236, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 3744, 0, 360, 3188, 143, 0, 0, 0, 0, 0, 0, 4096, 2195, 11, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 948, 0, 3519, 0, 0, 0, 2676, 76, 0, 0, 0, 0, 0, 0, 0, 3859, 0, 0, 4085, 2457, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 43, 4096, 0, 65, 0, 1813, 635, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8830, 9368, 8165, 7563, 7081, 7221, 10646, 6662] avg=8192.0000 max=10646 max_rank=6 min=6662 min_rank=7 imbalance_ratio=1.299561 max_load_reduction=54890 max_load_reduction_pct=0.837555 moved_to_echo=31610 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[17, 999, 22, 3, 15, 66, 0, 3841, 4081, 729, 0, 0, 0, 0, 0, 0, 47, 0, 0, 3263, 134, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 725, 2648, 1, 0, 4093, 312, 2700, 14, 0, 0, 0, 0, 0, 0, 0, 0, 32, 5, 0, 284, 14, 2250, 1619, 1, 3013, 74, 0, 0, 0, 0, 0, 0, 77, 844, 4064, 2, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 3962, 0, 1, 19, 1313, 33, 0, 0, 0, 0, 0, 0, 0, 283, 0, 222, 3825, 3378, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 905, 159, 1391, 32, 4084, 0, 3312, 0, 705, 856, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9773, 4062, 10493, 7292, 5544, 8888, 8040, 11444] avg=8192.0000 max=11444 max_rank=7 min=4062 min_rank=1 imbalance_ratio=1.396973 max_load_reduction=54092 max_load_reduction_pct=0.825378 moved_to_echo=33916 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 640, 640, 30208, 24704, 128, 128, 9472, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4480, 256, 9472, 128, 30976, 0, 256, 128, 19840, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 4736, 128, 256, 32384, 25728, 0, 128, 2560, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 1664, 384, 1280, 128, 128, 128, 23296, 32384, 6272, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1152, 3584, 128, 512, 14336, 25216, 21120, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1280, 1152, 2304, 4352, 32768, 128, 4224, 128, 13312, 6656, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 640, 256, 384, 128, 29440, 1792, 6400, 25984, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[27904, 16128, 19200, 1664, 384, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[59, 4081, 52, 9, 20, 0, 0, 329, 2119, 958, 0, 0, 1, 4084, 0, 4092, 0, 2733, 1, 4096, 0, 4083, 4046, 1, 0, 0, 0, 0, 4072, 0, 4086, 0, 3520, 12, 3951, 0, 5, 21, 17, 0, 0, 0, 2788, 50, 0, 4055, 2, 0, 0, 1, 43, 188, 0, 16, 0, 1, 4080, 0, 0, 0, 3981, 0, 0, 3883] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4076, 53, 0, 3, 0, 0, 845, 436, 2, 0, 0, 4067, 4090, 0, 4096, 0, 0, 0, 4093, 122, 4026, 3043, 0, 0, 0, 0, 0, 4038, 2, 4096, 1, 3606, 0, 1371, 1, 0, 148, 4, 58, 0, 1, 2603, 0, 0, 4086, 1, 0, 180, 2, 4096, 0, 0, 5, 0, 0, 4084, 0, 0, 0, 4093, 0, 12, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 1, 1, 4082, 0, 30, 0, 0, 4086, 26, 4096, 0, 0, 1259, 3227, 0, 4096, 1915, 279, 381, 17, 0, 0, 911, 0, 4048, 0, 420, 56, 4065, 20, 0, 1, 4067, 3429, 375, 0, 3374, 0, 58, 4096, 4, 0, 614, 173, 0, 654, 3, 0, 0, 10, 3908, 0, 0, 7, 3603, 3, 58, 3987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 2, 4087, 14, 0, 0, 105, 4096, 0, 4096, 0, 1, 15, 4096, 0, 4010, 4093, 345, 0, 1, 0, 0, 3545, 0, 4096, 0, 45, 0, 2677, 0, 0, 293, 4096, 0, 0, 25, 2646, 0, 0, 4065, 0, 0, 2450, 0, 84, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 174, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 37, 1, 1, 1329, 236, 1, 55, 53, 4094, 8, 4096, 8, 0, 5, 4096, 0, 3827, 4083, 35, 0, 0, 0, 0, 3391, 18, 4091, 0, 4072, 1, 4096, 0, 0, 4094, 577, 80, 0, 0, 10, 0, 3, 3780, 4, 0, 0, 0, 2994, 0, 20, 0, 0, 0, 4084, 0, 0, 7, 4018, 39, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[50, 3742, 0, 0, 24, 15, 0, 12, 30, 0, 0, 34, 0, 4055, 0, 4076, 0, 1, 4087, 4090, 0, 4085, 3945, 2, 0, 0, 1, 3, 4051, 62, 4094, 0, 4055, 0, 4090, 0, 12, 4035, 3, 5, 1, 0, 3, 0, 0, 4048, 0, 45, 713, 0, 43, 303, 0, 22, 143, 0, 4051, 0, 40, 0, 3366, 124, 3, 3972] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4084, 0, 0, 0, 0, 6, 1, 4094, 161, 0, 0, 821, 4040, 0, 4069, 0, 249, 0, 4042, 0, 3966, 3915, 91, 0, 32, 0, 0, 4092, 0, 4093, 0, 527, 0, 2654, 0, 0, 1804, 3505, 17, 0, 0, 2474, 10, 0, 4094, 1, 71, 3436, 0, 211, 22, 18, 144, 19, 0, 3962, 0, 13, 0, 955, 3, 0, 3840] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4091, 136, 0, 0, 0, 112, 4, 3583, 0, 0, 0, 10, 4096, 31, 4006, 0, 3416, 1, 3878, 5, 4056, 2268, 2, 1150, 0, 117, 0, 1204, 0, 4089, 11, 4, 0, 10, 0, 0, 4079, 1826, 1952, 0, 0, 590, 0, 0, 2721, 1863, 0, 3731, 0, 3, 0, 0, 143, 0, 0, 3953, 29, 0, 183, 3804, 4075, 0, 304] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[59, 4081, 52, 9, 20, 0, 0, 329, 2085, 1866, 0, 0, 0, 0, 0, 0, 2119, 958, 0, 0, 1, 819, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2733, 1, 3314, 0, 0, 4046, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4086, 0, 743, 0, 0, 0, 0, 0, 0, 0, 3520, 12, 2599, 0, 5, 21, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2788, 50, 0, 4055, 2, 0, 1180, 1352, 39, 0, 0, 0, 0, 0, 0, 1, 43, 188, 0, 16, 0, 1, 4083, 1633, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 3981, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8501, 7989, 10095, 8901, 6174, 9466, 5965, 8445] avg=8192.0000 max=10095 max_rank=2 min=5965 min_rank=6 imbalance_ratio=1.232300 max_load_reduction=55441 max_load_reduction_pct=0.845963 moved_to_echo=30050 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[50, 3742, 0, 0, 24, 15, 0, 12, 2070, 1853, 0, 0, 0, 0, 0, 0, 30, 0, 0, 34, 0, 813, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 3309, 0, 0, 3945, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4051, 62, 4094, 0, 742, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 2690, 0, 12, 4035, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 4048, 0, 45, 1172, 1400, 39, 0, 0, 0, 0, 0, 713, 0, 43, 303, 0, 22, 143, 0, 4085, 1622, 0, 0, 0, 0, 0, 0, 576, 0, 40, 0, 3366, 124, 3, 3972, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7766, 4953, 11344, 8953, 10800, 6708, 6931, 8081] avg=8192.0000 max=11344 max_rank=2 min=4953 min_rank=1 imbalance_ratio=1.384766 max_load_reduction=54192 max_load_reduction_pct=0.826904 moved_to_echo=32520 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 1, 1, 2086, 1787, 0, 0, 0, 0, 0, 0, 4082, 0, 30, 0, 0, 819, 26, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1259, 2610, 0, 0, 1915, 279, 0, 0, 0, 0, 0, 0, 0, 0, 381, 17, 0, 0, 911, 0, 4048, 0, 586, 0, 0, 0, 0, 0, 0, 0, 420, 56, 2674, 20, 0, 1, 4067, 3429, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 3374, 0, 58, 4096, 4, 0, 1181, 1391, 31, 0, 0, 0, 0, 0, 614, 173, 0, 654, 3, 0, 0, 10, 4096, 1565, 0, 0, 0, 0, 0, 0, 556, 0, 0, 7, 3603, 3, 58, 3987, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7971, 9053, 6063, 5943, 10667, 10510, 7115, 8214] avg=8192.0000 max=10667 max_rank=4 min=5943 min_rank=3 imbalance_ratio=1.302124 max_load_reduction=54869 max_load_reduction_pct=0.837234 moved_to_echo=36506 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4076, 53, 0, 3, 0, 0, 845, 2088, 1868, 0, 0, 0, 0, 0, 0, 436, 2, 0, 0, 4067, 820, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3311, 122, 0, 3043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 2, 4096, 1, 743, 0, 0, 0, 0, 0, 0, 0, 3606, 0, 902, 1, 0, 148, 4, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2603, 0, 0, 4086, 1, 0, 1182, 469, 39, 0, 0, 0, 0, 0, 180, 2, 4096, 0, 0, 5, 0, 0, 4026, 1635, 0, 0, 0, 0, 0, 0, 581, 0, 0, 0, 4093, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8933, 9421, 6476, 8880, 4719, 8381, 9944, 8782] avg=8192.0000 max=9944 max_rank=6 min=4719 min_rank=4 imbalance_ratio=1.213867 max_load_reduction=55592 max_load_reduction_pct=0.848267 moved_to_echo=31826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 0, 0, 0, 0, 6, 1, 2062, 1812, 0, 0, 0, 0, 0, 0, 4094, 161, 0, 0, 821, 810, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 3270, 0, 0, 3915, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 4092, 0, 4093, 0, 734, 0, 0, 0, 0, 0, 0, 0, 527, 0, 1746, 0, 0, 1804, 3505, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2474, 10, 0, 4094, 1, 71, 1168, 908, 38, 0, 0, 0, 0, 0, 3436, 0, 211, 22, 18, 144, 19, 0, 3966, 1586, 0, 0, 0, 0, 0, 0, 564, 0, 13, 0, 955, 3, 0, 3840, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7965, 9955, 7525, 8951, 7599, 8764, 9402, 5375] avg=8192.0000 max=9955 max_rank=1 min=5375 min_rank=7 imbalance_ratio=1.215210 max_load_reduction=55581 max_load_reduction_pct=0.848099 moved_to_echo=31140 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 37, 1, 1, 2090, 1868, 0, 0, 0, 0, 0, 0, 1329, 236, 1, 55, 53, 821, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 5, 3314, 0, 0, 4083, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3391, 18, 4091, 0, 743, 0, 0, 0, 0, 0, 0, 0, 4072, 1, 2694, 0, 0, 4094, 577, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3, 3780, 4, 0, 1183, 1402, 39, 0, 0, 0, 0, 0, 0, 0, 2994, 0, 20, 0, 0, 0, 3827, 1635, 0, 0, 0, 0, 0, 0, 581, 0, 0, 7, 4018, 39, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8093, 6599, 7445, 8243, 11518, 6421, 8476, 8741] avg=8192.0000 max=11518 max_rank=4 min=6421 min_rank=5 imbalance_ratio=1.406006 max_load_reduction=54018 max_load_reduction_pct=0.824249 moved_to_echo=35156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 2, 2091, 1872, 0, 0, 0, 0, 0, 0, 4087, 14, 0, 0, 105, 821, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 3314, 0, 0, 4093, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3545, 0, 4096, 0, 743, 0, 0, 0, 0, 0, 0, 0, 45, 0, 1761, 0, 0, 293, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2646, 0, 0, 4065, 0, 0, 1184, 916, 39, 0, 0, 0, 0, 0, 2450, 0, 84, 0, 0, 0, 0, 0, 4010, 1639, 0, 0, 0, 0, 0, 0, 583, 0, 0, 0, 4095, 174, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8059, 9123, 7768, 8385, 6195, 8875, 8183, 8948] avg=8192.0000 max=9123 max_rank=1 min=6195 min_rank=4 imbalance_ratio=1.113647 max_load_reduction=56413 max_load_reduction_pct=0.860794 moved_to_echo=32201 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 136, 0, 0, 0, 112, 4, 2094, 1811, 0, 0, 0, 0, 0, 0, 3583, 0, 0, 0, 10, 813, 31, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3416, 1, 3128, 5, 0, 2268, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 0, 117, 0, 1204, 0, 4089, 11, 709, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 0, 0, 4079, 1826, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 2721, 1863, 0, 1189, 7, 41, 0, 0, 0, 0, 0, 3731, 0, 3, 0, 0, 143, 0, 0, 4056, 1587, 0, 0, 0, 0, 0, 0, 555, 29, 0, 183, 3804, 4075, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8443, 8820, 7280, 7864, 6411, 9520, 8950] avg=8192.0000 max=9520 max_rank=6 min=6411 min_rank=5 imbalance_ratio=1.162109 max_load_reduction=56016 max_load_reduction_pct=0.854736 moved_to_echo=32745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 6400, 5376, 25600, 128, 0, 27392, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4608, 128, 128, 256, 28032, 4480, 128, 28288, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[19840, 1408, 128, 128, 5120, 6656, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[16256, 128, 15104, 128, 128, 14592, 14208, 5632, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 128, 14592, 128, 128, 30976, 1920, 128, 9472, 7936, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1536, 128, 128, 128, 25344, 128, 32768, 128, 5760, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32384, 256, 128, 128, 128, 128, 1280, 16768, 14848, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11136, 256, 7552, 1280, 128, 384, 256, 128, 32256, 12928, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4071, 0, 9, 0, 4008, 0, 3914, 4079, 41, 0, 58, 0, 17, 0, 0, 0, 0, 781, 0, 390, 8, 997, 0, 0, 0, 3340, 4074, 0, 4059, 0, 2, 54, 103, 20, 15, 4, 22, 0, 2209, 1141, 4, 0, 4096, 3949, 9, 1, 0, 87, 0, 4060, 4069, 0, 24, 1, 96, 4040, 3536, 0, 0, 4020, 2, 4079, 0, 47] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4096, 8, 0, 0, 4089, 0, 1711, 3543, 0, 2, 0, 310, 0, 0, 0, 0, 0, 242, 0, 13, 0, 32, 0, 0, 0, 0, 4090, 372, 2106, 1, 0, 0, 2019, 0, 1, 0, 4, 0, 2212, 287, 0, 1, 1625, 3666, 36, 673, 0, 4096, 0, 4094, 4095, 0, 4096, 1, 5, 3942, 4090, 0, 1, 4080, 3515, 2382, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3918, 61, 0, 7, 4096, 0, 233, 4055, 0, 40, 0, 0, 381, 213, 0, 31, 2, 0, 0, 0, 0, 110, 0, 0, 0, 0, 4092, 195, 808, 48, 0, 1201, 0, 0, 5, 0, 0, 123, 0, 3861, 1, 14, 3173, 3981, 1, 0, 0, 3070, 0, 4074, 4086, 0, 3456, 0, 0, 4071, 4096, 0, 12, 4093, 3886, 4042, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 4055, 54, 3985, 1, 607, 4096, 0, 69, 0, 0, 244, 59, 68, 0, 6, 492, 0, 2, 3, 0, 0, 19, 8, 0, 3919, 855, 4060, 0, 25, 40, 0, 0, 0, 1, 3, 1, 2937, 4094, 21, 0, 1888, 4039, 0, 17, 2, 4045, 0, 4057, 4094, 0, 2, 6, 110, 3722, 2808, 0, 0, 3735, 0, 3091, 9, 91] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4090, 4, 44, 368, 4096, 0, 3076, 4091, 4, 0, 0, 1, 3, 0, 0, 18, 5, 35, 0, 51, 34, 2, 3, 0, 0, 4096, 4077, 0, 3856, 0, 0, 566, 209, 1, 25, 0, 17, 0, 56, 4096, 0, 2, 1073, 4052, 0, 0, 0, 4093, 0, 4095, 4091, 0, 95, 0, 11, 3876, 4096, 0, 0, 4095, 209, 2824, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 215, 0, 4096, 0, 3925, 4096, 0, 0, 0, 0, 197, 0, 0, 0, 56, 4, 0, 0, 0, 0, 0, 0, 0, 7, 4094, 0, 2571, 0, 0, 35, 770, 0, 0, 0, 0, 0, 2, 3905, 0, 198, 4096, 3820, 0, 0, 0, 3357, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4087, 0, 0, 3400, 4085, 2131, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4072, 0, 5, 0, 4095, 0, 3603, 4094, 0, 0, 0, 368, 43, 0, 0, 33, 0, 104, 17, 0, 13, 0, 0, 0, 0, 2, 4095, 0, 4091, 138, 16, 11, 3, 0, 0, 0, 0, 0, 4, 4085, 0, 21, 4078, 4031, 33, 0, 0, 233, 7, 4096, 4056, 0, 0, 0, 0, 3984, 4091, 15, 0, 4095, 3953, 3948, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4086, 1118, 0, 0, 2465, 1, 1040, 4096, 0, 25, 338, 522, 209, 0, 0, 0, 7, 0, 299, 1308, 9, 0, 0, 0, 0, 4, 3, 0, 3023, 4, 0, 3955, 0, 51, 0, 0, 0, 0, 8, 4066, 26, 29, 620, 3987, 0, 0, 1, 4096, 220, 4096, 3739, 0, 4096, 0, 2493, 3358, 4096, 0, 0, 3970, 0, 4055, 0, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 4008, 0, 3914, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 58, 0, 17, 0, 0, 0, 4060, 3702, 0, 0, 0, 0, 0, 0, 0, 781, 0, 390, 8, 997, 0, 0, 4020, 2740, 560, 369, 0, 0, 0, 0, 0, 3340, 3723, 0, 4059, 0, 2, 54, 0, 0, 0, 0, 0, 0, 0, 0, 103, 20, 15, 4, 22, 0, 2209, 1141, 1353, 1025, 888, 351, 0, 0, 0, 0, 4, 0, 4096, 2596, 9, 1, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2484, 0, 24, 1, 96, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 0, 0, 0, 2, 4079, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9270, 7878, 9865, 11178, 7131, 6793, 6645, 6776] avg=8192.0000 max=11178 max_rank=3 min=6645 min_rank=6 imbalance_ratio=1.364502 max_load_reduction=54358 max_load_reduction_pct=0.829437 moved_to_echo=27345 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 61, 0, 7, 4096, 0, 233, 1331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 381, 213, 0, 31, 4074, 3562, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 110, 0, 0, 4093, 2724, 562, 356, 0, 0, 0, 0, 0, 0, 3739, 195, 808, 48, 0, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 123, 0, 3861, 1364, 1029, 1029, 353, 0, 0, 0, 0, 1, 14, 3173, 2617, 1, 0, 0, 3070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2495, 0, 3456, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 0, 12, 0, 3886, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5728, 8301, 7847, 5991, 7764, 8876, 10022, 11007] avg=8192.0000 max=11007 max_rank=7 min=5728 min_rank=0 imbalance_ratio=1.343628 max_load_reduction=54529 max_load_reduction_pct=0.832047 moved_to_echo=37669 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4055, 54, 3985, 1, 607, 1344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 244, 59, 68, 0, 4057, 3724, 0, 0, 0, 0, 0, 0, 6, 492, 0, 2, 3, 0, 0, 19, 3735, 2752, 563, 372, 0, 0, 0, 0, 8, 0, 3581, 855, 4060, 0, 25, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1, 2937, 4094, 1384, 1031, 705, 338, 0, 0, 0, 0, 21, 0, 1888, 2655, 0, 17, 2, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2500, 0, 2, 6, 110, 3722, 0, 0, 0, 0, 0, 0, 0, 0, 2103, 0, 0, 0, 0, 3091, 9, 91, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10046, 8221, 7944, 8569, 10494, 8628, 6340, 5294] avg=8192.0000 max=10494 max_rank=4 min=5294 min_rank=7 imbalance_ratio=1.281006 max_load_reduction=55042 max_load_reduction_pct=0.839874 moved_to_echo=30756 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 4089, 0, 1711, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 310, 0, 0, 0, 0, 4094, 3725, 0, 0, 0, 0, 0, 0, 0, 242, 0, 13, 0, 32, 0, 0, 4080, 2380, 563, 371, 0, 0, 0, 0, 0, 0, 3736, 372, 2106, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2019, 0, 1, 0, 4, 0, 2212, 287, 1256, 1032, 1027, 354, 0, 0, 0, 0, 0, 1, 1625, 2410, 36, 673, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2500, 0, 4096, 1, 5, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 3063, 0, 1, 0, 3515, 2382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6971, 8131, 7681, 6215, 8192, 8841, 10544, 8961] avg=8192.0000 max=10544 max_rank=6 min=6215 min_rank=3 imbalance_ratio=1.287109 max_load_reduction=54992 max_load_reduction_pct=0.839111 moved_to_echo=36538 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 4095, 0, 3603, 1344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 43, 0, 0, 33, 4096, 3702, 0, 0, 0, 0, 0, 0, 0, 104, 17, 0, 13, 0, 0, 0, 4095, 2750, 558, 370, 0, 0, 0, 0, 0, 2, 3742, 0, 4091, 138, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 4085, 1381, 1022, 1028, 353, 0, 0, 0, 0, 0, 21, 4078, 2650, 33, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2476, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 3063, 15, 0, 0, 3953, 3948, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9047, 8242, 7907, 8000, 7876, 7015, 6467, 10982] avg=8192.0000 max=10982 max_rank=7 min=6467 min_rank=6 imbalance_ratio=1.340576 max_load_reduction=54554 max_load_reduction_pct=0.832428 moved_to_echo=32340 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 44, 368, 4096, 0, 3076, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 3, 0, 0, 18, 4095, 3718, 0, 0, 0, 0, 0, 0, 5, 35, 0, 51, 34, 2, 3, 0, 4095, 2748, 563, 372, 0, 0, 0, 0, 0, 4096, 3725, 0, 3856, 0, 0, 566, 0, 0, 0, 0, 0, 0, 0, 0, 209, 1, 25, 0, 17, 0, 56, 4096, 1388, 1031, 1029, 352, 0, 0, 0, 0, 0, 2, 1073, 2664, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2497, 0, 95, 0, 11, 3876, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 0, 0, 0, 209, 2824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8931, 7839, 7908, 12243, 8204, 7832, 6479, 6100] avg=8192.0000 max=12243 max_rank=3 min=6100 min_rank=7 imbalance_ratio=1.494507 max_load_reduction=53293 max_load_reduction_pct=0.813187 moved_to_echo=28615 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 215, 0, 4096, 0, 3925, 1344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 4096, 3725, 0, 0, 0, 0, 0, 0, 56, 4, 0, 0, 0, 0, 0, 0, 3400, 2752, 563, 371, 0, 0, 0, 0, 0, 7, 3741, 0, 2571, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 770, 0, 0, 0, 0, 0, 2, 3905, 1309, 1032, 1027, 353, 0, 0, 0, 0, 0, 198, 4096, 2511, 0, 0, 0, 3357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2501, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3060, 0, 0, 0, 4085, 2131, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9580, 8018, 7146, 6354, 8398, 10162, 6597, 9281] avg=8192.0000 max=10162 max_rank=5 min=6354 min_rank=3 imbalance_ratio=1.240479 max_load_reduction=55374 max_load_reduction_pct=0.844940 moved_to_echo=34438 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1118, 0, 0, 2465, 1, 1040, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 338, 522, 209, 0, 0, 0, 4096, 3716, 0, 0, 0, 0, 0, 0, 7, 0, 299, 1308, 9, 0, 0, 0, 3970, 2757, 518, 370, 0, 0, 0, 0, 0, 4, 0, 0, 3023, 4, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 8, 4066, 1370, 946, 1033, 3, 0, 0, 0, 0, 26, 29, 620, 2617, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 2275, 0, 4096, 0, 2493, 3358, 0, 0, 0, 0, 0, 0, 0, 0, 3063, 0, 0, 0, 0, 4055, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5963, 8906, 9238, 6986, 7477, 7389, 12442, 7135] avg=8192.0000 max=12442 max_rank=6 min=5963 min_rank=0 imbalance_ratio=1.518799 max_load_reduction=53094 max_load_reduction_pct=0.810150 moved_to_echo=34443 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[23168, 128, 128, 0, 15744, 26624, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 0, 19840, 0, 11776, 128, 2816, 31104, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[3200, 128, 128, 128, 128, 128, 7552, 25600, 10880, 8192, 7808, 2560, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 1280, 4352, 512, 30976, 128, 18176, 10624, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 384, 20736, 20736, 128, 768, 128, 23168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 256, 512, 1280, 1152, 384, 128, 128, 32768, 29696, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 1664, 384, 1792, 128, 1152, 128, 128, 31488, 21632, 4480, 3072, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 7552, 26112, 1536, 24576, 256, 128, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 4044, 36, 2, 4041, 0, 3999, 2, 0, 36, 2, 5, 1, 0, 43, 0, 2, 1941, 3642, 0, 442, 3925, 0, 0, 73, 4001, 0, 0, 0, 5, 4093, 1, 0, 3249, 3953, 95, 3656, 0, 0, 4086, 0, 4094, 876, 0, 0, 17, 0, 231, 2348, 74, 0, 9, 153, 0, 0, 13, 4080, 0, 35, 0, 46, 0, 4096, 4089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[19, 4063, 58, 0, 2528, 0, 4086, 13, 122, 1, 0, 0, 12, 0, 0, 117, 0, 3462, 4088, 0, 0, 4053, 0, 0, 3, 4092, 23, 802, 0, 0, 3959, 24, 0, 2964, 4096, 0, 78, 0, 0, 4089, 2, 4092, 5, 1, 0, 0, 0, 3539, 1275, 0, 0, 5, 0, 0, 0, 6, 4096, 8, 0, 0, 0, 1632, 4084, 4039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 3825, 27, 2860, 3484, 0, 4096, 1, 85, 0, 0, 35, 0, 0, 95, 0, 0, 3638, 4077, 0, 145, 4037, 119, 0, 0, 1371, 0, 0, 0, 5, 4089, 0, 0, 3486, 4096, 0, 0, 0, 18, 4093, 0, 4066, 0, 0, 1, 0, 14, 1588, 0, 56, 0, 0, 45, 7, 8, 29, 4096, 0, 1470, 1, 4, 2570, 3967, 3932] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1, 4087, 91, 3932, 607, 0, 4015, 0, 0, 0, 0, 1, 0, 0, 0, 9, 0, 3375, 541, 17, 0, 29, 0, 2, 0, 4038, 4, 0, 9, 88, 1059, 0, 0, 3411, 4096, 409, 4000, 4096, 0, 4094, 0, 3005, 0, 0, 0, 1, 0, 3819, 90, 24, 8, 0, 0, 5, 55, 514, 4086, 0, 0, 3, 0, 3823, 4095, 3997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[20, 3835, 0, 51, 3878, 12, 4084, 37, 1051, 0, 12, 3, 0, 0, 0, 0, 3, 3141, 1461, 36, 0, 3735, 25, 0, 0, 4070, 0, 0, 1, 0, 2744, 0, 0, 4063, 4082, 0, 4048, 1645, 0, 4025, 508, 4096, 0, 0, 1, 0, 54, 2049, 197, 61, 138, 12, 0, 0, 0, 10, 4008, 0, 226, 0, 7, 1, 4043, 4063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 252, 3960, 5, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 2793, 3693, 0, 4010, 0, 0, 0, 1467, 0, 1009, 0, 0, 4096, 0, 0, 4089, 4095, 0, 4024, 1277, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1306, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 166, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[38, 4094, 45, 1734, 16, 0, 4053, 0, 552, 0, 0, 0, 1, 0, 0, 0, 25, 3494, 3869, 2920, 0, 0, 0, 0, 1, 2184, 2, 49, 0, 3520, 4096, 4, 0, 4089, 3965, 0, 0, 4086, 1, 4095, 1, 4072, 0, 17, 124, 0, 2, 552, 852, 0, 0, 0, 0, 3, 2, 0, 4074, 14, 0, 3, 0, 726, 4069, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[29, 3816, 4096, 3775, 2447, 82, 3181, 0, 160, 0, 0, 498, 0, 0, 0, 0, 0, 2577, 4081, 359, 0, 0, 0, 0, 0, 4058, 453, 28, 19, 0, 2450, 986, 0, 3720, 4045, 0, 2796, 841, 12, 4017, 0, 3532, 1149, 0, 0, 19, 30, 381, 1, 0, 0, 0, 56, 10, 17, 0, 4096, 0, 670, 0, 0, 0, 3648, 3401] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 36, 2, 4041, 0, 3462, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 2, 5, 1, 0, 43, 0, 4080, 3769, 0, 0, 0, 0, 0, 0, 2, 1941, 2333, 0, 442, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 4001, 0, 0, 0, 5, 4093, 1, 537, 342, 0, 0, 0, 0, 0, 0, 0, 3249, 658, 95, 3656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 876, 0, 0, 17, 0, 231, 1236, 967, 241, 0, 0, 0, 0, 0, 2348, 74, 0, 9, 153, 0, 0, 13, 4044, 3054, 317, 0, 0, 0, 0, 0, 0, 0, 35, 0, 46, 0, 2860, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7543, 7936, 8643, 9052, 7658, 7662, 10012, 7030] avg=8192.0000 max=10012 max_rank=6 min=7030 min_rank=7 imbalance_ratio=1.222168 max_load_reduction=55524 max_load_reduction_pct=0.847229 moved_to_echo=32362 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 27, 2860, 3484, 0, 3546, 1, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 35, 0, 0, 95, 0, 4096, 3775, 0, 0, 0, 0, 0, 0, 0, 3638, 2611, 0, 145, 4037, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1371, 0, 0, 0, 5, 4089, 0, 550, 383, 0, 0, 0, 0, 0, 0, 0, 3486, 681, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 1, 0, 14, 1588, 1197, 1083, 250, 0, 0, 0, 0, 0, 0, 56, 0, 0, 45, 7, 8, 29, 3825, 3165, 318, 0, 0, 0, 0, 0, 0, 0, 1470, 1, 4, 2570, 2770, 3932, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9918, 8086, 10550, 6398, 4185, 8199, 7453, 10747] avg=8192.0000 max=10747 max_rank=7 min=4185 min_rank=4 imbalance_ratio=1.311890 max_load_reduction=54789 max_load_reduction_pct=0.836014 moved_to_echo=30584 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 0, 51, 3878, 12, 3535, 37, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 0, 12, 3, 0, 0, 0, 0, 4008, 3712, 0, 0, 0, 0, 0, 0, 3, 3141, 936, 36, 0, 3735, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 1, 0, 2744, 0, 549, 137, 0, 0, 0, 0, 0, 0, 0, 4063, 679, 0, 4048, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 4096, 0, 0, 1, 0, 54, 2049, 1220, 388, 249, 0, 0, 0, 0, 0, 197, 61, 138, 12, 0, 0, 0, 10, 3835, 3154, 313, 0, 0, 0, 0, 0, 0, 0, 226, 0, 7, 1, 2823, 4063, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7533, 8786, 7876, 7501, 10435, 8565, 7720, 7120] avg=8192.0000 max=10435 max_rank=4 min=7120 min_rank=7 imbalance_ratio=1.273804 max_load_reduction=55101 max_load_reduction_pct=0.840775 moved_to_echo=33840 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 58, 0, 2528, 0, 3537, 13, 0, 0, 0, 0, 0, 0, 0, 0, 122, 1, 0, 0, 12, 0, 0, 117, 4096, 3772, 0, 0, 0, 0, 0, 0, 0, 3462, 2618, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 23, 802, 0, 0, 3959, 24, 549, 384, 0, 0, 0, 0, 0, 0, 0, 2964, 681, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 5, 1, 0, 0, 0, 3539, 1232, 1086, 250, 0, 0, 0, 0, 0, 1275, 0, 0, 5, 0, 0, 0, 6, 4063, 3165, 317, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1632, 2852, 4039, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6155, 8120, 10133, 9836, 3723, 10207, 8831, 8531] avg=8192.0000 max=10207 max_rank=5 min=3723 min_rank=4 imbalance_ratio=1.245972 max_load_reduction=55329 max_load_reduction_pct=0.844254 moved_to_echo=31292 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 91, 3932, 607, 0, 3476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 4086, 3776, 0, 0, 0, 0, 0, 0, 0, 3375, 348, 17, 0, 29, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 4, 0, 9, 88, 1059, 0, 539, 50, 0, 0, 0, 0, 0, 0, 0, 3411, 681, 409, 4000, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3005, 0, 0, 0, 1, 0, 3819, 1235, 143, 250, 0, 0, 0, 0, 0, 90, 24, 8, 0, 0, 5, 55, 514, 4087, 3165, 318, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3823, 2860, 3997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 7872, 3771, 5787, 12597, 8453, 8266, 10683] avg=8192.0000 max=12597 max_rank=4 min=3771 min_rank=2 imbalance_ratio=1.537720 max_load_reduction=52939 max_load_reduction_pct=0.807785 moved_to_echo=39999 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[38, 0, 45, 1734, 16, 0, 3509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 0, 0, 0, 1, 0, 0, 0, 4074, 3776, 0, 0, 0, 0, 0, 0, 25, 3494, 2477, 2920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2184, 2, 49, 0, 3520, 4096, 4, 544, 364, 0, 0, 0, 0, 0, 0, 0, 4089, 659, 0, 0, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4072, 0, 17, 124, 0, 2, 552, 1228, 1028, 242, 0, 0, 0, 0, 0, 852, 0, 0, 0, 0, 3, 2, 0, 4094, 3064, 319, 0, 0, 0, 0, 0, 0, 14, 0, 3, 0, 726, 2841, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5342, 8403, 8916, 10764, 8835, 7266, 8334, 7676] avg=8192.0000 max=10764 max_rank=3 min=5342 min_rank=0 imbalance_ratio=1.313965 max_load_reduction=54772 max_load_reduction_pct=0.835754 moved_to_echo=32111 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 252, 3960, 5, 0, 3542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3778, 0, 0, 0, 0, 0, 0, 0, 614, 1789, 3693, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1467, 0, 1009, 0, 0, 4096, 0, 550, 262, 0, 0, 0, 0, 0, 0, 0, 4089, 681, 0, 4024, 1277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1306, 1236, 742, 250, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 3164, 318, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 166, 2860, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7759, 7874, 10106, 7384, 10071, 7630, 7579, 7133] avg=8192.0000 max=10106 max_rank=2 min=7133 min_rank=7 imbalance_ratio=1.233643 max_load_reduction=55430 max_load_reduction_pct=0.845795 moved_to_echo=32413 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 4096, 3775, 2447, 82, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 498, 0, 0, 0, 0, 4096, 3705, 0, 0, 0, 0, 0, 0, 0, 2577, 2605, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 453, 28, 19, 0, 2450, 986, 431, 389, 0, 0, 0, 0, 0, 0, 0, 3720, 663, 0, 2796, 841, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 1149, 0, 0, 19, 30, 381, 1103, 1087, 253, 0, 0, 0, 0, 0, 1, 0, 0, 0, 56, 10, 17, 0, 3816, 3129, 312, 0, 0, 0, 0, 0, 0, 0, 670, 0, 0, 0, 2545, 3401, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13179, 8459, 5541, 8814, 8032, 7554, 7341, 6616] avg=8192.0000 max=13179 max_rank=0 min=5541 min_rank=2 imbalance_ratio=1.608765 max_load_reduction=52357 max_load_reduction_pct=0.798904 moved_to_echo=29543 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 29184, 5504, 512, 18688, 12032, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 22272, 15744, 7040, 640, 19840, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4864, 256, 256, 128, 256, 128, 128, 640, 31872, 25088, 2560, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 4608, 16384, 17024, 128, 27392, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 31104, 2048, 128, 128, 128, 128, 13568, 9728, 6528, 2048, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2048, 128, 128, 640, 128, 0, 256, 128, 32640, 30080, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 2432, 128, 128, 8960, 22528, 31744, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 25344, 512, 1920, 128, 3712, 26624, 1024, 4352, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 2923, 4096, 4036, 0, 338, 10, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 51, 0, 4096, 57, 4067, 324, 0, 4084, 0, 0, 0, 4094, 0, 4057, 0, 15, 4027, 7, 26, 4096, 0, 0, 0, 3521, 0, 59, 14, 4076, 58, 4044, 22, 0, 3, 0, 1169, 0, 5, 4089, 3962, 0, 0, 0, 0, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[9, 4009, 4094, 3964, 1, 0, 0, 0, 0, 0, 2, 4087, 0, 0, 1, 15, 0, 0, 0, 4066, 75, 4026, 3579, 3, 4096, 0, 6, 0, 4062, 62, 4064, 0, 40, 4054, 24, 0, 3223, 29, 819, 0, 1698, 0, 135, 0, 4096, 0, 4013, 0, 0, 121, 57, 2, 0, 2, 4096, 2762, 33, 5, 0, 10, 71, 0, 4, 21] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4028, 3647, 4087, 0, 0, 10, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 4050, 0, 4096, 0, 0, 4050, 3, 0, 0, 4083, 48, 4092, 57, 0, 4068, 0, 0, 4088, 0, 169, 0, 4069, 0, 0, 109, 4091, 2, 3977, 0, 2, 0, 4, 109, 0, 794, 4094, 3428, 8, 4, 0, 0, 0, 131, 30, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4038, 4093, 4096, 0, 0, 0, 0, 182, 0, 0, 2729, 46, 0, 0, 0, 1, 0, 0, 3828, 15, 3982, 1397, 5, 3871, 0, 0, 0, 3880, 191, 3124, 44, 262, 4096, 7, 0, 4057, 29, 0, 0, 3941, 4, 0, 0, 3553, 0, 3701, 0, 1, 0, 359, 0, 526, 932, 4096, 0, 0, 0, 1, 97, 4096, 0, 0, 256] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 3822, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 120, 0, 0, 0, 0, 0, 152, 4078, 0, 4096, 479, 0, 4096, 0, 0, 0, 4096, 15, 4096, 0, 3, 3908, 0, 0, 3127, 0, 1821, 0, 4094, 0, 0, 0, 2469, 0, 2184, 0, 0, 0, 0, 0, 1, 1, 4096, 2464, 0, 0, 0, 0, 4035, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 658, 4090, 4096, 0, 28, 4, 0, 26, 0, 0, 3866, 0, 0, 0, 0, 0, 23, 1, 4085, 0, 4094, 4083, 0, 3368, 0, 0, 0, 4095, 16, 4096, 0, 0, 4095, 3, 3, 3812, 0, 3964, 0, 3879, 0, 0, 0, 4007, 193, 4096, 188, 10, 0, 119, 12, 0, 0, 4093, 218, 0, 0, 0, 99, 112, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 3033, 4096, 4087, 0, 0, 0, 0, 2, 0, 0, 3945, 0, 0, 4, 4, 1, 0, 81, 4095, 0, 4095, 3995, 0, 4096, 0, 0, 0, 3652, 125, 4081, 3, 0, 3643, 0, 0, 4096, 20, 4095, 0, 571, 0, 13, 0, 3503, 5, 4096, 14, 0, 4, 173, 500, 209, 0, 4094, 999, 0, 1, 0, 92, 5, 0, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[8, 0, 3801, 2074, 0, 4, 0, 0, 0, 2, 0, 2608, 0, 72, 0, 1021, 0, 17, 1, 4079, 0, 4092, 3870, 7, 365, 0, 36, 50, 3996, 0, 3817, 107, 1701, 3700, 0, 412, 2947, 11, 4095, 619, 0, 0, 4055, 0, 1151, 0, 4088, 227, 0, 15, 2516, 3147, 0, 0, 4063, 17, 0, 285, 1621, 236, 261, 0, 340, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2923, 1544, 4036, 0, 338, 10, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2199, 1527, 701, 0, 0, 0, 0, 0, 0, 51, 0, 4096, 57, 1868, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 682, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2285, 7, 26, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3521, 0, 59, 14, 4076, 58, 1571, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1169, 0, 5, 4089, 3962, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3412, 2552, 946, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8863, 8523, 6396, 8823, 6429, 9321, 10269, 6912] avg=8192.0000 max=10269 max_rank=6 min=6396 min_rank=2 imbalance_ratio=1.253540 max_load_reduction=55267 max_load_reduction_pct=0.843307 moved_to_echo=32931 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4028, 1375, 4087, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 2215, 1502, 708, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 3, 0, 0, 680, 48, 4092, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2308, 0, 0, 4088, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 109, 4091, 2, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 109, 0, 794, 4094, 3428, 1052, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 131, 30, 10, 3403, 2272, 931, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9500, 8523, 5931, 8930, 6565, 9815, 9483, 6789] avg=8192.0000 max=9815 max_rank=5 min=5931 min_rank=2 imbalance_ratio=1.198120 max_load_reduction=55721 max_load_reduction_pct=0.850235 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4038, 1543, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 2729, 46, 0, 0, 0, 2153, 1398, 713, 0, 0, 0, 0, 0, 1, 0, 0, 3828, 15, 1829, 1397, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 0, 0, 0, 646, 191, 3124, 44, 0, 0, 0, 0, 0, 0, 0, 0, 262, 2324, 7, 0, 4057, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 4, 0, 0, 3553, 0, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 359, 0, 526, 932, 4096, 0, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 97, 4096, 0, 0, 256, 3234, 2550, 866, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9677, 7221, 7075, 7876, 6679, 8935, 6973, 11100] avg=8192.0000 max=11100 max_rank=7 min=6679 min_rank=4 imbalance_ratio=1.354980 max_load_reduction=54436 max_load_reduction_pct=0.830627 moved_to_echo=33687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 658, 1542, 4096, 0, 28, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3866, 0, 0, 0, 0, 2214, 1547, 713, 0, 0, 0, 0, 0, 0, 23, 1, 4085, 0, 1880, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3368, 0, 0, 0, 682, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 3, 3, 3812, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 0, 0, 0, 4007, 193, 1591, 188, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 119, 12, 0, 0, 4093, 218, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 112, 0, 4, 0, 3413, 2548, 958, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6328, 8366, 10072, 8162, 10105, 9858, 5511, 7134] avg=8192.0000 max=10105 max_rank=4 min=5511 min_rank=6 imbalance_ratio=1.233521 max_load_reduction=55431 max_load_reduction_pct=0.845810 moved_to_echo=32608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[9, 4009, 1543, 3964, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4087, 0, 0, 1, 15, 2177, 1516, 706, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 75, 1849, 3579, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 677, 62, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 2300, 24, 0, 3223, 29, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1698, 0, 135, 0, 4096, 0, 1558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 57, 2, 0, 2, 4096, 2762, 1048, 0, 0, 0, 0, 0, 0, 0, 33, 5, 0, 10, 71, 0, 4, 21, 3385, 2551, 939, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9526, 8504, 9572, 8905, 6435, 7487, 8088, 7019] avg=8192.0000 max=9572 max_rank=2 min=6435 min_rank=4 imbalance_ratio=1.168457 max_load_reduction=55964 max_load_reduction_pct=0.853943 moved_to_echo=29029 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3033, 1544, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3945, 0, 0, 4, 4, 2215, 1547, 634, 0, 0, 0, 0, 0, 1, 0, 81, 4095, 0, 1880, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 608, 125, 4081, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 0, 4096, 20, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 0, 13, 0, 3503, 5, 1591, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 173, 500, 209, 0, 4094, 999, 942, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 92, 5, 0, 8, 0, 3044, 2552, 958, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8664, 8351, 10052, 8913, 10278, 5697, 6921, 6660] avg=8192.0000 max=10278 max_rank=4 min=5697 min_rank=5 imbalance_ratio=1.254639 max_load_reduction=55258 max_load_reduction_pct=0.843170 moved_to_echo=29556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3822, 1544, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 120, 0, 0, 0, 2215, 825, 680, 0, 0, 0, 0, 0, 0, 0, 152, 4078, 0, 1881, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 682, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2218, 0, 0, 3127, 0, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2469, 0, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 2464, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 3414, 2552, 511, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9462, 7931, 6590, 8889, 7169, 7411, 7572, 10512] avg=8192.0000 max=10512 max_rank=7 min=6590 min_rank=2 imbalance_ratio=1.283203 max_load_reduction=55024 max_load_reduction_pct=0.839600 moved_to_echo=32664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 1430, 2074, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2608, 0, 72, 0, 1021, 2218, 1548, 648, 0, 0, 0, 0, 0, 0, 17, 1, 4079, 0, 1874, 3870, 7, 0, 0, 0, 0, 0, 0, 0, 0, 365, 0, 36, 50, 663, 0, 3817, 107, 0, 0, 0, 0, 0, 0, 0, 0, 1701, 2091, 0, 412, 2947, 11, 4095, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 1151, 0, 1579, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2516, 3147, 0, 0, 4063, 17, 961, 0, 0, 0, 0, 0, 0, 0, 0, 285, 1621, 236, 261, 0, 340, 2, 3333, 2371, 961, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3516, 8117, 9848, 5038, 11876, 7012, 10719, 9410] avg=8192.0000 max=11876 max_rank=4 min=3516 min_rank=0 imbalance_ratio=1.449707 max_load_reduction=53660 max_load_reduction_pct=0.818787 moved_to_echo=39017 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 384, 1664, 640, 8704, 256, 512, 384, 26752, 19968, 7168, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 22528, 12160, 30592, 128, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[21888, 128, 4352, 128, 27008, 384, 11776, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 256, 32384, 256, 14976, 17792, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 256, 3328, 4992, 768, 1792, 32768, 13952, 8192, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2048, 17920, 128, 512, 29568, 128, 14976, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[28032, 128, 128, 128, 5376, 512, 31488, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 128, 128, 29568, 256, 128, 128, 1152, 17664, 11520, 5504, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[100, 4094, 0, 0, 0, 1116, 0, 76, 4050, 0, 0, 0, 1, 26, 0, 0, 4096, 1, 0, 0, 42, 4079, 7, 1, 4083, 3, 0, 3943, 0, 7, 4095, 3970, 3674, 0, 0, 4091, 0, 5, 2666, 4090, 4057, 129, 126, 7, 0, 0, 0, 0, 0, 4, 185, 0, 0, 0, 1, 4041, 4056, 4092, 0, 505, 0, 17, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[2479, 4090, 0, 0, 303, 723, 0, 4, 4071, 1, 5, 0, 0, 93, 0, 0, 4096, 0, 29, 0, 31, 4083, 0, 0, 4096, 1, 0, 4037, 1, 0, 4096, 4094, 3655, 1, 1, 4091, 0, 13, 0, 4093, 4073, 0, 0, 0, 0, 34, 620, 0, 0, 0, 0, 0, 0, 0, 25, 4080, 3833, 4094, 0, 584, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4076, 4093, 0, 0, 0, 0, 0, 765, 4089, 0, 196, 0, 166, 0, 0, 0, 3700, 0, 0, 1, 616, 4021, 8, 0, 3, 14, 0, 4084, 0, 0, 3904, 4014, 3334, 0, 1, 4095, 0, 1, 57, 4095, 4095, 0, 2, 0, 3, 7, 0, 0, 1, 0, 0, 0, 0, 1, 164, 4096, 4009, 4096, 0, 3729, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[276, 4096, 0, 0, 532, 2223, 0, 29, 1739, 4, 0, 5, 103, 0, 0, 76, 3884, 0, 0, 0, 0, 4089, 0, 0, 4095, 1, 0, 4078, 11, 41, 4083, 3800, 3870, 0, 6, 4080, 0, 226, 4046, 4024, 3961, 52, 6, 0, 0, 0, 0, 107, 0, 0, 184, 0, 0, 0, 102, 3487, 4096, 4081, 37, 6, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[2, 3962, 119, 0, 2853, 1288, 63, 4095, 4093, 44, 39, 0, 0, 1362, 14, 0, 4096, 0, 70, 4, 247, 4079, 0, 9, 0, 49, 0, 3743, 0, 0, 2730, 4055, 4055, 0, 28, 4045, 8, 0, 3508, 4067, 4065, 1, 5, 0, 0, 0, 0, 15, 168, 0, 89, 1, 1, 0, 3, 4034, 178, 4031, 0, 174, 0, 0, 42, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[2209, 4082, 2, 0, 1, 0, 0, 1894, 2917, 0, 17, 0, 84, 7, 1, 0, 1679, 0, 0, 7, 6, 4096, 0, 0, 3826, 1, 0, 4096, 4, 0, 4095, 4094, 4049, 0, 2, 4057, 105, 29, 551, 4089, 4096, 18, 0, 9, 0, 0, 0, 0, 6, 96, 0, 0, 0, 2, 164, 4089, 3010, 4061, 0, 3985, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4007, 4096, 0, 0, 0, 2, 0, 1968, 3876, 0, 0, 0, 0, 103, 1, 0, 2628, 0, 3, 0, 0, 4095, 0, 0, 3987, 0, 0, 4096, 0, 0, 4066, 4096, 4096, 0, 0, 4096, 0, 0, 9, 4096, 4096, 0, 0, 0, 0, 4, 3, 0, 0, 0, 33, 0, 0, 0, 0, 4096, 3258, 4096, 0, 629, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1037, 3739, 10, 0, 1631, 0, 0, 1, 3953, 15, 8, 0, 3248, 1429, 46, 0, 2947, 20, 0, 0, 0, 4096, 0, 0, 3854, 310, 0, 2589, 265, 0, 3717, 4036, 1666, 7, 0, 4050, 4, 26, 4096, 4096, 4096, 0, 0, 2, 0, 6, 0, 192, 52, 213, 242, 1, 0, 11, 15, 4096, 855, 4096, 0, 761, 1, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[100, 4018, 0, 0, 0, 1116, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 1, 26, 0, 0, 2735, 963, 174, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 42, 4079, 7, 1, 399, 110, 76, 0, 0, 0, 0, 0, 4083, 3, 0, 3943, 0, 7, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3674, 0, 0, 2729, 0, 5, 2666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 129, 126, 7, 0, 0, 0, 0, 3796, 113, 0, 0, 0, 0, 0, 0, 0, 4, 185, 0, 0, 0, 1, 4041, 3977, 0, 0, 0, 0, 0, 0, 0, 4056, 3982, 0, 505, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5310, 7949, 8811, 9396, 9074, 8228, 8208, 8560] avg=8192.0000 max=9396 max_rank=3 min=5310 min_rank=0 imbalance_ratio=1.146973 max_load_reduction=56140 max_load_reduction_pct=0.856628 moved_to_echo=34070 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 3889, 119, 0, 2853, 1288, 63, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 44, 39, 0, 0, 1362, 14, 0, 1823, 952, 177, 0, 0, 0, 0, 0, 4096, 0, 70, 4, 247, 4079, 0, 9, 395, 109, 73, 0, 0, 0, 0, 0, 0, 49, 0, 3743, 0, 0, 907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 28, 2698, 8, 0, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 1, 5, 0, 0, 0, 0, 15, 3878, 111, 0, 0, 0, 0, 0, 0, 168, 0, 89, 1, 1, 0, 3, 4034, 3956, 0, 0, 0, 0, 0, 0, 0, 178, 3922, 0, 174, 0, 0, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12309, 8504, 9082, 4699, 10297, 8075, 8252, 4318] avg=8192.0000 max=12309 max_rank=0 min=4318 min_rank=7 imbalance_ratio=1.502563 max_load_reduction=53227 max_load_reduction_pct=0.812180 moved_to_echo=30942 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[276, 4020, 0, 0, 532, 2223, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 1739, 4, 0, 5, 103, 0, 0, 76, 2727, 960, 166, 0, 0, 0, 0, 0, 3884, 0, 0, 0, 0, 4089, 0, 0, 398, 110, 76, 0, 0, 0, 0, 0, 4095, 1, 0, 4078, 11, 41, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 0, 6, 2722, 0, 226, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 52, 6, 0, 0, 0, 0, 107, 3634, 110, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 102, 3487, 3914, 0, 0, 0, 0, 0, 0, 0, 4096, 3971, 37, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7080, 5780, 8557, 9582, 10870, 7870, 7687, 8110] avg=8192.0000 max=10870 max_rank=4 min=5780 min_rank=1 imbalance_ratio=1.326904 max_load_reduction=54666 max_load_reduction_pct=0.834137 moved_to_echo=34537 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2479, 4014, 0, 0, 303, 723, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 1, 5, 0, 0, 93, 0, 0, 2736, 963, 179, 0, 0, 0, 0, 0, 4096, 0, 29, 0, 31, 4083, 0, 0, 399, 110, 76, 0, 0, 0, 0, 0, 4096, 1, 0, 4037, 1, 0, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3655, 1, 1, 2729, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 34, 620, 0, 3915, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4080, 3981, 0, 0, 0, 0, 0, 0, 0, 3833, 3984, 0, 584, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7523, 8048, 8824, 9495, 6399, 8754, 8086, 8407] avg=8192.0000 max=9495 max_rank=3 min=6399 min_rank=4 imbalance_ratio=1.159058 max_load_reduction=56041 max_load_reduction_pct=0.855118 moved_to_echo=31646 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4076, 4017, 0, 0, 0, 0, 0, 765, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 196, 0, 166, 0, 0, 0, 2608, 964, 176, 0, 0, 0, 0, 0, 3700, 0, 0, 1, 616, 4021, 8, 0, 400, 111, 76, 0, 0, 0, 0, 0, 3, 14, 0, 4084, 0, 0, 1296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3334, 0, 1, 2731, 0, 1, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 3, 7, 0, 0, 3838, 113, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 164, 4096, 3982, 0, 0, 0, 0, 0, 0, 0, 4009, 3985, 0, 3729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8858, 8199, 8933, 5397, 6124, 8058, 8244, 11723] avg=8192.0000 max=11723 max_rank=7 min=5397 min_rank=3 imbalance_ratio=1.431030 max_load_reduction=53813 max_load_reduction_pct=0.821121 moved_to_echo=34149 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2209, 4007, 2, 0, 1, 0, 0, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 2917, 0, 17, 0, 84, 7, 1, 0, 2735, 955, 179, 0, 0, 0, 0, 0, 1679, 0, 0, 7, 6, 4096, 0, 0, 396, 110, 75, 0, 0, 0, 0, 0, 3826, 1, 0, 4096, 4, 0, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 2, 2706, 105, 29, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 9, 0, 0, 0, 0, 3915, 112, 0, 0, 0, 0, 0, 0, 6, 96, 0, 0, 0, 2, 164, 4089, 3977, 0, 0, 0, 0, 0, 0, 0, 3010, 3951, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 6895, 6369, 9287, 7442, 8150, 8334, 10946] avg=8192.0000 max=10946 max_rank=7 min=6369 min_rank=2 imbalance_ratio=1.336182 max_load_reduction=54590 max_load_reduction_pct=0.832977 moved_to_echo=34872 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4007, 4020, 0, 0, 0, 2, 0, 1968, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 0, 0, 0, 0, 103, 1, 0, 2716, 964, 179, 0, 0, 0, 0, 0, 2628, 0, 3, 0, 0, 4095, 0, 0, 400, 111, 76, 0, 0, 0, 0, 0, 3987, 0, 0, 4096, 0, 0, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2732, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 3, 0, 3917, 113, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 4096, 3983, 0, 0, 0, 0, 0, 0, 0, 3258, 3985, 0, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9997, 7839, 7313, 9433, 6837, 8133, 8112, 7872] avg=8192.0000 max=9997 max_rank=0 min=6837 min_rank=4 imbalance_ratio=1.220337 max_load_reduction=55539 max_load_reduction_pct=0.847458 moved_to_echo=30954 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1037, 3667, 10, 0, 1631, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 15, 8, 0, 3248, 1429, 46, 0, 2488, 958, 177, 0, 0, 0, 0, 0, 2947, 20, 0, 0, 0, 4096, 0, 0, 398, 114, 72, 0, 0, 0, 0, 0, 3854, 310, 0, 2589, 265, 0, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1666, 7, 0, 2694, 4, 26, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 6, 0, 192, 3859, 113, 0, 0, 0, 0, 0, 0, 52, 213, 242, 1, 0, 11, 15, 4096, 3983, 0, 0, 0, 0, 0, 0, 0, 855, 3982, 0, 761, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6346, 12322, 7647, 8247, 8493, 8268, 8613, 5600] avg=8192.0000 max=12322 max_rank=1 min=5600 min_rank=7 imbalance_ratio=1.504150 max_load_reduction=53214 max_load_reduction_pct=0.811981 moved_to_echo=30974 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[14208, 31744, 256, 0, 5376, 5376, 128, 8832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32640, 256, 256, 128, 128, 128, 640, 384, 30848, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[28416, 128, 128, 21760, 128, 384, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 384, 768, 128, 128, 128, 512, 32128, 31872, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[24064, 384, 0, 30720, 384, 128, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[27136, 128, 128, 128, 1024, 32640, 128, 128, 3200, 896, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[28800, 128, 384, 128, 3712, 3072, 128, 128, 20608, 7680, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[23296, 31872, 128, 10496, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[6, 3352, 0, 0, 0, 0, 324, 1261, 33, 375, 3887, 52, 1, 0, 0, 0, 0, 0, 3744, 0, 0, 3846, 26, 4089, 1, 0, 0, 4064, 820, 1429, 26, 0, 3443, 0, 2, 0, 0, 4056, 4096, 0, 9, 0, 4084, 26, 0, 0, 1722, 0, 0, 0, 0, 4075, 3906, 4089, 4, 4054, 4096, 0, 0, 273, 5, 44, 216, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[260, 4094, 276, 0, 38, 0, 2090, 4020, 0, 22, 3893, 0, 0, 23, 841, 0, 0, 94, 3669, 1, 0, 114, 891, 4052, 0, 0, 58, 22, 0, 1116, 4, 0, 4096, 59, 0, 0, 3956, 3982, 4064, 10, 2, 85, 4050, 0, 0, 40, 2, 0, 2, 0, 119, 3456, 3211, 3969, 27, 4089, 4056, 3, 0, 635, 45, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 2378, 31, 0, 0, 0, 4056, 115, 0, 1462, 0, 0, 919, 7, 3, 0, 303, 0, 3163, 0, 0, 4096, 0, 3893, 0, 0, 0, 4088, 0, 15, 0, 29, 4092, 0, 0, 0, 188, 4042, 4094, 0, 0, 0, 4029, 1, 0, 0, 13, 0, 6, 0, 33, 3944, 4081, 4038, 0, 4085, 4074, 0, 0, 3985, 0, 0, 273, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4015, 0, 0, 0, 0, 2146, 4024, 0, 236, 214, 0, 2, 0, 0, 0, 57, 0, 3982, 0, 0, 8, 613, 4096, 0, 0, 0, 4095, 3, 50, 6, 0, 4096, 0, 0, 0, 3688, 3718, 4096, 0, 0, 1742, 4096, 0, 2, 0, 114, 0, 0, 0, 0, 4096, 3967, 4084, 0, 4096, 4096, 0, 0, 98, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4094, 3, 0, 0, 0, 4095, 4096, 0, 19, 4091, 0, 0, 0, 0, 3400, 0, 58, 9, 0, 0, 4087, 518, 4095, 0, 29, 0, 450, 0, 0, 18, 0, 3910, 0, 6, 0, 40, 4096, 3617, 0, 0, 3916, 1943, 0, 0, 1, 0, 178, 1, 0, 6, 4070, 1276, 4095, 1, 4095, 4096, 0, 0, 175, 0, 952, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 3934, 0, 0, 0, 17, 1927, 4096, 0, 161, 30, 4, 0, 0, 0, 445, 3540, 268, 4010, 0, 32, 3649, 30, 3975, 8, 0, 0, 4037, 0, 74, 17, 96, 4082, 0, 3, 0, 137, 3685, 3880, 0, 0, 0, 4089, 0, 0, 0, 3, 0, 0, 0, 0, 4058, 1555, 4094, 0, 4073, 4070, 1, 6, 1321, 0, 2, 127, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 728, 0, 303, 4086, 0, 0, 27, 0, 2840, 0, 0, 2468, 30, 0, 4096, 0, 0, 2598, 1036, 4096, 0, 0, 25, 3864, 0, 0, 50, 0, 0, 0, 0, 0, 3524, 3550, 4085, 32, 0, 0, 4039, 0, 217, 0, 0, 0, 0, 0, 7, 3031, 3766, 4073, 231, 4096, 4096, 0, 0, 17, 134, 0, 0, 295] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[2, 184, 305, 0, 0, 0, 1745, 3865, 0, 4079, 2302, 6, 0, 0, 0, 17, 5, 756, 43, 0, 0, 3997, 5, 3417, 0, 0, 0, 26, 1621, 4085, 1092, 50, 3658, 0, 0, 0, 4096, 3209, 4080, 0, 0, 0, 4062, 782, 0, 47, 267, 2, 1, 0, 2325, 2912, 1, 4093, 0, 3802, 4062, 3, 0, 527, 5, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 2772, 0, 0, 0, 0, 324, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 33, 375, 3887, 52, 1, 0, 0, 0, 3888, 412, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 0, 0, 3846, 26, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4064, 820, 1429, 26, 0, 4089, 208, 0, 0, 0, 0, 0, 0, 3443, 0, 2, 0, 0, 2998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4084, 26, 0, 0, 1722, 0, 2218, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 3906, 0, 4, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 273, 5, 44, 216, 0, 1457, 1058, 580, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4363, 8648, 9185, 10637, 6443, 9122, 9409, 7729] avg=8192.0000 max=10637 max_rank=3 min=4363 min_rank=0 imbalance_ratio=1.298462 max_load_reduction=54899 max_load_reduction_pct=0.837692 moved_to_echo=32703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[260, 3386, 276, 0, 38, 0, 2090, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3893, 0, 0, 23, 841, 0, 3856, 416, 0, 0, 0, 0, 0, 0, 0, 94, 3669, 1, 0, 114, 891, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 22, 0, 1116, 4, 0, 3969, 208, 0, 0, 0, 0, 0, 0, 4096, 59, 0, 0, 3956, 2943, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 2, 85, 4050, 0, 0, 40, 2, 0, 2237, 1053, 0, 0, 0, 0, 0, 0, 2, 0, 119, 3456, 3211, 0, 27, 1436, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 3, 0, 635, 45, 0, 0, 0, 1444, 1039, 708, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10070, 9051, 6324, 5377, 11064, 7469, 8251, 7930] avg=8192.0000 max=11064 max_rank=4 min=5377 min_rank=3 imbalance_ratio=1.350586 max_load_reduction=54472 max_load_reduction_pct=0.831177 moved_to_echo=34714 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3254, 0, 0, 0, 17, 1927, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 30, 4, 0, 0, 0, 445, 3682, 414, 0, 0, 0, 0, 0, 0, 3540, 268, 4010, 0, 32, 3649, 30, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4037, 0, 74, 17, 96, 4094, 198, 0, 0, 0, 0, 0, 0, 4082, 0, 3, 0, 137, 2723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 3, 0, 2228, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 1555, 0, 0, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 1, 6, 1321, 0, 2, 127, 0, 1416, 962, 680, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9294, 4736, 13055, 8524, 6945, 7353, 7044, 8585] avg=8192.0000 max=13055 max_rank=2 min=4736 min_rank=1 imbalance_ratio=1.593628 max_load_reduction=52481 max_load_reduction_pct=0.800797 moved_to_echo=29927 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3387, 0, 0, 728, 0, 303, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 2840, 0, 0, 2468, 3876, 417, 0, 0, 0, 0, 0, 0, 30, 0, 4096, 0, 0, 2598, 1036, 1572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 3864, 0, 0, 50, 0, 4073, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 2624, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 217, 0, 0, 0, 2241, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3031, 3766, 0, 231, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 134, 0, 0, 295, 1459, 926, 709, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8504, 9628, 9332, 8221, 6180, 7562, 8473, 7636] avg=8192.0000 max=9628 max_rank=1 min=6180 min_rank=4 imbalance_ratio=1.175293 max_load_reduction=55908 max_load_reduction_pct=0.853088 moved_to_echo=29851 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1967, 31, 0, 0, 0, 4056, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1462, 0, 0, 919, 7, 3, 0, 3885, 415, 0, 0, 0, 0, 0, 0, 303, 0, 3163, 0, 0, 4096, 0, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 15, 0, 29, 4038, 209, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 188, 2987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 1, 0, 0, 13, 0, 2235, 1012, 0, 0, 0, 0, 0, 0, 6, 0, 33, 3944, 4081, 0, 0, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 3985, 0, 0, 273, 0, 1387, 1055, 411, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6169, 6691, 9056, 8379, 7267, 7290, 9499, 11185] avg=8192.0000 max=11185 max_rank=7 min=6169 min_rank=0 imbalance_ratio=1.365356 max_load_reduction=54351 max_load_reduction_pct=0.829330 moved_to_echo=35241 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3386, 3, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4091, 0, 0, 0, 0, 3400, 3433, 416, 0, 0, 0, 0, 0, 0, 0, 58, 9, 0, 0, 4087, 518, 1571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 450, 0, 0, 18, 0, 4095, 184, 0, 0, 0, 0, 0, 0, 3910, 0, 6, 0, 40, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3916, 1943, 0, 0, 1, 0, 178, 2240, 1065, 0, 0, 0, 0, 0, 0, 1, 0, 6, 4070, 1276, 0, 1, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 175, 0, 952, 0, 0, 1459, 1069, 708, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11580, 11359, 6243, 4776, 6983, 9343, 6793, 8459] avg=8192.0000 max=11580 max_rank=0 min=4776 min_rank=3 imbalance_ratio=1.413574 max_load_reduction=53956 max_load_reduction_pct=0.823303 moved_to_echo=31578 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3321, 0, 0, 0, 0, 2146, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 214, 0, 2, 0, 0, 0, 3887, 417, 0, 0, 0, 0, 0, 0, 57, 0, 3982, 0, 0, 8, 613, 1572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 50, 6, 0, 4084, 209, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3688, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 4096, 0, 2, 0, 114, 0, 2241, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3967, 0, 0, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 98, 0, 0, 0, 0, 1459, 970, 694, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9491, 4756, 6232, 8447, 10532, 9260, 9501, 7317] avg=8192.0000 max=10532 max_rank=4 min=4756 min_rank=1 imbalance_ratio=1.285645 max_load_reduction=55004 max_load_reduction_pct=0.839294 moved_to_echo=36610 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 148, 305, 0, 0, 0, 1745, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 2302, 6, 0, 0, 0, 17, 3872, 391, 0, 0, 0, 0, 0, 0, 5, 756, 43, 0, 0, 3997, 5, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 1621, 4085, 1092, 50, 4093, 208, 0, 0, 0, 0, 0, 0, 3658, 0, 0, 0, 4096, 2368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 782, 0, 47, 267, 2, 2084, 893, 0, 0, 0, 0, 0, 0, 1, 0, 2325, 2912, 1, 0, 0, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 3, 0, 527, 5, 0, 0, 0, 1221, 841, 36, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6065, 10667, 6109, 11175, 10122, 8137, 6566, 6695] avg=8192.0000 max=11175 max_rank=3 min=6065 min_rank=0 imbalance_ratio=1.364136 max_load_reduction=54361 max_load_reduction_pct=0.829483 moved_to_echo=31520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[384, 21632, 640, 0, 768, 128, 16768, 25600, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[27392, 128, 128, 0, 15744, 22528, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 7040, 256, 1024, 640, 384, 11392, 7936, 4608, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3968, 1280, 22784, 128, 128, 22400, 3200, 12288, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 2560, 29696, 21888, 0, 384, 11392, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 5760, 30464, 896, 256, 128, 2176, 256, 17792, 8320, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 20736, 2560, 6784, 1280, 256, 32640, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 6400, 14464, 128, 3840, 128, 896, 6400, 30464, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[3845, 0, 4091, 8, 328, 0, 4054, 4, 60, 0, 110, 0, 408, 0, 0, 1, 1583, 3982, 0, 0, 235, 0, 3721, 2218, 4095, 2705, 0, 4, 8, 4096, 2, 3906, 31, 21, 2, 3662, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4089, 45, 3965, 0, 0, 0, 42, 7, 0, 122, 564, 813, 391, 104, 0, 4049, 0, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4039, 0, 1, 0, 4096, 0, 0, 15, 0, 0, 0, 39, 116, 493, 4052, 4096, 0, 0, 3, 2941, 4024, 0, 2823, 0, 1, 1, 12, 4055, 175, 3961, 4091, 0, 119, 4076, 0, 4091, 62, 0, 0, 0, 26, 0, 248, 102, 0, 4096, 8, 2, 0, 0, 0, 0, 0, 679, 1285, 2556, 1221, 0, 4096, 36, 3799, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 38, 3991, 4030, 819, 0, 3715, 0, 0, 3775, 0, 0, 27, 0, 315, 0, 4096, 4096, 0, 0, 35, 101, 0, 0, 0, 0, 0, 6, 0, 3500, 624, 4090, 3853, 0, 0, 593, 0, 4096, 37, 0, 1, 0, 0, 28, 0, 0, 0, 3252, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 4039, 0, 4079, 4096, 0, 4035, 65] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3, 3, 4093, 0, 0, 2, 3809, 0, 1, 0, 0, 0, 0, 104, 43, 226, 655, 4093, 29, 1, 3866, 0, 2266, 185, 1122, 1, 15, 0, 0, 4095, 107, 3964, 1210, 0, 0, 4095, 70, 4091, 0, 0, 3, 0, 0, 3581, 0, 227, 0, 3996, 4, 0, 0, 2, 0, 27, 0, 3576, 3803, 582, 586, 2829, 4088, 0, 4083, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[33, 0, 4091, 5, 1353, 18, 4081, 0, 0, 2795, 0, 0, 161, 0, 0, 10, 3088, 4085, 912, 0, 0, 43, 3741, 3886, 4005, 2, 96, 5, 0, 3993, 5, 3115, 14, 0, 19, 3668, 0, 4092, 0, 1, 0, 362, 1, 98, 0, 3, 0, 4079, 0, 0, 32, 0, 0, 0, 0, 1149, 1, 4092, 1, 211, 4081, 7, 4096, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 0, 0, 24, 0, 0, 0, 0, 1, 0, 4070, 4096, 0, 0, 4081, 520, 14, 0, 4096, 0, 31, 0, 0, 4096, 0, 4096, 211, 0, 0, 3716, 0, 4096, 0, 0, 0, 244, 0, 11, 78, 3800, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 3688, 4054, 54, 0, 4086, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[3919, 0, 4096, 0, 0, 2, 4063, 5, 1, 15, 0, 0, 1, 24, 49, 0, 18, 4043, 0, 117, 0, 2184, 3510, 3849, 4096, 245, 206, 0, 0, 4088, 1, 4095, 0, 0, 1, 0, 0, 4074, 0, 0, 0, 0, 0, 3513, 71, 70, 0, 4059, 0, 20, 6, 0, 0, 39, 0, 306, 0, 3968, 2269, 291, 4096, 0, 4089, 37] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 33, 2375, 117, 3850, 0, 0, 1331, 1529, 0, 4093, 0, 3170, 1308, 0, 4090, 1909, 0, 234, 35, 4, 10, 4069, 103, 2, 0, 0, 3913, 1, 2638, 4048, 0, 395, 3915, 0, 2758, 103, 91, 1, 7, 26, 1111, 247, 17, 0, 1285, 0, 0, 0, 0, 0, 1239, 0, 1, 3551, 3963, 69, 0, 962, 57, 2778, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3845, 0, 2113, 8, 328, 0, 4054, 4, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 110, 0, 408, 0, 0, 1, 3194, 2304, 323, 124, 0, 0, 0, 0, 1583, 405, 0, 0, 235, 0, 3721, 2218, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2705, 0, 4, 8, 902, 2, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 31, 21, 2, 3662, 0, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 45, 3965, 1854, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 7, 0, 122, 564, 4095, 3254, 0, 0, 0, 0, 0, 0, 813, 391, 104, 0, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10352, 6524, 8162, 11622, 7276, 10463, 8084, 3053] avg=8192.0000 max=11622 max_rank=3 min=3053 min_rank=7 imbalance_ratio=1.418701 max_load_reduction=53914 max_load_reduction_pct=0.822662 moved_to_echo=28876 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 38, 2062, 4030, 819, 0, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 27, 0, 315, 0, 2729, 2331, 333, 121, 0, 0, 0, 0, 4096, 415, 0, 0, 35, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 771, 624, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 593, 0, 3583, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 28, 0, 0, 0, 3252, 1808, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4035, 3348, 0, 0, 0, 0, 0, 0, 4096, 4039, 0, 4079, 1765, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10664, 9631, 4647, 5491, 8066, 5602, 7391, 14044] avg=8192.0000 max=14044 max_rank=7 min=4647 min_rank=2 imbalance_ratio=1.714355 max_load_reduction=51492 max_load_reduction_pct=0.785706 moved_to_echo=35103 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2087, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 39, 116, 493, 3162, 2331, 333, 122, 0, 0, 0, 0, 4052, 415, 0, 0, 3, 2941, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2823, 0, 1, 1, 12, 893, 175, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 119, 4076, 0, 3578, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 248, 102, 0, 4096, 1830, 513, 0, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 679, 3799, 3348, 0, 0, 0, 0, 0, 0, 1285, 2556, 1221, 0, 1765, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6184, 6611, 11435, 7866, 11926, 6815, 7836, 6863] avg=8192.0000 max=11926 max_rank=4 min=6184 min_rank=0 imbalance_ratio=1.455811 max_load_reduction=53610 max_load_reduction_pct=0.818024 moved_to_echo=33440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3919, 0, 2116, 0, 0, 2, 4063, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 0, 0, 1, 24, 49, 0, 3188, 2331, 328, 124, 0, 0, 0, 0, 18, 411, 0, 117, 0, 2184, 3510, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 245, 206, 0, 0, 900, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3513, 71, 70, 0, 4059, 1856, 511, 0, 0, 0, 0, 0, 0, 0, 20, 6, 0, 0, 39, 0, 306, 4089, 3304, 0, 0, 0, 0, 0, 0, 0, 3968, 2269, 291, 1765, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10105, 6061, 10089, 9543, 3564, 10080, 7764, 8330] avg=8192.0000 max=10105 max_rank=0 min=3564 min_rank=4 imbalance_ratio=1.233521 max_load_reduction=55431 max_load_reduction_pct=0.845810 moved_to_echo=29738 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[33, 0, 2113, 5, 1353, 18, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2795, 0, 0, 161, 0, 0, 10, 3114, 2323, 332, 124, 0, 0, 0, 0, 3088, 414, 912, 0, 0, 43, 3741, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 2, 96, 5, 0, 879, 5, 3115, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 19, 3668, 0, 3579, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 1, 98, 0, 3, 0, 4079, 1854, 513, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 1149, 4096, 3339, 0, 0, 0, 0, 0, 0, 1, 4092, 1, 211, 1758, 7, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7603, 8859, 12084, 8107, 7281, 6910, 8616, 6076] avg=8192.0000 max=12084 max_rank=2 min=6076 min_rank=7 imbalance_ratio=1.475098 max_load_reduction=53452 max_load_reduction_pct=0.815613 moved_to_echo=28883 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 3, 2114, 0, 0, 2, 3809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 104, 43, 226, 3193, 2327, 332, 124, 0, 0, 0, 0, 655, 416, 29, 1, 3866, 0, 2266, 185, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 1, 15, 0, 0, 902, 107, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 1210, 0, 0, 4095, 70, 3578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3581, 0, 227, 0, 3996, 1855, 513, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 0, 27, 0, 3576, 4083, 3345, 0, 0, 0, 0, 0, 0, 3803, 582, 586, 2829, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5931, 6350, 7418, 6111, 8953, 10175, 11037, 9561] avg=8192.0000 max=11037 max_rank=6 min=5931 min_rank=0 imbalance_ratio=1.347290 max_load_reduction=54499 max_load_reduction_pct=0.831589 moved_to_echo=39726 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2116, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 1, 0, 3194, 2326, 333, 124, 0, 0, 0, 0, 4070, 415, 0, 0, 4081, 520, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 31, 0, 0, 902, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 0, 3716, 0, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 11, 78, 3800, 0, 4085, 1856, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3348, 0, 0, 0, 0, 0, 0, 3688, 4054, 54, 0, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6212, 6002, 9100, 9125, 7510, 10587, 7444, 9556] avg=8192.0000 max=10587 max_rank=5 min=6002 min_rank=1 imbalance_ratio=1.292358 max_load_reduction=54949 max_load_reduction_pct=0.838455 moved_to_echo=35097 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2110, 33, 2375, 117, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1331, 1529, 0, 4093, 0, 3170, 1308, 3055, 551, 335, 126, 0, 0, 0, 0, 0, 409, 1909, 0, 234, 35, 4, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 103, 2, 0, 0, 858, 1, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 395, 3915, 0, 2408, 103, 91, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 26, 1111, 247, 17, 0, 1285, 1860, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1239, 0, 1, 2778, 3346, 0, 0, 0, 0, 0, 0, 3551, 3963, 69, 0, 411, 57, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8485, 15498, 2601, 7671, 10960, 4904, 7364, 8053] avg=8192.0000 max=15498 max_rank=1 min=2601 min_rank=2 imbalance_ratio=1.891846 max_load_reduction=50038 max_load_reduction_pct=0.763519 moved_to_echo=31281 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 8064, 1664, 0, 4736, 256, 3712, 2048, 24832, 16896, 2688, 1024, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[17280, 23680, 4352, 7424, 12800, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[17664, 3328, 2944, 128, 8576, 5888, 17280, 10240, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[13568, 128, 640, 23808, 128, 27520, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 640, 128, 8448, 768, 8320, 128, 28928, 14848, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[24320, 3072, 384, 128, 128, 7040, 1024, 29952, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 1408, 128, 6400, 31104, 26752, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7808, 128, 16896, 4096, 4992, 256, 31872, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4094, 1516, 62, 11, 0, 12, 5, 2, 2586, 0, 4081, 4093, 3994, 2440, 14, 271, 0, 97, 0, 0, 0, 16, 2, 0, 8, 107, 0, 20, 84, 25, 0, 3789, 0, 4074, 0, 0, 42, 0, 1, 420, 4044, 0, 4080, 17, 0, 0, 0, 4069, 15, 0, 4041, 4056, 1, 3547, 0, 4053, 497, 0, 1157, 0, 4093, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4082, 1408, 0, 0, 4077, 0, 2, 116, 0, 0, 3915, 4089, 4087, 0, 25, 0, 0, 0, 0, 0, 1, 3544, 0, 0, 1, 171, 11, 0, 0, 0, 0, 4072, 389, 3966, 0, 0, 0, 0, 0, 225, 3760, 0, 4096, 1, 8, 0, 0, 4026, 16, 0, 4096, 3352, 0, 2296, 207, 4072, 0, 194, 0, 2, 4095, 1127, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[3, 4089, 0, 0, 0, 4096, 0, 47, 1, 206, 0, 3670, 4094, 4084, 2534, 0, 12, 0, 20, 1, 0, 0, 3860, 0, 0, 0, 186, 0, 7, 7, 0, 0, 4096, 0, 4057, 0, 0, 0, 59, 0, 75, 1621, 336, 4091, 0, 5, 0, 0, 4045, 4, 14, 3949, 3952, 4, 4095, 1, 4093, 0, 47, 0, 0, 4075, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4095, 72, 0, 0, 361, 2, 0, 10, 2232, 1, 4002, 4096, 4036, 0, 0, 554, 0, 0, 0, 0, 0, 39, 2, 0, 0, 14, 0, 2, 0, 0, 0, 4096, 0, 4093, 0, 109, 0, 0, 6, 4060, 3259, 0, 4082, 0, 1, 0, 111, 3003, 0, 0, 3961, 4078, 0, 4082, 17, 4089, 134, 2734, 0, 8, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 3585, 0, 0, 237, 0, 0, 0, 4091, 0, 4095, 4096, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 674, 0, 0, 0, 96, 0, 0, 0, 0, 0, 1765, 19, 4083, 0, 451, 0, 0, 0, 1271, 4083, 6, 3716, 0, 0, 0, 0, 4095, 0, 0, 4092, 4096, 0, 4096, 0, 4096, 0, 482, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4073, 28, 57, 0, 4084, 0, 0, 0, 3163, 24, 4057, 4084, 4096, 61, 0, 0, 33, 36, 0, 6, 0, 889, 37, 0, 0, 0, 76, 0, 13, 0, 27, 3997, 3, 3925, 0, 290, 0, 0, 70, 3212, 2665, 159, 1954, 22, 12, 0, 0, 4071, 306, 0, 3924, 3960, 5, 4015, 2, 4031, 0, 0, 0, 0, 4062, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[66, 4019, 0, 0, 0, 1902, 133, 0, 0, 0, 71, 4074, 4093, 3964, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 0, 1, 0, 0, 0, 0, 32, 0, 2518, 4096, 4042, 320, 0, 0, 0, 136, 4096, 4087, 59, 3468, 0, 0, 0, 469, 2234, 0, 0, 3957, 4010, 13, 4096, 55, 4087, 1143, 0, 8, 0, 4086, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[174, 27, 3618, 0, 0, 4096, 0, 0, 963, 23, 0, 2956, 4096, 4096, 0, 2005, 0, 0, 11, 3, 0, 0, 241, 3719, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 4096, 0, 3832, 0, 0, 0, 4096, 0, 3, 1992, 0, 36, 0, 10, 20, 0, 0, 4072, 1109, 0, 39, 279, 3908, 1, 179, 3558, 0, 4095, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 1408, 0, 0, 4077, 0, 2, 621, 384, 0, 0, 0, 0, 0, 0, 116, 0, 0, 3915, 0, 1776, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3544, 0, 2311, 1856, 1363, 735, 24, 0, 0, 0, 0, 1, 171, 11, 0, 0, 0, 0, 4089, 4072, 0, 0, 0, 0, 0, 0, 4072, 389, 2847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 3760, 0, 3475, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 16, 0, 0, 1496, 0, 2296, 207, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 194, 0, 2, 2732, 1127, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10574, 5832, 9834, 8344, 7308, 7469, 8041, 8134] avg=8192.0000 max=10574 max_rank=0 min=5832 min_rank=1 imbalance_ratio=1.290771 max_load_reduction=54962 max_load_reduction_pct=0.838654 moved_to_echo=30952 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 1516, 62, 11, 0, 12, 5, 619, 394, 0, 0, 0, 0, 0, 0, 2, 2586, 0, 4081, 0, 1735, 2440, 14, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 97, 0, 0, 0, 16, 2, 2259, 2246, 1362, 755, 23, 0, 0, 0, 0, 8, 107, 0, 20, 84, 25, 0, 4093, 4018, 0, 0, 0, 0, 0, 0, 3789, 0, 2925, 0, 0, 42, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 420, 4044, 0, 3461, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 15, 0, 0, 1810, 1, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 497, 0, 1157, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6713, 10858, 7031, 8355, 6757, 7942, 9442, 8438] avg=8192.0000 max=10858 max_rank=1 min=6713 min_rank=0 imbalance_ratio=1.325439 max_load_reduction=54678 max_load_reduction_pct=0.834320 moved_to_echo=32579 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[66, 4019, 0, 0, 0, 1902, 133, 0, 526, 391, 0, 0, 0, 0, 0, 0, 0, 0, 71, 4074, 0, 1722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 2242, 2220, 1360, 749, 22, 0, 0, 0, 0, 1, 0, 0, 0, 0, 32, 0, 4093, 3935, 0, 0, 0, 0, 0, 0, 2518, 4096, 2902, 320, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 59, 2942, 0, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 0, 0, 0, 1790, 13, 4096, 55, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1143, 0, 8, 0, 2726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7037, 5867, 6794, 8061, 9972, 11653, 8188, 7964] avg=8192.0000 max=11653 max_rank=5 min=5867 min_rank=1 imbalance_ratio=1.422485 max_load_reduction=53883 max_load_reduction_pct=0.822189 moved_to_echo=37777 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 4089, 0, 0, 0, 4096, 0, 47, 621, 393, 0, 0, 0, 0, 0, 0, 1, 206, 0, 3670, 0, 1774, 2534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 20, 1, 0, 0, 3860, 0, 2310, 2188, 1356, 751, 23, 0, 0, 0, 0, 0, 186, 0, 7, 7, 0, 0, 4094, 3926, 0, 0, 0, 0, 0, 0, 4096, 0, 2913, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 1621, 336, 3470, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 4, 14, 0, 1764, 4, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 47, 0, 0, 2719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9249, 8185, 10521, 8220, 7068, 5507, 9927, 6859] avg=8192.0000 max=10521 max_rank=2 min=5507 min_rank=5 imbalance_ratio=1.284302 max_load_reduction=55015 max_load_reduction_pct=0.839462 moved_to_echo=29361 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4073, 28, 57, 0, 4084, 0, 0, 296, 380, 0, 0, 0, 0, 0, 0, 0, 3163, 24, 4057, 0, 1779, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 36, 0, 6, 0, 889, 37, 2317, 2192, 1352, 727, 22, 0, 0, 0, 0, 0, 0, 76, 0, 13, 0, 27, 4084, 3902, 0, 0, 0, 0, 0, 0, 3997, 3, 2818, 0, 290, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 3212, 2665, 159, 1658, 22, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 306, 0, 0, 1768, 5, 4015, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 2710, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8918, 9084, 7611, 8102, 7178, 7728, 10167, 6748] avg=8192.0000 max=10167 max_rank=6 min=6748 min_rank=7 imbalance_ratio=1.241089 max_load_reduction=55369 max_load_reduction_pct=0.844864 moved_to_echo=31821 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 72, 0, 0, 361, 2, 0, 619, 396, 0, 0, 0, 0, 0, 0, 10, 2232, 1, 4002, 0, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 0, 0, 0, 39, 2, 2283, 2258, 1363, 758, 23, 0, 0, 0, 0, 0, 14, 0, 2, 0, 0, 0, 4096, 3938, 0, 0, 0, 0, 0, 0, 4096, 0, 2939, 0, 109, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 3259, 0, 3463, 0, 1, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 3003, 0, 0, 0, 1820, 0, 4082, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 134, 2734, 0, 8, 2732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5545, 7998, 7280, 8050, 7150, 10894, 8922, 9697] avg=8192.0000 max=10894 max_rank=5 min=5545 min_rank=0 imbalance_ratio=1.329834 max_load_reduction=54642 max_load_reduction_pct=0.833771 moved_to_echo=36663 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3585, 0, 0, 237, 0, 0, 564, 395, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4095, 0, 1779, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 674, 0, 2317, 2268, 1363, 756, 24, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 0, 0, 0, 1765, 19, 2932, 0, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 4083, 6, 3152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1828, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 482, 0, 0, 2733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8877, 9988, 7402, 8260, 5167, 8512, 10019, 7311] avg=8192.0000 max=10019 max_rank=6 min=5167 min_rank=4 imbalance_ratio=1.223022 max_load_reduction=55517 max_load_reduction_pct=0.847122 moved_to_echo=31009 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[174, 27, 3618, 0, 0, 4096, 0, 0, 306, 402, 0, 0, 0, 0, 0, 0, 963, 23, 0, 2956, 0, 1777, 0, 2005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3, 0, 0, 241, 3719, 2319, 617, 1367, 762, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4048, 0, 0, 0, 0, 0, 0, 4096, 4076, 2932, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 1686, 0, 36, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 492, 0, 39, 279, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 1, 179, 3558, 0, 2728, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8623, 7724, 9063, 8144, 14936, 5831, 830, 10385] avg=8192.0000 max=14936 max_rank=4 min=830 min_rank=6 imbalance_ratio=1.823242 max_load_reduction=50600 max_load_reduction_pct=0.772095 moved_to_echo=31982 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[896, 128, 256, 128, 256, 128, 9344, 3840, 18432, 15872, 11008, 6016, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25600, 384, 128, 0, 12800, 128, 26368, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32512, 1792, 3712, 4736, 128, 21888, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[17536, 23552, 640, 23424, 128, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[28544, 8704, 23296, 384, 4736, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1152, 12416, 128, 30976, 0, 14208, 5120, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 28672, 10240, 128, 128, 18944, 256, 128, 4224, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 640, 128, 128, 128, 128, 128, 32768, 32000, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1701, 0, 0, 2, 3708, 1960, 4082, 0, 4091, 0, 0, 16, 0, 0, 0, 0, 0, 3493, 0, 171, 0, 4096, 4067, 0, 266, 0, 0, 0, 13, 0, 0, 0, 0, 4070, 49, 3982, 3, 0, 22, 0, 4062, 3384, 6, 0, 1, 0, 0, 0, 8, 274, 60, 4037, 5, 4079, 1257, 4081, 5, 3951, 0, 0, 0, 942, 16, 3576] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4051, 617, 4094, 43, 4060, 0, 0, 0, 0, 8, 99, 0, 70, 3653, 265, 3973, 0, 4015, 3625, 3, 532, 0, 0, 0, 42, 0, 21, 0, 0, 4071, 0, 3977, 0, 0, 0, 0, 531, 1863, 0, 0, 1, 10, 4, 0, 0, 0, 14, 4096, 11, 4089, 2, 4084, 151, 3777, 2227, 0, 2, 628, 0, 2732] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[3203, 21, 0, 0, 2225, 3455, 4008, 0, 4054, 88, 0, 0, 1, 6, 0, 62, 0, 586, 65, 0, 19, 4083, 4049, 7, 1024, 587, 5, 7, 0, 0, 0, 0, 39, 4096, 2, 4064, 13, 0, 0, 0, 3497, 1138, 0, 0, 0, 3, 172, 49, 0, 0, 0, 4076, 0, 4089, 3908, 4091, 0, 4020, 682, 21, 0, 0, 97, 3924] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[32, 0, 0, 2, 121, 1600, 4094, 72, 4088, 0, 0, 1, 0, 0, 0, 27, 5, 1970, 96, 1184, 0, 4086, 4030, 0, 0, 7, 0, 5, 52, 1, 0, 0, 0, 4087, 3, 4089, 0, 0, 0, 0, 746, 4090, 0, 0, 0, 0, 0, 2, 0, 0, 1757, 3087, 3, 3993, 3553, 3414, 8, 4092, 3484, 2, 76, 3736, 8, 3833] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 477, 3744, 4069, 1216, 3975, 0, 37, 11, 0, 0, 174, 0, 0, 4095, 68, 2345, 0, 4096, 2771, 0, 559, 24, 0, 0, 0, 0, 7, 0, 0, 4091, 0, 4096, 0, 19, 119, 0, 878, 3928, 1, 0, 285, 0, 0, 0, 1, 4091, 0, 3826, 0, 4096, 38, 4072, 0, 4031, 0, 198, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1530, 27, 9, 0, 4093, 4052, 4092, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 1, 3625, 0, 0, 0, 3960, 4096, 2, 725, 1, 2, 0, 0, 0, 0, 4, 2, 3392, 0, 4029, 0, 20, 0, 38, 313, 83, 0, 0, 0, 0, 0, 0, 0, 11, 6, 4089, 0, 4096, 303, 4088, 0, 4072, 4084, 0, 46, 2453, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[60, 0, 0, 0, 4092, 2451, 4096, 617, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4033, 0, 4093, 0, 3936, 3593, 0, 13, 782, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 1, 4050, 0, 0, 0, 0, 0, 0, 14, 10, 0, 4096, 0, 4096, 0, 4096, 0, 4095, 0, 0, 1, 930, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 259, 0, 1, 2710, 91, 3764, 3772, 4095, 2, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 456, 4096, 3977, 0, 3370, 0, 18, 0, 0, 0, 3926, 0, 0, 4078, 0, 3234, 0, 2, 0, 512, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 173, 0, 4094, 0, 4096, 3852, 3986, 7, 3960, 53, 17, 43, 2794, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1701, 0, 0, 2, 3708, 1960, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 16, 0, 0, 0, 0, 2783, 1317, 0, 0, 0, 0, 0, 0, 0, 3493, 0, 171, 0, 134, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 13, 0, 0, 0, 4079, 2645, 0, 0, 0, 0, 0, 0, 0, 4070, 49, 3982, 3, 0, 22, 0, 141, 0, 0, 0, 0, 0, 0, 0, 4062, 3384, 6, 0, 1, 0, 0, 0, 2190, 1774, 107, 0, 0, 0, 0, 0, 8, 274, 60, 4037, 5, 0, 1257, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2036, 0, 0, 0, 942, 16, 3576, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8563, 8207, 7865, 7003, 8267, 11524, 7532, 6575] avg=8192.0000 max=11524 max_rank=5 min=6575 min_rank=7 imbalance_ratio=1.406738 max_load_reduction=54012 max_load_reduction_pct=0.824158 moved_to_echo=33898 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3203, 21, 0, 0, 2225, 3455, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 88, 0, 0, 1, 6, 0, 62, 2733, 1313, 0, 0, 0, 0, 0, 0, 0, 586, 65, 0, 19, 133, 4049, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 587, 5, 7, 0, 0, 0, 0, 4089, 2637, 0, 0, 0, 0, 0, 0, 39, 4096, 2, 4064, 13, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 3497, 1138, 0, 0, 0, 3, 172, 49, 2195, 1805, 105, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 3908, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2071, 682, 21, 0, 0, 97, 3924, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10074, 8257, 4859, 8349, 8358, 8964, 9880, 6795] avg=8192.0000 max=10074 max_rank=0 min=4859 min_rank=2 imbalance_ratio=1.229736 max_load_reduction=55462 max_load_reduction_pct=0.846283 moved_to_echo=33997 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4051, 617, 1196, 43, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 8, 99, 0, 2791, 1291, 0, 0, 0, 0, 0, 0, 70, 3653, 265, 3973, 0, 131, 3625, 3, 0, 0, 0, 0, 0, 0, 0, 0, 532, 0, 0, 0, 42, 0, 21, 0, 4089, 2593, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 3977, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 531, 1863, 0, 0, 1, 10, 4, 0, 2191, 1696, 107, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 11, 0, 2, 1893, 0, 0, 0, 0, 0, 0, 0, 0, 151, 1946, 2227, 0, 2, 628, 0, 2732, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10002, 8249, 11720, 7277, 8183, 6403, 6016, 7686] avg=8192.0000 max=11720 max_rank=2 min=6016 min_rank=6 imbalance_ratio=1.430664 max_load_reduction=53816 max_load_reduction_pct=0.821167 moved_to_echo=28288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 477, 3744, 1189, 1216, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 37, 11, 0, 0, 174, 0, 2774, 1317, 0, 0, 0, 0, 0, 0, 0, 4095, 68, 2345, 0, 134, 2771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 24, 0, 0, 0, 0, 7, 0, 4096, 2645, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 19, 119, 0, 144, 0, 0, 0, 0, 0, 0, 0, 878, 3928, 1, 0, 285, 0, 0, 0, 2185, 1810, 106, 0, 0, 0, 0, 0, 1, 4091, 0, 3826, 0, 0, 38, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 0, 198, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6628, 8288, 9413, 7331, 8469, 9193, 9843, 6371] avg=8192.0000 max=9843 max_rank=6 min=6371 min_rank=7 imbalance_ratio=1.201538 max_load_reduction=55693 max_load_reduction_pct=0.849808 moved_to_echo=33876 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1530, 27, 9, 0, 4093, 4052, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 2790, 1273, 0, 0, 0, 0, 0, 0, 1, 3625, 0, 0, 0, 129, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 725, 1, 2, 0, 0, 0, 0, 4, 4096, 2558, 0, 0, 0, 0, 0, 0, 2, 3392, 0, 4029, 0, 20, 0, 38, 146, 0, 0, 0, 0, 0, 0, 0, 313, 83, 0, 0, 0, 0, 0, 0, 2194, 1829, 107, 0, 0, 0, 0, 0, 0, 11, 6, 4089, 0, 0, 303, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2097, 4084, 0, 46, 2453, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10906, 8160, 7853, 7386, 7627, 4526, 6303, 12775] avg=8192.0000 max=12775 max_rank=7 min=4526 min_rank=5 imbalance_ratio=1.559448 max_load_reduction=52761 max_load_reduction_pct=0.805069 moved_to_echo=31231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[60, 0, 0, 0, 4092, 2451, 1196, 617, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2793, 1265, 0, 0, 0, 0, 0, 0, 2, 4033, 0, 4093, 0, 129, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 782, 0, 0, 0, 0, 0, 0, 4096, 2542, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 1, 4050, 0, 0, 0, 0, 0, 0, 2198, 1839, 107, 0, 0, 0, 0, 0, 14, 10, 0, 4096, 0, 0, 0, 1898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2109, 0, 0, 1, 930, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8416, 8154, 11850, 7433, 8334, 8195, 6018, 7136] avg=8192.0000 max=11850 max_rank=2 min=6018 min_rank=6 imbalance_ratio=1.446533 max_load_reduction=53686 max_load_reduction_pct=0.819183 moved_to_echo=29683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 0, 2, 121, 1600, 1196, 72, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 1, 0, 0, 0, 27, 2791, 1314, 0, 0, 0, 0, 0, 0, 5, 1970, 96, 1184, 0, 133, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 5, 52, 1, 0, 0, 3993, 2639, 0, 0, 0, 0, 0, 0, 0, 4087, 3, 4089, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 746, 4090, 0, 0, 0, 0, 0, 2, 1832, 1838, 107, 0, 0, 0, 0, 0, 0, 0, 1757, 3087, 3, 0, 3553, 1582, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2107, 3484, 2, 76, 3736, 8, 3833, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3023, 8221, 7418, 6697, 8326, 8615, 9982, 13254] avg=8192.0000 max=13254 max_rank=7 min=3023 min_rank=0 imbalance_ratio=1.617920 max_load_reduction=52282 max_load_reduction_pct=0.797760 moved_to_echo=40177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 259, 0, 1, 2710, 91, 1091, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 10, 2572, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 125, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3370, 0, 18, 0, 0, 0, 3926, 0, 4096, 2650, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 3234, 0, 2, 0, 512, 146, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 2143, 1784, 101, 0, 0, 0, 0, 0, 0, 173, 0, 4094, 0, 0, 3852, 1843, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2030, 53, 17, 43, 2794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7924, 8000, 4558, 14060, 7972, 8116, 9962, 4944] avg=8192.0000 max=14060 max_rank=3 min=4558 min_rank=2 imbalance_ratio=1.716309 max_load_reduction=51476 max_load_reduction_pct=0.785461 moved_to_echo=30994 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[14208, 18560, 128, 0, 384, 128, 256, 128, 17152, 14464, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 16512, 10624, 256, 256, 11520, 128, 26368, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[10624, 384, 128, 128, 21504, 18048, 9472, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32000, 128, 31616, 128, 128, 256, 640, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 4608, 1920, 31488, 128, 0, 12928, 14848, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 21504, 512, 11776, 512, 1152, 30208, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6528, 1408, 128, 128, 128, 128, 3968, 128, 32640, 20992, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32640, 128, 128, 128, 128, 128, 384, 128, 22144, 10496, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 38, 204, 0, 4042, 0, 1433, 4096, 4095, 0, 1, 0, 6, 0, 119, 3696, 38, 0, 0, 3490, 2647, 953, 11, 3971, 0, 0, 9, 0, 15, 0, 0, 4077, 72, 4017, 0, 12, 0, 0, 0, 0, 204, 4079, 102, 0, 4069, 0, 32, 3901, 3877, 2, 0, 4029, 0, 4039, 0, 55, 4080, 1, 0, 22, 0, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1, 77, 0, 0, 3428, 162, 224, 3989, 10, 0, 4040, 42, 0, 0, 19, 0, 3448, 0, 1, 525, 3188, 3964, 0, 3733, 0, 12, 0, 21, 0, 72, 35, 4087, 0, 4066, 1, 1, 1, 0, 1, 1274, 0, 4052, 3, 455, 4083, 0, 0, 4096, 4096, 0, 4095, 87, 2, 3953, 0, 20, 4076, 19, 63, 0, 0, 14, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[8, 0, 1, 0, 150, 0, 4076, 4096, 6, 0, 4075, 6, 0, 95, 0, 3261, 373, 0, 0, 3925, 4096, 0, 0, 3875, 0, 0, 0, 0, 1, 0, 0, 3884, 0, 4096, 23, 14, 29, 0, 0, 3275, 0, 433, 289, 54, 4059, 0, 0, 3844, 4096, 155, 790, 4088, 1, 4028, 0, 0, 4096, 0, 1, 0, 219, 18, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 5, 3, 4096, 0, 3855, 4071, 18, 0, 3758, 0, 2, 0, 0, 2435, 122, 0, 142, 0, 4096, 463, 0, 4096, 0, 1, 0, 5, 0, 30, 0, 4093, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 2689, 0, 0, 4094, 0, 0, 3364, 3990, 0, 4096, 3486, 0, 4044, 1, 0, 4088, 36, 0, 0, 0, 1, 0, 245] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 3471, 0, 1499, 3875, 0, 1, 4087, 0, 0, 0, 0, 182, 0, 0, 0, 4, 4094, 0, 47, 4096, 2, 0, 24, 0, 3, 8, 0, 4095, 0, 4094, 186, 0, 4, 4, 0, 0, 12, 4020, 0, 169, 4086, 0, 0, 4073, 4008, 467, 3861, 3502, 0, 4093, 1, 1, 4083, 19, 0, 0, 5, 3333, 25, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4059, 74, 3, 3861, 4021, 0, 3497, 0, 8, 0, 1231, 0, 0, 0, 0, 4096, 2115, 325, 0, 4060, 46, 0, 158, 2487, 0, 0, 10, 3679, 0, 4074, 0, 0, 0, 1077, 0, 0, 80, 3592, 215, 0, 3991, 22, 54, 3708, 4096, 0, 0, 2907, 0, 3961, 0, 0, 4024, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 29, 4047, 3, 25, 4096, 3959, 0, 4096, 0, 0, 8, 0, 1, 1393, 0, 0, 2644, 4063, 0, 0, 4096, 0, 0, 0, 1145, 0, 0, 0, 4096, 0, 2605, 0, 0, 2, 0, 0, 425, 0, 4096, 131, 2, 4096, 0, 0, 4096, 4086, 0, 4085, 24, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 4095, 0, 3382, 3793, 144, 0, 3238, 518, 0, 0, 0, 0, 3913, 0, 1, 2694, 3527, 0, 0, 4095, 0, 642, 2, 92, 0, 0, 8, 4087, 0, 4096, 0, 18, 386, 248, 0, 0, 0, 2282, 0, 0, 4095, 120, 1651, 5, 4095, 34, 2418, 2331, 0, 2832, 2, 0, 3021, 627, 1670, 0, 1366, 0, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 38, 204, 0, 4042, 0, 1433, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 6, 0, 119, 3696, 1132, 786, 0, 0, 0, 0, 0, 0, 38, 0, 0, 3490, 2647, 953, 11, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 15, 0, 0, 4077, 3429, 0, 0, 0, 0, 0, 0, 0, 72, 4017, 0, 12, 0, 0, 0, 0, 2925, 448, 0, 0, 0, 0, 0, 0, 204, 4079, 102, 0, 1201, 0, 32, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4029, 0, 3253, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 1, 0, 22, 0, 1, 1, 0, 2868, 408, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8678, 9835, 7777, 7530, 7474, 9519, 7339, 7384] avg=8192.0000 max=9835 max_rank=1 min=7339 min_rank=6 imbalance_ratio=1.200562 max_load_reduction=55701 max_load_reduction_pct=0.849930 moved_to_echo=31716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 1, 0, 150, 0, 4076, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4075, 6, 0, 95, 0, 3261, 1132, 784, 0, 0, 0, 0, 0, 0, 373, 0, 0, 3925, 4096, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3884, 3623, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 23, 14, 29, 0, 0, 3275, 2854, 473, 0, 0, 0, 0, 0, 0, 0, 433, 289, 54, 1198, 0, 0, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 790, 4088, 1, 3244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 219, 18, 0, 0, 2861, 398, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7196, 9359, 9017, 7508, 10764, 5818, 8278, 7596] avg=8192.0000 max=10764 max_rank=4 min=5818 min_rank=5 imbalance_ratio=1.313965 max_load_reduction=54772 max_load_reduction_pct=0.835754 moved_to_echo=32456 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 77, 0, 0, 3428, 162, 224, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4040, 42, 0, 0, 19, 0, 1103, 769, 0, 0, 0, 0, 0, 0, 3448, 0, 1, 525, 3188, 3964, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 21, 0, 72, 35, 4087, 3623, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 1, 1, 1, 0, 1, 1274, 2750, 473, 0, 0, 0, 0, 0, 0, 0, 4052, 3, 455, 1205, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 87, 2, 3184, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 19, 63, 0, 0, 14, 0, 0, 2878, 384, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6775, 5983, 11725, 7850, 8567, 9811, 7388, 7437] avg=8192.0000 max=11725 max_rank=2 min=5983 min_rank=1 imbalance_ratio=1.431274 max_load_reduction=53811 max_load_reduction_pct=0.821091 moved_to_echo=33203 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4059, 74, 3, 2791, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 3497, 0, 8, 0, 1231, 0, 1067, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2115, 325, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 158, 2487, 0, 0, 10, 3679, 3623, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 1077, 0, 0, 2990, 473, 0, 0, 0, 0, 0, 0, 80, 3592, 215, 0, 1178, 22, 54, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2907, 0, 3190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 0, 1, 0, 0, 0, 2813, 417, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6931, 10595, 7189, 10003, 8614, 8849, 6097, 7258] avg=8192.0000 max=10595 max_rank=1 min=6097 min_rank=6 imbalance_ratio=1.293335 max_load_reduction=54941 max_load_reduction_pct=0.838333 moved_to_echo=30818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 3471, 0, 1499, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 182, 1071, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 0, 47, 658, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 24, 0, 3, 8, 0, 4095, 3545, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 186, 0, 4, 4, 0, 0, 3017, 463, 0, 0, 0, 0, 0, 0, 12, 4020, 0, 169, 1206, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 3861, 3502, 0, 3296, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 19, 0, 0, 5, 3333, 25, 0, 2880, 421, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7773, 6138, 4803, 7677, 7768, 9480, 11128, 10769] avg=8192.0000 max=11128 max_rank=6 min=4803 min_rank=2 imbalance_ratio=1.358398 max_load_reduction=54408 max_load_reduction_pct=0.830200 moved_to_echo=39145 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 3, 4096, 0, 3855, 2943, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 3758, 0, 2, 0, 0, 2435, 1125, 787, 0, 0, 0, 0, 0, 0, 122, 0, 142, 0, 4096, 463, 0, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 30, 0, 4093, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 3017, 460, 0, 0, 0, 0, 0, 0, 0, 2689, 0, 0, 1208, 0, 0, 3364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3486, 0, 3257, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 36, 0, 0, 0, 1, 0, 245, 2886, 421, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10902, 8125, 5481, 7659, 7588, 7261, 10840, 7680] avg=8192.0000 max=10902 max_rank=0 min=5481 min_rank=2 imbalance_ratio=1.330811 max_load_reduction=54634 max_load_reduction_pct=0.833649 moved_to_echo=33369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 29, 4047, 3, 25, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 4096, 0, 0, 8, 0, 1, 1132, 796, 0, 0, 0, 0, 0, 0, 1393, 0, 0, 2644, 4063, 0, 0, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1145, 0, 0, 0, 4096, 3614, 0, 0, 0, 0, 0, 0, 0, 0, 2605, 0, 0, 2, 0, 0, 425, 3017, 472, 0, 0, 0, 0, 0, 0, 0, 4096, 131, 2, 1209, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 24, 0, 3295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2887, 421, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7065, 9992, 8758, 8855, 6521, 9534, 7404, 7407] avg=8192.0000 max=9992 max_rank=1 min=6521 min_rank=4 imbalance_ratio=1.219727 max_load_reduction=55544 max_load_reduction_pct=0.847534 moved_to_echo=30866 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 4095, 0, 3382, 2733, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 3238, 518, 0, 0, 0, 0, 1054, 555, 0, 0, 0, 0, 0, 0, 3913, 0, 1, 2694, 3527, 0, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 2, 92, 0, 0, 8, 4087, 3623, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 386, 248, 0, 0, 3020, 472, 0, 0, 0, 0, 0, 0, 0, 2282, 0, 0, 1206, 120, 1651, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 2418, 2331, 0, 2277, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3021, 627, 1670, 0, 1366, 0, 1, 1, 2889, 424, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10216, 5509, 10786, 8454, 8240, 5264, 7062, 10005] avg=8192.0000 max=10786 max_rank=2 min=5264 min_rank=5 imbalance_ratio=1.316650 max_load_reduction=54750 max_load_reduction_pct=0.835419 moved_to_echo=30571 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 256, 128, 27392, 256, 14592, 23040, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12288, 128, 26880, 640, 128, 128, 1408, 9600, 8832, 6144, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[9344, 0, 256, 17408, 27904, 5760, 128, 5248, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 768, 19456, 20480, 128, 25088, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 25344, 768, 768, 9728, 256, 1792, 27136, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[31616, 768, 1792, 128, 1664, 3456, 128, 256, 23040, 3328, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 768, 256, 3840, 128, 128, 128, 32128, 28672, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 31232, 256, 128, 512, 1408, 128, 4992, 23680, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 35, 0, 179, 0, 4085, 5, 0, 0, 0, 411, 1153, 0, 0, 0, 4096, 0, 4096, 0, 4085, 4092, 1652, 4022, 0, 8, 0, 8, 303, 3991, 0, 5, 4094, 0, 100, 0, 0, 0, 3645, 0, 2792, 0, 3996, 0, 1, 0, 21, 0, 3, 10, 4026, 16, 1698, 217, 0, 0, 4096, 0, 4055, 4074, 299, 0, 166, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3384, 8, 7, 0, 0, 236, 2872, 15, 429, 0, 4096, 11, 3788, 0, 4090, 4040, 1152, 4061, 0, 1943, 0, 6, 186, 3728, 0, 0, 3407, 750, 0, 0, 225, 0, 4096, 2, 1175, 0, 4096, 0, 0, 0, 61, 856, 4, 182, 0, 85, 0, 1197, 0, 0, 4092, 79, 4081, 4019, 3070, 0, 1, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3921, 241, 0, 0, 0, 1024, 2180, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 58, 3729, 0, 0, 3358, 0, 78, 0, 0, 0, 3969, 0, 756, 0, 4088, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 165, 3227, 0, 0, 4068, 0, 4096, 4096, 27, 0, 1808, 73, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[9, 0, 0, 2, 0, 1199, 51, 0, 6, 1, 93, 3792, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4069, 0, 4095, 0, 4046, 0, 0, 2, 3972, 0, 0, 214, 0, 81, 0, 1, 0, 4096, 4096, 2336, 0, 4096, 0, 0, 17, 0, 3858, 0, 0, 240, 0, 0, 605, 0, 0, 409, 0, 4088, 4091, 0, 2, 3672, 6, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 81, 1, 3357, 205, 1, 0, 0, 88, 0, 0, 0, 9, 4096, 0, 4091, 0, 4067, 3885, 516, 4096, 4, 30, 0, 0, 2324, 4096, 0, 0, 3708, 0, 3837, 0, 0, 3, 4096, 36, 1369, 0, 384, 0, 0, 0, 0, 716, 0, 0, 4042, 0, 0, 4067, 0, 0, 3937, 20, 4095, 4077, 4, 0, 181, 15, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 215, 223, 1121, 0, 20, 657, 0, 0, 0, 0, 3631, 2, 0, 51, 4059, 68, 3380, 0, 4044, 3859, 1, 4096, 0, 0, 0, 8, 4051, 3843, 0, 0, 4036, 0, 116, 0, 14, 31, 2804, 12, 3, 1, 3793, 26, 0, 118, 0, 3982, 0, 10, 0, 0, 5, 4089, 0, 0, 4096, 0, 3904, 4096, 0, 0, 1071, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[25, 69, 0, 160, 0, 19, 125, 0, 0, 0, 354, 66, 0, 50, 0, 4096, 0, 4055, 0, 4096, 3944, 0, 4044, 0, 1428, 1, 0, 3782, 1553, 410, 0, 4096, 0, 3547, 0, 0, 120, 93, 0, 2621, 0, 13, 0, 0, 0, 0, 2622, 0, 0, 4026, 17, 0, 4096, 0, 2, 3762, 7, 4096, 4042, 0, 0, 4096, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 18, 3967, 1, 0, 0, 0, 0, 4095, 694, 3, 0, 4096, 0, 4096, 0, 4092, 3600, 3821, 4096, 0, 5, 0, 0, 4087, 4096, 1057, 0, 293, 26, 4090, 0, 0, 0, 0, 0, 3637, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 4010, 0, 1, 2375, 0, 4001, 3554, 3, 0, 1372, 0, 288] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 35, 0, 179, 0, 4085, 5, 0, 4085, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 411, 1153, 0, 0, 0, 4096, 1213, 254, 0, 0, 0, 0, 0, 0, 0, 3455, 0, 0, 4092, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 303, 2557, 0, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 3645, 0, 2792, 1180, 297, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 1, 0, 21, 0, 3, 2852, 688, 0, 0, 0, 0, 0, 0, 10, 4026, 16, 1698, 217, 0, 0, 4096, 344, 0, 0, 0, 0, 0, 0, 0, 0, 2154, 4074, 299, 0, 166, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9559, 7127, 9199, 6975, 8014, 7561, 10407, 6694] avg=8192.0000 max=10407 max_rank=6 min=6694 min_rank=7 imbalance_ratio=1.270386 max_load_reduction=55129 max_load_reduction_pct=0.841202 moved_to_echo=32676 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3384, 8, 7, 4090, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 236, 2872, 15, 429, 0, 4096, 1221, 237, 0, 0, 0, 0, 0, 0, 11, 3195, 0, 0, 4040, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1943, 0, 6, 186, 2389, 0, 0, 3407, 0, 0, 0, 0, 0, 0, 0, 0, 750, 0, 0, 225, 0, 4096, 2, 1175, 1102, 275, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 61, 856, 4, 2881, 692, 0, 0, 0, 0, 0, 0, 182, 0, 85, 0, 1197, 0, 0, 4092, 318, 0, 0, 0, 0, 0, 0, 0, 79, 2168, 4019, 3070, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8669, 9106, 8398, 7931, 7625, 8590, 5874, 9343] avg=8192.0000 max=9343 max_rank=7 min=5874 min_rank=6 imbalance_ratio=1.140503 max_load_reduction=56193 max_load_reduction_pct=0.857437 moved_to_echo=31432 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[25, 69, 0, 160, 0, 19, 125, 0, 4096, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 354, 66, 0, 50, 0, 4096, 1226, 98, 0, 0, 0, 0, 0, 0, 0, 3420, 0, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1428, 1, 0, 3782, 996, 410, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 0, 0, 120, 93, 0, 2621, 459, 294, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 2622, 0, 2869, 695, 0, 0, 0, 0, 0, 0, 0, 4026, 17, 0, 4096, 0, 2, 3762, 341, 0, 0, 0, 0, 0, 0, 0, 7, 2175, 4042, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5669, 5890, 7364, 10713, 7134, 6199, 12244, 10323] avg=8192.0000 max=12244 max_rank=6 min=5669 min_rank=0 imbalance_ratio=1.494629 max_load_reduction=53292 max_load_reduction_pct=0.813171 moved_to_echo=35900 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 215, 223, 1121, 0, 20, 657, 0, 4044, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3631, 2, 0, 51, 4059, 1168, 244, 0, 0, 0, 0, 0, 0, 68, 2851, 0, 0, 3859, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4051, 2463, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 14, 31, 2804, 12, 3, 1136, 245, 0, 0, 0, 0, 0, 0, 1, 3793, 26, 0, 118, 0, 3982, 0, 2906, 662, 0, 0, 0, 0, 0, 0, 10, 0, 0, 5, 4089, 0, 0, 4096, 284, 0, 0, 0, 0, 0, 0, 0, 0, 2074, 4096, 0, 0, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7470, 9155, 6779, 10558, 4361, 11488, 8484, 7241] avg=8192.0000 max=11488 max_rank=5 min=4361 min_rank=4 imbalance_ratio=1.402344 max_load_reduction=54048 max_load_reduction_pct=0.824707 moved_to_echo=31574 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 81, 1, 3357, 205, 1, 4067, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 9, 4096, 1225, 260, 0, 0, 0, 0, 0, 0, 0, 3450, 0, 0, 3885, 516, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 2324, 2625, 0, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3837, 0, 0, 3, 4096, 36, 1369, 1211, 297, 0, 0, 0, 0, 0, 0, 0, 384, 0, 0, 0, 0, 716, 0, 2906, 694, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 4067, 0, 0, 3937, 344, 0, 0, 0, 0, 0, 0, 0, 20, 2176, 4077, 4, 0, 181, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8902, 5678, 7855, 8687, 10849, 4700, 12390, 6475] avg=8192.0000 max=12390 max_rank=6 min=4700 min_rank=5 imbalance_ratio=1.512451 max_load_reduction=53146 max_load_reduction_pct=0.810944 moved_to_echo=34414 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 2, 0, 1199, 51, 0, 4096, 1191, 0, 0, 0, 0, 0, 0, 6, 1, 93, 3792, 0, 0, 0, 4096, 1223, 252, 0, 0, 0, 0, 0, 0, 0, 3455, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 2, 2546, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 1, 0, 4096, 4096, 2336, 1174, 297, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 3858, 0, 2904, 693, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 605, 0, 0, 409, 344, 0, 0, 0, 0, 0, 0, 0, 0, 2172, 4091, 0, 2, 3672, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6548, 9463, 7524, 6808, 12081, 11568, 1598, 9946] avg=8192.0000 max=12081 max_rank=4 min=1598 min_rank=6 imbalance_ratio=1.474731 max_load_reduction=53455 max_load_reduction_pct=0.815659 moved_to_echo=35193 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3921, 241, 0, 4096, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 2180, 0, 0, 0, 4096, 1226, 237, 0, 0, 0, 0, 0, 0, 0, 3455, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 2390, 0, 0, 3358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 3969, 0, 756, 1102, 297, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4094, 0, 2905, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 3227, 0, 0, 4068, 344, 0, 0, 0, 0, 0, 0, 0, 0, 2175, 4096, 27, 0, 1808, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9449, 8763, 7551, 5806, 6202, 11782, 7804, 8179] avg=8192.0000 max=11782 max_rank=5 min=5806 min_rank=3 imbalance_ratio=1.438232 max_load_reduction=53754 max_load_reduction_pct=0.820221 moved_to_echo=33967 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 18, 3967, 1, 0, 4092, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 694, 3, 0, 4096, 1201, 265, 0, 0, 0, 0, 0, 0, 0, 3445, 0, 0, 3600, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4087, 2616, 1057, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4090, 0, 0, 0, 0, 0, 3637, 1215, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 2905, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 1, 2375, 349, 0, 0, 0, 0, 0, 0, 0, 0, 2118, 3554, 3, 0, 1372, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9270, 10354, 10866, 8058, 9270, 3648, 6735, 7335] avg=8192.0000 max=10866 max_rank=2 min=3648 min_rank=5 imbalance_ratio=1.326416 max_load_reduction=54670 max_load_reduction_pct=0.834198 moved_to_echo=26988 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 12416, 128, 1920, 21632, 0, 128, 26880, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 26752, 0, 0, 31616, 7168, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 20480, 128, 128, 256, 128, 16256, 128, 23168, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[7552, 128, 128, 14848, 18688, 1536, 128, 23296, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 17280, 32128, 3456, 128, 12416, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 2304, 17792, 768, 512, 128, 32768, 9728, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[896, 11904, 0, 256, 256, 22912, 4224, 14720, 8704, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 384, 256, 1664, 128, 19968, 1408, 128, 32768, 9600, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True + [2026-08-30 05:00:49] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 677.9 | throughput per GPU (TFLOP/s/GPU): 491.8 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.054679E+01 | loss scale: 1.0 | grad norm: 216.419 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[56, 1277, 2041, 27, 1877, 4, 732, 93, 1920, 62, 1101, 495, 665, 462, 13, 638, 323, 1935, 2415, 1595, 3290, 708, 557, 1625, 0, 3454, 182, 961, 545, 130, 732, 1236, 3332, 263, 105, 749, 1079, 51, 1414, 1107, 2015, 3235, 788, 306, 193, 1831, 1611, 1112, 341, 761, 869, 101, 44, 309, 3217, 360, 32, 1600, 3526, 1974, 571, 720, 675, 94] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[29, 765, 2668, 23, 2353, 5, 822, 16, 1218, 93, 726, 277, 833, 191, 7, 705, 182, 2844, 2162, 2086, 3330, 617, 844, 1450, 1, 2739, 145, 907, 695, 43, 563, 1338, 3622, 456, 45, 853, 976, 28, 1546, 1373, 2359, 3039, 801, 180, 102, 2204, 1479, 2067, 103, 482, 1219, 89, 96, 167, 3389, 277, 94, 1518, 3104, 1883, 200, 636, 427, 45] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[36, 758, 2319, 131, 1901, 13, 744, 49, 1689, 73, 1050, 736, 554, 279, 32, 491, 218, 2180, 2529, 1324, 3420, 1002, 803, 1718, 5, 3166, 157, 1151, 526, 94, 734, 1223, 3425, 172, 167, 1175, 824, 77, 1256, 1179, 1880, 3192, 682, 319, 177, 2556, 1057, 909, 318, 465, 1135, 146, 165, 626, 3743, 210, 16, 1925, 3475, 1569, 322, 638, 529, 102] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[40, 585, 1884, 66, 1833, 5, 807, 54, 1622, 59, 1172, 723, 534, 411, 24, 565, 224, 2177, 2427, 1822, 3526, 920, 572, 1372, 2, 3276, 165, 750, 441, 78, 688, 1376, 3464, 258, 129, 699, 474, 145, 1260, 1479, 2107, 3311, 969, 202, 226, 2634, 1584, 1357, 295, 710, 932, 146, 129, 467, 3566, 191, 16, 1700, 3479, 1585, 187, 999, 462, 174] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[85, 1150, 2062, 35, 2163, 10, 1177, 43, 1866, 36, 1709, 822, 553, 346, 73, 471, 336, 1732, 1869, 1517, 3571, 1330, 595, 1657, 15, 3200, 201, 781, 458, 134, 703, 1376, 2889, 198, 88, 916, 867, 201, 832, 1441, 2167, 2662, 937, 268, 259, 2345, 1588, 676, 341, 633, 834, 183, 201, 660, 3406, 221, 43, 1489, 3524, 1658, 488, 902, 386, 157] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[68, 1030, 2344, 52, 1596, 13, 869, 22, 1296, 93, 1609, 577, 420, 495, 16, 502, 215, 2063, 2191, 1831, 3471, 919, 746, 1443, 6, 3332, 150, 1316, 481, 278, 598, 1167, 3456, 103, 250, 642, 507, 187, 1257, 1470, 1833, 3265, 1082, 627, 155, 2609, 1369, 1828, 240, 568, 1025, 157, 159, 297, 2972, 164, 45, 1477, 2820, 1859, 158, 771, 854, 121] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[12, 1379, 2300, 94, 2201, 3, 754, 39, 1427, 69, 994, 453, 476, 276, 19, 350, 253, 2022, 2138, 1253, 3639, 1230, 528, 1445, 9, 3152, 178, 1057, 590, 259, 625, 1305, 3338, 166, 215, 1086, 311, 67, 1370, 1280, 2141, 2979, 1261, 373, 252, 2349, 1574, 2171, 93, 569, 1264, 80, 80, 345, 3021, 425, 14, 1289, 2761, 2522, 221, 623, 721, 46] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[23, 1127, 2680, 126, 1151, 9, 753, 44, 1838, 123, 1400, 530, 613, 356, 44, 462, 269, 2037, 2200, 1362, 3491, 865, 543, 1350, 3, 3019, 170, 1237, 666, 196, 897, 1500, 3229, 217, 77, 1255, 760, 86, 1228, 1800, 2243, 3349, 996, 411, 339, 2356, 1065, 1263, 274, 628, 609, 91, 81, 387, 3323, 167, 46, 1616, 3544, 1420, 335, 493, 713, 51] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[56, 1277, 2041, 27, 1877, 4, 732, 93, 1293, 823, 0, 0, 0, 0, 0, 0, 1920, 62, 1101, 495, 665, 462, 13, 638, 3061, 0, 0, 0, 0, 0, 0, 0, 323, 1935, 1199, 1595, 0, 708, 557, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3454, 182, 961, 545, 130, 732, 1236, 456, 393, 180, 0, 0, 0, 0, 0, 3332, 263, 105, 749, 1079, 51, 1414, 1107, 75, 0, 0, 0, 0, 0, 0, 0, 2015, 0, 788, 306, 193, 1375, 1611, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 341, 761, 869, 101, 44, 309, 3217, 360, 1942, 229, 0, 0, 0, 0, 0, 0, 32, 1600, 3271, 1974, 571, 720, 675, 94, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8417, 7942, 8269, 8175, 7400, 8173, 8937] avg=8192.0000 max=8937 max_rank=7 min=7400 min_rank=5 imbalance_ratio=1.090942 max_load_reduction=56599 max_load_reduction_pct=0.863632 moved_to_echo=32685 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[85, 1150, 2062, 35, 2163, 10, 1177, 43, 1063, 636, 0, 0, 0, 0, 0, 0, 1866, 36, 1709, 822, 553, 346, 73, 471, 3321, 0, 0, 0, 0, 0, 0, 0, 336, 1732, 929, 1517, 0, 1330, 595, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3200, 201, 781, 458, 134, 703, 1376, 584, 304, 180, 0, 0, 0, 0, 0, 2889, 198, 88, 916, 867, 201, 832, 1441, 75, 0, 0, 0, 0, 0, 0, 0, 2167, 0, 937, 268, 259, 1761, 1588, 676, 0, 0, 0, 0, 0, 0, 0, 0, 341, 633, 834, 183, 201, 660, 3406, 221, 1599, 250, 0, 0, 0, 0, 0, 0, 43, 1489, 3269, 1658, 488, 902, 386, 157, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8424, 9197, 8096, 7936, 7507, 7656, 8328, 8392] avg=8192.0000 max=9197 max_rank=1 min=7507 min_rank=4 imbalance_ratio=1.122681 max_load_reduction=56339 max_load_reduction_pct=0.859665 moved_to_echo=31883 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[68, 1030, 2344, 52, 1596, 13, 869, 22, 1304, 746, 0, 0, 0, 0, 0, 0, 1296, 93, 1609, 577, 420, 495, 16, 502, 3228, 0, 0, 0, 0, 0, 0, 0, 215, 2063, 1088, 1831, 0, 919, 746, 1443, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3332, 150, 1316, 481, 278, 598, 1167, 650, 357, 144, 0, 0, 0, 0, 0, 3456, 103, 250, 642, 507, 187, 1257, 1470, 60, 0, 0, 0, 0, 0, 0, 0, 1833, 0, 1082, 627, 155, 1959, 1369, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 240, 568, 1025, 157, 159, 297, 2972, 164, 1961, 243, 0, 0, 0, 0, 0, 0, 45, 1477, 2616, 1859, 158, 771, 854, 121, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8044, 8236, 8305, 8479, 7932, 8853, 7786, 7901] avg=8192.0000 max=8853 max_rank=5 min=7786 min_rank=6 imbalance_ratio=1.080688 max_load_reduction=56683 max_load_reduction_pct=0.864914 moved_to_echo=32472 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[29, 765, 2668, 23, 2353, 5, 822, 16, 1213, 736, 0, 0, 0, 0, 0, 0, 1218, 93, 726, 277, 833, 191, 7, 705, 3097, 0, 0, 0, 0, 0, 0, 0, 182, 2844, 1074, 2086, 0, 617, 844, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2739, 145, 907, 695, 43, 563, 1338, 549, 352, 158, 0, 0, 0, 0, 0, 3622, 456, 45, 853, 976, 28, 1546, 1373, 66, 0, 0, 0, 0, 0, 0, 0, 2359, 0, 801, 180, 102, 1655, 1479, 2067, 0, 0, 0, 0, 0, 0, 0, 0, 103, 482, 1219, 89, 96, 167, 3389, 277, 1826, 233, 0, 0, 0, 0, 0, 0, 94, 1518, 2880, 1883, 200, 636, 427, 45, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8630, 7147, 9097, 7490, 8965, 8643, 7881, 7683] avg=8192.0000 max=9097 max_rank=2 min=7147 min_rank=1 imbalance_ratio=1.110474 max_load_reduction=56439 max_load_reduction_pct=0.861191 moved_to_echo=33172 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[40, 585, 1884, 66, 1833, 5, 807, 54, 1322, 827, 0, 0, 0, 0, 0, 0, 1622, 59, 1172, 723, 534, 411, 24, 565, 3279, 0, 0, 0, 0, 0, 0, 0, 224, 2177, 1205, 1822, 0, 920, 572, 1372, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3276, 165, 750, 441, 78, 688, 1376, 656, 395, 178, 0, 0, 0, 0, 0, 3464, 258, 129, 699, 474, 145, 1260, 1479, 74, 0, 0, 0, 0, 0, 0, 0, 2107, 0, 969, 202, 226, 1978, 1584, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 295, 710, 932, 146, 129, 467, 3566, 191, 1989, 247, 0, 0, 0, 0, 0, 0, 16, 1700, 3227, 1585, 187, 999, 462, 174, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7423, 8389, 8292, 8005, 7982, 8423, 8672, 8350] avg=8192.0000 max=8672 max_rank=6 min=7423 min_rank=0 imbalance_ratio=1.058594 max_load_reduction=56864 max_load_reduction_pct=0.867676 moved_to_echo=33427 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[36, 758, 2319, 131, 1901, 13, 744, 49, 1275, 861, 0, 0, 0, 0, 0, 0, 1689, 73, 1050, 736, 554, 279, 32, 491, 3182, 0, 0, 0, 0, 0, 0, 0, 218, 2180, 1256, 1324, 0, 1002, 803, 1718, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3166, 157, 1151, 526, 94, 734, 1223, 637, 412, 177, 0, 0, 0, 0, 0, 3425, 172, 167, 1175, 824, 77, 1256, 1179, 74, 0, 0, 0, 0, 0, 0, 0, 1880, 0, 682, 319, 177, 1919, 1057, 909, 0, 0, 0, 0, 0, 0, 0, 0, 318, 465, 1135, 146, 165, 626, 3743, 210, 1917, 238, 0, 0, 0, 0, 0, 0, 16, 1925, 3224, 1569, 322, 638, 529, 102, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8086, 8501, 8282, 8349, 6943, 8963, 8325] avg=8192.0000 max=8963 max_rank=6 min=6943 min_rank=5 imbalance_ratio=1.094116 max_load_reduction=56573 max_load_reduction_pct=0.863235 moved_to_echo=32580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[12, 1379, 2300, 94, 2201, 3, 754, 39, 1190, 728, 0, 0, 0, 0, 0, 0, 1427, 69, 994, 453, 476, 276, 19, 350, 3386, 0, 0, 0, 0, 0, 0, 0, 253, 2022, 1062, 1253, 0, 1230, 528, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3152, 178, 1057, 590, 259, 625, 1305, 585, 348, 141, 0, 0, 0, 0, 0, 3338, 166, 215, 1086, 311, 67, 1370, 1280, 59, 0, 0, 0, 0, 0, 0, 0, 2141, 0, 1261, 373, 252, 1764, 1574, 2171, 0, 0, 0, 0, 0, 0, 0, 0, 93, 569, 1264, 80, 80, 345, 3021, 425, 1789, 253, 0, 0, 0, 0, 0, 0, 14, 1289, 2561, 2522, 221, 623, 721, 46, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8700, 7450, 7793, 8249, 7892, 9536, 7919, 7997] avg=8192.0000 max=9536 max_rank=5 min=7450 min_rank=1 imbalance_ratio=1.164062 max_load_reduction=56000 max_load_reduction_pct=0.854492 moved_to_echo=33344 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[23, 1127, 2680, 126, 1151, 9, 753, 44, 1338, 754, 0, 0, 0, 0, 0, 0, 1838, 123, 1400, 530, 613, 356, 44, 462, 3248, 0, 0, 0, 0, 0, 0, 0, 269, 2037, 1084, 1362, 0, 865, 543, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3019, 170, 1237, 666, 196, 897, 1500, 591, 362, 185, 0, 0, 0, 0, 0, 3229, 217, 77, 1255, 760, 86, 1228, 1800, 82, 0, 0, 0, 0, 0, 0, 0, 2243, 0, 996, 411, 339, 1765, 1065, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 274, 628, 609, 91, 81, 387, 3323, 167, 2011, 243, 0, 0, 0, 0, 0, 0, 46, 1616, 3277, 1420, 335, 493, 713, 51, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8005, 8614, 7510, 8826, 8734, 8082, 7814, 7951] avg=8192.0000 max=8826 max_rank=3 min=7510 min_rank=2 imbalance_ratio=1.077393 max_load_reduction=56710 max_load_reduction_pct=0.865326 moved_to_echo=32581 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 8192, 18304, 640, 15104, 128, 6784, 384, 10112, 6144, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2048, 17024, 8960, 12800, 0, 7680, 5248, 12160, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[26880, 1920, 1152, 7424, 5888, 896, 10240, 11136, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[16768, 0, 7552, 2688, 1792, 14208, 11392, 11392, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 12672, 24448, 14592, 2560, 5888, 4864, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2048, 4864, 7936, 1024, 1024, 3328, 26752, 2048, 15104, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 25344, 1408, 8192, 4480, 1280, 5632, 10624, 4736, 2944, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12928, 640, 9856, 4736, 4736, 2816, 256, 4224, 25856, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[23, 0, 0, 0, 7, 49, 0, 4070, 2902, 3084, 3710, 27, 4074, 2822, 4080, 3432, 6, 1, 7, 3847, 3120, 4090, 0, 3969, 3836, 669, 0, 0, 5, 0, 3683, 26, 8, 239, 1, 2947, 0, 0, 0, 0, 319, 0, 65, 29, 2, 0, 517, 0, 0, 69, 22, 3972, 0, 4016, 17, 0, 73, 0, 0, 349, 262, 1087, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[7, 0, 0, 2, 0, 12, 8, 3946, 2368, 2903, 2557, 0, 4034, 361, 4090, 2573, 0, 2, 9, 3990, 3274, 4043, 54, 3966, 723, 2804, 0, 5, 0, 0, 3652, 0, 0, 8, 0, 4080, 3, 0, 0, 56, 0, 0, 0, 0, 290, 5, 3024, 50, 0, 1, 3, 4080, 2, 3838, 58, 14, 36, 0, 0, 575, 2915, 1105, 8, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1, 1, 13, 0, 0, 8, 0, 4066, 2354, 3486, 4071, 1, 4049, 3197, 4095, 1432, 0, 0, 0, 4094, 267, 4017, 0, 4087, 245, 1378, 0, 0, 93, 0, 4083, 0, 115, 0, 2, 3921, 0, 0, 0, 0, 76, 0, 0, 0, 141, 0, 3402, 4, 0, 145, 0, 4096, 0, 4064, 6, 0, 0, 0, 6, 0, 453, 4067, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 5, 0, 0, 4096, 2841, 2323, 3640, 0, 4083, 70, 4096, 1520, 4, 3, 0, 2600, 2844, 4092, 39, 4049, 124, 3314, 7, 5, 0, 0, 3926, 0, 44, 4, 0, 4056, 0, 0, 0, 0, 660, 0, 5, 0, 11, 0, 2497, 24, 0, 0, 54, 4073, 0, 3974, 0, 0, 12, 4, 5, 547, 3908, 1976, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[6, 0, 6, 8, 0, 1, 0, 4051, 2046, 858, 3598, 0, 4077, 152, 4091, 3464, 0, 1, 0, 4083, 806, 4096, 3, 4087, 459, 3992, 0, 0, 0, 0, 4095, 1, 0, 0, 3, 4089, 9, 0, 0, 13, 0, 3, 3, 5, 481, 0, 920, 3, 0, 0, 0, 4082, 1, 4087, 1, 0, 1, 1, 1, 702, 3796, 3350, 1, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 6, 1, 0, 4090, 1461, 3365, 3416, 0, 3924, 395, 4093, 3038, 0, 0, 0, 4096, 3769, 4093, 0, 4081, 423, 3546, 1, 5, 0, 0, 4084, 0, 0, 1, 0, 2982, 2, 0, 0, 0, 6, 0, 1, 0, 1, 0, 901, 6, 0, 0, 2, 4047, 0, 4089, 5, 0, 0, 2, 0, 92, 1944, 3567, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3, 0, 3, 0, 0, 1, 2, 4094, 3409, 3284, 4030, 0, 4096, 412, 3778, 3776, 0, 0, 0, 3773, 3554, 4092, 0, 4009, 38, 1632, 0, 0, 0, 0, 4074, 0, 0, 3, 6, 4086, 1, 0, 0, 0, 0, 112, 2, 145, 386, 0, 2933, 15, 0, 0, 0, 3814, 1, 3562, 3, 0, 0, 0, 3, 46, 979, 1379, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4, 7, 6, 1, 6, 5, 16, 4076, 3493, 3140, 4080, 3, 3829, 1533, 4040, 1904, 4, 0, 46, 3978, 3807, 4087, 15, 3611, 458, 2220, 0, 10, 0, 14, 4078, 0, 0, 0, 8, 4088, 8, 2, 2, 1, 10, 5, 0, 3, 1005, 1, 58, 12, 7, 24, 0, 4045, 2, 3679, 0, 0, 0, 0, 0, 28, 594, 3475, 2, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[23, 0, 0, 0, 7, 49, 0, 4070, 2274, 1899, 0, 0, 0, 0, 0, 0, 2902, 2000, 0, 27, 0, 2822, 0, 3432, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 7, 3847, 3120, 0, 0, 1676, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 669, 0, 0, 5, 0, 3683, 26, 867, 188, 0, 0, 0, 0, 0, 0, 8, 239, 1, 2947, 0, 0, 0, 0, 2216, 2175, 0, 0, 0, 0, 0, 0, 319, 0, 65, 29, 2, 0, 517, 0, 4090, 1864, 0, 0, 0, 0, 0, 0, 0, 69, 22, 3972, 0, 4016, 17, 0, 217, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 349, 262, 1087, 3, 0, 2105, 1436, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8322, 11183, 8657, 9274, 7586, 6886, 8313, 5315] avg=8192.0000 max=11183 max_rank=1 min=5315 min_rank=7 imbalance_ratio=1.365112 max_load_reduction=54353 max_load_reduction_pct=0.829361 moved_to_echo=28100 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 2, 0, 12, 8, 3946, 1566, 1880, 0, 0, 0, 0, 0, 0, 2368, 1883, 0, 0, 0, 361, 0, 2573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 3990, 3274, 0, 54, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 723, 2804, 0, 5, 0, 0, 3652, 0, 816, 188, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4080, 3, 0, 0, 56, 2221, 2154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 5, 3024, 50, 4043, 1869, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4080, 2, 3838, 58, 14, 204, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 575, 2915, 1105, 8, 2, 2104, 991, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7421, 7185, 9003, 8188, 8522, 9281, 8200, 7736] avg=8192.0000 max=9281 max_rank=5 min=7185 min_rank=1 imbalance_ratio=1.132935 max_load_reduction=56255 max_load_reduction_pct=0.858383 moved_to_echo=33739 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 5, 0, 0, 4096, 2230, 1903, 0, 0, 0, 0, 0, 0, 2841, 1507, 0, 0, 0, 70, 0, 1520, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 2600, 2844, 0, 39, 1709, 0, 0, 0, 0, 0, 0, 0, 0, 124, 3314, 7, 5, 0, 0, 3926, 0, 653, 192, 0, 0, 0, 0, 0, 0, 44, 4, 0, 4056, 0, 0, 0, 0, 2224, 2180, 0, 0, 0, 0, 0, 0, 660, 0, 5, 0, 11, 0, 2497, 24, 4092, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 54, 4073, 0, 3974, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 12, 4, 5, 547, 3908, 1976, 0, 0, 2148, 1410, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 5938, 7199, 8221, 8508, 9161, 8264, 10010] avg=8192.0000 max=10010 max_rank=7 min=5938 min_rank=1 imbalance_ratio=1.221924 max_load_reduction=55526 max_load_reduction_pct=0.847260 moved_to_echo=35943 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 6, 8, 0, 1, 0, 4051, 2204, 1900, 0, 0, 0, 0, 0, 0, 2046, 557, 0, 0, 0, 152, 0, 3464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4083, 806, 0, 3, 1725, 0, 0, 0, 0, 0, 0, 0, 0, 459, 3992, 0, 0, 0, 0, 4095, 1, 241, 194, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4089, 9, 0, 0, 13, 2221, 2177, 0, 0, 0, 0, 0, 0, 0, 3, 3, 5, 481, 0, 920, 3, 4096, 1870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 1, 4087, 1, 0, 60, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 702, 3796, 3350, 1, 3, 2168, 1394, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 6219, 6618, 8982, 8512, 7381, 8231, 11417] avg=8192.0000 max=11417 max_rank=7 min=6219 min_rank=1 imbalance_ratio=1.393677 max_load_reduction=54119 max_load_reduction_pct=0.825790 moved_to_echo=35541 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 13, 0, 0, 8, 0, 4066, 2495, 1888, 0, 0, 0, 0, 0, 0, 2354, 2261, 0, 1, 0, 3197, 0, 1432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 267, 0, 0, 1725, 0, 0, 0, 0, 0, 0, 0, 0, 245, 1378, 0, 0, 93, 0, 4083, 0, 980, 194, 0, 0, 0, 0, 0, 0, 115, 0, 2, 3921, 0, 0, 0, 0, 2224, 2161, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 141, 0, 3402, 4, 4017, 1871, 0, 0, 0, 0, 0, 0, 0, 145, 0, 4096, 0, 4064, 6, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 453, 4067, 0, 0, 2168, 1576, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8472, 9245, 6086, 6973, 8423, 9511, 8556, 8270] avg=8192.0000 max=9511 max_rank=5 min=6086 min_rank=2 imbalance_ratio=1.161011 max_load_reduction=56025 max_load_reduction_pct=0.854874 moved_to_echo=34760 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 6, 1, 0, 4090, 2094, 1829, 0, 0, 0, 0, 0, 0, 1461, 2182, 0, 0, 0, 395, 0, 3038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3769, 0, 0, 1722, 0, 0, 0, 0, 0, 0, 0, 0, 423, 3546, 1, 5, 0, 0, 4084, 0, 946, 194, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2982, 2, 0, 0, 0, 2223, 2095, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 1, 0, 901, 6, 4093, 1870, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4047, 0, 4089, 5, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 92, 1944, 3567, 0, 0, 2165, 1322, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8021, 7076, 9587, 9199, 7303, 6878, 8380, 9092] avg=8192.0000 max=9587 max_rank=2 min=6878 min_rank=5 imbalance_ratio=1.170288 max_load_reduction=55949 max_load_reduction_pct=0.853714 moved_to_echo=31653 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 0, 0, 1, 2, 4094, 2470, 1909, 0, 0, 0, 0, 0, 0, 3409, 2130, 0, 0, 0, 412, 0, 3776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 3554, 0, 0, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 38, 1632, 0, 0, 0, 0, 4074, 0, 923, 190, 0, 0, 0, 0, 0, 0, 0, 3, 6, 4086, 1, 0, 0, 0, 2052, 2187, 0, 0, 0, 0, 0, 0, 0, 112, 2, 145, 386, 0, 2933, 15, 4092, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3814, 1, 3562, 3, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 46, 979, 1379, 0, 0, 2127, 1560, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8482, 9727, 9019, 6857, 8335, 9411, 7611, 6094] avg=8192.0000 max=9727 max_rank=1 min=6094 min_rank=7 imbalance_ratio=1.187378 max_load_reduction=55809 max_load_reduction_pct=0.851578 moved_to_echo=31451 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 7, 6, 1, 6, 5, 16, 4076, 2501, 1785, 0, 0, 0, 0, 0, 0, 3493, 2030, 0, 3, 0, 1533, 0, 1904, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 46, 3978, 3807, 0, 15, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 458, 2220, 0, 10, 0, 14, 4078, 0, 886, 176, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4088, 8, 2, 2, 1, 2194, 2044, 0, 0, 0, 0, 0, 0, 10, 5, 0, 3, 1005, 1, 58, 12, 4087, 1846, 0, 0, 0, 0, 0, 0, 7, 24, 0, 4045, 2, 3679, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 594, 3475, 2, 6, 1918, 1579, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8407, 8963, 9367, 7842, 8347, 7027, 7981, 7602] avg=8192.0000 max=9367 max_rank=2 min=7027 min_rank=5 imbalance_ratio=1.143433 max_load_reduction=56169 max_load_reduction_pct=0.857071 moved_to_echo=30957 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[20992, 14592, 0, 128, 0, 8960, 0, 21248, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 2432, 14976, 20096, 128, 128, 17024, 11392, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[6400, 19584, 128, 128, 128, 128, 31744, 128, 6400, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 128, 30464, 21504, 0, 128, 13440, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 256, 128, 30336, 128, 128, 128, 128, 17664, 17280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 256, 128, 32256, 128, 31360, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1152, 128, 128, 256, 2432, 128, 14336, 128, 32640, 14848, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 32512, 17920, 15104, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 74, 19, 3, 1, 1, 3777, 0, 0, 3694, 3862, 4096, 0, 0, 4096, 0, 0, 4068, 1, 0, 0, 4096, 0, 466, 0, 0, 4092, 6, 3, 4096, 4, 0, 0, 41, 3835, 10, 0, 3966, 3629, 31, 0, 0, 0, 0, 0, 221, 3703, 4096, 0, 0, 698, 0, 0, 4096, 0, 84, 0, 749, 0, 42, 3874] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 1, 3, 0, 184, 4090, 0, 0, 1768, 4062, 4066, 4, 122, 4093, 0, 0, 3907, 6, 0, 0, 4096, 0, 587, 0, 0, 4074, 1, 0, 4096, 0, 2, 0, 808, 371, 0, 8, 3055, 3178, 9, 47, 0, 1, 0, 0, 33, 4001, 4096, 0, 0, 4056, 0, 11, 4096, 0, 251, 0, 2355, 0, 0, 3995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 6, 0, 33, 1, 1, 34, 0, 4096, 7, 85, 456, 4072, 4096, 0, 1, 4092, 24, 1, 4050, 0, 0, 0, 4045, 0, 336, 0, 0, 4011, 63, 10, 4050, 38, 0, 0, 1359, 502, 0, 7, 4082, 3410, 154, 19, 0, 0, 14, 15, 408, 2809, 4078, 0, 0, 3580, 4, 18, 4090, 8, 1, 20, 3207, 0, 41, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 26, 0, 8, 10, 0, 0, 67, 4086, 0, 0, 1395, 4082, 4085, 13, 1, 4067, 0, 1, 4091, 1, 25, 4, 4079, 0, 473, 0, 0, 4076, 12, 0, 4086, 0, 1, 0, 3767, 713, 0, 9, 844, 2051, 25, 0, 15, 27, 0, 0, 573, 4074, 4091, 2, 0, 3262, 2, 0, 4093, 7, 215, 0, 3140, 0, 7, 3930] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 27, 1, 0, 0, 16, 0, 0, 4095, 2, 1, 282, 4093, 4096, 84, 28, 4091, 0, 1, 4020, 3, 2, 0, 4087, 3, 0, 0, 0, 4001, 12, 1, 4089, 0, 36, 0, 2741, 224, 0, 2, 4070, 3524, 0, 1, 6, 0, 0, 0, 5, 2603, 4095, 0, 0, 839, 0, 1, 4092, 15, 2055, 2, 4094, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 4, 0, 0, 2, 0, 1, 4088, 0, 65, 1197, 4095, 4094, 6, 141, 4063, 0, 0, 4092, 0, 0, 2, 4093, 0, 1285, 0, 2, 4084, 1, 0, 4076, 0, 4, 0, 3895, 133, 10, 12, 4073, 3502, 0, 1, 2, 3, 0, 0, 0, 1771, 4096, 0, 0, 1084, 0, 0, 4095, 0, 100, 0, 3263, 3, 1, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 13, 0, 0, 9, 0, 2, 4017, 0, 6, 2129, 4091, 4092, 0, 8, 4096, 0, 0, 4094, 0, 1, 0, 4087, 5, 16, 0, 0, 4095, 2, 0, 4079, 0, 0, 0, 1799, 888, 7, 0, 3631, 4075, 0, 0, 0, 7, 1, 0, 142, 1919, 4096, 0, 0, 3982, 0, 0, 4096, 0, 113, 0, 1837, 7, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[2, 9, 37, 0, 61, 1, 1, 0, 0, 4069, 0, 0, 22, 4043, 4086, 1, 29, 4091, 0, 1, 4096, 2, 0, 0, 4058, 9, 6, 0, 1, 4095, 0, 0, 4096, 0, 2, 9, 2414, 476, 0, 0, 4045, 3450, 0, 15, 1, 0, 0, 0, 1, 3557, 4095, 0, 0, 3690, 0, 3, 4088, 9, 1599, 2, 1233, 0, 0, 4031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 1, 3, 0, 4066, 2987, 1066, 0, 0, 0, 0, 0, 184, 4090, 0, 0, 1768, 2727, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 122, 4093, 0, 0, 3907, 6, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 587, 0, 0, 4074, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 808, 371, 0, 8, 731, 365, 0, 0, 0, 0, 0, 0, 3055, 3178, 9, 47, 0, 1, 0, 0, 717, 604, 0, 0, 0, 0, 0, 0, 33, 4001, 2313, 0, 0, 4056, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 0, 251, 0, 2355, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8773, 8131, 8390, 6381, 7611, 10414, 7710] avg=8192.0000 max=10414 max_rank=6 min=6381 min_rank=4 imbalance_ratio=1.271240 max_load_reduction=55122 max_load_reduction_pct=0.841095 moved_to_echo=32116 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 6, 0, 33, 1, 1, 34, 4096, 2982, 1061, 0, 0, 0, 0, 0, 0, 4096, 7, 85, 456, 2734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 24, 1, 4050, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3683, 0, 336, 0, 0, 4011, 63, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 38, 0, 0, 1359, 502, 0, 7, 732, 360, 0, 0, 0, 0, 0, 0, 4082, 3410, 154, 19, 0, 0, 14, 15, 714, 606, 0, 0, 0, 0, 0, 0, 408, 2809, 2303, 0, 0, 3580, 4, 18, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 8, 1, 20, 3207, 0, 41, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 7378, 8170, 8103, 7048, 9014, 9122, 8481] avg=8192.0000 max=9122 max_rank=6 min=7048 min_rank=4 imbalance_ratio=1.113525 max_load_reduction=56414 max_load_reduction_pct=0.860809 moved_to_echo=33665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 27, 1, 0, 0, 16, 0, 4096, 2984, 1066, 0, 0, 0, 0, 0, 0, 4095, 2, 1, 282, 2748, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4091, 0, 1, 4020, 3, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3720, 3, 0, 0, 0, 4001, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 36, 0, 2741, 224, 0, 2, 736, 364, 0, 0, 0, 0, 0, 0, 4070, 3524, 0, 1, 6, 0, 0, 0, 717, 609, 0, 0, 0, 0, 0, 0, 5, 2603, 2312, 0, 0, 839, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 15, 2055, 2, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7212, 8148, 7737, 8192, 8927, 5760, 11370] avg=8192.0000 max=11370 max_rank=7 min=5760 min_rank=6 imbalance_ratio=1.387939 max_load_reduction=54166 max_load_reduction_pct=0.826508 moved_to_echo=34249 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 74, 19, 3, 1, 4096, 2987, 1066, 0, 0, 0, 0, 0, 1, 3777, 0, 0, 3694, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4068, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 466, 0, 0, 4092, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 41, 3835, 10, 0, 695, 365, 0, 0, 0, 0, 0, 0, 3966, 3629, 31, 0, 0, 0, 0, 0, 717, 574, 0, 0, 0, 0, 0, 0, 221, 3703, 2313, 0, 0, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 0, 84, 0, 749, 0, 42, 3874, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 10065, 8168, 8295, 9046, 8917, 6935, 5858] avg=8192.0000 max=10065 max_rank=1 min=5858 min_rank=7 imbalance_ratio=1.228638 max_load_reduction=55471 max_load_reduction_pct=0.846420 moved_to_echo=30756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 26, 0, 8, 10, 0, 0, 4085, 2984, 1065, 0, 0, 0, 0, 0, 67, 4086, 0, 0, 1395, 2741, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4067, 0, 1, 4091, 1, 25, 4, 2, 0, 0, 0, 0, 0, 0, 0, 3714, 0, 473, 0, 0, 4076, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 1, 0, 3767, 713, 0, 9, 734, 363, 0, 0, 0, 0, 0, 0, 844, 2051, 25, 0, 15, 27, 0, 0, 716, 607, 0, 0, 0, 0, 0, 0, 573, 4074, 2310, 2, 0, 3262, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 7, 215, 0, 3140, 0, 7, 3930, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8302, 8192, 8275, 9673, 4285, 10223, 8408] avg=8192.0000 max=10223 max_rank=6 min=4285 min_rank=5 imbalance_ratio=1.247925 max_load_reduction=55313 max_load_reduction_pct=0.844009 moved_to_echo=32589 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 13, 0, 0, 9, 0, 4092, 2987, 1066, 0, 0, 0, 0, 0, 2, 4017, 0, 6, 2129, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 4094, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3720, 5, 16, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 1799, 888, 7, 0, 736, 364, 0, 0, 0, 0, 0, 0, 3631, 4075, 0, 0, 0, 7, 1, 0, 717, 608, 0, 0, 0, 0, 0, 0, 142, 1919, 2313, 0, 0, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 0, 113, 0, 1837, 7, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8901, 8202, 7838, 7873, 9039, 8356, 7159] avg=8192.0000 max=9039 max_rank=5 min=7159 min_rank=7 imbalance_ratio=1.103394 max_load_reduction=56497 max_load_reduction_pct=0.862076 moved_to_echo=32427 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4, 0, 0, 2, 0, 4094, 2986, 1066, 0, 0, 0, 0, 0, 1, 4088, 0, 65, 1197, 2749, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 141, 4063, 0, 0, 4092, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 3726, 0, 1285, 0, 2, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4, 0, 3895, 133, 10, 12, 737, 364, 0, 0, 0, 0, 0, 0, 4073, 3502, 0, 1, 2, 3, 0, 0, 717, 609, 0, 0, 0, 0, 0, 0, 0, 1771, 2313, 0, 0, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 0, 100, 0, 3263, 3, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8106, 8301, 9098, 9231, 8907, 5168, 8571] avg=8192.0000 max=9231 max_rank=4 min=5168 min_rank=6 imbalance_ratio=1.126831 max_load_reduction=56305 max_load_reduction_pct=0.859146 moved_to_echo=31877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 9, 37, 0, 61, 1, 1, 0, 4086, 2983, 1068, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 22, 2707, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4091, 0, 1, 4096, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3689, 9, 6, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 9, 2414, 476, 0, 0, 731, 364, 0, 0, 0, 0, 0, 0, 4045, 3450, 0, 15, 1, 0, 0, 0, 720, 605, 0, 0, 0, 0, 0, 0, 1, 3557, 2307, 0, 0, 3690, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 9, 1599, 2, 1233, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 6799, 8224, 7800, 8092, 8836, 9558, 7979] avg=8192.0000 max=9558 max_rank=6 min=6799 min_rank=1 imbalance_ratio=1.166748 max_load_reduction=55978 max_load_reduction_pct=0.854156 moved_to_echo=34465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1408, 24448, 18560, 128, 0, 21248, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[27776, 26880, 256, 128, 128, 128, 128, 128, 5760, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8960, 128, 4480, 128, 19968, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[29824, 128, 3200, 0, 128, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 32384, 128, 256, 11008, 21760, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[384, 32768, 128, 128, 32512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 256, 128, 128, 128, 32768, 23936, 8576, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 16896, 7168, 128, 128, 5888, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4078, 4016, 0, 2, 0, 4096, 0, 19, 0, 4095, 0, 11, 3, 0, 4019, 2, 37, 3688, 0, 241, 3967, 4080, 1242, 3, 4096, 3, 4, 9, 7, 0, 4091, 0, 51, 3566, 2798, 12, 1, 6, 0, 0, 11, 0, 3815, 51, 4096, 312, 0, 0, 639, 4, 0, 5, 0, 152, 0, 0, 15, 0, 4076, 0, 0, 4086, 31, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4093, 3958, 0, 27, 3, 4095, 5, 9, 0, 4086, 0, 1, 0, 19, 4061, 0, 0, 2557, 13, 3831, 4044, 4071, 29, 0, 4096, 15, 0, 0, 0, 1, 4087, 0, 52, 4083, 1821, 10, 0, 8, 0, 0, 10, 4, 4096, 0, 3938, 147, 7, 0, 40, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4092, 0, 4, 4093, 25, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4094, 4059, 0, 21, 0, 4096, 2, 4, 9, 4095, 0, 0, 1, 2, 4096, 0, 0, 4032, 2, 10, 4089, 4090, 3, 0, 4094, 3, 1, 0, 0, 7, 4092, 0, 0, 4096, 4068, 6, 0, 0, 0, 0, 0, 0, 4069, 5, 4080, 24, 1, 0, 52, 0, 0, 0, 5, 3, 0, 0, 0, 0, 4088, 0, 0, 4095, 0, 42] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4041, 4082, 0, 0, 0, 4081, 1, 10, 2, 4077, 1, 332, 7, 0, 4029, 16, 0, 3669, 0, 0, 4006, 4088, 0, 16, 4078, 1, 10, 3, 0, 0, 4086, 6, 1, 3734, 1650, 0, 42, 0, 0, 0, 0, 11, 3578, 10, 4047, 0, 1, 1, 3514, 44, 15, 0, 0, 0, 69, 18, 0, 5, 4056, 0, 2, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4095, 3273, 2, 0, 0, 4096, 1, 0, 0, 4091, 3, 33, 0, 1, 4091, 0, 0, 3864, 0, 1419, 4094, 4096, 566, 9, 4094, 0, 2, 0, 3, 2, 4093, 0, 0, 4094, 3837, 0, 4, 1, 0, 0, 0, 1, 2741, 0, 4069, 103, 0, 0, 556, 10, 2, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 4093, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4081, 3812, 0, 0, 0, 4095, 1, 2, 13, 4069, 1, 0, 2, 0, 4089, 3, 0, 3849, 5, 1, 4094, 4083, 565, 29, 4084, 0, 0, 0, 0, 0, 4088, 0, 0, 4077, 4058, 4, 0, 0, 3, 0, 0, 6, 4086, 0, 4089, 2, 0, 1, 16, 8, 10, 0, 0, 0, 4, 5, 0, 0, 4087, 2, 7, 4094, 2, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4081, 4096, 0, 0, 0, 4096, 0, 43, 0, 4095, 0, 0, 0, 6, 4094, 0, 0, 3161, 1, 4, 4096, 4051, 4, 50, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 4022, 3035, 0, 0, 0, 0, 0, 0, 0, 3667, 3, 4088, 16, 0, 0, 2248, 0, 0, 0, 0, 183, 1, 0, 0, 0, 4094, 11, 0, 4095, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4080, 3571, 0, 2, 0, 4069, 15, 0, 20, 4096, 0, 7, 0, 0, 4093, 1, 1, 4088, 0, 192, 4095, 4091, 45, 19, 4096, 30, 0, 0, 0, 7, 4083, 16, 0, 4072, 2501, 0, 235, 0, 10, 0, 0, 0, 4080, 1, 4080, 4, 0, 3, 1510, 0, 0, 0, 0, 0, 98, 0, 0, 0, 4087, 12, 8, 4092, 8, 18] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4078, 4016, 0, 2, 0, 234, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 11, 3, 0, 4019, 2, 0, 0, 0, 0, 0, 0, 0, 0, 37, 3688, 0, 241, 3456, 0, 1242, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 4, 9, 7, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 3566, 2798, 12, 1, 6, 0, 0, 900, 290, 0, 0, 0, 0, 0, 0, 11, 0, 3815, 51, 4096, 312, 0, 0, 221, 29, 10, 5, 0, 0, 0, 0, 639, 4, 0, 5, 0, 152, 0, 0, 4080, 2962, 0, 0, 0, 0, 0, 0, 15, 0, 4076, 0, 0, 4076, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 8101, 8667, 8205, 7624, 8550, 7842, 8198] avg=8192.0000 max=8667 max_rank=2 min=7624 min_rank=4 imbalance_ratio=1.057983 max_load_reduction=56869 max_load_reduction_pct=0.867752 moved_to_echo=32214 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4041, 4082, 0, 0, 0, 233, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4048, 1, 332, 7, 0, 4029, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3669, 0, 0, 3490, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 10, 3, 0, 0, 4086, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3734, 1650, 0, 42, 0, 0, 0, 897, 293, 0, 0, 0, 0, 0, 0, 0, 11, 3578, 10, 4047, 0, 1, 1, 223, 29, 11, 5, 0, 0, 0, 0, 3514, 44, 15, 0, 0, 0, 69, 18, 4088, 2951, 0, 0, 0, 0, 0, 0, 0, 5, 4056, 0, 2, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8367, 8435, 7175, 8179, 6617, 7916, 10699, 8148] avg=8192.0000 max=10699 max_rank=6 min=6617 min_rank=4 imbalance_ratio=1.306030 max_load_reduction=54837 max_load_reduction_pct=0.836746 moved_to_echo=33380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4093, 3958, 0, 27, 3, 233, 5, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 1, 0, 19, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2557, 13, 3831, 3523, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 15, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4083, 1821, 10, 0, 8, 0, 0, 900, 296, 0, 0, 0, 0, 0, 0, 10, 4, 4096, 0, 3938, 147, 7, 0, 225, 29, 11, 5, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 5, 0, 4071, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4, 4082, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8328, 8138, 9953, 8194, 7170, 8472, 7078, 8203] avg=8192.0000 max=9953 max_rank=2 min=7078 min_rank=6 imbalance_ratio=1.214966 max_load_reduction=55583 max_load_reduction_pct=0.848129 moved_to_echo=30923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 4059, 0, 21, 0, 234, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4066, 0, 0, 1, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 2, 10, 3562, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 1, 0, 0, 7, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 6, 0, 0, 0, 0, 900, 299, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 5, 4080, 24, 1, 0, 228, 29, 11, 5, 0, 0, 0, 0, 52, 0, 0, 0, 5, 3, 0, 0, 4090, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4084, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8414, 8174, 7609, 8192, 9369, 8452, 7112, 8214] avg=8192.0000 max=9369 max_rank=4 min=7112 min_rank=6 imbalance_ratio=1.143677 max_load_reduction=56167 max_load_reduction_pct=0.857040 moved_to_echo=33147 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4095, 3273, 2, 0, 0, 234, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 3, 33, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 1419, 3567, 0, 566, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 2, 0, 3, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3837, 0, 4, 1, 0, 0, 900, 299, 0, 0, 0, 0, 0, 0, 0, 1, 2741, 0, 4069, 103, 0, 0, 228, 29, 11, 5, 0, 0, 0, 0, 556, 10, 2, 0, 0, 0, 0, 0, 4096, 2962, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 4082, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7605, 8190, 9425, 8189, 9135, 7187, 7626, 8179] avg=8192.0000 max=9425 max_rank=2 min=7187 min_rank=5 imbalance_ratio=1.150513 max_load_reduction=56111 max_load_reduction_pct=0.856186 moved_to_echo=32127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4081, 4096, 0, 0, 0, 234, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3161, 1, 4, 3568, 0, 4, 50, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4022, 3035, 0, 0, 0, 0, 0, 900, 300, 0, 0, 0, 0, 0, 0, 0, 0, 3667, 3, 4088, 16, 0, 0, 228, 29, 11, 5, 0, 0, 0, 0, 2248, 0, 0, 0, 0, 183, 1, 0, 4051, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 11, 0, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8454, 8166, 6788, 8187, 8258, 8047, 9445, 8191] avg=8192.0000 max=9445 max_rank=6 min=6788 min_rank=2 imbalance_ratio=1.152954 max_load_reduction=56091 max_load_reduction_pct=0.855881 moved_to_echo=33941 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4081, 3812, 0, 0, 0, 233, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4040, 1, 0, 2, 0, 4089, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 5, 1, 3567, 0, 565, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4058, 4, 0, 0, 3, 0, 900, 299, 0, 0, 0, 0, 0, 0, 0, 6, 4086, 0, 4089, 2, 0, 1, 228, 29, 11, 5, 0, 0, 0, 0, 16, 8, 10, 0, 0, 0, 4, 5, 4083, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 7, 4083, 2, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8148, 8016, 8167, 9341, 8457, 7088, 8190] avg=8192.0000 max=9341 max_rank=4 min=7088 min_rank=6 imbalance_ratio=1.140259 max_load_reduction=56195 max_load_reduction_pct=0.857468 moved_to_echo=33076 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4080, 3571, 0, 2, 0, 222, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4063, 0, 7, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 0, 192, 3558, 0, 45, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 30, 0, 0, 0, 7, 4083, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 2501, 0, 235, 0, 10, 0, 900, 304, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 1, 4080, 4, 0, 3, 233, 33, 12, 9, 0, 0, 0, 0, 1510, 0, 0, 0, 0, 0, 98, 0, 4091, 2947, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 12, 8, 4080, 8, 18, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7890, 8184, 7903, 8223, 8022, 8455, 8646, 8213] avg=8192.0000 max=8646 max_rank=6 min=7890 min_rank=0 imbalance_ratio=1.055420 max_load_reduction=56890 max_load_reduction_pct=0.868073 moved_to_echo=33336 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32512, 128, 384, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 28928, 128, 5760, 28416, 0, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 30208, 128, 32512, 640, 128, 128, 1920, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8576, 128, 128, 128, 128, 384, 256, 128, 32768, 23680, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 30976, 128, 128, 128, 1920, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 31744, 23808, 128, 384, 128, 128, 0, 7296, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[18, 0, 3796, 1359, 7, 22, 0, 0, 8, 0, 104, 2, 4079, 0, 0, 4091, 4078, 4079, 0, 0, 23, 116, 4096, 0, 0, 0, 0, 0, 0, 0, 3882, 1, 418, 4075, 3988, 4081, 0, 6, 535, 4084, 0, 4, 3208, 0, 0, 0, 0, 13, 0, 169, 0, 49, 0, 10, 1, 21, 40, 4084, 12, 4090, 2952, 0, 3913, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1, 41, 2741, 2267, 0, 0, 6, 0, 1, 16, 13, 0, 4090, 0, 0, 4090, 4096, 4095, 1, 0, 0, 128, 4094, 0, 0, 0, 3, 0, 18, 0, 4095, 0, 0, 4077, 4089, 4076, 4, 0, 2, 4061, 0, 0, 4090, 3, 0, 1, 0, 0, 4, 40, 4, 24, 19, 0, 0, 1, 0, 4094, 3, 2972, 4091, 0, 4083, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3588, 1465, 0, 2, 0, 0, 0, 1, 0, 50, 4093, 0, 0, 4057, 4068, 4076, 5, 0, 14, 79, 3641, 0, 0, 0, 0, 0, 41, 0, 4095, 0, 0, 4092, 4095, 4082, 2, 0, 0, 4096, 26, 0, 3895, 0, 224, 0, 0, 3, 0, 38, 0, 1, 0, 0, 0, 1, 0, 4094, 0, 3443, 4080, 0, 4089, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 3645, 4096, 1, 0, 0, 0, 0, 0, 11, 12, 4096, 0, 0, 4090, 4095, 4096, 0, 4, 0, 0, 4077, 0, 0, 0, 26, 0, 34, 4, 4084, 0, 0, 4096, 4085, 4096, 0, 0, 325, 4096, 0, 5, 422, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 3762, 4096, 0, 4092, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[3, 86, 3840, 2966, 0, 0, 0, 15, 1, 0, 0, 0, 4087, 0, 0, 4094, 4076, 4096, 0, 0, 1, 67, 2951, 5, 0, 0, 729, 0, 0, 11, 4086, 0, 0, 3985, 4078, 4086, 4, 0, 31, 4094, 0, 0, 3641, 3, 8, 0, 2, 0, 0, 7, 0, 4, 515, 0, 0, 0, 0, 4030, 1, 1752, 4096, 0, 4085, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 11, 3964, 2978, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4080, 1, 0, 0, 1, 0, 1, 0, 4096, 0, 0, 4096, 4096, 4094, 0, 0, 0, 4096, 0, 0, 2533, 0, 9, 0, 0, 0, 0, 75, 5, 23, 39, 0, 0, 0, 0, 3932, 0, 2862, 4058, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 248, 2132, 4067, 0, 15, 14, 6, 0, 14, 2, 0, 4091, 0, 0, 4076, 3953, 4075, 5, 1, 0, 0, 3279, 0, 0, 0, 11, 0, 89, 21, 4087, 0, 0, 4085, 4096, 4068, 0, 0, 0, 4090, 4, 0, 4080, 0, 524, 6, 0, 0, 0, 47, 0, 0, 2, 0, 6, 0, 0, 4091, 2, 2057, 4095, 0, 4094, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[11, 1030, 643, 3706, 0, 0, 3, 0, 8, 13, 8, 0, 4088, 0, 0, 4083, 4003, 4076, 0, 2, 5, 0, 4094, 0, 0, 1, 3, 0, 32, 0, 4088, 0, 0, 4075, 4091, 4095, 0, 0, 26, 4085, 14, 0, 3670, 5, 1, 0, 0, 0, 2, 0, 0, 0, 178, 0, 0, 1, 8, 4096, 2, 3123, 4090, 0, 4074, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 3796, 1359, 7, 22, 0, 0, 1959, 124, 15, 0, 0, 0, 0, 0, 8, 0, 104, 2, 4064, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 288, 0, 0, 23, 116, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3882, 1, 3107, 891, 0, 0, 0, 0, 0, 0, 418, 4075, 3864, 0, 0, 6, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3208, 0, 0, 0, 0, 13, 3913, 684, 141, 0, 0, 0, 0, 0, 0, 169, 0, 49, 0, 10, 1, 21, 4084, 3940, 0, 0, 0, 0, 0, 0, 40, 1234, 12, 4090, 2952, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7300, 8269, 8601, 7881, 8898, 7963, 8274, 8350] avg=8192.0000 max=8898 max_rank=4 min=7300 min_rank=0 imbalance_ratio=1.086182 max_load_reduction=56638 max_load_reduction_pct=0.864227 moved_to_echo=33485 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3588, 1465, 0, 2, 0, 0, 1963, 127, 15, 0, 0, 0, 0, 0, 0, 1, 0, 50, 4078, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 288, 5, 0, 14, 79, 3641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 4095, 0, 3105, 893, 0, 0, 0, 0, 0, 0, 0, 4092, 3968, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 3895, 0, 224, 0, 0, 3, 4089, 683, 141, 0, 0, 0, 0, 0, 0, 38, 0, 1, 0, 0, 0, 1, 4096, 3941, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 3443, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7160, 8186, 8095, 8134, 8062, 9061, 8077, 8761] avg=8192.0000 max=9061 max_rank=5 min=7160 min_rank=0 imbalance_ratio=1.106079 max_load_reduction=56475 max_load_reduction_pct=0.861740 moved_to_echo=33961 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 86, 3840, 2966, 0, 0, 0, 15, 1933, 127, 15, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4072, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 289, 0, 0, 1, 67, 2951, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 729, 0, 0, 11, 4086, 0, 3120, 879, 0, 0, 0, 0, 0, 0, 0, 3985, 3951, 0, 4, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3641, 3, 8, 0, 2, 0, 4085, 687, 142, 0, 0, 0, 0, 0, 0, 7, 0, 4, 515, 0, 0, 0, 4094, 3944, 0, 0, 0, 0, 0, 0, 0, 1218, 1, 1752, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8985, 8167, 7389, 8825, 7971, 8568, 8564, 7067] avg=8192.0000 max=8985 max_rank=0 min=7067 min_rank=7 imbalance_ratio=1.096802 max_load_reduction=56551 max_load_reduction_pct=0.862900 moved_to_echo=32170 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3645, 4096, 1, 0, 0, 0, 1960, 127, 15, 0, 0, 0, 0, 0, 0, 0, 11, 12, 4081, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 289, 0, 4, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 34, 4, 4084, 0, 3120, 891, 0, 0, 0, 0, 0, 0, 0, 4096, 3958, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 422, 0, 0, 0, 0, 0, 4092, 687, 141, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 3955, 0, 0, 0, 0, 0, 0, 0, 1236, 0, 3762, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9845, 8194, 8465, 8159, 8379, 5347, 8053, 9094] avg=8192.0000 max=9845 max_rank=0 min=5347 min_rank=5 imbalance_ratio=1.201782 max_load_reduction=55691 max_load_reduction_pct=0.849777 moved_to_echo=30873 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 248, 2132, 4067, 0, 15, 14, 6, 1962, 127, 15, 0, 0, 0, 0, 0, 0, 14, 2, 0, 4076, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 288, 5, 1, 0, 0, 3279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 89, 21, 4087, 0, 3104, 892, 0, 0, 0, 0, 0, 0, 0, 4085, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4080, 0, 524, 6, 0, 0, 4094, 683, 141, 0, 0, 0, 0, 0, 0, 47, 0, 0, 2, 0, 6, 0, 4090, 3927, 0, 0, 0, 0, 0, 0, 0, 1237, 2, 2057, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8587, 8168, 7526, 8204, 8054, 9532, 8072, 7393] avg=8192.0000 max=9532 max_rank=5 min=7393 min_rank=7 imbalance_ratio=1.163574 max_load_reduction=56004 max_load_reduction_pct=0.854553 moved_to_echo=33051 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 41, 2741, 2267, 0, 0, 6, 0, 1963, 127, 15, 0, 0, 0, 0, 0, 1, 16, 13, 0, 4075, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 290, 1, 0, 0, 128, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 18, 0, 4095, 0, 3119, 893, 0, 0, 0, 0, 0, 0, 0, 4077, 3962, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 1, 0, 0, 4083, 686, 142, 0, 0, 0, 0, 0, 4, 40, 4, 24, 19, 0, 0, 1, 4061, 3934, 0, 0, 0, 0, 0, 0, 0, 1238, 3, 2972, 4091, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7161, 8195, 8609, 8128, 8045, 9005, 8087, 8306] avg=8192.0000 max=9005 max_rank=5 min=7161 min_rank=0 imbalance_ratio=1.099243 max_load_reduction=56531 max_load_reduction_pct=0.862595 moved_to_echo=33443 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 3964, 2978, 0, 0, 0, 6, 1886, 127, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 289, 0, 0, 0, 0, 4080, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4096, 0, 3120, 858, 0, 0, 0, 0, 0, 0, 0, 4096, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2533, 0, 9, 0, 0, 0, 4096, 687, 142, 0, 0, 0, 0, 0, 0, 75, 5, 23, 39, 0, 0, 0, 4096, 3952, 0, 0, 0, 0, 0, 0, 0, 1188, 0, 2862, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8987, 8177, 8466, 8076, 8065, 7467, 8190, 8108] avg=8192.0000 max=8987 max_rank=0 min=7467 min_rank=5 imbalance_ratio=1.097046 max_load_reduction=56549 max_load_reduction_pct=0.862869 moved_to_echo=31830 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[11, 1030, 643, 3706, 0, 0, 3, 0, 1968, 130, 20, 0, 0, 0, 0, 0, 8, 13, 8, 0, 4068, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 281, 0, 2, 5, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 32, 0, 4088, 0, 3108, 897, 0, 0, 0, 0, 0, 0, 0, 4075, 3961, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3670, 5, 1, 0, 0, 0, 4074, 687, 142, 0, 0, 0, 0, 0, 2, 0, 0, 0, 178, 0, 0, 1, 4085, 3953, 0, 0, 0, 0, 0, 0, 8, 1231, 2, 3123, 4090, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7511, 8180, 8385, 8129, 8062, 8593, 8219, 8457] avg=8192.0000 max=8593 max_rank=5 min=7511 min_rank=0 imbalance_ratio=1.048950 max_load_reduction=56943 max_load_reduction_pct=0.868881 moved_to_echo=33331 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 9856, 128, 24064, 31616, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 256, 128, 32640, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 896, 0, 256, 128, 32640, 128, 24960, 7168, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 25600, 128, 768, 128, 128, 128, 32640, 5504, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32512, 2304, 128, 128, 128, 512, 30336, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 32640, 31616, 0, 128, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 384, 128, 128, 768, 128, 128, 128, 32768, 31616, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 1536, 24448, 22912, 128, 128, 128, 128, 15616, 1024, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 3004, 1, 0, 3, 0, 0, 0, 2171, 3553, 327, 4093, 4096, 18, 3, 202, 36, 0, 0, 382, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 2959, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4085, 0, 4074, 4096, 0, 17, 20, 0, 0, 1, 0, 0, 3695, 26, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[33, 4082, 0, 0, 0, 4, 7, 2, 4088, 3700, 0, 4096, 4067, 2, 0, 171, 6, 4, 0, 49, 4096, 0, 0, 0, 4009, 0, 24, 3806, 4096, 0, 4093, 0, 0, 1139, 0, 0, 0, 0, 0, 20, 0, 4, 4096, 0, 0, 4092, 0, 18, 4084, 0, 4001, 4065, 0, 2, 0, 0, 0, 0, 2, 183, 3395, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[56, 4070, 0, 0, 0, 1, 0, 0, 4080, 2188, 162, 4048, 4040, 103, 0, 0, 109, 14, 23, 0, 4046, 18, 0, 0, 1751, 0, 287, 3997, 4079, 1, 4080, 3, 0, 3996, 0, 0, 42, 2, 59, 0, 0, 87, 4071, 19, 0, 4041, 0, 9, 4083, 11, 4078, 3870, 0, 0, 21, 3, 0, 0, 0, 0, 3968, 1, 9, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[10, 3952, 3, 0, 208, 0, 3, 0, 4092, 38, 1, 4061, 4080, 0, 0, 0, 0, 0, 0, 16, 4090, 1, 1, 3, 3710, 5, 41, 4084, 4096, 1, 4096, 0, 10, 3439, 0, 0, 0, 0, 3, 0, 101, 122, 4091, 0, 0, 4096, 0, 55, 4056, 0, 4096, 3927, 0, 196, 8, 0, 0, 0, 0, 548, 4094, 102, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[33, 4064, 0, 0, 25, 0, 52, 11, 4090, 3624, 0, 4096, 4081, 1324, 0, 0, 36, 8, 0, 8, 4079, 0, 0, 0, 1557, 0, 1, 4088, 4010, 0, 4088, 16, 0, 2409, 0, 7, 0, 7, 12, 0, 0, 3, 4096, 0, 0, 4086, 0, 0, 4053, 8, 3888, 4030, 0, 0, 0, 0, 0, 41, 1, 122, 3476, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[6, 3615, 0, 0, 2181, 0, 892, 0, 3939, 47, 0, 4086, 4088, 226, 0, 7, 0, 3, 2, 0, 4096, 0, 0, 4, 3980, 0, 0, 4058, 4085, 0, 3671, 97, 0, 2988, 0, 0, 1, 0, 5, 15, 0, 0, 4090, 0, 2, 4094, 3, 11, 4095, 129, 4088, 4094, 2, 159, 0, 1, 0, 14, 0, 483, 2179, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 3859, 0, 0, 0, 0, 0, 0, 3193, 4089, 0, 4095, 4095, 6, 7, 0, 0, 0, 0, 19, 4096, 0, 0, 3, 1867, 5, 0, 4095, 4074, 9, 4093, 0, 0, 4092, 0, 0, 11, 0, 41, 0, 0, 0, 4096, 0, 0, 4093, 0, 33, 4092, 0, 4055, 4096, 0, 16, 0, 0, 0, 0, 1, 179, 3126, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4085, 17, 0, 47, 1, 1, 7, 3244, 121, 0, 3879, 4073, 3172, 0, 0, 0, 0, 24, 62, 4096, 12, 36, 0, 2408, 0, 0, 4096, 4093, 0, 4092, 0, 0, 2937, 0, 24, 0, 0, 0, 0, 17, 0, 4095, 0, 0, 4091, 0, 0, 4096, 5, 4096, 4089, 0, 31, 0, 0, 0, 0, 0, 782, 3707, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[56, 4070, 0, 0, 0, 1, 0, 0, 2358, 1475, 45, 0, 0, 0, 0, 0, 4080, 2188, 162, 1690, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 14, 23, 0, 4046, 18, 0, 0, 2617, 1364, 0, 0, 0, 0, 0, 0, 1751, 0, 287, 1158, 0, 1, 4080, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 42, 2, 59, 0, 4079, 1075, 0, 0, 0, 0, 0, 0, 0, 87, 4026, 19, 0, 4041, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 34, 11, 4078, 3870, 0, 0, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 1, 9, 10, 2965, 1432, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8005, 8223, 8191, 7280, 9253, 8182, 8017, 8385] avg=8192.0000 max=9253 max_rank=4 min=7280 min_rank=3 imbalance_ratio=1.129517 max_load_reduction=56283 max_load_reduction_pct=0.858810 moved_to_echo=33837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[33, 4082, 0, 0, 0, 4, 7, 2, 2386, 1404, 46, 0, 0, 0, 0, 0, 4088, 3700, 0, 1710, 0, 2, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 49, 4096, 0, 0, 0, 2618, 1299, 0, 0, 0, 0, 0, 0, 4009, 0, 24, 1103, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1139, 0, 0, 0, 0, 0, 20, 4096, 1084, 0, 0, 0, 0, 0, 0, 0, 4, 4050, 0, 0, 4092, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4001, 4065, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 183, 3395, 0, 0, 0, 2983, 1432, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7964, 9671, 8072, 9229, 6339, 8164, 8102, 7995] avg=8192.0000 max=9671 max_rank=1 min=6339 min_rank=4 imbalance_ratio=1.180542 max_load_reduction=55865 max_load_reduction_pct=0.852432 moved_to_echo=30600 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[10, 3952, 3, 0, 208, 0, 3, 0, 2365, 1507, 45, 0, 0, 0, 0, 0, 4092, 38, 1, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4090, 1, 1, 3, 2600, 1394, 0, 0, 0, 0, 0, 0, 3710, 5, 41, 1183, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3439, 0, 0, 0, 0, 3, 0, 4096, 1086, 0, 0, 0, 0, 0, 0, 101, 122, 4046, 0, 0, 4096, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4096, 3927, 0, 196, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 4094, 102, 0, 0, 2994, 1422, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8093, 5827, 8105, 9036, 8634, 8420, 8261, 9160] avg=8192.0000 max=9160 max_rank=7 min=5827 min_rank=1 imbalance_ratio=1.118164 max_load_reduction=56376 max_load_reduction_pct=0.860229 moved_to_echo=34475 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[33, 4064, 0, 0, 25, 0, 52, 11, 2386, 1508, 46, 0, 0, 0, 0, 0, 4090, 3624, 0, 1710, 0, 1324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 8, 0, 8, 4079, 0, 0, 0, 2598, 1395, 0, 0, 0, 0, 0, 0, 1557, 0, 1, 1185, 0, 0, 4088, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2409, 0, 7, 0, 7, 12, 0, 4010, 1087, 0, 0, 0, 0, 0, 0, 0, 3, 4050, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 8, 3888, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 1, 122, 3476, 0, 6, 0, 2994, 1421, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 10748, 8124, 6847, 7532, 8139, 7960, 8061] avg=8192.0000 max=10748 max_rank=1 min=6847 min_rank=3 imbalance_ratio=1.312012 max_load_reduction=54788 max_load_reduction_pct=0.835999 moved_to_echo=31692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3004, 1, 0, 3, 0, 0, 0, 2384, 1511, 46, 0, 0, 0, 0, 0, 2171, 3553, 327, 1709, 0, 18, 3, 202, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 382, 4096, 0, 0, 0, 2618, 1398, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1187, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2959, 0, 0, 0, 0, 2, 0, 4096, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4074, 4096, 0, 17, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3695, 26, 0, 0, 3005, 1433, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6949, 7983, 8530, 9379, 8148, 8146, 8241, 8160] avg=8192.0000 max=9379 max_rank=3 min=6949 min_rank=0 imbalance_ratio=1.144897 max_load_reduction=56157 max_load_reduction_pct=0.856888 moved_to_echo=32695 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3859, 0, 0, 0, 0, 0, 0, 2385, 1511, 46, 0, 0, 0, 0, 0, 3193, 4089, 0, 1710, 0, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 3, 2623, 1398, 0, 0, 0, 0, 0, 0, 1867, 5, 0, 1186, 0, 9, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 11, 0, 41, 0, 4074, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 4093, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4055, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 179, 3126, 0, 0, 0, 3005, 1435, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7801, 9005, 8139, 7160, 9308, 8176, 8201, 7746] avg=8192.0000 max=9308 max_rank=4 min=7160 min_rank=3 imbalance_ratio=1.136230 max_load_reduction=56228 max_load_reduction_pct=0.857971 moved_to_echo=33431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[6, 3615, 0, 0, 2181, 0, 892, 0, 2380, 1497, 45, 0, 0, 0, 0, 0, 3939, 47, 0, 1706, 0, 226, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 4096, 0, 0, 4, 2625, 1385, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 1176, 0, 0, 3671, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2988, 0, 0, 1, 0, 5, 15, 4085, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 2, 4094, 3, 11, 0, 0, 0, 0, 0, 0, 0, 0, 34, 129, 4088, 4094, 2, 159, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 483, 2179, 0, 0, 0, 2999, 1436, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10616, 5925, 8115, 8924, 8183, 8155, 8507, 7111] avg=8192.0000 max=10616 max_rank=0 min=5925 min_rank=1 imbalance_ratio=1.295898 max_load_reduction=54920 max_load_reduction_pct=0.838013 moved_to_echo=31956 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4085, 17, 0, 47, 1, 1, 7, 2262, 1514, 49, 0, 0, 0, 0, 0, 3244, 121, 0, 1617, 0, 3172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 62, 4096, 12, 36, 0, 2629, 1401, 0, 0, 0, 0, 0, 0, 2408, 0, 0, 1181, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2937, 0, 24, 0, 0, 0, 0, 4093, 1085, 0, 0, 0, 0, 0, 0, 17, 0, 4046, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 5, 4096, 4089, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 3707, 0, 0, 0, 2988, 1441, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7983, 8154, 8260, 7681, 8139, 8154, 8247, 8918] avg=8192.0000 max=8918 max_rank=7 min=7681 min_rank=3 imbalance_ratio=1.088623 max_load_reduction=56618 max_load_reduction_pct=0.863922 moved_to_echo=33458 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 2304, 27648, 256, 128, 128, 23936, 11520, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 256, 32384, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 30848, 128, 0, 2560, 128, 1024, 128, 18944, 12032, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23424, 128, 384, 9472, 0, 128, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 256, 32384, 32384, 128, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 24064, 0, 128, 128, 128, 128, 128, 32640, 8704, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 128, 128, 640, 32768, 128, 128, 128, 20992, 11136, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[28928, 17408, 512, 13568, 0, 4864, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4033, 0, 0, 0, 4076, 0, 140, 4085, 0, 443, 1, 0, 0, 0, 0, 18, 4094, 4096, 3, 2, 0, 0, 0, 0, 0, 2331, 0, 20, 0, 0, 0, 4060, 31, 0, 28, 0, 4096, 2, 4042, 2621, 102, 1, 4094, 3720, 4081, 0, 4096, 6, 0, 17, 9, 0, 3894, 3816, 7, 2, 0, 0, 0, 4, 0, 1686, 1779, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4087, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4091, 1, 26, 0, 7, 0, 0, 0, 2059, 11, 0, 0, 0, 0, 3960, 76, 0, 13, 1, 4047, 0, 4092, 3749, 103, 0, 4073, 3690, 4093, 0, 4088, 0, 0, 18, 4, 0, 3860, 3879, 0, 146, 97, 0, 0, 1, 0, 105, 3029, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4094, 15, 4, 6, 4079, 0, 409, 4063, 1, 0, 44, 1, 0, 0, 0, 11, 4095, 4086, 8, 0, 0, 0, 2, 0, 11, 2771, 0, 1110, 0, 0, 0, 4096, 0, 0, 0, 3, 4011, 17, 4061, 3945, 5, 0, 4051, 4015, 4082, 0, 4096, 0, 19, 0, 0, 0, 4056, 360, 0, 166, 2, 0, 0, 36, 0, 181, 3524, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4089, 12, 0, 0, 4082, 2, 0, 4092, 0, 2, 0, 0, 0, 0, 9, 0, 4086, 4064, 0, 8, 0, 0, 12, 14, 0, 3674, 0, 640, 4, 0, 0, 4036, 0, 0, 23, 0, 4095, 0, 4088, 3594, 23, 0, 4070, 4096, 4090, 0, 4086, 0, 0, 1, 0, 0, 4036, 3433, 0, 14, 0, 0, 4, 1, 2, 119, 935, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4077, 0, 0, 0, 4075, 0, 342, 4021, 0, 0, 40, 14, 0, 28, 0, 13, 4082, 4069, 29, 35, 0, 0, 12, 0, 1, 4069, 16, 0, 26, 13, 0, 4093, 87, 4, 0, 0, 4058, 0, 3994, 4029, 0, 11, 4044, 3627, 4015, 0, 4085, 7, 0, 0, 0, 0, 56, 4081, 0, 42, 0, 23, 0, 82, 138, 9, 4089, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4083, 2, 13, 0, 4096, 0, 0, 4096, 0, 59, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 2293, 0, 0, 0, 0, 0, 2912, 0, 0, 58, 0, 4096, 0, 4072, 4096, 37, 0, 4061, 4083, 4096, 0, 4096, 0, 0, 0, 1, 0, 3977, 4096, 0, 0, 0, 0, 0, 0, 0, 1902, 1122, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4094, 1, 39, 0, 4082, 2, 0, 4094, 0, 34, 8, 0, 0, 2, 4, 9, 4085, 3999, 0, 2, 2, 6, 0, 13, 0, 3217, 0, 2, 7, 0, 0, 4088, 0, 0, 0, 0, 4094, 222, 4089, 2985, 0, 1, 4049, 4095, 4094, 0, 4084, 0, 13, 0, 0, 0, 1731, 3985, 0, 0, 0, 1, 5, 0, 0, 256, 4042, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4085, 0, 22, 0, 4060, 0, 15, 4046, 0, 58, 1, 0, 0, 2, 0, 0, 3983, 4094, 0, 0, 1, 0, 3, 2, 8, 4018, 0, 79, 5, 0, 0, 4082, 17, 13, 0, 0, 4096, 0, 4095, 4062, 1, 6, 4096, 4081, 4096, 0, 4091, 3, 2, 0, 0, 42, 17, 4042, 0, 9, 0, 0, 0, 22, 104, 0, 4077, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4007, 0, 140, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 1, 0, 0, 0, 0, 18, 4096, 3954, 0, 0, 0, 0, 0, 0, 4094, 4096, 3, 2, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 2331, 0, 20, 0, 0, 0, 4060, 930, 51, 0, 0, 0, 0, 0, 0, 31, 0, 28, 0, 420, 2, 4042, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 102, 1, 4094, 3720, 161, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 9, 0, 3894, 3816, 7, 2, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1686, 1779, 0, 3920, 773, 79, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8512, 8213, 7392, 7144, 8084, 9718, 8241] avg=8192.0000 max=9718 max_rank=6 min=7144 min_rank=4 imbalance_ratio=1.186279 max_load_reduction=55818 max_load_reduction_pct=0.851715 moved_to_echo=33187 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4017, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4014, 0, 0, 0, 0, 0, 0, 4087, 4091, 1, 26, 0, 7, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 2059, 11, 0, 0, 0, 0, 3960, 919, 52, 0, 0, 0, 0, 0, 0, 76, 0, 13, 1, 414, 0, 4092, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 4073, 3690, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4, 0, 3860, 3879, 0, 146, 1950, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 1, 0, 105, 3029, 6, 3931, 764, 81, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7959, 8102, 8230, 7001, 8345, 8028, 9857, 8014] avg=8192.0000 max=9857 max_rank=6 min=7001 min_rank=3 imbalance_ratio=1.203247 max_load_reduction=55679 max_load_reduction_pct=0.849594 moved_to_echo=34244 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4006, 0, 342, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 14, 0, 28, 0, 13, 4085, 3997, 0, 0, 0, 0, 0, 0, 4082, 4069, 29, 35, 0, 0, 12, 0, 18, 0, 0, 0, 0, 0, 0, 0, 1, 4069, 16, 0, 26, 13, 0, 4093, 922, 51, 0, 0, 0, 0, 0, 0, 87, 4, 0, 0, 415, 0, 3994, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4044, 3627, 159, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4081, 0, 42, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 82, 138, 9, 4089, 0, 3856, 766, 80, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8369, 8177, 8245, 9191, 8529, 7848, 6134, 9043] avg=8192.0000 max=9191 max_rank=3 min=6134 min_rank=6 imbalance_ratio=1.121948 max_load_reduction=56345 max_load_reduction_pct=0.859756 moved_to_echo=31554 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 0, 0, 4012, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 9, 0, 4086, 4009, 0, 0, 0, 0, 0, 0, 4086, 4064, 0, 8, 0, 0, 12, 14, 18, 0, 0, 0, 0, 0, 0, 0, 0, 3674, 0, 640, 4, 0, 0, 4036, 930, 52, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 419, 0, 4088, 3594, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4070, 4096, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4036, 3433, 0, 14, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 2, 119, 935, 0, 3929, 773, 80, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8106, 8202, 9336, 8124, 8350, 9457, 5843] avg=8192.0000 max=9457 max_rank=6 min=5843 min_rank=7 imbalance_ratio=1.154419 max_load_reduction=56079 max_load_reduction_pct=0.855698 moved_to_echo=31774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 4, 6, 4010, 0, 409, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 44, 1, 0, 0, 0, 11, 4096, 4014, 0, 0, 0, 0, 0, 0, 4095, 4086, 8, 0, 0, 0, 2, 0, 18, 0, 0, 0, 0, 0, 0, 0, 11, 2771, 0, 1110, 0, 0, 0, 4096, 911, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 411, 17, 4061, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4051, 4015, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4056, 360, 0, 166, 1932, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 36, 0, 181, 3524, 0, 3921, 757, 80, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8507, 8167, 8209, 8950, 8437, 8232, 6533, 8501] avg=8192.0000 max=8950 max_rank=3 min=6533 min_rank=6 imbalance_ratio=1.092529 max_load_reduction=56586 max_load_reduction_pct=0.863434 moved_to_echo=31703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 13, 0, 4026, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 4096, 4004, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 2293, 0, 0, 0, 0, 0, 2912, 930, 52, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 420, 0, 4072, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 4061, 4083, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3977, 4096, 0, 0, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1902, 1122, 0, 3934, 773, 79, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8159, 8207, 6187, 8646, 8343, 10047, 7810] avg=8192.0000 max=10047 max_rank=6 min=6187 min_rank=3 imbalance_ratio=1.226440 max_load_reduction=55489 max_load_reduction_pct=0.846695 moved_to_echo=34846 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 39, 0, 4012, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 8, 0, 0, 2, 4, 9, 4084, 4014, 0, 0, 0, 0, 0, 0, 4085, 3999, 0, 2, 2, 6, 0, 13, 18, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 2, 7, 0, 0, 4088, 930, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 222, 4089, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4049, 4095, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 1731, 3985, 0, 0, 1972, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 256, 4042, 0, 3932, 772, 80, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8155, 8125, 8296, 7716, 8307, 7701, 9088] avg=8192.0000 max=9088 max_rank=7 min=7701 min_rank=6 imbalance_ratio=1.109375 max_load_reduction=56448 max_load_reduction_pct=0.861328 moved_to_echo=32812 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 22, 0, 3983, 0, 15, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 1, 0, 0, 2, 0, 0, 4091, 4006, 0, 0, 0, 0, 0, 0, 3983, 4094, 0, 0, 1, 0, 3, 2, 22, 0, 0, 0, 0, 0, 0, 0, 8, 4018, 0, 79, 5, 0, 0, 4082, 936, 55, 0, 0, 0, 0, 0, 0, 17, 13, 0, 0, 408, 0, 4095, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4096, 4081, 157, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 42, 17, 4042, 0, 9, 1977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 104, 0, 4077, 0, 3939, 775, 79, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8158, 8105, 9183, 8595, 8344, 6089, 8996] avg=8192.0000 max=9183 max_rank=3 min=6089 min_rank=6 imbalance_ratio=1.120972 max_load_reduction=56353 max_load_reduction_pct=0.859879 moved_to_echo=32024 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 21632, 27776, 128, 384, 15744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 128, 32640, 31488, 1408, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 640, 128, 128, 0, 128, 128, 128, 32768, 32128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 256, 256, 4352, 22656, 128, 31488, 6272, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 32640, 128, 128, 128, 128, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 24448, 128, 1920, 128, 128, 0, 31360, 7424, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32128, 128, 1024, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 128, 128, 128, 3328, 256, 32640, 29184, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4096, 9, 0, 1453, 0, 172, 4086, 4, 0, 3633, 4027, 0, 0, 5, 0, 0, 0, 4094, 4096, 0, 0, 0, 199, 0, 3, 0, 0, 0, 4096, 3, 12, 0, 0, 1072, 11, 0, 4094, 0, 0, 2685, 3135, 4096, 0, 4, 4095, 4080, 8, 2, 4093, 0, 8, 0, 0, 4, 4088, 0, 0, 0, 0, 4063, 0, 0, 10, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4092, 0, 0, 2097, 5, 5, 4056, 1, 3, 4078, 4087, 0, 0, 158, 0, 2, 2, 4093, 4093, 0, 0, 179, 82, 0, 0, 0, 9, 4, 4087, 0, 0, 0, 4006, 1770, 2, 0, 4093, 0, 0, 0, 0, 4092, 16, 0, 4095, 4096, 0, 0, 3776, 2, 10, 3, 0, 0, 4091, 0, 0, 169, 89, 4089, 0, 0, 1, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4000, 8, 23, 11, 1, 0, 4086, 19, 0, 4096, 2541, 0, 0, 3606, 31, 12, 0, 4061, 4087, 0, 4, 942, 119, 0, 8, 0, 0, 5, 4084, 1, 3, 5, 419, 0, 0, 0, 4053, 0, 4, 622, 3682, 4077, 0, 0, 4095, 4096, 0, 0, 4075, 0, 0, 2, 0, 0, 4088, 0, 1, 1, 51, 4085, 3, 0, 429, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4096, 6, 0, 3445, 0, 0, 4095, 0, 4, 3989, 4067, 0, 0, 96, 0, 0, 1, 4096, 4092, 0, 0, 0, 0, 0, 22, 3, 0, 0, 4094, 26, 107, 0, 32, 355, 0, 0, 4093, 6, 0, 1239, 3611, 4092, 0, 6, 4075, 3485, 1, 0, 4096, 0, 0, 4, 0, 0, 4089, 0, 0, 0, 0, 4090, 23, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4074, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 1041, 0, 0, 3904, 0, 0, 0, 4096, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 268, 153, 0, 0, 4096, 0, 0, 4082, 3082, 4096, 0, 0, 4085, 3433, 17, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 422, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4086, 0, 0, 1381, 0, 194, 4053, 29, 5, 4000, 2527, 3, 0, 4051, 0, 19, 1, 4067, 4069, 13, 0, 0, 1, 1, 10, 0, 0, 0, 4067, 27, 41, 3, 1824, 2165, 44, 2, 4072, 6, 14, 17, 948, 4090, 0, 0, 4064, 3250, 0, 0, 4079, 39, 22, 14, 0, 0, 4056, 0, 28, 1, 0, 4069, 7, 0, 0, 77] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4086, 1, 16, 96, 6, 8, 4096, 0, 3, 4095, 1786, 0, 0, 4071, 2, 0, 1, 4093, 4096, 0, 0, 238, 0, 0, 0, 13, 0, 0, 4096, 0, 30, 0, 3, 2623, 0, 0, 4094, 0, 0, 803, 1457, 4095, 0, 0, 4096, 3988, 0, 1, 4090, 0, 0, 0, 0, 0, 4092, 0, 4, 0, 3, 4083, 0, 0, 1271, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4079, 4, 0, 18, 0, 0, 4080, 0, 2, 4096, 706, 17, 0, 3045, 2, 3, 0, 4090, 4094, 8, 11, 19, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 890, 373, 0, 17, 4044, 0, 0, 4010, 2823, 4078, 4, 0, 4096, 3455, 0, 0, 4087, 0, 0, 0, 0, 42, 4090, 0, 14, 0, 0, 4095, 55, 1, 912, 99] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 9, 0, 1453, 0, 172, 2994, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2912, 4027, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3882, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 3, 12, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 11, 0, 4094, 0, 0, 2685, 223, 176, 0, 0, 0, 0, 0, 0, 3135, 0, 0, 4, 2111, 4080, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 8, 0, 0, 4, 4088, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 10, 0, 1984, 1092, 498, 26, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8728, 6944, 8175, 8184, 8261, 9340, 8231, 7673] avg=8192.0000 max=9340 max_rank=5 min=6944 min_rank=1 imbalance_ratio=1.140137 max_load_reduction=56196 max_load_reduction_pct=0.857483 moved_to_echo=33505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 6, 0, 3445, 0, 0, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3198, 4067, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3, 0, 0, 4094, 26, 107, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 32, 355, 0, 0, 4093, 6, 0, 1239, 244, 176, 0, 0, 0, 0, 0, 0, 3611, 0, 0, 6, 2101, 3485, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4089, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 23, 0, 0, 0, 1974, 1095, 547, 27, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10547, 7365, 7975, 8317, 6145, 9204, 8227, 7756] avg=8192.0000 max=10547 max_rank=0 min=6145 min_rank=4 imbalance_ratio=1.287476 max_load_reduction=54989 max_load_reduction_pct=0.839066 moved_to_echo=31332 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 0, 1381, 0, 194, 2969, 29, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3207, 2527, 3, 0, 4051, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4067, 3857, 13, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4067, 27, 41, 3, 4063, 0, 0, 0, 0, 0, 0, 0, 1824, 2165, 44, 2, 4072, 6, 14, 17, 245, 175, 0, 0, 0, 0, 0, 0, 948, 0, 0, 0, 2095, 3250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 39, 22, 14, 0, 0, 4056, 0, 37, 0, 0, 0, 0, 0, 0, 0, 28, 1, 0, 4069, 7, 0, 0, 77, 1969, 1084, 548, 27, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8659, 9812, 7940, 8211, 8564, 6293, 8247, 7810] avg=8192.0000 max=9812 max_rank=1 min=6293 min_rank=5 imbalance_ratio=1.197754 max_load_reduction=55724 max_load_reduction_pct=0.850281 moved_to_echo=30914 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4000, 8, 23, 11, 1, 0, 2994, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3284, 2541, 0, 0, 3606, 31, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 3874, 0, 4, 942, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 5, 4084, 1, 3, 5, 4050, 0, 0, 0, 0, 0, 0, 0, 419, 0, 0, 0, 4053, 0, 4, 622, 251, 176, 0, 0, 0, 0, 0, 0, 3682, 0, 0, 0, 2111, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 2, 0, 0, 4088, 0, 37, 0, 0, 0, 0, 0, 0, 0, 1, 1, 51, 4085, 3, 0, 429, 0, 1984, 1092, 561, 27, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7056, 9474, 9000, 8156, 5525, 9889, 8202, 8234] avg=8192.0000 max=9889 max_rank=5 min=5525 min_rank=4 imbalance_ratio=1.207153 max_load_reduction=55647 max_load_reduction_pct=0.849106 moved_to_echo=31850 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4086, 1, 16, 96, 6, 8, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3283, 1786, 0, 0, 4071, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 3882, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4096, 0, 30, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 3, 2623, 0, 0, 4094, 0, 0, 803, 251, 176, 0, 0, 0, 0, 0, 0, 1457, 0, 0, 0, 2112, 3988, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4092, 0, 38, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 4083, 0, 0, 1271, 0, 1984, 1095, 561, 27, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7214, 9145, 8214, 8207, 7950, 7558, 8220, 9028] avg=8192.0000 max=9145 max_rank=1 min=7214 min_rank=0 imbalance_ratio=1.116333 max_load_reduction=56391 max_load_reduction_pct=0.860458 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 2097, 5, 5, 2972, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3269, 4087, 0, 0, 158, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 3879, 0, 0, 179, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 4087, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 4006, 1770, 2, 0, 4093, 0, 0, 0, 250, 176, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 2111, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3776, 2, 10, 3, 0, 0, 4091, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 169, 89, 4089, 0, 0, 1, 3, 1984, 1084, 559, 27, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9172, 7519, 8235, 8165, 10297, 6223, 7920, 8005] avg=8192.0000 max=10297 max_rank=4 min=6223 min_rank=5 imbalance_ratio=1.256958 max_load_reduction=55239 max_load_reduction_pct=0.842880 moved_to_echo=32445 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4074, 0, 0, 0, 0, 0, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3282, 1041, 0, 0, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3882, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 268, 153, 0, 0, 4096, 0, 0, 4082, 251, 176, 0, 0, 0, 0, 0, 0, 3082, 0, 0, 0, 2106, 3433, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 4096, 0, 0, 0, 0, 1979, 1095, 561, 26, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7075, 8227, 7994, 8167, 9026, 8638, 8230, 8179] avg=8192.0000 max=9026 max_rank=4 min=7075 min_rank=0 imbalance_ratio=1.101807 max_load_reduction=56510 max_load_reduction_pct=0.862274 moved_to_echo=34073 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4079, 4, 0, 18, 0, 0, 2984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3275, 706, 17, 0, 3045, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3875, 8, 11, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 890, 373, 0, 17, 4044, 0, 0, 4010, 255, 179, 0, 0, 0, 0, 0, 0, 2823, 0, 4, 0, 2109, 3455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 42, 4090, 0, 40, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4095, 55, 1, 912, 99, 1987, 1096, 566, 26, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7085, 7050, 8003, 8129, 9768, 8391, 8259, 8851] avg=8192.0000 max=9768 max_rank=4 min=7050 min_rank=1 imbalance_ratio=1.192383 max_load_reduction=55768 max_load_reduction_pct=0.850952 moved_to_echo=35269 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[18816, 0, 128, 128, 16896, 29952, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 8576, 128, 384, 23936, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 25728, 20864, 128, 0, 18944, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 31104, 128, 128, 1408, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32512, 128, 128, 128, 0, 128, 32768, 0, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 256, 128, 32512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[7552, 8576, 128, 128, 32640, 128, 128, 13568, 2048, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 256, 640, 32768, 128, 128, 2688, 256, 15872, 8832, 4480, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[3, 0, 0, 4092, 0, 8, 0, 4084, 3, 41, 6, 0, 4003, 11, 2, 4085, 4089, 4059, 0, 19, 0, 788, 0, 0, 0, 64, 4081, 0, 0, 35, 4083, 11, 1, 0, 4066, 0, 201, 0, 0, 0, 91, 0, 207, 4095, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 49, 0, 4096, 4072, 10, 4096, 35, 3258, 0, 3591, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 26, 584, 0, 0, 117, 3721, 0, 0, 29, 0, 2209, 0, 0, 4094, 4090, 2312, 0, 0, 0, 48, 0, 0, 0, 4092, 3954, 0, 10, 0, 4096, 0, 0, 0, 4090, 4, 3032, 0, 0, 0, 397, 0, 41, 4096, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 4096, 4096, 19, 4076, 0, 4044, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[35, 0, 0, 4087, 0, 0, 128, 4063, 0, 0, 0, 0, 323, 0, 0, 4096, 4089, 4033, 0, 0, 0, 54, 8, 0, 20, 3637, 3653, 8, 28, 5, 4094, 0, 0, 0, 4090, 0, 1365, 2, 0, 0, 0, 0, 10, 4076, 0, 3, 0, 0, 2, 0, 1, 4079, 0, 0, 18, 4096, 4094, 0, 4092, 13, 4096, 0, 3138, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 90, 3781, 12, 42, 0, 4096, 0, 50, 0, 20, 1601, 12, 110, 4065, 4042, 4064, 0, 0, 5, 0, 0, 23, 0, 3680, 3904, 0, 273, 0, 1834, 5, 0, 30, 4025, 8, 1047, 0, 4, 4, 170, 1, 70, 4070, 1, 10, 0, 0, 12, 13, 0, 4073, 6, 0, 0, 4062, 4044, 4, 4078, 0, 4082, 0, 4013, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 1, 59, 4081, 1, 9, 0, 0, 955, 0, 1, 4096, 4096, 3836, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 3565, 0, 0, 0, 4096, 0, 3917, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 47, 0, 4074, 4096, 0, 4096, 0, 4065, 0, 4053, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3793, 0, 0, 834, 4093, 0, 82, 0, 0, 3345, 0, 0, 4093, 4092, 4095, 2, 1, 7, 0, 12, 4, 0, 450, 4050, 1, 0, 0, 3011, 4, 97, 0, 4094, 0, 650, 1, 4, 5, 39, 0, 0, 4085, 0, 0, 2, 0, 1, 5, 0, 4088, 3, 0, 0, 4091, 4092, 9, 4096, 0, 4088, 7, 4095, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 7, 4092, 0, 0, 20, 4029, 0, 0, 6, 0, 1230, 1, 58, 4096, 4093, 3983, 0, 2, 0, 191, 0, 0, 0, 243, 2614, 0, 0, 0, 4095, 0, 0, 0, 4088, 1, 4096, 0, 0, 0, 5, 0, 5, 4089, 1, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 1, 4096, 4096, 0, 4094, 0, 4005, 0, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[28, 0, 0, 2367, 0, 65, 206, 4010, 0, 0, 0, 0, 4096, 3, 0, 4079, 4062, 2191, 18, 0, 0, 614, 0, 0, 0, 1, 4054, 0, 178, 4, 4087, 9, 0, 22, 4078, 0, 2701, 6, 0, 0, 318, 0, 0, 4095, 31, 15, 3, 0, 10, 1, 0, 4082, 0, 0, 81, 4056, 4094, 0, 3964, 0, 3852, 2, 4024, 29] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 90, 3781, 12, 42, 0, 4096, 1044, 92, 0, 0, 0, 0, 0, 0, 0, 50, 0, 20, 1601, 12, 110, 4065, 1829, 0, 0, 0, 0, 0, 0, 0, 4042, 4064, 0, 0, 5, 0, 0, 23, 314, 8, 0, 0, 0, 0, 0, 0, 0, 3680, 2546, 0, 273, 0, 1834, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4025, 8, 1047, 0, 4, 4, 1805, 158, 0, 0, 0, 0, 0, 0, 170, 1, 70, 4070, 1, 10, 0, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 12, 13, 0, 4065, 6, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 352, 0, 4082, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9157, 7687, 8456, 8338, 7081, 8208, 8158, 8451] avg=8192.0000 max=9157 max_rank=0 min=7081 min_rank=4 imbalance_ratio=1.117798 max_load_reduction=56379 max_load_reduction_pct=0.860275 moved_to_echo=31898 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 26, 584, 0, 0, 117, 3721, 1058, 92, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 2209, 0, 0, 4094, 1829, 0, 0, 0, 0, 0, 0, 0, 4090, 2312, 0, 0, 0, 48, 0, 0, 318, 8, 0, 0, 0, 0, 0, 0, 0, 4092, 2578, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 3032, 0, 0, 0, 1804, 160, 0, 0, 0, 0, 0, 0, 397, 0, 41, 4096, 0, 0, 0, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 351, 0, 4044, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5598, 8161, 6776, 10776, 9090, 8470, 8155, 8510] avg=8192.0000 max=10776 max_rank=3 min=5598 min_rank=0 imbalance_ratio=1.315430 max_load_reduction=54760 max_load_reduction_pct=0.835571 moved_to_echo=34225 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[35, 0, 0, 4087, 0, 0, 128, 4063, 977, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 4096, 1836, 0, 0, 0, 0, 0, 0, 0, 4089, 4033, 0, 0, 0, 54, 8, 0, 293, 8, 0, 0, 0, 0, 0, 0, 20, 3637, 2383, 8, 28, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1365, 2, 0, 0, 1811, 160, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4076, 0, 3, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4071, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 13, 4096, 0, 3138, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9382, 6255, 8485, 10175, 7428, 8023, 8188, 7600] avg=8192.0000 max=10175 max_rank=3 min=6255 min_rank=1 imbalance_ratio=1.242065 max_load_reduction=55361 max_load_reduction_pct=0.844742 moved_to_echo=31239 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 7, 4092, 0, 0, 20, 4029, 699, 92, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1230, 1, 58, 4096, 1837, 0, 0, 0, 0, 0, 0, 0, 4093, 3983, 0, 2, 0, 191, 0, 0, 210, 8, 0, 0, 0, 0, 0, 0, 0, 243, 1705, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 4096, 0, 0, 0, 1812, 160, 0, 0, 0, 0, 0, 0, 5, 0, 5, 4089, 1, 0, 0, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4088, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 4005, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8940, 7228, 8487, 6043, 10157, 8036, 8192, 8453] avg=8192.0000 max=10157 max_rank=4 min=6043 min_rank=3 imbalance_ratio=1.239868 max_load_reduction=55379 max_load_reduction_pct=0.845016 moved_to_echo=34838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3793, 0, 0, 834, 4093, 1083, 92, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 3345, 0, 0, 4093, 1837, 0, 0, 0, 0, 0, 0, 0, 4092, 4095, 2, 1, 7, 0, 12, 4, 325, 8, 0, 0, 0, 0, 0, 0, 0, 450, 2642, 1, 0, 0, 3011, 4, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 4094, 0, 650, 1, 4, 5, 1813, 160, 0, 0, 0, 0, 0, 0, 39, 0, 0, 4085, 0, 0, 2, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 4080, 3, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 354, 0, 4088, 7, 4095, 15, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9895, 9357, 8546, 6108, 6824, 8058, 8180, 8568] avg=8192.0000 max=9895 max_rank=0 min=6108 min_rank=3 imbalance_ratio=1.207886 max_load_reduction=55641 max_load_reduction_pct=0.849014 moved_to_echo=31630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 1, 59, 4081, 1096, 92, 0, 0, 0, 0, 0, 0, 1, 9, 0, 0, 955, 0, 1, 4096, 1837, 0, 0, 0, 0, 0, 0, 0, 4096, 3836, 0, 0, 0, 0, 0, 0, 329, 8, 0, 0, 0, 0, 0, 0, 0, 4089, 2671, 0, 0, 0, 3565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3917, 0, 0, 0, 1813, 161, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 47, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 4065, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5335, 6899, 8269, 10325, 9987, 8040, 8209, 8472] avg=8192.0000 max=10325 max_rank=3 min=5335 min_rank=0 imbalance_ratio=1.260376 max_load_reduction=55211 max_load_reduction_pct=0.842453 moved_to_echo=34708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4092, 0, 8, 0, 4084, 1091, 92, 0, 0, 0, 0, 0, 0, 3, 41, 6, 0, 4003, 11, 2, 4085, 1837, 0, 0, 0, 0, 0, 0, 0, 4089, 4059, 0, 19, 0, 788, 0, 0, 328, 8, 0, 0, 0, 0, 0, 0, 0, 64, 2662, 0, 0, 35, 4083, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4066, 0, 201, 0, 0, 0, 1813, 159, 0, 0, 0, 0, 0, 0, 91, 0, 207, 4095, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4088, 0, 49, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 354, 35, 3258, 0, 3591, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9370, 9988, 9291, 6855, 6240, 8306, 8236, 7250] avg=8192.0000 max=9988 max_rank=1 min=6240 min_rank=4 imbalance_ratio=1.219238 max_load_reduction=55548 max_load_reduction_pct=0.847595 moved_to_echo=30032 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 0, 2367, 0, 65, 206, 4010, 1088, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4079, 1783, 0, 0, 0, 0, 0, 0, 0, 4062, 2191, 18, 0, 0, 614, 0, 0, 329, 12, 0, 0, 0, 0, 0, 0, 0, 1, 2637, 0, 178, 4, 4087, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4078, 0, 2701, 6, 0, 0, 1761, 161, 0, 0, 0, 0, 0, 0, 318, 0, 0, 4095, 31, 15, 3, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 4070, 0, 0, 81, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 3852, 2, 4024, 29, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7859, 9961, 7226, 6916, 8729, 8395, 8218, 8232] avg=8192.0000 max=9961 max_rank=1 min=6916 min_rank=3 imbalance_ratio=1.215942 max_load_reduction=55575 max_load_reduction_pct=0.848007 moved_to_echo=33574 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 2816, 128, 31616, 128, 31232, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 256, 128, 128, 17792, 128, 256, 32768, 14720, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 16256, 19840, 128, 512, 128, 28928, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 22912, 128, 128, 1408, 32256, 8192, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32640, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1152, 128, 384, 32768, 128, 128, 128, 0, 31488, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32640, 128, 17024, 128, 128, 128, 14464, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[32768, 28672, 128, 128, 128, 1792, 128, 128, 2560, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 19, 30, 24, 4060, 4073, 0, 4095, 4069, 0, 0, 6, 4028, 0, 1, 0, 0, 4072, 0, 5, 0, 0, 2, 0, 6, 63, 0, 967, 14, 0, 6, 0, 0, 328, 13, 0, 0, 6, 0, 4069, 0, 0, 4096, 4096, 0, 0, 3842, 68, 4083, 12, 0, 3017, 4090, 4080, 4083, 0, 0, 2, 16, 0, 4090, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[11, 2, 0, 0, 3961, 4096, 0, 4096, 4082, 0, 0, 0, 4074, 0, 0, 0, 0, 4096, 0, 1165, 0, 8, 0, 0, 0, 629, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 1, 0, 1, 4096, 0, 0, 4036, 4096, 0, 0, 2371, 4, 4096, 0, 0, 4048, 4096, 4087, 4043, 0, 5, 0, 0, 0, 4085, 0, 81, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3, 0, 0, 12, 4076, 4096, 0, 4096, 4096, 0, 0, 0, 4070, 0, 7, 0, 0, 4096, 0, 2, 0, 4, 0, 0, 0, 557, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 3923, 0, 243, 4094, 4096, 0, 0, 3454, 1, 4096, 0, 0, 4084, 4096, 4096, 4095, 0, 0, 7, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4084, 4096, 0, 4095, 4093, 0, 0, 0, 4095, 0, 0, 1, 0, 4096, 0, 0, 12, 1, 0, 0, 3, 5, 0, 5, 0, 10, 0, 2, 0, 26, 0, 0, 0, 0, 0, 4091, 3, 0, 4096, 4094, 0, 45, 4082, 13, 4093, 0, 0, 4005, 4085, 4096, 4086, 0, 0, 2, 27, 0, 4093, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[33, 34, 0, 0, 3998, 4080, 0, 4072, 4096, 15, 38, 0, 4040, 0, 0, 68, 40, 3895, 0, 0, 1, 50, 0, 0, 0, 2476, 0, 296, 4, 0, 26, 0, 2, 0, 0, 0, 0, 0, 0, 4081, 25, 0, 4059, 4092, 0, 768, 604, 6, 4063, 2, 22, 3608, 4068, 4084, 4096, 19, 0, 589, 0, 4, 4067, 5, 7, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1, 5, 0, 0, 4080, 4095, 1, 4096, 4093, 0, 0, 2, 4093, 0, 12, 210, 0, 4062, 0, 0, 0, 21, 0, 0, 2, 414, 0, 273, 3, 0, 19, 1, 1, 0, 0, 7, 0, 4, 0, 4096, 9, 0, 4083, 4094, 0, 0, 4060, 0, 4091, 0, 0, 3257, 4074, 4088, 4035, 0, 58, 0, 0, 0, 4092, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[49, 0, 6, 188, 4094, 4088, 0, 4087, 4089, 2, 4, 3, 4082, 0, 19, 0, 7, 4094, 0, 0, 1, 0, 8, 0, 0, 2324, 0, 14, 1, 0, 0, 0, 0, 260, 1, 2, 1, 3, 0, 3719, 0, 0, 4050, 4091, 0, 16, 1760, 42, 4093, 0, 1, 4096, 4091, 4094, 3962, 0, 0, 0, 9, 0, 4065, 0, 0, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[71, 0, 0, 0, 4094, 4096, 0, 4085, 4085, 0, 20, 0, 4007, 37, 603, 0, 16, 4029, 0, 0, 7, 0, 0, 0, 0, 120, 0, 778, 0, 3, 0, 0, 91, 10, 0, 0, 0, 0, 17, 4079, 0, 0, 3984, 4096, 11, 13, 4094, 0, 4084, 12, 0, 2516, 4096, 4051, 4096, 0, 0, 0, 50, 120, 4064, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 2, 0, 0, 3961, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1165, 0, 8, 0, 0, 3157, 709, 87, 15, 0, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, 0, 4096, 2981, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 1, 0, 1, 4096, 2943, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 939, 0, 0, 2371, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 544, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4085, 0, 81, 0, 2834, 1153, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8055, 8069, 9237, 7706, 8326, 7350, 8635, 8158] avg=8192.0000 max=9237 max_rank=2 min=7350 min_rank=5 imbalance_ratio=1.127563 max_load_reduction=56299 max_load_reduction_pct=0.859055 moved_to_echo=32469 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[33, 34, 0, 0, 3998, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 15, 38, 0, 4040, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 40, 3895, 0, 0, 1, 50, 0, 0, 3154, 708, 88, 15, 0, 0, 0, 0, 0, 2476, 0, 296, 4, 0, 26, 0, 4072, 2956, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4081, 2922, 1107, 0, 0, 0, 0, 0, 0, 25, 0, 4059, 938, 0, 768, 604, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 22, 3608, 0, 544, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 4, 4067, 5, 7, 3, 2832, 1146, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8169, 7951, 9830, 8112, 6400, 8291, 8653] avg=8192.0000 max=9830 max_rank=3 min=6400 min_rank=5 imbalance_ratio=1.199951 max_load_reduction=55706 max_load_reduction_pct=0.850006 moved_to_echo=31456 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 5, 0, 0, 4080, 4080, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 2, 4093, 0, 12, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 21, 0, 0, 3156, 709, 88, 15, 0, 0, 0, 0, 2, 414, 0, 273, 3, 0, 19, 1, 4096, 2976, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 4, 0, 4096, 2927, 1115, 0, 0, 0, 0, 0, 0, 9, 0, 4083, 938, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3257, 0, 544, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 4092, 4, 0, 0, 2835, 1147, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8322, 8051, 7784, 8150, 9090, 7836, 8136] avg=8192.0000 max=9090 max_rank=5 min=7784 min_rank=3 imbalance_ratio=1.109619 max_load_reduction=56446 max_load_reduction_pct=0.861298 moved_to_echo=33212 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 30, 24, 4060, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 0, 6, 4028, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 5, 0, 0, 2, 0, 3157, 707, 87, 15, 0, 0, 0, 0, 6, 63, 0, 967, 14, 0, 6, 0, 4095, 2970, 0, 0, 0, 0, 0, 0, 0, 328, 13, 0, 0, 6, 0, 4069, 2939, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 939, 0, 0, 3842, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3017, 0, 543, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 16, 0, 4090, 0, 5, 0, 2830, 1151, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8017, 8045, 8121, 8468, 8945, 7655, 8094] avg=8192.0000 max=8945 max_rank=5 min=7655 min_rank=6 imbalance_ratio=1.091919 max_load_reduction=56591 max_load_reduction_pct=0.863510 moved_to_echo=33162 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4084, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 1, 0, 0, 3156, 710, 88, 15, 0, 0, 0, 0, 3, 5, 0, 5, 0, 10, 0, 2, 4095, 2978, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 4091, 2935, 1115, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 938, 0, 45, 4082, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 545, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 27, 0, 4093, 0, 0, 0, 2841, 1150, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8101, 8078, 7098, 8167, 9177, 8636, 8113] avg=8192.0000 max=9177 max_rank=5 min=7098 min_rank=3 imbalance_ratio=1.120239 max_load_reduction=56359 max_load_reduction_pct=0.859970 moved_to_echo=34093 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 12, 4076, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 4070, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4, 0, 0, 3157, 710, 88, 15, 0, 0, 0, 0, 0, 557, 0, 0, 0, 0, 0, 0, 4096, 2981, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 3923, 2943, 1115, 0, 0, 0, 0, 0, 0, 0, 243, 4094, 939, 0, 0, 3454, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 545, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 2841, 1153, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8085, 8072, 7634, 8021, 8731, 8724, 8097] avg=8192.0000 max=8731 max_rank=5 min=7634 min_rank=3 imbalance_ratio=1.065796 max_load_reduction=56805 max_load_reduction_pct=0.866776 moved_to_echo=33573 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[49, 0, 6, 188, 4094, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 2, 4, 3, 4082, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4094, 0, 0, 1, 0, 8, 0, 3153, 710, 88, 15, 0, 0, 0, 0, 0, 2324, 0, 14, 1, 0, 0, 0, 4087, 2979, 0, 0, 0, 0, 0, 0, 0, 260, 1, 2, 1, 3, 0, 3719, 2940, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 938, 0, 16, 1760, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 545, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4065, 0, 0, 20, 2839, 1151, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8410, 8111, 8076, 9405, 8040, 6806, 8604, 8084] avg=8192.0000 max=9405 max_rank=3 min=6806 min_rank=5 imbalance_ratio=1.148071 max_load_reduction=56131 max_load_reduction_pct=0.856491 moved_to_echo=31534 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[71, 0, 0, 0, 4094, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 20, 0, 4007, 37, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4029, 0, 0, 7, 0, 0, 0, 3161, 707, 90, 16, 0, 0, 0, 0, 0, 120, 0, 778, 0, 3, 0, 0, 4085, 2972, 0, 0, 0, 0, 0, 0, 91, 10, 0, 0, 0, 0, 17, 4079, 2943, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 935, 11, 13, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2516, 0, 531, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 120, 4064, 1, 0, 0, 2813, 1153, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8662, 8026, 7958, 8252, 9037, 7155, 8201] avg=8192.0000 max=9037 max_rank=5 min=7155 min_rank=6 imbalance_ratio=1.103149 max_load_reduction=56499 max_load_reduction_pct=0.862106 moved_to_echo=32645 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 256, 32512, 7552, 128, 896, 24320, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 640, 128, 128, 32768, 128, 128, 128, 22784, 9216, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 128, 128, 256, 32512, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32512, 0, 1280, 128, 128, 128, 0, 25344, 5760, 768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32000, 128, 128, 128, 32512, 128, 768, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 28672, 0, 4352, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 896, 128, 128, 128, 128, 128, 32256, 23552, 8960, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 6656, 0, 2432, 128, 128, 128, 128, 32768, 23808, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[75, 4040, 4066, 7, 23, 0, 0, 25, 30, 0, 0, 242, 4096, 4074, 4089, 0, 5, 34, 2241, 4091, 0, 0, 0, 350, 7, 0, 4, 0, 3, 28, 38, 4095, 8, 4064, 0, 4082, 4, 0, 0, 0, 4059, 0, 0, 0, 0, 1205, 0, 0, 4096, 0, 0, 4087, 4079, 0, 0, 4084, 0, 0, 10, 0, 13, 2, 4065, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4094, 68, 4096, 0, 0, 0, 3, 0, 12, 0, 0, 282, 3887, 4096, 4086, 0, 300, 0, 0, 4096, 0, 147, 126, 3488, 25, 0, 0, 0, 0, 26, 0, 4089, 0, 4003, 0, 3638, 26, 1, 0, 0, 3949, 0, 0, 20, 0, 290, 2, 8, 2570, 0, 0, 3636, 3975, 0, 0, 4095, 2, 651, 0, 0, 0, 1449, 4096, 204] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[3633, 3415, 4044, 19, 1, 0, 14, 25, 0, 0, 0, 893, 4066, 4058, 4055, 0, 7, 0, 358, 4082, 0, 238, 0, 8, 0, 0, 15, 1, 5, 0, 0, 3865, 25, 4025, 48, 1384, 0, 34, 0, 0, 3790, 0, 0, 1, 26, 3292, 27, 296, 3673, 0, 0, 3649, 4080, 9, 0, 4068, 15, 0, 21, 4, 0, 84, 4094, 89] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[3723, 0, 4096, 0, 0, 0, 0, 0, 525, 0, 0, 61, 4024, 4096, 4096, 0, 3565, 0, 0, 4096, 0, 0, 1, 431, 0, 0, 0, 0, 0, 7, 0, 3943, 0, 4096, 0, 4096, 120, 0, 0, 0, 4095, 0, 0, 0, 0, 62, 0, 4, 4007, 0, 0, 4096, 4065, 0, 0, 4096, 0, 39, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[2099, 472, 3999, 0, 0, 29, 1, 0, 2, 0, 0, 328, 4088, 4094, 4089, 30, 3721, 2, 926, 4084, 0, 0, 0, 811, 21, 0, 13, 14, 0, 67, 0, 4082, 0, 4058, 0, 3389, 0, 1, 0, 12, 3784, 2, 0, 0, 6, 674, 0, 172, 4094, 82, 0, 4035, 4096, 0, 0, 4087, 0, 0, 0, 4, 0, 0, 4068, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4080, 156, 4042, 0, 5, 0, 0, 0, 0, 3, 0, 14, 4092, 4093, 4092, 52, 1611, 0, 23, 4096, 10, 590, 26, 2703, 6, 0, 4, 0, 0, 0, 1, 4091, 0, 4080, 0, 3063, 2, 0, 1, 0, 4049, 0, 15, 5, 0, 461, 0, 0, 3779, 17, 2, 4096, 4096, 0, 0, 3980, 6, 0, 0, 0, 0, 0, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[3854, 11, 4084, 0, 0, 0, 0, 0, 0, 10, 0, 2783, 4096, 4092, 4096, 0, 0, 0, 1141, 4087, 7, 6, 7, 150, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 4082, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 100, 201, 0, 4031, 3, 0, 4096, 4094, 1, 0, 4096, 0, 5, 0, 0, 0, 0, 4095, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[3946, 0, 4096, 1, 0, 0, 6, 11, 110, 0, 0, 0, 4096, 4095, 4093, 0, 1661, 3, 18, 4094, 0, 0, 0, 1007, 10, 0, 0, 0, 0, 1194, 0, 3774, 1, 4094, 0, 4020, 1, 0, 2, 6, 4081, 0, 0, 0, 1, 65, 0, 0, 4096, 0, 0, 3971, 4092, 2, 1, 4095, 0, 286, 2, 0, 1, 420, 4084, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3633, 3415, 3942, 19, 1, 0, 14, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 893, 4066, 0, 3438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 358, 4082, 0, 238, 0, 8, 783, 103, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 5, 0, 0, 3865, 3973, 0, 0, 0, 0, 0, 0, 0, 25, 4025, 48, 1384, 0, 34, 0, 0, 514, 102, 0, 0, 0, 0, 0, 0, 3790, 0, 0, 1, 26, 3292, 27, 296, 2869, 472, 0, 0, 0, 0, 0, 0, 3673, 0, 0, 3649, 428, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 21, 4, 0, 84, 4094, 89, 3596, 85, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11049, 8397, 5579, 7859, 6132, 10773, 7759, 7988] avg=8192.0000 max=11049 max_rank=0 min=5579 min_rank=2 imbalance_ratio=1.348755 max_load_reduction=54487 max_load_reduction_pct=0.831406 moved_to_echo=32652 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[75, 4040, 3964, 7, 23, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 242, 4096, 0, 3467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 34, 2241, 4091, 0, 0, 0, 350, 783, 103, 0, 0, 0, 0, 0, 0, 7, 0, 4, 0, 3, 28, 38, 4095, 3988, 0, 0, 0, 0, 0, 0, 0, 8, 4064, 0, 4082, 4, 0, 0, 0, 519, 102, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 1205, 0, 0, 2868, 474, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 13, 2, 4065, 15, 3610, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 7835, 7607, 8163, 8779, 8606, 8611, 7801] avg=8192.0000 max=8779 max_rank=4 min=7607 min_rank=2 imbalance_ratio=1.071655 max_load_reduction=56757 max_load_reduction_pct=0.866043 moved_to_echo=33797 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4094, 68, 3993, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 282, 3887, 0, 3465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 4096, 0, 147, 126, 3488, 763, 103, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 26, 0, 4089, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 3638, 26, 1, 0, 0, 518, 103, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 20, 0, 290, 2, 8, 2795, 476, 0, 0, 0, 0, 0, 0, 2570, 0, 0, 3636, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 651, 0, 0, 0, 1449, 4096, 204, 3619, 85, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 7646, 9023, 8151, 8289, 7540, 6623, 10106] avg=8192.0000 max=10106 max_rank=7 min=6623 min_rank=6 imbalance_ratio=1.233643 max_load_reduction=55430 max_load_reduction_pct=0.845795 moved_to_echo=32558 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3723, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 61, 4024, 0, 3472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3565, 0, 0, 4096, 0, 0, 1, 431, 780, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3943, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 120, 0, 0, 0, 520, 103, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 62, 0, 4, 2859, 476, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 4096, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 4096, 0, 3620, 85, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7716, 8082, 8977, 7961, 8935, 7496, 8529, 7840] avg=8192.0000 max=8977 max_rank=2 min=7496 min_rank=5 imbalance_ratio=1.095825 max_load_reduction=56559 max_load_reduction_pct=0.863022 moved_to_echo=32800 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4080, 156, 3940, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 14, 4092, 0, 3469, 52, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 0, 23, 4096, 10, 590, 26, 2703, 786, 104, 0, 0, 0, 0, 0, 0, 6, 0, 4, 0, 0, 0, 1, 4091, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 3063, 2, 0, 1, 0, 519, 102, 0, 0, 0, 0, 0, 0, 4049, 0, 15, 5, 0, 461, 0, 0, 2880, 462, 0, 0, 0, 0, 0, 0, 3779, 17, 2, 4096, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4094, 0, 3518, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 7630, 9949, 8109, 7767, 7872, 8324, 7704] avg=8192.0000 max=9949 max_rank=2 min=7630 min_rank=1 imbalance_ratio=1.214478 max_load_reduction=55587 max_load_reduction_pct=0.848190 moved_to_echo=31667 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3854, 11, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2783, 4096, 0, 3472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1141, 4087, 7, 6, 7, 150, 786, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 0, 0, 0, 0, 520, 103, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 100, 201, 0, 2879, 475, 0, 0, 0, 0, 0, 0, 4031, 3, 0, 4096, 429, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4095, 22, 3621, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7846, 10361, 6288, 8100, 8801, 7751, 8560, 7829] avg=8192.0000 max=10361 max_rank=1 min=6288 min_rank=2 imbalance_ratio=1.264771 max_load_reduction=55175 max_load_reduction_pct=0.841904 moved_to_echo=32941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3946, 0, 3993, 1, 0, 0, 6, 11, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 4096, 0, 3470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1661, 3, 18, 4094, 0, 0, 0, 1007, 785, 104, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1194, 0, 3774, 4009, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 4020, 1, 0, 2, 6, 519, 103, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 1, 65, 0, 0, 2878, 475, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3971, 429, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 2, 0, 1, 420, 4084, 0, 3620, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7957, 7676, 7672, 8987, 8746, 7500, 8499, 8499] avg=8192.0000 max=8987 max_rank=3 min=7500 min_rank=5 imbalance_ratio=1.097046 max_load_reduction=56549 max_load_reduction_pct=0.862869 moved_to_echo=33244 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2099, 472, 3894, 0, 0, 29, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 328, 4088, 0, 3461, 30, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 2, 926, 4084, 0, 0, 0, 811, 791, 106, 0, 0, 0, 0, 0, 0, 21, 0, 13, 14, 0, 67, 0, 4082, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 3389, 0, 1, 0, 12, 522, 105, 0, 0, 0, 0, 0, 0, 3784, 2, 0, 0, 6, 674, 0, 172, 2885, 475, 0, 0, 0, 0, 0, 0, 4094, 82, 0, 4035, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4068, 0, 3612, 85, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6495, 7909, 10441, 8206, 8087, 7998, 8631, 7769] avg=8192.0000 max=10441 max_rank=2 min=6495 min_rank=0 imbalance_ratio=1.274536 max_load_reduction=55095 max_load_reduction_pct=0.840683 moved_to_echo=32485 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32000, 128, 128, 128, 128, 6272, 256, 512, 23040, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[25600, 8192, 31744, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[768, 128, 0, 4608, 32512, 0, 27776, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 1024, 128, 128, 128, 2048, 32768, 384, 28928, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[30464, 128, 128, 31744, 3456, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32512, 128, 27776, 256, 128, 128, 128, 4224, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[10880, 128, 4736, 32768, 128, 1024, 256, 8960, 6272, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 1408, 128, 32128, 32128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 3860, 4095, 0, 0, 4, 18, 4096, 0, 0, 0, 0, 7, 0, 2, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 4091, 4096, 238, 4096, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4072, 4096, 0, 0, 4096, 0, 0, 0, 0, 4008, 0, 88, 4090, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4, 4, 0, 0, 4092, 4076, 0, 0, 9, 550, 4092, 2, 1717, 17, 17, 0, 0, 0, 0, 0, 2, 4070, 6, 2, 4096, 0, 0, 3, 1, 20, 0, 4086, 0, 2, 152, 0, 0, 3771, 4091, 111, 4042, 30, 6, 0, 0, 0, 3688, 4090, 0, 0, 4094, 4090, 6, 0, 2886, 1, 0, 5, 48, 3441, 0, 79, 4037, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[22, 4, 350, 0, 340, 3839, 0, 0, 1, 59, 4096, 58, 4076, 0, 33, 0, 0, 2, 0, 0, 1, 3990, 10, 0, 3991, 0, 0, 3, 0, 49, 11, 4066, 0, 0, 0, 73, 0, 4096, 4016, 104, 4036, 3, 0, 0, 14, 0, 4042, 4084, 66, 31, 4070, 4074, 0, 27, 3605, 7, 2, 0, 0, 4082, 0, 0, 4083, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3, 13, 14, 0, 489, 3100, 5, 0, 0, 0, 4084, 0, 3903, 0, 15, 0, 10, 0, 0, 0, 0, 4083, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 4096, 0, 5, 909, 0, 7, 4085, 4039, 0, 4044, 0, 24, 0, 10, 0, 3931, 4093, 0, 11, 4080, 4091, 0, 0, 4079, 7, 0, 14, 8, 4096, 0, 0, 4080, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 2291, 3218, 0, 0, 0, 547, 4068, 0, 2778, 0, 0, 9, 0, 16, 2, 0, 0, 4087, 0, 10, 3839, 0, 0, 0, 0, 0, 0, 4064, 2, 0, 710, 0, 0, 4096, 4096, 73, 4096, 0, 0, 0, 0, 0, 2815, 4096, 0, 0, 4091, 4093, 0, 0, 4096, 0, 0, 0, 0, 4075, 0, 170, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 4096, 43, 0, 0, 0, 5, 0, 4096, 0, 0, 71, 0, 0, 3950, 4096, 24, 4096, 0, 0, 0, 0, 0, 2855, 4096, 0, 0, 4096, 4096, 0, 0, 1790, 0, 0, 0, 0, 3583, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 96, 0, 4056, 3579, 0, 0, 0, 2, 4096, 0, 1152, 2, 0, 0, 0, 0, 0, 1, 0, 4000, 0, 20, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 5, 0, 4094, 4085, 0, 4096, 0, 0, 0, 0, 0, 4076, 4095, 4, 0, 4096, 4094, 0, 0, 4096, 1, 13, 0, 0, 3484, 0, 13, 4086, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 4089, 3203, 0, 5, 9, 0, 4087, 44, 126, 5, 0, 0, 13, 0, 0, 0, 17, 4094, 0, 5, 4079, 0, 1, 0, 0, 707, 0, 4084, 0, 0, 0, 3, 0, 4095, 4092, 0, 4047, 0, 2, 7, 33, 0, 4011, 4094, 5, 0, 4081, 4030, 4, 0, 4085, 2, 0, 0, 0, 4095, 3, 174, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 3860, 4095, 0, 0, 1447, 108, 0, 0, 0, 0, 0, 0, 4, 18, 4096, 0, 0, 0, 0, 7, 1690, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 3676, 439, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4091, 3866, 238, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 88, 4090, 0, 122, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9515, 5815, 8211, 8145, 8197, 8610, 8688, 8355] avg=8192.0000 max=9515 max_rank=0 min=5815 min_rank=1 imbalance_ratio=1.161499 max_load_reduction=56021 max_load_reduction_pct=0.854813 moved_to_echo=33850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[22, 4, 350, 0, 340, 3839, 0, 0, 1447, 106, 0, 0, 0, 0, 0, 0, 1, 59, 4096, 58, 4076, 0, 33, 0, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 3990, 10, 0, 3667, 439, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 3, 0, 49, 11, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 4096, 3790, 104, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 3, 0, 0, 14, 0, 4042, 417, 0, 0, 0, 0, 0, 0, 0, 0, 66, 31, 495, 4074, 0, 27, 3605, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4082, 0, 0, 4083, 20, 120, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6108, 10012, 8109, 8074, 8063, 8512, 8305, 8353] avg=8192.0000 max=10012 max_rank=1 min=6108 min_rank=0 imbalance_ratio=1.222168 max_load_reduction=55524 max_load_reduction_pct=0.847229 moved_to_echo=33233 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 2291, 3218, 0, 0, 1454, 108, 0, 0, 0, 0, 0, 0, 0, 547, 4068, 0, 2778, 0, 0, 9, 1698, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2, 0, 0, 4087, 0, 10, 3678, 441, 0, 0, 0, 0, 0, 0, 3839, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 710, 0, 0, 4096, 3866, 73, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2815, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 170, 4096, 0, 122, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7073, 9100, 8234, 7857, 8747, 7329, 8687, 8509] avg=8192.0000 max=9100 max_rank=1 min=7073 min_rank=0 imbalance_ratio=1.110840 max_load_reduction=56436 max_load_reduction_pct=0.861145 moved_to_echo=33272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 13, 14, 0, 489, 3100, 5, 0, 1450, 107, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 3903, 0, 15, 0, 1693, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4083, 0, 0, 3675, 440, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 909, 0, 7, 4085, 3811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 24, 0, 10, 0, 3931, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 497, 4091, 0, 0, 4079, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 8, 4096, 0, 0, 4080, 16, 121, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5181, 9695, 8208, 8141, 8817, 8427, 8685, 8382] avg=8192.0000 max=9695 max_rank=1 min=5181 min_rank=0 imbalance_ratio=1.183472 max_load_reduction=55841 max_load_reduction_pct=0.852066 moved_to_echo=34311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 96, 0, 4056, 3579, 0, 0, 1456, 108, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 1152, 2, 0, 0, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4000, 0, 20, 3677, 442, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 4094, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4076, 418, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 498, 4094, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 3484, 0, 13, 4086, 0, 122, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9295, 6952, 8140, 8146, 7955, 8590, 8693, 7765] avg=8192.0000 max=9295 max_rank=0 min=6952 min_rank=1 imbalance_ratio=1.134644 max_load_reduction=56241 max_load_reduction_pct=0.858170 moved_to_echo=33003 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 4096, 0, 0, 1456, 108, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 3678, 442, 0, 0, 0, 0, 0, 0, 4096, 43, 0, 0, 0, 5, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 3950, 3866, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2855, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 4096, 0, 0, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3583, 0, 0, 4096, 0, 122, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9751, 9892, 8188, 8193, 7911, 7369, 6384, 7848] avg=8192.0000 max=9892 max_rank=1 min=6384 min_rank=6 imbalance_ratio=1.207520 max_load_reduction=55644 max_load_reduction_pct=0.849060 moved_to_echo=29512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4, 4, 0, 0, 4092, 4076, 0, 0, 1455, 108, 0, 0, 0, 0, 0, 0, 9, 550, 4092, 2, 1717, 17, 17, 0, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4070, 6, 2, 3672, 442, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 1, 20, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 152, 0, 0, 3771, 3861, 111, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 30, 6, 0, 0, 0, 3688, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 4090, 6, 0, 2886, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 48, 3441, 0, 79, 4037, 0, 122, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9739, 8103, 8194, 8159, 7897, 8184, 7481, 7779] avg=8192.0000 max=9739 max_rank=0 min=7481 min_rank=6 imbalance_ratio=1.188843 max_load_reduction=55797 max_load_reduction_pct=0.851395 moved_to_echo=31341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 4089, 3203, 0, 5, 1454, 114, 0, 0, 0, 0, 0, 0, 9, 0, 4087, 44, 126, 5, 0, 0, 1696, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 17, 4094, 0, 5, 3679, 444, 0, 0, 0, 0, 0, 0, 4079, 0, 1, 0, 0, 707, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 2, 7, 33, 0, 4011, 415, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 487, 4030, 4, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 174, 4096, 0, 127, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8874, 5967, 8252, 8821, 7949, 8515, 8613, 8545] avg=8192.0000 max=8874 max_rank=0 min=5967 min_rank=1 imbalance_ratio=1.083252 max_load_reduction=56662 max_load_reduction_pct=0.864594 moved_to_echo=33622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 128, 128, 0, 29568, 3456, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 3968, 32768, 128, 128, 28800, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 1920, 128, 128, 32384, 30848, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32384, 128, 128, 128, 128, 896, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 1280, 32768, 128, 17920, 128, 128, 128, 13568, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 30976, 128, 640, 32768, 128, 1024, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 512, 128, 23424, 29312, 128, 128, 11648, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 32512, 128, 128, 29440, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4082, 0, 2790, 0, 4080, 0, 3603, 0, 0, 0, 15, 3986, 0, 34, 0, 0, 0, 2, 4060, 16, 33, 4081, 0, 4043, 4082, 16, 1, 14, 376, 4075, 12, 4080, 7, 803, 4075, 8, 47, 1, 4069, 1, 4050, 21, 6, 338, 11, 0, 0, 12, 1, 0, 0, 480, 4043, 0, 0, 0, 0, 4068, 0, 14, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4094, 0, 2554, 0, 4061, 8, 874, 6, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 4088, 0, 0, 4096, 0, 3886, 4090, 0, 5, 0, 0, 4096, 0, 4096, 216, 712, 4096, 0, 0, 0, 4093, 0, 4096, 0, 0, 3698, 0, 2, 0, 0, 0, 0, 0, 378, 4091, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4095, 10, 988, 1, 4084, 0, 4068, 313, 3, 0, 0, 3300, 0, 0, 0, 0, 0, 0, 4096, 44, 0, 4086, 18, 4040, 4008, 0, 14, 0, 377, 4096, 0, 4096, 0, 88, 4096, 0, 0, 0, 4091, 0, 4096, 0, 0, 3165, 0, 0, 0, 0, 0, 0, 0, 136, 4076, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4077, 0, 1172, 1, 4080, 10, 4074, 11, 0, 0, 0, 3971, 0, 0, 28, 4, 0, 0, 4094, 315, 2, 4095, 10, 3740, 4049, 0, 1, 2, 0, 4096, 0, 4093, 424, 719, 4089, 0, 0, 170, 4091, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 1775, 4096, 0, 88, 0, 7, 4004, 2, 0, 0, 0, 10, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4082, 0, 803, 0, 4072, 0, 16, 724, 0, 0, 5, 3707, 21, 0, 0, 10, 0, 12, 4087, 0, 0, 4047, 13, 4080, 4056, 1, 0, 0, 5, 4075, 0, 4078, 163, 3976, 4006, 0, 36, 0, 4032, 0, 3369, 12, 20, 3608, 2, 0, 4, 25, 0, 10, 0, 109, 4063, 0, 74, 7, 30, 4012, 0, 1, 0, 18, 64, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4088, 3, 3, 0, 4095, 27, 4090, 2, 1, 2, 9, 4090, 0, 0, 0, 0, 0, 2, 3763, 15, 2, 4089, 1, 4055, 4089, 0, 0, 0, 659, 4083, 0, 4094, 367, 3687, 3402, 16, 0, 1356, 4079, 48, 3104, 123, 4, 0, 1, 0, 0, 0, 0, 3, 49, 0, 3920, 19, 0, 0, 0, 4085, 0, 0, 1, 10, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 355, 113, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 4096, 0, 3725, 4096, 0, 4, 0, 130, 4096, 0, 4096, 0, 3775, 4096, 0, 63, 0, 3968, 0, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4062, 0, 4085, 0, 3977, 32, 366, 16, 0, 0, 3, 4068, 13, 0, 38, 0, 5, 0, 3970, 0, 0, 4086, 0, 3653, 4084, 2, 0, 14, 0, 4093, 7, 4075, 293, 343, 4091, 0, 0, 13, 4096, 0, 4092, 0, 10, 2359, 20, 0, 0, 0, 14, 0, 0, 1182, 4096, 17, 119, 0, 0, 4096, 0, 24, 22, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 988, 1, 3806, 0, 4068, 313, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3300, 0, 0, 0, 0, 4096, 167, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 44, 0, 196, 18, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 14, 0, 377, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 4096, 0, 0, 0, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3165, 0, 0, 0, 0, 2046, 278, 271, 171, 0, 0, 0, 0, 0, 0, 0, 136, 4076, 0, 0, 0, 3436, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 6, 0, 3928, 183, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9186, 7566, 8394, 8324, 6229, 10027, 7648, 8162] avg=8192.0000 max=10027 max_rank=5 min=6229 min_rank=4 imbalance_ratio=1.223999 max_load_reduction=55509 max_load_reduction_pct=0.847000 moved_to_echo=32066 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 803, 0, 3794, 0, 16, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3707, 21, 0, 0, 10, 4075, 165, 0, 0, 0, 0, 0, 0, 0, 12, 4087, 0, 0, 195, 13, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 1, 0, 0, 5, 0, 0, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 163, 3976, 4006, 0, 36, 0, 2016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3369, 12, 20, 3608, 2, 0, 4, 25, 2016, 278, 268, 170, 0, 0, 0, 0, 0, 10, 0, 109, 4063, 0, 74, 7, 3403, 0, 0, 0, 0, 0, 0, 0, 30, 4012, 0, 1, 0, 18, 64, 1, 3917, 181, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5337, 7983, 8387, 7970, 10197, 9772, 7666, 8224] avg=8192.0000 max=10197 max_rank=4 min=5337 min_rank=0 imbalance_ratio=1.244751 max_load_reduction=55339 max_load_reduction_pct=0.844406 moved_to_echo=35859 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1172, 1, 3802, 10, 4074, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 28, 4, 4096, 165, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 315, 2, 197, 10, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 1, 2, 0, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 424, 719, 4089, 0, 0, 170, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 2046, 278, 272, 170, 0, 0, 0, 0, 47, 0, 0, 1775, 4096, 0, 88, 0, 3443, 0, 0, 0, 0, 0, 0, 0, 7, 4004, 2, 0, 0, 0, 10, 0, 3912, 183, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9070, 8264, 8358, 7975, 7447, 6855, 9449, 8118] avg=8192.0000 max=9449 max_rank=6 min=6855 min_rank=5 imbalance_ratio=1.153442 max_load_reduction=56087 max_load_reduction_pct=0.855820 moved_to_echo=31869 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2554, 0, 3784, 8, 874, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 166, 0, 0, 0, 0, 0, 0, 2, 0, 4088, 0, 0, 196, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 5, 0, 0, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 216, 712, 4096, 0, 0, 0, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3698, 0, 2, 0, 0, 2047, 277, 272, 171, 0, 0, 0, 0, 0, 0, 0, 378, 4091, 0, 0, 0, 3444, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4, 3, 3928, 184, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7226, 8356, 8172, 8020, 7070, 10563, 7913, 8216] avg=8192.0000 max=10563 max_rank=5 min=7070 min_rank=4 imbalance_ratio=1.289429 max_load_reduction=54973 max_load_reduction_pct=0.838821 moved_to_echo=33762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2790, 0, 3802, 0, 3603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3986, 0, 34, 0, 0, 4075, 165, 0, 0, 0, 0, 0, 0, 0, 2, 4060, 16, 33, 196, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 16, 1, 14, 376, 0, 12, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 7, 803, 4075, 8, 47, 1, 2034, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 21, 6, 338, 11, 0, 0, 12, 2035, 278, 271, 170, 0, 0, 0, 0, 1, 0, 0, 480, 4043, 0, 0, 0, 3431, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 14, 0, 0, 0, 0, 3917, 183, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10195, 8275, 8350, 8411, 6976, 7192, 7955, 8182] avg=8192.0000 max=10195 max_rank=0 min=6976 min_rank=4 imbalance_ratio=1.244507 max_load_reduction=55341 max_load_reduction_pct=0.844437 moved_to_echo=30305 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 355, 113, 3817, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 167, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 196, 0, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 130, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 4096, 0, 63, 0, 1984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3174, 0, 0, 0, 0, 0, 0, 0, 1984, 279, 272, 171, 0, 0, 0, 0, 0, 0, 0, 1078, 4087, 0, 0, 0, 3444, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3929, 184, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8380, 8359, 8026, 8155, 9918, 5880, 8609, 8209] avg=8192.0000 max=9918 max_rank=4 min=5880 min_rank=5 imbalance_ratio=1.210693 max_load_reduction=55618 max_load_reduction_pct=0.848663 moved_to_echo=32616 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3, 0, 3816, 27, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 9, 4090, 0, 0, 0, 0, 4083, 167, 0, 0, 0, 0, 0, 0, 0, 2, 3763, 15, 2, 197, 1, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 659, 0, 0, 3924, 0, 0, 0, 0, 0, 0, 0, 0, 367, 3687, 3402, 16, 0, 1356, 2039, 48, 0, 0, 0, 0, 0, 0, 0, 0, 3104, 123, 4, 0, 1, 0, 0, 0, 2040, 279, 271, 170, 0, 0, 0, 0, 0, 3, 49, 0, 3920, 19, 0, 0, 3438, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 1, 10, 0, 0, 3921, 183, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7941, 8352, 8035, 8672, 10915, 5992, 7429, 8200] avg=8192.0000 max=10915 max_rank=4 min=5992 min_rank=5 imbalance_ratio=1.332397 max_load_reduction=54621 max_load_reduction_pct=0.833450 moved_to_echo=32536 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4085, 0, 3702, 32, 366, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4068, 13, 0, 38, 0, 4093, 166, 0, 0, 0, 0, 0, 0, 5, 0, 3970, 0, 0, 186, 0, 3653, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 2, 0, 14, 0, 0, 7, 3902, 0, 0, 0, 0, 0, 0, 0, 0, 293, 343, 4091, 0, 0, 13, 2044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 10, 2359, 20, 0, 0, 0, 2052, 275, 274, 173, 0, 0, 0, 0, 14, 0, 0, 1182, 4096, 17, 119, 0, 3439, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 22, 0, 0, 0, 3896, 187, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8381, 7814, 8009, 6784, 9255, 8867, 8225] avg=8192.0000 max=9255 max_rank=5 min=6784 min_rank=4 imbalance_ratio=1.129761 max_load_reduction=56281 max_load_reduction_pct=0.858780 moved_to_echo=33131 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 12800, 128, 30336, 128, 21248, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32512, 128, 128, 128, 128, 128, 128, 31360, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1536, 14208, 32000, 128, 256, 1664, 16256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 5248, 32512, 128, 384, 128, 27520, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 128, 1664, 0, 128, 31360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32256, 512, 128, 1664, 128, 31232, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 31360, 128, 128, 128, 128, 32768, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[30080, 256, 128, 13184, 128, 128, 128, 128, 16384, 2304, 2176, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4055, 0, 0, 0, 4086, 3844, 4040, 0, 4096, 0, 0, 55, 0, 0, 0, 0, 411, 0, 1258, 0, 55, 0, 3988, 0, 0, 0, 4093, 0, 23, 0, 4096, 0, 29, 0, 0, 0, 4096, 0, 0, 0, 2701, 0, 0, 4094, 0, 3608, 85, 0, 0, 0, 4096, 0, 277, 164, 0, 0, 0, 9, 4085, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 3841, 19, 0, 4066, 0, 0, 26, 4064, 4096, 4084, 39, 4081, 7, 238, 0, 50, 8, 0, 10, 196, 1, 3904, 6, 0, 18, 10, 0, 0, 25, 3946, 18, 0, 15, 4089, 0, 0, 0, 3, 0, 4087, 0, 12, 0, 3, 0, 0, 4073, 0, 4087, 0, 0, 7, 0, 4079, 45, 3947, 211, 3, 0, 0, 0, 4072, 4050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1, 4072, 0, 2, 4089, 0, 5, 0, 4096, 3955, 4089, 0, 4077, 0, 20, 31, 7, 0, 0, 2, 0, 442, 3065, 0, 1, 0, 2523, 0, 2, 0, 3814, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 3798, 0, 0, 0, 565, 2, 0, 2606, 170, 4084, 133, 5, 0, 13, 4094, 0, 3443, 23, 8, 0, 0, 11, 4094, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 2908, 0, 0, 4044, 26, 42, 0, 4056, 2851, 4046, 0, 4086, 0, 579, 13, 70, 0, 0, 29, 70, 752, 3311, 0, 439, 7, 2653, 20, 0, 3, 3907, 0, 18, 0, 4085, 0, 0, 0, 0, 0, 3950, 27, 11, 35, 3960, 0, 20, 2432, 225, 3970, 100, 11, 14, 0, 4089, 157, 356, 0, 1, 0, 0, 33, 4062, 4068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4, 4049, 0, 0, 4092, 8, 10, 5, 4037, 4096, 4084, 19, 4082, 0, 333, 0, 0, 0, 0, 0, 38, 80, 0, 0, 0, 0, 1, 0, 0, 10, 4087, 1, 0, 4, 4089, 0, 0, 0, 1, 0, 4094, 0, 3, 6, 3993, 5, 0, 3827, 0, 4090, 0, 2, 2, 3, 4091, 10, 4078, 0, 0, 0, 29, 0, 4087, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 4083, 0, 0, 4096, 0, 0, 0, 4095, 4096, 4096, 0, 4096, 1, 0, 2, 0, 0, 0, 0, 0, 123, 2129, 0, 1, 0, 2535, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 1955, 0, 0, 3980, 0, 4073, 0, 0, 0, 0, 4096, 2, 1583, 1, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[5, 4096, 0, 0, 4096, 0, 2, 3, 4096, 4015, 4093, 0, 4092, 0, 7, 38, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 537, 0, 0, 0, 4096, 0, 0, 0, 4092, 10, 0, 0, 0, 0, 4070, 0, 0, 0, 104, 0, 0, 3450, 72, 4079, 2, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4, 1, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 3967, 0, 38, 4061, 0, 0, 0, 4094, 3933, 4096, 0, 4045, 0, 201, 0, 29, 0, 1, 1, 0, 22, 4063, 0, 0, 1, 4092, 0, 0, 0, 4042, 0, 3, 0, 4078, 0, 0, 0, 0, 0, 4082, 0, 0, 2, 22, 6, 6, 84, 28, 3520, 432, 1, 0, 0, 4085, 154, 4090, 64, 13, 0, 0, 4, 4096, 4080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4055, 0, 0, 0, 159, 46, 0, 0, 0, 0, 0, 0, 4086, 3844, 56, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 1258, 0, 4096, 1078, 0, 0, 0, 0, 0, 0, 55, 0, 3988, 0, 0, 0, 4093, 0, 1631, 439, 0, 0, 0, 0, 0, 0, 23, 0, 4096, 0, 29, 0, 0, 0, 2906, 1151, 0, 0, 0, 0, 0, 0, 3498, 0, 0, 0, 2701, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 85, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 164, 0, 0, 0, 9, 1303, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8356, 8041, 6843, 10206, 8205, 10293, 7743, 5849] avg=8192.0000 max=10293 max_rank=5 min=5849 min_rank=7 imbalance_ratio=1.256470 max_load_reduction=55243 max_load_reduction_pct=0.842941 moved_to_echo=32090 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 2908, 0, 0, 4044, 26, 42, 0, 153, 46, 0, 0, 0, 0, 0, 0, 4056, 2851, 56, 0, 0, 0, 579, 13, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 29, 70, 752, 3311, 0, 4086, 1080, 0, 0, 0, 0, 0, 0, 439, 7, 2653, 20, 0, 3, 3907, 0, 1621, 424, 0, 0, 0, 0, 0, 0, 18, 0, 4085, 0, 0, 0, 0, 0, 2910, 1145, 0, 0, 0, 0, 0, 0, 3373, 27, 11, 35, 3960, 0, 20, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 225, 3970, 100, 11, 14, 0, 4043, 157, 0, 0, 0, 0, 0, 0, 0, 0, 356, 0, 1, 0, 0, 33, 1296, 4068, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7219, 7555, 9398, 9074, 8158, 9858, 8520, 5754] avg=8192.0000 max=9858 max_rank=5 min=5754 min_rank=7 imbalance_ratio=1.203369 max_load_reduction=55678 max_load_reduction_pct=0.849579 moved_to_echo=32290 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 4072, 0, 2, 4089, 0, 5, 0, 147, 46, 0, 0, 0, 0, 0, 0, 4096, 3955, 57, 0, 0, 0, 20, 31, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 0, 442, 3065, 0, 4077, 1091, 0, 0, 0, 0, 0, 0, 1, 0, 2523, 0, 2, 0, 3814, 0, 1634, 407, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 2941, 1154, 0, 0, 0, 0, 0, 0, 3244, 0, 0, 0, 565, 2, 0, 2606, 0, 0, 0, 0, 0, 0, 0, 0, 170, 4084, 133, 5, 0, 13, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3443, 23, 8, 0, 0, 11, 1306, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8362, 8159, 8684, 8381, 8193, 6417, 8453, 8887] avg=8192.0000 max=8887 max_rank=7 min=6417 min_rank=5 imbalance_ratio=1.084839 max_load_reduction=56649 max_load_reduction_pct=0.864395 moved_to_echo=31950 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 3841, 19, 0, 4066, 0, 0, 26, 158, 46, 0, 0, 0, 0, 0, 0, 4064, 4096, 56, 39, 0, 7, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 8, 0, 10, 196, 1, 3904, 6, 4081, 1090, 0, 0, 0, 0, 0, 0, 0, 18, 10, 0, 0, 25, 3946, 18, 1625, 438, 0, 0, 0, 0, 0, 0, 0, 15, 4089, 0, 0, 0, 3, 0, 2938, 1148, 0, 0, 0, 0, 0, 0, 3491, 0, 12, 0, 3, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 7, 0, 4033, 45, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 211, 3, 0, 0, 0, 1299, 4050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8500, 9346, 6080, 8193, 7579, 8172, 9510] avg=8192.0000 max=9510 max_rank=7 min=6080 min_rank=3 imbalance_ratio=1.160889 max_load_reduction=56026 max_load_reduction_pct=0.854889 moved_to_echo=33454 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5, 4096, 0, 0, 4096, 0, 2, 3, 158, 46, 0, 0, 0, 0, 0, 0, 4096, 4015, 57, 0, 0, 0, 7, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4092, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 537, 0, 0, 0, 4096, 0, 1635, 437, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 10, 0, 0, 0, 0, 2944, 1154, 0, 0, 0, 0, 0, 0, 3475, 0, 0, 0, 104, 0, 0, 3450, 0, 0, 0, 0, 0, 0, 0, 0, 72, 4079, 2, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 1, 1307, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8406, 8213, 9276, 6705, 8200, 7029, 8203, 9504] avg=8192.0000 max=9504 max_rank=7 min=6705 min_rank=3 imbalance_ratio=1.160156 max_load_reduction=56032 max_load_reduction_pct=0.854980 moved_to_echo=32936 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 4049, 0, 0, 4092, 8, 10, 5, 159, 46, 0, 0, 0, 0, 0, 0, 4037, 4096, 56, 19, 0, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 80, 0, 0, 4082, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 4087, 1, 1631, 439, 0, 0, 0, 0, 0, 0, 0, 4, 4089, 0, 0, 0, 1, 0, 2938, 1152, 0, 0, 0, 0, 0, 0, 3496, 0, 3, 6, 3993, 5, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2, 2, 3, 4045, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 29, 0, 1304, 4086, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8373, 8541, 5290, 6169, 8184, 11330, 8152, 9497] avg=8192.0000 max=11330 max_rank=5 min=5290 min_rank=2 imbalance_ratio=1.383057 max_load_reduction=54206 max_load_reduction_pct=0.827118 moved_to_echo=37163 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4083, 0, 0, 4096, 0, 0, 0, 159, 46, 0, 0, 0, 0, 0, 0, 4095, 4096, 57, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 2129, 0, 4096, 1093, 0, 0, 0, 0, 0, 0, 1, 0, 2535, 0, 0, 0, 4093, 0, 1635, 439, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 0, 2946, 1154, 0, 0, 0, 0, 0, 0, 3498, 0, 0, 0, 1955, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 4050, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1583, 1, 0, 0, 0, 0, 1307, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8384, 8251, 7441, 8703, 8212, 9433, 8125, 6987] avg=8192.0000 max=9433 max_rank=5 min=6987 min_rank=7 imbalance_ratio=1.151489 max_load_reduction=56103 max_load_reduction_pct=0.856064 moved_to_echo=32757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 3967, 0, 38, 4061, 0, 0, 0, 162, 52, 0, 0, 0, 0, 0, 0, 4094, 3933, 48, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 1, 1, 0, 22, 4063, 0, 4045, 1097, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 0, 4042, 0, 1640, 443, 0, 0, 0, 0, 0, 0, 3, 0, 4078, 0, 0, 0, 0, 0, 2951, 1159, 0, 0, 0, 0, 0, 0, 3477, 0, 0, 2, 22, 6, 6, 84, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3520, 432, 1, 0, 0, 4033, 154, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 64, 13, 0, 0, 4, 1297, 4080, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8280, 8276, 9258, 10218, 8191, 3597, 8168, 9548] avg=8192.0000 max=10218 max_rank=3 min=3597 min_rank=5 imbalance_ratio=1.247314 max_load_reduction=55318 max_load_reduction_pct=0.844086 moved_to_echo=29504 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[27648, 128, 128, 128, 13312, 128, 128, 24576, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[21888, 512, 128, 0, 128, 128, 10496, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 30976, 512, 128, 0, 128, 1408, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 31232, 128, 128, 32640, 128, 128, 128, 1280, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 31616, 768, 128, 128, 128, 32384, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 128, 23552, 9344, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 128, 768, 1536, 21888, 128, 32768, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[512, 128, 16384, 128, 128, 128, 32128, 128, 13056, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 3, 4096, 0, 3, 26, 4, 0, 1795, 0, 0, 4095, 0, 0, 0, 0, 4096, 4076, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 83, 0, 4096, 3, 4093, 0, 3, 4093, 4096, 4019, 0, 0, 0, 0, 0, 0, 3970, 0, 0, 0, 424, 0, 0, 11, 0, 8, 4094, 0, 12, 1, 3437, 0, 4074, 2715, 4011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 209, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 3, 52, 4096, 0, 4012, 0, 0, 3441, 4096, 4096, 0, 1, 0, 0, 0, 0, 4096, 1, 0, 0, 233, 0, 0, 0, 1, 0, 4096, 0, 3071, 0, 732, 0, 4092, 589, 4040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[12, 15, 0, 4081, 12, 0, 40, 58, 0, 4096, 0, 0, 4073, 14, 34, 0, 1, 3749, 4047, 0, 0, 0, 0, 0, 1, 11, 2, 0, 4088, 0, 6, 4078, 0, 4074, 0, 0, 4084, 4073, 4040, 23, 7, 0, 22, 0, 0, 4096, 8, 25, 0, 76, 2, 0, 0, 0, 0, 4096, 0, 3502, 0, 1417, 2, 4095, 1268, 2208] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4, 8, 0, 243, 0, 4093, 0, 0, 4085, 0, 1, 14, 4, 4094, 3463, 0, 0, 0, 0, 28, 0, 0, 0, 0, 4095, 68, 117, 4092, 2, 4091, 0, 0, 3629, 4094, 3981, 0, 0, 0, 0, 0, 38, 4096, 0, 6, 0, 0, 5, 0, 0, 0, 0, 4054, 31, 3658, 0, 375, 0, 4036, 872, 4064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 1, 4096, 0, 12, 0, 291, 0, 4038, 0, 0, 4096, 0, 13, 0, 0, 4096, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 0, 4096, 10, 4096, 0, 0, 4081, 4092, 4033, 0, 0, 0, 0, 0, 3, 4072, 0, 0, 0, 79, 0, 1, 0, 6, 0, 4094, 30, 2700, 0, 4077, 0, 4054, 6, 4074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4, 0, 4055, 4, 0, 0, 399, 17, 426, 0, 0, 4085, 0, 45, 0, 54, 4047, 3684, 54, 5, 0, 0, 17, 0, 0, 0, 50, 4077, 0, 0, 4075, 0, 4096, 0, 0, 3965, 4030, 3700, 0, 0, 30, 0, 0, 0, 4065, 0, 0, 22, 0, 0, 4, 131, 60, 9, 3993, 102, 4028, 19, 4089, 0, 4075, 0, 4020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[5, 2, 0, 4087, 1, 0, 23, 0, 0, 813, 0, 2, 4092, 0, 11, 0, 1, 4071, 13, 4, 0, 0, 18, 0, 0, 0, 1, 6, 4093, 0, 0, 4074, 0, 4094, 0, 0, 4072, 4089, 4089, 0, 0, 6, 14, 3, 0, 4096, 0, 3, 2, 2441, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 3335, 3, 4087, 1605, 4090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 9, 1721, 5, 0, 4079, 0, 0, 87, 16, 4050, 4091, 0, 21, 13, 0, 121, 0, 0, 0, 0, 4066, 206, 410, 4091, 37, 4093, 1, 18, 3976, 4078, 4095, 22, 0, 0, 8, 0, 7, 4096, 0, 0, 1, 0, 38, 2, 0, 0, 0, 3756, 0, 3010, 0, 3947, 0, 3912, 967, 2392] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 4096, 0, 3, 26, 4, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 1795, 0, 0, 4095, 0, 0, 0, 1350, 222, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 0, 0, 0, 0, 0, 1016, 115, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3981, 83, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 4093, 255, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 0, 4093, 14, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 11, 0, 8, 4094, 3619, 106, 0, 0, 0, 0, 0, 0, 0, 12, 1, 3437, 0, 0, 2715, 1645, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 7462, 9303, 8163, 8373, 8077, 8262, 7810] avg=8192.0000 max=9303 max_rank=2 min=7462 min_rank=1 imbalance_ratio=1.135620 max_load_reduction=56233 max_load_reduction_pct=0.858047 moved_to_echo=32522 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4, 8, 0, 243, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4085, 0, 1, 14, 1368, 222, 0, 0, 0, 0, 0, 0, 4, 4094, 3463, 0, 0, 0, 0, 28, 1030, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 68, 117, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3629, 255, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 0, 6, 4091, 16, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4054, 3617, 105, 0, 0, 0, 0, 0, 0, 31, 3658, 0, 375, 0, 0, 872, 1666, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 9783, 8734, 8257, 7867, 8247, 7781, 6602] avg=8192.0000 max=9783 max_rank=1 min=6602 min_rank=7 imbalance_ratio=1.194214 max_load_reduction=55753 max_load_reduction_pct=0.850723 moved_to_echo=30497 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 4055, 4, 0, 0, 399, 3953, 0, 0, 0, 0, 0, 0, 0, 17, 426, 0, 0, 4085, 0, 45, 0, 1353, 218, 0, 0, 0, 0, 0, 0, 54, 4047, 3684, 54, 5, 0, 0, 17, 1018, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 3963, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 251, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 4065, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 22, 0, 0, 4, 131, 60, 9, 3993, 3561, 108, 0, 0, 0, 0, 0, 0, 102, 4028, 19, 4089, 0, 0, 0, 1649, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8415, 6144, 8993, 8088, 7916, 8205, 7888, 9887] avg=8192.0000 max=9887 max_rank=7 min=6144 min_rank=1 imbalance_ratio=1.206909 max_load_reduction=55649 max_load_reduction_pct=0.849136 moved_to_echo=33896 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4096, 0, 12, 0, 291, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 4096, 0, 13, 0, 1371, 222, 0, 0, 0, 0, 0, 0, 0, 4096, 1167, 0, 0, 0, 0, 0, 1032, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 26, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4081, 254, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4072, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 79, 0, 1, 0, 6, 0, 4094, 3616, 108, 0, 0, 0, 0, 0, 0, 30, 2700, 0, 4077, 0, 0, 6, 1671, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8332, 9740, 6410, 8103, 8378, 8185, 7904, 8484] avg=8192.0000 max=9740 max_rank=1 min=6410 min_rank=2 imbalance_ratio=1.188965 max_load_reduction=55796 max_load_reduction_pct=0.851379 moved_to_echo=32951 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[12, 15, 0, 4081, 12, 0, 40, 58, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4073, 14, 34, 0, 743, 221, 0, 0, 0, 0, 0, 0, 1, 3749, 4047, 0, 0, 0, 0, 0, 559, 115, 0, 0, 0, 0, 0, 0, 1, 11, 2, 0, 3973, 0, 6, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 253, 4040, 23, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 22, 0, 0, 4096, 8, 25, 4074, 16, 0, 0, 0, 0, 0, 0, 0, 76, 2, 0, 0, 0, 0, 4096, 3599, 107, 0, 0, 0, 0, 0, 0, 0, 3502, 0, 1417, 2, 0, 1268, 906, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 9181, 8471, 8071, 8400, 8248, 7880, 7095] avg=8192.0000 max=9181 max_rank=1 min=7095 min_rank=7 imbalance_ratio=1.120728 max_load_reduction=56355 max_load_reduction_pct=0.859909 moved_to_echo=31623 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 209, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 1360, 222, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 1024, 115, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3981, 3, 52, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3441, 255, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 1, 0, 4012, 14, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 1, 0, 4096, 3619, 107, 0, 0, 0, 0, 0, 0, 0, 3071, 0, 732, 0, 0, 589, 1656, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 9773, 9328, 8139, 7792, 8124, 8056, 6048] avg=8192.0000 max=9773 max_rank=1 min=6048 min_rank=7 imbalance_ratio=1.192993 max_load_reduction=55763 max_load_reduction_pct=0.850876 moved_to_echo=30020 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[5, 2, 0, 4087, 1, 0, 23, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 813, 0, 2, 4092, 0, 11, 0, 1377, 222, 0, 0, 0, 0, 0, 0, 1, 4071, 13, 4, 0, 0, 18, 0, 1036, 115, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 3978, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 254, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 14, 3, 0, 4096, 0, 3, 4094, 16, 0, 0, 0, 0, 0, 0, 2, 2441, 0, 0, 0, 0, 0, 4096, 3613, 107, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3335, 3, 0, 1605, 1677, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 6517, 5258, 8059, 8415, 8232, 10259, 10714] avg=8192.0000 max=10714 max_rank=7 min=5258 min_rank=2 imbalance_ratio=1.307861 max_load_reduction=54822 max_load_reduction_pct=0.836517 moved_to_echo=37620 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 3796, 0, 0, 0, 0, 0, 0, 0, 9, 1721, 5, 0, 4079, 0, 0, 87, 810, 225, 0, 0, 0, 0, 0, 0, 16, 4050, 4091, 0, 21, 13, 0, 121, 610, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 206, 410, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 1, 18, 3976, 246, 4095, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 7, 4096, 0, 0, 4093, 14, 0, 0, 0, 0, 0, 0, 1, 0, 38, 2, 0, 0, 0, 3756, 3607, 102, 0, 0, 0, 0, 0, 0, 0, 3010, 0, 3947, 0, 0, 967, 972, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7890, 6936, 9039, 8656, 8395, 8218, 7506, 8896] avg=8192.0000 max=9039 max_rank=2 min=6936 min_rank=1 imbalance_ratio=1.103394 max_load_reduction=56497 max_load_reduction_pct=0.862076 moved_to_echo=33015 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32384, 24704, 128, 128, 128, 128, 256, 7424, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 31872, 512, 640, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 1280, 31488, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 24192, 128, 21504, 128, 0, 8064, 11904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 3328, 128, 128, 256, 128, 128, 32384, 28928, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 21120, 128, 128, 32768, 128, 128, 128, 9856, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 31360, 2048, 32128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 32640, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 3645, 4016, 28, 0, 0, 4092, 0, 0, 0, 0, 0, 1514, 4096, 0, 1, 8, 0, 0, 4067, 0, 4071, 0, 2, 0, 4053, 5, 0, 206, 0, 0, 4096, 0, 4096, 0, 4096, 111, 13, 0, 4024, 0, 236, 4096, 0, 0, 0, 0, 3030, 0, 0, 0, 4083, 0, 51, 0, 0, 0, 3795, 0, 0, 27, 3978] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 4023, 2131, 1, 0, 4028, 4093, 0, 0, 0, 0, 0, 346, 4096, 0, 4, 14, 0, 0, 4095, 0, 4096, 23, 186, 0, 235, 13, 1865, 71, 0, 0, 4096, 0, 3272, 18, 4088, 885, 0, 0, 3192, 0, 3, 4096, 0, 39, 0, 0, 4096, 0, 0, 0, 4016, 14, 307, 0, 0, 0, 4096, 0, 0, 0, 3997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[24, 27, 0, 4025, 292, 3, 0, 281, 4085, 0, 47, 16, 0, 16, 0, 3963, 0, 0, 0, 0, 0, 4080, 1, 3043, 0, 4003, 0, 4063, 0, 315, 104, 38, 0, 4070, 0, 4096, 0, 4087, 342, 0, 19, 4078, 9, 0, 4052, 74, 119, 0, 10, 3872, 1, 13, 0, 4096, 0, 0, 0, 8, 4, 4060, 0, 21, 0, 4079] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[7, 2591, 0, 4093, 4096, 78, 5, 3346, 4092, 4, 0, 0, 5, 0, 60, 3992, 0, 0, 0, 0, 2, 4092, 3, 1037, 4, 0, 57, 4085, 0, 1095, 0, 0, 8, 4093, 0, 4096, 0, 4071, 0, 3, 3, 4091, 98, 0, 3965, 2331, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 25, 3, 2, 1773, 42, 0, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 3126, 0, 4096, 1228, 0, 0, 2925, 4096, 0, 0, 0, 0, 0, 3537, 4015, 0, 0, 87, 0, 0, 4095, 0, 4096, 0, 45, 38, 3473, 0, 1, 253, 0, 0, 4001, 280, 3654, 0, 4096, 0, 0, 0, 1358, 0, 2, 4096, 678, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4063, 5, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[16, 0, 4, 3527, 444, 0, 11, 187, 4070, 0, 0, 23, 0, 0, 3399, 4082, 0, 30, 2, 0, 10, 4052, 7, 4086, 21, 3387, 0, 1156, 206, 0, 2, 11, 0, 4090, 0, 3235, 0, 4045, 148, 104, 0, 1999, 0, 1825, 4096, 0, 10, 5, 0, 3982, 64, 1, 0, 3932, 55, 837, 0, 2, 3, 4007, 0, 0, 356, 4007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4088, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 3627, 4096, 0, 0, 1, 0, 0, 4085, 0, 4096, 0, 3981, 0, 3858, 0, 0, 54, 0, 0, 4096, 0, 3869, 0, 4095, 233, 0, 0, 802, 0, 0, 4096, 5, 7, 0, 0, 4096, 0, 1, 0, 4065, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 7, 4026, 0, 0, 1, 2517, 4096, 0, 18, 1, 0, 0, 0, 4017, 60, 0, 0, 30, 9, 4081, 0, 4080, 65, 401, 0, 4050, 118, 132, 1, 0, 0, 4088, 0, 2568, 0, 4074, 847, 1, 0, 3690, 0, 589, 4063, 0, 84, 0, 3, 4095, 0, 0, 0, 3658, 2, 2233, 0, 0, 0, 3948, 12, 0, 0, 3871] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3645, 4016, 28, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 2575, 0, 0, 0, 0, 0, 1514, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 4067, 0, 4071, 372, 129, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4053, 5, 0, 206, 0, 2993, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 130, 111, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 236, 4096, 0, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 3030, 0, 0, 0, 4083, 0, 51, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3795, 0, 0, 27, 3978, 371, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8013, 8185, 8648, 7259, 8446, 8806, 8008, 8171] avg=8192.0000 max=8806 max_rank=5 min=7259 min_rank=3 imbalance_ratio=1.074951 max_load_reduction=56730 max_load_reduction_pct=0.865631 moved_to_echo=33431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 4, 3527, 444, 0, 11, 187, 322, 0, 0, 0, 0, 0, 0, 0, 2561, 0, 0, 23, 0, 0, 3399, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 2, 0, 10, 4052, 7, 4086, 370, 127, 0, 0, 0, 0, 0, 0, 21, 3387, 0, 1156, 206, 0, 2, 11, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3235, 0, 130, 148, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1999, 0, 1825, 4096, 0, 10, 5, 448, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 64, 1, 0, 3932, 55, 837, 833, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4007, 0, 0, 356, 4007, 369, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4511, 10065, 8684, 7738, 7707, 8383, 9704, 8744] avg=8192.0000 max=10065 max_rank=1 min=4511 min_rank=0 imbalance_ratio=1.228638 max_load_reduction=55471 max_load_reduction_pct=0.846420 moved_to_echo=34538 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4023, 2131, 1, 0, 4028, 324, 0, 0, 0, 0, 0, 0, 0, 2573, 0, 0, 0, 0, 0, 346, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 14, 0, 0, 4095, 0, 4096, 373, 129, 0, 0, 0, 0, 0, 0, 23, 186, 0, 235, 13, 1865, 71, 0, 2987, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3272, 18, 130, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3192, 0, 3, 4096, 0, 39, 0, 451, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4016, 14, 307, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3997, 372, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10508, 7015, 8711, 5380, 8401, 7781, 9275, 8465] avg=8192.0000 max=10508 max_rank=0 min=5380 min_rank=3 imbalance_ratio=1.282715 max_load_reduction=55028 max_load_reduction_pct=0.839661 moved_to_echo=33922 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[24, 27, 0, 4025, 292, 3, 0, 281, 323, 0, 0, 0, 0, 0, 0, 0, 2569, 0, 47, 16, 0, 16, 0, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 1, 3043, 372, 129, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 4063, 0, 315, 104, 38, 2986, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4096, 0, 130, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4078, 9, 0, 4052, 74, 119, 0, 450, 0, 0, 0, 0, 0, 0, 0, 10, 3872, 1, 13, 0, 4096, 0, 0, 842, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 4060, 0, 21, 0, 4079, 371, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4975, 6611, 7625, 11509, 8638, 8801, 8834, 8543] avg=8192.0000 max=11509 max_rank=3 min=4975 min_rank=0 imbalance_ratio=1.404907 max_load_reduction=54027 max_load_reduction_pct=0.824387 moved_to_echo=34816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3126, 0, 4096, 1228, 0, 0, 2925, 324, 0, 0, 0, 0, 0, 0, 0, 2576, 0, 0, 0, 0, 0, 3537, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 4095, 0, 4096, 373, 129, 0, 0, 0, 0, 0, 0, 0, 45, 38, 3473, 0, 1, 253, 0, 2993, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 280, 3654, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 0, 2, 4096, 678, 0, 0, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 5, 0, 0, 4096, 372, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11699, 10128, 8780, 6803, 8065, 6585, 4940, 8536] avg=8192.0000 max=11699 max_rank=0 min=4940 min_rank=6 imbalance_ratio=1.428101 max_load_reduction=53837 max_load_reduction_pct=0.821487 moved_to_echo=28126 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[7, 2591, 0, 4093, 4096, 78, 5, 3346, 324, 0, 0, 0, 0, 0, 0, 0, 2575, 4, 0, 0, 5, 0, 60, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 3, 1037, 372, 128, 0, 0, 0, 0, 0, 0, 4, 0, 57, 4085, 0, 1095, 0, 0, 2974, 0, 0, 0, 0, 0, 0, 0, 8, 4093, 0, 4096, 0, 130, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 98, 0, 3965, 2331, 0, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 839, 0, 0, 0, 0, 0, 0, 0, 25, 3, 2, 1773, 42, 0, 0, 4093, 371, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14540, 6636, 5634, 8215, 8330, 10938, 4934, 6309] avg=8192.0000 max=14540 max_rank=0 min=4934 min_rank=6 imbalance_ratio=1.774902 max_load_reduction=50996 max_load_reduction_pct=0.778137 moved_to_echo=30511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4088, 0, 0, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 2575, 0, 0, 0, 0, 1, 3627, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4085, 0, 4096, 373, 129, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 3858, 0, 0, 54, 0, 2992, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3869, 0, 130, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 802, 0, 0, 4096, 5, 7, 0, 451, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4065, 0, 0, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4091, 372, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4412, 10299, 8684, 10885, 8328, 5361, 9006, 8561] avg=8192.0000 max=10885 max_rank=3 min=4412 min_rank=0 imbalance_ratio=1.328735 max_load_reduction=54651 max_load_reduction_pct=0.833908 moved_to_echo=31256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 4026, 0, 0, 1, 2517, 327, 0, 0, 0, 0, 0, 0, 0, 2561, 0, 18, 1, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 30, 9, 4081, 0, 4080, 377, 133, 0, 0, 0, 0, 0, 0, 65, 401, 0, 4050, 118, 132, 1, 0, 2980, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 2568, 0, 117, 847, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3690, 0, 589, 4063, 0, 84, 0, 455, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 3658, 2, 2233, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 12, 0, 0, 3871, 376, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6878, 6597, 8770, 7747, 7621, 8881, 10835, 8207] avg=8192.0000 max=10835 max_rank=6 min=6597 min_rank=1 imbalance_ratio=1.322632 max_load_reduction=54701 max_load_reduction_pct=0.834671 moved_to_echo=35544 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 23296, 128, 2688, 32640, 3200, 384, 128, 3712, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 5760, 128, 31616, 12288, 128, 128, 13312, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 12032, 128, 25088, 384, 3456, 768, 128, 23936, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 23296, 128, 128, 0, 32128, 128, 3456, 6784, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 28672, 3072, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32640, 384, 28928, 128, 1152, 2688, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 29952, 128, 128, 512, 32256, 3072, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[20608, 128, 128, 128, 128, 128, 12544, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[36, 1183, 2616, 35, 1550, 0, 807, 11, 1517, 49, 1216, 472, 566, 446, 13, 415, 150, 2065, 2102, 1611, 3659, 874, 802, 1451, 4, 3610, 146, 1203, 440, 166, 602, 1416, 3349, 186, 121, 347, 687, 128, 1043, 1224, 2391, 3223, 754, 284, 182, 1822, 1488, 2063, 129, 676, 1052, 184, 120, 587, 3035, 194, 57, 1597, 3443, 2295, 284, 855, 379, 124] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[15, 747, 2115, 39, 1807, 0, 886, 29, 1079, 104, 943, 948, 705, 258, 27, 470, 263, 1790, 2097, 1580, 3562, 1184, 534, 1562, 1, 3341, 186, 863, 597, 107, 743, 1422, 3546, 189, 155, 723, 509, 154, 1245, 1126, 2315, 3054, 944, 217, 253, 2468, 1646, 1887, 163, 515, 953, 168, 88, 275, 3691, 271, 17, 1833, 3267, 1749, 267, 924, 871, 49] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[32, 992, 2615, 70, 2037, 4, 786, 47, 1050, 105, 986, 691, 649, 303, 45, 443, 248, 1975, 2020, 1518, 3423, 810, 604, 1321, 36, 3042, 253, 957, 430, 201, 514, 1568, 3541, 199, 131, 806, 869, 151, 1199, 1410, 2415, 2783, 1083, 179, 132, 2327, 2002, 1492, 182, 752, 829, 130, 123, 721, 3456, 256, 20, 1594, 3101, 2068, 325, 1040, 314, 131] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[25, 963, 2094, 128, 2001, 2, 1023, 22, 1689, 28, 988, 372, 544, 305, 41, 404, 267, 2023, 2014, 1384, 3627, 1037, 616, 1779, 4, 3293, 189, 735, 167, 57, 636, 1421, 3486, 435, 199, 959, 345, 229, 1222, 1421, 2439, 2594, 607, 77, 275, 2331, 1685, 1999, 230, 560, 1140, 136, 74, 442, 3433, 246, 64, 1359, 3700, 2344, 318, 637, 545, 127] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[21, 1213, 2032, 86, 1929, 0, 836, 33, 1864, 49, 1197, 713, 1239, 259, 15, 517, 355, 1761, 2205, 1402, 3524, 1394, 637, 1595, 12, 3388, 165, 755, 392, 65, 743, 885, 3263, 182, 145, 696, 623, 121, 1211, 1384, 2250, 3018, 754, 236, 245, 2320, 1176, 1166, 148, 829, 1324, 94, 180, 446, 3513, 203, 71, 1784, 3561, 1853, 329, 682, 309, 139] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[22, 781, 1989, 34, 1342, 1, 865, 23, 2459, 129, 1565, 668, 609, 567, 18, 420, 169, 1436, 2322, 1104, 3522, 1477, 532, 1887, 1, 3152, 155, 978, 431, 87, 990, 1227, 3152, 206, 125, 970, 888, 65, 742, 1315, 2095, 3509, 1200, 431, 199, 2406, 985, 550, 194, 551, 1152, 172, 274, 440, 3244, 269, 13, 2324, 3661, 1473, 661, 386, 828, 94] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[43, 943, 2314, 45, 1700, 23, 889, 28, 1084, 140, 1303, 640, 498, 301, 50, 385, 209, 1947, 2165, 1477, 3400, 879, 543, 1579, 6, 2819, 158, 951, 647, 161, 721, 1955, 3292, 297, 274, 1114, 942, 168, 848, 1608, 2387, 2902, 808, 237, 312, 2442, 1706, 1465, 167, 816, 769, 159, 103, 459, 3104, 135, 27, 1622, 3131, 1835, 367, 934, 925, 178] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[34, 1029, 2301, 65, 1969, 0, 1142, 14, 1851, 112, 1297, 569, 477, 349, 15, 354, 200, 2325, 1677, 1178, 3675, 1197, 701, 1571, 10, 3449, 138, 907, 239, 56, 699, 1583, 3259, 262, 89, 1118, 443, 119, 1088, 1198, 2266, 3177, 633, 194, 219, 2183, 1693, 873, 93, 762, 1288, 130, 182, 693, 3069, 118, 90, 1667, 3640, 2419, 204, 734, 278, 172] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[36, 1183, 2616, 35, 1550, 0, 807, 11, 1945, 315, 0, 0, 0, 0, 0, 0, 1517, 49, 1216, 472, 566, 446, 13, 415, 3344, 0, 0, 0, 0, 0, 0, 0, 150, 2065, 1334, 1611, 0, 874, 802, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3610, 146, 1203, 440, 166, 602, 1416, 768, 342, 66, 0, 0, 0, 0, 0, 3349, 186, 121, 347, 687, 128, 1043, 1224, 385, 0, 0, 0, 0, 0, 0, 0, 1664, 0, 754, 284, 182, 1822, 1488, 2063, 0, 0, 0, 0, 0, 0, 0, 0, 129, 676, 1052, 184, 120, 587, 3035, 194, 1278, 753, 0, 0, 0, 0, 0, 0, 57, 1597, 2624, 2295, 284, 855, 379, 124, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8498, 8038, 8287, 8763, 7470, 8257, 8008, 8215] avg=8192.0000 max=8763 max_rank=3 min=7470 min_rank=4 imbalance_ratio=1.069702 max_load_reduction=56773 max_load_reduction_pct=0.866287 moved_to_echo=31950 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[15, 747, 2115, 39, 1807, 0, 886, 29, 1842, 306, 0, 0, 0, 0, 0, 0, 1079, 104, 943, 948, 705, 258, 27, 470, 3256, 0, 0, 0, 0, 0, 0, 0, 263, 1790, 1331, 1580, 0, 1184, 534, 1562, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3341, 186, 863, 597, 107, 743, 1422, 766, 331, 63, 0, 0, 0, 0, 0, 3546, 189, 155, 723, 509, 154, 1245, 1126, 373, 0, 0, 0, 0, 0, 0, 0, 1611, 0, 944, 217, 253, 2468, 1646, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 163, 515, 953, 168, 88, 275, 3691, 271, 1212, 714, 0, 0, 0, 0, 0, 0, 17, 1833, 2490, 1749, 267, 924, 871, 49, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7786, 7790, 8244, 8420, 8020, 9026, 8050, 8200] avg=8192.0000 max=9026 max_rank=5 min=7786 min_rank=0 imbalance_ratio=1.101807 max_load_reduction=56510 max_load_reduction_pct=0.862274 moved_to_echo=33296 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[22, 781, 1989, 34, 1342, 1, 865, 23, 2116, 302, 0, 0, 0, 0, 0, 0, 2459, 129, 1565, 668, 609, 567, 18, 420, 3220, 0, 0, 0, 0, 0, 0, 0, 169, 1436, 1474, 1104, 0, 1477, 532, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3152, 155, 978, 431, 87, 990, 1227, 848, 299, 71, 0, 0, 0, 0, 0, 3152, 206, 125, 970, 888, 65, 742, 1315, 338, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 1200, 431, 199, 2406, 985, 550, 0, 0, 0, 0, 0, 0, 0, 0, 194, 551, 1152, 172, 274, 440, 3244, 269, 1393, 801, 0, 0, 0, 0, 0, 0, 13, 2324, 2789, 1473, 661, 386, 828, 94, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7475, 9655, 8079, 8239, 7801, 7229, 8490, 8568] avg=8192.0000 max=9655 max_rank=1 min=7229 min_rank=5 imbalance_ratio=1.178589 max_load_reduction=55881 max_load_reduction_pct=0.852676 moved_to_echo=32088 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[32, 992, 2615, 70, 2037, 4, 786, 47, 1679, 295, 0, 0, 0, 0, 0, 0, 1050, 105, 986, 691, 649, 303, 45, 443, 3128, 0, 0, 0, 0, 0, 0, 0, 248, 1975, 1282, 1518, 0, 810, 604, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3042, 253, 957, 430, 201, 514, 1568, 738, 345, 60, 0, 0, 0, 0, 0, 3541, 199, 131, 806, 869, 151, 1199, 1410, 389, 0, 0, 0, 0, 0, 0, 0, 1681, 0, 1083, 179, 132, 2327, 2002, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 182, 752, 829, 130, 123, 721, 3456, 256, 1104, 678, 0, 0, 0, 0, 0, 0, 20, 1594, 2363, 2068, 325, 1040, 314, 131, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8557, 7400, 7758, 8144, 8695, 8896, 8231, 7855] avg=8192.0000 max=8896 max_rank=5 min=7400 min_rank=1 imbalance_ratio=1.085938 max_load_reduction=56640 max_load_reduction_pct=0.864258 moved_to_echo=33677 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[34, 1029, 2301, 65, 1969, 0, 1142, 14, 1916, 315, 0, 0, 0, 0, 0, 0, 1851, 112, 1297, 569, 477, 349, 15, 354, 3360, 0, 0, 0, 0, 0, 0, 0, 200, 2325, 1064, 1178, 0, 1197, 701, 1571, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3449, 138, 907, 239, 56, 699, 1583, 613, 324, 70, 0, 0, 0, 0, 0, 3259, 262, 89, 1118, 443, 119, 1088, 1198, 365, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 633, 194, 219, 2183, 1693, 873, 0, 0, 0, 0, 0, 0, 0, 0, 93, 762, 1288, 130, 182, 693, 3069, 118, 1261, 796, 0, 0, 0, 0, 0, 0, 90, 1667, 2774, 2419, 204, 734, 278, 172, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8785, 8384, 8236, 8088, 7941, 7372, 8392, 8338] avg=8192.0000 max=8785 max_rank=0 min=7372 min_rank=5 imbalance_ratio=1.072388 max_load_reduction=56751 max_load_reduction_pct=0.865952 moved_to_echo=32043 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[21, 1213, 2032, 86, 1929, 0, 836, 33, 1820, 303, 0, 0, 0, 0, 0, 0, 1864, 49, 1197, 713, 1239, 259, 15, 517, 3221, 0, 0, 0, 0, 0, 0, 0, 355, 1761, 1399, 1402, 0, 1394, 637, 1595, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3388, 165, 755, 392, 65, 743, 885, 806, 322, 69, 0, 0, 0, 0, 0, 3263, 182, 145, 696, 623, 121, 1211, 1384, 363, 0, 0, 0, 0, 0, 0, 0, 1565, 0, 754, 236, 245, 2320, 1176, 1166, 0, 0, 0, 0, 0, 0, 0, 0, 148, 829, 1324, 94, 180, 446, 3513, 203, 1198, 779, 0, 0, 0, 0, 0, 0, 71, 1784, 2713, 1853, 329, 682, 309, 139, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 9074, 8543, 7602, 7988, 7462, 8714, 7880] avg=8192.0000 max=9074 max_rank=1 min=7462 min_rank=5 imbalance_ratio=1.107666 max_load_reduction=56462 max_load_reduction_pct=0.861542 moved_to_echo=32044 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[25, 963, 2094, 128, 2001, 2, 1023, 22, 1564, 311, 0, 0, 0, 0, 0, 0, 1689, 28, 988, 372, 544, 305, 41, 404, 3316, 0, 0, 0, 0, 0, 0, 0, 267, 2023, 1278, 1384, 0, 1037, 616, 1779, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3293, 189, 735, 167, 57, 636, 1421, 736, 349, 71, 0, 0, 0, 0, 0, 3486, 435, 199, 959, 345, 229, 1222, 1421, 393, 0, 0, 0, 0, 0, 0, 0, 1697, 0, 607, 77, 275, 2331, 1685, 1999, 0, 0, 0, 0, 0, 0, 0, 0, 230, 560, 1140, 136, 74, 442, 3433, 246, 1030, 809, 0, 0, 0, 0, 0, 0, 64, 1359, 2820, 2344, 318, 637, 545, 127, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 7687, 8384, 7658, 8689, 8671, 8100, 8214] avg=8192.0000 max=8689 max_rank=4 min=7658 min_rank=3 imbalance_ratio=1.060669 max_load_reduction=56847 max_load_reduction_pct=0.867416 moved_to_echo=33674 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[43, 943, 2314, 45, 1700, 23, 889, 28, 1751, 293, 0, 0, 0, 0, 0, 0, 1084, 140, 1303, 640, 498, 301, 50, 385, 3107, 0, 0, 0, 0, 0, 0, 0, 209, 1947, 1371, 1477, 0, 879, 543, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2819, 158, 951, 647, 161, 721, 1955, 794, 345, 65, 0, 0, 0, 0, 0, 3292, 297, 274, 1114, 942, 168, 848, 1608, 389, 0, 0, 0, 0, 0, 0, 0, 1653, 0, 808, 237, 312, 2442, 1706, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 167, 816, 769, 159, 103, 459, 3104, 135, 1151, 688, 0, 0, 0, 0, 0, 0, 27, 1622, 2378, 1835, 367, 934, 925, 178, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8029, 7508, 8005, 8622, 8932, 8623, 7551, 8266] avg=8192.0000 max=8932 max_rank=4 min=7508 min_rank=1 imbalance_ratio=1.090332 max_load_reduction=56604 max_load_reduction_pct=0.863708 moved_to_echo=33372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 7936, 18176, 512, 14336, 128, 7296, 256, 14720, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[12928, 0, 6784, 1920, 1920, 18304, 12416, 11520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[27008, 2048, 1280, 6784, 5376, 1152, 8704, 10752, 3072, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1920, 15360, 10624, 11264, 0, 8960, 4992, 12800, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1408, 5504, 8576, 1280, 1152, 4096, 26624, 1792, 9728, 6144, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[384, 13824, 20992, 16128, 2816, 6272, 4480, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 26112, 1408, 7424, 3456, 1024, 5760, 11520, 6144, 2688, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12672, 768, 9600, 5120, 5376, 2816, 256, 3456, 25984, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4094, 3623, 912, 3757, 0, 4008, 943, 4095, 1049, 0, 0, 0, 4096, 2152, 4095, 0, 3889, 2870, 3833, 1, 0, 5, 0, 4003, 4, 20, 0, 15, 3428, 0, 0, 1, 0, 61, 19, 3, 2, 326, 0, 1362, 1, 0, 0, 3, 4096, 2, 4091, 0, 0, 0, 16, 0, 2, 689, 3970, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 0, 3, 0, 4086, 3975, 2952, 4093, 0, 3862, 292, 3548, 2901, 0, 0, 3, 3956, 3104, 4096, 0, 4041, 684, 3196, 0, 0, 0, 0, 4073, 0, 0, 14, 0, 3959, 1, 0, 0, 0, 18, 383, 6, 4, 196, 0, 652, 0, 0, 0, 0, 4084, 0, 3791, 0, 0, 0, 1, 0, 85, 194, 3275, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 1, 2, 0, 3926, 484, 3535, 2348, 1, 4091, 293, 4092, 3489, 0, 1, 80, 4089, 2860, 4056, 0, 2370, 159, 3590, 2, 0, 2, 0, 3055, 0, 5, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 2, 179, 1325, 0, 2467, 0, 0, 9, 3, 4075, 0, 4092, 2, 4, 0, 4, 0, 403, 3028, 3315, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 1, 69, 0, 4003, 1019, 2971, 4057, 1, 3877, 212, 4084, 3072, 0, 0, 13, 3955, 3492, 4093, 26, 4028, 910, 3484, 12, 0, 1, 0, 3886, 0, 9, 0, 1, 4019, 8, 0, 2, 0, 81, 46, 0, 13, 335, 3, 424, 100, 0, 3, 0, 3924, 0, 3989, 8, 1, 0, 5, 0, 166, 3804, 1324, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4092, 3143, 3610, 4087, 0, 4095, 22, 4096, 1827, 0, 0, 1, 4092, 2878, 4096, 0, 4077, 0, 3148, 0, 0, 6, 0, 4054, 1, 4, 1, 1, 4096, 0, 0, 0, 0, 250, 8, 0, 0, 40, 0, 1688, 0, 0, 6, 0, 4095, 0, 3972, 11, 0, 0, 1, 0, 47, 2087, 1880, 0, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3, 0, 0, 1, 0, 8, 0, 4092, 1933, 2271, 4005, 0, 4062, 468, 4084, 680, 1, 0, 0, 4070, 3517, 4094, 0, 3571, 444, 3789, 0, 0, 1, 0, 4047, 1, 24, 0, 0, 4089, 7, 4, 0, 0, 18, 4, 0, 21, 1421, 0, 2796, 5, 0, 3, 4, 4083, 0, 4000, 0, 0, 0, 24, 6, 247, 2730, 906, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 4095, 1294, 2973, 4016, 7, 4096, 71, 4096, 2941, 0, 0, 3, 4090, 1511, 4092, 0, 3980, 6, 1731, 0, 0, 0, 0, 4091, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 4, 2, 0, 16, 292, 0, 3351, 0, 0, 15, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 244, 3513, 2704, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[57, 0, 1, 0, 0, 0, 1, 4056, 3359, 1583, 4088, 0, 3991, 2, 4064, 1915, 0, 16, 1, 3901, 3842, 4053, 18, 4058, 70, 3199, 0, 3, 28, 0, 3715, 28, 300, 0, 0, 4080, 0, 1, 27, 0, 134, 18, 141, 47, 48, 16, 1077, 163, 21, 6, 2, 3355, 0, 4026, 1, 5, 21, 0, 0, 104, 2848, 3046, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4094, 2216, 1875, 0, 0, 0, 0, 0, 0, 3623, 912, 706, 0, 0, 943, 0, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1526, 2152, 0, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 3833, 1, 0, 5, 0, 4003, 4, 432, 0, 0, 0, 0, 0, 0, 0, 20, 0, 15, 3428, 0, 0, 1, 0, 2403, 1792, 0, 0, 0, 0, 0, 0, 61, 19, 3, 2, 326, 0, 1362, 1, 4095, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 2, 4091, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 2, 689, 3970, 0, 0, 1940, 1176, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 7233, 7567, 11148, 7659, 7561, 8390, 7793] avg=8192.0000 max=11148 max_rank=3 min=7233 min_rank=1 imbalance_ratio=1.360840 max_load_reduction=54388 max_load_reduction_pct=0.829895 moved_to_echo=31403 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 1, 2, 0, 3926, 2261, 1172, 0, 0, 0, 0, 0, 0, 484, 3535, 441, 1, 0, 293, 0, 3489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 80, 1523, 2860, 0, 0, 2370, 0, 0, 0, 0, 0, 0, 0, 0, 159, 3590, 2, 0, 2, 0, 3055, 0, 431, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4094, 0, 0, 1, 0, 2400, 1830, 0, 0, 0, 0, 0, 0, 0, 0, 2, 179, 1325, 0, 2467, 0, 4056, 1692, 0, 0, 0, 0, 0, 0, 0, 9, 3, 4075, 0, 4092, 2, 4, 198, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 403, 3028, 3315, 0, 0, 1937, 735, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7364, 8243, 6834, 7239, 8330, 9721, 8383, 9422] avg=8192.0000 max=9721 max_rank=5 min=6834 min_rank=2 imbalance_ratio=1.186646 max_load_reduction=55815 max_load_reduction_pct=0.851669 moved_to_echo=35856 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 0, 3, 0, 4086, 2134, 2043, 0, 0, 0, 0, 0, 0, 3975, 2952, 768, 0, 0, 292, 0, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1474, 3104, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 684, 3196, 0, 0, 0, 0, 4073, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3959, 1, 0, 0, 0, 2081, 1728, 0, 0, 0, 0, 0, 0, 18, 383, 6, 4, 196, 0, 652, 0, 4096, 1467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 3791, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 85, 194, 3275, 0, 4, 1874, 1282, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 10888, 8622, 8370, 7783, 6822, 8066, 6715] avg=8192.0000 max=10888 max_rank=1 min=6715 min_rank=7 imbalance_ratio=1.329102 max_load_reduction=54648 max_load_reduction_pct=0.833862 moved_to_echo=29386 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4092, 2263, 2040, 0, 0, 0, 0, 0, 0, 3143, 3610, 767, 0, 0, 22, 0, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1525, 2878, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3148, 0, 0, 6, 0, 4054, 1, 431, 0, 0, 0, 0, 0, 0, 0, 4, 1, 1, 4096, 0, 0, 0, 0, 2402, 1832, 0, 0, 0, 0, 0, 0, 250, 8, 0, 0, 40, 0, 1688, 0, 4096, 1694, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4095, 0, 3972, 11, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 47, 2087, 1880, 0, 23, 1938, 1280, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8396, 9369, 8481, 7640, 8336, 7776, 8282, 7256] avg=8192.0000 max=9369 max_rank=1 min=7256 min_rank=7 imbalance_ratio=1.143677 max_load_reduction=56167 max_load_reduction_pct=0.857040 moved_to_echo=31650 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 4095, 2264, 2004, 0, 0, 0, 0, 0, 0, 1294, 2973, 755, 7, 0, 71, 0, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1524, 1511, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1731, 0, 0, 0, 0, 4091, 0, 431, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 2403, 1832, 0, 0, 0, 0, 0, 0, 4, 2, 0, 16, 292, 0, 3351, 0, 4092, 1693, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4096, 0, 4094, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 3513, 2704, 0, 0, 1937, 1257, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8364, 8041, 7018, 6259, 8346, 9450, 8403, 9655] avg=8192.0000 max=9655 max_rank=7 min=6259 min_rank=3 imbalance_ratio=1.178589 max_load_reduction=55881 max_load_reduction_pct=0.852676 moved_to_echo=35854 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 1, 69, 0, 4003, 2144, 2025, 0, 0, 0, 0, 0, 0, 1019, 2971, 762, 1, 0, 212, 0, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1474, 3492, 0, 26, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 910, 3484, 12, 0, 1, 0, 3886, 0, 417, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1, 4019, 8, 0, 2, 0, 2396, 1733, 0, 0, 0, 0, 0, 0, 81, 46, 0, 13, 335, 3, 424, 100, 4093, 1688, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3924, 0, 3989, 8, 1, 191, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 166, 3804, 1324, 2, 0, 1873, 1270, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8037, 9033, 8710, 8168, 6783, 8116, 8444] avg=8192.0000 max=9033 max_rank=2 min=6783 min_rank=5 imbalance_ratio=1.102661 max_load_reduction=56503 max_load_reduction_pct=0.862167 moved_to_echo=31511 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 1, 0, 8, 0, 4092, 2245, 1999, 0, 0, 0, 0, 0, 0, 1933, 2271, 752, 0, 0, 468, 0, 680, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1516, 3517, 0, 0, 3571, 0, 0, 0, 0, 0, 0, 0, 0, 444, 3789, 0, 0, 1, 0, 4047, 1, 429, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4089, 7, 4, 0, 0, 2396, 1817, 0, 0, 0, 0, 0, 0, 18, 4, 0, 21, 1421, 0, 2796, 5, 4094, 1688, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4083, 0, 4000, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 24, 6, 247, 2730, 906, 0, 2, 1928, 1254, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8348, 6104, 8605, 8711, 8337, 10047, 8287, 7097] avg=8192.0000 max=10047 max_rank=5 min=6104 min_rank=1 imbalance_ratio=1.226440 max_load_reduction=55489 max_load_reduction_pct=0.846695 moved_to_echo=33768 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[57, 0, 1, 0, 0, 0, 1, 4056, 2207, 2042, 0, 0, 0, 0, 0, 0, 3359, 1583, 762, 0, 0, 2, 0, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 1441, 3842, 0, 18, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 70, 3199, 0, 3, 28, 0, 3715, 28, 416, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 4080, 0, 1, 27, 0, 2385, 1784, 0, 0, 0, 0, 0, 0, 134, 18, 141, 47, 48, 16, 1077, 163, 4053, 1679, 0, 0, 0, 0, 0, 0, 21, 6, 2, 3355, 0, 4026, 1, 5, 193, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 104, 2848, 3046, 0, 0, 1851, 1284, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8364, 7621, 9376, 7459, 8577, 7376, 7609, 9154] avg=8192.0000 max=9376 max_rank=2 min=7376 min_rank=5 imbalance_ratio=1.144531 max_load_reduction=56160 max_load_reduction_pct=0.856934 moved_to_echo=32716 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 12032, 23424, 0, 128, 30080, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5248, 25984, 128, 128, 128, 0, 30976, 128, 3456, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 512, 256, 384, 4096, 128, 13824, 384, 32768, 13312, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 1408, 18944, 20480, 128, 128, 15360, 9600, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[18944, 20864, 5760, 128, 0, 2304, 0, 17920, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 31872, 128, 32128, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 32512, 17792, 15232, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 128, 128, 31872, 128, 128, 128, 0, 18944, 14464, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 30, 0, 554, 15, 0, 4094, 0, 0, 416, 3969, 4096, 0, 0, 4087, 2, 0, 4096, 0, 0, 0, 4096, 4, 1, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 2182, 291, 0, 1, 4072, 2496, 3, 0, 0, 0, 0, 0, 421, 3542, 4096, 0, 0, 1727, 0, 0, 4095, 9, 1926, 0, 3145, 0, 0, 3871] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2, 0, 0, 1, 0, 4095, 0, 0, 1769, 3388, 4096, 2, 5, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4045, 0, 0, 4082, 0, 0, 0, 3574, 3268, 0, 0, 4092, 4074, 286, 0, 0, 0, 0, 0, 170, 3090, 4096, 0, 0, 207, 0, 0, 4096, 0, 8, 0, 704, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 16, 3976, 0, 4, 1558, 4096, 4096, 0, 2, 4095, 0, 0, 4094, 0, 0, 0, 4096, 0, 12, 0, 0, 4091, 34, 3, 4096, 0, 0, 0, 4050, 553, 0, 0, 3684, 3209, 110, 0, 0, 0, 0, 1, 0, 2399, 4096, 0, 0, 3923, 0, 0, 4096, 0, 338, 0, 727, 2, 1, 4078] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 20, 0, 0, 1, 0, 7, 2, 0, 4093, 0, 0, 153, 3936, 4096, 4, 41, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 209, 0, 0, 4096, 0, 0, 4087, 0, 0, 13, 3649, 3466, 0, 0, 4096, 3035, 2, 0, 0, 15, 0, 15, 291, 3894, 4096, 0, 0, 397, 0, 0, 4096, 0, 17, 0, 1325, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 0, 3, 0, 0, 4, 8, 0, 4077, 2, 2, 2108, 4054, 4092, 0, 0, 4090, 0, 0, 4094, 5, 0, 1, 4084, 0, 56, 28, 0, 3932, 0, 0, 4046, 1, 3, 61, 2051, 1755, 0, 1, 3841, 3935, 0, 2, 0, 108, 1, 15, 70, 1771, 4093, 0, 0, 1954, 0, 0, 4054, 0, 1253, 0, 1794, 0, 0, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[2, 5, 2, 0, 0, 1, 0, 0, 0, 4096, 0, 14, 16, 4093, 4089, 1, 10, 4095, 0, 1, 4093, 1, 0, 5, 4094, 0, 219, 0, 0, 4095, 1, 0, 4094, 0, 0, 0, 1408, 3918, 0, 0, 4096, 3901, 1, 0, 0, 0, 0, 0, 135, 2716, 4096, 0, 2, 331, 0, 2, 4094, 0, 11, 0, 3707, 0, 1, 4090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 36, 2, 7, 6, 0, 0, 3, 38, 4096, 0, 12, 160, 4067, 4076, 0, 0, 4039, 0, 0, 4093, 0, 0, 0, 4095, 0, 566, 5, 0, 4086, 2, 0, 4096, 0, 1, 0, 3662, 86, 0, 0, 3362, 4065, 7, 0, 0, 0, 1, 0, 1024, 23, 4096, 0, 0, 3603, 0, 0, 4095, 2, 199, 0, 4088, 0, 0, 3737] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 720, 0, 3, 3, 3, 0, 46, 0, 4088, 0, 2, 606, 3967, 4093, 13, 4, 4077, 16, 8, 4089, 0, 0, 32, 4063, 0, 36, 0, 5, 4068, 37, 0, 4042, 103, 0, 0, 1922, 1730, 4, 4, 4084, 3978, 1, 43, 6, 34, 2, 6, 120, 3223, 4060, 3, 21, 286, 33, 0, 4086, 39, 124, 22, 3490, 0, 0, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 30, 0, 554, 15, 4096, 2785, 696, 440, 0, 0, 0, 0, 0, 3398, 0, 0, 416, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 4, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3487, 0, 1, 0, 2182, 291, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 2496, 3, 0, 0, 0, 0, 0, 321, 169, 114, 2, 0, 0, 0, 0, 421, 3542, 3775, 0, 0, 1727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1310, 9, 1926, 0, 3145, 0, 0, 3871, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8622, 7783, 8183, 8083, 5962, 7177, 9465, 10261] avg=8192.0000 max=10261 max_rank=7 min=5962 min_rank=4 imbalance_ratio=1.252563 max_load_reduction=55275 max_load_reduction_pct=0.843430 moved_to_echo=32865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 5, 2, 0, 0, 1, 0, 0, 4089, 2784, 696, 440, 0, 0, 0, 0, 0, 3400, 0, 14, 16, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4095, 0, 1, 4091, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 219, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 0, 0, 0, 1408, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3901, 1, 0, 0, 0, 0, 0, 321, 169, 113, 2, 0, 0, 0, 0, 135, 2716, 3775, 0, 2, 331, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1310, 0, 11, 0, 3707, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8019, 7524, 8203, 8296, 8811, 8603, 6961, 9119] avg=8192.0000 max=9119 max_rank=7 min=6961 min_rank=6 imbalance_ratio=1.113159 max_load_reduction=56417 max_load_reduction_pct=0.860855 moved_to_echo=33494 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 2786, 675, 440, 0, 0, 0, 0, 16, 3301, 0, 4, 1558, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 12, 0, 0, 4091, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3487, 0, 0, 0, 4050, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3684, 3209, 110, 0, 0, 0, 0, 1, 321, 169, 114, 2, 0, 0, 0, 0, 0, 2399, 3775, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1310, 0, 338, 0, 727, 2, 1, 4078, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7997, 8975, 8189, 8122, 8090, 7610, 10097, 6456] avg=8192.0000 max=10097 max_rank=6 min=6456 min_rank=7 imbalance_ratio=1.232544 max_load_reduction=55439 max_load_reduction_pct=0.845932 moved_to_echo=32253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2, 0, 0, 1, 4096, 2786, 696, 438, 0, 0, 0, 0, 0, 3399, 0, 0, 1769, 3388, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3475, 0, 0, 0, 3574, 3268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4074, 286, 0, 0, 0, 0, 0, 321, 169, 114, 2, 0, 0, 0, 0, 170, 3090, 3775, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1310, 0, 8, 0, 704, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8021, 8558, 8195, 8027, 10317, 9058, 7242, 6118] avg=8192.0000 max=10317 max_rank=4 min=6118 min_rank=7 imbalance_ratio=1.259399 max_load_reduction=55219 max_load_reduction_pct=0.842575 moved_to_echo=32735 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 20, 0, 0, 1, 0, 7, 2, 4096, 2786, 695, 439, 0, 0, 0, 0, 0, 3398, 0, 0, 153, 3936, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 41, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 209, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3479, 0, 0, 13, 3649, 3466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3035, 2, 0, 0, 15, 0, 15, 321, 169, 114, 2, 0, 0, 0, 0, 291, 3894, 3775, 0, 0, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1310, 0, 17, 0, 1325, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 7491, 8231, 8287, 10607, 7769, 8357, 6748] avg=8192.0000 max=10607 max_rank=4 min=6748 min_rank=7 imbalance_ratio=1.294800 max_load_reduction=54929 max_load_reduction_pct=0.838150 moved_to_echo=33481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 3, 0, 0, 4, 8, 4092, 2757, 693, 434, 0, 0, 0, 0, 0, 3384, 2, 2, 2108, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4092, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 56, 28, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3445, 1, 3, 61, 2051, 1755, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 3935, 0, 2, 0, 108, 1, 15, 321, 167, 113, 2, 0, 0, 0, 0, 70, 1771, 3772, 0, 0, 1954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1297, 0, 1253, 0, 1794, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7993, 9550, 8188, 7987, 7317, 8505, 7567, 8429] avg=8192.0000 max=9550 max_rank=1 min=7317 min_rank=4 imbalance_ratio=1.165771 max_load_reduction=55986 max_load_reduction_pct=0.854279 moved_to_echo=31818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 36, 2, 7, 6, 0, 0, 3, 4076, 2785, 696, 440, 0, 0, 0, 0, 38, 3400, 0, 12, 160, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 566, 5, 0, 4086, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3487, 0, 1, 0, 3662, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3362, 4065, 7, 0, 0, 0, 1, 0, 321, 169, 114, 2, 0, 0, 0, 0, 1024, 23, 3775, 0, 0, 3603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1310, 2, 199, 0, 4088, 0, 0, 3737, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8051, 7677, 8130, 8640, 7236, 8041, 8425, 9336] avg=8192.0000 max=9336 max_rank=7 min=7236 min_rank=4 imbalance_ratio=1.139648 max_load_reduction=56200 max_load_reduction_pct=0.857544 moved_to_echo=33038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 720, 0, 3, 3, 3, 0, 46, 4093, 2783, 698, 438, 0, 0, 0, 0, 0, 3390, 0, 2, 606, 3967, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4077, 16, 8, 4080, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 36, 0, 5, 4068, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3433, 103, 0, 0, 1922, 1730, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 3978, 1, 43, 6, 34, 2, 6, 324, 171, 115, 9, 0, 0, 0, 0, 120, 3223, 3736, 3, 21, 286, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1303, 39, 124, 22, 3490, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8787, 7978, 8217, 8094, 7196, 8773, 7422, 9069] avg=8192.0000 max=9069 max_rank=7 min=7196 min_rank=4 imbalance_ratio=1.107056 max_load_reduction=56467 max_load_reduction_pct=0.861618 moved_to_echo=32460 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[10496, 128, 3968, 128, 19072, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[31360, 28800, 512, 128, 128, 256, 128, 128, 2688, 1408, 1024, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 27136, 128, 128, 6912, 31616, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31872, 128, 1152, 128, 128, 32512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[27904, 128, 128, 128, 22528, 15104, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 896, 128, 128, 128, 128, 640, 128, 32768, 22272, 5632, 3584, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2304, 20736, 30208, 128, 128, 12544, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66688 slack_slots=3584 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4078, 4082, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 1, 0, 0, 4092, 0, 0, 4096, 6, 18, 4094, 4063, 460, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 3135, 0, 4, 0, 0, 1, 0, 0, 4050, 0, 4093, 0, 0, 12, 372, 0, 0, 0, 0, 199, 0, 0, 0, 1, 4096, 24, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4072, 3879, 0, 0, 0, 4096, 0, 1, 0, 4085, 0, 297, 0, 0, 4095, 0, 0, 3075, 0, 3008, 4096, 4096, 1767, 0, 4096, 3, 0, 1, 0, 0, 4096, 0, 0, 2161, 2106, 0, 39, 0, 0, 1, 0, 0, 4089, 0, 4095, 0, 2, 0, 34, 0, 0, 0, 0, 47, 7, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4093, 4093, 0, 0, 0, 4094, 2, 0, 0, 4096, 1, 6, 0, 0, 4086, 0, 10, 2124, 1, 54, 4095, 4094, 10, 0, 4096, 0, 1, 0, 0, 0, 4091, 0, 1, 4090, 2963, 0, 46, 1, 0, 2, 0, 0, 4096, 0, 4096, 52, 0, 1, 2937, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4095, 5, 0, 4094, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4096, 4075, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2645, 0, 1683, 4096, 4096, 3, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3659, 0, 1, 0, 0, 0, 0, 0, 4069, 0, 4095, 4, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 3967, 0, 569, 4096, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3226, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4096, 3949, 0, 46, 0, 4090, 0, 5, 0, 4095, 1, 37, 0, 0, 4089, 1, 25, 4091, 32, 296, 4096, 4010, 0, 0, 4095, 0, 7, 1, 0, 0, 4087, 0, 13, 3957, 3073, 0, 0, 0, 10, 0, 0, 0, 4067, 9, 4096, 635, 0, 1, 142, 13, 0, 0, 0, 169, 1, 10, 0, 1, 4094, 0, 1, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4081, 3139, 0, 1, 0, 4096, 0, 0, 4, 4087, 0, 32, 0, 0, 4056, 0, 0, 2929, 0, 184, 4092, 4096, 207, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 2424, 18, 13, 0, 0, 11, 0, 0, 3655, 22, 4069, 0, 9, 2, 3849, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4059, 3758, 1, 10, 5, 4073, 43, 12, 43, 4050, 17, 0, 26, 140, 3912, 10, 0, 2630, 0, 161, 4057, 4066, 0, 0, 4083, 1, 1, 1, 4, 1, 4079, 2, 0, 4045, 3272, 0, 27, 66, 0, 0, 0, 18, 3264, 0, 4073, 0, 0, 11, 3005, 15, 4, 0, 30, 307, 0, 0, 8, 0, 4019, 0, 12, 4079, 13, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4078, 4082, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4059, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 18, 0, 3923, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3135, 0, 4, 0, 0, 1, 1261, 91, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 4093, 0, 0, 12, 49, 32, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 199, 0, 0, 4094, 2625, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 24, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8370, 8154, 8503, 8197, 8569, 8236, 7290, 8217] avg=8192.0000 max=8569 max_rank=4 min=7290 min_rank=6 imbalance_ratio=1.046021 max_load_reduction=56967 max_load_reduction_pct=0.869247 moved_to_echo=32312 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4093, 4093, 0, 0, 0, 210, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 1, 6, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2124, 1, 54, 0, 3953, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4091, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 2963, 0, 46, 1, 0, 2, 1260, 91, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 52, 0, 1, 50, 32, 0, 0, 0, 0, 0, 0, 2937, 0, 0, 0, 0, 7, 0, 0, 4095, 2624, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8398, 8152, 6152, 8193, 8454, 8327, 9663, 8197] avg=8192.0000 max=9663 max_rank=6 min=6152 min_rank=2 imbalance_ratio=1.179565 max_load_reduction=55873 max_load_reduction_pct=0.852554 moved_to_echo=34641 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4072, 3879, 0, 0, 0, 210, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 297, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3075, 0, 3008, 0, 3955, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 1, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2161, 2106, 0, 39, 0, 0, 1, 1261, 91, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4095, 0, 2, 0, 50, 32, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 47, 7, 0, 4096, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8440, 11805, 8201, 5659, 8268, 6809, 8192] avg=8192.0000 max=11805 max_rank=2 min=5659 min_rank=4 imbalance_ratio=1.441040 max_load_reduction=53731 max_load_reduction_pct=0.819870 moved_to_echo=28928 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 4095, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 569, 0, 3955, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3226, 0, 1, 0, 0, 0, 1261, 91, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 50, 32, 0, 0, 0, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 4096, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8401, 8154, 8492, 8197, 8675, 8274, 7150, 8193] avg=8192.0000 max=8675 max_rank=4 min=7150 min_rank=6 imbalance_ratio=1.058960 max_load_reduction=56861 max_load_reduction_pct=0.867630 moved_to_echo=32292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 4075, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2645, 0, 1683, 0, 3955, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3659, 0, 1, 0, 0, 0, 1261, 91, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4095, 4, 0, 0, 50, 32, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 4096, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8381, 8155, 8286, 8197, 9108, 8250, 6967, 8192] avg=8192.0000 max=9108 max_rank=4 min=6967 min_rank=6 imbalance_ratio=1.111816 max_load_reduction=56428 max_load_reduction_pct=0.861023 moved_to_echo=32517 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 3949, 0, 46, 0, 210, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 1, 37, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4091, 32, 296, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 7, 1, 0, 0, 4087, 0, 5, 0, 0, 0, 0, 0, 0, 0, 13, 3957, 3073, 0, 0, 0, 10, 0, 1259, 89, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 9, 4096, 635, 0, 1, 49, 32, 0, 0, 0, 0, 0, 0, 142, 13, 0, 0, 0, 169, 1, 10, 4096, 2621, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 8186, 8316, 8195, 8401, 8889, 7052, 8191] avg=8192.0000 max=8889 max_rank=5 min=7052 min_rank=6 imbalance_ratio=1.085083 max_load_reduction=56647 max_load_reduction_pct=0.864365 moved_to_echo=32533 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4081, 3139, 0, 1, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4050, 0, 32, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2929, 0, 184, 0, 3955, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 2424, 18, 13, 0, 0, 11, 1261, 91, 0, 0, 0, 0, 0, 0, 0, 0, 3655, 22, 4069, 0, 9, 2, 50, 32, 0, 0, 0, 0, 0, 0, 3849, 0, 0, 0, 0, 6, 0, 0, 4092, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7431, 8142, 7275, 8177, 7904, 7839, 10572, 8196] avg=8192.0000 max=10572 max_rank=6 min=7275 min_rank=2 imbalance_ratio=1.290527 max_load_reduction=54964 max_load_reduction_pct=0.838684 moved_to_echo=34511 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4059, 3758, 1, 10, 5, 199, 43, 12, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4005, 17, 0, 26, 140, 3912, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2630, 0, 161, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 1, 1, 4, 1, 4079, 2, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 3272, 0, 27, 66, 0, 0, 1259, 97, 0, 0, 0, 0, 0, 0, 0, 18, 3264, 0, 4073, 0, 0, 11, 53, 34, 0, 0, 0, 0, 0, 0, 3005, 15, 4, 0, 30, 307, 0, 0, 4057, 2615, 0, 0, 0, 0, 0, 0, 8, 0, 4019, 0, 12, 4079, 13, 23, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8153, 6707, 8179, 8766, 7453, 10033, 8158] avg=8192.0000 max=10033 max_rank=6 min=6707 min_rank=2 imbalance_ratio=1.224731 max_load_reduction=55503 max_load_reduction_pct=0.846909 moved_to_echo=34410 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32512, 128, 384, 128, 256, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 25600, 128, 6016, 0, 31488, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11136, 128, 128, 0, 128, 768, 128, 128, 32768, 20992, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 31488, 128, 32768, 768, 128, 128, 512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 31104, 128, 128, 128, 1792, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 30720, 23936, 128, 256, 128, 128, 128, 10112, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4072, 4012, 4, 0, 0, 0, 0, 5, 80, 123, 4093, 0, 0, 4087, 4096, 4089, 0, 10, 18, 0, 3757, 1, 1, 4, 0, 1, 3, 0, 4086, 0, 0, 4092, 4094, 4090, 0, 0, 10, 4096, 0, 2, 260, 2, 283, 0, 3, 3, 1, 5, 0, 0, 40, 0, 0, 0, 4, 4096, 0, 3726, 4079, 9, 4096, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4076, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 103, 3817, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 15, 4096, 4096, 4096, 0, 0, 0, 4096, 1, 0, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4095, 0, 4096, 3282, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 16, 4080, 2732, 0, 0, 0, 0, 0, 0, 7, 1, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 6, 3463, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 967, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1048, 0, 26, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 0, 0, 4096, 0, 3658, 4094, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1397, 3913, 2621, 0, 0, 0, 0, 0, 11, 120, 0, 4096, 0, 0, 4096, 4095, 4087, 0, 0, 31, 0, 4093, 0, 3, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 13, 0, 0, 4065, 24, 2, 3979, 0, 50, 0, 0, 0, 0, 0, 0, 18, 0, 1, 0, 0, 0, 4095, 0, 194, 4088, 0, 4059, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 1050, 4010, 992, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4079, 4096, 0, 0, 0, 0, 3973, 0, 0, 2, 31, 0, 1, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 154, 4096, 0, 0, 4052, 0, 53, 0, 0, 0, 1, 0, 0, 0, 89, 0, 0, 0, 0, 4039, 0, 2654, 3491, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4091, 2542, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 3, 4095, 4094, 4096, 1, 0, 0, 1, 4088, 0, 0, 1, 17, 0, 11, 0, 4094, 0, 6, 4078, 4096, 4096, 1, 0, 0, 4096, 0, 3, 1562, 0, 57, 0, 0, 0, 0, 0, 1, 0, 62, 0, 0, 0, 0, 4042, 0, 4040, 4054, 7, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 4095, 4010, 2, 2, 1, 0, 7, 0, 0, 0, 4096, 0, 0, 4091, 4072, 4094, 0, 2, 0, 8, 4026, 2, 0, 3, 0, 0, 0, 0, 4092, 0, 0, 4094, 4082, 4074, 0, 5, 13, 4089, 0, 0, 1707, 2, 48, 2, 2, 0, 1, 0, 0, 61, 12, 2, 1, 0, 0, 4093, 0, 2846, 3703, 1, 4093, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[5, 134, 4069, 1901, 51, 0, 1, 0, 23, 20, 0, 0, 4070, 31, 0, 4046, 3866, 4068, 24, 27, 5, 0, 4059, 9, 0, 7, 0, 0, 0, 15, 4039, 21, 19, 4082, 4051, 3968, 0, 5, 166, 4086, 33, 0, 2905, 4, 0, 11, 0, 8, 0, 0, 1, 108, 520, 0, 12, 40, 110, 4084, 10, 2856, 4073, 0, 3889, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1397, 3913, 2621, 0, 0, 0, 0, 792, 144, 43, 0, 0, 0, 0, 0, 0, 11, 120, 0, 4053, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 191, 0, 0, 31, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 4096, 0, 2122, 1962, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 4095, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 2, 3979, 0, 50, 0, 0, 0, 4095, 1774, 159, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 0, 0, 4096, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 4088, 0, 1305, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8910, 8280, 8410, 8185, 8060, 10083, 8021, 5587] avg=8192.0000 max=10083 max_rank=5 min=5587 min_rank=7 imbalance_ratio=1.230835 max_load_reduction=55453 max_load_reduction_pct=0.846146 moved_to_echo=31751 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4072, 4012, 4, 0, 0, 0, 799, 144, 43, 0, 0, 0, 0, 0, 0, 5, 80, 123, 4050, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 191, 0, 10, 18, 0, 3757, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 1, 3, 0, 4086, 0, 2123, 1979, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 4090, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 260, 2, 283, 0, 3, 3, 4096, 1775, 160, 0, 0, 0, 0, 0, 1, 5, 0, 0, 40, 0, 0, 0, 4092, 3936, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3726, 4079, 9, 1318, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9074, 8345, 8073, 8197, 8050, 6584, 8074, 9139] avg=8192.0000 max=9139 max_rank=7 min=6584 min_rank=5 imbalance_ratio=1.115601 max_load_reduction=56397 max_load_reduction_pct=0.860550 moved_to_echo=31847 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4095, 4010, 2, 2, 1, 0, 798, 144, 43, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4053, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 191, 0, 2, 0, 8, 4026, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4092, 0, 2126, 1978, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 4074, 0, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 2, 48, 2, 2, 0, 4093, 1777, 159, 0, 0, 0, 0, 0, 1, 0, 0, 61, 12, 2, 1, 0, 4094, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2846, 3703, 1, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9097, 8151, 8301, 8199, 8030, 7790, 8101, 7867] avg=8192.0000 max=9097 max_rank=0 min=7790 min_rank=5 imbalance_ratio=1.110474 max_load_reduction=56439 max_load_reduction_pct=0.861191 moved_to_echo=31788 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1050, 4010, 992, 0, 0, 0, 0, 799, 144, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 191, 0, 0, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 31, 0, 1, 0, 4096, 0, 2127, 1979, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 4096, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 53, 0, 0, 0, 4039, 1778, 160, 0, 0, 0, 0, 0, 1, 0, 0, 0, 89, 0, 0, 0, 4096, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2654, 3491, 0, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7038, 8150, 8243, 8236, 8202, 10082, 8122, 7463] avg=8192.0000 max=10082 max_rank=5 min=7038 min_rank=0 imbalance_ratio=1.230713 max_load_reduction=55454 max_load_reduction_pct=0.846161 moved_to_echo=33869 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 4080, 2732, 0, 0, 0, 0, 799, 144, 43, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4053, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 191, 0, 0, 0, 6, 3463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2127, 1979, 0, 0, 0, 0, 0, 0, 967, 0, 3952, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 26, 0, 0, 0, 4096, 1778, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 0, 4096, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 4094, 0, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7814, 8157, 7756, 8202, 9015, 7107, 8415, 9070] avg=8192.0000 max=9070 max_rank=7 min=7107 min_rank=5 imbalance_ratio=1.107178 max_load_reduction=56466 max_load_reduction_pct=0.861603 moved_to_echo=33607 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4076, 3805, 0, 0, 0, 0, 799, 144, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 191, 0, 0, 0, 103, 3817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 2127, 1979, 0, 0, 0, 0, 0, 0, 15, 0, 3952, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1279, 0, 0, 0, 0, 0, 4095, 1778, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3282, 0, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8867, 8149, 8207, 8204, 8063, 7313, 8037, 8696] avg=8192.0000 max=8867 max_rank=0 min=7313 min_rank=5 imbalance_ratio=1.082397 max_load_reduction=56669 max_load_reduction_pct=0.864700 moved_to_echo=32109 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 2542, 0, 0, 0, 0, 799, 144, 43, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4053, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 191, 1, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 0, 11, 0, 4094, 0, 2127, 1979, 0, 0, 0, 0, 0, 0, 6, 0, 3952, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1562, 0, 57, 0, 0, 0, 4042, 1778, 160, 0, 0, 0, 0, 0, 0, 0, 1, 0, 62, 0, 0, 0, 4078, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 4054, 7, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7619, 8160, 8375, 8229, 8055, 7602, 8077, 9419] avg=8192.0000 max=9419 max_rank=7 min=7602 min_rank=5 imbalance_ratio=1.149780 max_load_reduction=56117 max_load_reduction_pct=0.856277 moved_to_echo=33153 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 134, 4069, 1901, 51, 0, 1, 0, 762, 148, 46, 0, 0, 0, 0, 0, 23, 20, 0, 0, 4024, 31, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 181, 24, 27, 5, 0, 4059, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 15, 4039, 21, 2117, 1885, 0, 0, 0, 0, 0, 0, 19, 0, 3903, 3968, 0, 5, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 2905, 4, 0, 11, 0, 8, 4084, 1770, 160, 0, 0, 0, 0, 0, 0, 0, 1, 108, 520, 0, 12, 40, 4082, 3926, 0, 0, 0, 0, 0, 0, 110, 0, 10, 2856, 4073, 0, 1242, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7117, 8144, 8171, 8084, 8061, 8975, 8689, 8295] avg=8192.0000 max=8975 max_rank=5 min=7117 min_rank=0 imbalance_ratio=1.095581 max_load_reduction=56561 max_load_reduction_pct=0.863052 moved_to_echo=34020 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32512, 1536, 128, 128, 128, 128, 31360, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 24192, 30976, 128, 10496, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1024, 0, 31616, 32640, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 256, 1152, 128, 128, 128, 32768, 31488, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 16896, 128, 640, 128, 128, 128, 32640, 14208, 1280, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 2688, 32512, 22656, 128, 128, 128, 0, 6400, 1280, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 256, 256, 32512, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 128, 17024, 15744, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4063, 0, 0, 0, 0, 1, 0, 3673, 3620, 1, 4091, 4096, 0, 0, 1, 0, 0, 0, 99, 4096, 0, 0, 0, 2464, 0, 0, 4096, 4081, 0, 4035, 0, 0, 2298, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4095, 0, 4095, 4091, 0, 220, 0, 0, 0, 0, 0, 32, 4095, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 207, 0, 25, 0, 3958, 21, 756, 4047, 4096, 1605, 0, 93, 0, 0, 0, 132, 4086, 0, 0, 5, 4092, 0, 0, 4096, 4096, 0, 4096, 1, 0, 3904, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 4051, 4081, 0, 4, 0, 0, 0, 30, 0, 0, 1630, 38, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 3196, 0, 5, 758, 0, 3, 0, 3950, 2686, 15, 4080, 4093, 10, 0, 29, 2, 6, 0, 0, 4093, 1, 1, 0, 4061, 0, 1, 4096, 4072, 0, 4093, 19, 1, 1695, 51, 0, 2, 0, 0, 0, 197, 1, 4094, 0, 0, 4095, 1, 1, 4096, 0, 4094, 3841, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4023, 0, 0, 478, 0, 1, 0, 3757, 516, 1, 4085, 4096, 52, 0, 1, 0, 0, 0, 35, 4096, 0, 0, 0, 3581, 0, 14, 4096, 4094, 0, 3868, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4095, 4078, 0, 5, 0, 0, 0, 0, 0, 83, 4095, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4069, 0, 5, 4, 0, 9, 0, 4089, 3293, 26, 4090, 4089, 5, 0, 0, 0, 0, 3, 25, 4095, 6, 0, 0, 3397, 1, 0, 4079, 3904, 0, 4083, 0, 1, 1498, 0, 2, 0, 0, 8, 0, 16, 0, 4094, 0, 0, 4094, 1, 0, 4053, 1, 4074, 4091, 0, 8, 1, 0, 0, 6, 0, 305, 4007, 2, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 854, 0, 0, 0, 3718, 40, 0, 4095, 4096, 21, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 3967, 0, 313, 4096, 4095, 0, 4093, 0, 0, 3243, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 145, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 3985, 1, 0, 438, 0, 0, 0, 3777, 885, 203, 4093, 4095, 154, 10, 0, 0, 0, 0, 66, 4095, 0, 0, 0, 4095, 0, 0, 4094, 4096, 0, 3972, 0, 0, 3083, 4, 0, 13, 0, 0, 0, 5, 0, 4096, 0, 0, 4089, 0, 0, 4090, 0, 4087, 3975, 0, 0, 4, 0, 0, 3, 0, 19, 4007, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[63, 4094, 0, 0, 0, 24, 16, 11, 4056, 90, 338, 4071, 4077, 61, 0, 156, 9, 40, 13, 0, 4053, 44, 0, 0, 4089, 2, 0, 3917, 4084, 0, 4053, 17, 26, 3999, 0, 0, 0, 13, 0, 0, 0, 0, 4082, 2, 3, 4069, 18, 35, 4046, 0, 3742, 4003, 0, 0, 0, 23, 29, 0, 7, 0, 4047, 1, 13, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 207, 0, 25, 0, 2266, 1584, 28, 0, 0, 0, 0, 0, 3958, 21, 756, 2463, 0, 1605, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 4086, 0, 0, 5, 2624, 1435, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 395, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 0, 0, 0, 1, 0, 4096, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 4051, 4081, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 1630, 38, 0, 3, 2981, 1382, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8896, 8282, 8584, 9116, 8164, 8224, 6064] avg=8192.0000 max=9116 max_rank=4 min=6064 min_rank=7 imbalance_ratio=1.112793 max_load_reduction=56420 max_load_reduction_pct=0.860901 moved_to_echo=31568 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4069, 0, 5, 4, 0, 9, 0, 2160, 1601, 28, 0, 0, 0, 0, 0, 4089, 3293, 26, 2489, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 25, 4095, 6, 0, 0, 2598, 1367, 0, 0, 0, 0, 0, 0, 3397, 1, 0, 0, 377, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1498, 0, 2, 0, 0, 8, 0, 4089, 1109, 0, 0, 0, 0, 0, 0, 16, 0, 4066, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 1, 4074, 4091, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 305, 4007, 2, 0, 2, 2970, 1369, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7876, 9902, 8094, 7858, 6707, 8177, 8261, 8661] avg=8192.0000 max=9902 max_rank=1 min=6707 min_rank=4 imbalance_ratio=1.208740 max_load_reduction=55634 max_load_reduction_pct=0.848907 moved_to_echo=31806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 854, 0, 0, 0, 2266, 1603, 28, 0, 0, 0, 0, 0, 3718, 40, 0, 2492, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 2625, 1434, 0, 0, 0, 0, 0, 0, 3967, 0, 313, 0, 395, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3243, 1, 0, 0, 0, 0, 0, 4096, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 4096, 0, 0, 0, 2982, 1383, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8848, 6271, 8155, 8768, 8454, 8164, 8270, 8606] avg=8192.0000 max=8848 max_rank=0 min=6271 min_rank=1 imbalance_ratio=1.080078 max_load_reduction=56688 max_load_reduction_pct=0.864990 moved_to_echo=33494 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4063, 0, 0, 0, 0, 1, 0, 2258, 1601, 28, 0, 0, 0, 0, 0, 3673, 3620, 1, 2490, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 4096, 0, 0, 0, 2625, 1430, 0, 0, 0, 0, 0, 0, 2464, 0, 0, 0, 393, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2298, 0, 0, 0, 0, 0, 0, 4096, 1114, 0, 0, 0, 0, 0, 0, 0, 1, 4068, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 4095, 4091, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4095, 0, 0, 0, 2982, 1383, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7951, 9785, 8250, 6892, 7508, 8165, 8493, 8492] avg=8192.0000 max=9785 max_rank=1 min=6892 min_rank=3 imbalance_ratio=1.194458 max_load_reduction=55751 max_load_reduction_pct=0.850693 moved_to_echo=32658 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4023, 0, 0, 478, 0, 1, 0, 2265, 1599, 28, 0, 0, 0, 0, 0, 3757, 516, 1, 2486, 0, 52, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4096, 0, 0, 0, 2626, 1434, 0, 0, 0, 0, 0, 0, 3581, 0, 14, 0, 395, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 1114, 0, 0, 0, 0, 0, 0, 0, 4, 4068, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 4095, 4078, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 4095, 0, 0, 0, 2982, 1383, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8394, 6813, 8191, 7858, 9304, 8168, 8265, 8543] avg=8192.0000 max=9304 max_rank=4 min=6813 min_rank=1 imbalance_ratio=1.135742 max_load_reduction=56232 max_load_reduction_pct=0.858032 moved_to_echo=34280 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3196, 0, 5, 758, 0, 3, 0, 2253, 1597, 28, 0, 0, 0, 0, 0, 3950, 2686, 15, 2483, 0, 10, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 4093, 1, 1, 0, 2626, 1426, 0, 0, 0, 0, 0, 0, 4061, 0, 1, 0, 393, 0, 4093, 19, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1695, 51, 0, 2, 0, 0, 0, 4093, 1115, 0, 0, 0, 0, 0, 0, 197, 1, 4066, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 4094, 3841, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2981, 1383, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7840, 9173, 8155, 8567, 6957, 8361, 8023, 8460] avg=8192.0000 max=9173 max_rank=1 min=6957 min_rank=4 imbalance_ratio=1.119751 max_load_reduction=56363 max_load_reduction_pct=0.860031 moved_to_echo=31801 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3985, 1, 0, 438, 0, 0, 0, 2266, 1602, 28, 0, 0, 0, 0, 0, 3777, 885, 203, 2491, 0, 154, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 4095, 0, 0, 0, 2622, 1435, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 395, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3083, 4, 0, 13, 0, 0, 0, 4095, 1114, 0, 0, 0, 0, 0, 0, 5, 0, 4068, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 4087, 3975, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 19, 4007, 0, 2, 0, 2980, 1381, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 7520, 8218, 8462, 8309, 8162, 8153, 8392] avg=8192.0000 max=8462 max_rank=3 min=7520 min_rank=1 imbalance_ratio=1.032959 max_load_reduction=57074 max_load_reduction_pct=0.870880 moved_to_echo=33016 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[63, 4094, 0, 0, 0, 24, 16, 11, 2263, 1596, 34, 0, 0, 0, 0, 0, 4056, 90, 338, 2475, 0, 61, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 9, 40, 13, 0, 4053, 44, 0, 0, 2597, 1435, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 386, 0, 4053, 17, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3999, 0, 0, 0, 13, 0, 0, 4077, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 2, 3, 4069, 18, 35, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 3742, 4003, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 7, 0, 4047, 1, 13, 0, 2851, 1370, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 7176, 8191, 8547, 9181, 8175, 7847, 8318] avg=8192.0000 max=9181 max_rank=4 min=7176 min_rank=1 imbalance_ratio=1.120728 max_load_reduction=56355 max_load_reduction_pct=0.859909 moved_to_echo=33521 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31104, 11264, 1408, 19968, 0, 1920, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 640, 30080, 128, 128, 128, 23808, 11136, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 23936, 128, 128, 128, 128, 128, 0, 32768, 8960, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 31744, 128, 128, 2816, 128, 128, 128, 18048, 12800, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 128, 32384, 32256, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[29824, 128, 384, 0, 3200, 0, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 384, 32768, 128, 128, 128, 20992, 11520, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 128, 32640, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4091, 0, 23, 1, 4093, 0, 4, 4074, 3, 0, 0, 0, 6, 0, 1, 0, 4088, 4096, 0, 1, 0, 1, 0, 0, 0, 1738, 0, 3563, 0, 0, 1, 4090, 0, 0, 0, 3, 4092, 0, 4096, 4046, 8, 9, 4096, 4095, 4096, 0, 4096, 0, 7, 0, 5, 0, 7, 2992, 0, 23, 0, 3, 0, 0, 0, 0, 3988, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 4096, 0, 3, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 34, 1, 0, 2161, 0, 6, 1, 0, 0, 4036, 1, 0, 13, 0, 4093, 5, 4074, 4095, 2, 0, 4091, 4052, 4095, 0, 4096, 0, 0, 0, 1, 0, 3933, 2828, 0, 0, 0, 0, 0, 1, 0, 59, 3332, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4088, 37, 0, 3, 4096, 1, 0, 4096, 0, 0, 25, 1, 0, 0, 0, 8, 4091, 4070, 0, 0, 0, 0, 0, 0, 0, 2484, 0, 1009, 8, 4, 0, 4058, 0, 5, 4, 0, 4096, 0, 4094, 4095, 0, 0, 4094, 3855, 4094, 0, 4064, 0, 0, 24, 3, 0, 1331, 3361, 4, 0, 32, 0, 0, 0, 0, 218, 4083, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 604, 0, 39, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4094, 11, 0, 4096, 3639, 4096, 0, 4096, 0, 0, 0, 0, 0, 2482, 4091, 0, 0, 4, 0, 0, 0, 0, 1444, 4073, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 3, 1, 0, 0, 0, 0, 0, 4096, 4095, 0, 8, 0, 0, 0, 3, 0, 3549, 0, 21, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 4096, 4063, 0, 0, 4089, 2833, 4096, 0, 4083, 0, 0, 8, 0, 0, 4077, 2661, 0, 16, 0, 0, 0, 0, 58, 0, 3204, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4092, 5, 1, 0, 4096, 0, 3, 4085, 0, 8, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 1669, 0, 534, 0, 0, 0, 4089, 0, 0, 2, 15, 4096, 0, 4096, 4077, 0, 0, 4095, 4090, 4093, 0, 4096, 0, 0, 0, 0, 10, 1466, 4020, 0, 3, 2, 0, 0, 0, 43, 492, 4065, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 4096, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 126, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 1, 0, 4096, 4031, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4093, 0, 0, 3, 3993, 1, 472, 4050, 13, 0, 1, 0, 0, 27, 0, 0, 4016, 4074, 67, 0, 6, 0, 89, 0, 0, 143, 28, 891, 47, 0, 3, 4068, 0, 20, 0, 0, 4037, 18, 4049, 3957, 161, 61, 4078, 3983, 3954, 0, 4064, 54, 39, 3, 0, 0, 3096, 4070, 0, 184, 0, 1, 0, 6, 0, 0, 3596, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 23, 1, 4093, 0, 4, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6, 0, 1, 0, 4091, 4084, 0, 0, 0, 0, 0, 0, 4088, 4088, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 0, 3563, 0, 0, 1, 4090, 1227, 46, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3, 29, 0, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 0, 4095, 4096, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 5, 0, 7, 2992, 0, 23, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3988, 0, 4063, 21, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8185, 8178, 10673, 8174, 8463, 5627, 8087] avg=8192.0000 max=10673 max_rank=3 min=5627 min_rank=6 imbalance_ratio=1.302856 max_load_reduction=54863 max_load_reduction_pct=0.837143 moved_to_echo=30351 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 3, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 4091, 4080, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2161, 0, 6, 1, 0, 0, 4036, 1227, 46, 8, 0, 0, 0, 0, 0, 1, 0, 13, 0, 29, 5, 4074, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4052, 4095, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3933, 2828, 0, 0, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 59, 3332, 0, 4064, 21, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8219, 8219, 7485, 8217, 8404, 9355, 7488] avg=8192.0000 max=9355 max_rank=6 min=7485 min_rank=3 imbalance_ratio=1.141968 max_load_reduction=56181 max_load_reduction_pct=0.857254 moved_to_echo=33464 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 8, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3549, 0, 21, 0, 0, 0, 4096, 1223, 46, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4096, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2833, 4096, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4077, 2661, 0, 16, 2584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 3204, 0, 4063, 20, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8177, 8194, 8943, 8188, 7185, 9346, 7357] avg=8192.0000 max=9346 max_rank=6 min=7185 min_rank=5 imbalance_ratio=1.140869 max_load_reduction=56190 max_load_reduction_pct=0.857391 moved_to_echo=32076 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 4095, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 604, 0, 39, 0, 0, 0, 4096, 1227, 46, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3639, 4096, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2482, 4091, 0, 0, 2593, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1444, 4073, 0, 4067, 21, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8180, 8183, 6020, 8219, 8001, 9166, 9621] avg=8192.0000 max=9621 max_rank=7 min=6020 min_rank=3 imbalance_ratio=1.174438 max_load_reduction=55915 max_load_reduction_pct=0.853195 moved_to_echo=35007 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 37, 0, 3, 4096, 1, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 1, 0, 0, 0, 8, 4088, 4083, 0, 0, 0, 0, 0, 0, 4091, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2484, 0, 1009, 8, 4, 0, 4058, 1217, 46, 8, 0, 0, 0, 0, 0, 0, 5, 4, 0, 29, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 4094, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 3, 0, 1331, 3361, 4, 0, 2572, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 218, 4083, 0, 4067, 20, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8205, 8153, 8834, 8227, 8204, 7295, 8431] avg=8192.0000 max=8834 max_rank=3 min=7295 min_rank=6 imbalance_ratio=1.078369 max_load_reduction=56702 max_load_reduction_pct=0.865204 moved_to_echo=32157 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 126, 0, 0, 0, 4096, 1227, 46, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4031, 4096, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4067, 21, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8184, 8182, 9547, 8221, 8383, 6688, 8184] avg=8192.0000 max=9547 max_rank=3 min=6688 min_rank=6 imbalance_ratio=1.165405 max_load_reduction=55989 max_load_reduction_pct=0.854324 moved_to_echo=31476 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 1, 0, 4096, 0, 3, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1, 0, 4092, 4084, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1669, 0, 534, 0, 0, 0, 4089, 1227, 46, 8, 0, 0, 0, 0, 0, 0, 0, 2, 15, 29, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4093, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1466, 4020, 0, 3, 2593, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 43, 492, 4065, 0, 4067, 21, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8185, 8184, 7573, 8219, 8438, 8092, 8701] avg=8192.0000 max=8701 max_rank=7 min=7573 min_rank=3 imbalance_ratio=1.062134 max_load_reduction=56835 max_load_reduction_pct=0.867233 moved_to_echo=33450 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 3993, 1, 472, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 0, 27, 0, 0, 4093, 4067, 0, 0, 0, 0, 0, 0, 4016, 4065, 67, 0, 6, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 28, 891, 47, 0, 3, 4068, 1221, 51, 9, 0, 0, 0, 0, 0, 0, 20, 0, 0, 27, 18, 4049, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 161, 61, 0, 3983, 3954, 0, 245, 54, 0, 0, 0, 0, 0, 0, 0, 0, 39, 3, 0, 0, 3096, 4070, 0, 184, 2575, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 3596, 20, 4010, 23, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8468, 8201, 8243, 6461, 8071, 8458, 9967, 7667] avg=8192.0000 max=9967 max_rank=6 min=6461 min_rank=3 imbalance_ratio=1.216675 max_load_reduction=55569 max_load_reduction_pct=0.847916 moved_to_echo=34163 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 16512, 128, 6272, 128, 128, 128, 32640, 9856, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 512, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[256, 128, 0, 30592, 32640, 0, 2048, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 256, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 2304, 30464, 128, 32512, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 16512, 28160, 128, 256, 20736, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4094, 0, 2, 1006, 0, 0, 4096, 0, 0, 4096, 1682, 0, 0, 4040, 0, 0, 0, 4096, 4096, 0, 2, 169, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 1622, 1, 0, 4095, 1, 0, 3786, 11, 4096, 0, 0, 4096, 3954, 1, 2, 4095, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 5, 4095, 0, 0, 98, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 507, 0, 0, 4096, 0, 0, 4073, 4092, 0, 0, 309, 0, 0, 0, 4096, 4096, 0, 0, 112, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 595, 2299, 0, 0, 4096, 0, 0, 3681, 1306, 4096, 0, 0, 4096, 3497, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4084, 0, 0, 446, 0, 5, 4095, 0, 0, 4096, 2568, 0, 1, 1964, 0, 0, 1, 4087, 4090, 0, 6, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 2321, 4017, 0, 5, 4095, 0, 0, 122, 622, 4096, 32, 0, 4096, 4088, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 11, 0, 0, 0, 4096, 0, 0, 8, 191] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 1, 1029, 0, 43, 4076, 0, 5, 4092, 1274, 5, 2, 2897, 5, 0, 0, 4093, 4095, 2, 19, 375, 0, 0, 0, 0, 145, 0, 4096, 0, 32, 1, 387, 4096, 0, 0, 4092, 0, 0, 1629, 9, 4096, 0, 0, 4094, 3974, 12, 0, 4091, 0, 0, 1, 0, 1, 4095, 1, 0, 66, 0, 4095, 1, 0, 411, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4074, 0, 4, 0, 1, 1, 4088, 0, 0, 4061, 3558, 27, 10, 3958, 0, 0, 0, 4087, 4096, 2, 0, 6, 0, 0, 0, 2, 0, 7, 4092, 0, 31, 0, 660, 1849, 0, 0, 4094, 6, 0, 1996, 392, 4094, 0, 0, 4094, 3866, 0, 4, 4095, 0, 0, 0, 0, 0, 4091, 0, 2, 0, 0, 4090, 6, 2, 90, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1051, 0, 0, 4096, 0, 0, 4082, 3598, 0, 0, 3958, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 14, 421, 6, 0, 4095, 0, 0, 2079, 1252, 4096, 5, 0, 4096, 3918, 0, 0, 4090, 0, 0, 0, 0, 22, 4096, 1, 0, 0, 0, 4096, 2, 0, 53, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 72, 3442, 0, 0, 4096, 0, 0, 4095, 4086, 0, 0, 1541, 0, 0, 0, 4096, 4096, 0, 0, 1377, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 161, 367, 0, 0, 4096, 0, 0, 90, 265, 4096, 0, 0, 4096, 3919, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1061, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4034, 18, 0, 567, 4, 0, 4088, 0, 36, 3832, 3713, 1, 0, 18, 11, 4, 0, 4078, 4067, 30, 0, 0, 0, 0, 0, 1, 42, 0, 4038, 40, 0, 0, 3, 3726, 0, 0, 4060, 19, 24, 4089, 2, 4033, 27, 1, 4061, 3938, 0, 0, 4083, 6, 3, 8, 0, 32, 4067, 49, 2, 121, 0, 4060, 69, 1, 392, 38] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 2, 1006, 0, 0, 3087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2801, 1682, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3842, 0, 2, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 2, 1622, 1, 0, 3187, 1, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3813, 0, 0, 0, 3954, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4095, 0, 0, 98, 3, 1295, 1009, 908, 283, 254, 25, 9, 0] rerouted_tokens_per_ep_rank=[8189, 8523, 8109, 8167, 8599, 7781, 8184, 7984] avg=8192.0000 max=8599 max_rank=4 min=7781 min_rank=5 imbalance_ratio=1.049683 max_load_reduction=56937 max_load_reduction_pct=0.868790 moved_to_echo=32548 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4074, 0, 4, 0, 1, 1, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2777, 3558, 27, 10, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3842, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 4092, 0, 31, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 660, 1849, 0, 0, 3186, 6, 0, 1996, 0, 0, 0, 0, 0, 0, 0, 0, 392, 3811, 0, 0, 0, 3866, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4090, 6, 2, 90, 0, 1284, 1007, 908, 283, 254, 26, 9, 0] rerouted_tokens_per_ep_rank=[7161, 10330, 7937, 8200, 7697, 8073, 8177, 7961] avg=8192.0000 max=10330 max_rank=1 min=7161 min_rank=0 imbalance_ratio=1.260986 max_load_reduction=55206 max_load_reduction_pct=0.842377 moved_to_echo=31908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 507, 0, 0, 3087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2786, 4092, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3842, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 595, 2299, 0, 0, 3188, 0, 0, 3681, 0, 0, 0, 0, 0, 0, 0, 0, 1306, 3813, 0, 0, 0, 3497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4, 1287, 1009, 908, 283, 254, 25, 9, 0] rerouted_tokens_per_ep_rank=[7690, 7187, 8050, 8168, 9763, 8616, 8183, 7879] avg=8192.0000 max=9763 max_rank=4 min=7187 min_rank=1 imbalance_ratio=1.191772 max_load_reduction=55773 max_load_reduction_pct=0.851028 moved_to_echo=34441 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 1051, 0, 0, 3087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2792, 3598, 0, 0, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3842, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 14, 421, 6, 0, 3187, 0, 0, 2079, 0, 0, 0, 0, 0, 0, 0, 0, 1252, 3813, 5, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 22, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 53, 23, 1290, 1009, 908, 283, 254, 26, 9, 0] rerouted_tokens_per_ep_rank=[8233, 10348, 7940, 8167, 5707, 8988, 8200, 7953] avg=8192.0000 max=10348 max_rank=1 min=5707 min_rank=4 imbalance_ratio=1.263184 max_load_reduction=55188 max_load_reduction_pct=0.842102 moved_to_echo=30848 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 72, 3442, 0, 0, 3087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2801, 4086, 0, 0, 1541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3842, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 161, 367, 0, 0, 3188, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 265, 3813, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1061, 3, 1294, 1009, 908, 283, 254, 26, 9, 0] rerouted_tokens_per_ep_rank=[10697, 8428, 9315, 8167, 3806, 7997, 8183, 8943] avg=8192.0000 max=10697 max_rank=0 min=3806 min_rank=4 imbalance_ratio=1.305786 max_load_reduction=54839 max_load_reduction_pct=0.836777 moved_to_echo=28929 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 0, 446, 0, 5, 3087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2801, 2568, 0, 1, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 3837, 0, 6, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 2321, 4017, 0, 5, 3187, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 622, 3813, 32, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 1, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 191, 1295, 1008, 908, 283, 253, 25, 9, 0] rerouted_tokens_per_ep_rank=[7622, 7334, 7935, 8167, 9652, 8555, 8195, 8076] avg=8192.0000 max=9652 max_rank=4 min=7334 min_rank=1 imbalance_ratio=1.178223 max_load_reduction=55884 max_load_reduction_pct=0.852722 moved_to_echo=34478 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 1029, 0, 43, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2799, 1274, 5, 2, 2897, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3841, 2, 19, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 4096, 0, 32, 1, 4069, 0, 0, 0, 0, 0, 0, 0, 387, 4096, 0, 0, 3185, 0, 0, 1629, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3813, 0, 0, 0, 3974, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 4095, 1, 0, 411, 2, 1293, 1004, 907, 283, 254, 25, 9, 0] rerouted_tokens_per_ep_rank=[8241, 6987, 8330, 8343, 9297, 7808, 8180, 8350] avg=8192.0000 max=9297 max_rank=4 min=6987 min_rank=1 imbalance_ratio=1.134888 max_load_reduction=56239 max_load_reduction_pct=0.858139 moved_to_echo=33635 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4034, 18, 0, 567, 4, 0, 3080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 2616, 3713, 1, 0, 18, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3812, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 42, 0, 4038, 40, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 3, 3726, 0, 0, 3154, 19, 24, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3750, 27, 1, 0, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 6, 3, 8, 0, 32, 4067, 49, 0, 0, 0, 0, 0, 0, 0, 0, 2, 121, 0, 4060, 69, 1, 392, 38, 1216, 1008, 906, 283, 255, 25, 14, 0] rerouted_tokens_per_ep_rank=[7703, 6399, 7920, 8157, 11015, 7718, 8234, 8390] avg=8192.0000 max=11015 max_rank=4 min=6399 min_rank=1 imbalance_ratio=1.344604 max_load_reduction=54521 max_load_reduction_pct=0.831924 moved_to_echo=35357 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 22272, 24576, 128, 128, 18688, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3968, 30464, 128, 128, 0, 31232, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4224, 18432, 128, 128, 25472, 128, 128, 17536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 8064, 128, 128, 24704, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 256, 128, 32768, 128, 128, 2176, 384, 10368, 8064, 7296, 2304, 2048, 256, 128, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 30720, 128, 128, 2048, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 256, 128, 32768, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[12, 0, 0, 1424, 0, 8, 115, 4095, 0, 0, 2, 0, 318, 0, 0, 4096, 4096, 3572, 0, 0, 0, 0, 0, 0, 0, 3114, 4095, 0, 8, 0, 3984, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 65, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 2, 4096, 0, 1, 0, 4093, 4096, 0, 4096, 2, 4096, 0, 3762, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 886, 0, 32, 0, 4092, 0, 0, 0, 0, 3452, 0, 5, 4096, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 307, 4096, 0, 1, 0, 4070, 0, 0, 0, 4096, 0, 3411, 0, 0, 0, 180, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 4096, 0, 4096, 0, 4096, 0, 4088, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 3980, 3, 72, 4096, 4090, 3906, 0, 0, 2, 9, 2, 0, 5, 21, 3988, 1, 123, 0, 4053, 0, 35, 0, 4094, 0, 1765, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 4, 2, 0, 0, 0, 4090, 0, 0, 5, 4083, 4092, 0, 4086, 12, 4094, 20, 2520, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 2392, 0, 180, 351, 4086, 0, 0, 0, 0, 1717, 0, 21, 4096, 4094, 4074, 0, 0, 0, 30, 0, 0, 0, 2381, 4073, 0, 9, 0, 4096, 0, 0, 0, 4093, 0, 433, 0, 0, 0, 744, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4095, 0, 4096, 0, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[30, 1, 0, 4066, 2, 0, 93, 3995, 0, 8, 2, 1, 19, 0, 1, 4095, 4065, 99, 0, 1, 0, 47, 0, 0, 0, 3385, 1368, 0, 146, 24, 4095, 0, 0, 0, 4047, 0, 4090, 14, 1, 4, 3291, 0, 1, 4071, 0, 1, 1, 0, 2, 0, 11, 4092, 8, 0, 4, 4094, 4092, 0, 4049, 0, 4048, 0, 4072, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 1237, 0, 4, 8, 4096, 0, 0, 24, 0, 2790, 0, 46, 4096, 4095, 4096, 0, 0, 0, 2, 0, 0, 2, 142, 4090, 0, 0, 0, 4051, 0, 7, 0, 4071, 0, 3369, 3, 0, 0, 645, 0, 2, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 31, 0, 4069, 4096, 0, 4096, 0, 4096, 0, 4078, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4026, 0, 8, 79, 4096, 0, 0, 0, 0, 196, 0, 17, 4096, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 150, 4096, 0, 8, 0, 4076, 0, 0, 0, 4096, 0, 3943, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 3894, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 9, 363, 3363, 0, 0, 0, 4088, 0, 0, 111, 6, 4093, 0, 5, 4072, 4035, 3463, 50, 0, 49, 49, 47, 6, 0, 576, 3730, 0, 110, 4, 4063, 1, 0, 44, 4051, 5, 71, 0, 13, 31, 518, 0, 17, 4050, 0, 45, 0, 0, 1, 22, 0, 4043, 22, 95, 48, 3965, 4056, 16, 4008, 0, 4084, 34, 4001, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4090, 1041, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 3, 72, 4096, 1315, 674, 0, 0, 0, 0, 0, 0, 4090, 3906, 0, 0, 2, 9, 2, 0, 649, 0, 0, 0, 0, 0, 0, 0, 5, 21, 3988, 1, 123, 0, 3180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4094, 0, 1765, 0, 0, 0, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 4, 2, 3418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 5, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 12, 292, 20, 2520, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9450, 10140, 8658, 7318, 7340, 7521, 8179, 6930] avg=8192.0000 max=10140 max_rank=1 min=6930 min_rank=7 imbalance_ratio=1.237793 max_load_reduction=55396 max_load_reduction_pct=0.845276 moved_to_echo=29970 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 1424, 0, 8, 115, 4095, 1042, 219, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 318, 0, 0, 4096, 1315, 676, 0, 0, 0, 0, 0, 0, 4096, 3572, 0, 0, 0, 0, 0, 0, 638, 0, 0, 0, 0, 0, 0, 0, 0, 3114, 4095, 0, 8, 0, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 1447, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 4096, 0, 0, 0, 0, 3420, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 4096, 0, 1, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 292, 0, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6915, 6407, 8306, 10343, 9638, 7581, 8194, 8152] avg=8192.0000 max=10343 max_rank=3 min=6407 min_rank=1 imbalance_ratio=1.262573 max_load_reduction=55193 max_load_reduction_pct=0.842178 moved_to_echo=33565 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2392, 0, 180, 351, 4086, 1042, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1717, 0, 21, 4096, 1315, 675, 0, 0, 0, 0, 0, 0, 4094, 4074, 0, 0, 0, 30, 0, 0, 656, 0, 0, 0, 0, 0, 0, 0, 0, 2381, 4073, 0, 9, 0, 3214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 433, 0, 0, 0, 1447, 0, 0, 0, 0, 0, 0, 0, 744, 0, 0, 4093, 0, 0, 0, 0, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 292, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 7824, 8854, 9677, 5973, 8258, 8193, 8481] avg=8192.0000 max=9677 max_rank=3 min=5973 min_rank=4 imbalance_ratio=1.181274 max_load_reduction=55859 max_load_reduction_pct=0.852341 moved_to_echo=30905 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 886, 0, 32, 0, 4092, 1042, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3452, 0, 5, 4096, 1315, 676, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, 0, 0, 307, 4096, 0, 1, 0, 3193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3411, 0, 0, 0, 1447, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 4096, 0, 0, 0, 0, 3420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 292, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6276, 9544, 8825, 7597, 8954, 7696, 8168, 8476] avg=8192.0000 max=9544 max_rank=1 min=6276 min_rank=0 imbalance_ratio=1.165039 max_load_reduction=55992 max_load_reduction_pct=0.854370 moved_to_echo=33294 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4026, 0, 8, 79, 4096, 1042, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 17, 4096, 1315, 675, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 150, 4096, 0, 8, 0, 3198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3943, 0, 0, 0, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 292, 0, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9475, 6299, 8832, 7452, 9486, 7517, 8193, 8282] avg=8192.0000 max=9486 max_rank=4 min=6299 min_rank=1 imbalance_ratio=1.157959 max_load_reduction=56050 max_load_reduction_pct=0.855255 moved_to_echo=33478 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[30, 1, 0, 4066, 2, 0, 93, 3995, 1030, 225, 0, 0, 0, 0, 0, 0, 0, 8, 2, 1, 19, 0, 1, 4095, 1300, 675, 0, 0, 0, 0, 0, 0, 4065, 99, 0, 1, 0, 47, 0, 0, 656, 0, 0, 0, 0, 0, 0, 0, 0, 3385, 1368, 0, 146, 24, 3213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 4090, 14, 1, 4, 1430, 0, 0, 0, 0, 0, 0, 0, 3291, 0, 1, 4071, 0, 1, 1, 0, 3417, 0, 0, 0, 0, 0, 0, 0, 2, 0, 11, 4092, 8, 0, 4, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 288, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9442, 6101, 4868, 8136, 9586, 10782, 8212, 8409] avg=8192.0000 max=10782 max_rank=5 min=4868 min_rank=2 imbalance_ratio=1.316162 max_load_reduction=54754 max_load_reduction_pct=0.835480 moved_to_echo=36989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1237, 0, 4, 8, 4096, 1042, 223, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 2790, 0, 46, 4096, 1315, 676, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 2, 0, 0, 649, 0, 0, 0, 0, 0, 0, 0, 2, 142, 4090, 0, 0, 0, 3178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4071, 0, 3369, 3, 0, 0, 1447, 0, 0, 0, 0, 0, 0, 0, 645, 0, 2, 4096, 0, 0, 1, 0, 3420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 31, 0, 4069, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 292, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6611, 8947, 8842, 7412, 8897, 8164, 8197, 8466] avg=8192.0000 max=8947 max_rank=1 min=6611 min_rank=0 imbalance_ratio=1.092163 max_load_reduction=56589 max_load_reduction_pct=0.863480 moved_to_echo=33724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 363, 3363, 0, 0, 0, 4088, 1041, 227, 0, 0, 0, 0, 0, 0, 0, 0, 111, 6, 4093, 0, 5, 4072, 1318, 669, 0, 0, 0, 0, 0, 0, 4035, 3463, 50, 0, 49, 49, 47, 6, 652, 0, 0, 0, 0, 0, 0, 0, 0, 576, 3730, 0, 110, 4, 3180, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 4051, 5, 71, 0, 13, 31, 1447, 0, 0, 0, 0, 0, 0, 0, 518, 0, 17, 4050, 0, 45, 0, 0, 3387, 0, 0, 0, 0, 0, 0, 0, 1, 22, 0, 4043, 22, 95, 48, 3965, 4, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4008, 0, 278, 34, 4001, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9091, 10274, 8351, 7601, 5662, 8017, 8200, 8340] avg=8192.0000 max=10274 max_rank=1 min=5662 min_rank=4 imbalance_ratio=1.254150 max_load_reduction=55262 max_load_reduction_pct=0.843231 moved_to_echo=30219 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 10112, 29568, 128, 512, 128, 25600, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 27392, 128, 128, 128, 256, 128, 128, 5248, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 384, 21504, 128, 256, 768, 32640, 8448, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[5504, 0, 128, 32768, 128, 128, 128, 128, 27392, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32640, 128, 2432, 128, 30592, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 21248, 128, 128, 128, 11648, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 256, 128, 16640, 128, 256, 32768, 10624, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[499, 0, 0, 5, 4092, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 7, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4083, 4096, 0, 0, 4079, 5, 4096, 0, 0, 3690, 4093, 4096, 3990, 0, 0, 0, 17, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4094, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 7, 0, 4082, 3, 0, 0, 46, 0, 0, 0, 33, 0, 42, 0, 0, 0, 0, 0, 202, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 4096, 0, 1, 3836, 0, 4096, 0, 0, 3906, 4096, 4095, 4096, 0, 0, 142, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4082, 0, 4090, 4095, 0, 0, 0, 4096, 0, 215, 8, 0, 4095, 0, 27, 3, 0, 2, 0, 0, 1009, 31, 11, 7, 0, 25, 0, 8, 93, 0, 0, 0, 0, 0, 4092, 0, 1, 4026, 4015, 1, 0, 3121, 0, 4073, 4, 0, 4010, 4047, 4096, 3866, 2, 45, 75, 0, 0, 4066, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4087, 4094, 0, 4094, 4096, 0, 0, 1, 4095, 0, 873, 4, 2, 4093, 0, 0, 0, 3, 0, 0, 2, 187, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 3, 0, 4096, 4090, 3, 8, 4081, 8, 4092, 0, 1, 3051, 4089, 4085, 4089, 0, 0, 0, 0, 0, 4095, 8, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4087, 0, 4096, 4096, 0, 0, 0, 4096, 0, 178, 0, 0, 4095, 6, 3, 0, 0, 0, 12, 0, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4096, 4093, 0, 8, 3506, 0, 4096, 0, 0, 3976, 4085, 4096, 4081, 0, 1, 0, 4, 0, 4096, 0, 15, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 852, 3, 0, 4096, 0, 83, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 4096, 0, 10, 3156, 0, 4096, 0, 0, 4090, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 10, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 22, 4034, 0, 4096, 0, 0, 4073, 4096, 4096, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[217, 31, 19, 0, 4003, 4054, 20, 4070, 4008, 29, 51, 0, 4043, 0, 996, 0, 16, 3953, 0, 1, 1, 0, 5, 1, 0, 0, 0, 72, 41, 0, 0, 11, 99, 215, 0, 3, 46, 0, 0, 4057, 97, 0, 3994, 4044, 0, 0, 4003, 1, 4058, 124, 0, 2855, 4044, 4055, 4077, 11, 102, 56, 0, 0, 3952, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4094, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3710, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 3, 0, 0, 46, 0, 0, 3848, 248, 0, 0, 0, 0, 0, 0, 0, 33, 0, 42, 0, 0, 0, 0, 4096, 3847, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 0, 0, 4096, 3567, 386, 77, 0, 0, 0, 0, 0, 0, 0, 4083, 375, 0, 1, 3836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 423, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 4096, 0, 0, 0, 3721, 248, 106, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 7815, 8227, 8018, 8328, 8295, 8425, 8313] avg=8192.0000 max=8425 max_rank=6 min=7815 min_rank=1 imbalance_ratio=1.028442 max_load_reduction=57111 max_load_reduction_pct=0.871445 moved_to_echo=33361 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4087, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 3709, 0, 873, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 3, 0, 0, 3844, 247, 0, 0, 0, 0, 0, 0, 2, 187, 0, 0, 0, 1, 0, 0, 4094, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 3561, 386, 77, 0, 0, 0, 0, 0, 3, 0, 4096, 375, 3, 8, 4081, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3051, 422, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 8, 5, 0, 3715, 248, 106, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8683, 8189, 8122, 8122, 8574, 7563, 8177] avg=8192.0000 max=8683 max_rank=1 min=7563 min_rank=6 imbalance_ratio=1.059937 max_load_reduction=56853 max_load_reduction_pct=0.867508 moved_to_echo=32436 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3710, 0, 215, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 27, 3, 0, 2, 0, 3828, 248, 0, 0, 0, 0, 0, 0, 0, 1009, 31, 11, 7, 0, 25, 0, 4090, 3848, 0, 0, 0, 0, 0, 0, 8, 93, 0, 0, 0, 0, 0, 4092, 3524, 386, 77, 0, 0, 0, 0, 0, 0, 1, 4026, 368, 1, 0, 3121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4010, 418, 0, 3866, 2, 0, 0, 0, 0, 0, 0, 0, 0, 45, 75, 0, 0, 4066, 4, 0, 0, 3647, 245, 105, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8028, 8203, 9021, 8180, 7517, 8300, 8187] avg=8192.0000 max=9021 max_rank=3 min=7517 min_rank=5 imbalance_ratio=1.101196 max_load_reduction=56515 max_load_reduction_pct=0.862350 moved_to_echo=32184 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[499, 0, 0, 5, 4092, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 7, 0, 0, 3849, 248, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 3848, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 4096, 3565, 386, 77, 0, 0, 0, 0, 0, 0, 0, 4083, 375, 0, 0, 4079, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3690, 422, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4096, 0, 0, 0, 3721, 247, 106, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8615, 7806, 8201, 7956, 8127, 8542, 8102, 8187] avg=8192.0000 max=8615 max_rank=0 min=7806 min_rank=1 imbalance_ratio=1.051636 max_load_reduction=56921 max_load_reduction_pct=0.868546 moved_to_echo=32958 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3710, 0, 852, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 83, 0, 0, 0, 0, 3848, 247, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3567, 386, 77, 0, 0, 0, 0, 0, 0, 0, 4093, 375, 0, 10, 3156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 423, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3721, 248, 106, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8661, 8274, 7946, 8126, 7634, 8609, 8171] avg=8192.0000 max=8661 max_rank=1 min=7634 min_rank=5 imbalance_ratio=1.057251 max_load_reduction=56875 max_load_reduction_pct=0.867844 moved_to_echo=32540 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3710, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 3, 0, 0, 0, 12, 3849, 248, 0, 0, 0, 0, 0, 0, 0, 523, 0, 0, 0, 0, 0, 0, 4096, 3848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3558, 386, 77, 0, 0, 0, 0, 0, 1, 0, 4096, 375, 0, 8, 3506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 421, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 4096, 0, 15, 0, 3718, 247, 106, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 7984, 8213, 8467, 8115, 7986, 8478, 8187] avg=8192.0000 max=8478 max_rank=6 min=7984 min_rank=1 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=32766 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3710, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3849, 248, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 4096, 3848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3567, 386, 77, 0, 0, 0, 0, 0, 0, 0, 4096, 375, 0, 22, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 423, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3721, 247, 106, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 7817, 8193, 8003, 8126, 8527, 8584, 8170] avg=8192.0000 max=8584 max_rank=6 min=7817 min_rank=1 imbalance_ratio=1.047852 max_load_reduction=56952 max_load_reduction_pct=0.869019 moved_to_echo=33407 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[217, 31, 19, 0, 4003, 3973, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 29, 51, 0, 3658, 0, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3953, 0, 1, 1, 0, 5, 1, 3813, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 41, 0, 0, 11, 4070, 3809, 0, 0, 0, 0, 0, 0, 99, 215, 0, 3, 46, 0, 0, 4057, 3526, 385, 81, 0, 0, 0, 0, 0, 97, 0, 3994, 366, 0, 0, 4003, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 2855, 408, 0, 4077, 11, 0, 0, 0, 0, 0, 0, 0, 0, 102, 56, 0, 0, 3952, 0, 0, 1, 3678, 245, 110, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8742, 8036, 8003, 8412, 8461, 7475, 8144] avg=8192.0000 max=8742 max_rank=1 min=7475 min_rank=6 imbalance_ratio=1.067139 max_load_reduction=56794 max_load_reduction_pct=0.866608 moved_to_echo=32492 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 29696, 128, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 29696, 3456, 0, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 128, 128, 128, 32768, 32128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32640, 3072, 128, 128, 29824, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 384, 128, 0, 32640, 128, 128, 128, 29696, 2048, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 512, 0, 128, 128, 0, 128, 32768, 28544, 3200, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 1920, 128, 128, 128, 128, 128, 128, 32768, 30848, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32640, 128, 128, 128, 128, 128, 128, 30848, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[3938, 8, 4096, 0, 0, 0, 6, 0, 0, 0, 2, 0, 4057, 4082, 4096, 0, 704, 0, 887, 4094, 0, 0, 0, 1073, 1, 37, 0, 0, 4, 0, 12, 4096, 6, 4067, 0, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 1611, 24, 0, 4092, 0, 8, 4096, 4089, 0, 0, 4091, 0, 4, 0, 16, 0, 0, 4093, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[2710, 64, 4096, 0, 0, 0, 0, 0, 0, 16, 0, 1389, 4056, 4067, 4096, 0, 4082, 0, 43, 4096, 245, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 4077, 0, 3861, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 948, 2, 12, 3174, 1, 31, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4090, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[3862, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1295, 4096, 4096, 4096, 0, 3521, 0, 4, 4096, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4096, 0, 4090, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 450, 0, 0, 3079, 0, 0, 4075, 4095, 0, 0, 4095, 0, 74, 0, 0, 0, 47, 4096, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4091, 161, 4087, 0, 12, 0, 0, 5, 4, 172, 0, 6, 3949, 4069, 4096, 1, 923, 14, 3926, 4092, 0, 0, 32, 0, 63, 1, 1, 0, 3, 70, 7, 4090, 5, 4092, 0, 3052, 0, 0, 0, 2, 4034, 0, 3, 8, 0, 532, 0, 0, 4076, 0, 66, 3545, 4093, 0, 0, 4093, 0, 0, 0, 0, 10, 0, 4050, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4076, 2, 4096, 3, 0, 0, 0, 0, 0, 40, 0, 11, 4054, 4096, 4096, 0, 3307, 0, 812, 4083, 12, 77, 0, 240, 29, 0, 0, 0, 0, 0, 0, 3401, 0, 4095, 14, 4034, 47, 0, 0, 1, 4096, 0, 1, 0, 0, 255, 1, 0, 4052, 0, 98, 4020, 4093, 0, 0, 4078, 0, 67, 0, 0, 0, 51, 4093, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4063, 0, 4096, 0, 0, 0, 0, 0, 15, 0, 0, 10, 4096, 4096, 4096, 0, 2834, 0, 18, 4096, 0, 0, 0, 909, 0, 0, 0, 0, 0, 2, 0, 4081, 0, 4096, 0, 4076, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 7, 0, 0, 4085, 0, 0, 4096, 4096, 0, 0, 4096, 0, 380, 0, 0, 0, 1, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[3291, 0, 4096, 0, 0, 0, 0, 0, 551, 0, 0, 773, 4096, 4096, 4096, 0, 1, 0, 57, 4096, 0, 0, 0, 273, 0, 0, 0, 0, 0, 2, 0, 3586, 0, 4096, 0, 4078, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3808, 5, 0, 4056, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[3566, 507, 3955, 2, 6, 71, 51, 0, 1, 11, 50, 2, 3098, 4017, 4048, 43, 104, 213, 4066, 4050, 168, 71, 70, 158, 0, 0, 1, 0, 30, 0, 1, 4078, 18, 3925, 0, 3504, 0, 35, 6, 1, 3981, 11, 0, 0, 15, 908, 405, 0, 3801, 0, 0, 4027, 4084, 45, 19, 4080, 17, 0, 0, 165, 12, 0, 4039, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3938, 8, 4096, 0, 0, 0, 6, 0, 188, 114, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4057, 3710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 704, 0, 887, 4094, 0, 0, 0, 1073, 497, 0, 0, 0, 0, 0, 0, 0, 1, 37, 0, 0, 4, 0, 12, 4096, 4089, 136, 0, 0, 0, 0, 0, 0, 6, 4067, 0, 4096, 0, 0, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 1611, 24, 0, 3004, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 8, 4096, 0, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 16, 0, 0, 4093, 0, 3960, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8350, 7769, 7255, 8375, 8427, 8689, 8548, 8123] avg=8192.0000 max=8689 max_rank=5 min=7255 min_rank=2 imbalance_ratio=1.060669 max_load_reduction=56847 max_load_reduction_pct=0.867416 moved_to_echo=33787 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3862, 0, 4096, 0, 0, 0, 0, 0, 189, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1295, 4096, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3521, 0, 4, 4096, 0, 3, 0, 4, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4095, 137, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 1, 0, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 450, 0, 0, 3006, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 0, 4075, 0, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 47, 4096, 3, 3959, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 9114, 8126, 8302, 8446, 7552, 7506, 8229] avg=8192.0000 max=9114 max_rank=1 min=7506 min_rank=6 imbalance_ratio=1.112549 max_load_reduction=56422 max_load_reduction_pct=0.860931 moved_to_echo=31733 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2710, 64, 4096, 0, 0, 0, 0, 0, 189, 113, 0, 0, 0, 0, 0, 0, 0, 16, 0, 1389, 4056, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 43, 4096, 245, 0, 0, 1, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 4095, 137, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 3861, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 948, 2, 12, 3007, 0, 0, 0, 0, 0, 0, 0, 3174, 1, 31, 4096, 0, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 3959, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7172, 9158, 8965, 8268, 8195, 8022, 7654, 8102] avg=8192.0000 max=9158 max_rank=1 min=7172 min_rank=0 imbalance_ratio=1.117920 max_load_reduction=56378 max_load_reduction_pct=0.860260 moved_to_echo=31973 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4091, 161, 4087, 0, 12, 0, 0, 5, 188, 113, 0, 0, 0, 0, 0, 0, 4, 172, 0, 6, 3949, 3699, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 923, 14, 3926, 4092, 0, 0, 32, 0, 497, 0, 0, 0, 0, 0, 0, 0, 63, 1, 1, 0, 3, 70, 7, 4090, 4093, 137, 0, 0, 0, 0, 0, 0, 5, 4092, 0, 3052, 0, 0, 0, 2, 257, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 3, 8, 0, 532, 0, 0, 3005, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 66, 3545, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4050, 0, 3959, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8657, 7831, 9484, 8465, 7408, 7582, 8040, 8069] avg=8192.0000 max=9484 max_rank=2 min=7408 min_rank=4 imbalance_ratio=1.157715 max_load_reduction=56052 max_load_reduction_pct=0.855286 moved_to_echo=31099 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3291, 0, 4096, 0, 0, 0, 0, 0, 189, 114, 0, 0, 0, 0, 0, 0, 551, 0, 0, 773, 4096, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 57, 4096, 0, 0, 0, 273, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3586, 4096, 136, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4078, 0, 0, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3808, 5, 0, 3007, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 4096, 0, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3960, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7690, 9143, 4925, 7820, 8433, 10915, 8504, 8106] avg=8192.0000 max=10915 max_rank=5 min=4925 min_rank=2 imbalance_ratio=1.332397 max_load_reduction=54621 max_load_reduction_pct=0.833450 moved_to_echo=35958 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4076, 2, 4096, 3, 0, 0, 0, 0, 188, 114, 0, 0, 0, 0, 0, 0, 0, 40, 0, 11, 4054, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3307, 0, 812, 4083, 12, 77, 0, 240, 496, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 3401, 4093, 137, 0, 0, 0, 0, 0, 0, 0, 4095, 14, 4034, 47, 0, 0, 1, 259, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 255, 1, 0, 2994, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 98, 4020, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 51, 4093, 5, 3959, 49, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8479, 7828, 9027, 7660, 8450, 7347, 8521, 8224] avg=8192.0000 max=9027 max_rank=2 min=7347 min_rank=5 imbalance_ratio=1.101929 max_load_reduction=56509 max_load_reduction_pct=0.862259 moved_to_echo=32542 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4063, 0, 4096, 0, 0, 0, 0, 0, 189, 114, 0, 0, 0, 0, 0, 0, 15, 0, 0, 10, 4096, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2834, 0, 18, 4096, 0, 0, 0, 909, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4081, 4096, 137, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 0, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 7, 0, 0, 3007, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 0, 1, 4096, 0, 3959, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8462, 7844, 8355, 8316, 8431, 7109, 8533, 8486] avg=8192.0000 max=8533 max_rank=6 min=7109 min_rank=5 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=32559 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3566, 507, 3955, 2, 6, 71, 51, 0, 191, 116, 0, 0, 0, 0, 0, 0, 1, 11, 50, 2, 3098, 3644, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 104, 213, 4066, 4050, 168, 71, 70, 158, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 30, 0, 1, 4078, 4084, 136, 0, 0, 0, 0, 0, 0, 18, 3925, 0, 3504, 0, 35, 6, 1, 257, 0, 0, 0, 0, 0, 0, 0, 3981, 11, 0, 0, 15, 908, 405, 0, 3000, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 4027, 0, 45, 19, 338, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 165, 12, 0, 4039, 0, 3912, 52, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8465, 6849, 9399, 8330, 7746, 8320, 8230, 8197] avg=8192.0000 max=9399 max_rank=2 min=6849 min_rank=1 imbalance_ratio=1.147339 max_load_reduction=56137 max_load_reduction_pct=0.856583 moved_to_echo=32493 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[30464, 128, 256, 32128, 0, 128, 128, 2816, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 256, 128, 3584, 31616, 29696, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32512, 128, 128, 128, 128, 8576, 512, 128, 24064, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 31488, 32768, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[15488, 256, 9856, 32768, 512, 256, 128, 2688, 4096, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[29696, 768, 32640, 128, 128, 128, 128, 128, 1536, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 640, 0, 256, 128, 128, 32768, 128, 31744, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32640, 128, 30848, 128, 128, 128, 128, 2176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 653, 4096, 0, 0, 0, 0, 4087, 0, 4027, 0, 9, 5, 0, 0, 0, 0, 4, 4090, 0, 4, 4094, 0, 0, 0, 0, 0, 3, 4089, 0, 0, 647, 25, 2, 4095, 4089, 26, 4035, 0, 0, 0, 0, 1, 4092, 4096, 0, 0, 4096, 4077, 1, 0, 4092, 0, 0, 0, 9, 2885, 0, 0, 4096, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3836, 3272, 0, 0, 0, 38, 4096, 0, 1033, 2, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 4041, 0, 0, 26, 0, 0, 4052, 4096, 0, 4096, 0, 0, 0, 0, 0, 4089, 4095, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 66, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3497, 4074, 0, 0, 0, 687, 4096, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 26, 0, 4021, 0, 0, 0, 0, 0, 4079, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1084, 4095, 0, 0, 0, 2, 4096, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4085, 0, 0, 0, 0, 3197, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 178, 4090, 0, 0, 11, 0, 4094, 0, 4016, 0, 2, 3, 0, 0, 0, 0, 96, 4092, 0, 0, 4091, 5, 8, 0, 1, 385, 0, 4080, 5, 0, 0, 0, 0, 3815, 4096, 0, 4055, 3, 15, 0, 0, 0, 4096, 4077, 0, 3, 4093, 4096, 2, 0, 4075, 0, 0, 4, 0, 3813, 0, 29, 4083, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 754, 4095, 0, 0, 0, 1, 4096, 0, 3416, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 11, 0, 0, 4096, 4096, 29, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 3983, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3985, 1705, 0, 3, 0, 19, 4096, 2, 2540, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 35, 0, 3798, 0, 0, 136, 0, 0, 4094, 4088, 193, 4096, 5, 0, 0, 0, 0, 4070, 4095, 0, 0, 4096, 4096, 0, 0, 4001, 0, 0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[10, 0, 470, 0, 2031, 2576, 31, 0, 0, 1054, 4061, 0, 867, 0, 0, 12, 7, 38, 25, 43, 0, 4039, 0, 145, 3906, 0, 49, 0, 23, 0, 6, 4062, 12, 4, 1019, 0, 0, 4084, 3758, 2040, 4052, 0, 0, 0, 59, 28, 3134, 4049, 0, 24, 4043, 4040, 18, 47, 3765, 25, 0, 0, 1, 3525, 0, 286, 4066, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 653, 4096, 0, 0, 2629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4027, 0, 9, 5, 1315, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4090, 0, 4, 4043, 22, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3, 4089, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 647, 25, 2, 4095, 3653, 26, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 1, 4092, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4077, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2885, 0, 0, 4096, 1, 341, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7388, 9523, 8163, 8201, 8448, 8258, 8223, 7332] avg=8192.0000 max=9523 max_rank=1 min=7332 min_rank=7 imbalance_ratio=1.162476 max_load_reduction=56013 max_load_reduction_pct=0.854691 moved_to_echo=32261 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3836, 3272, 0, 0, 2628, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 0, 1033, 2, 0, 8, 1314, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4043, 22, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 4041, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 4052, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 66, 4096, 0, 341, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9736, 6571, 8161, 8173, 7738, 8316, 8245, 8596] avg=8192.0000 max=9736 max_rank=0 min=6571 min_rank=1 imbalance_ratio=1.188477 max_load_reduction=55800 max_load_reduction_pct=0.851440 moved_to_echo=32895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3497, 4074, 0, 0, 2629, 0, 0, 0, 0, 0, 0, 0, 0, 687, 4096, 0, 2, 0, 0, 0, 1315, 80, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4043, 22, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 26, 0, 4021, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 341, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10200, 6180, 8163, 8158, 7739, 8322, 8244, 8530] avg=8192.0000 max=10200 max_rank=0 min=6180 min_rank=1 imbalance_ratio=1.245117 max_load_reduction=55336 max_load_reduction_pct=0.844360 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 754, 4095, 0, 0, 2629, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 3416, 0, 0, 0, 1315, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4043, 22, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 3660, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 4096, 0, 341, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7478, 8908, 8161, 8206, 7796, 8322, 8245, 8420] avg=8192.0000 max=8908 max_rank=1 min=7478 min_rank=0 imbalance_ratio=1.087402 max_load_reduction=56628 max_load_reduction_pct=0.864075 moved_to_echo=32783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1084, 4095, 0, 0, 2629, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 3919, 0, 0, 0, 1315, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4043, 22, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4096, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3197, 0, 0, 4096, 0, 341, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7808, 9412, 8161, 8207, 7758, 8322, 8234, 7634] avg=8192.0000 max=9412 max_rank=1 min=7634 min_rank=7 imbalance_ratio=1.148926 max_load_reduction=56124 max_load_reduction_pct=0.856384 moved_to_echo=31948 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 178, 4090, 0, 0, 2617, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4094, 0, 4016, 0, 2, 3, 1309, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 4092, 0, 0, 4040, 22, 0, 0, 0, 0, 0, 0, 4091, 5, 8, 0, 1, 385, 0, 4080, 15, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3815, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 3, 15, 0, 0, 0, 4096, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 53, 4096, 2, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3813, 0, 29, 4083, 23, 341, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6886, 9515, 8250, 8585, 7480, 8298, 8229, 8293] avg=8192.0000 max=9515 max_rank=1 min=6886 min_rank=0 imbalance_ratio=1.161499 max_load_reduction=56021 max_load_reduction_pct=0.854813 moved_to_echo=32300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3985, 1705, 0, 3, 2628, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 2, 2540, 0, 1, 0, 1314, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4043, 22, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 35, 0, 3798, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 4094, 3652, 193, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 4070, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4096, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 341, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8321, 8052, 8161, 7942, 8075, 8302, 8150, 8533] avg=8192.0000 max=8533 max_rank=7 min=7942 min_rank=3 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=33060 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 470, 0, 2031, 2576, 31, 0, 2601, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 4061, 0, 867, 0, 0, 12, 1302, 79, 0, 0, 0, 0, 0, 0, 7, 38, 25, 43, 0, 4039, 0, 145, 3996, 23, 0, 0, 0, 0, 0, 0, 3906, 0, 49, 0, 23, 0, 6, 4062, 18, 0, 0, 0, 0, 0, 0, 0, 12, 4, 1019, 0, 0, 4084, 3343, 2040, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 59, 28, 3134, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 47, 4040, 18, 47, 3765, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3525, 0, 286, 4066, 2, 318, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7719, 7375, 8316, 8064, 10502, 7396, 7966, 8198] avg=8192.0000 max=10502 max_rank=4 min=7375 min_rank=1 imbalance_ratio=1.281982 max_load_reduction=55034 max_load_reduction_pct=0.839752 moved_to_echo=34062 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 1920, 128, 128, 32512, 29056, 2304, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 0, 128, 512, 128, 32384, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 29696, 0, 384, 32768, 128, 2816, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 512, 0, 16128, 28032, 128, 128, 20992, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 0, 128, 128, 31872, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 1920, 32768, 128, 19840, 128, 128, 128, 10624, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 256, 32384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 512, 32768, 128, 128, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 4010, 22, 3606, 360, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4, 4096, 0, 4096, 0, 3797, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 203, 4096, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 1, 15, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 1, 4094, 4095, 0, 0, 1, 3, 4096, 0, 4096, 682, 3430, 4096, 0, 3, 0, 4094, 7, 3644, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 375, 4096, 41, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 3180, 0, 4095, 0, 3333, 0, 0, 0, 0, 2810, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4068, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2609, 288, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4093, 0, 4096, 0, 4081, 22, 347, 23, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 4074, 8, 2466, 4096, 0, 0, 0, 0, 4092, 5, 4086, 567, 7, 4095, 0, 3, 0, 4096, 0, 4086, 0, 0, 4091, 0, 0, 0, 0, 13, 0, 0, 633, 4096, 0, 89, 0, 0, 4071, 0, 4, 0, 0, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4091, 0, 4096, 0, 4076, 0, 43, 0, 1, 11, 0, 3990, 14, 0, 0, 0, 0, 0, 4087, 0, 6, 4090, 15, 3564, 4096, 0, 0, 0, 0, 4081, 2, 4094, 371, 359, 4091, 1, 0, 409, 4095, 223, 4044, 2, 2, 3350, 5, 0, 0, 0, 0, 3, 0, 18, 4084, 8, 20, 0, 0, 4094, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 2949, 0, 4096, 0, 3078, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2087, 4096, 0, 0, 0, 0, 4096, 0, 4096, 62, 4051, 4096, 0, 0, 0, 4096, 39, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 2724, 0, 4094, 0, 1216, 55, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 4096, 30, 0, 4096, 0, 1193, 4096, 0, 1, 0, 0, 4068, 4, 4096, 673, 4096, 4096, 0, 0, 52, 4096, 6, 4069, 0, 0, 2382, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[3957, 0, 4059, 0, 3877, 133, 62, 0, 24, 0, 41, 4096, 50, 0, 0, 0, 0, 34, 4061, 244, 6, 4015, 0, 4093, 4030, 0, 0, 3, 0, 3919, 0, 3991, 378, 514, 4056, 0, 241, 0, 4063, 0, 4027, 0, 31, 3469, 21, 15, 13, 34, 11, 0, 107, 0, 4045, 0, 76, 0, 52, 3640, 10, 38, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 15, 0, 3540, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 188, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 0, 4096, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 1, 3, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 682, 3430, 4096, 0, 3, 0, 1245, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3644, 0, 0, 0, 0, 0, 0, 0, 1891, 556, 0, 0, 0, 0, 0, 0, 1, 0, 0, 375, 4096, 41, 0, 0, 2946, 958, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3866, 286, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7652, 8376, 9052, 8237, 9463, 6091, 8417, 8248] avg=8192.0000 max=9463 max_rank=4 min=6091 min_rank=5 imbalance_ratio=1.155151 max_load_reduction=56073 max_load_reduction_pct=0.855606 moved_to_echo=32219 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3527, 22, 347, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 4093, 188, 0, 0, 0, 0, 0, 0, 0, 0, 860, 0, 1, 4074, 8, 2466, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 4092, 5, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 567, 7, 4095, 0, 3, 0, 1246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4091, 0, 0, 0, 0, 1892, 554, 0, 0, 0, 0, 0, 0, 13, 0, 0, 633, 4096, 0, 89, 0, 2945, 958, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4, 0, 0, 7, 0, 3867, 286, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8015, 8378, 7409, 8224, 5918, 10623, 8734, 8235] avg=8192.0000 max=10623 max_rank=5 min=5918 min_rank=4 imbalance_ratio=1.296753 max_load_reduction=54913 max_load_reduction_pct=0.837906 moved_to_echo=33510 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 3466, 22, 3606, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 4096, 188, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 4, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3797, 4096, 0, 0, 0, 1246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 336, 0, 0, 0, 0, 1892, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 4096, 0, 4, 0, 2946, 958, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3867, 286, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7455, 8328, 9053, 8237, 9139, 6868, 8207, 8249] avg=8192.0000 max=9139 max_rank=4 min=6868 min_rank=5 imbalance_ratio=1.115601 max_load_reduction=56397 max_load_reduction_pct=0.860550 moved_to_echo=32463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3180, 0, 3539, 0, 3333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2810, 0, 0, 0, 0, 4096, 188, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 0, 4096, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2609, 288, 4096, 0, 0, 0, 1246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 93, 0, 0, 0, 0, 1892, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 1, 0, 2949, 958, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3867, 286, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10052, 7094, 9025, 8233, 8239, 6637, 8008, 8248] avg=8192.0000 max=10052 max_rank=0 min=6637 min_rank=5 imbalance_ratio=1.227051 max_load_reduction=55484 max_load_reduction_pct=0.846619 moved_to_echo=31132 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3523, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 3990, 14, 0, 0, 0, 4091, 188, 0, 0, 0, 0, 0, 0, 0, 0, 860, 0, 6, 4090, 15, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 4081, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 371, 359, 4091, 1, 0, 409, 1245, 223, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 2, 2, 3350, 5, 0, 0, 0, 1892, 553, 0, 0, 0, 0, 0, 0, 0, 3, 0, 18, 4084, 8, 20, 0, 2942, 958, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 3867, 285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7662, 8295, 8535, 8218, 6699, 9848, 8033, 8246] avg=8192.0000 max=9848 max_rank=5 min=6699 min_rank=4 imbalance_ratio=1.202148 max_load_reduction=55688 max_load_reduction_pct=0.849731 moved_to_echo=32826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2949, 0, 3540, 0, 3078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 188, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 0, 4096, 0, 2087, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 62, 4051, 4096, 0, 0, 0, 1246, 39, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 1892, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 5, 0, 2949, 958, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3867, 286, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9567, 8380, 7044, 8233, 9494, 6547, 8022, 8249] avg=8192.0000 max=9567 max_rank=0 min=6547 min_rank=5 imbalance_ratio=1.167847 max_load_reduction=55969 max_load_reduction_pct=0.854019 moved_to_echo=32312 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2724, 0, 3538, 0, 1216, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 4096, 188, 0, 0, 0, 0, 0, 0, 0, 0, 861, 30, 0, 4096, 0, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 1, 0, 0, 4068, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 673, 4096, 4096, 0, 0, 52, 1246, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 2382, 0, 0, 0, 0, 1892, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 0, 0, 0, 2949, 958, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 20, 3867, 286, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7533, 8326, 6180, 8210, 10169, 8899, 7994, 8225] avg=8192.0000 max=10169 max_rank=4 min=6180 min_rank=2 imbalance_ratio=1.241333 max_load_reduction=55367 max_load_reduction_pct=0.844833 moved_to_echo=35287 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4059, 0, 3346, 133, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 41, 4096, 50, 0, 0, 0, 3957, 191, 0, 0, 0, 0, 0, 0, 0, 34, 846, 244, 6, 4015, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 3, 0, 3919, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 378, 514, 4056, 0, 241, 0, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 31, 3469, 21, 15, 13, 34, 1882, 531, 0, 0, 0, 0, 0, 0, 11, 0, 107, 0, 4045, 0, 76, 0, 2927, 955, 0, 0, 0, 0, 0, 0, 52, 3640, 10, 38, 0, 0, 0, 0, 3808, 288, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7600, 8359, 9238, 7944, 6415, 10023, 8121, 7836] avg=8192.0000 max=10023 max_rank=5 min=6415 min_rank=4 imbalance_ratio=1.223511 max_load_reduction=55513 max_load_reduction_pct=0.847061 moved_to_echo=32395 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5376, 16640, 32768, 128, 256, 512, 9984, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 21120, 128, 28032, 256, 15872, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 31360, 128, 0, 0, 0, 32640, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 1280, 32768, 128, 256, 0, 23680, 7680, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[32256, 128, 128, 13824, 128, 128, 128, 128, 15232, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 0, 128, 128, 128, 32640, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32256, 128, 128, 0, 0, 128, 128, 30976, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 6912, 384, 128, 32768, 128, 25728, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4095, 4094, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 0, 0, 456, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 3796, 0, 0, 3623, 1, 4096, 0, 0, 0, 0, 4096, 1, 3022, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4, 4096, 0, 0, 0, 4077, 4096, 4080, 0, 4093, 0, 0, 25, 0, 0, 0, 0, 8, 0, 13, 0, 0, 0, 838, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4071, 0, 0, 1, 3182, 54, 0, 4050, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 81, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 4050, 4095, 0, 4096, 0, 403, 0, 0, 0, 0, 0, 6, 26, 1611, 0, 0, 0, 1646, 1, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 1992, 4, 0, 3319, 0, 4096, 0, 0, 0, 0, 4096, 0, 3345, 0, 0, 0, 2, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 4095, 4053, 4094, 0, 4092, 0, 0, 8, 0, 0, 0, 0, 21, 0, 1358, 0, 0, 0, 2309, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4090, 0, 0, 2, 3633, 0, 0, 2925, 0, 2034, 66, 0, 0, 0, 4096, 3, 4063, 0, 0, 0, 0, 17, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4095, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 1, 0, 741, 0, 0, 0, 2442, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2029, 0, 0, 4050, 0, 3107, 0, 0, 0, 0, 4096, 1, 4013, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 4093, 0, 0, 0, 4094, 4089, 4079, 0, 4090, 2, 0, 0, 0, 0, 5, 6, 8, 0, 3519, 1, 0, 0, 603, 0, 0, 0, 3523, 2, 0, 0, 4088, 0, 0, 2, 0, 0, 4082, 0, 0, 0, 2061, 3, 0, 4068, 0, 4096, 0, 0, 0, 8, 4096, 0, 2606, 26, 12, 0, 1, 0, 4083, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[6, 4094, 0, 0, 4093, 0, 1, 0, 4035, 4096, 4095, 0, 4074, 0, 0, 186, 0, 0, 1, 0, 0, 361, 4061, 0, 0, 0, 56, 142, 8, 2, 4088, 0, 0, 0, 4088, 31, 0, 0, 2, 0, 4011, 4, 3, 18, 3906, 0, 0, 110, 0, 3803, 75, 8, 0, 0, 4095, 0, 3784, 24, 11, 0, 2, 0, 4077, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 4049, 4, 0, 4023, 0, 39, 100, 4078, 3391, 4028, 12, 3995, 0, 705, 375, 0, 8, 59, 130, 0, 0, 4053, 47, 20, 0, 3616, 0, 43, 45, 3946, 0, 0, 3, 3927, 0, 0, 0, 6, 0, 4082, 0, 24, 0, 5, 0, 49, 345, 46, 4042, 0, 31, 14, 31, 4057, 353, 3621, 0, 1, 0, 0, 0, 4065, 4068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 3998, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 26, 1611, 0, 4096, 1927, 0, 0, 0, 0, 0, 0, 0, 0, 1646, 1, 0, 0, 4086, 0, 1628, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2168, 1954, 0, 0, 0, 0, 0, 0, 2796, 0, 0, 0, 1992, 4, 0, 3319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 249, 98, 3, 0, 0, 0, 0, 0, 3345, 0, 0, 0, 2, 0, 4096, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8451, 7666, 8399, 8218, 8111, 8542, 7957] avg=8192.0000 max=8542 max_rank=6 min=7666 min_rank=2 imbalance_ratio=1.042725 max_load_reduction=56994 max_load_reduction_pct=0.869659 moved_to_echo=32828 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3998, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 4096, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 456, 0, 0, 0, 4092, 0, 1628, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2168, 1954, 0, 0, 0, 0, 0, 0, 2761, 0, 0, 0, 3796, 0, 0, 3623, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 1, 246, 98, 3, 0, 0, 0, 0, 0, 3022, 0, 0, 0, 0, 0, 4096, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8095, 7480, 7201, 8218, 10180, 8541, 7632] avg=8192.0000 max=10180 max_rank=5 min=7201 min_rank=3 imbalance_ratio=1.242676 max_load_reduction=55356 max_load_reduction_pct=0.844666 moved_to_echo=34571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 3993, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 8, 0, 3519, 1, 4094, 1919, 0, 0, 0, 0, 0, 0, 0, 0, 603, 0, 0, 0, 3523, 2, 1628, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 2, 0, 0, 2160, 1954, 0, 0, 0, 0, 0, 0, 2796, 0, 0, 0, 2061, 3, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 4096, 0, 249, 97, 3, 0, 0, 0, 0, 0, 2606, 26, 12, 0, 1, 0, 4083, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8084, 9552, 6793, 8204, 8928, 8549, 7242] avg=8192.0000 max=9552 max_rank=2 min=6793 min_rank=3 imbalance_ratio=1.166016 max_load_reduction=55984 max_load_reduction_pct=0.854248 moved_to_echo=32923 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 3995, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 1358, 0, 4095, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 2309, 0, 0, 0, 4096, 0, 1628, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 2168, 1954, 0, 0, 0, 0, 0, 0, 2801, 0, 0, 2, 3633, 0, 0, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2034, 66, 0, 0, 0, 4096, 3, 250, 97, 3, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 17, 4096, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8056, 7400, 9072, 8219, 9361, 6549, 8690] avg=8192.0000 max=9361 max_rank=5 min=6549 min_rank=6 imbalance_ratio=1.142700 max_load_reduction=56175 max_load_reduction_pct=0.857162 moved_to_echo=32819 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3996, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 13, 0, 4077, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 838, 0, 0, 0, 4095, 0, 1628, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2160, 1954, 0, 0, 0, 0, 0, 0, 2787, 0, 0, 1, 3182, 54, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 249, 97, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 81, 4096, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8117, 6018, 7596, 8210, 10074, 8541, 8787] avg=8192.0000 max=10074 max_rank=5 min=6018 min_rank=2 imbalance_ratio=1.229736 max_load_reduction=55462 max_load_reduction_pct=0.846283 moved_to_echo=35612 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3998, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 741, 0, 4096, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 2442, 0, 0, 0, 4096, 0, 1628, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2169, 1954, 0, 0, 0, 0, 0, 0, 2805, 0, 0, 0, 2029, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3107, 0, 0, 0, 0, 4096, 1, 250, 98, 3, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 4096, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8095, 6764, 9207, 8219, 8884, 7555, 8623] avg=8192.0000 max=9207 max_rank=3 min=6764 min_rank=2 imbalance_ratio=1.123901 max_load_reduction=56329 max_load_reduction_pct=0.859512 moved_to_echo=33281 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 4091, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3977, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 361, 4061, 0, 4035, 1927, 0, 0, 0, 0, 0, 0, 0, 0, 56, 142, 8, 2, 4088, 0, 1623, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 31, 0, 0, 2, 0, 2168, 1948, 0, 0, 0, 0, 0, 0, 2747, 4, 3, 18, 3906, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 75, 8, 0, 0, 4095, 0, 245, 97, 3, 0, 0, 0, 0, 0, 3784, 24, 11, 0, 2, 0, 4077, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8259, 10385, 6938, 8237, 6788, 8326, 8412] avg=8192.0000 max=10385 max_rank=2 min=6788 min_rank=5 imbalance_ratio=1.267700 max_load_reduction=55151 max_load_reduction_pct=0.841537 moved_to_echo=31763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4043, 4, 0, 4023, 0, 39, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3391, 0, 12, 3896, 0, 705, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 59, 130, 0, 0, 4053, 47, 4078, 1896, 0, 0, 0, 0, 0, 0, 20, 0, 3616, 0, 43, 45, 3946, 0, 1619, 1041, 0, 0, 0, 0, 0, 0, 0, 3, 3927, 0, 0, 0, 6, 0, 2132, 1943, 0, 0, 0, 0, 0, 0, 2787, 0, 24, 0, 5, 0, 49, 345, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4042, 0, 31, 14, 31, 4057, 353, 254, 99, 6, 0, 0, 0, 0, 0, 3621, 0, 1, 0, 0, 0, 4065, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8379, 10271, 10330, 8011, 3210, 8933, 8193] avg=8192.0000 max=10330 max_rank=3 min=3210 min_rank=5 imbalance_ratio=1.260986 max_load_reduction=55206 max_load_reduction_pct=0.842377 moved_to_echo=28347 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[22400, 128, 128, 128, 20608, 128, 128, 22528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[28672, 128, 128, 0, 128, 128, 32768, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32640, 128, 0, 128, 128, 0, 17408, 15616, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 256, 128, 512, 16896, 128, 32768, 15488, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32000, 0, 128, 31872, 128, 1152, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 29440, 256, 128, 128, 128, 32768, 384, 2048, 896, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 12032, 256, 128, 128, 32128, 128, 13056, 8320, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 1, 0, 0, 0, 3272, 0, 0, 4096, 0, 0, 0, 0, 4096, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4096, 0, 4089, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 611, 0, 0, 0, 0, 0, 4096, 0, 2312, 0, 4093, 0, 4096, 268, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 2757, 0, 0, 4096, 0, 0, 0, 0, 4095, 2459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 3611, 0, 3461, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 5, 22, 2, 2121, 4, 0, 4093, 0, 0, 0, 0, 4090, 3958, 3, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 34, 277, 4094, 0, 4092, 0, 0, 4093, 4093, 4078, 0, 0, 1, 0, 0, 0, 4078, 0, 0, 0, 0, 3, 0, 16, 3, 0, 4092, 0, 2083, 0, 4096, 3, 4061, 73, 3773] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 8, 0, 623, 0, 0, 4096, 0, 0, 1, 0, 4096, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 95, 4096, 0, 4096, 0, 0, 4090, 4096, 4083, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 3702, 0, 2977, 0, 3878, 1301, 3940] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 8, 0, 4086, 12, 0, 0, 0, 6, 2236, 0, 8, 4088, 0, 2, 0, 0, 3995, 3694, 2, 8, 0, 0, 25, 0, 0, 3, 0, 4076, 0, 0, 4096, 0, 4096, 21, 0, 4079, 4089, 4087, 0, 0, 0, 0, 0, 0, 4084, 0, 11, 6, 51, 30, 0, 0, 0, 0, 4067, 0, 3185, 0, 3185, 18, 4096, 1, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 11, 0, 0, 3876, 0, 0, 4096, 0, 0, 0, 0, 4096, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 4084, 0, 7, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 14, 1, 4096, 0, 3983, 0, 64, 0, 4061, 870, 3626] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4095, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 2935, 0, 3946, 903, 4084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[8, 0, 5, 4046, 21, 0, 43, 39, 0, 3633, 51, 0, 4008, 0, 0, 0, 62, 3763, 4007, 0, 27, 23, 12, 210, 0, 0, 0, 1, 4028, 326, 628, 4075, 0, 4078, 391, 0, 4047, 4054, 3826, 1, 0, 21, 0, 0, 88, 4060, 0, 60, 2, 2, 0, 72, 0, 134, 24, 3647, 103, 1215, 38, 4060, 13, 3128, 36, 3420] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 5, 22, 4075, 8, 0, 0, 0, 0, 0, 0, 2, 2121, 4, 0, 4093, 0, 0, 0, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3958, 3, 0, 0, 6, 0, 970, 157, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 34, 277, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4078, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 16, 3, 0, 4092, 3773, 133, 18, 0, 0, 0, 0, 0, 0, 2083, 0, 4096, 3, 1613, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 7565, 9188, 8344, 8167, 8163, 8038, 7868] avg=8192.0000 max=9188 max_rank=2 min=7565 min_rank=1 imbalance_ratio=1.121582 max_load_reduction=56348 max_load_reduction_pct=0.859802 moved_to_echo=32236 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 1, 0, 0, 4079, 9, 0, 0, 0, 0, 0, 0, 0, 3272, 0, 0, 4096, 0, 0, 0, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1736, 0, 0, 0, 0, 0, 978, 157, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 611, 0, 0, 0, 0, 0, 4096, 4092, 134, 17, 0, 0, 0, 0, 0, 0, 2312, 0, 4093, 0, 1627, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8725, 6971, 8041, 8188, 8176, 8950, 8300] avg=8192.0000 max=8950 max_rank=6 min=6971 min_rank=2 imbalance_ratio=1.092529 max_load_reduction=56586 max_load_reduction_pct=0.863434 moved_to_echo=33614 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 8, 4079, 8, 0, 0, 0, 0, 0, 0, 0, 623, 0, 0, 4096, 0, 0, 1, 1284, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3952, 0, 0, 0, 0, 0, 926, 157, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 95, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3940, 127, 17, 0, 0, 0, 0, 0, 21, 3702, 0, 2977, 0, 1541, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 6004, 9135, 8132, 8169, 8183, 8180, 9542] avg=8192.0000 max=9542 max_rank=7 min=6004 min_rank=1 imbalance_ratio=1.164795 max_load_reduction=55994 max_load_reduction_pct=0.854401 moved_to_echo=34074 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4079, 9, 0, 0, 0, 0, 0, 0, 0, 2757, 0, 0, 4096, 0, 0, 0, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2459, 0, 0, 0, 0, 0, 826, 157, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 113, 17, 0, 0, 0, 0, 0, 0, 4095, 0, 3611, 0, 1376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 7999, 7541, 8037, 8187, 8183, 8323, 9082] avg=8192.0000 max=9082 max_rank=7 min=7541 min_rank=2 imbalance_ratio=1.108643 max_load_reduction=56454 max_load_reduction_pct=0.861420 moved_to_echo=33775 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4078, 8, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 405, 0, 0, 0, 0, 0, 942, 157, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 4084, 129, 18, 0, 0, 0, 0, 0, 0, 4095, 0, 2935, 0, 1568, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 9498, 5603, 8037, 8187, 8184, 8344, 9501] avg=8192.0000 max=9501 max_rank=7 min=5603 min_rank=2 imbalance_ratio=1.159790 max_load_reduction=56035 max_load_reduction_pct=0.855026 moved_to_echo=34216 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 4086, 12, 0, 0, 0, 4072, 9, 0, 0, 0, 0, 0, 0, 6, 2236, 0, 8, 4088, 0, 2, 0, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 3694, 2, 8, 0, 0, 25, 978, 157, 4, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4076, 0, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4075, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 11, 4087, 0, 0, 0, 0, 0, 0, 0, 6, 51, 30, 0, 0, 0, 0, 4067, 4085, 134, 17, 0, 0, 0, 0, 0, 0, 3185, 0, 3185, 18, 1627, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 7697, 8863, 8018, 8183, 8182, 8390, 8016] avg=8192.0000 max=8863 max_rank=2 min=7697 min_rank=1 imbalance_ratio=1.081909 max_load_reduction=56673 max_load_reduction_pct=0.864761 moved_to_echo=32771 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 11, 0, 4079, 9, 0, 0, 0, 0, 0, 0, 0, 3876, 0, 0, 4096, 0, 0, 0, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3991, 0, 0, 0, 0, 0, 970, 156, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 4096, 3626, 133, 17, 0, 0, 0, 0, 0, 0, 3983, 0, 64, 0, 1613, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 9317, 9217, 8028, 8195, 8167, 7887, 6530] avg=8192.0000 max=9317 max_rank=1 min=6530 min_rank=7 imbalance_ratio=1.137329 max_load_reduction=56219 max_load_reduction_pct=0.857834 moved_to_echo=30779 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 5, 4046, 21, 0, 43, 39, 4038, 9, 0, 0, 0, 0, 0, 0, 0, 3633, 51, 0, 4008, 0, 0, 0, 1039, 0, 0, 0, 0, 0, 0, 0, 62, 3763, 4007, 0, 27, 23, 12, 210, 750, 159, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4028, 326, 628, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 4042, 0, 3826, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 88, 4060, 0, 60, 4069, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 72, 0, 134, 24, 3647, 3420, 107, 16, 0, 0, 0, 0, 0, 103, 1215, 38, 4060, 13, 1232, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8731, 9018, 8899, 8260, 8298, 7424, 6697] avg=8192.0000 max=9018 max_rank=2 min=6697 min_rank=7 imbalance_ratio=1.100830 max_load_reduction=56518 max_load_reduction_pct=0.862396 moved_to_echo=30679 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 24704, 128, 25088, 128, 12288, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 22656, 128, 128, 32768, 0, 128, 128, 10240, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 384, 1024, 31488, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 768, 128, 128, 128, 256, 128, 32384, 31232, 1024, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 512, 128, 32768, 0, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32384, 24320, 128, 128, 128, 128, 256, 7424, 1280, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 33, 0, 4096, 3695, 0, 0, 3373, 4087, 0, 0, 0, 0, 0, 490, 4067, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4091, 0, 2616, 23, 0, 0, 4096, 0, 3887, 0, 4096, 0, 0, 0, 1627, 0, 0, 4096, 75, 9, 0, 0, 599, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 438, 1, 0, 1198, 4096, 0, 0, 0, 0, 0, 2400, 4096, 0, 0, 0, 0, 0, 4096, 0, 3587, 0, 710, 85, 3699, 0, 2189, 7, 0, 0, 4096, 0, 4093, 0, 4096, 1, 0, 0, 3942, 0, 0, 4096, 18, 0, 0, 0, 2490, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 3809, 0, 0, 4, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 3916, 2087, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 3934, 4094, 0, 1, 0, 1, 0, 4092, 0, 4070, 0, 149, 77, 353, 7, 3769, 896, 0, 0, 4093, 0, 998, 0, 4096, 21, 93, 0, 4096, 0, 30, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 4064, 0, 145, 0, 0, 0, 4096, 0, 0, 0, 4071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 8, 0, 4096, 0, 1, 0, 2526, 4096, 0, 0, 0, 0, 0, 2667, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 32, 88, 4088, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 4096, 2, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 2988, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 3977, 0, 0, 13, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4031, 0, 70, 4, 1, 4095, 0, 1, 2, 0, 0, 1937, 4093, 0, 0, 13, 0, 0, 4063, 0, 4090, 0, 3134, 0, 4093, 32, 106, 0, 0, 2, 4096, 0, 2470, 0, 4089, 408, 66, 0, 3033, 0, 0, 4095, 2, 0, 0, 0, 3961, 0, 0, 3, 4083, 0, 1986, 8, 0, 0, 3423, 0, 4, 0, 4042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4089, 1531, 0, 0, 3124, 4096, 0, 0, 0, 0, 0, 924, 4096, 0, 0, 17, 0, 0, 4096, 0, 3987, 0, 0, 0, 4058, 0, 793, 71, 0, 0, 4096, 12, 4096, 0, 4096, 12, 0, 0, 3522, 0, 0, 4089, 0, 1, 0, 0, 2456, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4082] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[3, 19, 0, 3970, 210, 283, 4, 0, 4086, 1, 0, 16, 0, 0, 4091, 4086, 0, 4, 0, 2, 0, 3934, 0, 4096, 11, 2271, 0, 3249, 49, 886, 0, 0, 1, 4093, 58, 4079, 0, 4092, 0, 8, 3, 1527, 0, 0, 4068, 0, 10, 0, 2, 4092, 0, 1, 0, 4096, 1, 4, 1, 6, 0, 4095, 0, 0, 8, 4020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4, 0, 5, 3903, 431, 0, 43, 8, 4024, 0, 0, 27, 6, 0, 1038, 4050, 0, 49, 1, 0, 41, 4036, 62, 578, 0, 1170, 0, 4055, 498, 2249, 0, 0, 0, 4090, 185, 2620, 0, 4026, 2046, 609, 2, 2023, 0, 590, 4067, 0, 3, 0, 30, 4057, 19, 3, 7, 3467, 82, 3239, 0, 26, 45, 3926, 0, 36, 120, 3940] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 33, 0, 4096, 3695, 0, 0, 3373, 1729, 50, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 490, 1909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2616, 23, 0, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 3887, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 0, 0, 4096, 75, 9, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 0, 4096, 0, 0, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 162, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12976, 6486, 8719, 8722, 8358, 6831, 5090, 8354] avg=8192.0000 max=12976 max_rank=0 min=5090 min_rank=6 imbalance_ratio=1.583984 max_load_reduction=52560 max_load_reduction_pct=0.802002 moved_to_echo=28633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4031, 0, 70, 4, 1, 1729, 50, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 2, 0, 0, 1937, 1922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4063, 0, 4090, 530, 0, 0, 0, 0, 0, 0, 0, 0, 3134, 0, 4093, 32, 106, 0, 0, 1992, 0, 0, 0, 0, 0, 0, 0, 2, 375, 0, 2470, 0, 4089, 408, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3033, 0, 0, 4095, 2, 0, 0, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 0, 0, 3, 4083, 0, 1986, 398, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3423, 0, 4, 0, 4042, 163, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5885, 7957, 8696, 9357, 7410, 8160, 10431, 7640] avg=8192.0000 max=10431 max_rank=6 min=5885 min_rank=0 imbalance_ratio=1.273315 max_load_reduction=55105 max_load_reduction_pct=0.840836 moved_to_echo=33641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3916, 2087, 0, 0, 0, 1728, 50, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 3934, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4092, 0, 4070, 530, 0, 0, 0, 0, 0, 0, 0, 0, 149, 77, 353, 7, 3769, 896, 0, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 998, 0, 4096, 21, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 30, 4096, 0, 0, 0, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4064, 0, 145, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 163, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7781, 9955, 8694, 7241, 5583, 9252, 8700, 8330] avg=8192.0000 max=9955 max_rank=1 min=5583 min_rank=4 imbalance_ratio=1.215210 max_load_reduction=55581 max_load_reduction_pct=0.848099 moved_to_echo=31865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 438, 1, 0, 1198, 1729, 51, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2400, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3587, 530, 0, 0, 0, 0, 0, 0, 0, 0, 710, 85, 3699, 0, 2189, 7, 0, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 4093, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 0, 0, 4096, 18, 0, 0, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 2490, 0, 0, 0, 4096, 0, 1, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3809, 0, 0, 4, 4096, 163, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7513, 8419, 8213, 8682, 8565, 9087, 6985, 8072] avg=8192.0000 max=9087 max_rank=5 min=6985 min_rank=6 imbalance_ratio=1.109253 max_load_reduction=56449 max_load_reduction_pct=0.861343 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 4096, 0, 1, 0, 2526, 1729, 51, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2667, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 530, 0, 0, 0, 0, 0, 0, 0, 0, 32, 88, 4088, 0, 0, 0, 0, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 4090, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 2988, 0, 0, 0, 4096, 0, 5, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 0, 13, 4096, 163, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8411, 8686, 8722, 6200, 8563, 9218, 7487, 8249] avg=8192.0000 max=9218 max_rank=5 min=6200 min_rank=3 imbalance_ratio=1.125244 max_load_reduction=56318 max_load_reduction_pct=0.859344 moved_to_echo=33517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4089, 1531, 0, 0, 3124, 1729, 51, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 924, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4096, 0, 3987, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 793, 71, 0, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 375, 12, 4096, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3522, 0, 0, 4089, 0, 1, 0, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 2456, 0, 0, 0, 4096, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4082, 163, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10524, 6943, 8630, 6914, 8591, 8643, 6950, 8341] avg=8192.0000 max=10524 max_rank=0 min=6914 min_rank=3 imbalance_ratio=1.284668 max_load_reduction=55012 max_load_reduction_pct=0.839417 moved_to_echo=32525 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 19, 0, 3970, 210, 283, 4, 0, 1728, 50, 0, 0, 0, 0, 0, 0, 4086, 1, 0, 16, 0, 0, 4091, 1919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 3934, 0, 4096, 529, 0, 0, 0, 0, 0, 0, 0, 11, 2271, 0, 3249, 49, 886, 0, 0, 1990, 0, 0, 0, 0, 0, 0, 0, 1, 375, 58, 4079, 0, 4092, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1527, 0, 0, 4068, 0, 10, 0, 1028, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 0, 1, 0, 4096, 1, 4, 397, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 4095, 0, 0, 8, 4020, 163, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6267, 10113, 8565, 8456, 8613, 6636, 8593, 8293] avg=8192.0000 max=10113 max_rank=1 min=6267 min_rank=0 imbalance_ratio=1.234497 max_load_reduction=55423 max_load_reduction_pct=0.845688 moved_to_echo=32135 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 5, 3903, 431, 0, 43, 8, 1731, 54, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 27, 6, 0, 1038, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 1, 0, 41, 4036, 62, 578, 530, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 0, 4055, 498, 2249, 0, 0, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 367, 185, 2620, 0, 4026, 2046, 609, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2023, 0, 590, 4067, 0, 3, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 30, 4057, 19, 3, 7, 3467, 82, 3239, 396, 0, 0, 0, 0, 0, 0, 0, 0, 26, 45, 3926, 0, 36, 120, 3940, 164, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6179, 6977, 5297, 9964, 9853, 7709, 11300, 8257] avg=8192.0000 max=11300 max_rank=6 min=5297 min_rank=2 imbalance_ratio=1.379395 max_load_reduction=54236 max_load_reduction_pct=0.827576 moved_to_echo=37119 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 31616, 0, 128, 256, 32512, 1408, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 3072, 256, 26368, 0, 32768, 2560, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 17536, 15360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 24832, 128, 128, 128, 32128, 128, 5504, 3200, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 23936, 0, 640, 32768, 128, 128, 0, 8320, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 32512, 128, 28672, 4352, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 7552, 256, 27776, 640, 12672, 1024, 0, 16000, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32256, 8448, 384, 128, 10240, 13952, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True + [2026-08-30 05:00:50] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 680.1 | throughput per GPU (TFLOP/s/GPU): 490.2 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.313556E+01 | loss scale: 1.0 | grad norm: 39.442 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[23, 127, 1937, 15, 748, 1, 972, 12, 2940, 127, 515, 884, 584, 288, 18, 924, 367, 1517, 2144, 990, 3832, 1035, 293, 1804, 1, 3424, 108, 654, 927, 81, 506, 1105, 3851, 114, 97, 1387, 1128, 27, 1095, 2142, 3285, 2448, 955, 263, 0, 2300, 640, 1155, 231, 1385, 393, 35, 48, 58, 2936, 292, 2, 2878, 3560, 1486, 368, 893, 1169, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[12, 66, 2415, 3, 942, 0, 932, 3, 2047, 86, 259, 828, 919, 276, 0, 792, 268, 2342, 1907, 1254, 3698, 868, 289, 1744, 1, 3540, 120, 824, 285, 22, 130, 1449, 3994, 164, 47, 1080, 1633, 18, 790, 2393, 2531, 1835, 563, 192, 3, 2403, 1288, 1877, 181, 1850, 438, 26, 3, 59, 3186, 453, 0, 3064, 3779, 1436, 158, 1063, 681, 27] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[19, 91, 2189, 1, 580, 3, 970, 3, 2886, 80, 329, 670, 595, 236, 0, 732, 370, 2059, 2090, 636, 3860, 939, 258, 1859, 0, 3639, 121, 610, 409, 24, 228, 1148, 4018, 80, 66, 1019, 1366, 28, 759, 1866, 3002, 2800, 810, 245, 2, 2509, 959, 2355, 138, 1372, 419, 4, 14, 12, 2970, 314, 0, 2906, 3794, 1877, 222, 1304, 656, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[29, 239, 2617, 5, 824, 5, 1330, 22, 2566, 56, 283, 965, 905, 238, 2, 857, 208, 2424, 1646, 961, 3765, 856, 295, 1512, 2, 3388, 107, 733, 413, 19, 176, 1440, 4032, 125, 53, 788, 1158, 20, 798, 2240, 2689, 2388, 359, 222, 7, 2571, 1216, 2206, 131, 1401, 352, 11, 15, 59, 2823, 212, 6, 3392, 3911, 1845, 226, 1034, 355, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[17, 149, 2321, 6, 693, 0, 989, 5, 2655, 74, 267, 1297, 1444, 77, 8, 713, 358, 1873, 2182, 689, 3858, 1035, 425, 1926, 4, 3124, 162, 716, 341, 63, 215, 1621, 3820, 121, 66, 1122, 1139, 23, 580, 2028, 3325, 2673, 787, 473, 28, 1743, 921, 2070, 213, 1584, 428, 19, 2, 61, 2592, 166, 0, 3272, 3798, 1293, 183, 999, 629, 71] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[16, 145, 2014, 12, 830, 0, 884, 4, 2601, 70, 407, 1119, 864, 232, 5, 812, 294, 1541, 2089, 825, 3846, 892, 339, 1663, 1, 3432, 55, 1170, 366, 20, 210, 1068, 4001, 59, 36, 963, 1185, 17, 559, 2079, 3201, 2396, 538, 205, 23, 2533, 1083, 2501, 112, 1709, 535, 2, 7, 40, 3015, 196, 2, 3284, 3752, 1591, 342, 1341, 393, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[16, 56, 2291, 0, 525, 0, 1173, 3, 2789, 66, 286, 815, 348, 265, 2, 838, 386, 2211, 1953, 949, 3909, 770, 447, 1486, 0, 3702, 90, 625, 355, 8, 108, 1409, 3993, 108, 58, 1099, 1527, 17, 477, 2463, 3209, 2316, 228, 283, 7, 2795, 1223, 2111, 108, 1534, 508, 26, 4, 13, 2816, 151, 0, 3343, 3703, 1651, 208, 1267, 386, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[28, 100, 2159, 11, 477, 0, 825, 8, 2812, 182, 463, 1070, 912, 424, 6, 897, 290, 2175, 2253, 779, 3754, 1074, 310, 1886, 2, 3149, 120, 857, 483, 23, 223, 1545, 3882, 105, 64, 1583, 1359, 18, 485, 2332, 3027, 2238, 361, 170, 10, 2370, 1079, 1563, 178, 1539, 414, 15, 19, 45, 2553, 263, 1, 3310, 3715, 1595, 359, 1022, 540, 25] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[23, 127, 1937, 15, 748, 1, 972, 12, 3832, 288, 0, 0, 0, 0, 0, 0, 2940, 127, 515, 884, 584, 288, 18, 924, 2114, 0, 0, 0, 0, 0, 0, 0, 367, 1517, 2079, 990, 0, 1035, 293, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3424, 108, 654, 927, 81, 506, 1105, 1244, 450, 65, 19, 0, 0, 0, 0, 2607, 114, 97, 1387, 1128, 27, 1095, 2142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2448, 955, 263, 0, 1850, 640, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 231, 1385, 393, 35, 48, 58, 2936, 292, 2997, 83, 0, 0, 0, 0, 0, 0, 2, 2878, 1344, 1486, 368, 893, 1169, 12, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7955, 8394, 8085, 8584, 8597, 7311, 8458, 8152] avg=8192.0000 max=8597 max_rank=4 min=7311 min_rank=5 imbalance_ratio=1.049438 max_load_reduction=56939 max_load_reduction_pct=0.868820 moved_to_echo=32518 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[29, 239, 2617, 5, 824, 5, 1330, 22, 3765, 235, 0, 0, 0, 0, 0, 0, 2566, 56, 283, 965, 905, 238, 2, 857, 2323, 0, 0, 0, 0, 0, 0, 0, 208, 2424, 1596, 961, 0, 856, 295, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3388, 107, 733, 413, 19, 176, 1440, 1302, 503, 50, 21, 0, 0, 0, 0, 2730, 125, 53, 788, 1158, 20, 798, 2240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2388, 359, 222, 7, 2068, 1216, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 131, 1401, 352, 11, 15, 59, 2823, 212, 2454, 91, 0, 0, 0, 0, 0, 0, 6, 3392, 1476, 1845, 226, 1034, 355, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9071, 8195, 7852, 8154, 7912, 8466, 7549, 8337] avg=8192.0000 max=9071 max_rank=0 min=7549 min_rank=6 imbalance_ratio=1.107300 max_load_reduction=56465 max_load_reduction_pct=0.861588 moved_to_echo=32264 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[12, 66, 2415, 3, 942, 0, 932, 3, 3698, 221, 0, 0, 0, 0, 0, 0, 2047, 86, 259, 828, 919, 276, 0, 792, 2244, 0, 0, 0, 0, 0, 0, 0, 268, 2342, 1849, 1254, 0, 868, 289, 1744, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3540, 120, 824, 285, 22, 130, 1449, 1290, 470, 58, 20, 0, 0, 0, 0, 2704, 164, 47, 1080, 1633, 18, 790, 2393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 563, 192, 3, 1933, 1288, 1877, 0, 0, 0, 0, 0, 0, 0, 0, 181, 1850, 438, 26, 3, 59, 3186, 453, 2310, 88, 0, 0, 0, 0, 0, 0, 0, 3064, 1427, 1436, 158, 1063, 681, 27, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8292, 7451, 8614, 8209, 8829, 7691, 8594, 7856] avg=8192.0000 max=8829 max_rank=4 min=7451 min_rank=1 imbalance_ratio=1.077759 max_load_reduction=56707 max_load_reduction_pct=0.865280 moved_to_echo=32970 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[17, 149, 2321, 6, 693, 0, 989, 5, 3858, 290, 0, 0, 0, 0, 0, 0, 2655, 74, 267, 1297, 1444, 77, 8, 713, 2256, 0, 0, 0, 0, 0, 0, 0, 358, 1873, 2116, 689, 0, 1035, 425, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3124, 162, 716, 341, 63, 215, 1621, 1234, 341, 66, 20, 0, 0, 0, 0, 2586, 121, 66, 1122, 1139, 23, 580, 2028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2673, 787, 473, 28, 1402, 921, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 213, 1584, 428, 19, 2, 61, 2592, 166, 3035, 88, 0, 0, 0, 0, 0, 0, 0, 3272, 1434, 1293, 183, 999, 629, 71, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8328, 8791, 8422, 7907, 7665, 8354, 8188, 7881] avg=8192.0000 max=8791 max_rank=1 min=7665 min_rank=4 imbalance_ratio=1.073120 max_load_reduction=56745 max_load_reduction_pct=0.865860 moved_to_echo=32088 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[19, 91, 2189, 1, 580, 3, 970, 3, 3860, 263, 0, 0, 0, 0, 0, 0, 2886, 80, 329, 670, 595, 236, 0, 732, 2253, 0, 0, 0, 0, 0, 0, 0, 370, 2059, 2026, 636, 0, 939, 258, 1859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3639, 121, 610, 409, 24, 228, 1148, 1298, 491, 64, 20, 0, 0, 0, 0, 2720, 80, 66, 1019, 1366, 28, 759, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2800, 810, 245, 2, 2018, 959, 2355, 0, 0, 0, 0, 0, 0, 0, 0, 138, 1372, 419, 4, 14, 12, 2970, 314, 2739, 88, 0, 0, 0, 0, 0, 0, 0, 2906, 1433, 1877, 222, 1304, 656, 16, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7979, 7781, 8147, 8052, 7904, 9189, 8070, 8414] avg=8192.0000 max=9189 max_rank=5 min=7781 min_rank=1 imbalance_ratio=1.121704 max_load_reduction=56347 max_load_reduction_pct=0.859787 moved_to_echo=33577 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 56, 2291, 0, 525, 0, 1173, 3, 3909, 280, 0, 0, 0, 0, 0, 0, 2789, 66, 286, 815, 348, 265, 2, 838, 2199, 0, 0, 0, 0, 0, 0, 0, 386, 2211, 1894, 949, 0, 770, 447, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3702, 90, 625, 355, 8, 108, 1409, 1290, 547, 59, 20, 0, 0, 0, 0, 2703, 108, 58, 1099, 1527, 17, 477, 2463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2316, 228, 283, 7, 2248, 1223, 2111, 0, 0, 0, 0, 0, 0, 0, 0, 108, 1534, 508, 26, 4, 13, 2816, 151, 2929, 86, 0, 0, 0, 0, 0, 0, 0, 3343, 1398, 1651, 208, 1267, 386, 23, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 7608, 8143, 8213, 8452, 8416, 8175, 8276] avg=8192.0000 max=8452 max_rank=4 min=7608 min_rank=1 imbalance_ratio=1.031738 max_load_reduction=57084 max_load_reduction_pct=0.871033 moved_to_echo=33319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 145, 2014, 12, 830, 0, 884, 4, 3846, 280, 0, 0, 0, 0, 0, 0, 2601, 70, 407, 1119, 864, 232, 5, 812, 2228, 0, 0, 0, 0, 0, 0, 0, 294, 1541, 2025, 825, 0, 892, 339, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3432, 55, 1170, 366, 20, 210, 1068, 1292, 495, 64, 20, 0, 0, 0, 0, 2709, 59, 36, 963, 1185, 17, 559, 2079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2396, 538, 205, 23, 2038, 1083, 2501, 0, 0, 0, 0, 0, 0, 0, 0, 112, 1709, 535, 2, 7, 40, 3015, 196, 2921, 87, 0, 0, 0, 0, 0, 0, 2, 3284, 1417, 1591, 342, 1341, 393, 10, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8031, 8338, 7579, 8193, 7607, 8784, 8624, 8380] avg=8192.0000 max=8784 max_rank=5 min=7579 min_rank=2 imbalance_ratio=1.072266 max_load_reduction=56752 max_load_reduction_pct=0.865967 moved_to_echo=33395 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[28, 100, 2159, 11, 477, 0, 825, 8, 3754, 265, 0, 0, 0, 0, 0, 0, 2812, 182, 463, 1070, 912, 424, 6, 897, 2212, 0, 0, 0, 0, 0, 0, 0, 290, 2175, 2180, 779, 0, 1074, 310, 1886, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3149, 120, 857, 483, 23, 223, 1545, 1258, 466, 73, 25, 0, 0, 0, 0, 2624, 105, 64, 1583, 1359, 18, 485, 2332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2238, 361, 170, 10, 1904, 1079, 1563, 0, 0, 0, 0, 0, 0, 0, 0, 178, 1539, 414, 15, 19, 45, 2553, 263, 2762, 90, 0, 0, 0, 0, 0, 0, 1, 3310, 1388, 1595, 359, 1022, 540, 25, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7627, 8978, 8694, 8224, 8570, 7325, 7878, 8240] avg=8192.0000 max=8978 max_rank=1 min=7325 min_rank=5 imbalance_ratio=1.095947 max_load_reduction=56558 max_load_reduction_pct=0.863007 moved_to_echo=32013 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[21504, 896, 512, 9088, 10496, 256, 5632, 17664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 19200, 4608, 2176, 128, 15488, 8448, 15872, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2560, 16256, 15872, 7168, 0, 7552, 2688, 13952, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 27520, 896, 6272, 3584, 384, 1920, 10880, 10240, 3840, 512, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 25472, 11392, 12800, 2176, 8960, 4864, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[21376, 768, 2816, 7680, 6656, 2048, 128, 6656, 17920, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1408, 12416, 3584, 256, 128, 384, 22912, 2048, 22272, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 1024, 18048, 128, 5632, 128, 8192, 128, 30592, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 2, 0, 0, 4, 4093, 5, 4, 4072, 0, 4093, 4088, 4093, 0, 0, 1, 1, 4096, 4064, 4094, 0, 0, 1193, 16, 0, 0, 118, 0, 0, 0, 4091, 4082, 0, 0, 0, 0, 2, 3952, 0, 0, 3, 4096, 4095, 0, 1, 0, 0, 4092, 1, 8, 0, 3, 29, 0, 0, 2, 0, 2939, 3, 4096, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 8, 0, 4044, 0, 3779, 4094, 4095, 0, 0, 1, 0, 4095, 4076, 4090, 1, 2, 4073, 7, 0, 0, 15, 1, 0, 1, 4096, 4089, 0, 0, 0, 2, 0, 4039, 1, 0, 0, 4096, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 1, 1, 1, 445, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 4096, 2, 45, 3674, 0, 4032, 4094, 4075, 0, 0, 0, 0, 4096, 4096, 4093, 0, 17, 3774, 6, 0, 0, 3, 28, 0, 0, 4096, 3889, 5, 0, 0, 1, 0, 4091, 0, 1, 0, 4086, 4094, 0, 0, 2, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 946, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 4096, 0, 2, 3941, 1, 4084, 4076, 4096, 1, 5, 1, 4, 4094, 4066, 4083, 0, 25, 3373, 22, 0, 2, 5, 0, 3, 0, 4094, 4037, 14, 0, 12, 7, 0, 3648, 0, 0, 1, 4096, 4094, 0, 0, 0, 0, 4093, 24, 0, 0, 4, 0, 0, 0, 1, 1, 1334, 0, 4093, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 4096, 33, 2, 3907, 0, 3487, 4096, 4095, 0, 0, 0, 0, 4096, 4091, 4096, 0, 11, 2646, 0, 0, 0, 57, 13, 0, 0, 4068, 4094, 0, 1, 0, 2, 2, 3981, 0, 1, 0, 4091, 4096, 0, 0, 0, 0, 4093, 0, 56, 0, 0, 5, 0, 0, 0, 0, 2221, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 20, 0, 4080, 0, 4083, 4096, 4096, 1, 0, 0, 0, 4096, 4094, 4096, 0, 2, 1237, 1, 0, 0, 5, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2868, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2, 2, 10, 0, 0, 6, 2, 4086, 0, 0, 2983, 2, 4008, 4045, 4072, 0, 24, 0, 2, 4046, 4085, 4082, 3, 18, 4054, 51, 6, 5, 298, 0, 12, 2, 4018, 4061, 13, 0, 0, 2, 0, 4087, 14, 0, 6, 4083, 4079, 1, 11, 77, 0, 4055, 23, 34, 1, 1, 50, 0, 13, 0, 0, 940, 0, 4059, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4, 0, 2, 0, 4, 0, 3, 4089, 49, 2, 4009, 0, 4033, 4094, 4093, 12, 0, 2, 0, 4094, 4084, 4096, 4, 7, 2322, 23, 11, 0, 507, 0, 0, 0, 4079, 4088, 0, 13, 0, 0, 0, 3896, 1, 3, 2, 4092, 4094, 0, 0, 28, 0, 4092, 0, 0, 1, 0, 1, 0, 1, 4, 0, 1517, 0, 4080, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 2, 0, 0, 4, 4093, 2972, 1125, 0, 0, 0, 0, 0, 0, 5, 4, 4072, 0, 4093, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 4064, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 16, 0, 0, 118, 0, 0, 0, 4093, 1117, 0, 0, 0, 0, 0, 0, 156, 4082, 0, 0, 0, 0, 2, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4084, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 8, 0, 3, 29, 0, 2810, 1270, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2939, 3, 4096, 3, 0, 2437, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8555, 8167, 6537, 8192, 8184, 8213, 9491] avg=8192.0000 max=9491 max_rank=7 min=6537 min_rank=3 imbalance_ratio=1.158569 max_load_reduction=56045 max_load_reduction_pct=0.855179 moved_to_echo=34080 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 2973, 1126, 0, 0, 0, 0, 0, 0, 20, 0, 4080, 0, 4083, 382, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1237, 1, 0, 0, 5, 0, 0, 0, 4096, 1117, 0, 0, 0, 0, 0, 0, 156, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2814, 1272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2868, 0, 4095, 0, 0, 2442, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8566, 8198, 6456, 8345, 8178, 8182, 9416] avg=8192.0000 max=9416 max_rank=7 min=6456 min_rank=3 imbalance_ratio=1.149414 max_load_reduction=56120 max_load_reduction_pct=0.856323 moved_to_echo=34121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 4096, 2971, 1126, 0, 0, 0, 0, 0, 0, 2, 45, 3674, 0, 4032, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 6, 0, 0, 3, 28, 0, 0, 4075, 1116, 0, 0, 0, 0, 0, 0, 156, 3889, 5, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 4083, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2814, 1272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 0, 4096, 0, 0, 2441, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8134, 8215, 9002, 8142, 8172, 8183, 7494] avg=8192.0000 max=9002 max_rank=3 min=7494 min_rank=7 imbalance_ratio=1.098877 max_load_reduction=56534 max_load_reduction_pct=0.862640 moved_to_echo=31991 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 4096, 2973, 1119, 0, 0, 0, 0, 0, 0, 33, 2, 3907, 0, 3487, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 3, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2646, 0, 0, 0, 57, 13, 0, 0, 4095, 1120, 0, 0, 0, 0, 0, 0, 155, 4094, 0, 1, 0, 2, 2, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 56, 0, 0, 5, 0, 2794, 1272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2221, 0, 4096, 0, 0, 2442, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7811, 8201, 7931, 8235, 8177, 8220, 8770] avg=8192.0000 max=8770 max_rank=7 min=7811 min_rank=1 imbalance_ratio=1.070557 max_load_reduction=56766 max_load_reduction_pct=0.866180 moved_to_echo=33402 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 2969, 1126, 0, 0, 0, 0, 0, 0, 8, 0, 4044, 0, 3779, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4076, 5, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 7, 0, 0, 15, 1, 0, 1, 4095, 1116, 0, 0, 0, 0, 0, 0, 156, 4089, 0, 0, 0, 2, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 2814, 1272, 0, 0, 0, 0, 0, 0, 1, 1, 1, 445, 0, 4095, 0, 0, 2441, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8212, 8180, 9308, 8286, 8182, 8183, 6995] avg=8192.0000 max=9308 max_rank=3 min=6995 min_rank=7 imbalance_ratio=1.136230 max_load_reduction=56228 max_load_reduction_pct=0.857971 moved_to_echo=31646 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 10, 0, 0, 6, 2, 4086, 2963, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 2983, 2, 4008, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 2, 4046, 4085, 6, 3, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 51, 6, 5, 298, 0, 12, 2, 4072, 1113, 0, 0, 0, 0, 0, 0, 153, 4061, 13, 0, 0, 2, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 6, 4083, 4068, 1, 11, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 23, 34, 1, 1, 50, 0, 2760, 1257, 0, 0, 0, 0, 0, 0, 13, 0, 0, 940, 0, 4059, 2, 0, 2411, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 7370, 8184, 9613, 8316, 8260, 8181, 7436] avg=8192.0000 max=9613 max_rank=3 min=7370 min_rank=1 imbalance_ratio=1.173462 max_load_reduction=55923 max_load_reduction_pct=0.853317 moved_to_echo=32193 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 4096, 2964, 1126, 0, 0, 0, 0, 0, 0, 0, 2, 3941, 1, 4084, 380, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 4, 4094, 4066, 5, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 3373, 22, 0, 2, 5, 0, 3, 0, 4096, 1114, 0, 0, 0, 0, 0, 0, 156, 4037, 14, 0, 12, 7, 0, 3648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 24, 0, 0, 4, 0, 0, 2812, 1266, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1334, 0, 4093, 0, 1, 2430, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8409, 8200, 8615, 7874, 8180, 8199, 7871] avg=8192.0000 max=8615 max_rank=3 min=7871 min_rank=7 imbalance_ratio=1.051636 max_load_reduction=56921 max_load_reduction_pct=0.868546 moved_to_echo=32124 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 2, 0, 4, 0, 3, 4089, 2978, 1125, 0, 0, 0, 0, 0, 0, 49, 2, 4009, 0, 4033, 374, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 4084, 0, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2322, 23, 11, 0, 507, 0, 0, 0, 4093, 1118, 0, 0, 0, 0, 0, 0, 149, 4088, 0, 13, 0, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 2, 4092, 4077, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 1, 0, 2805, 1276, 0, 0, 0, 0, 0, 0, 1, 4, 0, 1517, 0, 4080, 0, 0, 2444, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8479, 8191, 8074, 8146, 8203, 8175, 8063] avg=8192.0000 max=8479 max_rank=1 min=8063 min_rank=7 imbalance_ratio=1.035034 max_load_reduction=57057 max_load_reduction_pct=0.870621 moved_to_echo=32587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1280, 32512, 128, 128, 128, 128, 128, 31872, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 0, 22528, 10240, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[22784, 128, 128, 128, 1024, 128, 128, 128, 32768, 8960, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 32768, 23808, 9088, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 13312, 128, 32768, 128, 128, 19584, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 30720, 128, 31616, 3072, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4, 4095, 0, 4096, 0, 0, 4016, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 2264, 0, 1, 4096, 3939, 1, 0, 743, 0, 1, 0, 0, 3954, 0, 0, 4096, 0, 1, 4095, 4, 46, 0, 0, 2, 0, 0, 1243, 0, 10, 0, 161, 4093, 0, 4092, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4094, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 4094, 0, 0, 4021, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4023, 0, 0, 4096, 495, 52, 0, 998, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 1390, 0, 19, 0, 1024, 4094, 0, 4096, 0, 0, 262, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 53, 4064, 3, 4089, 0, 2, 3387, 4091, 4084, 1, 5, 5, 0, 3996, 0, 0, 4011, 24, 3, 4022, 2610, 142, 0, 3, 6, 1, 0, 15, 4090, 4, 0, 4045, 0, 0, 4072, 47, 0, 22, 0, 1, 51, 16, 1622, 14, 8, 12, 589, 4055, 0, 4072, 3, 5, 11, 5, 2, 4078, 1, 0, 0, 4079, 10, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 23, 4081, 0, 4096, 0, 6, 3656, 4096, 4078, 0, 0, 4, 0, 4096, 0, 0, 3921, 0, 0, 4096, 3794, 1, 0, 71, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 3, 0, 0, 0, 0, 0, 540, 0, 87, 0, 269, 4035, 0, 4096, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 22, 4080, 0, 4087, 0, 4, 3998, 4094, 4096, 1, 0, 5, 0, 4096, 0, 0, 4069, 0, 0, 4095, 3877, 3, 1, 74, 0, 16, 0, 0, 4091, 1, 0, 4094, 2, 0, 4093, 7, 0, 1, 0, 2, 0, 1, 178, 2, 0, 1, 51, 4090, 1, 4094, 1, 2, 18, 0, 0, 4095, 0, 0, 0, 4092, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 4089, 0, 4094, 0, 0, 3563, 4095, 4096, 0, 0, 1, 0, 4093, 0, 0, 4049, 0, 0, 4095, 3682, 220, 0, 59, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 4092, 3, 8, 0, 0, 0, 0, 1, 477, 0, 1, 1, 12, 4095, 1, 4096, 0, 0, 225, 0, 1, 4096, 0, 2, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4088, 0, 4096, 0, 0, 3140, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 3056, 0, 0, 4096, 3811, 203, 0, 971, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4084, 0, 3, 0, 0, 0, 0, 0, 897, 0, 36, 0, 222, 4056, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 23, 4096, 0, 4090, 0, 0, 1989, 4091, 4093, 4, 4, 0, 3, 4092, 0, 3, 4041, 6, 2, 4094, 3572, 108, 2, 1317, 0, 0, 0, 2, 4094, 0, 0, 4094, 0, 0, 4093, 0, 16, 0, 3, 0, 0, 0, 513, 2, 23, 0, 586, 4035, 3, 4093, 2, 2, 153, 1, 6, 4093, 0, 0, 0, 4092, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4095, 0, 618, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2264, 0, 1, 1197, 3939, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 743, 0, 1, 0, 0, 3954, 0, 0, 2899, 623, 57, 7, 0, 0, 0, 0, 4096, 0, 1, 4087, 4, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1243, 0, 10, 0, 161, 4082, 2855, 0, 0, 0, 0, 0, 0, 4093, 0, 4035, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8733, 8205, 7402, 8284, 8234, 8353, 8129, 8196] avg=8192.0000 max=8733 max_rank=0 min=7402 min_rank=2 imbalance_ratio=1.066040 max_load_reduction=56803 max_load_reduction_pct=0.866745 moved_to_echo=32912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 619, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 1197, 495, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 998, 0, 0, 0, 0, 4096, 0, 0, 2899, 622, 57, 7, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1390, 0, 19, 0, 1024, 4082, 2853, 0, 0, 0, 0, 0, 0, 4094, 0, 4039, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8735, 8206, 5767, 8679, 8192, 9368, 8395, 8194] avg=8192.0000 max=9368 max_rank=5 min=5767 min_rank=2 imbalance_ratio=1.143555 max_load_reduction=56168 max_load_reduction_pct=0.857056 moved_to_echo=34149 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 53, 4064, 3, 617, 0, 2, 3387, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4084, 1, 5, 5, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 24, 3, 1175, 2610, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 1, 0, 15, 4090, 4, 0, 2847, 622, 57, 7, 0, 0, 0, 0, 4045, 0, 0, 4064, 47, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 51, 16, 1622, 14, 8, 12, 589, 4077, 2850, 0, 0, 0, 0, 0, 0, 4055, 0, 4015, 3, 5, 11, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 0, 0, 4079, 10, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8105, 7965, 7652, 8178, 9240, 8096, 8173] avg=8192.0000 max=9240 max_rank=5 min=7652 min_rank=3 imbalance_ratio=1.127930 max_load_reduction=56296 max_load_reduction_pct=0.859009 moved_to_echo=33687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 23, 4081, 0, 618, 0, 6, 3656, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4078, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 0, 1197, 3794, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 4096, 0, 0, 2899, 623, 57, 7, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 0, 87, 0, 269, 4082, 2855, 0, 0, 0, 0, 0, 0, 4035, 0, 4039, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8384, 8192, 8913, 7753, 8186, 7833, 8080, 8195] avg=8192.0000 max=8913 max_rank=2 min=7753 min_rank=3 imbalance_ratio=1.088013 max_load_reduction=56623 max_load_reduction_pct=0.863998 moved_to_echo=32294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4088, 0, 618, 0, 0, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 0, 0, 1197, 3811, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 0, 0, 0, 0, 4096, 0, 0, 2899, 623, 57, 7, 0, 0, 0, 0, 4096, 0, 0, 4076, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 0, 36, 0, 222, 4082, 2855, 0, 0, 0, 0, 0, 0, 4056, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7846, 8206, 8267, 8653, 8175, 8092, 8095, 8202] avg=8192.0000 max=8653 max_rank=3 min=7846 min_rank=0 imbalance_ratio=1.056274 max_load_reduction=56883 max_load_reduction_pct=0.867966 moved_to_echo=32564 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 4080, 0, 617, 0, 4, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 1, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 1197, 3877, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 16, 0, 0, 4091, 1, 0, 2898, 621, 57, 7, 0, 0, 0, 0, 4094, 2, 0, 4085, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 178, 2, 0, 1, 51, 4080, 2849, 0, 0, 0, 0, 0, 0, 4090, 1, 4037, 1, 2, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4092, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8721, 8212, 9147, 7765, 8189, 7164, 8149, 8189] avg=8192.0000 max=9147 max_rank=2 min=7164 min_rank=5 imbalance_ratio=1.116577 max_load_reduction=56389 max_load_reduction_pct=0.860428 moved_to_echo=31691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4089, 0, 619, 0, 0, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 1197, 3682, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 4096, 0, 0, 2898, 622, 57, 7, 0, 0, 0, 0, 4095, 0, 1, 4084, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 477, 0, 1, 1, 12, 4081, 2853, 0, 0, 0, 0, 0, 0, 4095, 1, 4039, 0, 0, 225, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8204, 9148, 7739, 8191, 7426, 8361, 8195] avg=8192.0000 max=9148 max_rank=2 min=7426 min_rank=5 imbalance_ratio=1.116699 max_load_reduction=56388 max_load_reduction_pct=0.860413 moved_to_echo=32173 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 23, 4096, 0, 610, 0, 0, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4093, 4, 4, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4041, 6, 2, 1193, 3572, 108, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 2, 4094, 0, 0, 2901, 625, 64, 8, 0, 0, 0, 0, 4094, 0, 0, 4078, 0, 16, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 2, 23, 0, 586, 4081, 2855, 0, 0, 0, 0, 0, 0, 4035, 3, 4029, 2, 2, 153, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4092, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6718, 8206, 8927, 9011, 8191, 8060, 8231, 8192] avg=8192.0000 max=9011 max_rank=3 min=6718 min_rank=0 imbalance_ratio=1.099976 max_load_reduction=56525 max_load_reduction_pct=0.862503 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 4992, 0, 128, 27776, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 29440, 128, 128, 9600, 25856, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 6912, 128, 256, 128, 2944, 32768, 22912, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4352, 128, 128, 0, 128, 32640, 128, 0, 23168, 4992, 512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32640, 128, 32384, 128, 128, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 0, 4046, 2, 0, 0, 4084, 75, 4096, 0, 3979, 0, 0, 12, 0, 0, 4093, 0, 1, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 4096, 4093, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 22, 0, 47, 0, 0, 0, 4096, 4096, 0, 2, 0, 37, 0, 0, 4096, 2, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4094, 0, 3, 0, 4095, 1, 4096, 0, 4088, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 10, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4091, 0, 0, 4037, 0, 0, 0, 4096, 4, 4096, 0, 4050, 0, 0, 0, 0, 0, 4096, 0, 46, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4058, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 32, 0, 46, 0, 0, 0, 4096, 4096, 0, 0, 0, 20, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4091, 0, 13, 0, 4096, 2, 4096, 0, 4089, 0, 0, 0, 0, 1, 4094, 4, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4068, 4095, 0, 0, 6, 0, 1, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 2, 0, 13, 0, 0, 4096, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[3991, 1, 21, 4081, 0, 177, 8, 4075, 80, 4086, 14, 3826, 3, 0, 16, 5, 8, 4038, 0, 0, 9, 4093, 19, 0, 4082, 3, 0, 0, 1, 13, 4087, 27, 4083, 3942, 5, 0, 35, 0, 0, 0, 0, 14, 0, 7, 4071, 12, 0, 4, 0, 25, 15, 1, 0, 4088, 4081, 1, 0, 9, 194, 5, 0, 4083, 8, 4089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4052, 0, 2, 0, 4021, 0, 4096, 0, 4085, 0, 0, 2, 0, 0, 4090, 0, 3, 0, 4075, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4091, 0, 0, 86, 0, 9, 0, 0, 0, 4096, 4096, 0, 11, 0, 12, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4089, 0, 0, 4094, 0, 6, 0, 4095, 3, 4094, 0, 4096, 0, 2, 10, 0, 2, 4068, 7, 22, 0, 4093, 0, 0, 4094, 2, 0, 0, 0, 0, 4095, 0, 4086, 4070, 1, 0, 0, 2, 0, 0, 2, 0, 2, 1, 4093, 1, 0, 0, 0, 2, 0, 2, 0, 4094, 4094, 0, 0, 3, 50, 0, 0, 4068, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4092, 2, 0, 4069, 0, 0, 2, 4093, 23, 4092, 0, 4094, 1, 1, 2, 2, 0, 4059, 0, 0, 5, 4096, 3, 0, 4093, 2, 2, 11, 3, 4, 4090, 1, 4079, 4065, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 29, 3, 5, 3, 0, 6, 4088, 4092, 0, 0, 5, 25, 1, 0, 4093, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 4046, 2, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 75, 4096, 0, 3979, 0, 0, 12, 0, 14, 12, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 4090, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 1, 1, 0, 30, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 22, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 37, 0, 0, 4096, 2, 4054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8188, 8189, 8182, 8226, 8183, 8223, 8191] avg=8192.0000 max=8226 max_rank=4 min=8154 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4094, 0, 3, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4088, 0, 0, 0, 0, 14, 12, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 30, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8211, 8194, 8189, 8226, 8174, 8187, 8150] avg=8192.0000 max=8226 max_rank=4 min=8150 min_rank=7 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 4037, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4050, 0, 0, 0, 0, 14, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 46, 0, 4095, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4058, 0, 0, 0, 0, 1, 0, 30, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 32, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4096, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8176, 8242, 8189, 8190, 8207, 8222, 8170] avg=8192.0000 max=8242 max_rank=2 min=8140 min_rank=0 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3991, 1, 21, 4081, 0, 177, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 80, 4086, 14, 3826, 3, 0, 16, 5, 14, 12, 0, 0, 0, 0, 0, 0, 8, 4038, 0, 0, 9, 4093, 19, 0, 3, 2, 0, 0, 0, 0, 0, 0, 4079, 3, 0, 0, 1, 13, 4087, 27, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3942, 5, 0, 35, 0, 0, 0, 30, 5, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 4071, 12, 0, 4, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 25, 15, 1, 0, 4072, 4081, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 194, 5, 0, 4083, 8, 4047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8056, 8172, 8210, 8100, 8166, 8195, 8346] avg=8192.0000 max=8346 max_rank=7 min=8056 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4091, 0, 13, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4089, 0, 0, 0, 0, 14, 12, 0, 0, 0, 0, 0, 0, 1, 4094, 4, 0, 0, 4093, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4095, 0, 0, 6, 0, 1, 0, 30, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 13, 0, 0, 4096, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8213, 8197, 8189, 8205, 8182, 8175, 8163] avg=8192.0000 max=8213 max_rank=1 min=8163 min_rank=7 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4052, 0, 2, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4085, 0, 0, 2, 0, 14, 12, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3, 0, 4075, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 0, 0, 0, 30, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4091, 0, 0, 86, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 12, 0, 0, 4096, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8209, 8173, 8189, 8199, 8246, 8185, 8173] avg=8192.0000 max=8246 max_rank=5 min=8162 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 0, 4094, 0, 6, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 4096, 0, 2, 10, 0, 14, 12, 0, 0, 0, 0, 0, 0, 2, 4068, 7, 22, 0, 4093, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4070, 1, 0, 0, 2, 0, 0, 30, 5, 0, 0, 0, 0, 0, 0, 2, 0, 2, 1, 4093, 1, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 4078, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 50, 0, 0, 4068, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8231, 8197, 8188, 8194, 8177, 8176, 8172] avg=8192.0000 max=8231 max_rank=1 min=8172 min_rank=7 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4092, 2, 0, 4069, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4092, 0, 4094, 1, 1, 2, 2, 21, 16, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 5, 4096, 3, 0, 6, 3, 0, 0, 0, 0, 0, 0, 4087, 2, 2, 11, 3, 4, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4065, 11, 0, 0, 0, 0, 0, 31, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 29, 4077, 0, 0, 0, 0, 0, 0, 0, 3, 5, 3, 0, 6, 4064, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 25, 1, 0, 4093, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8252, 8172, 8200, 8196, 8201, 8173, 8171] avg=8192.0000 max=8252 max_rank=1 min=8171 min_rank=0 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32640, 128, 128, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 32768, 128, 32384, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 256, 128, 128, 128, 32640, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 256, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 32640, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 32512, 128, 0, 128, 128, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 384, 128, 0, 32768, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 5, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 4093, 2, 4096, 0, 0, 9, 4045, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 119, 4096, 2, 0, 2, 0, 4028, 4095, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 4084, 0, 0, 4096, 0, 4083, 0, 0, 0, 4039, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 62, 0, 4, 0, 4089, 4096, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4093, 0, 0, 0, 4060, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 21, 4096, 29, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[16, 0, 0, 4077, 10, 22, 20, 6, 0, 16, 7, 4068, 4066, 14, 0, 4080, 0, 4058, 15, 12, 32, 4064, 0, 12, 0, 4040, 0, 16, 4051, 34, 0, 0, 2, 4075, 0, 0, 0, 8, 4080, 4081, 0, 6, 0, 12, 4072, 3, 2, 11, 29, 4057, 2, 68, 0, 0, 1, 13, 4065, 22, 20, 9, 4078, 6, 4075, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 0, 0, 4096, 0, 4095, 0, 0, 0, 4091, 1, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4, 4096, 1, 0, 0, 1, 4096, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 4092, 4056, 0, 0, 4096, 0, 4076, 0, 1, 0, 4096, 2, 0, 0, 4086, 7, 1, 4091, 0, 0, 3, 0, 4096, 25, 6, 2, 5, 4096, 4094, 33, 2, 0, 0, 4094, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 4095, 0, 4044, 0, 0, 0, 3801, 0, 0, 0, 4080, 0, 0, 4096, 0, 0, 0, 275, 4096, 96, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[3, 4, 0, 4092, 0, 2, 4, 3, 0, 3, 41, 4089, 4091, 3, 0, 4088, 0, 4051, 0, 11, 0, 4092, 0, 2, 0, 4091, 0, 2, 4090, 6, 0, 1, 0, 4092, 4, 0, 0, 14, 4092, 4092, 0, 4, 0, 4, 4092, 3, 2, 0, 1, 4075, 0, 0, 0, 0, 0, 1, 4093, 4, 7, 0, 4093, 2, 4087, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 1, 4096, 0, 0, 0, 0, 4083, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 9, 4045, 0, 0, 59, 9, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 119, 4029, 2, 0, 2, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 4075, 11, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8197, 8220, 8190, 8180, 8185, 8183, 8191] avg=8192.0000 max=8220 max_rank=2 min=8180 min_rank=4 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32739 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4085, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4060, 0, 0, 59, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 21, 4029, 29, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4078, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8201, 8221, 8201, 8163, 8185, 8189, 8191] avg=8192.0000 max=8221 max_rank=2 min=8163 min_rank=4 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32728 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 4077, 10, 22, 20, 6, 4067, 5, 0, 0, 0, 0, 0, 0, 0, 16, 7, 4068, 4066, 14, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 15, 12, 32, 4064, 0, 12, 59, 9, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 16, 4051, 34, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 2, 4008, 0, 0, 0, 8, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 12, 4072, 3, 2, 11, 4062, 11, 0, 0, 0, 0, 0, 0, 29, 4057, 2, 68, 0, 0, 1, 13, 4076, 0, 0, 0, 0, 0, 0, 0, 4065, 22, 20, 9, 0, 6, 4074, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8180, 8261, 8150, 8098, 8179, 8246, 8199] avg=8192.0000 max=8261 max_rank=2 min=8098 min_rank=4 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32722 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4085, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4084, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4039, 0, 0, 59, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 4029, 62, 0, 4, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 4078, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8185, 8190, 8201, 8184, 8208, 8191, 8191] avg=8192.0000 max=8208 max_rank=5 min=8184 min_rank=4 imbalance_ratio=1.001953 max_load_reduction=57328 max_load_reduction_pct=0.874756 moved_to_echo=32774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4085, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4091, 1, 0, 59, 9, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 4, 4029, 1, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 4078, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8196, 8255, 8200, 8131, 8189, 8188, 8191] avg=8192.0000 max=8255 max_rank=2 min=8131 min_rank=4 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32699 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 4083, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 4056, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 1, 0, 4096, 2, 0, 59, 9, 0, 0, 0, 0, 0, 0, 0, 4086, 7, 1, 4091, 0, 0, 3, 8, 1, 0, 0, 0, 0, 0, 0, 0, 4029, 25, 6, 2, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 2, 0, 0, 4094, 0, 0, 0, 4078, 11, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8158, 8243, 8197, 8163, 8218, 8184, 8191] avg=8192.0000 max=8243 max_rank=2 min=8158 min_rank=1 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4096, 0, 0, 0, 0, 4085, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 3801, 0, 0, 59, 9, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 275, 4029, 96, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4077, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 1, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8188, 7913, 8185, 8496, 8184, 8191, 8191] avg=8192.0000 max=8496 max_rank=4 min=7913 min_rank=2 imbalance_ratio=1.037109 max_load_reduction=57040 max_load_reduction_pct=0.870361 moved_to_echo=33062 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 4, 0, 4092, 0, 2, 4, 3, 4083, 5, 0, 0, 0, 0, 0, 0, 0, 3, 41, 4089, 4091, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 11, 0, 4092, 0, 2, 66, 11, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2, 4090, 6, 0, 1, 15, 7, 0, 0, 0, 0, 0, 0, 0, 4011, 4, 0, 0, 14, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 4092, 3, 2, 0, 4073, 10, 0, 0, 0, 0, 0, 0, 1, 4075, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 7, 0, 0, 2, 4080, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8231, 8233, 8212, 8121, 8188, 8164, 8191] avg=8192.0000 max=8233 max_rank=2 min=8121 min_rank=4 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32664 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 32256, 256, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 32768, 128, 128, 32768, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32640, 128, 128, 128, 32384, 128, 128, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 0, 0, 3745, 4096, 4096, 0, 26, 4096, 0, 3921, 0, 0, 0, 4096, 0, 0, 557, 0, 27, 0, 1, 0, 4071, 0, 4096, 0, 37, 7, 4064, 0, 0, 0, 4027, 1, 0, 0, 4096, 2, 0, 4094, 0, 2, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 0, 0, 0, 4088, 4096, 4096, 0, 2, 4095, 0, 2840, 0, 0, 0, 4095, 0, 0, 705, 0, 0, 0, 0, 0, 4059, 0, 4096, 0, 407, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 3, 2, 0, 4096, 181, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3961, 4096, 4096, 0, 0, 4076, 0, 2293, 0, 0, 0, 4096, 0, 0, 2229, 0, 64, 0, 0, 0, 4091, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 3922, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4093, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3928] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 554, 0, 0, 0, 4096, 0, 0, 3514, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 4091, 0, 0, 0, 4096, 9, 0, 4096, 0, 1, 1, 0, 4095, 31, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[13, 4038, 0, 16, 16, 5, 0, 26, 4082, 4017, 4060, 18, 0, 4079, 13, 905, 0, 3, 13, 4058, 0, 31, 2739, 13, 6, 39, 0, 0, 4083, 33, 4062, 25, 550, 0, 3983, 9, 0, 4, 4086, 0, 0, 0, 4068, 64, 0, 4079, 3, 44, 19, 20, 3915, 172, 0, 0, 8, 18, 0, 20, 18, 0, 4054, 0, 0, 4009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 1, 0, 3, 4073, 4094, 4096, 3, 26, 4086, 1, 2852, 1, 1, 0, 4092, 0, 2, 779, 0, 0, 0, 0, 0, 4082, 0, 4094, 1, 6, 9, 4084, 0, 0, 0, 4096, 0, 0, 0, 4095, 4, 0, 4096, 0, 1, 98, 2, 4096, 380, 1, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4045, 4096, 4096, 0, 35, 4096, 0, 4035, 0, 0, 0, 4096, 0, 0, 309, 0, 36, 0, 0, 0, 4096, 0, 4096, 0, 2, 0, 4071, 0, 0, 0, 3832, 0, 0, 0, 4096, 3, 0, 4096, 0, 0, 1, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[13, 4085, 0, 5, 2, 0, 0, 1, 4067, 4091, 4080, 0, 81, 4092, 0, 1674, 1, 4, 0, 4084, 7, 6, 2235, 1, 4, 8, 0, 0, 4091, 3, 4090, 6, 244, 9, 4035, 2, 0, 3, 3751, 0, 4, 1, 4090, 0, 0, 4093, 0, 13, 2, 2, 4090, 273, 0, 4, 0, 4, 0, 3, 8, 0, 4088, 0, 0, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 3745, 1768, 0, 0, 26, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 557, 0, 2328, 125, 12, 0, 0, 0, 0, 0, 27, 0, 1, 0, 4071, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 7, 4043, 0, 0, 0, 4027, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4082, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4085, 13, 12, 9, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 9460, 7118, 8182, 8115, 8182, 8069, 8214] avg=8192.0000 max=9460 max_rank=1 min=7118 min_rank=2 imbalance_ratio=1.154785 max_load_reduction=56076 max_load_reduction_pct=0.855652 moved_to_echo=32580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 1768, 0, 0, 0, 0, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3514, 0, 2328, 123, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 4084, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 31, 0, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 13, 12, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 6418, 10073, 8174, 8162, 8190, 8098, 8227] avg=8192.0000 max=10073 max_rank=2 min=6418 min_rank=1 imbalance_ratio=1.229614 max_load_reduction=55463 max_load_reduction_pct=0.846298 moved_to_echo=32677 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4088, 1768, 0, 0, 2, 0, 0, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 705, 0, 2328, 125, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 2, 4074, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 181, 0, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 13, 12, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8698, 7265, 8142, 8579, 8183, 8247, 8227] avg=8192.0000 max=8698 max_rank=1 min=7265 min_rank=2 imbalance_ratio=1.061768 max_load_reduction=56838 max_load_reduction_pct=0.867279 moved_to_echo=33236 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[13, 4038, 0, 16, 16, 5, 0, 26, 4060, 2, 0, 0, 0, 0, 0, 0, 4082, 1734, 0, 18, 0, 0, 13, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 13, 4058, 0, 31, 2739, 13, 2283, 123, 12, 0, 0, 0, 0, 0, 6, 39, 0, 0, 4083, 33, 4049, 25, 0, 0, 0, 0, 0, 0, 0, 0, 550, 0, 3962, 9, 0, 4, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 64, 0, 4067, 3, 44, 0, 0, 0, 0, 0, 0, 0, 0, 19, 20, 3915, 172, 0, 0, 8, 18, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 20, 18, 0, 4054, 0, 0, 4009, 13, 12, 9, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 6752, 9275, 8235, 8611, 8246, 8106, 8135] avg=8192.0000 max=9275 max_rank=2 min=6752 min_rank=1 imbalance_ratio=1.132202 max_load_reduction=56261 max_load_reduction_pct=0.858475 moved_to_echo=33098 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3961, 1768, 0, 0, 0, 0, 0, 2293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2229, 0, 2328, 123, 12, 0, 0, 0, 0, 0, 64, 0, 0, 0, 4091, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4074, 0, 0, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 13, 0, 0, 0, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3928, 13, 12, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8022, 8788, 8238, 7998, 8180, 8056, 8059] avg=8192.0000 max=8788 max_rank=2 min=7998 min_rank=4 imbalance_ratio=1.072754 max_load_reduction=56748 max_load_reduction_pct=0.865906 moved_to_echo=32293 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4045, 1768, 0, 0, 35, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 309, 0, 2328, 123, 12, 0, 0, 0, 0, 0, 36, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4049, 0, 0, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 15, 0, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 13, 12, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 9883, 6868, 8215, 7883, 8183, 8083, 8227] avg=8192.0000 max=9883 max_rank=1 min=6868 min_rank=2 imbalance_ratio=1.206421 max_load_reduction=55653 max_load_reduction_pct=0.849197 moved_to_echo=32376 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 1, 0, 3, 4096, 2, 0, 0, 0, 0, 0, 0, 4073, 1767, 0, 3, 26, 0, 1, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4092, 0, 2, 779, 0, 2327, 125, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4081, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 9, 4062, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 4084, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 98, 2, 4096, 380, 1, 0, 0, 1, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4092, 13, 12, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8722, 7339, 8164, 8173, 8184, 8537, 8221] avg=8192.0000 max=8722 max_rank=1 min=7339 min_rank=2 imbalance_ratio=1.064697 max_load_reduction=56814 max_load_reduction_pct=0.866913 moved_to_echo=33115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[13, 4085, 0, 5, 2, 0, 0, 1, 4080, 4, 0, 0, 0, 0, 0, 0, 4067, 1759, 0, 0, 81, 0, 0, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4084, 7, 6, 2235, 1, 2332, 123, 17, 0, 0, 0, 0, 0, 4, 8, 0, 0, 4091, 3, 4074, 6, 0, 0, 0, 0, 0, 0, 0, 0, 244, 9, 4006, 2, 0, 3, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4090, 0, 0, 4080, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4090, 273, 0, 4, 0, 4, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 4088, 0, 0, 4086, 16, 13, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7581, 8810, 8186, 8015, 8188, 8340, 8226] avg=8192.0000 max=8810 max_rank=2 min=7581 min_rank=1 imbalance_ratio=1.075439 max_load_reduction=56726 max_load_reduction_pct=0.865570 moved_to_echo=32769 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 32512, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1280, 128, 32384, 128, 0, 128, 32000, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32256, 14208, 0, 128, 256, 0, 128, 19200, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 13184, 128, 18688, 1024, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32640, 1152, 128, 128, 128, 128, 31744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 128, 128, 0, 32768, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1069, 0, 0, 0, 1282, 0, 4096, 0, 0, 0, 388, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 1759, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3466, 0, 0, 4072, 0, 215, 0, 0, 4096, 0, 0, 4096, 52, 0, 0, 0, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[2352, 0, 0, 0, 2536, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 3838, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 2274, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 1, 31, 0, 0, 4014, 0, 133, 0, 0, 4095, 0, 0, 4096, 1154, 0, 0, 18, 4096, 4096, 27, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1722, 17, 15, 15, 3493, 22, 4073, 0, 22, 0, 46, 24, 59, 25, 3856, 26, 0, 0, 17, 3966, 6, 27, 7, 0, 23, 4066, 0, 4070, 3060, 0, 21, 4071, 4032, 0, 35, 1, 20, 1, 4075, 0, 0, 1, 4060, 0, 173, 20, 36, 4079, 0, 6, 0, 12, 4053, 0, 0, 4074, 0, 11, 25, 0, 4073, 4000, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[2876, 0, 0, 0, 190, 0, 4093, 0, 0, 0, 24, 2, 0, 0, 2531, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 2, 4095, 3679, 2, 0, 4096, 4096, 0, 1, 0, 0, 3, 4091, 2, 5, 0, 4096, 7, 2980, 0, 0, 4090, 0, 21, 0, 0, 4094, 0, 0, 4095, 1, 0, 0, 4, 4096, 4075, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1202, 0, 0, 0, 813, 0, 4096, 0, 0, 0, 21, 0, 0, 0, 3316, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3794, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 3124, 0, 0, 4096, 0, 3, 0, 0, 4096, 0, 0, 4096, 6, 0, 0, 10, 4096, 4094, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1014, 0, 0, 0, 597, 0, 4096, 0, 0, 0, 255, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 2676, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3806, 0, 0, 3737, 0, 129, 0, 0, 4096, 0, 0, 4096, 20, 0, 0, 38, 4096, 4096, 29, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3534, 0, 0, 0, 779, 0, 4096, 0, 0, 0, 131, 0, 0, 0, 4055, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3173, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 468, 0, 0, 4064, 0, 9, 0, 0, 4096, 0, 0, 4096, 31, 0, 0, 141, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[2580, 3, 0, 3, 2130, 6, 4089, 2, 6, 0, 237, 0, 0, 5, 3993, 6, 0, 0, 2, 4076, 4, 5, 0, 0, 0, 4090, 0, 4091, 2295, 0, 0, 4091, 4089, 3, 14, 5, 3, 0, 4091, 0, 10, 13, 4078, 0, 898, 5, 6, 4074, 0, 0, 0, 1, 4086, 0, 6, 4090, 1, 4, 5, 225, 4088, 4027, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2352, 0, 0, 0, 2536, 0, 4096, 0, 316, 198, 54, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3838, 0, 4096, 220, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 2618, 1493, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1258, 2274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2287, 1, 31, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 4095, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 0, 0, 18, 3898, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9552, 8160, 8207, 7628, 8193, 6334, 8269, 9193] avg=8192.0000 max=9552 max_rank=0 min=6334 min_rank=5 imbalance_ratio=1.166016 max_load_reduction=55984 max_load_reduction_pct=0.854248 moved_to_echo=31989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1722, 17, 15, 15, 3493, 22, 4073, 0, 313, 197, 54, 0, 0, 0, 0, 0, 22, 0, 46, 24, 59, 25, 3856, 26, 4071, 219, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3966, 6, 27, 7, 0, 2601, 1479, 0, 0, 0, 0, 0, 0, 23, 4066, 0, 1250, 3060, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 35, 1, 20, 1, 4075, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2268, 0, 173, 20, 36, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 12, 4053, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 25, 0, 3876, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9921, 8348, 8103, 8420, 8165, 6577, 8090, 7912] avg=8192.0000 max=9921 max_rank=0 min=6577 min_rank=5 imbalance_ratio=1.211060 max_load_reduction=55615 max_load_reduction_pct=0.848618 moved_to_echo=30744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1014, 0, 0, 0, 597, 0, 4096, 0, 316, 198, 54, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 4083, 0, 4096, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2617, 1493, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1258, 2676, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2287, 0, 3806, 0, 0, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 4096, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 38, 3898, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6275, 8654, 8206, 8031, 8193, 9830, 8266, 8081] avg=8192.0000 max=9830 max_rank=5 min=6275 min_rank=0 imbalance_ratio=1.199951 max_load_reduction=55706 max_load_reduction_pct=0.850006 moved_to_echo=34370 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1069, 0, 0, 0, 1282, 0, 4096, 0, 316, 198, 54, 0, 0, 0, 0, 0, 0, 0, 388, 0, 0, 0, 4081, 0, 4096, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2618, 1493, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1258, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2287, 0, 3466, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 4096, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 3898, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7015, 8785, 8207, 7113, 8193, 9825, 8352, 8046] avg=8192.0000 max=9825 max_rank=5 min=7015 min_rank=0 imbalance_ratio=1.199341 max_load_reduction=55711 max_load_reduction_pct=0.850082 moved_to_echo=34416 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3534, 0, 0, 0, 779, 0, 4096, 0, 316, 198, 54, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 4055, 0, 4095, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2618, 1493, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1258, 3173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2287, 0, 468, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 141, 3898, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8977, 8501, 8207, 8527, 8193, 6819, 8146, 8166] avg=8192.0000 max=8977 max_rank=0 min=6819 min_rank=5 imbalance_ratio=1.095825 max_load_reduction=56559 max_load_reduction_pct=0.863022 moved_to_echo=31324 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1202, 0, 0, 0, 813, 0, 4096, 0, 316, 198, 54, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 3316, 0, 4096, 220, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 2618, 1493, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1258, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2287, 0, 3124, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 10, 3898, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6679, 7653, 8209, 9148, 8192, 9507, 8140, 8008] avg=8192.0000 max=9507 max_rank=5 min=6679 min_rank=0 imbalance_ratio=1.160522 max_load_reduction=56029 max_load_reduction_pct=0.854935 moved_to_echo=33847 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2876, 0, 0, 0, 190, 0, 4093, 0, 316, 198, 54, 0, 0, 0, 0, 0, 0, 0, 24, 2, 0, 0, 2531, 0, 4096, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2617, 1493, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 1258, 3679, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 3, 4091, 2, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2287, 7, 2980, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 4094, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 3898, 4075, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7727, 6873, 8204, 9035, 8194, 9369, 8155, 7979] avg=8192.0000 max=9369 max_rank=5 min=6873 min_rank=1 imbalance_ratio=1.143677 max_load_reduction=56167 max_load_reduction_pct=0.857040 moved_to_echo=33697 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2580, 3, 0, 3, 2130, 6, 4089, 2, 321, 199, 57, 0, 0, 0, 0, 0, 6, 0, 237, 0, 0, 5, 3993, 6, 4091, 224, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4076, 4, 5, 0, 0, 2618, 1488, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1249, 2295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 14, 5, 3, 0, 4091, 0, 8, 0, 0, 0, 0, 0, 0, 0, 10, 13, 2269, 0, 898, 5, 6, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 0, 6, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 5, 225, 3889, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9390, 8562, 8193, 7634, 8213, 7275, 8118, 8151] avg=8192.0000 max=9390 max_rank=0 min=7275 min_rank=5 imbalance_ratio=1.146240 max_load_reduction=56146 max_load_reduction_pct=0.856720 moved_to_echo=31757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 640, 0, 128, 32768, 0, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1280, 128, 128, 512, 31232, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 10112, 22784, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 32640, 128, 128, 128, 0, 20992, 12032, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 18304, 128, 14976, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 1152, 128, 128, 128, 29824, 128, 32768, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[16384, 128, 128, 128, 11904, 128, 32768, 128, 2560, 1664, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 20, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 2, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 4094, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4066, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 6, 0, 1, 4082, 4094, 0, 0, 0, 4096, 4095, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4085, 0, 1, 0, 0, 0, 4094, 0, 4096, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 61, 2, 8, 0, 0, 0, 4044, 1, 1, 0, 4095, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4050, 4038, 0, 0, 0, 0, 4070, 26, 24, 115, 25, 10, 4064, 4061, 20, 0, 23, 4066, 4077, 30, 0, 0, 0, 1, 5, 35, 2, 0, 3978, 2, 432, 12, 46, 98, 4065, 1, 4069, 0, 19, 4058, 0, 4055, 0, 0, 20, 29, 27, 28, 4064, 18, 14, 1, 0, 0, 16, 5, 18, 0, 3552, 0, 1, 34, 4065, 4067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 101, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 4087, 4096, 0, 0, 0, 4096, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4091, 4089, 0, 0, 0, 2, 4088, 4, 7, 7, 5, 7, 4080, 4089, 3, 0, 0, 4089, 4091, 5, 1, 0, 5, 0, 2, 0, 0, 12, 4058, 6, 63, 0, 10, 1, 4090, 0, 4090, 0, 7, 4089, 0, 4084, 0, 0, 6, 4, 3, 9, 4092, 3, 3, 0, 14, 0, 28, 0, 0, 1, 4011, 3, 4, 4, 4088, 4088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 20, 4096, 0, 0, 0, 3995, 17, 12, 11, 2, 2, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4, 0, 0, 0, 4079, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 101, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8176, 8190, 8157, 8180, 8187, 8172, 8285] avg=8192.0000 max=8285 max_rank=7 min=8157 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32824 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3995, 17, 12, 11, 2, 2, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4079, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 101, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8181, 8190, 8135, 8180, 8186, 8181, 8293] avg=8192.0000 max=8293 max_rank=7 min=8135 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32840 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4095, 0, 0, 0, 3995, 17, 12, 11, 2, 2, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4079, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 101, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8151, 8190, 8179, 8180, 8186, 8181, 8279] avg=8192.0000 max=8279 max_rank=7 min=8151 min_rank=1 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32826 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4094, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3995, 17, 12, 11, 2, 2, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 4079, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 101, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8187, 8191, 8135, 8180, 8187, 8181, 8283] avg=8192.0000 max=8283 max_rank=7 min=8135 min_rank=3 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32831 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4048, 4038, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 24, 115, 25, 10, 4064, 4050, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4066, 4075, 30, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 35, 2, 0, 3978, 2, 432, 12, 3966, 16, 12, 11, 2, 2, 0, 0, 46, 98, 4053, 1, 0, 0, 19, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 20, 29, 27, 28, 4064, 0, 0, 0, 0, 0, 0, 0, 4064, 18, 14, 1, 0, 0, 16, 5, 4054, 5, 0, 0, 0, 0, 0, 0, 18, 0, 3552, 0, 1, 34, 4065, 101, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8308, 8195, 8475, 8275, 8223, 8177, 7771] avg=8192.0000 max=8475 max_rank=3 min=7771 min_rank=7 imbalance_ratio=1.034546 max_load_reduction=57061 max_load_reduction_pct=0.870682 moved_to_echo=32446 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 4096, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4087, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 1, 0, 3995, 18, 12, 11, 2, 2, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4078, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 0, 4096, 101, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8175, 8203, 8148, 8180, 8187, 8179, 8173] avg=8192.0000 max=8291 max_rank=0 min=8148 min_rank=3 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32719 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 4082, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4085, 0, 1, 0, 3993, 17, 12, 11, 2, 2, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 61, 2, 8, 0, 4077, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 1, 1, 0, 4095, 101, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8172, 8191, 8125, 8178, 8189, 8249, 8242] avg=8192.0000 max=8249 max_rank=6 min=8125 min_rank=3 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32858 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4087, 4089, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 5, 7, 4080, 4077, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4086, 5, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 12, 4058, 6, 63, 0, 3989, 17, 14, 12, 5, 4, 0, 0, 10, 1, 4076, 0, 0, 0, 7, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 6, 4, 3, 9, 4085, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 3, 0, 14, 0, 28, 0, 4071, 5, 0, 0, 0, 0, 0, 0, 0, 1, 4011, 3, 4, 4, 4088, 99, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8186, 8186, 8182, 8183, 8191, 8216, 8210] avg=8192.0000 max=8216 max_rank=6 min=8182 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=57320 max_load_reduction_pct=0.874634 moved_to_echo=32800 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 32768, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32000, 128, 128, 128, 32768, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 256, 128, 128, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 512, 128, 32000, 256, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66688 slack_slots=3584 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 3, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 255, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 4093, 0, 5, 0, 0, 0, 0, 0, 3996, 3938, 0, 10, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 190, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 65, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4036, 3508, 0, 16, 4096, 377, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 174, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 3946, 4065, 0, 2, 4096, 7, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4094, 0, 4095, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 7, 0, 0, 4091, 1, 4093, 0, 0, 4095, 0, 0, 1, 4095, 3, 4094, 0, 1044, 0, 2, 0, 2, 0, 4088, 0, 4094, 2, 4094, 5, 0, 0, 2, 2, 1, 1, 2753, 4096, 0, 151, 4095, 148, 4095, 0, 0, 0, 1, 3, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4036, 11, 4069, 0, 19, 0, 0, 4080, 0, 19, 0, 49, 0, 23, 41, 4069, 64, 28, 0, 4075, 0, 4070, 0, 11, 4072, 0, 0, 2, 4060, 0, 4065, 36, 241, 24, 0, 31, 26, 8, 4043, 41, 4076, 27, 4081, 0, 63, 0, 0, 0, 17, 26, 3578, 4077, 34, 0, 4059, 90, 4063, 14, 0, 0, 0, 0, 0, 18] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 94, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 826, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 3612, 3673, 0, 11, 4096, 24, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 3, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 164, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 4006, 4070, 0, 7, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4084, 0, 4091, 0, 0, 5, 3, 4091, 0, 0, 0, 0, 0, 5, 20, 4088, 2, 6, 0, 4091, 0, 4090, 8, 5, 4089, 4, 0, 3, 4090, 5, 4088, 7, 2715, 33, 0, 1, 0, 2, 4090, 6, 4091, 4, 4093, 0, 1, 0, 0, 0, 5, 6, 1809, 3612, 0, 0, 4087, 0, 4090, 3, 0, 0, 5, 2, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 3, 0, 0, 0, 0, 0, 0, 190, 0, 0, 4052, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 4096, 0, 3363, 38, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 3508, 0, 16, 730, 377, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8181, 8338, 8195, 7562, 8198, 8667, 8194] avg=8192.0000 max=8667 max_rank=6 min=7562 min_rank=4 imbalance_ratio=1.057983 max_load_reduction=56869 max_load_reduction_pct=0.867752 moved_to_echo=32621 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 9, 3, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 4095, 0, 3363, 38, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 5, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 3938, 0, 10, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8181, 8148, 8195, 7751, 8200, 8674, 8194] avg=8192.0000 max=8674 max_rank=6 min=7751 min_rank=4 imbalance_ratio=1.058838 max_load_reduction=56862 max_load_reduction_pct=0.867645 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4036, 11, 9, 0, 19, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 49, 0, 23, 41, 4069, 4055, 2, 0, 0, 0, 0, 0, 0, 64, 28, 0, 4032, 0, 4070, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 4060, 0, 4065, 36, 0, 0, 0, 0, 0, 0, 0, 0, 241, 24, 0, 31, 26, 8, 4043, 41, 3333, 38, 0, 0, 0, 0, 0, 0, 4076, 27, 4081, 0, 63, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 17, 26, 3578, 4077, 34, 0, 724, 90, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 14, 0, 0, 0, 0, 0, 18, 4069, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8258, 8205, 8166, 7785, 8252, 8546, 8169] avg=8192.0000 max=8546 max_rank=6 min=7785 min_rank=4 imbalance_ratio=1.043213 max_load_reduction=56990 max_load_reduction_pct=0.869598 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 9, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4052, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 0, 4096, 0, 3363, 38, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 4065, 0, 2, 730, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8181, 8149, 8195, 7671, 8196, 8750, 8194] avg=8192.0000 max=8750 max_rank=6 min=7671 min_rank=4 imbalance_ratio=1.068115 max_load_reduction=56786 max_load_reduction_pct=0.866486 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 3, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4052, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 4096, 0, 3363, 38, 0, 0, 0, 0, 0, 0, 4096, 0, 4036, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 4070, 0, 7, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8181, 8152, 8195, 7661, 8138, 8813, 8195] avg=8192.0000 max=8813 max_rank=6 min=7661 min_rank=4 imbalance_ratio=1.075806 max_load_reduction=56723 max_load_reduction_pct=0.865524 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 9, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4081, 3, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4048, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 4095, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 0, 2, 0, 2, 0, 4088, 0, 3362, 38, 0, 0, 0, 0, 0, 0, 4094, 2, 4094, 5, 0, 0, 2, 2, 5, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2753, 4096, 0, 151, 730, 148, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 3, 0, 1, 4092, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8178, 8149, 8196, 8536, 8204, 7880, 8197] avg=8192.0000 max=8536 max_rank=4 min=7880 min_rank=6 imbalance_ratio=1.041992 max_load_reduction=57000 max_load_reduction_pct=0.869751 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 9, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 3, 0, 0, 0, 0, 0, 0, 94, 0, 0, 4052, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 0, 0, 0, 0, 0, 4096, 0, 3363, 38, 0, 0, 0, 0, 0, 0, 4096, 0, 4049, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 3673, 0, 11, 730, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8181, 8242, 8195, 8323, 8151, 8050, 8194] avg=8192.0000 max=8323 max_rank=4 min=8050 min_rank=6 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 7, 0, 0, 5, 3, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 20, 4088, 4078, 4, 0, 0, 0, 0, 0, 0, 2, 6, 0, 4042, 0, 4090, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 3, 4090, 5, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2715, 33, 0, 1, 0, 2, 4090, 6, 3359, 41, 0, 0, 0, 0, 0, 0, 4091, 4, 4093, 0, 1, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 5, 6, 1809, 3612, 0, 0, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 5, 2, 6, 0, 4087, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8195, 8153, 8199, 10247, 8197, 6156, 8199] avg=8192.0000 max=10247 max_rank=4 min=6156 min_rank=6 imbalance_ratio=1.250854 max_load_reduction=55289 max_load_reduction_pct=0.843643 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 128, 128, 32384, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 32768, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 27776, 31104, 128, 256, 5888, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32640, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 0, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[5504, 128, 128, 128, 128, 128, 32768, 128, 26880, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 96, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 3088, 4096, 0, 0, 4096, 0, 0, 923, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 45, 115, 45, 4063, 0, 4028, 3990, 34, 0, 21, 0, 4066, 112, 0, 0, 22, 4042, 28, 3, 4057, 21, 4075, 4048, 5, 30, 4016, 1, 0, 0, 43, 4024, 3, 39, 4073, 1, 26, 0, 4051, 18, 15, 1, 48, 21, 0, 43, 40, 4048, 8, 0, 25, 56, 0, 4068, 0, 0, 3945, 0, 4058, 4, 0, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 1, 6, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 3988, 4096, 0, 0, 4083, 0, 0, 130, 0, 4065, 0, 0, 4090, 0, 0, 0, 4096, 0, 2, 0, 9, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 4095, 0, 2, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 1184, 0, 0, 0, 4093, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 4096, 0, 0, 4096, 0, 2747, 4090, 0, 0, 4096, 0, 4, 289, 0, 4095, 0, 0, 4094, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 3953, 0, 4096, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 51, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4095, 0, 0, 4096, 0, 3786, 4096, 0, 0, 4096, 0, 1, 310, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4042, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 859, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2893, 4096, 0, 0, 4096, 0, 1, 432, 0, 4048, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4055, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4083, 4096, 0, 0, 4096, 0, 2, 13, 0, 4095, 0, 0, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 4096, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 1390, 17, 0, 18, 4091, 10, 4057, 4085, 11, 0, 5, 0, 4087, 56, 0, 9, 4, 4088, 5, 10, 4083, 7, 2689, 4082, 0, 8, 4089, 1, 0, 0, 0, 4085, 0, 7, 4093, 0, 6, 0, 4086, 10, 2, 3, 0, 0, 3, 0, 9, 4084, 0, 0, 7, 0, 0, 4089, 0, 0, 4061, 1, 4086, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 96, 0, 0, 0, 3635, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 673, 0, 3088, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 4096, 0, 0, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 3423, 461, 210, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 43, 12, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7827, 8181, 7857, 8862, 8209, 8173, 8190, 8237] avg=8192.0000 max=8862 max_rank=3 min=7827 min_rank=0 imbalance_ratio=1.081787 max_load_reduction=56674 max_load_reduction_pct=0.864777 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 45, 115, 45, 3606, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 34, 0, 21, 0, 4054, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4042, 28, 3, 667, 21, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 3799, 5, 30, 4016, 1, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 3, 39, 17, 1, 26, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 18, 15, 1, 48, 21, 0, 43, 40, 4056, 3390, 457, 207, 0, 0, 0, 0, 4048, 8, 0, 25, 56, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 0, 4058, 4, 0, 11, 0, 42, 12, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7839, 8211, 8858, 7894, 8161, 8296, 8204, 8073] avg=8192.0000 max=8858 max_rank=2 min=7839 min_rank=0 imbalance_ratio=1.081299 max_load_reduction=56678 max_load_reduction_pct=0.864838 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1184, 0, 0, 0, 3632, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 673, 0, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 0, 0, 4096, 0, 4, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 17, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3423, 461, 209, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 0, 4096, 0, 2, 0, 0, 43, 12, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8912, 8199, 7516, 8227, 8213, 8170, 8191, 8108] avg=8192.0000 max=8912 max_rank=0 min=7516 min_rank=2 imbalance_ratio=1.087891 max_load_reduction=56624 max_load_reduction_pct=0.864014 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 0, 0, 0, 3635, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 673, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 4096, 0, 1, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4079, 3423, 461, 210, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 4096, 0, 0, 0, 0, 43, 12, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7782, 8181, 8554, 8250, 8209, 8175, 8190, 8195] avg=8192.0000 max=8554 max_rank=2 min=7782 min_rank=0 imbalance_ratio=1.044189 max_load_reduction=56982 max_load_reduction_pct=0.869476 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 6, 0, 3635, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 673, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 4083, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 9, 0, 0, 0, 0, 4073, 3423, 461, 210, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 2, 0, 1, 43, 12, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7738, 8177, 8755, 8056, 8178, 8178, 8204, 8250] avg=8192.0000 max=8755 max_rank=2 min=7738 min_rank=0 imbalance_ratio=1.068726 max_load_reduction=56781 max_load_reduction_pct=0.866409 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 859, 0, 0, 0, 3635, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 673, 0, 2893, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 4096, 0, 1, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 3423, 461, 210, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 4096, 0, 0, 0, 0, 43, 12, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8590, 8180, 7662, 8372, 8161, 8173, 8190, 8208] avg=8192.0000 max=8590 max_rank=0 min=7662 min_rank=2 imbalance_ratio=1.048584 max_load_reduction=56946 max_load_reduction_pct=0.868927 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3635, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 673, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 4096, 0, 2, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 17, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3423, 461, 210, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 1, 0, 0, 43, 12, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7731, 8180, 8852, 7954, 8209, 8172, 8190, 8248] avg=8192.0000 max=8852 max_rank=2 min=7731 min_rank=0 imbalance_ratio=1.080566 max_load_reduction=56684 max_load_reduction_pct=0.864929 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1390, 17, 0, 18, 3625, 10, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 11, 0, 5, 0, 4070, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 4088, 5, 10, 670, 7, 2689, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 0, 8, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 7, 12, 0, 6, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2, 3, 0, 0, 3, 0, 9, 4081, 3413, 466, 212, 0, 0, 0, 0, 4084, 0, 0, 7, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 1, 4086, 0, 0, 2, 0, 47, 17, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9117, 8227, 7482, 7921, 8196, 8199, 8177, 8217] avg=8192.0000 max=9117 max_rank=0 min=7482 min_rank=2 imbalance_ratio=1.112915 max_load_reduction=56419 max_load_reduction_pct=0.860886 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 5376, 128, 27392, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 0, 32640, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 3584, 128, 128, 128, 29056, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 32384, 128, 32768, 128, 128, 128, 128, 384, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[30720, 128, 128, 32768, 128, 128, 2176, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32640, 27392, 3712, 1792, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 128, 128, 256, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4024, 0, 0, 0, 0, 0, 0, 4096, 1, 7, 3, 0, 4096, 4096, 4096, 18, 135, 4096, 4096, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4010, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 4095, 4096, 0, 562, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3552, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4064, 4072, 25, 23, 0, 30, 57, 0, 4034, 0, 0, 0, 25, 3996, 4041, 4048, 1, 3407, 4064, 4023, 30, 4060, 4074, 0, 5, 0, 2, 23, 1, 0, 87, 4, 32, 0, 0, 0, 3983, 4063, 26, 28, 4, 0, 55, 0, 96, 3991, 0, 22, 0, 0, 0, 701, 0, 45, 77, 0, 31, 37, 4052, 41, 44, 4009, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4080, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 4096, 4096, 4096, 0, 8, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 3634, 0, 0, 0, 0, 0, 0, 4096, 0, 49, 0, 0, 4096, 4096, 4096, 0, 1096, 4096, 4096, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3498, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4013, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4094, 4022, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 47, 331, 4095, 4096, 0, 4094, 4089, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 4096, 4092, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 8, 1, 0, 4096, 2, 0, 3855, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 4096, 4096, 0, 1455, 4096, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2696, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4044, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4086, 4091, 7, 5, 0, 10, 7, 0, 4077, 0, 0, 0, 4, 4081, 4084, 4089, 1, 87, 4091, 4087, 15, 4087, 4091, 2, 0, 6, 2, 6, 0, 13, 20, 7, 6, 0, 0, 0, 4076, 4088, 5, 3, 12, 0, 10, 0, 0, 4070, 5, 9, 0, 0, 0, 4000, 0, 2, 0, 0, 6, 7, 4085, 6, 8, 4082, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4024, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 7, 3, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 18, 135, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3411, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 674, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4010, 0, 122, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8214, 7463, 8181, 8204, 8189, 8870, 8228] avg=8192.0000 max=8870 max_rank=6 min=7463 min_rank=2 imbalance_ratio=1.082764 max_load_reduction=56666 max_load_reduction_pct=0.864655 moved_to_echo=33491 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 0, 562, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3410, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4096, 674, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3552, 0, 122, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8205, 7872, 8180, 8204, 8187, 8860, 7770] avg=8192.0000 max=8860 max_rank=6 min=7770 min_rank=7 imbalance_ratio=1.081543 max_load_reduction=56676 max_load_reduction_pct=0.864807 moved_to_echo=33021 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4064, 4072, 25, 23, 0, 30, 57, 66, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 25, 3996, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3177, 1, 3407, 0, 4023, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 23, 1, 0, 87, 4064, 4048, 0, 0, 0, 0, 0, 0, 4, 32, 0, 0, 0, 3983, 4063, 26, 12, 0, 0, 0, 0, 0, 0, 0, 28, 4, 0, 55, 0, 96, 3991, 0, 3365, 673, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 701, 0, 45, 77, 4074, 665, 12, 0, 0, 0, 0, 0, 0, 31, 37, 4052, 41, 44, 4009, 3, 120, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8337, 8066, 10638, 8230, 8120, 8212, 5596, 8337] avg=8192.0000 max=10638 max_rank=2 min=5596 min_rank=6 imbalance_ratio=1.298584 max_load_reduction=54898 max_load_reduction_pct=0.837677 moved_to_echo=30265 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4080, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3411, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4095, 674, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 0, 122, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8227, 7318, 8180, 8204, 8188, 8869, 8307] avg=8192.0000 max=8869 max_rank=6 min=7318 min_rank=2 imbalance_ratio=1.082642 max_load_reduction=56667 max_load_reduction_pct=0.864670 moved_to_echo=33568 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3634, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 49, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 0, 1096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3411, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3498, 0, 0, 0, 4087, 674, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4013, 0, 122, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7797, 8252, 8406, 8188, 8204, 8188, 8270, 8231] avg=8192.0000 max=8406 max_rank=2 min=7797 min_rank=0 imbalance_ratio=1.026123 max_load_reduction=57130 max_load_reduction_pct=0.871735 moved_to_echo=32893 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 4022, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 47, 331, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 4083, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4092, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 3411, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 4089, 674, 11, 0, 0, 0, 0, 0, 8, 1, 0, 4096, 2, 0, 3855, 0, 122, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8202, 7688, 8181, 8201, 8190, 8807, 8084] avg=8192.0000 max=8807 max_rank=6 min=7688 min_rank=2 imbalance_ratio=1.075073 max_load_reduction=56729 max_load_reduction_pct=0.865616 moved_to_echo=33282 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4092, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 0, 1455, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3411, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2696, 0, 0, 0, 4094, 674, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4044, 0, 122, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8206, 8765, 8180, 8204, 8188, 7476, 8262] avg=8192.0000 max=8765 max_rank=2 min=7476 min_rank=6 imbalance_ratio=1.069946 max_load_reduction=56771 max_load_reduction_pct=0.866257 moved_to_echo=32130 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 4091, 7, 5, 0, 10, 7, 70, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4, 4081, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3196, 1, 87, 0, 4087, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 2, 6, 0, 13, 20, 4091, 4076, 0, 0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 4076, 4088, 5, 13, 0, 0, 0, 0, 0, 0, 0, 3, 12, 0, 10, 0, 0, 4070, 5, 3407, 687, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4000, 0, 2, 0, 4091, 675, 11, 0, 0, 0, 0, 0, 0, 6, 7, 4085, 6, 8, 4082, 0, 123, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8164, 7386, 8216, 8195, 8194, 8788, 8317] avg=8192.0000 max=8788 max_rank=6 min=7386 min_rank=2 imbalance_ratio=1.072754 max_load_reduction=56748 max_load_reduction_pct=0.865906 moved_to_echo=33494 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 32128, 128, 128, 0, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 27264, 0, 128, 128, 32768, 5504, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[25728, 128, 7168, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 128, 32640, 128, 27264, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 32640, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 31744, 128, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 3591, 0, 0, 0, 245, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 4084, 0, 4096, 0, 4020, 0, 0, 4065, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 4094, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3125, 0, 0, 0, 1010, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4020, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4086, 0, 0, 0, 322, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 2153, 0, 0, 4090, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1637, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2980, 0, 0, 0, 1542, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 2441, 0, 0, 4096, 0, 15, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1231, 0, 0, 0, 0, 4079, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[30, 11, 64, 0, 0, 3799, 2, 0, 6, 709, 4045, 0, 13, 0, 35, 0, 32, 4056, 0, 35, 0, 4022, 4070, 0, 4064, 0, 3120, 22, 29, 4082, 0, 31, 29, 35, 4053, 4067, 13, 4054, 0, 0, 3765, 32, 21, 0, 19, 65, 0, 4068, 16, 59, 78, 7, 25, 0, 549, 36, 74, 30, 31, 4068, 4058, 3964, 17, 26] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3977, 0, 0, 0, 96, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4074, 0, 0, 4082, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 50, 0, 1, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 0, 3411, 0, 1, 0, 927, 4096, 1, 0, 2, 0, 0, 0, 4096, 0, 0, 1, 4093, 4096, 0, 4095, 0, 3893, 0, 0, 4041, 3, 0, 0, 0, 4095, 4094, 0, 4094, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 8, 0, 0, 0, 26, 0, 0, 0, 1, 4093, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[21, 3, 53, 11, 0, 2785, 0, 0, 12, 1790, 4086, 0, 0, 0, 7, 0, 0, 4084, 0, 3, 0, 4077, 4091, 0, 4089, 0, 3358, 7, 6, 4092, 0, 63, 8, 8, 4085, 4090, 4, 4088, 0, 0, 3797, 11, 4, 0, 0, 5, 5, 4089, 0, 10, 42, 0, 3, 0, 356, 9, 0, 10, 2, 4088, 4081, 4083, 14, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 3591, 0, 0, 4074, 560, 57, 0, 0, 0, 0, 0, 0, 245, 4096, 0, 0, 0, 0, 0, 3267, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 4020, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 64, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 4096, 3536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 7608, 8203, 8793, 8192, 8257, 8010, 8189] avg=8192.0000 max=8793 max_rank=3 min=7608 min_rank=1 imbalance_ratio=1.073364 max_load_reduction=56743 max_load_reduction_pct=0.865829 moved_to_echo=32648 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4086, 0, 0, 4086, 559, 57, 0, 0, 0, 0, 0, 0, 322, 4096, 0, 0, 0, 0, 0, 3267, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 2153, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 64, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1637, 0, 4094, 3537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8788, 7685, 8202, 6951, 8194, 8257, 9268, 8191] avg=8192.0000 max=9268 max_rank=6 min=6951 min_rank=3 imbalance_ratio=1.131348 max_load_reduction=56268 max_load_reduction_pct=0.858582 moved_to_echo=33910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[30, 11, 64, 0, 0, 3799, 2, 0, 4060, 554, 57, 0, 0, 0, 0, 0, 6, 709, 4045, 0, 13, 0, 35, 0, 3242, 0, 0, 0, 0, 0, 0, 0, 32, 4056, 0, 35, 0, 4022, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 702, 0, 3120, 22, 29, 4082, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 29, 35, 4053, 2, 13, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3765, 32, 21, 0, 19, 65, 0, 4068, 63, 1, 0, 0, 0, 0, 0, 0, 16, 59, 78, 7, 25, 0, 549, 36, 4065, 3504, 0, 0, 0, 0, 0, 0, 74, 30, 31, 4067, 0, 3964, 17, 26, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8577, 8050, 8155, 7986, 8186, 8034, 8339, 8209] avg=8192.0000 max=8577 max_rank=0 min=7986 min_rank=3 imbalance_ratio=1.046997 max_load_reduction=56959 max_load_reduction_pct=0.869125 moved_to_echo=32768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3125, 0, 0, 4086, 559, 57, 0, 0, 0, 0, 0, 0, 1010, 4096, 0, 0, 0, 0, 0, 3267, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 4020, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 64, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 4094, 3537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7827, 8373, 8202, 8821, 8195, 8257, 7670, 8191] avg=8192.0000 max=8821 max_rank=3 min=7670 min_rank=6 imbalance_ratio=1.076782 max_load_reduction=56715 max_load_reduction_pct=0.865402 moved_to_echo=32313 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3977, 0, 0, 4086, 559, 57, 0, 0, 0, 0, 0, 0, 96, 4096, 0, 0, 0, 0, 0, 3267, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 4074, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 64, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 50, 0, 4094, 3537, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8679, 7459, 8202, 8864, 8194, 8257, 7689, 8192] avg=8192.0000 max=8864 max_rank=3 min=7459 min_rank=1 imbalance_ratio=1.082031 max_load_reduction=56672 max_load_reduction_pct=0.864746 moved_to_echo=32332 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2980, 0, 0, 4086, 560, 57, 0, 0, 0, 0, 0, 0, 1542, 4096, 0, 0, 0, 0, 0, 3267, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 2441, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 64, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 0, 4096, 3536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7683, 8905, 8202, 7260, 8192, 8257, 8863, 8174] avg=8192.0000 max=8905 max_rank=1 min=7260 min_rank=3 imbalance_ratio=1.087036 max_load_reduction=56631 max_load_reduction_pct=0.864120 moved_to_echo=33486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 0, 3411, 0, 1, 4086, 559, 57, 0, 0, 0, 0, 0, 0, 927, 4096, 1, 0, 2, 0, 0, 3267, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4093, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 707, 0, 3893, 0, 0, 4041, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 4094, 64, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 26, 0, 4093, 3537, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8293, 8200, 8644, 8190, 8238, 7664, 8189] avg=8192.0000 max=8644 max_rank=3 min=7664 min_rank=6 imbalance_ratio=1.055176 max_load_reduction=56892 max_load_reduction_pct=0.868103 moved_to_echo=32281 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[21, 3, 53, 11, 0, 2785, 0, 0, 4085, 558, 64, 0, 0, 0, 0, 0, 12, 1790, 4086, 0, 0, 0, 7, 0, 3268, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 3, 0, 4077, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, 3358, 7, 6, 4092, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 4085, 0, 4, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3797, 11, 4, 0, 0, 5, 5, 4089, 66, 2, 0, 0, 0, 0, 0, 0, 0, 10, 42, 0, 3, 0, 356, 9, 4090, 3523, 0, 0, 0, 0, 0, 0, 0, 10, 2, 4086, 0, 4083, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7580, 9163, 8170, 8217, 8193, 7979, 8033, 8201] avg=8192.0000 max=9163 max_rank=1 min=7580 min_rank=0 imbalance_ratio=1.118530 max_load_reduction=56373 max_load_reduction_pct=0.860184 moved_to_echo=32406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32128, 128, 128, 0, 128, 128, 128, 32768, 640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 0, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5760, 0, 27136, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 27776, 128, 128, 32768, 4480, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 256, 128, 128, 0, 4352, 128, 32768, 28288, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 6656, 32768, 128, 128, 128, 128, 0, 26112, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 1298, 0, 1, 0, 0, 0, 4096, 4096, 0, 4096, 2295, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4076, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 36, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 367, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 3, 0, 75, 0, 0, 0, 4096, 4096, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3843, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 115, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 32, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 66, 0, 21, 0, 0, 0, 4096, 4096, 0, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3720, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4096, 373, 4096, 0, 4096, 0, 4096, 4085, 0, 0, 0, 4096, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 289, 0, 7, 0, 0, 0, 4096, 4096, 0, 4096, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 22, 0, 3700, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 173, 4096, 0, 4096, 0, 4096, 4091, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 4094, 0, 0, 737, 0, 44, 0, 0, 0, 4095, 4096, 0, 4096, 3483, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 2, 0, 4096, 0, 4096, 0, 0, 2, 3807, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 122, 4096, 0, 4095, 0, 4091, 4095, 0, 0, 1, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4065, 0, 1, 22, 46, 0, 0, 0, 43, 4059, 4048, 48, 4054, 3955, 10, 17, 0, 226, 33, 0, 0, 40, 23, 50, 4, 31, 0, 39, 14, 4038, 0, 0, 4051, 34, 4046, 24, 0, 0, 3889, 50, 44, 4063, 38, 0, 0, 57, 39, 0, 4055, 45, 4043, 14, 4057, 0, 3980, 4055, 10, 0, 0, 4053, 0, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 349, 0, 4, 0, 0, 0, 4096, 4096, 0, 4096, 3671, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3970, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 186, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 1, 4088, 10, 0, 625, 11, 50, 2, 0, 9, 4084, 4086, 27, 4087, 3633, 8, 6, 0, 10, 5, 4, 19, 9, 13, 10, 0, 1, 0, 2, 8, 4086, 0, 0, 4086, 7, 4070, 3, 0, 0, 3645, 10, 12, 4088, 8, 0, 0, 12, 0, 0, 4085, 5, 4089, 0, 4085, 0, 4081, 4085, 16, 0, 0, 4086, 162, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 3, 0, 75, 3582, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 444, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4096, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4030, 57, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 0, 4096, 0, 0, 0, 0, 124, 70, 47, 0, 0, 0, 0, 0, 0, 0, 3972, 115, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4045, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7822, 8629, 8170, 8183, 8196, 8180, 8183, 8173] avg=8192.0000 max=8629 max_rank=1 min=7822 min_rank=0 imbalance_ratio=1.053345 max_load_reduction=56907 max_load_reduction_pct=0.868332 moved_to_echo=32732 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 1298, 0, 1, 3582, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 444, 2295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 4096, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4030, 57, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 0, 0, 0, 0, 124, 70, 47, 0, 0, 0, 0, 0, 0, 0, 3971, 36, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4045, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9043, 6835, 8255, 8183, 8196, 8413, 8103, 8508] avg=8192.0000 max=9043 max_rank=0 min=6835 min_rank=1 imbalance_ratio=1.103882 max_load_reduction=56493 max_load_reduction_pct=0.862015 moved_to_echo=33220 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 289, 0, 7, 3582, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 444, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4029, 57, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 22, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 4096, 0, 0, 0, 0, 124, 70, 47, 0, 0, 0, 0, 0, 0, 0, 3972, 173, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8041, 8546, 8135, 8182, 8218, 8037, 8241, 8136] avg=8192.0000 max=8546 max_rank=1 min=8037 min_rank=5 imbalance_ratio=1.043213 max_load_reduction=56990 max_load_reduction_pct=0.869598 moved_to_echo=32632 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4065, 0, 1, 22, 46, 0, 3546, 65, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 4048, 48, 439, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 10, 17, 0, 226, 33, 0, 0, 40, 4059, 4000, 0, 0, 0, 0, 0, 0, 23, 50, 4, 31, 0, 39, 14, 4038, 3988, 57, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 34, 4046, 24, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3889, 50, 44, 4063, 38, 0, 0, 57, 123, 69, 46, 0, 0, 0, 0, 0, 39, 0, 3932, 45, 4043, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 4005, 10, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7745, 8533, 8385, 8244, 8159, 8379, 8073, 8018] avg=8192.0000 max=8533 max_rank=1 min=7745 min_rank=0 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=32629 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 349, 0, 4, 3582, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 444, 3671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4030, 57, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 0, 4096, 0, 0, 0, 0, 124, 70, 47, 0, 0, 0, 0, 0, 0, 0, 3972, 186, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8211, 8147, 8183, 8196, 8307, 8254, 8141] avg=8192.0000 max=8307 max_rank=5 min=8097 min_rank=0 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32898 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 66, 0, 21, 3582, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 444, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4029, 57, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 0, 4086, 0, 0, 0, 0, 124, 70, 47, 0, 0, 0, 0, 0, 0, 0, 3972, 373, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4034, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7832, 8570, 8135, 8182, 8196, 8047, 8441, 8133] avg=8192.0000 max=8570 max_rank=1 min=7832 min_rank=0 imbalance_ratio=1.046143 max_load_reduction=56966 max_load_reduction_pct=0.869232 moved_to_echo=32817 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4094, 0, 0, 737, 0, 44, 3582, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 444, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4030, 57, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 3807, 0, 0, 4096, 0, 0, 0, 0, 124, 70, 47, 0, 0, 0, 0, 0, 0, 0, 3972, 122, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4044, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8524, 8023, 8135, 8184, 8200, 8144, 8190, 8136] avg=8192.0000 max=8524 max_rank=0 min=8023 min_rank=1 imbalance_ratio=1.040527 max_load_reduction=57012 max_load_reduction_pct=0.869934 moved_to_echo=32670 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4088, 10, 0, 625, 11, 50, 3580, 67, 0, 0, 0, 0, 0, 0, 2, 0, 9, 0, 4086, 27, 432, 3633, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 10, 5, 4, 19, 9, 4084, 4029, 0, 0, 0, 0, 0, 0, 13, 10, 0, 1, 0, 2, 8, 4086, 4019, 56, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 7, 4070, 3, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3645, 10, 12, 4088, 8, 0, 0, 12, 128, 75, 51, 0, 0, 0, 0, 0, 0, 0, 3957, 5, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4025, 16, 0, 0, 0, 162, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8432, 8189, 8174, 8195, 8175, 8029, 8051, 8291] avg=8192.0000 max=8432 max_rank=0 min=8029 min_rank=5 imbalance_ratio=1.029297 max_load_reduction=57104 max_load_reduction_pct=0.871338 moved_to_echo=32546 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 3584, 29184, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[30720, 128, 128, 32768, 128, 0, 0, 128, 1024, 640, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 32384, 128, 0, 128, 0, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 256, 128, 128, 256, 128, 32768, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 32768, 32256, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 31744, 1152, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 3456, 128, 256, 28672, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2276, 7, 4096, 0, 0, 0, 0, 0, 4096, 4096, 663, 0, 0, 1823, 0, 3499, 4096, 0, 4030, 0, 3, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 3, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3770, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 455, 0, 0, 3483, 0, 487, 4096, 0, 4092, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 4096, 1, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3709, 12, 4092, 0, 0, 0, 0, 0, 4096, 4095, 2557, 0, 0, 262, 0, 1655, 4096, 0, 4070, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 2, 0, 4095, 4094, 5, 0, 0, 0, 0, 0, 0, 4093, 1, 27, 2, 0, 0, 4096, 4091, 0, 4095, 1, 0, 0, 4096, 2, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[46, 42, 30, 0, 46, 0, 229, 0, 4028, 54, 40, 101, 0, 18, 4048, 4002, 3995, 0, 52, 3360, 22, 518, 4051, 0, 4048, 0, 0, 0, 0, 32, 4042, 42, 0, 31, 0, 26, 42, 0, 1, 4034, 4038, 0, 36, 0, 63, 0, 0, 95, 4052, 0, 0, 0, 27, 0, 4053, 4066, 8, 4056, 0, 13, 58, 4030, 0, 3961] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1372, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2290, 0, 0, 1684, 0, 3584, 4096, 0, 3352, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3616, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 1535, 0, 0, 1143, 0, 2080, 4096, 0, 3915, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1928, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2031, 0, 0, 1876, 0, 2400, 4096, 0, 4053, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[9, 8, 6, 0, 9, 0, 4075, 5, 4083, 13, 2, 18, 0, 5, 4086, 4081, 3766, 0, 11, 266, 0, 45, 4084, 0, 4069, 9, 0, 0, 0, 10, 4073, 8, 0, 6, 0, 8, 12, 0, 0, 4078, 4085, 0, 7, 0, 10, 0, 0, 47, 4085, 0, 11, 0, 8, 11, 4087, 4089, 3, 4088, 0, 4, 13, 4083, 0, 4060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2276, 7, 4096, 1455, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 663, 0, 0, 1823, 0, 3499, 2504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 3, 0, 0, 0, 4084, 0, 134, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2638, 1457, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2620, 1458, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7834, 8185, 8489, 8258, 8191, 8171, 8194, 8214] avg=8192.0000 max=8489 max_rank=2 min=7834 min_rank=0 imbalance_ratio=1.036255 max_load_reduction=57047 max_load_reduction_pct=0.870468 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1372, 0, 4096, 1455, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2290, 0, 0, 1684, 0, 3584, 2504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3352, 0, 0, 0, 0, 0, 4096, 0, 134, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2638, 1457, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2620, 1458, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6923, 8185, 10062, 7589, 8191, 8174, 8201, 8211] avg=8192.0000 max=10062 max_rank=2 min=6923 min_rank=0 imbalance_ratio=1.228271 max_load_reduction=55474 max_load_reduction_pct=0.846466 moved_to_echo=32777 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[46, 42, 30, 0, 46, 0, 229, 0, 4048, 1445, 0, 0, 0, 0, 0, 0, 4022, 54, 40, 101, 0, 18, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 52, 3360, 22, 518, 2477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 32, 4042, 42, 132, 6, 0, 0, 0, 0, 0, 0, 0, 31, 0, 26, 42, 0, 1, 4034, 2619, 1443, 0, 0, 0, 0, 0, 0, 4038, 0, 36, 0, 63, 0, 0, 95, 2594, 1442, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 27, 0, 4053, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 19, 0, 13, 58, 4030, 0, 3961, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5886, 8237, 10424, 8302, 8196, 8268, 8134, 8089] avg=8192.0000 max=10424 max_rank=2 min=5886 min_rank=0 imbalance_ratio=1.272461 max_load_reduction=55112 max_load_reduction_pct=0.840942 moved_to_echo=32687 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3770, 0, 4096, 1455, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 3483, 0, 487, 2504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4095, 0, 134, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2637, 1457, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2620, 1458, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9321, 8185, 6929, 8328, 8190, 8174, 8197, 8212] avg=8192.0000 max=9321 max_rank=0 min=6929 min_rank=2 imbalance_ratio=1.137817 max_load_reduction=56215 max_load_reduction_pct=0.857773 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3616, 0, 4096, 1455, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1535, 0, 0, 1143, 0, 2080, 2504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 0, 0, 0, 4095, 0, 134, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2638, 1457, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2620, 1458, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9167, 8185, 7262, 8151, 8191, 8174, 8195, 8211] avg=8192.0000 max=9167 max_rank=0 min=7262 min_rank=2 imbalance_ratio=1.119019 max_load_reduction=56369 max_load_reduction_pct=0.860123 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3709, 12, 4096, 1454, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2557, 0, 0, 262, 0, 1655, 2504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 1, 0, 0, 0, 4094, 0, 134, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 4095, 2635, 1457, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 0, 0, 0, 0, 2619, 1458, 0, 0, 0, 0, 0, 0, 4093, 1, 27, 2, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9271, 8180, 6978, 8306, 8190, 8176, 8221, 8214] avg=8192.0000 max=9271 max_rank=0 min=6978 min_rank=2 imbalance_ratio=1.131714 max_load_reduction=56265 max_load_reduction_pct=0.858536 moved_to_echo=32801 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1928, 0, 4096, 1456, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2031, 0, 0, 1876, 0, 2400, 2504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 4096, 0, 134, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2640, 1457, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2620, 1458, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7480, 8185, 8811, 8290, 8193, 8174, 8192, 8211] avg=8192.0000 max=8811 max_rank=2 min=7480 min_rank=0 imbalance_ratio=1.075562 max_load_reduction=56725 max_load_reduction_pct=0.865555 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[9, 8, 6, 0, 9, 0, 4075, 5, 4086, 1456, 0, 0, 0, 0, 0, 0, 4075, 13, 2, 18, 0, 5, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 0, 11, 266, 0, 45, 2493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 9, 0, 0, 0, 10, 4073, 8, 135, 8, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8, 12, 0, 0, 4078, 2633, 1457, 0, 0, 0, 0, 0, 0, 4085, 0, 7, 0, 10, 0, 0, 47, 2620, 1456, 0, 0, 0, 0, 0, 0, 4085, 0, 11, 0, 8, 11, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 11, 0, 4, 13, 4083, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9654, 8194, 6581, 8312, 8194, 8225, 8202, 8174] avg=8192.0000 max=9654 max_rank=0 min=6581 min_rank=2 imbalance_ratio=1.178467 max_load_reduction=55882 max_load_reduction_pct=0.852692 moved_to_echo=32795 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[17408, 0, 128, 13952, 128, 14336, 20096, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32640, 128, 128, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 256, 128, 128, 128, 32768, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 0, 256, 20992, 11648, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 128, 0, 20992, 128, 32768, 11648, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[31744, 128, 128, 0, 0, 128, 32768, 128, 1152, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 32768, 21120, 11648, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 565, 0, 0, 4096, 0, 0, 0, 0, 4096, 5, 4096, 3504, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 29, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[30, 20, 22, 1025, 0, 0, 0, 4021, 45, 4015, 3980, 42, 0, 2, 0, 4072, 0, 0, 4036, 0, 2951, 0, 1, 4044, 22, 0, 55, 0, 4060, 0, 4043, 4048, 1, 4053, 0, 59, 4038, 0, 45, 0, 27, 42, 53, 0, 4034, 75, 46, 4056, 40, 165, 0, 0, 0, 4051, 0, 52, 4049, 1, 31, 0, 39, 0, 9, 36] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3380, 2, 2, 0, 4096, 0, 4077, 4096, 0, 0, 0, 4, 4089, 0, 0, 4096, 1, 739, 0, 0, 4094, 0, 0, 0, 2, 4094, 2, 4096, 4078, 0, 4094, 2, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3878, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1980, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 4096, 2256, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 84, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 2808, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1437, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 3948, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[8, 5, 17, 3566, 0, 0, 0, 4080, 8, 4077, 4055, 9, 0, 1, 0, 4090, 0, 0, 4084, 0, 776, 0, 4, 4084, 2, 3, 12, 0, 4088, 0, 4087, 3808, 9, 4088, 0, 11, 4087, 0, 12, 0, 5, 9, 0, 0, 4085, 19, 1, 4088, 9, 47, 0, 0, 0, 4084, 0, 8, 4086, 1, 8, 0, 5, 0, 1, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 2101, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2140, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 4096, 3958, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4042, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 426, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 3724, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 4096, 961, 19, 3, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 565, 0, 0, 2734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 5, 4096, 3504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3669, 401, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 29, 0, 4069, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9173, 8212, 7395, 8032, 8189, 8175, 8166, 8194] avg=8192.0000 max=9173 max_rank=0 min=7395 min_rank=2 imbalance_ratio=1.119751 max_load_reduction=56363 max_load_reduction_pct=0.860031 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[30, 20, 22, 1025, 0, 0, 0, 4021, 949, 19, 3, 0, 0, 0, 0, 0, 45, 4015, 3980, 42, 0, 2, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 2951, 0, 1, 2699, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 55, 0, 423, 0, 4043, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4050, 0, 59, 4038, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 42, 53, 0, 4034, 75, 46, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 40, 165, 0, 0, 0, 4051, 0, 52, 3637, 396, 0, 0, 0, 0, 0, 0, 4049, 1, 31, 0, 39, 0, 9, 36, 4050, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6089, 8106, 9687, 8591, 8193, 8314, 8341, 8215] avg=8192.0000 max=9687 max_rank=2 min=6089 min_rank=0 imbalance_ratio=1.182495 max_load_reduction=55849 max_load_reduction_pct=0.852188 moved_to_echo=33063 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3878, 0, 0, 0, 4096, 961, 19, 3, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1980, 0, 0, 2734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 4096, 2256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3664, 401, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 84, 0, 4074, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8957, 8213, 8810, 6779, 8189, 8173, 8161, 8254] avg=8192.0000 max=8957 max_rank=0 min=6779 min_rank=3 imbalance_ratio=1.093384 max_load_reduction=56579 max_load_reduction_pct=0.863327 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3380, 2, 2, 0, 4096, 961, 19, 3, 0, 0, 0, 0, 0, 0, 4077, 4096, 0, 0, 0, 4, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 739, 0, 0, 2733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 427, 2, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3667, 400, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 0, 4067, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8463, 8199, 7569, 8605, 8193, 8170, 8163, 8174] avg=8192.0000 max=8605 max_rank=3 min=7569 min_rank=2 imbalance_ratio=1.050415 max_load_reduction=56931 max_load_reduction_pct=0.868698 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2808, 0, 0, 0, 4096, 961, 19, 3, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1437, 0, 0, 2734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 4096, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3666, 401, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4074, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7887, 8213, 8267, 8471, 8191, 8173, 8163, 8171] avg=8192.0000 max=8471 max_rank=3 min=7887 min_rank=0 imbalance_ratio=1.034058 max_load_reduction=57065 max_load_reduction_pct=0.870743 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2101, 0, 0, 0, 4096, 961, 19, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2140, 0, 0, 2734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 4096, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3663, 401, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7180, 8214, 8970, 8480, 8189, 8173, 8160, 8170] avg=8192.0000 max=8970 max_rank=2 min=7180 min_rank=0 imbalance_ratio=1.094971 max_load_reduction=56566 max_load_reduction_pct=0.863129 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4042, 0, 0, 0, 4096, 961, 19, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 426, 0, 0, 2734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 4096, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3669, 401, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9121, 8214, 7256, 8247, 8189, 8173, 8166, 8170] avg=8192.0000 max=9121 max_rank=0 min=7256 min_rank=2 imbalance_ratio=1.113403 max_load_reduction=56415 max_load_reduction_pct=0.860825 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 5, 17, 3566, 0, 0, 0, 4080, 964, 22, 4, 0, 0, 0, 0, 0, 8, 4077, 4055, 9, 0, 1, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 776, 0, 4, 2718, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 12, 0, 419, 0, 4087, 3808, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4084, 0, 11, 4087, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 9, 0, 0, 4085, 19, 1, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 9, 47, 0, 0, 0, 4084, 0, 8, 3666, 402, 0, 0, 0, 0, 0, 0, 4086, 1, 8, 0, 5, 0, 1, 9, 4075, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8666, 8165, 7582, 8331, 8203, 8185, 8216, 8188] avg=8192.0000 max=8666 max_rank=0 min=7582 min_rank=2 imbalance_ratio=1.057861 max_load_reduction=56870 max_load_reduction_pct=0.867767 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 11136, 0, 128, 21888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 24960, 128, 128, 0, 32768, 7680, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 3456, 128, 32768, 29440, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 32640, 128, 0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 256, 0, 0, 0, 32768, 0, 128, 29312, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 256, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4093, 4096, 4096, 0, 11, 4092, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4061, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 82, 0, 0, 4089, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4095, 4096, 4096, 0, 5, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4057, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 50, 0, 0, 4085, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 3952, 4096, 4096, 0, 0, 4095, 0, 3650, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3607, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1079, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 4093, 4096, 4096, 0, 12, 4096, 0, 3843, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3772, 4074, 0, 0, 0, 0, 4096, 0, 0, 0, 590, 0, 0, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[42, 58, 14, 34, 32, 304, 0, 0, 67, 0, 4, 3, 0, 13, 0, 4046, 0, 3700, 55, 0, 0, 18, 4032, 4051, 4061, 4051, 40, 105, 3846, 0, 3753, 0, 1, 0, 0, 87, 0, 575, 4004, 4048, 41, 0, 34, 0, 4010, 43, 4055, 26, 39, 2674, 3735, 45, 20, 50, 173, 3965, 13, 0, 0, 1475, 0, 0, 4054, 40] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 3995, 4096, 4096, 0, 106, 4096, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3905, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 311, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 5, 4096, 0, 2, 0, 0, 4094, 4096, 4096, 4095, 0, 1, 4094, 3, 3940, 1, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 4057, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 193, 2, 0, 4091, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4, 14, 3, 13, 5, 63, 0, 0, 18, 0, 0, 1, 0, 3, 0, 4087, 0, 4079, 4, 0, 0, 1, 4081, 4087, 4087, 4086, 20, 55, 4002, 0, 4084, 0, 0, 0, 0, 13, 0, 9, 4075, 4086, 9, 4, 7, 0, 4081, 7, 4085, 10, 9, 4019, 3754, 8, 5, 11, 43, 4050, 0, 0, 1, 357, 0, 0, 4088, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4016, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3902, 194, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 80, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 11, 4092, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4096, 0, 0, 0, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 4089, 0, 3531, 64, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8192, 8269, 8328, 8128, 8189, 8528, 7769] avg=8192.0000 max=8528 max_rank=6 min=7769 min_rank=7 imbalance_ratio=1.041016 max_load_reduction=57008 max_load_reduction_pct=0.869873 moved_to_echo=32614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 4096, 4016, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3902, 194, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 80, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 4095, 0, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3607, 4096, 0, 0, 0, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 0, 0, 4096, 0, 3531, 64, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8192, 8128, 7918, 8128, 8189, 8074, 8773] avg=8192.0000 max=8773 max_rank=7 min=7918 min_rank=3 imbalance_ratio=1.070923 max_load_reduction=56763 max_load_reduction_pct=0.866135 moved_to_echo=33164 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[42, 58, 14, 34, 32, 304, 0, 0, 4061, 3953, 21, 0, 0, 0, 0, 0, 67, 0, 4, 3, 0, 13, 0, 4046, 3860, 188, 0, 0, 0, 0, 0, 0, 0, 3700, 55, 0, 0, 18, 79, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 40, 105, 3846, 0, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 87, 0, 575, 4004, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 34, 0, 4010, 43, 4053, 26, 0, 0, 0, 0, 0, 0, 0, 0, 39, 2674, 3735, 45, 20, 50, 173, 359, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1475, 0, 0, 4054, 40, 3418, 64, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8519, 8181, 7903, 7914, 8651, 8207, 7095, 9066] avg=8192.0000 max=9066 max_rank=7 min=7095 min_rank=6 imbalance_ratio=1.106689 max_load_reduction=56470 max_load_reduction_pct=0.861664 moved_to_echo=33019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4016, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3902, 194, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 80, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 5, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 4093, 0, 0, 0, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 4085, 0, 3531, 64, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8192, 8271, 8369, 8128, 8189, 8521, 7733] avg=8192.0000 max=8521 max_rank=6 min=7733 min_rank=7 imbalance_ratio=1.040161 max_load_reduction=57015 max_load_reduction_pct=0.869980 moved_to_echo=32571 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4016, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3902, 194, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 80, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 106, 4096, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 4093, 0, 0, 0, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 4096, 0, 3531, 64, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8192, 8171, 8349, 8128, 8189, 8369, 8005] avg=8192.0000 max=8369 max_rank=6 min=8005 min_rank=7 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32691 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4016, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3902, 194, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 80, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 12, 4096, 0, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 4074, 0, 0, 0, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 4095, 0, 3531, 64, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8192, 8268, 8124, 8130, 8189, 8217, 8283] avg=8192.0000 max=8283 max_rank=7 min=8124 min_rank=3 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32819 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4014, 21, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 3901, 194, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 2, 0, 0, 80, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 1, 4094, 3, 3940, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 4094, 0, 0, 0, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 2, 0, 4091, 0, 3531, 64, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8193, 8279, 8212, 8130, 8185, 8522, 7884] avg=8192.0000 max=8522 max_rank=6 min=7884 min_rank=7 imbalance_ratio=1.040283 max_load_reduction=57014 max_load_reduction_pct=0.869965 moved_to_echo=32721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4, 14, 3, 13, 5, 63, 0, 0, 4087, 4005, 26, 0, 0, 0, 0, 0, 18, 0, 0, 1, 0, 3, 0, 4087, 3899, 194, 0, 0, 0, 0, 0, 0, 0, 4079, 4, 0, 0, 1, 76, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 20, 55, 4002, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 9, 4075, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 7, 0, 4081, 7, 4081, 10, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4019, 3754, 8, 5, 11, 43, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 357, 0, 0, 4088, 8, 3495, 70, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8202, 8247, 8322, 8113, 8199, 8210, 8023] avg=8192.0000 max=8322 max_rank=3 min=8023 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32545 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32384, 128, 128, 0, 128, 640, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 128, 384, 0, 0, 32768, 32128, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 30208, 32128, 128, 128, 128, 256, 3072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 4224, 128, 0, 32768, 128, 28160, 640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 0, 640, 32768, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 1408, 128, 384, 32512, 128, 31488, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 0, 32768, 31232, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2, 79, 2809, 5, 437, 0, 1189, 1, 2993, 44, 439, 1418, 924, 242, 0, 827, 216, 2445, 1716, 922, 3927, 659, 309, 1774, 0, 3581, 11, 596, 321, 15, 100, 1378, 4047, 98, 32, 770, 1391, 19, 397, 2309, 2903, 2379, 266, 240, 11, 2585, 1074, 1984, 59, 1504, 425, 24, 31, 78, 2489, 234, 8, 3219, 3703, 1813, 272, 1481, 307, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[11, 160, 2647, 13, 376, 0, 1113, 0, 2812, 155, 429, 1005, 1077, 285, 1, 953, 300, 2127, 2212, 760, 3882, 978, 222, 1363, 0, 3630, 69, 799, 293, 30, 181, 1434, 3925, 124, 43, 1222, 1063, 11, 589, 2392, 3188, 2217, 569, 427, 3, 2385, 1182, 2219, 57, 1491, 599, 13, 53, 57, 2733, 197, 6, 3052, 3900, 1409, 201, 625, 252, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[8, 266, 2545, 1, 402, 1, 666, 22, 2839, 119, 291, 848, 674, 125, 0, 860, 277, 1894, 2389, 702, 3683, 773, 305, 1615, 0, 3275, 143, 923, 403, 20, 270, 1538, 3973, 97, 100, 1006, 1629, 16, 756, 2309, 2752, 3020, 571, 261, 3, 2004, 1179, 2112, 189, 2016, 468, 30, 8, 36, 2471, 368, 1, 3036, 3545, 1696, 473, 922, 561, 51] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3, 132, 2442, 13, 737, 1, 1078, 1, 2803, 48, 277, 831, 970, 140, 0, 1411, 296, 2384, 1909, 583, 3803, 723, 296, 1865, 0, 3578, 102, 1142, 242, 21, 131, 1285, 3850, 127, 20, 1070, 1121, 18, 605, 2451, 2806, 2379, 526, 260, 7, 2297, 1103, 1679, 52, 1577, 526, 3, 6, 92, 2623, 214, 0, 3377, 3750, 2090, 142, 1242, 263, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2, 203, 1937, 9, 465, 0, 1147, 5, 3331, 43, 431, 1628, 1054, 328, 1, 1032, 240, 2205, 1825, 319, 3920, 1044, 183, 1607, 1, 3815, 4, 551, 216, 9, 156, 1420, 3955, 53, 30, 1572, 1105, 10, 412, 2027, 3395, 2248, 298, 124, 4, 2392, 1185, 2009, 46, 1982, 561, 4, 18, 28, 2475, 183, 2, 3596, 3912, 1468, 307, 691, 307, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[4, 81, 2390, 3, 434, 0, 938, 0, 2776, 77, 209, 816, 843, 190, 0, 1240, 266, 2252, 2203, 997, 3672, 795, 351, 1537, 0, 3593, 145, 758, 314, 34, 103, 1405, 4072, 82, 70, 1121, 1690, 23, 587, 1668, 3381, 2344, 253, 163, 2, 2207, 1715, 1874, 75, 1191, 727, 17, 14, 21, 3456, 395, 1, 3180, 3789, 1276, 214, 1105, 377, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[23, 63, 1697, 108, 637, 0, 827, 24, 2708, 348, 417, 403, 604, 298, 13, 1173, 331, 1801, 1622, 725, 3821, 1520, 359, 1808, 14, 3248, 118, 1190, 312, 21, 219, 1450, 3700, 133, 138, 1146, 1237, 22, 700, 1844, 2717, 2322, 564, 191, 63, 2327, 1462, 2576, 102, 1066, 907, 78, 58, 158, 2596, 261, 2, 3294, 3881, 2164, 326, 928, 563, 108] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2, 26, 2025, 0, 676, 1, 1152, 1, 2856, 71, 190, 1066, 409, 176, 1, 644, 184, 2662, 1831, 614, 3786, 1051, 178, 1882, 0, 3399, 39, 617, 399, 10, 221, 1776, 4029, 68, 24, 1017, 1377, 27, 661, 2829, 2999, 2706, 739, 202, 4, 2232, 1163, 2322, 61, 1718, 478, 9, 2, 18, 2340, 193, 5, 3407, 3716, 1334, 199, 1127, 579, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 79, 2809, 5, 437, 0, 1189, 1, 3781, 497, 0, 0, 0, 0, 0, 0, 2993, 44, 439, 1418, 924, 242, 0, 827, 1183, 618, 0, 0, 0, 0, 0, 0, 216, 2445, 1716, 922, 146, 659, 309, 1774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3581, 11, 596, 321, 15, 100, 1378, 1743, 32, 0, 0, 0, 0, 0, 0, 2832, 98, 32, 770, 1391, 19, 397, 2309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1761, 266, 240, 11, 2585, 1074, 1984, 0, 0, 0, 0, 0, 0, 0, 0, 59, 1504, 425, 24, 31, 78, 2489, 234, 2406, 437, 0, 0, 0, 0, 0, 0, 8, 3219, 1523, 1813, 272, 1481, 307, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8800, 8688, 8187, 7777, 7848, 7921, 7687, 8628] avg=8192.0000 max=8800 max_rank=0 min=7687 min_rank=6 imbalance_ratio=1.074219 max_load_reduction=56736 max_load_reduction_pct=0.865723 moved_to_echo=32084 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[8, 266, 2545, 1, 402, 1, 666, 22, 3546, 471, 0, 0, 0, 0, 0, 0, 2839, 119, 291, 848, 674, 125, 0, 860, 1161, 784, 0, 0, 0, 0, 0, 0, 277, 1894, 2389, 702, 137, 773, 305, 1615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3275, 143, 923, 403, 20, 270, 1538, 1669, 31, 0, 0, 0, 0, 0, 0, 2781, 97, 100, 1006, 1629, 16, 756, 2309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2236, 571, 261, 3, 2004, 1179, 2112, 0, 0, 0, 0, 0, 0, 0, 0, 189, 2016, 468, 30, 8, 36, 2471, 368, 2281, 418, 0, 0, 0, 0, 0, 0, 1, 3036, 1458, 1696, 473, 922, 561, 51, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7928, 7701, 8092, 8272, 8694, 8366, 8285, 8198] avg=8192.0000 max=8694 max_rank=4 min=7701 min_rank=1 imbalance_ratio=1.061279 max_load_reduction=56842 max_load_reduction_pct=0.867340 moved_to_echo=33543 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 132, 2442, 13, 737, 1, 1078, 1, 3662, 480, 0, 0, 0, 0, 0, 0, 2803, 48, 277, 831, 970, 140, 0, 1411, 1125, 618, 0, 0, 0, 0, 0, 0, 296, 2384, 1909, 583, 141, 723, 296, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 102, 1142, 242, 21, 131, 1285, 1765, 30, 0, 0, 0, 0, 0, 0, 2695, 127, 20, 1070, 1121, 18, 605, 2451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1761, 526, 260, 7, 2297, 1103, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1577, 526, 3, 6, 92, 2623, 214, 2326, 442, 0, 0, 0, 0, 0, 0, 0, 3377, 1543, 2090, 142, 1242, 263, 13, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8549, 8223, 8197, 8296, 8107, 7633, 7861, 8670] avg=8192.0000 max=8670 max_rank=7 min=7633 min_rank=5 imbalance_ratio=1.058350 max_load_reduction=56866 max_load_reduction_pct=0.867706 moved_to_echo=32271 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 160, 2647, 13, 376, 0, 1113, 0, 3738, 545, 0, 0, 0, 0, 0, 0, 2812, 155, 429, 1005, 1077, 285, 1, 953, 1147, 576, 0, 0, 0, 0, 0, 0, 300, 2127, 2212, 760, 144, 978, 222, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3630, 69, 799, 293, 30, 181, 1434, 1836, 31, 0, 0, 0, 0, 0, 0, 2747, 124, 43, 1222, 1063, 11, 589, 2392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1641, 569, 427, 3, 2385, 1182, 2219, 0, 0, 0, 0, 0, 0, 0, 0, 57, 1491, 599, 13, 53, 57, 2733, 197, 2643, 460, 0, 0, 0, 0, 0, 0, 6, 3052, 1604, 1409, 201, 625, 252, 15, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8603, 8440, 8106, 8303, 8191, 8426, 8303, 7164] avg=8192.0000 max=8603 max_rank=0 min=7164 min_rank=7 imbalance_ratio=1.050171 max_load_reduction=56933 max_load_reduction_pct=0.868729 moved_to_echo=32084 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 203, 1937, 9, 465, 0, 1147, 5, 3774, 581, 0, 0, 0, 0, 0, 0, 3331, 43, 431, 1628, 1054, 328, 1, 1032, 1156, 584, 0, 0, 0, 0, 0, 0, 240, 2205, 1825, 319, 146, 1044, 183, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3815, 4, 551, 216, 9, 156, 1420, 1841, 31, 0, 0, 0, 0, 0, 0, 2768, 53, 30, 1572, 1105, 10, 412, 2027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1664, 298, 124, 4, 2392, 1185, 2009, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1982, 561, 4, 18, 28, 2475, 183, 2814, 461, 0, 0, 0, 0, 0, 0, 2, 3596, 1610, 1468, 307, 691, 307, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 9588, 7569, 8044, 7977, 7676, 8572, 7987] avg=8192.0000 max=9588 max_rank=1 min=7569 min_rank=2 imbalance_ratio=1.170410 max_load_reduction=55948 max_load_reduction_pct=0.853699 moved_to_echo=32212 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[23, 63, 1697, 108, 637, 0, 827, 24, 3679, 466, 0, 0, 0, 0, 0, 0, 2708, 348, 417, 403, 604, 298, 13, 1173, 1081, 603, 0, 0, 0, 0, 0, 0, 331, 1801, 1622, 725, 142, 1520, 359, 1808, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3248, 118, 1190, 312, 21, 219, 1450, 1827, 29, 0, 0, 0, 0, 0, 0, 2590, 133, 138, 1146, 1237, 22, 700, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1719, 564, 191, 63, 2327, 1462, 2576, 0, 0, 0, 0, 0, 0, 0, 0, 102, 1066, 907, 78, 58, 158, 2596, 261, 2251, 458, 0, 0, 0, 0, 0, 0, 2, 3294, 1596, 2164, 326, 928, 563, 108, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7524, 7648, 8308, 8428, 7810, 8902, 7935, 8981] avg=8192.0000 max=8981 max_rank=7 min=7524 min_rank=0 imbalance_ratio=1.096313 max_load_reduction=56555 max_load_reduction_pct=0.862961 moved_to_echo=33628 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 81, 2390, 3, 434, 0, 938, 0, 3539, 579, 0, 0, 0, 0, 0, 0, 2776, 77, 209, 816, 843, 190, 0, 1240, 1193, 611, 0, 0, 0, 0, 0, 0, 266, 2252, 2203, 997, 133, 795, 351, 1537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3593, 145, 758, 314, 34, 103, 1405, 1788, 35, 0, 0, 0, 0, 0, 0, 2844, 82, 70, 1121, 1690, 23, 587, 1668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1733, 253, 163, 2, 2207, 1715, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 75, 1191, 727, 17, 14, 21, 3456, 395, 2802, 452, 0, 0, 0, 0, 0, 0, 1, 3180, 1549, 1276, 214, 1105, 377, 20, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7968, 7955, 8534, 8175, 8085, 7947, 9150, 7722] avg=8192.0000 max=9150 max_rank=6 min=7722 min_rank=7 imbalance_ratio=1.116943 max_load_reduction=56386 max_load_reduction_pct=0.860382 moved_to_echo=32904 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 26, 2025, 0, 676, 1, 1152, 1, 3645, 513, 0, 0, 0, 0, 0, 0, 2856, 71, 190, 1066, 409, 176, 1, 644, 1177, 703, 0, 0, 0, 0, 0, 0, 184, 2662, 1831, 614, 141, 1051, 178, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3399, 39, 617, 399, 10, 221, 1776, 1749, 31, 0, 0, 0, 0, 0, 0, 2821, 68, 24, 1017, 1377, 27, 661, 2829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 739, 202, 4, 2232, 1163, 2322, 0, 0, 0, 0, 0, 0, 0, 0, 61, 1718, 478, 9, 2, 18, 2340, 193, 2486, 438, 0, 0, 0, 0, 0, 0, 5, 3407, 1529, 1334, 199, 1127, 579, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8041, 7293, 8543, 8241, 8824, 8665, 7743, 8186] avg=8192.0000 max=8824 max_rank=4 min=7293 min_rank=1 imbalance_ratio=1.077148 max_load_reduction=56712 max_load_reduction_pct=0.865356 moved_to_echo=33418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2176, 17792, 15744, 5632, 1152, 7552, 2304, 13568, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 14592, 3840, 1920, 128, 18432, 10112, 16896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 26240, 12416, 13312, 2176, 8192, 3328, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[22144, 896, 512, 8960, 10624, 256, 4736, 17920, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[23168, 1024, 2688, 8064, 6656, 1792, 128, 8192, 9344, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 1024, 18560, 256, 4224, 128, 8192, 128, 29440, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 12672, 4736, 256, 256, 512, 21248, 2048, 20096, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 28160, 640, 6656, 2560, 256, 1408, 11776, 14336, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 3, 0, 8, 0, 1, 24, 4091, 9, 0, 3534, 29, 4067, 4073, 4094, 1, 0, 3, 0, 4084, 4090, 4089, 0, 0, 4036, 1, 12, 3, 127, 0, 26, 6, 4032, 4040, 0, 35, 0, 0, 0, 4077, 0, 0, 0, 4091, 4071, 0, 2, 12, 8, 4026, 0, 2, 2, 2, 1, 1, 23, 0, 0, 617, 0, 4078, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[125, 0, 1, 0, 51, 64, 27, 4056, 66, 33, 4080, 0, 4082, 4066, 4094, 4, 32, 0, 20, 4094, 4082, 4066, 2, 0, 2609, 3, 0, 2, 287, 0, 22, 4, 4078, 3249, 7, 1, 2, 0, 71, 3992, 0, 0, 0, 4086, 4017, 1, 25, 0, 0, 4030, 1, 0, 1, 9, 0, 0, 2, 0, 0, 1903, 1, 4088, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 4096, 62, 0, 4037, 0, 4040, 4095, 4093, 0, 0, 0, 1, 4095, 4084, 4093, 0, 3, 3567, 1, 0, 0, 22, 6, 0, 1, 4089, 4096, 0, 0, 0, 1, 0, 3925, 4, 4, 0, 4091, 4093, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 3, 0, 744, 0, 4092, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 6, 0, 0, 4096, 7, 1, 3980, 0, 3894, 4091, 4093, 4, 0, 0, 0, 4096, 4089, 4093, 0, 3, 3485, 0, 0, 0, 17, 0, 7, 0, 4096, 4089, 5, 2, 0, 0, 5, 2543, 0, 2, 0, 4096, 4096, 0, 2, 0, 0, 4092, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2441, 0, 4095, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 2, 16, 3632, 1, 4064, 4093, 4080, 0, 2, 0, 0, 4080, 4067, 4094, 0, 19, 2665, 0, 0, 0, 5, 5, 0, 0, 4094, 4069, 19, 0, 0, 0, 0, 3798, 0, 3, 0, 4082, 4095, 0, 1, 10, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 2238, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4093, 17, 7, 4000, 3, 4064, 4093, 4096, 0, 1, 0, 1, 4094, 4070, 4082, 0, 1, 4063, 2, 3, 0, 6, 3, 0, 1, 4094, 4089, 5, 0, 1, 1, 4, 4090, 0, 0, 0, 4091, 4094, 1, 1, 0, 0, 4092, 0, 2, 0, 1, 4, 0, 0, 0, 0, 184, 2, 4079, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 6, 1, 2, 1, 5, 0, 4088, 12, 13, 3850, 8, 4060, 4096, 4094, 9, 0, 0, 0, 4090, 4086, 4096, 0, 1, 4057, 27, 14, 1, 5, 3, 0, 0, 4076, 4092, 0, 1, 0, 0, 2, 4088, 0, 0, 1, 4096, 4096, 1, 7, 0, 0, 4082, 21, 9, 0, 4, 0, 0, 0, 0, 4, 235, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2349, 0, 4073, 4094, 4096, 0, 0, 0, 0, 4096, 4095, 4095, 0, 4, 3990, 3, 0, 0, 8, 0, 0, 0, 4096, 4063, 1, 0, 0, 0, 0, 4094, 2, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1897, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 3, 0, 8, 0, 1, 24, 4091, 3185, 834, 0, 0, 0, 0, 0, 0, 9, 0, 3534, 29, 4067, 428, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 4090, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 1, 12, 3, 127, 0, 26, 6, 4094, 460, 0, 0, 0, 0, 0, 0, 369, 4040, 0, 35, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4071, 0, 2, 12, 8, 0, 0, 0, 0, 0, 0, 0, 8, 4026, 0, 2, 2, 2, 1, 1, 3614, 478, 0, 0, 0, 0, 0, 0, 23, 0, 0, 617, 0, 4078, 4, 0, 2803, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8068, 8192, 8765, 8521, 8184, 8134, 7525] avg=8192.0000 max=8765 max_rank=3 min=7525 min_rank=7 imbalance_ratio=1.069946 max_load_reduction=56771 max_load_reduction_pct=0.866257 moved_to_echo=32364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4093, 3234, 838, 0, 0, 0, 0, 0, 0, 17, 7, 4000, 3, 4064, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 10, 4070, 4082, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 2, 3, 0, 6, 3, 0, 1, 4096, 461, 0, 0, 0, 0, 0, 0, 374, 4089, 5, 0, 1, 1, 4, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4094, 1, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 1, 4, 0, 3623, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 2, 4079, 0, 0, 2817, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8521, 8165, 8635, 8564, 8195, 8208, 7082] avg=8192.0000 max=8635 max_rank=3 min=7082 min_rank=7 imbalance_ratio=1.054077 max_load_reduction=56901 max_load_reduction_pct=0.868240 moved_to_echo=32049 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 4096, 3230, 838, 0, 0, 0, 0, 0, 0, 62, 0, 4037, 0, 4040, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 4084, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3567, 1, 0, 0, 22, 6, 0, 1, 4093, 461, 0, 0, 0, 0, 0, 0, 374, 4096, 0, 0, 0, 1, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 4091, 4093, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3624, 485, 0, 0, 0, 0, 0, 0, 1, 3, 0, 744, 0, 4092, 0, 1, 2818, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8570, 8191, 8151, 8396, 8200, 8204, 7659] avg=8192.0000 max=8570 max_rank=1 min=7659 min_rank=7 imbalance_ratio=1.046143 max_load_reduction=56966 max_load_reduction_pct=0.869232 moved_to_echo=32459 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 1, 2, 1, 5, 0, 4088, 3220, 838, 0, 0, 0, 0, 0, 0, 12, 13, 3850, 8, 4060, 431, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4086, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 27, 14, 1, 5, 3, 0, 0, 4094, 461, 0, 0, 0, 0, 0, 0, 373, 4092, 0, 1, 0, 0, 2, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 1, 7, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 21, 9, 0, 4, 0, 0, 3619, 483, 0, 0, 0, 0, 0, 0, 0, 0, 4, 235, 0, 4096, 0, 0, 2819, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8383, 8193, 8662, 8556, 8209, 8218, 7154] avg=8192.0000 max=8662 max_rank=3 min=7154 min_rank=7 imbalance_ratio=1.057373 max_load_reduction=56874 max_load_reduction_pct=0.867828 moved_to_echo=32137 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[125, 0, 1, 0, 51, 64, 27, 4056, 3221, 832, 0, 0, 0, 0, 0, 0, 66, 33, 4080, 0, 4082, 428, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 20, 10, 4082, 4066, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2609, 3, 0, 2, 287, 0, 22, 4, 4094, 461, 0, 0, 0, 0, 0, 0, 373, 3249, 7, 1, 2, 0, 71, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4017, 1, 25, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 1, 0, 1, 9, 0, 0, 3623, 484, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1903, 1, 4088, 0, 0, 2798, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8377, 8693, 8212, 7482, 7695, 8137, 8148, 8792] avg=8192.0000 max=8792 max_rank=7 min=7482 min_rank=3 imbalance_ratio=1.073242 max_load_reduction=56744 max_load_reduction_pct=0.865845 moved_to_echo=32772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 3234, 838, 0, 0, 0, 0, 0, 0, 2, 16, 3632, 1, 4064, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 10, 4067, 4094, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 0, 0, 0, 5, 5, 0, 0, 4080, 460, 0, 0, 0, 0, 0, 0, 374, 4069, 19, 0, 0, 0, 0, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4082, 4095, 0, 1, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 14, 0, 3610, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2238, 0, 4096, 0, 0, 2817, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8145, 8192, 7215, 8260, 8199, 8206, 9151] avg=8192.0000 max=9151 max_rank=7 min=7215 min_rank=3 imbalance_ratio=1.117065 max_load_reduction=56385 max_load_reduction_pct=0.860367 moved_to_echo=33816 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 6, 0, 0, 4096, 3236, 837, 0, 0, 0, 0, 0, 0, 7, 1, 3980, 0, 3894, 431, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4089, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 0, 0, 0, 17, 0, 7, 0, 4093, 461, 0, 0, 0, 0, 0, 0, 374, 4089, 5, 2, 0, 0, 5, 2543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 2, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 2, 3625, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2441, 0, 4095, 0, 7, 2815, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8317, 8195, 8063, 7018, 8204, 8205, 9358] avg=8192.0000 max=9358 max_rank=7 min=7018 min_rank=4 imbalance_ratio=1.142334 max_load_reduction=56178 max_load_reduction_pct=0.857208 moved_to_echo=32785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 3239, 841, 0, 0, 0, 0, 0, 0, 0, 0, 2349, 0, 4073, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 4095, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 3, 0, 0, 8, 0, 0, 0, 4096, 466, 0, 0, 0, 0, 0, 0, 368, 4063, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3628, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1897, 0, 4096, 0, 0, 2822, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 6839, 8196, 8563, 8526, 8208, 8213, 8815] avg=8192.0000 max=8815 max_rank=7 min=6839 min_rank=1 imbalance_ratio=1.076050 max_load_reduction=56721 max_load_reduction_pct=0.865494 moved_to_echo=33762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 32640, 128, 128, 128, 128, 128, 128, 29056, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 128, 29568, 128, 32384, 3456, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 32768, 25856, 6784, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 10368, 128, 32768, 128, 128, 22528, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[28544, 128, 128, 128, 512, 128, 128, 128, 32768, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3072, 31872, 128, 128, 128, 128, 128, 30720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 4025, 25, 4060, 8, 0, 3383, 4096, 4031, 0, 0, 0, 12, 4092, 4, 0, 4092, 2, 0, 4080, 4044, 190, 0, 128, 1, 0, 23, 0, 4083, 0, 0, 4088, 13, 0, 4054, 66, 189, 0, 0, 15, 0, 1, 201, 0, 65, 0, 71, 4093, 0, 4079, 0, 4, 4, 0, 9, 4095, 0, 0, 0, 4096, 0, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 14, 4078, 0, 4058, 1, 0, 4054, 4096, 4096, 0, 0, 10, 0, 4080, 0, 0, 4060, 0, 0, 4072, 3866, 36, 0, 9, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 2, 4095, 16, 30, 0, 0, 1, 0, 0, 104, 0, 12, 0, 24, 4085, 0, 4096, 0, 0, 156, 0, 3, 4096, 0, 0, 0, 4092, 0, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 5, 4094, 0, 4096, 5, 3, 4033, 4096, 4093, 0, 0, 0, 1, 4084, 0, 0, 2587, 0, 0, 4096, 4083, 10, 0, 132, 3, 0, 1, 0, 4095, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 0, 2, 0, 754, 0, 8, 0, 69, 4088, 0, 4096, 0, 0, 603, 0, 5, 4096, 0, 0, 2, 4096, 0, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 4079, 0, 4094, 0, 0, 3971, 4091, 4093, 0, 0, 0, 0, 4094, 0, 0, 3482, 0, 0, 4089, 3415, 8, 0, 128, 2, 2, 2, 0, 4094, 1, 0, 4095, 0, 0, 4095, 0, 357, 0, 0, 2, 3, 0, 927, 0, 6, 0, 17, 4092, 0, 4095, 0, 0, 7, 0, 0, 4096, 0, 1, 2, 4095, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4053, 0, 4096, 0, 0, 4075, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 3387, 0, 0, 4096, 3084, 217, 0, 561, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 236, 0, 0, 0, 0, 0, 400, 0, 54, 0, 322, 4079, 0, 4096, 0, 0, 6, 0, 0, 4096, 0, 3, 0, 4096, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4058, 0, 4094, 3, 0, 4075, 4094, 4092, 0, 0, 0, 0, 4096, 0, 0, 3452, 0, 0, 4086, 2713, 0, 0, 641, 1, 1, 0, 0, 4088, 0, 0, 4095, 0, 2, 4082, 0, 14, 0, 0, 1, 0, 1, 1211, 1, 0, 0, 194, 4089, 2, 4095, 0, 1, 56, 0, 0, 4096, 0, 0, 0, 4095, 1, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3993, 0, 4079, 27, 129, 3302, 4096, 4096, 0, 0, 0, 1, 4096, 0, 1, 3103, 1, 0, 4060, 744, 412, 4, 40, 0, 0, 0, 0, 3755, 0, 0, 4079, 10, 1, 4096, 0, 618, 1, 0, 0, 0, 0, 2431, 3, 156, 0, 1700, 3894, 0, 4095, 0, 0, 272, 0, 0, 4088, 1, 4, 0, 4094, 0, 51, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4095, 0, 0, 2851, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4071, 0, 0, 4096, 3724, 1, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 4096, 0, 5, 0, 0, 0, 0, 0, 1484, 0, 1, 0, 104, 4096, 0, 4096, 0, 0, 41, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4025, 25, 385, 8, 0, 3383, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4031, 0, 0, 0, 12, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 1342, 4044, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 1, 0, 23, 0, 4083, 0, 0, 2738, 878, 184, 111, 10, 0, 0, 0, 3904, 13, 0, 4054, 66, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 201, 0, 65, 0, 71, 4085, 2797, 0, 0, 0, 0, 0, 0, 4093, 0, 3968, 0, 4, 4, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4086, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7832, 8150, 9670, 8156, 8226, 7235, 8078, 8189] avg=8192.0000 max=9670 max_rank=2 min=7235 min_rank=5 imbalance_ratio=1.180420 max_load_reduction=55866 max_load_reduction_pct=0.852448 moved_to_echo=31728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 14, 4078, 0, 385, 1, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 10, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 1340, 3866, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4091, 0, 0, 2732, 878, 184, 112, 10, 0, 0, 0, 3912, 0, 2, 4095, 16, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 104, 0, 12, 0, 24, 4085, 2795, 0, 0, 0, 0, 0, 0, 4085, 0, 3984, 0, 0, 156, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4082, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8532, 8197, 9302, 8016, 8055, 7021, 8228, 8185] avg=8192.0000 max=9302 max_rank=2 min=7021 min_rank=5 imbalance_ratio=1.135498 max_load_reduction=56234 max_load_reduction_pct=0.858063 moved_to_echo=31489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4079, 0, 388, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4093, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3482, 0, 0, 1345, 3415, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 2, 2, 2, 0, 4094, 1, 0, 2744, 886, 184, 112, 10, 0, 0, 0, 3911, 0, 0, 4095, 0, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 927, 0, 6, 0, 17, 4080, 2820, 0, 0, 0, 0, 0, 0, 4092, 0, 3983, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 2, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8439, 8198, 8250, 8165, 8363, 7855, 8082, 8184] avg=8192.0000 max=8439 max_rank=0 min=7855 min_rank=5 imbalance_ratio=1.030151 max_load_reduction=57097 max_load_reduction_pct=0.871231 moved_to_echo=32484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4058, 0, 388, 3, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3452, 0, 0, 1344, 2713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 1, 1, 0, 0, 4088, 0, 0, 2742, 886, 184, 112, 10, 0, 0, 0, 3911, 0, 2, 4082, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1211, 1, 0, 0, 194, 4083, 2820, 0, 0, 0, 0, 0, 0, 4089, 2, 3983, 0, 1, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8524, 8199, 7509, 8665, 8009, 8311, 8131, 8188] avg=8192.0000 max=8665 max_rank=3 min=7509 min_rank=2 imbalance_ratio=1.057739 max_load_reduction=56871 max_load_reduction_pct=0.867783 moved_to_echo=32639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4053, 0, 388, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3387, 0, 0, 1348, 3084, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 0, 0, 0, 0, 4096, 0, 0, 2748, 886, 184, 112, 10, 0, 0, 0, 3912, 0, 0, 4096, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 54, 0, 322, 4085, 2822, 0, 0, 0, 0, 0, 0, 4079, 0, 3984, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4086, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8516, 8203, 8036, 8597, 8244, 7683, 8069, 8188] avg=8192.0000 max=8597 max_rank=3 min=7683 min_rank=5 imbalance_ratio=1.049438 max_load_reduction=56939 max_load_reduction_pct=0.868820 moved_to_echo=32184 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4094, 0, 388, 5, 3, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4093, 0, 0, 0, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2587, 0, 0, 1348, 4083, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 3, 0, 1, 0, 4095, 0, 0, 2748, 886, 184, 112, 10, 0, 0, 0, 3912, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 754, 0, 8, 0, 69, 4085, 2822, 0, 0, 0, 0, 0, 0, 4088, 0, 3984, 0, 0, 603, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4086, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8528, 8189, 8028, 8171, 8008, 7740, 8680, 8192] avg=8192.0000 max=8680 max_rank=6 min=7740 min_rank=5 imbalance_ratio=1.059570 max_load_reduction=56856 max_load_reduction_pct=0.867554 moved_to_echo=32620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3993, 0, 386, 27, 129, 3302, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3103, 1, 0, 1336, 744, 412, 4, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 3755, 0, 0, 2724, 883, 183, 112, 10, 0, 0, 0, 3896, 10, 1, 4096, 0, 618, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2431, 3, 156, 0, 1700, 4085, 2810, 0, 0, 0, 0, 0, 0, 3894, 0, 3983, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 4, 0, 4084, 0, 51, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7838, 8204, 5601, 7707, 8622, 11185, 8149, 8230] avg=8192.0000 max=11185 max_rank=5 min=5601 min_rank=2 imbalance_ratio=1.365356 max_load_reduction=54351 max_load_reduction_pct=0.829330 moved_to_echo=36186 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 380, 0, 0, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 1344, 3724, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 2752, 890, 189, 114, 14, 0, 0, 0, 3907, 0, 1, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 0, 1, 0, 104, 4092, 2825, 0, 0, 0, 0, 0, 0, 4096, 0, 3982, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7327, 8196, 9140, 8059, 8009, 8506, 8119, 8180] avg=8192.0000 max=9140 max_rank=2 min=7327 min_rank=0 imbalance_ratio=1.115723 max_load_reduction=56396 max_load_reduction_pct=0.860535 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32512, 128, 3200, 128, 256, 29824, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 28288, 128, 0, 10752, 25728, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 7552, 128, 384, 0, 2560, 32768, 22528, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 128, 31872, 0, 128, 1152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[1664, 128, 128, 128, 0, 32512, 128, 0, 22016, 7168, 1536, 1024, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31360, 128, 128, 32768, 128, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4086, 0, 0, 3959, 1, 1, 0, 4079, 0, 4094, 0, 3588, 0, 9, 158, 0, 2, 4096, 0, 22, 0, 4048, 1, 38, 4096, 0, 0, 0, 0, 0, 4060, 0, 3918, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 11, 19, 4036, 10, 3, 58, 25, 639, 0, 10, 0, 4089, 4096, 0, 60, 0, 0, 6, 0, 4093, 0, 4029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 0, 4043, 0, 0, 0, 4089, 278, 4095, 0, 3764, 0, 0, 4, 0, 0, 4096, 0, 26, 0, 4080, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4078, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 14, 0, 127, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4048, 4, 60, 0, 4081, 0, 4090, 0, 4081, 0, 0, 0, 0, 0, 4088, 14, 18, 0, 4084, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 2, 0, 0, 6, 1, 0, 0, 8, 0, 4094, 0, 0, 9, 0, 0, 0, 0, 3, 4093, 4096, 0, 3, 0, 0, 0, 0, 4085, 0, 4089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4050, 62, 125, 0, 4096, 0, 4084, 0, 4020, 0, 0, 0, 0, 0, 4048, 0, 57, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3830, 4, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4082, 0, 2, 58, 1, 50, 0, 0, 3, 4096, 4096, 0, 0, 0, 92, 0, 1, 4095, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4042, 2, 0, 0, 4096, 0, 4096, 0, 4069, 0, 1, 1, 0, 0, 4076, 0, 46, 0, 4087, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 4096, 3912, 0, 0, 1, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 0, 1, 10, 182, 0, 0, 1, 4096, 4096, 0, 5, 0, 18, 0, 0, 4090, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4079, 0, 4, 0, 4092, 8, 4095, 1, 4041, 0, 0, 48, 0, 2, 4091, 0, 16, 0, 4088, 1, 0, 4094, 0, 0, 6, 0, 0, 4093, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 2, 0, 2, 0, 0, 3, 4096, 4096, 2, 0, 0, 11, 0, 0, 4093, 1, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4084, 0, 0, 4086, 2, 1, 1, 4095, 15, 4095, 0, 4074, 1, 0, 26, 1, 2, 4095, 0, 0, 2, 4094, 0, 0, 4094, 0, 1, 0, 0, 0, 4096, 0, 4095, 4091, 1, 1, 0, 0, 1, 0, 1, 2, 0, 0, 4095, 0, 0, 2, 3, 1, 1, 1, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 4093, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4088, 0, 0, 0, 4096, 0, 4096, 0, 4041, 0, 0, 1, 0, 0, 4095, 0, 3, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 64, 0, 0, 0, 4096, 4096, 0, 0, 0, 6, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 0, 3959, 1, 1, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3588, 0, 9, 158, 0, 54, 12, 3, 0, 0, 0, 0, 0, 2, 4088, 0, 22, 0, 4048, 1, 38, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4060, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3918, 4096, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 19, 4036, 10, 3, 58, 4066, 0, 0, 0, 0, 0, 0, 0, 25, 639, 0, 10, 0, 4089, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 6, 0, 4081, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8060, 7918, 8199, 8161, 8098, 8203, 8721, 8176] avg=8192.0000 max=8721 max_rank=6 min=7918 min_rank=1 imbalance_ratio=1.064575 max_load_reduction=56815 max_load_reduction_pct=0.866928 moved_to_echo=33198 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 0, 4048, 4, 60, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4081, 0, 0, 0, 0, 54, 11, 3, 0, 0, 0, 0, 0, 0, 4080, 14, 18, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 2, 0, 0, 6, 1, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4094, 0, 0, 9, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4074, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8239, 8197, 8197, 8283, 8187, 8054, 8166] avg=8192.0000 max=8283 max_rank=4 min=8054 min_rank=6 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 0, 4079, 0, 4, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 1, 4041, 0, 0, 48, 0, 54, 12, 3, 0, 0, 0, 0, 0, 2, 4083, 0, 16, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 6, 0, 0, 4093, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 2, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 4096, 3958, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4081, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8262, 8190, 8198, 8270, 8172, 8061, 8188] avg=8192.0000 max=8270 max_rank=4 min=8061 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 0, 4043, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 278, 4095, 0, 3764, 0, 0, 4, 0, 54, 12, 3, 0, 0, 0, 0, 0, 0, 4088, 0, 26, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4078, 4062, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 14, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 4, 4096, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8210, 8194, 8197, 8224, 8194, 8185, 8180] avg=8192.0000 max=8224 max_rank=4 min=8152 min_rank=0 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 0, 4086, 2, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4095, 0, 4074, 1, 0, 26, 1, 54, 12, 3, 0, 0, 0, 0, 0, 2, 4087, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4095, 4091, 1, 1, 0, 0, 1, 0, 84, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 4095, 0, 0, 2, 4064, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1, 1, 0, 4094, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8281, 8185, 8196, 8273, 8164, 8057, 8175] avg=8192.0000 max=8281 max_rank=1 min=8057 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 0, 4050, 62, 125, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4020, 0, 0, 0, 0, 54, 12, 3, 0, 0, 0, 0, 0, 0, 4040, 0, 57, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 3830, 4, 0, 0, 0, 9, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 2, 58, 4076, 0, 0, 0, 0, 0, 0, 0, 1, 50, 0, 0, 3, 4096, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 1, 4083, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8353, 8173, 8193, 8197, 8023, 8218, 8108, 8271] avg=8192.0000 max=8353 max_rank=0 min=8023 min_rank=4 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 0, 4042, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4069, 0, 1, 1, 0, 54, 11, 3, 0, 0, 0, 0, 0, 0, 4068, 0, 46, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 3912, 0, 0, 1, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 0, 1, 4076, 0, 0, 0, 0, 0, 0, 0, 10, 182, 0, 0, 1, 4096, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 18, 0, 0, 4079, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8235, 8201, 8189, 8093, 8215, 8247, 8196] avg=8192.0000 max=8247 max_rank=6 min=8093 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4041, 0, 0, 1, 0, 59, 14, 7, 0, 0, 0, 0, 0, 0, 4078, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4082, 4, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 4096, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4082, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8218, 8177, 8201, 8272, 8183, 8103, 8184] avg=8192.0000 max=8272 max_rank=4 min=8103 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32640, 32384, 128, 128, 128, 0, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 256, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 1152, 128, 128, 128, 32768, 31744, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32640, 128, 256, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 0, 0, 32512, 128, 256, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[384, 32768, 128, 31744, 128, 128, 256, 128, 512, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4082, 4094, 0, 0, 4096, 1, 4075, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 3, 4096, 8, 0, 2, 0, 4096, 4096, 26, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 1, 4093, 4096, 0, 0, 4070, 0, 4090, 0, 0, 0, 4094, 0, 0, 0, 4060, 0, 0, 4096, 0, 0, 0, 41, 4096, 25, 12, 2, 0, 4080, 4096, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 6, 2, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 4093, 0, 0, 4083, 0, 4080, 0, 0, 0, 4091, 1, 0, 0, 4096, 2, 0, 4094, 0, 0, 0, 15, 4096, 11, 3, 0, 0, 4093, 4096, 6, 2, 4, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 4095, 0, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 18, 0, 4096, 0, 0, 0, 0, 88, 0, 0, 3961, 4069, 0, 0, 4055, 0, 4085, 0, 0, 0, 3892, 0, 0, 0, 4070, 0, 0, 4084, 0, 0, 1, 4, 4096, 68, 0, 14, 11, 4096, 4096, 47, 0, 0, 0, 4096, 0, 9, 0, 0, 4096, 0, 0, 226, 0, 0, 0, 4076, 0, 4, 0, 4093, 1, 4084, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 43, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4086, 0, 0, 4045, 0, 4007, 0, 0, 0, 3803, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 382, 7, 0, 1, 3958, 4096, 114, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 3, 25, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 4095, 0, 0, 0, 4016, 0, 0, 0, 4073, 0, 0, 4096, 0, 0, 0, 28, 4096, 102, 0, 1, 0, 4031, 4096, 22, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 16, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 1, 0, 0, 1, 4090, 4092, 0, 0, 4075, 2, 4095, 0, 0, 16, 3980, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 2, 59, 4095, 22, 0, 3, 0, 4096, 4095, 33, 0, 0, 0, 4095, 0, 1, 0, 1, 4092, 0, 1, 12, 0, 0, 0, 4096, 1, 0, 1, 4096, 0, 4096, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 3961, 28, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 3892, 0, 0, 63, 40, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 4084, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 4, 4021, 68, 0, 14, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 4096, 0, 9, 0, 4049, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 226, 0, 0, 0, 4001, 39, 0, 0, 0, 0, 0, 0, 4076, 0, 4, 0, 5, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8132, 8080, 8167, 8214, 8211, 8362, 8170] avg=8192.0000 max=8362 max_rank=6 min=8080 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32957 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 2, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 29, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 4091, 1, 0, 63, 40, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4094, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 15, 4021, 11, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 4, 0, 4096, 0, 0, 0, 4051, 11, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 2, 4024, 39, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 5, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8209, 8275, 8204, 8143, 8170, 8152, 8194] avg=8192.0000 max=8275 max_rank=2 min=8143 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 29, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4094, 0, 0, 63, 40, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 41, 4021, 25, 12, 2, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4052, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4027, 39, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8193, 8287, 8168, 8181, 8163, 8162, 8196] avg=8192.0000 max=8287 max_rank=2 min=8162 min_rank=6 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4082, 29, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4075, 0, 0, 0, 4096, 0, 0, 63, 40, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 3, 4021, 8, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 4096, 0, 0, 0, 4052, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4025, 39, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8208, 8275, 8200, 8130, 8185, 8160, 8197] avg=8192.0000 max=8275 max_rank=2 min=8130 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32672 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 10, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 28, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 3803, 0, 0, 63, 40, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 382, 7, 0, 1, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 4095, 0, 0, 0, 4052, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 25, 0, 0, 0, 4018, 42, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8168, 7913, 8206, 8369, 8271, 8184, 8190] avg=8192.0000 max=8369 max_rank=4 min=7913 min_rank=2 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=33014 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 29, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 16, 3980, 0, 0, 62, 40, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 59, 4021, 22, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 4095, 0, 1, 0, 4052, 11, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 1, 12, 0, 0, 0, 4023, 39, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 5, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8195, 8195, 8203, 8201, 8192, 8168, 8200] avg=8192.0000 max=8203 max_rank=3 min=8168 min_rank=6 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 29, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4016, 0, 0, 63, 40, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 28, 4021, 102, 0, 1, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 1, 0, 4096, 0, 0, 0, 4052, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 16, 0, 0, 0, 4027, 39, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8219, 8214, 8181, 8183, 8182, 8179, 8197] avg=8192.0000 max=8219 max_rank=1 min=8179 min_rank=6 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 64, 43, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 2, 4017, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4056, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4033, 40, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8212, 8297, 8207, 8115, 8162, 8169, 8192] avg=8192.0000 max=8297 max_rank=2 min=8115 min_rank=4 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32640, 0, 0, 128, 32128, 128, 0, 512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 32256, 640, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32640, 256, 0, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 32768, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 128, 128, 0, 32768, 0, 128, 0, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 128, 384, 0, 0, 128, 32256, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4088, 1, 0, 0, 0, 0, 0, 3955, 4096, 4096, 0, 66, 4051, 5, 3027, 0, 0, 0, 4096, 0, 0, 603, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 285, 2, 4067, 0, 0, 0, 3735, 0, 0, 0, 4096, 136, 0, 4085, 0, 18, 0, 0, 4096, 549, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 2, 0, 4084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 0, 0, 3739, 4094, 4096, 0, 70, 4095, 0, 2969, 0, 0, 0, 4094, 1, 0, 1557, 0, 37, 0, 0, 0, 4022, 0, 4096, 0, 0, 1, 4096, 2, 0, 0, 4018, 0, 0, 0, 4096, 0, 0, 4094, 2, 0, 0, 0, 4095, 13, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4001, 4091, 4096, 0, 5, 4096, 0, 3278, 0, 0, 0, 4096, 0, 0, 850, 0, 19, 0, 0, 0, 4096, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 4055, 0, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 0, 4094, 79, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 10, 0, 4052, 4096, 4096, 0, 13, 4096, 0, 3949, 0, 0, 0, 4096, 0, 0, 428, 0, 43, 0, 0, 0, 3967, 0, 4096, 0, 81, 0, 4071, 0, 0, 0, 3851, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 0, 0, 3612, 4096, 4096, 0, 107, 4096, 0, 2554, 0, 0, 0, 4096, 2, 0, 2151, 0, 147, 0, 0, 0, 3931, 0, 4096, 0, 46, 2, 4096, 0, 0, 0, 3836, 0, 0, 0, 4096, 7, 0, 4094, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 4081] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 1, 4094, 4093, 4095, 0, 5, 4093, 0, 3777, 0, 1, 0, 4095, 0, 1, 212, 1, 0, 0, 0, 0, 4096, 1, 4095, 0, 0, 0, 4096, 0, 0, 0, 4082, 1, 0, 0, 4095, 2, 0, 4096, 0, 3, 0, 0, 4095, 158, 0, 0, 1, 1, 1, 1, 1, 0, 4095, 0, 0, 4055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 1, 0, 4096, 4096, 4096, 0, 17, 4090, 10, 2704, 0, 0, 0, 4096, 0, 0, 1218, 0, 1, 0, 0, 0, 4084, 0, 4096, 0, 43, 0, 4096, 0, 0, 0, 3992, 0, 0, 0, 4096, 15, 0, 4094, 0, 35, 9, 0, 4096, 186, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4079, 4096, 4096, 0, 0, 4096, 0, 3971, 0, 0, 0, 4096, 0, 0, 293, 0, 0, 0, 0, 0, 4074, 0, 4096, 0, 55, 0, 4093, 0, 0, 0, 3882, 0, 0, 0, 4096, 0, 0, 4096, 0, 3, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4088, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 66, 916, 5, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 603, 0, 3024, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 13, 6, 0, 0, 0, 0, 0, 0, 285, 2, 4067, 0, 0, 0, 3735, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 136, 0, 4085, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 549, 0, 0, 0, 0, 3967, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 2, 0, 4084, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 7969, 7852, 8204, 8194, 8310, 8613, 8210] avg=8192.0000 max=8613 max_rank=6 min=7852 min_rank=2 imbalance_ratio=1.051392 max_load_reduction=56923 max_load_reduction_pct=0.868576 moved_to_echo=33327 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 0, 0, 70, 926, 0, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 1557, 0, 3057, 128, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 4022, 0, 4096, 0, 13, 6, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 2, 0, 0, 4018, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 13, 0, 0, 0, 0, 3966, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4054, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7704, 8837, 8174, 8223, 8167, 8075, 8163] avg=8192.0000 max=8837 max_rank=2 min=7704 min_rank=1 imbalance_ratio=1.078735 max_load_reduction=56699 max_load_reduction_pct=0.865158 moved_to_echo=32628 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 17, 925, 10, 2704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1218, 0, 3053, 128, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4084, 0, 4096, 0, 13, 6, 0, 0, 0, 0, 0, 0, 43, 0, 4096, 0, 0, 0, 3992, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 15, 0, 4094, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 186, 0, 0, 0, 0, 3968, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4075, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7752, 8495, 8200, 8237, 8215, 8260, 8183] avg=8192.0000 max=8495 max_rank=2 min=7752 min_rank=1 imbalance_ratio=1.036987 max_load_reduction=57041 max_load_reduction_pct=0.870377 moved_to_echo=32895 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 5, 926, 0, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 850, 0, 3058, 129, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 4096, 0, 13, 6, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4055, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 79, 0, 0, 0, 0, 3962, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8210, 8133, 8230, 8259, 8173, 8136, 8204] avg=8192.0000 max=8259 max_rank=4 min=8133 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 5, 926, 0, 3777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 212, 1, 3055, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4095, 0, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4082, 1, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 2, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 158, 0, 0, 1, 1, 3965, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 4095, 0, 0, 4055, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8802, 7493, 8211, 8285, 8171, 8221, 8165] avg=8192.0000 max=8802 max_rank=1 min=7493 min_rank=2 imbalance_ratio=1.074463 max_load_reduction=56734 max_load_reduction_pct=0.865692 moved_to_echo=32842 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 10, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 13, 926, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 428, 0, 3058, 129, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 3967, 0, 4096, 0, 13, 6, 0, 0, 0, 0, 0, 0, 81, 0, 4071, 0, 0, 0, 3851, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 3967, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8940, 7711, 8125, 8109, 8167, 8077, 8206] avg=8192.0000 max=8940 max_rank=1 min=7711 min_rank=2 imbalance_ratio=1.091309 max_load_reduction=56596 max_load_reduction_pct=0.863586 moved_to_echo=32559 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3612, 0, 0, 0, 107, 926, 0, 2554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 2151, 0, 3058, 129, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 3931, 0, 4096, 0, 13, 6, 0, 0, 0, 0, 0, 0, 46, 2, 4096, 0, 0, 0, 3836, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 7, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 4081, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7199, 9436, 8193, 8086, 8172, 8060, 8197] avg=8192.0000 max=9436 max_rank=2 min=7199 min_rank=1 imbalance_ratio=1.151855 max_load_reduction=56100 max_load_reduction_pct=0.856018 moved_to_echo=32515 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 910, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 293, 0, 3061, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4096, 0, 17, 12, 0, 0, 0, 0, 0, 0, 55, 0, 4093, 0, 0, 0, 3882, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 31, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8960, 7579, 8199, 8143, 8161, 8094, 8208] avg=8192.0000 max=8960 max_rank=1 min=7579 min_rank=2 imbalance_ratio=1.093750 max_load_reduction=56576 max_load_reduction_pct=0.863281 moved_to_echo=32606 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 32640, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31744, 0, 0, 0, 384, 7424, 128, 26240, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 32768, 1152, 0, 0, 128, 128, 31744, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32640, 256, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[256, 0, 0, 0, 32384, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 128, 32768, 128, 0, 0, 31488, 128, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 32768, 128, 128, 7424, 128, 24448, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[547, 0, 0, 0, 634, 0, 4096, 0, 0, 0, 174, 0, 1, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 2961, 1, 0, 4096, 4092, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 7, 3488, 0, 0, 4074, 0, 283, 1, 0, 4096, 2, 0, 4096, 101, 0, 0, 24, 4096, 4096, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[2761, 1, 0, 0, 49, 1, 4095, 2, 0, 2, 76, 1, 0, 0, 3652, 1, 0, 1, 0, 4092, 1, 0, 0, 0, 2, 4095, 0, 4094, 3378, 1, 0, 4095, 4092, 0, 0, 0, 1, 0, 4094, 5, 0, 0, 4093, 4, 2179, 0, 0, 4096, 2, 185, 3, 0, 4095, 0, 0, 4095, 2, 1, 0, 0, 4094, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[2328, 4, 0, 0, 455, 0, 4095, 0, 0, 19, 340, 0, 5, 0, 2872, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 4096, 3805, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 4082, 12, 0, 0, 4096, 0, 1839, 0, 0, 4026, 0, 548, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 149, 4096, 4087, 2, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3574, 0, 0, 0, 115, 0, 4096, 0, 0, 0, 69, 0, 0, 0, 3208, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3369, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 1872, 0, 0, 4023, 0, 39, 0, 0, 4096, 0, 0, 4096, 99, 0, 0, 1, 4096, 4096, 15, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3186, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 37, 0, 0, 0, 3811, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3736, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1343, 0, 0, 4075, 0, 108, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 85, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1217, 0, 0, 0, 3101, 0, 4096, 0, 0, 0, 2227, 0, 0, 0, 3755, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 1289, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 33, 0, 0, 4066, 0, 0, 0, 0, 4096, 0, 0, 4096, 578, 0, 0, 117, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[3489, 0, 0, 0, 25, 0, 4096, 0, 0, 0, 346, 0, 0, 0, 4021, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3732, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 588, 0, 0, 3983, 0, 6, 0, 0, 4096, 0, 0, 4096, 94, 0, 0, 53, 4096, 4096, 47, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[3039, 0, 0, 0, 26, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 3935, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2107, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 3165, 0, 0, 4096, 0, 6, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 1, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2328, 4, 0, 0, 455, 0, 4095, 0, 708, 172, 146, 0, 0, 0, 0, 0, 0, 19, 340, 0, 5, 0, 2872, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 3027, 1065, 0, 0, 0, 0, 0, 0, 0, 1049, 0, 0, 3805, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4082, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 1839, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 0, 0, 3949, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 3924, 4087, 2, 16, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7908, 7347, 8193, 8943, 8191, 8188, 8588, 8178] avg=8192.0000 max=8943 max_rank=3 min=7347 min_rank=1 imbalance_ratio=1.091675 max_load_reduction=56593 max_load_reduction_pct=0.863541 moved_to_echo=33145 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[547, 0, 0, 0, 634, 0, 4096, 0, 708, 172, 146, 0, 0, 0, 0, 0, 0, 0, 174, 0, 1, 0, 4090, 0, 4095, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3031, 1065, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 2961, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 7, 3488, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 1, 0, 3949, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 24, 3924, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6303, 8375, 8192, 8108, 8188, 9892, 8331, 8147] avg=8192.0000 max=9892 max_rank=5 min=6303 min_rank=0 imbalance_ratio=1.207520 max_load_reduction=55644 max_load_reduction_pct=0.849060 moved_to_echo=34558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2761, 1, 0, 0, 49, 1, 4095, 2, 707, 172, 145, 0, 0, 0, 0, 0, 0, 2, 76, 1, 0, 0, 3652, 1, 4094, 15, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 1, 0, 0, 0, 3030, 1064, 0, 0, 0, 0, 0, 0, 2, 1050, 0, 0, 3378, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 1, 0, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2322, 4, 2179, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 185, 3, 0, 3950, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 3922, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7933, 7841, 8188, 8526, 8192, 8601, 8235, 8020] avg=8192.0000 max=8601 max_rank=5 min=7841 min_rank=1 imbalance_ratio=1.049927 max_load_reduction=56935 max_load_reduction_pct=0.868759 moved_to_echo=33048 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3574, 0, 0, 0, 115, 0, 4096, 0, 708, 172, 146, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 3208, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3031, 1064, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 3369, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2322, 0, 1872, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 3949, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 1, 3924, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8811, 7388, 8191, 8517, 8193, 8217, 8084, 8135] avg=8192.0000 max=8811 max_rank=0 min=7388 min_rank=1 imbalance_ratio=1.075562 max_load_reduction=56725 max_load_reduction_pct=0.865555 moved_to_echo=32629 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3186, 0, 0, 0, 3, 0, 4096, 0, 708, 172, 146, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 3811, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3031, 1065, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 3736, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 1343, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 3949, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 85, 3924, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 7959, 8192, 8881, 8193, 7741, 8153, 8106] avg=8192.0000 max=8881 max_rank=3 min=7741 min_rank=5 imbalance_ratio=1.084106 max_load_reduction=56655 max_load_reduction_pct=0.864487 moved_to_echo=32193 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3489, 0, 0, 0, 25, 0, 4096, 0, 708, 172, 146, 0, 0, 0, 0, 0, 0, 0, 346, 0, 0, 0, 4021, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3031, 1065, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 3732, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 588, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3949, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 53, 3924, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8636, 8478, 8192, 8878, 8193, 6894, 8051, 8214] avg=8192.0000 max=8878 max_rank=3 min=6894 min_rank=5 imbalance_ratio=1.083740 max_load_reduction=56658 max_load_reduction_pct=0.864532 moved_to_echo=31352 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1217, 0, 0, 0, 3101, 0, 4096, 0, 708, 172, 146, 0, 0, 0, 0, 0, 0, 0, 2227, 0, 0, 0, 3755, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3031, 1065, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 1289, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 1, 33, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 578, 0, 0, 117, 3924, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9440, 10093, 8192, 6435, 8193, 6423, 8045, 8715] avg=8192.0000 max=10093 max_rank=1 min=6423 min_rank=5 imbalance_ratio=1.232056 max_load_reduction=55443 max_load_reduction_pct=0.845993 moved_to_echo=31376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3039, 0, 0, 0, 26, 0, 4096, 0, 712, 174, 147, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3935, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3033, 1067, 0, 0, 0, 0, 0, 0, 0, 1045, 0, 0, 2107, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2317, 2, 3165, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3947, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 3922, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8055, 8196, 7248, 8193, 9580, 8049, 8021] avg=8192.0000 max=9580 max_rank=5 min=7248 min_rank=3 imbalance_ratio=1.169434 max_load_reduction=55956 max_load_reduction_pct=0.853821 moved_to_echo=33843 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[896, 128, 0, 512, 31488, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 3328, 128, 128, 0, 29440, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[20224, 128, 0, 0, 4480, 128, 32768, 128, 5760, 1408, 1280, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 18688, 128, 14592, 0, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 8448, 0, 0, 24448, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 32768, 128, 0, 0, 0, 24320, 8576, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 1280, 128, 0, 31616, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 3974, 4096, 0, 0, 1, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 124, 4072, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 9, 0, 1, 0, 10, 1, 4090, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 16, 8, 7, 0, 4096, 0, 0, 0, 0, 0, 4076, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 7, 0, 19, 4071, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 49, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 4096, 4096, 0, 8, 6, 0, 0, 0, 0, 0, 5, 4081, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4094, 4093, 6, 1, 1, 1, 4094, 0, 0, 9, 0, 0, 4086, 4095, 0, 0, 0, 4095, 4096, 1, 0, 0, 0, 0, 1, 0, 2, 8, 4089, 0, 0, 1, 0, 1, 4095, 0, 4095, 0, 1, 4095, 0, 4095, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 1, 12, 0, 0, 0, 0, 2, 4078, 0, 0, 1, 4095, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4095, 4093, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4076, 4096, 0, 0, 7, 4096, 4096, 0, 20, 0, 0, 0, 0, 6, 0, 4, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4059, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4093, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 99, 0, 12, 0, 0, 0, 4006, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4093, 4096, 16, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 0, 0, 14, 6, 3, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 19, 4071, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 49, 0, 0, 0, 4054, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8195, 8187, 8174, 8193, 8192, 8217, 8158] avg=8192.0000 max=8220 max_rank=0 min=8158 min_rank=7 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 14, 6, 3, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4081, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4054, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8212, 8200, 8162, 8193, 8192, 8171, 8217] avg=8192.0000 max=8217 max_rank=7 min=8162 min_rank=3 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 14, 6, 3, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 4054, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8206, 8189, 8202, 8192, 8192, 8174, 8192] avg=8192.0000 max=8206 max_rank=1 min=8174 min_rank=6 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4092, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4076, 4096, 0, 0, 14, 6, 3, 0, 0, 0, 0, 0, 7, 4096, 4090, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4, 4096, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 1, 0, 0, 0, 4054, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8196, 8213, 8183, 8192, 8192, 8162, 8213] avg=8192.0000 max=8213 max_rank=2 min=8162 min_rank=6 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4091, 4093, 6, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4086, 4095, 0, 0, 14, 6, 3, 0, 0, 0, 0, 0, 0, 4095, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 8, 4089, 0, 0, 1, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 12, 0, 0, 0, 4054, 19, 0, 0, 0, 0, 0, 0, 0, 2, 4078, 0, 0, 1, 27, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8213, 8186, 8176, 8192, 8192, 8181, 8203] avg=8192.0000 max=8213 max_rank=1 min=8176 min_rank=3 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32768 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3974, 4096, 0, 0, 14, 6, 3, 0, 0, 0, 0, 0, 1, 4096, 4090, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 4072, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 9, 0, 1, 0, 4054, 19, 0, 0, 0, 0, 0, 0, 10, 1, 4090, 0, 0, 0, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8097, 8189, 8273, 8192, 8193, 8178, 8225] avg=8192.0000 max=8273 max_rank=3 min=8097 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4096, 0, 0, 14, 6, 3, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4093, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 99, 0, 12, 0, 4054, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8178, 8186, 8192, 8192, 8192, 8277, 8130] avg=8192.0000 max=8277 max_rank=6 min=8130 min_rank=7 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32791 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 4096, 0, 0, 15, 6, 10, 2, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4077, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4061, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 20, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8239, 8186, 8174, 8190, 8191, 8176, 8198] avg=8192.0000 max=8239 max_rank=1 min=8174 min_rank=3 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 32640, 32768, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32640, 0, 0, 128, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 128, 256, 0, 128, 0, 32512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 256, 32768, 0, 0, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 3, 0, 0, 4096, 3, 0, 5, 4094, 0, 4096, 0, 0, 4091, 0, 0, 0, 4096, 0, 4096, 0, 107, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 3935, 4087, 0, 0, 4096, 68, 4096, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 0, 2, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 1168, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 3014, 3989, 0, 5, 4096, 32, 4096, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 15, 0, 0, 4096, 803, 0, 42, 4095, 0, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 4096, 0, 668, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 4086, 2787, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 1, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 4096, 2, 0, 0, 4093, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 311, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 3639, 4095, 0, 31, 4096, 122, 4095, 0, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 11, 0, 0, 0, 4055, 0, 0, 0, 0, 1, 0, 0, 4096, 1, 0, 23, 4076, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 55, 0, 2, 0, 0, 0, 4090, 0, 4096, 0, 4035, 0, 0, 0, 0, 36, 0, 0, 4095, 4024, 0, 67, 4096, 1, 4096, 0, 0, 6, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 18, 0, 0, 4096, 20, 0, 0, 4092, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 4086, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4095, 1, 4094, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 4096, 0, 4095, 0, 1, 4096, 0, 0, 0, 4095, 0, 4095, 1, 150, 0, 1, 0, 1, 0, 4093, 1, 4095, 0, 4089, 2, 0, 2, 0, 0, 1, 0, 3914, 4057, 0, 29, 4095, 48, 4095, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 175, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3789, 4095, 0, 0, 4096, 133, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 1, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4083, 7, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 0, 0, 0, 4094, 0, 3687, 80, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 2, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3639, 4095, 0, 31, 402, 122, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8187, 8079, 8194, 8172, 8224, 8289, 8191] avg=8192.0000 max=8289 max_rank=6 min=8079 min_rank=2 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 4096, 4083, 7, 0, 0, 0, 0, 0, 0, 3, 0, 5, 4094, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 4096, 0, 3687, 80, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 4087, 0, 0, 402, 68, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8191, 8086, 8191, 7970, 8205, 8492, 8199] avg=8192.0000 max=8492 max_rank=6 min=7970 min_rank=4 imbalance_ratio=1.036621 max_load_reduction=57044 max_load_reduction_pct=0.870422 moved_to_echo=32866 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 11, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4083, 7, 0, 0, 0, 0, 0, 0, 1, 0, 23, 4076, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 2, 0, 0, 0, 4090, 0, 3687, 80, 0, 0, 0, 0, 0, 0, 4096, 0, 4035, 0, 0, 0, 0, 36, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4024, 0, 67, 402, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8187, 8084, 8194, 7914, 8199, 8589, 8195] avg=8192.0000 max=8589 max_rank=6 min=7914 min_rank=4 imbalance_ratio=1.048462 max_load_reduction=56947 max_load_reduction_pct=0.868942 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 7, 0, 0, 0, 0, 0, 0, 25, 0, 2, 4096, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 0, 0, 0, 0, 0, 4096, 0, 3687, 80, 0, 0, 0, 0, 0, 0, 4096, 0, 4056, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3014, 3989, 0, 5, 402, 32, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8186, 8107, 8196, 9031, 8184, 7442, 8190] avg=8192.0000 max=9031 max_rank=4 min=7442 min_rank=6 imbalance_ratio=1.102417 max_load_reduction=56505 max_load_reduction_pct=0.862198 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 1, 4094, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4083, 7, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4096, 0, 3983, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 1, 0, 1, 0, 4093, 1, 3686, 80, 0, 0, 0, 0, 0, 0, 4095, 0, 4089, 2, 0, 2, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3914, 4057, 0, 29, 402, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8185, 8082, 8196, 8012, 8220, 8451, 8187] avg=8192.0000 max=8451 max_rank=6 min=8012 min_rank=4 imbalance_ratio=1.031616 max_load_reduction=57085 max_load_reduction_pct=0.871048 moved_to_echo=32870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4096, 4083, 7, 0, 0, 0, 0, 0, 0, 803, 0, 42, 4095, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 0, 4094, 0, 3687, 80, 0, 0, 0, 0, 0, 0, 4096, 0, 3952, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 2787, 0, 2, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8201, 8924, 8176, 8529, 8080, 7277, 8192] avg=8192.0000 max=8924 max_rank=2 min=7277 min_rank=6 imbalance_ratio=1.089355 max_load_reduction=56612 max_load_reduction_pct=0.863831 moved_to_echo=32078 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 4083, 7, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4092, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 3687, 80, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4094, 0, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8204, 8096, 8196, 7866, 8199, 8582, 8189] avg=8192.0000 max=8582 max_rank=6 min=7866 min_rank=4 imbalance_ratio=1.047607 max_load_reduction=56954 max_load_reduction_pct=0.869049 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 4096, 0, 3690, 81, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 4095, 0, 0, 397, 133, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8195, 8078, 8193, 8042, 8225, 8414, 8193] avg=8192.0000 max=8414 max_rank=6 min=8042 min_rank=4 imbalance_ratio=1.027100 max_load_reduction=57122 max_load_reduction_pct=0.871613 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 32512, 128, 0, 128, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 128, 0, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 30592, 31232, 0, 256, 3328, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[896, 128, 128, 32768, 0, 31872, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2688, 0, 128, 0, 128, 0, 32768, 128, 29568, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 4083, 0, 4096, 4073, 0, 13, 0, 0, 4096, 0, 28, 0, 0, 4083, 0, 0, 4096, 0, 3663, 4096, 0, 0, 4096, 0, 44, 434, 0, 4000, 0, 0, 4054, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 96, 0, 4088, 0, 4096, 11, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1, 1400, 0, 9, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 45, 0, 0, 4095, 0, 0, 4096, 0, 1251, 4096, 0, 0, 4096, 0, 2, 1619, 0, 3894, 0, 0, 4071, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 128, 0, 0, 0, 4096, 0, 4096, 4087, 0, 0, 0, 0, 4096, 2, 26, 0, 0, 4096, 0, 0, 4096, 0, 3849, 4096, 0, 0, 4096, 0, 9, 123, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4063, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 4095, 0, 4095, 4095, 0, 0, 1, 0, 4095, 10, 0, 0, 1, 4094, 1, 0, 4095, 0, 4085, 4085, 1, 1, 4096, 0, 0, 10, 0, 4095, 1, 0, 4093, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 3, 1, 1, 0, 0, 4095, 0, 0, 4096, 0, 4095, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 449, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 4091, 0, 0, 4096, 0, 3369, 4096, 0, 0, 4096, 0, 7, 280, 0, 4089, 0, 0, 4092, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 506, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 4096, 0, 0, 4096, 0, 3297, 4096, 0, 0, 4096, 0, 6, 436, 0, 4073, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3964, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 4092, 0, 0, 4096, 0, 3842, 4096, 0, 0, 4095, 0, 32, 247, 0, 4064, 0, 0, 4090, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 4096, 0, 9, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 214, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3789, 4096, 0, 0, 4096, 0, 0, 92, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 4083, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 13, 0, 0, 4081, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 706, 0, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3680, 0, 44, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 4054, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 3390, 416, 338, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 11, 0, 0, 0, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7843, 8195, 8452, 8254, 8109, 8191, 8276, 8216] avg=8192.0000 max=8452 max_rank=2 min=7843 min_rank=0 imbalance_ratio=1.031738 max_load_reduction=57084 max_load_reduction_pct=0.871033 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 449, 0, 0, 0, 4096, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 706, 0, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3680, 0, 7, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4092, 0, 0, 4, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 3390, 416, 338, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8183, 8166, 8063, 8240, 8191, 8180, 8210] avg=8192.0000 max=8303 max_rank=0 min=8063 min_rank=3 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 4095, 0, 3757, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4080, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 706, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 1, 3680, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4093, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4040, 3389, 416, 338, 6, 0, 0, 0, 4095, 3, 1, 1, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 1, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7856, 8186, 8887, 7777, 8244, 8190, 8183, 8213] avg=8192.0000 max=8887 max_rank=2 min=7777 min_rank=3 imbalance_ratio=1.084839 max_load_reduction=56649 max_load_reduction_pct=0.864395 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 4096, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 706, 0, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3679, 0, 32, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4090, 0, 0, 2, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 3390, 416, 338, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 9, 0, 0, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7861, 8193, 8640, 8054, 8211, 8191, 8180, 8206] avg=8192.0000 max=8640 max_rank=2 min=7861 min_rank=0 imbalance_ratio=1.054688 max_load_reduction=56896 max_load_reduction_pct=0.868164 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 1400, 0, 9, 0, 4096, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4081, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 706, 0, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3680, 0, 2, 1619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3894, 0, 0, 4071, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 3390, 416, 338, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9264, 8221, 6052, 9397, 8020, 8191, 8180, 8211] avg=8192.0000 max=9397 max_rank=3 min=6052 min_rank=2 imbalance_ratio=1.147095 max_load_reduction=56139 max_load_reduction_pct=0.856613 moved_to_echo=32602 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 128, 0, 0, 0, 4096, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4081, 2, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 706, 0, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3680, 0, 9, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4041, 3390, 416, 338, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 4096, 0, 0, 0, 0, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7982, 8196, 8651, 7908, 8247, 8192, 8180, 8180] avg=8192.0000 max=8651 max_rank=2 min=7908 min_rank=3 imbalance_ratio=1.056030 max_load_reduction=56885 max_load_reduction_pct=0.867996 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 506, 0, 0, 0, 4096, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 706, 0, 3297, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3680, 0, 6, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4091, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 3390, 416, 338, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 4096, 0, 0, 0, 0, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8360, 8188, 8099, 8218, 8219, 8191, 8180, 8081] avg=8192.0000 max=8360 max_rank=0 min=8081 min_rank=7 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 214, 0, 0, 0, 4096, 0, 3757, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 704, 0, 3789, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3677, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 3392, 419, 339, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 16, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8067, 8174, 8589, 7865, 8246, 8199, 8177, 8219] avg=8192.0000 max=8589 max_rank=2 min=7865 min_rank=3 imbalance_ratio=1.048462 max_load_reduction=56947 max_load_reduction_pct=0.868942 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 2816, 0, 128, 0, 32768, 0, 30080, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32512, 128, 0, 32768, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32640, 0, 32768, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 128, 0, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 0, 5760, 0, 27264, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 29440, 0, 128, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 128, 0, 0, 0, 32384, 27136, 3456, 2816, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4067, 0, 0, 0, 0, 0, 0, 4096, 0, 39, 0, 0, 4096, 4096, 4096, 4, 56, 4096, 4096, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4040, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4078, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 4096, 4096, 4096, 0, 50, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 3842, 0, 0, 0, 0, 0, 0, 4096, 0, 208, 0, 0, 4096, 4096, 4096, 59, 273, 4096, 4096, 0, 4096, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3942, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4084, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 4096, 4096, 4095, 0, 140, 4096, 4096, 0, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 45, 0, 0, 0, 4096, 4, 0, 0, 2, 0, 3914, 9, 0, 1, 0, 0, 0, 4096, 0, 0, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4095, 4079, 1, 0, 0, 0, 0, 2, 4095, 0, 2, 0, 0, 4095, 4095, 4092, 0, 49, 4095, 4095, 0, 4095, 4083, 0, 0, 0, 1, 0, 1, 7, 0, 0, 0, 0, 0, 0, 4093, 4094, 1, 1, 0, 4, 1, 0, 3, 4089, 1, 1, 0, 0, 0, 4076, 0, 1, 12, 0, 1, 1, 4095, 1, 1, 4078, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 27, 0, 0, 4096, 4096, 4095, 0, 3, 4096, 4096, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 9, 0, 4096, 4096, 4096, 0, 139, 4096, 4095, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3981, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 842, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3736, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3616, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4084, 0, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 140, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 4096, 4, 4015, 75, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3914, 9, 0, 1, 4096, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 98, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8175, 8129, 8191, 8192, 8235, 8104, 8289] avg=8192.0000 max=8289 max_rank=7 min=8104 min_rank=6 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32820 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4067, 0, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 4, 56, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4014, 75, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4040, 0, 0, 0, 4096, 82, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 98, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8191, 8062, 8191, 8192, 8185, 8221, 8290] avg=8192.0000 max=8290 max_rank=7 min=8062 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32888 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 9, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 0, 139, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4013, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 4096, 82, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3981, 0, 98, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8163, 8155, 8192, 8192, 8184, 8245, 8175] avg=8192.0000 max=8245 max_rank=6 min=8155 min_rank=2 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32796 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4078, 0, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 0, 50, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4014, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 4096, 82, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0, 98, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8170, 8067, 8191, 8192, 8185, 8237, 8279] avg=8192.0000 max=8279 max_rank=7 min=8067 min_rank=2 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32893 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 4079, 1, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 4055, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 49, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 1, 0, 3, 4089, 1, 4013, 75, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4076, 0, 1, 12, 4095, 82, 1, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 1, 1, 4078, 0, 98, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8154, 8050, 8198, 8188, 8187, 8268, 8275] avg=8192.0000 max=8275 max_rank=7 min=8050 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32918 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3842, 0, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 59, 273, 0, 0, 0, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4015, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3942, 0, 98, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7979, 8360, 8219, 8191, 8192, 8186, 8273, 8136] avg=8192.0000 max=8360 max_rank=1 min=7979 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32787 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4094, 0, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 3, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4014, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 82, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0, 98, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8179, 8003, 8191, 8192, 8187, 8274, 8279] avg=8192.0000 max=8279 max_rank=7 min=8003 min_rank=2 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32932 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4094, 0, 0, 0, 0, 0, 44, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 842, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4015, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3736, 0, 0, 0, 4096, 81, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3616, 0, 101, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8144, 8851, 8191, 8196, 8187, 7914, 7813] avg=8192.0000 max=8851 max_rank=2 min=7813 min_rank=7 imbalance_ratio=1.080444 max_load_reduction=56685 max_load_reduction_pct=0.864944 moved_to_echo=32110 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31488, 128, 1664, 0, 0, 0, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 32512, 128, 0, 0, 0, 0, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 32000, 0, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 0, 128, 32768, 128, 32128, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 32128, 128, 128, 128, 32768, 768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 384, 128, 0, 32512, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 71, 0, 0, 4086, 0, 0, 0, 148, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 0, 4096, 0, 4079, 0, 0, 3945, 0, 21, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4036, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 41, 0, 3, 1609, 0, 0, 0, 182, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4083, 84, 4096, 0, 3848, 0, 0, 4041, 0, 2, 0, 0, 4096, 4096, 0, 4096, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 2563, 0, 1, 0, 0, 4016, 4094, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3940, 0, 0, 0, 102, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 2, 4096, 0, 3695, 0, 0, 4053, 0, 1, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 485, 0, 1, 0, 0, 4093, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4063, 0, 0, 0, 402, 4096, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4096, 4071, 0, 4096, 0, 3690, 0, 0, 4015, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 137, 0, 2, 0, 0, 4087, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 1898, 0, 0, 0, 1441, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4072, 6, 4096, 0, 3711, 0, 0, 4050, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1208, 0, 0, 0, 0, 4090, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4, 935, 1, 0, 1, 3357, 4094, 0, 0, 0, 1, 1, 0, 4094, 2, 0, 0, 4095, 4094, 0, 4095, 0, 3742, 1, 1, 4082, 0, 9, 0, 1, 4095, 4095, 1, 4095, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 4095, 0, 1, 0, 1, 1, 0, 157, 0, 0, 0, 2, 4095, 4095, 4095, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 0, 3986, 0, 0, 0, 88, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 3998, 0, 0, 4077, 0, 3, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 122, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2528, 0, 0, 0, 859, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4029, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 776, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 41, 0, 3, 1609, 0, 0, 4076, 683, 534, 0, 0, 0, 0, 0, 0, 182, 4096, 0, 0, 0, 0, 0, 3268, 3, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4083, 84, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 3848, 0, 0, 4041, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2563, 0, 4096, 3413, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4016, 4094, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6946, 7551, 8260, 8185, 8197, 8192, 10074, 8131] avg=8192.0000 max=10074 max_rank=6 min=6946 min_rank=0 imbalance_ratio=1.229736 max_load_reduction=55462 max_load_reduction_pct=0.846283 moved_to_echo=34594 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 1898, 0, 0, 4076, 683, 534, 0, 0, 0, 0, 0, 0, 1441, 4096, 0, 0, 0, 0, 0, 3268, 3, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4072, 6, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 3711, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1208, 0, 4096, 3413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7193, 8810, 8171, 8055, 8192, 8192, 8717, 8206] avg=8192.0000 max=8810 max_rank=1 min=7193 min_rank=0 imbalance_ratio=1.075439 max_load_reduction=56726 max_load_reduction_pct=0.865570 moved_to_echo=33307 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4063, 0, 0, 4076, 683, 534, 0, 0, 0, 0, 0, 0, 402, 4096, 0, 0, 0, 0, 0, 3268, 3, 2, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 3690, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 4096, 3413, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4087, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9356, 7771, 8168, 7999, 8199, 8192, 7646, 8205] avg=8192.0000 max=9356 max_rank=0 min=7646 min_rank=6 imbalance_ratio=1.142090 max_load_reduction=56180 max_load_reduction_pct=0.857239 moved_to_echo=32242 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 71, 0, 0, 4086, 0, 0, 4076, 684, 534, 0, 0, 0, 0, 0, 0, 148, 4096, 0, 0, 0, 0, 0, 3268, 3, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 4079, 0, 0, 3945, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9451, 7517, 8186, 8339, 8190, 8192, 7509, 8152] avg=8192.0000 max=9451 max_rank=0 min=7509 min_rank=6 imbalance_ratio=1.153687 max_load_reduction=56085 max_load_reduction_pct=0.855789 moved_to_echo=32043 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4, 935, 1, 0, 4075, 683, 534, 0, 0, 0, 0, 0, 1, 3357, 4094, 0, 0, 0, 1, 1, 3267, 3, 1, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 3742, 1, 1, 4082, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 157, 0, 4095, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 4095, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6234, 10725, 8187, 8129, 8191, 8190, 7667, 8213] avg=8192.0000 max=10725 max_rank=1 min=6234 min_rank=0 imbalance_ratio=1.309204 max_load_reduction=54811 max_load_reduction_pct=0.836349 moved_to_echo=32261 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3940, 0, 0, 4076, 683, 534, 0, 0, 0, 0, 0, 0, 102, 4096, 0, 0, 0, 0, 0, 3268, 3, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 3695, 0, 0, 4053, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 485, 0, 4096, 3413, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9234, 7471, 8190, 8043, 8190, 8192, 8006, 8210] avg=8192.0000 max=9234 max_rank=0 min=7471 min_rank=1 imbalance_ratio=1.127197 max_load_reduction=56302 max_load_reduction_pct=0.859100 moved_to_echo=32598 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 0, 3986, 0, 0, 4076, 683, 534, 0, 0, 0, 0, 0, 0, 88, 4096, 0, 0, 0, 0, 0, 3268, 3, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 3998, 0, 0, 4077, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 122, 0, 4096, 3413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9291, 7457, 8189, 8372, 8190, 8192, 7633, 8212] avg=8192.0000 max=9291 max_rank=0 min=7457 min_rank=1 imbalance_ratio=1.134155 max_load_reduction=56245 max_load_reduction_pct=0.858231 moved_to_echo=32227 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2528, 0, 0, 4081, 684, 538, 0, 0, 0, 0, 0, 0, 859, 4096, 0, 0, 0, 0, 0, 3269, 7, 3, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, 4029, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 0, 4096, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7831, 8234, 8185, 8414, 8187, 8194, 8284, 8207] avg=8192.0000 max=8414 max_rank=3 min=7831 min_rank=0 imbalance_ratio=1.027100 max_load_reduction=57122 max_load_reduction_pct=0.871613 moved_to_echo=32872 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 6656, 32768, 0, 0, 0, 128, 128, 26240, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 23168, 128, 0, 32640, 5504, 4352, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32640, 32768, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 0, 5504, 0, 32768, 27392, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 0, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2432, 0, 30848, 128, 128, 32384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 29, 0, 84, 0, 0, 0, 4086, 4096, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 3861, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 132, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 967, 0, 13, 0, 0, 0, 4096, 4096, 0, 4096, 3179, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3941, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4086, 97, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 129, 0, 36, 0, 0, 0, 4096, 4096, 0, 4096, 3540, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3973, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 520, 4095, 0, 4096, 0, 4096, 4092, 0, 0, 0, 4096, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 0, 9, 0, 379, 0, 0, 0, 4096, 4096, 0, 4096, 4013, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 3610, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 288, 4096, 0, 4096, 0, 4091, 3972, 0, 0, 0, 4096, 17, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 2, 0, 32, 0, 0, 0, 4096, 4096, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3942, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4081, 143, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 4096, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 3115, 1, 0, 0, 1, 1, 4095, 4095, 4, 4095, 758, 0, 1, 0, 1, 1, 1, 11, 0, 0, 1, 2, 0, 0, 0, 1, 4092, 0, 0, 4095, 0, 4095, 1, 35, 0, 4077, 1, 1, 4093, 1, 0, 0, 1, 0, 1, 4095, 194, 4095, 0, 4095, 0, 4094, 4095, 0, 0, 0, 4095, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 37, 0, 6, 0, 0, 0, 4096, 4096, 0, 4096, 3990, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3917, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 196, 4096, 0, 4096, 0, 4096, 4090, 0, 0, 0, 4096, 45, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 262, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3855, 0, 0, 0, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3445, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 418, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 18, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 29, 0, 84, 3436, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 657, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4067, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 0, 0, 4089, 0, 0, 0, 0, 244, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 132, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7668, 8835, 8168, 8191, 8188, 8201, 8080, 8205] avg=8192.0000 max=8835 max_rank=1 min=7668 min_rank=0 imbalance_ratio=1.078491 max_load_reduction=56701 max_load_reduction_pct=0.865189 moved_to_echo=32674 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 129, 0, 36, 3436, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 657, 0, 0, 3540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 30, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 0, 4093, 0, 0, 0, 0, 244, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 520, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7720, 8293, 8163, 8193, 8188, 8317, 8464, 8198] avg=8192.0000 max=8464 max_rank=6 min=7720 min_rank=0 imbalance_ratio=1.033203 max_load_reduction=57072 max_load_reduction_pct=0.870850 moved_to_echo=33167 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 967, 0, 13, 3436, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 657, 0, 0, 3179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 0, 4096, 0, 0, 0, 0, 244, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4086, 97, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8535, 7932, 8164, 8192, 8188, 8288, 8035, 8202] avg=8192.0000 max=8535 max_rank=0 min=7932 min_rank=1 imbalance_ratio=1.041870 max_load_reduction=57001 max_load_reduction_pct=0.869766 moved_to_echo=32713 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 2, 0, 32, 3436, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 657, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 0, 0, 4090, 0, 0, 0, 0, 244, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4081, 143, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7589, 8844, 8167, 8192, 8188, 8283, 8076, 8197] avg=8192.0000 max=8844 max_rank=1 min=7589 min_rank=0 imbalance_ratio=1.079590 max_load_reduction=56692 max_load_reduction_pct=0.865051 moved_to_echo=32744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 3115, 1, 0, 3435, 22, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 658, 4, 0, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 11, 0, 4095, 4066, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 1, 4092, 4066, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4091, 1, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 1, 1, 4093, 1, 0, 0, 1, 244, 4, 2, 0, 0, 0, 0, 0, 0, 1, 4095, 194, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10668, 5517, 8176, 8191, 8222, 8424, 8141, 8197] avg=8192.0000 max=10668 max_rank=0 min=5517 min_rank=1 imbalance_ratio=1.302246 max_load_reduction=54868 max_load_reduction_pct=0.837219 moved_to_echo=32984 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 0, 9, 0, 379, 3436, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 657, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 0, 0, 4096, 0, 0, 0, 0, 244, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 288, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3972, 0, 0, 0, 6, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7939, 8766, 8171, 8192, 8189, 7957, 8236, 8086] avg=8192.0000 max=8766 max_rank=1 min=7939 min_rank=0 imbalance_ratio=1.070068 max_load_reduction=56770 max_load_reduction_pct=0.866241 moved_to_echo=32468 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 37, 0, 6, 3436, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 657, 0, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 4096, 0, 0, 0, 0, 244, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 196, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 6, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7598, 8743, 8170, 8192, 8188, 8264, 8144, 8237] avg=8192.0000 max=8743 max_rank=1 min=7598 min_rank=0 imbalance_ratio=1.067261 max_load_reduction=56793 max_load_reduction_pct=0.866592 moved_to_echo=32833 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 262, 0, 0, 3437, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 655, 0, 0, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3445, 0, 0, 4096, 0, 0, 0, 0, 250, 7, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 418, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7819, 8606, 8357, 8193, 8185, 7802, 8360, 8214] avg=8192.0000 max=8606 max_rank=1 min=7802 min_rank=5 imbalance_ratio=1.050537 max_load_reduction=56930 max_load_reduction_pct=0.868683 moved_to_echo=32561 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 0, 0, 4608, 128, 640, 27520, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 2048, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 5376, 128, 0, 27520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[30848, 128, 128, 32768, 128, 0, 0, 128, 2048, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 32640, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 128, 256, 0, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 32768, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3731, 227, 4095, 0, 0, 0, 0, 0, 4087, 4096, 1177, 1, 0, 1614, 0, 1564, 4096, 0, 4054, 0, 21, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4051, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4040, 0, 4096, 0, 0, 0, 4096, 11, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3316, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 966, 0, 0, 2690, 0, 1256, 4096, 0, 4060, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 1, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3884, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3545, 0, 0, 319, 0, 717, 4096, 0, 3827, 0, 2, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 1, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4045, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2638, 0, 0, 1082, 0, 537, 4096, 0, 3986, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1595, 2, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2743, 0, 0, 341, 0, 3510, 4096, 0, 4093, 0, 43, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 28, 0, 0, 0, 0, 4096, 4039, 0, 4096, 0, 0, 0, 4096, 6, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1, 1, 1, 0, 0, 1, 3962, 34, 4095, 1, 0, 0, 0, 1, 4095, 4094, 1708, 0, 1, 2265, 1, 358, 4095, 2, 3955, 0, 0, 0, 0, 1, 4095, 1, 0, 1, 0, 1, 1, 0, 0, 4095, 4093, 0, 0, 0, 1, 0, 0, 7, 4094, 0, 0, 0, 1, 0, 4095, 4095, 0, 4095, 0, 0, 0, 4094, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3004, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 1471, 0, 0, 1275, 0, 2464, 4096, 0, 4078, 0, 6, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4089, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3373, 2, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2287, 0, 0, 511, 0, 2019, 4096, 0, 4096, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 4096, 1, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3731, 227, 4096, 698, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1177, 1, 0, 1614, 0, 1564, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 21, 0, 0, 0, 4095, 0, 69, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3378, 717, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 3398, 705, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 13, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8752, 8182, 7666, 8241, 8191, 8154, 8149, 8201] avg=8192.0000 max=8752 max_rank=0 min=7666 min_rank=2 imbalance_ratio=1.068359 max_load_reduction=56784 max_load_reduction_pct=0.866455 moved_to_echo=32695 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3316, 0, 4096, 698, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 966, 0, 0, 2690, 0, 1256, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 4095, 0, 69, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3378, 717, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3396, 705, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8194, 8222, 8226, 8191, 8197, 8205, 8191] avg=8192.0000 max=8226 max_rank=3 min=8110 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 0, 0, 1, 3962, 34, 4095, 698, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 1, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1708, 0, 1, 2265, 1, 358, 3309, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 1, 4095, 1, 69, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4095, 3377, 717, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 7, 3394, 705, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 13, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8793, 8194, 7644, 8123, 8192, 8200, 8203, 8187] avg=8192.0000 max=8793 max_rank=0 min=7644 min_rank=2 imbalance_ratio=1.073364 max_load_reduction=56743 max_load_reduction_pct=0.865829 moved_to_echo=32782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1595, 2, 4096, 701, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2743, 0, 0, 341, 0, 3510, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 43, 0, 0, 0, 4081, 0, 69, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3378, 717, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3395, 705, 0, 0, 0, 0, 0, 0, 4095, 28, 0, 0, 0, 0, 13, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6394, 8192, 9904, 8288, 8191, 8196, 8175, 8196] avg=8192.0000 max=9904 max_rank=2 min=6394 min_rank=0 imbalance_ratio=1.208984 max_load_reduction=55632 max_load_reduction_pct=0.848877 moved_to_echo=32758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3004, 0, 4096, 698, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1471, 0, 0, 1275, 0, 2464, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 6, 0, 0, 0, 4093, 0, 69, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3378, 717, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3395, 705, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 13, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7798, 8195, 8520, 8248, 8191, 8196, 8198, 8190] avg=8192.0000 max=8520 max_rank=2 min=7798 min_rank=0 imbalance_ratio=1.040039 max_load_reduction=57016 max_load_reduction_pct=0.869995 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3884, 0, 4096, 698, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 0, 0, 319, 0, 717, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3827, 0, 2, 0, 0, 0, 4091, 0, 69, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3378, 717, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 3398, 705, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8678, 8192, 7891, 7991, 8191, 8197, 8205, 8191] avg=8192.0000 max=8678 max_rank=0 min=7891 min_rank=2 imbalance_ratio=1.059326 max_load_reduction=56858 max_load_reduction_pct=0.867584 moved_to_echo=32784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3373, 2, 4096, 698, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2287, 0, 0, 511, 0, 2019, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4093, 0, 69, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3378, 717, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3395, 705, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 13, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8195, 8127, 8263, 8191, 8196, 8204, 8191] avg=8192.0000 max=8263 max_rank=3 min=8127 min_rank=2 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4045, 0, 4096, 701, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2638, 0, 0, 1082, 0, 537, 3305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 4096, 0, 71, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3382, 720, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3395, 709, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8842, 8192, 7562, 8156, 8198, 8200, 8197, 8189] avg=8192.0000 max=8842 max_rank=0 min=7562 min_rank=2 imbalance_ratio=1.079346 max_load_reduction=56694 max_load_reduction_pct=0.865082 moved_to_echo=32784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[16640, 128, 128, 10112, 128, 12544, 26496, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 32768, 27136, 5760, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32256, 0, 128, 0, 0, 128, 32768, 128, 640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 0, 128, 27264, 5760, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 128, 27008, 384, 32768, 5632, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1712, 0, 0, 0, 4096, 0, 4096, 4057, 0, 0, 0, 5, 4071, 0, 0, 4096, 0, 2615, 0, 0, 4096, 0, 0, 0, 0, 4055, 35, 4096, 3896, 0, 4096, 0, 0, 4094, 13, 0, 0, 0, 0, 0, 23, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3970, 0, 0, 0, 4096, 0, 4094, 4092, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1436, 0, 0, 4096, 0, 0, 0, 0, 4096, 10, 4096, 2779, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1, 1, 0, 4081, 0, 0, 0, 4095, 1, 4095, 4095, 1, 0, 1, 0, 4095, 0, 0, 4095, 0, 71, 0, 1, 4095, 0, 1, 1, 0, 4093, 0, 4095, 4012, 0, 4095, 0, 1, 4095, 0, 1, 0, 0, 0, 0, 2, 4095, 1, 1, 4095, 1, 0, 0, 0, 0, 4095, 0, 1, 4095, 0, 1, 0, 0, 1, 25, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3911, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 224, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 4055, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3388, 0, 0, 0, 4096, 0, 4088, 4096, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 735, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4076, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3156, 0, 0, 0, 4096, 0, 4066, 3884, 0, 0, 0, 0, 4023, 0, 0, 4096, 0, 971, 0, 0, 4096, 0, 0, 0, 0, 4070, 28, 4096, 3976, 0, 4096, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 26, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 369, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4089, 0, 0, 0, 4096, 0, 4094, 4095, 0, 2, 0, 0, 4093, 0, 0, 4096, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 4096, 7, 4096, 4092, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 621, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3617, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 3954, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1712, 0, 0, 0, 4096, 971, 6, 2, 0, 0, 0, 0, 0, 0, 45, 4057, 0, 0, 0, 5, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2615, 0, 0, 2887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 35, 240, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4051, 49, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3807, 238, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6787, 8178, 9598, 8226, 8201, 8209, 8196, 8141] avg=8192.0000 max=9598 max_rank=2 min=6787 min_rank=0 imbalance_ratio=1.171631 max_load_reduction=55938 max_load_reduction_pct=0.853546 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3970, 0, 0, 0, 4096, 971, 6, 2, 0, 0, 0, 0, 0, 0, 45, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1436, 0, 0, 2887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 240, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4049, 49, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 3807, 238, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9045, 8233, 8419, 7125, 8190, 8186, 8194, 8144] avg=8192.0000 max=9045 max_rank=0 min=7125 min_rank=3 imbalance_ratio=1.104126 max_load_reduction=56491 max_load_reduction_pct=0.861984 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 4081, 0, 0, 0, 4095, 970, 6, 2, 0, 0, 0, 0, 0, 1, 45, 4095, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 71, 0, 1, 2888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4093, 0, 241, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 1, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 1, 4050, 48, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 1, 25, 1, 3806, 237, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9156, 8238, 7055, 8348, 8190, 8188, 8195, 8166] avg=8192.0000 max=9156 max_rank=0 min=7055 min_rank=2 imbalance_ratio=1.117676 max_load_reduction=56380 max_load_reduction_pct=0.860291 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4089, 0, 0, 0, 4096, 971, 6, 2, 0, 0, 0, 0, 0, 0, 45, 4095, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 2887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 240, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4049, 49, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3807, 238, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9164, 8235, 6987, 8435, 8190, 8188, 8196, 8141] avg=8192.0000 max=9164 max_rank=0 min=6987 min_rank=2 imbalance_ratio=1.118652 max_load_reduction=56372 max_load_reduction_pct=0.860168 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3911, 0, 0, 0, 4096, 971, 6, 2, 0, 0, 0, 0, 0, 0, 45, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 224, 0, 0, 2887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 240, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4051, 49, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 3807, 238, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8986, 8237, 7207, 8388, 8190, 8186, 8196, 8146] avg=8192.0000 max=8986 max_rank=0 min=7207 min_rank=2 imbalance_ratio=1.096924 max_load_reduction=56550 max_load_reduction_pct=0.862885 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3388, 0, 0, 0, 4096, 971, 6, 2, 0, 0, 0, 0, 0, 0, 45, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 735, 0, 0, 2887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 240, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4043, 49, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 3807, 238, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8463, 8233, 7718, 8412, 8190, 8187, 8188, 8145] avg=8192.0000 max=8463 max_rank=0 min=7718 min_rank=2 imbalance_ratio=1.033081 max_load_reduction=57073 max_load_reduction_pct=0.870865 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3156, 0, 0, 0, 4096, 971, 6, 2, 0, 0, 0, 0, 0, 0, 44, 3884, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 971, 0, 0, 2887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 28, 240, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4022, 49, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 369, 0, 3807, 238, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 7951, 7954, 8314, 8196, 8212, 8168, 8510] avg=8192.0000 max=8510 max_rank=7 min=7951 min_rank=1 imbalance_ratio=1.038818 max_load_reduction=57026 max_load_reduction_pct=0.870148 moved_to_echo=33086 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 621, 0, 0, 0, 4096, 972, 12, 3, 0, 0, 0, 0, 0, 0, 39, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3617, 0, 0, 2885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 238, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4057, 50, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3808, 239, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5704, 8231, 10598, 8288, 8189, 8180, 8203, 8143] avg=8192.0000 max=10598 max_rank=2 min=5704 min_rank=0 imbalance_ratio=1.293701 max_load_reduction=54938 max_load_reduction_pct=0.838287 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 1920, 30848, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 0, 0, 128, 512, 128, 30464, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 24960, 0, 0, 0, 32768, 7808, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 384, 32512, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 0, 9728, 0, 128, 23168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 32768, 0, 128, 32384, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 4096, 4095, 4096, 4096, 0, 10, 4076, 2, 3850, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3847, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 506, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 4096, 4072, 4096, 4096, 0, 94, 4096, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2306, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 1805, 0, 0, 4021, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 4034, 4096, 4096, 0, 26, 4096, 0, 3878, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3812, 4078, 0, 0, 0, 0, 4096, 0, 0, 0, 583, 0, 0, 4070, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 1, 4095, 4095, 4094, 4094, 0, 20, 4092, 0, 3750, 0, 0, 0, 0, 2, 0, 3, 4093, 4095, 1, 0, 1, 0, 4095, 1, 4095, 1, 1, 3166, 4090, 1, 1, 1, 0, 4095, 0, 3, 1, 1249, 0, 1, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 3070, 4096, 4096, 0, 24, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4071, 4062, 0, 0, 0, 0, 4096, 0, 0, 0, 1068, 0, 0, 4089, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4074, 4096, 4096, 0, 19, 4096, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3593, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 787, 0, 0, 4031, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 47, 4096, 0, 3884, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4019, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 367, 0, 0, 3972, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 4094, 4096, 4096, 0, 22, 4096, 0, 3829, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3758, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 585, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3953, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3795, 301, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 143, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 94, 4096, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2306, 4095, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1805, 0, 0, 4021, 0, 3263, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8314, 8434, 8192, 8192, 6933, 9089] avg=8192.0000 max=9089 max_rank=7 min=6933 min_rank=6 imbalance_ratio=1.109497 max_load_reduction=56447 max_load_reduction_pct=0.861313 moved_to_echo=32406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3953, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3795, 301, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 143, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 10, 4076, 2, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 4096, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 0, 4096, 0, 3263, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8332, 8098, 8192, 8192, 8475, 7865] avg=8192.0000 max=8475 max_rank=6 min=7865 min_rank=7 imbalance_ratio=1.034546 max_load_reduction=57061 max_load_reduction_pct=0.870682 moved_to_echo=32724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 1, 10, 0, 0, 4094, 3952, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3793, 301, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 143, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 20, 4092, 0, 3750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4095, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3166, 4090, 1, 1, 1, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1249, 0, 1, 4095, 1, 3262, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8189, 8334, 8022, 8193, 8194, 7792, 8612] avg=8192.0000 max=8612 max_rank=7 min=7792 min_rank=6 imbalance_ratio=1.051270 max_load_reduction=56924 max_load_reduction_pct=0.868591 moved_to_echo=32791 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3953, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3795, 301, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 143, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 19, 4096, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3593, 4096, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 0, 0, 4031, 0, 3263, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8313, 8155, 8192, 8192, 8221, 8081] avg=8192.0000 max=8313 max_rank=2 min=8081 min_rank=7 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32686 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3953, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3795, 301, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 143, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 26, 4096, 0, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3812, 4078, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 0, 0, 4070, 0, 3263, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8272, 8160, 8192, 8192, 8422, 7916] avg=8192.0000 max=8422 max_rank=6 min=7916 min_rank=7 imbalance_ratio=1.028076 max_load_reduction=57114 max_load_reduction_pct=0.871490 moved_to_echo=32722 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3953, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3795, 301, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 143, 3070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 24, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4062, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 0, 0, 4089, 0, 3263, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 7309, 8376, 8192, 8192, 8665, 8420] avg=8192.0000 max=8665 max_rank=6 min=7309 min_rank=2 imbalance_ratio=1.057739 max_load_reduction=56871 max_load_reduction_pct=0.867783 moved_to_echo=33469 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3953, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3795, 301, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 143, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 47, 4096, 0, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 4095, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, 3972, 0, 3263, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8335, 8187, 8192, 8192, 8646, 7602] avg=8192.0000 max=8646 max_rank=6 min=7602 min_rank=7 imbalance_ratio=1.055420 max_load_reduction=56890 max_load_reduction_pct=0.868073 moved_to_echo=32632 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3958, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3797, 302, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 138, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 22, 4096, 0, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3758, 4096, 0, 0, 0, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 0, 0, 4096, 0, 3266, 2, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8195, 8327, 8104, 8191, 8190, 8382, 7951] avg=8192.0000 max=8382 max_rank=6 min=7951 min_rank=7 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32714 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 0, 32768, 31744, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 7040, 0, 128, 32512, 128, 26112, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 28672, 32768, 128, 128, 128, 0, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 32768, 30464, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 0, 0, 128, 1152, 31744, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 1280, 0, 384, 32768, 128, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True + [2026-08-30 05:00:51] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 676.1 | throughput per GPU (TFLOP/s/GPU): 493.1 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.037745E+01 | loss scale: 1.0 | grad norm: 16.731 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[8, 49, 934, 1, 293, 4, 1149, 7, 2802, 23, 108, 2111, 1298, 646, 0, 3361, 1208, 1354, 734, 802, 2041, 1298, 311, 1640, 11, 2643, 164, 1023, 450, 17, 179, 1848, 4020, 11, 69, 1390, 2418, 1, 1003, 1894, 3870, 584, 729, 517, 1, 1234, 1203, 2430, 186, 3469, 420, 6, 5, 4, 1006, 625, 0, 3017, 2983, 761, 556, 1337, 1238, 32] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[16, 12, 723, 0, 179, 0, 1049, 1, 2939, 48, 48, 2592, 1614, 289, 6, 3161, 974, 810, 420, 332, 2939, 1723, 162, 1427, 1, 2686, 20, 834, 359, 10, 229, 1795, 4095, 37, 67, 1550, 3092, 5, 468, 1952, 3695, 331, 524, 519, 6, 956, 1167, 2862, 87, 3830, 647, 10, 0, 7, 908, 565, 0, 3593, 3173, 1179, 279, 1721, 811, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[21, 10, 504, 2, 160, 0, 1395, 0, 2935, 79, 140, 2729, 896, 423, 1, 2836, 1107, 1042, 364, 209, 2633, 1324, 169, 1226, 1, 2681, 56, 1398, 755, 16, 203, 1689, 4084, 27, 57, 2566, 2650, 0, 468, 2384, 3886, 276, 657, 496, 4, 1093, 1142, 2700, 182, 3795, 671, 0, 4, 2, 658, 505, 0, 3393, 3117, 745, 444, 1211, 1299, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[12, 42, 1086, 0, 185, 0, 1563, 4, 2659, 111, 135, 2653, 1223, 637, 2, 2903, 693, 1601, 466, 407, 2388, 1076, 240, 1442, 6, 2335, 25, 1554, 656, 12, 151, 1566, 4080, 5, 91, 2044, 2622, 3, 469, 2831, 3416, 433, 733, 872, 0, 702, 1097, 1966, 159, 3835, 576, 0, 1, 7, 809, 854, 2, 3363, 2767, 1122, 490, 1332, 1019, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[24, 15, 989, 1, 196, 2, 1144, 0, 2925, 33, 84, 2013, 1094, 400, 3, 3260, 1234, 931, 665, 271, 2523, 1019, 220, 1606, 0, 2415, 9, 1304, 821, 22, 175, 1662, 3983, 10, 68, 1623, 3230, 0, 553, 2537, 3811, 413, 557, 592, 7, 1063, 1182, 2627, 158, 3751, 466, 1, 1, 12, 1159, 479, 0, 3363, 3092, 881, 554, 1143, 1153, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[13, 11, 568, 0, 144, 0, 1501, 0, 3133, 75, 93, 2267, 1424, 758, 0, 3411, 766, 870, 329, 495, 2653, 1637, 212, 1596, 0, 2615, 17, 1052, 534, 2, 283, 1639, 4081, 13, 57, 2207, 2726, 0, 538, 2159, 3532, 215, 766, 494, 0, 1096, 937, 1496, 156, 3868, 931, 0, 1, 5, 1015, 555, 0, 3381, 3392, 916, 558, 1207, 1133, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 10, 724, 5, 156, 0, 1102, 1, 2825, 133, 71, 2568, 879, 727, 0, 3661, 1214, 1647, 465, 343, 2581, 1433, 160, 1207, 0, 2668, 16, 1051, 475, 6, 136, 1847, 4073, 26, 39, 1720, 2866, 9, 368, 2302, 3365, 344, 400, 427, 0, 1286, 1294, 1656, 102, 3848, 494, 7, 5, 7, 1480, 628, 0, 3478, 3161, 1239, 525, 1192, 1069, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3, 2, 728, 1, 231, 0, 1501, 0, 2525, 71, 48, 2335, 730, 1072, 0, 3584, 1262, 957, 231, 197, 2562, 1706, 194, 1864, 0, 2950, 9, 1301, 410, 0, 160, 1587, 4096, 23, 75, 1416, 2802, 7, 518, 2427, 3600, 243, 714, 401, 0, 1129, 977, 2735, 137, 3880, 761, 0, 0, 1, 909, 412, 0, 3584, 3139, 905, 144, 1465, 803, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[8, 49, 934, 1, 293, 4, 1149, 7, 3396, 2320, 167, 0, 0, 0, 0, 0, 2802, 23, 108, 2111, 1298, 646, 0, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1208, 1354, 734, 802, 1632, 1298, 311, 1640, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2643, 164, 1023, 450, 17, 179, 1848, 1448, 409, 0, 0, 0, 0, 0, 0, 624, 11, 69, 1390, 2418, 1, 1003, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 2289, 584, 729, 517, 1, 1234, 1203, 2430, 0, 0, 0, 0, 0, 0, 0, 0, 186, 3469, 420, 6, 5, 4, 1006, 625, 1717, 133, 0, 0, 0, 0, 0, 0, 0, 1133, 2983, 761, 556, 1337, 1238, 32, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8328, 8029, 8979, 8192, 7410, 8987, 7571, 8040] avg=8192.0000 max=8987 max_rank=5 min=7410 min_rank=4 imbalance_ratio=1.097046 max_load_reduction=56549 max_load_reduction_pct=0.862869 moved_to_echo=32008 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 12, 723, 0, 179, 0, 1049, 1, 3459, 2182, 199, 0, 0, 0, 0, 0, 2939, 48, 48, 2592, 1614, 289, 6, 979, 0, 0, 0, 0, 0, 0, 0, 0, 974, 810, 420, 332, 2350, 1723, 162, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2686, 20, 834, 359, 10, 229, 1795, 1383, 589, 0, 0, 0, 0, 0, 0, 636, 37, 67, 1550, 3092, 5, 468, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 2185, 331, 524, 519, 6, 956, 1167, 2862, 0, 0, 0, 0, 0, 0, 0, 0, 87, 3830, 647, 10, 0, 7, 908, 565, 2045, 127, 0, 0, 0, 0, 0, 0, 0, 1349, 3173, 1179, 279, 1721, 811, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7820, 8515, 8198, 7906, 7807, 8550, 8226, 8514] avg=8192.0000 max=8550 max_rank=5 min=7807 min_rank=4 imbalance_ratio=1.043701 max_load_reduction=56986 max_load_reduction_pct=0.869537 moved_to_echo=33097 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[12, 42, 1086, 0, 185, 0, 1563, 4, 3447, 2004, 187, 0, 0, 0, 0, 0, 2659, 111, 135, 2653, 1223, 637, 2, 899, 0, 0, 0, 0, 0, 0, 0, 0, 693, 1601, 466, 407, 1909, 1076, 240, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2335, 25, 1554, 656, 12, 151, 1566, 1278, 479, 0, 0, 0, 0, 0, 0, 633, 5, 91, 2044, 2622, 3, 469, 2831, 0, 0, 0, 0, 0, 0, 0, 0, 2021, 433, 733, 872, 0, 702, 1097, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 159, 3835, 576, 0, 1, 7, 809, 854, 1914, 117, 0, 0, 0, 0, 0, 0, 2, 1262, 2767, 1122, 490, 1332, 1019, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8530, 8319, 7834, 8062, 8698, 7824, 8272, 7997] avg=8192.0000 max=8698 max_rank=4 min=7824 min_rank=5 imbalance_ratio=1.061768 max_load_reduction=56838 max_load_reduction_pct=0.867279 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[13, 11, 568, 0, 144, 0, 1501, 0, 3447, 2355, 188, 0, 0, 0, 0, 0, 3133, 75, 93, 2267, 1424, 758, 0, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 766, 870, 329, 495, 2121, 1637, 212, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2615, 17, 1052, 534, 2, 283, 1639, 1322, 532, 0, 0, 0, 0, 0, 0, 634, 13, 57, 2207, 2726, 0, 538, 2159, 0, 0, 0, 0, 0, 0, 0, 0, 2089, 215, 766, 494, 0, 1096, 937, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 156, 3868, 931, 0, 1, 5, 1015, 555, 1925, 121, 0, 0, 0, 0, 0, 0, 0, 1268, 3392, 916, 558, 1207, 1133, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8806, 8026, 7996, 8334, 7093, 8577, 8477] avg=8192.0000 max=8806 max_rank=1 min=7093 min_rank=5 imbalance_ratio=1.074951 max_load_reduction=56730 max_load_reduction_pct=0.865631 moved_to_echo=32481 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[24, 15, 989, 1, 196, 2, 1144, 0, 3365, 2250, 187, 0, 0, 0, 0, 0, 2925, 33, 84, 2013, 1094, 400, 3, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 931, 665, 271, 2017, 1019, 220, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2415, 9, 1304, 821, 22, 175, 1662, 1426, 506, 0, 0, 0, 0, 0, 0, 618, 10, 68, 1623, 3230, 0, 553, 2537, 0, 0, 0, 0, 0, 0, 0, 0, 2254, 413, 557, 592, 7, 1063, 1182, 2627, 0, 0, 0, 0, 0, 0, 0, 0, 158, 3751, 466, 1, 1, 12, 1159, 479, 1914, 131, 0, 0, 0, 0, 0, 0, 0, 1262, 3092, 881, 554, 1143, 1153, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 7562, 7963, 8340, 8639, 8695, 8072, 8092] avg=8192.0000 max=8695 max_rank=5 min=7562 min_rank=1 imbalance_ratio=1.061401 max_load_reduction=56841 max_load_reduction_pct=0.867325 moved_to_echo=33498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[21, 10, 504, 2, 160, 0, 1395, 0, 3450, 1958, 188, 0, 0, 0, 0, 0, 2935, 79, 140, 2729, 896, 423, 1, 878, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 1042, 364, 209, 2105, 1324, 169, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2681, 56, 1398, 755, 16, 203, 1689, 1454, 528, 0, 0, 0, 0, 0, 0, 634, 27, 57, 2566, 2650, 0, 468, 2384, 0, 0, 0, 0, 0, 0, 0, 0, 2299, 276, 657, 496, 4, 1093, 1142, 2700, 0, 0, 0, 0, 0, 0, 0, 0, 182, 3795, 671, 0, 4, 2, 658, 505, 1931, 133, 0, 0, 0, 0, 0, 0, 0, 1274, 3117, 745, 444, 1211, 1299, 16, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7688, 8081, 7546, 8781, 8786, 8667, 7881, 8106] avg=8192.0000 max=8786 max_rank=4 min=7546 min_rank=2 imbalance_ratio=1.072510 max_load_reduction=56750 max_load_reduction_pct=0.865936 moved_to_echo=33440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 724, 5, 156, 0, 1102, 1, 3445, 2531, 197, 0, 0, 0, 0, 0, 2825, 133, 71, 2568, 879, 727, 0, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 1647, 465, 343, 2061, 1433, 160, 1207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2668, 16, 1051, 475, 6, 136, 1847, 1263, 520, 0, 0, 0, 0, 0, 0, 628, 26, 39, 1720, 2866, 9, 368, 2302, 0, 0, 0, 0, 0, 0, 0, 0, 1984, 344, 400, 427, 0, 1286, 1294, 1656, 0, 0, 0, 0, 0, 0, 0, 0, 102, 3848, 494, 7, 5, 7, 1480, 628, 1985, 118, 0, 0, 0, 0, 0, 0, 0, 1296, 3161, 1239, 525, 1192, 1069, 15, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8333, 8530, 7982, 7958, 7391, 8674, 8497] avg=8192.0000 max=8674 max_rank=6 min=7391 min_rank=5 imbalance_ratio=1.058838 max_load_reduction=56862 max_load_reduction_pct=0.867645 moved_to_echo=32520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 2, 728, 1, 231, 0, 1501, 0, 3460, 2474, 199, 0, 0, 0, 0, 0, 2525, 71, 48, 2335, 730, 1072, 0, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 957, 231, 197, 2049, 1706, 194, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2950, 9, 1301, 410, 0, 160, 1587, 1347, 513, 0, 0, 0, 0, 0, 0, 636, 23, 75, 1416, 2802, 7, 518, 2427, 0, 0, 0, 0, 0, 0, 0, 0, 2130, 243, 714, 401, 0, 1129, 977, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 137, 3880, 761, 0, 0, 1, 909, 412, 2040, 123, 0, 0, 0, 0, 0, 0, 0, 1345, 3139, 905, 144, 1465, 803, 12, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8599, 7891, 8460, 8277, 7904, 8329, 8263, 7813] avg=8192.0000 max=8599 max_rank=0 min=7813 min_rank=7 imbalance_ratio=1.049683 max_load_reduction=56937 max_load_reduction_pct=0.868790 moved_to_echo=32309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 21120, 384, 9600, 4480, 128, 1536, 13696, 11008, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[17280, 2944, 5120, 4352, 128, 8576, 9088, 18560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[22784, 640, 768, 19328, 9216, 4992, 128, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8576, 9216, 3712, 3072, 16256, 11264, 1792, 12032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 256, 6272, 128, 1664, 128, 10496, 128, 27520, 18176, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 10240, 24832, 7808, 3584, 10624, 8576, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1280, 30336, 4992, 128, 128, 128, 8064, 4736, 15488, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[5120, 256, 640, 14592, 22528, 128, 4480, 18560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 2807, 0, 4092, 0, 6, 3415, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 14, 0, 4096, 0, 4096, 62, 0, 0, 4096, 1889, 5, 0, 0, 1, 4096, 4096, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 4096, 0, 2917, 0, 4084, 0, 0, 1285, 7, 15, 0, 0, 4095, 0, 0, 0, 0, 7, 0, 4096, 0, 4096, 48, 0, 2, 4091, 3949, 0, 0, 7, 0, 4087, 4096, 2, 0, 0, 4096, 4078, 0, 0, 4, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4081, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 4096, 0, 3664, 2, 4073, 0, 0, 1854, 0, 0, 0, 0, 4096, 4, 0, 0, 8, 25, 0, 4094, 0, 4094, 174, 0, 0, 4094, 2446, 25, 0, 1, 33, 4096, 4092, 0, 0, 0, 4090, 4094, 0, 0, 0, 0, 4096, 4094, 7, 0, 0, 3, 0, 1, 0, 0, 4087, 0, 4092, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4095, 0, 1832, 0, 4095, 0, 0, 3743, 0, 1, 0, 0, 4076, 3, 19, 0, 0, 6, 16, 4094, 2, 4095, 10, 0, 1, 4074, 2616, 1, 1, 0, 7, 4089, 4095, 4, 0, 0, 4094, 4094, 1, 0, 4, 0, 4096, 4078, 0, 0, 1, 0, 1, 0, 1, 0, 4095, 0, 4094, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 6, 1, 0, 4094, 0, 941, 12, 4078, 0, 1, 3912, 2, 16, 0, 0, 4092, 0, 0, 0, 0, 0, 1, 4082, 6, 4092, 3, 0, 1, 4095, 3297, 0, 3, 0, 44, 4094, 4094, 6, 0, 0, 4089, 4081, 2, 1, 0, 0, 4096, 4095, 0, 1, 0, 1, 0, 3, 0, 0, 4096, 2, 4092, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 3, 1, 6, 4096, 0, 2099, 0, 4091, 5, 7, 2464, 0, 20, 2, 0, 4090, 0, 7, 2, 6, 27, 0, 4090, 0, 4092, 264, 0, 9, 4096, 3351, 0, 0, 0, 1, 4074, 4095, 0, 2, 0, 4093, 4080, 0, 0, 0, 1, 4085, 4092, 1, 0, 0, 1, 1, 0, 6, 0, 4076, 1, 4095, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[16, 0, 1, 0, 0, 0, 0, 4092, 0, 1797, 16, 4063, 0, 0, 2988, 0, 0, 2, 0, 4086, 2, 2, 4, 1, 34, 0, 4095, 1, 4093, 113, 15, 1, 4090, 3309, 5, 0, 0, 4, 4087, 4093, 1, 1, 0, 4080, 4090, 3, 2, 0, 0, 4068, 4071, 10, 0, 3, 4, 3, 1, 0, 0, 4091, 2, 4093, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 2870, 0, 4093, 1, 18, 3758, 0, 4, 0, 0, 4094, 0, 0, 0, 1, 4, 0, 4096, 0, 4096, 10, 0, 0, 4095, 1539, 5, 0, 0, 0, 4089, 4095, 0, 0, 0, 4095, 4095, 0, 0, 1, 0, 4096, 4092, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 6, 1, 0, 4094, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 941, 12, 2886, 0, 1, 3912, 2, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4092, 0, 0, 0, 0, 2808, 1192, 82, 4, 0, 0, 0, 0, 0, 1, 4082, 6, 3992, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1287, 3297, 0, 3, 0, 44, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4089, 4081, 2, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 1, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 2, 4092, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7754, 8194, 8085, 8725, 8186, 8200, 8200] avg=8192.0000 max=8725 max_rank=4 min=7754 min_rank=1 imbalance_ratio=1.065063 max_load_reduction=56811 max_load_reduction_pct=0.866867 moved_to_echo=33311 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 2917, 0, 2890, 0, 0, 1285, 7, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4095, 0, 0, 0, 0, 2805, 1194, 82, 3, 0, 0, 0, 0, 7, 0, 4096, 0, 3996, 48, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1286, 3949, 0, 0, 7, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4078, 0, 0, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 7099, 8194, 8149, 9329, 8187, 8201, 8181] avg=8192.0000 max=9329 max_rank=4 min=7099 min_rank=1 imbalance_ratio=1.138794 max_load_reduction=56207 max_load_reduction_pct=0.857651 moved_to_echo=33898 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1832, 0, 2898, 0, 0, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4076, 3, 19, 0, 0, 2793, 1197, 82, 4, 0, 0, 0, 0, 6, 16, 4094, 2, 3995, 10, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1281, 2616, 1, 1, 0, 7, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4094, 4094, 1, 0, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 0, 1, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4094, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8473, 8175, 8124, 7995, 8204, 8183, 8195] avg=8192.0000 max=8473 max_rank=1 min=7995 min_rank=4 imbalance_ratio=1.034302 max_load_reduction=57063 max_load_reduction_pct=0.870712 moved_to_echo=32577 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 3664, 2, 2882, 0, 0, 1854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 8, 2807, 1191, 82, 3, 0, 0, 0, 0, 25, 0, 4094, 0, 3994, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1287, 2446, 25, 0, 1, 33, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4094, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 7, 0, 0, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4087, 0, 4092, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8402, 8191, 8287, 7888, 8191, 8207, 8184] avg=8192.0000 max=8402 max_rank=1 min=7888 min_rank=4 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32470 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 0, 2896, 1, 18, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4094, 0, 0, 0, 1, 2808, 1197, 82, 4, 0, 0, 0, 0, 4, 0, 4096, 0, 3996, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1287, 1539, 5, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 9543, 8190, 8106, 6920, 8198, 8195, 8197] avg=8192.0000 max=9543 max_rank=1 min=6920 min_rank=4 imbalance_ratio=1.164917 max_load_reduction=55993 max_load_reduction_pct=0.854385 moved_to_echo=31510 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 1, 0, 0, 0, 0, 4092, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 16, 2870, 0, 0, 2988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4086, 2, 2, 4, 1, 2809, 1193, 87, 3, 0, 0, 0, 0, 34, 0, 4095, 1, 3976, 113, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1281, 3309, 5, 0, 0, 4, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4080, 4090, 3, 2, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4071, 10, 0, 3, 4, 3, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 2, 4093, 2, 1, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 7671, 8189, 8235, 8686, 8188, 8168, 8200] avg=8192.0000 max=8686 max_rank=4 min=7671 min_rank=1 imbalance_ratio=1.060303 max_load_reduction=56850 max_load_reduction_pct=0.867462 moved_to_echo=33242 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 3, 1, 6, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2099, 0, 2895, 5, 7, 2464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 0, 4090, 0, 7, 2, 6, 2808, 1196, 82, 3, 0, 0, 0, 0, 27, 0, 4090, 0, 3992, 264, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1288, 3351, 0, 0, 0, 1, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 4080, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 4092, 1, 0, 0, 1, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4076, 1, 4095, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 7470, 8216, 8382, 8714, 8182, 8188, 8183] avg=8192.0000 max=8714 max_rank=4 min=7470 min_rank=1 imbalance_ratio=1.063721 max_load_reduction=56822 max_load_reduction_pct=0.867035 moved_to_echo=33267 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2807, 0, 2896, 0, 6, 3415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 2808, 1196, 82, 4, 0, 0, 0, 0, 14, 0, 4096, 0, 3996, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1288, 1889, 5, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 9124, 8187, 8168, 7279, 8200, 8194, 8196] avg=8192.0000 max=9124 max_rank=1 min=7279 min_rank=4 imbalance_ratio=1.113770 max_load_reduction=56412 max_load_reduction_pct=0.860779 moved_to_echo=31869 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 18944, 128, 23168, 128, 128, 23424, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 32000, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[10368, 22400, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 128, 22528, 9600, 768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1, 1, 2727, 4095, 5, 1, 8, 4095, 196, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4094, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 3, 0, 3956, 7, 2, 4094, 0, 1, 2, 0, 4095, 4095, 1, 4069, 0, 2987, 2, 0, 4096, 1, 1, 4090, 0, 0, 4095, 1, 4, 0, 4096, 0, 2423, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3973, 4096, 1, 0, 0, 4093, 79, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 2, 11, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3838, 0, 3838, 0, 0, 4090, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 574, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 3419, 4093, 2, 5, 16, 4090, 1676, 4095, 0, 0, 0, 0, 7, 0, 0, 0, 0, 1, 4, 0, 4090, 2, 4093, 0, 0, 1, 0, 0, 0, 2, 33, 0, 3921, 1, 0, 4088, 0, 1, 0, 2, 4084, 4091, 0, 4045, 1, 2453, 1, 14, 4080, 0, 0, 4095, 1, 7, 4093, 3, 1, 5, 4095, 1, 822, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3597, 4095, 0, 0, 1, 4095, 186, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 18, 33, 4096, 0, 0, 0, 6, 4093, 4096, 0, 4091, 0, 2670, 0, 0, 4084, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 0, 1750, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4069, 4096, 0, 0, 1, 4086, 97, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4086, 0, 5, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4094, 0, 3392, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 645, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 5, 4010, 4091, 0, 0, 17, 4022, 854, 4095, 0, 0, 0, 0, 0, 7, 0, 6, 0, 0, 0, 0, 4094, 0, 4090, 0, 4, 0, 0, 1, 0, 0, 53, 0, 4042, 3, 0, 4095, 4, 0, 0, 0, 4091, 4094, 0, 3543, 0, 2424, 0, 0, 4093, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 4091, 1, 1513, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3636, 4091, 0, 0, 28, 4095, 1000, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 1, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3708, 6, 17, 4096, 0, 0, 0, 1, 4095, 4089, 0, 4094, 0, 3288, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 610, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 3924, 4086, 1, 1, 26, 4072, 323, 4094, 3, 4, 0, 5, 0, 1, 0, 1, 0, 3, 0, 1, 4092, 0, 4094, 6, 0, 0, 0, 0, 1, 0, 151, 2, 4027, 17, 0, 4061, 1, 3, 0, 6, 4081, 4090, 0, 4089, 2, 1285, 0, 4, 4084, 0, 0, 4092, 2, 0, 4094, 1, 6, 0, 4092, 3, 2601, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3419, 427, 2, 5, 16, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1676, 4095, 0, 0, 0, 0, 7, 0, 2243, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4090, 2, 3664, 428, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 2, 4091, 2, 0, 0, 0, 0, 0, 0, 33, 0, 3921, 1, 0, 4088, 0, 1, 71, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1413, 0, 0, 4045, 1, 2453, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 4080, 0, 0, 4095, 1, 7, 3, 0, 0, 0, 0, 0, 0, 0, 2726, 3, 1, 5, 4095, 1, 822, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7960, 9314, 8189, 8189, 8115, 7914, 8201, 7654] avg=8192.0000 max=9314 max_rank=1 min=7654 min_rank=7 imbalance_ratio=1.136963 max_load_reduction=56222 max_load_reduction_pct=0.857880 moved_to_echo=31884 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3973, 427, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 79, 4096, 0, 0, 0, 0, 0, 0, 2249, 1294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3667, 430, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 2, 11, 4096, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 0, 3838, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2728, 0, 0, 0, 4096, 0, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8494, 7718, 8193, 8194, 8257, 9093, 8189, 7398] avg=8192.0000 max=9093 max_rank=5 min=7398 min_rank=7 imbalance_ratio=1.109985 max_load_reduction=56443 max_load_reduction_pct=0.861252 moved_to_echo=32937 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4010, 428, 0, 0, 17, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 854, 4095, 0, 0, 0, 0, 0, 7, 2247, 1294, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4094, 0, 3662, 429, 0, 0, 0, 0, 0, 0, 4090, 0, 4, 0, 0, 1, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 53, 0, 4042, 3, 0, 4095, 4, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1415, 0, 0, 3543, 0, 2424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2728, 0, 0, 1, 4091, 1, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8482, 8497, 8191, 8190, 8268, 7382, 8192, 8334] avg=8192.0000 max=8497 max_rank=1 min=7382 min_rank=5 imbalance_ratio=1.037231 max_load_reduction=57039 max_load_reduction_pct=0.870346 moved_to_echo=32176 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 2727, 427, 5, 1, 8, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 196, 4095, 0, 0, 0, 0, 0, 0, 2249, 1294, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4094, 0, 3666, 430, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 3, 0, 3956, 7, 2, 4094, 0, 1, 71, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1416, 0, 1, 4069, 0, 2987, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 1, 1, 4090, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2727, 1, 4, 0, 4096, 0, 2423, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7265, 7834, 8192, 8192, 8134, 8475, 8193, 9251] avg=8192.0000 max=9251 max_rank=7 min=7265 min_rank=0 imbalance_ratio=1.129272 max_load_reduction=56285 max_load_reduction_pct=0.858841 moved_to_echo=34053 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4069, 427, 0, 0, 1, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 97, 4096, 0, 0, 0, 0, 0, 0, 2249, 1294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3667, 430, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 5, 0, 4086, 0, 5, 4096, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 0, 4094, 0, 3392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2728, 0, 0, 0, 4096, 0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8583, 7736, 8193, 8194, 8263, 8903, 8195, 7469] avg=8192.0000 max=8903 max_rank=5 min=7469 min_rank=7 imbalance_ratio=1.086792 max_load_reduction=56633 max_load_reduction_pct=0.864151 moved_to_echo=32830 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3597, 427, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 186, 4096, 0, 0, 0, 0, 0, 0, 2248, 1294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3666, 429, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 18, 33, 4096, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1416, 0, 0, 4091, 0, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 2728, 0, 0, 0, 4096, 0, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 7824, 8191, 8194, 8266, 8183, 8184, 8574] avg=8192.0000 max=8574 max_rank=7 min=7824 min_rank=1 imbalance_ratio=1.046631 max_load_reduction=56962 max_load_reduction_pct=0.869171 moved_to_echo=33207 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3636, 428, 0, 0, 28, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 4096, 0, 0, 0, 0, 0, 0, 2249, 1294, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 1, 0, 4096, 0, 3662, 430, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3708, 6, 17, 4096, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1416, 0, 0, 4094, 0, 3288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2728, 0, 0, 0, 4096, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8639, 8195, 8188, 7898, 8799, 8195, 7434] avg=8192.0000 max=8799 max_rank=5 min=7434 min_rank=7 imbalance_ratio=1.074097 max_load_reduction=56737 max_load_reduction_pct=0.865738 moved_to_echo=32326 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3924, 419, 1, 1, 26, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 323, 4094, 3, 4, 0, 5, 0, 1, 2244, 1300, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 1, 4092, 0, 3663, 432, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 0, 0, 1, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 151, 2, 4027, 17, 0, 4061, 1, 3, 73, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1405, 0, 0, 4089, 2, 1285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4084, 0, 0, 4092, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2716, 1, 6, 0, 4092, 3, 2601, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8444, 7974, 8192, 8195, 8335, 6787, 8187, 9422] avg=8192.0000 max=9422 max_rank=7 min=6787 min_rank=5 imbalance_ratio=1.150146 max_load_reduction=56114 max_load_reduction_pct=0.856232 moved_to_echo=32731 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 11392, 0, 128, 31872, 128, 22400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 128, 31872, 128, 128, 32768, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[21888, 128, 128, 128, 32768, 128, 11008, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 29440, 3456, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 128, 29440, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4480, 32768, 128, 128, 0, 128, 128, 128, 18048, 10368, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 0, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 3, 1, 0, 4059, 0, 1, 0, 4093, 3, 0, 4, 1, 0, 4094, 0, 0, 1, 3489, 0, 1, 1, 4065, 3, 4041, 3, 2, 4094, 0, 4, 0, 0, 7, 0, 4096, 0, 4095, 8, 0, 1, 0, 0, 0, 4085, 5, 0, 2478, 4094, 176, 4088, 2, 4, 0, 4094, 4093, 0, 2113, 35, 4094, 2, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2499, 18, 0, 0, 4079, 0, 4096, 0, 0, 4018, 0, 0, 0, 0, 9, 0, 4087, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3715, 4096, 55, 4096, 0, 0, 32, 4096, 4096, 0, 1967, 0, 4096, 0, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3990, 1, 0, 0, 3994, 0, 4087, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3676, 4096, 248, 4096, 0, 0, 2, 4096, 4096, 0, 347, 53, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4033, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4042, 0, 0, 0, 3036, 0, 0, 0, 3988, 0, 4016, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3979, 4096, 924, 4096, 0, 0, 0, 4096, 4096, 0, 554, 2, 4096, 0, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4066, 1, 0, 0, 3440, 14, 0, 0, 4068, 1, 4046, 0, 0, 4087, 0, 0, 2, 0, 9, 0, 4075, 0, 4094, 5, 3, 0, 0, 0, 0, 4092, 0, 0, 3624, 4096, 56, 4091, 0, 0, 13, 4095, 4096, 0, 1181, 0, 4096, 0, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4087, 0, 1, 0, 4096, 6, 0, 0, 0, 0, 4058, 1, 1, 0, 4022, 3, 1, 0, 3998, 1, 4093, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4074, 0, 4095, 1, 0, 0, 0, 0, 0, 4092, 0, 1, 3008, 4095, 100, 4095, 1, 0, 47, 4094, 4095, 0, 1150, 12, 4095, 0, 0, 2, 0, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 2093, 12, 0, 0, 4006, 0, 4083, 0, 0, 4072, 0, 0, 0, 0, 12, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3570, 4096, 161, 4096, 0, 0, 13, 4096, 4096, 0, 2527, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 4091, 0, 3, 0, 4092, 0, 0, 6, 3, 0, 3990, 6, 0, 0, 3845, 10, 3, 0, 4063, 0, 4088, 4, 1, 4090, 0, 5, 4, 4, 0, 3, 4094, 0, 4093, 1, 1, 0, 0, 9, 5, 4081, 0, 2, 4005, 4090, 1, 4089, 3, 0, 0, 4092, 4092, 0, 364, 58, 4088, 5, 0, 43, 4, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 1, 0, 4059, 0, 1, 0, 4093, 14, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 4, 1, 0, 4094, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3489, 0, 1, 1, 4065, 3, 697, 150, 0, 0, 0, 0, 0, 0, 4041, 3, 2, 4094, 0, 4, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 4095, 8, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 5, 0, 2478, 592, 0, 0, 0, 0, 0, 0, 0, 0, 176, 4088, 2, 4, 0, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2113, 35, 4094, 2, 0, 0, 0, 3, 2791, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8228, 8407, 8183, 8208, 7160, 8141, 9038] avg=8192.0000 max=9038 max_rank=7 min=7160 min_rank=5 imbalance_ratio=1.103271 max_load_reduction=56498 max_load_reduction_pct=0.862091 moved_to_echo=32547 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4066, 1, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 14, 0, 0, 4068, 1, 697, 151, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 4087, 0, 0, 2, 0, 39, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4075, 0, 4094, 5, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3624, 593, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4091, 0, 0, 13, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 0, 4096, 0, 3, 0, 0, 0, 2792, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8187, 8371, 8174, 8187, 8309, 8031, 8072] avg=8192.0000 max=8371 max_rank=2 min=8031 min_rank=6 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 1, 0, 0, 3994, 0, 697, 151, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4087, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3676, 593, 0, 0, 0, 0, 0, 0, 0, 0, 248, 4096, 0, 0, 2, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 53, 4096, 0, 0, 0, 0, 0, 2792, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8223, 8833, 8213, 8190, 8365, 8218, 7288] avg=8192.0000 max=8833 max_rank=2 min=7288 min_rank=7 imbalance_ratio=1.078247 max_load_reduction=56703 max_load_reduction_pct=0.865219 moved_to_echo=32061 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4033, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4042, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3036, 0, 0, 0, 3988, 0, 697, 151, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 4095, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3979, 593, 0, 0, 0, 0, 0, 0, 0, 0, 924, 4096, 0, 0, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 2, 4096, 0, 0, 2, 0, 0, 2792, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8172, 7872, 8150, 8193, 8668, 8892, 7446] avg=8192.0000 max=8892 max_rank=6 min=7446 min_rank=7 imbalance_ratio=1.085449 max_load_reduction=56644 max_load_reduction_pct=0.864319 moved_to_echo=33199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2499, 18, 0, 0, 4079, 0, 697, 151, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4018, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4087, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3715, 593, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4096, 0, 0, 32, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1967, 0, 4096, 0, 0, 2, 0, 0, 2792, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8225, 7444, 8153, 8193, 8404, 8055, 8857] avg=8192.0000 max=8857 max_rank=7 min=7444 min_rank=2 imbalance_ratio=1.081177 max_load_reduction=56679 max_load_reduction_pct=0.864853 moved_to_echo=33509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4044, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2093, 12, 0, 0, 4006, 0, 697, 151, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 4072, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4085, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3570, 593, 0, 0, 0, 0, 0, 0, 0, 0, 161, 4096, 0, 0, 13, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2527, 0, 4096, 0, 0, 0, 0, 0, 2792, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8173, 6959, 8194, 8194, 8259, 8142, 9415] avg=8192.0000 max=9415 max_rank=7 min=6959 min_rank=2 imbalance_ratio=1.149292 max_load_reduction=56121 max_load_reduction_pct=0.856339 moved_to_echo=34010 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4087, 0, 1, 0, 4095, 14, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4058, 1, 33, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4022, 3, 1, 0, 3998, 1, 697, 150, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 1, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4095, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 3008, 593, 0, 0, 0, 0, 0, 0, 0, 0, 100, 4095, 1, 0, 47, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 12, 4095, 0, 0, 2, 0, 13, 2791, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8194, 8873, 8229, 8171, 7694, 8114, 8063] avg=8192.0000 max=8873 max_rank=2 min=7694 min_rank=5 imbalance_ratio=1.083130 max_load_reduction=56663 max_load_reduction_pct=0.864609 moved_to_echo=32042 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 4091, 0, 3, 0, 4092, 19, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 6, 3, 0, 3990, 6, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3845, 10, 3, 0, 4063, 0, 703, 153, 0, 0, 0, 0, 0, 0, 4088, 4, 1, 4090, 0, 5, 4, 4, 44, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 4093, 1, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 9, 5, 4081, 0, 2, 4005, 575, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 3, 0, 0, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 58, 4088, 5, 0, 43, 4, 2, 2793, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8134, 8777, 8240, 8200, 8677, 7943, 7357] avg=8192.0000 max=8777 max_rank=2 min=7357 min_rank=7 imbalance_ratio=1.071411 max_load_reduction=56759 max_load_reduction_pct=0.866074 moved_to_echo=32177 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 28160, 4736, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[10240, 256, 32768, 128, 128, 128, 128, 128, 22400, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 32512, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1792, 32768, 128, 128, 128, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 26496, 128, 128, 128, 32384, 128, 5632, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 128, 128, 32640, 0, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[7, 0, 1802, 4092, 0, 0, 2, 2, 4094, 0, 4090, 0, 0, 2, 1, 2, 0, 4091, 0, 5, 4095, 0, 164, 1, 0, 4095, 0, 4, 4092, 1, 4, 4092, 4092, 4093, 0, 0, 4093, 3, 611, 4094, 33, 4, 0, 2, 4092, 2, 0, 3, 2, 1, 1, 0, 4094, 0, 0, 0, 1303, 2, 181, 4, 4083, 1, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3751, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 1, 4096, 12, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 332, 0, 2, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3825, 4095, 0, 0, 1, 1, 4095, 0, 4095, 0, 0, 1, 0, 0, 0, 4091, 1, 0, 4082, 1, 0, 0, 31, 4092, 0, 1, 4094, 0, 1, 4095, 4095, 4095, 2, 0, 4095, 1, 6, 4066, 35, 1, 0, 1, 4095, 1, 0, 0, 0, 54, 0, 1, 4095, 1, 0, 1, 111, 0, 82, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2974, 4096, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 3971, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 2, 0, 4096, 0, 8, 4095, 649, 0, 0, 0, 4096, 0, 0, 0, 0, 111, 0, 0, 4096, 0, 0, 0, 484, 0, 5, 0, 4096, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 3175, 4096, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4031, 0, 6, 0, 0, 4092, 0, 0, 4096, 0, 0, 4095, 4096, 4096, 1, 0, 4096, 0, 505, 4095, 200, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 1, 0, 221, 0, 63, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3430, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 2, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 175, 4096, 25, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 331, 0, 137, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2983, 4096, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 61, 4092, 58, 0, 0, 0, 4096, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 0, 0, 986, 0, 7, 0, 4096, 0, 17, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3834, 4092, 2, 4, 4, 4, 4093, 7, 4088, 0, 0, 4, 0, 0, 0, 4066, 3, 6, 4009, 0, 7, 2, 6, 4092, 0, 0, 4091, 4, 0, 4089, 4091, 4091, 0, 0, 4092, 0, 40, 4092, 95, 6, 0, 3, 4090, 3, 0, 2, 1, 32, 4, 0, 4089, 0, 0, 5, 111, 0, 55, 14, 4091, 3, 13, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 1802, 4092, 0, 0, 2, 2, 724, 148, 0, 0, 0, 0, 0, 0, 4094, 0, 4090, 0, 0, 2, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 5, 4095, 0, 164, 1, 22, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 1, 4, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 0, 0, 0, 0, 3, 611, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4, 0, 2, 4092, 2, 0, 3, 3926, 28, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 4094, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 1303, 2, 181, 4, 4083, 1, 0, 4, 3368, 167, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6777, 8190, 8379, 8194, 8630, 8090, 8163, 9113] avg=8192.0000 max=9113 max_rank=7 min=6777 min_rank=0 imbalance_ratio=1.112427 max_load_reduction=56423 max_load_reduction_pct=0.860947 moved_to_echo=33996 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3175, 4096, 0, 0, 0, 0, 723, 148, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4031, 0, 6, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 1, 0, 0, 0, 505, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 4096, 0, 0, 0, 3928, 27, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 1, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 221, 0, 63, 0, 4096, 0, 0, 0, 3373, 168, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8189, 8148, 8185, 8527, 8251, 8172, 7921] avg=8192.0000 max=8527 max_rank=4 min=7921 min_rank=7 imbalance_ratio=1.040894 max_load_reduction=57009 max_load_reduction_pct=0.869888 moved_to_echo=32871 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3751, 4096, 0, 0, 0, 0, 723, 148, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 4096, 0, 0, 0, 3928, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 332, 0, 2, 0, 4096, 0, 0, 0, 3373, 168, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8718, 8193, 8212, 8190, 8023, 8064, 8165, 7971] avg=8192.0000 max=8718 max_rank=0 min=7971 min_rank=7 imbalance_ratio=1.064209 max_load_reduction=56818 max_load_reduction_pct=0.866974 moved_to_echo=32223 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3430, 4096, 0, 0, 0, 0, 723, 148, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 2, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 0, 0, 0, 0, 175, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4096, 0, 0, 0, 3929, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 331, 0, 137, 0, 4096, 0, 0, 0, 3373, 167, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8397, 8193, 8213, 8190, 8197, 8077, 8165, 8104] avg=8192.0000 max=8397 max_rank=0 min=8077 min_rank=5 imbalance_ratio=1.025024 max_load_reduction=57139 max_load_reduction_pct=0.871872 moved_to_echo=32543 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3825, 4095, 0, 0, 1, 1, 724, 148, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 4082, 1, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0, 31, 4092, 0, 1, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 0, 2, 0, 0, 1, 6, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1, 0, 1, 4095, 1, 0, 0, 3928, 28, 0, 0, 0, 0, 0, 0, 0, 54, 0, 1, 4095, 1, 0, 1, 4067, 0, 0, 0, 0, 0, 0, 0, 111, 0, 82, 0, 4096, 0, 0, 0, 3370, 167, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8794, 8192, 8198, 8218, 8000, 8089, 8219, 7826] avg=8192.0000 max=8794 max_rank=0 min=7826 min_rank=7 imbalance_ratio=1.073486 max_load_reduction=56742 max_load_reduction_pct=0.865814 moved_to_echo=32134 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2974, 4096, 0, 0, 0, 0, 724, 148, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 3971, 0, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 2, 0, 0, 0, 8, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 649, 0, 0, 0, 4096, 0, 0, 0, 3929, 28, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 4096, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 484, 0, 5, 0, 4096, 0, 5, 0, 3372, 167, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7942, 8188, 8079, 8190, 8031, 8702, 8275, 8129] avg=8192.0000 max=8702 max_rank=5 min=7942 min_rank=0 imbalance_ratio=1.062256 max_load_reduction=56834 max_load_reduction_pct=0.867218 moved_to_echo=33137 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2983, 4096, 0, 0, 0, 0, 724, 148, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 4087, 0, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 0, 0, 0, 0, 61, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 4096, 0, 0, 0, 3929, 28, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 986, 0, 7, 0, 4096, 0, 17, 0, 3372, 167, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7951, 8191, 8187, 8190, 8079, 8111, 8182, 8645] avg=8192.0000 max=8645 max_rank=7 min=7951 min_rank=0 imbalance_ratio=1.055298 max_load_reduction=56891 max_load_reduction_pct=0.868088 moved_to_echo=33017 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3834, 4092, 2, 4, 4, 4, 723, 151, 0, 0, 0, 0, 0, 0, 4093, 7, 4088, 0, 0, 4, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 3, 6, 4009, 0, 7, 2, 23, 4, 0, 0, 0, 0, 0, 0, 6, 4092, 0, 0, 0, 4, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 40, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 95, 6, 0, 3, 4090, 3, 0, 2, 3925, 28, 0, 0, 0, 0, 0, 0, 1, 32, 4, 0, 4089, 0, 0, 5, 4064, 0, 0, 0, 0, 0, 0, 0, 111, 0, 55, 14, 4091, 3, 13, 6, 3368, 166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8814, 8200, 8120, 8179, 8049, 8152, 8195, 7827] avg=8192.0000 max=8814 max_rank=0 min=7827 min_rank=7 imbalance_ratio=1.075928 max_load_reduction=56722 max_load_reduction_pct=0.865509 moved_to_echo=32223 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32768, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[3968, 128, 640, 128, 32768, 128, 128, 128, 27008, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1152, 128, 0, 128, 32768, 128, 0, 128, 31488, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 128, 32512, 128, 256, 128, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 25856, 32768, 128, 128, 128, 128, 5888, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31488, 0, 128, 0, 0, 128, 1408, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 256, 128, 128, 32768, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 3, 2, 0, 3749, 4093, 0, 4090, 2, 120, 0, 0, 1, 0, 4, 1, 4090, 0, 2, 4090, 2, 4092, 4, 0, 0, 4080, 1, 4087, 3, 4092, 4094, 371, 2, 0, 0, 5, 0, 3941, 1, 4, 4093, 0, 4094, 0, 4092, 2, 1, 2, 4091, 1, 3, 7, 3, 0, 3, 3, 0, 4092, 2, 1, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 16, 16, 0, 0, 0, 2, 3380, 4093, 0, 4096, 0, 44, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4082, 0, 4096, 0, 4096, 4092, 745, 0, 0, 0, 0, 0, 4057, 0, 0, 4096, 0, 4050, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 2, 0, 0, 0, 0, 3702, 4096, 1, 4096, 0, 100, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4079, 0, 4096, 0, 4096, 4096, 540, 0, 0, 0, 0, 0, 3874, 0, 0, 4096, 0, 4069, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 3415, 4096, 0, 4096, 0, 54, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 732, 0, 0, 0, 0, 0, 3987, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4, 5, 0, 0, 0, 0, 3654, 4096, 0, 4096, 0, 362, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 4093, 0, 4096, 0, 4096, 4094, 441, 0, 0, 0, 0, 0, 3723, 0, 0, 4096, 0, 4089, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2477, 4096, 0, 4096, 0, 322, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 1717, 0, 0, 0, 0, 0, 3675, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 1, 0, 3, 2413, 4095, 0, 4095, 0, 44, 0, 0, 2, 0, 1, 0, 4095, 0, 1, 4095, 1, 4095, 0, 0, 0, 4092, 1, 4094, 1, 4095, 4095, 1797, 1, 0, 0, 2, 0, 3884, 0, 6, 4095, 1, 4085, 0, 4095, 1, 1, 0, 4095, 0, 0, 0, 1, 50, 1, 1, 1, 4095, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[5, 0, 3, 0, 0, 2, 0, 530, 4092, 0, 4088, 6, 171, 0, 0, 0, 0, 6, 1, 4084, 0, 4, 4088, 4, 4089, 9, 0, 0, 4091, 5, 4086, 8, 4091, 4092, 3564, 4, 0, 0, 11, 0, 3923, 0, 0, 4091, 1, 4093, 0, 4091, 2, 4, 0, 4092, 0, 3, 3, 4, 0, 5, 1, 0, 4088, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 16, 0, 0, 0, 2, 3380, 4096, 1173, 0, 0, 0, 0, 0, 0, 3943, 0, 4096, 0, 44, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4082, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2857, 4092, 745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 4050, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 2916, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 2692, 1239, 150, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8683, 8092, 8192, 8185, 7694, 8332, 8179, 8179] avg=8192.0000 max=8683 max_rank=0 min=7694 min_rank=4 imbalance_ratio=1.059937 max_load_reduction=56853 max_load_reduction_pct=0.867508 moved_to_echo=32384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 3, 2, 0, 3749, 4093, 1172, 0, 0, 0, 0, 0, 0, 3943, 0, 4090, 2, 120, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4090, 0, 2, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 4080, 1, 4087, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2855, 4094, 371, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 1, 4, 0, 0, 4094, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 4091, 1, 3, 7, 3, 2913, 1178, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4092, 2, 1, 16, 2690, 1237, 150, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9023, 8156, 8189, 8182, 7327, 8264, 8201, 8194] avg=8192.0000 max=9023 max_rank=0 min=7327 min_rank=4 imbalance_ratio=1.101440 max_load_reduction=56513 max_load_reduction_pct=0.862320 moved_to_echo=31986 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 3415, 4096, 1173, 0, 0, 0, 0, 0, 0, 3944, 0, 4096, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2857, 4096, 732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 0, 0, 0, 4095, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2916, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 2692, 1239, 151, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8685, 8094, 8193, 8199, 7685, 8307, 8191, 8182] avg=8192.0000 max=8685 max_rank=0 min=7685 min_rank=4 imbalance_ratio=1.060181 max_load_reduction=56851 max_load_reduction_pct=0.867477 moved_to_echo=32365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2, 0, 0, 0, 0, 3702, 4096, 1173, 0, 0, 0, 0, 0, 0, 3944, 1, 4096, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4079, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2857, 4096, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 0, 0, 0, 0, 4069, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 2916, 1179, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 15, 2692, 1239, 151, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8974, 8141, 8193, 8182, 7493, 8168, 8188, 8197] avg=8192.0000 max=8974 max_rank=0 min=7493 min_rank=4 imbalance_ratio=1.095459 max_load_reduction=56562 max_load_reduction_pct=0.863068 moved_to_echo=32046 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 1, 0, 3, 2413, 4095, 1173, 0, 0, 0, 0, 0, 0, 3943, 0, 4095, 0, 44, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4092, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2857, 4095, 1797, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 6, 0, 1, 4085, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0, 1, 2916, 1179, 0, 0, 0, 0, 0, 0, 50, 1, 1, 1, 4095, 0, 1, 0, 2692, 1238, 151, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7689, 8084, 8194, 8194, 8752, 8200, 8193, 8230] avg=8192.0000 max=8752 max_rank=4 min=7689 min_rank=0 imbalance_ratio=1.068359 max_load_reduction=56784 max_load_reduction_pct=0.866455 moved_to_echo=33375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 5, 0, 0, 0, 0, 3654, 4096, 1173, 0, 0, 0, 0, 0, 0, 3944, 0, 4096, 0, 362, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2857, 4094, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3723, 0, 0, 0, 0, 4089, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 2917, 1179, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 12, 2692, 1239, 151, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8932, 8406, 8193, 8196, 7392, 8037, 8188, 8192] avg=8192.0000 max=8932 max_rank=0 min=7392 min_rank=4 imbalance_ratio=1.090332 max_load_reduction=56604 max_load_reduction_pct=0.863708 moved_to_echo=31809 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2477, 4096, 1173, 0, 0, 0, 0, 0, 0, 3944, 0, 4096, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2857, 4096, 1717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3675, 0, 0, 0, 0, 4096, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2916, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 2692, 1239, 151, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7746, 8362, 8192, 8199, 8670, 7996, 8191, 8180] avg=8192.0000 max=8670 max_rank=4 min=7746 min_rank=0 imbalance_ratio=1.058350 max_load_reduction=56866 max_load_reduction_pct=0.867706 moved_to_echo=33037 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 3, 0, 0, 2, 0, 530, 4091, 1173, 0, 0, 0, 0, 0, 0, 3936, 0, 4088, 6, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 4084, 0, 4, 4088, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4091, 5, 4086, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2852, 4092, 3564, 4, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 0, 1, 4093, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4092, 0, 3, 3, 4, 2916, 1181, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 4088, 1, 0, 0, 2695, 1239, 153, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5804, 8201, 8190, 8199, 10523, 8232, 8205, 8182] avg=8192.0000 max=10523 max_rank=4 min=5804 min_rank=0 imbalance_ratio=1.284546 max_load_reduction=55013 max_load_reduction_pct=0.839432 moved_to_echo=35142 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[22912, 32768, 9984, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31104, 128, 128, 0, 128, 32768, 0, 1792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31616, 128, 32768, 128, 1280, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32768, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 23424, 32768, 9472, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 128, 23424, 9472, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 128, 21632, 9984, 1280, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 52, 0, 0, 0, 163, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 3933, 4072, 42, 4006, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 19, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4, 3, 0, 1, 1, 3, 4092, 0, 4, 0, 55, 6, 0, 4, 176, 4, 0, 4092, 4, 4086, 0, 0, 0, 0, 0, 4089, 0, 4092, 4, 0, 5, 4091, 0, 0, 0, 3, 2, 0, 4093, 0, 3761, 4072, 0, 4039, 4, 3, 4, 0, 0, 4092, 0, 3, 4, 0, 1, 4092, 4093, 3, 3, 4092, 4093, 182, 4081, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 33, 0, 0, 0, 585, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 4096, 0, 3430, 4062, 113, 3807, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 233, 4096, 19] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 1, 4095, 0, 1, 2, 53, 1, 0, 1, 50, 1, 0, 4095, 1, 4090, 1, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 4095, 0, 1, 13, 1, 1, 0, 4095, 0, 3689, 4042, 0, 4072, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 4095, 0, 1, 4094, 4094, 370, 4095, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 16, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 4069, 3, 4080, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 24, 4096, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 71, 0, 0, 0, 108, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 3970, 4020, 37, 4013, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 4096, 0, 0, 4092, 4087, 58, 4096, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 127, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 3981, 4079, 263, 3770, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 42, 4096, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4, 4, 7, 5, 0, 7, 4090, 8, 0, 1, 36, 2, 7, 0, 1, 5, 0, 4070, 5, 4084, 4, 0, 0, 2, 2, 4088, 0, 4090, 6, 0, 8, 4091, 0, 0, 0, 5, 5, 0, 4091, 0, 4038, 4089, 0, 4059, 5, 4, 0, 0, 0, 4090, 0, 2, 5, 0, 0, 4091, 4091, 5, 4, 4088, 4087, 55, 4075, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 3, 0, 1, 1, 3, 4092, 0, 3780, 199, 119, 0, 0, 0, 0, 0, 4, 0, 55, 6, 0, 4, 176, 4, 4093, 3893, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 4086, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4092, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 4093, 0, 3893, 198, 0, 0, 0, 0, 0, 0, 3761, 292, 0, 4039, 4, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3, 4, 0, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 3969, 182, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8235, 8187, 8190, 8189, 8103, 8192, 8238] avg=8192.0000 max=8238 max_rank=7 min=8103 min_rank=5 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32722 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3777, 199, 119, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 16, 0, 4096, 3898, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3897, 198, 0, 0, 0, 0, 0, 0, 4080, 292, 3, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 24, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8022, 8197, 8192, 8191, 8455, 8192, 8096] avg=8192.0000 max=8455 max_rank=5 min=8022 min_rank=1 imbalance_ratio=1.032104 max_load_reduction=57081 max_load_reduction_pct=0.870987 moved_to_echo=32934 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3731, 199, 119, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 108, 0, 4096, 3898, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3893, 197, 0, 0, 0, 0, 0, 0, 3970, 289, 37, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 58, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8173, 8196, 8193, 8187, 8309, 8196, 8137] avg=8192.0000 max=8309 max_rank=5 min=8137 min_rank=7 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32829 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3780, 199, 119, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 163, 0, 4096, 3898, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3897, 198, 0, 0, 0, 0, 0, 0, 3933, 292, 42, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 19, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8209, 8197, 8192, 8192, 8273, 8192, 8087] avg=8192.0000 max=8273 max_rank=5 min=8087 min_rank=7 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 1, 4095, 0, 3752, 199, 119, 0, 0, 0, 0, 0, 1, 2, 53, 1, 0, 1, 50, 1, 4095, 3897, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4090, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 1, 1, 0, 4095, 0, 3895, 198, 0, 0, 0, 0, 0, 0, 3689, 290, 0, 4072, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3970, 370, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8101, 8192, 8190, 8204, 8052, 8190, 8438] avg=8192.0000 max=8438 max_rank=7 min=8052 min_rank=5 imbalance_ratio=1.030029 max_load_reduction=57098 max_load_reduction_pct=0.871246 moved_to_echo=32884 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3786, 199, 119, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 127, 0, 4096, 3898, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3897, 197, 0, 0, 0, 0, 0, 0, 3981, 293, 263, 3770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 42, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8146, 8197, 8193, 8190, 8307, 8192, 8111] avg=8192.0000 max=8307 max_rank=5 min=8111 min_rank=7 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32800 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3770, 200, 119, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 585, 0, 4096, 3899, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 3896, 197, 0, 0, 0, 0, 0, 0, 3430, 292, 113, 3807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 233, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8613, 8197, 8192, 8196, 7642, 8191, 8320] avg=8192.0000 max=8613 max_rank=1 min=7642 min_rank=5 imbalance_ratio=1.051392 max_load_reduction=56923 max_load_reduction_pct=0.868576 moved_to_echo=32349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 4, 7, 5, 0, 7, 4090, 8, 3800, 199, 126, 0, 0, 0, 0, 0, 0, 1, 36, 2, 7, 0, 1, 5, 4091, 3894, 0, 0, 0, 0, 0, 0, 0, 4070, 5, 4084, 4, 0, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 2, 4088, 0, 4090, 6, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 4091, 0, 3889, 197, 0, 0, 0, 0, 0, 0, 4038, 289, 0, 4059, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2, 5, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 3950, 55, 4075, 20, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8037, 8173, 8194, 8187, 8395, 8191, 8109] avg=8192.0000 max=8395 max_rank=5 min=8037 min_rank=1 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=32882 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 384, 128, 128, 128, 1280, 128, 32768, 31232, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 128, 30208, 1664, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 31744, 1024, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[30976, 2432, 512, 31872, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 0, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 0, 32768, 0, 31232, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3924, 0, 0, 4096, 0, 0, 0, 47, 0, 0, 3409, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4092, 4096, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 0, 0, 0, 0, 4096, 0, 0, 3853, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 0, 2, 3965, 0, 4, 4092, 4, 4, 0, 31, 4, 0, 3379, 4091, 4092, 4, 0, 4090, 1, 0, 0, 4075, 0, 4092, 3, 0, 0, 3, 2, 4093, 4090, 0, 4092, 2, 2, 4092, 0, 0, 0, 3, 3, 3, 3, 6, 0, 1, 4, 0, 842, 0, 0, 6, 0, 0, 4090, 4, 0, 4062, 4091, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 3848, 0, 0, 4096, 0, 0, 0, 24, 0, 0, 3916, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 0, 0, 0, 0, 0, 4096, 0, 0, 3956, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 1, 4043, 0, 1, 4095, 1, 1, 0, 0, 1, 0, 3812, 4095, 4095, 1, 0, 4095, 0, 2, 0, 4093, 2, 4095, 1, 0, 0, 1, 2, 4094, 4095, 0, 4095, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 467, 0, 0, 0, 0, 0, 4095, 1, 0, 3959, 4095, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4050, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4039, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 4067, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 3904, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 0, 0, 0, 4096, 0, 0, 3940, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3, 0, 6, 23, 0, 4, 4082, 0, 7, 4090, 7, 5, 6, 29, 6, 0, 3931, 4089, 4090, 6, 0, 4087, 0, 0, 0, 4079, 1, 4089, 0, 0, 6, 5, 6, 4091, 4085, 0, 4090, 0, 5, 4089, 0, 0, 0, 3, 5, 2, 4, 11, 1, 4, 6, 0, 137, 1, 0, 3, 0, 0, 4080, 5, 0, 4081, 4088, 4088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 0, 2, 3965, 0, 3726, 443, 0, 0, 0, 0, 0, 0, 4, 4092, 4, 4, 0, 31, 4, 0, 3799, 268, 0, 0, 0, 0, 0, 0, 3379, 288, 0, 4, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4092, 3, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4090, 0, 4092, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 6, 4092, 4089, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 842, 0, 0, 6, 4091, 3647, 3, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 4062, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8206, 7762, 8177, 8191, 8199, 8594, 8253] avg=8192.0000 max=8594 max_rank=6 min=7762 min_rank=2 imbalance_ratio=1.049072 max_load_reduction=56942 max_load_reduction_pct=0.868866 moved_to_echo=33237 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 3848, 0, 3731, 444, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 24, 0, 0, 3804, 268, 0, 0, 0, 0, 0, 0, 3916, 288, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 0, 0, 0, 4096, 3650, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3956, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8024, 8192, 8300, 8196, 8190, 8189, 8296, 8149] avg=8192.0000 max=8300 max_rank=2 min=8024 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32824 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4039, 0, 3731, 444, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 3804, 268, 0, 0, 0, 0, 0, 0, 4067, 288, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 4096, 3650, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8173, 8451, 8196, 8194, 8189, 7841, 8278] avg=8192.0000 max=8451 max_rank=2 min=7841 min_rank=6 imbalance_ratio=1.031616 max_load_reduction=57085 max_load_reduction_pct=0.871048 moved_to_echo=32502 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3924, 0, 3731, 443, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 47, 0, 0, 3804, 268, 0, 0, 0, 0, 0, 0, 3409, 288, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 0, 0, 4096, 3649, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3853, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8215, 7793, 8196, 8191, 8189, 8808, 8046] avg=8192.0000 max=8808 max_rank=6 min=7793 min_rank=2 imbalance_ratio=1.075195 max_load_reduction=56728 max_load_reduction_pct=0.865601 moved_to_echo=33234 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4095, 0, 3731, 444, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 3804, 268, 0, 0, 0, 0, 0, 0, 4050, 288, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 4096, 3650, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8169, 8434, 8196, 8194, 8190, 7813, 8270] avg=8192.0000 max=8434 max_rank=2 min=7813 min_rank=6 imbalance_ratio=1.029541 max_load_reduction=57102 max_load_reduction_pct=0.871307 moved_to_echo=32467 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 1, 4043, 0, 3730, 443, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 0, 1, 0, 3803, 268, 0, 0, 0, 0, 0, 0, 3812, 288, 0, 1, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 4095, 1, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4095, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 467, 0, 0, 0, 4095, 3648, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 3959, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8170, 8198, 8196, 8196, 8189, 8215, 8152] avg=8192.0000 max=8220 max_rank=0 min=8152 min_rank=7 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32752 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4091, 0, 3731, 444, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 3804, 268, 0, 0, 0, 0, 0, 0, 3904, 288, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 0, 4096, 3650, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3940, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8190, 8288, 8194, 8194, 8189, 8082, 8133] avg=8192.0000 max=8288 max_rank=2 min=8082 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32598 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 6, 23, 0, 4, 4082, 0, 3726, 446, 0, 0, 0, 0, 0, 0, 7, 4090, 7, 5, 6, 29, 6, 0, 3798, 273, 0, 0, 0, 0, 0, 0, 3931, 286, 0, 6, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 1, 4089, 0, 0, 6, 5, 5, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4085, 0, 4090, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 2, 4, 11, 4090, 4087, 0, 0, 0, 0, 0, 0, 1, 4, 6, 0, 137, 1, 0, 3, 4088, 3645, 2, 0, 0, 0, 0, 0, 0, 0, 4080, 5, 0, 4081, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 8221, 8310, 8185, 8186, 8202, 7887, 8255] avg=8192.0000 max=8310 max_rank=2 min=7887 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32530 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 0, 32128, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[30592, 2304, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 3584, 128, 0, 128, 32768, 29312, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 32768, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 256, 128, 0, 30464, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 32128, 0, 29952, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[753, 0, 4096, 18, 0, 0, 0, 0, 3345, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4075, 4091, 0, 0, 0, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[268, 0, 4096, 0, 0, 0, 0, 0, 3828, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[731, 0, 4096, 1, 0, 0, 0, 0, 3346, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[688, 0, 4096, 0, 0, 0, 0, 0, 3405, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[94, 0, 4090, 0, 3, 4, 0, 15, 3993, 0, 0, 4083, 0, 3, 5, 4090, 0, 4, 0, 1, 0, 0, 0, 3, 4092, 0, 0, 2, 4091, 4092, 4092, 5, 0, 3, 0, 3, 3, 3, 4093, 4, 4092, 4, 5, 0, 4090, 0, 0, 0, 1, 5, 4088, 7, 5, 4088, 4091, 4093, 4092, 2, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1877, 0, 4096, 0, 0, 0, 0, 0, 2220, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 4096, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1399, 0, 4095, 0, 1, 1, 0, 2, 2695, 1, 0, 4092, 0, 1, 1, 4095, 0, 1, 0, 1, 0, 4, 0, 0, 4095, 0, 0, 0, 4095, 4095, 4095, 1, 0, 2, 0, 0, 0, 0, 4093, 1, 4095, 0, 0, 0, 4095, 0, 0, 0, 1, 1, 4094, 0, 1, 4095, 4095, 4092, 4095, 1, 1, 2, 0, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[263, 5, 4088, 0, 6, 2, 0, 1, 3805, 0, 0, 4080, 0, 6, 7, 4088, 0, 7, 0, 6, 0, 2, 1, 6, 4087, 0, 5, 2, 4088, 4091, 4089, 8, 0, 9, 0, 0, 0, 4, 4091, 6, 4090, 6, 19, 0, 4090, 0, 5, 0, 2, 7, 4089, 8, 7, 4086, 4087, 4090, 4090, 5, 1, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[753, 0, 4096, 18, 0, 0, 0, 0, 3328, 2, 2, 0, 0, 0, 0, 0, 3345, 0, 0, 4096, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8209, 8190, 8193, 8186, 8194, 8173, 8192] avg=8192.0000 max=8209 max_rank=1 min=8173 min_rank=6 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[94, 0, 4090, 0, 3, 4, 0, 15, 3323, 2, 2, 0, 0, 0, 0, 0, 3993, 0, 0, 4083, 0, 3, 5, 767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 3, 4092, 4090, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 2, 4091, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 3, 3, 4093, 4, 4086, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 5, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 7, 5, 4088, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 2, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7533, 8851, 8190, 8190, 8195, 8189, 8199, 8189] avg=8192.0000 max=8851 max_rank=1 min=7533 min_rank=0 imbalance_ratio=1.080444 max_load_reduction=56685 max_load_reduction_pct=0.864944 moved_to_echo=32772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[268, 0, 4096, 0, 0, 0, 0, 0, 3328, 2, 2, 0, 0, 0, 0, 0, 3828, 0, 0, 4096, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7696, 8692, 8190, 8193, 8188, 8190, 8194, 8193] avg=8192.0000 max=8692 max_rank=1 min=7696 min_rank=0 imbalance_ratio=1.061035 max_load_reduction=56844 max_load_reduction_pct=0.867371 moved_to_echo=32765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[731, 0, 4096, 1, 0, 0, 0, 0, 3328, 2, 2, 0, 0, 0, 0, 0, 3346, 0, 0, 4096, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8210, 8190, 8193, 8206, 8190, 8194, 8193] avg=8192.0000 max=8210 max_rank=1 min=8160 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=57326 max_load_reduction_pct=0.874725 moved_to_echo=32783 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1399, 0, 4095, 0, 1, 1, 0, 2, 3327, 2, 2, 0, 0, 0, 0, 0, 2695, 1, 0, 4092, 0, 1, 1, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4093, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 1, 4095, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 2, 0, 1, 1, 0, 4094, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8829, 7558, 8195, 8192, 8186, 8188, 8192, 8196] avg=8192.0000 max=8829 max_rank=0 min=7558 min_rank=1 imbalance_ratio=1.077759 max_load_reduction=56707 max_load_reduction_pct=0.865280 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[688, 0, 4096, 0, 0, 0, 0, 0, 3328, 2, 2, 0, 0, 0, 0, 0, 3405, 0, 0, 4096, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8269, 8191, 8193, 8191, 8190, 8194, 8192] avg=8192.0000 max=8269 max_rank=1 min=8116 min_rank=0 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1877, 0, 4096, 0, 0, 0, 0, 0, 3328, 2, 3, 0, 0, 0, 0, 0, 2220, 0, 0, 4096, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9306, 7084, 8191, 8192, 8189, 8190, 8190, 8194] avg=8192.0000 max=9306 max_rank=0 min=7084 min_rank=1 imbalance_ratio=1.135986 max_load_reduction=56230 max_load_reduction_pct=0.858002 moved_to_echo=32763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[263, 5, 4088, 0, 6, 2, 0, 1, 3323, 5, 4, 0, 0, 0, 0, 0, 3805, 0, 0, 4080, 0, 6, 7, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 6, 0, 2, 1, 6, 4091, 4086, 0, 0, 0, 0, 0, 0, 4087, 0, 5, 2, 4088, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4, 4091, 6, 4085, 0, 0, 0, 0, 0, 0, 0, 4085, 6, 19, 0, 4090, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 0, 8, 7, 4086, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 5, 1, 0, 0, 0, 0, 1, 4089, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7697, 8663, 8199, 8190, 8195, 8205, 8200, 8187] avg=8192.0000 max=8663 max_rank=1 min=7697 min_rank=0 imbalance_ratio=1.057495 max_load_reduction=56873 max_load_reduction_pct=0.867813 moved_to_echo=32787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 0, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[6144, 128, 32768, 128, 128, 128, 0, 128, 26624, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26752, 128, 0, 32768, 0, 128, 128, 6144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4091, 0, 4063, 5, 0, 0, 0, 4087, 7, 0, 4, 0, 4092, 4092, 0, 4091, 4, 4, 5, 5, 4089, 3, 31, 4087, 0, 4, 4091, 3, 0, 0, 4068, 3, 1, 5, 4092, 4, 4, 3760, 4090, 6, 0, 2, 0, 0, 0, 0, 0, 4091, 1, 0, 5, 5, 0, 4092, 341, 0, 0, 2, 4091, 15, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4089, 7, 4093, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 67, 41, 3788, 0, 0, 0, 4096, 0, 0, 3176, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1124, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4095, 3, 4095, 0, 3, 2, 0, 4094, 2, 0, 1, 0, 4094, 4095, 0, 4095, 0, 0, 1, 1, 4095, 1, 0, 4094, 0, 1, 4094, 1, 27, 45, 3996, 0, 1, 1, 4095, 1, 0, 1922, 4095, 0, 0, 1, 1, 1, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 4095, 2194, 0, 0, 0, 4095, 1, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 4079, 0, 12, 4, 4078, 0, 0, 0, 4096, 0, 0, 286, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3831, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 4096, 0, 71, 54, 3812, 0, 0, 0, 4096, 0, 0, 2472, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1787, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4079, 0, 0, 4095, 0, 119, 26, 3759, 0, 0, 0, 4096, 0, 0, 2449, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1857, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 20, 0, 3977, 0, 0, 0, 4096, 0, 0, 451, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3746, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4089, 13, 4075, 8, 72, 15, 0, 4085, 6, 0, 6, 0, 4089, 4091, 0, 4090, 3, 6, 7, 10, 4084, 5, 0, 4088, 1, 7, 4087, 6, 0, 0, 3886, 0, 3, 8, 4090, 4, 1, 1295, 4087, 0, 2, 0, 0, 0, 3, 0, 5, 4087, 7, 0, 7, 9, 0, 4090, 2923, 0, 0, 0, 4086, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 117, 0, 67, 41, 3788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2118, 0, 0, 3176, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3978, 121, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1870, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2105, 1978, 6, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8193, 8183, 8109, 9390, 8194, 7090, 8191] avg=8192.0000 max=9390 max_rank=4 min=7090 min_rank=6 imbalance_ratio=1.146240 max_load_reduction=56146 max_load_reduction_pct=0.856720 moved_to_echo=32865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4085, 0, 4063, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 7, 0, 4, 0, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4, 4, 5, 5, 4089, 3, 31, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4, 117, 3, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 5, 2116, 4, 4, 3760, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 4091, 3974, 121, 0, 0, 0, 0, 0, 4091, 1, 0, 5, 5, 0, 1868, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4091, 15, 0, 0, 0, 2103, 1976, 6, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8191, 8226, 8279, 9983, 8194, 6311, 8199] avg=8192.0000 max=9983 max_rank=4 min=6311 min_rank=6 imbalance_ratio=1.218628 max_load_reduction=55553 max_load_reduction_pct=0.847672 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 117, 0, 12, 4, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2118, 0, 0, 286, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3962, 121, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1870, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2105, 1978, 6, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8193, 8186, 8305, 6500, 8178, 9797, 8191] avg=8192.0000 max=9797 max_rank=6 min=6500 min_rank=4 imbalance_ratio=1.195923 max_load_reduction=55739 max_load_reduction_pct=0.850510 moved_to_echo=32666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 3, 4095, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 117, 1, 27, 45, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2118, 1, 0, 1922, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 4093, 3977, 121, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 1869, 2194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 2105, 1977, 6, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8187, 8281, 8138, 8194, 8160, 8191] avg=8192.0000 max=8281 max_rank=3 min=8138 min_rank=4 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 117, 0, 20, 0, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2118, 0, 0, 451, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3977, 121, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1870, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2105, 1978, 6, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8193, 8186, 8210, 6665, 8193, 9712, 8191] avg=8192.0000 max=9712 max_rank=6 min=6665 min_rank=4 imbalance_ratio=1.185547 max_load_reduction=55824 max_load_reduction_pct=0.851807 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 117, 0, 119, 26, 3759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2118, 0, 0, 2449, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3978, 121, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1870, 1857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2105, 1978, 6, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8193, 8186, 8100, 8663, 8194, 7823, 8191] avg=8192.0000 max=8663 max_rank=4 min=7823 min_rank=6 imbalance_ratio=1.057495 max_load_reduction=56873 max_load_reduction_pct=0.867813 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 118, 0, 71, 54, 3812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2118, 0, 0, 2472, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3978, 121, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1870, 1787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2105, 1978, 6, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8192, 8186, 8149, 8686, 8194, 7753, 8191] avg=8192.0000 max=8686 max_rank=4 min=7753 min_rank=6 imbalance_ratio=1.060303 max_load_reduction=56850 max_load_reduction_pct=0.867462 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4079, 13, 4075, 8, 72, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 6, 0, 6, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3, 6, 7, 10, 4084, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 7, 115, 6, 0, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 2113, 4, 1, 1295, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 5, 4089, 3972, 124, 0, 0, 0, 0, 0, 4087, 7, 0, 7, 9, 0, 1857, 2923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 2109, 1977, 10, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8188, 8196, 8103, 7511, 8195, 8890, 8191] avg=8192.0000 max=8890 max_rank=6 min=7511 min_rank=4 imbalance_ratio=1.085205 max_load_reduction=56646 max_load_reduction_pct=0.864349 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 14976, 17920, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 128, 0, 16896, 15872, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 17024, 128, 128, 15872, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 32768, 31872, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 1024, 128, 384, 256, 31488, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4091, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4092, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 6, 3, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 3995, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 103, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4091, 4092, 5, 4, 0, 8, 4094, 0, 4086, 0, 4091, 3322, 1, 0, 0, 0, 4090, 4091, 4092, 0, 8, 4090, 4091, 0, 0, 0, 0, 1, 4088, 2, 7, 3, 5, 0, 0, 0, 0, 13, 4, 5, 31, 4079, 5, 2, 4092, 4046, 972, 0, 0, 0, 5, 0, 3896, 4, 0, 0, 5, 5, 0, 6, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4058, 0, 0, 0, 0, 4089, 0, 4094, 0, 4096, 4094, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 35, 7, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4, 1, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4076, 0, 0, 0, 0, 4095, 0, 4096, 0, 4096, 4043, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 37, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4095, 4095, 1, 1, 0, 0, 4095, 1, 4095, 0, 4095, 4083, 1, 0, 0, 1, 4095, 4095, 4095, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 4092, 1, 3, 0, 0, 0, 0, 0, 2, 4, 1, 1, 0, 4094, 2, 1, 4095, 4091, 63, 6, 1, 0, 1, 0, 4033, 1, 0, 0, 1, 1, 0, 1, 1, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4068, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 3967, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 162, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4088, 4089, 7, 5, 1, 11, 4092, 0, 4087, 0, 4087, 4082, 4, 0, 0, 0, 4084, 4085, 4090, 0, 10, 4088, 4089, 0, 0, 0, 0, 1, 4089, 5, 20, 0, 8, 0, 0, 0, 0, 6, 7, 7, 0, 4077, 14, 0, 4089, 4066, 170, 14, 8, 0, 9, 0, 3909, 4, 0, 0, 7, 8, 0, 8, 7, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 139, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3913, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4075, 4, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4094, 0, 0, 4086, 44, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8327, 8195, 8191, 8177, 8016, 8245, 8188] avg=8192.0000 max=8327 max_rank=1 min=8016 min_rank=5 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32626 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4092, 5, 4, 0, 8, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 139, 3322, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4091, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 2, 7, 3908, 181, 0, 0, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 13, 4, 4091, 4069, 5, 0, 0, 0, 0, 0, 5, 31, 3898, 5, 2, 0, 4046, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3896, 4, 0, 4081, 44, 19, 0, 0, 0, 0, 0, 0, 5, 5, 0, 6, 4, 0, 0, 4092, 4087, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 7548, 8191, 8187, 8190, 8959, 8049, 8199] avg=8192.0000 max=8959 max_rank=5 min=7548 min_rank=1 imbalance_ratio=1.093628 max_load_reduction=56577 max_load_reduction_pct=0.863297 moved_to_echo=33397 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4058, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 139, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3913, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 7, 0, 4096, 4075, 4, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 4096, 0, 0, 4086, 44, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8327, 8195, 8191, 8217, 8014, 8247, 8188] avg=8192.0000 max=8327 max_rank=1 min=8014 min_rank=5 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32666 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 139, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3913, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 4, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 0, 4094, 103, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 4086, 44, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8230, 8195, 8191, 8175, 8111, 8245, 8188] avg=8192.0000 max=8245 max_rank=6 min=8111 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32719 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4095, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 139, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4095, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 3, 3912, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 1, 4095, 4074, 5, 0, 0, 0, 0, 0, 1, 0, 3912, 2, 1, 0, 4091, 63, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 1, 0, 4033, 1, 0, 4085, 44, 18, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 2, 0, 4095, 4090, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8319, 8194, 8190, 8181, 8070, 8189, 8191] avg=8192.0000 max=8319 max_rank=1 min=8070 min_rank=5 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32631 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4076, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 139, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3913, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 4096, 4075, 5, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 0, 4096, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4086, 44, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8278, 8192, 8186, 8214, 8047, 8251, 8187] avg=8192.0000 max=8278 max_rank=1 min=8047 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32699 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4068, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 139, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3913, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 4096, 4075, 5, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 0, 4096, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 4086, 44, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8202, 8195, 8191, 8197, 8172, 8218, 8187] avg=8192.0000 max=8218 max_rank=6 min=8172 min_rank=5 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32774 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4089, 7, 5, 1, 11, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 132, 4082, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4085, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 5, 20, 3907, 187, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 6, 7, 4089, 4070, 5, 0, 0, 0, 0, 0, 7, 0, 3890, 14, 0, 0, 4066, 170, 0, 0, 0, 0, 0, 0, 0, 0, 14, 8, 0, 9, 0, 3909, 4, 0, 4082, 48, 18, 0, 0, 0, 0, 0, 0, 7, 8, 0, 8, 7, 4, 0, 4090, 4084, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8305, 8179, 8209, 8185, 8147, 8092, 8208] avg=8192.0000 max=8305 max_rank=1 min=8092 min_rank=6 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32632 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 31360, 128, 128, 0, 32768, 1536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 32768, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 1152, 31744, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 128, 32384, 128, 0, 32768, 384, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 128, 128, 128, 32768, 32640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 32768, 128, 128, 31360, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4048, 0, 0, 3, 5, 0, 0, 0, 1, 4090, 4090, 4, 1, 0, 3, 23, 0, 4090, 1, 0, 1919, 4049, 11, 0, 0, 4090, 0, 3, 4, 7, 0, 4091, 4, 5, 4088, 4092, 3, 4091, 0, 0, 10, 0, 0, 0, 0, 4092, 5, 2416, 5, 6, 4083, 6, 2, 0, 0, 6, 0, 5, 477, 4091, 4089, 0, 3424, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 3782, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 385, 0, 0, 0, 0, 4095, 0, 759, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 4080, 4096, 0, 3235, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 1, 2, 0, 4095, 0, 1, 3137, 4078, 1, 0, 3, 4094, 0, 0, 1, 1, 0, 4095, 0, 1, 4095, 4095, 1, 4095, 0, 0, 1000, 1, 0, 1, 0, 4095, 1, 1370, 0, 1, 4092, 1, 1, 0, 0, 1, 0, 1, 19, 4095, 4095, 0, 2679, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 1258, 4096, 0, 5, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 9, 326, 0, 0, 0, 0, 4092, 0, 3515, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 4085, 4096, 0, 3072, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 835, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 164, 0, 0, 0, 0, 4096, 0, 3592, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 4096, 0, 3600, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 2178, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 212, 0, 0, 0, 0, 4096, 0, 1883, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 149, 4077, 4096, 0, 3771, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1469, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 121, 0, 0, 0, 0, 4096, 0, 3524, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 31, 4094, 4096, 0, 3050, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4087, 1, 0, 1, 8, 0, 1, 0, 0, 4088, 4086, 6, 0, 0, 8, 18, 6, 4088, 0, 2, 1081, 4002, 5, 0, 5, 4086, 0, 4, 7, 11, 0, 4088, 0, 8, 4084, 4089, 4, 4087, 0, 0, 2, 5, 0, 0, 1, 4085, 7, 4022, 8, 9, 4063, 10, 8, 0, 0, 8, 0, 7, 183, 4089, 4088, 0, 2975, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4048, 0, 0, 3, 5, 0, 0, 0, 4092, 6, 0, 0, 0, 0, 0, 0, 1, 4086, 4090, 4, 1, 0, 3, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2149, 1, 0, 1919, 4049, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3, 4, 7, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 4088, 0, 3, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4092, 5, 2416, 1175, 4, 1, 0, 0, 0, 0, 0, 5, 6, 4083, 6, 2, 0, 0, 6, 3321, 766, 0, 0, 0, 0, 0, 0, 0, 5, 477, 4091, 762, 0, 3424, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8208, 8129, 8194, 8191, 7703, 8195, 8762] avg=8192.0000 max=8762 max_rank=7 min=7703 min_rank=5 imbalance_ratio=1.069580 max_load_reduction=56774 max_load_reduction_pct=0.866302 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 7, 0, 1258, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 326, 0, 0, 0, 0, 4092, 0, 3515, 1177, 4, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3327, 767, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4085, 763, 0, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8188, 7518, 8192, 8202, 9116, 8190, 7934] avg=8192.0000 max=9116 max_rank=5 min=7518 min_rank=2 imbalance_ratio=1.112793 max_load_reduction=56420 max_load_reduction_pct=0.860901 moved_to_echo=33442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 16, 0, 3782, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 0, 0, 0, 0, 4095, 0, 759, 1177, 4, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3327, 767, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4080, 763, 0, 3235, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8188, 10045, 8190, 8192, 6422, 8190, 8111] avg=8192.0000 max=10045 max_rank=2 min=6422 min_rank=5 imbalance_ratio=1.226196 max_load_reduction=55491 max_load_reduction_pct=0.846725 moved_to_echo=30915 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 0, 0, 835, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 4096, 0, 3592, 1177, 4, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3327, 767, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 763, 0, 3600, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8188, 7082, 8190, 8192, 9035, 8190, 8461] avg=8192.0000 max=9035 max_rank=5 min=7082 min_rank=2 imbalance_ratio=1.102905 max_load_reduction=56501 max_load_reduction_pct=0.862137 moved_to_echo=33878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 0, 0, 1469, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 0, 4096, 0, 3524, 1177, 4, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3327, 767, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4094, 763, 0, 3050, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8188, 7716, 8190, 8192, 8924, 8190, 7938] avg=8192.0000 max=8924 max_rank=5 min=7716 min_rank=2 imbalance_ratio=1.089355 max_load_reduction=56612 max_load_reduction_pct=0.863831 moved_to_echo=33244 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 1, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 0, 1, 3137, 4078, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 1, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1, 0, 1, 0, 4095, 1, 1370, 1177, 4, 2, 0, 0, 0, 0, 0, 0, 1, 4092, 1, 1, 0, 0, 1, 3326, 767, 0, 0, 0, 0, 0, 0, 0, 1, 19, 4095, 763, 0, 2679, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8189, 9368, 8192, 8192, 7651, 8189, 7558] avg=8192.0000 max=9368 max_rank=2 min=7558 min_rank=7 imbalance_ratio=1.143555 max_load_reduction=56168 max_load_reduction_pct=0.857056 moved_to_echo=31590 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 18, 0, 2178, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 4096, 0, 1883, 1177, 4, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3327, 767, 0, 0, 0, 0, 0, 0, 0, 0, 149, 4077, 763, 0, 3771, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8188, 8444, 8190, 8192, 7374, 8190, 8760] avg=8192.0000 max=8760 max_rank=7 min=7374 min_rank=5 imbalance_ratio=1.069336 max_load_reduction=56776 max_load_reduction_pct=0.866333 moved_to_echo=32516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4087, 1, 0, 1, 8, 0, 1, 0, 4089, 10, 0, 0, 0, 0, 0, 0, 0, 4081, 4086, 6, 0, 0, 8, 18, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2138, 0, 2, 1081, 4002, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4086, 0, 4, 7, 11, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4084, 0, 4, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 1, 4085, 7, 4022, 1179, 7, 3, 0, 0, 0, 0, 0, 8, 9, 4063, 10, 8, 0, 0, 8, 3325, 771, 0, 0, 0, 0, 0, 0, 0, 7, 183, 4089, 753, 0, 2975, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8199, 7234, 8198, 8183, 9311, 8202, 8012] avg=8192.0000 max=9311 max_rank=5 min=7234 min_rank=2 imbalance_ratio=1.136597 max_load_reduction=56225 max_load_reduction_pct=0.857925 moved_to_echo=33708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 17280, 128, 128, 15744, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[2304, 128, 0, 128, 128, 32768, 128, 21120, 9472, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 1024, 32768, 6144, 0, 25856, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 0, 128, 26624, 6144, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[3, 0, 4092, 6, 5, 0, 5, 0, 0, 4089, 5, 1, 4090, 8, 4087, 4087, 2, 0, 2, 4091, 4, 6, 0, 6, 0, 6, 0, 0, 0, 1, 4, 2, 0, 4090, 4091, 0, 3985, 5, 0, 0, 3, 8, 5, 0, 5, 4082, 3, 6, 1, 0, 111, 0, 4091, 6, 0, 0, 0, 4091, 4091, 4082, 0, 4090, 4088, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 0, 4094, 0, 2, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 61, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4079, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 123, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4076, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 4095, 1, 1, 0, 1, 0, 0, 4094, 1, 1, 4095, 2, 4094, 4094, 1, 0, 1, 4095, 1, 1, 5, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 4093, 4095, 0, 4069, 1, 0, 0, 0, 2, 0, 1, 1, 4094, 0, 1, 0, 0, 21, 0, 4095, 0, 0, 0, 0, 4095, 4095, 4095, 0, 4095, 4094, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 0, 4096, 0, 1, 0, 0, 4096, 4096, 4081, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 4089, 0, 14, 0, 0, 4091, 4096, 4084, 4, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4, 0, 4088, 6, 8, 1, 9, 0, 0, 4085, 7, 0, 4087, 19, 4084, 4082, 8, 7, 4, 4088, 7, 0, 0, 10, 6, 10, 0, 0, 0, 0, 6, 4, 0, 4086, 4087, 0, 4065, 4, 0, 0, 3, 14, 25, 0, 10, 4078, 2, 10, 0, 3, 6, 0, 4089, 7, 0, 0, 9, 4086, 4088, 4063, 0, 4088, 4083, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4092, 6, 5, 0, 5, 0, 4086, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 8, 4087, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4091, 4, 6, 0, 6, 4081, 4, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 4, 2, 4091, 4088, 0, 0, 0, 0, 0, 0, 0, 4090, 46, 0, 3985, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 5, 0, 5, 4082, 3, 6, 4075, 8, 1, 0, 0, 0, 0, 0, 1, 0, 111, 0, 4091, 6, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4082, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8188, 8196, 8192, 8126, 8196, 8253, 8186] avg=8192.0000 max=8253 max_rank=6 min=8126 min_rank=4 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32761 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 7, 1, 0, 0, 0, 0, 0, 0, 0, 61, 0, 4096, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8194, 8188, 8191, 8194, 8184, 8206, 8191] avg=8192.0000 max=8206 max_rank=6 min=8184 min_rank=5 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32775 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4075, 7, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4089, 0, 14, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4084, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8194, 8188, 8191, 8236, 8179, 8160, 8200] avg=8192.0000 max=8236 max_rank=4 min=8160 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32775 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 9, 1, 0, 0, 0, 0, 0, 0, 0, 123, 0, 4096, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4076, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8192, 8188, 8191, 8135, 8186, 8268, 8188] avg=8192.0000 max=8268 max_rank=6 min=8135 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 7, 1, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4094, 0, 2, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8189, 8190, 8217, 8184, 8166, 8208] avg=8192.0000 max=8217 max_rank=4 min=8166 min_rank=6 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32775 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4095, 1, 1, 0, 1, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1, 1, 5, 1, 4085, 4, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 46, 0, 4069, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 4094, 0, 1, 4079, 9, 1, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4095, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4095, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8194, 8191, 8209, 8188, 8164, 8206] avg=8192.0000 max=8209 max_rank=4 min=8164 min_rank=6 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32767 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 9, 1, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4096, 0, 1, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4081, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8187, 8190, 8231, 8186, 8167, 8193] avg=8192.0000 max=8231 max_rank=4 min=8167 min_rank=6 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4088, 6, 8, 1, 9, 0, 4083, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 19, 4084, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7, 4, 4088, 7, 0, 0, 10, 4078, 4, 0, 0, 0, 0, 0, 0, 6, 10, 0, 0, 0, 0, 6, 4, 4088, 4086, 0, 0, 0, 0, 0, 0, 0, 4086, 33, 0, 4065, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 14, 25, 0, 10, 4078, 2, 10, 4077, 7, 7, 0, 0, 0, 0, 0, 0, 3, 6, 0, 4089, 7, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 4063, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8192, 8206, 8200, 8188, 8233, 8152, 8164] avg=8192.0000 max=8233 max_rank=5 min=8152 min_rank=6 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32737 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 384, 0, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 32768, 128, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 384, 0, 32768, 128, 128, 0, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 128, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4, 3943, 8, 4090, 0, 4089, 0, 7, 0, 19, 0, 0, 4086, 0, 229, 4089, 4, 4066, 2, 4089, 4091, 0, 4079, 6, 4076, 0, 0, 5, 2, 5, 0, 7, 4, 13, 7, 0, 0, 0, 0, 0, 4, 5, 7, 0, 4086, 105, 4090, 0, 21, 0, 5, 2, 4090, 0, 6, 4091, 0, 6, 6, 4088, 0, 3904] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4092, 0, 12, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 7, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 0, 4089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 9, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 4096, 0, 4074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 1, 4094, 0, 4095, 0, 4095, 0, 1, 0, 5, 0, 0, 4095, 0, 6, 4092, 0, 4094, 0, 4095, 4095, 0, 4092, 1, 4094, 2, 0, 1, 0, 1, 0, 1, 0, 2, 2, 0, 0, 0, 0, 0, 1, 1, 1, 0, 4094, 0, 4095, 0, 11, 0, 1, 1, 4095, 1, 1, 4095, 0, 0, 2, 4094, 0, 4078] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 6, 4058, 10, 4085, 0, 4082, 0, 12, 0, 38, 0, 0, 4084, 0, 103, 4073, 13, 4018, 0, 4086, 4086, 0, 4037, 8, 4070, 0, 0, 7, 44, 9, 0, 9, 5, 20, 11, 0, 0, 0, 0, 0, 7, 9, 11, 4, 4082, 150, 4078, 0, 22, 0, 8, 1, 4087, 13, 10, 4089, 0, 0, 10, 4084, 0, 3895] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 3943, 8, 24, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 19, 0, 0, 4086, 0, 4066, 10, 7, 0, 0, 0, 0, 0, 229, 4060, 4, 4066, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 6, 4076, 0, 0, 5, 2, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 13, 7, 0, 0, 0, 4091, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 7, 0, 4086, 105, 4023, 29, 3, 0, 0, 0, 0, 0, 4090, 0, 21, 0, 5, 2, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4091, 0, 6, 6, 55, 0, 3904, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 8195, 8361, 8178, 8204, 8262, 8200, 8068] avg=8192.0000 max=8361 max_rank=2 min=8068 min_rank=0 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32734 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 24, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4072, 11, 7, 0, 0, 0, 0, 0, 0, 4067, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4031, 29, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 54, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8186, 8163, 8197, 8185, 8159, 8184, 8246] avg=8192.0000 max=8246 max_rank=7 min=8159 min_rank=5 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 24, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4072, 11, 7, 0, 0, 0, 0, 0, 0, 4067, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4031, 29, 3, 0, 0, 0, 0, 0, 4092, 0, 12, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 54, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8186, 8163, 8197, 8187, 8161, 8192, 8234] avg=8192.0000 max=8234 max_rank=7 min=8161 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 24, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4072, 11, 7, 0, 0, 0, 0, 0, 0, 4067, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4031, 29, 3, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 54, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8186, 8163, 8191, 8185, 8159, 8193, 8246] avg=8192.0000 max=8246 max_rank=7 min=8159 min_rank=5 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32783 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 4094, 0, 24, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 4095, 0, 4071, 11, 7, 0, 0, 0, 0, 0, 6, 4063, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 4094, 2, 0, 1, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 0, 0, 0, 4095, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 4094, 0, 4029, 29, 3, 0, 0, 0, 0, 0, 4095, 0, 11, 0, 1, 1, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 2, 54, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8190, 8163, 8196, 8188, 8158, 8196, 8230] avg=8192.0000 max=8230 max_rank=7 min=8158 min_rank=5 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 24, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4072, 11, 7, 0, 0, 0, 0, 0, 0, 4067, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4031, 29, 3, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 54, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8186, 8163, 8197, 8187, 8159, 8191, 8237] avg=8192.0000 max=8237 max_rank=7 min=8159 min_rank=5 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 24, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4072, 11, 7, 0, 0, 0, 0, 0, 21, 4067, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4031, 29, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 54, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8186, 8184, 8197, 8191, 8159, 8184, 8219] avg=8192.0000 max=8219 max_rank=7 min=8159 min_rank=5 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 6, 4058, 10, 18, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 38, 0, 0, 4084, 0, 4067, 12, 8, 0, 0, 0, 0, 0, 103, 4042, 13, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 8, 4070, 0, 0, 7, 44, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 9, 5, 20, 11, 0, 0, 0, 4086, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 11, 4, 4082, 150, 4020, 31, 5, 0, 0, 0, 0, 0, 4078, 0, 22, 0, 8, 1, 4074, 13, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4089, 0, 0, 10, 52, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8221, 8176, 8183, 8209, 8319, 8196, 8056] avg=8192.0000 max=8319 max_rank=5 min=8056 min_rank=7 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32780 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 32768, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 512, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32640, 128, 256, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 0, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[384, 32512, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 32768, 256, 32256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 3900, 1, 0, 2887, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3765, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1719, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 719, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4077, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3536, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 4091, 0, 0, 0, 0, 4089, 5, 0, 2, 5, 2, 5, 0, 4087, 0, 0, 855, 1, 4087, 4088, 0, 4088, 0, 2, 23, 6, 4091, 3, 4090, 0, 53, 8, 4039, 9, 0, 1, 0, 4, 0, 3, 3240, 0, 4090, 7, 4, 4090, 5, 4091, 4090, 0, 0, 4083, 0, 6, 0, 6, 7, 0, 3, 4071, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3828, 0, 0, 1025, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3344, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 7, 4094, 1, 1, 1, 1, 1, 1, 0, 4087, 0, 0, 228, 1, 4095, 4095, 0, 4095, 0, 0, 2, 1, 4084, 0, 4095, 0, 0, 0, 4088, 0, 0, 0, 1, 0, 0, 1, 3886, 0, 4095, 1, 0, 4095, 1, 4095, 4094, 0, 0, 4092, 0, 1, 0, 1, 2, 1, 1, 4095, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 3370, 0, 0, 903, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4060, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 3736, 0, 0, 967, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3739, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 3, 4085, 0, 0, 5, 2, 4085, 9, 3, 3, 8, 4, 7, 0, 4090, 0, 0, 161, 4, 4083, 4080, 0, 4082, 0, 0, 31, 9, 4086, 0, 4087, 0, 56, 12, 4032, 5, 13, 0, 10, 0, 0, 6, 3914, 0, 4085, 12, 1, 4086, 8, 4089, 4087, 0, 0, 4084, 0, 10, 0, 9, 7, 7, 6, 4061, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3900, 4096, 209, 0, 0, 0, 0, 0, 0, 1, 0, 2887, 0, 3133, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3765, 0, 4096, 1, 88, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3217, 875, 0, 0, 0, 0, 0, 0, 0, 0, 1719, 0, 4096, 0, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3884, 213, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8205, 10117, 7951, 8188, 6481, 8195, 8192] avg=8192.0000 max=10117 max_rank=2 min=6481 min_rank=5 imbalance_ratio=1.234985 max_load_reduction=55419 max_load_reduction_pct=0.845627 moved_to_echo=31056 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4090, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 5, 2, 5, 0, 4087, 4088, 209, 0, 0, 0, 0, 0, 0, 0, 0, 855, 1, 3127, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 23, 6, 4091, 3, 4090, 0, 87, 1, 0, 0, 0, 0, 0, 0, 53, 8, 4039, 9, 0, 1, 0, 4, 3212, 873, 0, 0, 0, 0, 0, 0, 0, 3, 3240, 0, 4090, 7, 4, 665, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4090, 0, 0, 4083, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 3, 4071, 0, 5, 3882, 213, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8403, 8071, 8303, 8199, 8009, 8184, 8187] avg=8192.0000 max=8403 max_rank=1 min=8009 min_rank=5 imbalance_ratio=1.025757 max_load_reduction=57133 max_load_reduction_pct=0.871780 moved_to_echo=32579 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 4096, 209, 0, 0, 0, 0, 0, 0, 0, 0, 719, 0, 3133, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 0, 88, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3217, 875, 0, 0, 0, 0, 0, 0, 0, 0, 3536, 0, 4096, 0, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3884, 213, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8261, 7948, 8262, 8188, 8298, 8195, 8193] avg=8192.0000 max=8298 max_rank=5 min=7948 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32874 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 4096, 209, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 0, 3133, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 88, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3217, 875, 0, 0, 0, 0, 0, 0, 0, 0, 3344, 0, 4096, 0, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3884, 213, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8133, 8254, 8276, 8188, 8106, 8195, 8193] avg=8192.0000 max=8276 max_rank=3 min=8106 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32682 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3370, 4096, 212, 0, 0, 0, 0, 0, 0, 0, 0, 903, 0, 3133, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 4096, 0, 88, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3217, 875, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 4096, 0, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 3884, 213, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 7678, 8132, 8245, 8188, 8716, 8193, 8186] avg=8192.0000 max=8716 max_rank=5 min=7678 min_rank=1 imbalance_ratio=1.063965 max_load_reduction=56820 max_load_reduction_pct=0.867004 moved_to_echo=33283 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3736, 4096, 209, 0, 0, 0, 0, 0, 0, 0, 0, 967, 0, 3133, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 4096, 0, 88, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3217, 875, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 4096, 0, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3885, 213, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8041, 8196, 7924, 8188, 8603, 8194, 8194] avg=8192.0000 max=8603 max_rank=5 min=7924 min_rank=3 imbalance_ratio=1.050171 max_load_reduction=56933 max_load_reduction_pct=0.868729 moved_to_echo=33179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 0, 7, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 4087, 4095, 209, 0, 0, 0, 0, 0, 0, 0, 0, 228, 1, 3132, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4084, 0, 4095, 0, 88, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 1, 0, 3216, 875, 0, 0, 0, 0, 0, 0, 0, 1, 3886, 0, 4095, 1, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4094, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 1, 4095, 0, 1, 3883, 213, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8397, 7456, 8271, 8180, 8649, 8191, 8197] avg=8192.0000 max=8649 max_rank=5 min=7456 min_rank=2 imbalance_ratio=1.055786 max_load_reduction=56887 max_load_reduction_pct=0.868027 moved_to_echo=33217 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4083, 0, 0, 5, 2, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 3, 8, 4, 7, 0, 4090, 4082, 212, 0, 0, 0, 0, 0, 0, 0, 0, 161, 4, 3117, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 9, 4086, 0, 4087, 0, 89, 2, 0, 0, 0, 0, 0, 0, 56, 12, 4032, 5, 13, 0, 10, 0, 3212, 877, 0, 0, 0, 0, 0, 0, 0, 6, 3914, 0, 4085, 12, 1, 656, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4087, 0, 0, 4084, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 7, 7, 6, 4061, 0, 9, 3877, 218, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8418, 7362, 8304, 8217, 8674, 8189, 8194] avg=8192.0000 max=8674 max_rank=5 min=7362 min_rank=2 imbalance_ratio=1.058838 max_load_reduction=56862 max_load_reduction_pct=0.867645 moved_to_echo=33274 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 128, 31104, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 7808, 128, 25088, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32640, 128, 128, 128, 128, 128, 25728, 7040, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 31104, 32768, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 27520, 0, 32768, 128, 128, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 32000, 128, 32768, 128, 768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[6, 7, 4, 0, 4, 4091, 0, 0, 8, 2, 2, 0, 0, 5, 0, 4089, 0, 4087, 4, 4084, 1, 4086, 4087, 2549, 4091, 2, 4090, 18, 0, 122, 4087, 4088, 0, 2, 0, 33, 0, 4092, 0, 4089, 6, 4085, 5, 0, 4088, 5, 3, 5, 6, 0, 0, 4, 1, 6, 4090, 0, 0, 1376, 0, 15, 0, 0, 6, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 721, 4096, 0, 4096, 0, 0, 89, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3278, 0, 0, 11, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4091, 4096, 1163, 4096, 0, 4096, 67, 0, 135, 4096, 4090, 0, 0, 0, 0, 0, 4069, 0, 4096, 0, 4096, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2813, 0, 0, 27, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4093, 4096, 533, 4096, 0, 4096, 6, 0, 228, 4096, 4090, 0, 0, 0, 0, 0, 4070, 0, 4096, 0, 4096, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3385, 0, 0, 29, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1, 1, 1, 0, 1, 4095, 0, 0, 0, 1, 2, 0, 0, 1, 1, 4095, 0, 4092, 1, 4095, 0, 4095, 4094, 1125, 4095, 1, 4095, 13, 0, 64, 4094, 4092, 0, 0, 0, 9, 0, 4079, 0, 4094, 1, 4094, 1, 0, 4094, 1, 4, 1, 1, 0, 0, 1, 1, 1, 4095, 0, 0, 2901, 0, 1, 0, 0, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4092, 4096, 2390, 4096, 0, 4096, 31, 0, 401, 4096, 4092, 0, 0, 0, 0, 0, 4065, 0, 4096, 0, 4092, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1356, 0, 0, 10, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[10, 12, 5, 0, 6, 4091, 0, 0, 13, 0, 4, 0, 0, 8, 0, 4084, 0, 4065, 5, 4081, 4, 4079, 4084, 2477, 4087, 4, 4086, 4, 0, 1, 4084, 4086, 0, 4, 0, 26, 0, 4090, 0, 4084, 8, 4080, 8, 0, 4082, 9, 7, 7, 9, 0, 0, 6, 5, 10, 4088, 0, 0, 1608, 0, 7, 0, 0, 10, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 258, 4096, 0, 4096, 0, 0, 503, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3332, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[6, 7, 4, 0, 4, 4091, 0, 0, 2516, 1593, 0, 0, 0, 0, 0, 0, 8, 2, 2, 0, 0, 5, 0, 4089, 2494, 1592, 0, 0, 0, 0, 0, 0, 0, 4087, 4, 0, 1, 2695, 0, 2549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 18, 0, 122, 3885, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 33, 0, 4092, 0, 4089, 4, 2, 0, 0, 0, 0, 0, 0, 6, 4085, 5, 0, 4084, 5, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4, 1, 6, 4090, 0, 2492, 1391, 200, 0, 0, 0, 0, 0, 0, 1376, 0, 15, 0, 0, 6, 5, 4091, 1574, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8192, 9336, 8115, 8222, 8193, 8190, 7067] avg=8192.0000 max=9336 max_rank=2 min=7067 min_rank=7 imbalance_ratio=1.139648 max_load_reduction=56200 max_load_reduction_pct=0.857544 moved_to_echo=31672 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4094, 0, 0, 2520, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2500, 1596, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2701, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 3893, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2500, 1395, 200, 0, 0, 0, 0, 0, 0, 3278, 0, 0, 11, 0, 0, 0, 4096, 1576, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8192, 7518, 8078, 8198, 8188, 8191, 8961] avg=8192.0000 max=8961 max_rank=7 min=7518 min_rank=2 imbalance_ratio=1.093872 max_load_reduction=56575 max_load_reduction_pct=0.863266 moved_to_echo=33538 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4037, 0, 0, 2520, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2499, 1596, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2699, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 228, 3893, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4096, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2500, 1394, 200, 0, 0, 0, 0, 0, 0, 3385, 0, 0, 29, 0, 0, 0, 4096, 1576, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8191, 7328, 8217, 8173, 8197, 8190, 9086] avg=8192.0000 max=9086 max_rank=7 min=7328 min_rank=2 imbalance_ratio=1.109131 max_load_reduction=56450 max_load_reduction_pct=0.861359 moved_to_echo=33646 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4019, 0, 0, 2521, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2499, 1597, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2698, 0, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 135, 3893, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4096, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2499, 1393, 200, 0, 0, 0, 0, 0, 0, 2813, 0, 0, 27, 0, 0, 0, 4096, 1575, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8192, 7957, 8185, 8172, 8194, 8188, 8511] avg=8192.0000 max=8511 max_rank=7 min=7957 min_rank=2 imbalance_ratio=1.038940 max_load_reduction=57025 max_load_reduction_pct=0.870132 moved_to_echo=33065 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 0, 1, 4095, 0, 0, 2519, 1596, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 1, 1, 4095, 2498, 1595, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 2701, 0, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 64, 3891, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4079, 0, 4094, 4, 3, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 4090, 1, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 4095, 0, 2500, 1394, 200, 0, 0, 0, 0, 0, 0, 2901, 0, 1, 0, 0, 1, 1, 4095, 1576, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8193, 7919, 8061, 8189, 8192, 8193, 8575] avg=8192.0000 max=8575 max_rank=7 min=7919 min_rank=2 imbalance_ratio=1.046753 max_load_reduction=56961 max_load_reduction_pct=0.869156 moved_to_echo=33149 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4041, 0, 0, 2520, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2500, 1596, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2699, 0, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 401, 3893, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 4096, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4092, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2500, 1393, 200, 0, 0, 0, 0, 0, 0, 1356, 0, 0, 10, 0, 0, 0, 4096, 1576, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8192, 9185, 8417, 8168, 8190, 8189, 7038] avg=8192.0000 max=9185 max_rank=2 min=7038 min_rank=7 imbalance_ratio=1.121216 max_load_reduction=56351 max_load_reduction_pct=0.859848 moved_to_echo=31585 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[10, 12, 5, 0, 6, 4091, 0, 0, 2515, 1592, 0, 0, 0, 0, 0, 0, 13, 0, 4, 0, 0, 8, 0, 4084, 2492, 1591, 0, 0, 0, 0, 0, 0, 0, 4065, 5, 0, 4, 2687, 0, 2477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 1, 3876, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 26, 0, 4090, 0, 4084, 7, 4, 0, 0, 0, 0, 0, 0, 8, 4080, 8, 0, 4075, 9, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 6, 5, 10, 4088, 0, 2490, 1392, 204, 0, 0, 0, 0, 0, 0, 1608, 0, 7, 0, 0, 10, 8, 4087, 1571, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8192, 9238, 7971, 8215, 8194, 8204, 7291] avg=8192.0000 max=9238 max_rank=2 min=7291 min_rank=7 imbalance_ratio=1.127686 max_load_reduction=56298 max_load_reduction_pct=0.859039 moved_to_echo=31904 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 2520, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2500, 1596, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2701, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 3893, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2500, 1395, 200, 0, 0, 0, 0, 0, 0, 3332, 0, 0, 3, 0, 0, 0, 4096, 1576, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8192, 7055, 8492, 8199, 8188, 8191, 9007] avg=8192.0000 max=9007 max_rank=7 min=7055 min_rank=2 imbalance_ratio=1.099487 max_load_reduction=56529 max_load_reduction_pct=0.862564 moved_to_echo=33585 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 32640, 0, 0, 20224, 12800, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 21632, 0, 11264, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 20096, 0, 128, 128, 0, 128, 128, 32768, 12672, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 256, 0, 1664, 31232, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 0, 32768, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 128, 128, 32768, 20096, 12800, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 128, 128, 32768, 0, 20096, 11264, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[8, 7, 1298, 3, 199, 0, 1500, 0, 3157, 84, 80, 2755, 1056, 384, 0, 3269, 1203, 920, 415, 599, 2525, 1300, 265, 1515, 2, 2602, 19, 1067, 614, 13, 320, 1939, 4031, 4, 49, 2019, 2603, 25, 629, 1718, 3398, 416, 1043, 804, 0, 867, 1244, 1969, 464, 3874, 427, 2, 1, 8, 682, 458, 0, 3475, 3102, 570, 283, 1023, 1201, 29] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[5, 27, 1042, 8, 243, 2, 999, 9, 2531, 69, 43, 1875, 964, 510, 0, 3346, 1487, 1625, 643, 275, 2554, 1410, 214, 1493, 0, 1514, 47, 1125, 698, 13, 236, 1788, 4079, 14, 83, 1790, 3371, 11, 691, 2224, 3649, 341, 518, 478, 0, 895, 1164, 2638, 107, 3589, 377, 5, 1, 3, 1141, 627, 1, 3553, 3251, 1125, 385, 1097, 1492, 41] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[3, 1, 884, 3, 106, 0, 1660, 0, 2909, 106, 66, 2221, 1127, 404, 0, 3303, 965, 1526, 328, 231, 2502, 1128, 122, 1515, 0, 2921, 6, 911, 639, 1, 284, 1643, 4088, 19, 49, 2197, 2578, 5, 326, 2275, 3630, 164, 504, 580, 6, 1248, 1166, 2622, 64, 3869, 390, 0, 2, 0, 948, 580, 0, 3558, 3344, 1079, 259, 1625, 842, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3, 7, 1024, 5, 210, 2, 1437, 1, 2606, 70, 49, 1984, 857, 440, 0, 3562, 1279, 866, 855, 377, 2151, 1181, 248, 1547, 0, 2226, 6, 1260, 442, 8, 196, 1970, 4093, 3, 45, 1792, 3313, 11, 532, 2195, 3396, 471, 690, 446, 1, 1048, 1736, 2753, 211, 3811, 511, 0, 4, 7, 797, 576, 2, 3306, 3206, 1026, 516, 1229, 934, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[11, 6, 1085, 0, 228, 0, 1134, 0, 2228, 56, 98, 2615, 1024, 462, 0, 3135, 1555, 1193, 733, 437, 2473, 1470, 193, 1353, 0, 2564, 35, 1111, 461, 6, 67, 1457, 4069, 10, 61, 1903, 2062, 7, 636, 2054, 3636, 326, 499, 337, 0, 1029, 1294, 2920, 142, 3864, 480, 0, 0, 0, 1141, 1204, 1, 2957, 3378, 1185, 459, 1515, 1169, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[7, 20, 961, 0, 214, 0, 1210, 0, 2604, 39, 77, 1961, 1305, 582, 0, 3160, 1129, 1212, 332, 502, 2758, 1293, 254, 1383, 0, 2525, 32, 1192, 415, 20, 284, 1891, 4089, 60, 64, 1623, 2708, 0, 1229, 2117, 3730, 514, 547, 468, 5, 1628, 1096, 2331, 104, 3775, 529, 0, 1, 15, 888, 522, 0, 3513, 2717, 1087, 315, 1726, 753, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[4, 21, 1135, 2, 148, 0, 1421, 2, 2777, 39, 58, 2240, 816, 580, 0, 2987, 1122, 1698, 310, 182, 2446, 1200, 237, 1583, 2, 2767, 3, 987, 755, 8, 160, 1727, 4070, 3, 103, 2038, 2318, 4, 412, 2507, 3331, 423, 452, 836, 4, 1079, 1087, 2567, 153, 3927, 519, 0, 1, 3, 902, 657, 2, 3535, 3299, 1046, 222, 1252, 1363, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[5, 26, 1250, 2, 175, 1, 1157, 24, 2843, 87, 58, 2689, 1057, 779, 3, 3356, 1692, 1207, 520, 518, 2672, 1270, 166, 1414, 5, 2350, 81, 1144, 745, 3, 193, 1788, 4091, 10, 60, 1493, 3260, 6, 620, 2072, 3433, 334, 606, 651, 0, 1011, 1048, 2167, 129, 3558, 551, 1, 24, 6, 1187, 710, 0, 3575, 2729, 476, 315, 1356, 775, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[8, 7, 1298, 3, 199, 0, 1500, 0, 3348, 2122, 90, 0, 0, 0, 0, 0, 3157, 84, 80, 2755, 1056, 384, 0, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 1203, 920, 415, 599, 1742, 1300, 265, 1515, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2602, 19, 1067, 614, 13, 320, 1939, 1206, 783, 0, 0, 0, 0, 0, 0, 683, 4, 49, 2019, 2603, 25, 629, 1718, 0, 0, 0, 0, 0, 0, 0, 0, 1744, 416, 1043, 804, 0, 867, 1244, 1969, 0, 0, 0, 0, 0, 0, 0, 0, 464, 3874, 427, 2, 1, 8, 682, 458, 1662, 448, 0, 0, 0, 0, 0, 0, 0, 1353, 3102, 570, 283, 1023, 1201, 29, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8575, 9033, 7959, 8565, 7730, 8087, 8026, 7561] avg=8192.0000 max=9033 max_rank=1 min=7561 min_rank=7 imbalance_ratio=1.102661 max_load_reduction=56503 max_load_reduction_pct=0.862167 moved_to_echo=31404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 27, 1042, 8, 243, 2, 999, 9, 3387, 2170, 92, 0, 0, 0, 0, 0, 2531, 69, 43, 1875, 964, 510, 0, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 1625, 643, 275, 1762, 1410, 214, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1514, 47, 1125, 698, 13, 236, 1788, 1295, 792, 0, 0, 0, 0, 0, 0, 692, 14, 83, 1790, 3371, 11, 691, 2224, 0, 0, 0, 0, 0, 0, 0, 0, 1873, 341, 518, 478, 0, 895, 1164, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 107, 3589, 377, 5, 1, 3, 1141, 627, 1701, 481, 0, 0, 0, 0, 0, 0, 1, 1383, 3251, 1125, 385, 1097, 1492, 41, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7984, 7545, 8909, 7508, 8876, 7907, 8032, 8775] avg=8192.0000 max=8909 max_rank=2 min=7508 min_rank=3 imbalance_ratio=1.087524 max_load_reduction=56627 max_load_reduction_pct=0.864059 moved_to_echo=33590 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 6, 1085, 0, 228, 0, 1134, 0, 3379, 1806, 86, 0, 0, 0, 0, 0, 2228, 56, 98, 2615, 1024, 462, 0, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 1555, 1193, 733, 437, 1706, 1470, 193, 1353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 35, 1111, 461, 6, 67, 1457, 1290, 767, 0, 0, 0, 0, 0, 0, 690, 10, 61, 1903, 2062, 7, 636, 2054, 0, 0, 0, 0, 0, 0, 0, 0, 1866, 326, 499, 337, 0, 1029, 1294, 2920, 0, 0, 0, 0, 0, 0, 0, 0, 142, 3864, 480, 0, 0, 0, 1141, 1204, 1594, 480, 0, 0, 0, 0, 0, 0, 1, 1151, 3378, 1185, 459, 1515, 1169, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7735, 7938, 8640, 7758, 7423, 8271, 8905, 8866] avg=8192.0000 max=8905 max_rank=6 min=7423 min_rank=4 imbalance_ratio=1.087036 max_load_reduction=56631 max_load_reduction_pct=0.864120 moved_to_echo=33465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 7, 1024, 5, 210, 2, 1437, 1, 3399, 2019, 98, 0, 0, 0, 0, 0, 2606, 70, 49, 1984, 857, 440, 0, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 1279, 866, 855, 377, 1484, 1181, 248, 1547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2226, 6, 1260, 442, 8, 196, 1970, 1205, 667, 0, 0, 0, 0, 0, 0, 694, 3, 45, 1792, 3313, 11, 532, 2195, 0, 0, 0, 0, 0, 0, 0, 0, 1743, 471, 690, 446, 1, 1048, 1736, 2753, 0, 0, 0, 0, 0, 0, 0, 0, 211, 3811, 511, 0, 4, 7, 797, 576, 1811, 448, 0, 0, 0, 0, 0, 0, 2, 1287, 3206, 1026, 516, 1229, 934, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 7659, 7837, 7980, 8585, 8888, 8176, 8206] avg=8192.0000 max=8888 max_rank=5 min=7659 min_rank=1 imbalance_ratio=1.084961 max_load_reduction=56648 max_load_reduction_pct=0.864380 moved_to_echo=33855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 20, 961, 0, 214, 0, 1210, 0, 3396, 2146, 87, 0, 0, 0, 0, 0, 2604, 39, 77, 1961, 1305, 582, 0, 1467, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 1212, 332, 502, 1902, 1293, 254, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2525, 32, 1192, 415, 20, 284, 1891, 1324, 856, 0, 0, 0, 0, 0, 0, 693, 60, 64, 1623, 2708, 0, 1229, 2117, 0, 0, 0, 0, 0, 0, 0, 0, 1914, 514, 547, 468, 5, 1628, 1096, 2331, 0, 0, 0, 0, 0, 0, 0, 0, 104, 3775, 529, 0, 1, 15, 888, 522, 1606, 492, 0, 0, 0, 0, 0, 0, 0, 1367, 2717, 1087, 315, 1726, 753, 20, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8041, 8035, 8007, 8539, 8494, 8503, 7932, 7985] avg=8192.0000 max=8539 max_rank=3 min=7932 min_rank=6 imbalance_ratio=1.042358 max_load_reduction=56997 max_load_reduction_pct=0.869705 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 21, 1135, 2, 148, 0, 1421, 2, 3380, 2159, 82, 0, 0, 0, 0, 0, 2777, 39, 58, 2240, 816, 580, 0, 1386, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 1698, 310, 182, 1687, 1200, 237, 1583, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2767, 3, 987, 755, 8, 160, 1727, 1182, 759, 0, 0, 0, 0, 0, 0, 690, 3, 103, 2038, 2318, 4, 412, 2507, 0, 0, 0, 0, 0, 0, 0, 0, 1710, 423, 452, 836, 4, 1079, 1087, 2567, 0, 0, 0, 0, 0, 0, 0, 0, 153, 3927, 519, 0, 1, 3, 902, 657, 1519, 439, 0, 0, 0, 0, 0, 0, 2, 1376, 3299, 1046, 222, 1252, 1363, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8354, 7896, 8019, 8350, 8075, 8158, 8120, 8564] avg=8192.0000 max=8564 max_rank=7 min=7896 min_rank=1 imbalance_ratio=1.045410 max_load_reduction=56972 max_load_reduction_pct=0.869324 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 26, 1250, 2, 175, 1, 1157, 24, 3397, 2183, 93, 0, 0, 0, 0, 0, 2843, 87, 58, 2689, 1057, 779, 3, 1557, 0, 0, 0, 0, 0, 0, 0, 0, 1692, 1207, 520, 518, 1843, 1270, 166, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2350, 81, 1144, 745, 3, 193, 1788, 1218, 829, 0, 0, 0, 0, 0, 0, 694, 10, 60, 1493, 3260, 6, 620, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 1762, 334, 606, 651, 0, 1011, 1048, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 129, 3558, 551, 1, 24, 6, 1187, 710, 1706, 453, 0, 0, 0, 0, 0, 0, 0, 1392, 2729, 476, 315, 1356, 775, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8313, 9073, 8630, 8356, 8215, 7579, 8325, 7045] avg=8192.0000 max=9073 max_rank=1 min=7045 min_rank=7 imbalance_ratio=1.107544 max_load_reduction=56463 max_load_reduction_pct=0.861557 moved_to_echo=31164 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 884, 3, 106, 0, 1660, 0, 3399, 2177, 96, 0, 0, 0, 0, 0, 2909, 106, 66, 2221, 1127, 404, 0, 1524, 0, 0, 0, 0, 0, 0, 0, 0, 965, 1526, 328, 231, 1720, 1128, 122, 1515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2921, 6, 911, 639, 1, 284, 1643, 1293, 782, 0, 0, 0, 0, 0, 0, 689, 19, 49, 2197, 2578, 5, 326, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 164, 504, 580, 6, 1248, 1166, 2622, 0, 0, 0, 0, 0, 0, 0, 0, 64, 3869, 390, 0, 2, 0, 948, 580, 1683, 484, 0, 0, 0, 0, 0, 0, 0, 1381, 3344, 1079, 259, 1625, 842, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8329, 8357, 7535, 8480, 8138, 8143, 8020, 8534] avg=8192.0000 max=8534 max_rank=7 min=7535 min_rank=2 imbalance_ratio=1.041748 max_load_reduction=57002 max_load_reduction_pct=0.869781 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[14592, 3072, 4864, 4608, 128, 8832, 9856, 19968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 19584, 256, 8832, 4864, 128, 1792, 14208, 10112, 6272, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5632, 128, 640, 14976, 22272, 128, 5120, 17280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[10496, 10368, 4224, 3200, 13952, 10368, 1792, 11904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[21760, 640, 640, 18432, 8320, 4224, 128, 12160, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 10752, 25088, 7680, 2816, 10880, 8576, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1408, 30336, 3840, 128, 128, 128, 7808, 5376, 13312, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 8704, 128, 1536, 128, 10624, 128, 27136, 16896, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 0, 4096, 0, 2755, 0, 4077, 0, 1, 3230, 0, 4, 0, 0, 4090, 0, 12, 0, 2, 0, 0, 4096, 0, 4096, 3, 0, 1, 4096, 2209, 8, 0, 0, 14, 4095, 4094, 3, 0, 0, 4096, 4081, 0, 0, 0, 0, 4089, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4093, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 4, 0, 0, 0, 0, 4092, 0, 3457, 0, 4092, 6, 6, 2984, 2, 2, 0, 2, 4077, 2, 0, 0, 0, 0, 0, 4090, 5, 4094, 17, 3, 7, 4090, 1737, 11, 0, 0, 16, 4082, 4081, 12, 0, 0, 4095, 4083, 0, 5, 0, 2, 4082, 4095, 4, 0, 0, 0, 0, 0, 7, 0, 4095, 0, 4094, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 2, 2, 6, 4091, 3, 3439, 23, 4069, 2, 45, 3058, 0, 3, 0, 2, 4078, 1, 0, 2, 0, 11, 7, 4091, 19, 4093, 1, 0, 1, 4058, 1682, 1, 0, 0, 0, 4094, 4085, 2, 0, 2, 4094, 4076, 0, 2, 21, 0, 4094, 4089, 0, 0, 0, 0, 0, 2, 1, 0, 4082, 2, 4096, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 2378, 0, 4073, 0, 6, 3607, 7, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 33, 0, 0, 4096, 2185, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 4093, 4, 2899, 1, 4089, 5, 16, 2856, 1, 2, 0, 0, 4093, 2, 0, 0, 4, 2, 3, 4093, 0, 4092, 206, 1, 0, 4078, 2233, 0, 0, 3, 1, 4081, 4085, 0, 0, 0, 4091, 4092, 0, 2, 1, 1, 4089, 4094, 10, 4, 3, 0, 4, 3, 4, 2, 4095, 1, 4093, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 4, 3657, 2, 4095, 0, 12, 3938, 1, 36, 0, 0, 4062, 0, 4, 0, 0, 4, 1, 4096, 0, 4096, 3, 0, 0, 4095, 558, 12, 0, 0, 0, 4089, 4096, 1, 0, 0, 4096, 4096, 0, 0, 8, 0, 4096, 4089, 0, 0, 0, 1, 0, 0, 0, 2, 4096, 0, 4094, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 0, 1, 1, 4093, 0, 2942, 0, 4089, 0, 14, 3125, 0, 0, 0, 0, 4095, 0, 1, 0, 6, 2, 1, 4096, 0, 4094, 23, 1, 0, 4095, 2109, 3, 2, 0, 1, 4081, 4095, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 3, 0, 0, 4095, 0, 4086, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 1, 0, 0, 4096, 1, 3532, 0, 4065, 0, 0, 3912, 0, 0, 0, 0, 4096, 0, 3, 0, 3, 10, 0, 4096, 0, 4096, 205, 0, 0, 4089, 572, 4, 0, 0, 2, 4096, 4096, 2, 0, 0, 4096, 4096, 0, 0, 0, 0, 4084, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 0, 4096, 2374, 1657, 61, 0, 0, 0, 0, 0, 0, 2755, 0, 1700, 0, 1, 3230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4090, 0, 12, 0, 2, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4030, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2209, 8, 0, 0, 14, 2438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 3, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7686, 8205, 8130, 8765, 8180, 8192, 8188] avg=8192.0000 max=8765 max_rank=4 min=7686 min_rank=1 imbalance_ratio=1.069946 max_load_reduction=56771 max_load_reduction_pct=0.866257 moved_to_echo=33325 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 2, 2, 6, 4091, 2370, 1656, 61, 0, 0, 0, 0, 0, 3, 3439, 23, 1696, 2, 45, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 4078, 1, 0, 2, 0, 4085, 3, 0, 0, 0, 0, 0, 0, 11, 7, 4091, 19, 4027, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 1682, 1, 0, 0, 0, 2438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4094, 4076, 0, 2, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4089, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 4082, 2, 4096, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8266, 8174, 8157, 8179, 8197, 8187, 8187] avg=8192.0000 max=8266 max_rank=1 min=8157 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 2372, 1657, 61, 0, 0, 0, 0, 0, 0, 2378, 0, 1698, 0, 6, 3607, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4030, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2185, 1, 0, 0, 0, 2439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 7696, 8196, 8159, 8721, 8191, 8195, 8192] avg=8192.0000 max=8721 max_rank=4 min=7696 min_rank=1 imbalance_ratio=1.064575 max_load_reduction=56815 max_load_reduction_pct=0.866928 moved_to_echo=33299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 4093, 2381, 1651, 61, 0, 0, 0, 0, 0, 4, 2899, 1, 1705, 5, 16, 2856, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 2, 0, 0, 4, 4085, 3, 0, 0, 0, 0, 0, 0, 2, 3, 4093, 0, 4026, 206, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 2233, 0, 0, 3, 1, 2430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4092, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 4094, 10, 4, 3, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 3, 4, 2, 4095, 1, 4093, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 7487, 8189, 8331, 8745, 8186, 8209, 8200] avg=8192.0000 max=8745 max_rank=4 min=7487 min_rank=1 imbalance_ratio=1.067505 max_load_reduction=56791 max_load_reduction_pct=0.866562 moved_to_echo=33340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 2385, 1654, 61, 0, 0, 0, 0, 0, 4, 3657, 2, 1707, 0, 12, 3938, 1, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4062, 0, 4, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4, 1, 4096, 0, 4030, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 558, 12, 0, 0, 0, 2435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 9321, 8201, 8134, 7100, 8201, 8190, 8193] avg=8192.0000 max=9321 max_rank=1 min=7100 min_rank=4 imbalance_ratio=1.137817 max_load_reduction=56215 max_load_reduction_pct=0.857773 moved_to_echo=31684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4, 0, 0, 0, 0, 4092, 2383, 1651, 61, 0, 0, 0, 0, 0, 0, 3457, 0, 1706, 6, 6, 2984, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4077, 2, 0, 0, 0, 4081, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 5, 4028, 17, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1737, 11, 0, 0, 16, 2431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4095, 4083, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4082, 4095, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4095, 0, 4094, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8161, 8167, 8150, 8285, 8195, 8187, 8198] avg=8192.0000 max=8285 max_rank=4 min=8150 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 0, 1, 1, 4093, 2381, 1651, 61, 0, 0, 0, 0, 0, 0, 2942, 0, 1705, 0, 14, 3125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 6, 4095, 3, 0, 0, 0, 0, 0, 0, 2, 1, 4096, 0, 4028, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2109, 3, 2, 0, 1, 2430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4095, 0, 4086, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7786, 8200, 8151, 8640, 8188, 8194, 8186] avg=8192.0000 max=8640 max_rank=4 min=7786 min_rank=1 imbalance_ratio=1.054688 max_load_reduction=56896 max_load_reduction_pct=0.868164 moved_to_echo=33208 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 1, 0, 0, 4096, 2371, 1662, 68, 0, 0, 0, 0, 0, 1, 3532, 0, 1688, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 3, 4096, 6, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 4013, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 572, 4, 0, 0, 2, 2434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4089, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 9133, 8204, 8324, 7101, 8198, 8182, 8192] avg=8192.0000 max=9133 max_rank=1 min=7101 min_rank=4 imbalance_ratio=1.114868 max_load_reduction=56403 max_load_reduction_pct=0.860641 moved_to_echo=31673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 13312, 128, 128, 128, 128, 19584, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 32768, 19072, 13312, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 25088, 128, 13696, 128, 128, 26752, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 32768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 32256, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3906, 4096, 0, 0, 0, 4091, 431, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4081, 6, 1, 4094, 0, 0, 0, 0, 4096, 4095, 0, 4089, 0, 2220, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 1645, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4071, 4096, 1, 0, 7, 4086, 1106, 4095, 0, 2, 0, 0, 1, 0, 0, 0, 15, 0, 0, 1, 4090, 0, 4094, 2, 0, 0, 0, 0, 1, 0, 1, 0, 4025, 8, 33, 4091, 0, 0, 3, 0, 4092, 4096, 0, 3777, 1, 2923, 0, 3, 4083, 0, 0, 4092, 0, 4, 4095, 0, 0, 0, 4093, 0, 448, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4043, 4096, 0, 0, 15, 4095, 479, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 1, 4033, 20, 4, 4094, 0, 0, 0, 2, 4093, 4096, 0, 4096, 0, 2803, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 892, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2715, 4094, 0, 0, 1, 4076, 345, 4095, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 2, 0, 4038, 9, 0, 4093, 2, 0, 0, 2, 4096, 4092, 0, 3859, 0, 3038, 1, 5, 4085, 0, 0, 4095, 0, 0, 4095, 0, 0, 1, 4086, 0, 2415, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 3331, 4091, 0, 1, 17, 4051, 515, 4091, 0, 6, 4, 5, 0, 0, 0, 0, 0, 4, 1, 0, 4094, 0, 4092, 0, 0, 0, 4, 1, 0, 0, 2, 0, 4016, 29, 0, 4089, 0, 2, 0, 0, 4085, 4092, 1, 3911, 5, 1520, 0, 8, 4093, 2, 0, 4094, 4, 7, 4092, 3, 0, 8, 4089, 1, 3074, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4012, 4096, 0, 0, 4, 4094, 1350, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 1, 4089, 0, 0, 0, 0, 4096, 4096, 0, 3979, 0, 2073, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 880, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4041, 4096, 0, 0, 3, 4096, 192, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 16, 43, 4095, 2, 0, 0, 0, 4096, 4096, 0, 3741, 0, 3391, 0, 0, 4084, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 0, 895, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4090, 4096, 0, 0, 0, 4095, 362, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4056, 13, 1, 4085, 0, 0, 0, 0, 4096, 4096, 0, 4070, 0, 3057, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 4096, 0, 739, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3906, 325, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 431, 4092, 0, 0, 0, 0, 0, 0, 2149, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3771, 324, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4081, 6, 1, 4094, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1623, 0, 0, 4089, 0, 2220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 0, 0, 4096, 0, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8322, 8022, 8191, 8191, 8219, 7932, 8193, 8466] avg=8192.0000 max=8466 max_rank=7 min=7932 min_rank=5 imbalance_ratio=1.033447 max_load_reduction=57070 max_load_reduction_pct=0.870819 moved_to_echo=32810 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3331, 325, 0, 1, 17, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 515, 4091, 0, 6, 4, 5, 0, 0, 2143, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 4094, 0, 3766, 323, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4016, 29, 0, 4089, 0, 2, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1619, 0, 1, 3911, 5, 1520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4093, 2, 0, 4094, 4, 7, 1, 0, 0, 0, 0, 0, 0, 0, 2723, 3, 0, 8, 4089, 1, 3074, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7726, 8112, 8188, 8189, 8158, 7056, 8209, 9898] avg=8192.0000 max=9898 max_rank=7 min=7056 min_rank=5 imbalance_ratio=1.208252 max_load_reduction=55638 max_load_reduction_pct=0.848969 moved_to_echo=33321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4071, 325, 1, 0, 7, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 1106, 4095, 0, 2, 0, 0, 1, 0, 2147, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1, 4090, 0, 3771, 323, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4025, 8, 33, 4091, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1622, 0, 0, 3777, 1, 2923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4083, 0, 0, 4092, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 0, 0, 4093, 0, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8490, 8700, 8200, 8193, 8178, 8326, 8183, 7266] avg=8192.0000 max=8700 max_rank=1 min=7266 min_rank=7 imbalance_ratio=1.062012 max_load_reduction=56836 max_load_reduction_pct=0.867249 moved_to_echo=31953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4012, 325, 0, 0, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1350, 4094, 0, 0, 0, 0, 0, 0, 2149, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3771, 324, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 1, 4089, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1623, 0, 0, 3979, 0, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 0, 0, 4096, 0, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8435, 8943, 8191, 8192, 8207, 7675, 8192, 7701] avg=8192.0000 max=8943 max_rank=1 min=7675 min_rank=5 imbalance_ratio=1.091675 max_load_reduction=56593 max_load_reduction_pct=0.863541 moved_to_echo=31775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2715, 325, 0, 0, 1, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 345, 4095, 1, 1, 1, 0, 0, 0, 2149, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3769, 324, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4038, 9, 0, 4093, 2, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1623, 0, 0, 3859, 0, 3038, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 4085, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 0, 1, 4086, 0, 2415, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7117, 7941, 8189, 8189, 8164, 8522, 8187, 9227] avg=8192.0000 max=9227 max_rank=7 min=7117 min_rank=0 imbalance_ratio=1.126343 max_load_reduction=56309 max_load_reduction_pct=0.859207 moved_to_echo=34100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4043, 325, 0, 0, 15, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 479, 4095, 0, 0, 0, 0, 0, 0, 2147, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3771, 323, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4033, 20, 4, 4094, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1623, 0, 0, 4096, 0, 2803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 0, 0, 4096, 0, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8478, 8070, 8190, 8196, 8172, 8524, 8193, 7713] avg=8192.0000 max=8524 max_rank=5 min=7713 min_rank=7 imbalance_ratio=1.040527 max_load_reduction=57012 max_load_reduction_pct=0.869934 moved_to_echo=32602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4041, 325, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 192, 4094, 0, 0, 0, 0, 0, 0, 2149, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3771, 324, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 16, 43, 4095, 2, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1623, 0, 0, 3741, 0, 3391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 0, 0, 4096, 0, 895, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8465, 7785, 8191, 8190, 8250, 8755, 8184, 7716] avg=8192.0000 max=8755 max_rank=5 min=7716 min_rank=7 imbalance_ratio=1.068726 max_load_reduction=56781 max_load_reduction_pct=0.866409 moved_to_echo=32905 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 318, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 362, 4095, 0, 0, 0, 0, 0, 0, 2151, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3774, 326, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 8, 0, 4056, 13, 1, 4085, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1619, 0, 0, 4070, 0, 3057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2713, 0, 1, 0, 4096, 0, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8503, 7963, 8196, 8196, 8188, 8746, 8195, 7549] avg=8192.0000 max=8746 max_rank=5 min=7549 min_rank=7 imbalance_ratio=1.067627 max_load_reduction=56790 max_load_reduction_pct=0.866547 moved_to_echo=32678 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 128, 32768, 0, 30208, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 30336, 2688, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 13056, 0, 128, 31616, 128, 21120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[21888, 128, 128, 128, 32768, 128, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4864, 32768, 128, 128, 128, 128, 128, 0, 17280, 10880, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32512, 128, 128, 32768, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 3374, 2, 0, 0, 4093, 0, 4078, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3738, 4096, 268, 4096, 0, 0, 6, 4096, 4096, 0, 877, 15, 4096, 0, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4088, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 3992, 0, 0, 4, 3762, 2, 0, 0, 4096, 0, 4088, 0, 1, 4070, 0, 0, 1, 0, 8, 0, 4088, 1, 4089, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3848, 4096, 0, 4096, 0, 0, 7, 4095, 4095, 2, 715, 6, 4093, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 3505, 1, 0, 0, 4093, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 3590, 4096, 48, 4096, 0, 0, 1, 4096, 4096, 0, 1055, 22, 4096, 0, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 3780, 14, 0, 0, 4068, 0, 4093, 0, 0, 4081, 0, 0, 0, 0, 11, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2525, 4096, 287, 4096, 0, 0, 24, 4096, 4096, 0, 1279, 323, 4096, 0, 0, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 4, 0, 8, 4036, 0, 5, 16, 4092, 21, 1, 3, 4, 0, 4091, 0, 0, 0, 3343, 0, 6, 14, 3950, 6, 3958, 0, 0, 4063, 2, 4, 5, 0, 0, 4, 4088, 5, 4091, 0, 1, 0, 0, 0, 4, 4088, 0, 0, 3823, 4090, 113, 4082, 4, 5, 0, 4091, 4091, 2, 1212, 0, 4091, 8, 0, 11, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3823, 8, 0, 0, 3994, 0, 4094, 0, 0, 4075, 0, 0, 5, 0, 16, 0, 4079, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 3984, 4096, 47, 4095, 0, 0, 24, 4096, 4096, 0, 414, 19, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4076, 0, 0, 0, 3669, 21, 0, 0, 4019, 0, 4094, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 4069, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3166, 4096, 174, 4096, 0, 0, 26, 4096, 4096, 0, 1234, 21, 4096, 0, 0, 44, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4075, 11, 0, 0, 4073, 0, 4051, 0, 0, 4073, 0, 0, 0, 0, 10, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3672, 4096, 59, 4096, 0, 0, 4, 4096, 4096, 0, 429, 4, 4096, 0, 0, 28, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4033, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3374, 2, 0, 0, 4093, 0, 405, 60, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 4094, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4091, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3738, 555, 0, 0, 0, 0, 0, 0, 0, 0, 268, 4096, 0, 0, 6, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 877, 15, 4096, 0, 0, 2, 0, 0, 3131, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8152, 7934, 8218, 8193, 8389, 8333, 8121] avg=8192.0000 max=8389 max_rank=5 min=7934 min_rank=2 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=33036 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4077, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3505, 1, 0, 0, 4093, 0, 405, 60, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 3590, 555, 0, 0, 0, 0, 0, 0, 0, 0, 48, 4096, 0, 0, 1, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 22, 4096, 0, 0, 2, 0, 0, 3131, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8196, 8064, 8230, 8196, 8242, 8108, 8306] avg=8192.0000 max=8306 max_rank=7 min=8064 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32852 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 8, 4036, 0, 5, 16, 4091, 5, 0, 0, 0, 0, 0, 0, 4092, 21, 1, 3, 4, 0, 4091, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3343, 0, 6, 14, 3950, 6, 404, 60, 0, 0, 0, 0, 0, 0, 3958, 0, 0, 4063, 2, 4, 5, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4088, 5, 4091, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4088, 0, 0, 3823, 554, 0, 0, 0, 0, 0, 0, 0, 0, 113, 4082, 4, 5, 0, 3958, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1212, 0, 4091, 8, 0, 11, 0, 0, 3127, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8235, 7783, 8078, 8193, 8469, 8164, 8449] avg=8192.0000 max=8469 max_rank=5 min=7783 min_rank=2 imbalance_ratio=1.033813 max_load_reduction=57067 max_load_reduction_pct=0.870773 moved_to_echo=33275 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4088, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 3992, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3762, 2, 0, 0, 4096, 0, 405, 60, 0, 0, 0, 0, 0, 0, 4088, 0, 1, 4070, 0, 0, 1, 0, 46, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4088, 1, 4089, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3848, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 3962, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 715, 6, 4093, 0, 0, 0, 0, 0, 3131, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8115, 8329, 8206, 8190, 8495, 8067, 7945] avg=8192.0000 max=8495 max_rank=5 min=7945 min_rank=7 imbalance_ratio=1.036987 max_load_reduction=57041 max_load_reduction_pct=0.870377 moved_to_echo=32697 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4076, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3669, 21, 0, 0, 4019, 0, 405, 60, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4053, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3166, 555, 0, 0, 0, 0, 0, 0, 0, 0, 174, 4096, 0, 0, 26, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 21, 4096, 0, 0, 44, 0, 4, 3131, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8197, 8174, 8193, 8169, 7817, 8259, 8530] avg=8192.0000 max=8530 max_rank=7 min=7817 min_rank=5 imbalance_ratio=1.041260 max_load_reduction=57006 max_load_reduction_pct=0.869843 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4092, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 8, 0, 0, 3994, 0, 405, 60, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4075, 0, 0, 5, 0, 46, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4079, 0, 4096, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3984, 555, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4095, 0, 0, 24, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 19, 4096, 0, 0, 0, 0, 0, 3131, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8209, 8290, 8220, 8196, 8635, 8129, 7660] avg=8192.0000 max=8635 max_rank=5 min=7660 min_rank=7 imbalance_ratio=1.054077 max_load_reduction=56901 max_load_reduction_pct=0.868240 moved_to_echo=32620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3780, 14, 0, 0, 4068, 0, 405, 60, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4081, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4088, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2525, 555, 0, 0, 0, 0, 0, 0, 0, 0, 287, 4096, 0, 0, 24, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1279, 323, 4096, 0, 0, 1, 0, 1, 3131, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8212, 8328, 8220, 8199, 7176, 8370, 8831] avg=8192.0000 max=8831 max_rank=7 min=7176 min_rank=5 imbalance_ratio=1.078003 max_load_reduction=56705 max_load_reduction_pct=0.865250 moved_to_echo=32576 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 11, 0, 0, 4073, 0, 409, 66, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 4073, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4085, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3672, 545, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4096, 0, 0, 4, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 4, 4096, 0, 0, 28, 0, 1, 3136, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8220, 8634, 8171, 8200, 8313, 8106, 7694] avg=8192.0000 max=8634 max_rank=2 min=7694 min_rank=7 imbalance_ratio=1.053955 max_load_reduction=56902 max_load_reduction_pct=0.868256 moved_to_echo=32313 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7296, 512, 32768, 128, 0, 128, 0, 128, 25088, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 29440, 128, 128, 128, 32512, 128, 3328, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 32640, 0, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 0, 128, 32640, 128, 128, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 28416, 4480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1024, 32768, 128, 128, 128, 31744, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3673, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4089, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 5, 0, 4096, 0, 74, 4096, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 232, 0, 82, 0, 4096, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[7, 0, 2785, 4090, 0, 3, 5, 5, 4092, 1, 4086, 5, 0, 5, 3, 0, 0, 4091, 2, 6, 3973, 0, 7, 0, 5, 4091, 0, 3, 4091, 6, 0, 4087, 4087, 4089, 0, 0, 4090, 15, 71, 4092, 8, 5, 0, 6, 4090, 4, 6, 2, 0, 1, 4, 0, 4078, 0, 0, 6, 1319, 5, 3, 0, 4091, 4, 6, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3797, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4027, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 23, 4096, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 318, 0, 11, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 3508, 4096, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4084, 0, 3, 0, 0, 4092, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 11, 0, 4096, 0, 119, 4082, 18, 0, 0, 0, 4096, 0, 0, 0, 0, 17, 0, 0, 4096, 0, 0, 0, 451, 0, 15, 0, 4095, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4028, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3966, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 1, 6, 4094, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 135, 0, 52, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3436, 4096, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 3, 0, 4095, 1, 74, 4094, 45, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 535, 0, 8, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3573, 4096, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 4087, 0, 5, 0, 0, 4094, 1, 0, 4096, 1, 0, 4096, 4096, 4096, 2, 3, 4093, 0, 22, 4088, 181, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 4095, 0, 0, 0, 169, 0, 154, 0, 4096, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3916, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 4088, 0, 0, 4090, 0, 15, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 133, 0, 10, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3673, 4096, 0, 0, 0, 0, 503, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4089, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4049, 0, 74, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 0, 4049, 6, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 232, 0, 82, 0, 4096, 0, 6, 0, 3593, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8193, 8222, 8193, 8224, 8179, 8196, 8056] avg=8192.0000 max=8273 max_rank=0 min=8056 min_rank=7 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32655 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 2785, 4090, 0, 3, 5, 5, 501, 1, 0, 0, 0, 0, 0, 0, 4092, 1, 4086, 5, 0, 5, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 6, 3973, 0, 7, 0, 44, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 0, 3, 1, 6, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 15, 71, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 6, 4090, 4, 6, 2, 4041, 6, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4078, 0, 0, 6, 4084, 0, 0, 0, 0, 0, 0, 0, 1319, 5, 3, 0, 4091, 4, 6, 5, 3586, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7397, 8194, 8123, 8193, 8221, 8168, 8173, 9067] avg=8192.0000 max=9067 max_rank=7 min=7397 min_rank=0 imbalance_ratio=1.106812 max_load_reduction=56469 max_load_reduction_pct=0.861649 moved_to_echo=33629 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3797, 4096, 0, 0, 0, 0, 503, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4027, 0, 1, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 4049, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 318, 0, 11, 0, 4096, 0, 0, 0, 3593, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8397, 8194, 8168, 8192, 8168, 8161, 8191, 8065] avg=8192.0000 max=8397 max_rank=0 min=8065 min_rank=7 imbalance_ratio=1.025024 max_load_reduction=57139 max_load_reduction_pct=0.871872 moved_to_echo=32585 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3508, 4096, 0, 0, 0, 0, 503, 1, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4084, 0, 3, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4049, 0, 119, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 0, 0, 4049, 6, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 451, 0, 15, 0, 4095, 0, 3, 0, 3593, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8184, 8218, 8188, 8261, 8169, 8203, 8204] avg=8192.0000 max=8261 max_rank=4 min=8109 min_rank=0 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4028, 4096, 0, 0, 0, 0, 503, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3966, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 1, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 4049, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 135, 0, 52, 0, 4096, 0, 0, 0, 3593, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8628, 8194, 8104, 8194, 8150, 8156, 8187, 7923] avg=8192.0000 max=8628 max_rank=0 min=7923 min_rank=7 imbalance_ratio=1.053223 max_load_reduction=56908 max_load_reduction_pct=0.868347 moved_to_echo=32416 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3436, 4096, 0, 0, 0, 0, 503, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4095, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4048, 1, 74, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 4096, 0, 0, 0, 4049, 6, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 535, 0, 8, 0, 4096, 0, 0, 0, 3593, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8036, 8188, 8232, 8192, 8220, 8196, 8193, 8279] avg=8192.0000 max=8279 max_rank=7 min=8036 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32888 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3573, 4096, 0, 0, 0, 0, 503, 1, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 4087, 0, 5, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4046, 0, 22, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 4096, 0, 0, 0, 4049, 6, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4095, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 169, 0, 154, 0, 4096, 0, 4, 0, 3593, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8189, 8230, 8192, 8161, 8332, 8196, 8063] avg=8192.0000 max=8332 max_rank=5 min=8063 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32752 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3916, 4096, 0, 0, 0, 0, 503, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4090, 0, 15, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 4049, 7, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 133, 0, 10, 0, 4096, 0, 0, 0, 3593, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8523, 8200, 8239, 8192, 8131, 8175, 8197, 7879] avg=8192.0000 max=8523 max_rank=0 min=7879 min_rank=7 imbalance_ratio=1.040405 max_load_reduction=57013 max_load_reduction_pct=0.869949 moved_to_echo=32382 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 28800, 32768, 0, 128, 128, 128, 4096, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 32512, 0, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 128, 0, 128, 32768, 128, 128, 128, 32384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 32384, 128, 512, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3328, 128, 384, 0, 32768, 128, 128, 128, 28800, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32768, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 21, 3153, 4096, 0, 4096, 0, 153, 0, 0, 2, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 0, 0, 0, 4089, 0, 4096, 0, 4096, 4096, 961, 0, 0, 0, 0, 0, 3896, 0, 0, 4096, 0, 4076, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 5, 3027, 4096, 0, 4096, 0, 111, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4095, 1035, 0, 0, 0, 0, 0, 4019, 0, 0, 4096, 0, 4085, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 16, 2, 0, 0, 0, 6, 2738, 4093, 0, 4096, 0, 202, 0, 0, 9, 0, 0, 0, 4092, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 0, 4096, 0, 4096, 4096, 1409, 0, 0, 0, 0, 0, 3845, 0, 0, 4096, 0, 4079, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4, 2, 0, 0, 0, 0, 2149, 4096, 0, 4096, 0, 201, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 0, 4096, 0, 4096, 4096, 1926, 0, 0, 0, 0, 0, 3927, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2519, 4096, 0, 4096, 0, 98, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 1578, 0, 0, 0, 0, 0, 3996, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[6, 0, 0, 1, 6, 4, 0, 2166, 4091, 0, 4083, 6, 0, 0, 0, 0, 0, 7, 0, 4088, 0, 9, 4090, 6, 4090, 13, 0, 0, 4091, 2, 4088, 8, 4090, 4090, 1924, 4, 0, 0, 11, 0, 4084, 1, 0, 4090, 0, 4092, 0, 4087, 1, 4, 0, 4089, 0, 5, 1, 6, 10, 5, 7, 0, 4077, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 5, 0, 0, 0, 0, 2210, 4096, 0, 4096, 0, 114, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 5, 3, 4088, 0, 4096, 0, 4096, 4094, 1834, 0, 0, 0, 0, 0, 4021, 0, 0, 4096, 0, 4090, 0, 4093, 0, 0, 0, 4092, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2404, 4096, 0, 4096, 0, 71, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4095, 1715, 0, 0, 0, 0, 0, 4003, 0, 0, 4096, 0, 4089, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 2, 0, 0, 0, 0, 2149, 4096, 1541, 0, 0, 0, 0, 0, 0, 3979, 0, 4096, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2549, 4096, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 0, 4095, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2555, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 2422, 1547, 116, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7792, 8276, 8179, 8190, 8571, 8153, 8193, 8182] avg=8192.0000 max=8571 max_rank=4 min=7792 min_rank=0 imbalance_ratio=1.046265 max_load_reduction=56965 max_load_reduction_pct=0.869217 moved_to_echo=33099 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 1, 6, 4, 0, 2166, 4090, 1538, 0, 0, 0, 0, 0, 0, 3974, 0, 4083, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4088, 0, 9, 4090, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4091, 2, 4088, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 4090, 1924, 4, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 0, 0, 0, 4092, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4089, 0, 5, 1, 6, 2552, 1539, 0, 0, 0, 0, 0, 0, 10, 5, 7, 0, 4077, 0, 3, 0, 2417, 1545, 116, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7811, 8063, 8201, 8202, 8574, 8308, 8197, 8180] avg=8192.0000 max=8574 max_rank=4 min=7811 min_rank=0 imbalance_ratio=1.046631 max_load_reduction=56962 max_load_reduction_pct=0.869171 moved_to_echo=33259 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2519, 4096, 1540, 0, 0, 0, 0, 0, 0, 3979, 0, 4096, 0, 98, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2549, 4096, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 4096, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2556, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 2422, 1547, 116, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8174, 8192, 8192, 8223, 8223, 8195, 8182] avg=8192.0000 max=8223 max_rank=4 min=8155 min_rank=0 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 21, 3153, 4096, 1540, 0, 0, 0, 0, 0, 0, 3979, 0, 4096, 0, 153, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2549, 4096, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 0, 0, 4076, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2556, 1543, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 8, 2422, 1547, 116, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8810, 8230, 8190, 8185, 7606, 8103, 8195, 8217] avg=8192.0000 max=8810 max_rank=0 min=7606 min_rank=4 imbalance_ratio=1.075439 max_load_reduction=56726 max_load_reduction_pct=0.865570 moved_to_echo=32121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 2, 0, 0, 0, 6, 2738, 4096, 1541, 0, 0, 0, 0, 0, 0, 3976, 0, 4096, 0, 202, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2549, 4096, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 4079, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 2555, 1543, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 1, 2422, 1547, 116, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8399, 8283, 8189, 8191, 8054, 8055, 8182, 8183] avg=8192.0000 max=8399 max_rank=0 min=8054 min_rank=4 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 5, 0, 0, 0, 0, 2210, 4096, 1541, 0, 0, 0, 0, 0, 0, 3979, 0, 4096, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2549, 4094, 1834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 4090, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 2554, 1542, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 14, 2420, 1547, 116, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7857, 8189, 8195, 8192, 8477, 8242, 8188, 8196] avg=8192.0000 max=8477 max_rank=4 min=7857 min_rank=0 imbalance_ratio=1.034790 max_load_reduction=57059 max_load_reduction_pct=0.870651 moved_to_echo=33103 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 5, 3027, 4096, 1541, 0, 0, 0, 0, 0, 0, 3979, 0, 4096, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2549, 4095, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 4085, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 2555, 1543, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 9, 2422, 1547, 116, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8671, 8186, 8193, 8192, 7679, 8235, 8189, 8191] avg=8192.0000 max=8671 max_rank=0 min=7679 min_rank=4 imbalance_ratio=1.058472 max_load_reduction=56865 max_load_reduction_pct=0.867691 moved_to_echo=32294 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2404, 4096, 1541, 0, 0, 0, 0, 0, 0, 3968, 0, 4096, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2542, 4095, 1715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 4089, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2555, 1546, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 5, 2425, 1554, 123, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8041, 8135, 8197, 8192, 8352, 8217, 8197, 8205] avg=8192.0000 max=8352 max_rank=4 min=8041 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32971 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31872, 128, 0, 0, 0, 32768, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 20480, 32768, 12416, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[20480, 32768, 12416, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31872, 0, 32768, 128, 1024, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 128, 19456, 12416, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 0, 128, 128, 128, 20480, 12416, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[8, 0, 0, 5, 5, 6, 4084, 7, 1, 0, 25, 13, 0, 9, 2, 7, 0, 4086, 6, 4078, 1, 9, 0, 0, 0, 4085, 0, 4088, 7, 0, 5, 4086, 0, 2, 0, 8, 7, 0, 4090, 0, 3955, 4066, 0, 4082, 3, 6, 0, 4, 0, 4081, 7, 4, 0, 0, 3, 4089, 4089, 6, 7, 4088, 4089, 143, 4084, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 11, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4053, 4069, 45, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 79, 0, 0, 0, 8, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 3, 0, 4096, 0, 0, 3, 0, 0, 0, 4095, 0, 4013, 4046, 10, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 4095, 0, 0, 4093, 4090, 0, 4096, 40] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 60, 0, 0, 0, 236, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 3925, 4034, 42, 3964, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 26, 4096, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 191, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 3946, 4057, 31, 4032, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 4096, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 434, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 3949, 4050, 379, 3452, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4092, 2, 4096, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 58, 0, 0, 0, 145, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 3877, 4067, 93, 3951, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4085, 108, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 57, 0, 0, 0, 81, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 3981, 4072, 9, 4064, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 24, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 5, 5, 6, 4084, 7, 3880, 182, 37, 0, 0, 0, 0, 0, 1, 0, 25, 13, 0, 9, 2, 7, 4089, 3905, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 4078, 1, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 2, 7, 0, 5, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8, 7, 0, 4090, 0, 3904, 183, 0, 0, 0, 0, 0, 0, 3955, 186, 0, 4082, 3, 6, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 7, 4, 0, 0, 3, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 4089, 143, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8051, 8181, 8185, 8194, 8236, 8184, 8291] avg=8192.0000 max=8291 max_rank=7 min=8051 min_rank=1 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32905 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3883, 183, 38, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 11, 0, 4096, 3912, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3911, 184, 0, 0, 0, 0, 0, 0, 4053, 186, 45, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8037, 8192, 8194, 8191, 8377, 8192, 8153] avg=8192.0000 max=8377 max_rank=5 min=8037 min_rank=1 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32913 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3871, 183, 38, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 191, 0, 4096, 3913, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3913, 183, 0, 0, 0, 0, 0, 0, 3946, 186, 31, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4057, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8228, 8193, 8192, 8193, 8195, 8192, 8155] avg=8192.0000 max=8228 max_rank=1 min=8155 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32735 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3861, 183, 38, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 8, 0, 4095, 3910, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4095, 0, 3911, 183, 0, 0, 0, 0, 0, 0, 4013, 185, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4057, 40, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8092, 8192, 8195, 8192, 8304, 8196, 8187] avg=8192.0000 max=8304 max_rank=5 min=8092 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32879 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3849, 183, 38, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 236, 0, 4096, 3913, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3913, 183, 0, 0, 0, 0, 0, 0, 3925, 185, 42, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 4057, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8305, 8193, 8191, 8192, 8116, 8192, 8181] avg=8192.0000 max=8305 max_rank=1 min=8116 min_rank=5 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32681 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3881, 183, 38, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 145, 0, 4096, 3912, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3911, 184, 0, 0, 0, 0, 0, 0, 3877, 186, 93, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 108, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8211, 8193, 8194, 8191, 8107, 8192, 8250] avg=8192.0000 max=8250 max_rank=7 min=8107 min_rank=5 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32740 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3865, 183, 38, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 434, 0, 4096, 3913, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 3912, 183, 0, 0, 0, 0, 0, 0, 3949, 185, 379, 3452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 4057, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8465, 8192, 8193, 8193, 7965, 8192, 8154] avg=8192.0000 max=8465 max_rank=1 min=7965 min_rank=5 imbalance_ratio=1.033325 max_load_reduction=57071 max_load_reduction_pct=0.870834 moved_to_echo=32504 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3890, 185, 39, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 81, 0, 4096, 3913, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3911, 183, 0, 0, 0, 0, 0, 0, 3981, 182, 9, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 24, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8147, 8200, 8192, 8190, 8236, 8196, 8165] avg=8192.0000 max=8236 max_rank=5 min=8147 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32787 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 32768, 384, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 384, 128, 0, 128, 1152, 128, 32768, 31360, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 0, 32768, 0, 31360, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 128, 128, 32768, 128, 31104, 1536, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 0, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31744, 1536, 640, 31744, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4020, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 3546, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 0, 4096, 0, 0, 4056, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 4090, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4060, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 4093, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 3, 4096, 0, 0, 4058, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3969, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 3664, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 4023, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 4091, 0, 0, 4052, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 32, 0, 0, 3984, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 4096, 0, 0, 4045, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[6, 0, 0, 14, 0, 6, 3949, 0, 8, 4089, 8, 7, 0, 0, 7, 0, 3974, 4085, 4090, 7, 0, 4081, 0, 2, 0, 4087, 0, 4088, 4, 0, 7, 7, 5, 4089, 4087, 0, 4090, 0, 6, 4085, 0, 0, 2, 4, 6, 5, 0, 10, 0, 5, 8, 0, 246, 0, 0, 4, 2, 0, 4084, 7, 4, 4091, 4086, 4084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3960, 0, 0, 4096, 0, 0, 0, 45, 0, 0, 3968, 4096, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 0, 0, 0, 4096, 0, 0, 3928, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4048, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 3940, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 0, 0, 4096, 0, 0, 4060, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4020, 0, 3860, 301, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 3893, 187, 0, 0, 0, 0, 0, 0, 3546, 203, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 4096, 3795, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4056, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8180, 7845, 8192, 8192, 8190, 8555, 8201] avg=8192.0000 max=8555 max_rank=6 min=7845 min_rank=2 imbalance_ratio=1.044312 max_load_reduction=56981 max_load_reduction_pct=0.869461 moved_to_echo=33138 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3969, 0, 3860, 301, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 3893, 187, 0, 0, 0, 0, 0, 0, 3664, 203, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, 0, 4096, 3792, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8177, 7963, 8192, 8195, 8190, 8448, 8241] avg=8192.0000 max=8448 max_rank=6 min=7963 min_rank=2 imbalance_ratio=1.031250 max_load_reduction=57088 max_load_reduction_pct=0.871094 moved_to_echo=33074 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 14, 0, 6, 3949, 0, 3849, 301, 0, 0, 0, 0, 0, 0, 8, 4089, 8, 7, 0, 0, 7, 0, 3883, 186, 0, 0, 0, 0, 0, 0, 3974, 202, 0, 7, 0, 4081, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4088, 4, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4089, 2, 0, 0, 0, 6, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 6, 5, 0, 10, 4090, 4088, 0, 0, 0, 0, 0, 0, 0, 5, 8, 0, 246, 0, 0, 4, 4086, 3784, 2, 0, 0, 0, 0, 0, 2, 0, 4084, 7, 4, 4091, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8188, 8266, 8193, 8187, 8205, 8135, 8237] avg=8192.0000 max=8266 max_rank=2 min=8125 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32764 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 4090, 0, 3860, 301, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3893, 187, 0, 0, 0, 0, 0, 0, 4060, 203, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 4096, 3792, 2, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4058, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8172, 8359, 8192, 8194, 8190, 7970, 8206] avg=8192.0000 max=8359 max_rank=2 min=7970 min_rank=6 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32560 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3860, 301, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 32, 0, 0, 3893, 187, 0, 0, 0, 0, 0, 0, 3984, 203, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 4096, 3792, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4045, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8208, 8283, 8192, 8195, 8190, 8021, 8190] avg=8192.0000 max=8283 max_rank=2 min=8021 min_rank=6 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32596 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4088, 0, 3860, 301, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 3893, 187, 0, 0, 0, 0, 0, 0, 4023, 203, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 4096, 3793, 2, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4052, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8201, 8322, 8192, 8194, 8190, 7996, 8192] avg=8192.0000 max=8322 max_rank=2 min=7996 min_rank=6 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32572 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3960, 0, 3860, 304, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 45, 0, 0, 3893, 187, 0, 0, 0, 0, 0, 0, 3968, 203, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 0, 4096, 3792, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3928, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8221, 8266, 8192, 8187, 8190, 8283, 8073] avg=8192.0000 max=8283 max_rank=6 min=8073 min_rank=7 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32733 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4048, 0, 3865, 304, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 3900, 191, 1, 0, 0, 0, 0, 0, 3940, 196, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 4096, 3792, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4060, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8189, 8232, 8191, 8192, 8191, 8128, 8196] avg=8192.0000 max=8232 max_rank=2 min=8128 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32707 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 32512, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31232, 1664, 0, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 2432, 0, 0, 128, 32768, 30336, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 0, 31232, 1536, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 0, 128, 32256, 0, 30976, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[276, 0, 4096, 0, 0, 0, 0, 0, 3820, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 4096, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4, 0, 4087, 0, 7, 6, 0, 26, 3975, 4, 0, 4077, 0, 7, 9, 4087, 0, 9, 0, 10, 0, 0, 0, 3, 4089, 0, 3, 8, 4085, 4088, 4088, 10, 0, 1, 0, 0, 0, 1, 4089, 9, 4089, 7, 116, 0, 4087, 0, 0, 0, 7, 9, 4061, 14, 8, 4082, 4087, 4089, 4089, 7, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[56, 0, 4096, 2, 0, 0, 0, 0, 4011, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4091, 4096, 0, 0, 27, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 4085, 4096, 0, 0, 11, 0, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1143, 0, 4096, 0, 0, 0, 0, 0, 2952, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1545, 0, 4096, 4, 0, 0, 0, 0, 2552, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4091, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[91, 0, 4096, 3, 0, 0, 0, 0, 4013, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4084, 4092, 0, 0, 5, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1717, 0, 4096, 0, 0, 0, 0, 0, 2379, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4092, 4096, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[314, 0, 4096, 0, 0, 0, 0, 0, 3782, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[276, 0, 4096, 0, 0, 0, 0, 0, 3435, 13, 0, 0, 0, 0, 0, 0, 3820, 0, 0, 4096, 0, 0, 0, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7820, 8577, 8191, 8192, 8189, 8180, 8195, 8192] avg=8192.0000 max=8577 max_rank=1 min=7820 min_rank=0 imbalance_ratio=1.046997 max_load_reduction=56959 max_load_reduction_pct=0.869125 moved_to_echo=32756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4087, 0, 7, 6, 0, 26, 3427, 13, 0, 0, 0, 0, 0, 0, 3975, 4, 0, 4077, 0, 7, 9, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 10, 0, 0, 0, 3, 4089, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 4085, 4088, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4089, 9, 4077, 2, 0, 0, 0, 0, 0, 0, 4076, 7, 116, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 4061, 14, 8, 5, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 7, 2, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7570, 8732, 8197, 8195, 8179, 8286, 8193, 8184] avg=8192.0000 max=8732 max_rank=1 min=7570 min_rank=0 imbalance_ratio=1.065918 max_load_reduction=56804 max_load_reduction_pct=0.866760 moved_to_echo=32842 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1143, 0, 4096, 0, 0, 0, 0, 0, 3435, 13, 0, 0, 0, 0, 0, 0, 2952, 0, 0, 4096, 0, 0, 0, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8687, 7709, 8191, 8194, 8189, 8180, 8197, 8189] avg=8192.0000 max=8687 max_rank=0 min=7709 min_rank=1 imbalance_ratio=1.060425 max_load_reduction=56849 max_load_reduction_pct=0.867447 moved_to_echo=32755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[91, 0, 4096, 3, 0, 0, 0, 0, 3435, 13, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 4096, 0, 0, 0, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 5, 0, 3, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7638, 8770, 8192, 8190, 8189, 8179, 8185, 8193] avg=8192.0000 max=8770 max_rank=1 min=7638 min_rank=0 imbalance_ratio=1.070557 max_load_reduction=56766 max_load_reduction_pct=0.866180 moved_to_echo=32746 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[56, 0, 4096, 2, 0, 0, 0, 0, 3435, 13, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 4096, 0, 0, 0, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 4096, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 5, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 6, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7602, 8768, 8191, 8187, 8216, 8179, 8185, 8208] avg=8192.0000 max=8768 max_rank=1 min=7602 min_rank=0 imbalance_ratio=1.070312 max_load_reduction=56768 max_load_reduction_pct=0.866211 moved_to_echo=32788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1545, 0, 4096, 4, 0, 0, 0, 0, 3435, 13, 0, 0, 0, 0, 0, 0, 2552, 0, 0, 4096, 0, 0, 0, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9093, 7309, 8191, 8192, 8189, 8179, 8192, 8191] avg=8192.0000 max=9093 max_rank=0 min=7309 min_rank=1 imbalance_ratio=1.109985 max_load_reduction=56443 max_load_reduction_pct=0.861252 moved_to_echo=32751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1717, 0, 4096, 0, 0, 0, 0, 0, 3435, 13, 0, 0, 0, 0, 0, 0, 2379, 0, 0, 4096, 0, 0, 0, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9261, 7136, 8191, 8194, 8192, 8179, 8193, 8190] avg=8192.0000 max=9261 max_rank=0 min=7136 min_rank=1 imbalance_ratio=1.130493 max_load_reduction=56275 max_load_reduction_pct=0.858688 moved_to_echo=32754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[314, 0, 4096, 0, 0, 0, 0, 0, 3437, 18, 0, 0, 0, 0, 0, 0, 3782, 0, 0, 4096, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 3, 2, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7865, 8535, 8192, 8192, 8193, 8174, 8196, 8189] avg=8192.0000 max=8535 max_rank=1 min=7865 min_rank=0 imbalance_ratio=1.041870 max_load_reduction=57001 max_load_reduction_pct=0.869766 moved_to_echo=32752 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5248, 0, 32768, 128, 128, 128, 0, 128, 27520, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 128, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[27520, 128, 0, 32768, 0, 128, 128, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 0, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3946, 0, 0, 8, 3612, 0, 0, 0, 4096, 0, 0, 2926, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1795, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4090, 0, 13, 43, 3684, 0, 0, 0, 4096, 0, 0, 2812, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1646, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4078, 0, 49, 0, 3999, 0, 0, 0, 4096, 0, 0, 2124, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2038, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4091, 4090, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 4086, 0, 0, 4094, 0, 22, 20, 3729, 0, 0, 0, 4096, 0, 0, 895, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3548, 0, 0, 0, 4096, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 12, 4096, 0, 0, 0, 0, 4096, 0, 0, 4086, 0, 0, 4087, 0, 73, 12, 2721, 0, 0, 0, 4096, 0, 0, 2546, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2847, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4087, 12, 4081, 13, 53, 2, 0, 4085, 1, 0, 8, 0, 4088, 4090, 0, 4087, 3, 8, 9, 1, 4083, 7, 9, 4082, 3, 9, 4023, 7, 0, 0, 3622, 0, 4, 8, 4089, 6, 4, 2970, 4085, 12, 1, 8, 8, 0, 0, 0, 0, 4086, 11, 0, 9, 9, 0, 4088, 1582, 0, 0, 0, 4081, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4083, 0, 0, 4096, 0, 20, 20, 3927, 0, 0, 0, 4096, 0, 0, 1306, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2934, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4091, 0, 12, 25, 3683, 0, 0, 0, 4096, 0, 0, 2109, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2368, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 4090, 0, 13, 43, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 2812, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3630, 469, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1750, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2211, 1877, 7, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8192, 8191, 8296, 8793, 8195, 7492, 8192] avg=8192.0000 max=8793 max_rank=4 min=7492 min_rank=6 imbalance_ratio=1.073364 max_load_reduction=56743 max_load_reduction_pct=0.865829 moved_to_echo=32672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 3946, 0, 0, 8, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 2926, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3630, 469, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1750, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2211, 1877, 7, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8193, 8191, 8032, 8907, 8194, 7641, 8192] avg=8192.0000 max=8907 max_rank=4 min=7641 min_rank=6 imbalance_ratio=1.087280 max_load_reduction=56629 max_load_reduction_pct=0.864090 moved_to_echo=32934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4091, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 4094, 0, 22, 20, 3729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 895, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3621, 469, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1750, 3548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 2211, 1877, 7, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8188, 8191, 8330, 6876, 8179, 9394, 8193] avg=8192.0000 max=9394 max_rank=6 min=6876 min_rank=4 imbalance_ratio=1.146729 max_load_reduction=56142 max_load_reduction_pct=0.856659 moved_to_echo=32642 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 4078, 0, 49, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 2124, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3630, 469, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1750, 2038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2211, 1877, 7, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8193, 8191, 8592, 8105, 8194, 7884, 8192] avg=8192.0000 max=8592 max_rank=3 min=7884 min_rank=6 imbalance_ratio=1.048828 max_load_reduction=56944 max_load_reduction_pct=0.868896 moved_to_echo=32375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 12, 4081, 13, 53, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 8, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 3, 8, 9, 1, 4083, 7, 9, 0, 0, 0, 0, 0, 0, 0, 0, 464, 3, 9, 4023, 7, 0, 0, 3622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 1881, 6, 4, 2970, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 8, 8, 0, 0, 0, 0, 4089, 3618, 468, 0, 0, 0, 0, 0, 4086, 11, 0, 9, 9, 0, 1747, 1582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 2, 0, 0, 0, 2208, 1873, 7, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8183, 8206, 8128, 8958, 8204, 7444, 8172] avg=8192.0000 max=8958 max_rank=4 min=7444 min_rank=6 imbalance_ratio=1.093506 max_load_reduction=56578 max_load_reduction_pct=0.863312 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 4087, 0, 73, 12, 2721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 2546, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3621, 469, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1750, 2847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2211, 1877, 7, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8204, 8191, 7358, 8527, 8186, 8693, 8192] avg=8192.0000 max=8693 max_rank=6 min=7358 min_rank=3 imbalance_ratio=1.061157 max_load_reduction=56843 max_load_reduction_pct=0.867355 moved_to_echo=33598 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 0, 0, 4096, 0, 20, 20, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 1306, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3619, 469, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1750, 2934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2211, 1877, 7, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8191, 8191, 8527, 7287, 8183, 8780, 8192] avg=8192.0000 max=8780 max_rank=6 min=7287 min_rank=4 imbalance_ratio=1.071777 max_load_reduction=56756 max_load_reduction_pct=0.866028 moved_to_echo=32442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 4091, 0, 12, 25, 3683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1878, 0, 0, 2109, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3634, 471, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1744, 2368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2218, 1881, 8, 8, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8192, 8184, 8273, 8083, 8201, 8208, 8211] avg=8192.0000 max=8273 max_rank=3 min=8083 min_rank=4 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 15104, 128, 128, 17792, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 32768, 128, 128, 0, 0, 17792, 15104, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 0, 0, 0, 32768, 29056, 3840, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 14080, 18816, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3840, 128, 128, 32512, 128, 256, 128, 29056, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 0, 4096, 0, 0, 4096, 4096, 13, 1, 0, 1, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4065, 0, 0, 0, 0, 4089, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 31, 3, 0, 0, 0, 4096, 0, 0, 4096, 4096, 51, 0, 1, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4086, 0, 0, 0, 0, 4093, 0, 4096, 0, 4096, 4059, 0, 0, 0, 3, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 33, 4, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4085, 4088, 9, 8, 3, 13, 4091, 0, 4086, 0, 4082, 3935, 5, 0, 0, 0, 4085, 4087, 4088, 0, 39, 4086, 4087, 0, 1, 0, 0, 5, 4071, 1, 26, 9, 8, 0, 0, 0, 0, 10, 8, 10, 23, 4068, 26, 0, 4088, 4059, 243, 128, 0, 0, 0, 0, 3829, 7, 0, 0, 9, 10, 0, 9, 11, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4072, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4049, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 82, 4, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4078, 0, 0, 0, 0, 4091, 0, 4096, 0, 4096, 4012, 0, 0, 0, 2, 4096, 4096, 4093, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 2, 0, 23, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 83, 4, 4, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4039, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 36, 25, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4078, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4085, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 61, 2, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4022, 74, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 4096, 4075, 4, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4079, 0, 0, 4083, 22, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8243, 8190, 8185, 8188, 8131, 8207, 8188] avg=8192.0000 max=8243 max_rank=1 min=8131 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32714 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 4088, 9, 8, 3, 13, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4082, 3935, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4088, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 4071, 1, 26, 4012, 74, 2, 0, 0, 0, 0, 0, 9, 8, 0, 0, 0, 0, 10, 8, 4087, 4064, 3, 0, 0, 0, 0, 0, 10, 23, 3994, 26, 0, 0, 4059, 243, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 3829, 7, 0, 4072, 22, 22, 0, 0, 0, 0, 0, 0, 9, 10, 0, 9, 11, 0, 0, 4088, 4084, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8074, 8210, 8192, 8189, 8355, 8080, 8211] avg=8192.0000 max=8355 max_rank=5 min=8074 min_rank=1 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32835 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4096, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4022, 74, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 3, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 4096, 36, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 4092, 0, 0, 4083, 22, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8187, 8190, 8194, 8174, 8154, 8243, 8189] avg=8192.0000 max=8243 max_rank=6 min=8154 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32760 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 4065, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4022, 74, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3, 0, 4096, 4075, 4, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 4096, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4046, 0, 0, 4083, 22, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8243, 8190, 8193, 8209, 8169, 8173, 8192] avg=8192.0000 max=8243 max_rank=1 min=8167 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32743 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 4072, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4096, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4022, 74, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 4096, 4075, 3, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 4096, 82, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4056, 0, 0, 4083, 22, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8197, 8190, 8194, 8199, 8200, 8186, 8189] avg=8192.0000 max=8200 max_rank=5 min=8181 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32774 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 4078, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4096, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4022, 74, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 23, 0, 0, 4096, 4075, 4, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 4096, 83, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 4096, 0, 0, 4083, 22, 21, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8160, 8189, 8184, 8200, 8201, 8230, 8190] avg=8192.0000 max=8230 max_rank=6 min=8160 min_rank=1 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32821 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 4086, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4096, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4022, 74, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 4075, 4, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4093, 0, 0, 4083, 22, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8207, 8193, 8193, 8189, 8151, 8223, 8188] avg=8192.0000 max=8223 max_rank=6 min=8151 min_rank=5 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32751 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4078, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4023, 78, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4096, 4075, 4, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 4096, 61, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4052, 0, 0, 4090, 29, 21, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8225, 8184, 8201, 8188, 8175, 8194, 8189] avg=8192.0000 max=8225 max_rank=1 min=8175 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32746 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 512, 0, 32768, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32256, 128, 0, 0, 32768, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 128, 128, 128, 32768, 32640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 128, 128, 0, 0, 32384, 128, 0, 32768, 256, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 32768, 128, 128, 32256, 640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 2859, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 510, 0, 0, 0, 0, 4096, 0, 1014, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 729, 4089, 4096, 0, 3082, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3253, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 221, 0, 0, 0, 0, 4096, 0, 1125, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 9, 4096, 4096, 0, 3584, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 1133, 4096, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 150, 0, 0, 0, 0, 4096, 0, 3179, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 15, 4089, 4096, 0, 3700, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 2283, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 599, 0, 0, 0, 0, 4096, 0, 1704, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 16, 4083, 4096, 0, 3591, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 3354, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 354, 0, 0, 0, 0, 4096, 0, 1341, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 51, 4080, 4096, 0, 3093, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4063, 0, 0, 17, 10, 0, 3, 0, 0, 4081, 4086, 8, 0, 0, 11, 5, 0, 4085, 0, 1, 1140, 3922, 22, 0, 4, 4085, 0, 7, 9, 12, 0, 4085, 0, 11, 4079, 4087, 5, 4085, 0, 0, 25, 0, 0, 0, 0, 4088, 0, 3257, 12, 11, 4080, 13, 5, 0, 0, 10, 0, 9, 340, 4087, 4083, 0, 3586, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 3740, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 678, 0, 0, 0, 0, 4096, 0, 1664, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 80, 4078, 4096, 0, 2031, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 3133, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 333, 0, 0, 0, 0, 4096, 0, 1733, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 4082, 4096, 0, 2985, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 5, 0, 2859, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 0, 0, 0, 0, 4096, 0, 1014, 1860, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3350, 742, 0, 0, 0, 0, 0, 0, 0, 0, 729, 4089, 743, 0, 3082, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8454, 8191, 8194, 7481, 8188, 8643] avg=8192.0000 max=8643 max_rank=7 min=7481 min_rank=5 imbalance_ratio=1.055054 max_load_reduction=56893 max_load_reduction_pct=0.868118 moved_to_echo=32506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 0, 0, 3253, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 0, 0, 0, 4096, 0, 1125, 1860, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 3350, 742, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 743, 0, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8843, 8191, 8194, 7303, 8189, 8432] avg=8192.0000 max=8843 max_rank=2 min=7303 min_rank=5 imbalance_ratio=1.079468 max_load_reduction=56693 max_load_reduction_pct=0.865067 moved_to_echo=32118 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4063, 0, 0, 17, 10, 0, 3, 0, 4085, 3, 0, 0, 0, 0, 0, 0, 0, 4081, 4086, 8, 0, 0, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1490, 0, 1, 1140, 3922, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4085, 0, 7, 9, 12, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4079, 2, 5, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 4088, 0, 3257, 1855, 1, 0, 0, 0, 0, 0, 0, 12, 11, 4080, 13, 5, 0, 0, 10, 3339, 740, 0, 0, 0, 0, 0, 0, 0, 9, 340, 4087, 741, 0, 3586, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8191, 6575, 8201, 8182, 9226, 8210, 8770] avg=8192.0000 max=9226 max_rank=5 min=6575 min_rank=2 imbalance_ratio=1.126221 max_load_reduction=56310 max_load_reduction_pct=0.859222 moved_to_echo=34388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 16, 0, 1133, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 4096, 0, 3179, 1860, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3350, 742, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4089, 743, 0, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 6745, 8191, 8194, 9286, 8188, 8547] avg=8192.0000 max=9286 max_rank=5 min=6745 min_rank=2 imbalance_ratio=1.133545 max_load_reduction=56250 max_load_reduction_pct=0.858307 moved_to_echo=34215 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 15, 0, 3354, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 4096, 0, 1341, 1860, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3350, 742, 0, 0, 0, 0, 0, 0, 0, 0, 51, 4080, 743, 0, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8959, 8191, 8194, 7652, 8188, 7967] avg=8192.0000 max=8959 max_rank=2 min=7652 min_rank=5 imbalance_ratio=1.093628 max_load_reduction=56577 max_load_reduction_pct=0.863297 moved_to_echo=32001 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 12, 0, 2283, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 599, 0, 0, 0, 0, 4096, 0, 1704, 1860, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3350, 742, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4083, 743, 0, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 7885, 8191, 8194, 8260, 8188, 8433] avg=8192.0000 max=8433 max_rank=7 min=7885 min_rank=2 imbalance_ratio=1.029419 max_load_reduction=57103 max_load_reduction_pct=0.871323 moved_to_echo=33075 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 15, 0, 3740, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 0, 0, 0, 0, 4096, 0, 1664, 1860, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3350, 742, 0, 0, 0, 0, 0, 0, 0, 0, 80, 4078, 743, 0, 2031, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 9347, 8191, 8192, 8299, 8188, 6932] avg=8192.0000 max=9347 max_rank=2 min=6932 min_rank=7 imbalance_ratio=1.140991 max_load_reduction=56189 max_load_reduction_pct=0.857376 moved_to_echo=31611 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 15, 0, 3133, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 0, 0, 4096, 0, 1733, 1865, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3354, 747, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4082, 738, 0, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8193, 8728, 8189, 8192, 8029, 8197, 7812] avg=8192.0000 max=8728 max_rank=2 min=7812 min_rank=7 imbalance_ratio=1.065430 max_load_reduction=56808 max_load_reduction_pct=0.866821 moved_to_echo=32230 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 32768, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2944, 0, 0, 0, 0, 32768, 0, 15104, 14976, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 12032, 128, 128, 20992, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 1280, 32768, 6016, 0, 25728, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 128, 128, 26880, 6016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[7, 0, 4086, 12, 7, 2, 12, 0, 0, 4084, 9, 0, 4084, 15, 4084, 4081, 11, 0, 6, 4085, 10, 11, 0, 14, 15, 12, 0, 0, 0, 0, 8, 0, 0, 4083, 4085, 0, 3984, 8, 0, 0, 13, 6, 26, 0, 11, 4068, 3, 12, 0, 1, 127, 0, 4087, 6, 0, 0, 0, 4076, 4086, 4034, 0, 4085, 4080, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4091, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 133, 0, 4096, 0, 4, 0, 0, 4094, 4096, 4047, 3, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 2, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 87, 0, 4096, 0, 1, 0, 0, 4085, 4096, 4088, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 180, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 32, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 92, 1, 4095, 0, 0, 0, 0, 4088, 4096, 4049, 10, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 55, 0, 4096, 0, 0, 0, 0, 4090, 4096, 4093, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 4095, 0, 1, 0, 0, 4096, 4096, 4088, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 4093, 0, 0, 0, 0, 4092, 4096, 4091, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 1, 0, 0, 4068, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 73, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 5, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 4096, 0, 1, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4088, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8194, 8189, 8190, 8193, 8189, 8194, 8201] avg=8192.0000 max=8201 max_rank=7 min=8186 min_rank=0 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4068, 13, 8, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 73, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 6, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 4096, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8194, 8188, 8190, 8085, 8188, 8286, 8220] avg=8192.0000 max=8286 max_rank=6 min=8085 min_rank=4 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32779 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4068, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 73, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 5, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 4096, 0, 4, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4047, 3, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8187, 8189, 8190, 8181, 8189, 8243, 8172] avg=8192.0000 max=8243 max_rank=6 min=8172 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32785 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4068, 13, 8, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 73, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 6, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 4096, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4093, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8194, 8187, 8191, 8219, 8188, 8161, 8211] avg=8192.0000 max=8219 max_rank=4 min=8161 min_rank=6 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32779 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 32, 0, 0, 4068, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 73, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 5, 0, 0, 0, 0, 0, 0, 0, 0, 92, 1, 4095, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4049, 10, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8192, 8189, 8190, 8186, 8187, 8198, 8175] avg=8192.0000 max=8219 max_rank=0 min=8175 min_rank=7 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32746 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4068, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 73, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4095, 0, 1, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8192, 8189, 8190, 8265, 8189, 8114, 8212] avg=8192.0000 max=8265 max_rank=4 min=8114 min_rank=6 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 1, 0, 0, 4072, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 67, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 5, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4093, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4091, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8189, 8190, 8258, 8187, 8120, 8207] avg=8192.0000 max=8258 max_rank=4 min=8120 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32772 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 4086, 12, 7, 2, 12, 0, 4052, 12, 8, 0, 0, 0, 0, 0, 0, 2, 9, 0, 0, 15, 4084, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 6, 4085, 10, 11, 0, 14, 4078, 1, 0, 0, 0, 0, 0, 0, 15, 12, 0, 0, 0, 0, 8, 0, 4086, 4084, 0, 0, 0, 0, 0, 0, 0, 4083, 74, 0, 3984, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6, 26, 0, 11, 4068, 3, 12, 4074, 6, 0, 0, 0, 0, 0, 0, 0, 1, 127, 0, 4087, 6, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4034, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8191, 8216, 8205, 8149, 8219, 8220, 8138] avg=8192.0000 max=8220 max_rank=6 min=8138 min_rank=7 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32726 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 32640, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 768, 128, 32768, 128, 128, 0, 32128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 640, 0, 32256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 32768, 128, 128, 128, 128, 0, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4095, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 3, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 6, 0, 0, 0, 4093, 0, 0, 4088, 0, 0, 0, 4096, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 4096, 0, 4096, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[2, 0, 8, 3988, 10, 4081, 0, 4078, 0, 16, 0, 44, 0, 0, 4079, 0, 109, 4050, 10, 3962, 0, 4082, 4085, 0, 4066, 7, 4050, 0, 0, 7, 39, 10, 24, 13, 6, 18, 14, 0, 1, 0, 0, 0, 8, 11, 15, 0, 4072, 115, 4084, 0, 253, 3, 9, 2, 4087, 19, 11, 4088, 0, 9, 8, 4079, 0, 3804] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 61, 4096, 0, 4094, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 44, 0, 0, 0, 4096, 0, 0, 4085, 0, 0, 0, 4096, 0, 4004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4096, 0, 4096, 0, 4096, 4096, 0, 4087, 0, 4096, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 9, 0, 0, 0, 0, 0, 0, 6, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4044, 37, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 52, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8187, 8197, 8194, 8186, 8177, 8156, 8233] avg=8192.0000 max=8233 max_rank=7 min=8156 min_rank=6 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32752 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 9, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4044, 37, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 52, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8187, 8191, 8194, 8183, 8177, 8154, 8244] avg=8192.0000 max=8244 max_rank=7 min=8154 min_rank=6 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32758 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 8, 3988, 10, 14, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 44, 0, 0, 4079, 0, 4067, 9, 0, 0, 0, 0, 0, 0, 109, 4049, 10, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 7, 4050, 0, 0, 7, 39, 10, 1, 1, 0, 0, 0, 0, 0, 0, 24, 13, 6, 18, 14, 0, 1, 0, 4085, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 8, 11, 15, 0, 4072, 115, 4028, 37, 0, 0, 0, 0, 0, 0, 4084, 0, 253, 3, 9, 2, 4049, 19, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4088, 0, 9, 8, 51, 0, 3804, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8215, 8130, 8181, 8234, 8286, 8419, 7971] avg=8192.0000 max=8419 max_rank=6 min=7971 min_rank=7 imbalance_ratio=1.027710 max_load_reduction=57117 max_load_reduction_pct=0.871536 moved_to_echo=32910 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 14, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 9, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4044, 37, 0, 0, 0, 0, 0, 0, 4092, 0, 6, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 52, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8186, 8191, 8197, 8190, 8177, 8153, 8235] avg=8192.0000 max=8235 max_rank=7 min=8153 min_rank=6 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 9, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4044, 37, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 52, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8187, 8192, 8193, 8183, 8177, 8154, 8244] avg=8192.0000 max=8244 max_rank=7 min=8154 min_rank=6 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32758 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 10, 0, 0, 0, 0, 0, 0, 61, 4095, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4044, 37, 0, 0, 0, 0, 0, 0, 4096, 0, 44, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 52, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8188, 8250, 8194, 8182, 8177, 8198, 8141] avg=8192.0000 max=8250 max_rank=2 min=8141 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32698 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 9, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4044, 37, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 52, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8187, 8194, 8185, 8196, 8177, 8153, 8238] avg=8192.0000 max=8238 max_rank=7 min=8153 min_rank=6 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32764 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 10, 4, 0, 0, 0, 0, 0, 2, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4051, 41, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 41, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8199, 8191, 8198, 8182, 8188, 8149, 8230] avg=8192.0000 max=8230 max_rank=7 min=8149 min_rank=6 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32749 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 32768, 0, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 0, 32768, 128, 32384, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 512, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 0, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[256, 32768, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 384, 128, 128, 128, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 312, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4063, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3818, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4, 4085, 0, 0, 4, 0, 4074, 11, 1, 5, 11, 6, 7, 0, 4088, 0, 0, 404, 5, 4079, 4083, 0, 4077, 0, 0, 34, 12, 3893, 0, 4084, 0, 72, 17, 4009, 19, 12, 0, 8, 0, 0, 12, 3873, 0, 4082, 15, 0, 4081, 11, 4087, 4085, 0, 0, 4076, 0, 10, 0, 10, 14, 0, 7, 4057, 0, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 1196, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4001, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3001, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3967, 5, 0, 439, 0, 4096, 4096, 5, 4096, 0, 0, 0, 0, 4088, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3785, 0, 4096, 0, 0, 4096, 0, 4096, 4095, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 469, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4047, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3702, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 3841, 5, 0, 1132, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3610, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 240, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 996, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3836, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3362, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4085, 0, 0, 4, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 5, 11, 6, 7, 0, 4088, 4083, 60, 0, 0, 0, 0, 0, 0, 0, 0, 404, 5, 3436, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 12, 3893, 0, 4084, 0, 136, 0, 0, 0, 0, 0, 0, 0, 72, 17, 4009, 19, 12, 0, 8, 0, 3571, 507, 0, 0, 0, 0, 0, 0, 0, 12, 3873, 0, 448, 15, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 4085, 0, 0, 4076, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 14, 0, 7, 4057, 0, 12, 4025, 63, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8272, 7922, 8159, 8215, 8429, 8184, 8188] avg=8192.0000 max=8429 max_rank=5 min=7922 min_rank=2 imbalance_ratio=1.028931 max_load_reduction=57107 max_load_reduction_pct=0.871384 moved_to_echo=33016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 3450, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 4096, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3583, 509, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 0, 450, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4033, 63, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8251, 7858, 8296, 8188, 8364, 8195, 8192] avg=8192.0000 max=8364 max_rank=5 min=7858 min_rank=2 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32939 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 1196, 0, 3450, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 4096, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3583, 509, 0, 0, 0, 0, 0, 0, 0, 0, 3001, 0, 450, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4036, 63, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8241, 8742, 8234, 8187, 7547, 8192, 8195] avg=8192.0000 max=8742 max_rank=2 min=7547 min_rank=5 imbalance_ratio=1.067139 max_load_reduction=56794 max_load_reduction_pct=0.866608 moved_to_echo=32121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 469, 0, 3450, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 4096, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3583, 509, 0, 0, 0, 0, 0, 0, 0, 0, 3702, 0, 450, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4033, 63, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8225, 8015, 8280, 8188, 8248, 8195, 8191] avg=8192.0000 max=8280 max_rank=3 min=8015 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 4096, 63, 0, 0, 0, 0, 0, 0, 5, 0, 1132, 0, 3450, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 0, 4096, 1, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3583, 509, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 450, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4033, 63, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8000, 8683, 7844, 8188, 8232, 8192, 8191] avg=8192.0000 max=8683 max_rank=2 min=7844 min_rank=3 imbalance_ratio=1.059937 max_load_reduction=56853 max_load_reduction_pct=0.867508 moved_to_echo=32803 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 3450, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3583, 509, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 450, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4036, 63, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8171, 7786, 8329, 8188, 8482, 8192, 8191] avg=8192.0000 max=8482 max_rank=5 min=7786 min_rank=2 imbalance_ratio=1.035400 max_load_reduction=57054 max_load_reduction_pct=0.870575 moved_to_echo=33053 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 4096, 60, 0, 0, 0, 0, 0, 0, 5, 0, 439, 0, 3450, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 2, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3583, 509, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 0, 450, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4034, 63, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8123, 7995, 8323, 8188, 8331, 8194, 8193] avg=8192.0000 max=8331 max_rank=5 min=7995 min_rank=2 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32906 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 63, 0, 0, 0, 0, 0, 0, 0, 0, 996, 0, 3443, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 4096, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3585, 513, 0, 0, 0, 0, 0, 0, 0, 0, 3362, 0, 445, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4033, 66, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8253, 8535, 8071, 8194, 7903, 8192, 8195] avg=8192.0000 max=8535 max_rank=2 min=7903 min_rank=5 imbalance_ratio=1.041870 max_load_reduction=57001 max_load_reduction_pct=0.869766 moved_to_echo=32484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 29184, 0, 3712, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 0, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 5248, 128, 27648, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 31744, 0, 32768, 128, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 32768, 0, 128, 32384, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 32256, 32768, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 128, 0, 28672, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[13, 15, 8, 0, 7, 4090, 0, 0, 17, 2, 2, 0, 2, 11, 0, 4083, 0, 4077, 7, 4074, 4, 4084, 4079, 2225, 4085, 5, 4086, 56, 0, 29, 4075, 4082, 0, 6, 0, 63, 0, 4090, 0, 4081, 11, 4077, 10, 0, 4076, 11, 3, 8, 12, 0, 0, 6, 0, 14, 4086, 0, 0, 1731, 1, 10, 0, 0, 11, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 1345, 4096, 0, 4096, 0, 0, 395, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2357, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 1141, 4096, 0, 4096, 3, 0, 323, 4096, 4095, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 4096, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2647, 0, 0, 6, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 4096, 4096, 2647, 4096, 0, 4096, 14, 0, 364, 4096, 4092, 0, 0, 0, 0, 0, 4073, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1151, 0, 0, 13, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 4096, 0, 4096, 4096, 1326, 4096, 0, 4096, 6, 0, 175, 4096, 4095, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2626, 0, 0, 16, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4094, 4096, 1718, 4096, 0, 4094, 0, 0, 139, 4096, 4095, 0, 0, 0, 0, 1, 4076, 1, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 2269, 0, 0, 20, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 968, 4096, 0, 4096, 8, 0, 355, 4096, 4096, 0, 0, 0, 0, 0, 4085, 0, 4096, 0, 4096, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2799, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 605, 4096, 0, 4096, 0, 0, 142, 4096, 4096, 0, 0, 0, 0, 0, 4060, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3384, 0, 0, 24, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4075, 0, 0, 2383, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2361, 1734, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2603, 0, 0, 0, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 323, 3850, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2362, 1493, 240, 0, 0, 0, 0, 0, 0, 2647, 0, 0, 6, 0, 0, 0, 4096, 1713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8191, 7840, 8271, 8194, 8194, 8191, 8462] avg=8192.0000 max=8462 max_rank=7 min=7840 min_rank=2 imbalance_ratio=1.032959 max_load_reduction=57074 max_load_reduction_pct=0.870880 moved_to_echo=33041 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[13, 15, 8, 0, 7, 4090, 0, 0, 2376, 1729, 0, 0, 0, 0, 0, 0, 17, 2, 2, 0, 2, 11, 0, 4083, 2355, 1727, 0, 0, 0, 0, 0, 0, 0, 4077, 7, 2589, 4, 0, 0, 2225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 56, 0, 29, 3831, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 63, 0, 4090, 0, 4081, 6, 1, 0, 0, 0, 0, 0, 0, 11, 4076, 10, 0, 4076, 11, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 6, 0, 14, 4086, 0, 2352, 1485, 238, 0, 0, 0, 0, 0, 0, 1731, 1, 10, 0, 0, 11, 11, 4085, 1710, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8199, 8902, 8003, 8247, 8195, 8193, 7559] avg=8192.0000 max=8902 max_rank=2 min=7559 min_rank=7 imbalance_ratio=1.086670 max_load_reduction=56634 max_load_reduction_pct=0.864166 moved_to_echo=32194 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 2382, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2361, 1734, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2603, 0, 0, 0, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 3850, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2362, 1493, 240, 0, 0, 0, 0, 0, 0, 2357, 0, 0, 0, 0, 0, 0, 4096, 1714, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8191, 8044, 8340, 8199, 8191, 8191, 8167] avg=8192.0000 max=8340 max_rank=3 min=8044 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32748 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4031, 0, 0, 2383, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2361, 1735, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2603, 0, 0, 0, 2647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 364, 3850, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 4096, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2361, 1493, 240, 0, 0, 0, 0, 0, 0, 1151, 0, 0, 13, 0, 0, 0, 4096, 1713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8192, 9345, 8320, 8176, 8191, 8190, 6973] avg=8192.0000 max=9345 max_rank=2 min=6973 min_rank=7 imbalance_ratio=1.140747 max_load_reduction=56191 max_load_reduction_pct=0.857407 moved_to_echo=31530 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4066, 0, 0, 2381, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2359, 1734, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2603, 0, 0, 0, 1718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 3850, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4076, 1, 4096, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 2362, 1493, 240, 0, 0, 0, 0, 0, 0, 2269, 0, 0, 20, 0, 0, 0, 4096, 1713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8189, 8417, 8084, 8181, 8191, 8194, 8098] avg=8192.0000 max=8417 max_rank=2 min=8084 min_rank=3 imbalance_ratio=1.027466 max_load_reduction=57119 max_load_reduction_pct=0.871567 moved_to_echo=32664 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4065, 0, 0, 2383, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2361, 1734, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2603, 0, 0, 0, 1326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 175, 3850, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2362, 1493, 240, 0, 0, 0, 0, 0, 0, 2626, 0, 0, 16, 0, 0, 0, 4096, 1713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8191, 8020, 8126, 8183, 8191, 8191, 8451] avg=8192.0000 max=8451 max_rank=7 min=8020 min_rank=2 imbalance_ratio=1.031616 max_load_reduction=57085 max_load_reduction_pct=0.871048 moved_to_echo=33016 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4073, 0, 0, 2382, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2361, 1734, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2603, 0, 0, 0, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 3850, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 4096, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2362, 1493, 240, 0, 0, 0, 0, 0, 0, 3384, 0, 0, 24, 0, 0, 0, 4096, 1714, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8191, 7304, 8088, 8163, 8191, 8191, 9218] avg=8192.0000 max=9218 max_rank=7 min=7304 min_rank=2 imbalance_ratio=1.125244 max_load_reduction=56318 max_load_reduction_pct=0.859344 moved_to_echo=33763 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4070, 0, 0, 2383, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2361, 1735, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2600, 0, 0, 0, 968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 355, 3845, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4096, 9, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2361, 1496, 242, 0, 0, 0, 0, 0, 0, 2799, 0, 0, 0, 0, 0, 0, 4096, 1713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8192, 7664, 8304, 8193, 8192, 8195, 8608] avg=8192.0000 max=8608 max_rank=7 min=7664 min_rank=2 imbalance_ratio=1.050781 max_load_reduction=56928 max_load_reduction_pct=0.868652 moved_to_echo=33188 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 32640, 0, 0, 19072, 13952, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 0, 2048, 30848, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 20864, 128, 0, 0, 12032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 128, 32768, 128, 18944, 12032, 1920, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 19072, 128, 128, 128, 0, 128, 128, 32768, 13824, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 32768, 18944, 13952, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True + [2026-08-30 05:00:51] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 682.2 | throughput per GPU (TFLOP/s/GPU): 488.7 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.034245E+01 | loss scale: 1.0 | grad norm: 29.702 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[26, 1, 65, 0, 9, 0, 1642, 3, 1106, 222, 16, 2200, 1771, 2319, 0, 3187, 2602, 78, 7, 595, 7, 2560, 154, 1190, 1, 63, 146, 1897, 1906, 2, 667, 2067, 4052, 13, 209, 3312, 2959, 0, 955, 263, 4057, 5, 1329, 2601, 3, 94, 1080, 824, 806, 3601, 963, 0, 1, 6, 1, 2997, 0, 1356, 469, 863, 1371, 2046, 2759, 32] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[37, 5, 26, 13, 8, 3, 1542, 0, 822, 115, 34, 1668, 2310, 1899, 2, 3564, 2848, 194, 13, 543, 4, 2511, 177, 1338, 25, 16, 147, 2094, 1989, 5, 539, 2174, 4073, 24, 267, 2244, 1881, 2, 1571, 1020, 4034, 4, 1439, 2197, 8, 160, 1325, 1255, 856, 3650, 1451, 0, 16, 15, 8, 3017, 1, 1442, 755, 545, 1426, 1619, 2491, 75] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[51, 1, 27, 2, 11, 0, 1553, 2, 865, 207, 3, 1925, 1829, 2204, 2, 3481, 3257, 115, 25, 659, 2, 3172, 138, 1242, 3, 24, 197, 1893, 1582, 7, 659, 2320, 4090, 28, 201, 3032, 2139, 4, 1483, 472, 4053, 0, 1098, 2336, 1, 158, 1432, 765, 816, 3562, 736, 1, 3, 5, 5, 3165, 0, 849, 716, 780, 1864, 1848, 2312, 124] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[44, 1, 83, 10, 20, 3, 1667, 1, 555, 109, 27, 1378, 2570, 2135, 3, 3531, 3189, 87, 23, 1249, 1, 2161, 221, 965, 0, 27, 218, 2384, 1603, 4, 651, 1788, 4091, 64, 141, 2782, 1787, 6, 1706, 765, 4050, 3, 1511, 2172, 5, 170, 1748, 1399, 938, 3483, 1083, 5, 2, 5, 27, 2766, 0, 971, 545, 389, 1692, 1754, 2633, 135] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[10, 0, 58, 1, 9, 0, 1835, 0, 694, 219, 12, 1519, 1642, 1912, 2, 3546, 3627, 202, 13, 626, 11, 2813, 181, 1068, 0, 65, 169, 1877, 1807, 1, 681, 2524, 4076, 10, 271, 2990, 2104, 1, 1126, 217, 4064, 0, 1200, 2318, 1, 114, 1791, 1311, 625, 3834, 860, 0, 0, 1, 0, 2963, 0, 1266, 631, 523, 1533, 1727, 2805, 50] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[44, 2, 57, 3, 13, 1, 1459, 3, 840, 168, 15, 1838, 1720, 2144, 3, 3628, 3527, 84, 18, 527, 1, 2577, 201, 1297, 5, 66, 174, 2269, 1455, 2, 591, 2007, 4092, 5, 163, 2693, 2296, 4, 1314, 420, 4067, 1, 1369, 2614, 2, 273, 1068, 1372, 1254, 3242, 1068, 0, 2, 2, 22, 2712, 1, 729, 595, 698, 1546, 2364, 2737, 72] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[37, 1, 48, 2, 22, 0, 1196, 0, 445, 212, 6, 1691, 2179, 2372, 0, 3661, 3703, 126, 13, 670, 1, 2613, 170, 1339, 1, 26, 177, 1967, 1945, 5, 565, 1806, 4092, 26, 202, 2719, 2316, 1, 1563, 959, 4081, 0, 1189, 2441, 0, 92, 1321, 1549, 611, 3548, 780, 1, 1, 2, 11, 3066, 0, 870, 516, 402, 1521, 1723, 2868, 66] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[130, 6, 66, 0, 133, 0, 1336, 8, 1202, 142, 9, 960, 2007, 1844, 0, 3424, 3427, 193, 91, 529, 1, 2215, 253, 1027, 4, 142, 200, 1818, 1389, 9, 735, 1719, 4080, 26, 562, 2861, 1500, 7, 1687, 389, 4051, 0, 1819, 3058, 1, 167, 1571, 1727, 934, 3817, 1243, 15, 4, 5, 6, 2846, 2, 565, 705, 634, 1709, 1669, 2636, 221] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[26, 1, 65, 0, 9, 0, 1642, 3, 3087, 2551, 786, 0, 0, 0, 0, 0, 1106, 222, 16, 2200, 1771, 2319, 0, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 2602, 78, 7, 595, 7, 2560, 154, 1190, 138, 0, 0, 0, 0, 0, 0, 0, 1, 63, 146, 1897, 1906, 2, 667, 2067, 1098, 241, 83, 0, 0, 0, 0, 0, 965, 13, 209, 3312, 2959, 0, 955, 263, 0, 0, 0, 0, 0, 0, 0, 0, 1506, 5, 1329, 2601, 3, 94, 1080, 824, 0, 0, 0, 0, 0, 0, 0, 0, 806, 3360, 963, 0, 1, 6, 1, 2997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1356, 469, 863, 1371, 2046, 2538, 32, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8937, 7331, 8171, 8676, 7442, 8134, 8675] avg=8192.0000 max=8937 max_rank=1 min=7331 min_rank=2 imbalance_ratio=1.090942 max_load_reduction=56599 max_load_reduction_pct=0.863632 moved_to_echo=32927 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[51, 1, 27, 2, 11, 0, 1553, 2, 3116, 2548, 859, 0, 0, 0, 0, 0, 865, 207, 3, 1925, 1829, 2204, 2, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 3257, 115, 25, 659, 2, 3172, 138, 1242, 115, 0, 0, 0, 0, 0, 0, 0, 3, 24, 197, 1893, 1582, 7, 659, 2320, 1200, 239, 70, 0, 0, 0, 0, 0, 974, 28, 201, 3032, 2139, 4, 1483, 472, 0, 0, 0, 0, 0, 0, 0, 0, 1505, 0, 1098, 2336, 1, 158, 1432, 765, 0, 0, 0, 0, 0, 0, 0, 0, 816, 3323, 736, 1, 3, 5, 5, 3165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 716, 780, 1864, 1848, 2127, 124, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8457, 8725, 8194, 8333, 7295, 8054, 8308] avg=8192.0000 max=8725 max_rank=2 min=7295 min_rank=5 imbalance_ratio=1.065063 max_load_reduction=56811 max_load_reduction_pct=0.866867 moved_to_echo=31990 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 58, 1, 9, 0, 1835, 0, 3105, 2555, 875, 0, 0, 0, 0, 0, 694, 219, 12, 1519, 1642, 1912, 2, 1449, 0, 0, 0, 0, 0, 0, 0, 0, 3627, 202, 13, 626, 11, 2813, 181, 1068, 140, 0, 0, 0, 0, 0, 0, 0, 0, 65, 169, 1877, 1807, 1, 681, 2524, 1222, 257, 85, 0, 0, 0, 0, 0, 971, 10, 271, 2990, 2104, 1, 1126, 217, 0, 0, 0, 0, 0, 0, 0, 0, 1509, 0, 1200, 2318, 1, 114, 1791, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 625, 3577, 860, 0, 0, 1, 0, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1266, 631, 523, 1533, 1727, 2580, 50, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8448, 7449, 8681, 8688, 7690, 8244, 8026, 8310] avg=8192.0000 max=8688 max_rank=3 min=7449 min_rank=1 imbalance_ratio=1.060547 max_load_reduction=56848 max_load_reduction_pct=0.867432 moved_to_echo=32270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[37, 5, 26, 13, 8, 3, 1542, 0, 3103, 2536, 879, 0, 0, 0, 0, 0, 822, 115, 34, 1668, 2310, 1899, 2, 1457, 0, 0, 0, 0, 0, 0, 0, 0, 2848, 194, 13, 543, 4, 2511, 177, 1338, 124, 0, 0, 0, 0, 0, 0, 0, 25, 16, 147, 2094, 1989, 5, 539, 2174, 1228, 245, 75, 0, 0, 0, 0, 0, 970, 24, 267, 2244, 1881, 2, 1571, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1498, 4, 1439, 2197, 8, 160, 1325, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 856, 3405, 1451, 0, 16, 15, 8, 3017, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1442, 755, 545, 1426, 1619, 2292, 75, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8307, 7752, 8537, 7979, 7886, 8768, 8155] avg=8192.0000 max=8768 max_rank=6 min=7752 min_rank=2 imbalance_ratio=1.070312 max_load_reduction=56768 max_load_reduction_pct=0.866211 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[44, 1, 83, 10, 20, 3, 1667, 1, 3116, 2547, 871, 0, 0, 0, 0, 0, 555, 109, 27, 1378, 2570, 2135, 3, 1443, 0, 0, 0, 0, 0, 0, 0, 0, 3189, 87, 23, 1249, 1, 2161, 221, 965, 132, 0, 0, 0, 0, 0, 0, 0, 0, 27, 218, 2384, 1603, 4, 651, 1788, 1217, 233, 79, 0, 0, 0, 0, 0, 975, 64, 141, 2782, 1787, 6, 1706, 765, 0, 0, 0, 0, 0, 0, 0, 0, 1503, 3, 1511, 2172, 5, 170, 1748, 1399, 0, 0, 0, 0, 0, 0, 0, 0, 938, 3250, 1083, 5, 2, 5, 27, 2766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 545, 389, 1692, 1754, 2422, 135, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8363, 8220, 8028, 8204, 8226, 8511, 8076, 7908] avg=8192.0000 max=8511 max_rank=5 min=7908 min_rank=7 imbalance_ratio=1.038940 max_load_reduction=57025 max_load_reduction_pct=0.870132 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[44, 2, 57, 3, 13, 1, 1459, 3, 3117, 2557, 895, 0, 0, 0, 0, 0, 840, 168, 15, 1838, 1720, 2144, 3, 1482, 0, 0, 0, 0, 0, 0, 0, 0, 3527, 84, 18, 527, 1, 2577, 201, 1297, 137, 0, 0, 0, 0, 0, 0, 0, 5, 66, 174, 2269, 1455, 2, 591, 2007, 1251, 217, 83, 0, 0, 0, 0, 0, 975, 5, 163, 2693, 2296, 4, 1314, 420, 0, 0, 0, 0, 0, 0, 0, 0, 1510, 1, 1369, 2614, 2, 273, 1068, 1372, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 3025, 1068, 0, 2, 2, 22, 2712, 0, 0, 0, 0, 0, 0, 0, 0, 1, 729, 595, 698, 1546, 2364, 2517, 72, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8210, 8369, 8120, 7870, 8209, 8085, 8522] avg=8192.0000 max=8522 max_rank=7 min=7870 min_rank=4 imbalance_ratio=1.040283 max_load_reduction=57014 max_load_reduction_pct=0.869965 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[37, 1, 48, 2, 22, 0, 1196, 0, 3117, 2566, 903, 0, 0, 0, 0, 0, 445, 212, 6, 1691, 2179, 2372, 0, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 3703, 126, 13, 670, 1, 2613, 170, 1339, 143, 0, 0, 0, 0, 0, 0, 0, 1, 26, 177, 1967, 1945, 5, 565, 1806, 1262, 238, 87, 0, 0, 0, 0, 0, 975, 26, 202, 2719, 2316, 1, 1563, 959, 0, 0, 0, 0, 0, 0, 0, 0, 1515, 0, 1189, 2441, 0, 92, 1321, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 611, 3310, 780, 1, 1, 2, 11, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 870, 516, 402, 1521, 1723, 2638, 66, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7892, 8401, 8778, 8079, 8761, 8107, 7782, 7736] avg=8192.0000 max=8778 max_rank=2 min=7736 min_rank=7 imbalance_ratio=1.071533 max_load_reduction=56758 max_load_reduction_pct=0.866058 moved_to_echo=32386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[130, 6, 66, 0, 133, 0, 1336, 8, 3111, 2552, 848, 0, 0, 0, 0, 0, 1202, 142, 9, 960, 2007, 1844, 0, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 3427, 193, 91, 529, 1, 2215, 253, 1027, 136, 0, 0, 0, 0, 0, 0, 0, 4, 142, 200, 1818, 1389, 9, 735, 1719, 1185, 259, 83, 0, 0, 0, 0, 0, 969, 26, 562, 2861, 1500, 7, 1687, 389, 0, 0, 0, 0, 0, 0, 0, 0, 1499, 0, 1819, 3058, 1, 167, 1571, 1727, 0, 0, 0, 0, 0, 0, 0, 0, 934, 3558, 1243, 15, 4, 5, 6, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 2, 565, 705, 634, 1709, 1669, 2417, 221, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7555, 7872, 7543, 8001, 9842, 8611, 7922] avg=8192.0000 max=9842 max_rank=5 min=7543 min_rank=3 imbalance_ratio=1.201416 max_load_reduction=55694 max_load_reduction_pct=0.849823 moved_to_echo=34376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[384, 128, 512, 128, 256, 128, 12288, 128, 24960, 20480, 7040, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[26240, 1152, 256, 5504, 128, 20736, 1536, 9472, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6912, 26880, 8192, 128, 128, 128, 128, 23552, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12160, 128, 11008, 19840, 128, 1280, 11392, 10240, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6656, 1408, 128, 13184, 16128, 16896, 128, 11520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 512, 1536, 16256, 13696, 128, 5120, 16512, 9728, 2048, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 8064, 4992, 4864, 12672, 14848, 19584, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7808, 256, 2048, 22656, 17024, 128, 11520, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4087, 1, 0, 0, 25, 4079, 2, 4095, 3, 135, 0, 4, 0, 0, 0, 4093, 3718, 1, 2, 0, 1, 0, 0, 4029, 4, 1, 4093, 0, 4096, 4091, 0, 6, 0, 4, 4095, 0, 2, 4090, 4092, 722, 53, 3, 1, 4091, 0, 1, 0, 33, 0, 0, 3576, 0, 4, 0, 8, 0, 0, 1, 3, 4095, 2, 4092, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4085, 0, 0, 2, 23, 4070, 0, 4092, 0, 511, 2, 0, 0, 0, 1, 4094, 2375, 2, 0, 73, 0, 0, 0, 4093, 2, 1, 4094, 0, 4094, 4092, 0, 135, 0, 0, 4093, 1, 0, 4093, 4090, 1052, 8, 2, 2, 4091, 0, 0, 0, 53, 1, 0, 4008, 4, 2, 2, 3, 0, 4, 0, 3, 4090, 3, 4086, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 3, 0, 0, 2, 4052, 0, 4096, 0, 365, 0, 10, 0, 0, 3, 3933, 3182, 0, 0, 59, 3, 0, 0, 4047, 0, 0, 4090, 66, 4096, 4084, 0, 231, 0, 3, 4078, 0, 31, 4089, 4093, 524, 1, 0, 0, 4096, 18, 0, 0, 5, 0, 0, 4048, 0, 0, 0, 0, 0, 21, 2, 3, 4013, 0, 4093, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4087, 0, 0, 1, 7, 4077, 0, 4096, 0, 266, 2, 1, 0, 0, 0, 4096, 3023, 1, 0, 12, 1, 0, 1, 4093, 1, 0, 4096, 0, 4094, 4082, 0, 73, 0, 0, 4094, 2, 2, 4093, 4094, 729, 13, 2, 0, 4096, 0, 0, 1, 119, 0, 0, 3982, 2, 2, 0, 2, 0, 1, 5, 0, 4094, 0, 4093, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4073, 4, 3, 0, 3, 4074, 0, 4090, 0, 100, 4, 3, 0, 1, 0, 4092, 3983, 1, 3, 2, 0, 0, 4, 4096, 1, 4, 4090, 0, 4091, 4094, 0, 21, 2, 0, 4092, 2, 0, 4090, 4088, 217, 15, 3, 0, 4091, 1, 2, 4, 6, 1, 1, 3869, 0, 0, 0, 0, 1, 16, 5, 3, 4095, 1, 4090, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4084, 0, 1, 2, 36, 4087, 3, 4086, 6, 214, 1, 16, 0, 0, 8, 4057, 2621, 3, 1, 474, 0, 1, 11, 4092, 3, 1, 4088, 0, 4089, 4037, 1, 63, 6, 13, 4084, 4, 0, 4084, 4092, 1042, 1, 0, 6, 4092, 0, 1, 0, 32, 7, 10, 3820, 8, 0, 3, 0, 0, 0, 0, 0, 4079, 0, 4060, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 14, 4094, 0, 4095, 0, 318, 0, 0, 1, 1, 1, 4096, 3426, 0, 0, 75, 0, 0, 0, 4095, 0, 1, 4096, 1, 4095, 4085, 1, 244, 1, 0, 4095, 0, 0, 4095, 4096, 433, 2, 0, 0, 4096, 0, 0, 0, 61, 0, 0, 3661, 0, 6, 0, 0, 0, 0, 3, 0, 4096, 0, 4057, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4082, 0, 0, 0, 2, 4079, 0, 4092, 0, 389, 2, 1, 0, 0, 0, 4087, 2703, 0, 0, 2, 0, 0, 0, 3966, 5, 2, 4096, 0, 4096, 4094, 0, 106, 0, 0, 4092, 0, 3, 4095, 4095, 1143, 3, 2, 0, 4094, 0, 0, 0, 75, 1, 6, 3929, 0, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4087, 1, 0, 0, 25, 4079, 2, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3, 135, 0, 4, 0, 0, 0, 4093, 3829, 0, 0, 0, 0, 0, 0, 0, 3718, 1, 2, 0, 1, 0, 0, 4029, 731, 90, 85, 0, 0, 0, 0, 0, 4, 1, 3989, 0, 0, 4091, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 2, 3359, 0, 722, 0, 0, 0, 0, 0, 0, 0, 0, 53, 3, 1, 4091, 0, 1, 0, 33, 3866, 167, 0, 0, 0, 0, 0, 0, 0, 0, 3576, 0, 4, 0, 8, 0, 4096, 226, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4095, 2, 4092, 1, 1, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8064, 8657, 8091, 8182, 8215, 7910, 8209] avg=8192.0000 max=8657 max_rank=2 min=7910 min_rank=6 imbalance_ratio=1.056763 max_load_reduction=56879 max_load_reduction_pct=0.867905 moved_to_echo=32516 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 0, 1, 7, 4077, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 2, 1, 0, 0, 0, 4096, 3830, 0, 0, 0, 0, 0, 0, 0, 3023, 1, 0, 12, 1, 0, 1, 4093, 732, 90, 85, 0, 0, 0, 0, 0, 1, 0, 3992, 0, 0, 4082, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 2, 3361, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2, 0, 4096, 0, 0, 1, 119, 3866, 167, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 2, 2, 0, 2, 0, 4094, 228, 0, 0, 0, 0, 0, 0, 1, 5, 0, 4094, 0, 4093, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8195, 8038, 8148, 8188, 8264, 8310, 8207] avg=8192.0000 max=8310 max_rank=6 min=8038 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32969 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 0, 2, 23, 4070, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 2, 0, 0, 0, 1, 4094, 3826, 0, 0, 0, 0, 0, 0, 0, 2375, 2, 0, 73, 0, 0, 0, 4093, 732, 90, 85, 0, 0, 0, 0, 0, 2, 1, 3990, 0, 0, 4092, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 3361, 0, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 2, 4091, 0, 0, 0, 53, 3863, 166, 0, 0, 0, 0, 0, 0, 1, 0, 4008, 4, 2, 2, 3, 0, 4094, 227, 0, 0, 0, 0, 0, 0, 4, 0, 3, 4090, 3, 4086, 0, 4, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8434, 7450, 8220, 8507, 8185, 8341, 8204] avg=8192.0000 max=8507 max_rank=4 min=7450 min_rank=2 imbalance_ratio=1.038452 max_load_reduction=57029 max_load_reduction_pct=0.870193 moved_to_echo=33237 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 1, 2, 36, 4087, 3, 14, 0, 0, 0, 0, 0, 0, 0, 0, 6, 214, 1, 16, 0, 0, 8, 4057, 3821, 0, 0, 0, 0, 0, 0, 0, 2621, 3, 1, 474, 0, 1, 11, 4092, 730, 90, 85, 0, 0, 0, 0, 0, 3, 1, 3984, 0, 0, 4037, 1, 63, 0, 0, 0, 0, 0, 0, 0, 0, 6, 13, 4084, 4, 0, 3354, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 4092, 0, 1, 0, 32, 3865, 166, 0, 0, 0, 0, 0, 0, 7, 10, 3820, 8, 0, 3, 0, 0, 4089, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4060, 0, 6, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8123, 8108, 8089, 8503, 8163, 8164, 8159] avg=8192.0000 max=8503 max_rank=4 min=8089 min_rank=3 imbalance_ratio=1.037964 max_load_reduction=57033 max_load_reduction_pct=0.870255 moved_to_echo=32989 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4073, 4, 3, 0, 3, 4074, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 4, 3, 0, 1, 0, 4092, 3824, 0, 0, 0, 0, 0, 0, 0, 3983, 1, 3, 2, 0, 0, 4, 4096, 731, 90, 85, 0, 0, 0, 0, 0, 1, 4, 3986, 0, 0, 4094, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 2, 0, 3359, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3, 0, 4091, 1, 2, 4, 6, 3862, 166, 0, 0, 0, 0, 0, 0, 1, 1, 3869, 0, 0, 0, 0, 1, 4091, 226, 0, 0, 0, 0, 0, 0, 16, 5, 3, 4095, 1, 4090, 0, 4, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8024, 8995, 8106, 7672, 8150, 8189, 8228] avg=8192.0000 max=8995 max_rank=2 min=7672 min_rank=4 imbalance_ratio=1.098022 max_load_reduction=56541 max_load_reduction_pct=0.862747 moved_to_echo=32239 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 14, 4094, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 1, 1, 1, 4096, 3829, 0, 0, 0, 0, 0, 0, 0, 3426, 0, 0, 75, 0, 0, 0, 4095, 732, 90, 85, 0, 0, 0, 0, 0, 0, 1, 3992, 1, 0, 4085, 1, 244, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 3363, 0, 433, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 61, 3869, 167, 0, 0, 0, 0, 0, 0, 0, 0, 3661, 0, 6, 0, 0, 0, 4095, 227, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4057, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8246, 8503, 8324, 7892, 8195, 7989, 8170] avg=8192.0000 max=8503 max_rank=2 min=7892 min_rank=4 imbalance_ratio=1.037964 max_load_reduction=57033 max_load_reduction_pct=0.870255 moved_to_echo=32246 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 3, 0, 0, 2, 4052, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 0, 10, 0, 0, 3, 3933, 3830, 0, 0, 0, 0, 0, 0, 0, 3182, 0, 0, 59, 3, 0, 0, 4047, 731, 90, 85, 0, 0, 0, 0, 0, 0, 0, 3986, 66, 0, 4084, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4078, 0, 31, 3358, 0, 524, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 18, 0, 0, 5, 3865, 167, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 4096, 228, 0, 0, 0, 0, 0, 0, 21, 2, 3, 4013, 0, 4093, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8141, 8197, 8367, 7994, 8152, 8372, 8146] avg=8192.0000 max=8372 max_rank=6 min=7994 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32664 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4082, 0, 0, 0, 2, 4079, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 389, 2, 1, 0, 0, 0, 4087, 3830, 0, 0, 0, 0, 0, 0, 0, 2703, 0, 0, 2, 0, 0, 0, 3966, 735, 91, 91, 0, 0, 0, 0, 0, 5, 2, 3984, 0, 0, 4094, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3, 3360, 0, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 4094, 0, 0, 0, 75, 3868, 170, 0, 0, 0, 0, 0, 0, 1, 6, 3929, 0, 2, 0, 0, 0, 4096, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 1, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8309, 7588, 8191, 8598, 8212, 8261, 8213] avg=8192.0000 max=8598 max_rank=4 min=7588 min_rank=2 imbalance_ratio=1.049561 max_load_reduction=56938 max_load_reduction_pct=0.868805 moved_to_echo=33284 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 128, 26880, 0, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32000, 128, 0, 32768, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 2304, 128, 128, 128, 128, 128, 32640, 30720, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[25088, 128, 128, 768, 128, 128, 128, 32512, 5888, 768, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 384, 30976, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 30976, 128, 128, 128, 128, 128, 32768, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 2, 4096, 5, 0, 4095, 0, 1, 3, 2939, 0, 0, 0, 3, 7, 0, 1, 3815, 1, 2, 0, 4093, 0, 0, 3, 0, 4091, 3, 0, 0, 4, 1, 4091, 0, 0, 4094, 1, 4092, 0, 0, 3544, 4093, 0, 1, 1984, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 4095, 4093, 0, 0, 4090, 1, 1, 4091, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 1, 4095, 0, 0, 0, 2904, 1, 3, 0, 0, 5, 0, 1, 4062, 1, 1, 0, 4095, 1, 28, 0, 0, 4092, 0, 0, 0, 2, 0, 4093, 0, 3, 4095, 0, 4095, 1, 1, 1588, 4092, 82, 1, 3800, 0, 4, 0, 0, 0, 0, 1, 4096, 0, 4095, 3914, 0, 0, 4091, 0, 1, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4, 0, 0, 4092, 0, 0, 4092, 1, 0, 0, 3232, 5, 0, 4, 1, 0, 3, 0, 3506, 1, 2, 4, 4090, 0, 0, 0, 0, 4092, 4, 4, 0, 0, 1, 4090, 1, 4, 4092, 0, 4093, 4, 4, 3274, 4092, 12, 2, 2253, 2, 0, 0, 0, 0, 3, 4, 4091, 3, 4092, 4088, 2, 5, 4095, 0, 1, 4091, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 15, 1535, 0, 0, 0, 0, 0, 0, 0, 3570, 0, 0, 0, 4096, 0, 21, 3, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 16, 4096, 0, 4096, 0, 0, 3881, 4079, 166, 0, 3296, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3904, 0, 3, 4091, 2, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 2, 4093, 0, 0, 4094, 2, 0, 0, 3033, 0, 0, 2, 0, 0, 0, 1, 3913, 0, 1, 0, 4094, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 3, 4091, 3, 1, 4094, 3, 4094, 1, 1, 2888, 4096, 24, 0, 2452, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 4095, 4068, 1, 0, 4094, 0, 2, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 3647, 0, 1, 0, 0, 1, 0, 0, 3774, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 2, 4096, 0, 0, 2925, 4095, 3, 0, 1940, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4093, 0, 0, 4095, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[2, 8, 3, 4095, 0, 0, 4095, 0, 4, 1, 3626, 0, 4, 0, 0, 4, 0, 2, 3952, 3, 1, 0, 4095, 0, 1, 8, 0, 4091, 3, 3, 0, 0, 0, 4096, 0, 1, 4095, 1, 4092, 0, 0, 3832, 4089, 4, 0, 867, 3, 21, 0, 0, 6, 0, 0, 4095, 0, 4089, 4067, 0, 0, 4086, 0, 3, 4088, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3689, 0, 0, 0, 0, 0, 0, 0, 2496, 0, 0, 0, 4096, 0, 4, 4, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 5, 4096, 0, 4096, 0, 0, 3524, 4096, 11, 0, 2726, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3937, 0, 0, 4089, 0, 0, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 2, 4096, 5, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 2939, 0, 0, 0, 3, 7, 4094, 1012, 0, 0, 0, 0, 0, 0, 0, 1, 3815, 1, 2, 0, 4093, 0, 454, 2, 1, 0, 0, 0, 0, 0, 0, 3, 0, 4091, 3, 0, 0, 4, 3001, 1084, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3544, 2555, 0, 1, 1984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4090, 1, 1, 78, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8059, 8369, 8186, 8183, 8084, 8190, 8266] avg=8192.0000 max=8369 max_rank=2 min=8059 min_rank=1 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2904, 1, 3, 0, 0, 5, 4095, 1013, 0, 0, 0, 0, 0, 0, 0, 1, 4062, 1, 1, 0, 4095, 1, 454, 2, 1, 0, 0, 0, 0, 0, 28, 0, 0, 4092, 0, 0, 0, 2, 3004, 1084, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1588, 2554, 82, 1, 3800, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 4091, 0, 1, 78, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8021, 8618, 8210, 8190, 8030, 8192, 8085] avg=8192.0000 max=8618 max_rank=2 min=8021 min_rank=1 imbalance_ratio=1.052002 max_load_reduction=56918 max_load_reduction_pct=0.868500 moved_to_echo=32497 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 4093, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3033, 0, 0, 2, 0, 0, 4094, 1013, 0, 0, 0, 0, 0, 0, 0, 1, 3913, 0, 1, 0, 4094, 0, 455, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 3003, 1085, 0, 0, 0, 0, 0, 0, 3, 4091, 3, 1, 0, 3, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2888, 2556, 24, 0, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 1, 0, 4094, 0, 2, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8142, 8467, 8186, 8194, 7921, 8192, 8243] avg=8192.0000 max=8467 max_rank=2 min=7921 min_rank=5 imbalance_ratio=1.033569 max_load_reduction=57069 max_load_reduction_pct=0.870804 moved_to_echo=32550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1535, 0, 0, 0, 0, 0, 4096, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 3570, 0, 0, 0, 4096, 0, 453, 2, 1, 0, 0, 0, 0, 0, 21, 3, 0, 4090, 0, 0, 0, 0, 3004, 1081, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 2545, 166, 0, 3296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 3, 4091, 2, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 6660, 8122, 8199, 8206, 9888, 8192, 8078] avg=8192.0000 max=9888 max_rank=5 min=6660 min_rank=1 imbalance_ratio=1.207031 max_load_reduction=55648 max_load_reduction_pct=0.849121 moved_to_echo=34364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 4092, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3232, 5, 0, 4, 1, 0, 4092, 1012, 0, 0, 0, 0, 0, 0, 3, 0, 3506, 1, 2, 4, 4090, 0, 454, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 4, 0, 0, 3001, 1084, 0, 0, 0, 0, 0, 0, 1, 4090, 1, 4, 0, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3274, 2554, 12, 2, 2253, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4091, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 5, 4095, 0, 1, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8346, 8063, 8185, 8191, 8101, 8193, 8269] avg=8192.0000 max=8346 max_rank=1 min=8063 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 8, 3, 4095, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 3626, 0, 4, 0, 0, 4, 4095, 1012, 0, 0, 0, 0, 0, 0, 0, 2, 3952, 3, 1, 0, 4095, 0, 454, 2, 1, 0, 0, 0, 0, 0, 1, 8, 0, 4091, 3, 3, 0, 0, 2998, 1083, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3832, 2552, 4, 0, 867, 3, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4095, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 4086, 0, 3, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8746, 8510, 8187, 8188, 7279, 8190, 8234] avg=8192.0000 max=8746 max_rank=1 min=7279 min_rank=5 imbalance_ratio=1.067627 max_load_reduction=56790 max_load_reduction_pct=0.866547 moved_to_echo=31891 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3647, 0, 1, 0, 0, 1, 4096, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 0, 0, 0, 4096, 0, 455, 2, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3004, 1085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 2555, 3, 0, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4095, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8759, 8328, 8186, 8193, 7423, 8191, 8266] avg=8192.0000 max=8759 max_rank=1 min=7423 min_rank=5 imbalance_ratio=1.069214 max_load_reduction=56777 max_load_reduction_pct=0.866348 moved_to_echo=32073 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 0, 0, 0, 0, 0, 4096, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 2496, 0, 0, 0, 4096, 0, 458, 6, 3, 0, 0, 0, 0, 0, 4, 4, 0, 4093, 0, 0, 0, 0, 3007, 1089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 2549, 11, 0, 2726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 3937, 0, 0, 4089, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8803, 7059, 8197, 8191, 8810, 8196, 8095] avg=8192.0000 max=8810 max_rank=5 min=7059 min_rank=2 imbalance_ratio=1.075439 max_load_reduction=56726 max_load_reduction_pct=0.865570 moved_to_echo=33292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 24704, 128, 128, 128, 128, 128, 32768, 8192, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 25472, 20480, 384, 128, 19328, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32256, 128, 128, 32768, 128, 128, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 128, 24064, 8704, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 29184, 128, 128, 128, 32768, 128, 3712, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66688 slack_slots=3584 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 3, 1, 1, 0, 0, 0, 1, 1, 0, 2, 0, 0, 0, 0, 4095, 1, 1, 0, 4088, 1, 0, 0, 0, 4093, 0, 1, 0, 1, 2, 1, 4089, 0, 4096, 4095, 0, 0, 0, 4095, 0, 143, 0, 0, 4092, 4095, 4095, 4095, 1, 0, 4095, 4095, 4095, 1, 3, 4095, 17, 3943, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[2, 2, 0, 1, 0, 1, 1, 0, 2, 0, 1, 1, 1, 0, 0, 0, 0, 1, 4094, 0, 2, 1, 4094, 2, 0, 0, 1, 4094, 2, 2, 0, 11, 0, 1, 4094, 0, 4094, 4093, 1, 1, 0, 4095, 1, 21, 0, 0, 4082, 4094, 4094, 4091, 2, 1, 4085, 4094, 4094, 0, 0, 4094, 0, 4088, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 2, 0, 0, 4094, 6, 0, 0, 4091, 0, 0, 2, 0, 4085, 0, 1, 0, 0, 4, 0, 4089, 0, 4092, 4096, 2, 1, 0, 4092, 0, 19, 0, 0, 4080, 4096, 4094, 4095, 0, 0, 4096, 4094, 4095, 0, 4, 4094, 5, 4090, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 4096, 1, 0, 0, 4012, 0, 0, 0, 0, 4087, 0, 0, 0, 19, 15, 0, 3989, 0, 4096, 4096, 0, 0, 0, 4096, 0, 657, 0, 0, 3996, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 65, 4096, 109, 3444, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 12, 0, 0, 0, 1, 0, 0, 0, 1, 4, 0, 6, 2, 0, 4095, 0, 0, 0, 4071, 0, 0, 7, 9, 4084, 0, 0, 0, 52, 8, 0, 4077, 3, 4092, 4096, 0, 2, 0, 4085, 0, 171, 0, 0, 4082, 4096, 4094, 4092, 0, 0, 4096, 4088, 4096, 2, 13, 4089, 9, 3899, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 0, 0, 4086, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4095, 0, 4096, 4096, 0, 0, 0, 4095, 0, 85, 0, 0, 4090, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1, 4095, 10, 4016, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 1, 5, 7, 0, 2, 4, 1, 4, 0, 0, 0, 0, 6, 4085, 0, 5, 5, 4084, 0, 0, 0, 1, 4087, 0, 5, 2, 22, 0, 3, 4091, 8, 4091, 4087, 0, 4, 0, 4091, 1, 2, 0, 0, 4092, 4089, 4090, 4092, 5, 0, 4073, 4091, 4091, 5, 0, 4091, 14, 4090, 0, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4095, 2, 0, 0, 4074, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 2, 0, 4086, 0, 4096, 4096, 0, 0, 0, 4095, 0, 169, 0, 0, 4090, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 9, 4096, 23, 3928, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 3, 1, 1, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2, 0, 0, 4095, 4077, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 4088, 1, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 1, 4072, 13, 0, 0, 0, 0, 0, 0, 2, 1, 4089, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 0, 143, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 4095, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 17, 3943, 0, 0, 0, 1, 116, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8180, 8204, 8180, 8203, 8196, 8197, 8183] avg=8192.0000 max=8204 max_rank=2 min=8180 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=57332 max_load_reduction_pct=0.874817 moved_to_echo=32779 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 0, 1, 0, 1, 1, 0, 4094, 4089, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 1, 0, 0, 0, 4094, 4073, 5, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 2, 1, 4094, 2, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 2, 2, 0, 11, 4070, 13, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 4094, 15, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 1, 21, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 1, 4085, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4088, 0, 0, 0, 0, 116, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8177, 8212, 8193, 8206, 8065, 8187, 8306] avg=8192.0000 max=8306 max_rank=7 min=8065 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32764 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 2, 4094, 4077, 5, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 4091, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4085, 0, 1, 0, 0, 4073, 13, 0, 0, 0, 0, 0, 0, 4, 0, 4089, 0, 4092, 15, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 19, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 5, 4090, 0, 1, 0, 0, 116, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8189, 8209, 8174, 8203, 8057, 8195, 8318] avg=8192.0000 max=8318 max_rank=7 min=8057 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32773 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 12, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4, 0, 6, 4094, 4074, 5, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 4071, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 4084, 0, 0, 0, 52, 4073, 13, 0, 0, 0, 0, 0, 0, 8, 0, 4077, 3, 4092, 15, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 0, 171, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4088, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4089, 9, 3899, 1, 0, 0, 0, 116, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8185, 8186, 8238, 8197, 8204, 8191, 8135] avg=8192.0000 max=8238 max_rank=3 min=8135 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32727 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4078, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4086, 0, 18, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4073, 13, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 0, 85, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 10, 4016, 0, 0, 0, 0, 116, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8179, 8200, 8183, 8207, 8136, 8197, 8246] avg=8192.0000 max=8246 max_rank=7 min=8136 min_rank=5 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32786 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 1, 5, 7, 4091, 4085, 0, 0, 0, 0, 0, 0, 0, 2, 4, 1, 4, 0, 0, 0, 4090, 4074, 4, 0, 0, 0, 0, 0, 0, 6, 4085, 0, 5, 5, 4084, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 5, 2, 22, 4064, 13, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 8, 4091, 15, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 1, 2, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 4073, 4091, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 14, 4090, 0, 0, 0, 5, 116, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8179, 8203, 8194, 8212, 8052, 8179, 8324] avg=8192.0000 max=8324 max_rank=7 min=8052 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32767 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 4096, 4078, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4012, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 19, 4073, 13, 0, 0, 0, 0, 0, 0, 15, 0, 3989, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 657, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4096, 109, 3444, 0, 0, 0, 0, 116, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8263, 8127, 8192, 8115, 8615, 8197, 7838] avg=8192.0000 max=8615 max_rank=5 min=7838 min_rank=7 imbalance_ratio=1.051636 max_load_reduction=56921 max_load_reduction_pct=0.868546 moved_to_echo=32765 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4079, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 4074, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4076, 16, 0, 0, 0, 0, 0, 0, 2, 0, 4086, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 0, 169, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 23, 3928, 0, 0, 0, 0, 119, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8184, 8195, 8182, 8193, 8211, 8193, 8186] avg=8192.0000 max=8211 max_rank=5 min=8182 min_rank=3 imbalance_ratio=1.002319 max_load_reduction=57325 max_load_reduction_pct=0.874710 moved_to_echo=32783 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 31744, 128, 1280, 0, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 32768, 32640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 32640, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 256, 31616, 128, 128, 0, 128, 1024, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32640, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4095, 4094, 4091, 1, 0, 1, 2, 1, 4094, 2, 4093, 1, 2, 2, 0, 2, 0, 0, 1, 4094, 17, 1, 4, 0, 2, 4095, 0, 0, 4092, 3, 0, 4076, 4094, 4094, 0, 0, 4094, 4, 0, 1, 0, 4094, 0, 2, 4091, 2, 0, 0, 0, 1, 1, 0, 4093, 0, 0, 1, 0, 0, 4094, 0, 0, 2, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4096, 4095, 4027, 0, 0, 0, 1, 1, 4093, 0, 4094, 1, 1, 1, 0, 2, 0, 0, 0, 4093, 2, 0, 4, 0, 2, 4093, 1, 0, 4095, 0, 1, 4081, 4095, 4095, 0, 0, 4095, 4, 0, 1, 84, 4095, 1, 1, 4089, 1, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 1, 0, 1, 4093, 0, 0, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4040, 4096, 4088, 9, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 38, 0, 0, 0, 0, 4070, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 18, 0, 4096, 0, 93, 0, 61, 4096, 1, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4088, 4096, 4092, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 12, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 6, 4095, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4079, 4096, 4080, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 4096, 4095, 4096, 4, 3, 4096, 3, 0, 0, 129, 4091, 0, 0, 3999, 0, 0, 0, 0, 2, 0, 1, 4096, 3, 5, 0, 0, 0, 4089, 4, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4091, 4090, 4065, 0, 0, 0, 5, 3, 4092, 4, 4089, 5, 5, 6, 0, 3, 0, 5, 4, 4092, 155, 0, 8, 0, 2, 4091, 0, 0, 4084, 9, 0, 4075, 4089, 4090, 0, 0, 4090, 13, 0, 0, 6, 4091, 0, 5, 3953, 4, 5, 0, 0, 0, 5, 0, 4089, 0, 0, 6, 0, 4, 4092, 0, 0, 5, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4057, 4096, 4059, 9, 0, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 4067, 0, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 0, 4095, 4096, 4096, 12, 6, 4094, 0, 28, 0, 108, 4084, 0, 0, 4066, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4088, 4096, 4033, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 8, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4095, 4096, 4096, 0, 0, 4096, 0, 4, 0, 106, 4096, 4, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 42, 4027, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4094, 1, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 4, 0, 4041, 22, 21, 0, 0, 0, 0, 0, 2, 4093, 1, 0, 12, 0, 1, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4074, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 84, 4073, 1, 1, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 1, 4081, 12, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 1, 0, 1, 4095, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8192, 8183, 8190, 8174, 8249, 8188, 8193] avg=8192.0000 max=8249 max_rank=5 min=8167 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4095, 42, 4091, 1, 0, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 4093, 1, 2, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 17, 1, 4, 0, 4040, 22, 21, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 12, 3, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4073, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 2, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4093, 0, 0, 1, 4078, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 2, 4094, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8196, 8200, 8188, 8172, 8167, 8186, 8194] avg=8192.0000 max=8233 max_rank=0 min=8167 min_rank=5 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4079, 42, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 0, 4042, 22, 21, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 3, 4075, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 4069, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4096, 3, 5, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 4, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8187, 8175, 8200, 8180, 8197, 8201, 8195] avg=8192.0000 max=8201 max_rank=0 min=8175 min_rank=2 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4040, 42, 4088, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 38, 0, 0, 0, 4042, 22, 21, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 4075, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4074, 1, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8191, 8115, 8178, 8282, 8219, 8190, 8182] avg=8192.0000 max=8282 max_rank=4 min=8115 min_rank=2 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32873 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4057, 42, 4059, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4067, 0, 0, 0, 0, 4042, 22, 21, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 12, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 6, 4073, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 4062, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8183, 8156, 8191, 8215, 8236, 8202, 8186] avg=8192.0000 max=8236 max_rank=5 min=8156 min_rank=2 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32839 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4088, 42, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 12, 0, 0, 0, 4042, 22, 21, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4073, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 1, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8192, 8187, 8203, 8171, 8175, 8192, 8194] avg=8192.0000 max=8222 max_rank=0 min=8171 min_rank=4 imbalance_ratio=1.003662 max_load_reduction=57314 max_load_reduction_pct=0.874542 moved_to_echo=32732 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4091, 42, 4065, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 4089, 5, 5, 6, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 4092, 155, 0, 8, 0, 4036, 22, 21, 0, 0, 0, 0, 0, 2, 4091, 0, 0, 12, 9, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4069, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4069, 0, 5, 3953, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4089, 0, 0, 6, 4070, 12, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 0, 0, 5, 0, 6, 4090, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8204, 8343, 8189, 8171, 8042, 8182, 8199] avg=8192.0000 max=8343 max_rank=2 min=8042 min_rank=5 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32594 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4088, 39, 4033, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 8, 0, 0, 0, 4043, 23, 25, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 7, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 4073, 4, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8191, 8177, 8197, 8171, 8251, 8195, 8193] avg=8192.0000 max=8251 max_rank=5 min=8161 min_rank=0 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32810 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 384, 32640, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32640, 256, 128, 128, 0, 32384, 256, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32768, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 32640, 128, 128, 32512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 32640, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[2, 0, 4095, 0, 0, 1, 0, 0, 4091, 0, 0, 1, 4095, 0, 0, 3, 2384, 1, 1, 4094, 0, 1, 4095, 1, 4095, 4095, 1, 0, 3098, 0, 0, 1, 4095, 4095, 0, 1, 0, 3, 4, 232, 4077, 1, 0, 4095, 0, 1, 2174, 4095, 1, 0, 0, 298, 0, 0, 4094, 1, 4095, 1, 0, 0, 0, 17, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4096, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 4096, 0, 0, 21, 1458, 0, 0, 4095, 1, 0, 4096, 0, 4096, 4089, 6, 0, 3858, 0, 0, 0, 4096, 4088, 0, 0, 0, 4, 0, 466, 4085, 0, 0, 4096, 0, 0, 2361, 4094, 0, 0, 0, 32, 0, 0, 4096, 0, 4096, 0, 0, 1, 18, 3, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[3, 0, 4094, 0, 0, 2, 0, 1, 4093, 0, 0, 2, 4091, 0, 0, 0, 2015, 2, 1, 4091, 0, 2, 4092, 0, 4094, 4094, 0, 1, 3420, 0, 0, 2, 4094, 4094, 3, 2, 1, 0, 2, 277, 4096, 2, 0, 4094, 0, 4, 2139, 4094, 2, 1, 0, 340, 0, 0, 4093, 2, 4094, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 3, 1454, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4095, 0, 0, 3829, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 170, 4090, 0, 0, 4096, 0, 0, 2726, 4096, 0, 0, 0, 9, 0, 0, 4096, 0, 4096, 0, 0, 0, 7, 2, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 34, 518, 0, 0, 4045, 0, 0, 4096, 0, 4096, 4096, 4, 0, 3941, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 38, 4044, 0, 0, 4096, 0, 0, 3548, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 3, 100, 114, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 4096, 0, 0, 15, 830, 0, 0, 4086, 6, 0, 4096, 0, 4096, 4084, 10, 0, 4027, 0, 0, 0, 4096, 4089, 11, 0, 0, 3, 0, 258, 4075, 0, 0, 4096, 0, 0, 3029, 4092, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 43, 25, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4, 0, 4089, 0, 0, 4, 0, 0, 4088, 0, 0, 5, 4085, 5, 0, 0, 2887, 6, 2, 4070, 0, 6, 4087, 6, 4090, 4092, 0, 0, 3894, 1, 0, 7, 4090, 4091, 0, 4, 0, 0, 6, 114, 4092, 4, 0, 4090, 0, 10, 1273, 4090, 5, 3, 0, 34, 0, 5, 4089, 6, 4089, 6, 4, 0, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 0, 11, 194, 0, 0, 4089, 0, 0, 4096, 0, 4096, 4095, 0, 0, 3966, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 269, 4084, 0, 0, 4096, 0, 0, 3746, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 3, 4, 26, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 4094, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 0, 4095, 1, 0, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 4089, 6, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 4088, 0, 0, 0, 4, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 2361, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 4096, 0, 2315, 1458, 229, 2, 0, 0, 0, 0, 4096, 0, 0, 1, 18, 3, 0, 8, 3754, 294, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8194, 8192, 8295, 8425, 7931, 8132, 8174] avg=8192.0000 max=8425 max_rank=4 min=7931 min_rank=5 imbalance_ratio=1.028442 max_load_reduction=57111 max_load_reduction_pct=0.871445 moved_to_echo=32662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4094, 0, 0, 2, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2015, 2, 1, 4091, 0, 2, 2636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 4094, 0, 1, 3420, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 4094, 3, 2, 1, 0, 2, 277, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4, 2139, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 340, 0, 0, 4093, 2, 2315, 1456, 229, 2, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 3752, 294, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8184, 8747, 7859, 8244, 7726, 8440, 8142] avg=8192.0000 max=8747 max_rank=2 min=7726 min_rank=5 imbalance_ratio=1.067749 max_load_reduction=56789 max_load_reduction_pct=0.866531 moved_to_echo=32552 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4095, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2384, 1, 1, 4094, 0, 1, 2638, 1, 0, 0, 0, 0, 0, 0, 0, 0, 342, 4095, 1, 0, 3098, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 4095, 0, 1, 0, 3, 4, 232, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 1, 0, 0, 0, 1, 2174, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 298, 0, 0, 4094, 1, 2315, 1457, 229, 2, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 17, 1, 0, 3753, 294, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8188, 9120, 7537, 8201, 7739, 8397, 8161] avg=8192.0000 max=9120 max_rank=2 min=7537 min_rank=3 imbalance_ratio=1.113281 max_load_reduction=56416 max_load_reduction_pct=0.860840 moved_to_echo=32498 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 4094, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 830, 0, 0, 4086, 6, 0, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 4084, 10, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 4089, 11, 0, 0, 3, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 3029, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 2314, 1458, 229, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 43, 25, 0, 4, 3754, 294, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8189, 7560, 8463, 8228, 8588, 8100, 8216] avg=8192.0000 max=8588 max_rank=5 min=7560 min_rank=2 imbalance_ratio=1.048340 max_load_reduction=56948 max_load_reduction_pct=0.868958 moved_to_echo=33132 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1454, 0, 0, 4096, 0, 0, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 4095, 0, 0, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 4096, 0, 0, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 2726, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 2316, 1458, 229, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 2, 0, 1, 3754, 294, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8188, 8266, 8133, 8302, 8110, 8154] avg=8192.0000 max=8302 max_rank=5 min=8110 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32699 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4094, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 0, 4045, 0, 0, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 4096, 4, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 4096, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 3548, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 2316, 1458, 229, 2, 0, 0, 0, 0, 4096, 0, 0, 3, 100, 114, 0, 0, 3754, 294, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8218, 7201, 8383, 8001, 9078, 8102, 8361] avg=8192.0000 max=9078 max_rank=5 min=7201 min_rank=2 imbalance_ratio=1.108154 max_load_reduction=56458 max_load_reduction_pct=0.861481 moved_to_echo=33542 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4089, 0, 0, 4, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 5, 4083, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2887, 6, 2, 4070, 0, 6, 2633, 6, 0, 0, 0, 0, 0, 0, 0, 0, 341, 4092, 0, 0, 3894, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 4091, 0, 4, 0, 0, 6, 114, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 10, 1273, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 34, 0, 5, 4089, 6, 2313, 1454, 229, 2, 0, 0, 0, 0, 4089, 6, 4, 0, 0, 0, 3, 0, 3749, 293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8181, 9610, 8335, 8076, 6863, 8140, 8144] avg=8192.0000 max=9610 max_rank=2 min=6863 min_rank=5 imbalance_ratio=1.173096 max_load_reduction=55926 max_load_reduction_pct=0.853363 moved_to_echo=31223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4092, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 4089, 0, 0, 2635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 4095, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 4096, 0, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 3746, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 2320, 1461, 233, 4, 0, 0, 0, 0, 4096, 0, 0, 3, 4, 26, 0, 0, 3758, 297, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8191, 6918, 8398, 8228, 9309, 8115, 8184] avg=8192.0000 max=9309 max_rank=5 min=6918 min_rank=2 imbalance_ratio=1.136353 max_load_reduction=56227 max_load_reduction_pct=0.857956 moved_to_echo=33836 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11776, 128, 128, 32768, 128, 128, 21120, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 0, 0, 128, 21120, 11904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[30976, 32768, 128, 128, 128, 128, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 768, 0, 128, 32768, 128, 18560, 11776, 1920, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 256, 256, 128, 128, 30080, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2816, 32768, 128, 128, 30080, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4084, 0, 4096, 20, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 3537, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 4058, 4096, 4096, 4096, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 130, 1, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 474] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 1, 4094, 4091, 1, 4095, 3, 2, 1, 0, 0, 1, 1, 4092, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 4011, 1, 0, 1, 1, 0, 0, 3, 1, 1, 0, 4095, 0, 4086, 4095, 4095, 4094, 0, 1, 10, 1, 0, 4092, 1, 1, 0, 8, 0, 4095, 4094, 1, 1, 4095, 0, 4, 4094, 189] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[3, 0, 3, 2, 2, 1, 4090, 4094, 1, 4094, 0, 0, 2, 2, 0, 2, 0, 4091, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 3791, 2, 2, 2, 0, 2, 0, 0, 3, 2, 0, 4094, 0, 4094, 4093, 4081, 4094, 1, 3, 16, 2, 0, 4092, 6, 2, 0, 0, 2, 4094, 4094, 2, 2, 4093, 0, 0, 4093, 327] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[7, 0, 10, 6, 0, 8, 4084, 4090, 4, 4090, 0, 0, 4, 8, 0, 6, 0, 4086, 1, 0, 0, 8, 0, 4010, 0, 0, 0, 4083, 7, 0, 6, 0, 0, 0, 0, 8, 6, 0, 4090, 0, 4089, 4088, 3784, 4089, 1, 7, 0, 6, 0, 4088, 3, 5, 0, 0, 71, 4090, 4089, 6, 6, 4088, 0, 0, 4088, 316] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4095, 1, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 1, 4045, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4094, 4096, 4070, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 155] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4086, 0, 4090, 2, 0, 0, 0, 12, 0, 0, 4085, 0, 11, 0, 0, 4, 4070, 0, 0, 0, 3726, 0, 4, 0, 0, 0, 0, 1, 0, 0, 4, 4091, 0, 4080, 4093, 4087, 4093, 0, 0, 0, 0, 4, 4096, 0, 0, 1, 24, 2, 4092, 4096, 0, 0, 4096, 0, 15, 4096, 379] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4088, 0, 4092, 4, 0, 0, 0, 13, 0, 0, 4093, 0, 7, 0, 0, 3, 4076, 0, 0, 0, 3713, 0, 12, 0, 0, 0, 0, 7, 0, 0, 0, 4094, 0, 4087, 4093, 4096, 4092, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 3, 0, 4095, 4096, 0, 0, 4096, 0, 2, 4096, 386] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 4096, 4094, 0, 4095, 4, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 2, 4078, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4085, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 21, 0, 4096, 4096, 0, 0, 4096, 0, 4, 4096, 130] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 1, 4094, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 3, 2, 1, 0, 0, 1, 3807, 278, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 0, 0, 0, 3980, 30, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 1, 0, 1, 1, 4094, 239, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1, 0, 4095, 0, 3856, 236, 0, 0, 0, 0, 0, 0, 4086, 0, 4095, 51, 0, 1, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 0, 8, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3799, 0, 4, 4094, 189, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8188, 8121, 8347, 8192, 8244, 8165, 8088] avg=8192.0000 max=8347 max_rank=3 min=8088 min_rank=7 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32689 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 2, 2, 1, 4090, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 2, 2, 0, 2, 3807, 278, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4060, 30, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 2, 2, 2, 0, 4094, 237, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 2, 0, 4094, 0, 3856, 236, 0, 0, 0, 0, 0, 0, 4094, 0, 4081, 51, 1, 3, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 6, 2, 0, 0, 2, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3797, 0, 0, 4093, 327, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8186, 8199, 8128, 8193, 8248, 8164, 8221] avg=8192.0000 max=8248 max_rank=5 min=8128 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32826 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 0, 0, 0, 8, 0, 3809, 278, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4055, 30, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3537, 0, 0, 0, 0, 4096, 238, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 3858, 236, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 51, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 130, 1, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 0, 0, 4096, 474, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8211, 8199, 7871, 8196, 8216, 8291, 8370] avg=8192.0000 max=8370 max_rank=7 min=7871 min_rank=3 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=33073 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 13, 0, 3805, 278, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 7, 0, 0, 3, 4076, 30, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3713, 0, 12, 0, 0, 4096, 237, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4094, 0, 3856, 236, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 3, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 0, 2, 4096, 386, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8192, 8227, 8058, 8193, 8234, 8162, 8284] avg=8192.0000 max=8284 max_rank=7 min=8058 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32873 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4086, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 12, 0, 3806, 278, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 11, 0, 0, 4, 4070, 30, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 0, 4, 0, 0, 4096, 237, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 4091, 0, 3856, 236, 0, 0, 0, 0, 0, 0, 4080, 0, 4087, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 1, 24, 2, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 0, 15, 4096, 379, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8188, 8218, 8063, 8188, 8218, 8187, 8290] avg=8192.0000 max=8290 max_rank=7 min=8063 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32883 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 10, 6, 0, 8, 4084, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 4, 4090, 0, 0, 4, 8, 0, 6, 3802, 278, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 0, 0, 8, 0, 4010, 30, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 7, 0, 6, 0, 4089, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 4090, 0, 3850, 236, 0, 0, 0, 0, 0, 0, 4089, 0, 3784, 51, 1, 7, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 5, 0, 0, 71, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 3792, 0, 0, 4088, 316, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8192, 8153, 8423, 8190, 7938, 8225, 8208] avg=8192.0000 max=8423 max_rank=3 min=7938 min_rank=5 imbalance_ratio=1.028198 max_load_reduction=57113 max_load_reduction_pct=0.871475 moved_to_echo=32561 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 1, 0, 3809, 278, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4045, 30, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 4096, 239, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 3857, 236, 0, 0, 0, 0, 0, 0, 4094, 0, 4070, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 0, 0, 4096, 155, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8184, 8189, 8344, 8191, 8215, 8169, 8051] avg=8192.0000 max=8344 max_rank=3 min=8051 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32626 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 4096, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 1, 0, 3813, 282, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4078, 34, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 4096, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3857, 240, 0, 0, 0, 0, 0, 0, 4085, 0, 4095, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 21, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 0, 4, 4096, 130, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8195, 8230, 8302, 8193, 8223, 8173, 8024] avg=8192.0000 max=8302 max_rank=3 min=8024 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32613 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 0, 30848, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 32384, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 0, 32512, 512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 30464, 0, 128, 32768, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 256, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 30848, 128, 128, 128, 128, 32768, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 30464, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4070, 0, 1, 2, 0, 4095, 0, 0, 0, 27, 0, 0, 1575, 4096, 4095, 0, 0, 4096, 0, 4094, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 2522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4001, 1, 2, 1, 1, 4095, 1, 1, 0, 1, 1, 0, 411, 4095, 4092, 1, 0, 4094, 0, 4092, 0, 4095, 0, 4095, 0, 0, 0, 1, 1, 4095, 0, 0, 1, 0, 0, 3778, 0, 0, 2, 0, 1, 1, 0, 2, 0, 0, 1, 0, 4095, 1, 0, 1, 4095, 0, 4095, 1, 0, 1, 4095, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4022, 0, 2, 8, 0, 4091, 0, 0, 0, 60, 0, 0, 2344, 4096, 4089, 0, 0, 4096, 0, 4087, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 1, 4095, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 4039, 2, 0, 0, 2, 4094, 2, 2, 0, 6, 2, 0, 1235, 4093, 4094, 2, 0, 4093, 1, 4094, 0, 4094, 0, 4093, 1, 3, 9, 1, 1, 4094, 0, 0, 2, 0, 2, 2896, 0, 0, 0, 0, 2, 2, 0, 3, 1, 1, 2, 0, 4094, 2, 0, 2, 4094, 0, 4094, 2, 0, 1, 4093, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4081, 0, 1, 9, 0, 4083, 0, 0, 0, 26, 0, 0, 2321, 4096, 4089, 0, 0, 4096, 0, 4055, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4087, 0, 1, 0, 0, 0, 1818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 4095, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2, 0, 0, 4088, 4069, 6, 0, 0, 7, 4090, 7, 6, 0, 3, 7, 0, 1448, 4088, 4090, 8, 0, 4088, 1, 4089, 0, 4088, 4, 4087, 0, 2, 21, 5, 14, 4090, 0, 0, 10, 0, 4, 2625, 0, 0, 0, 0, 6, 6, 0, 3, 0, 1, 7, 0, 4087, 6, 0, 4, 4089, 0, 4089, 7, 1, 12, 4086, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3955, 0, 0, 0, 0, 4096, 0, 0, 0, 141, 0, 0, 2658, 4096, 4092, 0, 0, 4096, 0, 3994, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4060, 0, 0, 0, 0, 4096, 0, 0, 0, 61, 0, 0, 3416, 4096, 4092, 0, 0, 4096, 0, 4090, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4001, 1, 2, 1, 52, 2, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 1, 1, 0, 4053, 2, 0, 0, 0, 0, 0, 0, 411, 0, 2191, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 0, 3778, 1849, 41, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 2, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 1, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8155, 6695, 8189, 9765, 8195, 8192, 8191] avg=8192.0000 max=9765 max_rank=4 min=6695 min_rank=2 imbalance_ratio=1.192017 max_load_reduction=55771 max_load_reduction_pct=0.850998 moved_to_echo=34343 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 4039, 2, 0, 0, 52, 2, 0, 0, 0, 0, 0, 0, 2, 4094, 2, 2, 0, 6, 2, 0, 4051, 1, 0, 0, 0, 0, 0, 0, 1235, 0, 2192, 2, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4093, 1, 3, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 2, 0, 2, 2896, 1850, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 3, 4094, 4092, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 4094, 2, 0, 2, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8160, 7524, 8199, 8887, 8193, 8196, 8189] avg=8192.0000 max=8887 max_rank=4 min=7524 min_rank=2 imbalance_ratio=1.084839 max_load_reduction=56649 max_load_reduction_pct=0.864395 moved_to_echo=33465 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4022, 0, 2, 8, 52, 2, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 60, 0, 0, 4053, 1, 0, 0, 0, 0, 0, 0, 2344, 0, 2190, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 1777, 1847, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8205, 8621, 8190, 7761, 8191, 8193, 8193] avg=8192.0000 max=8621 max_rank=2 min=7761 min_rank=4 imbalance_ratio=1.052368 max_load_reduction=56915 max_load_reduction_pct=0.868454 moved_to_echo=32338 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4081, 0, 1, 9, 52, 2, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 26, 0, 0, 4053, 1, 0, 0, 0, 0, 0, 0, 2321, 0, 2190, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 1, 0, 0, 0, 1818, 1847, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8163, 8566, 8190, 7796, 8191, 8191, 8198] avg=8192.0000 max=8566 max_rank=2 min=7796 min_rank=4 imbalance_ratio=1.045654 max_load_reduction=56970 max_load_reduction_pct=0.869293 moved_to_echo=32376 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4070, 0, 1, 2, 52, 2, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 27, 0, 0, 4054, 2, 0, 0, 0, 0, 0, 0, 1575, 0, 2193, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 2522, 1850, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8178, 7862, 8190, 8509, 8192, 8190, 8192] avg=8192.0000 max=8509 max_rank=4 min=7862 min_rank=2 imbalance_ratio=1.038696 max_load_reduction=57027 max_load_reduction_pct=0.870163 moved_to_echo=33083 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4088, 4069, 6, 0, 0, 52, 2, 0, 0, 0, 0, 0, 0, 7, 4090, 7, 6, 0, 3, 7, 0, 4047, 2, 0, 0, 0, 0, 0, 0, 1448, 0, 2190, 8, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4, 4087, 0, 2, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4090, 0, 0, 10, 0, 4, 2625, 1848, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 3, 4089, 4087, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 4087, 6, 0, 4, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 12, 4086, 4085, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8169, 7736, 8205, 8632, 8191, 8193, 8191] avg=8192.0000 max=8632 max_rank=4 min=7736 min_rank=2 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=33207 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3955, 0, 0, 0, 52, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 141, 0, 0, 4054, 2, 0, 0, 0, 0, 0, 0, 2658, 0, 2191, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1544, 1849, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8293, 8843, 8190, 7531, 8191, 8191, 8192] avg=8192.0000 max=8843 max_rank=2 min=7531 min_rank=4 imbalance_ratio=1.079468 max_load_reduction=56693 max_load_reduction_pct=0.865067 moved_to_echo=32105 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4060, 0, 0, 0, 57, 9, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 61, 0, 0, 4054, 2, 0, 0, 0, 0, 0, 0, 3416, 0, 2183, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 665, 1852, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8213, 9689, 8183, 6655, 8192, 8190, 8190] avg=8192.0000 max=9689 max_rank=2 min=6655 min_rank=4 imbalance_ratio=1.182739 max_load_reduction=55847 max_load_reduction_pct=0.852158 moved_to_echo=31227 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[15488, 0, 17536, 128, 0, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 0, 384, 128, 0, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 32768, 32384, 128, 128, 128, 512, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 128, 17664, 14848, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[442, 0, 4081, 0, 1, 0, 0, 2, 0, 0, 0, 4094, 0, 1, 1, 4095, 0, 1, 0, 1, 1, 1, 2, 1, 19, 0, 1, 1, 4095, 4094, 4095, 1, 0, 0, 0, 4094, 4093, 0, 0, 1, 4095, 0, 2, 0, 4095, 0, 3649, 1, 0, 1, 0, 3, 1, 22, 4095, 0, 4071, 1, 0, 4094, 4093, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[579, 0, 3967, 0, 0, 0, 7, 0, 0, 0, 6, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2453, 0, 0, 0, 12, 0, 0, 1242, 4096, 0, 4041, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[519, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1952, 0, 0, 0, 23, 79, 0, 1987, 4096, 0, 3895, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1293, 0, 4087, 0, 2, 0, 0, 5, 2, 0, 0, 4093, 0, 2, 3, 4074, 0, 2, 0, 3, 0, 2, 0, 1, 24, 1, 2, 2, 4092, 4094, 4094, 3, 0, 0, 0, 4092, 4092, 0, 4, 2, 4094, 0, 4, 0, 4094, 0, 2484, 2, 0, 3, 0, 2, 2, 313, 4093, 7, 4088, 2, 1, 4093, 4091, 0, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1253, 0, 4074, 0, 0, 0, 1, 0, 0, 0, 1, 4096, 0, 0, 0, 4073, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2430, 0, 0, 0, 12, 0, 0, 607, 4096, 0, 3933, 0, 1, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[664, 4, 4089, 0, 7, 0, 0, 8, 0, 0, 0, 4085, 0, 7, 7, 4088, 0, 7, 0, 10, 0, 9, 7, 4, 57, 0, 5, 6, 4086, 4089, 4088, 10, 0, 0, 0, 4088, 4082, 0, 14, 7, 4089, 1, 5, 0, 4089, 0, 2543, 5, 0, 8, 0, 12, 7, 818, 4082, 10, 4090, 6, 7, 4078, 4075, 0, 0, 4083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2240, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1434, 0, 0, 0, 3, 0, 0, 529, 4096, 0, 4023, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[803, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1986, 0, 0, 0, 6, 3, 0, 1509, 4096, 0, 4002, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[442, 0, 4081, 0, 1, 0, 0, 2, 2365, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 1, 4095, 6, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 2, 1, 4095, 4092, 0, 0, 0, 0, 0, 0, 19, 0, 1, 1, 4089, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 1729, 0, 3649, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 1, 22, 4095, 0, 3193, 3, 0, 0, 0, 0, 0, 0, 4071, 1, 0, 80, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7712, 8198, 8194, 8205, 8188, 9476, 7318, 8245] avg=8192.0000 max=9476 max_rank=5 min=7318 min_rank=6 imbalance_ratio=1.156738 max_load_reduction=56060 max_load_reduction_pct=0.855408 moved_to_echo=33227 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1293, 0, 4087, 0, 2, 0, 0, 5, 2364, 821, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 2, 3, 4074, 6, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 2, 0, 1, 4094, 4088, 0, 0, 0, 0, 0, 0, 24, 1, 2, 2, 4086, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4092, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4, 0, 1729, 0, 2484, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 2, 313, 4093, 7, 3192, 4, 0, 0, 0, 0, 0, 0, 4088, 2, 1, 80, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8572, 8181, 8190, 8212, 8190, 8313, 7616, 8262] avg=8192.0000 max=8572 max_rank=0 min=7616 min_rank=6 imbalance_ratio=1.046387 max_load_reduction=56964 max_load_reduction_pct=0.869202 moved_to_echo=32381 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1253, 0, 4074, 0, 0, 0, 1, 0, 2365, 821, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4073, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1730, 0, 2430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 607, 4096, 0, 3194, 3, 0, 0, 0, 0, 0, 0, 3933, 0, 1, 81, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8514, 8177, 8191, 8183, 8192, 8256, 7912, 8111] avg=8192.0000 max=8514 max_rank=0 min=7912 min_rank=6 imbalance_ratio=1.039307 max_load_reduction=57022 max_load_reduction_pct=0.870087 moved_to_echo=32471 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[579, 0, 3967, 0, 0, 0, 7, 0, 2365, 821, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 4090, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1730, 0, 2453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 1242, 4096, 0, 3194, 3, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 81, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7739, 8199, 8189, 8173, 8192, 8279, 8547, 8218] avg=8192.0000 max=8547 max_rank=6 min=7739 min_rank=0 imbalance_ratio=1.043335 max_load_reduction=56989 max_load_reduction_pct=0.869583 moved_to_echo=33236 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[519, 0, 3872, 0, 0, 0, 0, 0, 2365, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1730, 0, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 79, 0, 1987, 4096, 0, 3194, 3, 0, 0, 0, 0, 0, 0, 3895, 0, 0, 81, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7577, 8188, 8189, 8158, 8192, 7778, 9382, 8072] avg=8192.0000 max=9382 max_rank=6 min=7577 min_rank=0 imbalance_ratio=1.145264 max_load_reduction=56154 max_load_reduction_pct=0.856842 moved_to_echo=33424 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2240, 0, 4061, 0, 0, 0, 0, 0, 2365, 821, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4094, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1730, 0, 1434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 529, 4096, 0, 3194, 3, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 81, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9487, 8198, 8189, 8185, 8192, 7260, 7825, 8200] avg=8192.0000 max=9487 max_rank=0 min=7260 min_rank=5 imbalance_ratio=1.158081 max_load_reduction=56049 max_load_reduction_pct=0.855240 moved_to_echo=31477 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[664, 4, 4089, 0, 7, 0, 0, 8, 2361, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 7, 7, 4088, 6, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 10, 0, 9, 7, 4, 4088, 4080, 0, 0, 0, 0, 0, 0, 57, 0, 5, 6, 4080, 4089, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4082, 0, 14, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 5, 0, 1727, 0, 2543, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 12, 7, 818, 4082, 10, 3180, 3, 0, 0, 0, 0, 0, 0, 4090, 6, 7, 80, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7951, 8194, 8205, 8247, 8191, 8370, 8120, 8258] avg=8192.0000 max=8370 max_rank=5 min=7951 min_rank=0 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32939 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[803, 0, 3987, 0, 0, 0, 0, 0, 2369, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1722, 0, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 1509, 4096, 0, 3199, 3, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 72, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7984, 8201, 8189, 8173, 8199, 7804, 8816, 8170] avg=8192.0000 max=8816 max_rank=6 min=7804 min_rank=5 imbalance_ratio=1.076172 max_load_reduction=56720 max_load_reduction_pct=0.865479 moved_to_echo=32989 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 13952, 0, 18944, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 32768, 0, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32256, 128, 128, 640, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 32768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7808, 128, 32256, 0, 128, 0, 128, 128, 18944, 6656, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 7040, 32768, 128, 25600, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[2, 4096, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 4093, 0, 0, 4071, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4093, 4091, 0, 0, 0, 4096, 4096, 7, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[456, 4093, 1, 0, 3, 3, 0, 2, 4093, 3, 0, 2, 0, 4094, 4094, 41, 4094, 0, 3, 2, 2, 0, 2, 2, 0, 2, 2, 0, 2, 4091, 0, 117, 2, 2, 0, 4094, 2, 0, 0, 4092, 4094, 0, 2, 4090, 0, 2, 0, 1, 4092, 4, 0, 3, 0, 4092, 4094, 3481, 0, 0, 0, 4092, 4093, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[351, 4094, 1, 3, 1, 1, 0, 1, 4095, 0, 0, 1, 0, 4092, 4095, 801, 4095, 0, 1, 1, 1, 0, 1, 0, 2, 1, 1, 0, 1, 4095, 0, 31, 0, 1, 1, 4095, 1, 0, 0, 4094, 4095, 0, 1, 4092, 0, 1, 0, 0, 4095, 2, 0, 0, 0, 4095, 4095, 2912, 0, 1, 0, 4094, 4095, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[183, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 93, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3831, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[68, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4095, 4095, 90, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 8, 0, 0, 0, 4094, 0, 0, 1, 4096, 4095, 0, 0, 4068, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 4094, 3962, 0, 0, 0, 4096, 4096, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[187, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3889, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1166, 4087, 20, 0, 11, 8, 2, 6, 4083, 9, 0, 7, 0, 4089, 4090, 60, 4088, 3, 7, 7, 10, 0, 6, 5, 0, 5, 8, 0, 6, 4082, 0, 261, 4, 4, 0, 4090, 7, 0, 0, 4079, 4089, 0, 5, 4080, 0, 7, 2, 0, 4085, 13, 0, 11, 0, 4072, 4089, 2603, 0, 0, 0, 4082, 4088, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[93, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 31, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3966, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[351, 4094, 1, 3, 1, 1, 0, 1, 3591, 127, 59, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 3952, 0, 801, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 1, 4095, 0, 31, 4036, 4, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 1, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 4092, 0, 1, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 504, 2, 0, 0, 0, 4095, 0, 2912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8849, 8190, 8171, 8193, 8199, 7513, 8192] avg=8192.0000 max=8849 max_rank=1 min=7513 min_rank=6 imbalance_ratio=1.080200 max_load_reduction=56687 max_load_reduction_pct=0.864975 moved_to_echo=32097 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 4, 0, 0, 0, 0, 3592, 127, 58, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3955, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4035, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4071, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 504, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 7, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7879, 8079, 8187, 8137, 8187, 8174, 8691, 8202] avg=8192.0000 max=8691 max_rank=6 min=7879 min_rank=0 imbalance_ratio=1.060913 max_load_reduction=56845 max_load_reduction_pct=0.867386 moved_to_echo=33254 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[183, 4096, 0, 0, 0, 0, 0, 0, 3592, 127, 58, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3955, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4038, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 504, 0, 0, 0, 0, 4096, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8056, 8141, 8188, 8138, 8194, 8194, 8431, 8194] avg=8192.0000 max=8431 max_rank=6 min=8056 min_rank=0 imbalance_ratio=1.029175 max_load_reduction=57105 max_load_reduction_pct=0.871353 moved_to_echo=33013 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[68, 4096, 0, 0, 0, 0, 0, 0, 3592, 127, 58, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 3954, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 8, 4036, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4068, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 504, 0, 1, 0, 0, 4096, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7941, 8135, 8187, 8146, 8193, 8173, 8563, 8198] avg=8192.0000 max=8563 max_rank=6 min=7941 min_rank=0 imbalance_ratio=1.045288 max_load_reduction=56973 max_load_reduction_pct=0.869339 moved_to_echo=33127 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[187, 4096, 0, 0, 0, 0, 0, 0, 3592, 127, 59, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3954, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 4037, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 504, 0, 0, 0, 0, 4096, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8061, 8059, 8188, 8150, 8194, 8201, 8489, 8194] avg=8192.0000 max=8489 max_rank=6 min=8059 min_rank=1 imbalance_ratio=1.036255 max_load_reduction=57047 max_load_reduction_pct=0.870468 moved_to_echo=33078 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1166, 4087, 20, 0, 11, 8, 2, 6, 3582, 126, 59, 0, 0, 0, 0, 0, 4083, 9, 0, 7, 0, 3950, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 7, 7, 10, 0, 6, 5, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 0, 6, 4082, 0, 261, 4030, 3, 0, 0, 0, 0, 0, 0, 4, 4, 0, 4090, 7, 0, 0, 4079, 1, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 5, 4080, 0, 7, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 503, 13, 0, 11, 0, 4072, 0, 2603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4088, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9067, 8109, 8213, 8395, 8185, 8193, 7202, 8172] avg=8192.0000 max=9067 max_rank=0 min=7202 min_rank=6 imbalance_ratio=1.106812 max_load_reduction=56469 max_load_reduction_pct=0.861649 moved_to_echo=31752 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[456, 4093, 1, 0, 3, 3, 0, 2, 3588, 127, 58, 0, 0, 0, 0, 0, 4093, 3, 0, 2, 0, 3954, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 2, 2, 0, 2, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 2, 4091, 0, 117, 4036, 4, 0, 0, 0, 0, 0, 0, 2, 2, 0, 4094, 2, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 4090, 0, 2, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 504, 4, 0, 3, 0, 4092, 0, 3481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8331, 8093, 8195, 8254, 8193, 8199, 8084, 8187] avg=8192.0000 max=8331 max_rank=0 min=8084 min_rank=6 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32663 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[93, 4096, 0, 0, 0, 0, 0, 0, 3595, 129, 59, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3946, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 4037, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 501, 0, 0, 0, 0, 4096, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7972, 8071, 8188, 8145, 8197, 8203, 8563, 8197] avg=8192.0000 max=8563 max_rank=6 min=7972 min_rank=0 imbalance_ratio=1.045288 max_load_reduction=56973 max_load_reduction_pct=0.869339 moved_to_echo=33160 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4096, 128, 128, 128, 0, 32768, 0, 28800, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 512, 32384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 0, 31744, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[2560, 32768, 128, 128, 128, 128, 128, 128, 28800, 1024, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 32768, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4094, 1, 1, 1, 0, 0, 4095, 1, 1, 0, 4095, 0, 0, 1, 1, 0, 4094, 1, 2, 0, 2, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 4095, 5, 0, 4093, 0, 4095, 0, 0, 1, 1, 4095, 0, 4095, 0, 4095, 0, 4095, 4094, 4095, 4091, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[3, 4096, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 13, 0, 1, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 4096, 0, 4087, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4086, 12, 7, 9, 1, 19, 4091, 1, 8, 0, 4081, 0, 0, 6, 6, 0, 4083, 0, 7, 0, 97, 0, 4088, 0, 4, 0, 0, 4, 7, 7, 4089, 0, 2, 4071, 0, 4086, 0, 0, 7, 9, 4088, 0, 4090, 5, 4088, 0, 4087, 4087, 4087, 3994, 1, 2, 0, 6, 0, 0, 8, 9, 0, 8, 8, 0, 4080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 30, 0, 35, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4068, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 24, 0, 17, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 4096, 0, 4079, 0, 4096, 0, 4096, 4096, 4096, 4095, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4093, 3, 3, 2, 0, 5, 4094, 5, 3, 0, 4093, 0, 0, 2, 2, 0, 4090, 0, 2, 0, 31, 0, 4093, 0, 1, 0, 0, 0, 0, 2, 4094, 0, 0, 4088, 0, 4093, 0, 0, 2, 3, 4093, 0, 4094, 1, 4093, 0, 4092, 4093, 4094, 4063, 2, 2, 0, 2, 0, 0, 3, 3, 0, 2, 2, 2, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 6, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4090, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 1, 1, 1, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 2, 0, 2, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4095, 4073, 5, 0, 0, 0, 0, 0, 0, 5, 0, 4093, 0, 4095, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 1, 9, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 17, 4091, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 4092, 4086, 5, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8193, 8182, 8175, 8197, 8200, 8205, 8188] avg=8192.0000 max=8205 max_rank=6 min=8175 min_rank=3 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 4096, 0, 0, 0, 0, 0, 4083, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 0, 0, 0, 4091, 4074, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 5, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8191, 8180, 8184, 8196, 8192, 8218, 8189] avg=8192.0000 max=8218 max_rank=6 min=8180 min_rank=2 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32795 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 0, 4072, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 17, 0, 0, 0, 0, 4088, 4074, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4079, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 4095, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 5, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8191, 8180, 8208, 8197, 8184, 8214, 8189] avg=8192.0000 max=8214 max_rank=6 min=8173 min_rank=0 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 12, 7, 9, 1, 19, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 4081, 0, 0, 6, 6, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 7, 0, 97, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 7, 7, 4089, 4065, 5, 0, 0, 0, 0, 0, 0, 0, 2, 4071, 0, 4086, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 4090, 5, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 17, 3994, 1, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 0, 8, 8, 0, 4080, 4079, 5, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8189, 8263, 8181, 8169, 8200, 8107, 8198] avg=8192.0000 max=8263 max_rank=2 min=8107 min_rank=6 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32674 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4059, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 35, 0, 0, 0, 0, 4096, 4074, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4068, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 5, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8191, 8180, 8240, 8195, 8173, 8209, 8189] avg=8192.0000 max=8240 max_rank=3 min=8159 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 4096, 4074, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8180, 8179, 8195, 8200, 8209, 8188] avg=8192.0000 max=8209 max_rank=6 min=8179 min_rank=3 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 3, 3, 2, 0, 5, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 4093, 0, 0, 2, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2, 0, 31, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 4094, 4072, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4093, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 4094, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 17, 4063, 2, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 2, 2, 2, 4091, 4084, 5, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8197, 8204, 8174, 8186, 8199, 8179, 8193] avg=8192.0000 max=8204 max_rank=0 min=8174 min_rank=3 imbalance_ratio=1.001465 max_load_reduction=57332 max_load_reduction_pct=0.874817 moved_to_echo=32757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4090, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0, 0, 4096, 4081, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8192, 8167, 8195, 8201, 8188, 8195, 8202] avg=8192.0000 max=8202 max_rank=7 min=8167 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=57334 max_load_reduction_pct=0.874847 moved_to_echo=32786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 0, 256, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32640, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 32768, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 2, 0, 1, 0, 0, 9, 0, 1, 0, 0, 1, 4095, 1, 4094, 4095, 0, 0, 0, 2, 4091, 1, 4095, 4092, 1, 0, 1, 0, 4095, 4094, 1, 4094, 4093, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 2, 1, 4094, 1, 1, 1, 4095, 1, 0, 1, 4095, 0, 6, 0, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 2, 0, 2, 0, 0, 9, 1, 2, 0, 0, 7, 4094, 3, 4093, 4084, 0, 0, 0, 2, 4089, 2, 4093, 4090, 2, 0, 3, 0, 4093, 4092, 3, 4089, 4094, 2, 4093, 0, 0, 0, 1, 0, 0, 4, 4092, 0, 1, 2, 0, 4092, 3, 1, 2, 4093, 2, 0, 2, 4093, 4, 2, 0, 4093, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4074, 0, 4096, 8, 0, 0, 0, 23, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 1, 4043, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 4086, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 1, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 28, 0, 3846, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4089, 0, 4096, 1, 0, 0, 0, 10, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 5, 4071, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 14, 8, 0, 6, 1, 0, 23, 0, 7, 0, 0, 4, 4087, 9, 4084, 4075, 0, 0, 0, 6, 4073, 9, 4086, 4076, 5, 0, 12, 0, 4087, 4085, 9, 4061, 4090, 5, 4087, 0, 0, 0, 1, 0, 0, 16, 4081, 3, 17, 3, 10, 4084, 10, 1, 2, 4087, 8, 0, 6, 4088, 14, 0, 0, 4089, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 8, 0, 4086, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 2, 0, 1, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1, 0, 0, 1, 4095, 4045, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4095, 0, 0, 0, 2, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4092, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 4094, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4088, 9, 1, 0, 0, 0, 0, 0, 2, 1, 4094, 1, 1, 1, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 6, 0, 4095, 1, 32, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8151, 8195, 8190, 8197, 8183, 8194, 8232] avg=8192.0000 max=8232 max_rank=7 min=8151 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32806 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 2, 0, 2, 0, 4093, 4090, 0, 0, 0, 0, 0, 0, 0, 9, 1, 2, 0, 0, 7, 4094, 4043, 0, 0, 0, 0, 0, 0, 0, 3, 6, 4084, 0, 0, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 4090, 2, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 4089, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 4092, 0, 1, 4087, 9, 1, 0, 0, 0, 0, 0, 2, 0, 4092, 3, 1, 2, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 4, 2, 0, 4093, 1, 32, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8156, 8184, 8192, 8196, 8195, 8193, 8229] avg=8192.0000 max=8229 max_rank=7 min=8156 min_rank=1 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 4095, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 4074, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 4087, 0, 0, 4090, 9, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 1, 4043, 0, 32, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8192, 8198, 8194, 8186, 8210, 8190, 8177] avg=8192.0000 max=8210 max_rank=5 min=8177 min_rank=7 imbalance_ratio=1.002197 max_load_reduction=57326 max_load_reduction_pct=0.874725 moved_to_echo=32763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 4094, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 4089, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4081, 0, 0, 4090, 9, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 5, 4071, 0, 32, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8171, 8198, 8194, 8194, 8191, 8190, 8209] avg=8192.0000 max=8209 max_rank=7 min=8171 min_rank=1 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 4091, 9, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4086, 0, 33, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8153, 8197, 8192, 8198, 8197, 8190, 8218] avg=8192.0000 max=8218 max_rank=7 min=8153 min_rank=1 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32803 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 4096, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 4090, 9, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 28, 0, 3846, 0, 32, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8395, 8199, 8194, 8200, 8165, 8190, 8004] avg=8192.0000 max=8395 max_rank=1 min=8004 min_rank=7 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=32559 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 14, 8, 0, 6, 1, 4087, 4084, 0, 0, 0, 0, 0, 0, 0, 23, 0, 7, 0, 0, 4, 4087, 4036, 0, 0, 0, 0, 0, 0, 0, 9, 6, 4075, 0, 0, 0, 6, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4086, 4076, 5, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 4061, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 16, 4081, 3, 17, 4078, 9, 3, 0, 0, 0, 0, 0, 3, 10, 4084, 10, 1, 2, 4085, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4088, 14, 0, 0, 4089, 5, 32, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8157, 8169, 8188, 8173, 8208, 8203, 8236] avg=8192.0000 max=8236 max_rank=7 min=8157 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 4096, 13, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 4086, 0, 35, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8161, 8196, 8192, 8192, 8187, 8186, 8231] avg=8192.0000 max=8231 max_rank=7 min=8161 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 32512, 128, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 32768, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 512, 128, 128, 0, 0, 128, 32768, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 4095, 0, 1, 4092, 1, 0, 0, 4094, 1, 0, 4095, 2, 0, 659, 1, 0, 1, 3919, 1, 1, 4094, 2, 0, 2, 4095, 0, 2, 0, 1, 0, 0, 2, 4094, 0, 0, 1, 3612, 0, 2, 1, 4095, 0, 1, 4095, 0, 2, 0, 1, 0, 0, 2, 0, 0, 0, 1, 0, 4095, 4092, 4093, 4093, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1228, 0, 0, 0, 3820, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 22, 4096, 4, 0, 0, 3141, 0, 0, 0, 4074, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 2, 4090, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 927, 0, 0, 0, 3839, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 21, 4096, 8, 0, 0, 3425, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4074, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1419, 0, 0, 0, 3775, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 1, 0, 0, 3003, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4087, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 4093, 0, 2, 4087, 3, 0, 0, 4093, 3, 0, 4093, 9, 0, 287, 3, 0, 2, 4049, 2, 2, 4093, 3, 2, 3, 4092, 0, 0, 0, 2, 15, 0, 0, 4092, 0, 0, 1, 3849, 0, 3, 1, 4094, 0, 3, 4079, 3, 3, 0, 3, 0, 0, 5, 0, 0, 0, 2, 0, 4093, 4093, 4088, 4093, 4091, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[7, 0, 4088, 8, 6, 4082, 11, 0, 0, 4083, 9, 0, 4081, 19, 0, 259, 12, 0, 4, 3958, 8, 8, 4084, 11, 19, 10, 4077, 0, 0, 0, 6, 70, 0, 0, 4085, 0, 0, 5, 3944, 0, 14, 6, 4087, 0, 10, 4018, 4, 9, 0, 0, 0, 0, 15, 0, 0, 0, 10, 0, 4086, 4086, 4069, 4085, 4079, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 2, 4083, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2448, 0, 0, 0, 3597, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 20, 4096, 18, 0, 0, 2143, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4075, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2232, 0, 0, 0, 3768, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 5, 0, 0, 2192, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4095, 0, 1, 4092, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2913, 1, 0, 4095, 2, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3919, 1, 1, 4094, 2, 249, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 2, 0, 1, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 1, 3612, 0, 918, 1, 0, 0, 0, 0, 0, 0, 2, 1, 4095, 0, 1, 4095, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 4093, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7670, 8268, 8182, 8628, 8208, 8188, 8200] avg=8192.0000 max=8628 max_rank=4 min=7670 min_rank=1 imbalance_ratio=1.053223 max_load_reduction=56908 max_load_reduction_pct=0.868347 moved_to_echo=33224 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4093, 0, 2, 4087, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2912, 3, 0, 4093, 9, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 4049, 2, 2, 4093, 3, 249, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4092, 0, 0, 0, 2, 15, 4077, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 3849, 0, 918, 1, 0, 0, 0, 0, 0, 0, 3, 1, 4094, 0, 3, 4079, 3, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5, 0, 0, 0, 4093, 4091, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4088, 4093, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 7304, 8403, 8193, 8861, 8198, 8192, 8196] avg=8192.0000 max=8861 max_rank=4 min=7304 min_rank=1 imbalance_ratio=1.081665 max_load_reduction=56675 max_load_reduction_pct=0.864792 moved_to_echo=33447 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4090, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2913, 0, 0, 4096, 0, 0, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 0, 0, 4096, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 8, 0, 0, 3425, 0, 919, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4074, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 7936, 8185, 8181, 8470, 8197, 8197, 8183] avg=8192.0000 max=8470 max_rank=4 min=7936 min_rank=1 imbalance_ratio=1.033936 max_load_reduction=57066 max_load_reduction_pct=0.870758 moved_to_echo=33047 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2913, 0, 0, 4096, 0, 0, 1228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 0, 4096, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 4, 0, 0, 3141, 0, 919, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8237, 8166, 8180, 8183, 8182, 8190, 8205] avg=8192.0000 max=8237 max_rank=1 min=8166 min_rank=2 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4083, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2913, 0, 0, 4096, 0, 0, 2448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3597, 0, 0, 4096, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 18, 0, 0, 2143, 0, 919, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 9457, 7943, 8182, 7197, 8187, 8203, 8184] avg=8192.0000 max=9457 max_rank=1 min=7197 min_rank=4 imbalance_ratio=1.154419 max_load_reduction=56079 max_load_reduction_pct=0.855698 moved_to_echo=31771 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2913, 0, 0, 4096, 0, 0, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 4096, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4082, 3, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 1, 0, 0, 3003, 0, 919, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8428, 8121, 8181, 8025, 8203, 8190, 8195] avg=8192.0000 max=8428 max_rank=1 min=8025 min_rank=4 imbalance_ratio=1.028809 max_load_reduction=57108 max_load_reduction_pct=0.871399 moved_to_echo=32613 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 4088, 8, 6, 4082, 11, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2904, 9, 0, 4081, 19, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4, 3958, 8, 8, 4084, 11, 249, 0, 0, 0, 0, 0, 0, 0, 19, 10, 4077, 0, 0, 0, 6, 70, 4065, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 5, 3944, 0, 916, 1, 0, 0, 0, 0, 0, 0, 14, 6, 4087, 0, 10, 4018, 4, 9, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4086, 4084, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4069, 4085, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 7272, 8334, 8249, 8951, 8160, 8185, 8181] avg=8192.0000 max=8951 max_rank=4 min=7272 min_rank=1 imbalance_ratio=1.092651 max_load_reduction=56585 max_load_reduction_pct=0.863419 moved_to_echo=33477 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2904, 0, 0, 4096, 0, 0, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3768, 0, 0, 4096, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 5, 0, 0, 2192, 0, 920, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 9232, 8116, 8188, 7221, 8201, 8191, 8192] avg=8192.0000 max=9232 max_rank=1 min=7221 min_rank=4 imbalance_ratio=1.126953 max_load_reduction=56304 max_load_reduction_pct=0.859131 moved_to_echo=31805 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 0, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 30848, 128, 128, 32768, 128, 2048, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 25344, 0, 7424, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 23296, 128, 0, 32768, 128, 0, 9472, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[2, 4095, 1, 4094, 12, 4094, 0, 4071, 0, 1, 0, 1048, 1, 4092, 0, 0, 4095, 4023, 0, 0, 1, 0, 4095, 0, 0, 1, 4091, 0, 0, 1, 0, 1, 38, 1, 1, 4095, 2, 4093, 1, 1, 84, 4094, 1, 1, 2, 0, 0, 4095, 0, 0, 4095, 1, 1, 0, 1, 2592, 2, 422, 0, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 255, 0, 0, 4096, 0, 4096, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3001, 0, 835, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 91, 0, 4096, 0, 0, 4096, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 137, 0, 0, 4093, 0, 4096, 0, 0, 123, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3374, 0, 370, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 4093, 2, 4089, 34, 4093, 0, 4007, 0, 3, 0, 718, 3, 4090, 0, 0, 4092, 3841, 12, 0, 0, 0, 4093, 0, 0, 3, 4081, 0, 0, 3, 4, 2, 28, 3, 2, 4094, 7, 4085, 1, 2, 29, 4089, 2, 2, 4, 0, 0, 4094, 0, 0, 4092, 0, 2, 2, 3, 3145, 3, 520, 0, 0, 2, 4060, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 18, 0, 4096, 0, 0, 4096, 4096, 0, 6, 0, 0, 4096, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 414, 0, 0, 4090, 0, 4096, 0, 0, 206, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2842, 0, 608, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 9, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 48, 0, 0, 4096, 0, 4096, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 4019, 0, 6, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 4086, 4, 4070, 97, 4086, 0, 3773, 0, 15, 0, 2050, 10, 4078, 0, 0, 4085, 3594, 36, 0, 0, 0, 4087, 0, 0, 9, 4053, 0, 0, 6, 19, 8, 3, 9, 8, 4088, 27, 4065, 5, 8, 64, 4067, 7, 9, 11, 0, 0, 4088, 3, 0, 4083, 3, 7, 6, 14, 2339, 10, 497, 0, 1, 0, 3947, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 39, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 87, 0, 0, 4096, 0, 4096, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3374, 0, 592, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 41, 1, 4094, 12, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1048, 1, 4092, 0, 0, 3181, 417, 0, 0, 0, 0, 0, 0, 4095, 4023, 0, 0, 1, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 1, 0, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 38, 1, 1, 3968, 2, 4093, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 84, 4094, 1, 1, 2, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 1, 2592, 814, 127, 64, 0, 0, 0, 0, 0, 2, 422, 0, 0, 0, 4094, 0, 0, 3636, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8740, 8219, 8189, 8105, 8213, 7695, 8154] avg=8192.0000 max=8740 max_rank=1 min=7695 min_rank=6 imbalance_ratio=1.066895 max_load_reduction=56796 max_load_reduction_pct=0.866638 moved_to_echo=32167 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 41, 2, 4089, 34, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 718, 3, 4090, 0, 0, 3180, 417, 0, 0, 0, 0, 0, 0, 4092, 3841, 12, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4081, 0, 0, 3, 4, 2, 4093, 1, 0, 0, 0, 0, 0, 0, 28, 3, 2, 3967, 7, 4085, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4089, 2, 2, 4, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 2, 3, 3145, 814, 127, 64, 0, 0, 0, 0, 0, 3, 520, 0, 0, 2, 4060, 2, 0, 3634, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8411, 8044, 8187, 8095, 8156, 8249, 8221] avg=8192.0000 max=8411 max_rank=1 min=8044 min_rank=2 imbalance_ratio=1.026733 max_load_reduction=57125 max_load_reduction_pct=0.871658 moved_to_echo=32721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 41, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 4096, 0, 0, 3182, 417, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 137, 0, 0, 3966, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 4096, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3374, 814, 127, 64, 0, 0, 0, 0, 0, 0, 370, 0, 0, 0, 4096, 0, 0, 3636, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 7786, 8293, 8197, 8199, 8251, 8475, 8102] avg=8192.0000 max=8475 max_rank=6 min=7786 min_rank=1 imbalance_ratio=1.034546 max_load_reduction=57061 max_load_reduction_pct=0.870682 moved_to_echo=33027 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 41, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 0, 0, 3182, 417, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 414, 0, 0, 3963, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 4096, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2842, 814, 127, 64, 0, 0, 0, 0, 0, 0, 608, 0, 0, 0, 4096, 0, 0, 3637, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 7713, 8298, 8201, 8473, 8334, 7943, 8341] avg=8192.0000 max=8473 max_rank=4 min=7713 min_rank=1 imbalance_ratio=1.034302 max_load_reduction=57063 max_load_reduction_pct=0.870712 moved_to_echo=33091 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 41, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3182, 417, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 255, 0, 0, 3969, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3001, 814, 127, 64, 0, 0, 0, 0, 0, 0, 835, 0, 0, 0, 4096, 0, 0, 3637, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 7695, 8292, 8193, 8320, 8133, 8102, 8568] avg=8192.0000 max=8568 max_rank=7 min=7695 min_rank=1 imbalance_ratio=1.045898 max_load_reduction=56968 max_load_reduction_pct=0.869263 moved_to_echo=33123 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 41, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 3182, 417, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 48, 0, 0, 3969, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4019, 814, 127, 64, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 3637, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 7704, 8292, 8193, 8113, 8141, 9121, 7739] avg=8192.0000 max=9121 max_rank=6 min=7704 min_rank=1 imbalance_ratio=1.113403 max_load_reduction=56415 max_load_reduction_pct=0.860825 moved_to_echo=33114 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 41, 4, 4070, 97, 0, 0, 3773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 2050, 10, 4078, 0, 0, 3175, 416, 0, 0, 0, 0, 0, 0, 4085, 3594, 36, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4053, 0, 0, 6, 19, 8, 4086, 1, 0, 0, 0, 0, 0, 0, 3, 9, 8, 3961, 27, 4065, 5, 8, 0, 0, 0, 0, 0, 0, 0, 0, 64, 4067, 7, 9, 11, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4083, 3, 7, 6, 14, 2339, 812, 127, 64, 0, 0, 0, 0, 0, 10, 497, 0, 1, 0, 3947, 1, 0, 3628, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7985, 9744, 7815, 8182, 8086, 8182, 7458, 8084] avg=8192.0000 max=9744 max_rank=1 min=7458 min_rank=6 imbalance_ratio=1.189453 max_load_reduction=55792 max_load_reduction_pct=0.851318 moved_to_echo=31810 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 33, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 4096, 0, 0, 3186, 422, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 87, 0, 0, 3962, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3374, 819, 134, 70, 0, 0, 0, 0, 0, 0, 592, 0, 0, 0, 4096, 0, 0, 3639, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 7743, 8283, 8194, 8145, 8126, 8493, 8327] avg=8192.0000 max=8493 max_rank=6 min=7743 min_rank=1 imbalance_ratio=1.036743 max_load_reduction=57043 max_load_reduction_pct=0.870407 moved_to_echo=33091 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32000, 128, 128, 128, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 384, 128, 32768, 256, 0, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 128, 128, 31744, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 32768, 128, 128, 128, 0, 0, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 3968, 0, 128, 128, 32640, 128, 0, 29184, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 0, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 24704, 6528, 1024, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 4096, 128, 32768, 0, 0, 25472, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4036, 0, 0, 0, 0, 0, 4083, 4090, 3, 3, 3, 3, 2, 2, 0, 7, 4088, 0, 0, 2, 0, 58, 4092, 0, 0, 0, 4094, 3, 4093, 0, 4093, 0, 4093, 5, 0, 4093, 2, 3, 6, 0, 0, 12, 0, 0, 4093, 4, 0, 0, 3, 4093, 4092, 4086, 2, 0, 4087, 0, 2, 2, 3, 0, 2, 4091, 4, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4067, 0, 3, 0, 0, 0, 4094, 4094, 1, 1, 1, 1, 0, 1, 0, 2, 4094, 0, 0, 1, 0, 25, 4094, 0, 0, 0, 4095, 2, 4095, 0, 4094, 0, 4095, 1, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 1, 4095, 4094, 4093, 1, 0, 4093, 2, 2, 1, 0, 2, 1, 4093, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4050, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 46, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 6, 4096, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[3784, 0, 19, 4, 0, 0, 4031, 4077, 9, 11, 10, 8, 4, 6, 0, 50, 4032, 0, 0, 7, 0, 306, 4081, 0, 0, 0, 4088, 11, 4084, 0, 4085, 0, 4087, 18, 0, 4088, 0, 8, 26, 0, 1, 68, 0, 0, 4086, 12, 0, 0, 9, 4087, 4086, 4058, 6, 0, 4065, 1, 8, 3, 9, 11, 5, 4077, 0, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 4096, 0, 4095, 0, 4096, 6, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 20, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4067, 0, 3, 0, 0, 0, 4094, 69, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 2, 4095, 4083, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 25, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 4025, 46, 10, 5, 0, 0, 0, 0, 1, 0, 9, 4093, 1, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 2, 1, 4093, 2, 1, 4075, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8185, 8168, 8191, 8188, 8183, 8199, 8189] avg=8192.0000 max=8233 max_rank=0 min=8168 min_rank=2 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4036, 0, 0, 0, 0, 0, 4083, 69, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 2, 2, 0, 7, 4093, 4081, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 2, 0, 58, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 5, 0, 4093, 2, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4093, 4, 0, 0, 4021, 46, 9, 5, 0, 0, 0, 0, 3, 0, 10, 4086, 2, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3, 0, 2, 4091, 4, 3, 4073, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8197, 8194, 8190, 8197, 8190, 8188, 8192] avg=8192.0000 max=8197 max_rank=1 min=8188 min_rank=0 imbalance_ratio=1.000610 max_load_reduction=57339 max_load_reduction_pct=0.874924 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 0, 0, 4096, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4027, 46, 10, 5, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8178, 8146, 8200, 8187, 8184, 8201, 8185] avg=8192.0000 max=8255 max_rank=0 min=8146 min_rank=2 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 0, 0, 4096, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4050, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4027, 46, 10, 5, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 0, 4077, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8178, 8155, 8199, 8188, 8175, 8201, 8205] avg=8192.0000 max=8235 max_rank=0 min=8155 min_rank=2 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 4096, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4050, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4027, 46, 10, 5, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8180, 8148, 8192, 8187, 8184, 8201, 8185] avg=8192.0000 max=8259 max_rank=0 min=8148 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3784, 0, 19, 4, 0, 0, 4031, 68, 0, 0, 0, 0, 0, 0, 0, 0, 9, 11, 10, 8, 4, 6, 0, 50, 4087, 4075, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 7, 0, 306, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4084, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 18, 0, 4088, 0, 8, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 68, 0, 0, 4086, 12, 0, 0, 4009, 46, 9, 5, 0, 0, 0, 0, 9, 0, 10, 4058, 6, 0, 4065, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 9, 11, 5, 4077, 0, 10, 4067, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7906, 8260, 8380, 8180, 8222, 8236, 8149, 8203] avg=8192.0000 max=8380 max_rank=2 min=7906 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32810 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4050, 0, 0, 0, 0, 0, 4096, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4050, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4027, 46, 10, 5, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8178, 8192, 8192, 8187, 8184, 8201, 8187] avg=8192.0000 max=8215 max_rank=0 min=8178 min_rank=1 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 0, 0, 4096, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4048, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4032, 48, 12, 12, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4080, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8180, 8153, 8192, 8180, 8200, 8196, 8190] avg=8192.0000 max=8245 max_rank=0 min=8153 min_rank=2 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32384, 0, 128, 128, 0, 0, 32768, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 0, 32256, 384, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 0, 512, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4070, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 26, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 4090, 0, 4096, 0, 0, 4096, 4096, 4076, 0, 0, 0, 0, 20, 4088, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 4096, 0, 4096, 7, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3, 4, 2, 0, 1, 16, 0, 0, 9, 1, 4, 0, 3, 2, 0, 4092, 4093, 0, 4093, 4091, 2, 4092, 0, 2, 4093, 3, 4093, 0, 0, 3875, 4078, 4093, 0, 2, 0, 219, 0, 4094, 0, 4088, 3, 0, 3, 0, 4087, 0, 2, 0, 4, 2, 0, 0, 0, 3, 4093, 0, 0, 4089, 3, 4093, 0, 0, 4, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[2, 1, 1, 0, 1, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 4094, 4087, 1, 4095, 4094, 1, 4094, 0, 1, 4094, 2, 4094, 0, 0, 3985, 4093, 4095, 0, 1, 0, 117, 0, 4094, 0, 4094, 1, 2, 1, 0, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 2, 4094, 0, 0, 4094, 1, 4095, 0, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[10, 8, 7, 0, 5, 60, 0, 0, 51, 3, 12, 0, 11, 7, 0, 4083, 4083, 0, 4087, 4080, 6, 4085, 0, 15, 4086, 7, 4087, 0, 0, 3445, 4033, 4087, 0, 8, 0, 644, 0, 4088, 0, 4042, 10, 0, 9, 0, 4069, 0, 1, 1, 13, 3, 0, 0, 5, 11, 4086, 0, 0, 4073, 8, 4085, 0, 0, 13, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 4095, 0, 4096, 0, 0, 4096, 4096, 4085, 0, 0, 0, 0, 11, 4090, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 4096, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 6, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 4, 2, 0, 1, 16, 0, 0, 4093, 4081, 0, 0, 0, 0, 0, 0, 9, 1, 4, 0, 3, 2, 0, 4092, 3961, 120, 4, 0, 0, 0, 0, 0, 4093, 0, 0, 4091, 2, 8, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3875, 4078, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 219, 0, 3974, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 4087, 0, 2, 0, 4084, 12, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 3, 4093, 0, 4080, 9, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 4093, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8196, 8196, 8088, 8283, 8191, 8191, 8191] avg=8192.0000 max=8283 max_rank=4 min=8088 min_rank=3 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32856 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 1, 0, 1, 1, 0, 0, 4095, 4082, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 2, 1, 0, 4094, 3962, 120, 5, 0, 0, 0, 0, 0, 4087, 1, 0, 4094, 1, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3985, 4093, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 117, 0, 3974, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 4093, 0, 0, 0, 4086, 12, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 4094, 0, 4082, 8, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4095, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8187, 8191, 8213, 8186, 8195, 8188, 8193] avg=8192.0000 max=8213 max_rank=3 min=8183 min_rank=0 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32762 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3963, 120, 5, 0, 0, 0, 0, 0, 4070, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3976, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8184, 8173, 8325, 8098, 8196, 8188, 8192] avg=8192.0000 max=8325 max_rank=3 min=8098 min_rank=4 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32674 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3953, 120, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3970, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 4084, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8174, 8199, 8324, 8077, 8194, 8195, 8193] avg=8192.0000 max=8324 max_rank=3 min=8077 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32659 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3944, 120, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 3968, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4081, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 4084, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8165, 8199, 8324, 8084, 8189, 8196, 8199] avg=8192.0000 max=8324 max_rank=3 min=8084 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32668 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3963, 120, 5, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3976, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8184, 8198, 8325, 8073, 8196, 8188, 8192] avg=8192.0000 max=8325 max_rank=3 min=8073 min_rank=4 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32649 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[10, 8, 7, 0, 5, 60, 0, 0, 4087, 4075, 0, 0, 0, 0, 0, 0, 51, 3, 12, 0, 11, 7, 0, 4083, 3955, 120, 4, 0, 0, 0, 0, 0, 4083, 0, 0, 4080, 6, 8, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3445, 4033, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 644, 0, 3968, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 9, 0, 4069, 0, 1, 1, 4078, 12, 0, 0, 0, 0, 0, 0, 13, 3, 0, 0, 5, 11, 4086, 0, 4073, 8, 0, 0, 0, 0, 0, 0, 0, 4073, 8, 4085, 0, 0, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8246, 8192, 7617, 8662, 8180, 8199, 8188] avg=8192.0000 max=8662 max_rank=4 min=7617 min_rank=3 imbalance_ratio=1.057373 max_load_reduction=56874 max_load_reduction_pct=0.867828 moved_to_echo=33229 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3968, 125, 7, 4, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3971, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8200, 8188, 8320, 8073, 8195, 8191, 8188] avg=8192.0000 max=8320 max_rank=3 min=8073 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32647 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 1024, 128, 31872, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 31872, 32768, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 32768, 31744, 1024, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 128, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[53, 1, 39, 8, 8, 2, 1410, 1, 737, 150, 11, 1525, 1591, 2574, 0, 3816, 3604, 164, 12, 561, 1, 2616, 179, 1401, 0, 69, 177, 1900, 1275, 10, 616, 2077, 4087, 9, 94, 2574, 1834, 5, 1239, 761, 4014, 1, 1618, 2077, 0, 178, 1739, 1120, 1273, 3729, 1206, 2, 0, 22, 13, 3056, 0, 833, 606, 647, 1625, 2063, 2395, 128] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[18, 0, 39, 0, 8, 1, 1843, 3, 481, 115, 5, 1315, 1507, 2009, 1, 3420, 3544, 129, 4, 641, 2, 3156, 147, 1263, 4, 112, 199, 1846, 1894, 3, 506, 2249, 4089, 4, 82, 2857, 1891, 3, 1440, 793, 4079, 0, 1069, 2510, 1, 140, 2183, 1209, 554, 3490, 935, 0, 0, 1, 0, 3190, 0, 812, 616, 678, 1276, 2237, 2867, 66] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[141, 3, 39, 2, 21, 1, 1540, 0, 1570, 217, 28, 2101, 2083, 2674, 0, 3600, 3439, 95, 12, 440, 16, 2660, 191, 1236, 2, 82, 139, 1977, 1251, 2, 888, 2284, 3956, 9, 124, 2619, 2340, 11, 866, 470, 4014, 1, 1305, 2371, 1, 280, 1067, 736, 898, 3430, 938, 4, 7, 10, 2, 2718, 0, 1393, 848, 849, 1501, 1304, 2520, 210] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[56, 1, 59, 0, 29, 0, 1704, 1, 701, 219, 3, 1765, 2132, 2766, 0, 3306, 3096, 100, 7, 873, 0, 2533, 159, 1236, 0, 149, 203, 2496, 2054, 2, 794, 1818, 4059, 23, 195, 2959, 2238, 1, 1320, 675, 4084, 0, 1459, 2029, 0, 217, 1157, 1099, 816, 3755, 980, 0, 1, 5, 2, 2801, 0, 909, 429, 506, 1591, 1435, 2457, 72] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[82, 4, 97, 5, 58, 0, 1162, 0, 683, 152, 2, 1423, 2505, 1687, 0, 3740, 3730, 134, 17, 1208, 0, 2249, 171, 1212, 0, 19, 215, 2192, 1482, 1, 461, 1700, 4082, 33, 126, 2516, 2220, 2, 2049, 456, 4064, 0, 1276, 2290, 0, 114, 1336, 1558, 1084, 3644, 1128, 0, 0, 30, 6, 3150, 0, 1012, 432, 530, 1790, 1440, 2700, 77] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[17, 0, 99, 0, 4, 0, 1057, 0, 768, 137, 10, 1684, 2096, 2388, 0, 3451, 3373, 87, 39, 717, 0, 2301, 173, 1218, 0, 25, 185, 2244, 1499, 6, 656, 2711, 4093, 1, 264, 2742, 2348, 0, 1305, 509, 4044, 0, 1258, 2559, 0, 194, 1198, 1100, 777, 3356, 1184, 0, 0, 0, 22, 2916, 0, 830, 636, 526, 1467, 2465, 2727, 70] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[43, 9, 45, 1, 66, 0, 1766, 0, 700, 153, 3, 1387, 1475, 2204, 0, 3699, 3792, 124, 9, 566, 23, 2628, 163, 1200, 4, 98, 103, 1887, 1562, 3, 480, 2206, 4086, 12, 98, 2712, 2581, 5, 1271, 993, 4083, 0, 1339, 2079, 0, 155, 1771, 1199, 572, 3856, 993, 0, 5, 1, 4, 3110, 0, 917, 541, 745, 1434, 1805, 2658, 112] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[42, 0, 32, 2, 6, 0, 1435, 0, 1225, 182, 9, 1800, 1712, 1773, 0, 3271, 3200, 190, 23, 1332, 8, 3054, 130, 1142, 1, 66, 113, 2009, 2113, 0, 1007, 1864, 4089, 4, 297, 2944, 2510, 7, 1152, 432, 4039, 1, 1305, 2680, 2, 98, 986, 959, 781, 3671, 1176, 2, 0, 1, 7, 2990, 0, 1021, 532, 469, 1698, 1472, 2436, 34] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[53, 1, 39, 8, 8, 2, 1410, 1, 3018, 2217, 1420, 0, 0, 0, 0, 0, 737, 150, 11, 1525, 1591, 2574, 0, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 3320, 164, 12, 561, 1, 2616, 179, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 177, 1900, 1275, 10, 616, 2077, 899, 356, 273, 0, 0, 0, 0, 0, 1069, 9, 94, 2574, 1834, 5, 1239, 761, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 1, 1618, 2077, 0, 178, 1739, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 1273, 3373, 1206, 2, 0, 22, 13, 3056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 606, 647, 1625, 2063, 2395, 128, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8085, 8254, 7652, 7585, 8530, 8945, 8308] avg=8192.0000 max=8945 max_rank=6 min=7585 min_rank=4 imbalance_ratio=1.091919 max_load_reduction=56591 max_load_reduction_pct=0.863510 moved_to_echo=33368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[141, 3, 39, 2, 21, 1, 1540, 0, 2921, 2217, 1339, 0, 0, 0, 0, 0, 1570, 217, 28, 2101, 2083, 2674, 0, 1413, 0, 0, 0, 0, 0, 0, 0, 0, 3169, 95, 12, 440, 16, 2660, 191, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 2, 82, 139, 1977, 1251, 2, 888, 2284, 848, 328, 260, 0, 0, 0, 0, 0, 1035, 9, 124, 2619, 2340, 11, 866, 470, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 1, 1305, 2371, 1, 280, 1067, 736, 0, 0, 0, 0, 0, 0, 0, 0, 898, 3102, 938, 4, 7, 10, 2, 2718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1393, 848, 849, 1501, 1304, 2520, 210, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 10086, 7819, 8061, 7474, 7558, 7679, 8635] avg=8192.0000 max=10086 max_rank=1 min=7474 min_rank=4 imbalance_ratio=1.231201 max_load_reduction=55450 max_load_reduction_pct=0.846100 moved_to_echo=31346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 39, 0, 8, 1, 1843, 3, 3019, 2253, 1272, 0, 0, 0, 0, 0, 481, 115, 5, 1315, 1507, 2009, 1, 1342, 0, 0, 0, 0, 0, 0, 0, 0, 3265, 129, 4, 641, 2, 3156, 147, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 4, 112, 199, 1846, 1894, 3, 506, 2249, 806, 333, 268, 0, 0, 0, 0, 0, 1070, 4, 82, 2857, 1891, 3, 1440, 793, 0, 0, 0, 0, 0, 0, 0, 0, 1826, 0, 1069, 2510, 1, 140, 2183, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 554, 3157, 935, 0, 0, 1, 0, 3190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 812, 616, 678, 1276, 2237, 2867, 66, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8456, 6775, 8607, 8220, 8140, 8938, 7837, 8563] avg=8192.0000 max=8938 max_rank=5 min=6775 min_rank=1 imbalance_ratio=1.091064 max_load_reduction=56598 max_load_reduction_pct=0.863617 moved_to_echo=33478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[82, 4, 97, 5, 58, 0, 1162, 0, 3014, 2244, 1391, 0, 0, 0, 0, 0, 683, 152, 2, 1423, 2505, 1687, 0, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 3437, 134, 17, 1208, 0, 2249, 171, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 215, 2192, 1482, 1, 461, 1700, 881, 348, 282, 0, 0, 0, 0, 0, 1068, 33, 126, 2516, 2220, 2, 2049, 456, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 0, 1276, 2290, 0, 114, 1336, 1558, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 3296, 1128, 0, 0, 30, 6, 3150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 432, 530, 1790, 1440, 2700, 77, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8057, 7920, 8428, 7581, 8470, 8394, 8694, 7992] avg=8192.0000 max=8694 max_rank=6 min=7581 min_rank=3 imbalance_ratio=1.061279 max_load_reduction=56842 max_load_reduction_pct=0.867340 moved_to_echo=33550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[56, 1, 59, 0, 29, 0, 1704, 1, 2997, 2255, 1230, 0, 0, 0, 0, 0, 701, 219, 3, 1765, 2132, 2766, 0, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 2853, 100, 7, 873, 0, 2533, 159, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 203, 2496, 2054, 2, 794, 1818, 779, 359, 234, 0, 0, 0, 0, 0, 1062, 23, 195, 2959, 2238, 1, 1320, 675, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 0, 1459, 2029, 0, 217, 1157, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 816, 3396, 980, 0, 1, 5, 2, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 429, 506, 1591, 1435, 2457, 72, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8332, 8883, 7761, 8888, 8473, 7790, 8001, 7408] avg=8192.0000 max=8888 max_rank=3 min=7408 min_rank=7 imbalance_ratio=1.084961 max_load_reduction=56648 max_load_reduction_pct=0.864380 moved_to_echo=31672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 99, 0, 4, 0, 1057, 0, 3022, 2233, 1284, 0, 0, 0, 0, 0, 768, 137, 10, 1684, 2096, 2388, 0, 1354, 0, 0, 0, 0, 0, 0, 0, 0, 3108, 87, 39, 717, 0, 2301, 173, 1218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 185, 2244, 1499, 6, 656, 2711, 813, 321, 255, 0, 0, 0, 0, 0, 1071, 1, 264, 2742, 2348, 0, 1305, 509, 0, 0, 0, 0, 0, 0, 0, 0, 1811, 0, 1258, 2559, 0, 194, 1198, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 777, 3035, 1184, 0, 0, 0, 22, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 636, 526, 1467, 2465, 2727, 70, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7716, 8437, 7643, 8715, 8240, 8120, 7934, 8731] avg=8192.0000 max=8731 max_rank=7 min=7643 min_rank=2 imbalance_ratio=1.065796 max_load_reduction=56805 max_load_reduction_pct=0.866776 moved_to_echo=33025 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[43, 9, 45, 1, 66, 0, 1766, 0, 3017, 2255, 1376, 0, 0, 0, 0, 0, 700, 153, 3, 1387, 1475, 2204, 0, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 3494, 124, 9, 566, 23, 2628, 163, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 4, 98, 103, 1887, 1562, 3, 480, 2206, 871, 368, 287, 0, 0, 0, 0, 0, 1069, 12, 98, 2712, 2581, 5, 1271, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1828, 0, 1339, 2079, 0, 155, 1771, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 572, 3488, 993, 0, 5, 1, 4, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 541, 745, 1434, 1805, 2658, 112, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8578, 7374, 8207, 7869, 8741, 8371, 8173, 8223] avg=8192.0000 max=8741 max_rank=4 min=7374 min_rank=1 imbalance_ratio=1.067017 max_load_reduction=56795 max_load_reduction_pct=0.866623 moved_to_echo=33508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[42, 0, 32, 2, 6, 0, 1435, 0, 3022, 2235, 1222, 0, 0, 0, 0, 0, 1225, 182, 9, 1800, 1712, 1773, 0, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 2938, 190, 23, 1332, 8, 3054, 130, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 1, 66, 113, 2009, 2113, 0, 1007, 1864, 774, 355, 248, 0, 0, 0, 0, 0, 1067, 4, 297, 2944, 2510, 7, 1152, 432, 0, 0, 0, 0, 0, 0, 0, 0, 1804, 1, 1305, 2680, 2, 98, 986, 959, 0, 0, 0, 0, 0, 0, 0, 0, 781, 3316, 1176, 2, 0, 1, 7, 2990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 532, 469, 1698, 1472, 2436, 34, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7996, 7976, 8817, 8550, 8413, 7835, 8273, 7676] avg=8192.0000 max=8817 max_rank=2 min=7676 min_rank=7 imbalance_ratio=1.076294 max_load_reduction=56719 max_load_reduction_pct=0.865463 moved_to_echo=32197 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 128, 512, 128, 256, 128, 12032, 128, 24064, 17920, 10624, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[14592, 128, 10752, 18688, 128, 1408, 11520, 9088, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6912, 1408, 128, 13056, 15104, 18176, 128, 11136, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 7808, 4736, 4992, 12416, 14336, 20864, 896, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8576, 128, 1280, 22016, 18048, 128, 10752, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[25600, 1024, 128, 6400, 128, 21248, 1408, 9984, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 640, 1408, 16640, 13184, 128, 5504, 17024, 6784, 2816, 2176, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6784, 26240, 8576, 128, 128, 128, 128, 23936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4094, 1, 1, 0, 13, 4087, 0, 4094, 0, 81, 0, 0, 0, 0, 0, 4094, 3419, 1, 0, 37, 2, 1, 0, 4079, 1, 0, 4095, 0, 4095, 4089, 0, 299, 4, 0, 4095, 1, 3, 4093, 4095, 377, 16, 1, 1, 4095, 1, 1, 1, 191, 0, 0, 3839, 0, 1, 1, 7, 1, 5, 0, 1, 4092, 1, 4029, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 12, 4089, 1, 4096, 0, 189, 0, 1, 0, 0, 0, 4095, 3597, 0, 0, 8, 0, 0, 0, 4095, 2, 0, 4096, 0, 4096, 4093, 0, 47, 1, 0, 4096, 0, 2, 4096, 4096, 302, 1, 0, 0, 4095, 1, 0, 0, 24, 0, 0, 4026, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4089, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 24, 4088, 2, 4096, 0, 224, 0, 0, 2, 1, 0, 4095, 3518, 1, 0, 25, 2, 0, 0, 4089, 0, 0, 4094, 3, 4094, 4085, 1, 292, 5, 0, 4094, 2, 0, 4094, 4092, 72, 2, 0, 2, 4092, 1, 1, 1, 68, 0, 0, 3991, 1, 0, 2, 4, 2, 0, 3, 0, 4094, 0, 4082, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 4094, 0, 4096, 0, 119, 0, 0, 0, 0, 0, 4090, 3715, 0, 0, 12, 0, 0, 0, 4071, 0, 0, 4096, 0, 4096, 4095, 0, 52, 1, 0, 4096, 0, 3, 4096, 4096, 258, 2, 0, 0, 4096, 0, 0, 0, 47, 0, 0, 4040, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4076, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4072, 0, 0, 0, 27, 4096, 1, 4096, 0, 65, 0, 9, 0, 0, 1, 4092, 3894, 0, 0, 33, 0, 0, 0, 4072, 1, 3, 4093, 8, 4096, 4078, 0, 93, 1, 4, 4087, 0, 15, 4092, 4096, 130, 0, 0, 0, 4096, 15, 0, 0, 14, 0, 0, 3980, 0, 0, 1, 0, 0, 1, 3, 0, 4083, 0, 4087, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 1, 4096, 0, 4096, 0, 464, 0, 0, 0, 0, 0, 4096, 2872, 0, 0, 9, 0, 0, 0, 4088, 0, 0, 4096, 0, 4096, 4092, 0, 85, 0, 0, 4096, 0, 0, 4096, 4096, 1136, 1, 0, 0, 4096, 3, 0, 0, 16, 0, 0, 3623, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4093, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 1, 4096, 0, 4096, 0, 605, 0, 0, 0, 0, 0, 4087, 2836, 0, 0, 182, 0, 0, 0, 4091, 4, 0, 4096, 0, 4096, 4096, 0, 47, 0, 0, 4096, 0, 1, 4096, 4094, 595, 1, 0, 0, 4093, 0, 0, 0, 28, 0, 0, 3914, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 4094, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 2, 4090, 0, 4096, 1, 91, 0, 9, 0, 0, 0, 4084, 3086, 0, 0, 3, 0, 0, 0, 4090, 1, 0, 4096, 0, 4096, 4091, 0, 73, 1, 0, 4095, 0, 2, 4093, 4094, 1267, 5, 0, 0, 4095, 0, 0, 0, 39, 0, 0, 3702, 0, 3, 0, 17, 0, 0, 0, 0, 4096, 0, 4023, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 24, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 2, 1, 0, 4095, 3864, 0, 0, 0, 0, 0, 0, 0, 3518, 1, 0, 25, 2, 0, 0, 4089, 519, 96, 84, 0, 0, 0, 0, 0, 0, 0, 3976, 3, 0, 4085, 1, 292, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4094, 2, 0, 3575, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4092, 1, 1, 1, 68, 3892, 144, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 1, 0, 2, 4, 2, 4096, 202, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4094, 0, 4082, 2, 0, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8186, 8334, 8357, 7748, 8203, 8298, 8203] avg=8192.0000 max=8357 max_rank=3 min=7748 min_rank=4 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32452 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4094, 1, 1, 0, 13, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 4094, 3867, 0, 0, 0, 0, 0, 0, 0, 3419, 1, 0, 37, 2, 1, 0, 4079, 519, 96, 84, 0, 0, 0, 0, 0, 1, 0, 3977, 0, 0, 4089, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4095, 1, 3, 3574, 0, 377, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 1, 4095, 1, 1, 1, 191, 3893, 144, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 0, 1, 1, 7, 1, 4094, 202, 0, 0, 0, 0, 0, 0, 5, 0, 1, 4092, 1, 4029, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8042, 8238, 8366, 8054, 8344, 8145, 8151] avg=8192.0000 max=8366 max_rank=3 min=8042 min_rank=1 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32694 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 4090, 3870, 0, 0, 0, 0, 0, 0, 0, 3715, 0, 0, 12, 0, 0, 0, 4071, 519, 96, 84, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 4095, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 3, 3577, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 47, 3894, 142, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 1, 0, 0, 0, 4096, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4076, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8079, 8497, 8125, 7935, 8181, 8339, 8194] avg=8192.0000 max=8497 max_rank=2 min=7935 min_rank=4 imbalance_ratio=1.037231 max_load_reduction=57039 max_load_reduction_pct=0.870346 moved_to_echo=32649 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, 0, 0, 0, 0, 0, 4087, 3866, 0, 0, 0, 0, 0, 0, 0, 2836, 0, 0, 182, 0, 0, 0, 4091, 519, 96, 84, 0, 0, 0, 0, 0, 4, 0, 3978, 0, 0, 4096, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 3577, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 28, 3894, 144, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 0, 0, 0, 4096, 202, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 4094, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8558, 7808, 8125, 8269, 8160, 8212, 8217] avg=8192.0000 max=8558 max_rank=1 min=7808 min_rank=2 imbalance_ratio=1.044678 max_load_reduction=56978 max_load_reduction_pct=0.869415 moved_to_echo=32858 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 12, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 1, 0, 0, 0, 4095, 3870, 0, 0, 0, 0, 0, 0, 0, 3597, 0, 0, 8, 0, 0, 0, 4095, 519, 96, 84, 0, 0, 0, 0, 0, 2, 0, 3978, 0, 0, 4093, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 2, 3577, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 0, 0, 24, 3895, 142, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 4096, 201, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4089, 0, 1, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8155, 8399, 8120, 7978, 8158, 8323, 8209] avg=8192.0000 max=8399 max_rank=2 min=7978 min_rank=4 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32668 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 4096, 3868, 0, 0, 0, 0, 0, 0, 0, 2872, 0, 0, 9, 0, 0, 0, 4088, 519, 96, 85, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 4092, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3577, 0, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 3, 0, 0, 16, 3894, 143, 0, 0, 0, 0, 0, 0, 0, 0, 3623, 0, 0, 0, 0, 0, 4096, 202, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4093, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8428, 7669, 8155, 8809, 8153, 7921, 8213] avg=8192.0000 max=8809 max_rank=4 min=7669 min_rank=2 imbalance_ratio=1.075317 max_load_reduction=56727 max_load_reduction_pct=0.865585 moved_to_echo=33096 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4072, 0, 0, 0, 27, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 9, 0, 0, 1, 4092, 3868, 0, 0, 0, 0, 0, 0, 0, 3894, 0, 0, 33, 0, 0, 0, 4072, 519, 96, 86, 0, 0, 0, 0, 0, 1, 3, 3975, 8, 0, 4078, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 4087, 0, 15, 3573, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 14, 3894, 142, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 1, 0, 0, 4096, 202, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4083, 0, 4087, 1, 0, 22, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8035, 8700, 8158, 7810, 8161, 8279, 8198] avg=8192.0000 max=8700 max_rank=2 min=7810 min_rank=4 imbalance_ratio=1.062012 max_load_reduction=56836 max_load_reduction_pct=0.867249 moved_to_echo=32448 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 2, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 91, 0, 9, 0, 0, 0, 4084, 3868, 0, 0, 0, 0, 0, 0, 0, 3086, 0, 0, 3, 0, 0, 0, 4090, 525, 103, 84, 0, 0, 0, 0, 0, 1, 0, 3965, 0, 0, 4091, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 2, 3568, 0, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4095, 0, 0, 0, 39, 3895, 142, 0, 0, 0, 0, 0, 0, 0, 0, 3702, 0, 3, 0, 17, 0, 4096, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4023, 0, 0, 28, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8053, 7891, 8130, 8933, 8176, 8019, 8151] avg=8192.0000 max=8933 max_rank=4 min=7891 min_rank=2 imbalance_ratio=1.090454 max_load_reduction=56603 max_load_reduction_pct=0.863693 moved_to_echo=33279 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 28672, 0, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 128, 32640, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 31872, 128, 0, 32768, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 512, 31232, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[27008, 128, 0, 384, 128, 128, 0, 32768, 4224, 896, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 31232, 128, 128, 128, 128, 128, 32768, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 1920, 0, 128, 128, 128, 128, 32768, 30976, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1668, 0, 0, 0, 0, 0, 0, 0, 3715, 0, 0, 0, 4096, 0, 156, 0, 0, 4095, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 3121, 4096, 57, 0, 3714, 0, 1, 0, 1, 0, 0, 0, 4096, 0, 4096, 3961, 0, 0, 4096, 0, 0, 4089, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 3523, 0, 0, 0, 0, 0, 0, 0, 3673, 0, 0, 0, 4096, 0, 4, 1, 1, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 3481, 4096, 0, 0, 1612, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 4095, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 0, 4091, 2, 0, 4094, 0, 0, 4, 3418, 0, 0, 0, 2, 0, 0, 0, 3722, 2, 0, 1, 4094, 0, 2, 0, 0, 4094, 0, 1, 2, 2, 0, 4091, 6, 0, 4093, 0, 4093, 0, 2, 3031, 4094, 0, 2, 2127, 0, 0, 0, 0, 4, 1, 2, 4094, 3, 4094, 4074, 2, 0, 4094, 0, 2, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 2, 1, 4094, 0, 0, 0, 3451, 1, 0, 0, 1, 0, 0, 0, 4035, 1, 1, 1, 4093, 1, 7, 0, 0, 4094, 1, 1, 2, 0, 1, 4092, 2, 0, 4092, 3, 4094, 1, 2, 2181, 4095, 0, 1, 2630, 0, 0, 1, 0, 2, 1, 1, 4094, 0, 4094, 4069, 1, 4, 4095, 0, 1, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3246, 0, 0, 0, 0, 0, 0, 0, 3912, 0, 0, 0, 4096, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 3634, 4096, 10, 0, 1502, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4071, 0, 0, 4095, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3305, 0, 0, 0, 0, 0, 0, 0, 3626, 0, 0, 0, 4096, 0, 49, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 2788, 4096, 2, 0, 2568, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4046, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3359, 0, 5, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 4096, 0, 36, 4, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 4096, 0, 0, 2547, 4091, 59, 0, 2621, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4093, 4036, 0, 0, 4092, 0, 0, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2534, 0, 0, 0, 0, 2, 0, 0, 3694, 0, 0, 0, 4096, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 4092, 0, 2, 4096, 0, 4096, 0, 0, 2780, 4096, 4, 0, 3294, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4079, 0, 0, 4095, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1668, 0, 0, 0, 0, 0, 4096, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 0, 0, 0, 4096, 0, 332, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 4095, 0, 0, 0, 0, 3020, 1042, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3121, 2722, 57, 0, 3714, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 0, 0, 44, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6796, 8143, 8313, 8190, 9615, 8193, 8094] avg=8192.0000 max=9615 max_rank=5 min=6796 min_rank=1 imbalance_ratio=1.173706 max_load_reduction=55921 max_load_reduction_pct=0.853287 moved_to_echo=34092 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4091, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3418, 0, 0, 0, 2, 0, 4092, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 3722, 2, 0, 1, 4094, 0, 332, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 1, 2, 2, 3019, 1041, 0, 0, 0, 0, 0, 0, 0, 4091, 6, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3031, 2721, 0, 2, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 2, 4094, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 2, 0, 44, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8547, 8151, 8161, 8191, 7883, 8198, 8216] avg=8192.0000 max=8547 max_rank=1 min=7883 min_rank=5 imbalance_ratio=1.043335 max_load_reduction=56989 max_load_reduction_pct=0.869583 moved_to_echo=32488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 2, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3451, 1, 0, 0, 1, 0, 4093, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 1, 1, 1, 4093, 1, 332, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4094, 1, 1, 2, 0, 3020, 1041, 0, 0, 0, 0, 0, 0, 1, 4092, 2, 0, 4092, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2181, 2722, 0, 1, 2630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 1, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 1, 4, 43, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8578, 8464, 8166, 8192, 7536, 8193, 8214] avg=8192.0000 max=8578 max_rank=1 min=7536 min_rank=5 imbalance_ratio=1.047119 max_load_reduction=56958 max_load_reduction_pct=0.869110 moved_to_echo=32135 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3246, 0, 0, 0, 0, 0, 4095, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 0, 0, 0, 4096, 0, 332, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 3020, 1042, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 2722, 10, 0, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 43, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8373, 8340, 8168, 8193, 7868, 8192, 8210] avg=8192.0000 max=8373 max_rank=1 min=7868 min_rank=5 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 0, 0, 0, 0, 0, 4096, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 3673, 0, 0, 0, 4096, 0, 332, 0, 0, 0, 0, 0, 0, 0, 4, 1, 1, 4096, 0, 0, 0, 0, 3020, 1042, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3481, 2722, 0, 0, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 43, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8651, 8101, 8164, 8192, 7815, 8192, 8230] avg=8192.0000 max=8651 max_rank=1 min=7815 min_rank=5 imbalance_ratio=1.056030 max_load_reduction=56885 max_load_reduction_pct=0.867996 moved_to_echo=32429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3305, 0, 0, 0, 0, 0, 4096, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 3626, 0, 0, 0, 4096, 0, 332, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 4096, 0, 0, 0, 0, 3020, 1042, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2788, 2722, 2, 0, 2568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 44, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8433, 8054, 8207, 8192, 8080, 8192, 8186] avg=8192.0000 max=8433 max_rank=1 min=8054 min_rank=2 imbalance_ratio=1.029419 max_load_reduction=57103 max_load_reduction_pct=0.871323 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3359, 0, 5, 0, 0, 0, 4096, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 4096, 0, 332, 0, 0, 0, 0, 0, 0, 0, 36, 4, 0, 4096, 0, 0, 0, 0, 3017, 1040, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2547, 2719, 59, 0, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 44, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8491, 8156, 8193, 8196, 7946, 8188, 8174] avg=8192.0000 max=8491 max_rank=1 min=7946 min_rank=5 imbalance_ratio=1.036499 max_load_reduction=57045 max_load_reduction_pct=0.870438 moved_to_echo=32504 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2534, 0, 0, 0, 0, 2, 4096, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 3694, 0, 0, 0, 4096, 0, 336, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 3023, 1045, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2780, 2715, 4, 0, 3294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 37, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 7667, 8127, 8164, 8190, 8793, 8188, 8212] avg=8192.0000 max=8793 max_rank=5 min=7667 min_rank=1 imbalance_ratio=1.073364 max_load_reduction=56743 max_load_reduction_pct=0.865829 moved_to_echo=33383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32512, 128, 128, 384, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 23680, 21888, 256, 128, 20096, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 24576, 128, 128, 0, 128, 128, 32768, 8320, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[384, 128, 128, 32768, 128, 128, 128, 128, 24192, 8448, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 30208, 128, 128, 128, 32768, 128, 2688, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[2, 3, 0, 0, 0, 2, 2, 3, 0, 2, 1, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 3, 2, 4088, 0, 0, 0, 0, 4094, 0, 2, 1, 18, 0, 2, 4096, 0, 4094, 4091, 0, 1, 0, 4094, 2, 0, 0, 1, 4094, 4093, 4093, 4086, 2, 0, 4092, 4094, 4093, 3, 0, 4094, 0, 4090, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4066, 0, 4096, 4096, 0, 0, 0, 4096, 0, 143, 0, 0, 4088, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 30, 3963, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[2, 2, 0, 2, 0, 2, 2, 0, 0, 3, 0, 1, 1, 0, 1, 0, 0, 1, 4094, 0, 2, 2, 4095, 0, 0, 0, 0, 4092, 1, 1, 0, 2, 0, 2, 4095, 2, 4094, 4094, 0, 1, 0, 4095, 2, 0, 0, 0, 4093, 4093, 4093, 4084, 2, 0, 4091, 4094, 4094, 0, 0, 4094, 10, 4095, 0, 0, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 4094, 0, 0, 0, 22, 0, 0, 4090, 0, 4096, 4096, 0, 0, 0, 4096, 0, 43, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 6, 4035, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 105, 0, 0, 4061, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 15, 4014, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 4, 0, 0, 4045, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 6, 0, 4066, 0, 4094, 4096, 0, 0, 0, 4091, 0, 171, 0, 0, 3987, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 12, 4096, 38, 4041, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 1, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 4096, 0, 0, 0, 4096, 0, 46, 0, 0, 4090, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1, 4096, 15, 4046, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 4092, 0, 4096, 4096, 0, 0, 0, 4096, 0, 76, 0, 0, 4036, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 6, 4075, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 143, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 3963, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8195, 8193, 8185, 8171, 8283, 8196, 8123] avg=8192.0000 max=8283 max_rank=5 min=8123 min_rank=7 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32773 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 0, 0, 0, 2, 2, 3, 4093, 4089, 1, 0, 0, 0, 0, 0, 0, 2, 1, 0, 2, 0, 0, 0, 4094, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 2, 4088, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 1, 18, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 9, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 2, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4086, 2, 0, 4092, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4090, 0, 0, 0, 2, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8191, 8195, 8201, 8199, 8148, 8187, 8220] avg=8192.0000 max=8220 max_rank=7 min=8148 min_rank=5 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32754 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 0, 2, 0, 2, 2, 0, 4093, 4089, 1, 0, 0, 0, 0, 0, 0, 3, 0, 1, 1, 0, 1, 0, 4094, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 2, 2, 4095, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 0, 2, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 2, 9, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4084, 2, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 10, 4095, 0, 0, 1, 1, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8193, 8203, 8182, 8203, 8146, 8181, 8235] avg=8192.0000 max=8235 max_rank=7 min=8146 min_rank=5 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32765 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 76, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 4075, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8191, 8198, 8190, 8197, 8164, 8196, 8211] avg=8192.0000 max=8211 max_rank=7 min=8164 min_rank=5 imbalance_ratio=1.002319 max_load_reduction=57325 max_load_reduction_pct=0.874710 moved_to_echo=32768 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4096, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 22, 4087, 1, 4, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 43, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 4035, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8190, 8198, 8208, 8195, 8191, 8192, 8171] avg=8192.0000 max=8208 max_rank=3 min=8171 min_rank=7 imbalance_ratio=1.001953 max_load_reduction=57328 max_load_reduction_pct=0.874756 moved_to_echo=32749 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 105, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 4014, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8191, 8198, 8184, 8201, 8218, 8196, 8159] avg=8192.0000 max=8218 max_rank=5 min=8159 min_rank=7 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32774 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 0, 0, 4096, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4045, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 4085, 1, 0, 0, 0, 0, 0, 0, 6, 0, 4066, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 171, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 38, 4041, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8193, 8153, 8193, 8177, 8206, 8196, 8221] avg=8192.0000 max=8221 max_rank=7 min=8153 min_rank=2 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32800 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4090, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 3, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 46, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 15, 4046, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8198, 8193, 8193, 8180, 8192, 8196] avg=8192.0000 max=8198 max_rank=2 min=8180 min_rank=5 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32761 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32512, 128, 640, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 32384, 0, 0, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 32768, 128, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4096, 4096, 4016, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 26, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 65, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 4096, 4095, 2, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 1, 0, 4096, 0, 0, 0, 3, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4096, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4093, 4096, 4096, 1, 0, 4096, 0, 2, 0, 10, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4080, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 20, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 4092, 4096, 4096, 6, 0, 4096, 0, 6, 0, 18, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 4090, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4092, 4096, 4056, 3, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 51, 4096, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4094, 4093, 4091, 5, 0, 0, 2, 3, 4094, 1, 4092, 2, 3, 2, 0, 0, 0, 1, 2, 4094, 0, 0, 3, 1, 0, 4094, 0, 0, 4091, 2, 0, 4093, 4093, 4093, 0, 0, 4093, 8, 0, 2, 3, 4093, 0, 3, 4079, 2, 1, 0, 3, 0, 2, 0, 4092, 0, 0, 3, 0, 2, 4094, 0, 2, 2, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4094, 4094, 4094, 1, 0, 1, 2, 2, 4094, 0, 4094, 1, 2, 2, 0, 0, 0, 0, 2, 4094, 1, 0, 6, 1, 0, 4094, 0, 0, 4093, 1, 0, 4088, 4094, 4093, 0, 0, 4094, 0, 2, 2, 0, 4095, 0, 2, 4092, 2, 2, 0, 2, 0, 2, 0, 4093, 0, 0, 1, 0, 2, 4094, 0, 0, 1, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4094, 4096, 4084, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 13, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4094, 4096, 4096, 0, 0, 4096, 0, 1, 0, 9, 4096, 1, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 19, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4, 0, 0, 0, 4075, 15, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4081, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8192, 8179, 8193, 8193, 8187, 8190, 8192] avg=8192.0000 max=8210 max_rank=0 min=8179 min_rank=2 imbalance_ratio=1.002197 max_load_reduction=57326 max_load_reduction_pct=0.874725 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 19, 4091, 5, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4092, 2, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4094, 0, 0, 3, 1, 4072, 15, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4093, 0, 0, 0, 8, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4078, 0, 3, 4079, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 4092, 0, 0, 3, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 2, 2, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8194, 8190, 8193, 8194, 8166, 8189, 8196] avg=8192.0000 max=8214 max_rank=0 min=8166 min_rank=5 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 19, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 26, 0, 0, 0, 4075, 15, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4081, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8192, 8206, 8196, 8190, 8239, 8190, 8192] avg=8192.0000 max=8239 max_rank=5 min=8131 min_rank=0 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32811 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 19, 4094, 1, 0, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 1, 0, 6, 1, 4073, 15, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4, 1, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4093, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 2, 4092, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 4093, 0, 0, 1, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 1, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8193, 8194, 8187, 8189, 8178, 8189, 8193] avg=8192.0000 max=8213 max_rank=0 min=8178 min_rank=5 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4092, 19, 4056, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4075, 15, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 4081, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8192, 8189, 8196, 8190, 8217, 8190, 8192] avg=8192.0000 max=8217 max_rank=5 min=8170 min_rank=0 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4080, 19, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 20, 0, 0, 0, 4075, 15, 2, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 6, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4081, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 8183, 8186, 8202, 8195, 8197, 8186] avg=8192.0000 max=8202 max_rank=4 min=8183 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=57334 max_load_reduction_pct=0.874847 moved_to_echo=32780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 19, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 4075, 15, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4081, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8192, 8190, 8196, 8191, 8173, 8190, 8192] avg=8192.0000 max=8212 max_rank=0 min=8173 min_rank=5 imbalance_ratio=1.002441 max_load_reduction=57324 max_load_reduction_pct=0.874695 moved_to_echo=32746 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 13, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 13, 0, 0, 0, 4076, 21, 6, 3, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4075, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8189, 8205, 8189, 8187, 8181, 8201, 8193] avg=8192.0000 max=8205 max_rank=2 min=8181 min_rank=5 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 0, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 32640, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 256, 32640, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 128, 128, 32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 896, 0, 0, 4090, 0, 0, 4096, 0, 4096, 4096, 0, 0, 3696, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 120, 4068, 0, 0, 4096, 0, 0, 3257, 4096, 0, 0, 0, 215, 0, 0, 4096, 0, 4096, 0, 0, 0, 7, 35, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 2, 0, 0, 4094, 0, 0, 2, 4092, 1, 0, 0, 927, 3, 2, 4092, 0, 3, 4091, 3, 4093, 4094, 0, 2, 3783, 1, 0, 2, 4093, 4094, 0, 2, 0, 0, 0, 183, 4094, 2, 0, 4093, 0, 4, 3204, 4093, 4, 1, 0, 91, 0, 2, 4093, 3, 4093, 3, 3, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2, 0, 4094, 0, 0, 1, 0, 0, 4094, 0, 0, 2, 4093, 1, 0, 4, 1984, 2, 1, 4090, 0, 3, 4090, 2, 4094, 4094, 0, 1, 3488, 2, 0, 2, 4094, 4094, 0, 2, 0, 0, 0, 25, 4095, 1, 0, 4093, 0, 3, 2533, 4093, 2, 1, 1, 158, 0, 2, 4093, 2, 4094, 2, 3, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 603, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4060, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 434, 4093, 0, 0, 4096, 0, 0, 3092, 4096, 0, 0, 0, 8, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 1, 1212, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 3789, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 24, 4091, 0, 0, 4096, 0, 0, 3163, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 9, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2, 0, 4088, 0, 0, 0, 4096, 0, 0, 7, 709, 0, 0, 4085, 0, 0, 4096, 0, 4096, 4093, 3, 0, 4045, 0, 0, 0, 4096, 4092, 0, 0, 0, 8, 0, 54, 4079, 0, 0, 4096, 0, 0, 3341, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 16, 43, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 2, 836, 0, 0, 4093, 0, 0, 4096, 0, 4096, 4096, 0, 0, 3737, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 723, 4095, 0, 0, 4096, 0, 0, 2805, 4096, 0, 0, 0, 90, 0, 0, 4096, 0, 4096, 0, 0, 0, 2, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 4, 917, 0, 0, 4088, 0, 0, 4096, 0, 4096, 4095, 0, 0, 3349, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 200, 4080, 0, 0, 4096, 0, 0, 3735, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 4096, 0, 0, 0, 2, 10, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 896, 0, 0, 4090, 0, 0, 3089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 4096, 0, 0, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 4096, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 964, 0, 0, 3257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 4096, 0, 2797, 1007, 219, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 35, 0, 0, 3743, 335, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8075, 8145, 8093, 8289, 8334, 8216] avg=8192.0000 max=8334 max_rank=6 min=8075 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32932 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 3, 2, 4092, 0, 3, 3086, 3, 0, 0, 0, 0, 0, 0, 0, 0, 353, 4094, 0, 2, 3783, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 4094, 0, 2, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 963, 0, 4, 3204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 91, 0, 2, 4093, 3, 2795, 1005, 219, 0, 0, 0, 0, 0, 4093, 3, 3, 0, 0, 0, 1, 0, 3740, 335, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8189, 8116, 8235, 8153, 8267, 8213, 8175] avg=8192.0000 max=8267 max_rank=5 min=8116 min_rank=2 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4094, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 4093, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1984, 2, 1, 4090, 0, 3, 3085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 353, 4094, 0, 1, 3488, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 4094, 0, 2, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 963, 0, 3, 2533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 158, 0, 2, 4093, 2, 2795, 1005, 219, 0, 0, 0, 0, 0, 4094, 2, 3, 0, 0, 0, 1, 0, 3741, 335, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8194, 9167, 7940, 7996, 7595, 8278, 8176] avg=8192.0000 max=9167 max_rank=2 min=7595 min_rank=5 imbalance_ratio=1.119019 max_load_reduction=56369 max_load_reduction_pct=0.860123 moved_to_echo=32045 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 603, 0, 0, 4096, 0, 0, 3089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 4096, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 4096, 0, 0, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 964, 0, 0, 3092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 2797, 1007, 219, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3743, 335, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8190, 7788, 8509, 8407, 8149, 8127, 8174] avg=8192.0000 max=8509 max_rank=3 min=7788 min_rank=2 imbalance_ratio=1.038696 max_load_reduction=57027 max_load_reduction_pct=0.870163 moved_to_echo=32857 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 709, 0, 0, 4085, 0, 0, 3089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 4093, 3, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 4092, 0, 0, 0, 8, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 964, 0, 0, 3341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2797, 1007, 219, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 43, 0, 7, 3743, 335, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8191, 7883, 8494, 8031, 8384, 8119, 8240] avg=8192.0000 max=8494 max_rank=3 min=7883 min_rank=2 imbalance_ratio=1.036865 max_load_reduction=57042 max_load_reduction_pct=0.870392 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1212, 0, 0, 4096, 0, 0, 3089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 4096, 0, 0, 3789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 4096, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 964, 0, 0, 3163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2797, 1007, 219, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 3743, 335, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8397, 8238, 7997, 8218, 8119, 8183] avg=8192.0000 max=8397 max_rank=2 min=7997 min_rank=4 imbalance_ratio=1.025024 max_load_reduction=57139 max_load_reduction_pct=0.871872 moved_to_echo=32517 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 836, 0, 0, 4093, 0, 0, 3089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 4096, 0, 0, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 4095, 0, 0, 0, 0, 0, 723, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 964, 0, 0, 2805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 4096, 0, 2797, 1007, 219, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 3, 0, 0, 3743, 335, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8018, 8186, 8695, 7864, 8209, 8179] avg=8192.0000 max=8695 max_rank=4 min=7864 min_rank=5 imbalance_ratio=1.061401 max_load_reduction=56841 max_load_reduction_pct=0.867325 moved_to_echo=32947 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 917, 0, 0, 4088, 0, 0, 3087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 4095, 0, 0, 3349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 4096, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 955, 0, 0, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 2803, 1009, 226, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 10, 0, 0, 3747, 338, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8195, 8092, 7789, 8164, 8770, 8137, 8193] avg=8192.0000 max=8770 max_rank=5 min=7789 min_rank=3 imbalance_ratio=1.070557 max_load_reduction=56766 max_load_reduction_pct=0.866180 moved_to_echo=33264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31104, 32768, 0, 128, 0, 128, 0, 1792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 0, 0, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 128, 128, 32768, 0, 128, 24704, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 128, 128, 29952, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 7808, 0, 128, 25216, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 640, 0, 128, 32768, 128, 22400, 8064, 1792, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2816, 32768, 128, 128, 29952, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 3271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 4095, 4096, 4096, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 24, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 826] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3, 0, 3, 2, 0, 3, 4089, 4093, 1, 4094, 0, 1, 2, 3, 0, 3, 0, 4093, 2, 0, 0, 0, 0, 4052, 1, 0, 0, 3796, 2, 0, 2, 0, 1, 1, 0, 2, 2, 0, 4094, 0, 4094, 4094, 4042, 4093, 0, 0, 13, 2, 0, 4093, 3, 2, 0, 0, 23, 4093, 4093, 2, 2, 4093, 0, 0, 4093, 356] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4, 0, 6, 3, 0, 3, 4086, 4093, 0, 4093, 0, 3, 0, 3, 0, 3, 0, 4088, 3, 0, 0, 1, 0, 4074, 0, 1, 0, 3872, 3, 0, 3, 0, 2, 0, 0, 3, 3, 0, 4093, 0, 4093, 4092, 3940, 4093, 0, 0, 16, 3, 0, 4092, 6, 2, 0, 0, 16, 4093, 4091, 3, 3, 4092, 0, 0, 4093, 365] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4095, 4061, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 262] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4093, 4096, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 277] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4088, 0, 4090, 10, 0, 0, 0, 13, 0, 0, 4095, 0, 2, 0, 0, 0, 4066, 0, 0, 0, 3198, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4081, 4096, 4094, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 28, 0, 4096, 4096, 0, 0, 4096, 0, 3, 4096, 902] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 29] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 4096, 4096, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 8, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 236] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 3693, 398, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4086, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 0, 0, 4096, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3724, 370, 0, 0, 0, 0, 0, 0, 4077, 32, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 24, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3690, 0, 0, 0, 826, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8188, 8202, 7739, 8190, 8209, 8204, 8612] avg=8192.0000 max=8612 max_rank=7 min=7739 min_rank=3 imbalance_ratio=1.051270 max_load_reduction=56924 max_load_reduction_pct=0.868591 moved_to_echo=33215 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 6, 3, 0, 3, 4086, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 3, 0, 3, 3690, 397, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 1, 0, 4074, 12, 8, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3872, 3, 0, 3, 0, 4093, 371, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 3, 0, 4093, 0, 3722, 370, 0, 0, 0, 0, 0, 0, 4093, 32, 3940, 0, 0, 0, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 6, 2, 0, 0, 16, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 3, 3687, 0, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8189, 8186, 8343, 8193, 8084, 8197, 8149] avg=8192.0000 max=8343 max_rank=3 min=8084 min_rank=5 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32623 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 2, 0, 3, 4089, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 2, 3, 0, 3, 3692, 397, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 4052, 12, 8, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3796, 2, 0, 2, 0, 4093, 371, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2, 2, 0, 4094, 0, 3722, 370, 0, 0, 0, 0, 0, 0, 4094, 32, 4042, 0, 0, 0, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 2, 0, 0, 23, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 2, 3688, 0, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8193, 8167, 8265, 8192, 8183, 8202, 8141] avg=8192.0000 max=8265 max_rank=3 min=8141 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32718 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3694, 398, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3829, 0, 0, 0, 0, 4096, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3725, 370, 0, 0, 0, 0, 0, 0, 4096, 32, 4093, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3690, 0, 0, 0, 277, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8188, 8201, 8296, 8191, 8225, 8180, 8063] avg=8192.0000 max=8296 max_rank=3 min=8063 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32659 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3693, 398, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 0, 0, 0, 0, 4096, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3724, 370, 0, 0, 0, 0, 0, 0, 4095, 32, 4061, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3690, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8187, 8209, 8339, 8190, 8189, 8182, 8048] avg=8192.0000 max=8339 max_rank=3 min=8048 min_rank=7 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32609 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3694, 398, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4070, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3725, 370, 0, 0, 0, 0, 0, 0, 4096, 32, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3690, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8188, 8186, 8560, 8191, 8224, 8180, 7815] avg=8192.0000 max=8560 max_rank=3 min=7815 min_rank=7 imbalance_ratio=1.044922 max_load_reduction=56976 max_load_reduction_pct=0.869385 moved_to_echo=32410 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 10, 0, 0, 0, 13, 0, 3694, 398, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 4066, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3198, 0, 0, 0, 0, 4096, 372, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 3724, 370, 0, 0, 0, 0, 0, 0, 4081, 32, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 28, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3690, 0, 3, 0, 902, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8205, 8183, 7666, 8192, 8207, 8208, 8691] avg=8192.0000 max=8691 max_rank=7 min=7666 min_rank=3 imbalance_ratio=1.060913 max_load_reduction=56845 max_load_reduction_pct=0.867386 moved_to_echo=33298 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 3699, 401, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4083, 13, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 4096, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3724, 377, 0, 0, 0, 0, 0, 0, 4094, 20, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3685, 0, 0, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8198, 8202, 8328, 8197, 8215, 8183, 8017] avg=8192.0000 max=8328 max_rank=3 min=8017 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32612 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 0, 128, 0, 32640, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 256, 32640, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 29568, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 29568, 0, 128, 0, 3328, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 29824, 128, 0, 128, 0, 32768, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 0, 29824, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3951, 0, 0, 0, 0, 4096, 0, 0, 0, 45, 0, 0, 2486, 4096, 4095, 0, 0, 4096, 0, 4056, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4029, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 0, 0, 2643, 4096, 4096, 0, 0, 4096, 0, 4095, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4083, 0, 0, 0, 0, 4095, 0, 0, 0, 12, 0, 0, 1643, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[2, 0, 0, 4093, 4052, 2, 0, 0, 2, 4094, 3, 2, 0, 14, 3, 0, 1388, 4093, 4094, 3, 1, 4092, 1, 4091, 0, 4093, 0, 4092, 1, 0, 10, 2, 2, 4094, 0, 0, 3, 0, 2, 2724, 0, 0, 0, 1, 2, 2, 0, 2, 0, 0, 3, 0, 4093, 2, 0, 1, 4093, 0, 4094, 2, 0, 3, 4092, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4049, 0, 0, 0, 0, 4096, 0, 0, 0, 62, 0, 0, 1895, 4096, 4095, 0, 0, 4096, 0, 4087, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 2197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4070, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 2063, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3962, 0, 4, 2, 0, 4094, 0, 0, 0, 102, 0, 0, 2102, 4096, 4092, 0, 0, 4096, 0, 4069, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3, 0, 0, 4092, 4047, 3, 0, 0, 4, 4093, 3, 3, 1, 9, 3, 0, 1676, 4091, 4093, 4, 1, 4092, 0, 4093, 0, 4092, 0, 4092, 0, 0, 24, 3, 3, 4093, 0, 0, 0, 0, 3, 2427, 0, 0, 1, 1, 3, 3, 0, 2, 0, 2, 4, 0, 4092, 2, 0, 2, 4093, 0, 4092, 3, 1, 5, 4091, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3951, 0, 0, 0, 61, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 45, 0, 0, 4060, 1, 0, 0, 0, 0, 0, 0, 2486, 0, 2119, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1751, 1915, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8202, 8661, 8189, 7798, 8192, 8191, 8192] avg=8192.0000 max=8661 max_rank=2 min=7798 min_rank=4 imbalance_ratio=1.057251 max_load_reduction=56875 max_load_reduction_pct=0.867844 moved_to_echo=32373 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4093, 4052, 2, 0, 0, 61, 3, 0, 0, 0, 0, 0, 0, 2, 4094, 3, 2, 0, 14, 3, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 1388, 0, 2119, 3, 1, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4092, 1, 0, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 3, 0, 2, 2724, 1914, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 2, 4093, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 2, 0, 1, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 3, 4092, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8175, 7603, 8195, 8775, 8192, 8192, 8191] avg=8192.0000 max=8775 max_rank=4 min=7603 min_rank=2 imbalance_ratio=1.071167 max_load_reduction=56761 max_load_reduction_pct=0.866104 moved_to_echo=33350 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4029, 0, 0, 0, 61, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 2643, 0, 2120, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1505, 1915, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8171, 8858, 8189, 7553, 8192, 8192, 8192] avg=8192.0000 max=8858 max_rank=2 min=7553 min_rank=4 imbalance_ratio=1.081299 max_load_reduction=56678 max_load_reduction_pct=0.864838 moved_to_echo=32129 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4070, 0, 0, 0, 61, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 2063, 0, 2120, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2040, 1915, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8174, 8279, 8189, 8087, 8191, 8193, 8193] avg=8192.0000 max=8279 max_rank=2 min=8087 min_rank=4 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32664 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4083, 0, 0, 0, 61, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 12, 0, 0, 4060, 1, 0, 0, 0, 0, 0, 0, 1643, 0, 2120, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2455, 1915, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8168, 7859, 8189, 8502, 8192, 8191, 8192] avg=8192.0000 max=8502 max_rank=4 min=7859 min_rank=2 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=33077 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3962, 0, 4, 2, 61, 3, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 102, 0, 0, 4059, 1, 0, 0, 0, 0, 0, 0, 2102, 0, 2118, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2053, 1913, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8256, 8289, 8189, 8099, 8192, 8191, 8192] avg=8192.0000 max=8289 max_rank=2 min=8099 min_rank=4 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32674 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4049, 0, 0, 0, 64, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 62, 0, 0, 4060, 1, 0, 0, 0, 0, 0, 0, 1895, 0, 2113, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 2197, 1918, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8219, 8095, 8188, 8246, 8192, 8191, 8192] avg=8192.0000 max=8246 max_rank=4 min=8095 min_rank=2 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32821 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4092, 4047, 3, 0, 0, 61, 3, 0, 0, 0, 0, 0, 0, 4, 4093, 3, 3, 1, 9, 3, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 1676, 0, 2118, 4, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4092, 0, 0, 24, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 3, 2427, 1914, 36, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 3, 0, 2, 4093, 4090, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4092, 2, 0, 2, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1, 5, 4091, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8171, 7892, 8208, 8476, 8193, 8195, 8192] avg=8192.0000 max=8476 max_rank=4 min=7892 min_rank=2 imbalance_ratio=1.034668 max_load_reduction=57060 max_load_reduction_pct=0.870667 moved_to_echo=33056 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 0, 128, 0, 128, 17152, 15360, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[16000, 0, 17024, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 384, 128, 0, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 32768, 32256, 128, 128, 128, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2439, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1754, 0, 0, 0, 1, 0, 0, 198, 4096, 0, 3805, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1821, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1992, 0, 0, 0, 1, 0, 0, 390, 4096, 0, 3993, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[673, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1873, 0, 0, 0, 0, 0, 0, 1668, 4096, 0, 4013, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[495, 1, 4072, 0, 2, 0, 0, 7, 1, 0, 0, 4092, 1, 2, 3, 4088, 0, 3, 0, 3, 0, 0, 2, 2, 33, 0, 2, 2, 4093, 4094, 4093, 3, 0, 0, 0, 4093, 4089, 0, 3, 3, 4093, 0, 3, 0, 4094, 0, 3509, 2, 0, 3, 0, 11, 3, 65, 4093, 6, 4093, 2, 2, 4091, 4093, 0, 0, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[969, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2621, 0, 0, 0, 0, 0, 0, 532, 4096, 0, 4072, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[687, 0, 4084, 0, 3, 0, 0, 9, 0, 0, 0, 4091, 0, 3, 4, 4077, 0, 4, 0, 4, 0, 3, 1, 2, 16, 0, 2, 3, 4092, 4093, 4092, 4, 1, 0, 1, 4093, 4088, 0, 7, 3, 4093, 0, 8, 0, 4093, 0, 3083, 2, 0, 4, 0, 11, 3, 313, 4092, 1, 4093, 3, 0, 4091, 4087, 0, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1740, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1202, 0, 0, 0, 25, 2, 0, 1307, 4096, 0, 3995, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[64, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3865, 0, 0, 0, 0, 14, 0, 271, 4096, 0, 4006, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2439, 0, 4092, 0, 0, 0, 0, 0, 2488, 514, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1608, 0, 1754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 198, 4096, 0, 3493, 1, 0, 0, 0, 0, 0, 0, 3805, 0, 0, 89, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9535, 8195, 8191, 8186, 8192, 7458, 7789, 7990] avg=8192.0000 max=9535 max_rank=0 min=7458 min_rank=5 imbalance_ratio=1.163940 max_load_reduction=56001 max_load_reduction_pct=0.854507 moved_to_echo=31429 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[495, 1, 4072, 0, 2, 0, 0, 7, 2487, 514, 2, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 1, 2, 3, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 2, 2, 4093, 4090, 0, 0, 0, 0, 0, 0, 33, 0, 2, 2, 4089, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4089, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 1607, 0, 3509, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 11, 3, 65, 4093, 6, 3489, 1, 0, 0, 0, 0, 0, 0, 4093, 2, 2, 88, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7580, 8189, 8193, 8223, 8188, 9214, 7671, 8278] avg=8192.0000 max=9214 max_rank=5 min=7580 min_rank=0 imbalance_ratio=1.124756 max_load_reduction=56322 max_load_reduction_pct=0.859406 moved_to_echo=33351 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[969, 0, 4094, 0, 0, 0, 0, 0, 2488, 514, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1608, 0, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 4096, 0, 3493, 1, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 89, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8067, 8195, 8191, 8187, 8192, 8325, 8122, 8257] avg=8192.0000 max=8325 max_rank=5 min=8067 min_rank=0 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32896 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1821, 0, 4091, 0, 0, 0, 0, 0, 2488, 514, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1608, 0, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 390, 4096, 0, 3493, 1, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 89, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8916, 8195, 8191, 8187, 8192, 7696, 7981, 8178] avg=8192.0000 max=8916 max_rank=0 min=7696 min_rank=5 imbalance_ratio=1.088379 max_load_reduction=56620 max_load_reduction_pct=0.863953 moved_to_echo=32047 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[687, 0, 4084, 0, 3, 0, 0, 9, 2486, 514, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3, 4, 4077, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 3, 1, 2, 4092, 4091, 0, 0, 0, 0, 0, 0, 16, 0, 2, 3, 4088, 4093, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4093, 4088, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 8, 0, 1607, 0, 3083, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 11, 3, 313, 4092, 1, 3489, 1, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 88, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7785, 8177, 8197, 8206, 8193, 8793, 7914, 8271] avg=8192.0000 max=8793 max_rank=5 min=7785 min_rank=0 imbalance_ratio=1.073364 max_load_reduction=56743 max_load_reduction_pct=0.865829 moved_to_echo=33171 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[673, 0, 4061, 0, 0, 0, 0, 0, 2488, 514, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1608, 0, 1873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1668, 4096, 0, 3493, 1, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 89, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7738, 8195, 8191, 8187, 8192, 7577, 9258, 8198] avg=8192.0000 max=9258 max_rank=6 min=7577 min_rank=5 imbalance_ratio=1.130127 max_load_reduction=56278 max_load_reduction_pct=0.858734 moved_to_echo=33225 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1740, 0, 4024, 0, 0, 0, 0, 0, 2488, 514, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1608, 0, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2, 0, 1307, 4096, 0, 3493, 1, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 89, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8768, 8195, 8191, 8180, 8192, 6906, 8924, 8180] avg=8192.0000 max=8924 max_rank=6 min=6906 min_rank=5 imbalance_ratio=1.089355 max_load_reduction=56612 max_load_reduction_pct=0.863831 moved_to_echo=32202 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[64, 0, 4070, 0, 0, 0, 0, 0, 2490, 519, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1606, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 271, 4096, 0, 3495, 1, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 82, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7147, 8195, 8191, 8180, 8195, 9567, 7877, 8184] avg=8192.0000 max=9567 max_rank=5 min=7147 min_rank=0 imbalance_ratio=1.167847 max_load_reduction=55969 max_load_reduction_pct=0.854019 moved_to_echo=33823 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32128, 128, 128, 768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 32768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8960, 128, 32640, 0, 128, 0, 0, 128, 19968, 4224, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 0, 12928, 0, 19968, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 4864, 32768, 128, 28032, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[62, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 226, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3807, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[385, 4091, 7, 0, 5, 3, 0, 3, 4087, 3, 0, 3, 0, 4093, 4093, 44, 4092, 2, 4, 3, 5, 0, 3, 1, 0, 2, 4, 0, 3, 4089, 0, 58, 3, 2, 0, 4093, 3, 0, 0, 4091, 4093, 0, 3, 4087, 0, 3, 0, 1, 4090, 4, 0, 6, 2, 4090, 4093, 3607, 0, 1, 0, 4089, 4092, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[33, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 91, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3973, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[452, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3614, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[246, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 58, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3792, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[391, 4092, 5, 0, 3, 3, 0, 2, 4090, 1, 0, 3, 0, 4093, 4094, 81, 4093, 1, 3, 3, 3, 0, 2, 2, 1, 1, 3, 0, 2, 4089, 0, 7, 1, 2, 0, 4094, 2, 2, 0, 4092, 4093, 0, 2, 4093, 0, 2, 0, 3, 4092, 3, 0, 6, 0, 4091, 4094, 3612, 0, 0, 0, 4091, 4093, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[53, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4027, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[91, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 226, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3780, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[62, 4096, 0, 0, 0, 0, 0, 0, 3777, 91, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4002, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 4096, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8038, 8324, 8190, 8185, 8192, 8191, 8222, 8194] avg=8192.0000 max=8324 max_rank=1 min=8038 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32799 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[385, 4091, 7, 0, 5, 3, 0, 3, 3772, 91, 11, 0, 0, 0, 0, 0, 4087, 3, 0, 3, 0, 3999, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 4, 3, 5, 0, 3, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 3, 4089, 0, 58, 4082, 3, 0, 0, 0, 0, 0, 0, 3, 2, 0, 4093, 3, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 4087, 0, 3, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 318, 4, 0, 6, 2, 4090, 0, 3607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 4092, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8368, 8136, 8200, 8241, 8192, 8188, 8027, 8184] avg=8192.0000 max=8368 max_rank=0 min=8027 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=57168 max_load_reduction_pct=0.872314 moved_to_echo=32591 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[391, 4092, 5, 0, 3, 3, 0, 2, 3773, 91, 11, 0, 0, 0, 0, 0, 4090, 1, 0, 3, 0, 3999, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 3, 3, 3, 0, 2, 2, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 0, 2, 4089, 0, 7, 4083, 3, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4094, 2, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 4093, 0, 2, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 319, 3, 0, 6, 0, 4091, 0, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8371, 8174, 8198, 8189, 8193, 8194, 8031, 8186] avg=8192.0000 max=8371 max_rank=0 min=8031 min_rank=6 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32604 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[33, 4096, 0, 0, 0, 0, 0, 0, 3777, 91, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4002, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 4096, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8008, 8189, 8189, 8184, 8192, 8192, 8388, 8194] avg=8192.0000 max=8388 max_rank=6 min=8008 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32966 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[452, 4096, 0, 0, 0, 0, 0, 0, 3777, 91, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4002, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 4096, 0, 3614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8428, 8126, 8190, 8187, 8192, 8190, 8029, 8194] avg=8192.0000 max=8428 max_rank=0 min=8029 min_rank=6 imbalance_ratio=1.028809 max_load_reduction=57108 max_load_reduction_pct=0.871399 moved_to_echo=32605 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[246, 4096, 0, 0, 0, 0, 0, 0, 3777, 91, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4002, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 4096, 0, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8156, 8189, 8184, 8192, 8193, 8207, 8194] avg=8192.0000 max=8221 max_rank=0 min=8156 min_rank=1 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[53, 4096, 0, 0, 0, 0, 0, 0, 3782, 92, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3997, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 4096, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8035, 8107, 8190, 8184, 8191, 8196, 8437, 8196] avg=8192.0000 max=8437 max_rank=6 min=8035 min_rank=0 imbalance_ratio=1.029907 max_load_reduction=57099 max_load_reduction_pct=0.871262 moved_to_echo=33020 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[91, 4096, 0, 0, 0, 0, 0, 0, 3777, 91, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4002, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 319, 0, 0, 0, 0, 4096, 0, 3780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8067, 8324, 8190, 8182, 8192, 8192, 8195, 8194] avg=8192.0000 max=8324 max_rank=1 min=8067 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 32768, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2560, 128, 0, 128, 128, 32768, 0, 30336, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 0, 32128, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1792, 32768, 128, 0, 128, 128, 0, 128, 30336, 768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 32768, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 32768, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4092, 5, 2, 2, 1, 4, 4094, 9, 1, 0, 4090, 0, 0, 2, 2, 0, 4091, 1, 2, 1, 33, 0, 4093, 0, 1, 0, 0, 2, 0, 0, 4093, 1, 3, 4085, 0, 4092, 0, 0, 3, 3, 4093, 0, 4094, 0, 4093, 0, 4092, 4092, 4093, 4061, 3, 1, 0, 2, 3, 0, 3, 3, 0, 2, 3, 0, 4090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4092, 6, 4, 3, 0, 4, 4094, 5, 4, 0, 4088, 0, 0, 3, 2, 0, 4087, 2, 4, 0, 37, 0, 4092, 0, 2, 0, 0, 2, 0, 1, 4093, 0, 3, 4087, 0, 4091, 0, 0, 4, 4, 4092, 0, 4093, 0, 4092, 0, 4091, 4092, 4092, 4056, 2, 1, 0, 3, 3, 0, 4, 4, 0, 4, 4, 0, 4089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 9, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4089, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 5, 2, 2, 1, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 4090, 0, 0, 2, 2, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 2, 1, 33, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4093, 4084, 5, 0, 0, 0, 0, 0, 0, 1, 3, 4085, 0, 4092, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 4094, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 9, 4061, 3, 1, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 2, 3, 0, 4090, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8196, 8215, 8185, 8185, 8191, 8171, 8193] avg=8192.0000 max=8215 max_rank=2 min=8171 min_rank=6 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32740 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4087, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8191, 8185, 8192, 8193, 8194, 8201, 8191] avg=8192.0000 max=8201 max_rank=6 min=8185 min_rank=2 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 6, 4, 3, 0, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 4088, 0, 0, 3, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 4, 0, 37, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 1, 4093, 4083, 5, 0, 0, 0, 0, 0, 0, 0, 3, 4087, 0, 4091, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 4093, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 9, 4056, 2, 1, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 4, 4, 0, 4089, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8192, 8216, 8186, 8186, 8190, 8166, 8197] avg=8192.0000 max=8216 max_rank=2 min=8166 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=57320 max_load_reduction_pct=0.874634 moved_to_echo=32739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8185, 8189, 8193, 8194, 8201, 8191] avg=8192.0000 max=8201 max_rank=6 min=8185 min_rank=2 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4096, 4087, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8191, 8185, 8191, 8193, 8193, 8201, 8191] avg=8192.0000 max=8201 max_rank=6 min=8185 min_rank=2 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7, 0, 0, 0, 0, 4096, 4087, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8191, 8185, 8205, 8193, 8187, 8201, 8191] avg=8192.0000 max=8205 max_rank=3 min=8183 min_rank=0 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8185, 8189, 8193, 8194, 8201, 8191] avg=8192.0000 max=8201 max_rank=6 min=8185 min_rank=2 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4093, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8193, 8180, 8199, 8200, 8193, 8194, 8191] avg=8192.0000 max=8200 max_rank=4 min=8180 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 32768, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4037, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 7, 0, 4071, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 5, 4, 0, 3, 1, 0, 19, 0, 4, 0, 0, 5, 4092, 4, 4091, 4075, 0, 0, 0, 2, 4088, 1, 4092, 4088, 2, 0, 5, 0, 4091, 4090, 4, 4084, 4093, 2, 4092, 0, 0, 0, 0, 0, 0, 9, 4089, 0, 6, 0, 6, 4090, 5, 0, 2, 4092, 4, 0, 3, 4092, 4, 1, 0, 4093, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 4090, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 4, 3, 0, 2, 1, 0, 15, 0, 3, 0, 0, 5, 4093, 3, 4092, 4087, 0, 0, 0, 0, 4089, 2, 4092, 4091, 1, 0, 4, 0, 4093, 4092, 3, 4086, 4094, 2, 4093, 0, 0, 0, 2, 0, 0, 0, 4087, 0, 1, 2, 2, 4092, 4, 2, 2, 4093, 3, 0, 2, 4093, 5, 0, 0, 4093, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4088, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 0, 4080, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4093, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4037, 0, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8234, 8198, 8192, 8195, 8191, 8191, 8144] avg=8192.0000 max=8234 max_rank=1 min=8144 min_rank=7 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 5, 4, 0, 3, 1, 4093, 4089, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4, 0, 0, 5, 4092, 4070, 0, 0, 0, 0, 0, 0, 0, 4, 5, 4075, 0, 0, 0, 2, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 4088, 2, 0, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 4084, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4089, 0, 6, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 6, 4090, 5, 0, 2, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 4, 1, 0, 4093, 2, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8190, 8174, 8190, 8183, 8199, 8198, 8206] avg=8192.0000 max=8206 max_rank=7 min=8174 min_rank=2 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4, 3, 0, 2, 1, 4094, 4091, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 0, 0, 5, 4093, 4071, 0, 0, 0, 0, 0, 0, 0, 3, 5, 4087, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 4091, 1, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 4086, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4087, 0, 1, 4090, 6, 0, 0, 0, 0, 0, 0, 2, 2, 4092, 4, 2, 2, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 5, 0, 0, 4093, 2, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8187, 8184, 8192, 8186, 8186, 8199, 8206] avg=8192.0000 max=8206 max_rank=7 min=8184 min_rank=2 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32777 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4093, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8180, 8198, 8194, 8195, 8185, 8191, 8203] avg=8192.0000 max=8203 max_rank=7 min=8180 min_rank=1 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4093, 6, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4071, 0, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8193, 8198, 8192, 8195, 8193, 8191, 8185] avg=8192.0000 max=8198 max_rank=2 min=8185 min_rank=7 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4093, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4090, 0, 11, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8176, 8194, 8192, 8195, 8197, 8191, 8199] avg=8192.0000 max=8199 max_rank=7 min=8176 min_rank=1 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4093, 6, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8183, 8198, 8192, 8195, 8192, 8191, 8195] avg=8192.0000 max=8198 max_rank=2 min=8183 min_rank=1 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 4080, 0, 10, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8192, 8192, 8192, 8193, 8184, 8198] avg=8192.0000 max=8198 max_rank=7 min=8184 min_rank=6 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 32768, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 256, 0, 128, 0, 0, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 636, 0, 0, 0, 3997, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3559, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 951, 0, 0, 0, 3931, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3310, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[3, 0, 4092, 2, 3, 4092, 5, 0, 0, 4091, 4, 0, 4089, 10, 0, 566, 4, 1, 2, 3901, 4, 3, 4091, 5, 5, 5, 4090, 0, 0, 0, 3, 12, 0, 0, 4090, 0, 0, 2, 3715, 1, 4, 2, 4092, 0, 4, 4074, 1, 5, 0, 4, 0, 0, 8, 0, 0, 0, 4, 0, 4092, 4091, 4084, 4091, 4088, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 4093, 1, 2, 4092, 4, 0, 0, 4092, 3, 0, 4091, 7, 0, 174, 4, 1, 2, 3990, 3, 3, 4092, 4, 3, 4, 4091, 0, 0, 0, 2, 20, 0, 0, 4092, 0, 0, 2, 4019, 0, 4, 2, 4093, 0, 3, 4073, 0, 3, 0, 3, 0, 0, 5, 0, 0, 0, 3, 0, 4093, 4092, 4086, 4093, 4090, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 529, 0, 0, 0, 3910, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 3753, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1214, 0, 0, 0, 3938, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 3040, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4095, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1039, 0, 0, 0, 3899, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 21, 4096, 8, 0, 0, 3245, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4078, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1998, 0, 0, 0, 3714, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 2, 0, 0, 2480, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4095, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3209, 0, 0, 4096, 0, 0, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 4096, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3559, 0, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7941, 8275, 8187, 8357, 8195, 8191, 8198] avg=8192.0000 max=8357 max_rank=4 min=7941 min_rank=1 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4093, 1, 2, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 3, 0, 4091, 7, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 2, 3990, 3, 3, 4092, 4, 182, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4091, 0, 0, 0, 2, 20, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 2, 4019, 0, 701, 0, 0, 0, 0, 0, 0, 0, 4, 2, 4093, 0, 3, 4073, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5, 0, 0, 0, 4093, 4091, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4086, 4093, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7481, 8281, 8205, 8814, 8181, 8192, 8188] avg=8192.0000 max=8814 max_rank=4 min=7481 min_rank=1 imbalance_ratio=1.075928 max_load_reduction=56722 max_load_reduction_pct=0.865509 moved_to_echo=33375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3209, 0, 0, 4096, 0, 0, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 4096, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3310, 0, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8256, 8209, 8187, 8108, 8195, 8191, 8198] avg=8192.0000 max=8256 max_rank=1 min=8108 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4092, 2, 3, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3205, 4, 0, 4089, 10, 0, 566, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 2, 3901, 4, 3, 4091, 5, 182, 0, 0, 0, 0, 0, 0, 0, 5, 5, 4090, 0, 0, 0, 3, 12, 4082, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 2, 3715, 1, 701, 0, 0, 0, 0, 0, 0, 0, 4, 2, 4092, 0, 4, 4074, 1, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8, 0, 0, 0, 4092, 4090, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4084, 4091, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 7874, 8193, 8198, 8509, 8185, 8194, 8186] avg=8192.0000 max=8509 max_rank=4 min=7874 min_rank=1 imbalance_ratio=1.038696 max_load_reduction=57027 max_load_reduction_pct=0.870163 moved_to_echo=33074 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3209, 0, 0, 4096, 0, 0, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 0, 4096, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 3040, 0, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8519, 8216, 8187, 7839, 8195, 8191, 8197] avg=8192.0000 max=8519 max_rank=1 min=7839 min_rank=4 imbalance_ratio=1.039917 max_load_reduction=57017 max_load_reduction_pct=0.870010 moved_to_echo=32422 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3209, 0, 0, 4096, 0, 0, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 0, 0, 4096, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 3753, 0, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7834, 8188, 8187, 8552, 8194, 8191, 8198] avg=8192.0000 max=8552 max_rank=4 min=7834 min_rank=1 imbalance_ratio=1.043945 max_load_reduction=56984 max_load_reduction_pct=0.869507 moved_to_echo=33135 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3209, 0, 0, 4096, 0, 0, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 4096, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 8, 0, 0, 3245, 0, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8344, 8177, 8187, 8072, 8189, 8195, 8180] avg=8192.0000 max=8344 max_rank=1 min=8072 min_rank=4 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32636 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3193, 0, 0, 4096, 0, 0, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3714, 0, 0, 4096, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 5, 0, 0, 0, 0, 0, 0, 1, 4096, 2, 0, 0, 2480, 0, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 9287, 7997, 8198, 7285, 8202, 8191, 8191] avg=8192.0000 max=9287 max_rank=1 min=7285 min_rank=4 imbalance_ratio=1.133667 max_load_reduction=56249 max_load_reduction_pct=0.858292 moved_to_echo=31869 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 0, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 27136, 128, 5632, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 25728, 128, 0, 32768, 128, 0, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 31360, 128, 128, 32768, 128, 1536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 235, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 269, 0, 0, 4096, 0, 4096, 0, 0, 202, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2429, 0, 961, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 117, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 4096, 0, 0, 26, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3927, 0, 14, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 4091, 3, 4086, 53, 4091, 0, 3927, 0, 3, 0, 869, 4, 4089, 0, 0, 4091, 3887, 16, 0, 0, 0, 4092, 2, 0, 4, 4076, 0, 0, 4, 6, 4, 41, 5, 3, 4093, 12, 4082, 2, 4, 389, 4085, 3, 4, 6, 0, 0, 4092, 0, 0, 4090, 2, 4, 2, 7, 2752, 5, 415, 0, 0, 0, 4038, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 4092, 2, 4087, 44, 4092, 0, 3988, 0, 3, 0, 1014, 2, 4089, 0, 0, 4092, 3918, 16, 0, 0, 0, 4093, 1, 0, 3, 4078, 0, 0, 3, 10, 3, 18, 3, 2, 4093, 8, 4082, 2, 3, 19, 4088, 2, 3, 4, 0, 0, 4093, 1, 0, 4092, 1, 3, 3, 5, 2740, 4, 594, 1, 0, 0, 4041, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 171, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 161, 0, 0, 4096, 0, 4096, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3604, 0, 149, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 12, 0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 77, 0, 0, 4096, 0, 4096, 0, 0, 85, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3297, 0, 627, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 36, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 58, 0, 0, 4096, 0, 4096, 0, 0, 63, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3877, 0, 59, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 42, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 304, 0, 0, 4096, 0, 4096, 0, 0, 366, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 3340, 0, 43, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 26, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 4096, 0, 0, 3785, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3978, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 0, 0, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3927, 584, 144, 118, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 3465, 285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 7998, 8239, 8192, 8086, 8074, 8869, 7860] avg=8192.0000 max=8869 max_rank=6 min=7860 min_rank=7 imbalance_ratio=1.082642 max_load_reduction=56667 max_load_reduction_pct=0.864670 moved_to_echo=32889 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 26, 2, 4087, 44, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1014, 2, 4089, 0, 0, 3781, 0, 0, 0, 0, 0, 0, 0, 4092, 3918, 16, 0, 0, 0, 46, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4078, 0, 0, 3, 10, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 18, 3, 2, 3975, 8, 4082, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4088, 2, 3, 4, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 1, 3, 3, 5, 2740, 584, 144, 118, 0, 0, 0, 0, 0, 4, 594, 1, 0, 0, 4041, 1, 0, 3463, 285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8889, 8073, 8189, 8093, 8065, 7691, 8389] avg=8192.0000 max=8889 max_rank=1 min=7691 min_rank=6 imbalance_ratio=1.085083 max_load_reduction=56647 max_load_reduction_pct=0.864365 moved_to_echo=32238 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 26, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 4096, 0, 0, 3785, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 269, 0, 0, 3978, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 4096, 0, 0, 0, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2429, 584, 144, 118, 0, 0, 0, 0, 0, 0, 961, 0, 0, 0, 4096, 0, 0, 3465, 285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8116, 8239, 8192, 8343, 8250, 7371, 8807] avg=8192.0000 max=8807 max_rank=7 min=7371 min_rank=6 imbalance_ratio=1.075073 max_load_reduction=56729 max_load_reduction_pct=0.865616 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 26, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 0, 0, 3785, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 3978, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 4096, 0, 0, 0, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3297, 584, 144, 118, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 4096, 0, 0, 3465, 285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 7893, 8237, 8192, 8151, 8133, 8239, 8473] avg=8192.0000 max=8473 max_rank=7 min=7893 min_rank=1 imbalance_ratio=1.034302 max_load_reduction=57063 max_load_reduction_pct=0.870712 moved_to_echo=32996 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 26, 3, 4086, 53, 0, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 869, 4, 4089, 0, 0, 3780, 0, 0, 0, 0, 0, 0, 0, 4091, 3887, 16, 0, 0, 0, 46, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4076, 0, 0, 4, 6, 4, 4091, 0, 0, 0, 0, 0, 0, 0, 41, 5, 3, 3975, 12, 4082, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 389, 4085, 3, 4, 6, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 4, 2, 7, 2752, 584, 144, 118, 0, 0, 0, 0, 0, 5, 415, 0, 0, 0, 4038, 2, 0, 3462, 285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8745, 8042, 8185, 8124, 8435, 7703, 8207] avg=8192.0000 max=8745 max_rank=1 min=7703 min_rank=6 imbalance_ratio=1.067505 max_load_reduction=56791 max_load_reduction_pct=0.866562 moved_to_echo=32469 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 26, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 4096, 0, 0, 3785, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 3978, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3604, 584, 144, 118, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 4096, 0, 0, 3465, 285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8052, 8239, 8192, 8235, 8059, 8546, 7995] avg=8192.0000 max=8546 max_rank=6 min=7995 min_rank=7 imbalance_ratio=1.043213 max_load_reduction=56990 max_load_reduction_pct=0.869598 moved_to_echo=32835 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 26, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4096, 0, 0, 3785, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 3978, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 4096, 0, 0, 0, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3877, 584, 144, 118, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 4096, 0, 0, 3465, 285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 7917, 8239, 8195, 8132, 8111, 8819, 7905] avg=8192.0000 max=8819 max_rank=6 min=7905 min_rank=7 imbalance_ratio=1.076538 max_load_reduction=56717 max_load_reduction_pct=0.865433 moved_to_echo=32967 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 4096, 0, 0, 3788, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 304, 0, 0, 3972, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 4096, 0, 0, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3340, 588, 149, 124, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 4096, 0, 0, 3472, 289, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 7926, 8228, 8199, 8372, 8409, 8298, 7900] avg=8192.0000 max=8409 max_rank=5 min=7900 min_rank=7 imbalance_ratio=1.026489 max_load_reduction=57127 max_load_reduction_pct=0.871689 moved_to_echo=32979 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 256, 128, 32768, 128, 0, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1280, 32768, 128, 128, 128, 0, 0, 31616, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 0, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 32384, 128, 0, 0, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1024, 128, 128, 31872, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 2944, 128, 0, 0, 32768, 128, 0, 27776, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 25984, 4736, 1280, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 2560, 128, 32768, 0, 0, 30336, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4080, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4081, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4010, 0, 3, 0, 0, 0, 4079, 4090, 3, 4, 3, 3, 1, 2, 0, 11, 4088, 0, 0, 2, 0, 80, 4091, 0, 1, 0, 4093, 4, 4092, 0, 4092, 0, 4093, 5, 0, 4093, 1, 3, 10, 0, 2, 22, 1, 0, 4092, 4, 0, 0, 3, 4093, 4092, 4078, 3, 0, 4084, 1, 3, 1, 4, 2, 1, 4089, 1, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[3966, 0, 8, 0, 0, 0, 4083, 4088, 4, 5, 5, 4, 2, 3, 0, 18, 4075, 0, 0, 3, 0, 119, 4087, 0, 2, 0, 4092, 5, 4090, 0, 4091, 0, 4092, 7, 0, 4092, 0, 4, 8, 0, 2, 27, 1, 0, 4091, 6, 0, 0, 4, 4092, 4091, 4078, 3, 0, 4081, 0, 4, 0, 6, 4, 2, 4087, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4089, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 1, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4080, 0, 0, 0, 0, 0, 4096, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4071, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4060, 25, 3, 1, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8185, 8183, 8192, 8189, 8185, 8199, 8191] avg=8192.0000 max=8212 max_rank=0 min=8183 min_rank=2 imbalance_ratio=1.002441 max_load_reduction=57324 max_load_reduction_pct=0.874695 moved_to_echo=32764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4060, 25, 3, 1, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8185, 8167, 8194, 8189, 8185, 8199, 8189] avg=8192.0000 max=8228 max_rank=0 min=8167 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4010, 0, 3, 0, 0, 0, 4079, 35, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 3, 3, 1, 2, 0, 11, 4093, 4086, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 2, 0, 80, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 4, 4092, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 5, 0, 4093, 1, 3, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 22, 1, 0, 4092, 4, 0, 0, 4055, 25, 3, 1, 0, 0, 0, 0, 3, 0, 7, 4078, 3, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4, 2, 1, 4089, 1, 3, 4084, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8206, 8236, 8191, 8202, 8205, 8176, 8193] avg=8192.0000 max=8236 max_rank=2 min=8127 min_rank=0 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3966, 0, 8, 0, 0, 0, 4083, 35, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 5, 4, 2, 3, 0, 18, 4092, 4085, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 3, 0, 119, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 5, 4090, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 7, 0, 4092, 0, 4, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 27, 1, 0, 4091, 6, 0, 0, 4053, 25, 3, 1, 0, 0, 0, 0, 4, 0, 7, 4078, 3, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 4, 2, 4087, 0, 4, 4083, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8092, 8218, 8259, 8190, 8200, 8209, 8173, 8195] avg=8192.0000 max=8259 max_rank=2 min=8092 min_rank=0 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32777 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4081, 0, 0, 0, 0, 0, 4096, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4071, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4060, 25, 3, 1, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8185, 8181, 8192, 8190, 8185, 8199, 8191] avg=8192.0000 max=8213 max_rank=0 min=8181 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 0, 4096, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4071, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4060, 25, 3, 1, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4088, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8185, 8168, 8193, 8189, 8185, 8199, 8192] avg=8192.0000 max=8225 max_rank=0 min=8168 min_rank=2 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 0, 0, 0, 0, 4096, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4071, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4060, 25, 3, 1, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8185, 8174, 8192, 8189, 8185, 8199, 8191] avg=8192.0000 max=8221 max_rank=0 min=8174 min_rank=2 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4069, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4066, 27, 4, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4092, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8187, 8168, 8192, 8188, 8197, 8192, 8194] avg=8192.0000 max=8218 max_rank=0 min=8168 min_rank=2 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 0, 256, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 0, 0, 32512, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32512, 0, 128, 0, 0, 0, 32768, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 2, 1, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4, 3, 2, 0, 1, 17, 0, 0, 14, 2, 6, 0, 4, 3, 0, 4090, 4092, 0, 4093, 4090, 2, 4092, 0, 3, 4093, 3, 4093, 0, 0, 3805, 4076, 4092, 0, 3, 0, 289, 0, 4093, 0, 4081, 3, 0, 3, 0, 4086, 1, 1, 1, 4, 1, 0, 0, 0, 4, 4092, 0, 0, 4091, 3, 4092, 0, 0, 5, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[5, 3, 3, 0, 2, 17, 0, 0, 27, 2, 4, 0, 5, 4, 0, 4090, 4090, 0, 4092, 4089, 3, 4091, 0, 5, 4092, 4, 4091, 0, 0, 3706, 4066, 4092, 0, 3, 0, 387, 0, 4092, 0, 4079, 5, 0, 4, 0, 4084, 3, 2, 1, 5, 0, 0, 0, 0, 5, 4091, 0, 0, 4088, 4, 4091, 0, 0, 6, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 11, 4095, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4005, 82, 1, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4014, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8184, 8193, 8283, 8113, 8193, 8191, 8192] avg=8192.0000 max=8283 max_rank=3 min=8113 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32689 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4, 3, 2, 0, 1, 17, 0, 0, 4093, 4087, 0, 0, 0, 0, 0, 0, 14, 2, 6, 0, 4, 3, 0, 4090, 4002, 82, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 4090, 2, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 91, 0, 0, 3805, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 289, 0, 4011, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 4086, 1, 1, 1, 4087, 6, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 4, 4092, 0, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 4092, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8204, 8191, 7975, 8384, 8188, 8193, 8194] avg=8192.0000 max=8384 max_rank=4 min=7975 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=32959 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 3, 3, 0, 2, 17, 0, 0, 4092, 4086, 0, 0, 0, 0, 0, 0, 27, 2, 4, 0, 5, 4, 0, 4090, 4000, 82, 1, 0, 0, 0, 0, 0, 4090, 0, 0, 4089, 3, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 91, 0, 0, 3706, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 387, 0, 4010, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 0, 4084, 3, 2, 1, 4087, 6, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 5, 4091, 0, 4086, 5, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 4091, 0, 0, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8215, 8191, 7867, 8479, 8192, 8192, 8192] avg=8192.0000 max=8479 max_rank=4 min=7867 min_rank=3 imbalance_ratio=1.035034 max_load_reduction=57057 max_load_reduction_pct=0.870621 moved_to_echo=33055 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4005, 82, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4014, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8184, 8194, 8283, 8112, 8193, 8192, 8192] avg=8192.0000 max=8283 max_rank=3 min=8112 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32689 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4005, 82, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4014, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8184, 8195, 8283, 8111, 8192, 8192, 8192] avg=8192.0000 max=8283 max_rank=3 min=8111 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32687 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4005, 82, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8184, 8195, 8283, 8110, 8193, 8192, 8192] avg=8192.0000 max=8283 max_rank=3 min=8110 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32687 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4005, 82, 1, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4013, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8184, 8187, 8283, 8118, 8192, 8192, 8193] avg=8192.0000 max=8283 max_rank=3 min=8118 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32695 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4009, 84, 4, 4, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4012, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8197, 8190, 8279, 8109, 8193, 8192, 8189] avg=8192.0000 max=8279 max_rank=3 min=8109 min_rank=4 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32683 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 768, 128, 32128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 768, 0, 0, 32128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 32768, 32128, 768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True + [2026-08-30 05:00:52] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 706.8 | throughput per GPU (TFLOP/s/GPU): 471.7 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 1.003656E+01 | loss scale: 1.0 | grad norm: 6.969 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[585, 3, 20, 28, 47, 0, 2993, 4, 438, 2162, 17, 135, 547, 2188, 0, 651, 483, 742, 39, 2771, 0, 680, 373, 2266, 0, 2, 890, 1922, 3382, 21, 2975, 629, 745, 220, 2329, 255, 374, 29, 2994, 1260, 3721, 0, 3591, 2379, 0, 140, 1628, 1266, 3577, 700, 3206, 0, 0, 145, 0, 612, 1, 59, 260, 2007, 3590, 1012, 1258, 1185] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[954, 2, 49, 21, 48, 1, 3244, 0, 454, 2230, 22, 149, 803, 2628, 0, 417, 729, 295, 54, 2324, 0, 825, 513, 2331, 0, 3, 931, 1814, 3335, 35, 2798, 561, 732, 205, 1967, 182, 350, 12, 3274, 1003, 3775, 0, 3626, 2575, 0, 279, 1432, 1383, 3263, 410, 3081, 0, 0, 77, 0, 397, 1, 73, 446, 2129, 3504, 713, 1747, 1330] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[465, 4, 26, 10, 284, 0, 3066, 2, 658, 1137, 28, 122, 976, 1058, 0, 706, 1129, 316, 50, 2389, 0, 711, 580, 2178, 14, 5, 1381, 1540, 3122, 10, 2397, 729, 1965, 156, 2290, 283, 388, 29, 3121, 1013, 3616, 0, 3395, 1971, 0, 330, 2222, 1217, 3217, 249, 2855, 1, 1, 8, 1, 533, 0, 125, 932, 1978, 3370, 1237, 1845, 2095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[512, 0, 31, 3, 47, 3, 3142, 5, 819, 2034, 38, 259, 843, 1747, 0, 702, 1017, 684, 37, 2207, 0, 860, 453, 2276, 1, 4, 1228, 1520, 3252, 16, 2720, 427, 696, 215, 2204, 203, 386, 18, 3051, 1022, 3769, 0, 3601, 1980, 0, 209, 1810, 735, 3554, 453, 3179, 5, 0, 28, 0, 397, 0, 129, 525, 2226, 3715, 1286, 1382, 1871] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[681, 0, 72, 29, 107, 0, 2379, 7, 510, 1646, 72, 198, 801, 2848, 0, 607, 1331, 492, 31, 2611, 0, 984, 238, 2022, 11, 11, 674, 1831, 3309, 10, 2957, 600, 1222, 128, 2343, 232, 813, 39, 3231, 651, 3684, 0, 3121, 2484, 0, 150, 1912, 1033, 3607, 306, 2750, 0, 3, 17, 0, 407, 0, 75, 431, 1573, 3541, 1807, 1593, 1314] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[461, 0, 69, 3, 50, 0, 3131, 0, 380, 1867, 21, 209, 814, 1904, 0, 410, 787, 480, 61, 2677, 0, 691, 328, 2184, 0, 4, 1319, 1679, 3577, 10, 2934, 755, 1351, 243, 1975, 256, 868, 14, 2987, 1021, 3732, 0, 3445, 2725, 0, 196, 2199, 913, 3335, 387, 2541, 1, 0, 9, 4, 707, 4, 60, 346, 1605, 3597, 1162, 1764, 1284] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[593, 4, 162, 10, 188, 0, 2788, 0, 433, 1642, 12, 213, 1058, 1767, 0, 665, 1238, 627, 58, 2196, 0, 688, 469, 2397, 2, 1, 763, 1852, 3369, 10, 2653, 745, 1419, 361, 2438, 255, 520, 17, 3243, 655, 3708, 0, 3146, 1861, 0, 223, 1759, 1419, 3490, 121, 2190, 1, 0, 13, 58, 560, 1, 149, 913, 1732, 3747, 1388, 1868, 1678] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[642, 7, 25, 11, 67, 0, 3035, 0, 265, 1835, 5, 212, 716, 2554, 0, 689, 1050, 411, 37, 1999, 0, 849, 316, 2231, 2, 3, 1209, 1738, 3252, 51, 2520, 663, 786, 143, 2124, 206, 410, 24, 3304, 1232, 3680, 0, 3636, 2153, 0, 316, 1408, 959, 3196, 445, 3028, 3, 1, 48, 1, 736, 0, 63, 487, 1896, 3592, 1613, 1806, 1846] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[512, 0, 31, 3, 47, 3, 3142, 5, 3769, 742, 0, 0, 0, 0, 0, 0, 819, 2034, 38, 259, 843, 1747, 0, 702, 1775, 429, 0, 0, 0, 0, 0, 0, 1017, 684, 37, 2207, 0, 860, 453, 2276, 626, 235, 0, 0, 0, 0, 0, 0, 1, 4, 1228, 1520, 1952, 16, 2720, 427, 0, 0, 0, 0, 0, 0, 0, 0, 696, 215, 2204, 203, 386, 18, 2816, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2911, 1980, 0, 209, 1810, 735, 0, 0, 0, 0, 0, 0, 0, 0, 3554, 453, 3179, 5, 0, 28, 0, 397, 871, 64, 0, 0, 0, 0, 0, 0, 0, 129, 525, 2226, 1198, 1286, 1382, 1871, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8646, 8395, 7868, 7560, 7645, 8551, 8617] avg=8192.0000 max=8646 max_rank=1 min=7560 min_rank=4 imbalance_ratio=1.055420 max_load_reduction=56890 max_load_reduction_pct=0.868073 moved_to_echo=32373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[954, 2, 49, 21, 48, 1, 3244, 0, 3775, 700, 0, 0, 0, 0, 0, 0, 454, 2230, 22, 149, 803, 2628, 0, 417, 1675, 440, 0, 0, 0, 0, 0, 0, 729, 295, 54, 2324, 0, 825, 513, 2331, 630, 252, 0, 0, 0, 0, 0, 0, 0, 3, 931, 1814, 2002, 35, 2798, 561, 0, 0, 0, 0, 0, 0, 0, 0, 732, 205, 1967, 182, 350, 12, 3022, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2931, 2575, 0, 279, 1432, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 3263, 410, 3081, 0, 0, 77, 0, 397, 893, 65, 0, 0, 0, 0, 0, 0, 1, 73, 446, 2129, 1129, 713, 1747, 1330, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8794, 8818, 7953, 8144, 7473, 8600, 8186, 7568] avg=8192.0000 max=8818 max_rank=1 min=7473 min_rank=4 imbalance_ratio=1.076416 max_load_reduction=56718 max_load_reduction_pct=0.865448 moved_to_echo=31827 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[461, 0, 69, 3, 50, 0, 3131, 0, 3732, 719, 0, 0, 0, 0, 0, 0, 380, 1867, 21, 209, 814, 1904, 0, 410, 1719, 472, 0, 0, 0, 0, 0, 0, 787, 480, 61, 2677, 0, 691, 328, 2184, 599, 230, 0, 0, 0, 0, 0, 0, 0, 4, 1319, 1679, 2147, 10, 2934, 755, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 243, 1975, 256, 868, 14, 2757, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2785, 2725, 0, 196, 2199, 913, 0, 0, 0, 0, 0, 0, 0, 0, 3335, 387, 2541, 1, 0, 9, 4, 707, 958, 61, 0, 0, 0, 0, 0, 0, 4, 60, 346, 1605, 1159, 1162, 1764, 1284, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 7796, 8037, 8848, 8485, 8818, 8003, 7384] avg=8192.0000 max=8848 max_rank=3 min=7384 min_rank=7 imbalance_ratio=1.080078 max_load_reduction=56688 max_load_reduction_pct=0.864990 moved_to_echo=32690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[585, 3, 20, 28, 47, 0, 2993, 4, 3721, 717, 0, 0, 0, 0, 0, 0, 438, 2162, 17, 135, 547, 2188, 0, 651, 1716, 447, 0, 0, 0, 0, 0, 0, 483, 742, 39, 2771, 0, 680, 373, 2266, 624, 231, 0, 0, 0, 0, 0, 0, 0, 2, 890, 1922, 2029, 21, 2975, 629, 0, 0, 0, 0, 0, 0, 0, 0, 745, 220, 2329, 255, 374, 29, 2763, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2903, 2379, 0, 140, 1628, 1266, 0, 0, 0, 0, 0, 0, 0, 0, 3577, 700, 3206, 0, 0, 145, 0, 612, 906, 64, 0, 0, 0, 0, 0, 0, 1, 59, 260, 2007, 1157, 1012, 1258, 1185, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8301, 8209, 8468, 7975, 8316, 9210, 6939] avg=8192.0000 max=9210 max_rank=6 min=6939 min_rank=7 imbalance_ratio=1.124268 max_load_reduction=56326 max_load_reduction_pct=0.859467 moved_to_echo=32440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[593, 4, 162, 10, 188, 0, 2788, 0, 3708, 749, 0, 0, 0, 0, 0, 0, 433, 1642, 12, 213, 1058, 1767, 0, 665, 1791, 445, 0, 0, 0, 0, 0, 0, 1238, 627, 58, 2196, 0, 688, 469, 2397, 547, 250, 0, 0, 0, 0, 0, 0, 2, 1, 763, 1852, 2022, 10, 2653, 745, 0, 0, 0, 0, 0, 0, 0, 0, 1419, 361, 2438, 255, 520, 17, 2993, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2543, 1861, 0, 223, 1759, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 3490, 121, 2190, 1, 0, 13, 58, 560, 902, 56, 0, 0, 0, 0, 0, 0, 1, 149, 913, 1732, 1207, 1388, 1868, 1678, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8026, 8470, 8048, 8658, 7805, 7391, 8936] avg=8192.0000 max=8936 max_rank=7 min=7391 min_rank=6 imbalance_ratio=1.090820 max_load_reduction=56600 max_load_reduction_pct=0.863647 moved_to_echo=32790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[465, 4, 26, 10, 284, 0, 3066, 2, 3616, 673, 0, 0, 0, 0, 0, 0, 658, 1137, 28, 122, 976, 1058, 0, 706, 1611, 412, 0, 0, 0, 0, 0, 0, 1129, 316, 50, 2389, 0, 711, 580, 2178, 590, 240, 0, 0, 0, 0, 0, 0, 14, 5, 1381, 1540, 1874, 10, 2397, 729, 0, 0, 0, 0, 0, 0, 0, 0, 1965, 156, 2290, 283, 388, 29, 2881, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2744, 1971, 0, 330, 2222, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 249, 2855, 1, 1, 8, 1, 533, 836, 61, 0, 0, 0, 0, 0, 0, 0, 125, 932, 1978, 1086, 1237, 1845, 2095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 6708, 8183, 7950, 9005, 8484, 7762, 9298] avg=8192.0000 max=9298 max_rank=7 min=6708 min_rank=1 imbalance_ratio=1.135010 max_load_reduction=56238 max_load_reduction_pct=0.858124 moved_to_echo=34549 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[681, 0, 72, 29, 107, 0, 2379, 7, 3684, 708, 0, 0, 0, 0, 0, 0, 510, 1646, 72, 198, 801, 2848, 0, 607, 1692, 437, 0, 0, 0, 0, 0, 0, 1331, 492, 31, 2611, 0, 984, 238, 2022, 542, 249, 0, 0, 0, 0, 0, 0, 11, 11, 674, 1831, 1986, 10, 2957, 600, 0, 0, 0, 0, 0, 0, 0, 0, 1222, 128, 2343, 232, 813, 39, 2982, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2523, 2484, 0, 150, 1912, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 3607, 306, 2750, 0, 3, 17, 0, 407, 886, 56, 0, 0, 0, 0, 0, 0, 0, 75, 431, 1573, 1141, 1807, 1593, 1314, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7667, 8811, 8500, 8080, 8410, 8102, 8032, 7934] avg=8192.0000 max=8811 max_rank=1 min=7667 min_rank=0 imbalance_ratio=1.075562 max_load_reduction=56725 max_load_reduction_pct=0.865555 moved_to_echo=32478 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[642, 7, 25, 11, 67, 0, 3035, 0, 3680, 723, 0, 0, 0, 0, 0, 0, 265, 1835, 5, 212, 716, 2554, 0, 689, 1720, 434, 0, 0, 0, 0, 0, 0, 1050, 411, 37, 1999, 0, 849, 316, 2231, 637, 259, 0, 0, 0, 0, 0, 0, 2, 3, 1209, 1738, 1944, 51, 2520, 663, 0, 0, 0, 0, 0, 0, 0, 0, 786, 143, 2124, 206, 410, 24, 3045, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2930, 2153, 0, 316, 1408, 959, 0, 0, 0, 0, 0, 0, 0, 0, 3196, 445, 3028, 3, 1, 48, 1, 736, 874, 69, 0, 0, 0, 0, 0, 0, 0, 63, 487, 1896, 1149, 1613, 1806, 1846, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8430, 7789, 8130, 7970, 7766, 8401, 8860] avg=8192.0000 max=8860 max_rank=7 min=7766 min_rank=5 imbalance_ratio=1.081543 max_load_reduction=56676 max_load_reduction_pct=0.864807 moved_to_echo=32997 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 8448, 13952, 16000, 256, 22016, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3968, 14592, 256, 1536, 6656, 16768, 0, 4864, 13824, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 22272, 18176, 0, 1920, 14464, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[27264, 3072, 22912, 128, 128, 384, 128, 4352, 7168, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 768, 4352, 15232, 9344, 10240, 13312, 12672, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8960, 1792, 17792, 1920, 4224, 256, 23296, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4992, 128, 512, 128, 896, 128, 23808, 128, 29696, 5760, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7808, 4096, 384, 19200, 0, 6400, 3328, 17920, 4864, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[3, 1, 0, 0, 4093, 2, 0, 1, 0, 4096, 0, 3611, 1, 0, 0, 63, 1, 0, 0, 1, 1, 0, 2, 3, 4, 1, 546, 4093, 4094, 0, 0, 4094, 0, 3, 4, 0, 4088, 2356, 39, 1870, 4094, 0, 0, 0, 1, 1, 3, 4096, 0, 0, 0, 0, 4093, 0, 4096, 1, 4094, 4092, 3825, 2, 1, 1, 0, 4065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 0, 1, 4094, 0, 3946, 3, 0, 1, 100, 24, 0, 0, 0, 0, 0, 2, 7, 2, 0, 282, 4096, 4093, 0, 0, 4095, 0, 0, 28, 1, 4093, 2029, 0, 2078, 4096, 0, 0, 0, 0, 1, 0, 4095, 4, 0, 1, 0, 4095, 0, 4096, 0, 4092, 4095, 3804, 0, 0, 1, 0, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 8, 0, 0, 4096, 0, 3758, 0, 0, 0, 126, 5, 0, 0, 1, 0, 0, 0, 3, 2, 0, 106, 4089, 4096, 0, 0, 4096, 0, 0, 2, 0, 4096, 2671, 0, 1834, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 4095, 4090, 3832, 0, 0, 0, 0, 4054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 5, 0, 1, 0, 4096, 0, 3976, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 188, 4096, 4096, 0, 0, 4095, 0, 0, 9, 0, 4085, 1733, 0, 2173, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 4096, 0, 4096, 4096, 4080, 1, 0, 0, 0, 4075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 3705, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 36, 4079, 4096, 0, 0, 4093, 0, 0, 3, 0, 4093, 2663, 0, 1634, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 4096, 4065, 3, 0, 0, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 3, 3, 0, 1, 4096, 0, 4070, 1, 0, 0, 159, 4, 0, 0, 0, 0, 0, 0, 12, 0, 0, 35, 4096, 4096, 7, 0, 4092, 0, 0, 6, 0, 4096, 3001, 1, 959, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4091, 0, 4096, 4095, 4035, 4, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 3751, 0, 0, 0, 12, 2, 0, 0, 0, 0, 0, 0, 11, 5, 183, 38, 4096, 4096, 0, 0, 4095, 0, 0, 1, 0, 4095, 2041, 7, 2242, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 4096, 4096, 4009, 1, 0, 0, 0, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 4040, 0, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 31, 4094, 4096, 0, 0, 4096, 0, 0, 1, 0, 4094, 2440, 2, 1525, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4088, 0, 4096, 4096, 4048, 0, 0, 0, 0, 4079] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 8, 0, 3941, 138, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 3758, 0, 0, 0, 126, 67, 40, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 0, 3, 4096, 4086, 0, 0, 0, 0, 0, 0, 2, 0, 106, 4089, 0, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4027, 2671, 0, 1834, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 149, 3832, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8087, 8191, 8115, 8534, 8192, 8190, 8035] avg=8192.0000 max=8534 max_rank=4 min=8035 min_rank=7 imbalance_ratio=1.041748 max_load_reduction=57002 max_load_reduction_pct=0.869781 moved_to_echo=32951 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 3, 3, 0, 3946, 137, 9, 0, 0, 0, 0, 0, 1, 4096, 0, 4070, 1, 0, 0, 159, 67, 40, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 12, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4096, 0, 7, 0, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4027, 3001, 1, 959, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 149, 4035, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8434, 8199, 8053, 7994, 8189, 8189, 8284] avg=8192.0000 max=8434 max_rank=1 min=7994 min_rank=4 imbalance_ratio=1.029541 max_load_reduction=57102 max_load_reduction_pct=0.871307 moved_to_echo=32656 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 0, 3946, 138, 8, 0, 0, 0, 0, 0, 1, 4094, 0, 3946, 3, 0, 1, 100, 67, 40, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 2, 7, 4093, 4084, 0, 0, 0, 0, 0, 0, 2, 0, 282, 4096, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1, 4024, 2029, 0, 2078, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 4095, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 149, 3804, 0, 0, 1, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8252, 8210, 8297, 8160, 8192, 8198, 8039] avg=8192.0000 max=8297 max_rank=3 min=8039 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32589 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 0, 0, 4093, 2, 0, 1, 3943, 137, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 3611, 1, 0, 0, 63, 67, 40, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 2, 3, 4094, 4086, 0, 0, 0, 0, 0, 0, 4, 1, 546, 4093, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 4019, 2356, 39, 1870, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 1, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 149, 3825, 2, 1, 1, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 7878, 8188, 8561, 8291, 8195, 8192, 8043] avg=8192.0000 max=8561 max_rank=3 min=7878 min_rank=1 imbalance_ratio=1.045044 max_load_reduction=56975 max_load_reduction_pct=0.869370 moved_to_echo=32721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 3947, 137, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 3705, 0, 0, 0, 198, 67, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 4, 36, 4079, 0, 0, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4024, 2663, 0, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 149, 4065, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8106, 8185, 8035, 8324, 8192, 8193, 8310] avg=8192.0000 max=8324 max_rank=4 min=8035 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=33019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 5, 0, 1, 3947, 138, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 3976, 0, 0, 0, 153, 67, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 188, 4096, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4016, 1733, 0, 2173, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 149, 4080, 1, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8332, 8184, 8201, 7931, 8192, 8195, 8305] avg=8192.0000 max=8332 max_rank=1 min=7931 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32623 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 3947, 138, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 3751, 0, 0, 0, 12, 67, 40, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 11, 4096, 4087, 0, 0, 0, 0, 0, 0, 5, 183, 38, 4096, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4026, 2041, 7, 2242, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 149, 4009, 1, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7966, 8196, 8239, 8317, 8192, 8190, 8245] avg=8192.0000 max=8317 max_rank=4 min=7966 min_rank=1 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32944 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 0, 0, 3948, 141, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 4040, 0, 0, 0, 226, 72, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 2, 31, 4094, 0, 0, 0, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4017, 2440, 2, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 148, 4048, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8481, 8183, 8035, 7985, 8192, 8189, 8275] avg=8192.0000 max=8481 max_rank=1 min=7985 min_rank=4 imbalance_ratio=1.035278 max_load_reduction=57055 max_load_reduction_pct=0.870590 moved_to_echo=32641 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32256, 18944, 128, 14336, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 0, 30976, 128, 0, 128, 1152, 640, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 256, 1280, 32768, 0, 128, 0, 31360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 128, 128, 31616, 1152, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 1280, 31744, 128, 128, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 122, 0, 0, 4074, 3444, 0, 4095, 1, 4096, 3906, 0, 0, 0, 0, 0, 1036, 0, 0, 4096, 0, 4096, 0, 0, 1, 4093, 3, 0, 33, 0, 3986, 0, 4096, 0, 0, 0, 6, 0, 4096, 0, 0, 3883, 4085, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 101, 0, 0, 4090, 2898, 0, 4096, 0, 4096, 4069, 0, 0, 0, 0, 0, 1615, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 6, 0, 4036, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 3715, 4044, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 3, 4095, 0, 1, 3, 0, 0, 4096, 0, 0, 4096, 3, 0, 78, 0, 0, 4092, 3317, 0, 4085, 3, 4092, 4059, 0, 0, 0, 0, 0, 850, 0, 3, 4095, 0, 4094, 0, 0, 1, 4092, 10, 0, 9, 0, 4091, 0, 4093, 1, 1, 0, 7, 0, 4094, 0, 1, 3991, 4079, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 90, 0, 0, 4067, 3441, 0, 4094, 0, 4094, 4093, 0, 0, 0, 0, 0, 1103, 1, 0, 4096, 1, 4096, 0, 0, 0, 4096, 0, 0, 24, 0, 4086, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 3583, 4089, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 699, 0, 0, 4060, 2933, 0, 4096, 0, 4095, 4096, 1, 0, 0, 0, 0, 959, 0, 0, 4095, 0, 4096, 0, 0, 0, 4095, 18, 0, 19, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3629, 4071, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 4095, 1, 0, 514, 0, 0, 4080, 3762, 0, 4093, 1, 4095, 4089, 1, 0, 0, 1, 0, 269, 1, 0, 4095, 1, 4096, 0, 0, 0, 4093, 2, 0, 13, 0, 4093, 0, 4096, 1, 0, 0, 1, 0, 4096, 0, 0, 3686, 4067, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 55, 0, 0, 4096, 3830, 0, 4096, 0, 4095, 4070, 1, 0, 0, 0, 0, 246, 0, 0, 4096, 1, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 296, 0, 0, 4094, 2462, 0, 4096, 0, 4095, 4094, 0, 0, 0, 0, 0, 2056, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3389, 4085, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 4058, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 4074, 3444, 0, 604, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 38, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 0, 0, 4096, 0, 3080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 3, 0, 33, 0, 3986, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 4085, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8244, 8040, 8212, 8123, 8197, 8336] avg=8192.0000 max=8336 max_rank=7 min=8040 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32868 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 4058, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 4090, 2898, 0, 604, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 38, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 0, 0, 4096, 0, 3080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 6, 0, 4036, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 4044, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 7693, 8203, 8791, 8146, 8192, 8127] avg=8192.0000 max=8791 max_rank=4 min=7693 min_rank=2 imbalance_ratio=1.073120 max_load_reduction=56745 max_load_reduction_pct=0.865860 moved_to_echo=33256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 1, 4057, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 514, 0, 0, 4080, 3762, 0, 604, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 38, 4089, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 1, 0, 4095, 1, 3080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 13, 0, 4093, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 4067, 0, 0, 1, 0, 368, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8961, 8223, 7446, 8208, 8193, 8122] avg=8192.0000 max=8961 max_rank=2 min=7446 min_rank=4 imbalance_ratio=1.093872 max_load_reduction=56575 max_load_reduction_pct=0.863266 moved_to_echo=31969 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 3, 4095, 0, 4054, 38, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 78, 0, 0, 4092, 3317, 0, 604, 0, 0, 0, 0, 0, 0, 0, 4085, 3, 38, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 850, 0, 3, 4095, 0, 3079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 10, 0, 9, 0, 4091, 5, 1, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 1, 0, 7, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3991, 4079, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8196, 8094, 8185, 8027, 8209, 8195, 8439] avg=8192.0000 max=8439 max_rank=7 min=8027 min_rank=4 imbalance_ratio=1.030151 max_load_reduction=57097 max_load_reduction_pct=0.871231 moved_to_echo=32870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 4057, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 699, 0, 0, 4060, 2933, 0, 604, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 38, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 959, 0, 0, 4095, 0, 3080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 18, 0, 19, 0, 4094, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 4071, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8296, 8231, 8134, 8233, 8191, 8068] avg=8192.0000 max=8296 max_rank=2 min=8068 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 4056, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 4067, 3441, 0, 604, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 38, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 1, 0, 4096, 1, 3080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 24, 0, 4086, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3583, 4089, 0, 0, 1, 0, 368, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8202, 8225, 8281, 8213, 8192, 8041] avg=8192.0000 max=8281 max_rank=4 min=8041 min_rank=7 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 4057, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 4096, 3830, 0, 604, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 38, 4070, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 4096, 1, 3080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4087, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8585, 8205, 7423, 8191, 8191, 8558] avg=8192.0000 max=8585 max_rank=2 min=7423 min_rank=4 imbalance_ratio=1.047974 max_load_reduction=56951 max_load_reduction_pct=0.869003 moved_to_echo=32363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 4061, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 4094, 2462, 0, 609, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 34, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2056, 0, 0, 4096, 0, 3070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 1, 0, 4096, 7, 7, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3389, 4085, 0, 0, 1, 0, 370, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8189, 7461, 8224, 9222, 8213, 8185, 7845] avg=8192.0000 max=9222 max_rank=4 min=7461 min_rank=2 imbalance_ratio=1.125732 max_load_reduction=56314 max_load_reduction_pct=0.859283 moved_to_echo=33465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 32640, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 30080, 32640, 0, 0, 128, 0, 3072, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 8192, 128, 128, 32768, 128, 24704, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 384, 32512, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 0, 128, 32768, 128, 32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 2048, 0, 0, 32768, 26112, 0, 4864, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 47, 4092, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 4096, 0, 0, 0, 4096, 4062, 4096, 0, 0, 0, 3999, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4085, 4096, 4096, 4096, 0, 106, 4096, 7, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 4095, 0, 0, 0, 4095, 4089, 4096, 1, 0, 0, 4084, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4095, 4096, 0, 10, 4095, 7, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 20, 1, 1, 0, 0, 0, 4096, 0, 10, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 4083, 4095, 0, 0, 0, 3600, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 3985, 4096, 4096, 0, 587, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 4096, 0, 0, 0, 4096, 4081, 4091, 0, 0, 0, 4019, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 97, 4096, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4096, 0, 2, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 4093, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 5, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 5, 0, 1, 0, 0, 0, 4096, 0, 2, 4096, 0, 12, 7, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4094, 3, 0, 0, 4093, 4078, 4096, 2, 0, 0, 3836, 0, 4096, 0, 1, 0, 4096, 4096, 0, 0, 4085, 4096, 4094, 4094, 0, 252, 4092, 21, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 4096, 4089, 4096, 0, 0, 0, 4080, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 12, 4096, 13, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4096, 0, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 4096, 4070, 4096, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4093, 4096, 4096, 4096, 0, 17, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 47, 4070, 11, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 2, 0, 0, 0, 0, 4080, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4062, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 4096, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 4096, 7, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8224, 8186, 8179, 8174, 8218, 8199, 8163] avg=8192.0000 max=8224 max_rank=1 min=8163 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4069, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 4079, 8, 5, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 4095, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4089, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4095, 7, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8177, 8193, 8194, 8202, 8303, 8210, 8066] avg=8192.0000 max=8303 max_rank=5 min=8066 min_rank=7 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32781 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 4070, 11, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4080, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 13, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8190, 8184, 8189, 8201, 8299, 8210, 8075] avg=8192.0000 max=8299 max_rank=5 min=8075 min_rank=7 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32785 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 5, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 2, 4068, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 7, 0, 0, 0, 0, 4077, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4094, 8, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 16, 4078, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 4096, 0, 1, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 4092, 21, 0, 0, 0, 1, 0, 3954, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8178, 8205, 8190, 8195, 8056, 8199, 8320] avg=8192.0000 max=8320 max_rank=7 min=8056 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32770 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 4070, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4080, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4081, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 4096, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 4096, 2, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8189, 8191, 8184, 8188, 8238, 8210, 8149] avg=8192.0000 max=8238 max_rank=5 min=8149 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32785 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4096, 0, 2, 4070, 11, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 4080, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 1, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8181, 8188, 8199, 8204, 8310, 8210, 8056] avg=8192.0000 max=8310 max_rank=5 min=8056 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32780 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 20, 4096, 4091, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4096, 0, 10, 4070, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4080, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4083, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3600, 0, 4096, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3985, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 587, 4096, 1, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8189, 8191, 8199, 8194, 7819, 8099, 8638] avg=8192.0000 max=8638 max_rank=7 min=7819 min_rank=5 imbalance_ratio=1.054443 max_load_reduction=56898 max_load_reduction_pct=0.868195 moved_to_echo=32750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4096, 4092, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 25, 4072, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4084, 14, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4070, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8208, 8198, 8202, 8178, 8293, 8199, 8069] avg=8192.0000 max=8293 max_rank=5 min=8069 min_rank=7 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32739 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 128, 32768, 0, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1152, 32768, 128, 0, 0, 0, 128, 0, 31744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 31872, 0, 32768, 0, 128, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 4096, 4055, 0, 0, 0, 1189, 0, 0, 16, 4096, 4, 0, 4096, 4096, 153, 0, 4096, 4093, 4096, 0, 0, 0, 0, 0, 0, 142, 0, 0, 2, 0, 2655, 0, 0, 0, 4096, 4085, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 4095, 4038, 0, 1, 0, 1082, 0, 0, 15, 4096, 1, 0, 4096, 4096, 601, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 101, 0, 0, 3, 0, 2361, 0, 0, 0, 4095, 4091, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 4059, 0, 0, 0, 2134, 0, 0, 0, 4096, 0, 0, 4096, 4096, 47, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 46, 0, 0, 7, 0, 1901, 0, 0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 4096, 4090, 0, 0, 0, 3052, 0, 0, 4, 4096, 5, 0, 4096, 4096, 96, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 902, 0, 0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 4091, 0, 0, 0, 357, 0, 0, 11, 4096, 0, 0, 4096, 4096, 2632, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1101, 0, 0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 0, 4096, 4011, 0, 0, 0, 1695, 0, 0, 18, 4096, 0, 0, 4096, 4096, 289, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 172, 0, 0, 33, 0, 1991, 0, 0, 0, 4096, 4088, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 1, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4091, 0, 4096, 4024, 1, 4, 6, 936, 0, 0, 13, 4096, 8, 0, 4092, 4096, 1335, 0, 4096, 4093, 4096, 0, 0, 0, 0, 0, 0, 12, 0, 0, 36, 0, 1840, 0, 0, 0, 4093, 4091, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 4096, 4076, 0, 0, 0, 1354, 0, 0, 10, 4096, 0, 0, 4096, 4096, 246, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 137, 0, 0, 10, 0, 2365, 0, 0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 1982, 1435, 677, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2661, 4055, 0, 0, 0, 1189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3411, 4, 0, 4096, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 2, 0, 2655, 0, 4096, 2114, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8187, 7905, 7680, 8189, 9009, 8184] avg=8192.0000 max=9009 max_rank=6 min=7680 min_rank=4 imbalance_ratio=1.099731 max_load_reduction=56527 max_load_reduction_pct=0.862534 moved_to_echo=33062 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 1982, 1435, 677, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2660, 4038, 0, 1, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3411, 1, 0, 4096, 0, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 3, 0, 2361, 0, 4096, 2112, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4091, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8190, 8194, 7781, 8124, 8194, 8673, 8190] avg=8192.0000 max=8673 max_rank=6 min=7781 min_rank=3 imbalance_ratio=1.058716 max_load_reduction=56863 max_load_reduction_pct=0.867661 moved_to_echo=33181 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 1982, 1435, 677, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2661, 4090, 0, 0, 0, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3411, 5, 0, 4096, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 902, 0, 4096, 2112, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8193, 9803, 7612, 8194, 7157, 8195] avg=8192.0000 max=9803 max_rank=3 min=7157 min_rank=6 imbalance_ratio=1.196655 max_load_reduction=55733 max_load_reduction_pct=0.850418 moved_to_echo=31158 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 1, 0, 0, 0, 1982, 1435, 677, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4091, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2661, 4024, 1, 4, 6, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3411, 8, 0, 4092, 0, 1335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 36, 0, 1840, 0, 4096, 2112, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4091, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8187, 8190, 7632, 8859, 8191, 8096, 8189] avg=8192.0000 max=8859 max_rank=4 min=7632 min_rank=3 imbalance_ratio=1.081421 max_load_reduction=56677 max_load_reduction_pct=0.864822 moved_to_echo=33335 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 1982, 1435, 677, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2661, 4059, 0, 0, 0, 2134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3411, 0, 0, 4096, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 7, 0, 1901, 0, 4096, 2114, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8196, 8854, 7554, 8191, 8164, 8195] avg=8192.0000 max=8854 max_rank=3 min=7554 min_rank=4 imbalance_ratio=1.080811 max_load_reduction=56682 max_load_reduction_pct=0.864899 moved_to_echo=32104 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 1982, 1435, 677, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2661, 4011, 0, 0, 0, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3411, 0, 0, 4096, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 33, 0, 1991, 0, 4096, 2113, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8188, 8367, 7814, 8193, 8405, 8187] avg=8192.0000 max=8405 max_rank=6 min=7814 min_rank=4 imbalance_ratio=1.026001 max_load_reduction=57131 max_load_reduction_pct=0.871750 moved_to_echo=32599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 1982, 1435, 677, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2660, 4091, 0, 0, 0, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3411, 0, 0, 4096, 0, 2632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1101, 0, 4096, 2114, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8197, 7108, 10150, 8192, 7312, 8195] avg=8192.0000 max=10150 max_rank=4 min=7108 min_rank=3 imbalance_ratio=1.239014 max_load_reduction=55386 max_load_reduction_pct=0.845123 moved_to_echo=33849 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 1984, 1440, 684, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2656, 4076, 0, 0, 0, 1354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3391, 0, 0, 4096, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 10, 0, 2365, 0, 4096, 2112, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8199, 8191, 8086, 7743, 8192, 8720, 8201] avg=8192.0000 max=8720 max_rank=6 min=7743 min_rank=4 imbalance_ratio=1.064453 max_load_reduction=56816 max_load_reduction_pct=0.866943 moved_to_echo=32856 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 27392, 128, 0, 32768, 0, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 32768, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 32768, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 768, 0, 0, 128, 0, 15232, 0, 32768, 17024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[21376, 32512, 128, 128, 128, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 0, 0, 15872, 11520, 5504, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 73, 0, 0, 0, 0, 23, 0, 0, 4096, 4096, 0, 0, 0, 4079, 0, 15, 0, 4096, 0, 9, 0, 0, 0, 0, 4085, 4096, 0, 0, 0, 0, 4096, 4064, 4096, 0, 0, 0, 4096, 0, 0, 4028, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 2864, 0, 0, 0, 0, 48, 0, 0, 4096, 4096, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3575, 4096, 0, 0, 0, 4096, 0, 0, 1754, 4047, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 1, 0, 4095, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 1, 0, 0, 2023, 0, 0, 0, 0, 48, 0, 0, 4096, 4096, 0, 0, 0, 4084, 0, 5, 0, 4096, 0, 12, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 4096, 3515, 4096, 0, 0, 0, 4096, 0, 0, 3055, 3638, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 3, 4, 4096, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 470, 0, 0, 0, 0, 18, 0, 0, 4096, 4096, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 4096, 3948, 4096, 0, 0, 0, 4096, 1, 0, 3778, 4069, 0, 0, 0, 4095, 0, 0, 4096, 0, 3, 0, 0, 4, 1, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 1315, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3970, 4096, 0, 0, 0, 4096, 1, 0, 3073, 3926, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 357, 0, 0, 0, 0, 35, 0, 0, 4096, 4096, 0, 0, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4068, 4096, 0, 0, 0, 4096, 0, 0, 3804, 4039, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 953, 0, 0, 0, 0, 24, 0, 0, 4094, 4096, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3990, 4096, 0, 0, 0, 4096, 0, 0, 3300, 4030, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 529, 0, 0, 0, 0, 80, 0, 0, 4093, 4096, 0, 0, 0, 4086, 0, 3, 0, 4096, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3955, 4096, 0, 0, 0, 4096, 0, 0, 3642, 4077, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 73, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 4096, 3885, 172, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4079, 0, 15, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 9, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4064, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 0, 0, 4028, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7191, 8181, 8196, 8187, 8371, 9022, 8192, 8196] avg=8192.0000 max=9022 max_rank=5 min=7191 min_rank=0 imbalance_ratio=1.101318 max_load_reduction=56514 max_load_reduction_pct=0.862335 moved_to_echo=33781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 470, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 3885, 172, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 11, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3948, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 1, 0, 3778, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 1, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7588, 8175, 8187, 8197, 8255, 8747, 8191, 8196] avg=8192.0000 max=8747 max_rank=5 min=7588 min_rank=0 imbalance_ratio=1.067749 max_load_reduction=56789 max_load_reduction_pct=0.866531 moved_to_echo=33389 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 1, 0, 0, 2023, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 4096, 3885, 172, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4084, 0, 5, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 12, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3515, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 0, 0, 3055, 3638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4096, 0, 4, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9142, 8205, 8191, 8195, 7822, 7592, 8190, 8199] avg=8192.0000 max=9142 max_rank=0 min=7592 min_rank=5 imbalance_ratio=1.115967 max_load_reduction=56394 max_load_reduction_pct=0.860504 moved_to_echo=31803 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 2864, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 4096, 3885, 172, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3575, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 0, 0, 1754, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4095, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9982, 8206, 8193, 8191, 7882, 6700, 8192, 8190] avg=8192.0000 max=9982 max_rank=0 min=6700 min_rank=5 imbalance_ratio=1.218506 max_load_reduction=55554 max_load_reduction_pct=0.847687 moved_to_echo=30964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 529, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 4093, 3885, 172, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 3, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3955, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 0, 0, 3642, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7647, 8234, 8191, 8199, 8262, 8618, 8192, 8193] avg=8192.0000 max=8618 max_rank=5 min=7647 min_rank=0 imbalance_ratio=1.052002 max_load_reduction=56918 max_load_reduction_pct=0.868500 moved_to_echo=33265 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 1315, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3885, 172, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3970, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 1, 0, 3073, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8433, 8157, 8198, 8193, 8277, 7899, 8192, 8187] avg=8192.0000 max=8433 max_rank=0 min=7899 min_rank=5 imbalance_ratio=1.029419 max_load_reduction=57103 max_load_reduction_pct=0.871323 moved_to_echo=32555 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 357, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 4096, 3885, 172, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4081, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 0, 0, 3804, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7475, 8192, 8183, 8189, 8375, 8742, 8192, 8188] avg=8192.0000 max=8742 max_rank=5 min=7475 min_rank=0 imbalance_ratio=1.067139 max_load_reduction=56794 max_load_reduction_pct=0.866608 moved_to_echo=33497 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 953, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4094, 3890, 173, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 4088, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3990, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 0, 0, 3300, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8078, 8186, 8197, 8185, 8292, 8216, 8195, 8187] avg=8192.0000 max=8292 max_rank=4 min=8078 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 0, 128, 0, 0, 8704, 24192, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 7296, 128, 0, 26496, 32000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 32768, 31104, 1792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 0, 32768, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 384, 0, 0, 32768, 31104, 1408, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4073, 0, 0, 4096, 0, 4096, 4096, 1965, 0, 0, 29, 0, 2943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4030, 0, 1, 0, 0, 3858, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 1759, 0, 0, 0, 1822, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4084, 0, 1, 4095, 0, 4088, 4096, 68, 0, 0, 45, 0, 3799, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 3861, 0, 0, 0, 0, 3907, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 2098, 9, 0, 0, 2616, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 0, 1, 4091, 0, 4094, 4096, 60, 0, 0, 10, 0, 3205, 0, 0, 1, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 4090, 0, 0, 4069, 2, 0, 0, 0, 4067, 0, 4096, 0, 4095, 0, 0, 4096, 4095, 2533, 0, 0, 0, 2446, 0, 3, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4095, 0, 4096, 4096, 132, 0, 0, 4, 0, 3969, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3990, 0, 0, 0, 0, 4041, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 1746, 0, 0, 0, 2504, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4069, 0, 1, 4089, 0, 4090, 4096, 1096, 0, 0, 56, 0, 3771, 0, 0, 6, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 4096, 4086, 0, 0, 4047, 3, 6, 0, 0, 4044, 0, 4096, 2, 4092, 0, 0, 4096, 4092, 2415, 0, 0, 0, 970, 0, 7, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4055, 0, 0, 4095, 0, 4096, 4096, 1404, 0, 0, 11, 0, 3779, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3909, 0, 3, 0, 0, 3952, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 1950, 0, 0, 0, 1417, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4073, 0, 0, 4092, 0, 4096, 4096, 338, 0, 0, 23, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3780, 0, 0, 0, 0, 3931, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 1895, 0, 0, 0, 2508, 0, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 4096, 109, 0, 0, 29, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4083, 0, 0, 0, 0, 4062, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 2920, 0, 0, 0, 1157, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4084, 0, 1, 4095, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3428, 68, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3799, 0, 0, 0, 0, 0, 4, 0, 4096, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3683, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 0, 0, 0, 0, 3907, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1933, 0, 0, 0, 4096, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2616, 0, 0, 0, 4096, 1751, 412, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 7629, 8312, 8191, 8190, 8004, 8127, 8884] avg=8192.0000 max=8884 max_rank=7 min=7629 min_rank=1 imbalance_ratio=1.084473 max_load_reduction=56652 max_load_reduction_pct=0.864441 moved_to_echo=33205 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 0, 1, 4091, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3428, 60, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3205, 0, 0, 1, 0, 0, 11, 0, 4096, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3684, 412, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 2, 0, 0, 0, 4067, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1933, 0, 0, 0, 4095, 2533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2446, 0, 3, 0, 4096, 1750, 412, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 7592, 7725, 8192, 8188, 8374, 8561, 8707] avg=8192.0000 max=8707 max_rank=7 min=7592 min_rank=1 imbalance_ratio=1.062866 max_load_reduction=56829 max_load_reduction_pct=0.867142 moved_to_echo=33830 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4073, 0, 0, 4092, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3428, 338, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 4096, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3684, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 0, 0, 3931, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1933, 0, 0, 0, 4096, 1895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2508, 0, 1, 0, 4096, 1751, 412, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 7885, 8443, 8192, 8193, 7947, 7924, 8768] avg=8192.0000 max=8768 max_rank=7 min=7885 min_rank=1 imbalance_ratio=1.070312 max_load_reduction=56768 max_load_reduction_pct=0.866211 moved_to_echo=32832 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4073, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3428, 1965, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2943, 0, 0, 0, 0, 0, 0, 0, 4096, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3683, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 1, 0, 0, 3858, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1933, 0, 0, 0, 4096, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1822, 0, 0, 0, 4096, 1751, 412, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 9518, 7452, 8191, 8193, 8125, 7788, 8081] avg=8192.0000 max=9518 max_rank=1 min=7452 min_rank=2 imbalance_ratio=1.161865 max_load_reduction=56018 max_load_reduction_pct=0.854767 moved_to_echo=32187 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4069, 0, 1, 4089, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3428, 1096, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3771, 0, 0, 6, 0, 0, 14, 0, 4096, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3684, 412, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 4086, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 3, 6, 0, 0, 4044, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1931, 0, 0, 0, 4092, 2415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 0, 7, 0, 4096, 1749, 412, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8670, 8299, 8192, 8187, 8336, 8440, 7234] avg=8192.0000 max=8670 max_rank=1 min=7234 min_rank=7 imbalance_ratio=1.058350 max_load_reduction=56866 max_load_reduction_pct=0.867706 moved_to_echo=32197 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4055, 0, 0, 4095, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3428, 1404, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 1, 0, 0, 0, 0, 0, 0, 4096, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3683, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 0, 3, 0, 0, 3952, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1933, 0, 0, 0, 4096, 1950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 0, 0, 0, 4096, 1751, 412, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8939, 8289, 8191, 8193, 8100, 7979, 7676] avg=8192.0000 max=8939 max_rank=1 min=7676 min_rank=7 imbalance_ratio=1.091187 max_load_reduction=56597 max_load_reduction_pct=0.863602 moved_to_echo=31948 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4095, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3428, 132, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 3, 0, 4096, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3684, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 0, 0, 0, 4041, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1933, 0, 0, 0, 4096, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2504, 0, 0, 0, 4096, 1751, 412, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 7660, 8480, 8192, 8193, 8267, 7775, 8763] avg=8192.0000 max=8763 max_rank=7 min=7660 min_rank=1 imbalance_ratio=1.069702 max_load_reduction=56773 max_load_reduction_pct=0.866287 moved_to_echo=32998 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3409, 109, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 4096, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3683, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4062, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1926, 0, 0, 0, 4096, 2920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 0, 0, 0, 4096, 1754, 416, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 7643, 8536, 8195, 8199, 8383, 8942, 7423] avg=8192.0000 max=8942 max_rank=6 min=7423 min_rank=7 imbalance_ratio=1.091553 max_load_reduction=56594 max_load_reduction_pct=0.863556 moved_to_echo=32947 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 15488, 0, 128, 0, 32768, 14080, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 15488, 0, 0, 0, 32768, 17408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 27520, 5248, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 32640, 0, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[29440, 128, 0, 128, 0, 0, 128, 0, 32768, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 32768, 0, 0, 29568, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 31872, 128, 128, 0, 0, 31872, 0, 1920, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4087, 0, 0, 4096, 0, 4096, 7, 4096, 0, 4096, 0, 977, 4096, 0, 0, 4014, 0, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 1837, 4096, 0, 0, 3933, 0, 2422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 1, 4091, 0, 0, 4096, 0, 4095, 2, 4096, 0, 4096, 0, 1009, 4095, 0, 0, 3923, 0, 3260, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 4096, 0, 4096, 0, 4096, 0, 332, 4096, 0, 1, 3942, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 0, 4096, 3, 4085, 0, 0, 4092, 0, 4095, 5, 4096, 0, 4096, 0, 213, 4091, 0, 0, 3955, 0, 4020, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4088, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 1354, 4096, 0, 0, 3531, 0, 3291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 4096, 0, 4096, 0, 4096, 0, 123, 4096, 0, 0, 4053, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 517, 4096, 0, 0, 4003, 0, 3664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3300, 0, 977, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 3181, 0, 0, 0, 0, 796, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8188, 8380, 7994, 8192, 8206, 8192] avg=8192.0000 max=8380 max_rank=3 min=7994 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3300, 0, 1837, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 2422, 0, 0, 0, 0, 796, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8197, 9233, 7154, 8192, 8184, 8192] avg=8192.0000 max=9233 max_rank=3 min=7154 min_rank=4 imbalance_ratio=1.127075 max_load_reduction=56303 max_load_reduction_pct=0.859116 moved_to_echo=31722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3300, 0, 1009, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 3260, 0, 0, 0, 1, 796, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8193, 8406, 7983, 8192, 8183, 8196] avg=8192.0000 max=8406 max_rank=3 min=7983 min_rank=4 imbalance_ratio=1.026123 max_load_reduction=57130 max_load_reduction_pct=0.871735 moved_to_echo=32554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4085, 0, 0, 4092, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3300, 0, 213, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 4020, 0, 0, 0, 3, 796, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8189, 8185, 7609, 8777, 8192, 8194, 8198] avg=8192.0000 max=8777 max_rank=4 min=7609 min_rank=3 imbalance_ratio=1.071411 max_load_reduction=56759 max_load_reduction_pct=0.866074 moved_to_echo=33361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3300, 0, 332, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3942, 0, 3918, 0, 0, 0, 0, 796, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8196, 7728, 8660, 8192, 8184, 8192] avg=8192.0000 max=8660 max_rank=4 min=7728 min_rank=3 imbalance_ratio=1.057129 max_load_reduction=56876 max_load_reduction_pct=0.867859 moved_to_echo=33228 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3300, 0, 1354, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3531, 0, 3291, 0, 0, 0, 0, 796, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8189, 8750, 7621, 8192, 8208, 8192] avg=8192.0000 max=8750 max_rank=3 min=7621 min_rank=4 imbalance_ratio=1.068115 max_load_reduction=56786 max_load_reduction_pct=0.866486 moved_to_echo=32213 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3300, 0, 123, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 4017, 0, 0, 0, 0, 796, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8196, 7519, 8869, 8192, 8184, 8192] avg=8192.0000 max=8869 max_rank=4 min=7519 min_rank=3 imbalance_ratio=1.082642 max_load_reduction=56667 max_load_reduction_pct=0.864670 moved_to_echo=33437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3298, 0, 517, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 3664, 0, 0, 0, 0, 798, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8196, 8192, 7911, 8478, 8192, 8193, 8182] avg=8192.0000 max=8478 max_rank=4 min=7911 min_rank=3 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=33045 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65536 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 26496, 0, 6400, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65536 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 0, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 31360, 0, 27776, 0, 0, 0, 128, 6400, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65536 slack_slots=4736 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65536 slack_slots=4736 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 4068, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 28, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 7, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 4006, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 90, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 0, 0, 0, 4077, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 30, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 16, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 3998, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 46, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 80, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 0, 0, 1, 0, 4095, 0, 4, 4096, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4095, 4076, 0, 0, 0, 4036, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 43, 0, 0, 0, 4096, 4095, 0, 0, 0, 4095, 0, 0, 4096, 37, 0, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 7, 0, 0, 2, 0, 4092, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4089, 4080, 0, 0, 0, 4012, 4096, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 4096, 0, 0, 0, 4084, 0, 18, 0, 0, 0, 4096, 4095, 0, 0, 1, 4096, 0, 0, 4096, 84, 0, 3, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 4061, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 14, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 36, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4026, 44, 23, 1, 0, 0, 0, 0, 4090, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 70, 4081, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 90, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8191, 8191, 8157, 8192, 8185, 8192, 8238] avg=8192.0000 max=8238 max_rank=7 min=8157 min_rank=3 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4026, 44, 23, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 70, 4089, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4096, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8192, 8191, 8227, 8192, 8197, 8193, 8155] avg=8192.0000 max=8227 max_rank=3 min=8155 min_rank=7 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 0, 0, 1, 0, 4025, 44, 23, 0, 0, 0, 0, 0, 4095, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 70, 4076, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4092, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 37, 0, 1, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8195, 8194, 8182, 8191, 8208, 8190, 8186] avg=8192.0000 max=8208 max_rank=5 min=8182 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=57328 max_load_reduction_pct=0.874756 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 7, 0, 0, 2, 0, 4019, 44, 23, 0, 0, 0, 0, 0, 4092, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 70, 4080, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4084, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 84, 0, 3, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8195, 8196, 8162, 8190, 8175, 8192, 8235] avg=8192.0000 max=8235 max_rank=7 min=8162 min_rank=3 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4026, 44, 23, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 70, 4067, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4096, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8191, 8214, 8193, 8199, 8193, 8164] avg=8192.0000 max=8214 max_rank=3 min=8164 min_rank=7 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4026, 44, 23, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 70, 4068, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4096, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 80, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8191, 8136, 8192, 8215, 8192, 8228] avg=8192.0000 max=8228 max_rank=7 min=8136 min_rank=3 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32827 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 0, 0, 0, 0, 4026, 44, 23, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 70, 4096, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8191, 8252, 8192, 8177, 8192, 8149] avg=8192.0000 max=8252 max_rank=3 min=8149 min_rank=7 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4032, 47, 26, 1, 3, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 64, 4081, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 36, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8187, 8191, 8206, 8194, 8180, 8192, 8181] avg=8192.0000 max=8206 max_rank=3 min=8180 min_rank=5 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 384, 0, 128, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 128, 0, 32256, 384, 256, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[640, 32640, 0, 0, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 32768, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32640, 0, 0, 0, 32768, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4089, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 7, 122, 3998, 0, 0, 4034, 0, 0, 26, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 3, 4094, 0, 12, 0, 0, 0, 0, 4096, 4096, 4096, 2, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 26, 4082, 0, 0, 4065, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4092, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 29, 4060, 0, 0, 4072, 0, 0, 21, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 4096, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 17, 4068, 0, 0, 4094, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 53, 4083, 0, 0, 4041, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 3982, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4062, 0, 0, 4076, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4055, 0, 0, 4090, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4032, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 53, 4096, 4032, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 4041, 0, 0, 4, 0, 62, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 4030, 62, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8192, 8192, 8190, 8193, 8196, 8191] avg=8192.0000 max=8196 max_rank=6 min=8190 min_rank=0 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4032, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 29, 4096, 4032, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 4072, 0, 0, 21, 0, 62, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 2, 0, 0, 4030, 62, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8192, 8170, 8215, 8191, 8191, 8192] avg=8192.0000 max=8215 max_rank=4 min=8170 min_rank=3 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4032, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 7, 122, 4096, 4032, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 4034, 0, 0, 26, 0, 62, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 4094, 0, 12, 0, 0, 4030, 62, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8182, 8192, 8270, 8120, 8187, 8204, 8198] avg=8192.0000 max=8270 max_rank=3 min=8120 min_rank=4 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4032, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 17, 4096, 4033, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 4094, 0, 0, 6, 0, 62, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4030, 62, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8194, 8191, 8153, 8230, 8193, 8188, 8191] avg=8192.0000 max=8230 max_rank=4 min=8153 min_rank=3 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32802 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4032, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 26, 4096, 4032, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 4065, 0, 0, 14, 0, 62, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4030, 62, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8194, 8192, 8159, 8223, 8193, 8188, 8191] avg=8192.0000 max=8223 max_rank=4 min=8159 min_rank=3 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32795 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4032, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 4096, 4032, 0, 0, 0, 0, 0, 0, 3982, 0, 0, 4096, 0, 0, 1, 0, 64, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4032, 64, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8193, 8192, 8241, 8143, 8193, 8192, 8191] avg=8192.0000 max=8241 max_rank=3 min=8143 min_rank=4 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4032, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4096, 4032, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 4076, 0, 0, 2, 0, 62, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4030, 62, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8196, 8192, 8178, 8202, 8193, 8188, 8191] avg=8192.0000 max=8202 max_rank=4 min=8178 min_rank=3 imbalance_ratio=1.001221 max_load_reduction=57334 max_load_reduction_pct=0.874847 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4033, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4096, 4032, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 4090, 0, 0, 2, 0, 66, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4030, 63, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8193, 8173, 8213, 8193, 8189, 8191] avg=8192.0000 max=8213 max_rank=4 min=8173 min_rank=3 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32512, 0, 0, 32640, 0, 0, 128, 0, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 0, 128, 0, 0, 32256, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 512, 32768, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 0, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 0, 0, 32384, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 16, 0, 0, 0, 0, 4096, 4080, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 4089, 4096, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4089, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 4096, 21, 0, 0, 0, 0, 4093, 4080, 4096, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 4096, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 4095, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4096, 7, 0, 0, 0, 0, 4094, 4090, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 9, 0, 0, 0, 0, 4096, 4088, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 4090, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 9, 4080, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8193, 8192, 8199, 8185, 8192, 8190, 8192] avg=8192.0000 max=8199 max_rank=3 min=8185 min_rank=4 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 9, 4089, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8192, 8186, 8191, 8193, 8197, 8191] avg=8192.0000 max=8197 max_rank=6 min=8186 min_rank=3 imbalance_ratio=1.000610 max_load_reduction=57339 max_load_reduction_pct=0.874924 moved_to_echo=32772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 7, 0, 0, 0, 0, 4083, 2, 0, 0, 0, 0, 0, 0, 10, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8191, 8191, 8196, 8192, 8192, 8192] avg=8192.0000 max=8196 max_rank=4 min=8190 min_rank=0 imbalance_ratio=1.000488 max_load_reduction=57340 max_load_reduction_pct=0.874939 moved_to_echo=32772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 21, 0, 0, 0, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 10, 4080, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8190, 8191, 8206, 8183, 8191, 8197, 8191] avg=8192.0000 max=8206 max_rank=3 min=8183 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8192, 8184, 8198, 8192, 8193, 8193] avg=8192.0000 max=8198 max_rank=4 min=8184 min_rank=3 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8192, 8183, 8200, 8192, 8191, 8192] avg=8192.0000 max=8200 max_rank=4 min=8183 min_rank=3 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 9, 4088, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8192, 8192, 8192, 8192, 8190, 8192] avg=8192.0000 max=8194 max_rank=0 min=8190 min_rank=6 imbalance_ratio=1.000244 max_load_reduction=57342 max_load_reduction_pct=0.874969 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4086, 7, 0, 0, 0, 0, 0, 0, 6, 4090, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8194, 8195, 8191, 8192, 8186, 8193] avg=8192.0000 max=8195 max_rank=3 min=8186 min_rank=6 imbalance_ratio=1.000366 max_load_reduction=57341 max_load_reduction_pct=0.874954 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 128, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 32768, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 92, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 4096, 0, 4096, 64, 0, 0, 0, 0, 3964, 0, 4096, 4068, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 38, 0, 0, 0, 0, 3961, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 79, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 4095, 0, 4096, 22, 0, 0, 0, 0, 4071, 0, 4096, 4086, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 7, 4096, 0, 4095, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 4090, 4089, 0, 4096, 117, 0, 0, 0, 0, 4028, 0, 4096, 4089, 0, 0, 0, 4096, 4096, 4093, 0, 3, 0, 0, 0, 2, 0, 0, 0, 4096, 2, 4093, 0, 0, 0, 4096, 0, 4067, 0, 0, 0, 4081, 0, 0, 0, 4096, 0, 4, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 22, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 138, 0, 0, 0, 0, 4009, 0, 4096, 4086, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4033, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 47, 0, 0, 0, 0, 4080, 0, 4096, 4059, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 40, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 33, 0, 0, 0, 0, 4026, 0, 4096, 4083, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 41, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 110, 0, 0, 0, 0, 3936, 0, 4096, 4076, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 1, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 0, 0, 0, 0, 4071, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4067, 27, 0, 0, 0, 0, 0, 0, 0, 7, 4081, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4080, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8192, 8204, 8198, 8191, 8183, 8206, 8191] avg=8192.0000 max=8206 max_rank=6 min=8171 min_rank=0 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 92, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 64, 0, 0, 0, 0, 3964, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4068, 29, 0, 0, 0, 0, 0, 0, 0, 7, 4081, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4080, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8190, 8139, 8180, 8193, 8184, 8207, 8190] avg=8192.0000 max=8253 max_rank=0 min=8139 min_rank=2 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 38, 0, 0, 0, 0, 3961, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4068, 27, 0, 0, 0, 0, 0, 0, 0, 79, 4081, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8196, 8110, 8208, 8191, 8256, 8207, 8189] avg=8192.0000 max=8256 max_rank=5 min=8110 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32843 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 0, 1, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4090, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 117, 0, 0, 0, 0, 4028, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 2, 0, 4068, 27, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4067, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 4080, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8182, 8256, 8201, 8193, 8176, 8163, 8193] avg=8192.0000 max=8256 max_rank=2 min=8163 min_rank=6 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 138, 0, 0, 0, 0, 4009, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4068, 27, 0, 0, 0, 0, 0, 0, 0, 1, 4081, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4033, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8196, 8258, 8198, 8190, 8178, 8144, 8189] avg=8192.0000 max=8258 max_rank=2 min=8144 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32701 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 4080, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4068, 27, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8196, 8238, 8171, 8191, 8177, 8206, 8189] avg=8192.0000 max=8238 max_rank=2 min=8168 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 40, 0, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 33, 0, 0, 0, 0, 4026, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4068, 28, 0, 0, 0, 0, 0, 0, 0, 10, 4081, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8192, 8170, 8195, 8192, 8187, 8207, 8189] avg=8192.0000 max=8207 max_rank=6 min=8170 min_rank=2 imbalance_ratio=1.001831 max_load_reduction=57329 max_load_reduction_pct=0.874771 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 41, 0, 0, 0, 0, 0, 0, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 110, 0, 0, 0, 0, 3936, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4072, 27, 0, 0, 0, 0, 0, 0, 0, 29, 4070, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4083, 20, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8192, 8161, 8185, 8195, 8195, 8196, 8206] avg=8192.0000 max=8206 max_rank=0 min=8161 min_rank=2 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 128, 0, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 32768, 640, 0, 0, 0, 0, 32128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 0, 128, 0, 0, 0, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 256, 32640, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 256, 0, 0, 0, 128, 0, 0, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4070, 0, 4092, 0, 3062, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 8, 0, 0, 1039, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4092, 4096, 0, 4096, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 3466, 0, 4096, 0, 2654, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2079, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4080, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 3646, 0, 4084, 0, 2134, 0, 0, 0, 0, 0, 0, 0, 5, 0, 15, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 15, 4096, 0, 0, 0, 4095, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 13, 0, 0, 2404, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4078, 4096, 0, 4096, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4085, 0, 4096, 0, 2507, 0, 0, 0, 0, 0, 0, 0, 33, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1575, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4076, 4096, 0, 4096, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 3795, 0, 4096, 0, 2873, 0, 0, 0, 0, 0, 0, 0, 98, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 1416, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4093, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 3884, 0, 4096, 0, 2694, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 1389, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 3658, 0, 4096, 0, 1352, 0, 0, 0, 0, 0, 0, 0, 91, 0, 14, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 12, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 3091, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4073, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4074, 0, 4096, 0, 2973, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1098, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4088, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4070, 0, 1826, 0, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4, 4096, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4025, 71, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 8, 0, 0, 2266, 1761, 58, 6, 0, 0, 0, 0, 1039, 0, 4096, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 13, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8958, 8140, 8192, 8191, 8192, 8194, 7470, 8199] avg=8192.0000 max=8958 max_rank=0 min=7470 min_rank=6 imbalance_ratio=1.093506 max_load_reduction=56578 max_load_reduction_pct=0.863312 moved_to_echo=32055 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3658, 0, 1828, 0, 1352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 14, 4096, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4025, 71, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2268, 1761, 58, 6, 0, 0, 0, 0, 3091, 0, 4096, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6838, 8226, 8192, 8198, 8192, 8186, 9522, 8182] avg=8192.0000 max=9522 max_rank=6 min=6838 min_rank=0 imbalance_ratio=1.162354 max_load_reduction=56014 max_load_reduction_pct=0.854706 moved_to_echo=34082 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3646, 0, 1822, 0, 2134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 15, 4096, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4090, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4025, 71, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 13, 0, 0, 2262, 1761, 58, 6, 0, 0, 0, 0, 2404, 0, 4096, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 13, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7602, 8141, 8192, 8200, 8194, 8189, 8835, 8183] avg=8192.0000 max=8835 max_rank=6 min=7602 min_rank=0 imbalance_ratio=1.078491 max_load_reduction=56701 max_load_reduction_pct=0.865189 moved_to_echo=33401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4085, 0, 1828, 0, 2507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 1, 4096, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4025, 71, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2268, 1761, 58, 6, 0, 0, 0, 0, 1575, 0, 4096, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 13, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8420, 8155, 8192, 8199, 8192, 8189, 8006, 8183] avg=8192.0000 max=8420 max_rank=0 min=8006 min_rank=6 imbalance_ratio=1.027832 max_load_reduction=57116 max_load_reduction_pct=0.871521 moved_to_echo=32570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3466, 0, 1828, 0, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4025, 71, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2268, 1761, 58, 6, 0, 0, 0, 0, 2079, 0, 4096, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7948, 8128, 8192, 8188, 8192, 8189, 8510, 8189] avg=8192.0000 max=8510 max_rank=6 min=7948 min_rank=0 imbalance_ratio=1.038818 max_load_reduction=57026 max_load_reduction_pct=0.870148 moved_to_echo=33080 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3795, 0, 1828, 0, 2873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 5, 4096, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4025, 71, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 2268, 1761, 58, 6, 0, 0, 0, 0, 1416, 0, 4096, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8496, 8224, 8192, 8192, 8192, 8191, 7847, 8202] avg=8192.0000 max=8496 max_rank=0 min=7847 min_rank=6 imbalance_ratio=1.037109 max_load_reduction=57040 max_load_reduction_pct=0.870361 moved_to_echo=32432 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3884, 0, 1828, 0, 2694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 4096, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4025, 70, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 2268, 1761, 58, 6, 0, 0, 0, 0, 1389, 0, 4096, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8406, 8345, 8192, 8187, 8191, 8190, 7820, 8205] avg=8192.0000 max=8406 max_rank=0 min=7820 min_rank=6 imbalance_ratio=1.026123 max_load_reduction=57130 max_load_reduction_pct=0.871735 moved_to_echo=32406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4074, 0, 1821, 0, 2973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 4096, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4025, 70, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2275, 1764, 62, 11, 0, 0, 0, 0, 1098, 0, 4096, 2332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8868, 8177, 8192, 8181, 8191, 8208, 7526, 8193] avg=8192.0000 max=8868 max_rank=0 min=7526 min_rank=6 imbalance_ratio=1.082520 max_load_reduction=56668 max_load_reduction_pct=0.864685 moved_to_echo=32118 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 30720, 0, 14720, 0, 20352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[14208, 0, 32768, 18688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 0, 0, 128, 0, 0, 18176, 14208, 512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65536 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0, 32256, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 640, 0, 128, 32768, 32256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65536 slack_slots=4736 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3101, 4096, 0, 0, 0, 4096, 0, 0, 0, 1125, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 4096, 1, 4088, 0, 0, 3804, 4096, 0, 0, 0, 4096, 0, 4, 0, 317, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 4093, 33, 0, 0, 0, 0, 1, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 3541, 4096, 0, 0, 0, 4096, 0, 0, 0, 689, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 4096, 4096, 1, 0, 0, 0, 4095, 0, 0, 1, 0, 4096, 0, 4095, 0, 0, 2756, 4096, 0, 0, 0, 4096, 0, 1, 0, 1387, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 4095, 8, 0, 0, 0, 0, 1, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 2272, 4096, 0, 0, 0, 4096, 0, 0, 0, 1848, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3547, 4096, 0, 0, 0, 4095, 0, 0, 0, 601, 0, 4083, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 3430, 4096, 0, 0, 0, 4096, 0, 0, 0, 710, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2943, 4096, 0, 0, 0, 4096, 0, 0, 0, 1188, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4035, 62, 0, 0, 0, 0, 0, 0, 924, 0, 4096, 0, 0, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1125, 0, 3110, 2, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 0, 0, 0, 0, 4096, 61, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8121, 8333, 8183, 8206, 8192, 8116] avg=8192.0000 max=8333 max_rank=3 min=8116 min_rank=7 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32697 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 4034, 62, 0, 0, 0, 0, 0, 0, 924, 1, 4088, 0, 0, 3804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 317, 0, 3110, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4093, 33, 0, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 4, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8189, 8817, 7529, 8195, 8215, 8191, 8201] avg=8192.0000 max=8817 max_rank=2 min=7529 min_rank=3 imbalance_ratio=1.076294 max_load_reduction=56719 max_load_reduction_pct=0.865463 moved_to_echo=32802 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4034, 62, 0, 0, 0, 0, 0, 0, 924, 0, 4092, 0, 0, 3541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 689, 0, 3110, 2, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8557, 7897, 8196, 8189, 8192, 8121] avg=8192.0000 max=8557 max_rank=2 min=7897 min_rank=3 imbalance_ratio=1.044556 max_load_reduction=56979 max_load_reduction_pct=0.869431 moved_to_echo=32698 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4034, 62, 0, 0, 0, 0, 0, 0, 924, 0, 4095, 0, 0, 2756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 1387, 0, 3110, 2, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 8, 0, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 1, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 7775, 8596, 8193, 8192, 8191, 8204] avg=8192.0000 max=8596 max_rank=3 min=7775 min_rank=2 imbalance_ratio=1.049316 max_load_reduction=56940 max_load_reduction_pct=0.868835 moved_to_echo=32780 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4035, 62, 0, 0, 0, 0, 0, 0, 924, 0, 4094, 0, 0, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1848, 0, 3110, 2, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 4096, 61, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 7290, 9056, 8195, 8184, 8192, 8234] avg=8192.0000 max=9056 max_rank=3 min=7290 min_rank=2 imbalance_ratio=1.105469 max_load_reduction=56480 max_load_reduction_pct=0.861816 moved_to_echo=32805 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4034, 62, 0, 0, 0, 0, 0, 0, 924, 0, 4093, 0, 0, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 710, 0, 3110, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8447, 7918, 8198, 8187, 8192, 8210] avg=8192.0000 max=8447 max_rank=2 min=7918 min_rank=3 imbalance_ratio=1.031128 max_load_reduction=57089 max_load_reduction_pct=0.871109 moved_to_echo=32787 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4034, 62, 0, 0, 0, 0, 0, 0, 924, 0, 4096, 0, 0, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 601, 0, 3110, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8567, 7808, 8185, 8184, 8192, 8216] avg=8192.0000 max=8567 max_rank=2 min=7808 min_rank=3 imbalance_ratio=1.045776 max_load_reduction=56969 max_load_reduction_pct=0.869278 moved_to_echo=32777 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4035, 62, 0, 0, 0, 0, 0, 0, 923, 0, 4096, 0, 0, 2943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1188, 0, 3111, 4, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 10, 6, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 4096, 61, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8193, 7962, 8399, 8191, 8179, 8194, 8234] avg=8192.0000 max=8399 max_rank=3 min=7962 min_rank=2 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32798 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 0, 32768, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 32768, 32768, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[7424, 128, 32768, 0, 0, 25472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 128, 0, 32384, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32384, 0, 0, 0, 128, 0, 0, 32768, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 0, 128, 0, 7936, 0, 24960, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4050, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4076, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 23, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 1, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4070, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 2, 3, 0, 0, 4096, 0, 0, 0, 4096, 3, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4061, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 34, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4066, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 28, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4050, 0, 0, 0, 0, 0, 33, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 14, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4079, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8191, 8189, 8192, 8192, 8193, 8193, 8192] avg=8192.0000 max=8194 max_rank=0 min=8189 min_rank=2 imbalance_ratio=1.000244 max_load_reduction=57342 max_load_reduction_pct=0.874969 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4061, 0, 0, 0, 0, 0, 4, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 14, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4095, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 1, 0, 4079, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8192, 8209, 8193, 8192, 8192, 8191, 8192] avg=8192.0000 max=8209 max_rank=2 min=8175 min_rank=0 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4076, 0, 0, 0, 0, 0, 9, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 14, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 8195, 8192, 8192, 8191, 8187, 8192] avg=8192.0000 max=8195 max_rank=0 min=8187 min_rank=6 imbalance_ratio=1.000366 max_load_reduction=57341 max_load_reduction_pct=0.874954 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4070, 0, 0, 0, 0, 0, 26, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 14, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4095, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 4079, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8192, 8180, 8195, 8192, 8189, 8190, 8192] avg=8192.0000 max=8206 max_rank=0 min=8180 min_rank=2 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4072, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 14, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4081, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8191, 8199, 8192, 8193, 8193, 8193, 8192] avg=8192.0000 max=8199 max_rank=2 min=8183 min_rank=0 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 2, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 14, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8192, 8180, 8192, 8192, 8191, 8195, 8192] avg=8192.0000 max=8202 max_rank=0 min=8180 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=57334 max_load_reduction_pct=0.874847 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4066, 0, 0, 0, 0, 0, 2, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 14, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8192, 8204, 8192, 8192, 8191, 8195, 8192] avg=8192.0000 max=8204 max_rank=2 min=8178 min_rank=0 imbalance_ratio=1.001465 max_load_reduction=57332 max_load_reduction_pct=0.874817 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 0, 0, 1, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 17, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 16, 4, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8194, 8180, 8188, 8191, 8196, 8192, 8192] avg=8192.0000 max=8203 max_rank=0 min=8180 min_rank=2 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32640, 0, 0, 0, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 32640, 0, 0, 0, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 32768, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65536 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 128, 0, 32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65536 slack_slots=4736 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 6, 0, 4051, 0, 3939, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 185, 0, 4096, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 2, 0, 4096, 0, 4096, 4078, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 9, 0, 4081, 0, 3999, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 105, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4, 0, 4096, 0, 4096, 4088, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4059, 0, 3934, 3, 0, 4094, 0, 4096, 1, 0, 0, 1, 4076, 0, 0, 4092, 4096, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 215, 4, 4096, 0, 0, 0, 4096, 8, 4090, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 19, 0, 4096, 0, 4096, 4041, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 9, 0, 4070, 0, 3932, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 198, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 4096, 4077, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 2, 0, 4083, 0, 4074, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 26, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4084, 0, 3936, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 172, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 4096, 0, 4096, 4088, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 13, 0, 4067, 0, 4035, 1, 0, 4095, 0, 4096, 1, 0, 0, 1, 4095, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 95, 0, 4096, 0, 0, 0, 4096, 3, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 3, 0, 4096, 0, 4096, 4067, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4075, 0, 3975, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 127, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 2, 0, 4096, 0, 4096, 4090, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 4051, 0, 3939, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 185, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 2, 4078, 2, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8194, 8192, 8234, 8193, 8203, 8196, 8192] avg=8192.0000 max=8234 max_rank=3 min=8132 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 4070, 0, 3932, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 198, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4078, 2, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8195, 8193, 8249, 8192, 8192, 8177, 8191] avg=8192.0000 max=8249 max_rank=3 min=8147 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 0, 4067, 0, 4035, 1, 0, 136, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 95, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 3, 4078, 2, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8196, 8192, 8149, 8193, 8187, 8186, 8181] avg=8192.0000 max=8252 max_rank=0 min=8149 min_rank=3 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 4081, 0, 3999, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 105, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4, 4078, 2, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8196, 8193, 8161, 8192, 8185, 8182, 8202] avg=8192.0000 max=8225 max_rank=0 min=8161 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4084, 0, 3936, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 172, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 4078, 4, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8191, 8192, 8225, 8192, 8187, 8191, 8202] avg=8192.0000 max=8225 max_rank=3 min=8156 min_rank=0 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4083, 0, 4074, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 26, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 4078, 2, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8196, 8193, 8081, 8192, 8188, 8181, 8210] avg=8192.0000 max=8295 max_rank=0 min=8081 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4059, 0, 3934, 3, 0, 136, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4, 1, 0, 0, 1, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 215, 4, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 19, 4078, 2, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8176, 8189, 8268, 8194, 8204, 8218, 8155] avg=8192.0000 max=8268 max_rank=3 min=8132 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4075, 0, 3975, 0, 0, 143, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 127, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 2, 4079, 2, 6, 0, 0, 0, 0, 0, 0, 17, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8192, 8192, 8169, 8188, 8190, 8205, 8203] avg=8192.0000 max=8205 max_rank=6 min=8169 min_rank=3 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 0, 32768, 128, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 256, 0, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 32640, 0, 31872, 128, 0, 1152, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 0, 0, 1152, 128, 31744, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[712, 0, 53, 0, 62, 0, 2853, 3, 555, 1824, 30, 181, 1136, 1800, 0, 480, 1179, 523, 32, 2550, 0, 612, 250, 2023, 1, 1, 1351, 1927, 3329, 24, 2886, 522, 864, 263, 2126, 268, 259, 17, 3314, 959, 3811, 0, 3490, 2385, 0, 205, 1397, 859, 3466, 380, 2959, 0, 1, 22, 0, 643, 0, 274, 338, 2217, 3577, 694, 2090, 1759] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1136, 0, 9, 8, 16, 1, 3531, 3, 652, 2522, 12, 122, 657, 2144, 0, 454, 664, 179, 19, 2376, 0, 581, 388, 2165, 1, 1, 852, 1407, 3680, 37, 2687, 554, 554, 148, 2367, 548, 615, 13, 3172, 607, 3978, 0, 3617, 2754, 0, 257, 1616, 844, 3702, 378, 2897, 0, 0, 17, 2, 391, 1, 92, 202, 2002, 3544, 864, 2142, 1354] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[822, 16, 36, 2, 25, 0, 3628, 2, 389, 2373, 22, 132, 601, 2249, 0, 444, 1039, 526, 33, 2268, 0, 596, 420, 1928, 0, 25, 1131, 1584, 3643, 53, 2894, 771, 855, 280, 1504, 468, 455, 10, 3183, 759, 3485, 1, 3694, 2396, 2, 219, 1892, 901, 3714, 381, 2755, 0, 1, 44, 2, 327, 0, 164, 255, 2167, 3318, 1363, 1987, 1302] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1096, 5, 48, 39, 68, 0, 3544, 3, 766, 1551, 47, 338, 918, 1574, 0, 481, 560, 412, 23, 2634, 0, 721, 556, 1920, 0, 16, 1458, 1734, 3578, 14, 2762, 496, 705, 302, 2220, 343, 475, 15, 2691, 1169, 3703, 0, 3481, 2179, 12, 252, 1373, 764, 3592, 561, 3246, 0, 11, 78, 0, 383, 0, 119, 425, 2471, 3603, 980, 1211, 1810] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[597, 0, 47, 0, 51, 0, 3255, 0, 699, 2338, 27, 126, 1176, 2407, 0, 411, 601, 638, 27, 2111, 0, 684, 393, 2238, 0, 1, 1205, 1957, 3551, 33, 3007, 756, 761, 148, 1630, 562, 307, 20, 3168, 755, 3544, 0, 3587, 2211, 0, 226, 1931, 824, 3378, 491, 2613, 0, 0, 55, 7, 376, 1, 106, 279, 2048, 3668, 832, 2028, 1644] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1093, 5, 71, 10, 76, 4, 3338, 2, 719, 1927, 39, 172, 617, 1931, 5, 894, 768, 504, 85, 2621, 0, 698, 653, 2599, 4, 2, 968, 1624, 3126, 17, 2739, 493, 938, 164, 1915, 411, 281, 45, 3171, 403, 3537, 3, 3523, 2229, 0, 308, 1739, 507, 3563, 500, 3044, 0, 0, 15, 10, 333, 0, 195, 478, 1945, 3705, 1313, 1281, 2176] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[925, 21, 30, 15, 95, 2, 3255, 3, 382, 1430, 24, 148, 741, 2048, 4, 500, 927, 632, 17, 2847, 0, 771, 417, 2014, 7, 6, 822, 1376, 3186, 5, 2799, 730, 1338, 308, 2066, 171, 205, 39, 3049, 883, 3752, 0, 3494, 2224, 5, 239, 1649, 905, 3338, 326, 3102, 1, 2, 50, 3, 556, 2, 217, 459, 2174, 3168, 1450, 1932, 2250] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[902, 15, 96, 6, 25, 0, 3460, 6, 563, 1970, 21, 141, 1056, 1720, 0, 699, 588, 468, 29, 2254, 0, 1073, 465, 1901, 1, 10, 1415, 1736, 3534, 20, 2721, 573, 695, 240, 1896, 653, 135, 12, 2924, 924, 3380, 0, 3738, 2252, 7, 168, 1908, 1087, 3493, 269, 3156, 4, 2, 29, 0, 444, 0, 102, 445, 1861, 3323, 1105, 1817, 1999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[712, 0, 53, 0, 62, 0, 2853, 3, 3811, 156, 0, 0, 0, 0, 0, 0, 555, 1824, 30, 181, 1136, 1800, 0, 480, 2069, 0, 0, 0, 0, 0, 0, 0, 1179, 523, 32, 2550, 0, 612, 250, 2023, 712, 439, 0, 0, 0, 0, 0, 0, 1, 1, 1351, 1927, 1847, 24, 2886, 522, 0, 0, 0, 0, 0, 0, 0, 0, 864, 263, 2126, 268, 259, 17, 3314, 959, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3025, 2385, 0, 205, 1397, 859, 0, 0, 0, 0, 0, 0, 0, 0, 3466, 380, 2959, 0, 1, 22, 0, 643, 770, 3, 0, 0, 0, 0, 0, 0, 0, 274, 338, 2217, 913, 694, 2090, 1759, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7650, 8075, 8320, 8559, 8532, 7871, 8244, 8285] avg=8192.0000 max=8559 max_rank=3 min=7650 min_rank=0 imbalance_ratio=1.044800 max_load_reduction=56977 max_load_reduction_pct=0.869400 moved_to_echo=32932 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1136, 0, 9, 8, 16, 1, 3531, 3, 3978, 154, 0, 0, 0, 0, 0, 0, 652, 2522, 12, 122, 657, 2144, 0, 454, 2050, 0, 0, 0, 0, 0, 0, 0, 664, 179, 19, 2376, 0, 581, 388, 2165, 787, 434, 0, 0, 0, 0, 0, 0, 1, 1, 852, 1407, 2042, 37, 2687, 554, 0, 0, 0, 0, 0, 0, 0, 0, 554, 148, 2367, 548, 615, 13, 3172, 607, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3135, 2754, 0, 257, 1616, 844, 0, 0, 0, 0, 0, 0, 0, 0, 3702, 378, 2897, 0, 0, 17, 2, 391, 851, 3, 0, 0, 0, 0, 0, 0, 1, 92, 202, 2002, 906, 864, 2142, 1354, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8836, 8613, 7593, 7581, 8503, 8606, 8241, 7563] avg=8192.0000 max=8836 max_rank=0 min=7563 min_rank=7 imbalance_ratio=1.078613 max_load_reduction=56700 max_load_reduction_pct=0.865173 moved_to_echo=32913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[597, 0, 47, 0, 51, 0, 3255, 0, 3544, 160, 0, 0, 0, 0, 0, 0, 699, 2338, 27, 126, 1176, 2407, 0, 411, 2122, 0, 0, 0, 0, 0, 0, 0, 601, 638, 27, 2111, 0, 684, 393, 2238, 759, 450, 0, 0, 0, 0, 0, 0, 0, 1, 1205, 1957, 1970, 33, 3007, 756, 0, 0, 0, 0, 0, 0, 0, 0, 761, 148, 1630, 562, 307, 20, 3168, 755, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 2211, 0, 226, 1931, 824, 0, 0, 0, 0, 0, 0, 0, 0, 3378, 491, 2613, 0, 0, 55, 7, 376, 822, 3, 0, 0, 0, 0, 0, 0, 1, 106, 279, 2048, 936, 832, 2028, 1644, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7654, 9306, 7901, 8929, 7826, 8301, 7745, 7874] avg=8192.0000 max=9306 max_rank=1 min=7654 min_rank=0 imbalance_ratio=1.135986 max_load_reduction=56230 max_load_reduction_pct=0.858002 moved_to_echo=31746 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[822, 16, 36, 2, 25, 0, 3628, 2, 3485, 145, 0, 0, 0, 0, 0, 0, 389, 2373, 22, 132, 601, 2249, 0, 444, 1919, 0, 0, 0, 0, 0, 0, 0, 1039, 526, 33, 2268, 0, 596, 420, 1928, 779, 407, 0, 0, 0, 0, 0, 0, 0, 25, 1131, 1584, 2021, 53, 2894, 771, 0, 0, 0, 0, 0, 0, 0, 0, 855, 280, 1504, 468, 455, 10, 3183, 759, 489, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3202, 2396, 2, 219, 1892, 901, 0, 0, 0, 0, 0, 0, 0, 0, 3714, 381, 2755, 0, 1, 44, 2, 327, 843, 3, 0, 0, 0, 0, 0, 0, 0, 164, 255, 2167, 847, 1363, 1987, 1302, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8129, 7996, 8479, 8003, 8613, 8070, 8085] avg=8192.0000 max=8613 max_rank=5 min=7996 min_rank=2 imbalance_ratio=1.051392 max_load_reduction=56923 max_load_reduction_pct=0.868576 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1096, 5, 48, 39, 68, 0, 3544, 3, 3703, 157, 0, 0, 0, 0, 0, 0, 766, 1551, 47, 338, 918, 1574, 0, 481, 2084, 0, 0, 0, 0, 0, 0, 0, 560, 412, 23, 2634, 0, 721, 556, 1920, 765, 442, 0, 0, 0, 0, 0, 0, 0, 16, 1458, 1734, 1985, 14, 2762, 496, 0, 0, 0, 0, 0, 0, 0, 0, 705, 302, 2220, 343, 475, 15, 2691, 1169, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3017, 2179, 12, 252, 1373, 764, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 561, 3246, 0, 11, 78, 0, 383, 828, 3, 0, 0, 0, 0, 0, 0, 0, 119, 425, 2471, 920, 980, 1211, 1810, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8663, 7759, 8033, 8465, 8381, 7597, 8702, 7936] avg=8192.0000 max=8702 max_rank=6 min=7597 min_rank=5 imbalance_ratio=1.062256 max_load_reduction=56834 max_load_reduction_pct=0.867218 moved_to_echo=32616 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1093, 5, 71, 10, 76, 4, 3338, 2, 3537, 161, 0, 0, 0, 0, 0, 0, 719, 1927, 39, 172, 617, 1931, 5, 894, 2143, 0, 0, 0, 0, 0, 0, 0, 768, 504, 85, 2621, 0, 698, 653, 2599, 668, 454, 0, 0, 0, 0, 0, 0, 4, 2, 968, 1624, 1735, 17, 2739, 493, 0, 0, 0, 0, 0, 0, 0, 0, 938, 164, 1915, 411, 281, 45, 3171, 403, 466, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3054, 2229, 0, 308, 1739, 507, 0, 0, 0, 0, 0, 0, 0, 0, 3563, 500, 3044, 0, 0, 15, 10, 333, 723, 3, 0, 0, 0, 0, 0, 0, 0, 195, 478, 1945, 947, 1313, 1281, 2176, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8447, 9050, 7582, 7794, 7840, 8191, 8335] avg=8192.0000 max=9050 max_rank=2 min=7582 min_rank=3 imbalance_ratio=1.104736 max_load_reduction=56486 max_load_reduction_pct=0.861908 moved_to_echo=32160 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[925, 21, 30, 15, 95, 2, 3255, 3, 3752, 138, 0, 0, 0, 0, 0, 0, 382, 1430, 24, 148, 741, 2048, 4, 500, 1833, 0, 0, 0, 0, 0, 0, 0, 927, 632, 17, 2847, 0, 771, 417, 2014, 681, 388, 0, 0, 0, 0, 0, 0, 7, 6, 822, 1376, 1768, 5, 2799, 730, 0, 0, 0, 0, 0, 0, 0, 0, 1338, 308, 2066, 171, 205, 39, 3049, 883, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 2224, 5, 239, 1649, 905, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 326, 3102, 1, 2, 50, 3, 556, 737, 3, 0, 0, 0, 0, 0, 0, 2, 217, 459, 2174, 809, 1450, 1932, 2250, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 7110, 8694, 7513, 8521, 8051, 8118, 9293] avg=8192.0000 max=9293 max_rank=7 min=7110 min_rank=1 imbalance_ratio=1.134399 max_load_reduction=56243 max_load_reduction_pct=0.858200 moved_to_echo=33983 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[902, 15, 96, 6, 25, 0, 3460, 6, 3380, 149, 0, 0, 0, 0, 0, 0, 563, 1970, 21, 141, 1056, 1720, 0, 699, 1927, 0, 0, 0, 0, 0, 0, 0, 588, 468, 29, 2254, 0, 1073, 465, 1901, 760, 411, 0, 0, 0, 0, 0, 0, 1, 10, 1415, 1736, 1952, 20, 2721, 573, 0, 0, 0, 0, 0, 0, 0, 0, 695, 240, 1896, 653, 135, 12, 2924, 924, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3235, 2252, 7, 168, 1908, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 3493, 269, 3156, 4, 2, 29, 0, 444, 822, 6, 0, 0, 0, 0, 0, 0, 0, 102, 445, 1861, 836, 1105, 1817, 1999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8039, 8097, 7949, 8428, 7976, 8657, 8225, 8165] avg=8192.0000 max=8657 max_rank=5 min=7949 min_rank=2 imbalance_ratio=1.056763 max_load_reduction=56879 max_load_reduction_pct=0.867905 moved_to_echo=33023 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4736, 16000, 256, 1408, 6912, 16000, 128, 4480, 16256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 1280, 2944, 16896, 7168, 8704, 14592, 14336, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 24832, 18688, 128, 1920, 13568, 6784, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6400, 3968, 384, 19712, 0, 5760, 3584, 16896, 6016, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 9216, 13440, 15360, 256, 22528, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6784, 1920, 15744, 3456, 2816, 256, 24704, 6528, 3840, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[28288, 3328, 23808, 128, 128, 384, 128, 3456, 6400, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[7296, 128, 512, 128, 512, 128, 26880, 128, 29312, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 3, 2, 0, 0, 4094, 0, 4092, 0, 0, 0, 199, 3, 1, 0, 0, 1, 0, 1, 8, 0, 0, 143, 4094, 4094, 3, 0, 4096, 3, 2, 18, 1, 4072, 1743, 0, 2142, 4095, 1, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4094, 1, 4095, 0, 4095, 4095, 3976, 0, 1, 0, 1, 4074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 9, 4, 0, 4096, 0, 4069, 0, 0, 0, 168, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 50, 4094, 4096, 0, 0, 4092, 0, 0, 8, 0, 4095, 2456, 0, 1596, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 4096, 3934, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 4015, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 86, 4087, 4096, 1, 0, 4095, 0, 0, 10, 0, 4089, 1910, 0, 2159, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 4096, 3994, 2, 0, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4092, 1, 3, 13, 0, 4092, 1, 2847, 0, 2, 0, 114, 0, 0, 0, 2, 0, 0, 0, 3, 14, 2, 78, 4089, 4096, 3, 0, 4091, 0, 0, 4, 0, 4095, 3358, 44, 1935, 4083, 1, 2, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 4087, 0, 4096, 4094, 3914, 3, 0, 2, 0, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 5, 0, 2, 4092, 12, 4, 11, 0, 4084, 0, 3793, 14, 0, 1, 96, 0, 2, 0, 2, 2, 3, 0, 0, 1, 2, 227, 4086, 4089, 0, 0, 4094, 0, 10, 10, 6, 4073, 2643, 57, 1579, 4094, 2, 0, 10, 2, 1, 0, 4084, 0, 0, 8, 0, 4090, 7, 4090, 0, 4088, 4085, 3921, 1, 1, 1, 2, 4049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 5, 0, 4096, 0, 4007, 0, 0, 0, 48, 12, 0, 0, 0, 0, 0, 0, 0, 0, 6, 147, 4093, 4096, 0, 0, 4093, 0, 0, 7, 0, 4078, 1210, 0, 3074, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4094, 4096, 3801, 0, 0, 0, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 5, 0, 3, 0, 4095, 0, 4070, 0, 0, 0, 577, 2, 0, 0, 0, 0, 0, 1, 3, 12, 0, 20, 4096, 4092, 2, 0, 4096, 1, 0, 1, 0, 4060, 863, 0, 2708, 4096, 2, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 0, 4094, 1, 4095, 4092, 4072, 0, 0, 0, 1, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4091, 3, 9, 0, 11, 4094, 0, 4050, 0, 2, 0, 69, 0, 0, 0, 5, 0, 0, 0, 1, 0, 5, 109, 4090, 4095, 0, 0, 4089, 0, 0, 28, 3, 4092, 1970, 3, 1990, 4091, 0, 0, 5, 0, 0, 0, 4095, 0, 3, 0, 0, 4089, 1, 4089, 0, 4094, 4093, 4075, 0, 0, 4, 0, 4083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 3949, 106, 27, 5, 0, 0, 0, 0, 0, 4096, 0, 4015, 0, 0, 0, 120, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 86, 4087, 0, 1, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4006, 1910, 0, 2159, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 147, 3994, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8284, 8197, 8163, 8085, 8193, 8193, 8237] avg=8192.0000 max=8284 max_rank=1 min=8085 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32708 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 9, 4, 3949, 106, 27, 4, 0, 0, 0, 0, 0, 4096, 0, 4069, 0, 0, 0, 168, 53, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 50, 4094, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4012, 2456, 0, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 147, 3934, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8386, 8189, 8131, 8072, 8193, 8194, 8176] avg=8192.0000 max=8386 max_rank=1 min=8072 min_rank=4 imbalance_ratio=1.023682 max_load_reduction=57150 max_load_reduction_pct=0.872040 moved_to_echo=32635 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 3, 2, 0, 3948, 106, 26, 4, 0, 0, 0, 0, 0, 4094, 0, 4092, 0, 0, 0, 199, 52, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 1, 0, 1, 8, 4094, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 143, 4094, 0, 3, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 18, 1, 3991, 1743, 0, 2142, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 1, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 147, 3976, 0, 1, 0, 1, 4074, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8437, 8199, 8230, 7900, 8194, 8192, 8199] avg=8192.0000 max=8437 max_rank=1 min=7900 min_rank=4 imbalance_ratio=1.029907 max_load_reduction=57099 max_load_reduction_pct=0.871262 moved_to_echo=32485 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 5, 0, 3, 3945, 106, 26, 4, 0, 0, 0, 0, 0, 4095, 0, 4070, 0, 0, 0, 577, 52, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 3, 4092, 4091, 0, 0, 0, 0, 0, 0, 12, 0, 20, 4096, 0, 2, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3979, 863, 0, 2708, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 147, 4072, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8794, 8189, 8120, 7552, 8196, 8192, 8307] avg=8192.0000 max=8794 max_rank=1 min=7552 min_rank=4 imbalance_ratio=1.073486 max_load_reduction=56742 max_load_reduction_pct=0.865814 moved_to_echo=32247 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4092, 1, 3, 13, 3947, 106, 27, 5, 0, 0, 0, 0, 0, 4092, 1, 2847, 0, 2, 0, 114, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 4096, 4091, 0, 0, 0, 0, 0, 0, 14, 2, 78, 4089, 0, 3, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4012, 3358, 44, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 2, 0, 1, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 147, 3914, 3, 0, 2, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 7109, 8192, 8171, 9353, 8180, 8185, 8151] avg=8192.0000 max=9353 max_rank=4 min=7109 min_rank=1 imbalance_ratio=1.141724 max_load_reduction=56183 max_load_reduction_pct=0.857285 moved_to_echo=33869 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 2, 4092, 12, 4, 11, 3939, 106, 26, 4, 0, 0, 0, 0, 0, 4084, 0, 3793, 14, 0, 1, 96, 52, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 2, 3, 0, 0, 4089, 4084, 0, 0, 0, 0, 0, 0, 1, 2, 227, 4086, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 10, 6, 3992, 2643, 57, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 10, 2, 1, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4090, 7, 4090, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 146, 3921, 1, 1, 1, 2, 4049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8040, 8182, 8304, 8297, 8194, 8197, 8121] avg=8192.0000 max=8304 max_rank=3 min=8040 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 5, 3949, 106, 26, 4, 0, 0, 0, 0, 0, 4096, 0, 4007, 0, 0, 0, 48, 52, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 6, 147, 4093, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3997, 1210, 0, 3074, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 147, 3801, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8203, 8198, 8233, 8288, 8193, 8194, 8041] avg=8192.0000 max=8288 max_rank=4 min=8041 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32716 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4091, 3, 9, 0, 3951, 109, 31, 5, 0, 0, 0, 0, 11, 4094, 0, 4050, 0, 2, 0, 69, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 4095, 4089, 0, 0, 0, 0, 0, 0, 0, 5, 109, 4090, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3, 3995, 1970, 3, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 5, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4089, 1, 4089, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 142, 4075, 0, 0, 4, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8283, 8190, 8184, 7989, 8193, 8189, 8304] avg=8192.0000 max=8304 max_rank=7 min=7989 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32675 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 896, 32768, 0, 128, 0, 32000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 1280, 31744, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32000, 16256, 128, 17280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 128, 31616, 896, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 30976, 128, 128, 128, 1408, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 4093, 1, 1, 4094, 0, 1, 555, 1, 1, 4094, 3423, 1, 4090, 0, 4093, 4089, 0, 0, 0, 0, 0, 694, 0, 1, 4095, 4, 4095, 1, 1, 0, 4095, 0, 1, 8, 0, 4003, 0, 4095, 1, 1, 1, 3, 1, 4095, 0, 1, 3611, 4095, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 96, 0, 0, 4084, 3627, 0, 4096, 0, 4094, 4091, 0, 0, 0, 0, 0, 766, 1, 0, 4096, 1, 4096, 0, 0, 0, 4096, 0, 0, 12, 0, 4064, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 3757, 4077, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 293, 0, 0, 4043, 3975, 0, 4090, 0, 4096, 4060, 0, 0, 0, 0, 0, 234, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 47, 0, 4060, 0, 4096, 0, 0, 0, 11, 0, 4096, 0, 0, 3769, 4082, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 115, 0, 0, 4090, 3832, 0, 4095, 0, 4096, 4093, 0, 0, 0, 0, 0, 452, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 14, 0, 3923, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 3985, 4068, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 65, 0, 0, 4068, 3548, 0, 4090, 6, 4096, 4052, 0, 0, 0, 0, 0, 658, 0, 5, 4095, 0, 4096, 0, 0, 16, 4087, 10, 0, 15, 0, 4084, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 3990, 4076, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 1, 4095, 1, 0, 37, 0, 0, 4086, 3506, 0, 4092, 0, 4096, 4006, 0, 0, 1, 1, 3, 1326, 2, 1, 4096, 0, 4095, 0, 0, 0, 4095, 7, 0, 18, 0, 3995, 0, 4095, 0, 1, 1, 5, 1, 4095, 0, 0, 3508, 4076, 1, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4, 0, 5, 0, 0, 0, 4091, 0, 5, 0, 21, 0, 4092, 3, 3, 4085, 0, 1, 133, 3, 2, 4075, 3434, 0, 4082, 0, 4081, 4045, 6, 0, 0, 0, 0, 556, 12, 0, 4091, 11, 4093, 2, 3, 3, 4091, 15, 2, 25, 0, 4074, 4, 4093, 2, 4, 3, 0, 3, 4092, 0, 5, 4077, 4088, 0, 2, 13, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 4, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 340, 0, 0, 4078, 3515, 1, 4009, 11, 4094, 4024, 0, 0, 0, 0, 0, 951, 0, 6, 4096, 3, 4096, 0, 0, 2, 4088, 3, 0, 57, 0, 3746, 0, 4090, 0, 0, 0, 29, 0, 4094, 0, 0, 3818, 4091, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4094, 0, 4042, 51, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 1, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 555, 1, 1, 4094, 3423, 1, 301, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 51, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, 0, 1, 4095, 4, 3387, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 8, 0, 4003, 64, 6, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 3, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3611, 4095, 0, 0, 0, 1, 292, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8189, 8377, 8230, 8182, 8179, 8191, 8000] avg=8192.0000 max=8377 max_rank=2 min=8000 min_rank=7 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32552 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 4043, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 4084, 3627, 0, 301, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 51, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 766, 1, 0, 4096, 1, 3388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 4064, 64, 6, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 4077, 0, 0, 1, 0, 292, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8191, 8108, 8238, 8252, 8243, 8187, 8127] avg=8192.0000 max=8252 max_rank=4 min=8108 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 4096, 0, 4045, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 4090, 3832, 0, 301, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 51, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 4096, 0, 3388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 3923, 64, 6, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 4068, 0, 0, 0, 0, 292, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8191, 8338, 8239, 7936, 8104, 8187, 8345] avg=8192.0000 max=8345 max_rank=7 min=7936 min_rank=4 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 4096, 0, 4045, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 4043, 3975, 0, 301, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 51, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 4096, 0, 3388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 47, 0, 4060, 64, 6, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3769, 4082, 0, 0, 0, 0, 292, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8191, 8612, 8201, 7718, 8274, 8197, 8143] avg=8192.0000 max=8612 max_rank=2 min=7718 min_rank=4 imbalance_ratio=1.051270 max_load_reduction=56924 max_load_reduction_pct=0.868591 moved_to_echo=32332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 4095, 0, 4045, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 37, 0, 0, 4086, 3506, 0, 301, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 51, 4006, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1326, 2, 1, 4096, 0, 3387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 18, 0, 3995, 64, 6, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 5, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3508, 4076, 1, 0, 2, 0, 292, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8190, 7931, 8151, 8815, 8186, 8192, 7879] avg=8192.0000 max=8815 max_rank=4 min=7879 min_rank=7 imbalance_ratio=1.076050 max_load_reduction=56721 max_load_reduction_pct=0.865494 moved_to_echo=33072 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 4045, 51, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 4068, 3548, 0, 301, 0, 0, 0, 0, 0, 0, 0, 4090, 6, 51, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 0, 5, 4095, 0, 3388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4087, 10, 0, 15, 0, 4084, 64, 6, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 4076, 0, 0, 0, 0, 292, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 7982, 8199, 8146, 8283, 8184, 8358] avg=8192.0000 max=8358 max_rank=7 min=7982 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 5, 0, 0, 0, 4091, 0, 4030, 51, 0, 0, 0, 0, 0, 0, 5, 0, 21, 0, 4091, 3, 3, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 133, 3, 2, 4075, 3434, 0, 301, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 51, 4045, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 12, 0, 4091, 11, 3385, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 4091, 15, 2, 25, 0, 4074, 64, 6, 1, 0, 0, 0, 0, 0, 4, 4093, 2, 4, 3, 0, 3, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4077, 4088, 0, 2, 13, 1, 292, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8208, 7949, 8184, 8057, 8284, 8195, 8478] avg=8192.0000 max=8478 max_rank=7 min=7949 min_rank=2 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=33014 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 4, 4096, 0, 4044, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 4078, 3515, 1, 305, 0, 0, 0, 0, 0, 0, 0, 4009, 11, 50, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 6, 4096, 3, 3374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4088, 3, 0, 57, 0, 3746, 70, 10, 7, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 29, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 4091, 0, 0, 2, 0, 295, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8184, 8239, 8094, 8430, 7983, 8203, 8206] avg=8192.0000 max=8430 max_rank=4 min=7983 min_rank=5 imbalance_ratio=1.029053 max_load_reduction=57106 max_load_reduction_pct=0.871368 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 1664, 128, 128, 32640, 28928, 128, 2432, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 0, 32384, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 512, 32512, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 256, 0, 32000, 640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 5760, 128, 128, 32768, 128, 27136, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 30592, 32768, 128, 128, 128, 128, 2432, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 33, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 4073, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4086, 4096, 4096, 4096, 0, 3, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 4095, 0, 0, 0, 4082, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 14, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 16, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 32, 4090, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 4096, 5, 0, 0, 4096, 4061, 4096, 0, 0, 0, 3964, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4092, 4096, 4094, 4096, 0, 133, 4092, 5, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 2, 0, 1, 1, 10, 5, 3, 0, 1, 1, 4092, 0, 0, 4091, 0, 2, 0, 1, 1, 0, 0, 0, 1, 0, 4086, 0, 1, 0, 4095, 0, 1, 1, 4095, 4095, 4095, 0, 0, 0, 4095, 1, 4095, 0, 0, 0, 4095, 4095, 0, 1, 4092, 4093, 4095, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 1, 4096, 0, 22, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4083, 0, 0, 0, 4095, 1, 0, 0, 4095, 4081, 4094, 1, 0, 0, 4083, 0, 4096, 0, 1, 0, 4096, 4096, 0, 0, 4087, 4095, 4095, 4096, 0, 26, 4095, 1, 1, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4, 7, 0, 3, 0, 5, 6, 13, 0, 1, 0, 2, 0, 4078, 0, 0, 4088, 0, 0, 4, 6, 4, 4, 10, 0, 1, 0, 4071, 1, 5, 2, 4092, 0, 5, 6, 4091, 4092, 4085, 0, 1, 9, 4053, 4, 4089, 0, 0, 0, 4089, 4091, 0, 5, 4090, 4069, 4091, 4090, 2, 45, 4088, 27, 2, 0, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 70, 4092, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 3988, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4049, 4096, 4091, 4096, 0, 119, 4096, 8, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 50, 4088, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 4089, 6, 0, 0, 4096, 3997, 4096, 1, 0, 0, 3971, 0, 4096, 0, 0, 0, 4096, 4096, 6, 0, 4083, 4096, 4090, 4096, 0, 212, 4087, 8, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 4096, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 4082, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 1, 0, 0, 0, 0, 0, 4018, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8170, 8190, 8211, 8210, 8235, 8206, 8133] avg=8192.0000 max=8235 max_rank=5 min=8133 min_rank=7 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32784 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 7, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 33, 4071, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 4082, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 4096, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 1, 0, 0, 0, 0, 0, 4018, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8200, 8190, 8203, 8211, 8226, 8196, 8122] avg=8192.0000 max=8226 max_rank=5 min=8122 min_rank=7 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 2, 0, 1, 1, 10, 4095, 4084, 0, 0, 0, 0, 0, 0, 5, 3, 0, 1, 1, 4092, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2, 0, 1, 1, 0, 0, 4081, 11, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 0, 1, 0, 4095, 21, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 20, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4095, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 4093, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4017, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8172, 8188, 8204, 8212, 8248, 8200, 8117] avg=8192.0000 max=8248 max_rank=5 min=8117 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32777 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 2, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 22, 4070, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4081, 11, 1, 0, 0, 0, 0, 0, 1, 0, 0, 4083, 0, 0, 0, 4095, 21, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 20, 4081, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 1, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4095, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4095, 1, 1, 0, 1, 0, 0, 4018, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8189, 8187, 8200, 8197, 8237, 8196, 8146] avg=8192.0000 max=8237 max_rank=5 min=8146 min_rank=7 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32776 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 16, 0, 0, 8, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 32, 4071, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 0, 0, 4080, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 20, 4061, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 4096, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 4092, 5, 0, 0, 0, 0, 0, 4018, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8199, 8188, 8191, 8182, 8117, 8202, 8252] avg=8192.0000 max=8252 max_rank=7 min=8117 min_rank=5 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 70, 4071, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 9, 0, 0, 0, 0, 4077, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 4096, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 4096, 8, 0, 0, 0, 0, 0, 4018, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8237, 8190, 8170, 8211, 8141, 8159, 8245] avg=8192.0000 max=8245 max_rank=7 min=8141 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 7, 0, 3, 0, 5, 6, 13, 4091, 4079, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 4078, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4, 6, 4, 4, 10, 4077, 11, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4071, 1, 5, 2, 4092, 20, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 20, 4092, 4085, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4053, 4, 4089, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4090, 4069, 14, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4088, 27, 2, 0, 0, 0, 5, 4011, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8147, 8205, 8192, 8209, 8213, 8180, 8182] avg=8192.0000 max=8213 max_rank=5 min=8147 min_rank=1 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32784 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 10, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 50, 4076, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 16, 0, 0, 0, 0, 4078, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 4089, 22, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4, 3997, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 4096, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4083, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 4087, 8, 0, 0, 0, 0, 0, 4022, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8222, 8198, 8165, 8104, 8119, 8197, 8339] avg=8192.0000 max=8339 max_rank=7 min=8104 min_rank=4 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 32768, 128, 128, 0, 128, 0, 128, 32256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32384, 128, 32768, 0, 128, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 256, 32640, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32640, 128, 128, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 128, 128, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 32768, 0, 256, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 4096, 4021, 0, 0, 0, 395, 0, 0, 23, 4096, 0, 0, 4096, 4096, 1244, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 37, 0, 0, 3, 0, 2475, 0, 0, 0, 4096, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4076, 0, 0, 0, 2440, 0, 0, 0, 4096, 0, 0, 4096, 4096, 15, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 1567, 0, 0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[2, 4095, 0, 0, 0, 0, 1, 1, 4095, 0, 4094, 1, 1, 1, 0, 0, 0, 0, 1, 0, 4095, 1, 4095, 1, 4094, 4095, 0, 0, 0, 3131, 0, 0, 2, 4095, 1, 0, 4095, 4095, 23, 1, 4092, 4095, 4095, 1, 0, 1, 0, 0, 0, 114, 1, 0, 0, 0, 827, 1, 0, 1, 4095, 4095, 0, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4075, 0, 0, 0, 2136, 0, 0, 0, 4096, 0, 0, 4096, 4096, 567, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 228, 0, 0, 8, 0, 1179, 0, 0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4091, 0, 4096, 4004, 0, 1, 1, 2546, 0, 0, 4, 4096, 4, 0, 4095, 4096, 27, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 310, 0, 0, 14, 0, 1299, 0, 0, 0, 4095, 4089, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[6, 4089, 3, 0, 0, 0, 6, 5, 4092, 0, 4089, 6, 6, 5, 0, 1, 0, 0, 5, 0, 4090, 4, 4079, 4, 4090, 4073, 0, 1, 8, 1400, 0, 0, 16, 4089, 0, 0, 4090, 4090, 348, 2, 4082, 4085, 4088, 6, 0, 5, 6, 0, 3, 11, 4, 0, 44, 0, 2303, 4, 0, 7, 4091, 4088, 0, 6, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4079, 0, 4096, 4057, 0, 0, 2, 1542, 0, 0, 27, 4096, 17, 0, 4096, 4096, 616, 0, 4096, 4091, 4096, 0, 0, 0, 0, 0, 0, 142, 0, 0, 18, 0, 1827, 0, 1, 0, 4094, 4070, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4055, 0, 4091, 4033, 0, 0, 1, 1280, 0, 0, 63, 4096, 16, 5, 4096, 4096, 292, 0, 4096, 4090, 4096, 0, 0, 0, 0, 0, 0, 72, 0, 0, 35, 0, 2467, 0, 0, 0, 4096, 4080, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 1880, 1817, 395, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 2279, 4075, 0, 0, 0, 2136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3685, 0, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 8, 0, 1179, 0, 4096, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8193, 8202, 8490, 8348, 8195, 7723, 8197] avg=8192.0000 max=8490 max_rank=3 min=7723 min_rank=6 imbalance_ratio=1.036377 max_load_reduction=57046 max_load_reduction_pct=0.870453 moved_to_echo=32463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 4095, 0, 0, 0, 0, 1, 1, 1880, 1816, 395, 0, 0, 0, 0, 0, 4095, 0, 4094, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 4095, 1, 10, 0, 0, 0, 0, 0, 0, 0, 2278, 4095, 0, 0, 0, 3131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 1, 0, 3684, 0, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4095, 4, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 1, 0, 0, 0, 827, 1, 4095, 2211, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4095, 0, 1, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8193, 8203, 9504, 7806, 8193, 7249, 8198] avg=8192.0000 max=9504 max_rank=3 min=7249 min_rank=6 imbalance_ratio=1.160156 max_load_reduction=56032 max_load_reduction_pct=0.854980 moved_to_echo=31446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 1881, 1817, 395, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 2279, 4076, 0, 0, 0, 2440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3685, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 1567, 0, 4096, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8193, 8202, 8795, 7796, 8196, 7968, 8197] avg=8192.0000 max=8795 max_rank=3 min=7796 min_rank=4 imbalance_ratio=1.073608 max_load_reduction=56741 max_load_reduction_pct=0.865799 moved_to_echo=32157 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 1881, 1817, 395, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 10, 0, 0, 0, 0, 0, 0, 0, 2279, 4021, 0, 0, 0, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4096, 0, 0, 3685, 0, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 3, 0, 2475, 0, 4096, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8193, 8197, 6695, 9048, 8196, 8822, 8196] avg=8192.0000 max=9048 max_rank=4 min=6695 min_rank=3 imbalance_ratio=1.104492 max_load_reduction=56488 max_load_reduction_pct=0.861938 moved_to_echo=34262 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 1881, 1817, 395, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4091, 0, 10, 0, 0, 0, 0, 0, 0, 0, 2279, 4004, 0, 1, 1, 2546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4, 0, 3684, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 14, 0, 1299, 0, 4096, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4089, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8189, 8196, 8831, 7815, 8195, 7930, 8190] avg=8192.0000 max=8831 max_rank=3 min=7815 min_rank=4 imbalance_ratio=1.078003 max_load_reduction=56705 max_load_reduction_pct=0.865250 moved_to_echo=32130 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 1882, 1817, 395, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4079, 0, 10, 0, 0, 0, 0, 0, 0, 0, 2279, 4057, 0, 0, 2, 1542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4096, 17, 0, 3685, 0, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 18, 0, 1827, 0, 4096, 2214, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 4070, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8187, 8184, 7880, 8441, 8187, 8297, 8170] avg=8192.0000 max=8441 max_rank=4 min=7880 min_rank=3 imbalance_ratio=1.030396 max_load_reduction=57095 max_load_reduction_pct=0.871201 moved_to_echo=33095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 4089, 3, 0, 0, 0, 6, 5, 1877, 1815, 395, 0, 0, 0, 0, 0, 4092, 0, 4089, 6, 6, 5, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4090, 4, 4079, 4, 9, 0, 0, 0, 0, 0, 0, 0, 2275, 4073, 0, 1, 8, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4089, 0, 0, 3681, 0, 348, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4085, 4, 6, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 11, 4, 0, 44, 0, 2303, 4, 4090, 2207, 0, 0, 0, 0, 0, 0, 0, 7, 4091, 4088, 0, 6, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8200, 8191, 7757, 8136, 8188, 8666, 8202] avg=8192.0000 max=8666 max_rank=6 min=7757 min_rank=3 imbalance_ratio=1.057861 max_load_reduction=56870 max_load_reduction_pct=0.867767 moved_to_echo=33192 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 0, 0, 0, 1885, 1821, 401, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4055, 0, 11, 0, 0, 0, 0, 0, 0, 0, 2270, 4033, 0, 0, 1, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 4096, 16, 5, 3674, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 35, 0, 2467, 0, 4096, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 4, 0, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8188, 8161, 7584, 8146, 8186, 8881, 8186] avg=8192.0000 max=8881 max_rank=6 min=7584 min_rank=3 imbalance_ratio=1.084106 max_load_reduction=56655 max_load_reduction_pct=0.864487 moved_to_echo=33399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 1024, 128, 0, 128, 0, 13952, 128, 32768, 17792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[18304, 32512, 0, 128, 128, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 32768, 128, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 32768, 128, 128, 29568, 0, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 128, 128, 15104, 14592, 3200, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 397, 0, 0, 0, 0, 18, 0, 0, 4096, 4096, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4096, 4045, 4096, 0, 0, 0, 4096, 0, 0, 3734, 4093, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 495, 0, 0, 0, 0, 26, 0, 0, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4047, 4096, 0, 0, 0, 4096, 0, 0, 3624, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4095, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 4088, 3, 0, 4, 0, 808, 2, 0, 0, 7, 9, 1, 0, 4086, 4088, 7, 2, 0, 4062, 6, 20, 7, 4089, 4, 30, 1, 6, 4, 0, 4081, 4089, 5, 1, 5, 0, 4087, 3896, 4087, 7, 4, 3, 4089, 0, 8, 3468, 4064, 8, 3, 0, 4089, 1, 4, 4089, 6, 0, 7, 7, 11, 0, 4081, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 0, 0, 1, 0, 582, 0, 0, 0, 1, 38, 0, 0, 4095, 4095, 1, 1, 0, 4078, 0, 0, 1, 4095, 0, 27, 1, 1, 1, 1, 4095, 4095, 0, 0, 1, 0, 4095, 4044, 4095, 1, 0, 0, 4095, 0, 1, 3500, 4094, 1, 1, 0, 4095, 0, 0, 4095, 1, 12, 1, 1, 6, 0, 4092, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 169, 0, 0, 0, 0, 46, 0, 0, 4096, 4096, 0, 0, 0, 4090, 0, 6, 0, 4096, 0, 8, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 4096, 4067, 4096, 0, 0, 0, 4096, 0, 0, 3911, 4093, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 1, 1, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 678, 0, 0, 0, 0, 55, 0, 0, 4096, 4096, 0, 0, 0, 4081, 0, 14, 0, 4096, 0, 19, 0, 0, 0, 0, 4085, 4096, 0, 0, 0, 0, 4096, 3953, 4096, 0, 0, 0, 4096, 0, 0, 3520, 4067, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 7, 1, 4096, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 928, 0, 0, 0, 0, 60, 0, 0, 4096, 4096, 0, 0, 0, 4081, 0, 20, 0, 4096, 0, 10, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 4096, 3645, 4096, 0, 0, 0, 4096, 0, 0, 3627, 4004, 0, 0, 0, 4096, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 124, 0, 0, 0, 0, 73, 0, 0, 4096, 4096, 0, 0, 0, 4078, 0, 14, 0, 4096, 0, 18, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 4096, 3936, 4096, 0, 0, 0, 4096, 7, 0, 4054, 4082, 0, 0, 0, 4090, 0, 0, 4096, 0, 1, 0, 0, 11, 2, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 397, 3566, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 3954, 95, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4045, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 3734, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8170, 8190, 8186, 8283, 8262, 8192, 8187] avg=8192.0000 max=8283 max_rank=4 min=8066 min_rank=0 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32924 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 495, 3566, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 4096, 3954, 95, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4047, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 3624, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8178, 8192, 8185, 8285, 8154, 8192, 8186] avg=8192.0000 max=8285 max_rank=4 min=8154 min_rank=5 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4095, 0, 0, 1, 0, 582, 3565, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 0, 0, 4095, 3953, 95, 7, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 4078, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 27, 1, 1, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4044, 142, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 435, 0, 1, 3500, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 1, 6, 0, 4092, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8189, 8178, 8212, 8282, 8031, 8193, 8200] avg=8192.0000 max=8282 max_rank=4 min=8031 min_rank=5 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32706 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 169, 3566, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 4096, 3954, 95, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 8, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 3911, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7838, 8198, 8194, 8184, 8305, 8439, 8191, 8187] avg=8192.0000 max=8439 max_rank=5 min=7838 min_rank=0 imbalance_ratio=1.030151 max_load_reduction=57097 max_load_reduction_pct=0.871231 moved_to_echo=33122 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4088, 3, 0, 4, 0, 808, 3560, 7, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 9, 1, 0, 4086, 3945, 94, 7, 0, 0, 0, 0, 0, 4088, 7, 2, 0, 4062, 6, 20, 7, 2, 0, 0, 0, 0, 0, 0, 0, 4080, 4, 30, 1, 6, 4, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 5, 0, 4087, 3896, 142, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 3, 435, 0, 8, 3468, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 4089, 1, 4, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 11, 0, 4081, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8472, 8151, 8194, 8206, 8136, 7989, 8200, 8188] avg=8192.0000 max=8472 max_rank=0 min=7989 min_rank=5 imbalance_ratio=1.034180 max_load_reduction=57064 max_load_reduction_pct=0.870728 moved_to_echo=32513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 678, 3566, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 4096, 3954, 95, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4081, 0, 14, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 19, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3953, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 3520, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4096, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8347, 8208, 8193, 8191, 8191, 8022, 8191, 8193] avg=8192.0000 max=8347 max_rank=0 min=8022 min_rank=5 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32597 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 928, 3566, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 4096, 3954, 95, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4081, 0, 20, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 10, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3645, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 3627, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8597, 8212, 8199, 8190, 7883, 8066, 8192, 8197] avg=8192.0000 max=8597 max_rank=0 min=7883 min_rank=4 imbalance_ratio=1.049438 max_load_reduction=56939 max_load_reduction_pct=0.868820 moved_to_echo=32338 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 124, 3570, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 4096, 3957, 96, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4078, 0, 14, 0, 7, 1, 0, 0, 0, 0, 0, 0, 4078, 0, 18, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3936, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 430, 7, 0, 4054, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 2, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7801, 8230, 8196, 8182, 8171, 8573, 8185, 8198] avg=8192.0000 max=8573 max_rank=5 min=7801 min_rank=0 imbalance_ratio=1.046509 max_load_reduction=56963 max_load_reduction_pct=0.869186 moved_to_echo=33127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 0, 32768, 31744, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 4224, 28544, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 0, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 384, 128, 0, 32768, 31744, 768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 3584, 128, 128, 29440, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4074, 0, 0, 4095, 0, 4096, 4096, 352, 0, 0, 61, 0, 3607, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3812, 0, 2, 0, 0, 4055, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 2445, 0, 0, 0, 2072, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[8, 0, 9, 7, 4061, 8, 0, 4090, 0, 4089, 4088, 324, 5, 0, 0, 7, 2490, 0, 6, 0, 0, 0, 2, 0, 0, 1, 0, 2, 6, 4087, 7, 0, 4, 5, 0, 7, 7, 4078, 4090, 0, 2, 4031, 3, 3, 3, 2, 4062, 7, 4081, 0, 4089, 6, 0, 4088, 4087, 2339, 0, 6, 6, 3136, 4, 9, 0, 4084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 2, 1, 4089, 0, 0, 4095, 0, 4095, 4095, 241, 0, 1, 9, 1, 4083, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 1, 1, 0, 1, 1, 4095, 4095, 0, 0, 4049, 0, 0, 0, 0, 4018, 1, 4088, 0, 4095, 1, 0, 4095, 4095, 689, 0, 1, 1, 3303, 0, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4057, 0, 0, 4096, 0, 4095, 4096, 178, 0, 0, 63, 0, 3474, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4022, 0, 0, 0, 0, 3986, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 1397, 0, 0, 0, 3300, 0, 3, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 0, 5, 4086, 0, 4080, 4096, 121, 0, 0, 21, 0, 3892, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4093, 4088, 0, 0, 3992, 4, 0, 0, 0, 3885, 0, 4091, 0, 4091, 0, 0, 4096, 4096, 2243, 2, 0, 0, 2251, 0, 11, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 0, 4091, 4096, 125, 0, 0, 52, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4086, 0, 0, 0, 0, 4063, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 1252, 0, 0, 0, 2744, 0, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 1, 4088, 0, 4085, 4096, 39, 0, 0, 62, 0, 3994, 1, 0, 2, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4095, 4093, 0, 0, 3964, 1, 0, 0, 0, 4069, 0, 4096, 1, 4095, 0, 0, 4096, 4096, 1006, 0, 0, 0, 3263, 0, 3, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4081, 0, 6, 4086, 0, 4076, 4096, 42, 0, 0, 55, 0, 3979, 0, 0, 6, 0, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 4096, 4089, 0, 0, 4092, 5, 1, 0, 0, 4091, 0, 4091, 0, 4095, 0, 0, 4096, 4090, 582, 6, 0, 0, 3556, 0, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 1, 4089, 0, 0, 4095, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3885, 241, 0, 1, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 1, 0, 0, 0, 0, 0, 4095, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 3703, 392, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4095, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 4018, 1, 153, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 1, 0, 0, 2607, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3303, 0, 0, 0, 4095, 1096, 41, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8232, 8571, 8192, 8195, 8221, 7385, 8537] avg=8192.0000 max=8571 max_rank=2 min=7385 min_rank=6 imbalance_ratio=1.046265 max_load_reduction=56965 max_load_reduction_pct=0.869217 moved_to_echo=32338 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4074, 0, 0, 4095, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3886, 352, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3607, 1, 0, 0, 0, 0, 0, 0, 4096, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3704, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3812, 0, 2, 0, 0, 4055, 0, 153, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2608, 2445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2072, 0, 0, 0, 4096, 1096, 41, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8395, 8096, 8192, 8193, 8022, 9149, 7305] avg=8192.0000 max=9149 max_rank=6 min=7305 min_rank=7 imbalance_ratio=1.116821 max_load_reduction=56387 max_load_reduction_pct=0.860397 moved_to_echo=32669 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3886, 125, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4096, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3705, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4063, 0, 153, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2608, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2744, 0, 1, 0, 4096, 1096, 41, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8154, 8557, 8193, 8193, 8302, 7956, 7978] avg=8192.0000 max=8557 max_rank=2 min=7956 min_rank=6 imbalance_ratio=1.044556 max_load_reduction=56979 max_load_reduction_pct=0.869431 moved_to_echo=32429 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 1, 4088, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 3886, 39, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 1, 0, 2, 0, 0, 6, 0, 4096, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3704, 392, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 1, 0, 0, 0, 4069, 0, 153, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 2608, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3263, 0, 3, 0, 4096, 1096, 41, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8072, 8490, 8192, 8190, 8187, 7711, 8499] avg=8192.0000 max=8499 max_rank=7 min=7711 min_rank=6 imbalance_ratio=1.037476 max_load_reduction=57037 max_load_reduction_pct=0.870316 moved_to_echo=32587 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4057, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3886, 178, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3474, 0, 0, 0, 0, 0, 1, 0, 4096, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3704, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 3986, 0, 153, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2608, 1397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3300, 0, 3, 0, 4096, 1096, 41, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8222, 7963, 8192, 8193, 8161, 8101, 8536] avg=8192.0000 max=8536 max_rank=7 min=7963 min_rank=2 imbalance_ratio=1.041992 max_load_reduction=57000 max_load_reduction_pct=0.869751 moved_to_echo=32991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 9, 7, 4061, 8, 0, 4090, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3879, 324, 5, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2490, 0, 6, 0, 0, 0, 2, 0, 4088, 392, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 6, 4087, 7, 0, 3697, 391, 0, 0, 0, 0, 0, 0, 4, 5, 0, 7, 7, 4078, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 4031, 3, 3, 3, 2, 4062, 7, 152, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 6, 0, 0, 2602, 2339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 3136, 4, 9, 0, 4084, 1094, 41, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8304, 6978, 8191, 8192, 8265, 9028, 8380] avg=8192.0000 max=9028 max_rank=6 min=6978 min_rank=2 imbalance_ratio=1.102051 max_load_reduction=56508 max_load_reduction_pct=0.862244 moved_to_echo=33865 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 0, 5, 4086, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 3886, 121, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 1, 0, 0, 9, 0, 4096, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3699, 392, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 4088, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 4, 0, 0, 0, 3885, 0, 153, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 2608, 2243, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2251, 0, 11, 0, 4096, 1096, 41, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8108, 8390, 8187, 8184, 8034, 8942, 7497] avg=8192.0000 max=8942 max_rank=6 min=7497 min_rank=7 imbalance_ratio=1.091553 max_load_reduction=56594 max_load_reduction_pct=0.863556 moved_to_echo=32657 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4081, 0, 6, 4086, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 3876, 42, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 6, 0, 0, 18, 0, 4096, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3703, 398, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 4089, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 1, 0, 0, 4091, 0, 155, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 2591, 582, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3556, 0, 1, 0, 4096, 1101, 44, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8049, 8491, 8197, 8196, 8344, 7264, 8804] avg=8192.0000 max=8804 max_rank=7 min=7264 min_rank=6 imbalance_ratio=1.074707 max_load_reduction=56732 max_load_reduction_pct=0.865662 moved_to_echo=32608 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[29696, 128, 128, 128, 0, 0, 128, 0, 32768, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 0, 0, 20864, 12032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 32640, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32128, 128, 128, 128, 128, 32256, 128, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 23680, 128, 128, 0, 32768, 8832, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 31104, 1536, 128, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 32768, 128, 0, 29696, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 4096, 0, 4096, 0, 4096, 0, 1548, 4096, 0, 1, 3941, 0, 2703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 1613, 4096, 0, 0, 3810, 0, 2769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 1, 0, 4095, 1, 4095, 1, 1, 0, 4095, 1, 0, 4095, 0, 4086, 1, 1, 4095, 2, 4094, 0, 4095, 0, 4095, 1, 596, 4095, 1, 9, 3931, 0, 3663, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 1, 0, 4, 4095, 0, 0, 0, 4091, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 4096, 0, 0, 4096, 0, 4084, 0, 0, 4096, 0, 4096, 15, 4096, 0, 4096, 0, 396, 4096, 0, 0, 3966, 0, 3825, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4090, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 296, 4096, 0, 0, 3911, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[9, 0, 4, 4089, 1, 7, 0, 4087, 8, 4089, 8, 7, 0, 4088, 7, 0, 4088, 0, 4089, 6, 4, 4088, 0, 4084, 4, 4087, 4, 4087, 9, 125, 4089, 5, 1, 4058, 0, 3949, 9, 0, 1, 0, 0, 0, 0, 0, 7, 6, 0, 5, 28, 2, 9, 0, 4086, 8, 0, 5, 0, 3, 4088, 1, 0, 18, 4079, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4096, 0, 0, 4096, 1, 4088, 0, 0, 4096, 0, 4095, 7, 4096, 0, 4096, 0, 1690, 4095, 0, 0, 3662, 0, 2834, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 1, 4096, 0, 4089, 0, 0, 0, 4096, 0, 0, 4096, 0, 4073, 0, 0, 4096, 0, 4095, 5, 4096, 0, 4096, 0, 428, 4096, 0, 0, 3935, 0, 3829, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3256, 0, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 0, 2769, 0, 0, 0, 0, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8201, 8965, 7419, 8191, 8184, 8192] avg=8192.0000 max=8965 max_rank=3 min=7419 min_rank=4 imbalance_ratio=1.094360 max_load_reduction=56571 max_load_reduction_pct=0.863205 moved_to_echo=31986 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3256, 0, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3941, 0, 2703, 0, 0, 0, 0, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8201, 8900, 7485, 8192, 8182, 8192] avg=8192.0000 max=8900 max_rank=3 min=7485 min_rank=4 imbalance_ratio=1.086426 max_load_reduction=56636 max_load_reduction_pct=0.864197 moved_to_echo=32051 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 1, 9, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3256, 1, 596, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3931, 0, 3663, 1, 0, 0, 0, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 1, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8194, 8193, 7949, 8443, 8192, 8184, 8190] avg=8192.0000 max=8443 max_rank=4 min=7949 min_rank=3 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=33009 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 0, 0, 9, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 3256, 0, 1690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3662, 0, 2834, 0, 0, 0, 1, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 1, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8190, 8193, 9049, 7337, 8190, 8188, 8197] avg=8192.0000 max=9049 max_rank=3 min=7337 min_rank=4 imbalance_ratio=1.104614 max_load_reduction=56487 max_load_reduction_pct=0.861923 moved_to_echo=31912 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3256, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 3984, 0, 0, 0, 0, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8195, 7648, 8735, 8192, 8190, 8192] avg=8192.0000 max=8735 max_rank=4 min=7648 min_rank=3 imbalance_ratio=1.066284 max_load_reduction=56801 max_load_reduction_pct=0.866714 moved_to_echo=33309 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 3256, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 3825, 0, 0, 0, 5, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8187, 8189, 7763, 8636, 8192, 8184, 8193] avg=8192.0000 max=8636 max_rank=4 min=7763 min_rank=3 imbalance_ratio=1.054199 max_load_reduction=56900 max_load_reduction_pct=0.868225 moved_to_echo=33205 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 4, 4089, 1, 7, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4089, 8, 7, 0, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 4, 9, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4087, 4, 3249, 9, 125, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4058, 0, 3949, 9, 0, 1, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 5, 4088, 4089, 0, 0, 0, 0, 0, 0, 28, 2, 9, 0, 4086, 8, 0, 5, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4088, 1, 0, 18, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8207, 8192, 7483, 8856, 8195, 8217, 8189] avg=8192.0000 max=8856 max_rank=4 min=7483 min_rank=3 imbalance_ratio=1.081055 max_load_reduction=56680 max_load_reduction_pct=0.864868 moved_to_echo=33457 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 3250, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 3829, 0, 0, 0, 4, 841, 7, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8182, 8172, 7779, 8625, 8192, 8207, 8191] avg=8192.0000 max=8625 max_rank=4 min=7779 min_rank=3 imbalance_ratio=1.052856 max_load_reduction=56911 max_load_reduction_pct=0.868393 moved_to_echo=33215 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 31232, 0, 27648, 128, 0, 128, 128, 6784, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 26112, 128, 6784, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 4081, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 6, 0, 2, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 10, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 0, 0, 0, 4010, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 66, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 84, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 0, 0, 1, 0, 4095, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4095, 4074, 0, 0, 0, 4049, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 52, 0, 0, 0, 4096, 4095, 0, 0, 0, 4095, 0, 0, 4096, 17, 0, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 1, 0, 0, 3, 4095, 0, 0, 4095, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 4095, 4094, 1, 1, 0, 4095, 4095, 1, 1, 0, 0, 4094, 4095, 0, 0, 1, 4095, 0, 0, 0, 4092, 0, 0, 1, 0, 1, 4095, 4095, 1, 0, 0, 4095, 2, 1, 4095, 2, 0, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 4053, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 43, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 16, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 7, 4096, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 4091, 4064, 0, 0, 0, 4015, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 65, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 4096, 48, 0, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4068, 0, 0, 0, 6, 0, 0, 19, 4078, 0, 0, 4085, 0, 8, 9, 0, 1, 8, 0, 9, 0, 6, 1, 5, 4088, 4085, 8, 7, 0, 4085, 4087, 11, 1, 10, 1, 4084, 4082, 0, 6, 9, 4089, 0, 9, 0, 4088, 0, 6, 5, 0, 9, 4078, 4089, 8, 0, 3, 4088, 16, 7, 4087, 0, 0, 0, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 0, 4089, 0, 10, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4087, 4039, 0, 0, 0, 3952, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 93, 0, 0, 0, 4096, 4096, 0, 0, 3, 4091, 0, 0, 4096, 108, 0, 8, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4020, 38, 36, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 76, 4092, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8193, 8189, 8249, 8193, 8160, 8194, 8165] avg=8192.0000 max=8249 max_rank=3 min=8160 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 1, 0, 0, 3, 4019, 38, 36, 3, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 4095, 4090, 0, 0, 0, 0, 0, 0, 76, 4094, 1, 1, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 2, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4058, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8193, 8188, 8268, 8191, 8150, 8194, 8157] avg=8192.0000 max=8268 max_rank=3 min=8150 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4020, 38, 36, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 76, 4080, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 4096, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8193, 8187, 8209, 8192, 8197, 8194, 8171] avg=8192.0000 max=8209 max_rank=3 min=8171 min_rank=7 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 0, 0, 1, 0, 4019, 38, 36, 3, 0, 0, 0, 0, 4095, 0, 1, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 76, 4074, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 4094, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 17, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8193, 8188, 8199, 8192, 8204, 8193, 8173] avg=8192.0000 max=8204 max_rank=5 min=8173 min_rank=7 imbalance_ratio=1.001465 max_load_reduction=57332 max_load_reduction_pct=0.874817 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4020, 38, 36, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 76, 4032, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 4096, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 84, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8193, 8189, 8118, 8192, 8220, 8192, 8239] avg=8192.0000 max=8239 max_rank=7 min=8118 min_rank=3 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4068, 0, 0, 0, 6, 0, 0, 19, 4012, 38, 36, 3, 0, 0, 0, 0, 4078, 0, 0, 4085, 0, 8, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 9, 0, 6, 1, 5, 4087, 4086, 0, 0, 0, 0, 0, 0, 76, 4085, 8, 7, 0, 4085, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 1, 4084, 4082, 0, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 9, 0, 4088, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4078, 0, 8, 0, 3, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 16, 7, 4050, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8181, 8203, 8272, 8193, 8159, 8186, 8160] avg=8192.0000 max=8272 max_rank=3 min=8159 min_rank=5 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4015, 38, 36, 3, 0, 0, 0, 0, 4089, 0, 7, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 76, 4064, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 4090, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 48, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8193, 8198, 8155, 8192, 8213, 8193, 8204] avg=8192.0000 max=8213 max_rank=5 min=8155 min_rank=3 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 0, 4012, 43, 39, 5, 0, 0, 0, 0, 4089, 0, 10, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 75, 4039, 0, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 4087, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 108, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8197, 8194, 8066, 8191, 8233, 8190, 8267] avg=8192.0000 max=8267 max_rank=7 min=8066 min_rank=3 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32512, 384, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 32640, 128, 128, 0, 32384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 32768, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32512, 0, 128, 0, 32768, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 128, 128, 32256, 384, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4067, 0, 0, 4096, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4095, 4092, 4095, 4093, 0, 0, 1, 1, 0, 68, 0, 1, 0, 4095, 4095, 4092, 4095, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 92, 3944, 1, 0, 4093, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 1, 0, 4095, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 1, 4095, 4090, 4095, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4094, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 15, 20, 4043, 0, 0, 4072, 0, 0, 30, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 4096, 4096, 4096, 2, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4095, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4082, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 65, 4047, 0, 0, 4069, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 14, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4087, 4070, 4088, 4083, 11, 10, 6, 7, 0, 236, 0, 8, 0, 4086, 4089, 4053, 4085, 3, 8, 9, 10, 1, 7, 3, 0, 8, 9, 0, 7, 0, 0, 16, 3813, 4, 0, 4091, 8, 0, 55, 0, 4087, 0, 7, 0, 0, 9, 0, 4083, 0, 11, 0, 4087, 3, 8, 36, 0, 0, 3, 4081, 4051, 4086, 0, 0, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4084, 4096, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4086, 4088, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 13, 41, 4020, 0, 0, 4056, 0, 0, 18, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 4096, 4096, 4096, 4, 0, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4090, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 5, 30, 4070, 0, 0, 4069, 0, 0, 7, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 1, 4095, 0, 4, 0, 0, 0, 0, 4096, 4096, 4096, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4063, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4043, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 4047, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 4096, 0, 0, 6, 0, 55, 29, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4036, 50, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8160, 8188, 8167, 8253, 8194, 8182, 8195] avg=8192.0000 max=8253 max_rank=4 min=8160 min_rank=1 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32824 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 4095, 5, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 1, 0, 4064, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 1, 0, 1, 0, 4042, 50, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 92, 4095, 4045, 0, 0, 0, 0, 0, 0, 3944, 1, 0, 4093, 1, 0, 1, 0, 55, 29, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 1, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4033, 50, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4090, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8225, 8192, 8236, 8124, 8194, 8179, 8192] avg=8192.0000 max=8236 max_rank=3 min=8124 min_rank=4 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32689 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4065, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4043, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4046, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 4095, 0, 0, 0, 0, 55, 29, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4035, 52, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8171, 8189, 8148, 8261, 8194, 8183, 8193] avg=8192.0000 max=8261 max_rank=4 min=8148 min_rank=3 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32831 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4083, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4043, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 15, 20, 4096, 4044, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 4072, 0, 0, 30, 0, 55, 29, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 7, 0, 0, 4038, 53, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8155, 8189, 8198, 8229, 8188, 8195, 8200] avg=8192.0000 max=8229 max_rank=4 min=8155 min_rank=1 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32812 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4043, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 65, 4096, 4046, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 4069, 0, 0, 0, 0, 55, 29, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 4036, 50, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8156, 8189, 8209, 8200, 8194, 8196, 8195] avg=8192.0000 max=8209 max_rank=3 min=8156 min_rank=1 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32785 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4070, 4088, 6, 11, 10, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 8, 0, 4055, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 3, 8, 9, 10, 1, 7, 3, 4033, 50, 0, 0, 0, 0, 0, 0, 0, 8, 9, 0, 7, 0, 0, 16, 4087, 4039, 0, 0, 0, 0, 0, 0, 3813, 4, 0, 4091, 8, 0, 55, 0, 54, 29, 0, 0, 0, 0, 0, 0, 4087, 0, 7, 0, 0, 9, 0, 4083, 2, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4087, 3, 8, 36, 0, 4023, 50, 0, 0, 0, 0, 0, 0, 0, 3, 4081, 4051, 3, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8352, 8209, 8166, 8054, 8188, 8218, 8151] avg=8192.0000 max=8352 max_rank=1 min=8054 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32611 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4064, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4043, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 5, 30, 4096, 4046, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 4069, 0, 0, 7, 0, 55, 29, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 4, 0, 0, 4036, 50, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8161, 8188, 8191, 8230, 8191, 8187, 8197] avg=8192.0000 max=8230 max_rank=4 min=8161 min_rank=1 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32805 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4047, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4046, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 13, 41, 4096, 4038, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 4056, 0, 0, 18, 0, 57, 34, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 4039, 50, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8156, 8192, 8221, 8185, 8193, 8196, 8213] avg=8192.0000 max=8221 max_rank=3 min=8156 min_rank=1 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32787 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 384, 0, 128, 0, 32512, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 128, 32384, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 384, 32768, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 0, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 0, 32384, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32128, 128, 0, 32768, 128, 0, 128, 0, 512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2, 0, 0, 4096, 4096, 4095, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 46, 0, 0, 0, 0, 4096, 4051, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 4096, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4091, 4087, 2, 1, 1, 0, 4095, 101, 4095, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 4, 0, 4095, 0, 1, 4094, 13, 0, 0, 0, 10, 4095, 4095, 4086, 0, 3987, 0, 0, 1, 1, 4095, 0, 7, 0, 4081, 0, 4094, 4095, 4095, 0, 1, 1, 8, 1, 0, 0, 1, 1, 0, 1, 1, 1, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 4093, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 4096, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 9, 0, 0, 0, 0, 4096, 4087, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 4096, 41, 0, 0, 0, 0, 4096, 4055, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4050, 4006, 4, 8, 5, 0, 4083, 418, 4087, 10, 1, 0, 0, 0, 8, 8, 0, 0, 0, 11, 6, 36, 0, 4088, 0, 4, 4079, 17, 0, 0, 0, 83, 4087, 4086, 4087, 0, 3662, 0, 0, 9, 10, 4086, 0, 35, 0, 4083, 0, 4085, 4080, 4086, 0, 9, 7, 0, 7, 0, 0, 9, 10, 0, 11, 10, 5, 4060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4084, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 4096, 83, 0, 0, 0, 0, 4096, 4016, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 4096, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4048, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 4004, 48, 3, 2, 0, 0, 0, 0, 0, 4096, 91, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8192, 8190, 8155, 8280, 8192, 8191, 8192] avg=8192.0000 max=8280 max_rank=4 min=8144 min_rank=0 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32855 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4043, 4087, 2, 1, 1, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 0, 4095, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 13, 0, 0, 0, 10, 3995, 48, 3, 3, 0, 0, 0, 0, 0, 4095, 91, 0, 3987, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 0, 4081, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4092, 0, 1, 1, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 4093, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8192, 8194, 8167, 8174, 8183, 8198, 8193] avg=8192.0000 max=8235 max_rank=0 min=8167 min_rank=3 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32748 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4048, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4005, 48, 3, 3, 0, 0, 0, 0, 0, 4093, 91, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8191, 8189, 8157, 8279, 8190, 8193, 8193] avg=8192.0000 max=8279 max_rank=4 min=8144 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32855 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4048, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 9, 0, 0, 0, 0, 4005, 48, 3, 3, 0, 0, 0, 0, 0, 4087, 91, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8191, 8187, 8165, 8274, 8192, 8191, 8192] avg=8192.0000 max=8274 max_rank=4 min=8144 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32849 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4048, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 0, 0, 0, 0, 4005, 48, 3, 2, 0, 0, 0, 0, 0, 4051, 91, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8191, 8191, 8200, 8237, 8191, 8190, 8192] avg=8192.0000 max=8237 max_rank=4 min=8144 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32810 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4048, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 41, 0, 0, 0, 0, 4005, 48, 3, 2, 0, 0, 0, 0, 0, 4055, 91, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8192, 8184, 8201, 8242, 8192, 8189, 8192] avg=8192.0000 max=8242 max_rank=4 min=8144 min_rank=0 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32815 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4002, 4006, 4, 8, 5, 0, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 10, 1, 0, 0, 0, 8, 8, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 36, 0, 4088, 4081, 1, 0, 0, 0, 0, 0, 0, 0, 4, 4079, 17, 0, 0, 0, 83, 3996, 48, 3, 2, 0, 0, 0, 0, 0, 4086, 91, 0, 3662, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 35, 0, 4083, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4083, 0, 9, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 10, 0, 11, 10, 5, 4060, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8443, 8199, 8223, 8232, 7848, 8213, 8186, 8192] avg=8192.0000 max=8443 max_rank=0 min=7848 min_rank=4 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32439 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4042, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 83, 0, 0, 0, 0, 4006, 54, 5, 2, 7, 0, 0, 0, 0, 4016, 90, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8188, 8178, 8259, 8202, 8183, 8198, 8190] avg=8192.0000 max=8259 max_rank=3 min=8138 min_rank=0 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32773 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32384, 32768, 128, 128, 128, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 256, 0, 128, 0, 128, 32128, 512, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 128, 0, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32640, 768, 0, 32256, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 19, 0, 0, 0, 0, 4047, 0, 4096, 4095, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 92, 0, 2, 1, 0, 1, 0, 0, 4095, 0, 1, 0, 0, 4095, 4095, 2, 4094, 267, 0, 0, 0, 0, 3570, 1, 4095, 4047, 0, 1, 2, 4094, 4095, 4094, 1, 0, 1, 1, 3, 0, 0, 0, 207, 4092, 0, 4095, 0, 0, 0, 4095, 1, 4093, 1, 1, 1, 4094, 1, 0, 1, 4095, 2, 0, 0, 6, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 67, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 78, 0, 0, 0, 0, 3973, 0, 4096, 4073, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 4096, 0, 0, 0, 4093, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 55, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 0, 4028, 0, 4096, 4095, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 4096, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 13, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 4095, 0, 4096, 5, 0, 0, 0, 0, 3973, 0, 4096, 4090, 0, 0, 0, 4096, 4096, 4096, 0, 1, 0, 0, 0, 1, 0, 0, 114, 4096, 0, 4095, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 33, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 4096, 0, 4096, 155, 0, 0, 0, 0, 3940, 0, 4096, 4042, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 7, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 5, 0, 15, 9, 0, 6, 2, 0, 4085, 0, 9, 0, 0, 4086, 4089, 12, 4084, 1292, 0, 0, 0, 20, 2380, 11, 4085, 3550, 2, 6, 10, 4080, 4086, 4078, 10, 0, 40, 5, 22, 0, 0, 0, 933, 4052, 0, 4086, 0, 5, 0, 4087, 4, 4029, 12, 7, 7, 4085, 9, 0, 4, 4087, 10, 0, 0, 35, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 38, 0, 0, 0, 6, 0, 0, 0, 4095, 0, 0, 0, 0, 4080, 4091, 0, 4096, 181, 0, 0, 0, 0, 3907, 0, 4096, 4084, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 4093, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 11, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 4047, 116, 0, 0, 0, 0, 0, 0, 0, 0, 85, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4053, 35, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4011, 43, 33, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8199, 8278, 8276, 8184, 8058, 8201, 8183] avg=8192.0000 max=8278 max_rank=2 min=8058 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32626 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 92, 0, 2, 1, 0, 1, 0, 4095, 4053, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 267, 0, 0, 0, 0, 3570, 116, 0, 0, 0, 0, 0, 0, 0, 1, 85, 4047, 0, 1, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 1, 1, 3, 0, 0, 4052, 35, 0, 0, 0, 0, 0, 0, 0, 207, 4092, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 4093, 1, 1, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 2, 0, 0, 6, 1, 4010, 43, 33, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8198, 8049, 8230, 8187, 8235, 8202, 8191] avg=8192.0000 max=8244 max_rank=0 min=8049 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32815 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 55, 0, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4028, 116, 0, 0, 0, 0, 0, 0, 0, 0, 85, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4052, 35, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4011, 43, 33, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8199, 8241, 8276, 8183, 8047, 8202, 8183] avg=8192.0000 max=8276 max_rank=3 min=8047 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 0, 0, 0, 1, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 3973, 116, 0, 0, 0, 0, 0, 0, 0, 0, 85, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 1, 0, 4052, 35, 0, 0, 0, 0, 0, 0, 0, 114, 4096, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4011, 43, 33, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8195, 8190, 8271, 8185, 8146, 8200, 8185] avg=8192.0000 max=8271 max_rank=3 min=8146 min_rank=5 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 67, 0, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 78, 0, 0, 0, 0, 3973, 116, 0, 0, 0, 0, 0, 0, 0, 0, 85, 4073, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4051, 35, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4011, 43, 33, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8199, 8263, 8254, 8182, 8043, 8195, 8183] avg=8192.0000 max=8263 max_rank=2 min=8043 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32603 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 33, 0, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 155, 0, 0, 0, 0, 3940, 116, 0, 0, 0, 0, 0, 0, 0, 0, 85, 4042, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4053, 42, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 4011, 43, 34, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8184, 8307, 8223, 8191, 8058, 8199, 8191] avg=8192.0000 max=8307 max_rank=2 min=8058 min_rank=5 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32639 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 15, 9, 0, 6, 2, 4086, 4043, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 0, 0, 4086, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4084, 1292, 0, 0, 0, 20, 2380, 116, 0, 0, 0, 0, 0, 0, 0, 11, 85, 3550, 2, 6, 10, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 10, 0, 40, 5, 22, 0, 0, 4045, 35, 0, 0, 0, 0, 0, 0, 0, 933, 4052, 0, 3927, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 4029, 12, 7, 7, 4085, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4087, 10, 0, 0, 35, 5, 4000, 43, 33, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8191, 7904, 7744, 8235, 8917, 8162, 8217] avg=8192.0000 max=8917 max_rank=5 min=7744 min_rank=3 imbalance_ratio=1.088501 max_load_reduction=56619 max_load_reduction_pct=0.863937 moved_to_echo=33531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 38, 0, 0, 0, 6, 0, 0, 4096, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 181, 0, 0, 0, 0, 3907, 120, 0, 0, 0, 0, 0, 0, 0, 0, 82, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4058, 35, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 4014, 44, 38, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8171, 8304, 8262, 8189, 8032, 8175, 8203] avg=8192.0000 max=8304 max_rank=2 min=8032 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32599 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 384, 0, 128, 128, 128, 128, 128, 32768, 32512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 2048, 0, 0, 0, 128, 29824, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 1408, 32768, 0, 31488, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 0, 32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 768, 32128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 128, 128, 32128, 384, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 3685, 0, 4096, 0, 1576, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 2887, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4032, 0, 4096, 0, 2462, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1660, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 3941, 0, 4096, 0, 2114, 0, 0, 0, 0, 0, 0, 0, 11, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2126, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4022, 0, 4093, 0, 2089, 0, 0, 0, 0, 0, 0, 0, 42, 0, 8, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 7, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3, 0, 0, 2030, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4092, 4096, 0, 4096, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1, 3791, 0, 4095, 1, 2455, 1, 0, 0, 0, 1, 0, 0, 263, 0, 0, 3, 0, 1, 4095, 1, 1, 4094, 1, 0, 1, 39, 4093, 0, 0, 1, 4095, 6, 0, 0, 4092, 0, 1, 0, 1, 3, 6, 4095, 0, 1, 0, 0, 1, 1638, 0, 4093, 4086, 1, 24, 1, 0, 1, 0, 4095, 4083, 4085, 5, 4090, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4025, 0, 4081, 0, 1899, 0, 0, 0, 0, 0, 0, 0, 9, 0, 16, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 27, 4096, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 8, 0, 0, 2264, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4068, 4096, 0, 4096, 4088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[9, 2826, 0, 4089, 5, 3141, 10, 0, 0, 0, 10, 0, 0, 1286, 9, 0, 26, 0, 6, 4086, 9, 10, 4080, 12, 13, 12, 106, 4070, 0, 0, 7, 4085, 58, 0, 0, 4064, 0, 4, 5, 16, 19, 28, 4074, 0, 11, 0, 0, 8, 785, 0, 4048, 3950, 17, 184, 3, 0, 11, 0, 4085, 4034, 3980, 107, 4059, 4079] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4045, 0, 4082, 0, 1452, 0, 0, 0, 0, 0, 0, 0, 81, 0, 14, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 19, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 10, 0, 0, 2622, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4079, 4096, 0, 4096, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3685, 0, 2247, 0, 1576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 4096, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3871, 222, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 2007, 1849, 208, 24, 5, 0, 0, 0, 2887, 0, 2089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7508, 8009, 8187, 8172, 8189, 8190, 9072, 8209] avg=8192.0000 max=9072 max_rank=6 min=7508 min_rank=0 imbalance_ratio=1.107422 max_load_reduction=56464 max_load_reduction_pct=0.861572 moved_to_echo=33660 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4032, 0, 2247, 0, 2462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 4096, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3871, 223, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2007, 1849, 208, 24, 5, 0, 0, 0, 1660, 0, 2089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8741, 8007, 8187, 8168, 8190, 8189, 7845, 8209] avg=8192.0000 max=8741 max_rank=0 min=7845 min_rank=6 imbalance_ratio=1.067017 max_load_reduction=56795 max_load_reduction_pct=0.866623 moved_to_echo=32433 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 3791, 0, 2247, 1, 2455, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 263, 0, 0, 4095, 3872, 0, 0, 0, 0, 0, 0, 3, 0, 1, 4090, 1, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 39, 4093, 0, 0, 1, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4092, 0, 1, 0, 1, 3866, 223, 0, 0, 0, 0, 0, 0, 3, 6, 4095, 0, 1, 0, 0, 1, 2006, 1848, 207, 24, 5, 0, 0, 0, 1638, 0, 2087, 4086, 1, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4083, 4085, 5, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8496, 8231, 8191, 8205, 8189, 8196, 7837, 8191] avg=8192.0000 max=8496 max_rank=0 min=7837 min_rank=6 imbalance_ratio=1.037109 max_load_reduction=57040 max_load_reduction_pct=0.870361 moved_to_echo=32413 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4022, 0, 2246, 0, 2089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 8, 4096, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 1, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3871, 223, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3, 0, 0, 2007, 1847, 208, 24, 5, 0, 0, 0, 2030, 0, 2089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8357, 8017, 8187, 8176, 8190, 8189, 8215, 8205] avg=8192.0000 max=8357 max_rank=0 min=8017 min_rank=1 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32799 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3941, 0, 2247, 0, 2114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 2, 4096, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3871, 223, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2007, 1849, 208, 24, 5, 0, 0, 0, 2126, 0, 2089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8302, 7980, 8187, 8168, 8190, 8189, 8311, 8209] avg=8192.0000 max=8311 max_rank=6 min=7980 min_rank=1 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32899 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4025, 0, 2239, 0, 1899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 16, 4096, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4096, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 3871, 221, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 8, 0, 0, 2007, 1842, 208, 24, 5, 0, 0, 0, 2264, 0, 2089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4096, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 7988, 8187, 8195, 8190, 8183, 8449, 8181] avg=8192.0000 max=8449 max_rank=6 min=7988 min_rank=1 imbalance_ratio=1.031372 max_load_reduction=57087 max_load_reduction_pct=0.871078 moved_to_echo=33003 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[9, 2826, 0, 2244, 5, 3141, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1286, 9, 0, 4085, 3858, 0, 0, 0, 0, 0, 0, 26, 0, 6, 4081, 9, 10, 4080, 12, 0, 0, 0, 0, 0, 0, 0, 0, 13, 12, 106, 4070, 0, 0, 7, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 4064, 0, 4, 5, 16, 3836, 221, 0, 0, 0, 0, 0, 0, 19, 28, 4074, 0, 11, 0, 0, 8, 1983, 1845, 206, 24, 5, 0, 0, 0, 785, 0, 2065, 3950, 17, 184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4034, 3980, 107, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 9248, 8224, 8269, 8204, 8203, 7004, 8149] avg=8192.0000 max=9248 max_rank=1 min=7004 min_rank=6 imbalance_ratio=1.128906 max_load_reduction=56288 max_load_reduction_pct=0.858887 moved_to_echo=31560 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4045, 0, 2237, 0, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 14, 4096, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3877, 221, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 10, 0, 0, 2011, 1845, 211, 27, 6, 0, 0, 0, 2622, 0, 2085, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7734, 8056, 8186, 8183, 8194, 8197, 8803, 8183] avg=8192.0000 max=8803 max_rank=6 min=7734 min_rank=0 imbalance_ratio=1.074585 max_load_reduction=56733 max_load_reduction_pct=0.865677 moved_to_echo=33377 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 1792, 128, 128, 32768, 30976, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 30464, 0, 18048, 128, 17280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 256, 32768, 128, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[16128, 0, 16768, 32640, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 0, 128, 16128, 14848, 1664, 256, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 32768, 0, 128, 128, 128, 30976, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 3533, 4096, 0, 0, 0, 4096, 0, 0, 0, 582, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3811, 4096, 0, 0, 0, 4096, 0, 0, 0, 346, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4095, 4086, 0, 1, 0, 0, 4095, 2, 0, 0, 1, 4095, 0, 4095, 0, 0, 2534, 4095, 9, 0, 1, 3990, 0, 0, 1, 1665, 1, 4086, 2, 1, 7, 3, 4092, 1, 1, 4094, 0, 1, 1, 2, 0, 0, 4095, 0, 4094, 4094, 0, 1, 2, 1, 0, 1, 4095, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 3293, 4096, 0, 0, 0, 4096, 0, 6, 0, 922, 0, 4036, 0, 0, 0, 0, 4096, 0, 0, 4090, 21, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 5, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3236, 4096, 0, 0, 0, 4096, 0, 0, 0, 951, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 4095, 4096, 4096, 1, 0, 0, 0, 4095, 0, 1, 0, 0, 4096, 0, 4095, 1, 0, 3629, 4096, 0, 0, 0, 4096, 0, 1, 0, 498, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[18, 0, 0, 0, 4088, 4083, 4032, 0, 2, 10, 0, 4089, 16, 0, 0, 10, 4085, 0, 4086, 0, 0, 832, 4085, 87, 0, 11, 3535, 6, 0, 9, 3801, 9, 4005, 11, 8, 57, 25, 4034, 4, 11, 4075, 0, 7, 10, 17, 0, 0, 4086, 10, 4082, 4079, 0, 9, 0, 4, 9, 8, 4087, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 4096, 0, 4086, 0, 0, 4004, 4096, 0, 0, 0, 4096, 0, 10, 0, 146, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 4089, 34, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4038, 54, 0, 0, 0, 0, 0, 0, 980, 0, 4095, 0, 0, 3533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 582, 0, 3059, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 10, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 4096, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8188, 8608, 7737, 8204, 8182, 8192, 8226] avg=8192.0000 max=8608 max_rank=2 min=7737 min_rank=3 imbalance_ratio=1.050781 max_load_reduction=56928 max_load_reduction_pct=0.868652 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4038, 54, 0, 0, 0, 0, 0, 0, 980, 0, 4096, 0, 0, 3811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 346, 0, 3059, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 10, 3, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 4096, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8188, 8887, 7501, 8205, 8190, 8192, 8174] avg=8192.0000 max=8887 max_rank=2 min=7501 min_rank=3 imbalance_ratio=1.084839 max_load_reduction=56649 max_load_reduction_pct=0.864395 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 4095, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 2, 0, 0, 1, 4037, 54, 0, 0, 0, 0, 0, 0, 980, 0, 4095, 0, 0, 2534, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3990, 0, 0, 1, 1665, 1, 3058, 0, 0, 0, 0, 0, 0, 0, 4086, 2, 1, 7, 3, 4092, 1, 1, 10, 3, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 1, 2, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 4095, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8190, 7618, 8716, 8206, 8183, 8192, 8243] avg=8192.0000 max=8716 max_rank=3 min=7618 min_rank=2 imbalance_ratio=1.063965 max_load_reduction=56820 max_load_reduction_pct=0.867004 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 7, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 4037, 54, 0, 0, 0, 0, 0, 0, 980, 0, 4095, 1, 0, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 498, 0, 3059, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 10, 3, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 1, 0, 0, 4096, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8187, 8705, 7654, 8204, 8183, 8191, 8207] avg=8192.0000 max=8705 max_rank=2 min=7654 min_rank=3 imbalance_ratio=1.062622 max_load_reduction=56831 max_load_reduction_pct=0.867172 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 6, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4038, 54, 0, 0, 0, 0, 0, 0, 980, 0, 4089, 0, 0, 3293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 922, 0, 3059, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 4096, 0, 0, 10, 3, 0, 0, 0, 0, 0, 0, 4090, 21, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 5, 0, 0, 4096, 52, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8188, 8362, 8083, 8145, 8197, 8192, 8162] avg=8192.0000 max=8362 max_rank=2 min=8083 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4038, 54, 0, 0, 0, 0, 0, 0, 980, 0, 4096, 0, 0, 3236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 951, 0, 3059, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 10, 3, 0, 0, 0, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 4096, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8188, 8312, 8106, 8202, 8211, 8192, 8126] avg=8192.0000 max=8312 max_rank=2 min=8106 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 0, 0, 8, 4083, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 10, 0, 4089, 16, 0, 0, 10, 4030, 53, 0, 0, 0, 0, 0, 0, 978, 0, 4086, 0, 0, 832, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3535, 6, 0, 9, 3801, 9, 3051, 0, 0, 0, 0, 0, 0, 0, 4005, 11, 8, 57, 25, 4034, 4, 11, 10, 3, 0, 0, 0, 0, 0, 0, 4075, 0, 7, 10, 17, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4082, 4079, 0, 9, 0, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4087, 0, 0, 0, 0, 4, 0, 4085, 50, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8210, 5983, 10422, 8168, 8185, 8193, 8234] avg=8192.0000 max=10422 max_rank=3 min=5983 min_rank=2 imbalance_ratio=1.272217 max_load_reduction=55114 max_load_reduction_pct=0.840973 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 4045, 56, 0, 0, 0, 0, 0, 0, 971, 0, 4086, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 146, 0, 3065, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 14, 4, 0, 0, 0, 0, 0, 0, 4089, 34, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 6, 0, 0, 4096, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8197, 9061, 7317, 8202, 8205, 8192, 8164] avg=8192.0000 max=9061 max_rank=2 min=7317 min_rank=3 imbalance_ratio=1.106079 max_load_reduction=56475 max_load_reduction_pct=0.861740 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7936, 0, 32768, 128, 0, 24960, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 128, 32384, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32384, 0, 0, 0, 128, 128, 0, 32768, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32128, 128, 128, 128, 8960, 128, 24576, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32640, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 2, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4062, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 5, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 57, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4021, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4092, 13, 4020, 0, 0, 0, 0, 4095, 108, 0, 1, 0, 4095, 2, 3, 0, 4094, 0, 4095, 39, 0, 4095, 4, 1, 4095, 4095, 0, 4095, 0, 0, 0, 2, 0, 0, 1, 4095, 4082, 0, 1, 0, 4094, 0, 1, 1, 1, 1, 1, 4091, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4058, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 36, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 4096, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 3999, 14, 0, 0, 4, 0, 0, 10, 11, 10, 10, 5, 2, 4076, 147, 3451, 0, 0, 5, 0, 4088, 545, 0, 2, 0, 4086, 13, 17, 0, 4085, 0, 4086, 6, 0, 4087, 28, 12, 4086, 4081, 9, 4085, 3, 0, 0, 15, 0, 0, 10, 4086, 4072, 0, 9, 0, 4078, 6, 11, 9, 15, 1, 2, 4065, 4084, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4027, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4093, 64, 0, 0, 0, 4096, 0, 0, 0, 4096, 4, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4091, 0, 0, 0, 0, 6, 0, 0, 4096, 4069, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 0, 0, 0, 4096, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4049, 12, 8, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4057, 37, 0, 0, 0, 0, 0, 0, 0, 10, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8168, 8184, 8189, 8186, 8190, 8201, 8192] avg=8192.0000 max=8226 max_rank=0 min=8168 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32769 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 0, 0, 0, 0, 0, 0, 4096, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4049, 12, 8, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4058, 37, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8168, 8194, 8189, 8184, 8191, 8202, 8192] avg=8192.0000 max=8216 max_rank=0 min=8168 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=57320 max_load_reduction_pct=0.874634 moved_to_echo=32769 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4021, 0, 1, 0, 0, 0, 0, 4095, 38, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4092, 13, 4048, 12, 8, 3, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 4083, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 2, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 39, 0, 0, 4, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 4057, 37, 0, 0, 0, 0, 0, 0, 1, 10, 4082, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 4091, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8181, 8211, 8192, 8226, 8191, 8188, 8192] avg=8192.0000 max=8226 max_rank=4 min=8155 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32797 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4058, 0, 0, 0, 0, 0, 9, 4096, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4049, 12, 8, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 1, 0, 4058, 37, 0, 0, 0, 0, 0, 0, 0, 10, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8168, 8215, 8190, 8184, 8192, 8194, 8192] avg=8192.0000 max=8215 max_rank=2 min=8168 min_rank=1 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4062, 0, 0, 0, 0, 0, 33, 4096, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4049, 12, 8, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4057, 37, 0, 0, 0, 0, 0, 0, 0, 10, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8168, 8181, 8194, 8184, 8190, 8197, 8192] avg=8192.0000 max=8230 max_rank=0 min=8168 min_rank=1 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4039, 0, 0, 0, 0, 0, 0, 4096, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4049, 12, 8, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4058, 37, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8168, 8237, 8189, 8184, 8191, 8202, 8192] avg=8192.0000 max=8237 max_rank=2 min=8168 min_rank=1 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32769 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3999, 14, 0, 0, 4, 0, 0, 4087, 39, 0, 0, 0, 0, 0, 0, 10, 11, 10, 10, 5, 2, 4076, 147, 4039, 12, 8, 2, 0, 0, 0, 0, 3451, 0, 0, 5, 0, 4076, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4084, 13, 17, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 6, 0, 0, 28, 12, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4085, 3, 0, 0, 15, 0, 0, 4047, 37, 0, 0, 0, 0, 0, 0, 10, 10, 4072, 0, 9, 0, 4078, 6, 0, 0, 0, 0, 0, 0, 0, 0, 11, 9, 15, 1, 2, 4065, 4084, 10, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8332, 8077, 8201, 8205, 8196, 8185, 8197] avg=8192.0000 max=8332 max_rank=1 min=8077 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32783 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4027, 0, 0, 0, 0, 0, 30, 4096, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4053, 16, 9, 4, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 4077, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 6, 0, 4057, 41, 0, 0, 0, 0, 0, 0, 0, 2, 4069, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8183, 8237, 8192, 8183, 8195, 8167, 8187] avg=8192.0000 max=8237 max_rank=2 min=8167 min_rank=6 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32732 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32128, 0, 0, 128, 0, 32768, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 0, 32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32384, 128, 128, 0, 128, 0, 128, 32768, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 256, 32512, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66816 slack_slots=3456 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4088, 0, 4091, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 11, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 65, 1, 4022, 1, 4095, 0, 0, 4095, 1, 4095, 0, 2, 0, 0, 4092, 17, 0, 4095, 4094, 1, 4094, 0, 1, 3, 1, 4095, 0, 0, 207, 0, 4095, 0, 1, 0, 4095, 0, 4095, 0, 0, 1, 0, 1, 0, 0, 2, 1, 1, 0, 1, 2, 0, 0, 2, 4095, 5, 0, 4090, 1, 4094, 3874, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4086, 0, 3998, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 106, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 4096, 0, 4096, 4095, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 3, 0, 4087, 0, 3969, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 130, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 4096, 0, 4096, 4093, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4062, 0, 3978, 0, 0, 4095, 0, 4096, 1, 0, 0, 0, 4095, 0, 0, 4093, 4096, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 130, 1, 4096, 0, 0, 0, 4096, 4, 4094, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 0, 4096, 0, 4096, 4093, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 27, 0, 4057, 0, 3938, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 252, 2, 4096, 0, 0, 0, 4096, 8, 4090, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 5, 0, 4096, 0, 4096, 4004, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 442, 9, 3776, 4, 4089, 0, 0, 4087, 8, 4084, 0, 15, 0, 0, 4069, 271, 0, 4084, 4073, 7, 4081, 7, 18, 12, 10, 4087, 0, 0, 35, 0, 4082, 0, 9, 2, 4088, 0, 4085, 0, 0, 12, 0, 11, 0, 0, 0, 14, 4, 0, 12, 9, 0, 0, 14, 4085, 3, 0, 4025, 8, 4082, 3697, 0, 16, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 21, 0, 4047, 0, 3914, 3, 0, 4095, 0, 4096, 0, 0, 0, 1, 4086, 0, 0, 4090, 4096, 0, 4096, 0, 0, 0, 0, 4081, 0, 0, 181, 6, 4096, 0, 0, 0, 4096, 9, 4088, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 34, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4088, 0, 4091, 0, 0, 49, 33, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 11, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4001, 77, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 95, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8198, 8159, 8075, 8192, 8183, 8181, 8287] avg=8192.0000 max=8287 max_rank=7 min=8075 min_rank=3 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32843 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4086, 0, 3998, 0, 0, 49, 33, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 106, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 4001, 77, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 95, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8196, 8159, 8170, 8192, 8183, 8184, 8286] avg=8192.0000 max=8286 max_rank=7 min=8159 min_rank=2 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32845 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 65, 1, 4022, 1, 4095, 0, 0, 49, 33, 0, 0, 0, 0, 0, 0, 6, 1, 4095, 0, 2, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4062, 4094, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4095, 0, 0, 207, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 2, 1, 1, 4094, 4086, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 2, 4095, 5, 3999, 77, 3, 0, 0, 0, 0, 0, 0, 4090, 1, 95, 3874, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8196, 8175, 8275, 8191, 8186, 8184, 8063] avg=8192.0000 max=8275 max_rank=3 min=8063 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32624 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4062, 0, 3978, 0, 0, 49, 33, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 130, 1, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 4001, 77, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 95, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8198, 8156, 8191, 8194, 8189, 8202, 8284] avg=8192.0000 max=8284 max_rank=7 min=8122 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32869 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 0, 4057, 0, 3938, 0, 0, 49, 33, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 252, 2, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 5, 4001, 77, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 95, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8184, 8158, 8307, 8194, 8208, 8186, 8195] avg=8192.0000 max=8307 max_rank=3 min=8104 min_rank=0 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32783 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 4087, 0, 3969, 0, 0, 49, 33, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 130, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 4001, 77, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 95, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8198, 8159, 8185, 8192, 8183, 8194, 8284] avg=8192.0000 max=8284 max_rank=7 min=8141 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32853 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 442, 9, 3776, 4, 4089, 0, 0, 48, 33, 0, 0, 0, 0, 0, 0, 7, 8, 4084, 0, 15, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 4051, 4073, 7, 0, 7, 18, 0, 0, 0, 0, 0, 0, 0, 0, 12, 10, 4087, 0, 0, 35, 0, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 4088, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 11, 0, 0, 0, 14, 4, 4081, 4078, 0, 0, 0, 0, 0, 0, 0, 12, 9, 0, 0, 14, 4085, 3, 3987, 77, 2, 0, 0, 0, 0, 0, 0, 4025, 8, 95, 3697, 0, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8401, 8183, 8427, 8101, 8184, 8200, 8189, 7851] avg=8192.0000 max=8427 max_rank=2 min=7851 min_rank=7 imbalance_ratio=1.028687 max_load_reduction=57109 max_load_reduction_pct=0.871414 moved_to_echo=32424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 21, 0, 4047, 0, 3914, 3, 0, 51, 39, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 181, 6, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4088, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 34, 4002, 81, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 94, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8075, 8183, 8143, 8232, 8197, 8204, 8216, 8286] avg=8192.0000 max=8286 max_rank=7 min=8075 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32903 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 0, 32512, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 768, 32128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 0, 1152, 128, 31744, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 128, 32768, 128, 32000, 640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 640, 128, 32256, 128, 32128, 128, 0, 512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True + [2026-08-30 05:00:53] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 687.7 | throughput per GPU (TFLOP/s/GPU): 484.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.503458E+00 | loss scale: 1.0 | grad norm: 5.736 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2781, 23, 253, 44, 126, 0, 2531, 15, 819, 3455, 132, 289, 1161, 2232, 0, 254, 338, 1095, 81, 2019, 0, 702, 1224, 1502, 3, 5, 2687, 1432, 3471, 267, 3118, 309, 28, 692, 2213, 282, 153, 62, 2128, 1742, 1384, 0, 3102, 2302, 0, 913, 1037, 1070, 2846, 74, 1885, 2, 1, 269, 0, 360, 0, 163, 708, 1340, 3355, 1357, 1133, 2567] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2502, 25, 174, 76, 418, 1, 2436, 22, 753, 3294, 124, 194, 810, 1863, 0, 287, 454, 1375, 162, 1686, 0, 619, 970, 1469, 4, 10, 2937, 1421, 3322, 107, 2946, 562, 6, 770, 2884, 271, 152, 92, 2406, 1194, 2274, 0, 2791, 2053, 1, 995, 1454, 885, 3374, 42, 2004, 1, 16, 310, 8, 307, 2, 389, 457, 1367, 3202, 916, 1132, 2758] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2702, 5, 189, 38, 442, 0, 2647, 8, 558, 3014, 82, 434, 1382, 1338, 0, 208, 298, 1117, 273, 1611, 0, 944, 1051, 1583, 0, 20, 2862, 1233, 3399, 166, 2752, 934, 24, 699, 3090, 114, 188, 133, 2745, 1563, 1597, 0, 2823, 1502, 0, 771, 1616, 845, 3253, 98, 1612, 0, 9, 324, 0, 427, 0, 61, 639, 1636, 3603, 1103, 877, 2894] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2761, 43, 209, 48, 251, 0, 2034, 72, 1013, 3184, 86, 193, 1387, 1110, 0, 120, 239, 1055, 63, 1853, 3, 710, 955, 1738, 9, 14, 2727, 1308, 3238, 193, 3170, 799, 13, 1286, 2892, 509, 491, 54, 2394, 1182, 2152, 0, 3021, 1619, 9, 869, 1241, 541, 3366, 27, 1637, 0, 24, 172, 0, 670, 0, 224, 815, 1760, 3474, 409, 1087, 3013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2632, 5, 91, 92, 383, 1, 2536, 8, 426, 3521, 187, 398, 895, 1733, 0, 295, 236, 1537, 127, 1523, 0, 790, 894, 1461, 0, 40, 3183, 1648, 3143, 78, 2458, 426, 31, 1201, 2804, 238, 182, 128, 2661, 1607, 1831, 0, 2897, 1578, 2, 757, 1359, 1132, 3569, 50, 1587, 5, 3, 260, 0, 493, 0, 174, 686, 1634, 3006, 1278, 1073, 2563] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2725, 33, 151, 54, 339, 1, 1838, 69, 818, 2910, 120, 272, 878, 1630, 1, 312, 499, 1348, 149, 1784, 0, 528, 1245, 1728, 26, 6, 2890, 1009, 2805, 81, 3141, 669, 29, 861, 2445, 106, 540, 140, 3441, 1074, 1981, 0, 3415, 1936, 1, 968, 1032, 1160, 3099, 26, 1660, 1, 3, 174, 2, 563, 0, 93, 746, 1279, 3399, 1277, 1183, 2843] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2573, 17, 95, 95, 562, 0, 2786, 49, 1073, 3379, 86, 462, 1274, 1262, 0, 220, 462, 799, 101, 1869, 0, 527, 868, 1385, 25, 21, 2682, 1074, 3381, 175, 3032, 584, 23, 1001, 2486, 417, 335, 188, 2593, 1099, 1881, 0, 3114, 1940, 9, 972, 1357, 754, 3383, 50, 1941, 1, 17, 264, 3, 556, 3, 147, 794, 1534, 2998, 836, 1114, 2808] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2165, 33, 267, 49, 345, 0, 2334, 32, 264, 2944, 61, 328, 1441, 1063, 0, 400, 669, 1301, 105, 1934, 0, 511, 1411, 1776, 20, 7, 2803, 1214, 3183, 112, 2793, 797, 35, 1103, 2704, 214, 217, 71, 2771, 963, 2595, 1, 2828, 1855, 0, 693, 1591, 835, 3216, 56, 1761, 1, 4, 163, 0, 754, 0, 64, 562, 1722, 3379, 1030, 1147, 2839] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2781, 23, 253, 44, 126, 0, 2531, 15, 2716, 0, 0, 0, 0, 0, 0, 0, 819, 3455, 132, 289, 1161, 2232, 0, 254, 838, 64, 0, 0, 0, 0, 0, 0, 338, 1095, 81, 2019, 0, 702, 1224, 1502, 946, 439, 0, 0, 0, 0, 0, 0, 3, 5, 2687, 1432, 341, 267, 3118, 309, 0, 0, 0, 0, 0, 0, 0, 0, 28, 692, 2149, 282, 153, 62, 2128, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 0, 1318, 2302, 0, 913, 1037, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 2846, 74, 1885, 2, 1, 269, 0, 360, 1988, 414, 0, 0, 0, 0, 0, 0, 0, 163, 708, 1340, 928, 1357, 1133, 2567, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8489, 9244, 8346, 8162, 7236, 8024, 7839, 8196] avg=8192.0000 max=9244 max_rank=1 min=7236 min_rank=4 imbalance_ratio=1.128418 max_load_reduction=56292 max_load_reduction_pct=0.858948 moved_to_echo=31295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2702, 5, 189, 38, 442, 0, 2647, 8, 2659, 0, 0, 0, 0, 0, 0, 0, 558, 3014, 82, 434, 1382, 1338, 0, 208, 762, 89, 0, 0, 0, 0, 0, 0, 298, 1117, 273, 1611, 0, 944, 1051, 1583, 861, 471, 0, 0, 0, 0, 0, 0, 0, 20, 2862, 1233, 335, 166, 2752, 934, 0, 0, 0, 0, 0, 0, 0, 0, 24, 699, 3001, 114, 188, 133, 2745, 1563, 0, 0, 0, 0, 0, 0, 0, 0, 1597, 0, 1200, 1502, 0, 771, 1616, 845, 0, 0, 0, 0, 0, 0, 0, 0, 3253, 98, 1612, 0, 9, 324, 0, 427, 2135, 405, 0, 0, 0, 0, 0, 0, 0, 61, 639, 1636, 997, 1103, 877, 2894, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8690, 7867, 8209, 8302, 8467, 7531, 8263, 8207] avg=8192.0000 max=8690 max_rank=0 min=7531 min_rank=5 imbalance_ratio=1.060791 max_load_reduction=56846 max_load_reduction_pct=0.867401 moved_to_echo=32468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2761, 43, 209, 48, 251, 0, 2034, 72, 2533, 0, 0, 0, 0, 0, 0, 0, 1013, 3184, 86, 193, 1387, 1110, 0, 120, 816, 83, 0, 0, 0, 0, 0, 0, 239, 1055, 63, 1853, 3, 710, 955, 1738, 922, 454, 0, 0, 0, 0, 0, 0, 9, 14, 2727, 1308, 319, 193, 3170, 799, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1286, 2809, 509, 491, 54, 2394, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 0, 1283, 1619, 9, 869, 1241, 541, 0, 0, 0, 0, 0, 0, 0, 0, 3366, 27, 1637, 0, 24, 172, 0, 670, 2059, 386, 0, 0, 0, 0, 0, 0, 0, 224, 815, 1760, 961, 409, 1087, 3013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7951, 7992, 7992, 8539, 8738, 7714, 8341, 8269] avg=8192.0000 max=8738 max_rank=4 min=7714 min_rank=5 imbalance_ratio=1.066650 max_load_reduction=56798 max_load_reduction_pct=0.866669 moved_to_echo=33062 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2502, 25, 174, 76, 418, 1, 2436, 22, 2599, 0, 0, 0, 0, 0, 0, 0, 753, 3294, 124, 194, 810, 1863, 0, 287, 754, 83, 0, 0, 0, 0, 0, 0, 454, 1375, 162, 1686, 0, 619, 970, 1469, 851, 419, 0, 0, 0, 0, 0, 0, 4, 10, 2937, 1421, 327, 107, 2946, 562, 0, 0, 0, 0, 0, 0, 0, 0, 6, 770, 2801, 271, 152, 92, 2406, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 2274, 0, 1186, 2053, 1, 995, 1454, 885, 0, 0, 0, 0, 0, 0, 0, 0, 3374, 42, 2004, 1, 16, 310, 8, 307, 1897, 396, 0, 0, 0, 0, 0, 0, 2, 389, 457, 1367, 886, 916, 1132, 2758, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8162, 8005, 8314, 7692, 8848, 8355, 7907] avg=8192.0000 max=8848 max_rank=5 min=7692 min_rank=4 imbalance_ratio=1.080078 max_load_reduction=56688 max_load_reduction_pct=0.864990 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2725, 33, 151, 54, 339, 1, 1838, 69, 2194, 0, 0, 0, 0, 0, 0, 0, 818, 2910, 120, 272, 878, 1630, 1, 312, 922, 70, 0, 0, 0, 0, 0, 0, 499, 1348, 149, 1784, 0, 528, 1245, 1728, 1042, 444, 0, 0, 0, 0, 0, 0, 26, 6, 2890, 1009, 277, 81, 3141, 669, 0, 0, 0, 0, 0, 0, 0, 0, 29, 861, 2375, 106, 540, 140, 3441, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1981, 0, 1451, 1936, 1, 968, 1032, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 3099, 26, 1660, 1, 3, 174, 2, 563, 2014, 334, 0, 0, 0, 0, 0, 0, 0, 93, 746, 1279, 941, 1277, 1183, 2843, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7404, 7933, 8767, 8099, 8566, 8529, 7876, 8362] avg=8192.0000 max=8767 max_rank=2 min=7404 min_rank=0 imbalance_ratio=1.070190 max_load_reduction=56769 max_load_reduction_pct=0.866226 moved_to_echo=33333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2632, 5, 91, 92, 383, 1, 2536, 8, 2459, 0, 0, 0, 0, 0, 0, 0, 426, 3521, 187, 398, 895, 1733, 0, 295, 782, 81, 0, 0, 0, 0, 0, 0, 236, 1537, 127, 1523, 0, 790, 894, 1461, 884, 393, 0, 0, 0, 0, 0, 0, 0, 40, 3183, 1648, 310, 78, 2458, 426, 0, 0, 0, 0, 0, 0, 0, 0, 31, 1201, 2723, 238, 182, 128, 2661, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 1831, 0, 1231, 1578, 2, 757, 1359, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 50, 1587, 5, 3, 260, 0, 493, 1781, 374, 0, 0, 0, 0, 0, 0, 0, 174, 686, 1634, 832, 1278, 1073, 2563, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8318, 7845, 8143, 8771, 7890, 8122, 8240] avg=8192.0000 max=8771 max_rank=4 min=7845 min_rank=2 imbalance_ratio=1.070679 max_load_reduction=56765 max_load_reduction_pct=0.866165 moved_to_echo=33023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2573, 17, 95, 95, 562, 0, 2786, 49, 2645, 0, 0, 0, 0, 0, 0, 0, 1073, 3379, 86, 462, 1274, 1262, 0, 220, 841, 72, 0, 0, 0, 0, 0, 0, 462, 799, 101, 1869, 0, 527, 868, 1385, 950, 392, 0, 0, 0, 0, 0, 0, 25, 21, 2682, 1074, 333, 175, 3032, 584, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1001, 2414, 417, 335, 188, 2593, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 1881, 0, 1323, 1940, 9, 972, 1357, 754, 0, 0, 0, 0, 0, 0, 0, 0, 3383, 50, 1941, 1, 17, 264, 3, 556, 1777, 403, 0, 0, 0, 0, 0, 0, 3, 147, 794, 1534, 829, 836, 1114, 2808, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8822, 8669, 7353, 7926, 8070, 8236, 8395, 8065] avg=8192.0000 max=8822 max_rank=0 min=7353 min_rank=2 imbalance_ratio=1.076904 max_load_reduction=56714 max_load_reduction_pct=0.865387 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2165, 33, 267, 49, 345, 0, 2334, 32, 2495, 0, 0, 0, 0, 0, 0, 0, 264, 2944, 61, 328, 1441, 1063, 0, 400, 768, 82, 0, 0, 0, 0, 0, 0, 669, 1301, 105, 1934, 0, 511, 1411, 1776, 866, 446, 0, 0, 0, 0, 0, 0, 20, 7, 2803, 1214, 305, 112, 2793, 797, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1103, 2622, 214, 217, 71, 2771, 963, 0, 0, 0, 0, 0, 0, 0, 0, 2595, 1, 1194, 1855, 0, 693, 1591, 835, 0, 0, 0, 0, 0, 0, 0, 0, 3216, 56, 1761, 1, 4, 163, 0, 754, 2007, 383, 0, 0, 0, 0, 0, 0, 0, 64, 562, 1722, 926, 1030, 1147, 2839, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7720, 7351, 9019, 8051, 7996, 8764, 8345, 8290] avg=8192.0000 max=9019 max_rank=2 min=7351 min_rank=1 imbalance_ratio=1.100952 max_load_reduction=56517 max_load_reduction_pct=0.862381 moved_to_echo=33395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[15744, 128, 10240, 14848, 128, 7040, 10752, 7296, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5760, 25728, 896, 2688, 9344, 12288, 128, 2176, 6528, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 1408, 5504, 12288, 7424, 8320, 8832, 22400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3200, 9728, 1152, 14336, 128, 5376, 8704, 12672, 7424, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[26112, 512, 14208, 128, 128, 2048, 128, 4224, 15744, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 7680, 20992, 2176, 2304, 896, 21248, 10496, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 128, 22784, 10368, 2560, 1280, 23424, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[20864, 256, 1536, 512, 2944, 128, 19200, 384, 20352, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 4, 4083, 0, 3814, 4096, 0, 240, 8, 0, 0, 2082, 2839, 0, 0, 5, 0, 0, 0, 39, 3440, 4074, 0, 4087, 0, 2, 1, 3787, 6, 1, 0, 0, 398, 4, 2, 1417, 4094, 20, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 6, 4096, 0, 15, 0, 4089, 4090, 2427, 0, 0, 0, 0, 4077] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 0, 3933, 0, 2995, 4093, 0, 4, 210, 0, 1, 1132, 2329, 1, 0, 0, 0, 1, 6, 40, 3296, 4056, 3, 4088, 0, 41, 0, 4086, 0, 0, 2, 0, 1434, 1, 0, 3322, 4094, 2, 0, 0, 0, 1, 0, 4095, 0, 0, 1, 9, 4095, 0, 18, 0, 4096, 4095, 2062, 2, 0, 1, 1, 3793] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4094, 0, 3936, 4096, 0, 38, 272, 0, 0, 1450, 2873, 0, 0, 0, 0, 0, 1, 29, 3062, 4062, 0, 4093, 2, 9, 0, 4077, 3, 0, 1, 0, 682, 0, 0, 2317, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 37, 0, 4096, 4096, 1736, 0, 0, 0, 0, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 3, 4056, 0, 3906, 4095, 0, 51, 203, 0, 0, 2214, 3434, 0, 1, 0, 0, 0, 0, 22, 2848, 4057, 3, 4093, 3, 18, 0, 4038, 3, 0, 1, 0, 512, 0, 0, 1218, 4092, 6, 0, 1, 0, 0, 0, 4095, 1, 4, 3, 4, 4095, 0, 51, 0, 4094, 4094, 2311, 1, 0, 0, 0, 3812] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 2, 1, 4093, 2, 3985, 0, 3546, 4095, 0, 33, 243, 0, 2, 1965, 2784, 0, 0, 2, 0, 0, 0, 1, 2563, 4066, 6, 4092, 3, 1, 8, 4042, 4, 0, 4, 0, 919, 0, 0, 2380, 4088, 0, 0, 0, 1, 0, 0, 4096, 2, 0, 0, 3, 4091, 0, 3, 0, 4096, 4092, 2581, 0, 0, 0, 0, 3640] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4065, 0, 3616, 4096, 0, 21, 140, 0, 0, 2000, 3136, 0, 0, 0, 0, 0, 0, 204, 2552, 4044, 5, 4082, 0, 35, 0, 3998, 4, 0, 0, 0, 432, 1, 0, 2261, 4094, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 90, 0, 4096, 4096, 2833, 0, 0, 0, 0, 3342] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 1, 3931, 0, 3660, 4092, 0, 26, 202, 1, 0, 824, 2555, 0, 0, 0, 0, 0, 3, 7, 2986, 4082, 4, 4095, 1, 24, 0, 4085, 0, 2, 0, 0, 2285, 0, 0, 2942, 4089, 0, 1, 0, 0, 0, 0, 4096, 0, 2, 0, 9, 4096, 0, 10, 0, 4079, 4096, 1414, 0, 0, 0, 0, 3741] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 20, 3953, 0, 3555, 4096, 0, 7, 359, 0, 0, 965, 2227, 0, 0, 0, 0, 0, 0, 74, 3053, 4067, 1, 4094, 0, 38, 0, 4094, 0, 0, 0, 0, 1617, 14, 0, 3147, 4083, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 4096, 0, 21, 0, 4096, 4070, 1617, 0, 0, 0, 0, 3974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 0, 3933, 0, 79, 0, 0, 0, 0, 0, 0, 0, 2995, 2728, 0, 4, 210, 0, 1, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 2329, 1, 0, 0, 0, 1, 6, 40, 4096, 1272, 0, 0, 0, 0, 0, 0, 3296, 1164, 3, 0, 0, 41, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1434, 1, 0, 3322, 2816, 1953, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 4095, 0, 18, 0, 1832, 1286, 939, 0, 0, 0, 0, 0, 0, 2263, 2062, 2, 0, 1, 1, 3793, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 7070, 7745, 8590, 9528, 8192, 8180, 8122] avg=8192.0000 max=9528 max_rank=4 min=7070 min_rank=1 imbalance_ratio=1.163086 max_load_reduction=56008 max_load_reduction_pct=0.854614 moved_to_echo=34022 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4094, 0, 79, 0, 0, 0, 0, 0, 0, 0, 3936, 2730, 0, 38, 272, 0, 0, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 0, 0, 0, 0, 0, 1, 29, 4096, 1273, 0, 0, 0, 0, 0, 0, 3062, 1166, 0, 0, 2, 9, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 682, 0, 0, 2317, 2820, 1956, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 37, 0, 1833, 1287, 940, 0, 0, 0, 0, 0, 0, 2263, 1736, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8426, 8272, 8316, 7779, 8192, 8192, 8090] avg=8192.0000 max=8426 max_rank=1 min=7779 min_rank=4 imbalance_ratio=1.028564 max_load_reduction=57110 max_load_reduction_pct=0.871429 moved_to_echo=32253 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 20, 3953, 0, 79, 0, 0, 0, 0, 0, 0, 0, 3555, 2730, 0, 7, 359, 0, 0, 965, 0, 0, 0, 0, 0, 0, 0, 0, 2227, 0, 0, 0, 0, 0, 0, 74, 4096, 1273, 0, 0, 0, 0, 0, 0, 3053, 1167, 1, 0, 0, 38, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1617, 14, 0, 3147, 2821, 1958, 0, 0, 0, 0, 0, 0, 4083, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 21, 0, 1821, 1287, 942, 0, 0, 0, 0, 0, 0, 2249, 1617, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 7616, 7670, 8353, 9557, 8180, 8172, 7840] avg=8192.0000 max=9557 max_rank=4 min=7616 min_rank=1 imbalance_ratio=1.166626 max_load_reduction=55979 max_load_reduction_pct=0.854172 moved_to_echo=33749 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 3, 4056, 0, 79, 0, 0, 0, 0, 0, 0, 0, 3906, 2729, 0, 51, 203, 0, 0, 2214, 0, 0, 0, 0, 0, 0, 0, 0, 3434, 0, 1, 0, 0, 0, 0, 22, 4094, 1273, 0, 0, 0, 0, 0, 0, 2848, 1165, 3, 0, 3, 18, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 512, 0, 0, 1218, 2820, 1953, 0, 0, 0, 0, 0, 0, 4092, 6, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 3, 4, 4095, 0, 51, 0, 1832, 1287, 939, 0, 0, 0, 0, 0, 0, 2262, 2311, 1, 0, 0, 0, 3812, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 9103, 8824, 8075, 6507, 8194, 8216, 8386] avg=8192.0000 max=9103 max_rank=1 min=6507 min_rank=4 imbalance_ratio=1.111206 max_load_reduction=56433 max_load_reduction_pct=0.861099 moved_to_echo=31303 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2, 1, 4093, 2, 3985, 0, 79, 0, 0, 0, 0, 0, 0, 0, 3546, 2729, 0, 33, 243, 0, 2, 1965, 0, 0, 0, 0, 0, 0, 0, 0, 2784, 0, 0, 2, 0, 0, 0, 1, 4096, 1274, 0, 0, 0, 0, 0, 0, 2563, 1167, 6, 0, 3, 1, 8, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 919, 0, 0, 2380, 2818, 1958, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 4091, 0, 3, 0, 1831, 1287, 941, 0, 0, 0, 0, 0, 0, 2261, 2581, 0, 0, 0, 0, 3640, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8518, 8157, 7790, 8083, 8185, 8158, 8482] avg=8192.0000 max=8518 max_rank=1 min=7790 min_rank=3 imbalance_ratio=1.039795 max_load_reduction=57018 max_load_reduction_pct=0.870026 moved_to_echo=32908 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4065, 0, 79, 0, 0, 0, 0, 0, 0, 0, 3616, 2730, 0, 21, 140, 0, 0, 2000, 0, 0, 0, 0, 0, 0, 0, 0, 3136, 0, 0, 0, 0, 0, 0, 204, 4096, 1270, 0, 0, 0, 0, 0, 0, 2552, 1161, 5, 0, 0, 35, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 432, 1, 0, 2261, 2812, 1947, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 90, 0, 1833, 1287, 936, 0, 0, 0, 0, 0, 0, 2263, 2833, 0, 0, 0, 0, 3342, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8507, 8706, 7751, 7457, 8192, 8245, 8438] avg=8192.0000 max=8706 max_rank=2 min=7457 min_rank=4 imbalance_ratio=1.062744 max_load_reduction=56830 max_load_reduction_pct=0.867157 moved_to_echo=32332 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 1, 3931, 0, 86, 0, 0, 0, 0, 0, 0, 0, 3660, 2709, 0, 26, 202, 1, 0, 824, 0, 0, 0, 0, 0, 0, 0, 0, 2555, 0, 0, 0, 0, 0, 3, 7, 4079, 1275, 0, 0, 0, 0, 0, 0, 2986, 1162, 4, 0, 1, 24, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2285, 0, 0, 2942, 2820, 1970, 0, 0, 0, 0, 0, 0, 4089, 0, 1, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 9, 4096, 0, 10, 0, 1837, 1291, 950, 0, 0, 0, 0, 0, 0, 2259, 1414, 0, 0, 0, 0, 3741, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 7422, 7919, 8262, 10019, 8192, 8195, 7414] avg=8192.0000 max=10019 max_rank=4 min=7414 min_rank=7 imbalance_ratio=1.223022 max_load_reduction=55517 max_load_reduction_pct=0.847122 moved_to_echo=33820 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 4, 4083, 0, 79, 0, 0, 0, 0, 0, 0, 0, 3814, 2730, 0, 240, 8, 0, 0, 2082, 0, 0, 0, 0, 0, 0, 0, 0, 2839, 0, 0, 5, 0, 0, 0, 39, 4089, 1271, 0, 0, 0, 0, 0, 0, 3440, 1169, 0, 0, 0, 2, 1, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 398, 4, 2, 1417, 2816, 1962, 0, 0, 0, 0, 0, 0, 4094, 20, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 4096, 0, 15, 0, 1830, 1287, 943, 0, 0, 0, 0, 0, 0, 2260, 2427, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8874, 8243, 8399, 6606, 8209, 8178, 8764] avg=8192.0000 max=8874 max_rank=1 min=6606 min_rank=4 imbalance_ratio=1.083252 max_load_reduction=56662 max_load_reduction_pct=0.864594 moved_to_echo=31757 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 18176, 17024, 128, 0, 128, 128, 30592, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23808, 9344, 128, 0, 128, 256, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 32128, 0, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[29056, 21888, 0, 512, 1664, 128, 128, 12672, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 256, 0, 14720, 10368, 7552, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 8320, 128, 128, 19072, 22656, 15744, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[22272, 128, 128, 128, 0, 128, 128, 512, 32768, 10240, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4021, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3969, 3, 0, 774, 148, 4096, 0, 4088, 0, 0, 0, 2153, 4096, 0, 0, 4091, 0, 4096, 0, 0, 76, 80, 4096, 0, 4096, 0, 22, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1052, 0, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 4094, 0, 4095, 0, 1, 4095, 0, 1, 4095, 1, 1, 4089, 0, 0, 246, 9, 4090, 0, 4095, 1, 0, 0, 3019, 4090, 1, 0, 4092, 9, 4094, 1, 1, 0, 581, 4095, 0, 4089, 0, 0, 1, 4095, 1, 1, 0, 1, 1, 4095, 1, 1, 255, 0, 0, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 3, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 360, 3, 4090, 0, 4096, 0, 0, 0, 2782, 4092, 1, 0, 4096, 14, 4095, 0, 0, 0, 74, 4095, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 875, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4075, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4040, 0, 0, 423, 103, 4095, 0, 4095, 0, 0, 0, 3051, 4096, 0, 1, 4095, 0, 4096, 0, 0, 0, 161, 4096, 0, 4096, 0, 19, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 418, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4082, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4086, 4, 0, 416, 119, 4094, 0, 4089, 0, 0, 0, 2467, 4096, 0, 4, 4096, 9, 4095, 0, 0, 0, 76, 4096, 0, 4096, 0, 7, 0, 4096, 0, 0, 0, 5, 0, 4094, 0, 0, 1022, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 4, 0, 0, 4096, 0, 0, 0, 4079, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 4087, 0, 0, 614, 55, 4091, 0, 4092, 0, 0, 0, 1978, 4096, 0, 0, 4096, 23, 4096, 0, 0, 2, 195, 4096, 0, 4096, 0, 7, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 1250, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 5, 4096, 0, 0, 0, 4028, 0, 4096, 0, 0, 4096, 4, 0, 4096, 0, 0, 4085, 4, 0, 446, 66, 4092, 0, 4086, 0, 0, 0, 2616, 4096, 0, 4, 4096, 3, 4096, 0, 0, 5, 165, 4091, 0, 4096, 0, 26, 0, 4093, 0, 0, 0, 9, 0, 4092, 0, 0, 844, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4055, 0, 4096, 0, 0, 4096, 1, 0, 4095, 0, 0, 4039, 8, 0, 299, 65, 4094, 0, 4084, 0, 0, 0, 2909, 4096, 0, 8, 4090, 5, 4096, 0, 0, 2, 133, 4094, 0, 4096, 0, 15, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 768, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4094, 0, 2625, 781, 509, 173, 0, 0, 0, 0, 0, 1, 4094, 0, 4095, 0, 1, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4089, 0, 0, 30, 1, 0, 0, 0, 0, 0, 0, 246, 9, 3581, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3019, 1465, 1, 0, 4092, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 581, 4095, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 1, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 255, 0, 0, 0, 2, 1, 4094, 3283, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8222, 8218, 7932, 8587, 8563, 8194, 7637] avg=8192.0000 max=8587 max_rank=4 min=7637 min_rank=7 imbalance_ratio=1.048218 max_load_reduction=56949 max_load_reduction_pct=0.868973 moved_to_echo=32981 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 2629, 781, 510, 173, 0, 0, 0, 0, 0, 0, 4021, 0, 4096, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3969, 3, 0, 30, 1, 0, 0, 0, 0, 0, 0, 774, 148, 3586, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2153, 1467, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 80, 4096, 0, 3893, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 0, 3, 0, 0, 0, 4096, 3284, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8148, 8099, 8596, 7711, 8167, 8191, 8435] avg=8192.0000 max=8596 max_rank=3 min=7711 min_rank=4 imbalance_ratio=1.049316 max_load_reduction=56940 max_load_reduction_pct=0.868835 moved_to_echo=32504 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 2627, 781, 509, 173, 0, 0, 0, 0, 0, 0, 4092, 3, 4096, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 30, 1, 0, 0, 0, 0, 0, 0, 360, 3, 3581, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2782, 1465, 1, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 4095, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 875, 0, 0, 0, 1, 0, 4095, 3284, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8222, 8223, 8040, 8358, 8061, 8191, 8255] avg=8192.0000 max=8358 max_rank=4 min=8040 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 2629, 781, 510, 173, 0, 0, 0, 0, 0, 0, 4075, 0, 4096, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4040, 0, 0, 30, 1, 0, 0, 0, 0, 0, 0, 423, 103, 3585, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3051, 1467, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 4096, 0, 3893, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 0, 0, 0, 0, 4096, 3284, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8202, 8167, 8206, 8614, 8169, 8191, 7798] avg=8192.0000 max=8614 max_rank=4 min=7798 min_rank=7 imbalance_ratio=1.051514 max_load_reduction=56922 max_load_reduction_pct=0.868561 moved_to_echo=32772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4, 0, 0, 4096, 0, 2629, 781, 509, 173, 0, 0, 0, 0, 0, 0, 4079, 0, 4096, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4087, 0, 0, 30, 1, 0, 0, 0, 0, 0, 0, 614, 55, 3582, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1978, 1467, 0, 0, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 195, 4096, 0, 3893, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 0, 0, 0, 0, 0, 4096, 3284, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8206, 8213, 8343, 7564, 8193, 8193, 8630] avg=8192.0000 max=8630 max_rank=7 min=7564 min_rank=4 imbalance_ratio=1.053467 max_load_reduction=56906 max_load_reduction_pct=0.868317 moved_to_echo=32580 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 4096, 0, 2629, 781, 509, 173, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 4, 0, 30, 1, 0, 0, 0, 0, 0, 0, 416, 119, 3585, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2467, 1467, 0, 4, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 4096, 0, 3893, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 5, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 0, 2, 0, 0, 0, 4095, 3284, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8209, 8217, 8209, 8043, 8072, 8194, 8403] avg=8192.0000 max=8403 max_rank=7 min=8043 min_rank=4 imbalance_ratio=1.025757 max_load_reduction=57133 max_load_reduction_pct=0.871780 moved_to_echo=32712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 5, 4096, 0, 2629, 781, 509, 173, 0, 0, 0, 0, 0, 0, 4028, 0, 4096, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 4085, 4, 0, 30, 1, 0, 0, 0, 0, 0, 0, 446, 66, 3583, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2616, 1467, 0, 4, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 165, 4091, 0, 3893, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 9, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 844, 0, 0, 0, 2, 0, 4096, 3284, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8155, 8220, 8181, 8186, 8180, 8193, 8226] avg=8192.0000 max=8226 max_rank=7 min=8155 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 4096, 0, 2635, 788, 513, 178, 0, 0, 0, 0, 0, 0, 4055, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4039, 8, 0, 31, 5, 0, 0, 0, 0, 0, 0, 299, 65, 3581, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2909, 1461, 0, 8, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 133, 4094, 0, 3887, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 768, 0, 1, 0, 0, 0, 4096, 3287, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8172, 8179, 8029, 8473, 8131, 8189, 8152] avg=8192.0000 max=8473 max_rank=4 min=8029 min_rank=3 imbalance_ratio=1.034302 max_load_reduction=57063 max_load_reduction_pct=0.870712 moved_to_echo=32945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32640, 128, 32768, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 32512, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 1536, 32768, 0, 31232, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[3584, 640, 28672, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[20992, 11776, 128, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 6528, 0, 128, 0, 128, 128, 32768, 26368, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 0, 128, 32768, 0, 21120, 6272, 4096, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4083, 4049, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 17, 7, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4083, 34, 0, 19, 0, 4096, 4085, 11, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 4087, 4062, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 4096, 0, 16, 3, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4090, 25, 0, 13, 0, 4096, 4086, 9, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 4084, 4075, 0, 0, 0, 4096, 0, 4096, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 30, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 4051, 23, 1, 21, 0, 4096, 4093, 16, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4059, 3872, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 184, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4035, 84, 0, 43, 0, 4096, 4076, 31, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4095, 0, 4096, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 4063, 12, 0, 29, 0, 4092, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4087, 1, 1, 4095, 4093, 4086, 0, 1, 0, 4089, 0, 4095, 4094, 0, 0, 4095, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 1, 1, 4095, 4, 0, 0, 0, 0, 5, 1, 4094, 0, 0, 0, 4092, 4095, 0, 1, 4095, 1, 0, 22, 0, 4089, 4095, 5, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4092, 4083, 3993, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 4096, 0, 98, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4077, 25, 5, 12, 0, 4096, 4079, 19, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4073, 4027, 0, 0, 0, 4096, 0, 4096, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4096, 0, 25, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4049, 42, 0, 79, 0, 4096, 4068, 21, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4084, 4075, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 30, 0, 0, 4016, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 4051, 23, 1, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8237, 8188, 8194, 8172, 8190, 8159, 8204] avg=8192.0000 max=8237 max_rank=1 min=8159 min_rank=6 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4059, 3872, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 184, 0, 0, 4016, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 4035, 84, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4076, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8009, 8195, 8194, 8326, 8192, 8225, 8202] avg=8192.0000 max=8326 max_rank=4 min=8009 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32945 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4087, 1, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4093, 4086, 0, 1, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 1, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4, 0, 0, 0, 4009, 30, 0, 0, 0, 0, 0, 0, 0, 5, 1, 4094, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 1, 4095, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4095, 5, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8259, 8194, 8193, 8140, 8192, 8182, 8190] avg=8192.0000 max=8259 max_rank=1 min=8140 min_rank=4 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4015, 30, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 4063, 12, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8270, 8192, 8194, 8141, 8196, 8167, 8188] avg=8192.0000 max=8270 max_rank=1 min=8141 min_rank=4 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4083, 4049, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 7, 0, 4016, 30, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 4083, 34, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4085, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8210, 8195, 8188, 8166, 8194, 8199, 8191] avg=8192.0000 max=8210 max_rank=1 min=8166 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=57326 max_load_reduction_pct=0.874725 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4087, 4062, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 16, 3, 0, 4016, 30, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 4090, 25, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4086, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8227, 8190, 8191, 8163, 8193, 8191, 8190] avg=8192.0000 max=8227 max_rank=1 min=8163 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 4083, 3993, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 98, 4, 0, 4016, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 4077, 25, 5, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4079, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8154, 8190, 8188, 8248, 8192, 8182, 8193] avg=8192.0000 max=8248 max_rank=4 min=8154 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 7, 5, 0, 0, 0, 0, 0, 0, 4073, 4027, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 25, 4, 0, 4023, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 4049, 42, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4068, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8170, 8192, 8194, 8180, 8187, 8231, 8178] avg=8192.0000 max=8231 max_rank=6 min=8170 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 32768, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32512, 0, 128, 32640, 256, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 128, 384, 128, 0, 32128, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 32384, 0, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 32768, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 32768, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 4095, 0, 1, 0, 0, 1, 1, 4096, 0, 4090, 1, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 12, 0, 4095, 1, 0, 0, 0, 4095, 1, 0, 0, 4093, 2, 0, 15, 4095, 1, 0, 4094, 4095, 4095, 1, 0, 1, 1, 0, 0, 4094, 1, 0, 0, 0, 4092, 0, 4076, 1, 4095, 4095, 0, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 10, 0, 0, 0, 4086, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 6, 0, 4080, 0, 0, 11, 4096, 103, 0, 0, 4096, 4, 0, 4096, 4089, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4005, 0, 4095, 4086, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 10, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 4096, 0, 1, 0, 0, 4088, 0, 0, 8, 4096, 38, 0, 1, 4096, 11, 0, 4096, 4089, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 4068, 0, 4096, 4083, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 8, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 0, 0, 2, 4096, 14, 0, 14, 4096, 4, 0, 4096, 4084, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4087, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 3993, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 22, 4096, 118, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4082, 0, 4096, 4075, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 9, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4088, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 7, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 4090, 0, 0, 6, 4096, 13, 1, 9, 4096, 3, 0, 4096, 4088, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 4083, 0, 4092, 4089, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 9, 0, 0, 0, 4064, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 4096, 0, 2, 0, 0, 4076, 0, 0, 14, 4096, 92, 0, 8, 4096, 24, 0, 4096, 4088, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4096, 0, 4043, 0, 4094, 4056, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 1, 0, 0, 1, 1, 4052, 37, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 1, 1, 1, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 12, 0, 4089, 3, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4095, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 15, 4030, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 6, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 1, 40, 4095, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8211, 8200, 8198, 8141, 8198, 8187, 8214] avg=8192.0000 max=8214 max_rank=7 min=8141 min_rank=4 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 4053, 37, 0, 0, 0, 0, 0, 0, 3993, 0, 4096, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 118, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 40, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8111, 8189, 8195, 8266, 8198, 8192, 8197] avg=8192.0000 max=8266 max_rank=4 min=8111 min_rank=1 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 10, 0, 0, 0, 4053, 37, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4088, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 38, 0, 1, 4030, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 40, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8201, 8188, 8192, 8184, 8191, 8193, 8191] avg=8192.0000 max=8201 max_rank=1 min=8184 min_rank=4 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 0, 0, 0, 4053, 37, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 9, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 40, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8212, 8197, 8199, 8127, 8198, 8192, 8224] avg=8192.0000 max=8224 max_rank=7 min=8127 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 10, 0, 0, 0, 4052, 37, 0, 0, 0, 0, 0, 0, 4086, 0, 4095, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4080, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 103, 0, 0, 4030, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 40, 4086, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8203, 8187, 8189, 8244, 8191, 8192, 8135] avg=8192.0000 max=8244 max_rank=4 min=8135 min_rank=7 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 8, 0, 0, 0, 4053, 37, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 14, 0, 14, 4030, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 40, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8207, 8189, 8185, 8160, 8186, 8192, 8223] avg=8192.0000 max=8223 max_rank=7 min=8160 min_rank=4 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 7, 0, 0, 0, 4049, 37, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4090, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 13, 1, 9, 4030, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 40, 4089, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8208, 8189, 8194, 8158, 8190, 8194, 8215] avg=8192.0000 max=8215 max_rank=7 min=8158 min_rank=4 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 9, 0, 0, 0, 4053, 43, 0, 0, 0, 0, 0, 0, 4064, 0, 4096, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4076, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 14, 4096, 92, 0, 8, 4022, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 34, 4056, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8183, 8197, 8184, 8256, 8184, 8194, 8137] avg=8192.0000 max=8256 max_rank=4 min=8137 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 384, 128, 128, 32256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 128, 384, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 0, 128, 128, 0, 128, 128, 32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 0, 32768, 128, 128, 128, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 32768, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 1651, 0, 0, 1, 4096, 0, 0, 0, 3999, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4077, 4096, 0, 1442, 0, 0, 4096, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1272, 0, 3928, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 873, 0, 0, 10, 4096, 0, 0, 0, 4070, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 2647, 0, 1, 4096, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1365, 0, 3321, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 1, 0, 1, 0, 4095, 0, 0, 0, 1, 960, 0, 1, 2, 4095, 1, 0, 0, 4078, 0, 0, 1, 4095, 1, 4095, 1, 1, 0, 0, 4095, 4095, 0, 1704, 1, 4, 4095, 1, 4095, 1, 1, 0, 4095, 4093, 1, 0, 7, 1, 1, 0, 4095, 0, 0, 4095, 1, 0, 1, 0, 2033, 0, 3495, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 287, 0, 1, 48, 4096, 0, 0, 0, 4048, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 4096, 0, 162, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 3849, 0, 3905, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 290, 0, 0, 33, 4096, 0, 0, 0, 4062, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 0, 4089, 4096, 0, 597, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 0, 0, 0, 0, 3409, 0, 3902, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 207, 0, 0, 30, 4096, 0, 0, 0, 4050, 0, 7, 0, 4096, 0, 4096, 0, 0, 0, 0, 4089, 4096, 0, 1037, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3641, 0, 3322, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 574, 0, 1, 15, 4096, 0, 0, 0, 4075, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 0, 4089, 4096, 0, 1692, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4095, 0, 0, 0, 0, 2728, 1, 3209, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4, 0, 0, 0, 4090, 0, 0, 0, 0, 535, 0, 1, 9, 4096, 0, 0, 0, 3967, 0, 9, 0, 4096, 0, 4096, 0, 0, 0, 0, 4086, 4096, 0, 439, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 3563, 0, 3773, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1651, 0, 0, 1, 4096, 3404, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3999, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1442, 0, 0, 4096, 1, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 0, 3928, 0, 0, 1165, 683, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 9152, 8143, 8182, 8421, 8192, 8193, 7048] avg=8192.0000 max=9152 max_rank=1 min=7048 min_rank=7 imbalance_ratio=1.117188 max_load_reduction=56384 max_load_reduction_pct=0.860352 moved_to_echo=31854 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 873, 0, 0, 10, 4096, 3404, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4070, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2647, 0, 1, 4096, 1, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1365, 0, 3321, 0, 1, 1165, 683, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8383, 8214, 8200, 9627, 8193, 8193, 6535] avg=8192.0000 max=9627 max_rank=4 min=6535 min_rank=7 imbalance_ratio=1.175171 max_load_reduction=55909 max_load_reduction_pct=0.853104 moved_to_echo=32548 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4095, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 960, 0, 1, 2, 4095, 3404, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4078, 0, 0, 1, 48, 0, 0, 0, 0, 0, 0, 0, 9, 1, 4095, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1704, 1, 4, 4095, 1, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 4093, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2033, 0, 3495, 0, 1, 1165, 682, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8463, 8223, 8202, 8686, 8198, 8194, 7377] avg=8192.0000 max=8686 max_rank=4 min=7377 min_rank=7 imbalance_ratio=1.060303 max_load_reduction=56850 max_load_reduction_pct=0.867462 moved_to_echo=32455 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 1, 48, 4096, 3404, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4048, 0, 4, 0, 48, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 4096, 0, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 0, 3905, 0, 8, 1165, 683, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 7836, 8196, 8188, 7140, 8192, 8190, 9610] avg=8192.0000 max=9610 max_rank=7 min=7140 min_rank=4 imbalance_ratio=1.173096 max_load_reduction=55926 max_load_reduction_pct=0.853363 moved_to_echo=33132 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 30, 4096, 3404, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4050, 0, 7, 0, 48, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 0, 0, 4096, 0, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3641, 0, 3322, 0, 1, 1165, 683, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7737, 8201, 8194, 8015, 8192, 8193, 8812] avg=8192.0000 max=8812 max_rank=7 min=7737 min_rank=1 imbalance_ratio=1.075684 max_load_reduction=56724 max_load_reduction_pct=0.865540 moved_to_echo=33212 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 33, 4096, 3404, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4062, 0, 4, 0, 48, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 597, 0, 0, 4096, 0, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3409, 0, 3902, 0, 3, 1165, 683, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7823, 8210, 8194, 7575, 8192, 8190, 9162] avg=8192.0000 max=9162 max_rank=7 min=7575 min_rank=4 imbalance_ratio=1.118408 max_load_reduction=56374 max_load_reduction_pct=0.860199 moved_to_echo=33119 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 1, 15, 4096, 3404, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4075, 0, 5, 0, 48, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1692, 0, 0, 4096, 0, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2728, 1, 3209, 0, 9, 1165, 683, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8090, 8224, 8194, 8670, 8192, 8188, 7795] avg=8192.0000 max=8670 max_rank=4 min=7795 min_rank=7 imbalance_ratio=1.058350 max_load_reduction=56866 max_load_reduction_pct=0.867706 moved_to_echo=32845 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4, 0, 0, 0, 4090, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 1, 9, 4096, 3411, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3967, 0, 9, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 0, 0, 4096, 0, 2867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3563, 0, 3773, 0, 5, 1171, 685, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8052, 8125, 8182, 7402, 8185, 8195, 9197] avg=8192.0000 max=9197 max_rank=7 min=7402 min_rank=4 imbalance_ratio=1.122681 max_load_reduction=56339 max_load_reduction_pct=0.859665 moved_to_echo=32979 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32768, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 5504, 0, 128, 256, 32768, 27264, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 21888, 128, 28928, 0, 128, 9344, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 0, 32384, 0, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 9728, 128, 128, 32768, 128, 23040, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 82, 0, 0, 4091, 4062, 5, 4095, 4095, 0, 0, 4094, 1, 4095, 0, 1, 135, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 1, 0, 0, 1, 1, 4092, 0, 0, 1, 3907, 0, 0, 0, 1, 3, 1, 4094, 1, 4095, 1, 0, 4095, 4095, 0, 0, 1, 1, 4, 0, 4095, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 4093, 4074, 2, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 478, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3636, 0, 3, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 67, 0, 0, 4050, 4070, 0, 4096, 4096, 0, 0, 4096, 0, 3998, 0, 0, 137, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 22, 0, 3, 4080, 3884, 0, 4096, 4094, 0, 0, 4096, 0, 4086, 0, 0, 415, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4091, 0, 0, 0, 3891, 5, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 0, 0, 4088, 3981, 0, 4096, 4093, 0, 0, 4096, 0, 4071, 0, 0, 425, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4092, 0, 0, 0, 3792, 4, 0, 0, 0, 7, 0, 4096, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 20, 0, 0, 4080, 3991, 0, 4096, 4096, 0, 0, 4096, 0, 4081, 0, 0, 545, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 4095, 0, 0, 0, 3644, 0, 0, 2, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 52, 0, 0, 4072, 4028, 0, 4096, 4096, 0, 0, 4096, 0, 4076, 0, 0, 385, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 3762, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 72, 0, 0, 4065, 3892, 0, 4096, 4096, 0, 0, 4096, 0, 4034, 0, 0, 868, 0, 0, 4096, 0, 0, 0, 29, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 4088, 0, 0, 0, 3423, 3, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 67, 0, 0, 4050, 3677, 394, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 4096, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 137, 0, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3769, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3768, 325, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 4096, 328, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8265, 7837, 8185, 8186, 8489, 8194, 8192] avg=8192.0000 max=8489 max_rank=5 min=7837 min_rank=2 imbalance_ratio=1.036255 max_load_reduction=57047 max_load_reduction_pct=0.870468 moved_to_echo=33061 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 82, 0, 0, 4091, 3675, 394, 0, 0, 0, 0, 0, 0, 4062, 5, 0, 4095, 0, 0, 99, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 135, 0, 0, 3701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 3768, 320, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 4092, 0, 0, 3768, 326, 0, 0, 0, 0, 0, 0, 1, 3907, 0, 0, 0, 1, 3, 1, 4095, 327, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8262, 7932, 8185, 8189, 8335, 8192, 8196] avg=8192.0000 max=8335 max_rank=5 min=7932 min_rank=2 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32912 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 4093, 3677, 394, 0, 0, 0, 0, 0, 0, 4074, 2, 0, 4096, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 478, 0, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3769, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3768, 325, 0, 0, 0, 0, 0, 0, 0, 3636, 0, 3, 0, 0, 0, 0, 4096, 328, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8271, 8276, 8185, 8189, 8063, 8193, 8192] avg=8192.0000 max=8276 max_rank=2 min=8063 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32637 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 20, 0, 0, 4080, 3677, 394, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 4096, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 545, 0, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 3769, 320, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4095, 0, 0, 3768, 325, 0, 0, 0, 0, 0, 0, 0, 3644, 0, 0, 2, 0, 0, 0, 4096, 328, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8186, 8328, 8198, 8197, 8070, 8194, 8192] avg=8192.0000 max=8328 max_rank=2 min=8070 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32653 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 22, 0, 3, 4080, 3677, 394, 0, 0, 0, 0, 0, 0, 3884, 0, 0, 4094, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 415, 0, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 3769, 320, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4091, 0, 0, 3769, 325, 0, 0, 0, 0, 0, 0, 0, 3891, 5, 0, 0, 0, 0, 0, 4096, 327, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8077, 8203, 8187, 8191, 8319, 8191, 8192] avg=8192.0000 max=8319 max_rank=5 min=8077 min_rank=1 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32893 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 0, 0, 4088, 3677, 394, 0, 0, 0, 0, 0, 0, 3981, 0, 0, 4093, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 425, 0, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 3769, 320, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4092, 0, 0, 3769, 327, 0, 0, 0, 0, 0, 0, 0, 3792, 4, 0, 0, 0, 7, 0, 4096, 327, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8173, 8198, 8188, 8194, 8226, 8191, 8193] avg=8192.0000 max=8226 max_rank=5 min=8173 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32804 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 52, 0, 0, 4072, 3677, 394, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 4096, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 385, 0, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3771, 320, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 3769, 325, 0, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 0, 0, 0, 0, 4096, 327, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8223, 8163, 8187, 8199, 8185, 8192, 8192] avg=8192.0000 max=8223 max_rank=1 min=8163 min_rank=2 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32768 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 72, 0, 0, 4065, 3680, 399, 5, 0, 0, 0, 0, 0, 3892, 0, 0, 4096, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 868, 0, 0, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 4096, 0, 0, 3771, 325, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4088, 0, 0, 3769, 325, 0, 0, 0, 0, 0, 0, 0, 3423, 3, 0, 0, 0, 0, 0, 4096, 327, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8079, 8599, 8221, 8191, 7849, 8189, 8187] avg=8192.0000 max=8599 max_rank=2 min=7849 min_rank=5 imbalance_ratio=1.049683 max_load_reduction=56937 max_load_reduction_pct=0.868790 moved_to_echo=32416 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32000, 128, 0, 32768, 0, 0, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 0, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32640, 0, 128, 3456, 0, 0, 29696, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 384, 0, 128, 32640, 29440, 3200, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 32768, 128, 0, 30208, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 0, 0, 30208, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 30208, 128, 128, 128, 128, 128, 128, 32768, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 753, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 3992, 4094, 0, 4096, 4096, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2857, 0, 0, 0, 0, 4096, 4096, 0, 0, 620, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 1723, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 4058, 4096, 0, 4096, 4096, 0, 4073, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 767, 0, 0, 0, 0, 4096, 4096, 0, 0, 1662, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1616, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4096, 4096, 4094, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1120, 0, 0, 0, 0, 4096, 4096, 0, 0, 1362, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 1, 0, 4095, 1, 4095, 1, 1, 0, 531, 1, 0, 4091, 0, 0, 1, 1, 0, 0, 4095, 4093, 4094, 4094, 4095, 1, 4095, 4095, 1, 4095, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4094, 1, 1, 0, 1442, 1, 0, 1, 0, 4095, 4095, 0, 1, 2127, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 1441, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4050, 4096, 0, 4096, 4096, 0, 4082, 0, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1932, 0, 0, 0, 0, 4096, 4096, 0, 0, 775, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 482, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 4093, 4096, 4096, 4060, 4096, 0, 4096, 4091, 0, 4074, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1626, 0, 0, 0, 0, 4096, 4096, 0, 0, 2041, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 971, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4091, 4096, 0, 4096, 4093, 0, 4085, 0, 6, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2157, 0, 0, 0, 0, 4096, 4096, 0, 0, 981, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4085, 0, 0, 0, 508, 0, 23, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4001, 4096, 0, 4096, 4093, 0, 4067, 0, 9, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2069, 0, 0, 0, 0, 4096, 4096, 0, 0, 1621, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 753, 0, 18, 3089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 3992, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 4094, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2857, 0, 0, 0, 1398, 950, 0, 0, 0, 0, 0, 0, 0, 2698, 4096, 0, 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7955, 8192, 8132, 8158, 8192, 9301, 7414] avg=8192.0000 max=9301 max_rank=6 min=7414 min_rank=7 imbalance_ratio=1.135376 max_load_reduction=56235 max_load_reduction_pct=0.858078 moved_to_echo=33065 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 1441, 0, 3, 3090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4050, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 7, 5, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1932, 0, 0, 0, 1398, 951, 0, 0, 0, 0, 0, 0, 0, 2698, 4096, 0, 0, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8623, 8192, 8189, 8202, 8192, 8377, 7569] avg=8192.0000 max=8623 max_rank=1 min=7569 min_rank=7 imbalance_ratio=1.052612 max_load_reduction=56913 max_load_reduction_pct=0.868423 moved_to_echo=32340 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1616, 0, 0, 3088, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1120, 0, 0, 0, 1398, 950, 0, 0, 0, 0, 0, 0, 0, 2698, 4096, 0, 0, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8800, 8192, 8236, 8204, 8192, 7564, 8156] avg=8192.0000 max=8800 max_rank=1 min=7564 min_rank=6 imbalance_ratio=1.074219 max_load_reduction=56736 max_load_reduction_pct=0.865723 moved_to_echo=32116 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 531, 1, 0, 3088, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 4094, 0, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 1, 0, 0, 0, 4095, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 4093, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 1442, 1, 0, 1, 1398, 950, 0, 0, 0, 0, 0, 0, 0, 2697, 4095, 0, 1, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7718, 8188, 8235, 8205, 8190, 7888, 8920] avg=8192.0000 max=8920 max_rank=7 min=7718 min_rank=1 imbalance_ratio=1.088867 max_load_reduction=56616 max_load_reduction_pct=0.863892 moved_to_echo=33203 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 971, 0, 0, 3090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4091, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 6, 0, 0, 0, 0, 3, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2157, 0, 0, 0, 1398, 951, 0, 0, 0, 0, 0, 0, 0, 2698, 4096, 0, 0, 981, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8151, 8192, 8227, 8202, 8192, 8602, 7778] avg=8192.0000 max=8602 max_rank=6 min=7778 min_rank=7 imbalance_ratio=1.050049 max_load_reduction=56934 max_load_reduction_pct=0.868744 moved_to_echo=32774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1723, 0, 6, 3089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 4058, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 1, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 767, 0, 0, 0, 1398, 950, 0, 0, 0, 0, 0, 0, 0, 2698, 4096, 0, 0, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8913, 8192, 8198, 8182, 8192, 7211, 8456] avg=8192.0000 max=8913 max_rank=1 min=7211 min_rank=6 imbalance_ratio=1.088013 max_load_reduction=56623 max_load_reduction_pct=0.863998 moved_to_echo=32041 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 482, 0, 7, 3090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4060, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4, 0, 0, 0, 5, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1626, 0, 0, 0, 1398, 951, 0, 0, 0, 0, 0, 0, 0, 2698, 4096, 0, 0, 2041, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7668, 8189, 8194, 8191, 8192, 8071, 8839] avg=8192.0000 max=8839 max_rank=7 min=7668 min_rank=1 imbalance_ratio=1.078979 max_load_reduction=56697 max_load_reduction_pct=0.865128 moved_to_echo=33293 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 508, 0, 23, 3092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4001, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 9, 0, 0, 0, 0, 1, 4096, 18, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2069, 0, 0, 0, 1402, 955, 0, 0, 0, 0, 0, 0, 0, 2687, 4096, 0, 0, 1621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7708, 8199, 8125, 8192, 8194, 8522, 8404] avg=8192.0000 max=8522 max_rank=6 min=7708 min_rank=1 imbalance_ratio=1.040283 max_load_reduction=57014 max_load_reduction_pct=0.869965 moved_to_echo=33312 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 14080, 128, 0, 128, 11264, 7680, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 384, 32512, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 32768, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 21632, 32768, 0, 128, 11264, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 0, 128, 128, 128, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 0, 8064, 128, 128, 24832, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 0, 0, 0, 3, 0, 8, 0, 0, 0, 0, 0, 1689, 4087, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2369, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 33, 4096, 0, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 2275, 4087, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1789, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 4096, 4096, 0, 0, 0, 4094, 0, 0, 4095, 4096, 0, 7, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 89, 4096, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 542, 4077, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3489, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4043, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 32, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 1, 0, 0, 4, 22, 0, 1, 4094, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 2774, 4095, 1, 1, 1, 5, 4095, 2, 1, 4095, 0, 1341, 4091, 0, 0, 1, 4095, 3, 0, 0, 0, 0, 4094, 0, 4092, 1, 4080, 4095, 1, 0, 0, 4095, 1, 1, 4095, 4064, 0, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 31, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2622, 4078, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1442, 4096, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 12, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 0, 0, 0, 4, 0, 9, 0, 0, 0, 0, 0, 1335, 4083, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2718, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4092, 4096, 0, 12, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2781, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1317, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4093, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 69, 4096, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 2073, 4083, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1959, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4058, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 23, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4077, 9, 7, 0, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 0, 0, 0, 2045, 2002, 6, 0, 0, 0, 0, 0, 3, 0, 8, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 1689, 4087, 0, 0, 0, 0, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2044, 0, 2369, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8191, 8194, 7870, 8509, 8192, 8201, 8190] avg=8192.0000 max=8509 max_rank=4 min=7870 min_rank=3 imbalance_ratio=1.038696 max_load_reduction=57027 max_load_reduction_pct=0.870163 moved_to_echo=33092 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4077, 8, 7, 0, 0, 0, 0, 0, 0, 0, 89, 4096, 0, 0, 0, 0, 2045, 2002, 6, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 542, 4077, 0, 0, 0, 0, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2044, 0, 3489, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8238, 8198, 6713, 9631, 8192, 8158, 8218] avg=8192.0000 max=9631 max_rank=4 min=6713 min_rank=3 imbalance_ratio=1.175659 max_load_reduction=55905 max_load_reduction_pct=0.853043 moved_to_echo=34199 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 1, 0, 0, 4, 4077, 9, 7, 0, 0, 0, 0, 0, 22, 0, 1, 4094, 0, 1, 1, 0, 2045, 2001, 6, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 4095, 4086, 0, 0, 0, 0, 0, 0, 2774, 4095, 1, 1, 1, 5, 2094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2043, 0, 1341, 4091, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 4080, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4089, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8171, 8184, 8973, 7477, 8192, 8192, 8155] avg=8192.0000 max=8973 max_rank=3 min=7477 min_rank=4 imbalance_ratio=1.095337 max_load_reduction=56563 max_load_reduction_pct=0.863083 moved_to_echo=32016 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 0, 0, 4075, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2045, 2002, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 2781, 4095, 0, 0, 0, 0, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2044, 0, 1317, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8149, 8183, 8970, 7457, 8193, 8210, 8186] avg=8192.0000 max=8970 max_rank=3 min=7457 min_rank=4 imbalance_ratio=1.094971 max_load_reduction=56566 max_load_reduction_pct=0.863129 moved_to_echo=32046 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4077, 9, 7, 0, 0, 0, 0, 0, 0, 0, 33, 4096, 0, 0, 0, 0, 2045, 2002, 6, 0, 0, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 2275, 4087, 0, 0, 0, 0, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2044, 0, 1789, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8182, 8189, 8456, 7929, 8192, 8207, 8192] avg=8192.0000 max=8456 max_rank=3 min=7929 min_rank=4 imbalance_ratio=1.032227 max_load_reduction=57080 max_load_reduction_pct=0.870972 moved_to_echo=32520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 0, 4096, 0, 0, 0, 0, 4077, 9, 7, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 0, 0, 0, 0, 2045, 2002, 6, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 2622, 4078, 0, 0, 0, 0, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2044, 0, 1442, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8180, 8186, 8794, 7585, 8192, 8198, 8198] avg=8192.0000 max=8794 max_rank=3 min=7585 min_rank=4 imbalance_ratio=1.073486 max_load_reduction=56742 max_load_reduction_pct=0.865814 moved_to_echo=32173 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4077, 9, 7, 0, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 0, 0, 0, 2045, 2002, 6, 0, 0, 0, 0, 0, 4, 0, 9, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 1335, 4083, 0, 0, 0, 0, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2044, 0, 2718, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8191, 8196, 7512, 8859, 8192, 8203, 8194] avg=8192.0000 max=8859 max_rank=4 min=7512 min_rank=3 imbalance_ratio=1.081421 max_load_reduction=56677 max_load_reduction_pct=0.864822 moved_to_echo=33448 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4083, 9, 10, 0, 0, 0, 0, 0, 0, 0, 69, 4096, 0, 0, 0, 0, 2052, 2004, 12, 1, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 2073, 4083, 0, 0, 0, 0, 2092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2034, 0, 1959, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8234, 8206, 8248, 8089, 8191, 8167, 8203] avg=8192.0000 max=8248 max_rank=3 min=8089 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32650 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 0, 128, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[16128, 32768, 128, 128, 128, 128, 16768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32640, 128, 32768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 384, 32768, 0, 128, 128, 0, 16384, 16128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 16384, 0, 16512, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 4095, 1, 0, 1, 1, 0, 4095, 0, 1, 0, 34, 2, 4092, 4095, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 4092, 4095, 0, 1, 239, 4, 1, 0, 4095, 0, 4094, 0, 1, 0, 0, 1, 0, 4095, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 1, 4094, 3822, 4095, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 1, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 13, 4092, 0, 1, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 5, 4087, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 5, 4095, 0, 7, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 1, 4092, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 4, 4090, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 6, 4092, 0, 4, 0, 0, 0, 0, 4096, 4096, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 3, 4090, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 6, 4092, 0, 4, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 1, 4084, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 1, 0, 0, 0, 4088, 0, 0, 7, 4090, 0, 11, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4095, 4095, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 34, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 4092, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 239, 4, 1, 0, 4095, 0, 4060, 3, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 1, 0, 4095, 3, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 3822, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8221, 8194, 8188, 8403, 8196, 8185, 7952] avg=8192.0000 max=8403 max_rank=4 min=7952 min_rank=7 imbalance_ratio=1.025757 max_load_reduction=57133 max_load_reduction_pct=0.871780 moved_to_echo=32736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4061, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4092, 0, 1, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8189, 8191, 8189, 8161, 8187, 8196, 8227] avg=8192.0000 max=8227 max_rank=7 min=8161 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 5, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4061, 4, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4088, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4095, 0, 7, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8189, 8192, 8197, 8161, 8183, 8196, 8227] avg=8192.0000 max=8227 max_rank=7 min=8161 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4061, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8189, 8191, 8189, 8161, 8198, 8187, 8227] avg=8192.0000 max=8227 max_rank=7 min=8161 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 3, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4061, 4, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4092, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4092, 0, 4, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8189, 8192, 8193, 8161, 8191, 8191, 8227] avg=8192.0000 max=8227 max_rank=7 min=8161 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4061, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8189, 8192, 8189, 8161, 8195, 8191, 8227] avg=8192.0000 max=8227 max_rank=7 min=8161 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 4, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4061, 4, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4092, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4092, 0, 4, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 35, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8188, 8192, 8195, 8161, 8188, 8191, 8228] avg=8192.0000 max=8228 max_rank=7 min=8161 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4066, 5, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 4088, 10, 4, 1, 0, 0, 0, 0, 0, 0, 0, 7, 4090, 0, 11, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8182, 8192, 8196, 8167, 8198, 8199, 8221] avg=8192.0000 max=8221 max_rank=7 min=8167 min_rank=4 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 32512, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 0, 128, 128, 0, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 256, 128, 128, 0, 32768, 0, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4059, 1577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 2454, 39, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 88, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4077, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 4096, 4096, 0, 3934, 14, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 6, 0, 18, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4087, 0, 7, 1, 1, 0, 4095, 4095, 1042, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 2, 1, 11, 0, 4095, 0, 1, 4094, 4093, 0, 3100, 1, 2, 4095, 4095, 4071, 0, 0, 0, 0, 1, 1, 4095, 4094, 3, 0, 1, 0, 1, 4094, 4095, 0, 0, 1, 4092, 1, 0, 0, 1, 1, 0, 1, 1, 0, 4060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4075, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 4096, 4096, 0, 3625, 19, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 69, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4073] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4092, 4060, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 4096, 4096, 0, 3710, 1, 0, 4096, 4096, 4096, 0, 1, 0, 0, 0, 0, 4096, 4086, 0, 0, 4, 0, 40, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3807, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4092, 4084, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 4096, 4096, 0, 3340, 4, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 4, 0, 21, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4092, 4058, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 4096, 4096, 0, 3427, 21, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 4, 0, 65, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 0, 0, 0, 0, 0, 4095, 3437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3442, 619, 32, 0, 0, 0, 0, 0, 1, 0, 654, 4096, 0, 2454, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 4092, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4062, 3455, 654, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 9109, 8189, 7244, 8197, 8244, 8193, 8183] avg=8192.0000 max=9109 max_rank=1 min=7244 min_rank=3 imbalance_ratio=1.111938 max_load_reduction=56427 max_load_reduction_pct=0.861008 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 7, 1, 1, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1, 1, 0, 0, 0, 1, 1, 4095, 3437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 11, 0, 4095, 3440, 618, 32, 0, 0, 0, 0, 0, 0, 1, 654, 4093, 0, 3100, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 4071, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4063, 4094, 3, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 4060, 3447, 654, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8578, 8199, 7851, 8171, 8163, 8188, 8165] avg=8192.0000 max=8578 max_rank=1 min=7851 min_rank=3 imbalance_ratio=1.047119 max_load_reduction=56958 max_load_reduction_pct=0.869110 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 4095, 3437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3442, 619, 32, 0, 0, 0, 0, 0, 9, 0, 654, 4096, 0, 3934, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 4088, 0, 0, 6, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3455, 654, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 7689, 8189, 8707, 8197, 8176, 8193, 8190] avg=8192.0000 max=8707 max_rank=3 min=7689 min_rank=1 imbalance_ratio=1.062866 max_load_reduction=56829 max_load_reduction_pct=0.867142 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 4095, 3437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3442, 619, 32, 0, 0, 0, 0, 0, 0, 0, 654, 4096, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3455, 654, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 7826, 8189, 8557, 8192, 8160, 8193, 8205] avg=8192.0000 max=8557 max_rank=3 min=7826 min_rank=1 imbalance_ratio=1.044556 max_load_reduction=56979 max_load_reduction_pct=0.869431 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0, 4096, 3439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3442, 619, 32, 0, 0, 0, 0, 0, 4, 0, 654, 4096, 0, 3625, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 4077, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4073, 3455, 657, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7965, 8189, 8398, 8192, 8210, 8192, 8197] avg=8192.0000 max=8398 max_rank=3 min=7965 min_rank=1 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 0, 0, 4092, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 0, 0, 0, 0, 4096, 3437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3442, 619, 32, 0, 0, 0, 0, 0, 9, 0, 654, 4096, 0, 3710, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 4086, 0, 0, 4, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4085, 3455, 654, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 7920, 8189, 8470, 8198, 8194, 8192, 8199] avg=8192.0000 max=8470 max_rank=3 min=7920 min_rank=1 imbalance_ratio=1.033936 max_load_reduction=57066 max_load_reduction_pct=0.870758 moved_to_echo=32783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 0, 0, 0, 4092, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 746, 0, 0, 0, 0, 0, 0, 0, 4095, 3437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3442, 619, 32, 0, 0, 0, 0, 0, 5, 0, 654, 4096, 0, 3340, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 4091, 0, 0, 4, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4087, 3455, 654, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8278, 8189, 8099, 8197, 8180, 8193, 8202] avg=8192.0000 max=8278 max_rank=1 min=8099 min_rank=3 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 0, 0, 0, 0, 4092, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 633, 0, 0, 0, 0, 0, 0, 0, 4096, 3442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3445, 626, 36, 0, 0, 0, 0, 0, 15, 0, 651, 4096, 0, 3427, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4080, 0, 0, 4, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4068, 3456, 654, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8171, 8203, 8210, 8192, 8209, 8192, 8195] avg=8192.0000 max=8210 max_rank=3 min=8164 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=57326 max_load_reduction_pct=0.874725 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 32640, 27648, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5376, 128, 128, 0, 0, 0, 128, 128, 32768, 27520, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 0, 128, 128, 128, 0, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 32768, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 32768, 27648, 4992, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 5248, 32768, 0, 27520, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32512, 32768, 128, 0, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 24, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1239, 0, 211, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4084, 1, 4096, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 492, 0, 12, 3607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4093, 0, 7, 1, 4077, 1, 0, 0, 3665, 0, 0, 0, 0, 4095, 4095, 10, 4089, 4090, 0, 0, 0, 271, 0, 1, 4095, 0, 0, 1, 2, 4094, 4095, 0, 1, 0, 1, 1, 0, 0, 0, 0, 4095, 4088, 0, 4095, 1271, 1, 249, 2753, 0, 0, 1, 1, 1, 4, 0, 0, 1, 4095, 2, 4089, 0, 4, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4087, 21, 4096, 4096, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4096, 2238, 0, 149, 1685, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 4091, 4085, 8, 4096, 4096, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 4096, 1543, 0, 46, 2500, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 8, 4096, 4096, 0, 0, 0, 24, 0, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 124, 0, 118, 3822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 4091, 7, 4096, 4096, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 1377, 0, 227, 2481, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 4076, 8, 4096, 4096, 0, 0, 0, 19, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 231, 0, 64, 3790, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 7, 1, 4077, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3665, 0, 0, 0, 0, 64, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4089, 4039, 0, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 2896, 4088, 0, 0, 1271, 1, 249, 0, 0, 0, 0, 0, 0, 0, 0, 2753, 0, 0, 1, 1, 1, 4, 0, 4031, 1199, 51, 0, 0, 0, 0, 0, 0, 1, 4094, 2, 4089, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 7824, 8409, 8193, 8193, 8505, 8041, 8191] avg=8192.0000 max=8505 max_rank=5 min=7824 min_rank=1 imbalance_ratio=1.038208 max_load_reduction=57031 max_load_reduction_pct=0.870224 moved_to_echo=32930 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 64, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4044, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 2897, 4096, 0, 0, 124, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 0, 0, 0, 0, 4032, 1199, 52, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8255, 8172, 8193, 8191, 7235, 9106, 8191] avg=8192.0000 max=9106 max_rank=6 min=7235 min_rank=5 imbalance_ratio=1.111572 max_load_reduction=56430 max_load_reduction_pct=0.861053 moved_to_echo=32723 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 64, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 4044, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2897, 4096, 2, 0, 2238, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 1685, 0, 0, 0, 0, 0, 0, 0, 4031, 1199, 52, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8247, 8169, 8192, 8192, 9382, 6967, 8194] avg=8192.0000 max=9382 max_rank=5 min=6967 min_rank=6 imbalance_ratio=1.145264 max_load_reduction=56154 max_load_reduction_pct=0.856842 moved_to_echo=32735 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 64, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4044, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2897, 4096, 0, 0, 492, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3607, 0, 0, 0, 0, 0, 0, 0, 4032, 1199, 52, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8237, 8143, 8192, 8193, 7497, 8890, 8191] avg=8192.0000 max=8890 max_rank=6 min=7497 min_rank=5 imbalance_ratio=1.085205 max_load_reduction=56646 max_load_reduction_pct=0.864349 moved_to_echo=32771 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 64, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4044, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2897, 4096, 4, 0, 1377, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 0, 0, 0, 0, 0, 4028, 1199, 52, 0, 0, 0, 0, 0, 4, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8251, 8152, 8192, 8192, 8601, 7760, 8195] avg=8192.0000 max=8601 max_rank=5 min=7760 min_rank=6 imbalance_ratio=1.049927 max_load_reduction=56935 max_load_reduction_pct=0.868759 moved_to_echo=32748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 64, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4044, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2897, 4096, 5, 0, 1543, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 2500, 0, 0, 0, 0, 0, 0, 0, 4027, 1199, 52, 0, 0, 0, 0, 0, 4, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8238, 8161, 8192, 8192, 8587, 7778, 8195] avg=8192.0000 max=8587 max_rank=5 min=7778 min_rank=6 imbalance_ratio=1.048218 max_load_reduction=56949 max_load_reduction_pct=0.868973 moved_to_echo=32752 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 59, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4043, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2891, 4096, 4, 0, 231, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 3790, 0, 0, 0, 0, 0, 0, 0, 4033, 1205, 53, 4, 0, 0, 0, 0, 4, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8231, 8166, 8190, 8192, 7286, 9085, 8188] avg=8192.0000 max=9085 max_rank=6 min=7286 min_rank=5 imbalance_ratio=1.109009 max_load_reduction=56451 max_load_reduction_pct=0.861374 moved_to_echo=32751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 64, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 2897, 4096, 0, 0, 1239, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 2625, 0, 0, 0, 0, 0, 0, 0, 4032, 1199, 52, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8253, 8164, 8192, 8191, 8443, 7909, 8191] avg=8192.0000 max=8443 max_rank=5 min=7909 min_rank=6 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32734 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 0, 128, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 32768, 32384, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 32384, 0, 0, 0, 0, 512, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 23296, 32768, 128, 0, 8576, 128, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[23296, 0, 0, 128, 128, 128, 128, 0, 32256, 9600, 512, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4070, 0, 0, 0, 0, 691, 0, 0, 0, 0, 4096, 0, 3038, 0, 0, 0, 3682, 0, 0, 4096, 0, 772, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 4096, 0, 6, 4096, 4096, 33, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4073, 0, 0, 0, 0, 810, 0, 0, 0, 0, 4096, 0, 3289, 0, 0, 0, 3728, 0, 0, 4096, 0, 344, 0, 10, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 17, 4087, 1, 0, 7, 4096, 0, 8, 4095, 4096, 20, 0, 4092, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4084, 0, 0, 0, 0, 310, 0, 0, 0, 0, 4096, 0, 3196, 0, 0, 0, 3955, 0, 0, 4096, 0, 715, 0, 6, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 9, 4096, 0, 4, 4095, 4096, 19, 0, 4092, 0, 0, 0, 4095, 4096, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 1, 4095, 1, 0, 1, 0, 137, 0, 1, 0, 0, 4095, 0, 2952, 2, 0, 1, 2650, 1, 1, 4088, 1, 2415, 1, 3, 4093, 2, 0, 1, 4095, 4095, 0, 0, 4092, 0, 1, 0, 5, 4, 33, 4090, 0, 1, 0, 4092, 0, 0, 4092, 4094, 0, 1, 4095, 1, 0, 1, 4094, 4095, 0, 10, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4041, 0, 0, 0, 0, 123, 0, 0, 0, 0, 4096, 0, 2988, 0, 0, 0, 3462, 0, 0, 4096, 0, 1574, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 4096, 0, 15, 4096, 4096, 70, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 224, 0, 0, 0, 0, 4096, 0, 2998, 0, 0, 0, 3894, 0, 0, 4096, 0, 1060, 0, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 7, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4077, 0, 0, 0, 0, 592, 0, 0, 0, 0, 4096, 0, 2617, 0, 0, 0, 3371, 0, 0, 4096, 0, 1586, 0, 6, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 5, 4096, 0, 18, 4095, 4096, 27, 0, 4095, 0, 0, 0, 4092, 4096, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4044, 0, 0, 0, 0, 126, 0, 0, 0, 0, 4096, 0, 2610, 0, 0, 0, 3040, 0, 0, 4096, 0, 2363, 0, 9, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 14, 4084, 1, 0, 5, 4096, 0, 31, 4092, 4096, 61, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 4095, 1, 0, 1, 0, 4094, 23, 0, 0, 0, 0, 0, 0, 137, 0, 1, 0, 0, 4095, 0, 2952, 758, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 2650, 1, 1, 4088, 1, 576, 36, 9, 1, 0, 0, 0, 0, 2415, 1, 3, 4093, 2, 0, 1, 2738, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4092, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4, 33, 4090, 0, 1, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 4059, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4094, 0, 10, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 7943, 7366, 9253, 8193, 8211, 8153, 8201] avg=8192.0000 max=9253 max_rank=3 min=7366 min_rank=2 imbalance_ratio=1.129517 max_load_reduction=56283 max_load_reduction_pct=0.858810 moved_to_echo=32758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4073, 0, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 0, 0, 810, 0, 0, 0, 0, 4096, 0, 3289, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 0, 4096, 0, 576, 36, 9, 1, 0, 0, 0, 0, 344, 0, 10, 4096, 0, 0, 0, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4087, 1, 0, 7, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 20, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8953, 8446, 7189, 8192, 8199, 8179, 8186] avg=8192.0000 max=8953 max_rank=1 min=7189 min_rank=3 imbalance_ratio=1.092896 max_load_reduction=56583 max_load_reduction_pct=0.863388 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4041, 0, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 4096, 0, 2988, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3462, 0, 0, 4096, 0, 576, 36, 9, 1, 0, 0, 0, 0, 1574, 0, 0, 4096, 0, 0, 0, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 70, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 7965, 8180, 8409, 8192, 8198, 8241, 8191] avg=8192.0000 max=8409 max_rank=3 min=7965 min_rank=1 imbalance_ratio=1.026489 max_load_reduction=57127 max_load_reduction_pct=0.871689 moved_to_echo=32822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4070, 0, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 0, 0, 691, 0, 0, 0, 0, 4096, 0, 3038, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3682, 0, 0, 4096, 0, 576, 36, 9, 1, 0, 0, 0, 0, 772, 0, 0, 4096, 0, 0, 0, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 33, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8583, 8400, 7607, 8192, 8179, 8195, 8191] avg=8192.0000 max=8583 max_rank=1 min=7607 min_rank=3 imbalance_ratio=1.047729 max_load_reduction=56953 max_load_reduction_pct=0.869034 moved_to_echo=32757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 4096, 0, 2998, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3894, 0, 0, 4096, 0, 576, 36, 9, 1, 0, 0, 0, 0, 1060, 0, 1, 4096, 0, 0, 0, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8076, 8612, 7896, 8192, 8193, 8156, 8198] avg=8192.0000 max=8612 max_rank=2 min=7896 min_rank=3 imbalance_ratio=1.051270 max_load_reduction=56924 max_load_reduction_pct=0.868591 moved_to_echo=32739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4084, 0, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 0, 0, 310, 0, 0, 0, 0, 4096, 0, 3196, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 4096, 0, 576, 36, 9, 1, 0, 0, 0, 0, 715, 0, 6, 4096, 0, 0, 0, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 9, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 19, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8360, 8673, 7556, 8192, 8186, 8174, 8192] avg=8192.0000 max=8673 max_rank=2 min=7556 min_rank=3 imbalance_ratio=1.058716 max_load_reduction=56863 max_load_reduction_pct=0.867661 moved_to_echo=32744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4077, 0, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 0, 0, 592, 0, 0, 0, 0, 4096, 0, 2617, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3371, 0, 0, 4096, 0, 576, 36, 9, 1, 0, 0, 0, 0, 1586, 0, 6, 4096, 0, 0, 0, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 5, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4095, 0, 27, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8063, 8089, 8427, 8192, 8182, 8199, 8188] avg=8192.0000 max=8427 max_rank=3 min=8063 min_rank=1 imbalance_ratio=1.028687 max_load_reduction=57109 max_load_reduction_pct=0.871414 moved_to_echo=32761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4044, 0, 0, 0, 0, 4096, 27, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 4096, 0, 2610, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3040, 0, 0, 4096, 0, 577, 41, 12, 3, 1, 0, 0, 0, 2363, 0, 9, 4096, 0, 0, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4084, 1, 0, 5, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4092, 0, 61, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 7593, 7770, 9199, 8191, 8188, 8239, 8189] avg=8192.0000 max=9199 max_rank=3 min=7593 min_rank=1 imbalance_ratio=1.122925 max_load_reduction=56337 max_load_reduction_pct=0.859634 moved_to_echo=32807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 256, 128, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 32640, 128, 0, 128, 0, 32768, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 32768, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[10880, 128, 128, 32768, 128, 0, 128, 22016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3072, 0, 128, 0, 0, 32768, 0, 23808, 6144, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 27904, 128, 128, 32768, 128, 4736, 384, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 32768, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66816 slack_slots=3456 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4092, 4096, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4078, 62, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4033, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4095, 4095, 4084, 0, 1, 1, 0, 4095, 1, 0, 0, 1, 4095, 0, 204, 0, 0, 4022, 4095, 4095, 0, 1, 0, 4, 0, 1, 4095, 1, 0, 0, 1, 3, 2, 3989, 1, 1, 4089, 0, 1, 1, 2, 0, 0, 4095, 0, 4094, 4094, 0, 1, 28, 1, 0, 1, 962, 0, 3089, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 4095, 4096, 4090, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 33, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 118, 0, 3946, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4093, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4096, 0, 63, 0, 0, 4096, 4096, 4091, 0, 0, 2, 0, 13, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 72, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 49, 0, 3912, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4092, 4096, 4089, 0, 0, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 61, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 4036, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 30, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4066, 148, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 43, 0, 3890, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 4096, 0, 2, 0, 0, 4096, 4096, 4087, 0, 0, 0, 0, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4062, 134, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 29, 0, 3931, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4095, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 0, 0, 1, 4005, 86, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 4022, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 1, 4095, 1, 4006, 49, 21, 3, 0, 0, 0, 0, 0, 0, 1, 3, 2, 3989, 1, 1, 4017, 90, 0, 0, 0, 0, 0, 0, 4089, 0, 1, 1, 2, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4094, 0, 1, 28, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 962, 0, 3089, 0, 0, 1, 0, 4095, 78, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8190, 8300, 8181, 8104, 8139, 8215, 8226] avg=8192.0000 max=8300 max_rank=2 min=8104 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32684 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4005, 86, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4007, 49, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4018, 91, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 3, 0, 0, 0, 4096, 78, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8187, 8165, 8176, 8205, 8146, 8189, 8273] avg=8192.0000 max=8273 max_rank=7 min=8146 min_rank=5 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32813 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4005, 86, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4095, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4096, 0, 4007, 49, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4018, 91, 0, 0, 0, 0, 0, 0, 4066, 148, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 3890, 0, 0, 0, 0, 4096, 78, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8187, 8186, 8206, 8205, 8261, 8189, 8107] avg=8192.0000 max=8261 max_rank=5 min=8107 min_rank=7 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32762 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 4005, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4096, 0, 4007, 49, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4018, 91, 0, 0, 0, 0, 0, 0, 4082, 61, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4036, 0, 0, 0, 0, 4096, 78, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8195, 8160, 8190, 8205, 8190, 8189, 8213] avg=8192.0000 max=8213 max_rank=7 min=8160 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32797 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4006, 89, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 4007, 49, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4018, 90, 0, 0, 0, 0, 0, 0, 4078, 62, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4033, 0, 0, 0, 0, 4096, 78, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8172, 8192, 8204, 8187, 8189, 8209] avg=8192.0000 max=8209 max_rank=7 min=8172 min_rank=2 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32789 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4006, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 4007, 49, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4018, 90, 0, 0, 0, 0, 0, 0, 4089, 33, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 3946, 0, 0, 0, 0, 4096, 78, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8194, 8164, 8183, 8204, 8169, 8189, 8238] avg=8192.0000 max=8238 max_rank=7 min=8164 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32800 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4006, 86, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 4096, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 13, 0, 4096, 0, 4007, 49, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4018, 90, 0, 0, 0, 0, 0, 0, 4083, 72, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 3912, 0, 2, 0, 0, 4096, 78, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8229, 8191, 8204, 8202, 8189, 8137] avg=8192.0000 max=8229 max_rank=2 min=8137 min_rank=7 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32732 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 4006, 89, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4096, 0, 4007, 50, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4019, 90, 0, 0, 0, 0, 0, 0, 4062, 134, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 3931, 0, 0, 0, 0, 4096, 77, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8200, 8160, 8217, 8205, 8242, 8187, 8133] avg=8192.0000 max=8242 max_rank=5 min=8133 min_rank=7 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32767 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 384, 0, 0, 32768, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 640, 128, 128, 128, 0, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 1280, 0, 30976, 128, 128, 128, 0, 32768, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 32768, 128, 128, 32256, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 128, 32768, 128, 32128, 512, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 128, 128, 32128, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[205, 684, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 4092, 15, 0, 0, 0, 1, 0, 4095, 113, 0, 0, 0, 4095, 1, 1, 0, 3440, 0, 4095, 4094, 0, 4095, 4094, 1, 3820, 4094, 4, 4095, 0, 0, 0, 2, 4019, 0, 1, 4095, 0, 0, 1, 0, 4094, 0, 1, 1, 1, 0, 0, 4083, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[662, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 205, 0, 0, 0, 13, 4096, 0, 7, 0, 0, 4096, 0, 0, 0, 2196, 1, 4096, 4094, 0, 4096, 4076, 0, 3943, 4096, 0, 4092, 0, 1, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[329, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 0, 13, 4093, 0, 22, 0, 0, 4096, 0, 0, 0, 3051, 3, 4096, 4093, 0, 4096, 4072, 0, 4063, 4096, 0, 4090, 0, 3, 0, 0, 4096, 7, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[410, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 240, 0, 0, 0, 35, 4096, 0, 103, 0, 0, 4096, 0, 0, 0, 3138, 0, 4096, 4096, 0, 4096, 4090, 0, 3855, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1527, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 95, 0, 0, 0, 22, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 2305, 0, 4096, 4096, 0, 4096, 4075, 0, 3923, 4096, 0, 4089, 0, 2, 0, 0, 4096, 8, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[831, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 173, 0, 0, 0, 1, 4096, 0, 13, 0, 0, 4096, 0, 0, 0, 2617, 0, 4096, 4096, 0, 4096, 4085, 0, 3983, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[665, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 88, 0, 0, 0, 4, 4096, 0, 10, 0, 0, 4096, 0, 0, 0, 2772, 0, 4096, 4096, 0, 4096, 4093, 0, 4056, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[521, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 86, 0, 0, 0, 20, 4096, 0, 21, 0, 0, 4096, 0, 0, 0, 3309, 1, 4096, 4096, 0, 4096, 4057, 0, 4016, 4096, 0, 4086, 0, 3, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[205, 684, 1, 1, 0, 1, 0, 0, 4095, 2854, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4092, 15, 2852, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 113, 0, 2688, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 3440, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 4094, 1, 3820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 2, 4019, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 4083, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7841, 8205, 8139, 8778, 8071, 8128, 8191, 8183] avg=8192.0000 max=8778 max_rank=3 min=7841 min_rank=0 imbalance_ratio=1.071533 max_load_reduction=56758 max_load_reduction_pct=0.866058 moved_to_echo=32573 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[410, 417, 0, 0, 0, 0, 0, 0, 4096, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2852, 1242, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 35, 4096, 0, 103, 2689, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3138, 0, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 4090, 0, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7777, 8190, 8407, 8476, 8101, 8200, 8192, 8193] avg=8192.0000 max=8476 max_rank=3 min=7777 min_rank=0 imbalance_ratio=1.034668 max_load_reduction=57060 max_load_reduction_pct=0.870667 moved_to_echo=32686 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1527, 337, 0, 0, 0, 0, 0, 0, 4096, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2852, 1242, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 22, 4096, 1, 0, 2689, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2305, 0, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 4075, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 2, 0, 0, 4096, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8814, 8190, 8147, 7643, 8154, 8203, 8192, 8193] avg=8192.0000 max=8814 max_rank=0 min=7643 min_rank=3 imbalance_ratio=1.075928 max_load_reduction=56722 max_load_reduction_pct=0.865509 moved_to_echo=32742 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[665, 601, 0, 0, 0, 0, 0, 0, 4096, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2852, 1242, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 4, 4096, 0, 10, 2689, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2772, 0, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 4093, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8190, 8131, 8110, 8305, 8199, 8192, 8193] avg=8192.0000 max=8305 max_rank=4 min=8110 min_rank=3 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32889 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[662, 1187, 0, 0, 0, 0, 0, 0, 4096, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2853, 1242, 0, 0, 0, 0, 0, 0, 205, 0, 0, 0, 13, 4096, 0, 7, 2688, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2196, 1, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 4076, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 0, 4096, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8799, 8191, 8252, 7534, 8175, 8200, 8192, 8193] avg=8192.0000 max=8799 max_rank=0 min=7534 min_rank=3 imbalance_ratio=1.074097 max_load_reduction=56737 max_load_reduction_pct=0.865738 moved_to_echo=32760 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[329, 712, 0, 0, 0, 0, 0, 0, 4096, 2855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2853, 1241, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 13, 4093, 0, 22, 2687, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3051, 3, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 4072, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3, 0, 0, 4096, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7992, 8190, 8083, 8391, 8291, 8204, 8192, 8193] avg=8192.0000 max=8391 max_rank=3 min=7992 min_rank=0 imbalance_ratio=1.024292 max_load_reduction=57145 max_load_reduction_pct=0.871964 moved_to_echo=32880 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[831, 585, 0, 0, 0, 0, 0, 0, 4096, 2855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2853, 1241, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 1, 4096, 0, 13, 2689, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2617, 0, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 4085, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8367, 8190, 8215, 7955, 8224, 8200, 8192, 8193] avg=8192.0000 max=8367 max_rank=0 min=7955 min_rank=3 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32809 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[521, 258, 0, 0, 0, 0, 0, 0, 4096, 2855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2853, 1241, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 20, 4096, 0, 21, 2696, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3309, 1, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 4057, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 3, 0, 0, 4096, 6, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7730, 8190, 8162, 8649, 8215, 8202, 8193, 8195] avg=8192.0000 max=8649 max_rank=3 min=7730 min_rank=0 imbalance_ratio=1.055786 max_load_reduction=56887 max_load_reduction_pct=0.868027 moved_to_echo=32805 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5248, 4864, 128, 128, 0, 128, 0, 0, 32768, 22912, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 0, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 1280, 0, 0, 32768, 128, 31744, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 22912, 128, 9984, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 22912, 9984, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1024, 0, 0, 128, 128, 32768, 128, 256, 21632, 9984, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 128, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 11, 0, 4096, 0, 4090, 14, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 63, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 17, 0, 4096, 0, 4095, 1, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 24, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 4096, 25, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 22, 0, 4096, 0, 4096, 14, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 5, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 1, 67, 0, 0, 4095, 1, 4095, 0, 2, 0, 0, 0, 4095, 0, 9, 4093, 1, 4094, 0, 3, 2, 1, 6, 0, 0, 4066, 0, 4095, 0, 1, 0, 4095, 0, 4095, 0, 0, 1, 0, 1, 0, 0, 54, 1, 1, 1, 1, 4089, 0, 0, 1, 4095, 4002, 0, 4086, 0, 4094, 4093, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 11, 0, 4065, 4, 0, 0, 0, 0, 0, 0, 4091, 0, 4090, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4079, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8195, 8191, 8194, 8196, 8184, 8206, 8194] avg=8192.0000 max=8206 max_rank=6 min=8176 min_rank=0 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 4065, 4, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8187, 8193, 8194, 8212, 8184, 8206, 8194] avg=8192.0000 max=8212 max_rank=4 min=8166 min_rank=0 imbalance_ratio=1.002441 max_load_reduction=57324 max_load_reduction_pct=0.874695 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 63, 0, 4065, 4, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4033, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8187, 8194, 8194, 8149, 8184, 8206, 8194] avg=8192.0000 max=8228 max_rank=0 min=8149 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 24, 0, 4065, 4, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8188, 8192, 8194, 8187, 8188, 8206, 8192] avg=8192.0000 max=8206 max_rank=6 min=8187 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 1, 67, 0, 0, 4064, 3, 0, 0, 0, 0, 0, 0, 4090, 1, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 9, 4093, 1, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 6, 0, 0, 4066, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4095, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 54, 1, 1, 4094, 4089, 0, 0, 0, 0, 0, 0, 1, 1, 4089, 0, 0, 1, 14, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4093, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8188, 8203, 8172, 8211, 8241, 8108, 8182] avg=8192.0000 max=8241 max_rank=5 min=8108 min_rank=6 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 6, 0, 4065, 4, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4071, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8212, 8187, 8194, 8191, 8184, 8203, 8194] avg=8192.0000 max=8212 max_rank=1 min=8171 min_rank=0 imbalance_ratio=1.002441 max_load_reduction=57324 max_load_reduction_pct=0.874695 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 17, 0, 4065, 4, 0, 0, 0, 0, 0, 0, 4091, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4079, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8187, 8190, 8194, 8199, 8184, 8206, 8194] avg=8192.0000 max=8206 max_rank=6 min=8182 min_rank=0 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 22, 0, 4070, 5, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4063, 0, 0, 21, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8186, 8200, 8191, 8187, 8195, 8192] avg=8192.0000 max=8200 max_rank=3 min=8186 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 32768, 128, 32640, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 256, 0, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 0, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2705, 35, 124, 24, 309, 0, 2703, 29, 854, 3297, 121, 152, 1249, 1796, 1, 204, 287, 963, 133, 1863, 4, 756, 1003, 1417, 6, 6, 3216, 1159, 3430, 56, 2903, 472, 25, 586, 2788, 186, 155, 89, 2735, 1246, 2246, 3, 2838, 1756, 4, 852, 1446, 837, 3472, 55, 1785, 0, 1, 137, 0, 290, 2, 115, 1143, 1367, 3274, 802, 952, 3072] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2574, 17, 255, 77, 408, 0, 2086, 10, 590, 3160, 114, 229, 611, 2109, 0, 352, 595, 1277, 179, 1478, 0, 563, 979, 1282, 2, 22, 2904, 1114, 2696, 64, 2353, 637, 104, 798, 2591, 167, 184, 192, 3205, 1587, 1977, 0, 2963, 1638, 0, 883, 1577, 1041, 2919, 88, 2402, 0, 1, 312, 0, 691, 0, 137, 598, 2093, 3264, 1268, 1607, 2512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2471, 22, 413, 42, 386, 0, 2299, 54, 593, 3080, 136, 490, 1530, 1269, 0, 404, 259, 963, 170, 2354, 0, 403, 1262, 1614, 1, 21, 2706, 1414, 3180, 88, 2521, 580, 33, 1175, 2490, 197, 510, 148, 2739, 980, 2153, 0, 2987, 1955, 0, 582, 1366, 990, 3394, 49, 1531, 4, 14, 278, 2, 640, 0, 141, 569, 1246, 3472, 1171, 1198, 2797] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2549, 4, 87, 49, 420, 0, 2407, 12, 1049, 3248, 152, 290, 906, 1464, 0, 227, 221, 1430, 184, 2136, 0, 852, 917, 1902, 7, 16, 2609, 1002, 3319, 96, 2940, 511, 15, 695, 2852, 345, 188, 124, 2511, 1164, 1941, 0, 3241, 2256, 0, 697, 1490, 871, 3277, 113, 1783, 1, 4, 232, 0, 393, 0, 91, 741, 1155, 3224, 1099, 1331, 2696] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2025, 26, 281, 90, 596, 2, 2143, 10, 711, 2948, 130, 324, 1735, 1791, 1, 369, 240, 1352, 132, 1953, 0, 547, 1243, 2078, 1, 3, 2669, 1525, 3299, 86, 2982, 657, 26, 1202, 2370, 108, 218, 72, 2420, 1764, 2255, 0, 2544, 1546, 3, 839, 1389, 820, 3245, 22, 1279, 0, 8, 259, 0, 416, 2, 122, 1107, 1215, 3424, 1251, 767, 2894] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2739, 38, 213, 74, 411, 0, 2687, 21, 695, 3038, 93, 336, 1250, 1648, 0, 239, 431, 1020, 81, 2001, 4, 551, 1418, 1454, 7, 3, 2791, 1563, 3200, 135, 2938, 514, 35, 850, 2361, 172, 310, 42, 2961, 1591, 2066, 3, 2680, 1634, 1, 1327, 1224, 930, 3077, 16, 1901, 3, 5, 213, 5, 284, 0, 153, 827, 1362, 3218, 668, 1243, 2781] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2421, 5, 101, 47, 317, 0, 2788, 49, 549, 3395, 192, 460, 1405, 1478, 0, 275, 286, 1322, 75, 1764, 1, 575, 949, 1540, 0, 26, 2826, 1078, 3238, 82, 2809, 861, 20, 948, 2451, 277, 123, 90, 2513, 1240, 1518, 0, 2875, 1720, 0, 1225, 1710, 789, 3354, 57, 1854, 0, 2, 254, 1, 469, 0, 113, 698, 1477, 3379, 1582, 1007, 2876] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1806, 98, 157, 55, 404, 11, 1990, 26, 815, 3138, 106, 145, 826, 895, 0, 252, 286, 1731, 41, 1935, 1, 526, 1209, 1580, 7, 23, 2715, 1187, 3541, 102, 2854, 697, 18, 978, 2406, 177, 299, 61, 2615, 1150, 2048, 1, 3284, 1764, 6, 800, 1576, 1117, 3264, 104, 2124, 20, 20, 47, 7, 654, 3, 119, 631, 1964, 3147, 1647, 1251, 3075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2574, 17, 255, 77, 408, 0, 2086, 10, 2637, 0, 0, 0, 0, 0, 0, 0, 590, 3160, 114, 229, 611, 2109, 0, 352, 837, 0, 0, 0, 0, 0, 0, 0, 595, 1277, 179, 1478, 0, 563, 979, 1282, 919, 260, 0, 0, 0, 0, 0, 0, 2, 22, 2904, 1114, 518, 64, 2353, 637, 0, 0, 0, 0, 0, 0, 0, 0, 104, 798, 2591, 167, 184, 192, 3205, 1587, 109, 0, 0, 0, 0, 0, 0, 0, 1977, 0, 1098, 1638, 0, 883, 1577, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 2919, 88, 2402, 0, 1, 312, 0, 691, 1918, 33, 0, 0, 0, 0, 0, 0, 0, 137, 598, 2093, 594, 1268, 1607, 2512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8064, 8002, 7532, 7614, 8937, 8214, 8364, 8809] avg=8192.0000 max=8937 max_rank=4 min=7532 min_rank=2 imbalance_ratio=1.090942 max_load_reduction=56599 max_load_reduction_pct=0.863632 moved_to_echo=34324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2705, 35, 124, 24, 309, 0, 2703, 29, 2645, 0, 0, 0, 0, 0, 0, 0, 854, 3297, 121, 152, 1249, 1796, 1, 204, 802, 0, 0, 0, 0, 0, 0, 0, 287, 963, 133, 1863, 4, 756, 1003, 1417, 880, 331, 0, 0, 0, 0, 0, 0, 6, 6, 3216, 1159, 659, 56, 2903, 472, 0, 0, 0, 0, 0, 0, 0, 0, 25, 586, 2788, 186, 155, 89, 2735, 1246, 104, 0, 0, 0, 0, 0, 0, 0, 2246, 3, 1052, 1756, 4, 852, 1446, 837, 0, 0, 0, 0, 0, 0, 0, 0, 3472, 55, 1785, 0, 1, 137, 0, 290, 2440, 33, 0, 0, 0, 0, 0, 0, 2, 115, 1143, 1367, 596, 802, 952, 3072, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8574, 8476, 7637, 8477, 7914, 8196, 8213, 8049] avg=8192.0000 max=8574 max_rank=0 min=7637 min_rank=2 imbalance_ratio=1.046631 max_load_reduction=56962 max_load_reduction_pct=0.869171 moved_to_echo=32372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2025, 26, 281, 90, 596, 2, 2143, 10, 2766, 0, 0, 0, 0, 0, 0, 0, 711, 2948, 130, 324, 1735, 1791, 1, 369, 719, 0, 0, 0, 0, 0, 0, 0, 240, 1352, 132, 1953, 0, 547, 1243, 2078, 789, 318, 0, 0, 0, 0, 0, 0, 1, 3, 2669, 1525, 634, 86, 2982, 657, 0, 0, 0, 0, 0, 0, 0, 0, 26, 1202, 2370, 108, 218, 72, 2420, 1764, 93, 0, 0, 0, 0, 0, 0, 0, 2255, 0, 943, 1546, 3, 839, 1389, 820, 0, 0, 0, 0, 0, 0, 0, 0, 3245, 22, 1279, 0, 8, 259, 0, 416, 2347, 35, 0, 0, 0, 0, 0, 0, 2, 122, 1107, 1215, 623, 1251, 767, 2894, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7939, 8728, 8652, 8557, 8273, 7795, 7611, 7981] avg=8192.0000 max=8728 max_rank=1 min=7611 min_rank=6 imbalance_ratio=1.065430 max_load_reduction=56808 max_load_reduction_pct=0.866821 moved_to_echo=31660 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2471, 22, 413, 42, 386, 0, 2299, 54, 2805, 0, 0, 0, 0, 0, 0, 0, 593, 3080, 136, 490, 1530, 1269, 0, 404, 844, 0, 0, 0, 0, 0, 0, 0, 259, 963, 170, 2354, 0, 403, 1262, 1614, 926, 306, 0, 0, 0, 0, 0, 0, 1, 21, 2706, 1414, 611, 88, 2521, 580, 0, 0, 0, 0, 0, 0, 0, 0, 33, 1175, 2490, 197, 510, 148, 2739, 980, 110, 0, 0, 0, 0, 0, 0, 0, 2153, 0, 1107, 1955, 0, 582, 1366, 990, 0, 0, 0, 0, 0, 0, 0, 0, 3394, 49, 1531, 4, 14, 278, 2, 640, 2263, 35, 0, 0, 0, 0, 0, 0, 0, 141, 569, 1246, 632, 1171, 1198, 2797, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8492, 8346, 8257, 7942, 8382, 8153, 8210, 7754] avg=8192.0000 max=8492 max_rank=0 min=7754 min_rank=7 imbalance_ratio=1.036621 max_load_reduction=57044 max_load_reduction_pct=0.870422 moved_to_echo=32499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2739, 38, 213, 74, 411, 0, 2687, 21, 2600, 0, 0, 0, 0, 0, 0, 0, 695, 3038, 93, 336, 1250, 1648, 0, 239, 757, 0, 0, 0, 0, 0, 0, 0, 431, 1020, 81, 2001, 4, 551, 1418, 1454, 831, 308, 0, 0, 0, 0, 0, 0, 7, 3, 2791, 1563, 615, 135, 2938, 514, 0, 0, 0, 0, 0, 0, 0, 0, 35, 850, 2361, 172, 310, 42, 2961, 1591, 98, 0, 0, 0, 0, 0, 0, 0, 2066, 3, 994, 1634, 1, 1327, 1224, 930, 0, 0, 0, 0, 0, 0, 0, 0, 3077, 16, 1901, 3, 5, 213, 5, 284, 2277, 32, 0, 0, 0, 0, 0, 0, 0, 153, 827, 1362, 586, 668, 1243, 2781, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8783, 8056, 8099, 8566, 8420, 8179, 7813, 7620] avg=8192.0000 max=8783 max_rank=0 min=7620 min_rank=7 imbalance_ratio=1.072144 max_load_reduction=56753 max_load_reduction_pct=0.865982 moved_to_echo=32032 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2549, 4, 87, 49, 420, 0, 2407, 12, 2605, 0, 0, 0, 0, 0, 0, 0, 1049, 3248, 152, 290, 906, 1464, 0, 227, 916, 0, 0, 0, 0, 0, 0, 0, 221, 1430, 184, 2136, 0, 852, 917, 1902, 1005, 320, 0, 0, 0, 0, 0, 0, 7, 16, 2609, 1002, 638, 96, 2940, 511, 0, 0, 0, 0, 0, 0, 0, 0, 15, 695, 2852, 345, 188, 124, 2511, 1164, 119, 0, 0, 0, 0, 0, 0, 0, 1941, 0, 1201, 2256, 0, 697, 1490, 871, 0, 0, 0, 0, 0, 0, 0, 0, 3277, 113, 1783, 1, 4, 232, 0, 393, 2361, 32, 0, 0, 0, 0, 0, 0, 0, 91, 741, 1155, 587, 1099, 1331, 2696, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8252, 8967, 7819, 8013, 8456, 8196, 7700] avg=8192.0000 max=8967 max_rank=2 min=7700 min_rank=7 imbalance_ratio=1.094604 max_load_reduction=56569 max_load_reduction_pct=0.863174 moved_to_echo=32365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1806, 98, 157, 55, 404, 11, 1990, 26, 2542, 0, 0, 0, 0, 0, 0, 0, 815, 3138, 106, 145, 826, 895, 0, 252, 928, 0, 0, 0, 0, 0, 0, 0, 286, 1731, 41, 1935, 1, 526, 1209, 1580, 1018, 341, 0, 0, 0, 0, 0, 0, 7, 23, 2715, 1187, 681, 102, 2854, 697, 0, 0, 0, 0, 0, 0, 0, 0, 18, 978, 2406, 177, 299, 61, 2615, 1150, 121, 0, 0, 0, 0, 0, 0, 0, 2048, 1, 1217, 1764, 6, 800, 1576, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 104, 2124, 20, 20, 47, 7, 654, 2519, 32, 0, 0, 0, 0, 0, 0, 3, 119, 631, 1964, 573, 1647, 1251, 3075, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7089, 7105, 8668, 8266, 7825, 8529, 8791, 9263] avg=8192.0000 max=9263 max_rank=7 min=7089 min_rank=0 imbalance_ratio=1.130737 max_load_reduction=56273 max_load_reduction_pct=0.858658 moved_to_echo=34408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2421, 5, 101, 47, 317, 0, 2788, 49, 2734, 0, 0, 0, 0, 0, 0, 0, 549, 3395, 192, 460, 1405, 1478, 0, 275, 817, 0, 0, 0, 0, 0, 0, 0, 286, 1322, 75, 1764, 1, 575, 949, 1540, 896, 316, 0, 0, 0, 0, 0, 0, 0, 26, 2826, 1078, 613, 82, 2809, 861, 0, 0, 0, 0, 0, 0, 0, 0, 20, 948, 2451, 277, 123, 90, 2513, 1240, 110, 0, 0, 0, 0, 0, 0, 0, 1518, 0, 1052, 1720, 0, 1225, 1710, 789, 0, 0, 0, 0, 0, 0, 0, 0, 3354, 57, 1854, 0, 2, 254, 1, 469, 2309, 38, 0, 0, 0, 0, 0, 0, 0, 113, 698, 1477, 607, 1582, 1007, 2876, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8462, 8571, 7724, 8295, 7772, 8014, 8338, 8360] avg=8192.0000 max=8571 max_rank=1 min=7724 min_rank=2 imbalance_ratio=1.046265 max_load_reduction=56965 max_load_reduction_pct=0.869217 moved_to_echo=32484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[16256, 128, 8704, 14336, 128, 7296, 11904, 7424, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 1024, 6400, 11904, 4864, 9600, 9472, 22784, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 22528, 10112, 4992, 768, 22400, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[26112, 512, 14720, 128, 128, 1792, 128, 3840, 18560, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5888, 25344, 1152, 2432, 9600, 12544, 128, 2432, 6656, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[19328, 256, 1664, 512, 3328, 128, 19200, 256, 21376, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 7296, 20352, 1664, 2048, 896, 21760, 10752, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2688, 10112, 1024, 15488, 128, 4864, 9088, 12928, 7296, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4082, 0, 3867, 4093, 0, 47, 32, 0, 0, 1792, 3101, 0, 2, 0, 0, 1, 8, 63, 3156, 4003, 1, 4066, 0, 13, 0, 3956, 6, 0, 1, 1, 971, 6, 1, 1889, 4092, 0, 0, 0, 0, 2, 0, 4092, 0, 4, 0, 0, 4094, 0, 9, 0, 4096, 4090, 1883, 0, 1, 2, 0, 3918] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 3, 3926, 1, 3377, 4093, 0, 5, 146, 0, 0, 738, 2735, 0, 0, 0, 0, 0, 0, 3, 3544, 4086, 0, 4090, 0, 10, 1, 4059, 0, 0, 0, 0, 1327, 0, 0, 2886, 4093, 3, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 113, 4096, 0, 52, 2, 4093, 4096, 2017, 3, 0, 0, 0, 3746] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[15, 5, 5, 0, 4077, 2, 3828, 2, 3891, 4086, 0, 37, 692, 1, 0, 426, 3313, 1, 1, 24, 9, 11, 0, 16, 3026, 4095, 0, 4095, 8, 0, 0, 4089, 0, 5, 0, 11, 706, 0, 18, 2686, 4083, 2, 0, 1, 4, 0, 0, 4092, 0, 0, 0, 2, 4085, 2, 2, 0, 4083, 4080, 2173, 0, 7, 2, 1, 3736] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 3957, 0, 3936, 4096, 0, 181, 140, 0, 0, 2466, 3435, 0, 0, 0, 0, 0, 3, 33, 1556, 4069, 0, 4074, 3, 3, 0, 4086, 1, 0, 1, 0, 492, 0, 0, 2179, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 4095, 0, 74, 0, 4096, 4096, 2153, 0, 0, 0, 0, 4004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 3833, 0, 3561, 4096, 0, 19, 16, 0, 0, 452, 3224, 0, 0, 0, 0, 0, 1, 5, 3062, 4087, 0, 4096, 2, 0, 0, 4078, 0, 0, 0, 0, 1447, 0, 0, 2685, 4096, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 3, 0, 4095, 4096, 2232, 0, 0, 0, 0, 4056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 3906, 0, 3894, 4096, 0, 36, 156, 0, 0, 2131, 2925, 0, 0, 0, 0, 0, 0, 1, 2068, 4096, 0, 4087, 1, 19, 0, 4087, 0, 0, 1, 0, 1017, 0, 0, 1949, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 107, 0, 4092, 4096, 2318, 0, 0, 0, 0, 4074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 4078, 0, 3468, 4096, 0, 171, 256, 0, 0, 923, 2887, 0, 0, 0, 0, 0, 0, 13, 2247, 4085, 0, 4095, 0, 14, 0, 4060, 0, 0, 3, 0, 1097, 0, 0, 2790, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 4096, 2708, 0, 0, 0, 0, 3965] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4089, 0, 3850, 4096, 0, 81, 109, 0, 0, 1759, 3324, 0, 1, 0, 0, 0, 17, 18, 2759, 4048, 0, 4075, 5, 7, 0, 3979, 1, 0, 3, 0, 231, 0, 0, 1517, 4095, 3, 0, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 4094, 0, 17, 0, 4095, 4095, 2990, 0, 0, 0, 0, 3983] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 3, 3926, 1, 132, 0, 0, 0, 0, 0, 0, 0, 3377, 2861, 0, 5, 146, 0, 0, 738, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 0, 0, 0, 0, 0, 0, 3, 4093, 953, 0, 0, 0, 0, 0, 0, 3544, 1461, 0, 0, 0, 10, 1, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 0, 0, 2886, 3137, 1824, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 113, 4096, 0, 52, 2, 2149, 1099, 801, 0, 0, 0, 0, 0, 0, 1947, 2017, 3, 0, 0, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 7127, 7784, 9075, 9174, 8193, 8316, 7713] avg=8192.0000 max=9174 max_rank=4 min=7127 min_rank=1 imbalance_ratio=1.119873 max_load_reduction=56362 max_load_reduction_pct=0.860016 moved_to_echo=33396 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4082, 0, 132, 0, 0, 0, 0, 0, 0, 0, 3867, 2861, 0, 47, 32, 0, 0, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 3101, 0, 2, 0, 0, 1, 8, 63, 4096, 948, 0, 0, 0, 0, 0, 0, 3156, 1431, 1, 0, 0, 13, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 1, 971, 6, 1, 1889, 3118, 1787, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 2, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4094, 0, 9, 0, 2145, 1099, 785, 0, 0, 0, 0, 0, 0, 1945, 1883, 0, 1, 2, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8309, 8599, 8219, 8557, 7780, 8187, 8136, 7749] avg=8192.0000 max=8599 max_rank=1 min=7749 min_rank=7 imbalance_ratio=1.049683 max_load_reduction=56937 max_load_reduction_pct=0.868790 moved_to_echo=31852 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 4078, 0, 132, 0, 0, 0, 0, 0, 0, 0, 3468, 2863, 0, 171, 256, 0, 0, 923, 0, 0, 0, 0, 0, 0, 0, 0, 2887, 0, 0, 0, 0, 0, 0, 13, 4096, 954, 0, 0, 0, 0, 0, 0, 2247, 1460, 0, 0, 0, 14, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1097, 0, 0, 2790, 3141, 1824, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 2149, 1100, 801, 0, 0, 0, 0, 0, 0, 1947, 2708, 0, 0, 0, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 7681, 7950, 7781, 8855, 8193, 8149, 8620] avg=8192.0000 max=8855 max_rank=4 min=7681 min_rank=1 imbalance_ratio=1.080933 max_load_reduction=56681 max_load_reduction_pct=0.864883 moved_to_echo=33817 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[15, 5, 5, 0, 4077, 2, 3828, 2, 132, 0, 0, 0, 0, 0, 0, 0, 3891, 2855, 0, 37, 692, 1, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 1, 1, 24, 9, 11, 0, 16, 4083, 954, 0, 0, 0, 0, 0, 0, 3026, 1464, 0, 0, 8, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 11, 706, 0, 18, 2686, 3141, 1828, 0, 0, 0, 0, 0, 0, 4083, 2, 0, 1, 4, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4085, 2, 2, 0, 2140, 1098, 803, 0, 0, 0, 0, 0, 0, 1940, 2173, 0, 7, 2, 1, 3736, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 7902, 8412, 8587, 8395, 8183, 8132, 7859] avg=8192.0000 max=8587 max_rank=3 min=7859 min_rank=7 imbalance_ratio=1.048218 max_load_reduction=56949 max_load_reduction_pct=0.868973 moved_to_echo=32569 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 3833, 0, 132, 0, 0, 0, 0, 0, 0, 0, 3561, 2863, 0, 19, 16, 0, 0, 452, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 0, 0, 0, 0, 0, 1, 5, 4095, 954, 0, 0, 0, 0, 0, 0, 3062, 1461, 0, 0, 2, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1447, 0, 0, 2685, 3142, 1825, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 3, 0, 2149, 1100, 801, 0, 0, 0, 0, 0, 0, 1947, 2232, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8061, 6911, 8279, 8603, 9099, 8198, 8150, 8235] avg=8192.0000 max=9099 max_rank=4 min=6911 min_rank=1 imbalance_ratio=1.110718 max_load_reduction=56437 max_load_reduction_pct=0.861160 moved_to_echo=33682 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 3957, 0, 132, 0, 0, 0, 0, 0, 0, 0, 3936, 2863, 0, 181, 140, 0, 0, 2466, 0, 0, 0, 0, 0, 0, 0, 0, 3435, 0, 0, 0, 0, 0, 3, 33, 4096, 950, 0, 0, 0, 0, 0, 0, 1556, 1455, 0, 0, 3, 3, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 492, 0, 0, 2179, 3124, 1817, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4095, 0, 74, 0, 2149, 1100, 797, 0, 0, 0, 0, 0, 0, 1947, 2153, 0, 0, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 9586, 8517, 7103, 7614, 8194, 8233, 8104] avg=8192.0000 max=9586 max_rank=1 min=7103 min_rank=3 imbalance_ratio=1.170166 max_load_reduction=55950 max_load_reduction_pct=0.853729 moved_to_echo=32145 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 3906, 0, 132, 0, 0, 0, 0, 0, 0, 0, 3894, 2863, 0, 36, 156, 0, 0, 2131, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 0, 0, 0, 0, 0, 0, 1, 4092, 952, 0, 0, 0, 0, 0, 0, 2068, 1464, 0, 0, 1, 19, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1017, 0, 0, 1949, 3135, 1829, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 107, 0, 2149, 1100, 803, 0, 0, 0, 0, 0, 0, 1947, 2318, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 9080, 7970, 7639, 7931, 8188, 8255, 8339] avg=8192.0000 max=9080 max_rank=1 min=7639 min_rank=3 imbalance_ratio=1.108398 max_load_reduction=56456 max_load_reduction_pct=0.861450 moved_to_echo=32713 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4089, 0, 135, 0, 0, 0, 0, 0, 0, 0, 3850, 2851, 0, 81, 109, 0, 0, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 3324, 0, 1, 0, 0, 0, 17, 18, 4095, 950, 0, 0, 0, 0, 0, 0, 2759, 1441, 0, 0, 5, 7, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 231, 0, 0, 1517, 3125, 1811, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 1, 1, 1, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 17, 0, 2151, 1106, 796, 0, 0, 0, 0, 0, 0, 1944, 2990, 0, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8650, 8405, 8191, 6688, 8200, 8165, 8917] avg=8192.0000 max=8917 max_rank=7 min=6688 min_rank=4 imbalance_ratio=1.088501 max_load_reduction=56619 max_load_reduction_pct=0.863937 moved_to_echo=31970 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 0, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[21504, 11648, 128, 0, 128, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 15616, 18560, 128, 128, 128, 128, 31488, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[29952, 22912, 0, 640, 1664, 128, 0, 10752, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[24960, 128, 128, 128, 128, 128, 128, 256, 32768, 7680, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 256, 32768, 128, 384, 128, 17280, 8832, 6400, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 31744, 128, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 7296, 128, 128, 18688, 25088, 14592, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 4042, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 4055, 7, 0, 761, 144, 4096, 0, 4085, 0, 0, 0, 2787, 4096, 0, 10, 4090, 1, 4092, 0, 0, 0, 174, 4096, 0, 4096, 0, 35, 0, 4095, 0, 0, 0, 4, 0, 4096, 0, 0, 276, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 4, 4096, 0, 0, 0, 4081, 0, 4096, 0, 0, 4096, 2, 1, 4095, 0, 0, 4077, 6, 0, 170, 64, 4095, 0, 4089, 0, 0, 0, 2740, 4093, 0, 4, 4095, 2, 4092, 0, 0, 1, 165, 4096, 0, 4096, 1, 7, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 975, 0, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4086, 0, 0, 1234, 8, 4094, 0, 4096, 0, 0, 0, 2545, 4095, 2, 0, 4096, 30, 4095, 0, 0, 0, 29, 4095, 0, 4095, 0, 20, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 249, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4092, 0, 0, 76, 32, 4094, 0, 4096, 0, 0, 0, 3186, 4096, 1, 0, 4096, 7, 4096, 0, 0, 0, 33, 4096, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 776, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 4082, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4058, 0, 0, 57, 29, 4096, 0, 4094, 0, 0, 0, 3584, 4093, 0, 0, 4093, 0, 4096, 0, 0, 0, 54, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 426, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 0, 20, 4092, 0, 0, 10, 4077, 0, 4092, 2, 0, 4088, 5, 4, 4076, 1, 4, 4020, 0, 5, 197, 144, 4091, 0, 4096, 0, 0, 0, 592, 4088, 7, 2, 3925, 45, 4092, 0, 4, 29, 30, 4092, 4, 4057, 0, 4, 3, 4091, 6, 4, 6, 11, 1, 4089, 0, 6, 3304, 5, 0, 0, 12, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 0, 0, 4096, 0, 0, 0, 4056, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4071, 5, 0, 620, 107, 4095, 0, 4089, 0, 0, 0, 2214, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 75, 4090, 0, 4096, 0, 19, 0, 4096, 0, 0, 0, 4, 0, 4096, 0, 0, 1112, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4086, 1, 0, 66, 13, 4095, 0, 4095, 0, 0, 0, 2670, 4096, 0, 1, 4096, 6, 4095, 0, 0, 1, 145, 4096, 0, 4096, 0, 7, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 1207, 0, 1, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 15, 0, 0, 4096, 0, 2531, 1022, 460, 75, 0, 0, 0, 0, 0, 0, 4042, 0, 21, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4055, 7, 0, 22, 2, 0, 0, 0, 0, 0, 0, 761, 144, 3636, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2787, 1565, 0, 10, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 3999, 0, 4096, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 0, 0, 0, 0, 4092, 3053, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8158, 8182, 8626, 8453, 8304, 8193, 7421] avg=8192.0000 max=8626 max_rank=3 min=7421 min_rank=7 imbalance_ratio=1.052979 max_load_reduction=56910 max_load_reduction_pct=0.868378 moved_to_echo=32371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 2531, 1022, 460, 75, 0, 0, 0, 0, 0, 0, 4090, 0, 21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 22, 2, 0, 0, 0, 0, 0, 0, 76, 32, 3634, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3186, 1565, 1, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3999, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 0, 0, 0, 0, 0, 4096, 3053, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8207, 8212, 7838, 8855, 8125, 8190, 7925] avg=8192.0000 max=8855 max_rank=4 min=7838 min_rank=3 imbalance_ratio=1.080933 max_load_reduction=56681 max_load_reduction_pct=0.864883 moved_to_echo=33095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 4, 4096, 0, 2529, 1022, 460, 75, 0, 0, 0, 0, 0, 0, 4081, 0, 21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4095, 0, 0, 4077, 6, 0, 22, 2, 0, 0, 0, 0, 0, 0, 170, 64, 3635, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2740, 1564, 0, 4, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 165, 3999, 0, 4096, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 0, 0, 0, 3, 0, 4092, 3053, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8198, 8205, 7958, 8405, 8269, 8188, 8123] avg=8192.0000 max=8405 max_rank=4 min=7958 min_rank=3 imbalance_ratio=1.026001 max_load_reduction=57131 max_load_reduction_pct=0.871750 moved_to_echo=32985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 0, 20, 4092, 0, 2526, 1021, 459, 75, 0, 0, 0, 0, 0, 10, 4077, 0, 21, 2, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 4076, 1, 4, 4020, 0, 5, 22, 2, 0, 0, 0, 0, 0, 0, 197, 144, 3632, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 1562, 7, 2, 3925, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 29, 30, 3995, 4, 4057, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4089, 6, 4, 6, 11, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3304, 5, 0, 0, 12, 0, 4092, 3050, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8198, 8139, 8069, 6133, 8123, 8209, 10469] avg=8192.0000 max=10469 max_rank=7 min=6133 min_rank=4 imbalance_ratio=1.277954 max_load_reduction=55067 max_load_reduction_pct=0.840256 moved_to_echo=32934 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 2530, 1022, 460, 75, 0, 0, 0, 0, 0, 0, 4089, 1, 21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 0, 0, 22, 2, 0, 0, 0, 0, 0, 0, 1234, 8, 3634, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 1565, 2, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 3998, 0, 4095, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 0, 1, 0, 4095, 3053, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8207, 8206, 8972, 8238, 8142, 8190, 7398] avg=8192.0000 max=8972 max_rank=3 min=7398 min_rank=7 imbalance_ratio=1.095215 max_load_reduction=56564 max_load_reduction_pct=0.863098 moved_to_echo=31968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 4096, 0, 2529, 1022, 460, 75, 0, 0, 0, 0, 0, 0, 4082, 0, 21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4058, 0, 0, 22, 2, 0, 0, 0, 0, 0, 0, 57, 29, 3636, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3584, 1564, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 3999, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 0, 0, 0, 0, 4096, 3053, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8199, 8178, 7816, 9241, 8149, 8191, 7575] avg=8192.0000 max=9241 max_rank=4 min=7575 min_rank=7 imbalance_ratio=1.128052 max_load_reduction=56295 max_load_reduction_pct=0.858994 moved_to_echo=33156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 0, 0, 4096, 0, 2531, 1022, 460, 75, 0, 0, 0, 0, 0, 0, 4056, 1, 21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4071, 5, 0, 22, 2, 0, 0, 0, 0, 0, 0, 620, 107, 3635, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 1565, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 3993, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 0, 0, 0, 0, 0, 4096, 3053, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8174, 8196, 8451, 7875, 8183, 8194, 8261] avg=8192.0000 max=8451 max_rank=3 min=7875 min_rank=4 imbalance_ratio=1.031616 max_load_reduction=57085 max_load_reduction_pct=0.871048 moved_to_echo=32513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 2533, 1025, 463, 78, 0, 0, 0, 0, 0, 0, 4087, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 1, 0, 26, 9, 0, 0, 0, 0, 0, 0, 66, 13, 3632, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2670, 1563, 0, 1, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 145, 3992, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1207, 0, 1, 0, 2, 0, 4095, 3059, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8195, 8218, 7806, 8336, 8241, 8181, 8364] avg=8192.0000 max=8364 max_rank=7 min=7806 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=33122 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 32768, 0, 20352, 8192, 3712, 640, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32640, 128, 256, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 128, 128, 32640, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[20352, 12544, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 768, 32000, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 8448, 128, 128, 0, 128, 0, 32768, 24448, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[3200, 640, 29184, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4087, 4092, 3927, 0, 0, 0, 4096, 0, 4096, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 6, 0, 0, 4096, 0, 91, 2, 0, 0, 3, 0, 4096, 0, 0, 0, 4096, 4096, 4, 0, 4062, 87, 0, 34, 0, 4096, 4080, 16, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4092, 4079, 4072, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 4092, 0, 11, 4, 0, 1, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 4096, 21, 0, 19, 0, 4096, 4081, 13, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 5, 0, 1, 4090, 9, 8, 4077, 4042, 4051, 0, 4, 0, 4088, 4, 4087, 4086, 0, 0, 4086, 5, 5, 3, 8, 0, 0, 0, 14, 0, 4, 0, 4071, 2, 6, 7, 4082, 1, 107, 0, 0, 42, 17, 4, 4091, 0, 0, 0, 4089, 4092, 3, 5, 4082, 0, 8, 42, 1, 4092, 3982, 26, 3, 0, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4096, 4096, 4095, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4081, 16, 0, 6, 0, 4095, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 4096, 4079, 0, 0, 0, 4095, 0, 4096, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4087, 22, 0, 8, 0, 4094, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4084, 4053, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 47, 3, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 4045, 39, 0, 11, 0, 4096, 4096, 19, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 4071, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 26, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4081, 17, 0, 3, 0, 4096, 4084, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 4093, 4093, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4073, 26, 0, 8, 0, 4093, 4095, 3, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 4079, 4072, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 5, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4092, 0, 11, 4, 0, 4039, 20, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 4096, 21, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 13, 0, 0, 0, 1, 0, 6, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8203, 8193, 8189, 8169, 8185, 8206, 8200] avg=8192.0000 max=8206 max_rank=6 min=8169 min_rank=4 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32760 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4038, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 4081, 16, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 1, 0, 0, 0, 0, 0, 6, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8243, 8193, 8197, 8154, 8185, 8173, 8201] avg=8192.0000 max=8243 max_rank=1 min=8154 min_rank=4 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32713 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4087, 3, 0, 0, 0, 0, 0, 0, 0, 4092, 3927, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 5, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 91, 2, 0, 4039, 20, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4, 0, 4062, 87, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 16, 0, 2, 0, 0, 0, 6, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8071, 8189, 8188, 8254, 8188, 8257, 8203] avg=8192.0000 max=8257 max_rank=6 min=8071 min_rank=1 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32902 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 1, 4090, 9, 8, 4077, 3, 0, 0, 0, 0, 0, 0, 0, 4042, 4051, 0, 4, 0, 52, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4086, 5, 5, 3, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4, 0, 4071, 4087, 5, 0, 0, 0, 0, 0, 0, 2, 6, 7, 4082, 1, 107, 0, 0, 4031, 20, 0, 0, 0, 0, 0, 0, 42, 17, 4, 4091, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 3, 5, 4082, 0, 8, 42, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3982, 26, 3, 0, 0, 0, 4, 6, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8153, 8194, 8181, 8256, 8236, 8207, 8116] avg=8192.0000 max=8256 max_rank=4 min=8116 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32815 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4038, 20, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 4087, 22, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 1, 0, 0, 0, 0, 0, 6, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8227, 8191, 8197, 8154, 8187, 8187, 8200] avg=8192.0000 max=8227 max_rank=1 min=8154 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32728 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4084, 4053, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 47, 3, 0, 4039, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 4045, 39, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 19, 0, 0, 0, 0, 0, 6, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8189, 8191, 8187, 8205, 8184, 8165, 8220] avg=8192.0000 max=8220 max_rank=7 min=8165 min_rank=6 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4094, 4071, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 26, 0, 0, 4039, 20, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 4081, 17, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 1, 0, 0, 0, 0, 0, 6, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8217, 8193, 8197, 8181, 8192, 8171, 8190] avg=8192.0000 max=8217 max_rank=1 min=8171 min_rank=6 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32734 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 8, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 1, 0, 0, 4041, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 4073, 26, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4095, 3, 0, 0, 0, 0, 0, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8233, 8192, 8200, 8163, 8179, 8170, 8206] avg=8192.0000 max=8233 max_rank=1 min=8163 min_rank=4 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32718 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 128, 128, 32640, 256, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 384, 128, 0, 32384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 32640, 128, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 0, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32512, 0, 128, 0, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 7, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4084, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4086, 0, 4092, 0, 0, 0, 1, 0, 4, 0, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 4, 0, 4091, 0, 0, 8, 4096, 40, 0, 0, 4096, 5, 0, 4096, 4092, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4077, 0, 4092, 4089, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 15, 0, 0, 0, 4022, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 2, 0, 4055, 0, 0, 15, 4096, 184, 1, 0, 4096, 1, 0, 4096, 4084, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 4096, 0, 4004, 0, 4094, 4083, 0, 0, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 26, 0, 9, 4096, 7, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 4095, 0, 4096, 4074, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 10, 0, 3, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4081, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 18, 0, 0, 0, 4079, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 81, 0, 1, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4032, 0, 4096, 4078, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[8, 4091, 0, 0, 0, 0, 5, 5, 4077, 0, 4090, 6, 5, 4, 0, 0, 0, 2, 3, 0, 4091, 1, 0, 2, 4090, 8, 0, 3, 6, 4003, 0, 0, 21, 4085, 140, 0, 0, 4092, 70, 3, 4090, 4089, 4089, 5, 0, 5, 4, 0, 0, 4087, 5, 0, 2, 0, 4089, 1, 4084, 4, 4092, 3969, 0, 4, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 6, 0, 4096, 0, 0, 1, 0, 4095, 0, 0, 1, 4096, 5, 0, 8, 4096, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 4080, 0, 4095, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4040, 46, 0, 0, 0, 0, 0, 0, 4086, 0, 4092, 0, 0, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4084, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4091, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 40, 0, 0, 4018, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 50, 4089, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8196, 8183, 8206, 8167, 8191, 8192, 8219] avg=8192.0000 max=8219 max_rank=7 min=8167 min_rank=4 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 0, 0, 0, 4044, 46, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 7, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 50, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8205, 8193, 8207, 8127, 8195, 8192, 8230] avg=8192.0000 max=8230 max_rank=7 min=8127 min_rank=4 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 15, 0, 0, 0, 4042, 46, 0, 0, 0, 0, 0, 0, 4022, 0, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 2, 0, 4055, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 184, 1, 0, 4018, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 50, 4083, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8135, 8191, 8169, 8315, 8183, 8199, 8146] avg=8192.0000 max=8315 max_rank=4 min=8135 min_rank=1 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[8, 4091, 0, 0, 0, 0, 5, 5, 4040, 46, 0, 0, 0, 0, 0, 0, 4077, 0, 4090, 6, 5, 4, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 4091, 1, 0, 2, 4087, 2, 0, 0, 0, 0, 0, 0, 4090, 8, 0, 3, 6, 4003, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 21, 4085, 140, 0, 0, 4015, 70, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4089, 4089, 5, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 2, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4, 50, 3969, 0, 4, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8198, 8188, 8125, 8334, 8195, 8184, 8117] avg=8192.0000 max=8334 max_rank=4 min=8117 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4044, 46, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 0, 9, 4018, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 50, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8196, 8190, 8200, 8156, 8195, 8194, 8219] avg=8192.0000 max=8219 max_rank=7 min=8156 min_rank=4 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 4044, 46, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 3, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 50, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8207, 8196, 8207, 8127, 8192, 8192, 8227] avg=8192.0000 max=8227 max_rank=7 min=8127 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 18, 0, 0, 0, 4044, 46, 0, 0, 0, 0, 0, 0, 4079, 0, 4095, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 81, 0, 1, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 50, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8191, 8191, 8207, 8196, 8195, 8192, 8160] avg=8192.0000 max=8207 max_rank=3 min=8160 min_rank=7 imbalance_ratio=1.001831 max_load_reduction=57329 max_load_reduction_pct=0.874771 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 4045, 53, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 6, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4095, 0, 0, 21, 2, 0, 0, 0, 0, 0, 0, 1, 4096, 5, 0, 8, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 43, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8208, 8204, 8215, 8114, 8190, 8191, 8218] avg=8192.0000 max=8218 max_rank=7 min=8114 min_rank=4 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 512, 128, 128, 32128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 0, 32768, 128, 128, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 128, 512, 32640, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 32640, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 0, 128, 0, 128, 128, 32384, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 1292, 0, 0, 8, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1664, 0, 1, 4096, 5, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2253, 0, 2975, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 342, 0, 0, 48, 4096, 0, 0, 0, 4073, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 0, 4085, 4096, 0, 185, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 3915, 1, 3735, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 1377, 0, 1, 3, 4096, 0, 0, 0, 3983, 0, 11, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 4096, 0, 1757, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 3210, 0, 1960, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 502, 0, 0, 13, 4096, 0, 0, 0, 3987, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4091, 4096, 0, 986, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 3273, 0, 3531, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 653, 0, 0, 6, 4096, 0, 0, 0, 4055, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 852, 0, 1, 4096, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3191, 0, 3529, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4089, 7, 0, 2, 0, 4088, 4, 1, 0, 5, 1534, 5, 23, 36, 4090, 6, 0, 0, 4054, 7, 17, 5, 4090, 7, 4088, 0, 6, 0, 3, 4087, 4082, 4, 1914, 3, 0, 4087, 2, 4089, 6, 1, 9, 4089, 4090, 7, 0, 0, 8, 1, 0, 4067, 0, 2, 4090, 6, 0, 6, 3, 2290, 0, 2426, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1197, 0, 0, 9, 4096, 0, 0, 0, 4057, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 4096, 0, 1078, 0, 0, 4096, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2119, 0, 3827, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 288, 0, 0, 23, 4096, 0, 0, 0, 4061, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 4096, 0, 443, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 1, 0, 0, 3615, 0, 3866, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4091, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1292, 0, 0, 8, 4096, 3176, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1664, 0, 1, 2989, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2253, 0, 2975, 0, 0, 1056, 915, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8572, 8235, 8197, 8755, 8191, 8194, 7199] avg=8192.0000 max=8755 max_rank=4 min=7199 min_rank=7 imbalance_ratio=1.068726 max_load_reduction=56781 max_load_reduction_pct=0.866409 moved_to_echo=32339 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4091, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 0, 0, 48, 4096, 3176, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4073, 0, 4, 0, 44, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 185, 0, 0, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 1, 3735, 0, 5, 1056, 915, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7662, 8217, 8186, 7270, 8191, 8190, 9627] avg=8192.0000 max=9627 max_rank=7 min=7270 min_rank=4 imbalance_ratio=1.175171 max_load_reduction=55909 max_load_reduction_pct=0.853104 moved_to_echo=33278 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4082, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 1, 3, 4096, 3176, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3983, 0, 11, 0, 44, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1757, 0, 0, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3210, 0, 1960, 0, 16, 1056, 915, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8653, 8134, 8184, 8842, 8191, 8191, 7157] avg=8192.0000 max=8842 max_rank=4 min=7157 min_rank=7 imbalance_ratio=1.079346 max_load_reduction=56694 max_load_reduction_pct=0.865082 moved_to_echo=32381 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 7, 0, 2, 0, 4088, 6, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 5, 1534, 5, 23, 36, 4089, 3171, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 0, 4054, 7, 17, 5, 44, 0, 0, 0, 0, 0, 0, 0, 6, 7, 4088, 0, 6, 0, 3, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4, 1914, 3, 0, 2982, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 9, 4089, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 4067, 0, 2, 4090, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 6, 3, 2290, 0, 2426, 0, 0, 1054, 913, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8868, 8223, 8197, 8987, 8201, 8176, 6692] avg=8192.0000 max=8987 max_rank=4 min=6692 min_rank=7 imbalance_ratio=1.097046 max_load_reduction=56549 max_load_reduction_pct=0.862869 moved_to_echo=32056 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4088, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 0, 0, 13, 4096, 3176, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3987, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 986, 0, 0, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3273, 0, 3531, 0, 6, 1056, 915, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7787, 8127, 8192, 8071, 8192, 8194, 8783] avg=8192.0000 max=8783 max_rank=7 min=7787 min_rank=1 imbalance_ratio=1.072144 max_load_reduction=56753 max_load_reduction_pct=0.865982 moved_to_echo=33240 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 653, 0, 0, 6, 4096, 3176, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4055, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 852, 0, 1, 2989, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3191, 0, 3529, 0, 1, 1056, 915, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 7931, 8195, 8196, 7939, 8192, 8194, 8692] avg=8192.0000 max=8692 max_rank=7 min=7931 min_rank=1 imbalance_ratio=1.061035 max_load_reduction=56844 max_load_reduction_pct=0.867371 moved_to_echo=33017 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 0, 0, 9, 4096, 3176, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4057, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1078, 0, 0, 2989, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2119, 0, 3827, 0, 2, 1056, 915, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8478, 8197, 8195, 8164, 8191, 8194, 7919] avg=8192.0000 max=8478 max_rank=1 min=7919 min_rank=7 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=32468 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4086, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 23, 4096, 3178, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4061, 0, 1, 0, 50, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 443, 0, 0, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 7, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3615, 0, 3866, 0, 5, 1063, 917, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 7585, 8208, 8189, 7508, 8187, 8203, 9467] avg=8192.0000 max=9467 max_rank=7 min=7508 min_rank=4 imbalance_ratio=1.155640 max_load_reduction=56069 max_load_reduction_pct=0.855545 moved_to_echo=33365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 0, 32384, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 8960, 128, 128, 23936, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 23936, 128, 25856, 0, 128, 8576, 7424, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 0, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 7296, 128, 128, 256, 32768, 25472, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 0, 0, 4094, 4092, 2, 4096, 4095, 0, 0, 4096, 0, 4096, 0, 0, 49, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4043, 0, 2, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[7, 0, 8, 6, 160, 2, 0, 4049, 4022, 4, 4090, 4091, 8, 0, 4086, 7, 4018, 0, 5, 203, 0, 3, 4088, 0, 0, 0, 25, 5, 5, 4088, 6, 0, 4, 0, 37, 5, 6, 4074, 1, 0, 2, 3846, 0, 0, 4, 1, 19, 6, 4073, 0, 4089, 5, 0, 4089, 4080, 0, 0, 5, 4, 14, 10, 4089, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 34, 0, 1, 4067, 3974, 0, 4096, 4091, 0, 0, 4096, 0, 4086, 0, 0, 400, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 4095, 0, 0, 0, 3815, 5, 0, 0, 0, 0, 0, 4096, 4, 4091, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 59, 0, 8, 4085, 3988, 0, 4096, 4092, 0, 0, 4096, 0, 4002, 0, 0, 159, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 16, 0, 0, 4078, 0, 0, 0, 4082, 7, 2, 0, 0, 0, 0, 4096, 1, 4089, 0, 0, 4096, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 4080, 3850, 0, 4096, 4096, 0, 0, 4096, 0, 4088, 0, 0, 522, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3816, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 57, 0, 0, 4096, 3909, 0, 4096, 4096, 0, 0, 4096, 0, 4039, 0, 0, 286, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 0, 4095, 4072, 1, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 474, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3639, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 1, 4086, 3963, 0, 4096, 4094, 0, 0, 4096, 0, 4093, 0, 0, 779, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 3458, 1, 1, 0, 0, 0, 0, 4096, 1, 4094, 0, 0, 4096, 4095, 0, 0, 0, 0, 2, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 34, 0, 1, 4067, 3733, 327, 2, 0, 0, 0, 0, 0, 3974, 0, 0, 4091, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 400, 0, 0, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 3837, 251, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4095, 0, 0, 3843, 249, 0, 0, 0, 0, 0, 0, 0, 3815, 5, 0, 0, 0, 0, 0, 4096, 253, 0, 0, 0, 0, 0, 0, 4096, 4, 4091, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8177, 8255, 8190, 8194, 8169, 8197, 8190] avg=8192.0000 max=8255 max_rank=2 min=8164 min_rank=0 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 59, 0, 8, 4085, 3733, 327, 2, 0, 0, 0, 0, 0, 3988, 0, 0, 4092, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 159, 0, 0, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3841, 251, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4078, 0, 0, 3842, 250, 0, 0, 0, 0, 0, 0, 0, 4082, 7, 2, 0, 0, 0, 0, 4096, 254, 0, 0, 0, 0, 0, 0, 4096, 1, 4089, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8192, 7930, 8188, 8186, 8441, 8191, 8194] avg=8192.0000 max=8441 max_rank=5 min=7930 min_rank=2 imbalance_ratio=1.030396 max_load_reduction=57095 max_load_reduction_pct=0.871201 moved_to_echo=33012 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 8, 6, 160, 2, 0, 4049, 3724, 327, 2, 0, 0, 0, 0, 0, 4022, 4, 0, 4091, 8, 0, 112, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 5, 203, 0, 3, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 5, 5, 4088, 6, 0, 3826, 250, 0, 0, 0, 0, 0, 0, 4, 0, 37, 5, 6, 4074, 1, 0, 3835, 249, 0, 0, 0, 0, 0, 0, 2, 3846, 0, 0, 4, 1, 19, 6, 4090, 254, 0, 0, 0, 0, 0, 0, 4073, 0, 4089, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 14, 10, 4087, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8244, 7990, 8205, 8211, 8222, 8172, 8207] avg=8192.0000 max=8285 max_rank=0 min=7990 min_rank=2 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32812 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 0, 0, 4094, 3733, 327, 2, 0, 0, 0, 0, 0, 4092, 2, 0, 4095, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 49, 0, 0, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3841, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3843, 252, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 2, 0, 0, 0, 0, 4096, 253, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8301, 7914, 8188, 8191, 8394, 8194, 8190] avg=8192.0000 max=8394 max_rank=5 min=7914 min_rank=2 imbalance_ratio=1.024658 max_load_reduction=57142 max_load_reduction_pct=0.871918 moved_to_echo=32969 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 0, 4095, 3733, 327, 2, 0, 0, 0, 0, 0, 4072, 1, 0, 4096, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 474, 0, 0, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3841, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3842, 250, 0, 0, 0, 0, 0, 0, 0, 3639, 0, 0, 0, 0, 0, 0, 4096, 254, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8281, 8339, 8188, 8188, 7989, 8197, 8190] avg=8192.0000 max=8339 max_rank=2 min=7989 min_rank=5 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32564 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 4080, 3733, 327, 2, 0, 0, 0, 0, 0, 3850, 0, 0, 4096, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 522, 0, 0, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 3841, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3842, 250, 0, 0, 0, 0, 0, 0, 0, 3816, 0, 0, 0, 0, 0, 0, 4096, 254, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8058, 8379, 8195, 8188, 8166, 8197, 8190] avg=8192.0000 max=8379 max_rank=2 min=8058 min_rank=1 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32741 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 57, 0, 0, 4096, 3733, 327, 2, 0, 0, 0, 0, 0, 3909, 0, 0, 4096, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 286, 0, 0, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3841, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3843, 250, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 4096, 253, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8117, 8094, 8188, 8189, 8346, 8197, 8190] avg=8192.0000 max=8346 max_rank=5 min=8094 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32922 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 1, 4086, 3735, 333, 9, 0, 0, 0, 0, 0, 3963, 0, 0, 4094, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 779, 0, 0, 3763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3846, 252, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 3843, 249, 0, 0, 0, 0, 0, 0, 0, 3458, 1, 1, 0, 0, 0, 0, 4096, 253, 0, 0, 0, 0, 0, 0, 4096, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4087, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8166, 8635, 8194, 8189, 7809, 8191, 8185] avg=8192.0000 max=8635 max_rank=2 min=7809 min_rank=5 imbalance_ratio=1.054077 max_load_reduction=56901 max_load_reduction_pct=0.868240 moved_to_echo=32374 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31872, 128, 0, 32768, 128, 0, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 0, 128, 2944, 0, 128, 30208, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 30720, 128, 128, 128, 128, 128, 128, 32768, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 128, 32768, 128, 0, 30720, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 384, 128, 128, 32768, 29952, 2688, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 32768, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 128, 0, 30848, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[8, 0, 2, 4089, 1, 6, 0, 4089, 7, 4092, 7, 7, 0, 228, 7, 0, 4089, 0, 4, 0, 6, 0, 0, 4077, 4083, 4088, 3770, 4088, 7, 4087, 4090, 3, 4065, 6, 6, 0, 6, 0, 0, 0, 0, 0, 16, 0, 6, 5, 0, 8, 4085, 0, 7, 0, 3064, 7, 0, 3, 0, 4089, 4089, 0, 1, 1138, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4081, 0, 0, 0, 397, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 3950, 4096, 0, 4096, 4091, 0, 4039, 0, 5, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2417, 0, 0, 0, 0, 4096, 4096, 0, 0, 1477, 13, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 2459, 0, 6, 4096, 0, 0, 0, 0, 1, 0, 4094, 4096, 4096, 4063, 4096, 0, 4096, 4087, 0, 4083, 0, 7, 7, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 742, 0, 0, 0, 0, 4096, 4096, 0, 0, 936, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 637, 0, 1, 4095, 0, 1, 2, 0, 0, 0, 4096, 4096, 4094, 4095, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1578, 0, 0, 0, 0, 4096, 4096, 0, 0, 1881, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 77, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 4059, 4096, 0, 4096, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1728, 0, 0, 0, 0, 4096, 4096, 0, 1, 2332, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 407, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4051, 4096, 0, 4096, 4096, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1668, 0, 0, 0, 0, 4096, 4096, 0, 0, 2098, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1224, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4096, 4096, 4095, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1846, 0, 0, 0, 0, 4096, 4096, 0, 0, 1027, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 1416, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 4095, 4096, 0, 4096, 4094, 0, 4095, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 843, 0, 0, 0, 0, 4096, 4096, 0, 0, 1843, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 397, 0, 1, 3241, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 76, 0, 3950, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 5, 0, 0, 0, 0, 8, 4096, 10, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2417, 0, 0, 0, 1590, 769, 0, 0, 0, 0, 0, 0, 0, 2505, 4096, 0, 0, 1477, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7720, 8193, 8117, 8160, 8191, 8872, 8091] avg=8192.0000 max=8872 max_rank=6 min=7720 min_rank=1 imbalance_ratio=1.083008 max_load_reduction=56664 max_load_reduction_pct=0.864624 moved_to_echo=33314 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 2459, 0, 6, 3241, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 76, 0, 4063, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 7, 7, 0, 0, 0, 4, 4096, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 742, 0, 0, 0, 1590, 769, 0, 0, 0, 0, 0, 0, 0, 2505, 4096, 0, 0, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 9794, 8192, 8226, 8210, 8189, 7196, 7537] avg=8192.0000 max=9794 max_rank=1 min=7196 min_rank=6 imbalance_ratio=1.195557 max_load_reduction=55742 max_load_reduction_pct=0.850555 moved_to_echo=31132 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 2, 4089, 1, 6, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4092, 7, 7, 0, 228, 7, 0, 3230, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4, 0, 6, 0, 0, 4077, 1, 0, 0, 0, 0, 0, 0, 0, 77, 0, 3770, 0, 7, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 6, 6, 0, 6, 0, 0, 0, 4088, 10, 2, 0, 0, 0, 0, 0, 0, 0, 16, 0, 6, 5, 0, 8, 4088, 4085, 0, 0, 0, 0, 0, 0, 4085, 0, 7, 0, 3064, 7, 0, 3, 1588, 766, 0, 0, 0, 0, 0, 0, 0, 2500, 4089, 0, 1, 1138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 7578, 8177, 7947, 8183, 8208, 9520, 7728] avg=8192.0000 max=9520 max_rank=6 min=7578 min_rank=1 imbalance_ratio=1.162109 max_load_reduction=56016 max_load_reduction_pct=0.854736 moved_to_echo=33639 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 637, 0, 1, 3241, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 2, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 76, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1578, 0, 0, 0, 1590, 769, 0, 0, 0, 0, 0, 0, 0, 2505, 4096, 0, 0, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7975, 8195, 8267, 8202, 8190, 8033, 8482] avg=8192.0000 max=8482 max_rank=7 min=7975 min_rank=1 imbalance_ratio=1.035400 max_load_reduction=57054 max_load_reduction_pct=0.870575 moved_to_echo=32907 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 407, 0, 1, 3241, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 76, 0, 4051, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1668, 0, 0, 0, 1590, 769, 0, 0, 0, 0, 0, 0, 0, 2505, 4096, 0, 0, 2098, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7739, 8193, 8223, 8172, 8190, 8123, 8704] avg=8192.0000 max=8704 max_rank=7 min=7739 min_rank=1 imbalance_ratio=1.062500 max_load_reduction=56832 max_load_reduction_pct=0.867188 moved_to_echo=33189 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 77, 0, 5, 3241, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 76, 0, 4059, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4095, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1728, 0, 0, 0, 1590, 769, 0, 0, 0, 0, 0, 0, 0, 2505, 4096, 0, 1, 2332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7419, 8193, 8231, 8194, 8190, 8183, 8934] avg=8192.0000 max=8934 max_rank=7 min=7419 min_rank=1 imbalance_ratio=1.090576 max_load_reduction=56602 max_load_reduction_pct=0.863678 moved_to_echo=33501 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1224, 0, 0, 3241, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 76, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1846, 0, 0, 0, 1590, 769, 0, 0, 0, 0, 0, 0, 0, 2505, 4096, 0, 0, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8561, 8193, 8268, 8204, 8189, 8301, 7628] avg=8192.0000 max=8561 max_rank=1 min=7628 min_rank=7 imbalance_ratio=1.045044 max_load_reduction=56975 max_load_reduction_pct=0.869370 moved_to_echo=32322 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 1416, 0, 2, 3242, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 6, 3, 0, 0, 0, 0, 0, 0, 68, 0, 4095, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 1, 4096, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 843, 0, 0, 0, 1597, 772, 0, 0, 0, 0, 0, 0, 0, 2493, 4096, 0, 0, 1843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8750, 8200, 8257, 8211, 8189, 7308, 8432] avg=8192.0000 max=8750 max_rank=1 min=7308 min_rank=6 imbalance_ratio=1.068115 max_load_reduction=56786 max_load_reduction_pct=0.866486 moved_to_echo=32140 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 0, 13952, 128, 0, 128, 12800, 6272, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 0, 6912, 128, 128, 25984, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 128, 128, 128, 128, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 20096, 32768, 0, 128, 12800, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[640, 0, 32256, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[13, 0, 0, 4085, 2, 0, 0, 58, 2, 0, 42, 4078, 0, 6, 7, 0, 0, 7, 2, 9, 0, 1, 0, 5, 839, 4087, 7, 6, 0, 3, 4088, 9, 1, 4089, 0, 3273, 4066, 0, 7, 8, 4090, 0, 15, 2, 0, 0, 4087, 7, 3976, 7, 4081, 4090, 7, 0, 7, 4089, 10, 6, 4087, 4087, 0, 1, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 21, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 4089, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3561, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 5, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 51, 4096, 0, 0, 0, 0, 5, 0, 18, 0, 0, 0, 0, 0, 2518, 4077, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1522, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4072, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4091, 4096, 0, 24, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2019, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2081, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3020, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1077, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[12, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 28, 4096, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 2323, 4084, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1757, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 4079, 0, 4096, 4094, 0, 0, 0, 4094, 0, 0, 4092, 4091, 0, 10, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[19, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 30, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 4075, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3452, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4070, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1394, 4090, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2701, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 4093, 0, 0, 0, 4096, 0, 0, 4095, 4095, 0, 3, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 4096, 0, 0, 0, 0, 4072, 7, 4, 0, 0, 0, 0, 0, 0, 0, 51, 4096, 0, 0, 0, 0, 2418, 1653, 2, 0, 0, 0, 0, 0, 5, 0, 18, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 2518, 4077, 0, 0, 0, 0, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1671, 0, 1522, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8220, 8211, 9038, 7289, 8192, 8192, 8209] avg=8192.0000 max=9038 max_rank=3 min=7289 min_rank=4 imbalance_ratio=1.103271 max_load_reduction=56498 max_load_reduction_pct=0.862091 moved_to_echo=31882 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4072, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2418, 1653, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 2019, 4096, 0, 0, 0, 0, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1671, 0, 2081, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8169, 8188, 8558, 7848, 8192, 8216, 8186] avg=8192.0000 max=8558 max_rank=3 min=7848 min_rank=4 imbalance_ratio=1.044678 max_load_reduction=56978 max_load_reduction_pct=0.869415 moved_to_echo=32442 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 4085, 2, 0, 0, 58, 4066, 7, 4, 0, 0, 0, 0, 0, 2, 0, 42, 4078, 0, 6, 7, 0, 2414, 1649, 2, 0, 0, 0, 0, 0, 0, 7, 2, 9, 0, 1, 0, 5, 4089, 4083, 0, 0, 0, 0, 0, 0, 839, 4087, 7, 6, 0, 3, 2439, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1668, 0, 3273, 4066, 0, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 15, 2, 0, 0, 4087, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 7, 4081, 24, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 6, 4085, 4087, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8200, 8196, 7390, 9023, 8201, 8102, 8189] avg=8192.0000 max=9023 max_rank=4 min=7390 min_rank=3 imbalance_ratio=1.101440 max_load_reduction=56513 max_load_reduction_pct=0.862320 moved_to_echo=33515 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 4096, 0, 0, 4, 0, 4070, 7, 5, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 0, 0, 0, 0, 2418, 1653, 2, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 4094, 4091, 0, 0, 0, 0, 0, 0, 2323, 4084, 0, 0, 0, 0, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1671, 0, 1757, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4091, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8197, 8191, 8850, 7524, 8190, 8199, 8191] avg=8192.0000 max=8850 max_rank=3 min=7524 min_rank=4 imbalance_ratio=1.080322 max_load_reduction=56686 max_load_reduction_pct=0.864960 moved_to_echo=32104 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 0, 0, 4072, 7, 5, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 0, 0, 0, 0, 2418, 1653, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 523, 4089, 0, 0, 0, 0, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1671, 0, 3561, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8190, 8187, 7055, 9328, 8192, 8208, 8195] avg=8192.0000 max=9328 max_rank=4 min=7055 min_rank=3 imbalance_ratio=1.138672 max_load_reduction=56208 max_load_reduction_pct=0.857666 moved_to_echo=33923 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4071, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2418, 1653, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 3020, 4096, 0, 0, 0, 0, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1671, 0, 1077, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8169, 8187, 9559, 6844, 8192, 8216, 8190] avg=8192.0000 max=9559 max_rank=3 min=6844 min_rank=4 imbalance_ratio=1.166870 max_load_reduction=55977 max_load_reduction_pct=0.854141 moved_to_echo=31442 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 0, 4096, 0, 0, 0, 0, 4072, 7, 5, 0, 0, 0, 0, 0, 0, 0, 30, 4096, 0, 0, 0, 0, 2418, 1653, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 642, 4075, 0, 0, 0, 0, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1671, 0, 3452, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8199, 8187, 7160, 9219, 8192, 8190, 8190] avg=8192.0000 max=9219 max_rank=4 min=7160 min_rank=3 imbalance_ratio=1.125366 max_load_reduction=56317 max_load_reduction_pct=0.859329 moved_to_echo=33791 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 4071, 11, 5, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 2421, 1654, 7, 7, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 1394, 4090, 0, 0, 0, 0, 2442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1664, 0, 2701, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4095, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8192, 8189, 7926, 8461, 8185, 8213, 8186] avg=8192.0000 max=8461 max_rank=4 min=7926 min_rank=3 imbalance_ratio=1.032837 max_load_reduction=57075 max_load_reduction_pct=0.870895 moved_to_echo=33045 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 128, 32768, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 13440, 0, 19456, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 256, 32768, 0, 128, 128, 0, 19456, 13312, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 128, 128, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[13312, 32768, 128, 128, 0, 128, 19584, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4089, 0, 4089, 4087, 4, 4, 15, 6, 6, 4089, 0, 7, 0, 69, 17, 4065, 4084, 0, 7, 7, 10, 1, 6, 0, 3, 4, 7, 0, 5, 0, 4080, 4086, 0, 4, 75, 23, 7, 0, 4087, 0, 4071, 0, 6, 39, 0, 8, 0, 4083, 0, 12, 0, 4082, 8, 22, 10, 0, 0, 4, 4077, 3895, 4088, 0, 0, 4088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 1, 4088, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 8, 4092, 0, 6, 0, 0, 0, 0, 4096, 4096, 4096, 3, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 6, 4085, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 7, 4089, 0, 10, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 1, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 1, 4094, 4096, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 4095, 0, 1, 0, 0, 0, 0, 4096, 4096, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4069, 8, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8185, 8190, 8191, 8179, 8194, 8186, 8219] avg=8192.0000 max=8219 max_rank=7 min=8179 min_rank=4 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4085, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4069, 8, 6, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4086, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 10, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8185, 8190, 8202, 8179, 8178, 8196, 8219] avg=8192.0000 max=8219 max_rank=7 min=8178 min_rank=5 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4069, 8, 6, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4090, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4092, 0, 6, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 27, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8187, 8191, 8196, 8179, 8181, 8195, 8222] avg=8192.0000 max=8222 max_rank=7 min=8179 min_rank=4 imbalance_ratio=1.003662 max_load_reduction=57314 max_load_reduction_pct=0.874542 moved_to_echo=32777 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 4087, 4, 4, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4082, 0, 7, 0, 69, 17, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 7, 7, 10, 1, 6, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 3, 4, 7, 0, 5, 0, 4080, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 75, 23, 7, 0, 4087, 0, 4061, 8, 6, 0, 0, 0, 0, 0, 4071, 0, 6, 39, 0, 8, 0, 4083, 1, 1, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4082, 8, 22, 10, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 0, 4, 4077, 3895, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8246, 8202, 8184, 8271, 8209, 8216, 8003] avg=8192.0000 max=8271 max_rank=4 min=8003 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32699 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4069, 9, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8185, 8191, 8191, 8180, 8194, 8185, 8219] avg=8192.0000 max=8219 max_rank=7 min=8180 min_rank=4 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4069, 8, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8185, 8190, 8191, 8179, 8194, 8185, 8219] avg=8192.0000 max=8219 max_rank=7 min=8179 min_rank=4 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32777 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4069, 8, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8185, 8191, 8191, 8179, 8192, 8187, 8219] avg=8192.0000 max=8219 max_rank=7 min=8179 min_rank=4 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32777 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4073, 9, 12, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 1, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8178, 8191, 8190, 8190, 8194, 8186, 8216] avg=8192.0000 max=8216 max_rank=7 min=8178 min_rank=1 imbalance_ratio=1.002930 max_load_reduction=57320 max_load_reduction_pct=0.874634 moved_to_echo=32786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 0, 32768, 0, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 32640, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4091, 4078, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 4096, 4096, 0, 3910, 9, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 0, 4096, 4093, 0, 0, 2, 0, 36, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4083, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3875, 4, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 46, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3549, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4089, 4036, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 4096, 4096, 0, 2469, 10, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 7, 0, 159, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3260, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4061, 0, 1, 7, 4, 0, 4079, 4092, 1706, 8, 0, 0, 0, 0, 7, 7, 0, 2, 0, 24, 6, 97, 0, 4089, 0, 3, 4086, 4088, 0, 2442, 0, 29, 4087, 4089, 4073, 0, 0, 0, 0, 7, 8, 4089, 4038, 19, 0, 15, 0, 9, 4074, 4087, 0, 53, 6, 4066, 5, 0, 0, 8, 8, 0, 9, 7, 4, 3937] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4087, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 2708, 4, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 31, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4070] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4091, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 3922, 1, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 17, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 4096, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3275, 773, 31, 1, 0, 0, 0, 0, 5, 0, 4096, 821, 0, 3910, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4093, 0, 0, 2, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4085, 3301, 814, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7538, 8176, 8841, 8197, 8196, 8191, 8206] avg=8192.0000 max=8841 max_rank=3 min=7538 min_rank=1 imbalance_ratio=1.079224 max_load_reduction=56695 max_load_reduction_pct=0.865097 moved_to_echo=32790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 550, 0, 0, 0, 0, 0, 0, 0, 4096, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3275, 773, 31, 1, 0, 0, 0, 0, 0, 0, 4096, 821, 0, 3549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3305, 814, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 7924, 8176, 8466, 8193, 8161, 8191, 8215] avg=8192.0000 max=8466 max_rank=3 min=7924 min_rank=1 imbalance_ratio=1.033447 max_load_reduction=57070 max_load_reduction_pct=0.870819 moved_to_echo=32760 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 18, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 1528, 0, 0, 0, 0, 0, 0, 0, 4096, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3275, 772, 31, 1, 0, 0, 0, 0, 14, 0, 4096, 821, 0, 2469, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4081, 0, 0, 7, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4063, 3299, 814, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8902, 8175, 7410, 8196, 8312, 8191, 8200] avg=8192.0000 max=8902 max_rank=1 min=7410 min_rank=3 imbalance_ratio=1.086670 max_load_reduction=56634 max_load_reduction_pct=0.864166 moved_to_echo=32899 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4061, 0, 1, 7, 4, 0, 18, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1706, 8, 0, 0, 0, 0, 7, 7, 4087, 3272, 0, 0, 0, 0, 0, 0, 0, 2, 0, 24, 6, 97, 0, 4089, 3269, 770, 31, 1, 0, 0, 0, 0, 0, 3, 4086, 819, 0, 2442, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 5, 4073, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4058, 4038, 19, 0, 15, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 53, 6, 4066, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 0, 9, 7, 4, 3937, 3291, 812, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 9087, 8289, 7379, 8172, 8147, 8203, 8076] avg=8192.0000 max=9087 max_rank=1 min=7379 min_rank=3 imbalance_ratio=1.109253 max_load_reduction=56449 max_load_reduction_pct=0.861343 moved_to_echo=32598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 18, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 4096, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3275, 773, 31, 1, 0, 0, 0, 0, 0, 0, 4096, 821, 0, 3875, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4094, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4089, 3305, 814, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 7563, 8176, 8796, 8196, 8205, 8191, 8212] avg=8192.0000 max=8796 max_rank=3 min=7563 min_rank=1 imbalance_ratio=1.073730 max_load_reduction=56740 max_load_reduction_pct=0.865784 moved_to_echo=32804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 835, 0, 0, 0, 0, 0, 0, 0, 4096, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3275, 773, 31, 1, 0, 0, 0, 0, 0, 0, 4096, 821, 0, 3260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3305, 814, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8209, 8176, 8177, 8194, 8165, 8191, 8214] avg=8192.0000 max=8214 max_rank=7 min=8165 min_rank=5 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 18, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 1383, 0, 0, 0, 0, 0, 0, 0, 4096, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3275, 773, 31, 1, 0, 0, 0, 0, 0, 0, 4096, 821, 0, 2708, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4096, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4070, 3305, 814, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8757, 8176, 7629, 8196, 8192, 8191, 8194] avg=8192.0000 max=8757 max_rank=1 min=7629 min_rank=3 imbalance_ratio=1.068970 max_load_reduction=56779 max_load_reduction_pct=0.866379 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 7, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 4096, 3280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3278, 779, 34, 5, 0, 0, 0, 0, 1, 0, 4096, 818, 0, 3922, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 3309, 816, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7556, 8192, 8838, 8192, 8158, 8187, 8219] avg=8192.0000 max=8838 max_rank=3 min=7556 min_rank=1 imbalance_ratio=1.078857 max_load_reduction=56698 max_load_reduction_pct=0.865143 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 6656, 0, 26240, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32512, 32768, 128, 0, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 32768, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 32640, 26496, 6528, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 32768, 26240, 6272, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6656, 128, 0, 0, 0, 0, 128, 128, 32768, 26240, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4038, 0, 0, 0, 0, 4096, 4095, 68, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 298, 0, 181, 3608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 10, 4096, 4096, 0, 0, 0, 47, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 438, 0, 21, 3582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4050, 0, 23, 7, 3839, 4, 0, 0, 4026, 1, 6, 0, 0, 4083, 4091, 56, 4087, 4088, 0, 2, 0, 875, 0, 8, 4086, 0, 0, 6, 11, 4084, 4087, 0, 8, 0, 13, 6, 0, 0, 5, 0, 4088, 4050, 0, 4086, 521, 6, 176, 2825, 0, 0, 9, 2, 2, 31, 0, 0, 3, 4089, 8, 4073, 0, 10, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 4095, 2, 4096, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1364, 0, 43, 2693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 4089, 4085, 0, 4096, 4096, 0, 0, 0, 36, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 4096, 251, 0, 37, 3787, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 0, 0, 4091, 4083, 18, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 4, 4096, 2313, 0, 60, 1718, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4090, 38, 4096, 4096, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 776, 0, 60, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 4095, 4094, 8, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 1311, 0, 59, 2731, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 31, 1, 1, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 26, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 68, 4096, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3113, 298, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 0, 0, 0, 0, 0, 0, 0, 4070, 983, 112, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8159, 8117, 8191, 8189, 7688, 8776, 8191] avg=8192.0000 max=8776 max_rank=6 min=7688 min_rank=5 imbalance_ratio=1.071289 max_load_reduction=56760 max_load_reduction_pct=0.866089 moved_to_echo=32844 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4050, 0, 23, 7, 3839, 4, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 1, 6, 0, 0, 26, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4087, 3945, 0, 2, 0, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4086, 0, 0, 6, 11, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 13, 6, 0, 0, 5, 4088, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 3105, 521, 6, 176, 0, 0, 0, 0, 0, 0, 0, 0, 2825, 0, 0, 9, 2, 2, 31, 0, 4057, 981, 112, 2, 0, 0, 0, 0, 0, 3, 4089, 8, 4073, 0, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7954, 8150, 8965, 8195, 8205, 7858, 8021, 8188] avg=8192.0000 max=8965 max_rank=2 min=7858 min_rank=5 imbalance_ratio=1.094360 max_load_reduction=56571 max_load_reduction_pct=0.863205 moved_to_echo=32272 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 31, 1, 1, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 26, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3953, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 3113, 251, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 3787, 0, 0, 0, 0, 0, 0, 0, 4063, 983, 112, 2, 0, 0, 0, 0, 7, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8198, 8085, 8191, 8190, 7502, 8947, 8198] avg=8192.0000 max=8947 max_rank=6 min=7502 min_rank=5 imbalance_ratio=1.092163 max_load_reduction=56589 max_load_reduction_pct=0.863480 moved_to_echo=32837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 31, 1, 1, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 26, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 3113, 2313, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 1718, 0, 0, 0, 0, 0, 0, 0, 4065, 983, 112, 2, 0, 0, 0, 0, 4, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8195, 8067, 8196, 8192, 9586, 6880, 8195] avg=8192.0000 max=9586 max_rank=5 min=6880 min_rank=6 imbalance_ratio=1.170166 max_load_reduction=55950 max_load_reduction_pct=0.853729 moved_to_echo=32853 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 31, 1, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 26, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 3953, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3113, 438, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 3582, 0, 0, 0, 0, 0, 0, 0, 4070, 983, 112, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8216, 8106, 8191, 8190, 7668, 8749, 8191] avg=8192.0000 max=8749 max_rank=6 min=7668 min_rank=5 imbalance_ratio=1.067993 max_load_reduction=56787 max_load_reduction_pct=0.866501 moved_to_echo=32798 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 31, 1, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 26, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 3953, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3113, 1364, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 2693, 0, 0, 0, 0, 0, 0, 0, 4070, 983, 112, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8211, 8052, 8191, 8190, 8616, 7860, 8191] avg=8192.0000 max=8616 max_rank=5 min=7860 min_rank=6 imbalance_ratio=1.051758 max_load_reduction=56920 max_load_reduction_pct=0.868530 moved_to_echo=32857 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 31, 1, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 26, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 3953, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3113, 776, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 0, 0, 0, 0, 0, 0, 0, 4070, 983, 112, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8210, 8094, 8191, 8190, 8045, 8390, 8191] avg=8192.0000 max=8390 max_rank=6 min=8045 min_rank=5 imbalance_ratio=1.024170 max_load_reduction=57146 max_load_reduction_pct=0.871979 moved_to_echo=32816 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 36, 2, 2, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 19, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3106, 1311, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 4076, 990, 114, 2, 0, 0, 0, 0, 1, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8197, 8050, 8190, 8190, 8573, 7913, 8191] avg=8192.0000 max=8573 max_rank=5 min=7913 min_rank=6 imbalance_ratio=1.046509 max_load_reduction=56963 max_load_reduction_pct=0.869186 moved_to_echo=32867 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 0, 128, 128, 32512, 128, 0, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32640, 128, 128, 0, 0, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[256, 32768, 31616, 0, 128, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 128, 24960, 7296, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[24192, 0, 0, 128, 128, 128, 128, 0, 32640, 7936, 1024, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 293, 0, 0, 0, 0, 4096, 0, 2653, 0, 0, 0, 2921, 0, 0, 4096, 0, 2319, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 1, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 5, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[8, 0, 4, 4092, 3, 1, 7, 0, 456, 0, 8, 0, 0, 4089, 1, 1081, 20, 0, 5, 4003, 8, 8, 4085, 10, 2072, 11, 62, 4071, 0, 0, 6, 4087, 4090, 0, 0, 4061, 0, 2, 0, 36, 13, 564, 4053, 1, 7, 0, 4088, 0, 0, 4081, 4076, 13, 3, 4089, 8, 0, 10, 4026, 4086, 0, 2, 29, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4038, 0, 0, 0, 0, 70, 0, 0, 0, 0, 4096, 0, 3337, 0, 0, 0, 3006, 0, 0, 4096, 0, 1744, 0, 10, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4084, 2, 0, 13, 4096, 0, 7, 4089, 4096, 82, 0, 4094, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 678, 0, 0, 0, 0, 4096, 0, 3082, 0, 0, 0, 3664, 0, 0, 4096, 0, 750, 0, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 15, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4079, 0, 0, 0, 0, 48, 0, 0, 0, 0, 4096, 0, 3582, 0, 0, 0, 3787, 0, 0, 4096, 0, 769, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 5, 4096, 4096, 17, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4077, 0, 0, 0, 0, 1085, 0, 0, 0, 0, 4096, 0, 3023, 0, 0, 0, 3039, 0, 0, 4096, 0, 1012, 0, 8, 4096, 0, 0, 0, 4095, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 27, 4088, 1, 0, 7, 4092, 0, 6, 4096, 4096, 23, 0, 4092, 0, 0, 0, 4096, 4096, 0, 4, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4072, 0, 0, 0, 0, 662, 0, 0, 0, 0, 4095, 0, 2953, 0, 0, 0, 3790, 0, 0, 4096, 0, 780, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 3, 4096, 0, 4, 4096, 4096, 23, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 0, 0, 0, 0, 924, 0, 0, 0, 0, 4096, 0, 2029, 0, 0, 0, 3248, 0, 0, 4096, 0, 1989, 0, 2, 4096, 1, 0, 0, 4096, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 2, 4096, 0, 1, 4096, 4096, 3, 0, 4095, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 4095, 15, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 4096, 0, 2653, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2921, 0, 0, 4096, 0, 571, 72, 15, 0, 0, 0, 0, 0, 2319, 0, 0, 4096, 0, 0, 0, 2659, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 1, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 5, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 7893, 7675, 9074, 8193, 8124, 8174, 8199] avg=8192.0000 max=9074 max_rank=3 min=7675 min_rank=2 imbalance_ratio=1.107666 max_load_reduction=56462 max_load_reduction_pct=0.861542 moved_to_echo=32690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4038, 0, 0, 0, 0, 4094, 15, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 4096, 0, 3337, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3006, 0, 0, 4096, 0, 571, 72, 15, 0, 0, 0, 0, 0, 1744, 0, 10, 4096, 0, 0, 0, 2659, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4084, 2, 0, 13, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 4078, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8354, 7760, 8509, 8193, 8124, 8256, 8193] avg=8192.0000 max=8509 max_rank=3 min=7760 min_rank=2 imbalance_ratio=1.038696 max_load_reduction=57027 max_load_reduction_pct=0.870163 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4, 4092, 3, 1, 7, 0, 4089, 15, 0, 0, 0, 0, 0, 0, 456, 0, 8, 0, 0, 4089, 1, 1081, 849, 0, 0, 0, 0, 0, 0, 0, 20, 0, 5, 4003, 8, 8, 4085, 10, 569, 72, 15, 0, 0, 0, 0, 0, 2072, 11, 62, 4071, 0, 0, 6, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4061, 0, 2, 0, 36, 1, 0, 0, 0, 0, 0, 0, 0, 13, 564, 4053, 1, 7, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4059, 13, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4026, 4086, 0, 2, 29, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 6484, 8795, 8876, 8190, 8654, 8164, 8154] avg=8192.0000 max=8876 max_rank=3 min=6484 min_rank=1 imbalance_ratio=1.083496 max_load_reduction=56660 max_load_reduction_pct=0.864563 moved_to_echo=33162 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4077, 0, 0, 0, 0, 4092, 15, 0, 0, 0, 0, 0, 0, 1085, 0, 0, 0, 0, 4096, 0, 3023, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3039, 0, 0, 4096, 0, 571, 72, 15, 0, 0, 0, 0, 0, 1012, 0, 8, 4096, 0, 0, 0, 2658, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4088, 1, 0, 7, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4078, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 9055, 7793, 7774, 8186, 8143, 8203, 8198] avg=8192.0000 max=9055 max_rank=1 min=7774 min_rank=3 imbalance_ratio=1.105347 max_load_reduction=56481 max_load_reduction_pct=0.861832 moved_to_echo=32730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 678, 0, 0, 0, 0, 4096, 0, 3082, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3664, 0, 0, 4096, 0, 571, 72, 15, 0, 0, 0, 0, 0, 750, 0, 1, 4096, 0, 0, 0, 2659, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 15, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8707, 8418, 7506, 8193, 8123, 8174, 8209] avg=8192.0000 max=8707 max_rank=1 min=7506 min_rank=3 imbalance_ratio=1.062866 max_load_reduction=56829 max_load_reduction_pct=0.867142 moved_to_echo=32699 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4079, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 4096, 0, 3582, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3787, 0, 0, 4096, 0, 571, 72, 15, 0, 0, 0, 0, 0, 769, 0, 0, 4096, 0, 0, 0, 2659, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4078, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8577, 8541, 7524, 8193, 8121, 8196, 8194] avg=8192.0000 max=8577 max_rank=1 min=7524 min_rank=3 imbalance_ratio=1.046997 max_load_reduction=56959 max_load_reduction_pct=0.869125 moved_to_echo=32704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4072, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 4095, 0, 2953, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3790, 0, 0, 4096, 0, 571, 72, 15, 0, 0, 0, 0, 0, 780, 0, 0, 4096, 0, 0, 0, 2659, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 3, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4078, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8561, 8544, 7535, 8193, 8124, 8201, 8195] avg=8192.0000 max=8561 max_rank=1 min=7535 min_rank=3 imbalance_ratio=1.045044 max_load_reduction=56975 max_load_reduction_pct=0.869370 moved_to_echo=32713 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 0, 0, 0, 0, 4095, 17, 0, 0, 0, 0, 0, 0, 924, 0, 0, 0, 0, 4096, 0, 2029, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3248, 0, 0, 4096, 0, 573, 73, 20, 0, 0, 0, 0, 0, 1989, 0, 2, 4096, 1, 0, 0, 2650, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 2, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4068, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 7905, 8010, 8738, 8195, 8123, 8168, 8194] avg=8192.0000 max=8738 max_rank=3 min=7905 min_rank=1 imbalance_ratio=1.066650 max_load_reduction=56798 max_load_reduction_pct=0.866669 moved_to_echo=32680 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 640, 32768, 128, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[11520, 128, 128, 32768, 128, 0, 128, 21376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 32768, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 27520, 128, 128, 32768, 128, 4608, 640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 32640, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4224, 0, 128, 0, 0, 32768, 128, 21760, 6912, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32640, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 110, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 3981, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 4093, 4096, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 33, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 35, 0, 4015, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 6, 0, 4096, 1, 0, 0, 0, 4090, 4095, 4089, 0, 0, 1, 0, 13, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 24, 0, 0, 0, 0, 5, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 4070, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 4096, 0, 69, 0, 0, 4095, 4096, 4086, 0, 0, 0, 0, 25, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4071, 74, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 22, 0, 3932, 0, 5, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 10, 0, 4087, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4081, 0, 0, 0, 4088, 4084, 4039, 0, 3, 11, 0, 4090, 16, 0, 0, 9, 4087, 0, 1169, 0, 0, 4024, 4087, 4088, 0, 8, 0, 111, 0, 8, 4085, 8, 0, 1, 7, 8, 12, 3443, 4, 9, 3968, 40, 5, 8, 9, 3, 0, 4088, 11, 4085, 4083, 0, 7, 36, 0, 6, 7, 2579, 0, 1018, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4094, 4095, 4090, 0, 0, 1, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 46, 0, 0, 0, 0, 1, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4053, 1, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 6, 0, 3913, 178, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4090, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 4092, 0, 3908, 139, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3990, 183, 0, 0, 0, 0, 0, 0, 4083, 24, 0, 0, 0, 0, 5, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4070, 0, 4, 0, 0, 4096, 105, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8192, 8041, 8179, 8269, 8186, 8188, 8283] avg=8192.0000 max=8283 max_rank=7 min=8041 min_rank=2 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32926 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 3913, 178, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 4095, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4095, 0, 3908, 139, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3991, 183, 0, 0, 0, 0, 0, 0, 4071, 74, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 3932, 0, 5, 0, 0, 4096, 105, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8197, 8111, 8193, 8270, 8219, 8188, 8160] avg=8192.0000 max=8270 max_rank=4 min=8111 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32837 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3912, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3908, 139, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3990, 184, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4087, 1, 0, 0, 0, 4096, 106, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8186, 8049, 8169, 8269, 8172, 8189, 8300] avg=8192.0000 max=8300 max_rank=7 min=8049 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32930 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 4088, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 11, 0, 4090, 16, 0, 0, 9, 3902, 177, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 0, 0, 4024, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 111, 0, 8, 4085, 8, 3894, 139, 22, 4, 0, 0, 0, 0, 0, 1, 7, 8, 12, 3443, 4, 9, 3983, 182, 0, 0, 0, 0, 0, 0, 3968, 40, 5, 8, 9, 3, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4081, 4083, 0, 7, 36, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2579, 0, 1018, 0, 0, 3, 0, 4087, 104, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8208, 9142, 8279, 7649, 8099, 8224, 7798] avg=8192.0000 max=9142 max_rank=2 min=7649 min_rank=4 imbalance_ratio=1.115967 max_load_reduction=56394 max_load_reduction_pct=0.860504 moved_to_echo=31770 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3913, 178, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4093, 0, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 3908, 139, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3991, 183, 0, 0, 0, 0, 0, 0, 4090, 33, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4015, 0, 0, 0, 0, 4096, 105, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8187, 8066, 8175, 8270, 8197, 8188, 8251] avg=8192.0000 max=8270 max_rank=4 min=8066 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32906 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3913, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4096, 0, 3908, 139, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3991, 183, 0, 0, 0, 0, 0, 0, 4083, 110, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3981, 0, 0, 0, 0, 4096, 105, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8187, 8049, 8186, 8270, 8267, 8188, 8187] avg=8192.0000 max=8270 max_rank=4 min=8049 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32912 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3913, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3908, 139, 22, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3991, 183, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 2, 0, 0, 0, 4096, 105, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8187, 8037, 8169, 8270, 8184, 8188, 8299] avg=8192.0000 max=8299 max_rank=7 min=8037 min_rank=2 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32941 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 3913, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4095, 0, 3911, 143, 25, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3990, 183, 0, 0, 0, 0, 0, 0, 4094, 46, 0, 0, 0, 0, 1, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4053, 1, 1, 0, 0, 4096, 105, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 8041, 8186, 8269, 8212, 8183, 8258] avg=8192.0000 max=8269 max_rank=4 min=8041 min_rank=2 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32922 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 2688, 0, 29312, 128, 128, 128, 0, 32768, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 384, 128, 128, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 128, 128, 31360, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 32128, 128, 128, 32000, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 128, 32768, 128, 31360, 1152, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 1280, 0, 0, 32768, 0, 31616, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1541, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 179, 0, 0, 0, 1, 4096, 0, 4, 0, 0, 4096, 0, 0, 0, 2295, 0, 4096, 4096, 0, 4096, 4094, 0, 3768, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[992, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 0, 2, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 2629, 0, 4096, 4096, 0, 4096, 4086, 0, 4026, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[591, 212, 1, 0, 0, 4, 11, 0, 9, 8, 9, 7, 6, 0, 4071, 52, 0, 3, 0, 3, 0, 4090, 501, 0, 0, 0, 4088, 11, 0, 0, 3671, 0, 4089, 4073, 0, 4087, 4089, 10, 3874, 4083, 47, 4087, 0, 0, 0, 13, 3414, 3, 7, 4089, 0, 0, 7, 0, 4080, 6, 8, 8, 10, 0, 2, 4007, 4086, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[794, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 143, 0, 0, 0, 3, 4096, 0, 33, 0, 0, 4096, 0, 0, 0, 3126, 0, 4096, 4096, 0, 4096, 4078, 0, 3804, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[228, 597, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 222, 0, 0, 0, 38, 4092, 0, 55, 0, 0, 4096, 0, 0, 0, 3194, 5, 4096, 4095, 0, 4096, 4059, 0, 3895, 4096, 0, 4089, 0, 0, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[194, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 64, 0, 0, 0, 9, 4096, 0, 35, 0, 0, 4096, 0, 0, 0, 3268, 0, 4096, 4096, 0, 4096, 4085, 0, 3935, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[982, 223, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 27, 0, 0, 0, 12, 4092, 0, 19, 0, 0, 4096, 0, 0, 0, 2900, 5, 4096, 4091, 0, 4096, 4079, 0, 4042, 4096, 0, 4090, 0, 5, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[700, 737, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 60, 0, 0, 0, 2, 4095, 2, 13, 0, 0, 4096, 0, 0, 0, 2792, 1, 4096, 4095, 0, 4096, 4040, 0, 3941, 4096, 0, 4095, 0, 1, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[992, 532, 0, 0, 0, 0, 0, 0, 4096, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2871, 1218, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 2, 4096, 0, 1, 2694, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2629, 0, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 4086, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8498, 8185, 8038, 7835, 8319, 8270, 8192, 8199] avg=8192.0000 max=8498 max_rank=0 min=7835 min_rank=3 imbalance_ratio=1.037354 max_load_reduction=57038 max_load_reduction_pct=0.870331 moved_to_echo=32980 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[228, 597, 0, 0, 0, 0, 0, 1, 4096, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2871, 1218, 0, 0, 0, 0, 0, 0, 222, 0, 0, 0, 38, 4092, 0, 55, 2694, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3194, 5, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 4059, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 6, 78, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7800, 8185, 8326, 8405, 8160, 8269, 8192, 8199] avg=8192.0000 max=8405 max_rank=3 min=7800 min_rank=0 imbalance_ratio=1.026001 max_load_reduction=57131 max_load_reduction_pct=0.871750 moved_to_echo=32820 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[591, 212, 1, 0, 0, 4, 11, 0, 4089, 2871, 0, 0, 0, 0, 0, 0, 9, 8, 9, 7, 6, 0, 4071, 52, 2862, 1216, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 4090, 501, 0, 2679, 1221, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 11, 0, 0, 3671, 0, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 4089, 10, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4087, 0, 0, 0, 13, 3414, 3, 77, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 0, 7, 0, 4080, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 10, 0, 2, 4007, 4086, 9, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7779, 8240, 8497, 8874, 8179, 7641, 8189, 8137] avg=8192.0000 max=8874 max_rank=3 min=7641 min_rank=5 imbalance_ratio=1.083252 max_load_reduction=56662 max_load_reduction_pct=0.864594 moved_to_echo=32146 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1541, 407, 0, 0, 0, 0, 0, 0, 4096, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2872, 1217, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 1, 4096, 0, 4, 2694, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2295, 0, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 4094, 0, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8923, 8185, 8198, 7501, 8069, 8269, 8192, 8199] avg=8192.0000 max=8923 max_rank=0 min=7501 min_rank=3 imbalance_ratio=1.089233 max_load_reduction=56613 max_load_reduction_pct=0.863846 moved_to_echo=32729 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[794, 307, 0, 0, 0, 0, 0, 0, 4096, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2871, 1218, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 3, 4096, 0, 33, 2694, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3126, 0, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 4078, 0, 3804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8075, 8185, 8194, 8332, 8089, 8270, 8192, 8199] avg=8192.0000 max=8332 max_rank=3 min=8075 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32750 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[982, 223, 0, 0, 0, 0, 0, 3, 4096, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2872, 1218, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 12, 4092, 0, 19, 2691, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2900, 5, 1109, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 4079, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 5, 0, 0, 4096, 6, 77, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8186, 8065, 8110, 8328, 8274, 8192, 8199] avg=8192.0000 max=8328 max_rank=4 min=8065 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32993 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[194, 698, 0, 0, 0, 0, 0, 0, 4096, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2871, 1218, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 9, 4096, 0, 35, 2694, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3268, 0, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 4085, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7866, 8185, 8123, 8474, 8227, 8270, 8192, 8199] avg=8192.0000 max=8474 max_rank=3 min=7866 min_rank=0 imbalance_ratio=1.034424 max_load_reduction=57062 max_load_reduction_pct=0.870697 moved_to_echo=32888 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[700, 737, 0, 0, 0, 0, 0, 1, 4096, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2872, 1217, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 2, 4095, 2, 13, 2699, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2792, 1, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 4040, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 4096, 1, 80, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8413, 8185, 8095, 8005, 8165, 8273, 8195, 8205] avg=8192.0000 max=8413 max_rank=0 min=8005 min_rank=3 imbalance_ratio=1.026978 max_load_reduction=57123 max_load_reduction_pct=0.871628 moved_to_echo=32838 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 23936, 128, 8960, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[6144, 3840, 128, 0, 0, 128, 128, 128, 32768, 23040, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 1664, 0, 0, 32640, 128, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 128, 0, 128, 128, 32768, 512, 256, 21632, 9856, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 0, 128, 32128, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 23040, 9856, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 46, 0, 4094, 0, 4096, 6, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 6, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 18, 0, 4093, 0, 4094, 8, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 4096, 5, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 24, 0, 4096, 0, 4096, 23, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 9, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 3, 0, 4095, 0, 4096, 5, 0, 0, 1, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4089, 7, 0, 3, 307, 0, 0, 4089, 7, 4086, 0, 13, 0, 0, 2, 4086, 0, 96, 4069, 5, 4083, 4, 15, 7, 6, 37, 17, 0, 3979, 0, 4081, 0, 7, 1, 4089, 0, 4082, 0, 0, 10, 0, 9, 0, 0, 132, 14, 8, 1, 10, 4001, 0, 0, 11, 4087, 3740, 0, 4052, 5, 4084, 4085, 0, 12, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4031, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 54, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8187, 8186, 8199, 8207, 8173, 8246, 8197] avg=8192.0000 max=8246 max_rank=6 min=8141 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 46, 0, 4031, 14, 0, 0, 0, 0, 0, 0, 4094, 0, 4091, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 4050, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 54, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8191, 8176, 8199, 8167, 8173, 8246, 8197] avg=8192.0000 max=8246 max_rank=6 min=8167 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32783 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 18, 0, 4031, 14, 0, 0, 0, 0, 0, 0, 4093, 0, 4089, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4077, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 54, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8190, 8179, 8200, 8193, 8173, 8246, 8197] avg=8192.0000 max=8246 max_rank=6 min=8158 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32809 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 4031, 15, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 54, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8187, 8185, 8199, 8207, 8176, 8246, 8193] avg=8192.0000 max=8246 max_rank=6 min=8143 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 9, 0, 4031, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 54, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8188, 8186, 8199, 8197, 8173, 8246, 8197] avg=8192.0000 max=8246 max_rank=6 min=8150 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 7, 0, 3, 307, 0, 0, 4022, 14, 0, 0, 0, 0, 0, 0, 4089, 7, 4081, 0, 13, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 96, 4063, 5, 0, 4, 15, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 37, 17, 0, 3979, 0, 4081, 5, 2, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4089, 0, 4082, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 10, 0, 9, 0, 0, 132, 14, 8, 4083, 4066, 0, 0, 0, 0, 0, 0, 1, 10, 4001, 0, 0, 11, 54, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 5, 4084, 5, 0, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8442, 8192, 8269, 8134, 8194, 8322, 7817, 8166] avg=8192.0000 max=8442 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.030518 max_load_reduction=57094 max_load_reduction_pct=0.871185 moved_to_echo=32499 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 24, 0, 4031, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4052, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 54, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8210, 8183, 8199, 8163, 8173, 8246, 8197] avg=8192.0000 max=8246 max_rank=6 min=8163 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 3, 0, 4033, 19, 0, 0, 0, 0, 0, 0, 4095, 0, 4090, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4090, 0, 0, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 51, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8191, 8172, 8207, 8208, 8173, 8243, 8192] avg=8192.0000 max=8243 max_rank=6 min=8150 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32816 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 0, 256, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 512, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 384, 128, 0, 32256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 32640, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True + [2026-08-30 05:00:53] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 676.4 | throughput per GPU (TFLOP/s/GPU): 492.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.257265E+00 | loss scale: 1.0 | grad norm: 5.579 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2967, 24, 320, 175, 1051, 1, 2127, 44, 920, 2969, 304, 397, 1050, 1587, 0, 217, 418, 2053, 296, 1284, 0, 596, 1319, 1570, 6, 13, 2677, 1192, 2818, 297, 2270, 451, 14, 1277, 2777, 248, 466, 361, 2475, 1253, 1687, 2, 2517, 2060, 1, 975, 1579, 1404, 2918, 46, 1393, 0, 12, 362, 1, 624, 0, 135, 1285, 1317, 2105, 1150, 1019, 2660] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2446, 14, 188, 166, 716, 0, 2253, 34, 1305, 3311, 325, 531, 1238, 2013, 0, 238, 442, 1217, 233, 1229, 0, 611, 1707, 1577, 11, 105, 2735, 1421, 2943, 494, 2873, 553, 3, 1320, 2837, 265, 441, 180, 1816, 1450, 959, 0, 2592, 2216, 0, 1200, 935, 884, 2821, 42, 1716, 0, 36, 668, 0, 383, 0, 115, 987, 1379, 2710, 1249, 721, 2682] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2084, 19, 655, 133, 937, 1, 1784, 57, 779, 2910, 270, 659, 1574, 1381, 0, 328, 370, 1468, 282, 2022, 0, 787, 1329, 1978, 63, 31, 2574, 1083, 3184, 242, 2589, 886, 7, 1810, 2636, 148, 607, 304, 2532, 1070, 1749, 2, 2347, 1937, 2, 1034, 1325, 1226, 2866, 9, 1528, 1, 21, 362, 4, 597, 3, 167, 498, 1316, 2801, 1272, 829, 2067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2750, 37, 201, 80, 1072, 0, 2569, 31, 726, 3202, 177, 609, 1200, 1336, 0, 320, 199, 1086, 297, 1617, 0, 788, 1688, 1257, 12, 17, 2609, 1071, 3055, 323, 2837, 751, 13, 1516, 2367, 276, 536, 164, 2518, 1176, 1083, 0, 2533, 1596, 0, 1029, 1367, 819, 2896, 41, 1848, 1, 3, 610, 0, 603, 0, 160, 1056, 1483, 3154, 903, 1050, 2818] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2864, 37, 345, 128, 530, 0, 2446, 130, 922, 2841, 381, 781, 1102, 1003, 0, 209, 247, 1439, 301, 1815, 1, 671, 1722, 1200, 57, 26, 2881, 1178, 3069, 288, 2956, 556, 10, 1127, 2826, 604, 430, 240, 2250, 1686, 1322, 0, 2592, 1941, 0, 1062, 1227, 782, 2840, 46, 1881, 0, 63, 486, 1, 423, 1, 160, 819, 904, 2966, 788, 1177, 2756] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3255, 30, 210, 115, 625, 0, 2129, 114, 966, 3391, 370, 827, 1246, 1832, 1, 216, 447, 1150, 260, 1045, 1, 1094, 1491, 1121, 4, 16, 3028, 1102, 2713, 326, 2632, 362, 6, 1087, 2790, 451, 386, 255, 2528, 671, 1086, 0, 2209, 1767, 0, 1585, 1035, 951, 3261, 33, 1929, 2, 39, 408, 2, 638, 0, 305, 1120, 1374, 2464, 987, 1089, 2959] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3071, 60, 387, 57, 677, 1, 2763, 46, 320, 3432, 304, 494, 1465, 1249, 0, 194, 328, 1210, 201, 1720, 0, 743, 1818, 1477, 0, 34, 2977, 960, 2965, 187, 2425, 624, 14, 1589, 2762, 435, 296, 134, 2255, 1521, 1084, 2, 2222, 1711, 0, 1084, 1154, 665, 3298, 21, 1843, 0, 22, 374, 0, 895, 0, 104, 782, 1070, 3053, 1382, 968, 2607] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2990, 77, 359, 119, 1021, 0, 2064, 61, 785, 3175, 399, 757, 1713, 952, 0, 246, 560, 1320, 303, 1742, 1, 587, 1700, 1493, 26, 6, 2665, 1039, 2702, 192, 2245, 619, 10, 1486, 2434, 217, 482, 206, 2325, 1237, 910, 2, 2284, 1746, 8, 1154, 1268, 909, 2899, 62, 1627, 8, 21, 449, 11, 736, 0, 119, 1222, 1595, 2965, 1377, 683, 3166] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2967, 24, 320, 175, 1051, 1, 2127, 44, 1252, 87, 0, 0, 0, 0, 0, 0, 920, 2969, 304, 397, 1050, 1587, 0, 217, 270, 0, 0, 0, 0, 0, 0, 0, 418, 2053, 296, 1284, 0, 596, 1319, 1570, 602, 257, 0, 0, 0, 0, 0, 0, 6, 13, 2677, 1192, 714, 297, 2270, 451, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1277, 2250, 248, 466, 361, 2475, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 1687, 2, 1828, 2060, 1, 975, 1579, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 2918, 46, 1393, 0, 12, 362, 1, 624, 2104, 122, 0, 0, 0, 0, 0, 0, 0, 135, 1285, 1317, 731, 1150, 1019, 2660, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8048, 7714, 8395, 7620, 8344, 9536, 7582, 8297] avg=8192.0000 max=9536 max_rank=5 min=7582 min_rank=6 imbalance_ratio=1.164062 max_load_reduction=56000 max_load_reduction_pct=0.854492 moved_to_echo=33759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2750, 37, 201, 80, 1072, 0, 2569, 31, 1876, 87, 0, 0, 0, 0, 0, 0, 726, 3202, 177, 609, 1200, 1336, 0, 320, 230, 0, 0, 0, 0, 0, 0, 0, 199, 1086, 297, 1617, 0, 788, 1688, 1257, 606, 219, 0, 0, 0, 0, 0, 0, 12, 17, 2609, 1071, 774, 323, 2837, 751, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1516, 1918, 276, 536, 164, 2518, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 0, 1840, 1596, 0, 1029, 1367, 819, 0, 0, 0, 0, 0, 0, 0, 0, 2896, 41, 1848, 1, 3, 610, 0, 603, 2281, 183, 0, 0, 0, 0, 0, 0, 0, 160, 1056, 1483, 1095, 903, 1050, 2818, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8703, 7800, 7757, 8394, 8117, 7734, 8466, 8565] avg=8192.0000 max=8703 max_rank=0 min=7734 min_rank=5 imbalance_ratio=1.062378 max_load_reduction=56833 max_load_reduction_pct=0.867203 moved_to_echo=32882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2084, 19, 655, 133, 937, 1, 1784, 57, 1666, 81, 0, 0, 0, 0, 0, 0, 779, 2910, 270, 659, 1574, 1381, 0, 328, 256, 0, 0, 0, 0, 0, 0, 0, 370, 1468, 282, 2022, 0, 787, 1329, 1978, 561, 244, 0, 0, 0, 0, 0, 0, 63, 31, 2574, 1083, 807, 242, 2589, 886, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1810, 2136, 148, 607, 304, 2532, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1749, 2, 1705, 1937, 2, 1034, 1325, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 2866, 9, 1528, 1, 21, 362, 4, 597, 2377, 163, 0, 0, 0, 0, 0, 0, 3, 167, 498, 1316, 972, 1272, 829, 2067, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7417, 8157, 9041, 8275, 8614, 8980, 7928, 7124] avg=8192.0000 max=9041 max_rank=2 min=7124 min_rank=7 imbalance_ratio=1.103638 max_load_reduction=56495 max_load_reduction_pct=0.862045 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2864, 37, 345, 128, 530, 0, 2446, 130, 1764, 89, 0, 0, 0, 0, 0, 0, 922, 2841, 381, 781, 1102, 1003, 0, 209, 275, 0, 0, 0, 0, 0, 0, 0, 247, 1439, 301, 1815, 1, 671, 1722, 1200, 620, 262, 0, 0, 0, 0, 0, 0, 57, 26, 2881, 1178, 778, 288, 2956, 556, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1127, 2289, 604, 430, 240, 2250, 1686, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 0, 1883, 1941, 0, 1062, 1227, 782, 0, 0, 0, 0, 0, 0, 0, 0, 2840, 46, 1881, 0, 63, 486, 1, 423, 2291, 173, 0, 0, 0, 0, 0, 0, 1, 160, 819, 904, 1029, 788, 1177, 2756, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8333, 7514, 8278, 8720, 8636, 8217, 8204, 7634] avg=8192.0000 max=8720 max_rank=3 min=7514 min_rank=1 imbalance_ratio=1.064453 max_load_reduction=56816 max_load_reduction_pct=0.866943 moved_to_echo=32691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2446, 14, 188, 166, 716, 0, 2253, 34, 1612, 89, 0, 0, 0, 0, 0, 0, 1305, 3311, 325, 531, 1238, 2013, 0, 238, 276, 0, 0, 0, 0, 0, 0, 0, 442, 1217, 233, 1229, 0, 611, 1707, 1577, 620, 263, 0, 0, 0, 0, 0, 0, 11, 105, 2735, 1421, 746, 494, 2873, 553, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1320, 2298, 265, 441, 180, 1816, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 959, 0, 1883, 2216, 0, 1200, 935, 884, 0, 0, 0, 0, 0, 0, 0, 0, 2821, 42, 1716, 0, 36, 668, 0, 383, 2197, 158, 0, 0, 0, 0, 0, 0, 0, 115, 987, 1379, 940, 1249, 721, 2682, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7518, 9237, 7899, 8938, 7773, 8077, 8021, 8073] avg=8192.0000 max=9237 max_rank=1 min=7518 min_rank=0 imbalance_ratio=1.127563 max_load_reduction=56299 max_load_reduction_pct=0.859055 moved_to_echo=31944 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3255, 30, 210, 115, 625, 0, 2129, 114, 1466, 76, 0, 0, 0, 0, 0, 0, 966, 3391, 370, 827, 1246, 1832, 1, 216, 271, 0, 0, 0, 0, 0, 0, 0, 447, 1150, 260, 1045, 1, 1094, 1491, 1121, 528, 259, 0, 0, 0, 0, 0, 0, 4, 16, 3028, 1102, 688, 326, 2632, 362, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1087, 2260, 451, 386, 255, 2528, 671, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 0, 1605, 1767, 0, 1585, 1035, 951, 0, 0, 0, 0, 0, 0, 0, 0, 3261, 33, 1929, 2, 39, 408, 2, 638, 2025, 143, 0, 0, 0, 0, 0, 0, 0, 305, 1120, 1374, 855, 987, 1089, 2959, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8020, 9120, 7396, 8158, 7644, 8029, 8480, 8689] avg=8192.0000 max=9120 max_rank=1 min=7396 min_rank=2 imbalance_ratio=1.113281 max_load_reduction=56416 max_load_reduction_pct=0.860840 moved_to_echo=32842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3071, 60, 387, 57, 677, 1, 2763, 46, 1816, 76, 0, 0, 0, 0, 0, 0, 320, 3432, 304, 494, 1465, 1249, 0, 194, 268, 0, 0, 0, 0, 0, 0, 0, 328, 1210, 201, 1720, 0, 743, 1818, 1477, 531, 256, 0, 0, 0, 0, 0, 0, 0, 34, 2977, 960, 751, 187, 2425, 624, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1589, 2238, 435, 296, 134, 2255, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 2, 1615, 1711, 0, 1084, 1154, 665, 0, 0, 0, 0, 0, 0, 0, 0, 3298, 21, 1843, 0, 22, 374, 0, 895, 2214, 178, 0, 0, 0, 0, 0, 0, 0, 104, 782, 1070, 1059, 1382, 968, 2607, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8954, 7726, 8284, 7958, 8482, 7315, 8845, 7972] avg=8192.0000 max=8954 max_rank=0 min=7315 min_rank=5 imbalance_ratio=1.093018 max_load_reduction=56582 max_load_reduction_pct=0.863373 moved_to_echo=32614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2990, 77, 359, 119, 1021, 0, 2064, 61, 1769, 83, 0, 0, 0, 0, 0, 0, 785, 3175, 399, 757, 1713, 952, 0, 246, 241, 0, 0, 0, 0, 0, 0, 0, 560, 1320, 303, 1742, 1, 587, 1700, 1493, 550, 230, 0, 0, 0, 0, 0, 0, 26, 6, 2665, 1039, 681, 192, 2245, 619, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1486, 1963, 217, 482, 206, 2325, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 910, 2, 1651, 1746, 8, 1154, 1268, 909, 0, 0, 0, 0, 0, 0, 0, 0, 2899, 62, 1627, 8, 21, 449, 11, 736, 2021, 176, 0, 0, 0, 0, 0, 0, 0, 119, 1222, 1595, 1020, 1377, 683, 3166, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8543, 8268, 8486, 7473, 7926, 7648, 8010, 9182] avg=8192.0000 max=9182 max_rank=7 min=7473 min_rank=3 imbalance_ratio=1.120850 max_load_reduction=56354 max_load_reduction_pct=0.859894 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9984, 128, 14080, 14976, 128, 9216, 9984, 7680, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 11264, 17408, 2688, 3712, 1920, 18816, 10112, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3072, 11008, 2176, 12544, 128, 5888, 12800, 11776, 4736, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 1280, 7808, 10496, 7808, 9216, 7552, 21760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[23808, 384, 13824, 128, 256, 3840, 128, 4992, 17536, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 256, 22272, 9088, 6016, 2432, 20864, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6784, 25344, 2560, 5120, 10624, 11392, 128, 2048, 2176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[22528, 384, 2688, 1024, 6656, 128, 18176, 640, 13312, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4060, 0, 4022, 4094, 1, 306, 4049, 1, 1, 2981, 4088, 0, 0, 1, 0, 0, 11, 258, 1058, 4084, 0, 4093, 0, 71, 0, 1010, 2, 0, 2, 0, 406, 0, 1, 2948, 4095, 0, 1, 0, 0, 0, 0, 4096, 3, 0, 0, 187, 4094, 0, 0, 0, 4037, 4096, 848, 0, 0, 0, 0, 2436] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4091, 0, 4090, 1, 4092, 4091, 0, 1138, 4090, 3, 1, 2753, 4060, 3, 2, 0, 2, 0, 53, 389, 2896, 4010, 2, 4089, 0, 465, 0, 200, 12, 0, 5, 3, 272, 0, 0, 838, 4089, 0, 1, 3, 0, 1, 1, 4093, 0, 0, 3, 4, 4093, 3, 0, 3, 3955, 4093, 1088, 0, 3, 1, 3, 2445] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4004, 0, 4013, 4096, 0, 87, 4006, 0, 0, 3401, 4041, 0, 0, 0, 0, 0, 38, 808, 2612, 3893, 0, 3941, 0, 795, 0, 763, 1, 0, 0, 0, 189, 0, 0, 1662, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 20, 4096, 0, 0, 0, 3989, 4096, 360, 0, 0, 0, 0, 2335] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 4096, 0, 4089, 0, 4090, 4096, 0, 844, 4062, 0, 0, 3368, 4047, 0, 0, 0, 0, 0, 23, 576, 2206, 3898, 0, 3905, 0, 330, 0, 548, 1, 0, 4, 0, 66, 0, 0, 1249, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 3968, 4096, 957, 0, 0, 0, 0, 2713] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4014, 0, 4081, 4095, 0, 281, 4056, 0, 0, 1997, 4074, 0, 0, 0, 0, 0, 0, 247, 1415, 4094, 0, 4087, 0, 200, 0, 1157, 0, 0, 0, 0, 593, 1, 0, 2891, 4096, 4, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 50, 4095, 0, 3, 0, 4006, 4096, 688, 0, 0, 0, 0, 3024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 4060, 0, 4072, 4094, 0, 1859, 3916, 0, 0, 2167, 4059, 0, 2, 1, 0, 0, 78, 359, 2122, 3729, 3, 4067, 0, 325, 0, 469, 15, 2, 0, 0, 513, 1, 0, 1580, 4096, 0, 0, 0, 1, 0, 0, 4096, 2, 0, 0, 15, 4090, 0, 3, 0, 3986, 4094, 310, 0, 0, 1, 0, 3255] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 4080, 1, 4079, 4096, 1, 1019, 4000, 0, 3, 3289, 4056, 0, 0, 1, 0, 0, 18, 968, 1707, 4006, 8, 4000, 0, 146, 0, 539, 5, 0, 0, 0, 117, 0, 1, 1529, 4094, 4, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 8, 4095, 0, 20, 0, 4010, 4095, 811, 0, 0, 0, 0, 2534] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 6, 4075, 0, 4024, 4096, 0, 219, 4071, 0, 0, 2479, 3937, 0, 0, 0, 0, 0, 13, 603, 1940, 4077, 5, 4071, 0, 507, 0, 696, 4, 0, 1, 0, 402, 0, 0, 2767, 4095, 31, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 390, 4096, 0, 17, 0, 4021, 4091, 545, 0, 0, 0, 0, 2064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4060, 0, 31, 4, 0, 0, 0, 0, 0, 0, 631, 0, 1, 306, 4049, 1, 1, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 1, 0, 0, 11, 258, 2849, 783, 0, 0, 0, 0, 0, 0, 1058, 4084, 0, 1213, 0, 71, 0, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 406, 0, 1, 2948, 4094, 1820, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 187, 4094, 0, 0, 0, 2418, 1571, 0, 0, 0, 0, 0, 0, 4037, 895, 848, 0, 0, 0, 0, 2436, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7970, 7990, 7436, 9273, 8188, 8273, 8216] avg=8192.0000 max=9273 max_rank=4 min=7436 min_rank=3 imbalance_ratio=1.131958 max_load_reduction=56263 max_load_reduction_pct=0.858505 moved_to_echo=33950 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4004, 0, 30, 4, 0, 0, 0, 0, 0, 0, 629, 0, 0, 87, 4006, 0, 0, 3401, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 38, 808, 2743, 783, 0, 0, 0, 0, 0, 0, 2612, 3893, 0, 1168, 0, 795, 0, 763, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 189, 0, 0, 1662, 4096, 1816, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 0, 0, 0, 2418, 1568, 0, 0, 0, 0, 0, 0, 3989, 895, 360, 0, 0, 0, 0, 2335, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8123, 8413, 9231, 7764, 8190, 8102, 7579] avg=8192.0000 max=9231 max_rank=3 min=7579 min_rank=7 imbalance_ratio=1.126831 max_load_reduction=56305 max_load_reduction_pct=0.859146 moved_to_echo=31635 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4014, 0, 31, 4, 0, 0, 0, 0, 0, 0, 641, 0, 0, 281, 4056, 0, 0, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 247, 2844, 783, 0, 0, 0, 0, 0, 0, 1415, 4094, 0, 1212, 0, 200, 0, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 593, 1, 0, 2891, 4095, 1846, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 50, 4095, 0, 3, 0, 2418, 1594, 0, 0, 0, 0, 0, 0, 4006, 895, 688, 0, 0, 0, 0, 3024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 6975, 7948, 8078, 9426, 8191, 8161, 8613] avg=8192.0000 max=9426 max_rank=4 min=6975 min_rank=1 imbalance_ratio=1.150635 max_load_reduction=56110 max_load_reduction_pct=0.856171 moved_to_echo=34391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4091, 0, 4090, 1, 31, 4, 0, 0, 0, 0, 0, 0, 642, 0, 0, 1138, 4090, 3, 1, 2753, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 3, 2, 0, 2, 0, 53, 389, 2846, 783, 0, 0, 0, 0, 0, 0, 2896, 4010, 2, 1212, 0, 465, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 5, 3, 272, 0, 0, 838, 4091, 1851, 0, 0, 0, 0, 0, 0, 4089, 0, 1, 3, 0, 1, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4093, 3, 0, 3, 2417, 1599, 0, 0, 0, 0, 0, 0, 3955, 893, 1088, 0, 3, 1, 3, 2445, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8627, 8138, 8785, 7072, 8184, 8122, 8388] avg=8192.0000 max=8785 max_rank=3 min=7072 min_rank=4 imbalance_ratio=1.072388 max_load_reduction=56751 max_load_reduction_pct=0.865952 moved_to_echo=31766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 4060, 0, 31, 4, 0, 0, 0, 0, 0, 0, 639, 0, 0, 1859, 3916, 0, 0, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 2, 1, 0, 0, 78, 359, 2830, 783, 0, 0, 0, 0, 0, 0, 2122, 3729, 3, 1206, 0, 325, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2, 0, 0, 513, 1, 0, 1580, 4094, 1842, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 15, 4090, 0, 3, 0, 2417, 1591, 0, 0, 0, 0, 0, 0, 3986, 894, 310, 0, 0, 1, 0, 3255, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8581, 8112, 7854, 8047, 8189, 8118, 8446] avg=8192.0000 max=8581 max_rank=1 min=7854 min_rank=3 imbalance_ratio=1.047485 max_load_reduction=56955 max_load_reduction_pct=0.869064 moved_to_echo=32800 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 4096, 0, 4089, 0, 30, 4, 0, 0, 0, 0, 0, 0, 642, 0, 0, 844, 4062, 0, 0, 3368, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 23, 576, 2718, 783, 0, 0, 0, 0, 0, 0, 2206, 3898, 0, 1157, 0, 330, 0, 548, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 66, 0, 0, 1249, 4096, 1850, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 2418, 1598, 0, 0, 0, 0, 0, 0, 3968, 895, 957, 0, 0, 0, 0, 2713, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8916, 8147, 8139, 7266, 8188, 8125, 8533] avg=8192.0000 max=8916 max_rank=1 min=7266 min_rank=4 imbalance_ratio=1.088379 max_load_reduction=56620 max_load_reduction_pct=0.863953 moved_to_echo=32112 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 4080, 1, 31, 4, 0, 0, 0, 0, 0, 0, 640, 0, 1, 1019, 4000, 0, 3, 3289, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 1, 0, 0, 18, 968, 2784, 783, 0, 0, 0, 0, 0, 0, 1707, 4006, 8, 1185, 0, 146, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 117, 0, 1, 1529, 4096, 1845, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 2, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 20, 0, 2418, 1594, 0, 0, 0, 0, 0, 0, 4010, 894, 811, 0, 0, 0, 0, 2534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8952, 8610, 7591, 7593, 8192, 8135, 8249] avg=8192.0000 max=8952 max_rank=1 min=7591 min_rank=3 imbalance_ratio=1.092773 max_load_reduction=56584 max_load_reduction_pct=0.863403 moved_to_echo=32169 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 6, 4075, 0, 37, 9, 0, 0, 0, 0, 0, 0, 623, 0, 0, 219, 4071, 0, 0, 2479, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 13, 603, 2837, 788, 0, 0, 0, 0, 0, 0, 1940, 4077, 5, 1197, 0, 507, 0, 696, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 402, 0, 0, 2767, 4096, 1825, 0, 0, 0, 0, 0, 0, 4095, 31, 0, 0, 1, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 4096, 0, 17, 0, 2421, 1576, 0, 0, 0, 0, 0, 0, 4021, 882, 545, 0, 0, 0, 0, 2064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 7392, 8178, 8422, 9095, 8214, 8500, 7512] avg=8192.0000 max=9095 max_rank=4 min=7392 min_rank=1 imbalance_ratio=1.110229 max_load_reduction=56441 max_load_reduction_pct=0.861221 moved_to_echo=33321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32000, 7168, 5632, 0, 128, 128, 128, 20864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 768, 32768, 128, 128, 128, 19456, 12800, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32384, 128, 128, 128, 128, 0, 256, 4224, 22528, 6272, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[16000, 31872, 128, 9600, 0, 2944, 0, 5504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 32512, 128, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 2560, 128, 128, 15488, 32768, 14720, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5120, 0, 128, 5760, 32256, 128, 128, 22528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 0, 3450, 0, 0, 0, 4079, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 3967, 0, 0, 12, 4015, 4095, 0, 4094, 0, 0, 1, 2006, 3952, 0, 2, 504, 3132, 4096, 0, 0, 0, 0, 3515, 0, 4094, 0, 16, 0, 4095, 0, 0, 0, 1, 0, 4095, 0, 0, 24, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 4088, 0, 0, 0, 4090, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3879, 0, 0, 15, 4074, 4092, 0, 4096, 0, 0, 0, 817, 3979, 1, 0, 354, 3359, 4096, 0, 0, 1, 0, 4009, 0, 4087, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 3, 4056, 0, 0, 0, 4071, 32, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 3766, 0, 0, 15, 4077, 4092, 0, 4091, 0, 0, 0, 2029, 4055, 0, 4, 551, 2002, 4091, 0, 0, 0, 0, 3991, 0, 4095, 0, 2, 0, 4095, 0, 0, 0, 3, 0, 4095, 0, 0, 25, 0, 1, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 0, 3719, 0, 0, 0, 4078, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4012, 1, 0, 3, 4079, 4092, 0, 4093, 0, 0, 0, 1526, 4081, 0, 4, 590, 3488, 4096, 0, 0, 0, 0, 3028, 0, 4096, 0, 20, 0, 4095, 0, 0, 0, 2, 0, 4096, 0, 0, 38, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 4043, 7, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3989, 0, 1, 4, 4061, 4083, 0, 4095, 0, 0, 0, 1229, 3935, 2, 0, 306, 3177, 4095, 0, 0, 0, 0, 3841, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 19, 0, 2, 4074, 0, 0, 0, 4052, 2, 4096, 0, 0, 4096, 2, 0, 4096, 0, 0, 3826, 4, 0, 2, 4086, 4079, 0, 4090, 0, 0, 0, 886, 3967, 0, 5, 727, 3084, 4094, 0, 0, 13, 0, 3947, 0, 4096, 0, 2, 0, 4094, 0, 0, 0, 3, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3, 0, 0, 1, 0, 0, 4044, 1, 0, 1, 4093, 11, 4090, 0, 3, 4087, 0, 0, 4093, 3, 3, 4001, 0, 9, 6, 4010, 4090, 0, 4092, 3, 2, 0, 2391, 4054, 2, 0, 541, 1717, 4092, 3, 3, 0, 4, 3730, 3, 4081, 0, 0, 2, 4092, 3, 3, 1, 0, 3, 4092, 3, 3, 60, 0, 0, 3, 1, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3904, 0, 0, 0, 4047, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4013, 0, 0, 2, 4086, 4096, 0, 4083, 0, 0, 0, 1841, 4040, 0, 5, 325, 2822, 4095, 0, 0, 45, 0, 3535, 0, 4096, 0, 5, 0, 4096, 0, 0, 0, 15, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 0, 3450, 0, 4060, 196, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3967, 0, 0, 157, 2, 0, 0, 0, 0, 0, 0, 12, 4015, 4095, 0, 34, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 3219, 0, 2, 504, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3515, 0, 4094, 0, 16, 380, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 1, 0, 4096, 4073, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7710, 8196, 8222, 8157, 8863, 8005, 8189, 8194] avg=8192.0000 max=8863 max_rank=4 min=7710 min_rank=0 imbalance_ratio=1.081909 max_load_reduction=56673 max_load_reduction_pct=0.864761 moved_to_echo=33251 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 0, 3719, 0, 4059, 202, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4012, 1, 0, 162, 2, 0, 0, 0, 0, 0, 0, 3, 4079, 4092, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1526, 3325, 0, 4, 590, 3488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3028, 0, 4096, 0, 20, 392, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7991, 8195, 8273, 8208, 8933, 7536, 8191, 8209] avg=8192.0000 max=8933 max_rank=4 min=7536 min_rank=5 imbalance_ratio=1.090454 max_load_reduction=56603 max_load_reduction_pct=0.863693 moved_to_echo=32869 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 4088, 0, 4062, 197, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3879, 0, 0, 158, 2, 0, 0, 0, 0, 0, 0, 15, 4074, 4092, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 817, 3241, 1, 0, 354, 3359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4009, 0, 4087, 0, 4, 383, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 1, 0, 4096, 4075, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 8208, 8135, 8215, 7772, 8484, 8190, 8183] avg=8192.0000 max=8484 max_rank=5 min=7772 min_rank=4 imbalance_ratio=1.035645 max_load_reduction=57052 max_load_reduction_pct=0.870544 moved_to_echo=32629 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 3, 4056, 0, 4057, 201, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 32, 21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3766, 0, 0, 161, 2, 0, 0, 0, 0, 0, 0, 15, 4077, 4092, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2029, 3303, 0, 4, 551, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 4095, 0, 2, 390, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 1, 0, 1, 0, 4091, 4075, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8323, 8220, 8024, 8218, 7889, 8478, 8191, 8193] avg=8192.0000 max=8478 max_rank=5 min=7889 min_rank=4 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=32751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 4091, 0, 4061, 195, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 7, 21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3989, 0, 1, 156, 2, 0, 0, 0, 0, 0, 0, 4, 4061, 4083, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1229, 3206, 2, 0, 306, 3177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 4090, 0, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4095, 4075, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8348, 8167, 8244, 8182, 7920, 8309, 8190, 8176] avg=8192.0000 max=8348 max_rank=0 min=7920 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32595 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 19, 0, 2, 4074, 0, 4056, 197, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 2, 21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 3826, 4, 0, 158, 2, 0, 0, 0, 0, 0, 0, 2, 4086, 4079, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 3231, 0, 5, 727, 3084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3947, 0, 4096, 0, 2, 381, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4075, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8348, 8171, 8088, 8201, 7933, 8439, 8187, 8169] avg=8192.0000 max=8439 max_rank=5 min=7933 min_rank=4 imbalance_ratio=1.030151 max_load_reduction=57097 max_load_reduction_pct=0.871231 moved_to_echo=32728 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 1, 0, 0, 4044, 1, 4058, 201, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 11, 21, 0, 3, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 3, 4001, 0, 9, 161, 2, 0, 0, 0, 0, 0, 0, 6, 4010, 4090, 0, 34, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2391, 3302, 2, 0, 541, 1717, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 3730, 3, 4081, 0, 0, 390, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 3, 3, 1, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 60, 0, 0, 3, 1, 3, 4092, 4069, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8216, 8272, 8145, 7956, 8211, 8194, 8234] avg=8192.0000 max=8308 max_rank=0 min=7956 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32595 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3904, 0, 4051, 204, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 20, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4013, 0, 0, 166, 3, 0, 0, 0, 0, 0, 0, 2, 4086, 4096, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1841, 3277, 0, 5, 325, 2822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 3535, 0, 4096, 0, 5, 393, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 15, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 4076, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8163, 8278, 8210, 8270, 8074, 8204, 8178] avg=8192.0000 max=8278 max_rank=2 min=8074 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32726 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 29696, 128, 32768, 0, 128, 3200, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32640, 128, 256, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 256, 0, 128, 128, 128, 128, 32768, 32640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 32768, 128, 128, 31488, 128, 128, 1280, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12800, 26112, 128, 128, 3968, 22784, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32512, 32768, 0, 384, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 128, 31488, 128, 32512, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 4091, 4081, 0, 0, 0, 4094, 0, 4096, 4094, 0, 62, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4095, 0, 298, 78, 0, 1, 1, 0, 3682, 0, 0, 0, 4080, 4096, 0, 0, 4095, 21, 0, 5, 0, 4096, 4074, 13, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 4095, 4094, 0, 0, 0, 4095, 0, 4096, 4096, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 198, 32, 0, 0, 3, 0, 3885, 0, 0, 0, 4070, 4096, 0, 0, 4088, 17, 0, 1, 0, 4095, 4092, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4092, 4091, 4091, 0, 0, 0, 4095, 0, 4096, 4094, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 4096, 0, 1436, 76, 0, 0, 0, 0, 2618, 0, 0, 0, 4063, 4096, 1, 0, 4040, 69, 0, 3, 0, 4096, 4080, 18, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4093, 4090, 4081, 0, 0, 0, 4096, 0, 4096, 4093, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 245, 127, 0, 2, 0, 0, 3763, 0, 0, 0, 4064, 4096, 0, 0, 4038, 81, 0, 6, 0, 4096, 4070, 18, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[3, 4, 0, 4, 4073, 3, 0, 4093, 4093, 4093, 0, 3, 0, 4082, 3, 4088, 4092, 0, 73, 4092, 4, 4, 5, 0, 0, 0, 0, 0, 3, 1, 3, 4093, 0, 3, 8, 4092, 4, 463, 140, 0, 1, 24, 3, 3427, 3, 0, 0, 4060, 4092, 0, 3, 4091, 0, 0, 18, 0, 4084, 4091, 11, 3, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 4088, 4076, 0, 0, 0, 4096, 0, 4096, 4095, 0, 37, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 4096, 0, 1084, 60, 0, 0, 1, 0, 3073, 0, 0, 0, 3995, 4096, 0, 0, 4074, 41, 0, 2, 0, 4096, 4037, 12, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4092, 0, 4096, 4096, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 903, 190, 0, 0, 1, 0, 3007, 0, 0, 0, 4085, 4096, 0, 0, 4032, 66, 0, 0, 0, 4094, 4096, 11, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4093, 4054, 4083, 0, 0, 0, 4096, 0, 4096, 4096, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 860, 100, 0, 1, 0, 0, 3297, 0, 0, 0, 4096, 4096, 0, 0, 4027, 38, 0, 12, 0, 4096, 3978, 34, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4091, 4081, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 62, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 298, 78, 0, 3308, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3682, 0, 0, 0, 4080, 763, 15, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4095, 21, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4074, 13, 0, 0, 0, 1, 0, 9, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8191, 8228, 8195, 7780, 8542, 8206, 8201] avg=8192.0000 max=8542 max_rank=5 min=7780 min_rank=4 imbalance_ratio=1.042725 max_load_reduction=56994 max_load_reduction_pct=0.869659 moved_to_echo=32729 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 198, 32, 0, 3309, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3885, 0, 0, 0, 4070, 763, 15, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4088, 17, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4092, 1, 0, 0, 0, 0, 0, 9, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8208, 8173, 8194, 7635, 8736, 8191, 8205] avg=8192.0000 max=8736 max_rank=5 min=7635 min_rank=4 imbalance_ratio=1.066406 max_load_reduction=56800 max_load_reduction_pct=0.866699 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 4091, 4091, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 6, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 1436, 76, 0, 3309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 0, 4063, 763, 15, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 4040, 69, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 18, 0, 0, 0, 1, 0, 9, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8201, 8172, 8185, 8918, 7459, 8198, 8212] avg=8192.0000 max=8918 max_rank=4 min=7459 min_rank=5 imbalance_ratio=1.088623 max_load_reduction=56618 max_load_reduction_pct=0.863922 moved_to_echo=32787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4088, 4076, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 37, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 1084, 60, 0, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3073, 0, 0, 0, 3995, 763, 15, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4074, 41, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4037, 12, 0, 0, 0, 0, 0, 9, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8183, 8204, 8194, 8551, 7847, 8202, 8162] avg=8192.0000 max=8551 max_rank=4 min=7847 min_rank=5 imbalance_ratio=1.043823 max_load_reduction=56985 max_load_reduction_pct=0.869522 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 4090, 4081, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 245, 127, 0, 3310, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3763, 0, 0, 0, 4064, 763, 15, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4038, 81, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4070, 18, 0, 1, 0, 0, 0, 9, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8190, 8169, 8188, 7778, 8607, 8210, 8202] avg=8192.0000 max=8607 max_rank=5 min=7778 min_rank=4 imbalance_ratio=1.050659 max_load_reduction=56929 max_load_reduction_pct=0.868668 moved_to_echo=32797 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 903, 190, 0, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3007, 0, 0, 0, 4085, 763, 15, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4032, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 11, 0, 0, 0, 0, 0, 9, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8210, 8171, 8196, 8496, 7871, 8183, 8218] avg=8192.0000 max=8496 max_rank=4 min=7871 min_rank=5 imbalance_ratio=1.037109 max_load_reduction=57040 max_load_reduction_pct=0.870361 moved_to_echo=32768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 4, 0, 4, 4073, 3, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 3, 0, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 73, 4069, 4, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 4093, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 3, 8, 4092, 4, 463, 140, 0, 3299, 0, 0, 0, 0, 0, 0, 0, 1, 24, 3, 3427, 3, 0, 0, 4060, 761, 15, 0, 0, 0, 0, 0, 0, 4081, 0, 3, 4091, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4091, 11, 3, 0, 0, 0, 3, 8, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8210, 8247, 8192, 8009, 8294, 8193, 8208] avg=8192.0000 max=8294 max_rank=5 min=8009 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4093, 10, 0, 0, 0, 0, 0, 0, 0, 4054, 4083, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 860, 100, 0, 3313, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3297, 0, 0, 0, 4096, 768, 18, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4027, 38, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3978, 34, 0, 0, 0, 0, 0, 10, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8143, 8172, 8192, 8369, 8180, 8153, 8128] avg=8192.0000 max=8369 max_rank=4 min=8128 min_rank=7 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32830 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 0, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 26752, 128, 0, 0, 32640, 6144, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 256, 32640, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 32640, 128, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 128, 5504, 896, 0, 26496, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 32512, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 66, 0, 0, 0, 4072, 4081, 7, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1026, 0, 22, 0, 4096, 0, 0, 3, 0, 4091, 0, 0, 4, 4090, 3600, 0, 3, 4095, 4, 0, 4096, 4091, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 3513, 0, 4095, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 26, 0, 0, 0, 3991, 4088, 18, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1020, 0, 13, 0, 4096, 0, 0, 5, 0, 4088, 0, 0, 3, 4030, 3956, 0, 2, 4096, 0, 0, 4096, 4090, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3353, 0, 4095, 4085, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 23, 0, 0, 0, 4065, 4091, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 0, 44, 0, 4095, 0, 0, 1, 1, 4093, 0, 0, 5, 4084, 3959, 0, 0, 4096, 9, 0, 4096, 4088, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 2, 0, 4096, 0, 3683, 0, 4095, 4094, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 16, 0, 0, 0, 4070, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 8, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4053, 3650, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3928, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 30, 0, 0, 0, 4088, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 0, 32, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4069, 3652, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3745, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3, 4093, 0, 0, 0, 0, 3, 4, 4084, 4078, 26, 3, 3, 4, 0, 2, 0, 2, 3, 9, 2219, 3, 76, 0, 4092, 2, 0, 3, 0, 4093, 1, 2, 0, 4084, 3893, 0, 18, 4092, 2, 1, 4088, 4093, 4089, 3, 0, 3, 4, 0, 0, 4092, 3, 0, 0, 0, 4090, 1, 1976, 3, 4092, 4092, 3, 3, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 57, 0, 0, 0, 4050, 4079, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 898, 0, 1, 0, 4096, 0, 0, 6, 0, 4091, 0, 0, 5, 4089, 3674, 0, 0, 4096, 0, 0, 4096, 4089, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3626, 0, 4092, 4086, 0, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 15, 0, 0, 0, 3979, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 1, 0, 4096, 0, 0, 6, 0, 4081, 0, 0, 8, 4090, 4061, 0, 0, 4096, 0, 0, 4096, 4081, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4047, 0, 4096, 4086, 0, 0, 10, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 26, 0, 0, 0, 3438, 628, 0, 0, 0, 0, 0, 0, 3991, 4088, 18, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1020, 0, 13, 0, 4090, 3162, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4088, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 4030, 3956, 0, 2, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3353, 0, 615, 4085, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8138, 8286, 8190, 8297, 8192, 8192, 8055] avg=8192.0000 max=8297 max_rank=4 min=8055 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 66, 0, 0, 0, 3438, 628, 0, 0, 0, 0, 0, 0, 4072, 4081, 7, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1026, 0, 22, 0, 4090, 3161, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 4090, 3600, 0, 3, 306, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3513, 0, 615, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8201, 8301, 8191, 8007, 8193, 8194, 8220] avg=8192.0000 max=8301 max_rank=2 min=8007 min_rank=4 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 23, 0, 0, 0, 3438, 628, 0, 0, 0, 0, 0, 0, 4065, 4091, 14, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 511, 0, 44, 0, 4090, 3162, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 4084, 3959, 0, 0, 306, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3683, 0, 615, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8211, 7807, 8191, 8363, 8190, 8195, 8395] avg=8192.0000 max=8395 max_rank=7 min=7807 min_rank=2 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=33143 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 16, 0, 0, 0, 3439, 628, 0, 0, 0, 0, 0, 0, 4070, 4095, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 8, 0, 4090, 3162, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 3650, 0, 0, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3928, 0, 615, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8206, 7920, 8193, 8009, 8198, 8192, 8639] avg=8192.0000 max=8639 max_rank=7 min=7920 min_rank=2 imbalance_ratio=1.054565 max_load_reduction=56897 max_load_reduction_pct=0.868179 moved_to_echo=33038 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 30, 0, 0, 0, 3439, 628, 0, 0, 0, 0, 0, 0, 4088, 4090, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 0, 32, 0, 4090, 3162, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 3652, 0, 0, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3745, 0, 615, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8219, 8058, 8193, 8027, 8198, 8192, 8456] avg=8192.0000 max=8456 max_rank=7 min=8027 min_rank=4 imbalance_ratio=1.032227 max_load_reduction=57080 max_load_reduction_pct=0.870972 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 4093, 0, 0, 0, 0, 3, 4, 3436, 628, 0, 0, 0, 0, 0, 0, 4084, 4078, 26, 3, 3, 4, 0, 2, 41, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 9, 2219, 3, 76, 0, 4083, 3159, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 3, 0, 4093, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 3893, 0, 18, 305, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4093, 6, 3, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1976, 3, 614, 4092, 3, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8241, 9554, 8194, 8303, 8197, 8186, 6694] avg=8192.0000 max=9554 max_rank=2 min=6694 min_rank=7 imbalance_ratio=1.166260 max_load_reduction=55982 max_load_reduction_pct=0.854218 moved_to_echo=31380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 57, 0, 0, 0, 3436, 628, 0, 0, 0, 0, 0, 0, 4050, 4079, 16, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 898, 0, 1, 0, 4090, 3162, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 4089, 3674, 0, 0, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3626, 0, 614, 4086, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8186, 8151, 8194, 8074, 8191, 8192, 8331] avg=8192.0000 max=8331 max_rank=7 min=8074 min_rank=4 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 15, 0, 0, 0, 3443, 633, 0, 0, 0, 0, 0, 0, 3979, 4096, 17, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 1, 0, 4096, 3166, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4081, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 8, 4090, 4061, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 603, 4086, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8134, 7459, 8190, 8456, 8177, 8193, 8746] avg=8192.0000 max=8746 max_rank=7 min=7459 min_rank=2 imbalance_ratio=1.067627 max_load_reduction=56790 max_load_reduction_pct=0.866547 moved_to_echo=33572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32640, 30464, 0, 128, 2560, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[27904, 128, 4992, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32512, 32768, 128, 128, 128, 128, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 256, 0, 128, 128, 27520, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 7424, 128, 256, 0, 32768, 25344, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4058, 0, 2, 7, 4096, 0, 0, 0, 3984, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 4090, 4096, 0, 4069, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 0, 0, 0, 0, 166, 0, 7, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3774, 0, 0, 23, 4096, 0, 0, 0, 3706, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4071, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 707, 0, 5, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 4072, 0, 1, 12, 4096, 0, 0, 0, 3934, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 4085, 4096, 0, 4079, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 180, 0, 32, 0, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 3932, 0, 0, 34, 4096, 0, 0, 0, 3319, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 4074, 0, 2, 4096, 2, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 967, 0, 0, 0, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 4022, 0, 0, 41, 4096, 0, 0, 0, 3291, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 0, 4090, 4096, 0, 4085, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4094, 0, 0, 0, 0, 890, 1, 5, 0, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 3, 4092, 0, 0, 3, 0, 4068, 1, 0, 0, 4, 3964, 0, 1, 16, 4092, 4, 3, 0, 3917, 1, 0, 2, 4092, 1, 4091, 0, 4, 3, 3, 4090, 4092, 1, 4019, 3, 1, 4091, 1, 4092, 4, 3, 7, 4092, 4092, 5, 0, 3, 5, 3, 0, 4092, 3, 1, 4089, 4, 1, 4, 4, 337, 0, 0, 0, 32] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 4083, 0, 0, 0, 0, 3964, 0, 3, 29, 4096, 0, 0, 0, 2522, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 0, 4087, 4096, 0, 4056, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4090, 0, 0, 0, 0, 1710, 0, 10, 0, 21] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 7, 0, 0, 0, 4081, 0, 0, 0, 0, 4065, 0, 0, 8, 4096, 0, 0, 0, 3900, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4081, 4096, 0, 4048, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 201, 0, 78, 0, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4090, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 2, 7, 3518, 670, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3984, 0, 1, 0, 522, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4069, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 7, 0, 9, 4096, 3419, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8255, 8603, 8193, 8200, 8191, 8190, 7697] avg=8192.0000 max=8603 max_rank=2 min=7697 min_rank=7 imbalance_ratio=1.050171 max_load_reduction=56933 max_load_reduction_pct=0.868729 moved_to_echo=32278 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 0, 0, 23, 3518, 670, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3706, 0, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4071, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 707, 0, 5, 0, 2, 4096, 3419, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 7985, 8324, 8199, 8202, 8191, 8193, 8229] avg=8192.0000 max=8324 max_rank=2 min=7985 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32815 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4075, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 1, 12, 3518, 670, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3934, 0, 1, 0, 522, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 32, 0, 13, 4096, 3419, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8273, 8553, 8188, 8210, 8191, 8189, 7740] avg=8192.0000 max=8553 max_rank=2 min=7740 min_rank=7 imbalance_ratio=1.044067 max_load_reduction=56983 max_load_reduction_pct=0.869492 moved_to_echo=32330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4046, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 41, 3518, 670, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3291, 0, 2, 0, 522, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4085, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 890, 1, 5, 0, 12, 4096, 3419, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8251, 7911, 8193, 8216, 8191, 8188, 8423] avg=8192.0000 max=8423 max_rank=7 min=7911 min_rank=2 imbalance_ratio=1.028198 max_load_reduction=57113 max_load_reduction_pct=0.871475 moved_to_echo=33018 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4044, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 34, 3518, 670, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3319, 0, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4074, 0, 2, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 967, 0, 0, 0, 11, 4096, 3419, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8154, 7937, 8198, 8209, 8191, 8193, 8493] avg=8192.0000 max=8493 max_rank=7 min=7937 min_rank=2 imbalance_ratio=1.036743 max_load_reduction=57043 max_load_reduction_pct=0.870407 moved_to_echo=33086 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 4083, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 3, 29, 3518, 670, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2522, 0, 4, 0, 522, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4056, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1710, 0, 10, 0, 21, 4096, 3420, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8184, 7144, 8189, 8187, 8191, 8183, 9257] avg=8192.0000 max=9257 max_rank=7 min=7144 min_rank=2 imbalance_ratio=1.130005 max_load_reduction=56279 max_load_reduction_pct=0.858749 moved_to_echo=33818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4092, 0, 0, 3, 0, 4068, 21, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 3964, 0, 1, 16, 3514, 669, 0, 0, 0, 0, 0, 0, 4092, 4, 3, 0, 3917, 1, 0, 2, 522, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4091, 0, 4, 3, 3, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1, 4019, 3, 1, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 7, 4092, 4091, 5, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 4092, 3, 1, 4089, 4, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4, 4, 337, 0, 0, 0, 32, 4092, 3416, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8169, 8541, 8199, 8151, 8205, 8198, 7886] avg=8192.0000 max=8541 max_rank=2 min=7886 min_rank=7 imbalance_ratio=1.042603 max_load_reduction=56995 max_load_reduction_pct=0.869675 moved_to_echo=32440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 7, 0, 0, 0, 4081, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 8, 3521, 671, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3900, 0, 0, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4048, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 78, 0, 11, 4096, 3425, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8265, 8523, 8177, 8161, 8185, 8202, 7811] avg=8192.0000 max=8523 max_rank=2 min=7811 min_rank=7 imbalance_ratio=1.040405 max_load_reduction=57013 max_load_reduction_pct=0.869949 moved_to_echo=32359 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 5248, 128, 256, 0, 128, 32768, 27392, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 28672, 128, 128, 128, 4224, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 31872, 0, 128, 256, 28160, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 32640, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 128, 32512, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 19, 0, 0, 4059, 417, 0, 4096, 4094, 0, 0, 4096, 0, 3814, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 560, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 4086, 0, 0, 0, 3409, 2, 1, 47, 0, 0, 0, 4096, 11, 4052, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 43, 0, 1, 4047, 714, 0, 4096, 4094, 0, 0, 4096, 0, 4061, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 185, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4093, 0, 0, 0, 2826, 2, 0, 476, 0, 0, 0, 4096, 6, 4023, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 4045, 1007, 0, 4096, 4095, 0, 0, 4096, 0, 3853, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 625, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 4094, 0, 0, 0, 2874, 1, 0, 132, 0, 0, 0, 4096, 2, 3822, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 1, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 73, 0, 0, 4070, 434, 0, 4096, 4096, 0, 0, 4096, 0, 4034, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 608, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3230, 0, 0, 613, 0, 0, 0, 4096, 1, 3321, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 37, 0, 2, 4024, 1777, 0, 4096, 4093, 0, 0, 4096, 0, 4042, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 893, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 4091, 0, 0, 0, 1808, 5, 0, 424, 0, 0, 0, 4096, 0, 3368, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4, 0, 12, 4, 27, 2, 0, 4068, 521, 6, 4090, 4092, 1, 0, 4087, 4, 4029, 0, 4, 4019, 0, 0, 4091, 0, 0, 3, 277, 4, 4, 4088, 4, 0, 3, 1, 1, 4, 4, 4071, 0, 0, 4, 3270, 0, 0, 245, 0, 3, 4, 4090, 0, 4020, 3, 0, 4089, 4090, 1, 0, 4, 3, 3, 0, 4091, 0, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 4072, 982, 0, 4096, 4096, 0, 0, 4096, 0, 4052, 0, 0, 4095, 12, 0, 4096, 0, 0, 0, 790, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1275, 0, 0, 1413, 0, 0, 0, 4096, 0, 3783, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 28, 0, 0, 4050, 1536, 0, 4096, 4096, 0, 0, 4096, 0, 3858, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1257, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 4077, 0, 0, 0, 1595, 0, 0, 302, 0, 0, 0, 4096, 0, 3769, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 19, 0, 0, 4059, 2994, 1109, 0, 0, 0, 0, 0, 0, 417, 0, 0, 4094, 0, 0, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3814, 0, 0, 4096, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 4096, 0, 0, 2523, 922, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4086, 0, 0, 2850, 1247, 0, 0, 0, 0, 0, 0, 0, 3409, 2, 1, 47, 0, 0, 0, 4096, 1102, 0, 0, 0, 0, 0, 0, 0, 11, 4052, 0, 0, 326, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 7685, 8047, 8101, 8188, 8657, 8485, 8192] avg=8192.0000 max=8657 max_rank=5 min=7685 min_rank=1 imbalance_ratio=1.056763 max_load_reduction=56879 max_load_reduction_pct=0.867905 moved_to_echo=33522 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 4045, 2994, 1109, 0, 0, 0, 0, 0, 0, 1007, 0, 0, 4095, 0, 0, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 4096, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 625, 0, 0, 4096, 0, 0, 2523, 922, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4094, 0, 0, 2850, 1247, 0, 0, 0, 0, 0, 0, 0, 2874, 1, 0, 132, 0, 0, 0, 4096, 1102, 0, 0, 0, 0, 0, 0, 0, 2, 3822, 0, 0, 326, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8276, 8086, 8166, 8196, 8205, 8245, 8193] avg=8192.0000 max=8276 max_rank=1 min=8086 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32839 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 73, 0, 0, 4070, 2994, 1109, 0, 0, 0, 0, 0, 0, 434, 0, 0, 4096, 0, 0, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 4096, 1, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 608, 0, 0, 4096, 0, 0, 2523, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2850, 1247, 0, 0, 0, 0, 0, 0, 0, 3230, 0, 0, 613, 0, 0, 0, 4096, 1102, 0, 0, 0, 0, 0, 0, 0, 1, 3321, 0, 0, 326, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 7704, 8268, 8149, 8192, 9041, 7744, 8192] avg=8192.0000 max=9041 max_rank=5 min=7704 min_rank=1 imbalance_ratio=1.103638 max_load_reduction=56495 max_load_reduction_pct=0.862045 moved_to_echo=33169 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 43, 0, 1, 4047, 2994, 1109, 0, 0, 0, 0, 0, 0, 714, 0, 0, 4094, 0, 0, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 4096, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 4096, 0, 0, 2523, 922, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4093, 0, 0, 2850, 1247, 0, 0, 0, 0, 0, 0, 0, 2826, 2, 0, 476, 0, 0, 0, 4096, 1102, 0, 0, 0, 0, 0, 0, 0, 6, 4023, 0, 0, 326, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7982, 8294, 7726, 8196, 8502, 8450, 8192] avg=8192.0000 max=8502 max_rank=5 min=7726 min_rank=3 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=33340 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 4072, 2994, 1109, 0, 0, 0, 0, 0, 0, 982, 0, 0, 4096, 0, 0, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 4095, 12, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 790, 0, 0, 4096, 0, 0, 2523, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2850, 1247, 0, 0, 0, 0, 0, 0, 0, 1275, 0, 0, 1413, 0, 0, 0, 4096, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 3783, 0, 0, 326, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8252, 8296, 8331, 8193, 7886, 8205, 8192] avg=8192.0000 max=8331 max_rank=3 min=7886 min_rank=5 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32476 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 37, 0, 2, 4024, 2995, 1109, 0, 0, 0, 0, 0, 0, 1777, 0, 0, 4093, 0, 0, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 4096, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 893, 0, 0, 4096, 0, 0, 2523, 922, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4091, 0, 0, 2850, 1247, 0, 0, 0, 0, 0, 0, 0, 1808, 5, 0, 424, 0, 0, 0, 4096, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 3368, 0, 0, 326, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 9044, 8275, 8434, 8199, 7434, 7790, 8193] avg=8192.0000 max=9044 max_rank=1 min=7434 min_rank=5 imbalance_ratio=1.104004 max_load_reduction=56492 max_load_reduction_pct=0.862000 moved_to_echo=31616 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 12, 4, 27, 2, 0, 4068, 2989, 1107, 0, 0, 0, 0, 0, 0, 521, 6, 0, 4092, 1, 0, 3167, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 4, 4019, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 277, 4, 4, 4088, 4, 0, 2519, 920, 0, 0, 0, 0, 0, 0, 3, 1, 1, 4, 4, 4071, 0, 0, 2847, 1245, 0, 0, 0, 0, 0, 0, 4, 3270, 0, 0, 245, 0, 3, 4, 4090, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 3, 0, 325, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 3, 0, 4091, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 7791, 8189, 7819, 8176, 8717, 8439, 8192] avg=8192.0000 max=8717 max_rank=5 min=7791 min_rank=1 imbalance_ratio=1.064087 max_load_reduction=56819 max_load_reduction_pct=0.866989 moved_to_echo=33524 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 28, 0, 0, 4050, 2995, 1112, 0, 0, 0, 0, 0, 0, 1536, 0, 0, 4096, 0, 0, 3170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3858, 0, 0, 4096, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 0, 0, 4096, 0, 0, 2529, 926, 2, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4077, 0, 0, 2857, 1254, 0, 0, 0, 0, 0, 0, 0, 1595, 0, 0, 302, 0, 0, 0, 4096, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 3769, 0, 0, 313, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8802, 8081, 8810, 8196, 7094, 8178, 8190] avg=8192.0000 max=8810 max_rank=3 min=7094 min_rank=5 imbalance_ratio=1.075439 max_load_reduction=56726 max_load_reduction_pct=0.865570 moved_to_echo=31658 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 30208, 128, 0, 2688, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 5248, 128, 128, 32768, 128, 0, 20224, 7424, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7424, 128, 0, 32768, 128, 0, 25472, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 20352, 128, 128, 3712, 0, 128, 128, 32768, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 0, 22912, 9984, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 256, 128, 128, 32512, 24064, 8960, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31744, 0, 128, 32768, 128, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 0, 7, 27, 0, 4096, 4096, 0, 0, 4089, 0, 0, 837, 4095, 4096, 49, 4031, 406, 0, 4096, 4091, 0, 4087, 0, 4, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 69, 2778, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 8, 4, 0, 4096, 4094, 0, 0, 4096, 0, 0, 607, 4096, 4096, 11, 4095, 2910, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2, 555, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 6, 19, 0, 4096, 4096, 0, 0, 4096, 0, 0, 1133, 4096, 4096, 34, 4087, 2008, 0, 4096, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 15, 892, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4096, 0, 4087, 0, 0, 21, 39, 0, 4096, 4096, 0, 0, 4083, 0, 0, 973, 4096, 4096, 152, 4071, 1882, 0, 4096, 4087, 0, 4085, 0, 13, 0, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 4095, 4096, 0, 4, 1064, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4096, 0, 4087, 0, 0, 1, 13, 0, 4096, 4096, 0, 0, 4085, 0, 0, 593, 4095, 4096, 38, 4087, 2132, 0, 4096, 4085, 0, 4090, 0, 11, 1, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 4096, 0, 15, 1326, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 0, 0, 2, 49, 0, 4095, 4096, 0, 0, 4090, 0, 0, 619, 4096, 4096, 133, 3739, 1114, 0, 4096, 4094, 0, 4066, 0, 7, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 4096, 0, 74, 2491, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2, 0, 2, 4092, 0, 4, 0, 4091, 4, 4093, 4, 4, 17, 0, 4, 4040, 4083, 0, 2, 4092, 3, 0, 1570, 4090, 4061, 26, 4063, 1166, 4, 4092, 4092, 3, 4093, 1, 8, 0, 4, 0, 0, 0, 0, 1, 31, 0, 4, 3, 0, 0, 4092, 3, 4, 0, 4, 4, 0, 3, 0, 4088, 4092, 1, 11, 1383, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4080, 0, 0, 4, 35, 0, 4096, 4096, 0, 0, 4081, 0, 0, 711, 4096, 4096, 103, 4059, 1630, 0, 4096, 4078, 0, 4075, 0, 14, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 4096, 0, 74, 1611, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 7, 27, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 0, 0, 4089, 0, 0, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4031, 406, 0, 0, 2439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4, 0, 0, 0, 0, 1, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 1652, 1544, 872, 19, 1, 0, 0, 0, 0, 4096, 2552, 0, 69, 2778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8200, 8150, 6925, 8194, 8192, 8188, 9495] avg=8192.0000 max=9495 max_rank=7 min=6925 min_rank=3 imbalance_ratio=1.159058 max_load_reduction=56041 max_load_reduction_pct=0.855118 moved_to_echo=34069 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 6, 19, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 0, 0, 4096, 0, 0, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4087, 2008, 0, 0, 2439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1651, 1544, 872, 19, 1, 0, 0, 0, 0, 4096, 2552, 0, 15, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8198, 8453, 8568, 8198, 8189, 8183, 7555] avg=8192.0000 max=8568 max_rank=3 min=7555 min_rank=7 imbalance_ratio=1.045898 max_load_reduction=56968 max_load_reduction_pct=0.869263 moved_to_echo=32125 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 2, 49, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 0, 0, 4090, 0, 0, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 3739, 1114, 0, 0, 2441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 7, 0, 0, 0, 0, 2, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 1653, 1544, 872, 19, 2, 0, 0, 0, 0, 4096, 2552, 0, 74, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8214, 7933, 7427, 8176, 8186, 8195, 9213] avg=8192.0000 max=9213 max_rank=7 min=7427 min_rank=3 imbalance_ratio=1.124634 max_load_reduction=56323 max_load_reduction_pct=0.859421 moved_to_echo=33770 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 8, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3222, 0, 0, 4096, 0, 0, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4095, 2910, 0, 0, 2442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1654, 1544, 872, 19, 1, 0, 0, 0, 0, 4096, 2552, 0, 2, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8185, 7925, 9458, 8198, 8187, 8186, 7205] avg=8192.0000 max=9458 max_rank=3 min=7205 min_rank=7 imbalance_ratio=1.154541 max_load_reduction=56078 max_load_reduction_pct=0.855682 moved_to_echo=31776 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 1, 13, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 0, 0, 4085, 0, 0, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4087, 2132, 0, 0, 2436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 11, 1, 0, 0, 0, 1, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 1649, 1544, 872, 19, 2, 0, 0, 0, 0, 4096, 2552, 0, 15, 1326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8178, 7902, 8693, 8204, 8187, 8190, 7989] avg=8192.0000 max=8693 max_rank=3 min=7902 min_rank=2 imbalance_ratio=1.061157 max_load_reduction=56843 max_load_reduction_pct=0.867355 moved_to_echo=32570 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 21, 39, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 0, 0, 4083, 0, 0, 973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 4071, 1882, 0, 0, 2437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 13, 0, 0, 0, 0, 1, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 1650, 1544, 872, 19, 1, 0, 0, 0, 0, 4095, 2552, 0, 4, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8224, 8280, 8542, 8201, 8190, 8191, 7715] avg=8192.0000 max=8542 max_rank=3 min=7715 min_rank=7 imbalance_ratio=1.042725 max_load_reduction=56994 max_load_reduction_pct=0.869659 moved_to_echo=32297 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 2, 4092, 0, 4, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4074, 4, 4, 17, 0, 4, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 0, 2, 4092, 3, 0, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4063, 1166, 4, 0, 2440, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 8, 0, 4, 0, 0, 0, 4060, 7, 0, 0, 0, 0, 0, 0, 0, 1, 31, 0, 4, 3, 0, 0, 4092, 4083, 0, 0, 0, 0, 0, 0, 4092, 3, 4, 0, 4, 4, 0, 3, 1652, 1542, 869, 19, 1, 0, 0, 0, 0, 4088, 2550, 1, 11, 1383, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8147, 8881, 7702, 8173, 8214, 8193, 8035] avg=8192.0000 max=8881 max_rank=2 min=7702 min_rank=3 imbalance_ratio=1.084106 max_load_reduction=56655 max_load_reduction_pct=0.864487 moved_to_echo=32615 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 4, 35, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3220, 0, 0, 4081, 0, 0, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 4059, 1630, 0, 0, 2429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 14, 0, 0, 0, 0, 0, 4095, 8, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 0, 0, 0, 1649, 1548, 876, 25, 1, 1, 0, 0, 0, 4096, 2548, 0, 74, 1611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8190, 8012, 8221, 8192, 8191, 8210, 8329] avg=8192.0000 max=8329 max_rank=7 min=8012 min_rank=2 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32922 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 20480, 128, 384, 12160, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25856, 0, 128, 32768, 128, 0, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 640, 32256, 13312, 128, 0, 19584, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32640, 128, 128, 128, 256, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 128, 13312, 12416, 7040, 256, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66688 slack_slots=3584 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 1753, 4096, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4057, 0, 0, 4075, 0, 0, 0, 0, 0, 4096, 0, 4079, 0, 1759, 4094, 0, 0, 1, 4095, 0, 0, 4094, 4096, 0, 657, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2305, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 4096, 4070, 0, 0, 4092, 1, 1, 1, 0, 0, 4096, 0, 4096, 0, 1226, 4094, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 595, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 721, 4096, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4019, 0, 0, 4086, 0, 0, 0, 0, 0, 4096, 0, 4064, 0, 3034, 4096, 0, 1, 0, 4096, 0, 0, 4095, 4096, 0, 458, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1958, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4069, 0, 0, 4077, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 1160, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 1031, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2514, 4096, 0, 0, 0, 0, 4, 0, 9, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4010, 0, 0, 4065, 0, 0, 0, 0, 0, 4096, 0, 4070, 0, 1307, 4096, 0, 0, 0, 4096, 0, 0, 4092, 4096, 0, 418, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 1742, 4096, 0, 0, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 4022, 0, 0, 4079, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 1734, 4095, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 717, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 3, 0, 0, 14, 14, 0, 96, 4068, 0, 4, 4, 0, 0, 4, 0, 2, 23, 1, 1, 3, 15, 4091, 4, 4, 2, 46, 4090, 11, 3, 4091, 1, 0, 4080, 4056, 3, 4, 4081, 170, 83, 3, 0, 0, 4085, 3, 4088, 4, 3737, 4092, 4, 0, 0, 4092, 5, 4, 4092, 3995, 0, 79, 0, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1783, 4096, 0, 0, 0, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4004, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 4048, 0, 1724, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 730, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 2, 0, 4037, 46, 11, 0, 0, 0, 0, 0, 0, 0, 1753, 4096, 0, 0, 0, 0, 1895, 571, 16, 3, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 4095, 4085, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 1759, 2153, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 4096, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8334, 8190, 8185, 8212, 8155, 7992, 8276] avg=8192.0000 max=8334 max_rank=1 min=7992 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32635 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4037, 46, 10, 0, 0, 0, 0, 0, 0, 0, 721, 4096, 0, 0, 0, 0, 1896, 572, 16, 3, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 4096, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 3034, 2154, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 4096, 0, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 7304, 8194, 8179, 8174, 8166, 9253, 8077] avg=8192.0000 max=9253 max_rank=6 min=7304 min_rank=1 imbalance_ratio=1.129517 max_load_reduction=56283 max_load_reduction_pct=0.858810 moved_to_echo=33670 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4037, 46, 11, 0, 0, 0, 0, 0, 0, 0, 1958, 4096, 0, 0, 0, 0, 1896, 572, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1160, 2154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 4095, 0, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8541, 8181, 8189, 8224, 8157, 7404, 8650] avg=8192.0000 max=8650 max_rank=7 min=7404 min_rank=6 imbalance_ratio=1.055908 max_load_reduction=56886 max_load_reduction_pct=0.868011 moved_to_echo=32435 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 2, 0, 4037, 46, 11, 0, 0, 0, 0, 0, 0, 0, 1742, 4096, 0, 0, 0, 0, 1896, 571, 16, 3, 0, 0, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 4095, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1734, 2153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 4096, 0, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8324, 8191, 8182, 8176, 8159, 7976, 8336] avg=8192.0000 max=8336 max_rank=7 min=7976 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32647 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4037, 46, 11, 0, 0, 0, 0, 0, 0, 0, 2514, 4096, 0, 0, 0, 0, 1896, 571, 16, 3, 0, 0, 0, 0, 4, 0, 9, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 4096, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 1307, 2154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3521, 4096, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 9096, 8194, 8180, 8165, 8145, 7531, 8035] avg=8192.0000 max=9096 max_rank=1 min=7531 min_rank=6 imbalance_ratio=1.110352 max_load_reduction=56440 max_load_reduction_pct=0.861206 moved_to_echo=31876 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 3, 0, 0, 14, 4032, 46, 11, 0, 0, 0, 0, 0, 14, 0, 96, 4068, 0, 4, 4, 0, 1894, 571, 16, 3, 0, 0, 0, 0, 0, 4, 0, 2, 23, 1, 1, 3, 4092, 4080, 0, 0, 0, 0, 0, 0, 15, 4091, 4, 4, 2, 46, 4087, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3, 59, 1, 0, 4080, 4056, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 170, 83, 3, 0, 0, 4069, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 3737, 2152, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 3521, 3995, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 6670, 8206, 8260, 8206, 8409, 9985, 7604] avg=8192.0000 max=9985 max_rank=6 min=6670 min_rank=1 imbalance_ratio=1.218872 max_load_reduction=55551 max_load_reduction_pct=0.847641 moved_to_echo=34204 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4037, 46, 11, 0, 0, 0, 0, 0, 0, 0, 2305, 4096, 0, 0, 0, 0, 1895, 572, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 1, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 1, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1226, 2153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 4093, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8887, 8181, 8190, 8226, 8175, 7475, 8212] avg=8192.0000 max=8887 max_rank=1 min=7475 min_rank=6 imbalance_ratio=1.084839 max_load_reduction=56649 max_load_reduction_pct=0.864395 moved_to_echo=32088 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4043, 47, 11, 0, 0, 0, 0, 0, 0, 0, 1783, 4096, 0, 0, 0, 0, 1901, 576, 18, 6, 0, 0, 0, 0, 1, 0, 17, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 4096, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 1724, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3520, 4096, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8380, 8199, 8171, 8153, 8170, 7920, 8346] avg=8192.0000 max=8380 max_rank=1 min=7920 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32589 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 128, 128, 32384, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 512, 128, 128, 32768, 32384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 256, 128, 128, 0, 0, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 28288, 32768, 0, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 15744, 17280, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 12928, 32768, 0, 128, 128, 0, 15232, 4608, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4090, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 186, 0, 0, 0, 4096, 0, 4085, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 3846, 4089, 0, 5, 0, 0, 0, 0, 4096, 14, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 26, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4033, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 123, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3912, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 30, 0, 0, 0, 4096, 0, 4085, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4066, 4093, 0, 2, 0, 0, 0, 0, 4096, 13, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 38, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4054, 4094, 0, 3, 0, 0, 0, 0, 4095, 6, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4092, 0, 4092, 4051, 7, 3, 4, 4, 0, 4092, 0, 4, 0, 15, 9, 4069, 4088, 3, 4, 5, 5, 2, 4, 161, 0, 4, 4, 0, 3, 0, 4081, 4090, 0, 3, 1503, 16, 4, 0, 4089, 0, 4059, 0, 4, 16, 0, 4, 0, 4092, 0, 7, 2473, 4092, 0, 2, 1, 0, 0, 3, 4089, 0, 4092, 0, 0, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4090, 4096, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 138, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3943, 4092, 0, 4, 0, 0, 0, 0, 4096, 8, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 182, 0, 0, 0, 4095, 0, 4071, 0, 0, 2, 0, 0, 0, 4091, 0, 0, 3898, 4087, 0, 9, 0, 0, 0, 0, 4096, 23, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4096, 0, 3767, 48, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4090, 7, 4, 1, 0, 0, 0, 0, 0, 0, 0, 4054, 4094, 0, 3, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8193, 8150, 8194, 7949, 8194, 8468, 8191] avg=8192.0000 max=8468 max_rank=6 min=7949 min_rank=4 imbalance_ratio=1.033691 max_load_reduction=57068 max_load_reduction_pct=0.870789 moved_to_echo=32802 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 62, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 4096, 0, 3767, 49, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 7, 4, 1, 0, 0, 0, 0, 0, 0, 0, 3912, 4096, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8191, 8205, 8192, 8035, 8203, 8325, 8185] avg=8192.0000 max=8325 max_rank=6 min=8035 min_rank=4 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32748 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 37, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 4096, 0, 3767, 49, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 7, 4, 1, 0, 0, 0, 0, 0, 0, 0, 4033, 4096, 0, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8191, 8180, 8192, 7938, 8204, 8446, 8185] avg=8192.0000 max=8446 max_rank=6 min=7938 min_rank=4 imbalance_ratio=1.031006 max_load_reduction=57090 max_load_reduction_pct=0.871124 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 4096, 0, 3767, 49, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4090, 7, 4, 1, 0, 0, 0, 0, 0, 0, 0, 4066, 4093, 0, 2, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8191, 8144, 8196, 7942, 8187, 8478, 8199] avg=8192.0000 max=8478 max_rank=6 min=7942 min_rank=4 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=32806 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 19, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 4096, 0, 3767, 49, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4091, 7, 4, 1, 0, 0, 0, 0, 0, 0, 0, 3943, 4092, 0, 4, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8198, 8162, 8194, 8050, 8189, 8356, 8193] avg=8192.0000 max=8356 max_rank=6 min=8050 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32788 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 72, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 4096, 0, 3767, 48, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4088, 7, 9, 1, 0, 0, 0, 0, 0, 0, 0, 3846, 4089, 0, 5, 0, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8192, 8216, 8196, 8097, 8190, 8260, 8199] avg=8192.0000 max=8260 max_rank=6 min=8097 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32746 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 4092, 4051, 7, 3, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4, 0, 15, 9, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 4, 5, 5, 2, 4, 161, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 3, 0, 4081, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1503, 16, 4, 0, 4089, 0, 3764, 48, 0, 0, 0, 0, 0, 0, 4059, 0, 4, 16, 0, 4, 0, 4092, 6, 4, 0, 0, 0, 0, 0, 0, 0, 7, 2473, 4092, 0, 2, 1, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4089, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8189, 8316, 8182, 9427, 8185, 6892, 8177] avg=8192.0000 max=9427 max_rank=4 min=6892 min_rank=6 imbalance_ratio=1.150757 max_load_reduction=56109 max_load_reduction_pct=0.856155 moved_to_echo=32681 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 19, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 4095, 0, 3773, 48, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 2, 0, 0, 0, 4091, 8, 6, 2, 4, 0, 0, 0, 0, 0, 0, 3898, 4087, 0, 9, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 23, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8163, 8190, 8098, 8184, 8311, 8207] avg=8192.0000 max=8311 max_rank=6 min=8098 min_rank=4 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32800 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 2304, 128, 128, 0, 32768, 0, 30208, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 30336, 32768, 0, 128, 128, 0, 2560, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 32768, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 128, 0, 128, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 384, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 3886, 0, 0, 0, 0, 0, 0, 4096, 2496, 0, 0, 1, 0, 2, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3905, 0, 0, 0, 1996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 1, 4096, 0, 4096, 4096, 4055, 0, 0, 0, 0, 0, 0, 4096, 4042, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 86] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 4062, 0, 0, 0, 0, 0, 0, 4096, 2087, 0, 0, 1, 0, 11, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 2042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4092, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 5, 0, 4096, 4096, 0, 1, 4096, 0, 4096, 4096, 3936, 0, 0, 0, 0, 0, 0, 4096, 4027, 0, 0, 0, 0, 8, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 221] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 3946, 0, 0, 0, 0, 0, 0, 4096, 3763, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 4044, 0, 0, 0, 0, 0, 0, 4096, 4000, 0, 0, 0, 0, 6, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 151] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4045, 0, 24, 4, 4, 0, 4091, 4093, 226, 5, 3, 0, 0, 0, 4, 4, 0, 0, 0, 25, 4, 70, 0, 4092, 0, 3, 4075, 4090, 0, 171, 4080, 15, 4091, 4092, 3745, 0, 0, 0, 0, 5, 5, 4091, 3977, 18, 0, 0, 0, 0, 4088, 4091, 0, 9, 4, 4077, 1, 0, 0, 5, 5, 3424, 5, 4, 2, 669] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 4096, 4096, 0, 1, 4096, 0, 4096, 4096, 4003, 0, 0, 0, 0, 0, 0, 4096, 2877, 0, 0, 0, 0, 20, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1292] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 7, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4055, 31, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 137, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2496, 0, 0, 1, 0, 2, 656, 22, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 0, 0, 1996, 3303, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8163, 8182, 8173, 8119, 7273, 8192, 9237] avg=8192.0000 max=9237 max_rank=7 min=7273 min_rank=5 imbalance_ratio=1.127563 max_load_reduction=56299 max_load_reduction_pct=0.859055 moved_to_echo=32821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 7, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4055, 28, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 137, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2087, 0, 0, 1, 0, 11, 656, 22, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 2042, 3303, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8161, 8179, 8175, 8295, 6873, 8195, 9465] avg=8192.0000 max=9465 max_rank=7 min=6873 min_rank=5 imbalance_ratio=1.155396 max_load_reduction=56071 max_load_reduction_pct=0.855576 moved_to_echo=32828 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4056, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 137, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4042, 0, 0, 0, 0, 0, 656, 22, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 86, 3303, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8169, 8180, 8171, 8288, 8816, 8195, 7518] avg=8192.0000 max=8816 max_rank=5 min=7518 min_rank=7 imbalance_ratio=1.076172 max_load_reduction=56720 max_load_reduction_pct=0.865479 moved_to_echo=32817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 6, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4052, 28, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 1, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 137, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4027, 0, 0, 0, 0, 8, 656, 22, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 221, 3303, 34, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8165, 8178, 8176, 8169, 8809, 8192, 7654] avg=8192.0000 max=8809 max_rank=5 min=7654 min_rank=7 imbalance_ratio=1.075317 max_load_reduction=56727 max_load_reduction_pct=0.865585 moved_to_echo=32824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 7, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4055, 28, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 137, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4000, 0, 0, 0, 0, 6, 656, 22, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 151, 3303, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8161, 8179, 8173, 8277, 8780, 8195, 7578] avg=8192.0000 max=8780 max_rank=5 min=7578 min_rank=7 imbalance_ratio=1.071777 max_load_reduction=56756 max_load_reduction_pct=0.866028 moved_to_echo=32830 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4045, 0, 24, 4, 4, 0, 7, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 226, 5, 3, 0, 0, 0, 4, 4, 4090, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4, 70, 0, 4092, 4051, 30, 0, 0, 0, 0, 0, 0, 0, 3, 4075, 0, 0, 171, 4058, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 138, 3745, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 3977, 18, 0, 0, 0, 0, 655, 22, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 9, 4, 4077, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 3424, 5, 4, 2, 669, 3299, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8392, 8272, 8322, 7979, 8768, 8180, 7446] avg=8192.0000 max=8768 max_rank=5 min=7446 min_rank=7 imbalance_ratio=1.070312 max_load_reduction=56768 max_load_reduction_pct=0.866211 moved_to_echo=32373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4056, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 137, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3763, 0, 0, 0, 0, 4, 656, 22, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 488, 3303, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8161, 8180, 8170, 8179, 8541, 8195, 7911] avg=8192.0000 max=8541 max_rank=5 min=7911 min_rank=7 imbalance_ratio=1.042603 max_load_reduction=56995 max_load_reduction_pct=0.869675 moved_to_echo=32826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4062, 28, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 1, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 131, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2877, 0, 0, 0, 0, 20, 660, 23, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1292, 3305, 34, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8164, 8186, 8176, 8230, 7676, 8192, 8727] avg=8192.0000 max=8727 max_rank=7 min=7676 min_rank=5 imbalance_ratio=1.065308 max_load_reduction=56809 max_load_reduction_pct=0.866837 moved_to_echo=32825 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 0, 256, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 1152, 31744, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 32000, 128, 128, 128, 7040, 26496, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 32768, 32512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 27392, 128, 0, 128, 0, 128, 5376, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 128, 128, 0, 0, 0, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 4094, 24, 4092, 4096, 4096, 0, 0, 0, 2617, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 201, 0, 2882, 0, 2463, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 212, 4088, 4096, 4096, 0, 0, 0, 2046, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 868, 0, 3810, 0, 1254, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 208, 4095, 4096, 4096, 0, 0, 0, 2442, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 273, 0, 3816, 0, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4094, 62, 4090, 4096, 4096, 0, 0, 0, 2874, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 2, 4096, 537, 0, 3506, 0, 1211, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 233, 4095, 4096, 4096, 0, 0, 0, 1719, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 618, 0, 3443, 0, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4080, 0, 47, 4, 3782, 3, 3, 0, 13, 2, 4, 0, 0, 4092, 426, 3966, 4041, 4084, 0, 0, 0, 3754, 0, 5, 4091, 0, 0, 4, 9, 4088, 4090, 0, 5, 0, 6, 3, 0, 0, 0, 0, 4091, 4063, 0, 4090, 630, 4, 3718, 64, 64, 0, 6, 3, 2, 2, 0, 0, 3, 4092, 7, 4081, 0, 11, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 4094, 43, 4091, 4096, 4096, 0, 0, 0, 3457, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 304, 0, 3826, 0, 561, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 144, 4079, 4096, 4096, 0, 0, 0, 2025, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 159, 0, 4003, 0, 1869, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 4096, 0, 0, 29, 3, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4094, 24, 2562, 1365, 0, 0, 0, 0, 0, 0, 4092, 1505, 0, 0, 0, 0, 2617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 26, 201, 0, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2463, 0, 0, 0, 0, 0, 0, 4096, 2705, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8052, 8214, 8192, 8192, 7206, 9264, 8191] avg=8192.0000 max=9264 max_rank=6 min=7206 min_rank=5 imbalance_ratio=1.130859 max_load_reduction=56272 max_load_reduction_pct=0.858643 moved_to_echo=32853 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 29, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 233, 2562, 1365, 0, 0, 0, 0, 0, 0, 4095, 1505, 0, 0, 0, 0, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 26, 618, 0, 3443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2180, 0, 0, 0, 0, 0, 0, 4096, 2705, 1, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8256, 7319, 8192, 8191, 8183, 8982, 8189] avg=8192.0000 max=8982 max_rank=6 min=7319 min_rank=2 imbalance_ratio=1.096436 max_load_reduction=56554 max_load_reduction_pct=0.862946 moved_to_echo=33545 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 29, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4094, 43, 2562, 1365, 0, 0, 0, 0, 0, 0, 4091, 1505, 0, 0, 0, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 26, 304, 0, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 0, 0, 0, 0, 0, 0, 4096, 2705, 1, 0, 0, 0, 0, 0, 4, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8067, 9053, 8192, 8191, 8253, 7363, 8193] avg=8192.0000 max=9053 max_rank=2 min=7363 min_rank=6 imbalance_ratio=1.105103 max_load_reduction=56483 max_load_reduction_pct=0.861862 moved_to_echo=32000 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 29, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4094, 62, 2562, 1365, 0, 0, 0, 0, 0, 0, 4090, 1505, 0, 0, 0, 0, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 26, 537, 0, 3506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1211, 0, 0, 0, 0, 0, 0, 4096, 2705, 1, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8087, 8469, 8192, 8192, 8167, 8013, 8192] avg=8192.0000 max=8469 max_rank=2 min=8013 min_rank=6 imbalance_ratio=1.033813 max_load_reduction=57067 max_load_reduction_pct=0.870773 moved_to_echo=32564 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 29, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 212, 2562, 1365, 0, 0, 0, 0, 0, 0, 4088, 1505, 0, 0, 0, 0, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 26, 868, 0, 3810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 0, 0, 0, 0, 0, 0, 4096, 2705, 0, 0, 0, 0, 0, 0, 7, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8237, 7639, 8192, 8192, 8801, 8055, 8196] avg=8192.0000 max=8801 max_rank=5 min=7639 min_rank=2 imbalance_ratio=1.074341 max_load_reduction=56735 max_load_reduction_pct=0.865707 moved_to_echo=33244 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 29, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 208, 2562, 1365, 0, 0, 0, 0, 0, 0, 4095, 1505, 0, 0, 0, 0, 2442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 26, 273, 0, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1454, 0, 0, 0, 0, 0, 0, 4096, 2705, 1, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8231, 8042, 8192, 8191, 8211, 8256, 8189] avg=8192.0000 max=8256 max_rank=6 min=8042 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32847 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 0, 47, 4, 3782, 3, 3, 29, 3, 0, 0, 0, 0, 0, 0, 0, 13, 2, 4, 0, 0, 4092, 426, 2528, 1363, 0, 0, 0, 0, 0, 0, 3966, 1484, 0, 0, 0, 0, 3754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 9, 4088, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 3, 0, 0, 0, 4091, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 26, 630, 4, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 64, 64, 0, 6, 3, 2, 2, 0, 4084, 2701, 1, 0, 0, 0, 0, 0, 0, 3, 4089, 7, 4081, 0, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7951, 8428, 9204, 8196, 8195, 8441, 6927, 8194] avg=8192.0000 max=9204 max_rank=2 min=6927 min_rank=6 imbalance_ratio=1.123535 max_load_reduction=56332 max_load_reduction_pct=0.859558 moved_to_echo=31757 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 35, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 144, 2569, 1369, 0, 0, 0, 0, 0, 0, 4079, 1492, 0, 0, 0, 0, 2025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 159, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1869, 0, 0, 0, 0, 0, 0, 4096, 2711, 0, 0, 0, 0, 0, 0, 9, 0, 4087, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8178, 7596, 8188, 8192, 8274, 8676, 8192] avg=8192.0000 max=8676 max_rank=6 min=7596 min_rank=2 imbalance_ratio=1.059082 max_load_reduction=56860 max_load_reduction_pct=0.867615 moved_to_echo=33334 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 256, 3712, 128, 29056, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 0, 0, 32768, 1408, 20480, 11008, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 128, 128, 32512, 128, 128, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 11136, 0, 128, 128, 128, 128, 0, 32768, 21760, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32640, 12032, 0, 0, 0, 0, 20992, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 3986, 0, 6, 0, 0, 0, 1, 0, 0, 4096, 0, 3772, 0, 2, 4096, 0, 0, 0, 4095, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 4039, 4093, 488, 0, 3, 4096, 0, 2, 4095, 4096, 3, 0, 4095, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3546, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4079, 4096, 588, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3735, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4072, 4096, 413, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4061, 0, 22, 0, 0, 0, 0, 0, 0, 4096, 0, 3931, 0, 1, 4096, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4066, 4091, 208, 0, 5, 4096, 0, 1, 4096, 4096, 2, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4035, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 3369, 0, 2, 4096, 0, 0, 0, 4096, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 4082, 4092, 788, 0, 5, 4096, 0, 3, 4094, 4096, 5, 0, 4093, 0, 0, 0, 4095, 4096, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4014, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 3422, 0, 4, 4096, 0, 0, 0, 4096, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 3986, 4091, 854, 0, 6, 4096, 0, 4, 4096, 4096, 6, 0, 4090, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4, 0, 4, 396, 3, 1, 6, 0, 3817, 0, 5, 0, 0, 4092, 1, 0, 13, 0, 4, 42, 4, 4, 4049, 7, 3656, 5, 0, 4075, 2, 0, 4, 4092, 4072, 0, 0, 4065, 0, 3, 0, 38, 21, 4085, 4074, 330, 4, 0, 4088, 0, 0, 4089, 4077, 0, 6, 4092, 4, 0, 5, 4083, 4091, 0, 0, 20, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4063, 0, 28, 0, 0, 0, 0, 0, 0, 4096, 0, 3454, 0, 7, 4096, 0, 0, 0, 4096, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 4035, 4087, 704, 0, 9, 4096, 0, 7, 4096, 4096, 12, 0, 4081, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4046, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3611, 433, 50, 0, 0, 0, 0, 0, 3546, 0, 0, 4096, 0, 0, 0, 485, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 3589, 588, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8241, 8190, 8127, 8195, 8257, 8192, 8192] avg=8192.0000 max=8257 max_rank=5 min=8127 min_rank=3 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32836 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4046, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4061, 0, 22, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3610, 432, 50, 0, 0, 0, 0, 0, 3931, 0, 1, 4096, 0, 0, 0, 485, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 3585, 208, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8248, 8188, 8513, 8195, 7864, 8194, 8192] avg=8192.0000 max=8513 max_rank=3 min=7864 min_rank=5 imbalance_ratio=1.039185 max_load_reduction=57023 max_load_reduction_pct=0.870102 moved_to_echo=32445 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4044, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4035, 0, 14, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3611, 432, 50, 0, 0, 0, 0, 0, 3369, 0, 2, 4096, 0, 0, 0, 485, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 3586, 788, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 5, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8216, 8189, 7952, 8189, 8461, 8197, 8192] avg=8192.0000 max=8461 max_rank=5 min=7952 min_rank=3 imbalance_ratio=1.032837 max_load_reduction=57075 max_load_reduction_pct=0.870895 moved_to_echo=33039 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4046, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4069, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3611, 433, 50, 0, 0, 0, 0, 0, 3735, 0, 0, 4096, 0, 0, 0, 485, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 3589, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8236, 8190, 8316, 8195, 8074, 8191, 8192] avg=8192.0000 max=8316 max_rank=3 min=8074 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32652 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4047, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4014, 0, 12, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3611, 432, 49, 0, 0, 0, 0, 0, 3422, 0, 4, 4096, 0, 0, 0, 485, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 3585, 854, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 6, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8193, 8188, 8007, 8189, 8431, 8196, 8189] avg=8192.0000 max=8431 max_rank=5 min=8007 min_rank=3 imbalance_ratio=1.029175 max_load_reduction=57105 max_load_reduction_pct=0.871353 moved_to_echo=33005 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4, 396, 3, 1, 6, 0, 4088, 4040, 0, 0, 0, 0, 0, 0, 3817, 0, 5, 0, 0, 4092, 1, 0, 71, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4, 42, 4, 4, 4049, 7, 3607, 430, 49, 0, 0, 0, 0, 0, 3656, 5, 0, 4075, 2, 0, 4, 485, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 4065, 0, 3, 0, 38, 2, 0, 0, 0, 0, 0, 0, 0, 21, 4085, 3571, 330, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 6, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4083, 4091, 0, 0, 20, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8542, 7986, 8209, 8227, 8180, 8011, 8179, 8202] avg=8192.0000 max=8542 max_rank=0 min=7986 min_rank=1 imbalance_ratio=1.042725 max_load_reduction=56994 max_load_reduction_pct=0.869659 moved_to_echo=32572 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4047, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4063, 0, 28, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3614, 435, 49, 0, 0, 0, 0, 0, 3454, 0, 7, 4096, 0, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 4, 6, 5, 0, 0, 0, 0, 0, 0, 4035, 3573, 704, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4091, 12, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8262, 8194, 8039, 8200, 8321, 8191, 8186] avg=8192.0000 max=8321 max_rank=5 min=8039 min_rank=3 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32898 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4046, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 3986, 0, 6, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 3610, 432, 49, 0, 0, 0, 0, 0, 3772, 0, 2, 4096, 0, 0, 0, 485, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 3587, 488, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8154, 8188, 8355, 8193, 8117, 8196, 8191] avg=8192.0000 max=8355 max_rank=3 min=8117 min_rank=5 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32697 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 0, 32768, 0, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[28928, 128, 128, 32768, 128, 0, 128, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32512, 28672, 4480, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32512, 0, 128, 0, 0, 32512, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 512, 128, 128, 128, 0, 32768, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 128, 28928, 3584, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1047, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2737, 0, 0, 60, 4045, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 4020, 0, 309, 75, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4091, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 4096, 0, 0, 0, 0, 1251, 4094, 4084, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 2533, 0, 0, 7, 4067, 0, 0, 0, 0, 4, 4096, 0, 4096, 4096, 0, 0, 4052, 0, 0, 0, 4034, 0, 329, 99, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 467, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2895, 0, 0, 18, 4087, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4074, 0, 0, 0, 4088, 0, 705, 49, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 4096, 0, 0, 0, 0, 1296, 4095, 4087, 0, 0, 0, 0, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 2287, 0, 0, 4, 4052, 0, 0, 0, 0, 4, 4096, 0, 4096, 4096, 0, 0, 3835, 0, 0, 0, 4048, 0, 836, 3, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 4092, 4089, 4042, 0, 3, 5, 0, 4092, 7, 0, 0, 4, 4091, 0, 289, 0, 0, 2973, 4092, 4092, 0, 4, 0, 45, 0, 4, 4091, 4, 0, 1, 4, 9, 11, 132, 3, 5, 25, 2598, 3, 4, 4, 1, 0, 4092, 3, 4088, 4086, 0, 4, 4092, 5, 0, 4, 4091, 0, 735, 1429, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 4096, 0, 0, 0, 0, 814, 4095, 4081, 0, 0, 0, 0, 12, 0, 4096, 0, 0, 0, 0, 0, 0, 2246, 0, 0, 47, 4076, 0, 0, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 4048, 0, 0, 0, 3990, 0, 940, 201, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 4096, 0, 0, 0, 0, 549, 4096, 4085, 0, 0, 0, 0, 20, 0, 4096, 0, 0, 0, 0, 0, 0, 2593, 0, 0, 28, 4008, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 3976, 0, 0, 0, 4054, 0, 1115, 26, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 379, 4095, 4091, 0, 0, 0, 0, 6, 0, 4095, 0, 0, 0, 0, 0, 0, 3062, 0, 0, 75, 4071, 0, 0, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 4087, 0, 0, 0, 4060, 0, 589, 55, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 2313, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1296, 2971, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4096, 0, 2317, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2287, 0, 0, 4096, 1783, 0, 0, 0, 0, 0, 0, 4, 4052, 0, 0, 0, 0, 4, 4096, 185, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4096, 0, 0, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 836, 3, 1, 0, 0, 2266, 939, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8194, 8354, 8195, 8166, 8341, 7996, 8093] avg=8192.0000 max=8354 max_rank=2 min=7996 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32596 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 2314, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 2972, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2317, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2895, 0, 0, 4096, 1782, 0, 0, 0, 0, 0, 0, 18, 4087, 0, 0, 0, 0, 0, 4096, 185, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4096, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 705, 49, 0, 0, 0, 2266, 939, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8181, 7535, 8178, 8773, 8386, 8235, 8047] avg=8192.0000 max=8773 max_rank=4 min=7535 min_rank=2 imbalance_ratio=1.070923 max_load_reduction=56763 max_load_reduction_pct=0.866135 moved_to_echo=33441 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 2313, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 2970, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 2317, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2533, 0, 0, 4096, 1783, 0, 0, 0, 0, 0, 0, 7, 4067, 0, 0, 0, 0, 4, 4096, 185, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4096, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 329, 99, 1, 0, 0, 2266, 939, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8194, 8305, 8189, 8412, 8359, 8213, 7668] avg=8192.0000 max=8412 max_rank=4 min=7668 min_rank=7 imbalance_ratio=1.026855 max_load_reduction=57124 max_load_reduction_pct=0.871643 moved_to_echo=32652 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2314, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 2972, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2317, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2737, 0, 0, 4096, 1782, 0, 0, 0, 0, 0, 0, 60, 4045, 0, 0, 0, 0, 0, 4096, 185, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 309, 75, 0, 0, 0, 2266, 939, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8180, 8113, 8178, 8615, 8386, 8254, 7609] avg=8192.0000 max=8615 max_rank=4 min=7609 min_rank=7 imbalance_ratio=1.051636 max_load_reduction=56921 max_load_reduction_pct=0.868546 moved_to_echo=32864 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 2314, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 814, 2971, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 0, 2317, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2246, 0, 0, 4096, 1782, 0, 0, 0, 0, 0, 0, 47, 4076, 0, 0, 0, 0, 1, 4096, 185, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4096, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 940, 201, 0, 0, 0, 2266, 939, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8206, 7866, 8190, 8124, 8405, 8209, 8336] avg=8192.0000 max=8405 max_rank=5 min=7866 min_rank=2 imbalance_ratio=1.026001 max_load_reduction=57131 max_load_reduction_pct=0.871750 moved_to_echo=33074 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 4092, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 4092, 7, 0, 0, 4, 2310, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, 2973, 2969, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 45, 0, 4, 4091, 4, 2310, 1762, 0, 0, 0, 0, 0, 0, 0, 1, 4, 9, 11, 132, 3, 5, 4091, 1779, 0, 0, 0, 0, 0, 0, 25, 2598, 3, 4, 4, 1, 0, 4092, 185, 0, 0, 0, 0, 0, 0, 0, 3, 64, 4086, 0, 4, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4091, 0, 735, 1429, 0, 3, 0, 2262, 938, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8186, 10323, 8220, 6035, 6912, 8254, 9462] avg=8192.0000 max=10323 max_rank=2 min=6035 min_rank=4 imbalance_ratio=1.260132 max_load_reduction=55213 max_load_reduction_pct=0.842484 moved_to_echo=30663 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 2314, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 2965, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 0, 2318, 1769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2593, 0, 0, 4096, 1782, 0, 0, 0, 0, 0, 0, 28, 4008, 0, 0, 0, 0, 0, 4096, 187, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4096, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 1115, 26, 0, 0, 0, 2272, 944, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8209, 7599, 8203, 8471, 8319, 8127, 8411] avg=8192.0000 max=8471 max_rank=4 min=7599 min_rank=2 imbalance_ratio=1.034058 max_load_reduction=57065 max_load_reduction_pct=0.870743 moved_to_echo=33328 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 2314, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 2971, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4095, 0, 2317, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3062, 0, 0, 4096, 1782, 0, 0, 0, 0, 0, 0, 75, 4071, 0, 0, 0, 0, 1, 4096, 185, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 589, 55, 1, 0, 0, 2266, 939, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8186, 7441, 8183, 8940, 8428, 8248, 7910] avg=8192.0000 max=8940 max_rank=4 min=7441 min_rank=2 imbalance_ratio=1.091309 max_load_reduction=56596 max_load_reduction_pct=0.863586 moved_to_echo=33526 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 512, 32768, 0, 128, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[384, 31104, 128, 128, 128, 128, 128, 32768, 1536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 384, 0, 0, 8832, 23808, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32512, 0, 5632, 2048, 128, 128, 0, 18176, 7552, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 18560, 128, 128, 32768, 14336, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 128, 128, 18560, 14208, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 128, 32768, 128, 18560, 14208, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4052, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 3119, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4087, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1016, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4067, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 1491, 4096, 1, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4089, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2620, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4085, 11, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 2793, 4095, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 2, 4093, 4095, 0, 4096, 4077, 0, 0, 4096, 0, 4092, 0, 3, 0, 0, 1312, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4027, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 2461, 4093, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 6, 4089, 4096, 0, 4096, 4063, 0, 0, 4096, 0, 4089, 0, 3, 0, 0, 1721, 17, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4053, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 3073, 4095, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 1, 4091, 4096, 0, 4096, 4061, 0, 0, 4096, 0, 4092, 0, 3, 0, 0, 943, 36, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4043, 45, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 2735, 4094, 0, 4073, 0, 0, 4096, 0, 0, 0, 0, 3, 4093, 4095, 0, 4096, 4065, 0, 0, 4096, 0, 4090, 0, 5, 0, 0, 1413, 9, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[3888, 611, 4, 0, 0, 3, 0, 0, 5, 5, 5, 4, 4, 0, 4085, 90, 0, 0, 4, 0, 2129, 4092, 602, 3885, 0, 0, 4092, 6, 3, 0, 2, 0, 4092, 4089, 0, 4091, 4092, 5, 103, 4087, 156, 4092, 0, 0, 0, 7, 906, 0, 4, 4092, 1, 0, 4, 0, 4086, 0, 4, 4, 7, 1, 3, 3995, 4091, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4052, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2095, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4049, 0, 0, 4096, 0, 4087, 0, 9, 0, 0, 2049, 22, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4052, 9, 0, 0, 0, 0, 0, 0, 4039, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 2, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3119, 1561, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1016, 0, 2525, 43, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8181, 8781, 8190, 8197, 7680, 8192, 8202] avg=8192.0000 max=8781 max_rank=2 min=7680 min_rank=5 imbalance_ratio=1.071899 max_load_reduction=56755 max_load_reduction_pct=0.866013 moved_to_echo=32271 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4053, 126, 0, 0, 0, 0, 0, 0, 4039, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 1, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 3073, 1561, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 4091, 14, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3, 0, 0, 943, 36, 2524, 43, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8180, 8732, 8192, 8166, 7641, 8192, 8202] avg=8192.0000 max=8732 max_rank=2 min=7641 min_rank=5 imbalance_ratio=1.065918 max_load_reduction=56804 max_load_reduction_pct=0.866760 moved_to_echo=32201 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4085, 11, 0, 0, 0, 0, 0, 1, 4038, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2793, 1561, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 4093, 14, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3, 0, 0, 1312, 6, 2524, 43, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8180, 8458, 8192, 8184, 7980, 8192, 8201] avg=8192.0000 max=8458 max_rank=2 min=7980 min_rank=5 imbalance_ratio=1.032471 max_load_reduction=57078 max_load_reduction_pct=0.870941 moved_to_echo=32557 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4067, 16, 0, 0, 0, 0, 0, 0, 4039, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1491, 1561, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2620, 2, 2525, 43, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8181, 7151, 8190, 8199, 9286, 8192, 8202] avg=8192.0000 max=9286 max_rank=5 min=7151 min_rank=2 imbalance_ratio=1.133545 max_load_reduction=56250 max_load_reduction_pct=0.858307 moved_to_echo=33879 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4043, 45, 0, 0, 0, 0, 0, 1, 4038, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2735, 1561, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 14, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 5, 0, 0, 1413, 9, 2523, 43, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8180, 8373, 8194, 8172, 8083, 8192, 8202] avg=8192.0000 max=8373 max_rank=2 min=8083 min_rank=5 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32649 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4027, 4, 0, 0, 0, 0, 0, 0, 4039, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2461, 1560, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 4089, 14, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3, 0, 0, 1721, 17, 2523, 43, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8084, 8180, 8120, 8196, 8166, 8396, 8192, 8202] avg=8192.0000 max=8396 max_rank=5 min=8084 min_rank=0 imbalance_ratio=1.024902 max_load_reduction=57140 max_load_reduction_pct=0.871887 moved_to_echo=32956 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3888, 611, 4, 0, 0, 3, 0, 0, 4032, 13, 0, 0, 0, 0, 0, 0, 5, 5, 5, 4, 4, 0, 4085, 90, 4074, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2129, 1560, 602, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 6, 3, 0, 2, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4092, 14, 0, 0, 4092, 5, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 4092, 0, 0, 0, 7, 906, 0, 2522, 43, 0, 0, 0, 0, 0, 0, 4, 4092, 1, 0, 4, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 7, 1, 3, 3995, 4091, 5, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8551, 8274, 8180, 8192, 8306, 7726, 8187, 8120] avg=8192.0000 max=8551 max_rank=0 min=7726 min_rank=5 imbalance_ratio=1.043823 max_load_reduction=56985 max_load_reduction_pct=0.869522 moved_to_echo=32339 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4052, 21, 0, 0, 0, 0, 0, 0, 4046, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2095, 1550, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 9, 0, 0, 2049, 22, 2528, 49, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8180, 7741, 8190, 8146, 8744, 8197, 8205] avg=8192.0000 max=8744 max_rank=5 min=7741 min_rank=2 imbalance_ratio=1.067383 max_load_reduction=56792 max_load_reduction_pct=0.866577 moved_to_echo=33292 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 0, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32384, 896, 128, 0, 0, 128, 0, 128, 32384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 32768, 0, 128, 0, 128, 12032, 128, 20224, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 19968, 12544, 640, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 38, 0, 4094, 0, 4095, 4088, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4068, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 36, 0, 4096, 0, 4094, 4065, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4092, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 63, 0, 4095, 0, 4090, 4081, 0, 0, 0, 0, 4085, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4066, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 151, 0, 4095, 0, 4091, 3974, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4075, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 202, 0, 4096, 0, 4091, 3918, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4081, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 448, 0, 4096, 0, 4096, 3658, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4087, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[9, 4092, 4, 1, 3, 131, 1384, 0, 4092, 4, 4090, 2372, 8, 0, 0, 0, 4091, 0, 37, 4037, 4, 4088, 0, 8, 6, 1, 18, 2, 0, 4026, 0, 4087, 0, 4, 3, 4092, 0, 242, 0, 0, 6, 0, 5, 0, 0, 528, 5, 4, 12, 5, 4076, 0, 0, 6, 4091, 3608, 0, 4066, 3, 4086, 4085, 0, 9, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 334, 0, 4093, 0, 4094, 3945, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3923, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 36, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 336, 0, 4094, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4000, 58, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 96, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7881, 8495, 8198, 8198, 8160, 8124, 8284, 8196] avg=8192.0000 max=8495 max_rank=1 min=7881 min_rank=0 imbalance_ratio=1.036987 max_load_reduction=57041 max_load_reduction_pct=0.870377 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 63, 0, 3747, 0, 0, 0, 0, 0, 0, 0, 336, 0, 4090, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4000, 58, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 96, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7904, 8507, 8187, 8199, 8161, 8124, 8258, 8196] avg=8192.0000 max=8507 max_rank=1 min=7904 min_rank=0 imbalance_ratio=1.038452 max_load_reduction=57029 max_load_reduction_pct=0.870193 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 38, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 336, 0, 4095, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4000, 58, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 96, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7880, 8519, 8196, 8200, 8161, 8124, 8260, 8196] avg=8192.0000 max=8519 max_rank=1 min=7880 min_rank=0 imbalance_ratio=1.039917 max_load_reduction=57017 max_load_reduction_pct=0.870010 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 151, 0, 3747, 0, 0, 0, 0, 0, 0, 0, 336, 0, 4091, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4000, 58, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 96, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7991, 8401, 8197, 8198, 8162, 8124, 8267, 8196] avg=8192.0000 max=8401 max_rank=1 min=7991 min_rank=0 imbalance_ratio=1.025513 max_load_reduction=57135 max_load_reduction_pct=0.871811 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 202, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 336, 0, 4091, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4000, 58, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 96, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 8345, 8188, 8198, 8161, 8132, 8273, 8193] avg=8192.0000 max=8345 max_rank=1 min=8046 min_rank=0 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[9, 4092, 4, 1, 3, 131, 1384, 0, 3744, 0, 0, 0, 0, 0, 0, 0, 336, 4, 4090, 2372, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 37, 4037, 4, 6, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 18, 2, 0, 4026, 0, 4087, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 4092, 0, 242, 0, 0, 3995, 58, 6, 0, 0, 0, 0, 0, 6, 0, 5, 0, 0, 528, 5, 4, 4083, 4024, 0, 0, 0, 0, 0, 0, 12, 5, 4076, 0, 0, 6, 96, 3608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 3, 3, 4085, 0, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9368, 6810, 8183, 8146, 8400, 8655, 7803, 8171] avg=8192.0000 max=9368 max_rank=0 min=6810 min_rank=1 imbalance_ratio=1.143555 max_load_reduction=56168 max_load_reduction_pct=0.857056 moved_to_echo=33029 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 448, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 336, 0, 4096, 3658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4000, 58, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 96, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8090, 8198, 8198, 8160, 8124, 8279, 8196] avg=8192.0000 max=8291 max_rank=0 min=8090 min_rank=1 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 334, 0, 3747, 0, 0, 0, 0, 0, 0, 0, 330, 0, 4094, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4003, 63, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 93, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8369, 8189, 8199, 8171, 8129, 8112, 8192] avg=8192.0000 max=8369 max_rank=1 min=8112 min_rank=6 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32604 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 0, 640, 128, 128, 32768, 32256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 256, 0, 0, 32000, 512, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 768, 32000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 256, 2688, 0, 30080, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2688, 128, 32768, 30208, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2250, 12, 368, 123, 350, 0, 2056, 13, 779, 3042, 228, 660, 2009, 2167, 0, 197, 254, 1521, 217, 1224, 0, 631, 1410, 1776, 6, 14, 2554, 1274, 2836, 216, 2612, 730, 15, 1938, 3019, 209, 471, 251, 1806, 1763, 1754, 0, 2028, 1507, 0, 1108, 1379, 1054, 3233, 31, 1355, 0, 15, 827, 0, 577, 0, 174, 799, 1352, 2840, 1127, 954, 2421] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3172, 16, 355, 132, 548, 0, 1964, 146, 1008, 3028, 193, 266, 1571, 780, 0, 124, 228, 1193, 147, 1895, 0, 600, 2132, 1460, 61, 48, 3092, 1361, 3109, 155, 2646, 718, 4, 1772, 2359, 563, 278, 192, 2200, 1197, 1731, 0, 2427, 1554, 0, 790, 892, 809, 3129, 65, 1976, 6, 14, 274, 0, 490, 0, 331, 1303, 1020, 3006, 769, 1026, 3211] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3020, 50, 331, 184, 684, 0, 2173, 71, 875, 3440, 186, 659, 1380, 1409, 0, 251, 509, 1044, 224, 1771, 2, 754, 1392, 1338, 14, 26, 2879, 1337, 3084, 124, 2559, 675, 18, 1885, 2407, 162, 331, 144, 2414, 1214, 772, 2, 2240, 1540, 0, 1362, 900, 736, 3200, 9, 1871, 2, 18, 526, 14, 1162, 0, 184, 968, 1442, 2907, 1111, 885, 2665] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2545, 13, 214, 192, 615, 0, 2571, 28, 602, 3483, 286, 392, 1152, 2145, 2, 222, 322, 1076, 282, 1258, 0, 747, 1392, 1501, 20, 18, 2907, 1169, 3354, 260, 2676, 733, 3, 1518, 2616, 258, 407, 133, 2213, 1590, 1027, 0, 2403, 1846, 0, 1602, 1328, 1315, 3260, 16, 1844, 2, 7, 419, 1, 325, 1, 116, 667, 1203, 2454, 930, 985, 2870] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2342, 47, 258, 219, 455, 0, 1655, 18, 722, 3431, 410, 505, 1037, 2763, 0, 324, 570, 1692, 299, 1574, 1, 820, 853, 1568, 15, 189, 2085, 974, 3282, 492, 2709, 667, 5, 1084, 3110, 272, 539, 292, 2231, 1937, 1107, 0, 1895, 2549, 0, 1315, 1134, 1087, 3144, 31, 1505, 0, 4, 634, 0, 540, 6, 117, 616, 996, 2753, 1232, 1216, 2209] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2059, 12, 426, 189, 859, 1, 1859, 91, 894, 2455, 260, 441, 1513, 1372, 0, 249, 429, 1349, 256, 1568, 0, 785, 1647, 1590, 66, 4, 2395, 858, 3037, 272, 2688, 593, 12, 1486, 2587, 167, 438, 196, 2505, 1395, 1665, 0, 2520, 1679, 0, 1436, 1477, 1583, 3315, 50, 1469, 0, 14, 654, 0, 528, 1, 248, 1200, 1454, 2391, 929, 1070, 2850] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2493, 58, 231, 130, 1125, 0, 1992, 117, 748, 2955, 298, 359, 1151, 1686, 0, 259, 404, 969, 267, 1503, 0, 1212, 1935, 1466, 47, 84, 2940, 990, 2632, 200, 2521, 934, 12, 1249, 2880, 153, 257, 255, 2108, 1179, 1298, 0, 2333, 1857, 0, 1449, 1051, 981, 2792, 61, 1879, 1, 18, 574, 1, 661, 0, 130, 1181, 1515, 2726, 1716, 883, 2630] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2349, 70, 691, 64, 1038, 3, 1696, 23, 741, 2343, 342, 630, 1817, 1569, 0, 366, 577, 1446, 331, 1955, 0, 861, 1213, 1793, 18, 42, 2723, 840, 2475, 71, 2671, 646, 8, 1485, 2842, 114, 662, 210, 2396, 1437, 1720, 0, 2022, 1861, 0, 1078, 1352, 865, 3291, 8, 1385, 3, 9, 438, 4, 511, 0, 182, 1414, 1183, 2633, 1083, 925, 3011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2250, 12, 368, 123, 350, 0, 2056, 13, 2131, 97, 0, 0, 0, 0, 0, 0, 779, 3042, 228, 660, 2009, 2167, 0, 197, 188, 0, 0, 0, 0, 0, 0, 0, 254, 1521, 217, 1224, 0, 631, 1410, 1776, 547, 266, 0, 0, 0, 0, 0, 0, 6, 14, 2554, 1274, 705, 216, 2612, 730, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1938, 2284, 209, 471, 251, 1806, 1763, 0, 0, 0, 0, 0, 0, 0, 0, 1754, 0, 1361, 1507, 0, 1108, 1379, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 3233, 31, 1355, 0, 15, 827, 0, 577, 1528, 401, 0, 0, 0, 0, 0, 0, 0, 174, 799, 1352, 2840, 1127, 954, 796, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7400, 9270, 7846, 8111, 8737, 8163, 7967, 8042] avg=8192.0000 max=9270 max_rank=1 min=7400 min_rank=0 imbalance_ratio=1.131592 max_load_reduction=56266 max_load_reduction_pct=0.858551 moved_to_echo=32909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3172, 16, 355, 132, 548, 0, 1964, 146, 2336, 129, 0, 0, 0, 0, 0, 0, 1008, 3028, 193, 266, 1571, 780, 0, 124, 147, 0, 0, 0, 0, 0, 0, 0, 228, 1193, 147, 1895, 0, 600, 2132, 1460, 427, 318, 0, 0, 0, 0, 0, 0, 61, 48, 3092, 1361, 773, 155, 2646, 718, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1772, 1785, 563, 278, 192, 2200, 1197, 0, 0, 0, 0, 0, 0, 0, 0, 1731, 0, 1629, 1554, 0, 790, 892, 809, 0, 0, 0, 0, 0, 0, 0, 0, 3129, 65, 1976, 6, 14, 274, 0, 490, 2027, 480, 0, 0, 0, 0, 0, 0, 0, 331, 1303, 1020, 3006, 769, 1026, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8798, 7117, 8400, 8854, 7991, 7405, 8461, 8510] avg=8192.0000 max=8854 max_rank=3 min=7117 min_rank=1 imbalance_ratio=1.080811 max_load_reduction=56682 max_load_reduction_pct=0.864899 moved_to_echo=32367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2342, 47, 258, 219, 455, 0, 1655, 18, 2466, 88, 0, 0, 0, 0, 0, 0, 722, 3431, 410, 505, 1037, 2763, 0, 324, 193, 0, 0, 0, 0, 0, 0, 0, 570, 1692, 299, 1574, 1, 820, 853, 1568, 563, 248, 0, 0, 0, 0, 0, 0, 15, 189, 2085, 974, 816, 492, 2709, 667, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1084, 2354, 272, 539, 292, 2231, 1937, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 0, 1272, 2549, 0, 1315, 1134, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 3144, 31, 1505, 0, 4, 634, 0, 540, 1394, 375, 0, 0, 0, 0, 0, 0, 6, 117, 616, 996, 2753, 1232, 1216, 727, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7548, 9385, 8188, 7947, 8714, 8464, 7627, 7663] avg=8192.0000 max=9385 max_rank=1 min=7548 min_rank=0 imbalance_ratio=1.145630 max_load_reduction=56151 max_load_reduction_pct=0.856796 moved_to_echo=32468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3020, 50, 331, 184, 684, 0, 2173, 71, 2317, 107, 0, 0, 0, 0, 0, 0, 875, 3440, 186, 659, 1380, 1409, 0, 251, 150, 0, 0, 0, 0, 0, 0, 0, 509, 1044, 224, 1771, 2, 754, 1392, 1338, 436, 294, 0, 0, 0, 0, 0, 0, 14, 26, 2879, 1337, 767, 124, 2559, 675, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1885, 1821, 162, 331, 144, 2414, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 772, 2, 1503, 1540, 0, 1362, 900, 736, 0, 0, 0, 0, 0, 0, 0, 0, 3200, 9, 1871, 2, 18, 526, 14, 1162, 1682, 443, 0, 0, 0, 0, 0, 0, 0, 184, 968, 1442, 2907, 1111, 885, 876, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8937, 8350, 7764, 8381, 7989, 6815, 8927, 8373] avg=8192.0000 max=8937 max_rank=0 min=6815 min_rank=5 imbalance_ratio=1.090942 max_load_reduction=56599 max_load_reduction_pct=0.863632 moved_to_echo=32104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2545, 13, 214, 192, 615, 0, 2571, 28, 2520, 115, 0, 0, 0, 0, 0, 0, 602, 3483, 286, 392, 1152, 2145, 2, 222, 163, 0, 0, 0, 0, 0, 0, 0, 322, 1076, 282, 1258, 0, 747, 1392, 1501, 474, 315, 0, 0, 0, 0, 0, 0, 20, 18, 2907, 1169, 834, 260, 2676, 733, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1518, 1979, 258, 407, 133, 2213, 1590, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 0, 1613, 1846, 0, 1602, 1328, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 3260, 16, 1844, 2, 7, 419, 1, 325, 1812, 475, 0, 0, 0, 0, 0, 0, 1, 116, 667, 1203, 2454, 930, 985, 943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8813, 8447, 7367, 8617, 8101, 8731, 8161, 7299] avg=8192.0000 max=8813 max_rank=0 min=7299 min_rank=7 imbalance_ratio=1.075806 max_load_reduction=56723 max_load_reduction_pct=0.865524 moved_to_echo=32292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2493, 58, 231, 130, 1125, 0, 1992, 117, 1977, 105, 0, 0, 0, 0, 0, 0, 748, 2955, 298, 359, 1151, 1686, 0, 259, 179, 0, 0, 0, 0, 0, 0, 0, 404, 969, 267, 1503, 0, 1212, 1935, 1466, 522, 306, 0, 0, 0, 0, 0, 0, 47, 84, 2940, 990, 655, 200, 2521, 934, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1249, 2179, 153, 257, 255, 2108, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 1298, 0, 1566, 1857, 0, 1449, 1051, 981, 0, 0, 0, 0, 0, 0, 0, 0, 2792, 61, 1879, 1, 18, 574, 1, 661, 1660, 461, 0, 0, 0, 0, 0, 0, 0, 130, 1181, 1515, 2726, 1716, 883, 865, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 7635, 8584, 8371, 7392, 8202, 8108, 9016] avg=8192.0000 max=9016 max_rank=7 min=7392 min_rank=4 imbalance_ratio=1.100586 max_load_reduction=56520 max_load_reduction_pct=0.862427 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2059, 12, 426, 189, 859, 1, 1859, 91, 2282, 114, 0, 0, 0, 0, 0, 0, 894, 2455, 260, 441, 1513, 1372, 0, 249, 161, 0, 0, 0, 0, 0, 0, 0, 429, 1349, 256, 1568, 0, 785, 1647, 1590, 469, 330, 0, 0, 0, 0, 0, 0, 66, 4, 2395, 858, 755, 272, 2688, 593, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1486, 1957, 167, 438, 196, 2505, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 1665, 0, 1692, 1679, 0, 1436, 1477, 1583, 0, 0, 0, 0, 0, 0, 0, 0, 3315, 50, 1469, 0, 14, 654, 0, 528, 1799, 498, 0, 0, 0, 0, 0, 0, 1, 248, 1200, 1454, 2391, 929, 1070, 937, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7892, 7345, 8423, 7631, 8156, 9532, 8327, 8230] avg=8192.0000 max=9532 max_rank=5 min=7345 min_rank=1 imbalance_ratio=1.163574 max_load_reduction=56004 max_load_reduction_pct=0.854553 moved_to_echo=34245 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2349, 70, 691, 64, 1038, 3, 1696, 23, 1862, 124, 0, 0, 0, 0, 0, 0, 741, 2343, 342, 630, 1817, 1569, 0, 366, 179, 0, 0, 0, 0, 0, 0, 0, 577, 1446, 331, 1955, 0, 861, 1213, 1793, 519, 269, 0, 0, 0, 0, 0, 0, 18, 42, 2723, 840, 613, 71, 2671, 646, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1485, 2144, 114, 662, 210, 2396, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 0, 1348, 1861, 0, 1078, 1352, 865, 0, 0, 0, 0, 0, 0, 0, 0, 3291, 8, 1385, 3, 9, 438, 4, 511, 1904, 405, 0, 0, 0, 0, 0, 0, 0, 182, 1414, 1183, 2633, 1083, 925, 983, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7920, 7987, 8964, 7624, 8456, 8224, 7958, 8403] avg=8192.0000 max=8964 max_rank=2 min=7624 min_rank=3 imbalance_ratio=1.094238 max_load_reduction=56572 max_load_reduction_pct=0.863220 moved_to_echo=33041 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11136, 128, 12032, 14464, 0, 10240, 9600, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 1536, 8192, 10240, 21760, 8960, 8064, 7296, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[25472, 384, 13312, 128, 128, 4352, 128, 4864, 13824, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[20352, 384, 2944, 1280, 5760, 128, 16000, 512, 17920, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 12544, 16512, 1920, 3456, 1792, 17920, 11776, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 512, 21632, 8832, 6016, 1792, 21120, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3328, 10368, 2048, 12800, 128, 6528, 12032, 12544, 3968, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6400, 24192, 2304, 3968, 11648, 13952, 128, 2048, 1408, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 3951, 0, 3951, 4096, 0, 75, 3956, 0, 1, 2128, 4066, 0, 0, 0, 0, 0, 0, 262, 2787, 3953, 7, 4087, 0, 309, 0, 1673, 3, 1, 4, 0, 615, 2, 0, 3134, 4092, 10, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 352, 4096, 0, 1, 0, 3699, 4092, 189, 0, 0, 0, 0, 1749] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 0, 4079, 4096, 0, 1058, 4054, 0, 0, 3723, 4064, 0, 0, 0, 0, 0, 0, 46, 777, 4047, 0, 4092, 1, 689, 0, 203, 0, 0, 1, 0, 26, 0, 0, 1753, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 107, 4096, 0, 0, 0, 4056, 4096, 1287, 0, 0, 0, 0, 2795] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4096, 1, 4078, 3, 4062, 4093, 0, 1079, 3937, 0, 0, 3679, 3882, 0, 0, 0, 0, 0, 10, 638, 1988, 3144, 0, 4087, 2, 794, 3, 318, 0, 6, 0, 0, 46, 2, 0, 2480, 4093, 2, 0, 0, 5, 0, 0, 4096, 2, 0, 0, 55, 4093, 0, 2, 0, 3918, 4093, 924, 2, 0, 2, 0, 1819] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 4074, 0, 4088, 4093, 0, 247, 4055, 0, 0, 2693, 4084, 0, 0, 0, 0, 0, 19, 559, 2254, 3937, 4, 4095, 0, 114, 0, 515, 0, 0, 2, 0, 177, 0, 0, 2555, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 132, 4095, 0, 2, 0, 4089, 4096, 1159, 0, 0, 0, 0, 2112] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 0, 4064, 0, 3877, 4094, 0, 562, 3903, 0, 0, 3154, 4027, 0, 0, 0, 0, 0, 0, 1030, 2712, 3291, 1, 4039, 3, 665, 1, 405, 12, 0, 0, 0, 131, 1, 3, 2664, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 4096, 0, 7, 0, 4028, 4096, 570, 0, 0, 0, 0, 1794] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 3854, 0, 4087, 4096, 0, 275, 3761, 0, 0, 3078, 4037, 0, 0, 0, 0, 0, 174, 399, 2744, 3731, 0, 3899, 0, 487, 2, 733, 6, 0, 2, 0, 757, 0, 0, 2186, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 64, 4096, 0, 1, 0, 3918, 4096, 450, 0, 0, 0, 0, 2316] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 0, 4021, 0, 4003, 4096, 0, 623, 4034, 0, 1, 3344, 3955, 0, 0, 0, 1, 1, 9, 210, 1318, 4049, 4, 4068, 0, 196, 1, 663, 0, 0, 2, 0, 415, 0, 0, 3217, 4096, 7, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 284, 4095, 0, 0, 0, 3998, 4094, 719, 0, 0, 0, 0, 1820] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4091, 7, 4074, 0, 4085, 4079, 0, 310, 4086, 0, 5, 2812, 3984, 5, 0, 6, 3, 0, 19, 173, 2353, 4063, 0, 4079, 0, 172, 0, 1309, 15, 0, 0, 5, 740, 0, 2, 2328, 4091, 4, 0, 0, 0, 0, 3, 4091, 0, 0, 3, 88, 4090, 4, 15, 5, 4018, 4091, 590, 0, 5, 0, 4, 1626] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 0, 64, 3, 0, 0, 0, 0, 0, 0, 621, 0, 0, 1058, 4054, 0, 0, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 46, 2545, 1200, 0, 0, 0, 0, 0, 0, 777, 4047, 0, 1150, 1, 689, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 26, 0, 0, 1753, 4096, 1202, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 4096, 0, 0, 0, 2256, 1742, 0, 0, 0, 0, 0, 0, 4056, 1487, 1287, 0, 0, 0, 0, 2795, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 9456, 7855, 6867, 7078, 8195, 8201, 9625] avg=8192.0000 max=9625 max_rank=7 min=6867 min_rank=3 imbalance_ratio=1.174927 max_load_reduction=55911 max_load_reduction_pct=0.853134 moved_to_echo=33099 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 0, 4064, 0, 64, 3, 0, 0, 0, 0, 0, 0, 591, 0, 0, 562, 3903, 0, 0, 3154, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 1030, 2545, 1184, 0, 0, 0, 0, 0, 0, 2712, 3291, 1, 1136, 3, 665, 1, 405, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 131, 1, 3, 2664, 4094, 1142, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 7, 0, 2144, 1719, 0, 0, 0, 0, 0, 0, 4028, 1487, 570, 0, 0, 0, 0, 1794, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8210, 8786, 8214, 8047, 8195, 7977, 7879] avg=8192.0000 max=8786 max_rank=2 min=7879 min_rank=7 imbalance_ratio=1.072510 max_load_reduction=56750 max_load_reduction_pct=0.865936 moved_to_echo=32098 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 3854, 0, 64, 3, 0, 0, 0, 0, 0, 0, 623, 0, 0, 275, 3761, 0, 0, 3078, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 174, 399, 2545, 1143, 0, 0, 0, 0, 0, 0, 2744, 3731, 0, 1096, 0, 487, 2, 733, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 757, 0, 0, 2186, 4096, 1204, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 4096, 0, 1, 0, 2260, 1660, 0, 0, 0, 0, 0, 0, 3918, 1487, 450, 0, 0, 0, 0, 2316, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8018, 7737, 8298, 8793, 8251, 8187, 8081, 8171] avg=8192.0000 max=8793 max_rank=3 min=7737 min_rank=1 imbalance_ratio=1.073364 max_load_reduction=56743 max_load_reduction_pct=0.865829 moved_to_echo=32690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 3951, 0, 64, 3, 0, 0, 0, 0, 0, 0, 602, 0, 0, 75, 3956, 0, 1, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 262, 2543, 1198, 0, 0, 0, 0, 0, 0, 2787, 3953, 7, 1149, 0, 309, 0, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4, 0, 615, 2, 0, 3134, 4096, 1164, 0, 0, 0, 0, 0, 0, 4092, 10, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 4096, 0, 1, 0, 2185, 1740, 0, 0, 0, 0, 0, 0, 3699, 1485, 189, 0, 0, 0, 0, 1749, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 6762, 8069, 9878, 9019, 8196, 8374, 7122] avg=8192.0000 max=9878 max_rank=3 min=6762 min_rank=1 imbalance_ratio=1.205811 max_load_reduction=55658 max_load_reduction_pct=0.849274 moved_to_echo=32711 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 4074, 0, 64, 3, 0, 0, 0, 0, 0, 0, 623, 0, 0, 247, 4055, 0, 0, 2693, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 19, 559, 2545, 1201, 0, 0, 0, 0, 0, 0, 2254, 3937, 4, 1151, 0, 114, 0, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 177, 0, 0, 2555, 4093, 1204, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 4095, 0, 2, 0, 2261, 1743, 0, 0, 0, 0, 0, 0, 4089, 1487, 1159, 0, 0, 0, 0, 2112, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 7618, 8408, 7975, 8031, 8189, 8233, 8847] avg=8192.0000 max=8847 max_rank=7 min=7618 min_rank=1 imbalance_ratio=1.079956 max_load_reduction=56689 max_load_reduction_pct=0.865005 moved_to_echo=33300 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4096, 1, 4078, 3, 64, 3, 0, 0, 0, 0, 0, 0, 619, 0, 0, 1079, 3937, 0, 0, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 3882, 0, 0, 0, 0, 0, 10, 638, 2543, 1198, 0, 0, 0, 0, 0, 0, 1988, 3144, 0, 1149, 2, 794, 3, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 46, 2, 0, 2480, 4093, 1197, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 5, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 55, 4093, 0, 2, 0, 2246, 1740, 0, 0, 0, 0, 0, 0, 3918, 1486, 924, 2, 0, 2, 0, 1819, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 9314, 8271, 7398, 7824, 8193, 8138, 8151] avg=8192.0000 max=9314 max_rank=1 min=7398 min_rank=3 imbalance_ratio=1.136963 max_load_reduction=56222 max_load_reduction_pct=0.857880 moved_to_echo=32306 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 0, 4021, 0, 64, 3, 0, 0, 0, 0, 0, 0, 610, 0, 0, 623, 4034, 0, 1, 3344, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 1, 1, 9, 210, 2544, 1193, 0, 0, 0, 0, 0, 0, 1318, 4049, 4, 1143, 0, 196, 1, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 415, 0, 0, 3217, 4096, 1179, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 4095, 0, 0, 0, 2214, 1732, 0, 0, 0, 0, 0, 0, 3998, 1486, 719, 0, 0, 0, 0, 1820, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8612, 7913, 7374, 8909, 8196, 8325, 8023] avg=8192.0000 max=8909 max_rank=4 min=7374 min_rank=3 imbalance_ratio=1.087524 max_load_reduction=56627 max_load_reduction_pct=0.864059 moved_to_echo=33453 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4091, 7, 4074, 0, 70, 4, 0, 0, 0, 0, 0, 0, 614, 0, 0, 310, 4086, 0, 5, 2812, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 5, 0, 6, 3, 0, 19, 173, 2546, 1200, 0, 0, 0, 0, 0, 0, 2353, 4063, 0, 1140, 0, 172, 0, 1309, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 5, 740, 0, 2, 2328, 4079, 1208, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 3, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 88, 4090, 4, 15, 5, 2263, 1739, 0, 0, 0, 0, 0, 0, 4018, 1475, 590, 0, 5, 0, 4, 1626, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 7827, 7936, 9037, 8377, 8185, 8207, 7718] avg=8192.0000 max=9037 max_rank=3 min=7718 min_rank=7 imbalance_ratio=1.103149 max_load_reduction=56499 max_load_reduction_pct=0.862106 moved_to_echo=32487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4992, 0, 0, 4352, 31872, 0, 128, 24704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31744, 11904, 5888, 128, 128, 128, 128, 16128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 32256, 128, 640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 1152, 32768, 128, 128, 128, 17920, 13824, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32128, 128, 0, 128, 128, 128, 256, 3328, 20480, 9600, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 2944, 128, 128, 20352, 32768, 9600, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[17024, 30336, 128, 9216, 128, 3456, 128, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 0, 0, 4057, 0, 0, 0, 4047, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3762, 0, 1, 1, 4078, 4095, 0, 4095, 0, 0, 0, 2111, 3836, 1, 0, 326, 2352, 4095, 0, 0, 6, 0, 4082, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 36, 0, 4, 4096, 0, 0, 0, 4086, 35, 4096, 0, 0, 4096, 0, 0, 4094, 0, 0, 3926, 3, 0, 0, 4088, 4095, 0, 4093, 0, 0, 0, 844, 4059, 0, 3, 565, 3005, 4095, 0, 0, 4, 0, 3908, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 9, 0, 4095, 0, 0, 8, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 4086, 0, 0, 0, 4074, 7, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3829, 1, 1, 19, 4088, 4092, 0, 4094, 0, 0, 0, 1606, 3829, 0, 0, 446, 3247, 4096, 0, 0, 0, 0, 3432, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 1, 3722, 0, 0, 1, 4079, 2, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 4068, 0, 1, 2, 4035, 4089, 0, 4096, 0, 0, 0, 1362, 3852, 0, 0, 139, 3718, 4096, 0, 0, 0, 0, 3689, 0, 4096, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 1, 4084, 0, 0, 0, 4074, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3961, 0, 0, 23, 4072, 4093, 0, 4092, 0, 0, 0, 1149, 3927, 0, 2, 230, 3362, 4096, 0, 0, 7, 0, 3758, 0, 4096, 0, 10, 0, 4096, 0, 0, 0, 3, 0, 4095, 0, 0, 6, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 23, 0, 0, 3706, 0, 0, 0, 4073, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4050, 1, 0, 4, 4083, 4090, 0, 4092, 0, 0, 0, 2428, 3754, 0, 6, 59, 3203, 4096, 0, 0, 5, 0, 3293, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 4068, 11, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3989, 2, 4, 3, 4045, 4091, 0, 4094, 0, 0, 0, 1383, 3915, 0, 2, 258, 3110, 4093, 0, 0, 2, 0, 3851, 0, 4094, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[5, 0, 5, 33, 0, 0, 3352, 0, 0, 0, 4086, 8, 4087, 0, 5, 4089, 0, 5, 4090, 0, 4, 4064, 0, 70, 5, 4053, 4082, 4, 4090, 0, 0, 0, 1909, 3754, 7, 0, 218, 3766, 4088, 4, 5, 0, 9, 3301, 4, 4083, 0, 0, 5, 4091, 6, 5, 0, 0, 2, 4091, 5, 5, 28, 8, 0, 4, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 4086, 0, 4060, 219, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 7, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3829, 1, 1, 125, 2, 0, 0, 0, 0, 0, 0, 19, 4088, 4092, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1606, 3062, 0, 0, 446, 3247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3432, 0, 4096, 0, 1, 423, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 4080, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8374, 8193, 8054, 8225, 8361, 7952, 8191, 8186] avg=8192.0000 max=8374 max_rank=0 min=7952 min_rank=5 imbalance_ratio=1.022217 max_load_reduction=57162 max_load_reduction_pct=0.872223 moved_to_echo=32690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 0, 0, 4057, 0, 4061, 220, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 1, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3762, 0, 1, 125, 2, 0, 0, 0, 0, 0, 0, 1, 4078, 4095, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 3067, 1, 0, 326, 2352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4082, 0, 4094, 0, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4095, 4080, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8352, 8160, 7986, 8200, 7857, 8606, 8190, 8185] avg=8192.0000 max=8606 max_rank=5 min=7857 min_rank=4 imbalance_ratio=1.050537 max_load_reduction=56930 max_load_reduction_pct=0.868683 moved_to_echo=32838 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 1, 4084, 0, 4058, 225, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3961, 0, 0, 128, 2, 0, 0, 0, 0, 0, 0, 23, 4072, 4093, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1149, 3140, 0, 2, 230, 3362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3758, 0, 4096, 0, 10, 434, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 4096, 4080, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8378, 8186, 8187, 8215, 7883, 8305, 8192, 8190] avg=8192.0000 max=8378 max_rank=0 min=7883 min_rank=4 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4039, 0, 4060, 224, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 11, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3989, 2, 4, 128, 2, 0, 0, 0, 0, 0, 0, 3, 4045, 4091, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1383, 3130, 0, 2, 258, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3851, 0, 4094, 0, 2, 433, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4080, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8323, 8191, 8221, 8165, 7883, 8382, 8190, 8181] avg=8192.0000 max=8382 max_rank=5 min=7883 min_rank=4 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 36, 0, 4, 4096, 0, 4059, 233, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 35, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3926, 3, 0, 132, 2, 0, 0, 0, 0, 0, 0, 0, 4088, 4095, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 844, 3245, 0, 3, 565, 3005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3908, 0, 4096, 0, 1, 449, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 9, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4095, 4080, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8428, 8233, 8157, 8210, 7662, 8458, 8198, 8190] avg=8192.0000 max=8458 max_rank=5 min=7662 min_rank=4 imbalance_ratio=1.032471 max_load_reduction=57078 max_load_reduction_pct=0.870941 moved_to_echo=32508 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 1, 3722, 0, 4062, 221, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 2, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4068, 0, 1, 126, 2, 0, 0, 0, 0, 0, 0, 2, 4035, 4089, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1362, 3079, 0, 0, 139, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 0, 4096, 0, 1, 426, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2, 0, 4096, 4080, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8007, 8193, 8293, 8152, 8298, 8212, 8190, 8191] avg=8192.0000 max=8298 max_rank=4 min=8007 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32891 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 23, 0, 0, 3706, 0, 4058, 215, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4050, 1, 0, 122, 2, 0, 0, 0, 0, 0, 0, 4, 4083, 4090, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2428, 3002, 0, 6, 59, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3293, 0, 4087, 0, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8002, 8185, 8271, 8204, 8698, 7800, 8193, 8183] avg=8192.0000 max=8698 max_rank=4 min=7800 min_rank=5 imbalance_ratio=1.061768 max_load_reduction=56838 max_load_reduction_pct=0.867279 moved_to_echo=32874 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 5, 33, 0, 0, 3352, 0, 4058, 219, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 8, 4087, 0, 5, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4090, 0, 4, 4064, 0, 70, 126, 8, 0, 0, 0, 0, 0, 0, 5, 4053, 4082, 4, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1909, 2990, 7, 0, 218, 3766, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 9, 3301, 4, 4083, 0, 0, 419, 0, 0, 0, 0, 0, 0, 0, 5, 4083, 6, 5, 0, 0, 2, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 28, 8, 0, 4, 1, 0, 4088, 4080, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7672, 8195, 8367, 8165, 8894, 7821, 8192, 8230] avg=8192.0000 max=8894 max_rank=4 min=7672 min_rank=0 imbalance_ratio=1.085693 max_load_reduction=56642 max_load_reduction_pct=0.864288 moved_to_echo=33137 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 29440, 128, 32768, 0, 128, 3456, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12800, 24832, 128, 128, 2304, 25856, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32640, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 0, 128, 31744, 128, 128, 1024, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32640, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 32768, 32640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 128, 31232, 0, 32512, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 4096, 0, 0, 0, 4096, 0, 4096, 4091, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 666, 155, 0, 0, 0, 0, 3318, 0, 0, 0, 4086, 4096, 0, 0, 4021, 44, 0, 2, 0, 4096, 4095, 4, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 4091, 4096, 0, 0, 0, 4095, 0, 4096, 4096, 0, 32, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1220, 30, 0, 0, 0, 0, 2991, 0, 0, 0, 3952, 4096, 0, 0, 4001, 64, 0, 0, 0, 4096, 4095, 9, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 4080, 4085, 0, 0, 0, 4096, 0, 4096, 4094, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4096, 0, 252, 42, 0, 0, 0, 0, 3781, 0, 0, 0, 4096, 4096, 0, 0, 4088, 55, 0, 9, 0, 4096, 4073, 15, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4092, 4091, 4087, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 428, 18, 0, 3, 0, 0, 3793, 0, 0, 0, 4030, 4096, 0, 0, 4039, 11, 0, 11, 0, 4096, 4069, 13, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 4093, 4075, 0, 0, 0, 4094, 0, 4096, 4096, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4091, 1, 0, 0, 4095, 0, 563, 108, 0, 2, 2, 0, 3562, 0, 0, 0, 3971, 4096, 0, 0, 4057, 30, 0, 0, 0, 4096, 4083, 9, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[5, 6, 0, 3, 4075, 6, 1, 4091, 4091, 4091, 0, 5, 0, 4081, 0, 4082, 4088, 0, 115, 4089, 6, 5, 7, 0, 0, 0, 0, 1, 4, 4, 2, 4091, 0, 6, 12, 4090, 8, 996, 25, 0, 0, 29, 5, 2941, 1, 0, 2, 4075, 4091, 0, 5, 4088, 0, 2, 13, 1, 4087, 4090, 6, 9, 0, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4093, 4085, 4086, 0, 0, 0, 4096, 0, 4096, 4092, 0, 29, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3, 0, 0, 4095, 0, 165, 284, 0, 2, 0, 0, 3674, 0, 0, 0, 4096, 4096, 0, 0, 4028, 69, 0, 7, 0, 4096, 4052, 17, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 4093, 4095, 0, 0, 0, 4096, 0, 4096, 4091, 0, 75, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 348, 42, 0, 0, 0, 0, 3627, 0, 0, 0, 4086, 4096, 0, 0, 4081, 25, 0, 4, 0, 4096, 4092, 15, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 666, 155, 0, 3427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3318, 0, 0, 0, 4086, 651, 25, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4021, 44, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8203, 8157, 8195, 8344, 8080, 8165, 8197] avg=8192.0000 max=8344 max_rank=4 min=8080 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 32, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1220, 30, 0, 3426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2991, 0, 0, 0, 3952, 650, 25, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4001, 64, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 9, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8202, 8192, 8196, 8772, 7618, 8163, 8202] avg=8192.0000 max=8772 max_rank=4 min=7618 min_rank=5 imbalance_ratio=1.070801 max_load_reduction=56764 max_load_reduction_pct=0.866150 moved_to_echo=32755 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4080, 4085, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 252, 42, 0, 3427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3781, 0, 0, 0, 4096, 651, 25, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 55, 0, 9, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4073, 15, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8179, 8162, 8195, 7818, 8553, 8250, 8186] avg=8192.0000 max=8553 max_rank=5 min=7818 min_rank=4 imbalance_ratio=1.044067 max_load_reduction=56983 max_load_reduction_pct=0.869492 moved_to_echo=32807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 4091, 4087, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 428, 18, 0, 3427, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3793, 0, 0, 0, 4030, 651, 25, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4039, 11, 0, 11, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 13, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8192, 8160, 8187, 7969, 8502, 8159, 8180] avg=8192.0000 max=8502 max_rank=5 min=7969 min_rank=4 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=32810 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 4075, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 27, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4091, 4096, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 563, 108, 0, 3425, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 3562, 0, 0, 0, 3971, 650, 25, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4057, 30, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 9, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8183, 8187, 8192, 8192, 8212, 8185, 8191] avg=8192.0000 max=8212 max_rank=5 min=8183 min_rank=1 imbalance_ratio=1.002441 max_load_reduction=57324 max_load_reduction_pct=0.874695 moved_to_echo=32780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 4095, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 75, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 348, 42, 0, 3427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3627, 0, 0, 0, 4086, 651, 25, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 25, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 15, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8202, 8230, 8196, 7913, 8389, 8208, 8205] avg=8192.0000 max=8389 max_rank=5 min=7913 min_rank=4 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=32715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 4085, 4086, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 29, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4096, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4095, 0, 165, 284, 0, 3427, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3674, 0, 0, 0, 4096, 651, 25, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4028, 69, 0, 7, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4052, 17, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8185, 8185, 8181, 7974, 8448, 8202, 8169] avg=8192.0000 max=8448 max_rank=5 min=7974 min_rank=4 imbalance_ratio=1.031250 max_load_reduction=57088 max_load_reduction_pct=0.871094 moved_to_echo=32793 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[5, 6, 0, 3, 4075, 6, 1, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 4091, 4091, 0, 5, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 115, 4042, 6, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 4, 2, 4091, 4082, 10, 0, 0, 0, 0, 0, 0, 0, 6, 12, 4090, 8, 996, 25, 0, 3417, 0, 0, 0, 0, 0, 0, 0, 0, 29, 5, 2941, 1, 0, 2, 4075, 651, 30, 0, 0, 0, 0, 0, 0, 4091, 0, 5, 4088, 0, 2, 13, 1, 4, 0, 0, 0, 0, 0, 0, 0, 4087, 4090, 6, 9, 0, 0, 0, 5, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8190, 8263, 8194, 8554, 7734, 8204, 8206] avg=8192.0000 max=8554 max_rank=4 min=7734 min_rank=5 imbalance_ratio=1.044189 max_load_reduction=56982 max_load_reduction_pct=0.869476 moved_to_echo=32698 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 384, 32512, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 4736, 768, 0, 27520, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 27776, 128, 0, 128, 32512, 5248, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 32768, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 0, 128, 32512, 384, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 32768, 128, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 36, 0, 0, 0, 4085, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 923, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 4009, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3241, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 11, 0, 0, 0, 4034, 4080, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 15, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 4004, 3488, 0, 1, 4096, 0, 0, 4096, 4092, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3798, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 16, 0, 0, 0, 4087, 4080, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 0, 71, 0, 4096, 0, 1, 1, 0, 4095, 0, 0, 2, 3846, 3679, 0, 0, 4096, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3511, 0, 4096, 4095, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 8, 0, 0, 0, 4086, 4093, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 0, 7, 0, 4096, 0, 0, 8, 0, 4084, 0, 0, 2, 4069, 3832, 0, 0, 4096, 7, 0, 4096, 4089, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3627, 0, 4096, 4094, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 23, 0, 0, 0, 4065, 4085, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 6, 0, 4096, 0, 0, 8, 0, 4087, 0, 0, 0, 4021, 3979, 0, 1, 4096, 0, 0, 4096, 4091, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3889, 0, 4096, 4088, 0, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 28, 0, 0, 0, 4030, 4072, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 0, 20, 0, 4096, 0, 1, 5, 0, 4090, 0, 0, 5, 4066, 3193, 0, 4, 4096, 0, 0, 4096, 4090, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3429, 0, 4095, 4090, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 1, 0, 0, 0, 4068, 4089, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 13, 0, 4095, 0, 0, 0, 1, 4091, 0, 0, 2, 4088, 3691, 0, 1, 4096, 2, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4096, 0, 3801, 0, 4095, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4, 4090, 0, 1, 3, 0, 6, 5, 4090, 4084, 6, 5, 6, 6, 0, 0, 0, 2, 0, 7, 2883, 5, 96, 4, 4090, 11, 0, 0, 0, 4091, 0, 0, 0, 3966, 3967, 0, 1, 4091, 8, 6, 4087, 4091, 4089, 6, 0, 5, 2, 0, 1, 4089, 5, 0, 0, 0, 4085, 5, 1341, 5, 4091, 4090, 0, 5, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 11, 0, 0, 0, 3305, 774, 0, 0, 0, 0, 0, 0, 4034, 4080, 5, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 15, 0, 4092, 2882, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 3488, 0, 1, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 768, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8142, 8035, 8194, 7933, 8192, 8192, 8662] avg=8192.0000 max=8662 max_rank=7 min=7933 min_rank=4 imbalance_ratio=1.057373 max_load_reduction=56874 max_load_reduction_pct=0.867828 moved_to_echo=32979 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 8, 0, 0, 0, 3305, 774, 0, 0, 0, 0, 0, 0, 4086, 4093, 13, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 0, 7, 0, 4092, 2882, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4069, 3832, 0, 0, 440, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3627, 0, 768, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8215, 7728, 8188, 8350, 8189, 8192, 8491] avg=8192.0000 max=8491 max_rank=7 min=7728 min_rank=2 imbalance_ratio=1.036499 max_load_reduction=57045 max_load_reduction_pct=0.870438 moved_to_echo=33222 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 36, 0, 0, 0, 3305, 774, 0, 0, 0, 0, 0, 0, 4085, 4092, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 923, 0, 5, 0, 4092, 2882, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4009, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3241, 0, 768, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8200, 7902, 8192, 8538, 8196, 8192, 8105] avg=8192.0000 max=8538 max_rank=4 min=7902 min_rank=2 imbalance_ratio=1.042236 max_load_reduction=56998 max_load_reduction_pct=0.869720 moved_to_echo=33031 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 16, 0, 0, 0, 3305, 774, 0, 0, 0, 0, 0, 0, 4087, 4080, 3, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 0, 71, 0, 4092, 2882, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3846, 3679, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3511, 0, 768, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8193, 8232, 8193, 7967, 8194, 8192, 8375] avg=8192.0000 max=8375 max_rank=7 min=7967 min_rank=4 imbalance_ratio=1.022339 max_load_reduction=57161 max_load_reduction_pct=0.872208 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 23, 0, 0, 0, 3305, 774, 0, 0, 0, 0, 0, 0, 4065, 4085, 18, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 6, 0, 4092, 2882, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 3979, 0, 1, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 0, 768, 4088, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8191, 7382, 8191, 8441, 8191, 8192, 8750] avg=8192.0000 max=8750 max_rank=7 min=7382 min_rank=2 imbalance_ratio=1.068115 max_load_reduction=56786 max_load_reduction_pct=0.866486 moved_to_echo=33574 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 28, 0, 0, 0, 3304, 774, 0, 0, 0, 0, 0, 0, 4030, 4072, 48, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 0, 20, 0, 4094, 2882, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 5, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4066, 3193, 0, 4, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3429, 0, 768, 4090, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8173, 8590, 8192, 7708, 8188, 8192, 8291] avg=8192.0000 max=8590 max_rank=2 min=7708 min_rank=4 imbalance_ratio=1.048584 max_load_reduction=56946 max_load_reduction_pct=0.868927 moved_to_echo=32379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 1, 0, 0, 0, 3304, 774, 0, 0, 0, 0, 0, 0, 4068, 4089, 21, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 13, 0, 4092, 2882, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4088, 3691, 0, 1, 440, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 768, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8201, 7695, 8187, 8224, 8195, 8195, 8663] avg=8192.0000 max=8663 max_rank=7 min=7695 min_rank=2 imbalance_ratio=1.057495 max_load_reduction=56873 max_load_reduction_pct=0.867813 moved_to_echo=33277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4, 4090, 0, 1, 3, 0, 6, 5, 3303, 778, 0, 0, 0, 0, 0, 0, 4090, 4084, 6, 5, 6, 6, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 2883, 5, 96, 4, 4089, 2886, 0, 0, 0, 0, 0, 0, 4090, 11, 0, 0, 0, 4091, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 3967, 0, 1, 427, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4091, 0, 6, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 5, 0, 0, 0, 4085, 5, 4, 0, 0, 0, 0, 0, 0, 0, 1341, 5, 753, 4090, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8221, 9972, 8199, 8375, 8191, 8189, 6199] avg=8192.0000 max=9972 max_rank=2 min=6199 min_rank=7 imbalance_ratio=1.217285 max_load_reduction=55564 max_load_reduction_pct=0.847839 moved_to_echo=30954 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32256, 29952, 0, 128, 3584, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 9600, 128, 256, 128, 32768, 23168, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 128, 128, 26496, 6272, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[26752, 128, 6144, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 32768, 128, 128, 128, 128, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3, 0, 0, 0, 4069, 0, 0, 0, 0, 3960, 0, 0, 35, 4096, 0, 0, 0, 3523, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4093, 4096, 0, 4086, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3, 0, 0, 700, 0, 6, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 4019, 0, 0, 0, 0, 4025, 0, 0, 19, 4096, 0, 0, 0, 3761, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4095, 0, 4040, 0, 1, 4096, 2, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 503, 0, 3, 0, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4, 0, 0, 0, 4084, 0, 0, 0, 0, 3999, 0, 5, 33, 4096, 0, 0, 0, 3961, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 0, 4089, 4096, 0, 4032, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4091, 0, 0, 0, 0, 257, 0, 6, 0, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2, 0, 0, 0, 4060, 0, 0, 0, 0, 3527, 0, 2, 18, 4096, 0, 0, 0, 3097, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 4089, 4096, 0, 3930, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 1732, 0, 14, 0, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4, 0, 0, 0, 4077, 0, 0, 0, 0, 3896, 0, 0, 41, 4096, 0, 0, 0, 3947, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4093, 4096, 0, 3849, 0, 1, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 554, 0, 2, 0, 18] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 3982, 0, 3, 51, 4096, 0, 0, 0, 3163, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 4096, 0, 4042, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4093, 0, 0, 0, 0, 1138, 0, 29, 0, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 1, 0, 0, 4059, 0, 0, 0, 0, 4031, 0, 0, 30, 4096, 0, 0, 0, 3877, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 4095, 0, 4082, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 274, 1, 4, 0, 34] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1, 5, 4090, 1, 0, 5, 0, 4069, 3, 0, 2, 6, 4050, 0, 0, 1, 4090, 6, 3, 0, 3943, 0, 0, 9, 4090, 7, 4089, 1, 6, 2, 3, 4091, 4088, 3, 4089, 5, 5, 4085, 0, 4089, 6, 0, 2, 4088, 4088, 7, 0, 20, 8, 5, 0, 4089, 0, 1, 4089, 6, 0, 6, 14, 154, 0, 0, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 4019, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 19, 3542, 684, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3761, 0, 0, 1, 434, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 4040, 0, 1, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 3, 0, 11, 4096, 3407, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8270, 8292, 8196, 8217, 8191, 8193, 8020] avg=8192.0000 max=8292 max_rank=2 min=8020 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32621 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3, 0, 0, 0, 4069, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 35, 3543, 684, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3523, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 4086, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 700, 0, 6, 0, 2, 4096, 3407, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8222, 8053, 8194, 8260, 8191, 8193, 8214] avg=8192.0000 max=8260 max_rank=4 min=8053 min_rank=2 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32858 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 2, 0, 0, 0, 4060, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3527, 0, 2, 18, 3543, 684, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3097, 0, 1, 0, 434, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 3930, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1732, 0, 14, 0, 10, 4096, 3407, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 7774, 7628, 8190, 8104, 8191, 8191, 9259] avg=8192.0000 max=9259 max_rank=7 min=7628 min_rank=2 imbalance_ratio=1.130249 max_load_reduction=56277 max_load_reduction_pct=0.858719 moved_to_echo=33745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4, 0, 0, 0, 4077, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 41, 3543, 684, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3947, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 3849, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 2, 0, 18, 4096, 3407, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8164, 8477, 8194, 8024, 8191, 8191, 8077] avg=8192.0000 max=8477 max_rank=2 min=8024 min_rank=4 imbalance_ratio=1.034790 max_load_reduction=57059 max_load_reduction_pct=0.870651 moved_to_echo=32483 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4, 0, 0, 0, 4084, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 5, 33, 3543, 684, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3961, 0, 4, 0, 434, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 4032, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 6, 0, 12, 4096, 3407, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8264, 8495, 8190, 8206, 8191, 8187, 7778] avg=8192.0000 max=8495 max_rank=2 min=7778 min_rank=7 imbalance_ratio=1.036987 max_load_reduction=57041 max_load_reduction_pct=0.870377 moved_to_echo=32362 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 3975, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 3, 51, 3543, 684, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3163, 0, 3, 0, 434, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 4042, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1138, 0, 29, 0, 17, 4096, 3411, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8263, 7696, 8180, 8216, 8191, 8187, 8691] avg=8192.0000 max=8691 max_rank=7 min=7696 min_rank=2 imbalance_ratio=1.060913 max_load_reduction=56845 max_load_reduction_pct=0.867386 moved_to_echo=33285 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 1, 0, 0, 4059, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 30, 3542, 684, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3877, 0, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 4082, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 1, 4, 0, 34, 4096, 3407, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8287, 8407, 8193, 8256, 8191, 8189, 7816] avg=8192.0000 max=8407 max_rank=2 min=7816 min_rank=7 imbalance_ratio=1.026245 max_load_reduction=57129 max_load_reduction_pct=0.871719 moved_to_echo=32452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 5, 4090, 1, 0, 5, 0, 4069, 43, 5, 0, 0, 0, 0, 0, 0, 3, 0, 2, 6, 4050, 0, 0, 1, 3542, 688, 0, 0, 0, 0, 0, 0, 4090, 6, 3, 0, 3943, 0, 0, 9, 437, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 1, 6, 2, 3, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 66, 3, 4089, 5, 5, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 4088, 4076, 7, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 4089, 0, 1, 4089, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 6, 14, 154, 0, 0, 0, 16, 4089, 3402, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8292, 8488, 8199, 8253, 8199, 8205, 7681] avg=8192.0000 max=8488 max_rank=2 min=7681 min_rank=7 imbalance_ratio=1.036133 max_load_reduction=57048 max_load_reduction_pct=0.870483 moved_to_echo=32338 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 0, 29312, 0, 128, 128, 3584, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 128, 32256, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 32512, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 5376, 128, 128, 0, 128, 32768, 27264, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 31488, 0, 128, 256, 28416, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 160, 0, 0, 4060, 562, 0, 4096, 4096, 0, 0, 4096, 0, 3949, 0, 0, 4095, 5, 0, 4096, 0, 0, 0, 869, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1985, 0, 0, 919, 0, 0, 0, 4096, 0, 3876, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 4092, 181, 0, 4096, 4096, 0, 0, 4096, 0, 4068, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 574, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4088, 0, 0, 0, 3486, 0, 0, 114, 0, 0, 0, 4096, 0, 3874, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 22, 0, 3, 4073, 868, 0, 4096, 4092, 0, 0, 4096, 0, 4048, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 730, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 4093, 0, 0, 0, 1829, 2, 0, 768, 0, 0, 0, 4096, 0, 4037, 0, 0, 4096, 4095, 0, 0, 0, 0, 1, 3, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 111, 0, 0, 4060, 1413, 0, 4096, 4095, 0, 0, 4096, 0, 4050, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 615, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4093, 0, 0, 0, 1137, 2, 0, 1121, 0, 0, 0, 4096, 0, 3873, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 25, 0, 1, 4054, 1332, 0, 4096, 4094, 0, 0, 4096, 0, 3981, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 595, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 2162, 0, 1, 211, 0, 0, 0, 4096, 4, 4021, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 1, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 45, 0, 0, 4076, 444, 0, 4096, 4096, 0, 0, 4096, 0, 4011, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1143, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 3170, 0, 0, 40, 0, 0, 0, 4096, 0, 3445, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 37, 0, 0, 4087, 683, 0, 4096, 4096, 0, 0, 4096, 0, 3998, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 927, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4090, 0, 0, 0, 2125, 0, 0, 618, 0, 0, 0, 4096, 7, 3907, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[8, 0, 22, 6, 26, 4, 0, 4091, 1061, 9, 4087, 4090, 4, 1, 4080, 6, 3938, 0, 6, 3904, 0, 1, 4088, 0, 0, 3, 775, 3, 6, 4089, 6, 0, 5, 3, 0, 6, 6, 4048, 0, 0, 7, 2607, 0, 0, 21, 0, 5, 6, 4083, 0, 4058, 5, 0, 4081, 4088, 2, 0, 6, 6, 4, 0, 4088, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 160, 0, 0, 4060, 2673, 1386, 0, 0, 0, 0, 0, 0, 562, 0, 0, 3279, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 4095, 5, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 0, 0, 4096, 0, 0, 2499, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2791, 1309, 0, 0, 0, 0, 0, 0, 0, 1985, 0, 0, 919, 0, 0, 0, 4096, 1305, 0, 0, 0, 0, 0, 0, 211, 0, 3876, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 7937, 8163, 8281, 8196, 8305, 8183, 8192] avg=8192.0000 max=8305 max_rank=5 min=7937 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32876 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 4092, 2673, 1386, 0, 0, 0, 0, 0, 0, 181, 0, 0, 3279, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 4096, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 0, 4096, 0, 0, 2499, 817, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4088, 0, 0, 2790, 1309, 0, 0, 0, 0, 0, 0, 0, 3486, 0, 0, 114, 0, 0, 0, 4096, 1306, 0, 0, 0, 0, 0, 0, 211, 0, 3874, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 7556, 8278, 7986, 8189, 9002, 8181, 8192] avg=8192.0000 max=9002 max_rank=5 min=7556 min_rank=1 imbalance_ratio=1.098877 max_load_reduction=56534 max_load_reduction_pct=0.862640 moved_to_echo=33564 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 111, 0, 0, 4060, 2673, 1386, 0, 0, 0, 0, 0, 0, 1413, 0, 0, 3279, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 4096, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 615, 0, 0, 4096, 0, 0, 2499, 816, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4093, 0, 0, 2790, 1309, 0, 0, 0, 0, 0, 0, 0, 1137, 2, 0, 1121, 0, 0, 0, 4096, 1306, 0, 0, 0, 0, 0, 0, 211, 0, 3873, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8788, 8260, 8026, 8198, 7662, 8180, 8192] avg=8192.0000 max=8788 max_rank=1 min=7662 min_rank=5 imbalance_ratio=1.072754 max_load_reduction=56748 max_load_reduction_pct=0.865906 moved_to_echo=32232 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 22, 0, 3, 4073, 2673, 1386, 0, 0, 0, 0, 0, 0, 868, 0, 0, 3276, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 4096, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 0, 0, 4096, 0, 0, 2499, 816, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4093, 0, 0, 2790, 1309, 0, 0, 0, 0, 0, 0, 0, 1829, 2, 0, 768, 0, 0, 0, 4096, 1305, 0, 0, 0, 0, 0, 0, 211, 0, 4037, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8240, 8258, 8141, 8200, 8000, 8344, 8196] avg=8192.0000 max=8344 max_rank=6 min=8000 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32740 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 37, 0, 0, 4087, 2673, 1386, 0, 0, 0, 0, 0, 0, 683, 0, 0, 3279, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 4096, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 0, 0, 4096, 0, 0, 2499, 817, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4090, 0, 0, 2790, 1309, 0, 0, 0, 0, 0, 0, 0, 2125, 0, 0, 618, 0, 0, 0, 4096, 1306, 0, 0, 0, 0, 0, 0, 211, 7, 3907, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8058, 8208, 8339, 8190, 8145, 8221, 8192] avg=8192.0000 max=8339 max_rank=3 min=8058 min_rank=1 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 45, 0, 0, 4076, 2673, 1386, 0, 0, 0, 0, 0, 0, 444, 0, 0, 3279, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 4096, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 0, 0, 4096, 0, 0, 2499, 817, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 2790, 1309, 0, 0, 0, 0, 0, 0, 0, 3170, 0, 0, 40, 0, 0, 0, 4096, 1306, 0, 0, 0, 0, 0, 0, 211, 0, 3445, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 7819, 8221, 8555, 8205, 8612, 7752, 8192] avg=8192.0000 max=8612 max_rank=5 min=7752 min_rank=6 imbalance_ratio=1.051270 max_load_reduction=56924 max_load_reduction_pct=0.868591 moved_to_echo=32761 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 25, 0, 1, 4054, 2673, 1386, 0, 0, 0, 0, 0, 0, 1332, 0, 0, 3278, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 0, 4096, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 0, 4096, 0, 0, 2499, 816, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 2790, 1309, 0, 0, 0, 0, 0, 0, 0, 2162, 0, 1, 211, 0, 0, 0, 4096, 1304, 0, 0, 0, 0, 0, 0, 211, 4, 4021, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8706, 8191, 8006, 8195, 7774, 8332, 8193] avg=8192.0000 max=8706 max_rank=1 min=7774 min_rank=5 imbalance_ratio=1.062744 max_load_reduction=56830 max_load_reduction_pct=0.867157 moved_to_echo=32494 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 22, 6, 26, 4, 0, 4091, 2670, 1389, 0, 0, 0, 0, 0, 0, 1061, 9, 0, 3271, 4, 1, 4080, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 6, 3904, 0, 1, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 775, 3, 6, 4089, 6, 0, 2497, 819, 4, 0, 0, 0, 0, 0, 5, 3, 0, 6, 6, 4048, 0, 0, 2785, 1310, 0, 0, 0, 0, 0, 0, 7, 2607, 0, 0, 21, 0, 5, 6, 4087, 1303, 0, 0, 0, 0, 0, 0, 197, 0, 4058, 5, 0, 4081, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 4, 0, 4084, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8432, 7957, 8202, 8163, 8036, 8343, 8187] avg=8192.0000 max=8432 max_rank=1 min=7957 min_rank=2 imbalance_ratio=1.029297 max_load_reduction=57104 max_load_reduction_pct=0.871338 moved_to_echo=32729 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6656, 128, 0, 26240, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 18560, 128, 128, 3840, 0, 128, 128, 32768, 10496, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 512, 128, 128, 32640, 21504, 11136, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1792, 128, 31104, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 0, 22400, 10496, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32128, 0, 128, 32640, 128, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 6272, 128, 128, 32768, 128, 0, 20096, 6656, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 3, 0, 4096, 4095, 0, 0, 4096, 0, 0, 749, 4096, 4096, 35, 4076, 1132, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 150, 2043, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4090, 0, 0, 4, 141, 0, 4096, 4096, 0, 0, 4095, 0, 0, 715, 4096, 4096, 118, 4093, 2356, 0, 4096, 4088, 0, 4092, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 9, 769, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 80, 0, 4096, 4096, 0, 0, 4089, 0, 0, 398, 4096, 4096, 13, 3970, 629, 0, 4096, 4095, 0, 4081, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 4096, 4096, 0, 218, 2892, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4096, 0, 4090, 0, 0, 0, 14, 0, 4096, 4096, 0, 0, 4094, 0, 0, 617, 4095, 4096, 38, 4078, 617, 0, 4096, 4090, 0, 4086, 0, 0, 1, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 2, 2826, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 0, 19, 40, 0, 4096, 4096, 0, 0, 4083, 0, 0, 957, 4095, 4096, 107, 4088, 2116, 0, 4096, 4085, 0, 4068, 0, 7, 0, 0, 0, 0, 4, 0, 0, 13, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 6, 882, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4091, 0, 0, 52, 40, 0, 4096, 4096, 0, 0, 4088, 0, 0, 976, 4096, 4096, 36, 4070, 1411, 0, 4096, 4088, 0, 4086, 0, 8, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 56, 1565, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 0, 0, 3, 13, 0, 4095, 4096, 0, 0, 4093, 0, 0, 1491, 4096, 4096, 36, 4080, 792, 0, 4096, 4096, 0, 4093, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 13, 1775, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4, 0, 0, 4089, 0, 6, 0, 4088, 7, 4091, 6, 6, 3, 0, 6, 4016, 4081, 0, 5, 4090, 5, 0, 1490, 4089, 3985, 13, 3964, 190, 6, 4089, 4090, 0, 4090, 25, 11, 0, 7, 0, 0, 0, 0, 7, 8, 0, 6, 5, 0, 4, 4087, 0, 7, 0, 0, 6, 0, 5, 0, 4087, 4090, 1, 447, 2224, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 4, 141, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3178, 0, 0, 4095, 0, 0, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2944, 118, 4093, 2356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 5, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1983, 1152, 918, 38, 1, 0, 0, 0, 0, 4096, 2113, 0, 9, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8293, 7988, 9511, 8189, 8188, 8188, 6987] avg=8192.0000 max=9511 max_rank=3 min=6987 min_rank=7 imbalance_ratio=1.161011 max_load_reduction=56025 max_load_reduction_pct=0.854874 moved_to_echo=31552 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 4, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3177, 0, 0, 4096, 0, 0, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2944, 35, 4076, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1983, 1152, 918, 38, 0, 0, 0, 0, 0, 4096, 2113, 0, 150, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8161, 8022, 8187, 8197, 8188, 8187, 8402] avg=8192.0000 max=8402 max_rank=7 min=8022 min_rank=2 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32974 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 52, 40, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3178, 0, 0, 4088, 0, 0, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2944, 36, 4070, 1411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 8, 0, 0, 0, 0, 0, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 1983, 1152, 918, 38, 1, 0, 0, 0, 0, 4096, 2113, 0, 56, 1565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8241, 8242, 8461, 8186, 8189, 8195, 7830] avg=8192.0000 max=8461 max_rank=3 min=7830 min_rank=7 imbalance_ratio=1.032837 max_load_reduction=57075 max_load_reduction_pct=0.870895 moved_to_echo=32400 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 80, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3178, 0, 0, 4089, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2944, 13, 3970, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 7, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 1983, 1152, 918, 38, 1, 0, 0, 0, 0, 4096, 2113, 0, 218, 2892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8226, 7665, 7556, 8187, 8188, 8203, 9319] avg=8192.0000 max=9319 max_rank=7 min=7556 min_rank=3 imbalance_ratio=1.137573 max_load_reduction=56217 max_load_reduction_pct=0.857803 moved_to_echo=33897 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 3, 13, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3178, 0, 0, 4093, 0, 0, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2944, 36, 4080, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 1, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1983, 1152, 918, 38, 1, 0, 0, 0, 0, 4096, 2113, 0, 13, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8161, 8762, 7852, 8197, 8187, 8188, 7997] avg=8192.0000 max=8762 max_rank=2 min=7852 min_rank=3 imbalance_ratio=1.069580 max_load_reduction=56774 max_load_reduction_pct=0.866302 moved_to_echo=32569 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3178, 0, 0, 4094, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2944, 38, 4078, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 1, 0, 0, 0, 0, 4089, 5, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 1983, 1152, 918, 38, 1, 0, 0, 0, 0, 4096, 2113, 0, 2, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8162, 7889, 7677, 8181, 8196, 8201, 9037] avg=8192.0000 max=9037 max_rank=7 min=7677 min_rank=3 imbalance_ratio=1.103149 max_load_reduction=56499 max_load_reduction_pct=0.862106 moved_to_echo=33615 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 19, 40, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3178, 0, 0, 4083, 0, 0, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2944, 107, 4088, 2116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 7, 0, 0, 0, 0, 4, 4085, 6, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 1983, 1152, 918, 38, 0, 0, 0, 0, 0, 4096, 2113, 0, 6, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8206, 8218, 9255, 8170, 8198, 8200, 7097] avg=8192.0000 max=9255 max_rank=3 min=7097 min_rank=7 imbalance_ratio=1.129761 max_load_reduction=56281 max_load_reduction_pct=0.858780 moved_to_echo=31665 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 4089, 0, 6, 0, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 7, 4042, 6, 6, 3, 0, 6, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 3160, 0, 5, 4090, 5, 0, 1490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2864, 13, 3964, 190, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 25, 11, 0, 7, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 7, 8, 0, 6, 5, 0, 4, 4089, 4083, 0, 0, 0, 0, 0, 0, 4087, 0, 7, 0, 0, 6, 0, 5, 1982, 1121, 921, 45, 0, 0, 0, 0, 0, 4087, 2108, 1, 447, 2224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8086, 8750, 7037, 8229, 8202, 8174, 8867] avg=8192.0000 max=8867 max_rank=7 min=7037 min_rank=3 imbalance_ratio=1.082397 max_load_reduction=56669 max_load_reduction_pct=0.864700 moved_to_echo=33472 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 17024, 128, 1024, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32512, 128, 128, 128, 384, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23552, 512, 32512, 9344, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 0, 128, 0, 128, 128, 0, 128, 15872, 9216, 7424, 384, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[25472, 0, 128, 32768, 128, 0, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2295, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4080, 0, 0, 4084, 0, 0, 1, 0, 1, 4096, 0, 4096, 0, 1393, 4095, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 435, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 2026, 4096, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 3983, 0, 0, 4021, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 1962, 4096, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 314, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2174, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4056, 0, 0, 4081, 0, 0, 0, 0, 8, 4096, 0, 4094, 0, 848, 4093, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 1128, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 1860, 4096, 0, 0, 0, 0, 3, 0, 9, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 3980, 0, 0, 4032, 0, 0, 0, 0, 0, 4096, 0, 4071, 0, 2030, 4095, 0, 0, 0, 4095, 0, 0, 4094, 4096, 0, 412, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 1271, 4096, 0, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 3979, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 4081, 0, 2407, 4090, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 568, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1966, 4096, 0, 0, 0, 0, 5, 0, 10, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4029, 0, 0, 3954, 0, 0, 0, 0, 0, 4096, 0, 4075, 0, 1862, 4095, 0, 0, 0, 4096, 0, 0, 4091, 4096, 0, 501, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1942, 4096, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4065, 0, 0, 4092, 0, 0, 0, 0, 1, 4096, 0, 4095, 0, 1561, 4092, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 632, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4086, 0, 0, 0, 19, 26, 0, 80, 4053, 0, 7, 6, 0, 0, 7, 0, 9, 12, 3, 6, 5, 141, 4089, 6, 6, 5, 42, 4088, 22, 5, 4089, 0, 2, 4071, 4045, 4, 7, 4088, 127, 56, 1, 0, 0, 4078, 5, 4088, 7, 3361, 4090, 6, 0, 0, 4090, 8, 6, 4089, 4021, 0, 387, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 4026, 58, 9, 0, 0, 0, 0, 0, 0, 0, 2026, 4096, 0, 0, 0, 0, 1858, 537, 0, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4096, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 1962, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3557, 4096, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8517, 8193, 8165, 8149, 8136, 8219, 7967] avg=8192.0000 max=8517 max_rank=1 min=7967 min_rank=7 imbalance_ratio=1.039673 max_load_reduction=57019 max_load_reduction_pct=0.870041 moved_to_echo=32471 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4026, 58, 10, 0, 0, 0, 0, 0, 0, 0, 2295, 4096, 0, 0, 0, 0, 1857, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 1, 0, 1, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1393, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3559, 4095, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8785, 8182, 8174, 8246, 8201, 7669, 8089] avg=8192.0000 max=8785 max_rank=1 min=7669 min_rank=6 imbalance_ratio=1.072388 max_load_reduction=56751 max_load_reduction_pct=0.865952 moved_to_echo=32205 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 0, 0, 4026, 58, 10, 0, 0, 0, 0, 0, 0, 0, 2174, 4096, 0, 0, 0, 0, 1856, 537, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4096, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 8, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 848, 2179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3559, 4096, 0, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8663, 8183, 8166, 8222, 8204, 7121, 8783] avg=8192.0000 max=8783 max_rank=7 min=7121 min_rank=6 imbalance_ratio=1.072144 max_load_reduction=56753 max_load_reduction_pct=0.865982 moved_to_echo=32330 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4026, 58, 10, 0, 0, 0, 0, 0, 0, 0, 1942, 4096, 0, 0, 0, 0, 1856, 537, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 1, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1561, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3557, 4096, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8431, 8187, 8170, 8231, 8208, 7834, 8285] avg=8192.0000 max=8431 max_rank=1 min=7834 min_rank=6 imbalance_ratio=1.029175 max_load_reduction=57105 max_load_reduction_pct=0.871353 moved_to_echo=32558 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 4026, 58, 10, 0, 0, 0, 0, 0, 0, 0, 1860, 4096, 0, 0, 0, 0, 1857, 537, 0, 0, 0, 0, 0, 0, 3, 0, 9, 0, 0, 0, 0, 0, 4095, 4086, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4095, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 2030, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3557, 4096, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8350, 8193, 8164, 8145, 8147, 8281, 8065] avg=8192.0000 max=8350 max_rank=1 min=8065 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32638 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4026, 58, 10, 0, 0, 0, 0, 0, 0, 0, 1966, 4096, 0, 0, 0, 0, 1857, 536, 0, 0, 0, 0, 0, 0, 5, 0, 10, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4096, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 1862, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3555, 4096, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8455, 8197, 8161, 8195, 8069, 8117, 8152] avg=8192.0000 max=8455 max_rank=1 min=8069 min_rank=5 imbalance_ratio=1.032104 max_load_reduction=57081 max_load_reduction_pct=0.870987 moved_to_echo=32533 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4096, 0, 0, 1, 0, 4026, 58, 10, 0, 0, 0, 0, 0, 0, 0, 1271, 4096, 0, 0, 0, 0, 1855, 537, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4096, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 2407, 2177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 4096, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7759, 8192, 8169, 8145, 8191, 8665, 8222] avg=8192.0000 max=8665 max_rank=6 min=7759 min_rank=1 imbalance_ratio=1.057739 max_load_reduction=56871 max_load_reduction_pct=0.867783 moved_to_echo=33223 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4086, 0, 0, 0, 19, 4020, 63, 10, 0, 0, 0, 0, 0, 26, 0, 80, 4053, 0, 7, 6, 0, 1859, 538, 7, 0, 0, 0, 0, 0, 0, 7, 0, 9, 12, 3, 6, 5, 4090, 4077, 0, 0, 0, 0, 0, 0, 141, 4089, 6, 6, 5, 42, 4056, 22, 0, 0, 0, 0, 0, 0, 0, 0, 5, 69, 0, 2, 4071, 4045, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 127, 56, 1, 0, 0, 4078, 5, 25, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 3361, 2168, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 3551, 4021, 0, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 6576, 8209, 8367, 8203, 8380, 9630, 7973] avg=8192.0000 max=9630 max_rank=6 min=6576 min_rank=1 imbalance_ratio=1.175537 max_load_reduction=55906 max_load_reduction_pct=0.853058 moved_to_echo=34186 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 128, 128, 32256, 512, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 640, 0, 128, 32768, 32256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 15488, 17536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 28544, 32768, 0, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 13696, 32768, 0, 128, 128, 0, 14976, 4352, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32512, 128, 128, 128, 0, 128, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 32768, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 164, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3607, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 2, 4092, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 24, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4070, 4094, 0, 4, 0, 0, 0, 0, 4096, 10, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 90, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4003, 4092, 0, 4, 0, 0, 0, 0, 4096, 15, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 33, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4050, 4096, 0, 0, 0, 0, 0, 0, 4096, 6, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 20, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4076, 4095, 0, 1, 0, 0, 0, 0, 4096, 6, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4091, 4096, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 0, 48, 0, 0, 0, 4096, 0, 4077, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4045, 4091, 0, 3, 0, 0, 0, 0, 4096, 19, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4092, 4096, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 112, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3979, 4095, 0, 4, 0, 0, 0, 0, 4096, 8, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4089, 0, 4090, 4074, 5, 4, 13, 6, 0, 4089, 0, 6, 0, 50, 9, 4037, 4081, 4, 7, 7, 8, 0, 6, 208, 0, 6, 7, 0, 5, 0, 4063, 4089, 0, 5, 1486, 34, 6, 0, 4080, 0, 4062, 0, 6, 1, 0, 6, 0, 4090, 0, 17, 2420, 4090, 0, 0, 4, 0, 0, 5, 4086, 0, 4088, 0, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 13, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 4096, 0, 3774, 71, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4090, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 4096, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 8134, 8193, 7974, 8198, 8458, 8192] avg=8192.0000 max=8458 max_rank=6 min=7974 min_rank=4 imbalance_ratio=1.032471 max_load_reduction=57078 max_load_reduction_pct=0.870941 moved_to_echo=32822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 325, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, 4096, 0, 3774, 71, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3607, 4096, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8446, 8193, 8105, 8206, 8017, 8186] avg=8192.0000 max=8446 max_rank=2 min=8017 min_rank=6 imbalance_ratio=1.031006 max_load_reduction=57090 max_load_reduction_pct=0.871124 moved_to_echo=32514 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 2, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4096, 0, 3774, 72, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4093, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4094, 0, 4, 0, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8122, 8195, 7966, 8192, 8480, 8196] avg=8192.0000 max=8480 max_rank=6 min=7966 min_rank=4 imbalance_ratio=1.035156 max_load_reduction=57056 max_load_reduction_pct=0.870605 moved_to_echo=32834 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 3774, 72, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4095, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4095, 0, 1, 0, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8191, 8122, 8194, 7962, 8198, 8484, 8192] avg=8192.0000 max=8484 max_rank=6 min=7962 min_rank=4 imbalance_ratio=1.035645 max_load_reduction=57052 max_load_reduction_pct=0.870544 moved_to_echo=32836 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 4096, 0, 3774, 71, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4089, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 4092, 0, 4, 0, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8196, 8123, 8194, 8031, 8185, 8411, 8201] avg=8192.0000 max=8411 max_rank=6 min=8031 min_rank=4 imbalance_ratio=1.026733 max_load_reduction=57125 max_load_reduction_pct=0.871658 moved_to_echo=32828 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 4096, 0, 3777, 71, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 4094, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 4091, 0, 3, 0, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8195, 8124, 8198, 7992, 8184, 8451, 8205] avg=8192.0000 max=8451 max_rank=6 min=7992 min_rank=4 imbalance_ratio=1.031616 max_load_reduction=57085 max_load_reduction_pct=0.871048 moved_to_echo=32832 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 4096, 0, 3774, 71, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4093, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 4095, 0, 4, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8192, 8127, 8194, 8053, 8194, 8392, 8195] avg=8192.0000 max=8392 max_rank=6 min=8053 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=57144 max_load_reduction_pct=0.871948 moved_to_echo=32834 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 4074, 5, 4, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 6, 0, 50, 9, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4, 7, 7, 8, 0, 6, 208, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 5, 0, 4063, 4089, 4, 1, 0, 0, 0, 0, 0, 0, 0, 5, 1486, 34, 6, 0, 4080, 0, 3771, 71, 0, 0, 0, 0, 0, 0, 4062, 0, 6, 1, 0, 6, 0, 4090, 6, 8, 0, 0, 0, 0, 0, 0, 0, 17, 2420, 4090, 0, 0, 4, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4086, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8187, 8338, 8175, 9453, 8179, 6843, 8169] avg=8192.0000 max=9453 max_rank=4 min=6843 min_rank=6 imbalance_ratio=1.153931 max_load_reduction=56083 max_load_reduction_pct=0.855759 moved_to_echo=32644 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 128, 640, 32256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 30336, 32768, 0, 128, 128, 0, 2560, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 2048, 128, 128, 0, 32768, 0, 30208, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4093, 4088, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 5, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 4080, 1, 0, 0, 0, 0, 0, 4096, 3942, 0, 0, 0, 0, 11, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 159] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 4072, 0, 0, 0, 0, 0, 0, 4096, 1952, 0, 0, 0, 0, 9, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 2182] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 3938, 0, 0, 0, 0, 0, 0, 4096, 3680, 0, 0, 0, 0, 9, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 566] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 3987, 0, 0, 0, 0, 0, 0, 4096, 3661, 0, 0, 1, 0, 3, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 563] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 4044, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 82] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3, 4096, 0, 4096, 4096, 4074, 0, 0, 0, 0, 0, 0, 4096, 3891, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 230] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 3874, 0, 0, 1, 0, 0, 0, 4096, 2489, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 1895] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4013, 0, 44, 7, 7, 0, 4089, 4091, 282, 8, 5, 0, 0, 0, 6, 6, 0, 0, 0, 27, 6, 120, 0, 4089, 0, 5, 4064, 4085, 0, 65, 4060, 24, 4088, 4089, 3798, 0, 0, 0, 0, 7, 8, 4087, 2655, 28, 0, 1, 0, 0, 4084, 4088, 0, 62, 6, 4065, 5, 0, 0, 7, 8, 2319, 9, 7, 0, 3012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4041, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 114, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3891, 0, 0, 0, 0, 0, 786, 5, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 230, 3196, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8156, 8174, 8190, 8284, 8781, 8189, 7561] avg=8192.0000 max=8781 max_rank=5 min=7561 min_rank=7 imbalance_ratio=1.071899 max_load_reduction=56755 max_load_reduction_pct=0.866013 moved_to_echo=32815 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 9, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4041, 38, 0, 0, 0, 0, 0, 0, 4, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 114, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3680, 0, 0, 0, 0, 9, 786, 5, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 566, 3196, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8154, 8175, 8191, 8148, 8579, 8189, 7903] avg=8192.0000 max=8579 max_rank=5 min=7903 min_rank=7 imbalance_ratio=1.047241 max_load_reduction=56957 max_load_reduction_pct=0.869095 moved_to_echo=32819 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4041, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 114, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 786, 5, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 82, 3196, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8156, 8174, 8187, 8254, 8955, 8189, 7420] avg=8192.0000 max=8955 max_rank=5 min=7420 min_rank=7 imbalance_ratio=1.093140 max_load_reduction=56581 max_load_reduction_pct=0.863358 moved_to_echo=32818 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 9, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4040, 37, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 114, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3661, 0, 0, 1, 0, 3, 786, 5, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 563, 3196, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8156, 8173, 8188, 8197, 8555, 8189, 7880] avg=8192.0000 max=8555 max_rank=5 min=7880 min_rank=7 imbalance_ratio=1.044312 max_load_reduction=56981 max_load_reduction_pct=0.869461 moved_to_echo=32821 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 5, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 4096, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4039, 37, 0, 0, 0, 0, 0, 0, 5, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 114, 4080, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3942, 0, 0, 0, 0, 11, 786, 5, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 159, 3196, 49, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8159, 8173, 8192, 8291, 8843, 8189, 7500] avg=8192.0000 max=8843 max_rank=5 min=7500 min_rank=7 imbalance_ratio=1.079468 max_load_reduction=56693 max_load_reduction_pct=0.865067 moved_to_echo=32823 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 9, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4040, 37, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 114, 3874, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2489, 0, 0, 0, 0, 4, 786, 5, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 1895, 3196, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8157, 8173, 8188, 8085, 7383, 8189, 9165] avg=8192.0000 max=9165 max_rank=7 min=7383 min_rank=5 imbalance_ratio=1.118774 max_load_reduction=56371 max_load_reduction_pct=0.860153 moved_to_echo=32822 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 9, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4041, 37, 0, 0, 0, 0, 0, 0, 5, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 114, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1952, 0, 0, 0, 0, 9, 786, 5, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 2182, 3196, 46, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8155, 8174, 8192, 8282, 6851, 8189, 9501] avg=8192.0000 max=9501 max_rank=7 min=6851 min_rank=5 imbalance_ratio=1.159790 max_load_reduction=56035 max_load_reduction_pct=0.855026 moved_to_echo=32823 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4013, 0, 44, 7, 7, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 282, 8, 5, 0, 0, 0, 6, 6, 4088, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 6, 120, 0, 4089, 4041, 37, 0, 0, 0, 0, 0, 0, 0, 5, 4054, 0, 0, 65, 4060, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 102, 3798, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4087, 2655, 28, 0, 1, 0, 0, 791, 10, 9, 0, 0, 0, 0, 0, 4075, 0, 0, 62, 6, 4065, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 2319, 9, 7, 0, 3012, 3196, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8443, 8320, 8208, 7995, 7589, 8213, 8606] avg=8192.0000 max=8606 max_rank=7 min=7589 min_rank=5 imbalance_ratio=1.050537 max_load_reduction=56930 max_load_reduction_pct=0.868683 moved_to_echo=32403 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 26368, 128, 0, 128, 0, 128, 6400, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 30976, 128, 128, 0, 8704, 25600, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 128, 128, 0, 0, 128, 128, 128, 32768, 32512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 1024, 31872, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 32768, 32384, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 86, 4090, 4096, 4096, 0, 0, 0, 3396, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 339, 0, 3877, 0, 489, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 71, 4096, 4096, 4096, 0, 0, 0, 2953, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 715, 0, 3796, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 85, 4092, 4096, 4096, 0, 0, 0, 3186, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 190, 0, 3724, 0, 1009, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 243, 4091, 4096, 4096, 0, 0, 0, 1761, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 2, 4096, 584, 0, 3698, 0, 1904, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4092, 219, 4089, 4096, 4096, 0, 0, 0, 2298, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 3, 4096, 845, 0, 3902, 0, 925, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 127, 4089, 4096, 4096, 0, 0, 0, 2067, 0, 0, 4096, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 1, 4095, 391, 0, 3762, 0, 1846, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4066, 0, 78, 7, 3658, 5, 4, 0, 10, 0, 3, 0, 0, 4090, 535, 4036, 4046, 4072, 0, 0, 0, 4007, 0, 7, 4088, 0, 0, 6, 17, 4083, 4087, 0, 8, 0, 11, 5, 0, 0, 0, 0, 4089, 4028, 0, 4088, 101, 7, 4004, 90, 37, 0, 10, 0, 6, 7, 0, 0, 5, 4089, 10, 4008, 0, 22, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 4085, 4096, 4096, 0, 0, 0, 2908, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 612, 0, 3861, 0, 776, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 71, 2807, 968, 148, 0, 0, 0, 0, 0, 4096, 1289, 0, 0, 0, 0, 2953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 715, 0, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 4096, 3128, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8090, 8338, 8191, 8190, 8410, 7883, 8196] avg=8192.0000 max=8410 max_rank=5 min=7883 min_rank=6 imbalance_ratio=1.026611 max_load_reduction=57126 max_load_reduction_pct=0.871674 moved_to_echo=32679 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4094, 243, 2807, 967, 148, 0, 0, 0, 0, 0, 4091, 1289, 0, 0, 0, 0, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 2, 0, 584, 0, 3698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1904, 0, 0, 0, 0, 0, 0, 4096, 3129, 2, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8261, 7141, 8191, 8191, 8183, 9131, 8200] avg=8192.0000 max=9131 max_rank=6 min=7141 min_rank=2 imbalance_ratio=1.114624 max_load_reduction=56405 max_load_reduction_pct=0.860672 moved_to_echo=33705 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 85, 2807, 967, 148, 0, 0, 0, 0, 0, 4092, 1289, 0, 0, 0, 0, 3186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 1, 0, 190, 0, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 0, 0, 0, 0, 0, 0, 4096, 3129, 2, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8102, 8567, 8191, 8190, 7814, 8236, 8198] avg=8192.0000 max=8567 max_rank=2 min=7814 min_rank=5 imbalance_ratio=1.045776 max_load_reduction=56969 max_load_reduction_pct=0.869278 moved_to_echo=32438 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 86, 2807, 968, 148, 0, 0, 0, 0, 0, 4090, 1289, 0, 0, 0, 0, 3396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 339, 0, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 0, 4096, 3128, 2, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8111, 8775, 8191, 8190, 8115, 7715, 8201] avg=8192.0000 max=8775 max_rank=2 min=7715 min_rank=6 imbalance_ratio=1.071167 max_load_reduction=56761 max_load_reduction_pct=0.866104 moved_to_echo=32221 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 4096, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 127, 2807, 968, 148, 0, 0, 0, 0, 0, 4089, 1289, 0, 0, 0, 0, 2067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 1, 0, 391, 0, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1846, 0, 0, 0, 0, 0, 0, 4095, 3128, 2, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8147, 7445, 8193, 8191, 8053, 9071, 8197] avg=8192.0000 max=9071 max_rank=6 min=7445 min_rank=2 imbalance_ratio=1.107300 max_load_reduction=56465 max_load_reduction_pct=0.861588 moved_to_echo=33512 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4092, 219, 2807, 968, 148, 0, 0, 0, 0, 0, 4089, 1289, 0, 0, 0, 0, 2298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 3, 0, 845, 0, 3902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 925, 0, 0, 0, 0, 0, 0, 4096, 3128, 1, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8236, 7676, 8191, 8194, 8649, 8150, 8202] avg=8192.0000 max=8649 max_rank=5 min=7676 min_rank=2 imbalance_ratio=1.055786 max_load_reduction=56887 max_load_reduction_pct=0.868027 moved_to_echo=33195 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4066, 0, 78, 7, 3658, 5, 4, 51, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3, 0, 0, 4090, 535, 2777, 963, 146, 0, 0, 0, 0, 0, 4036, 1269, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 6, 17, 4083, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 11, 5, 0, 0, 0, 4089, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 0, 0, 101, 7, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 90, 37, 0, 10, 0, 6, 7, 0, 4088, 3109, 1, 0, 0, 0, 0, 0, 0, 5, 4089, 10, 4008, 0, 22, 6, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7869, 8524, 9312, 8197, 8200, 7940, 7348, 8146] avg=8192.0000 max=9312 max_rank=2 min=7348 min_rank=6 imbalance_ratio=1.136719 max_load_reduction=56224 max_load_reduction_pct=0.857910 moved_to_echo=31634 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 2807, 968, 148, 0, 0, 0, 0, 0, 4085, 1289, 0, 0, 0, 0, 2908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 612, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 0, 0, 0, 0, 0, 0, 4096, 3128, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8065, 8282, 8191, 8190, 8372, 8002, 8196] avg=8192.0000 max=8372 max_rank=5 min=8002 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32760 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 128, 128, 32384, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 31232, 128, 0, 3840, 128, 30720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 7680, 0, 128, 0, 128, 128, 0, 32768, 25088, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 1536, 22528, 7808, 1280, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 10368, 0, 0, 0, 0, 22656, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4068, 0, 15, 0, 0, 0, 0, 0, 0, 4096, 0, 3546, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4040, 4096, 619, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4046, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 3931, 0, 3, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 4093, 197, 0, 4, 4096, 0, 1, 4095, 4096, 4, 0, 4092, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4002, 0, 19, 0, 0, 0, 1, 0, 0, 4096, 0, 3674, 0, 5, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 4091, 506, 0, 5, 4096, 0, 3, 4094, 4096, 6, 0, 4093, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3912, 0, 5, 0, 0, 0, 0, 0, 0, 4095, 0, 3430, 0, 1, 4096, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4044, 4095, 895, 0, 4, 4096, 0, 1, 4094, 4096, 3, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 3930, 0, 8, 0, 0, 0, 5, 0, 0, 4096, 0, 3199, 0, 1, 4096, 0, 0, 0, 4095, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 3705, 4094, 1443, 0, 2, 4096, 0, 2, 4096, 4096, 5, 0, 4094, 0, 0, 0, 4095, 4096, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4005, 0, 23, 0, 0, 0, 0, 0, 0, 4096, 0, 3422, 0, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3930, 4091, 907, 0, 5, 4096, 0, 2, 4096, 4096, 5, 0, 4089, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[7, 0, 7, 492, 5, 0, 9, 0, 3686, 0, 8, 0, 0, 4087, 0, 0, 18, 0, 6, 60, 6, 7, 4033, 12, 3729, 9, 0, 4065, 0, 0, 6, 4089, 4089, 0, 0, 4050, 0, 4, 0, 43, 26, 4064, 4082, 301, 7, 0, 4070, 0, 0, 4081, 4066, 0, 0, 4089, 7, 0, 8, 4072, 4086, 0, 0, 34, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3341, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3951, 4096, 988, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4036, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4068, 0, 15, 127, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3533, 500, 57, 0, 0, 0, 0, 0, 3546, 0, 0, 4096, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 3469, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8307, 8186, 8205, 8193, 8128, 8195, 8190] avg=8192.0000 max=8307 max_rank=1 min=8128 min_rank=5 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32706 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4035, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4046, 0, 18, 127, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3533, 500, 57, 0, 0, 0, 0, 0, 3931, 0, 3, 4096, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3466, 197, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4096, 4, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8286, 8186, 8593, 8193, 7761, 8196, 8190] avg=8192.0000 max=8593 max_rank=3 min=7761 min_rank=5 imbalance_ratio=1.048950 max_load_reduction=56943 max_load_reduction_pct=0.868881 moved_to_echo=32340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4034, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3912, 0, 5, 127, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3533, 500, 57, 0, 0, 0, 0, 0, 3430, 0, 1, 4096, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 3468, 895, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4096, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8141, 8185, 8090, 8192, 8411, 8197, 8190] avg=8192.0000 max=8411 max_rank=5 min=8090 min_rank=3 imbalance_ratio=1.026733 max_load_reduction=57125 max_load_reduction_pct=0.871658 moved_to_echo=32990 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4036, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4005, 0, 23, 127, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3533, 500, 57, 0, 0, 0, 0, 0, 3422, 0, 1, 4096, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 3464, 907, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4096, 5, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8252, 8186, 8082, 8193, 8306, 8195, 8190] avg=8192.0000 max=8306 max_rank=5 min=8082 min_rank=3 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32884 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4036, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4002, 0, 19, 127, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 3533, 500, 58, 0, 0, 0, 0, 0, 3674, 0, 5, 4096, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 3464, 506, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 6, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8239, 8188, 8338, 8193, 8060, 8198, 8188] avg=8192.0000 max=8338 max_rank=3 min=8060 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32639 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4036, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 3930, 0, 8, 127, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 3532, 500, 57, 0, 0, 0, 0, 0, 3199, 0, 1, 4096, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3705, 3467, 1443, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4096, 5, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8157, 8190, 7859, 8191, 8617, 8200, 8190] avg=8192.0000 max=8617 max_rank=5 min=7859 min_rank=3 imbalance_ratio=1.051880 max_load_reduction=56919 max_load_reduction_pct=0.868515 moved_to_echo=33198 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4036, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4010, 0, 0, 127, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3533, 500, 57, 0, 0, 0, 0, 0, 3341, 0, 0, 4096, 0, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 3469, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8234, 8186, 8000, 8193, 8408, 8193, 8190] avg=8192.0000 max=8408 max_rank=5 min=8000 min_rank=3 imbalance_ratio=1.026367 max_load_reduction=57128 max_load_reduction_pct=0.871704 moved_to_echo=32984 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 7, 492, 5, 0, 9, 0, 4070, 4025, 0, 0, 0, 0, 0, 0, 3686, 0, 8, 0, 0, 4087, 0, 0, 133, 6, 0, 0, 0, 0, 0, 0, 18, 0, 6, 60, 6, 7, 4033, 12, 3529, 502, 56, 0, 0, 0, 0, 0, 3729, 9, 0, 4065, 0, 0, 6, 560, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4050, 0, 4, 0, 43, 2, 0, 0, 0, 0, 0, 0, 0, 26, 4064, 3447, 301, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4072, 4078, 0, 0, 34, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8615, 7920, 8229, 8369, 8188, 7845, 8162, 8208] avg=8192.0000 max=8615 max_rank=0 min=7845 min_rank=5 imbalance_ratio=1.051636 max_load_reduction=56921 max_load_reduction_pct=0.868546 moved_to_echo=32403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 512, 128, 0, 128, 0, 32768, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32000, 27776, 5888, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 32768, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32384, 0, 128, 0, 0, 32128, 0, 128, 1024, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[28288, 128, 128, 32768, 0, 0, 128, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 128, 28288, 4096, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 278, 4096, 4094, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 2860, 0, 0, 307, 4072, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4043, 0, 0, 0, 4080, 0, 721, 16, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 1416, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2110, 0, 0, 21, 4087, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 4070, 0, 546, 31, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 588, 4096, 4086, 0, 0, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 2907, 0, 0, 5, 4080, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4073, 0, 0, 0, 4008, 0, 693, 17, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 426, 4096, 4090, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 2952, 0, 0, 106, 4078, 0, 0, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 4077, 0, 0, 0, 4076, 0, 649, 18, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 4096, 0, 0, 0, 0, 658, 4095, 4091, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 2748, 0, 0, 73, 4092, 0, 0, 0, 0, 2, 4096, 0, 4096, 4096, 0, 0, 4027, 0, 0, 0, 4068, 0, 686, 14, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4087, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 4096, 0, 0, 0, 0, 801, 4095, 4087, 0, 0, 0, 0, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 2855, 0, 0, 156, 4056, 0, 0, 0, 0, 7, 4096, 0, 4096, 4096, 0, 0, 4023, 0, 0, 0, 4056, 0, 401, 11, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 17, 0, 4096, 1, 0, 0, 0, 473, 4094, 4091, 1, 0, 1, 0, 6, 0, 4094, 0, 0, 0, 0, 0, 0, 1903, 0, 0, 119, 3979, 0, 0, 0, 0, 3, 4096, 0, 4095, 4096, 0, 0, 4069, 0, 0, 0, 4055, 0, 1739, 30, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4076, 0, 0, 0, 4089, 4085, 4030, 0, 5, 9, 0, 4090, 13, 0, 0, 6, 4087, 0, 408, 0, 2, 1826, 4089, 4089, 0, 7, 0, 53, 0, 7, 4088, 7, 0, 0, 7, 16, 21, 8, 5, 8, 4, 2396, 4, 7, 6, 1, 0, 4089, 7, 4084, 4080, 0, 7, 4088, 9, 0, 6, 4090, 0, 1545, 1977, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2298, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 3244, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 2298, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2860, 0, 0, 4096, 1798, 0, 0, 0, 0, 0, 0, 307, 4072, 0, 0, 0, 0, 0, 4096, 139, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4096, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 721, 16, 0, 0, 0, 2278, 713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8180, 7616, 8187, 8754, 8614, 8173, 7808] avg=8192.0000 max=8754 max_rank=4 min=7616 min_rank=2 imbalance_ratio=1.068604 max_load_reduction=56782 max_load_reduction_pct=0.866425 moved_to_echo=33349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 2298, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 3243, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 2298, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2748, 0, 0, 4096, 1798, 0, 0, 0, 0, 0, 0, 73, 4092, 0, 0, 0, 0, 2, 4096, 139, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4096, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 686, 14, 0, 0, 0, 2278, 713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8197, 7992, 8187, 8642, 8402, 8157, 7759] avg=8192.0000 max=8642 max_rank=4 min=7759 min_rank=7 imbalance_ratio=1.054932 max_load_reduction=56894 max_load_reduction_pct=0.868134 moved_to_echo=32960 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 2298, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1416, 3244, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2298, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2110, 0, 0, 4096, 1798, 0, 0, 0, 0, 0, 0, 21, 4087, 0, 0, 0, 0, 0, 4096, 139, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 546, 31, 0, 0, 0, 2278, 713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8191, 8755, 8178, 8004, 8343, 8224, 7638] avg=8192.0000 max=8755 max_rank=2 min=7638 min_rank=7 imbalance_ratio=1.068726 max_load_reduction=56781 max_load_reduction_pct=0.866409 moved_to_echo=32209 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 2298, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 3244, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 2298, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2952, 0, 0, 4096, 1798, 0, 0, 0, 0, 0, 0, 106, 4078, 0, 0, 0, 0, 1, 4096, 139, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 649, 18, 0, 0, 0, 2278, 713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8185, 7760, 8181, 8846, 8420, 8207, 7734] avg=8192.0000 max=8846 max_rank=4 min=7734 min_rank=7 imbalance_ratio=1.079834 max_load_reduction=56690 max_load_reduction_pct=0.865021 moved_to_echo=33207 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 2298, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 3244, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4096, 0, 2298, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2907, 0, 0, 4096, 1798, 0, 0, 0, 0, 0, 0, 5, 4080, 0, 0, 0, 0, 0, 4096, 139, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4096, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 693, 17, 0, 0, 0, 2278, 713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8189, 7918, 8192, 8801, 8320, 8203, 7709] avg=8192.0000 max=8801 max_rank=4 min=7709 min_rank=7 imbalance_ratio=1.074341 max_load_reduction=56735 max_load_reduction_pct=0.865707 moved_to_echo=33033 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 2299, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 3243, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4096, 0, 2298, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2855, 0, 0, 4096, 1797, 0, 0, 0, 0, 0, 0, 156, 4056, 0, 0, 0, 0, 7, 4096, 139, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4096, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 401, 11, 0, 0, 0, 2278, 713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8201, 8131, 8195, 8748, 8454, 8153, 7459] avg=8192.0000 max=8748 max_rank=4 min=7459 min_rank=7 imbalance_ratio=1.067871 max_load_reduction=56788 max_load_reduction_pct=0.866516 moved_to_echo=32814 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 17, 0, 2298, 1786, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 473, 3242, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6, 0, 4094, 0, 2298, 1783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1903, 0, 0, 4096, 1798, 0, 0, 0, 0, 0, 0, 119, 3979, 0, 0, 0, 0, 3, 4096, 139, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4096, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 1739, 30, 1, 0, 0, 2277, 713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8196, 7807, 8183, 7797, 8336, 8200, 8815] avg=8192.0000 max=8815 max_rank=7 min=7797 min_rank=4 imbalance_ratio=1.076050 max_load_reduction=56721 max_load_reduction_pct=0.865494 moved_to_echo=33148 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 4089, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 9, 0, 4090, 13, 0, 0, 6, 2293, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 2, 1826, 3232, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 53, 0, 7, 4088, 7, 2289, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 7, 16, 21, 8, 5, 8, 4087, 1792, 0, 0, 0, 0, 0, 0, 4, 2396, 4, 7, 6, 1, 0, 4089, 140, 0, 0, 0, 0, 0, 0, 0, 7, 28, 4080, 0, 7, 4088, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4090, 0, 1545, 1977, 0, 5, 0, 2274, 717, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8197, 9557, 8233, 5944, 6647, 8219, 10614] avg=8192.0000 max=10614 max_rank=7 min=5944 min_rank=4 imbalance_ratio=1.295654 max_load_reduction=54922 max_load_reduction_pct=0.838043 moved_to_echo=31424 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 384, 32768, 0, 128, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32512, 0, 7040, 2176, 128, 128, 0, 18304, 5760, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 128, 128, 18432, 14336, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 18432, 128, 128, 32768, 14464, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[896, 30848, 128, 128, 128, 128, 128, 32768, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 128, 18432, 14336, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 512, 0, 128, 6528, 25984, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4069, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 2470, 4096, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4065, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1652, 7, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4034, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 1129, 4096, 0, 4047, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4077, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3057, 5, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4063, 113, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 3082, 4095, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 2, 4095, 4095, 0, 4096, 4090, 0, 0, 4096, 0, 4093, 0, 4, 0, 0, 924, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4068, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 0, 2484, 4093, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 4, 4089, 4096, 0, 4096, 4069, 0, 0, 4096, 0, 4089, 0, 5, 0, 0, 1598, 12, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4070, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 2177, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 4096, 4048, 0, 0, 4096, 0, 4089, 0, 5, 0, 0, 1525, 25, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[3980, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 2478, 4096, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 2, 4094, 4096, 0, 4096, 4050, 0, 0, 4096, 0, 4090, 0, 3, 0, 0, 1588, 13, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4087, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 3487, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 4061, 0, 0, 4096, 0, 4094, 0, 1, 0, 0, 587, 3, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[3819, 2292, 16, 0, 0, 5, 0, 0, 9, 8, 8, 6, 6, 0, 4072, 148, 1, 0, 1, 1, 895, 4091, 769, 3721, 0, 0, 4089, 10, 0, 0, 0, 0, 4089, 4088, 0, 4088, 4088, 8, 144, 4083, 218, 4090, 0, 0, 2, 12, 404, 0, 7, 4089, 0, 5, 6, 0, 4081, 0, 7, 7, 9, 1, 4, 3944, 4087, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4069, 21, 0, 0, 0, 0, 0, 0, 3749, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2470, 1773, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4065, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1652, 7, 2308, 335, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7858, 8173, 8343, 8192, 8173, 8398, 8192, 8207] avg=8192.0000 max=8398 max_rank=5 min=7858 min_rank=0 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32970 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3980, 178, 0, 0, 0, 0, 0, 0, 3749, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2478, 1773, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4050, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3, 0, 0, 1588, 13, 2308, 335, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7926, 8173, 8353, 8194, 8156, 8337, 8192, 8205] avg=8192.0000 max=8353 max_rank=2 min=7926 min_rank=0 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32890 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4087, 55, 0, 0, 0, 0, 0, 0, 3749, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 3487, 1773, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4061, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 587, 3, 2308, 335, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7910, 8173, 9366, 8192, 8168, 7328, 8192, 8207] avg=8192.0000 max=9366 max_rank=2 min=7328 min_rank=5 imbalance_ratio=1.143311 max_load_reduction=56170 max_load_reduction_pct=0.857086 moved_to_echo=31895 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4063, 113, 0, 0, 0, 0, 0, 1, 3749, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 3082, 1772, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4090, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4, 0, 0, 924, 4, 2308, 335, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7945, 8172, 8961, 8194, 8197, 7668, 8192, 8207] avg=8192.0000 max=8961 max_rank=2 min=7668 min_rank=5 imbalance_ratio=1.093872 max_load_reduction=56575 max_load_reduction_pct=0.863266 moved_to_echo=32264 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4068, 52, 0, 0, 0, 0, 0, 0, 3749, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 1, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 2484, 1772, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4069, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 5, 0, 0, 1598, 12, 2306, 335, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7889, 8173, 8365, 8195, 8170, 8345, 8192, 8207] avg=8192.0000 max=8365 max_rank=2 min=7889 min_rank=0 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32914 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4070, 441, 0, 0, 0, 0, 0, 0, 3749, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2177, 1773, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4048, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 5, 0, 0, 1525, 25, 2308, 335, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8173, 8055, 8192, 8151, 8287, 8192, 8207] avg=8192.0000 max=8287 max_rank=5 min=8055 min_rank=2 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4034, 34, 0, 0, 0, 0, 0, 0, 3749, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1129, 1773, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4077, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3057, 5, 2308, 335, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7837, 8173, 6950, 8191, 8185, 9801, 8192, 8207] avg=8192.0000 max=9801 max_rank=5 min=6950 min_rank=2 imbalance_ratio=1.196411 max_load_reduction=55735 max_load_reduction_pct=0.850449 moved_to_echo=34385 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3819, 2292, 16, 0, 0, 5, 0, 0, 3740, 20, 0, 0, 0, 0, 0, 0, 9, 8, 8, 6, 6, 0, 4072, 148, 4068, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 895, 1755, 769, 3721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 10, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4088, 8, 144, 7, 0, 0, 0, 0, 0, 0, 0, 0, 218, 4090, 0, 0, 2, 12, 404, 0, 2310, 336, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 5, 6, 0, 4081, 0, 4, 0, 0, 0, 0, 0, 0, 0, 7, 7, 9, 1, 4, 3944, 4087, 8, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9892, 8326, 7143, 8186, 8336, 7372, 8192, 8089] avg=8192.0000 max=9892 max_rank=0 min=7143 min_rank=2 imbalance_ratio=1.207520 max_load_reduction=55644 max_load_reduction_pct=0.849060 moved_to_echo=31989 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 32768, 0, 128, 128, 128, 11392, 128, 18560, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 0, 32640, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32256, 3200, 128, 0, 0, 128, 0, 128, 30080, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32640, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 256, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 18304, 14208, 896, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4084, 0, 0, 0, 0, 219, 0, 4096, 0, 4096, 3923, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4062, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4089, 0, 0, 0, 0, 93, 0, 4096, 0, 4089, 4025, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4090, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 122, 0, 4095, 0, 4095, 4050, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4022, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 123, 0, 4096, 0, 4084, 4008, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4074, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 0, 0, 363, 0, 4096, 0, 4080, 3759, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4090, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 0, 0, 111, 0, 4095, 0, 4095, 4019, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4095, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4067, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 316, 0, 4095, 0, 4084, 3885, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4007, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[14, 4089, 6, 100, 5, 161, 757, 0, 4089, 6, 4087, 2543, 12, 0, 0, 0, 4086, 0, 70, 3975, 5, 4084, 1, 16, 3, 5, 25, 12, 0, 3992, 0, 4083, 0, 7, 5, 4090, 0, 434, 0, 0, 8, 0, 8, 0, 0, 989, 8, 5, 20, 9, 4064, 0, 0, 11, 4088, 3337, 0, 4045, 1, 4084, 4077, 0, 13, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 0, 0, 0, 93, 0, 3762, 40, 0, 0, 0, 0, 0, 0, 326, 0, 4089, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3929, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 127, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7984, 8440, 8197, 8200, 8137, 8066, 8313, 8199] avg=8192.0000 max=8440 max_rank=1 min=7984 min_rank=0 imbalance_ratio=1.030273 max_load_reduction=57096 max_load_reduction_pct=0.871216 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 0, 0, 0, 0, 219, 0, 3762, 40, 0, 0, 0, 0, 0, 0, 326, 0, 4096, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3929, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 127, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8345, 8199, 8200, 8137, 8066, 8285, 8199] avg=8192.0000 max=8345 max_rank=1 min=8066 min_rank=5 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 123, 0, 3762, 40, 0, 0, 0, 0, 0, 0, 326, 0, 4084, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 3929, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 127, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8020, 8418, 8197, 8200, 8139, 8066, 8297, 8199] avg=8192.0000 max=8418 max_rank=1 min=8020 min_rank=0 imbalance_ratio=1.027588 max_load_reduction=57118 max_load_reduction_pct=0.871552 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 122, 0, 3761, 40, 0, 0, 0, 0, 0, 0, 326, 0, 4095, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3929, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 127, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8019, 8471, 8197, 8201, 8138, 8066, 8245, 8199] avg=8192.0000 max=8471 max_rank=1 min=8019 min_rank=0 imbalance_ratio=1.034058 max_load_reduction=57065 max_load_reduction_pct=0.870743 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 0, 0, 111, 0, 3761, 40, 0, 0, 0, 0, 0, 0, 326, 0, 4095, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4095, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3929, 112, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4089, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 127, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8004, 8440, 8198, 8200, 8138, 8067, 8290, 8199] avg=8192.0000 max=8440 max_rank=1 min=8004 min_rank=0 imbalance_ratio=1.030273 max_load_reduction=57096 max_load_reduction_pct=0.871216 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 0, 0, 363, 0, 3762, 40, 0, 0, 0, 0, 0, 0, 326, 0, 4080, 3759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 3929, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 127, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8165, 8196, 8200, 8140, 8066, 8313, 8199] avg=8192.0000 max=8313 max_rank=6 min=8066 min_rank=5 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 316, 0, 3761, 40, 0, 0, 0, 0, 0, 0, 326, 0, 4084, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3929, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 127, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8295, 8198, 8201, 8137, 8066, 8230, 8199] avg=8192.0000 max=8295 max_rank=1 min=8066 min_rank=5 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[14, 4089, 6, 100, 5, 161, 757, 0, 3761, 44, 0, 0, 0, 0, 0, 0, 314, 6, 4087, 2543, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 70, 3975, 5, 1, 1, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 25, 12, 0, 3992, 0, 4083, 14, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 4090, 0, 434, 0, 0, 3922, 112, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 989, 8, 5, 4084, 3971, 0, 0, 0, 0, 0, 0, 20, 9, 4064, 0, 0, 11, 122, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 1, 0, 4077, 0, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8937, 6962, 8154, 8134, 8570, 9073, 7563, 8143] avg=8192.0000 max=9073 max_rank=5 min=6962 min_rank=1 imbalance_ratio=1.107544 max_load_reduction=56463 max_load_reduction_pct=0.861557 moved_to_echo=33349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 1024, 128, 128, 32768, 31872, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 512, 0, 0, 31488, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 1024, 31872, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2688, 128, 32768, 30336, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 256, 2176, 0, 30208, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True + [2026-08-30 05:00:54] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 672.1 | throughput per GPU (TFLOP/s/GPU): 496.1 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.098960E+00 | loss scale: 1.0 | grad norm: 5.401 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1984, 132, 760, 348, 1064, 1, 2142, 76, 665, 3325, 586, 779, 1883, 1924, 1, 230, 291, 1296, 510, 948, 0, 565, 1608, 1495, 64, 25, 3065, 1195, 2663, 612, 2111, 776, 7, 1703, 2038, 182, 521, 713, 2250, 1967, 494, 6, 1924, 1287, 1, 1624, 1386, 896, 2512, 36, 1303, 1, 52, 897, 1, 754, 0, 266, 875, 1374, 2281, 1293, 1281, 2487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2386, 41, 965, 390, 1069, 0, 2099, 138, 1097, 3252, 648, 542, 1697, 1597, 1, 310, 371, 1362, 352, 1361, 0, 688, 1700, 1092, 78, 33, 2843, 1089, 2777, 475, 2742, 666, 5, 1888, 2368, 316, 382, 852, 2058, 1685, 679, 0, 1806, 1431, 0, 1458, 1350, 874, 2607, 27, 1445, 3, 33, 789, 4, 449, 0, 448, 1474, 604, 2403, 878, 951, 2408] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2833, 96, 512, 300, 1088, 0, 1914, 72, 1392, 2741, 834, 769, 1338, 1823, 0, 313, 501, 880, 700, 763, 0, 944, 1630, 1368, 152, 75, 2651, 1125, 1723, 631, 2335, 785, 1, 1126, 2467, 170, 502, 855, 1643, 1461, 915, 0, 1776, 2164, 0, 1928, 784, 813, 2560, 32, 1934, 21, 184, 894, 1, 487, 4, 352, 1645, 1298, 2384, 1269, 1267, 2311] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2445, 42, 391, 244, 824, 0, 1691, 82, 1424, 3414, 670, 456, 1313, 2175, 0, 240, 666, 1675, 333, 1202, 0, 1009, 1891, 1728, 44, 52, 2474, 938, 2295, 677, 2186, 813, 2, 1949, 2250, 271, 509, 688, 2101, 1397, 745, 0, 1556, 1909, 0, 1588, 1030, 1195, 2678, 51, 1588, 6, 11, 998, 0, 398, 0, 318, 1037, 658, 2278, 952, 1285, 2694] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2175, 77, 577, 342, 1100, 0, 1861, 199, 574, 2819, 335, 1233, 2122, 1502, 0, 425, 334, 1337, 911, 1583, 0, 609, 2306, 1491, 67, 99, 2656, 1331, 2955, 731, 2123, 672, 13, 1348, 2209, 213, 458, 713, 2003, 1482, 820, 0, 2019, 1214, 1, 1355, 1005, 1175, 2908, 33, 1407, 2, 85, 966, 31, 624, 0, 185, 1017, 681, 2537, 1010, 816, 2660] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2282, 58, 602, 156, 1601, 1, 2456, 130, 657, 2652, 560, 579, 1851, 1152, 0, 175, 256, 1557, 1085, 1313, 1, 682, 2492, 1350, 53, 36, 2967, 1563, 2190, 513, 2489, 1160, 5, 1798, 1624, 168, 507, 551, 1782, 1752, 707, 6, 1905, 1003, 2, 1713, 1185, 860, 2637, 73, 1163, 5, 87, 687, 0, 671, 3, 245, 1278, 1138, 2702, 972, 1033, 2655] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2661, 85, 393, 572, 1128, 0, 1534, 256, 880, 2738, 284, 959, 1406, 2061, 0, 345, 515, 1173, 662, 633, 0, 718, 1648, 1227, 69, 19, 2850, 1405, 2091, 738, 2424, 902, 3, 1365, 2529, 349, 749, 861, 2120, 1655, 627, 2, 1934, 1628, 1, 1475, 1146, 911, 2929, 27, 1391, 9, 102, 1224, 9, 1110, 0, 259, 1022, 953, 2400, 1017, 1445, 1908] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2038, 124, 702, 521, 1012, 6, 1701, 129, 1042, 2813, 514, 507, 2008, 1255, 0, 424, 517, 1224, 876, 1286, 1, 735, 2077, 1648, 55, 30, 2715, 1001, 2347, 723, 2494, 758, 11, 1920, 2291, 215, 361, 649, 1719, 1413, 756, 4, 1925, 1563, 1, 1161, 1708, 774, 2351, 8, 1700, 28, 51, 797, 6, 913, 7, 381, 1491, 1238, 2327, 1073, 881, 2530] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2833, 96, 512, 300, 1088, 0, 1914, 72, 1083, 428, 0, 0, 0, 0, 0, 0, 1392, 2075, 834, 769, 1338, 1823, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 501, 880, 700, 763, 0, 944, 1630, 1368, 318, 238, 0, 0, 0, 0, 0, 0, 152, 75, 640, 1125, 1723, 631, 2335, 785, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1126, 1687, 170, 502, 855, 1643, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 915, 0, 1776, 2164, 0, 1928, 784, 813, 462, 0, 0, 0, 0, 0, 0, 0, 2560, 32, 1934, 21, 184, 894, 1, 487, 2011, 221, 0, 0, 0, 0, 0, 0, 4, 352, 1645, 1298, 2384, 1269, 1267, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8326, 8544, 7342, 7466, 7445, 8842, 8345, 9226] avg=8192.0000 max=9226 max_rank=7 min=7342 min_rank=2 imbalance_ratio=1.126221 max_load_reduction=56310 max_load_reduction_pct=0.859222 moved_to_echo=33858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2386, 41, 965, 390, 1069, 0, 2099, 138, 1129, 508, 0, 0, 0, 0, 0, 0, 1097, 2462, 648, 542, 1697, 1597, 1, 310, 0, 0, 0, 0, 0, 0, 0, 0, 371, 1362, 352, 1361, 0, 688, 1700, 1092, 305, 282, 0, 0, 0, 0, 0, 0, 78, 33, 686, 1089, 2777, 475, 2742, 666, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1888, 1620, 316, 382, 852, 2058, 1685, 0, 0, 0, 0, 0, 0, 0, 0, 679, 0, 1806, 1431, 0, 1458, 1350, 874, 443, 0, 0, 0, 0, 0, 0, 0, 2607, 27, 1445, 3, 33, 789, 4, 449, 2157, 231, 0, 0, 0, 0, 0, 0, 0, 448, 1474, 604, 2403, 878, 951, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8725, 8354, 7513, 8546, 8806, 8041, 7745, 7806] avg=8192.0000 max=8806 max_rank=4 min=7513 min_rank=2 imbalance_ratio=1.074951 max_load_reduction=56730 max_load_reduction_pct=0.865631 moved_to_echo=32398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2175, 77, 577, 342, 1100, 0, 1861, 199, 1247, 440, 0, 0, 0, 0, 0, 0, 574, 2135, 335, 1233, 2122, 1502, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 334, 1337, 911, 1583, 0, 609, 2306, 1491, 285, 244, 0, 0, 0, 0, 0, 0, 67, 99, 641, 1331, 2955, 731, 2123, 672, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1348, 1511, 213, 458, 713, 2003, 1482, 0, 0, 0, 0, 0, 0, 0, 0, 820, 0, 2019, 1214, 1, 1355, 1005, 1175, 413, 0, 0, 0, 0, 0, 0, 0, 2908, 33, 1407, 2, 85, 966, 31, 624, 2015, 255, 0, 0, 0, 0, 0, 0, 0, 185, 1017, 681, 2537, 1010, 816, 1158, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8018, 8326, 9100, 8619, 7741, 8002, 8326, 7404] avg=8192.0000 max=9100 max_rank=2 min=7404 min_rank=7 imbalance_ratio=1.110840 max_load_reduction=56436 max_load_reduction_pct=0.861145 moved_to_echo=31473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1984, 132, 760, 348, 1064, 1, 2142, 76, 1166, 519, 0, 0, 0, 0, 0, 0, 665, 2518, 586, 779, 1883, 1924, 1, 230, 0, 0, 0, 0, 0, 0, 0, 0, 291, 1296, 510, 948, 0, 565, 1608, 1495, 263, 288, 0, 0, 0, 0, 0, 0, 64, 25, 740, 1195, 2663, 612, 2111, 776, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1703, 1394, 182, 521, 713, 2250, 1967, 0, 0, 0, 0, 0, 0, 0, 0, 494, 6, 1924, 1287, 1, 1624, 1386, 896, 381, 0, 0, 0, 0, 0, 0, 0, 2512, 36, 1303, 1, 52, 897, 1, 754, 2325, 238, 0, 0, 0, 0, 0, 0, 0, 266, 875, 1374, 2281, 1293, 1281, 1083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8586, 7264, 8186, 8737, 7999, 8119, 8453] avg=8192.0000 max=8737 max_rank=4 min=7264 min_rank=2 imbalance_ratio=1.066528 max_load_reduction=56799 max_load_reduction_pct=0.866684 moved_to_echo=33308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2445, 42, 391, 244, 824, 0, 1691, 82, 1263, 533, 0, 0, 0, 0, 0, 0, 1424, 2585, 670, 456, 1313, 2175, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 666, 1675, 333, 1202, 0, 1009, 1891, 1728, 290, 296, 0, 0, 0, 0, 0, 0, 44, 52, 597, 938, 2295, 677, 2186, 813, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1949, 1539, 271, 509, 688, 2101, 1397, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 1556, 1909, 0, 1588, 1030, 1195, 421, 0, 0, 0, 0, 0, 0, 0, 2678, 51, 1588, 6, 11, 998, 0, 398, 1877, 258, 0, 0, 0, 0, 0, 0, 0, 318, 1037, 658, 2278, 952, 1285, 1173, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7515, 8863, 9090, 7602, 8456, 8444, 7865, 7701] avg=8192.0000 max=9090 max_rank=2 min=7515 min_rank=0 imbalance_ratio=1.109619 max_load_reduction=56446 max_load_reduction_pct=0.861298 moved_to_echo=32466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2282, 58, 602, 156, 1601, 1, 2456, 130, 1245, 414, 0, 0, 0, 0, 0, 0, 657, 2008, 560, 579, 1851, 1152, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 256, 1557, 1085, 1313, 1, 682, 2492, 1350, 209, 230, 0, 0, 0, 0, 0, 0, 53, 36, 716, 1563, 2190, 513, 2489, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1798, 1111, 168, 507, 551, 1782, 1752, 0, 0, 0, 0, 0, 0, 0, 0, 707, 6, 1905, 1003, 2, 1713, 1185, 860, 304, 0, 0, 0, 0, 0, 0, 0, 2637, 73, 1163, 5, 87, 687, 0, 671, 2251, 254, 0, 0, 0, 0, 0, 0, 3, 245, 1278, 1138, 2702, 972, 1033, 1156, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8945, 6982, 9175, 8720, 7674, 7685, 7828, 8527] avg=8192.0000 max=9175 max_rank=2 min=6982 min_rank=1 imbalance_ratio=1.119995 max_load_reduction=56361 max_load_reduction_pct=0.860001 moved_to_echo=31714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2661, 85, 393, 572, 1128, 0, 1534, 256, 894, 427, 0, 0, 0, 0, 0, 0, 880, 2074, 284, 959, 1406, 2061, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 515, 1173, 662, 633, 0, 718, 1648, 1227, 326, 237, 0, 0, 0, 0, 0, 0, 69, 19, 688, 1405, 2091, 738, 2424, 902, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1365, 1730, 349, 749, 861, 2120, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 627, 2, 1934, 1628, 1, 1475, 1146, 911, 473, 0, 0, 0, 0, 0, 0, 0, 2929, 27, 1391, 9, 102, 1224, 9, 1110, 2162, 183, 0, 0, 0, 0, 0, 0, 0, 259, 1022, 953, 2400, 1017, 1445, 831, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7950, 8009, 7139, 8336, 8832, 8197, 9146, 7927] avg=8192.0000 max=9146 max_rank=6 min=7139 min_rank=2 imbalance_ratio=1.116455 max_load_reduction=56390 max_load_reduction_pct=0.860443 moved_to_echo=34102 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2038, 124, 702, 521, 1012, 6, 1701, 129, 1190, 442, 0, 0, 0, 0, 0, 0, 1042, 2122, 514, 507, 2008, 1255, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 517, 1224, 876, 1286, 1, 735, 2077, 1648, 300, 249, 0, 0, 0, 0, 0, 0, 55, 30, 653, 1001, 2347, 723, 2494, 758, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1920, 1557, 215, 361, 649, 1719, 1413, 0, 0, 0, 0, 0, 0, 0, 0, 756, 4, 1925, 1563, 1, 1161, 1708, 774, 434, 0, 0, 0, 0, 0, 0, 0, 2351, 8, 1700, 28, 51, 797, 6, 913, 2062, 246, 0, 0, 0, 0, 0, 0, 7, 381, 1491, 1238, 2327, 1073, 881, 1094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7865, 7872, 8913, 8061, 7845, 8326, 8162, 8492] avg=8192.0000 max=8913 max_rank=2 min=7845 min_rank=4 imbalance_ratio=1.088013 max_load_reduction=56623 max_load_reduction_pct=0.863998 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5760, 128, 14848, 12288, 128, 12416, 9600, 7552, 3456, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[21248, 384, 12032, 128, 640, 7296, 128, 5504, 16896, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3456, 10624, 5504, 9216, 128, 6016, 15360, 11520, 2304, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 13184, 12160, 1920, 4096, 5888, 15744, 12928, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7808, 18048, 4480, 5888, 13696, 13568, 128, 2560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[640, 384, 5376, 9728, 19072, 5120, 18944, 6656, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[18816, 768, 4992, 2944, 8960, 128, 15488, 1152, 9344, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 2560, 9856, 8064, 19328, 8576, 8960, 8576, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4094, 0, 2786, 0, 3922, 4095, 0, 1308, 2790, 0, 2, 857, 3351, 1, 0, 0, 1, 0, 4081, 3100, 12, 237, 1, 2400, 2, 3514, 0, 68, 3403, 2, 0, 1, 494, 0, 0, 972, 3786, 3583, 0, 0, 0, 1, 0, 4094, 0, 0, 1, 4075, 4094, 1, 13, 0, 9, 4094, 247, 0, 2, 2, 2, 35] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 2047, 0, 3437, 4095, 0, 914, 3363, 0, 0, 248, 4076, 0, 0, 0, 0, 0, 4047, 3332, 13, 1050, 0, 3063, 0, 3111, 2, 3, 1899, 0, 0, 0, 713, 0, 0, 1679, 4091, 3596, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 4095, 4096, 0, 0, 0, 0, 4092, 242, 0, 0, 0, 0, 25] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 2647, 0, 3802, 4096, 0, 2082, 3281, 0, 0, 1724, 3953, 0, 0, 0, 0, 0, 4096, 3881, 13, 54, 0, 1276, 0, 3564, 0, 0, 2819, 0, 1, 0, 167, 0, 0, 509, 4029, 2895, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4039, 4095, 0, 0, 0, 12, 4096, 170, 0, 0, 0, 0, 42] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 4093, 1, 2783, 1, 3953, 4092, 0, 2795, 3504, 0, 0, 1056, 3084, 0, 4, 0, 0, 0, 4075, 3611, 142, 84, 0, 1000, 0, 4085, 0, 1, 1948, 0, 0, 0, 45, 0, 0, 783, 4012, 3591, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 4096, 0, 1, 0, 0, 4096, 348, 0, 0, 0, 0, 60] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4095, 5, 3258, 1, 3649, 4089, 0, 1767, 2718, 0, 0, 1109, 3505, 1, 0, 4, 1, 1, 4094, 3784, 25, 136, 0, 2004, 0, 3894, 5, 34, 1509, 0, 2, 1, 436, 0, 0, 926, 4043, 3548, 1, 0, 0, 0, 0, 4095, 1, 0, 1, 4080, 4092, 1, 1, 1, 65, 4095, 324, 1, 1, 0, 1, 131] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4095, 0, 2305, 0, 2473, 4089, 0, 640, 2788, 0, 0, 108, 3841, 1, 0, 2, 1, 1, 4095, 3669, 54, 452, 0, 3534, 0, 3519, 1, 6, 2384, 0, 3, 1, 1409, 1, 5, 2205, 3856, 3518, 0, 0, 0, 1, 0, 4095, 2, 0, 0, 3973, 4094, 1, 0, 1, 120, 4095, 78, 0, 1, 0, 1, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1404, 0, 3864, 4096, 0, 1649, 3379, 0, 0, 301, 3728, 0, 0, 0, 0, 0, 4032, 2060, 10, 1005, 0, 3328, 0, 3166, 30, 18, 1471, 0, 0, 0, 1040, 0, 0, 2145, 4054, 4008, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 4095, 0, 15, 0, 3, 4096, 162, 0, 0, 0, 0, 100] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 13, 2698, 0, 3287, 4096, 0, 754, 3036, 0, 0, 247, 3779, 0, 0, 2, 0, 0, 4004, 3839, 5, 27, 0, 2516, 6, 3244, 0, 35, 2693, 0, 7, 0, 1172, 5, 3, 1349, 3706, 3966, 0, 0, 0, 0, 4, 4090, 1, 3, 0, 4084, 4096, 0, 18, 0, 25, 4091, 506, 0, 0, 6, 0, 27] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4094, 0, 2786, 0, 969, 703, 0, 0, 0, 0, 0, 0, 3195, 0, 0, 1308, 2790, 0, 2, 857, 0, 0, 0, 0, 0, 0, 0, 0, 3351, 1, 0, 0, 1, 0, 1121, 3100, 0, 0, 0, 0, 0, 0, 0, 0, 12, 237, 1, 2400, 2, 3514, 0, 68, 1855, 0, 0, 0, 0, 0, 0, 0, 3403, 2, 0, 1, 494, 0, 0, 972, 3918, 0, 0, 0, 0, 0, 0, 0, 3786, 3583, 0, 0, 0, 1, 0, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4075, 4094, 1, 13, 0, 24, 0, 0, 0, 0, 0, 0, 0, 9, 4094, 247, 0, 2, 2, 2, 35, 3438, 177, 136, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8555, 8152, 7574, 8089, 8790, 8026, 8208, 8142] avg=8192.0000 max=8790 max_rank=4 min=7574 min_rank=2 imbalance_ratio=1.072998 max_load_reduction=56746 max_load_reduction_pct=0.865875 moved_to_echo=33166 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 2047, 0, 960, 616, 0, 0, 0, 0, 0, 0, 2800, 0, 0, 914, 3363, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 1113, 3332, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1050, 0, 3063, 0, 3111, 2, 3, 1839, 0, 0, 0, 0, 0, 0, 0, 1899, 0, 0, 0, 713, 0, 0, 1679, 3918, 0, 0, 0, 0, 0, 0, 0, 4091, 3596, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4095, 4096, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 242, 0, 0, 0, 0, 25, 3439, 177, 135, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7719, 7325, 8521, 9081, 8209, 8344, 8227, 8110] avg=8192.0000 max=9081 max_rank=3 min=7325 min_rank=1 imbalance_ratio=1.108521 max_load_reduction=56455 max_load_reduction_pct=0.861435 moved_to_echo=32890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 2647, 0, 972, 682, 0, 0, 0, 0, 0, 0, 3097, 0, 0, 2082, 3281, 0, 0, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 0, 0, 0, 0, 0, 1125, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 13, 54, 0, 1276, 0, 3564, 0, 0, 1862, 0, 0, 0, 0, 0, 0, 0, 2819, 0, 1, 0, 167, 0, 0, 509, 3919, 0, 0, 0, 0, 0, 0, 0, 4029, 2895, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 4095, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 170, 0, 0, 0, 0, 42, 3439, 177, 137, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8398, 10184, 8959, 6769, 7415, 7581, 8157, 8073] avg=8192.0000 max=10184 max_rank=1 min=6769 min_rank=3 imbalance_ratio=1.243164 max_load_reduction=55352 max_load_reduction_pct=0.844604 moved_to_echo=31226 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 4093, 1, 2783, 1, 967, 709, 0, 0, 0, 0, 0, 0, 3220, 0, 0, 2795, 3504, 0, 0, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 3084, 0, 4, 0, 0, 0, 1120, 3611, 0, 0, 0, 0, 0, 0, 0, 0, 142, 84, 0, 1000, 0, 4085, 0, 1, 1852, 0, 0, 0, 0, 0, 0, 0, 1948, 0, 0, 0, 45, 0, 0, 783, 3915, 0, 0, 0, 0, 0, 0, 0, 4012, 3591, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 1, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 348, 0, 0, 0, 0, 60, 3439, 177, 136, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8556, 10575, 7819, 7164, 6691, 8260, 8215, 8256] avg=8192.0000 max=10575 max_rank=1 min=6691 min_rank=4 imbalance_ratio=1.290894 max_load_reduction=54961 max_load_reduction_pct=0.838638 moved_to_echo=31422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4095, 5, 3258, 1, 972, 654, 0, 0, 0, 0, 0, 0, 2973, 0, 0, 1767, 2718, 0, 0, 1109, 0, 0, 0, 0, 0, 0, 0, 0, 3505, 1, 0, 4, 1, 1, 1124, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 25, 136, 0, 2004, 0, 3894, 5, 34, 1861, 0, 0, 0, 0, 0, 0, 0, 1509, 0, 2, 1, 436, 0, 0, 926, 3912, 0, 0, 0, 0, 0, 0, 0, 4043, 3548, 1, 0, 0, 0, 0, 656, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4080, 4092, 1, 1, 1, 22, 0, 0, 0, 0, 0, 0, 0, 65, 4095, 324, 1, 1, 0, 1, 131, 3439, 177, 137, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8986, 8567, 8420, 7959, 6786, 8248, 8199, 8371] avg=8192.0000 max=8986 max_rank=0 min=6786 min_rank=4 imbalance_ratio=1.096924 max_load_reduction=56550 max_load_reduction_pct=0.862885 moved_to_echo=31604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4095, 0, 2305, 0, 972, 443, 0, 0, 0, 0, 0, 0, 2015, 0, 0, 640, 2788, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 1, 0, 2, 1, 1, 1125, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 54, 452, 0, 3534, 0, 3519, 1, 6, 1861, 0, 0, 0, 0, 0, 0, 0, 2384, 0, 3, 1, 1409, 1, 5, 2205, 3913, 0, 0, 0, 0, 0, 0, 0, 3856, 3518, 0, 0, 0, 1, 0, 656, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3973, 4094, 1, 0, 1, 15, 0, 0, 0, 0, 0, 0, 0, 120, 4095, 78, 0, 1, 0, 1, 17, 3439, 176, 137, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7816, 5551, 8640, 9427, 9921, 8031, 8086, 8064] avg=8192.0000 max=9921 max_rank=4 min=5551 min_rank=1 imbalance_ratio=1.211060 max_load_reduction=55615 max_load_reduction_pct=0.848618 moved_to_echo=34102 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1404, 0, 957, 693, 0, 0, 0, 0, 0, 0, 3148, 0, 0, 1649, 3379, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 0, 0, 1107, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1005, 0, 3328, 0, 3166, 30, 18, 1833, 0, 0, 0, 0, 0, 0, 0, 1471, 0, 0, 0, 1040, 0, 0, 2145, 3919, 0, 0, 0, 0, 0, 0, 0, 4054, 4008, 0, 0, 0, 0, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4095, 0, 15, 0, 23, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 162, 0, 0, 0, 0, 100, 3439, 177, 135, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7150, 8477, 6895, 9390, 8575, 8719, 8218, 8112] avg=8192.0000 max=9390 max_rank=3 min=6895 min_rank=2 imbalance_ratio=1.146240 max_load_reduction=56146 max_load_reduction_pct=0.856720 moved_to_echo=33624 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 13, 2698, 0, 954, 595, 0, 0, 0, 0, 0, 0, 2668, 0, 0, 754, 3036, 0, 0, 247, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 0, 0, 2, 0, 0, 1088, 3839, 0, 0, 0, 0, 0, 0, 0, 0, 5, 27, 0, 2516, 6, 3244, 0, 35, 1824, 0, 0, 0, 0, 0, 0, 0, 2693, 0, 7, 0, 1172, 5, 3, 1349, 3920, 0, 0, 0, 0, 0, 0, 0, 3706, 3966, 0, 0, 0, 0, 4, 651, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4084, 4096, 0, 18, 0, 24, 0, 0, 0, 0, 0, 0, 0, 25, 4091, 506, 0, 0, 6, 0, 27, 3439, 176, 138, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8356, 6705, 8708, 7657, 9149, 8327, 8226, 8408] avg=8192.0000 max=9149 max_rank=4 min=6705 min_rank=1 imbalance_ratio=1.116821 max_load_reduction=56387 max_load_reduction_pct=0.860397 moved_to_echo=34110 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23168, 0, 0, 12032, 24960, 0, 128, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[18176, 128, 128, 128, 5504, 128, 128, 10624, 31360, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31616, 28800, 128, 0, 0, 128, 128, 5248, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32640, 32768, 128, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 19968, 128, 7808, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[29440, 128, 128, 128, 128, 128, 8960, 27392, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 3072, 128, 19200, 128, 28160, 128, 256, 14848, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 32768, 2176, 128, 128, 128, 128, 512, 27520, 1536, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 2878, 0, 0, 2, 0, 0, 0, 3958, 4092, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 1455, 0, 4093, 171, 4085, 3422, 0, 4095, 0, 0, 0, 3785, 606, 0, 0, 4, 4035, 4096, 0, 0, 112, 0, 24, 0, 4090, 0, 29, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 2709, 0, 0, 2, 0, 0, 0, 3839, 4079, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 467, 0, 4089, 23, 4096, 4058, 0, 4094, 0, 0, 0, 3934, 733, 0, 0, 2, 3963, 4096, 0, 0, 298, 0, 65, 0, 4092, 0, 19, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 398, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 2575, 0, 0, 5, 0, 0, 0, 3909, 4094, 4095, 0, 1, 4095, 0, 1, 4095, 1, 1, 935, 0, 4095, 6, 4095, 4019, 0, 4089, 1, 0, 0, 3749, 1104, 5, 0, 37, 3983, 4092, 1, 1, 164, 0, 8, 1, 4050, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 4095, 1, 1, 29, 0, 2, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 1995, 0, 1, 17, 0, 0, 0, 3816, 4067, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 794, 2, 4069, 2, 4096, 3478, 0, 4093, 0, 0, 0, 3898, 2595, 0, 1, 4, 3818, 4095, 0, 0, 71, 0, 0, 0, 4095, 0, 15, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 30, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 2379, 0, 0, 4, 0, 0, 1, 3875, 4091, 4091, 0, 1, 4095, 0, 1, 4095, 1, 1, 991, 0, 4093, 0, 4094, 3622, 0, 4091, 1, 0, 0, 3432, 1910, 1, 0, 8, 3983, 4094, 1, 1, 241, 0, 37, 1, 4088, 0, 2, 1, 4094, 0, 1, 0, 1, 1, 4094, 1, 1, 13, 0, 0, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1865, 0, 0, 45, 0, 0, 0, 3814, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 500, 0, 4096, 74, 4086, 3441, 0, 4096, 0, 0, 0, 3502, 2799, 4, 0, 17, 4058, 4095, 0, 0, 383, 0, 2, 0, 4059, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 2, 3071, 0, 0, 1, 0, 2, 0, 3932, 4078, 4095, 0, 1, 4094, 0, 2, 4094, 1, 1, 815, 0, 4080, 7, 4088, 3857, 0, 4096, 1, 0, 0, 3974, 547, 8, 0, 2, 4082, 4094, 2, 1, 140, 0, 21, 0, 4040, 0, 38, 0, 4095, 2, 1, 2, 1, 1, 4095, 2, 0, 65, 0, 0, 0, 1, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 5, 0, 2242, 0, 7, 16, 0, 0, 0, 3743, 4068, 4096, 0, 0, 4085, 3, 0, 4096, 0, 0, 1191, 10, 4064, 97, 4096, 3934, 0, 4076, 0, 0, 0, 3674, 1215, 0, 11, 16, 4058, 4090, 0, 0, 278, 0, 12, 0, 4078, 0, 37, 0, 4089, 0, 0, 0, 11, 0, 4086, 0, 0, 34, 0, 18, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2709, 0, 0, 2, 0, 3848, 1790, 78, 0, 0, 0, 0, 0, 0, 0, 3839, 4079, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3211, 0, 0, 467, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 23, 325, 4058, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 733, 0, 0, 2, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 65, 0, 4092, 0, 19, 1981, 994, 885, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 4096, 4018, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8427, 8166, 7767, 8500, 7638, 8334, 8192, 8512] avg=8192.0000 max=8512 max_rank=7 min=7638 min_rank=4 imbalance_ratio=1.039062 max_load_reduction=57024 max_load_reduction_pct=0.870117 moved_to_echo=32676 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 2878, 0, 0, 2, 0, 3848, 1785, 79, 0, 0, 0, 0, 0, 0, 0, 3958, 4092, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3211, 0, 0, 1455, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 171, 324, 3422, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 606, 0, 0, 4, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 24, 0, 4090, 0, 29, 1976, 1012, 885, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4096, 4017, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8593, 8298, 8759, 8012, 7418, 8128, 8192, 8136] avg=8192.0000 max=8759 max_rank=2 min=7418 min_rank=4 imbalance_ratio=1.069214 max_load_reduction=56777 max_load_reduction_pct=0.866348 moved_to_echo=31874 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 2, 3071, 0, 0, 1, 0, 3846, 1786, 78, 0, 0, 0, 0, 0, 2, 0, 3932, 4078, 0, 0, 1, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3210, 1, 1, 815, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 7, 325, 3857, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 547, 8, 0, 2, 3058, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 140, 0, 21, 0, 4040, 0, 38, 1977, 1024, 884, 0, 0, 0, 0, 0, 0, 4095, 2, 1, 2, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 65, 0, 0, 0, 1, 2, 4095, 4016, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8786, 8261, 8109, 8286, 7591, 8125, 8197, 8181] avg=8192.0000 max=8786 max_rank=0 min=7591 min_rank=4 imbalance_ratio=1.072510 max_load_reduction=56750 max_load_reduction_pct=0.865936 moved_to_echo=32094 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 2575, 0, 0, 5, 0, 3847, 1789, 79, 0, 0, 0, 0, 0, 0, 0, 3909, 4094, 0, 0, 1, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3210, 1, 1, 935, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 6, 326, 4019, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3749, 1104, 5, 0, 37, 2984, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 164, 0, 8, 1, 4050, 0, 0, 1980, 999, 885, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 29, 0, 2, 1, 0, 1, 4095, 4013, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8252, 8243, 8441, 7880, 8088, 8193, 8143] avg=8192.0000 max=8441 max_rank=3 min=7880 min_rank=4 imbalance_ratio=1.030396 max_load_reduction=57095 max_load_reduction_pct=0.871201 moved_to_echo=32304 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1865, 0, 0, 45, 0, 3848, 1785, 78, 0, 0, 0, 0, 0, 0, 0, 3814, 4096, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3211, 0, 0, 500, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 74, 325, 3441, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 2799, 4, 0, 17, 3040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 0, 2, 0, 4059, 0, 0, 1976, 1018, 885, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 1, 0, 4096, 4017, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7621, 8158, 7807, 7936, 9362, 8323, 8192, 8137] avg=8192.0000 max=9362 max_rank=4 min=7621 min_rank=0 imbalance_ratio=1.142822 max_load_reduction=56174 max_load_reduction_pct=0.857147 moved_to_echo=34014 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1995, 0, 1, 17, 0, 3848, 1790, 78, 0, 0, 0, 0, 0, 0, 0, 3816, 4067, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3211, 0, 0, 794, 2, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 2, 325, 3478, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 2595, 0, 1, 4, 2860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 4095, 0, 15, 1981, 958, 885, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 1, 0, 4096, 4017, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7730, 8131, 8077, 7898, 9358, 8005, 8193, 8144] avg=8192.0000 max=9358 max_rank=4 min=7730 min_rank=0 imbalance_ratio=1.142334 max_load_reduction=56178 max_load_reduction_pct=0.857208 moved_to_echo=33700 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 2379, 0, 0, 4, 0, 3847, 1789, 78, 0, 0, 0, 0, 0, 0, 1, 3875, 4091, 0, 0, 1, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3210, 1, 1, 991, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 3622, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3432, 1910, 1, 0, 8, 2984, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 241, 0, 37, 1, 4088, 0, 2, 1980, 999, 885, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 13, 0, 0, 1, 0, 1, 4091, 4016, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8216, 8297, 8039, 8336, 8234, 8192, 8124] avg=8192.0000 max=8336 max_rank=4 min=8039 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32886 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 2242, 0, 7, 16, 0, 3842, 1794, 79, 0, 0, 0, 0, 0, 0, 0, 3743, 4068, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3209, 0, 0, 1191, 10, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 97, 317, 3934, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3674, 1215, 0, 11, 16, 3037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 12, 0, 4078, 0, 37, 1985, 1021, 887, 1, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 11, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 18, 0, 0, 0, 4096, 4011, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7985, 8054, 8477, 8424, 7953, 8299, 8185, 8159] avg=8192.0000 max=8477 max_rank=2 min=7953 min_rank=4 imbalance_ratio=1.034790 max_load_reduction=57059 max_load_reduction_pct=0.870651 moved_to_echo=32596 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 19840, 0, 128, 128, 0, 30848, 14336, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[384, 2688, 29952, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[29952, 11520, 128, 128, 128, 24064, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 25728, 128, 128, 7168, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 30976, 32768, 0, 0, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 640, 0, 128, 128, 128, 128, 32768, 32128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 1792, 0, 256, 128, 32640, 0, 256, 15872, 8064, 7168, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 4076, 0, 0, 0, 4087, 0, 4096, 4096, 0, 3930, 4096, 0, 0, 0, 420, 0, 0, 0, 0, 0, 0, 0, 3605, 0, 0, 0, 4096, 0, 4096, 4096, 0, 2170, 2, 0, 1, 0, 0, 0, 8, 4096, 0, 0, 2118, 44, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4095, 4094, 4095, 0, 0, 0, 4076, 0, 4096, 4096, 0, 3997, 4096, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 3497, 0, 0, 0, 4096, 0, 4070, 4096, 0, 1201, 56, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2236, 1247, 0, 0, 0, 4096, 7, 30, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4093, 1, 0, 4095, 4015, 4095, 0, 1, 0, 3858, 0, 4093, 4095, 0, 3914, 4095, 1, 1, 1, 275, 0, 0, 0, 0, 1, 0, 0, 4000, 0, 1, 2, 4095, 0, 4091, 4093, 1, 1203, 1, 1, 0, 1, 0, 0, 0, 4095, 0, 1, 3025, 54, 1, 1, 0, 4093, 38, 101, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 4039, 4046, 0, 0, 0, 4081, 0, 4096, 4096, 0, 3886, 4096, 0, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 3373, 0, 0, 0, 4096, 0, 4091, 4096, 0, 2589, 6, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 1877, 42, 0, 1, 0, 4095, 0, 87, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[2, 2, 0, 4, 4093, 1, 2, 4094, 4084, 4091, 0, 2, 0, 3946, 0, 4092, 4094, 0, 3917, 4094, 2, 2, 2, 161, 0, 0, 0, 0, 0, 2, 0, 3700, 0, 2, 4, 4094, 1, 4094, 4093, 0, 1770, 20, 2, 0, 0, 0, 0, 0, 4094, 0, 2, 2665, 184, 1, 0, 1, 4092, 7, 21, 0, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4089, 1, 1, 4095, 4095, 4093, 0, 1, 0, 4031, 1, 4093, 4095, 0, 4061, 4095, 1, 1, 1, 234, 0, 0, 0, 0, 1, 0, 1, 3730, 0, 1, 1, 4095, 0, 4081, 4093, 0, 1216, 45, 1, 6, 0, 0, 0, 0, 4095, 0, 1, 2711, 357, 0, 3, 0, 4094, 2, 10, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 4033, 4060, 0, 0, 0, 4085, 0, 4096, 4094, 0, 4001, 4096, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 3052, 0, 0, 0, 4095, 0, 4092, 4096, 0, 1143, 4, 0, 1, 0, 0, 0, 6, 4096, 0, 0, 3363, 746, 0, 3, 0, 4096, 2, 40, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4088, 4072, 4029, 0, 0, 0, 4071, 0, 4096, 4090, 0, 3984, 4096, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 3520, 9, 0, 0, 4096, 0, 4090, 4096, 0, 1676, 2, 0, 19, 0, 0, 0, 31, 4096, 0, 0, 2665, 272, 0, 20, 0, 4096, 0, 68, 0, 4, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4093, 98, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 3930, 0, 0, 0, 0, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 2986, 1647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2170, 2, 0, 1, 0, 0, 0, 8, 4096, 2449, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2118, 44, 0, 0, 0, 1039, 102, 0, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 2939, 1103, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8278, 8343, 8238, 8199, 8726, 7399, 8160] avg=8192.0000 max=8726 max_rank=5 min=7399 min_rank=6 imbalance_ratio=1.065186 max_load_reduction=56810 max_load_reduction_pct=0.866852 moved_to_echo=32484 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4039, 97, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 3886, 0, 0, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3373, 2986, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2589, 6, 0, 0, 0, 0, 0, 1, 4096, 2450, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1877, 42, 0, 1, 0, 1031, 102, 0, 0, 0, 0, 0, 0, 4095, 0, 87, 0, 0, 0, 0, 0, 2918, 1103, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8217, 8435, 8005, 8194, 9142, 7149, 8203] avg=8192.0000 max=9142 max_rank=5 min=7149 min_rank=6 imbalance_ratio=1.115967 max_load_reduction=56394 max_load_reduction_pct=0.860504 moved_to_echo=32688 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 0, 4, 4093, 1, 2, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4084, 98, 0, 2, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 3917, 0, 2, 2, 2, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3700, 2984, 1644, 0, 0, 0, 0, 0, 0, 0, 2, 4, 7, 1, 4094, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1770, 20, 2, 0, 0, 0, 0, 0, 4094, 2448, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 2665, 184, 1, 0, 1, 1043, 102, 0, 0, 0, 0, 0, 0, 4092, 7, 21, 0, 0, 0, 0, 2, 2950, 1103, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8130, 8075, 8330, 8201, 8334, 8092, 8175] avg=8192.0000 max=8334 max_rank=5 min=8075 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32802 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 98, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 3997, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3497, 2986, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4070, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1201, 56, 0, 0, 0, 0, 0, 0, 4096, 2450, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2236, 1247, 0, 0, 0, 1044, 102, 0, 0, 0, 0, 0, 0, 4096, 7, 30, 0, 0, 0, 0, 0, 2953, 1103, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8268, 8058, 8129, 8173, 7803, 8725, 8189] avg=8192.0000 max=8725 max_rank=6 min=7803 min_rank=5 imbalance_ratio=1.065063 max_load_reduction=56811 max_load_reduction_pct=0.866867 moved_to_echo=32890 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4033, 97, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 4001, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3052, 2985, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 4, 0, 1, 0, 0, 0, 6, 4096, 2450, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3363, 746, 0, 3, 0, 1035, 102, 0, 0, 0, 0, 0, 0, 4096, 2, 40, 0, 0, 0, 0, 0, 2928, 1103, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8215, 8037, 7683, 8195, 7700, 9345, 8169] avg=8192.0000 max=9345 max_rank=6 min=7683 min_rank=3 imbalance_ratio=1.140747 max_load_reduction=56191 max_load_reduction_pct=0.857407 moved_to_echo=33409 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4093, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4015, 98, 0, 1, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 3914, 0, 1, 1, 1, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4000, 2985, 1646, 0, 0, 0, 0, 0, 0, 0, 1, 2, 7, 0, 4091, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1203, 1, 1, 0, 1, 0, 0, 0, 4095, 2447, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 3025, 54, 1, 1, 0, 1044, 102, 0, 0, 0, 0, 0, 0, 4093, 38, 101, 1, 0, 0, 0, 1, 2953, 1103, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7972, 8184, 8632, 8195, 7748, 8323, 8290] avg=8192.0000 max=8632 max_rank=3 min=7748 min_rank=5 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=32556 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4089, 1, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 98, 0, 1, 0, 4031, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 4061, 0, 1, 1, 1, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3730, 2985, 1646, 0, 0, 0, 0, 0, 0, 0, 1, 1, 7, 0, 4081, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 45, 1, 6, 0, 0, 0, 0, 4095, 2447, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 2711, 357, 0, 3, 0, 1043, 102, 0, 0, 0, 0, 0, 0, 4094, 2, 10, 1, 0, 0, 0, 1, 2952, 1103, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8226, 8290, 8363, 8183, 7810, 8312, 8163] avg=8192.0000 max=8363 max_rank=3 min=7810 min_rank=5 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32468 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4088, 5, 0, 0, 0, 0, 0, 0, 0, 4072, 87, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 3984, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3520, 2989, 1647, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1676, 2, 0, 19, 0, 0, 0, 31, 4096, 2449, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2665, 272, 0, 20, 0, 1032, 106, 0, 0, 0, 0, 0, 0, 4096, 0, 68, 0, 4, 0, 3, 0, 2910, 1106, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8230, 8114, 8156, 8196, 8273, 8191, 8187] avg=8192.0000 max=8273 max_rank=5 min=8114 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32847 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32000, 0, 31744, 0, 128, 128, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 384, 128, 128, 0, 128, 128, 23552, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[13056, 256, 128, 128, 128, 0, 0, 128, 32768, 19712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 0, 128, 20736, 3072, 128, 128, 128, 8320, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 896, 0, 128, 0, 32256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 28544, 23936, 13184, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 2677, 0, 0, 4096, 0, 0, 0, 475, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 4096, 669, 0, 4096, 3837, 4096, 0, 0, 0, 0, 199, 0, 4096, 0, 0, 1, 0, 4096, 0, 339, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 3245, 0, 0, 4080, 0, 0, 0, 215, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 5, 0, 4095, 0, 0, 0, 24, 4095, 0, 0, 4096, 69, 0, 4096, 4032, 4096, 0, 0, 0, 0, 422, 2, 4096, 0, 0, 0, 0, 4096, 0, 208, 0, 4096, 4090, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[2, 3059, 0, 1, 4047, 0, 2, 2, 530, 4093, 0, 2, 2, 2, 0, 4066, 0, 2, 0, 2, 3, 2, 4094, 0, 4094, 2, 0, 0, 0, 4095, 0, 0, 3, 0, 4092, 0, 2, 4094, 106, 0, 4093, 4041, 4093, 2, 0, 2, 0, 114, 31, 4094, 2, 0, 0, 0, 4092, 2, 378, 2, 4094, 4093, 1, 1, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 1970, 0, 0, 4096, 0, 0, 0, 1818, 4094, 11, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 4088, 0, 4095, 0, 1, 34, 0, 4095, 0, 0, 1, 14, 4096, 0, 1, 4096, 73, 0, 4096, 3954, 4096, 0, 0, 0, 0, 130, 0, 4096, 0, 0, 1, 0, 4096, 0, 206, 0, 4095, 4090, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 2881, 0, 0, 4094, 0, 1, 1, 657, 4092, 1, 1, 1, 1, 0, 4070, 0, 1, 0, 0, 0, 1, 4081, 0, 4095, 0, 5, 26, 0, 4095, 0, 0, 7, 0, 4091, 0, 0, 4095, 117, 0, 4095, 4087, 4095, 1, 1, 1, 1, 263, 0, 4095, 1, 0, 0, 0, 4093, 0, 195, 1, 4095, 4095, 1, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 2295, 0, 0, 4089, 0, 1, 1, 1116, 4089, 0, 1, 1, 1, 0, 4075, 0, 1, 1, 0, 0, 1, 4095, 0, 4094, 1, 0, 8, 0, 4095, 0, 0, 6, 1, 4095, 0, 1, 4095, 15, 0, 4093, 4072, 4095, 1, 0, 1, 1, 348, 3, 4095, 1, 0, 0, 0, 4094, 0, 360, 1, 4095, 4095, 1, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 2131, 0, 1, 4096, 0, 0, 0, 1536, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 17, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4022, 4096, 0, 0, 0, 0, 62, 0, 4096, 0, 0, 1, 0, 4096, 0, 426, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 2664, 0, 0, 4096, 0, 0, 0, 601, 4079, 27, 0, 0, 0, 0, 4057, 12, 0, 0, 0, 0, 0, 4066, 0, 4096, 0, 6, 25, 0, 4076, 0, 0, 21, 43, 4096, 0, 0, 4096, 259, 0, 4096, 3923, 4096, 0, 0, 0, 0, 259, 0, 4096, 0, 0, 2, 0, 4096, 0, 474, 0, 4089, 4068, 0, 0, 17, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2677, 0, 0, 4096, 0, 0, 0, 850, 320, 177, 125, 12, 4, 0, 0, 475, 3240, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 3919, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 3837, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 3776, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 7810, 8193, 8180, 8685, 8007, 8189, 8211] avg=8192.0000 max=8685 max_rank=4 min=7810 min_rank=1 imbalance_ratio=1.060181 max_load_reduction=56851 max_load_reduction_pct=0.867477 moved_to_echo=33092 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3245, 0, 0, 4080, 0, 0, 0, 850, 320, 177, 125, 12, 4, 0, 0, 215, 3241, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 5, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4095, 0, 0, 3919, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 4032, 0, 0, 0, 0, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 3776, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8813, 7553, 8190, 8184, 8107, 8425, 8190, 8074] avg=8192.0000 max=8813 max_rank=0 min=7553 min_rank=1 imbalance_ratio=1.075806 max_load_reduction=56723 max_load_reduction_pct=0.865524 moved_to_echo=32796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 3059, 0, 1, 4047, 0, 2, 2, 849, 320, 177, 125, 12, 4, 0, 0, 530, 3239, 0, 2, 2, 2, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 3, 2, 4094, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 0, 2, 3917, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 4041, 0, 2, 0, 2, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4090, 2, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 378, 2, 3774, 4093, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8600, 7841, 8201, 8179, 8120, 8127, 8217, 8251] avg=8192.0000 max=8600 max_rank=0 min=7841 min_rank=1 imbalance_ratio=1.049805 max_load_reduction=56936 max_load_reduction_pct=0.868774 moved_to_echo=32715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2131, 0, 1, 4096, 0, 0, 0, 850, 320, 177, 125, 12, 4, 0, 0, 1536, 3241, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 17, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 4022, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 3776, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7716, 8871, 8197, 8195, 8015, 8055, 8189, 8298] avg=8192.0000 max=8871 max_rank=1 min=7716 min_rank=0 imbalance_ratio=1.082886 max_load_reduction=56665 max_load_reduction_pct=0.864639 moved_to_echo=32557 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1970, 0, 0, 4096, 0, 0, 0, 849, 320, 177, 125, 12, 4, 0, 0, 1818, 3240, 11, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4088, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4083, 0, 1, 34, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 4096, 0, 1, 3919, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 3954, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 3775, 4090, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7553, 9157, 8191, 8213, 8104, 8055, 8189, 8074] avg=8192.0000 max=9157 max_rank=1 min=7553 min_rank=0 imbalance_ratio=1.117798 max_load_reduction=56379 max_load_reduction_pct=0.860275 moved_to_echo=32422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2881, 0, 0, 4094, 0, 1, 1, 849, 320, 177, 125, 12, 4, 0, 0, 657, 3238, 1, 1, 1, 1, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4081, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 4083, 0, 5, 26, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4091, 0, 0, 3918, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 4087, 0, 1, 1, 1, 1, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 1, 3775, 4095, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8464, 7969, 8183, 8209, 8133, 8324, 8185, 8069] avg=8192.0000 max=8464 max_rank=0 min=7969 min_rank=1 imbalance_ratio=1.033203 max_load_reduction=57072 max_load_reduction_pct=0.870850 moved_to_echo=32711 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2295, 0, 0, 4089, 0, 1, 1, 848, 320, 177, 125, 12, 4, 0, 0, 1116, 3236, 0, 1, 1, 1, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 4095, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 4082, 1, 0, 8, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 4095, 0, 1, 3918, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 4072, 0, 1, 0, 1, 1, 348, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 1, 3775, 4095, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7872, 8430, 8198, 8186, 8036, 8391, 8189, 8234] avg=8192.0000 max=8430 max_rank=1 min=7872 min_rank=0 imbalance_ratio=1.029053 max_load_reduction=57106 max_load_reduction_pct=0.871368 moved_to_echo=32850 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2664, 0, 0, 4096, 0, 0, 0, 850, 323, 179, 126, 13, 6, 0, 0, 601, 3220, 27, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4066, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 4083, 0, 6, 25, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 43, 4096, 0, 0, 3917, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 3923, 0, 0, 0, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 3766, 4068, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 7905, 8183, 8190, 8336, 8152, 8188, 8325] avg=8192.0000 max=8336 max_rank=4 min=7905 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=33001 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2688, 128, 30208, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[7040, 25984, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[31872, 32000, 0, 128, 128, 128, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 31360, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 20992, 0, 128, 32768, 0, 128, 128, 6912, 2688, 1536, 1024, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66688 slack_slots=3584 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4091, 0, 0, 0, 191, 0, 0, 0, 0, 4096, 0, 3, 1871, 4096, 0, 0, 0, 62, 0, 0, 604, 4096, 0, 4096, 0, 0, 0, 0, 4096, 446, 0, 4093, 0, 0, 4096, 801, 4096, 0, 0, 87, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2693, 0, 247, 0, 1139, 64, 0, 0, 0, 4088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 105, 0, 0, 0, 0, 4096, 0, 3, 496, 4096, 0, 0, 0, 133, 0, 0, 244, 4096, 0, 4096, 0, 0, 0, 0, 4092, 1182, 0, 4089, 0, 1, 4096, 2033, 4096, 0, 0, 12, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3022, 0, 563, 0, 422, 110, 0, 0, 0, 3973] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 4095, 0, 1, 0, 12, 0, 0, 4, 1, 4038, 0, 14, 345, 4095, 1, 1, 0, 205, 0, 0, 629, 4095, 1, 4095, 0, 1, 1, 1, 4091, 1007, 0, 4015, 1, 24, 4095, 772, 4094, 1, 1, 21, 4094, 4095, 1, 0, 0, 0, 1, 0, 4095, 1, 0, 3903, 1, 199, 1, 913, 292, 0, 0, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4088, 0, 2, 0, 21, 1, 0, 0, 2, 4067, 0, 72, 1557, 4094, 2, 0, 0, 139, 1, 0, 963, 4094, 2, 4094, 1, 2, 0, 2, 4093, 636, 2, 4064, 2, 9, 4093, 455, 4093, 2, 0, 39, 4094, 4093, 2, 4, 0, 3, 2, 0, 4094, 0, 0, 3200, 2, 153, 2, 634, 452, 0, 0, 0, 4015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 46, 1112, 4096, 0, 0, 0, 4, 0, 1, 12, 4096, 0, 4096, 0, 0, 0, 0, 4091, 1829, 0, 3994, 0, 26, 4096, 320, 4096, 0, 0, 168, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3116, 0, 934, 0, 391, 347, 0, 20, 0, 4080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 4093, 0, 1, 0, 28, 0, 0, 9, 1, 4086, 0, 48, 1341, 4095, 1, 1, 0, 20, 0, 0, 110, 4095, 1, 4095, 0, 1, 1, 1, 4091, 1079, 0, 4084, 1, 67, 4095, 274, 4095, 1, 1, 6, 4095, 4095, 1, 0, 1, 0, 1, 0, 4095, 1, 0, 2475, 1, 934, 1, 983, 842, 0, 0, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 49, 0, 0, 5, 0, 4093, 0, 216, 420, 4096, 0, 0, 0, 160, 0, 11, 273, 4096, 0, 4096, 0, 0, 0, 0, 4059, 924, 0, 4064, 0, 0, 4096, 625, 4096, 0, 0, 23, 4096, 4096, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 3741, 0, 176, 0, 954, 702, 0, 15, 0, 4080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 10, 0, 0, 28, 0, 4091, 0, 1, 1535, 4096, 0, 0, 0, 25, 0, 0, 21, 4096, 0, 4096, 0, 0, 0, 0, 4094, 498, 0, 4050, 0, 108, 4096, 1222, 4096, 0, 0, 76, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2250, 0, 236, 0, 382, 1856, 0, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4047, 4096, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 496, 2517, 451, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 133, 0, 0, 244, 3645, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1182, 0, 4089, 0, 1, 0, 2033, 2344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4042, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3022, 0, 942, 54, 49, 0, 0, 0, 0, 0, 563, 0, 422, 110, 0, 0, 0, 3973, 1572, 810, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 7563, 8118, 8195, 9649, 8150, 8163, 7450] avg=8192.0000 max=9649 max_rank=4 min=7450 min_rank=7 imbalance_ratio=1.177856 max_load_reduction=55887 max_load_reduction_pct=0.852768 moved_to_echo=33412 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4047, 4091, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 1871, 2517, 452, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 62, 0, 0, 604, 3644, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 4093, 0, 0, 0, 801, 2344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 4042, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2693, 0, 942, 54, 49, 0, 0, 0, 0, 0, 247, 0, 1139, 64, 0, 0, 0, 4088, 1572, 810, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8329, 8939, 8406, 8199, 7684, 8225, 7834, 7920] avg=8192.0000 max=8939 max_rank=1 min=7684 min_rank=4 imbalance_ratio=1.091187 max_load_reduction=56597 max_load_reduction_pct=0.863602 moved_to_echo=31663 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4045, 4088, 0, 2, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 4067, 0, 72, 1557, 2516, 452, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 139, 1, 0, 963, 3641, 0, 0, 0, 0, 0, 0, 0, 7, 2, 4094, 1, 2, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 636, 2, 4064, 2, 9, 0, 455, 2342, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 39, 4040, 4093, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 4094, 0, 0, 3200, 2, 941, 54, 49, 0, 0, 0, 0, 0, 153, 2, 634, 452, 0, 0, 0, 4015, 1571, 810, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8667, 8840, 8201, 7510, 8180, 8345, 7637] avg=8192.0000 max=8840 max_rank=2 min=7510 min_rank=4 imbalance_ratio=1.079102 max_load_reduction=56696 max_load_reduction_pct=0.865112 moved_to_echo=31672 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4046, 4095, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4038, 0, 14, 345, 2516, 452, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 205, 0, 0, 629, 3643, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4095, 0, 1, 1, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 0, 4015, 1, 24, 0, 772, 2342, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 21, 4040, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 3903, 1, 942, 54, 49, 0, 0, 0, 0, 0, 199, 1, 913, 292, 0, 0, 0, 4087, 1572, 810, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 7370, 8574, 8197, 8161, 8159, 9046, 7874] avg=8192.0000 max=9046 max_rank=6 min=7370 min_rank=1 imbalance_ratio=1.104248 max_load_reduction=56490 max_load_reduction_pct=0.861969 moved_to_echo=33240 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4047, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 46, 1112, 2517, 452, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 1, 12, 3644, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 0, 3994, 0, 26, 0, 320, 2344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 4042, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3116, 0, 942, 54, 49, 0, 0, 0, 0, 0, 934, 0, 391, 347, 0, 20, 0, 4080, 1572, 810, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8215, 7757, 8194, 8513, 8306, 8256, 8154] avg=8192.0000 max=8513 max_rank=4 min=7757 min_rank=2 imbalance_ratio=1.039185 max_load_reduction=57023 max_load_reduction_pct=0.870102 moved_to_echo=33229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4047, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 4091, 0, 1, 1535, 2517, 451, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 21, 3645, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 498, 0, 4050, 0, 108, 0, 1222, 2344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 4042, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2250, 0, 942, 54, 49, 0, 0, 0, 0, 0, 236, 0, 382, 1856, 0, 0, 0, 4094, 1572, 810, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8623, 7787, 8197, 8222, 8213, 7391, 8950] avg=8192.0000 max=8950 max_rank=7 min=7391 min_rank=6 imbalance_ratio=1.092529 max_load_reduction=56586 max_load_reduction_pct=0.863434 moved_to_echo=32776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4046, 4093, 0, 1, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 4086, 0, 48, 1341, 2516, 452, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 20, 0, 0, 110, 3643, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4095, 0, 1, 1, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 0, 4084, 1, 67, 0, 274, 2343, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 6, 4041, 4095, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 2475, 1, 942, 54, 49, 0, 0, 0, 0, 0, 934, 1, 983, 842, 0, 0, 0, 4093, 1572, 810, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8453, 7870, 8197, 7848, 8146, 7618, 9235] avg=8192.0000 max=9235 max_rank=7 min=7618 min_rank=6 imbalance_ratio=1.127319 max_load_reduction=56301 max_load_reduction_pct=0.859085 moved_to_echo=32847 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4041, 4095, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4093, 0, 216, 420, 2520, 452, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 160, 0, 11, 273, 3644, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 924, 0, 4064, 0, 0, 0, 625, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4038, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 3741, 0, 946, 58, 55, 0, 0, 0, 0, 0, 176, 0, 954, 702, 0, 15, 0, 4080, 1575, 814, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 7706, 8184, 8156, 7949, 8157, 8883, 8316] avg=8192.0000 max=8883 max_rank=6 min=7706 min_rank=1 imbalance_ratio=1.084351 max_load_reduction=56653 max_load_reduction_pct=0.864456 moved_to_echo=33305 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32384, 32768, 0, 128, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 512, 32384, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 0, 768, 128, 128, 2944, 29184, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 512, 8704, 20224, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7680, 128, 32512, 128, 256, 0, 6528, 18816, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3456, 128, 5888, 4736, 0, 128, 0, 32512, 12672, 6528, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 24448, 128, 7552, 512, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 49, 0, 0, 1, 38, 0, 4096, 4096, 0, 3832, 4096, 0, 2, 0, 0, 4096, 4096, 0, 4096, 0, 0, 3918, 4096, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 18, 0, 0, 0, 328, 0, 0, 4094, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 11, 2, 234, 1, 0, 2, 230, 3, 4092, 4094, 1, 3853, 4089, 2, 25, 0, 2, 4054, 4094, 1, 4093, 0, 0, 3679, 4094, 2, 2, 4090, 2, 0, 1, 0, 29, 3, 2, 13, 1, 0, 2, 111, 0, 0, 4094, 0, 1, 2, 4090, 4091, 6, 1, 0, 4092, 54, 0, 0, 2, 2, 1, 0, 4091, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 45, 0, 0, 0, 101, 0, 4096, 4096, 0, 3988, 4096, 0, 5, 0, 0, 4096, 4094, 0, 4096, 0, 0, 3989, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 62, 0, 0, 4096, 0, 0, 0, 4096, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 174, 0, 0, 0, 272, 0, 4096, 4096, 0, 3916, 4096, 0, 5, 0, 0, 4095, 4096, 0, 4096, 0, 0, 3781, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 46, 0, 0, 4096, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 302, 0, 0, 0, 159, 0, 4096, 4094, 0, 3745, 4096, 0, 47, 0, 0, 4096, 4085, 0, 4096, 0, 0, 3776, 4096, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 3, 0, 0, 0, 172, 1, 0, 4093, 0, 0, 0, 4096, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 2, 1, 101, 0, 0, 0, 140, 1, 4094, 4094, 0, 4018, 4094, 1, 2, 0, 1, 4073, 4094, 0, 4095, 0, 0, 3796, 4095, 1, 1, 4095, 1, 0, 1, 0, 33, 1, 1, 38, 0, 0, 1, 70, 0, 1, 4095, 1, 1, 1, 4094, 4095, 1, 1, 0, 4094, 13, 0, 0, 1, 1, 1, 0, 4095, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 248, 1, 0, 0, 459, 1, 4095, 4093, 0, 3908, 4094, 1, 5, 0, 1, 4079, 4095, 0, 4095, 0, 0, 3502, 4095, 1, 1, 4094, 1, 0, 1, 0, 9, 1, 1, 3, 0, 0, 1, 49, 0, 2, 4095, 0, 0, 1, 4094, 4080, 4, 1, 0, 4094, 35, 0, 0, 1, 1, 1, 0, 4095, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 213, 0, 10, 0, 148, 0, 4096, 4083, 0, 3970, 4096, 0, 9, 0, 0, 4096, 4079, 0, 4096, 0, 0, 3644, 4096, 0, 0, 4089, 0, 0, 0, 0, 29, 0, 0, 23, 0, 0, 0, 166, 9, 0, 4086, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 7, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 11, 2, 234, 1, 0, 2, 4088, 3892, 27, 0, 0, 0, 0, 0, 230, 3, 0, 4094, 1, 3853, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 2, 4054, 4094, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3679, 336, 2, 2, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 29, 3, 2, 13, 1, 0, 4092, 4065, 0, 0, 0, 0, 0, 0, 2, 111, 0, 0, 4094, 0, 1, 2, 3758, 198, 10, 0, 0, 0, 0, 0, 4080, 4091, 6, 1, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 0, 4091, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8184, 8179, 8111, 8206, 8176, 8232, 8189] avg=8192.0000 max=8259 max_rank=0 min=8111 min_rank=3 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32803 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 49, 0, 0, 1, 4095, 3895, 27, 0, 0, 0, 0, 0, 38, 0, 0, 4096, 0, 3832, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 337, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 18, 0, 0, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 328, 0, 0, 4094, 0, 0, 0, 3759, 198, 10, 1, 0, 0, 0, 0, 4086, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8067, 7967, 8197, 8351, 8192, 8390, 8181, 8191] avg=8192.0000 max=8390 max_rank=5 min=7967 min_rank=1 imbalance_ratio=1.024170 max_load_reduction=57146 max_load_reduction_pct=0.871979 moved_to_echo=32954 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 174, 0, 0, 0, 4095, 3895, 27, 0, 0, 0, 0, 0, 272, 0, 0, 4096, 0, 3916, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4095, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3781, 337, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 4096, 0, 0, 0, 3759, 198, 10, 1, 0, 0, 0, 0, 4086, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8285, 8199, 8213, 8166, 8110, 8181, 8191] avg=8192.0000 max=8285 max_rank=1 min=8110 min_rank=5 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32648 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 45, 0, 0, 0, 4096, 3895, 28, 0, 0, 0, 0, 0, 101, 0, 0, 4096, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 337, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 4096, 0, 0, 0, 3759, 201, 10, 1, 0, 0, 0, 0, 4086, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8064, 8185, 8195, 8422, 8173, 8129, 8177, 8191] avg=8192.0000 max=8422 max_rank=3 min=8064 min_rank=0 imbalance_ratio=1.028076 max_load_reduction=57114 max_load_reduction_pct=0.871490 moved_to_echo=32670 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 302, 0, 0, 0, 4096, 3895, 27, 0, 0, 0, 0, 0, 159, 0, 0, 4094, 0, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 4096, 4085, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3776, 337, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3, 0, 0, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 172, 1, 0, 4093, 0, 0, 0, 3759, 198, 10, 1, 0, 0, 0, 0, 4086, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 7998, 8231, 8209, 8172, 8234, 8179, 8193] avg=8192.0000 max=8320 max_rank=0 min=7998 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 1, 101, 0, 0, 0, 4094, 3896, 28, 0, 0, 0, 0, 0, 140, 1, 0, 4094, 0, 4018, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4073, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 336, 1, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 33, 1, 1, 38, 0, 0, 4094, 4066, 0, 0, 0, 0, 0, 0, 1, 70, 0, 1, 4095, 1, 1, 1, 3759, 199, 10, 0, 0, 0, 0, 0, 4084, 4095, 1, 1, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8254, 8170, 8230, 8234, 8138, 8194, 8193] avg=8192.0000 max=8254 max_rank=1 min=8123 min_rank=0 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 248, 1, 0, 0, 4094, 3894, 27, 0, 0, 0, 0, 0, 459, 1, 0, 4093, 0, 3908, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 4079, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 336, 1, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 1, 1, 3, 0, 0, 4095, 4067, 0, 0, 0, 0, 0, 0, 1, 49, 0, 2, 4095, 0, 0, 1, 3759, 200, 10, 0, 0, 0, 0, 0, 4084, 4080, 4, 1, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8462, 8181, 7935, 8177, 8117, 8204, 8193] avg=8192.0000 max=8462 max_rank=1 min=7935 min_rank=3 imbalance_ratio=1.032959 max_load_reduction=57074 max_load_reduction_pct=0.870880 moved_to_echo=32691 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 213, 0, 10, 0, 4096, 3898, 28, 0, 0, 0, 0, 0, 148, 0, 0, 4083, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3644, 332, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 23, 0, 0, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 166, 9, 0, 4086, 0, 0, 0, 3764, 198, 15, 4, 0, 0, 0, 0, 4081, 4095, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8201, 8184, 8065, 8216, 8242, 8188, 8195] avg=8192.0000 max=8245 max_rank=0 min=8065 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32841 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 30208, 2688, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 0, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 1024, 128, 128, 32768, 128, 128, 128, 30080, 1664, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1664, 128, 0, 32768, 128, 31232, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 32768, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 1408, 128, 128, 128, 32768, 31232, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3, 0, 1, 4093, 0, 2, 0, 4094, 1, 3341, 2, 2, 4089, 0, 2, 4075, 4088, 0, 3, 4093, 2, 0, 4083, 4094, 1, 0, 11, 0, 2, 4094, 119, 0, 4081, 3, 2194, 0, 2, 0, 0, 0, 0, 0, 3303, 0, 2, 1, 0, 0, 4093, 1, 2, 0, 0, 2, 0, 2, 0, 4093, 4094, 0, 3365, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 1, 0, 4095, 1, 3519, 1, 1, 4090, 0, 1, 4087, 4082, 0, 0, 4095, 1, 0, 4095, 4095, 7, 0, 6, 3, 1, 4095, 93, 0, 4071, 5, 1269, 0, 1, 0, 0, 0, 0, 0, 3713, 0, 1, 1, 0, 0, 4094, 1, 1, 0, 1, 1, 0, 1, 0, 4095, 4095, 1, 3719, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3909, 0, 0, 4096, 9, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4064, 4096, 85, 0, 0, 9, 0, 4096, 240, 0, 3952, 0, 1983, 0, 0, 0, 0, 0, 0, 0, 3177, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3052, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3970, 0, 0, 4089, 2, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4090, 4096, 51, 0, 0, 3, 0, 4096, 177, 0, 3889, 0, 1586, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2874, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3778, 0, 0, 4091, 12, 0, 4096, 4096, 0, 0, 4095, 0, 0, 4072, 4096, 121, 0, 0, 31, 0, 4096, 172, 0, 3986, 0, 3831, 0, 0, 0, 0, 1, 0, 0, 4007, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 473, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4096, 0, 2613, 0, 0, 4096, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4095, 4096, 46, 0, 1, 1, 0, 4096, 56, 0, 4093, 0, 2523, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3190, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 1, 1, 0, 4095, 1, 3372, 1, 1, 4094, 0, 1, 4084, 4092, 0, 0, 4095, 1, 0, 4087, 4095, 13, 0, 2, 3, 1, 4095, 46, 0, 4093, 1, 2415, 0, 1, 0, 0, 0, 0, 0, 3784, 0, 1, 1, 0, 0, 4095, 1, 1, 0, 2, 1, 0, 1, 0, 4093, 4095, 0, 2673, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 15, 0, 0, 4096, 0, 3568, 0, 0, 4080, 38, 0, 4096, 4096, 0, 0, 4084, 0, 0, 4026, 4090, 201, 1, 0, 43, 0, 4096, 47, 0, 3609, 0, 2197, 0, 0, 0, 0, 12, 5, 0, 3773, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 4096, 4096, 0, 2964, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 0, 4089, 2, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 3, 0, 4096, 177, 0, 3078, 823, 0, 0, 0, 0, 0, 0, 3889, 0, 1586, 0, 0, 0, 0, 0, 1965, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 4096, 413, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3660, 434, 0, 0, 0, 0, 0, 0, 0, 4096, 1308, 0, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8645, 8209, 8228, 7442, 8354, 8190, 8278] avg=8192.0000 max=8645 max_rank=1 min=7442 min_rank=4 imbalance_ratio=1.055298 max_load_reduction=56891 max_load_reduction_pct=0.868088 moved_to_echo=32264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 1, 4093, 0, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3341, 2, 2, 4089, 0, 2, 582, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 3, 23, 2, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 2, 4094, 119, 0, 3062, 822, 0, 0, 0, 0, 0, 0, 4081, 3, 2194, 0, 2, 0, 0, 0, 1964, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 0, 2, 1, 0, 0, 4094, 412, 0, 0, 0, 0, 0, 0, 4093, 1, 2, 0, 0, 2, 0, 2, 3658, 431, 0, 0, 0, 0, 0, 0, 0, 4093, 1308, 0, 3365, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8019, 8199, 8111, 8246, 7812, 8189, 8769] avg=8192.0000 max=8769 max_rank=7 min=7812 min_rank=5 imbalance_ratio=1.070435 max_load_reduction=56767 max_load_reduction_pct=0.866196 moved_to_echo=33016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 0, 0, 4096, 9, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 9, 0, 4096, 240, 0, 3078, 823, 0, 0, 0, 0, 0, 0, 3952, 0, 1983, 0, 0, 0, 0, 0, 1965, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3177, 0, 0, 0, 0, 0, 4096, 413, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3660, 434, 0, 0, 0, 0, 0, 0, 0, 4096, 1308, 0, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8598, 8183, 8331, 7902, 7686, 8190, 8456] avg=8192.0000 max=8598 max_rank=1 min=7686 min_rank=5 imbalance_ratio=1.049561 max_load_reduction=56938 max_load_reduction_pct=0.868805 moved_to_echo=32234 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3519, 1, 1, 4090, 0, 1, 583, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 23, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 6, 3, 1, 4095, 93, 0, 3071, 823, 0, 0, 0, 0, 0, 0, 4071, 5, 1269, 0, 1, 0, 0, 0, 1964, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3713, 0, 1, 1, 0, 0, 4095, 413, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 1, 1, 0, 1, 3659, 433, 0, 0, 0, 0, 0, 0, 0, 4095, 1308, 1, 3719, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8196, 8201, 8099, 7312, 8223, 8191, 9124] avg=8192.0000 max=9124 max_rank=7 min=7312 min_rank=4 imbalance_ratio=1.113770 max_load_reduction=56412 max_load_reduction_pct=0.860779 moved_to_echo=32850 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 0, 0, 4096, 1, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 1, 1, 0, 4096, 56, 0, 3078, 823, 0, 0, 0, 0, 0, 0, 4093, 0, 2523, 0, 0, 0, 0, 0, 1965, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 4096, 413, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3660, 434, 0, 0, 0, 0, 0, 0, 0, 4096, 1308, 0, 3190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7294, 8214, 8101, 8583, 8369, 8190, 8594] avg=8192.0000 max=8594 max_rank=7 min=7294 min_rank=1 imbalance_ratio=1.049072 max_load_reduction=56942 max_load_reduction_pct=0.868866 moved_to_echo=33736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 0, 4091, 12, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 31, 0, 4096, 172, 0, 3078, 823, 0, 0, 0, 0, 0, 0, 3986, 0, 3831, 0, 0, 0, 0, 1, 1965, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 4096, 413, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 3659, 434, 0, 0, 0, 0, 0, 0, 0, 4096, 1308, 0, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8465, 8191, 8321, 9785, 8516, 8191, 5877] avg=8192.0000 max=9785 max_rank=4 min=5877 min_rank=7 imbalance_ratio=1.194458 max_load_reduction=55751 max_load_reduction_pct=0.850693 moved_to_echo=32369 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 1, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3372, 1, 1, 4094, 0, 1, 583, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 23, 1, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 2, 3, 1, 4095, 46, 0, 3069, 823, 0, 0, 0, 0, 0, 0, 4093, 1, 2415, 0, 1, 0, 0, 0, 1964, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3784, 0, 1, 1, 0, 0, 4095, 413, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 2, 1, 0, 1, 3659, 432, 0, 0, 0, 0, 0, 0, 0, 4093, 1308, 0, 2673, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8053, 8203, 8052, 8476, 8294, 8192, 8075] avg=8192.0000 max=8476 max_rank=4 min=8052 min_rank=3 imbalance_ratio=1.034668 max_load_reduction=57060 max_load_reduction_pct=0.870667 moved_to_echo=33037 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 15, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3568, 0, 0, 4080, 38, 0, 580, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 1, 0, 43, 0, 4096, 47, 0, 3080, 825, 0, 0, 0, 0, 0, 0, 3609, 0, 2197, 0, 0, 0, 0, 12, 1968, 4, 0, 0, 0, 0, 0, 0, 5, 0, 3773, 0, 0, 0, 0, 0, 4090, 414, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 3656, 436, 0, 0, 0, 0, 0, 0, 0, 4096, 1303, 0, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8266, 8136, 8293, 7790, 8282, 8203, 8363] avg=8192.0000 max=8363 max_rank=7 min=7790 min_rank=4 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 256, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 10496, 128, 22400, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 128, 29312, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 29568, 0, 128, 128, 0, 0, 32768, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 128, 128, 128, 128, 32768, 1024, 0, 24704, 6656, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31872, 128, 18048, 0, 128, 0, 0, 128, 15744, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 28160, 128, 128, 32768, 128, 128, 4736, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 0, 23, 2, 0, 3767, 3938, 0, 2, 2, 1, 0, 2, 2, 4, 4087, 2, 4, 2, 3, 4093, 2, 2, 1, 33, 4094, 5, 1, 4092, 0, 0, 4083, 0, 0, 2, 102, 4049, 1918, 5, 1, 4084, 4091, 0, 191, 2, 0, 12, 2, 0, 0, 4094, 2, 2, 4094, 2550, 0, 3903, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4068, 4045, 0, 0, 0, 0, 35, 0, 9, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4035, 893, 0, 0, 4092, 4096, 0, 25, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 3287, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4064, 3911, 0, 0, 0, 0, 0, 0, 21, 0, 3967, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4084, 1113, 0, 0, 4096, 4096, 0, 28, 0, 1, 1, 0, 0, 0, 4096, 0, 0, 4095, 3291, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 8, 1, 0, 3926, 4030, 0, 1, 1, 0, 0, 1, 54, 1, 4067, 1, 0, 1, 1, 4094, 1, 1, 1, 1, 4095, 3, 1, 4095, 0, 0, 4086, 0, 0, 1, 19, 4093, 498, 2, 0, 4094, 4093, 0, 91, 1, 0, 4, 1, 0, 0, 4095, 1, 1, 4095, 3769, 0, 4017, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4093, 4077, 0, 0, 0, 0, 66, 0, 20, 0, 4053, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 4070, 531, 6, 0, 4094, 4096, 0, 489, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 4090, 3093, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 4055, 0, 0, 0, 0, 1, 0, 10, 0, 3833, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 301, 0, 0, 4096, 4096, 0, 1278, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 2836, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 0, 8, 1, 0, 4000, 4012, 0, 1, 1, 0, 0, 1, 37, 1, 4083, 0, 0, 1, 0, 4094, 1, 1, 1, 15, 4095, 3, 1, 4095, 0, 0, 4089, 0, 0, 1, 16, 4091, 621, 6, 0, 4094, 4093, 0, 239, 1, 0, 24, 1, 0, 0, 4095, 1, 1, 4095, 3397, 0, 4028, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 4093, 4049, 0, 0, 0, 0, 56, 0, 87, 0, 3946, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4086, 744, 0, 0, 4096, 4096, 0, 418, 0, 2, 1, 0, 0, 0, 4089, 0, 0, 4085, 3039, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 0, 23, 4062, 24, 0, 0, 0, 0, 0, 0, 2, 0, 3767, 3938, 0, 2, 2, 1, 164, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4, 4087, 2, 4, 2, 4092, 29, 0, 0, 0, 0, 0, 0, 3, 4093, 2, 2, 1, 33, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 4083, 0, 0, 2, 1, 1, 0, 0, 0, 0, 0, 0, 102, 4049, 1918, 5, 1, 3263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 2, 0, 12, 2, 0, 0, 4094, 3089, 656, 0, 0, 0, 0, 0, 0, 2, 2, 981, 2550, 0, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 7876, 8224, 8232, 8180, 9338, 8046, 7438] avg=8192.0000 max=9338 max_rank=5 min=7438 min_rank=7 imbalance_ratio=1.139893 max_load_reduction=56198 max_load_reduction_pct=0.857513 moved_to_echo=33002 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4068, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 3911, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 3967, 0, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 4084, 1113, 0, 0, 3273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 1, 1, 0, 0, 0, 4096, 3090, 658, 0, 0, 0, 0, 0, 0, 0, 0, 981, 3291, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8139, 8112, 8190, 8195, 8470, 7874, 8368] avg=8192.0000 max=8470 max_rank=5 min=7874 min_rank=6 imbalance_ratio=1.033936 max_load_reduction=57066 max_load_reduction_pct=0.870758 moved_to_echo=32907 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4067, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4045, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 35, 0, 9, 0, 4090, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 4035, 893, 0, 0, 3270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 4096, 3088, 657, 0, 0, 0, 0, 0, 0, 0, 0, 981, 3287, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8277, 8259, 8190, 8195, 8198, 7866, 8364] avg=8192.0000 max=8364 max_rank=7 min=7866 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32623 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4067, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4055, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 0, 3833, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 301, 0, 0, 3273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 0, 0, 0, 0, 4096, 3085, 658, 0, 0, 0, 0, 0, 0, 0, 0, 980, 2836, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8298, 7969, 8189, 8195, 7669, 9117, 7912] avg=8192.0000 max=9117 max_rank=6 min=7669 min_rank=5 imbalance_ratio=1.112915 max_load_reduction=56419 max_load_reduction_pct=0.860886 moved_to_echo=32893 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 4067, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4077, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 66, 0, 20, 0, 4053, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 4070, 531, 6, 0, 3272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 2, 0, 0, 0, 0, 4095, 3086, 657, 0, 0, 0, 0, 0, 0, 0, 0, 980, 3093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8334, 8264, 8177, 8196, 7879, 8329, 8169] avg=8192.0000 max=8334 max_rank=1 min=7879 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32573 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 8, 4065, 24, 0, 0, 0, 0, 0, 0, 1, 0, 3926, 4030, 0, 1, 1, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 1, 54, 1, 4067, 1, 0, 1, 4095, 28, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 1, 1, 1, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4086, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 19, 4093, 498, 2, 0, 3272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 1, 0, 4, 1, 0, 0, 4095, 3090, 657, 0, 0, 0, 0, 0, 0, 1, 1, 981, 3769, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8123, 8248, 8196, 8185, 7884, 7939, 8769] avg=8192.0000 max=8769 max_rank=7 min=7884 min_rank=5 imbalance_ratio=1.070435 max_load_reduction=56767 max_load_reduction_pct=0.866196 moved_to_echo=32777 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 0, 8, 4064, 24, 0, 0, 0, 0, 0, 0, 1, 0, 4000, 4012, 0, 1, 1, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 1, 37, 1, 4083, 0, 0, 1, 4095, 29, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 1, 15, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4089, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 16, 4091, 621, 6, 0, 3272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 1, 0, 24, 1, 0, 0, 4095, 3090, 657, 0, 0, 0, 0, 0, 0, 1, 1, 981, 3397, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8179, 8247, 8209, 8188, 8006, 8107, 8408] avg=8192.0000 max=8408 max_rank=7 min=8006 min_rank=5 imbalance_ratio=1.026367 max_load_reduction=57128 max_load_reduction_pct=0.871704 moved_to_echo=32709 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 11, 0, 4068, 25, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4049, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 56, 0, 87, 0, 3946, 0, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 4086, 744, 0, 0, 3262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 2, 1, 0, 0, 0, 4089, 3087, 661, 0, 0, 0, 0, 0, 0, 0, 0, 973, 3039, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8310, 8213, 8153, 8202, 8092, 8258, 8108] avg=8192.0000 max=8310 max_rank=1 min=8092 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32660 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 128, 128, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 7936, 25344, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 0, 32768, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 32640, 6656, 128, 128, 26240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 32128, 32128, 0, 128, 128, 128, 1408, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 128, 256, 128, 32128, 128, 128, 128, 32768, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2816, 128, 128, 128, 128, 0, 0, 32768, 24832, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4093, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 4058, 2286, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3360, 0, 0, 0, 3926, 0, 0, 0, 0, 7, 0, 0, 0, 2766, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4094, 4094, 0, 0, 0, 0, 0, 4095, 8, 0, 0, 0, 0, 4046, 2324, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 2991, 0, 0, 0, 4014, 0, 0, 0, 0, 40, 0, 0, 0, 3119, 0, 0, 4096, 3939, 0, 1, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4090, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4081, 2728, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3753, 0, 0, 0, 3993, 0, 0, 0, 0, 1, 0, 0, 0, 1832, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 3545, 1, 0, 3, 1, 0, 4095, 0, 1, 0, 2, 0, 4016, 3090, 1, 1, 1, 1, 1, 1, 4095, 0, 1, 1, 0, 1, 0, 4092, 4095, 0, 1, 2968, 3, 1, 0, 4094, 0, 0, 0, 1, 138, 0, 1, 0, 2616, 0, 2, 4091, 4095, 0, 4, 0, 0, 0, 1, 4095, 0, 4095, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4094, 0, 4094, 3538, 1, 0, 10, 2, 2, 4094, 0, 2, 0, 1, 3, 3981, 2710, 0, 2, 2, 2, 0, 2, 4093, 3, 2, 2, 0, 2, 0, 4088, 4092, 0, 1, 2891, 7, 2, 0, 4067, 0, 0, 0, 2, 246, 0, 2, 2, 3042, 0, 6, 4084, 4079, 1, 0, 2, 0, 0, 1, 4092, 0, 4094, 0, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 3392, 0, 0, 1, 1, 0, 4095, 0, 1, 0, 10, 0, 4000, 2901, 1, 1, 1, 1, 1, 1, 4095, 1, 1, 1, 0, 1, 0, 4092, 4095, 0, 1, 3041, 2, 1, 0, 4070, 0, 0, 0, 1, 175, 0, 1, 0, 2888, 0, 3, 4093, 4087, 0, 4, 0, 0, 0, 1, 4095, 0, 4095, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4045, 2936, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3068, 0, 0, 0, 4079, 0, 0, 0, 0, 82, 0, 0, 0, 2245, 0, 0, 4096, 4023, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4078, 4092, 0, 0, 0, 0, 0, 4094, 61, 0, 0, 0, 0, 4089, 1578, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 4096, 4086, 0, 0, 3804, 0, 0, 0, 4087, 0, 0, 0, 0, 11, 0, 0, 0, 2776, 0, 0, 4096, 4056, 0, 21, 0, 0, 0, 0, 4096, 18, 4096, 2, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[231, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 4058, 43, 0, 0, 0, 0, 0, 0, 0, 2286, 0, 0, 0, 0, 0, 0, 4096, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3360, 0, 0, 0, 3926, 0, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2766, 4096, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8420, 8212, 7907, 8192, 8199, 8216, 8199, 8191] avg=8192.0000 max=8420 max_rank=0 min=7907 min_rank=2 imbalance_ratio=1.027832 max_load_reduction=57116 max_load_reduction_pct=0.871521 moved_to_echo=32805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[231, 0, 4094, 3538, 1, 0, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 2, 0, 1, 3, 3981, 43, 0, 0, 0, 0, 0, 0, 0, 2710, 0, 2, 2, 2, 0, 2, 4093, 1524, 0, 0, 0, 0, 0, 0, 0, 3, 2, 2, 0, 2, 0, 4088, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2891, 7, 2, 0, 4067, 0, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 246, 0, 2, 2, 3042, 4094, 1346, 0, 0, 0, 0, 0, 0, 0, 6, 4084, 4079, 1, 0, 2, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7876, 8126, 8335, 8189, 7881, 8734, 8210, 8185] avg=8192.0000 max=8734 max_rank=5 min=7876 min_rank=0 imbalance_ratio=1.066162 max_load_reduction=56802 max_load_reduction_pct=0.866730 moved_to_echo=33010 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[231, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4045, 43, 0, 0, 0, 0, 0, 0, 0, 2936, 0, 0, 0, 0, 0, 0, 4096, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3068, 0, 0, 0, 4079, 0, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 2245, 4096, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4023, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8423, 8186, 8557, 8192, 8060, 7770, 8157, 8191] avg=8192.0000 max=8557 max_rank=2 min=7770 min_rank=5 imbalance_ratio=1.044556 max_load_reduction=56979 max_load_reduction_pct=0.869431 moved_to_echo=32178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[231, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4081, 43, 0, 0, 0, 0, 0, 0, 0, 2728, 0, 0, 0, 0, 0, 0, 4096, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 0, 0, 0, 3993, 0, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1832, 4096, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8417, 8226, 8349, 8192, 8659, 7276, 8226, 8191] avg=8192.0000 max=8659 max_rank=4 min=7276 min_rank=5 imbalance_ratio=1.057007 max_load_reduction=56877 max_load_reduction_pct=0.867874 moved_to_echo=32352 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[231, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 8, 0, 0, 0, 0, 4046, 43, 0, 0, 0, 0, 0, 0, 0, 2324, 0, 0, 0, 0, 0, 0, 4096, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2991, 0, 0, 0, 4014, 0, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 3119, 4096, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3939, 0, 1, 0, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8419, 8192, 7945, 8193, 7918, 8602, 8074, 8193] avg=8192.0000 max=8602 max_rank=5 min=7918 min_rank=4 imbalance_ratio=1.050049 max_load_reduction=56934 max_load_reduction_pct=0.868744 moved_to_echo=32787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[231, 0, 4095, 3545, 1, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 2, 0, 4016, 43, 0, 0, 0, 0, 0, 0, 0, 3090, 1, 1, 1, 1, 1, 1, 4095, 1524, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 4092, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2968, 3, 1, 0, 4094, 0, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 138, 0, 1, 0, 2616, 4095, 1347, 0, 0, 0, 0, 0, 0, 0, 2, 4091, 4095, 0, 4, 0, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7876, 8157, 8715, 8190, 7980, 8198, 8230, 8190] avg=8192.0000 max=8715 max_rank=2 min=7876 min_rank=0 imbalance_ratio=1.063843 max_load_reduction=56821 max_load_reduction_pct=0.867020 moved_to_echo=32598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[231, 0, 4095, 3392, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 10, 0, 4000, 43, 0, 0, 0, 0, 0, 0, 0, 2901, 1, 1, 1, 1, 1, 1, 4095, 1524, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 4092, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3041, 2, 1, 0, 4070, 0, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 175, 0, 1, 0, 2888, 4095, 1347, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 4087, 0, 4, 0, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7720, 8149, 8526, 8191, 8028, 8507, 8225, 8190] avg=8192.0000 max=8526 max_rank=2 min=7720 min_rank=0 imbalance_ratio=1.040771 max_load_reduction=57010 max_load_reduction_pct=0.869904 moved_to_echo=32950 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[215, 0, 4078, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 61, 0, 0, 0, 0, 4089, 44, 0, 0, 0, 0, 0, 0, 0, 1578, 0, 0, 0, 0, 0, 0, 4096, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 4086, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3804, 0, 0, 0, 4087, 0, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 2776, 4096, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4056, 0, 21, 0, 0, 39, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 2, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8385, 8288, 7202, 8197, 8811, 8233, 8215, 8205] avg=8192.0000 max=8811 max_rank=4 min=7202 min_rank=2 imbalance_ratio=1.075562 max_load_reduction=56725 max_load_reduction_pct=0.865555 moved_to_echo=33464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 32512, 128, 128, 128, 0, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 768, 0, 128, 128, 21376, 32768, 10880, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1920, 0, 32768, 30976, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 32384, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 25984, 128, 128, 0, 32384, 0, 7424, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[20608, 128, 128, 128, 128, 128, 128, 32768, 12288, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 2687, 11, 0, 0, 0, 0, 0, 4096, 1146, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 552, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 720, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1391, 0, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 2537, 0, 0, 0, 0, 0, 0, 4096, 1543, 0, 0, 0, 0, 13, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1988, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3357, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 2828, 0, 0, 0, 0, 0, 0, 4096, 318, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1395, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4073, 0, 5, 1, 1, 0, 4095, 4095, 724, 1, 1, 0, 0, 0, 1, 1, 0, 0, 3862, 42, 1, 36, 0, 4095, 0, 1, 4091, 4094, 0, 6, 4095, 6, 4094, 4095, 2162, 0, 0, 0, 0, 1, 1, 4095, 1067, 3, 0, 0, 0, 0, 4094, 4095, 0, 10, 1, 4090, 1, 0, 362, 1, 1, 4034, 0, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4032, 0, 20, 1, 2, 0, 4094, 4094, 288, 2, 1, 0, 0, 0, 2, 2, 0, 0, 3733, 137, 2, 91, 2, 4094, 0, 2, 4082, 4092, 0, 3, 4094, 12, 4093, 4094, 2663, 0, 0, 0, 0, 2, 2, 4085, 1357, 11, 0, 0, 0, 1, 4093, 4094, 0, 91, 2, 4073, 0, 1, 65, 2, 2, 3919, 1, 2, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 2760, 0, 0, 0, 0, 0, 0, 4096, 2677, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1656, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4076, 0, 14, 1, 1, 0, 4095, 4095, 847, 1, 1, 0, 0, 0, 1, 1, 0, 0, 3266, 29, 1, 64, 0, 4095, 0, 1, 4090, 4094, 0, 2, 4095, 2, 4095, 4095, 1821, 0, 0, 0, 0, 1, 1, 4093, 1940, 5, 0, 1, 0, 0, 4095, 4095, 0, 5, 1, 4088, 0, 0, 294, 1, 1, 4029, 1, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4089, 4078, 575, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2897, 8, 0, 0, 0, 4095, 15, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 2944, 0, 0, 0, 0, 0, 0, 4094, 411, 0, 0, 0, 0, 23, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1345, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 4096, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 3357, 0, 0, 0, 0, 4096, 1137, 8, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1545, 2828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 318, 0, 0, 0, 0, 1, 1414, 1369, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1395, 0, 0, 4096, 0, 0, 0, 0, 2715, 461, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8024, 8600, 8190, 8469, 7198, 8184, 8667] avg=8192.0000 max=8667 max_rank=7 min=7198 min_rank=5 imbalance_ratio=1.057983 max_load_reduction=56869 max_load_reduction_pct=0.867752 moved_to_echo=32518 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 4096, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 0, 0, 0, 0, 4096, 1137, 8, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1545, 2687, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1146, 0, 0, 0, 0, 0, 1414, 1369, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 0, 0, 4096, 0, 0, 0, 0, 2715, 460, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 7918, 8853, 8190, 8339, 8025, 8184, 7823] avg=8192.0000 max=8853 max_rank=2 min=7823 min_rank=7 imbalance_ratio=1.080688 max_load_reduction=56683 max_load_reduction_pct=0.864914 moved_to_echo=32371 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4032, 0, 20, 1, 2, 0, 12, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 288, 2, 1, 0, 0, 0, 2, 2, 4094, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 3733, 137, 2, 91, 2, 4094, 1136, 8, 2, 0, 0, 0, 0, 0, 0, 2, 4082, 4090, 0, 3, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1545, 2663, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4085, 1357, 11, 0, 0, 0, 1, 1413, 1368, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 91, 2, 4073, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 65, 2, 2, 3919, 1, 2, 1, 0, 2714, 459, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8024, 9205, 8189, 8303, 8237, 8252, 7165] avg=8192.0000 max=9205 max_rank=2 min=7165 min_rank=7 imbalance_ratio=1.123657 max_load_reduction=56331 max_load_reduction_pct=0.859543 moved_to_echo=31957 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 11, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 0, 0, 0, 1, 0, 0, 4096, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 1391, 0, 0, 0, 0, 4096, 1137, 8, 2, 0, 0, 0, 0, 0, 1, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1545, 2537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1543, 0, 0, 0, 0, 13, 1414, 1369, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1988, 0, 0, 4096, 0, 0, 0, 0, 2715, 460, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8453, 6634, 8191, 8178, 8435, 8184, 9259] avg=8192.0000 max=9259 max_rank=7 min=6634 min_rank=2 imbalance_ratio=1.130249 max_load_reduction=56277 max_load_reduction_pct=0.858719 moved_to_echo=34056 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 4096, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 0, 0, 0, 0, 4096, 1137, 8, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1545, 2760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2677, 0, 0, 0, 0, 0, 1414, 1369, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1656, 0, 0, 4096, 0, 0, 0, 0, 2715, 461, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 7769, 6304, 8190, 8401, 9556, 8184, 8928] avg=8192.0000 max=9556 max_rank=5 min=6304 min_rank=2 imbalance_ratio=1.166504 max_load_reduction=55980 max_load_reduction_pct=0.854187 moved_to_echo=35069 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 5, 1, 1, 0, 11, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 724, 1, 1, 0, 0, 0, 1, 1, 4095, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 3862, 42, 1, 36, 0, 4095, 1136, 8, 2, 0, 0, 0, 0, 0, 0, 1, 4091, 4092, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1546, 2162, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1067, 3, 0, 0, 0, 0, 1413, 1369, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 10, 1, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 1, 1, 4034, 0, 1, 1, 0, 2715, 460, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8458, 9182, 8196, 7803, 7948, 8188, 7575] avg=8192.0000 max=9182 max_rank=2 min=7575 min_rank=7 imbalance_ratio=1.120850 max_load_reduction=56354 max_load_reduction_pct=0.859894 moved_to_echo=31514 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4076, 0, 14, 1, 1, 0, 11, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 847, 1, 1, 0, 0, 0, 1, 1, 4095, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 3266, 29, 1, 64, 0, 4095, 1136, 8, 2, 0, 0, 0, 0, 0, 0, 1, 4090, 4092, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1546, 1821, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 1940, 5, 0, 1, 0, 0, 1413, 1369, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 5, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 1, 1, 4029, 1, 1, 1, 0, 2715, 460, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8581, 8601, 8187, 7463, 8822, 8181, 7503] avg=8192.0000 max=8822 max_rank=5 min=7463 min_rank=4 imbalance_ratio=1.076904 max_load_reduction=56714 max_load_reduction_pct=0.865387 moved_to_echo=31969 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 2, 0, 0, 4096, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 2897, 8, 0, 0, 0, 4095, 1140, 13, 4, 0, 0, 0, 0, 0, 15, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1540, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 411, 0, 0, 0, 0, 23, 1416, 1371, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1345, 0, 0, 4096, 0, 0, 0, 0, 2715, 460, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8309, 8157, 8203, 8580, 7315, 8179, 8616] avg=8192.0000 max=8616 max_rank=7 min=7315 min_rank=5 imbalance_ratio=1.051758 max_load_reduction=56920 max_load_reduction_pct=0.868530 moved_to_echo=32690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[7680, 128, 128, 32512, 128, 128, 128, 0, 21760, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 23296, 256, 128, 256, 128, 32768, 9216, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 10496, 128, 0, 128, 0, 128, 11392, 11008, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 12416, 20480, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3712, 128, 128, 0, 0, 128, 128, 128, 32768, 29184, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4092, 0, 63, 1, 3082, 1, 1, 0, 1, 0, 1, 0, 0, 4095, 64, 4083, 3987, 4094, 0, 0, 0, 973, 0, 1, 4095, 0, 0, 1, 3, 4094, 4095, 0, 1, 0, 0, 1, 0, 0, 0, 0, 4095, 4093, 0, 4095, 3890, 1, 316, 15, 4019, 0, 1, 1, 1, 1, 0, 0, 1, 4095, 1, 4079, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 35, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4089, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 4096, 0, 0, 0, 83, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4093, 0, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4090, 0, 94, 2, 2712, 1, 0, 0, 6, 0, 1, 0, 0, 4094, 179, 4093, 4005, 4092, 0, 0, 0, 1438, 0, 2, 4094, 0, 0, 2, 10, 4093, 4094, 0, 2, 0, 4, 2, 0, 0, 1, 0, 4094, 4078, 0, 4093, 3780, 2, 131, 22, 4071, 0, 3, 0, 2, 1, 0, 0, 2, 4094, 3, 4042, 0, 5, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4092, 0, 61, 1, 3034, 1, 1, 0, 1, 0, 0, 0, 0, 4095, 100, 4094, 3991, 4094, 0, 0, 0, 1057, 0, 1, 4095, 0, 0, 1, 3, 4094, 4095, 0, 1, 0, 1, 1, 0, 0, 0, 0, 4095, 4090, 0, 4095, 3972, 1, 104, 14, 4059, 0, 1, 1, 1, 1, 0, 0, 1, 4095, 2, 4086, 0, 3, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 4096, 4096, 0, 0, 0, 24, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4079, 4096, 4096, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 13, 4096, 4096, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4067, 0, 0, 407, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 3686, 4096, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 4089, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 12, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8585, 8150, 7817, 8191, 8191, 8201, 8205, 8196] avg=8192.0000 max=8585 max_rank=0 min=7817 min_rank=2 imbalance_ratio=1.047974 max_load_reduction=56951 max_load_reduction_pct=0.869003 moved_to_echo=32793 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 94, 2, 2712, 1, 0, 407, 14, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 4094, 179, 4051, 0, 0, 0, 0, 0, 0, 0, 3683, 4005, 0, 0, 0, 0, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 2, 10, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 2, 0, 0, 1, 4094, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 16, 3780, 2, 131, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4071, 0, 3, 0, 2, 1, 0, 4092, 12, 1, 0, 0, 0, 0, 0, 0, 2, 4094, 3, 4042, 0, 5, 2, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7320, 8331, 9126, 8200, 8196, 8007, 8204, 8152] avg=8192.0000 max=9126 max_rank=2 min=7320 min_rank=0 imbalance_ratio=1.114014 max_load_reduction=56410 max_load_reduction_pct=0.860748 moved_to_echo=32559 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4017, 0, 0, 407, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 3685, 4096, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 12, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8535, 8150, 7864, 8191, 8191, 8207, 8203, 8195] avg=8192.0000 max=8535 max_rank=0 min=7864 min_rank=2 imbalance_ratio=1.041870 max_load_reduction=57001 max_load_reduction_pct=0.869766 moved_to_echo=32796 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 407, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 3686, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 12, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8614, 8150, 7783, 8191, 8191, 8207, 8204, 8196] avg=8192.0000 max=8614 max_rank=0 min=7783 min_rank=2 imbalance_ratio=1.051514 max_load_reduction=56922 max_load_reduction_pct=0.868561 moved_to_echo=32798 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 63, 1, 3082, 1, 1, 406, 14, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4095, 64, 4053, 0, 0, 0, 0, 0, 0, 0, 3674, 3987, 0, 0, 0, 0, 973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 1, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 16, 3890, 1, 316, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4019, 0, 1, 1, 1, 1, 0, 4094, 12, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 4079, 0, 2, 1, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7660, 8214, 8634, 8193, 8192, 8316, 8144, 8183] avg=8192.0000 max=8634 max_rank=2 min=7660 min_rank=0 imbalance_ratio=1.053955 max_load_reduction=56902 max_load_reduction_pct=0.868256 moved_to_echo=32835 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 61, 1, 3034, 1, 1, 407, 14, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 100, 4053, 0, 0, 0, 0, 0, 0, 0, 3684, 3991, 0, 0, 0, 0, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 1, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 16, 3972, 1, 104, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4059, 0, 1, 1, 1, 1, 0, 4094, 12, 1, 0, 0, 0, 0, 0, 0, 1, 4095, 2, 4086, 0, 3, 1, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7611, 8249, 8732, 8193, 8192, 8183, 8184, 8192] avg=8192.0000 max=8732 max_rank=2 min=7611 min_rank=0 imbalance_ratio=1.065918 max_load_reduction=56804 max_load_reduction_pct=0.866760 moved_to_echo=32751 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4074, 0, 0, 407, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 3684, 4096, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 12, 1, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8592, 8149, 7804, 8191, 8191, 8208, 8202, 8199] avg=8192.0000 max=8592 max_rank=0 min=7804 min_rank=2 imbalance_ratio=1.048828 max_load_reduction=56944 max_load_reduction_pct=0.868896 moved_to_echo=32800 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4095, 0, 0, 410, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 3664, 4096, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 0, 4096, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8619, 8143, 7776, 8186, 8192, 8207, 8190, 8223] avg=8192.0000 max=8619 max_rank=0 min=7776 min_rank=2 imbalance_ratio=1.052124 max_load_reduction=56917 max_load_reduction_pct=0.868484 moved_to_echo=32812 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 0, 256, 128, 29184, 128, 128, 3328, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 128, 128, 32128, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[29568, 32512, 0, 0, 0, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32640, 0, 128, 128, 128, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 384, 32512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 1194, 0, 4067, 0, 0, 4096, 0, 0, 0, 4096, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 15, 4096, 0, 0, 4096, 4096, 0, 0, 2547, 0, 0, 0, 4096, 4096, 0, 0, 4085, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 0, 0, 0, 172, 0, 4042, 0, 1, 4096, 0, 0, 0, 4095, 4038, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 92, 4096, 0, 1, 4096, 4096, 3, 0, 3536, 0, 0, 0, 4096, 4096, 0, 0, 4073, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 0, 0, 0, 590, 0, 4074, 0, 0, 4096, 0, 0, 0, 4096, 4083, 0, 0, 4096, 0, 0, 10, 0, 0, 4096, 4096, 4096, 0, 18, 4096, 0, 0, 4096, 4096, 0, 0, 3219, 0, 0, 0, 4096, 4096, 0, 0, 4090, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 2, 152, 0, 1, 3, 0, 3964, 0, 2, 0, 0, 290, 0, 11, 8, 0, 2, 50, 2, 2, 130, 4, 4034, 3, 0, 4085, 9, 0, 2, 4094, 3775, 0, 0, 4074, 0, 0, 2, 58, 7, 4094, 4088, 4071, 2, 0, 4087, 0, 0, 4084, 4091, 0, 0, 3981, 2, 0, 2, 4084, 4093, 1, 0, 4083, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 392, 0, 0, 0, 0, 0, 0, 481, 0, 4073, 0, 0, 4096, 0, 0, 0, 4096, 4084, 0, 0, 4096, 0, 0, 2, 0, 0, 4096, 4096, 4096, 0, 435, 4096, 0, 0, 4096, 4096, 0, 0, 2824, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 1, 102, 1, 0, 1, 0, 4032, 0, 1, 0, 0, 288, 0, 31, 1, 0, 1, 13, 1, 1, 449, 1, 4071, 1, 6, 4092, 3, 0, 1, 4095, 3738, 0, 0, 4084, 0, 1, 4, 30, 4, 4095, 4089, 4081, 1, 92, 4092, 0, 0, 4087, 4093, 0, 0, 3586, 1, 0, 1, 4095, 4095, 0, 0, 4071, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 1, 88, 1, 0, 1, 0, 4046, 0, 1, 0, 0, 138, 0, 234, 3, 0, 1, 19, 1, 1, 818, 1, 4061, 1, 7, 4092, 2, 0, 1, 4095, 3817, 0, 0, 4085, 0, 1, 9, 25, 3, 4095, 4090, 4090, 1, 2, 4092, 0, 0, 4087, 4093, 0, 0, 3241, 1, 0, 1, 4095, 4095, 0, 0, 3997, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 616, 0, 0, 0, 0, 0, 0, 367, 0, 4036, 0, 12, 4096, 0, 0, 0, 4096, 4034, 0, 0, 4096, 0, 0, 3, 0, 0, 4096, 4081, 4096, 0, 43, 4096, 0, 17, 4096, 4096, 28, 0, 3189, 0, 0, 0, 4085, 4096, 0, 0, 4065, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4053, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 300, 3220, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 4068, 3543, 43, 0, 0, 0, 0, 0, 4074, 0, 0, 4096, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 4096, 0, 0, 10, 0, 79, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 829, 0, 0, 3219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 28, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8133, 8244, 8206, 8315, 8131, 8144, 8214] avg=8192.0000 max=8315 max_rank=4 min=8131 min_rank=5 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32804 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 388, 3220, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 0, 4068, 3543, 42, 0, 0, 0, 0, 0, 4067, 0, 0, 4096, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 79, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 829, 0, 0, 2547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 28, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8221, 8847, 8199, 8310, 8128, 7472, 8209] avg=8192.0000 max=8847 max_rank=2 min=7472 min_rank=6 imbalance_ratio=1.079956 max_load_reduction=56689 max_load_reduction_pct=0.865005 moved_to_echo=32119 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 2, 152, 0, 1, 3, 0, 4094, 4045, 0, 0, 0, 0, 0, 0, 3964, 0, 2, 0, 0, 290, 0, 11, 3216, 517, 0, 0, 0, 0, 0, 0, 8, 0, 2, 50, 2, 2, 130, 4, 4065, 3541, 42, 0, 0, 0, 0, 0, 4034, 3, 0, 4085, 9, 0, 2, 36, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 4074, 0, 0, 2, 58, 78, 47, 0, 0, 0, 0, 0, 0, 7, 0, 4010, 4071, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 828, 0, 0, 3981, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4084, 28, 1, 0, 4083, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8299, 8000, 7846, 8169, 8034, 8090, 8895, 8203] avg=8192.0000 max=8895 max_rank=6 min=7846 min_rank=2 imbalance_ratio=1.085815 max_load_reduction=56641 max_load_reduction_pct=0.864273 moved_to_echo=33222 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 392, 3220, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 4068, 3543, 42, 0, 0, 0, 0, 0, 4073, 0, 0, 4096, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 0, 2, 0, 79, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 4096, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 829, 0, 0, 2824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 28, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8225, 8134, 8205, 8308, 8548, 7749, 8217] avg=8192.0000 max=8548 max_rank=5 min=7749 min_rank=6 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=32822 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 102, 1, 0, 1, 0, 4095, 4050, 0, 0, 0, 0, 0, 0, 4032, 0, 1, 0, 0, 288, 0, 31, 3218, 517, 0, 0, 0, 0, 0, 0, 1, 0, 1, 13, 1, 1, 449, 1, 4067, 3542, 42, 0, 0, 0, 0, 0, 4071, 1, 6, 4092, 3, 0, 1, 36, 0, 0, 0, 0, 0, 0, 0, 0, 3738, 0, 0, 4084, 0, 1, 4, 30, 78, 47, 0, 0, 0, 0, 0, 0, 4, 0, 4011, 4081, 1, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 828, 0, 0, 3586, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 28, 0, 0, 4071, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8087, 8118, 8210, 7982, 8189, 8502, 8197] avg=8192.0000 max=8502 max_rank=6 min=7982 min_rank=4 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=32870 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 428, 3220, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 4068, 3542, 42, 0, 0, 0, 0, 0, 4042, 0, 1, 4096, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 4096, 0, 0, 0, 0, 79, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 4096, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 829, 3, 0, 3536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 28, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8261, 7824, 8175, 8260, 8204, 8465, 8197] avg=8192.0000 max=8465 max_rank=6 min=7824 min_rank=2 imbalance_ratio=1.033325 max_load_reduction=57071 max_load_reduction_pct=0.870834 moved_to_echo=33126 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 88, 1, 0, 1, 0, 4095, 4050, 0, 0, 0, 0, 0, 0, 4046, 0, 1, 0, 0, 138, 0, 234, 3218, 517, 0, 0, 0, 0, 0, 0, 3, 0, 1, 19, 1, 1, 818, 1, 4067, 3542, 42, 0, 0, 0, 0, 0, 4061, 1, 7, 4092, 2, 0, 1, 36, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 0, 0, 4085, 0, 1, 9, 25, 78, 47, 0, 0, 0, 0, 0, 0, 3, 0, 4012, 4090, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 828, 0, 0, 3241, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 28, 0, 0, 3997, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8154, 8495, 8200, 8062, 8108, 8157, 8123] avg=8192.0000 max=8495 max_rank=2 min=8062 min_rank=4 imbalance_ratio=1.036987 max_load_reduction=57041 max_load_reduction_pct=0.870377 moved_to_echo=32450 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 616, 3224, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 4070, 3549, 42, 0, 0, 0, 0, 0, 4036, 0, 12, 4096, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 4096, 0, 0, 3, 0, 82, 50, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 4096, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 822, 28, 0, 3189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 26, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8455, 8028, 8172, 8265, 8138, 8152, 8176] avg=8192.0000 max=8455 max_rank=1 min=8028 min_rank=2 imbalance_ratio=1.032104 max_load_reduction=57081 max_load_reduction_pct=0.870987 moved_to_echo=32731 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 6656, 128, 0, 26240, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 256, 128, 0, 32640, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32512, 128, 128, 32768, 128, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 32128, 32768, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31744, 0, 0, 32768, 0, 128, 128, 128, 640, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 128, 4224, 128, 32640, 28416, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 384, 128, 128, 128, 0, 32768, 32512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 0, 128, 0, 0, 768, 0, 2432, 25856, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 934, 0, 4095, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2066, 0, 0, 0, 0, 4095, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 2463, 0, 2730, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 629, 0, 4090, 0, 0, 0, 0, 21, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3114, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 2978, 0, 1456, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 823, 0, 4083, 0, 0, 0, 0, 72, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 656, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4083, 0, 0, 0, 3458, 0, 3207, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 98, 4095, 4079, 0, 1, 2, 0, 4095, 2, 0, 4012, 1, 4095, 0, 21, 0, 0, 547, 194, 4095, 0, 1, 0, 12, 0, 1, 4095, 1, 0, 0, 1, 2, 2, 0, 1, 1, 0, 1606, 1, 1, 1, 0, 4025, 4095, 0, 4094, 4094, 0, 1, 4095, 0, 0, 1, 4009, 0, 1897, 4068, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 135, 4095, 4075, 0, 1, 1, 0, 4095, 2, 0, 4009, 1, 4095, 0, 31, 0, 0, 664, 188, 4095, 0, 1, 0, 12, 0, 1, 4095, 1, 0, 0, 1, 2, 2, 0, 1, 1, 0, 1206, 1, 1, 1, 0, 4064, 4095, 0, 4094, 4094, 0, 1, 4095, 0, 0, 1, 3925, 0, 2223, 4036, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 175, 0, 4055, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 0, 0, 0, 0, 4020, 4096, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 3467, 0, 3653, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 254, 4093, 4067, 0, 2, 3, 0, 4094, 3, 0, 3964, 2, 4094, 0, 76, 0, 0, 516, 318, 4094, 0, 2, 0, 25, 0, 2, 4094, 2, 0, 0, 2, 5, 5, 0, 2, 2, 0, 2376, 0, 2, 1, 0, 3972, 4094, 2, 4092, 4092, 0, 2, 4094, 2, 0, 2, 3827, 0, 1146, 4018, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1284, 0, 4079, 0, 0, 0, 0, 62, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2629, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4064, 0, 0, 0, 2524, 0, 1729, 4096, 5, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4046, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 35, 0, 0, 0, 0, 0, 0, 0, 3307, 0, 0, 0, 0, 934, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 1792, 1478, 789, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 3, 0, 0, 0, 0, 0, 0, 2066, 0, 0, 0, 0, 4095, 2301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2463, 0, 2730, 2618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8227, 8336, 8171, 8188, 8462, 8199, 7811] avg=8192.0000 max=8462 max_rank=5 min=7811 min_rank=7 imbalance_ratio=1.032959 max_load_reduction=57074 max_load_reduction_pct=0.870880 moved_to_echo=32660 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4046, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 35, 0, 0, 0, 0, 0, 0, 0, 3307, 0, 0, 0, 0, 629, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4096, 0, 1792, 1478, 789, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 3, 0, 0, 0, 0, 0, 0, 3114, 0, 0, 0, 0, 4096, 2301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2978, 0, 1456, 2618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8227, 8026, 8191, 8188, 9511, 8199, 7052] avg=8192.0000 max=9511 max_rank=5 min=7052 min_rank=7 imbalance_ratio=1.161011 max_load_reduction=56025 max_load_reduction_pct=0.854874 moved_to_echo=32950 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4046, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 35, 0, 0, 0, 0, 0, 0, 0, 3307, 0, 0, 0, 0, 175, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 1792, 1478, 789, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 3, 0, 0, 0, 0, 0, 0, 1010, 0, 0, 0, 0, 4020, 2301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3467, 0, 3653, 2618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8227, 7537, 8175, 8188, 7331, 8198, 9738] avg=8192.0000 max=9738 max_rank=7 min=7331 min_rank=5 imbalance_ratio=1.188721 max_load_reduction=55798 max_load_reduction_pct=0.851410 moved_to_echo=33455 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4043, 0, 0, 0, 98, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4095, 2, 0, 4012, 1, 35, 0, 0, 0, 0, 0, 0, 0, 3306, 0, 21, 0, 0, 547, 194, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 1, 4095, 1, 1792, 1468, 789, 15, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 1, 1, 4095, 4087, 3, 0, 0, 0, 0, 0, 0, 1606, 1, 1, 1, 0, 4025, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4094, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4009, 0, 1897, 2600, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8148, 8163, 8174, 8192, 7934, 8197, 8508] avg=8192.0000 max=8508 max_rank=7 min=7934 min_rank=5 imbalance_ratio=1.038574 max_load_reduction=57028 max_load_reduction_pct=0.870178 moved_to_echo=32831 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4046, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 35, 0, 0, 0, 0, 0, 0, 0, 3307, 0, 0, 0, 0, 823, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 4096, 0, 1792, 1478, 789, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4089, 3, 0, 0, 0, 0, 0, 0, 656, 0, 0, 0, 0, 4096, 2301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3458, 0, 3207, 2618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8227, 8213, 8242, 8190, 7053, 8186, 9283] avg=8192.0000 max=9283 max_rank=7 min=7053 min_rank=5 imbalance_ratio=1.133179 max_load_reduction=56253 max_load_reduction_pct=0.858353 moved_to_echo=32712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4044, 0, 0, 0, 135, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 2, 0, 4009, 1, 35, 0, 0, 0, 0, 0, 0, 0, 3306, 0, 31, 0, 0, 664, 188, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 1, 4095, 1, 1792, 1457, 789, 15, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 1, 1, 4095, 4087, 3, 0, 0, 0, 0, 0, 0, 1206, 1, 1, 1, 0, 4064, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4094, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3925, 0, 2223, 2579, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8144, 8284, 8163, 8192, 7573, 8197, 8729] avg=8192.0000 max=8729 max_rank=7 min=7573 min_rank=5 imbalance_ratio=1.065552 max_load_reduction=56807 max_load_reduction_pct=0.866806 moved_to_echo=32691 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4041, 0, 0, 0, 254, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 4094, 3, 0, 3964, 2, 35, 0, 0, 0, 0, 0, 0, 0, 3306, 0, 76, 0, 0, 516, 318, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 25, 0, 2, 4094, 2, 1791, 1450, 788, 15, 0, 0, 0, 0, 0, 0, 2, 5, 5, 0, 2, 2, 4093, 4085, 3, 0, 0, 0, 0, 0, 0, 2376, 0, 2, 1, 0, 3972, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 4092, 0, 2, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3827, 0, 1146, 2568, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8362, 8103, 8310, 8169, 8197, 8651, 8199, 7545] avg=8192.0000 max=8651 max_rank=5 min=7545 min_rank=7 imbalance_ratio=1.056030 max_load_reduction=56885 max_load_reduction_pct=0.867996 moved_to_echo=32592 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4034, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 41, 0, 0, 0, 0, 0, 0, 0, 3304, 0, 0, 0, 0, 1284, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 4096, 0, 1796, 1484, 792, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 4095, 4, 0, 0, 0, 0, 0, 0, 2629, 0, 0, 0, 0, 4096, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2524, 0, 1729, 2612, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8233, 8667, 8251, 8201, 9021, 8161, 6870] avg=8192.0000 max=9021 max_rank=5 min=6870 min_rank=7 imbalance_ratio=1.101196 max_load_reduction=56515 max_load_reduction_pct=0.862350 moved_to_echo=32253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 26752, 0, 18048, 20864, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 32512, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32384, 0, 0, 0, 512, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 14720, 128, 128, 128, 0, 32512, 18432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[26496, 0, 128, 0, 0, 5632, 768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 256, 128, 32768, 128, 14464, 11776, 6400, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4084, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1314, 0, 0, 0, 2707, 4096, 74, 508, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 11, 0, 0, 4096, 4094, 4065, 0, 2424, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4088, 1580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1882, 0, 0, 0, 1839, 4096, 36, 1277, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 29, 0, 0, 4096, 4094, 4054, 0, 1609, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4092, 1833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 324, 0, 0, 0, 2062, 4096, 12, 740, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 23, 0, 0, 4096, 4095, 4038, 0, 3261, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4022, 1652, 45, 0, 0, 1, 0, 0, 3, 2, 2, 2, 2, 0, 4087, 21, 33, 0, 2, 1, 2095, 4094, 933, 176, 0, 0, 4094, 3, 1, 0, 0, 0, 109, 4093, 0, 4094, 931, 2, 10, 4092, 4061, 4093, 0, 2577, 0, 3, 0, 4075, 2, 4094, 0, 2, 2, 0, 4091, 0, 2, 2, 3, 0, 0, 3831, 4094, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4068, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 826, 0, 0, 0, 1892, 4095, 33, 450, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 5, 0, 0, 4096, 4088, 3985, 0, 3443, 0, 0, 9, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4080, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4070, 2529, 10, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 4091, 17, 280, 0, 1, 0, 1542, 4095, 650, 170, 0, 0, 4095, 1, 1, 0, 0, 0, 53, 4095, 0, 4095, 356, 1, 7, 4094, 4078, 4095, 0, 2772, 0, 1, 0, 4086, 1, 4095, 0, 1, 1, 0, 4094, 0, 1, 0, 1, 2, 1, 3952, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4066, 1612, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 4093, 17, 70, 0, 1, 0, 2989, 4095, 547, 115, 0, 0, 4095, 1, 1, 0, 0, 0, 54, 4095, 0, 4095, 494, 1, 9, 4094, 4054, 4090, 0, 2544, 0, 1, 0, 4086, 1, 4095, 0, 1, 1, 0, 4094, 0, 1, 0, 1, 2, 1, 3917, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4042, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 905, 0, 0, 0, 2594, 4086, 66, 651, 0, 3, 4096, 0, 0, 0, 0, 21, 0, 4095, 0, 4096, 4, 0, 0, 4096, 4083, 4009, 0, 3137, 0, 0, 12, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4059, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4092, 1833, 0, 0, 0, 0, 0, 0, 2483, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 324, 0, 0, 0, 2062, 4096, 12, 740, 192, 181, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 23, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 4038, 0, 3261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 79, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8413, 8190, 7607, 8189, 7955, 8719, 8192, 8271] avg=8192.0000 max=8719 max_rank=5 min=7607 min_rank=2 imbalance_ratio=1.064331 max_load_reduction=56817 max_load_reduction_pct=0.866959 moved_to_echo=33137 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4022, 1652, 45, 0, 0, 1, 0, 0, 2462, 5, 0, 0, 0, 0, 0, 0, 3, 2, 2, 2, 2, 0, 4087, 21, 4070, 3, 0, 0, 0, 0, 0, 0, 33, 0, 2, 1, 2095, 4094, 933, 176, 191, 181, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 1, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 109, 3833, 0, 0, 931, 2, 10, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1408, 4093, 0, 2577, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 2, 2, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3, 0, 0, 3831, 4094, 2, 79, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8192, 7706, 8189, 8977, 8081, 8191, 8013] avg=8192.0000 max=8977 max_rank=4 min=7706 min_rank=2 imbalance_ratio=1.095825 max_load_reduction=56559 max_load_reduction_pct=0.863022 moved_to_echo=33262 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4084, 1202, 0, 0, 0, 0, 0, 0, 2482, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 1314, 0, 0, 0, 2707, 4096, 74, 508, 192, 181, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 4065, 0, 2424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 79, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7774, 8189, 9072, 8189, 7943, 7909, 8192, 8268] avg=8192.0000 max=9072 max_rank=2 min=7774 min_rank=0 imbalance_ratio=1.107422 max_load_reduction=56464 max_load_reduction_pct=0.861572 moved_to_echo=32312 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4088, 1580, 0, 0, 0, 0, 0, 0, 2482, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 1882, 0, 0, 0, 1839, 4096, 36, 1277, 192, 181, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 29, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 4054, 0, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 79, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8190, 9503, 8189, 7961, 7083, 8192, 8263] avg=8192.0000 max=9503 max_rank=2 min=7083 min_rank=5 imbalance_ratio=1.160034 max_load_reduction=56033 max_load_reduction_pct=0.854996 moved_to_echo=31499 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4070, 2529, 10, 0, 0, 1, 0, 0, 2472, 6, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4091, 17, 4080, 3, 0, 0, 0, 0, 0, 0, 280, 0, 1, 0, 1542, 4095, 650, 170, 191, 181, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 53, 3835, 0, 0, 356, 1, 7, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1415, 4095, 0, 2772, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2, 1, 3952, 4095, 1, 79, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9088, 8196, 7110, 8189, 8346, 8283, 8192, 8132] avg=8192.0000 max=9088 max_rank=0 min=7110 min_rank=2 imbalance_ratio=1.109375 max_load_reduction=56448 max_load_reduction_pct=0.861328 moved_to_echo=32953 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4068, 1697, 0, 0, 0, 0, 0, 0, 2478, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 826, 0, 0, 0, 1892, 4095, 33, 450, 192, 181, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1418, 3985, 0, 3443, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 79, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8190, 7669, 8190, 7937, 8855, 8192, 8255] avg=8192.0000 max=8855 max_rank=5 min=7669 min_rank=2 imbalance_ratio=1.080933 max_load_reduction=56681 max_load_reduction_pct=0.864883 moved_to_echo=33239 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4066, 1612, 1, 0, 0, 1, 0, 0, 2458, 5, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4093, 17, 4081, 3, 0, 0, 0, 0, 0, 0, 70, 0, 1, 0, 2989, 4095, 547, 115, 190, 181, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 54, 3835, 0, 0, 494, 1, 9, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 4090, 0, 2544, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2, 1, 3917, 4095, 1, 79, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8199, 8188, 8189, 8487, 8041, 8192, 8097] avg=8192.0000 max=8487 max_rank=4 min=8041 min_rank=5 imbalance_ratio=1.036011 max_load_reduction=57049 max_load_reduction_pct=0.870499 moved_to_echo=32817 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4042, 1001, 0, 0, 0, 0, 0, 0, 2479, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 905, 0, 0, 0, 2594, 4086, 66, 651, 197, 182, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 21, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1407, 4009, 0, 3137, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4096, 0, 82, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7528, 8190, 8681, 8212, 7930, 8565, 8193, 8237] avg=8192.0000 max=8681 max_rank=2 min=7528 min_rank=0 imbalance_ratio=1.059692 max_load_reduction=56855 max_load_reduction_pct=0.867538 moved_to_echo=32925 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 30720, 0, 0, 1920, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 13184, 128, 0, 0, 128, 0, 0, 19840, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11392, 32512, 0, 21888, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32128, 32768, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[5760, 0, 128, 128, 17792, 32768, 2432, 4096, 1664, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4065, 155, 2, 2850, 2, 44, 3959, 0, 4094, 2, 1594, 3355, 3, 0, 0, 0, 4094, 0, 13, 0, 2, 4093, 0, 5, 1, 1, 4, 3975, 0, 4078, 0, 4091, 0, 2, 1, 4094, 0, 20, 0, 0, 2, 0, 2, 0, 0, 489, 2, 2, 6, 2, 4070, 0, 0, 4, 4094, 0, 0, 4076, 0, 4092, 4090, 0, 4, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[3993, 9, 0, 4066, 0, 0, 4096, 0, 4096, 0, 125, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4018, 72, 1, 3375, 1, 22, 4025, 0, 4095, 1, 819, 3733, 2, 0, 0, 0, 4095, 0, 6, 0, 1, 4094, 0, 1, 1, 0, 3, 4062, 0, 4090, 0, 4090, 0, 1, 1, 4095, 0, 4, 1, 0, 1, 0, 1, 0, 0, 362, 1, 1, 3, 1, 4083, 0, 0, 1, 4095, 0, 0, 4087, 0, 4094, 4094, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4029, 72, 1, 3167, 1, 29, 4031, 0, 4095, 1, 1267, 3596, 2, 0, 0, 0, 4095, 0, 6, 0, 1, 4094, 0, 2, 0, 0, 1, 3981, 0, 4080, 0, 4094, 0, 1, 1, 4095, 0, 7, 0, 0, 1, 0, 1, 0, 0, 318, 1, 1, 3, 1, 4086, 0, 0, 1, 4095, 0, 0, 4089, 1, 4094, 4092, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3918, 38, 0, 3996, 0, 0, 4096, 0, 4096, 0, 264, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4072, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4048, 57, 0, 3875, 0, 0, 4096, 0, 4095, 0, 231, 4096, 0, 0, 1, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4075, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[3764, 4, 0, 3697, 0, 0, 4096, 0, 4096, 0, 816, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4005, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4040, 40, 0, 3613, 0, 0, 4096, 0, 4088, 0, 495, 4096, 0, 0, 0, 0, 4093, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 4078, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3993, 9, 0, 4066, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3596, 0, 125, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3532, 500, 86, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4091, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8632, 7817, 8188, 8247, 8215, 8045, 8195, 8197] avg=8192.0000 max=8632 max_rank=0 min=7817 min_rank=1 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4065, 155, 2, 2850, 2, 44, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3594, 2, 1594, 3355, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 13, 0, 2, 4089, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 3975, 0, 4078, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4094, 0, 20, 0, 0, 3414, 500, 86, 1, 0, 0, 0, 0, 2, 0, 2, 0, 0, 489, 2, 2, 4087, 3947, 0, 0, 0, 0, 0, 0, 6, 2, 4070, 0, 0, 4, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 5, 4090, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7663, 8548, 8203, 8117, 8118, 8531, 8179, 8177] avg=8192.0000 max=8548 max_rank=1 min=7663 min_rank=0 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=33005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3918, 38, 0, 3996, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3596, 0, 264, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4096, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3532, 500, 86, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8516, 7956, 8188, 8226, 8215, 8043, 8195, 8197] avg=8192.0000 max=8516 max_rank=0 min=7956 min_rank=1 imbalance_ratio=1.039551 max_load_reduction=57020 max_load_reduction_pct=0.870056 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4018, 72, 1, 3375, 1, 22, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3595, 1, 819, 3733, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 0, 1, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4062, 0, 4090, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 4, 1, 0, 3471, 500, 86, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 362, 1, 1, 4089, 3946, 0, 0, 0, 0, 0, 0, 3, 1, 4083, 0, 0, 1, 4095, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 5, 4094, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8043, 8150, 8193, 8214, 8160, 8401, 8186, 8189] avg=8192.0000 max=8401 max_rank=5 min=8043 min_rank=0 imbalance_ratio=1.025513 max_load_reduction=57135 max_load_reduction_pct=0.871811 moved_to_echo=32936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3764, 4, 0, 3697, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3596, 0, 816, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 4096, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3532, 500, 86, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4091, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8029, 8508, 8188, 8159, 8215, 8044, 8196, 8197] avg=8192.0000 max=8508 max_rank=1 min=8029 min_rank=0 imbalance_ratio=1.038574 max_load_reduction=57028 max_load_reduction_pct=0.870178 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4048, 57, 0, 3875, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3595, 0, 231, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4096, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3532, 500, 86, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4091, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8544, 7923, 8189, 8229, 8215, 8044, 8195, 8197] avg=8192.0000 max=8544 max_rank=0 min=7923 min_rank=1 imbalance_ratio=1.042969 max_load_reduction=56992 max_load_reduction_pct=0.869629 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4029, 72, 1, 3167, 1, 29, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3595, 1, 1267, 3596, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 0, 1, 4090, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3981, 0, 4080, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 7, 0, 0, 3476, 500, 86, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 318, 1, 1, 4089, 3950, 0, 0, 0, 0, 0, 0, 3, 1, 4086, 0, 0, 1, 4095, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 5, 4092, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7854, 8461, 8194, 8120, 8167, 8361, 8189, 8190] avg=8192.0000 max=8461 max_rank=1 min=7854 min_rank=0 imbalance_ratio=1.032837 max_load_reduction=57075 max_load_reduction_pct=0.870895 moved_to_echo=32907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4040, 40, 0, 3613, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3582, 0, 495, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 16, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 3534, 506, 89, 3, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8173, 8193, 8224, 8231, 8067, 8201, 8192] avg=8192.0000 max=8255 max_rank=0 min=8067 min_rank=5 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32000, 512, 128, 28672, 128, 128, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 1280, 128, 128, 32768, 31616, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32384, 0, 32768, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[28800, 128, 5632, 31232, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 128, 0, 28032, 4096, 768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2659, 73, 1049, 478, 985, 0, 2023, 319, 1234, 3022, 486, 539, 1564, 1165, 0, 166, 667, 1276, 677, 1794, 0, 845, 2059, 1065, 69, 27, 3074, 1010, 2787, 408, 2386, 734, 11, 1803, 2146, 179, 461, 428, 2065, 1501, 621, 1, 1501, 1515, 0, 1340, 1182, 943, 2686, 9, 1184, 4, 157, 575, 8, 776, 0, 440, 1459, 845, 2261, 1331, 703, 2761] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2707, 108, 592, 268, 1041, 0, 1529, 482, 1595, 3003, 886, 1171, 1518, 1796, 0, 378, 322, 1256, 731, 1247, 0, 736, 1973, 1319, 76, 121, 2511, 1257, 2815, 1226, 2328, 404, 2, 1553, 2599, 577, 763, 405, 1622, 1188, 468, 0, 2048, 1884, 0, 1294, 871, 1028, 2400, 35, 1632, 23, 147, 875, 19, 782, 0, 394, 1093, 680, 2047, 584, 859, 2268] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3170, 67, 564, 301, 1270, 0, 1633, 139, 1125, 2797, 775, 802, 1289, 1335, 0, 283, 389, 1180, 463, 914, 0, 773, 1973, 1233, 60, 37, 2600, 1821, 2762, 732, 2339, 624, 1, 1690, 2443, 557, 728, 486, 1494, 984, 508, 0, 1632, 1879, 0, 1509, 1253, 1036, 2552, 40, 1485, 1, 186, 880, 1, 676, 0, 473, 1324, 1215, 2346, 834, 1116, 2757] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2299, 45, 516, 328, 1165, 0, 2563, 51, 651, 3431, 930, 493, 976, 2073, 0, 88, 356, 1744, 517, 951, 0, 1056, 1321, 941, 59, 78, 2817, 933, 2046, 581, 2456, 863, 4, 1283, 2480, 457, 456, 590, 1623, 1699, 412, 0, 2026, 1730, 0, 1786, 1361, 759, 2822, 42, 962, 1, 160, 1273, 0, 744, 0, 327, 1015, 1743, 2249, 1763, 885, 2556] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2763, 25, 1081, 297, 1059, 0, 1391, 118, 898, 2426, 557, 984, 1742, 726, 0, 117, 280, 1770, 823, 903, 3, 697, 2079, 1876, 77, 24, 2478, 1558, 2536, 395, 2562, 1007, 45, 2299, 2339, 441, 561, 415, 1717, 947, 532, 0, 1726, 1780, 8, 1451, 1122, 708, 2546, 42, 1028, 1, 110, 749, 11, 657, 0, 576, 1683, 1394, 2409, 1075, 1273, 2639] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3013, 41, 312, 417, 920, 0, 1894, 115, 1452, 3223, 770, 1269, 1332, 1627, 10, 261, 462, 930, 789, 754, 1, 798, 2453, 1069, 73, 38, 2676, 1069, 2184, 698, 2246, 836, 0, 1369, 2862, 335, 668, 773, 1687, 750, 707, 0, 1636, 1471, 1, 1678, 1074, 628, 2988, 65, 1450, 6, 162, 821, 0, 567, 3, 489, 1341, 1291, 2564, 757, 1174, 2487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2248, 66, 836, 266, 1540, 0, 1807, 236, 926, 2912, 622, 735, 1510, 1234, 0, 184, 654, 1340, 1216, 1784, 0, 715, 1191, 1183, 68, 60, 2662, 1209, 2495, 421, 2411, 730, 2, 1857, 1769, 350, 811, 648, 2275, 1492, 745, 0, 1764, 1500, 0, 1633, 1213, 1083, 2962, 19, 1261, 1, 48, 497, 1, 661, 0, 311, 1077, 1134, 2198, 1171, 1201, 2591] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2770, 68, 630, 387, 1379, 1, 2018, 104, 1208, 2993, 685, 1080, 1168, 1421, 0, 195, 424, 1053, 417, 904, 2, 901, 1504, 1137, 107, 26, 2732, 1063, 3040, 625, 2104, 473, 1, 1395, 2295, 338, 514, 1119, 2079, 1538, 595, 3, 1901, 2086, 1, 1531, 850, 1307, 2847, 40, 1743, 1, 54, 908, 0, 539, 1, 744, 1211, 635, 2296, 759, 1255, 2331] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2659, 73, 1049, 478, 985, 0, 2023, 319, 1260, 0, 0, 0, 0, 0, 0, 0, 1234, 2476, 486, 539, 1564, 1165, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 667, 1276, 677, 1794, 0, 845, 2059, 1065, 502, 152, 11, 0, 0, 0, 0, 0, 69, 27, 623, 1010, 2787, 408, 2386, 734, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1803, 1644, 179, 461, 428, 2065, 1501, 0, 0, 0, 0, 0, 0, 0, 0, 621, 1, 1501, 1515, 0, 1340, 1182, 943, 535, 0, 0, 0, 0, 0, 0, 0, 2686, 9, 1184, 4, 157, 575, 8, 776, 2451, 328, 0, 0, 0, 0, 0, 0, 0, 440, 1459, 845, 2261, 1331, 703, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8846, 7630, 9048, 8044, 8092, 7638, 8178, 8060] avg=8192.0000 max=9048 max_rank=2 min=7630 min_rank=1 imbalance_ratio=1.104492 max_load_reduction=56488 max_load_reduction_pct=0.861938 moved_to_echo=31968 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3170, 67, 564, 301, 1270, 0, 1633, 139, 1258, 0, 0, 0, 0, 0, 0, 0, 1125, 2292, 775, 802, 1289, 1335, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 389, 1180, 463, 914, 0, 773, 1973, 1233, 572, 152, 10, 0, 0, 0, 0, 0, 60, 37, 527, 1821, 2762, 732, 2339, 624, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1690, 1871, 557, 728, 486, 1494, 984, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 1632, 1879, 0, 1509, 1253, 1036, 495, 0, 0, 0, 0, 0, 0, 0, 2552, 40, 1485, 1, 186, 880, 1, 676, 2073, 328, 0, 0, 0, 0, 0, 0, 0, 473, 1324, 1215, 2346, 834, 1116, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8402, 7901, 7659, 8902, 7811, 8312, 8222, 8327] avg=8192.0000 max=8902 max_rank=3 min=7659 min_rank=2 imbalance_ratio=1.086670 max_load_reduction=56634 max_load_reduction_pct=0.864166 moved_to_echo=32672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2707, 108, 592, 268, 1041, 0, 1529, 482, 1035, 0, 0, 0, 0, 0, 0, 0, 1595, 2462, 886, 1171, 1518, 1796, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 322, 1256, 731, 1247, 0, 736, 1973, 1319, 608, 125, 10, 0, 0, 0, 0, 0, 76, 121, 509, 1257, 2815, 1226, 2328, 404, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1553, 1991, 577, 763, 405, 1622, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 2048, 1884, 0, 1294, 871, 1028, 531, 0, 0, 0, 0, 0, 0, 0, 2400, 35, 1632, 23, 147, 875, 19, 782, 2002, 269, 0, 0, 0, 0, 0, 0, 0, 394, 1093, 680, 2047, 584, 859, 839, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7762, 9806, 8327, 8736, 8101, 8124, 8184, 6496] avg=8192.0000 max=9806 max_rank=1 min=6496 min_rank=7 imbalance_ratio=1.197021 max_load_reduction=55730 max_load_reduction_pct=0.850372 moved_to_echo=30905 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3013, 41, 312, 417, 920, 0, 1894, 115, 1135, 0, 0, 0, 0, 0, 0, 0, 1452, 2642, 770, 1269, 1332, 1627, 10, 261, 0, 0, 0, 0, 0, 0, 0, 0, 462, 930, 789, 754, 1, 798, 2453, 1069, 670, 137, 11, 0, 0, 0, 0, 0, 73, 38, 542, 1069, 2184, 698, 2246, 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 2192, 335, 668, 773, 1687, 750, 0, 0, 0, 0, 0, 0, 0, 0, 707, 0, 1636, 1471, 1, 1678, 1074, 628, 570, 0, 0, 0, 0, 0, 0, 0, 2988, 65, 1450, 6, 162, 821, 0, 567, 2134, 295, 0, 0, 0, 0, 0, 0, 3, 489, 1341, 1291, 2564, 757, 1174, 920, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7847, 9363, 8074, 7686, 7774, 7765, 8488, 8539] avg=8192.0000 max=9363 max_rank=1 min=7686 min_rank=3 imbalance_ratio=1.142944 max_load_reduction=56173 max_load_reduction_pct=0.857132 moved_to_echo=32566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2299, 45, 516, 328, 1165, 0, 2563, 51, 1167, 0, 0, 0, 0, 0, 0, 0, 651, 2812, 930, 493, 976, 2073, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 356, 1744, 517, 951, 0, 1056, 1321, 941, 580, 141, 12, 0, 0, 0, 0, 0, 59, 78, 571, 933, 2046, 581, 2456, 863, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1283, 1900, 457, 456, 590, 1623, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 412, 0, 2026, 1730, 0, 1786, 1361, 759, 607, 0, 0, 0, 0, 0, 0, 0, 2822, 42, 962, 1, 160, 1273, 0, 744, 2246, 304, 0, 0, 0, 0, 0, 0, 0, 327, 1015, 1743, 2249, 1763, 885, 944, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8023, 7619, 7587, 8012, 8681, 8554, 8926] avg=8192.0000 max=8926 max_rank=7 min=7587 min_rank=3 imbalance_ratio=1.089600 max_load_reduction=56610 max_load_reduction_pct=0.863800 moved_to_echo=34173 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2248, 66, 836, 266, 1540, 0, 1807, 236, 1183, 0, 0, 0, 0, 0, 0, 0, 926, 2387, 622, 735, 1510, 1234, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 654, 1340, 1216, 1784, 0, 715, 1191, 1183, 414, 143, 10, 0, 0, 0, 0, 0, 68, 60, 539, 1209, 2495, 421, 2411, 730, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1857, 1355, 350, 811, 648, 2275, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 1764, 1500, 0, 1633, 1213, 1083, 515, 0, 0, 0, 0, 0, 0, 0, 2962, 19, 1261, 1, 48, 497, 1, 661, 2123, 308, 0, 0, 0, 0, 0, 0, 0, 311, 1077, 1134, 2198, 1171, 1201, 957, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 7598, 8650, 7933, 8790, 8453, 7881, 8049] avg=8192.0000 max=8790 max_rank=4 min=7598 min_rank=1 imbalance_ratio=1.072998 max_load_reduction=56746 max_load_reduction_pct=0.865875 moved_to_echo=33173 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2770, 68, 630, 387, 1379, 1, 2018, 104, 1064, 0, 0, 0, 0, 0, 0, 0, 1208, 2454, 685, 1080, 1168, 1421, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 424, 1053, 417, 904, 2, 901, 1504, 1137, 537, 128, 10, 0, 0, 0, 0, 0, 107, 26, 553, 1063, 3040, 625, 2104, 473, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1395, 1758, 338, 514, 1119, 2079, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 595, 3, 1901, 2086, 1, 1531, 850, 1307, 529, 0, 0, 0, 0, 0, 0, 0, 2847, 40, 1743, 1, 54, 908, 0, 539, 2179, 277, 0, 0, 0, 0, 0, 0, 1, 744, 1211, 635, 2296, 759, 1255, 862, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8421, 8211, 7017, 7991, 8742, 8803, 8588, 7763] avg=8192.0000 max=8803 max_rank=5 min=7017 min_rank=2 imbalance_ratio=1.074585 max_load_reduction=56733 max_load_reduction_pct=0.865677 moved_to_echo=33896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2763, 25, 1081, 297, 1059, 0, 1391, 118, 1208, 0, 0, 0, 0, 0, 0, 0, 898, 1980, 557, 984, 1742, 726, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 280, 1770, 823, 903, 3, 697, 2079, 1876, 550, 148, 13, 0, 0, 0, 0, 0, 77, 24, 498, 1558, 2536, 395, 2562, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 45, 2299, 1789, 441, 561, 415, 1717, 947, 0, 0, 0, 0, 0, 0, 0, 0, 532, 0, 1726, 1780, 8, 1451, 1122, 708, 433, 0, 0, 0, 0, 0, 0, 0, 2546, 42, 1028, 1, 110, 749, 11, 657, 1980, 317, 0, 0, 0, 0, 0, 0, 0, 576, 1683, 1394, 2409, 1075, 1273, 966, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7942, 7004, 9142, 8657, 8214, 7760, 7441, 9376] avg=8192.0000 max=9376 max_rank=7 min=7004 min_rank=1 imbalance_ratio=1.144531 max_load_reduction=56160 max_load_reduction_pct=0.856934 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4608, 128, 14336, 13952, 128, 12288, 8960, 7552, 4224, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 13312, 14592, 3328, 4992, 4864, 14592, 10112, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3584, 10624, 5760, 9344, 128, 6528, 14592, 9856, 4480, 1152, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9216, 19584, 5760, 7168, 11136, 11392, 128, 1792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[21632, 512, 5632, 2816, 9472, 128, 14976, 1664, 9344, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 512, 4480, 9984, 20736, 5120, 18944, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[21888, 384, 10752, 128, 1024, 6656, 128, 5504, 17280, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 3840, 10240, 8960, 18432, 8320, 8576, 7552, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 3013, 0, 3617, 4096, 0, 1706, 2686, 0, 0, 455, 3842, 0, 0, 0, 0, 0, 4096, 3343, 109, 79, 0, 2319, 0, 3519, 22, 1, 2490, 2, 0, 0, 1117, 0, 0, 1568, 4030, 2718, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 4096, 0, 0, 0, 26, 4096, 200, 0, 0, 0, 0, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 2458, 0, 3784, 4096, 0, 1615, 2523, 0, 0, 1769, 3337, 0, 0, 0, 0, 0, 4091, 3848, 25, 87, 0, 1743, 0, 3160, 0, 1, 1685, 0, 1, 0, 825, 0, 0, 1943, 4079, 3678, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4094, 4096, 0, 1, 0, 24, 4096, 279, 0, 0, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 1724, 0, 3770, 4095, 0, 2925, 3466, 0, 0, 457, 3845, 0, 0, 0, 0, 0, 4095, 3160, 18, 845, 0, 2057, 0, 3727, 0, 0, 1698, 0, 0, 0, 551, 2, 0, 679, 3920, 3604, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4004, 4096, 0, 1, 0, 115, 4096, 339, 0, 0, 0, 0, 54] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 2268, 0, 3492, 4095, 0, 2683, 3037, 0, 0, 1123, 3805, 0, 0, 0, 0, 0, 4093, 3662, 91, 234, 0, 1047, 2, 4024, 10, 7, 1426, 1, 3, 0, 238, 1, 0, 1270, 4083, 3348, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4008, 4094, 0, 0, 0, 0, 4096, 1062, 0, 0, 0, 0, 39] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 3608, 0, 3729, 4096, 0, 2659, 2861, 0, 0, 382, 3550, 0, 0, 0, 0, 0, 4093, 3887, 3, 35, 0, 1074, 0, 4061, 2, 14, 2626, 0, 1, 0, 122, 0, 0, 584, 4054, 3164, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4066, 4096, 0, 0, 0, 19, 4096, 383, 0, 0, 0, 0, 78] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 3796, 0, 3986, 4096, 0, 3307, 2491, 0, 0, 817, 3403, 0, 0, 0, 0, 0, 4094, 3426, 13, 30, 0, 1188, 0, 3945, 0, 0, 2207, 0, 1, 0, 57, 0, 0, 628, 4019, 2759, 0, 0, 1, 0, 0, 4096, 0, 1, 0, 4077, 4094, 0, 0, 0, 1, 4095, 751, 0, 0, 0, 0, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 2, 2935, 0, 3587, 4084, 0, 1239, 2745, 0, 0, 1370, 3602, 0, 0, 0, 0, 0, 4092, 3464, 10, 593, 0, 1610, 0, 3871, 0, 0, 2633, 1, 1, 0, 213, 0, 0, 1381, 3861, 3175, 0, 0, 0, 0, 2, 4096, 0, 1, 0, 4088, 4095, 0, 5, 0, 15, 4095, 525, 2, 0, 0, 0, 49] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 0, 1758, 0, 3708, 4090, 0, 1554, 3271, 1, 1, 215, 3664, 1, 4, 1, 1, 0, 4055, 4064, 105, 32, 0, 2250, 0, 3781, 2, 8, 2955, 0, 3, 1, 535, 0, 0, 1370, 3787, 3524, 0, 0, 0, 1, 0, 4093, 10, 0, 0, 4074, 4090, 1, 0, 0, 125, 4081, 165, 1, 1, 0, 0, 57] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 3013, 0, 1365, 0, 0, 0, 0, 0, 0, 0, 2218, 0, 0, 1706, 2686, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 0, 0, 0, 0, 956, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 109, 79, 0, 2319, 0, 3519, 22, 1, 2470, 2, 0, 0, 0, 0, 0, 0, 2490, 2, 0, 0, 1117, 0, 0, 1568, 4096, 246, 0, 0, 0, 0, 0, 0, 4030, 2718, 0, 0, 0, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 200, 0, 0, 0, 0, 20, 2894, 659, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8476, 7065, 8141, 8521, 9519, 7715, 8204, 7895] avg=8192.0000 max=9519 max_rank=4 min=7065 min_rank=1 imbalance_ratio=1.161987 max_load_reduction=56017 max_load_reduction_pct=0.854752 moved_to_echo=33333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 2458, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 2319, 0, 0, 1615, 2523, 0, 0, 1769, 0, 0, 0, 0, 0, 0, 0, 0, 3337, 0, 0, 0, 0, 0, 955, 3848, 0, 0, 0, 0, 0, 0, 0, 0, 25, 87, 0, 1743, 0, 3160, 0, 1, 2470, 2, 0, 0, 0, 0, 0, 0, 1685, 0, 1, 0, 825, 0, 0, 1943, 4096, 246, 0, 0, 0, 0, 0, 0, 4079, 3678, 0, 0, 0, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 4096, 0, 1, 0, 34, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 279, 0, 0, 0, 0, 7, 2890, 659, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7981, 8226, 8140, 7488, 8796, 8724, 8226, 7955] avg=8192.0000 max=8796 max_rank=4 min=7488 min_rank=3 imbalance_ratio=1.073730 max_load_reduction=56740 max_load_reduction_pct=0.865784 moved_to_echo=33701 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 1724, 0, 1423, 0, 0, 0, 0, 0, 0, 0, 2311, 0, 0, 2925, 3466, 0, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 956, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 18, 845, 0, 2057, 0, 3727, 0, 0, 2470, 2, 0, 0, 0, 0, 0, 0, 1698, 0, 0, 0, 551, 2, 0, 679, 4095, 246, 0, 0, 0, 0, 0, 0, 3920, 3604, 0, 0, 0, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 4096, 0, 1, 0, 34, 0, 0, 0, 0, 0, 0, 0, 115, 4096, 339, 0, 0, 0, 0, 54, 2893, 659, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7244, 9159, 7961, 9119, 7271, 8491, 8135, 8156] avg=8192.0000 max=9159 max_rank=1 min=7244 min_rank=0 imbalance_ratio=1.118042 max_load_reduction=56377 max_load_reduction_pct=0.860245 moved_to_echo=32053 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 2268, 0, 1318, 0, 0, 0, 0, 0, 0, 0, 2141, 0, 0, 2683, 3037, 0, 0, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 3805, 0, 0, 0, 0, 0, 955, 3662, 0, 0, 0, 0, 0, 0, 0, 0, 91, 234, 0, 1047, 2, 4024, 10, 7, 2470, 2, 0, 0, 0, 0, 0, 0, 1426, 1, 3, 0, 238, 1, 0, 1270, 4095, 246, 0, 0, 0, 0, 0, 0, 4083, 3348, 0, 0, 0, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4008, 4094, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1062, 0, 0, 0, 0, 39, 2892, 659, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7682, 8984, 8422, 7887, 7280, 8398, 8135, 8748] avg=8192.0000 max=8984 max_rank=1 min=7280 min_rank=4 imbalance_ratio=1.096680 max_load_reduction=56552 max_load_reduction_pct=0.862915 moved_to_echo=32561 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 3608, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 2286, 0, 0, 2659, 2861, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 3550, 0, 0, 0, 0, 0, 955, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 3, 35, 0, 1074, 0, 4061, 2, 14, 2470, 2, 0, 0, 0, 0, 0, 0, 2626, 0, 1, 0, 122, 0, 0, 584, 4096, 246, 0, 0, 0, 0, 0, 0, 4054, 3164, 0, 0, 0, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4096, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 383, 0, 0, 0, 0, 78, 2892, 659, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9113, 8188, 8392, 7661, 7675, 8185, 8195, 8127] avg=8192.0000 max=9113 max_rank=0 min=7661 min_rank=3 imbalance_ratio=1.112427 max_load_reduction=56423 max_load_reduction_pct=0.860947 moved_to_echo=32182 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 3796, 0, 1505, 0, 0, 0, 0, 0, 0, 0, 2443, 0, 0, 3307, 2491, 0, 0, 817, 0, 0, 0, 0, 0, 0, 0, 0, 3403, 0, 0, 0, 0, 0, 956, 3426, 0, 0, 0, 0, 0, 0, 0, 0, 13, 30, 0, 1188, 0, 3945, 0, 0, 2470, 2, 0, 0, 0, 0, 0, 0, 2207, 0, 1, 0, 57, 0, 0, 628, 4096, 246, 0, 0, 0, 0, 0, 0, 4019, 2759, 0, 0, 1, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4077, 4094, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 751, 0, 0, 0, 0, 61, 2892, 659, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9397, 9058, 7785, 7648, 7235, 7746, 8208, 8459] avg=8192.0000 max=9397 max_rank=0 min=7235 min_rank=4 imbalance_ratio=1.147095 max_load_reduction=56139 max_load_reduction_pct=0.856613 moved_to_echo=31648 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 2, 2935, 0, 1354, 0, 0, 0, 0, 0, 0, 0, 2199, 0, 0, 1239, 2745, 0, 0, 1370, 0, 0, 0, 0, 0, 0, 0, 0, 3602, 0, 0, 0, 0, 0, 955, 3464, 0, 0, 0, 0, 0, 0, 0, 0, 10, 593, 0, 1610, 0, 3871, 0, 0, 2470, 2, 0, 0, 0, 0, 0, 0, 2633, 1, 1, 0, 213, 0, 0, 1381, 4084, 246, 0, 0, 0, 0, 0, 0, 3861, 3175, 0, 0, 0, 0, 2, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 4095, 0, 5, 0, 32, 0, 0, 0, 0, 0, 0, 0, 15, 4095, 525, 2, 0, 0, 0, 49, 2891, 659, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8385, 7553, 8021, 8556, 8559, 8005, 8221, 8236] avg=8192.0000 max=8559 max_rank=4 min=7553 min_rank=1 imbalance_ratio=1.044800 max_load_reduction=56977 max_load_reduction_pct=0.869400 moved_to_echo=33021 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 0, 1758, 0, 1404, 0, 0, 0, 0, 0, 0, 0, 2261, 0, 0, 1554, 3271, 1, 1, 215, 0, 0, 0, 0, 0, 0, 0, 0, 3664, 1, 4, 1, 1, 0, 939, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 105, 32, 0, 2250, 0, 3781, 2, 8, 2472, 6, 0, 0, 0, 0, 0, 0, 2955, 0, 3, 1, 535, 0, 0, 1370, 4090, 247, 0, 0, 0, 0, 0, 0, 3787, 3524, 0, 0, 0, 1, 0, 960, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4074, 4090, 1, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 125, 4081, 165, 1, 1, 0, 0, 57, 2869, 661, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7258, 7303, 8674, 8656, 9201, 8272, 8212, 7960] avg=8192.0000 max=9201 max_rank=4 min=7258 min_rank=0 imbalance_ratio=1.123169 max_load_reduction=56335 max_load_reduction_pct=0.859604 moved_to_echo=33645 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31872, 25984, 0, 0, 128, 128, 128, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[18304, 0, 0, 17792, 23168, 128, 128, 6656, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 2048, 0, 13312, 128, 30208, 128, 128, 19840, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[29056, 128, 128, 128, 128, 0, 7680, 28928, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 32768, 3712, 128, 128, 0, 0, 384, 23168, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 21632, 0, 11264, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32512, 32768, 128, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[17792, 128, 128, 128, 3712, 128, 0, 9472, 32768, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3061, 0, 0, 5, 0, 0, 0, 3769, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 737, 0, 4096, 9, 4092, 3844, 0, 4096, 0, 0, 0, 3803, 1058, 2, 0, 1, 3980, 4095, 0, 0, 196, 0, 5, 0, 4059, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 51, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 0, 3535, 0, 1, 41, 0, 0, 0, 3830, 4091, 4095, 0, 0, 4096, 0, 0, 4095, 0, 0, 331, 1, 4086, 2, 4096, 3839, 0, 4090, 0, 0, 0, 3846, 932, 0, 1, 33, 3988, 4096, 0, 0, 80, 0, 5, 0, 4079, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 4094, 0, 0, 45, 0, 9, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3469, 0, 0, 0, 0, 0, 0, 3915, 4085, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 458, 3, 4092, 16, 4096, 3753, 0, 4096, 0, 0, 0, 3914, 688, 0, 2, 26, 4093, 4089, 0, 0, 52, 0, 107, 0, 4080, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 13, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3492, 0, 0, 51, 0, 0, 0, 3600, 4091, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 735, 0, 4095, 8, 4095, 3005, 0, 4093, 0, 0, 0, 3923, 1607, 2, 0, 18, 3959, 4094, 0, 0, 88, 0, 4, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 3343, 0, 1, 8, 0, 0, 0, 3788, 4093, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 166, 0, 4092, 18, 4096, 4034, 0, 4094, 0, 0, 0, 4048, 785, 0, 0, 0, 3961, 4095, 0, 0, 192, 0, 26, 0, 4077, 0, 1, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 134, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 2911, 0, 0, 121, 0, 0, 0, 3809, 4095, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 1025, 1, 4094, 6, 4091, 3752, 0, 4090, 0, 0, 0, 3221, 1629, 1, 1, 134, 3835, 4096, 0, 0, 30, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 19, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3454, 0, 0, 10, 0, 0, 0, 3915, 4086, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 331, 5, 4079, 6, 4096, 3707, 0, 4091, 0, 0, 1, 3828, 1035, 0, 6, 20, 3992, 4095, 0, 0, 162, 0, 15, 0, 4095, 0, 6, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 19, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 1, 2603, 0, 2, 0, 0, 0, 0, 2881, 4068, 4095, 0, 1, 4094, 0, 0, 4095, 1, 0, 680, 4, 4095, 33, 4095, 3886, 0, 4087, 1, 1, 1, 3651, 2162, 0, 8, 57, 4062, 4084, 1, 1, 350, 0, 13, 1, 4050, 0, 16, 1, 4094, 0, 1, 0, 13, 1, 4088, 1, 1, 153, 0, 0, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3061, 0, 0, 5, 0, 3732, 1137, 55, 0, 0, 0, 0, 0, 0, 0, 3769, 4096, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 0, 0, 737, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 360, 3844, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 1058, 2, 0, 1, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 5, 0, 4059, 0, 0, 2542, 846, 554, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 1, 0, 4096, 4040, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7990, 8282, 8375, 8309, 7998, 8203, 8191, 8188] avg=8192.0000 max=8375 max_rank=2 min=7990 min_rank=0 imbalance_ratio=1.022339 max_load_reduction=57161 max_load_reduction_pct=0.872208 moved_to_echo=32580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3469, 0, 0, 0, 0, 3736, 1137, 54, 0, 0, 0, 0, 0, 0, 0, 3915, 4085, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 0, 0, 458, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 16, 360, 3753, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 688, 0, 2, 26, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 107, 0, 4080, 0, 9, 2542, 870, 554, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4096, 4035, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8396, 8417, 8095, 8225, 7853, 8215, 8191, 8144] avg=8192.0000 max=8417 max_rank=1 min=7853 min_rank=4 imbalance_ratio=1.027466 max_load_reduction=57119 max_load_reduction_pct=0.871567 moved_to_echo=32403 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 3535, 0, 1, 41, 0, 3736, 1137, 55, 0, 0, 0, 0, 0, 0, 0, 3830, 4091, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3541, 0, 0, 331, 1, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 2, 360, 3839, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 932, 0, 1, 33, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 5, 0, 4079, 0, 1, 2542, 848, 554, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 9, 0, 0, 0, 4096, 4041, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8508, 8337, 7959, 8291, 7952, 8109, 8189, 8191] avg=8192.0000 max=8508 max_rank=0 min=7952 min_rank=4 imbalance_ratio=1.038574 max_load_reduction=57028 max_load_reduction_pct=0.870178 moved_to_echo=32441 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 3343, 0, 1, 8, 0, 3736, 1137, 55, 0, 0, 0, 0, 0, 0, 0, 3788, 4093, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 0, 0, 166, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 18, 360, 4034, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 785, 0, 0, 0, 3119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 26, 0, 4077, 0, 1, 2542, 842, 554, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 1, 0, 4096, 4040, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8298, 7800, 8506, 7952, 8235, 8193, 8271] avg=8192.0000 max=8506 max_rank=3 min=7800 min_rank=2 imbalance_ratio=1.038330 max_load_reduction=57030 max_load_reduction_pct=0.870209 moved_to_echo=32651 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3492, 0, 0, 51, 0, 3735, 1137, 55, 0, 0, 0, 0, 0, 0, 0, 3600, 4091, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 0, 0, 735, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 8, 360, 3005, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 1607, 2, 0, 18, 3118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 4, 0, 4085, 0, 0, 2542, 841, 554, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 0, 0, 0, 4096, 4039, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8470, 8108, 8372, 7466, 8668, 8115, 8191, 8146] avg=8192.0000 max=8668 max_rank=4 min=7466 min_rank=3 imbalance_ratio=1.058105 max_load_reduction=56868 max_load_reduction_pct=0.867737 moved_to_echo=33120 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2911, 0, 0, 121, 0, 3731, 1137, 55, 0, 0, 0, 0, 0, 0, 0, 3809, 4095, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 0, 0, 1025, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 6, 360, 3752, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3221, 1629, 1, 1, 134, 3020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 4088, 0, 0, 2542, 815, 554, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 6, 0, 4096, 4041, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7955, 8321, 8662, 8208, 8006, 8030, 8192, 8162] avg=8192.0000 max=8662 max_rank=2 min=7955 min_rank=0 imbalance_ratio=1.057373 max_load_reduction=56874 max_load_reduction_pct=0.867828 moved_to_echo=32390 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3454, 0, 0, 10, 0, 3736, 1137, 56, 0, 0, 0, 0, 0, 0, 0, 3915, 4086, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 0, 0, 331, 5, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 6, 360, 3707, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 1035, 0, 6, 20, 3144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 15, 0, 4095, 0, 6, 2542, 848, 554, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 4039, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8393, 8418, 7957, 8165, 8033, 8223, 8193, 8154] avg=8192.0000 max=8418 max_rank=1 min=7957 min_rank=2 imbalance_ratio=1.027588 max_load_reduction=57118 max_load_reduction_pct=0.871552 moved_to_echo=32603 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 2603, 0, 2, 0, 0, 3738, 1143, 55, 0, 0, 0, 0, 0, 0, 0, 2881, 4068, 405, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3536, 1, 0, 680, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 33, 357, 3886, 0, 4087, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3651, 2162, 0, 8, 57, 3195, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 350, 0, 13, 1, 4050, 0, 16, 2547, 867, 559, 2, 0, 0, 0, 0, 1, 4092, 0, 1, 0, 13, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 153, 0, 0, 1, 0, 1, 4094, 4029, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7543, 7355, 8316, 8366, 9074, 8406, 8196, 8280] avg=8192.0000 max=9074 max_rank=4 min=7355 min_rank=1 imbalance_ratio=1.107666 max_load_reduction=56462 max_load_reduction_pct=0.861542 moved_to_echo=33956 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 28416, 128, 0, 4480, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 512, 0, 128, 128, 128, 128, 32768, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 2944, 29824, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 29568, 32768, 3328, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[30336, 9984, 128, 128, 384, 25216, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 25984, 0, 128, 256, 0, 29952, 9216, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 1152, 0, 256, 128, 32640, 0, 128, 20352, 6784, 4480, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4079, 4073, 0, 0, 0, 4070, 0, 4096, 4096, 0, 4024, 4096, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 0, 0, 4096, 0, 4051, 4096, 0, 558, 10, 0, 1, 0, 0, 0, 5, 4096, 0, 0, 3332, 328, 0, 1, 0, 4096, 0, 45, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4074, 4081, 0, 0, 0, 4071, 0, 4096, 4096, 0, 4055, 4096, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 0, 0, 4096, 0, 4079, 4096, 0, 1478, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2484, 127, 0, 0, 0, 4095, 4, 11, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 4096, 4095, 0, 0, 0, 4035, 0, 4096, 4096, 0, 4087, 4096, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 0, 0, 4096, 0, 4079, 4096, 0, 1371, 31, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 2228, 830, 0, 0, 0, 4095, 4, 15, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 4077, 4090, 0, 0, 0, 4087, 0, 4096, 4095, 0, 4055, 4096, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 3617, 1, 0, 0, 4095, 0, 4032, 4096, 0, 854, 63, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 3279, 368, 0, 3, 0, 4096, 10, 19, 0, 1, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4079, 4018, 4093, 0, 0, 0, 4076, 0, 4096, 4096, 0, 3977, 4096, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 3472, 0, 0, 0, 4096, 0, 4096, 4095, 0, 2661, 11, 0, 1, 0, 0, 0, 2, 4096, 0, 0, 1792, 270, 0, 1, 0, 4096, 0, 32, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4093, 4045, 4089, 0, 0, 0, 4091, 0, 4096, 4090, 0, 4037, 4096, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, 0, 3637, 2, 0, 0, 4096, 0, 4093, 4096, 0, 1186, 8, 0, 10, 0, 0, 0, 9, 4096, 0, 0, 2493, 326, 0, 10, 0, 4096, 20, 27, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 0, 0, 4095, 4089, 4094, 0, 0, 0, 4058, 0, 4096, 4093, 0, 4072, 4096, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 3443, 0, 0, 0, 4096, 0, 4018, 4096, 0, 1379, 179, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 2718, 404, 0, 4, 0, 4096, 90, 41, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4094, 1, 0, 4095, 3931, 4086, 0, 1, 0, 3870, 0, 4095, 4092, 0, 4056, 4095, 1, 1, 1, 126, 0, 0, 0, 0, 1, 1, 0, 3899, 0, 1, 1, 4095, 1, 4081, 4094, 0, 943, 4, 1, 0, 0, 0, 0, 11, 4095, 0, 1, 3202, 261, 0, 11, 1, 4095, 0, 188, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4074, 97, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 4055, 0, 0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 3170, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1478, 2, 0, 0, 0, 0, 0, 0, 4096, 2746, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2484, 127, 0, 0, 0, 853, 181, 0, 0, 0, 0, 0, 0, 4095, 4, 11, 0, 0, 0, 0, 0, 3131, 896, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8242, 8453, 8240, 8205, 8322, 7741, 8137] avg=8192.0000 max=8453 max_rank=2 min=7741 min_rank=6 imbalance_ratio=1.031860 max_load_reduction=57083 max_load_reduction_pct=0.871017 moved_to_echo=32405 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4079, 96, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 4024, 0, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 3170, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4051, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 10, 0, 1, 0, 0, 0, 5, 4096, 2747, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3332, 328, 0, 1, 0, 852, 181, 0, 0, 0, 0, 0, 0, 4096, 0, 45, 0, 0, 0, 0, 0, 3125, 896, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8245, 8126, 8423, 8177, 7417, 8790, 8162] avg=8192.0000 max=8790 max_rank=6 min=7417 min_rank=5 imbalance_ratio=1.072998 max_load_reduction=56746 max_load_reduction_pct=0.865875 moved_to_echo=32546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 4077, 97, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 0, 4055, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3617, 3169, 1349, 0, 0, 0, 0, 0, 0, 1, 0, 0, 31, 0, 4032, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 854, 63, 0, 0, 0, 0, 0, 5, 4096, 2747, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3279, 368, 0, 3, 0, 855, 181, 0, 0, 0, 0, 0, 0, 4096, 10, 19, 0, 1, 0, 1, 0, 3138, 895, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8261, 8078, 8135, 8160, 7765, 8782, 8160] avg=8192.0000 max=8782 max_rank=6 min=7765 min_rank=5 imbalance_ratio=1.072021 max_load_reduction=56754 max_load_reduction_pct=0.865997 moved_to_echo=32867 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4079, 4, 0, 0, 0, 0, 0, 0, 0, 4018, 97, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 3977, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3472, 3170, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2661, 11, 0, 1, 0, 0, 0, 2, 4096, 2747, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1792, 270, 0, 1, 0, 856, 181, 0, 0, 0, 0, 0, 0, 4096, 0, 32, 0, 0, 0, 0, 0, 3140, 896, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8191, 8076, 7991, 8221, 9518, 7196, 8164] avg=8192.0000 max=9518 max_rank=5 min=7196 min_rank=6 imbalance_ratio=1.161865 max_load_reduction=56018 max_load_reduction_pct=0.854767 moved_to_echo=33099 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 97, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 4087, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 3170, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1371, 31, 0, 6, 0, 0, 0, 0, 4096, 2747, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2228, 830, 0, 0, 0, 856, 181, 0, 0, 0, 0, 0, 0, 4095, 4, 15, 0, 0, 0, 0, 0, 3142, 896, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8228, 8110, 8205, 8205, 8251, 8191, 8152] avg=8192.0000 max=8251 max_rank=5 min=8110 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32799 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 4089, 97, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 0, 4072, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3443, 3170, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4018, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1379, 179, 0, 0, 0, 0, 0, 8, 4096, 2747, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2718, 404, 0, 4, 0, 856, 181, 0, 0, 0, 0, 0, 0, 4096, 90, 41, 0, 1, 0, 0, 0, 3141, 896, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8244, 8067, 7962, 8144, 8409, 8259, 8265] avg=8192.0000 max=8409 max_rank=5 min=7962 min_rank=3 imbalance_ratio=1.026489 max_load_reduction=57127 max_load_reduction_pct=0.871689 moved_to_echo=33077 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 4045, 97, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 4037, 0, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3637, 3170, 1350, 0, 0, 0, 0, 0, 0, 2, 0, 0, 30, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1186, 8, 0, 10, 0, 0, 0, 9, 4096, 2746, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2493, 326, 0, 10, 0, 855, 181, 0, 0, 0, 0, 0, 0, 4096, 20, 27, 0, 0, 0, 0, 0, 3137, 896, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8233, 8540, 8157, 8221, 8055, 7961, 8176] avg=8192.0000 max=8540 max_rank=2 min=7961 min_rank=6 imbalance_ratio=1.042480 max_load_reduction=56996 max_load_reduction_pct=0.869690 moved_to_echo=32413 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4094, 1, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 3931, 90, 0, 1, 0, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 0, 4056, 0, 1, 1, 1, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3899, 3172, 1350, 0, 0, 0, 0, 0, 0, 0, 1, 1, 25, 1, 4081, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 4, 1, 0, 0, 0, 0, 11, 4095, 2745, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 3202, 261, 0, 11, 1, 859, 186, 0, 0, 0, 0, 0, 0, 4095, 0, 188, 1, 0, 0, 0, 1, 3137, 898, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 7892, 8086, 8423, 8203, 7799, 8616, 8320] avg=8192.0000 max=8616 max_rank=6 min=7799 min_rank=5 imbalance_ratio=1.051758 max_load_reduction=56920 max_load_reduction_pct=0.868530 moved_to_echo=32938 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[10496, 384, 128, 128, 0, 0, 0, 128, 32768, 22016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32512, 768, 0, 128, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31360, 0, 32384, 0, 128, 128, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 29440, 25472, 10880, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 21632, 2944, 0, 128, 128, 6912, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 256, 128, 32640, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 384, 128, 128, 0, 128, 128, 25216, 7296, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 2028, 0, 0, 4096, 0, 0, 0, 1470, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 4079, 0, 4095, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 3957, 4096, 0, 0, 0, 0, 161, 9, 4096, 0, 0, 0, 0, 4096, 0, 576, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 2345, 0, 0, 4096, 0, 0, 0, 1598, 4094, 7, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 1, 0, 4091, 0, 4095, 0, 1, 18, 0, 4094, 0, 0, 2, 6, 4096, 0, 0, 4096, 3, 0, 4096, 4068, 4096, 0, 0, 0, 0, 132, 3, 4096, 0, 0, 0, 0, 4096, 0, 28, 0, 4095, 4093, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 3068, 0, 0, 4096, 0, 0, 0, 797, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4081, 0, 4096, 0, 0, 4, 0, 4094, 0, 0, 3, 0, 4096, 0, 0, 4096, 20, 0, 4096, 4035, 4096, 0, 0, 0, 0, 188, 1, 4096, 0, 0, 0, 0, 4096, 0, 94, 0, 4096, 4092, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 3119, 0, 0, 4096, 0, 0, 0, 751, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 35, 0, 4096, 4079, 4096, 0, 0, 0, 0, 186, 0, 4096, 0, 0, 0, 0, 4096, 0, 22, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 3554, 0, 0, 4094, 0, 0, 0, 305, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 2, 26, 0, 4093, 0, 0, 1, 0, 4096, 0, 2, 4096, 0, 0, 4096, 3903, 4096, 0, 0, 0, 0, 190, 2, 4096, 0, 0, 1, 0, 4096, 0, 217, 0, 4096, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 3279, 0, 0, 4096, 0, 0, 0, 324, 4088, 21, 0, 0, 0, 0, 4086, 3, 0, 0, 0, 2, 0, 4090, 0, 4096, 0, 5, 35, 0, 4091, 0, 0, 6, 8, 4096, 0, 0, 4096, 5, 0, 4096, 4008, 4096, 0, 0, 0, 0, 222, 0, 4096, 0, 0, 0, 0, 4096, 0, 300, 0, 4096, 4090, 0, 0, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 1388, 0, 0, 4096, 0, 0, 0, 2654, 4093, 1, 0, 0, 0, 0, 4059, 1, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 0, 109, 0, 4094, 0, 0, 2, 3, 4076, 0, 5, 4096, 0, 0, 4096, 3881, 4096, 0, 0, 0, 0, 115, 0, 4096, 0, 0, 0, 0, 4096, 0, 104, 0, 4096, 4094, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1, 2214, 0, 1, 4093, 0, 1, 1, 1698, 4086, 1, 1, 1, 1, 0, 4071, 0, 1, 0, 0, 0, 1, 4086, 0, 4094, 1, 7, 6, 0, 4085, 0, 0, 16, 11, 4095, 0, 0, 4095, 37, 0, 4094, 4055, 4095, 1, 0, 1, 1, 128, 11, 4095, 1, 0, 0, 0, 4094, 0, 64, 1, 4095, 4081, 1, 1, 11, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3068, 0, 0, 4096, 0, 0, 0, 1185, 174, 67, 23, 17, 3, 0, 0, 797, 2905, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4079, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 4035, 0, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 3922, 4092, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8633, 7799, 8183, 8171, 8198, 8252, 8190, 8110] avg=8192.0000 max=8633 max_rank=0 min=7799 min_rank=1 imbalance_ratio=1.053833 max_load_reduction=56903 max_load_reduction_pct=0.868271 moved_to_echo=32750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2345, 0, 0, 4096, 0, 0, 0, 1184, 174, 67, 23, 17, 3, 0, 0, 1598, 2904, 7, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4072, 0, 1, 18, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 4096, 0, 0, 4079, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 4068, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 3921, 4093, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7909, 8597, 8194, 8185, 8186, 8229, 8192, 8044] avg=8192.0000 max=8597 max_rank=1 min=7909 min_rank=0 imbalance_ratio=1.049438 max_load_reduction=56939 max_load_reduction_pct=0.868820 moved_to_echo=32651 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3119, 0, 0, 4096, 0, 0, 0, 1185, 174, 67, 23, 17, 3, 0, 0, 751, 2905, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4079, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 4079, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 3922, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8684, 7751, 8197, 8171, 8210, 8294, 8189, 8040] avg=8192.0000 max=8684 max_rank=0 min=7751 min_rank=1 imbalance_ratio=1.060059 max_load_reduction=56852 max_load_reduction_pct=0.867493 moved_to_echo=32733 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3554, 0, 0, 4094, 0, 0, 0, 1185, 174, 67, 23, 17, 3, 0, 0, 305, 2905, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4073, 0, 2, 26, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 2, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 3903, 0, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 3922, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9117, 7306, 8196, 8194, 8178, 8122, 8192, 8231] avg=8192.0000 max=9117 max_rank=0 min=7306 min_rank=1 imbalance_ratio=1.112915 max_load_reduction=56419 max_load_reduction_pct=0.860886 moved_to_echo=32723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2028, 0, 0, 4096, 0, 0, 0, 1185, 174, 67, 23, 17, 3, 0, 0, 1470, 2905, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4079, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 3957, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 576, 0, 3922, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7593, 8471, 8186, 8172, 8175, 8147, 8198, 8594] avg=8192.0000 max=8594 max_rank=7 min=7593 min_rank=0 imbalance_ratio=1.049072 max_load_reduction=56942 max_load_reduction_pct=0.868866 moved_to_echo=33114 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3279, 0, 0, 4096, 0, 0, 0, 1182, 174, 67, 23, 17, 3, 0, 0, 324, 2900, 21, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 4090, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4073, 0, 5, 35, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8, 4096, 0, 0, 4079, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 4008, 0, 0, 0, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 3922, 4090, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8841, 7331, 8197, 8204, 8194, 8259, 8189, 8321] avg=8192.0000 max=8841 max_rank=0 min=7331 min_rank=1 imbalance_ratio=1.079224 max_load_reduction=56695 max_load_reduction_pct=0.865097 moved_to_echo=32963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1388, 0, 0, 4096, 0, 0, 0, 1184, 174, 67, 23, 17, 3, 0, 0, 2654, 2903, 1, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4087, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 109, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4076, 0, 5, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 3881, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 3922, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6952, 9617, 8190, 8276, 8165, 8025, 8189, 8122] avg=8192.0000 max=9617 max_rank=1 min=6952 min_rank=0 imbalance_ratio=1.173950 max_load_reduction=55919 max_load_reduction_pct=0.853256 moved_to_echo=32501 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 2214, 0, 1, 4093, 0, 1, 1, 1185, 181, 72, 30, 24, 4, 0, 0, 1698, 2891, 1, 1, 1, 1, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4086, 0, 4095, 10, 0, 0, 0, 0, 0, 0, 4064, 1, 7, 6, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 11, 4095, 0, 0, 4071, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 4055, 0, 1, 0, 1, 1, 128, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4091, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1, 3914, 4081, 1, 1, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7807, 8664, 8193, 8163, 8230, 8208, 8197, 8074] avg=8192.0000 max=8664 max_rank=1 min=7807 min_rank=0 imbalance_ratio=1.057617 max_load_reduction=56872 max_load_reduction_pct=0.867798 moved_to_echo=32709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1408, 128, 31488, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 128, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32256, 32000, 0, 128, 0, 128, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32640, 128, 128, 256, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9600, 23296, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 21120, 0, 128, 32768, 0, 128, 128, 9600, 1408, 640, 256, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66688 slack_slots=3584 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 7, 0, 4054, 0, 7, 1631, 4096, 0, 0, 0, 51, 0, 1, 31, 4096, 0, 4096, 0, 0, 0, 0, 4090, 1517, 0, 4043, 0, 26, 4096, 113, 4096, 0, 0, 41, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2635, 0, 945, 0, 571, 722, 0, 2, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 7, 0, 4091, 0, 8, 581, 4096, 0, 0, 0, 260, 0, 0, 370, 4096, 0, 4096, 0, 0, 0, 0, 4095, 1531, 0, 4048, 0, 1, 4096, 430, 4096, 0, 0, 14, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3512, 0, 70, 0, 858, 631, 0, 0, 0, 4068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 1, 0, 0, 15, 0, 4089, 0, 2, 1025, 4096, 0, 0, 0, 36, 0, 0, 84, 4096, 0, 4096, 0, 0, 0, 0, 4093, 1229, 0, 4041, 0, 9, 4096, 382, 4096, 0, 0, 44, 4096, 4096, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 3284, 0, 976, 0, 299, 902, 0, 1, 0, 4063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 29, 0, 4064, 0, 2, 1555, 4096, 0, 0, 0, 7, 0, 0, 14, 4096, 0, 4096, 0, 0, 0, 0, 4095, 1658, 0, 3965, 0, 211, 4096, 106, 4096, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2853, 0, 718, 0, 324, 896, 0, 0, 0, 4076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 1, 0, 0, 42, 0, 4094, 0, 1, 674, 4096, 0, 0, 0, 60, 0, 0, 112, 4096, 0, 4095, 0, 0, 0, 0, 4090, 2245, 0, 4041, 0, 3, 4096, 400, 4096, 0, 0, 9, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2898, 0, 649, 0, 605, 575, 0, 1, 0, 4078] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 2, 0, 0, 9, 0, 4080, 0, 2, 455, 4096, 0, 0, 0, 18, 0, 0, 48, 4096, 0, 4096, 0, 0, 0, 0, 4086, 1893, 0, 3996, 0, 191, 4096, 846, 4096, 0, 0, 11, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2326, 0, 852, 0, 612, 1054, 0, 1, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 2, 8, 0, 0, 3, 0, 4075, 0, 4, 1620, 4096, 0, 0, 0, 7, 0, 1, 19, 4096, 0, 4096, 0, 0, 0, 0, 4081, 1303, 0, 4080, 0, 5, 4096, 219, 4096, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2103, 0, 1189, 0, 1040, 712, 0, 9, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 4093, 0, 1, 0, 8, 0, 0, 8, 1, 4083, 0, 50, 529, 4095, 1, 1, 0, 392, 0, 7, 210, 4095, 1, 4095, 0, 1, 0, 1, 4083, 554, 0, 4067, 1, 3, 4095, 555, 4095, 1, 0, 5, 4095, 4095, 1, 1, 0, 1, 1, 0, 4095, 1, 0, 3111, 1, 71, 1, 218, 2548, 0, 2, 0, 4067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4091, 0, 8, 581, 2863, 216, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 260, 0, 0, 370, 3880, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1531, 0, 4048, 0, 1, 2229, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3512, 0, 1237, 16, 1, 0, 0, 0, 0, 0, 70, 0, 858, 631, 0, 0, 0, 4068, 1225, 630, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7766, 8606, 8199, 8239, 8190, 8862, 7482] avg=8192.0000 max=8862 max_rank=6 min=7482 min_rank=7 imbalance_ratio=1.081787 max_load_reduction=56674 max_load_reduction_pct=0.864777 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4054, 0, 7, 1631, 2863, 216, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 51, 0, 1, 31, 3880, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1517, 0, 4043, 0, 26, 2229, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2635, 0, 1237, 16, 1, 0, 0, 0, 0, 0, 945, 0, 571, 722, 0, 2, 0, 4092, 1225, 630, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8778, 8059, 8194, 7928, 8217, 7984, 8187] avg=8192.0000 max=8778 max_rank=1 min=7928 min_rank=4 imbalance_ratio=1.071533 max_load_reduction=56758 max_load_reduction_pct=0.866058 moved_to_echo=32316 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4089, 0, 2, 1025, 2863, 216, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 36, 0, 0, 84, 3880, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1229, 0, 4041, 0, 9, 2229, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 4096, 4080, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3284, 0, 1237, 16, 1, 0, 0, 0, 0, 0, 976, 0, 299, 902, 0, 1, 0, 4063, 1225, 630, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8210, 8096, 8197, 7890, 8222, 8634, 8096] avg=8192.0000 max=8634 max_rank=6 min=7890 min_rank=4 imbalance_ratio=1.053955 max_load_reduction=56902 max_load_reduction_pct=0.868256 moved_to_echo=32842 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 4094, 0, 1, 674, 2863, 216, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 60, 0, 0, 112, 3880, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4095, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2245, 0, 4041, 0, 3, 2229, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2898, 0, 1237, 16, 1, 0, 0, 0, 0, 0, 649, 0, 605, 575, 0, 1, 0, 4078, 1225, 630, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7890, 8148, 8193, 8918, 8185, 8248, 7763] avg=8192.0000 max=8918 max_rank=4 min=7763 min_rank=7 imbalance_ratio=1.088623 max_load_reduction=56618 max_load_reduction_pct=0.863922 moved_to_echo=33114 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4064, 0, 2, 1555, 2863, 216, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 14, 3880, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1658, 0, 3965, 0, 211, 2229, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2853, 0, 1237, 16, 1, 0, 0, 0, 0, 0, 718, 0, 324, 896, 0, 0, 0, 4076, 1225, 630, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8729, 7997, 8199, 8169, 8180, 8203, 7869] avg=8192.0000 max=8729 max_rank=1 min=7869 min_rank=7 imbalance_ratio=1.065552 max_load_reduction=56807 max_load_reduction_pct=0.866806 moved_to_echo=32421 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4080, 0, 2, 455, 2863, 216, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 48, 3880, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 1893, 0, 3996, 0, 191, 2229, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2326, 0, 1237, 16, 1, 0, 0, 0, 0, 0, 852, 0, 612, 1054, 0, 1, 0, 4095, 1225, 630, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7625, 8042, 8190, 9155, 8187, 7676, 8469] avg=8192.0000 max=9155 max_rank=4 min=7625 min_rank=1 imbalance_ratio=1.117554 max_load_reduction=56381 max_load_reduction_pct=0.860306 moved_to_echo=33487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 4096, 0, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4075, 0, 4, 1620, 2863, 216, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 1, 19, 3880, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 1303, 0, 4080, 0, 5, 2229, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2103, 0, 1237, 16, 1, 0, 0, 0, 0, 0, 1189, 0, 1040, 712, 0, 9, 0, 4094, 1227, 630, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8781, 8003, 8183, 7836, 8180, 7451, 8901] avg=8192.0000 max=8901 max_rank=7 min=7451 min_rank=6 imbalance_ratio=1.086548 max_load_reduction=56635 max_load_reduction_pct=0.864182 moved_to_echo=32368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4087, 4093, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 4083, 0, 50, 529, 2870, 216, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 392, 0, 7, 210, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 1, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 4067, 1, 3, 2221, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 4095, 4072, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 0, 3111, 1, 1237, 23, 8, 0, 0, 0, 0, 0, 71, 1, 218, 2548, 0, 2, 0, 4067, 1225, 637, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7757, 8585, 8181, 7401, 8175, 8478, 8769] avg=8192.0000 max=8769 max_rank=7 min=7401 min_rank=4 imbalance_ratio=1.070435 max_load_reduction=56767 max_load_reduction_pct=0.866196 moved_to_echo=32823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 256, 32768, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 22784, 128, 9984, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 896, 0, 128, 896, 31104, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5504, 128, 4608, 8064, 0, 128, 0, 32640, 9856, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12032, 0, 32384, 128, 512, 17920, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 32640, 0, 128, 8192, 22912, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 0, 105, 0, 4096, 4096, 0, 4087, 4096, 0, 6, 0, 0, 4096, 4096, 0, 4096, 0, 0, 3936, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 108, 0, 1, 0, 365, 0, 4096, 4094, 0, 3879, 4096, 0, 9, 0, 0, 4096, 4095, 0, 4096, 0, 0, 3791, 4096, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 23, 0, 0, 0, 8, 1, 0, 4095, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 1, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 175, 0, 0, 0, 95, 0, 4096, 4096, 0, 3618, 4096, 0, 52, 0, 0, 4096, 4093, 0, 4096, 0, 0, 3898, 4096, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 118, 0, 0, 0, 234, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 206, 0, 0, 2, 306, 0, 4096, 4096, 0, 3794, 4096, 0, 4, 0, 0, 4096, 4094, 0, 4096, 0, 0, 3821, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 34, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 145, 0, 0, 2, 390, 0, 4096, 4094, 0, 3879, 4096, 0, 32, 0, 0, 4096, 4094, 0, 4096, 0, 0, 3726, 4096, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 7, 0, 0, 0, 10, 0, 0, 4095, 0, 0, 0, 4096, 4087, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 1, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 80, 0, 0, 0, 236, 0, 4096, 4090, 0, 3843, 4096, 0, 10, 0, 0, 4096, 4085, 0, 4096, 0, 0, 3912, 4095, 0, 0, 4095, 0, 0, 0, 0, 8, 0, 0, 31, 0, 0, 0, 62, 3, 0, 4092, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 3, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 501, 0, 0, 0, 243, 0, 4095, 4095, 0, 4010, 4094, 1, 21, 0, 1, 4088, 4095, 0, 4095, 0, 0, 3246, 4095, 1, 1, 4091, 1, 0, 1, 0, 38, 1, 1, 119, 0, 0, 1, 25, 0, 0, 4089, 0, 2, 1, 4091, 4091, 1, 1, 0, 4095, 3, 0, 0, 1, 1, 0, 7, 4095, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 60, 0, 0, 0, 373, 0, 4096, 4095, 0, 3951, 4096, 0, 60, 0, 0, 4095, 4096, 0, 4096, 0, 0, 3638, 4096, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 77, 0, 0, 0, 36, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 0, 4096, 3873, 63, 0, 0, 0, 0, 0, 105, 0, 0, 4047, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4076, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 351, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 3745, 223, 49, 20, 3, 1, 0, 0, 4093, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8037, 8239, 8178, 8383, 8163, 8153, 8192, 8191] avg=8192.0000 max=8383 max_rank=3 min=8037 min_rank=0 imbalance_ratio=1.023315 max_load_reduction=57153 max_load_reduction_pct=0.872086 moved_to_echo=32699 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 108, 0, 1, 0, 4096, 3872, 63, 0, 0, 0, 0, 0, 365, 0, 0, 4045, 0, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4076, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 351, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 23, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 4095, 0, 0, 0, 3745, 224, 49, 20, 3, 1, 0, 0, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8289, 8180, 8238, 8162, 8146, 8188, 8193] avg=8192.0000 max=8289 max_rank=1 min=8140 min_rank=0 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32689 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 175, 0, 0, 0, 4096, 3872, 63, 0, 0, 0, 0, 0, 95, 0, 0, 4047, 0, 3618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 4076, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 351, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 118, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 4096, 0, 0, 0, 3745, 224, 49, 20, 3, 1, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 7760, 8221, 8345, 8252, 8372, 8189, 8191] avg=8192.0000 max=8372 max_rank=5 min=7760 min_rank=1 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=33004 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 206, 0, 0, 2, 4096, 3872, 63, 0, 0, 0, 0, 0, 306, 0, 0, 4047, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4076, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 351, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 4096, 0, 0, 0, 3745, 224, 49, 20, 3, 1, 0, 0, 4093, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8147, 8174, 8268, 8155, 8172, 8190, 8191] avg=8192.0000 max=8268 max_rank=3 min=8147 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32708 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 145, 0, 0, 2, 4096, 3872, 63, 0, 0, 0, 0, 0, 390, 0, 0, 4045, 0, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 4076, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 351, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4095, 0, 0, 0, 3745, 224, 49, 20, 3, 1, 0, 0, 4093, 4087, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8314, 8202, 8173, 8142, 8147, 8188, 8192] avg=8192.0000 max=8314 max_rank=1 min=8142 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32669 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 60, 0, 0, 0, 4096, 3872, 63, 0, 0, 0, 0, 0, 373, 0, 0, 4046, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 4075, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3638, 351, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 77, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4096, 0, 0, 0, 3745, 224, 49, 20, 3, 1, 0, 0, 4093, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8091, 8370, 8231, 8084, 8207, 8174, 8189, 8190] avg=8192.0000 max=8370 max_rank=1 min=8084 min_rank=3 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32760 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 80, 0, 0, 0, 4096, 3873, 63, 0, 0, 0, 0, 0, 236, 0, 0, 4041, 0, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4076, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 351, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 31, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 62, 3, 0, 4092, 0, 0, 0, 3744, 223, 49, 20, 3, 1, 0, 0, 4093, 4095, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8120, 8171, 8358, 8168, 8197, 8216, 8194] avg=8192.0000 max=8358 max_rank=3 min=8112 min_rank=0 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32775 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 501, 0, 0, 0, 4095, 3871, 63, 0, 0, 0, 0, 0, 243, 0, 0, 4043, 0, 4010, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 1, 4062, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3246, 347, 1, 1, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 38, 1, 1, 119, 0, 0, 4095, 4032, 0, 0, 0, 0, 0, 0, 1, 25, 0, 0, 4089, 0, 2, 1, 3748, 223, 52, 26, 3, 5, 0, 0, 4088, 4091, 1, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 7, 4095, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8533, 8297, 8179, 7687, 8287, 8175, 8184, 8194] avg=8192.0000 max=8533 max_rank=0 min=7687 min_rank=3 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=32840 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 512, 0, 0, 32768, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 32768, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 30080, 2816, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[256, 0, 128, 32640, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 512, 128, 0, 32768, 0, 128, 128, 30080, 1792, 512, 256, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 1280, 0, 128, 128, 32768, 31104, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[2176, 0, 0, 32384, 0, 31104, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66688 slack_slots=3584 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1924, 0, 0, 4096, 4, 0, 4096, 4094, 0, 0, 4096, 0, 0, 4088, 4096, 175, 0, 0, 1, 0, 4096, 22, 0, 4075, 0, 3438, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2808, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3019, 0, 0, 4091, 4, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4086, 4096, 36, 0, 0, 9, 0, 4096, 157, 0, 3955, 0, 3489, 0, 0, 0, 0, 0, 0, 0, 3260, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2470, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3657, 0, 0, 4096, 5, 0, 4096, 4095, 0, 0, 4096, 0, 0, 4095, 4096, 24, 0, 0, 3, 0, 4096, 100, 0, 4025, 0, 1829, 0, 0, 0, 0, 0, 0, 0, 3610, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3133, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2, 0, 0, 4096, 0, 3146, 0, 0, 4090, 20, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4085, 4096, 29, 0, 0, 8, 0, 4096, 26, 0, 4021, 0, 2858, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2328, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3517, 0, 0, 4094, 5, 0, 4096, 4095, 0, 0, 4095, 0, 0, 4088, 4095, 20, 0, 0, 6, 0, 4096, 158, 0, 4078, 0, 2468, 0, 0, 0, 0, 1, 0, 0, 3796, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 2, 0, 0, 0, 0, 4096, 4096, 0, 2345, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3200, 0, 0, 4094, 4, 0, 4096, 4096, 0, 0, 4095, 0, 0, 4090, 4096, 212, 0, 3, 10, 0, 4096, 154, 0, 4089, 0, 3118, 0, 0, 0, 0, 1, 0, 0, 3480, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2122, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3782, 0, 0, 4093, 15, 0, 4096, 4096, 0, 0, 4091, 0, 0, 4048, 4095, 71, 1, 0, 28, 0, 4096, 94, 0, 4047, 0, 2633, 0, 0, 0, 0, 3, 0, 0, 3818, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 4096, 0, 1944, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 1, 0, 4095, 1, 2786, 1, 1, 4072, 10, 1, 4087, 4095, 0, 0, 4088, 1, 0, 4084, 4094, 23, 0, 7, 13, 1, 4095, 18, 0, 4059, 0, 2188, 0, 1, 0, 0, 0, 0, 0, 4029, 0, 1, 1, 0, 0, 4094, 1, 1, 0, 2, 1, 0, 1, 0, 4095, 4095, 1, 3295, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3657, 0, 0, 4096, 5, 0, 965, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 3, 0, 4096, 100, 0, 2759, 1160, 0, 0, 0, 0, 0, 0, 4025, 0, 1829, 0, 0, 0, 0, 0, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 0, 0, 0, 0, 0, 4096, 358, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3721, 372, 0, 0, 0, 0, 0, 0, 0, 1542, 4096, 0, 3133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8723, 8207, 8142, 7248, 8064, 8189, 8771] avg=8192.0000 max=8771 max_rank=7 min=7248 min_rank=4 imbalance_ratio=1.070679 max_load_reduction=56765 max_load_reduction_pct=0.866165 moved_to_echo=32272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3146, 0, 0, 4090, 20, 0, 965, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 8, 0, 4096, 26, 0, 2759, 1160, 0, 0, 0, 0, 0, 0, 4021, 0, 2858, 0, 0, 0, 0, 0, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 4096, 358, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3722, 372, 0, 0, 0, 0, 0, 0, 0, 1542, 4096, 0, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8221, 8197, 8078, 8273, 8417, 8190, 7966] avg=8192.0000 max=8417 max_rank=5 min=7966 min_rank=7 imbalance_ratio=1.027466 max_load_reduction=57119 max_load_reduction_pct=0.871567 moved_to_echo=32846 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3019, 0, 0, 4091, 4, 0, 965, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 9, 0, 4096, 157, 0, 2759, 1160, 0, 0, 0, 0, 0, 0, 3955, 0, 3489, 0, 0, 0, 0, 0, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3260, 0, 0, 0, 0, 0, 4096, 358, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3722, 372, 0, 0, 0, 0, 0, 0, 0, 1542, 4096, 0, 2470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8079, 8198, 8217, 8838, 7714, 8190, 8108] avg=8192.0000 max=8838 max_rank=4 min=7714 min_rank=5 imbalance_ratio=1.078857 max_load_reduction=56698 max_load_reduction_pct=0.865143 moved_to_echo=32850 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3517, 0, 0, 4094, 5, 0, 965, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4095, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 6, 0, 4096, 158, 0, 2759, 1160, 0, 0, 0, 0, 0, 0, 4078, 0, 2468, 0, 0, 0, 0, 1, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 0, 0, 0, 0, 0, 4095, 358, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 2, 0, 0, 0, 3721, 372, 0, 0, 0, 0, 0, 0, 0, 1542, 4096, 0, 2345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8581, 8199, 8199, 7941, 8249, 8192, 7983] avg=8192.0000 max=8581 max_rank=1 min=7941 min_rank=4 imbalance_ratio=1.047485 max_load_reduction=56955 max_load_reduction_pct=0.869064 moved_to_echo=32365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1924, 0, 0, 4096, 4, 0, 965, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 1, 0, 4096, 22, 0, 2759, 1160, 0, 0, 0, 0, 0, 0, 4075, 0, 3438, 0, 0, 0, 0, 0, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 4096, 358, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3721, 372, 0, 0, 0, 0, 0, 0, 0, 1542, 4096, 0, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6989, 8199, 8213, 8907, 8401, 8189, 8446] avg=8192.0000 max=8907 max_rank=4 min=6989 min_rank=1 imbalance_ratio=1.087280 max_load_reduction=56629 max_load_reduction_pct=0.864090 moved_to_echo=33943 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3200, 0, 0, 4094, 4, 0, 965, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4095, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 3, 10, 0, 4096, 154, 0, 2759, 1160, 0, 0, 0, 0, 0, 0, 4089, 0, 3118, 0, 0, 0, 0, 1, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 0, 0, 0, 0, 0, 4096, 358, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3722, 372, 0, 0, 0, 0, 0, 0, 0, 1542, 4096, 0, 2122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8263, 8201, 8394, 8602, 7934, 8190, 7760] avg=8192.0000 max=8602 max_rank=4 min=7760 min_rank=7 imbalance_ratio=1.050049 max_load_reduction=56934 max_load_reduction_pct=0.868744 moved_to_echo=32486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3782, 0, 0, 4093, 15, 0, 965, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4091, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 1, 0, 28, 0, 4096, 94, 0, 2759, 1160, 0, 0, 0, 0, 0, 0, 4047, 0, 2633, 0, 0, 0, 0, 3, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 0, 0, 0, 0, 0, 4095, 358, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 3722, 372, 0, 0, 0, 0, 0, 0, 0, 1542, 4096, 0, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8855, 8155, 8209, 8077, 8271, 8195, 7582] avg=8192.0000 max=8855 max_rank=1 min=7582 min_rank=7 imbalance_ratio=1.080933 max_load_reduction=56681 max_load_reduction_pct=0.864883 moved_to_echo=32125 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2786, 1, 1, 4072, 10, 1, 953, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4088, 1, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 7, 13, 1, 4095, 18, 0, 2760, 1167, 0, 0, 0, 0, 0, 0, 4059, 0, 2188, 0, 1, 0, 0, 0, 1401, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 1, 1, 0, 0, 4094, 361, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 2, 1, 0, 1, 3727, 374, 0, 0, 0, 0, 0, 0, 0, 1527, 4095, 1, 3295, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7825, 8180, 8084, 7650, 8486, 8201, 8920] avg=8192.0000 max=8920 max_rank=7 min=7650 min_rank=4 imbalance_ratio=1.088867 max_load_reduction=56616 max_load_reduction_pct=0.863892 moved_to_echo=33257 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 12416, 32768, 128, 20480, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 128, 29824, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32384, 0, 22144, 0, 128, 0, 0, 128, 11264, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 128, 128, 128, 128, 32768, 768, 0, 22144, 9344, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 25088, 128, 128, 32768, 128, 128, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 29952, 0, 128, 128, 0, 0, 32768, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4066, 4056, 0, 0, 0, 0, 0, 0, 13, 0, 4061, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 1421, 11, 0, 4096, 4096, 0, 626, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 2129, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4065, 4043, 0, 0, 0, 0, 4, 0, 36, 0, 3999, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4093, 181, 1, 0, 4096, 4096, 0, 472, 0, 2, 3, 0, 1, 0, 4095, 0, 0, 4092, 3586, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 3974, 0, 0, 0, 0, 0, 0, 23, 0, 4048, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 1935, 2, 0, 4096, 4096, 0, 428, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 1888, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4088, 3959, 0, 0, 0, 0, 21, 0, 48, 0, 3898, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 4084, 792, 1, 0, 4096, 4096, 0, 336, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4093, 3253, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4077, 3907, 0, 0, 0, 0, 1, 0, 22, 0, 4016, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 10, 0, 0, 0, 4094, 786, 0, 0, 4096, 4096, 0, 488, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 3081, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[7, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4006, 4036, 0, 0, 0, 0, 2, 0, 32, 0, 3917, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 564, 2, 0, 4096, 4096, 0, 997, 0, 1, 1, 0, 0, 0, 4095, 0, 0, 4072, 2849, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 1, 0, 14, 0, 0, 3949, 4055, 0, 1, 1, 1, 1, 1, 77, 1, 4051, 2, 0, 1, 0, 4086, 1, 1, 1, 0, 4095, 1, 0, 4094, 0, 0, 4090, 2, 0, 1, 31, 4093, 587, 7, 0, 4095, 4094, 0, 258, 1, 0, 9, 1, 0, 0, 4095, 1, 1, 4094, 3485, 0, 3966, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[6, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4084, 4075, 0, 0, 0, 0, 36, 0, 13, 0, 3980, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4094, 523, 0, 0, 4096, 4096, 0, 200, 0, 1, 1, 0, 0, 0, 4094, 0, 0, 4089, 3487, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4038, 54, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4056, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4061, 0, 0, 0, 4096, 58, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1421, 11, 0, 0, 3245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 1, 0, 0, 0, 0, 4096, 2898, 723, 0, 0, 0, 0, 0, 0, 0, 0, 1144, 2129, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8247, 8228, 8195, 8192, 8773, 8344, 7369] avg=8192.0000 max=8773 max_rank=5 min=7369 min_rank=7 imbalance_ratio=1.070923 max_load_reduction=56763 max_load_reduction_pct=0.866135 moved_to_echo=32678 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 1, 0, 4038, 54, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4043, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 4, 0, 36, 0, 3999, 0, 0, 0, 4096, 58, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 181, 1, 0, 0, 3245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 0, 2, 3, 0, 1, 0, 4095, 2895, 723, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 3586, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8233, 8193, 8192, 8192, 7520, 8191, 8825] avg=8192.0000 max=8825 max_rank=7 min=7520 min_rank=5 imbalance_ratio=1.077271 max_load_reduction=56711 max_load_reduction_pct=0.865341 moved_to_echo=32728 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 4037, 54, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3959, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 21, 0, 48, 0, 3898, 0, 0, 0, 4096, 59, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 792, 1, 0, 0, 3245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 0, 2, 0, 0, 0, 0, 4096, 2895, 723, 0, 0, 0, 0, 0, 0, 0, 0, 1144, 3253, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8172, 8122, 8193, 8194, 8122, 8052, 8493] avg=8192.0000 max=8493 max_rank=7 min=8052 min_rank=6 imbalance_ratio=1.036743 max_load_reduction=57043 max_load_reduction_pct=0.870407 moved_to_echo=32861 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4038, 54, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3907, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 1, 0, 22, 0, 4016, 0, 0, 0, 4096, 58, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 786, 0, 0, 0, 3245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 0, 0, 0, 0, 0, 0, 4096, 2897, 723, 0, 0, 0, 0, 0, 0, 0, 0, 1144, 3081, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8109, 8193, 8194, 8202, 8125, 8204, 8321] avg=8192.0000 max=8321 max_rank=7 min=8109 min_rank=1 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32852 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4038, 54, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3974, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4048, 0, 0, 0, 4096, 58, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1935, 2, 0, 0, 3245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 0, 0, 0, 0, 0, 4096, 2896, 723, 0, 0, 0, 0, 0, 0, 0, 0, 1144, 1888, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8188, 8225, 8195, 8192, 9277, 8143, 7128] avg=8192.0000 max=9277 max_rank=5 min=7128 min_rank=7 imbalance_ratio=1.132446 max_load_reduction=56259 max_load_reduction_pct=0.858444 moved_to_echo=32740 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 4096, 0, 0, 1, 0, 4038, 53, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 4036, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 2, 0, 32, 0, 3917, 0, 0, 0, 4096, 58, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 564, 2, 0, 0, 3245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 0, 1, 1, 0, 0, 0, 4095, 2881, 723, 0, 0, 0, 0, 0, 0, 0, 0, 1138, 2849, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8167, 8105, 8190, 8192, 7906, 8698, 8083] avg=8192.0000 max=8698 max_rank=6 min=7906 min_rank=5 imbalance_ratio=1.061768 max_load_reduction=56838 max_load_reduction_pct=0.867279 moved_to_echo=32879 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 4096, 0, 0, 2, 0, 4038, 54, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4075, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 36, 0, 13, 0, 3980, 0, 0, 0, 4096, 58, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 523, 0, 0, 0, 3245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 1, 1, 0, 0, 0, 4094, 2893, 723, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 3487, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8284, 8183, 8182, 8192, 7862, 7912, 8725] avg=8192.0000 max=8725 max_rank=7 min=7862 min_rank=5 imbalance_ratio=1.065063 max_load_reduction=56811 max_load_reduction_pct=0.866867 moved_to_echo=32691 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 1, 0, 14, 4037, 56, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 4055, 0, 1, 1, 1, 129, 0, 0, 0, 0, 0, 0, 0, 1, 1, 77, 1, 4051, 2, 0, 1, 4095, 58, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 1, 1, 0, 4095, 1, 3, 7, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4090, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4093, 587, 7, 0, 0, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 1, 0, 9, 1, 0, 0, 4095, 2899, 729, 0, 0, 0, 0, 0, 0, 1, 1, 1139, 3485, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8136, 8287, 8195, 8180, 7951, 7992, 8592] avg=8192.0000 max=8592 max_rank=7 min=7951 min_rank=5 imbalance_ratio=1.048828 max_load_reduction=56944 max_load_reduction_pct=0.868896 moved_to_echo=32715 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 32768, 0, 128, 128, 128, 32384, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3840, 128, 128, 128, 128, 128, 0, 32768, 23168, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 9216, 23808, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 6912, 128, 0, 0, 25984, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 32512, 32128, 0, 128, 128, 128, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 384, 128, 32000, 128, 0, 128, 32768, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 0, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 4084, 0, 0, 0, 0, 0, 4095, 24, 0, 0, 0, 0, 3991, 2517, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 2935, 0, 0, 0, 3845, 0, 0, 0, 0, 9, 0, 0, 0, 3093, 0, 0, 4096, 4077, 0, 1, 0, 0, 0, 0, 4096, 2, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 4055, 2125, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3133, 0, 0, 0, 4062, 0, 0, 0, 0, 6, 0, 0, 0, 3008, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4082, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 4081, 2893, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3274, 0, 0, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 2237, 0, 0, 4096, 4089, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 4090, 3330, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3335, 0, 0, 0, 4040, 0, 0, 0, 0, 4, 0, 0, 0, 1574, 0, 0, 4096, 4081, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4086, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 4081, 1815, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3669, 0, 0, 0, 3771, 0, 0, 0, 0, 10, 0, 0, 0, 3081, 0, 0, 4096, 4057, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 4095, 46, 0, 0, 0, 0, 4069, 2485, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 4095, 0, 0, 3732, 0, 0, 0, 3915, 0, 0, 0, 0, 1, 0, 0, 0, 2147, 0, 0, 4096, 4084, 0, 1, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4090, 4096, 0, 0, 0, 0, 0, 4094, 29, 0, 0, 0, 0, 4093, 3004, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 3482, 0, 0, 0, 4025, 0, 0, 0, 0, 5, 0, 0, 0, 1786, 0, 0, 4096, 4053, 0, 4, 0, 0, 0, 0, 4096, 6, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 3373, 1, 1, 7, 1, 0, 4095, 0, 1, 0, 0, 0, 3922, 3213, 1, 1, 1, 1, 0, 1, 4095, 0, 1, 1, 0, 1, 0, 4094, 4086, 0, 1, 2950, 0, 1, 0, 4044, 0, 0, 0, 1, 155, 0, 1, 2, 2877, 1, 1, 4095, 4024, 1, 10, 0, 0, 0, 1, 4094, 0, 4095, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[97, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 4055, 29, 0, 0, 0, 0, 0, 0, 0, 2125, 0, 0, 0, 0, 0, 0, 4096, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3133, 0, 0, 0, 4062, 0, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3008, 4096, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8289, 8187, 7644, 8192, 8122, 8706, 8204, 8192] avg=8192.0000 max=8706 max_rank=5 min=7644 min_rank=2 imbalance_ratio=1.062744 max_load_reduction=56830 max_load_reduction_pct=0.867157 moved_to_echo=33224 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[97, 0, 4095, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 24, 0, 0, 0, 0, 3991, 29, 0, 0, 0, 0, 0, 0, 0, 2517, 0, 0, 0, 0, 0, 0, 4094, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2935, 0, 0, 0, 3845, 0, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3093, 4096, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 1, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8139, 8034, 8193, 7707, 8794, 8198, 8195] avg=8192.0000 max=8794 max_rank=5 min=7707 min_rank=4 imbalance_ratio=1.073486 max_load_reduction=56742 max_load_reduction_pct=0.865814 moved_to_echo=32894 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[97, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 4081, 29, 0, 0, 0, 0, 0, 0, 0, 2893, 0, 0, 0, 0, 0, 0, 4096, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3274, 0, 0, 0, 3905, 0, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2237, 4096, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 2, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8219, 8412, 8192, 8106, 7929, 8211, 8192] avg=8192.0000 max=8412 max_rank=2 min=7929 min_rank=5 imbalance_ratio=1.026855 max_load_reduction=57124 max_load_reduction_pct=0.871643 moved_to_echo=32438 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[97, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 4081, 29, 0, 0, 0, 0, 0, 0, 0, 1815, 0, 0, 0, 0, 0, 0, 4095, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3669, 0, 0, 0, 3771, 0, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 3081, 4096, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4057, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8213, 7333, 8192, 8367, 8783, 8177, 8192] avg=8192.0000 max=8783 max_rank=5 min=7333 min_rank=2 imbalance_ratio=1.072144 max_load_reduction=56753 max_load_reduction_pct=0.865982 moved_to_echo=33519 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[97, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 29, 0, 0, 0, 0, 4093, 29, 0, 0, 0, 0, 0, 0, 0, 3004, 0, 0, 0, 0, 0, 0, 4096, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3482, 0, 0, 0, 4025, 0, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1786, 4096, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4053, 0, 4, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8245, 8523, 8193, 8434, 7483, 8177, 8198] avg=8192.0000 max=8523 max_rank=2 min=7483 min_rank=5 imbalance_ratio=1.040405 max_load_reduction=57013 max_load_reduction_pct=0.869949 moved_to_echo=32292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[97, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 46, 0, 0, 0, 0, 4069, 29, 0, 0, 0, 0, 0, 0, 0, 2485, 0, 0, 0, 0, 0, 0, 4096, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3732, 0, 0, 0, 3915, 0, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2147, 4096, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 1, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8239, 8004, 8192, 8574, 7840, 8205, 8194] avg=8192.0000 max=8574 max_rank=4 min=7840 min_rank=5 imbalance_ratio=1.046631 max_load_reduction=56962 max_load_reduction_pct=0.869171 moved_to_echo=32813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[80, 0, 4095, 3373, 1, 1, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 3922, 35, 0, 0, 0, 0, 0, 0, 0, 3213, 1, 1, 1, 1, 0, 1, 4095, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 4094, 4086, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2950, 0, 1, 0, 4044, 0, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 155, 0, 1, 2, 2877, 4095, 1600, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4024, 1, 10, 0, 0, 28, 2, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7558, 8053, 8737, 8190, 7924, 8731, 8162, 8181] avg=8192.0000 max=8737 max_rank=2 min=7558 min_rank=0 imbalance_ratio=1.066528 max_load_reduction=56799 max_load_reduction_pct=0.866684 moved_to_echo=32998 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[97, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 4090, 29, 0, 0, 0, 0, 0, 0, 0, 3330, 0, 0, 0, 0, 0, 0, 4096, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3335, 0, 0, 0, 4040, 0, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1574, 4096, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 1, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8241, 8849, 8192, 8302, 7270, 8202, 8192] avg=8192.0000 max=8849 max_rank=2 min=7270 min_rank=5 imbalance_ratio=1.080200 max_load_reduction=56687 max_load_reduction_pct=0.864975 moved_to_echo=31966 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 0, 32768, 32128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 32640, 128, 128, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 256, 128, 0, 0, 0, 32384, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 256, 0, 128, 128, 19840, 32768, 12800, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 26624, 0, 128, 0, 31616, 0, 7424, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[21504, 128, 128, 128, 128, 0, 128, 32768, 11392, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 2826, 0, 0, 0, 0, 0, 0, 4096, 722, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 620, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1837, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 3090, 0, 0, 0, 0, 0, 0, 4095, 978, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2076, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 198, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1667, 1, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 2751, 0, 0, 0, 0, 0, 0, 4094, 1616, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1958, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2516, 0, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 2413, 0, 0, 0, 0, 0, 0, 4096, 607, 0, 0, 0, 0, 7, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2315, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2237, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 3649, 0, 0, 0, 0, 0, 0, 4096, 852, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1333, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 3118, 0, 0, 0, 0, 0, 0, 4092, 3047, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 4096, 0, 0, 778, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4094, 4091, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2256, 3, 0, 0, 0, 4096, 4, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 2506, 0, 0, 0, 0, 0, 0, 4096, 2290, 0, 0, 0, 0, 13, 4096, 4096, 0, 0, 0, 4096, 0, 0, 760, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4085, 0, 1, 1, 1, 0, 4095, 4095, 600, 1, 1, 0, 0, 0, 1, 1, 0, 0, 3994, 58, 1, 37, 0, 4095, 0, 1, 4093, 4095, 0, 3, 4095, 7, 4095, 4095, 1787, 0, 0, 0, 0, 1, 1, 4093, 1474, 3, 0, 2, 0, 2, 4093, 4095, 0, 8, 1, 4091, 0, 0, 300, 1, 1, 4029, 1, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 4096, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 0, 0, 0, 0, 4096, 1647, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 4096, 2826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 722, 0, 0, 0, 0, 0, 1524, 1120, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 620, 0, 0, 4096, 0, 0, 0, 0, 2570, 267, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8060, 9633, 8191, 8251, 7462, 8192, 7553] avg=8192.0000 max=9633 max_rank=2 min=7462 min_rank=5 imbalance_ratio=1.175903 max_load_reduction=55903 max_load_reduction_pct=0.853012 moved_to_echo=31458 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 1, 0, 0, 4096, 3828, 0, 0, 0, 0, 0, 0, 0, 0, 1667, 1, 0, 0, 0, 4096, 1647, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 4096, 2751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1616, 0, 0, 0, 0, 3, 1524, 1120, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1958, 0, 0, 4096, 0, 0, 0, 0, 2568, 268, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8123, 7412, 8192, 8176, 8357, 8192, 8890] avg=8192.0000 max=8890 max_rank=7 min=7412 min_rank=2 imbalance_ratio=1.085205 max_load_reduction=56646 max_load_reduction_pct=0.864349 moved_to_echo=33615 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 4096, 3828, 0, 0, 0, 0, 0, 0, 0, 0, 2516, 0, 0, 0, 0, 4096, 1647, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 4096, 2413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 607, 0, 0, 0, 0, 7, 1524, 1120, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2315, 0, 0, 4096, 0, 0, 0, 0, 2569, 268, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8258, 8260, 8192, 7838, 7354, 8192, 9248] avg=8192.0000 max=9248 max_rank=7 min=7354 min_rank=5 imbalance_ratio=1.128906 max_load_reduction=56288 max_load_reduction_pct=0.858887 moved_to_echo=32632 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 0, 0, 0, 4096, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 1837, 0, 0, 0, 0, 4096, 1647, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 4096, 3090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 978, 0, 0, 0, 0, 0, 1524, 1120, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2076, 0, 0, 4096, 0, 0, 0, 0, 2569, 267, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8137, 7581, 8191, 8515, 7717, 8192, 9008] avg=8192.0000 max=9008 max_rank=7 min=7581 min_rank=2 imbalance_ratio=1.099609 max_load_reduction=56528 max_load_reduction_pct=0.862549 moved_to_echo=33432 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 4096, 3828, 0, 0, 0, 0, 0, 0, 0, 0, 2237, 0, 0, 0, 0, 4096, 1647, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 4096, 3649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 852, 0, 0, 0, 0, 1, 1524, 1120, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 0, 0, 4096, 0, 0, 0, 0, 2569, 268, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8044, 7981, 8191, 9074, 7593, 8192, 8266] avg=8192.0000 max=9074 max_rank=4 min=7593 min_rank=5 imbalance_ratio=1.107666 max_load_reduction=56462 max_load_reduction_pct=0.861542 moved_to_echo=33125 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 4096, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1, 0, 0, 0, 4096, 1647, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 4096, 3118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3047, 0, 0, 0, 0, 3, 1524, 1120, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 0, 0, 4096, 0, 0, 0, 0, 2569, 267, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8095, 6824, 8192, 8543, 9786, 8192, 7710] avg=8192.0000 max=9786 max_rank=5 min=6824 min_rank=2 imbalance_ratio=1.194580 max_load_reduction=55750 max_load_reduction_pct=0.850677 moved_to_echo=34231 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 0, 4096, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 2256, 3, 0, 0, 0, 4096, 1647, 1, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 4096, 2506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2290, 0, 0, 0, 0, 13, 1525, 1120, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 0, 0, 4096, 0, 0, 0, 0, 2569, 267, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8292, 8003, 8195, 7931, 9044, 8192, 7692] avg=8192.0000 max=9044 max_rank=5 min=7692 min_rank=7 imbalance_ratio=1.104004 max_load_reduction=56492 max_load_reduction_pct=0.862000 moved_to_echo=32859 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 1, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 600, 1, 1, 0, 0, 0, 1, 1, 4095, 3828, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 58, 1, 37, 0, 4095, 1649, 7, 1, 0, 0, 0, 0, 0, 0, 1, 4093, 4088, 0, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 4095, 1787, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 1474, 3, 0, 2, 0, 2, 1527, 1121, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 8, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 1, 1, 4029, 1, 1, 1, 0, 2568, 267, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8527, 9842, 8192, 7208, 8223, 8192, 7169] avg=8192.0000 max=9842 max_rank=2 min=7169 min_rank=7 imbalance_ratio=1.201416 max_load_reduction=55694 max_load_reduction_pct=0.849823 moved_to_echo=30792 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[10752, 32768, 22144, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2176, 128, 128, 0, 0, 128, 128, 128, 32768, 30720, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 11648, 128, 0, 128, 0, 128, 12288, 9088, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[10240, 128, 128, 32768, 128, 128, 128, 0, 20608, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 19584, 128, 128, 128, 0, 32768, 13184, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 20, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 4096, 4096, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 1, 4096, 4096, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4084, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4088, 4096, 4096, 0, 0, 0, 19, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 29, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4092, 4096, 4096, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4092, 0, 54, 1, 3041, 1, 0, 0, 5, 0, 1, 0, 0, 4093, 69, 4095, 4039, 4095, 0, 0, 1, 1042, 0, 1, 4095, 0, 0, 1, 1, 4094, 4095, 0, 1, 0, 1, 1, 0, 0, 0, 0, 4095, 4088, 0, 4095, 4022, 1, 141, 21, 3958, 0, 1, 1, 0, 1, 0, 0, 1, 4095, 1, 4092, 0, 3, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4085, 4096, 4096, 0, 0, 0, 82, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4096, 4076, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4093, 0, 0, 131, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 11, 0, 0, 0, 0, 0, 0, 3954, 4096, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8325, 8184, 8070, 8192, 8192, 8174, 8211, 8188] avg=8192.0000 max=8325 max_rank=0 min=8070 min_rank=2 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32765 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4084, 0, 0, 130, 3, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4076, 11, 0, 0, 0, 0, 0, 0, 3947, 4096, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 20, 1, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8315, 8184, 8062, 8192, 8191, 8189, 8211, 8192] avg=8192.0000 max=8315 max_rank=0 min=8062 min_rank=2 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32783 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4087, 0, 0, 131, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4077, 11, 0, 0, 0, 0, 0, 0, 3952, 4096, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 4, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8319, 8183, 8058, 8192, 8193, 8191, 8206, 8194] avg=8192.0000 max=8319 max_rank=0 min=8058 min_rank=2 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32784 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4081, 0, 0, 131, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 11, 0, 0, 0, 0, 0, 0, 3954, 4096, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 20, 1, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8313, 8183, 8079, 8192, 8191, 8188, 8199, 8191] avg=8192.0000 max=8313 max_rank=0 min=8079 min_rank=2 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32769 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4090, 0, 0, 131, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 11, 0, 0, 0, 0, 0, 0, 3954, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8322, 8184, 8057, 8192, 8192, 8190, 8210, 8189] avg=8192.0000 max=8322 max_rank=0 min=8057 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32781 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 131, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4077, 11, 0, 0, 0, 0, 0, 0, 3950, 4096, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 4, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8328, 8183, 8059, 8192, 8192, 8190, 8198, 8194] avg=8192.0000 max=8328 max_rank=0 min=8059 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32774 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4049, 0, 0, 130, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4077, 11, 0, 0, 0, 0, 0, 0, 3944, 4096, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 7, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8280, 8177, 8122, 8192, 8192, 8171, 8205, 8197] avg=8192.0000 max=8280 max_rank=0 min=8122 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32765 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 54, 1, 3041, 1, 0, 134, 9, 2, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 4093, 69, 4076, 14, 0, 0, 0, 0, 0, 0, 3947, 4039, 0, 0, 0, 1, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4022, 1, 141, 0, 0, 0, 0, 0, 0, 0, 0, 21, 3958, 0, 1, 1, 0, 1, 0, 4095, 19, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 1, 4092, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7334, 8258, 9029, 8192, 8193, 8243, 8096, 8191] avg=8192.0000 max=9029 max_rank=2 min=7334 min_rank=0 imbalance_ratio=1.102173 max_load_reduction=56507 max_load_reduction_pct=0.862228 moved_to_echo=32723 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32640, 0, 128, 128, 0, 128, 0, 32768, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[31616, 32768, 0, 0, 0, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 0, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 32768, 0, 128, 128, 31744, 128, 0, 1152, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 529, 0, 0, 0, 0, 0, 0, 228, 0, 4077, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 4096, 4096, 4096, 0, 81, 4096, 0, 0, 4096, 4096, 0, 0, 3274, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 238, 0, 0, 0, 0, 0, 0, 758, 0, 4045, 0, 0, 4096, 0, 0, 0, 4096, 4066, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 52, 4096, 0, 0, 4096, 4096, 0, 0, 3142, 0, 0, 0, 4096, 4096, 0, 0, 4078, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 673, 0, 0, 0, 1, 0, 0, 794, 0, 4072, 0, 2, 4096, 0, 0, 0, 4095, 4082, 0, 0, 4096, 0, 0, 9, 0, 0, 4096, 4095, 4096, 0, 68, 4096, 0, 1, 4096, 4096, 3, 0, 2588, 0, 0, 0, 4095, 4096, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 365, 0, 4076, 0, 0, 4096, 0, 0, 0, 4096, 4087, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 11, 4096, 0, 0, 4096, 4096, 0, 0, 3697, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 264, 0, 4051, 0, 4, 4096, 0, 0, 0, 4096, 4081, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 23, 4096, 0, 3, 4093, 4096, 11, 0, 3378, 0, 0, 0, 4096, 4096, 0, 0, 4078, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 63, 0, 0, 0, 0, 0, 0, 191, 0, 4078, 0, 0, 4096, 0, 0, 0, 4096, 4091, 0, 0, 4096, 0, 0, 7, 0, 0, 4096, 4096, 4096, 0, 10, 4096, 0, 0, 4096, 4096, 0, 0, 3850, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 212, 0, 4057, 0, 1, 4096, 0, 0, 0, 4096, 4089, 0, 0, 4096, 0, 0, 2, 0, 0, 4096, 4096, 4096, 0, 232, 4096, 0, 1, 4096, 4096, 3, 0, 3612, 0, 0, 0, 4095, 4096, 0, 0, 4086, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 1, 73, 1, 0, 1, 0, 4016, 0, 1, 0, 0, 129, 0, 28, 4, 0, 1, 34, 1, 1, 352, 1, 4070, 1, 14, 4092, 7, 0, 1, 4095, 3868, 0, 0, 4089, 0, 0, 10, 19, 3, 4095, 4082, 4095, 1, 1, 4093, 0, 0, 4082, 4094, 0, 2, 3876, 1, 0, 1, 4087, 4095, 0, 0, 4017, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 238, 3428, 365, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 758, 0, 4078, 3703, 10, 0, 0, 0, 0, 0, 4045, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4096, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 52, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 668, 0, 0, 3142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8156, 8549, 8169, 8195, 8187, 7906, 8192] avg=8192.0000 max=8549 max_rank=2 min=7906 min_rank=6 imbalance_ratio=1.043579 max_load_reduction=56987 max_load_reduction_pct=0.869553 moved_to_echo=32480 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 52, 3428, 365, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 0, 4078, 3703, 9, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 668, 0, 0, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 7965, 8155, 8200, 8216, 8146, 8461, 8210] avg=8192.0000 max=8461 max_rank=6 min=7965 min_rank=1 imbalance_ratio=1.032837 max_load_reduction=57075 max_load_reduction_pct=0.870895 moved_to_echo=33033 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 673, 3428, 365, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 794, 0, 4078, 3702, 10, 0, 0, 0, 0, 0, 4072, 0, 2, 4096, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 4096, 0, 0, 9, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 68, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 668, 3, 0, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 18, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8586, 8585, 8198, 8220, 8202, 7356, 8207] avg=8192.0000 max=8586 max_rank=1 min=7356 min_rank=6 imbalance_ratio=1.048096 max_load_reduction=56950 max_load_reduction_pct=0.868988 moved_to_echo=31985 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 529, 3428, 365, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 4078, 3703, 9, 0, 0, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 81, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 668, 0, 0, 3274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8442, 8018, 8201, 8228, 8216, 8038, 8210] avg=8192.0000 max=8442 max_rank=1 min=8018 min_rank=2 imbalance_ratio=1.030518 max_load_reduction=57094 max_load_reduction_pct=0.871185 moved_to_echo=32692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 63, 3428, 365, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 4078, 3703, 10, 0, 0, 0, 0, 0, 4078, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 7, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 668, 0, 0, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 7978, 7982, 8202, 8227, 8145, 8614, 8206] avg=8192.0000 max=8614 max_rank=6 min=7978 min_rank=1 imbalance_ratio=1.051514 max_load_reduction=56922 max_load_reduction_pct=0.868561 moved_to_echo=33192 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 496, 3428, 365, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 4078, 3703, 9, 0, 0, 0, 0, 0, 4051, 0, 4, 4096, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4096, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 23, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 668, 11, 0, 3378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8409, 8054, 8179, 8210, 8156, 8153, 8192] avg=8192.0000 max=8409 max_rank=1 min=8054 min_rank=2 imbalance_ratio=1.026489 max_load_reduction=57127 max_load_reduction_pct=0.871689 moved_to_echo=32711 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 91, 3428, 365, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 4078, 3703, 9, 0, 0, 0, 0, 0, 4057, 0, 1, 4096, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 2, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 232, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 668, 3, 0, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 18, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8003, 8002, 8182, 8220, 8367, 8380, 8199] avg=8192.0000 max=8380 max_rank=6 min=8002 min_rank=2 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=33166 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 73, 1, 0, 1, 0, 4095, 4086, 0, 0, 0, 0, 0, 0, 4016, 0, 1, 0, 0, 129, 0, 28, 3427, 367, 29, 0, 0, 0, 0, 0, 4, 0, 1, 34, 1, 1, 352, 1, 4080, 3708, 9, 0, 0, 0, 0, 0, 4070, 1, 14, 4092, 7, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 0, 0, 4089, 0, 0, 10, 19, 34, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4082, 0, 1, 1, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 667, 0, 2, 3876, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 15, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 7997, 8191, 8205, 8020, 8117, 8628, 8120] avg=8192.0000 max=8628 max_rank=6 min=7997 min_rank=1 imbalance_ratio=1.053223 max_load_reduction=56908 max_load_reduction_pct=0.868347 moved_to_echo=32885 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 32768, 256, 0, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 32768, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 32768, 0, 128, 512, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 5376, 128, 128, 27520, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 0, 0, 256, 0, 2176, 27520, 2944, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32512, 0, 0, 32768, 0, 0, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 128, 3200, 128, 32640, 29696, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 786, 0, 4093, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1901, 0, 2839, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1084, 0, 4075, 0, 0, 1, 0, 22, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 2774, 0, 3163, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 471, 0, 4085, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2476, 0, 0, 0, 0, 4091, 4096, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 3249, 0, 2009, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1348, 0, 4070, 0, 0, 0, 0, 29, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 962, 0, 0, 0, 0, 4095, 4096, 0, 4096, 4096, 0, 0, 4090, 0, 0, 0, 3064, 0, 2817, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 566, 1, 4083, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1727, 0, 0, 0, 0, 4095, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 2440, 0, 3463, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 446, 0, 4005, 0, 0, 0, 0, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 864, 0, 0, 0, 0, 4059, 4096, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 3395, 0, 3599, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 627, 0, 4086, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1794, 0, 0, 0, 0, 4083, 4096, 0, 4096, 4096, 0, 0, 4088, 0, 0, 0, 2426, 0, 3360, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 126, 4095, 4078, 0, 1, 2, 0, 4095, 2, 0, 4012, 1, 4095, 0, 7, 0, 0, 603, 246, 4095, 0, 1, 0, 12, 0, 1, 4095, 1, 0, 0, 1, 1, 2, 0, 1, 1, 0, 2199, 1, 1, 1, 0, 4090, 4095, 0, 4094, 4094, 0, 1, 4095, 0, 0, 1, 3711, 0, 1423, 4061, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3345, 0, 0, 0, 0, 786, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 1721, 1605, 751, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 1, 0, 0, 0, 0, 0, 0, 2658, 0, 0, 0, 0, 4096, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1901, 0, 2839, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8201, 8224, 8187, 8190, 9128, 8195, 7231] avg=8192.0000 max=9128 max_rank=5 min=7231 min_rank=7 imbalance_ratio=1.114258 max_load_reduction=56408 max_load_reduction_pct=0.860718 moved_to_echo=32744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3345, 0, 0, 0, 0, 1084, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 22, 0, 4095, 0, 1721, 1605, 751, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 1, 0, 0, 0, 0, 0, 0, 1170, 0, 0, 0, 0, 4096, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2774, 0, 3163, 2491, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8201, 8504, 8198, 8190, 7640, 8194, 8429] avg=8192.0000 max=8504 max_rank=2 min=7640 min_rank=5 imbalance_ratio=1.038086 max_load_reduction=57032 max_load_reduction_pct=0.870239 moved_to_echo=32453 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3345, 0, 0, 0, 0, 471, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 1721, 1605, 751, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 1, 0, 0, 0, 0, 0, 0, 2476, 0, 0, 0, 0, 4091, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3249, 0, 2009, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8201, 7901, 8182, 8190, 8941, 8192, 7749] avg=8192.0000 max=8941 max_rank=5 min=7749 min_rank=7 imbalance_ratio=1.091431 max_load_reduction=56595 max_load_reduction_pct=0.863571 moved_to_echo=33072 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3345, 0, 0, 0, 0, 627, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 1721, 1605, 751, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 1, 0, 0, 0, 0, 0, 0, 1794, 0, 0, 0, 0, 4083, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2426, 0, 3360, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8201, 8058, 8192, 8190, 8251, 8187, 8277] avg=8192.0000 max=8277 max_rank=7 min=8058 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32905 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3345, 0, 0, 0, 0, 1348, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4096, 0, 1721, 1605, 751, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4093, 1, 0, 0, 0, 0, 0, 0, 962, 0, 0, 0, 0, 4095, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3064, 0, 2817, 2491, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8201, 8763, 8205, 8193, 7431, 8189, 8373] avg=8192.0000 max=8763 max_rank=2 min=7431 min_rank=5 imbalance_ratio=1.069702 max_load_reduction=56773 max_load_reduction_pct=0.866287 moved_to_echo=32186 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3345, 0, 0, 0, 0, 446, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4096, 0, 1721, 1605, 751, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 1, 0, 0, 0, 0, 0, 0, 864, 0, 0, 0, 0, 4059, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3395, 0, 3599, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8201, 7796, 8193, 8190, 7297, 8194, 9485] avg=8192.0000 max=9485 max_rank=7 min=7297 min_rank=5 imbalance_ratio=1.157837 max_load_reduction=56051 max_load_reduction_pct=0.855270 moved_to_echo=33166 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3345, 0, 0, 0, 0, 566, 1, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 1721, 1605, 751, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 1, 0, 0, 0, 0, 0, 0, 1727, 0, 0, 0, 0, 4095, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2440, 0, 3463, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8201, 7995, 8185, 8190, 8196, 8195, 8394] avg=8192.0000 max=8394 max_rank=7 min=7995 min_rank=2 imbalance_ratio=1.024658 max_load_reduction=57142 max_load_reduction_pct=0.871918 moved_to_echo=32975 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4071, 0, 0, 0, 126, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4095, 2, 0, 4012, 1, 16, 0, 0, 0, 0, 0, 0, 0, 3344, 0, 7, 0, 0, 603, 246, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 1, 4095, 1, 1727, 1599, 751, 7, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 1, 1, 4095, 4094, 8, 0, 0, 0, 0, 0, 0, 2199, 1, 1, 1, 0, 4090, 2360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3711, 0, 1423, 2462, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8129, 8295, 8194, 8203, 8652, 8190, 7598] avg=8192.0000 max=8652 max_rank=5 min=7598 min_rank=7 imbalance_ratio=1.056152 max_load_reduction=56884 max_load_reduction_pct=0.867981 moved_to_echo=32643 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[26880, 0, 128, 0, 0, 6016, 256, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 13952, 128, 128, 128, 0, 32768, 19072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 23040, 0, 22784, 19968, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 128, 32768, 128, 13824, 12928, 6016, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65664 slack_slots=4608 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4074, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2229, 0, 0, 0, 2074, 4095, 34, 566, 0, 1, 4096, 0, 0, 0, 0, 2, 0, 4095, 1, 4096, 93, 0, 0, 4096, 4085, 4057, 0, 1906, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4086, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1517, 0, 0, 0, 1947, 4096, 262, 524, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 28, 0, 0, 4096, 4096, 4072, 0, 1622, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4087, 2080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 314, 0, 0, 0, 1817, 4096, 8, 714, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 4096, 4073, 4038, 0, 3352, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4094, 2006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 396, 0, 0, 0, 2997, 4096, 61, 338, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 11, 0, 0, 4096, 4095, 4064, 0, 2418, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4087, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 572, 0, 0, 0, 2852, 4096, 9, 345, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 7, 0, 0, 4096, 4096, 4080, 0, 2095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4036, 2234, 36, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4091, 5, 199, 0, 1, 0, 2442, 4095, 514, 85, 0, 0, 4095, 1, 2, 0, 0, 0, 62, 4093, 0, 4095, 375, 1, 5, 4094, 4084, 4079, 0, 2533, 0, 1, 0, 4091, 1, 4095, 0, 1, 1, 0, 4094, 0, 1, 1, 1, 2, 1, 3888, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4070, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1278, 0, 0, 0, 1906, 4096, 42, 760, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4095, 0, 4096, 28, 0, 0, 4096, 4075, 4066, 0, 2146, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4075, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4080, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 332, 0, 0, 0, 2623, 4093, 15, 1130, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 7, 0, 0, 4096, 4094, 4021, 0, 2765, 0, 0, 9, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4071, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4074, 1367, 0, 0, 0, 0, 0, 0, 2119, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2229, 0, 0, 0, 2074, 4095, 34, 566, 190, 43, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 2, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 93, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 1776, 4057, 0, 1906, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7561, 8191, 9231, 8195, 8209, 7741, 8192, 8216] avg=8192.0000 max=9231 max_rank=2 min=7561 min_rank=0 imbalance_ratio=1.126831 max_load_reduction=56305 max_load_reduction_pct=0.859146 moved_to_echo=32358 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4094, 2006, 0, 0, 0, 0, 0, 0, 2125, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 2997, 4096, 61, 338, 190, 43, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 1780, 4064, 0, 2418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8192, 8121, 8191, 8127, 8262, 8192, 8225] avg=8192.0000 max=8262 max_rank=5 min=8121 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32806 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4087, 2080, 0, 0, 0, 0, 0, 0, 2113, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 1817, 4096, 8, 714, 189, 43, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 1771, 4038, 0, 3352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8191, 7181, 8192, 8119, 9161, 8192, 8219] avg=8192.0000 max=9161 max_rank=5 min=7181 min_rank=2 imbalance_ratio=1.118286 max_load_reduction=56375 max_load_reduction_pct=0.860214 moved_to_echo=33691 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4087, 2338, 0, 0, 0, 0, 0, 0, 2125, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 0, 2852, 4096, 9, 345, 190, 43, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 1781, 4080, 0, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8551, 8191, 8107, 8192, 8123, 7956, 8192, 8224] avg=8192.0000 max=8551 max_rank=0 min=7956 min_rank=5 imbalance_ratio=1.043823 max_load_reduction=56985 max_load_reduction_pct=0.869522 moved_to_echo=32495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4086, 2328, 0, 0, 0, 0, 0, 0, 2125, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1517, 0, 0, 0, 1947, 4096, 262, 524, 190, 43, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 28, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 1781, 4072, 0, 1622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8540, 8191, 8579, 8192, 8144, 7475, 8192, 8223] avg=8192.0000 max=8579 max_rank=2 min=7475 min_rank=5 imbalance_ratio=1.047241 max_load_reduction=56957 max_load_reduction_pct=0.869095 moved_to_echo=32034 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4080, 1427, 0, 0, 0, 0, 0, 0, 2124, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 332, 0, 0, 0, 2623, 4093, 15, 1130, 190, 43, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 1780, 4021, 0, 2765, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7633, 8192, 8426, 8195, 8123, 8575, 8192, 8200] avg=8192.0000 max=8575 max_rank=5 min=7633 min_rank=0 imbalance_ratio=1.046753 max_load_reduction=56961 max_load_reduction_pct=0.869156 moved_to_echo=33090 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4070, 2128, 0, 0, 0, 0, 0, 0, 2114, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 0, 1906, 4096, 42, 760, 189, 43, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 28, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 1772, 4066, 0, 2146, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8313, 8192, 8314, 8192, 8143, 7986, 8192, 8204] avg=8192.0000 max=8314 max_rank=2 min=7986 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32525 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4036, 2234, 36, 0, 0, 0, 0, 0, 2123, 2, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4091, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 199, 0, 1, 0, 2442, 4095, 514, 85, 194, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 2, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 62, 4093, 0, 0, 375, 1, 5, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 1767, 4079, 0, 2533, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 1, 3888, 4095, 1, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8431, 8196, 7577, 8187, 8548, 8380, 8192, 8025] avg=8192.0000 max=8548 max_rank=4 min=7577 min_rank=2 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=33145 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 640, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32640, 16000, 128, 0, 0, 0, 0, 0, 17024, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[14208, 32512, 0, 18944, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 32512, 32768, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[6912, 0, 128, 0, 18688, 32768, 1024, 4480, 1536, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4077, 21, 0, 3945, 0, 0, 4096, 0, 4096, 0, 147, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3874, 4, 0, 3641, 0, 0, 4096, 0, 4095, 0, 697, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4064, 0, 4096, 0, 4095, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[3990, 5, 0, 3820, 0, 0, 4096, 0, 4096, 0, 437, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4033, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4057, 15, 0, 3908, 0, 0, 4096, 0, 4095, 0, 242, 4096, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 4064, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4078, 3, 0, 3993, 0, 0, 4096, 0, 4096, 0, 140, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4066, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[3955, 15, 0, 3741, 0, 0, 4096, 0, 4095, 0, 621, 4096, 0, 0, 1, 0, 4095, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 3954, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4009, 50, 0, 3695, 0, 0, 4096, 0, 4092, 0, 583, 4096, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 3940, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4092, 116, 1, 2590, 1, 9, 3989, 0, 4095, 1, 1551, 3730, 2, 0, 1, 0, 4095, 1, 1, 1, 1, 4094, 0, 2, 0, 0, 3, 4051, 0, 4092, 0, 4086, 0, 1, 1, 4095, 0, 0, 0, 0, 1, 10, 1, 1, 0, 353, 1, 1, 2, 1, 4085, 0, 0, 2, 4095, 0, 0, 4089, 0, 4095, 4094, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4077, 21, 0, 3945, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 0, 147, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 64, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3599, 506, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 4032, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8525, 7833, 8191, 8254, 8202, 8145, 8193, 8193] avg=8192.0000 max=8525 max_rank=0 min=7833 min_rank=1 imbalance_ratio=1.040649 max_load_reduction=57011 max_load_reduction_pct=0.869919 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3874, 4, 0, 3641, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 0, 697, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 64, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3599, 506, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4095, 4032, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8001, 8382, 8192, 8223, 8203, 8150, 8192, 8193] avg=8192.0000 max=8382 max_rank=1 min=8001 min_rank=0 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4078, 3, 0, 3993, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 0, 140, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 64, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3599, 506, 1, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4095, 4032, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8556, 7826, 8191, 8225, 8202, 8151, 8192, 8193] avg=8192.0000 max=8556 max_rank=0 min=7826 min_rank=1 imbalance_ratio=1.044434 max_load_reduction=56980 max_load_reduction_pct=0.869446 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4057, 15, 0, 3908, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 0, 242, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 64, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3599, 506, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4032, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8462, 7927, 8194, 8224, 8202, 8142, 8192, 8193] avg=8192.0000 max=8462 max_rank=0 min=7927 min_rank=1 imbalance_ratio=1.032959 max_load_reduction=57074 max_load_reduction_pct=0.870880 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3990, 5, 0, 3820, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 0, 437, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 64, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3599, 506, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 4032, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8123, 8191, 8193, 8202, 8145, 8192, 8193] avg=8192.0000 max=8297 max_rank=0 min=8123 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4009, 50, 0, 3695, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3587, 0, 583, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 64, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3599, 505, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4032, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8266, 8202, 8100, 8201, 8147, 8192, 8192] avg=8192.0000 max=8266 max_rank=1 min=8100 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3955, 15, 0, 3741, 0, 0, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 0, 621, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 64, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3599, 506, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 4032, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8307, 8191, 8114, 8202, 8143, 8193, 8193] avg=8192.0000 max=8307 max_rank=1 min=8114 min_rank=3 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4092, 116, 1, 2590, 1, 9, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3587, 1, 1551, 3730, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 1, 4083, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4051, 0, 63, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 3511, 508, 6, 0, 0, 0, 0, 0, 1, 10, 1, 1, 0, 353, 1, 1, 4095, 4029, 21, 0, 0, 0, 0, 0, 2, 1, 4085, 0, 0, 2, 4095, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4094, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7266, 8872, 8184, 8203, 8122, 8513, 8190, 8186] avg=8192.0000 max=8872 max_rank=1 min=7266 min_rank=0 imbalance_ratio=1.083008 max_load_reduction=56664 max_load_reduction_pct=0.864624 moved_to_echo=33011 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32256, 256, 128, 29440, 128, 128, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32384, 0, 512, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 384, 128, 128, 32768, 32256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[28800, 128, 4480, 32512, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 0, 28800, 4096, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True + [2026-08-30 05:00:55] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 670.2 | throughput per GPU (TFLOP/s/GPU): 497.5 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.980397E+00 | loss scale: 1.0 | grad norm: 5.201 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2334, 307, 982, 550, 2108, 0, 1066, 390, 772, 2842, 1310, 1698, 1846, 1843, 5, 319, 268, 1409, 1031, 625, 0, 985, 1957, 1102, 252, 73, 2287, 863, 1419, 1334, 1602, 1311, 14, 2098, 1909, 505, 461, 1162, 1526, 1745, 424, 0, 1117, 988, 2, 1470, 894, 635, 2103, 52, 1327, 16, 295, 1488, 7, 1145, 3, 1119, 1340, 902, 1584, 992, 1185, 2138] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1938, 149, 1391, 594, 1842, 4, 1211, 837, 825, 2064, 1135, 1519, 1774, 863, 0, 527, 558, 1403, 1507, 1275, 2, 875, 2005, 1213, 142, 36, 2301, 935, 1730, 733, 1519, 1125, 8, 2393, 1959, 201, 950, 1228, 1559, 913, 501, 5, 1056, 996, 3, 1769, 881, 587, 2350, 89, 789, 91, 427, 1351, 9, 1211, 0, 731, 1956, 821, 1871, 1469, 999, 2331] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2628, 138, 1153, 801, 1083, 0, 1095, 407, 1014, 2939, 1538, 719, 1096, 2107, 0, 315, 848, 1348, 1169, 620, 0, 971, 2006, 943, 108, 282, 2567, 989, 1495, 1523, 1543, 851, 1, 1930, 1554, 792, 755, 1063, 1530, 1013, 319, 2, 1232, 1415, 0, 1512, 1189, 1158, 2499, 72, 1627, 1, 277, 1563, 0, 633, 0, 869, 1029, 1223, 1633, 1341, 970, 2038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2097, 139, 1134, 644, 1526, 0, 1751, 519, 1292, 2380, 1670, 976, 2043, 1054, 0, 517, 354, 1247, 1363, 811, 0, 1029, 1764, 1095, 179, 65, 2257, 1083, 1758, 754, 2321, 1333, 3, 1951, 1993, 448, 483, 608, 1489, 903, 284, 0, 1100, 1368, 0, 1260, 826, 717, 1797, 26, 2365, 3, 407, 1852, 5, 1216, 1, 839, 1569, 1003, 1711, 962, 857, 2335] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2223, 153, 1336, 669, 2172, 1, 1198, 392, 917, 2522, 1214, 803, 1986, 744, 1, 388, 371, 1208, 1652, 928, 0, 1042, 2011, 1321, 116, 62, 2553, 954, 1725, 1050, 1967, 1087, 12, 2640, 1677, 330, 782, 769, 2000, 522, 630, 2, 897, 1104, 0, 1656, 681, 824, 2319, 55, 1176, 5, 176, 1368, 1, 1554, 0, 1031, 1766, 678, 2041, 906, 885, 2283] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2680, 113, 1142, 771, 1673, 0, 1430, 648, 1076, 2983, 1415, 1222, 1195, 1588, 1, 199, 711, 1427, 1385, 611, 2, 1060, 1309, 917, 109, 133, 2297, 908, 1769, 746, 1510, 838, 1, 2092, 2030, 232, 814, 1861, 1902, 1109, 251, 0, 1062, 1521, 7, 1796, 1291, 925, 2022, 28, 1531, 6, 211, 1505, 1, 962, 1, 855, 1107, 1109, 1272, 934, 946, 2284] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2058, 106, 1045, 556, 2277, 0, 1432, 486, 954, 1843, 1137, 1121, 1595, 834, 1, 518, 518, 2050, 1250, 1000, 0, 971, 2207, 1182, 80, 56, 2571, 1175, 1866, 684, 1620, 901, 4, 2905, 1766, 124, 514, 1588, 1807, 706, 427, 0, 1053, 1776, 6, 1290, 1217, 988, 2671, 69, 912, 6, 220, 1028, 3, 998, 1, 736, 1396, 1002, 1742, 887, 806, 2794] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2284, 124, 1118, 670, 1543, 2, 1427, 521, 1278, 3206, 1020, 1357, 1629, 1580, 0, 245, 480, 962, 1159, 606, 0, 984, 1723, 1354, 113, 236, 2332, 1405, 2250, 1198, 1930, 1007, 3, 2376, 1762, 756, 716, 1213, 1393, 878, 395, 0, 1159, 1785, 0, 1413, 913, 639, 2159, 61, 1414, 7, 244, 1022, 2, 994, 0, 640, 1031, 651, 1583, 1182, 1126, 2276] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2334, 307, 982, 550, 2108, 0, 1066, 390, 281, 0, 0, 0, 0, 0, 0, 0, 772, 1470, 1310, 1698, 1846, 1843, 5, 319, 0, 0, 0, 0, 0, 0, 0, 0, 268, 1409, 1031, 625, 0, 985, 1957, 1102, 154, 0, 0, 0, 0, 0, 0, 0, 252, 73, 1276, 863, 1419, 1334, 1602, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1313, 1909, 505, 461, 1162, 1526, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 1117, 988, 2, 1470, 894, 635, 1372, 725, 0, 0, 0, 0, 0, 0, 2103, 52, 1327, 16, 295, 1488, 7, 1145, 952, 350, 286, 0, 0, 0, 0, 0, 3, 1119, 1340, 902, 1584, 992, 1185, 1186, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8018, 9263, 7531, 8130, 8635, 7627, 8021, 8311] avg=8192.0000 max=9263 max_rank=1 min=7531 min_rank=2 imbalance_ratio=1.130737 max_load_reduction=56273 max_load_reduction_pct=0.858658 moved_to_echo=32594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1938, 149, 1391, 594, 1842, 4, 1211, 837, 321, 0, 0, 0, 0, 0, 0, 0, 825, 1067, 1135, 1519, 1774, 863, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 558, 1403, 1507, 1275, 2, 875, 2005, 1213, 175, 0, 0, 0, 0, 0, 0, 0, 142, 36, 1284, 935, 1730, 733, 1519, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1497, 1959, 201, 950, 1228, 1559, 913, 0, 0, 0, 0, 0, 0, 0, 0, 501, 5, 1056, 996, 3, 1769, 881, 587, 997, 729, 0, 0, 0, 0, 0, 0, 2350, 89, 789, 91, 427, 1351, 9, 1211, 1038, 400, 288, 0, 0, 0, 0, 0, 0, 731, 1956, 821, 1871, 1469, 999, 1293, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 7710, 9013, 7504, 8315, 7524, 8043, 9140] avg=8192.0000 max=9140 max_rank=7 min=7504 min_rank=3 imbalance_ratio=1.115723 max_load_reduction=56396 max_load_reduction_pct=0.860535 moved_to_echo=33022 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2223, 153, 1336, 669, 2172, 1, 1198, 392, 354, 0, 0, 0, 0, 0, 0, 0, 917, 1304, 1214, 803, 1986, 744, 1, 388, 0, 0, 0, 0, 0, 0, 0, 0, 371, 1208, 1652, 928, 0, 1042, 2011, 1321, 194, 0, 0, 0, 0, 0, 0, 0, 116, 62, 1425, 954, 1725, 1050, 1967, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1651, 1677, 330, 782, 769, 2000, 522, 0, 0, 0, 0, 0, 0, 0, 0, 630, 2, 897, 1104, 0, 1656, 681, 824, 1218, 809, 0, 0, 0, 0, 0, 0, 2319, 55, 1176, 5, 176, 1368, 1, 1554, 1017, 441, 319, 0, 0, 0, 0, 0, 0, 1031, 1766, 678, 2041, 906, 885, 1266, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8498, 7357, 8727, 8386, 7743, 7821, 8431, 8573] avg=8192.0000 max=8727 max_rank=2 min=7357 min_rank=1 imbalance_ratio=1.065308 max_load_reduction=56809 max_load_reduction_pct=0.866837 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2097, 139, 1134, 644, 1526, 0, 1751, 519, 261, 0, 0, 0, 0, 0, 0, 0, 1292, 1231, 1670, 976, 2043, 1054, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 354, 1247, 1363, 811, 0, 1029, 1764, 1095, 143, 0, 0, 0, 0, 0, 0, 0, 179, 65, 1259, 1083, 1758, 754, 2321, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1221, 1993, 448, 483, 608, 1489, 903, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 1100, 1368, 0, 1260, 826, 717, 1149, 716, 0, 0, 0, 0, 0, 0, 1797, 26, 2365, 3, 407, 1852, 5, 1216, 1040, 326, 282, 0, 0, 0, 0, 0, 1, 839, 1569, 1003, 1711, 962, 857, 1295, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8783, 7806, 8752, 7148, 7420, 9319, 8237] avg=8192.0000 max=9319 max_rank=6 min=7148 min_rank=4 imbalance_ratio=1.137573 max_load_reduction=56217 max_load_reduction_pct=0.857803 moved_to_echo=32124 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2680, 113, 1142, 771, 1673, 0, 1430, 648, 280, 0, 0, 0, 0, 0, 0, 0, 1076, 1542, 1415, 1222, 1195, 1588, 1, 199, 0, 0, 0, 0, 0, 0, 0, 0, 711, 1427, 1385, 611, 2, 1060, 1309, 917, 153, 0, 0, 0, 0, 0, 0, 0, 109, 133, 1282, 908, 1769, 746, 1510, 838, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1310, 2030, 232, 814, 1861, 1902, 1109, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 1062, 1521, 7, 1796, 1291, 925, 1441, 728, 0, 0, 0, 0, 0, 0, 2022, 28, 1531, 6, 211, 1505, 1, 962, 1017, 349, 287, 0, 0, 0, 0, 0, 1, 855, 1107, 1109, 1272, 934, 946, 1267, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8737, 8238, 7575, 7295, 9259, 9022, 7919, 7491] avg=8192.0000 max=9259 max_rank=4 min=7295 min_rank=3 imbalance_ratio=1.130249 max_load_reduction=56277 max_load_reduction_pct=0.858719 moved_to_echo=33691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2628, 138, 1153, 801, 1083, 0, 1095, 407, 264, 0, 0, 0, 0, 0, 0, 0, 1014, 1516, 1538, 719, 1096, 2107, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 848, 1348, 1169, 620, 0, 971, 2006, 943, 146, 0, 0, 0, 0, 0, 0, 0, 108, 282, 1422, 989, 1495, 1523, 1543, 851, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1193, 1554, 792, 755, 1063, 1530, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 319, 2, 1232, 1415, 0, 1512, 1189, 1158, 1423, 819, 0, 0, 0, 0, 0, 0, 2499, 72, 1627, 1, 277, 1563, 0, 633, 913, 327, 326, 0, 0, 0, 0, 0, 0, 869, 1029, 1223, 1633, 1341, 970, 1125, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7569, 8305, 8051, 8213, 7901, 9069, 8238, 8190] avg=8192.0000 max=9069 max_rank=5 min=7569 min_rank=0 imbalance_ratio=1.107056 max_load_reduction=56467 max_load_reduction_pct=0.861618 moved_to_echo=33398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2058, 106, 1045, 556, 2277, 0, 1432, 486, 389, 0, 0, 0, 0, 0, 0, 0, 954, 953, 1137, 1121, 1595, 834, 1, 518, 0, 0, 0, 0, 0, 0, 0, 0, 518, 2050, 1250, 1000, 0, 971, 2207, 1182, 213, 0, 0, 0, 0, 0, 0, 0, 80, 56, 1434, 1175, 1866, 684, 1620, 901, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1818, 1766, 124, 514, 1588, 1807, 706, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 1053, 1776, 6, 1290, 1217, 988, 890, 815, 0, 0, 0, 0, 0, 0, 2671, 69, 912, 6, 220, 1028, 3, 998, 1245, 485, 322, 0, 0, 0, 0, 0, 1, 736, 1396, 1002, 1742, 887, 806, 1549, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 7113, 9391, 7816, 8327, 8462, 7959, 8119] avg=8192.0000 max=9391 max_rank=2 min=7113 min_rank=1 imbalance_ratio=1.146362 max_load_reduction=56145 max_load_reduction_pct=0.856705 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2284, 124, 1118, 670, 1543, 2, 1427, 521, 318, 0, 0, 0, 0, 0, 0, 0, 1278, 1658, 1020, 1357, 1629, 1580, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 480, 962, 1159, 606, 0, 984, 1723, 1354, 174, 0, 0, 0, 0, 0, 0, 0, 113, 236, 1301, 1405, 2250, 1198, 1930, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1487, 1762, 756, 716, 1213, 1393, 878, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 1159, 1785, 0, 1413, 913, 639, 1548, 739, 0, 0, 0, 0, 0, 0, 2159, 61, 1414, 7, 244, 1022, 2, 994, 1014, 397, 292, 0, 0, 0, 0, 0, 0, 640, 1031, 651, 1583, 1182, 1126, 1262, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8007, 8767, 7442, 9440, 8208, 8591, 7606, 7475] avg=8192.0000 max=9440 max_rank=3 min=7442 min_rank=2 imbalance_ratio=1.152344 max_load_reduction=56096 max_load_reduction_pct=0.855957 moved_to_echo=31880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[17920, 512, 11264, 256, 2304, 11264, 128, 8832, 8320, 3200, 2432, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3328, 128, 8704, 11008, 128, 12288, 7936, 6528, 10112, 6144, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 11520, 14720, 3456, 5504, 9600, 13312, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1152, 1024, 10752, 8320, 14080, 8064, 14080, 8576, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[18304, 1280, 9344, 5376, 14336, 128, 10624, 4224, 2560, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 11136, 10624, 6528, 128, 7936, 15104, 9216, 1408, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 6912, 11264, 7424, 13440, 8704, 7808, 10368, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 10752, 10496, 9472, 13184, 10624, 128, 3072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 4083, 0, 354, 1, 2376, 451, 0, 3582, 1129, 0, 14, 395, 1750, 5, 0, 21, 7, 0, 4080, 4061, 0, 50, 12, 262, 0, 3927, 4038, 4, 4086, 15, 1, 2, 3728, 6, 8, 3723, 20, 4090, 0, 9, 0, 0, 1, 4086, 1605, 0, 1, 4089, 4085, 6, 0, 0, 1, 4087, 1142, 0, 6, 0, 2, 129] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 4095, 0, 833, 0, 2986, 247, 0, 3902, 117, 0, 6, 859, 2510, 0, 6, 3, 1, 0, 4086, 4066, 3, 7, 3, 0, 0, 4068, 4054, 0, 4096, 0, 1, 0, 3177, 0, 3, 2397, 78, 4091, 3, 4, 4, 0, 0, 4092, 1915, 1, 1, 4068, 4092, 3, 12, 3, 0, 4092, 1529, 0, 3, 0, 1, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 5, 1, 0, 4096, 2, 736, 0, 2450, 560, 1, 3132, 825, 0, 0, 592, 2689, 0, 1, 0, 0, 0, 4094, 4096, 30, 0, 0, 2, 0, 4043, 4087, 0, 4096, 1, 0, 0, 3847, 2, 0, 1660, 45, 4095, 0, 0, 0, 1, 0, 4096, 3392, 3, 0, 4079, 4096, 0, 0, 0, 0, 4095, 515, 0, 0, 0, 0, 71] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 1034, 0, 3342, 330, 0, 3935, 376, 0, 1, 1233, 2096, 0, 2, 0, 0, 0, 4096, 3958, 44, 1, 0, 2, 0, 4028, 3686, 0, 4096, 0, 0, 0, 2246, 15, 0, 2095, 688, 4091, 0, 0, 0, 0, 0, 4096, 1443, 6, 0, 4096, 4094, 0, 19, 0, 0, 4074, 2134, 1, 0, 0, 0, 83] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 494, 0, 3116, 643, 0, 3793, 1481, 0, 0, 1249, 1998, 0, 0, 0, 0, 0, 4096, 4093, 0, 1, 0, 6, 0, 4095, 3892, 0, 4095, 0, 1, 0, 2791, 0, 0, 2028, 148, 4069, 0, 0, 0, 0, 0, 4096, 1784, 0, 0, 4092, 4095, 0, 0, 0, 0, 4096, 1070, 0, 0, 0, 0, 117] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 823, 0, 1710, 832, 0, 3431, 847, 0, 0, 85, 2687, 0, 8, 2, 0, 0, 4057, 4084, 39, 54, 0, 336, 0, 4066, 4059, 5, 4096, 3, 4, 0, 3735, 0, 0, 3190, 26, 4092, 0, 0, 0, 0, 0, 4096, 2320, 0, 0, 4096, 4096, 0, 19, 0, 0, 4095, 402, 0, 0, 0, 0, 45] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 10, 438, 1, 2191, 108, 0, 3488, 606, 0, 0, 403, 3055, 2, 0, 5, 2, 3, 4094, 4094, 28, 2, 0, 233, 0, 4057, 3985, 4, 4094, 0, 2, 0, 3717, 18, 4, 2759, 28, 4088, 2, 0, 3, 0, 1, 4093, 2562, 2, 0, 4091, 4091, 2, 6, 1, 3, 4083, 954, 1, 2, 2, 2, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 2060, 0, 2414, 941, 2, 3806, 415, 3, 0, 1018, 2391, 0, 1, 0, 0, 0, 4091, 4093, 7, 1, 1, 4, 0, 4091, 3869, 1, 4096, 2, 3, 1, 1603, 0, 0, 2130, 286, 4089, 0, 1, 0, 0, 0, 4095, 1601, 1, 0, 4094, 4091, 0, 1, 0, 6, 4094, 1950, 2, 0, 0, 0, 85] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 1, 0, 4096, 2, 736, 0, 2372, 880, 0, 0, 0, 0, 0, 0, 2450, 560, 1, 3132, 825, 0, 0, 592, 14, 0, 0, 0, 0, 0, 0, 0, 2689, 0, 1, 0, 0, 0, 1722, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 2, 0, 4043, 4087, 0, 43, 0, 0, 0, 0, 0, 0, 0, 2579, 1, 0, 0, 3847, 2, 0, 1660, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4095, 0, 0, 0, 1, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 3392, 3, 0, 4079, 2018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 515, 0, 0, 0, 0, 71, 1517, 1198, 100, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8092, 7574, 8508, 8205, 8089, 8080, 9492, 7496] avg=8192.0000 max=9492 max_rank=6 min=7496 min_rank=7 imbalance_ratio=1.158691 max_load_reduction=56044 max_load_reduction_pct=0.855164 moved_to_echo=33157 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 4083, 0, 354, 1, 2364, 877, 0, 0, 0, 0, 0, 0, 2376, 451, 0, 3582, 1129, 0, 14, 395, 14, 0, 0, 0, 0, 0, 0, 0, 1750, 5, 0, 21, 7, 0, 1716, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 12, 262, 0, 3927, 4038, 4, 43, 0, 0, 0, 0, 0, 0, 0, 2572, 15, 1, 2, 3728, 6, 8, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4090, 0, 9, 0, 0, 1, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 1605, 0, 1, 4089, 2013, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 1142, 0, 6, 0, 2, 129, 1514, 1195, 100, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7685, 7961, 7560, 8336, 10055, 8049, 7714, 8176] avg=8192.0000 max=10055 max_rank=4 min=7560 min_rank=2 imbalance_ratio=1.227417 max_load_reduction=55481 max_load_reduction_pct=0.846573 moved_to_echo=33994 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 494, 0, 2373, 879, 0, 0, 0, 0, 0, 0, 3116, 643, 0, 3793, 1481, 0, 0, 1249, 14, 0, 0, 0, 0, 0, 0, 0, 1998, 0, 0, 0, 0, 0, 1723, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 4095, 3892, 0, 43, 0, 0, 0, 0, 0, 0, 0, 2578, 0, 1, 0, 2791, 0, 0, 2028, 0, 0, 0, 0, 0, 0, 0, 0, 148, 4069, 0, 0, 0, 0, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 1784, 0, 0, 4092, 2018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1070, 0, 0, 0, 0, 117, 1517, 1198, 100, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7843, 10296, 7814, 8037, 7398, 8156, 7894, 8098] avg=8192.0000 max=10296 max_rank=1 min=7398 min_rank=4 imbalance_ratio=1.256836 max_load_reduction=55240 max_load_reduction_pct=0.842896 moved_to_echo=31546 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 1034, 0, 2373, 879, 0, 0, 0, 0, 0, 0, 3342, 330, 0, 3935, 376, 0, 1, 1233, 14, 0, 0, 0, 0, 0, 0, 0, 2096, 0, 2, 0, 0, 0, 1723, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1, 0, 2, 0, 4028, 3686, 0, 43, 0, 0, 0, 0, 0, 0, 0, 2579, 0, 0, 0, 2246, 15, 0, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 688, 4091, 0, 0, 0, 0, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 1443, 6, 0, 4096, 2017, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 2134, 1, 0, 0, 0, 83, 1517, 1198, 100, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8381, 9231, 7779, 7804, 6935, 8718, 7581, 9107] avg=8192.0000 max=9231 max_rank=1 min=6935 min_rank=4 imbalance_ratio=1.126831 max_load_reduction=56305 max_load_reduction_pct=0.859146 moved_to_echo=32341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 823, 0, 2351, 880, 0, 0, 0, 0, 0, 0, 1710, 832, 0, 3431, 847, 0, 0, 85, 14, 0, 0, 0, 0, 0, 0, 0, 2687, 0, 8, 2, 0, 0, 1706, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 39, 54, 0, 336, 0, 4066, 4059, 5, 43, 0, 0, 0, 0, 0, 0, 0, 2579, 3, 4, 0, 3735, 0, 0, 3190, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4092, 0, 0, 0, 0, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 2320, 0, 0, 4096, 2018, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 402, 0, 0, 0, 0, 45, 1517, 1198, 100, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 6919, 8487, 8602, 9511, 8057, 8453, 7357] avg=8192.0000 max=9511 max_rank=4 min=6919 min_rank=1 imbalance_ratio=1.161011 max_load_reduction=56025 max_load_reduction_pct=0.854874 moved_to_echo=33378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 4095, 0, 833, 0, 2368, 879, 0, 0, 0, 0, 0, 0, 2986, 247, 0, 3902, 117, 0, 6, 859, 14, 0, 0, 0, 0, 0, 0, 0, 2510, 0, 6, 3, 1, 0, 1718, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 3, 0, 0, 4068, 4054, 0, 43, 0, 0, 0, 0, 0, 0, 0, 2579, 0, 1, 0, 3177, 0, 3, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 78, 4091, 3, 4, 4, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 1915, 1, 1, 4068, 2016, 3, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1529, 0, 3, 0, 1, 15, 1517, 1197, 100, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8131, 8304, 8178, 8157, 8115, 8019, 8454] avg=8192.0000 max=8454 max_rank=7 min=8019 min_rank=6 imbalance_ratio=1.031982 max_load_reduction=57082 max_load_reduction_pct=0.871002 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 10, 438, 1, 2372, 879, 0, 0, 0, 0, 0, 0, 2191, 108, 0, 3488, 606, 0, 0, 403, 14, 0, 0, 0, 0, 0, 0, 0, 3055, 2, 0, 5, 2, 3, 1722, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 28, 2, 0, 233, 0, 4057, 3985, 4, 43, 0, 0, 0, 0, 0, 0, 0, 2578, 0, 2, 0, 3717, 18, 4, 2759, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4088, 2, 0, 3, 0, 1, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 2562, 2, 0, 4091, 2015, 2, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4083, 954, 1, 2, 2, 2, 22, 1516, 1197, 100, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7794, 6810, 8883, 8352, 9078, 8058, 8679, 7882] avg=8192.0000 max=9078 max_rank=4 min=6810 min_rank=1 imbalance_ratio=1.108154 max_load_reduction=56458 max_load_reduction_pct=0.861481 moved_to_echo=33697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 2060, 0, 2375, 882, 0, 0, 0, 0, 0, 0, 2414, 941, 2, 3806, 415, 3, 0, 1018, 15, 0, 0, 0, 0, 0, 0, 0, 2391, 0, 1, 0, 0, 0, 1716, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 1, 4, 0, 4091, 3869, 1, 48, 0, 0, 0, 0, 0, 0, 0, 2574, 2, 3, 1, 1603, 0, 0, 2130, 0, 0, 0, 0, 0, 0, 0, 0, 286, 4089, 0, 1, 0, 0, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 1601, 1, 0, 4094, 2007, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4094, 1950, 2, 0, 0, 0, 85, 1522, 1202, 105, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9413, 8614, 8201, 8022, 6313, 8303, 7704, 8966] avg=8192.0000 max=9413 max_rank=0 min=6313 min_rank=4 imbalance_ratio=1.149048 max_load_reduction=56123 max_load_reduction_pct=0.856369 moved_to_echo=31286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[20736, 128, 128, 128, 24960, 128, 128, 20096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[19200, 128, 128, 128, 128, 128, 13824, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1408, 32768, 128, 128, 128, 128, 128, 31488, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 128, 128, 896, 0, 32384, 31744, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[16640, 128, 128, 32768, 16128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 9728, 128, 128, 128, 128, 640, 12160, 9600, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 6784, 128, 19072, 7040, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[20608, 4224, 128, 29184, 5888, 128, 128, 5888, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 76, 0, 4088, 0, 6, 1, 0, 0, 0, 92, 4081, 4089, 0, 0, 4096, 0, 0, 4096, 0, 0, 376, 1, 4092, 1502, 4094, 392, 0, 4090, 0, 0, 0, 1144, 422, 964, 1, 6, 2983, 1949, 0, 0, 4086, 0, 0, 0, 3400, 0, 28, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 3134, 0, 4057, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[6, 3, 6, 4087, 0, 1, 0, 0, 1, 7, 185, 4083, 4018, 0, 6, 4084, 2, 8, 4084, 1, 6, 739, 0, 4096, 2304, 3756, 223, 3, 4093, 0, 0, 0, 2057, 177, 2165, 0, 0, 2377, 732, 6, 2, 4087, 0, 0, 4, 3177, 0, 0, 3, 4090, 12, 6, 10, 4, 3, 4063, 6, 8, 2726, 38, 3981, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 4093, 0, 0, 4, 0, 0, 0, 11, 4096, 3811, 0, 0, 4096, 0, 0, 4096, 0, 0, 450, 0, 4096, 2279, 3965, 449, 0, 4096, 0, 0, 0, 1147, 491, 2137, 0, 46, 2520, 1222, 0, 0, 4085, 0, 0, 0, 3504, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2563, 0, 4081, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 4093, 0, 0, 6, 0, 0, 0, 24, 4095, 4021, 0, 0, 4093, 0, 0, 4096, 0, 0, 470, 1, 4093, 1820, 4066, 439, 0, 4084, 0, 0, 1, 907, 562, 2163, 1, 10, 3454, 512, 0, 0, 4093, 0, 0, 0, 3258, 1, 28, 0, 4095, 0, 0, 0, 4, 0, 4095, 0, 0, 2865, 0, 4081, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 0, 13, 0, 0, 3, 418, 4096, 3852, 1, 0, 4095, 1, 0, 4094, 0, 0, 265, 0, 4095, 827, 4006, 120, 1, 4074, 0, 1, 1, 630, 967, 2758, 1, 23, 3411, 371, 0, 0, 4061, 0, 1, 0, 3490, 0, 13, 0, 4096, 0, 2, 4, 2, 0, 4093, 0, 1, 3475, 0, 4075, 0, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 12, 41, 0, 0, 0, 134, 4089, 3593, 0, 0, 4096, 0, 0, 4095, 0, 0, 108, 0, 4073, 1726, 4092, 211, 0, 4049, 0, 0, 0, 912, 507, 2838, 3, 3, 3215, 1299, 0, 0, 3575, 0, 10, 0, 3731, 0, 21, 0, 4095, 0, 0, 0, 0, 0, 4090, 0, 0, 2746, 0, 4077, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[2, 46, 2, 4081, 0, 7, 0, 1, 0, 0, 108, 4090, 4043, 1, 2, 4093, 0, 3, 4094, 2, 2, 100, 0, 4092, 2188, 3865, 79, 1, 4092, 2, 0, 1, 1531, 2002, 1761, 0, 2, 3409, 833, 2, 2, 4058, 0, 0, 2, 1912, 2, 85, 2, 4093, 0, 2, 0, 0, 0, 4092, 2, 3, 2679, 0, 4061, 2, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 0, 4066, 0, 6, 0, 0, 0, 0, 32, 4086, 4043, 0, 0, 4092, 0, 0, 4096, 0, 0, 211, 7, 4085, 1564, 4059, 35, 0, 4084, 0, 0, 0, 529, 1015, 2233, 14, 8, 3681, 718, 0, 0, 3955, 0, 0, 0, 3517, 0, 32, 0, 4095, 0, 0, 0, 5, 0, 4096, 0, 0, 3163, 0, 4005, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[6, 3, 6, 4087, 0, 1, 0, 0, 4043, 22, 0, 0, 0, 0, 0, 0, 1, 7, 185, 4083, 4018, 0, 6, 41, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 3750, 1, 6, 739, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2304, 3756, 223, 3, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2057, 177, 2165, 0, 0, 2377, 732, 6, 87, 0, 0, 0, 0, 0, 0, 0, 2, 4087, 0, 0, 4, 3177, 0, 0, 668, 247, 0, 0, 0, 0, 0, 0, 3, 4090, 12, 6, 10, 4, 3, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8, 2726, 38, 3981, 0, 0, 0, 1213, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8341, 8602, 8476, 7601, 8185, 8191, 7972] avg=8192.0000 max=8602 max_rank=2 min=7601 min_rank=4 imbalance_ratio=1.050049 max_load_reduction=56934 max_load_reduction_pct=0.868744 moved_to_echo=31949 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4093, 0, 0, 6, 0, 4052, 22, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4095, 4021, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3760, 0, 0, 470, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1820, 4066, 439, 0, 2186, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 907, 562, 2163, 1, 10, 3454, 512, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3258, 1, 28, 666, 248, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2865, 0, 4081, 0, 4, 0, 1210, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8181, 8324, 8512, 7697, 8294, 8194, 8160] avg=8192.0000 max=8512 max_rank=3 min=7697 min_rank=4 imbalance_ratio=1.039062 max_load_reduction=57024 max_load_reduction_pct=0.870117 moved_to_echo=32345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4093, 0, 0, 4, 0, 4055, 22, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 3811, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3760, 0, 0, 450, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2279, 3965, 449, 0, 2192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 491, 2137, 0, 46, 2520, 1222, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 3504, 0, 1, 668, 248, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2563, 0, 4081, 0, 3, 0, 1214, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 7959, 8306, 8885, 7651, 8506, 8192, 7861] avg=8192.0000 max=8885 max_rank=3 min=7651 min_rank=4 imbalance_ratio=1.084595 max_load_reduction=56651 max_load_reduction_pct=0.864426 moved_to_echo=32210 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 12, 41, 0, 4055, 22, 0, 0, 0, 0, 0, 0, 0, 0, 134, 4089, 3593, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3760, 0, 0, 108, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 1726, 4092, 211, 0, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 507, 2838, 3, 3, 3215, 1299, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 3575, 0, 10, 0, 3731, 0, 21, 660, 247, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2746, 0, 4077, 0, 3, 0, 1200, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 7857, 7941, 8196, 8865, 8244, 8185, 8026] avg=8192.0000 max=8865 max_rank=4 min=7857 min_rank=1 imbalance_ratio=1.082153 max_load_reduction=56671 max_load_reduction_pct=0.864731 moved_to_echo=33320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 76, 0, 4088, 0, 6, 1, 0, 4055, 22, 0, 0, 0, 0, 0, 0, 0, 0, 92, 4081, 4089, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3760, 0, 0, 376, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 4094, 392, 0, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1144, 422, 964, 1, 6, 2983, 1949, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 3400, 0, 28, 667, 248, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3134, 0, 4057, 0, 0, 0, 1212, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8303, 8229, 8177, 7557, 8429, 8190, 8403] avg=8192.0000 max=8429 max_rank=5 min=7557 min_rank=4 imbalance_ratio=1.028931 max_load_reduction=57107 max_load_reduction_pct=0.871384 moved_to_echo=32579 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 0, 13, 0, 4054, 22, 0, 0, 0, 0, 0, 0, 0, 3, 418, 4096, 3852, 1, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3759, 0, 0, 265, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 827, 4006, 120, 1, 2179, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 630, 967, 2758, 1, 23, 3411, 371, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 1, 0, 3490, 0, 13, 665, 247, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3475, 0, 4075, 0, 9, 0, 1208, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8411, 8120, 7135, 8249, 8477, 8197, 8768] avg=8192.0000 max=8768 max_rank=7 min=7135 min_rank=3 imbalance_ratio=1.070312 max_load_reduction=56768 max_load_reduction_pct=0.866211 moved_to_echo=33691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 46, 2, 4081, 0, 7, 0, 1, 4052, 22, 0, 0, 0, 0, 0, 0, 0, 0, 108, 4090, 4043, 1, 2, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3759, 2, 2, 100, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2188, 3865, 79, 1, 2189, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1531, 2002, 1761, 0, 2, 3409, 833, 2, 88, 0, 0, 0, 0, 0, 0, 0, 2, 4058, 0, 0, 2, 1912, 2, 85, 668, 247, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 2, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 2679, 0, 4061, 2, 0, 2, 1213, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8285, 7958, 8325, 9628, 6976, 8189, 7962] avg=8192.0000 max=9628 max_rank=4 min=6976 min_rank=5 imbalance_ratio=1.175293 max_load_reduction=55908 max_load_reduction_pct=0.853088 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 4066, 0, 6, 0, 0, 4054, 27, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4086, 4043, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 0, 0, 211, 7, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 4059, 35, 0, 2172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 1015, 2233, 14, 8, 3681, 718, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 3517, 0, 32, 670, 251, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 5, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3163, 0, 4005, 0, 1, 0, 1215, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8199, 8056, 7830, 8288, 8425, 8198, 8384] avg=8192.0000 max=8425 max_rank=5 min=7830 min_rank=3 imbalance_ratio=1.028442 max_load_reduction=57111 max_load_reduction_pct=0.871445 moved_to_echo=33295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 30080, 128, 128, 2816, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[14336, 32000, 2048, 128, 17536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 23424, 128, 32512, 128, 128, 128, 9728, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 256, 128, 32768, 0, 128, 128, 128, 32512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8960, 6144, 17024, 128, 128, 25088, 7680, 128, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32000, 0, 128, 128, 26112, 128, 256, 5376, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 1024, 32768, 31488, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[6, 9, 0, 0, 3877, 11, 6, 3723, 98, 4090, 0, 7, 0, 6, 0, 3202, 4085, 0, 4072, 4086, 9, 8, 7, 4090, 0, 6, 0, 19, 0, 7, 0, 1, 0, 9, 21, 4086, 7, 4082, 4085, 0, 4088, 4088, 8, 0, 0, 1, 0, 0, 4088, 0, 8, 500, 3558, 7, 1, 0, 4084, 0, 1377, 5, 0, 0, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3920, 0, 0, 3366, 4, 4087, 0, 0, 0, 3, 0, 3249, 4095, 0, 4093, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 4096, 4096, 0, 4096, 4080, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 248, 3893, 0, 4, 0, 4092, 0, 1730, 0, 1, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4011, 0, 0, 3220, 6, 4077, 0, 0, 0, 17, 0, 3656, 4096, 0, 4077, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4095, 4096, 0, 4096, 4090, 0, 1, 0, 0, 0, 4, 4096, 0, 0, 11, 4071, 0, 17, 0, 4096, 0, 1413, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3990, 0, 0, 3654, 7, 4096, 0, 0, 0, 2, 0, 3580, 4094, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 34, 4006, 0, 0, 0, 4090, 3, 1118, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4085, 0, 0, 3604, 2, 4034, 0, 0, 0, 16, 0, 4013, 4096, 0, 4077, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 310, 3103, 0, 0, 0, 4096, 0, 1325, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3229, 0, 0, 3730, 91, 4094, 0, 0, 0, 1, 0, 2598, 4090, 0, 4062, 4096, 0, 0, 0, 4092, 0, 2, 0, 2, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 4096, 4096, 0, 4095, 4093, 0, 2, 0, 0, 0, 4, 4096, 2, 0, 142, 4063, 0, 6, 0, 4094, 0, 2560, 0, 1, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2, 2, 0, 4, 3772, 2, 0, 3788, 29, 4038, 0, 2, 0, 0, 1, 2846, 4092, 0, 4074, 4093, 2, 2, 2, 4094, 0, 0, 0, 0, 3, 0, 2, 0, 0, 2, 3, 4094, 5, 4094, 4093, 0, 4094, 4085, 2, 0, 2, 0, 0, 0, 4093, 0, 2, 416, 3736, 0, 1, 0, 4091, 1, 1864, 2, 4, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3947, 0, 0, 3573, 161, 4058, 0, 0, 0, 33, 0, 3430, 4095, 0, 4093, 4096, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 4096, 4096, 0, 4096, 4078, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 150, 4032, 0, 6, 0, 4096, 0, 1113, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[6, 9, 0, 0, 3877, 11, 6, 3723, 749, 0, 0, 0, 0, 0, 0, 0, 98, 4090, 0, 7, 0, 6, 0, 3202, 736, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4072, 0, 9, 8, 7, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 19, 0, 7, 0, 1, 4086, 4083, 0, 0, 0, 0, 0, 0, 0, 9, 21, 4086, 7, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4088, 8, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4088, 0, 8, 500, 3558, 7, 1, 0, 56, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 1377, 5, 0, 0, 0, 8, 2526, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8381, 8139, 8203, 8202, 8203, 8188, 8218, 8002] avg=8192.0000 max=8381 max_rank=0 min=8002 min_rank=7 imbalance_ratio=1.023071 max_load_reduction=57155 max_load_reduction_pct=0.872116 moved_to_echo=32611 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4011, 0, 0, 3220, 751, 0, 0, 0, 0, 0, 0, 0, 6, 4077, 0, 0, 0, 17, 0, 3656, 738, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4077, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 1, 0, 0, 0, 4, 2, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 4071, 0, 17, 0, 56, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1413, 0, 0, 0, 1, 0, 2532, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7982, 8494, 8191, 8190, 8190, 8194, 8251, 8044] avg=8192.0000 max=8494 max_rank=1 min=7982 min_rank=0 imbalance_ratio=1.036865 max_load_reduction=57042 max_load_reduction_pct=0.870392 moved_to_echo=32679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3920, 0, 0, 3366, 751, 0, 0, 0, 0, 0, 0, 0, 4, 4087, 0, 0, 0, 3, 0, 3249, 738, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4093, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 1, 2, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 248, 3893, 0, 4, 0, 56, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1730, 0, 1, 0, 1, 0, 2532, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8037, 8081, 8203, 8190, 8190, 8180, 8297, 8358] avg=8192.0000 max=8358 max_rank=7 min=8037 min_rank=0 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=33025 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3990, 0, 0, 3654, 750, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 2, 0, 3580, 737, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 34, 4006, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 1118, 1, 0, 0, 0, 0, 2531, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8394, 8422, 8211, 8190, 8190, 8192, 8192, 7745] avg=8192.0000 max=8422 max_rank=1 min=7745 min_rank=7 imbalance_ratio=1.028076 max_load_reduction=57114 max_load_reduction_pct=0.871490 moved_to_echo=32319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3229, 0, 0, 3730, 750, 0, 0, 0, 0, 0, 0, 0, 91, 4094, 0, 0, 0, 1, 0, 2598, 737, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4062, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 0, 2, 0, 0, 0, 4, 2, 1, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 142, 4063, 0, 6, 0, 56, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2560, 0, 1, 0, 3, 0, 2529, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7709, 7521, 8171, 8194, 8190, 8197, 8365, 9189] avg=8192.0000 max=9189 max_rank=7 min=7521 min_rank=1 imbalance_ratio=1.121704 max_load_reduction=56347 max_load_reduction_pct=0.859787 moved_to_echo=33941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 0, 4, 3772, 2, 0, 3788, 750, 0, 0, 0, 0, 0, 0, 0, 29, 4038, 0, 2, 0, 0, 1, 2846, 737, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4074, 0, 2, 2, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 4093, 4091, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4094, 5, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4085, 2, 0, 2, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 416, 3736, 0, 1, 0, 56, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 1864, 2, 4, 0, 0, 2, 2531, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 7653, 8191, 8189, 8196, 8186, 8304, 8497] avg=8192.0000 max=8497 max_rank=7 min=7653 min_rank=1 imbalance_ratio=1.037231 max_load_reduction=57039 max_load_reduction_pct=0.870346 moved_to_echo=33183 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4085, 0, 0, 3604, 751, 0, 0, 0, 0, 0, 0, 0, 2, 4034, 0, 0, 0, 16, 0, 4013, 738, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4077, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 8, 2, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 310, 3103, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1325, 0, 0, 0, 0, 0, 2532, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8440, 8803, 8191, 8190, 8190, 8202, 7565, 7955] avg=8192.0000 max=8803 max_rank=1 min=7565 min_rank=6 imbalance_ratio=1.074585 max_load_reduction=56733 max_load_reduction_pct=0.865677 moved_to_echo=31912 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3947, 0, 0, 3573, 753, 0, 0, 0, 0, 0, 0, 0, 161, 4058, 0, 0, 0, 33, 0, 3430, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 0, 0, 0, 0, 11, 5, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 150, 4032, 0, 6, 0, 60, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1113, 0, 0, 0, 2, 0, 2534, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8423, 8175, 8191, 8187, 8197, 8344, 7746] avg=8192.0000 max=8423 max_rank=1 min=7746 min_rank=7 imbalance_ratio=1.028198 max_load_reduction=57113 max_load_reduction_pct=0.871475 moved_to_echo=32474 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 1920, 30464, 128, 128, 0, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 30848, 128, 128, 28672, 6016, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 12544, 128, 128, 0, 128, 128, 20352, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 32640, 0, 128, 0, 128, 128, 26624, 6016, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[15, 5, 0, 1, 4006, 0, 9, 2, 28, 4078, 0, 8, 8, 8, 0, 4087, 0, 8, 0, 43, 1, 7, 4054, 0, 4087, 10, 19, 4087, 0, 4088, 0, 0, 0, 0, 13, 0, 0, 4088, 125, 9, 4071, 5, 4086, 9, 0, 8, 3, 4058, 4088, 4072, 8, 0, 0, 0, 4084, 4, 0, 7, 4088, 4029, 5, 7, 1, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 4094, 7, 0, 0, 1, 0, 0, 3951, 0, 4096, 0, 10, 4096, 0, 4078, 0, 0, 4, 4, 27, 0, 4, 4096, 3, 0, 4096, 0, 4096, 0, 0, 0, 0, 4091, 4091, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 4092, 3960, 0, 0, 253, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 4093, 6, 0, 0, 7, 0, 0, 3964, 0, 4090, 0, 1, 4096, 0, 4093, 0, 1, 2, 1, 8, 1, 0, 4095, 17, 0, 4096, 1, 4096, 0, 0, 0, 0, 4093, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4094, 0, 0, 108, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4080, 0, 4095, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 1, 4096, 0, 0, 4095, 1, 4095, 0, 0, 0, 0, 4094, 4084, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 28, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 18, 0, 0, 4074, 0, 4093, 0, 22, 4096, 0, 4069, 0, 0, 1, 0, 81, 0, 1, 4096, 64, 0, 4096, 0, 4096, 0, 0, 0, 0, 4089, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3957, 0, 0, 16, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 4094, 0, 0, 0, 0, 4087, 4, 0, 0, 0, 0, 4095, 3, 0, 0, 3, 0, 0, 4055, 0, 4091, 0, 4, 4096, 0, 4089, 0, 8, 0, 5, 31, 0, 0, 4094, 8, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 4074, 4096, 0, 0, 2, 0, 4096, 0, 1, 0, 4092, 4067, 0, 0, 56, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1, 4, 1, 0, 4093, 0, 2, 2, 0, 4085, 2, 2, 2, 2, 0, 4086, 2, 2, 0, 7, 0, 2, 4087, 0, 4093, 2, 7, 4066, 1, 4087, 1, 2, 15, 0, 30, 0, 5, 4094, 17, 0, 4093, 1, 4089, 2, 0, 2, 2, 4079, 4093, 4094, 2, 0, 0, 0, 4090, 1, 0, 2, 4094, 4060, 2, 2, 24, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 9, 0, 0, 0, 0, 4085, 17, 0, 0, 2, 0, 0, 3973, 0, 4092, 0, 1, 4096, 0, 4063, 0, 0, 3, 2, 114, 0, 0, 4095, 17, 0, 4096, 0, 4096, 0, 0, 0, 0, 4077, 4096, 4096, 0, 0, 6, 0, 4096, 0, 6, 0, 4094, 4015, 0, 0, 99, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[15, 5, 0, 1, 4006, 0, 9, 2, 4022, 73, 0, 0, 0, 0, 0, 0, 28, 4078, 0, 8, 8, 8, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 43, 1, 7, 4054, 0, 4077, 57, 0, 0, 0, 0, 0, 0, 8, 10, 19, 4087, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4088, 125, 9, 3999, 13, 0, 0, 0, 0, 0, 0, 4071, 5, 14, 9, 0, 8, 3, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4072, 8, 0, 0, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4074, 4029, 5, 7, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8218, 8247, 8212, 8247, 8168, 8179, 8132] avg=8192.0000 max=8247 max_rank=2 min=8132 min_rank=7 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4031, 74, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 3951, 0, 4089, 58, 0, 0, 0, 0, 0, 0, 7, 0, 10, 4096, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 27, 0, 4, 4096, 3, 0, 4009, 13, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4078, 3960, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8191, 8106, 8191, 8160, 8200, 8194, 8293] avg=8192.0000 max=8293 max_rank=7 min=8106 min_rank=2 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4025, 74, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7, 0, 0, 3964, 0, 4089, 57, 0, 0, 0, 0, 0, 0, 8, 0, 1, 4096, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 8, 1, 0, 4095, 17, 0, 4009, 13, 0, 0, 0, 0, 0, 0, 4096, 1, 13, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4094, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8189, 8123, 8199, 8146, 8203, 8198, 8283] avg=8192.0000 max=8283 max_rank=7 min=8123 min_rank=2 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4096, 0, 0, 0, 4030, 74, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4080, 0, 4088, 58, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4008, 13, 0, 0, 0, 0, 0, 0, 4095, 1, 13, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8193, 8227, 8200, 8118, 8203, 8187, 8206] avg=8192.0000 max=8227 max_rank=2 min=8118 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 1, 0, 4093, 0, 2, 2, 4028, 73, 0, 0, 0, 0, 0, 0, 0, 4085, 2, 2, 2, 2, 0, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 7, 0, 2, 4087, 0, 4083, 58, 0, 0, 0, 0, 0, 0, 7, 2, 7, 4066, 1, 4087, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 30, 0, 5, 4094, 17, 0, 4002, 13, 0, 0, 0, 0, 0, 0, 4093, 1, 14, 2, 0, 2, 2, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 2, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4080, 4060, 2, 2, 24, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8180, 8241, 8173, 8176, 8193, 8197, 8172] avg=8192.0000 max=8241 max_rank=2 min=8172 min_rank=7 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 4094, 0, 0, 0, 4026, 74, 0, 0, 0, 0, 0, 0, 0, 4087, 4, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 4055, 0, 4089, 57, 0, 0, 0, 0, 0, 0, 8, 0, 4, 4096, 0, 4089, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 31, 0, 0, 4094, 8, 0, 4009, 13, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 0, 0, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4078, 4067, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8187, 8207, 8205, 8160, 8192, 8179, 8202] avg=8192.0000 max=8207 max_rank=2 min=8160 min_rank=4 imbalance_ratio=1.001831 max_load_reduction=57329 max_load_reduction_pct=0.874771 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 4028, 74, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 18, 0, 0, 4074, 0, 4089, 58, 0, 0, 0, 0, 0, 0, 7, 0, 22, 4096, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 81, 0, 1, 4096, 64, 0, 4009, 13, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 3957, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8188, 8242, 8194, 8265, 8198, 8198, 8055] avg=8192.0000 max=8265 max_rank=4 min=8055 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4029, 76, 0, 0, 0, 0, 0, 0, 0, 4094, 9, 0, 0, 0, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 2, 0, 0, 3973, 0, 4090, 61, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4096, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 114, 0, 0, 4095, 17, 0, 4014, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4073, 4015, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8190, 8143, 8162, 8264, 8179, 8204, 8193] avg=8192.0000 max=8264 max_rank=4 min=8143 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 128, 128, 32256, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 384, 128, 128, 32768, 256, 128, 32128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 128, 32256, 0, 32768, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32640, 32384, 128, 128, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 4093, 3524, 0, 2, 8, 0, 0, 1, 4081, 2, 32, 0, 4094, 11, 4093, 2, 2, 0, 0, 1, 1, 1634, 4094, 2, 18, 0, 2, 1, 2, 4072, 0, 0, 0, 2, 4080, 4093, 1365, 4091, 2, 3, 4077, 4090, 2673, 2, 0, 3, 1, 2, 0, 4093, 2, 0, 1, 2, 3328, 2, 3779, 34, 0, 0, 0, 37] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[3, 6, 4086, 2948, 0, 8, 23, 0, 3, 0, 4086, 9, 2, 0, 4087, 0, 4087, 10, 2, 0, 0, 4, 0, 1386, 4087, 10, 78, 0, 9, 1, 7, 4094, 0, 9, 1, 4, 4028, 3883, 592, 4076, 9, 0, 4087, 4083, 3527, 12, 6, 4, 15, 7, 0, 4068, 0, 182, 0, 9, 3777, 9, 4046, 13, 0, 0, 0, 53] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3288, 0, 0, 85, 0, 0, 0, 4081, 0, 15, 0, 4096, 259, 4096, 0, 0, 0, 0, 0, 0, 905, 4096, 0, 8, 0, 0, 0, 0, 3946, 1, 0, 0, 0, 4067, 4037, 652, 4096, 0, 0, 4088, 4096, 3323, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 4073, 0, 3923, 79, 0, 3, 0, 28] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3018, 0, 0, 19, 0, 0, 0, 4093, 0, 91, 0, 4096, 26, 4096, 0, 0, 0, 0, 0, 1, 1746, 4096, 0, 21, 0, 0, 0, 0, 4081, 1, 0, 0, 0, 4094, 4095, 1264, 4096, 0, 0, 4094, 4095, 3444, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 3174, 0, 3566, 11, 0, 2, 0, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1607, 0, 0, 13, 0, 0, 0, 4096, 0, 21, 0, 4095, 52, 4096, 0, 0, 0, 0, 0, 0, 2439, 4096, 0, 30, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 4078, 4071, 1977, 4095, 0, 0, 4094, 4096, 2469, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 3950, 0, 3793, 30, 0, 0, 0, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2090, 0, 0, 0, 0, 0, 0, 4068, 0, 8, 0, 4096, 12, 4096, 0, 0, 0, 0, 0, 0, 3061, 4096, 0, 88, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 3969, 4032, 1577, 4096, 0, 0, 4091, 4096, 2904, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 3705, 0, 3123, 22, 0, 6, 0, 48] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2837, 6, 0, 12, 0, 0, 0, 4094, 0, 76, 0, 4084, 418, 4096, 0, 0, 0, 2, 0, 2, 200, 4096, 0, 33, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 4089, 4094, 2350, 4096, 0, 0, 4095, 4090, 2546, 0, 1, 0, 0, 0, 0, 4081, 0, 64, 1, 0, 4035, 0, 3815, 5, 0, 1, 0, 93] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3057, 1, 0, 1, 0, 0, 0, 4089, 0, 21, 0, 4096, 242, 4096, 0, 0, 0, 0, 0, 2, 176, 4096, 0, 60, 0, 0, 0, 0, 4020, 2, 0, 0, 0, 4084, 4094, 1922, 4094, 0, 0, 4067, 4096, 3194, 0, 0, 0, 0, 0, 0, 4059, 0, 68, 0, 0, 3966, 0, 3695, 75, 0, 13, 0, 54] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 6, 4086, 2948, 0, 8, 23, 0, 887, 382, 0, 0, 0, 0, 0, 0, 3, 0, 4086, 9, 2, 0, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 10, 2, 0, 0, 4, 0, 1386, 2612, 32, 0, 0, 0, 0, 0, 0, 4087, 10, 78, 0, 9, 1, 7, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 4, 2656, 3883, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4087, 1089, 3527, 12, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 15, 7, 0, 4068, 0, 182, 0, 9, 4044, 0, 0, 0, 0, 0, 0, 0, 3777, 9, 4046, 13, 0, 0, 0, 53, 485, 146, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8343, 8038, 8133, 8289, 7145, 8734, 8325, 8529] avg=8192.0000 max=8734 max_rank=5 min=7145 min_rank=4 imbalance_ratio=1.066162 max_load_reduction=56802 max_load_reduction_pct=0.866730 moved_to_echo=32733 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3018, 0, 0, 19, 0, 901, 383, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 91, 0, 3947, 26, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 1746, 2620, 31, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 4081, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2700, 4095, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1092, 3444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 3174, 0, 3566, 11, 0, 2, 0, 23, 493, 146, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8417, 8157, 8494, 8201, 8060, 8630, 8162, 7415] avg=8192.0000 max=8630 max_rank=5 min=7415 min_rank=7 imbalance_ratio=1.053467 max_load_reduction=56906 max_load_reduction_pct=0.868317 moved_to_echo=32267 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3288, 0, 0, 85, 0, 895, 383, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 15, 0, 3947, 259, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 905, 2621, 31, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 3946, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2682, 4037, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1092, 3323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 3923, 79, 0, 3, 0, 28, 490, 146, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8747, 8302, 7653, 8053, 7372, 8503, 8164, 8742] avg=8192.0000 max=8747 max_rank=0 min=7372 min_rank=4 imbalance_ratio=1.067749 max_load_reduction=56789 max_load_reduction_pct=0.866531 moved_to_echo=32781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 1607, 0, 0, 13, 0, 898, 383, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 3946, 52, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2439, 2621, 31, 0, 0, 0, 0, 0, 0, 4096, 0, 30, 0, 0, 0, 0, 4072, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2689, 4071, 1977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1092, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 3950, 0, 3793, 30, 0, 0, 0, 61, 491, 146, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6997, 8115, 9187, 8201, 8737, 7655, 8173, 8471] avg=8192.0000 max=9187 max_rank=2 min=6997 min_rank=0 imbalance_ratio=1.121460 max_load_reduction=56349 max_load_reduction_pct=0.859818 moved_to_echo=33036 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4093, 3524, 0, 2, 8, 0, 898, 383, 0, 0, 0, 0, 0, 0, 0, 1, 4081, 2, 32, 0, 3945, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 2, 0, 0, 1, 1, 1634, 2617, 32, 0, 0, 0, 0, 0, 0, 4094, 2, 18, 0, 2, 1, 2, 4072, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2691, 4093, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4077, 1090, 2673, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4093, 2, 0, 1, 2, 4059, 0, 0, 0, 0, 0, 0, 0, 3328, 2, 3779, 34, 0, 0, 0, 37, 491, 146, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8910, 8072, 8382, 8194, 8151, 7850, 8160, 7817] avg=8192.0000 max=8910 max_rank=0 min=7817 min_rank=7 imbalance_ratio=1.087646 max_load_reduction=56626 max_load_reduction_pct=0.864044 moved_to_echo=31978 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 2837, 6, 0, 12, 0, 900, 383, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 76, 0, 3936, 418, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 2, 200, 2617, 31, 0, 0, 0, 0, 0, 0, 4096, 0, 33, 0, 0, 0, 0, 4028, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2697, 4094, 2350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1090, 2546, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 64, 1, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 3815, 5, 0, 1, 0, 93, 492, 145, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8524, 6948, 8160, 9141, 7732, 8211, 8586] avg=8192.0000 max=9141 max_rank=4 min=6948 min_rank=2 imbalance_ratio=1.115845 max_load_reduction=56395 max_load_reduction_pct=0.860519 moved_to_echo=33670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 2090, 0, 0, 0, 0, 874, 383, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 8, 0, 3947, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3061, 2621, 31, 0, 0, 0, 0, 0, 0, 4096, 0, 88, 0, 0, 0, 0, 4069, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 4032, 1577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1092, 2904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 3705, 0, 3123, 22, 0, 6, 0, 48, 478, 146, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7443, 8035, 9809, 8256, 8226, 8087, 8152, 7528] avg=8192.0000 max=9809 max_rank=2 min=7443 min_rank=0 imbalance_ratio=1.197388 max_load_reduction=55727 max_load_reduction_pct=0.850327 moved_to_echo=31993 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3057, 1, 0, 1, 0, 902, 388, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 21, 0, 3941, 242, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 176, 2624, 32, 0, 0, 0, 0, 0, 0, 4096, 0, 60, 0, 0, 0, 0, 4020, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2686, 4094, 1922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 1084, 3194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 68, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 3695, 75, 0, 13, 0, 54, 496, 149, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8445, 8293, 6930, 8182, 8704, 8345, 8189, 8448] avg=8192.0000 max=8704 max_rank=4 min=6930 min_rank=2 imbalance_ratio=1.062500 max_load_reduction=56832 max_load_reduction_pct=0.867188 moved_to_echo=33686 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 22400, 128, 128, 256, 0, 7168, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 384, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 21504, 32512, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[30080, 128, 29824, 384, 0, 128, 0, 512, 3968, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 128, 11648, 20992, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 8832, 24192, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 384, 0, 31616, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 384, 0, 128, 128, 128, 32384, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4046, 0, 0, 0, 2214, 0, 4096, 4094, 0, 4079, 4096, 0, 1, 0, 0, 1693, 4093, 0, 323, 0, 0, 4046, 4096, 0, 0, 4091, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 3915, 0, 1, 4091, 0, 0, 0, 4096, 4096, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 5, 4095, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 2573, 1, 4096, 4093, 0, 4094, 4096, 0, 1, 0, 0, 455, 4096, 0, 1026, 0, 0, 4066, 4096, 0, 0, 4093, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 4072, 0, 3, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 1, 4079, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[11, 0, 69, 9, 4082, 12, 0, 27, 3875, 14, 4085, 4088, 12, 4067, 4012, 8, 47, 0, 9, 143, 4082, 2, 22, 0, 0, 4024, 4084, 10, 9, 4074, 9, 0, 7, 0, 57, 7, 9, 0, 1, 0, 8, 4080, 0, 0, 4087, 0, 6, 9, 4072, 4083, 5, 8, 0, 16, 0, 4, 0, 8, 6, 0, 25, 4075, 0, 4077] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4081, 0, 0, 0, 3454, 0, 4096, 4085, 0, 4086, 4096, 0, 0, 0, 0, 536, 4096, 0, 492, 0, 0, 4046, 4096, 0, 0, 4092, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 3719, 1, 4, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 3, 3998, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 3492, 0, 4096, 4075, 0, 4061, 4096, 0, 1, 0, 0, 246, 4093, 0, 513, 0, 0, 4008, 4096, 0, 0, 4096, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 3906, 4, 23, 4089, 0, 0, 0, 4096, 4096, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 8, 4070, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 2, 0, 3135, 0, 4096, 4082, 0, 4069, 4096, 0, 0, 0, 0, 332, 4094, 0, 610, 0, 0, 3870, 4096, 0, 0, 4086, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 3964, 2, 0, 4091, 0, 0, 0, 4096, 4096, 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 11, 4073, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 1, 0, 2685, 0, 4096, 4089, 0, 4000, 4096, 0, 0, 0, 0, 334, 4085, 0, 1104, 0, 0, 4012, 4096, 0, 0, 4083, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 4052, 2, 0, 4093, 0, 0, 0, 4096, 4096, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 11, 4092, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3, 0, 6, 2, 4093, 1, 0, 0, 2914, 4, 4094, 4093, 1, 4058, 4091, 1, 40, 0, 2, 540, 4086, 0, 699, 0, 0, 3916, 4094, 3, 2, 4090, 2, 0, 2, 0, 93, 3, 2, 0, 0, 0, 2, 3859, 0, 3, 4093, 3, 0, 2, 4093, 4092, 1, 2, 1, 253, 0, 1, 0, 2, 2, 1, 17, 4081, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 3994, 58, 38, 0, 0, 0, 0, 0, 2573, 1, 0, 0, 0, 4094, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 455, 4096, 0, 1026, 0, 2954, 8, 0, 0, 0, 0, 0, 0, 0, 4066, 102, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 4096, 4055, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 3, 4096, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 4096, 3928, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 7752, 8540, 8261, 8181, 8316, 8109, 8191] avg=8192.0000 max=8540 max_rank=2 min=7752 min_rank=1 imbalance_ratio=1.042480 max_load_reduction=56996 max_load_reduction_pct=0.869690 moved_to_echo=32797 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 69, 9, 4082, 12, 0, 27, 3983, 57, 38, 0, 0, 0, 0, 0, 3875, 14, 0, 0, 12, 4067, 1062, 8, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 9, 143, 4082, 2, 22, 0, 2893, 8, 0, 0, 0, 0, 0, 0, 0, 4024, 101, 10, 9, 4074, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 57, 7, 9, 0, 1, 0, 4085, 4050, 0, 0, 0, 0, 0, 0, 8, 4080, 0, 0, 4087, 0, 6, 9, 144, 0, 0, 0, 0, 0, 0, 0, 4072, 3916, 5, 8, 0, 16, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 25, 4075, 0, 4077, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 9038, 7206, 8227, 8216, 8334, 8021, 8206] avg=8192.0000 max=9038 max_rank=1 min=7206 min_rank=2 imbalance_ratio=1.103271 max_load_reduction=56498 max_load_reduction_pct=0.862091 moved_to_echo=32777 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4046, 0, 0, 0, 3994, 58, 38, 0, 0, 0, 0, 0, 2214, 0, 0, 0, 0, 4079, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1693, 4093, 0, 323, 0, 2954, 8, 0, 0, 0, 0, 0, 0, 0, 4046, 102, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 4096, 4056, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 1, 4091, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 4096, 3928, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4095, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 7377, 9072, 8239, 8177, 8152, 8172, 8211] avg=8192.0000 max=9072 max_rank=2 min=7377 min_rank=1 imbalance_ratio=1.107422 max_load_reduction=56464 max_load_reduction_pct=0.861572 moved_to_echo=32712 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 1, 0, 3994, 58, 37, 0, 0, 0, 0, 0, 2685, 0, 0, 0, 0, 4000, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 4085, 0, 1104, 0, 2954, 8, 0, 0, 0, 0, 0, 0, 0, 4012, 102, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 4096, 4052, 0, 0, 0, 0, 0, 0, 0, 4052, 2, 0, 4093, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 4096, 3928, 0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4092, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 7769, 8485, 8197, 8231, 8292, 8166, 8214] avg=8192.0000 max=8485 max_rank=2 min=7769 min_rank=1 imbalance_ratio=1.035767 max_load_reduction=57051 max_load_reduction_pct=0.870529 moved_to_echo=32903 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 6, 2, 4093, 1, 0, 0, 3993, 58, 38, 0, 0, 0, 0, 0, 2914, 4, 0, 0, 1, 4058, 1083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 2, 540, 4086, 0, 699, 0, 2950, 8, 0, 0, 0, 0, 0, 0, 0, 3916, 101, 3, 2, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 93, 3, 2, 0, 0, 0, 4094, 4055, 0, 0, 0, 0, 0, 0, 2, 3859, 0, 3, 4093, 3, 0, 2, 145, 0, 0, 0, 0, 0, 0, 0, 4093, 3924, 1, 2, 1, 253, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 17, 4081, 0, 4093, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8061, 8325, 8114, 8249, 8107, 8275, 8211] avg=8192.0000 max=8325 max_rank=2 min=8061 min_rank=1 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32842 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 3994, 58, 38, 0, 0, 0, 0, 0, 3492, 0, 0, 0, 0, 4061, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 246, 4093, 0, 513, 0, 2954, 8, 0, 0, 0, 0, 0, 0, 0, 4008, 102, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 4096, 4037, 0, 0, 0, 0, 0, 0, 0, 3906, 4, 23, 4089, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 4096, 3928, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4070, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8637, 7815, 8206, 8166, 8167, 8173, 8189] avg=8192.0000 max=8637 max_rank=1 min=7815 min_rank=2 imbalance_ratio=1.054321 max_load_reduction=56899 max_load_reduction_pct=0.868210 moved_to_echo=32695 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4081, 0, 0, 0, 3994, 58, 38, 0, 0, 0, 0, 0, 3454, 0, 0, 0, 0, 4086, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 4096, 0, 492, 0, 2954, 8, 0, 0, 0, 0, 0, 0, 0, 4046, 102, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 4047, 0, 0, 0, 0, 0, 0, 0, 3719, 1, 4, 4095, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 4096, 3928, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3998, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8624, 8086, 8240, 8162, 7964, 8177, 8112] avg=8192.0000 max=8624 max_rank=1 min=7964 min_rank=5 imbalance_ratio=1.052734 max_load_reduction=56912 max_load_reduction_pct=0.868408 moved_to_echo=32415 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 2, 0, 4000, 64, 38, 0, 0, 0, 0, 0, 3135, 0, 0, 0, 0, 4069, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 4094, 0, 610, 0, 2958, 13, 0, 0, 0, 0, 0, 0, 0, 3870, 96, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 3964, 2, 0, 4091, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 4096, 3914, 0, 0, 0, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4073, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8278, 8007, 8052, 8154, 8204, 8443, 8202] avg=8192.0000 max=8443 max_rank=6 min=8007 min_rank=2 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=33003 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[24448, 128, 0, 0, 128, 32640, 8704, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 384, 128, 128, 0, 128, 0, 32768, 32512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 128, 128, 32000, 512, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 4352, 32768, 128, 4864, 0, 23680, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 128, 32640, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 31616, 128, 128, 32768, 128, 128, 128, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32000, 896, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 31488, 128, 128, 128, 1408, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[12, 0, 0, 4086, 3515, 9, 0, 4085, 3, 21, 10, 9, 4076, 0, 9, 4002, 3932, 0, 10, 4087, 6, 0, 4087, 4086, 0, 0, 4, 0, 9, 4076, 72, 0, 497, 315, 4080, 0, 10, 0, 0, 0, 0, 8, 4082, 0, 9, 7, 8, 1, 4069, 0, 11, 0, 14, 9, 0, 8, 0, 4015, 4087, 0, 4072, 24, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4079, 0, 0, 4096, 0, 0, 0, 0, 4095, 28, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4068, 4096, 5, 0, 0, 18, 0, 4096, 0, 0, 2, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 0, 4092, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4084, 0, 0, 4096, 0, 0, 0, 0, 4095, 7, 0, 4096, 4096, 0, 0, 4095, 0, 0, 4090, 4095, 17, 0, 0, 10, 0, 4096, 0, 0, 35, 64, 4096, 1, 0, 0, 0, 1, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 3999, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2, 4093, 3970, 2, 0, 4093, 0, 2, 3, 2, 4082, 0, 2, 4089, 4081, 0, 1, 4094, 2, 0, 4073, 4092, 0, 0, 1, 8, 2, 4093, 18, 1, 181, 57, 4093, 0, 2, 0, 1, 0, 0, 3, 4093, 0, 2, 2, 0, 1, 4092, 2, 3, 0, 0, 3, 0, 2, 0, 4072, 4093, 0, 4024, 2, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4090, 20, 0, 4096, 4096, 0, 0, 4092, 0, 0, 4080, 4092, 9, 0, 0, 8, 0, 4096, 0, 0, 257, 372, 4096, 4, 0, 7, 0, 4, 1, 0, 4090, 0, 0, 0, 3, 0, 4090, 0, 0, 0, 11, 0, 0, 0, 0, 4030, 4096, 0, 3508, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 35, 39, 4096, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4096, 0, 4060, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4074, 0, 0, 4096, 0, 0, 0, 0, 4091, 26, 0, 4096, 4096, 0, 0, 4093, 0, 0, 4064, 4096, 31, 0, 0, 43, 0, 4096, 0, 0, 476, 963, 4096, 1, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3792, 4096, 0, 2917, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4083, 0, 0, 4096, 0, 0, 0, 0, 4093, 29, 0, 4096, 4096, 0, 0, 4093, 0, 0, 4069, 4094, 23, 0, 0, 14, 0, 4096, 0, 0, 418, 121, 4096, 0, 0, 0, 0, 3, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 4035, 4096, 0, 3592, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 96, 3515, 9, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 3, 21, 10, 9, 4074, 0, 9, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 0, 10, 38, 6, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 9, 4076, 72, 0, 3954, 99, 0, 0, 0, 0, 0, 0, 497, 315, 4080, 0, 10, 0, 0, 0, 3600, 2, 0, 0, 0, 0, 0, 0, 0, 8, 4082, 0, 9, 7, 8, 1, 4086, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 11, 0, 14, 9, 0, 8, 4049, 36, 0, 0, 0, 0, 0, 0, 0, 4015, 388, 0, 4072, 24, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7717, 8128, 8073, 8214, 8504, 8201, 8196, 8503] avg=8192.0000 max=8504 max_rank=4 min=7717 min_rank=0 imbalance_ratio=1.038086 max_load_reduction=57032 max_load_reduction_pct=0.870239 moved_to_echo=33404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 96, 4084, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 38, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 10, 0, 4096, 0, 0, 3964, 99, 0, 0, 0, 0, 0, 0, 35, 64, 4096, 1, 0, 0, 0, 1, 3608, 2, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4057, 36, 1, 0, 0, 0, 0, 0, 0, 4078, 389, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8196, 8224, 8186, 7807, 8191, 8190, 8466] avg=8192.0000 max=8466 max_rank=7 min=7807 min_rank=4 imbalance_ratio=1.033447 max_load_reduction=57070 max_load_reduction_pct=0.870819 moved_to_echo=32654 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 96, 4074, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 26, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 38, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 43, 0, 4096, 0, 0, 3964, 99, 0, 0, 0, 0, 0, 0, 476, 963, 4096, 1, 0, 0, 0, 5, 3608, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4055, 36, 1, 0, 0, 0, 0, 0, 0, 3792, 389, 0, 2917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8211, 8198, 8233, 9151, 8191, 8188, 7098] avg=8192.0000 max=9151 max_rank=4 min=7098 min_rank=7 imbalance_ratio=1.117065 max_load_reduction=56385 max_load_reduction_pct=0.860367 moved_to_echo=32628 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 96, 4079, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 28, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 38, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 18, 0, 4096, 0, 0, 3964, 99, 0, 0, 0, 0, 0, 0, 2, 27, 4096, 0, 0, 0, 0, 0, 3608, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4058, 36, 0, 0, 0, 0, 0, 0, 0, 4067, 389, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8217, 8202, 8182, 7735, 8191, 8190, 8548] avg=8192.0000 max=8548 max_rank=7 min=7735 min_rank=4 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=32664 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 96, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 38, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3964, 99, 0, 0, 0, 0, 0, 0, 35, 39, 4096, 0, 0, 10, 0, 0, 3608, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4058, 36, 1, 0, 0, 0, 0, 0, 0, 4051, 389, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8190, 8229, 8159, 7790, 8191, 8191, 8500] avg=8192.0000 max=8500 max_rank=7 min=7790 min_rank=4 imbalance_ratio=1.037598 max_load_reduction=57036 max_load_reduction_pct=0.870300 moved_to_echo=32672 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 96, 3970, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 2, 4080, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 1, 38, 2, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 2, 4093, 18, 1, 3961, 99, 0, 0, 0, 0, 0, 0, 181, 57, 4093, 0, 2, 0, 1, 0, 3606, 2, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 2, 2, 0, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 3, 0, 0, 3, 0, 2, 4056, 36, 0, 0, 0, 0, 0, 0, 0, 4072, 388, 0, 4024, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8178, 8195, 8183, 7942, 8193, 8194, 8488] avg=8192.0000 max=8488 max_rank=7 min=7942 min_rank=4 imbalance_ratio=1.036133 max_load_reduction=57048 max_load_reduction_pct=0.870483 moved_to_echo=32817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 96, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 20, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 38, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8, 0, 4096, 0, 0, 3964, 99, 0, 0, 0, 0, 0, 0, 257, 372, 4096, 4, 0, 7, 0, 4, 3608, 2, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 3, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 11, 0, 0, 0, 4054, 36, 0, 0, 0, 0, 0, 0, 0, 4030, 389, 0, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8204, 8214, 8176, 8350, 8186, 8191, 7927] avg=8192.0000 max=8350 max_rank=4 min=7927 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32654 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 90, 4083, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 29, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 36, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 14, 0, 4096, 0, 0, 3966, 104, 0, 0, 0, 0, 0, 0, 418, 121, 4096, 0, 0, 0, 0, 3, 3613, 6, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4057, 40, 0, 0, 0, 0, 0, 0, 0, 4035, 379, 0, 3592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8212, 8201, 8203, 8257, 8192, 8196, 8006] avg=8192.0000 max=8269 max_rank=0 min=8006 min_rank=7 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32651 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32256, 3200, 0, 30336, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 128, 32512, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 768, 32000, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32640, 0, 128, 384, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 128, 128, 31744, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[1920, 2048, 32768, 128, 128, 128, 128, 128, 28928, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 400, 2, 0, 0, 0, 0, 4049, 0, 4096, 0, 4082, 0, 6, 0, 0, 4091, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 3749, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 11, 0, 4095, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 4083, 0, 0, 0, 329, 48, 0, 12, 0, 0, 10, 10, 0, 3146, 10, 3995, 15, 4056, 11, 2363, 7, 0, 4085, 10, 9, 9, 360, 3360, 166, 5, 4085, 1, 0, 3957, 0, 12, 11, 47, 4085, 4090, 2991, 0, 4076, 4069, 0, 0, 10, 0, 15, 8, 0, 0, 4086, 13, 9, 695, 0, 0, 3129, 0, 4047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[10, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 277, 0, 0, 0, 0, 0, 4055, 0, 4096, 0, 4090, 0, 9, 0, 0, 4085, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4057, 3895, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 27, 0, 4096, 0, 4072] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 4093, 1, 0, 0, 101, 0, 0, 35, 0, 0, 2, 3, 0, 3437, 3, 4093, 3, 4091, 1, 1109, 3, 1, 4086, 2, 2, 2, 58, 4002, 24, 2, 4094, 2, 0, 4060, 0, 0, 3, 14, 4093, 4094, 3842, 0, 4091, 4091, 0, 1, 3, 0, 4, 2, 0, 0, 4093, 3, 2, 74, 3, 0, 3719, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[19, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 1597, 9, 0, 0, 0, 0, 3992, 0, 4096, 0, 4060, 0, 180, 0, 0, 4072, 0, 0, 0, 0, 4023, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4094, 4056, 2523, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 43, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 0, 0, 4033, 0, 4096, 0, 4071, 0, 43, 0, 1, 4094, 0, 0, 0, 0, 4082, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 3796, 0, 4096, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[29, 0, 0, 4096, 0, 0, 10, 0, 0, 8, 396, 1, 0, 0, 0, 0, 3761, 0, 4096, 0, 4062, 0, 378, 0, 0, 4086, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4071, 4069, 3679, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 0, 0, 0, 30, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 14, 0, 0, 16, 1992, 7, 0, 0, 0, 0, 2699, 0, 4096, 0, 3997, 0, 137, 0, 0, 4082, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4072, 3445, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 36, 0, 4092, 0, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4083, 0, 0, 0, 329, 3395, 619, 0, 0, 0, 0, 0, 0, 48, 0, 12, 0, 0, 10, 10, 0, 4076, 3433, 0, 0, 0, 0, 0, 0, 3146, 10, 0, 15, 4004, 11, 2363, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 10, 9, 9, 360, 3360, 166, 33, 12, 0, 0, 0, 0, 0, 0, 5, 4085, 1, 0, 3957, 0, 12, 11, 19, 0, 0, 0, 0, 0, 0, 0, 47, 617, 4090, 2991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 15, 8, 0, 0, 4086, 3376, 636, 0, 0, 0, 0, 0, 0, 13, 9, 695, 0, 0, 3129, 0, 4047, 61, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8427, 7589, 9556, 8044, 8090, 7745, 8131, 7954] avg=8192.0000 max=9556 max_rank=2 min=7589 min_rank=1 imbalance_ratio=1.166504 max_load_reduction=55980 max_load_reduction_pct=0.854187 moved_to_echo=31920 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4, 0, 3404, 633, 0, 0, 0, 0, 0, 0, 0, 0, 400, 2, 0, 0, 0, 0, 4096, 3455, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 4030, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4093, 0, 33, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 619, 4088, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 4095, 3463, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4095, 0, 4096, 61, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 7953, 8085, 8229, 8211, 8456, 8202, 8263] avg=8192.0000 max=8456 max_rank=5 min=7953 min_rank=1 imbalance_ratio=1.032227 max_load_reduction=57080 max_load_reduction_pct=0.870972 moved_to_echo=33132 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 0, 0, 3404, 633, 0, 0, 0, 0, 0, 0, 0, 0, 349, 0, 0, 0, 0, 0, 4096, 3455, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 4019, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 4082, 0, 33, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 619, 4096, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 3463, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 61, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 7900, 8095, 8222, 8210, 8511, 8207, 8254] avg=8192.0000 max=8511 max_rank=5 min=7900 min_rank=1 imbalance_ratio=1.038940 max_load_reduction=57025 max_load_reduction_pct=0.870132 moved_to_echo=33182 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 0, 4096, 0, 0, 3, 0, 3403, 633, 0, 0, 0, 0, 0, 0, 0, 0, 1597, 9, 0, 0, 0, 0, 4096, 3455, 0, 0, 0, 0, 0, 0, 3992, 0, 0, 0, 4008, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 4023, 0, 33, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 618, 4056, 2523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 3463, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 4096, 0, 4096, 61, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 9157, 8180, 8140, 8211, 7197, 8201, 8296] avg=8192.0000 max=9157 max_rank=1 min=7197 min_rank=5 imbalance_ratio=1.117798 max_load_reduction=56379 max_load_reduction_pct=0.860275 moved_to_echo=31905 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 4096, 0, 0, 0, 0, 3404, 634, 0, 0, 0, 0, 0, 0, 0, 1, 277, 0, 0, 0, 0, 0, 4096, 3456, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 4038, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 4094, 0, 33, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 619, 4057, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3462, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 0, 4072, 61, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 7830, 8102, 8224, 8211, 8571, 8198, 8256] avg=8192.0000 max=8571 max_rank=5 min=7830 min_rank=1 imbalance_ratio=1.046265 max_load_reduction=56965 max_load_reduction_pct=0.869217 moved_to_echo=33236 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4093, 1, 0, 0, 101, 3402, 634, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 2, 3, 0, 4091, 3452, 0, 0, 0, 0, 0, 0, 3437, 3, 0, 3, 4039, 1, 1109, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 2, 2, 2, 58, 4002, 24, 33, 12, 0, 0, 0, 0, 0, 0, 2, 4094, 2, 0, 4060, 0, 0, 3, 19, 0, 0, 0, 0, 0, 0, 0, 14, 618, 4094, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4, 2, 0, 0, 4093, 3459, 639, 0, 0, 0, 0, 0, 0, 3, 2, 74, 3, 0, 3719, 0, 4093, 61, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 7583, 8595, 8222, 8180, 8568, 8201, 7955] avg=8192.0000 max=8595 max_rank=2 min=7583 min_rank=1 imbalance_ratio=1.049194 max_load_reduction=56941 max_load_reduction_pct=0.868851 moved_to_echo=32904 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 0, 4096, 0, 0, 10, 0, 3383, 634, 0, 0, 0, 0, 0, 0, 0, 8, 396, 1, 0, 0, 0, 0, 4096, 3456, 0, 0, 0, 0, 0, 0, 3761, 0, 0, 0, 4010, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4092, 0, 33, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 616, 4069, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4093, 3462, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4096, 0, 4096, 60, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 7957, 8149, 8223, 8211, 8364, 8198, 8282] avg=8192.0000 max=8364 max_rank=5 min=7957 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=33055 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 14, 0, 3409, 634, 0, 0, 0, 0, 0, 0, 0, 16, 1992, 7, 0, 0, 0, 0, 4096, 3456, 0, 0, 0, 0, 0, 0, 2699, 0, 0, 0, 3938, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4094, 0, 39, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 607, 4072, 3445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3462, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4092, 0, 4085, 63, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 9567, 6774, 8232, 8212, 8124, 8198, 8276] avg=8192.0000 max=9567 max_rank=1 min=6774 min_rank=2 imbalance_ratio=1.167847 max_load_reduction=55969 max_load_reduction_pct=0.854019 moved_to_echo=32810 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 4992, 32640, 28032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 128, 512, 27264, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[29184, 128, 0, 128, 32128, 128, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 32640, 0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 5120, 128, 0, 128, 128, 0, 32768, 27648, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 32768, 27648, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 896, 256, 0, 31488, 0, 32768, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 512, 31872, 256, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4085, 0, 4087, 3936, 8, 5, 69, 9, 114, 4086, 3938, 10, 0, 74, 10, 3859, 3287, 2, 10, 11, 9, 0, 9, 4084, 10, 9, 10, 0, 7, 0, 4023, 4074, 0, 7, 4086, 72, 9, 0, 221, 0, 0, 0, 10, 4076, 0, 10, 0, 637, 0, 72, 4051, 187, 0, 2, 7, 0, 0, 7, 4080, 0, 4085, 0, 1, 4081] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3877, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 93, 0, 4095, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 118, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 6, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4096, 3571, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 4095, 224, 0, 4091, 0, 0, 0, 68, 0, 0, 0, 0, 4096, 0, 0, 2, 233, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4093, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3657, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 207, 0, 4093, 0, 0, 0, 102, 0, 0, 0, 0, 4096, 0, 0, 0, 123, 0, 0, 4096, 0, 0, 13, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4089, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3492, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 4096, 4092, 383, 0, 4084, 0, 0, 0, 15, 0, 0, 0, 0, 4094, 0, 0, 0, 179, 0, 0, 4096, 0, 0, 38, 0, 0, 0, 0, 4096, 7, 4096, 3, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4089, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 4096, 3897, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4095, 127, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 49, 0, 0, 4096, 0, 0, 25, 0, 0, 0, 0, 4096, 5, 4096, 0, 1, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4093, 0, 4093, 3988, 5, 1, 22, 3, 85, 4093, 4061, 3, 0, 11, 5, 4069, 3515, 1, 3, 3, 3, 1, 2, 4093, 13, 2, 3, 0, 2, 0, 4087, 4093, 7, 2, 4093, 3, 3, 1, 244, 0, 0, 0, 2, 4091, 0, 2, 0, 310, 4, 7, 4090, 9, 0, 36, 1, 0, 0, 2, 4090, 0, 4093, 0, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4086, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 4096, 3920, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4094, 76, 0, 4094, 0, 0, 0, 16, 0, 0, 0, 0, 4096, 0, 0, 1, 43, 0, 0, 4096, 0, 0, 43, 0, 0, 0, 0, 4096, 10, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 4087, 3936, 8, 5, 69, 9, 0, 0, 0, 0, 0, 0, 0, 0, 114, 19, 3938, 10, 0, 74, 10, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 3287, 2, 10, 11, 9, 0, 9, 4084, 438, 0, 0, 0, 0, 0, 0, 0, 10, 9, 10, 0, 7, 0, 4023, 4074, 5, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4086, 72, 9, 0, 221, 0, 3617, 238, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4076, 0, 10, 0, 637, 3829, 48, 0, 0, 0, 0, 0, 0, 0, 72, 4051, 187, 0, 2, 7, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4080, 0, 0, 0, 1, 4081, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8024, 7850, 8138, 8250, 8610, 8356, 8169] avg=8192.0000 max=8610 max_rank=5 min=7850 min_rank=2 imbalance_ratio=1.051025 max_load_reduction=56926 max_load_reduction_pct=0.868622 moved_to_echo=33385 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 0, 0, 0, 0, 0, 0, 4096, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 224, 0, 4091, 0, 0, 0, 68, 0, 3627, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 233, 3837, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8211, 8106, 8197, 8249, 8216, 8149, 8193] avg=8192.0000 max=8249 max_rank=4 min=8106 min_rank=2 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 0, 0, 0, 4096, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 93, 0, 4095, 0, 0, 0, 3, 0, 3627, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 118, 3838, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8211, 8412, 8197, 8057, 8100, 8145, 8198] avg=8192.0000 max=8412 max_rank=2 min=8057 min_rank=4 imbalance_ratio=1.026855 max_load_reduction=57124 max_load_reduction_pct=0.871643 moved_to_echo=32500 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 0, 0, 0, 0, 0, 0, 4096, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 127, 0, 4096, 0, 0, 0, 0, 0, 3627, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 49, 3836, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 25, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8211, 8432, 8198, 8089, 8029, 8169, 8198] avg=8192.0000 max=8432 max_rank=2 min=8029 min_rank=5 imbalance_ratio=1.029297 max_load_reduction=57104 max_load_reduction_pct=0.871338 moved_to_echo=32485 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3657, 0, 0, 0, 0, 0, 0, 4096, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 207, 0, 4093, 0, 0, 0, 102, 0, 3627, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 123, 3838, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 13, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8211, 8192, 8197, 8268, 8105, 8157, 8193] avg=8192.0000 max=8268 max_rank=4 min=8105 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[26, 0, 4093, 3988, 5, 1, 22, 3, 0, 0, 0, 0, 0, 0, 0, 0, 85, 19, 4061, 3, 0, 11, 5, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 3515, 1, 3, 3, 3, 1, 2, 4093, 438, 0, 0, 0, 0, 0, 0, 0, 13, 2, 3, 0, 2, 0, 4087, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 7, 2, 4093, 3, 3, 1, 244, 0, 3624, 239, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4091, 0, 2, 0, 310, 3835, 48, 0, 0, 0, 0, 0, 0, 4, 7, 4090, 9, 0, 36, 1, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8253, 8059, 8205, 8216, 8288, 8192, 8185] avg=8192.0000 max=8288 max_rank=5 min=8059 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32881 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3492, 0, 0, 0, 0, 0, 0, 4096, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 383, 0, 4084, 0, 0, 0, 15, 0, 3627, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 179, 3838, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 38, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8211, 8027, 8197, 8348, 8159, 8182, 8202] avg=8192.0000 max=8348 max_rank=4 min=8027 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32891 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 0, 0, 0, 0, 0, 0, 4096, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4094, 10, 5, 0, 0, 0, 0, 0, 0, 76, 0, 4094, 0, 0, 0, 16, 0, 3631, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 43, 3840, 49, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 43, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8204, 8458, 8207, 8059, 8029, 8186, 8198] avg=8192.0000 max=8458 max_rank=2 min=8029 min_rank=5 imbalance_ratio=1.032471 max_load_reduction=57078 max_load_reduction_pct=0.870941 moved_to_echo=32472 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 0, 32768, 32512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 256, 32640, 128, 0, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 256, 0, 256, 128, 0, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1152, 128, 32768, 128, 128, 128, 768, 0, 29056, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 32768, 0, 128, 128, 1792, 30720, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[29312, 128, 128, 128, 128, 128, 128, 32768, 3584, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4080, 0, 0, 0, 4018, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 12, 64, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[3833, 0, 285, 8, 9, 0, 4085, 4089, 169, 12, 7, 0, 0, 0, 9, 9, 0, 1, 3501, 4087, 10, 680, 0, 4086, 0, 7, 4036, 4081, 0, 0, 4083, 40, 4084, 4086, 0, 0, 0, 0, 0, 11, 12, 656, 0, 68, 0, 7, 0, 0, 4079, 4084, 0, 572, 9, 3911, 0, 0, 3251, 11, 12, 3531, 15, 10, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3982, 4091, 0, 0, 0, 4094, 1, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 4, 106, 0, 1, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 3, 57, 0, 0, 0, 0, 0, 27, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4001, 0, 110, 2, 2, 0, 4093, 4094, 101, 3, 2, 0, 0, 0, 3, 2, 0, 1, 3910, 4092, 3, 208, 2, 4093, 0, 2, 4083, 4092, 0, 0, 4093, 10, 4092, 4093, 0, 2, 0, 0, 1, 3, 3, 124, 3, 16, 0, 3, 0, 0, 4091, 4093, 0, 215, 2, 4057, 0, 0, 3872, 3, 3, 3852, 3, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3800, 4096, 0, 0, 0, 4092, 1, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 64, 148, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4094, 4092, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3831, 4096, 0, 0, 0, 4079, 3, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 31, 233, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4093, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 4093, 0, 0, 0, 4092, 6, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 4, 80, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 2, 3, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3833, 0, 285, 8, 9, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 169, 12, 7, 0, 0, 0, 9, 9, 4084, 4050, 0, 0, 0, 0, 0, 0, 0, 1, 3501, 111, 10, 680, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4036, 4081, 0, 0, 3, 40, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 3987, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 12, 656, 0, 68, 0, 7, 0, 0, 4080, 3922, 35, 0, 0, 0, 0, 0, 4011, 0, 0, 572, 9, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3251, 11, 12, 3531, 15, 10, 0, 0, 99, 68, 54, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8340, 8389, 8167, 8082, 8780, 8503, 7055] avg=8192.0000 max=8780 max_rank=5 min=7055 min_rank=7 imbalance_ratio=1.071777 max_load_reduction=56756 max_load_reduction_pct=0.866028 moved_to_echo=32420 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 112, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3997, 3, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4, 0, 0, 0, 0, 0, 4093, 3930, 36, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 99, 69, 54, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8159, 8210, 8195, 8153, 8090, 8123, 8417] avg=8192.0000 max=8417 max_rank=7 min=8090 min_rank=5 imbalance_ratio=1.027466 max_load_reduction=57119 max_load_reduction_pct=0.871567 moved_to_echo=32783 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 111, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3997, 4, 106, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4093, 3926, 36, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 99, 69, 54, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8156, 8187, 8196, 8204, 8066, 8123, 8418] avg=8192.0000 max=8418 max_rank=7 min=8066 min_rank=5 imbalance_ratio=1.027588 max_load_reduction=57118 max_load_reduction_pct=0.871552 moved_to_echo=32811 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 111, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3997, 12, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 4093, 3915, 36, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 99, 69, 54, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8156, 8188, 8195, 8169, 8101, 8123, 8418] avg=8192.0000 max=8418 max_rank=7 min=8101 min_rank=5 imbalance_ratio=1.027588 max_load_reduction=57118 max_load_reduction_pct=0.871552 moved_to_echo=32811 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4058, 0, 0, 0, 0, 0, 0, 1, 0, 3800, 112, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3997, 64, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 2, 4093, 3930, 36, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 99, 69, 54, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8154, 8005, 8196, 8305, 8149, 8123, 8418] avg=8192.0000 max=8418 max_rank=7 min=8005 min_rank=2 imbalance_ratio=1.027588 max_load_reduction=57118 max_load_reduction_pct=0.871552 moved_to_echo=32995 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4001, 0, 110, 2, 2, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 101, 3, 2, 0, 0, 0, 3, 2, 4093, 4058, 0, 0, 0, 0, 0, 0, 0, 1, 3910, 111, 3, 208, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4083, 4092, 0, 0, 3, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3994, 0, 2, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 124, 3, 16, 0, 3, 0, 0, 4090, 3927, 35, 0, 0, 0, 0, 0, 4023, 0, 0, 215, 2, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 3, 3, 3852, 3, 3, 0, 0, 99, 68, 54, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8262, 8328, 8190, 8092, 8201, 8297, 7961] avg=8192.0000 max=8328 max_rank=2 min=7961 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32551 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 111, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3997, 4, 80, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 3, 4093, 3928, 36, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 99, 69, 54, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8153, 8215, 8201, 8180, 8064, 8123, 8418] avg=8192.0000 max=8418 max_rank=7 min=8064 min_rank=5 imbalance_ratio=1.027588 max_load_reduction=57118 max_load_reduction_pct=0.871552 moved_to_echo=32785 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 3831, 104, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3991, 31, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4, 4095, 3934, 35, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 105, 71, 58, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8156, 8014, 8196, 8351, 8085, 8121, 8431] avg=8192.0000 max=8431 max_rank=7 min=8014 min_rank=2 imbalance_ratio=1.029175 max_load_reduction=57105 max_load_reduction_pct=0.871353 moved_to_echo=32988 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[31744, 128, 128, 32000, 128, 128, 0, 0, 896, 640, 512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 32768, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32512, 0, 512, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 1024, 128, 128, 0, 128, 128, 128, 32768, 31488, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32000, 128, 768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32256, 0, 0, 896, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 128, 128, 0, 0, 128, 128, 128, 32768, 32512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 31104, 896, 128, 896, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[118, 4096, 0, 823, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4091, 739, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 4096, 4095, 0, 3061, 0, 4077, 0, 0, 0, 0, 0, 0, 117, 0, 4096, 0, 3356, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[2065, 4096, 1, 638, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4091, 700, 4096, 0, 0, 0, 4089, 0, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4095, 4096, 0, 3659, 0, 4091, 0, 0, 0, 0, 0, 0, 66, 0, 4096, 0, 1072, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[463, 4096, 0, 1168, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4089, 2771, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4096, 4096, 0, 3480, 0, 4083, 0, 0, 0, 0, 0, 0, 28, 0, 4096, 0, 298, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1405, 4096, 0, 747, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 810, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 3826, 0, 4094, 0, 0, 0, 0, 0, 0, 60, 0, 4096, 0, 1345, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[68, 4096, 0, 1558, 0, 21, 0, 0, 2, 0, 0, 0, 0, 0, 4091, 1, 4087, 490, 4096, 0, 0, 0, 4066, 0, 0, 4096, 8, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 4096, 5, 4094, 4095, 0, 2972, 0, 4081, 0, 0, 0, 0, 0, 0, 62, 0, 4096, 0, 3067, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4017, 0, 4079, 10, 0, 6, 0, 0, 10, 0, 0, 0, 0, 4090, 748, 4088, 16, 4072, 0, 1, 0, 4086, 0, 11, 4084, 0, 0, 10, 77, 4076, 4083, 0, 11, 0, 19, 8, 2, 0, 2, 0, 4086, 3985, 0, 4083, 3697, 10, 3948, 95, 3864, 0, 13, 3, 4, 5, 0, 0, 6, 4085, 15, 2, 0, 21, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[422, 4088, 0, 3145, 3, 0, 2, 2, 0, 8, 0, 0, 0, 0, 4092, 288, 4094, 351, 4092, 0, 0, 0, 4093, 0, 3, 4093, 0, 0, 2, 19, 4091, 4093, 0, 3, 0, 4, 3, 0, 0, 4, 1, 4093, 4073, 0, 4092, 4050, 3, 3231, 33, 3907, 0, 4, 3, 0, 2, 0, 0, 2, 4093, 3, 943, 0, 6, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[393, 4096, 0, 840, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4080, 943, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 4096, 4095, 0, 3438, 0, 4074, 0, 0, 0, 0, 0, 0, 39, 0, 4096, 0, 2567, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2065, 4096, 1, 638, 0, 9, 0, 0, 1009, 839, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 4091, 700, 0, 0, 0, 0, 3250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 700, 1, 0, 4096, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4095, 42, 4, 0, 0, 0, 0, 0, 66, 0, 4096, 0, 1072, 0, 0, 0, 2387, 84, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8668, 8064, 8041, 8182, 8188, 8456, 8232, 7705] avg=8192.0000 max=8668 max_rank=0 min=7705 min_rank=7 imbalance_ratio=1.058105 max_load_reduction=56868 max_load_reduction_pct=0.867737 moved_to_echo=32581 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[118, 4096, 0, 823, 0, 3, 0, 0, 1009, 841, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 4091, 739, 0, 0, 0, 0, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 700, 5, 0, 4095, 0, 3061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 4096, 40, 4, 0, 0, 0, 0, 0, 117, 0, 4096, 0, 3356, 0, 0, 0, 2387, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6901, 8061, 8085, 8181, 8188, 7861, 8217, 10042] avg=8192.0000 max=10042 max_rank=7 min=6901 min_rank=0 imbalance_ratio=1.225830 max_load_reduction=55494 max_load_reduction_pct=0.846771 moved_to_echo=34308 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4017, 0, 4079, 10, 0, 6, 0, 982, 839, 11, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4090, 748, 3948, 0, 0, 0, 0, 0, 0, 0, 4088, 16, 0, 0, 1, 0, 3247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 10, 77, 4076, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 19, 8, 2, 0, 2, 4084, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 681, 0, 0, 3697, 10, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 95, 3864, 0, 13, 3, 4, 5, 0, 4083, 41, 3, 0, 0, 0, 0, 0, 0, 6, 4085, 15, 2, 0, 21, 8, 2322, 83, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9944, 8796, 7352, 8246, 8209, 8336, 8111, 6542] avg=8192.0000 max=9944 max_rank=0 min=6542 min_rank=7 imbalance_ratio=1.213867 max_load_reduction=55592 max_load_reduction_pct=0.848267 moved_to_echo=31198 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[463, 4096, 0, 1168, 0, 3, 0, 0, 1009, 841, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 4089, 2771, 0, 0, 0, 0, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 700, 2, 0, 4096, 0, 3480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4096, 42, 4, 0, 0, 0, 0, 0, 28, 0, 4096, 0, 298, 0, 0, 0, 2387, 84, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7591, 8065, 10115, 8181, 8188, 8278, 8225, 6893] avg=8192.0000 max=10115 max_rank=2 min=6893 min_rank=7 imbalance_ratio=1.234741 max_load_reduction=55421 max_load_reduction_pct=0.845657 moved_to_echo=31584 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[68, 4096, 0, 1558, 0, 21, 0, 0, 1009, 834, 11, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4091, 1, 3970, 0, 0, 0, 0, 0, 0, 0, 4087, 490, 0, 0, 0, 0, 3232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 700, 5, 0, 4095, 0, 2972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 4094, 42, 3, 0, 0, 0, 0, 0, 62, 0, 4096, 0, 3067, 0, 0, 0, 2387, 84, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7597, 8064, 7809, 8189, 8189, 7772, 8220, 9696] avg=8192.0000 max=9696 max_rank=7 min=7597 min_rank=0 imbalance_ratio=1.183594 max_load_reduction=55840 max_load_reduction_pct=0.852051 moved_to_echo=33877 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[422, 4088, 0, 3145, 3, 0, 2, 2, 1003, 840, 11, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4092, 288, 3968, 0, 0, 0, 0, 0, 0, 0, 4094, 351, 0, 0, 0, 0, 3253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 19, 4091, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 3, 0, 0, 4, 4093, 4090, 0, 0, 0, 0, 0, 0, 1, 0, 696, 0, 0, 4050, 3, 3231, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3907, 0, 4, 3, 0, 2, 0, 4092, 40, 3, 0, 0, 0, 0, 0, 0, 2, 4093, 3, 943, 0, 6, 2, 2374, 84, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9516, 8356, 7698, 8197, 8197, 7981, 8084, 7507] avg=8192.0000 max=9516 max_rank=0 min=7507 min_rank=7 imbalance_ratio=1.161621 max_load_reduction=56020 max_load_reduction_pct=0.854797 moved_to_echo=31769 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[393, 4096, 0, 840, 0, 12, 0, 0, 1013, 843, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 4080, 943, 0, 0, 0, 0, 3252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 692, 5, 0, 4095, 0, 3438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 4096, 40, 3, 0, 0, 0, 0, 0, 39, 0, 4096, 0, 2567, 0, 0, 0, 2391, 84, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7210, 8063, 8275, 8179, 8189, 8230, 8213, 9177] avg=8192.0000 max=9177 max_rank=7 min=7210 min_rank=0 imbalance_ratio=1.120239 max_load_reduction=56359 max_load_reduction_pct=0.859970 moved_to_echo=33809 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1405, 4096, 0, 747, 0, 0, 0, 0, 1009, 841, 11, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 4096, 810, 0, 0, 0, 0, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 700, 0, 0, 4096, 0, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 40, 4, 0, 0, 0, 0, 0, 60, 0, 4096, 0, 1345, 0, 0, 0, 2387, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8067, 8161, 8181, 8188, 8622, 8234, 7974] avg=8192.0000 max=8622 max_rank=5 min=7974 min_rank=7 imbalance_ratio=1.052490 max_load_reduction=56914 max_load_reduction_pct=0.868439 moved_to_echo=33018 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 5632, 128, 0, 32384, 128, 27648, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32384, 0, 128, 128, 128, 128, 0, 32768, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 6912, 0, 0, 128, 0, 26112, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4992, 32768, 128, 13056, 128, 128, 128, 128, 8064, 6784, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 128, 32768, 128, 12672, 0, 128, 128, 19072, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 0, 32768, 1152, 31744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 1, 0, 0, 0, 1047, 0, 1, 0, 0, 0, 0, 1, 0, 2, 0, 1, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4090, 3075, 0, 4095, 4095, 4096, 0, 4088, 4096, 0, 1, 4095, 4096, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4090, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[10, 0, 11, 417, 0, 0, 12, 0, 3462, 0, 11, 0, 0, 346, 0, 10, 26, 0, 9, 119, 9, 11, 0, 18, 341, 14, 2, 4037, 9, 0, 9, 4086, 26, 0, 0, 3984, 0, 5, 4084, 4078, 57, 4085, 4067, 3968, 7, 3710, 4082, 0, 0, 4069, 4028, 0, 0, 46, 11, 0, 12, 4071, 4080, 4, 0, 4085, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 1822, 0, 0, 0, 0, 0, 0, 1, 0, 86, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4081, 2220, 0, 4096, 4096, 4096, 0, 4087, 4096, 0, 0, 4095, 4096, 9, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 1, 0, 0, 0, 1426, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 4073, 2707, 0, 4096, 4093, 4096, 0, 4086, 4096, 0, 6, 4093, 4096, 9, 0, 0, 0, 4, 0, 4093, 4096, 0, 5, 4090, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 2423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 1741, 0, 4096, 4096, 4096, 0, 4037, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 1752, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 4096, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 4057, 2390, 0, 4096, 4095, 4096, 0, 4085, 4096, 0, 5, 4095, 4096, 12, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4091, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 489, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 3, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4030, 3665, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 11, 4093, 4096, 5, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[3, 0, 3, 162, 2, 1, 3, 0, 3788, 0, 3, 0, 0, 96, 0, 20, 6, 0, 2, 58, 2, 3, 0, 4, 208, 3, 12, 4080, 9, 0, 2, 4093, 7, 0, 0, 4065, 0, 2, 4052, 4070, 16, 4093, 4083, 4090, 0, 3951, 4086, 0, 0, 4067, 4090, 0, 0, 17, 3, 0, 3, 4086, 4093, 3, 0, 4092, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 3999, 30, 0, 0, 0, 0, 0, 4084, 0, 0, 1, 0, 0, 0, 1047, 2967, 76, 10, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4095, 4066, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4096, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 0, 0, 4090, 3075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8185, 8163, 8118, 8294, 8270, 8195, 8191] avg=8192.0000 max=8294 max_rank=4 min=8118 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32950 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 11, 417, 0, 0, 12, 0, 4075, 3972, 30, 0, 0, 0, 0, 0, 3462, 0, 11, 0, 0, 346, 0, 10, 2886, 76, 10, 0, 0, 0, 0, 0, 26, 0, 9, 119, 9, 11, 0, 18, 4085, 4052, 0, 0, 0, 0, 0, 0, 341, 14, 2, 4037, 9, 0, 9, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 1098, 0, 5, 4084, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 85, 3968, 7, 3710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4028, 0, 0, 46, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4071, 5, 4, 0, 4085, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8527, 6801, 8329, 8422, 9291, 7827, 8154, 8185] avg=8192.0000 max=9291 max_rank=4 min=6801 min_rank=1 imbalance_ratio=1.134155 max_load_reduction=56245 max_load_reduction_pct=0.858231 moved_to_echo=33457 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 4000, 30, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 489, 2967, 76, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 0, 0, 4030, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4093, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 7631, 8168, 8119, 8824, 8278, 8205, 8190] avg=8192.0000 max=8824 max_rank=4 min=7631 min_rank=1 imbalance_ratio=1.077148 max_load_reduction=56712 max_load_reduction_pct=0.865356 moved_to_echo=33497 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 4000, 30, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 2423, 2967, 76, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 0, 0, 4096, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 4096, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 9563, 8162, 8116, 6966, 8219, 8192, 8197] avg=8192.0000 max=9563 max_rank=1 min=6966 min_rank=4 imbalance_ratio=1.167358 max_load_reduction=55973 max_load_reduction_pct=0.854080 moved_to_echo=31574 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 4091, 3997, 30, 0, 0, 0, 0, 0, 4079, 0, 0, 1, 0, 0, 0, 1426, 2967, 76, 10, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 0, 0, 4073, 2707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4093, 4096, 9, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 5, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8559, 8165, 8115, 7909, 8268, 8208, 8193] avg=8192.0000 max=8559 max_rank=1 min=7909 min_rank=4 imbalance_ratio=1.044800 max_load_reduction=56977 max_load_reduction_pct=0.869400 moved_to_echo=32578 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 4000, 30, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 1822, 2967, 76, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 86, 0, 0, 4096, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 0, 0, 4081, 2220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8954, 8163, 8202, 7430, 8269, 8200, 8197] avg=8192.0000 max=8954 max_rank=1 min=7430 min_rank=4 imbalance_ratio=1.093018 max_load_reduction=56582 max_load_reduction_pct=0.863373 moved_to_echo=32096 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 162, 2, 1, 3, 0, 4088, 3987, 31, 0, 0, 0, 0, 0, 3788, 0, 3, 0, 0, 96, 0, 20, 2945, 76, 10, 0, 0, 0, 0, 0, 6, 0, 2, 58, 2, 3, 0, 4, 4093, 4055, 0, 0, 0, 0, 0, 0, 208, 3, 12, 4080, 9, 0, 2, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1120, 0, 2, 4052, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 86, 4090, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4090, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4086, 5, 3, 0, 4092, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8280, 6938, 8223, 8331, 9251, 8143, 8177, 8193] avg=8192.0000 max=9251 max_rank=4 min=6938 min_rank=1 imbalance_ratio=1.129272 max_load_reduction=56285 max_load_reduction_pct=0.858841 moved_to_echo=33764 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 4003, 31, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 1752, 2972, 79, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 0, 0, 4057, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 4096, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4092, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8905, 8163, 8113, 7571, 8262, 8205, 8190] avg=8192.0000 max=8905 max_rank=1 min=7571 min_rank=4 imbalance_ratio=1.087036 max_load_reduction=56631 max_load_reduction_pct=0.864120 moved_to_echo=32228 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 9088, 0, 128, 32640, 24064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 768, 32640, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 640, 128, 128, 128, 0, 32768, 32000, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[31872, 0, 128, 128, 0, 512, 0, 9088, 23680, 640, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 256, 128, 128, 128, 128, 32768, 32512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 128, 128, 32768, 128, 0, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1467, 0, 0, 0, 291, 4077, 3966, 0, 8, 24, 0, 4087, 21, 0, 3746, 9, 4084, 0, 344, 0, 1, 2721, 120, 3219, 0, 10, 0, 98, 18, 11, 4043, 10, 0, 0, 10, 17, 23, 0, 7, 12, 0, 2524, 0, 11, 15, 2, 3832, 4086, 12, 4080, 4075, 0, 10, 4083, 7, 0, 10, 4085, 0, 2262, 3992, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[100, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 780, 0, 12, 0, 0, 0, 0, 4007, 0, 3320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4074, 0, 4095, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[382, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3317, 0, 4, 0, 0, 0, 0, 3916, 0, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4045, 0, 3587, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[85, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 2102, 0, 0, 0, 0, 0, 0, 3970, 0, 2124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 4037, 0, 4068, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[118, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 7, 0, 3, 0, 520, 0, 0, 5, 0, 2, 0, 4077, 0, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 0, 4091, 4096, 0, 0, 4092, 0, 0, 0, 4071, 0, 4086, 4094, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[210, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1337, 0, 0, 0, 0, 0, 0, 4080, 0, 2684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 4057, 0, 4016, 4096, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1574, 2, 0, 0, 101, 4092, 4053, 0, 2, 5, 0, 4094, 5, 0, 3994, 3, 4093, 0, 77, 0, 1, 1948, 35, 2183, 0, 3, 0, 27, 1180, 3, 3281, 2, 0, 0, 3, 5, 5, 0, 2, 3, 0, 3245, 0, 3, 4, 0, 4019, 4093, 1, 4090, 4091, 0, 3, 4093, 0, 0, 3, 4091, 0, 2957, 4064, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[135, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1520, 0, 0, 0, 0, 0, 0, 4093, 0, 2712, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 4096, 0, 4094, 4096, 0, 0, 4094, 0, 0, 0, 3997, 0, 3935, 4096, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1467, 0, 0, 0, 291, 3546, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 24, 0, 4087, 21, 0, 3746, 9, 48, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 344, 0, 1, 2721, 120, 3219, 1058, 483, 0, 0, 0, 0, 0, 0, 0, 10, 0, 98, 18, 11, 4043, 10, 2093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 17, 23, 0, 7, 12, 4080, 3730, 345, 0, 0, 0, 0, 0, 0, 2524, 0, 11, 15, 2, 3832, 356, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4080, 4075, 0, 10, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4085, 0, 2262, 496, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9270, 7943, 12030, 6283, 8224, 6740, 8187, 6859] avg=8192.0000 max=12030 max_rank=2 min=6283 min_rank=3 imbalance_ratio=1.468506 max_load_reduction=53506 max_load_reduction_pct=0.816437 moved_to_echo=30010 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[100, 0, 0, 0, 0, 3563, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 780, 0, 12, 1085, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 3320, 0, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3739, 354, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4095, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7759, 8240, 6458, 9475, 8186, 8545, 8195, 8678] avg=8192.0000 max=9475 max_rank=3 min=6458 min_rank=2 imbalance_ratio=1.156616 max_load_reduction=56061 max_load_reduction_pct=0.855423 moved_to_echo=33604 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[85, 0, 0, 0, 0, 3563, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 2102, 0, 0, 1085, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 2124, 0, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3739, 354, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 4068, 509, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7744, 8240, 7770, 8242, 8184, 8546, 8195, 8615] avg=8192.0000 max=8615 max_rank=7 min=7744 min_rank=0 imbalance_ratio=1.051636 max_load_reduction=56921 max_load_reduction_pct=0.868546 moved_to_echo=33540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[210, 0, 0, 0, 0, 3563, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1337, 0, 0, 1085, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 2684, 0, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3739, 354, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 4016, 509, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7869, 8240, 7003, 8912, 8184, 8549, 8195, 8584] avg=8192.0000 max=8912 max_rank=3 min=7003 min_rank=2 imbalance_ratio=1.087891 max_load_reduction=56624 max_load_reduction_pct=0.864014 moved_to_echo=33512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[382, 0, 0, 0, 0, 3563, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3317, 0, 4, 1085, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 1236, 0, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3739, 354, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 0, 4096, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 3587, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8041, 8240, 8987, 7300, 8186, 8446, 8195, 8141] avg=8192.0000 max=8987 max_rank=2 min=7300 min_rank=3 imbalance_ratio=1.097046 max_load_reduction=56549 max_load_reduction_pct=0.862869 moved_to_echo=32968 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[118, 0, 0, 0, 0, 3563, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 3, 0, 520, 0, 0, 1085, 485, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 4077, 0, 3508, 0, 2147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3739, 354, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4086, 508, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7777, 8237, 6196, 9739, 8182, 8546, 8190, 8669] avg=8192.0000 max=9739 max_rank=3 min=6196 min_rank=2 imbalance_ratio=1.188843 max_load_reduction=55797 max_load_reduction_pct=0.851395 moved_to_echo=33587 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1574, 2, 0, 0, 101, 3560, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 4094, 5, 0, 3994, 3, 48, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 77, 0, 1, 1948, 35, 2183, 1077, 484, 0, 0, 0, 0, 0, 0, 0, 3, 0, 27, 1180, 3, 3281, 2, 2131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 5, 0, 2, 3, 4090, 3736, 351, 0, 0, 0, 0, 0, 0, 3245, 0, 3, 4, 0, 4019, 357, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 4091, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 0, 2957, 505, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9290, 8151, 9898, 6627, 8195, 7628, 8188, 7559] avg=8192.0000 max=9898 max_rank=2 min=6627 min_rank=3 imbalance_ratio=1.208252 max_load_reduction=55638 max_load_reduction_pct=0.848969 moved_to_echo=31570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[135, 0, 0, 0, 0, 3555, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 53, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1520, 0, 0, 1090, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2712, 0, 2153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 3744, 356, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 3935, 497, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7786, 8245, 7194, 8958, 8195, 8536, 8191, 8431] avg=8192.0000 max=8958 max_rank=3 min=7194 min_rank=2 imbalance_ratio=1.093506 max_load_reduction=56578 max_load_reduction_pct=0.863312 moved_to_echo=33353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 32384, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 25344, 128, 22912, 128, 17152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32512, 0, 29056, 4096, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 30336, 0, 128, 128, 128, 32512, 2944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4096, 128, 0, 0, 512, 28544, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 128, 32768, 29952, 2944, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 512, 128, 128, 14336, 256, 5504, 8704, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4071, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 26, 4095, 4087, 42, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4087, 4038, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 58, 4086, 4091, 10, 0, 4, 4096, 0, 0, 0, 0, 10, 0, 4092, 1, 4095, 0, 0, 0, 4096, 4091, 0, 0, 4096, 0, 0, 4, 4096, 0, 4093, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4059, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 84, 4091, 4095, 34, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 4094, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[3880, 4026, 212, 0, 0, 7, 0, 0, 13, 10, 12, 9, 10, 0, 4046, 66, 3217, 1, 8, 0, 4, 4088, 4083, 186, 0, 0, 4085, 18, 0, 0, 0, 0, 84, 4079, 0, 4086, 789, 13, 37, 4078, 4085, 357, 0, 3614, 0, 17, 0, 4002, 10, 4087, 0, 7, 9, 0, 4075, 0, 9, 11, 12, 0, 0, 0, 4083, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4087, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 30, 4095, 4096, 13, 0, 1, 4096, 0, 0, 0, 0, 3, 0, 4095, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 1, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4087, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 33, 4096, 4096, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[3990, 4058, 68, 0, 0, 2, 0, 0, 3, 2, 3, 3, 2, 0, 4085, 13, 3962, 0, 3, 0, 5, 4094, 4093, 15, 0, 0, 4093, 3, 5, 0, 0, 0, 37, 4087, 0, 4093, 225, 3, 10, 4092, 4093, 108, 0, 3905, 0, 4, 0, 4077, 2, 4094, 0, 2, 2, 0, 4091, 0, 3, 2, 7, 1, 0, 0, 4093, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4044, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 98, 4090, 4093, 79, 0, 2, 4096, 0, 0, 0, 0, 10, 0, 4090, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4, 4096, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4059, 4012, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4025, 58, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 84, 39, 4095, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 4060, 27, 3, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8179, 8348, 8195, 8050, 8230, 8192, 8189] avg=8192.0000 max=8348 max_rank=2 min=8050 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3880, 4026, 212, 0, 0, 7, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 13, 10, 12, 9, 10, 0, 4046, 66, 4022, 58, 0, 0, 0, 0, 0, 0, 3217, 1, 8, 0, 4, 39, 4083, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 18, 0, 0, 0, 0, 4049, 27, 4, 0, 0, 0, 0, 0, 84, 4079, 0, 0, 789, 13, 37, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 36, 357, 0, 3614, 0, 17, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4087, 0, 7, 9, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 11, 12, 0, 0, 0, 4083, 11, 4082, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8246, 7538, 8183, 8940, 8026, 8188, 8208] avg=8192.0000 max=8940 max_rank=4 min=7538 min_rank=2 imbalance_ratio=1.091309 max_load_reduction=56596 max_load_reduction_pct=0.863586 moved_to_echo=33362 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4087, 4061, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4029, 58, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 30, 39, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3, 4060, 27, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8183, 8273, 8190, 8051, 8229, 8191, 8189] avg=8192.0000 max=8273 max_rank=2 min=8051 min_rank=4 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32660 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4087, 4056, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4030, 58, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 33, 39, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4060, 27, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8184, 8279, 8186, 8053, 8228, 8192, 8189] avg=8192.0000 max=8279 max_rank=2 min=8053 min_rank=4 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4087, 4038, 0, 0, 0, 0, 0, 3, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4020, 58, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 58, 39, 4091, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 10, 4055, 27, 4, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4096, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8175, 8294, 8196, 8049, 8232, 8193, 8187] avg=8192.0000 max=8294 max_rank=2 min=8049 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4071, 4054, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4029, 58, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 26, 39, 4087, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 4060, 27, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8183, 8290, 8194, 8052, 8230, 8192, 8188] avg=8192.0000 max=8290 max_rank=2 min=8052 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3990, 4058, 68, 0, 0, 2, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 3, 2, 3, 3, 2, 0, 4085, 13, 4028, 58, 0, 0, 0, 0, 0, 0, 3962, 0, 3, 0, 5, 39, 4093, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 5, 0, 0, 0, 4057, 27, 4, 0, 0, 0, 0, 0, 37, 4087, 0, 0, 225, 3, 10, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 36, 108, 0, 3905, 0, 4, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 2, 2, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 7, 1, 0, 0, 4093, 3, 4089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8197, 8117, 8189, 8314, 8130, 8191, 8198] avg=8192.0000 max=8314 max_rank=4 min=8117 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32833 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4044, 3973, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4028, 65, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 98, 32, 4093, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 10, 4061, 30, 4, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 4096, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8189, 8397, 8203, 8027, 8231, 8197, 8188] avg=8192.0000 max=8397 max_rank=2 min=8027 min_rank=4 imbalance_ratio=1.025024 max_load_reduction=57139 max_load_reduction_pct=0.871872 moved_to_echo=32643 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 512, 0, 32128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31872, 128, 128, 0, 384, 384, 32768, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32384, 32384, 384, 0, 0, 128, 0, 128, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 1024, 128, 128, 31616, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 0, 128, 32512, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 128, 32256, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3332, 0, 0, 4096, 0, 3098, 0, 0, 3968, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1879, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 11, 4096, 0, 4095, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 1, 0, 3709, 0, 0, 4096, 0, 2340, 0, 0, 3323, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2915, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4095, 0, 0, 3828, 0, 0, 4096, 0, 2112, 0, 0, 4003, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2337, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 6, 4096, 0, 4095, 4096, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4090, 128, 2, 3111, 2, 92, 3950, 0, 3869, 3, 228, 761, 5, 0, 4, 0, 4093, 0, 7, 0, 2, 4091, 0, 3, 2, 0, 5, 4088, 0, 4085, 0, 3644, 0, 3, 2, 4094, 0, 16, 2, 0, 4, 4056, 3, 0, 0, 746, 4, 3, 26, 3, 4068, 0, 0, 4, 4093, 0, 0, 4073, 0, 3973, 4089, 0, 6, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4078, 313, 9, 2696, 6, 272, 3681, 0, 4062, 9, 496, 185, 17, 0, 0, 0, 4083, 0, 66, 0, 8, 4077, 0, 21, 17, 3, 17, 4067, 0, 3999, 0, 3264, 0, 10, 7, 4086, 0, 82, 1, 0, 12, 3803, 12, 0, 0, 1697, 13, 8, 127, 13, 4009, 0, 0, 18, 4084, 0, 0, 4008, 0, 3996, 4071, 0, 22, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4093, 0, 0, 3209, 0, 0, 4096, 0, 3397, 0, 1, 3711, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 1942, 0, 0, 0, 4094, 4, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 15, 13, 4096, 0, 4091, 4096, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 1, 0, 2765, 0, 0, 4096, 0, 3118, 0, 0, 3990, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2404, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 9, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3744, 0, 0, 4096, 0, 3051, 0, 0, 3868, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1606, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 19, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[765, 0, 0, 3332, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3098, 0, 0, 3968, 0, 0, 0, 0, 1973, 9, 8, 0, 0, 0, 0, 0, 4088, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1608, 0, 4096, 0, 1879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4066, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3312, 515, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 4095, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 9056, 8185, 7583, 8181, 7923, 8183, 8232] avg=8192.0000 max=9056 max_rank=1 min=7583 min_rank=3 imbalance_ratio=1.105469 max_load_reduction=56480 max_load_reduction_pct=0.861816 moved_to_echo=32519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[765, 1, 0, 3709, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2340, 0, 0, 3323, 0, 0, 0, 0, 1973, 9, 8, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1608, 0, 4096, 0, 2915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4066, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3312, 515, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8571, 7653, 8184, 8619, 8181, 7923, 8183, 8222] avg=8192.0000 max=8619 max_rank=3 min=7653 min_rank=1 imbalance_ratio=1.052124 max_load_reduction=56917 max_load_reduction_pct=0.868484 moved_to_echo=32509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[761, 313, 9, 2696, 6, 272, 3681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 9, 496, 185, 17, 0, 0, 0, 1959, 9, 8, 0, 0, 0, 0, 0, 4075, 0, 66, 0, 8, 4077, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3, 17, 1597, 0, 3999, 0, 3264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 7, 4086, 0, 82, 1, 0, 4041, 19, 0, 0, 0, 0, 0, 0, 12, 3803, 12, 0, 0, 1697, 13, 8, 3298, 511, 0, 0, 0, 0, 0, 0, 127, 13, 4009, 0, 0, 18, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 3996, 30, 0, 22, 11, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7738, 6745, 8247, 8897, 8246, 9354, 8242, 8067] avg=8192.0000 max=9354 max_rank=5 min=6745 min_rank=1 imbalance_ratio=1.141846 max_load_reduction=56182 max_load_reduction_pct=0.857269 moved_to_echo=33909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[765, 0, 0, 3828, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2112, 0, 0, 4003, 0, 0, 0, 0, 1973, 9, 8, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1608, 0, 4096, 0, 2337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4066, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3311, 515, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4095, 30, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8689, 8105, 8184, 8041, 8182, 7922, 8184, 8229] avg=8192.0000 max=8689 max_rank=0 min=7922 min_rank=5 imbalance_ratio=1.060669 max_load_reduction=56847 max_load_reduction_pct=0.867416 moved_to_echo=32517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[764, 128, 2, 3111, 2, 92, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3869, 3, 228, 761, 5, 0, 4, 0, 1969, 9, 8, 0, 0, 0, 0, 0, 4085, 0, 7, 0, 2, 4091, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 1605, 0, 4085, 0, 3644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 4094, 0, 16, 2, 0, 4059, 19, 0, 0, 0, 0, 0, 0, 4, 4056, 3, 0, 0, 746, 4, 3, 3307, 514, 0, 0, 0, 0, 0, 0, 26, 3, 4068, 0, 0, 4, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 3973, 30, 0, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8049, 6856, 8188, 9341, 8195, 8637, 8185, 8085] avg=8192.0000 max=9341 max_rank=3 min=6856 min_rank=1 imbalance_ratio=1.140259 max_load_reduction=56195 max_load_reduction_pct=0.857468 moved_to_echo=33102 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[765, 0, 0, 3744, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3051, 0, 0, 3868, 0, 0, 0, 0, 1973, 9, 8, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1608, 0, 4096, 0, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4066, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3312, 515, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8605, 8909, 8184, 7310, 8181, 7923, 8183, 8241] avg=8192.0000 max=8909 max_rank=1 min=7310 min_rank=3 imbalance_ratio=1.087524 max_load_reduction=56627 max_load_reduction_pct=0.864059 moved_to_echo=32528 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[764, 0, 0, 3209, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 1, 3711, 0, 0, 0, 0, 1973, 9, 8, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1608, 0, 4096, 2, 1942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 4066, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 3310, 515, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 15, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 4091, 30, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8069, 9099, 8184, 7648, 8183, 7922, 8198, 8233] avg=8192.0000 max=9099 max_rank=1 min=7648 min_rank=3 imbalance_ratio=1.110718 max_load_reduction=56437 max_load_reduction_pct=0.861160 moved_to_echo=32536 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[760, 1, 0, 2765, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3118, 0, 0, 3990, 0, 0, 0, 0, 1979, 14, 12, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1597, 0, 4096, 0, 2404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4070, 20, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3316, 520, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 26, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7622, 9113, 8180, 8097, 8187, 7932, 8178, 8227] avg=8192.0000 max=9113 max_rank=1 min=7622 min_rank=0 imbalance_ratio=1.112427 max_load_reduction=56423 max_load_reduction_pct=0.860947 moved_to_echo=32524 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 128, 32768, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6144, 512, 128, 26496, 128, 384, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 32640, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[25088, 128, 768, 23936, 128, 0, 128, 0, 15872, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32512, 128, 128, 0, 2560, 128, 128, 26496, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 12928, 0, 32768, 128, 20096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 128, 0, 32512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2160, 163, 1198, 598, 1192, 0, 1239, 433, 1723, 2909, 1815, 1005, 1410, 1811, 0, 457, 409, 1314, 1739, 353, 0, 1239, 1850, 1117, 65, 106, 2441, 848, 1329, 801, 2031, 985, 1, 2086, 1819, 470, 563, 917, 1342, 1149, 263, 1, 1068, 1817, 0, 1857, 953, 352, 1832, 41, 1545, 12, 165, 1595, 1, 792, 1, 1300, 2148, 797, 1597, 1150, 956, 2206] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2341, 168, 1225, 862, 1727, 0, 1293, 486, 951, 2420, 1578, 1128, 1323, 1405, 0, 293, 551, 1304, 1573, 794, 1, 1037, 2521, 1207, 186, 85, 2661, 862, 1647, 1170, 1792, 624, 5, 2040, 1682, 601, 719, 1020, 1398, 433, 284, 0, 1316, 1230, 0, 1846, 996, 1065, 2061, 85, 1484, 13, 198, 1135, 0, 1197, 0, 712, 1568, 1418, 1401, 1234, 890, 2290] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1957, 147, 1149, 679, 1460, 0, 1745, 362, 972, 2991, 1181, 1133, 1635, 1503, 0, 262, 567, 1149, 1719, 723, 0, 783, 2241, 1107, 92, 139, 2372, 1258, 1989, 1444, 1436, 1127, 3, 1594, 1875, 243, 719, 1486, 1019, 806, 308, 0, 1282, 1458, 0, 1741, 1532, 824, 2280, 79, 1315, 3, 311, 1933, 2, 970, 1, 1070, 1048, 662, 1692, 1121, 768, 2069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2169, 215, 1129, 404, 1705, 2, 1150, 822, 1233, 2713, 1546, 1569, 1634, 1366, 2, 247, 551, 1066, 1495, 910, 1, 808, 2436, 720, 175, 66, 2837, 882, 1704, 1171, 1817, 1091, 4, 1604, 1299, 424, 845, 1151, 1569, 247, 512, 2, 1210, 2167, 0, 2322, 988, 684, 2270, 29, 1352, 49, 394, 656, 4, 924, 3, 650, 1436, 763, 1542, 1121, 1611, 2068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1570, 128, 2152, 771, 1959, 1, 783, 519, 434, 1762, 1121, 1742, 2134, 1359, 0, 601, 1364, 1961, 2227, 865, 0, 840, 2171, 1526, 54, 9, 2554, 1834, 471, 582, 1256, 1137, 4, 1458, 2450, 38, 1613, 988, 1840, 407, 481, 0, 679, 1102, 0, 1246, 904, 1106, 2034, 32, 876, 6, 167, 787, 29, 1613, 8, 1067, 1853, 744, 1869, 1488, 688, 2072] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2587, 238, 655, 483, 1510, 0, 1153, 1108, 1409, 2741, 1294, 667, 913, 1932, 0, 515, 489, 954, 1574, 785, 0, 1218, 2320, 1113, 167, 232, 2034, 870, 1639, 1353, 2542, 724, 4, 1834, 1775, 850, 659, 1361, 1584, 1283, 495, 1, 1887, 1340, 1, 1776, 946, 833, 1847, 42, 1063, 52, 229, 1012, 4, 776, 0, 590, 1257, 411, 2184, 587, 1134, 2500] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2203, 101, 1490, 994, 1446, 0, 1620, 413, 846, 2981, 1140, 1602, 1814, 1364, 3, 338, 445, 1336, 1424, 694, 1, 860, 2349, 1065, 83, 224, 2345, 1162, 2082, 854, 1680, 1235, 2, 2131, 2097, 689, 912, 986, 1126, 881, 365, 0, 905, 1598, 0, 1869, 1105, 877, 1753, 51, 886, 1, 357, 1582, 2, 729, 7, 360, 1374, 475, 2091, 1575, 689, 1867] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2318, 158, 900, 760, 1712, 0, 1667, 259, 723, 2578, 1286, 1352, 1635, 1462, 1, 436, 392, 1482, 946, 644, 0, 774, 2488, 1062, 225, 156, 2263, 922, 1179, 779, 1651, 1057, 13, 2796, 1505, 253, 614, 1334, 1642, 953, 358, 1, 1412, 1283, 1, 1998, 1062, 630, 2454, 94, 1235, 23, 252, 1418, 2, 1257, 0, 1000, 1778, 1061, 2026, 726, 657, 2431] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2160, 163, 1198, 598, 1192, 0, 1239, 433, 203, 225, 0, 0, 0, 0, 0, 0, 1723, 1136, 1815, 1005, 1410, 1811, 0, 457, 0, 0, 0, 0, 0, 0, 0, 0, 409, 1314, 1739, 353, 0, 1239, 1537, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 65, 106, 1559, 848, 1329, 801, 2031, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1861, 1819, 470, 563, 917, 1342, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 263, 1, 1068, 1817, 0, 1857, 953, 352, 882, 383, 110, 0, 0, 0, 0, 0, 1832, 41, 1545, 12, 165, 1595, 1, 792, 1773, 665, 0, 0, 0, 0, 0, 0, 1, 1300, 2148, 797, 1597, 1150, 956, 1158, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7411, 9357, 7708, 7724, 8122, 7686, 8421, 9107] avg=8192.0000 max=9357 max_rank=1 min=7411 min_rank=0 imbalance_ratio=1.142212 max_load_reduction=56179 max_load_reduction_pct=0.857224 moved_to_echo=33336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2587, 238, 655, 483, 1510, 0, 1153, 1108, 254, 198, 0, 0, 0, 0, 0, 0, 1409, 1070, 1294, 667, 913, 1932, 0, 515, 0, 0, 0, 0, 0, 0, 0, 0, 489, 954, 1574, 785, 0, 1218, 1928, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 167, 232, 1299, 870, 1639, 1353, 2542, 724, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1636, 1775, 850, 659, 1361, 1584, 1283, 0, 0, 0, 0, 0, 0, 0, 0, 495, 1, 1887, 1340, 1, 1776, 946, 833, 735, 435, 138, 0, 0, 0, 0, 0, 1847, 42, 1063, 52, 229, 1012, 4, 776, 1671, 754, 0, 0, 0, 0, 0, 0, 0, 590, 1257, 411, 2184, 587, 1134, 1311, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 7800, 8061, 8826, 9152, 8587, 7450, 7474] avg=8192.0000 max=9152 max_rank=4 min=7450 min_rank=6 imbalance_ratio=1.117188 max_load_reduction=56384 max_load_reduction_pct=0.860352 moved_to_echo=32663 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2341, 168, 1225, 862, 1727, 0, 1293, 486, 276, 220, 0, 0, 0, 0, 0, 0, 951, 945, 1578, 1128, 1323, 1405, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 551, 1304, 1573, 794, 1, 1037, 2095, 1207, 0, 0, 0, 0, 0, 0, 0, 0, 186, 85, 1700, 862, 1647, 1170, 1792, 624, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1820, 1682, 601, 719, 1020, 1398, 433, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 1316, 1230, 0, 1846, 996, 1065, 961, 398, 150, 0, 0, 0, 0, 0, 2061, 85, 1484, 13, 198, 1135, 0, 1197, 1475, 690, 0, 0, 0, 0, 0, 0, 0, 712, 1568, 1418, 1401, 1234, 890, 1202, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8598, 7623, 8562, 8066, 7678, 8246, 8338, 8425] avg=8192.0000 max=8598 max_rank=0 min=7623 min_rank=1 imbalance_ratio=1.049561 max_load_reduction=56938 max_load_reduction_pct=0.868805 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1957, 147, 1149, 679, 1460, 0, 1745, 362, 246, 172, 0, 0, 0, 0, 0, 0, 972, 1168, 1181, 1133, 1635, 1503, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 567, 1149, 1719, 723, 0, 783, 1861, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 92, 139, 1515, 1258, 1989, 1444, 1436, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1422, 1875, 243, 719, 1486, 1019, 806, 0, 0, 0, 0, 0, 0, 0, 0, 308, 0, 1282, 1458, 0, 1741, 1532, 824, 857, 360, 134, 0, 0, 0, 0, 0, 2280, 79, 1315, 3, 311, 1933, 2, 970, 1823, 624, 0, 0, 0, 0, 0, 0, 1, 1070, 1048, 662, 1692, 1121, 768, 1085, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7917, 7854, 7909, 9000, 7573, 8496, 9340, 7447] avg=8192.0000 max=9340 max_rank=6 min=7447 min_rank=7 imbalance_ratio=1.140137 max_load_reduction=56196 max_load_reduction_pct=0.857483 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2169, 215, 1129, 404, 1705, 2, 1150, 822, 267, 173, 0, 0, 0, 0, 0, 0, 1233, 1059, 1546, 1569, 1634, 1366, 2, 247, 0, 0, 0, 0, 0, 0, 0, 0, 551, 1066, 1495, 910, 1, 808, 2024, 720, 0, 0, 0, 0, 0, 0, 0, 0, 175, 66, 1812, 882, 1704, 1171, 1817, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1431, 1299, 424, 845, 1151, 1569, 247, 0, 0, 0, 0, 0, 0, 0, 0, 512, 2, 1210, 2167, 0, 2322, 988, 684, 1025, 359, 145, 0, 0, 0, 0, 0, 2270, 29, 1352, 49, 394, 656, 4, 924, 1654, 623, 0, 0, 0, 0, 0, 0, 3, 650, 1436, 763, 1542, 1121, 1611, 1086, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8036, 8656, 7575, 8718, 6970, 9414, 7955, 8212] avg=8192.0000 max=9414 max_rank=5 min=6970 min_rank=4 imbalance_ratio=1.149170 max_load_reduction=56122 max_load_reduction_pct=0.856354 moved_to_echo=32551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1570, 128, 2152, 771, 1959, 1, 783, 519, 242, 159, 0, 0, 0, 0, 0, 0, 434, 683, 1121, 1742, 2134, 1359, 0, 601, 0, 0, 0, 0, 0, 0, 0, 0, 1364, 1961, 2227, 865, 0, 840, 1795, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 54, 9, 1627, 1834, 471, 582, 1256, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1299, 2450, 38, 1613, 988, 1840, 407, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 679, 1102, 0, 1246, 904, 1106, 927, 364, 134, 0, 0, 0, 0, 0, 2034, 32, 876, 6, 167, 787, 29, 1613, 1079, 628, 0, 0, 0, 0, 0, 0, 8, 1067, 1853, 744, 1869, 1488, 688, 1080, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8074, 10578, 6970, 8639, 6943, 7251, 8797] avg=8192.0000 max=10578 max_rank=2 min=6943 min_rank=5 imbalance_ratio=1.291260 max_load_reduction=54958 max_load_reduction_pct=0.838593 moved_to_echo=31630 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2203, 101, 1490, 994, 1446, 0, 1620, 413, 258, 230, 0, 0, 0, 0, 0, 0, 846, 1164, 1140, 1602, 1814, 1364, 3, 338, 0, 0, 0, 0, 0, 0, 0, 0, 445, 1336, 1424, 694, 1, 860, 1951, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 83, 224, 1498, 1162, 2082, 854, 1680, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1901, 2097, 689, 912, 986, 1126, 881, 0, 0, 0, 0, 0, 0, 0, 0, 365, 0, 905, 1598, 0, 1869, 1105, 877, 847, 324, 140, 0, 0, 0, 0, 0, 1753, 51, 886, 1, 357, 1582, 2, 729, 1817, 563, 0, 0, 0, 0, 0, 0, 7, 360, 1374, 475, 2091, 1575, 689, 980, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8755, 8271, 7776, 8818, 8594, 8030, 7741, 7551] avg=8192.0000 max=8818 max_rank=3 min=7551 min_rank=7 imbalance_ratio=1.076416 max_load_reduction=56718 max_load_reduction_pct=0.865448 moved_to_echo=31916 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2318, 158, 900, 760, 1712, 0, 1667, 259, 273, 302, 0, 0, 0, 0, 0, 0, 723, 1006, 1286, 1352, 1635, 1462, 1, 436, 0, 0, 0, 0, 0, 0, 0, 0, 392, 1482, 946, 644, 0, 774, 2067, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 225, 156, 1445, 922, 1179, 779, 1651, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2494, 1505, 253, 614, 1334, 1642, 953, 0, 0, 0, 0, 0, 0, 0, 0, 358, 1, 1412, 1283, 1, 1998, 1062, 630, 818, 423, 148, 0, 0, 0, 0, 0, 2454, 94, 1235, 23, 252, 1418, 2, 1257, 1572, 733, 0, 0, 0, 0, 0, 0, 0, 1000, 1778, 1061, 2026, 726, 657, 1275, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 7901, 7367, 7414, 8808, 8134, 9040, 8523] avg=8192.0000 max=9040 max_rank=6 min=7367 min_rank=2 imbalance_ratio=1.103516 max_load_reduction=56496 max_load_reduction_pct=0.862061 moved_to_echo=34505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[17408, 1408, 9984, 5632, 12800, 128, 10752, 4480, 2048, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3072, 128, 9856, 12032, 128, 14720, 8576, 6400, 7168, 3072, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1152, 1024, 12544, 8704, 12160, 8192, 14208, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 11008, 10240, 12544, 12288, 128, 3200, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 13952, 14592, 3584, 6656, 9344, 11520, 6272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[16640, 512, 9856, 256, 2176, 10240, 128, 8320, 12928, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4864, 10624, 12800, 5888, 128, 7680, 15360, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 6784, 12544, 6400, 14464, 9088, 7424, 9216, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 1, 236, 0, 2964, 1316, 0, 3672, 581, 0, 0, 794, 1974, 0, 4, 0, 0, 0, 4094, 4047, 27, 2, 0, 138, 0, 4090, 3639, 17, 4096, 0, 0, 0, 3721, 1, 0, 2996, 114, 4088, 1, 0, 0, 0, 0, 4096, 1274, 0, 1, 4095, 4094, 0, 4, 1, 2, 4084, 1092, 0, 0, 0, 1, 82] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4094, 0, 1404, 2, 3540, 673, 0, 3842, 280, 0, 0, 599, 1858, 0, 1, 1, 0, 2, 4094, 4072, 6, 1, 1, 26, 2, 4056, 3899, 0, 4095, 1, 9, 0, 2921, 5, 0, 988, 181, 4084, 0, 2, 0, 0, 0, 4096, 2449, 0, 0, 4083, 4089, 0, 6, 0, 10, 4094, 1899, 0, 0, 0, 0, 70] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 4095, 0, 1212, 0, 2591, 658, 0, 3650, 501, 0, 1, 731, 2732, 1, 0, 1, 1, 0, 4080, 4079, 6, 16, 3, 15, 0, 4038, 3999, 9, 4095, 0, 3, 1, 2817, 0, 0, 2030, 367, 4086, 1, 1, 0, 0, 2, 4095, 1625, 7, 0, 4071, 4091, 1, 15, 0, 8, 4095, 1669, 0, 1, 0, 1, 27] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4094, 0, 1137, 0, 2882, 445, 2, 3688, 591, 0, 0, 811, 2922, 0, 4, 0, 0, 0, 4095, 4066, 36, 4, 3, 174, 0, 4038, 3767, 2, 4096, 2, 1, 0, 3089, 1, 2, 2629, 228, 4092, 0, 0, 0, 0, 0, 4096, 392, 0, 0, 4096, 4096, 0, 7, 0, 0, 4095, 1660, 0, 0, 0, 0, 191] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 0, 1, 4095, 1, 232, 1, 2583, 297, 0, 3893, 500, 0, 12, 668, 3377, 1, 1, 1, 1, 0, 4095, 4071, 2, 1, 0, 32, 0, 4038, 3952, 1, 4095, 0, 2, 1, 2539, 2, 0, 2926, 390, 4087, 1, 0, 0, 0, 1, 4095, 961, 0, 0, 4090, 4095, 1, 7, 0, 3, 4095, 2155, 0, 1, 0, 1, 130] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 0, 321, 0, 2183, 1948, 0, 2646, 1234, 0, 0, 5, 1829, 0, 2, 1, 1, 0, 4084, 4061, 2, 134, 12, 1097, 0, 3887, 4093, 3, 4096, 1, 0, 0, 3608, 0, 0, 1763, 33, 4091, 0, 0, 0, 0, 0, 4095, 3291, 0, 1, 4094, 4096, 0, 5, 0, 1, 4095, 602, 1, 0, 1, 0, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4096, 6, 896, 0, 3358, 303, 0, 3253, 499, 0, 0, 982, 3108, 0, 0, 0, 0, 1, 4095, 4078, 29, 0, 2, 1, 0, 4014, 3926, 0, 4095, 0, 0, 0, 2625, 0, 1, 1677, 479, 4093, 0, 1, 2, 0, 0, 4096, 2525, 0, 0, 4096, 4095, 0, 11, 1, 0, 4087, 943, 0, 0, 1, 0, 59] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4096, 1, 784, 0, 2090, 513, 0, 2891, 583, 0, 0, 1073, 2562, 0, 0, 2, 1, 0, 4094, 3969, 4, 7, 0, 503, 1, 4086, 4068, 75, 4090, 20, 0, 0, 3472, 0, 0, 3174, 105, 4096, 1, 1, 0, 0, 0, 4094, 1994, 0, 0, 4095, 4096, 0, 0, 0, 0, 4087, 783, 0, 0, 2, 0, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 1, 236, 0, 2504, 811, 0, 0, 0, 0, 0, 0, 2964, 1316, 0, 3567, 581, 0, 0, 794, 0, 0, 0, 0, 0, 0, 0, 0, 1974, 0, 4, 0, 0, 0, 1590, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2, 0, 138, 0, 4034, 3639, 17, 0, 0, 0, 0, 0, 0, 0, 0, 2815, 0, 0, 0, 3721, 1, 0, 2996, 0, 0, 0, 0, 0, 0, 0, 0, 114, 4088, 1, 0, 0, 0, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 1274, 0, 1, 4095, 2281, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4084, 1092, 0, 0, 0, 1, 82, 1281, 1002, 231, 105, 56, 0, 0, 0] rerouted_tokens_per_ep_rank=[7649, 9222, 7615, 7857, 9533, 8068, 7656, 7936] avg=8192.0000 max=9533 max_rank=4 min=7615 min_rank=2 imbalance_ratio=1.163696 max_load_reduction=56003 max_load_reduction_pct=0.854538 moved_to_echo=33193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4094, 0, 1404, 2, 2504, 810, 0, 0, 0, 0, 0, 0, 3540, 673, 0, 3732, 280, 0, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 1858, 0, 1, 1, 0, 2, 1590, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 1, 26, 2, 4000, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2814, 1, 9, 0, 2921, 5, 0, 988, 0, 0, 0, 0, 0, 0, 0, 0, 181, 4084, 0, 2, 0, 0, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 2449, 0, 0, 4083, 2278, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4094, 1899, 0, 0, 0, 0, 70, 1281, 1001, 231, 110, 56, 0, 0, 0] rerouted_tokens_per_ep_rank=[8815, 8824, 7524, 7935, 6738, 8132, 8816, 8752] avg=8192.0000 max=8824 max_rank=1 min=6738 min_rank=4 imbalance_ratio=1.077148 max_load_reduction=56712 max_load_reduction_pct=0.865356 moved_to_echo=32438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 1, 4095, 1, 232, 1, 2505, 811, 0, 0, 0, 0, 0, 0, 2583, 297, 0, 3782, 500, 0, 12, 668, 0, 0, 0, 0, 0, 0, 0, 0, 3377, 1, 1, 1, 1, 0, 1590, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 32, 0, 3982, 3952, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2814, 0, 2, 1, 2539, 2, 0, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 390, 4087, 1, 0, 0, 0, 1, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 961, 0, 0, 4090, 2282, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 2155, 0, 1, 0, 1, 130, 1281, 1002, 231, 111, 56, 0, 0, 0] rerouted_tokens_per_ep_rank=[7648, 7842, 9042, 7970, 8284, 8343, 7341, 9066] avg=8192.0000 max=9066 max_rank=7 min=7341 min_rank=6 imbalance_ratio=1.106689 max_load_reduction=56470 max_load_reduction_pct=0.861664 moved_to_echo=33034 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4096, 6, 896, 0, 2505, 811, 0, 0, 0, 0, 0, 0, 3358, 303, 0, 3160, 499, 0, 0, 982, 0, 0, 0, 0, 0, 0, 0, 0, 3108, 0, 0, 0, 0, 1, 1590, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 2, 1, 0, 3959, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2814, 0, 0, 0, 2625, 0, 1, 1677, 0, 0, 0, 0, 0, 0, 0, 0, 479, 4093, 0, 1, 2, 0, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 2525, 0, 0, 4096, 2282, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 943, 0, 0, 1, 0, 59, 1281, 1002, 231, 93, 55, 0, 0, 0] rerouted_tokens_per_ep_rank=[8316, 8302, 8777, 7917, 7117, 8440, 8915, 7752] avg=8192.0000 max=8915 max_rank=6 min=7117 min_rank=4 imbalance_ratio=1.088257 max_load_reduction=56621 max_load_reduction_pct=0.863968 moved_to_echo=32224 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4096, 1, 784, 0, 2504, 812, 0, 0, 0, 0, 0, 0, 2090, 513, 0, 2809, 583, 0, 0, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 2562, 0, 0, 2, 1, 0, 1590, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 503, 1, 4030, 4068, 75, 0, 0, 0, 0, 0, 0, 0, 0, 2811, 20, 0, 0, 3472, 0, 0, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 105, 4096, 1, 1, 0, 0, 0, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 1994, 0, 0, 4095, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 783, 0, 0, 2, 0, 22, 1279, 1003, 231, 82, 56, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 7068, 8124, 8688, 9477, 8066, 8370, 7545] avg=8192.0000 max=9477 max_rank=4 min=7068 min_rank=1 imbalance_ratio=1.156860 max_load_reduction=56059 max_load_reduction_pct=0.855392 moved_to_echo=33458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4094, 0, 1137, 0, 2505, 812, 0, 0, 0, 0, 0, 0, 2882, 445, 2, 3583, 591, 0, 0, 811, 0, 0, 0, 0, 0, 0, 0, 0, 2922, 0, 4, 0, 0, 0, 1590, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4, 3, 174, 0, 3982, 3767, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2815, 2, 1, 0, 3089, 1, 2, 2629, 0, 0, 0, 0, 0, 0, 0, 0, 228, 4092, 0, 0, 0, 0, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 0, 4096, 2281, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1660, 0, 0, 0, 0, 191, 1281, 1003, 231, 105, 56, 0, 0, 0] rerouted_tokens_per_ep_rank=[8550, 8314, 8582, 7968, 8539, 8185, 6776, 8622] avg=8192.0000 max=8622 max_rank=7 min=6776 min_rank=6 imbalance_ratio=1.052490 max_load_reduction=56914 max_load_reduction_pct=0.868439 moved_to_echo=32122 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 4095, 0, 1212, 0, 2496, 811, 0, 0, 0, 0, 0, 0, 2591, 658, 0, 3546, 501, 0, 1, 731, 0, 0, 0, 0, 0, 0, 0, 0, 2732, 1, 0, 1, 1, 0, 1584, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 6, 16, 3, 15, 0, 3982, 3999, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2814, 0, 3, 1, 2817, 0, 0, 2030, 0, 0, 0, 0, 0, 0, 0, 0, 367, 4086, 1, 1, 0, 0, 2, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 1625, 7, 0, 4071, 2279, 1, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 1669, 0, 1, 0, 1, 27, 1281, 1001, 231, 104, 56, 0, 0, 0] rerouted_tokens_per_ep_rank=[8622, 8028, 8398, 8030, 7665, 8321, 7998, 8474] avg=8192.0000 max=8622 max_rank=0 min=7665 min_rank=4 imbalance_ratio=1.052490 max_load_reduction=56914 max_load_reduction_pct=0.868439 moved_to_echo=32458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 0, 321, 0, 2504, 816, 0, 0, 0, 0, 0, 0, 2183, 1948, 0, 2566, 1234, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 0, 2, 1, 1, 0, 1580, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 2, 134, 12, 1097, 0, 3830, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2811, 1, 0, 0, 3608, 0, 0, 1763, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4091, 0, 0, 0, 0, 0, 3857, 0, 0, 0, 0, 0, 0, 0, 0, 3291, 0, 1, 4094, 2273, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 602, 1, 0, 1, 0, 22, 1285, 1007, 238, 80, 57, 0, 0, 0] rerouted_tokens_per_ep_rank=[7738, 7936, 7474, 9171, 8183, 7981, 9664, 7389] avg=8192.0000 max=9664 max_rank=6 min=7389 min_rank=7 imbalance_ratio=1.179688 max_load_reduction=55872 max_load_reduction_pct=0.852539 moved_to_echo=33217 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[20480, 128, 128, 128, 128, 128, 12800, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 6272, 128, 20096, 6528, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[22272, 6272, 128, 26752, 4864, 0, 128, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1920, 32768, 128, 128, 128, 0, 128, 30976, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 256, 128, 2048, 128, 31872, 31360, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 32768, 10880, 128, 128, 128, 128, 640, 10368, 8064, 1920, 896, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[22528, 128, 128, 128, 24832, 128, 128, 18304, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[14592, 128, 128, 32768, 18304, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66688 slack_slots=3584 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 35, 0, 4085, 0, 3, 1, 0, 0, 0, 32, 4096, 3827, 0, 0, 4095, 1, 0, 4096, 0, 0, 166, 0, 4084, 2973, 4094, 270, 0, 4093, 0, 0, 1, 976, 350, 1732, 2, 4, 2652, 833, 0, 0, 4054, 0, 0, 0, 3722, 0, 51, 0, 4095, 0, 0, 1, 0, 0, 4090, 0, 0, 2925, 1, 4095, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 27, 0, 4094, 0, 2, 7, 0, 0, 0, 91, 4093, 3831, 0, 0, 4096, 0, 0, 4096, 0, 0, 229, 0, 4095, 1009, 4077, 95, 0, 4086, 0, 0, 0, 1197, 1015, 2001, 3, 15, 3411, 960, 0, 0, 4056, 0, 0, 0, 3532, 0, 5, 0, 4093, 0, 0, 0, 0, 0, 4078, 0, 0, 3224, 0, 4013, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 1, 4095, 0, 0, 5, 0, 0, 1, 43, 4094, 3870, 0, 1, 4095, 0, 1, 4095, 1, 0, 122, 0, 4095, 1172, 4044, 86, 0, 4076, 1, 0, 0, 952, 549, 2342, 0, 17, 3491, 747, 1, 1, 4047, 0, 0, 1, 3612, 0, 0, 1, 4094, 0, 1, 0, 1, 1, 4093, 1, 1, 3619, 0, 4062, 1, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4088, 0, 4, 11, 0, 0, 0, 15, 4095, 3617, 0, 0, 4095, 1, 0, 4095, 0, 0, 250, 1, 4096, 2746, 3931, 243, 0, 4082, 0, 0, 0, 998, 667, 2766, 1, 8, 3205, 153, 0, 0, 4028, 1, 2, 0, 3371, 0, 0, 0, 4095, 1, 1, 1, 2, 0, 4093, 0, 1, 2723, 0, 4045, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 0, 4090, 0, 25, 1, 0, 1, 0, 45, 4095, 4056, 0, 0, 4096, 0, 0, 4095, 0, 1, 680, 0, 4079, 3056, 4013, 111, 0, 4064, 0, 0, 0, 1151, 629, 1693, 5, 1, 3858, 276, 0, 1, 4087, 0, 16, 0, 2303, 0, 290, 1, 4095, 0, 0, 0, 6, 0, 4094, 0, 1, 2729, 0, 3785, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 1, 4095, 0, 1, 1, 0, 0, 0, 294, 4095, 3964, 0, 1, 4095, 0, 1, 4095, 1, 0, 557, 1, 4093, 1992, 4043, 266, 1, 4091, 1, 0, 0, 1292, 448, 1893, 1, 40, 2927, 1093, 1, 1, 4079, 0, 0, 1, 3562, 0, 24, 1, 4095, 0, 1, 0, 3, 1, 4094, 0, 1, 2196, 0, 4089, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 0, 4092, 0, 2, 8, 0, 0, 0, 76, 4094, 3976, 0, 0, 4096, 2, 0, 4096, 0, 0, 105, 2, 4089, 2504, 4035, 66, 0, 4080, 0, 0, 0, 2043, 197, 2889, 5, 9, 2815, 476, 0, 0, 4018, 0, 0, 0, 2826, 0, 21, 0, 4094, 0, 0, 0, 4, 0, 4092, 0, 0, 2715, 0, 4004, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 49, 0, 4092, 1, 0, 1, 0, 0, 0, 130, 4095, 4093, 0, 0, 4096, 0, 0, 4096, 0, 0, 1725, 0, 4096, 2263, 4089, 39, 0, 4088, 0, 1, 1, 1069, 955, 1789, 1, 0, 3593, 311, 0, 0, 4096, 0, 5, 0, 1360, 0, 223, 0, 4095, 0, 0, 0, 2, 0, 4092, 0, 0, 3205, 0, 3778, 0, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 35, 0, 4085, 0, 3, 1, 0, 3993, 82, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 3827, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3622, 0, 0, 166, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 2973, 4094, 270, 0, 1795, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 976, 350, 1732, 2, 4, 2652, 833, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 3722, 0, 51, 926, 93, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 1, 4095, 0, 0, 1, 1290, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8057, 7873, 9133, 6927, 8846, 8189, 8312] avg=8192.0000 max=9133 max_rank=3 min=6927 min_rank=4 imbalance_ratio=1.114868 max_load_reduction=56403 max_load_reduction_pct=0.860641 moved_to_echo=32274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 0, 4094, 0, 2, 7, 0, 3994, 82, 0, 0, 0, 0, 0, 0, 0, 0, 91, 4093, 3831, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3622, 0, 0, 229, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 4077, 95, 0, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 1015, 2001, 3, 15, 3411, 960, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 3532, 0, 5, 924, 93, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4078, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 0, 4013, 0, 5, 0, 1288, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8117, 7946, 6973, 8980, 8610, 8174, 8530] avg=8192.0000 max=8980 max_rank=4 min=6973 min_rank=3 imbalance_ratio=1.096191 max_load_reduction=56556 max_load_reduction_pct=0.862976 moved_to_echo=34294 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 4095, 0, 0, 5, 0, 3993, 82, 0, 0, 0, 0, 0, 0, 0, 1, 43, 4094, 3870, 0, 1, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3621, 1, 0, 122, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 4044, 86, 0, 1787, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 952, 549, 2342, 0, 17, 3491, 747, 1, 378, 0, 0, 0, 0, 0, 0, 0, 1, 4047, 0, 0, 1, 3612, 0, 0, 922, 93, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 1, 1, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3619, 0, 4062, 1, 1, 1, 1285, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8111, 7840, 7090, 8477, 8676, 8194, 8971] avg=8192.0000 max=8971 max_rank=7 min=7090 min_rank=3 imbalance_ratio=1.095093 max_load_reduction=56565 max_load_reduction_pct=0.863113 moved_to_echo=34318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 4090, 0, 25, 1, 0, 3994, 82, 0, 0, 0, 0, 0, 0, 1, 0, 45, 4095, 4056, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3621, 0, 1, 680, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 4013, 111, 0, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1151, 629, 1693, 5, 1, 3858, 276, 0, 378, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 16, 0, 2303, 0, 290, 919, 93, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 6, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2729, 0, 3785, 0, 1, 0, 1281, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8299, 8381, 8962, 7991, 7709, 8199, 7797] avg=8192.0000 max=8962 max_rank=3 min=7709 min_rank=5 imbalance_ratio=1.093994 max_load_reduction=56574 max_load_reduction_pct=0.863251 moved_to_echo=31696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4088, 0, 4, 11, 0, 3993, 82, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4095, 3617, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3621, 0, 0, 250, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2746, 3931, 243, 0, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 998, 667, 2766, 1, 8, 3205, 153, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 1, 2, 0, 3371, 0, 0, 923, 93, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 2, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2723, 0, 4045, 0, 4, 0, 1286, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 7829, 7969, 8711, 8176, 8418, 8196, 8059] avg=8192.0000 max=8711 max_rank=3 min=7829 min_rank=1 imbalance_ratio=1.063354 max_load_reduction=56825 max_load_reduction_pct=0.867081 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 4095, 0, 1, 1, 0, 3993, 82, 0, 0, 0, 0, 0, 0, 0, 0, 294, 4095, 3964, 0, 1, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3621, 1, 0, 557, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 4043, 266, 1, 1795, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1292, 448, 1893, 1, 40, 2927, 1093, 1, 378, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 0, 0, 1, 3562, 0, 24, 925, 93, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 3, 1, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2196, 0, 4089, 0, 2, 1, 1289, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8456, 8274, 8098, 8073, 8685, 8198, 7578] avg=8192.0000 max=8685 max_rank=5 min=7578 min_rank=7 imbalance_ratio=1.060181 max_load_reduction=56851 max_load_reduction_pct=0.867477 moved_to_echo=32534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 4092, 0, 2, 8, 0, 3994, 82, 0, 0, 0, 0, 0, 0, 0, 0, 76, 4094, 3976, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3622, 0, 0, 105, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 2504, 4035, 66, 0, 1789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2043, 197, 2889, 5, 9, 2815, 476, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 2826, 0, 21, 923, 93, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2715, 0, 4004, 0, 2, 0, 1286, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8248, 7820, 8394, 8812, 7881, 8193, 8007] avg=8192.0000 max=8812 max_rank=4 min=7820 min_rank=2 imbalance_ratio=1.075684 max_load_reduction=56724 max_load_reduction_pct=0.865540 moved_to_echo=32893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 49, 0, 4092, 1, 0, 1, 0, 3995, 85, 0, 0, 0, 0, 0, 0, 0, 0, 130, 4095, 4093, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 0, 0, 1725, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2263, 4089, 39, 0, 1782, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 955, 1789, 1, 0, 3593, 311, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 1360, 0, 223, 929, 98, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3205, 0, 3778, 0, 7, 0, 1292, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8419, 9433, 8175, 8100, 6711, 8193, 8282] avg=8192.0000 max=9433 max_rank=2 min=6711 min_rank=5 imbalance_ratio=1.151489 max_load_reduction=56103 max_load_reduction_pct=0.856064 moved_to_echo=31286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 29056, 128, 128, 3840, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32512, 128, 128, 128, 24320, 0, 640, 7424, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[9728, 4864, 17152, 128, 128, 25984, 4864, 128, 3072, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[17792, 32384, 1280, 128, 14336, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 0, 32000, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 23424, 128, 31872, 128, 128, 128, 10368, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 768, 32768, 31360, 0, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4032, 0, 0, 3103, 25, 4029, 0, 0, 0, 0, 0, 3915, 4093, 0, 4073, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 0, 4096, 4096, 0, 4096, 4093, 0, 2, 0, 0, 0, 10, 4096, 0, 0, 29, 4039, 0, 3, 0, 4096, 0, 1320, 0, 1, 2, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3475, 0, 0, 3854, 30, 4086, 0, 0, 0, 2, 0, 3188, 4095, 0, 4073, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 4096, 4096, 0, 4090, 4096, 0, 1, 0, 1, 0, 5, 4096, 0, 0, 66, 3850, 0, 1, 0, 4096, 0, 1950, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3820, 0, 0, 3528, 9, 4087, 0, 0, 0, 9, 0, 3220, 4093, 0, 4081, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 4096, 4096, 0, 4092, 4076, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 214, 3962, 0, 7, 0, 4096, 0, 1660, 0, 4, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 3969, 1, 0, 3738, 21, 4094, 0, 1, 0, 14, 6, 3458, 4094, 0, 4087, 4095, 1, 1, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 4095, 0, 4095, 4094, 1, 4094, 4094, 1, 0, 1, 0, 0, 0, 4095, 0, 1, 164, 4003, 0, 3, 0, 4093, 0, 1017, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3828, 0, 0, 3508, 14, 4083, 0, 0, 0, 52, 0, 3398, 4091, 0, 4091, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 4096, 4096, 0, 4094, 4094, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 290, 3687, 0, 4, 0, 4096, 0, 1624, 0, 2, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4056, 0, 0, 3678, 26, 3995, 0, 1, 0, 10, 0, 3303, 4093, 0, 4044, 4095, 1, 1, 1, 4087, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 4096, 0, 4095, 4095, 0, 4096, 4082, 1, 3, 0, 0, 0, 28, 4095, 0, 1, 101, 3934, 1, 1, 0, 4095, 0, 1416, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 3819, 32, 3883, 0, 0, 0, 21, 0, 3264, 4094, 0, 4080, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 4096, 4096, 0, 4096, 4083, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 2030, 754, 0, 3, 0, 4092, 0, 2614, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 3724, 1, 1, 3463, 8, 4095, 0, 1, 0, 4, 0, 3309, 4094, 0, 4047, 4095, 1, 1, 1, 4093, 0, 0, 3, 0, 1, 0, 0, 1, 0, 1, 1, 4095, 3, 4095, 4095, 0, 4091, 4095, 1, 0, 1, 0, 0, 0, 4095, 0, 1, 78, 3888, 0, 6, 0, 4094, 0, 1949, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4032, 0, 0, 3103, 729, 0, 0, 0, 0, 0, 0, 0, 25, 4029, 0, 0, 0, 0, 0, 3915, 730, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4073, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 2, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 4039, 0, 3, 0, 206, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1320, 0, 1, 2, 1, 0, 2400, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7864, 8699, 8194, 8193, 8192, 8201, 8373, 7820] avg=8192.0000 max=8699 max_rank=1 min=7820 min_rank=7 imbalance_ratio=1.061890 max_load_reduction=56837 max_load_reduction_pct=0.867264 moved_to_echo=32586 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3820, 0, 0, 3528, 729, 0, 0, 0, 0, 0, 0, 0, 9, 4087, 0, 0, 0, 9, 0, 3220, 730, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4081, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4076, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 214, 3962, 0, 7, 0, 206, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1660, 0, 4, 0, 2, 0, 2400, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8077, 8055, 8202, 8191, 8192, 8171, 8485, 8163] avg=8192.0000 max=8485 max_rank=6 min=8055 min_rank=1 imbalance_ratio=1.035767 max_load_reduction=57051 max_load_reduction_pct=0.870529 moved_to_echo=33011 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 3969, 1, 0, 3738, 730, 0, 0, 0, 0, 0, 0, 0, 21, 4094, 0, 1, 0, 14, 6, 3458, 730, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4087, 0, 1, 1, 1, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 164, 4003, 0, 3, 0, 206, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1017, 1, 0, 0, 0, 1, 2401, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8441, 8324, 8212, 8191, 8192, 8190, 8472, 7514] avg=8192.0000 max=8472 max_rank=6 min=7514 min_rank=7 imbalance_ratio=1.034180 max_load_reduction=57064 max_load_reduction_pct=0.870728 moved_to_echo=32368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4056, 0, 0, 3678, 728, 0, 0, 0, 0, 0, 0, 0, 26, 3995, 0, 1, 0, 10, 0, 3303, 729, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4044, 0, 1, 1, 1, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 1, 3, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 101, 3934, 1, 1, 0, 205, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1416, 1, 0, 0, 0, 0, 2396, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8463, 8064, 8169, 8191, 8192, 8210, 8338, 7909] avg=8192.0000 max=8463 max_rank=0 min=7909 min_rank=7 imbalance_ratio=1.033081 max_load_reduction=57073 max_load_reduction_pct=0.870865 moved_to_echo=32649 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3475, 0, 0, 3854, 730, 0, 0, 0, 0, 0, 0, 0, 30, 4086, 0, 0, 0, 2, 0, 3188, 730, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4073, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 1, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 66, 3850, 0, 1, 0, 206, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1950, 0, 0, 0, 0, 0, 2401, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8059, 8036, 8197, 8192, 8192, 8193, 8219, 8448] avg=8192.0000 max=8448 max_rank=7 min=8036 min_rank=1 imbalance_ratio=1.031250 max_load_reduction=57088 max_load_reduction_pct=0.871094 moved_to_echo=33052 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3828, 0, 0, 3508, 730, 0, 0, 0, 0, 0, 0, 0, 14, 4083, 0, 0, 0, 52, 0, 3398, 730, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4091, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 290, 3687, 0, 4, 0, 206, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1624, 0, 2, 0, 2, 0, 2400, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8277, 8210, 8191, 8192, 8192, 8283, 8125] avg=8192.0000 max=8283 max_rank=6 min=8066 min_rank=0 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32792 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 3724, 1, 1, 3463, 729, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 1, 0, 4, 0, 3309, 730, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4047, 0, 1, 1, 1, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 1, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 78, 3888, 0, 6, 0, 206, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1949, 1, 0, 0, 0, 1, 2400, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7920, 8147, 8172, 8195, 8195, 8188, 8274, 8445] avg=8192.0000 max=8445 max_rank=7 min=7920 min_rank=0 imbalance_ratio=1.030884 max_load_reduction=57091 max_load_reduction_pct=0.871140 moved_to_echo=33102 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 3819, 735, 0, 0, 0, 0, 0, 0, 0, 32, 3883, 0, 0, 0, 21, 0, 3264, 734, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4080, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 2, 7, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2030, 754, 0, 3, 0, 209, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2614, 0, 0, 0, 1, 0, 2405, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8646, 7934, 8180, 8192, 8189, 8191, 7092, 9112] avg=8192.0000 max=9112 max_rank=7 min=7092 min_rank=6 imbalance_ratio=1.112305 max_load_reduction=56424 max_load_reduction_pct=0.860962 moved_to_echo=32584 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 32640, 0, 128, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 0, 13568, 128, 128, 128, 128, 128, 19328, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 3072, 28160, 128, 128, 0, 1664, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 32384, 0, 128, 0, 128, 128, 27136, 5888, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 31104, 128, 128, 28800, 5888, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 4091, 3, 0, 0, 1, 0, 0, 4073, 0, 4093, 0, 13, 4096, 0, 4081, 0, 0, 5, 3, 36, 2, 0, 4095, 6, 0, 4096, 0, 4096, 0, 0, 0, 0, 4086, 4095, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 4093, 4074, 0, 0, 14, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4096, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 4093, 3, 0, 0, 1, 0, 0, 4069, 0, 4095, 0, 10, 4058, 1, 4075, 0, 0, 7, 1, 23, 1, 0, 4095, 163, 0, 4096, 0, 4095, 0, 0, 0, 0, 4086, 4094, 4096, 0, 0, 3, 0, 4096, 0, 0, 0, 4094, 3966, 0, 0, 22, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 1, 1, 2, 4090, 2, 1, 1, 1, 0, 4095, 0, 1, 0, 1, 0, 1, 4084, 1, 4094, 0, 7, 4095, 0, 4095, 0, 0, 2, 0, 5, 0, 0, 4095, 0, 0, 4094, 1, 4094, 1, 0, 1, 1, 4088, 4088, 4095, 1, 0, 0, 0, 4093, 0, 1, 1, 4095, 4044, 1, 1, 64, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 4, 0, 0, 4041, 0, 4096, 0, 4, 4096, 1, 4092, 0, 1, 1, 1, 35, 0, 0, 4095, 4, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4016, 0, 0, 94, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4096, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 4068, 0, 4092, 0, 4, 4096, 0, 4094, 0, 0, 4, 2, 38, 0, 0, 4096, 6, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 4093, 4096, 0, 0, 1, 0, 4096, 0, 3, 0, 4094, 4012, 0, 0, 70, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4094, 0, 1, 0, 0, 4093, 1, 0, 1, 1, 0, 4093, 0, 0, 1, 1, 0, 1, 3982, 1, 4095, 1, 27, 4048, 0, 4046, 0, 0, 16, 0, 223, 2, 1, 4095, 44, 1, 4095, 0, 4094, 1, 0, 1, 1, 4093, 4095, 4095, 1, 0, 0, 0, 4095, 0, 0, 0, 4095, 3890, 0, 1, 108, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 1, 1, 0, 4091, 3, 1, 1, 1, 0, 4090, 0, 1, 0, 2, 0, 1, 4043, 0, 4095, 1, 0, 4095, 0, 4095, 0, 1, 4, 0, 3, 0, 0, 4095, 7, 0, 4094, 0, 4095, 1, 0, 1, 1, 4092, 4095, 4095, 1, 0, 0, 0, 4095, 0, 0, 1, 4095, 4033, 1, 1, 107, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4050, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 4095, 2, 0, 0, 200, 0, 0, 4014, 0, 4094, 0, 90, 3845, 1, 4089, 0, 0, 0, 1, 399, 0, 1, 4096, 367, 0, 4096, 0, 4096, 0, 0, 0, 0, 4087, 4093, 4079, 0, 0, 1, 0, 4096, 0, 0, 0, 4095, 3444, 0, 0, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4064, 36, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 4073, 0, 4091, 25, 0, 0, 0, 0, 0, 0, 37, 0, 13, 4096, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 36, 2, 0, 4095, 6, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 4074, 0, 0, 14, 0, 99, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8189, 8193, 8227, 8065, 8189, 8196, 8282] avg=8192.0000 max=8282 max_rank=7 min=8065 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 4096, 0, 0, 0, 4064, 36, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 4069, 0, 4091, 25, 0, 0, 0, 0, 0, 0, 37, 0, 10, 4058, 1, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 23, 1, 0, 4095, 163, 0, 3920, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4095, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3966, 0, 0, 22, 0, 99, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8191, 8189, 8181, 8210, 8188, 8198, 8181] avg=8192.0000 max=8210 max_rank=4 min=8181 min_rank=3 imbalance_ratio=1.002197 max_load_reduction=57326 max_load_reduction_pct=0.874725 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 1, 1, 4062, 36, 0, 0, 0, 0, 0, 0, 2, 4090, 2, 1, 1, 1, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 4084, 1, 4089, 25, 0, 0, 0, 0, 0, 0, 37, 0, 7, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 4095, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4094, 1, 0, 1, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4095, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4044, 1, 1, 64, 1, 99, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8195, 8202, 8234, 8021, 8193, 8188, 8307] avg=8192.0000 max=8307 max_rank=7 min=8021 min_rank=4 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4094, 0, 1, 0, 4063, 36, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 1, 1, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 3982, 1, 4091, 25, 0, 0, 0, 0, 0, 0, 37, 1, 27, 4048, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 223, 2, 1, 4095, 44, 1, 3920, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4094, 1, 0, 1, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3890, 0, 1, 108, 1, 99, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 8102, 8159, 8302, 8197, 8195, 8194] avg=8192.0000 max=8302 max_rank=4 min=8102 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 0, 4064, 36, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4041, 0, 4091, 25, 0, 0, 0, 0, 0, 0, 37, 0, 4, 4096, 1, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 35, 0, 0, 4095, 4, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4016, 0, 0, 94, 0, 99, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8194, 8161, 8231, 8057, 8197, 8196, 8304] avg=8192.0000 max=8304 max_rank=7 min=8057 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 4096, 0, 0, 0, 4064, 36, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4068, 0, 4091, 25, 0, 0, 0, 0, 0, 0, 37, 0, 4, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 38, 0, 0, 4096, 6, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4094, 4012, 0, 0, 70, 0, 99, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8190, 8185, 8231, 8063, 8195, 8195, 8278] avg=8192.0000 max=8278 max_rank=7 min=8063 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 1, 1, 4062, 36, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 1, 1, 1, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 1, 4043, 0, 4091, 28, 0, 0, 0, 0, 0, 0, 37, 1, 0, 4095, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 0, 0, 4095, 7, 0, 3920, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4095, 1, 0, 1, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4033, 1, 1, 107, 1, 99, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8190, 8166, 8229, 8029, 8194, 8195, 8338] avg=8192.0000 max=8338 max_rank=7 min=8029 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4050, 0, 0, 0, 4070, 40, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 200, 0, 0, 4014, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 19, 0, 90, 3845, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 399, 0, 1, 4096, 367, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4079, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3444, 0, 0, 8, 0, 105, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8195, 8338, 8044, 8789, 8183, 8173, 7652] avg=8192.0000 max=8789 max_rank=4 min=7652 min_rank=7 imbalance_ratio=1.072876 max_load_reduction=56747 max_load_reduction_pct=0.865891 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 32768, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 31488, 128, 128, 512, 128, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 128, 128, 32640, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 768, 128, 128, 32768, 640, 128, 31360, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 128, 256, 32512, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 256, 0, 128, 32384, 128, 32768, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3341, 1, 0, 4, 0, 0, 0, 4072, 0, 100, 0, 4095, 79, 4096, 0, 0, 0, 0, 0, 1, 416, 4096, 0, 29, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 4039, 4076, 1434, 4096, 0, 0, 4090, 4094, 3472, 0, 0, 0, 0, 0, 0, 4094, 0, 17, 1, 0, 4025, 0, 3530, 29, 1, 3, 0, 38] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2828, 0, 0, 0, 0, 0, 0, 4090, 0, 531, 0, 4093, 92, 4096, 0, 0, 0, 1, 0, 1, 3688, 4096, 0, 9, 0, 0, 0, 0, 4064, 0, 0, 1, 0, 3306, 4089, 2463, 4096, 0, 0, 4096, 4095, 1588, 0, 0, 0, 0, 0, 0, 4084, 0, 13, 0, 0, 2562, 0, 3344, 38, 1, 0, 0, 75] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3399, 1, 0, 4, 0, 0, 0, 4092, 0, 79, 0, 4095, 93, 4096, 0, 0, 0, 0, 0, 0, 1133, 4096, 0, 76, 0, 0, 0, 0, 4014, 2, 0, 0, 0, 4092, 4093, 599, 4096, 0, 0, 4072, 4096, 3247, 0, 0, 0, 0, 0, 0, 4075, 0, 65, 0, 0, 4069, 0, 3600, 15, 0, 2, 0, 139] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 2757, 0, 1, 22, 0, 0, 0, 4078, 1, 6, 0, 4095, 26, 4095, 1, 1, 0, 0, 0, 0, 1306, 4095, 0, 6, 1, 1, 1, 1, 4083, 0, 0, 0, 1, 4092, 4051, 1626, 4094, 1, 1, 4093, 4094, 2655, 1, 0, 0, 0, 1, 0, 4091, 1, 98, 0, 1, 4000, 1, 3904, 31, 0, 0, 0, 26] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4095, 2769, 0, 1, 24, 0, 1, 0, 4091, 1, 313, 1, 4095, 60, 4095, 1, 0, 0, 0, 0, 1, 1748, 4095, 1, 56, 0, 1, 0, 0, 4028, 0, 1, 0, 0, 3901, 4086, 1397, 4095, 1, 0, 4095, 4095, 3077, 1, 0, 1, 1, 1, 0, 4058, 0, 7, 0, 1, 3501, 1, 3586, 91, 0, 0, 0, 62] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2519, 0, 0, 6, 0, 0, 0, 4065, 0, 114, 0, 4096, 53, 4096, 0, 0, 0, 0, 0, 2, 2383, 4096, 0, 18, 0, 0, 0, 0, 4075, 2, 0, 0, 0, 3997, 4036, 1320, 4093, 0, 0, 4086, 4096, 2845, 0, 0, 0, 0, 0, 0, 4082, 0, 32, 0, 0, 4009, 0, 3246, 34, 0, 2, 0, 37] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2993, 1, 0, 8, 0, 0, 0, 4075, 0, 103, 0, 4095, 156, 4096, 0, 0, 0, 0, 0, 1, 1674, 4096, 0, 13, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 4062, 4091, 1451, 4096, 0, 0, 4096, 4095, 2420, 0, 2, 0, 0, 0, 0, 4095, 0, 6, 0, 0, 4076, 0, 3554, 12, 1, 0, 0, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 2981, 0, 1, 173, 0, 0, 0, 4078, 1, 10, 0, 4095, 47, 4095, 1, 1, 0, 0, 0, 0, 1177, 4095, 1, 24, 0, 1, 0, 1, 4015, 0, 0, 0, 1, 4032, 4052, 684, 4092, 1, 1, 4091, 4095, 3362, 1, 2, 0, 1, 1, 0, 4095, 0, 15, 0, 1, 3991, 1, 3959, 12, 0, 0, 0, 153] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3341, 1, 0, 4, 0, 674, 443, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 100, 0, 3879, 79, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 416, 2383, 20, 0, 0, 0, 0, 0, 0, 4096, 0, 29, 0, 0, 0, 0, 4071, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 2883, 1434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1268, 3472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 17, 1, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 3530, 29, 1, 3, 0, 38, 519, 200, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8559, 8130, 6916, 8212, 8356, 8830, 8188, 8345] avg=8192.0000 max=8830 max_rank=5 min=6916 min_rank=2 imbalance_ratio=1.077881 max_load_reduction=56706 max_load_reduction_pct=0.865265 moved_to_echo=33719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3399, 1, 0, 4, 0, 677, 443, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 79, 0, 3879, 93, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1133, 2384, 20, 0, 0, 0, 0, 0, 0, 4096, 0, 76, 0, 0, 0, 0, 4014, 16, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4092, 2895, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 1269, 3247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 65, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 3600, 15, 0, 2, 0, 139, 521, 200, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8620, 8143, 7633, 8202, 7588, 8588, 8216, 8546] avg=8192.0000 max=8620 max_rank=0 min=7588 min_rank=4 imbalance_ratio=1.052246 max_load_reduction=56916 max_load_reduction_pct=0.868469 moved_to_echo=32938 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 2769, 0, 1, 24, 0, 675, 443, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 1, 313, 1, 3879, 60, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 1, 1748, 2384, 19, 0, 0, 0, 0, 0, 0, 4095, 1, 56, 0, 1, 0, 0, 4028, 16, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3901, 2891, 1397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1268, 3077, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4058, 0, 7, 0, 1, 4076, 0, 0, 0, 0, 0, 0, 0, 3501, 1, 3586, 91, 0, 0, 0, 62, 520, 200, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8007, 8346, 8248, 8197, 8190, 8443, 8144, 7961] avg=8192.0000 max=8443 max_rank=5 min=7961 min_rank=7 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32738 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4095, 2757, 0, 1, 22, 0, 670, 443, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 6, 0, 3879, 26, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 1306, 2383, 20, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 1, 1, 1, 1, 4083, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 2865, 1626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4093, 1268, 2655, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 1, 98, 0, 1, 4074, 0, 0, 0, 0, 0, 0, 0, 4000, 1, 3904, 31, 0, 0, 0, 26, 516, 200, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7989, 7990, 7806, 8204, 8584, 8019, 8266, 8678] avg=8192.0000 max=8678 max_rank=7 min=7806 min_rank=2 imbalance_ratio=1.059326 max_load_reduction=56858 max_load_reduction_pct=0.867584 moved_to_echo=33547 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 2993, 1, 0, 8, 0, 676, 443, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 103, 0, 3879, 156, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 1674, 2384, 20, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 4052, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 2894, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1268, 2420, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 3554, 12, 1, 0, 0, 20, 521, 200, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8213, 8175, 8177, 8407, 7786, 8177, 8384] avg=8192.0000 max=8407 max_rank=4 min=7786 min_rank=5 imbalance_ratio=1.026245 max_load_reduction=57129 max_load_reduction_pct=0.871719 moved_to_echo=32754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 2519, 0, 0, 6, 0, 667, 443, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 114, 0, 3880, 53, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 2383, 2384, 20, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 0, 0, 4075, 16, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3997, 2855, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1269, 2845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 32, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 3246, 34, 0, 2, 0, 37, 514, 200, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7731, 8112, 8885, 8205, 8174, 8200, 8187, 8042] avg=8192.0000 max=8885 max_rank=2 min=7731 min_rank=0 imbalance_ratio=1.084595 max_load_reduction=56651 max_load_reduction_pct=0.864426 moved_to_echo=32603 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4095, 2981, 0, 1, 173, 0, 670, 443, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 10, 0, 3879, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 1177, 2384, 20, 0, 0, 0, 0, 0, 0, 4095, 1, 24, 0, 1, 0, 1, 4015, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4032, 2866, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4091, 1268, 3362, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 15, 0, 1, 4072, 0, 0, 0, 0, 0, 0, 0, 3991, 1, 3959, 12, 0, 0, 0, 153, 516, 200, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8364, 8015, 7678, 8153, 7583, 8726, 8185, 8832] avg=8192.0000 max=8832 max_rank=7 min=7583 min_rank=4 imbalance_ratio=1.078125 max_load_reduction=56704 max_load_reduction_pct=0.865234 moved_to_echo=33326 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 2828, 0, 0, 0, 0, 679, 446, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 531, 0, 3874, 92, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 1, 3688, 2389, 20, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 4064, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3306, 2884, 2463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1260, 1588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 13, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 2562, 0, 3344, 38, 1, 0, 0, 75, 526, 202, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8049, 8587, 10195, 8186, 8654, 6944, 8173, 6748] avg=8192.0000 max=10195 max_rank=2 min=6748 min_rank=7 imbalance_ratio=1.244507 max_load_reduction=55341 max_load_reduction_pct=0.844437 moved_to_echo=30519 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 31616, 23040, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 32768, 128, 1280, 128, 31104, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 256, 128, 128, 128, 128, 32512, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 10240, 22784, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 128, 13568, 19200, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 256, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[30336, 128, 28800, 384, 128, 128, 0, 640, 4224, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 23680, 128, 128, 256, 0, 5504, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 3059, 0, 4096, 4088, 0, 4078, 4096, 0, 2, 0, 0, 504, 4095, 0, 319, 0, 0, 4046, 4096, 0, 0, 4095, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 4045, 1, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 10, 4088, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 2, 1, 4094, 0, 0, 0, 2458, 2, 4095, 4093, 1, 4042, 4093, 1, 6, 0, 1, 1073, 4095, 0, 550, 0, 0, 3997, 4095, 1, 1, 4092, 1, 0, 1, 0, 56, 1, 1, 0, 0, 0, 1, 4074, 0, 3, 4095, 0, 1, 1, 4094, 4095, 0, 1, 0, 132, 0, 0, 0, 1, 1, 2, 5, 4081, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 1, 0, 3351, 0, 4096, 4088, 0, 4064, 4096, 0, 0, 0, 0, 109, 4095, 0, 570, 0, 0, 4030, 4096, 0, 0, 4090, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 4024, 1, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 292, 0, 0, 0, 0, 0, 1, 6, 4031, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 2, 0, 1756, 0, 4096, 4089, 0, 4078, 4096, 0, 0, 0, 0, 1061, 4094, 0, 1225, 0, 0, 4062, 4096, 0, 0, 4079, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 3931, 3, 2, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 47, 4091, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4069, 0, 2, 0, 3300, 0, 4096, 4084, 0, 4088, 4096, 0, 0, 0, 0, 273, 4094, 0, 805, 0, 0, 3919, 4096, 0, 0, 4084, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 3838, 1, 0, 4093, 0, 0, 0, 4096, 4096, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 17, 4047, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 0, 0, 0, 3571, 0, 4095, 4092, 1, 4008, 4095, 0, 0, 0, 1, 265, 4093, 0, 109, 0, 0, 3967, 4095, 0, 1, 4095, 1, 0, 1, 0, 171, 1, 1, 0, 1, 0, 1, 4046, 0, 0, 4083, 0, 0, 1, 4095, 4094, 0, 1, 0, 260, 0, 0, 0, 1, 1, 1, 9, 4086, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 0, 3567, 0, 4096, 4091, 0, 4074, 4096, 0, 1, 0, 0, 744, 4095, 0, 75, 0, 0, 4080, 4096, 0, 0, 4096, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 3553, 1, 0, 4091, 0, 0, 0, 4096, 4096, 0, 0, 0, 130, 0, 0, 0, 0, 0, 1, 175, 4081, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4091, 2, 0, 1, 3047, 0, 4095, 4092, 1, 3986, 4095, 1, 1, 0, 1, 435, 4095, 0, 612, 0, 0, 3947, 4095, 1, 1, 4086, 1, 0, 1, 0, 208, 0, 1, 0, 0, 0, 1, 4007, 0, 3, 4095, 0, 1, 1, 4095, 4095, 0, 1, 0, 146, 0, 0, 0, 1, 1, 0, 11, 4084, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 4000, 70, 29, 0, 0, 0, 0, 0, 3059, 0, 0, 1125, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 504, 4095, 0, 319, 0, 2934, 69, 0, 0, 0, 0, 0, 0, 0, 4046, 96, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 4026, 0, 0, 0, 0, 0, 0, 0, 4045, 1, 0, 4094, 0, 0, 0, 143, 13, 0, 0, 0, 0, 0, 0, 3884, 4096, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4088, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8262, 7923, 8237, 8144, 8296, 8297, 8181] avg=8192.0000 max=8297 max_rank=6 min=7923 min_rank=2 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32918 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 1, 4094, 0, 0, 0, 3999, 69, 29, 0, 0, 0, 0, 0, 2458, 2, 0, 1127, 1, 4042, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 1073, 4095, 0, 550, 0, 2937, 69, 0, 0, 0, 0, 0, 0, 0, 3997, 96, 1, 1, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 56, 1, 1, 0, 0, 0, 4095, 4024, 0, 0, 0, 0, 0, 0, 1, 4074, 0, 3, 4095, 0, 1, 1, 143, 13, 0, 0, 0, 0, 0, 0, 3882, 4095, 0, 1, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 5, 4081, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 7631, 8731, 8188, 8178, 8331, 8110, 8172] avg=8192.0000 max=8731 max_rank=2 min=7631 min_rank=1 imbalance_ratio=1.065796 max_load_reduction=56805 max_load_reduction_pct=0.866776 moved_to_echo=32791 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4069, 0, 2, 0, 4000, 69, 29, 0, 0, 0, 0, 0, 3300, 0, 0, 1124, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 4094, 0, 805, 0, 2931, 69, 0, 0, 0, 0, 0, 0, 0, 3919, 96, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 4096, 4027, 0, 0, 0, 0, 0, 0, 0, 3838, 1, 0, 4093, 0, 0, 0, 143, 13, 0, 0, 0, 0, 0, 0, 3884, 4096, 0, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4047, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8512, 8172, 8099, 8148, 8088, 8201, 8147] avg=8192.0000 max=8512 max_rank=1 min=8088 min_rank=5 imbalance_ratio=1.039062 max_load_reduction=57024 max_load_reduction_pct=0.870117 moved_to_echo=32584 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 0, 0, 0, 3999, 69, 29, 0, 0, 0, 0, 0, 3571, 0, 0, 1127, 1, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 265, 4093, 0, 109, 0, 2936, 69, 0, 0, 0, 0, 0, 0, 0, 3967, 96, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 171, 1, 1, 0, 1, 0, 4095, 4026, 0, 0, 0, 0, 0, 0, 1, 4046, 0, 0, 4083, 0, 0, 1, 143, 13, 0, 0, 0, 0, 0, 0, 3883, 4094, 0, 1, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 9, 4086, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8707, 7473, 8160, 8296, 8287, 8238, 8180] avg=8192.0000 max=8707 max_rank=1 min=7473 min_rank=2 imbalance_ratio=1.062866 max_load_reduction=56829 max_load_reduction_pct=0.867142 moved_to_echo=33001 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 1, 0, 4000, 70, 29, 0, 0, 0, 0, 0, 3351, 0, 0, 1125, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 4095, 0, 570, 0, 2934, 69, 0, 0, 0, 0, 0, 0, 0, 4030, 96, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 4026, 0, 0, 0, 0, 0, 0, 0, 4024, 1, 0, 4095, 0, 0, 0, 143, 13, 0, 0, 0, 0, 0, 0, 3884, 4096, 0, 0, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4031, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8540, 7777, 8216, 8139, 8276, 8272, 8121] avg=8192.0000 max=8540 max_rank=1 min=7777 min_rank=2 imbalance_ratio=1.042480 max_load_reduction=56996 max_load_reduction_pct=0.869690 moved_to_echo=32808 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 2, 0, 4000, 70, 29, 0, 0, 0, 0, 0, 1756, 0, 0, 1126, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 4094, 0, 1225, 0, 2934, 69, 0, 0, 0, 0, 0, 0, 0, 4062, 96, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 4096, 4026, 0, 0, 0, 0, 0, 0, 0, 3931, 3, 2, 4094, 0, 0, 0, 143, 13, 0, 0, 0, 0, 0, 0, 3884, 4096, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4091, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 6960, 9383, 8237, 8166, 8186, 8189, 8221] avg=8192.0000 max=9383 max_rank=2 min=6960 min_rank=1 imbalance_ratio=1.145386 max_load_reduction=56153 max_load_reduction_pct=0.856827 moved_to_echo=32762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4091, 2, 0, 1, 3999, 70, 29, 0, 0, 0, 0, 0, 3047, 0, 0, 1127, 1, 3986, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 435, 4095, 0, 612, 0, 2936, 69, 0, 0, 0, 0, 0, 0, 0, 3947, 96, 1, 1, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 208, 0, 1, 0, 0, 0, 4095, 4025, 0, 0, 0, 0, 0, 0, 1, 4007, 0, 3, 4095, 0, 1, 1, 143, 13, 0, 0, 0, 0, 0, 0, 3883, 4095, 0, 1, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 11, 4084, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8162, 8149, 8132, 8330, 8264, 8125, 8179] avg=8192.0000 max=8330 max_rank=4 min=8125 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32898 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 0, 4005, 70, 30, 0, 0, 0, 0, 0, 3567, 0, 0, 1121, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 744, 4095, 0, 75, 0, 2940, 73, 0, 0, 0, 0, 0, 0, 0, 4080, 91, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4096, 4026, 0, 0, 0, 0, 0, 0, 0, 3553, 1, 0, 4091, 0, 0, 0, 145, 18, 0, 0, 0, 0, 0, 0, 3878, 4096, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 175, 4081, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8762, 7928, 8267, 8135, 7808, 8104, 8335] avg=8192.0000 max=8762 max_rank=1 min=7808 min_rank=5 imbalance_ratio=1.069580 max_load_reduction=56774 max_load_reduction_pct=0.866302 moved_to_echo=32382 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[31104, 32768, 0, 128, 0, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 640, 128, 128, 0, 128, 0, 32768, 32256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[24192, 128, 0, 9088, 128, 32512, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 4480, 32768, 0, 4352, 0, 23552, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32128, 768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 31616, 128, 128, 32768, 0, 128, 128, 1152, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 384, 32640, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 128, 128, 32128, 640, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4092, 10, 0, 4096, 4096, 0, 0, 4094, 0, 0, 4081, 4095, 8, 0, 0, 7, 0, 4096, 0, 0, 89, 10, 4096, 2, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 4093, 4096, 0, 3994, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4050, 1, 0, 4095, 1, 0, 1, 1, 4095, 40, 1, 4079, 4095, 0, 1, 4094, 1, 0, 4053, 4093, 44, 0, 0, 16, 1, 4095, 1, 0, 5, 35, 4095, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 2, 4095, 0, 1, 0, 4, 1, 0, 1, 0, 4087, 4095, 0, 4064, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 3931, 1, 0, 4095, 1, 0, 1, 1, 4093, 5, 1, 4093, 4093, 0, 0, 4094, 1, 0, 4071, 4094, 15, 0, 2, 6, 1, 4095, 2, 0, 249, 232, 4095, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 1, 1, 0, 1, 4095, 1, 1, 0, 3, 1, 0, 1, 0, 3928, 4095, 0, 3946, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4005, 1, 0, 4095, 0, 2, 1, 1, 4094, 0, 1, 4089, 4078, 0, 0, 4095, 1, 0, 4093, 4095, 0, 0, 0, 3, 1, 4095, 14, 0, 121, 260, 4095, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 1, 1, 0, 0, 4095, 1, 1, 0, 4, 1, 0, 1, 0, 4026, 4095, 1, 3878, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4090, 0, 0, 4096, 0, 0, 0, 0, 4090, 12, 0, 4096, 4096, 0, 0, 4094, 0, 0, 4083, 4094, 8, 0, 0, 11, 0, 4096, 0, 0, 93, 90, 4096, 0, 0, 0, 0, 3, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4048, 4096, 0, 3952, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4089, 0, 0, 4096, 0, 0, 0, 0, 4093, 19, 0, 4096, 4096, 0, 0, 4095, 0, 0, 4079, 4095, 18, 0, 0, 21, 0, 4096, 0, 0, 98, 317, 4096, 1, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 4096, 0, 3692, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4080, 0, 0, 4096, 0, 0, 0, 0, 4092, 7, 0, 4096, 4096, 0, 0, 4094, 0, 0, 4069, 4096, 30, 0, 0, 18, 0, 4096, 0, 0, 6, 121, 4096, 0, 0, 0, 0, 4, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 4096, 0, 4011, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 4093, 6, 0, 4096, 4096, 0, 0, 4095, 0, 0, 4073, 4094, 21, 1, 0, 8, 0, 4096, 0, 0, 0, 4, 4096, 1, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 4089, 4096, 0, 4090, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4050, 1, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4088, 40, 1, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 24, 1, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 16, 1, 4095, 1, 0, 4030, 32, 0, 0, 0, 0, 0, 0, 5, 35, 4095, 0, 1, 0, 0, 0, 3870, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 4, 1, 0, 1, 4070, 22, 0, 0, 0, 0, 0, 0, 0, 4087, 193, 0, 4064, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8211, 8174, 8219, 8013, 8191, 8194, 8345] avg=8192.0000 max=8345 max_rank=7 min=8013 min_rank=4 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 24, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7, 0, 4096, 0, 0, 4031, 32, 0, 0, 0, 0, 0, 0, 89, 10, 4096, 2, 0, 0, 0, 1, 3870, 7, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 4070, 22, 0, 0, 0, 0, 0, 0, 0, 4093, 194, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8191, 8201, 8174, 8075, 8191, 8188, 8281] avg=8192.0000 max=8281 max_rank=7 min=8075 min_rank=4 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32735 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4090, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 12, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 24, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 11, 0, 4096, 0, 0, 4031, 32, 0, 0, 0, 0, 0, 0, 93, 90, 4096, 0, 0, 0, 0, 3, 3870, 7, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 4070, 22, 0, 0, 0, 0, 0, 0, 0, 4048, 194, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8191, 8203, 8178, 8159, 8192, 8190, 8194] avg=8192.0000 max=8229 max_rank=0 min=8159 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32735 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4005, 1, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 4087, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 24, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4095, 14, 0, 4030, 32, 0, 0, 0, 0, 0, 0, 121, 260, 4095, 0, 1, 0, 0, 0, 3870, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 4, 1, 0, 1, 4071, 22, 0, 0, 0, 0, 0, 0, 0, 4026, 193, 1, 3878, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8181, 8196, 8175, 8354, 8191, 8196, 8099] avg=8192.0000 max=8354 max_rank=4 min=8099 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32840 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4089, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 19, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 24, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 21, 0, 4096, 0, 0, 4031, 32, 0, 0, 0, 0, 0, 0, 98, 317, 4096, 1, 0, 0, 0, 1, 3870, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 22, 0, 0, 0, 0, 0, 0, 0, 4054, 194, 0, 3692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8201, 8199, 8198, 8390, 8191, 8189, 7940] avg=8192.0000 max=8390 max_rank=4 min=7940 min_rank=7 imbalance_ratio=1.024170 max_load_reduction=57146 max_load_reduction_pct=0.871979 moved_to_echo=32710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 3931, 1, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4086, 5, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 24, 1, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 2, 6, 1, 4095, 2, 0, 4030, 32, 0, 0, 0, 0, 0, 0, 249, 232, 4095, 0, 1, 0, 0, 0, 3870, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 1, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 3, 1, 0, 1, 4070, 22, 0, 0, 0, 0, 0, 0, 0, 3928, 193, 0, 3946, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 8188, 8189, 8183, 8454, 8190, 8194, 8068] avg=8192.0000 max=8454 max_rank=4 min=8068 min_rank=7 imbalance_ratio=1.031982 max_load_reduction=57082 max_load_reduction_pct=0.871002 moved_to_echo=32906 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4080, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 24, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 18, 0, 4096, 0, 0, 4031, 32, 0, 0, 0, 0, 0, 0, 6, 121, 4096, 0, 0, 0, 0, 4, 3870, 7, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4070, 22, 0, 0, 0, 0, 0, 0, 0, 4042, 194, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8188, 8189, 8207, 8104, 8194, 8188, 8247] avg=8192.0000 max=8247 max_rank=7 min=8104 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32733 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4095, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 0, 8, 0, 4096, 0, 0, 4038, 38, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 1, 0, 0, 0, 1, 3875, 10, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 4073, 27, 0, 0, 0, 0, 0, 0, 0, 4089, 183, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8185, 8185, 8202, 7987, 8196, 8197, 8362] avg=8192.0000 max=8362 max_rank=7 min=7987 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32384, 1664, 128, 31744, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 32768, 32512, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 256, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 32768, 0, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 128, 128, 128, 128, 32768, 128, 0, 32256, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 128, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 1152, 32768, 128, 128, 0, 0, 128, 30976, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 0, 56, 6, 0, 113, 0, 0, 1, 1, 0, 3639, 1, 4095, 1, 4084, 1, 1061, 1, 0, 4093, 1, 1, 1, 39, 4031, 9, 1, 4095, 1, 0, 4074, 0, 0, 1, 2, 4095, 4095, 3430, 0, 4095, 4093, 0, 0, 1, 0, 4, 1, 0, 0, 4095, 1, 1, 111, 2, 0, 3913, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 10, 0, 0, 1, 1638, 3, 0, 0, 0, 0, 4065, 0, 4096, 0, 4075, 0, 14, 0, 0, 4086, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 4058, 2508, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 35, 0, 4096, 0, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 3, 0, 0, 6, 970, 1, 0, 0, 0, 0, 3747, 0, 4096, 0, 4035, 0, 109, 0, 0, 4083, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4089, 4063, 3456, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 25, 0, 4092, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 5, 0, 0, 2, 438, 3, 0, 0, 0, 0, 3974, 0, 4096, 0, 4045, 0, 36, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 4094, 4089, 3793, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 4094, 0, 0, 0, 12, 0, 4093, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[3, 0, 0, 4094, 0, 0, 0, 14, 0, 2, 257, 0, 0, 1, 1, 0, 4044, 1, 4095, 1, 4095, 1, 353, 1, 0, 4089, 1, 1, 1, 0, 4094, 1, 0, 4095, 0, 0, 4090, 0, 0, 1, 12, 4095, 4053, 3677, 0, 4095, 4095, 0, 0, 1, 0, 5, 1, 0, 0, 4095, 1, 1, 4, 115, 0, 3855, 1, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 3, 0, 0, 1, 164, 5, 0, 0, 0, 0, 4080, 0, 4096, 0, 4082, 0, 8, 0, 0, 4082, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 3949, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 31, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[3, 0, 0, 4095, 0, 0, 0, 54, 0, 0, 182, 0, 0, 1, 1, 1, 3794, 1, 4095, 1, 4092, 0, 852, 1, 0, 4092, 1, 1, 1, 2, 3941, 5, 1, 4095, 0, 0, 4075, 0, 0, 1, 8, 4095, 4086, 3494, 0, 4095, 4094, 0, 7, 1, 0, 3, 1, 0, 0, 4095, 1, 1, 54, 71, 0, 3947, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 3, 0, 0, 5, 51, 1, 0, 0, 0, 0, 4094, 0, 4096, 0, 4095, 0, 2, 0, 0, 4074, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 4092, 4077, 4070, 0, 4095, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 0, 4095, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 10, 0, 3528, 484, 64, 0, 0, 0, 0, 0, 0, 1, 1638, 3, 0, 0, 0, 0, 4096, 3615, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 3958, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4095, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 567, 4058, 2508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 3612, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4096, 0, 4085, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 9353, 8037, 8214, 8198, 7133, 8189, 8230] avg=8192.0000 max=9353 max_rank=1 min=7133 min_rank=5 imbalance_ratio=1.141724 max_load_reduction=56183 max_load_reduction_pct=0.857285 moved_to_echo=31750 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 0, 56, 3528, 482, 64, 0, 0, 0, 0, 0, 6, 0, 113, 0, 0, 1, 1, 0, 4095, 3614, 0, 0, 0, 0, 0, 0, 3639, 1, 0, 1, 3967, 1, 1061, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 1, 1, 39, 4031, 9, 33, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 4074, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 2, 567, 4095, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 1, 0, 0, 4095, 3611, 481, 0, 0, 0, 0, 0, 0, 1, 1, 111, 2, 0, 3913, 0, 4094, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 7830, 8671, 8208, 8178, 8094, 8193, 8136] avg=8192.0000 max=8671 max_rank=2 min=7830 min_rank=1 imbalance_ratio=1.058472 max_load_reduction=56865 max_load_reduction_pct=0.867691 moved_to_echo=32601 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4094, 0, 0, 0, 14, 3528, 483, 64, 0, 0, 0, 0, 0, 0, 2, 257, 0, 0, 1, 1, 0, 4095, 3613, 0, 0, 0, 0, 0, 0, 4044, 1, 0, 1, 3978, 1, 353, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 1, 1, 0, 4094, 1, 33, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4090, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 12, 567, 4053, 3677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 1, 0, 0, 4095, 3612, 482, 0, 0, 0, 0, 0, 0, 1, 1, 4, 115, 0, 3855, 1, 4094, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 7969, 8379, 8220, 8192, 8309, 8196, 8085] avg=8192.0000 max=8379 max_rank=2 min=7969 min_rank=1 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32782 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 3, 0, 3523, 483, 63, 0, 0, 0, 0, 0, 0, 6, 970, 1, 0, 0, 0, 0, 4096, 3615, 0, 0, 0, 0, 0, 0, 3747, 0, 0, 0, 3920, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4086, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 566, 4063, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 3613, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4092, 0, 4095, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8688, 7776, 8202, 8198, 8085, 8190, 8225] avg=8192.0000 max=8688 max_rank=1 min=7776 min_rank=2 imbalance_ratio=1.060547 max_load_reduction=56848 max_load_reduction_pct=0.867432 moved_to_echo=32698 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 5, 0, 3527, 483, 64, 0, 0, 0, 0, 0, 0, 2, 438, 3, 0, 0, 0, 0, 4096, 3615, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 3928, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 567, 4089, 3793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4094, 3613, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4093, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8154, 7938, 8215, 8195, 8449, 8191, 8215] avg=8192.0000 max=8449 max_rank=5 min=7938 min_rank=2 imbalance_ratio=1.031372 max_load_reduction=57087 max_load_reduction_pct=0.871078 moved_to_echo=33050 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 3, 0, 3529, 483, 64, 0, 0, 0, 0, 0, 0, 1, 164, 5, 0, 0, 0, 0, 4096, 3615, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 3965, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4080, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 567, 4088, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 3613, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 4096, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 7881, 8053, 8195, 8198, 8604, 8189, 8237] avg=8192.0000 max=8604 max_rank=5 min=7881 min_rank=1 imbalance_ratio=1.050293 max_load_reduction=56932 max_load_reduction_pct=0.868713 moved_to_echo=33228 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4095, 0, 0, 0, 54, 3528, 484, 64, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 1, 1, 1, 4095, 3614, 0, 0, 0, 0, 0, 0, 3794, 1, 0, 1, 3975, 0, 852, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 1, 2, 3941, 5, 33, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4075, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 8, 567, 4086, 3494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 3, 1, 0, 0, 4095, 3610, 481, 0, 0, 0, 0, 0, 0, 1, 1, 54, 71, 0, 3947, 0, 4095, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 7894, 8624, 8076, 8178, 8155, 8198, 8183] avg=8192.0000 max=8624 max_rank=2 min=7894 min_rank=1 imbalance_ratio=1.052734 max_load_reduction=56912 max_load_reduction_pct=0.868408 moved_to_echo=32714 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 3, 0, 3532, 484, 69, 0, 0, 0, 0, 0, 0, 5, 51, 1, 0, 0, 0, 0, 4096, 3614, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3962, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 4094, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 560, 4077, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 3612, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4095, 0, 4093, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 7767, 8058, 8206, 8199, 8707, 8190, 8225] avg=8192.0000 max=8707 max_rank=5 min=7767 min_rank=1 imbalance_ratio=1.062866 max_load_reduction=56829 max_load_reduction_pct=0.867142 moved_to_echo=33321 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 32768, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[31488, 128, 0, 128, 31744, 128, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 4608, 32640, 28416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 256, 384, 0, 32256, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 128, 128, 28288, 3968, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 3840, 128, 0, 128, 128, 128, 32768, 28928, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 32768, 28928, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 32640, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4096, 3850, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4096, 170, 0, 4089, 0, 0, 0, 46, 0, 0, 0, 0, 4095, 0, 0, 1, 22, 0, 0, 4096, 0, 0, 16, 0, 0, 0, 0, 4095, 1, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4092, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 4096, 3815, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4094, 36, 0, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 2, 218, 0, 0, 4096, 0, 0, 27, 0, 0, 0, 0, 4096, 4, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 4054, 1, 0, 13, 1, 29, 4095, 4085, 1, 0, 3, 1, 4077, 3846, 1, 1, 1, 1, 0, 1, 4094, 1, 1, 1, 0, 1, 0, 4093, 4095, 37, 1, 4091, 7, 1, 0, 49, 0, 0, 0, 1, 4094, 0, 1, 0, 100, 0, 5, 4094, 31, 0, 51, 0, 0, 0, 1, 4095, 0, 4095, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4091, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 4096, 3965, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 4093, 84, 0, 4095, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 0, 1, 21, 0, 0, 4096, 0, 0, 14, 0, 0, 0, 0, 4096, 4, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4093, 4096, 0, 0, 0, 0, 2, 4095, 4096, 0, 0, 0, 0, 4096, 3154, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 4095, 645, 0, 4093, 0, 0, 0, 4, 0, 0, 0, 0, 4095, 0, 0, 0, 287, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 4096, 1, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 4073, 0, 1, 16, 1, 0, 4095, 4083, 1, 0, 0, 1, 4075, 3767, 0, 1, 1, 1, 0, 1, 4095, 2, 1, 1, 0, 1, 0, 4094, 4095, 0, 0, 4095, 0, 0, 0, 253, 0, 0, 0, 1, 4095, 0, 1, 16, 12, 0, 4, 4094, 0, 2, 82, 1, 0, 0, 1, 4093, 0, 4095, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4096, 3533, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 4096, 6, 0, 4092, 0, 0, 0, 64, 0, 0, 0, 0, 4095, 0, 0, 0, 474, 0, 0, 4096, 0, 0, 23, 0, 0, 0, 0, 4096, 2, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 4048, 4, 0, 21, 1, 10, 4095, 4077, 1, 0, 0, 1, 4069, 3926, 1, 1, 1, 1, 0, 1, 4095, 0, 1, 1, 0, 1, 0, 4094, 4095, 25, 1, 4093, 2, 1, 0, 18, 0, 0, 0, 1, 4095, 0, 1, 14, 51, 2, 2, 4094, 29, 1, 83, 0, 0, 0, 1, 4093, 0, 4095, 0, 4, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 0, 0, 0, 0, 0, 0, 4096, 360, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 4089, 0, 0, 0, 46, 0, 3726, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 22, 3897, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8200, 8307, 8194, 8221, 8062, 8161, 8192] avg=8192.0000 max=8307 max_rank=2 min=8062 min_rank=5 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 4095, 4073, 0, 1, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4083, 1, 0, 0, 1, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 0, 1, 1, 1, 0, 1, 4095, 360, 1, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 1, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 253, 0, 3725, 190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 16, 12, 3897, 46, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 0, 2, 82, 1, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8168, 8227, 8194, 8263, 8068, 8232, 8188] avg=8192.0000 max=8263 max_rank=4 min=8068 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 0, 0, 0, 0, 0, 0, 4096, 360, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 4095, 0, 0, 0, 14, 0, 3726, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 21, 3896, 46, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8201, 8422, 8192, 8108, 8060, 8160, 8196] avg=8192.0000 max=8422 max_rank=2 min=8060 min_rank=5 imbalance_ratio=1.028076 max_load_reduction=57114 max_load_reduction_pct=0.871490 moved_to_echo=32524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 4095, 4054, 1, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 29, 8, 4085, 1, 0, 3, 1, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 1, 1, 1, 1, 0, 1, 4094, 360, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 4091, 7, 1, 0, 49, 0, 3725, 190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 100, 3897, 46, 0, 0, 0, 0, 0, 0, 0, 5, 4094, 31, 0, 51, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8204, 8306, 8192, 8101, 8139, 8230, 8190] avg=8192.0000 max=8306 max_rank=2 min=8101 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32660 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3815, 0, 0, 0, 0, 0, 0, 4096, 360, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4093, 0, 0, 0, 2, 0, 3726, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 218, 3896, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 27, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8201, 8272, 8192, 8046, 8259, 8172, 8196] avg=8192.0000 max=8272 max_rank=2 min=8046 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3154, 0, 0, 0, 0, 0, 0, 4096, 360, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 645, 0, 4093, 0, 0, 0, 4, 0, 3726, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 287, 3897, 46, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8202, 7611, 8194, 8658, 8325, 8154, 8193] avg=8192.0000 max=8658 max_rank=4 min=7611 min_rank=2 imbalance_ratio=1.056885 max_load_reduction=56878 max_load_reduction_pct=0.867889 moved_to_echo=33330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 4095, 4048, 4, 0, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8, 4077, 1, 0, 0, 1, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 1, 1, 1, 1, 0, 1, 4095, 360, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 25, 1, 4093, 2, 1, 0, 18, 0, 3725, 190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 14, 51, 3897, 47, 0, 0, 0, 0, 0, 0, 2, 2, 4094, 29, 1, 83, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 4094, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8166, 8387, 8192, 8055, 8106, 8259, 8192] avg=8192.0000 max=8387 max_rank=2 min=8055 min_rank=4 imbalance_ratio=1.023804 max_load_reduction=57149 max_load_reduction_pct=0.872025 moved_to_echo=32612 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3533, 0, 0, 0, 0, 0, 0, 4096, 362, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4092, 0, 0, 0, 64, 0, 3730, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 474, 3901, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8194, 8004, 8186, 8084, 8517, 8168, 8189] avg=8192.0000 max=8517 max_rank=5 min=8004 min_rank=2 imbalance_ratio=1.039673 max_load_reduction=57019 max_load_reduction_pct=0.870041 moved_to_echo=32958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 32768, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 32768, 0, 128, 128, 1280, 31232, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 384, 128, 0, 32512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1024, 128, 32768, 128, 128, 0, 512, 0, 29824, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[29952, 128, 128, 128, 128, 0, 128, 32768, 2944, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4093, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3970, 4036, 0, 0, 0, 4055, 3, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 34, 152, 0, 1, 0, 0, 0, 39, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 4092, 0, 0, 0, 4091, 2, 0, 4096, 4096, 0, 0, 4096, 0, 4095, 4096, 11, 191, 0, 1, 0, 0, 0, 22, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4094, 4092, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4004, 4095, 0, 0, 0, 4090, 2, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 9, 63, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4073, 0, 31, 1, 1, 0, 4095, 4095, 46, 1, 1, 1, 0, 0, 1, 1, 0, 0, 3919, 4095, 1, 63, 0, 4093, 0, 1, 4090, 4095, 0, 10, 4095, 2, 4094, 4095, 0, 9, 0, 0, 1, 1, 1, 158, 3, 5, 0, 1, 0, 0, 4094, 4095, 0, 84, 1, 4079, 0, 0, 3952, 1, 1, 4049, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 4096, 0, 0, 0, 4082, 3, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 13, 131, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4084, 0, 1, 1, 1, 0, 4094, 4096, 12, 1, 1, 0, 0, 0, 1, 1, 0, 2, 3993, 4094, 1, 52, 0, 4095, 0, 1, 4093, 4095, 0, 0, 4095, 8, 4095, 4095, 26, 3, 0, 0, 0, 1, 1, 168, 0, 5, 0, 2, 0, 0, 4094, 4095, 0, 73, 1, 4074, 0, 0, 3948, 1, 1, 4030, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4079, 0, 10, 1, 1, 0, 4095, 4095, 64, 1, 1, 0, 0, 0, 1, 1, 0, 0, 3870, 4095, 1, 45, 1, 4095, 0, 1, 4090, 4095, 0, 4, 4095, 3, 4095, 4095, 0, 7, 0, 0, 0, 1, 1, 130, 49, 3, 0, 1, 0, 0, 4094, 4095, 0, 59, 1, 4082, 0, 0, 4006, 1, 1, 4069, 0, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4086, 4007, 0, 0, 0, 3966, 1, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 18, 150, 0, 1, 0, 0, 0, 58, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 139, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3995, 34, 152, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 1, 4092, 3897, 60, 17, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 41, 20, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8176, 8164, 8193, 8278, 8106, 8172, 8255] avg=8192.0000 max=8278 max_rank=4 min=8106 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32811 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 141, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3995, 9, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 3, 4091, 3954, 60, 17, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 41, 20, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8176, 8235, 8192, 8163, 8155, 8172, 8252] avg=8192.0000 max=8252 max_rank=7 min=8155 min_rank=5 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32742 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 31, 1, 1, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 1, 1, 0, 0, 1, 1, 4095, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 141, 1, 63, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 4093, 0, 10, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3994, 0, 9, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 158, 3, 5, 0, 1, 0, 0, 4092, 3954, 60, 17, 0, 0, 0, 0, 4075, 0, 0, 84, 1, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 1, 1, 4049, 0, 1, 0, 0, 41, 19, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8224, 8217, 8196, 8099, 8291, 8239, 8066] avg=8192.0000 max=8291 max_rank=5 min=8066 min_rank=7 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32695 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 1, 1, 1, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 1, 0, 0, 0, 1, 1, 4095, 4078, 0, 0, 0, 0, 0, 0, 0, 2, 3993, 141, 1, 52, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 4093, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3994, 26, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 168, 0, 5, 0, 2, 0, 0, 4091, 3953, 60, 17, 0, 0, 0, 0, 4075, 0, 0, 73, 1, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 1, 1, 4030, 0, 1, 0, 0, 41, 19, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8189, 8284, 8195, 8119, 8297, 8223, 8043] avg=8192.0000 max=8297 max_rank=5 min=8043 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32682 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 141, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3995, 11, 191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 1, 4092, 3951, 60, 17, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 41, 20, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8178, 8111, 8192, 8293, 8143, 8172, 8255] avg=8192.0000 max=8293 max_rank=4 min=8111 min_rank=2 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32863 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 141, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3995, 13, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2, 4091, 3955, 60, 17, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 41, 20, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8175, 8184, 8193, 8235, 8130, 8172, 8254] avg=8192.0000 max=8254 max_rank=7 min=8130 min_rank=5 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32791 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4079, 0, 10, 1, 1, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1, 1, 0, 0, 0, 1, 1, 4095, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 141, 1, 45, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 4093, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3994, 0, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 130, 49, 3, 0, 1, 0, 0, 4093, 3954, 60, 17, 0, 0, 0, 0, 4075, 0, 0, 59, 1, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 1, 1, 4069, 0, 1, 1, 0, 41, 19, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8241, 8153, 8191, 8097, 8308, 8217, 8141] avg=8192.0000 max=8308 max_rank=5 min=8097 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 136, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3987, 18, 150, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 3, 4095, 3871, 63, 16, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 46, 23, 9, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8177, 8188, 8184, 8252, 8106, 8169, 8270] avg=8192.0000 max=8270 max_rank=7 min=8106 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32797 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32640, 0, 0, 256, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 31744, 1152, 128, 256, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32000, 128, 768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 640, 128, 128, 0, 128, 0, 128, 32768, 31616, 512, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32384, 128, 128, 32640, 0, 128, 128, 0, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[278, 4096, 0, 459, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4084, 1206, 4096, 0, 0, 0, 4084, 0, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 4095, 4096, 0, 3715, 0, 4086, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 2547, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[318, 4093, 0, 1817, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4094, 179, 4095, 523, 4095, 0, 0, 0, 4095, 0, 1, 4095, 0, 0, 1, 6, 4095, 4095, 0, 1, 0, 1, 1, 0, 0, 0, 0, 4095, 4085, 0, 4095, 4059, 1, 3325, 10, 3974, 0, 1, 1, 0, 1, 0, 28, 0, 4095, 1, 2154, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[230, 4096, 0, 1403, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4090, 1170, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4095, 4096, 0, 3404, 0, 4085, 0, 0, 0, 0, 0, 0, 76, 0, 4096, 0, 1924, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1589, 4096, 1, 806, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 10, 4087, 880, 4096, 0, 0, 0, 4094, 0, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 2, 4095, 4093, 0, 2484, 0, 4084, 0, 0, 0, 0, 0, 1, 97, 0, 4096, 0, 2349, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[165, 4090, 0, 2109, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 4095, 156, 4095, 518, 4094, 0, 0, 1, 4095, 0, 1, 4095, 0, 0, 1, 5, 4094, 4095, 0, 1, 0, 1, 1, 0, 0, 0, 0, 4095, 4093, 0, 4095, 4050, 1, 3909, 17, 3942, 0, 1, 1, 0, 1, 0, 0, 1, 4095, 2, 1514, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[60, 4096, 0, 939, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 4091, 2717, 4096, 0, 0, 0, 4095, 0, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 4096, 4095, 0, 3099, 0, 4081, 0, 0, 0, 0, 0, 0, 54, 0, 4096, 0, 1338, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1793, 4096, 0, 94, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 28, 4094, 2503, 4096, 0, 0, 0, 4093, 0, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4095, 4095, 0, 3606, 0, 4093, 0, 0, 0, 0, 0, 0, 33, 0, 4096, 0, 141, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[171, 4092, 0, 2244, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4095, 189, 4095, 569, 4095, 0, 0, 0, 4095, 0, 1, 4095, 0, 0, 1, 3, 4094, 4095, 0, 1, 0, 1, 1, 1, 0, 1, 0, 4095, 4092, 0, 4095, 4056, 1, 3562, 8, 3912, 0, 1, 1, 1, 0, 0, 4, 1, 4095, 1, 1665, 0, 3, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[278, 4096, 0, 459, 0, 10, 0, 0, 1250, 1029, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 4084, 1206, 0, 0, 0, 0, 2834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 723, 3, 0, 4096, 0, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4096, 59, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 2547, 0, 0, 0, 2344, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7124, 8120, 8124, 8191, 8192, 8537, 8241, 9007] avg=8192.0000 max=9007 max_rank=7 min=7124 min_rank=0 imbalance_ratio=1.099487 max_load_reduction=56529 max_load_reduction_pct=0.862564 moved_to_echo=33977 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[230, 4096, 0, 1403, 0, 3, 0, 0, 1254, 1029, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 4090, 1170, 0, 0, 0, 0, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 723, 4, 0, 4096, 0, 3404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4096, 58, 1, 0, 0, 0, 0, 0, 76, 0, 4096, 0, 1924, 0, 0, 0, 2344, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8017, 8120, 8101, 8190, 8191, 8227, 8240, 8450] avg=8192.0000 max=8450 max_rank=7 min=8017 min_rank=0 imbalance_ratio=1.031494 max_load_reduction=57086 max_load_reduction_pct=0.871063 moved_to_echo=33108 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[318, 4093, 0, 1817, 1, 0, 0, 0, 1254, 1026, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4094, 179, 4027, 0, 0, 0, 0, 0, 0, 0, 4095, 523, 0, 0, 0, 0, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 1, 6, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 722, 0, 0, 4059, 1, 3325, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3974, 0, 1, 1, 0, 1, 0, 4095, 58, 1, 0, 0, 0, 0, 0, 28, 0, 4095, 1, 2154, 0, 2, 1, 2337, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8510, 8302, 7459, 8197, 8192, 8107, 8141, 8628] avg=8192.0000 max=8628 max_rank=7 min=7459 min_rank=2 imbalance_ratio=1.053223 max_load_reduction=56908 max_load_reduction_pct=0.868347 moved_to_echo=33068 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1589, 4096, 1, 806, 0, 3, 0, 0, 1253, 1029, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 10, 4027, 0, 0, 0, 0, 0, 0, 0, 4087, 880, 0, 0, 0, 0, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 723, 2, 0, 4093, 0, 2484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 1, 4095, 58, 1, 0, 0, 0, 0, 0, 97, 0, 4096, 0, 2349, 0, 0, 0, 2344, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8779, 8130, 7808, 8191, 8191, 7302, 8239, 8896] avg=8192.0000 max=8896 max_rank=7 min=7302 min_rank=5 imbalance_ratio=1.085938 max_load_reduction=56640 max_load_reduction_pct=0.864258 moved_to_echo=32628 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[165, 4090, 0, 2109, 1, 0, 1, 1, 1254, 1028, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 156, 4027, 0, 0, 0, 0, 0, 0, 0, 4095, 518, 0, 0, 0, 1, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 1, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 723, 0, 0, 4050, 1, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3942, 0, 1, 1, 0, 1, 0, 4095, 58, 1, 0, 0, 0, 0, 0, 0, 1, 4095, 2, 1514, 0, 2, 1, 2342, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8650, 8279, 7455, 8195, 8191, 8683, 8116, 7967] avg=8192.0000 max=8683 max_rank=5 min=7455 min_rank=2 imbalance_ratio=1.059937 max_load_reduction=56853 max_load_reduction_pct=0.867508 moved_to_echo=32957 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[60, 4096, 0, 939, 0, 5, 0, 0, 1254, 1029, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 4091, 2717, 0, 0, 0, 0, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 723, 3, 0, 4095, 0, 3099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 4096, 58, 0, 0, 0, 0, 0, 0, 54, 0, 4096, 0, 1338, 0, 0, 0, 2344, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7385, 8121, 9649, 8191, 8192, 7920, 8235, 7843] avg=8192.0000 max=9649 max_rank=2 min=7385 min_rank=0 imbalance_ratio=1.177856 max_load_reduction=55887 max_load_reduction_pct=0.852768 moved_to_echo=32190 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[171, 4092, 0, 2244, 1, 0, 1, 0, 1254, 1028, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 189, 4027, 0, 0, 0, 0, 0, 0, 0, 4095, 569, 0, 0, 0, 0, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 1, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 723, 0, 0, 4056, 1, 3562, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3912, 0, 1, 1, 1, 0, 0, 4095, 59, 0, 0, 0, 0, 0, 0, 4, 1, 4095, 1, 1665, 0, 3, 1, 2341, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8792, 8312, 7505, 8193, 8195, 8342, 8077, 8120] avg=8192.0000 max=8792 max_rank=0 min=7505 min_rank=2 imbalance_ratio=1.073242 max_load_reduction=56744 max_load_reduction_pct=0.865845 moved_to_echo=32734 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1793, 4096, 0, 94, 0, 2, 0, 0, 1255, 1034, 5, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 28, 4028, 0, 0, 0, 0, 0, 0, 0, 4094, 2503, 0, 0, 0, 0, 2838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 716, 1, 0, 4095, 0, 3606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 58, 0, 0, 0, 0, 0, 0, 33, 0, 4096, 0, 141, 0, 0, 0, 2346, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8152, 9435, 8188, 8192, 8418, 8247, 6625] avg=8192.0000 max=9435 max_rank=2 min=6625 min_rank=7 imbalance_ratio=1.151733 max_load_reduction=56101 max_load_reduction_pct=0.856033 moved_to_echo=31482 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 5888, 128, 0, 32640, 128, 27136, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[4608, 32768, 128, 9984, 128, 128, 128, 128, 10112, 8320, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 32384, 0, 128, 128, 128, 128, 128, 32768, 512, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 10112, 0, 0, 0, 128, 22784, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 0, 128, 32768, 640, 32256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 128, 32768, 128, 13696, 0, 128, 128, 18816, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 1763, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4083, 2413, 0, 4096, 4095, 4096, 0, 4087, 4095, 0, 3, 4096, 4096, 6, 0, 0, 0, 1, 0, 4095, 4096, 0, 0, 4090, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 1, 80, 1, 0, 1, 0, 3887, 0, 1, 0, 0, 45, 0, 601, 2, 0, 1, 34, 1, 1, 0, 1, 93, 1, 5, 4090, 4, 0, 1, 4095, 6, 0, 0, 4084, 0, 1, 4092, 3562, 4, 4095, 4090, 4080, 1, 4008, 4092, 0, 0, 4087, 4092, 0, 2, 8, 1, 0, 1, 4092, 4095, 0, 0, 4095, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 1440, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4096, 1, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4087, 2703, 0, 4096, 4095, 4096, 0, 4090, 4096, 0, 2, 4094, 4096, 3, 0, 0, 0, 1, 0, 4095, 4096, 0, 1, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 1, 0, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 4061, 2938, 0, 4096, 4096, 4096, 0, 4063, 4096, 0, 4, 4096, 4096, 4, 0, 0, 0, 1, 0, 4094, 4096, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 1, 50, 0, 0, 1, 0, 3908, 0, 1, 0, 0, 82, 0, 57, 2, 0, 1, 56, 1, 1, 0, 1, 85, 1, 14, 4092, 3, 0, 1, 4095, 4, 0, 0, 4089, 0, 0, 4022, 4071, 4, 4095, 4087, 4094, 0, 4057, 4093, 0, 0, 4078, 4094, 0, 1, 6, 1, 0, 1, 4095, 4095, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 0, 1220, 0, 0, 0, 2, 0, 0, 0, 0, 32, 0, 2, 4096, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 4076, 2867, 0, 4096, 4096, 4096, 0, 4095, 4096, 0, 3, 4095, 4096, 5, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 1, 0, 0, 0, 0, 330, 0, 1, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4092, 3416, 0, 4096, 4095, 4096, 0, 4094, 4096, 0, 1, 4096, 4096, 3, 0, 0, 0, 1, 0, 4095, 4096, 0, 1, 4094, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 1, 72, 1, 0, 1, 0, 3977, 0, 1, 0, 0, 39, 0, 510, 4, 0, 1, 24, 1, 1, 0, 1, 15, 1, 4, 4090, 7, 0, 1, 4095, 3, 0, 0, 4087, 0, 1, 4087, 3629, 3, 4095, 4089, 4088, 0, 4037, 4092, 0, 0, 4088, 4093, 0, 3, 7, 1, 0, 1, 4095, 4095, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 4092, 4059, 17, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 1763, 3179, 72, 4, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 0, 0, 4083, 2413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4094, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 9050, 8179, 8121, 7413, 8215, 8200, 8189] avg=8192.0000 max=9050 max_rank=1 min=7413 min_rank=4 imbalance_ratio=1.104736 max_load_reduction=56486 max_load_reduction_pct=0.861908 moved_to_echo=32017 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 80, 1, 0, 1, 0, 4091, 4054, 15, 0, 0, 0, 0, 0, 3887, 0, 1, 0, 0, 45, 0, 601, 3170, 72, 4, 2, 0, 0, 0, 0, 2, 0, 1, 34, 1, 1, 0, 1, 4095, 4077, 0, 0, 0, 0, 0, 0, 93, 1, 5, 4090, 4, 0, 1, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 914, 0, 1, 4092, 3562, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 32, 4080, 1, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4090, 0, 2, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 4, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 7782, 8212, 8217, 8575, 8125, 8188, 8193] avg=8192.0000 max=8575 max_rank=4 min=7782 min_rank=1 imbalance_ratio=1.046753 max_load_reduction=56961 max_load_reduction_pct=0.869156 moved_to_echo=33081 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 50, 0, 0, 1, 0, 4091, 4051, 15, 0, 0, 0, 0, 0, 3908, 0, 1, 0, 0, 82, 0, 57, 3174, 72, 4, 2, 0, 0, 0, 0, 2, 0, 1, 56, 1, 1, 0, 1, 4095, 4078, 0, 0, 0, 0, 0, 0, 85, 1, 14, 4092, 3, 0, 1, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 915, 0, 0, 4022, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 32, 4094, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4092, 0, 1, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 7300, 8235, 8219, 9012, 8187, 8178, 8195] avg=8192.0000 max=9012 max_rank=4 min=7300 min_rank=1 imbalance_ratio=1.100098 max_load_reduction=56524 max_load_reduction_pct=0.862488 moved_to_echo=33572 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 4060, 16, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 1, 0, 1222, 3179, 72, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 0, 0, 4061, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4094, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8574, 8176, 8122, 7916, 8191, 8199, 8190] avg=8192.0000 max=8574 max_rank=1 min=7916 min_rank=4 imbalance_ratio=1.046631 max_load_reduction=56962 max_load_reduction_pct=0.869171 moved_to_echo=32496 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 4059, 16, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 1440, 3179, 72, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 0, 0, 4087, 2703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 4094, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8757, 8177, 8121, 7707, 8218, 8194, 8195] avg=8192.0000 max=8757 max_rank=1 min=7707 min_rank=4 imbalance_ratio=1.068970 max_load_reduction=56779 max_load_reduction_pct=0.866379 moved_to_echo=32314 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 4060, 16, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 0, 1220, 3179, 72, 4, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 32, 0, 2, 4096, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 0, 0, 4076, 2867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8567, 8178, 8152, 7860, 8223, 8197, 8191] avg=8192.0000 max=8567 max_rank=1 min=7860 min_rank=4 imbalance_ratio=1.045776 max_load_reduction=56969 max_load_reduction_pct=0.869278 moved_to_echo=32471 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 72, 1, 0, 1, 0, 4092, 4053, 16, 0, 0, 0, 0, 0, 3977, 0, 1, 0, 0, 39, 0, 510, 3172, 72, 4, 2, 0, 0, 0, 0, 4, 0, 1, 24, 1, 1, 0, 1, 4095, 4076, 0, 0, 0, 0, 0, 0, 15, 1, 4, 4090, 7, 0, 1, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 915, 0, 1, 4087, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 32, 4088, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4091, 0, 3, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 3, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 7777, 8203, 8141, 8635, 8160, 8190, 8193] avg=8192.0000 max=8635 max_rank=4 min=7777 min_rank=1 imbalance_ratio=1.054077 max_load_reduction=56901 max_load_reduction_pct=0.868240 moved_to_echo=33178 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4060, 17, 0, 0, 0, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 517, 3186, 79, 8, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 330, 0, 1, 4096, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 910, 0, 0, 4092, 3416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4089, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 7729, 8176, 8443, 8418, 8217, 8190, 8190] avg=8192.0000 max=8443 max_rank=3 min=7729 min_rank=1 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=33015 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 256, 32768, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 256, 128, 0, 128, 0, 32768, 32512, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 7424, 0, 128, 32640, 25600, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 128, 128, 32768, 128, 0, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32000, 0, 128, 128, 0, 256, 0, 7424, 25472, 640, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[264, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 2, 0, 1, 0, 1889, 0, 13, 0, 0, 1, 0, 4094, 0, 2284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4096, 0, 0, 4094, 0, 0, 0, 4060, 0, 3780, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[168, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 2, 0, 1, 0, 1177, 0, 5, 1, 0, 1, 0, 4086, 0, 2859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4096, 0, 0, 4095, 0, 0, 0, 4049, 0, 4057, 4093, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[92, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 946, 0, 0, 0, 0, 0, 0, 4083, 0, 3217, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4096, 0, 0, 4093, 0, 0, 0, 3991, 0, 4070, 4096, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1000, 0, 0, 0, 23, 4095, 4076, 0, 0, 3, 0, 4095, 2, 0, 4080, 1, 4095, 0, 14, 0, 0, 1768, 8, 1950, 0, 1, 0, 19, 1533, 1, 3220, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 3712, 0, 1, 2, 0, 4086, 4095, 1, 4093, 4094, 0, 1, 4095, 0, 0, 1, 4093, 0, 3200, 4072, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[934, 1, 0, 0, 38, 4095, 4078, 0, 1, 2, 0, 4095, 2, 0, 4056, 1, 4095, 0, 28, 0, 0, 1151, 17, 1290, 0, 1, 0, 10, 2342, 1, 3452, 1, 0, 0, 1, 1, 2, 0, 1, 1, 0, 3795, 0, 1, 1, 0, 4083, 4095, 0, 4094, 4094, 0, 1, 4095, 1, 0, 1, 4090, 0, 3418, 4069, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[403, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 1486, 0, 44, 1, 0, 0, 0, 4048, 0, 2529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 4096, 0, 4094, 4096, 0, 0, 4092, 0, 0, 0, 3977, 0, 3913, 4096, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[245, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 2, 0, 1, 0, 2612, 0, 54, 1, 0, 1, 0, 3494, 0, 2345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4096, 0, 0, 4094, 0, 0, 0, 3975, 0, 3998, 4095, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1522, 1, 0, 0, 44, 4095, 4081, 0, 1, 2, 0, 4095, 1, 0, 4057, 1, 4095, 0, 17, 0, 0, 1998, 12, 728, 0, 1, 0, 14, 2277, 1, 2358, 1, 0, 0, 1, 1, 2, 0, 1, 1, 0, 3795, 0, 1, 1, 0, 4081, 4095, 0, 4093, 4095, 0, 1, 4095, 1, 0, 1, 4092, 0, 3700, 4076, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[264, 0, 0, 0, 0, 3512, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 1, 0, 1889, 0, 13, 1372, 574, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 2284, 0, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3920, 177, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 3780, 385, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7872, 8201, 7947, 8541, 8190, 8367, 8192, 8226] avg=8192.0000 max=8541 max_rank=3 min=7872 min_rank=0 imbalance_ratio=1.042603 max_load_reduction=56995 max_load_reduction_pct=0.869675 moved_to_echo=32975 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[92, 0, 0, 0, 0, 3512, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 946, 0, 0, 1372, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 3217, 0, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 3920, 177, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 4070, 385, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7700, 8202, 6989, 9462, 8192, 8352, 8191, 8448] avg=8192.0000 max=9462 max_rank=3 min=6989 min_rank=2 imbalance_ratio=1.155029 max_load_reduction=56074 max_load_reduction_pct=0.855621 moved_to_echo=33183 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1000, 0, 0, 0, 23, 3511, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 2, 0, 4080, 1, 10, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 14, 0, 0, 1768, 8, 1950, 1364, 574, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 1533, 1, 3220, 1, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 4091, 3919, 176, 0, 0, 0, 0, 0, 0, 3712, 0, 1, 2, 0, 4086, 176, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 3, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 3200, 383, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8610, 8191, 9773, 6924, 8190, 7977, 8193, 7678] avg=8192.0000 max=9773 max_rank=2 min=6924 min_rank=3 imbalance_ratio=1.192993 max_load_reduction=55763 max_load_reduction_pct=0.850876 moved_to_echo=32038 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[934, 1, 0, 0, 38, 3511, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4095, 2, 0, 4056, 1, 10, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 28, 0, 0, 1151, 17, 1290, 1363, 574, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 2342, 1, 3452, 1, 2147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 1, 1, 4091, 3919, 176, 0, 0, 0, 0, 0, 0, 3795, 0, 1, 1, 0, 4083, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 3418, 383, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8562, 8167, 8518, 7954, 8192, 8056, 8194, 7893] avg=8192.0000 max=8562 max_rank=0 min=7893 min_rank=7 imbalance_ratio=1.045166 max_load_reduction=56974 max_load_reduction_pct=0.869354 moved_to_echo=32335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[168, 0, 0, 0, 0, 3512, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 1, 0, 1177, 0, 5, 1371, 574, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4086, 0, 2859, 0, 2160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3920, 177, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 4057, 385, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7776, 8201, 7226, 9107, 8190, 8351, 8193, 8492] avg=8192.0000 max=9107 max_rank=3 min=7226 min_rank=2 imbalance_ratio=1.111694 max_load_reduction=56429 max_load_reduction_pct=0.861038 moved_to_echo=33226 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[403, 0, 0, 0, 0, 3512, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 1486, 0, 44, 1372, 574, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4048, 0, 2529, 0, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3920, 177, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 3913, 385, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8011, 8202, 7574, 8740, 8190, 8353, 8189, 8277] avg=8192.0000 max=8740 max_rank=3 min=7574 min_rank=2 imbalance_ratio=1.066895 max_load_reduction=56796 max_load_reduction_pct=0.866638 moved_to_echo=33009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1522, 1, 0, 0, 44, 3511, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4095, 1, 0, 4057, 1, 10, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 17, 0, 0, 1998, 12, 728, 1366, 574, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 2277, 1, 2358, 1, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 1, 1, 4092, 3919, 176, 0, 0, 0, 0, 0, 0, 3795, 0, 1, 1, 0, 4081, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 3700, 383, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9159, 8167, 8790, 6803, 8193, 8054, 8193, 8177] avg=8192.0000 max=9159 max_rank=0 min=6803 min_rank=3 imbalance_ratio=1.118042 max_load_reduction=56377 max_load_reduction_pct=0.860245 moved_to_echo=32617 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[245, 0, 0, 0, 0, 3505, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 1, 0, 2612, 0, 54, 1377, 577, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3494, 0, 2345, 0, 2164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3922, 183, 0, 0, 0, 0, 0, 0, 3756, 0, 0, 0, 0, 4096, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 3998, 371, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7846, 8205, 8719, 8005, 8199, 8026, 8191, 8345] avg=8192.0000 max=8719 max_rank=2 min=7846 min_rank=0 imbalance_ratio=1.064331 max_load_reduction=56817 max_load_reduction_pct=0.866959 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32384, 0, 30208, 3072, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4736, 128, 0, 0, 128, 28160, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 25984, 128, 22272, 128, 17280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 31488, 0, 128, 128, 0, 32768, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 128, 32768, 31360, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 0, 13056, 128, 4096, 11008, 4608, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4079, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 30, 4093, 4095, 17, 0, 1, 4096, 0, 0, 0, 0, 3, 0, 4094, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 2, 4096, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4068, 4058, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 37, 4092, 4096, 27, 0, 1, 4096, 0, 0, 0, 0, 5, 0, 4095, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 1, 4096, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[3979, 3993, 11, 0, 0, 0, 5, 0, 1, 1, 1, 1, 1, 0, 4092, 3, 4078, 0, 1, 0, 16, 4095, 4095, 119, 0, 0, 4095, 1, 1, 0, 0, 0, 11, 4092, 0, 4095, 101, 1, 3, 4095, 4095, 10, 0, 4048, 0, 1, 9, 4094, 1, 4095, 1, 0, 1, 0, 4094, 1, 1, 1, 1, 0, 0, 0, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4068, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 87, 4092, 4095, 16, 0, 2, 4096, 0, 0, 0, 0, 7, 0, 4093, 0, 4096, 0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4087, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 21, 4094, 4093, 8, 0, 2, 4096, 0, 0, 0, 0, 5, 0, 4094, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4078, 4085, 18, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 4093, 7, 4026, 0, 1, 0, 2, 4095, 4095, 35, 0, 0, 4095, 1, 1, 0, 0, 0, 6, 4095, 0, 4095, 94, 1, 4, 4094, 4095, 29, 0, 4000, 0, 1, 0, 4091, 1, 4095, 0, 1, 1, 0, 4094, 0, 1, 1, 1, 2, 1, 0, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4085, 4060, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 26, 4094, 4094, 22, 0, 1, 4096, 0, 0, 0, 0, 2, 0, 4095, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 1, 4096, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4069, 4057, 37, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 4091, 8, 4060, 0, 1, 0, 4, 4095, 4095, 33, 0, 0, 4095, 1, 1, 0, 0, 0, 9, 4095, 0, 4095, 99, 1, 2, 4094, 4095, 20, 0, 3986, 0, 1, 0, 4092, 1, 4095, 0, 1, 1, 0, 4094, 0, 1, 1, 3, 0, 0, 0, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4079, 4066, 0, 0, 0, 0, 0, 0, 39, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 30, 4049, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3, 4071, 17, 3, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8191, 8192, 8191, 8151, 8218, 8192, 8190] avg=8192.0000 max=8218 max_rank=5 min=8151 min_rank=4 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3979, 3993, 11, 0, 0, 0, 5, 0, 38, 27, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4092, 3, 4092, 1, 0, 0, 0, 0, 0, 0, 4078, 0, 1, 0, 16, 4051, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 4071, 17, 3, 0, 0, 0, 0, 0, 11, 4092, 0, 0, 101, 1, 3, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 24, 10, 0, 4048, 0, 1, 9, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 1, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 4095, 1, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8053, 8193, 8265, 8188, 8265, 8186, 8193, 8193] avg=8192.0000 max=8265 max_rank=2 min=8053 min_rank=0 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4068, 4022, 0, 0, 0, 0, 0, 0, 39, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 87, 4048, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 7, 4070, 17, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8189, 8244, 8195, 8150, 8218, 8193, 8191] avg=8192.0000 max=8244 max_rank=2 min=8150 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32752 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4068, 4058, 0, 0, 0, 0, 0, 1, 39, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 37, 4048, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 5, 4071, 17, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8190, 8208, 8193, 8152, 8217, 8192, 8191] avg=8192.0000 max=8217 max_rank=5 min=8152 min_rank=4 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32752 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4078, 4085, 18, 0, 0, 1, 0, 0, 38, 27, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4093, 7, 4092, 1, 0, 0, 0, 0, 0, 0, 4026, 0, 1, 0, 2, 4051, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 4071, 17, 3, 0, 0, 0, 0, 0, 6, 4095, 0, 0, 94, 1, 4, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 24, 29, 0, 4000, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 1, 0, 4095, 1, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8197, 8115, 8188, 8256, 8145, 8192, 8196] avg=8192.0000 max=8256 max_rank=4 min=8115 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32789 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4087, 4074, 0, 0, 0, 0, 0, 0, 39, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 4050, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 5, 4072, 17, 3, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8187, 8175, 8195, 8151, 8218, 8192, 8191] avg=8192.0000 max=8227 max_rank=0 min=8151 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32752 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4069, 4057, 37, 0, 0, 1, 1, 0, 38, 27, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4091, 8, 4093, 2, 0, 0, 0, 0, 0, 0, 4060, 0, 1, 0, 4, 4051, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 4071, 17, 2, 0, 0, 0, 0, 0, 9, 4095, 0, 0, 99, 1, 2, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 24, 20, 0, 3986, 0, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 0, 0, 0, 4095, 1, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8199, 8149, 8187, 8262, 8123, 8192, 8194] avg=8192.0000 max=8262 max_rank=4 min=8123 min_rank=5 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32771 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4085, 4060, 0, 0, 0, 0, 0, 1, 42, 29, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 26, 4044, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 2, 4077, 21, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8190, 8188, 8199, 8149, 8211, 8190, 8190] avg=8192.0000 max=8219 max_rank=0 min=8149 min_rank=4 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32740 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32640, 32512, 128, 0, 0, 128, 128, 128, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 0, 256, 32512, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 128, 0, 32640, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 0, 384, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 0, 128, 32640, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4095, 0, 0, 3584, 0, 0, 4096, 0, 2892, 0, 0, 3457, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2345, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 6, 4096, 0, 4096, 4096, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3986, 0, 0, 4096, 0, 2076, 0, 0, 3939, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2275, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 4, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4094, 46, 1, 3465, 1, 21, 4068, 0, 3428, 1, 93, 1639, 2, 0, 0, 0, 4095, 0, 0, 0, 1, 4094, 0, 2, 2, 0, 1, 4094, 0, 4093, 0, 3354, 0, 1, 1, 4095, 0, 0, 0, 0, 1, 4072, 1, 0, 0, 359, 1, 1, 7, 1, 4089, 0, 0, 2, 4095, 0, 0, 4088, 0, 4030, 4094, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 1, 0, 3361, 0, 0, 4096, 0, 3016, 0, 1, 3940, 0, 0, 1, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1956, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 11, 4096, 0, 4093, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4094, 62, 1, 3140, 0, 5, 4050, 0, 3733, 1, 104, 1661, 1, 0, 0, 0, 4095, 0, 2, 0, 1, 4094, 0, 2, 1, 1, 2, 4094, 0, 4094, 0, 3271, 0, 1, 0, 4095, 0, 0, 0, 0, 1, 4078, 1, 7, 0, 424, 1, 1, 3, 1, 4081, 0, 0, 1, 4095, 0, 0, 4090, 0, 4049, 4095, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4095, 0, 0, 3819, 0, 0, 4096, 0, 3125, 0, 0, 3788, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1541, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 5, 4096, 0, 4096, 4096, 8, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4053, 0, 0, 4096, 0, 3983, 0, 1, 4083, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 145, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4, 14, 4096, 0, 4095, 4096, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4095, 42, 1, 3524, 1, 29, 4059, 0, 3259, 1, 93, 1761, 2, 0, 0, 0, 4095, 0, 4, 0, 1, 4094, 0, 1, 0, 0, 1, 4094, 0, 4091, 0, 3350, 0, 1, 1, 4095, 0, 3, 1, 0, 1, 4058, 1, 0, 0, 351, 1, 1, 10, 1, 4091, 0, 0, 1, 4095, 0, 0, 4085, 0, 4044, 4094, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[468, 0, 0, 3584, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2892, 0, 0, 3457, 0, 0, 0, 0, 1930, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1818, 0, 4096, 0, 2345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4079, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3612, 348, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 17, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8280, 8191, 8259, 8191, 8056, 8192, 8219] avg=8192.0000 max=8280 max_rank=1 min=8056 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32658 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[468, 1, 0, 3361, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3016, 0, 1, 3940, 0, 0, 1, 0, 1930, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1818, 0, 4096, 0, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4079, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3613, 348, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 4093, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7926, 8889, 8193, 7870, 8193, 8057, 8191, 8217] avg=8192.0000 max=8889 max_rank=1 min=7870 min_rank=3 imbalance_ratio=1.085083 max_load_reduction=56647 max_load_reduction_pct=0.864365 moved_to_echo=32658 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[468, 62, 1, 3140, 0, 5, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3733, 1, 104, 1661, 1, 0, 0, 0, 1929, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 1, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 1817, 0, 4094, 0, 3271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4078, 15, 0, 0, 0, 0, 0, 0, 1, 4078, 1, 7, 0, 424, 1, 1, 3611, 348, 0, 0, 0, 0, 0, 0, 3, 1, 4081, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4049, 17, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7726, 7430, 8194, 9186, 8189, 8472, 8180, 8159] avg=8192.0000 max=9186 max_rank=3 min=7430 min_rank=1 imbalance_ratio=1.121338 max_load_reduction=56350 max_load_reduction_pct=0.859833 moved_to_echo=33000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[468, 42, 1, 3524, 1, 29, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3259, 1, 93, 1761, 2, 0, 0, 0, 1929, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1817, 0, 4091, 0, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 3, 1, 0, 4077, 15, 0, 0, 0, 0, 0, 0, 1, 4058, 1, 0, 0, 351, 1, 1, 3612, 348, 0, 0, 0, 0, 0, 0, 10, 1, 4091, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4044, 17, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 7046, 8195, 9259, 8193, 8373, 8197, 8149] avg=8192.0000 max=9259 max_rank=3 min=7046 min_rank=1 imbalance_ratio=1.130249 max_load_reduction=56277 max_load_reduction_pct=0.858719 moved_to_echo=32912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[468, 46, 1, 3465, 1, 21, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3428, 1, 93, 1639, 2, 0, 0, 0, 1929, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1817, 0, 4093, 0, 3354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 4077, 15, 0, 0, 0, 0, 0, 0, 1, 4072, 1, 0, 0, 359, 1, 1, 3611, 348, 0, 0, 0, 0, 0, 0, 7, 1, 4089, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4030, 17, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 7093, 8192, 9267, 8189, 8394, 8193, 8138] avg=8192.0000 max=9267 max_rank=3 min=7093 min_rank=1 imbalance_ratio=1.131226 max_load_reduction=56269 max_load_reduction_pct=0.858597 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[468, 0, 0, 3986, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2076, 0, 0, 3939, 0, 0, 0, 0, 1930, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1818, 0, 4096, 0, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4079, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3613, 348, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8550, 7946, 8192, 8189, 8192, 8057, 8197, 8213] avg=8192.0000 max=8550 max_rank=0 min=7946 min_rank=1 imbalance_ratio=1.043701 max_load_reduction=56986 max_load_reduction_pct=0.869537 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[468, 0, 0, 3819, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3125, 0, 0, 3788, 0, 0, 0, 0, 1930, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1818, 0, 4096, 0, 1541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4079, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3612, 348, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 4096, 17, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8383, 8844, 8192, 7455, 8190, 8056, 8194, 8222] avg=8192.0000 max=8844 max_rank=1 min=7455 min_rank=3 imbalance_ratio=1.079590 max_load_reduction=56692 max_load_reduction_pct=0.865051 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[460, 0, 0, 4053, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 1, 4083, 0, 0, 1, 0, 1931, 4, 5, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1810, 0, 4096, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4084, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 3618, 355, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 4095, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8609, 10008, 8187, 6051, 8199, 8071, 8192, 8219] avg=8192.0000 max=10008 max_rank=1 min=6051 min_rank=3 imbalance_ratio=1.221680 max_load_reduction=55528 max_load_reduction_pct=0.847290 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3840, 256, 128, 29056, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 128, 0, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 14592, 0, 32768, 0, 18304, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 0, 1152, 128, 128, 28928, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 32640, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[25600, 128, 384, 24320, 128, 0, 128, 0, 15488, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True + [2026-08-30 05:00:55] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 674.2 | throughput per GPU (TFLOP/s/GPU): 494.5 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.856404E+00 | loss scale: 1.0 | grad norm: 5.294 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 05:00:55 +[rank0]:[W830 05:00:57.395787218 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..1b89178530af928764b96d42e2b34e025503d38c --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1/mem_samples.csv @@ -0,0 +1,248 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 26 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1895 +1, 1895 +2, 1895 +3, 1895 +4, 1895 +5, 1895 +6, 1895 +7, 1895 +0, 38603 +1, 38595 +2, 38595 +3, 38595 +4, 38595 +5, 38595 +6, 38595 +7, 38595 +0, 39193 +1, 39151 +2, 39145 +3, 39155 +4, 39171 +5, 39167 +6, 39211 +7, 39187 +0, 41799 +1, 41783 +2, 41783 +3, 41783 +4, 41783 +5, 41783 +6, 41783 +7, 41783 +0, 42023 +1, 42007 +2, 42007 +3, 42007 +4, 42007 +5, 42007 +6, 42007 +7, 42007 +0, 42025 +1, 42009 +2, 42009 +3, 42009 +4, 42009 +5, 42009 +6, 42009 +7, 42009 +0, 42951 +1, 42935 +2, 42935 +3, 42935 +4, 42935 +5, 42935 +6, 42935 +7, 42935 +0, 43501 +1, 43485 +2, 43485 +3, 43485 +4, 43485 +5, 43485 +6, 43485 +7, 43485 +0, 44549 +1, 44533 +2, 44533 +3, 44533 +4, 44533 +5, 44533 +6, 44533 +7, 44533 +0, 45349 +1, 45333 +2, 45333 +3, 45333 +4, 45333 +5, 45333 +6, 45333 +7, 45333 +0, 78963 +1, 78877 +2, 78877 +3, 78877 +4, 79001 +5, 78877 +6, 78877 +7, 78877 +0, 78963 +1, 78877 +2, 78877 +3, 78877 +4, 79001 +5, 78877 +6, 78877 +7, 78877 +0, 78965 +1, 78879 +2, 78879 +3, 78879 +4, 79003 +5, 78879 +6, 78879 +7, 78879 +0, 79077 +1, 78991 +2, 78991 +3, 78991 +4, 79115 +5, 78991 +6, 78991 +7, 78991 +0, 97449 +1, 97291 +2, 97291 +3, 97291 +4, 97487 +5, 97291 +6, 97291 +7, 97291 +0, 98341 +1, 98161 +2, 98179 +3, 98157 +4, 98387 +5, 98233 +6, 98229 +7, 98215 +0, 99131 +1, 98973 +2, 98973 +3, 98973 +4, 99169 +5, 98973 +6, 98973 +7, 98973 +0, 99723 +1, 99555 +2, 99575 +3, 99583 +4, 99819 +5, 99623 +6, 99583 +7, 99589 +0, 122783 +1, 122791 +2, 122811 +3, 122791 +4, 122899 +5, 122791 +6, 122787 +7, 122791 +0, 122783 +1, 122791 +2, 122811 +3, 122791 +4, 122899 +5, 122791 +6, 122787 +7, 122791 +0, 122783 +1, 122791 +2, 122811 +3, 122791 +4, 122899 +5, 122791 +6, 122787 +7, 122791 +0, 122783 +1, 122791 +2, 122811 +3, 122791 +4, 122899 +5, 122791 +6, 122787 +7, 122791 +0, 121857 +1, 121865 +2, 121885 +3, 121865 +4, 3176 +5, 2152 +6, 121861 +7, 121865 +0, 121854 +1, 3173 +2, 1893 +3, 1893 +4, 1896 +5, 1896 +6, 1896 +7, 2152 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1/run.env b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..3d8c3a721aa711995f6e28d92a44b140463c34cd --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1/run.env @@ -0,0 +1,70 @@ +backend=echo +config=G3 +family=tierA +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=1792 +num_experts=64 +topk=16 +experts_per_rank=8 +expert_params=22020096 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=65536 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.072265625 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1/train.log b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f83730be060840ab081ab4e663473d5337cd1d86 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1/train.log @@ -0,0 +1,16605 @@ +[05:01:03] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G3 / echo (slots/rank=8 policy=matched) + E=64 K=16 H'=1792 m=8 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.072265625 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0830 05:01:05.092000 463303 torch/distributed/run.py:766] +W0830 05:01:05.092000 463303 torch/distributed/run.py:766] ***************************************** +W0830 05:01:05.092000 463303 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 05:01:05.092000 463303 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1792 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.072265625 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 16 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 56 is connected to 7 is connected to 7 is connected to 7 peer ranks. 7 peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 7 +7 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank1]:[W830 05:01:22.741775135 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 05:01:22.741821729 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 05:01:22.741860206 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 05:01:22.741953645 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 05:01:22.744239521 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 05:01:22.744289198 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 05:01:22.745691996 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:01:22.948778595 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.963 seconds +time to initialize megatron (seconds): 7.219 +[after megatron is initialized] datetime: 2026-08-30 05:01:28 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3906080768 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42217472 elements, 42217472 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_proj.weight +Params for bucket 3 (42213376 elements, 42213376 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight +Params for bucket 4 (42213376 elements, 42213376 padded size): + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight +Params for bucket 5 (42213376 elements, 42213376 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_proj.weight +Params for bucket 6 (42213376 elements, 42213376 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.weight +Params for bucket 7 (42213376 elements, 42213376 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.weight +Params for bucket 8 (42213376 elements, 42213376 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight +Params for bucket 9 (42213376 elements, 42213376 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight +Params for bucket 10 (42213376 elements, 42213376 padded size): + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_qkv.weight +Params for bucket 11 (42213376 elements, 42213376 padded size): + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.mlp.router.weight +Params for bucket 12 (42213376 elements, 42213376 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight +Params for bucket 13 (42213376 elements, 42213376 padded size): + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight +Params for bucket 14 (42213376 elements, 42213376 padded size): + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_qkv.weight +Params for bucket 15 (42213376 elements, 42213376 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight +Params for bucket 16 (42213376 elements, 42213376 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight +Params for bucket 17 (42213376 elements, 42213376 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 64 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 +Params for bucket 6 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 +Params for bucket 10 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 +Params for bucket 17 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 18 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 +Params for bucket 24 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 28 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 +Params for bucket 30 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 34 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 35 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 42 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 +Params for bucket 43 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 +Params for bucket 47 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 48 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 +Params for bucket 49 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 55 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 +Params for bucket 56 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 61 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 +Params for bucket 62 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 +Params for bucket 63 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:01:29 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:01:29 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (488.25, 494.59) + train/valid/test-data-iterators-setup ..........: (66.22, 73.01) +[before the start of training step] datetime: 2026-08-30 05:01:29 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1028, 965, 1028, 1051, 1021, 1071, 977, 1049, 1003, 1011, 1000, 1008, 1002, 994, 967, 1049, 1058, 1095, 1031, 1001, 1035, 1023, 1050, 1032, 1052, 1015, 1030, 1026, 1070, 1002, 1032, 1003, 982, 1014, 1012, 1015, 1018, 974, 1103, 1023, 1014, 1100, 1013, 978, 1046, 1038, 987, 1080, 1021, 1027, 1043, 959, 1036, 1048, 1020, 1019, 1008, 1075, 1033, 1038, 1002, 1014, 963, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1021, 1060, 1038, 1023, 1033, 971, 1003, 1054, 1027, 997, 1024, 1036, 1027, 1034, 1007, 974, 1052, 1040, 1030, 1023, 1026, 1016, 1009, 1069, 1007, 1058, 1046, 1040, 1033, 1056, 1046, 1026, 1041, 1044, 1002, 1042, 1019, 1077, 996, 1042, 1051, 1063, 1012, 1027, 965, 1009, 1062, 1042, 1044, 1030, 1000, 1057, 1021, 1009, 992, 1015, 988, 1001, 1003, 969, 1005, 989, 1003, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1049, 1031, 1042, 1003, 1070, 973, 1004, 1009, 1021, 1039, 1004, 1025, 1000, 1042, 1029, 1060, 1011, 1029, 1018, 1027, 1021, 1105, 1045, 1010, 1045, 993, 1010, 1018, 1015, 970, 958, 1041, 1079, 1046, 1058, 1039, 1003, 1034, 999, 1027, 1037, 1031, 1042, 1018, 956, 1028, 995, 1008, 1017, 1037, 999, 1009, 1014, 1008, 1030, 1037, 1058, 1028, 1032, 1039, 1043, 1010, 1041, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1020, 1027, 1033, 984, 1057, 1057, 1075, 1070, 1006, 1044, 1039, 944, 1018, 998, 996, 1061, 1065, 1007, 1011, 1011, 1001, 983, 969, 1000, 991, 1005, 1069, 1062, 1066, 1006, 1095, 1028, 1031, 1020, 1037, 1036, 1011, 1021, 1006, 979, 1052, 1035, 1060, 984, 998, 1001, 1038, 1047, 1010, 1036, 1064, 1013, 981, 986, 1055, 1041, 1010, 1015, 1039, 1024, 1078, 985, 1054, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1056, 966, 1022, 1034, 1035, 1025, 1042, 1062, 1015, 1067, 1005, 1049, 1031, 1003, 1053, 1057, 1001, 1037, 1011, 1013, 974, 1034, 1057, 1034, 1021, 1021, 1046, 984, 1041, 1021, 1069, 1015, 1028, 997, 985, 1003, 983, 985, 1058, 999, 1042, 984, 1044, 999, 1024, 1046, 1040, 1029, 1025, 982, 1022, 1030, 1028, 1036, 1052, 1001, 995, 1052, 1033, 1025, 1035, 1040, 1006, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1045, 1025, 1037, 1013, 1024, 1059, 1023, 1028, 1034, 1078, 1012, 1032, 1000, 1040, 978, 1035, 1049, 950, 1102, 1000, 1037, 1034, 1033, 1011, 1039, 1012, 1026, 1064, 1014, 1003, 1072, 994, 1038, 1043, 977, 1009, 1022, 1032, 1021, 1037, 986, 1003, 1039, 1062, 1012, 1020, 1010, 1018, 1028, 1026, 1042, 1015, 1013, 1027, 978, 1026, 1002, 979, 989, 1042, 1052, 1034, 1015, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1058, 992, 1043, 1031, 1004, 1039, 1062, 1043, 1032, 1058, 1038, 1058, 1063, 1043, 1016, 1042, 1070, 1021, 1027, 1034, 1057, 1020, 1021, 1028, 1020, 1073, 1006, 973, 1048, 1034, 1016, 1030, 1040, 990, 1025, 985, 1003, 972, 1061, 1063, 1010, 986, 986, 996, 1037, 976, 1007, 1005, 1091, 962, 1012, 1071, 1013, 1031, 991, 1018, 999, 1016, 1015, 1045, 1057, 1001, 1016, 956] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1080, 1024, 1031, 1035, 980, 1063, 1024, 1010, 1027, 1006, 1020, 1067, 1026, 985, 1036, 1029, 1004, 1052, 1022, 987, 1013, 1031, 1022, 1001, 1008, 930, 1012, 1045, 1041, 1014, 980, 1019, 1074, 1014, 1042, 1011, 1023, 1068, 1013, 1023, 1060, 1033, 1002, 1032, 1005, 1073, 1023, 1008, 1036, 1012, 1026, 1011, 1036, 1016, 1069, 1058, 1002, 988, 1004, 1009, 1065, 1019, 1035, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[983, 965, 1028, 1051, 1021, 1071, 977, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1011, 1000, 1008, 1002, 994, 967, 1049, 1, 0, 0, 0, 0, 0, 0, 0, 1040, 1095, 1031, 1001, 1035, 1023, 1050, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1015, 1030, 1026, 1063, 1002, 1032, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1014, 1012, 1015, 1018, 974, 1103, 1023, 10, 0, 0, 0, 0, 0, 0, 0, 1014, 1100, 1013, 978, 1046, 1038, 987, 1080, 8, 6, 0, 0, 0, 0, 0, 0, 1021, 1027, 1043, 959, 1036, 1048, 1020, 1019, 6, 2, 0, 0, 0, 0, 0, 0, 1008, 1075, 1033, 1038, 1002, 1014, 963, 1054, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8035, 8307, 8223, 8151, 8270, 8181, 8224] avg=8192.0000 max=8307 max_rank=2 min=8035 min_rank=1 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[976, 1060, 1038, 1023, 1033, 971, 1003, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 997, 1024, 1036, 1027, 1034, 1007, 974, 1, 0, 0, 0, 0, 0, 0, 0, 1034, 1040, 1030, 1023, 1026, 1016, 1009, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1058, 1046, 1040, 1026, 1056, 1046, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1044, 1002, 1042, 1019, 1077, 996, 1042, 10, 0, 0, 0, 0, 0, 0, 0, 1051, 1063, 1012, 1027, 965, 1009, 1062, 1042, 8, 6, 0, 0, 0, 0, 0, 0, 1044, 1030, 1000, 1057, 1021, 1009, 992, 1015, 6, 2, 0, 0, 0, 0, 0, 0, 988, 1001, 1003, 969, 1005, 989, 1003, 1010, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8127, 8247, 8305, 8273, 8245, 8176, 8005] avg=8192.0000 max=8305 max_rank=3 min=8005 min_rank=7 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1010, 966, 1022, 1034, 1035, 1025, 1042, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1067, 1005, 1049, 1031, 1003, 1053, 1057, 1, 0, 0, 0, 0, 0, 0, 0, 984, 1037, 1011, 1013, 974, 1034, 1057, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1021, 1046, 984, 1034, 1021, 1069, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 997, 985, 1003, 983, 985, 1058, 999, 9, 0, 0, 0, 0, 0, 0, 0, 1042, 984, 1044, 999, 1024, 1046, 1040, 1029, 8, 6, 0, 0, 0, 0, 0, 0, 1025, 982, 1022, 1030, 1028, 1036, 1052, 1001, 6, 2, 0, 0, 0, 0, 0, 0, 995, 1052, 1033, 1025, 1035, 1040, 1006, 1027, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8281, 8144, 8211, 8047, 8222, 8184, 8251] avg=8192.0000 max=8281 max_rank=1 min=8047 min_rank=4 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[994, 1031, 1042, 1003, 1070, 973, 1004, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1039, 1004, 1025, 1000, 1042, 1029, 1060, 8, 0, 0, 0, 0, 0, 0, 0, 981, 1029, 1018, 1027, 1021, 1105, 1045, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 993, 1010, 1018, 995, 970, 958, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1046, 1058, 1039, 1003, 1034, 999, 1027, 14, 0, 0, 0, 0, 0, 0, 0, 1037, 1031, 1042, 1018, 956, 1028, 995, 1008, 13, 10, 0, 0, 0, 0, 0, 0, 1017, 1037, 999, 1009, 1014, 1008, 1030, 1037, 12, 6, 0, 0, 0, 0, 0, 0, 1058, 1028, 1032, 1039, 1043, 1010, 1041, 1017, 42, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8228, 8236, 8030, 8299, 8138, 8169, 8310] avg=8192.0000 max=8310 max_rank=7 min=8030 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32916 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[975, 1027, 1033, 984, 1057, 1057, 1075, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1044, 1039, 944, 1018, 998, 996, 1061, 1, 0, 0, 0, 0, 0, 0, 0, 1047, 1007, 1011, 1011, 1001, 983, 969, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1005, 1069, 1062, 1059, 1006, 1095, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1020, 1037, 1036, 1011, 1021, 1006, 979, 10, 0, 0, 0, 0, 0, 0, 0, 1052, 1035, 1060, 984, 998, 1001, 1038, 1047, 8, 6, 0, 0, 0, 0, 0, 0, 1010, 1036, 1064, 1013, 981, 986, 1055, 1041, 6, 2, 0, 0, 0, 0, 0, 0, 1010, 1015, 1039, 1024, 1078, 985, 1054, 991, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8107, 8029, 8315, 8151, 8229, 8194, 8233] avg=8192.0000 max=8315 max_rank=3 min=8029 min_rank=2 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1012, 992, 1043, 1031, 1004, 1039, 1062, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1058, 1038, 1058, 1063, 1043, 1016, 1042, 1, 0, 0, 0, 0, 0, 0, 0, 1052, 1021, 1027, 1034, 1057, 1020, 1021, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1073, 1006, 973, 1041, 1034, 1016, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 990, 1025, 985, 1003, 972, 1061, 1063, 10, 0, 0, 0, 0, 0, 0, 0, 1010, 986, 986, 996, 1037, 976, 1007, 1005, 8, 6, 0, 0, 0, 0, 0, 0, 1091, 962, 1012, 1071, 1013, 1031, 991, 1018, 6, 2, 0, 0, 0, 0, 0, 0, 999, 1016, 1015, 1045, 1057, 1001, 1016, 956, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8351, 8260, 8193, 8149, 8017, 8197, 8143] avg=8192.0000 max=8351 max_rank=1 min=8017 min_rank=5 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[999, 1025, 1037, 1013, 1024, 1059, 1023, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1078, 1012, 1032, 1000, 1040, 978, 1035, 1, 0, 0, 0, 0, 0, 0, 0, 1031, 950, 1102, 1000, 1037, 1034, 1033, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1012, 1026, 1064, 1007, 1003, 1072, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1043, 977, 1009, 1022, 1032, 1021, 1037, 10, 0, 0, 0, 0, 0, 0, 0, 986, 1003, 1039, 1062, 1012, 1020, 1010, 1018, 8, 6, 0, 0, 0, 0, 0, 0, 1028, 1026, 1042, 1015, 1013, 1027, 978, 1026, 6, 2, 0, 0, 0, 0, 0, 0, 1002, 979, 989, 1042, 1052, 1034, 1015, 1036, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8210, 8198, 8217, 8189, 8164, 8163, 8187] avg=8192.0000 max=8217 max_rank=3 min=8163 min_rank=6 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1032, 1024, 1031, 1035, 980, 1063, 1024, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1006, 1020, 1067, 1026, 985, 1036, 1029, 1, 0, 0, 0, 0, 0, 0, 0, 987, 1052, 1022, 987, 1013, 1031, 1022, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 930, 1012, 1045, 1034, 1014, 980, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1014, 1042, 1011, 1023, 1068, 1013, 1023, 9, 0, 0, 0, 0, 0, 0, 0, 1060, 1033, 1002, 1032, 1005, 1073, 1023, 1008, 9, 6, 0, 0, 0, 0, 0, 0, 1036, 1012, 1026, 1011, 1036, 1016, 1069, 1058, 6, 2, 0, 0, 0, 0, 0, 0, 1002, 988, 1004, 1009, 1065, 1019, 1035, 1022, 39, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8197, 8115, 8042, 8277, 8251, 8272, 8183] avg=8192.0000 max=8277 max_rank=4 min=8042 min_rank=3 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32983 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8064, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8320, 8192, 8320, 8320, 8320, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8192, 8448, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[997, 1084, 1010, 1012, 1005, 1025, 1019, 1051, 1038, 1012, 1033, 1050, 1007, 1048, 1063, 1049, 982, 1045, 1010, 1042, 1037, 993, 1042, 980, 1011, 1044, 1046, 1011, 1065, 1040, 989, 1038, 1130, 962, 1067, 1040, 962, 1036, 989, 981, 1049, 1037, 998, 1016, 1013, 990, 1019, 1017, 1031, 1039, 1059, 1008, 1017, 1009, 1009, 1030, 986, 1032, 1042, 1009, 1002, 1037, 1030, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1018, 1000, 1001, 1019, 998, 956, 1033, 1032, 1058, 1032, 1030, 1065, 1019, 1012, 957, 1039, 1005, 946, 1014, 1026, 1006, 1026, 1057, 1085, 1011, 974, 990, 1039, 1049, 1017, 1011, 1051, 1050, 1000, 1101, 1049, 1014, 1003, 1038, 1026, 1049, 1024, 1021, 1012, 1027, 999, 1010, 1034, 1034, 1007, 1021, 1016, 1080, 1041, 1054, 1036, 1061, 994, 1052, 1067, 1051, 987, 1005, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1021, 1050, 1048, 937, 997, 1077, 1002, 977, 1021, 1029, 1052, 1019, 995, 1024, 1028, 1010, 1042, 1073, 1020, 1036, 1050, 1030, 1064, 1009, 1073, 1020, 973, 1023, 1007, 1018, 1015, 1045, 1007, 986, 1031, 1017, 1061, 997, 1069, 1021, 1025, 986, 1013, 1045, 977, 1010, 1016, 1030, 1074, 996, 1007, 1020, 1036, 1037, 1007, 1059, 1008, 1063, 1019, 1043, 1075, 1022, 937, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1012, 1062, 1029, 1035, 1046, 1010, 1071, 1056, 1031, 994, 1046, 1040, 1049, 1039, 1038, 1016, 1064, 1010, 972, 996, 1024, 1012, 1010, 996, 1006, 1049, 1021, 1019, 993, 1055, 1013, 1024, 1005, 961, 1056, 1023, 1040, 1000, 978, 1047, 1049, 986, 1033, 1043, 1045, 1048, 1076, 998, 1005, 971, 1005, 1042, 1059, 1049, 1035, 995, 1048, 1019, 962, 1000, 1021, 1034, 1028, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1012, 1033, 990, 1001, 1030, 1025, 992, 1034, 1002, 1051, 1035, 1072, 1034, 1015, 1041, 968, 1038, 1019, 1032, 1028, 1088, 1000, 1040, 993, 1038, 1043, 1068, 1053, 1014, 1007, 1052, 977, 1007, 1015, 1038, 995, 1000, 1035, 1014, 1055, 1005, 1019, 1045, 1028, 1003, 1036, 1017, 1005, 1060, 999, 1037, 1031, 1000, 1050, 983, 1068, 1026, 1027, 1049, 1034, 1009, 989, 1064, 968] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1040, 1023, 1034, 1029, 1044, 979, 1037, 1017, 1003, 1015, 1039, 999, 1054, 1054, 1014, 1006, 1043, 1021, 1066, 972, 973, 1029, 1063, 1007, 1036, 1002, 997, 1032, 1079, 1098, 992, 989, 1073, 1068, 1008, 982, 1049, 1053, 1056, 1024, 1005, 959, 1046, 1061, 966, 1043, 1015, 1024, 1031, 1016, 976, 991, 997, 1049, 1061, 1028, 1004, 1049, 972, 1022, 1046, 1018, 982, 1076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1012, 1048, 1058, 1054, 1038, 1029, 956, 1063, 982, 1033, 992, 1014, 1045, 1037, 1011, 1032, 973, 1045, 1033, 1023, 1007, 1016, 1039, 1014, 1047, 998, 1097, 989, 1070, 1054, 1007, 1056, 1013, 1036, 1057, 1000, 1026, 1030, 986, 1028, 1027, 1057, 1046, 1024, 1021, 941, 1026, 1066, 970, 983, 1047, 1036, 998, 983, 1035, 1035, 1015, 1035, 1032, 1052, 1022, 1017, 1008, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1002, 1053, 1023, 1023, 1026, 1022, 1081, 1024, 1052, 1017, 1024, 1001, 1026, 990, 1054, 1063, 1025, 1017, 1049, 1038, 1059, 1035, 1013, 1011, 1000, 1021, 1048, 1054, 1054, 972, 1035, 1012, 995, 1002, 1038, 1013, 1001, 1018, 998, 989, 1038, 999, 1038, 1057, 1032, 1012, 1063, 996, 1024, 1011, 1010, 1001, 1032, 1013, 1032, 1060, 983, 1046, 1022, 1006, 996, 1020, 1035, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[997, 1084, 1010, 1012, 1005, 1025, 1019, 1051, 1, 0, 0, 0, 0, 0, 0, 0, 1038, 1012, 1033, 1029, 1007, 1048, 1063, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1045, 1010, 1042, 1037, 993, 1042, 980, 2, 0, 0, 0, 0, 0, 0, 0, 1011, 1044, 1046, 1011, 1041, 1040, 989, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1130, 962, 1067, 1040, 962, 1036, 989, 981, 11, 0, 0, 0, 0, 0, 0, 0, 1049, 1037, 998, 1016, 1013, 990, 1019, 1017, 8, 7, 0, 0, 0, 0, 0, 0, 1031, 1039, 1059, 1008, 1017, 1009, 1009, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1032, 1042, 1009, 1002, 1037, 1030, 1012, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8279, 8133, 8220, 8178, 8154, 8202, 8166] avg=8192.0000 max=8279 max_rank=1 min=8133 min_rank=2 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1012, 1062, 1029, 1035, 1046, 1010, 1071, 1056, 1, 0, 0, 0, 0, 0, 0, 0, 1031, 994, 1046, 1020, 1049, 1039, 1038, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1010, 972, 996, 1024, 1012, 1010, 996, 2, 0, 0, 0, 0, 0, 0, 0, 1006, 1049, 1021, 1019, 971, 1055, 1013, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 961, 1056, 1023, 1040, 1000, 978, 1047, 10, 0, 0, 0, 0, 0, 0, 0, 1049, 986, 1033, 1043, 1045, 1048, 1076, 998, 7, 7, 0, 0, 0, 0, 0, 0, 1005, 971, 1005, 1042, 1059, 1049, 1035, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1019, 962, 1000, 1021, 1034, 1028, 1037, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8322, 8233, 8086, 8158, 8120, 8292, 8161, 8164] avg=8192.0000 max=8322 max_rank=0 min=8086 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1018, 1000, 1001, 1019, 998, 956, 1033, 1032, 1, 0, 0, 0, 0, 0, 0, 0, 1058, 1032, 1030, 1044, 1019, 1012, 957, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 946, 1014, 1026, 1006, 1026, 1057, 1085, 2, 0, 0, 0, 0, 0, 0, 0, 1011, 974, 990, 1039, 1025, 1017, 1011, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1000, 1101, 1049, 1014, 1003, 1038, 1026, 11, 0, 0, 0, 0, 0, 0, 0, 1049, 1024, 1021, 1012, 1027, 999, 1010, 1034, 8, 7, 0, 0, 0, 0, 0, 0, 1034, 1007, 1021, 1016, 1080, 1041, 1054, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 994, 1052, 1067, 1051, 987, 1005, 997, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8058, 8191, 8167, 8118, 8292, 8191, 8289, 8230] avg=8192.0000 max=8292 max_rank=4 min=8058 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=33002 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1021, 1050, 1048, 937, 997, 1077, 1002, 977, 1, 0, 0, 0, 0, 0, 0, 0, 1021, 1029, 1052, 999, 995, 1024, 1028, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1073, 1020, 1036, 1050, 1030, 1064, 1009, 2, 0, 0, 0, 0, 0, 0, 0, 1073, 1020, 973, 1023, 985, 1018, 1015, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 986, 1031, 1017, 1061, 997, 1069, 1021, 10, 0, 0, 0, 0, 0, 0, 0, 1025, 986, 1013, 1045, 977, 1010, 1016, 1030, 7, 7, 0, 0, 0, 0, 0, 0, 1074, 996, 1007, 1020, 1036, 1037, 1007, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1063, 1019, 1043, 1075, 1022, 937, 1057, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8158, 8326, 8152, 8199, 8116, 8236, 8239] avg=8192.0000 max=8326 max_rank=2 min=8110 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1040, 1023, 1034, 1029, 1044, 979, 1037, 1017, 7, 0, 0, 0, 0, 0, 0, 0, 1003, 1015, 1039, 958, 1054, 1054, 1014, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1021, 1066, 972, 973, 1029, 1063, 1007, 9, 0, 0, 0, 0, 0, 0, 0, 1036, 1002, 997, 1032, 1046, 1098, 992, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1068, 1008, 982, 1049, 1053, 1056, 1024, 14, 0, 0, 0, 0, 0, 0, 0, 1005, 959, 1046, 1061, 966, 1043, 1015, 1024, 11, 10, 0, 0, 0, 0, 0, 0, 1031, 1016, 976, 991, 997, 1049, 1061, 1028, 1, 0, 0, 0, 0, 0, 0, 0, 1004, 1049, 972, 1022, 1046, 1018, 982, 1076, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8143, 8183, 8192, 8327, 8140, 8150, 8191] avg=8192.0000 max=8327 max_rank=4 min=8140 min_rank=5 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1012, 1048, 1058, 1054, 1038, 1029, 956, 1063, 1, 0, 0, 0, 0, 0, 0, 0, 982, 1033, 992, 995, 1045, 1037, 1011, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1045, 1033, 1023, 1007, 1016, 1039, 1014, 2, 0, 0, 0, 0, 0, 0, 0, 1047, 998, 1097, 989, 1046, 1054, 1007, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1036, 1057, 1000, 1026, 1030, 986, 1028, 10, 0, 0, 0, 0, 0, 0, 0, 1027, 1057, 1046, 1024, 1021, 941, 1026, 1066, 8, 6, 0, 0, 0, 0, 0, 0, 970, 983, 1047, 1036, 998, 983, 1035, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1035, 1032, 1052, 1022, 1017, 1008, 1012, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8127, 8152, 8294, 8186, 8222, 8087, 8209] avg=8192.0000 max=8294 max_rank=3 min=8087 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1012, 1033, 990, 1001, 1030, 1025, 992, 1034, 1, 0, 0, 0, 0, 0, 0, 0, 1002, 1051, 1035, 1051, 1034, 1015, 1041, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1019, 1032, 1028, 1088, 1000, 1040, 993, 2, 0, 0, 0, 0, 0, 0, 0, 1038, 1043, 1068, 1053, 992, 1007, 1052, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1015, 1038, 995, 1000, 1035, 1014, 1055, 11, 0, 0, 0, 0, 0, 0, 0, 1005, 1019, 1045, 1028, 1003, 1036, 1017, 1005, 7, 7, 0, 0, 0, 0, 0, 0, 1060, 999, 1037, 1031, 1000, 1050, 983, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1027, 1049, 1034, 1009, 989, 1064, 968, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8197, 8240, 8230, 8170, 8172, 8228, 8181] avg=8192.0000 max=8240 max_rank=2 min=8118 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1002, 1053, 1023, 1023, 1026, 1022, 1081, 1024, 1, 0, 0, 0, 0, 0, 0, 0, 1052, 1017, 1024, 982, 1026, 990, 1054, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1017, 1049, 1038, 1059, 1035, 1013, 1011, 2, 0, 0, 0, 0, 0, 0, 0, 1000, 1021, 1048, 1054, 1030, 972, 1035, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1002, 1038, 1013, 1001, 1018, 998, 989, 10, 0, 0, 0, 0, 0, 0, 0, 1038, 999, 1038, 1057, 1032, 1012, 1063, 996, 8, 6, 0, 0, 0, 0, 0, 0, 1024, 1011, 1010, 1001, 1032, 1013, 1032, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1046, 1022, 1006, 996, 1020, 1035, 1032, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8208, 8249, 8172, 8064, 8249, 8183, 8156] avg=8192.0000 max=8255 max_rank=0 min=8064 min_rank=4 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8192, 8192, 8320, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8448, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8448, 8192, 8192, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8320, 8192, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1064, 1041, 1038, 997, 998, 1009, 1037, 1017, 1015, 983, 998, 1045, 1027, 1011, 1013, 1007, 1021, 1041, 1036, 988, 1009, 979, 1041, 1041, 1013, 1032, 1027, 979, 1072, 1007, 1019, 1040, 1051, 1027, 1021, 1054, 1042, 1017, 1047, 1065, 1005, 1013, 994, 1073, 1019, 1010, 1029, 1004, 980, 1044, 1025, 1024, 1015, 1066, 1080, 1001, 1032, 1074, 1047, 1040, 1048, 973, 1017, 954] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1014, 1023, 1002, 985, 1047, 1004, 1034, 1022, 1022, 1024, 1025, 1021, 1026, 1007, 1017, 1021, 1051, 1049, 1003, 1064, 993, 1030, 1086, 990, 1033, 1053, 1055, 1023, 981, 1022, 1007, 1022, 1002, 1023, 1065, 1002, 1038, 1008, 1003, 1016, 1058, 999, 1019, 1015, 1047, 1052, 1013, 1033, 1031, 1012, 1038, 1003, 1037, 1074, 1041, 1066, 1033, 1005, 1048, 987, 998, 1034, 1006, 974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1069, 1047, 1025, 975, 1033, 1025, 1047, 1012, 1029, 1003, 1017, 1004, 1048, 1051, 983, 1039, 976, 1035, 1049, 1029, 1027, 998, 1043, 1013, 1027, 1046, 1016, 995, 1063, 967, 1061, 1035, 1052, 985, 1050, 980, 996, 1063, 1004, 1032, 1001, 980, 1018, 1084, 1043, 1022, 1003, 1052, 995, 990, 967, 1041, 1061, 1038, 1043, 1032, 1034, 1047, 1013, 1018, 1047, 1029, 1018, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[991, 1007, 1020, 1062, 1035, 1032, 1014, 1046, 1062, 1030, 1032, 1034, 996, 1020, 946, 1040, 1030, 1024, 998, 1044, 1008, 1046, 1010, 1012, 1006, 1035, 1036, 1063, 1080, 1058, 1016, 1088, 1016, 1023, 1022, 1027, 972, 1037, 979, 983, 1026, 1026, 1059, 981, 995, 986, 1030, 1030, 1016, 1011, 985, 1048, 1018, 1016, 1076, 1061, 1019, 1037, 1023, 1018, 1026, 1010, 1024, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1037, 1026, 1005, 1079, 1004, 1021, 1025, 1047, 1010, 1001, 1042, 968, 1049, 1037, 965, 1048, 1023, 1043, 1055, 976, 1054, 1063, 1045, 1028, 1090, 1007, 1029, 1036, 1051, 1022, 994, 1006, 1049, 952, 979, 1026, 1014, 984, 1032, 982, 1037, 1088, 993, 1029, 999, 1015, 1024, 1020, 1046, 1006, 1002, 1076, 1006, 1025, 1015, 990, 1049, 1015, 1038, 1025, 1043, 1051, 1026, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1042, 1066, 1009, 990, 1021, 1035, 1020, 1028, 1062, 1044, 1039, 1017, 990, 1036, 1009, 1034, 968, 1021, 968, 971, 1058, 1011, 1020, 1025, 1026, 980, 991, 983, 1055, 1029, 1016, 1032, 991, 1008, 1055, 1033, 1081, 1053, 1037, 1043, 1060, 1061, 962, 1022, 1064, 1016, 1047, 1025, 1017, 1034, 1030, 1057, 990, 1044, 1020, 1035, 1031, 1007, 1064, 1016, 1029, 947, 1030, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1025, 997, 1045, 1062, 1023, 985, 1005, 1064, 1000, 1004, 1010, 1040, 1025, 1006, 1024, 994, 1004, 1083, 985, 1063, 1056, 1005, 1066, 967, 1007, 1009, 1011, 1042, 1019, 998, 996, 1045, 1004, 1021, 990, 1037, 1081, 1063, 985, 1040, 1002, 1070, 1052, 992, 1037, 1021, 1048, 1018, 973, 1034, 1025, 1034, 1015, 1010, 998, 1000, 1015, 1039, 1024, 1058, 1042, 1033, 1086, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1059, 980, 1058, 1096, 1015, 1017, 1042, 1031, 1034, 1037, 948, 972, 1035, 1032, 1079, 1067, 991, 1071, 1036, 1057, 1023, 1052, 971, 984, 1000, 1035, 1009, 947, 1019, 982, 1008, 991, 1058, 1041, 978, 1013, 1047, 1021, 1052, 1054, 1016, 1041, 1011, 1026, 1033, 1079, 1030, 1034, 1019, 1023, 1009, 1020, 1045, 1026, 1027, 1033, 1010, 988, 984, 1049, 1045, 1031, 1001, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1040, 1041, 1038, 997, 998, 1009, 1037, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 983, 998, 1045, 1027, 1011, 1013, 1007, 24, 10, 0, 0, 0, 0, 0, 0, 1021, 1041, 1036, 988, 1009, 979, 1041, 1041, 3, 0, 0, 0, 0, 0, 0, 0, 1013, 1032, 1027, 979, 1072, 1007, 1019, 1040, 8, 3, 0, 0, 0, 0, 0, 0, 1051, 1027, 1021, 1054, 1042, 1017, 1047, 1065, 3, 0, 0, 0, 0, 0, 0, 0, 1005, 995, 994, 1073, 1019, 1010, 1029, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1044, 1025, 1015, 1015, 1066, 1080, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1074, 1047, 1040, 1048, 973, 1017, 954, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8133, 8159, 8200, 8327, 8129, 8226, 8185] avg=8192.0000 max=8327 max_rank=4 min=8129 min_rank=5 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1044, 1047, 1025, 975, 1033, 1025, 1047, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1003, 1017, 1004, 1048, 1051, 983, 1039, 25, 10, 0, 0, 0, 0, 0, 0, 976, 1035, 1049, 1029, 1027, 998, 1043, 1013, 3, 0, 0, 0, 0, 0, 0, 0, 1027, 1046, 1016, 995, 1063, 967, 1061, 1035, 8, 3, 0, 0, 0, 0, 0, 0, 1052, 985, 1050, 980, 996, 1063, 1004, 1032, 3, 0, 0, 0, 0, 0, 0, 0, 1001, 962, 1018, 1084, 1043, 1022, 1003, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 995, 990, 967, 1032, 1061, 1038, 1043, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1047, 1013, 1018, 1047, 1029, 1018, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8209, 8173, 8221, 8165, 8185, 8158, 8217] avg=8192.0000 max=8221 max_rank=3 min=8158 min_rank=6 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[991, 1023, 1002, 985, 1047, 1004, 1034, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1024, 1025, 1021, 1026, 1007, 1017, 1021, 23, 10, 0, 0, 0, 0, 0, 0, 1051, 1049, 1003, 1064, 993, 1030, 1086, 990, 3, 0, 0, 0, 0, 0, 0, 0, 1033, 1053, 1055, 1023, 981, 1022, 1007, 1022, 8, 3, 0, 0, 0, 0, 0, 0, 1002, 1023, 1065, 1002, 1038, 1008, 1003, 1016, 3, 0, 0, 0, 0, 0, 0, 0, 1058, 981, 1019, 1015, 1047, 1052, 1013, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1012, 1038, 994, 1037, 1074, 1041, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1005, 1048, 987, 998, 1034, 1006, 974, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8196, 8269, 8207, 8160, 8218, 8293, 8085] avg=8192.0000 max=8293 max_rank=6 min=8085 min_rank=7 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1009, 1026, 1005, 1079, 1004, 1021, 1025, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1001, 1042, 968, 1049, 1037, 965, 1048, 28, 15, 0, 0, 0, 0, 0, 0, 1023, 1043, 1055, 976, 1054, 1063, 1045, 1028, 6, 2, 0, 0, 0, 0, 0, 0, 1090, 1007, 1029, 1036, 1051, 1022, 994, 1006, 12, 5, 0, 0, 0, 0, 0, 0, 1049, 952, 979, 1026, 1014, 984, 1032, 982, 9, 0, 0, 0, 0, 0, 0, 0, 1037, 1061, 993, 1029, 999, 1015, 1024, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1006, 1002, 1056, 1006, 1025, 1015, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1015, 1038, 1025, 1041, 1051, 1026, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8163, 8295, 8252, 8027, 8178, 8146, 8259] avg=8192.0000 max=8295 max_rank=2 min=8027 min_rank=4 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32610 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1018, 1066, 1009, 990, 1021, 1035, 1020, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1044, 1039, 1017, 990, 1036, 1009, 1034, 24, 11, 0, 0, 0, 0, 0, 0, 968, 1021, 968, 971, 1058, 1011, 1020, 1025, 3, 0, 0, 0, 0, 0, 0, 0, 1026, 980, 991, 983, 1055, 1029, 1016, 1032, 8, 3, 0, 0, 0, 0, 0, 0, 991, 1008, 1055, 1033, 1081, 1053, 1037, 1043, 3, 0, 0, 0, 0, 0, 0, 0, 1060, 1042, 962, 1022, 1064, 1016, 1047, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1034, 1030, 1048, 990, 1044, 1020, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1007, 1064, 1016, 1029, 947, 1030, 1031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8266, 8045, 8123, 8304, 8238, 8218, 8155] avg=8192.0000 max=8304 max_rank=4 min=8045 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32915 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1001, 997, 1045, 1062, 1023, 985, 1005, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1004, 1010, 1040, 1025, 1006, 1024, 994, 24, 11, 0, 0, 0, 0, 0, 0, 1004, 1083, 985, 1063, 1056, 1005, 1066, 967, 3, 0, 0, 0, 0, 0, 0, 0, 1007, 1009, 1011, 1042, 1019, 998, 996, 1045, 8, 3, 0, 0, 0, 0, 0, 0, 1004, 1021, 990, 1037, 1081, 1063, 985, 1040, 3, 0, 0, 0, 0, 0, 0, 0, 1002, 1051, 1052, 992, 1037, 1021, 1048, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1034, 1025, 1025, 1015, 1010, 998, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1039, 1024, 1058, 1042, 1033, 1086, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8138, 8232, 8138, 8224, 8221, 8080, 8321] avg=8192.0000 max=8321 max_rank=7 min=8080 min_rank=6 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1035, 980, 1058, 1096, 1015, 1017, 1042, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1037, 948, 972, 1035, 1032, 1079, 1067, 24, 10, 0, 0, 0, 0, 0, 0, 991, 1071, 1036, 1057, 1023, 1052, 971, 984, 3, 0, 0, 0, 0, 0, 0, 0, 1000, 1035, 1009, 947, 1019, 982, 1008, 991, 8, 3, 0, 0, 0, 0, 0, 0, 1058, 1041, 978, 1013, 1047, 1021, 1052, 1054, 3, 0, 0, 0, 0, 0, 0, 0, 1016, 1023, 1011, 1026, 1033, 1079, 1030, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1023, 1009, 1011, 1045, 1026, 1027, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 988, 984, 1049, 1045, 1031, 1001, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8238, 8188, 8002, 8267, 8252, 8193, 8122] avg=8192.0000 max=8274 max_rank=0 min=8002 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[968, 1007, 1020, 1062, 1035, 1032, 1014, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1030, 1032, 1034, 996, 1020, 946, 1040, 23, 10, 0, 0, 0, 0, 0, 0, 1030, 1024, 998, 1044, 1008, 1046, 1010, 1012, 3, 0, 0, 0, 0, 0, 0, 0, 1006, 1035, 1036, 1063, 1080, 1058, 1016, 1088, 8, 3, 0, 0, 0, 0, 0, 0, 1016, 1023, 1022, 1027, 972, 1037, 979, 983, 3, 0, 0, 0, 0, 0, 0, 0, 1026, 1008, 1059, 981, 995, 986, 1030, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1011, 985, 1039, 1018, 1016, 1076, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1037, 1023, 1018, 1026, 1010, 1024, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8193, 8175, 8393, 8062, 8115, 8222, 8192] avg=8192.0000 max=8393 max_rank=3 min=8062 min_rank=4 imbalance_ratio=1.024536 max_load_reduction=57143 max_load_reduction_pct=0.871933 moved_to_echo=32591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8064, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8192, 8192, 8448, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8448, 8192, 8192, 8320, 8192, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1059, 1059, 1042, 974, 1016, 1046, 1041, 1043, 1051, 1032, 1008, 980, 1023, 1061, 989, 1033, 1002, 1046, 1031, 1034, 999, 1008, 1048, 977, 1038, 1065, 1036, 1022, 1003, 1043, 1060, 1000, 991, 1000, 1058, 1032, 1025, 1014, 1043, 1005, 978, 1031, 1015, 987, 1053, 986, 1045, 993, 1014, 1033, 1054, 1042, 1098, 1053, 1013, 1043, 1020, 1041, 986, 1027, 1013, 999, 993, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[966, 1066, 1003, 1032, 1041, 1037, 1033, 1002, 1014, 1058, 986, 1016, 975, 1030, 1000, 1023, 1006, 1083, 1066, 987, 1048, 1043, 1051, 1007, 1033, 1016, 1027, 1052, 1053, 998, 985, 1019, 1046, 1098, 1078, 1034, 1038, 1003, 1069, 1068, 1078, 1005, 978, 1041, 1005, 1033, 1038, 1017, 1000, 1001, 964, 989, 1054, 1019, 1009, 984, 994, 1040, 1036, 1007, 957, 1058, 1038, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1021, 1022, 1056, 999, 998, 1068, 1028, 1009, 1059, 1015, 1052, 1032, 995, 1038, 1013, 1043, 1017, 1013, 1008, 1014, 1019, 1045, 1043, 1016, 990, 1034, 1055, 1011, 1005, 1004, 1044, 997, 1023, 1036, 985, 1036, 1035, 1027, 1020, 999, 994, 1017, 1062, 1003, 1032, 1022, 1035, 1029, 1021, 1055, 1024, 1032, 1031, 1058, 1006, 999, 1031, 1021, 1053, 998, 1016, 1039, 1021, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1054, 985, 1004, 1039, 998, 1079, 1032, 981, 1063, 1033, 1031, 1060, 1001, 997, 990, 1039, 1009, 1036, 1016, 983, 978, 1039, 1015, 999, 1047, 1041, 1041, 1066, 960, 1022, 1038, 1065, 986, 989, 1037, 1016, 1010, 1038, 1022, 1034, 1006, 1021, 1058, 1005, 1004, 998, 1019, 1001, 1022, 1045, 1066, 1001, 1012, 1045, 1050, 1029, 1062, 1064, 1012, 1013, 1085, 1000, 1033, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1036, 1010, 1006, 1041, 977, 1007, 1050, 1006, 1027, 1037, 996, 1004, 1072, 1005, 1004, 1069, 1051, 983, 952, 1066, 1051, 1033, 1050, 1023, 983, 1003, 1031, 1007, 1030, 1065, 1051, 1117, 1052, 1021, 1027, 1033, 1076, 995, 1014, 1011, 1045, 1016, 1043, 1036, 1054, 1027, 1033, 1013, 967, 1045, 1018, 996, 1027, 1015, 1066, 952, 1026, 975, 999, 1039, 1008, 1041, 988, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1074, 1065, 1030, 1030, 1028, 1045, 1022, 996, 1043, 1034, 985, 1014, 1017, 1011, 992, 1053, 1025, 1048, 1060, 1066, 1000, 1011, 1004, 1053, 1052, 1048, 995, 992, 1065, 1004, 989, 1023, 995, 1026, 1021, 1004, 1010, 1044, 1036, 1037, 1043, 985, 990, 986, 971, 1015, 1024, 994, 1033, 1046, 1038, 1036, 1026, 1027, 1001, 1023, 1070, 1022, 1061, 1038, 1015, 1065, 931, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1029, 1006, 1031, 958, 1009, 1007, 1020, 1042, 1049, 980, 988, 1062, 1062, 1063, 995, 993, 977, 1021, 1053, 1039, 990, 1062, 1031, 1054, 1009, 1077, 995, 1017, 1035, 1051, 1025, 1067, 1028, 1004, 925, 1019, 1013, 986, 1018, 1003, 970, 1043, 1046, 1044, 1075, 1067, 1027, 1045, 1033, 1027, 1065, 1029, 1010, 1014, 1027, 992, 998, 1017, 1093, 1014, 1032, 987, 1062, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1058, 991, 1044, 1094, 1009, 1012, 1057, 1019, 1008, 1021, 966, 1056, 1064, 1016, 1050, 971, 1050, 1053, 988, 1014, 1004, 998, 1021, 988, 1012, 1072, 1046, 994, 1012, 1031, 1026, 1011, 1000, 1039, 984, 1009, 996, 1061, 1003, 1052, 1019, 971, 978, 1075, 979, 1013, 1050, 1046, 1038, 1001, 1067, 986, 1027, 1033, 1072, 1018, 1048, 1050, 1007, 1008, 993, 1070, 1037, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1021, 1022, 1056, 999, 998, 1056, 1028, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1015, 1052, 1032, 995, 1038, 1013, 1043, 6, 4, 0, 0, 0, 0, 0, 0, 1017, 1013, 1008, 1014, 1019, 1045, 1043, 1016, 3, 0, 0, 0, 0, 0, 0, 0, 990, 1002, 1055, 1011, 1005, 1004, 1044, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1036, 985, 1036, 1035, 1027, 1020, 999, 6, 5, 0, 0, 0, 0, 0, 0, 994, 1017, 1062, 1003, 1032, 1022, 1035, 1029, 27, 0, 0, 0, 0, 0, 0, 0, 1021, 1055, 1015, 1032, 1031, 1058, 1006, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1021, 1053, 998, 1016, 1039, 1021, 1013, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8257, 8178, 8108, 8172, 8221, 8217, 8194] avg=8192.0000 max=8257 max_rank=1 min=8108 min_rank=3 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[966, 1066, 1003, 1032, 1041, 1025, 1033, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1058, 986, 1016, 975, 1030, 1000, 1023, 6, 4, 0, 0, 0, 0, 0, 0, 1006, 1083, 1066, 987, 1048, 1043, 1051, 1007, 3, 0, 0, 0, 0, 0, 0, 0, 1033, 984, 1027, 1052, 1053, 998, 985, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1098, 1078, 1034, 1038, 1003, 1069, 1068, 6, 5, 0, 0, 0, 0, 0, 0, 1078, 1005, 978, 1041, 1005, 1033, 1038, 1017, 27, 0, 0, 0, 0, 0, 0, 0, 1000, 1001, 956, 989, 1054, 1019, 1009, 984, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1040, 1036, 1007, 957, 1058, 1038, 1001, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8112, 8294, 8151, 8445, 8222, 8012, 8132] avg=8192.0000 max=8445 max_rank=4 min=8012 min_rank=6 imbalance_ratio=1.030884 max_load_reduction=57091 max_load_reduction_pct=0.871140 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1059, 1059, 1042, 974, 1016, 1034, 1041, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1032, 1008, 980, 1023, 1061, 989, 1033, 6, 4, 0, 0, 0, 0, 0, 0, 1002, 1046, 1031, 1034, 999, 1008, 1048, 977, 3, 0, 0, 0, 0, 0, 0, 0, 1038, 1032, 1036, 1022, 1003, 1043, 1060, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1000, 1058, 1032, 1025, 1014, 1043, 1005, 6, 5, 0, 0, 0, 0, 0, 0, 978, 1031, 1015, 987, 1053, 986, 1045, 993, 28, 0, 0, 0, 0, 0, 0, 0, 1014, 1033, 1045, 1042, 1098, 1053, 1013, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1041, 986, 1027, 1013, 999, 993, 982, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8187, 8148, 8234, 8179, 8116, 8341, 8063] avg=8192.0000 max=8341 max_rank=6 min=8063 min_rank=7 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1036, 1010, 1006, 1041, 977, 986, 1050, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1037, 996, 1004, 1072, 1005, 1004, 1069, 10, 9, 0, 0, 0, 0, 0, 0, 1051, 983, 952, 1066, 1051, 1033, 1050, 1023, 10, 0, 0, 0, 0, 0, 0, 0, 983, 962, 1031, 1007, 1030, 1065, 1051, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1021, 1027, 1033, 1076, 995, 1014, 1011, 11, 10, 0, 0, 0, 0, 0, 0, 1045, 1016, 1043, 1036, 1054, 1027, 1033, 1013, 31, 0, 0, 0, 0, 0, 0, 0, 967, 1045, 995, 996, 1027, 1015, 1066, 952, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 975, 999, 1039, 1008, 1041, 988, 1035, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8233, 8219, 8246, 8250, 8298, 8063, 8115] avg=8192.0000 max=8298 max_rank=5 min=8063 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1058, 991, 1044, 1094, 1009, 1000, 1057, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1021, 966, 1056, 1064, 1016, 1050, 971, 6, 4, 0, 0, 0, 0, 0, 0, 1050, 1053, 988, 1014, 1004, 998, 1021, 988, 3, 0, 0, 0, 0, 0, 0, 0, 1012, 1039, 1046, 994, 1012, 1031, 1026, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1039, 984, 1009, 996, 1061, 1003, 1052, 6, 5, 0, 0, 0, 0, 0, 0, 1019, 971, 978, 1075, 979, 1013, 1050, 1046, 28, 0, 0, 0, 0, 0, 0, 0, 1038, 1001, 1058, 986, 1027, 1033, 1072, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1050, 1007, 1008, 993, 1070, 1037, 1050, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8162, 8119, 8171, 8155, 8159, 8233, 8265] avg=8192.0000 max=8272 max_rank=0 min=8119 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1054, 985, 1004, 1039, 998, 1066, 1032, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1033, 1031, 1060, 1001, 997, 990, 1039, 6, 4, 0, 0, 0, 0, 0, 0, 1009, 1036, 1016, 983, 978, 1039, 1015, 999, 3, 0, 0, 0, 0, 0, 0, 0, 1047, 1009, 1041, 1066, 960, 1022, 1038, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 986, 989, 1037, 1016, 1010, 1038, 1022, 1034, 7, 5, 0, 0, 0, 0, 0, 0, 1006, 1021, 1058, 1005, 1004, 998, 1019, 1001, 27, 0, 0, 0, 0, 0, 0, 0, 1022, 1045, 1057, 1001, 1012, 1045, 1050, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1064, 1012, 1013, 1085, 1000, 1033, 1012, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8224, 8078, 8248, 8144, 8139, 8261, 8283] avg=8192.0000 max=8283 max_rank=7 min=8078 min_rank=2 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1029, 1006, 1031, 958, 1009, 995, 1020, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 980, 988, 1062, 1062, 1063, 995, 993, 6, 4, 0, 0, 0, 0, 0, 0, 977, 1021, 1053, 1039, 990, 1062, 1031, 1054, 3, 0, 0, 0, 0, 0, 0, 0, 1009, 1044, 995, 1017, 1035, 1051, 1025, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1004, 925, 1019, 1013, 986, 1018, 1003, 6, 5, 0, 0, 0, 0, 0, 0, 970, 1043, 1046, 1044, 1075, 1067, 1027, 1045, 28, 0, 0, 0, 0, 0, 0, 0, 1033, 1027, 1056, 1029, 1010, 1014, 1027, 992, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1017, 1093, 1014, 1032, 987, 1062, 1026, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8202, 8230, 8243, 8007, 8345, 8188, 8231] avg=8192.0000 max=8345 max_rank=5 min=8007 min_rank=4 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1074, 1065, 1030, 1030, 1028, 1033, 1022, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1034, 985, 1014, 1017, 1011, 992, 1053, 6, 4, 0, 0, 0, 0, 0, 0, 1025, 1048, 1060, 1066, 1000, 1011, 1004, 1053, 3, 0, 0, 0, 0, 0, 0, 0, 1052, 1015, 995, 992, 1065, 1004, 989, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1026, 1021, 1004, 1010, 1044, 1036, 1037, 6, 5, 0, 0, 0, 0, 0, 0, 1043, 985, 990, 986, 971, 1015, 1024, 994, 28, 0, 0, 0, 0, 0, 0, 0, 1033, 1046, 1029, 1036, 1026, 1027, 1001, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1022, 1061, 1038, 1015, 1065, 931, 1049, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8159, 8270, 8135, 8184, 8036, 8221, 8253] avg=8192.0000 max=8278 max_rank=0 min=8036 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8064, 8320, 8320, 8320, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1040, 1010, 1031, 1044, 978, 979, 1005, 1040, 1047, 1005, 1000, 1006, 1059, 1024, 1043, 1022, 1062, 1056, 995, 1020, 1043, 994, 1043, 975, 1056, 1023, 1040, 1007, 1022, 1006, 988, 1020, 987, 1046, 972, 1049, 1041, 991, 1027, 1023, 1055, 1006, 1009, 1030, 1050, 1026, 983, 1030, 989, 1005, 1055, 1067, 1035, 1022, 1018, 1002, 1037, 1078, 1095, 1031, 995, 1050, 1025, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[988, 1003, 990, 990, 1058, 1039, 1042, 1038, 1052, 1030, 1054, 970, 1012, 1044, 1005, 1009, 1051, 1006, 1021, 1028, 1003, 1018, 1009, 1076, 1002, 986, 1056, 1044, 1074, 1009, 1054, 975, 1024, 1040, 1016, 1054, 1040, 987, 999, 1053, 1012, 975, 1025, 1034, 1102, 1032, 1063, 1015, 999, 1005, 999, 1105, 1031, 1043, 1039, 986, 1035, 1040, 1038, 1011, 987, 1006, 970, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1002, 1007, 1025, 1039, 1015, 1037, 1034, 1016, 1020, 1060, 1035, 1062, 1012, 995, 1021, 1054, 1043, 1005, 981, 1014, 1076, 1000, 1028, 1029, 1075, 1025, 973, 1025, 1015, 1021, 1024, 1035, 985, 1031, 1022, 1015, 1066, 1063, 989, 1077, 1011, 1029, 1022, 1048, 1029, 1027, 998, 1016, 1049, 994, 970, 1051, 1008, 1029, 1048, 1034, 970, 974, 975, 1016, 1049, 1061, 1042, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1076, 1018, 1023, 1012, 1054, 1039, 1026, 1036, 1011, 1007, 987, 1034, 995, 1036, 988, 1005, 1064, 1066, 980, 1013, 1045, 1030, 1030, 979, 1030, 1038, 1033, 1020, 1044, 1074, 1065, 1036, 996, 1095, 1091, 1012, 1033, 1000, 988, 1046, 1029, 990, 1036, 1006, 1034, 993, 1025, 1030, 991, 1004, 986, 994, 1018, 1043, 1029, 1020, 1029, 1037, 1024, 1044, 999, 992, 996, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1030, 1025, 1075, 1040, 1023, 1036, 1029, 1039, 1033, 964, 1018, 1014, 1015, 1005, 1026, 1046, 1016, 1038, 1012, 1036, 1015, 1075, 1052, 1038, 1059, 1050, 1036, 1014, 976, 1008, 1044, 1062, 986, 1015, 991, 1052, 964, 1012, 1012, 1013, 1017, 1014, 977, 986, 1050, 1046, 1025, 1038, 1042, 1024, 984, 1075, 1006, 1009, 1034, 1036, 1044, 1061, 1015, 997, 1012, 1032, 996, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1017, 994, 1013, 999, 1019, 1031, 1045, 1045, 1049, 1076, 1027, 1072, 1020, 1059, 1054, 987, 1012, 1036, 1050, 1023, 990, 1002, 996, 1047, 984, 1016, 1063, 1049, 975, 997, 1016, 1065, 969, 992, 1037, 1064, 1013, 1002, 1039, 985, 947, 1084, 1007, 1001, 1036, 1037, 1043, 1031, 1030, 1027, 1049, 1016, 1040, 1022, 1012, 1014, 1137, 1034, 1007, 1020, 991, 1035, 981, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1008, 1006, 1019, 995, 1040, 1022, 997, 1036, 968, 1028, 1026, 1057, 1052, 996, 1031, 1010, 1034, 974, 1033, 1043, 1045, 1043, 1039, 1002, 1021, 1003, 1027, 1032, 1094, 1069, 1052, 1042, 1059, 1014, 1007, 991, 1067, 1014, 1043, 1026, 1029, 983, 1058, 1013, 1038, 1009, 1010, 989, 1035, 1031, 1006, 1008, 1038, 1015, 1004, 1010, 1006, 1016, 1041, 1014, 1063, 997, 1053, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[990, 1088, 1071, 1022, 1037, 1043, 1074, 1015, 1048, 1048, 993, 1027, 986, 1014, 1024, 994, 969, 1006, 1038, 1027, 1048, 979, 1045, 1007, 1055, 1029, 1054, 1046, 982, 1076, 1032, 1023, 988, 1012, 1045, 1002, 1030, 1011, 1011, 982, 1045, 1063, 1010, 1055, 1037, 1024, 998, 1029, 1094, 1054, 1023, 986, 1043, 1000, 987, 955, 1030, 1041, 986, 1021, 1070, 1027, 979, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1040, 1010, 1031, 1044, 978, 979, 1005, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1005, 1000, 1006, 1059, 1024, 1043, 1022, 5, 2, 0, 0, 0, 0, 0, 0, 1062, 1056, 995, 1020, 1041, 994, 1043, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1023, 1040, 1007, 1022, 1006, 988, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1046, 972, 1049, 1041, 991, 1027, 1023, 29, 0, 0, 0, 0, 0, 0, 0, 1055, 1006, 1009, 1030, 1050, 1026, 983, 1030, 2, 2, 0, 0, 0, 0, 0, 0, 989, 1005, 1055, 1067, 1035, 1022, 1018, 1002, 20, 0, 0, 0, 0, 0, 0, 0, 1037, 1078, 1095, 1031, 995, 1050, 1025, 1024, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8213, 8186, 8111, 8165, 8193, 8213, 8342] avg=8192.0000 max=8342 max_rank=7 min=8111 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1002, 1007, 1025, 1039, 1015, 1037, 1034, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1060, 1035, 1062, 1012, 995, 1021, 1054, 5, 2, 0, 0, 0, 0, 0, 0, 1043, 1005, 981, 1014, 1074, 1000, 1028, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1025, 973, 1025, 1015, 1021, 1024, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1031, 1022, 1015, 1066, 1063, 989, 1077, 29, 0, 0, 0, 0, 0, 0, 0, 1011, 1029, 1022, 1048, 1029, 1027, 998, 1016, 2, 2, 0, 0, 0, 0, 0, 0, 1049, 994, 970, 1051, 1008, 1029, 1048, 1034, 20, 0, 0, 0, 0, 0, 0, 0, 970, 974, 975, 1016, 1049, 1061, 1042, 1035, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8266, 8174, 8142, 8277, 8184, 8203, 8129] avg=8192.0000 max=8277 max_rank=4 min=8129 min_rank=7 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1076, 1018, 1023, 1012, 1054, 1039, 1026, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1007, 987, 1034, 995, 1036, 988, 1005, 5, 2, 0, 0, 0, 0, 0, 0, 1064, 1066, 980, 1013, 1043, 1030, 1030, 979, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1038, 1033, 1020, 1044, 1074, 1065, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1095, 1091, 1012, 1033, 1000, 988, 1046, 28, 0, 0, 0, 0, 0, 0, 0, 1029, 990, 1036, 1006, 1034, 993, 1025, 1030, 2, 2, 0, 0, 0, 0, 0, 0, 991, 1004, 986, 994, 1018, 1043, 1029, 1020, 19, 0, 0, 0, 0, 0, 0, 0, 1029, 1037, 1024, 1044, 999, 992, 996, 1032, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8070, 8205, 8291, 8289, 8147, 8104, 8160] avg=8192.0000 max=8291 max_rank=3 min=8070 min_rank=1 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[990, 1088, 1071, 1022, 1037, 1043, 1074, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1048, 993, 1027, 986, 1014, 1024, 994, 5, 2, 0, 0, 0, 0, 0, 0, 969, 1006, 1038, 1027, 1046, 979, 1045, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1029, 1054, 1046, 982, 1076, 1032, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1012, 1045, 1002, 1030, 1011, 1011, 982, 29, 0, 0, 0, 0, 0, 0, 0, 1045, 1063, 1010, 1055, 1037, 1024, 998, 1029, 2, 2, 0, 0, 0, 0, 0, 0, 1094, 1054, 1023, 986, 1043, 1000, 987, 955, 20, 0, 0, 0, 0, 0, 0, 0, 1030, 1041, 986, 1021, 1070, 1027, 979, 1008, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8326, 8141, 8117, 8246, 8110, 8265, 8162, 8169] avg=8192.0000 max=8326 max_rank=0 min=8110 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[988, 1003, 990, 990, 1058, 1039, 1042, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1030, 1054, 970, 1012, 1044, 1005, 1009, 9, 7, 0, 0, 0, 0, 0, 0, 1051, 1006, 1021, 1028, 1000, 1018, 1009, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 941, 986, 1056, 1044, 1074, 1009, 1054, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1040, 1016, 1054, 1040, 987, 999, 1053, 31, 0, 0, 0, 0, 0, 0, 0, 1012, 975, 1025, 1034, 1102, 1032, 1063, 1015, 6, 3, 0, 0, 0, 0, 0, 0, 999, 1005, 999, 1105, 1031, 1043, 1039, 986, 23, 0, 0, 0, 0, 0, 0, 0, 1035, 1040, 1038, 1011, 987, 1006, 970, 1035, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8192, 8209, 8139, 8244, 8267, 8230, 8130] avg=8192.0000 max=8267 max_rank=5 min=8125 min_rank=0 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1008, 1006, 1019, 995, 1040, 1022, 997, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1028, 1026, 1057, 1052, 996, 1031, 1010, 5, 2, 0, 0, 0, 0, 0, 0, 1034, 974, 1033, 1043, 1043, 1043, 1039, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1003, 1027, 1032, 1094, 1069, 1052, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1014, 1007, 991, 1067, 1014, 1043, 1026, 28, 0, 0, 0, 0, 0, 0, 0, 1029, 983, 1058, 1013, 1038, 1009, 1010, 989, 2, 2, 0, 0, 0, 0, 0, 0, 1035, 1031, 1006, 1008, 1038, 1015, 1004, 1010, 19, 0, 0, 0, 0, 0, 0, 0, 1006, 1016, 1041, 1014, 1063, 997, 1053, 1005, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8175, 8211, 8291, 8249, 8133, 8166, 8202] avg=8192.0000 max=8291 max_rank=3 min=8109 min_rank=0 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1030, 1025, 1075, 1040, 1023, 1036, 1029, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 964, 1018, 1014, 1015, 1005, 1026, 1046, 5, 2, 0, 0, 0, 0, 0, 0, 1016, 1038, 1012, 1036, 1013, 1075, 1052, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1050, 1036, 1014, 976, 1008, 1044, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1015, 991, 1052, 964, 1012, 1012, 1013, 29, 0, 0, 0, 0, 0, 0, 0, 1017, 1014, 977, 986, 1050, 1046, 1025, 1038, 2, 2, 0, 0, 0, 0, 0, 0, 1042, 1024, 984, 1075, 1006, 1009, 1034, 1036, 20, 0, 0, 0, 0, 0, 0, 0, 1044, 1061, 1015, 997, 1012, 1032, 996, 1022, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8128, 8280, 8198, 8074, 8157, 8230, 8186] avg=8192.0000 max=8283 max_rank=0 min=8074 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1017, 994, 1013, 999, 1019, 1031, 1045, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1076, 1027, 1072, 1020, 1059, 1054, 987, 5, 2, 0, 0, 0, 0, 0, 0, 1012, 1036, 1050, 1023, 988, 1002, 996, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 937, 1016, 1063, 1049, 975, 997, 1016, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 969, 992, 1037, 1064, 1013, 1002, 1039, 985, 27, 0, 0, 0, 0, 0, 0, 0, 947, 1084, 1007, 1001, 1036, 1037, 1043, 1031, 2, 2, 0, 0, 0, 0, 0, 0, 1030, 1027, 1049, 1016, 1040, 1022, 1012, 1014, 18, 0, 0, 0, 0, 0, 0, 0, 1137, 1034, 1007, 1020, 991, 1035, 981, 1006, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8351, 8154, 8118, 8128, 8190, 8228, 8218] avg=8192.0000 max=8351 max_rank=1 min=8118 min_rank=3 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8448, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7936, 8192, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8064, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8064, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1012, 1048, 1063, 988, 1044, 1069, 1076, 1016, 1007, 997, 1024, 996, 1019, 1019, 977, 963, 1000, 1038, 1037, 1065, 1033, 1004, 991, 1034, 1077, 1083, 1046, 1012, 1017, 1014, 1043, 1069, 1097, 975, 1005, 999, 1063, 1012, 1046, 996, 972, 947, 1053, 990, 1017, 1042, 1029, 1015, 1025, 1029, 1050, 1043, 1002, 970, 1015, 1045, 1007, 1025, 1018, 1069, 997, 1005, 1051, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1088, 1016, 1050, 1021, 1000, 999, 983, 1035, 1046, 1027, 1015, 1012, 1028, 1039, 1022, 1017, 1007, 1025, 975, 1000, 1049, 1042, 1063, 1033, 1025, 1040, 1032, 993, 1028, 1024, 987, 1054, 1023, 985, 1063, 1026, 1052, 1007, 1018, 1050, 1015, 1038, 1028, 1077, 1083, 994, 992, 1018, 1007, 1040, 1027, 1013, 1039, 997, 1014, 1026, 1000, 1010, 989, 1045, 1048, 996, 1035, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[991, 998, 1027, 1058, 1035, 974, 1101, 1028, 1017, 1027, 1050, 973, 1040, 1010, 1013, 1024, 1050, 1058, 1002, 1001, 973, 1072, 1029, 1056, 1028, 1013, 1039, 1031, 1042, 987, 1011, 1055, 1084, 1023, 1041, 1022, 1006, 1014, 1080, 1012, 1014, 998, 1037, 1026, 999, 1057, 1008, 983, 1039, 1023, 1035, 1000, 1028, 1004, 1044, 1031, 971, 1037, 1042, 1018, 1024, 1017, 1014, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[989, 1010, 1022, 1022, 1005, 1005, 1023, 1043, 1043, 1016, 1009, 1037, 1073, 1025, 1037, 983, 1002, 1073, 1041, 1014, 1027, 1034, 1010, 1052, 1093, 1026, 1027, 1018, 1044, 1058, 1046, 1000, 1016, 1021, 992, 995, 1039, 1030, 1008, 1017, 1068, 1017, 1015, 1053, 1020, 1017, 999, 1025, 966, 1041, 1010, 1040, 1024, 984, 1027, 1004, 1051, 974, 1020, 1027, 995, 1062, 1032, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1034, 1018, 1013, 1002, 997, 1041, 1067, 1055, 1046, 1053, 1035, 1038, 1060, 1046, 1010, 1049, 956, 971, 996, 1036, 1024, 1030, 1022, 987, 1044, 1054, 1042, 1068, 1001, 984, 991, 1033, 1037, 982, 1009, 1002, 1011, 980, 970, 1076, 1073, 1021, 1046, 1039, 1020, 1053, 1074, 990, 1080, 988, 994, 983, 1004, 1028, 1050, 1011, 1049, 1080, 998, 1008, 986, 1092, 988, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1029, 985, 1052, 1032, 1020, 1083, 1054, 1046, 1027, 1086, 1053, 1030, 1018, 1025, 1014, 1050, 1063, 1031, 1041, 1003, 1019, 1001, 992, 1066, 1067, 1028, 1076, 1023, 1017, 1008, 1053, 1023, 1027, 1003, 1031, 1023, 971, 1036, 1033, 1017, 1026, 1014, 1057, 969, 1002, 1027, 1052, 1061, 988, 1011, 971, 1026, 1031, 1004, 1041, 948, 1035, 993, 1021, 1009, 1029, 1001, 1007, 957] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[992, 1020, 967, 1042, 993, 985, 1022, 1022, 1021, 1017, 1021, 1048, 1047, 1044, 1005, 1024, 1006, 1014, 1047, 1076, 1031, 1060, 1068, 1002, 1031, 1006, 1020, 1013, 988, 988, 1014, 1001, 992, 1056, 1010, 1047, 1026, 1015, 1014, 1014, 1057, 1010, 1010, 1035, 954, 989, 1026, 1012, 1042, 1047, 1050, 997, 1035, 1051, 1018, 1049, 1015, 1047, 1020, 1029, 1072, 1092, 1030, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1028, 1007, 1041, 1042, 996, 1026, 1052, 1004, 1000, 985, 981, 1067, 1006, 989, 1057, 1013, 1059, 994, 1024, 1052, 1011, 1007, 1041, 1042, 1054, 1039, 1037, 1010, 1082, 1025, 1047, 1042, 968, 1041, 1018, 1015, 1019, 989, 1076, 1030, 1001, 1062, 1009, 1040, 1039, 999, 1008, 1023, 1013, 1064, 1047, 1020, 989, 1061, 995, 1037, 1059, 1013, 972, 984, 1015, 1018, 1001, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1012, 1048, 1063, 988, 1044, 1069, 1068, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 997, 1024, 996, 1018, 1019, 977, 963, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1038, 1037, 1065, 1033, 1004, 991, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1083, 1046, 1012, 1017, 1014, 1043, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 975, 1005, 999, 1063, 1012, 1046, 996, 23, 0, 0, 0, 0, 0, 0, 0, 972, 947, 1053, 990, 1017, 1042, 1029, 1015, 8, 1, 1, 0, 0, 0, 0, 0, 1025, 1029, 1050, 1043, 1002, 970, 1015, 1045, 28, 0, 0, 0, 0, 0, 0, 0, 1007, 1025, 1018, 1069, 997, 1005, 1051, 1046, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8001, 8187, 8306, 8216, 8075, 8207, 8236] avg=8192.0000 max=8308 max_rank=0 min=8001 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1029, 985, 1052, 1032, 1020, 1083, 1046, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1086, 1053, 1030, 1017, 1025, 1014, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1031, 1041, 1003, 1019, 1001, 992, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1028, 1076, 1023, 1017, 1008, 1053, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1003, 1031, 1023, 971, 1036, 1033, 1017, 22, 0, 0, 0, 0, 0, 0, 0, 1026, 1014, 1057, 969, 1002, 1027, 1052, 1061, 8, 1, 1, 0, 0, 0, 0, 0, 988, 1011, 971, 1026, 1031, 1004, 1041, 948, 28, 0, 0, 0, 0, 0, 0, 0, 1035, 993, 1021, 1009, 1029, 1001, 1007, 957, 15, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8302, 8200, 8241, 8163, 8218, 8048, 8071] avg=8192.0000 max=8302 max_rank=1 min=8048 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1088, 1016, 1050, 1021, 1000, 999, 975, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1027, 1015, 1012, 1027, 1039, 1022, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1025, 975, 1000, 1049, 1042, 1063, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1040, 1032, 993, 1028, 1024, 987, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 985, 1063, 1026, 1052, 1007, 1018, 1050, 22, 0, 0, 0, 0, 0, 0, 0, 1015, 1038, 1028, 1077, 1083, 994, 992, 1018, 8, 1, 0, 0, 0, 0, 0, 0, 1007, 1040, 1027, 1013, 1039, 997, 1014, 1026, 26, 0, 0, 0, 0, 0, 0, 0, 1000, 1010, 989, 1045, 1048, 996, 1035, 1006, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8205, 8180, 8131, 8246, 8254, 8189, 8147] avg=8192.0000 max=8254 max_rank=5 min=8131 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1034, 1018, 1013, 1002, 997, 1041, 1054, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1053, 1035, 1038, 1053, 1046, 1010, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 956, 971, 996, 1036, 1024, 1030, 1022, 967, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1054, 1042, 1068, 1001, 984, 991, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 982, 1009, 1002, 1011, 980, 970, 1076, 25, 0, 0, 0, 0, 0, 0, 0, 1073, 1021, 1046, 1039, 1020, 1053, 1074, 990, 13, 7, 3, 0, 0, 0, 0, 0, 1080, 988, 994, 983, 1004, 1028, 1050, 1011, 31, 0, 0, 0, 0, 0, 0, 0, 1049, 1080, 998, 1008, 986, 1092, 988, 1011, 17, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8330, 8002, 8156, 8092, 8339, 8169, 8234] avg=8192.0000 max=8339 max_rank=5 min=8002 min_rank=2 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[991, 998, 1027, 1058, 1035, 974, 1092, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1027, 1050, 973, 1039, 1010, 1013, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1058, 1002, 1001, 973, 1072, 1029, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1013, 1039, 1031, 1042, 987, 1011, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 1023, 1041, 1022, 1006, 1014, 1080, 1012, 22, 0, 0, 0, 0, 0, 0, 0, 1014, 998, 1037, 1026, 999, 1057, 1008, 983, 9, 1, 1, 0, 0, 0, 0, 0, 1039, 1023, 1035, 1000, 1028, 1004, 1044, 1031, 26, 0, 0, 0, 0, 0, 0, 0, 971, 1037, 1042, 1018, 1024, 1017, 1014, 992, 15, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8153, 8225, 8154, 8304, 8133, 8230, 8134] avg=8192.0000 max=8304 max_rank=4 min=8133 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[989, 1010, 1022, 1022, 1005, 1005, 1015, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1016, 1009, 1037, 1072, 1025, 1037, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1073, 1041, 1014, 1027, 1034, 1010, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1026, 1027, 1018, 1044, 1058, 1046, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1021, 992, 995, 1039, 1030, 1008, 1017, 23, 0, 0, 0, 0, 0, 0, 0, 1068, 1017, 1015, 1053, 1020, 1017, 999, 1025, 8, 1, 1, 0, 0, 0, 0, 0, 966, 1041, 1010, 1040, 1024, 984, 1027, 1004, 28, 0, 0, 0, 0, 0, 0, 0, 1051, 974, 1020, 1027, 995, 1062, 1032, 1040, 15, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8222, 8237, 8257, 8141, 8224, 8124, 8220] avg=8192.0000 max=8257 max_rank=3 min=8111 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[992, 1020, 967, 1042, 993, 985, 1014, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1017, 1021, 1048, 1046, 1044, 1005, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1014, 1047, 1076, 1031, 1060, 1068, 988, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1006, 1020, 1013, 988, 988, 1014, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1056, 1010, 1047, 1026, 1015, 1014, 1014, 22, 0, 0, 0, 0, 0, 0, 0, 1057, 1010, 1010, 1035, 954, 989, 1026, 1012, 8, 1, 0, 0, 0, 0, 0, 0, 1042, 1047, 1050, 997, 1035, 1051, 1018, 1049, 27, 0, 0, 0, 0, 0, 0, 0, 1015, 1047, 1020, 1029, 1072, 1092, 1030, 1040, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8035, 8226, 8290, 8008, 8196, 8102, 8316, 8363] avg=8192.0000 max=8363 max_rank=7 min=8008 min_rank=3 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32977 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1028, 1007, 1041, 1042, 996, 1026, 1044, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 985, 981, 1067, 1005, 989, 1057, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 994, 1024, 1052, 1011, 1007, 1041, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1039, 1037, 1010, 1082, 1025, 1047, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1041, 1018, 1015, 1019, 989, 1076, 1030, 22, 0, 0, 0, 0, 0, 0, 0, 1001, 1062, 1009, 1040, 1039, 999, 1008, 1023, 8, 1, 1, 0, 0, 0, 0, 0, 1013, 1064, 1047, 1020, 989, 1061, 995, 1037, 27, 0, 0, 0, 0, 0, 0, 0, 1059, 1013, 972, 984, 1015, 1018, 1001, 1051, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8097, 8215, 8283, 8178, 8191, 8253, 8131] avg=8192.0000 max=8283 max_rank=3 min=8097 min_rank=1 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1030, 1051, 1000, 1046, 1009, 1001, 978, 1052, 1043, 1043, 1054, 1028, 1003, 1022, 967, 1039, 972, 993, 1003, 1019, 997, 994, 1052, 1050, 991, 1057, 1000, 964, 1069, 998, 1025, 1046, 1007, 1031, 989, 1046, 1027, 1012, 975, 1033, 1063, 1037, 1040, 1017, 1017, 1011, 1029, 1052, 993, 1027, 1012, 1008, 1068, 1056, 1041, 1103, 1021, 1048, 1017, 1038, 1039, 1027, 1012, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[968, 987, 1043, 997, 1033, 987, 1030, 1037, 1036, 1034, 1062, 1027, 1028, 978, 992, 1030, 971, 1002, 1025, 1038, 1041, 1049, 1042, 986, 1010, 1011, 1042, 1020, 1005, 1048, 1031, 991, 1009, 988, 1043, 1014, 1079, 1013, 1002, 991, 1015, 1058, 998, 1041, 1044, 1055, 1055, 1019, 1006, 1013, 1027, 1038, 999, 1017, 1016, 1088, 1107, 1004, 990, 1081, 1050, 1044, 1017, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[994, 1000, 1076, 999, 1080, 1089, 1052, 979, 1032, 1000, 1051, 1046, 1020, 1048, 1089, 1068, 997, 960, 1006, 1059, 1031, 1066, 1060, 1027, 1024, 1040, 1002, 965, 1013, 1045, 1002, 1035, 1023, 967, 996, 1017, 1006, 1023, 1059, 975, 987, 1040, 1038, 1046, 980, 1050, 1044, 1050, 1023, 993, 1015, 1011, 1019, 1007, 993, 1030, 997, 1006, 1080, 1007, 974, 1030, 1031, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1035, 1057, 1002, 1040, 1026, 1065, 1052, 1011, 1026, 997, 1039, 1075, 984, 958, 957, 997, 1000, 1033, 1006, 1005, 1007, 998, 996, 1029, 1006, 1036, 1038, 1038, 1049, 994, 998, 1083, 1008, 1012, 1035, 1016, 1051, 1044, 1047, 999, 1054, 1049, 1029, 1055, 1081, 1050, 1024, 1029, 1001, 1006, 1008, 982, 1068, 963, 1020, 988, 1031, 1044, 1026, 1024, 1047, 1041, 1023, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[985, 1010, 1048, 1058, 1032, 1024, 1028, 1028, 1058, 1054, 1044, 1043, 996, 1035, 1039, 1052, 1064, 1029, 1002, 1025, 999, 1010, 969, 1088, 1038, 1025, 991, 1035, 1051, 999, 1054, 1023, 1010, 1034, 991, 995, 1008, 1003, 1044, 1014, 1031, 1049, 1052, 1020, 1004, 980, 1036, 1017, 982, 985, 1031, 1088, 1032, 1030, 982, 1045, 1051, 1031, 1013, 1008, 965, 1017, 1062, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1044, 1037, 1055, 1022, 1040, 1033, 1041, 1047, 1003, 995, 981, 1025, 1044, 1020, 1037, 1051, 1029, 1056, 1036, 1047, 976, 1056, 1005, 1029, 997, 1058, 974, 953, 982, 1006, 1016, 1051, 1030, 1025, 1036, 1031, 1003, 1001, 985, 1015, 1016, 1022, 1022, 1020, 1039, 999, 1034, 1015, 1035, 1043, 988, 1042, 1045, 1039, 1019, 1018, 1012, 1048, 1053, 1036, 1008, 1080, 1003, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1036, 1018, 1045, 1026, 1040, 1026, 1034, 997, 1034, 980, 1043, 1026, 927, 1017, 1036, 1064, 1057, 1024, 987, 1035, 1012, 1008, 1060, 998, 994, 1009, 1024, 1052, 1016, 1042, 1051, 1026, 1022, 1051, 1022, 1006, 1039, 1015, 1039, 1005, 1032, 937, 1040, 1002, 1089, 1020, 1047, 1036, 1019, 1036, 1008, 1065, 1047, 1067, 1040, 1021, 959, 1061, 983, 1002, 1034, 979, 1035, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1070, 1040, 974, 1032, 1056, 993, 1043, 1001, 1035, 1005, 1059, 1042, 1022, 1026, 1049, 987, 1063, 1005, 1011, 985, 1014, 1038, 1045, 1061, 1032, 1003, 1074, 1009, 1020, 1051, 1059, 1059, 1018, 1015, 1045, 1015, 1008, 1006, 1011, 1035, 1009, 1003, 1033, 1043, 1020, 1026, 1044, 1042, 1011, 990, 1017, 965, 1027, 1020, 1043, 1030, 1030, 1034, 1031, 1012, 973, 989, 1030, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[968, 987, 1043, 997, 1006, 987, 1030, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1034, 1054, 1027, 1028, 978, 992, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1002, 1025, 1038, 1041, 1049, 1042, 986, 24, 0, 0, 0, 0, 0, 0, 0, 1010, 1011, 1042, 1020, 1005, 1048, 1031, 991, 8, 2, 0, 0, 0, 0, 0, 0, 1009, 988, 1043, 1014, 1079, 1013, 1002, 991, 50, 2, 0, 0, 0, 0, 0, 0, 1015, 1058, 998, 1041, 1044, 1055, 1005, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1013, 1027, 1038, 999, 1017, 1016, 1088, 9, 1, 0, 0, 0, 0, 0, 0, 1107, 993, 990, 1081, 1050, 1044, 1017, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8055, 8179, 8178, 8168, 8191, 8235, 8214, 8316] avg=8192.0000 max=8316 max_rank=7 min=8055 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32956 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1030, 1051, 1000, 1046, 982, 1001, 978, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1043, 1046, 1028, 1003, 1022, 967, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 972, 993, 1003, 1019, 997, 994, 1052, 1050, 24, 0, 0, 0, 0, 0, 0, 0, 991, 1057, 1000, 964, 1069, 998, 1025, 1046, 8, 2, 0, 0, 0, 0, 0, 0, 1007, 1031, 989, 1046, 1027, 1012, 975, 1033, 49, 2, 0, 0, 0, 0, 0, 0, 1063, 1037, 1040, 1017, 1017, 1011, 980, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1027, 1012, 1008, 1068, 1056, 1041, 1103, 9, 1, 0, 0, 0, 0, 0, 0, 1021, 1037, 1017, 1038, 1039, 1027, 1012, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8191, 8104, 8160, 8171, 8217, 8318, 8235] avg=8192.0000 max=8318 max_rank=6 min=8104 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32941 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[994, 1000, 1076, 999, 1052, 1089, 1052, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1000, 1043, 1046, 1020, 1048, 1089, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 997, 960, 1006, 1059, 1031, 1066, 1060, 1027, 25, 0, 0, 0, 0, 0, 0, 0, 1024, 1040, 1002, 965, 1013, 1045, 1002, 1035, 8, 2, 0, 0, 0, 0, 0, 0, 1023, 967, 996, 1017, 1006, 1023, 1059, 975, 50, 2, 0, 0, 0, 0, 0, 0, 987, 1040, 1038, 1046, 980, 1050, 994, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 993, 1015, 1011, 1019, 1007, 993, 1030, 9, 1, 0, 0, 0, 0, 0, 0, 997, 995, 1080, 1007, 974, 1030, 1031, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8346, 8231, 8136, 8118, 8185, 8101, 8178] avg=8192.0000 max=8346 max_rank=1 min=8101 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1070, 1040, 974, 1032, 1028, 993, 1043, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1005, 1051, 1042, 1022, 1026, 1049, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1005, 1011, 985, 1014, 1038, 1045, 1061, 25, 0, 0, 0, 0, 0, 0, 0, 1032, 1003, 1074, 1009, 1020, 1051, 1059, 1059, 8, 2, 0, 0, 0, 0, 0, 0, 1018, 1015, 1045, 1015, 1008, 1006, 1011, 1035, 50, 2, 0, 0, 0, 0, 0, 0, 1009, 1003, 1033, 1043, 1020, 1026, 994, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 990, 1017, 965, 1027, 1020, 1043, 1030, 9, 1, 0, 0, 0, 0, 0, 0, 1030, 1023, 1031, 1012, 973, 989, 1030, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8217, 8247, 8317, 8205, 8170, 8113, 8086] avg=8192.0000 max=8317 max_rank=3 min=8086 min_rank=7 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32574 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1044, 1037, 1055, 1022, 998, 1033, 1041, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 995, 971, 1025, 1044, 1020, 1037, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1056, 1036, 1047, 976, 1056, 1005, 1029, 29, 0, 0, 0, 0, 0, 0, 0, 997, 1058, 974, 953, 982, 1006, 1016, 1051, 10, 6, 0, 0, 0, 0, 0, 0, 1030, 1025, 1036, 1031, 1003, 1001, 985, 1015, 53, 8, 0, 0, 0, 0, 0, 0, 1016, 1022, 1022, 1020, 1039, 999, 981, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1043, 988, 1042, 1045, 1039, 1019, 1018, 12, 5, 0, 0, 0, 0, 0, 0, 1012, 1030, 1053, 1036, 1008, 1080, 1003, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8146, 8263, 8053, 8187, 8114, 8246, 8250] avg=8192.0000 max=8277 max_rank=0 min=8053 min_rank=3 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[985, 1010, 1048, 1058, 1005, 1024, 1028, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1054, 1036, 1043, 996, 1035, 1039, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1029, 1002, 1025, 999, 1010, 969, 1088, 24, 0, 0, 0, 0, 0, 0, 0, 1038, 1025, 991, 1035, 1051, 999, 1054, 1023, 8, 2, 0, 0, 0, 0, 0, 0, 1010, 1034, 991, 995, 1008, 1003, 1044, 1014, 49, 2, 0, 0, 0, 0, 0, 0, 1031, 1049, 1052, 1020, 1004, 980, 987, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 982, 985, 1031, 1088, 1032, 1030, 982, 1045, 9, 1, 0, 0, 0, 0, 0, 0, 1051, 1020, 1013, 1008, 965, 1017, 1062, 990, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8313, 8210, 8226, 8150, 8140, 8185, 8126] avg=8192.0000 max=8313 max_rank=1 min=8126 min_rank=7 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1036, 1018, 1045, 1026, 1013, 1026, 1034, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 980, 1035, 1026, 927, 1017, 1036, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1024, 987, 1035, 1012, 1008, 1060, 998, 24, 0, 0, 0, 0, 0, 0, 0, 994, 1009, 1024, 1052, 1016, 1042, 1051, 1026, 8, 2, 0, 0, 0, 0, 0, 0, 1022, 1051, 1022, 1006, 1039, 1015, 1039, 1005, 50, 2, 0, 0, 0, 0, 0, 0, 1032, 937, 1040, 1002, 1089, 1020, 997, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1036, 1008, 1065, 1047, 1067, 1040, 1021, 9, 1, 0, 0, 0, 0, 0, 0, 959, 1050, 983, 1002, 1034, 979, 1035, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8119, 8205, 8224, 8251, 8153, 8313, 8076] avg=8192.0000 max=8313 max_rank=6 min=8076 min_rank=7 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1035, 1057, 1002, 1040, 999, 1065, 1052, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 997, 1031, 1075, 984, 958, 957, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1033, 1006, 1005, 1007, 998, 996, 1029, 24, 0, 0, 0, 0, 0, 0, 0, 1006, 1036, 1038, 1038, 1049, 994, 998, 1083, 8, 2, 0, 0, 0, 0, 0, 0, 1008, 1012, 1035, 1016, 1051, 1044, 1047, 999, 49, 2, 0, 0, 0, 0, 0, 0, 1054, 1049, 1029, 1055, 1081, 1050, 975, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1006, 1008, 982, 1068, 963, 1020, 988, 9, 1, 0, 0, 0, 0, 0, 0, 1031, 1033, 1026, 1024, 1047, 1041, 1023, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8025, 8098, 8252, 8263, 8322, 8046, 8269] avg=8192.0000 max=8322 max_rank=5 min=8025 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32900 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8192, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8192, 8448, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8064, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 7936, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1006, 1005, 1006, 1017, 995, 1021, 1014, 999, 1028, 1053, 1016, 1050, 1030, 1063, 1041, 1067, 1039, 1012, 1084, 1031, 1081, 1037, 1036, 1029, 1014, 1031, 973, 1014, 1043, 1043, 1015, 1004, 1034, 1022, 1083, 1022, 1003, 981, 986, 1020, 975, 966, 1060, 1033, 1030, 997, 994, 1011, 1018, 1047, 1055, 1019, 1042, 1044, 1058, 1040, 1040, 1033, 1001, 1035, 927, 1001, 1017, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1018, 974, 1037, 1031, 1085, 989, 951, 1019, 960, 1016, 1049, 1065, 1050, 988, 983, 1059, 1062, 1028, 1008, 970, 999, 1010, 1052, 1030, 1001, 994, 1056, 1012, 1019, 1019, 1069, 1010, 982, 1066, 1045, 1018, 1001, 1037, 1033, 1049, 1046, 1023, 979, 1012, 1012, 1032, 1065, 993, 1025, 1019, 1034, 1024, 1061, 1037, 1007, 968, 1043, 1049, 1064, 1057, 1026, 1055, 989, 1072] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[985, 969, 1009, 1027, 1037, 989, 1067, 1041, 990, 1007, 1036, 1017, 1020, 1033, 1000, 1021, 1038, 981, 1054, 977, 1068, 1039, 1017, 981, 1055, 1020, 1012, 995, 982, 1005, 1041, 1040, 996, 1033, 991, 1011, 1026, 1039, 1027, 1004, 998, 1081, 1030, 1028, 1063, 1063, 1074, 1038, 1033, 1044, 965, 1073, 1007, 1022, 1013, 1086, 1046, 983, 1018, 1000, 1093, 1019, 1055, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[993, 1028, 991, 977, 989, 1034, 1016, 1018, 1022, 1038, 1032, 983, 1048, 1065, 991, 1053, 1044, 1055, 1029, 1031, 1034, 1023, 1018, 1016, 1032, 1083, 995, 1017, 1028, 1016, 1028, 1018, 1067, 1006, 1020, 996, 1067, 1016, 1021, 994, 1009, 997, 1042, 1009, 1048, 1042, 1044, 1045, 1000, 1057, 988, 1051, 1024, 1032, 1043, 1030, 1039, 987, 1047, 988, 1042, 997, 979, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1013, 1043, 1014, 1014, 1038, 1028, 1021, 1066, 1006, 1010, 1036, 1033, 1047, 995, 1035, 970, 1068, 1062, 1024, 985, 971, 1057, 1063, 998, 983, 1037, 1020, 1025, 1010, 989, 1079, 1033, 1060, 1063, 1038, 1043, 1025, 1003, 1042, 1057, 1023, 985, 1001, 1029, 1079, 1068, 1022, 1051, 950, 1030, 1041, 997, 1014, 989, 1033, 995, 1036, 1038, 1037, 988, 996, 1025, 1029, 976] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1014, 1066, 1012, 1003, 985, 1015, 1041, 1047, 1034, 1020, 1017, 1032, 1006, 980, 1009, 987, 1073, 1009, 1010, 1005, 1074, 1044, 1020, 1047, 1041, 998, 993, 1022, 1004, 1052, 1005, 990, 1021, 1044, 1021, 1019, 1044, 1005, 1019, 1055, 1002, 1047, 1062, 1043, 1035, 1007, 1007, 1009, 1049, 1080, 1047, 1039, 1004, 999, 1038, 1046, 1015, 989, 1008, 1022, 1056, 975, 1044, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1013, 1043, 1026, 1033, 1016, 1011, 989, 1037, 1013, 990, 1014, 1004, 1024, 1058, 1035, 1063, 981, 1062, 1014, 1014, 1041, 1027, 992, 1036, 1013, 1028, 988, 1002, 1044, 992, 1022, 996, 1053, 1012, 1052, 1041, 1005, 1043, 1067, 1054, 989, 1045, 1072, 979, 1035, 1016, 1020, 1031, 1033, 1024, 1017, 983, 1016, 1040, 1022, 1019, 1020, 1035, 958, 1065, 1055, 1078, 1007, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1111, 1018, 1075, 988, 991, 1046, 1012, 1054, 997, 1051, 1006, 1007, 1015, 980, 990, 1057, 1038, 1013, 986, 991, 1019, 973, 1036, 1049, 1022, 1018, 1012, 1005, 1047, 1008, 995, 1011, 967, 1058, 1051, 1060, 1017, 1041, 1033, 1090, 1031, 1023, 984, 1040, 1055, 995, 1001, 1030, 1048, 977, 1079, 1118, 1048, 992, 1027, 1023, 991, 1033, 1038, 979, 1043, 1010, 1017, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1006, 1005, 1006, 1017, 995, 1021, 1014, 999, 29, 9, 0, 0, 0, 0, 0, 0, 1028, 1053, 1016, 1050, 1030, 1063, 1041, 1067, 10, 6, 0, 0, 0, 0, 0, 0, 1020, 1012, 1084, 1031, 1081, 1037, 1036, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1031, 973, 1014, 1043, 1043, 1015, 1004, 44, 0, 0, 0, 0, 0, 0, 0, 1034, 1022, 1083, 1022, 1003, 981, 986, 976, 0, 0, 0, 0, 0, 0, 0, 0, 975, 966, 1060, 1033, 1017, 997, 994, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1047, 1055, 990, 1042, 1044, 1058, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1033, 1001, 1035, 927, 1001, 1017, 1045, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8364, 8330, 8181, 8107, 8053, 8294, 8106] avg=8192.0000 max=8364 max_rank=1 min=8053 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32560 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1013, 1043, 1014, 1014, 1038, 1028, 1021, 1066, 29, 9, 0, 0, 0, 0, 0, 0, 1006, 1010, 1036, 1033, 1047, 995, 1035, 970, 10, 7, 0, 0, 0, 0, 0, 0, 1049, 1062, 1024, 985, 971, 1057, 1063, 998, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1037, 1020, 1025, 1010, 989, 1079, 1033, 46, 0, 0, 0, 0, 0, 0, 0, 1060, 1063, 1038, 1043, 1025, 1003, 1042, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 985, 1001, 1029, 1064, 1068, 1022, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 950, 1030, 1041, 968, 1014, 989, 1033, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1038, 1037, 988, 996, 1025, 1029, 976, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8149, 8209, 8222, 8285, 8243, 8020, 8133] avg=8192.0000 max=8285 max_rank=4 min=8020 min_rank=6 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[985, 969, 1009, 1027, 1037, 989, 1067, 1041, 31, 9, 0, 0, 0, 0, 0, 0, 990, 1007, 1036, 1017, 1020, 1033, 1000, 1021, 10, 7, 0, 0, 0, 0, 0, 0, 1019, 981, 1054, 977, 1068, 1039, 1017, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1020, 1012, 995, 982, 1005, 1041, 1040, 43, 0, 0, 0, 0, 0, 0, 0, 996, 1033, 991, 1011, 1026, 1039, 1027, 961, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1081, 1030, 1028, 1048, 1063, 1074, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1044, 965, 1042, 1007, 1022, 1013, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 983, 1018, 1000, 1093, 1019, 1055, 1024, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8141, 8136, 8193, 8084, 8360, 8212, 8246] avg=8192.0000 max=8360 max_rank=5 min=8084 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32902 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1014, 1066, 1012, 1003, 985, 1015, 1041, 1047, 30, 9, 0, 0, 0, 0, 0, 0, 1034, 1020, 1017, 1032, 1006, 980, 1009, 987, 10, 6, 0, 0, 0, 0, 0, 0, 1054, 1009, 1010, 1005, 1074, 1044, 1020, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 998, 993, 1022, 1004, 1052, 1005, 990, 46, 0, 0, 0, 0, 0, 0, 0, 1021, 1044, 1021, 1019, 1044, 1005, 1019, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1047, 1062, 1043, 1022, 1007, 1007, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1080, 1047, 1009, 1004, 999, 1038, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 989, 1008, 1022, 1056, 975, 1044, 1030, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8101, 8263, 8151, 8182, 8199, 8272, 8146] avg=8192.0000 max=8272 max_rank=6 min=8101 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[993, 1028, 991, 977, 989, 1034, 1016, 1018, 30, 9, 0, 0, 0, 0, 0, 0, 1022, 1038, 1032, 983, 1048, 1065, 991, 1053, 10, 7, 0, 0, 0, 0, 0, 0, 1025, 1055, 1029, 1031, 1034, 1023, 1018, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1083, 995, 1017, 1028, 1016, 1028, 1018, 43, 0, 0, 0, 0, 0, 0, 0, 1067, 1006, 1020, 996, 1067, 1016, 1021, 951, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 997, 1042, 1009, 1034, 1042, 1044, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1057, 988, 1021, 1024, 1032, 1043, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 987, 1047, 988, 1042, 997, 979, 1064, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8085, 8249, 8231, 8260, 8144, 8222, 8195, 8150] avg=8192.0000 max=8260 max_rank=3 min=8085 min_rank=0 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1018, 974, 1037, 1031, 1085, 989, 951, 1019, 29, 9, 0, 0, 0, 0, 0, 0, 960, 1016, 1049, 1065, 1050, 988, 983, 1059, 10, 6, 0, 0, 0, 0, 0, 0, 1043, 1028, 1008, 970, 999, 1010, 1052, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 994, 1056, 1012, 1019, 1019, 1069, 1010, 45, 0, 0, 0, 0, 0, 0, 0, 982, 1066, 1045, 1018, 1001, 1037, 1033, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1023, 979, 1012, 999, 1032, 1065, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1019, 1034, 995, 1061, 1037, 1007, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1049, 1064, 1057, 1026, 1055, 989, 1072, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8186, 8140, 8225, 8186, 8149, 8146, 8362] avg=8192.0000 max=8362 max_rank=7 min=8140 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1111, 1018, 1075, 988, 991, 1046, 1012, 1054, 36, 12, 0, 0, 0, 0, 0, 0, 997, 1051, 1006, 1007, 1015, 980, 990, 1057, 16, 11, 0, 0, 0, 0, 0, 0, 1010, 1013, 986, 991, 1019, 973, 1036, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1018, 1012, 1005, 1047, 1008, 995, 1011, 51, 5, 0, 0, 0, 0, 0, 0, 967, 1058, 1051, 1060, 1017, 1041, 1033, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1023, 984, 1040, 1032, 995, 1001, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 977, 1079, 1077, 1048, 992, 1027, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1033, 1038, 979, 1043, 1010, 1017, 1016, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8343, 8130, 8077, 8174, 8266, 8136, 8271, 8139] avg=8192.0000 max=8343 max_rank=0 min=8077 min_rank=2 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1013, 1043, 1026, 1033, 1016, 1011, 989, 1037, 28, 8, 0, 0, 0, 0, 0, 0, 1013, 990, 1014, 1004, 1024, 1058, 1035, 1063, 9, 6, 0, 0, 0, 0, 0, 0, 964, 1062, 1014, 1014, 1041, 1027, 992, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1028, 988, 1002, 1044, 992, 1022, 996, 45, 0, 0, 0, 0, 0, 0, 0, 1053, 1012, 1052, 1041, 1005, 1043, 1067, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1045, 1072, 979, 1022, 1016, 1020, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1024, 1017, 955, 1016, 1040, 1022, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1035, 958, 1065, 1055, 1078, 1007, 1029, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8216, 8150, 8130, 8282, 8174, 8126, 8254] avg=8192.0000 max=8282 max_rank=4 min=8126 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8064, 8192, 8192, 8192, 8320, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1066, 1038, 1007, 1003, 977, 1028, 1004, 1041, 1012, 1006, 1080, 1009, 1048, 991, 1017, 979, 975, 1021, 1020, 962, 1006, 1100, 1001, 1023, 1041, 995, 1042, 1018, 1000, 1046, 1041, 1072, 1081, 1013, 1003, 1015, 1029, 1078, 995, 1015, 989, 1052, 1037, 1001, 994, 1012, 1015, 993, 1008, 1041, 997, 1043, 1029, 1043, 1051, 1071, 1040, 1045, 1004, 1057, 1020, 1006, 1030, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1013, 971, 1007, 1034, 992, 1057, 991, 1005, 1027, 1053, 1038, 993, 1006, 993, 1030, 1049, 1009, 1003, 1009, 1018, 1089, 1063, 1050, 1041, 1066, 1014, 1009, 1006, 1021, 1041, 1008, 1039, 1009, 1052, 1004, 1018, 1071, 1026, 1035, 1015, 967, 1022, 995, 1079, 1072, 1002, 1055, 1014, 1002, 1027, 1060, 1052, 994, 1051, 1051, 991, 1049, 1045, 1005, 1017, 991, 971, 1006, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1055, 1033, 1047, 1013, 1030, 1007, 1050, 996, 1039, 1020, 1038, 1000, 1012, 972, 999, 1024, 1021, 985, 1065, 1043, 1030, 1018, 1017, 1010, 1032, 1064, 1123, 1039, 1019, 1060, 1018, 971, 1038, 1045, 999, 1045, 1015, 992, 990, 1028, 1015, 1058, 1056, 1020, 996, 983, 1064, 1031, 1068, 1055, 1019, 1013, 1013, 1044, 1020, 952, 993, 1061, 998, 1022, 1035, 985, 1029, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1028, 994, 1014, 996, 1036, 1002, 1001, 1021, 1050, 1076, 1023, 995, 1010, 1048, 1020, 981, 1010, 1003, 1044, 1006, 1018, 1038, 1017, 1000, 1033, 1001, 1047, 992, 1051, 1042, 1039, 975, 1031, 1072, 995, 1005, 1011, 1007, 1039, 1012, 1054, 1024, 1081, 974, 1022, 1049, 1038, 1084, 996, 1007, 1098, 1004, 1056, 1016, 1076, 982, 961, 1030, 996, 1025, 1069, 1023, 1011, 1077] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1020, 1052, 978, 1028, 984, 960, 1060, 1062, 1019, 1042, 1057, 1045, 1050, 1015, 1019, 1024, 1026, 1031, 1056, 1030, 1077, 965, 1025, 1015, 1011, 1058, 1033, 1021, 997, 1017, 1053, 1019, 1008, 1037, 1069, 1016, 1021, 997, 1006, 1042, 1037, 1049, 1038, 981, 996, 971, 1011, 976, 1029, 985, 981, 1077, 1041, 1039, 1049, 1056, 1000, 1031, 980, 1008, 1037, 1036, 1036, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1025, 1053, 1008, 1028, 1011, 997, 1002, 1051, 991, 1032, 1007, 1020, 1042, 1020, 1006, 989, 1060, 1002, 1008, 1037, 1030, 1004, 1003, 1032, 1041, 1012, 1051, 1027, 1046, 1034, 1083, 977, 1030, 1020, 1022, 1011, 1004, 1011, 1044, 1044, 1022, 1004, 1056, 1058, 954, 1027, 1065, 1057, 974, 1098, 974, 1034, 1045, 1045, 1004, 1063, 1061, 972, 1003, 1020, 977, 1009, 1026, 1073] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1013, 970, 1014, 1068, 1017, 1036, 971, 1023, 1045, 1018, 1047, 1068, 1026, 995, 1004, 984, 1027, 1002, 1021, 1016, 1026, 1016, 989, 1055, 1027, 999, 974, 1039, 1003, 990, 1025, 1045, 1046, 1008, 1037, 1004, 1088, 1069, 1034, 1036, 1001, 1042, 1003, 1050, 1056, 1027, 1016, 1006, 1008, 1042, 1011, 1014, 1008, 1045, 1058, 1022, 1023, 1061, 1024, 1001, 1034, 1017, 1051, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1071, 981, 997, 1014, 997, 1016, 993, 1031, 1066, 1025, 1059, 1051, 1038, 1028, 1019, 1034, 1042, 1015, 1014, 1035, 1069, 1016, 982, 1053, 1018, 1048, 1080, 1014, 1006, 1045, 964, 1015, 1016, 1004, 1047, 1052, 1039, 1021, 1037, 1004, 1029, 1010, 1056, 1015, 1043, 1038, 988, 989, 1016, 1001, 1037, 1021, 1024, 1001, 1065, 997, 990, 1038, 1048, 1036, 1037, 1014, 996, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1028, 994, 1014, 996, 1036, 1002, 1001, 1021, 33, 23, 0, 0, 0, 0, 0, 0, 1050, 1076, 1023, 995, 1010, 1048, 1020, 981, 1, 0, 0, 0, 0, 0, 0, 0, 1010, 1003, 1044, 1006, 1018, 1038, 1017, 1000, 5, 0, 0, 0, 0, 0, 0, 0, 1033, 1001, 1023, 992, 1051, 1042, 1039, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1072, 995, 1005, 991, 1007, 1039, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1024, 1081, 974, 1022, 1049, 1038, 1084, 2, 0, 0, 0, 0, 0, 0, 0, 996, 1007, 1098, 1004, 1056, 1016, 1043, 982, 0, 0, 0, 0, 0, 0, 0, 0, 961, 1030, 996, 1025, 1069, 1023, 1011, 1077, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8204, 8141, 8156, 8152, 8328, 8202, 8205] avg=8192.0000 max=8328 max_rank=5 min=8141 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32887 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1066, 1038, 1007, 1003, 977, 1028, 1004, 1041, 32, 23, 0, 0, 0, 0, 0, 0, 1012, 1006, 1080, 1009, 1048, 991, 1017, 979, 1, 0, 0, 0, 0, 0, 0, 0, 975, 1021, 1020, 962, 1006, 1100, 1001, 1023, 5, 0, 0, 0, 0, 0, 0, 0, 1041, 995, 1018, 1018, 1000, 1046, 1041, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1013, 1003, 1015, 1009, 1078, 995, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1052, 1037, 1001, 994, 1012, 1015, 993, 2, 0, 0, 0, 0, 0, 0, 0, 1008, 1041, 997, 1043, 1029, 1043, 1019, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1045, 1004, 1057, 1020, 1006, 1030, 1060, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8143, 8113, 8231, 8209, 8095, 8251, 8275] avg=8192.0000 max=8275 max_rank=7 min=8095 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1020, 1052, 978, 1028, 984, 960, 1060, 1062, 32, 23, 0, 0, 0, 0, 0, 0, 1019, 1042, 1057, 1045, 1050, 1015, 1019, 1024, 1, 0, 0, 0, 0, 0, 0, 0, 1026, 1031, 1056, 1030, 1077, 965, 1025, 1015, 5, 0, 0, 0, 0, 0, 0, 0, 1011, 1058, 1009, 1021, 997, 1017, 1053, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1037, 1069, 1016, 1001, 997, 1006, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1049, 1038, 981, 996, 971, 1011, 976, 2, 0, 0, 0, 0, 0, 0, 0, 1029, 985, 981, 1077, 1041, 1039, 1017, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1031, 980, 1008, 1037, 1036, 1036, 1047, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8272, 8230, 8185, 8176, 8061, 8225, 8188] avg=8192.0000 max=8272 max_rank=1 min=8061 min_rank=5 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1055, 1033, 1047, 1013, 1030, 1007, 1050, 996, 31, 25, 0, 0, 0, 0, 0, 0, 1039, 1020, 1038, 1000, 1012, 972, 999, 1024, 1, 0, 0, 0, 0, 0, 0, 0, 1021, 985, 1065, 1043, 1030, 1018, 1017, 1010, 5, 0, 0, 0, 0, 0, 0, 0, 1032, 1064, 1097, 1039, 1019, 1060, 1018, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1045, 999, 1045, 995, 992, 990, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1058, 1056, 1020, 996, 983, 1064, 1031, 2, 0, 0, 0, 0, 0, 0, 0, 1068, 1055, 1019, 1013, 1013, 1044, 989, 952, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1061, 998, 1022, 1035, 985, 1029, 1004, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8105, 8194, 8300, 8132, 8225, 8153, 8140] avg=8192.0000 max=8300 max_rank=3 min=8105 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1013, 971, 1007, 1034, 992, 1057, 991, 1005, 36, 27, 0, 0, 0, 0, 0, 0, 1027, 1053, 1038, 993, 1006, 993, 1030, 1049, 6, 0, 0, 0, 0, 0, 0, 0, 1009, 1003, 1009, 1018, 1089, 1063, 1050, 1041, 7, 0, 0, 0, 0, 0, 0, 0, 1066, 1014, 978, 1006, 1021, 1041, 1008, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1052, 1004, 1018, 1041, 1026, 1035, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1022, 995, 1079, 1072, 1002, 1055, 1014, 3, 0, 0, 0, 0, 0, 0, 0, 1002, 1027, 1060, 1052, 994, 1051, 1015, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1045, 1005, 1017, 991, 971, 1006, 1043, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8195, 8289, 8173, 8200, 8209, 8192, 8145] avg=8192.0000 max=8289 max_rank=2 min=8133 min_rank=0 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1071, 981, 997, 1014, 997, 1016, 993, 1031, 32, 24, 0, 0, 0, 0, 0, 0, 1066, 1025, 1059, 1051, 1038, 1028, 1019, 1034, 1, 0, 0, 0, 0, 0, 0, 0, 1042, 1015, 1014, 1035, 1069, 1016, 982, 1053, 5, 0, 0, 0, 0, 0, 0, 0, 1018, 1048, 1055, 1014, 1006, 1045, 964, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1004, 1047, 1052, 1019, 1021, 1037, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1010, 1056, 1015, 1043, 1038, 988, 989, 2, 0, 0, 0, 0, 0, 0, 0, 1016, 1001, 1037, 1021, 1024, 1001, 1033, 997, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1038, 1048, 1036, 1037, 1014, 996, 991, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8321, 8231, 8165, 8200, 8170, 8130, 8163] avg=8192.0000 max=8321 max_rank=1 min=8130 min_rank=6 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32663 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1013, 970, 1014, 1068, 1017, 1036, 971, 1023, 32, 22, 0, 0, 0, 0, 0, 0, 1045, 1018, 1047, 1068, 1026, 995, 1004, 984, 1, 0, 0, 0, 0, 0, 0, 0, 1027, 1002, 1021, 1016, 1026, 1016, 989, 1055, 5, 0, 0, 0, 0, 0, 0, 0, 1027, 999, 951, 1039, 1003, 990, 1025, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1008, 1037, 1004, 1067, 1069, 1034, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1042, 1003, 1050, 1056, 1027, 1016, 1006, 2, 0, 0, 0, 0, 0, 0, 0, 1008, 1042, 1011, 1014, 1008, 1045, 1026, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1061, 1024, 1001, 1034, 1017, 1051, 1041, 13, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8188, 8157, 8079, 8301, 8203, 8176, 8266] avg=8192.0000 max=8301 max_rank=4 min=8079 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32946 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 1053, 1008, 1028, 1011, 997, 1002, 1051, 30, 23, 0, 0, 0, 0, 0, 0, 991, 1032, 1007, 1020, 1042, 1020, 1006, 989, 1, 0, 0, 0, 0, 0, 0, 0, 1060, 1002, 1008, 1037, 1030, 1004, 1003, 1032, 5, 0, 0, 0, 0, 0, 0, 0, 1041, 1012, 1027, 1027, 1046, 1034, 1083, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1020, 1022, 1011, 984, 1011, 1044, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1004, 1056, 1058, 954, 1027, 1065, 1057, 2, 0, 0, 0, 0, 0, 0, 0, 974, 1098, 974, 1034, 1045, 1045, 974, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 972, 1003, 1020, 977, 1009, 1026, 1073, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8108, 8181, 8247, 8166, 8245, 8207, 8154] avg=8192.0000 max=8247 max_rank=3 min=8108 min_rank=1 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8192, 8320, 8064, 8192, 8448, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8448, 8192, 8320, 8064, 8192, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8064, 8192, 8192, 8320, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8448, 8192, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8320, 8192, 8448, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1042, 1038, 988, 1042, 1034, 1010, 1027, 1014, 1015, 1020, 972, 991, 1004, 1017, 986, 1057, 974, 1028, 1060, 1041, 1012, 1063, 1054, 1006, 1023, 1048, 1015, 1006, 1051, 1019, 1032, 1030, 1036, 1044, 1022, 1005, 1038, 1083, 1057, 1014, 1031, 1019, 969, 1029, 976, 1024, 1064, 1037, 1067, 966, 1019, 1025, 1006, 1029, 1050, 1000, 1034, 1035, 1005, 986, 1043, 1040, 1036, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1063, 1009, 1087, 1030, 1066, 1012, 1028, 985, 1076, 1003, 1045, 1034, 998, 1004, 1000, 1038, 1023, 1046, 1015, 1008, 969, 1040, 1021, 973, 997, 1031, 1007, 998, 987, 993, 1026, 980, 1021, 1057, 1032, 1021, 1049, 1003, 1034, 997, 997, 1012, 1062, 1049, 978, 1056, 1030, 992, 1033, 1056, 1034, 1032, 1002, 1077, 1019, 1044, 1021, 1052, 1013, 1014, 1014, 1030, 1045, 1068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1088, 1022, 1053, 1064, 1032, 1044, 1019, 1017, 958, 1052, 1052, 1027, 1000, 1037, 1002, 1023, 990, 1041, 1030, 1037, 997, 1046, 1024, 1015, 1010, 1054, 1009, 1034, 1040, 1013, 975, 1031, 1032, 1022, 1008, 1013, 1022, 1041, 1028, 1032, 1029, 1031, 1034, 1068, 1014, 973, 1005, 1030, 973, 993, 1007, 1000, 1033, 1015, 1070, 995, 988, 1035, 1029, 994, 1077, 1027, 1028, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1066, 997, 1016, 1017, 1061, 1045, 1005, 994, 1057, 1077, 1058, 989, 1050, 961, 1009, 1031, 1008, 1014, 1004, 1018, 1021, 988, 1043, 1023, 986, 1004, 1014, 1008, 988, 1035, 1014, 1011, 1061, 1014, 999, 1001, 971, 1041, 1051, 1061, 1029, 1020, 1025, 1016, 1013, 1046, 1052, 1042, 1040, 1018, 1054, 1077, 1043, 994, 1067, 979, 1015, 1063, 1052, 1006, 1038, 981, 990, 1065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1049, 982, 1030, 1070, 1006, 1055, 995, 1011, 1019, 1020, 1029, 1025, 1028, 1028, 967, 993, 1035, 1053, 1015, 1049, 1058, 1015, 1044, 1035, 1002, 1044, 1034, 1034, 1012, 1074, 983, 1056, 1065, 1019, 986, 1002, 1053, 1006, 1071, 1008, 1051, 985, 1034, 1027, 1000, 976, 983, 1025, 1037, 1030, 1037, 1054, 995, 1037, 1035, 998, 1041, 1000, 1052, 962, 1035, 1031, 975, 1076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1010, 1055, 1041, 1034, 1040, 1021, 1041, 1017, 1007, 1030, 1020, 1024, 1059, 1049, 1008, 981, 1022, 1001, 1005, 1049, 1001, 1041, 1033, 1013, 1044, 1043, 1061, 1044, 1015, 999, 1021, 998, 1040, 1051, 1008, 1025, 1043, 1016, 1051, 1001, 1046, 1009, 1007, 1083, 976, 997, 994, 986, 1004, 991, 1010, 993, 1035, 1027, 1028, 1066, 1032, 1012, 997, 1074, 1053, 977, 1032, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1043, 999, 1023, 1031, 1026, 1072, 1025, 996, 995, 1017, 1009, 1054, 1013, 1006, 990, 990, 979, 1026, 1063, 982, 1034, 1055, 1051, 1041, 1081, 1042, 1037, 1036, 1026, 1058, 980, 975, 1033, 999, 1025, 968, 1007, 1067, 1031, 1037, 1035, 994, 1065, 982, 1036, 1021, 1048, 1063, 1045, 1028, 1051, 1009, 1001, 1022, 1014, 1011, 954, 1031, 1027, 1069, 1037, 1032, 978, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1044, 1054, 1027, 1037, 974, 982, 1024, 1024, 1036, 972, 1054, 1046, 1020, 1059, 1011, 1007, 993, 1051, 1025, 1074, 1001, 1050, 1047, 1046, 1008, 1014, 1092, 1044, 990, 985, 1014, 1009, 1010, 1032, 993, 1042, 1024, 1048, 1004, 1018, 1067, 1035, 1056, 957, 1032, 1057, 992, 985, 1019, 1014, 1007, 1031, 1034, 1040, 981, 1047, 996, 979, 1057, 1046, 987, 1034, 1060, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1003, 1038, 988, 1042, 1034, 1010, 1027, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1020, 972, 991, 1004, 1017, 986, 1057, 39, 1, 0, 0, 0, 0, 0, 0, 974, 1028, 1060, 1041, 1012, 1063, 1044, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1048, 1015, 1006, 1051, 1019, 1032, 1030, 18, 7, 0, 0, 0, 0, 0, 0, 1036, 1044, 1022, 1005, 1038, 1083, 1038, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1019, 969, 1029, 976, 1024, 1064, 1037, 10, 10, 1, 0, 0, 0, 0, 0, 1067, 966, 1019, 1025, 1006, 1029, 1049, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1035, 1005, 986, 1043, 1040, 1036, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8102, 8228, 8249, 8280, 8170, 8161, 8190] avg=8192.0000 max=8280 max_rank=4 min=8102 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[972, 1055, 1041, 1034, 1040, 1021, 1041, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1030, 1020, 1024, 1059, 1049, 1008, 981, 38, 1, 0, 0, 0, 0, 0, 0, 1022, 1001, 1005, 1049, 1001, 1041, 1023, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1043, 1061, 1044, 1015, 999, 1021, 998, 18, 8, 0, 0, 0, 0, 0, 0, 1040, 1051, 1008, 1025, 1043, 1016, 1032, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1009, 1007, 1083, 976, 997, 994, 986, 10, 10, 1, 0, 0, 0, 0, 0, 1004, 991, 1010, 993, 1035, 1027, 1027, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1012, 997, 1074, 1053, 977, 1032, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8217, 8155, 8251, 8216, 8119, 8153, 8204] avg=8192.0000 max=8251 max_rank=3 min=8119 min_rank=5 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1047, 1022, 1053, 1064, 1032, 1044, 1019, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 958, 1052, 1052, 1027, 1000, 1037, 1002, 1023, 41, 1, 0, 0, 0, 0, 0, 0, 990, 1041, 1030, 1037, 997, 1046, 1014, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1054, 1009, 1034, 1040, 1013, 975, 1031, 18, 8, 0, 0, 0, 0, 0, 0, 1032, 1022, 1008, 1013, 1022, 1041, 1009, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1031, 1034, 1068, 1014, 973, 1005, 1030, 10, 10, 1, 0, 0, 0, 0, 0, 973, 993, 1007, 1000, 1033, 1015, 1069, 995, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1035, 1029, 994, 1077, 1027, 1028, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8193, 8170, 8192, 8179, 8205, 8085, 8214] avg=8192.0000 max=8298 max_rank=0 min=8085 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1021, 1009, 1087, 1030, 1066, 1012, 1028, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1003, 1045, 1034, 998, 1004, 1000, 1038, 42, 3, 0, 0, 0, 0, 0, 0, 1023, 1046, 1015, 1008, 969, 1040, 1008, 973, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1031, 1007, 998, 987, 993, 1026, 980, 22, 11, 0, 0, 0, 0, 0, 0, 1021, 1057, 1032, 1021, 1049, 1003, 1009, 997, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1012, 1062, 1049, 978, 1056, 1030, 992, 15, 13, 5, 0, 0, 0, 0, 0, 1033, 1056, 1034, 1032, 1002, 1077, 1014, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1052, 1013, 1014, 1014, 1030, 1045, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8243, 8082, 8052, 8189, 8209, 8292, 8231] avg=8192.0000 max=8292 max_rank=6 min=8052 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1026, 997, 1016, 1017, 1061, 1045, 1005, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1077, 1058, 989, 1050, 961, 1009, 1031, 40, 1, 0, 0, 0, 0, 0, 0, 1008, 1014, 1004, 1018, 1021, 988, 1033, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1004, 1014, 1008, 988, 1035, 1014, 1011, 18, 8, 0, 0, 0, 0, 0, 0, 1061, 1014, 999, 1001, 971, 1041, 1032, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1020, 1025, 1016, 1013, 1046, 1052, 1042, 10, 10, 1, 0, 0, 0, 0, 0, 1040, 1018, 1054, 1077, 1043, 994, 1066, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1063, 1052, 1006, 1038, 981, 990, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8273, 8109, 8086, 8180, 8264, 8271, 8192] avg=8192.0000 max=8273 max_rank=1 min=8086 min_rank=3 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1010, 982, 1030, 1070, 1006, 1055, 995, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1020, 1029, 1025, 1028, 1028, 967, 993, 39, 1, 0, 0, 0, 0, 0, 0, 1035, 1053, 1015, 1049, 1058, 1015, 1034, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1044, 1034, 1034, 1012, 1074, 983, 1056, 18, 8, 0, 0, 0, 0, 0, 0, 1065, 1019, 986, 1002, 1053, 1006, 1052, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 985, 1034, 1027, 1000, 976, 983, 1025, 10, 10, 1, 0, 0, 0, 0, 0, 1037, 1030, 1037, 1054, 995, 1037, 1034, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1000, 1052, 962, 1035, 1031, 975, 1058, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8149, 8294, 8265, 8191, 8102, 8222, 8154] avg=8192.0000 max=8294 max_rank=2 min=8102 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1004, 999, 1023, 1031, 1026, 1072, 1025, 996, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1017, 1009, 1054, 1013, 1006, 990, 990, 39, 1, 0, 0, 0, 0, 0, 0, 979, 1026, 1063, 982, 1034, 1055, 1041, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1042, 1037, 1036, 1026, 1058, 980, 975, 18, 8, 0, 0, 0, 0, 0, 0, 1033, 999, 1025, 968, 1007, 1067, 1012, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 994, 1065, 982, 1036, 1021, 1048, 1063, 10, 10, 1, 0, 0, 0, 0, 0, 1045, 1028, 1051, 1009, 1001, 1022, 1013, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1031, 1027, 1069, 1037, 1032, 978, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8114, 8221, 8261, 8148, 8265, 8180, 8171] avg=8192.0000 max=8265 max_rank=5 min=8114 min_rank=1 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1005, 1054, 1027, 1037, 974, 982, 1024, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 972, 1054, 1046, 1020, 1059, 1011, 1007, 39, 1, 0, 0, 0, 0, 0, 0, 993, 1051, 1025, 1074, 1001, 1050, 1037, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1014, 1092, 1044, 990, 985, 1014, 1009, 17, 7, 0, 0, 0, 0, 0, 0, 1010, 1032, 993, 1042, 1024, 1048, 986, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1035, 1056, 957, 1032, 1057, 992, 985, 10, 10, 1, 0, 0, 0, 0, 0, 1019, 1014, 1007, 1031, 1034, 1040, 980, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 996, 979, 1057, 1046, 987, 1034, 1060, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8245, 8277, 8180, 8153, 8202, 8172, 8180] avg=8192.0000 max=8277 max_rank=2 min=8127 min_rank=0 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8448, 8320, 8320, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8064, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8064, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1048, 1033, 1015, 1002, 1101, 996, 1065, 1056, 1017, 989, 1013, 1024, 986, 1027, 986, 990, 1034, 1001, 1052, 969, 1009, 1059, 990, 1061, 1044, 1004, 1020, 1047, 998, 1069, 977, 992, 1029, 1028, 1047, 1045, 1031, 1019, 1025, 1005, 1039, 1011, 1056, 1058, 1058, 974, 1033, 1021, 1028, 1026, 1057, 1036, 1009, 1002, 1036, 1021, 998, 1074, 968, 1028, 1009, 1059, 1008, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1026, 1010, 1040, 1013, 1032, 1052, 981, 1007, 1059, 1030, 1021, 1028, 1017, 1025, 1018, 1030, 1003, 1025, 1057, 1024, 963, 994, 1054, 1022, 1015, 1030, 1040, 1024, 1025, 1032, 1021, 1016, 985, 981, 1025, 1014, 1000, 999, 1026, 1026, 1063, 994, 1031, 1032, 1012, 1019, 1039, 1039, 1029, 1025, 1082, 1043, 1017, 1040, 1033, 1021, 1016, 1018, 1026, 1009, 1001, 1050, 1024, 1083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1032, 1023, 1012, 1016, 1040, 982, 997, 1045, 1023, 1029, 1008, 1038, 997, 1040, 1012, 1030, 1001, 1029, 1057, 1007, 1000, 996, 1048, 993, 1005, 1063, 1086, 1002, 1052, 1022, 1003, 1011, 1019, 1052, 1038, 1052, 1032, 1034, 1014, 1063, 1020, 1069, 991, 941, 1004, 1032, 1059, 1027, 1025, 1042, 1024, 1006, 1011, 995, 1032, 1042, 1022, 1033, 1052, 1018, 1010, 1028, 1025, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1048, 1056, 1025, 1015, 1034, 1051, 1018, 1046, 1051, 1013, 1059, 1003, 992, 1036, 1048, 1008, 1032, 970, 1018, 1003, 988, 1030, 1010, 1030, 1034, 1050, 1065, 1055, 969, 1041, 1025, 1052, 1012, 1000, 994, 1033, 1017, 1040, 1043, 1071, 1046, 1042, 1018, 1015, 1037, 1051, 1014, 1056, 995, 1007, 1020, 1023, 1008, 1054, 1030, 998, 1006, 1048, 1011, 963, 1015, 1010, 1031, 953] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1026, 998, 1055, 986, 1014, 1026, 998, 1082, 1027, 1019, 982, 1034, 1050, 1052, 1039, 993, 1005, 1079, 996, 1055, 1017, 1063, 1021, 988, 1047, 931, 1018, 1083, 983, 1046, 1033, 1005, 1015, 1019, 1032, 986, 1040, 1040, 1098, 1027, 1024, 982, 1001, 1006, 1014, 1067, 1047, 1052, 998, 1046, 1058, 1071, 1012, 972, 1027, 1019, 1081, 968, 1015, 964, 999, 1050, 1046, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1036, 1005, 1013, 1037, 1078, 1070, 1010, 1045, 1003, 1072, 1030, 1029, 1021, 1002, 1042, 1061, 990, 1057, 1105, 1021, 1062, 981, 1007, 1035, 992, 963, 1001, 1029, 1023, 1005, 1021, 1002, 1036, 1039, 1023, 1061, 982, 1071, 994, 1017, 1006, 1025, 999, 1020, 1013, 1031, 988, 1024, 1054, 1028, 1013, 1053, 1050, 1047, 990, 966, 1016, 946, 1043, 1010, 1021, 1085, 1011, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1090, 984, 1022, 1001, 992, 1015, 951, 1002, 1051, 989, 1007, 1026, 1050, 1035, 1047, 1054, 1044, 1035, 976, 1027, 1073, 1009, 1048, 1014, 966, 1052, 1042, 1038, 1022, 1036, 1040, 1035, 985, 988, 1077, 1035, 1009, 1041, 1056, 1030, 1070, 1029, 1039, 1019, 1015, 1025, 1013, 1046, 1028, 1068, 1042, 1009, 1006, 1006, 1024, 1018, 979, 1025, 1009, 1011, 1072, 1026, 989, 974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[986, 1013, 1052, 1050, 1038, 1055, 969, 1004, 1027, 1011, 1025, 992, 1057, 1034, 1011, 1021, 1043, 988, 1040, 1062, 1058, 1027, 1040, 1003, 1029, 1043, 1024, 1002, 1036, 1016, 1005, 1040, 995, 1032, 986, 1047, 962, 1047, 1025, 1025, 1013, 1040, 1023, 1027, 1009, 1071, 1003, 998, 1023, 1007, 980, 1063, 1054, 1013, 1049, 1037, 1038, 1073, 970, 1017, 1010, 1076, 1020, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1048, 1033, 1015, 1002, 1090, 996, 1065, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 989, 1013, 1024, 986, 1027, 986, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1001, 1052, 969, 1009, 1059, 990, 1061, 4, 0, 0, 0, 0, 0, 0, 0, 1044, 1004, 1020, 1047, 998, 1069, 977, 992, 4, 0, 0, 0, 0, 0, 0, 0, 1029, 1028, 1047, 1045, 1031, 1010, 1025, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1011, 1056, 1058, 1058, 974, 1033, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1026, 1057, 1023, 1009, 1002, 1036, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1074, 968, 1028, 1009, 1059, 1008, 1024, 13, 13, 11, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 8032, 8179, 8155, 8220, 8237, 8202, 8206] avg=8192.0000 max=8305 max_rank=0 min=8032 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1026, 1010, 1040, 1013, 1022, 1052, 981, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1030, 1021, 1028, 1017, 1025, 1018, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1025, 1057, 1024, 963, 994, 1054, 1022, 4, 0, 0, 0, 0, 0, 0, 0, 1015, 1030, 1040, 1024, 1025, 1032, 1021, 1016, 4, 0, 0, 0, 0, 0, 0, 0, 985, 981, 1025, 1014, 1000, 990, 1026, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 994, 1031, 1032, 1012, 1019, 1039, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1025, 1082, 1029, 1017, 1040, 1033, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1018, 1026, 1009, 1001, 1050, 1024, 1083, 14, 13, 10, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8228, 8146, 8207, 8047, 8216, 8276, 8265] avg=8192.0000 max=8276 max_rank=6 min=8047 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1032, 1023, 1012, 1016, 1030, 982, 997, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1029, 1008, 1038, 997, 1040, 1012, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1029, 1057, 1007, 1000, 996, 1048, 993, 4, 0, 0, 0, 0, 0, 0, 0, 1005, 1063, 1086, 1002, 1052, 1022, 1003, 1011, 4, 0, 0, 0, 0, 0, 0, 0, 1019, 1052, 1038, 1052, 1032, 1025, 1014, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1069, 991, 941, 1004, 1032, 1059, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1042, 1024, 993, 1011, 995, 1032, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1033, 1052, 1018, 1010, 1028, 1025, 1025, 13, 12, 10, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8177, 8135, 8248, 8295, 8131, 8164, 8249] avg=8192.0000 max=8295 max_rank=4 min=8131 min_rank=5 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[986, 1013, 1052, 1050, 1028, 1055, 969, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1011, 1025, 992, 1057, 1034, 1011, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 988, 1040, 1062, 1058, 1027, 1040, 1003, 4, 0, 0, 0, 0, 0, 0, 0, 1029, 1043, 1024, 1002, 1036, 1016, 1005, 1040, 4, 0, 0, 0, 0, 0, 0, 0, 995, 1032, 986, 1047, 962, 1038, 1025, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1040, 1023, 1027, 1009, 1071, 1003, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1007, 980, 1049, 1054, 1013, 1049, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1073, 970, 1017, 1010, 1076, 1020, 1002, 14, 12, 10, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8178, 8265, 8199, 8110, 8172, 8212, 8243] avg=8192.0000 max=8265 max_rank=2 min=8110 min_rank=4 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1048, 1056, 1025, 1015, 1024, 1051, 1018, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1013, 1059, 1003, 992, 1036, 1048, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 970, 1018, 1003, 988, 1030, 1010, 1030, 4, 0, 0, 0, 0, 0, 0, 0, 1034, 1050, 1065, 1055, 969, 1041, 1025, 1052, 4, 0, 0, 0, 0, 0, 0, 0, 1012, 1000, 994, 1033, 1017, 1031, 1043, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1042, 1018, 1015, 1037, 1051, 1014, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1007, 1020, 1010, 1008, 1054, 1030, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1048, 1011, 963, 1015, 1010, 1031, 953, 13, 13, 10, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8210, 8085, 8295, 8201, 8266, 8122, 8074] avg=8192.0000 max=8295 max_rank=3 min=8074 min_rank=7 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32663 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1026, 998, 1055, 986, 1004, 1026, 998, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1019, 982, 1034, 1050, 1052, 1039, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1079, 996, 1055, 1017, 1063, 1021, 988, 4, 0, 0, 0, 0, 0, 0, 0, 1047, 931, 1018, 1083, 983, 1046, 1033, 1005, 4, 0, 0, 0, 0, 0, 0, 0, 1015, 1019, 1032, 986, 1040, 1031, 1098, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 982, 1001, 1006, 1014, 1067, 1047, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1046, 1058, 1057, 1012, 972, 1027, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 968, 1015, 964, 999, 1050, 1046, 1009, 14, 12, 10, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8196, 8228, 8150, 8248, 8181, 8189, 8169] avg=8192.0000 max=8248 max_rank=4 min=8150 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1036, 1005, 1013, 1037, 1068, 1070, 1010, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1072, 1030, 1029, 1021, 1002, 1042, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1057, 1105, 1021, 1062, 981, 1007, 1035, 4, 0, 0, 0, 0, 0, 0, 0, 992, 963, 1001, 1029, 1023, 1005, 1021, 1002, 5, 0, 0, 0, 0, 0, 0, 0, 1036, 1039, 1023, 1061, 982, 1061, 994, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1025, 999, 1020, 1013, 1031, 988, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1028, 1013, 1039, 1050, 1047, 990, 966, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 946, 1043, 1010, 1021, 1085, 1011, 1026, 14, 12, 10, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8260, 8262, 8041, 8213, 8094, 8187, 8195] avg=8192.0000 max=8284 max_rank=0 min=8041 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1090, 984, 1022, 1001, 979, 1015, 951, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 989, 1007, 1026, 1050, 1035, 1047, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1035, 976, 1027, 1073, 1009, 1048, 1014, 6, 4, 0, 0, 0, 0, 0, 0, 966, 1052, 1042, 1038, 1022, 1036, 1040, 1035, 10, 0, 0, 0, 0, 0, 0, 0, 985, 988, 1077, 1035, 1009, 1022, 1056, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1029, 1039, 1019, 1015, 1025, 1013, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1068, 1042, 992, 1006, 1006, 1024, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1025, 1009, 1011, 1072, 1026, 989, 974, 17, 17, 13, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8044, 8255, 8236, 8241, 8202, 8239, 8184, 8135] avg=8192.0000 max=8255 max_rank=1 min=8044 min_rank=0 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8064, 8192, 8448, 8192, 8192, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1006, 1065, 1076, 1022, 1024, 989, 1046, 1006, 1007, 996, 1043, 988, 1060, 1047, 1032, 965, 1039, 1016, 1027, 1025, 1023, 1067, 1001, 1015, 997, 1026, 1028, 1038, 1079, 1014, 1017, 993, 966, 954, 1079, 1002, 1050, 1022, 1058, 983, 1037, 1044, 1105, 1055, 1020, 1024, 993, 1018, 1033, 1032, 1008, 1067, 1038, 992, 1012, 1008, 1038, 1011, 1086, 1008, 985, 1041, 1018, 972] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1006, 1071, 1006, 1038, 1059, 972, 1050, 995, 1037, 1046, 1020, 952, 1058, 1023, 1009, 1015, 1035, 994, 1007, 1052, 1000, 1042, 1058, 1031, 1034, 1055, 1016, 1009, 1001, 983, 1031, 968, 1043, 1000, 1031, 1063, 1022, 1049, 1060, 1035, 976, 1045, 1046, 1005, 1019, 1073, 990, 1016, 1039, 993, 1039, 994, 1045, 992, 1050, 996, 1036, 1023, 1048, 1029, 997, 1045, 1046, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[992, 1048, 949, 1063, 1015, 1042, 1032, 1030, 1006, 1015, 962, 1021, 1070, 986, 1016, 1019, 1001, 994, 966, 1017, 1021, 1048, 1041, 1009, 1062, 993, 1003, 1000, 1060, 1070, 1018, 1055, 1038, 1038, 1061, 1025, 1083, 951, 1034, 997, 1037, 1047, 1051, 1072, 1024, 1057, 1021, 1037, 1062, 1060, 983, 972, 1005, 1032, 1022, 1005, 1019, 1017, 1052, 1006, 977, 1046, 1042, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1055, 1028, 1028, 999, 1073, 1012, 1033, 1026, 1023, 995, 1022, 1021, 1071, 1021, 1028, 1040, 1046, 1023, 1061, 1021, 1033, 1082, 979, 960, 1033, 1037, 1053, 1032, 1062, 1023, 998, 1040, 978, 993, 1062, 1016, 996, 1039, 1003, 1054, 1006, 1009, 965, 1045, 1040, 1030, 1029, 999, 1041, 1039, 1014, 1065, 1016, 989, 1037, 1054, 1008, 1047, 976, 1000, 1029, 967, 1017, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1037, 1067, 990, 1029, 998, 1054, 1055, 1028, 1021, 974, 1037, 1079, 967, 1054, 1012, 1055, 1067, 986, 1072, 1032, 994, 1036, 1006, 1028, 985, 1037, 1012, 988, 974, 1007, 1019, 1023, 992, 1054, 1023, 992, 989, 1016, 1021, 1038, 1019, 1016, 1059, 1021, 1037, 1013, 1004, 1017, 1018, 1056, 1016, 1002, 1042, 1046, 994, 1037, 1063, 1009, 1037, 1034, 1045, 1019, 1050, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1001, 1027, 1005, 1064, 1036, 1006, 980, 1002, 1035, 1030, 1018, 1066, 1039, 1031, 976, 999, 1005, 1058, 994, 1011, 1019, 1049, 1002, 1099, 1038, 1034, 976, 998, 1038, 1050, 977, 1017, 1010, 1016, 999, 1039, 995, 999, 974, 1043, 992, 999, 1014, 1064, 987, 1051, 1049, 1057, 1043, 1011, 1057, 1077, 1079, 1014, 1025, 1047, 1021, 1007, 1042, 1054, 1035, 1033, 987, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1009, 1042, 1040, 1065, 1009, 1000, 994, 1029, 1002, 1043, 1063, 1054, 1040, 1013, 1013, 1070, 1068, 1040, 1058, 1024, 1015, 1033, 974, 1046, 987, 1062, 963, 989, 1018, 927, 1077, 1057, 1026, 1028, 996, 1047, 1029, 1021, 978, 992, 1021, 999, 1022, 1059, 1001, 1011, 1045, 1051, 990, 1002, 972, 1071, 1027, 1027, 1033, 1051, 993, 1036, 1051, 1010, 997, 1048, 1015, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1021, 1050, 1030, 1025, 967, 1044, 1064, 1053, 1020, 958, 1001, 1013, 1030, 1080, 1023, 1030, 1022, 992, 1005, 1008, 1044, 1043, 1071, 998, 1003, 1057, 1018, 998, 1018, 1022, 1025, 1013, 1016, 1007, 1030, 1083, 1001, 1010, 1023, 1006, 1001, 984, 1050, 1029, 1008, 1043, 1075, 1028, 960, 1032, 1050, 1049, 1007, 1048, 1022, 1033, 1026, 1071, 988, 1008, 1003, 1051, 1020, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1006, 1048, 1076, 1022, 1024, 989, 1046, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 996, 1043, 988, 1060, 1047, 1032, 965, 5, 4, 0, 0, 0, 0, 0, 0, 1039, 1016, 1027, 1025, 1023, 1055, 1001, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1026, 1028, 1038, 1079, 1014, 1017, 993, 28, 9, 0, 0, 0, 0, 0, 0, 966, 954, 1079, 1002, 1050, 1022, 1058, 983, 17, 8, 7, 0, 0, 0, 0, 0, 1037, 1044, 1105, 1027, 1020, 1024, 993, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1032, 1008, 1050, 1038, 992, 1012, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1011, 1082, 1008, 985, 1041, 1018, 972, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8147, 8201, 8229, 8146, 8268, 8173, 8155] avg=8192.0000 max=8268 max_rank=5 min=8146 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1006, 1047, 1006, 1038, 1059, 972, 1050, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1046, 1020, 952, 1058, 1023, 1009, 1015, 10, 8, 0, 0, 0, 0, 0, 0, 1035, 994, 1007, 1052, 1000, 1024, 1058, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1055, 1016, 1009, 1001, 983, 1031, 968, 31, 13, 0, 0, 0, 0, 0, 0, 1043, 1000, 1031, 1063, 1022, 1049, 1060, 1035, 20, 11, 8, 0, 0, 0, 0, 0, 976, 1045, 1046, 974, 1019, 1073, 990, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 993, 1039, 974, 1045, 992, 1050, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1023, 1040, 1029, 997, 1045, 1046, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8178, 8201, 8141, 8342, 8139, 8128, 8234] avg=8192.0000 max=8342 max_rank=4 min=8128 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1021, 1033, 1030, 1025, 967, 1044, 1064, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 958, 1001, 1013, 1030, 1080, 1023, 1030, 4, 4, 0, 0, 0, 0, 0, 0, 1022, 992, 1005, 1008, 1044, 1032, 1071, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1057, 1018, 998, 1018, 1022, 1025, 1013, 27, 9, 0, 0, 0, 0, 0, 0, 1016, 1007, 1030, 1083, 1001, 1010, 1023, 1006, 17, 8, 7, 0, 0, 0, 0, 0, 1001, 984, 1050, 1002, 1008, 1043, 1075, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 960, 1032, 1050, 1032, 1007, 1048, 1022, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1071, 984, 1008, 1003, 1051, 1020, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8163, 8172, 8190, 8208, 8191, 8184, 8191] avg=8192.0000 max=8237 max_rank=0 min=8163 min_rank=1 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[992, 1031, 949, 1063, 1015, 1042, 1032, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1015, 962, 1021, 1070, 986, 1016, 1019, 4, 4, 0, 0, 0, 0, 0, 0, 1001, 994, 966, 1017, 1021, 1037, 1041, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 993, 1003, 1000, 1060, 1070, 1018, 1055, 28, 9, 0, 0, 0, 0, 0, 0, 1038, 1038, 1061, 1025, 1083, 951, 1034, 997, 15, 8, 7, 0, 0, 0, 0, 0, 1037, 1047, 1051, 1044, 1024, 1057, 1021, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1060, 983, 957, 1005, 1032, 1022, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1017, 1048, 1006, 977, 1046, 1042, 1039, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8103, 8086, 8298, 8257, 8318, 8126, 8194] avg=8192.0000 max=8318 max_rank=5 min=8086 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1009, 1025, 1040, 1065, 1009, 1000, 994, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1043, 1063, 1054, 1040, 1013, 1013, 1070, 4, 4, 0, 0, 0, 0, 0, 0, 1068, 1040, 1058, 1024, 1015, 1022, 974, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1062, 963, 989, 1018, 927, 1077, 1057, 28, 9, 0, 0, 0, 0, 0, 0, 1026, 1028, 996, 1047, 1029, 1021, 978, 992, 17, 8, 7, 0, 0, 0, 0, 0, 1021, 999, 1022, 1031, 1001, 1011, 1045, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1002, 972, 1054, 1027, 1027, 1033, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1036, 1047, 1010, 997, 1048, 1015, 1063, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8306, 8247, 8117, 8149, 8181, 8156, 8209] avg=8192.0000 max=8306 max_rank=1 min=8117 min_rank=3 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1001, 1011, 1005, 1064, 1036, 1006, 980, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1030, 1018, 1066, 1039, 1031, 976, 999, 4, 4, 0, 0, 0, 0, 0, 0, 1005, 1058, 994, 1011, 1019, 1038, 1002, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1034, 976, 998, 1038, 1050, 977, 1017, 28, 9, 0, 0, 0, 0, 0, 0, 1010, 1016, 999, 1039, 995, 999, 974, 1043, 17, 7, 7, 0, 0, 0, 0, 0, 992, 999, 1014, 1036, 987, 1051, 1049, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1011, 1057, 1060, 1079, 1014, 1025, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1007, 1038, 1054, 1035, 1033, 987, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8202, 8226, 8165, 8106, 8185, 8336, 8211] avg=8192.0000 max=8336 max_rank=6 min=8105 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1055, 1012, 1028, 999, 1073, 1012, 1033, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 995, 1022, 1021, 1071, 1021, 1028, 1040, 5, 4, 0, 0, 0, 0, 0, 0, 1046, 1023, 1061, 1021, 1033, 1070, 979, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1037, 1053, 1032, 1062, 1023, 998, 1040, 28, 9, 0, 0, 0, 0, 0, 0, 978, 993, 1062, 1016, 996, 1039, 1003, 1054, 17, 7, 7, 0, 0, 0, 0, 0, 1006, 1009, 965, 1017, 1040, 1030, 1029, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1039, 1014, 1048, 1016, 989, 1037, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1047, 972, 1000, 1029, 967, 1017, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8230, 8193, 8315, 8172, 8095, 8238, 8055] avg=8192.0000 max=8315 max_rank=3 min=8055 min_rank=7 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32560 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1037, 1050, 990, 1029, 998, 1054, 1055, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 974, 1037, 1079, 967, 1054, 1012, 1055, 4, 4, 0, 0, 0, 0, 0, 0, 1067, 986, 1072, 1032, 994, 1025, 1006, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1037, 1012, 988, 974, 1007, 1019, 1023, 27, 9, 0, 0, 0, 0, 0, 0, 992, 1054, 1023, 992, 989, 1016, 1021, 1038, 16, 8, 7, 0, 0, 0, 0, 0, 1019, 1016, 1059, 994, 1037, 1013, 1004, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1056, 1016, 986, 1042, 1046, 994, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1009, 1033, 1034, 1045, 1019, 1050, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8207, 8210, 8081, 8156, 8159, 8195, 8287] avg=8192.0000 max=8287 max_rank=7 min=8081 min_rank=3 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8064, 8192, 8320, 8448, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8064, 8320, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[999, 1000, 993, 1027, 1037, 1057, 1013, 1043, 984, 1009, 1010, 1067, 1033, 1052, 1020, 1016, 1015, 1042, 999, 1023, 1042, 995, 1009, 1016, 966, 1059, 977, 1000, 1035, 1064, 1049, 1009, 1010, 1060, 1045, 1014, 1044, 1052, 1025, 984, 1019, 1032, 1065, 1009, 1029, 1044, 1070, 1037, 1055, 1058, 982, 961, 1062, 1013, 1070, 1009, 1034, 1013, 1015, 1052, 985, 1016, 994, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1023, 1027, 1002, 1037, 1026, 994, 1018, 983, 1026, 1044, 1001, 1050, 1020, 1038, 986, 1061, 1051, 1038, 1035, 1045, 1021, 1017, 1032, 992, 1044, 1018, 1030, 1042, 1026, 1000, 1072, 1013, 988, 1006, 1018, 1060, 1049, 1032, 1017, 1026, 1001, 1015, 1043, 1061, 991, 1086, 1040, 1034, 1009, 989, 1000, 1025, 1042, 1069, 1033, 1022, 1000, 971, 1033, 1003, 997, 1004, 1010, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1026, 1026, 1053, 1085, 1052, 979, 1044, 1028, 1011, 1001, 1035, 1018, 1043, 1082, 1010, 1021, 1075, 1072, 1007, 1090, 1007, 1020, 1024, 1006, 1031, 1004, 962, 1017, 1072, 1021, 1039, 1016, 999, 1014, 1001, 1018, 1063, 1062, 1055, 1024, 962, 1016, 1053, 1009, 1015, 1031, 1024, 1016, 1051, 998, 972, 1033, 1005, 1032, 1045, 985, 1010, 972, 1048, 1023, 1068, 999, 973, 983] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[992, 1002, 1036, 996, 1031, 1011, 1027, 1028, 1043, 1069, 1049, 1011, 1001, 1027, 1062, 1025, 1004, 1041, 996, 1046, 996, 981, 1041, 1024, 1015, 1008, 1068, 1032, 1007, 1009, 1005, 1016, 1023, 1049, 1013, 991, 1053, 1041, 1022, 987, 1040, 998, 1058, 1027, 1060, 1006, 1026, 1004, 1026, 1016, 1061, 1027, 1028, 970, 1002, 1047, 1062, 1025, 1048, 1047, 1015, 981, 1059, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1073, 1047, 1009, 1002, 1021, 1047, 1057, 998, 1041, 996, 1025, 1036, 1021, 1082, 1036, 1041, 1030, 996, 1026, 1063, 1031, 1001, 1020, 1033, 992, 994, 1040, 1033, 1031, 1011, 1042, 1002, 1014, 1025, 1038, 971, 1032, 1012, 1012, 1008, 1018, 1082, 962, 1043, 1038, 1064, 1003, 991, 1032, 1061, 975, 1101, 1003, 1039, 1006, 1031, 995, 1026, 956, 1056, 1050, 1030, 998, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[980, 989, 1023, 989, 1054, 1028, 1016, 1024, 994, 1026, 1057, 1033, 1051, 955, 1011, 1036, 988, 1023, 1010, 1054, 1028, 1003, 1005, 1019, 1055, 1049, 1067, 1048, 1041, 1042, 1072, 976, 1023, 1006, 1021, 1031, 1036, 1075, 1009, 1036, 997, 1031, 1046, 1002, 1031, 1029, 991, 1021, 1033, 1050, 1004, 1006, 1036, 999, 1047, 1005, 1053, 1042, 1023, 1014, 1019, 1013, 1033, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[981, 1004, 1018, 1001, 1010, 996, 1029, 954, 1042, 1041, 1031, 1003, 1021, 1061, 1049, 1033, 1007, 984, 1006, 1030, 1012, 1021, 1044, 1048, 1061, 1070, 1053, 1037, 1039, 1000, 1022, 998, 1129, 1019, 1035, 1059, 976, 975, 984, 1025, 1036, 1004, 1036, 1066, 1023, 1014, 994, 1051, 1001, 988, 1003, 1031, 1015, 1058, 1044, 995, 975, 1089, 1049, 1007, 1094, 1005, 1077, 973] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1026, 1026, 1053, 1085, 1052, 979, 1044, 1028, 13, 8, 8, 0, 0, 0, 0, 0, 1011, 1001, 1035, 1018, 1043, 1040, 1010, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1072, 1007, 1083, 1007, 1020, 1024, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1004, 962, 1017, 1072, 1021, 1026, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1014, 1001, 1018, 1063, 1054, 1055, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1016, 1053, 1009, 1015, 1027, 1024, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 998, 972, 1033, 1005, 1032, 1045, 985, 7, 4, 0, 0, 0, 0, 0, 0, 1010, 972, 1048, 1023, 1068, 999, 973, 983, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8322, 8179, 8294, 8149, 8228, 8122, 8132, 8110] avg=8192.0000 max=8322 max_rank=0 min=8110 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32592 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[999, 1000, 993, 1027, 1037, 1057, 1013, 1043, 13, 8, 7, 0, 0, 0, 0, 0, 984, 1009, 1010, 1067, 1033, 1012, 1020, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1042, 999, 1017, 1042, 995, 1009, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1059, 977, 1000, 1035, 1064, 1036, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1060, 1045, 1014, 1044, 1044, 1025, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1032, 1065, 1009, 1029, 1040, 1070, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1058, 982, 961, 1062, 1013, 1070, 1009, 6, 4, 0, 0, 0, 0, 0, 0, 1034, 1013, 1015, 1052, 985, 1016, 994, 1018, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8151, 8135, 8146, 8226, 8301, 8220, 8160] avg=8192.0000 max=8301 max_rank=5 min=8135 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[981, 1004, 1018, 1001, 1010, 996, 1029, 954, 13, 7, 8, 0, 0, 0, 0, 0, 1042, 1041, 1031, 1003, 1021, 1019, 1049, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 984, 1006, 1024, 1012, 1021, 1044, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1070, 1053, 1037, 1039, 1000, 1009, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 1019, 1035, 1059, 976, 968, 984, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1004, 1036, 1066, 1023, 1010, 994, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 988, 1003, 1031, 1015, 1058, 1044, 995, 6, 4, 0, 0, 0, 0, 0, 0, 975, 1089, 1049, 1007, 1094, 1005, 1077, 973, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8021, 8239, 8146, 8267, 8195, 8220, 8145, 8303] avg=8192.0000 max=8303 max_rank=7 min=8021 min_rank=0 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32863 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1023, 1027, 1002, 1037, 1026, 994, 1018, 983, 13, 7, 7, 0, 0, 0, 0, 0, 1026, 1044, 1001, 1050, 1020, 998, 986, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1038, 1035, 1039, 1021, 1017, 1032, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1018, 1030, 1042, 1026, 1000, 1059, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1006, 1018, 1060, 1049, 1025, 1017, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1015, 1043, 1061, 991, 1082, 1040, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 989, 1000, 1025, 1042, 1069, 1033, 1022, 6, 4, 0, 0, 0, 0, 0, 0, 1000, 971, 1033, 1003, 997, 1004, 1010, 1050, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8186, 8225, 8232, 8189, 8267, 8199, 8101] avg=8192.0000 max=8267 max_rank=5 min=8101 min_rank=7 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[992, 1002, 1036, 996, 1031, 1011, 1027, 1028, 12, 7, 7, 0, 0, 0, 0, 0, 1043, 1069, 1049, 1011, 1001, 987, 1062, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1041, 996, 1040, 996, 981, 1041, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1008, 1068, 1032, 1007, 1009, 993, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1049, 1013, 991, 1053, 1034, 1022, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 998, 1058, 1027, 1060, 1002, 1026, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1016, 1061, 1027, 1028, 970, 1002, 1047, 6, 4, 0, 0, 0, 0, 0, 0, 1062, 1025, 1048, 1047, 1015, 981, 1059, 1025, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8247, 8123, 8148, 8172, 8215, 8187, 8295] avg=8192.0000 max=8295 max_rank=7 min=8123 min_rank=2 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1073, 1047, 1009, 1002, 1021, 1047, 1057, 998, 17, 11, 12, 0, 0, 0, 0, 0, 1041, 996, 1025, 1036, 1021, 1032, 1036, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 996, 1026, 1052, 1031, 1001, 1020, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 992, 994, 1040, 1033, 1031, 1011, 1025, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1025, 1038, 971, 1032, 999, 1012, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1082, 962, 1043, 1038, 1056, 1003, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1061, 975, 1101, 1003, 1039, 1006, 1031, 11, 8, 2, 0, 0, 0, 0, 0, 995, 1026, 956, 1056, 1050, 1030, 998, 987, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 8228, 8189, 8128, 8099, 8193, 8269, 8136] avg=8192.0000 max=8294 max_rank=0 min=8099 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[980, 989, 1023, 989, 1054, 1028, 1016, 1024, 13, 8, 7, 0, 0, 0, 0, 0, 994, 1026, 1057, 1033, 1051, 918, 1011, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1023, 1010, 1048, 1028, 1003, 1005, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1049, 1067, 1048, 1041, 1042, 1059, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1006, 1021, 1031, 1036, 1067, 1009, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1031, 1046, 1002, 1031, 1025, 991, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1050, 1004, 1006, 1036, 999, 1047, 1005, 6, 4, 0, 0, 0, 0, 0, 0, 1053, 1042, 1023, 1014, 1019, 1013, 1033, 1028, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8126, 8124, 8337, 8229, 8144, 8190, 8255] avg=8192.0000 max=8337 max_rank=3 min=8124 min_rank=2 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1059, 1021, 1026, 1112, 1017, 1047, 973, 1003, 1026, 1058, 1021, 1030, 1030, 1037, 1005, 1013, 1027, 981, 1019, 1081, 1017, 1029, 1080, 1073, 1032, 1042, 1008, 1006, 1068, 972, 1036, 978, 1060, 978, 1049, 1008, 997, 1020, 1042, 1051, 1009, 1010, 1004, 962, 1024, 1047, 993, 1029, 978, 1018, 1045, 990, 1014, 1044, 1045, 1049, 971, 1043, 1025, 996, 1010, 1035, 1044, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1059, 1021, 1026, 1112, 1017, 1047, 973, 1003, 13, 7, 7, 0, 0, 0, 0, 0, 1026, 1058, 1021, 1030, 1030, 997, 1005, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 981, 1019, 1074, 1017, 1029, 1080, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1042, 1008, 1006, 1068, 972, 1023, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 978, 1049, 1008, 997, 1013, 1042, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1010, 1004, 962, 1024, 1043, 993, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1018, 1045, 990, 1014, 1044, 1045, 1049, 7, 4, 0, 0, 0, 0, 0, 0, 971, 1043, 1025, 996, 1010, 1035, 1044, 1019, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8180, 8300, 8129, 8198, 8074, 8194, 8176] avg=8192.0000 max=8300 max_rank=2 min=8074 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32642 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8064, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8448, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8064, 8192, 8320, 8320, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8192, 8064, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1004, 997, 1014, 1033, 981, 1030, 1028, 1064, 1080, 1045, 1008, 1041, 975, 1067, 1051, 1054, 1044, 1031, 1015, 1050, 1049, 1001, 1043, 1000, 1024, 981, 993, 980, 1010, 1047, 1014, 1024, 1008, 1025, 1005, 1056, 1049, 1045, 1010, 1014, 1070, 1014, 1012, 1031, 1022, 993, 1042, 1053, 1012, 1012, 1040, 1092, 1012, 1062, 964, 1025, 1005, 1025, 1010, 1031, 1011, 968, 1004, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1076, 1043, 1045, 1058, 1023, 1030, 1059, 1027, 1021, 1003, 994, 1049, 972, 1104, 1017, 1006, 1026, 1054, 1064, 1013, 1029, 1021, 1042, 1033, 1052, 1043, 1067, 1010, 1044, 1015, 1021, 1040, 1049, 1029, 1022, 1034, 992, 987, 1066, 1021, 995, 1026, 962, 1005, 999, 1014, 1025, 1026, 996, 979, 1038, 1011, 994, 995, 1000, 1048, 1011, 1056, 1019, 999, 1044, 1052, 960, 981] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1030, 1025, 1069, 1019, 1027, 1021, 1042, 1063, 1042, 1034, 1018, 959, 997, 971, 1032, 1028, 980, 1018, 1000, 1048, 1020, 991, 991, 1060, 951, 1026, 1008, 1044, 1069, 1036, 1051, 995, 1050, 1044, 1011, 1032, 1007, 982, 1035, 1006, 1008, 998, 1025, 1051, 1034, 1041, 1083, 1053, 1071, 974, 990, 1048, 1029, 1067, 1073, 995, 1041, 1011, 1012, 1000, 1027, 1076, 984, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1027, 1066, 1066, 1049, 1017, 994, 1022, 1064, 982, 1055, 1073, 1021, 995, 1006, 1017, 1026, 987, 1049, 991, 997, 1000, 957, 1044, 1002, 986, 1045, 1093, 1035, 1079, 1032, 992, 1045, 1047, 1029, 1034, 1063, 1065, 976, 1043, 1018, 1014, 1017, 1005, 1025, 1076, 999, 1050, 961, 1008, 1032, 1053, 1034, 1041, 990, 1021, 1032, 987, 1025, 995, 1004, 1025, 1002, 1012, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1065, 1021, 1032, 1043, 1017, 1059, 1022, 984, 1041, 986, 1057, 1040, 1022, 1010, 1020, 1018, 1022, 1014, 1074, 991, 1052, 1021, 992, 1041, 1014, 1010, 1039, 990, 1019, 1022, 979, 1064, 1064, 1013, 1016, 1046, 1027, 1018, 996, 957, 1044, 1079, 1028, 972, 1043, 990, 1057, 1047, 1033, 1007, 986, 1006, 1057, 1036, 1035, 1014, 1067, 1046, 990, 1015, 1013, 1002, 1046, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1000, 977, 1028, 1028, 1043, 1018, 1081, 1002, 1052, 1059, 992, 1049, 1090, 1003, 1062, 1038, 1065, 1018, 1019, 982, 1084, 1015, 998, 1038, 1031, 1039, 1035, 1071, 1010, 1003, 983, 1046, 1012, 1001, 977, 992, 986, 1031, 1041, 1012, 986, 1002, 994, 1036, 1007, 975, 1007, 1010, 1015, 1038, 1002, 1025, 1009, 1022, 988, 1053, 1047, 1073, 1034, 1027, 1026, 1063, 999, 1087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1007, 1002, 991, 1047, 1003, 1037, 1028, 996, 1050, 1000, 996, 1040, 1030, 995, 1070, 1034, 1041, 1018, 1057, 1001, 982, 1106, 1025, 1096, 987, 1042, 1017, 972, 1007, 1019, 1011, 1072, 1056, 1071, 983, 1063, 986, 993, 1064, 997, 1000, 1075, 1048, 1034, 1043, 1048, 999, 1036, 991, 1010, 1009, 1025, 1028, 1020, 981, 1005, 1038, 1013, 1007, 1031, 1026, 1044, 1033, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1036, 1089, 1028, 1033, 1047, 1029, 1051, 975, 1003, 1058, 1040, 986, 1008, 1007, 1050, 1018, 1014, 1022, 1029, 988, 968, 993, 1079, 1057, 1016, 1072, 1009, 1008, 1032, 1062, 1026, 983, 1055, 1017, 1045, 969, 1050, 1005, 1024, 1006, 1071, 1043, 1016, 1006, 1005, 1012, 1007, 976, 1045, 1000, 990, 1035, 994, 1030, 1040, 1018, 1018, 1031, 1043, 1096, 1042, 990, 1005, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1004, 997, 1014, 1033, 981, 1030, 981, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 1045, 1008, 1041, 975, 1067, 1035, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1031, 1015, 1050, 1049, 1001, 1043, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 981, 993, 980, 1010, 1047, 1014, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1025, 1005, 1056, 1049, 1045, 1010, 1014, 9, 4, 0, 0, 0, 0, 0, 0, 1070, 1014, 1012, 1031, 1022, 993, 1042, 1053, 13, 0, 0, 0, 0, 0, 0, 0, 1012, 1012, 1040, 1092, 1012, 1062, 964, 1025, 30, 0, 0, 0, 0, 0, 0, 0, 1005, 1025, 1010, 1031, 1011, 968, 1004, 1036, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8305, 8232, 8073, 8225, 8250, 8249, 8098] avg=8192.0000 max=8305 max_rank=1 min=8073 min_rank=3 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1065, 1021, 1032, 1043, 1017, 1059, 975, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 986, 1057, 1040, 1022, 1010, 1005, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1014, 1074, 991, 1052, 1021, 992, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1010, 1039, 990, 1019, 1022, 979, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1013, 1016, 1046, 1027, 1018, 996, 957, 9, 4, 0, 0, 0, 0, 0, 0, 1044, 1079, 1028, 972, 1043, 990, 1057, 1047, 13, 0, 0, 0, 0, 0, 0, 0, 1033, 1007, 986, 1006, 1057, 1036, 1035, 1014, 30, 0, 0, 0, 0, 0, 0, 0, 1067, 1046, 990, 1015, 1013, 1002, 1046, 1005, 6, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8179, 8205, 8137, 8150, 8273, 8204, 8192] avg=8192.0000 max=8273 max_rank=5 min=8137 min_rank=3 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1000, 977, 1028, 1028, 1043, 1018, 1031, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1059, 992, 1049, 1090, 1003, 1046, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1018, 1019, 982, 1084, 1015, 998, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1039, 1035, 1071, 1010, 1003, 983, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1001, 977, 992, 986, 1031, 1041, 1012, 9, 4, 0, 0, 0, 0, 0, 0, 986, 1002, 994, 1036, 1007, 975, 1007, 1010, 14, 0, 0, 0, 0, 0, 0, 0, 1015, 1038, 1002, 1025, 1009, 1022, 988, 1053, 32, 0, 0, 0, 0, 0, 0, 0, 1047, 1073, 1034, 1027, 1026, 1063, 999, 1087, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8329, 8218, 8218, 8065, 8031, 8184, 8364] avg=8192.0000 max=8364 max_rank=7 min=8031 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1027, 1066, 1066, 1049, 1017, 994, 975, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1055, 1073, 1021, 995, 1006, 1002, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1049, 991, 997, 1000, 957, 1044, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1045, 1093, 1035, 1079, 1032, 992, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1029, 1034, 1063, 1065, 976, 1043, 1018, 9, 4, 0, 0, 0, 0, 0, 0, 1014, 1017, 1005, 1025, 1076, 999, 1050, 961, 13, 0, 0, 0, 0, 0, 0, 0, 1008, 1032, 1053, 1034, 1041, 990, 1021, 1032, 30, 0, 0, 0, 0, 0, 0, 0, 987, 1025, 995, 1004, 1025, 1002, 1012, 1039, 6, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8160, 8026, 8307, 8288, 8160, 8241, 8096] avg=8192.0000 max=8307 max_rank=3 min=8026 min_rank=2 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1030, 1025, 1069, 1019, 1027, 1021, 981, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1034, 1018, 959, 997, 971, 1010, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1018, 1000, 1048, 1020, 991, 991, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1026, 1008, 1044, 1063, 1036, 1051, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1044, 1011, 1032, 1007, 982, 1035, 1006, 12, 6, 0, 0, 0, 0, 0, 0, 1008, 998, 1025, 1051, 1034, 1041, 1083, 1053, 18, 0, 0, 0, 0, 0, 0, 0, 1071, 974, 990, 1048, 1029, 1067, 1073, 995, 37, 0, 0, 0, 0, 0, 0, 0, 1041, 1011, 1012, 1000, 1027, 1076, 984, 1013, 10, 6, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8059, 8102, 8174, 8185, 8311, 8284, 8186] avg=8192.0000 max=8311 max_rank=5 min=8059 min_rank=1 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32966 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1036, 1089, 1028, 1033, 1047, 1029, 1003, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1058, 1040, 986, 1008, 1007, 1034, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1022, 1029, 988, 968, 993, 1079, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1072, 1009, 1008, 1032, 1062, 1026, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1017, 1045, 969, 1050, 1005, 1024, 1006, 9, 4, 0, 0, 0, 0, 0, 0, 1071, 1043, 1016, 1006, 1005, 1012, 1007, 976, 13, 0, 0, 0, 0, 0, 0, 0, 1045, 1000, 990, 1035, 994, 1030, 1040, 1018, 31, 0, 0, 0, 0, 0, 0, 0, 1018, 1031, 1043, 1096, 1042, 990, 1005, 1036, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8154, 8148, 8208, 8184, 8149, 8183, 8270] avg=8192.0000 max=8270 max_rank=7 min=8148 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1076, 1043, 1045, 1058, 1023, 1030, 1010, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1003, 994, 1049, 972, 1104, 1002, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1054, 1064, 1013, 1029, 1021, 1042, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1043, 1067, 1010, 1044, 1015, 1021, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1029, 1022, 1034, 992, 987, 1066, 1021, 9, 4, 0, 0, 0, 0, 0, 0, 995, 1026, 962, 1005, 999, 1014, 1025, 1026, 14, 0, 0, 0, 0, 0, 0, 0, 996, 979, 1038, 1011, 994, 995, 1000, 1048, 31, 0, 0, 0, 0, 0, 0, 0, 1011, 1056, 1019, 999, 1044, 1052, 960, 981, 6, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8312, 8151, 8281, 8292, 8213, 8066, 8092, 8129] avg=8192.0000 max=8312 max_rank=0 min=8066 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1007, 1002, 991, 1047, 1003, 1037, 981, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1000, 996, 1040, 1030, 995, 1054, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1018, 1057, 1001, 982, 1106, 1025, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1042, 1017, 972, 1007, 1019, 1011, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1071, 983, 1063, 986, 993, 1064, 997, 9, 4, 0, 0, 0, 0, 0, 0, 1000, 1075, 1048, 1034, 1043, 1048, 999, 1036, 13, 0, 0, 0, 0, 0, 0, 0, 991, 1010, 1009, 1025, 1028, 1020, 981, 1005, 30, 0, 0, 0, 0, 0, 0, 0, 1038, 1013, 1007, 1031, 1026, 1044, 1033, 1000, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8064, 8199, 8324, 8127, 8226, 8296, 8099, 8201] avg=8192.0000 max=8324 max_rank=2 min=8064 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8320, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8320, 8192, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8320, 8192, 8320, 8192, 8064, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8320, 8064, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1039, 1032, 995, 1075, 1035, 1030, 1017, 1001, 1016, 992, 1015, 1020, 1004, 1064, 1052, 1020, 954, 1024, 1074, 1092, 1040, 1062, 996, 1015, 1046, 1007, 982, 1032, 1032, 1020, 1022, 1045, 1035, 1064, 1069, 1024, 976, 1018, 982, 1038, 999, 983, 1056, 1005, 1029, 1025, 1026, 974, 1040, 1035, 1034, 995, 1041, 1009, 998, 982, 1022, 1019, 971, 1037, 1027, 1068, 1098, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1021, 1037, 956, 1037, 1031, 1027, 1029, 1023, 1030, 997, 1015, 1053, 1020, 1015, 1020, 1042, 974, 1081, 1059, 984, 1007, 1039, 993, 1047, 1028, 1032, 1043, 999, 1066, 1024, 1064, 1030, 1004, 1014, 1020, 977, 1030, 991, 1020, 1050, 1039, 1031, 1005, 1040, 1023, 1050, 1014, 1030, 1003, 979, 1025, 1023, 1033, 1038, 1052, 1049, 972, 1025, 1080, 1024, 1004, 1023, 1036, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[977, 1018, 1026, 1051, 1015, 1038, 1067, 1017, 966, 1044, 1020, 1023, 979, 1040, 989, 1039, 1005, 1041, 1036, 1011, 990, 1066, 1018, 1001, 1005, 1020, 980, 1042, 1016, 1095, 973, 1030, 1002, 1104, 1018, 1040, 1038, 1040, 1037, 1022, 1033, 1021, 1062, 1037, 1008, 959, 1061, 1029, 1061, 998, 1046, 1033, 1023, 1016, 1002, 1042, 988, 1013, 1021, 1042, 971, 1023, 1096, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1028, 998, 1034, 1019, 985, 1006, 1002, 1021, 1014, 994, 1019, 983, 1056, 991, 1001, 1007, 1042, 1095, 1042, 1054, 1021, 1071, 1065, 1067, 998, 938, 1057, 1044, 1034, 1025, 993, 1013, 997, 1055, 1023, 1064, 1031, 983, 968, 1030, 1096, 1080, 1029, 1037, 1012, 1012, 1021, 977, 991, 1030, 1012, 1029, 1047, 1042, 999, 999, 999, 1067, 1028, 1042, 1065, 1009, 1054, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1098, 1037, 1052, 1052, 1034, 988, 1051, 1019, 1035, 1035, 1033, 1090, 1054, 1062, 1031, 985, 1040, 1037, 1041, 1029, 1005, 995, 965, 1032, 1035, 992, 995, 975, 1030, 1039, 1016, 1000, 999, 998, 1052, 994, 1031, 1028, 978, 1017, 983, 1005, 1038, 1040, 1009, 1048, 1014, 1041, 1060, 1025, 1045, 1029, 976, 998, 1051, 1030, 1011, 1033, 1024, 1005, 1010, 981, 1037, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1051, 955, 1066, 1011, 1060, 1052, 1046, 995, 982, 1019, 1044, 1043, 1011, 1016, 941, 1043, 1028, 1039, 998, 1024, 1046, 1046, 1018, 1026, 1024, 1064, 997, 1006, 1066, 1034, 972, 1029, 962, 1049, 1119, 1085, 1058, 981, 1044, 1041, 1039, 1075, 992, 1008, 983, 996, 1021, 1021, 1012, 1051, 994, 1014, 1012, 1048, 1022, 1071, 1009, 1003, 1011, 990, 1034, 1016, 1030, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1000, 1005, 1003, 1058, 994, 1055, 988, 1017, 1052, 1012, 1062, 1067, 993, 1027, 1047, 1008, 1077, 999, 981, 1020, 1033, 1012, 1028, 1054, 979, 1067, 1005, 1005, 1019, 1048, 1010, 1015, 1013, 1019, 1038, 1027, 980, 1058, 1012, 1037, 1008, 1043, 1018, 1065, 1021, 990, 955, 1008, 1063, 1021, 1003, 1019, 1009, 1061, 1087, 1020, 1054, 1062, 1034, 1039, 1002, 1034, 967, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[994, 1037, 1033, 998, 1045, 1051, 1012, 1000, 1007, 1075, 1022, 988, 971, 1015, 1047, 1010, 1075, 999, 1015, 978, 976, 1073, 1019, 1047, 1054, 1032, 1025, 1063, 1052, 1036, 1026, 1012, 1045, 1034, 1022, 1041, 975, 1013, 1000, 1044, 1024, 1018, 996, 1010, 990, 1046, 1028, 1015, 953, 1041, 1033, 1071, 1014, 1010, 1033, 994, 1040, 1034, 1033, 1053, 1040, 1066, 1038, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1028, 998, 1034, 1019, 985, 1006, 1002, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 994, 1019, 983, 1056, 991, 1001, 1007, 12, 8, 0, 0, 0, 0, 0, 0, 1042, 1095, 1042, 1054, 1021, 1036, 1065, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 998, 938, 1057, 1044, 1034, 1025, 993, 1013, 4, 2, 1, 0, 0, 0, 0, 0, 997, 1055, 1021, 1064, 1031, 983, 968, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1096, 1080, 1029, 1037, 1012, 1012, 1021, 977, 23, 0, 0, 0, 0, 0, 0, 0, 991, 1030, 1012, 1029, 1047, 1042, 995, 999, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1067, 1028, 1042, 1065, 1009, 1045, 991, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8093, 8085, 8422, 8109, 8149, 8287, 8145, 8246] avg=8192.0000 max=8422 max_rank=2 min=8085 min_rank=1 imbalance_ratio=1.028076 max_load_reduction=57114 max_load_reduction_pct=0.871490 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[977, 1018, 1026, 1050, 1015, 1038, 1067, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1044, 1020, 1023, 979, 1040, 989, 1039, 12, 9, 0, 0, 0, 0, 0, 0, 1005, 1041, 1036, 1011, 990, 1031, 1018, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1020, 980, 1042, 1016, 1095, 973, 1030, 4, 2, 1, 1, 0, 0, 0, 0, 1002, 1104, 1016, 1040, 1038, 1040, 1037, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1021, 1062, 1037, 1008, 959, 1061, 1029, 23, 0, 0, 0, 0, 0, 0, 0, 1061, 998, 1046, 1033, 1023, 1016, 998, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1013, 1021, 1042, 971, 1023, 1086, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8121, 8133, 8169, 8299, 8233, 8217, 8156] avg=8192.0000 max=8299 max_rank=4 min=8121 min_rank=1 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32905 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1051, 955, 1066, 1011, 1060, 1052, 1046, 995, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1019, 1044, 1043, 1011, 1016, 941, 1043, 12, 8, 0, 0, 0, 0, 0, 0, 1028, 1039, 998, 1024, 1046, 1011, 1018, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1064, 997, 1006, 1066, 1034, 972, 1029, 4, 2, 1, 0, 0, 0, 0, 0, 962, 1049, 1117, 1085, 1058, 981, 1044, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1075, 992, 1008, 983, 996, 1021, 1021, 23, 0, 0, 0, 0, 0, 0, 0, 1012, 1051, 994, 1014, 1012, 1048, 1018, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1003, 1011, 990, 1034, 1016, 1021, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8119, 8190, 8199, 8337, 8158, 8220, 8077] avg=8192.0000 max=8337 max_rank=4 min=8077 min_rank=7 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1021, 1037, 956, 1037, 1031, 1027, 1029, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 997, 1015, 1053, 1020, 1015, 1020, 1042, 12, 8, 0, 0, 0, 0, 0, 0, 974, 1081, 1059, 984, 1007, 1005, 993, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1032, 1043, 999, 1066, 1024, 1064, 1030, 5, 2, 1, 0, 0, 0, 0, 0, 1004, 1014, 1018, 977, 1030, 991, 1020, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1031, 1005, 1040, 1023, 1050, 1014, 1030, 22, 0, 0, 0, 0, 0, 0, 0, 1003, 979, 1025, 1023, 1033, 1038, 1047, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1025, 1080, 1024, 1004, 1023, 1027, 1009, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8212, 8150, 8294, 8104, 8254, 8197, 8164] avg=8192.0000 max=8294 max_rank=3 min=8104 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1098, 1037, 1052, 1051, 1034, 988, 1051, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1035, 1033, 1090, 1054, 1062, 1031, 985, 11, 8, 0, 0, 0, 0, 0, 0, 1040, 1037, 1041, 1029, 1005, 962, 965, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 992, 995, 975, 1030, 1039, 1016, 1000, 5, 2, 1, 1, 0, 0, 0, 0, 999, 998, 1050, 994, 1031, 1028, 978, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1005, 1038, 1040, 1009, 1048, 1014, 1041, 22, 0, 0, 0, 0, 0, 0, 0, 1060, 1025, 1045, 1029, 976, 998, 1046, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1033, 1024, 1005, 1010, 981, 1028, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8330, 8344, 8111, 8091, 8095, 8200, 8209, 8156] avg=8192.0000 max=8344 max_rank=1 min=8091 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32660 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1039, 1032, 995, 1074, 1035, 1030, 1017, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 992, 1015, 1020, 1004, 1064, 1052, 1020, 12, 9, 0, 0, 0, 0, 0, 0, 954, 1024, 1074, 1092, 1040, 1027, 996, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1007, 982, 1032, 1032, 1020, 1022, 1045, 4, 2, 1, 1, 0, 0, 0, 0, 1035, 1064, 1067, 1024, 976, 1018, 982, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 999, 983, 1056, 1005, 1029, 1025, 1026, 974, 23, 0, 0, 0, 0, 0, 0, 0, 1040, 1035, 1034, 995, 1041, 1009, 994, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1019, 971, 1037, 1027, 1068, 1088, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8204, 8222, 8194, 8204, 8120, 8130, 8239] avg=8192.0000 max=8239 max_rank=7 min=8120 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1000, 1005, 1003, 1057, 994, 1055, 988, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1012, 1062, 1067, 993, 1027, 1047, 1008, 12, 7, 0, 0, 0, 0, 0, 0, 1077, 999, 981, 1020, 1033, 978, 1028, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1067, 1005, 1005, 1019, 1048, 1010, 1015, 5, 2, 1, 1, 0, 0, 0, 0, 1013, 1019, 1036, 1027, 980, 1058, 1012, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1043, 1018, 1065, 1021, 990, 955, 1008, 22, 0, 0, 0, 0, 0, 0, 0, 1063, 1021, 1003, 1019, 1009, 1061, 1082, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1062, 1034, 1039, 1002, 1034, 959, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8287, 8170, 8157, 8182, 8130, 8278, 8213] avg=8192.0000 max=8287 max_rank=1 min=8119 min_rank=0 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[994, 1037, 1033, 994, 1045, 1051, 1012, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1075, 1022, 988, 971, 1015, 1047, 1010, 17, 12, 0, 0, 0, 0, 0, 0, 1075, 999, 1015, 978, 976, 1029, 1019, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1032, 1025, 1063, 1052, 1036, 1026, 1012, 9, 8, 2, 4, 0, 0, 0, 0, 1045, 1034, 1014, 1041, 975, 1013, 1000, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1018, 996, 1010, 990, 1046, 1028, 1015, 27, 0, 0, 0, 0, 0, 0, 0, 953, 1041, 1033, 1071, 1014, 1010, 1024, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1034, 1033, 1053, 1040, 1066, 1024, 995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8164, 8138, 8323, 8166, 8154, 8140, 8285] avg=8192.0000 max=8323 max_rank=3 min=8138 min_rank=2 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8448, 8192, 8192, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8448, 8448, 8320, 8192, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1060, 1008, 1019, 961, 965, 1080, 1031, 1065, 1002, 1036, 1037, 1047, 1018, 1039, 1031, 1022, 926, 997, 1015, 1031, 1059, 1058, 1005, 1029, 1030, 1022, 1005, 1038, 1058, 1030, 1032, 1024, 1089, 991, 1014, 1059, 978, 1012, 1040, 1039, 1031, 968, 1039, 1011, 1068, 1015, 975, 988, 947, 1033, 988, 1028, 1082, 1006, 1020, 1011, 1015, 1078, 1030, 1032, 1026, 1009, 1094, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[987, 1024, 1025, 981, 1041, 980, 1052, 1053, 1030, 1032, 982, 1047, 1023, 1033, 1014, 1017, 957, 1012, 1043, 1011, 999, 1049, 1039, 1069, 1032, 1048, 999, 975, 1033, 1029, 1048, 981, 960, 1026, 1019, 1043, 1073, 989, 1016, 990, 1052, 1006, 1025, 1045, 1061, 1034, 1002, 1018, 1035, 1029, 1065, 1056, 1055, 1058, 1021, 1034, 1064, 1016, 1013, 996, 1026, 1053, 1037, 974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[991, 1041, 1017, 1004, 1023, 1022, 1039, 1058, 1058, 1061, 1023, 1001, 1034, 1017, 982, 1023, 992, 1013, 994, 1041, 1060, 1101, 1036, 972, 1006, 1038, 999, 1008, 1055, 1080, 1006, 1014, 1014, 1034, 987, 1023, 987, 1068, 1005, 1021, 1023, 1016, 1084, 1039, 1060, 971, 1001, 1025, 1017, 1035, 1019, 1007, 1052, 1007, 1049, 1017, 1058, 983, 1021, 1018, 1061, 996, 979, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1016, 1035, 1012, 996, 964, 1024, 986, 980, 1040, 1016, 1025, 1022, 1011, 1050, 1053, 1014, 997, 1016, 1041, 989, 1011, 1071, 1015, 1033, 1003, 1058, 1002, 1050, 1053, 1035, 1037, 1006, 994, 999, 1030, 1070, 1017, 1027, 963, 971, 979, 1077, 1042, 1017, 1052, 1057, 1018, 1022, 1095, 1041, 1029, 1058, 997, 972, 1007, 1051, 1047, 1045, 1057, 1018, 1056, 1035, 998, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[993, 979, 1056, 1031, 1034, 1007, 1060, 1029, 1004, 1010, 1066, 990, 1013, 1025, 1053, 1039, 998, 1069, 1042, 1067, 1012, 1054, 983, 1006, 1031, 966, 1011, 1002, 1003, 1068, 987, 1029, 1034, 995, 977, 1032, 1032, 1005, 1069, 1018, 1083, 1020, 1028, 1029, 1060, 999, 1032, 1004, 988, 1012, 1025, 1043, 1000, 1070, 989, 1093, 1017, 1041, 987, 1025, 1028, 1050, 987, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1029, 1015, 1014, 1023, 1061, 1071, 1022, 994, 972, 1032, 1031, 1083, 1034, 1046, 1020, 1018, 996, 1030, 1013, 1058, 1050, 1053, 1016, 1022, 1056, 1044, 1003, 1043, 992, 1056, 1021, 1000, 1049, 993, 1001, 1005, 1035, 1021, 998, 1052, 968, 1022, 1017, 1020, 992, 1020, 1021, 990, 1038, 1054, 992, 1014, 1027, 1050, 1036, 997, 1055, 1003, 1075, 1019, 1025, 1063, 978, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1004, 1017, 1001, 1007, 1065, 990, 1034, 1016, 1036, 1038, 1015, 978, 1038, 1040, 1063, 1020, 1054, 995, 985, 1008, 1064, 1036, 1043, 1087, 1047, 1072, 1000, 983, 1021, 1045, 1007, 1005, 1028, 1040, 1009, 1005, 1041, 1052, 1021, 985, 1046, 1009, 1078, 967, 1041, 1063, 959, 1000, 1069, 1014, 1018, 1032, 1032, 984, 1035, 1000, 987, 1055, 1026, 1039, 1033, 1002, 1041, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1023, 1043, 1016, 1043, 994, 1036, 1027, 1018, 1047, 1046, 1048, 1038, 998, 1009, 1027, 1072, 976, 960, 951, 1019, 1002, 1014, 1054, 1061, 1074, 1021, 1046, 1034, 1002, 1068, 1017, 1011, 1030, 1048, 998, 1043, 1021, 1027, 1017, 1026, 1046, 1099, 1037, 1014, 1042, 1000, 990, 997, 1053, 1073, 1002, 1026, 979, 1017, 1045, 978, 1044, 1041, 1018, 1019, 1022, 982, 987, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[991, 1041, 1017, 1004, 1023, 1022, 1039, 1058, 17, 13, 0, 0, 0, 0, 0, 0, 1058, 1029, 1023, 1001, 1034, 1017, 982, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1013, 994, 1041, 1060, 1101, 1036, 972, 5, 4, 0, 0, 0, 0, 0, 0, 1006, 1038, 999, 1008, 1055, 1073, 1006, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1034, 987, 1023, 987, 1068, 1005, 1021, 32, 3, 0, 0, 0, 0, 0, 0, 1023, 1016, 1084, 1039, 1060, 971, 1001, 1025, 2, 0, 0, 0, 0, 0, 0, 0, 1017, 1015, 1019, 1007, 1052, 1007, 1049, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 983, 1021, 1018, 1061, 996, 979, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8167, 8218, 8199, 8174, 8221, 8183, 8149] avg=8192.0000 max=8225 max_rank=0 min=8149 min_rank=7 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1016, 1035, 1012, 996, 964, 1024, 986, 980, 17, 12, 0, 0, 0, 0, 0, 0, 1040, 985, 1025, 1022, 1011, 1050, 1053, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1016, 1041, 989, 1011, 1071, 1015, 1033, 5, 4, 0, 0, 0, 0, 0, 0, 1003, 1058, 1002, 1050, 1053, 1028, 1037, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 994, 999, 1030, 1070, 1017, 1027, 963, 971, 31, 3, 0, 0, 0, 0, 0, 0, 979, 1077, 1042, 1017, 1052, 1057, 1018, 1022, 2, 0, 0, 0, 0, 0, 0, 0, 1095, 1021, 1029, 1058, 997, 972, 1007, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1045, 1057, 1018, 1056, 1035, 998, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8042, 8200, 8182, 8237, 8105, 8266, 8230, 8274] avg=8192.0000 max=8274 max_rank=7 min=8042 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1060, 1008, 1019, 961, 965, 1080, 1031, 1065, 17, 12, 0, 0, 0, 0, 0, 0, 1002, 1005, 1037, 1047, 1018, 1039, 1031, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 926, 997, 1015, 1031, 1059, 1058, 1005, 1029, 5, 4, 0, 0, 0, 0, 0, 0, 1030, 1022, 1005, 1038, 1058, 1023, 1032, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 991, 1014, 1059, 978, 1012, 1040, 1039, 31, 3, 0, 0, 0, 0, 0, 0, 1031, 968, 1039, 1011, 1068, 1015, 975, 988, 2, 0, 0, 0, 0, 0, 0, 0, 947, 1013, 988, 1028, 1082, 1006, 1020, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1078, 1030, 1032, 1026, 1009, 1094, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8201, 8129, 8232, 8256, 8097, 8095, 8308] avg=8192.0000 max=8308 max_rank=7 min=8095 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[987, 1024, 1025, 981, 1041, 980, 1052, 1053, 17, 13, 0, 0, 0, 0, 0, 0, 1030, 1001, 982, 1047, 1023, 1033, 1014, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 957, 1012, 1043, 1011, 999, 1049, 1039, 1069, 5, 4, 0, 0, 0, 0, 0, 0, 1032, 1048, 999, 975, 1033, 1022, 1048, 981, 0, 0, 0, 0, 0, 0, 0, 0, 960, 1026, 1019, 1043, 1073, 989, 1016, 990, 31, 3, 0, 0, 0, 0, 0, 0, 1052, 1006, 1025, 1045, 1061, 1034, 1002, 1018, 2, 0, 0, 0, 0, 0, 0, 0, 1035, 1009, 1065, 1056, 1055, 1058, 1021, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1016, 1013, 996, 1026, 1053, 1037, 974, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8147, 8188, 8138, 8150, 8245, 8333, 8162] avg=8192.0000 max=8333 max_rank=6 min=8138 min_rank=3 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1004, 1017, 1001, 1007, 1065, 990, 1034, 1016, 17, 12, 0, 0, 0, 0, 0, 0, 1036, 1007, 1015, 978, 1038, 1040, 1063, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 995, 985, 1008, 1064, 1036, 1043, 1087, 5, 4, 0, 0, 0, 0, 0, 0, 1047, 1072, 1000, 983, 1021, 1038, 1007, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1040, 1009, 1005, 1041, 1052, 1021, 985, 31, 3, 0, 0, 0, 0, 0, 0, 1046, 1009, 1078, 967, 1041, 1063, 959, 1000, 2, 0, 0, 0, 0, 0, 0, 0, 1069, 994, 1018, 1032, 1032, 984, 1035, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1055, 1026, 1039, 1033, 1002, 1041, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8197, 8281, 8173, 8215, 8165, 8164, 8178] avg=8192.0000 max=8281 max_rank=2 min=8163 min_rank=0 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[993, 979, 1056, 1031, 1034, 1007, 1060, 1029, 17, 12, 0, 0, 0, 0, 0, 0, 1004, 980, 1066, 990, 1013, 1025, 1053, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1069, 1042, 1067, 1012, 1054, 983, 1006, 5, 4, 0, 0, 0, 0, 0, 0, 1031, 966, 1011, 1002, 1003, 1061, 987, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 995, 977, 1032, 1032, 1005, 1069, 1018, 30, 3, 0, 0, 0, 0, 0, 0, 1083, 1020, 1028, 1029, 1060, 999, 1032, 1004, 2, 0, 0, 0, 0, 0, 0, 0, 988, 992, 1025, 1043, 1000, 1070, 989, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1041, 987, 1025, 1028, 1050, 987, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8170, 8240, 8090, 8195, 8257, 8200, 8166] avg=8192.0000 max=8257 max_rank=5 min=8090 min_rank=3 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1029, 1015, 1014, 1023, 1061, 1071, 1022, 994, 18, 12, 0, 0, 0, 0, 0, 0, 972, 1001, 1031, 1083, 1034, 1046, 1020, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1030, 1013, 1058, 1050, 1053, 1016, 1022, 5, 4, 0, 0, 0, 0, 0, 0, 1056, 1044, 1003, 1043, 992, 1049, 1021, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 993, 1001, 1005, 1035, 1021, 998, 1052, 31, 3, 0, 0, 0, 0, 0, 0, 968, 1022, 1017, 1020, 992, 1020, 1021, 990, 2, 0, 0, 0, 0, 0, 0, 0, 1038, 1033, 992, 1014, 1027, 1050, 1036, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1003, 1075, 1019, 1025, 1063, 978, 988, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8205, 8247, 8208, 8188, 8052, 8187, 8190] avg=8192.0000 max=8259 max_rank=0 min=8052 min_rank=5 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32617 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1023, 1043, 1016, 1043, 994, 1036, 1027, 1018, 22, 16, 0, 0, 0, 0, 0, 0, 1047, 1010, 1048, 1038, 998, 1009, 1027, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 976, 960, 951, 1019, 1002, 1014, 1054, 1061, 6, 8, 0, 0, 0, 0, 0, 0, 1074, 1021, 1046, 1034, 1002, 1054, 1017, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1048, 998, 1043, 1021, 1027, 1017, 1026, 36, 7, 0, 0, 0, 0, 0, 0, 1046, 1099, 1037, 1014, 1042, 1000, 990, 997, 8, 0, 0, 0, 0, 0, 0, 0, 1053, 1044, 1002, 1026, 979, 1017, 1045, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1041, 1018, 1019, 1022, 982, 987, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8249, 8051, 8259, 8253, 8233, 8144, 8109] avg=8192.0000 max=8259 max_rank=3 min=8051 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8448, 8192, 8448, 8192, 8064, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8064, 8192, 8320, 8320, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8448, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7936, 8192, 8192, 8320, 8320, 8448, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8064, 8320, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1044, 1030, 988, 1014, 1037, 1072, 1006, 1024, 1032, 1032, 1014, 1065, 998, 969, 1015, 990, 1041, 1063, 1004, 988, 989, 1002, 980, 1032, 1038, 1121, 1048, 1051, 1035, 1035, 1050, 1010, 1023, 1013, 1024, 1025, 1032, 1022, 1062, 1041, 1016, 994, 1028, 1036, 991, 963, 1038, 1004, 1060, 988, 1070, 998, 1061, 961, 1049, 1030, 1015, 1029, 1031, 1029, 1042, 1035, 1017, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1056, 1059, 1056, 1051, 1034, 1010, 1035, 1049, 1013, 1084, 1024, 998, 1044, 1032, 999, 1009, 1005, 1014, 1006, 1033, 1054, 968, 982, 1019, 1035, 1063, 977, 1028, 1029, 1045, 1014, 1006, 1018, 1051, 970, 1071, 1047, 1044, 1016, 1034, 1048, 1046, 1033, 962, 1018, 1027, 1029, 998, 1018, 1044, 1012, 1044, 1004, 1057, 1027, 1009, 1037, 1006, 990, 964, 1014, 1023, 1024, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1037, 1028, 1049, 1065, 1022, 1001, 979, 991, 1065, 1074, 1008, 1026, 955, 1025, 1003, 1003, 992, 1057, 1013, 1005, 1046, 985, 1036, 1009, 1041, 1043, 986, 1006, 973, 1027, 1055, 1039, 989, 978, 1080, 1035, 1047, 1076, 1011, 1053, 1068, 1067, 1064, 1052, 1009, 1051, 1013, 988, 1036, 1040, 1041, 974, 1023, 972, 1072, 1034, 1002, 1052, 1022, 983, 1003, 1030, 1026, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1040, 1026, 1028, 1052, 1035, 1013, 1022, 1029, 1002, 1031, 978, 1017, 990, 1009, 1009, 1021, 1059, 1035, 1045, 1060, 1001, 1010, 1044, 1085, 1027, 1001, 1029, 1035, 1044, 1052, 1044, 1030, 998, 1031, 1014, 1020, 1059, 1024, 1013, 1014, 1080, 1037, 1011, 982, 1034, 1034, 1019, 1024, 1024, 1031, 1004, 1102, 965, 1026, 1018, 1010, 1021, 1033, 1066, 996, 1034, 994, 965, 950] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1061, 1031, 1021, 1062, 1073, 995, 1035, 1066, 1041, 1027, 991, 999, 1031, 1005, 1044, 989, 1034, 1022, 1021, 1050, 1030, 974, 1030, 1034, 1043, 1030, 1022, 1028, 1021, 1003, 1064, 1025, 1014, 1025, 1018, 1053, 1046, 968, 1009, 1041, 1051, 1010, 1030, 983, 1045, 1011, 1058, 1033, 1054, 976, 1045, 957, 1027, 994, 1039, 968, 1034, 983, 1081, 987, 1043, 1002, 1035, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1051, 1059, 989, 1001, 1061, 1014, 1057, 995, 1003, 1020, 1027, 1017, 989, 1026, 1022, 1008, 1033, 1026, 939, 1002, 1020, 1059, 1032, 1024, 1099, 1007, 1002, 994, 1043, 1039, 1064, 1036, 1030, 1041, 1043, 1047, 990, 1002, 1076, 993, 1060, 1003, 1019, 1072, 1046, 1018, 1005, 1040, 1034, 1030, 1032, 1035, 994, 1029, 1022, 991, 982, 990, 1048, 1030, 998, 1019, 1008, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[985, 1032, 1111, 1017, 1071, 1014, 1024, 1040, 1022, 956, 998, 1025, 1026, 986, 1032, 1079, 1046, 1048, 1069, 1053, 1002, 1027, 975, 1060, 1041, 992, 999, 1001, 1002, 1009, 1009, 1027, 1024, 1033, 1041, 1026, 997, 995, 977, 1018, 996, 1028, 988, 1067, 998, 1035, 1049, 1079, 1052, 1076, 1024, 1056, 1057, 1046, 1045, 1020, 985, 992, 1056, 1009, 1010, 953, 1026, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[987, 1033, 1043, 1050, 986, 1016, 991, 997, 1025, 981, 1040, 993, 1033, 1042, 1004, 998, 1018, 1091, 1001, 998, 1086, 1024, 1049, 984, 1043, 992, 987, 999, 1025, 994, 1045, 1032, 1040, 1056, 1072, 1081, 1027, 1003, 985, 1024, 1071, 1066, 1028, 1015, 1039, 1025, 1006, 1021, 1006, 1010, 1066, 955, 1066, 1035, 1010, 1039, 1050, 989, 1016, 1012, 1005, 1035, 1025, 1071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1051, 1059, 989, 1001, 1009, 1014, 1057, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1020, 1027, 1017, 989, 1026, 1022, 1008, 26, 24, 10, 5, 0, 0, 0, 0, 1033, 1026, 939, 1002, 1020, 1059, 1032, 1024, 1, 0, 0, 0, 0, 0, 0, 0, 1073, 1007, 1002, 994, 1043, 1039, 1064, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1041, 1043, 1023, 990, 1002, 1076, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1003, 1019, 1072, 1046, 1018, 1005, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1030, 1026, 1035, 994, 1029, 1022, 991, 0, 0, 0, 0, 0, 0, 0, 0, 982, 990, 1048, 1030, 998, 1019, 1008, 1051, 52, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8177, 8136, 8258, 8198, 8236, 8161, 8195] avg=8192.0000 max=8258 max_rank=3 min=8136 min_rank=2 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1056, 1059, 1056, 1051, 983, 1010, 1035, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1084, 1024, 998, 1044, 1032, 999, 1009, 24, 24, 10, 5, 0, 0, 0, 0, 1005, 1014, 1006, 1033, 1054, 968, 982, 1019, 1, 0, 0, 0, 0, 0, 0, 0, 1011, 1063, 977, 1028, 1029, 1045, 1014, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1051, 970, 1047, 1047, 1044, 1016, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1046, 1033, 962, 1018, 1027, 1029, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1044, 1006, 1044, 1004, 1057, 1027, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1006, 990, 964, 1014, 1023, 1024, 1020, 51, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8299, 8266, 8082, 8173, 8227, 8134, 8209, 8146] avg=8192.0000 max=8299 max_rank=0 min=8082 min_rank=2 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1061, 1031, 1021, 1062, 1020, 995, 1035, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1027, 991, 999, 1031, 1005, 1044, 989, 24, 24, 10, 5, 0, 0, 0, 0, 1034, 1022, 1021, 1050, 1030, 974, 1030, 1034, 1, 0, 0, 0, 0, 0, 0, 0, 1019, 1030, 1022, 1028, 1021, 1003, 1064, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1025, 1018, 1029, 1046, 968, 1009, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1010, 1030, 983, 1045, 1011, 1058, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 976, 1039, 957, 1027, 994, 1039, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 983, 1081, 987, 1043, 1002, 1035, 1014, 53, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8190, 8196, 8212, 8150, 8194, 8054, 8249] avg=8192.0000 max=8291 max_rank=0 min=8054 min_rank=6 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1044, 1030, 988, 1014, 986, 1072, 1006, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1032, 1014, 1065, 998, 969, 1015, 990, 24, 23, 10, 5, 0, 0, 0, 0, 1041, 1063, 1004, 988, 989, 1002, 980, 1032, 1, 0, 0, 0, 0, 0, 0, 0, 1014, 1121, 1048, 1051, 1035, 1035, 1050, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1013, 1024, 1002, 1032, 1022, 1062, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 990, 994, 1028, 1036, 991, 963, 1038, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 988, 1064, 998, 1061, 961, 1049, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1029, 1031, 1029, 1042, 1035, 1017, 992, 51, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8177, 8100, 8364, 8219, 8044, 8211, 8257] avg=8192.0000 max=8364 max_rank=3 min=8044 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[985, 1032, 1111, 1017, 1018, 1014, 1024, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 956, 998, 1025, 1026, 986, 1032, 1079, 24, 23, 10, 5, 0, 0, 0, 0, 1046, 1048, 1069, 1053, 1002, 1027, 975, 1060, 1, 0, 0, 0, 0, 0, 0, 0, 1017, 992, 999, 1001, 1002, 1009, 1009, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1033, 1041, 1003, 997, 995, 977, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1028, 988, 1067, 998, 1035, 1049, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1076, 1018, 1056, 1057, 1046, 1045, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 985, 992, 1056, 1009, 1010, 953, 1026, 1000, 53, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8186, 8281, 8056, 8088, 8214, 8370, 8100] avg=8192.0000 max=8370 max_rank=6 min=8056 min_rank=3 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1040, 1026, 1028, 1052, 984, 1013, 1022, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1031, 978, 1017, 990, 1009, 1009, 1021, 24, 23, 11, 5, 0, 0, 0, 0, 1059, 1035, 1045, 1060, 1001, 1010, 1044, 1085, 1, 0, 0, 0, 0, 0, 0, 0, 1003, 1001, 1029, 1035, 1044, 1052, 1044, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1031, 1014, 997, 1059, 1024, 1013, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1037, 1011, 982, 1034, 1034, 1019, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1031, 998, 1102, 965, 1026, 1018, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1033, 1066, 996, 1034, 994, 965, 950, 51, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8120, 8340, 8238, 8150, 8193, 8174, 8127] avg=8192.0000 max=8340 max_rank=2 min=8120 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1037, 1028, 1049, 1065, 972, 1001, 979, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1074, 1008, 1026, 955, 1025, 1003, 1003, 24, 24, 10, 5, 0, 0, 0, 0, 992, 1057, 1013, 1005, 1046, 985, 1036, 1009, 1, 0, 0, 0, 0, 0, 0, 0, 1017, 1043, 986, 1006, 973, 1027, 1055, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 989, 978, 1080, 1011, 1047, 1076, 1011, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1067, 1064, 1052, 1009, 1051, 1013, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1040, 1035, 974, 1023, 972, 1072, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1052, 1022, 983, 1003, 1030, 1026, 1001, 50, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8222, 8144, 8146, 8245, 8285, 8186, 8186] avg=8192.0000 max=8285 max_rank=5 min=8122 min_rank=0 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32902 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[987, 1033, 1043, 1050, 933, 1016, 991, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 981, 1040, 993, 1033, 1042, 1004, 998, 28, 29, 15, 10, 0, 0, 0, 0, 1018, 1091, 1001, 998, 1086, 1024, 1049, 984, 6, 0, 0, 0, 0, 0, 0, 0, 1015, 992, 987, 999, 1025, 994, 1045, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1056, 1072, 1052, 1027, 1003, 985, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1066, 1028, 1015, 1039, 1025, 1006, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1010, 1050, 955, 1066, 1035, 1010, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 989, 1016, 1012, 1005, 1035, 1025, 1071, 53, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8050, 8198, 8257, 8089, 8259, 8236, 8171, 8276] avg=8192.0000 max=8276 max_rank=7 min=8050 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32942 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8320, 8320, 8320, 7936, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8192, 8192, 256, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8192, 8192, 8320, 8448, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8064, 8192, 8192, 8192, 8192, 512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8448, 8192, 8192, 8320, 8064, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[985, 1025, 1024, 1037, 998, 976, 976, 1021, 1025, 1016, 1051, 1040, 1048, 1002, 1092, 1045, 1052, 1031, 1013, 1015, 961, 1045, 1021, 1030, 1017, 1036, 1052, 1016, 1034, 1061, 1026, 1068, 987, 1052, 1044, 1043, 1015, 980, 982, 1039, 1077, 1030, 1078, 1002, 1019, 1066, 1049, 1033, 1011, 1045, 983, 1061, 1023, 1031, 1020, 1038, 1004, 991, 979, 999, 1019, 984, 1019, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1023, 1040, 1018, 1047, 1011, 1042, 1029, 1022, 1006, 1060, 1042, 1043, 1042, 1032, 1047, 982, 1023, 1055, 1008, 1044, 1052, 1045, 997, 1030, 1000, 1030, 1015, 1060, 1008, 1007, 1031, 1003, 1050, 991, 1010, 956, 989, 1016, 1010, 1010, 1028, 1017, 1028, 1051, 988, 1043, 953, 1046, 1021, 1040, 1019, 1044, 1035, 1006, 1047, 1033, 1010, 1040, 1047, 1004, 1049, 995, 1021, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1057, 1035, 1002, 976, 1058, 1083, 990, 994, 1017, 1024, 1024, 1034, 1023, 997, 1019, 985, 1018, 1040, 988, 1027, 1000, 988, 995, 1066, 1032, 1028, 1048, 1010, 1073, 1026, 1032, 1030, 1026, 1038, 1027, 1025, 1079, 1067, 1073, 1017, 1028, 1040, 993, 1007, 982, 1024, 1048, 1042, 1029, 1030, 1014, 1038, 1051, 993, 1079, 1027, 1032, 1012, 1007, 1000, 947, 1008, 1026, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1017, 1025, 980, 1043, 1037, 1003, 1019, 1050, 947, 1044, 1039, 1028, 1024, 1070, 1037, 1026, 1032, 1000, 1024, 1074, 1017, 1052, 1017, 987, 964, 1011, 959, 1043, 1057, 992, 960, 1030, 976, 1056, 1026, 1033, 993, 1040, 1010, 1035, 1045, 998, 1038, 1050, 1001, 1025, 984, 1060, 992, 1049, 1014, 1053, 1026, 1036, 1045, 1031, 1066, 1054, 1048, 984, 1042, 1025, 1092, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1022, 1013, 1048, 979, 1104, 1017, 999, 1001, 1063, 962, 1038, 1055, 1008, 1041, 1019, 1072, 978, 1010, 1018, 964, 1042, 998, 996, 1011, 1066, 1017, 980, 1051, 1066, 1023, 1009, 1003, 1064, 1040, 965, 1035, 1034, 1065, 987, 1041, 1015, 1043, 1029, 1015, 1054, 1002, 1035, 1005, 1012, 1032, 1086, 1047, 1008, 1007, 1035, 1007, 1041, 1046, 1005, 1045, 1031, 1028, 974, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1064, 1041, 1048, 1000, 1018, 1019, 1017, 1007, 1008, 1011, 1043, 1057, 1026, 1003, 981, 1049, 1053, 992, 1043, 1056, 1024, 1059, 1035, 1024, 1044, 1015, 962, 1068, 1025, 1043, 1034, 1010, 1059, 1017, 1055, 1007, 1061, 998, 1033, 1087, 1007, 1035, 1013, 1030, 993, 1062, 1063, 983, 1016, 933, 1013, 1027, 1064, 1021, 981, 1021, 983, 1007, 1031, 1015, 1004, 1006, 1027, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[985, 1026, 1020, 1057, 1002, 1016, 1062, 1060, 1033, 999, 1021, 1016, 1056, 1040, 1024, 1012, 1045, 997, 1046, 1092, 1006, 1013, 985, 1031, 1027, 1016, 1006, 1001, 1038, 1000, 1030, 997, 966, 1009, 1030, 1002, 1061, 1024, 1024, 995, 1018, 1060, 982, 1088, 1060, 1054, 1030, 1016, 992, 1020, 1024, 1057, 1065, 1026, 989, 1069, 1018, 988, 968, 1065, 1031, 1004, 1037, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1035, 1006, 1050, 991, 1020, 1020, 1054, 1058, 1001, 1014, 1007, 991, 990, 1047, 1034, 963, 1051, 1041, 1048, 1062, 1019, 956, 1034, 1069, 1027, 1083, 1017, 1020, 1049, 1059, 982, 1031, 1017, 1013, 1027, 1000, 1061, 975, 957, 1019, 1023, 1017, 993, 1016, 993, 1058, 1030, 1000, 971, 1072, 1041, 999, 1019, 1052, 1037, 1014, 1072, 1052, 997, 1051, 1030, 1060, 992, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1023, 1040, 1018, 1047, 1011, 1042, 1029, 1022, 2, 1, 0, 0, 0, 0, 0, 0, 1006, 1060, 1035, 1043, 1042, 1032, 1047, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1055, 1008, 1043, 1052, 1045, 997, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1030, 1015, 1060, 1006, 1007, 1031, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 991, 1010, 956, 989, 1016, 1010, 1010, 7, 7, 0, 0, 0, 0, 0, 0, 1028, 1017, 1028, 1051, 988, 1024, 953, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1040, 1019, 1021, 1035, 1006, 1047, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1040, 1047, 1004, 1049, 995, 1021, 1045, 23, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8247, 8253, 8152, 8046, 8135, 8222, 8246] avg=8192.0000 max=8253 max_rank=2 min=8046 min_rank=4 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[985, 1025, 1024, 1037, 998, 976, 976, 1021, 2, 1, 0, 0, 0, 0, 0, 0, 1025, 1016, 1044, 1040, 1048, 1002, 1092, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1031, 1013, 1014, 961, 1045, 1021, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1036, 1052, 1016, 1032, 1061, 1026, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1052, 1044, 1043, 1015, 980, 982, 1039, 7, 7, 0, 0, 0, 0, 0, 0, 1077, 1030, 1078, 1002, 1019, 1046, 1049, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1045, 983, 1038, 1023, 1031, 1020, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 991, 979, 999, 1019, 984, 1019, 994, 23, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8045, 8312, 8167, 8308, 8156, 8334, 8189, 8025] avg=8192.0000 max=8334 max_rank=5 min=8025 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1057, 1035, 1002, 976, 1058, 1083, 990, 994, 2, 1, 0, 0, 0, 0, 0, 0, 1017, 1024, 1018, 1034, 1023, 997, 1019, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1040, 988, 1026, 1000, 988, 995, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1028, 1048, 1010, 1071, 1026, 1032, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1038, 1027, 1025, 1079, 1067, 1073, 1017, 7, 6, 0, 0, 0, 0, 0, 0, 1028, 1040, 993, 1007, 982, 1005, 1048, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1030, 1014, 1015, 1051, 993, 1079, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1012, 1007, 1000, 947, 1008, 1026, 1008, 23, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8117, 8121, 8277, 8365, 8145, 8238, 8075] avg=8192.0000 max=8365 max_rank=4 min=8075 min_rank=7 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1022, 1013, 1048, 979, 1104, 1017, 999, 1001, 2, 1, 0, 0, 0, 0, 0, 0, 1063, 962, 1031, 1055, 1008, 1041, 1019, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1010, 1018, 963, 1042, 998, 996, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1017, 980, 1051, 1064, 1023, 1009, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1040, 965, 1035, 1034, 1065, 987, 1041, 6, 7, 0, 0, 0, 0, 0, 0, 1015, 1043, 1029, 1015, 1054, 984, 1035, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1032, 1086, 1024, 1008, 1007, 1035, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1046, 1005, 1045, 1031, 1028, 974, 1030, 23, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8251, 8016, 8213, 8244, 8180, 8211, 8235] avg=8192.0000 max=8251 max_rank=1 min=8016 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[985, 1026, 1020, 1057, 1002, 1016, 1062, 1060, 2, 1, 0, 0, 0, 0, 0, 0, 1033, 999, 1015, 1016, 1056, 1040, 1024, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 997, 1046, 1091, 1006, 1013, 985, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1016, 1006, 1001, 1036, 1000, 1030, 997, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1009, 1030, 1002, 1061, 1024, 1024, 995, 7, 6, 0, 0, 0, 0, 0, 0, 1018, 1060, 982, 1088, 1060, 1035, 1030, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1020, 1024, 1034, 1065, 1026, 989, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 988, 968, 1065, 1031, 1004, 1037, 1005, 23, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8195, 8214, 8113, 8124, 8289, 8219, 8151] avg=8192.0000 max=8289 max_rank=5 min=8113 min_rank=3 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1064, 1041, 1048, 1000, 1018, 1019, 1017, 1007, 8, 2, 3, 0, 0, 0, 0, 0, 1008, 1011, 1030, 1057, 1026, 1003, 981, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 992, 1043, 1054, 1024, 1059, 1035, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1015, 962, 1068, 1017, 1043, 1034, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1017, 1055, 1007, 1061, 998, 1033, 1087, 9, 10, 0, 0, 0, 0, 0, 0, 1007, 1035, 1013, 1030, 993, 1036, 1063, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 933, 1013, 1002, 1064, 1021, 981, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1007, 1031, 1015, 1004, 1006, 1027, 1005, 25, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8165, 8284, 8193, 8336, 8160, 8051, 8120] avg=8192.0000 max=8336 max_rank=4 min=8051 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1017, 1025, 980, 1043, 1037, 1003, 1019, 1050, 2, 1, 0, 0, 0, 0, 0, 0, 947, 1044, 1032, 1028, 1024, 1070, 1037, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1000, 1024, 1073, 1017, 1052, 1017, 987, 0, 0, 0, 0, 0, 0, 0, 0, 964, 1011, 959, 1043, 1055, 992, 960, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1056, 1026, 1033, 993, 1040, 1010, 1035, 7, 7, 0, 0, 0, 0, 0, 0, 1045, 998, 1038, 1050, 1001, 1006, 984, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1049, 1014, 1030, 1026, 1036, 1045, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1054, 1048, 984, 1042, 1025, 1092, 1001, 23, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8208, 8202, 8014, 8183, 8182, 8223, 8347] avg=8192.0000 max=8347 max_rank=7 min=8014 min_rank=3 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32935 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1035, 1006, 1050, 991, 1020, 1020, 1054, 1058, 2, 1, 0, 0, 0, 0, 0, 0, 1001, 1014, 1001, 991, 990, 1047, 1034, 963, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1041, 1048, 1061, 1019, 956, 1034, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1083, 1017, 1020, 1047, 1059, 982, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1013, 1027, 1000, 1061, 975, 957, 1019, 7, 6, 0, 0, 0, 0, 0, 0, 1023, 1017, 993, 1016, 993, 1039, 1030, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1072, 1041, 977, 1019, 1052, 1037, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1052, 997, 1051, 1030, 1060, 992, 1049, 22, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8041, 8279, 8266, 8082, 8111, 8183, 8337] avg=8192.0000 max=8337 max_rank=7 min=8041 min_rank=1 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8320, 8448, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8448, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[994, 1025, 1003, 1022, 990, 997, 1067, 1001, 1051, 994, 1052, 1021, 1028, 1010, 1067, 979, 1049, 1020, 1026, 986, 1014, 998, 1048, 962, 1066, 1059, 1029, 1016, 1000, 1046, 1024, 1051, 1041, 1041, 1052, 1044, 982, 1054, 995, 1010, 996, 1002, 1049, 1020, 1032, 977, 1040, 977, 1106, 1047, 1050, 1069, 1012, 1043, 983, 1036, 1029, 977, 1041, 1050, 1026, 1024, 1038, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1021, 1029, 1030, 1026, 1034, 1043, 990, 1019, 1018, 1049, 1035, 1027, 1058, 1038, 991, 1027, 1059, 1077, 1030, 1054, 1065, 1062, 987, 1038, 977, 1015, 1044, 1017, 961, 975, 1056, 953, 1066, 1078, 1000, 984, 1055, 1063, 969, 1061, 1022, 1023, 1021, 958, 1010, 1042, 1038, 998, 1053, 1024, 1029, 1044, 986, 1021, 937, 1074, 996, 1020, 1038, 990, 1025, 1042, 1056, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1044, 1013, 1014, 1050, 1001, 1077, 960, 1035, 1001, 1059, 1036, 1002, 1032, 1008, 1015, 1009, 995, 1013, 1040, 1049, 1034, 986, 1051, 1044, 1040, 1046, 1061, 1029, 1025, 1062, 1016, 1051, 1041, 971, 982, 1068, 1025, 1022, 997, 1018, 988, 1051, 1031, 1048, 961, 1035, 1027, 1010, 1038, 1000, 1031, 1090, 1018, 993, 988, 978, 1024, 1006, 1068, 1022, 1025, 1034, 1002, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[995, 1037, 1022, 1059, 1060, 1009, 1000, 1011, 1024, 1004, 1021, 1021, 1021, 991, 1066, 997, 997, 1008, 1055, 1059, 1020, 1046, 1013, 1002, 1040, 1011, 1023, 983, 992, 1024, 1038, 986, 1033, 1029, 1080, 1024, 1008, 1024, 1046, 1067, 1043, 1067, 1044, 1000, 1014, 1015, 1054, 981, 1002, 1005, 1034, 1013, 1027, 1030, 1018, 1025, 1025, 1023, 1049, 1025, 1007, 1057, 1024, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1091, 995, 1034, 991, 1020, 1031, 1061, 1033, 1045, 986, 1033, 985, 1046, 1059, 1033, 993, 997, 1037, 1040, 1042, 1035, 1055, 1040, 999, 1027, 1010, 1032, 1060, 1064, 994, 1006, 1063, 1033, 1041, 996, 1026, 982, 983, 1018, 982, 995, 993, 1016, 1013, 1037, 1026, 1023, 1042, 1024, 1018, 1041, 1035, 1021, 987, 1035, 1036, 989, 1057, 1038, 1023, 1019, 1030, 994, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1036, 1016, 1055, 981, 988, 981, 985, 1029, 1000, 1036, 973, 995, 1059, 1037, 1029, 972, 1015, 1015, 1009, 1034, 1051, 1055, 1035, 1020, 1090, 1054, 1026, 1005, 1064, 987, 1069, 1014, 1042, 982, 1005, 1045, 970, 1044, 1040, 1014, 1061, 1066, 1054, 999, 1061, 978, 995, 1021, 1046, 1094, 1070, 1050, 985, 1051, 1029, 982, 1019, 1005, 1025, 981, 993, 1046, 994, 1074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1016, 1004, 979, 1021, 1067, 1002, 1027, 1023, 1027, 1040, 1067, 1022, 1052, 1084, 1013, 1059, 1071, 987, 1002, 1035, 1028, 997, 1028, 1012, 1014, 1065, 960, 1016, 971, 1046, 1016, 1045, 1044, 1003, 1010, 1023, 1047, 1030, 1045, 986, 1049, 1022, 1002, 1028, 1051, 1033, 1043, 1000, 1058, 975, 1013, 1051, 995, 986, 1012, 1066, 1003, 1064, 1048, 1036, 964, 1007, 1010, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1032, 976, 981, 1030, 1005, 1048, 1004, 1091, 990, 1052, 1025, 1000, 1069, 982, 1027, 997, 1031, 1022, 1017, 984, 1007, 1029, 1061, 1027, 1035, 1025, 1017, 1075, 1036, 1043, 1018, 1038, 993, 1054, 1031, 1004, 1014, 998, 1050, 995, 1056, 1026, 1004, 1011, 1036, 1030, 1004, 1016, 1080, 1032, 1064, 981, 991, 1056, 1002, 1022, 991, 1007, 1039, 1050, 1060, 1024, 1029, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[995, 1037, 1022, 1059, 1060, 1009, 1000, 1011, 20, 7, 0, 0, 0, 0, 0, 0, 1024, 1004, 1021, 1021, 1021, 991, 1066, 997, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1008, 1055, 1059, 1020, 1046, 993, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1011, 1023, 983, 992, 1024, 1038, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1029, 1080, 1024, 1008, 1024, 1046, 1067, 9, 0, 0, 0, 0, 0, 0, 0, 1043, 1067, 1044, 1000, 1014, 1015, 1054, 981, 13, 7, 0, 0, 0, 0, 0, 0, 984, 1005, 1034, 1013, 1027, 1030, 1018, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1023, 1049, 1025, 1007, 1057, 1024, 1008, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8145, 8180, 8077, 8320, 8238, 8136, 8220] avg=8192.0000 max=8320 max_rank=4 min=8077 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[994, 1025, 1003, 1022, 990, 997, 1067, 1001, 21, 7, 0, 0, 0, 0, 0, 0, 1051, 994, 1052, 1021, 1028, 1010, 1067, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1020, 1026, 986, 1014, 998, 1027, 962, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1059, 1029, 1016, 1000, 1046, 1024, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1041, 1052, 1044, 982, 1054, 995, 1010, 9, 0, 0, 0, 0, 0, 0, 0, 996, 1002, 1049, 1020, 1032, 977, 1040, 977, 13, 8, 0, 0, 0, 0, 0, 0, 1087, 1047, 1050, 1069, 1012, 1043, 983, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 977, 1041, 1050, 1026, 1024, 1038, 998, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8202, 8082, 8271, 8228, 8114, 8327, 8185] avg=8192.0000 max=8327 max_rank=6 min=8082 min_rank=2 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1044, 1013, 1014, 1050, 1001, 1077, 960, 1035, 21, 7, 0, 0, 0, 0, 0, 0, 1001, 1059, 1036, 1002, 1032, 1008, 1015, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1013, 1040, 1049, 1034, 986, 1030, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1046, 1061, 1029, 1025, 1062, 1016, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 971, 982, 1068, 1025, 1022, 997, 1018, 9, 0, 0, 0, 0, 0, 0, 0, 988, 1051, 1031, 1048, 961, 1035, 1027, 1010, 13, 7, 0, 0, 0, 0, 0, 0, 1020, 1000, 1031, 1090, 1018, 993, 988, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1006, 1068, 1022, 1025, 1034, 1002, 1046, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8162, 8191, 8310, 8133, 8171, 8118, 8229] avg=8192.0000 max=8310 max_rank=3 min=8118 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1036, 1016, 1055, 981, 988, 981, 985, 1029, 21, 7, 0, 0, 0, 0, 0, 0, 1000, 1036, 973, 995, 1059, 1037, 1029, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1015, 1009, 1034, 1051, 1055, 1014, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1054, 1026, 1005, 1064, 987, 1069, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 982, 1005, 1045, 970, 1044, 1040, 1014, 9, 0, 0, 0, 0, 0, 0, 0, 1061, 1066, 1054, 999, 1061, 978, 995, 1021, 14, 7, 0, 0, 0, 0, 0, 0, 1028, 1094, 1070, 1050, 985, 1051, 1029, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1005, 1025, 981, 993, 1046, 994, 1074, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8101, 8213, 8288, 8151, 8256, 8289, 8139] avg=8192.0000 max=8289 max_rank=6 min=8099 min_rank=0 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1016, 1004, 979, 1021, 1067, 1002, 1027, 1023, 20, 6, 0, 0, 0, 0, 0, 0, 1027, 1040, 1067, 1022, 1052, 1084, 1013, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 987, 1002, 1035, 1028, 997, 1008, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1065, 960, 1016, 971, 1046, 1016, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1003, 1010, 1023, 1047, 1030, 1045, 986, 9, 0, 0, 0, 0, 0, 0, 0, 1049, 1022, 1002, 1028, 1051, 1033, 1043, 1000, 13, 7, 0, 0, 0, 0, 0, 0, 1040, 975, 1013, 1051, 995, 986, 1012, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1064, 1048, 1036, 964, 1007, 1010, 1036, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8364, 8140, 8114, 8197, 8248, 8138, 8170] avg=8192.0000 max=8364 max_rank=1 min=8114 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1021, 1029, 1030, 1026, 1034, 1043, 990, 1019, 20, 6, 0, 0, 0, 0, 0, 0, 1018, 1049, 1035, 1027, 1058, 1038, 991, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1077, 1030, 1054, 1065, 1062, 967, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 959, 1015, 1044, 1017, 961, 975, 1056, 953, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1078, 1000, 984, 1055, 1063, 969, 1061, 9, 0, 0, 0, 0, 0, 0, 0, 1022, 1023, 1021, 958, 1010, 1042, 1038, 998, 12, 7, 0, 0, 0, 0, 0, 0, 1035, 1024, 1029, 1044, 986, 1021, 937, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1020, 1038, 990, 1025, 1042, 1056, 1008, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8243, 8352, 7980, 8285, 8131, 8150, 8177] avg=8192.0000 max=8352 max_rank=2 min=7980 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1091, 995, 1034, 991, 1020, 1031, 1061, 1033, 21, 7, 0, 0, 0, 0, 0, 0, 1045, 986, 1033, 985, 1046, 1059, 1033, 993, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1037, 1040, 1042, 1035, 1055, 1019, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1010, 1032, 1060, 1064, 994, 1006, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1041, 996, 1026, 982, 983, 1018, 982, 9, 0, 0, 0, 0, 0, 0, 0, 995, 993, 1016, 1013, 1037, 1026, 1023, 1042, 13, 7, 0, 0, 0, 0, 0, 0, 1006, 1018, 1041, 1035, 1021, 987, 1035, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1057, 1038, 1023, 1019, 1030, 994, 1046, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8180, 8224, 8236, 8070, 8165, 8179, 8198] avg=8192.0000 max=8284 max_rank=0 min=8070 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32612 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1032, 976, 981, 1030, 1005, 1048, 1004, 1091, 24, 10, 0, 0, 0, 0, 0, 0, 990, 1052, 1025, 1000, 1066, 982, 1027, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1022, 1017, 984, 1007, 1029, 1037, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1025, 1017, 1075, 1036, 1043, 1018, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1054, 1031, 1004, 1014, 998, 1050, 995, 13, 0, 0, 0, 0, 0, 0, 0, 1056, 1026, 1004, 1011, 1036, 1030, 1004, 1016, 17, 13, 0, 0, 0, 0, 0, 0, 1051, 1032, 1064, 981, 991, 1056, 1002, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1007, 1039, 1050, 1060, 1024, 1029, 1012, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8139, 8154, 8260, 8152, 8213, 8199, 8218] avg=8192.0000 max=8260 max_rank=3 min=8139 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8448, 8192, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8448, 8448, 8064, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8448, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1034, 1007, 1024, 1006, 954, 985, 1023, 1018, 1002, 1040, 1101, 1022, 1027, 1007, 988, 1052, 1039, 1032, 1049, 980, 1098, 1032, 1039, 1012, 1045, 1019, 1032, 1033, 1080, 1030, 1014, 1014, 984, 1012, 988, 997, 998, 1031, 1013, 1057, 988, 1001, 1008, 1049, 1038, 1014, 1037, 1079, 1025, 1039, 1048, 1020, 992, 1053, 1034, 1058, 999, 1041, 998, 999, 1021, 1073, 967, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1047, 1023, 1023, 1031, 1033, 1012, 1082, 1083, 1081, 1051, 1008, 1038, 966, 1034, 1032, 1024, 1050, 999, 1030, 973, 1044, 1071, 1045, 1013, 996, 1019, 1033, 1012, 1006, 999, 991, 1028, 1036, 1009, 1005, 1027, 1038, 989, 1015, 998, 1061, 1006, 973, 1005, 1036, 1000, 1023, 1009, 1027, 1047, 1004, 1065, 992, 988, 1006, 1025, 1045, 1024, 1043, 1025, 1125, 1030, 1010, 973] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1083, 1012, 1050, 1008, 1006, 1011, 1022, 1040, 1048, 1041, 1047, 1004, 999, 962, 1033, 1061, 1012, 1076, 1030, 1046, 1003, 1027, 1046, 965, 966, 1007, 1057, 1040, 1027, 1039, 1034, 997, 994, 1020, 1006, 1013, 994, 1013, 1049, 1014, 1033, 1048, 1008, 1007, 1028, 998, 1013, 1014, 1060, 977, 1071, 1052, 1051, 1027, 1035, 1005, 1044, 981, 1048, 1039, 1024, 1038, 991, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1050, 1047, 1046, 1033, 1057, 998, 1007, 1007, 1033, 1016, 974, 983, 1062, 1010, 1017, 1064, 1032, 1036, 996, 1046, 1057, 1010, 992, 1068, 1052, 1079, 1049, 1028, 1059, 1017, 1063, 1006, 1016, 1018, 984, 1055, 1066, 991, 1033, 1024, 1039, 953, 1048, 1008, 1022, 1017, 1026, 981, 1020, 1002, 959, 1010, 1036, 998, 1074, 1029, 1008, 1032, 1010, 989, 996, 1059, 1026, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1029, 1027, 1005, 998, 998, 1026, 986, 1017, 1018, 999, 1010, 1062, 961, 1042, 1055, 1087, 978, 1033, 1059, 1013, 999, 1032, 1056, 1052, 1029, 1022, 1020, 993, 1049, 1068, 1020, 998, 1031, 1056, 1031, 1037, 1009, 1039, 1017, 1058, 1031, 1016, 1045, 994, 1073, 984, 1001, 1016, 1019, 1012, 1023, 1000, 1010, 1037, 1021, 1024, 1039, 968, 1041, 1086, 999, 1010, 1054, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1001, 1007, 1018, 1020, 1060, 1006, 987, 1052, 1002, 1011, 1024, 1007, 1041, 984, 1024, 1016, 1028, 994, 1036, 1062, 1100, 1006, 1030, 1037, 1031, 1033, 1049, 1015, 1115, 1011, 996, 1029, 1027, 1021, 1046, 1027, 1028, 1046, 1002, 1028, 1019, 1007, 1052, 993, 1026, 972, 1028, 1040, 1003, 1023, 1073, 991, 1044, 1005, 1021, 1038, 1002, 1007, 1036, 1020, 1027, 1006, 1019, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1037, 1074, 1127, 1002, 1020, 1029, 1038, 1022, 1027, 1045, 969, 1029, 1001, 1042, 982, 1021, 1036, 1032, 961, 962, 1054, 1019, 1017, 1037, 1036, 1017, 989, 1037, 1051, 1022, 1050, 994, 1034, 1064, 1047, 993, 1042, 1046, 1047, 1057, 1017, 1029, 1009, 984, 999, 1006, 1062, 1025, 1004, 998, 1000, 1036, 987, 1027, 1063, 1031, 1056, 970, 981, 1035, 1047, 993, 1016, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1055, 1057, 1004, 1022, 1025, 1045, 969, 1023, 1001, 988, 1013, 1017, 1036, 943, 968, 1006, 1011, 1025, 1022, 1036, 1059, 1002, 1045, 1023, 1068, 970, 1001, 1075, 1063, 1027, 994, 1078, 1041, 1021, 997, 1042, 1024, 1064, 1038, 1049, 1002, 991, 1023, 1040, 990, 1020, 1049, 996, 1016, 1003, 1036, 1053, 977, 1058, 1037, 1025, 1037, 1051, 1052, 1026, 1049, 1046, 1033, 979] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1021, 1007, 1024, 1006, 954, 985, 1023, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1040, 1101, 1022, 1027, 1007, 988, 1052, 16, 13, 5, 0, 0, 0, 0, 0, 1039, 1032, 1049, 980, 1064, 1032, 1039, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1019, 1032, 1033, 1044, 1030, 1014, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1012, 988, 997, 998, 1031, 1013, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1001, 1008, 1049, 1038, 1014, 1037, 1079, 36, 18, 0, 0, 0, 0, 0, 0, 1025, 1039, 1048, 1020, 992, 1053, 1034, 1058, 1, 0, 0, 0, 0, 0, 0, 0, 999, 1041, 998, 999, 1021, 1073, 967, 1037, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8038, 8273, 8247, 8231, 8073, 8268, 8270, 8136] avg=8192.0000 max=8273 max_rank=1 min=8038 min_rank=0 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1033, 1023, 1023, 1031, 1033, 1012, 1082, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1051, 1008, 1038, 966, 1034, 1032, 1024, 15, 14, 4, 0, 0, 0, 0, 0, 1050, 999, 1030, 973, 1012, 1071, 1045, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1019, 1033, 1012, 973, 999, 991, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1009, 1005, 1027, 1038, 989, 1015, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1006, 973, 1005, 1036, 1000, 1023, 1009, 33, 17, 0, 0, 0, 0, 0, 0, 1027, 1047, 1004, 1065, 992, 988, 1006, 1025, 1, 0, 0, 0, 0, 0, 0, 0, 1045, 1024, 1043, 1025, 1125, 1030, 1010, 973, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8267, 8193, 8051, 8112, 8163, 8155, 8275] avg=8192.0000 max=8320 max_rank=0 min=8051 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1069, 1012, 1050, 1008, 1006, 1011, 1022, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1041, 1047, 1004, 999, 962, 1033, 1061, 15, 14, 4, 0, 0, 0, 0, 0, 1012, 1076, 1030, 1046, 972, 1027, 1046, 965, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1007, 1057, 1040, 993, 1039, 1034, 997, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1020, 1006, 1013, 994, 1013, 1049, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1048, 1008, 1007, 1028, 998, 1013, 1014, 34, 16, 0, 0, 0, 0, 0, 0, 1060, 977, 1071, 1052, 1051, 1027, 1035, 1005, 1, 0, 0, 0, 0, 0, 0, 0, 1044, 981, 1048, 1039, 1024, 1038, 991, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8228, 8174, 8133, 8098, 8199, 8279, 8207] avg=8192.0000 max=8279 max_rank=6 min=8098 min_rank=4 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1036, 1047, 1046, 1033, 1057, 998, 1007, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1016, 974, 983, 1062, 1010, 1017, 1064, 15, 14, 4, 0, 0, 0, 0, 0, 1032, 1036, 996, 1046, 1025, 1010, 992, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1079, 1049, 1028, 1024, 1017, 1063, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1018, 984, 1055, 1066, 991, 1033, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 953, 1048, 1008, 1022, 1017, 1026, 981, 35, 17, 0, 0, 0, 0, 0, 0, 1020, 1002, 959, 1010, 1036, 998, 1074, 1029, 1, 0, 0, 0, 0, 0, 0, 0, 1008, 1032, 1010, 989, 996, 1059, 1026, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8192, 8205, 8318, 8182, 8146, 8129, 8133] avg=8192.0000 max=8318 max_rank=3 min=8129 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32590 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1016, 1027, 1005, 998, 998, 1026, 986, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 999, 1010, 1062, 961, 1042, 1055, 1087, 14, 13, 5, 0, 0, 0, 0, 0, 978, 1033, 1059, 1013, 969, 1032, 1056, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1022, 1020, 993, 1014, 1068, 1020, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1056, 1031, 1037, 1009, 1039, 1017, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1016, 1045, 994, 1073, 984, 1001, 1016, 35, 16, 0, 0, 0, 0, 0, 0, 1019, 1012, 1023, 1000, 1010, 1037, 1021, 1024, 1, 0, 0, 0, 0, 0, 0, 0, 1039, 968, 1041, 1086, 999, 1010, 1054, 1014, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8073, 8266, 8192, 8164, 8271, 8211, 8147, 8212] avg=8192.0000 max=8271 max_rank=4 min=8073 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[984, 1007, 1018, 1020, 1060, 1006, 987, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1011, 1024, 1007, 1041, 984, 1024, 1016, 21, 17, 8, 0, 0, 0, 0, 0, 1028, 994, 1036, 1062, 1055, 1006, 1030, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1033, 1049, 1015, 1073, 1011, 996, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1021, 1046, 1027, 1028, 1046, 1002, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1007, 1052, 993, 1026, 972, 1028, 1040, 42, 24, 0, 0, 0, 0, 0, 0, 1003, 1023, 1073, 991, 1044, 1005, 1021, 1038, 5, 0, 0, 0, 0, 0, 0, 0, 1002, 1007, 1036, 1020, 1027, 1006, 1019, 1027, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8155, 8248, 8237, 8208, 8203, 8203, 8148] avg=8192.0000 max=8248 max_rank=2 min=8134 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1041, 1057, 1004, 1022, 1025, 1045, 969, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 988, 1013, 1017, 1036, 943, 968, 1006, 15, 14, 5, 0, 0, 0, 0, 0, 1011, 1025, 1022, 1036, 1027, 1002, 1045, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 970, 1001, 1075, 1028, 1027, 994, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1021, 997, 1042, 1024, 1064, 1038, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 991, 1023, 1040, 990, 1020, 1049, 996, 35, 17, 0, 0, 0, 0, 0, 0, 1016, 1003, 1036, 1053, 977, 1058, 1037, 1025, 1, 0, 0, 0, 0, 0, 0, 0, 1037, 1051, 1052, 1026, 1049, 1046, 1033, 979, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8006, 8191, 8241, 8269, 8163, 8206, 8274] avg=8192.0000 max=8274 max_rank=7 min=8006 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1024, 1074, 1127, 1002, 1020, 1029, 1038, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1045, 969, 1029, 1001, 1042, 982, 1021, 15, 13, 5, 0, 0, 0, 0, 0, 1036, 1032, 961, 962, 1022, 1019, 1017, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1017, 989, 1037, 1016, 1022, 1050, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1064, 1047, 993, 1042, 1046, 1047, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1029, 1009, 984, 999, 1006, 1062, 1025, 35, 17, 0, 0, 0, 0, 0, 0, 1004, 998, 1000, 1036, 987, 1027, 1063, 1031, 1, 0, 0, 0, 0, 0, 0, 0, 1056, 970, 981, 1035, 1047, 993, 1016, 1052, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8336, 8149, 8086, 8161, 8323, 8183, 8147, 8151] avg=8192.0000 max=8336 max_rank=0 min=8086 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8064, 8192, 8448, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8192, 8192, 8320, 8064, 8320, 8192, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1038, 1014, 979, 1076, 1045, 1004, 1051, 1004, 984, 988, 975, 1004, 1058, 1004, 1002, 1069, 1001, 1092, 1017, 1053, 1035, 1005, 1058, 1013, 982, 1024, 997, 1039, 1000, 1027, 1015, 1025, 1067, 1027, 1034, 1036, 1014, 1015, 1033, 982, 1035, 1041, 1055, 1075, 1023, 1009, 1035, 1030, 1043, 987, 1068, 1021, 1003, 986, 1019, 1027, 1041, 996, 1000, 1059, 1034, 1013, 1054, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1042, 1057, 994, 1044, 1033, 994, 1025, 1058, 1049, 1016, 1014, 1036, 991, 1021, 996, 1054, 1008, 1028, 1018, 1029, 1039, 1029, 1002, 1028, 1017, 1017, 1047, 1047, 1011, 1040, 973, 1012, 1004, 1005, 1077, 996, 1026, 1043, 1023, 986, 993, 1059, 1027, 985, 997, 1002, 1037, 1009, 1004, 1044, 1038, 1018, 1034, 1037, 991, 1018, 1053, 1047, 1069, 1025, 1031, 1063, 1021, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[999, 1036, 996, 1023, 1057, 1027, 1007, 973, 1058, 1039, 1040, 1056, 992, 1050, 1043, 1015, 1027, 1041, 1008, 1059, 1017, 1060, 1062, 1037, 996, 1052, 964, 986, 1020, 1042, 1035, 1084, 1023, 1016, 1024, 978, 968, 1033, 1018, 1035, 1033, 1004, 997, 1045, 1053, 999, 1016, 1041, 1031, 1028, 1039, 1015, 995, 996, 1002, 976, 992, 1039, 1026, 1018, 1115, 1044, 1084, 952] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[983, 1024, 1048, 1066, 1057, 1021, 1020, 1026, 1022, 1000, 1012, 997, 1052, 1029, 990, 1080, 1009, 977, 1034, 1012, 1027, 957, 1014, 1025, 1021, 1074, 1012, 1064, 1005, 1023, 987, 1085, 1033, 1046, 1012, 996, 1028, 1001, 1028, 1021, 1056, 1054, 976, 1072, 1048, 1063, 967, 1041, 990, 1014, 960, 994, 1018, 1035, 1097, 1029, 1068, 977, 1005, 998, 1017, 1053, 1030, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[994, 972, 1005, 1012, 1027, 1050, 998, 1066, 1045, 1005, 1035, 993, 1013, 997, 1068, 1075, 1005, 1071, 1055, 1019, 1005, 1059, 1054, 1025, 1057, 1045, 1021, 1037, 1020, 1022, 999, 973, 991, 980, 1047, 1060, 1041, 1020, 972, 1013, 1021, 1036, 1009, 1007, 1047, 1051, 993, 1088, 1021, 1041, 1029, 1005, 1059, 1003, 972, 1012, 995, 1048, 1009, 1022, 1020, 1058, 1011, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1019, 1072, 997, 1011, 1010, 1033, 997, 1045, 1008, 1013, 1024, 1032, 1005, 1018, 1052, 997, 974, 972, 1018, 1030, 1015, 1059, 1068, 1030, 1022, 992, 998, 1020, 1011, 992, 1011, 1041, 996, 1054, 1077, 1036, 1027, 1026, 1033, 1057, 1050, 1005, 990, 1024, 1092, 1012, 1035, 1043, 1038, 1033, 1037, 1055, 1025, 1009, 1017, 973, 1040, 1086, 1007, 1030, 1007, 1025, 992, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1001, 1022, 1016, 1036, 970, 990, 1069, 1021, 1002, 1012, 993, 1010, 1040, 1000, 981, 1005, 1050, 989, 1035, 1058, 1041, 1026, 1046, 1013, 967, 986, 1012, 1083, 1019, 1017, 1000, 1000, 1002, 1005, 1034, 959, 1072, 1024, 1028, 1037, 1006, 1034, 1008, 1017, 1069, 1081, 1062, 1033, 1028, 1021, 1004, 993, 1036, 1037, 1039, 1029, 995, 1090, 1031, 1049, 1076, 1035, 1032, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[971, 999, 1028, 1041, 1028, 1028, 1080, 1043, 1027, 1058, 1011, 1020, 1009, 1034, 1054, 1032, 1039, 996, 1015, 1068, 1000, 1037, 986, 1005, 1014, 1062, 1008, 1042, 1005, 1041, 1040, 994, 981, 1043, 1031, 1032, 989, 1053, 1048, 1049, 989, 1045, 991, 1049, 1027, 1046, 1013, 992, 1012, 1037, 1015, 1030, 998, 1039, 1027, 1004, 1057, 1009, 1031, 1040, 990, 1053, 987, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1019, 1072, 997, 1011, 1010, 1033, 997, 1045, 7, 0, 0, 0, 0, 0, 0, 0, 1008, 1013, 1024, 1032, 1005, 1018, 1052, 997, 17, 0, 0, 0, 0, 0, 0, 0, 974, 972, 1018, 1013, 1015, 1059, 1068, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 992, 998, 1020, 1011, 992, 1011, 1041, 21, 11, 0, 0, 0, 0, 0, 0, 996, 1054, 1077, 1036, 1027, 1026, 1033, 1057, 10, 0, 0, 0, 0, 0, 0, 0, 1050, 1005, 990, 1024, 1054, 1012, 1035, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1033, 1037, 1055, 1025, 1009, 1017, 973, 36, 0, 0, 0, 0, 0, 0, 0, 1040, 1086, 1007, 1030, 1007, 978, 992, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8166, 8149, 8119, 8316, 8213, 8223, 8159] avg=8192.0000 max=8316 max_rank=4 min=8119 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32911 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1042, 1057, 994, 1044, 1033, 994, 1025, 1058, 7, 0, 0, 0, 0, 0, 0, 0, 1049, 1016, 1014, 1036, 991, 1021, 996, 1054, 15, 0, 0, 0, 0, 0, 0, 0, 1008, 1028, 1018, 1012, 1039, 1029, 1002, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1017, 1047, 1047, 1011, 1040, 973, 1012, 19, 12, 0, 0, 0, 0, 0, 0, 1004, 1005, 1077, 996, 1026, 1043, 1023, 986, 10, 0, 0, 0, 0, 0, 0, 0, 993, 1059, 1027, 985, 963, 1002, 1037, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1044, 1038, 1018, 1034, 1037, 991, 1018, 38, 0, 0, 0, 0, 0, 0, 0, 1053, 1047, 1069, 1025, 1031, 1013, 1021, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8192, 8164, 8195, 8170, 8075, 8222, 8264] avg=8192.0000 max=8264 max_rank=7 min=8075 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[999, 1036, 996, 1023, 1057, 1027, 1007, 973, 8, 0, 0, 0, 0, 0, 0, 0, 1058, 1039, 1040, 1056, 992, 1050, 1043, 1015, 16, 0, 0, 0, 0, 0, 0, 0, 1027, 1041, 1008, 1041, 1017, 1060, 1062, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1052, 964, 986, 1020, 1042, 1035, 1084, 21, 11, 0, 0, 0, 0, 0, 0, 1023, 1016, 1024, 978, 968, 1033, 1018, 1035, 10, 0, 0, 0, 0, 0, 0, 0, 1033, 1004, 997, 1045, 1016, 999, 1016, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1028, 1039, 1015, 995, 996, 1002, 976, 37, 0, 0, 0, 0, 0, 0, 0, 992, 1039, 1026, 1018, 1115, 996, 1084, 952, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8309, 8293, 8211, 8105, 8151, 8119, 8222] avg=8192.0000 max=8309 max_rank=1 min=8105 min_rank=4 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[983, 1024, 1048, 1066, 1057, 1021, 1020, 1026, 7, 0, 0, 0, 0, 0, 0, 0, 1022, 1000, 1012, 997, 1052, 1029, 990, 1080, 16, 0, 0, 0, 0, 0, 0, 0, 1009, 977, 1034, 995, 1027, 957, 1014, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1074, 1012, 1064, 1005, 1023, 987, 1085, 20, 11, 0, 0, 0, 0, 0, 0, 1033, 1046, 1012, 996, 1028, 1001, 1028, 1021, 10, 0, 0, 0, 0, 0, 0, 0, 1056, 1054, 976, 1072, 1012, 1063, 967, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1014, 960, 994, 1018, 1035, 1097, 1029, 37, 0, 0, 0, 0, 0, 0, 0, 1068, 977, 1005, 998, 1017, 1005, 1030, 1056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8198, 8038, 8302, 8175, 8241, 8174, 8156] avg=8192.0000 max=8302 max_rank=3 min=8038 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[994, 972, 1005, 1012, 1027, 1050, 998, 1066, 11, 0, 0, 0, 0, 0, 0, 0, 1045, 1005, 1035, 993, 1013, 997, 1068, 1075, 20, 0, 0, 0, 0, 0, 0, 0, 1005, 1071, 1055, 994, 1005, 1059, 1054, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1045, 1021, 1037, 1020, 1022, 999, 973, 25, 17, 0, 0, 0, 0, 0, 0, 991, 980, 1047, 1060, 1041, 1020, 972, 1013, 14, 7, 0, 0, 0, 0, 0, 0, 1021, 1036, 1009, 1007, 995, 1051, 993, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1041, 1029, 1005, 1059, 1003, 972, 1012, 43, 0, 0, 0, 0, 0, 0, 0, 995, 1048, 1009, 1022, 1020, 998, 1011, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8251, 8268, 8216, 8145, 8200, 8185, 8136] avg=8192.0000 max=8268 max_rank=2 min=8135 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1001, 1022, 1016, 1036, 970, 990, 1069, 1021, 8, 0, 0, 0, 0, 0, 0, 0, 1002, 1012, 993, 1010, 1040, 1000, 981, 1005, 16, 0, 0, 0, 0, 0, 0, 0, 1050, 989, 1035, 1040, 1041, 1026, 1046, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 967, 986, 1012, 1083, 1019, 1017, 1000, 1000, 21, 11, 0, 0, 0, 0, 0, 0, 1002, 1005, 1034, 959, 1072, 1024, 1028, 1037, 10, 0, 0, 0, 0, 0, 0, 0, 1006, 1034, 1008, 1017, 1032, 1081, 1062, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1021, 1004, 993, 1036, 1037, 1039, 1029, 37, 0, 0, 0, 0, 0, 0, 0, 995, 1090, 1031, 1049, 1076, 987, 1032, 1060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8059, 8240, 8116, 8171, 8273, 8224, 8320] avg=8192.0000 max=8320 max_rank=7 min=8059 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32988 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[971, 999, 1028, 1041, 1028, 1028, 1080, 1043, 8, 0, 0, 0, 0, 0, 0, 0, 1027, 1058, 1011, 1020, 1009, 1034, 1054, 1032, 16, 0, 0, 0, 0, 0, 0, 0, 1039, 996, 1015, 1050, 1000, 1037, 986, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1062, 1008, 1042, 1005, 1041, 1040, 994, 20, 11, 0, 0, 0, 0, 0, 0, 981, 1043, 1031, 1032, 989, 1053, 1048, 1049, 10, 0, 0, 0, 0, 0, 0, 0, 989, 1045, 991, 1049, 991, 1046, 1013, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1037, 1015, 1030, 998, 1039, 1027, 1004, 37, 0, 0, 0, 0, 0, 0, 0, 1057, 1009, 1031, 1040, 990, 1005, 987, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8261, 8128, 8237, 8236, 8116, 8199, 8133] avg=8192.0000 max=8261 max_rank=1 min=8116 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1038, 1014, 979, 1076, 1045, 1004, 1051, 1004, 8, 0, 0, 0, 0, 0, 0, 0, 984, 988, 975, 1004, 1058, 1004, 1002, 1069, 16, 0, 0, 0, 0, 0, 0, 0, 1001, 1092, 1017, 1035, 1035, 1005, 1058, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1024, 997, 1039, 1000, 1027, 1015, 1025, 20, 11, 0, 0, 0, 0, 0, 0, 1067, 1027, 1034, 1036, 1014, 1015, 1033, 982, 10, 0, 0, 0, 0, 0, 0, 0, 1035, 1041, 1055, 1075, 987, 1009, 1035, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 987, 1068, 1021, 1003, 986, 1019, 1027, 36, 0, 0, 0, 0, 0, 0, 0, 1041, 996, 1000, 1059, 1034, 966, 1054, 996, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8100, 8256, 8140, 8218, 8267, 8190, 8146] avg=8192.0000 max=8267 max_rank=5 min=8100 min_rank=1 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8320, 8064, 8320, 8064, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8448, 8192, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8448, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8064, 8320, 8192, 8320, 8064, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8064, 8320, 8320, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8064, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1013, 1059, 1075, 1030, 1008, 1026, 971, 1014, 1070, 1071, 1028, 991, 1075, 1038, 1032, 986, 1052, 1063, 1059, 1014, 1044, 1052, 989, 955, 1033, 996, 996, 1064, 1032, 1070, 990, 1038, 1023, 1000, 925, 1069, 1012, 1001, 1017, 1007, 1016, 1044, 1048, 1044, 996, 1071, 985, 970, 1017, 987, 1027, 1031, 1073, 1052, 992, 1025, 1004, 1059, 991, 1037, 1044, 990, 1030, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[995, 1033, 1031, 1039, 1010, 1041, 1042, 1006, 1019, 1015, 1035, 1047, 1051, 1001, 1037, 1031, 1049, 1038, 1037, 1003, 982, 1005, 1027, 965, 1059, 998, 986, 1034, 992, 1039, 1025, 1003, 1037, 1006, 1067, 1049, 1063, 1062, 1014, 1013, 1052, 1018, 990, 1026, 1026, 989, 1014, 992, 1007, 1027, 1022, 1032, 1067, 1011, 1000, 1023, 1000, 1058, 1077, 1026, 1053, 1018, 994, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[978, 977, 1048, 1061, 1039, 1039, 1012, 1025, 1038, 1004, 1015, 1040, 1025, 994, 1024, 1040, 1043, 1022, 1013, 1014, 1027, 1007, 1028, 1001, 1058, 1051, 1060, 1037, 1053, 995, 1017, 1031, 1012, 1002, 1008, 1014, 1010, 989, 1058, 977, 1031, 1097, 1015, 1033, 1027, 1006, 1021, 1003, 999, 996, 1069, 1006, 1060, 1016, 1035, 1015, 1020, 1015, 1066, 1023, 1031, 1042, 1012, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1023, 1059, 1054, 1008, 997, 987, 1052, 996, 1064, 1007, 1024, 1032, 1046, 1101, 1035, 987, 1002, 1001, 1036, 983, 1016, 1058, 1054, 988, 1051, 1054, 983, 1011, 1012, 1032, 1043, 1044, 1041, 1014, 1009, 1081, 1053, 1041, 1021, 1005, 1019, 1034, 1012, 1025, 1007, 1032, 1060, 1050, 1020, 1025, 1064, 1026, 1002, 1048, 1000, 1013, 1020, 990, 999, 960, 1024, 1007, 998, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1007, 1000, 1033, 1011, 1029, 1003, 1018, 1069, 1008, 1040, 1054, 1011, 1037, 1018, 1052, 1013, 1003, 1028, 1022, 1049, 1011, 1028, 1015, 1027, 1057, 1031, 1021, 1029, 1076, 1013, 1064, 1028, 1028, 960, 1032, 1034, 1001, 1044, 957, 1040, 1009, 1032, 975, 979, 977, 1012, 1015, 1071, 985, 1014, 1042, 1054, 1031, 997, 1034, 1004, 1019, 1025, 1035, 1044, 1017, 1008, 1087, 1069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1056, 991, 1054, 1052, 1055, 986, 1005, 1085, 1038, 1019, 1051, 1005, 984, 1045, 1055, 1033, 1021, 967, 1011, 1045, 1043, 1024, 977, 1029, 1012, 1024, 1038, 1023, 1006, 1042, 1044, 1041, 1017, 1078, 964, 976, 1006, 1059, 986, 1022, 1031, 1002, 1048, 1029, 1019, 1034, 1029, 1007, 1027, 1049, 1031, 1063, 1002, 1060, 1034, 1029, 1018, 1014, 1006, 988, 989, 1017, 985, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1013, 979, 995, 977, 1023, 1100, 1056, 1010, 1089, 1013, 1041, 1008, 1039, 1051, 990, 1017, 1001, 1035, 1031, 1013, 1027, 1037, 1055, 996, 1042, 1061, 1051, 1005, 1012, 997, 1022, 1005, 1004, 997, 1033, 999, 1028, 1018, 1063, 1027, 1021, 1039, 1046, 1024, 996, 1021, 1024, 1106, 1083, 1024, 1038, 1002, 981, 1042, 969, 1026, 1026, 1058, 1032, 996, 1011, 1005, 1004, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1027, 1036, 1056, 1067, 1013, 1006, 996, 1017, 1077, 1019, 1028, 995, 1056, 1018, 1007, 1006, 1090, 969, 1030, 1032, 1039, 990, 984, 984, 971, 993, 1025, 1057, 1065, 996, 1024, 1046, 957, 981, 1026, 1023, 1036, 1060, 1034, 1057, 1067, 1013, 1053, 1003, 1034, 1036, 1024, 997, 1006, 1063, 1010, 1009, 1049, 1007, 1035, 1009, 1048, 1025, 1059, 997, 994, 1034, 1032, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1013, 1059, 1071, 1030, 1008, 1026, 971, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1071, 1028, 991, 1075, 1038, 1032, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1063, 1059, 1014, 1044, 1052, 989, 955, 37, 0, 0, 0, 0, 0, 0, 0, 1000, 996, 996, 1064, 1032, 1070, 990, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1000, 925, 1069, 1012, 1001, 1017, 1007, 25, 11, 0, 0, 0, 0, 0, 0, 1016, 1044, 1048, 1044, 996, 1071, 985, 970, 1, 0, 0, 0, 0, 0, 0, 0, 1017, 987, 1020, 1031, 1073, 1052, 992, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1059, 991, 1037, 1044, 990, 1030, 1015, 8, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8243, 8265, 8186, 8090, 8175, 8197, 8188] avg=8192.0000 max=8265 max_rank=2 min=8090 min_rank=4 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1007, 1000, 1029, 1011, 1029, 1003, 1018, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1040, 1054, 1011, 1037, 1018, 1052, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1028, 1022, 1049, 1011, 1028, 1015, 1027, 35, 0, 0, 0, 0, 0, 0, 0, 1023, 1031, 1021, 1029, 1076, 1013, 1064, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 960, 1032, 1034, 1001, 1044, 957, 1040, 25, 10, 0, 0, 0, 0, 0, 0, 1009, 1032, 975, 979, 977, 1012, 1015, 1071, 1, 0, 0, 0, 0, 0, 0, 0, 985, 1014, 1035, 1054, 1031, 997, 1034, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1025, 1035, 1044, 1017, 1008, 1087, 1069, 9, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8188, 8218, 8285, 8131, 8071, 8154, 8323] avg=8192.0000 max=8323 max_rank=7 min=8071 min_rank=5 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[978, 977, 1044, 1061, 1039, 1039, 1012, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1004, 1015, 1040, 1025, 994, 1024, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1022, 1013, 1014, 1027, 1007, 1028, 1001, 36, 0, 0, 0, 0, 0, 0, 0, 1024, 1051, 1060, 1037, 1053, 995, 1017, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1002, 1008, 1014, 1010, 989, 1058, 977, 25, 10, 0, 0, 0, 0, 0, 0, 1031, 1097, 1015, 1033, 1027, 1006, 1021, 1003, 1, 0, 0, 0, 0, 0, 0, 0, 999, 996, 1062, 1006, 1060, 1016, 1035, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1015, 1066, 1023, 1031, 1042, 1012, 1012, 9, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8134, 8191, 8268, 8105, 8234, 8189, 8240] avg=8192.0000 max=8268 max_rank=3 min=8105 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[995, 1033, 1027, 1039, 1010, 1041, 1042, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1015, 1035, 1047, 1051, 1001, 1037, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1038, 1037, 1003, 982, 1005, 1027, 965, 35, 0, 0, 0, 0, 0, 0, 0, 1025, 998, 986, 1034, 992, 1039, 1025, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1006, 1067, 1049, 1063, 1062, 1014, 1013, 25, 10, 0, 0, 0, 0, 0, 0, 1052, 1018, 990, 1026, 1026, 989, 1014, 992, 1, 0, 0, 0, 0, 0, 0, 0, 1007, 1027, 1015, 1032, 1067, 1011, 1000, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1058, 1077, 1026, 1053, 1018, 994, 1028, 9, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8191, 8141, 8102, 8346, 8108, 8182, 8273] avg=8192.0000 max=8346 max_rank=4 min=8102 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1027, 1036, 1052, 1067, 1013, 1006, 996, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1019, 1028, 995, 1056, 1018, 1007, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 969, 1030, 1032, 1039, 990, 984, 984, 37, 0, 0, 0, 0, 0, 0, 0, 940, 993, 1025, 1057, 1065, 996, 1024, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 957, 981, 1026, 1023, 1036, 1060, 1034, 1057, 23, 11, 0, 0, 0, 0, 0, 0, 1067, 1013, 1053, 1003, 1034, 1036, 1024, 997, 1, 0, 0, 0, 0, 0, 0, 0, 1006, 1063, 1003, 1009, 1049, 1007, 1035, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1025, 1059, 997, 994, 1034, 1032, 1039, 8, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8158, 8155, 8146, 8208, 8228, 8181, 8246] avg=8192.0000 max=8246 max_rank=7 min=8146 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32863 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1013, 979, 991, 977, 1023, 1100, 1056, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1013, 1041, 1008, 1039, 1051, 990, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1035, 1031, 1013, 1027, 1037, 1055, 996, 38, 0, 0, 0, 0, 0, 0, 0, 1009, 1061, 1051, 1005, 1012, 997, 1022, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 997, 1033, 999, 1028, 1018, 1063, 1027, 25, 11, 0, 0, 0, 0, 0, 0, 1021, 1039, 1046, 1024, 996, 1021, 1024, 1106, 1, 0, 0, 0, 0, 0, 0, 0, 1083, 1024, 1031, 1002, 981, 1042, 969, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1058, 1032, 996, 1011, 1005, 1004, 1002, 8, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8199, 8233, 8162, 8205, 8278, 8158, 8152] avg=8192.0000 max=8278 max_rank=5 min=8149 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1023, 1059, 1050, 1008, 997, 987, 1052, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1007, 1024, 1032, 1046, 1101, 1035, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1001, 1036, 983, 1016, 1058, 1054, 988, 37, 0, 0, 0, 0, 0, 0, 0, 1017, 1054, 983, 1011, 1012, 1032, 1043, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1014, 1009, 1081, 1053, 1041, 1021, 1005, 25, 11, 0, 0, 0, 0, 0, 0, 1019, 1034, 1012, 1025, 1007, 1032, 1060, 1050, 1, 0, 0, 0, 0, 0, 0, 0, 1020, 1025, 1057, 1026, 1002, 1048, 1000, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 990, 999, 960, 1024, 1007, 998, 996, 9, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8248, 8175, 8196, 8301, 8240, 8191, 8013] avg=8192.0000 max=8301 max_rank=4 min=8013 min_rank=7 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1056, 991, 1045, 1052, 1055, 986, 1005, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1019, 1051, 1005, 984, 1045, 1055, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 967, 1011, 1045, 1043, 1024, 977, 1029, 41, 0, 0, 0, 0, 0, 0, 0, 973, 1024, 1038, 1023, 1006, 1042, 1044, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1078, 964, 976, 1006, 1059, 986, 1022, 28, 14, 0, 0, 0, 0, 0, 0, 1031, 1002, 1048, 1029, 1019, 1034, 1029, 1007, 3, 0, 0, 0, 0, 0, 0, 0, 1027, 1049, 1020, 1063, 1002, 1060, 1034, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1014, 1006, 988, 989, 1017, 985, 1056, 11, 11, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8175, 8158, 8191, 8150, 8202, 8284, 8101] avg=8192.0000 max=8284 max_rank=6 min=8101 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8064, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8064, 8320, 8320, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[996, 1033, 994, 989, 1015, 1031, 1026, 935, 1017, 1053, 1045, 1032, 994, 1076, 1107, 1014, 1056, 1012, 983, 1036, 989, 992, 1066, 1011, 1041, 1054, 1038, 1031, 972, 1065, 973, 1009, 969, 1044, 994, 1060, 981, 1020, 1010, 1046, 1073, 1004, 1008, 1033, 985, 967, 1065, 1035, 1009, 1056, 1042, 1034, 1025, 1059, 1026, 1069, 1019, 1025, 1042, 1079, 999, 1027, 1043, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1037, 1034, 1069, 1009, 1038, 989, 1038, 1024, 1076, 1031, 1014, 1020, 988, 1016, 1039, 1064, 1051, 1020, 1055, 1024, 1000, 989, 1008, 1030, 1036, 992, 959, 1012, 1025, 1018, 1009, 996, 989, 1012, 1043, 989, 1025, 1061, 1026, 1030, 997, 1058, 1036, 1021, 1036, 1013, 1002, 1060, 1042, 1042, 1004, 1064, 1025, 1031, 1017, 1004, 1040, 1018, 1033, 1023, 1012, 1060, 1025, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1046, 993, 1075, 1023, 1037, 1023, 1036, 991, 1002, 1037, 1028, 966, 1024, 1033, 991, 1045, 1057, 1024, 984, 1051, 1028, 1079, 1039, 1052, 1023, 1065, 1056, 1009, 1013, 998, 1032, 1091, 1026, 998, 1030, 997, 976, 1019, 1001, 992, 1021, 1023, 1064, 1040, 1042, 984, 1057, 996, 1030, 1022, 1015, 1034, 1019, 1030, 1044, 1046, 992, 1061, 1006, 991, 979, 1031, 993, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[991, 1020, 994, 1052, 986, 1030, 1029, 1056, 1051, 997, 971, 969, 1021, 1049, 1057, 1037, 1032, 1009, 1002, 1034, 1041, 1042, 1024, 1032, 1021, 1045, 965, 1030, 1056, 1046, 1074, 1032, 1000, 961, 1040, 969, 1065, 1047, 1034, 1011, 1024, 1055, 1026, 985, 1054, 1008, 1008, 1005, 1050, 1004, 1043, 1011, 998, 1029, 1072, 1041, 1040, 1019, 999, 1034, 1012, 1040, 1027, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[992, 1045, 1022, 1029, 1000, 1029, 1038, 1022, 1055, 1037, 1057, 1029, 1022, 977, 1073, 1014, 977, 1012, 1023, 1035, 1043, 1060, 1103, 1048, 1026, 1039, 1019, 1031, 1032, 1049, 1044, 1032, 1018, 961, 1004, 1029, 1052, 1000, 1016, 989, 958, 1015, 1029, 1005, 1020, 995, 1060, 1020, 1041, 996, 1029, 988, 989, 1053, 1032, 1025, 1043, 1057, 1053, 1049, 1027, 1014, 987, 968] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1007, 1014, 1002, 1029, 1037, 939, 1008, 1036, 975, 1050, 1013, 993, 992, 1007, 1035, 1027, 1025, 980, 1076, 981, 1039, 1048, 1029, 994, 1046, 1048, 1037, 1025, 1034, 987, 1012, 1020, 1041, 1062, 1020, 1022, 1032, 1034, 1027, 1024, 1052, 1010, 1023, 1014, 1002, 1083, 1028, 1027, 1028, 1069, 1030, 1006, 964, 1011, 1047, 1037, 1042, 1060, 1029, 1062, 1087, 1040, 1024, 954] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[998, 1013, 1108, 1005, 1029, 990, 1007, 1008, 1062, 1038, 1019, 1023, 1020, 1051, 1001, 1010, 986, 1088, 1025, 1018, 1024, 1023, 1036, 992, 1034, 1060, 1010, 1049, 1055, 1064, 1023, 1021, 992, 1015, 1050, 1000, 1055, 1046, 1032, 1024, 1003, 1028, 1099, 994, 989, 1046, 995, 969, 977, 990, 1048, 1052, 1033, 1038, 1023, 1055, 966, 996, 1005, 1034, 979, 1017, 1049, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1009, 1024, 1070, 1050, 1058, 1015, 1050, 1072, 1038, 1031, 1019, 1057, 1032, 1027, 975, 1032, 1012, 1018, 1000, 1062, 1046, 1004, 1040, 1054, 1019, 1033, 1036, 1059, 1009, 985, 1034, 1040, 995, 1019, 1039, 1028, 1011, 1010, 1004, 1036, 998, 1016, 1041, 991, 1050, 1014, 1033, 988, 1007, 1045, 1000, 1010, 950, 1017, 1023, 1045, 1027, 987, 988, 1072, 1028, 1023, 996, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[991, 1020, 994, 1052, 986, 1030, 1029, 1056, 12, 5, 0, 0, 0, 0, 0, 0, 1051, 997, 971, 969, 1021, 1049, 1043, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1009, 1002, 1034, 1041, 1042, 999, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1013, 965, 1030, 1056, 1046, 1074, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 961, 1040, 969, 1065, 1047, 1034, 1011, 32, 14, 0, 0, 0, 0, 0, 0, 1024, 1055, 1026, 985, 1054, 1008, 1008, 1005, 11, 7, 0, 0, 0, 0, 0, 0, 1050, 1004, 1043, 1011, 998, 1029, 1072, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1019, 999, 1034, 1012, 1040, 1027, 1030, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8138, 8191, 8237, 8173, 8183, 8229, 8210] avg=8192.0000 max=8237 max_rank=3 min=8138 min_rank=1 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1037, 1034, 1069, 1009, 1038, 989, 1038, 1024, 11, 5, 0, 0, 0, 0, 0, 0, 1076, 1031, 1014, 1020, 988, 1016, 1025, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1020, 1055, 1024, 1000, 989, 983, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 961, 959, 1012, 1025, 1018, 1009, 996, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1012, 1043, 989, 1025, 1061, 1026, 1030, 31, 14, 0, 0, 0, 0, 0, 0, 997, 1058, 1036, 1021, 1036, 1013, 1002, 1060, 11, 7, 0, 0, 0, 0, 0, 0, 1042, 1042, 1004, 1064, 1025, 1031, 1017, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1018, 1033, 1023, 1012, 1060, 1025, 988, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8234, 8152, 8016, 8220, 8241, 8211, 8208] avg=8192.0000 max=8254 max_rank=0 min=8016 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[996, 1033, 994, 989, 1015, 1031, 1026, 935, 12, 5, 0, 0, 0, 0, 0, 0, 1017, 1053, 1045, 1032, 994, 1076, 1092, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1012, 983, 1036, 989, 992, 1039, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1021, 1038, 1031, 972, 1065, 973, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1044, 994, 1060, 981, 1020, 1010, 1046, 33, 15, 0, 0, 0, 0, 0, 0, 1073, 1004, 1008, 1033, 985, 967, 1065, 1035, 12, 7, 0, 0, 0, 0, 0, 0, 1009, 1056, 1042, 1034, 1025, 1059, 1026, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1025, 1042, 1079, 999, 1027, 1043, 1003, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8036, 8323, 8118, 8150, 8172, 8189, 8301, 8247] avg=8192.0000 max=8323 max_rank=1 min=8036 min_rank=0 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[992, 1045, 1022, 1029, 1000, 1029, 1038, 1022, 12, 5, 0, 0, 0, 0, 0, 0, 1055, 1037, 1057, 1029, 1022, 977, 1059, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1012, 1023, 1035, 1043, 1060, 1075, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1007, 1019, 1031, 1032, 1049, 1044, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 961, 1004, 1029, 1052, 1000, 1016, 989, 32, 16, 0, 0, 0, 0, 0, 0, 958, 1015, 1029, 1005, 1020, 995, 1060, 1020, 12, 7, 0, 0, 0, 0, 0, 0, 1041, 996, 1029, 988, 989, 1053, 1032, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1057, 1053, 1049, 1027, 1014, 987, 968, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8250, 8273, 8240, 8117, 8121, 8134, 8207] avg=8192.0000 max=8273 max_rank=2 min=8117 min_rank=4 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32579 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1046, 993, 1075, 1023, 1037, 1023, 1036, 991, 15, 9, 0, 0, 0, 0, 0, 0, 1002, 1037, 1028, 966, 1024, 1033, 970, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1024, 984, 1051, 1028, 1079, 1005, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1028, 1056, 1009, 1013, 998, 1032, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 998, 1030, 997, 976, 1019, 1001, 992, 37, 18, 0, 0, 0, 0, 0, 0, 1021, 1023, 1064, 1040, 1042, 984, 1057, 996, 16, 12, 0, 0, 0, 0, 0, 0, 1030, 1022, 1015, 1034, 1019, 1030, 1044, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1061, 1006, 991, 979, 1031, 993, 1026, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8105, 8280, 8250, 8094, 8255, 8213, 8091] avg=8192.0000 max=8280 max_rank=2 min=8091 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32653 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[998, 1013, 1108, 1005, 1029, 990, 1007, 1008, 12, 5, 0, 0, 0, 0, 0, 0, 1062, 1038, 1019, 1023, 1020, 1051, 987, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1088, 1025, 1018, 1024, 1023, 1010, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1027, 1010, 1049, 1055, 1064, 1023, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1015, 1050, 1000, 1055, 1046, 1032, 1024, 33, 15, 0, 0, 0, 0, 0, 0, 1003, 1028, 1099, 994, 989, 1046, 995, 969, 11, 7, 0, 0, 0, 0, 0, 0, 977, 990, 1048, 1052, 1033, 1038, 1023, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 966, 996, 1005, 1034, 979, 1017, 1049, 1047, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8210, 8166, 8283, 8262, 8141, 8197, 8102] avg=8192.0000 max=8283 max_rank=3 min=8102 min_rank=7 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1007, 1014, 1002, 1029, 1037, 939, 1008, 1036, 12, 5, 0, 0, 0, 0, 0, 0, 975, 1050, 1013, 993, 992, 1007, 1021, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 980, 1076, 981, 1039, 1048, 1003, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1016, 1037, 1025, 1034, 987, 1012, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1062, 1020, 1022, 1032, 1034, 1027, 1024, 32, 15, 0, 0, 0, 0, 0, 0, 1052, 1010, 1023, 1014, 1002, 1083, 1028, 1027, 11, 7, 0, 0, 0, 0, 0, 0, 1028, 1069, 1030, 1006, 964, 1011, 1047, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1060, 1029, 1062, 1087, 1040, 1024, 954, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 8078, 8146, 8177, 8309, 8257, 8173, 8307] avg=8192.0000 max=8309 max_rank=4 min=8078 min_rank=1 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=33046 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1009, 1024, 1070, 1050, 1058, 1015, 1050, 1072, 12, 5, 0, 0, 0, 0, 0, 0, 1038, 1031, 1019, 1057, 1032, 1027, 962, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1018, 1000, 1062, 1046, 1004, 1014, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1001, 1036, 1059, 1009, 985, 1034, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1019, 1039, 1028, 1011, 1010, 1004, 1036, 32, 15, 0, 0, 0, 0, 0, 0, 998, 1016, 1041, 991, 1050, 1014, 1033, 988, 11, 7, 0, 0, 0, 0, 0, 0, 1007, 1045, 1000, 1010, 950, 1017, 1023, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 987, 988, 1072, 1028, 1023, 996, 1035, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8365, 8198, 8210, 8183, 8189, 8149, 8078, 8164] avg=8192.0000 max=8365 max_rank=0 min=8078 min_rank=6 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8064, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8192, 8320, 8192, 8320, 8320, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8448, 8192, 8320, 8064, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8448, 8192, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8448, 8192, 8320, 8192, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1098, 1038, 1016, 1020, 1062, 1016, 1027, 1038, 1029, 1053, 1051, 991, 1019, 1007, 1034, 1032, 1005, 1021, 1052, 991, 1018, 1019, 1039, 1001, 1023, 999, 1032, 1026, 1025, 1020, 1022, 1040, 1061, 1011, 1007, 986, 1061, 1029, 998, 1058, 1013, 1024, 1019, 998, 980, 1017, 991, 1007, 1020, 1024, 1024, 1023, 947, 1028, 1073, 1020, 1037, 1004, 1041, 1060, 1066, 996, 1063, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1045, 1072, 1026, 1017, 1001, 1033, 1102, 1056, 1039, 983, 1110, 1010, 1023, 1014, 1024, 1043, 1007, 1041, 1023, 1022, 1018, 949, 1011, 1022, 1066, 988, 1000, 1003, 1030, 1026, 986, 1047, 996, 1077, 1015, 1036, 1028, 1025, 968, 1065, 979, 1007, 1032, 1031, 1025, 1019, 1003, 981, 1007, 1038, 1018, 975, 1021, 1045, 1052, 1006, 1035, 987, 1020, 1035, 1032, 1042, 1038, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[994, 1028, 1028, 1000, 1009, 976, 1046, 1042, 1013, 999, 1046, 1053, 1025, 1023, 1027, 998, 1012, 1010, 1061, 1065, 1063, 979, 1047, 1058, 960, 1032, 1009, 1068, 1020, 1010, 978, 1049, 1008, 1053, 1037, 1004, 1064, 982, 1020, 997, 1032, 1014, 1038, 1033, 1087, 1044, 1023, 1047, 989, 1052, 1013, 1050, 1080, 1029, 1002, 1053, 961, 992, 1008, 999, 1043, 992, 1048, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1029, 1039, 1015, 1037, 1045, 999, 1033, 1031, 1011, 1044, 1037, 1057, 1019, 1026, 1012, 1045, 1015, 1028, 1027, 1014, 1111, 1028, 1026, 967, 1011, 1061, 1009, 1002, 1019, 1055, 995, 1019, 1044, 987, 1038, 1052, 1026, 1011, 966, 1041, 993, 1023, 1028, 1039, 1046, 1012, 1028, 992, 1055, 1022, 1017, 1057, 1025, 1000, 1072, 1029, 995, 1006, 1002, 998, 1038, 1034, 1016, 978] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1010, 1018, 1015, 1055, 1054, 1026, 1026, 969, 1021, 1002, 1005, 1012, 1030, 1052, 1045, 1081, 1022, 1062, 997, 1039, 1010, 1036, 1060, 999, 1070, 1048, 1060, 991, 1030, 1029, 1078, 1038, 1086, 1014, 965, 1066, 1052, 976, 994, 982, 1025, 1039, 1023, 1029, 1053, 1025, 998, 1023, 970, 1044, 1050, 1008, 993, 1056, 1002, 995, 999, 1008, 989, 986, 1031, 1030, 1032, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1026, 1065, 1028, 1023, 1026, 998, 1005, 975, 959, 1054, 1014, 1043, 1020, 1018, 1028, 997, 1026, 1017, 1048, 1008, 1016, 1015, 1029, 1000, 1046, 1017, 1012, 986, 1034, 1045, 1018, 1042, 1028, 1001, 1026, 1025, 1050, 999, 1001, 1030, 1052, 1067, 1085, 1035, 1036, 971, 982, 1078, 1037, 1056, 1042, 1070, 1076, 1030, 1039, 1028, 960, 1003, 1017, 1038, 1007, 994, 1019, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[994, 1032, 1033, 1008, 1012, 1060, 1030, 1039, 1021, 986, 1054, 1062, 1024, 1041, 994, 1068, 964, 1022, 946, 1086, 1006, 1024, 1005, 1023, 984, 980, 989, 1048, 1028, 1040, 988, 992, 1046, 1010, 1054, 1025, 1025, 1032, 991, 1061, 1026, 997, 1029, 1049, 1075, 1046, 1035, 1075, 1047, 1005, 978, 1023, 1024, 1024, 1083, 975, 972, 1053, 1019, 1049, 990, 1017, 1055, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1015, 1032, 1014, 1063, 989, 969, 991, 1042, 1035, 1031, 979, 1023, 1016, 1000, 1088, 1013, 1041, 1045, 991, 1025, 1000, 1029, 1019, 996, 1033, 1045, 1058, 1074, 1064, 984, 992, 1053, 1066, 1027, 1006, 1002, 1034, 1005, 981, 1078, 1022, 1028, 1072, 1054, 1052, 993, 1024, 1071, 1004, 1068, 1027, 1015, 999, 990, 1049, 979, 989, 1004, 1041, 991, 1050, 1034, 1007, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1098, 1023, 1016, 1020, 1062, 1016, 1027, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1053, 1029, 991, 1019, 1007, 1034, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1021, 1052, 991, 1018, 1019, 1039, 1001, 15, 7, 0, 0, 0, 0, 0, 0, 1023, 999, 1032, 1026, 1025, 1020, 1022, 1040, 1, 0, 0, 0, 0, 0, 0, 0, 1061, 1011, 1007, 986, 1061, 1029, 998, 1058, 6, 0, 0, 0, 0, 0, 0, 0, 1013, 1024, 1019, 998, 953, 1017, 991, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1024, 1024, 1023, 947, 1028, 1059, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1004, 1041, 1060, 1066, 996, 1063, 986, 27, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8194, 8168, 8188, 8217, 8022, 8145, 8302] avg=8192.0000 max=8302 max_rank=7 min=8022 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1045, 1057, 1026, 1017, 1001, 1033, 1102, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 983, 1087, 1010, 1023, 1014, 1024, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1041, 1023, 1022, 1018, 949, 1011, 1022, 15, 7, 0, 0, 0, 0, 0, 0, 1066, 988, 1000, 1003, 1030, 1026, 986, 1047, 1, 0, 0, 0, 0, 0, 0, 0, 996, 1077, 1015, 1036, 1028, 1025, 968, 1065, 5, 0, 0, 0, 0, 0, 0, 0, 979, 1007, 1032, 1031, 996, 1019, 1003, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1038, 1018, 975, 1021, 1045, 1039, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 987, 1020, 1035, 1032, 1042, 1038, 1061, 29, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8337, 8223, 8115, 8147, 8215, 8048, 8149, 8302] avg=8192.0000 max=8337 max_rank=0 min=8048 min_rank=5 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1010, 1004, 1015, 1055, 1054, 1026, 1026, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1002, 984, 1012, 1030, 1052, 1045, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1062, 997, 1039, 1010, 1036, 1060, 999, 14, 7, 0, 0, 0, 0, 0, 0, 1070, 1048, 1060, 991, 1030, 1029, 1078, 1038, 1, 0, 0, 0, 0, 0, 0, 0, 1086, 1014, 965, 1066, 1052, 976, 994, 982, 5, 0, 0, 0, 0, 0, 0, 0, 1025, 1039, 1023, 1029, 1024, 1025, 998, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1044, 1050, 1008, 993, 1056, 989, 995, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1008, 989, 986, 1031, 1030, 1032, 1003, 29, 21, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8227, 8246, 8345, 8140, 8186, 8105, 8128] avg=8192.0000 max=8345 max_rank=3 min=8105 min_rank=6 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 1050, 1028, 1023, 1026, 998, 1005, 975, 0, 0, 0, 0, 0, 0, 0, 0, 959, 1054, 993, 1043, 1020, 1018, 1028, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1017, 1048, 1008, 1016, 1015, 1029, 1000, 15, 7, 0, 0, 0, 0, 0, 0, 1046, 1017, 1012, 986, 1034, 1045, 1018, 1042, 1, 0, 0, 0, 0, 0, 0, 0, 1028, 1001, 1026, 1025, 1050, 999, 1001, 1030, 5, 0, 0, 0, 0, 0, 0, 0, 1052, 1067, 1085, 1035, 1007, 971, 982, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1056, 1042, 1070, 1076, 1030, 1026, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 960, 1003, 1017, 1038, 1007, 994, 1019, 1016, 29, 21, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8112, 8181, 8201, 8165, 8277, 8365, 8104] avg=8192.0000 max=8365 max_rank=6 min=8104 min_rank=7 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32911 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1029, 1024, 1015, 1037, 1045, 999, 1033, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1044, 1015, 1057, 1019, 1026, 1012, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1028, 1027, 1014, 1111, 1028, 1026, 967, 15, 7, 0, 0, 0, 0, 0, 0, 1011, 1061, 1009, 1002, 1019, 1055, 995, 1019, 1, 0, 0, 0, 0, 0, 0, 0, 1044, 987, 1038, 1052, 1026, 1011, 966, 1041, 6, 0, 0, 0, 0, 0, 0, 0, 993, 1023, 1028, 1039, 1017, 1012, 1028, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1022, 1017, 1057, 1025, 1000, 1058, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1006, 1002, 998, 1038, 1034, 1016, 978, 29, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8229, 8238, 8172, 8171, 8132, 8263, 8118] avg=8192.0000 max=8263 max_rank=6 min=8118 min_rank=7 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[994, 1014, 1028, 1000, 1009, 976, 1046, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 999, 1024, 1053, 1025, 1023, 1027, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1010, 1061, 1065, 1063, 979, 1047, 1058, 14, 7, 0, 0, 0, 0, 0, 0, 960, 1032, 1009, 1068, 1020, 1010, 978, 1049, 1, 0, 0, 0, 0, 0, 0, 0, 1008, 1053, 1037, 1004, 1064, 982, 1020, 997, 5, 0, 0, 0, 0, 0, 0, 0, 1032, 1014, 1038, 1033, 1057, 1044, 1023, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1052, 1013, 1050, 1080, 1029, 989, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 961, 992, 1008, 999, 1043, 992, 1048, 1014, 30, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8162, 8316, 8127, 8170, 8288, 8255, 8109] avg=8192.0000 max=8316 max_rank=2 min=8109 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[994, 1011, 1033, 1008, 1012, 1060, 1030, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 986, 1028, 1062, 1024, 1041, 994, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 964, 1022, 946, 1086, 1006, 1024, 1005, 1023, 18, 10, 0, 0, 0, 0, 0, 0, 984, 980, 989, 1048, 1028, 1040, 988, 992, 3, 0, 0, 0, 0, 0, 0, 0, 1046, 1010, 1054, 1025, 1025, 1032, 991, 1061, 10, 0, 0, 0, 0, 0, 0, 0, 1026, 997, 1029, 1049, 1039, 1046, 1035, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1005, 978, 1023, 1024, 1024, 1060, 975, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1053, 1019, 1049, 990, 1017, 1055, 1063, 36, 26, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8224, 8104, 8052, 8254, 8296, 8136, 8283] avg=8192.0000 max=8296 max_rank=5 min=8052 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32969 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1015, 1017, 1014, 1063, 989, 969, 991, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1031, 959, 1023, 1016, 1000, 1088, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1045, 991, 1025, 1000, 1029, 1019, 996, 15, 7, 0, 0, 0, 0, 0, 0, 1033, 1045, 1058, 1074, 1064, 984, 992, 1053, 1, 0, 0, 0, 0, 0, 0, 0, 1066, 1027, 1006, 1002, 1034, 1005, 981, 1078, 5, 0, 0, 0, 0, 0, 0, 0, 1022, 1028, 1072, 1054, 1023, 993, 1024, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1068, 1027, 1015, 999, 990, 1036, 979, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1004, 1041, 991, 1050, 1034, 1007, 1025, 29, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8165, 8168, 8304, 8204, 8287, 8118, 8190] avg=8192.0000 max=8304 max_rank=3 min=8100 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8064, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8448, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8064, 8192, 8192, 8320, 8192, 8320, 8192, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8448, 8192, 8192, 8320, 8448, 8064, 7936, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1072, 1016, 1007, 1023, 1033, 1023, 1011, 998, 1007, 1074, 1012, 1018, 961, 1049, 974, 1001, 1076, 1012, 1051, 1032, 1035, 1020, 1051, 1033, 1044, 1000, 1029, 1031, 1050, 1026, 1086, 1024, 1015, 1079, 1063, 996, 1036, 990, 1033, 1014, 1017, 1000, 1054, 997, 1037, 992, 1016, 1004, 997, 1054, 998, 995, 1043, 1041, 1001, 1021, 990, 1073, 1032, 1023, 1042, 1038, 971, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1035, 1054, 1059, 1044, 1081, 1014, 1055, 1061, 1008, 1062, 986, 982, 1034, 1041, 993, 979, 1067, 1054, 1060, 996, 1010, 1001, 1026, 1076, 979, 1005, 1021, 1049, 1041, 996, 1067, 1017, 1003, 984, 1047, 1074, 1029, 1033, 1064, 1015, 1018, 988, 1016, 985, 1004, 1051, 1055, 1052, 997, 1001, 1016, 1010, 1033, 1014, 1018, 1038, 1027, 1017, 1003, 968, 986, 1030, 978, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1017, 976, 1044, 1028, 1004, 1026, 989, 1022, 1063, 1039, 975, 1037, 1011, 1041, 1000, 1026, 1023, 996, 995, 997, 1018, 1036, 1051, 1040, 1028, 1043, 1058, 1037, 1055, 1035, 1027, 961, 1060, 1029, 973, 1028, 1021, 996, 1026, 1033, 1033, 976, 1029, 1015, 1010, 1024, 1061, 1038, 1015, 1069, 1031, 1023, 1014, 1014, 1055, 1055, 1043, 1001, 1028, 977, 1044, 1044, 1001, 1072] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1009, 1003, 1011, 978, 1016, 994, 1040, 1089, 1026, 989, 1008, 1048, 1034, 980, 1005, 1026, 983, 1028, 1032, 1076, 1057, 1036, 1014, 1006, 1034, 998, 1064, 1019, 1003, 996, 1059, 1046, 1048, 1031, 1027, 1039, 1069, 1033, 1035, 1029, 1007, 1016, 1006, 1022, 1041, 1018, 976, 997, 1092, 976, 1056, 1028, 1018, 1013, 1027, 1031, 1059, 1015, 993, 1032, 1003, 1061, 998, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1010, 1032, 1001, 1002, 1020, 1051, 1019, 964, 1025, 1037, 1018, 1015, 1063, 1077, 1041, 1039, 989, 955, 1029, 1044, 1049, 964, 1034, 999, 1040, 1028, 992, 1010, 1018, 1047, 953, 995, 1061, 968, 1026, 1029, 1058, 1029, 1063, 1001, 1009, 1066, 1071, 1049, 982, 983, 1016, 1015, 1055, 1069, 1030, 1029, 1024, 1035, 1046, 1008, 1029, 1030, 974, 1033, 1078, 1037, 1031, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1030, 1007, 1024, 1043, 1038, 995, 1040, 1031, 1017, 1056, 977, 1054, 1040, 1042, 981, 1027, 1057, 1014, 1010, 1003, 1058, 1038, 1044, 1047, 1024, 1063, 989, 1015, 999, 1067, 1074, 1003, 1028, 1034, 1007, 1022, 1025, 999, 1007, 1011, 1038, 997, 1047, 1033, 1002, 1007, 1088, 1055, 1000, 1015, 1039, 1003, 1006, 1012, 1120, 1022, 1028, 966, 1063, 1049, 992, 941, 980, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1030, 1051, 1011, 1029, 1015, 1052, 980, 997, 1040, 1021, 1035, 985, 1001, 1047, 988, 1006, 1066, 1023, 1008, 1012, 1002, 1067, 1013, 1065, 1008, 1004, 1066, 1030, 992, 1039, 1022, 1031, 1068, 1047, 1029, 953, 1053, 981, 1001, 1053, 1032, 994, 1038, 1030, 1011, 1035, 1052, 988, 1032, 1066, 1051, 994, 1052, 1039, 996, 1022, 1024, 1036, 1039, 1038, 976, 1055, 1005, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[978, 956, 1037, 1026, 1048, 1000, 1045, 1001, 1055, 1063, 987, 1010, 980, 1013, 1035, 1022, 1004, 1014, 1005, 1068, 1046, 1018, 1009, 1014, 1014, 998, 1060, 1063, 1023, 1047, 1068, 1040, 1045, 1007, 982, 1023, 1021, 1010, 1012, 1109, 1007, 998, 1007, 1045, 995, 980, 998, 1047, 1015, 1063, 990, 1043, 1058, 1017, 1024, 1033, 973, 1064, 1022, 1050, 1076, 1040, 965, 1070] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1072, 1016, 1007, 1023, 1033, 1023, 1011, 998, 17, 0, 0, 0, 0, 0, 0, 0, 1007, 1074, 1012, 1018, 961, 1049, 974, 1001, 27, 3, 0, 0, 0, 0, 0, 0, 1076, 1012, 1051, 1032, 1035, 1020, 1051, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1000, 1029, 1031, 1050, 1026, 1060, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1079, 1063, 996, 1036, 990, 1033, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1000, 1054, 997, 1037, 992, 1016, 1004, 33, 0, 0, 0, 0, 0, 0, 0, 997, 1021, 998, 995, 1043, 1041, 1001, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1073, 1032, 1023, 1042, 1038, 971, 995, 23, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8126, 8283, 8264, 8205, 8150, 8117, 8191] avg=8192.0000 max=8283 max_rank=2 min=8117 min_rank=6 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32663 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1010, 1032, 1001, 1002, 1020, 1051, 1019, 964, 17, 0, 0, 0, 0, 0, 0, 0, 1025, 1037, 1018, 1015, 1063, 1077, 1041, 1039, 26, 3, 0, 0, 0, 0, 0, 0, 989, 955, 1029, 1044, 1049, 964, 1034, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1028, 992, 1010, 1018, 1047, 930, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 968, 1026, 1029, 1058, 1029, 1063, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1066, 1071, 1049, 982, 983, 1016, 1015, 34, 0, 0, 0, 0, 0, 0, 0, 1055, 1035, 1030, 1029, 1024, 1035, 1046, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1030, 974, 1033, 1078, 1037, 1031, 1042, 20, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8344, 8037, 8060, 8214, 8225, 8262, 8278] avg=8192.0000 max=8344 max_rank=1 min=8037 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32979 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1030, 1051, 1011, 1029, 1015, 1052, 980, 997, 18, 0, 0, 0, 0, 0, 0, 0, 1040, 1021, 1035, 985, 1001, 1047, 988, 1006, 28, 3, 0, 0, 0, 0, 0, 0, 1066, 1023, 1008, 1012, 1002, 1067, 1013, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1004, 1066, 1030, 992, 1039, 997, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1047, 1029, 953, 1053, 981, 1001, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 994, 1038, 1030, 1011, 1035, 1052, 988, 34, 0, 0, 0, 0, 0, 0, 0, 1032, 1032, 1051, 994, 1052, 1039, 996, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1036, 1039, 1038, 976, 1055, 1005, 1010, 22, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8154, 8228, 8167, 8163, 8214, 8218, 8209] avg=8192.0000 max=8228 max_rank=2 min=8154 min_rank=1 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32804 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1017, 976, 1044, 1028, 1004, 1026, 989, 1022, 17, 0, 0, 0, 0, 0, 0, 0, 1063, 1039, 975, 1037, 1011, 1041, 1000, 1026, 27, 3, 0, 0, 0, 0, 0, 0, 1023, 996, 995, 997, 1018, 1036, 1051, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1043, 1058, 1037, 1055, 1035, 1002, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1029, 973, 1028, 1021, 996, 1026, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 976, 1029, 1015, 1010, 1024, 1061, 1038, 34, 0, 0, 0, 0, 0, 0, 0, 1015, 1035, 1031, 1023, 1014, 1014, 1055, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1001, 1028, 977, 1044, 1044, 1001, 1072, 22, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8222, 8129, 8219, 8145, 8220, 8242, 8236] avg=8192.0000 max=8242 max_rank=6 min=8123 min_rank=0 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32843 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[978, 956, 1037, 1026, 1048, 1000, 1045, 1001, 17, 0, 0, 0, 0, 0, 0, 0, 1055, 1063, 987, 1010, 980, 1013, 1035, 1022, 26, 3, 0, 0, 0, 0, 0, 0, 1004, 1014, 1005, 1068, 1046, 1018, 1009, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 998, 1060, 1063, 1023, 1047, 1042, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1007, 982, 1023, 1021, 1010, 1012, 1109, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 998, 1007, 1045, 995, 980, 998, 1047, 34, 0, 0, 0, 0, 0, 0, 0, 1015, 1029, 990, 1043, 1058, 1017, 1024, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1064, 1022, 1050, 1076, 1040, 965, 1070, 23, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8194, 8152, 8287, 8188, 8111, 8209, 8287] avg=8192.0000 max=8287 max_rank=3 min=8108 min_rank=0 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32795 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1030, 1007, 1024, 1043, 1038, 995, 1040, 1031, 17, 0, 0, 0, 0, 0, 0, 0, 1017, 1056, 977, 1054, 1040, 1042, 981, 1027, 27, 3, 0, 0, 0, 0, 0, 0, 1057, 1014, 1010, 1003, 1058, 1038, 1044, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1063, 989, 1015, 999, 1067, 1048, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1034, 1007, 1022, 1025, 999, 1007, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 997, 1047, 1033, 1002, 1007, 1088, 1055, 32, 0, 0, 0, 0, 0, 0, 0, 1000, 983, 1039, 1003, 1006, 1012, 1120, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 966, 1063, 1049, 992, 941, 980, 993, 23, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8224, 8244, 8208, 8112, 8299, 8185, 8039] avg=8192.0000 max=8299 max_rank=5 min=8039 min_rank=7 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32635 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1009, 1003, 1011, 978, 1016, 994, 1040, 1089, 17, 0, 0, 0, 0, 0, 0, 0, 1026, 989, 1008, 1048, 1034, 980, 1005, 1026, 26, 3, 0, 0, 0, 0, 0, 0, 983, 1028, 1032, 1076, 1057, 1036, 1014, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 998, 1064, 1019, 1003, 996, 1033, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1031, 1027, 1039, 1069, 1033, 1035, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1016, 1006, 1022, 1041, 1018, 976, 997, 31, 0, 0, 0, 0, 0, 0, 0, 1092, 945, 1056, 1028, 1018, 1013, 1027, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1015, 993, 1032, 1003, 1061, 998, 1033, 23, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8145, 8206, 8193, 8290, 8114, 8210, 8221] avg=8192.0000 max=8290 max_rank=4 min=8114 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32835 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1035, 1054, 1059, 1044, 1081, 1014, 1055, 1061, 21, 0, 0, 0, 0, 0, 0, 0, 1008, 1062, 986, 982, 1034, 1041, 993, 979, 33, 9, 0, 0, 0, 0, 0, 0, 1067, 1054, 1060, 996, 1010, 1001, 1026, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1005, 1021, 1049, 1041, 996, 1030, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 973, 984, 1047, 1074, 1029, 1033, 1064, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 988, 1016, 985, 1004, 1051, 1055, 1052, 34, 0, 0, 0, 0, 0, 0, 0, 997, 967, 1016, 1010, 1033, 1014, 1018, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1017, 1003, 968, 986, 1030, 978, 1029, 28, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8424, 8127, 8257, 8138, 8219, 8203, 8093, 8075] avg=8192.0000 max=8424 max_rank=0 min=8075 min_rank=7 imbalance_ratio=1.028320 max_load_reduction=57112 max_load_reduction_pct=0.871460 moved_to_echo=32590 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 7936, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8448, 8064, 8192, 8192, 8320, 8064, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8320, 8192, 8192, 8192, 8320, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[995, 1046, 1007, 1071, 1016, 1058, 1039, 995, 1045, 1039, 1014, 1025, 1075, 1025, 1073, 1019, 1014, 1007, 977, 1024, 1011, 1034, 978, 1040, 1005, 1033, 1028, 1038, 1034, 1022, 1024, 1026, 1034, 1079, 1053, 995, 1011, 1034, 1046, 1027, 1017, 1053, 991, 984, 1026, 1047, 1006, 972, 1041, 1000, 991, 990, 1073, 996, 1070, 995, 996, 1073, 987, 1041, 1040, 990, 1018, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1017, 1064, 1051, 987, 1063, 1063, 1044, 988, 1008, 1031, 1011, 1034, 1051, 980, 999, 1001, 1049, 1008, 992, 998, 1052, 1002, 1065, 1054, 991, 1024, 970, 983, 1055, 983, 1004, 1069, 1009, 1002, 1046, 1041, 1067, 990, 1021, 1054, 1043, 1028, 997, 1002, 1049, 1000, 1004, 985, 1005, 971, 1039, 1010, 1024, 977, 972, 1047, 1023, 1072, 1039, 1073, 1072, 1060, 1073, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1014, 1025, 1029, 1029, 1023, 1029, 1036, 1046, 1048, 1010, 1019, 1037, 1029, 1043, 979, 1088, 1024, 1054, 986, 1026, 1025, 1039, 1025, 1055, 1034, 1024, 994, 1011, 1032, 1008, 1017, 1009, 996, 1043, 1018, 988, 1050, 1011, 1008, 1045, 995, 1037, 1096, 1008, 987, 1018, 1005, 1079, 997, 1029, 1042, 995, 1039, 1037, 1015, 1048, 975, 980, 1060, 1015, 1017, 989, 1023, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1040, 1017, 1049, 989, 1028, 1043, 1027, 1047, 1031, 1019, 1056, 1021, 1035, 1086, 1060, 1069, 993, 1021, 1058, 1063, 968, 1008, 999, 1022, 1018, 996, 1035, 1015, 1016, 1067, 1040, 1047, 1019, 1016, 1083, 1012, 1019, 1025, 949, 1032, 949, 1068, 1062, 1048, 996, 1015, 1029, 1036, 1028, 975, 1049, 948, 1016, 980, 1023, 1001, 1004, 1018, 974, 1005, 1035, 1037, 1051, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1011, 1057, 1044, 1017, 1003, 1032, 1047, 1099, 1060, 1019, 1031, 975, 997, 1004, 1041, 994, 1060, 1041, 1016, 1045, 1016, 1001, 1034, 1028, 1022, 1009, 1028, 1019, 1004, 1028, 975, 1031, 1020, 990, 1033, 1038, 1039, 989, 1011, 1007, 1018, 973, 1009, 1018, 1002, 1052, 1078, 1004, 1004, 1027, 1031, 1047, 1013, 1018, 989, 1114, 1052, 1008, 1023, 999, 1025, 1045, 1071, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1062, 1026, 1022, 1018, 1008, 1007, 1002, 1007, 1024, 1027, 1013, 1005, 1023, 978, 1023, 998, 991, 1013, 1015, 1042, 1079, 1014, 1004, 1014, 1027, 1054, 995, 1006, 1063, 959, 1062, 1063, 1048, 1031, 1028, 1051, 1015, 1060, 1045, 1051, 1029, 1023, 1000, 1015, 1037, 1050, 1004, 1009, 1034, 1032, 1048, 1006, 1051, 988, 1042, 1078, 991, 1002, 989, 998, 1039, 1053, 1051, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1052, 1042, 1008, 1063, 1087, 1028, 1037, 1007, 1036, 1002, 1043, 1004, 1067, 1063, 1010, 1022, 1060, 1028, 1060, 1017, 1030, 1006, 1013, 1050, 1013, 1028, 1050, 1056, 1004, 968, 1026, 1031, 1029, 965, 982, 1030, 1028, 968, 1066, 1031, 1007, 1073, 998, 1007, 1050, 997, 1034, 1011, 975, 996, 1030, 1004, 1031, 1019, 968, 1014, 1046, 998, 1039, 984, 1002, 1038, 1056, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1013, 989, 1034, 1028, 1048, 1018, 1017, 1020, 994, 1005, 1055, 1007, 1012, 1005, 1012, 1026, 1056, 1041, 1025, 1024, 995, 1035, 1069, 1022, 1032, 1003, 1046, 1030, 997, 1093, 1014, 988, 1047, 1013, 1005, 1011, 1032, 1012, 1047, 999, 1023, 1037, 1022, 1011, 1016, 1031, 1067, 1043, 1006, 1034, 1070, 1030, 1025, 1038, 1072, 1006, 1031, 990, 1006, 998, 971, 991, 1042, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1017, 1064, 1051, 987, 1063, 1013, 1044, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1031, 1011, 1034, 1039, 980, 999, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1008, 992, 998, 1052, 1002, 1065, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1024, 970, 983, 1055, 983, 1004, 1069, 15, 5, 0, 0, 0, 0, 0, 0, 1009, 1002, 1046, 1041, 1066, 990, 1021, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1028, 997, 1002, 1049, 1000, 1004, 985, 10, 7, 1, 1, 0, 0, 0, 0, 1005, 971, 1039, 1010, 1024, 977, 972, 1047, 35, 0, 0, 0, 0, 0, 0, 0, 1023, 1072, 1039, 1073, 1072, 1060, 1072, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8103, 8210, 8099, 8229, 8127, 8080, 8461] avg=8192.0000 max=8461 max_rank=7 min=8080 min_rank=6 imbalance_ratio=1.032837 max_load_reduction=57075 max_load_reduction_pct=0.870895 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1062, 1026, 1022, 1018, 1008, 960, 1002, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1027, 1013, 1005, 1011, 978, 1023, 998, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1013, 1015, 1042, 1079, 1014, 1004, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1054, 995, 1006, 1063, 959, 1062, 1063, 14, 5, 0, 0, 0, 0, 0, 0, 1048, 1031, 1028, 1051, 1015, 1060, 1045, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1023, 1000, 1015, 1037, 1050, 1004, 1009, 9, 7, 1, 0, 0, 0, 0, 0, 1034, 1032, 1048, 1006, 1051, 988, 1042, 1078, 33, 0, 0, 0, 0, 0, 0, 0, 991, 1002, 989, 998, 1039, 1053, 1050, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8079, 8163, 8248, 8329, 8184, 8312, 8116] avg=8192.0000 max=8329 max_rank=4 min=8079 min_rank=1 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32941 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[995, 1046, 1007, 1071, 1016, 1009, 1039, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1039, 1014, 1025, 1063, 1025, 1073, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1007, 977, 1024, 1011, 1034, 978, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1033, 1028, 1038, 1034, 1022, 1024, 1026, 15, 5, 0, 0, 0, 0, 0, 0, 1034, 1079, 1053, 995, 1011, 1034, 1046, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1053, 991, 984, 1026, 1047, 1006, 972, 9, 7, 1, 0, 0, 0, 0, 0, 1041, 1000, 991, 990, 1073, 996, 1070, 995, 34, 0, 0, 0, 0, 0, 0, 0, 996, 1073, 987, 1041, 1040, 990, 1017, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8303, 8076, 8230, 8279, 8113, 8190, 8167] avg=8192.0000 max=8303 max_rank=1 min=8076 min_rank=2 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1014, 1025, 1029, 1029, 1023, 982, 1036, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1010, 1019, 1037, 1017, 1043, 979, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1054, 986, 1026, 1025, 1039, 1025, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1024, 994, 1011, 1032, 1008, 1017, 1009, 14, 5, 0, 0, 0, 0, 0, 0, 996, 1043, 1018, 988, 1049, 1011, 1008, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1037, 1096, 1008, 987, 1018, 1005, 1079, 10, 7, 1, 1, 0, 0, 0, 0, 997, 1029, 1042, 995, 1039, 1037, 1015, 1048, 33, 0, 0, 0, 0, 0, 0, 0, 975, 980, 1060, 1015, 1017, 989, 1022, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8241, 8224, 8148, 8158, 8244, 8235, 8102] avg=8192.0000 max=8244 max_rank=5 min=8102 min_rank=7 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1052, 1042, 1008, 1063, 1087, 981, 1037, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1002, 1043, 1004, 1055, 1063, 1010, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1028, 1060, 1017, 1030, 1006, 1013, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1028, 1050, 1056, 1004, 968, 1026, 1031, 14, 5, 0, 0, 0, 0, 0, 0, 1029, 965, 982, 1030, 1028, 968, 1066, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1073, 998, 1007, 1050, 997, 1034, 1011, 10, 7, 1, 0, 0, 0, 0, 0, 975, 996, 1030, 1004, 1031, 1019, 968, 1014, 33, 0, 0, 0, 0, 0, 0, 0, 1046, 998, 1039, 984, 1002, 1038, 1055, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8235, 8254, 8195, 8099, 8195, 8070, 8211] avg=8192.0000 max=8277 max_rank=0 min=8070 min_rank=6 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1013, 989, 1034, 1028, 1048, 971, 1017, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1005, 1055, 1007, 1000, 1005, 1012, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1041, 1025, 1024, 995, 1035, 1069, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1003, 1046, 1030, 997, 1093, 1014, 988, 14, 5, 0, 0, 0, 0, 0, 0, 1047, 1013, 1005, 1011, 1032, 1012, 1047, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1037, 1022, 1011, 1016, 1031, 1067, 1043, 9, 7, 1, 0, 0, 0, 0, 0, 1006, 1034, 1070, 1030, 1025, 1038, 1072, 1006, 33, 0, 0, 0, 0, 0, 0, 0, 1031, 990, 1006, 998, 971, 991, 1041, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8104, 8258, 8222, 8166, 8267, 8314, 8085] avg=8192.0000 max=8314 max_rank=6 min=8085 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1011, 1057, 1044, 1017, 1003, 975, 1047, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1019, 1031, 975, 982, 1004, 1041, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1041, 1016, 1045, 1016, 1001, 1034, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1009, 1028, 1019, 1004, 1028, 975, 1031, 19, 6, 0, 0, 0, 0, 0, 0, 1020, 990, 1033, 1038, 1033, 989, 1011, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 973, 1009, 1018, 1002, 1052, 1078, 1004, 13, 9, 4, 6, 0, 0, 0, 0, 1004, 1027, 1031, 1047, 1013, 1018, 989, 1114, 38, 0, 0, 0, 0, 0, 0, 0, 1052, 1008, 1023, 999, 1025, 1045, 1067, 1001, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8106, 8228, 8141, 8121, 8186, 8281, 8220] avg=8192.0000 max=8281 max_rank=6 min=8106 min_rank=1 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1040, 1017, 1049, 989, 1028, 995, 1027, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1019, 1056, 1021, 1023, 1086, 1060, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1021, 1058, 1063, 968, 1008, 999, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 996, 1035, 1015, 1016, 1067, 1040, 1047, 14, 5, 0, 0, 0, 0, 0, 0, 1019, 1016, 1083, 1012, 1019, 1025, 949, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 949, 1068, 1062, 1048, 996, 1015, 1029, 1036, 9, 7, 1, 0, 0, 0, 0, 0, 1028, 975, 1049, 948, 1016, 980, 1023, 1001, 34, 0, 0, 0, 0, 0, 0, 0, 1004, 1018, 974, 1005, 1035, 1037, 1050, 1051, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8365, 8123, 8253, 8155, 8220, 8054, 8174] avg=8192.0000 max=8365 max_rank=1 min=8054 min_rank=6 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32603 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 7936, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8064, 8320, 8064, 8192, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8448, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1041, 1012, 1064, 1015, 1042, 1017, 1042, 1030, 1032, 998, 1031, 1053, 1039, 1070, 1063, 1053, 1011, 1041, 994, 971, 967, 994, 1000, 1006, 1005, 1040, 987, 1044, 986, 1008, 1008, 1062, 992, 1039, 1037, 1052, 1025, 1048, 1025, 1077, 1048, 1030, 1008, 1050, 1039, 987, 999, 1040, 1061, 1005, 1006, 1032, 1000, 1002, 1017, 1033, 1048, 984, 1048, 991, 1054, 1031, 1005, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1028, 998, 1006, 1060, 984, 1038, 1023, 1084, 997, 1017, 1032, 991, 1023, 1021, 1021, 1021, 996, 1037, 1111, 1075, 984, 1011, 1025, 1011, 1001, 1026, 983, 1045, 988, 1077, 1047, 1044, 1032, 1035, 1004, 1033, 1014, 992, 1072, 1028, 1077, 972, 1037, 1011, 1022, 988, 1004, 1023, 988, 1046, 1025, 1054, 1042, 986, 1048, 1019, 1006, 984, 998, 1034, 1034, 1044, 1016, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1059, 1000, 1026, 1051, 1021, 1028, 979, 1038, 1028, 1067, 1011, 1026, 1063, 1053, 1069, 1060, 975, 1010, 991, 974, 970, 1023, 1018, 1036, 983, 1069, 998, 1037, 1034, 1069, 1044, 1054, 1015, 1014, 988, 1011, 1028, 1012, 1046, 1052, 1026, 1032, 1006, 1041, 1058, 1015, 1025, 1007, 997, 1062, 1030, 1048, 1042, 985, 1040, 947, 1011, 976, 1006, 1029, 1033, 977, 1083, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1050, 1041, 1031, 978, 1035, 1069, 1023, 1068, 1035, 991, 1080, 1001, 1044, 992, 983, 994, 1019, 1062, 1044, 1097, 1019, 995, 971, 1036, 1031, 984, 1046, 1025, 1054, 1099, 1033, 1026, 994, 1009, 1042, 1011, 1032, 990, 1002, 1008, 1053, 1043, 1074, 976, 1034, 1037, 983, 1014, 1036, 1042, 996, 1007, 973, 1004, 978, 988, 1057, 1003, 1013, 1041, 1034, 1035, 1023, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1044, 1023, 1022, 1076, 1077, 1029, 995, 1008, 1001, 1004, 1024, 1001, 980, 970, 1045, 1021, 997, 1003, 994, 998, 1022, 1031, 971, 1049, 1064, 1036, 1006, 961, 995, 1000, 1035, 1051, 971, 1021, 1060, 1010, 1075, 1006, 1023, 986, 1070, 1022, 1004, 1004, 1007, 1042, 1014, 1005, 1032, 1017, 989, 1083, 1024, 1083, 1069, 1042, 1052, 1048, 999, 1070, 1059, 1011, 1086, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1005, 1052, 1035, 978, 1039, 993, 1046, 1017, 1061, 1034, 1012, 1002, 1037, 1044, 1059, 1083, 1065, 957, 1031, 963, 1006, 1026, 1039, 959, 1016, 1039, 993, 1003, 1045, 1036, 967, 1120, 997, 1030, 1010, 1027, 1074, 1033, 1026, 1052, 1014, 1018, 1056, 997, 1018, 1014, 989, 1024, 1012, 1030, 1057, 1050, 991, 990, 1034, 1001, 1031, 995, 1020, 1044, 1018, 1065, 1038, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[998, 1016, 1063, 1015, 1000, 1032, 1001, 1018, 1064, 998, 1007, 1000, 1042, 1038, 1010, 992, 1008, 1068, 1052, 990, 1070, 996, 1025, 977, 1055, 1055, 1032, 1052, 1019, 1009, 1060, 1063, 1054, 993, 1027, 988, 970, 1049, 1024, 983, 1039, 1067, 1039, 1013, 1042, 1005, 1006, 1017, 966, 1018, 995, 1048, 1025, 1017, 1018, 1066, 1016, 1044, 1059, 1040, 1018, 1023, 1012, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1023, 953, 1022, 1060, 1029, 1006, 1023, 1001, 982, 1021, 980, 1028, 1019, 1029, 1007, 1023, 982, 1028, 1042, 1004, 1096, 1034, 1054, 1042, 1004, 1004, 1029, 1058, 1020, 1014, 1026, 1034, 1014, 1017, 1023, 1011, 1059, 1024, 1021, 1048, 979, 979, 1043, 998, 1069, 1010, 1082, 1025, 1038, 1028, 1068, 964, 1035, 1039, 1047, 999, 1072, 1036, 1016, 1075, 1060, 995, 998, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1005, 1052, 1018, 978, 1039, 993, 1046, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1034, 1012, 1002, 1037, 1044, 1054, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 957, 1031, 963, 1006, 1026, 1039, 959, 40, 22, 0, 0, 0, 0, 0, 0, 1016, 1039, 993, 1003, 1045, 1036, 967, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1030, 1010, 1027, 1074, 1033, 1026, 1052, 8, 0, 0, 0, 0, 0, 0, 0, 1014, 1018, 1056, 997, 1018, 1014, 989, 1024, 5, 3, 0, 0, 0, 0, 0, 0, 1012, 1030, 1057, 1050, 991, 990, 1034, 1001, 9, 6, 0, 0, 0, 0, 0, 0, 1031, 995, 1020, 1013, 1018, 1065, 1038, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8327, 8108, 8179, 8257, 8138, 8180, 8199] avg=8192.0000 max=8327 max_rank=1 min=8108 min_rank=2 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1044, 1023, 1006, 1076, 1077, 1029, 995, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1004, 1024, 1001, 980, 970, 1040, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1003, 994, 998, 1022, 1031, 971, 1049, 37, 23, 0, 0, 0, 0, 0, 0, 1064, 1036, 1006, 961, 995, 1000, 1035, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1021, 1060, 1010, 1075, 1006, 1023, 986, 8, 0, 0, 0, 0, 0, 0, 0, 1070, 1022, 1004, 1004, 1007, 1042, 1014, 1005, 5, 3, 0, 0, 0, 0, 0, 0, 1032, 1017, 989, 1083, 1024, 1083, 1069, 1042, 9, 5, 0, 0, 0, 0, 0, 0, 1052, 1048, 999, 1038, 1059, 1011, 1086, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8041, 8125, 8111, 8160, 8176, 8353, 8312] avg=8192.0000 max=8353 max_rank=6 min=8041 min_rank=1 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=33001 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1028, 998, 990, 1060, 984, 1038, 1023, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1017, 1032, 991, 1023, 1021, 1016, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1037, 1111, 1075, 984, 1011, 1025, 1011, 37, 22, 0, 0, 0, 0, 0, 0, 1001, 1026, 983, 1045, 988, 1077, 1047, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1035, 1004, 1033, 1014, 992, 1072, 1028, 8, 0, 0, 0, 0, 0, 0, 0, 1077, 972, 1037, 1011, 1022, 988, 1004, 1023, 5, 3, 0, 0, 0, 0, 0, 0, 988, 1046, 1025, 1054, 1042, 986, 1048, 1019, 9, 5, 0, 0, 0, 0, 0, 0, 1006, 984, 998, 1003, 1034, 1044, 1016, 1063, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8118, 8309, 8174, 8218, 8142, 8222, 8148] avg=8192.0000 max=8309 max_rank=2 min=8118 min_rank=1 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1023, 953, 1006, 1060, 1029, 1006, 1023, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1021, 980, 1028, 1019, 1029, 1002, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1028, 1042, 1004, 1096, 1034, 1054, 1042, 36, 23, 0, 0, 0, 0, 0, 0, 1004, 1004, 1029, 1058, 1020, 1014, 1026, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1017, 1023, 1011, 1059, 1024, 1021, 1048, 8, 0, 0, 0, 0, 0, 0, 0, 979, 979, 1043, 998, 1069, 1010, 1082, 1025, 5, 3, 0, 0, 0, 0, 0, 0, 1038, 1028, 1068, 964, 1035, 1039, 1047, 999, 9, 5, 0, 0, 0, 0, 0, 0, 1072, 1036, 1016, 1043, 1060, 995, 998, 987, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8084, 8341, 8153, 8225, 8193, 8232, 8207] avg=8192.0000 max=8341 max_rank=2 min=8084 min_rank=1 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1050, 1041, 1015, 978, 1035, 1069, 1023, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 991, 1080, 1001, 1044, 992, 979, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1062, 1044, 1097, 1019, 995, 971, 1036, 36, 22, 0, 0, 0, 0, 0, 0, 1031, 984, 1046, 1025, 1054, 1099, 1033, 990, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1009, 1042, 1011, 1032, 990, 1002, 1008, 8, 0, 0, 0, 0, 0, 0, 0, 1053, 1043, 1074, 976, 1034, 1037, 983, 1014, 4, 3, 0, 0, 0, 0, 0, 0, 1036, 1042, 996, 1007, 973, 1004, 978, 988, 9, 5, 0, 0, 0, 0, 0, 0, 1057, 1003, 1013, 1010, 1034, 1035, 1023, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8116, 8301, 8262, 8096, 8221, 8038, 8223] avg=8192.0000 max=8301 max_rank=2 min=8038 min_rank=6 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32578 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1059, 1000, 999, 1051, 1021, 1028, 979, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1067, 1011, 1026, 1063, 1053, 1062, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1010, 991, 974, 970, 1023, 1018, 1036, 42, 26, 0, 0, 0, 0, 0, 0, 983, 1069, 998, 1037, 1034, 1069, 1044, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1014, 988, 1011, 1028, 1012, 1046, 1052, 13, 0, 0, 0, 0, 0, 0, 0, 1026, 1032, 1006, 1041, 1058, 1015, 1025, 1007, 7, 4, 0, 0, 0, 0, 0, 0, 997, 1062, 1030, 1048, 1042, 985, 1040, 947, 12, 10, 0, 0, 0, 0, 0, 0, 1011, 976, 1006, 991, 1033, 977, 1083, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8370, 8065, 8246, 8179, 8221, 8173, 8107] avg=8192.0000 max=8370 max_rank=1 min=8065 min_rank=2 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1041, 1012, 1046, 1015, 1042, 1017, 1042, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 998, 1031, 1053, 1039, 1070, 1058, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1041, 994, 971, 967, 994, 1000, 1006, 37, 21, 0, 0, 0, 0, 0, 0, 1005, 1040, 987, 1044, 986, 1008, 1008, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1039, 1037, 1052, 1025, 1048, 1025, 1077, 9, 0, 0, 0, 0, 0, 0, 0, 1048, 1030, 1008, 1050, 1039, 987, 999, 1040, 5, 3, 0, 0, 0, 0, 0, 0, 1061, 1005, 1006, 1032, 1000, 1002, 1017, 1033, 8, 6, 0, 0, 0, 0, 0, 0, 1048, 984, 1048, 962, 1054, 1031, 1005, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8334, 8042, 8103, 8304, 8209, 8170, 8129] avg=8192.0000 max=8334 max_rank=1 min=8042 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[998, 1016, 1045, 1015, 1000, 1032, 1001, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 998, 1007, 1000, 1042, 1038, 1005, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1068, 1052, 990, 1070, 996, 1025, 977, 37, 22, 0, 0, 0, 0, 0, 0, 1055, 1055, 1032, 1052, 1019, 1009, 1060, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 993, 1027, 988, 970, 1049, 1024, 983, 9, 0, 0, 0, 0, 0, 0, 0, 1039, 1067, 1039, 1013, 1042, 1005, 1006, 1017, 5, 3, 0, 0, 0, 0, 0, 0, 966, 1018, 995, 1048, 1025, 1017, 1018, 1066, 9, 6, 0, 0, 0, 0, 0, 0, 1016, 1044, 1059, 1009, 1018, 1023, 1012, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8146, 8245, 8308, 8097, 8236, 8168, 8211] avg=8192.0000 max=8308 max_rank=3 min=8097 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8320, 8192, 8192, 8192, 8192, 8192, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1024, 1039, 1090, 1007, 1040, 1034, 1007, 993, 1020, 1054, 1003, 1014, 1020, 1064, 989, 1034, 1009, 1055, 1020, 1007, 1023, 984, 1061, 1042, 997, 996, 998, 972, 1007, 1013, 1054, 983, 1082, 994, 946, 1060, 1061, 1000, 967, 1093, 1097, 1007, 1044, 1037, 1036, 1022, 993, 997, 1047, 1000, 1032, 1051, 979, 1077, 1004, 1062, 1049, 1014, 994, 1001, 1025, 1048, 996, 1068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1010, 1058, 963, 1028, 1030, 1058, 974, 1034, 1015, 954, 994, 1019, 990, 1073, 1029, 957, 1009, 1083, 1019, 1068, 1071, 1057, 973, 1009, 1077, 1030, 1039, 989, 1028, 1039, 1012, 1062, 1025, 1039, 1035, 1010, 1067, 989, 1031, 1006, 1009, 1028, 1079, 1016, 986, 1002, 1027, 1056, 997, 1032, 1009, 1012, 1002, 1007, 1003, 1048, 1046, 957, 1042, 1019, 1064, 1078, 1053, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[983, 1007, 1048, 1020, 955, 1037, 1045, 1011, 1031, 984, 1013, 1053, 1029, 1089, 1025, 1036, 1037, 1002, 1004, 1060, 1011, 1014, 1023, 1013, 1069, 1052, 1038, 1043, 987, 1021, 1041, 1013, 1008, 980, 1014, 1023, 998, 1009, 1048, 1048, 1042, 1105, 1022, 1015, 1018, 1057, 1015, 1007, 1079, 1035, 999, 1022, 1011, 968, 1032, 1068, 1010, 1041, 1027, 1019, 996, 1026, 1008, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1007, 1003, 1016, 1037, 1037, 979, 1041, 997, 986, 1016, 1047, 1053, 1043, 1046, 1039, 980, 1030, 1048, 1034, 1044, 1020, 994, 1048, 1025, 1060, 1064, 1064, 1037, 1036, 994, 1014, 991, 994, 1025, 1028, 1027, 1080, 1026, 1026, 1050, 1027, 1065, 1042, 1006, 1032, 966, 974, 999, 1012, 995, 1050, 1011, 1034, 1022, 1030, 1035, 1017, 1013, 1011, 1053, 1069, 1016, 969, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1038, 1049, 1029, 1053, 1065, 1020, 999, 986, 1052, 1001, 1021, 1079, 1072, 1074, 1054, 1020, 990, 975, 1048, 1026, 1056, 1004, 986, 1005, 1014, 1037, 1025, 1014, 1014, 1012, 1017, 1016, 964, 1061, 1048, 977, 1035, 1035, 1003, 986, 1012, 1062, 1027, 1047, 1030, 1013, 1034, 991, 1002, 985, 1029, 1036, 1027, 993, 1021, 1058, 1029, 1078, 1017, 1026, 1016, 1030, 1023, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1009, 1107, 1036, 1092, 1001, 1015, 1020, 1074, 1095, 997, 1017, 1004, 1039, 1037, 1042, 970, 1000, 1000, 1030, 1012, 993, 996, 1033, 1006, 1029, 997, 1021, 1013, 1079, 1059, 1006, 972, 1016, 1031, 1079, 1008, 996, 1044, 1065, 978, 1032, 1010, 1057, 1028, 1023, 1032, 993, 1085, 1054, 1008, 1030, 959, 1012, 978, 1028, 1015, 1003, 1003, 1042, 994, 1020, 1049, 1047, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1003, 1026, 1060, 998, 1008, 1019, 984, 1011, 1025, 1016, 1050, 1056, 994, 1041, 1046, 1047, 992, 1015, 1048, 1032, 1031, 1043, 1039, 972, 1007, 1029, 1001, 1061, 954, 1058, 993, 1075, 966, 1044, 1077, 1046, 1045, 1046, 1026, 998, 976, 1023, 1056, 1010, 980, 1062, 1008, 1028, 1031, 974, 1029, 1008, 1009, 985, 1009, 1028, 1026, 1083, 1037, 1051, 1023, 1038, 1054, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1030, 1041, 1004, 1034, 1038, 1069, 1018, 999, 1054, 1029, 968, 1034, 1081, 1011, 1104, 1007, 1006, 1041, 1032, 976, 994, 992, 983, 1000, 1044, 1022, 1027, 1014, 1014, 1025, 987, 1019, 1017, 1081, 1029, 1008, 978, 1008, 1041, 1022, 1035, 1068, 988, 1053, 1014, 1072, 1002, 1067, 1032, 988, 1032, 1083, 1025, 987, 985, 951, 1025, 1065, 999, 1023, 1028, 1047, 1019, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1024, 1038, 1090, 1007, 1040, 1034, 1007, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1054, 1003, 1014, 1020, 1027, 989, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1055, 1020, 1007, 1023, 984, 1061, 1042, 19, 15, 0, 0, 0, 0, 0, 0, 997, 996, 998, 972, 1007, 1013, 1054, 983, 4, 1, 1, 0, 0, 0, 0, 0, 1082, 994, 946, 1060, 1060, 1000, 967, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 983, 1044, 1037, 1036, 1022, 993, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1000, 1032, 1051, 979, 1077, 1004, 1062, 37, 9, 0, 0, 0, 0, 0, 0, 1049, 1014, 994, 1001, 1025, 1025, 996, 1068, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8161, 8235, 8026, 8202, 8209, 8298, 8172] avg=8192.0000 max=8298 max_rank=6 min=8026 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1009, 1106, 1036, 1092, 1001, 1015, 1020, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 997, 1017, 1004, 1039, 1001, 1042, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1000, 1030, 1012, 993, 996, 1033, 1006, 19, 15, 0, 0, 0, 0, 0, 0, 1029, 997, 1021, 1013, 1079, 1059, 1006, 972, 4, 1, 1, 0, 0, 0, 0, 0, 1016, 1031, 1079, 1008, 995, 1044, 1065, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 986, 1057, 1028, 1023, 1032, 993, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1008, 1030, 959, 1012, 978, 1028, 1015, 36, 9, 0, 0, 0, 0, 0, 0, 1003, 1003, 1042, 994, 1020, 1026, 1047, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8353, 8165, 8104, 8182, 8216, 8236, 8129, 8151] avg=8192.0000 max=8353 max_rank=0 min=8104 min_rank=2 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1010, 1057, 963, 1028, 1030, 1058, 974, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 954, 994, 1019, 990, 1035, 1029, 957, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1083, 1019, 1068, 1071, 1057, 973, 1009, 19, 15, 0, 0, 0, 0, 0, 0, 1077, 1030, 1039, 989, 1028, 1039, 1012, 1062, 5, 1, 1, 0, 0, 0, 0, 0, 1025, 1039, 1035, 1010, 1066, 989, 1031, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1004, 1079, 1016, 986, 1002, 1027, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1032, 1009, 1012, 1002, 1007, 1003, 1048, 38, 9, 0, 0, 0, 0, 0, 0, 1046, 957, 1042, 1019, 1064, 1054, 1053, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 7993, 8323, 8283, 8201, 8179, 8157, 8246] avg=8192.0000 max=8323 max_rank=2 min=7993 min_rank=1 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[983, 1006, 1048, 1020, 955, 1037, 1045, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 984, 1013, 1053, 1029, 1051, 1025, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1002, 1004, 1060, 1011, 1014, 1023, 1013, 18, 17, 0, 0, 0, 0, 0, 0, 1069, 1052, 1038, 1043, 987, 1021, 1041, 1013, 4, 1, 1, 0, 0, 0, 0, 0, 1008, 980, 1014, 1023, 997, 1009, 1048, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1078, 1022, 1015, 1018, 1057, 1015, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1035, 999, 1022, 1011, 968, 1032, 1068, 38, 10, 0, 0, 0, 0, 0, 0, 1010, 1041, 1027, 1019, 996, 1004, 1008, 992, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8222, 8199, 8270, 8127, 8254, 8262, 8097] avg=8192.0000 max=8270 max_rank=3 min=8097 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1038, 1048, 1029, 1053, 1065, 1020, 999, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1001, 1021, 1079, 1072, 1036, 1054, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 990, 975, 1048, 1026, 1056, 1004, 986, 1005, 18, 16, 0, 0, 0, 0, 0, 0, 1014, 1037, 1025, 1014, 1014, 1012, 1017, 1016, 4, 1, 1, 0, 0, 0, 0, 0, 964, 1061, 1048, 977, 1034, 1035, 1003, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1036, 1027, 1047, 1030, 1013, 1034, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 985, 1029, 1036, 1027, 993, 1021, 1058, 38, 10, 0, 0, 0, 0, 0, 0, 1029, 1078, 1017, 1026, 1016, 1008, 1023, 990, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8335, 8124, 8155, 8108, 8190, 8199, 8187] avg=8192.0000 max=8335 max_rank=1 min=8108 min_rank=4 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1007, 1002, 1016, 1037, 1037, 979, 1041, 997, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1016, 1047, 1053, 1043, 1009, 1039, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1048, 1034, 1044, 1020, 994, 1048, 1025, 18, 16, 0, 0, 0, 0, 0, 0, 1060, 1064, 1064, 1037, 1036, 994, 1014, 991, 4, 1, 1, 0, 0, 0, 0, 0, 994, 1025, 1028, 1027, 1079, 1026, 1026, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1039, 1042, 1006, 1032, 966, 974, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 995, 1050, 1011, 1034, 1022, 1030, 1035, 37, 10, 0, 0, 0, 0, 0, 0, 1017, 1013, 1011, 1053, 1069, 994, 969, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8173, 8277, 8266, 8255, 8085, 8236, 8128] avg=8192.0000 max=8277 max_rank=2 min=8085 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1003, 1025, 1060, 998, 1008, 1019, 984, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1016, 1050, 1056, 994, 1004, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1015, 1048, 1032, 1031, 1043, 1039, 972, 19, 15, 0, 0, 0, 0, 0, 0, 1007, 1029, 1001, 1061, 954, 1058, 993, 1075, 4, 1, 1, 0, 0, 0, 0, 0, 966, 1044, 1077, 1046, 1044, 1046, 1026, 998, 0, 0, 0, 0, 0, 0, 0, 0, 976, 999, 1056, 1010, 980, 1062, 1008, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 974, 1029, 1008, 1009, 985, 1009, 1028, 37, 9, 0, 0, 0, 0, 0, 0, 1026, 1083, 1037, 1051, 1023, 1015, 1054, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8238, 8206, 8184, 8247, 8119, 8119, 8315] avg=8192.0000 max=8315 max_rank=7 min=8108 min_rank=0 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1030, 1037, 1004, 1034, 1038, 1069, 1018, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1029, 968, 1034, 1081, 972, 1104, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1041, 1032, 976, 994, 992, 983, 1000, 23, 21, 0, 0, 0, 0, 0, 0, 1044, 1022, 1027, 1014, 1014, 1025, 987, 1019, 10, 4, 4, 0, 0, 0, 0, 0, 1017, 1081, 1029, 1008, 974, 1008, 1041, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1033, 988, 1053, 1014, 1072, 1002, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 988, 1032, 1083, 1025, 987, 985, 951, 39, 14, 0, 0, 0, 0, 0, 0, 1025, 1065, 999, 1023, 1028, 1014, 1019, 1067, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8249, 8068, 8170, 8180, 8264, 8136, 8240] avg=8192.0000 max=8264 max_rank=5 min=8068 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8192, 8192, 8064, 8192, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8320, 8320, 8192, 8448, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8192, 8192, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1024, 1041, 1025, 1029, 1028, 1008, 1027, 979, 1016, 1068, 1050, 1038, 1058, 1012, 998, 1028, 968, 1031, 1004, 980, 1083, 1009, 1027, 1033, 1066, 1033, 1049, 938, 1042, 1066, 1009, 1044, 1019, 1034, 1017, 1031, 1077, 1053, 1064, 961, 1007, 997, 1018, 1036, 1031, 1008, 963, 1005, 1004, 1029, 969, 1013, 1038, 1022, 972, 1023, 1024, 1047, 1024, 1027, 1073, 1034, 1088, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1022, 1036, 1057, 1069, 940, 1024, 1025, 1029, 993, 1020, 1009, 990, 998, 1054, 1023, 1009, 1044, 997, 1004, 1085, 1013, 1039, 1041, 997, 1019, 1025, 1000, 994, 1050, 1032, 1026, 1044, 1061, 1044, 1011, 1045, 1020, 1027, 1044, 1014, 1040, 997, 1026, 993, 1045, 965, 999, 1023, 1057, 1028, 1029, 1044, 1078, 936, 1032, 1019, 1050, 1035, 1030, 1027, 1003, 1033, 1053, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1051, 1023, 974, 1024, 1039, 1039, 1017, 1003, 1078, 1002, 1043, 1054, 1029, 1013, 1005, 981, 1018, 995, 1115, 1079, 989, 985, 1006, 1007, 1032, 1019, 1022, 1010, 1067, 982, 989, 1020, 1058, 1048, 1022, 1069, 1023, 1009, 1071, 1046, 1042, 1062, 992, 1031, 1070, 1007, 1005, 1028, 1038, 986, 1005, 1038, 999, 1050, 1008, 988, 1019, 1041, 1049, 1049, 1011, 1007, 973, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1024, 996, 1022, 1016, 1023, 1017, 1028, 972, 1043, 978, 1021, 1070, 1009, 1082, 1043, 1092, 997, 1026, 978, 998, 1027, 988, 1025, 995, 1035, 1036, 901, 1024, 1012, 992, 1040, 1021, 1022, 1015, 1029, 1016, 999, 989, 1068, 1012, 1036, 1026, 1055, 1010, 1059, 1103, 1005, 1004, 1038, 1013, 1025, 1039, 1063, 1016, 1034, 1044, 993, 1048, 1047, 1006, 1046, 1006, 1082, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[998, 988, 983, 1037, 1013, 1072, 1044, 998, 1015, 1003, 1000, 1033, 1004, 1026, 1029, 1059, 1056, 980, 1052, 1045, 1020, 1017, 1021, 1027, 1016, 1041, 1049, 1036, 1057, 993, 1061, 994, 1047, 951, 1012, 1022, 963, 1007, 1062, 1033, 1005, 1057, 985, 1031, 996, 1008, 970, 1059, 1064, 1012, 1049, 1018, 1010, 1026, 1037, 1060, 1057, 1067, 1016, 1043, 1051, 1052, 1017, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1038, 1069, 1002, 1034, 1002, 1037, 1063, 1037, 1021, 1020, 1031, 991, 1034, 956, 1037, 981, 1009, 1055, 1069, 1058, 1055, 982, 1046, 995, 1051, 1003, 1037, 1011, 1078, 985, 1025, 1031, 1028, 1028, 1023, 1038, 1038, 1036, 978, 1018, 1070, 1046, 991, 1050, 990, 1018, 997, 1023, 1043, 1035, 993, 1009, 992, 1028, 1015, 1048, 1008, 1012, 1044, 1032, 1031, 998, 1017, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1032, 1029, 967, 1023, 1001, 1008, 1013, 1060, 1060, 1047, 1019, 1054, 1021, 1030, 1068, 1019, 986, 992, 1000, 988, 997, 1001, 1021, 1035, 1049, 1063, 1020, 989, 1053, 1019, 1005, 1032, 1067, 1057, 1016, 1017, 996, 966, 1035, 1057, 1020, 1018, 1068, 1010, 1049, 1032, 1003, 1011, 1055, 1062, 1039, 1009, 988, 980, 995, 1014, 1056, 1024, 1039, 1047, 1011, 1049, 1027, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[968, 1075, 1021, 986, 1051, 1046, 1043, 1040, 1003, 991, 988, 1063, 1020, 1069, 1070, 1015, 974, 1038, 1011, 1008, 1067, 1048, 1054, 1010, 1013, 1056, 976, 988, 1024, 1025, 989, 1078, 1040, 1025, 975, 1070, 1069, 1022, 1082, 1001, 994, 1009, 1018, 1046, 1059, 1035, 997, 1024, 1067, 1050, 1019, 1017, 1017, 953, 1013, 1018, 994, 1081, 1065, 1020, 942, 987, 996, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1022, 1036, 1057, 1069, 940, 1024, 1025, 1029, 15, 0, 0, 0, 0, 0, 0, 0, 993, 1020, 1009, 974, 998, 1054, 1023, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 997, 1004, 1085, 1013, 1039, 1041, 997, 29, 0, 0, 0, 0, 0, 0, 0, 1019, 1025, 1000, 994, 1050, 1032, 1026, 1044, 5, 0, 0, 0, 0, 0, 0, 0, 1061, 1044, 1011, 1045, 1020, 1027, 1013, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 997, 1026, 993, 1045, 965, 999, 1023, 13, 6, 0, 0, 0, 0, 0, 0, 1057, 1028, 1029, 1044, 1078, 936, 1032, 1019, 11, 3, 0, 0, 0, 0, 0, 0, 1050, 1000, 1030, 1027, 1003, 1033, 1053, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8080, 8249, 8195, 8235, 8107, 8237, 8216] avg=8192.0000 max=8249 max_rank=2 min=8080 min_rank=1 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[998, 988, 983, 1037, 1013, 1072, 1044, 998, 16, 0, 0, 0, 0, 0, 0, 0, 1015, 1003, 1000, 1015, 1004, 1026, 1029, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 980, 1052, 1045, 1020, 1017, 1021, 1027, 30, 0, 0, 0, 0, 0, 0, 0, 1016, 1041, 1049, 1036, 1057, 993, 1061, 994, 6, 0, 0, 0, 0, 0, 0, 0, 1047, 951, 1012, 1022, 963, 1007, 1030, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1057, 985, 1031, 996, 1008, 970, 1059, 13, 6, 0, 0, 0, 0, 0, 0, 1064, 1012, 1049, 1018, 1010, 1026, 1037, 1060, 12, 3, 0, 0, 0, 0, 0, 0, 1057, 1031, 1016, 1043, 1051, 1052, 1017, 982, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8151, 8248, 8253, 8065, 8130, 8291, 8249] avg=8192.0000 max=8291 max_rank=6 min=8065 min_rank=4 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1024, 996, 1022, 1016, 1023, 1017, 1028, 972, 16, 0, 0, 0, 0, 0, 0, 0, 1043, 978, 1021, 1052, 1009, 1082, 1043, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1026, 978, 998, 1027, 988, 1025, 995, 29, 0, 0, 0, 0, 0, 0, 0, 1035, 1036, 901, 1024, 1012, 992, 1040, 1021, 6, 0, 0, 0, 0, 0, 0, 0, 1022, 1015, 1029, 1016, 999, 989, 1036, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1026, 1055, 1010, 1059, 1103, 1005, 1004, 13, 6, 0, 0, 0, 0, 0, 0, 1038, 1013, 1025, 1039, 1063, 1016, 1034, 1044, 12, 3, 0, 0, 0, 0, 0, 0, 993, 1013, 1047, 1006, 1046, 1006, 1082, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8320, 8063, 8067, 8118, 8317, 8287, 8250] avg=8192.0000 max=8320 max_rank=1 min=8063 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1024, 1041, 1025, 1029, 1028, 1008, 1027, 979, 16, 0, 0, 0, 0, 0, 0, 0, 1016, 1068, 1050, 1020, 1058, 1012, 998, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1031, 1004, 980, 1083, 1009, 1027, 1033, 29, 0, 0, 0, 0, 0, 0, 0, 1066, 1033, 1049, 938, 1042, 1066, 1009, 1044, 6, 0, 0, 0, 0, 0, 0, 0, 1019, 1034, 1017, 1031, 1077, 1053, 1032, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 997, 1018, 1036, 1031, 1008, 963, 1005, 13, 6, 0, 0, 0, 0, 0, 0, 1004, 1029, 969, 1013, 1038, 1022, 972, 1023, 12, 3, 0, 0, 0, 0, 0, 0, 1024, 1012, 1024, 1027, 1073, 1034, 1088, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8250, 8164, 8253, 8224, 8084, 8085, 8299] avg=8192.0000 max=8299 max_rank=7 min=8084 min_rank=5 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1051, 1023, 974, 1024, 1039, 1039, 1017, 1003, 16, 0, 0, 0, 0, 0, 0, 0, 1078, 1002, 1043, 1036, 1029, 1013, 1005, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 995, 1115, 1079, 989, 985, 1006, 1007, 29, 0, 0, 0, 0, 0, 0, 0, 1032, 1019, 1022, 1010, 1067, 982, 989, 1020, 6, 0, 0, 0, 0, 0, 0, 0, 1058, 1048, 1022, 1069, 1023, 1009, 1039, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1062, 992, 1031, 1070, 1007, 1005, 1028, 13, 6, 0, 0, 0, 0, 0, 0, 1038, 986, 1005, 1038, 999, 1050, 1008, 988, 12, 3, 0, 0, 0, 0, 0, 0, 1019, 1006, 1049, 1049, 1011, 1007, 973, 982, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8187, 8223, 8147, 8314, 8256, 8127, 8096] avg=8192.0000 max=8314 max_rank=4 min=8096 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1032, 1029, 967, 1023, 1001, 1008, 1013, 1060, 15, 0, 0, 0, 0, 0, 0, 0, 1060, 1047, 1019, 1036, 1021, 1030, 1068, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 986, 992, 1000, 988, 997, 1001, 1021, 1035, 28, 0, 0, 0, 0, 0, 0, 0, 1049, 1063, 1020, 989, 1053, 1019, 1005, 1032, 6, 0, 0, 0, 0, 0, 0, 0, 1067, 1057, 1016, 1017, 996, 966, 1004, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1018, 1068, 1010, 1049, 1032, 1003, 1011, 13, 6, 0, 0, 0, 0, 0, 0, 1055, 1062, 1039, 1009, 988, 980, 995, 1014, 12, 3, 0, 0, 0, 0, 0, 0, 1056, 990, 1039, 1047, 1011, 1049, 1027, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8300, 8048, 8236, 8180, 8230, 8157, 8237] avg=8192.0000 max=8300 max_rank=1 min=8048 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[968, 1075, 1021, 986, 1051, 1046, 1043, 1040, 16, 0, 0, 0, 0, 0, 0, 0, 1003, 991, 988, 1045, 1020, 1069, 1070, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1038, 1011, 1008, 1067, 1048, 1054, 1010, 30, 0, 0, 0, 0, 0, 0, 0, 1013, 1056, 976, 988, 1024, 1025, 989, 1078, 6, 0, 0, 0, 0, 0, 0, 0, 1040, 1025, 975, 1070, 1069, 1022, 1050, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1009, 1018, 1046, 1059, 1035, 997, 1024, 13, 6, 0, 0, 0, 0, 0, 0, 1067, 1050, 1019, 1017, 1017, 953, 1013, 1018, 12, 3, 0, 0, 0, 0, 0, 0, 994, 1045, 1065, 1020, 942, 987, 996, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8201, 8240, 8155, 8252, 8201, 8169, 8072] avg=8192.0000 max=8252 max_rank=4 min=8072 min_rank=7 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1038, 1069, 1002, 1034, 1002, 1037, 1063, 1037, 17, 0, 0, 0, 0, 0, 0, 0, 1021, 1020, 1031, 967, 1034, 956, 1037, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1055, 1069, 1058, 1055, 982, 1046, 995, 32, 0, 0, 0, 0, 0, 0, 0, 1051, 1003, 1037, 1011, 1078, 985, 1025, 1031, 9, 0, 0, 0, 0, 0, 0, 0, 1028, 1028, 1023, 1038, 1038, 1036, 939, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1046, 991, 1050, 990, 1018, 997, 1023, 17, 9, 0, 0, 0, 0, 0, 0, 1043, 1035, 993, 1009, 992, 1028, 1015, 1048, 15, 5, 0, 0, 0, 0, 0, 0, 1008, 971, 1044, 1032, 1031, 998, 1017, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8299, 8047, 8301, 8230, 8148, 8211, 8183, 8117] avg=8192.0000 max=8301 max_rank=2 min=8047 min_rank=1 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8320, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8064, 8448, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8448, 8320, 8192, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8320, 8192, 8192, 8192, 8064, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8064, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1013, 1015, 1023, 1005, 1050, 971, 1049, 1036, 994, 1026, 1019, 1017, 1071, 1049, 1047, 1023, 988, 1013, 1047, 1030, 993, 999, 1038, 1047, 1024, 982, 1069, 1013, 1056, 1010, 1045, 1028, 1054, 1041, 1046, 1002, 1034, 1006, 1028, 1046, 989, 966, 1085, 1027, 1049, 1003, 998, 1001, 1021, 1005, 1026, 1014, 974, 1003, 1038, 1020, 1014, 1054, 1035, 1007, 1047, 1037, 1006, 1070] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1049, 1011, 1038, 997, 1075, 1035, 949, 997, 1079, 1012, 1027, 1043, 1030, 1007, 994, 1075, 1008, 966, 1034, 1014, 1047, 1038, 1007, 1025, 1033, 967, 1020, 993, 1009, 978, 1037, 1036, 1026, 1063, 1003, 1081, 996, 1045, 1034, 1013, 980, 1033, 1073, 1037, 1014, 1046, 1041, 1053, 1020, 1017, 1060, 983, 1030, 1039, 1019, 1042, 985, 1037, 1019, 1016, 1069, 987, 1006, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1014, 1017, 1034, 1050, 1015, 1014, 1011, 1004, 1009, 1016, 981, 1034, 975, 1039, 1027, 1028, 1023, 1016, 1044, 1036, 1017, 1024, 1023, 987, 1072, 1033, 1010, 1085, 1038, 1002, 986, 990, 1068, 1032, 986, 1019, 1024, 1027, 1020, 994, 990, 1017, 1028, 1009, 1006, 990, 987, 1082, 1037, 1066, 1032, 1036, 1059, 1043, 1014, 1033, 1044, 1019, 1067, 1022, 1046, 1050, 1018, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1032, 1033, 1005, 1021, 1057, 1019, 1038, 1062, 1000, 1017, 1045, 1045, 1032, 1034, 1009, 965, 999, 1010, 1055, 1047, 1026, 1037, 1064, 1024, 995, 1061, 1029, 1018, 1045, 1036, 941, 1007, 1050, 1048, 1029, 1030, 1017, 981, 1043, 981, 1039, 1029, 1024, 1010, 1023, 1045, 1000, 1013, 1015, 993, 997, 1023, 1024, 993, 952, 1026, 1061, 1051, 1054, 1029, 1048, 974, 1067, 1059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1006, 987, 1057, 976, 1039, 991, 1099, 999, 956, 1020, 1031, 1023, 1023, 1014, 1010, 1024, 1014, 992, 1014, 1041, 1011, 993, 1038, 987, 998, 1025, 1021, 1033, 1069, 1017, 1050, 1051, 1065, 1012, 1063, 1062, 1061, 1050, 988, 1036, 974, 1018, 971, 1069, 1026, 1009, 1039, 1053, 1010, 1032, 1081, 1061, 1004, 991, 1047, 1020, 1026, 1031, 1013, 1065, 1037, 1021, 993, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1007, 969, 1024, 1034, 1038, 1055, 1002, 1014, 992, 1059, 1017, 1018, 1011, 1035, 1010, 990, 1003, 1061, 1034, 1011, 1063, 1034, 1013, 1028, 1016, 1025, 1027, 1035, 1017, 976, 1064, 1044, 1049, 1031, 1039, 1033, 1001, 1063, 993, 1060, 1062, 1001, 1035, 1008, 1025, 964, 1054, 1038, 1003, 1048, 980, 1012, 1005, 1017, 990, 1062, 1053, 990, 998, 977, 1078, 1040, 1063, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1057, 1008, 971, 982, 1019, 992, 1020, 1016, 1064, 1009, 1000, 1030, 998, 993, 1006, 1043, 1028, 1028, 990, 1063, 1056, 1015, 1014, 1014, 1003, 1021, 1036, 1075, 996, 1020, 963, 1072, 1059, 1024, 1021, 1046, 1010, 1025, 1039, 1105, 1044, 1019, 1064, 1043, 1030, 1034, 1027, 1010, 1044, 999, 1050, 1008, 1008, 1028, 1072, 983, 1017, 982, 1014, 986, 1038, 1044, 1009, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1048, 1021, 1028, 1003, 1029, 1105, 1078, 995, 1022, 984, 1040, 1029, 1010, 1031, 1053, 1033, 1021, 988, 1006, 987, 1001, 1034, 992, 998, 1012, 1039, 1056, 997, 972, 1000, 1084, 1068, 1061, 1016, 1017, 974, 1006, 1059, 1011, 986, 1031, 1012, 1011, 1048, 1025, 1049, 1029, 1036, 1024, 1042, 997, 1011, 1000, 1019, 1012, 1004, 1014, 1081, 1030, 1055, 1024, 1026, 1028, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1013, 1015, 1023, 1005, 1050, 971, 1049, 1036, 15, 0, 0, 0, 0, 0, 0, 0, 994, 1026, 1019, 1017, 1071, 1049, 1047, 1023, 23, 0, 0, 0, 0, 0, 0, 0, 988, 1013, 1047, 1030, 993, 999, 1038, 1047, 29, 0, 0, 0, 0, 0, 0, 0, 1024, 982, 1069, 1013, 1056, 1010, 1045, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1041, 1046, 1002, 1034, 1006, 1028, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 989, 966, 1085, 1027, 1049, 1003, 998, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1005, 1026, 1014, 974, 1003, 1038, 1020, 19, 7, 0, 0, 0, 0, 0, 0, 1014, 1054, 1035, 1007, 1002, 1037, 1006, 1070, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8269, 8184, 8227, 8209, 8118, 8127, 8225] avg=8192.0000 max=8269 max_rank=1 min=8118 min_rank=5 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1049, 1011, 1038, 997, 1075, 1035, 949, 997, 15, 0, 0, 0, 0, 0, 0, 0, 1079, 1012, 1027, 1043, 1030, 1007, 994, 1075, 24, 0, 0, 0, 0, 0, 0, 0, 1008, 966, 1034, 1014, 1047, 1038, 1007, 1025, 28, 0, 0, 0, 0, 0, 0, 0, 1033, 967, 1020, 993, 1009, 978, 1037, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1063, 1003, 1081, 996, 1045, 1034, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1033, 1073, 1037, 1014, 1046, 1041, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1017, 1060, 983, 1030, 1039, 1019, 1042, 19, 7, 0, 0, 0, 0, 0, 0, 985, 1037, 1019, 1016, 1023, 987, 1006, 1039, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8291, 8167, 8073, 8214, 8277, 8236, 8112] avg=8192.0000 max=8291 max_rank=1 min=8073 min_rank=3 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1014, 1017, 1034, 1050, 1015, 1014, 1011, 1004, 15, 0, 0, 0, 0, 0, 0, 0, 1009, 1016, 981, 1034, 975, 1039, 1027, 1028, 23, 0, 0, 0, 0, 0, 0, 0, 1023, 1016, 1044, 1036, 1017, 1024, 1023, 987, 30, 0, 0, 0, 0, 0, 0, 0, 1072, 1033, 1010, 1085, 1038, 1002, 986, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1032, 986, 1019, 1024, 1027, 1020, 994, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1017, 1028, 1009, 1006, 990, 987, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1066, 1032, 1036, 1059, 1043, 1014, 1033, 20, 7, 0, 0, 0, 0, 0, 0, 1044, 1019, 1067, 1022, 1001, 1050, 1018, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8132, 8200, 8216, 8120, 8109, 8347, 8238] avg=8192.0000 max=8347 max_rank=6 min=8109 min_rank=5 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1057, 1008, 971, 982, 1019, 992, 1020, 1016, 15, 0, 0, 0, 0, 0, 0, 0, 1064, 1009, 1000, 1030, 998, 993, 1006, 1043, 23, 0, 0, 0, 0, 0, 0, 0, 1028, 1028, 990, 1063, 1056, 1015, 1014, 1014, 29, 0, 0, 0, 0, 0, 0, 0, 1003, 1021, 1036, 1075, 996, 1020, 963, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1024, 1021, 1046, 1010, 1025, 1039, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1019, 1064, 1043, 1030, 1034, 1027, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 999, 1050, 1008, 1008, 1028, 1072, 983, 19, 7, 0, 0, 0, 0, 0, 0, 1017, 982, 1014, 986, 993, 1044, 1009, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8080, 8166, 8237, 8186, 8281, 8271, 8218, 8097] avg=8192.0000 max=8281 max_rank=4 min=8080 min_rank=0 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1007, 969, 1024, 1034, 1038, 1055, 1002, 1014, 16, 0, 0, 0, 0, 0, 0, 0, 992, 1059, 1017, 1018, 1011, 1035, 1010, 990, 24, 0, 0, 0, 0, 0, 0, 0, 1003, 1061, 1034, 1011, 1063, 1034, 1013, 1028, 29, 0, 0, 0, 0, 0, 0, 0, 1016, 1025, 1027, 1035, 1017, 976, 1064, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1031, 1039, 1033, 1001, 1063, 993, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1001, 1035, 1008, 1025, 964, 1054, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1048, 980, 1012, 1005, 1017, 990, 1062, 19, 7, 0, 0, 0, 0, 0, 0, 1053, 990, 998, 977, 1031, 1040, 1063, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8156, 8276, 8204, 8221, 8187, 8143, 8190] avg=8192.0000 max=8276 max_rank=2 min=8143 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1006, 987, 1057, 976, 1039, 991, 1099, 999, 19, 3, 0, 0, 0, 0, 0, 0, 956, 1020, 1031, 1023, 1023, 1014, 1010, 1024, 26, 0, 0, 0, 0, 0, 0, 0, 1014, 992, 1014, 1041, 1011, 993, 1038, 987, 35, 0, 0, 0, 0, 0, 0, 0, 998, 1025, 1021, 1033, 1069, 1017, 1050, 1051, 6, 0, 0, 0, 0, 0, 0, 0, 1006, 1012, 1063, 1062, 1061, 1050, 988, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1018, 962, 1069, 1026, 1009, 1039, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1032, 1081, 1061, 1004, 991, 1047, 1020, 24, 11, 0, 0, 0, 0, 0, 0, 1026, 1031, 1013, 1065, 981, 1021, 993, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8127, 8125, 8270, 8278, 8150, 8281, 8129] avg=8192.0000 max=8281 max_rank=6 min=8125 min_rank=2 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1032, 1033, 1005, 1021, 1057, 1019, 1038, 1062, 15, 0, 0, 0, 0, 0, 0, 0, 1000, 1017, 1045, 1045, 1032, 1034, 1009, 965, 23, 0, 0, 0, 0, 0, 0, 0, 999, 1010, 1055, 1047, 1026, 1037, 1064, 1024, 29, 0, 0, 0, 0, 0, 0, 0, 995, 1061, 1029, 1018, 1045, 1036, 941, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1048, 1029, 1030, 1017, 981, 1043, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1029, 1024, 1010, 1023, 1045, 1000, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 993, 997, 1023, 1024, 993, 952, 1026, 19, 7, 0, 0, 0, 0, 0, 0, 1061, 1051, 1054, 1029, 1003, 974, 1067, 1059, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8170, 8291, 8132, 8131, 8183, 8049, 8298] avg=8192.0000 max=8298 max_rank=7 min=8049 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1048, 1021, 1028, 1003, 1029, 1105, 1078, 995, 15, 0, 0, 0, 0, 0, 0, 0, 1022, 984, 1040, 1029, 1010, 1031, 1053, 1033, 23, 0, 0, 0, 0, 0, 0, 0, 1021, 988, 1006, 987, 1001, 1034, 992, 998, 29, 0, 0, 0, 0, 0, 0, 0, 1012, 1039, 1056, 997, 972, 1000, 1084, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1016, 1017, 974, 1006, 1059, 1011, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1012, 1011, 1048, 1025, 1049, 1029, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1042, 997, 1011, 1000, 1019, 1012, 1004, 19, 7, 0, 0, 0, 0, 0, 0, 1014, 1081, 1030, 1055, 979, 1026, 1028, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8322, 8225, 8056, 8228, 8082, 8241, 8135, 8247] avg=8192.0000 max=8322 max_rank=0 min=8056 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 8064, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8192, 8192, 8448, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8064, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1060, 998, 1005, 1069, 1035, 1072, 996, 1003, 1003, 994, 1042, 1057, 1040, 1015, 1031, 1032, 1046, 988, 1067, 1077, 1010, 1035, 987, 999, 1035, 995, 1077, 1026, 1000, 1025, 974, 988, 1044, 1013, 996, 1014, 1013, 1051, 1020, 1029, 1101, 1092, 1002, 1050, 1041, 1080, 1013, 1028, 994, 964, 1025, 1036, 989, 1011, 963, 1024, 1043, 991, 1043, 1010, 1044, 981, 1037, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1034, 1019, 1053, 1020, 966, 1030, 1024, 1020, 1015, 1023, 1043, 1010, 1023, 1110, 1042, 993, 986, 983, 1035, 1046, 1041, 1001, 979, 1012, 1013, 1011, 1073, 1040, 1013, 1052, 1004, 1039, 954, 1026, 1019, 1011, 1044, 1035, 1032, 1043, 1023, 1013, 1085, 1020, 1015, 1006, 1063, 1012, 980, 1031, 1056, 1056, 1046, 997, 994, 1036, 1024, 1012, 999, 1007, 1051, 1050, 1021, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1032, 1011, 996, 1017, 971, 1039, 1001, 1073, 994, 1001, 1048, 1087, 1007, 1013, 1068, 1020, 979, 1068, 1044, 998, 1006, 986, 1028, 1089, 1006, 1048, 1028, 1026, 978, 990, 1004, 1021, 1040, 1044, 1026, 1019, 1056, 999, 1028, 1047, 1040, 1064, 1003, 995, 1019, 991, 1026, 1086, 979, 1044, 1051, 1070, 1034, 1039, 1046, 1043, 1014, 1029, 1002, 1006, 951, 1024, 1029, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[991, 1049, 1021, 994, 1011, 1007, 993, 1021, 1001, 1051, 1012, 967, 1014, 1051, 1008, 1001, 1014, 1050, 997, 1023, 1045, 1006, 976, 1028, 1039, 990, 1016, 1017, 1017, 1039, 1016, 1081, 1058, 1033, 1063, 1063, 988, 998, 949, 1039, 1008, 1063, 1006, 1030, 1060, 1018, 986, 1060, 1035, 1073, 1022, 1002, 1037, 1021, 1058, 1018, 1021, 1004, 1048, 1033, 1070, 1021, 1048, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1074, 1028, 1010, 1037, 1002, 1048, 1026, 1039, 1022, 1030, 1029, 1059, 1009, 981, 1028, 1040, 1019, 1024, 995, 1051, 984, 1024, 1071, 1071, 1023, 1010, 980, 1007, 1011, 1003, 1062, 1064, 1026, 1007, 976, 1026, 1040, 1012, 1021, 1047, 995, 1018, 1024, 1019, 1047, 1027, 1037, 1025, 1047, 1031, 986, 1019, 1047, 991, 992, 1047, 1020, 1057, 1015, 1017, 927, 1094, 1033, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[971, 1035, 1033, 1042, 1001, 959, 1060, 1008, 977, 1043, 1023, 965, 1036, 1006, 1036, 1053, 1039, 1000, 1010, 1032, 1055, 1016, 1061, 1025, 1010, 1030, 1015, 1058, 1057, 1033, 1046, 983, 1002, 1045, 996, 993, 1019, 1016, 1085, 1070, 1033, 1042, 1047, 971, 1056, 1030, 1082, 1051, 1047, 1025, 1031, 1041, 1029, 1019, 1064, 988, 944, 1028, 1011, 1023, 1006, 1037, 1030, 957] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1034, 1046, 1037, 990, 987, 1027, 1048, 1006, 1012, 1064, 1048, 1007, 1045, 999, 1048, 1044, 1000, 1024, 1037, 1044, 1050, 1042, 1010, 957, 1066, 1040, 1004, 1070, 1042, 980, 999, 1023, 1012, 1007, 1027, 1000, 1043, 1037, 1032, 1011, 988, 1017, 1017, 1057, 1001, 1053, 1037, 998, 1076, 1001, 1008, 1040, 1002, 994, 1059, 975, 1010, 1063, 1064, 1004, 1062, 970, 1030, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[977, 1065, 1054, 987, 1051, 1026, 1045, 1015, 969, 956, 991, 1029, 992, 978, 1038, 1005, 1058, 1040, 1024, 1032, 1028, 1014, 1013, 982, 1023, 1023, 956, 1059, 1072, 1081, 1034, 1056, 1003, 993, 1017, 1029, 1048, 1040, 1039, 1023, 1015, 1013, 1015, 987, 1084, 1040, 1020, 1017, 1060, 1072, 1065, 1009, 990, 1072, 1063, 1054, 1039, 996, 982, 990, 1016, 1004, 1026, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1034, 1019, 1053, 1020, 966, 1030, 1024, 1020, 17, 0, 0, 0, 0, 0, 0, 0, 1015, 1023, 1043, 1010, 1023, 1110, 1042, 993, 15, 4, 0, 0, 0, 0, 0, 0, 986, 983, 1035, 1046, 1041, 1001, 979, 1012, 8, 1, 0, 0, 0, 0, 0, 0, 1013, 1011, 1073, 1032, 1013, 1052, 1004, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1026, 1019, 1011, 1044, 1035, 1032, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1013, 1085, 1020, 964, 1006, 1063, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1031, 1056, 1034, 1046, 997, 994, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1012, 999, 1007, 1051, 1050, 1021, 1022, 36, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8278, 8092, 8237, 8164, 8186, 8174, 8222] avg=8192.0000 max=8278 max_rank=1 min=8092 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32746 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1060, 998, 1005, 1069, 1035, 1072, 996, 1003, 17, 0, 0, 0, 0, 0, 0, 0, 1003, 994, 1042, 1057, 1040, 1015, 1031, 1032, 15, 4, 0, 0, 0, 0, 0, 0, 1046, 988, 1067, 1077, 1010, 1035, 987, 999, 8, 1, 0, 0, 0, 0, 0, 0, 1035, 995, 1077, 1018, 1000, 1025, 974, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1013, 996, 1014, 1013, 1051, 1020, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 1092, 1002, 1050, 989, 1080, 1013, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 994, 964, 1025, 1014, 989, 1011, 963, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 991, 1043, 1010, 1044, 981, 1037, 1013, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8233, 8218, 8112, 8180, 8355, 7984, 8199] avg=8192.0000 max=8355 max_rank=5 min=7984 min_rank=6 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32718 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1032, 1011, 996, 1017, 971, 1039, 1001, 1073, 17, 0, 0, 0, 0, 0, 0, 0, 994, 1001, 1048, 1087, 1007, 1013, 1068, 1020, 15, 4, 0, 0, 0, 0, 0, 0, 979, 1068, 1044, 998, 1006, 986, 1028, 1089, 8, 1, 0, 0, 0, 0, 0, 0, 1006, 1048, 1028, 1018, 978, 990, 1004, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1044, 1026, 1019, 1056, 999, 1028, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1064, 1003, 995, 967, 991, 1026, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1044, 1051, 1048, 1034, 1039, 1046, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1029, 1002, 1006, 951, 1024, 1029, 1015, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8257, 8207, 8093, 8259, 8172, 8284, 8107] avg=8192.0000 max=8284 max_rank=6 min=8093 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32822 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[991, 1049, 1021, 994, 1011, 1007, 993, 1021, 16, 0, 0, 0, 0, 0, 0, 0, 1001, 1051, 1012, 967, 1014, 1051, 1008, 1001, 15, 4, 0, 0, 0, 0, 0, 0, 1014, 1050, 997, 1023, 1045, 1006, 976, 1028, 7, 1, 0, 0, 0, 0, 0, 0, 1039, 990, 1016, 1010, 1017, 1039, 1016, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1033, 1063, 1063, 988, 998, 949, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1063, 1006, 1030, 1007, 1018, 986, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1073, 1022, 981, 1037, 1021, 1058, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1004, 1048, 1033, 1070, 1021, 1048, 1057, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8124, 8147, 8208, 8191, 8178, 8245, 8340] avg=8192.0000 max=8340 max_rank=7 min=8103 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32954 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[971, 1035, 1033, 1042, 1001, 959, 1060, 1008, 17, 0, 0, 0, 0, 0, 0, 0, 977, 1043, 1023, 965, 1036, 1006, 1036, 1053, 15, 4, 0, 0, 0, 0, 0, 0, 1039, 1000, 1010, 1032, 1055, 1016, 1061, 1025, 8, 1, 0, 0, 0, 0, 0, 0, 1010, 1030, 1015, 1050, 1057, 1033, 1046, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1045, 996, 993, 1019, 1016, 1085, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1042, 1047, 971, 1003, 1030, 1082, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1025, 1031, 1019, 1029, 1019, 1064, 988, 0, 0, 0, 0, 0, 0, 0, 0, 944, 1028, 1011, 1023, 1006, 1037, 1030, 957, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8158, 8247, 8224, 8226, 8259, 8222, 8074] avg=8192.0000 max=8259 max_rank=5 min=8074 min_rank=7 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32781 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1074, 1028, 1010, 1037, 1002, 1048, 1026, 1039, 16, 0, 0, 0, 0, 0, 0, 0, 1022, 1030, 1029, 1059, 1009, 981, 1028, 1040, 15, 4, 0, 0, 0, 0, 0, 0, 1019, 1024, 995, 1051, 984, 1024, 1071, 1071, 7, 1, 0, 0, 0, 0, 0, 0, 1023, 1010, 980, 1000, 1011, 1003, 1062, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1007, 976, 1026, 1040, 1012, 1021, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1018, 1024, 1019, 994, 1027, 1037, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1031, 986, 998, 1047, 991, 992, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1057, 1015, 1017, 927, 1094, 1033, 1005, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8280, 8217, 8247, 8153, 8155, 8139, 8139, 8206] avg=8192.0000 max=8280 max_rank=0 min=8139 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32639 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1034, 1046, 1037, 990, 987, 1027, 1048, 1006, 17, 0, 0, 0, 0, 0, 0, 0, 1012, 1064, 1048, 1007, 1045, 999, 1048, 1044, 14, 4, 0, 0, 0, 0, 0, 0, 1000, 1024, 1037, 1044, 1050, 1042, 1010, 957, 8, 1, 0, 0, 0, 0, 0, 0, 1066, 1040, 1004, 1062, 1042, 980, 999, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1007, 1027, 1000, 1043, 1037, 1032, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1017, 1017, 1057, 951, 1053, 1037, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1001, 1008, 1018, 1002, 994, 1059, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1063, 1064, 1004, 1062, 970, 1030, 1011, 36, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8285, 8173, 8216, 8169, 8118, 8133, 8250] avg=8192.0000 max=8285 max_rank=1 min=8118 min_rank=5 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32670 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[977, 1065, 1054, 987, 1051, 1026, 1045, 1015, 20, 0, 0, 0, 0, 0, 0, 0, 969, 956, 991, 1029, 992, 978, 1038, 1005, 19, 7, 0, 0, 0, 0, 0, 0, 1058, 1040, 1024, 1032, 1028, 1014, 1013, 982, 11, 3, 0, 0, 0, 0, 0, 0, 1023, 1023, 956, 1048, 1072, 1081, 1034, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 993, 1017, 1029, 1048, 1040, 1039, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1013, 1015, 987, 1022, 1040, 1020, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1072, 1065, 979, 990, 1072, 1063, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 996, 982, 990, 1016, 1004, 1026, 1042, 43, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 7984, 8205, 8293, 8192, 8129, 8355, 8138] avg=8192.0000 max=8355 max_rank=6 min=7984 min_rank=1 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32814 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8192, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8064, 8320, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8448, 8320, 8192, 7936, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[989, 1045, 1007, 1013, 1031, 1006, 1020, 986, 1018, 1051, 1017, 1041, 1011, 999, 1024, 1057, 1007, 991, 1012, 1025, 1071, 1056, 1061, 1014, 1003, 1001, 1006, 1067, 977, 1032, 985, 1029, 1046, 1024, 974, 1054, 1025, 1013, 1048, 1033, 1040, 1042, 959, 1070, 1021, 1020, 1069, 1034, 1023, 1051, 977, 1007, 1056, 996, 1050, 990, 969, 1052, 1009, 1065, 994, 1050, 1027, 1096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1010, 1074, 1026, 1061, 1040, 1062, 1022, 977, 997, 999, 1005, 1001, 1015, 1039, 1016, 1044, 1033, 998, 1061, 987, 1021, 1087, 1051, 999, 1057, 1029, 999, 1027, 1042, 977, 1030, 1033, 999, 1058, 1009, 1038, 985, 998, 1022, 995, 1040, 1054, 1048, 1018, 1025, 1000, 1058, 1000, 1059, 993, 1061, 1016, 1031, 1041, 1044, 1062, 967, 1005, 1001, 1021, 1064, 1040, 976, 989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1019, 1050, 1041, 1039, 1007, 1045, 1019, 1084, 988, 1033, 1033, 1064, 1003, 1034, 1025, 1004, 1005, 1001, 986, 1007, 1038, 1039, 1006, 983, 1055, 1053, 995, 1042, 1042, 996, 1053, 1064, 1044, 1065, 996, 1034, 1013, 1044, 993, 1027, 1069, 1059, 1048, 1026, 971, 1067, 1020, 995, 1054, 1038, 1015, 988, 981, 1002, 1003, 1020, 1005, 1046, 1025, 1051, 977, 1007, 967, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1076, 1057, 989, 998, 1030, 1039, 1055, 984, 1019, 1005, 1006, 1007, 1031, 1037, 1067, 1014, 1058, 1034, 1016, 1023, 1031, 998, 1002, 1031, 1014, 1014, 1053, 978, 1013, 1031, 1048, 1002, 1071, 1041, 1046, 1069, 1018, 973, 1051, 1000, 1014, 1043, 1017, 1028, 983, 1037, 1047, 1030, 1063, 985, 997, 990, 1064, 1030, 1020, 1028, 1012, 1015, 1021, 1024, 1028, 967, 1067, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1061, 1022, 1063, 1019, 978, 1052, 1044, 1020, 1049, 1030, 1050, 1015, 1025, 1009, 1019, 1047, 1001, 1032, 1063, 1012, 1085, 1020, 1021, 1010, 1030, 947, 1034, 1083, 1017, 1024, 987, 993, 934, 1006, 1006, 1057, 1040, 1031, 980, 1053, 1038, 1001, 1031, 1024, 1045, 1045, 1028, 1037, 1034, 1051, 1016, 1014, 1023, 1028, 991, 1087, 1022, 1003, 988, 1002, 1003, 1013, 1052, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1001, 1029, 1062, 1028, 1045, 1034, 1042, 1023, 1056, 1005, 973, 1043, 1009, 1038, 1065, 1026, 1033, 1014, 1043, 1033, 1046, 1043, 1014, 1035, 1021, 1011, 1011, 1039, 1035, 1001, 1039, 1009, 1025, 1017, 1049, 1014, 991, 1000, 1010, 1004, 1037, 1043, 1055, 1043, 949, 952, 1004, 1027, 1042, 1012, 983, 1007, 1028, 1004, 1035, 995, 990, 1065, 994, 1063, 1046, 1031, 1095, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1073, 981, 981, 998, 1006, 1018, 1053, 990, 1030, 1050, 1014, 1009, 997, 974, 991, 1020, 1044, 1028, 1013, 1027, 1044, 1051, 1033, 1031, 987, 1031, 1040, 1007, 1027, 1068, 1028, 1008, 993, 1034, 1013, 1071, 1028, 1021, 1060, 1043, 1056, 994, 997, 1007, 985, 1017, 1059, 1055, 994, 1047, 991, 1070, 1049, 1017, 1015, 1023, 1036, 985, 1072, 983, 1030, 1055, 1041, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1012, 985, 969, 994, 1062, 1006, 1024, 1034, 1041, 1031, 994, 1066, 1025, 1005, 1008, 1047, 1011, 1040, 1031, 1016, 1045, 1010, 1032, 1012, 1025, 1003, 1032, 1038, 1026, 1016, 1008, 1038, 1023, 1038, 999, 1083, 1032, 1006, 1067, 998, 1013, 1053, 983, 996, 1062, 1015, 1035, 1045, 997, 1042, 1035, 1034, 1022, 1024, 1044, 1001, 1018, 1004, 1031, 1063, 1036, 1024, 1006, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[989, 1045, 1007, 1013, 1031, 1006, 1013, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1051, 1017, 1041, 1011, 999, 1024, 1057, 6, 2, 0, 0, 0, 0, 0, 0, 1007, 991, 1012, 1025, 1050, 1056, 1061, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1001, 1006, 1067, 977, 1032, 985, 1029, 12, 3, 0, 0, 0, 0, 0, 0, 1046, 1024, 974, 1054, 1025, 1013, 1048, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1042, 959, 1070, 1021, 1020, 1051, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1051, 977, 1007, 1056, 996, 1050, 990, 5, 0, 0, 0, 0, 0, 0, 0, 969, 1052, 1009, 1065, 994, 1050, 1027, 1096, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8226, 8216, 8115, 8217, 8237, 8155, 8280] avg=8192.0000 max=8280 max_rank=7 min=8090 min_rank=0 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32889 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1019, 1050, 1041, 1039, 1007, 1045, 1012, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1033, 1033, 1064, 1003, 1034, 1025, 1004, 5, 2, 0, 0, 0, 0, 0, 0, 1005, 1001, 986, 1007, 1018, 1039, 1006, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1053, 995, 1042, 1042, 996, 1053, 1064, 12, 3, 0, 0, 0, 0, 0, 0, 1044, 1065, 996, 1034, 1013, 1044, 993, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1059, 1048, 1026, 971, 1067, 1003, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1038, 1015, 988, 981, 1002, 1003, 1020, 5, 0, 0, 0, 0, 0, 0, 0, 1005, 1046, 1025, 1051, 977, 1007, 967, 1033, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8191, 8045, 8315, 8216, 8238, 8106, 8128] avg=8192.0000 max=8315 max_rank=3 min=8045 min_rank=2 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1010, 1074, 1026, 1061, 1040, 1062, 1015, 977, 0, 0, 0, 0, 0, 0, 0, 0, 997, 999, 1005, 1001, 1015, 1039, 1016, 1044, 5, 2, 0, 0, 0, 0, 0, 0, 1033, 998, 1061, 987, 1001, 1087, 1051, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1029, 999, 1027, 1042, 977, 1030, 1033, 12, 3, 0, 0, 0, 0, 0, 0, 999, 1058, 1009, 1038, 985, 998, 1022, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1054, 1048, 1018, 1025, 1000, 1041, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 993, 1061, 1016, 1031, 1041, 1044, 1062, 5, 0, 0, 0, 0, 0, 0, 0, 967, 1005, 1001, 1021, 1064, 1040, 976, 989, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8123, 8217, 8209, 8104, 8226, 8312, 8080] avg=8192.0000 max=8312 max_rank=6 min=8080 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1001, 1029, 1062, 1028, 1045, 1034, 1034, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1005, 973, 1043, 1009, 1038, 1065, 1026, 5, 3, 0, 0, 0, 0, 0, 0, 1033, 1014, 1043, 1033, 1026, 1043, 1014, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1011, 1011, 1039, 1035, 1001, 1039, 1009, 12, 3, 0, 0, 0, 0, 0, 0, 1025, 1017, 1049, 1014, 991, 1000, 1010, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1043, 1055, 1043, 949, 952, 987, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1012, 983, 1007, 1028, 1004, 1035, 995, 5, 0, 0, 0, 0, 0, 0, 0, 990, 1065, 994, 1063, 1046, 1031, 1095, 1020, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8223, 8241, 8181, 8110, 8093, 8111, 8321] avg=8192.0000 max=8321 max_rank=7 min=8093 min_rank=5 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1076, 1057, 989, 998, 1030, 1039, 1033, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1005, 1006, 1007, 1031, 1037, 1067, 1014, 11, 7, 0, 0, 0, 0, 0, 0, 1058, 1034, 1016, 1023, 1004, 998, 1002, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1014, 1053, 978, 1013, 1031, 1048, 1002, 16, 5, 0, 0, 0, 0, 0, 0, 1071, 1041, 1046, 1069, 1018, 973, 1051, 1000, 4, 0, 0, 0, 0, 0, 0, 0, 1014, 1043, 1017, 1028, 983, 1037, 1020, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 985, 997, 990, 1064, 1030, 1020, 1028, 11, 0, 0, 0, 0, 0, 0, 0, 1012, 1015, 1021, 1024, 1028, 967, 1067, 997, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8204, 8166, 8174, 8273, 8172, 8188, 8153] avg=8192.0000 max=8273 max_rank=4 min=8153 min_rank=7 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1012, 985, 969, 994, 1062, 1006, 1017, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1031, 994, 1066, 1025, 1005, 1008, 1047, 5, 2, 0, 0, 0, 0, 0, 0, 1011, 1040, 1031, 1016, 1025, 1010, 1032, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1003, 1032, 1038, 1026, 1016, 1008, 1038, 12, 3, 0, 0, 0, 0, 0, 0, 1023, 1038, 999, 1083, 1032, 1006, 1067, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1053, 983, 996, 1062, 1015, 1018, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1042, 1035, 1034, 1022, 1024, 1044, 1001, 5, 0, 0, 0, 0, 0, 0, 0, 1018, 1004, 1031, 1063, 1036, 1024, 1006, 1021, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8079, 8224, 8177, 8201, 8246, 8185, 8204, 8220] avg=8192.0000 max=8246 max_rank=4 min=8079 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1061, 1022, 1063, 1019, 978, 1052, 1036, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1030, 1050, 1015, 1025, 1009, 1019, 1047, 5, 3, 0, 0, 0, 0, 0, 0, 1001, 1032, 1063, 1012, 1064, 1020, 1021, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 947, 1034, 1083, 1017, 1024, 987, 993, 12, 3, 0, 0, 0, 0, 0, 0, 934, 1006, 1006, 1057, 1040, 1031, 980, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1001, 1031, 1024, 1045, 1045, 1011, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1051, 1016, 1014, 1023, 1028, 991, 1087, 5, 0, 0, 0, 0, 0, 0, 0, 1022, 1003, 988, 1002, 1003, 1013, 1052, 991, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8252, 8223, 8130, 8107, 8232, 8249, 8092] avg=8192.0000 max=8252 max_rank=1 min=8092 min_rank=7 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1073, 981, 981, 998, 1006, 1018, 1045, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1050, 1014, 1009, 997, 974, 991, 1020, 5, 3, 0, 0, 0, 0, 0, 0, 1044, 1028, 1013, 1027, 1024, 1051, 1033, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1031, 1040, 1007, 1027, 1068, 1028, 1008, 12, 3, 0, 0, 0, 0, 0, 0, 993, 1034, 1013, 1071, 1028, 1021, 1060, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 994, 997, 1007, 985, 1017, 1042, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1047, 991, 1070, 1049, 1017, 1015, 1023, 5, 0, 0, 0, 0, 0, 0, 0, 1036, 985, 1072, 983, 1030, 1055, 1041, 1043, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8092, 8093, 8251, 8211, 8263, 8153, 8211, 8262] avg=8192.0000 max=8263 max_rank=4 min=8092 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32889 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8448, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8320, 8192, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 8064, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True + [2026-08-30 05:02:08] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 39170.6 | throughput per GPU (TFLOP/s/GPU): 8.5 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.101816E+01 | loss scale: 1.0 | grad norm: 43.805 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 63755.8447265625 | max allocated: 88295.66845703125 | reserved: 91652.0 | max reserved: 91652.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1001, 1021, 1058, 1030, 1049, 1003, 991, 1018, 1033, 1020, 1070, 963, 1033, 1007, 1043, 1033, 1012, 947, 1057, 1005, 1048, 1036, 985, 977, 1047, 1035, 989, 979, 1025, 1023, 1048, 1025, 1037, 1072, 1020, 1037, 1001, 1054, 978, 1049, 1005, 991, 1082, 1016, 1015, 1042, 1034, 1003, 1033, 1056, 1054, 1040, 1029, 992, 1061, 990, 1021, 1024, 1027, 1027, 1003, 1049, 1052, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1044, 1043, 1030, 1036, 1033, 1037, 996, 1030, 1043, 1085, 1016, 1033, 1039, 1017, 1023, 1044, 1035, 988, 985, 1080, 1016, 1032, 1039, 1084, 1038, 1004, 1029, 994, 1046, 1020, 1008, 1044, 1064, 1030, 1052, 1017, 1000, 982, 1015, 1072, 965, 1035, 1012, 1008, 1037, 1020, 1003, 1034, 1023, 1030, 966, 1020, 1010, 1021, 1038, 1023, 993, 1012, 1003, 1003, 997, 1007, 1007, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1015, 986, 957, 996, 1041, 1027, 1024, 1020, 1012, 996, 1015, 990, 1049, 989, 1015, 1071, 1038, 1032, 1039, 1033, 1007, 1048, 943, 1020, 1041, 1016, 1007, 1038, 1090, 1102, 1006, 983, 1095, 1074, 991, 966, 1019, 1006, 978, 1015, 1042, 1000, 1020, 1018, 1054, 1008, 1024, 1044, 1033, 998, 1039, 1033, 1036, 1036, 1032, 1011, 1015, 1057, 1094, 1054, 1000, 1079, 989, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1021, 1033, 1048, 1082, 985, 1033, 982, 1050, 1075, 1007, 1022, 982, 1000, 1017, 1035, 960, 1073, 1017, 1043, 992, 1009, 1015, 1026, 999, 1038, 1043, 1013, 1002, 1006, 1058, 986, 1100, 1022, 1001, 1010, 1051, 1028, 1060, 1014, 1005, 977, 1091, 978, 1037, 1047, 1020, 985, 1046, 1006, 1021, 1015, 1034, 1029, 1045, 1036, 1064, 985, 1060, 1057, 1018, 1021, 996, 1045, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1031, 1036, 1030, 973, 1011, 1018, 1010, 1002, 1010, 996, 1057, 1044, 1037, 1055, 1001, 1006, 1073, 1033, 984, 1035, 1029, 1061, 1036, 1010, 1001, 1062, 1003, 1008, 1026, 1066, 1040, 1042, 1041, 1015, 989, 1027, 990, 984, 1019, 1046, 1046, 1035, 1041, 1034, 1035, 1021, 1047, 1081, 1003, 1003, 1051, 998, 988, 991, 1002, 978, 1040, 1020, 1031, 1069, 1008, 1075, 1002, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1029, 1046, 1026, 1067, 1018, 1062, 1072, 1043, 990, 1003, 1011, 1047, 1027, 1006, 1047, 1059, 1047, 1065, 1017, 1026, 1022, 1032, 1032, 1025, 1067, 1046, 1041, 1020, 1007, 1060, 999, 995, 1042, 1006, 1043, 1007, 1031, 1022, 1055, 1052, 988, 1051, 1032, 1026, 1000, 996, 1016, 960, 992, 997, 976, 991, 1029, 1032, 1037, 1040, 1057, 1023, 997, 985, 1035, 981, 1017, 966] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1068, 1013, 1031, 1021, 1046, 1021, 980, 1054, 1009, 1003, 1014, 991, 1058, 1015, 1060, 1063, 1004, 1000, 1040, 1027, 1041, 1007, 1019, 987, 1052, 1028, 1012, 1035, 1040, 1074, 1016, 1029, 1049, 988, 996, 985, 991, 1002, 1024, 1016, 974, 1035, 1020, 989, 1031, 1032, 1070, 1019, 990, 1000, 1038, 1017, 1076, 1049, 1034, 998, 1029, 991, 1049, 1066, 1032, 1038, 1046, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1057, 1045, 980, 1009, 1023, 1025, 1048, 1038, 1007, 1075, 959, 1047, 1076, 1027, 1063, 1012, 1008, 1058, 1036, 1018, 1032, 992, 1031, 1017, 1043, 1039, 986, 1034, 1009, 1016, 1048, 990, 1001, 1034, 1049, 960, 1025, 1030, 1045, 1007, 1001, 999, 1012, 1038, 1060, 1029, 995, 1000, 1035, 947, 1055, 1021, 1024, 961, 1034, 1054, 1059, 1021, 1027, 1029, 1045, 996, 1063, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1031, 1043, 1030, 1036, 1033, 1037, 996, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1085, 1016, 1033, 1031, 1017, 1023, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 988, 985, 1080, 1016, 1032, 1039, 1084, 3, 0, 0, 0, 0, 0, 0, 0, 1038, 1004, 1029, 994, 1046, 987, 1008, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1030, 1052, 1017, 1000, 982, 1015, 1072, 13, 4, 0, 0, 0, 0, 0, 0, 965, 1035, 1012, 1008, 1037, 1020, 1003, 1034, 9, 5, 0, 0, 0, 0, 0, 0, 1023, 1030, 966, 1020, 1010, 1021, 1038, 1023, 29, 0, 0, 0, 0, 0, 0, 0, 993, 1012, 994, 1003, 997, 1007, 1007, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8292, 8262, 8150, 8249, 8128, 8160, 8059] avg=8192.0000 max=8292 max_rank=1 min=8059 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[988, 1021, 1058, 1030, 1049, 1003, 991, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1020, 1070, 963, 1025, 1007, 1043, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 947, 1057, 1005, 1048, 1036, 985, 977, 3, 0, 0, 0, 0, 0, 0, 0, 1047, 1035, 989, 979, 1025, 990, 1048, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1072, 1020, 1037, 1001, 1054, 978, 1049, 13, 4, 0, 0, 0, 0, 0, 0, 1005, 991, 1082, 1016, 1015, 1042, 1034, 1003, 9, 5, 0, 0, 0, 0, 0, 0, 1033, 1056, 1054, 1040, 1029, 992, 1061, 990, 29, 0, 0, 0, 0, 0, 0, 0, 1021, 1024, 1018, 1027, 1003, 1049, 1052, 1031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8194, 8070, 8138, 8265, 8202, 8284, 8225] avg=8192.0000 max=8284 max_rank=6 min=8070 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1002, 986, 957, 996, 1041, 1027, 1024, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 996, 1015, 990, 1040, 989, 1015, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1032, 1039, 1033, 1007, 1048, 943, 1020, 4, 0, 0, 0, 0, 0, 0, 0, 1041, 1016, 1007, 1038, 1090, 1066, 1006, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 1074, 991, 966, 1019, 1006, 978, 1015, 13, 5, 0, 0, 0, 0, 0, 0, 1042, 1000, 1020, 1018, 1054, 1008, 1024, 1044, 10, 5, 0, 0, 0, 0, 0, 0, 1033, 998, 1039, 1033, 1036, 1036, 1032, 1011, 31, 0, 0, 0, 0, 0, 0, 0, 1015, 1057, 1084, 1054, 1000, 1079, 989, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8053, 8128, 8164, 8247, 8162, 8225, 8249, 8308] avg=8192.0000 max=8308 max_rank=7 min=8053 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32944 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1008, 1033, 1048, 1082, 985, 1033, 982, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1007, 1022, 982, 992, 1017, 1035, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1017, 1043, 992, 1009, 1015, 1026, 999, 3, 0, 0, 0, 0, 0, 0, 0, 1038, 1043, 1013, 1002, 1006, 1023, 986, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1001, 1010, 1051, 1028, 1060, 1014, 1005, 13, 5, 0, 0, 0, 0, 0, 0, 977, 1091, 978, 1037, 1047, 1020, 985, 1046, 10, 5, 0, 0, 0, 0, 0, 0, 1006, 1021, 1015, 1034, 1029, 1045, 1036, 1064, 30, 0, 0, 0, 0, 0, 0, 0, 985, 1060, 1047, 1018, 1021, 996, 1045, 980, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8090, 8177, 8211, 8209, 8196, 8280, 8152] avg=8192.0000 max=8280 max_rank=6 min=8090 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1016, 1046, 1026, 1067, 1018, 1062, 1072, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1003, 1011, 1047, 1019, 1006, 1047, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1065, 1017, 1026, 1022, 1032, 1032, 1025, 3, 0, 0, 0, 0, 0, 0, 0, 1067, 1046, 1041, 1020, 1007, 1025, 999, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1006, 1043, 1007, 1031, 1022, 1055, 1052, 13, 5, 0, 0, 0, 0, 0, 0, 988, 1051, 1032, 1026, 1000, 996, 1016, 960, 9, 5, 0, 0, 0, 0, 0, 0, 992, 997, 976, 991, 1029, 1032, 1037, 1040, 30, 0, 0, 0, 0, 0, 0, 0, 1057, 1023, 988, 985, 1035, 981, 1017, 966, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8350, 8182, 8269, 8200, 8276, 8083, 8124, 8052] avg=8192.0000 max=8350 max_rank=0 min=8052 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32535 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1018, 1036, 1030, 973, 1011, 1018, 1010, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 996, 1057, 1044, 1029, 1055, 1001, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1033, 984, 1035, 1029, 1061, 1036, 1010, 3, 0, 0, 0, 0, 0, 0, 0, 1001, 1062, 1003, 1008, 1026, 1031, 1040, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1015, 989, 1027, 990, 984, 1019, 1046, 13, 5, 0, 0, 0, 0, 0, 0, 1046, 1035, 1041, 1034, 1035, 1021, 1047, 1081, 9, 5, 0, 0, 0, 0, 0, 0, 1003, 1003, 1051, 998, 988, 991, 1002, 978, 30, 0, 0, 0, 0, 0, 0, 0, 1040, 1020, 1022, 1069, 1008, 1075, 1002, 1000, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8198, 8264, 8213, 8129, 8354, 8044, 8236] avg=8192.0000 max=8354 max_rank=5 min=8044 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1043, 1013, 1031, 1021, 1046, 1021, 980, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1003, 1014, 991, 1040, 1015, 1060, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1000, 1040, 1027, 1041, 1007, 1019, 987, 9, 0, 0, 0, 0, 0, 0, 0, 1052, 1028, 1012, 1035, 1040, 1033, 1016, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 988, 996, 985, 991, 1002, 1024, 1016, 18, 8, 0, 0, 0, 0, 0, 0, 974, 1035, 1020, 989, 1031, 1032, 1070, 1019, 14, 9, 7, 0, 0, 0, 0, 0, 990, 1000, 1038, 1017, 1076, 1049, 1034, 998, 33, 0, 0, 0, 0, 0, 0, 0, 1029, 991, 1035, 1066, 1032, 1038, 1046, 1004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8195, 8134, 8245, 8077, 8200, 8235, 8241] avg=8192.0000 max=8245 max_rank=3 min=8077 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1043, 1045, 980, 1009, 1023, 1025, 1048, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1075, 959, 1047, 1067, 1027, 1063, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1058, 1036, 1018, 1032, 992, 1031, 1017, 4, 0, 0, 0, 0, 0, 0, 0, 1043, 1039, 986, 1034, 1009, 983, 1048, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1034, 1049, 960, 1025, 1030, 1045, 1007, 14, 4, 0, 0, 0, 0, 0, 0, 1001, 999, 1012, 1038, 1060, 1029, 995, 1000, 9, 5, 0, 0, 0, 0, 0, 0, 1035, 947, 1055, 1021, 1024, 961, 1034, 1054, 29, 0, 0, 0, 0, 0, 0, 0, 1059, 1021, 1018, 1029, 1045, 996, 1063, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8257, 8196, 8132, 8169, 8148, 8160, 8263] avg=8192.0000 max=8263 max_rank=7 min=8132 min_rank=3 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8320, 8192, 8064, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8192, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1093, 1047, 1000, 1008, 1008, 1024, 1048, 1002, 1051, 995, 1029, 1027, 1065, 1009, 994, 1070, 990, 1006, 1001, 1003, 1018, 1036, 1023, 1056, 1011, 1010, 1034, 1017, 1039, 1018, 1025, 976, 1001, 1008, 1076, 1028, 1015, 1075, 1022, 1044, 1028, 967, 1051, 963, 1030, 1024, 1031, 1012, 983, 1032, 1034, 997, 1012, 1012, 1064, 1049, 1027, 1048, 971, 1064, 1061, 1067, 1004, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1002, 1009, 1016, 1031, 1018, 1029, 1010, 1023, 1060, 994, 1036, 1031, 1022, 1016, 997, 1011, 1024, 973, 942, 1040, 1030, 1017, 1018, 1020, 1016, 990, 1048, 1044, 1012, 1039, 998, 1002, 1074, 1010, 976, 1072, 1052, 1036, 1020, 1062, 1027, 1039, 1063, 1057, 1038, 1000, 976, 1092, 1042, 1036, 1051, 979, 1057, 1083, 1017, 1050, 1043, 1002, 994, 1009, 1018, 1034, 1028, 981] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[999, 1084, 1044, 1024, 1048, 1001, 995, 1018, 1068, 1010, 1009, 1024, 1002, 1003, 1012, 1035, 1066, 994, 970, 1007, 1060, 1031, 1027, 986, 1026, 998, 1038, 1057, 1024, 963, 1043, 1054, 991, 1016, 1055, 1038, 1052, 1039, 1023, 1051, 995, 1059, 1009, 996, 1008, 1017, 972, 1023, 1063, 1043, 1002, 1030, 1067, 1011, 1031, 984, 1013, 976, 1062, 1042, 1032, 1049, 1064, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[998, 1049, 1039, 990, 989, 1010, 1047, 986, 991, 1058, 1020, 1068, 1001, 994, 1028, 1010, 944, 999, 1044, 998, 1069, 1043, 990, 1062, 1016, 1015, 1065, 999, 980, 1019, 1025, 1034, 1040, 1064, 1052, 1027, 1048, 1021, 1033, 1056, 1072, 979, 1052, 1068, 1002, 1010, 1039, 1024, 992, 1024, 1043, 998, 1014, 1017, 1022, 1058, 1028, 1025, 1013, 983, 1009, 1058, 1060, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[992, 1038, 974, 1026, 1013, 998, 1045, 1016, 1020, 1013, 1035, 1012, 1016, 1008, 1016, 1032, 1011, 1018, 1026, 1030, 1082, 1040, 1045, 1019, 987, 1033, 1023, 1012, 963, 1023, 989, 988, 1025, 1029, 1035, 1029, 1039, 1074, 1029, 1006, 1072, 1030, 1045, 1002, 1040, 983, 1039, 985, 1037, 1026, 1048, 1065, 997, 1032, 1033, 1024, 1051, 987, 1042, 1066, 1046, 1041, 987, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1020, 983, 1031, 1036, 1004, 1007, 1093, 1015, 999, 1007, 1079, 1066, 1048, 997, 1035, 1013, 1013, 1050, 1047, 1046, 1016, 1028, 1021, 996, 1028, 1008, 1021, 1023, 1005, 1042, 1015, 1037, 1034, 1047, 1042, 1038, 965, 1013, 985, 1023, 1054, 1030, 1045, 1033, 958, 1027, 975, 1058, 1005, 1018, 1022, 1037, 1002, 1022, 1051, 994, 1039, 1004, 1014, 1019, 1048, 1002, 1071, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1071, 1013, 1046, 1038, 1042, 1058, 961, 1005, 1021, 991, 1032, 1016, 992, 1052, 989, 1039, 1050, 1035, 1010, 1022, 1012, 993, 1035, 1017, 1000, 1027, 1042, 992, 1017, 1055, 1034, 1029, 1061, 1035, 1031, 1009, 1068, 1013, 982, 1063, 1021, 1016, 1024, 1044, 999, 1018, 970, 1053, 1042, 1009, 1042, 1075, 987, 1009, 1024, 1057, 1025, 1060, 1013, 982, 1013, 1003, 1037, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1032, 982, 1057, 991, 1032, 1037, 1008, 1037, 1001, 1044, 1063, 1004, 1022, 1077, 1023, 1031, 1005, 988, 1012, 1021, 1045, 1047, 1004, 1039, 1026, 1040, 997, 1025, 1030, 1016, 1049, 1018, 977, 979, 1003, 983, 1072, 1067, 1022, 1059, 1036, 975, 1037, 1077, 1014, 1004, 1014, 1115, 988, 1014, 1056, 1023, 977, 966, 1031, 995, 1035, 1029, 959, 1075, 1023, 1030, 1057, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1002, 1009, 1016, 1031, 1018, 1029, 1010, 1023, 12, 5, 0, 0, 0, 0, 0, 0, 1060, 994, 1036, 1031, 1022, 1016, 997, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 973, 942, 1040, 1030, 1017, 1018, 1020, 21, 6, 0, 0, 0, 0, 0, 0, 1016, 990, 1048, 1044, 1012, 1039, 998, 1002, 38, 0, 0, 0, 0, 0, 0, 0, 1074, 1010, 976, 1072, 1052, 1036, 1020, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1039, 1063, 1057, 1038, 1000, 976, 1092, 2, 0, 0, 0, 0, 0, 0, 0, 1042, 1036, 1044, 979, 1057, 1083, 1017, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1002, 994, 1009, 1018, 1034, 1010, 981, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8167, 8091, 8187, 8243, 8294, 8308, 8091] avg=8192.0000 max=8308 max_rank=6 min=8091 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1093, 1047, 1000, 1008, 1008, 1024, 1048, 1002, 12, 5, 0, 0, 0, 0, 0, 0, 1051, 995, 1029, 1027, 1065, 1009, 994, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1006, 1001, 1003, 1018, 1036, 1023, 1056, 21, 6, 0, 0, 0, 0, 0, 0, 1011, 1010, 1034, 1017, 1039, 1018, 1025, 976, 38, 0, 0, 0, 0, 0, 0, 0, 1001, 1008, 1076, 1028, 1015, 1075, 1022, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 967, 1051, 963, 1030, 1024, 1031, 1012, 2, 0, 0, 0, 0, 0, 0, 0, 983, 1032, 1027, 997, 1012, 1012, 1064, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1048, 971, 1064, 1061, 1067, 986, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8240, 8160, 8168, 8210, 8108, 8176, 8227] avg=8192.0000 max=8247 max_rank=0 min=8108 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[992, 1038, 974, 1026, 1013, 998, 1045, 1016, 11, 5, 0, 0, 0, 0, 0, 0, 1020, 1013, 1035, 1012, 1016, 1008, 1016, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1018, 1026, 1030, 1082, 1040, 1045, 1019, 20, 6, 0, 0, 0, 0, 0, 0, 987, 1033, 1023, 1012, 963, 1023, 989, 988, 36, 0, 0, 0, 0, 0, 0, 0, 1025, 1029, 1035, 1029, 1039, 1074, 1029, 950, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1030, 1045, 1002, 1040, 983, 1039, 985, 2, 0, 0, 0, 0, 0, 0, 0, 1037, 1026, 1041, 1065, 997, 1032, 1033, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 987, 1042, 1066, 1046, 1041, 970, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8152, 8297, 8054, 8210, 8198, 8255, 8252] avg=8192.0000 max=8297 max_rank=2 min=8054 min_rank=3 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32915 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1071, 1013, 1046, 1038, 1042, 1058, 961, 1005, 12, 5, 0, 0, 0, 0, 0, 0, 1021, 991, 1032, 1016, 992, 1052, 989, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1035, 1010, 1022, 1012, 993, 1035, 1017, 21, 6, 0, 0, 0, 0, 0, 0, 1000, 1027, 1042, 992, 1017, 1055, 1034, 1029, 39, 0, 0, 0, 0, 0, 0, 0, 1061, 1035, 1031, 1009, 1068, 1013, 982, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1016, 1024, 1044, 999, 1018, 970, 1053, 2, 0, 0, 0, 0, 0, 0, 0, 1042, 1009, 1035, 1075, 987, 1009, 1024, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1060, 1013, 982, 1013, 1003, 1019, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8132, 8201, 8235, 8202, 8147, 8238, 8130] avg=8192.0000 max=8251 max_rank=0 min=8130 min_rank=7 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[999, 1084, 1044, 1024, 1048, 1001, 995, 1018, 12, 5, 0, 0, 0, 0, 0, 0, 1068, 1010, 1009, 1024, 1002, 1003, 1012, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 994, 970, 1007, 1060, 1031, 1027, 986, 21, 7, 0, 0, 0, 0, 0, 0, 1026, 998, 1038, 1057, 1024, 963, 1043, 1054, 38, 0, 0, 0, 0, 0, 0, 0, 991, 1016, 1055, 1038, 1052, 1039, 1023, 992, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1059, 1009, 996, 1008, 1017, 972, 1023, 2, 0, 0, 0, 0, 0, 0, 0, 1063, 1043, 995, 1030, 1067, 1011, 1031, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 976, 1062, 1042, 1032, 1049, 1045, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8163, 8169, 8241, 8206, 8081, 8224, 8222] avg=8192.0000 max=8241 max_rank=3 min=8081 min_rank=5 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1020, 983, 1031, 1036, 1004, 1007, 1093, 1015, 12, 5, 0, 0, 0, 0, 0, 0, 999, 1007, 1079, 1066, 1048, 997, 1035, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1050, 1047, 1046, 1016, 1028, 1021, 996, 20, 7, 0, 0, 0, 0, 0, 0, 1028, 1008, 1021, 1023, 1005, 1042, 1015, 1037, 37, 0, 0, 0, 0, 0, 0, 0, 1034, 1047, 1042, 1038, 965, 1013, 985, 966, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1030, 1045, 1033, 958, 1027, 975, 1058, 2, 0, 0, 0, 0, 0, 0, 0, 1005, 1018, 1015, 1037, 1002, 1022, 1051, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1004, 1014, 1019, 1048, 1002, 1052, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8244, 8244, 8216, 8090, 8182, 8144, 8210] avg=8192.0000 max=8244 max_rank=1 min=8090 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[998, 1049, 1039, 990, 989, 1010, 1047, 986, 17, 11, 0, 0, 0, 0, 0, 0, 991, 1058, 1020, 1068, 1001, 994, 1028, 1010, 3, 0, 0, 0, 0, 0, 0, 0, 944, 999, 1044, 998, 1069, 1043, 990, 1062, 26, 11, 0, 0, 0, 0, 0, 0, 1016, 1015, 1065, 999, 980, 1019, 1025, 1034, 43, 0, 0, 0, 0, 0, 0, 0, 1040, 1064, 1052, 1027, 1048, 1021, 1033, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 979, 1052, 1068, 1002, 1010, 1039, 1024, 6, 0, 0, 0, 0, 0, 0, 0, 992, 1024, 1023, 998, 1014, 1017, 1022, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1025, 1013, 983, 1009, 1058, 1032, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8173, 8186, 8196, 8272, 8252, 8148, 8173] avg=8192.0000 max=8272 max_rank=4 min=8136 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1032, 982, 1057, 991, 1032, 1037, 1008, 1037, 12, 5, 0, 0, 0, 0, 0, 0, 1001, 1044, 1063, 1004, 1022, 1077, 1023, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 988, 1012, 1021, 1045, 1047, 1004, 1039, 21, 6, 0, 0, 0, 0, 0, 0, 1026, 1040, 997, 1025, 1030, 1016, 1049, 1018, 38, 0, 0, 0, 0, 0, 0, 0, 977, 979, 1003, 983, 1072, 1067, 1022, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 975, 1037, 1077, 1014, 1004, 1014, 1115, 2, 0, 0, 0, 0, 0, 0, 0, 988, 1014, 1049, 1023, 977, 966, 1031, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1029, 959, 1075, 1023, 1030, 1039, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8265, 8188, 8239, 8103, 8274, 8043, 8231] avg=8192.0000 max=8274 max_rank=5 min=8043 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8064, 8192, 8448, 8320, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8448, 8320, 8192, 8192, 8064, 8448, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8448, 8192, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1001, 1001, 977, 1067, 1035, 1011, 997, 1042, 1024, 1062, 996, 1038, 1009, 1017, 1006, 1024, 1053, 1035, 1075, 1003, 994, 994, 1008, 1031, 979, 988, 1054, 968, 1016, 1050, 1000, 1011, 1050, 1043, 1056, 1025, 1012, 1067, 1013, 1027, 1051, 1064, 1038, 1042, 1029, 1056, 1071, 1026, 1007, 1020, 1063, 984, 1034, 1028, 1032, 1026, 1032, 974, 1038, 1000, 1009, 1046, 1015, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1032, 1030, 1040, 1044, 1041, 1051, 1053, 1031, 1001, 994, 1018, 1024, 979, 1007, 1042, 1016, 996, 993, 1027, 1010, 1079, 1017, 1013, 1026, 1029, 1027, 1010, 1015, 1029, 1045, 1018, 1038, 1057, 1020, 1034, 1006, 1028, 997, 975, 1003, 1062, 1026, 996, 1001, 1069, 1026, 1095, 1034, 1003, 1028, 1029, 1017, 1019, 1062, 1013, 970, 1039, 986, 1056, 973, 1044, 1027, 1063, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1002, 1029, 1035, 1069, 1054, 1049, 998, 1030, 1019, 999, 1060, 995, 1011, 965, 1015, 1005, 987, 1023, 1023, 1095, 995, 1008, 1052, 1029, 1009, 1072, 1032, 1042, 1040, 983, 1077, 1035, 1095, 996, 1022, 986, 986, 1034, 1070, 1057, 1016, 1031, 994, 1020, 999, 984, 1043, 1019, 1001, 1061, 1070, 980, 1001, 1014, 1023, 1067, 1034, 1035, 997, 1025, 993, 1010, 1017, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[990, 1002, 1034, 1093, 1028, 991, 1006, 1034, 994, 1035, 1018, 1024, 1019, 1064, 1045, 1083, 986, 1080, 1032, 977, 1036, 1052, 1033, 973, 1038, 1028, 1049, 1026, 1029, 1052, 1005, 993, 1023, 1009, 1052, 1058, 993, 972, 980, 1029, 987, 1063, 1017, 1027, 1030, 1025, 1014, 1022, 1010, 1048, 1070, 1019, 997, 1024, 1029, 986, 1021, 983, 1048, 1002, 1019, 1051, 1063, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1011, 991, 1060, 1032, 1008, 1040, 1009, 1017, 985, 1086, 1002, 965, 1033, 1038, 1040, 980, 1037, 1059, 1007, 1002, 1071, 1044, 1017, 997, 1041, 1032, 1036, 1068, 1050, 998, 1048, 1053, 1010, 1035, 1005, 1018, 1034, 1020, 983, 1014, 1002, 1014, 1028, 1035, 1009, 999, 1051, 1049, 1009, 1031, 988, 1058, 1026, 1002, 1038, 1009, 1037, 1091, 1036, 1008, 1001, 1032, 941, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1029, 1000, 1060, 1000, 983, 1006, 1076, 1036, 1013, 1071, 1015, 1033, 1042, 1023, 1037, 1041, 1039, 1023, 1050, 1017, 1012, 942, 979, 979, 1050, 1014, 1011, 1036, 990, 1068, 982, 1021, 991, 1014, 1030, 1068, 1057, 1012, 1076, 1059, 1014, 1032, 1004, 1031, 997, 984, 998, 1031, 982, 1075, 1083, 1076, 1056, 1027, 1031, 986, 1032, 1013, 1011, 1034, 1001, 1043, 990, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1018, 1030, 1038, 1085, 1015, 1092, 1002, 1038, 1007, 1014, 1033, 998, 1058, 1018, 973, 949, 998, 1011, 1033, 1061, 1034, 1005, 1035, 1036, 1063, 1002, 1069, 1011, 1032, 1007, 999, 1018, 1049, 1037, 1014, 1069, 991, 1002, 984, 1041, 1046, 1013, 1014, 980, 1009, 976, 1066, 996, 1067, 1059, 1059, 1041, 1074, 1009, 1039, 1016, 976, 982, 1026, 1044, 994, 1034, 1027, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1040, 1061, 986, 992, 1021, 1062, 1026, 997, 1031, 1039, 1016, 990, 992, 1071, 1102, 1032, 976, 982, 1015, 997, 1037, 1076, 1013, 1032, 1007, 1013, 1030, 1046, 987, 1014, 1041, 1046, 1006, 1036, 1000, 945, 1039, 1011, 1050, 988, 1034, 1011, 1025, 1028, 1063, 983, 1013, 1024, 1007, 1036, 1031, 1009, 1034, 976, 1043, 1008, 1021, 1067, 1077, 1006, 1045, 1054, 1066, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1001, 1001, 977, 1040, 1035, 1011, 997, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1062, 996, 1038, 1009, 1017, 1006, 1024, 25, 0, 0, 0, 0, 0, 0, 0, 1053, 1035, 1075, 1003, 994, 994, 1008, 1031, 20, 2, 0, 0, 0, 0, 0, 0, 979, 988, 1038, 968, 1016, 1050, 1000, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1043, 1056, 1025, 1012, 1067, 1013, 1027, 9, 0, 0, 0, 0, 0, 0, 0, 1051, 1064, 1038, 1042, 1029, 1056, 1071, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1020, 1041, 984, 1034, 1028, 1032, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 974, 1038, 1000, 1009, 1046, 1015, 992, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8201, 8215, 8050, 8302, 8377, 8172, 8115] avg=8192.0000 max=8377 max_rank=5 min=8050 min_rank=3 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32966 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1032, 1030, 1040, 1017, 1041, 1051, 1053, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 994, 1018, 1024, 979, 1007, 1042, 1016, 25, 0, 0, 0, 0, 0, 0, 0, 996, 993, 1027, 1010, 1079, 1017, 1013, 1026, 20, 2, 0, 0, 0, 0, 0, 0, 1029, 1027, 995, 1015, 1029, 1045, 1018, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1020, 1034, 1006, 1028, 997, 975, 1003, 8, 0, 0, 0, 0, 0, 0, 0, 1062, 1026, 996, 1001, 1069, 1026, 1095, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1028, 1007, 1017, 1019, 1062, 1013, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 986, 1056, 973, 1044, 1027, 1063, 1003, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8106, 8183, 8196, 8128, 8309, 8119, 8200] avg=8192.0000 max=8309 max_rank=5 min=8106 min_rank=1 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32756 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1011, 991, 1060, 1006, 1008, 1040, 1009, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1086, 1002, 965, 1033, 1038, 1040, 980, 24, 0, 0, 0, 0, 0, 0, 0, 1037, 1059, 1007, 1002, 1071, 1044, 1017, 997, 19, 2, 0, 0, 0, 0, 0, 0, 1041, 1032, 1020, 1068, 1050, 998, 1048, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1035, 1005, 1018, 1034, 1020, 983, 1014, 9, 0, 0, 0, 0, 0, 0, 0, 1002, 1014, 1028, 1035, 1009, 999, 1051, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1031, 967, 1058, 1026, 1002, 1038, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1091, 1036, 1008, 1001, 1032, 941, 1066, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8153, 8255, 8310, 8128, 8187, 8140, 8221] avg=8192.0000 max=8310 max_rank=3 min=8128 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32676 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1002, 1029, 1035, 1042, 1054, 1049, 998, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 999, 1060, 995, 1011, 965, 1015, 1005, 25, 0, 0, 0, 0, 0, 0, 0, 987, 1023, 1023, 1095, 995, 1008, 1052, 1029, 20, 2, 0, 0, 0, 0, 0, 0, 1009, 1072, 1016, 1042, 1040, 983, 1077, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 996, 1022, 986, 986, 1034, 1070, 1057, 9, 0, 0, 0, 0, 0, 0, 0, 1016, 1031, 994, 1020, 999, 984, 1043, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1061, 1048, 980, 1001, 1014, 1023, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1035, 997, 1025, 993, 1010, 1017, 1019, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8094, 8234, 8274, 8255, 8106, 8195, 8139] avg=8192.0000 max=8274 max_rank=3 min=8094 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32695 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[990, 1002, 1034, 1065, 1028, 991, 1006, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1035, 1018, 1024, 1019, 1064, 1045, 1083, 26, 0, 0, 0, 0, 0, 0, 0, 986, 1080, 1032, 977, 1036, 1052, 1033, 973, 20, 2, 0, 0, 0, 0, 0, 0, 1038, 1028, 1033, 1026, 1029, 1052, 1005, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1009, 1052, 1058, 993, 972, 980, 1029, 9, 0, 0, 0, 0, 0, 0, 0, 987, 1063, 1017, 1027, 1030, 1025, 1014, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1048, 1048, 1019, 997, 1024, 1029, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 983, 1048, 1002, 1019, 1051, 1063, 1016, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8308, 8191, 8204, 8125, 8185, 8161, 8212] avg=8192.0000 max=8308 max_rank=1 min=8125 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32683 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1029, 1000, 1060, 975, 983, 1006, 1076, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1071, 1015, 1033, 1042, 1023, 1037, 1041, 23, 0, 0, 0, 0, 0, 0, 0, 1039, 1023, 1050, 1017, 1012, 942, 979, 979, 21, 2, 0, 0, 0, 0, 0, 0, 1050, 1014, 996, 1036, 990, 1068, 982, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1014, 1030, 1068, 1057, 1012, 1076, 1059, 8, 0, 0, 0, 0, 0, 0, 0, 1014, 1032, 1004, 1031, 997, 984, 998, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1075, 1060, 1076, 1056, 1027, 1031, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1013, 1011, 1034, 1001, 1043, 990, 1020, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8298, 8064, 8157, 8315, 8091, 8293, 8153] avg=8192.0000 max=8315 max_rank=4 min=8064 min_rank=2 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32852 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1040, 1061, 986, 958, 1021, 1062, 1026, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1039, 1016, 990, 992, 1071, 1102, 1032, 27, 0, 0, 0, 0, 0, 0, 0, 976, 982, 1015, 997, 1037, 1076, 1013, 1032, 24, 7, 0, 0, 0, 0, 0, 0, 1007, 1013, 1006, 1046, 987, 1014, 1041, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1036, 1000, 945, 1039, 1011, 1050, 988, 12, 0, 0, 0, 0, 0, 0, 0, 1034, 1011, 1025, 1028, 1063, 983, 1013, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1036, 1001, 1009, 1034, 976, 1043, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1067, 1077, 1006, 1045, 1054, 1066, 1030, 12, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8300, 8159, 8160, 8087, 8181, 8114, 8384] avg=8192.0000 max=8384 max_rank=7 min=8087 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=32766 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1018, 1030, 1038, 1057, 1015, 1092, 1002, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1014, 1033, 998, 1058, 1018, 973, 949, 26, 0, 0, 0, 0, 0, 0, 0, 998, 1011, 1033, 1061, 1034, 1005, 1035, 1036, 20, 2, 0, 0, 0, 0, 0, 0, 1063, 1002, 1053, 1011, 1032, 1007, 999, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1037, 1014, 1069, 991, 1002, 984, 1041, 9, 0, 0, 0, 0, 0, 0, 0, 1046, 1013, 1014, 980, 1009, 976, 1066, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1059, 1037, 1041, 1074, 1009, 1039, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 976, 982, 1026, 1044, 994, 1034, 1027, 1020, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 8076, 8235, 8185, 8196, 8100, 8342, 8112] avg=8192.0000 max=8342 max_rank=6 min=8076 min_rank=1 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32750 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8064, 8448, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1054, 1010, 1040, 954, 1047, 1063, 982, 992, 1045, 1030, 1016, 1107, 984, 956, 984, 1031, 1025, 972, 1034, 984, 1041, 1018, 981, 1032, 1005, 1029, 1041, 1031, 1001, 1023, 977, 1051, 1029, 1057, 1041, 985, 1002, 1002, 1074, 1068, 1047, 1070, 1017, 1055, 1024, 1037, 995, 1048, 995, 1031, 1002, 1043, 1023, 1050, 1031, 1034, 1037, 1031, 973, 1051, 997, 1073, 1022, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[999, 1009, 1073, 1013, 1062, 1015, 988, 1017, 987, 1032, 1044, 1003, 1036, 1042, 1063, 1032, 993, 1060, 1004, 999, 1092, 986, 1049, 1037, 1003, 1015, 1055, 1020, 1006, 988, 1000, 986, 1016, 1009, 1060, 1059, 1063, 1061, 1041, 995, 1029, 992, 1014, 1057, 986, 1004, 1015, 997, 1026, 1061, 1023, 1026, 1052, 1016, 1034, 1039, 1028, 1056, 1043, 1021, 1015, 967, 1044, 979] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1032, 1018, 999, 1029, 944, 1016, 1039, 1058, 1006, 1065, 1016, 1021, 1078, 968, 1015, 983, 1041, 997, 1026, 1024, 1044, 989, 1058, 999, 1024, 1018, 1015, 1055, 1034, 1003, 1066, 1034, 1033, 976, 1050, 1009, 1065, 1019, 1047, 1016, 1041, 1018, 1040, 982, 1056, 1057, 1021, 1054, 1015, 1006, 1030, 1001, 1015, 1033, 1031, 1057, 1032, 1042, 1042, 1034, 1019, 999, 995, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1019, 1028, 990, 989, 1080, 1034, 1064, 1015, 1043, 1038, 1048, 1062, 1040, 1042, 1024, 1044, 968, 1029, 955, 1040, 972, 1071, 1030, 1012, 1036, 1011, 992, 1078, 1079, 1022, 1035, 1019, 1060, 996, 991, 1039, 989, 1012, 1024, 988, 1005, 998, 1054, 1006, 982, 1011, 994, 1042, 1044, 1016, 1008, 1025, 988, 1054, 974, 1026, 1011, 1053, 1063, 1019, 1020, 1023, 1070, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[988, 1012, 1029, 1083, 1036, 1037, 986, 1050, 1017, 1021, 1005, 996, 1017, 1043, 1043, 1048, 1033, 987, 1018, 1029, 1075, 1023, 1061, 1092, 1033, 1046, 1103, 1037, 1063, 1038, 1066, 986, 999, 1031, 1021, 991, 1001, 997, 1040, 1033, 1046, 974, 1044, 987, 995, 975, 1016, 1060, 984, 1025, 963, 1003, 1025, 1055, 1019, 1037, 975, 1024, 1001, 1014, 1012, 988, 1069, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1090, 987, 1004, 985, 1052, 1024, 1011, 1089, 1014, 1005, 992, 1056, 1030, 1007, 1048, 945, 993, 1057, 1027, 1039, 1035, 988, 1027, 1024, 1051, 1068, 1028, 1076, 1043, 1013, 995, 1039, 1054, 1065, 1016, 993, 984, 1012, 1037, 1016, 1023, 1010, 1005, 1064, 995, 1071, 1059, 1020, 1004, 966, 981, 1043, 1004, 1019, 1037, 1082, 1062, 1001, 981, 984, 1001, 1036, 1022, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1063, 987, 1031, 1049, 1020, 1025, 1054, 1008, 1067, 1028, 1013, 1014, 1057, 1019, 1019, 975, 976, 992, 984, 977, 1045, 1027, 1000, 1080, 1005, 1022, 1034, 1044, 965, 1085, 969, 1057, 1018, 1037, 999, 1006, 1043, 1057, 1051, 1099, 1026, 1021, 984, 1039, 1007, 1012, 1011, 1022, 988, 1003, 1001, 1066, 1038, 1029, 1016, 1020, 1032, 1053, 1065, 1025, 1045, 1009, 1020, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1035, 1035, 1017, 1008, 1027, 1061, 995, 1037, 998, 985, 1009, 1048, 1024, 1064, 1021, 1062, 1028, 989, 994, 1046, 1002, 988, 1046, 1034, 1008, 1071, 988, 1046, 1018, 1068, 1029, 1009, 1008, 998, 1010, 990, 1037, 980, 1046, 1067, 997, 1022, 1035, 994, 1000, 1065, 1026, 1043, 1011, 1045, 1028, 1055, 1027, 1013, 1010, 1028, 1044, 1015, 1005, 1047, 1056, 1038, 1000, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[994, 1009, 1073, 1013, 1062, 1015, 988, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1032, 1044, 999, 1036, 1042, 1063, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1060, 1004, 999, 1092, 986, 1049, 1037, 31, 0, 0, 0, 0, 0, 0, 0, 1003, 1015, 1055, 982, 1006, 988, 1000, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1009, 1060, 1059, 1063, 1061, 1032, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 992, 1014, 1057, 986, 1004, 1015, 997, 9, 7, 0, 0, 0, 0, 0, 0, 1026, 1061, 1023, 1026, 1052, 1016, 1034, 1039, 5, 4, 1, 0, 0, 0, 0, 0, 1028, 1055, 1043, 1021, 1015, 967, 1044, 979, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8235, 8251, 8035, 8295, 8110, 8287, 8152] avg=8192.0000 max=8295 max_rank=4 min=8035 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1048, 1010, 1040, 954, 1047, 1063, 982, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1030, 1016, 1102, 984, 956, 984, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 972, 1034, 984, 1041, 1018, 981, 1032, 31, 0, 0, 0, 0, 0, 0, 0, 1005, 1029, 1041, 993, 1001, 1023, 977, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1057, 1041, 985, 1002, 1002, 1065, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1070, 1017, 1055, 1024, 1037, 995, 1048, 9, 7, 0, 0, 0, 0, 0, 0, 995, 1031, 1002, 1043, 1023, 1050, 1031, 1034, 6, 5, 1, 0, 0, 0, 0, 0, 1037, 1030, 973, 1051, 997, 1073, 1022, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8148, 8118, 8120, 8249, 8309, 8221, 8235] avg=8192.0000 max=8309 max_rank=5 min=8118 min_rank=2 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=33014 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1084, 987, 1004, 985, 1052, 1024, 1011, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1005, 992, 1052, 1030, 1007, 1048, 945, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1057, 1027, 1039, 1035, 988, 1027, 1024, 33, 0, 0, 0, 0, 0, 0, 0, 1051, 1068, 1028, 1036, 1043, 1013, 995, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1065, 1016, 993, 984, 1012, 1028, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1010, 1005, 1064, 995, 1071, 1059, 1020, 9, 7, 0, 0, 0, 0, 0, 0, 1004, 966, 981, 1043, 1004, 1019, 1037, 1082, 6, 4, 1, 0, 0, 0, 0, 0, 1062, 1000, 981, 984, 1001, 1036, 1022, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8093, 8223, 8273, 8168, 8263, 8147, 8133] avg=8192.0000 max=8273 max_rank=3 min=8093 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1027, 1018, 999, 1029, 944, 1016, 1039, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1065, 1016, 1017, 1078, 968, 1015, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 997, 1026, 1024, 1044, 989, 1058, 999, 32, 0, 0, 0, 0, 0, 0, 0, 1024, 1018, 1015, 1016, 1034, 1003, 1066, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 976, 1050, 1009, 1065, 1019, 1038, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1018, 1040, 982, 1056, 1057, 1021, 1054, 9, 7, 0, 0, 0, 0, 0, 0, 1015, 1006, 1030, 1001, 1015, 1033, 1031, 1057, 5, 4, 1, 0, 0, 0, 0, 0, 1032, 1041, 1042, 1034, 1019, 999, 995, 987, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8148, 8210, 8210, 8206, 8285, 8198, 8149] avg=8192.0000 max=8285 max_rank=5 min=8130 min_rank=0 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1007, 1028, 990, 989, 1080, 1034, 1064, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1038, 1048, 1052, 1040, 1042, 1024, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1029, 955, 1040, 972, 1071, 1030, 1012, 36, 0, 0, 0, 0, 0, 0, 0, 1036, 1011, 992, 1030, 1079, 1022, 1035, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 996, 991, 1039, 989, 1012, 1011, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 998, 1054, 1006, 982, 1011, 994, 1042, 13, 12, 3, 0, 0, 0, 0, 0, 1044, 1016, 1008, 1025, 988, 1054, 974, 1026, 9, 10, 8, 0, 0, 0, 0, 0, 1011, 1045, 1063, 1019, 1020, 1023, 1070, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8331, 8113, 8224, 8086, 8120, 8162, 8293] avg=8192.0000 max=8331 max_rank=1 min=8086 min_rank=4 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[983, 1012, 1029, 1083, 1036, 1037, 986, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1021, 1005, 992, 1017, 1043, 1043, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 987, 1018, 1029, 1075, 1023, 1061, 1092, 31, 0, 0, 0, 0, 0, 0, 0, 1033, 1046, 1103, 999, 1063, 1038, 1066, 986, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1031, 1021, 991, 1001, 997, 1031, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 974, 1044, 987, 995, 975, 1016, 1060, 9, 7, 0, 0, 0, 0, 0, 0, 984, 1025, 963, 1003, 1025, 1055, 1019, 1037, 5, 4, 1, 0, 0, 0, 0, 0, 975, 1023, 1001, 1014, 1012, 988, 1069, 1031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8186, 8349, 8334, 8104, 8113, 8121, 8113] avg=8192.0000 max=8349 max_rank=2 min=8104 min_rank=4 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1057, 987, 1031, 1049, 1020, 1025, 1054, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1028, 1013, 1010, 1057, 1019, 1019, 975, 0, 0, 0, 0, 0, 0, 0, 0, 976, 992, 984, 977, 1045, 1027, 1000, 1080, 32, 0, 0, 0, 0, 0, 0, 0, 1005, 1022, 1034, 1005, 965, 1085, 969, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1037, 999, 1006, 1043, 1057, 1042, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1021, 984, 1039, 1007, 1012, 1011, 1022, 9, 7, 0, 0, 0, 0, 0, 0, 988, 1003, 1001, 1066, 1038, 1029, 1016, 1020, 6, 4, 1, 0, 0, 0, 0, 0, 1032, 1052, 1065, 1025, 1045, 1009, 1020, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8188, 8113, 8142, 8301, 8138, 8172, 8251] avg=8192.0000 max=8301 max_rank=4 min=8113 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32862 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1029, 1035, 1017, 1008, 1027, 1061, 995, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 998, 985, 1009, 1044, 1024, 1064, 1021, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 989, 994, 1046, 1002, 988, 1046, 1034, 32, 0, 0, 0, 0, 0, 0, 0, 1008, 1071, 988, 1007, 1018, 1068, 1029, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 998, 1010, 990, 1037, 980, 1037, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1022, 1035, 994, 1000, 1065, 1026, 1043, 9, 7, 0, 0, 0, 0, 0, 0, 1011, 1045, 1028, 1055, 1027, 1013, 1010, 1028, 6, 4, 1, 0, 0, 0, 0, 0, 1044, 1014, 1005, 1047, 1056, 1038, 1000, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8207, 8159, 8198, 8127, 8198, 8228, 8210] avg=8192.0000 max=8228 max_rank=6 min=8127 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8064, 8320, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8064, 8320, 8192, 8320, 8192, 8448, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8064, 8192, 8320, 8192, 8320, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1044, 982, 1026, 1015, 1011, 1021, 1019, 982, 1000, 1036, 999, 988, 1029, 1075, 1056, 1049, 1082, 1014, 1075, 945, 994, 1037, 982, 1027, 1032, 1004, 1049, 998, 1055, 995, 1018, 1013, 1071, 1041, 1031, 1031, 1014, 1002, 994, 1083, 1056, 1064, 1055, 1050, 1002, 988, 1061, 1029, 953, 1075, 978, 1034, 974, 1028, 1001, 1016, 1058, 1007, 1054, 1015, 1036, 1006, 1047, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1027, 1024, 1016, 1052, 1039, 1004, 1016, 1039, 1025, 1032, 988, 1034, 1055, 1038, 1064, 1058, 967, 1029, 1061, 976, 1017, 1018, 1006, 1044, 1013, 1010, 1029, 1008, 1113, 1024, 1003, 1035, 1038, 1057, 994, 1000, 1001, 998, 1007, 991, 1041, 1003, 1031, 1006, 1046, 1015, 1038, 1041, 990, 1008, 1043, 965, 1011, 1008, 1042, 1034, 1026, 1047, 995, 1037, 1042, 1066, 1016, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1035, 1011, 1015, 1020, 1031, 1027, 1098, 1045, 978, 1034, 971, 1067, 1026, 986, 1057, 1007, 1027, 1064, 1034, 1043, 1023, 1020, 1043, 1004, 992, 998, 1038, 1021, 1010, 1029, 1046, 1050, 989, 1061, 1028, 966, 1031, 1023, 1036, 979, 1053, 1012, 1036, 977, 1027, 1014, 1038, 1015, 1028, 991, 1039, 1057, 1019, 1005, 1056, 1021, 1018, 1015, 1028, 1119, 1025, 979, 1026, 975] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1054, 1053, 1042, 1003, 1034, 1029, 1017, 1054, 1023, 1065, 1014, 1038, 1039, 1032, 1026, 1023, 1018, 1029, 1032, 1067, 1037, 1026, 991, 1016, 1079, 997, 1031, 1043, 990, 994, 975, 1005, 1056, 1054, 1004, 997, 1058, 1004, 1067, 1051, 1029, 982, 986, 1057, 984, 988, 1012, 1046, 1033, 973, 998, 1010, 991, 1034, 1005, 979, 1034, 1063, 1016, 1052, 992, 1085, 1003, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[971, 1028, 994, 1099, 1050, 1012, 1003, 1040, 1048, 1004, 1028, 997, 979, 1005, 1009, 1038, 1005, 1011, 1064, 1000, 1045, 1022, 981, 1011, 1024, 969, 1025, 1062, 1035, 1002, 1045, 1000, 1011, 1038, 1072, 1019, 1004, 1078, 1039, 1057, 1027, 1019, 1028, 1061, 1022, 1027, 1011, 1014, 995, 1085, 1002, 1024, 1026, 1009, 1000, 1053, 1019, 1046, 1003, 1041, 989, 1022, 1025, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1011, 1045, 1026, 1022, 940, 1068, 1048, 1006, 1033, 990, 1031, 1027, 1035, 1014, 1045, 976, 1018, 1028, 1015, 1076, 1018, 978, 1072, 1030, 1033, 1045, 1041, 1004, 1005, 982, 1045, 996, 1013, 1021, 991, 964, 1000, 1023, 1013, 1019, 1049, 1036, 1045, 1023, 1001, 1069, 1027, 1066, 1034, 1018, 972, 1067, 1048, 1051, 1030, 1038, 986, 1017, 1020, 997, 1083, 1016, 1069, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1005, 1049, 1054, 1040, 986, 1023, 1029, 1004, 1003, 1013, 1004, 998, 1024, 999, 1030, 969, 974, 1033, 1067, 1019, 1060, 1043, 1008, 1005, 1035, 1011, 1005, 1018, 994, 1033, 1079, 987, 1003, 1015, 1032, 1044, 1027, 998, 1038, 1019, 1015, 1074, 1014, 1056, 1027, 1033, 1087, 1000, 1050, 996, 1028, 1043, 1058, 1034, 1027, 1040, 1003, 1054, 995, 1033, 1027, 1023, 1059, 983] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[999, 1014, 1036, 993, 1039, 999, 1027, 1028, 1029, 1021, 1031, 974, 1064, 993, 1044, 991, 1056, 989, 1034, 1077, 1017, 1028, 1041, 1029, 986, 1025, 1038, 1056, 1085, 996, 988, 1080, 1062, 1012, 1037, 1010, 1038, 1030, 1025, 1053, 995, 1037, 1040, 1065, 1030, 999, 1025, 1050, 1001, 1019, 997, 1010, 1054, 962, 1012, 1016, 1024, 991, 995, 1052, 1026, 1040, 1003, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[971, 1028, 994, 1099, 1050, 1012, 999, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1004, 1028, 997, 979, 1005, 1009, 1038, 19, 2, 0, 0, 0, 0, 0, 0, 1005, 1011, 1057, 1000, 1045, 1022, 981, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 969, 1025, 1062, 1035, 1002, 1045, 1000, 5, 4, 3, 0, 0, 0, 0, 0, 1011, 1035, 1072, 1019, 1004, 1078, 1039, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1019, 1028, 1061, 1022, 1027, 973, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1085, 1002, 1024, 1026, 1009, 1000, 1053, 38, 2, 0, 0, 0, 0, 0, 0, 1019, 1046, 1003, 1020, 989, 1022, 1025, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8129, 8132, 8174, 8315, 8171, 8234, 8188] avg=8192.0000 max=8315 max_rank=4 min=8129 min_rank=1 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32908 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1027, 1024, 1016, 1052, 1039, 1004, 1012, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1032, 988, 1034, 1055, 1038, 1064, 1058, 19, 2, 0, 0, 0, 0, 0, 0, 967, 1029, 1054, 976, 1017, 1018, 1006, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1010, 1029, 1008, 1113, 1024, 1003, 1035, 5, 4, 3, 0, 0, 0, 0, 0, 1038, 1054, 994, 1000, 1001, 998, 1007, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1003, 1031, 1006, 1046, 1015, 999, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1008, 1043, 965, 1011, 1008, 1042, 1034, 39, 2, 0, 0, 0, 0, 0, 0, 1026, 1047, 995, 1016, 1042, 1066, 1016, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8315, 8111, 8247, 8083, 8182, 8142, 8243] avg=8192.0000 max=8315 max_rank=1 min=8083 min_rank=4 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1035, 1011, 1015, 1020, 1031, 1027, 1094, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1034, 971, 1067, 1026, 986, 1057, 1007, 21, 2, 0, 0, 0, 0, 0, 0, 1027, 1064, 1027, 1043, 1023, 1020, 1043, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 992, 998, 1038, 1021, 1010, 1029, 1046, 1050, 5, 4, 3, 0, 0, 0, 0, 0, 989, 1058, 1028, 966, 1031, 1023, 1036, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1012, 1036, 977, 1027, 1014, 999, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 991, 1039, 1057, 1019, 1005, 1056, 1021, 39, 2, 0, 0, 0, 0, 0, 0, 1018, 1015, 1028, 1096, 1025, 979, 1026, 975, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8149, 8251, 8196, 8110, 8133, 8257, 8162] avg=8192.0000 max=8278 max_rank=0 min=8110 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1044, 982, 1026, 1015, 1011, 1021, 1015, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1036, 999, 988, 1029, 1075, 1056, 1049, 19, 2, 0, 0, 0, 0, 0, 0, 1082, 1014, 1067, 945, 994, 1037, 982, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1004, 1049, 998, 1055, 995, 1018, 1013, 6, 4, 3, 0, 0, 0, 0, 0, 1071, 1038, 1031, 1031, 1014, 1002, 994, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1064, 1055, 1050, 1002, 988, 1021, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 953, 1075, 978, 1034, 974, 1028, 1001, 1016, 40, 2, 0, 0, 0, 0, 0, 0, 1058, 1007, 1054, 994, 1036, 1006, 1047, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8253, 8148, 8177, 8264, 8265, 8101, 8232] avg=8192.0000 max=8265 max_rank=5 min=8096 min_rank=0 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32862 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1054, 1053, 1042, 1003, 1034, 1029, 1013, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1065, 1014, 1038, 1039, 1032, 1026, 1023, 20, 2, 0, 0, 0, 0, 0, 0, 1018, 1029, 1025, 1067, 1037, 1026, 991, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 997, 1031, 1043, 990, 994, 975, 1005, 5, 4, 3, 0, 0, 0, 0, 0, 1056, 1051, 1004, 997, 1058, 1004, 1067, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 982, 986, 1057, 984, 988, 974, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 973, 998, 1010, 991, 1034, 1005, 979, 38, 2, 0, 0, 0, 0, 0, 0, 1034, 1063, 1016, 1030, 992, 1085, 1003, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8282, 8209, 8126, 8288, 8046, 8063, 8240] avg=8192.0000 max=8288 max_rank=4 min=8046 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1011, 1045, 1026, 1022, 940, 1068, 1039, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 990, 1031, 1027, 1035, 1014, 1045, 976, 22, 5, 0, 0, 0, 0, 0, 0, 1018, 1028, 999, 1076, 1018, 978, 1072, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1045, 1041, 1004, 1005, 982, 1045, 996, 11, 9, 5, 0, 0, 0, 0, 0, 1013, 1016, 991, 964, 1000, 1023, 1013, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1036, 1045, 1023, 1001, 1069, 983, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1018, 972, 1067, 1048, 1051, 1030, 1038, 44, 6, 0, 0, 0, 0, 0, 0, 986, 1017, 1020, 969, 1083, 1016, 1069, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8178, 8219, 8176, 8039, 8272, 8308, 8187] avg=8192.0000 max=8308 max_rank=6 min=8039 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1005, 1049, 1054, 1040, 986, 1023, 1025, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1013, 1004, 998, 1024, 999, 1030, 969, 19, 2, 0, 0, 0, 0, 0, 0, 974, 1033, 1060, 1019, 1060, 1043, 1008, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1011, 1005, 1018, 994, 1033, 1079, 987, 5, 4, 3, 0, 0, 0, 0, 0, 1003, 1012, 1032, 1044, 1027, 998, 1038, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1074, 1014, 1056, 1027, 1033, 1046, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 996, 1028, 1043, 1058, 1034, 1027, 1040, 41, 2, 0, 0, 0, 0, 0, 0, 1003, 1054, 995, 1012, 1027, 1023, 1059, 983, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8061, 8202, 8174, 8173, 8265, 8319, 8156] avg=8192.0000 max=8319 max_rank=6 min=8061 min_rank=1 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[999, 1014, 1036, 993, 1039, 999, 1023, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1021, 1031, 974, 1064, 993, 1044, 991, 20, 2, 0, 0, 0, 0, 0, 0, 1056, 989, 1027, 1077, 1017, 1028, 1041, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1025, 1038, 1056, 1085, 996, 988, 1080, 5, 4, 3, 0, 0, 0, 0, 0, 1062, 1009, 1037, 1010, 1038, 1030, 1025, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1037, 1040, 1065, 1030, 999, 986, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1019, 997, 1010, 1054, 962, 1012, 1016, 39, 2, 0, 0, 0, 0, 0, 0, 1024, 991, 995, 1030, 1026, 1040, 1003, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8169, 8264, 8266, 8264, 8202, 8112, 8128] avg=8192.0000 max=8266 max_rank=3 min=8112 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8064, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8064, 8320, 8320, 8320, 8064, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8064, 8320, 8192, 8192, 8192, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8448, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1024, 1015, 1005, 1030, 1010, 1016, 1075, 1049, 1003, 1001, 992, 1061, 1017, 1079, 983, 1011, 1032, 1033, 986, 990, 1035, 1027, 1011, 1011, 977, 1049, 1052, 1041, 1022, 1002, 1039, 1042, 1012, 978, 1053, 1050, 1019, 1041, 1047, 1053, 976, 1045, 1003, 998, 1038, 1074, 1020, 973, 1068, 1062, 1021, 1026, 1011, 1013, 1054, 1027, 1078, 1022, 1036, 998, 968, 1018, 980, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1046, 999, 1015, 1028, 1018, 992, 1010, 1001, 1041, 1022, 1041, 969, 1055, 1010, 1017, 1039, 1003, 1028, 1016, 986, 980, 1019, 1044, 995, 1048, 1087, 982, 959, 1049, 1023, 1015, 1018, 1002, 980, 1051, 1062, 1052, 1063, 1026, 1114, 1031, 1057, 1068, 1061, 1002, 1016, 1048, 1048, 989, 1040, 976, 958, 1014, 1016, 1026, 1015, 977, 1038, 1050, 1042, 1078, 1019, 1015, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1004, 1047, 1029, 1030, 1022, 1017, 1060, 980, 1061, 1008, 1007, 1072, 1016, 1055, 1025, 978, 1048, 1038, 1075, 1023, 1037, 992, 1042, 1045, 1039, 1040, 1072, 1039, 1023, 1058, 1051, 975, 1004, 948, 1014, 1037, 1006, 1016, 1036, 1008, 1023, 967, 1024, 1022, 1035, 987, 967, 1028, 974, 1087, 1011, 1018, 1072, 995, 1021, 1012, 983, 1050, 1014, 1004, 1039, 1058, 1063, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1053, 1042, 1027, 1034, 993, 968, 1001, 1015, 1032, 1048, 1021, 1093, 1001, 1028, 995, 1024, 1034, 984, 1004, 1018, 1028, 1022, 1033, 1005, 971, 1003, 1053, 1046, 1026, 1038, 1009, 1065, 1012, 1031, 1036, 1062, 1064, 1016, 1032, 1041, 1042, 1043, 996, 1049, 977, 1027, 1009, 1002, 1029, 1022, 1033, 988, 1024, 1034, 1045, 1027, 1078, 1013, 971, 1025, 1012, 1033, 1019, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[990, 1004, 1011, 1025, 1002, 978, 1041, 1002, 997, 987, 1052, 1023, 970, 1045, 1000, 1019, 1038, 1040, 1000, 1060, 1004, 1001, 991, 1094, 1059, 1014, 991, 1018, 1039, 1043, 1042, 1033, 1034, 1072, 1001, 964, 1034, 1009, 991, 1074, 1015, 1031, 986, 971, 988, 1011, 1064, 991, 1062, 1022, 1054, 1041, 1035, 1088, 1030, 1069, 1066, 1034, 1044, 1048, 1063, 993, 1040, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1019, 1034, 1018, 1000, 1046, 1044, 1041, 1065, 1020, 1091, 1026, 1023, 1052, 1021, 1024, 1067, 1040, 964, 1038, 1030, 1022, 1063, 1004, 1024, 989, 998, 1026, 986, 1024, 1045, 1039, 968, 1031, 1013, 1004, 1019, 1025, 1000, 1060, 1001, 1047, 1063, 1043, 983, 1046, 991, 1041, 1016, 987, 1019, 976, 1046, 1045, 1000, 1021, 1042, 983, 1020, 1034, 1010, 1034, 1012, 1051, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1045, 1006, 1026, 984, 1021, 1012, 1009, 1033, 1011, 1041, 996, 1063, 1019, 999, 1057, 1015, 994, 1017, 985, 978, 1001, 1031, 1051, 1067, 992, 1087, 1061, 1040, 1007, 1013, 1044, 1011, 984, 980, 990, 999, 1015, 1054, 1057, 981, 1045, 1055, 1035, 1029, 1029, 999, 1040, 1066, 1022, 1124, 963, 1017, 1042, 990, 1100, 1039, 1007, 1013, 989, 1034, 1008, 1025, 1034, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[996, 1048, 1001, 1016, 980, 1026, 1019, 994, 981, 1048, 1060, 1085, 1018, 1047, 1064, 1033, 1053, 1014, 1033, 1021, 978, 995, 1033, 1039, 1023, 1045, 1027, 1024, 998, 1035, 1020, 989, 1018, 979, 991, 1061, 1024, 1022, 1038, 1004, 1032, 1035, 1043, 1027, 1023, 1040, 1034, 1018, 1052, 1041, 1026, 1017, 1025, 1075, 1014, 981, 1035, 999, 1007, 1003, 1024, 1028, 1043, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1024, 1015, 1005, 1030, 1010, 1016, 1075, 1049, 29, 13, 0, 0, 0, 0, 0, 0, 1003, 1001, 992, 1034, 1017, 1079, 983, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1033, 986, 990, 1035, 1027, 1011, 1011, 14, 11, 0, 0, 0, 0, 0, 0, 977, 1037, 1052, 1041, 1022, 1002, 1039, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 978, 1053, 1050, 1019, 1041, 1047, 1053, 4, 0, 0, 0, 0, 0, 0, 0, 976, 1045, 1003, 998, 1038, 1074, 1020, 973, 3, 1, 0, 0, 0, 0, 0, 0, 1068, 1033, 1021, 1026, 1011, 1013, 1054, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1022, 1036, 998, 968, 1018, 973, 1054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8120, 8150, 8212, 8257, 8131, 8253, 8147] avg=8192.0000 max=8266 max_rank=0 min=8120 min_rank=1 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1046, 999, 1015, 1028, 1018, 992, 1010, 1001, 29, 12, 0, 0, 0, 0, 0, 0, 1041, 1022, 1041, 944, 1055, 1010, 1017, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1028, 1016, 986, 980, 1019, 1044, 995, 13, 12, 0, 0, 0, 0, 0, 0, 1048, 1074, 982, 959, 1049, 1023, 1015, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 980, 1051, 1062, 1052, 1063, 1026, 1114, 5, 0, 0, 0, 0, 0, 0, 0, 1031, 1057, 1068, 1061, 1002, 1016, 1048, 1048, 3, 1, 0, 0, 0, 0, 0, 0, 989, 1011, 976, 958, 1014, 1016, 1026, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1038, 1050, 1042, 1078, 1019, 1007, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8169, 8096, 8168, 8355, 8335, 8005, 8258] avg=8192.0000 max=8355 max_rank=4 min=8005 min_rank=6 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1053, 1042, 1027, 1034, 993, 968, 1001, 1015, 28, 14, 0, 0, 0, 0, 0, 0, 1032, 1048, 1021, 1065, 1001, 1028, 995, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 984, 1004, 1018, 1028, 1022, 1033, 1005, 14, 11, 0, 0, 0, 0, 0, 0, 971, 991, 1053, 1046, 1026, 1038, 1009, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1031, 1036, 1062, 1064, 1016, 1032, 1041, 5, 0, 0, 0, 0, 0, 0, 0, 1042, 1043, 996, 1049, 977, 1027, 1009, 1002, 3, 1, 0, 0, 0, 0, 0, 0, 1029, 994, 1033, 988, 1024, 1034, 1045, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1013, 971, 1025, 1012, 1033, 1011, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8214, 8153, 8199, 8299, 8149, 8174, 8173] avg=8192.0000 max=8299 max_rank=4 min=8149 min_rank=5 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[990, 1004, 1011, 1025, 1002, 978, 1041, 1002, 28, 13, 0, 0, 0, 0, 0, 0, 997, 987, 1052, 996, 970, 1045, 1000, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1040, 1000, 1060, 1004, 1001, 991, 1094, 14, 11, 0, 0, 0, 0, 0, 0, 1059, 1002, 991, 1018, 1039, 1043, 1042, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1072, 1001, 964, 1034, 1009, 991, 1074, 5, 0, 0, 0, 0, 0, 0, 0, 1015, 1031, 986, 971, 988, 1011, 1064, 991, 3, 1, 0, 0, 0, 0, 0, 0, 1062, 994, 1054, 1041, 1035, 1088, 1030, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1034, 1044, 1048, 1063, 993, 1032, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8094, 8066, 8253, 8227, 8184, 8061, 8373, 8278] avg=8192.0000 max=8373 max_rank=6 min=8061 min_rank=5 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1019, 1034, 1018, 1000, 1046, 1044, 1041, 1065, 28, 13, 0, 0, 0, 0, 0, 0, 1020, 1091, 1026, 996, 1052, 1021, 1024, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 964, 1038, 1030, 1022, 1063, 1004, 1024, 14, 11, 0, 0, 0, 0, 0, 0, 989, 986, 1026, 986, 1024, 1045, 1039, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1013, 1004, 1019, 1025, 1000, 1060, 1001, 5, 0, 0, 0, 0, 0, 0, 0, 1047, 1063, 1043, 983, 1046, 991, 1041, 1016, 3, 1, 0, 0, 0, 0, 0, 0, 987, 991, 976, 1046, 1045, 1000, 1021, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1020, 1034, 1010, 1034, 1012, 1043, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8297, 8210, 8063, 8158, 8234, 8108, 8158] avg=8192.0000 max=8308 max_rank=0 min=8063 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32658 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1004, 1047, 1029, 1030, 1022, 1017, 1060, 980, 35, 17, 0, 0, 0, 0, 0, 0, 1061, 1008, 1007, 1037, 1016, 1055, 1025, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1038, 1075, 1023, 1037, 992, 1042, 1045, 18, 15, 0, 0, 0, 0, 0, 0, 1039, 1019, 1072, 1039, 1023, 1058, 1051, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 948, 1014, 1037, 1006, 1016, 1036, 1008, 7, 0, 0, 0, 0, 0, 0, 0, 1023, 967, 1024, 1022, 1035, 987, 967, 1028, 9, 6, 0, 0, 0, 0, 0, 0, 974, 1052, 1011, 1018, 1072, 995, 1021, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1050, 1014, 1004, 1039, 1058, 1047, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8187, 8333, 8276, 8076, 8068, 8155, 8200] avg=8192.0000 max=8333 max_rank=2 min=8068 min_rank=5 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1045, 1006, 1026, 984, 1021, 1012, 1009, 1033, 31, 13, 0, 0, 0, 0, 0, 0, 1011, 1041, 996, 1036, 1019, 999, 1057, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1017, 985, 978, 1001, 1031, 1051, 1067, 14, 12, 0, 0, 0, 0, 0, 0, 992, 1074, 1061, 1040, 1007, 1013, 1044, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 984, 980, 990, 999, 1015, 1054, 1057, 981, 5, 0, 0, 0, 0, 0, 0, 0, 1045, 1055, 1035, 1029, 1029, 999, 1040, 1066, 3, 1, 0, 0, 0, 0, 0, 0, 1022, 1093, 963, 1017, 1042, 990, 1100, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1013, 989, 1034, 1008, 1025, 1026, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8174, 8150, 8242, 8065, 8302, 8266, 8157] avg=8192.0000 max=8302 max_rank=5 min=8065 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[996, 1048, 1001, 1016, 980, 1026, 1019, 994, 29, 13, 0, 0, 0, 0, 0, 0, 981, 1048, 1060, 1058, 1018, 1047, 1064, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1014, 1033, 1021, 978, 995, 1033, 1039, 14, 11, 0, 0, 0, 0, 0, 0, 1023, 1033, 1027, 1024, 998, 1035, 1020, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 979, 991, 1061, 1024, 1022, 1038, 1004, 5, 0, 0, 0, 0, 0, 0, 0, 1032, 1035, 1043, 1027, 1023, 1040, 1034, 1018, 3, 1, 0, 0, 0, 0, 0, 0, 1052, 1012, 1026, 1017, 1025, 1075, 1014, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 999, 1007, 1003, 1024, 1028, 1035, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8309, 8191, 8149, 8142, 8256, 8202, 8165] avg=8192.0000 max=8309 max_rank=1 min=8122 min_rank=0 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8064, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8192, 8320, 8320, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8064, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1011, 1025, 1053, 1028, 1042, 1020, 972, 993, 1021, 1059, 1025, 988, 1057, 1038, 1006, 1039, 1037, 1069, 1006, 1009, 1024, 1027, 1031, 1039, 1035, 1010, 1044, 1050, 1017, 1018, 1041, 1021, 1004, 970, 1049, 1010, 1042, 1024, 1067, 1010, 1068, 997, 1014, 980, 1007, 1044, 1004, 1020, 1004, 1040, 1018, 1048, 1006, 1048, 1024, 991, 987, 1024, 996, 1036, 1033, 1020, 1049, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[964, 1034, 1055, 1042, 1052, 1080, 1008, 1046, 973, 1008, 1021, 1045, 1004, 1058, 1008, 1004, 995, 1018, 985, 1050, 1045, 992, 1064, 1081, 1016, 1069, 1014, 1031, 1097, 1015, 1009, 981, 1030, 990, 1029, 1028, 1038, 1019, 981, 1002, 1029, 982, 1040, 985, 1035, 1050, 1051, 962, 1004, 1031, 1014, 1043, 1027, 1048, 1052, 1051, 1043, 1044, 993, 1044, 1036, 1014, 980, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1014, 1057, 1011, 1025, 1011, 1026, 1052, 1014, 1012, 1032, 999, 1016, 1007, 986, 1044, 1012, 1058, 993, 993, 1042, 1016, 1033, 1014, 1011, 978, 1053, 1013, 997, 1037, 1024, 1048, 1049, 1048, 1038, 1040, 972, 1032, 1028, 982, 999, 1045, 1031, 994, 1033, 1079, 977, 1024, 1088, 996, 1041, 1042, 1035, 1038, 1050, 946, 1022, 989, 1031, 1088, 1052, 1031, 1072, 988, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1026, 1039, 1021, 1003, 1018, 1012, 1011, 1004, 1070, 1031, 1004, 1035, 1048, 1005, 1026, 1027, 1010, 992, 1008, 1030, 1024, 992, 1060, 1070, 1060, 1005, 1019, 986, 997, 1030, 1048, 1088, 1044, 1012, 1078, 1001, 1042, 1008, 1060, 997, 971, 1007, 1026, 1044, 1002, 1036, 1002, 1033, 993, 1028, 997, 1018, 1040, 1058, 1013, 1006, 1033, 1030, 1024, 1030, 1057, 1025, 1037, 985] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1054, 1049, 1006, 964, 1068, 1020, 993, 1003, 1056, 1032, 1039, 1076, 1022, 1043, 1042, 969, 1029, 1008, 1049, 954, 1013, 1051, 1011, 1041, 1039, 1017, 1027, 996, 977, 1045, 1013, 991, 1011, 1013, 1034, 1040, 1038, 1015, 1020, 1074, 1023, 1033, 999, 1032, 1045, 1004, 1018, 1022, 1031, 1013, 1031, 1044, 1010, 1042, 1019, 998, 1060, 992, 1059, 1022, 1037, 998, 1031, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1007, 1005, 1029, 995, 1039, 1002, 1045, 1014, 1028, 983, 1041, 1014, 1048, 1045, 1012, 997, 1046, 1046, 981, 1093, 1010, 1027, 1062, 1033, 1069, 1027, 1038, 1018, 1021, 1029, 1031, 975, 1007, 973, 1027, 1027, 1056, 999, 1043, 1013, 1015, 967, 1023, 1043, 1001, 961, 1035, 992, 1060, 1040, 1017, 1077, 1068, 994, 1003, 1065, 967, 1064, 1025, 1066, 1014, 1023, 1061, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1042, 989, 1075, 1027, 1075, 999, 1010, 1023, 1017, 1047, 1008, 1027, 1014, 1034, 990, 955, 1043, 1028, 994, 1052, 974, 1029, 1025, 1027, 1022, 1033, 1010, 1060, 1075, 1067, 1056, 1015, 1025, 1029, 1028, 1014, 1015, 1043, 1044, 1001, 1027, 1023, 1043, 1041, 1004, 999, 944, 1019, 1063, 1067, 981, 1077, 1023, 1037, 966, 1028, 1017, 1019, 1078, 1002, 1002, 995, 1040, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1039, 1044, 996, 979, 986, 998, 1034, 1006, 1052, 1048, 1041, 1010, 1024, 1018, 993, 1096, 994, 1017, 1036, 997, 1089, 980, 1020, 1044, 1022, 1008, 992, 1032, 1006, 1053, 1046, 1034, 1023, 1020, 1032, 1046, 1030, 1030, 1028, 1001, 1044, 1004, 982, 1027, 1035, 1025, 976, 1043, 1027, 990, 1046, 1051, 1031, 1126, 988, 1037, 1002, 1019, 1046, 985, 997, 1035, 1019, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1011, 1025, 1053, 1028, 1042, 1020, 972, 993, 10, 8, 0, 0, 0, 0, 0, 0, 1021, 1059, 1025, 988, 1057, 1038, 1006, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1069, 1006, 1009, 1024, 1027, 1031, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1010, 1044, 1050, 1017, 1018, 1016, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 970, 1049, 1010, 1042, 1024, 1067, 1010, 7, 0, 0, 0, 0, 0, 0, 0, 1068, 997, 1014, 980, 1007, 1044, 1004, 1020, 35, 17, 0, 0, 0, 0, 0, 0, 1004, 1040, 1018, 1048, 1006, 1013, 1024, 991, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1024, 989, 1036, 1033, 1020, 1049, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8233, 8232, 8211, 8183, 8186, 8144, 8185] avg=8192.0000 max=8233 max_rank=1 min=8144 min_rank=6 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32698 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1014, 1057, 1011, 1025, 1011, 1026, 1052, 1014, 10, 8, 0, 0, 0, 0, 0, 0, 1012, 1032, 999, 1016, 1007, 986, 1044, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 993, 993, 1042, 1016, 1033, 1014, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1053, 1013, 997, 1037, 1024, 1023, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1038, 1040, 972, 1032, 1028, 982, 999, 8, 0, 0, 0, 0, 0, 0, 0, 1045, 1031, 994, 1033, 1079, 977, 1024, 1088, 35, 17, 0, 0, 0, 0, 0, 0, 996, 1041, 1042, 1035, 1038, 1015, 946, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1031, 1080, 1052, 1031, 1072, 988, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8108, 8150, 8174, 8147, 8323, 8135, 8271] avg=8192.0000 max=8323 max_rank=5 min=8108 min_rank=1 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32876 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[964, 1034, 1055, 1042, 1052, 1080, 1008, 1046, 11, 8, 0, 0, 0, 0, 0, 0, 973, 1008, 1021, 1045, 1004, 1058, 1008, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1018, 985, 1050, 1045, 992, 1064, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1069, 1014, 1031, 1097, 1015, 984, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 990, 1029, 1028, 1038, 1019, 981, 1002, 7, 0, 0, 0, 0, 0, 0, 0, 1029, 982, 1040, 985, 1035, 1050, 1051, 962, 35, 17, 0, 0, 0, 0, 0, 0, 1004, 1031, 1014, 1043, 1027, 1013, 1052, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1044, 986, 1044, 1036, 1014, 980, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8121, 8219, 8207, 8124, 8186, 8235, 8144] avg=8192.0000 max=8300 max_rank=0 min=8121 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32689 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1042, 989, 1075, 1027, 1075, 999, 1010, 1023, 10, 8, 0, 0, 0, 0, 0, 0, 1017, 1047, 1008, 1027, 1014, 1034, 990, 955, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1028, 994, 1052, 974, 1029, 1025, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1033, 1010, 1060, 1075, 1067, 1031, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1029, 1028, 1014, 1015, 1043, 1044, 1001, 8, 0, 0, 0, 0, 0, 0, 0, 1027, 1023, 1043, 1041, 1004, 999, 944, 1019, 35, 17, 0, 0, 0, 0, 0, 0, 1063, 1067, 981, 1077, 1023, 1002, 966, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1019, 1070, 1002, 1002, 995, 1040, 1000, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8092, 8162, 8313, 8207, 8152, 8207, 8145] avg=8192.0000 max=8313 max_rank=3 min=8092 min_rank=1 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32711 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1054, 1049, 1006, 964, 1068, 1020, 993, 1003, 10, 8, 0, 0, 0, 0, 0, 0, 1056, 1032, 1039, 1076, 1022, 1043, 1042, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1008, 1049, 954, 1013, 1051, 1011, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1017, 1027, 996, 977, 1045, 988, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1013, 1034, 1040, 1038, 1015, 1020, 1074, 8, 0, 0, 0, 0, 0, 0, 0, 1023, 1033, 999, 1032, 1045, 1004, 1018, 1022, 35, 17, 0, 0, 0, 0, 0, 0, 1031, 1013, 1031, 1044, 1010, 1007, 1019, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 992, 1051, 1022, 1037, 998, 1031, 1031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8279, 8146, 8080, 8253, 8228, 8153, 8222] avg=8192.0000 max=8279 max_rank=1 min=8080 min_rank=3 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32856 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 1039, 1021, 1003, 1018, 1012, 1011, 1004, 14, 11, 0, 0, 0, 0, 0, 0, 1070, 1031, 1004, 1035, 1048, 1005, 1026, 1027, 7, 0, 0, 0, 0, 0, 0, 0, 1010, 992, 1008, 1030, 1024, 992, 1060, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1005, 1019, 986, 997, 1030, 1015, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1012, 1078, 1001, 1042, 1008, 1060, 997, 11, 0, 0, 0, 0, 0, 0, 0, 971, 1007, 1026, 1044, 1002, 1036, 1002, 1033, 40, 22, 0, 0, 0, 0, 0, 0, 993, 1028, 997, 1018, 1040, 1018, 1013, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1030, 1006, 1030, 1057, 1025, 1037, 985, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8253, 8172, 8200, 8253, 8183, 8113, 8203] avg=8192.0000 max=8253 max_rank=1 min=8113 min_rank=6 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32752 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1007, 1005, 1029, 995, 1039, 1002, 1045, 1014, 10, 8, 0, 0, 0, 0, 0, 0, 1028, 983, 1041, 1014, 1048, 1045, 1012, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1046, 981, 1093, 1010, 1027, 1062, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1027, 1038, 1018, 1021, 1029, 1006, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 973, 1027, 1027, 1056, 999, 1043, 1013, 7, 0, 0, 0, 0, 0, 0, 0, 1015, 967, 1023, 1043, 1001, 961, 1035, 992, 33, 17, 0, 0, 0, 0, 0, 0, 1060, 1040, 1017, 1077, 1068, 961, 1003, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1064, 1018, 1066, 1014, 1023, 1061, 1000, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8168, 8288, 8183, 8152, 8087, 8291, 8213] avg=8192.0000 max=8291 max_rank=6 min=8087 min_rank=5 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32743 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1039, 1044, 996, 979, 986, 998, 1034, 1006, 10, 8, 0, 0, 0, 0, 0, 0, 1052, 1048, 1041, 1010, 1024, 1018, 993, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1017, 1036, 997, 1089, 980, 1020, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1008, 992, 1032, 1006, 1053, 1021, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1020, 1032, 1046, 1030, 1030, 1028, 1001, 7, 0, 0, 0, 0, 0, 0, 0, 1044, 1004, 982, 1027, 1035, 1025, 976, 1043, 38, 17, 0, 0, 0, 0, 0, 0, 1027, 990, 1046, 1051, 1031, 1088, 988, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1019, 1039, 985, 997, 1035, 1019, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8282, 8167, 8168, 8217, 8191, 8258, 8153] avg=8192.0000 max=8282 max_rank=1 min=8100 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32819 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8192, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8192, 8192, 8320, 8192, 8064, 8192, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8448, 8320, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[988, 1040, 1050, 991, 1003, 987, 985, 1080, 966, 963, 1050, 1101, 1040, 1017, 1027, 1025, 1076, 1014, 972, 1029, 1039, 1007, 1027, 988, 1044, 1059, 1063, 1026, 1011, 1031, 1067, 989, 1016, 980, 989, 995, 1033, 1028, 1035, 1040, 1013, 1054, 1067, 1022, 1051, 1035, 1038, 1007, 1070, 1003, 1031, 1010, 988, 1024, 1041, 1009, 1019, 1000, 1008, 1027, 1023, 1023, 1051, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1023, 1032, 1046, 1032, 1035, 1028, 998, 1039, 989, 1033, 1024, 1030, 1030, 1010, 1028, 1030, 1027, 999, 1034, 1011, 1010, 999, 1054, 1006, 1007, 1042, 1047, 1004, 1011, 1067, 1064, 1018, 1051, 1008, 1027, 1052, 945, 1024, 1013, 1035, 971, 1021, 1015, 1018, 1024, 1065, 1002, 1026, 1042, 1009, 1008, 1023, 1032, 1007, 1025, 1023, 979, 1030, 1058, 1035, 985, 1041, 1123, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[994, 1010, 1016, 1046, 1022, 1046, 1018, 1028, 979, 1022, 1004, 1017, 1032, 1023, 1014, 1007, 1022, 1028, 1033, 1015, 1071, 1031, 1035, 1005, 1027, 1027, 1069, 1091, 994, 1018, 1055, 1017, 1063, 1057, 990, 1013, 1031, 1043, 1040, 1013, 1018, 1033, 1030, 1065, 1004, 1018, 1011, 1062, 1021, 1041, 988, 1046, 988, 992, 1076, 1019, 996, 1022, 997, 1024, 1047, 1013, 972, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1049, 1082, 1013, 1058, 997, 994, 1035, 1044, 1001, 1038, 1067, 1010, 1071, 957, 1065, 1042, 1034, 1041, 1012, 1000, 1027, 1069, 997, 1004, 1011, 994, 1026, 1041, 1020, 1006, 1009, 1042, 1037, 982, 1082, 1041, 1054, 961, 1019, 962, 1011, 1073, 1008, 1064, 1008, 998, 1010, 987, 1018, 981, 991, 1041, 1040, 1068, 1047, 1056, 1021, 1051, 1056, 1014, 981, 996, 1010, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1049, 1060, 1011, 982, 1020, 1007, 1010, 1070, 1020, 1076, 1027, 1026, 1028, 1015, 1035, 991, 1046, 1017, 1021, 1026, 1013, 1031, 1039, 1040, 1030, 991, 949, 1044, 1030, 1037, 1019, 1019, 1030, 1027, 997, 1030, 1040, 1033, 974, 991, 1081, 1003, 1032, 1028, 1002, 1000, 1035, 1037, 1056, 1020, 1030, 1034, 1031, 1060, 1029, 1020, 1026, 987, 1041, 1036, 1057, 1010, 1005, 975] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1029, 1011, 1016, 1017, 1025, 1021, 1043, 999, 1022, 1080, 1043, 1039, 994, 1033, 1043, 1039, 973, 1007, 1026, 1015, 1000, 1058, 1036, 1049, 1036, 1036, 1019, 1024, 1009, 1026, 1071, 1017, 1011, 1009, 1021, 1054, 1034, 983, 956, 991, 998, 1017, 1026, 1033, 1051, 1067, 994, 1049, 1035, 989, 1052, 1014, 1041, 1043, 1056, 969, 1077, 1034, 993, 1032, 1037, 1024, 980, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1034, 1061, 1025, 1042, 1018, 1011, 1064, 972, 1026, 1020, 1012, 994, 1056, 1055, 1030, 1008, 1029, 1031, 1019, 1024, 1048, 1004, 1031, 982, 1040, 1025, 1082, 1024, 993, 1011, 1047, 988, 1027, 1008, 985, 1048, 1043, 1023, 1047, 1020, 972, 999, 1031, 1020, 1041, 1041, 1073, 1022, 1044, 1002, 994, 1014, 995, 996, 1023, 1003, 1040, 1026, 1020, 1046, 1048, 1034, 1027, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1035, 1002, 1047, 1012, 1035, 1021, 991, 1007, 1064, 983, 1069, 1025, 1057, 1010, 975, 1001, 1042, 1013, 1073, 1004, 1001, 1060, 1017, 997, 981, 1009, 1046, 996, 1059, 1051, 1038, 1031, 981, 1048, 1012, 996, 1086, 978, 1112, 1124, 1061, 994, 1022, 975, 1040, 1024, 1039, 981, 1019, 1027, 1029, 1040, 1030, 996, 1037, 950, 1051, 983, 1028, 1009, 1041, 1008, 1026, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[994, 1008, 1016, 1046, 1022, 1046, 1018, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1022, 1004, 1017, 1024, 1023, 1014, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1028, 1033, 1015, 1071, 1031, 1035, 1005, 3, 2, 0, 0, 0, 0, 0, 0, 1027, 1027, 1069, 1091, 994, 1018, 1026, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1057, 990, 1013, 1031, 1043, 1040, 1013, 19, 0, 0, 0, 0, 0, 0, 0, 1018, 1033, 1030, 1065, 1004, 1009, 1011, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1041, 988, 1046, 988, 992, 1076, 1019, 6, 5, 0, 0, 0, 0, 0, 0, 996, 1022, 997, 1024, 1047, 1013, 972, 987, 10, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8090, 8245, 8269, 8269, 8232, 8182, 8071] avg=8192.0000 max=8269 max_rank=3 min=8071 min_rank=7 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[988, 1038, 1050, 991, 1003, 987, 985, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 966, 963, 1050, 1101, 1032, 1017, 1027, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1014, 972, 1029, 1039, 1007, 1027, 988, 3, 2, 0, 0, 0, 0, 0, 0, 1044, 1059, 1063, 1026, 1011, 1031, 1037, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 980, 989, 995, 1033, 1028, 1035, 1040, 20, 0, 0, 0, 0, 0, 0, 0, 1013, 1054, 1067, 1022, 1051, 1025, 1038, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1003, 1031, 1010, 988, 1024, 1041, 1009, 7, 5, 0, 0, 0, 0, 0, 0, 1019, 1000, 1008, 1027, 1023, 1023, 1051, 1051, 10, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8181, 8157, 8260, 8136, 8277, 8188, 8215] avg=8192.0000 max=8277 max_rank=5 min=8122 min_rank=0 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1049, 1058, 1011, 982, 1020, 1007, 1010, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1076, 1027, 1026, 1020, 1015, 1035, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1017, 1021, 1026, 1013, 1031, 1039, 1040, 3, 2, 0, 0, 0, 0, 0, 0, 1030, 991, 949, 1044, 1030, 1037, 991, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1027, 997, 1030, 1040, 1033, 974, 991, 19, 0, 0, 0, 0, 0, 0, 0, 1081, 1003, 1032, 1028, 1002, 991, 1035, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1020, 1030, 1034, 1031, 1060, 1029, 1020, 6, 5, 0, 0, 0, 0, 0, 0, 1026, 987, 1041, 1036, 1057, 1010, 1005, 975, 9, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8210, 8238, 8091, 8141, 8209, 8291, 8149] avg=8192.0000 max=8291 max_rank=6 min=8091 min_rank=3 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1023, 1030, 1046, 1032, 1035, 1028, 998, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1033, 1024, 1030, 1022, 1010, 1028, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 999, 1034, 1011, 1010, 999, 1054, 1006, 3, 2, 0, 0, 0, 0, 0, 0, 1007, 1042, 1047, 1004, 1011, 1067, 1034, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1008, 1027, 1052, 945, 1024, 1013, 1035, 20, 0, 0, 0, 0, 0, 0, 0, 971, 1021, 1015, 1018, 1024, 1055, 1002, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1009, 1008, 1023, 1032, 1007, 1025, 1023, 7, 5, 0, 0, 0, 0, 0, 0, 979, 1030, 1058, 1035, 985, 1041, 1123, 1012, 10, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8166, 8145, 8230, 8175, 8132, 8181, 8276] avg=8192.0000 max=8276 max_rank=7 min=8132 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1049, 1080, 1013, 1058, 997, 994, 1035, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1038, 1067, 1010, 1063, 957, 1065, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1041, 1012, 1000, 1027, 1069, 997, 1004, 3, 2, 0, 0, 0, 0, 0, 0, 1011, 994, 1026, 1041, 1020, 1006, 981, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 982, 1082, 1041, 1054, 961, 1019, 962, 19, 0, 0, 0, 0, 0, 0, 0, 1011, 1073, 1008, 1064, 1008, 989, 1010, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 981, 991, 1041, 1040, 1068, 1047, 1056, 6, 5, 0, 0, 0, 0, 0, 0, 1021, 1051, 1056, 1014, 981, 996, 1010, 1012, 9, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8243, 8189, 8121, 8157, 8150, 8253, 8153] avg=8192.0000 max=8270 max_rank=0 min=8121 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1029, 1005, 1016, 1017, 1025, 1021, 1043, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1080, 1043, 1039, 978, 1033, 1043, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1007, 1026, 1015, 1000, 1058, 1036, 1049, 7, 6, 0, 0, 0, 0, 0, 0, 1036, 1036, 1019, 1024, 1009, 1026, 1035, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1009, 1021, 1054, 1034, 983, 956, 991, 23, 0, 0, 0, 0, 0, 0, 0, 998, 1017, 1026, 1033, 1051, 1052, 994, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 989, 1052, 1014, 1041, 1043, 1056, 969, 11, 9, 0, 0, 0, 0, 0, 0, 1077, 1034, 993, 1032, 1037, 1024, 980, 1010, 13, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8277, 8177, 8202, 8082, 8220, 8219, 8204] avg=8192.0000 max=8277 max_rank=1 min=8082 min_rank=4 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1034, 1059, 1025, 1042, 1018, 1011, 1064, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1020, 1012, 994, 1048, 1055, 1030, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1031, 1019, 1024, 1048, 1004, 1031, 982, 3, 2, 0, 0, 0, 0, 0, 0, 1040, 1025, 1082, 1024, 993, 1011, 1018, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1008, 985, 1048, 1043, 1023, 1047, 1020, 19, 0, 0, 0, 0, 0, 0, 0, 972, 999, 1031, 1020, 1041, 1031, 1073, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1002, 994, 1014, 995, 996, 1023, 1003, 7, 5, 0, 0, 0, 0, 0, 0, 1040, 1026, 1020, 1046, 1048, 1034, 1027, 1018, 10, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8193, 8173, 8181, 8220, 8189, 8083, 8272] avg=8192.0000 max=8272 max_rank=7 min=8083 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1035, 1000, 1047, 1012, 1035, 1021, 991, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 983, 1069, 1025, 1049, 1010, 975, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1013, 1073, 1004, 1001, 1060, 1017, 997, 3, 2, 0, 0, 0, 0, 0, 0, 981, 1009, 1046, 996, 1059, 1051, 1009, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1048, 1012, 996, 1086, 978, 1112, 1124, 19, 0, 0, 0, 0, 0, 0, 0, 1061, 994, 1022, 975, 1040, 1015, 1039, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1027, 1029, 1040, 1030, 996, 1037, 950, 6, 5, 0, 0, 0, 0, 0, 0, 1051, 983, 1028, 1009, 1041, 1008, 1026, 1037, 10, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8176, 8212, 8182, 8356, 8127, 8139, 8196] avg=8192.0000 max=8356 max_rank=4 min=8127 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8448, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1036, 1023, 1038, 1023, 1029, 1027, 1052, 997, 1025, 1035, 1040, 988, 1046, 1022, 1018, 1043, 1022, 1074, 995, 1010, 1012, 1055, 1038, 1037, 1060, 1006, 1047, 1022, 978, 1028, 1041, 1010, 1010, 1024, 1014, 1020, 1046, 955, 1030, 994, 1034, 1097, 1023, 978, 990, 1047, 1063, 975, 1007, 1007, 1078, 1063, 1060, 1021, 982, 1025, 1005, 1022, 980, 1043, 1007, 978, 1055, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1042, 1052, 1038, 988, 1044, 1005, 987, 1049, 1035, 1027, 980, 1001, 1023, 1013, 1022, 1049, 1031, 1051, 1038, 1021, 1002, 1035, 1029, 1025, 1035, 1030, 1047, 986, 968, 1039, 1012, 1035, 1031, 994, 1038, 985, 1032, 1054, 1060, 1039, 1053, 998, 1038, 1011, 1044, 1034, 1074, 988, 1011, 1037, 1032, 1034, 1048, 1028, 1014, 993, 1029, 1043, 1010, 986, 1041, 996, 991, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1070, 968, 985, 992, 1045, 1055, 1047, 1022, 1004, 990, 1073, 1078, 1013, 1055, 972, 1040, 991, 1046, 1042, 1032, 1026, 1001, 1004, 989, 1018, 1000, 1021, 1050, 1015, 1051, 1017, 1028, 1025, 1047, 1030, 1019, 1050, 990, 1035, 984, 1051, 1031, 1008, 1041, 1016, 1015, 986, 1030, 1032, 1012, 1022, 1086, 1011, 1034, 962, 1000, 1031, 1037, 1041, 1022, 1031, 1039, 1054, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1032, 1042, 1015, 1011, 1028, 1000, 1012, 1022, 1054, 1016, 1000, 1005, 1004, 1022, 986, 1044, 991, 1008, 1055, 1029, 1032, 1051, 983, 1019, 995, 1013, 1026, 1020, 1063, 1029, 1058, 1015, 1042, 1052, 1052, 1015, 1029, 1041, 1011, 1014, 1021, 996, 1048, 1010, 1028, 1067, 1011, 1002, 1062, 1016, 1007, 1035, 1025, 1022, 1059, 1009, 1040, 989, 1001, 1033, 1071, 1011, 1017, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1061, 951, 992, 1037, 1071, 1020, 1037, 996, 1040, 1052, 1005, 1014, 1015, 1079, 1061, 1054, 1027, 1033, 1054, 1046, 1002, 1022, 1000, 1040, 982, 1023, 972, 1041, 1056, 1027, 1028, 1051, 1023, 1060, 1015, 1014, 1008, 1012, 1070, 987, 1027, 1054, 1048, 990, 1031, 982, 1017, 967, 1013, 1005, 1020, 1055, 1018, 1055, 1024, 1047, 1053, 993, 1037, 1044, 1007, 1006, 975, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1054, 1000, 1051, 985, 1030, 1064, 969, 1041, 942, 1060, 1007, 1059, 1018, 1045, 1023, 1030, 1006, 1050, 1045, 999, 1026, 1010, 1042, 982, 1019, 980, 1026, 1001, 1024, 1033, 1010, 1047, 1020, 1004, 996, 1030, 995, 1026, 1022, 1041, 1028, 1032, 1043, 1051, 1015, 1054, 1060, 1005, 999, 1019, 1032, 1035, 1002, 978, 1011, 1043, 1083, 1083, 1040, 1010, 1064, 992, 1018, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1023, 1009, 1048, 1007, 1045, 1023, 969, 1026, 1031, 1036, 1030, 1015, 1001, 1053, 1008, 1048, 1040, 1032, 1034, 975, 1083, 1046, 1040, 974, 960, 1075, 1021, 1022, 1034, 1029, 1030, 1041, 1023, 1028, 1007, 963, 1018, 1048, 1047, 1054, 1033, 1040, 998, 1051, 1003, 991, 1094, 1009, 1012, 1003, 998, 1009, 1060, 1033, 973, 1040, 1013, 1027, 1046, 1012, 997, 1062, 1024, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1001, 1009, 993, 1012, 1090, 1036, 1051, 983, 1047, 1054, 1081, 952, 1054, 1064, 1081, 1028, 1006, 1016, 988, 1016, 1051, 995, 1001, 1014, 1028, 1069, 1023, 1035, 1039, 1033, 1000, 991, 1064, 996, 1049, 983, 1048, 1045, 960, 1060, 1074, 1022, 999, 1023, 1038, 1024, 1079, 1018, 1000, 1060, 987, 985, 1017, 1045, 1003, 992, 997, 1047, 1019, 997, 992, 1004, 1082, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1036, 1023, 1038, 1023, 1029, 1027, 1052, 997, 6, 3, 0, 0, 0, 0, 0, 0, 1025, 1035, 1040, 988, 1046, 990, 1018, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1074, 995, 1010, 1012, 1055, 1038, 1037, 8, 0, 0, 0, 0, 0, 0, 0, 1060, 1006, 1047, 1022, 978, 1028, 1041, 1010, 2, 0, 0, 0, 0, 0, 0, 0, 1010, 1024, 1014, 1020, 1046, 955, 1030, 994, 7, 0, 0, 0, 0, 0, 0, 0, 1034, 1097, 1023, 978, 990, 1047, 1043, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1007, 1078, 1063, 1060, 1021, 982, 1025, 15, 0, 0, 0, 0, 0, 0, 0, 1005, 1022, 980, 1043, 1007, 978, 1055, 1026, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8185, 8251, 8194, 8100, 8187, 8258, 8127] avg=8192.0000 max=8258 max_rank=6 min=8100 min_rank=4 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1042, 1052, 1038, 988, 1044, 1005, 987, 1049, 6, 3, 0, 0, 0, 0, 0, 0, 1035, 1027, 980, 1001, 1023, 981, 1022, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1051, 1038, 1021, 1002, 1035, 1029, 1025, 8, 0, 0, 0, 0, 0, 0, 0, 1035, 1030, 1047, 986, 968, 1039, 1012, 1035, 2, 0, 0, 0, 0, 0, 0, 0, 1031, 994, 1038, 985, 1032, 1054, 1060, 1039, 7, 0, 0, 0, 0, 0, 0, 0, 1053, 998, 1038, 1011, 1044, 1034, 1054, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1037, 1032, 1034, 1048, 1028, 1014, 993, 15, 0, 0, 0, 0, 0, 0, 0, 1029, 1043, 1010, 986, 1041, 996, 991, 1031, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8118, 8240, 8154, 8240, 8220, 8212, 8138] avg=8192.0000 max=8240 max_rank=2 min=8118 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1070, 968, 985, 992, 1045, 1055, 1047, 1022, 6, 2, 0, 0, 0, 0, 0, 0, 1004, 990, 1073, 1078, 1013, 1022, 972, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1046, 1042, 1032, 1026, 1001, 1004, 989, 7, 0, 0, 0, 0, 0, 0, 0, 1018, 1000, 1021, 1050, 1015, 1051, 1017, 1028, 2, 0, 0, 0, 0, 0, 0, 0, 1025, 1047, 1030, 1019, 1050, 990, 1035, 984, 6, 0, 0, 0, 0, 0, 0, 0, 1051, 1031, 1008, 1041, 1016, 1015, 969, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1012, 1022, 1086, 1011, 1034, 962, 1000, 16, 0, 0, 0, 0, 0, 0, 0, 1031, 1037, 1041, 1022, 1031, 1039, 1054, 1024, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8138, 8202, 8186, 8161, 8175, 8290] avg=8192.0000 max=8290 max_rank=7 min=8138 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1032, 1042, 1015, 1011, 1028, 1000, 1012, 1022, 6, 2, 0, 0, 0, 0, 0, 0, 1054, 1016, 1000, 1005, 1004, 990, 986, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1008, 1055, 1029, 1032, 1051, 983, 1019, 8, 0, 0, 0, 0, 0, 0, 0, 995, 1013, 1026, 1020, 1063, 1029, 1058, 1015, 2, 0, 0, 0, 0, 0, 0, 0, 1042, 1052, 1052, 1015, 1029, 1041, 1011, 1014, 6, 0, 0, 0, 0, 0, 0, 0, 1021, 996, 1048, 1010, 1028, 1067, 993, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1016, 1007, 1035, 1025, 1022, 1059, 1009, 15, 0, 0, 0, 0, 0, 0, 0, 1040, 989, 1001, 1033, 1071, 1011, 1017, 1020, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8099, 8176, 8221, 8262, 8165, 8250, 8193] avg=8192.0000 max=8262 max_rank=4 min=8099 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1061, 951, 992, 1037, 1071, 1020, 1037, 996, 6, 2, 0, 0, 0, 0, 0, 0, 1040, 1052, 1005, 1014, 1015, 1045, 1061, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1033, 1054, 1046, 1002, 1022, 1000, 1040, 8, 0, 0, 0, 0, 0, 0, 0, 982, 1023, 972, 1041, 1056, 1027, 1028, 1051, 2, 0, 0, 0, 0, 0, 0, 0, 1023, 1060, 1015, 1014, 1008, 1012, 1070, 987, 7, 0, 0, 0, 0, 0, 0, 0, 1027, 1054, 1048, 990, 1031, 982, 998, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1005, 1020, 1055, 1018, 1055, 1024, 1047, 16, 0, 0, 0, 0, 0, 0, 0, 1053, 993, 1037, 1044, 1007, 1006, 975, 990, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8286, 8232, 8182, 8196, 8097, 8253, 8117] avg=8192.0000 max=8286 max_rank=1 min=8097 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32663 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1023, 1009, 1048, 1007, 1045, 1023, 969, 1026, 6, 3, 0, 0, 0, 0, 0, 0, 1031, 1036, 1030, 1015, 1001, 1020, 1008, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1032, 1034, 975, 1083, 1046, 1040, 974, 8, 0, 0, 0, 0, 0, 0, 0, 960, 1075, 1021, 1022, 1034, 1029, 1030, 1041, 3, 0, 0, 0, 0, 0, 0, 0, 1023, 1028, 1007, 963, 1018, 1048, 1047, 1054, 7, 0, 0, 0, 0, 0, 0, 0, 1033, 1040, 998, 1051, 1003, 991, 1073, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1003, 998, 1009, 1060, 1033, 973, 1040, 16, 0, 0, 0, 0, 0, 0, 0, 1013, 1027, 1046, 1012, 997, 1062, 1024, 1012, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8189, 8232, 8215, 8195, 8198, 8144, 8204] avg=8192.0000 max=8232 max_rank=2 min=8144 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1054, 1000, 1051, 985, 1030, 1064, 969, 1041, 10, 6, 0, 0, 0, 0, 0, 0, 942, 1060, 1007, 1059, 1018, 1000, 1023, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1050, 1045, 999, 1026, 1010, 1042, 982, 12, 0, 0, 0, 0, 0, 0, 0, 1019, 980, 1026, 1001, 1024, 1033, 1010, 1047, 8, 0, 0, 0, 0, 0, 0, 0, 1020, 1004, 996, 1030, 995, 1026, 1022, 1041, 11, 0, 0, 0, 0, 0, 0, 0, 1028, 1032, 1043, 1051, 1015, 1054, 1023, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1019, 1032, 1035, 1002, 978, 1011, 1043, 20, 0, 0, 0, 0, 0, 0, 0, 1083, 1083, 1040, 1010, 1064, 992, 1018, 1027, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8139, 8172, 8148, 8145, 8251, 8139, 8332] avg=8192.0000 max=8332 max_rank=7 min=8139 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1001, 1009, 993, 1012, 1090, 1036, 1051, 983, 6, 3, 0, 0, 0, 0, 0, 0, 1047, 1054, 1081, 952, 1054, 1031, 1081, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1016, 988, 1016, 1051, 995, 1001, 1014, 8, 0, 0, 0, 0, 0, 0, 0, 1028, 1069, 1023, 1035, 1039, 1033, 1000, 991, 2, 0, 0, 0, 0, 0, 0, 0, 1064, 996, 1049, 983, 1048, 1045, 960, 1060, 7, 0, 0, 0, 0, 0, 0, 0, 1074, 1022, 999, 1023, 1038, 1024, 1059, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1060, 987, 985, 1017, 1045, 1003, 992, 16, 0, 0, 0, 0, 0, 0, 0, 997, 1047, 1019, 997, 992, 1004, 1082, 986, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8328, 8095, 8220, 8212, 8257, 8105, 8135] avg=8192.0000 max=8328 max_rank=1 min=8095 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8064, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8320, 8320, 8192, 8192, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8064, 8448, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8064, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[992, 1014, 1029, 992, 1041, 1040, 1037, 994, 1034, 987, 986, 993, 1068, 1038, 1027, 1032, 1074, 1062, 1001, 1035, 1014, 995, 1011, 1028, 1022, 1051, 1081, 1050, 977, 1045, 1009, 1022, 998, 1048, 1048, 1037, 1019, 1043, 1054, 1028, 1028, 998, 1056, 1023, 1008, 978, 1031, 1054, 1049, 994, 1015, 999, 963, 1041, 1072, 1004, 1037, 1015, 1006, 1028, 992, 1083, 1022, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[999, 1072, 1059, 1014, 1012, 1027, 980, 1070, 996, 1022, 1061, 1008, 1058, 1040, 1049, 1051, 1009, 1006, 1038, 1045, 967, 1055, 1043, 1007, 1062, 1026, 1015, 1021, 1058, 1007, 1036, 1009, 976, 997, 1032, 1009, 1051, 1015, 1030, 1032, 986, 1050, 947, 1006, 1031, 1032, 974, 1001, 1081, 991, 1047, 1010, 1051, 1054, 1006, 1034, 1050, 1048, 1003, 1049, 998, 1025, 1012, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1031, 993, 1036, 1073, 1001, 1025, 994, 1017, 1080, 1029, 1039, 1039, 999, 995, 1000, 994, 1020, 1058, 1029, 1065, 1023, 1075, 999, 1001, 1000, 1028, 1038, 1043, 1039, 986, 974, 1069, 1032, 1006, 993, 1027, 1023, 1002, 1054, 1058, 996, 1011, 1051, 1043, 1047, 1034, 1013, 960, 979, 1046, 1025, 1046, 1095, 1070, 1012, 1003, 1057, 936, 1041, 1078, 985, 1025, 985, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1043, 1031, 1049, 1057, 1051, 1056, 1059, 992, 997, 1020, 977, 994, 1024, 1019, 1017, 1019, 977, 1026, 999, 1028, 1099, 1043, 1042, 1004, 1015, 1029, 1027, 1023, 1036, 1027, 1042, 974, 1039, 1043, 978, 994, 995, 1017, 1050, 1021, 1003, 1041, 1047, 1057, 1067, 995, 1048, 1045, 1009, 980, 1012, 1004, 1002, 1022, 1049, 1021, 1023, 1039, 1006, 1042, 988, 1000, 1048, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1014, 1050, 1004, 994, 1069, 1009, 1012, 1040, 992, 1004, 1062, 1065, 1007, 1051, 1039, 997, 1026, 1019, 992, 997, 1003, 998, 1031, 1041, 988, 1055, 1088, 1017, 1047, 1012, 1015, 1017, 988, 988, 964, 1007, 995, 1013, 1034, 996, 1002, 1042, 1050, 1019, 1012, 1002, 1082, 1048, 1089, 1023, 1009, 1049, 1040, 1028, 1008, 949, 1010, 1034, 1043, 1077, 1064, 1024, 1030, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[979, 1103, 1022, 954, 1011, 1054, 1027, 971, 1079, 1027, 993, 1010, 996, 975, 984, 1001, 1028, 995, 1020, 1037, 1008, 1075, 1020, 1012, 1033, 1046, 1042, 1026, 1041, 1009, 1080, 1062, 1053, 1032, 1059, 994, 1027, 1034, 1028, 1024, 1050, 1061, 972, 1021, 960, 1027, 1011, 1009, 1030, 1009, 1040, 1039, 1027, 1004, 1038, 1022, 1042, 1036, 1027, 1040, 1016, 1015, 1021, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1002, 1008, 1046, 1045, 986, 1012, 1052, 1022, 1016, 1022, 993, 1050, 983, 1009, 1002, 1015, 1054, 1024, 1054, 959, 1048, 1066, 1032, 1066, 1021, 1044, 1014, 1049, 988, 1010, 1036, 1015, 1036, 1028, 1037, 1023, 1068, 986, 1036, 1023, 1049, 1021, 1023, 1024, 1040, 1003, 1033, 1029, 1034, 1016, 1037, 1010, 1008, 992, 1031, 991, 994, 1038, 1010, 1037, 1079, 1009, 1024, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1017, 1025, 1008, 1035, 1047, 993, 992, 1035, 1036, 1057, 1005, 979, 1086, 1035, 977, 1009, 1009, 997, 1025, 1038, 1028, 1085, 1054, 950, 1045, 1016, 1017, 1043, 946, 1053, 1009, 1088, 1023, 1042, 996, 1008, 1015, 1052, 1064, 1032, 1043, 1024, 1024, 1028, 992, 984, 1030, 1093, 1035, 1031, 1048, 983, 1008, 1028, 1003, 985, 1042, 1003, 1029, 1022, 1026, 1047, 1004, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1014, 1050, 1004, 994, 1069, 1009, 1012, 1040, 2, 0, 0, 0, 0, 0, 0, 0, 992, 1004, 1062, 1065, 1007, 1051, 1039, 997, 36, 1, 0, 0, 0, 0, 0, 0, 1026, 1019, 992, 997, 1003, 984, 1031, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1055, 1052, 1017, 1047, 1012, 1015, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 988, 988, 964, 1007, 995, 1013, 1034, 996, 5, 4, 0, 0, 0, 0, 0, 0, 1002, 1042, 1050, 1019, 1012, 1002, 1082, 1048, 7, 0, 0, 0, 0, 0, 0, 0, 1089, 1023, 1009, 1049, 1040, 1028, 1008, 949, 13, 0, 0, 0, 0, 0, 0, 0, 1010, 1034, 1043, 1059, 1064, 1024, 1030, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8254, 8093, 8203, 7994, 8264, 8208, 8326] avg=8192.0000 max=8326 max_rank=7 min=7994 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[992, 1014, 1029, 992, 1041, 1040, 1037, 994, 2, 0, 0, 0, 0, 0, 0, 0, 1034, 987, 986, 993, 1068, 1038, 1027, 1032, 35, 1, 0, 0, 0, 0, 0, 0, 1074, 1062, 1001, 1035, 1014, 981, 1011, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1051, 1046, 1050, 977, 1045, 1009, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1048, 1048, 1037, 1019, 1043, 1054, 1028, 5, 4, 0, 0, 0, 0, 0, 0, 1028, 998, 1056, 1023, 1008, 978, 1031, 1054, 7, 0, 0, 0, 0, 0, 0, 0, 1049, 994, 1015, 999, 963, 1041, 1072, 1004, 13, 0, 0, 0, 0, 0, 0, 0, 1037, 1015, 1006, 1010, 992, 1083, 1022, 984, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8201, 8206, 8222, 8284, 8183, 8150, 8149] avg=8192.0000 max=8284 max_rank=4 min=8141 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[979, 1103, 1022, 954, 1011, 1054, 1027, 971, 2, 0, 0, 0, 0, 0, 0, 0, 1079, 1027, 993, 1010, 996, 975, 984, 1001, 34, 1, 0, 0, 0, 0, 0, 0, 1028, 995, 1020, 1037, 1008, 1059, 1020, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1046, 1008, 1026, 1041, 1009, 1080, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1032, 1059, 994, 1027, 1034, 1028, 1024, 6, 4, 0, 0, 0, 0, 0, 0, 1050, 1061, 972, 1021, 960, 1027, 1011, 1009, 8, 0, 0, 0, 0, 0, 0, 0, 1030, 1009, 1040, 1039, 1027, 1004, 1038, 1022, 13, 0, 0, 0, 0, 0, 0, 0, 1042, 1036, 1027, 1022, 1016, 1015, 1021, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8100, 8179, 8305, 8261, 8119, 8222, 8227] avg=8192.0000 max=8305 max_rank=3 min=8100 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1043, 1031, 1049, 1057, 1051, 1056, 1059, 992, 2, 0, 0, 0, 0, 0, 0, 0, 997, 1020, 977, 994, 1024, 1019, 1017, 1019, 34, 1, 0, 0, 0, 0, 0, 0, 977, 1026, 999, 1028, 1099, 1028, 1042, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1029, 993, 1023, 1036, 1027, 1042, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1043, 978, 994, 995, 1017, 1050, 1021, 5, 4, 0, 0, 0, 0, 0, 0, 1003, 1041, 1047, 1057, 1067, 995, 1048, 1045, 8, 0, 0, 0, 0, 0, 0, 0, 1009, 980, 1012, 1004, 1002, 1022, 1049, 1021, 13, 0, 0, 0, 0, 0, 0, 0, 1023, 1039, 1006, 1024, 988, 1000, 1048, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8340, 8102, 8203, 8139, 8146, 8311, 8112, 8183] avg=8192.0000 max=8340 max_rank=0 min=8102 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[999, 1072, 1059, 1014, 1012, 1027, 980, 1070, 2, 0, 0, 0, 0, 0, 0, 0, 996, 1022, 1061, 1008, 1058, 1040, 1049, 1051, 33, 1, 0, 0, 0, 0, 0, 0, 1009, 1006, 1038, 1045, 967, 1039, 1043, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1026, 982, 1021, 1058, 1007, 1036, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 976, 997, 1032, 1009, 1051, 1015, 1030, 1032, 6, 4, 0, 0, 0, 0, 0, 0, 986, 1050, 947, 1006, 1031, 1032, 974, 1001, 8, 0, 0, 0, 0, 0, 0, 0, 1081, 991, 1047, 1010, 1051, 1054, 1006, 1034, 13, 0, 0, 0, 0, 0, 0, 0, 1050, 1048, 1003, 1031, 998, 1025, 1012, 986, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8319, 8154, 8201, 8152, 8035, 8287, 8153] avg=8192.0000 max=8319 max_rank=1 min=8035 min_rank=5 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1031, 993, 1036, 1073, 1001, 1025, 994, 1017, 4, 0, 0, 0, 0, 0, 0, 0, 1080, 1029, 1039, 1039, 999, 995, 1000, 994, 39, 4, 0, 0, 0, 0, 0, 0, 1020, 1058, 1029, 1065, 1023, 1049, 999, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1028, 999, 1043, 1039, 986, 974, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1006, 993, 1027, 1023, 1002, 1054, 1058, 9, 6, 0, 0, 0, 0, 0, 0, 996, 1011, 1051, 1043, 1047, 1034, 1013, 960, 13, 0, 0, 0, 0, 0, 0, 0, 979, 1046, 1025, 1046, 1095, 1070, 1012, 1003, 16, 0, 0, 0, 0, 0, 0, 0, 1057, 936, 1041, 1052, 985, 1025, 985, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8218, 8244, 8138, 8210, 8168, 8292, 8092] avg=8192.0000 max=8292 max_rank=6 min=8092 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1002, 1008, 1046, 1045, 986, 1012, 1052, 1022, 2, 0, 0, 0, 0, 0, 0, 0, 1016, 1022, 993, 1050, 983, 1009, 1002, 1015, 33, 1, 0, 0, 0, 0, 0, 0, 1054, 1024, 1054, 959, 1048, 1050, 1032, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1044, 981, 1049, 988, 1010, 1036, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1028, 1037, 1023, 1068, 986, 1036, 1023, 6, 4, 0, 0, 0, 0, 0, 0, 1049, 1021, 1023, 1024, 1040, 1003, 1033, 1029, 8, 0, 0, 0, 0, 0, 0, 0, 1034, 1016, 1037, 1010, 1008, 992, 1031, 991, 13, 0, 0, 0, 0, 0, 0, 0, 994, 1038, 1010, 1019, 1079, 1009, 1024, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8124, 8287, 8144, 8247, 8230, 8132, 8197] avg=8192.0000 max=8287 max_rank=2 min=8124 min_rank=1 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1017, 1025, 1008, 1035, 1047, 993, 992, 1035, 2, 0, 0, 0, 0, 0, 0, 0, 1036, 1057, 1005, 979, 1086, 1035, 977, 1009, 33, 1, 0, 0, 0, 0, 0, 0, 1009, 997, 1025, 1038, 1028, 1069, 1054, 950, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1016, 984, 1043, 946, 1053, 1009, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1042, 996, 1008, 1015, 1052, 1064, 1032, 6, 4, 0, 0, 0, 0, 0, 0, 1043, 1024, 1024, 1028, 992, 984, 1030, 1093, 8, 0, 0, 0, 0, 0, 0, 0, 1035, 1031, 1048, 983, 1008, 1028, 1003, 985, 12, 0, 0, 0, 0, 0, 0, 0, 1042, 1003, 1029, 1005, 1026, 1047, 1004, 1053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8218, 8170, 8184, 8242, 8226, 8133, 8209] avg=8192.0000 max=8242 max_rank=4 min=8133 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8448, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8320, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8064, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1050, 1013, 993, 1001, 1027, 998, 1024, 927, 1014, 1033, 1028, 1012, 1028, 1012, 1005, 1033, 992, 1046, 1022, 1053, 1019, 1101, 1038, 1012, 1029, 972, 1046, 1017, 1062, 1036, 1075, 993, 1016, 1086, 1025, 993, 1031, 1029, 1017, 1048, 1033, 1063, 1002, 1059, 1041, 998, 999, 1037, 1034, 1030, 1002, 988, 1096, 1044, 998, 992, 1055, 1047, 998, 1017, 1008, 1029, 1036, 974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1018, 1035, 1048, 1039, 1043, 983, 1023, 1046, 987, 1038, 1067, 1004, 1067, 1058, 1005, 1023, 1032, 1006, 1025, 1060, 1022, 1035, 1014, 1104, 1052, 1051, 1022, 1031, 1010, 997, 1060, 1007, 970, 1045, 999, 982, 1061, 999, 1005, 1054, 1041, 1018, 1019, 991, 998, 1014, 1006, 1018, 1015, 987, 1092, 1036, 989, 1043, 1003, 1020, 1023, 1053, 1004, 994, 984, 1030, 997, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[976, 1009, 1021, 1051, 1052, 1025, 1009, 1024, 1016, 1094, 1076, 1033, 1031, 996, 1061, 1048, 977, 1014, 1032, 1033, 1027, 976, 1042, 1007, 990, 1033, 1036, 1033, 1034, 1020, 1055, 1003, 1018, 995, 1012, 1031, 1012, 1101, 1025, 1001, 982, 1063, 1033, 1013, 1020, 1016, 1033, 979, 1004, 1035, 1038, 1012, 999, 1022, 1015, 1043, 1026, 1054, 982, 1057, 1022, 1005, 1024, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1034, 982, 1067, 1053, 990, 1048, 1052, 983, 1040, 1065, 966, 1003, 1055, 1028, 1048, 1002, 1012, 1009, 1026, 1026, 995, 1043, 1011, 1021, 1023, 1032, 992, 1048, 1036, 1006, 1039, 1007, 1048, 1009, 1015, 1012, 1052, 1031, 1021, 1024, 1032, 1014, 959, 1033, 1078, 996, 1018, 1030, 1017, 1010, 1053, 1059, 999, 1032, 1025, 984, 1007, 1039, 993, 1049, 1020, 1073, 1002, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1018, 1040, 1039, 1063, 1022, 1049, 1018, 974, 1031, 994, 1002, 1063, 1030, 1025, 1065, 1034, 1050, 1060, 1020, 1033, 1050, 1018, 985, 1035, 1037, 1016, 1038, 1023, 1031, 1007, 1040, 1036, 1034, 1022, 1014, 989, 1059, 1005, 1010, 987, 1091, 1048, 1044, 1017, 998, 1031, 995, 1036, 957, 1040, 1043, 997, 1031, 970, 1051, 1005, 992, 1006, 1039, 1021, 989, 1008, 996, 1065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[992, 1022, 1056, 986, 1034, 1010, 992, 985, 1035, 1007, 1004, 1005, 1048, 1005, 1064, 1047, 1028, 1071, 1012, 1039, 1041, 1003, 1009, 1040, 1034, 1035, 1040, 1018, 1051, 1009, 1036, 961, 1032, 1032, 1020, 1004, 1063, 989, 1008, 1000, 1053, 1034, 1041, 1029, 1013, 1046, 1003, 1027, 1032, 1038, 1039, 1026, 1051, 982, 1028, 1049, 1003, 995, 985, 1013, 1021, 1056, 1037, 1068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[972, 1017, 1003, 1024, 1058, 1009, 1006, 1012, 958, 1048, 997, 1057, 1048, 1032, 1012, 1034, 1056, 983, 981, 1044, 997, 1026, 981, 988, 985, 1052, 1006, 991, 1042, 1108, 1049, 1052, 1019, 1010, 1055, 1046, 1031, 1029, 1066, 1025, 997, 1010, 1027, 1035, 1029, 1041, 1046, 1004, 1040, 1005, 1002, 1033, 1061, 1042, 1018, 1037, 1047, 1013, 1048, 1020, 1047, 972, 1008, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1058, 1064, 1002, 1009, 1005, 1042, 1058, 1050, 996, 1052, 956, 1029, 1035, 1035, 1029, 1021, 1022, 972, 1056, 1040, 1000, 1104, 1038, 1022, 1013, 989, 1024, 994, 1038, 1060, 1045, 947, 1041, 1037, 1030, 1047, 1050, 973, 1019, 1031, 997, 1036, 1036, 1058, 1016, 1019, 1024, 1018, 1008, 1069, 1012, 1010, 1046, 952, 1049, 989, 1045, 986, 1027, 991, 985, 1053, 1024, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1050, 1013, 993, 1001, 1027, 998, 1024, 927, 25, 0, 0, 0, 0, 0, 0, 0, 1014, 1033, 1028, 1012, 999, 1012, 1005, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1046, 1022, 1041, 1019, 1101, 1038, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 972, 1046, 1017, 1062, 1036, 1061, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1086, 1025, 993, 1031, 1029, 1017, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1063, 1002, 1059, 1041, 998, 999, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1030, 1002, 988, 1096, 1044, 998, 992, 12, 0, 0, 0, 0, 0, 0, 0, 1055, 1047, 998, 1017, 1008, 1029, 1036, 974, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8058, 8136, 8271, 8216, 8245, 8232, 8196, 8182] avg=8192.0000 max=8271 max_rank=2 min=8058 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1018, 1035, 1048, 1039, 1043, 983, 1023, 1046, 25, 0, 0, 0, 0, 0, 0, 0, 987, 1038, 1067, 1004, 1038, 1058, 1005, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1006, 1025, 1048, 1022, 1035, 1014, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1051, 1022, 1031, 1010, 997, 1046, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1045, 999, 982, 1061, 999, 1005, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1018, 1019, 991, 998, 1014, 1006, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 987, 1092, 1036, 989, 1043, 1003, 1020, 12, 0, 0, 0, 0, 0, 0, 0, 1023, 1053, 1004, 994, 984, 1030, 997, 1034, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8220, 8286, 8216, 8115, 8105, 8197, 8137] avg=8192.0000 max=8286 max_rank=2 min=8105 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[976, 1009, 1021, 1051, 1052, 1025, 1009, 1024, 25, 0, 0, 0, 0, 0, 0, 0, 1016, 1094, 1076, 1033, 1002, 996, 1061, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1014, 1032, 1021, 1027, 976, 1042, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1033, 1036, 1033, 1034, 1020, 1041, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 995, 1012, 1031, 1012, 1101, 1025, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1063, 1033, 1013, 1020, 1016, 1033, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1035, 1038, 1012, 999, 1022, 1015, 1043, 12, 0, 0, 0, 0, 0, 0, 0, 1026, 1054, 982, 1057, 1022, 1005, 1024, 1030, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8326, 8096, 8190, 8195, 8139, 8180, 8218] avg=8192.0000 max=8326 max_rank=1 min=8096 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[992, 1022, 1056, 986, 1034, 1010, 992, 985, 25, 0, 0, 0, 0, 0, 0, 0, 1035, 1007, 1004, 1005, 1019, 1005, 1064, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1071, 1012, 1027, 1041, 1003, 1009, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1035, 1040, 1018, 1051, 1009, 1022, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1032, 1020, 1004, 1063, 989, 1008, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1034, 1041, 1029, 1013, 1046, 1003, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1038, 1039, 1026, 1051, 982, 1028, 1049, 12, 0, 0, 0, 0, 0, 0, 0, 1003, 995, 985, 1013, 1021, 1056, 1037, 1068, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8186, 8231, 8170, 8148, 8246, 8257, 8196] avg=8192.0000 max=8257 max_rank=6 min=8102 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1034, 982, 1067, 1053, 990, 1048, 1052, 983, 25, 0, 0, 0, 0, 0, 0, 0, 1040, 1065, 966, 1003, 1026, 1028, 1048, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1009, 1026, 1014, 995, 1043, 1011, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1032, 992, 1048, 1036, 1006, 1025, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1009, 1015, 1012, 1052, 1031, 1021, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1014, 959, 1033, 1078, 996, 1018, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1010, 1053, 1059, 999, 1032, 1025, 984, 12, 0, 0, 0, 0, 0, 0, 0, 1007, 1039, 993, 1049, 1020, 1073, 1002, 1060, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8178, 8131, 8169, 8212, 8160, 8191, 8261] avg=8192.0000 max=8261 max_rank=7 min=8131 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1018, 1040, 1039, 1063, 1022, 1049, 1018, 974, 28, 0, 0, 0, 0, 0, 0, 0, 1031, 994, 1002, 1063, 995, 1025, 1065, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1060, 1020, 1017, 1050, 1018, 985, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1016, 1038, 1023, 1031, 1007, 1020, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1022, 1014, 989, 1051, 1005, 1010, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 1048, 1044, 1017, 998, 1031, 995, 1036, 1, 0, 0, 0, 0, 0, 0, 0, 957, 1040, 1043, 997, 1031, 970, 1051, 1005, 16, 7, 1, 0, 0, 0, 0, 0, 992, 1006, 1039, 1021, 989, 1008, 996, 1065, 19, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8209, 8235, 8208, 8112, 8261, 8118, 8142] avg=8192.0000 max=8261 max_rank=5 min=8112 min_rank=4 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[972, 1017, 1003, 1024, 1058, 1009, 1006, 1012, 25, 0, 0, 0, 0, 0, 0, 0, 958, 1048, 997, 1057, 1019, 1032, 1012, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 983, 981, 1032, 997, 1026, 981, 988, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1052, 1006, 991, 1042, 1108, 1035, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1010, 1055, 1046, 1031, 1029, 1066, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1010, 1027, 1035, 1029, 1041, 1046, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1005, 1002, 1033, 1061, 1042, 1018, 1037, 12, 0, 0, 0, 0, 0, 0, 0, 1047, 1013, 1048, 1020, 1047, 972, 1008, 1045, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8157, 8044, 8271, 8281, 8189, 8250, 8218] avg=8192.0000 max=8281 max_rank=4 min=8044 min_rank=2 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32938 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1058, 1064, 1002, 1009, 1005, 1042, 1058, 1050, 25, 0, 0, 0, 0, 0, 0, 0, 996, 1052, 956, 1029, 1006, 1035, 1029, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 972, 1056, 1028, 1000, 1104, 1038, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 989, 1024, 994, 1038, 1060, 1031, 947, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1037, 1030, 1047, 1050, 973, 1019, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1036, 1036, 1058, 1016, 1019, 1024, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1069, 1012, 1010, 1046, 952, 1049, 989, 12, 0, 0, 0, 0, 0, 0, 0, 1045, 986, 1027, 991, 985, 1053, 1024, 1053, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8313, 8124, 8242, 8096, 8228, 8204, 8147, 8182] avg=8192.0000 max=8313 max_rank=0 min=8096 min_rank=3 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8064, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8448, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8448, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1074, 1055, 1036, 1013, 1019, 999, 1003, 1007, 998, 1031, 965, 983, 1057, 1016, 1008, 1004, 1044, 970, 1045, 998, 995, 1039, 1081, 1007, 1026, 1058, 1067, 1008, 1017, 982, 1055, 1033, 1032, 1043, 1033, 1069, 1057, 1046, 1025, 1031, 999, 1058, 976, 1018, 1049, 1009, 1009, 1085, 1014, 1015, 1038, 1011, 1044, 1017, 1024, 983, 1021, 1009, 1007, 995, 1075, 1046, 1011, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1088, 1021, 1057, 1059, 1067, 1057, 1023, 998, 1001, 979, 989, 992, 1052, 1064, 1007, 1002, 1026, 1048, 1067, 1055, 1014, 1002, 1032, 991, 1048, 993, 1007, 1066, 1024, 1036, 1063, 1060, 979, 1016, 1028, 1010, 1031, 1043, 1022, 997, 1011, 995, 1048, 1018, 1056, 1061, 1010, 957, 975, 1034, 1070, 1001, 1014, 999, 1018, 992, 1013, 994, 1028, 1053, 1033, 1045, 1007, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1070, 1052, 975, 1002, 1022, 1034, 1039, 1041, 1010, 1011, 1031, 1058, 1018, 1040, 999, 1032, 1055, 1062, 998, 1038, 1029, 1020, 1032, 1036, 978, 1003, 1061, 974, 1049, 1027, 1026, 1065, 1013, 1050, 986, 1051, 988, 1015, 1033, 990, 1053, 1014, 1009, 990, 1026, 1010, 1033, 1045, 1039, 1017, 1044, 1035, 988, 1044, 1035, 1041, 1041, 979, 1004, 1031, 1008, 1017, 1014, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[980, 1054, 1028, 1028, 1054, 1026, 1069, 1009, 1068, 1033, 1034, 1009, 1035, 1010, 1051, 999, 1048, 1061, 1096, 1012, 1035, 1023, 1055, 1055, 1087, 991, 1012, 1014, 1039, 1013, 970, 987, 992, 1023, 986, 1020, 1023, 1051, 982, 1027, 1014, 1021, 1008, 1012, 1015, 1023, 1036, 1013, 975, 1008, 1031, 1032, 1001, 1001, 1029, 1027, 1057, 1022, 1061, 1039, 1009, 989, 987, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1024, 996, 1066, 1010, 1045, 1011, 979, 1015, 1016, 976, 1056, 1097, 1018, 1022, 1026, 1042, 1061, 1018, 1133, 1050, 1048, 964, 1038, 1083, 985, 1045, 1038, 1006, 1067, 1015, 1043, 1009, 1001, 1032, 1009, 1020, 986, 1011, 1017, 1004, 976, 996, 1015, 993, 1009, 1037, 991, 1043, 990, 1053, 1046, 994, 1059, 1034, 1038, 1017, 1070, 1015, 992, 990, 1051, 1016, 973, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1012, 1019, 1030, 1049, 1038, 1036, 1002, 1063, 1002, 1054, 1005, 1038, 1044, 1025, 1036, 1044, 1077, 1033, 1062, 1017, 1023, 1026, 1033, 996, 1003, 1000, 1046, 1019, 1007, 1011, 1034, 978, 1021, 1057, 998, 1046, 1040, 1067, 1026, 1021, 1034, 1023, 960, 1067, 1017, 991, 987, 1025, 1015, 974, 1009, 1012, 1045, 1022, 1041, 1000, 1002, 1057, 1022, 989, 963, 1033, 1075, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1005, 1036, 1043, 1052, 1015, 996, 991, 997, 1055, 1043, 986, 1017, 1045, 1040, 1010, 1045, 1001, 998, 997, 1022, 1007, 982, 1016, 1071, 1022, 1005, 1076, 1047, 1000, 1018, 1009, 1060, 1080, 1022, 994, 1050, 999, 1033, 992, 1031, 1053, 995, 1046, 997, 1019, 989, 1044, 1001, 1014, 993, 1065, 1023, 1036, 1054, 990, 1021, 1033, 1021, 1083, 1069, 995, 1046, 1030, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[986, 1024, 1043, 1016, 999, 1053, 1025, 1003, 1034, 997, 1018, 999, 975, 984, 1051, 1044, 1008, 1025, 1000, 1014, 1045, 1061, 1052, 1082, 1051, 1014, 1042, 1047, 1024, 993, 1032, 985, 1030, 1008, 1032, 982, 969, 1012, 1070, 1071, 1025, 1005, 1030, 1033, 990, 1007, 999, 1065, 982, 1064, 1061, 1013, 970, 1031, 1016, 1066, 1045, 1067, 981, 1028, 1026, 1023, 1018, 1091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[980, 1054, 1004, 1028, 1054, 1026, 1069, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1033, 1034, 1009, 1035, 1010, 1051, 999, 9, 3, 0, 0, 0, 0, 0, 0, 1048, 1061, 1022, 1012, 1035, 1023, 1055, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 991, 1009, 1014, 1039, 1013, 970, 987, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1023, 986, 1020, 1023, 1051, 982, 1027, 14, 0, 0, 0, 0, 0, 0, 0, 1014, 1021, 1008, 1012, 1015, 1023, 1036, 1013, 51, 0, 0, 0, 0, 0, 0, 0, 975, 1008, 1031, 1032, 1001, 1001, 1029, 1027, 23, 1, 0, 0, 0, 0, 0, 0, 1057, 1022, 1061, 1039, 1009, 989, 987, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8251, 8311, 8110, 8118, 8193, 8128, 8201] avg=8192.0000 max=8311 max_rank=2 min=8110 min_rank=3 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1070, 1052, 952, 1002, 1022, 1034, 1039, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1011, 1031, 1058, 1018, 1040, 999, 1032, 9, 3, 0, 0, 0, 0, 0, 0, 1055, 1062, 931, 1038, 1029, 1020, 1032, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1003, 1057, 974, 1049, 1027, 1026, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1050, 986, 1051, 988, 1015, 1033, 990, 13, 0, 0, 0, 0, 0, 0, 0, 1053, 1014, 1009, 990, 1026, 1010, 1033, 1045, 46, 0, 0, 0, 0, 0, 0, 0, 1039, 1017, 1044, 1035, 988, 1044, 1035, 1041, 21, 1, 0, 0, 0, 0, 0, 0, 1041, 979, 1004, 1031, 1008, 1017, 1014, 1006, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8211, 8203, 8179, 8139, 8226, 8265, 8101] avg=8192.0000 max=8265 max_rank=6 min=8101 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1012, 1019, 1006, 1049, 1038, 1036, 1002, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1054, 1005, 1038, 1044, 1025, 1036, 1044, 9, 3, 0, 0, 0, 0, 0, 0, 1077, 1033, 990, 1017, 1023, 1026, 1033, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1000, 1042, 1019, 1007, 1011, 1034, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1057, 998, 1046, 1040, 1067, 1026, 1021, 14, 0, 0, 0, 0, 0, 0, 0, 1034, 1023, 960, 1067, 1017, 991, 987, 1025, 49, 0, 0, 0, 0, 0, 0, 0, 1015, 974, 1009, 1012, 1045, 1022, 1041, 1000, 23, 1, 0, 0, 0, 0, 0, 0, 1002, 1057, 1022, 989, 963, 1033, 1075, 1035, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8260, 8195, 8094, 8290, 8153, 8142, 8177] avg=8192.0000 max=8290 max_rank=4 min=8094 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1074, 1055, 1011, 1013, 1019, 999, 1003, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1031, 965, 983, 1057, 1016, 1008, 1004, 10, 3, 0, 0, 0, 0, 0, 0, 1044, 970, 975, 998, 995, 1039, 1081, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1058, 1063, 1008, 1017, 982, 1055, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1043, 1033, 1069, 1057, 1046, 1025, 1031, 14, 0, 0, 0, 0, 0, 0, 0, 999, 1058, 976, 1018, 1049, 1009, 1009, 1085, 48, 0, 0, 0, 0, 0, 0, 0, 1014, 1015, 1038, 1011, 1044, 1017, 1024, 983, 22, 1, 0, 0, 0, 0, 0, 0, 1021, 1009, 1007, 995, 1075, 1046, 1011, 994, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8075, 8109, 8242, 8350, 8251, 8169, 8159] avg=8192.0000 max=8350 max_rank=4 min=8075 min_rank=1 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1024, 996, 1041, 1010, 1045, 1011, 979, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 976, 1056, 1097, 1018, 1022, 1026, 1042, 10, 3, 0, 0, 0, 0, 0, 0, 1061, 1018, 1056, 1050, 1048, 964, 1038, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1045, 1035, 1006, 1067, 1015, 1043, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1032, 1009, 1020, 986, 1011, 1017, 1004, 14, 0, 0, 0, 0, 0, 0, 0, 976, 996, 1015, 993, 1009, 1037, 991, 1043, 53, 0, 0, 0, 0, 0, 0, 0, 990, 1053, 1046, 994, 1059, 1034, 1038, 1017, 24, 1, 0, 0, 0, 0, 0, 0, 1070, 1015, 992, 990, 1051, 1016, 973, 1056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8266, 8318, 8205, 8094, 8113, 8256, 8163] avg=8192.0000 max=8318 max_rank=2 min=8094 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1088, 1021, 1032, 1059, 1067, 1057, 1023, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 979, 989, 992, 1052, 1064, 1007, 1002, 10, 3, 0, 0, 0, 0, 0, 0, 1026, 1048, 995, 1055, 1014, 1002, 1032, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 993, 1004, 1066, 1024, 1036, 1063, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1016, 1028, 1010, 1031, 1043, 1022, 997, 14, 0, 0, 0, 0, 0, 0, 0, 1011, 995, 1048, 1018, 1056, 1061, 1010, 957, 49, 0, 0, 0, 0, 0, 0, 0, 975, 1034, 1070, 1001, 1014, 999, 1018, 992, 23, 1, 0, 0, 0, 0, 0, 0, 1013, 994, 1028, 1053, 1033, 1045, 1007, 990, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8345, 8099, 8163, 8294, 8140, 8205, 8127, 8163] avg=8192.0000 max=8345 max_rank=0 min=8099 min_rank=1 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1005, 1036, 1012, 1052, 1015, 996, 991, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1043, 986, 1017, 1045, 1040, 1010, 1045, 13, 5, 0, 0, 0, 0, 0, 0, 1001, 998, 920, 1022, 1007, 982, 1016, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1005, 1066, 1047, 1000, 1018, 1009, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 1022, 994, 1050, 999, 1033, 992, 1031, 16, 0, 0, 0, 0, 0, 0, 0, 1053, 995, 1046, 997, 1019, 989, 1044, 1001, 51, 0, 0, 0, 0, 0, 0, 0, 1014, 993, 1065, 1023, 1036, 1054, 990, 1021, 26, 2, 0, 0, 0, 0, 0, 0, 1033, 1021, 1083, 1069, 995, 1046, 1030, 1011, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8259, 8017, 8227, 8217, 8195, 8224, 8293] avg=8192.0000 max=8293 max_rank=7 min=8017 min_rank=2 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[986, 1024, 1018, 1016, 999, 1053, 1025, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 997, 1018, 999, 975, 984, 1051, 1044, 10, 3, 0, 0, 0, 0, 0, 0, 1008, 1025, 933, 1014, 1045, 1061, 1052, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1014, 1039, 1047, 1024, 993, 1032, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1008, 1032, 982, 969, 1012, 1070, 1071, 14, 0, 0, 0, 0, 0, 0, 0, 1025, 1005, 1030, 1033, 990, 1007, 999, 1065, 46, 0, 0, 0, 0, 0, 0, 0, 982, 1064, 1061, 1013, 970, 1031, 1016, 1066, 21, 1, 0, 0, 0, 0, 0, 0, 1045, 1067, 981, 1028, 1026, 1023, 1018, 1091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8115, 8220, 8185, 8188, 8200, 8225, 8279] avg=8192.0000 max=8279 max_rank=7 min=8115 min_rank=1 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32892 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 7936, 8320, 8320, 8192, 8448, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8448, 8192, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1080, 1030, 986, 1041, 1038, 1012, 977, 1046, 1028, 1019, 992, 1040, 1014, 1010, 1037, 1030, 1066, 992, 1012, 1043, 1067, 997, 1008, 1034, 1012, 1043, 982, 983, 1046, 1045, 1057, 995, 1013, 1072, 1022, 1003, 1049, 1063, 1040, 963, 1019, 1027, 1060, 1074, 1072, 991, 1035, 1024, 1070, 1056, 996, 1013, 990, 971, 1038, 1017, 1008, 1036, 972, 981, 1069, 998, 993, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1023, 973, 1062, 988, 1008, 1013, 1046, 984, 1022, 1047, 1045, 1006, 991, 1071, 1075, 1014, 981, 1020, 1033, 1006, 1046, 1027, 987, 1058, 1066, 1044, 990, 1052, 1004, 1050, 1062, 1056, 1033, 944, 1009, 999, 1002, 1005, 1045, 1016, 1043, 1017, 1014, 1044, 1039, 1042, 986, 1015, 996, 1030, 979, 1025, 1076, 1020, 993, 1077, 1051, 1037, 1037, 972, 1027, 1062, 988, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1045, 1008, 1004, 1039, 1028, 1048, 973, 997, 1031, 998, 1046, 1011, 1035, 1052, 1043, 1018, 1035, 1026, 986, 983, 1016, 1005, 1007, 990, 1032, 1040, 1051, 1012, 1058, 964, 1010, 1047, 1029, 1071, 1025, 1034, 1024, 1029, 999, 1051, 1023, 1038, 1041, 1009, 980, 1032, 1006, 1087, 1029, 1000, 1023, 1028, 1039, 1029, 982, 1043, 1041, 1027, 1020, 989, 1045, 1034, 1037, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1027, 1007, 1003, 1028, 1001, 1044, 1027, 1039, 1107, 1030, 1038, 1021, 982, 1009, 988, 1086, 1005, 1035, 1062, 1001, 999, 1008, 1016, 1018, 1022, 1010, 1033, 1019, 1050, 1041, 1018, 1051, 1091, 1039, 1045, 1023, 1040, 1055, 1063, 986, 1010, 1027, 1021, 1076, 944, 1058, 971, 1002, 981, 1001, 1022, 1037, 1055, 1034, 993, 1031, 1020, 1024, 1022, 999, 1027, 1013, 1009, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[987, 1001, 981, 1073, 1002, 1000, 1052, 1046, 1009, 966, 998, 1009, 1009, 1004, 1050, 1018, 1017, 1091, 1047, 1048, 1062, 989, 1052, 1026, 1013, 1039, 1006, 1023, 1023, 1073, 1040, 1042, 1021, 1028, 1020, 1016, 1014, 1042, 1058, 1022, 1030, 993, 1024, 1039, 988, 1045, 954, 1008, 1026, 1019, 1050, 1030, 1039, 1007, 1040, 1009, 1056, 1036, 1043, 957, 1022, 1012, 1089, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1006, 974, 1031, 1067, 1013, 1021, 1034, 987, 1025, 1026, 992, 993, 1050, 1026, 1016, 974, 1041, 1006, 1053, 1028, 981, 1002, 1035, 1013, 1022, 1049, 1035, 1057, 987, 1004, 988, 1044, 1049, 1001, 1036, 1053, 1036, 1038, 976, 1033, 1076, 1006, 1049, 1022, 1013, 1063, 1075, 1014, 1049, 1024, 980, 1000, 1033, 1035, 1011, 991, 993, 1029, 1049, 1024, 1034, 1108, 1023, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1045, 1064, 1043, 1047, 1032, 1011, 1037, 1059, 990, 1020, 1020, 1016, 1028, 1048, 1020, 999, 985, 1005, 1065, 1030, 1009, 1093, 1052, 997, 951, 1014, 1007, 1049, 1057, 1042, 991, 992, 1035, 1027, 1100, 1060, 1052, 997, 1048, 981, 1030, 1006, 1029, 997, 1061, 968, 1025, 999, 1058, 1004, 1054, 1034, 973, 1011, 1029, 1010, 1055, 975, 1029, 995, 1009, 992, 1049, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[992, 1018, 997, 1015, 953, 1029, 1000, 1057, 1039, 1081, 1063, 1027, 1049, 1033, 1027, 1020, 1013, 1080, 1013, 1021, 1011, 985, 1006, 1046, 1021, 1074, 1009, 1031, 1010, 1074, 1021, 1066, 1038, 1022, 1029, 1061, 1021, 973, 1042, 984, 1018, 1026, 1066, 1019, 1026, 1029, 1000, 1029, 1039, 1063, 1025, 970, 993, 1080, 1022, 978, 1065, 1014, 1031, 990, 1042, 1016, 997, 947] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1080, 1030, 986, 1041, 1038, 1012, 977, 1046, 29, 0, 0, 0, 0, 0, 0, 0, 1028, 1019, 992, 1040, 1014, 1010, 1032, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 992, 1012, 1043, 1067, 997, 1008, 1034, 5, 2, 0, 0, 0, 0, 0, 0, 1012, 1011, 982, 983, 1046, 1045, 1057, 995, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1072, 1022, 1003, 1049, 1063, 1040, 963, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1027, 1058, 1074, 1072, 991, 1035, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1056, 996, 1013, 990, 971, 1038, 1017, 18, 3, 0, 0, 0, 0, 0, 0, 1008, 1036, 972, 981, 1069, 998, 993, 1039, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8165, 8226, 8131, 8194, 8300, 8172, 8109] avg=8192.0000 max=8300 max_rank=5 min=8109 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1023, 973, 1062, 988, 1008, 1013, 1046, 984, 29, 0, 0, 0, 0, 0, 0, 0, 1022, 1047, 1045, 1006, 991, 1071, 1070, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1020, 1033, 1006, 1046, 1027, 987, 1058, 5, 2, 0, 0, 0, 0, 0, 0, 1066, 1012, 990, 1052, 1004, 1050, 1062, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 944, 1009, 999, 1002, 1005, 1045, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1017, 1012, 1044, 1039, 1042, 986, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1030, 979, 1025, 1076, 1020, 993, 1077, 18, 3, 0, 0, 0, 0, 0, 0, 1051, 1037, 1037, 972, 1027, 1062, 988, 1063, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8266, 8165, 8292, 8022, 8198, 8217, 8250] avg=8192.0000 max=8292 max_rank=3 min=8022 min_rank=4 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1045, 1008, 1004, 1039, 1028, 1048, 973, 997, 29, 0, 0, 0, 0, 0, 0, 0, 1031, 998, 1046, 1011, 1035, 1052, 1038, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1026, 986, 983, 1016, 1005, 1007, 990, 5, 2, 0, 0, 0, 0, 0, 0, 1032, 1008, 1051, 1012, 1058, 964, 1010, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1071, 1025, 1034, 1024, 1029, 999, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1038, 1039, 1009, 980, 1032, 1006, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1000, 1023, 1028, 1039, 1029, 982, 1043, 18, 3, 0, 0, 0, 0, 0, 0, 1041, 1027, 1020, 989, 1045, 1034, 1037, 1054, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8229, 8055, 8182, 8231, 8214, 8194, 8260] avg=8192.0000 max=8260 max_rank=7 min=8055 min_rank=2 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1045, 1064, 1043, 1047, 1032, 1011, 1037, 1059, 28, 0, 0, 0, 0, 0, 0, 0, 990, 1020, 1020, 1016, 1028, 1048, 1015, 999, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1005, 1065, 1030, 1009, 1093, 1052, 997, 5, 2, 0, 0, 0, 0, 0, 0, 951, 983, 1007, 1049, 1057, 1042, 991, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1027, 1100, 1060, 1052, 997, 1048, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1006, 1027, 997, 1061, 968, 1025, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1004, 1054, 1034, 973, 1011, 1029, 1010, 18, 3, 0, 0, 0, 0, 0, 0, 1055, 975, 1029, 995, 1009, 992, 1049, 1026, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8366, 8136, 8243, 8072, 8269, 8113, 8194, 8143] avg=8192.0000 max=8366 max_rank=0 min=8072 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1027, 1007, 1003, 1028, 1001, 1044, 1027, 1039, 28, 0, 0, 0, 0, 0, 0, 0, 1107, 1030, 1038, 1021, 982, 1009, 983, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1035, 1062, 1001, 999, 1008, 1016, 1018, 5, 2, 0, 0, 0, 0, 0, 0, 1022, 979, 1033, 1019, 1050, 1041, 1018, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1039, 1045, 1023, 1040, 1055, 1063, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1027, 1019, 1076, 944, 1058, 971, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1001, 1022, 1037, 1055, 1034, 993, 1031, 19, 3, 0, 0, 0, 0, 0, 0, 1020, 1024, 1022, 999, 1027, 1013, 1009, 992, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8256, 8151, 8213, 8309, 8107, 8176, 8120] avg=8192.0000 max=8309 max_rank=4 min=8107 min_rank=5 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1006, 974, 1031, 1067, 1013, 1021, 1034, 987, 29, 0, 0, 0, 0, 0, 0, 0, 1025, 1026, 992, 993, 1050, 1026, 1011, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1006, 1053, 1028, 981, 1002, 1035, 1013, 5, 2, 0, 0, 0, 0, 0, 0, 1022, 1017, 1035, 1057, 987, 1004, 988, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1001, 1036, 1053, 1036, 1038, 976, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1006, 1047, 1022, 1013, 1063, 1075, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1024, 980, 1000, 1033, 1035, 1011, 991, 18, 3, 0, 0, 0, 0, 0, 0, 993, 1029, 1049, 1024, 1034, 1108, 1023, 1033, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8097, 8166, 8154, 8191, 8316, 8144, 8306] avg=8192.0000 max=8316 max_rank=5 min=8097 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32957 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[987, 1001, 981, 1073, 1002, 1000, 1052, 1046, 35, 0, 0, 0, 0, 0, 0, 0, 1009, 966, 998, 1009, 1009, 1004, 1040, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1091, 1047, 1048, 1062, 989, 1052, 1026, 10, 4, 2, 0, 0, 0, 0, 0, 1013, 999, 1006, 1023, 1023, 1073, 1040, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1028, 1020, 1016, 1014, 1042, 1058, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 993, 1020, 1039, 988, 1045, 954, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1019, 1050, 1030, 1039, 1007, 1040, 1009, 23, 5, 0, 0, 0, 0, 0, 0, 1056, 1036, 1043, 957, 1022, 1012, 1089, 1003, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8053, 8348, 8219, 8181, 8077, 8248, 8233] avg=8192.0000 max=8348 max_rank=2 min=8053 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[992, 1018, 997, 1015, 953, 1029, 1000, 1057, 30, 0, 0, 0, 0, 0, 0, 0, 1039, 1081, 1063, 1027, 1049, 1033, 1022, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1080, 1013, 1021, 1011, 985, 1006, 1046, 5, 2, 0, 0, 0, 0, 0, 0, 1021, 1041, 1009, 1031, 1010, 1074, 1021, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1022, 1029, 1061, 1021, 973, 1042, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1026, 1064, 1019, 1026, 1029, 1000, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1063, 1025, 970, 993, 1080, 1022, 978, 18, 3, 0, 0, 0, 0, 0, 0, 1065, 1014, 1031, 990, 1042, 1016, 997, 947, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8091, 8334, 8182, 8273, 8139, 8211, 8191, 8115] avg=8192.0000 max=8334 max_rank=1 min=8091 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 8320, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 7936, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1075, 1028, 1044, 1051, 1021, 1031, 1007, 952, 1035, 1044, 1012, 1056, 976, 1003, 1036, 989, 983, 1027, 995, 1024, 997, 1028, 1009, 1058, 997, 1019, 989, 985, 1058, 1062, 1040, 1065, 1043, 994, 1004, 1025, 1082, 964, 1044, 1033, 1039, 1070, 1072, 1029, 1018, 1027, 1056, 1022, 1018, 1013, 1016, 1051, 1023, 978, 1039, 1047, 1002, 1001, 1010, 1029, 1015, 1021, 1040, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1034, 1011, 1017, 980, 1042, 988, 990, 1032, 1055, 1031, 1045, 1048, 1022, 1050, 1041, 1020, 1025, 1053, 1005, 1022, 981, 1017, 1078, 1041, 1059, 1054, 1018, 1033, 1012, 1054, 974, 1033, 1049, 1016, 989, 1040, 1007, 1010, 1032, 1034, 1017, 993, 981, 975, 1037, 1028, 1030, 978, 1007, 1009, 1030, 1021, 1075, 969, 1056, 1039, 1037, 1052, 1011, 1037, 1039, 1042, 970, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1039, 980, 996, 1046, 1019, 1069, 1039, 1046, 1082, 1005, 1046, 1011, 932, 1042, 1073, 1039, 1029, 968, 1047, 1026, 1001, 1010, 1026, 1011, 1003, 1008, 1063, 979, 1019, 1044, 1004, 1061, 1032, 1015, 1014, 1030, 1020, 1029, 1082, 978, 1020, 995, 1048, 969, 983, 1017, 988, 1072, 1044, 993, 1037, 1034, 977, 1075, 1056, 990, 1045, 1019, 1014, 1029, 1075, 1056, 991, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1024, 947, 1062, 964, 1028, 1047, 1038, 1061, 1034, 1023, 1049, 1005, 1049, 1025, 1008, 978, 994, 1035, 1052, 1040, 1031, 1008, 1031, 1025, 1024, 1037, 1061, 1024, 1080, 1015, 954, 1009, 1028, 1018, 1015, 1048, 991, 1020, 1041, 1050, 1049, 1015, 985, 983, 997, 1052, 1041, 1040, 1017, 983, 1044, 1018, 1009, 1055, 1060, 1009, 1053, 1016, 1038, 1005, 1002, 1046, 1045, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[982, 1019, 1006, 1034, 1049, 1015, 1029, 1070, 964, 1050, 1032, 1064, 1013, 1020, 1032, 1012, 1003, 992, 1027, 1031, 1030, 1044, 1002, 1010, 997, 1044, 1008, 1069, 1069, 1009, 1032, 1051, 1006, 1001, 1008, 985, 1003, 1003, 1020, 993, 1057, 1041, 1007, 1032, 999, 1042, 1034, 1035, 967, 1028, 1048, 998, 1025, 1055, 1066, 1007, 1045, 1011, 1000, 1017, 1050, 1063, 1026, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1019, 1018, 1061, 1011, 1024, 975, 1035, 1022, 1020, 1026, 981, 1069, 1075, 1024, 1016, 1027, 1014, 1002, 1020, 1049, 1094, 1032, 1002, 1008, 1003, 1058, 1016, 998, 998, 1007, 1008, 1021, 1038, 1043, 1042, 1015, 1035, 1061, 1032, 1055, 988, 1002, 1016, 1023, 1019, 1010, 1005, 1030, 1034, 1024, 1025, 1011, 1041, 960, 1044, 1069, 1014, 997, 1032, 1079, 1007, 999, 1036, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1048, 1005, 1036, 1049, 1014, 992, 1020, 1017, 1024, 1017, 1052, 1021, 1016, 1074, 1096, 989, 996, 1012, 1022, 1015, 1023, 1033, 1007, 1000, 961, 996, 1014, 1023, 1045, 1005, 1048, 1017, 1047, 1063, 1030, 1036, 1026, 1009, 1044, 986, 1020, 1059, 1062, 981, 1018, 1016, 993, 1024, 989, 1030, 1006, 1031, 1039, 1046, 1055, 1027, 1026, 1036, 1050, 983, 1035, 1024, 1032, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[962, 1054, 1066, 1038, 1025, 987, 1036, 1018, 1063, 1048, 1003, 1062, 1015, 986, 1014, 1053, 990, 1043, 1019, 975, 1025, 992, 1041, 1044, 1050, 969, 974, 1070, 1018, 1052, 1012, 1047, 1057, 1048, 1003, 1042, 981, 1000, 1023, 1031, 1059, 1032, 1014, 1055, 1001, 1009, 1022, 1013, 1026, 991, 1051, 1023, 1011, 996, 1027, 1029, 1085, 951, 1040, 996, 1029, 1051, 1059, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1075, 1028, 1044, 1051, 1021, 1031, 1007, 952, 7, 3, 0, 0, 0, 0, 0, 0, 1035, 1044, 1012, 1020, 976, 1003, 1036, 989, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1027, 995, 1024, 997, 1028, 1009, 1058, 33, 0, 0, 0, 0, 0, 0, 0, 997, 1019, 989, 985, 1058, 1062, 1040, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 994, 1004, 1025, 1082, 964, 1044, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1070, 1072, 1029, 1018, 1027, 1056, 1022, 20, 3, 0, 0, 0, 0, 0, 0, 1018, 1013, 1016, 1051, 1023, 978, 1036, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1001, 1010, 1029, 1015, 1021, 1040, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8115, 8154, 8215, 8189, 8356, 8182, 8106] avg=8192.0000 max=8356 max_rank=5 min=8106 min_rank=7 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1048, 1005, 1036, 1049, 1014, 992, 1020, 1017, 7, 4, 0, 0, 0, 0, 0, 0, 1024, 1017, 1052, 987, 1016, 1074, 1096, 989, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1012, 1022, 1015, 1023, 1033, 1007, 1000, 32, 0, 0, 0, 0, 0, 0, 0, 961, 996, 1014, 1023, 1045, 1005, 1048, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1063, 1030, 1036, 1026, 1009, 1044, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1059, 1062, 981, 1018, 1016, 993, 1024, 20, 2, 0, 0, 0, 0, 0, 0, 989, 1030, 1006, 1031, 1039, 1046, 1051, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1036, 1050, 983, 1035, 1024, 1032, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8255, 8140, 8109, 8241, 8195, 8219, 8185] avg=8192.0000 max=8255 max_rank=1 min=8109 min_rank=3 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1034, 1011, 1017, 980, 1042, 988, 990, 1032, 7, 4, 0, 0, 0, 0, 0, 0, 1055, 1031, 1045, 1013, 1022, 1050, 1041, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1053, 1005, 1022, 981, 1017, 1078, 1041, 32, 0, 0, 0, 0, 0, 0, 0, 1059, 1054, 1018, 1033, 1012, 1054, 974, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1016, 989, 1040, 1007, 1010, 1032, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 993, 981, 975, 1037, 1028, 1030, 978, 20, 3, 0, 0, 0, 0, 0, 0, 1007, 1009, 1030, 1021, 1075, 969, 1052, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1052, 1011, 1037, 1039, 1042, 970, 1061, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8277, 8254, 8237, 8177, 8062, 8202, 8222] avg=8192.0000 max=8277 max_rank=1 min=8062 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32663 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1024, 947, 1062, 964, 1028, 1047, 1038, 1061, 7, 4, 0, 0, 0, 0, 0, 0, 1034, 1023, 1049, 972, 1049, 1025, 1008, 978, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1035, 1052, 1040, 1031, 1008, 1031, 1025, 31, 0, 0, 0, 0, 0, 0, 0, 1024, 1037, 1061, 1024, 1080, 1015, 954, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1018, 1015, 1048, 991, 1020, 1041, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1015, 985, 983, 997, 1052, 1041, 1040, 21, 2, 0, 0, 0, 0, 0, 0, 1017, 983, 1044, 1018, 1009, 1055, 1056, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1016, 1038, 1005, 1002, 1046, 1045, 1001, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8138, 8247, 8204, 8211, 8185, 8191, 8178] avg=8192.0000 max=8247 max_rank=2 min=8138 min_rank=1 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1019, 1018, 1061, 1011, 1024, 975, 1035, 1022, 7, 3, 0, 0, 0, 0, 0, 0, 1020, 1026, 981, 1033, 1075, 1024, 1016, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1002, 1020, 1049, 1094, 1032, 1002, 1008, 33, 0, 0, 0, 0, 0, 0, 0, 1003, 1058, 1016, 998, 998, 1007, 1008, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1043, 1042, 1015, 1035, 1061, 1032, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1002, 1016, 1023, 1019, 1010, 1005, 1030, 20, 3, 0, 0, 0, 0, 0, 0, 1034, 1024, 1025, 1011, 1041, 960, 1041, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 987, 997, 1032, 1079, 1007, 999, 1036, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8202, 8254, 8109, 8321, 8116, 8205, 8154] avg=8192.0000 max=8321 max_rank=4 min=8109 min_rank=3 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1039, 980, 996, 1046, 1019, 1069, 1039, 1046, 7, 4, 0, 0, 0, 0, 0, 0, 1082, 1005, 1046, 978, 932, 1042, 1073, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 968, 1047, 1026, 1001, 1010, 1026, 1011, 31, 0, 0, 0, 0, 0, 0, 0, 1003, 1008, 1063, 979, 1019, 1044, 1004, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1015, 1014, 1030, 1020, 1029, 1082, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 995, 1048, 969, 983, 1017, 988, 1072, 21, 2, 0, 0, 0, 0, 0, 0, 1044, 993, 1037, 1034, 977, 1075, 1052, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1019, 1014, 1029, 1075, 1056, 991, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8197, 8149, 8181, 8200, 8115, 8202, 8247] avg=8192.0000 max=8247 max_rank=7 min=8115 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[962, 1054, 1066, 1038, 1025, 987, 1036, 1018, 11, 6, 0, 0, 0, 0, 0, 0, 1063, 1048, 1003, 1019, 1015, 986, 1014, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1043, 1019, 975, 1025, 992, 1041, 1044, 37, 0, 0, 0, 0, 0, 0, 0, 1050, 969, 974, 1070, 1018, 1052, 1012, 1047, 7, 3, 0, 0, 0, 0, 0, 0, 1057, 1048, 1003, 1042, 981, 1000, 1016, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1032, 1014, 1055, 1001, 1009, 1022, 1013, 25, 6, 0, 0, 0, 0, 0, 0, 1026, 991, 1051, 1023, 1011, 996, 1018, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 951, 1040, 996, 1029, 1051, 1059, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8201, 8166, 8202, 8178, 8236, 8145, 8205] avg=8192.0000 max=8236 max_rank=5 min=8145 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[982, 1019, 1006, 1034, 1049, 1015, 1029, 1070, 7, 4, 0, 0, 0, 0, 0, 0, 964, 1050, 1032, 1028, 1013, 1020, 1032, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 992, 1027, 1031, 1030, 1044, 1002, 1010, 33, 0, 0, 0, 0, 0, 0, 0, 997, 1044, 1008, 1069, 1069, 1009, 1032, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1001, 1008, 985, 1003, 1003, 1020, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1041, 1007, 1032, 999, 1042, 1034, 1035, 21, 3, 0, 0, 0, 0, 0, 0, 967, 1028, 1048, 998, 1025, 1055, 1062, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1011, 1000, 1017, 1050, 1063, 1026, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8151, 8172, 8279, 8019, 8271, 8190, 8239] avg=8192.0000 max=8279 max_rank=3 min=8019 min_rank=4 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8320, 8192, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8448, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8064, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8064, 8192, 8320, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[984, 1038, 1018, 1002, 1015, 1043, 1021, 1062, 995, 1029, 1039, 1055, 1074, 1003, 1013, 1004, 1028, 1010, 1016, 948, 1046, 1026, 1020, 1021, 1017, 1007, 1060, 1019, 999, 985, 1015, 1076, 958, 1048, 995, 1060, 1010, 1053, 1035, 1023, 1071, 1019, 983, 1046, 1020, 1002, 1005, 986, 1029, 1023, 1064, 1035, 1019, 1051, 1008, 1023, 1043, 978, 1057, 1073, 1046, 1061, 1003, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1032, 1027, 1058, 1013, 1060, 1048, 997, 1037, 1032, 1016, 1016, 1031, 1050, 1013, 1036, 983, 1078, 1017, 1058, 1006, 1047, 1025, 1067, 1041, 1032, 1000, 1015, 979, 1013, 982, 971, 1002, 1052, 1027, 1013, 1025, 1022, 1082, 998, 1006, 1026, 1036, 1048, 992, 1007, 1035, 1008, 1015, 996, 1052, 1054, 1024, 1015, 1026, 1018, 1064, 1015, 1019, 987, 1066, 999, 974, 993, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1016, 1033, 1018, 1034, 1010, 1023, 994, 1047, 1098, 1032, 1012, 1062, 982, 1110, 1038, 1032, 1012, 1018, 1027, 1019, 1031, 1051, 1032, 1064, 1017, 1016, 1050, 1010, 1060, 991, 989, 1040, 1010, 981, 996, 1040, 1053, 998, 1035, 1031, 972, 1011, 1001, 1012, 1005, 1002, 996, 1025, 1041, 1040, 1044, 1007, 1004, 1024, 995, 1037, 978, 1061, 1030, 1015, 1037, 1002, 1054, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[993, 1008, 1051, 1005, 1058, 1009, 1038, 1043, 1016, 1050, 1048, 1021, 982, 993, 1109, 1073, 1067, 959, 978, 1052, 1063, 1032, 1048, 1048, 993, 974, 1007, 1000, 1029, 1039, 995, 1041, 1033, 1010, 978, 1052, 972, 997, 1082, 1030, 1036, 1025, 1035, 1026, 1037, 1046, 1024, 1043, 949, 1035, 1043, 1012, 1069, 990, 1071, 1018, 1035, 978, 1006, 999, 1029, 1021, 1014, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1003, 1018, 1044, 1042, 1038, 1027, 1059, 1035, 1020, 992, 1047, 1034, 1024, 1009, 1026, 1033, 977, 1018, 1038, 1017, 1033, 988, 1018, 1056, 999, 1043, 1050, 1018, 1050, 976, 993, 1020, 1044, 975, 978, 1005, 1017, 1014, 1040, 1041, 1077, 1030, 986, 1017, 1042, 1032, 1046, 1018, 1042, 1033, 1059, 1013, 1031, 1042, 1049, 1001, 1090, 1046, 984, 960, 1017, 1043, 1035, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1041, 1054, 1035, 1048, 1009, 1037, 1054, 1010, 987, 1030, 1054, 1006, 1029, 1060, 1042, 1042, 1015, 1014, 1011, 1044, 1049, 1025, 1025, 1019, 1004, 1032, 1040, 1030, 1016, 1050, 1020, 1040, 1019, 1007, 1006, 999, 1025, 1038, 994, 1045, 1021, 1024, 984, 1006, 1050, 1040, 1036, 1004, 1025, 1030, 1023, 982, 990, 993, 1102, 978, 1012, 1038, 963, 1013, 1016, 1017, 1042, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1042, 1014, 997, 1043, 1053, 983, 993, 1056, 1059, 1040, 977, 1063, 1027, 1016, 1056, 1047, 1012, 1024, 1015, 1046, 999, 1027, 999, 988, 1022, 1025, 1069, 993, 1055, 1021, 1022, 1004, 1034, 994, 1055, 1034, 1043, 1007, 1027, 984, 983, 1063, 1019, 1036, 999, 1026, 1027, 1046, 1051, 1011, 989, 1051, 1026, 1060, 998, 1000, 1020, 1031, 1035, 997, 1018, 1043, 1005, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1060, 1006, 994, 1021, 1008, 1057, 999, 994, 1025, 1054, 998, 1005, 1000, 1010, 1022, 1029, 1075, 1004, 998, 1036, 1030, 1014, 1070, 1053, 1027, 966, 1026, 1049, 1010, 1041, 1054, 1024, 1002, 1079, 1077, 1004, 1046, 987, 997, 1009, 1012, 1041, 1062, 1025, 1046, 988, 990, 1037, 994, 1047, 1033, 1048, 1082, 973, 1037, 1017, 1011, 1031, 1054, 1003, 1011, 1013, 1006, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[984, 1038, 1018, 1002, 1015, 1043, 1021, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1029, 1039, 1055, 1074, 1003, 968, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1010, 1016, 948, 1027, 1026, 1020, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1007, 1060, 1019, 999, 985, 1015, 1076, 39, 0, 0, 0, 0, 0, 0, 0, 958, 1048, 995, 1060, 1010, 1053, 1035, 1023, 19, 6, 0, 0, 0, 0, 0, 0, 1071, 1019, 983, 1046, 1020, 1002, 1005, 986, 16, 0, 0, 0, 0, 0, 0, 0, 1029, 1023, 1045, 1035, 1019, 1051, 1008, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 978, 1057, 1073, 1046, 1061, 1003, 1021, 19, 3, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8167, 8096, 8217, 8207, 8148, 8233, 8306] avg=8192.0000 max=8306 max_rank=7 min=8096 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32894 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1016, 1033, 1018, 1034, 1010, 1023, 994, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1098, 1032, 1012, 1062, 982, 1110, 992, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1018, 1027, 1019, 1012, 1051, 1032, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1016, 1050, 1010, 1060, 991, 989, 1040, 40, 0, 0, 0, 0, 0, 0, 0, 1010, 981, 996, 1040, 1053, 998, 1035, 1031, 19, 6, 0, 0, 0, 0, 0, 0, 972, 1011, 1001, 1012, 1005, 1002, 996, 1025, 16, 0, 0, 0, 0, 0, 0, 0, 1041, 1040, 1025, 1007, 1004, 1024, 995, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1061, 1030, 1015, 1037, 1002, 1054, 1031, 19, 3, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8320, 8235, 8213, 8169, 8040, 8173, 8232] avg=8192.0000 max=8320 max_rank=1 min=8040 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1032, 1027, 1058, 1013, 1060, 1048, 997, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1016, 1016, 1031, 1050, 1013, 990, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1017, 1058, 1006, 1028, 1025, 1067, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1000, 1015, 979, 1013, 982, 971, 1002, 40, 0, 0, 0, 0, 0, 0, 0, 1052, 1027, 1013, 1025, 1022, 1082, 998, 1006, 19, 6, 0, 0, 0, 0, 0, 0, 1026, 1036, 1048, 992, 1007, 1035, 1008, 1015, 16, 0, 0, 0, 0, 0, 0, 0, 996, 1052, 1035, 1024, 1015, 1026, 1018, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1019, 987, 1066, 999, 974, 993, 1060, 19, 3, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8131, 8320, 8034, 8250, 8183, 8230, 8137] avg=8192.0000 max=8320 max_rank=2 min=8034 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[993, 1008, 1051, 1005, 1058, 1009, 1038, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1050, 1048, 1021, 982, 993, 1060, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 959, 978, 1052, 1044, 1032, 1048, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 993, 974, 1007, 1000, 1029, 1039, 995, 1041, 43, 0, 0, 0, 0, 0, 0, 0, 1033, 1010, 978, 1052, 972, 997, 1082, 1030, 19, 6, 0, 0, 0, 0, 0, 0, 1036, 1025, 1035, 1026, 1037, 1046, 1024, 1043, 16, 0, 0, 0, 0, 0, 0, 0, 949, 1035, 1024, 1012, 1069, 990, 1071, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 978, 1006, 999, 1029, 1021, 1014, 1019, 19, 3, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8243, 8228, 8121, 8179, 8288, 8168, 8125] avg=8192.0000 max=8288 max_rank=5 min=8121 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1060, 1006, 994, 1021, 1008, 1057, 999, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1054, 998, 1005, 1000, 1010, 977, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1004, 998, 1036, 1011, 1014, 1070, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 966, 1026, 1049, 1010, 1041, 1054, 1024, 39, 0, 0, 0, 0, 0, 0, 0, 1002, 1079, 1077, 1004, 1046, 987, 997, 1009, 18, 6, 0, 0, 0, 0, 0, 0, 1012, 1041, 1062, 1025, 1046, 988, 990, 1037, 15, 0, 0, 0, 0, 0, 0, 0, 994, 1047, 1015, 1048, 1082, 973, 1037, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1031, 1054, 1003, 1011, 1013, 1006, 1015, 19, 3, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8098, 8261, 8236, 8225, 8216, 8213, 8168] avg=8192.0000 max=8261 max_rank=2 min=8098 min_rank=1 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1003, 1018, 1044, 1042, 1038, 1027, 1059, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 992, 1047, 1034, 1024, 1009, 981, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1018, 1038, 1017, 1014, 988, 1018, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1043, 1050, 1018, 1050, 976, 993, 1020, 39, 0, 0, 0, 0, 0, 0, 0, 1044, 975, 978, 1005, 1017, 1014, 1040, 1041, 19, 6, 0, 0, 0, 0, 0, 0, 1077, 1030, 986, 1017, 1042, 1032, 1046, 1018, 16, 0, 0, 0, 0, 0, 0, 0, 1042, 1033, 1040, 1013, 1031, 1042, 1049, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 1046, 984, 960, 1017, 1043, 1035, 984, 19, 3, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8140, 8126, 8188, 8139, 8264, 8251, 8183] avg=8192.0000 max=8264 max_rank=5 min=8126 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1042, 1014, 997, 1043, 1053, 983, 993, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1040, 977, 1063, 1027, 1016, 1004, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1024, 1015, 1046, 976, 1027, 999, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1025, 1069, 993, 1055, 1021, 1022, 1004, 44, 0, 0, 0, 0, 0, 0, 0, 1034, 994, 1055, 1034, 1043, 1007, 1027, 984, 24, 8, 0, 0, 0, 0, 0, 0, 983, 1063, 1019, 1036, 999, 1026, 1027, 1046, 18, 0, 0, 0, 0, 0, 0, 0, 1051, 1011, 966, 1051, 1026, 1060, 998, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1031, 1035, 997, 1018, 1043, 1005, 1037, 23, 5, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8233, 8087, 8255, 8210, 8217, 8163, 8218] avg=8192.0000 max=8255 max_rank=3 min=8087 min_rank=2 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1041, 1054, 1035, 1048, 1009, 1037, 1054, 990, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1030, 1054, 1006, 1029, 1060, 996, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1014, 1011, 1044, 1030, 1025, 1025, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1032, 1040, 1030, 1016, 1050, 1020, 1040, 40, 0, 0, 0, 0, 0, 0, 0, 1019, 1007, 1006, 999, 1025, 1038, 994, 1045, 18, 6, 0, 0, 0, 0, 0, 0, 1021, 1024, 984, 1006, 1050, 1040, 1036, 1004, 15, 0, 0, 0, 0, 0, 0, 0, 1025, 1030, 1005, 982, 990, 993, 1102, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1038, 963, 1013, 1016, 1017, 1042, 1042, 19, 3, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8204, 8183, 8272, 8157, 8180, 8105, 8167] avg=8192.0000 max=8272 max_rank=3 min=8105 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8064, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8192, 8320, 8192, 8064, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[991, 1013, 1018, 1044, 1030, 974, 1035, 990, 1023, 1035, 971, 1060, 955, 1033, 1027, 1009, 1052, 1064, 1040, 1034, 1068, 1003, 1046, 1033, 1041, 1019, 1018, 1005, 1033, 942, 1080, 1037, 1043, 984, 1031, 1017, 947, 1031, 1055, 1026, 1033, 1013, 1037, 1039, 1004, 1053, 1075, 1015, 1080, 1036, 1029, 976, 1037, 1089, 1035, 992, 1018, 1008, 1023, 1037, 988, 1008, 1026, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1061, 1061, 1015, 1043, 1017, 1013, 1053, 1013, 1002, 978, 1040, 1017, 983, 1005, 980, 1035, 1043, 1014, 1041, 973, 987, 987, 1043, 979, 1038, 1042, 970, 1001, 1033, 1012, 1019, 1025, 1004, 1089, 1075, 1022, 1066, 1081, 1054, 1011, 1021, 974, 1005, 1061, 1044, 1032, 980, 1036, 1018, 1077, 1012, 996, 1016, 1059, 1064, 1031, 987, 1050, 1002, 1035, 992, 1040, 1036, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1068, 1010, 1069, 1078, 1025, 999, 1009, 1073, 1100, 1036, 1020, 1028, 1012, 995, 969, 1018, 1006, 1018, 981, 1040, 1033, 1018, 990, 1016, 1031, 1094, 1029, 1065, 1034, 972, 1001, 959, 1041, 998, 1071, 1016, 1027, 1071, 1024, 1075, 1003, 997, 1027, 1050, 998, 997, 1014, 1038, 1000, 1013, 1026, 1048, 950, 968, 1030, 1028, 1060, 1040, 1042, 976, 1065, 1026, 1008, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1027, 1047, 1052, 1015, 1002, 1043, 982, 1023, 1015, 1057, 1061, 1011, 1017, 1022, 1015, 1047, 1045, 967, 1001, 1082, 1031, 1011, 1018, 1060, 1067, 967, 1010, 1080, 1041, 1003, 1059, 1010, 996, 1017, 1026, 1010, 988, 1028, 1043, 1073, 1000, 1017, 1017, 1032, 1004, 1046, 1024, 967, 1085, 1008, 1018, 1022, 995, 1040, 1057, 998, 1056, 1034, 980, 993, 1010, 1011, 1017, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[966, 1065, 995, 1056, 1005, 1042, 1033, 1025, 1030, 998, 997, 1068, 1100, 1026, 1041, 1003, 1061, 1067, 1011, 1001, 1051, 1023, 1013, 964, 1006, 1029, 1021, 1003, 1040, 1064, 1038, 1020, 1019, 1003, 1027, 1000, 968, 1012, 1056, 1040, 1045, 1010, 1041, 1006, 1042, 1019, 1056, 1060, 1019, 1030, 1020, 1031, 1005, 979, 1039, 1005, 1026, 1027, 1028, 1019, 1021, 999, 1007, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1001, 974, 1008, 1026, 1085, 1069, 974, 1045, 1037, 960, 972, 1002, 1003, 1028, 1059, 982, 998, 1061, 1082, 1033, 1047, 1005, 1022, 1013, 1017, 1049, 1037, 997, 1010, 1027, 1032, 1043, 1018, 1006, 1024, 1011, 1056, 1031, 1004, 946, 997, 1022, 1041, 1042, 1053, 1018, 1078, 963, 1027, 1005, 1022, 1030, 1033, 1017, 1069, 1018, 1045, 1105, 983, 1053, 1039, 1033, 998, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1060, 1045, 1014, 982, 1046, 1016, 1028, 1001, 1013, 1020, 1048, 1015, 1019, 1042, 1048, 1009, 1063, 954, 991, 1001, 1010, 1014, 992, 1018, 990, 1046, 1009, 1055, 1018, 1030, 1000, 1025, 1063, 1042, 1025, 1036, 1048, 1054, 1007, 1014, 962, 1039, 1077, 1044, 1039, 1079, 997, 1025, 1010, 1007, 1013, 1058, 1017, 1083, 990, 1047, 1029, 1024, 1015, 977, 1039, 1003, 1029, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1069, 1044, 979, 1063, 1016, 996, 1059, 1056, 1020, 991, 1020, 1032, 1059, 1005, 1039, 1001, 1058, 1035, 1004, 989, 1006, 1063, 999, 1093, 986, 1031, 1023, 1035, 1048, 1019, 1073, 1047, 1023, 999, 1059, 1012, 1000, 1015, 947, 1022, 1038, 1024, 1068, 1034, 968, 1018, 1072, 1019, 996, 1015, 1057, 1001, 977, 1053, 986, 1068, 1023, 984, 1013, 1019, 1014, 1025, 1002, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1061, 1061, 1015, 1018, 1017, 1013, 1053, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 978, 1040, 1017, 983, 1005, 980, 1035, 25, 8, 0, 0, 0, 0, 0, 0, 1043, 1014, 1041, 973, 987, 987, 1043, 979, 5, 3, 0, 0, 0, 0, 0, 0, 1038, 1042, 970, 1001, 1033, 1012, 1012, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1089, 1064, 1022, 1066, 1081, 1054, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 974, 993, 1061, 1044, 1032, 980, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1077, 1012, 996, 1016, 1056, 1064, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1050, 1002, 1035, 992, 1040, 1036, 1043, 11, 4, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8073, 8075, 8133, 8391, 8141, 8270, 8202] avg=8192.0000 max=8391 max_rank=4 min=8073 min_rank=1 imbalance_ratio=1.024292 max_load_reduction=57145 max_load_reduction_pct=0.871964 moved_to_echo=33004 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1001, 974, 1008, 1002, 1085, 1069, 974, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 960, 972, 1002, 1003, 1028, 1059, 982, 24, 9, 0, 0, 0, 0, 0, 0, 998, 1061, 1082, 1033, 1047, 1005, 1022, 1013, 5, 2, 0, 0, 0, 0, 0, 0, 1017, 1049, 1037, 997, 1010, 1027, 1025, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1006, 1013, 1011, 1056, 1031, 1004, 946, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1022, 1027, 1042, 1053, 1018, 1078, 963, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1005, 1022, 1030, 1033, 1015, 1069, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1105, 983, 1053, 1039, 1033, 998, 1051, 11, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8076, 8268, 8205, 8085, 8200, 8219, 8325] avg=8192.0000 max=8325 max_rank=7 min=8076 min_rank=1 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1027, 1047, 1052, 991, 1002, 1043, 982, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1057, 1061, 1011, 1017, 1022, 1015, 1047, 24, 8, 0, 0, 0, 0, 0, 0, 1045, 967, 1001, 1082, 1031, 1011, 1018, 1060, 5, 3, 0, 0, 0, 0, 0, 0, 1067, 967, 1010, 1080, 1041, 1003, 1052, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1017, 1015, 1010, 988, 1028, 1043, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1017, 1005, 1032, 1004, 1046, 1024, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 1008, 1018, 1022, 995, 1037, 1057, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1034, 980, 993, 1010, 1011, 1017, 1036, 11, 4, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8277, 8223, 8230, 8170, 8095, 8220, 8154] avg=8192.0000 max=8277 max_rank=1 min=8095 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[991, 1013, 1018, 1019, 1030, 974, 1035, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1035, 971, 1060, 955, 1033, 1027, 1009, 25, 9, 0, 0, 0, 0, 0, 0, 1052, 1064, 1040, 1034, 1068, 1003, 1046, 1033, 5, 3, 0, 0, 0, 0, 0, 0, 1041, 1019, 1018, 1005, 1033, 942, 1073, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 984, 1020, 1017, 947, 1031, 1055, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1013, 1024, 1039, 1004, 1053, 1075, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 1036, 1029, 976, 1037, 1086, 1035, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1008, 1023, 1037, 988, 1008, 1026, 1028, 11, 4, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 8147, 8348, 8168, 8123, 8256, 8271, 8153] avg=8192.0000 max=8348 max_rank=2 min=8070 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1068, 1010, 1069, 1053, 1025, 999, 1009, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 1036, 1020, 1028, 1012, 995, 969, 1018, 25, 9, 0, 0, 0, 0, 0, 0, 1006, 1018, 981, 1040, 1033, 1018, 990, 1016, 5, 2, 0, 0, 0, 0, 0, 0, 1031, 1094, 1029, 1065, 1034, 972, 994, 959, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 998, 1060, 1016, 1027, 1071, 1024, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 997, 1014, 1050, 998, 997, 1014, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1013, 1026, 1048, 950, 966, 1030, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1040, 1042, 976, 1065, 1026, 1008, 1013, 11, 4, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 8212, 8109, 8178, 8312, 8111, 8061, 8247] avg=8192.0000 max=8312 max_rank=4 min=8061 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[966, 1065, 995, 1031, 1005, 1042, 1033, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 998, 997, 1068, 1100, 1026, 1041, 1003, 25, 9, 0, 0, 0, 0, 0, 0, 1061, 1067, 1011, 1001, 1051, 1023, 1013, 964, 5, 2, 0, 0, 0, 0, 0, 0, 1006, 1029, 1021, 1003, 1040, 1064, 1031, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1003, 1016, 1000, 968, 1012, 1056, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1010, 1027, 1006, 1042, 1019, 1056, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1030, 1020, 1031, 1005, 977, 1039, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1027, 1028, 1019, 1021, 999, 1007, 1015, 11, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8297, 8198, 8214, 8114, 8265, 8126, 8160] avg=8192.0000 max=8297 max_rank=1 min=8114 min_rank=4 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1060, 1045, 1014, 955, 1046, 1016, 1028, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1020, 1048, 1015, 1019, 1042, 1048, 1009, 27, 12, 0, 0, 0, 0, 0, 0, 1063, 954, 991, 1001, 1010, 1014, 992, 1018, 11, 6, 0, 0, 0, 0, 0, 0, 990, 1046, 1009, 1055, 1018, 1030, 980, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1042, 1011, 1036, 1048, 1054, 1007, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1039, 1056, 1044, 1039, 1079, 997, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1007, 1013, 1058, 1017, 1077, 990, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1024, 1015, 977, 1039, 1003, 1029, 1022, 14, 9, 9, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8253, 8060, 8153, 8275, 8241, 8219, 8170] avg=8192.0000 max=8275 max_rank=4 min=8060 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32905 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1069, 1044, 979, 1038, 1016, 996, 1059, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 991, 1020, 1032, 1059, 1005, 1039, 1001, 25, 9, 0, 0, 0, 0, 0, 0, 1058, 1035, 1004, 989, 1006, 1063, 999, 1093, 5, 3, 0, 0, 0, 0, 0, 0, 986, 1031, 1023, 1035, 1048, 1019, 1066, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 999, 1048, 1012, 1000, 1015, 947, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1024, 1054, 1034, 968, 1018, 1072, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1015, 1057, 1001, 977, 1050, 986, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 984, 1013, 1019, 1014, 1025, 1002, 1027, 11, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8201, 8255, 8255, 8066, 8227, 8150, 8125] avg=8192.0000 max=8257 max_rank=0 min=8066 min_rank=4 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8448, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8192, 8192, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8064, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1025, 1044, 1041, 1014, 973, 1056, 1001, 961, 1005, 1010, 1025, 1018, 1035, 1024, 1044, 1024, 1028, 1111, 1036, 987, 1008, 1059, 984, 1045, 1005, 963, 1025, 980, 1022, 991, 996, 987, 1041, 1007, 1022, 1019, 1035, 1012, 1025, 1027, 1041, 1023, 1032, 1028, 963, 1046, 1005, 1075, 1010, 1038, 1040, 1028, 1044, 997, 1059, 1057, 1030, 1055, 1042, 1031, 1081, 1011, 1039, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1023, 1007, 1017, 1040, 1041, 1043, 1006, 1034, 992, 1036, 1021, 1048, 992, 1014, 975, 997, 1007, 1022, 1015, 1026, 1058, 1000, 1005, 1042, 1034, 1002, 1052, 1024, 1019, 1057, 1013, 1032, 1024, 1048, 1029, 1061, 1029, 1002, 1065, 1073, 1023, 1047, 1021, 1021, 1088, 1003, 1007, 997, 1071, 1002, 1014, 1047, 1012, 1007, 1015, 993, 1004, 976, 982, 1038, 1007, 1055, 1052, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1007, 1015, 1016, 1028, 1013, 1053, 1015, 1005, 1043, 999, 1035, 972, 1067, 978, 1024, 1004, 1026, 1015, 1008, 1042, 1068, 1010, 1043, 974, 978, 1029, 1041, 1017, 1034, 985, 1034, 1057, 1059, 1015, 1027, 989, 1014, 1013, 1076, 1006, 1034, 1005, 1089, 1044, 1039, 1071, 984, 1068, 1016, 1008, 1066, 1034, 1029, 998, 997, 1010, 1047, 1045, 1000, 1018, 1042, 1018, 1015, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1013, 1012, 1038, 969, 1054, 1016, 1032, 1059, 1074, 995, 1014, 1024, 996, 1034, 998, 1023, 967, 965, 1018, 1058, 989, 1024, 1007, 1001, 992, 1016, 1080, 1035, 1047, 1043, 995, 1065, 1030, 1001, 1042, 1061, 1023, 998, 1041, 1008, 1034, 1065, 986, 1031, 973, 1054, 995, 1019, 1027, 1027, 1064, 1059, 1007, 1039, 1040, 1015, 1017, 1012, 1040, 1004, 1023, 1019, 1065, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1037, 1013, 1064, 985, 1050, 1027, 1052, 1029, 989, 995, 1007, 1050, 1005, 1008, 1038, 1055, 995, 996, 1058, 989, 991, 995, 1025, 977, 1062, 1010, 1041, 1007, 986, 1020, 1028, 1044, 1059, 1047, 1007, 1019, 1038, 1044, 1033, 1044, 1000, 988, 1051, 1036, 1020, 1050, 1050, 1025, 987, 1018, 1015, 981, 982, 1008, 1013, 1057, 1033, 1044, 1028, 1060, 1067, 1001, 1066, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1034, 1029, 1055, 1024, 1012, 1026, 1010, 1011, 1042, 1013, 1005, 1028, 1034, 1039, 1014, 1017, 1039, 992, 1039, 1008, 1010, 978, 1062, 1024, 1083, 1023, 1060, 1022, 1062, 990, 1007, 999, 1076, 1044, 1062, 1056, 1052, 965, 1025, 1036, 1037, 951, 1000, 1058, 1027, 1024, 1003, 1046, 1007, 1026, 1057, 998, 996, 1004, 1011, 978, 1031, 1052, 1004, 1015, 1003, 1056, 991, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1025, 1052, 1006, 1024, 997, 988, 1014, 992, 998, 1011, 1034, 1015, 1037, 1075, 976, 1078, 1044, 1000, 1023, 974, 970, 1071, 1065, 998, 1019, 967, 1028, 1050, 1071, 1004, 1040, 1007, 1036, 976, 999, 1008, 1029, 1031, 1036, 1014, 1026, 1047, 1085, 1007, 1033, 983, 988, 1031, 1008, 1052, 1018, 1067, 994, 1051, 1085, 1116, 967, 1014, 1018, 1062, 1008, 1082, 994, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1008, 995, 1031, 986, 1037, 1063, 1020, 1026, 982, 1002, 1014, 1040, 1025, 1025, 1077, 989, 1026, 1049, 1035, 1014, 1025, 999, 1053, 1023, 1089, 1051, 1031, 993, 1006, 1002, 1014, 1017, 994, 999, 1044, 1024, 1055, 1060, 1013, 1042, 1040, 977, 1024, 1072, 1041, 1079, 1095, 974, 1030, 1052, 1046, 978, 988, 957, 1025, 1063, 977, 1018, 1040, 1069, 1012, 981, 1012, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1025, 1044, 1041, 1014, 973, 1056, 1001, 961, 17, 0, 0, 0, 0, 0, 0, 0, 1005, 1010, 1025, 1018, 1035, 1024, 1044, 1024, 30, 2, 1, 0, 0, 0, 0, 0, 1028, 1111, 1036, 987, 1008, 1059, 984, 1045, 42, 0, 0, 0, 0, 0, 0, 0, 1005, 963, 1025, 980, 1022, 991, 996, 987, 6, 0, 0, 0, 0, 0, 0, 0, 998, 1007, 1022, 1019, 1035, 1012, 1025, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1023, 1032, 1028, 963, 1016, 1005, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1038, 1040, 1028, 1044, 997, 1059, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1055, 1042, 1006, 1081, 1011, 1039, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8218, 8300, 7975, 8145, 8183, 8273, 8310] avg=8192.0000 max=8310 max_rank=7 min=7975 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32911 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1013, 1012, 1038, 969, 1054, 1016, 1032, 1059, 17, 0, 0, 0, 0, 0, 0, 0, 1074, 995, 1014, 1024, 996, 1034, 998, 1023, 30, 2, 1, 0, 0, 0, 0, 0, 967, 965, 1018, 1058, 989, 1024, 1007, 1001, 42, 0, 0, 0, 0, 0, 0, 0, 992, 1016, 1080, 1035, 1047, 1043, 995, 1065, 6, 0, 0, 0, 0, 0, 0, 0, 987, 1001, 1042, 1061, 1023, 998, 1041, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1065, 986, 1031, 973, 1024, 995, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1027, 1064, 1059, 1007, 1039, 1040, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1012, 1040, 979, 1023, 1019, 1065, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8191, 8071, 8279, 8161, 8127, 8278, 8219] avg=8192.0000 max=8279 max_rank=3 min=8071 min_rank=2 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1007, 1015, 1016, 1028, 1013, 1053, 1015, 1005, 17, 0, 0, 0, 0, 0, 0, 0, 1043, 999, 1035, 972, 1067, 978, 1024, 1004, 31, 2, 1, 0, 0, 0, 0, 0, 1026, 1015, 1008, 1042, 1068, 1010, 1043, 974, 43, 0, 0, 0, 0, 0, 0, 0, 978, 1029, 1041, 1017, 1034, 985, 1034, 1057, 6, 0, 0, 0, 0, 0, 0, 0, 1015, 1015, 1027, 989, 1014, 1013, 1076, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1005, 1089, 1044, 1039, 1040, 984, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1008, 1066, 1034, 1029, 998, 997, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1045, 1000, 993, 1042, 1018, 1015, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8156, 8229, 8181, 8155, 8303, 8158, 8185] avg=8192.0000 max=8303 max_rank=5 min=8155 min_rank=4 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1023, 1007, 1017, 1040, 1041, 1043, 1006, 1034, 17, 0, 0, 0, 0, 0, 0, 0, 992, 1036, 1021, 1048, 992, 1014, 975, 997, 29, 3, 1, 0, 0, 0, 0, 0, 1007, 1022, 1015, 1026, 1058, 1000, 1005, 1042, 41, 0, 0, 0, 0, 0, 0, 0, 1034, 1002, 1052, 1024, 1019, 1057, 1013, 1032, 6, 0, 0, 0, 0, 0, 0, 0, 982, 1048, 1029, 1061, 1029, 1002, 1065, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1047, 1021, 1021, 1088, 974, 1007, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1002, 1014, 1047, 1012, 1007, 1015, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 976, 982, 1012, 1007, 1055, 1052, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8108, 8216, 8239, 8289, 8178, 8161, 8117] avg=8192.0000 max=8289 max_rank=4 min=8108 min_rank=1 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1037, 1013, 1064, 985, 1050, 1027, 1052, 1029, 18, 0, 0, 0, 0, 0, 0, 0, 989, 995, 1007, 1050, 1005, 1008, 1038, 1055, 30, 3, 1, 0, 0, 0, 0, 0, 995, 996, 1058, 989, 991, 995, 1025, 977, 43, 0, 0, 0, 0, 0, 0, 0, 1062, 1010, 1041, 1007, 986, 1020, 1028, 1044, 6, 0, 0, 0, 0, 0, 0, 0, 1015, 1047, 1007, 1019, 1038, 1044, 1033, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 988, 1051, 1036, 1020, 1020, 1050, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1018, 1015, 981, 982, 1008, 1013, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1044, 1028, 1033, 1067, 1001, 1066, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8181, 8069, 8204, 8247, 8190, 8061, 8309] avg=8192.0000 max=8309 max_rank=7 min=8061 min_rank=6 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1034, 1029, 1055, 1024, 1012, 1026, 1010, 1011, 17, 0, 0, 0, 0, 0, 0, 0, 1042, 1013, 1005, 1028, 1034, 1039, 1014, 1017, 29, 2, 1, 0, 0, 0, 0, 0, 1039, 992, 1039, 1008, 1010, 978, 1062, 1024, 44, 0, 0, 0, 0, 0, 0, 0, 1083, 1023, 1060, 1022, 1062, 990, 1007, 999, 6, 0, 0, 0, 0, 0, 0, 0, 1031, 1044, 1062, 1056, 1052, 965, 1025, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 951, 1000, 1058, 1027, 995, 1003, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1026, 1057, 998, 996, 1004, 1011, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1052, 1004, 990, 1003, 1056, 991, 1054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8224, 8196, 8252, 8271, 8117, 8077, 8181] avg=8192.0000 max=8271 max_rank=4 min=8077 min_rank=6 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1025, 1052, 1006, 1024, 997, 988, 1014, 992, 18, 0, 0, 0, 0, 0, 0, 0, 998, 1011, 1034, 1015, 1037, 1075, 976, 1078, 28, 3, 1, 0, 0, 0, 0, 0, 1044, 1000, 1023, 974, 970, 1071, 1065, 998, 42, 0, 0, 0, 0, 0, 0, 0, 1019, 967, 1028, 1050, 1071, 1004, 1040, 1007, 6, 0, 0, 0, 0, 0, 0, 0, 993, 976, 999, 1008, 1029, 1031, 1036, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1047, 1085, 1007, 1033, 955, 988, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1052, 1018, 1067, 994, 1051, 1085, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1014, 1018, 1035, 1008, 1082, 994, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8256, 8187, 8192, 8086, 8172, 8391, 8136] avg=8192.0000 max=8391 max_rank=6 min=8086 min_rank=4 imbalance_ratio=1.024292 max_load_reduction=57145 max_load_reduction_pct=0.871964 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1008, 995, 1031, 986, 1037, 1063, 1020, 1026, 22, 0, 0, 0, 0, 0, 0, 0, 982, 1002, 1014, 1040, 1025, 1025, 1077, 989, 36, 7, 5, 0, 0, 0, 0, 0, 1026, 1049, 1035, 1014, 1025, 999, 1053, 1023, 44, 0, 0, 0, 0, 0, 0, 0, 1089, 1051, 1031, 993, 1006, 1002, 1014, 1017, 9, 2, 0, 0, 0, 0, 0, 0, 945, 999, 1044, 1024, 1055, 1060, 1013, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 977, 1024, 1072, 1041, 1043, 1095, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1052, 1044, 978, 988, 957, 1025, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1018, 1040, 1031, 1012, 981, 1012, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8202, 8268, 8214, 8182, 8266, 8137, 8079] avg=8192.0000 max=8268 max_rank=2 min=8079 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8192, 8320, 8192, 8320, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 8064, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8064, 8448, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1061, 1040, 1008, 1030, 984, 1046, 1008, 1037, 989, 983, 1039, 1038, 1039, 1024, 1015, 1022, 1049, 1019, 1034, 972, 982, 1004, 1038, 1021, 1029, 1097, 1050, 1060, 1046, 1032, 1038, 1031, 1050, 1022, 1011, 960, 1036, 1008, 1024, 1009, 1066, 1070, 1039, 1042, 996, 977, 1016, 1024, 990, 1026, 997, 1010, 1051, 1020, 973, 1011, 1031, 1014, 1040, 990, 1064, 1014, 1009, 1081] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1030, 1009, 1051, 1028, 996, 1008, 1025, 1047, 1002, 997, 1041, 1039, 1024, 1039, 984, 1047, 1021, 974, 1030, 1079, 1038, 1005, 1080, 1059, 958, 1031, 1068, 989, 997, 1034, 997, 1049, 1018, 1001, 1031, 1004, 1045, 972, 1035, 1049, 1043, 1066, 999, 1031, 1049, 1045, 1004, 969, 1014, 1026, 1062, 1034, 1046, 1055, 1019, 1000, 1012, 1024, 1006, 993, 991, 1013, 1067, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1004, 1007, 1058, 1012, 1026, 1010, 1048, 998, 992, 980, 1021, 1050, 1048, 1058, 996, 1049, 1015, 1065, 1048, 979, 1009, 1035, 1034, 1010, 1058, 1044, 1010, 1027, 1010, 1027, 1015, 1028, 1040, 993, 983, 1008, 1037, 1000, 1054, 1029, 1015, 1044, 993, 1075, 1036, 997, 1109, 1027, 1020, 1090, 992, 1037, 1018, 1002, 1050, 1015, 977, 1049, 1012, 1028, 982, 1053, 1001, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1044, 1017, 1033, 1026, 1027, 1048, 1000, 973, 1005, 1038, 1032, 1049, 1091, 1008, 1102, 1020, 979, 1032, 1026, 1029, 991, 1019, 1014, 969, 994, 1031, 1019, 1037, 985, 1040, 956, 977, 1011, 997, 1000, 1039, 1033, 1014, 1014, 1048, 1012, 1041, 1074, 1007, 1011, 1036, 1024, 1020, 1023, 1002, 1009, 1021, 1016, 1069, 1002, 1000, 1012, 1053, 1021, 1009, 1084, 1080, 1044, 1099] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1013, 996, 1024, 1020, 1045, 1034, 1009, 1049, 993, 1033, 985, 1018, 1026, 1046, 1052, 1036, 1029, 1037, 1026, 1031, 1047, 1023, 1042, 1006, 1039, 1017, 1033, 996, 969, 1031, 1049, 1080, 1079, 1006, 1046, 1021, 1013, 1065, 1009, 1057, 983, 1003, 1003, 1011, 1005, 999, 1031, 1048, 988, 1060, 1021, 981, 1068, 1035, 1040, 1015, 1007, 1063, 1043, 1015, 992, 978, 1043, 974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[988, 1068, 1046, 1052, 1002, 1063, 991, 1047, 998, 1050, 1052, 1005, 1025, 994, 1051, 1033, 1067, 1001, 1034, 1014, 1038, 1054, 1014, 990, 987, 1034, 975, 1014, 1044, 971, 1055, 1008, 1000, 1001, 1027, 994, 1051, 1013, 1040, 1019, 1016, 1015, 995, 1034, 1040, 1034, 997, 990, 1018, 1029, 1013, 1040, 1034, 1032, 976, 1033, 1020, 1120, 1004, 1036, 1063, 1018, 1032, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1023, 1003, 1040, 1019, 1017, 1028, 1016, 998, 1039, 1018, 1005, 1012, 1039, 974, 1044, 998, 992, 1091, 1025, 1044, 1031, 972, 1031, 1036, 1003, 1008, 1045, 1044, 1046, 1014, 1059, 1060, 1048, 1015, 1050, 1083, 1010, 1015, 1049, 1006, 1033, 1050, 1029, 978, 1097, 1016, 1011, 1021, 996, 1044, 1009, 978, 1031, 993, 973, 1032, 1044, 1030, 1032, 1051, 1009, 979, 1007, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[997, 1028, 1048, 1038, 1007, 1060, 1024, 1037, 1006, 998, 986, 1027, 1019, 1064, 1055, 1029, 1032, 1007, 948, 991, 1044, 965, 1023, 1020, 1043, 989, 1057, 963, 1036, 974, 1050, 988, 1006, 1021, 1035, 1030, 1035, 1061, 999, 1023, 1010, 1045, 1058, 1037, 1043, 1059, 994, 1023, 1045, 1006, 1062, 1013, 1075, 1064, 992, 1031, 1014, 977, 1052, 1042, 1032, 1043, 1007, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1061, 1040, 1004, 1030, 984, 1046, 1008, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 989, 983, 1039, 1038, 1035, 1024, 1015, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1019, 1034, 972, 982, 1004, 1038, 1021, 20, 0, 0, 0, 0, 0, 0, 0, 1029, 1097, 1050, 1060, 1046, 1032, 1038, 1031, 7, 4, 0, 0, 0, 0, 0, 0, 1050, 1022, 1011, 960, 1036, 1008, 1024, 1009, 4, 0, 0, 0, 0, 0, 0, 0, 1066, 1054, 1039, 1042, 996, 977, 1016, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1026, 997, 1010, 1051, 1020, 973, 1011, 9, 4, 0, 0, 0, 0, 0, 0, 1031, 990, 1040, 990, 1064, 1014, 1009, 1081, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8145, 8139, 8394, 8124, 8214, 8091, 8219] avg=8192.0000 max=8394 max_rank=3 min=8091 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=57142 max_load_reduction_pct=0.871918 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1004, 1007, 1054, 1012, 1026, 1010, 1048, 998, 0, 0, 0, 0, 0, 0, 0, 0, 992, 980, 1021, 1050, 1044, 1058, 996, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1065, 1048, 979, 1009, 1035, 1034, 1010, 21, 0, 0, 0, 0, 0, 0, 0, 1058, 1044, 1010, 1027, 1010, 1027, 1015, 1028, 7, 4, 0, 0, 0, 0, 0, 0, 1040, 993, 983, 1008, 1037, 1000, 1054, 1029, 4, 0, 0, 0, 0, 0, 0, 0, 1015, 1028, 993, 1075, 1036, 997, 1109, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1090, 992, 1037, 1018, 1002, 1050, 1015, 9, 4, 0, 0, 0, 0, 0, 0, 977, 1024, 1012, 1028, 982, 1053, 1001, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8190, 8216, 8230, 8148, 8280, 8237, 8076] avg=8192.0000 max=8280 max_rank=5 min=8076 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1044, 1017, 1029, 1026, 1027, 1048, 1000, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1038, 1032, 1049, 1087, 1008, 1102, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1032, 1026, 1029, 991, 1019, 1014, 969, 21, 0, 0, 0, 0, 0, 0, 0, 994, 1031, 1019, 1037, 985, 1040, 956, 977, 7, 4, 0, 0, 0, 0, 0, 0, 1011, 997, 1000, 1039, 1033, 1014, 1014, 1048, 4, 0, 0, 0, 0, 0, 0, 0, 1012, 1026, 1074, 1007, 1011, 1036, 1024, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1002, 1009, 1021, 1016, 1069, 1002, 1000, 8, 4, 0, 0, 0, 0, 0, 0, 1012, 1028, 1021, 1009, 1084, 1080, 1044, 1099, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8341, 8080, 8050, 8160, 8210, 8154, 8377] avg=8192.0000 max=8377 max_rank=7 min=8050 min_rank=3 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32901 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1030, 1009, 1047, 1028, 996, 1008, 1025, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 997, 1041, 1039, 1020, 1039, 984, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 974, 1030, 1079, 1038, 1005, 1080, 1059, 21, 0, 0, 0, 0, 0, 0, 0, 958, 1031, 1068, 989, 997, 1034, 997, 1049, 7, 4, 0, 0, 0, 0, 0, 0, 1018, 1001, 1031, 1004, 1045, 972, 1035, 1049, 4, 0, 0, 0, 0, 0, 0, 0, 1043, 1050, 999, 1031, 1049, 1045, 1004, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1026, 1062, 1034, 1046, 1055, 1019, 1000, 9, 4, 0, 0, 0, 0, 0, 0, 1012, 999, 1006, 993, 991, 1013, 1067, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8169, 8307, 8134, 8159, 8190, 8269, 8118] avg=8192.0000 max=8307 max_rank=2 min=8118 min_rank=7 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[988, 1068, 1042, 1052, 1002, 1063, 991, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1050, 1052, 1005, 1021, 994, 1051, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1001, 1034, 1014, 1038, 1054, 1014, 990, 23, 0, 0, 0, 0, 0, 0, 0, 987, 1034, 975, 1014, 1044, 971, 1055, 1008, 6, 4, 0, 0, 0, 0, 0, 0, 1000, 1001, 1027, 994, 1051, 1013, 1040, 1019, 4, 0, 0, 0, 0, 0, 0, 0, 1016, 1001, 995, 1034, 1040, 1034, 997, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1029, 1013, 1040, 1034, 1032, 976, 1033, 8, 4, 0, 0, 0, 0, 0, 0, 1020, 1093, 1004, 1036, 1063, 1018, 1032, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8204, 8235, 8098, 8149, 8107, 8187, 8303] avg=8192.0000 max=8303 max_rank=7 min=8098 min_rank=3 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[997, 1028, 1044, 1038, 1007, 1060, 1024, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 998, 986, 1027, 1015, 1064, 1055, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1007, 948, 991, 1044, 965, 1023, 1020, 20, 0, 0, 0, 0, 0, 0, 0, 1043, 989, 1057, 963, 1036, 974, 1050, 988, 7, 4, 0, 0, 0, 0, 0, 0, 1006, 1021, 1035, 1030, 1035, 1061, 999, 1023, 4, 0, 0, 0, 0, 0, 0, 0, 1010, 1029, 1058, 1037, 1043, 1059, 994, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1006, 1062, 1013, 1075, 1064, 992, 1031, 9, 4, 0, 0, 0, 0, 0, 0, 1014, 953, 1052, 1042, 1032, 1043, 1007, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8180, 8050, 8111, 8214, 8253, 8301, 8192] avg=8192.0000 max=8301 max_rank=6 min=8050 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32960 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1023, 1003, 1036, 1019, 1017, 1028, 1016, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1018, 1005, 1012, 1035, 974, 1044, 998, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1091, 1025, 1044, 1031, 972, 1031, 1036, 21, 0, 0, 0, 0, 0, 0, 0, 1003, 1008, 1045, 1044, 1046, 1014, 1059, 1060, 7, 4, 0, 0, 0, 0, 0, 0, 1048, 1015, 1050, 1083, 1010, 1015, 1049, 1006, 4, 0, 0, 0, 0, 0, 0, 0, 1033, 1034, 1029, 978, 1097, 1016, 1011, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1044, 1009, 978, 1031, 993, 973, 1032, 9, 4, 0, 0, 0, 0, 0, 0, 1044, 1005, 1032, 1051, 1009, 979, 1007, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8125, 8243, 8290, 8280, 8219, 8069, 8170] avg=8192.0000 max=8290 max_rank=3 min=8069 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1013, 996, 1019, 1020, 1045, 1034, 1009, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1033, 985, 1018, 1019, 1046, 1052, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1037, 1026, 1031, 1047, 1023, 1042, 1006, 25, 0, 0, 0, 0, 0, 0, 0, 1039, 1017, 1033, 996, 969, 1031, 1049, 1080, 9, 6, 0, 0, 0, 0, 0, 0, 1079, 1006, 1046, 1021, 1013, 1065, 1009, 1057, 5, 1, 0, 0, 0, 0, 0, 0, 983, 983, 1003, 1011, 1005, 999, 1031, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1060, 1021, 981, 1068, 1035, 1040, 1015, 11, 9, 0, 0, 0, 0, 0, 0, 1007, 1029, 1043, 1015, 992, 978, 1043, 974, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8182, 8266, 8229, 8302, 8063, 8228, 8081] avg=8192.0000 max=8302 max_rank=4 min=8063 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8192, 8192, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8448, 8320, 8064, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1076, 1031, 999, 1057, 1026, 991, 1047, 1032, 1018, 1021, 1021, 1005, 1033, 1025, 1016, 1036, 998, 1055, 1019, 1013, 996, 1014, 1054, 1026, 1032, 1069, 1026, 1056, 1040, 1038, 974, 990, 999, 1081, 969, 1023, 1039, 1009, 1070, 1002, 1023, 1072, 1013, 1020, 1019, 1033, 1051, 992, 1012, 980, 1080, 1009, 993, 1073, 986, 1054, 1036, 1017, 1010, 978, 1065, 975, 1000, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1058, 1028, 1010, 1006, 1030, 998, 1043, 1078, 1011, 1071, 1050, 1023, 1074, 1019, 1055, 968, 1077, 1039, 999, 978, 1011, 946, 1060, 1007, 953, 1050, 974, 1044, 995, 1042, 1039, 1062, 1059, 1009, 979, 1053, 971, 1053, 1005, 1025, 1020, 1041, 1015, 1021, 1003, 1018, 999, 994, 1037, 1037, 1029, 1044, 987, 1089, 1006, 1071, 1007, 1003, 1035, 1036, 1010, 1042, 1016, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1018, 1039, 1048, 1000, 1050, 996, 994, 1005, 1074, 1031, 985, 1033, 1062, 1056, 1029, 1061, 1005, 986, 982, 1011, 1061, 1001, 998, 1007, 1063, 1036, 1055, 1004, 1025, 993, 1025, 1051, 1015, 1027, 1032, 992, 1044, 1010, 955, 1042, 1042, 1003, 1042, 1036, 1038, 1021, 1048, 1023, 994, 1026, 1078, 990, 1041, 964, 1012, 1058, 973, 1032, 1045, 1037, 1010, 1067, 1004, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1020, 1030, 1021, 1054, 1015, 962, 1023, 1019, 1074, 1046, 990, 1057, 982, 1038, 1052, 1002, 1001, 1048, 984, 1020, 1014, 1025, 1019, 1054, 1039, 1006, 1038, 1062, 1010, 1019, 1063, 1070, 1005, 1001, 982, 1040, 1058, 1020, 1017, 1032, 1068, 976, 995, 1013, 1016, 970, 1026, 1046, 999, 1060, 997, 1049, 1058, 981, 1067, 1020, 996, 1067, 1068, 992, 1009, 1019, 1040, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1053, 1066, 1044, 1024, 1030, 1018, 1034, 1002, 1026, 1001, 976, 1036, 991, 1002, 1059, 1053, 1048, 1047, 999, 1036, 999, 1024, 985, 989, 1052, 1045, 1046, 996, 1003, 1038, 1042, 1026, 1070, 1051, 1001, 965, 1041, 1010, 1003, 992, 1040, 991, 1004, 1030, 1025, 1051, 1033, 1004, 1085, 1049, 1010, 1023, 998, 1057, 1010, 1044, 1012, 1050, 1004, 1009, 1019, 1030, 1011, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1051, 989, 983, 1003, 1020, 1041, 1009, 1044, 1072, 1042, 1054, 1036, 1007, 995, 1011, 1016, 994, 1002, 1021, 1027, 1048, 988, 1013, 1053, 1004, 1036, 1002, 1026, 1013, 1049, 1075, 1014, 1015, 1037, 1025, 1067, 1000, 1048, 1074, 1022, 1049, 1045, 1042, 1033, 1021, 1047, 1024, 1004, 969, 955, 994, 1081, 1051, 999, 1022, 1034, 1018, 977, 970, 1051, 1062, 1033, 1024, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[995, 1023, 1003, 1012, 1044, 1024, 1048, 1016, 1016, 1009, 1030, 1030, 989, 990, 1020, 1020, 1025, 1022, 1087, 1045, 988, 1046, 1054, 1034, 1024, 1053, 980, 1078, 1007, 1020, 1028, 966, 1053, 1030, 1030, 1000, 1017, 1028, 1079, 1029, 1023, 1042, 1033, 1055, 1021, 1046, 1044, 1003, 1041, 1010, 1056, 989, 1024, 997, 1005, 1023, 1001, 1019, 1047, 1001, 1021, 985, 1027, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1029, 1003, 961, 1024, 1028, 1019, 1028, 1067, 1025, 1019, 1037, 1030, 1068, 1025, 982, 1052, 1051, 1031, 1060, 1029, 995, 1003, 1027, 1000, 1015, 1055, 1030, 1023, 1005, 1019, 1012, 1081, 1016, 1072, 1019, 1020, 991, 984, 1049, 1026, 1012, 1038, 1001, 1078, 1017, 1098, 1017, 1034, 1023, 974, 1012, 978, 1021, 1018, 971, 1024, 1050, 1043, 1009, 1017, 1007, 1070, 985, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1058, 1028, 1010, 1006, 1030, 998, 1043, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1071, 1050, 1023, 1074, 1019, 1055, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 1039, 999, 978, 1011, 946, 1060, 1007, 32, 0, 0, 0, 0, 0, 0, 0, 953, 1017, 974, 1044, 995, 1042, 1039, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1009, 979, 1053, 971, 1053, 1005, 1025, 6, 0, 0, 0, 0, 0, 0, 0, 1020, 1041, 1015, 1002, 1003, 1018, 999, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1037, 1029, 1044, 987, 1089, 1006, 1071, 13, 0, 0, 0, 0, 0, 0, 0, 1007, 1003, 1035, 1036, 1010, 1042, 1016, 1024, 24, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8247, 8149, 8126, 8160, 8092, 8313, 8198] avg=8192.0000 max=8313 max_rank=6 min=8092 min_rank=5 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1076, 1031, 999, 1057, 1026, 991, 1047, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1021, 1021, 1005, 1033, 1025, 1016, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1055, 1019, 1013, 996, 1014, 1054, 1026, 33, 0, 0, 0, 0, 0, 0, 0, 1032, 1035, 1026, 1056, 1040, 1038, 974, 990, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1081, 969, 1023, 1039, 1009, 1070, 1002, 6, 0, 0, 0, 0, 0, 0, 0, 1023, 1072, 1013, 1001, 1019, 1033, 1051, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 980, 1080, 1009, 993, 1073, 986, 1054, 13, 0, 0, 0, 0, 0, 0, 0, 1036, 1017, 1010, 978, 1065, 975, 1000, 1019, 24, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8151, 8208, 8191, 8198, 8204, 8200, 8125] avg=8192.0000 max=8259 max_rank=0 min=8125 min_rank=7 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1018, 1039, 1048, 1000, 1050, 996, 994, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1031, 985, 1033, 1062, 1056, 1029, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 986, 982, 1011, 1061, 1001, 998, 1007, 32, 0, 0, 0, 0, 0, 0, 0, 1063, 1003, 1055, 1004, 1025, 993, 1025, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1027, 1032, 992, 1044, 1010, 955, 1042, 6, 0, 0, 0, 0, 0, 0, 0, 1042, 1003, 1042, 1017, 1038, 1021, 1048, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1026, 1078, 990, 1041, 964, 1012, 1058, 13, 0, 0, 0, 0, 0, 0, 0, 973, 1032, 1045, 1037, 1010, 1067, 1004, 1051, 25, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8306, 8083, 8219, 8123, 8234, 8176, 8245] avg=8192.0000 max=8306 max_rank=1 min=8083 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1053, 1066, 1044, 1024, 1030, 1018, 1034, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1001, 976, 1036, 991, 1002, 1059, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1047, 999, 1036, 999, 1024, 985, 989, 32, 0, 0, 0, 0, 0, 0, 0, 1052, 1012, 1046, 996, 1003, 1038, 1042, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1051, 1001, 965, 1041, 1010, 1003, 992, 6, 0, 0, 0, 0, 0, 0, 0, 1040, 991, 1004, 1011, 1025, 1051, 1033, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 1049, 1010, 1023, 998, 1057, 1010, 1044, 13, 0, 0, 0, 0, 0, 0, 0, 1012, 1050, 1004, 1009, 1019, 1030, 1011, 1024, 24, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8120, 8159, 8215, 8139, 8159, 8289, 8184] avg=8192.0000 max=8289 max_rank=6 min=8120 min_rank=1 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1020, 1030, 1021, 1054, 1015, 962, 1023, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1046, 990, 1057, 982, 1038, 1052, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1048, 984, 1020, 1014, 1025, 1019, 1054, 31, 0, 0, 0, 0, 0, 0, 0, 1039, 974, 1038, 1062, 1010, 1019, 1063, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1001, 982, 1040, 1058, 1020, 1017, 1032, 6, 0, 0, 0, 0, 0, 0, 0, 1068, 976, 995, 994, 1016, 970, 1026, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1060, 997, 1049, 1058, 981, 1067, 1020, 13, 0, 0, 0, 0, 0, 0, 0, 996, 1067, 1068, 992, 1009, 1019, 1040, 992, 25, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8216, 8196, 8275, 8161, 8091, 8244, 8209] avg=8192.0000 max=8275 max_rank=3 min=8091 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1051, 989, 983, 1003, 1020, 1041, 1009, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1042, 1054, 1036, 1007, 995, 1011, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1002, 1021, 1027, 1048, 988, 1013, 1053, 32, 0, 0, 0, 0, 0, 0, 0, 1004, 1003, 1002, 1026, 1013, 1049, 1075, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1037, 1025, 1067, 1000, 1048, 1074, 1022, 6, 0, 0, 0, 0, 0, 0, 0, 1049, 1045, 1042, 1014, 1021, 1047, 1024, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 969, 955, 994, 1081, 1051, 999, 1022, 1034, 13, 0, 0, 0, 0, 0, 0, 0, 1018, 977, 970, 1051, 1062, 1033, 1024, 1005, 25, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8208, 8178, 8186, 8294, 8246, 8118, 8166] avg=8192.0000 max=8294 max_rank=4 min=8118 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[995, 1023, 1003, 1012, 1044, 1024, 1048, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1009, 1030, 1030, 989, 990, 1020, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1022, 1087, 1045, 988, 1046, 1054, 1034, 32, 0, 0, 0, 0, 0, 0, 0, 1024, 1020, 980, 1078, 1007, 1020, 1028, 966, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1030, 1030, 1000, 1017, 1028, 1079, 1029, 6, 0, 0, 0, 0, 0, 0, 0, 1023, 1042, 1033, 1036, 1021, 1046, 1044, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1010, 1056, 989, 1024, 997, 1005, 1023, 13, 0, 0, 0, 0, 0, 0, 0, 1001, 1019, 1047, 1001, 1021, 985, 1027, 1031, 24, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8080, 8333, 8123, 8272, 8248, 8158, 8157] avg=8192.0000 max=8333 max_rank=2 min=8080 min_rank=1 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1026, 1003, 961, 1024, 1028, 1019, 1028, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1019, 1037, 1030, 1068, 1025, 982, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1031, 1060, 1029, 995, 1003, 1027, 1000, 34, 0, 0, 0, 0, 0, 0, 0, 1015, 1016, 1030, 1023, 1005, 1019, 1012, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1072, 1019, 1020, 991, 984, 1049, 1026, 9, 3, 0, 0, 0, 0, 0, 0, 1012, 1038, 1001, 1045, 1017, 1098, 1017, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 974, 1012, 978, 1021, 1018, 971, 1024, 17, 0, 0, 0, 0, 0, 0, 0, 1050, 1043, 1009, 1017, 1007, 1070, 985, 1029, 30, 5, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8208, 8230, 8201, 8189, 8262, 8038, 8252] avg=8192.0000 max=8262 max_rank=5 min=8038 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8064, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8192, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8448, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1007, 1073, 1037, 1055, 1049, 967, 996, 1021, 1020, 1046, 1020, 992, 1019, 1012, 1060, 1013, 1023, 1020, 1020, 1022, 1047, 1027, 1062, 1047, 1052, 989, 1004, 1068, 1025, 1027, 1043, 1016, 1076, 985, 985, 1011, 1060, 1034, 1013, 1038, 990, 1083, 1045, 1032, 1002, 1016, 1010, 1006, 1034, 1032, 1032, 1034, 1023, 1022, 971, 985, 1003, 985, 1018, 1015, 990, 1019, 1030, 1078] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1075, 1035, 1053, 1042, 1029, 989, 1051, 1015, 991, 1006, 1016, 1005, 1067, 1047, 1044, 1000, 1045, 1004, 1020, 1070, 1008, 1042, 1033, 1042, 1052, 1050, 1026, 1048, 1039, 1076, 994, 1044, 987, 981, 986, 985, 1051, 1047, 1013, 1003, 1017, 1025, 1016, 1023, 1020, 1026, 1063, 1015, 1021, 1037, 1010, 1013, 1027, 1037, 1027, 986, 995, 1009, 1020, 1052, 1010, 1020, 987, 969] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1056, 973, 1019, 1045, 1083, 1021, 1031, 1014, 1019, 1036, 990, 1015, 1029, 1025, 1052, 1023, 1070, 1015, 1044, 979, 1017, 1035, 1038, 1053, 975, 1054, 1007, 1010, 1005, 1037, 1017, 1005, 979, 1048, 1052, 953, 1015, 1055, 1057, 1015, 1029, 998, 1032, 986, 1012, 1045, 1053, 1055, 1018, 1003, 1035, 1027, 1017, 1039, 1035, 1016, 1049, 1038, 1038, 979, 1010, 1018, 983, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[975, 1046, 1020, 1031, 1023, 1019, 1032, 992, 1050, 971, 997, 1052, 1027, 1031, 1025, 1071, 1062, 1036, 993, 987, 1043, 1007, 993, 1005, 1052, 1059, 1004, 995, 1026, 989, 1025, 1024, 1001, 1037, 1040, 1034, 982, 1052, 996, 1007, 994, 1066, 1032, 1069, 995, 1039, 1084, 1080, 1059, 1006, 1037, 1037, 1033, 1008, 983, 985, 1064, 1021, 1044, 1025, 995, 1052, 1002, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1039, 1007, 1028, 1046, 1079, 1029, 1069, 1064, 1011, 1019, 999, 1020, 1025, 1048, 1031, 1001, 1023, 991, 1004, 1007, 1077, 979, 1011, 1039, 1048, 1015, 999, 1040, 1036, 1057, 1029, 1040, 1052, 1014, 1034, 1029, 1013, 1011, 985, 1057, 989, 1000, 986, 1030, 985, 1022, 975, 1064, 996, 1016, 1017, 1038, 957, 1021, 1084, 964, 1017, 1052, 1048, 1019, 1000, 1031, 1025, 1095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1039, 1066, 1073, 1024, 1024, 943, 990, 1008, 1021, 983, 1025, 1054, 1038, 1043, 1036, 990, 1058, 1048, 1028, 1054, 1049, 1010, 1041, 1021, 1023, 1017, 1018, 999, 1011, 1014, 1044, 1013, 1017, 1038, 1014, 977, 992, 1050, 994, 1070, 1048, 1017, 1051, 1002, 991, 1015, 1071, 1054, 1013, 1032, 1026, 1016, 1002, 1061, 1009, 995, 1037, 997, 1026, 1012, 1001, 1056, 1056, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1005, 1039, 986, 1032, 1019, 1035, 1031, 1008, 1012, 1069, 999, 1034, 991, 1049, 1023, 1013, 1003, 1032, 1037, 1077, 1009, 1026, 1032, 1010, 1087, 1022, 1031, 1003, 1024, 1036, 1020, 1028, 1045, 1024, 1039, 1008, 1008, 1007, 1019, 1021, 990, 1007, 1041, 1041, 1019, 1006, 1035, 1049, 1041, 1060, 1004, 989, 1044, 1030, 988, 1051, 968, 1004, 1019, 1053, 1020, 1037, 1021, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1059, 1052, 1035, 1027, 1075, 1021, 1006, 1000, 1017, 1044, 989, 1038, 1058, 1004, 1007, 987, 1021, 1000, 1068, 1040, 985, 983, 1012, 1006, 1042, 976, 1007, 1045, 1056, 994, 1013, 1034, 969, 1047, 989, 1010, 1038, 1039, 1042, 1077, 973, 1045, 1060, 1053, 1051, 1010, 1055, 983, 1003, 1012, 980, 1051, 1043, 999, 1051, 1086, 1005, 1033, 1022, 1026, 993, 1036, 1047, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1007, 1073, 1037, 1055, 1013, 967, 996, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1046, 1020, 992, 1019, 1012, 1060, 1013, 10, 0, 0, 0, 0, 0, 0, 0, 1005, 1020, 1020, 1022, 1047, 1027, 1062, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 989, 1004, 1068, 1025, 1027, 1043, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 985, 985, 1011, 1060, 1034, 1013, 1038, 28, 0, 0, 0, 0, 0, 0, 0, 990, 1083, 1045, 1032, 1002, 1016, 994, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1032, 1032, 1034, 1023, 1022, 971, 985, 18, 8, 0, 0, 0, 0, 0, 0, 1003, 985, 1018, 1015, 990, 1019, 1030, 1078, 16, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8192, 8250, 8210, 8230, 8168, 8159, 8158] avg=8192.0000 max=8250 max_rank=2 min=8158 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[975, 1046, 1020, 1031, 988, 1019, 1032, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 971, 997, 1052, 1027, 1031, 1025, 1071, 10, 0, 0, 0, 0, 0, 0, 0, 1043, 1036, 993, 987, 1043, 1007, 993, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1059, 1004, 995, 1026, 989, 1025, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1037, 1040, 1034, 982, 1052, 996, 1007, 27, 0, 0, 0, 0, 0, 0, 0, 994, 1066, 1032, 1069, 995, 1039, 1067, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1006, 1037, 1037, 1033, 1008, 983, 985, 19, 8, 0, 0, 0, 0, 0, 0, 1064, 1021, 1044, 1025, 995, 1052, 1002, 1015, 17, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8234, 8107, 8160, 8176, 8342, 8175, 8239] avg=8192.0000 max=8342 max_rank=5 min=8103 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32932 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1056, 973, 1019, 1045, 1046, 1021, 1031, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1036, 990, 1015, 1029, 1025, 1052, 1023, 10, 0, 0, 0, 0, 0, 0, 0, 1051, 1015, 1044, 979, 1017, 1035, 1038, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 961, 1054, 1007, 1010, 1005, 1037, 1017, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1048, 1052, 953, 1015, 1055, 1057, 1015, 29, 0, 0, 0, 0, 0, 0, 0, 1029, 998, 1032, 986, 1012, 1045, 1037, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1003, 1035, 1027, 1017, 1039, 1035, 1016, 19, 8, 0, 0, 0, 0, 0, 0, 1049, 1038, 1038, 979, 1010, 1018, 983, 1055, 16, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8199, 8232, 8096, 8203, 8194, 8217, 8190] avg=8192.0000 max=8232 max_rank=2 min=8096 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1039, 1007, 1028, 1046, 1042, 1029, 1069, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1019, 999, 1020, 1025, 1048, 1031, 1001, 10, 0, 0, 0, 0, 0, 0, 0, 1005, 991, 1004, 1007, 1077, 979, 1011, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1015, 999, 1040, 1036, 1057, 1029, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1014, 1034, 1029, 1013, 1011, 985, 1057, 29, 0, 0, 0, 0, 0, 0, 0, 989, 1000, 986, 1030, 985, 1022, 960, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1016, 1017, 1038, 957, 1021, 1084, 964, 18, 8, 0, 0, 0, 0, 0, 0, 1017, 1052, 1048, 1019, 1000, 1031, 1025, 1095, 15, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 8164, 8113, 8250, 8224, 8036, 8119, 8306] avg=8192.0000 max=8324 max_rank=0 min=8036 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1075, 1035, 1053, 1042, 993, 989, 1051, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1006, 1016, 1005, 1067, 1047, 1044, 1000, 10, 0, 0, 0, 0, 0, 0, 0, 1026, 1004, 1020, 1070, 1008, 1042, 1033, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1050, 1026, 1048, 1039, 1076, 994, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 987, 981, 986, 985, 1051, 1047, 1013, 1003, 28, 0, 0, 0, 0, 0, 0, 0, 1017, 1025, 1016, 1023, 1020, 1026, 1047, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1037, 1010, 1013, 1027, 1037, 1027, 986, 19, 8, 0, 0, 0, 0, 0, 0, 995, 1009, 1020, 1052, 1010, 1020, 987, 969, 16, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8186, 8245, 8315, 8081, 8189, 8185, 8082] avg=8192.0000 max=8315 max_rank=3 min=8081 min_rank=4 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32537 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1005, 1039, 986, 1032, 984, 1035, 1031, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1069, 999, 1034, 991, 1049, 1023, 1013, 11, 0, 0, 0, 0, 0, 0, 0, 985, 1032, 1037, 1077, 1009, 1026, 1032, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1022, 1031, 1003, 1024, 1036, 1020, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1024, 1039, 1008, 1008, 1007, 1019, 1021, 27, 0, 0, 0, 0, 0, 0, 0, 990, 1007, 1041, 1041, 1019, 1006, 1019, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1060, 1004, 989, 1044, 1030, 988, 1051, 18, 8, 0, 0, 0, 0, 0, 0, 968, 1004, 1019, 1053, 1020, 1037, 1021, 1026, 16, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8201, 8208, 8236, 8198, 8172, 8233, 8168] avg=8192.0000 max=8236 max_rank=3 min=8120 min_rank=0 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1039, 1066, 1073, 1024, 981, 943, 990, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 983, 1025, 1054, 1038, 1043, 1036, 990, 16, 0, 0, 0, 0, 0, 0, 0, 1033, 1048, 1028, 1054, 1049, 1010, 1041, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1017, 1018, 999, 1011, 1014, 1044, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1038, 1014, 977, 992, 1050, 994, 1070, 32, 0, 0, 0, 0, 0, 0, 0, 1048, 1017, 1051, 1002, 991, 1015, 1043, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1032, 1026, 1016, 1002, 1061, 1009, 995, 25, 11, 7, 0, 0, 0, 0, 0, 1037, 997, 1026, 1012, 1001, 1056, 1056, 991, 21, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8206, 8284, 8116, 8184, 8221, 8197, 8204] avg=8192.0000 max=8284 max_rank=2 min=8116 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1059, 1052, 1035, 1027, 1038, 1021, 1006, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1044, 989, 1038, 1058, 1004, 1007, 987, 10, 0, 0, 0, 0, 0, 0, 0, 1003, 1000, 1068, 1040, 985, 983, 1012, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 976, 1007, 1045, 1056, 994, 1013, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1047, 989, 1010, 1038, 1039, 1042, 1077, 29, 0, 0, 0, 0, 0, 0, 0, 973, 1045, 1060, 1053, 1051, 1010, 1039, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1012, 980, 1051, 1043, 999, 1051, 1086, 18, 8, 0, 0, 0, 0, 0, 0, 1005, 1033, 1022, 1026, 993, 1036, 1047, 1007, 16, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8154, 8097, 8153, 8240, 8214, 8251, 8189] avg=8192.0000 max=8251 max_rank=6 min=8097 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32894 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8064, 8320, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8064, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8064, 8192, 8320, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8064, 8320, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1036, 1016, 1019, 1020, 1021, 1023, 1018, 1016, 1040, 1040, 1008, 1027, 1008, 1035, 1065, 1054, 1004, 987, 1051, 1010, 992, 1002, 1039, 1024, 1030, 1018, 1066, 1018, 1031, 1051, 1005, 998, 1025, 1029, 1054, 1013, 998, 1020, 1037, 995, 1011, 1006, 1028, 989, 1033, 1009, 1046, 1072, 1022, 1037, 1038, 1019, 1073, 1070, 1017, 1059, 1017, 981, 984, 1027, 980, 1061, 1010, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[963, 1050, 1011, 1032, 1016, 1065, 1024, 1050, 1072, 1050, 1063, 1017, 965, 947, 1006, 1005, 1039, 1071, 1021, 1019, 971, 1020, 1033, 1041, 1052, 1008, 1043, 1034, 1037, 1020, 988, 1032, 986, 1036, 1065, 1048, 1004, 1032, 1040, 1026, 1025, 1004, 1025, 1029, 1009, 1058, 1031, 935, 1070, 1007, 1031, 1045, 1048, 1047, 1046, 1028, 1011, 1001, 1034, 1048, 971, 1031, 1000, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1025, 1028, 1016, 1010, 1015, 1044, 1032, 1033, 959, 935, 1049, 965, 1020, 1023, 1015, 1058, 1030, 1012, 1063, 1011, 1003, 1052, 1069, 1024, 1020, 992, 1060, 1045, 1040, 1057, 1027, 991, 1020, 1056, 989, 1019, 1000, 1063, 1020, 1044, 1046, 1045, 1044, 1000, 1019, 1057, 1032, 1011, 1025, 1067, 1038, 1041, 992, 973, 977, 1060, 1070, 1002, 983, 1034, 1059, 983, 1011, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1004, 1040, 1063, 1053, 1012, 1014, 1002, 1063, 1048, 1051, 1029, 1015, 1018, 1013, 1020, 1054, 1007, 997, 1062, 1008, 1042, 1034, 1029, 1009, 1047, 1010, 1017, 1029, 1016, 1027, 1006, 1021, 1016, 1030, 1040, 995, 1048, 1014, 1045, 1006, 967, 1041, 1070, 1043, 968, 1025, 1054, 986, 1022, 970, 1023, 1050, 1003, 995, 1033, 988, 1034, 1005, 1037, 1051, 1025, 1026, 999, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[997, 1031, 1027, 993, 982, 1005, 1040, 1027, 1049, 1007, 1034, 996, 1009, 1057, 1034, 1046, 1008, 1018, 1022, 1074, 1005, 1008, 1019, 1018, 1030, 1045, 1069, 1022, 1038, 1024, 1001, 1025, 1066, 1026, 972, 1030, 1009, 1006, 1063, 1017, 1087, 993, 983, 1009, 1000, 1025, 1022, 1068, 1055, 1034, 1036, 1067, 1061, 996, 1022, 1036, 1001, 1031, 987, 1016, 1013, 1006, 1022, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1076, 1044, 1005, 978, 1049, 1079, 1009, 1008, 1024, 1018, 1002, 1005, 971, 1011, 1057, 1056, 1037, 1010, 1027, 1026, 1009, 1004, 1005, 964, 993, 993, 1097, 1066, 1001, 1047, 1035, 1045, 1040, 1041, 1023, 1012, 1003, 981, 1027, 1037, 1082, 999, 1045, 999, 1021, 1015, 1044, 1059, 1035, 1065, 1061, 1059, 991, 980, 1041, 979, 991, 1071, 1017, 1012, 1048, 1014, 997, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1039, 1032, 1031, 1004, 1022, 1020, 1019, 959, 1026, 1039, 1056, 1006, 1042, 1012, 1035, 1008, 1049, 1030, 988, 1042, 998, 1020, 1011, 995, 1047, 1031, 1028, 1056, 1001, 980, 1015, 1023, 966, 1034, 1039, 1029, 1054, 1018, 1073, 1063, 1058, 981, 1004, 1064, 1035, 1068, 970, 1040, 980, 1007, 1009, 1029, 1032, 1029, 1045, 1080, 985, 1027, 1006, 1033, 1044, 1008, 1020, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1071, 1047, 997, 1010, 1086, 989, 966, 976, 972, 1082, 1009, 1007, 1000, 980, 1051, 1058, 1056, 1004, 1027, 1042, 1033, 1030, 999, 1003, 1025, 1039, 1062, 1043, 1036, 1025, 969, 1011, 992, 998, 1048, 1017, 1016, 1087, 1031, 1027, 1012, 1027, 1017, 1025, 1003, 1024, 1046, 1065, 1044, 1025, 1068, 1032, 1029, 996, 1027, 1018, 997, 1010, 1049, 965, 1024, 1038, 1042, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[997, 1031, 1027, 993, 982, 1005, 1040, 1027, 9, 1, 0, 0, 0, 0, 0, 0, 1049, 1007, 1034, 996, 1009, 1057, 1034, 1046, 13, 0, 0, 0, 0, 0, 0, 0, 1008, 1018, 1022, 1074, 1005, 1008, 1019, 1018, 21, 0, 0, 0, 0, 0, 0, 0, 1030, 1045, 1042, 1022, 1038, 1024, 1001, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1026, 972, 1030, 1009, 1006, 1049, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 993, 983, 1009, 1000, 1025, 1022, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1034, 1036, 1023, 1061, 996, 1022, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1031, 987, 1016, 1013, 1006, 1022, 1017, 44, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8245, 8193, 8227, 8175, 8178, 8263, 8143] avg=8192.0000 max=8263 max_rank=6 min=8112 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1004, 1040, 1063, 1053, 1012, 1014, 1002, 1063, 8, 1, 0, 0, 0, 0, 0, 0, 1048, 1051, 1029, 1015, 1018, 1013, 1020, 1054, 12, 0, 0, 0, 0, 0, 0, 0, 1007, 997, 1062, 1008, 1042, 1034, 1029, 1009, 20, 0, 0, 0, 0, 0, 0, 0, 1047, 1010, 991, 1029, 1016, 1027, 1006, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1030, 1040, 995, 1048, 1014, 1032, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 959, 1041, 1070, 1043, 968, 1025, 1054, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 970, 1023, 1007, 1003, 995, 1033, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1005, 1037, 1051, 1025, 1026, 999, 1067, 43, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8260, 8208, 8147, 8181, 8146, 8041, 8293] avg=8192.0000 max=8293 max_rank=7 min=8041 min_rank=6 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1025, 1028, 1016, 1010, 1015, 1044, 1032, 1033, 9, 1, 0, 0, 0, 0, 0, 0, 959, 935, 1049, 965, 1020, 1023, 1015, 1058, 12, 0, 0, 0, 0, 0, 0, 0, 1030, 1012, 1063, 1011, 1003, 1052, 1069, 1024, 21, 0, 0, 0, 0, 0, 0, 0, 1020, 992, 1033, 1045, 1040, 1057, 1027, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1056, 989, 1019, 1000, 1063, 1007, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1045, 1044, 1000, 1019, 1057, 1032, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1067, 1038, 998, 992, 973, 977, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1002, 983, 1034, 1059, 983, 1011, 1033, 43, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8036, 8285, 8205, 8198, 8245, 8130, 8224] avg=8192.0000 max=8285 max_rank=2 min=8036 min_rank=1 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1036, 1016, 1019, 1020, 1021, 1023, 1018, 1016, 8, 1, 0, 0, 0, 0, 0, 0, 1040, 1040, 1008, 1027, 1008, 1035, 1065, 1054, 12, 0, 0, 0, 0, 0, 0, 0, 1004, 987, 1051, 1010, 992, 1002, 1039, 1024, 21, 0, 0, 0, 0, 0, 0, 0, 1030, 1018, 1039, 1018, 1031, 1051, 1005, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1029, 1054, 1013, 998, 1020, 1024, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1006, 1028, 989, 1033, 1009, 1046, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1037, 1038, 977, 1073, 1070, 1017, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 981, 984, 1027, 980, 1061, 1010, 1004, 42, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8289, 8130, 8190, 8158, 8186, 8293, 8112] avg=8192.0000 max=8293 max_rank=6 min=8112 min_rank=7 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1076, 1044, 1005, 978, 1049, 1079, 1009, 1008, 9, 1, 0, 0, 0, 0, 0, 0, 1024, 1018, 1002, 1005, 971, 1011, 1057, 1056, 12, 0, 0, 0, 0, 0, 0, 0, 1037, 1010, 1027, 1026, 1009, 1004, 1005, 964, 22, 0, 0, 0, 0, 0, 0, 0, 993, 993, 1069, 1066, 1001, 1047, 1035, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1041, 1023, 1012, 1003, 981, 1014, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 999, 1045, 999, 1021, 1015, 1044, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1065, 1061, 1016, 991, 980, 1041, 979, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1071, 1017, 1012, 1048, 1014, 997, 996, 43, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8156, 8104, 8249, 8151, 8255, 8168, 8195] avg=8192.0000 max=8258 max_rank=0 min=8104 min_rank=2 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[963, 1050, 1011, 1032, 1016, 1065, 1024, 1050, 8, 1, 0, 0, 0, 0, 0, 0, 1072, 1050, 1063, 1017, 965, 947, 1006, 1005, 12, 0, 0, 0, 0, 0, 0, 0, 1039, 1071, 1021, 1019, 971, 1020, 1033, 1041, 21, 0, 0, 0, 0, 0, 0, 0, 1052, 1008, 1016, 1034, 1037, 1020, 988, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1036, 1065, 1048, 1004, 1032, 1027, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1004, 1025, 1029, 1009, 1058, 1031, 935, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1007, 1031, 1002, 1048, 1047, 1046, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1001, 1034, 1048, 971, 1031, 1000, 1000, 43, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8137, 8236, 8187, 8224, 8108, 8279, 8145] avg=8192.0000 max=8279 max_rank=6 min=8108 min_rank=5 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1039, 1032, 1031, 1004, 1022, 1020, 1019, 959, 13, 5, 0, 0, 0, 0, 0, 0, 1026, 1039, 1056, 1006, 1042, 1012, 1035, 1008, 18, 0, 0, 0, 0, 0, 0, 0, 1049, 1030, 988, 1042, 998, 1020, 1011, 995, 25, 7, 0, 0, 0, 0, 0, 0, 1047, 1031, 995, 1056, 1001, 980, 1015, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1034, 1039, 1029, 1054, 1018, 1043, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 981, 1004, 1064, 1035, 1068, 970, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1007, 1009, 983, 1032, 1029, 1045, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1027, 1006, 1033, 1044, 1008, 1020, 1042, 46, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8242, 8165, 8148, 8246, 8207, 8165, 8219] avg=8192.0000 max=8246 max_rank=4 min=8144 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1071, 1047, 997, 1010, 1086, 989, 966, 976, 8, 1, 0, 0, 0, 0, 0, 0, 972, 1082, 1009, 1007, 1000, 980, 1051, 1058, 12, 0, 0, 0, 0, 0, 0, 0, 1056, 1004, 1027, 1042, 1033, 1030, 999, 1003, 21, 0, 0, 0, 0, 0, 0, 0, 1025, 1039, 1035, 1043, 1036, 1025, 969, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 992, 998, 1048, 1017, 1016, 1087, 1018, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1027, 1017, 1025, 1003, 1024, 1046, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1025, 1068, 990, 1029, 996, 1027, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1010, 1049, 965, 1024, 1038, 1042, 1032, 42, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8171, 8215, 8183, 8203, 8211, 8197, 8205] avg=8192.0000 max=8215 max_rank=2 min=8151 min_rank=0 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 8064, 8192, 8320, 8448, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8064, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8320, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1028, 1030, 1008, 1096, 1014, 1045, 1076, 1034, 1023, 1089, 1027, 1013, 1051, 1020, 996, 1012, 1022, 1063, 1011, 1006, 1036, 1046, 1040, 1051, 1000, 961, 1022, 1036, 1005, 1055, 1008, 1016, 1001, 1057, 1057, 1025, 984, 1008, 989, 1030, 1007, 1041, 1042, 1028, 1004, 1024, 1010, 1073, 948, 1019, 1052, 1015, 994, 1052, 988, 1007, 1003, 1008, 1038, 963, 1032, 1026, 1013, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1030, 996, 1033, 1053, 1047, 1019, 1033, 1029, 1007, 1021, 1017, 1014, 1001, 1042, 1006, 1040, 1000, 961, 1027, 1046, 1032, 1060, 1023, 1023, 1024, 1053, 1002, 1106, 995, 1065, 992, 1003, 1018, 1012, 1062, 1035, 1017, 1018, 1005, 974, 1016, 1044, 1035, 959, 1055, 1007, 1043, 1068, 1029, 1003, 1060, 1008, 1007, 1074, 1005, 1041, 993, 1035, 1020, 990, 959, 1053, 1033, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1024, 1051, 1018, 996, 1010, 988, 1040, 1021, 1045, 1050, 1051, 1069, 1044, 1026, 1038, 1047, 1023, 1024, 1014, 1016, 1005, 981, 1066, 1027, 1023, 1019, 1067, 1016, 1021, 1020, 1015, 986, 1024, 1034, 1003, 1004, 1062, 977, 1031, 1071, 1067, 1025, 1008, 1033, 1015, 1008, 998, 1049, 1036, 1034, 1027, 1001, 1063, 1074, 1072, 965, 1050, 1012, 974, 1015, 1028, 958, 992, 985] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1002, 1027, 1003, 1044, 995, 1057, 1008, 1004, 1010, 1012, 1016, 1044, 996, 1026, 1043, 1021, 966, 1017, 1020, 1042, 957, 995, 1055, 1012, 1045, 1047, 1037, 1021, 991, 1043, 1039, 1043, 998, 1007, 979, 1044, 1091, 1011, 1013, 1048, 1040, 1008, 999, 1017, 1038, 1069, 1073, 1049, 1037, 960, 1054, 996, 1033, 1018, 1047, 1071, 1030, 980, 1022, 1037, 1054, 1043, 991, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1013, 1006, 1054, 1044, 1001, 1020, 1037, 1012, 1020, 1053, 1065, 1003, 1003, 1000, 1009, 1010, 1000, 1066, 1085, 1034, 1011, 1052, 978, 1039, 1050, 991, 1037, 1047, 995, 1030, 1052, 1070, 996, 1024, 1029, 1021, 1016, 1012, 1036, 1018, 1073, 1006, 1056, 1040, 1043, 996, 1020, 988, 1020, 979, 1055, 1044, 999, 1018, 989, 1046, 1021, 1024, 1036, 1048, 1028, 1008, 976, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[998, 1059, 1034, 1000, 1024, 991, 1007, 1081, 1030, 998, 1020, 1027, 990, 1039, 997, 1021, 1042, 1033, 1032, 1010, 1028, 1077, 978, 1019, 1058, 1060, 1041, 1047, 1023, 1022, 1023, 1000, 1012, 1026, 1051, 1039, 1036, 1060, 1020, 1041, 1020, 1021, 995, 987, 1052, 1025, 1000, 1034, 1039, 1015, 995, 1037, 1039, 1009, 986, 1027, 1026, 1024, 970, 1029, 1032, 1044, 1047, 989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1059, 972, 933, 1035, 1026, 1091, 1027, 1047, 1026, 1056, 1020, 1051, 1021, 1018, 1015, 1022, 1018, 1036, 1016, 1077, 982, 1046, 1062, 966, 1043, 1014, 1027, 1075, 1006, 980, 1044, 1018, 1011, 1040, 1021, 994, 1013, 1003, 1094, 1008, 1012, 1015, 1030, 1054, 1012, 1003, 1006, 1039, 1001, 1034, 1012, 1008, 1062, 1045, 989, 1037, 1026, 1013, 1041, 1032, 1015, 996, 1009, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1007, 1014, 1058, 1039, 1042, 1068, 1073, 1040, 1031, 999, 1023, 1033, 1023, 1007, 1001, 1047, 1010, 1028, 995, 1004, 1049, 1070, 995, 1048, 1029, 1031, 980, 999, 1007, 1042, 1020, 997, 1034, 977, 1002, 1051, 1013, 999, 1051, 995, 1006, 983, 994, 993, 1031, 986, 1026, 1003, 1052, 1011, 998, 1011, 1009, 1067, 1028, 1014, 1089, 1064, 1043, 1045, 1007, 1029, 1084, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1028, 1030, 1008, 1066, 1014, 1045, 1076, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1082, 1027, 1013, 1051, 1020, 996, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1063, 1011, 1006, 1036, 1044, 1040, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 961, 1022, 1025, 1005, 1055, 1008, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1057, 1057, 1025, 984, 1008, 989, 1030, 8, 4, 0, 0, 0, 0, 0, 0, 1007, 1041, 1042, 1028, 1004, 1024, 1010, 1073, 2, 2, 0, 0, 0, 0, 0, 0, 948, 1019, 1052, 1015, 994, 1052, 988, 1007, 5, 3, 0, 0, 0, 0, 0, 0, 1003, 1008, 1038, 963, 1032, 1026, 1013, 1058, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8224, 8273, 8092, 8163, 8233, 8083, 8167] avg=8192.0000 max=8301 max_rank=0 min=8083 min_rank=6 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1013, 1006, 1054, 1015, 1001, 1020, 1037, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1046, 1065, 1003, 1003, 1000, 1009, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1066, 1085, 1034, 1011, 1050, 978, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 991, 1037, 1036, 995, 1030, 1052, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1024, 1029, 1021, 1016, 1012, 1036, 1018, 8, 4, 0, 0, 0, 0, 0, 0, 1073, 1006, 1056, 1040, 1043, 996, 1020, 988, 2, 2, 0, 0, 0, 0, 0, 0, 1020, 979, 1055, 1044, 999, 1018, 989, 1046, 5, 3, 0, 0, 0, 0, 0, 0, 1021, 1024, 1036, 1048, 1028, 1008, 976, 984, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8156, 8263, 8261, 8164, 8226, 8158, 8150] avg=8192.0000 max=8263 max_rank=2 min=8150 min_rank=7 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1030, 996, 1033, 1024, 1047, 1019, 1033, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1014, 1017, 1014, 1001, 1042, 1006, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 961, 1027, 1046, 1032, 1058, 1023, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1053, 1002, 1094, 995, 1065, 992, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1012, 1062, 1035, 1017, 1018, 1005, 974, 9, 4, 0, 0, 0, 0, 0, 0, 1016, 1044, 1035, 959, 1055, 1007, 1043, 1068, 2, 2, 0, 0, 0, 0, 0, 0, 1029, 1003, 1060, 1008, 1007, 1074, 1005, 1041, 5, 3, 0, 0, 0, 0, 0, 0, 993, 1035, 1020, 990, 959, 1053, 1033, 1058, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8141, 8170, 8228, 8154, 8231, 8235, 8166] avg=8192.0000 max=8235 max_rank=6 min=8141 min_rank=1 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1024, 1051, 1018, 969, 1010, 988, 1040, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1043, 1051, 1069, 1044, 1026, 1038, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1024, 1014, 1016, 1005, 979, 1066, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1019, 1067, 1005, 1021, 1020, 1015, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1034, 1003, 1004, 1062, 977, 1031, 1071, 8, 4, 0, 0, 0, 0, 0, 0, 1067, 1025, 1008, 1033, 1015, 1008, 998, 1049, 2, 2, 0, 0, 0, 0, 0, 0, 1036, 1034, 1027, 1001, 1063, 1074, 1072, 965, 5, 3, 0, 0, 0, 0, 0, 0, 1050, 1012, 974, 1015, 1028, 958, 992, 985, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8363, 8154, 8156, 8218, 8207, 8280, 8037] avg=8192.0000 max=8363 max_rank=1 min=8037 min_rank=7 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[998, 1059, 1034, 973, 1024, 991, 1007, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 992, 1020, 1027, 990, 1039, 997, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1033, 1032, 1010, 1028, 1075, 978, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1060, 1041, 1036, 1023, 1022, 1023, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1026, 1051, 1039, 1036, 1060, 1020, 1041, 8, 4, 0, 0, 0, 0, 0, 0, 1020, 1021, 995, 987, 1052, 1025, 1000, 1034, 2, 2, 0, 0, 0, 0, 0, 0, 1039, 1015, 995, 1037, 1039, 1009, 986, 1027, 4, 3, 0, 0, 0, 0, 0, 0, 1026, 1024, 970, 1029, 1032, 1044, 1047, 989, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8116, 8217, 8263, 8297, 8138, 8154, 8184] avg=8192.0000 max=8297 max_rank=4 min=8116 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1002, 1027, 1003, 1015, 995, 1057, 1008, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1005, 1016, 1044, 996, 1026, 1043, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1017, 1020, 1042, 957, 993, 1055, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1047, 1037, 1010, 991, 1043, 1039, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1007, 979, 1044, 1091, 1011, 1013, 1048, 8, 4, 0, 0, 0, 0, 0, 0, 1040, 1008, 999, 1017, 1038, 1069, 1073, 1049, 2, 2, 0, 0, 0, 0, 0, 0, 1037, 960, 1054, 996, 1033, 1018, 1047, 1071, 5, 3, 0, 0, 0, 0, 0, 0, 1030, 980, 1022, 1037, 1054, 1043, 991, 1041, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8161, 8062, 8255, 8203, 8297, 8224, 8223] avg=8192.0000 max=8297 max_rank=5 min=8062 min_rank=2 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32947 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1059, 972, 933, 999, 1026, 1091, 1027, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1044, 1020, 1051, 1021, 1018, 1015, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1036, 1016, 1077, 982, 1043, 1062, 966, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1014, 1027, 1055, 1006, 980, 1044, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1040, 1021, 994, 1013, 1003, 1094, 1008, 11, 8, 0, 0, 0, 0, 0, 0, 1012, 1015, 1030, 1054, 1012, 1003, 1006, 1039, 4, 3, 0, 0, 0, 0, 0, 0, 1001, 1034, 1012, 1008, 1062, 1045, 989, 1037, 8, 9, 0, 0, 0, 0, 0, 0, 1026, 1013, 1041, 1032, 1015, 996, 1009, 1032, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8217, 8200, 8187, 8203, 8178, 8205, 8192] avg=8192.0000 max=8217 max_rank=1 min=8154 min_rank=0 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1007, 1014, 1058, 1011, 1042, 1068, 1073, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 993, 1023, 1033, 1023, 1007, 1001, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1028, 995, 1004, 1049, 1068, 995, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1031, 980, 988, 1007, 1042, 1020, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 977, 1002, 1051, 1013, 999, 1051, 995, 8, 4, 0, 0, 0, 0, 0, 0, 1006, 983, 994, 993, 1031, 986, 1026, 1003, 2, 2, 0, 0, 0, 0, 0, 0, 1052, 1011, 998, 1011, 1009, 1067, 1028, 1014, 4, 3, 0, 0, 0, 0, 0, 0, 1089, 1064, 1043, 1045, 1007, 1029, 1084, 1032, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8313, 8158, 8197, 8094, 8134, 8026, 8197, 8417] avg=8192.0000 max=8417 max_rank=7 min=8026 min_rank=5 imbalance_ratio=1.027466 max_load_reduction=57119 max_load_reduction_pct=0.871567 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8064, 8320, 8192, 8320, 8448, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1017, 1067, 1027, 987, 1048, 1029, 1034, 1004, 975, 1012, 1025, 1051, 985, 1002, 1009, 1052, 993, 1020, 1053, 1066, 975, 1031, 1037, 1016, 997, 1040, 1000, 977, 985, 1023, 1033, 1050, 968, 1030, 1066, 1039, 1008, 1080, 1076, 1070, 979, 1028, 1065, 975, 966, 1039, 997, 1082, 1053, 1001, 1018, 1085, 961, 1071, 1012, 979, 1008, 1067, 1030, 984, 1008, 1081, 1006, 1084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[982, 1017, 1019, 1008, 1051, 968, 1030, 1024, 1036, 992, 1011, 988, 1059, 1037, 1009, 1006, 970, 944, 1016, 1037, 981, 1013, 1038, 1051, 1080, 1090, 1049, 1009, 1065, 1027, 995, 996, 1079, 1033, 1064, 1012, 1017, 1004, 1078, 1004, 1031, 1025, 965, 1052, 1035, 1028, 997, 1048, 1015, 1003, 1013, 1051, 1031, 1005, 1038, 1058, 1012, 1044, 1064, 1007, 1017, 1047, 1039, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1076, 1029, 1004, 1047, 997, 984, 1037, 1011, 1043, 1074, 1033, 1024, 961, 1011, 1009, 996, 1006, 1013, 1044, 1031, 993, 988, 972, 1010, 1043, 1019, 1047, 998, 1032, 1090, 973, 1041, 1040, 1029, 1025, 1040, 1028, 1073, 1010, 1005, 1052, 1041, 1070, 1021, 1067, 1013, 1049, 1045, 1040, 1027, 1018, 969, 995, 1018, 997, 997, 1003, 1019, 1057, 1061, 979, 1038, 1034, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1031, 1038, 991, 1011, 989, 1042, 1043, 1053, 1033, 1035, 1049, 1041, 1025, 1069, 1026, 1040, 1023, 1008, 1025, 1016, 979, 1080, 989, 1047, 995, 975, 1041, 1026, 1048, 1063, 1019, 974, 1013, 1014, 1024, 1017, 970, 1063, 1043, 1021, 1017, 1005, 1060, 1036, 1016, 1011, 993, 1050, 1045, 989, 979, 1041, 1011, 1031, 1022, 985, 1040, 1014, 1035, 1029, 1052, 1061, 1024, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1046, 1008, 1057, 1051, 1076, 1023, 1048, 1049, 1065, 1011, 1044, 1022, 1012, 996, 993, 1042, 1039, 975, 1027, 1019, 1035, 1035, 997, 1062, 1016, 1039, 1016, 977, 1013, 1039, 1066, 1029, 1000, 1001, 1059, 1003, 1022, 1024, 1070, 1021, 976, 1017, 1028, 1037, 1057, 1010, 997, 1031, 980, 974, 1041, 964, 1056, 1045, 1029, 1037, 1055, 1034, 981, 1046, 999, 1007, 976, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[961, 1024, 997, 1053, 1003, 1016, 1021, 1002, 1027, 1034, 1012, 1020, 1037, 1031, 1049, 1026, 982, 1008, 1032, 1035, 1080, 1032, 1070, 1039, 1020, 1042, 1007, 1036, 1062, 1019, 1034, 1012, 1037, 1024, 1016, 1014, 1011, 963, 1052, 1007, 1015, 1040, 1028, 1063, 1018, 1029, 1008, 1025, 1014, 1033, 986, 1061, 1013, 1008, 1068, 1017, 1045, 1011, 1038, 1022, 964, 1016, 1033, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1044, 1011, 1021, 1014, 1006, 1009, 1022, 1037, 976, 1041, 991, 1054, 1043, 1035, 1029, 1021, 1032, 994, 1013, 950, 1061, 1034, 1033, 993, 1017, 1044, 1013, 1016, 1061, 1007, 1006, 1022, 1027, 993, 1034, 1045, 1008, 1032, 1028, 990, 1028, 993, 1081, 1048, 969, 971, 1029, 1042, 1041, 1005, 1035, 1038, 1060, 1059, 1037, 1038, 1042, 1050, 1013, 1071, 1035, 1027, 990, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1030, 1057, 1078, 954, 1024, 1013, 1059, 1067, 1061, 1001, 1003, 974, 994, 1031, 1021, 985, 1020, 977, 1007, 994, 1019, 1019, 1006, 1050, 995, 1075, 1026, 1028, 1066, 1035, 1008, 1072, 1037, 953, 1028, 1010, 1023, 972, 1001, 1021, 1021, 1084, 1033, 1035, 1015, 1027, 1014, 1050, 994, 1009, 1040, 1064, 1021, 1033, 1018, 1082, 1040, 1050, 1014, 1056, 1015, 1021, 1027, 979] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[982, 1017, 1019, 1008, 1051, 968, 1026, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 992, 1011, 988, 1059, 1037, 1009, 1006, 17, 0, 0, 0, 0, 0, 0, 0, 970, 944, 1016, 1037, 981, 1013, 1038, 1051, 27, 23, 0, 0, 0, 0, 0, 0, 1080, 1090, 1049, 1009, 1042, 1027, 995, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1033, 1064, 1012, 1017, 1004, 1074, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1025, 965, 1052, 1035, 1028, 997, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1003, 1013, 1051, 1031, 1005, 1038, 1058, 4, 4, 0, 0, 0, 0, 0, 0, 1012, 1044, 1064, 1007, 1017, 1020, 1039, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8155, 8100, 8288, 8287, 8164, 8222, 8225] avg=8192.0000 max=8288 max_rank=3 min=8095 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1031, 1038, 991, 1011, 989, 1042, 1038, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1035, 1049, 1041, 1025, 1069, 1026, 1040, 17, 0, 0, 0, 0, 0, 0, 0, 1023, 1008, 1025, 1016, 979, 1080, 989, 1047, 28, 22, 0, 0, 0, 0, 0, 0, 995, 975, 1041, 1026, 1026, 1063, 1019, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1014, 1024, 1017, 970, 1063, 1039, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1005, 1060, 1036, 1016, 1011, 993, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 989, 979, 1041, 1011, 1031, 1022, 985, 5, 4, 0, 0, 0, 0, 0, 0, 1040, 1014, 1035, 1029, 1052, 1033, 1024, 1001, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8335, 8217, 8119, 8161, 8171, 8112, 8228] avg=8192.0000 max=8335 max_rank=1 min=8112 min_rank=6 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1017, 1067, 1027, 987, 1048, 1029, 1030, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1012, 1025, 1051, 985, 1002, 1009, 1052, 17, 0, 0, 0, 0, 0, 0, 0, 993, 1020, 1053, 1066, 975, 1031, 1037, 1016, 28, 21, 0, 0, 0, 0, 0, 0, 997, 1040, 1000, 977, 964, 1023, 1033, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1030, 1066, 1039, 1008, 1080, 1072, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1028, 1065, 975, 966, 1039, 997, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1001, 1018, 1085, 961, 1071, 1012, 979, 4, 4, 0, 0, 0, 0, 0, 0, 1008, 1067, 1030, 984, 1008, 1053, 1006, 1084, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8128, 8240, 8084, 8333, 8114, 8188, 8240] avg=8192.0000 max=8333 max_rank=4 min=8084 min_rank=3 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1076, 1029, 1004, 1047, 997, 984, 1032, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1074, 1033, 1024, 961, 1011, 1009, 996, 16, 0, 0, 0, 0, 0, 0, 0, 1006, 1013, 1044, 1031, 993, 988, 972, 1010, 27, 22, 0, 0, 0, 0, 0, 0, 1043, 1019, 1047, 998, 1010, 1090, 973, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1029, 1025, 1040, 1028, 1073, 1007, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1041, 1070, 1021, 1067, 1013, 1049, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1027, 1018, 969, 995, 1018, 997, 997, 5, 3, 0, 0, 0, 0, 0, 0, 1003, 1019, 1057, 1061, 979, 1011, 1034, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8167, 8106, 8221, 8247, 8342, 8069, 8204] avg=8192.0000 max=8342 max_rank=5 min=8069 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32862 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[961, 1024, 997, 1053, 1003, 1016, 1017, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1034, 1012, 1020, 1037, 1031, 1049, 1026, 16, 0, 0, 0, 0, 0, 0, 0, 982, 1008, 1032, 1035, 1080, 1032, 1070, 1039, 27, 23, 0, 0, 0, 0, 0, 0, 1020, 1042, 1007, 1036, 1039, 1019, 1034, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1024, 1016, 1014, 1011, 963, 1048, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1040, 1028, 1063, 1018, 1029, 1008, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1033, 986, 1061, 1013, 1008, 1068, 1017, 4, 4, 0, 0, 0, 0, 0, 0, 1045, 1011, 1038, 1022, 964, 989, 1033, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8073, 8252, 8328, 8209, 8120, 8210, 8208, 8136] avg=8192.0000 max=8328 max_rank=2 min=8073 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1044, 1011, 1021, 1014, 1006, 1009, 1018, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1041, 991, 1054, 1043, 1035, 1029, 1021, 16, 0, 0, 0, 0, 0, 0, 0, 1032, 994, 1013, 950, 1061, 1034, 1033, 993, 27, 23, 0, 0, 0, 0, 0, 0, 1017, 1044, 1013, 1016, 1038, 1007, 1006, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 993, 1034, 1045, 1008, 1032, 1024, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 993, 1081, 1048, 969, 971, 1029, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1005, 1035, 1038, 1060, 1059, 1037, 1038, 4, 4, 0, 0, 0, 0, 0, 0, 1042, 1050, 1013, 1071, 1035, 1000, 990, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8206, 8160, 8163, 8153, 8145, 8321, 8228] avg=8192.0000 max=8321 max_rank=6 min=8145 min_rank=5 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1046, 1008, 1057, 1051, 1076, 1023, 1039, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1011, 1044, 1022, 1012, 996, 993, 1042, 20, 1, 0, 0, 0, 0, 0, 0, 1039, 975, 1027, 1019, 1035, 1035, 997, 1062, 30, 24, 0, 0, 0, 0, 0, 0, 1016, 1039, 1016, 977, 988, 1039, 1066, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1001, 1059, 1003, 1022, 1024, 1063, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1017, 1028, 1037, 1057, 1010, 997, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 980, 974, 1041, 964, 1056, 1045, 1029, 1037, 9, 7, 5, 0, 0, 0, 0, 0, 1055, 1034, 981, 1046, 999, 977, 976, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8349, 8206, 8243, 8170, 8193, 8128, 8147, 8100] avg=8192.0000 max=8349 max_rank=0 min=8100 min_rank=7 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1030, 1057, 1078, 954, 1024, 1013, 1054, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1001, 1003, 974, 994, 1031, 1021, 985, 17, 0, 0, 0, 0, 0, 0, 0, 1020, 977, 1007, 994, 1019, 1019, 1006, 1050, 27, 23, 0, 0, 0, 0, 0, 0, 995, 1075, 1026, 1028, 1043, 1035, 1008, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 953, 1028, 1010, 1023, 972, 998, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1084, 1033, 1035, 1015, 1027, 1014, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1009, 1040, 1064, 1021, 1033, 1018, 1082, 5, 3, 0, 0, 0, 0, 0, 0, 1040, 1050, 1014, 1056, 1015, 994, 1027, 979, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8087, 8142, 8282, 8042, 8262, 8269, 8175] avg=8192.0000 max=8282 max_rank=3 min=8042 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8064, 8320, 8192, 8192, 8320, 8192, 8320, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8448, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8064, 8192, 8320, 8192, 8320, 8320, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1004, 1038, 1014, 1079, 999, 1005, 1031, 1040, 973, 992, 988, 1012, 1026, 1069, 1043, 1062, 1002, 1040, 983, 972, 1053, 1015, 1081, 1030, 1071, 1052, 984, 1044, 995, 995, 1027, 1037, 1052, 1028, 941, 1010, 1024, 1023, 1016, 1039, 1024, 1058, 1059, 1071, 1006, 1051, 1010, 1060, 996, 993, 980, 1066, 1031, 1027, 1030, 1015, 1016, 1010, 1035, 1001, 1011, 1042, 1012, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1049, 1024, 1033, 979, 1036, 1084, 994, 1068, 973, 1038, 1041, 982, 977, 1013, 1074, 1022, 973, 1032, 1033, 1016, 1059, 991, 1048, 1008, 1051, 1020, 1028, 1011, 1045, 1038, 1026, 1056, 1020, 1014, 1037, 1033, 1006, 1010, 1021, 1059, 1016, 1024, 995, 1027, 982, 1003, 1037, 1060, 1085, 1001, 1024, 1007, 1027, 961, 1047, 1065, 994, 983, 1033, 1033, 1038, 1094, 946, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1017, 1034, 975, 1042, 1055, 982, 1006, 999, 1041, 1006, 1043, 984, 1042, 1019, 941, 1021, 1004, 1034, 1017, 1060, 1088, 1036, 1056, 1027, 1052, 973, 1077, 1062, 1042, 1025, 984, 1056, 1038, 1038, 1021, 960, 1027, 1023, 1035, 1008, 1068, 1051, 992, 1034, 993, 1043, 980, 1027, 1037, 1028, 1042, 1029, 1020, 1021, 1036, 1043, 1026, 1006, 1007, 1021, 1020, 1036, 1034, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1072, 997, 1027, 993, 1028, 1013, 1027, 1049, 1025, 1036, 1042, 1071, 1023, 1044, 1032, 1032, 1051, 1053, 1053, 980, 1032, 1036, 1031, 1028, 1060, 1000, 999, 1026, 1015, 1033, 1020, 1021, 996, 1015, 1007, 1019, 995, 1003, 946, 995, 1018, 983, 1045, 1050, 1038, 966, 999, 1026, 1014, 1033, 1038, 1063, 1027, 996, 1053, 1041, 1020, 1036, 1007, 1046, 1040, 991, 1032, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1050, 1024, 1053, 959, 1050, 1014, 1025, 1045, 1030, 1042, 1018, 1014, 1038, 1057, 1045, 957, 1055, 1075, 1045, 1007, 1038, 1055, 1033, 976, 1063, 1027, 981, 1051, 1032, 1039, 974, 996, 994, 1024, 1060, 1054, 1010, 997, 1045, 1037, 1010, 1020, 990, 1007, 1045, 1004, 1054, 1001, 1038, 1032, 973, 1061, 1020, 1051, 1027, 1034, 1019, 1055, 1008, 944, 1006, 1013, 1032, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1020, 1027, 1026, 1008, 1024, 1048, 1029, 1016, 1085, 1044, 1051, 1024, 1027, 949, 1032, 974, 1046, 1039, 1015, 1008, 997, 1045, 1060, 1092, 993, 995, 1026, 986, 1038, 1054, 997, 1031, 994, 1040, 1024, 979, 1011, 1033, 991, 1018, 1036, 1014, 1017, 1048, 1021, 1015, 1033, 1056, 999, 1019, 1027, 988, 1054, 1040, 1021, 989, 1028, 1077, 1041, 961, 1001, 1038, 1076, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[984, 996, 1041, 1011, 1052, 938, 1046, 1057, 1023, 1066, 978, 1039, 1043, 1020, 1043, 1008, 990, 980, 1006, 1043, 988, 985, 1000, 1019, 981, 1037, 1039, 992, 1051, 1019, 1033, 1043, 1064, 1023, 1058, 1021, 1011, 1048, 1019, 1020, 1005, 1055, 1072, 1012, 1021, 1005, 1022, 966, 1022, 1033, 1013, 1045, 1058, 1037, 1016, 1009, 1033, 1096, 1084, 1033, 1051, 1012, 1014, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[984, 1015, 1015, 1019, 976, 1023, 1011, 991, 1041, 1059, 1039, 985, 1006, 1051, 1025, 1029, 1025, 1007, 1048, 1080, 1046, 1035, 1019, 1056, 1048, 1005, 1030, 1049, 1034, 958, 1011, 1072, 1111, 1075, 1034, 1061, 962, 1036, 1006, 1025, 1016, 1039, 1001, 981, 1002, 1000, 1010, 1061, 1034, 1036, 1040, 996, 1010, 1002, 1016, 1027, 993, 1014, 1038, 1034, 970, 1045, 1053, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1049, 1024, 1033, 979, 1036, 1084, 994, 1068, 13, 7, 0, 0, 0, 0, 0, 0, 973, 1038, 1041, 982, 977, 1013, 1074, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1032, 1033, 1016, 1059, 991, 1011, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1020, 1028, 1011, 1045, 1038, 1026, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1014, 1037, 1033, 1006, 1010, 1021, 1059, 24, 0, 0, 0, 0, 0, 0, 0, 1016, 1024, 995, 1027, 982, 1003, 1037, 1060, 6, 5, 0, 0, 0, 0, 0, 0, 1085, 1001, 1024, 995, 1027, 961, 1047, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 994, 983, 1033, 1033, 1038, 1094, 946, 1032, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8120, 8123, 8268, 8224, 8155, 8205, 8154] avg=8192.0000 max=8287 max_rank=0 min=8120 min_rank=1 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1072, 997, 1027, 993, 1028, 1013, 1027, 1049, 13, 7, 0, 0, 0, 0, 0, 0, 1025, 1036, 1042, 1071, 1023, 1044, 1032, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1053, 1053, 980, 1032, 1036, 995, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1000, 999, 1026, 1015, 1033, 1020, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1015, 1007, 1019, 995, 1003, 946, 995, 23, 0, 0, 0, 0, 0, 0, 0, 1018, 983, 1045, 1050, 1038, 966, 999, 1026, 6, 6, 0, 0, 0, 0, 0, 0, 1014, 1033, 1038, 1050, 1027, 996, 1053, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1036, 1007, 1046, 1040, 991, 1032, 1049, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8305, 8228, 8167, 7999, 8137, 8252, 8222] avg=8192.0000 max=8305 max_rank=1 min=7999 min_rank=4 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32610 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1004, 1038, 1014, 1079, 999, 1005, 1031, 1040, 13, 7, 0, 0, 0, 0, 0, 0, 973, 992, 988, 1012, 1026, 1069, 1043, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1040, 983, 972, 1053, 1015, 1044, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1052, 984, 1044, 995, 995, 1027, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1028, 941, 1010, 1024, 1023, 1016, 1039, 24, 0, 0, 0, 0, 0, 0, 0, 1024, 1058, 1059, 1071, 1006, 1051, 1010, 1060, 6, 6, 0, 0, 0, 0, 0, 0, 996, 993, 980, 1053, 1031, 1027, 1030, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1010, 1035, 1001, 1011, 1042, 1012, 1043, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8165, 8139, 8198, 8157, 8351, 8125, 8171] avg=8192.0000 max=8351 max_rank=5 min=8125 min_rank=6 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[984, 996, 1041, 1011, 1052, 938, 1046, 1057, 12, 7, 0, 0, 0, 0, 0, 0, 1023, 1066, 978, 1039, 1043, 1020, 1043, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 990, 980, 1006, 1043, 988, 985, 965, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1037, 1039, 992, 1051, 1019, 1033, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1023, 1058, 1021, 1011, 1048, 1019, 1020, 23, 0, 0, 0, 0, 0, 0, 0, 1005, 1055, 1072, 1012, 1021, 1005, 1022, 966, 6, 6, 0, 0, 0, 0, 0, 0, 1022, 1033, 1013, 1032, 1058, 1037, 1016, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1096, 1084, 1033, 1051, 1012, 1014, 1007, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8220, 7976, 8188, 8287, 8170, 8220, 8331] avg=8192.0000 max=8331 max_rank=7 min=7976 min_rank=2 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=33008 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1020, 1027, 1026, 1008, 1024, 1048, 1029, 1016, 13, 7, 0, 0, 0, 0, 0, 0, 1085, 1044, 1051, 1024, 1027, 949, 1032, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1039, 1015, 1008, 997, 1045, 1023, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 986, 995, 1026, 986, 1038, 1054, 997, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1040, 1024, 979, 1011, 1033, 991, 1018, 24, 0, 0, 0, 0, 0, 0, 0, 1036, 1014, 1017, 1048, 1021, 1015, 1033, 1056, 6, 5, 0, 0, 0, 0, 0, 0, 999, 1019, 1027, 976, 1054, 1040, 1021, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1077, 1041, 961, 1001, 1038, 1076, 1041, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8186, 8265, 8113, 8114, 8251, 8125, 8264] avg=8192.0000 max=8265 max_rank=2 min=8113 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1017, 1034, 975, 1042, 1055, 982, 1006, 999, 13, 7, 0, 0, 0, 0, 0, 0, 1041, 1006, 1043, 984, 1042, 1019, 941, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1034, 1017, 1060, 1088, 1036, 1019, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 973, 1077, 1062, 1042, 1025, 984, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1038, 1021, 960, 1027, 1023, 1035, 1008, 24, 0, 0, 0, 0, 0, 0, 0, 1068, 1051, 992, 1034, 993, 1043, 980, 1027, 6, 5, 0, 0, 0, 0, 0, 0, 1037, 1028, 1042, 1017, 1020, 1021, 1036, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1006, 1007, 1021, 1020, 1036, 1034, 992, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8097, 8285, 8264, 8174, 8199, 8244, 8143] avg=8192.0000 max=8285 max_rank=2 min=8097 min_rank=1 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1050, 1024, 1053, 959, 1050, 1014, 1025, 1045, 17, 10, 0, 0, 0, 0, 0, 0, 1030, 1042, 1018, 1014, 1038, 1057, 1045, 957, 7, 0, 0, 0, 0, 0, 0, 0, 1055, 1075, 1045, 1007, 1038, 1055, 989, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1027, 981, 1051, 1032, 1039, 974, 996, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1024, 1060, 1054, 1010, 997, 1045, 1037, 27, 0, 0, 0, 0, 0, 0, 0, 1010, 1020, 990, 1007, 1045, 1004, 1054, 1001, 11, 10, 0, 0, 0, 0, 0, 0, 1038, 1032, 973, 1041, 1020, 1051, 1027, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1055, 1008, 944, 1006, 1013, 1032, 1003, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8208, 8240, 8138, 8248, 8152, 8216, 8087] avg=8192.0000 max=8248 max_rank=4 min=8087 min_rank=7 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[984, 1015, 1015, 1019, 976, 1023, 1011, 991, 13, 7, 0, 0, 0, 0, 0, 0, 1041, 1059, 1039, 985, 1006, 1051, 1025, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1007, 1048, 1080, 1046, 1035, 983, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1005, 1030, 1049, 1034, 958, 1011, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 1075, 1034, 1061, 962, 1036, 1006, 1025, 23, 0, 0, 0, 0, 0, 0, 0, 1016, 1039, 1001, 981, 1002, 1000, 1010, 1061, 6, 5, 0, 0, 0, 0, 0, 0, 1034, 1036, 1040, 984, 1010, 1002, 1016, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1014, 1038, 1034, 970, 1045, 1053, 1016, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8054, 8235, 8280, 8200, 8333, 8121, 8149, 8164] avg=8192.0000 max=8333 max_rank=4 min=8054 min_rank=0 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8064, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1023, 1033, 1005, 1043, 1035, 1038, 1075, 1011, 1059, 1013, 1006, 1047, 1001, 1071, 1021, 1047, 999, 1054, 1020, 1069, 1034, 1041, 1019, 1021, 1024, 1014, 1003, 982, 1064, 1015, 1022, 1031, 1013, 1034, 1023, 1027, 948, 1022, 1005, 1082, 954, 977, 1025, 1036, 1046, 1042, 1025, 986, 1039, 995, 1049, 1039, 980, 1010, 1041, 1012, 1024, 1035, 1049, 1014, 1033, 1033, 1006, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[998, 976, 986, 1087, 1059, 1034, 1074, 1042, 973, 984, 1053, 1048, 1040, 1004, 1047, 1003, 1003, 1039, 998, 994, 994, 1004, 983, 1015, 1040, 992, 1065, 1073, 1020, 1030, 1034, 1018, 1023, 1051, 1042, 1038, 1001, 988, 1003, 1070, 1004, 1009, 1019, 1058, 1036, 1050, 1033, 1046, 1042, 999, 997, 1054, 983, 1040, 993, 1011, 1018, 1000, 1016, 1079, 1014, 1069, 1046, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1020, 1027, 1029, 977, 1004, 1020, 1047, 1029, 997, 1056, 1088, 954, 1028, 1046, 1031, 1040, 985, 1013, 970, 1063, 1015, 1060, 1054, 983, 1052, 1030, 1048, 988, 1023, 1062, 1036, 1077, 1054, 1077, 1004, 1080, 974, 1039, 1035, 1010, 1073, 941, 1015, 983, 1032, 1000, 1033, 1003, 1020, 1014, 1012, 1046, 967, 1025, 1021, 1019, 1054, 1031, 997, 1015, 991, 1027, 1060, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1082, 1022, 1058, 960, 1057, 973, 1044, 1001, 1078, 1035, 1022, 1079, 991, 1040, 1000, 1041, 1006, 1029, 988, 1026, 1016, 1000, 1052, 1002, 1011, 1027, 1042, 1024, 1054, 1039, 1083, 1033, 1027, 1083, 992, 1040, 1028, 985, 1012, 971, 1010, 1009, 1020, 1042, 1038, 1031, 990, 1028, 1030, 1020, 984, 1004, 1024, 1009, 1055, 1012, 1025, 1033, 1027, 1024, 1052, 1034, 999, 983] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1053, 1029, 1049, 1018, 978, 988, 1048, 978, 1052, 988, 1017, 1023, 1065, 1037, 1015, 986, 1035, 1041, 1025, 957, 998, 1038, 1019, 1027, 976, 1065, 1013, 1058, 1038, 1047, 1005, 1056, 1022, 1028, 1033, 1004, 976, 1011, 1066, 1038, 1024, 1054, 1093, 1010, 988, 1019, 1060, 1004, 1079, 1060, 1051, 1005, 1071, 1057, 1058, 961, 1005, 986, 1030, 1028, 952, 1022, 1001, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1063, 1039, 986, 1071, 1004, 986, 1018, 1015, 986, 1011, 1055, 1045, 1019, 1023, 1070, 1019, 1043, 991, 1044, 1030, 1063, 1019, 1068, 1037, 959, 1044, 1026, 964, 1008, 1027, 1033, 1017, 1033, 1085, 1025, 1025, 1066, 1025, 1056, 1021, 975, 1062, 1020, 1046, 1033, 1000, 1014, 988, 1069, 1024, 999, 1045, 1066, 1032, 975, 950, 1051, 990, 1047, 1004, 1000, 1010, 1007, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1005, 1063, 1018, 1039, 1081, 1003, 981, 1008, 1010, 1097, 972, 1000, 986, 1001, 1015, 1083, 990, 1007, 1003, 1028, 1025, 984, 970, 1028, 1034, 1007, 1034, 987, 1058, 1058, 1025, 1040, 1060, 1011, 1034, 1029, 1047, 1004, 1016, 1029, 1052, 999, 1049, 1011, 1005, 1030, 990, 1023, 1006, 1001, 1030, 986, 1019, 1049, 1007, 1037, 1032, 1070, 1046, 1080, 1016, 1005, 1099, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1024, 1008, 1052, 1019, 1020, 1059, 1015, 996, 1022, 1026, 1043, 1025, 1014, 971, 1046, 1023, 1079, 1027, 1064, 994, 1002, 993, 1034, 1070, 1008, 1039, 1033, 1007, 988, 1028, 1033, 1080, 1018, 1046, 1006, 1034, 1051, 1036, 1001, 1051, 1007, 1037, 1033, 1015, 999, 1060, 1032, 998, 1037, 1021, 986, 965, 1013, 1084, 1020, 964, 1025, 1016, 1037, 1003, 1019, 1002, 1025, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1023, 1033, 1005, 1043, 1035, 1038, 1070, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1013, 988, 1047, 1001, 1071, 1021, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1054, 1020, 1069, 1034, 1041, 1019, 1021, 24, 6, 0, 0, 0, 0, 0, 0, 1024, 1014, 1003, 982, 1064, 1015, 1022, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1007, 1023, 1027, 948, 1022, 1005, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 954, 977, 1025, 1036, 1046, 1042, 1025, 986, 18, 3, 3, 0, 0, 0, 0, 0, 1039, 995, 1049, 1039, 980, 1010, 1041, 1012, 32, 0, 0, 0, 0, 0, 0, 0, 1024, 1035, 1049, 1014, 1033, 1033, 1006, 992, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8247, 8287, 8117, 8127, 8115, 8197, 8188] avg=8192.0000 max=8287 max_rank=2 min=8115 min_rank=5 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1020, 1027, 1029, 977, 1004, 1020, 1042, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1056, 1068, 954, 1028, 1046, 1031, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1013, 970, 1063, 1015, 1060, 1054, 983, 25, 6, 0, 0, 0, 0, 0, 0, 1052, 1030, 1048, 988, 1023, 1062, 1036, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1049, 1004, 1080, 974, 1039, 1035, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 941, 1015, 983, 1032, 1000, 1033, 1003, 20, 3, 3, 0, 0, 0, 0, 0, 1020, 1014, 1012, 1046, 967, 1025, 1021, 1019, 33, 0, 0, 0, 0, 0, 0, 0, 1054, 1031, 997, 1015, 991, 1027, 1060, 1032, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8220, 8174, 8277, 8245, 8106, 8157, 8209] avg=8192.0000 max=8277 max_rank=3 min=8106 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1053, 1029, 1049, 1018, 978, 988, 1043, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 988, 999, 1023, 1065, 1037, 1015, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1041, 1025, 957, 998, 1038, 1019, 1027, 24, 6, 0, 0, 0, 0, 0, 0, 976, 1065, 1013, 1058, 1038, 1047, 1005, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1001, 1033, 1004, 976, 1011, 1066, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1054, 1093, 1010, 988, 1019, 1060, 1004, 18, 3, 3, 0, 0, 0, 0, 0, 1079, 1060, 1051, 1005, 1071, 1057, 1058, 961, 33, 0, 0, 0, 0, 0, 0, 0, 1005, 986, 1030, 1028, 952, 1022, 1001, 1018, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8165, 8170, 8219, 8151, 8276, 8375, 8044] avg=8192.0000 max=8375 max_rank=6 min=8044 min_rank=7 imbalance_ratio=1.022339 max_load_reduction=57161 max_load_reduction_pct=0.872208 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1082, 1022, 1058, 960, 1057, 973, 1039, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1035, 1004, 1079, 991, 1040, 1000, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1029, 988, 1026, 1016, 1000, 1052, 1002, 25, 6, 0, 0, 0, 0, 0, 0, 1011, 1027, 1042, 1024, 1054, 1039, 1083, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1055, 992, 1040, 1028, 985, 1012, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1009, 1020, 1042, 1038, 1031, 990, 1028, 18, 3, 3, 0, 0, 0, 0, 0, 1030, 1020, 984, 1004, 1024, 1009, 1055, 1012, 32, 0, 0, 0, 0, 0, 0, 0, 1025, 1033, 1027, 1024, 1052, 1034, 999, 983, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8268, 8150, 8275, 8110, 8192, 8170, 8179] avg=8192.0000 max=8275 max_rank=3 min=8110 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[998, 976, 986, 1087, 1059, 1034, 1069, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 973, 984, 1034, 1048, 1040, 1004, 1047, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1039, 998, 994, 994, 1004, 983, 1015, 25, 6, 0, 0, 0, 0, 0, 0, 1040, 992, 1065, 1073, 1020, 1030, 1034, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1023, 1042, 1038, 1001, 988, 1003, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1009, 1019, 1058, 1036, 1050, 1033, 1046, 19, 3, 3, 0, 0, 0, 0, 0, 1042, 999, 997, 1054, 983, 1040, 993, 1011, 32, 0, 0, 0, 0, 0, 0, 0, 1018, 1000, 1016, 1079, 1014, 1069, 1046, 994, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8133, 8061, 8234, 8188, 8280, 8151, 8238] avg=8192.0000 max=8280 max_rank=5 min=8061 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1005, 1063, 1018, 1039, 1081, 1003, 976, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1097, 955, 1000, 986, 1001, 1015, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1007, 1003, 1028, 1025, 984, 970, 1028, 24, 6, 0, 0, 0, 0, 0, 0, 1034, 1007, 1034, 987, 1058, 1058, 1025, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 984, 1034, 1029, 1047, 1004, 1016, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 999, 1049, 1011, 1005, 1030, 990, 1023, 17, 3, 3, 0, 0, 0, 0, 0, 1006, 1001, 1030, 986, 1019, 1049, 1007, 1037, 32, 0, 0, 0, 0, 0, 0, 0, 1032, 1070, 1046, 1080, 1016, 1005, 1099, 1024, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8147, 8065, 8205, 8203, 8182, 8167, 8374] avg=8192.0000 max=8374 max_rank=7 min=8065 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=57162 max_load_reduction_pct=0.872223 moved_to_echo=32926 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1063, 1039, 986, 1071, 1004, 986, 1006, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1011, 1032, 1045, 1019, 1023, 1070, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 991, 1044, 1030, 1063, 1019, 1068, 1037, 30, 10, 0, 0, 0, 0, 0, 0, 959, 1044, 1026, 964, 1008, 1027, 1033, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1046, 1025, 1025, 1066, 1025, 1056, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1062, 1020, 1046, 1033, 1000, 1014, 988, 23, 9, 9, 0, 0, 0, 0, 0, 1069, 1024, 999, 1045, 1066, 1032, 975, 950, 35, 0, 0, 0, 0, 0, 0, 0, 1051, 990, 1047, 1004, 1000, 1010, 1007, 1010, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8205, 8335, 8033, 8297, 8179, 8195, 8122] avg=8192.0000 max=8335 max_rank=2 min=8033 min_rank=3 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1024, 1008, 1052, 1019, 1020, 1059, 1010, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1026, 1024, 1025, 1014, 971, 1046, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1027, 1064, 994, 1002, 993, 1034, 1070, 25, 6, 0, 0, 0, 0, 0, 0, 1008, 1039, 1033, 1007, 988, 1028, 1033, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1018, 1006, 1034, 1051, 1036, 1001, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1037, 1033, 1015, 999, 1060, 1032, 998, 19, 3, 3, 0, 0, 0, 0, 0, 1037, 1021, 986, 965, 1013, 1084, 1020, 964, 34, 0, 0, 0, 0, 0, 0, 0, 1025, 1016, 1037, 1003, 1019, 1002, 1025, 1053, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8151, 8294, 8176, 8215, 8206, 8124, 8182] avg=8192.0000 max=8294 max_rank=2 min=8124 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8192, 8192, 8192, 8320, 8192, 8064, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1015, 1039, 983, 1008, 983, 967, 1008, 1006, 1028, 1027, 1008, 1076, 1074, 1049, 1028, 1042, 1051, 1003, 987, 982, 998, 997, 1007, 1029, 1013, 1023, 1048, 1012, 1059, 1044, 1007, 1040, 1033, 1060, 1012, 1038, 1016, 1009, 1041, 1026, 1009, 1038, 1016, 1074, 989, 1011, 991, 1011, 1048, 1055, 1063, 1011, 1026, 1023, 1030, 1052, 1004, 1020, 1042, 1041, 1004, 992, 1058, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1054, 1001, 1068, 1002, 1088, 1015, 995, 1064, 1052, 1006, 1021, 995, 1049, 1030, 1032, 995, 1071, 1010, 1017, 1045, 1006, 1022, 1014, 1045, 1039, 1002, 1010, 1002, 1039, 986, 983, 1011, 1002, 1034, 1049, 1025, 966, 1015, 973, 1031, 1002, 994, 1049, 1005, 1041, 1053, 1051, 1022, 1009, 1067, 1026, 1007, 980, 1047, 1030, 1031, 980, 1026, 1020, 1006, 1056, 1095, 1042, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1018, 988, 1006, 992, 1035, 1043, 998, 999, 1045, 974, 1037, 1005, 1022, 1035, 1035, 1027, 1040, 1038, 1024, 1010, 1002, 1006, 1005, 1037, 1062, 1045, 1064, 1024, 1011, 972, 1027, 1074, 1014, 995, 1046, 1024, 1018, 1013, 1045, 1025, 1001, 1043, 1051, 1042, 1028, 1047, 1065, 1033, 987, 1020, 1012, 1037, 1028, 987, 1018, 1001, 1019, 1010, 1000, 1030, 1057, 1037, 1053, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1056, 1019, 1033, 996, 1014, 1023, 1062, 1052, 1024, 1019, 1064, 1029, 1013, 1048, 1066, 1018, 976, 1012, 1040, 1051, 1030, 996, 1062, 1012, 1034, 1036, 1000, 1027, 1007, 1030, 989, 1026, 1077, 1037, 1038, 1023, 1047, 1022, 995, 1065, 1044, 981, 1035, 1005, 1028, 1028, 1002, 1046, 993, 1023, 1039, 1000, 1035, 1064, 1024, 1025, 989, 956, 1048, 1011, 1009, 1031, 1012, 940] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1005, 1054, 999, 1007, 1028, 995, 1048, 1050, 1006, 1030, 1028, 1046, 1058, 1038, 1026, 1050, 1061, 1090, 1034, 1027, 1003, 1001, 988, 972, 1007, 1008, 1042, 1018, 1049, 989, 1003, 963, 1020, 1038, 971, 1057, 1033, 1018, 1043, 1037, 1056, 1059, 987, 1023, 1041, 1004, 1026, 990, 1000, 1042, 1030, 1018, 1038, 1033, 989, 1045, 1053, 1034, 1002, 1019, 1021, 1013, 1066, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1006, 1037, 998, 1060, 1069, 1064, 1100, 1033, 1042, 1066, 1033, 938, 1060, 1049, 1031, 1011, 1042, 985, 1018, 1004, 993, 1009, 1027, 1000, 1035, 1026, 1031, 1057, 1013, 1065, 1036, 1006, 1004, 1070, 1015, 1022, 1052, 996, 993, 1001, 1007, 1014, 1046, 1003, 1042, 1055, 972, 1011, 1032, 1017, 1063, 1011, 1016, 1016, 1039, 1002, 976, 998, 1024, 1014, 1049, 1040, 991, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1019, 1006, 1041, 1018, 996, 1019, 1029, 1015, 1028, 1015, 1024, 998, 984, 1046, 1031, 1056, 1011, 1004, 1047, 1035, 1028, 1008, 1014, 1008, 1066, 1033, 1017, 999, 1034, 1055, 1028, 1067, 958, 998, 1062, 1014, 1065, 997, 981, 1033, 989, 1032, 995, 1094, 1030, 1054, 1021, 1039, 1035, 988, 1012, 1029, 1014, 995, 1052, 1039, 1060, 1043, 999, 1027, 1018, 1036, 994, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[983, 1026, 1042, 982, 1041, 1019, 1052, 1004, 1005, 1055, 1021, 1039, 1030, 987, 1031, 1031, 979, 1012, 1054, 1004, 1024, 1050, 1024, 1017, 1071, 1046, 1012, 999, 1043, 962, 1054, 1076, 1018, 1069, 1012, 1040, 1035, 994, 1013, 1077, 1039, 1063, 1014, 1008, 1002, 1035, 1003, 1046, 1036, 1058, 1008, 1040, 996, 1002, 1038, 1044, 1043, 997, 1029, 1019, 978, 1002, 1003, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1018, 988, 1006, 992, 1035, 1043, 998, 999, 4, 1, 1, 0, 0, 0, 0, 0, 1045, 974, 1037, 1005, 982, 1035, 1035, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1038, 1024, 1010, 1002, 1006, 1005, 1037, 40, 1, 0, 0, 0, 0, 0, 0, 1047, 1045, 1064, 1024, 1011, 972, 1027, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 994, 1046, 1024, 1018, 1013, 1045, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1043, 1051, 1042, 1028, 1035, 1065, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1016, 1012, 1037, 1028, 987, 1018, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1010, 1000, 1030, 1057, 1037, 1053, 1050, 14, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8085, 8140, 8203, 8264, 8179, 8298, 8086, 8281] avg=8192.0000 max=8298 max_rank=5 min=8085 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1054, 1001, 1068, 1002, 1088, 1015, 995, 1064, 5, 1, 1, 0, 0, 0, 0, 0, 1052, 1006, 1021, 995, 1008, 1030, 1032, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1010, 1017, 1045, 1006, 1022, 1014, 1045, 41, 0, 0, 0, 0, 0, 0, 0, 1026, 1002, 1010, 1002, 1039, 986, 983, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1033, 1049, 1025, 966, 1015, 973, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 994, 1049, 1005, 1041, 1041, 1051, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1062, 1026, 1007, 980, 1047, 1030, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1026, 1020, 1006, 1056, 1095, 1042, 1033, 13, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 8139, 8271, 8059, 8094, 8205, 8192, 8282] avg=8192.0000 max=8294 max_rank=0 min=8059 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1005, 1054, 999, 1007, 1028, 995, 1048, 1050, 4, 1, 0, 0, 0, 0, 0, 0, 1006, 1030, 1028, 1046, 1016, 1038, 1026, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1090, 1034, 1027, 1003, 1001, 988, 972, 42, 0, 0, 0, 0, 0, 0, 0, 994, 1008, 1042, 1018, 1049, 989, 1003, 963, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1037, 971, 1057, 1033, 1018, 1043, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1059, 987, 1023, 1041, 993, 1026, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1038, 1030, 1018, 1038, 1033, 989, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1034, 1002, 1019, 1021, 1013, 1066, 1007, 13, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8240, 8218, 8066, 8216, 8175, 8191, 8239] avg=8192.0000 max=8240 max_rank=1 min=8066 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1015, 1039, 983, 1008, 983, 967, 1008, 1006, 4, 1, 0, 0, 0, 0, 0, 0, 1028, 1027, 1008, 1076, 1032, 1049, 1028, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1003, 987, 982, 998, 997, 1007, 1029, 42, 0, 0, 0, 0, 0, 0, 0, 1000, 1023, 1048, 1012, 1059, 1044, 1007, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1059, 1012, 1038, 1016, 1009, 1041, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1038, 1016, 1074, 989, 1000, 991, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1051, 1063, 1011, 1026, 1023, 1030, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1020, 1042, 1041, 1004, 992, 1058, 1052, 13, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8014, 8290, 8096, 8233, 8234, 8128, 8304, 8237] avg=8192.0000 max=8304 max_rank=6 min=8014 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1056, 1019, 1033, 996, 1014, 1023, 1062, 1052, 4, 1, 0, 0, 0, 0, 0, 0, 1024, 1019, 1064, 1029, 973, 1048, 1066, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1012, 1040, 1051, 1030, 996, 1062, 1012, 40, 0, 0, 0, 0, 0, 0, 0, 1021, 1036, 1000, 1027, 1007, 1030, 989, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 1036, 1038, 1023, 1047, 1022, 995, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 981, 1035, 1005, 1028, 1017, 1002, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1019, 1039, 1000, 1035, 1064, 1024, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 989, 956, 1048, 1011, 1009, 1031, 1012, 940, 13, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8241, 8219, 8136, 8303, 8158, 8199, 8020] avg=8192.0000 max=8303 max_rank=4 min=8020 min_rank=7 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1006, 1037, 998, 1060, 1069, 1064, 1100, 1033, 4, 1, 1, 0, 0, 0, 0, 0, 1042, 1066, 1033, 938, 1018, 1049, 1031, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 985, 1018, 1004, 993, 1009, 1027, 1000, 42, 0, 0, 0, 0, 0, 0, 0, 1022, 1026, 1031, 1057, 1013, 1065, 1036, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1069, 1015, 1022, 1052, 996, 993, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1014, 1046, 1003, 1042, 1043, 972, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1013, 1063, 1011, 1016, 1016, 1039, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 976, 998, 1024, 1014, 1049, 1040, 991, 1001, 13, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8373, 8188, 8120, 8256, 8152, 8138, 8192, 8117] avg=8192.0000 max=8373 max_rank=0 min=8117 min_rank=7 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1019, 1006, 1041, 1018, 996, 1019, 1029, 1015, 10, 7, 5, 0, 0, 0, 0, 0, 1028, 1015, 1024, 998, 942, 1046, 1031, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1004, 1047, 1035, 1028, 1008, 1014, 1008, 42, 6, 0, 0, 0, 0, 0, 0, 1041, 1033, 1017, 999, 1034, 1055, 1028, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 958, 991, 1062, 1014, 1065, 997, 981, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1032, 995, 1094, 1030, 1035, 1021, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 978, 1012, 1029, 1014, 995, 1052, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1043, 999, 1027, 1018, 1036, 994, 1054, 19, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8140, 8203, 8274, 8101, 8235, 8154, 8264] avg=8192.0000 max=8274 max_rank=3 min=8101 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[983, 1026, 1042, 982, 1041, 1019, 1052, 1004, 4, 1, 0, 0, 0, 0, 0, 0, 1005, 1055, 1021, 1039, 989, 987, 1031, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1012, 1054, 1004, 1024, 1050, 1024, 1017, 41, 1, 0, 0, 0, 0, 0, 0, 1056, 1046, 1012, 999, 1043, 962, 1054, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1068, 1012, 1040, 1035, 994, 1013, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1063, 1014, 1008, 1002, 1024, 1003, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1054, 1008, 1040, 996, 1002, 1038, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 997, 1029, 1019, 978, 1002, 1003, 1000, 14, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8158, 8206, 8248, 8257, 8199, 8218, 8096] avg=8192.0000 max=8257 max_rank=4 min=8096 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8064, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8064, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1026, 1031, 1047, 1025, 1038, 1007, 1003, 1015, 986, 1022, 1025, 1043, 1006, 1038, 1054, 1022, 1044, 1033, 1036, 1036, 1053, 1006, 1058, 1014, 1045, 1028, 959, 1033, 1004, 1041, 1024, 999, 1014, 989, 1028, 998, 1008, 1020, 1000, 1059, 1026, 1012, 1016, 1026, 1036, 1066, 1052, 1007, 1027, 1013, 1043, 980, 1016, 1016, 1054, 1020, 1021, 1060, 1033, 1016, 1048, 1013, 988, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[987, 1039, 1039, 1019, 1025, 1059, 994, 1055, 1030, 1001, 992, 1029, 998, 1053, 1027, 1069, 1046, 1026, 1005, 1039, 1018, 1055, 1024, 1024, 993, 1027, 1027, 1026, 1035, 998, 1055, 1007, 1045, 1015, 1057, 1046, 1003, 1029, 1027, 1019, 1025, 1018, 1023, 1026, 1016, 1011, 1024, 1029, 1005, 1065, 1031, 1014, 1027, 1019, 1015, 1010, 1029, 1027, 994, 1045, 975, 1004, 986, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[998, 1056, 1106, 1035, 1005, 1045, 1017, 991, 994, 1020, 1053, 939, 1028, 1003, 1036, 1015, 1014, 1025, 1030, 1044, 994, 1054, 1018, 1002, 1010, 1080, 1001, 990, 1002, 1062, 1043, 1000, 1037, 1020, 1003, 1008, 1060, 998, 1027, 1010, 1045, 1055, 1081, 1000, 1095, 1013, 1081, 1039, 1034, 1067, 1016, 1027, 987, 989, 1001, 1054, 1035, 992, 1036, 1001, 980, 1002, 1002, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[993, 974, 999, 1009, 1008, 1026, 1026, 1071, 1026, 1085, 1031, 1018, 1055, 1040, 992, 1035, 990, 1031, 1015, 1038, 1019, 989, 1027, 1077, 1007, 1033, 973, 1014, 1044, 1013, 1011, 1046, 1036, 974, 1074, 1025, 1018, 1042, 1014, 1012, 1037, 1042, 1017, 1016, 1002, 1020, 1029, 986, 1008, 1084, 1039, 1004, 1050, 1024, 1045, 1001, 1009, 1035, 997, 1037, 1077, 1036, 1025, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[998, 1017, 1035, 1041, 1062, 999, 991, 1026, 976, 1030, 1007, 1073, 1018, 1050, 1038, 1008, 1001, 987, 989, 1043, 1035, 981, 1009, 1048, 1017, 1010, 1059, 1013, 1046, 1090, 1043, 1019, 1037, 1021, 1013, 1021, 990, 1038, 1069, 1024, 1025, 1063, 989, 992, 1013, 971, 1036, 1020, 1017, 1041, 1038, 1009, 1065, 1020, 1016, 1019, 1006, 1011, 1035, 1028, 1018, 1056, 1041, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1014, 1031, 1000, 995, 1014, 1033, 1038, 1012, 1095, 1065, 997, 1005, 1011, 1042, 962, 1015, 1015, 1027, 988, 987, 1018, 1030, 1007, 1012, 1016, 1028, 1055, 1036, 1028, 1046, 1052, 1024, 966, 1055, 1051, 1013, 1075, 1020, 1002, 1048, 1033, 1052, 999, 1040, 1009, 992, 1047, 1045, 999, 1012, 1016, 998, 1058, 1005, 1081, 994, 1065, 1033, 1071, 984, 1037, 1037, 981, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1003, 1044, 1030, 988, 1035, 1027, 988, 1055, 997, 1009, 1038, 1052, 1029, 1012, 1011, 1075, 985, 1038, 1008, 1014, 991, 1027, 1048, 1071, 1073, 1024, 996, 1041, 1052, 1009, 1029, 1043, 1047, 1033, 984, 1039, 1016, 1041, 1031, 1008, 1010, 1042, 1001, 1061, 987, 1010, 1013, 1052, 1044, 1030, 1017, 980, 1017, 1010, 969, 1003, 1007, 1004, 1057, 1012, 1015, 1060, 1073, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1051, 1013, 1051, 1102, 1011, 1008, 1032, 1085, 1009, 1050, 1056, 1008, 1008, 1078, 1016, 1051, 1074, 1053, 1034, 1020, 1026, 982, 1046, 1030, 1028, 993, 1036, 1021, 1036, 1070, 1035, 1009, 1002, 995, 1006, 1075, 991, 997, 1004, 992, 1057, 992, 1038, 1013, 1049, 973, 978, 986, 1019, 1025, 1002, 1050, 1047, 972, 1062, 1025, 1036, 1018, 973, 991, 1034, 978, 1009, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[987, 1039, 1039, 1019, 1025, 1059, 994, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1001, 992, 1029, 998, 1039, 1027, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1026, 1005, 1039, 1018, 1055, 1024, 1024, 2, 0, 0, 0, 0, 0, 0, 0, 993, 1027, 1027, 1026, 1035, 979, 1055, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1015, 1057, 1046, 1003, 1029, 1027, 1019, 10, 3, 0, 0, 0, 0, 0, 0, 1025, 1016, 1023, 1026, 1016, 1011, 1024, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1065, 1031, 1014, 1027, 1019, 1015, 1010, 7, 4, 0, 0, 0, 0, 0, 0, 1029, 1027, 994, 1045, 975, 1004, 986, 1056, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8185, 8239, 8149, 8254, 8170, 8197, 8132] avg=8192.0000 max=8254 max_rank=4 min=8132 min_rank=7 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1026, 1031, 1047, 1025, 1038, 1007, 1003, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1022, 1025, 1043, 1006, 1024, 1054, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1033, 1036, 1036, 1053, 1006, 1058, 1014, 2, 0, 0, 0, 0, 0, 0, 0, 1045, 1028, 959, 1033, 1004, 1021, 1024, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 989, 1028, 998, 1008, 1020, 1000, 1059, 10, 3, 0, 0, 0, 0, 0, 0, 1026, 1010, 1016, 1026, 1036, 1066, 1052, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1013, 1043, 980, 1016, 1016, 1054, 1020, 6, 4, 0, 0, 0, 0, 0, 0, 1021, 1060, 1033, 1016, 1048, 1013, 988, 1030, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8182, 8282, 8113, 8129, 8239, 8179, 8226] avg=8192.0000 max=8282 max_rank=2 min=8113 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[993, 974, 999, 1009, 1008, 1026, 1026, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1085, 1031, 1018, 1055, 1026, 992, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1031, 1015, 1038, 1019, 989, 1027, 1077, 2, 0, 0, 0, 0, 0, 0, 0, 1007, 1033, 973, 1014, 1044, 994, 1011, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 974, 1074, 1025, 1018, 1042, 1014, 1012, 10, 3, 0, 0, 0, 0, 0, 0, 1037, 1040, 1017, 1016, 1002, 1020, 1029, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1084, 1039, 1004, 1050, 1024, 1045, 1001, 7, 4, 0, 0, 0, 0, 0, 0, 1009, 1035, 997, 1037, 1077, 1036, 1025, 1006, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8268, 8188, 8122, 8208, 8147, 8266, 8238] avg=8192.0000 max=8268 max_rank=1 min=8099 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[998, 1017, 1035, 1041, 1062, 999, 991, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1030, 1007, 1073, 1018, 1036, 1038, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 987, 989, 1043, 1035, 981, 1009, 1048, 2, 0, 0, 0, 0, 0, 0, 0, 1017, 1010, 1059, 1013, 1046, 1068, 1043, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1021, 1013, 1021, 990, 1038, 1069, 1024, 10, 4, 0, 0, 0, 0, 0, 0, 1025, 1061, 989, 992, 1013, 971, 1036, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1041, 1038, 1009, 1065, 1020, 1016, 1019, 6, 4, 0, 0, 0, 0, 0, 0, 1006, 1011, 1035, 1028, 1018, 1056, 1041, 1035, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8186, 8095, 8275, 8227, 8107, 8235, 8248] avg=8192.0000 max=8275 max_rank=3 min=8095 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[998, 1056, 1106, 1035, 1005, 1045, 1017, 985, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1020, 1053, 939, 1028, 989, 1036, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1025, 1030, 1044, 994, 1054, 1018, 1002, 2, 0, 0, 0, 0, 0, 0, 0, 1010, 1080, 1001, 990, 1002, 1041, 1043, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1020, 1003, 1008, 1060, 998, 1027, 1010, 10, 4, 0, 0, 0, 0, 0, 0, 1045, 1053, 1081, 1000, 1095, 1013, 1081, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1067, 1016, 1027, 987, 989, 1001, 1054, 6, 4, 0, 0, 0, 0, 0, 0, 1035, 992, 1036, 1001, 980, 1002, 1002, 1031, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8074, 8183, 8167, 8177, 8407, 8185, 8096] avg=8192.0000 max=8407 max_rank=5 min=8074 min_rank=1 imbalance_ratio=1.026245 max_load_reduction=57129 max_load_reduction_pct=0.871719 moved_to_echo=32865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1003, 1044, 1030, 988, 1035, 1027, 988, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1009, 1038, 1052, 1029, 998, 1011, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1038, 1008, 1014, 991, 1027, 1048, 1071, 2, 0, 0, 0, 0, 0, 0, 0, 1073, 1024, 996, 1041, 1052, 990, 1029, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1033, 984, 1039, 1016, 1041, 1031, 1008, 10, 3, 0, 0, 0, 0, 0, 0, 1010, 1040, 1001, 1061, 987, 1010, 1013, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1030, 1017, 980, 1017, 1010, 969, 1003, 7, 4, 0, 0, 0, 0, 0, 0, 1007, 1004, 1057, 1012, 1015, 1060, 1073, 1021, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8209, 8184, 8248, 8212, 8174, 8081, 8265] avg=8192.0000 max=8265 max_rank=7 min=8081 min_rank=6 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1014, 1031, 1000, 995, 1014, 1033, 1038, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 1065, 997, 1005, 1011, 1021, 962, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1027, 988, 987, 1018, 1030, 1007, 1012, 9, 5, 0, 0, 0, 0, 0, 0, 1016, 1028, 1055, 1036, 1028, 1016, 1052, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1055, 1051, 1013, 1075, 1020, 1002, 1048, 14, 8, 0, 0, 0, 0, 0, 0, 1033, 1043, 999, 1040, 1009, 992, 1047, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1012, 1016, 998, 1058, 1005, 1081, 994, 10, 7, 0, 0, 0, 0, 0, 0, 1065, 1033, 1071, 984, 1037, 1037, 981, 1020, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8171, 8098, 8255, 8252, 8208, 8180, 8250] avg=8192.0000 max=8255 max_rank=3 min=8098 min_rank=2 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1051, 1013, 1051, 1102, 1011, 1008, 1032, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1050, 1056, 1008, 1008, 1063, 1016, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1053, 1034, 1020, 1026, 982, 1046, 1030, 2, 0, 0, 0, 0, 0, 0, 0, 1028, 993, 1036, 1021, 1036, 1049, 1035, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 995, 1006, 1075, 991, 997, 1004, 992, 11, 4, 0, 0, 0, 0, 0, 0, 1057, 990, 1038, 1013, 1049, 973, 978, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1025, 1002, 1050, 1047, 972, 1062, 1025, 7, 4, 0, 0, 0, 0, 0, 0, 1036, 1018, 973, 991, 1034, 978, 1009, 1025, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8346, 8261, 8267, 8207, 8077, 8084, 8213, 8081] avg=8192.0000 max=8346 max_rank=0 min=8077 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8064, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8320, 8064, 8320, 8064, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[999, 949, 1050, 1014, 1027, 1033, 1014, 1002, 1016, 1021, 1064, 1000, 1013, 1028, 1005, 1022, 1039, 1054, 1006, 1041, 1026, 1036, 1007, 998, 1035, 1033, 1034, 1022, 1016, 962, 1013, 1002, 1078, 1054, 1050, 1013, 1043, 1017, 1019, 1030, 1030, 1050, 1062, 1036, 1007, 1022, 995, 1025, 1062, 1042, 1031, 1052, 987, 1012, 982, 1040, 964, 1051, 1031, 1022, 1055, 992, 1073, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[999, 972, 1018, 1021, 1086, 1041, 1039, 1001, 1005, 983, 1023, 1019, 993, 1050, 1034, 1024, 1014, 1017, 1048, 1041, 1060, 1043, 1022, 1020, 1038, 1042, 1024, 1000, 1000, 1026, 1013, 1007, 998, 1051, 1032, 1032, 1044, 1024, 1073, 977, 1042, 1012, 998, 997, 1039, 1012, 1073, 1008, 1029, 1011, 995, 987, 1031, 1045, 1033, 1035, 1063, 1009, 1030, 1032, 1034, 1002, 1056, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1023, 1082, 1000, 1067, 1036, 1015, 1023, 1008, 986, 996, 1039, 1021, 1056, 1061, 1063, 1057, 965, 1012, 1053, 1017, 1033, 1009, 1002, 1006, 995, 1016, 1040, 1032, 1021, 1008, 1040, 977, 1019, 1005, 1019, 1036, 1002, 1069, 1063, 991, 992, 1047, 1045, 1040, 1026, 1044, 1064, 998, 1018, 1037, 1031, 1022, 988, 996, 1018, 995, 1047, 1016, 1003, 999, 1049, 1038, 1033, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1002, 995, 1049, 1050, 1015, 1014, 1033, 1077, 1053, 1023, 1041, 1011, 1058, 988, 985, 1040, 1018, 1017, 1027, 1030, 1026, 1031, 1000, 995, 1037, 1067, 1014, 1025, 1025, 1023, 981, 1045, 959, 1042, 1058, 1062, 1035, 1031, 1018, 1036, 1017, 940, 1046, 1036, 1029, 1024, 1084, 1038, 1031, 1031, 1028, 998, 1033, 1025, 1076, 955, 1027, 967, 1027, 1005, 1009, 1041, 1012, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1025, 1034, 1023, 992, 1006, 989, 1002, 992, 1031, 1015, 1010, 1038, 1001, 1046, 1061, 966, 1034, 1072, 1002, 1059, 971, 1040, 989, 1005, 1008, 1053, 1024, 1057, 1054, 1054, 1017, 1015, 1023, 1036, 1049, 1040, 1025, 1007, 992, 1050, 1022, 977, 1004, 1021, 1040, 1029, 1025, 1022, 1048, 1065, 1015, 1029, 1001, 1021, 1030, 1011, 1027, 1035, 1070, 1011, 1021, 1041, 999, 1065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1047, 1039, 1035, 1037, 1042, 1042, 1030, 997, 982, 1007, 1020, 995, 1020, 1018, 1072, 1016, 1035, 992, 1013, 1067, 1052, 986, 1005, 960, 995, 971, 972, 1031, 988, 1033, 1026, 1031, 1056, 1087, 1038, 1024, 1046, 1001, 1039, 1034, 1044, 1044, 1039, 1025, 1004, 1042, 1027, 1017, 1031, 997, 1018, 1062, 1059, 1034, 1064, 1025, 1037, 996, 1002, 1060, 1016, 981, 1012, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[999, 1055, 1004, 1050, 1050, 1031, 1047, 1038, 1011, 1076, 1050, 1004, 1014, 1017, 1013, 1029, 1015, 1038, 990, 1034, 1064, 1035, 1028, 1047, 1028, 1086, 1034, 985, 1010, 1069, 1053, 1003, 979, 1017, 1021, 1016, 1014, 1031, 973, 1034, 1030, 982, 985, 1042, 1014, 996, 1023, 1023, 1057, 978, 982, 1023, 1014, 1054, 1026, 1005, 1038, 971, 1036, 1060, 990, 1008, 1082, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1012, 995, 1078, 1071, 1006, 993, 1001, 1020, 970, 1062, 1018, 1047, 1025, 1030, 1045, 1073, 1025, 1037, 1025, 1073, 1019, 983, 1045, 1065, 1045, 1064, 989, 1018, 1020, 1025, 960, 987, 998, 1032, 1054, 970, 1038, 961, 1058, 993, 1042, 1061, 1044, 1008, 1029, 1011, 1031, 1008, 963, 996, 1029, 1003, 1022, 1024, 1064, 1040, 1053, 1019, 1022, 1021, 1022, 1029, 1050, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[999, 949, 1050, 1014, 1027, 1033, 1014, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1021, 1064, 1000, 1013, 1028, 1003, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1054, 1006, 1041, 1026, 1036, 1007, 998, 2, 0, 0, 0, 0, 0, 0, 0, 1035, 1033, 1034, 1022, 1016, 962, 1013, 1002, 22, 4, 0, 0, 0, 0, 0, 0, 1078, 1032, 1050, 1013, 1043, 1017, 1019, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1050, 1062, 1036, 1007, 1022, 989, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1042, 1031, 1052, 987, 1012, 982, 1040, 6, 3, 0, 0, 0, 0, 0, 0, 964, 1051, 1031, 1022, 1055, 992, 1066, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8088, 8167, 8209, 8143, 8282, 8221, 8217, 8209] avg=8192.0000 max=8282 max_rank=4 min=8088 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32929 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[999, 972, 1018, 1021, 1086, 1041, 1039, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 983, 1023, 1019, 993, 1050, 1032, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1017, 1048, 1041, 1060, 1043, 1022, 1020, 2, 0, 0, 0, 0, 0, 0, 0, 1038, 1042, 1024, 1000, 1000, 1026, 1013, 1007, 22, 4, 0, 0, 0, 0, 0, 0, 998, 1029, 1032, 1032, 1044, 1024, 1073, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1012, 998, 997, 1039, 1012, 1067, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1011, 995, 987, 1031, 1045, 1033, 1035, 6, 3, 0, 0, 0, 0, 0, 0, 1063, 1009, 1030, 1032, 1034, 1002, 1049, 1009, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8129, 8267, 8176, 8209, 8175, 8175, 8228] avg=8192.0000 max=8267 max_rank=2 min=8129 min_rank=1 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32787 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1002, 995, 1049, 1050, 1015, 1014, 1033, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1023, 1041, 1011, 1058, 988, 983, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1017, 1027, 1030, 1026, 1031, 1000, 995, 2, 0, 0, 0, 0, 0, 0, 0, 1037, 1067, 1014, 1025, 1025, 1023, 981, 1045, 22, 4, 0, 0, 0, 0, 0, 0, 959, 1020, 1058, 1062, 1035, 1031, 1018, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 940, 1046, 1036, 1029, 1024, 1078, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1031, 1028, 998, 1033, 1025, 1076, 955, 6, 3, 0, 0, 0, 0, 0, 0, 1027, 967, 1027, 1005, 1009, 1041, 1005, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8197, 8146, 8243, 8219, 8208, 8186, 8102] avg=8192.0000 max=8243 max_rank=3 min=8102 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32715 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1012, 995, 1078, 1071, 1006, 993, 1001, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1062, 1018, 1047, 1025, 1030, 1043, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1037, 1025, 1073, 1019, 983, 1045, 1065, 2, 0, 0, 0, 0, 0, 0, 0, 1045, 1064, 989, 1018, 1020, 1025, 960, 987, 22, 4, 0, 0, 0, 0, 0, 0, 998, 1010, 1054, 970, 1038, 961, 1058, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1061, 1044, 1008, 1029, 1011, 1025, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 963, 996, 1029, 1003, 1022, 1024, 1064, 1040, 6, 3, 0, 0, 0, 0, 0, 0, 1053, 1019, 1022, 1021, 1022, 1029, 1043, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8268, 8274, 8134, 8082, 8228, 8150, 8224] avg=8192.0000 max=8274 max_rank=2 min=8082 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32684 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 1034, 1023, 992, 1006, 989, 1002, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1015, 1010, 1038, 1001, 1046, 1051, 966, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1072, 1002, 1059, 971, 1040, 989, 1005, 4, 0, 0, 0, 0, 0, 0, 0, 1008, 1053, 1024, 1057, 1054, 1054, 1017, 1015, 27, 10, 0, 0, 0, 0, 0, 0, 1023, 1009, 1049, 1040, 1025, 1007, 992, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 977, 1004, 1021, 1040, 1029, 1014, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1065, 1015, 1029, 1001, 1021, 1030, 1011, 11, 9, 6, 0, 0, 0, 0, 0, 1027, 1035, 1070, 1011, 1021, 1041, 980, 1065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8063, 8158, 8176, 8319, 8195, 8129, 8246, 8250] avg=8192.0000 max=8319 max_rank=3 min=8063 min_rank=0 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32820 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1047, 1039, 1035, 1037, 1042, 1042, 1030, 997, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1007, 1020, 995, 1020, 1018, 1070, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 992, 1013, 1067, 1052, 986, 1005, 960, 2, 0, 0, 0, 0, 0, 0, 0, 995, 971, 972, 1031, 988, 1033, 1026, 1031, 23, 4, 0, 0, 0, 0, 0, 0, 1056, 1064, 1038, 1024, 1046, 1001, 1039, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1044, 1039, 1025, 1004, 1042, 1021, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 997, 1018, 1062, 1059, 1034, 1064, 1025, 6, 3, 0, 0, 0, 0, 0, 0, 1037, 996, 1002, 1060, 1016, 981, 1005, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8128, 8112, 8074, 8302, 8236, 8299, 8116] avg=8192.0000 max=8302 max_rank=4 min=8074 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32953 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[999, 1055, 1004, 1050, 1050, 1031, 1047, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1076, 1050, 1004, 1014, 1017, 1011, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1038, 990, 1034, 1064, 1035, 1028, 1047, 2, 0, 0, 0, 0, 0, 0, 0, 1028, 1086, 1034, 985, 1010, 1069, 1053, 1003, 21, 4, 0, 0, 0, 0, 0, 0, 979, 996, 1021, 1016, 1014, 1031, 973, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 982, 985, 1042, 1014, 996, 1017, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 978, 982, 1023, 1014, 1054, 1026, 1005, 6, 4, 0, 0, 0, 0, 0, 0, 1038, 971, 1036, 1060, 990, 1008, 1074, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8212, 8253, 8293, 8064, 8089, 8149, 8202] avg=8192.0000 max=8293 max_rank=3 min=8064 min_rank=4 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32504 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1023, 1082, 1000, 1067, 1036, 1015, 1023, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 986, 996, 1039, 1021, 1056, 1061, 1061, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 965, 1012, 1053, 1017, 1033, 1009, 1002, 1006, 2, 0, 0, 0, 0, 0, 0, 0, 995, 1016, 1040, 1032, 1021, 1008, 1040, 977, 21, 4, 0, 0, 0, 0, 0, 0, 1019, 984, 1019, 1036, 1002, 1069, 1063, 991, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1047, 1045, 1040, 1026, 1044, 1058, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1037, 1031, 1022, 988, 996, 1018, 995, 6, 3, 0, 0, 0, 0, 0, 0, 1047, 1016, 1003, 999, 1049, 1038, 1026, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8277, 8099, 8154, 8183, 8250, 8114, 8205] avg=8192.0000 max=8277 max_rank=1 min=8099 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32752 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8448, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8448, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8448, 8192, 8192, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1061, 1015, 995, 1042, 1017, 973, 1010, 1022, 975, 1031, 1080, 981, 1069, 1020, 1051, 1034, 1057, 1065, 1008, 1033, 1017, 1000, 1025, 1029, 999, 1006, 1010, 971, 989, 1008, 1045, 1044, 1078, 1034, 1020, 1015, 1030, 1033, 1030, 1038, 1018, 1012, 1055, 986, 1030, 1005, 1050, 1032, 1043, 973, 1034, 981, 1046, 1087, 1020, 999, 989, 1035, 1048, 1003, 1045, 1006, 1055, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1001, 997, 1033, 1061, 1077, 987, 1045, 1045, 1012, 1044, 1076, 1031, 1008, 1067, 1015, 1001, 1001, 1056, 1031, 967, 992, 1016, 1024, 1044, 974, 1000, 1052, 1004, 989, 1049, 1040, 1010, 1002, 1054, 996, 1033, 1004, 1018, 1008, 1053, 997, 991, 1078, 972, 1023, 1031, 1036, 1009, 998, 1090, 982, 1058, 1063, 1053, 1018, 1079, 1030, 992, 994, 1044, 1014, 980, 1032, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1010, 1008, 1071, 1033, 1009, 1038, 1044, 994, 1038, 1049, 1035, 1043, 1004, 1021, 1010, 1010, 1008, 1021, 1027, 1033, 962, 1015, 1021, 1022, 1064, 960, 1018, 1045, 991, 1030, 994, 1015, 976, 1016, 1027, 1027, 1025, 1029, 1042, 1055, 1026, 1025, 1047, 1023, 1000, 1026, 1038, 1061, 1051, 1033, 1003, 977, 1101, 1045, 1019, 1023, 998, 1047, 1060, 998, 1074, 1009, 1017, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1047, 982, 1028, 1060, 1014, 1051, 1014, 1000, 1013, 1020, 994, 1010, 1060, 993, 984, 1026, 980, 982, 1042, 1048, 1036, 986, 1028, 1030, 1045, 1031, 1018, 1031, 1003, 1056, 1014, 990, 1031, 1013, 974, 1061, 947, 1029, 996, 1053, 1055, 964, 1049, 1059, 984, 1058, 1028, 1049, 1077, 1052, 975, 1019, 1082, 1025, 1048, 1005, 1061, 1072, 1020, 1012, 999, 1047, 1027, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1089, 1060, 979, 997, 1087, 1035, 1039, 1010, 1046, 1027, 1036, 1033, 1014, 1018, 990, 1031, 1047, 978, 993, 1077, 1022, 1076, 994, 1026, 1048, 1032, 1026, 974, 1037, 1052, 1049, 1035, 980, 1014, 1028, 1024, 1020, 1027, 981, 1044, 1056, 1021, 1004, 988, 977, 1035, 1033, 1013, 1061, 1059, 993, 1028, 997, 1044, 986, 987, 1062, 1020, 1027, 1074, 1007, 986, 1011, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1012, 1053, 1013, 1050, 1015, 992, 1067, 968, 1029, 1021, 1007, 1028, 1045, 1025, 1047, 1040, 1083, 1001, 1055, 1037, 1019, 1033, 1031, 986, 1023, 1041, 1047, 1043, 1036, 1038, 1034, 1046, 1041, 1045, 1041, 1043, 1019, 1014, 974, 1015, 965, 1028, 1018, 1024, 1068, 997, 1030, 999, 1027, 1051, 1003, 1015, 1004, 1018, 1019, 1006, 1059, 983, 992, 1080, 961, 997, 1014, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1040, 1054, 1006, 1012, 1042, 1016, 1050, 999, 987, 998, 965, 1017, 1014, 1107, 1033, 1041, 1050, 999, 1043, 1042, 1060, 977, 955, 1033, 1032, 1049, 1076, 997, 1026, 996, 985, 1021, 991, 1065, 997, 1020, 1015, 1060, 1026, 1066, 1015, 1032, 979, 1039, 1033, 1048, 1039, 1035, 1003, 993, 1054, 1023, 1037, 998, 1050, 1032, 1000, 1033, 992, 996, 1050, 1001, 1035, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1042, 979, 995, 1038, 1039, 1041, 978, 1000, 1025, 1019, 1062, 1002, 1020, 1006, 1036, 1017, 1023, 1062, 1000, 999, 1018, 1069, 1036, 1077, 1005, 1038, 1027, 985, 1034, 993, 1035, 998, 1058, 1050, 1015, 989, 1031, 1017, 1002, 1063, 962, 1024, 1034, 1008, 1015, 1014, 1005, 1026, 1027, 1041, 1032, 1043, 995, 1035, 1057, 1023, 1082, 1031, 1040, 981, 1031, 1054, 1004, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[996, 997, 1033, 1061, 1077, 987, 1045, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1044, 1076, 1031, 1008, 1060, 1015, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1056, 1031, 967, 992, 1016, 1024, 1044, 3, 0, 0, 0, 0, 0, 0, 0, 974, 1000, 1052, 1004, 989, 1049, 1040, 1010, 23, 0, 0, 0, 0, 0, 0, 0, 1002, 1054, 996, 1033, 1004, 1018, 1008, 1053, 1, 0, 0, 0, 0, 0, 0, 0, 997, 991, 1078, 972, 1023, 1031, 1036, 1009, 10, 7, 2, 0, 0, 0, 0, 0, 998, 1090, 982, 1058, 1030, 1053, 1018, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 992, 994, 1044, 1014, 980, 1032, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8247, 8134, 8141, 8169, 8156, 8308, 8140] avg=8192.0000 max=8308 max_rank=6 min=8134 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1056, 1015, 995, 1042, 1017, 973, 1010, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1031, 1080, 981, 1069, 1014, 1051, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1065, 1008, 1033, 1017, 1000, 1025, 1029, 3, 0, 0, 0, 0, 0, 0, 0, 999, 1006, 1010, 971, 989, 1008, 1045, 1044, 22, 0, 0, 0, 0, 0, 0, 0, 1078, 1034, 1020, 1015, 1030, 1033, 1030, 1038, 1, 0, 0, 0, 0, 0, 0, 0, 1018, 1012, 1055, 986, 1030, 1005, 1050, 1032, 10, 6, 2, 0, 0, 0, 0, 0, 1043, 973, 1034, 981, 1014, 1087, 1020, 999, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1035, 1048, 1003, 1045, 1006, 1055, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8235, 8237, 8094, 8279, 8206, 8151, 8204] avg=8192.0000 max=8279 max_rank=4 min=8094 min_rank=3 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1042, 982, 1028, 1060, 1014, 1051, 1014, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1020, 994, 1010, 1060, 987, 984, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 980, 982, 1042, 1048, 1036, 986, 1028, 1030, 3, 0, 0, 0, 0, 0, 0, 0, 1045, 1031, 1018, 1031, 1003, 1056, 1014, 990, 23, 0, 0, 0, 0, 0, 0, 0, 1031, 1013, 974, 1061, 947, 1029, 996, 1053, 1, 0, 0, 0, 0, 0, 0, 0, 1055, 964, 1049, 1059, 984, 1058, 1028, 1049, 10, 6, 2, 0, 0, 0, 0, 0, 1077, 1052, 975, 1019, 1049, 1025, 1048, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1072, 1020, 1012, 999, 1047, 1027, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8094, 8135, 8211, 8105, 8264, 8250, 8286] avg=8192.0000 max=8286 max_rank=7 min=8094 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32905 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1035, 1054, 1006, 1012, 1042, 1016, 1050, 999, 0, 0, 0, 0, 0, 0, 0, 0, 987, 998, 965, 1017, 1014, 1100, 1033, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 999, 1043, 1042, 1060, 977, 955, 1033, 3, 0, 0, 0, 0, 0, 0, 0, 1032, 1049, 1076, 997, 1026, 996, 985, 1021, 22, 0, 0, 0, 0, 0, 0, 0, 991, 1065, 997, 1020, 1015, 1060, 1026, 1066, 1, 0, 0, 0, 0, 0, 0, 0, 1015, 1032, 979, 1039, 1033, 1048, 1039, 1035, 10, 7, 2, 0, 0, 0, 0, 0, 1003, 993, 1054, 1023, 1005, 998, 1050, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1033, 992, 996, 1050, 1001, 1035, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8155, 8162, 8204, 8241, 8239, 8158, 8163] avg=8192.0000 max=8241 max_rank=4 min=8155 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1007, 1053, 1013, 1050, 1015, 992, 1067, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1021, 1007, 1028, 1045, 1019, 1047, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 1001, 1055, 1037, 1019, 1033, 1031, 986, 3, 0, 0, 0, 0, 0, 0, 0, 1023, 1041, 1047, 1043, 1036, 1038, 1034, 1046, 22, 0, 0, 0, 0, 0, 0, 0, 1041, 1045, 1041, 1043, 1019, 1014, 974, 1015, 1, 0, 0, 0, 0, 0, 0, 0, 965, 1028, 1018, 1024, 1068, 997, 1030, 999, 9, 6, 2, 0, 0, 0, 0, 0, 1027, 1051, 1003, 1015, 973, 1018, 1019, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 983, 992, 1080, 961, 997, 1014, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8236, 8248, 8330, 8193, 8146, 8112, 8106] avg=8192.0000 max=8330 max_rank=3 min=8106 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32557 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1079, 1060, 979, 997, 1087, 1035, 1039, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1027, 1036, 1033, 1014, 1007, 990, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 978, 993, 1077, 1022, 1076, 994, 1026, 5, 3, 0, 0, 0, 0, 0, 0, 1048, 1032, 1026, 974, 1037, 1052, 1049, 1035, 26, 0, 0, 0, 0, 0, 0, 0, 980, 1014, 1028, 1024, 1020, 1027, 981, 1044, 7, 0, 0, 0, 0, 0, 0, 0, 1056, 1021, 1004, 988, 977, 1035, 1033, 1013, 13, 11, 5, 0, 0, 0, 0, 0, 1061, 1059, 993, 1028, 958, 1044, 986, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1020, 1027, 1074, 1007, 986, 1011, 992, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8184, 8221, 8279, 8125, 8156, 8116, 8169] avg=8192.0000 max=8286 max_rank=0 min=8116 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1005, 1008, 1071, 1033, 1009, 1038, 1044, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1049, 1035, 1043, 1004, 1015, 1010, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1021, 1027, 1033, 962, 1015, 1021, 1022, 3, 0, 0, 0, 0, 0, 0, 0, 1064, 960, 1018, 1045, 991, 1030, 994, 1015, 24, 0, 0, 0, 0, 0, 0, 0, 976, 1016, 1027, 1027, 1025, 1029, 1042, 1055, 1, 0, 0, 0, 0, 0, 0, 0, 1026, 1025, 1047, 1023, 1000, 1026, 1038, 1061, 10, 6, 2, 0, 0, 0, 0, 0, 1051, 1033, 1003, 977, 1067, 1045, 1019, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1047, 1060, 998, 1074, 1009, 1017, 995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8204, 8112, 8141, 8198, 8264, 8218, 8197] avg=8192.0000 max=8264 max_rank=5 min=8112 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1037, 979, 995, 1038, 1039, 1041, 978, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1019, 1062, 1002, 1020, 1000, 1036, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1062, 1000, 999, 1018, 1069, 1036, 1077, 3, 0, 0, 0, 0, 0, 0, 0, 1005, 1038, 1027, 985, 1034, 993, 1035, 998, 21, 0, 0, 0, 0, 0, 0, 0, 1058, 1050, 1015, 989, 1031, 1017, 1002, 1063, 1, 0, 0, 0, 0, 0, 0, 0, 962, 1024, 1034, 1008, 1015, 1014, 1005, 1026, 9, 6, 2, 0, 0, 0, 0, 0, 1027, 1041, 1032, 1043, 965, 1035, 1057, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1031, 1040, 981, 1031, 1054, 1004, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8181, 8287, 8136, 8226, 8105, 8223, 8271] avg=8192.0000 max=8287 max_rank=2 min=8105 min_rank=5 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8064, 8192, 8320, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8064, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8064, 8448, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1000, 1060, 1011, 1031, 942, 1029, 1025, 1018, 1047, 1016, 997, 1010, 1053, 1032, 1037, 998, 1049, 1026, 992, 1018, 1027, 1034, 1048, 1071, 1005, 1058, 1039, 1034, 1064, 1050, 1012, 984, 1031, 1032, 1054, 981, 996, 1005, 1067, 1008, 981, 1035, 1055, 1022, 1011, 995, 1012, 1007, 1031, 1030, 1021, 1057, 1056, 1046, 1000, 1008, 1026, 1002, 1049, 992, 982, 1055, 1040, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[974, 1031, 1057, 989, 972, 1051, 993, 1022, 1088, 1008, 1030, 977, 1025, 1040, 1021, 1057, 1039, 995, 1064, 1027, 1031, 1079, 1015, 1066, 991, 1011, 1057, 1007, 1061, 1069, 1000, 1062, 961, 995, 1067, 983, 1055, 1058, 981, 1050, 1050, 1051, 1026, 1033, 1032, 1000, 999, 1014, 998, 1021, 1043, 1054, 990, 1081, 995, 1022, 1027, 1032, 983, 1045, 1000, 1010, 979, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1023, 1033, 987, 987, 1005, 1015, 1066, 1014, 1039, 982, 1008, 1013, 1015, 1003, 1053, 1000, 1012, 1022, 1017, 995, 997, 1002, 1039, 1025, 1030, 1023, 1007, 1058, 1034, 1011, 1072, 1038, 1052, 999, 976, 1037, 1055, 1071, 1015, 1061, 1080, 1064, 996, 997, 1045, 1048, 1043, 978, 1045, 1012, 1050, 1029, 1030, 1056, 1064, 1012, 998, 975, 1018, 999, 1057, 1038, 978, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1039, 1050, 1011, 1003, 1029, 989, 1070, 982, 1058, 1005, 1051, 1006, 1010, 1024, 1060, 1056, 1007, 965, 1022, 1058, 1069, 1007, 970, 1023, 1045, 999, 987, 1016, 1010, 1064, 1050, 1039, 1053, 1059, 1047, 1003, 1017, 983, 1011, 1033, 1007, 1036, 1085, 1030, 1068, 1022, 1060, 1010, 1008, 1068, 1037, 974, 1018, 1001, 1026, 1018, 1041, 957, 1005, 990, 997, 1031, 1026, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1024, 1045, 1043, 1021, 1015, 994, 999, 958, 1048, 1031, 1061, 1028, 1030, 996, 1056, 1053, 1039, 1007, 1050, 998, 1046, 1007, 992, 1045, 1038, 1032, 991, 1068, 1039, 1015, 1028, 1026, 1057, 1019, 1036, 951, 1025, 998, 1014, 969, 1057, 980, 985, 1018, 1042, 1043, 1054, 1017, 1041, 984, 1087, 1051, 1000, 1055, 1071, 1017, 988, 1006, 1033, 1035, 1010, 983, 1081, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1002, 1040, 1079, 1040, 1040, 999, 1031, 1037, 988, 1000, 1035, 1013, 1091, 1000, 1006, 1004, 989, 1053, 1044, 1021, 969, 1071, 1020, 1045, 1010, 1006, 1015, 1029, 1032, 982, 985, 1006, 1048, 1025, 1077, 1002, 1005, 1023, 1000, 969, 980, 1036, 1076, 1029, 1105, 1047, 1029, 1048, 1028, 1068, 1040, 1015, 1073, 968, 1002, 1043, 990, 1008, 1046, 1011, 995, 1033, 1011, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1027, 1006, 1022, 1052, 1005, 1053, 1022, 1065, 1010, 1019, 1030, 1033, 1064, 984, 976, 1004, 1057, 1073, 964, 1005, 1054, 1031, 1044, 1096, 983, 1023, 1038, 1046, 1040, 1033, 985, 970, 1013, 1019, 990, 1027, 1018, 1013, 1018, 1020, 1065, 1032, 1032, 983, 1038, 1050, 1054, 1018, 992, 1025, 996, 970, 1016, 1015, 1057, 1038, 1058, 1084, 984, 1021, 1036, 973, 1053, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[970, 1011, 1017, 1009, 1049, 1045, 1041, 999, 1054, 1031, 1065, 1056, 1058, 1041, 990, 1024, 993, 1004, 1002, 1045, 1053, 1007, 995, 992, 989, 1071, 1044, 983, 1051, 1008, 1002, 1041, 1042, 1019, 1013, 1003, 1033, 1026, 1008, 1020, 1029, 959, 1070, 1074, 1029, 1018, 1032, 1013, 1029, 1024, 992, 1090, 1049, 1051, 1065, 1037, 1004, 986, 988, 1021, 1020, 1023, 1024, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[970, 1011, 1017, 1009, 1049, 1045, 1041, 999, 25, 8, 0, 0, 0, 0, 0, 0, 1054, 1031, 1065, 1056, 1039, 1041, 990, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1004, 1002, 1045, 1053, 1007, 995, 982, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1071, 1044, 983, 1044, 1008, 1002, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1019, 1013, 1003, 1033, 1026, 1008, 1020, 11, 10, 7, 0, 0, 0, 0, 0, 1029, 959, 1070, 1074, 981, 1018, 1032, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1024, 992, 1090, 1049, 1051, 1021, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 986, 988, 1021, 1020, 1023, 1024, 1005, 48, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8300, 8081, 8182, 8192, 8176, 8293, 8138] avg=8192.0000 max=8300 max_rank=1 min=8081 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1024, 1045, 1043, 1021, 1015, 994, 999, 958, 25, 8, 0, 0, 0, 0, 0, 0, 1048, 1031, 1061, 1028, 1011, 996, 1056, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1007, 1050, 998, 1046, 1007, 992, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1032, 991, 1068, 1032, 1015, 1028, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1019, 1036, 951, 1025, 998, 1014, 969, 11, 10, 7, 0, 0, 0, 0, 0, 1057, 980, 985, 1018, 993, 1043, 1054, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 984, 1087, 1051, 1000, 1055, 1027, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1006, 1033, 1035, 1010, 983, 1081, 1006, 49, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8284, 8174, 8230, 8097, 8147, 8262, 8210] avg=8192.0000 max=8284 max_rank=1 min=8097 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[974, 1031, 1057, 989, 972, 1051, 993, 1022, 24, 8, 0, 0, 0, 0, 0, 0, 1088, 1008, 1030, 977, 1006, 1040, 1021, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 995, 1064, 1027, 1031, 1079, 1015, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1011, 1057, 1007, 1054, 1069, 1000, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 961, 995, 1067, 983, 1055, 1058, 981, 1050, 11, 10, 7, 0, 0, 0, 0, 0, 1050, 1051, 1026, 1033, 983, 1000, 999, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1021, 1043, 1054, 990, 1081, 953, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1032, 983, 1045, 1000, 1010, 979, 992, 49, 18, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8227, 8306, 8251, 8178, 8156, 8162, 8135] avg=8192.0000 max=8306 max_rank=2 min=8121 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1000, 1060, 1011, 1031, 942, 1029, 1025, 1018, 24, 8, 0, 0, 0, 0, 0, 0, 1047, 1016, 997, 1010, 1034, 1032, 1037, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1026, 992, 1018, 1027, 1034, 1048, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1058, 1039, 1034, 1057, 1050, 1012, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1032, 1054, 981, 996, 1005, 1067, 1008, 11, 11, 7, 0, 0, 0, 0, 0, 981, 1035, 1055, 1022, 963, 995, 1012, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1030, 1021, 1057, 1056, 1046, 958, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1002, 1049, 992, 982, 1055, 1040, 1032, 48, 18, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8171, 8254, 8239, 8203, 8070, 8207, 8244] avg=8192.0000 max=8254 max_rank=2 min=8070 min_rank=5 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1039, 1050, 1011, 1003, 1029, 989, 1070, 982, 24, 8, 0, 0, 0, 0, 0, 0, 1058, 1005, 1051, 1006, 991, 1024, 1060, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 965, 1022, 1058, 1069, 1007, 970, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 999, 987, 1016, 1003, 1064, 1050, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1059, 1047, 1003, 1017, 983, 1011, 1033, 11, 10, 7, 0, 0, 0, 0, 0, 1007, 1036, 1085, 1030, 1018, 1022, 1060, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1068, 1037, 974, 1018, 1001, 983, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 957, 1005, 990, 997, 1031, 1026, 1041, 50, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8251, 8111, 8203, 8234, 8268, 8107, 8157] avg=8192.0000 max=8268 max_rank=5 min=8107 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1023, 1033, 987, 987, 1005, 1015, 1066, 1014, 25, 8, 0, 0, 0, 0, 0, 0, 1039, 982, 1008, 1013, 996, 1003, 1053, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1022, 1017, 995, 997, 1002, 1039, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1023, 1007, 1058, 1027, 1011, 1072, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 999, 976, 1037, 1055, 1071, 1015, 1061, 11, 10, 7, 0, 0, 0, 0, 0, 1080, 1064, 996, 997, 996, 1048, 1043, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1012, 1050, 1029, 1030, 1056, 1020, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 998, 975, 1018, 999, 1057, 1038, 978, 1033, 49, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8094, 8099, 8266, 8294, 8202, 8254, 8164] avg=8192.0000 max=8294 max_rank=4 min=8094 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1002, 1040, 1079, 1040, 1040, 999, 1031, 1037, 28, 12, 0, 0, 0, 0, 0, 0, 988, 1000, 1035, 1013, 1062, 1000, 1006, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1053, 1044, 1021, 969, 1071, 1020, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1006, 1015, 1029, 1021, 982, 985, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1025, 1077, 1002, 1005, 1023, 1000, 969, 17, 14, 11, 0, 0, 0, 0, 0, 980, 1036, 1076, 1029, 1049, 1047, 1029, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1068, 1040, 1015, 1073, 968, 951, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1008, 1046, 1011, 995, 1033, 1011, 1024, 56, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8108, 8198, 8054, 8191, 8294, 8186, 8197] avg=8192.0000 max=8308 max_rank=0 min=8054 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32868 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1027, 1006, 1022, 1052, 1005, 1053, 1022, 1065, 25, 8, 0, 0, 0, 0, 0, 0, 1010, 1019, 1030, 1033, 1045, 984, 976, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1073, 964, 1005, 1054, 1031, 1044, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1023, 1038, 1046, 1033, 1033, 985, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1019, 990, 1027, 1018, 1013, 1018, 1020, 11, 11, 7, 0, 0, 0, 0, 0, 1065, 1032, 1032, 983, 989, 1050, 1054, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1025, 996, 970, 1016, 1015, 1013, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1084, 984, 1021, 1036, 973, 1053, 1014, 49, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8101, 8313, 8111, 8147, 8223, 8065, 8291] avg=8192.0000 max=8313 max_rank=2 min=8065 min_rank=6 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8192, 8064, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8192, 8192, 8192, 8192, 8192, 8192, 512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8320, 7936, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8448, 8192, 8064, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8064, 8320, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1045, 1026, 1056, 1049, 1035, 997, 1006, 948, 1026, 1036, 985, 996, 991, 990, 987, 998, 1014, 1000, 1040, 1050, 1012, 1025, 1011, 1049, 994, 983, 1093, 1039, 1023, 1057, 1065, 1005, 984, 1047, 1003, 1026, 1041, 1039, 961, 1035, 1052, 1022, 976, 1064, 1002, 1014, 1003, 1049, 1038, 1032, 1035, 1080, 1061, 1014, 1016, 1107, 1044, 994, 1024, 1043, 1043, 1000, 1015, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1030, 1062, 1062, 1004, 1070, 1018, 1040, 973, 1008, 1029, 1014, 1037, 1031, 1036, 1039, 987, 1040, 1080, 1053, 1000, 1006, 978, 1023, 1074, 1002, 1086, 1002, 1023, 1051, 1019, 1068, 1069, 1015, 1035, 986, 1011, 1012, 1060, 1001, 1061, 1027, 1019, 1054, 1009, 984, 1030, 976, 1032, 1059, 990, 1036, 1010, 992, 978, 1029, 1045, 1032, 1002, 999, 950, 1024, 1029, 1033, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1042, 1006, 1001, 1059, 1032, 1038, 1020, 1013, 983, 1003, 1053, 1112, 1090, 1014, 1012, 1046, 979, 1033, 1033, 1018, 979, 1034, 1029, 1032, 1015, 1020, 990, 1025, 964, 1012, 1073, 1019, 1068, 1030, 971, 1047, 995, 1050, 1024, 1003, 997, 1041, 1014, 1037, 1048, 1061, 1052, 991, 976, 1019, 1069, 1002, 993, 1012, 1006, 991, 1072, 1063, 1020, 993, 1002, 1093, 1009, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1066, 1076, 982, 1039, 1024, 1018, 990, 1051, 989, 973, 994, 972, 988, 1010, 1059, 1007, 1008, 980, 992, 1015, 988, 983, 1089, 1012, 1068, 1011, 1055, 1083, 1060, 1051, 1019, 1012, 1021, 1009, 1041, 1015, 1048, 1035, 966, 1012, 981, 1030, 1027, 1030, 1064, 1002, 1044, 1036, 1069, 1028, 1015, 1006, 1090, 1056, 997, 1014, 1026, 1091, 1029, 1030, 1041, 975, 1008, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1039, 1012, 1040, 995, 1010, 1070, 1008, 1053, 1018, 1039, 1047, 1001, 1011, 1007, 1056, 1002, 1026, 1026, 967, 1004, 997, 1032, 1012, 1046, 1007, 1021, 1005, 1013, 1033, 1027, 986, 1059, 1018, 1035, 1019, 1056, 1019, 1001, 1009, 1045, 974, 1000, 1023, 1016, 1055, 1012, 1014, 1023, 1039, 1059, 1001, 1067, 1030, 991, 1014, 1054, 1026, 1038, 1027, 1059, 1047, 1027, 1045, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[964, 1030, 1049, 1042, 1019, 1034, 1041, 1055, 1017, 1038, 1000, 1068, 967, 1039, 1058, 1055, 994, 1045, 1069, 1057, 998, 1051, 1040, 955, 1025, 1020, 1030, 1028, 1025, 1010, 1006, 1080, 1039, 1002, 991, 1008, 1050, 1056, 1020, 993, 976, 1040, 1018, 994, 1078, 1045, 994, 1094, 1020, 1088, 1039, 984, 1069, 969, 988, 1006, 1006, 1055, 983, 1032, 1002, 996, 1022, 970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[994, 1038, 1033, 1025, 1001, 973, 1015, 1027, 1047, 980, 1058, 1023, 1010, 1028, 1049, 1014, 1010, 1010, 986, 1013, 966, 1013, 969, 1067, 1056, 1046, 1050, 1014, 1027, 1039, 1060, 1048, 1041, 1018, 1026, 1066, 966, 1066, 1018, 984, 1013, 1068, 1041, 995, 1059, 1016, 1035, 1021, 1017, 970, 958, 1016, 1013, 1022, 1056, 1045, 1001, 1069, 1003, 1070, 975, 1105, 1025, 1069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1004, 1003, 1060, 1042, 1026, 1015, 961, 1007, 1030, 1021, 1016, 1008, 1058, 958, 1083, 1011, 995, 1092, 1011, 1001, 1029, 1067, 1069, 1068, 1047, 1003, 1038, 1036, 972, 1022, 1016, 1014, 996, 1002, 982, 1047, 995, 1001, 1037, 1023, 1024, 969, 1030, 1044, 1040, 1024, 991, 1031, 994, 1050, 1027, 992, 1028, 1055, 1052, 985, 1067, 1041, 1034, 1043, 986, 1031, 1037, 1095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1045, 1026, 1053, 1049, 1035, 997, 1006, 948, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1036, 985, 996, 991, 990, 987, 998, 15, 11, 0, 0, 0, 0, 0, 0, 1014, 1000, 1040, 1050, 1012, 1025, 1011, 1049, 17, 3, 2, 0, 0, 0, 0, 0, 994, 983, 1093, 1039, 1023, 1057, 1065, 960, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1047, 1003, 1026, 1041, 1039, 961, 1035, 30, 0, 0, 0, 0, 0, 0, 0, 1052, 1022, 976, 1064, 1002, 1014, 1003, 1049, 1, 0, 0, 0, 0, 0, 0, 0, 1038, 1032, 1035, 1080, 1058, 1014, 1016, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 966, 1024, 1043, 1043, 1000, 1015, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8035, 8223, 8214, 8166, 8183, 8380, 8176] avg=8192.0000 max=8380 max_rank=6 min=8035 min_rank=1 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32905 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1039, 1012, 1037, 995, 1010, 1070, 1008, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1039, 1047, 1001, 1011, 1007, 1056, 1002, 16, 11, 0, 0, 0, 0, 0, 0, 1026, 1026, 967, 1004, 997, 1032, 1012, 1046, 18, 3, 1, 0, 0, 0, 0, 0, 1007, 1021, 1005, 1013, 1033, 1027, 986, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1035, 1019, 1056, 1019, 1001, 1009, 1045, 32, 0, 0, 0, 0, 0, 0, 0, 974, 1000, 1023, 1016, 1055, 1012, 1014, 1023, 1, 0, 0, 0, 0, 0, 0, 0, 1039, 1059, 1001, 1067, 1028, 991, 1014, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1009, 1027, 1059, 1047, 1027, 1045, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8208, 8132, 8103, 8234, 8118, 8253, 8264] avg=8192.0000 max=8264 max_rank=7 min=8103 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[964, 1030, 1046, 1042, 1019, 1034, 1041, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1038, 1000, 1068, 967, 1039, 1058, 1055, 16, 12, 0, 0, 0, 0, 0, 0, 994, 1045, 1069, 1057, 998, 1051, 1040, 955, 18, 3, 2, 0, 0, 0, 0, 0, 1025, 1020, 1030, 1028, 1025, 1010, 1006, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1002, 991, 1008, 1050, 1056, 1020, 993, 33, 0, 0, 0, 0, 0, 0, 0, 976, 1040, 1018, 994, 1078, 1045, 994, 1094, 1, 0, 0, 0, 0, 0, 0, 0, 1020, 1088, 1039, 984, 1066, 969, 988, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1025, 983, 1032, 1002, 996, 1022, 970, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8270, 8232, 8175, 8192, 8240, 8160, 8036] avg=8192.0000 max=8270 max_rank=1 min=8036 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1042, 1006, 998, 1059, 1032, 1038, 1020, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1003, 1053, 1112, 1090, 1014, 1012, 1046, 15, 12, 0, 0, 0, 0, 0, 0, 979, 1033, 1033, 1018, 979, 1034, 1029, 1032, 18, 3, 1, 0, 0, 0, 0, 0, 1015, 1020, 990, 1025, 964, 1012, 1073, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1030, 971, 1047, 995, 1050, 1024, 1003, 31, 0, 0, 0, 0, 0, 0, 0, 997, 1041, 1014, 1037, 1048, 1061, 1052, 991, 1, 0, 0, 0, 0, 0, 0, 0, 976, 1019, 1069, 1002, 991, 1012, 1006, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1033, 1020, 993, 1002, 1093, 1009, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8340, 8159, 8072, 8219, 8242, 8066, 8230] avg=8192.0000 max=8340 max_rank=1 min=8066 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1030, 1062, 1059, 1004, 1070, 1018, 1040, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1029, 1014, 1037, 1031, 1036, 1039, 987, 16, 11, 0, 0, 0, 0, 0, 0, 1040, 1080, 1053, 1000, 1006, 978, 1023, 1074, 17, 3, 1, 0, 0, 0, 0, 0, 1002, 1086, 1002, 1023, 1051, 1019, 1068, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1035, 986, 1011, 1012, 1060, 1001, 1061, 32, 0, 0, 0, 0, 0, 0, 0, 1027, 1019, 1054, 1009, 984, 1030, 976, 1032, 1, 0, 0, 0, 0, 0, 0, 0, 1059, 990, 1036, 1010, 990, 978, 1029, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 974, 999, 950, 1024, 1029, 1033, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8208, 8275, 8272, 8213, 8132, 8137, 8043] avg=8192.0000 max=8275 max_rank=2 min=8043 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32525 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1066, 1076, 979, 1039, 1024, 1018, 990, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 989, 973, 994, 972, 988, 1010, 1059, 1007, 15, 12, 0, 0, 0, 0, 0, 0, 1008, 980, 992, 1015, 988, 983, 1089, 1012, 19, 3, 2, 0, 0, 0, 0, 0, 1068, 1011, 1055, 1083, 1060, 1051, 1019, 966, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1009, 1041, 1015, 1048, 1035, 966, 1012, 31, 0, 0, 0, 0, 0, 0, 0, 981, 1030, 1027, 1030, 1064, 1002, 1044, 1036, 1, 0, 0, 0, 0, 0, 0, 0, 1069, 1028, 1015, 1006, 1087, 1056, 997, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1060, 1029, 1030, 1041, 975, 1008, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8019, 8091, 8313, 8178, 8215, 8272, 8205] avg=8192.0000 max=8313 max_rank=3 min=8019 min_rank=1 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1004, 1003, 1057, 1042, 1026, 1015, 961, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1021, 1016, 1008, 1058, 958, 1083, 1011, 15, 11, 0, 0, 0, 0, 0, 0, 995, 1092, 1011, 1001, 1029, 1067, 1069, 1068, 18, 3, 1, 0, 0, 0, 0, 0, 1047, 1003, 1038, 1036, 972, 1022, 1016, 968, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1002, 982, 1047, 995, 1001, 1037, 1023, 31, 0, 0, 0, 0, 0, 0, 0, 1024, 969, 1030, 1044, 1040, 1024, 991, 1031, 1, 0, 0, 0, 0, 0, 0, 0, 994, 1050, 1027, 992, 1026, 1055, 1052, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1012, 1034, 1043, 986, 1031, 1037, 1095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8211, 8354, 8102, 8114, 8154, 8181, 8305] avg=8192.0000 max=8354 max_rank=2 min=8102 min_rank=3 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[994, 1038, 1027, 1025, 1001, 973, 1015, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 980, 1058, 1023, 1010, 1028, 1049, 1014, 20, 16, 0, 0, 0, 0, 0, 0, 1010, 1010, 986, 1013, 966, 1013, 969, 1067, 24, 6, 6, 0, 0, 0, 0, 0, 1056, 1046, 1050, 1014, 1027, 1039, 1060, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1018, 1026, 1066, 966, 1066, 1018, 984, 35, 0, 0, 0, 0, 0, 0, 0, 1013, 1068, 1041, 995, 1059, 1016, 1035, 1021, 4, 0, 0, 0, 0, 0, 0, 0, 1017, 970, 958, 1016, 1003, 1022, 1056, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1029, 1003, 1070, 975, 1105, 1025, 1069, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8245, 8070, 8285, 8220, 8252, 8087, 8277] avg=8192.0000 max=8285 max_rank=3 min=8070 min_rank=2 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8192, 8448, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8320, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8064, 8192, 8320, 8320, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8192, 8064, 8320, 8192, 8320, 8064, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8448, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1091, 1048, 988, 1032, 1025, 1067, 1076, 1008, 1021, 1025, 1003, 995, 1042, 961, 1007, 1012, 1041, 1021, 992, 1023, 1063, 994, 1022, 1053, 977, 1075, 990, 1046, 1084, 979, 1015, 1027, 1032, 1087, 1048, 1023, 1032, 971, 993, 1029, 1100, 1001, 1040, 1047, 1008, 1011, 1039, 1032, 1008, 1022, 1001, 993, 1044, 1026, 1050, 1063, 1035, 952, 1065, 1006, 1008, 993, 981, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1042, 1046, 1000, 1066, 1049, 1040, 1018, 1009, 1061, 1036, 1049, 1012, 972, 1058, 979, 1005, 1048, 1063, 1032, 1024, 1003, 991, 1068, 1040, 1032, 1042, 1027, 1031, 1063, 1020, 971, 989, 1020, 996, 1073, 1049, 1008, 1020, 1005, 1050, 988, 1026, 1024, 1032, 953, 1022, 1025, 1042, 1033, 1011, 986, 1022, 1014, 1012, 1045, 1022, 1069, 1021, 1054, 1001, 1024, 996, 991, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1041, 1003, 1052, 1047, 1017, 1027, 1032, 1031, 1016, 1014, 977, 985, 1015, 1044, 1002, 1000, 1036, 1004, 1016, 1068, 1038, 1025, 1060, 1026, 1021, 1032, 1008, 961, 994, 1006, 1051, 970, 1025, 1039, 1015, 1058, 1035, 1043, 1048, 1074, 1036, 1059, 998, 1017, 1018, 1007, 1026, 983, 1035, 1020, 1058, 1082, 1023, 1057, 1025, 995, 1026, 1006, 1024, 1025, 1015, 1011, 991, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1017, 1033, 1022, 1054, 1046, 1044, 1042, 1034, 1051, 947, 1020, 1029, 1021, 1021, 1058, 997, 1000, 1065, 1002, 1034, 1048, 1045, 1050, 998, 1038, 1019, 1006, 1002, 1054, 1019, 1036, 1029, 1021, 1004, 1002, 1039, 1043, 1003, 1013, 1070, 1010, 1029, 1032, 1049, 1057, 1006, 1029, 1010, 1010, 1029, 1043, 975, 1026, 1043, 1057, 998, 999, 1012, 1031, 950, 973, 1024, 1015, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1032, 1001, 1024, 1017, 1029, 1026, 994, 1066, 1022, 964, 1037, 1008, 1032, 994, 1078, 1062, 1036, 1005, 997, 1042, 998, 1035, 1030, 995, 1020, 1000, 1050, 1013, 1015, 1031, 1015, 1024, 1055, 1018, 1045, 952, 1044, 995, 1008, 1034, 1061, 1033, 1048, 1005, 1035, 1046, 998, 1020, 1038, 985, 999, 967, 1056, 1069, 1039, 1080, 1014, 1020, 1064, 1016, 1039, 1003, 1032, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1003, 1003, 994, 976, 1024, 967, 1022, 1051, 1008, 967, 1002, 1038, 1051, 1013, 990, 1022, 1031, 994, 1067, 1017, 1042, 1026, 982, 994, 1071, 1039, 1050, 1017, 1059, 1039, 1050, 1024, 1006, 1008, 1014, 985, 1055, 1043, 1083, 1037, 989, 1059, 1019, 1025, 1028, 1052, 1074, 1047, 1057, 1041, 1005, 986, 1022, 997, 1048, 1006, 1045, 1056, 1008, 1020, 1035, 1041, 1033, 979] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[950, 1018, 1051, 1064, 1002, 1055, 1013, 1027, 1028, 1026, 1028, 1030, 1069, 1056, 963, 998, 1018, 1048, 1021, 1019, 1066, 998, 990, 1060, 960, 1020, 1045, 983, 1016, 1054, 1026, 1003, 1071, 960, 1025, 1000, 1047, 989, 998, 1033, 1021, 1055, 983, 1021, 1026, 1037, 1025, 1030, 1010, 1003, 1014, 1001, 1010, 1039, 999, 1039, 1061, 1009, 1030, 1043, 1081, 1080, 1061, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1009, 1032, 982, 1006, 1029, 1045, 1081, 1039, 984, 1024, 994, 983, 1022, 999, 995, 980, 1025, 1028, 1035, 1025, 1002, 963, 1017, 1061, 992, 1044, 1013, 998, 1025, 1021, 1078, 1035, 1036, 1047, 986, 1031, 1016, 1033, 1036, 1031, 1115, 981, 1014, 1004, 1051, 1053, 1045, 1081, 1031, 1057, 1015, 1028, 1046, 1032, 1001, 978, 1072, 1029, 1019, 1058, 1034, 991, 975, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1091, 1048, 988, 1032, 1025, 1067, 1045, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1025, 1003, 995, 1042, 961, 1007, 1012, 39, 31, 11, 0, 0, 0, 0, 0, 1041, 1021, 992, 1023, 1049, 994, 1022, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1075, 990, 1046, 1084, 979, 1015, 1027, 7, 3, 0, 0, 0, 0, 0, 0, 1032, 1087, 1048, 1023, 1032, 971, 993, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1001, 1040, 1047, 1008, 1011, 1039, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1022, 1001, 993, 1044, 1026, 1050, 1063, 1, 0, 0, 0, 0, 0, 0, 0, 1035, 952, 1065, 1006, 1008, 993, 981, 993, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8304, 8147, 8195, 8203, 8197, 8239, 8208, 8043] avg=8192.0000 max=8304 max_rank=0 min=8043 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1042, 1046, 1000, 1066, 1049, 1040, 989, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1036, 1049, 1012, 972, 1058, 979, 1005, 35, 29, 11, 0, 0, 0, 0, 0, 1048, 1063, 1032, 1024, 989, 991, 1068, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1042, 1027, 1031, 1063, 1020, 971, 989, 7, 3, 0, 0, 0, 0, 0, 0, 1020, 996, 1073, 1049, 1008, 1020, 1005, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 953, 1026, 1024, 1032, 953, 1022, 1025, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1011, 986, 1022, 1014, 1012, 1045, 1022, 1, 0, 0, 0, 0, 0, 0, 0, 1069, 1021, 1054, 1001, 1024, 996, 991, 1016, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8247, 8255, 8185, 8203, 8077, 8146, 8182] avg=8192.0000 max=8255 max_rank=2 min=8077 min_rank=5 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1041, 1003, 1052, 1047, 1017, 1027, 1002, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1014, 977, 985, 1015, 1044, 1002, 1000, 37, 30, 11, 0, 0, 0, 0, 0, 1036, 1004, 1016, 1068, 1024, 1025, 1060, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1032, 1008, 961, 994, 1006, 1051, 970, 8, 3, 0, 0, 0, 0, 0, 0, 1025, 1039, 1015, 1058, 1035, 1043, 1048, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1059, 998, 1017, 1018, 1007, 1026, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1020, 1058, 1082, 1023, 1057, 1025, 995, 1, 0, 0, 0, 0, 0, 0, 0, 1026, 1006, 1024, 1025, 1015, 1011, 991, 1043, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8131, 8259, 8054, 8318, 8107, 8296, 8151] avg=8192.0000 max=8318 max_rank=4 min=8054 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1017, 1033, 1022, 1054, 1046, 1044, 1012, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 947, 1020, 1029, 1021, 1021, 1058, 997, 36, 30, 11, 0, 0, 0, 0, 0, 1000, 1065, 1002, 1034, 1034, 1045, 1050, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1019, 1006, 1002, 1054, 1019, 1036, 1029, 8, 3, 0, 0, 0, 0, 0, 0, 1021, 1004, 1002, 1039, 1043, 1003, 1013, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1029, 1032, 1049, 1057, 1006, 1029, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1029, 1043, 975, 1026, 1043, 1057, 998, 1, 0, 0, 0, 0, 0, 0, 0, 999, 1012, 1031, 950, 973, 1024, 1015, 1053, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8221, 8228, 8214, 8176, 8186, 8182, 8067] avg=8192.0000 max=8262 max_rank=0 min=8067 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32611 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1003, 1003, 994, 976, 1024, 967, 992, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 967, 1002, 1038, 1051, 1013, 990, 1022, 35, 30, 11, 0, 0, 0, 0, 0, 1031, 994, 1067, 1017, 1028, 1026, 982, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1039, 1050, 1017, 1059, 1039, 1050, 1024, 7, 3, 0, 0, 0, 0, 0, 0, 1006, 1008, 1014, 985, 1055, 1043, 1083, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1059, 1019, 1025, 1028, 1052, 1074, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1041, 1005, 986, 1022, 997, 1048, 1006, 1, 0, 0, 0, 0, 0, 0, 0, 1045, 1056, 1008, 1020, 1035, 1041, 1033, 979, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8010, 8167, 8139, 8359, 8213, 8258, 8163, 8227] avg=8192.0000 max=8359 max_rank=3 min=8010 min_rank=0 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32861 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1032, 1001, 1024, 1017, 1029, 1026, 965, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 964, 1037, 1008, 1032, 994, 1078, 1062, 38, 29, 11, 0, 0, 0, 0, 0, 1036, 1005, 997, 1042, 984, 1035, 1030, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1000, 1050, 1013, 1015, 1031, 1015, 1024, 7, 3, 0, 0, 0, 0, 0, 0, 1055, 1018, 1045, 952, 1044, 995, 1008, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1033, 1048, 1005, 1035, 1046, 998, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 985, 999, 967, 1056, 1069, 1039, 1080, 1, 0, 0, 0, 0, 0, 0, 0, 1014, 1020, 1064, 1016, 1039, 1003, 1032, 1026, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8275, 8124, 8178, 8133, 8208, 8234, 8224] avg=8192.0000 max=8275 max_rank=1 min=8124 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[950, 1018, 1051, 1064, 1002, 1055, 984, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1026, 1028, 1030, 1069, 1056, 963, 998, 36, 29, 11, 0, 0, 0, 0, 0, 1018, 1048, 1021, 1019, 1052, 998, 990, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 960, 1020, 1045, 983, 1016, 1054, 1026, 1003, 7, 3, 0, 0, 0, 0, 0, 0, 1071, 960, 1025, 1000, 1047, 989, 998, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1055, 983, 1021, 1026, 1037, 1025, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1003, 1014, 1001, 1010, 1039, 999, 1039, 1, 0, 0, 0, 0, 0, 0, 0, 1061, 1009, 1030, 1043, 1081, 1080, 1061, 1030, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8274, 8206, 8117, 8105, 8162, 8116, 8405] avg=8192.0000 max=8405 max_rank=7 min=8105 min_rank=4 imbalance_ratio=1.026001 max_load_reduction=57131 max_load_reduction_pct=0.871750 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1009, 1032, 982, 1006, 1029, 1045, 1046, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1024, 994, 983, 1022, 999, 995, 980, 45, 35, 13, 0, 0, 0, 0, 0, 1025, 1028, 1035, 1025, 976, 963, 1017, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1044, 1013, 998, 1025, 1021, 1078, 1035, 12, 8, 0, 0, 0, 0, 0, 0, 1036, 1047, 986, 1031, 1016, 1033, 1036, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 981, 1014, 1004, 1051, 1053, 1045, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1057, 1015, 1028, 1046, 1032, 1001, 978, 3, 0, 0, 0, 0, 0, 0, 0, 1072, 1029, 1019, 1058, 1034, 991, 975, 1044, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8074, 8130, 8226, 8191, 8299, 8191, 8237] avg=8192.0000 max=8299 max_rank=5 min=8074 min_rank=1 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8064, 8192, 8192, 8320, 8192, 8192, 8192, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8064, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8064, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True + [2026-08-30 05:02:09] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1216.5 | throughput per GPU (TFLOP/s/GPU): 274.1 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101405E+01 | loss scale: 1.0 | grad norm: 45.664 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1017, 1057, 1006, 1040, 1031, 977, 1066, 1056, 1009, 1026, 1061, 992, 992, 1018, 1028, 1003, 1044, 1028, 1001, 1054, 1042, 1023, 1043, 1006, 975, 1028, 1006, 971, 1024, 1012, 1008, 1007, 1020, 1040, 1049, 1019, 1000, 998, 1035, 1054, 1026, 1034, 1044, 1049, 994, 1015, 1044, 1011, 982, 1051, 1029, 1057, 1006, 1026, 1048, 1037, 1030, 1029, 1046, 1072, 984, 1036, 1006, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1061, 1033, 1023, 955, 1034, 1025, 1016, 1025, 1045, 1023, 1035, 1064, 1025, 1040, 986, 1027, 1018, 1076, 995, 1029, 1010, 1019, 1042, 1003, 1021, 1027, 996, 1052, 1033, 1078, 996, 1039, 1057, 1036, 1028, 1040, 948, 1018, 1028, 1014, 1009, 1033, 1014, 964, 1030, 1052, 1040, 1020, 998, 981, 1050, 981, 1053, 952, 1037, 1026, 1055, 1024, 1031, 1032, 1082, 1046, 997, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1059, 1024, 965, 1052, 1003, 991, 1036, 1042, 990, 1042, 973, 991, 1086, 1068, 1006, 984, 1049, 1021, 1079, 1046, 1027, 1040, 1046, 1072, 1041, 1010, 1044, 1014, 1025, 1003, 1012, 993, 1030, 1009, 1031, 1001, 1057, 960, 996, 1027, 1015, 1020, 1018, 1007, 1048, 1053, 1016, 978, 998, 1029, 1012, 1039, 1017, 1048, 1048, 1060, 1064, 999, 1011, 980, 997, 1025, 1091, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1026, 1043, 998, 995, 1023, 1036, 1013, 1042, 981, 958, 1018, 1028, 1002, 1016, 1025, 993, 1020, 1039, 1060, 999, 1056, 1023, 1044, 1036, 1063, 997, 1018, 1005, 1032, 1059, 1004, 1040, 1042, 1079, 1032, 1045, 1020, 1121, 1018, 1031, 1000, 988, 991, 980, 1040, 1046, 994, 1049, 980, 1042, 1004, 974, 1033, 1055, 1028, 989, 1079, 1014, 972, 1038, 1062, 1031, 1037, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[996, 1009, 1045, 994, 1025, 1026, 1044, 1052, 1068, 1000, 1048, 1040, 1020, 1032, 1023, 1037, 993, 1045, 1013, 1020, 1068, 993, 1023, 1030, 1081, 1045, 1036, 1031, 971, 1011, 1045, 1024, 989, 1066, 1043, 1053, 997, 1031, 982, 1034, 1028, 1032, 1008, 1027, 1057, 991, 1042, 1001, 969, 980, 1022, 1059, 1002, 1041, 1023, 1060, 1024, 999, 1038, 1009, 1018, 997, 1039, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1065, 1057, 990, 1021, 1041, 1020, 993, 1025, 1023, 1080, 1069, 998, 1063, 971, 1002, 1054, 1041, 1005, 1045, 1000, 1007, 997, 1008, 1024, 982, 1050, 1007, 1060, 1028, 1024, 1025, 1002, 983, 1048, 1036, 1037, 1044, 1022, 990, 1025, 1005, 1035, 1000, 981, 1044, 1016, 1035, 1049, 1000, 1063, 1015, 1014, 1015, 1063, 1034, 987, 976, 1048, 1025, 1074, 1050, 985, 1038, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[997, 1004, 989, 1057, 1013, 1053, 1002, 1056, 1014, 1034, 1041, 1027, 1025, 1008, 1026, 1036, 998, 1080, 1072, 1055, 986, 1016, 1006, 1027, 1036, 1013, 996, 1023, 1003, 998, 1042, 1064, 998, 1030, 1013, 1062, 1058, 1076, 988, 1053, 1004, 1044, 986, 1017, 1007, 1039, 1011, 1033, 1044, 1023, 955, 1012, 983, 1069, 1032, 1056, 989, 1059, 1029, 1021, 1009, 1013, 1033, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1011, 1023, 982, 1098, 1023, 1013, 1060, 1010, 985, 1018, 1007, 1046, 1027, 1002, 1051, 984, 1053, 1071, 1033, 1043, 1019, 1050, 1006, 1060, 999, 1004, 1025, 1068, 979, 1037, 1034, 977, 1075, 1010, 1013, 1006, 1051, 997, 1009, 1069, 1035, 1045, 954, 989, 1057, 1072, 1013, 1036, 988, 1066, 1010, 992, 998, 1029, 1035, 1021, 1011, 1016, 1026, 1017, 1009, 1019, 1052, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1017, 1057, 1006, 1040, 1031, 977, 1066, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1026, 1061, 992, 992, 1018, 1028, 1003, 8, 5, 3, 0, 0, 0, 0, 0, 1044, 975, 1001, 1054, 1042, 1023, 1043, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1028, 1006, 971, 1024, 1012, 1008, 1007, 21, 1, 0, 0, 0, 0, 0, 0, 1020, 1011, 1049, 1019, 1000, 998, 1035, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1034, 1044, 1049, 994, 1015, 1044, 1011, 27, 0, 0, 0, 0, 0, 0, 0, 982, 1051, 1029, 1057, 1006, 1026, 1048, 1037, 25, 0, 0, 0, 0, 0, 0, 0, 1030, 1029, 1046, 1072, 984, 1036, 1001, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8145, 8188, 8053, 8186, 8244, 8261, 8212] avg=8192.0000 max=8261 max_rank=6 min=8053 min_rank=3 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1061, 1033, 1023, 955, 1034, 1025, 1016, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1023, 1035, 1064, 1025, 1040, 986, 1027, 8, 5, 3, 0, 0, 0, 0, 0, 1018, 1021, 995, 1029, 1010, 1019, 1042, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1027, 996, 1052, 1033, 1078, 996, 1039, 21, 1, 0, 0, 0, 0, 0, 0, 1057, 1007, 1028, 1040, 948, 1018, 1028, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1033, 1014, 964, 1030, 1052, 1040, 1020, 28, 0, 0, 0, 0, 0, 0, 0, 998, 981, 1050, 981, 1053, 952, 1037, 1026, 26, 0, 0, 0, 0, 0, 0, 0, 1055, 1024, 1031, 1032, 1082, 1046, 992, 1009, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8261, 8137, 8264, 8140, 8190, 8104, 8271] avg=8192.0000 max=8271 max_rank=7 min=8104 min_rank=6 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1059, 1024, 965, 1052, 1003, 991, 1036, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1042, 973, 991, 1086, 1068, 1006, 984, 8, 6, 3, 0, 0, 0, 0, 0, 1049, 969, 1079, 1046, 1027, 1040, 1046, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1010, 1044, 1014, 1025, 1003, 1012, 993, 20, 1, 0, 0, 0, 0, 0, 0, 1030, 981, 1031, 1001, 1057, 960, 996, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1020, 1018, 1007, 1048, 1053, 1016, 978, 26, 0, 0, 0, 0, 0, 0, 0, 998, 1029, 1012, 1039, 1017, 1048, 1048, 1060, 25, 0, 0, 0, 0, 0, 0, 0, 1064, 999, 1011, 980, 997, 1025, 1085, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8157, 8328, 8163, 8083, 8181, 8276, 8179] avg=8192.0000 max=8328 max_rank=2 min=8083 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[997, 1004, 989, 1057, 1013, 1053, 1002, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1034, 1041, 1027, 1025, 1008, 1026, 1036, 8, 5, 3, 0, 0, 0, 0, 0, 998, 1025, 1072, 1055, 986, 1016, 1006, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1013, 996, 1023, 1003, 998, 1042, 1064, 20, 1, 0, 0, 0, 0, 0, 0, 998, 1002, 1013, 1062, 1058, 1076, 988, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1044, 986, 1017, 1007, 1039, 1011, 1033, 28, 0, 0, 0, 0, 0, 0, 0, 1044, 1023, 955, 1012, 983, 1069, 1032, 1056, 26, 0, 0, 0, 0, 0, 0, 0, 989, 1059, 1029, 1021, 1009, 1013, 1028, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8227, 8185, 8196, 8250, 8169, 8200, 8141] avg=8192.0000 max=8250 max_rank=4 min=8141 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1026, 1043, 998, 995, 1023, 1036, 1013, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 981, 958, 1018, 1028, 1002, 1016, 1025, 993, 8, 6, 3, 0, 0, 0, 0, 0, 1020, 986, 1060, 999, 1056, 1023, 1044, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 997, 1018, 1005, 1032, 1059, 1004, 1040, 21, 1, 0, 0, 0, 0, 0, 0, 1042, 1050, 1032, 1045, 1020, 1121, 1018, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 988, 991, 980, 1040, 1046, 994, 1049, 27, 0, 0, 0, 0, 0, 0, 0, 980, 1042, 1004, 974, 1033, 1055, 1028, 989, 25, 0, 0, 0, 0, 0, 0, 0, 1079, 1014, 972, 1038, 1062, 1031, 1031, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8038, 8224, 8240, 8359, 8115, 8130, 8257] avg=8192.0000 max=8359 max_rank=4 min=8038 min_rank=1 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32861 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[996, 1009, 1045, 994, 1025, 1026, 1044, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1000, 1048, 1040, 1020, 1032, 1023, 1037, 8, 6, 3, 0, 0, 0, 0, 0, 993, 992, 1013, 1020, 1068, 993, 1023, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1045, 1036, 1031, 971, 1011, 1045, 1024, 21, 1, 0, 0, 0, 0, 0, 0, 989, 1037, 1043, 1053, 997, 1031, 982, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1032, 1008, 1027, 1057, 991, 1042, 1001, 27, 0, 0, 0, 0, 0, 0, 0, 969, 980, 1022, 1059, 1002, 1041, 1023, 1060, 25, 0, 0, 0, 0, 0, 0, 0, 1024, 999, 1038, 1009, 1018, 997, 1033, 987, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8285, 8132, 8266, 8166, 8213, 8181, 8105] avg=8192.0000 max=8285 max_rank=1 min=8105 min_rank=7 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1065, 1057, 990, 1021, 1041, 1020, 993, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1080, 1069, 998, 1063, 971, 1002, 1054, 8, 6, 3, 0, 0, 0, 0, 0, 1041, 954, 1045, 1000, 1007, 997, 1008, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1050, 1007, 1060, 1028, 1024, 1025, 1002, 21, 1, 0, 0, 0, 0, 0, 0, 983, 1019, 1036, 1037, 1044, 1022, 990, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1035, 1000, 981, 1044, 1016, 1035, 1049, 26, 0, 0, 0, 0, 0, 0, 0, 1000, 1063, 1015, 1014, 1015, 1063, 1034, 987, 24, 0, 0, 0, 0, 0, 0, 0, 976, 1048, 1025, 1074, 1050, 985, 1032, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8277, 8076, 8200, 8156, 8191, 8215, 8212] avg=8192.0000 max=8277 max_rank=1 min=8076 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1011, 1023, 982, 1098, 1023, 1013, 1060, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1018, 1007, 1046, 1027, 1002, 1051, 984, 10, 9, 7, 0, 0, 0, 0, 0, 1053, 1002, 1033, 1043, 1019, 1050, 1006, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1004, 1025, 1068, 979, 1037, 1034, 977, 24, 7, 0, 0, 0, 0, 0, 0, 1075, 976, 1013, 1006, 1051, 997, 1009, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1045, 954, 989, 1057, 1072, 1013, 1036, 32, 0, 0, 0, 0, 0, 0, 0, 988, 1066, 1010, 992, 998, 1029, 1035, 1021, 30, 0, 0, 0, 0, 0, 0, 0, 1011, 1016, 1026, 1017, 1009, 1019, 1043, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8146, 8266, 8154, 8196, 8233, 8169, 8159] avg=8192.0000 max=8266 max_rank=2 min=8146 min_rank=1 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8064, 8320, 8320, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 7936, 8320, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8192, 8192, 8320, 8320, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1024, 988, 1010, 1056, 1046, 1028, 1035, 966, 1007, 1042, 1014, 1067, 991, 988, 982, 1085, 1030, 1022, 1056, 1006, 1034, 1035, 1042, 995, 1022, 1035, 1060, 1008, 1051, 1005, 1007, 1011, 1069, 998, 984, 1058, 1033, 1067, 1025, 1075, 1035, 1011, 1001, 959, 1037, 1062, 1013, 992, 1022, 1042, 1001, 1008, 1059, 1005, 1000, 994, 984, 1024, 1033, 1042, 1037, 1015, 1056, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1084, 988, 1007, 972, 1006, 1033, 1006, 1066, 1006, 1023, 1010, 997, 1028, 1006, 1015, 992, 975, 1051, 1016, 1007, 1058, 1007, 1048, 1079, 1065, 1060, 1048, 989, 1044, 994, 978, 1040, 1040, 1023, 1016, 1060, 992, 984, 1097, 1050, 981, 985, 998, 1042, 1033, 1038, 1037, 1018, 997, 991, 1036, 1044, 1003, 1006, 1075, 1077, 1020, 968, 1035, 1057, 1038, 1024, 1014, 1059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1023, 1027, 985, 1068, 991, 984, 1025, 999, 997, 1049, 1031, 1004, 1002, 1054, 1016, 1013, 1041, 1032, 1002, 1012, 1058, 1024, 1072, 1045, 970, 1024, 999, 1043, 1032, 1049, 1018, 1021, 1002, 1068, 1033, 998, 1039, 1011, 1012, 1011, 1089, 1002, 1017, 1051, 1016, 1013, 1036, 1028, 1024, 1038, 1050, 988, 1041, 1019, 1018, 987, 1051, 1080, 1055, 983, 1014, 1049, 1005, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1002, 1044, 1025, 1006, 1047, 1011, 1038, 1031, 1022, 1035, 1005, 1049, 1022, 993, 1066, 1002, 1071, 1005, 982, 1022, 1075, 1023, 961, 1006, 969, 1002, 1012, 995, 1095, 990, 1066, 1064, 1071, 1026, 1018, 1043, 1066, 1030, 1016, 971, 1035, 1053, 1032, 1022, 1088, 1017, 969, 1026, 993, 1043, 1018, 1044, 980, 992, 994, 994, 1044, 994, 1009, 1036, 992, 1073, 1054, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1034, 1065, 1047, 974, 1043, 1001, 1017, 996, 1042, 1013, 1073, 1003, 1022, 990, 1049, 1027, 1047, 980, 1050, 1010, 1039, 992, 978, 1095, 1032, 1019, 1000, 1028, 1060, 1026, 1044, 1031, 996, 989, 969, 1024, 991, 1041, 1042, 1040, 1014, 977, 1037, 993, 1027, 1004, 1044, 996, 1015, 1024, 1073, 1012, 1040, 1060, 1054, 1016, 1002, 1009, 1096, 1017, 1002, 1015, 1066, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[998, 1032, 1023, 1058, 1020, 1036, 1025, 1037, 983, 1032, 1014, 1019, 1054, 1039, 976, 1029, 1023, 1051, 1075, 1023, 1017, 1022, 978, 1046, 1030, 1032, 1026, 1085, 959, 1010, 1046, 1033, 1023, 1069, 956, 1012, 1018, 1045, 1011, 981, 1041, 975, 1029, 990, 1017, 995, 1051, 1049, 1000, 1004, 1063, 1009, 1063, 1026, 1045, 1027, 962, 1023, 1045, 1017, 1043, 1040, 1025, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[937, 1024, 1059, 1032, 997, 1023, 1008, 1051, 1037, 1025, 1008, 996, 1016, 979, 1072, 1045, 982, 1019, 1026, 1071, 1042, 1015, 1003, 1001, 977, 1002, 1008, 1061, 1052, 1034, 1053, 986, 1095, 1024, 951, 1049, 1000, 1039, 1068, 1029, 1048, 1021, 982, 1024, 1021, 1043, 1047, 985, 999, 1051, 1034, 1020, 1003, 1002, 1029, 1040, 992, 1079, 1018, 1086, 1034, 1018, 1009, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1043, 1009, 1005, 1066, 1008, 1028, 1001, 1022, 1019, 1058, 1022, 1030, 961, 1029, 1035, 1064, 1027, 995, 1036, 938, 1037, 1001, 1048, 1030, 1049, 1059, 1063, 992, 1008, 1043, 1008, 1019, 1012, 1013, 1011, 1031, 1036, 952, 1068, 1012, 1025, 995, 1024, 1033, 1026, 1078, 1042, 1011, 1030, 998, 1015, 1054, 1037, 1034, 989, 1074, 1019, 997, 992, 991, 1049, 1051, 1033, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1024, 988, 1010, 1056, 1046, 1028, 1035, 966, 24, 0, 0, 0, 0, 0, 0, 0, 1007, 1042, 1014, 1067, 991, 988, 982, 1085, 15, 4, 0, 0, 0, 0, 0, 0, 1030, 1022, 1056, 1006, 1028, 1035, 1042, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1035, 1060, 1008, 1039, 1005, 1007, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 998, 984, 1058, 1033, 1067, 1020, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1011, 1001, 959, 1037, 1062, 1013, 992, 8, 6, 4, 0, 0, 0, 0, 0, 1022, 1042, 1001, 1008, 1059, 1005, 1000, 994, 1, 0, 0, 0, 0, 0, 0, 0, 984, 1024, 1033, 1042, 1037, 1015, 1056, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8195, 8214, 8187, 8304, 8128, 8132, 8199] avg=8192.0000 max=8304 max_rank=4 min=8128 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1084, 988, 1007, 972, 1006, 1033, 1006, 1066, 24, 0, 0, 0, 0, 0, 0, 0, 1006, 1023, 1010, 997, 1028, 1006, 1015, 992, 15, 4, 0, 0, 0, 0, 0, 0, 975, 1051, 1016, 1007, 1052, 1007, 1048, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1060, 1048, 989, 1032, 994, 978, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1023, 1016, 1060, 992, 984, 1092, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 981, 985, 998, 1042, 1033, 1038, 1037, 1018, 8, 6, 4, 0, 0, 0, 0, 0, 997, 991, 1036, 1044, 1003, 1006, 1075, 1077, 1, 0, 0, 0, 0, 0, 0, 0, 1020, 968, 1035, 1057, 1038, 1024, 1014, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8096, 8235, 8206, 8257, 8150, 8230, 8176] avg=8192.0000 max=8257 max_rank=4 min=8096 min_rank=1 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1034, 1065, 1047, 974, 1043, 1001, 1017, 996, 24, 0, 0, 0, 0, 0, 0, 0, 1042, 1013, 1073, 1003, 1022, 990, 1049, 1027, 15, 4, 0, 0, 0, 0, 0, 0, 1047, 980, 1050, 1010, 1033, 992, 978, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1019, 1000, 1028, 1048, 1026, 1044, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 996, 989, 969, 1024, 991, 1041, 1037, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 977, 1037, 993, 1027, 1004, 1044, 996, 8, 6, 4, 0, 0, 0, 0, 0, 1015, 1024, 1073, 1012, 1040, 1060, 1054, 1016, 1, 0, 0, 0, 0, 0, 0, 0, 1002, 1009, 1096, 1017, 1002, 1015, 1066, 985, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8238, 8185, 8228, 8087, 8110, 8295, 8192] avg=8192.0000 max=8295 max_rank=6 min=8087 min_rank=4 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[998, 1032, 1023, 1058, 1020, 1036, 1025, 1037, 24, 0, 0, 0, 0, 0, 0, 0, 983, 1032, 1014, 1019, 1054, 1039, 976, 1029, 15, 4, 0, 0, 0, 0, 0, 0, 1023, 1051, 1075, 1023, 1011, 1022, 978, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1032, 1026, 1085, 948, 1010, 1046, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1069, 956, 1012, 1018, 1045, 1006, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 975, 1029, 990, 1017, 995, 1051, 1049, 7, 6, 4, 0, 0, 0, 0, 0, 1000, 1004, 1063, 1009, 1063, 1026, 1045, 1027, 1, 0, 0, 0, 0, 0, 0, 0, 962, 1023, 1045, 1017, 1043, 1040, 1025, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8165, 8229, 8210, 8110, 8164, 8238, 8167] avg=8192.0000 max=8253 max_rank=0 min=8110 min_rank=4 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1002, 1044, 1025, 1006, 1047, 1011, 1038, 1031, 29, 0, 0, 0, 0, 0, 0, 0, 1022, 1035, 1005, 1049, 1022, 993, 1066, 1002, 21, 8, 0, 0, 0, 0, 0, 0, 1071, 1005, 982, 1022, 1064, 1023, 961, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1002, 1012, 995, 1073, 990, 1066, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1026, 1018, 1043, 1066, 1030, 1004, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1053, 1032, 1022, 1088, 1017, 969, 1026, 14, 11, 6, 0, 0, 0, 0, 0, 993, 1043, 1018, 1044, 980, 992, 994, 994, 6, 0, 0, 0, 0, 0, 0, 0, 1044, 994, 1009, 1036, 992, 1073, 1054, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8223, 8134, 8171, 8229, 8273, 8064, 8209] avg=8192.0000 max=8273 max_rank=5 min=8064 min_rank=6 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1023, 1027, 985, 1068, 991, 984, 1025, 999, 23, 0, 0, 0, 0, 0, 0, 0, 997, 1049, 1031, 1004, 1002, 1054, 1016, 1013, 15, 4, 0, 0, 0, 0, 0, 0, 1041, 1032, 1002, 1012, 1052, 1024, 1072, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1024, 999, 1043, 1020, 1049, 1018, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1068, 1033, 998, 1039, 1011, 1007, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 1002, 1017, 1051, 1016, 1013, 1036, 1028, 8, 6, 4, 0, 0, 0, 0, 0, 1024, 1038, 1050, 988, 1041, 1019, 1018, 987, 1, 0, 0, 0, 0, 0, 0, 0, 1051, 1080, 1055, 983, 1014, 1049, 1005, 960, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8185, 8280, 8144, 8169, 8270, 8166, 8197] avg=8192.0000 max=8280 max_rank=2 min=8125 min_rank=0 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[937, 1024, 1059, 1032, 997, 1023, 1008, 1051, 24, 0, 0, 0, 0, 0, 0, 0, 1037, 1025, 1008, 996, 1016, 979, 1072, 1045, 15, 4, 0, 0, 0, 0, 0, 0, 982, 1019, 1026, 1071, 1036, 1015, 1003, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1002, 1008, 1061, 1040, 1034, 1053, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 1024, 951, 1049, 1000, 1039, 1063, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1021, 982, 1024, 1021, 1043, 1047, 985, 8, 6, 4, 0, 0, 0, 0, 0, 999, 1051, 1034, 1020, 1003, 1002, 1029, 1040, 1, 0, 0, 0, 0, 0, 0, 0, 992, 1079, 1018, 1086, 1034, 1018, 1009, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8197, 8153, 8161, 8250, 8189, 8179, 8252] avg=8192.0000 max=8252 max_rank=7 min=8153 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1043, 1009, 1005, 1066, 1008, 1028, 1001, 1022, 24, 0, 0, 0, 0, 0, 0, 0, 1019, 1058, 1022, 1030, 961, 1029, 1035, 1064, 15, 4, 0, 0, 0, 0, 0, 0, 1027, 995, 1036, 938, 1031, 1001, 1048, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1059, 1063, 992, 996, 1043, 1008, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1013, 1011, 1031, 1036, 952, 1063, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 995, 1024, 1033, 1026, 1078, 1042, 1011, 8, 6, 4, 0, 0, 0, 0, 0, 1030, 998, 1015, 1054, 1037, 1034, 989, 1074, 1, 0, 0, 0, 0, 0, 0, 0, 1019, 997, 992, 991, 1049, 1051, 1033, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8237, 8106, 8229, 8130, 8252, 8232, 8144] avg=8192.0000 max=8252 max_rank=5 min=8106 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8064, 8192, 8192, 8320, 8320, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1008, 1029, 1024, 1063, 986, 1046, 1025, 1009, 1030, 1018, 1044, 993, 1009, 1018, 959, 999, 962, 1027, 1032, 1042, 1008, 1025, 1031, 1015, 1015, 1051, 1003, 998, 1028, 1048, 1029, 1033, 1018, 1060, 1015, 1023, 1042, 1015, 1005, 1049, 1003, 1061, 1063, 1039, 1051, 1027, 1017, 1061, 1043, 986, 1032, 987, 1015, 1020, 1060, 980, 1014, 1037, 1049, 1052, 1032, 1013, 1031, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[963, 979, 1074, 1031, 1023, 1007, 1006, 1019, 1012, 1015, 1002, 1064, 1006, 1006, 997, 980, 1048, 1080, 1017, 1013, 1010, 1048, 993, 987, 1037, 1008, 1003, 1018, 1028, 1026, 1035, 1041, 1049, 1052, 1054, 1012, 1017, 1044, 1012, 1033, 1023, 1058, 1044, 1028, 1009, 941, 997, 1064, 1025, 1007, 1043, 1049, 1013, 1049, 1034, 1041, 1037, 1013, 1058, 1008, 1058, 1034, 1052, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[986, 982, 1077, 975, 982, 1004, 997, 1006, 1026, 999, 1005, 1040, 1044, 1041, 1015, 1018, 1034, 1044, 1060, 998, 1019, 1023, 1045, 1056, 977, 1035, 993, 1029, 1046, 1012, 990, 1013, 1000, 1071, 1040, 1059, 994, 1058, 1004, 1025, 992, 1021, 1091, 1011, 1062, 1030, 1004, 1079, 1056, 1020, 1030, 986, 1027, 1046, 1014, 1055, 1022, 1027, 1008, 1087, 1016, 1043, 986, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1050, 996, 1031, 1084, 990, 1040, 995, 973, 999, 1041, 1007, 990, 1011, 1034, 1002, 1072, 1052, 1031, 1003, 1016, 1049, 1025, 1013, 1049, 1020, 1058, 1078, 1044, 1020, 1012, 996, 1029, 1018, 1004, 1024, 1006, 1000, 1025, 1016, 1037, 959, 999, 1067, 1005, 1023, 1063, 1016, 1039, 1041, 1010, 1017, 1008, 1037, 1023, 1035, 1008, 1014, 1050, 999, 1048, 1032, 1061, 1040, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1081, 1047, 1064, 1051, 995, 957, 1041, 1060, 1058, 1017, 1014, 1010, 1081, 1015, 1034, 975, 1015, 1020, 1009, 1025, 1029, 1033, 987, 982, 1086, 1016, 1049, 1036, 1011, 1019, 976, 1076, 1049, 989, 1032, 1046, 1040, 1020, 982, 1012, 1022, 1034, 1030, 1045, 987, 998, 1054, 1031, 1035, 1020, 1042, 987, 1012, 1012, 1027, 1040, 1038, 1048, 1007, 1013, 951, 1002, 1044, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1024, 1047, 985, 1057, 1001, 1020, 1005, 1037, 1023, 1066, 997, 1027, 986, 1071, 985, 1079, 1007, 1052, 986, 998, 953, 1078, 1000, 1067, 1069, 1011, 1008, 1044, 989, 1007, 1024, 1038, 1014, 1052, 1034, 1014, 1013, 1067, 994, 1021, 975, 1056, 1074, 975, 962, 1002, 1055, 1001, 1069, 1096, 1051, 1012, 1028, 1031, 1049, 985, 1004, 1013, 1033, 993, 971, 1021, 1047, 1083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1062, 996, 1087, 992, 1044, 999, 1029, 1031, 1070, 1029, 1007, 1011, 1039, 1067, 1071, 1032, 990, 1026, 996, 1014, 1100, 974, 1002, 966, 1043, 1056, 1014, 1019, 1041, 980, 998, 1012, 1077, 1011, 1010, 1025, 1050, 1067, 1074, 1003, 1025, 997, 970, 996, 1036, 1035, 1060, 1035, 1030, 1049, 1015, 989, 1036, 991, 1057, 1052, 985, 1026, 1010, 1037, 998, 980, 989, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[989, 1003, 1017, 1058, 1028, 1072, 1071, 1044, 1064, 1028, 1010, 1050, 975, 979, 1017, 1034, 1093, 993, 1000, 1004, 1016, 1045, 1044, 1003, 1011, 1006, 1002, 1027, 1058, 1025, 994, 984, 1016, 1072, 1009, 985, 1008, 988, 1007, 1054, 1040, 1013, 1022, 1039, 1025, 1005, 1038, 1101, 1028, 1080, 1029, 1049, 1022, 1042, 1061, 981, 1009, 1026, 1042, 1044, 1008, 1048, 924, 977] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1008, 1029, 1024, 1063, 986, 1046, 1025, 1009, 10, 0, 0, 0, 0, 0, 0, 0, 1030, 1018, 1044, 993, 1009, 1018, 959, 999, 13, 1, 0, 0, 0, 0, 0, 0, 962, 1027, 1032, 1042, 1008, 1025, 1031, 1015, 22, 0, 0, 0, 0, 0, 0, 0, 1015, 1051, 1003, 998, 1028, 1048, 1029, 1033, 6, 0, 0, 0, 0, 0, 0, 0, 1018, 1038, 1015, 1023, 1042, 1015, 1005, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1061, 1063, 1039, 1051, 1027, 1017, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 986, 1032, 987, 1015, 1020, 1027, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1037, 1049, 1052, 1032, 1013, 1031, 1029, 11, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8084, 8164, 8211, 8205, 8305, 8090, 8277] avg=8192.0000 max=8305 max_rank=5 min=8084 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[963, 979, 1074, 1031, 1023, 1007, 1006, 1019, 10, 0, 0, 0, 0, 0, 0, 0, 1012, 1015, 1002, 1064, 1006, 1006, 997, 980, 13, 1, 0, 0, 0, 0, 0, 0, 1048, 1080, 1017, 1013, 1010, 1048, 993, 987, 21, 0, 0, 0, 0, 0, 0, 0, 1037, 1008, 1003, 1018, 1028, 1026, 1035, 1041, 6, 0, 0, 0, 0, 0, 0, 0, 1049, 1030, 1054, 1012, 1017, 1044, 1012, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1058, 1044, 1028, 1009, 941, 997, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1007, 1043, 1049, 1013, 1049, 1002, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1013, 1058, 1008, 1058, 1034, 1052, 1002, 11, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8096, 8217, 8202, 8251, 8147, 8229, 8282] avg=8192.0000 max=8282 max_rank=7 min=8096 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1050, 996, 1031, 1084, 990, 1040, 995, 973, 10, 0, 0, 0, 0, 0, 0, 0, 999, 1041, 1007, 990, 1011, 1034, 1002, 1072, 12, 1, 0, 0, 0, 0, 0, 0, 1052, 1031, 1003, 1016, 1049, 1025, 1013, 1049, 21, 0, 0, 0, 0, 0, 0, 0, 1020, 1058, 1078, 1044, 1020, 1012, 996, 1029, 6, 0, 0, 0, 0, 0, 0, 0, 1018, 984, 1024, 1006, 1000, 1025, 1016, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 959, 999, 1067, 1005, 1023, 1063, 1016, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1010, 1017, 1008, 1037, 1023, 1003, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1050, 999, 1048, 1032, 1061, 1040, 1002, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8169, 8259, 8263, 8110, 8154, 8147, 8265] avg=8192.0000 max=8265 max_rank=7 min=8110 min_rank=4 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[986, 982, 1077, 975, 982, 1004, 997, 1006, 10, 0, 0, 0, 0, 0, 0, 0, 1026, 999, 1005, 1040, 1044, 1041, 1015, 1018, 13, 1, 0, 0, 0, 0, 0, 0, 1034, 1044, 1060, 998, 1019, 1023, 1045, 1056, 21, 0, 0, 0, 0, 0, 0, 0, 977, 1035, 993, 1029, 1046, 1012, 990, 1013, 6, 0, 0, 0, 0, 0, 0, 0, 1000, 1049, 1040, 1059, 994, 1058, 1004, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1021, 1091, 1011, 1062, 1030, 1004, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1020, 1030, 986, 1027, 1046, 982, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1027, 1008, 1087, 1016, 1043, 986, 1001, 11, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8019, 8202, 8300, 8101, 8229, 8273, 8202, 8210] avg=8192.0000 max=8300 max_rank=2 min=8019 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1024, 1047, 985, 1057, 1001, 1020, 1005, 1037, 9, 0, 0, 0, 0, 0, 0, 0, 1023, 1066, 997, 1027, 986, 1071, 985, 1079, 13, 1, 0, 0, 0, 0, 0, 0, 1007, 1052, 986, 998, 953, 1078, 1000, 1067, 21, 0, 0, 0, 0, 0, 0, 0, 1069, 1011, 1008, 1044, 989, 1007, 1024, 1038, 6, 0, 0, 0, 0, 0, 0, 0, 1014, 1030, 1034, 1014, 1013, 1067, 994, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1056, 1074, 975, 962, 1002, 1055, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1096, 1051, 1012, 1028, 1031, 1017, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1013, 1033, 993, 971, 1021, 1047, 1083, 11, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8248, 8162, 8196, 8187, 8084, 8289, 8185] avg=8192.0000 max=8289 max_rank=6 min=8084 min_rank=5 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1081, 1047, 1064, 1051, 995, 957, 1041, 1060, 13, 0, 0, 0, 0, 0, 0, 0, 1058, 1017, 1014, 1010, 1081, 1015, 1034, 975, 17, 6, 0, 0, 0, 0, 0, 0, 1015, 1020, 1009, 1025, 1029, 1033, 987, 982, 24, 0, 0, 0, 0, 0, 0, 0, 1086, 1016, 1049, 1036, 1011, 1019, 976, 1076, 11, 0, 0, 0, 0, 0, 0, 0, 1049, 960, 1032, 1046, 1040, 1020, 982, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1034, 1030, 1045, 987, 998, 1054, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1020, 1042, 987, 1012, 1012, 986, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1048, 1007, 1013, 951, 1002, 1044, 1018, 17, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8309, 8227, 8124, 8280, 8141, 8171, 8134, 8150] avg=8192.0000 max=8309 max_rank=0 min=8124 min_rank=2 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1062, 996, 1087, 992, 1044, 999, 1029, 1031, 10, 0, 0, 0, 0, 0, 0, 0, 1070, 1029, 1007, 1011, 1039, 1067, 1071, 1032, 12, 1, 0, 0, 0, 0, 0, 0, 990, 1026, 996, 1014, 1100, 974, 1002, 966, 22, 0, 0, 0, 0, 0, 0, 0, 1043, 1056, 1014, 1019, 1041, 980, 998, 1012, 6, 0, 0, 0, 0, 0, 0, 0, 1077, 990, 1010, 1025, 1050, 1067, 1074, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 997, 970, 996, 1036, 1035, 1060, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1049, 1015, 989, 1036, 991, 1024, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1026, 1010, 1037, 998, 980, 989, 1024, 11, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8339, 8090, 8169, 8296, 8137, 8186, 8069] avg=8192.0000 max=8339 max_rank=1 min=8069 min_rank=7 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[989, 1003, 1017, 1058, 1028, 1072, 1071, 1044, 10, 0, 0, 0, 0, 0, 0, 0, 1064, 1028, 1010, 1050, 975, 979, 1017, 1034, 13, 1, 0, 0, 0, 0, 0, 0, 1093, 993, 1000, 1004, 1016, 1045, 1044, 1003, 22, 0, 0, 0, 0, 0, 0, 0, 1011, 1006, 1002, 1027, 1058, 1025, 994, 984, 7, 0, 0, 0, 0, 0, 0, 0, 1016, 1050, 1009, 985, 1008, 988, 1007, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1013, 1022, 1039, 1025, 1005, 1038, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1080, 1029, 1049, 1022, 1042, 1028, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1026, 1042, 1044, 1008, 1048, 924, 977, 11, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8292, 8171, 8220, 8114, 8117, 8265, 8259, 8098] avg=8192.0000 max=8292 max_rank=0 min=8098 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8448, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8064, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8448, 8320, 8064, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1070, 1055, 997, 970, 1027, 1019, 994, 982, 996, 1049, 1018, 1003, 1047, 995, 1022, 1016, 1041, 1060, 981, 1023, 1011, 1028, 996, 1061, 1013, 970, 998, 1043, 1041, 1052, 1054, 1013, 1019, 1035, 1035, 1064, 1017, 989, 1019, 1042, 1050, 1061, 1049, 1037, 1021, 1046, 1026, 1084, 1003, 1021, 1004, 1066, 1015, 1014, 1050, 1013, 1012, 1026, 1016, 1049, 1048, 973, 990, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1067, 998, 1025, 1003, 1034, 1022, 1018, 1012, 996, 1037, 1041, 1021, 1021, 1086, 1000, 1039, 1029, 989, 1073, 1028, 1069, 999, 1004, 1032, 996, 1029, 1020, 1063, 1023, 1020, 998, 1019, 1045, 1030, 1051, 1030, 998, 1028, 1040, 1011, 1031, 1004, 1019, 1038, 1044, 998, 1081, 1032, 957, 1005, 1033, 1050, 998, 1004, 964, 1026, 1034, 1045, 1020, 1024, 1021, 1017, 1013, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[985, 1000, 1003, 1058, 1038, 1042, 991, 1062, 1046, 1030, 1045, 1014, 1047, 1085, 1013, 1016, 1009, 984, 961, 1020, 1007, 1026, 1004, 1023, 1029, 1011, 1002, 999, 1086, 1016, 1035, 1069, 1058, 1031, 1021, 992, 1005, 1014, 1032, 995, 954, 1020, 1049, 1060, 1072, 1051, 1049, 1043, 1006, 1062, 1039, 1043, 1038, 999, 1039, 1011, 1004, 1052, 981, 1042, 986, 975, 1004, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1012, 1007, 1050, 1045, 1023, 1018, 1082, 1022, 988, 981, 1023, 1034, 1005, 1010, 1042, 1097, 1020, 1060, 1009, 992, 1050, 995, 1049, 1012, 1027, 1043, 1019, 1014, 1018, 1059, 1038, 982, 1034, 1037, 1014, 1033, 1032, 1027, 1021, 1037, 1040, 1008, 1022, 967, 1050, 1052, 1046, 1016, 1003, 1023, 1057, 1004, 1034, 1042, 1020, 992, 1030, 1044, 996, 1046, 998, 990, 990, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1040, 1029, 1038, 1068, 948, 1067, 1038, 1006, 1038, 1026, 1035, 979, 1048, 1077, 1033, 1068, 926, 1024, 1069, 1018, 1008, 1011, 1015, 1037, 1015, 1022, 999, 1037, 993, 1003, 1000, 1022, 990, 979, 988, 1023, 1030, 1045, 1026, 992, 1011, 1051, 1051, 1051, 955, 996, 995, 1036, 1088, 1019, 1013, 1015, 1094, 1035, 1032, 996, 1049, 1051, 1013, 996, 1031, 1055, 1060, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1052, 1016, 1006, 1024, 1020, 1032, 1010, 971, 1039, 1017, 966, 999, 1038, 1058, 1043, 997, 954, 1022, 1072, 994, 1011, 1020, 980, 1053, 1033, 1064, 1025, 1043, 1055, 1074, 1036, 1059, 976, 1048, 1022, 1029, 1056, 1008, 1031, 1066, 1048, 1003, 1055, 973, 1038, 1054, 1044, 1002, 1022, 958, 996, 1010, 1002, 991, 1033, 1020, 1045, 1042, 1007, 1009, 1062, 1022, 1056, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1000, 1017, 1046, 1006, 1031, 1031, 1044, 1033, 1039, 1005, 1015, 1001, 1066, 987, 1039, 995, 1043, 984, 1005, 1027, 1043, 1044, 1061, 998, 1007, 1019, 1037, 1042, 968, 1005, 1016, 1038, 987, 1075, 1026, 1045, 1042, 994, 1071, 1013, 997, 1048, 1038, 1017, 1057, 1047, 1034, 1027, 1006, 1040, 939, 1035, 1054, 1002, 1078, 1003, 1064, 989, 1000, 1043, 1051, 1002, 1029, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1046, 970, 1056, 1005, 1046, 1013, 1016, 1039, 989, 1040, 1018, 1025, 1011, 1055, 1061, 1034, 1061, 961, 963, 1063, 1006, 975, 985, 1040, 1057, 980, 1087, 1044, 1045, 1031, 1067, 1014, 1038, 1025, 1005, 1044, 992, 1027, 1031, 990, 1053, 975, 1041, 1048, 1054, 1049, 1026, 1023, 1002, 1041, 1037, 979, 1038, 997, 1031, 1012, 1013, 1019, 1061, 990, 994, 987, 1044, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1070, 1055, 997, 970, 1027, 1019, 994, 982, 3, 1, 0, 0, 0, 0, 0, 0, 996, 1049, 1018, 1003, 1047, 976, 1022, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1060, 981, 1023, 1011, 1028, 996, 1061, 52, 0, 0, 0, 0, 0, 0, 0, 1013, 970, 998, 1018, 1041, 1052, 1054, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1035, 1035, 1064, 1017, 989, 1018, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1061, 990, 1037, 1021, 1046, 1026, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1021, 1004, 1066, 1015, 1014, 1050, 1013, 25, 7, 2, 0, 0, 0, 0, 0, 1012, 1026, 1016, 1049, 1048, 973, 990, 997, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8127, 8253, 8159, 8219, 8315, 8220, 8125] avg=8192.0000 max=8315 max_rank=5 min=8118 min_rank=0 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32879 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1067, 998, 1025, 1003, 1034, 1022, 1018, 1012, 3, 1, 0, 0, 0, 0, 0, 0, 996, 1037, 1041, 1021, 1021, 1066, 1000, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 989, 1073, 1028, 1069, 999, 1004, 1032, 51, 0, 0, 0, 0, 0, 0, 0, 996, 1029, 1020, 1038, 1023, 1020, 998, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1030, 1051, 1030, 998, 1028, 1039, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1004, 961, 1038, 1044, 998, 1081, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 957, 1005, 1033, 1050, 998, 1004, 964, 1026, 25, 7, 2, 0, 0, 0, 0, 0, 1034, 1045, 1020, 1024, 1021, 1017, 1013, 1034, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8221, 8274, 8143, 8232, 8189, 8071, 8223] avg=8192.0000 max=8274 max_rank=2 min=8071 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[985, 1000, 1003, 1058, 1038, 1042, 991, 1062, 3, 1, 0, 0, 0, 0, 0, 0, 1046, 1030, 1045, 1014, 1047, 1065, 1013, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 984, 961, 1020, 1007, 1026, 1004, 1023, 52, 0, 0, 0, 0, 0, 0, 0, 1029, 1011, 1002, 975, 1086, 1016, 1035, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1031, 1021, 992, 1005, 1014, 1031, 995, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1020, 990, 1060, 1072, 1051, 1049, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1062, 1039, 1043, 1038, 999, 1039, 1011, 24, 7, 2, 0, 0, 0, 0, 0, 1004, 1052, 981, 1042, 986, 975, 1004, 1053, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8276, 8086, 8223, 8147, 8239, 8270, 8112] avg=8192.0000 max=8276 max_rank=1 min=8086 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1012, 1007, 1050, 1045, 1023, 1018, 1082, 1022, 3, 1, 0, 0, 0, 0, 0, 0, 988, 981, 1023, 1034, 1005, 991, 1042, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1060, 1009, 992, 1050, 995, 1049, 1012, 51, 0, 0, 0, 0, 0, 0, 0, 1027, 1043, 1019, 990, 1018, 1059, 1038, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1037, 1014, 1033, 1032, 1027, 1020, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1008, 964, 967, 1050, 1052, 1046, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1023, 1057, 1004, 1034, 1042, 1020, 992, 24, 7, 2, 0, 0, 0, 0, 0, 1030, 1044, 996, 1046, 998, 990, 990, 1005, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8161, 8238, 8176, 8234, 8143, 8208, 8113] avg=8192.0000 max=8263 max_rank=0 min=8113 min_rank=7 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1052, 1016, 1006, 1024, 1020, 1032, 1010, 971, 3, 1, 0, 0, 0, 0, 0, 0, 1039, 1017, 966, 999, 1038, 1038, 1043, 997, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1022, 1072, 994, 1011, 1020, 980, 1053, 52, 0, 0, 0, 0, 0, 0, 0, 1033, 1064, 1025, 1018, 1055, 1074, 1036, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1048, 1022, 1029, 1056, 1008, 1030, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1003, 996, 973, 1038, 1054, 1044, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 958, 996, 1010, 1002, 991, 1033, 1020, 25, 7, 2, 0, 0, 0, 0, 0, 1045, 1042, 1007, 1009, 1062, 1022, 1056, 1025, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8137, 8158, 8364, 8235, 8158, 8066, 8283] avg=8192.0000 max=8364 max_rank=3 min=8066 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1040, 1029, 1038, 1068, 948, 1067, 1038, 1006, 3, 1, 0, 0, 0, 0, 0, 0, 1038, 1026, 1035, 979, 1048, 1057, 1033, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 926, 1024, 1069, 1018, 1008, 1011, 1015, 1037, 52, 0, 0, 0, 0, 0, 0, 0, 1015, 1022, 999, 1012, 993, 1003, 1000, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 990, 979, 988, 1023, 1030, 1045, 1025, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1051, 992, 1051, 955, 996, 995, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 1019, 1013, 1015, 1094, 1035, 1032, 996, 25, 7, 2, 0, 0, 0, 0, 0, 1049, 1051, 1013, 996, 1031, 1055, 1060, 1033, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8284, 8160, 8066, 8072, 8087, 8326, 8303] avg=8192.0000 max=8326 max_rank=6 min=8066 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1000, 1017, 1046, 1006, 1031, 1031, 1044, 1033, 3, 1, 0, 0, 0, 0, 0, 0, 1039, 1005, 1015, 1001, 1066, 968, 1039, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 984, 1005, 1027, 1043, 1044, 1061, 998, 52, 0, 0, 0, 0, 0, 0, 0, 1007, 1019, 1037, 1017, 968, 1005, 1016, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1075, 1026, 1045, 1042, 994, 1070, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1048, 979, 1017, 1057, 1047, 1034, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1040, 939, 1035, 1054, 1002, 1078, 1003, 25, 7, 2, 0, 0, 0, 0, 0, 1064, 989, 1000, 1043, 1051, 1002, 1029, 991, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8128, 8257, 8107, 8252, 8206, 8191, 8183] avg=8192.0000 max=8257 max_rank=2 min=8107 min_rank=3 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1046, 970, 1056, 1005, 1046, 1013, 1016, 1039, 6, 7, 0, 0, 0, 0, 0, 0, 989, 1040, 1018, 1025, 1011, 1024, 1061, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 961, 963, 1063, 1006, 975, 985, 1040, 56, 0, 0, 0, 0, 0, 0, 0, 1057, 980, 1087, 1017, 1045, 1031, 1067, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1025, 1005, 1044, 992, 1027, 1024, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 975, 971, 1048, 1054, 1049, 1026, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1041, 1037, 979, 1038, 997, 1031, 1012, 27, 14, 6, 0, 0, 0, 0, 0, 1013, 1019, 1061, 990, 994, 987, 1044, 1067, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8202, 8110, 8298, 8145, 8199, 8184, 8194] avg=8192.0000 max=8298 max_rank=3 min=8110 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8192, 7936, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8064, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1073, 985, 986, 1052, 999, 1018, 1022, 1041, 1046, 1021, 1052, 1018, 1003, 1007, 1019, 991, 1034, 1021, 991, 1017, 1040, 1059, 1000, 987, 994, 1029, 1069, 1022, 1039, 1028, 982, 1038, 1029, 1024, 1009, 1011, 1003, 1050, 1058, 1019, 1040, 1030, 1003, 1021, 1019, 1016, 1007, 1043, 1043, 1017, 983, 1037, 1041, 1053, 1007, 1028, 1017, 1024, 1035, 1069, 1009, 1049, 1000, 1059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1036, 1072, 1050, 1018, 1011, 1035, 1019, 1014, 1064, 995, 999, 1018, 998, 1012, 1063, 1024, 1060, 1043, 990, 1076, 965, 1013, 1013, 1019, 960, 943, 1037, 1005, 1005, 1010, 1044, 1018, 1005, 1019, 1036, 1003, 1024, 1004, 1010, 1003, 1063, 1050, 1010, 1035, 1037, 1044, 1039, 1027, 1015, 1030, 1018, 994, 1060, 1072, 1017, 1062, 1040, 1042, 1047, 989, 1058, 1006, 1013, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1000, 1051, 1000, 1010, 1038, 1042, 1030, 1007, 988, 1038, 1051, 983, 1016, 1033, 1031, 1037, 1032, 1007, 1065, 1044, 1016, 1080, 1010, 990, 1032, 997, 1040, 996, 1032, 1008, 1019, 993, 991, 1033, 998, 1049, 1050, 1012, 1011, 1011, 1086, 1058, 1084, 1028, 1024, 1040, 954, 1020, 980, 1003, 1032, 1072, 1007, 1050, 1016, 1031, 1040, 1035, 1002, 991, 1002, 1037, 1069, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1034, 998, 1026, 985, 1041, 976, 1033, 1045, 1013, 1066, 1053, 990, 1040, 1011, 1045, 975, 1045, 1021, 994, 1029, 1013, 1023, 1046, 1061, 997, 975, 1024, 1047, 1040, 1033, 1019, 1078, 1021, 1007, 968, 1042, 1016, 992, 1079, 1022, 1009, 1019, 1002, 974, 1075, 1037, 1075, 996, 1022, 1059, 1091, 1022, 1087, 986, 1037, 1051, 1008, 1015, 986, 1051, 1013, 1016, 1015, 967] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1022, 1010, 1003, 1022, 1023, 1037, 1028, 1009, 1022, 965, 1050, 1018, 976, 988, 1016, 995, 994, 1043, 1029, 1014, 1064, 979, 1051, 993, 1051, 1038, 986, 1029, 972, 1076, 1040, 1030, 962, 1040, 1038, 1055, 1070, 967, 1028, 991, 1017, 1067, 1037, 997, 1021, 1034, 1033, 996, 1057, 1015, 1046, 1066, 1007, 1081, 1027, 1048, 1014, 989, 1019, 1026, 1053, 1067, 1047, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1043, 1022, 1018, 1012, 1018, 1026, 1017, 1024, 1020, 1022, 1038, 1036, 1045, 1028, 1048, 1052, 996, 992, 1049, 1062, 1058, 1032, 1036, 1006, 1012, 1030, 1019, 1040, 1017, 993, 1021, 998, 1015, 991, 1026, 1045, 1012, 1037, 1083, 969, 1022, 1012, 1056, 1076, 1035, 974, 1051, 1038, 1028, 971, 1023, 951, 1050, 996, 1040, 1049, 1011, 1026, 1044, 1044, 1044, 1047, 994, 946] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1037, 1045, 982, 1014, 1009, 1019, 987, 1044, 1021, 1002, 1006, 1029, 1003, 1041, 1034, 1053, 1047, 1013, 1040, 1054, 1059, 1073, 999, 1027, 1039, 1004, 996, 996, 1025, 1033, 1061, 1009, 987, 939, 1028, 1047, 1054, 1022, 1002, 1075, 1030, 996, 1017, 1045, 1020, 1029, 975, 1012, 1024, 1028, 1098, 1010, 1093, 1032, 1014, 1021, 1024, 1059, 991, 988, 975, 1036, 1012, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1009, 1032, 1031, 1023, 1049, 1039, 1022, 1021, 1025, 1071, 1002, 1059, 990, 1018, 1027, 1003, 994, 1047, 1016, 1077, 1058, 973, 1028, 1021, 1067, 1041, 1018, 992, 1035, 988, 1027, 998, 994, 1030, 1017, 1049, 1002, 1030, 984, 982, 1031, 1039, 1053, 996, 1006, 1049, 1051, 1005, 1035, 995, 1066, 1066, 1043, 968, 1037, 1044, 1017, 1036, 1022, 1055, 1028, 1054, 966, 985] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1073, 985, 986, 1052, 999, 1018, 1022, 1041, 11, 0, 0, 0, 0, 0, 0, 0, 1046, 1021, 1052, 1018, 1003, 1007, 1019, 991, 13, 3, 0, 0, 0, 0, 0, 0, 1034, 1021, 991, 995, 1040, 1059, 1000, 987, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1029, 1069, 1022, 1039, 1028, 982, 1038, 19, 17, 0, 0, 0, 0, 0, 0, 1029, 1024, 1009, 1011, 1003, 1050, 1058, 1019, 44, 0, 0, 0, 0, 0, 0, 0, 1010, 1030, 1003, 1021, 1019, 1016, 1007, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1017, 983, 1037, 978, 1053, 1007, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1024, 1035, 1069, 1009, 1049, 1000, 1059, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8173, 8127, 8237, 8247, 8149, 8146, 8270] avg=8192.0000 max=8270 max_rank=7 min=8127 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1036, 1072, 1050, 1018, 1011, 1035, 1019, 1014, 11, 0, 0, 0, 0, 0, 0, 0, 1064, 995, 999, 1018, 998, 1012, 1063, 1024, 13, 3, 0, 0, 0, 0, 0, 0, 1060, 1043, 990, 1053, 965, 1013, 1013, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 960, 943, 1037, 1005, 1005, 1010, 1044, 1018, 20, 18, 0, 0, 0, 0, 0, 0, 1005, 1019, 1036, 1003, 1024, 1004, 1010, 1003, 44, 0, 0, 0, 0, 0, 0, 0, 1032, 1050, 1010, 1035, 1037, 1044, 1039, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1030, 1018, 994, 996, 1072, 1017, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1042, 1047, 989, 1058, 1006, 1013, 1035, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8189, 8156, 8060, 8148, 8274, 8204, 8239] avg=8192.0000 max=8274 max_rank=5 min=8060 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1000, 1051, 1000, 1010, 1038, 1042, 1030, 1007, 11, 0, 0, 0, 0, 0, 0, 0, 988, 1038, 1051, 983, 1016, 1033, 1031, 1037, 14, 3, 0, 0, 0, 0, 0, 0, 1032, 1007, 1065, 1021, 1016, 1080, 1010, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 997, 1040, 996, 1032, 1008, 1019, 993, 19, 18, 0, 0, 0, 0, 0, 0, 991, 1033, 998, 1049, 1050, 1012, 1011, 1011, 42, 0, 0, 0, 0, 0, 0, 0, 1054, 1058, 1084, 1028, 1024, 1040, 954, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1003, 1032, 1072, 946, 1050, 1016, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1035, 1002, 991, 1002, 1037, 1069, 1004, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8194, 8221, 8154, 8197, 8262, 8130, 8189] avg=8192.0000 max=8262 max_rank=5 min=8130 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1022, 1010, 1003, 1022, 1023, 1037, 1028, 1009, 11, 0, 0, 0, 0, 0, 0, 0, 1022, 965, 1050, 1018, 976, 988, 1016, 995, 13, 3, 0, 0, 0, 0, 0, 0, 994, 1043, 1029, 992, 1064, 979, 1051, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1038, 986, 1029, 972, 1076, 1040, 1030, 19, 17, 0, 0, 0, 0, 0, 0, 962, 1040, 1038, 1055, 1070, 967, 1028, 991, 42, 0, 0, 0, 0, 0, 0, 0, 987, 1067, 1037, 997, 1021, 1034, 1033, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1015, 1046, 1066, 946, 1081, 1027, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 989, 1019, 1026, 1053, 1067, 1047, 1048, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8046, 8145, 8258, 8193, 8172, 8286, 8271] avg=8192.0000 max=8286 max_rank=6 min=8046 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32922 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1034, 998, 1026, 985, 1041, 976, 1033, 1045, 11, 0, 0, 0, 0, 0, 0, 0, 1013, 1066, 1053, 990, 1040, 1011, 1045, 975, 13, 3, 0, 0, 0, 0, 0, 0, 1045, 1021, 994, 1006, 1013, 1023, 1046, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 997, 975, 1024, 1047, 1040, 1033, 1019, 1078, 20, 17, 0, 0, 0, 0, 0, 0, 1021, 1007, 968, 1042, 1016, 992, 1079, 1022, 46, 0, 0, 0, 0, 0, 0, 0, 979, 1019, 1002, 974, 1075, 1037, 1075, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1059, 1091, 1022, 1021, 986, 1037, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1015, 986, 1051, 1013, 1016, 1015, 967, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8209, 8209, 8250, 8193, 8157, 8289, 8080] avg=8192.0000 max=8289 max_rank=6 min=8080 min_rank=7 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1043, 1022, 1018, 1012, 1018, 1026, 1017, 1024, 11, 0, 0, 0, 0, 0, 0, 0, 1020, 1022, 1038, 1036, 1045, 1028, 1048, 1052, 13, 3, 0, 0, 0, 0, 0, 0, 996, 992, 1049, 1039, 1058, 1032, 1036, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1030, 1019, 1040, 1017, 993, 1021, 998, 20, 17, 0, 0, 0, 0, 0, 0, 1015, 991, 1026, 1045, 1012, 1037, 1083, 969, 44, 0, 0, 0, 0, 0, 0, 0, 992, 1012, 1056, 1076, 1035, 974, 1051, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 971, 1023, 951, 986, 996, 1040, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1026, 1044, 1044, 1044, 1047, 994, 946, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8305, 8208, 8167, 8222, 8234, 8044, 8165] avg=8192.0000 max=8305 max_rank=1 min=8044 min_rank=6 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1009, 1032, 1031, 1023, 1049, 1039, 1022, 1021, 15, 0, 0, 0, 0, 0, 0, 0, 1025, 1071, 1002, 1059, 990, 1018, 1027, 1003, 15, 5, 0, 0, 0, 0, 0, 0, 994, 1047, 1016, 1044, 1058, 973, 1028, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1041, 1018, 992, 1035, 988, 1027, 998, 23, 21, 0, 0, 0, 0, 0, 0, 994, 1030, 1017, 1049, 1002, 1030, 984, 982, 48, 0, 0, 0, 0, 0, 0, 0, 995, 1039, 1053, 996, 1006, 1049, 1051, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 995, 1066, 1066, 972, 968, 1037, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1036, 1022, 1055, 1028, 1054, 966, 985, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8215, 8181, 8210, 8136, 8194, 8183, 8176] avg=8192.0000 max=8241 max_rank=0 min=8136 min_rank=4 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1037, 1045, 982, 1014, 1009, 1019, 987, 1044, 11, 0, 0, 0, 0, 0, 0, 0, 1021, 1002, 1006, 1029, 1003, 1041, 1034, 1053, 13, 3, 0, 0, 0, 0, 0, 0, 1047, 1013, 1040, 1031, 1059, 1073, 999, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1004, 996, 996, 1025, 1033, 1061, 1009, 20, 17, 0, 0, 0, 0, 0, 0, 987, 939, 1028, 1047, 1054, 1022, 1002, 1075, 46, 0, 0, 0, 0, 0, 0, 0, 1000, 996, 1017, 1045, 1020, 1029, 975, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1028, 1098, 1010, 1027, 1032, 1014, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1059, 991, 988, 975, 1036, 1012, 1052, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8205, 8289, 8200, 8200, 8094, 8254, 8146] avg=8192.0000 max=8289 max_rank=2 min=8094 min_rank=5 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8448, 8320, 7936, 8320, 8320, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8192, 8192, 8192, 8320, 8192, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1048, 1044, 1024, 1039, 1033, 995, 1043, 1015, 999, 1006, 988, 996, 1054, 1040, 1007, 1047, 1074, 1014, 1064, 1019, 1006, 1020, 1062, 1012, 970, 1053, 1051, 1048, 1049, 1030, 1106, 1027, 1031, 996, 1017, 1065, 1045, 995, 1033, 992, 968, 1012, 978, 983, 1012, 996, 1055, 1019, 996, 1047, 1026, 961, 1038, 1005, 1086, 1021, 1007, 1030, 1022, 1007, 1027, 1026, 1022, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1014, 1033, 1056, 1008, 1036, 1016, 1014, 1021, 1009, 1023, 995, 1051, 1004, 1040, 1022, 1034, 973, 1031, 1038, 1022, 1020, 1036, 1017, 1001, 1010, 991, 1036, 1024, 1043, 1098, 1018, 1009, 993, 994, 1066, 1026, 1042, 1042, 1081, 963, 1029, 1106, 956, 1073, 1043, 989, 975, 979, 1008, 1079, 1043, 1034, 1002, 1037, 1010, 993, 1000, 1035, 992, 1064, 1020, 1001, 1031, 1087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1043, 1041, 998, 1084, 1034, 1004, 1007, 996, 943, 1035, 1050, 1038, 1010, 960, 991, 1051, 1040, 1018, 1073, 1006, 1023, 1000, 1031, 999, 1048, 991, 1054, 1006, 1036, 981, 1006, 986, 984, 1091, 1026, 1042, 1053, 1007, 1060, 1048, 1003, 1030, 1019, 1003, 949, 1036, 1012, 992, 1043, 1052, 1001, 1087, 1017, 1074, 937, 1071, 1017, 1017, 1059, 1011, 1003, 1046, 1076, 1087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[971, 1029, 1075, 998, 1055, 1041, 1096, 1022, 1008, 1053, 1033, 1045, 1011, 1011, 1066, 1015, 1054, 1017, 962, 1024, 1009, 1029, 1025, 1018, 1044, 1030, 1070, 1052, 1038, 1002, 981, 1048, 1004, 1014, 1064, 996, 1005, 1005, 996, 1061, 995, 980, 1015, 973, 1008, 1040, 1050, 1039, 1045, 1004, 1066, 973, 986, 1015, 1012, 1045, 998, 1028, 1045, 1002, 1006, 1052, 1027, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1075, 1086, 1011, 982, 1022, 1017, 1025, 985, 1014, 1046, 1054, 1037, 1049, 1005, 1019, 1049, 991, 1005, 1028, 1024, 1049, 1017, 1055, 1006, 1035, 1023, 1042, 1035, 1006, 1063, 1011, 1028, 956, 1005, 1041, 1052, 1049, 1064, 946, 1001, 1037, 1014, 1052, 995, 1005, 1013, 1031, 1033, 983, 1046, 1029, 1042, 999, 1039, 1020, 1043, 1006, 1035, 980, 981, 1053, 1021, 1076, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1046, 1016, 1019, 1024, 1054, 990, 1021, 1076, 972, 1041, 1027, 1093, 1022, 992, 1019, 993, 1046, 1021, 989, 996, 998, 1021, 1066, 1010, 1011, 1052, 1045, 1032, 1065, 975, 1037, 1039, 1042, 1017, 990, 1009, 990, 966, 1034, 1052, 1009, 1040, 1007, 1030, 1054, 1009, 1032, 1043, 1051, 1011, 1046, 1021, 1022, 1000, 1009, 1072, 1061, 1020, 1030, 1039, 1025, 984, 1022, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[989, 1059, 1042, 1035, 1010, 1020, 1029, 1010, 1040, 1028, 1068, 1038, 970, 995, 1037, 1019, 1066, 1011, 1001, 1061, 1052, 1000, 1029, 998, 977, 1056, 1030, 1014, 996, 1023, 1044, 1057, 980, 1035, 1053, 1038, 1035, 1023, 1019, 1018, 1043, 1102, 1007, 1052, 1010, 1039, 1012, 1056, 986, 996, 998, 1017, 1012, 1017, 1015, 1031, 1026, 977, 1027, 1021, 1018, 1019, 1037, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[998, 1069, 1043, 1025, 1012, 986, 1047, 1041, 1027, 1030, 1041, 1031, 992, 1000, 1044, 1012, 1063, 1029, 1014, 1053, 985, 1019, 1032, 990, 1022, 1003, 1023, 1034, 987, 1007, 1006, 1027, 981, 1021, 1040, 999, 1057, 999, 1024, 1014, 1046, 1069, 1003, 1051, 1034, 1038, 1092, 1044, 1028, 1015, 997, 1024, 991, 1023, 987, 1018, 1062, 1046, 1010, 1035, 980, 1024, 1016, 1076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1043, 1010, 998, 1084, 1034, 1004, 1007, 996, 0, 0, 0, 0, 0, 0, 0, 0, 943, 1035, 1050, 1038, 1010, 960, 991, 1051, 8, 4, 0, 0, 0, 0, 0, 0, 1040, 1018, 1073, 1006, 1023, 1000, 1031, 999, 9, 0, 0, 0, 0, 0, 0, 0, 1048, 991, 1030, 1006, 1036, 981, 1006, 986, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1091, 1026, 1042, 1053, 1007, 1060, 1048, 18, 0, 0, 0, 0, 0, 0, 0, 1003, 1030, 1019, 1003, 949, 1036, 1012, 992, 14, 0, 0, 0, 0, 0, 0, 0, 1043, 1052, 1001, 1087, 1017, 1074, 937, 1071, 13, 2, 0, 0, 0, 0, 0, 0, 1017, 1017, 1059, 1011, 1003, 1046, 1076, 1074, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8090, 8199, 8084, 8329, 8058, 8297, 8303] avg=8192.0000 max=8329 max_rank=4 min=8058 min_rank=5 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32987 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1048, 1013, 1024, 1039, 1033, 995, 1043, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1006, 988, 996, 1054, 1040, 1007, 1047, 8, 3, 0, 0, 0, 0, 0, 0, 1074, 1014, 1064, 1019, 1006, 1020, 1062, 1012, 9, 0, 0, 0, 0, 0, 0, 0, 970, 1053, 1027, 1048, 1049, 1030, 1106, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 996, 1017, 1065, 1045, 995, 1033, 992, 18, 0, 0, 0, 0, 0, 0, 0, 968, 1012, 978, 983, 1012, 996, 1055, 1019, 14, 0, 0, 0, 0, 0, 0, 0, 996, 1047, 1026, 961, 1038, 1005, 1086, 1021, 13, 2, 0, 0, 0, 0, 0, 0, 1007, 1030, 1022, 1007, 1027, 1026, 1022, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8148, 8280, 8310, 8192, 8037, 8195, 8164] avg=8192.0000 max=8310 max_rank=3 min=8037 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32588 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1075, 1053, 1011, 982, 1022, 1017, 1025, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1046, 1054, 1037, 1049, 1005, 1019, 1049, 8, 3, 0, 0, 0, 0, 0, 0, 991, 1005, 1028, 1024, 1049, 1017, 1055, 1006, 8, 0, 0, 0, 0, 0, 0, 0, 1035, 1023, 1018, 1035, 1006, 1063, 1011, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1005, 1041, 1052, 1049, 1064, 946, 1001, 19, 0, 0, 0, 0, 0, 0, 0, 1037, 1014, 1052, 995, 1005, 1013, 1031, 1033, 14, 0, 0, 0, 0, 0, 0, 0, 983, 1046, 1029, 1042, 999, 1039, 1020, 1043, 14, 2, 0, 0, 0, 0, 0, 0, 1006, 1035, 980, 981, 1053, 1021, 1076, 984, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8284, 8183, 8219, 8133, 8194, 8217, 8136] avg=8192.0000 max=8284 max_rank=1 min=8133 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1014, 1002, 1056, 1008, 1036, 1016, 1014, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1023, 995, 1051, 1004, 1040, 1022, 1034, 8, 4, 0, 0, 0, 0, 0, 0, 973, 1031, 1038, 1022, 1020, 1036, 1017, 1001, 9, 0, 0, 0, 0, 0, 0, 0, 1010, 991, 1012, 1024, 1043, 1098, 1018, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 993, 994, 1066, 1026, 1042, 1042, 1081, 963, 18, 0, 0, 0, 0, 0, 0, 0, 1029, 1106, 956, 1073, 1043, 989, 975, 979, 14, 0, 0, 0, 0, 0, 0, 0, 1008, 1079, 1043, 1034, 1002, 1037, 1010, 993, 13, 2, 0, 0, 0, 0, 0, 0, 1000, 1035, 992, 1064, 1020, 1001, 1031, 1074, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8190, 8147, 8205, 8225, 8164, 8221, 8217] avg=8192.0000 max=8225 max_rank=4 min=8147 min_rank=2 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[971, 998, 1075, 998, 1055, 1041, 1096, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1053, 1033, 1045, 1011, 1011, 1066, 1015, 8, 3, 0, 0, 0, 0, 0, 0, 1054, 1017, 962, 1024, 1009, 1029, 1025, 1018, 9, 0, 0, 0, 0, 0, 0, 0, 1044, 1030, 1046, 1052, 1038, 1002, 981, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1014, 1064, 996, 1005, 1005, 996, 1061, 18, 0, 0, 0, 0, 0, 0, 0, 995, 980, 1015, 973, 1008, 1040, 1050, 1039, 14, 0, 0, 0, 0, 0, 0, 0, 1045, 1004, 1066, 973, 986, 1015, 1012, 1045, 13, 2, 0, 0, 0, 0, 0, 0, 998, 1028, 1045, 1002, 1006, 1052, 1027, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8253, 8147, 8241, 8163, 8114, 8161, 8201] avg=8192.0000 max=8256 max_rank=0 min=8114 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1046, 985, 1019, 1024, 1054, 990, 1021, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1041, 1027, 1093, 1022, 992, 1019, 993, 8, 3, 0, 0, 0, 0, 0, 0, 1046, 1021, 989, 996, 998, 1021, 1066, 1010, 8, 0, 0, 0, 0, 0, 0, 0, 1011, 1052, 1021, 1032, 1065, 975, 1037, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1017, 990, 1009, 990, 966, 1034, 1052, 18, 0, 0, 0, 0, 0, 0, 0, 1009, 1040, 1007, 1030, 1054, 1009, 1032, 1043, 14, 0, 0, 0, 0, 0, 0, 0, 1051, 1011, 1046, 1021, 1022, 1000, 1009, 1072, 13, 2, 0, 0, 0, 0, 0, 0, 1061, 1020, 1030, 1039, 1025, 984, 1022, 980, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8170, 8155, 8232, 8118, 8238, 8247, 8161] avg=8192.0000 max=8247 max_rank=6 min=8118 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[989, 1019, 1042, 1035, 1010, 1020, 1029, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1028, 1068, 1038, 970, 995, 1037, 1019, 10, 7, 0, 0, 0, 0, 0, 0, 1066, 1011, 1001, 1061, 1052, 1000, 1029, 998, 13, 0, 0, 0, 0, 0, 0, 0, 977, 1056, 993, 1014, 996, 1023, 1044, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1035, 1053, 1038, 1035, 1023, 1019, 1018, 21, 0, 0, 0, 0, 0, 0, 0, 1043, 1102, 1007, 1052, 1010, 1039, 1012, 1056, 19, 0, 0, 0, 0, 0, 0, 0, 986, 996, 998, 1017, 1012, 1017, 1015, 1031, 19, 8, 0, 0, 0, 0, 0, 0, 1026, 977, 1027, 1021, 1018, 1019, 1037, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8212, 8231, 8160, 8222, 8340, 8099, 8118] avg=8192.0000 max=8340 max_rank=5 min=8099 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[998, 1036, 1043, 1025, 1012, 986, 1047, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1030, 1041, 1031, 992, 1000, 1044, 1012, 8, 4, 0, 0, 0, 0, 0, 0, 1063, 1029, 1014, 1053, 985, 1019, 1032, 990, 9, 0, 0, 0, 0, 0, 0, 0, 1022, 1003, 999, 1034, 987, 1007, 1006, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1021, 1040, 999, 1057, 999, 1024, 1014, 19, 0, 0, 0, 0, 0, 0, 0, 1046, 1069, 1003, 1051, 1034, 1038, 1092, 1044, 14, 0, 0, 0, 0, 0, 0, 0, 1028, 1015, 997, 1024, 991, 1023, 987, 1018, 14, 2, 0, 0, 0, 0, 0, 0, 1062, 1046, 1010, 1035, 980, 1024, 1016, 1063, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8189, 8194, 8085, 8154, 8391, 8099, 8236] avg=8192.0000 max=8391 max_rank=5 min=8085 min_rank=3 imbalance_ratio=1.024292 max_load_reduction=57145 max_load_reduction_pct=0.871964 moved_to_echo=32880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8320, 8448, 8192, 8064, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8448, 8064, 8192, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1067, 1019, 985, 970, 1019, 971, 984, 1025, 957, 1004, 1005, 1063, 1039, 975, 1064, 1021, 1023, 995, 1030, 1022, 1024, 992, 1051, 985, 1037, 1064, 1028, 1098, 1018, 1039, 1034, 1003, 991, 1054, 1015, 983, 1050, 1051, 1043, 1018, 1031, 1016, 999, 1049, 1029, 998, 987, 1000, 1031, 1023, 1000, 1040, 1064, 1044, 1038, 1058, 1050, 1065, 1062, 1026, 1042, 969, 1037, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1052, 1007, 1031, 1043, 1043, 1024, 1027, 1024, 1040, 1001, 1052, 1044, 973, 1006, 1057, 1011, 1015, 1039, 1020, 1022, 1025, 973, 982, 1019, 1003, 1058, 1013, 1027, 1037, 1014, 1031, 1011, 986, 1023, 1003, 1034, 1020, 1080, 1058, 1061, 978, 1016, 1059, 1056, 971, 990, 1039, 1036, 1034, 1030, 1049, 971, 1015, 1101, 973, 1019, 1045, 955, 1019, 1034, 1023, 1036, 1019, 1079] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1066, 1045, 987, 962, 1051, 1083, 960, 1024, 1011, 964, 1059, 997, 1019, 1014, 1038, 998, 1038, 986, 978, 1008, 1021, 1017, 1019, 1011, 963, 979, 1054, 1042, 1018, 1071, 1021, 943, 1041, 1076, 1028, 1064, 1025, 1031, 1052, 1032, 1024, 980, 1040, 1066, 1049, 995, 1020, 1019, 1004, 999, 1004, 1052, 1033, 1055, 1070, 993, 1069, 1072, 1077, 1010, 994, 1061, 1011, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1030, 1016, 1035, 1033, 1014, 1019, 982, 1041, 1018, 1058, 1018, 1027, 1045, 996, 1000, 1019, 1037, 985, 1035, 1029, 1020, 1063, 1042, 1043, 1054, 963, 1087, 982, 985, 1001, 1066, 1012, 1054, 1012, 996, 1048, 1045, 1026, 1029, 1041, 1064, 995, 1097, 983, 1001, 1063, 1008, 1027, 1112, 1014, 1037, 956, 1015, 986, 1036, 994, 1008, 992, 1039, 1046, 1009, 1042, 994, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1021, 1062, 1043, 1006, 1042, 1033, 979, 1026, 1062, 979, 1009, 1052, 1038, 1031, 1031, 985, 1050, 1070, 1048, 1029, 1007, 1027, 994, 1037, 1043, 1006, 1036, 1013, 1019, 1017, 1026, 1023, 1046, 994, 966, 1040, 1039, 1034, 1052, 997, 1039, 1033, 1009, 1001, 999, 1004, 1040, 1020, 1022, 1016, 1029, 1002, 992, 1027, 1020, 1031, 1057, 994, 1035, 1054, 1036, 1034, 1030, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1027, 985, 1000, 1013, 981, 1083, 989, 1043, 1000, 995, 1020, 1021, 1006, 1043, 984, 1043, 1031, 1015, 1044, 1014, 1061, 1005, 1031, 982, 1023, 1075, 1034, 1023, 971, 1012, 1068, 1046, 1032, 1036, 1037, 1079, 991, 1021, 1022, 1010, 1023, 1027, 1054, 1017, 1053, 995, 1003, 1020, 1017, 1025, 1030, 1011, 997, 986, 1031, 1081, 1041, 1042, 1036, 1029, 1007, 1025, 1043, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1050, 990, 1017, 1050, 1010, 1019, 1003, 987, 1017, 1038, 1066, 1008, 1030, 1020, 1020, 1009, 1038, 1117, 1022, 1034, 1018, 1037, 1015, 1011, 1038, 995, 1008, 995, 1035, 983, 1042, 1024, 1034, 1059, 998, 1023, 1002, 1035, 1047, 1040, 1019, 1043, 1051, 1036, 1051, 1021, 992, 968, 1037, 1016, 1042, 1025, 957, 994, 1031, 1027, 1069, 1019, 977, 1049, 1030, 1074, 1008, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[997, 1070, 1001, 1051, 1009, 1029, 1055, 994, 1001, 1032, 1009, 1037, 966, 1059, 1012, 1028, 1066, 1031, 990, 1019, 1025, 1064, 1010, 1005, 1054, 1068, 1007, 1008, 1054, 1046, 1064, 967, 1042, 1029, 1024, 1026, 966, 1026, 965, 1025, 1033, 1025, 978, 1037, 987, 963, 1060, 1007, 1024, 1023, 1028, 1007, 1102, 1062, 1002, 1055, 1014, 1015, 1072, 1019, 1029, 1075, 994, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1067, 1019, 985, 970, 1019, 971, 984, 1025, 21, 7, 0, 0, 0, 0, 0, 0, 957, 1004, 1005, 1063, 1039, 975, 1064, 1021, 36, 0, 0, 0, 0, 0, 0, 0, 1023, 995, 1030, 1022, 1024, 992, 1051, 985, 4, 0, 0, 0, 0, 0, 0, 0, 1037, 1064, 1028, 1098, 1018, 1039, 1030, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1054, 1015, 983, 1050, 1017, 1043, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1016, 999, 1049, 1029, 998, 987, 1000, 17, 13, 0, 0, 0, 0, 0, 0, 1024, 1023, 1000, 1040, 1064, 1044, 1038, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1065, 1062, 1026, 1042, 969, 1037, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 8164, 8126, 8317, 8171, 8139, 8291, 8260] avg=8192.0000 max=8317 max_rank=3 min=8068 min_rank=0 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32861 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1052, 1007, 1031, 1043, 1043, 1024, 1027, 1024, 21, 7, 0, 0, 0, 0, 0, 0, 1040, 1001, 1052, 1044, 973, 1006, 1057, 1011, 36, 0, 0, 0, 0, 0, 0, 0, 1015, 1039, 1020, 1022, 1025, 973, 982, 1019, 4, 0, 0, 0, 0, 0, 0, 0, 1003, 1058, 1013, 1027, 1037, 1014, 1027, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1023, 1003, 1034, 1020, 1046, 1058, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1016, 1059, 1056, 971, 990, 1039, 1036, 17, 13, 0, 0, 0, 0, 0, 0, 1027, 1030, 1049, 971, 1015, 1101, 973, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 992, 955, 1019, 1034, 1023, 1036, 1019, 1079, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8220, 8099, 8190, 8231, 8175, 8185, 8157] avg=8192.0000 max=8279 max_rank=0 min=8099 min_rank=2 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1021, 1062, 1043, 1006, 1042, 1033, 979, 1026, 20, 7, 0, 0, 0, 0, 0, 0, 1062, 979, 1009, 1052, 1038, 1031, 1031, 985, 36, 0, 0, 0, 0, 0, 0, 0, 1050, 1070, 1048, 1029, 1007, 1027, 994, 1037, 4, 0, 0, 0, 0, 0, 0, 0, 1043, 1006, 1036, 1013, 1019, 1017, 1022, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 994, 966, 1040, 1039, 1001, 1052, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1033, 1009, 1001, 999, 1004, 1040, 1020, 17, 13, 0, 0, 0, 0, 0, 0, 1015, 1016, 1029, 1002, 992, 1027, 1020, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 994, 1035, 1054, 1036, 1034, 1030, 1000, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8223, 8266, 8179, 8135, 8175, 8132, 8187] avg=8192.0000 max=8266 max_rank=2 min=8132 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1066, 1045, 987, 962, 1051, 1083, 960, 1024, 20, 7, 0, 0, 0, 0, 0, 0, 1011, 964, 1059, 997, 1019, 1014, 1038, 998, 37, 0, 0, 0, 0, 0, 0, 0, 1038, 986, 978, 1008, 1021, 1017, 1019, 1011, 4, 0, 0, 0, 0, 0, 0, 0, 963, 979, 1054, 1042, 1018, 1071, 1017, 943, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1076, 1028, 1064, 1025, 998, 1052, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 980, 1040, 1066, 1049, 995, 1020, 1019, 17, 13, 0, 0, 0, 0, 0, 0, 997, 999, 1004, 1052, 1033, 1055, 1070, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1072, 1077, 1010, 994, 1061, 1011, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8137, 8082, 8087, 8316, 8223, 8203, 8283] avg=8192.0000 max=8316 max_rank=4 min=8082 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=33025 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1030, 1016, 1035, 1033, 1014, 1019, 982, 1041, 20, 8, 0, 0, 0, 0, 0, 0, 1018, 1058, 1018, 1027, 1045, 996, 1000, 1019, 35, 0, 0, 0, 0, 0, 0, 0, 1037, 985, 1035, 1029, 1020, 1063, 1042, 1043, 5, 0, 0, 0, 0, 0, 0, 0, 1054, 963, 1087, 982, 985, 1001, 1061, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1012, 996, 1048, 1045, 994, 1029, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 995, 1097, 983, 1001, 1063, 1008, 1027, 16, 12, 0, 0, 0, 0, 0, 0, 1104, 1014, 1037, 956, 1015, 986, 1036, 994, 0, 0, 0, 0, 0, 0, 0, 0, 957, 992, 1039, 1046, 1009, 1042, 994, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8216, 8259, 8145, 8219, 8266, 8142, 8091] avg=8192.0000 max=8266 max_rank=5 min=8091 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1027, 985, 1000, 1013, 981, 1083, 989, 1043, 20, 7, 0, 0, 0, 0, 0, 0, 1000, 995, 1020, 1021, 1006, 1043, 984, 1043, 36, 0, 0, 0, 0, 0, 0, 0, 1031, 1015, 1044, 1014, 1061, 1005, 1031, 982, 5, 0, 0, 0, 0, 0, 0, 0, 1023, 1075, 1034, 1023, 971, 1012, 1063, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1036, 1037, 1079, 991, 989, 1022, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1027, 1054, 1017, 1053, 995, 1003, 1020, 17, 12, 0, 0, 0, 0, 0, 0, 1010, 1025, 1030, 1011, 997, 986, 1031, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1042, 1036, 1029, 1007, 1025, 1043, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8148, 8188, 8247, 8196, 8221, 8171, 8217] avg=8192.0000 max=8247 max_rank=3 min=8148 min_rank=0 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1050, 990, 1017, 1050, 1010, 1019, 1003, 987, 24, 13, 3, 0, 0, 0, 0, 0, 1017, 1038, 1066, 1008, 1030, 1020, 1020, 1009, 41, 0, 0, 0, 0, 0, 0, 0, 1038, 1117, 1022, 1034, 1018, 1037, 1015, 1011, 9, 0, 0, 0, 0, 0, 0, 0, 1038, 995, 1008, 995, 1035, 983, 1030, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1059, 998, 1023, 1002, 994, 1047, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1043, 1051, 1036, 1051, 1021, 992, 968, 21, 17, 0, 0, 0, 0, 0, 0, 1024, 1016, 1042, 1025, 957, 994, 1031, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1019, 977, 1049, 1030, 1074, 1008, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8249, 8301, 8108, 8197, 8219, 8116, 8180] avg=8192.0000 max=8301 max_rank=2 min=8108 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[997, 1070, 1001, 1051, 1009, 1029, 1055, 994, 20, 7, 0, 0, 0, 0, 0, 0, 1001, 1032, 1009, 1037, 966, 1059, 1012, 1028, 35, 0, 0, 0, 0, 0, 0, 0, 1066, 1031, 990, 1019, 1025, 1064, 1010, 1005, 5, 0, 0, 0, 0, 0, 0, 0, 1054, 1068, 1007, 1008, 1054, 1046, 1059, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1029, 1024, 1026, 966, 994, 965, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1025, 978, 1037, 987, 963, 1060, 1007, 16, 12, 0, 0, 0, 0, 0, 0, 1017, 1023, 1028, 1007, 1102, 1062, 1002, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1015, 1072, 1019, 1029, 1075, 994, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8179, 8215, 8263, 8071, 8118, 8296, 8161] avg=8192.0000 max=8296 max_rank=6 min=8071 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8064, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[7936, 8192, 8320, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8064, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8064, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1012, 1015, 1010, 1045, 1054, 1040, 1095, 986, 1000, 1034, 1046, 1049, 1038, 1023, 981, 1016, 988, 1050, 1037, 1012, 1010, 1064, 1023, 960, 1059, 1043, 1041, 1041, 1041, 976, 1031, 1038, 982, 996, 1051, 1064, 1029, 1035, 1034, 1023, 1041, 1011, 1035, 1034, 1007, 988, 1073, 998, 1034, 978, 1030, 1011, 1061, 1034, 1050, 1057, 949, 971, 1011, 986, 1010, 1035, 1032, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1025, 1006, 955, 1056, 1025, 1009, 1019, 1036, 1018, 1003, 1039, 984, 1056, 1033, 1041, 1005, 994, 982, 1071, 1037, 1024, 982, 1017, 961, 1039, 1026, 1035, 1060, 1018, 1021, 1062, 1033, 1031, 1114, 998, 985, 1028, 1025, 1065, 1026, 1025, 988, 1009, 1033, 1016, 1072, 1019, 1026, 1047, 993, 1024, 1039, 999, 1086, 1091, 1001, 999, 1035, 1024, 1032, 988, 994, 1036, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1016, 1031, 1065, 1034, 1056, 998, 1035, 1023, 1006, 1025, 986, 1025, 1042, 997, 1007, 1012, 1027, 1055, 1042, 1064, 1022, 1001, 1000, 1086, 1048, 1037, 984, 1028, 1045, 1011, 988, 1048, 1052, 1012, 1026, 1015, 978, 1015, 1047, 962, 999, 1043, 1018, 1038, 1022, 1065, 1088, 1012, 972, 1018, 1024, 1051, 1057, 965, 992, 979, 995, 991, 1018, 1030, 1081, 1017, 1050, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1011, 1029, 990, 1040, 1003, 1073, 1026, 1036, 1017, 1060, 1002, 1001, 996, 1031, 1029, 1025, 1008, 1065, 1019, 1033, 999, 1007, 1041, 984, 1033, 1014, 1011, 1033, 1022, 989, 984, 1021, 960, 1040, 1022, 1037, 1056, 990, 1100, 1045, 1045, 1026, 1017, 1068, 1021, 1009, 1049, 1018, 1074, 1019, 1056, 968, 1024, 1001, 1017, 1022, 1041, 1067, 1021, 996, 1048, 1026, 1035, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1060, 1013, 1062, 998, 1028, 943, 1036, 1022, 1006, 983, 1051, 1043, 1042, 1003, 985, 1050, 1033, 1088, 1045, 1022, 1025, 1008, 999, 1043, 997, 956, 1017, 1006, 1027, 1026, 1055, 1031, 1021, 1021, 1006, 1042, 994, 1003, 970, 1016, 1025, 1078, 1040, 1009, 1052, 1036, 1007, 1044, 1044, 1018, 1046, 1010, 1054, 1012, 1070, 1081, 1017, 1021, 1006, 987, 1010, 1069, 1043, 981] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[990, 1027, 1046, 1016, 981, 1045, 1004, 983, 1011, 1028, 1047, 1018, 994, 1030, 1031, 1077, 985, 994, 1041, 1048, 1069, 1053, 1023, 1020, 1021, 1052, 1022, 1062, 1030, 1046, 1041, 1025, 1041, 1014, 1030, 1036, 1053, 1018, 997, 1041, 1016, 979, 1036, 1008, 1042, 974, 988, 1005, 1056, 1075, 1013, 1080, 1051, 1012, 1002, 1035, 1009, 1023, 1005, 1012, 1000, 1007, 1008, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1074, 994, 1030, 1025, 1085, 1030, 1088, 1030, 1036, 1020, 1039, 1039, 1007, 1015, 1019, 1078, 1002, 1077, 1003, 1037, 1003, 1017, 989, 1052, 1067, 1008, 961, 1001, 1032, 944, 1036, 1025, 1010, 1015, 1036, 1018, 1055, 1029, 1017, 1004, 1063, 1016, 1021, 1023, 1058, 990, 1043, 1037, 958, 1016, 1065, 1018, 1002, 1021, 1034, 1063, 965, 999, 1020, 1013, 1045, 1040, 1008, 971] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1074, 1007, 1019, 1034, 1044, 1032, 1029, 1018, 1024, 1057, 1055, 1005, 987, 1029, 1045, 1009, 992, 1028, 977, 1011, 1059, 1025, 1035, 1008, 1068, 1043, 1026, 1009, 993, 1058, 1007, 1011, 1038, 1062, 960, 997, 1038, 1005, 963, 1081, 1056, 963, 986, 1036, 1040, 985, 1035, 1100, 1041, 1006, 1051, 1009, 1022, 1038, 1039, 1006, 1029, 1003, 1011, 1021, 1033, 989, 999, 1076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 1006, 955, 1056, 1025, 1009, 997, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1003, 1039, 984, 1056, 1033, 1041, 1005, 3, 2, 0, 0, 0, 0, 0, 0, 994, 982, 1071, 1037, 1024, 982, 1017, 961, 7, 0, 0, 0, 0, 0, 0, 0, 1036, 1026, 1035, 1060, 1018, 1021, 1062, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1114, 998, 985, 1028, 1025, 1065, 1026, 7, 0, 0, 0, 0, 0, 0, 0, 1025, 988, 1009, 1033, 1016, 1072, 1000, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 993, 989, 1039, 999, 1086, 1091, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1035, 1024, 1032, 988, 994, 1036, 1016, 35, 22, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8184, 8075, 8291, 8279, 8169, 8245, 8184] avg=8192.0000 max=8291 max_rank=3 min=8075 min_rank=2 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1016, 1031, 1065, 1034, 1056, 998, 1013, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1025, 986, 1025, 1042, 997, 1007, 1012, 3, 2, 0, 0, 0, 0, 0, 0, 1027, 1055, 1042, 1064, 1022, 1001, 1000, 1086, 7, 0, 0, 0, 0, 0, 0, 0, 1045, 1037, 984, 1028, 1045, 1011, 988, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1012, 1026, 1015, 978, 1015, 1047, 962, 8, 0, 0, 0, 0, 0, 0, 0, 999, 1043, 1018, 1038, 1022, 1065, 1068, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1018, 989, 1051, 1057, 965, 992, 979, 0, 0, 0, 0, 0, 0, 0, 0, 995, 991, 1018, 1030, 1081, 1017, 1050, 1060, 35, 22, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8105, 8304, 8186, 8115, 8265, 8023, 8302] avg=8192.0000 max=8304 max_rank=2 min=8023 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1012, 1015, 1010, 1045, 1054, 1040, 1071, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1034, 1046, 1049, 1038, 1023, 981, 1016, 3, 2, 0, 0, 0, 0, 0, 0, 988, 1050, 1037, 1012, 1010, 1064, 1023, 960, 7, 0, 0, 0, 0, 0, 0, 0, 1056, 1043, 1041, 1041, 1041, 976, 1031, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 982, 996, 1051, 1064, 1029, 1035, 1034, 1023, 8, 0, 0, 0, 0, 0, 0, 0, 1041, 1011, 1035, 1034, 1007, 988, 1053, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 978, 995, 1011, 1061, 1034, 1050, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 949, 971, 1011, 986, 1010, 1035, 1032, 1028, 35, 24, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8192, 8151, 8267, 8222, 8167, 8220, 8084] avg=8192.0000 max=8267 max_rank=3 min=8084 min_rank=7 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1011, 1029, 990, 1040, 1003, 1073, 1004, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1060, 1002, 1001, 996, 1031, 1029, 1025, 3, 2, 0, 0, 0, 0, 0, 0, 1008, 1065, 1019, 1033, 999, 1007, 1041, 984, 7, 0, 0, 0, 0, 0, 0, 0, 1030, 1014, 1011, 1033, 1022, 989, 984, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 960, 1040, 1022, 1037, 1056, 990, 1100, 1045, 7, 0, 0, 0, 0, 0, 0, 0, 1045, 1026, 1017, 1068, 1021, 1009, 1030, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1019, 1020, 968, 1024, 1001, 1017, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1067, 1021, 996, 1048, 1026, 1035, 986, 36, 22, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8166, 8163, 8104, 8257, 8234, 8145, 8281] avg=8192.0000 max=8281 max_rank=7 min=8104 min_rank=3 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1060, 1013, 1062, 998, 1028, 943, 1013, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 983, 1051, 1043, 1042, 1003, 985, 1050, 3, 2, 0, 0, 0, 0, 0, 0, 1033, 1088, 1045, 1022, 1025, 1008, 999, 1043, 7, 0, 0, 0, 0, 0, 0, 0, 994, 956, 1017, 1006, 1027, 1026, 1055, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1021, 1006, 1042, 994, 1003, 970, 1016, 7, 0, 0, 0, 0, 0, 0, 0, 1025, 1078, 1040, 1009, 1052, 1036, 988, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1018, 1010, 1010, 1054, 1012, 1070, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1021, 1006, 987, 1010, 1069, 1043, 981, 36, 23, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8168, 8270, 8112, 8080, 8272, 8299, 8196] avg=8192.0000 max=8299 max_rank=6 min=8080 min_rank=4 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1074, 994, 1030, 1025, 1085, 1030, 1064, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1020, 1039, 1039, 1007, 1015, 1019, 1078, 3, 2, 0, 0, 0, 0, 0, 0, 1002, 1077, 1003, 1037, 1003, 1017, 989, 1052, 7, 0, 0, 0, 0, 0, 0, 0, 1064, 1008, 961, 1001, 1032, 944, 1036, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1015, 1036, 1018, 1055, 1029, 1017, 1004, 7, 0, 0, 0, 0, 0, 0, 0, 1063, 1016, 1021, 1023, 1058, 990, 1024, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 958, 1016, 1029, 1018, 1002, 1021, 1034, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 965, 999, 1020, 1013, 1045, 1040, 1008, 971, 36, 24, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8332, 8258, 8187, 8071, 8191, 8232, 8141, 8124] avg=8192.0000 max=8332 max_rank=0 min=8071 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[990, 1027, 1046, 1016, 981, 1045, 978, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1028, 1047, 1018, 994, 1030, 1031, 1077, 6, 5, 0, 0, 0, 0, 0, 0, 985, 994, 1041, 1048, 1069, 1053, 1023, 1020, 11, 0, 0, 0, 0, 0, 0, 0, 1015, 1052, 1022, 1062, 1030, 1046, 1041, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1014, 1030, 1036, 1053, 1018, 997, 1041, 11, 0, 0, 0, 0, 0, 0, 0, 1016, 979, 1036, 1008, 1042, 974, 955, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1075, 976, 1080, 1051, 1012, 1002, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1023, 1005, 1012, 1000, 1007, 1008, 1010, 37, 26, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8247, 8244, 8293, 8241, 8015, 8287, 8143] avg=8192.0000 max=8293 max_rank=3 min=8015 min_rank=5 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1074, 1007, 1019, 1034, 1044, 1032, 1007, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1057, 1055, 1005, 987, 1029, 1045, 1009, 3, 2, 0, 0, 0, 0, 0, 0, 992, 1028, 977, 1011, 1059, 1025, 1035, 1008, 7, 0, 0, 0, 0, 0, 0, 0, 1065, 1043, 1026, 1009, 993, 1058, 1007, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1062, 960, 997, 1038, 1005, 963, 1081, 7, 0, 0, 0, 0, 0, 0, 0, 1056, 963, 986, 1036, 1040, 985, 1016, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1006, 1015, 1009, 1022, 1038, 1039, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1003, 1011, 1021, 1033, 989, 999, 1076, 36, 22, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8216, 8142, 8212, 8151, 8182, 8176, 8222] avg=8192.0000 max=8235 max_rank=0 min=8142 min_rank=2 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8448, 8320, 8320, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8064, 8192, 8192, 8192, 8320, 8192, 8320, 8192, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1046, 1014, 1037, 1029, 1006, 1020, 1053, 992, 1025, 1042, 992, 1031, 956, 992, 1043, 1012, 1013, 1032, 994, 1005, 1015, 1074, 950, 1049, 1043, 1026, 1052, 1048, 1016, 1043, 1044, 989, 1025, 1002, 1004, 1045, 991, 1039, 1049, 1055, 1068, 1062, 969, 1048, 998, 1021, 950, 1034, 1008, 1021, 959, 1048, 1081, 1052, 1079, 994, 1037, 1004, 1057, 1039, 1013, 1055, 1039, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1026, 1050, 1012, 1017, 1020, 1030, 1032, 1020, 1032, 1047, 1018, 999, 1019, 1061, 955, 1017, 991, 1077, 1015, 1030, 1054, 1005, 1070, 1030, 1015, 1006, 1050, 1051, 991, 969, 1033, 1041, 988, 1067, 1033, 999, 1059, 1039, 1062, 1045, 954, 1035, 1006, 965, 1008, 938, 1041, 989, 1085, 1014, 1038, 1049, 990, 1025, 1060, 1043, 1039, 1033, 1024, 1048, 1011, 1031, 1019, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[994, 1068, 1048, 1049, 1049, 996, 997, 1004, 995, 1026, 1040, 1031, 1047, 1076, 998, 1036, 1007, 1040, 983, 995, 1088, 940, 1062, 997, 1025, 1069, 1087, 1037, 1003, 976, 1008, 1017, 1072, 1026, 1093, 1017, 1027, 1023, 989, 1021, 1006, 1013, 1003, 1014, 1041, 992, 1057, 1000, 997, 1029, 972, 1026, 1065, 1018, 1011, 1005, 999, 1024, 1037, 1035, 1052, 1013, 1080, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1068, 973, 993, 995, 1035, 1009, 1024, 1057, 1016, 1006, 1047, 1019, 1013, 1045, 971, 999, 1020, 1010, 1045, 1037, 1036, 1005, 1018, 1060, 1068, 958, 1021, 1108, 1065, 1059, 1029, 1009, 1046, 1003, 1042, 1019, 991, 1009, 1061, 1005, 966, 978, 1054, 1025, 1079, 1005, 1038, 1007, 1043, 1033, 1005, 976, 1031, 1027, 1022, 1014, 1068, 1038, 1004, 1005, 1048, 1056, 1032, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1028, 990, 1037, 1019, 988, 1071, 1092, 1015, 1023, 1020, 1034, 1044, 1005, 1042, 982, 1001, 1073, 1050, 1022, 977, 1065, 1006, 1047, 1001, 983, 1002, 1031, 995, 1040, 1047, 999, 1030, 1013, 1067, 992, 1036, 964, 1011, 1010, 1023, 1020, 1032, 1032, 1043, 972, 1008, 1004, 1031, 1028, 1039, 1021, 987, 1020, 1024, 968, 1086, 1037, 1026, 1042, 1119, 1062, 981, 1022, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1044, 1027, 1003, 971, 1048, 1045, 1005, 1052, 1044, 1025, 1045, 998, 1026, 1019, 1004, 1021, 1017, 1057, 1030, 1013, 1026, 965, 1099, 979, 974, 1056, 984, 1042, 1043, 979, 1058, 1035, 1012, 1028, 987, 1002, 1053, 1015, 1021, 1017, 1043, 1025, 1074, 1049, 975, 1012, 993, 991, 1031, 1017, 1001, 1051, 1016, 1030, 1020, 1033, 1015, 1024, 1000, 1043, 992, 1099, 1065, 1068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1039, 1061, 1028, 1012, 1043, 1019, 1075, 984, 1003, 1050, 1043, 1032, 1030, 1041, 984, 1039, 1019, 1025, 1034, 1036, 1079, 1076, 1011, 1028, 991, 1036, 1038, 1036, 1046, 1027, 1016, 1057, 1049, 1025, 987, 1003, 1032, 1007, 986, 1002, 1022, 989, 992, 990, 1004, 976, 991, 1080, 995, 1076, 1045, 1041, 993, 998, 999, 1035, 1063, 1048, 992, 1038, 987, 990, 1069, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[998, 991, 1001, 1020, 1046, 1036, 979, 1015, 1040, 1003, 1113, 1002, 1028, 1030, 1003, 1033, 1009, 1015, 1011, 1059, 1012, 1036, 1030, 986, 997, 1004, 971, 1035, 1032, 1080, 1018, 1108, 993, 1015, 1040, 1026, 1050, 1024, 1066, 1034, 1021, 1047, 944, 1000, 1008, 1018, 986, 1073, 1010, 1005, 1041, 1032, 1020, 981, 1043, 1038, 1014, 1010, 1086, 1028, 1013, 1020, 1052, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1046, 1014, 1037, 1029, 1006, 1020, 1052, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1042, 992, 1031, 956, 992, 1043, 1012, 12, 1, 0, 0, 0, 0, 0, 0, 1013, 1032, 994, 1005, 1003, 1074, 950, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1026, 1052, 1022, 1016, 1043, 1044, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1002, 1004, 1045, 991, 1039, 1049, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1062, 969, 1048, 998, 1021, 950, 1034, 64, 26, 0, 0, 0, 0, 0, 0, 1008, 1021, 959, 1048, 1081, 1052, 1079, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1004, 1057, 1039, 1013, 1055, 975, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8106, 8120, 8235, 8210, 8240, 8242, 8187] avg=8192.0000 max=8242 max_rank=6 min=8106 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32879 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1068, 973, 993, 995, 1035, 1009, 1023, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1006, 1047, 1019, 1013, 1045, 971, 999, 12, 1, 0, 0, 0, 0, 0, 0, 1020, 1010, 1045, 1037, 1024, 1005, 1018, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 958, 1021, 1081, 1065, 1059, 1029, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1003, 1042, 1019, 991, 1009, 1061, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 966, 978, 1054, 1025, 1079, 1005, 1038, 1007, 63, 27, 0, 0, 0, 0, 0, 0, 1043, 1033, 1005, 976, 1031, 1027, 1022, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1038, 1004, 1005, 1048, 1056, 969, 988, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8129, 8219, 8290, 8176, 8242, 8151, 8176] avg=8192.0000 max=8290 max_rank=3 min=8129 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32745 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 1050, 1012, 1017, 1020, 1030, 1031, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1047, 1018, 999, 1019, 1061, 955, 1017, 13, 1, 1, 0, 0, 0, 0, 0, 991, 1077, 1015, 1030, 1041, 1005, 1070, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1006, 1050, 1025, 991, 969, 1033, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1067, 1033, 999, 1059, 1039, 1062, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1035, 1006, 965, 1008, 938, 1041, 989, 63, 26, 0, 0, 0, 0, 0, 0, 1085, 1014, 1038, 1049, 990, 1025, 1060, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1033, 1024, 1048, 1011, 1031, 956, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8163, 8259, 8130, 8292, 8025, 8303, 8158] avg=8192.0000 max=8303 max_rank=6 min=8025 min_rank=5 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32778 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[994, 1068, 1048, 1049, 1049, 996, 996, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1026, 1040, 1031, 1047, 1076, 998, 1036, 13, 1, 0, 0, 0, 0, 0, 0, 1007, 1040, 983, 995, 1075, 940, 1062, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1069, 1087, 1012, 1003, 976, 1008, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1026, 1093, 1017, 1027, 1023, 989, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1013, 1003, 1014, 1041, 992, 1057, 1000, 66, 25, 0, 0, 0, 0, 0, 0, 997, 1029, 972, 1026, 1065, 1018, 1011, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1024, 1037, 1035, 1052, 1013, 1014, 991, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8263, 8099, 8197, 8268, 8217, 8123, 8165] avg=8192.0000 max=8268 max_rank=4 min=8099 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32773 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1044, 1027, 1003, 971, 1048, 1045, 1004, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1025, 1045, 998, 1026, 1019, 1004, 1021, 12, 1, 1, 0, 0, 0, 0, 0, 1017, 1057, 1030, 1013, 1014, 965, 1099, 979, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1056, 984, 1017, 1043, 979, 1058, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1028, 987, 1002, 1053, 1015, 1021, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1025, 1074, 1049, 975, 1012, 993, 991, 65, 25, 0, 0, 0, 0, 0, 0, 1031, 1017, 1001, 1051, 1016, 1030, 1020, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1024, 1000, 1043, 992, 1099, 1000, 1068, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8196, 8174, 8146, 8135, 8252, 8198, 8241] avg=8192.0000 max=8252 max_rank=5 min=8135 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32826 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1028, 990, 1037, 1019, 988, 1071, 1090, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1020, 1034, 1044, 1005, 1042, 982, 1001, 17, 2, 4, 2, 0, 0, 0, 0, 1073, 1050, 1022, 977, 1048, 1006, 1047, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1002, 1031, 964, 1040, 1047, 999, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1067, 992, 1036, 964, 1011, 1010, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1032, 1032, 1043, 972, 1008, 1004, 1031, 67, 29, 0, 0, 0, 0, 0, 0, 1028, 1039, 1021, 987, 1020, 1024, 968, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1026, 1042, 1119, 1062, 981, 955, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8176, 8224, 8096, 8116, 8238, 8169, 8279] avg=8192.0000 max=8279 max_rank=7 min=8096 min_rank=3 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32802 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1039, 1061, 1028, 1012, 1043, 1019, 1074, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1050, 1043, 1032, 1030, 1041, 984, 1039, 13, 1, 1, 0, 0, 0, 0, 0, 1019, 1025, 1034, 1036, 1066, 1076, 1011, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1036, 1038, 1011, 1046, 1027, 1016, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1025, 987, 1003, 1032, 1007, 986, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 989, 992, 990, 1004, 976, 991, 1080, 66, 25, 0, 0, 0, 0, 0, 0, 995, 1076, 1045, 1041, 993, 998, 999, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1048, 992, 1038, 987, 990, 1003, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8237, 8295, 8222, 8091, 8135, 8181, 8115] avg=8192.0000 max=8295 max_rank=2 min=8091 min_rank=4 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32522 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[998, 991, 1001, 1020, 1046, 1036, 978, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1003, 1113, 1002, 1028, 1030, 1003, 1033, 12, 1, 1, 0, 0, 0, 0, 0, 1009, 1015, 1011, 1059, 1000, 1036, 1030, 986, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1004, 971, 1010, 1032, 1080, 1018, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1015, 1040, 1026, 1050, 1024, 1066, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1047, 944, 1000, 1008, 1018, 986, 1073, 65, 25, 0, 0, 0, 0, 0, 0, 1010, 1005, 1041, 1032, 1020, 981, 1043, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1010, 1086, 1028, 1013, 1020, 987, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8085, 8266, 8146, 8220, 8248, 8187, 8169, 8215] avg=8192.0000 max=8266 max_rank=1 min=8085 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32819 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8064, 8064, 8320, 640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8448, 8192, 8192, 8320, 8064, 8192, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8448, 8192, 8320, 7936, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1000, 1010, 1045, 990, 1035, 1009, 1007, 1065, 1026, 1008, 1016, 1043, 957, 1004, 1005, 1052, 1070, 1047, 1008, 1043, 1009, 987, 1009, 1055, 1006, 1039, 1029, 999, 1038, 1050, 982, 989, 1013, 1012, 1026, 1049, 992, 1072, 1021, 1017, 1012, 1015, 1041, 1009, 982, 1037, 1066, 1028, 1052, 1046, 1047, 1064, 1045, 1015, 1038, 976, 993, 1051, 1038, 1042, 1033, 1018, 1024, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1030, 988, 1018, 1065, 1070, 993, 1086, 1039, 991, 1027, 1046, 1049, 1024, 994, 971, 1025, 1020, 1002, 966, 1044, 993, 1037, 1003, 1057, 1062, 1035, 1035, 1078, 1022, 976, 999, 1009, 1004, 988, 1018, 982, 1023, 1019, 962, 1026, 988, 1029, 1057, 1042, 1060, 1053, 1038, 1008, 1042, 1033, 1074, 999, 1045, 1046, 1026, 1038, 1011, 1023, 1022, 978, 1057, 1006, 1051, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1067, 1024, 1054, 1075, 997, 997, 997, 1037, 1006, 1015, 996, 977, 1006, 1027, 1016, 1027, 1073, 1065, 1007, 983, 1041, 994, 1041, 1057, 1037, 1008, 1046, 1038, 1015, 983, 1031, 1006, 1016, 1019, 1022, 1024, 1040, 1042, 1041, 1044, 1000, 1000, 959, 995, 1019, 1093, 1046, 1060, 1024, 1038, 1016, 1017, 1060, 993, 1002, 1007, 1052, 1018, 1019, 1017, 1021, 1045, 1012, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[999, 1073, 1055, 923, 1027, 996, 1022, 1064, 1115, 1007, 1002, 1033, 1051, 1015, 1041, 1040, 1015, 1000, 1005, 1043, 1046, 1042, 1024, 1014, 1024, 1019, 1028, 1041, 1005, 1010, 965, 1014, 1034, 1021, 1057, 995, 1033, 980, 1038, 1074, 995, 1048, 984, 1005, 1021, 1067, 1059, 999, 1012, 1025, 1017, 1015, 1038, 1007, 1026, 1032, 1045, 1006, 1038, 1030, 1051, 1006, 1015, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1028, 965, 1036, 1027, 1000, 1042, 1053, 1031, 1070, 1003, 1011, 1041, 1011, 992, 1023, 1031, 1022, 1069, 1034, 1052, 1033, 1054, 993, 1023, 1051, 1030, 1012, 988, 1052, 1047, 1006, 1035, 1033, 981, 1021, 1027, 1030, 1024, 1045, 960, 1026, 1052, 1029, 1047, 1033, 1008, 997, 1040, 1032, 997, 1083, 1025, 1053, 988, 1000, 1037, 1019, 987, 1044, 954, 1050, 1033, 1032, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[998, 1039, 1025, 1030, 979, 1028, 1015, 1025, 991, 1009, 994, 1060, 1066, 1024, 1015, 1017, 1027, 1063, 1062, 1034, 1046, 1053, 1088, 1047, 1074, 1011, 1002, 1035, 1051, 1009, 1025, 1011, 1031, 1031, 1022, 1052, 1034, 1090, 1012, 963, 1027, 1008, 953, 1032, 1032, 964, 1037, 1031, 982, 1068, 1019, 1040, 965, 1036, 1054, 982, 1033, 980, 1039, 1004, 965, 1018, 1055, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[993, 1005, 1010, 1003, 1038, 1036, 1004, 1054, 1035, 1039, 1065, 1017, 1006, 1029, 1005, 1049, 1030, 1070, 1024, 1034, 1044, 1055, 1034, 1026, 996, 1037, 963, 981, 1068, 1002, 994, 1057, 1067, 1037, 1019, 1038, 1045, 1014, 989, 1016, 1037, 1030, 1016, 1038, 1011, 1069, 1052, 979, 1024, 976, 1023, 983, 1048, 1052, 1018, 1028, 1036, 993, 967, 1050, 997, 993, 1066, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1011, 1047, 1022, 989, 1005, 1047, 1008, 1004, 1022, 1040, 1022, 1047, 1029, 1066, 989, 1019, 1020, 1042, 1040, 1058, 1002, 1042, 1024, 1067, 1041, 1018, 1062, 983, 1080, 1056, 994, 1012, 1009, 1020, 1015, 1013, 1024, 1019, 978, 1046, 1038, 1046, 999, 1015, 1004, 1019, 1016, 1014, 997, 1042, 1006, 987, 1030, 1073, 1037, 1028, 993, 1023, 1036, 1030, 1071, 984, 1037, 979] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1000, 1010, 1045, 990, 1035, 1009, 1007, 1065, 8, 1, 0, 0, 0, 0, 0, 0, 1026, 1008, 1016, 1043, 957, 1004, 1005, 1052, 10, 0, 0, 0, 0, 0, 0, 0, 1070, 979, 1008, 1043, 1009, 987, 1009, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1039, 1029, 999, 1038, 1050, 982, 989, 16, 0, 0, 0, 0, 0, 0, 0, 1013, 1012, 1026, 1049, 992, 1072, 1021, 1017, 15, 0, 0, 0, 0, 0, 0, 0, 1012, 1015, 1041, 1009, 982, 1037, 1066, 1028, 2, 0, 0, 0, 0, 0, 0, 0, 1052, 1046, 1037, 1064, 1045, 1015, 1038, 976, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1051, 1038, 1042, 1033, 1018, 1024, 1030, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8121, 8160, 8148, 8217, 8192, 8273, 8255] avg=8192.0000 max=8273 max_rank=6 min=8121 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32937 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1030, 988, 1018, 1065, 1070, 993, 1086, 1039, 9, 1, 0, 0, 0, 0, 0, 0, 991, 1027, 1046, 1049, 1024, 994, 971, 1025, 10, 0, 0, 0, 0, 0, 0, 0, 1020, 935, 966, 1044, 993, 1037, 1003, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1035, 1035, 1078, 1022, 976, 999, 1009, 16, 0, 0, 0, 0, 0, 0, 0, 1004, 988, 1018, 982, 1023, 1019, 962, 1026, 15, 0, 0, 0, 0, 0, 0, 0, 988, 1029, 1057, 1042, 1060, 1053, 1038, 1008, 2, 0, 0, 0, 0, 0, 0, 0, 1042, 1033, 1063, 999, 1045, 1046, 1026, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1023, 1022, 978, 1057, 1006, 1051, 1034, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8299, 8137, 8055, 8232, 8037, 8277, 8292, 8207] avg=8192.0000 max=8299 max_rank=0 min=8037 min_rank=4 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[999, 1073, 1055, 923, 1027, 996, 1022, 1064, 8, 1, 0, 0, 0, 0, 0, 0, 1115, 1007, 1002, 1033, 1051, 1015, 1041, 1040, 10, 0, 0, 0, 0, 0, 0, 0, 1015, 933, 1005, 1043, 1046, 1042, 1024, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1019, 1028, 1041, 1005, 1010, 965, 1014, 16, 0, 0, 0, 0, 0, 0, 0, 1034, 1021, 1057, 995, 1033, 980, 1038, 1074, 15, 0, 0, 0, 0, 0, 0, 0, 995, 1048, 984, 1005, 1021, 1067, 1059, 999, 2, 0, 0, 0, 0, 0, 0, 0, 1012, 1025, 1007, 1015, 1038, 1007, 1026, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1006, 1038, 1030, 1051, 1006, 1015, 1005, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8314, 8122, 8122, 8247, 8180, 8162, 8221] avg=8192.0000 max=8314 max_rank=1 min=8122 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1067, 1024, 1054, 1075, 997, 997, 997, 1037, 8, 1, 0, 0, 0, 0, 0, 0, 1006, 1015, 996, 977, 1006, 1027, 1016, 1027, 11, 0, 0, 0, 0, 0, 0, 0, 1073, 993, 1007, 983, 1041, 994, 1041, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1008, 1046, 1038, 1015, 983, 1031, 1006, 17, 0, 0, 0, 0, 0, 0, 0, 1016, 1019, 1022, 1024, 1040, 1042, 1041, 1044, 16, 0, 0, 0, 0, 0, 0, 0, 1000, 1000, 959, 995, 1019, 1093, 1046, 1060, 2, 0, 0, 0, 0, 0, 0, 0, 1024, 1038, 1006, 1017, 1060, 993, 1002, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1018, 1019, 1017, 1021, 1045, 1012, 1032, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8081, 8189, 8181, 8264, 8174, 8147, 8243] avg=8192.0000 max=8264 max_rank=4 min=8081 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1028, 965, 1036, 1027, 1000, 1042, 1053, 1031, 9, 1, 0, 0, 0, 0, 0, 0, 1070, 1003, 1011, 1041, 1011, 992, 1023, 1031, 11, 0, 0, 0, 0, 0, 0, 0, 1022, 997, 1034, 1052, 1033, 1054, 993, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1030, 1012, 988, 1052, 1047, 1006, 1035, 17, 0, 0, 0, 0, 0, 0, 0, 1033, 981, 1021, 1027, 1030, 1024, 1045, 960, 16, 0, 0, 0, 0, 0, 0, 0, 1026, 1052, 1029, 1047, 1033, 1008, 997, 1040, 2, 0, 0, 0, 0, 0, 0, 0, 1032, 997, 1072, 1025, 1053, 988, 1000, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 987, 1044, 954, 1050, 1033, 1032, 984, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8208, 8238, 8137, 8234, 8204, 8130] avg=8192.0000 max=8238 max_rank=3 min=8130 min_rank=7 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[998, 1039, 1025, 1030, 979, 1028, 1015, 1025, 8, 1, 0, 0, 0, 0, 0, 0, 991, 1009, 994, 1060, 1066, 1024, 1015, 1017, 11, 0, 0, 0, 0, 0, 0, 0, 1027, 991, 1062, 1034, 1046, 1053, 1088, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1011, 1002, 1035, 1051, 1009, 1025, 1011, 17, 0, 0, 0, 0, 0, 0, 0, 1031, 1031, 1022, 1052, 1034, 1090, 1012, 963, 16, 0, 0, 0, 0, 0, 0, 0, 1027, 1008, 953, 1032, 1032, 964, 1037, 1031, 2, 0, 0, 0, 0, 0, 0, 0, 982, 1068, 1009, 1040, 965, 1036, 1054, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 980, 1039, 1004, 965, 1018, 1055, 1024, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8187, 8348, 8235, 8251, 8086, 8136, 8145] avg=8192.0000 max=8348 max_rank=2 min=8086 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32618 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[993, 1005, 1010, 1003, 1038, 1036, 1004, 1054, 12, 5, 0, 0, 0, 0, 0, 0, 1035, 1039, 1065, 1017, 1006, 1029, 1005, 1049, 14, 0, 0, 0, 0, 0, 0, 0, 1030, 980, 1024, 1034, 1044, 1055, 1034, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1037, 963, 981, 1068, 1002, 994, 1057, 20, 0, 0, 0, 0, 0, 0, 0, 1067, 1037, 1019, 1038, 1045, 1014, 989, 1016, 19, 0, 0, 0, 0, 0, 0, 0, 1037, 1030, 1016, 1038, 1011, 1069, 1052, 979, 8, 0, 0, 0, 0, 0, 0, 0, 1024, 976, 1003, 983, 1048, 1052, 1018, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 993, 967, 1050, 997, 993, 1066, 1022, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8259, 8227, 8118, 8244, 8240, 8132, 8156] avg=8192.0000 max=8259 max_rank=1 min=8118 min_rank=3 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1011, 1047, 1022, 989, 1005, 1047, 1008, 1004, 8, 1, 0, 0, 0, 0, 0, 0, 1022, 1040, 1022, 1047, 1029, 1066, 989, 1019, 10, 0, 0, 0, 0, 0, 0, 0, 1020, 974, 1040, 1058, 1002, 1042, 1024, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1018, 1062, 983, 1080, 1056, 994, 1012, 16, 0, 0, 0, 0, 0, 0, 0, 1009, 1020, 1015, 1013, 1024, 1019, 978, 1046, 15, 0, 0, 0, 0, 0, 0, 0, 1038, 1046, 999, 1015, 1004, 1019, 1016, 1014, 2, 0, 0, 0, 0, 0, 0, 0, 997, 1042, 996, 987, 1030, 1073, 1037, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1023, 1036, 1030, 1071, 984, 1037, 979, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8244, 8227, 8262, 8139, 8153, 8190, 8179] avg=8192.0000 max=8262 max_rank=3 min=8139 min_rank=4 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 7808, 8192, 8320, 8320, 8320, 8320, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8192, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8448, 8192, 8064, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1011, 1057, 960, 993, 1018, 981, 1046, 1032, 1032, 1041, 1067, 1001, 1010, 1044, 1006, 1085, 1048, 1063, 1032, 1044, 1024, 996, 1026, 1009, 1026, 1050, 1022, 1057, 1035, 998, 1008, 1046, 1075, 1003, 991, 1059, 1036, 1047, 1023, 1020, 1008, 1010, 1005, 1040, 1011, 1059, 1022, 970, 1010, 1024, 1002, 1007, 1027, 996, 1003, 981, 1050, 1009, 1035, 1000, 1023, 1074, 1034, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1015, 1057, 1000, 1033, 1039, 1058, 1065, 995, 1004, 986, 1070, 1020, 1012, 1035, 1057, 1037, 980, 1055, 1024, 999, 985, 1036, 998, 1048, 1042, 1029, 1017, 1040, 1027, 1026, 982, 1031, 989, 983, 1074, 1013, 1000, 1011, 1006, 1024, 990, 1041, 1030, 989, 986, 1024, 1050, 1005, 1038, 1017, 1060, 1031, 1013, 1034, 1075, 1017, 1079, 1002, 979, 1039, 1020, 1009, 1046, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1040, 1020, 995, 998, 1005, 1049, 1027, 1024, 1008, 1024, 1020, 993, 1022, 1055, 1040, 1032, 1005, 1055, 1006, 1056, 1061, 1047, 1033, 1013, 1007, 975, 1065, 1035, 1029, 998, 1073, 1020, 1054, 1057, 1055, 980, 1006, 1020, 1057, 1022, 992, 1049, 1052, 1034, 1050, 984, 1050, 1024, 995, 1038, 1040, 1006, 1054, 1040, 992, 1003, 1023, 989, 990, 1019, 1030, 966, 1023, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1028, 1025, 1045, 1085, 1046, 974, 1003, 1033, 1053, 1034, 993, 1029, 1052, 960, 996, 1044, 1001, 1024, 1031, 1004, 1073, 1026, 1046, 1042, 1018, 994, 1037, 1026, 936, 1023, 1042, 1049, 1077, 1013, 1015, 1005, 1049, 998, 1045, 1029, 986, 1053, 1069, 1010, 1044, 1062, 1045, 1048, 1057, 999, 1035, 977, 1029, 992, 1044, 1038, 1040, 1023, 978, 989, 1026, 1003, 1014, 972] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1035, 1023, 1056, 1032, 982, 1042, 1010, 1015, 1004, 1036, 1044, 1029, 1006, 1023, 1023, 994, 1037, 959, 1056, 1046, 1025, 1020, 1081, 983, 1051, 1013, 1016, 1014, 1064, 1035, 1018, 1053, 1023, 1036, 1001, 1022, 1010, 1071, 988, 1023, 979, 1051, 980, 1064, 1053, 1021, 995, 1032, 1068, 1056, 1021, 997, 1020, 1006, 1055, 1005, 1025, 1033, 1049, 1045, 1026, 954, 979, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[964, 971, 1032, 1004, 1041, 991, 1061, 1002, 993, 1030, 1036, 1029, 1023, 1053, 1046, 1031, 1039, 1010, 1023, 1029, 1006, 1006, 986, 1044, 1004, 1037, 1009, 999, 1019, 1044, 1007, 1008, 1001, 1076, 1014, 1004, 1003, 1033, 1003, 1037, 1051, 1045, 999, 1051, 1071, 1009, 1039, 1018, 1009, 1052, 1043, 1034, 1061, 1037, 991, 1084, 1024, 1024, 1007, 1049, 1003, 1052, 996, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1010, 1005, 1037, 1044, 1073, 983, 991, 1037, 1056, 1017, 1003, 1037, 1012, 1022, 1010, 1043, 1016, 1015, 1004, 990, 1008, 1043, 1010, 1024, 1014, 1013, 978, 1009, 1041, 1044, 1051, 1013, 1023, 1027, 1012, 1070, 963, 982, 1017, 1056, 1001, 996, 1028, 1044, 1036, 1043, 1009, 1014, 1040, 992, 1039, 1028, 1051, 1072, 982, 1056, 972, 992, 1039, 1044, 1057, 1059, 1084, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1006, 1033, 997, 1055, 1038, 1034, 1015, 978, 1034, 1020, 1034, 992, 1059, 1038, 1008, 1021, 1005, 1014, 1018, 1029, 1008, 1020, 1074, 1010, 1016, 1009, 1047, 1006, 978, 1062, 1040, 1020, 1049, 1059, 1047, 969, 1070, 1020, 1046, 1021, 1045, 945, 1000, 1006, 1063, 1054, 1004, 1034, 1029, 1009, 1027, 1004, 1048, 1038, 993, 1060, 1062, 1038, 1025, 990, 1032, 1008, 978, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1011, 1057, 960, 993, 1018, 981, 1046, 1032, 17, 4, 0, 0, 0, 0, 0, 0, 1032, 1041, 1067, 1001, 1010, 1044, 1006, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1063, 1032, 1044, 1024, 996, 1026, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1050, 1022, 1057, 1035, 998, 1008, 1046, 3, 1, 0, 0, 0, 0, 0, 0, 1066, 1003, 991, 1059, 1036, 1047, 1023, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1010, 1005, 1040, 1008, 1059, 1022, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1024, 1002, 1007, 1010, 996, 1003, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1009, 1035, 1000, 1023, 1074, 1034, 1014, 12, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8270, 8242, 8246, 8245, 8122, 8033, 8259] avg=8192.0000 max=8270 max_rank=1 min=8033 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1040, 1020, 995, 998, 1005, 1049, 1027, 1024, 18, 4, 0, 0, 0, 0, 0, 0, 1008, 1024, 1020, 993, 1022, 1055, 1040, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1055, 1006, 1056, 1061, 1047, 1033, 1013, 1, 0, 0, 0, 0, 0, 0, 0, 1007, 975, 1065, 1035, 1029, 998, 1073, 1020, 3, 1, 0, 0, 0, 0, 0, 0, 1045, 1057, 1055, 980, 1006, 1020, 1057, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1049, 1052, 1034, 1046, 984, 1050, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1038, 1040, 1006, 1036, 1040, 992, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 989, 990, 1019, 1030, 966, 1023, 1012, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8179, 8277, 8206, 8242, 8231, 8150, 8071] avg=8192.0000 max=8277 max_rank=2 min=8071 min_rank=7 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1035, 1023, 1056, 1032, 982, 1042, 1010, 1015, 17, 4, 0, 0, 0, 0, 0, 0, 1004, 1036, 1044, 1029, 1006, 1023, 1023, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 959, 1056, 1046, 1025, 1020, 1081, 983, 1, 0, 0, 0, 0, 0, 0, 0, 1051, 1013, 1016, 1014, 1064, 1035, 1018, 1053, 3, 1, 0, 0, 0, 0, 0, 0, 1015, 1036, 1001, 1022, 1010, 1071, 988, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1051, 980, 1064, 1049, 1021, 995, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1056, 1021, 997, 1003, 1006, 1055, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1033, 1049, 1045, 1026, 954, 979, 1023, 11, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8144, 8208, 8268, 8166, 8171, 8211, 8152] avg=8192.0000 max=8268 max_rank=3 min=8144 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1015, 1057, 1000, 1033, 1039, 1058, 1065, 995, 17, 4, 0, 0, 0, 0, 0, 0, 1004, 986, 1070, 1020, 1012, 1035, 1057, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1055, 1024, 999, 985, 1036, 998, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1029, 1017, 1040, 1027, 1026, 982, 1031, 3, 1, 0, 0, 0, 0, 0, 0, 981, 983, 1074, 1013, 1000, 1011, 1006, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1041, 1030, 989, 983, 1024, 1050, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1017, 1060, 1031, 996, 1034, 1075, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1002, 979, 1039, 1020, 1009, 1046, 1060, 11, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8206, 8125, 8198, 8092, 8112, 8268, 8252] avg=8192.0000 max=8283 max_rank=0 min=8092 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1028, 1025, 1045, 1085, 1046, 974, 1003, 1033, 17, 4, 0, 0, 0, 0, 0, 0, 1053, 1034, 993, 1029, 1052, 960, 996, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1024, 1031, 1004, 1073, 1026, 1046, 1042, 1, 0, 0, 0, 0, 0, 0, 0, 1018, 994, 1037, 1026, 936, 1023, 1042, 1049, 3, 1, 0, 0, 0, 0, 0, 0, 1068, 1013, 1015, 1005, 1049, 998, 1045, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1053, 1069, 1010, 1040, 1062, 1045, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 999, 1035, 977, 1012, 992, 1044, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1023, 978, 989, 1026, 1003, 1014, 972, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8146, 8248, 8129, 8222, 8313, 8154, 8064] avg=8192.0000 max=8313 max_rank=5 min=8064 min_rank=7 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[964, 971, 1032, 1004, 1041, 991, 1061, 1002, 23, 9, 0, 0, 0, 0, 0, 0, 993, 1030, 1036, 1029, 1023, 1053, 1046, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1010, 1023, 1029, 1006, 1006, 986, 1044, 4, 0, 0, 0, 0, 0, 0, 0, 1004, 1037, 1009, 999, 1019, 1044, 1007, 1008, 7, 6, 0, 0, 0, 0, 0, 0, 984, 1076, 1014, 1004, 1003, 1033, 1003, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1045, 999, 1051, 1060, 1009, 1039, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1052, 1043, 1034, 1038, 1037, 991, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1024, 1007, 1049, 1003, 1052, 996, 1039, 17, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8215, 8147, 8140, 8154, 8272, 8288, 8222] avg=8192.0000 max=8288 max_rank=6 min=8098 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1010, 1005, 1037, 1044, 1073, 983, 991, 1037, 18, 4, 0, 0, 0, 0, 0, 0, 1056, 1017, 1003, 1037, 1012, 1022, 1010, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1015, 1004, 990, 1008, 1043, 1010, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1013, 978, 1009, 1041, 1044, 1051, 1013, 3, 1, 0, 0, 0, 0, 0, 0, 1015, 1027, 1012, 1070, 963, 982, 1017, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 996, 1028, 1044, 1033, 1043, 1009, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 992, 1039, 1028, 1033, 1072, 982, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 972, 992, 1039, 1044, 1057, 1059, 1084, 1055, 11, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8185, 8110, 8167, 8142, 8168, 8242, 8320] avg=8192.0000 max=8320 max_rank=7 min=8110 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1006, 1033, 997, 1055, 1038, 1034, 1015, 978, 18, 4, 0, 0, 0, 0, 0, 0, 1034, 1020, 1034, 992, 1059, 1038, 1008, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1014, 1018, 1029, 1008, 1020, 1074, 1010, 1, 0, 0, 0, 0, 0, 0, 0, 1016, 1009, 1047, 1006, 978, 1062, 1040, 1020, 3, 1, 0, 0, 0, 0, 0, 0, 1041, 1059, 1047, 969, 1070, 1020, 1046, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 945, 1000, 1006, 1059, 1054, 1004, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1009, 1027, 1004, 1030, 1038, 993, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1038, 1025, 990, 1032, 1008, 978, 1045, 11, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8191, 8179, 8182, 8273, 8147, 8190, 8196] avg=8192.0000 max=8273 max_rank=4 min=8147 min_rank=5 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1035, 1031, 1105, 995, 1062, 993, 1039, 1051, 1018, 1003, 1046, 1027, 995, 1038, 1034, 1017, 1018, 1018, 1028, 978, 998, 1016, 1016, 984, 1014, 984, 1025, 1060, 993, 1062, 995, 993, 971, 1050, 1062, 1028, 1020, 1032, 1031, 1084, 1015, 1006, 1048, 1021, 1002, 1060, 980, 980, 1034, 1011, 1049, 1034, 1015, 1034, 1026, 985, 1044, 1038, 996, 1006, 1032, 1068, 1018, 1085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1007, 1042, 1028, 989, 1047, 1009, 1066, 975, 986, 1057, 1040, 1034, 992, 963, 1008, 1051, 968, 1012, 1054, 1039, 971, 1087, 1079, 982, 987, 957, 1045, 1064, 1021, 991, 1019, 1064, 1072, 1078, 1063, 1039, 996, 1047, 1030, 1061, 1035, 1031, 1043, 1017, 1019, 1032, 999, 1034, 1033, 992, 1009, 1023, 1023, 1014, 1065, 968, 1041, 998, 999, 999, 1010, 1063, 1046, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1039, 999, 1007, 1006, 1039, 1060, 1042, 1070, 964, 1034, 1004, 1022, 1055, 1026, 1010, 982, 1088, 1031, 1025, 1068, 1003, 985, 956, 1060, 1003, 1024, 991, 1042, 978, 991, 989, 948, 1020, 1052, 1056, 1051, 1030, 1058, 996, 1084, 1083, 995, 1053, 1009, 1026, 1004, 1000, 1011, 1014, 1062, 1053, 1060, 1034, 1030, 1038, 1041, 1045, 1029, 1034, 952, 1032, 958, 1060, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1024, 1053, 969, 1032, 991, 981, 1043, 1007, 965, 1038, 1058, 968, 1019, 1065, 1047, 1051, 986, 1022, 991, 1011, 1078, 1021, 1000, 1065, 991, 1023, 1052, 995, 954, 1054, 977, 1039, 999, 987, 1029, 1044, 1078, 1016, 1022, 1010, 1010, 1010, 1062, 994, 1051, 978, 992, 1029, 1044, 1044, 1067, 1039, 1026, 1113, 1014, 1038, 1026, 1023, 1010, 1023, 1048, 1054, 1021, 1065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1023, 1046, 968, 1032, 1040, 1053, 1003, 1055, 1017, 1083, 1063, 1040, 1015, 967, 1003, 1005, 1017, 1002, 1048, 1064, 1029, 1029, 1040, 1066, 1008, 1067, 1006, 1041, 1092, 1032, 1004, 1027, 992, 1021, 1004, 1022, 1017, 1044, 1065, 965, 1011, 1031, 1075, 982, 1048, 1046, 996, 996, 1062, 1038, 1045, 1065, 1011, 983, 1025, 1008, 1001, 1048, 965, 997, 1047, 997, 961, 983] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1041, 1007, 1024, 999, 991, 938, 1029, 1074, 1038, 1060, 1061, 1021, 1002, 1004, 1033, 997, 1017, 1000, 1056, 997, 1068, 1045, 1035, 1005, 999, 1049, 1024, 994, 1067, 1032, 1016, 1033, 1007, 1017, 1041, 1006, 1033, 976, 985, 1033, 1052, 1107, 1094, 1043, 1024, 1018, 1026, 993, 1020, 1050, 1054, 1024, 999, 1005, 990, 1016, 980, 1011, 1004, 1021, 1021, 1055, 1043, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1037, 1059, 1071, 1022, 1010, 1070, 1020, 1065, 1046, 1015, 1071, 981, 986, 1049, 1086, 1016, 1033, 968, 1012, 1035, 1013, 1075, 1034, 1029, 995, 997, 977, 1052, 1017, 1026, 1019, 1028, 1060, 1022, 987, 1011, 1018, 1026, 1032, 990, 984, 993, 1055, 966, 1008, 1029, 1026, 1013, 1068, 1028, 1050, 1033, 978, 965, 1025, 1046, 1090, 1053, 963, 999, 1024, 1039, 1031, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1036, 1002, 980, 1028, 1070, 980, 1010, 1053, 1023, 1050, 1015, 1057, 1031, 1050, 1018, 1008, 1007, 1056, 1032, 1040, 1044, 1023, 1031, 983, 1030, 972, 1002, 1040, 1043, 1058, 1038, 1079, 1025, 1062, 1004, 1013, 1073, 1022, 992, 1043, 1027, 1006, 1016, 1050, 1041, 994, 1040, 1004, 995, 992, 996, 1013, 982, 995, 1010, 1056, 1031, 1048, 1087, 1024, 970, 972, 1074, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1039, 999, 1007, 1006, 1039, 1060, 1042, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 964, 1034, 1004, 1022, 1055, 1026, 1010, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 1031, 1025, 1068, 1003, 982, 956, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1024, 991, 1042, 978, 991, 989, 948, 34, 10, 0, 0, 0, 0, 0, 0, 1020, 1018, 1056, 1051, 1030, 1058, 996, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 995, 1053, 1009, 1026, 1004, 1000, 1011, 7, 7, 0, 0, 0, 0, 0, 0, 1014, 1062, 1038, 1060, 1034, 1030, 1038, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1029, 1034, 952, 1032, 958, 1060, 1025, 8, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8097, 8213, 8010, 8313, 8195, 8317, 8146] avg=8192.0000 max=8317 max_rank=6 min=8010 min_rank=3 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1007, 1042, 1028, 989, 1047, 1009, 1066, 960, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1057, 1040, 1034, 992, 963, 1008, 1051, 1, 0, 0, 0, 0, 0, 0, 0, 968, 1012, 1054, 1039, 971, 1083, 1079, 982, 0, 0, 0, 0, 0, 0, 0, 0, 987, 957, 1045, 1064, 1021, 991, 1019, 1064, 34, 9, 0, 0, 0, 0, 0, 0, 1072, 1044, 1063, 1039, 996, 1047, 1030, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1031, 1043, 1017, 1019, 1032, 999, 1034, 6, 6, 0, 0, 0, 0, 0, 0, 1033, 992, 995, 1023, 1023, 1014, 1065, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 998, 999, 999, 1010, 1063, 1046, 1023, 8, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8132, 8188, 8191, 8352, 8222, 8113, 8190] avg=8192.0000 max=8352 max_rank=4 min=8113 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1035, 1031, 1105, 995, 1062, 993, 1039, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1003, 1046, 1027, 995, 1038, 1034, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1018, 1028, 978, 998, 1013, 1016, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 984, 1025, 1060, 993, 1062, 995, 993, 33, 9, 0, 0, 0, 0, 0, 0, 971, 1017, 1062, 1028, 1020, 1032, 1031, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1006, 1048, 1021, 1002, 1060, 980, 980, 7, 7, 0, 0, 0, 0, 0, 0, 1034, 1011, 1034, 1034, 1015, 1034, 1026, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1038, 996, 1006, 1032, 1068, 1018, 1085, 8, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8178, 8053, 8168, 8245, 8126, 8173, 8298] avg=8192.0000 max=8298 max_rank=7 min=8053 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1041, 1007, 1024, 999, 991, 938, 1029, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1060, 1061, 1021, 1002, 1004, 1033, 997, 1, 0, 0, 0, 0, 0, 0, 0, 1017, 1000, 1056, 997, 1068, 1041, 1035, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1049, 1024, 994, 1067, 1032, 1016, 1033, 32, 10, 0, 0, 0, 0, 0, 0, 1007, 985, 1041, 1006, 1033, 976, 985, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1107, 1094, 1043, 1024, 1018, 1026, 993, 7, 7, 0, 0, 0, 0, 0, 0, 1020, 1050, 1039, 1024, 999, 1005, 990, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1011, 1004, 1021, 1021, 1055, 1043, 1032, 8, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 8217, 8219, 8256, 8066, 8371, 8143, 8178] avg=8192.0000 max=8371 max_rank=5 min=8066 min_rank=4 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1023, 1046, 968, 1032, 1040, 1053, 1003, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1083, 1063, 1040, 1015, 967, 1003, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1002, 1048, 1064, 1029, 1026, 1040, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1067, 1006, 1041, 1092, 1032, 1004, 1027, 33, 9, 0, 0, 0, 0, 0, 0, 992, 988, 1004, 1022, 1017, 1044, 1065, 965, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1031, 1075, 982, 1048, 1046, 996, 996, 7, 7, 0, 0, 0, 0, 0, 0, 1062, 1038, 1030, 1065, 1011, 983, 1025, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1048, 965, 997, 1047, 997, 961, 983, 8, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8193, 8292, 8319, 8097, 8199, 8222, 8010] avg=8192.0000 max=8319 max_rank=3 min=8010 min_rank=7 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32528 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1037, 1059, 1071, 1022, 1010, 1070, 1020, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1015, 1071, 981, 986, 1049, 1086, 1016, 1, 0, 0, 0, 0, 0, 0, 0, 1033, 968, 1012, 1035, 1013, 1071, 1034, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 995, 997, 977, 1052, 1017, 1026, 1019, 1028, 33, 10, 0, 0, 0, 0, 0, 0, 1060, 989, 987, 1011, 1018, 1026, 1032, 990, 0, 0, 0, 0, 0, 0, 0, 0, 984, 993, 1055, 966, 1008, 1029, 1026, 1013, 7, 7, 0, 0, 0, 0, 0, 0, 1068, 1028, 1035, 1033, 978, 965, 1025, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 1053, 963, 999, 1024, 1039, 1031, 1010, 8, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8337, 8251, 8195, 8154, 8113, 8088, 8178, 8220] avg=8192.0000 max=8337 max_rank=0 min=8088 min_rank=5 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1024, 1053, 969, 1032, 991, 981, 1043, 985, 0, 0, 0, 0, 0, 0, 0, 0, 965, 1038, 1058, 968, 1019, 1065, 1047, 1051, 5, 0, 0, 0, 0, 0, 0, 0, 986, 1022, 991, 1011, 1078, 1010, 1000, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1023, 1052, 995, 954, 1054, 977, 1039, 35, 13, 0, 0, 0, 0, 0, 0, 999, 952, 1029, 1044, 1078, 1016, 1022, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1010, 1062, 994, 1051, 978, 992, 1029, 9, 9, 0, 0, 0, 0, 0, 0, 1044, 1044, 1047, 1039, 1026, 1113, 1014, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1023, 1010, 1023, 1048, 1054, 1021, 1065, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8078, 8216, 8163, 8133, 8150, 8144, 8365, 8287] avg=8192.0000 max=8365 max_rank=6 min=8078 min_rank=0 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32946 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1036, 1002, 980, 1028, 1070, 980, 1010, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1050, 1015, 1057, 1031, 1050, 1018, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1056, 1032, 1040, 1044, 1020, 1031, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 972, 1002, 1040, 1043, 1058, 1038, 1079, 34, 9, 0, 0, 0, 0, 0, 0, 1025, 1028, 1004, 1013, 1073, 1022, 992, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1006, 1016, 1050, 1041, 994, 1040, 1004, 7, 6, 0, 0, 0, 0, 0, 0, 995, 992, 982, 1013, 982, 995, 1010, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1048, 1087, 1024, 970, 972, 1074, 990, 8, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8252, 8213, 8305, 8200, 8191, 8025, 8207] avg=8192.0000 max=8305 max_rank=3 min=8025 min_rank=6 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8448, 8448, 8192, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8448, 8192, 8320, 8192, 8064, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8064, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8192, 8320, 8192, 8320, 8064, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1008, 1074, 1017, 1022, 1009, 1027, 1027, 992, 1006, 984, 1016, 1004, 1040, 1043, 1056, 1074, 1025, 1021, 991, 990, 1028, 1032, 959, 1041, 968, 1034, 999, 1034, 983, 1021, 1031, 1013, 1031, 1062, 1018, 972, 996, 1055, 1052, 1043, 1007, 1100, 1091, 1087, 986, 1020, 1038, 1006, 998, 961, 1052, 1071, 1034, 1018, 1077, 998, 987, 1033, 1061, 977, 1058, 1013, 1035, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1065, 1023, 1032, 1061, 1026, 1022, 1001, 1040, 1031, 964, 1035, 1013, 1029, 1036, 1040, 999, 1055, 1090, 1033, 994, 1033, 1002, 964, 1017, 1061, 1020, 953, 1032, 1066, 1013, 1008, 1032, 1045, 1000, 1045, 985, 1003, 1022, 1038, 1031, 1062, 1023, 1023, 1045, 1022, 1045, 1037, 1050, 1012, 997, 1027, 1012, 992, 1040, 1010, 980, 1064, 973, 1020, 1065, 1013, 1025, 1019, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[991, 1008, 1039, 1060, 1089, 972, 1004, 1030, 1020, 1043, 1084, 994, 1009, 1056, 1013, 1031, 1038, 955, 1021, 1017, 1053, 1050, 975, 985, 1040, 967, 1003, 977, 1054, 1015, 1001, 1030, 1045, 1018, 1035, 1056, 1023, 1009, 1042, 1046, 1036, 1021, 992, 1020, 983, 1000, 1023, 1029, 994, 1010, 1073, 1064, 1027, 1074, 1048, 1045, 1007, 1052, 1002, 1067, 1016, 1034, 1010, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1021, 1009, 1053, 1026, 998, 1015, 1009, 1052, 1000, 993, 1075, 1058, 1067, 1000, 1028, 1021, 995, 995, 1017, 1045, 1017, 995, 1008, 1047, 1007, 988, 1106, 1034, 1037, 1046, 1040, 1067, 1016, 1004, 1021, 1002, 1021, 1004, 1010, 1033, 1036, 1026, 1004, 1001, 1033, 1014, 1032, 1036, 970, 1068, 967, 1030, 1034, 1024, 993, 1044, 1042, 1037, 1048, 1026, 1008, 1066, 992, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1016, 1016, 1021, 1009, 993, 996, 1040, 1026, 1041, 1102, 991, 974, 1033, 1012, 953, 1043, 1037, 1015, 959, 988, 1076, 1012, 1031, 1026, 1028, 1011, 1028, 1046, 1026, 1006, 1022, 988, 1038, 1056, 1018, 1017, 1053, 1033, 1071, 996, 1042, 1042, 1025, 1000, 1014, 1014, 994, 1046, 1050, 1035, 1008, 1088, 1079, 1038, 973, 974, 1035, 1053, 1040, 1090, 1013, 1030, 993, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1010, 992, 978, 984, 1003, 1033, 1060, 1051, 1065, 995, 982, 1043, 1043, 1053, 1025, 1017, 1044, 1023, 995, 1015, 1004, 1085, 1042, 1023, 1040, 1019, 1045, 994, 1019, 1007, 1037, 1020, 1045, 1039, 1015, 1005, 994, 1006, 1017, 1066, 1018, 1073, 1022, 993, 1030, 1011, 1043, 1006, 1025, 1036, 1035, 1016, 984, 1055, 1031, 1012, 1026, 1058, 1011, 1026, 1033, 1037, 1003, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1024, 1034, 1047, 1007, 1040, 997, 1024, 1038, 1006, 1039, 1065, 1015, 976, 1054, 1043, 1035, 1007, 1053, 1008, 948, 1005, 1018, 1045, 1018, 1025, 1036, 992, 987, 989, 1018, 1014, 1057, 1078, 1016, 1035, 1045, 1015, 1047, 1064, 1000, 1026, 1013, 1061, 984, 993, 1053, 1031, 1066, 1028, 1005, 1022, 990, 1016, 1002, 1022, 1023, 1020, 1037, 1011, 1004, 1050, 1029, 1041, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1080, 1027, 1021, 1006, 1013, 1071, 1033, 971, 1026, 1010, 998, 1019, 996, 1050, 1029, 1026, 999, 1080, 1040, 1035, 1035, 1027, 1042, 1043, 1012, 1007, 1021, 1006, 1029, 1049, 1029, 1014, 1039, 1018, 1035, 1021, 989, 1018, 1003, 1004, 1036, 1048, 1023, 1000, 1020, 1011, 1033, 1022, 1049, 1011, 995, 974, 1007, 1055, 1077, 1013, 1015, 1064, 1072, 1022, 1012, 1018, 991, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1008, 1074, 1017, 1022, 1009, 1027, 1027, 992, 6, 0, 0, 0, 0, 0, 0, 0, 1006, 984, 1016, 1004, 1040, 1033, 1056, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1021, 991, 990, 1028, 1032, 959, 1041, 25, 8, 0, 0, 0, 0, 0, 0, 968, 1034, 999, 1034, 983, 1021, 1031, 1013, 16, 13, 0, 0, 0, 0, 0, 0, 1015, 1062, 1018, 972, 996, 1055, 1052, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1076, 1091, 1087, 986, 1020, 1038, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 998, 961, 1052, 1071, 1034, 1018, 1077, 998, 4, 3, 0, 0, 0, 0, 0, 0, 987, 1008, 1061, 977, 1058, 1013, 1035, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8213, 8120, 8112, 8213, 8311, 8216, 8169] avg=8192.0000 max=8311 max_rank=5 min=8112 min_rank=3 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1065, 1023, 1032, 1061, 1026, 1022, 1001, 1040, 6, 0, 0, 0, 0, 0, 0, 0, 1031, 964, 1035, 1013, 1029, 1027, 1040, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1090, 1033, 994, 1033, 1002, 964, 1017, 24, 7, 0, 0, 0, 0, 0, 0, 1061, 1020, 953, 1032, 1066, 1013, 1008, 1032, 15, 13, 0, 0, 0, 0, 0, 0, 1028, 1000, 1045, 985, 1003, 1022, 1038, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1001, 1023, 1045, 1022, 1045, 1037, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 997, 1027, 1012, 992, 1040, 1010, 980, 3, 4, 0, 0, 0, 0, 0, 0, 1064, 949, 1020, 1065, 1013, 1025, 1019, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8138, 8219, 8213, 8152, 8285, 8077, 8176] avg=8192.0000 max=8285 max_rank=5 min=8077 min_rank=6 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[991, 1008, 1039, 1060, 1089, 972, 1004, 1030, 6, 0, 0, 0, 0, 0, 0, 0, 1020, 1043, 1084, 994, 1009, 1046, 1013, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 955, 1021, 1017, 1053, 1050, 975, 985, 26, 7, 0, 0, 0, 0, 0, 0, 1040, 967, 1003, 977, 1054, 1015, 1001, 1030, 15, 13, 0, 0, 0, 0, 0, 0, 1028, 1018, 1035, 1056, 1023, 1009, 1042, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 999, 992, 1020, 983, 1000, 1023, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1010, 1073, 1064, 1027, 1074, 1048, 1045, 4, 4, 0, 0, 0, 0, 0, 0, 1007, 1026, 1002, 1067, 1016, 1034, 1010, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8240, 8127, 8115, 8257, 8082, 8343, 8173] avg=8192.0000 max=8343 max_rank=6 min=8082 min_rank=5 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1021, 1009, 1053, 1026, 998, 1015, 1009, 1052, 6, 0, 0, 0, 0, 0, 0, 0, 1000, 993, 1075, 1058, 1067, 991, 1028, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 995, 995, 1017, 1045, 1017, 995, 1008, 1047, 26, 7, 0, 0, 0, 0, 0, 0, 1007, 988, 1106, 1034, 1037, 1046, 1040, 1067, 15, 13, 0, 0, 0, 0, 0, 0, 1000, 1004, 1021, 1002, 1021, 1004, 1010, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1004, 1004, 1001, 1033, 1014, 1032, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1068, 967, 1030, 1034, 1024, 993, 1044, 3, 3, 0, 0, 0, 0, 0, 0, 1042, 1011, 1048, 1026, 1008, 1066, 992, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8233, 8152, 8353, 8095, 8160, 8136, 8218] avg=8192.0000 max=8353 max_rank=3 min=8095 min_rank=4 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1016, 1016, 1021, 1009, 993, 996, 1040, 1026, 6, 0, 0, 0, 0, 0, 0, 0, 1041, 1102, 991, 974, 1033, 1003, 953, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1015, 959, 988, 1076, 1012, 1031, 1026, 26, 7, 0, 0, 0, 0, 0, 0, 1028, 1011, 1028, 1046, 1026, 1006, 1022, 988, 15, 13, 0, 0, 0, 0, 0, 0, 1022, 1056, 1018, 1017, 1053, 1033, 1071, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1020, 1025, 1000, 1014, 1014, 994, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1035, 1008, 1088, 1079, 1038, 973, 974, 3, 3, 0, 0, 0, 0, 0, 0, 1035, 1027, 1040, 1090, 1013, 1030, 993, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8140, 8177, 8183, 8266, 8155, 8251, 8241] avg=8192.0000 max=8266 max_rank=4 min=8123 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1010, 992, 978, 984, 1003, 1033, 1060, 1051, 11, 0, 0, 0, 0, 0, 0, 0, 1065, 995, 982, 1043, 1043, 1035, 1025, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1023, 995, 1015, 1004, 1085, 1042, 1023, 30, 11, 0, 0, 0, 0, 0, 0, 1040, 1019, 1045, 994, 1019, 1007, 1037, 1020, 18, 19, 0, 0, 0, 0, 0, 0, 1018, 1039, 1015, 1005, 994, 1006, 1017, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1044, 1022, 993, 1030, 1011, 1043, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1036, 1035, 1016, 984, 1055, 1031, 1012, 7, 8, 0, 0, 0, 0, 0, 0, 1026, 1028, 1011, 1026, 1033, 1037, 1003, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8205, 8272, 8218, 8160, 8167, 8209, 8183] avg=8192.0000 max=8272 max_rank=2 min=8122 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1024, 1034, 1047, 1007, 1040, 997, 1024, 1038, 6, 0, 0, 0, 0, 0, 0, 0, 1006, 1039, 1065, 1015, 976, 1044, 1043, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1053, 1008, 948, 1005, 1018, 1045, 1018, 26, 7, 0, 0, 0, 0, 0, 0, 1025, 1036, 992, 987, 989, 1018, 1014, 1057, 15, 14, 0, 0, 0, 0, 0, 0, 1060, 1016, 1035, 1045, 1015, 1047, 1064, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 991, 1061, 984, 993, 1053, 1031, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1005, 1022, 990, 1016, 1002, 1022, 1023, 4, 4, 0, 0, 0, 0, 0, 0, 1020, 1011, 1011, 1004, 1050, 1029, 1041, 1045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8223, 8135, 8147, 8282, 8205, 8116, 8211] avg=8192.0000 max=8282 max_rank=4 min=8116 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1080, 1027, 1021, 1006, 1013, 1071, 1033, 971, 6, 0, 0, 0, 0, 0, 0, 0, 1026, 1010, 998, 1019, 996, 1040, 1029, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1080, 1040, 1035, 1035, 1027, 1042, 1043, 26, 7, 0, 0, 0, 0, 0, 0, 1012, 1007, 1021, 1006, 1029, 1049, 1029, 1014, 15, 13, 0, 0, 0, 0, 0, 0, 1023, 1018, 1035, 1021, 989, 1018, 1003, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1026, 1023, 1000, 1020, 1011, 1033, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1011, 995, 974, 1007, 1055, 1077, 1013, 4, 3, 0, 0, 0, 0, 0, 0, 1015, 1038, 1072, 1022, 1012, 1018, 991, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8144, 8334, 8195, 8111, 8171, 8188, 8165] avg=8192.0000 max=8334 max_rank=2 min=8111 min_rank=4 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8064, 8064, 8320, 8320, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1042, 1012, 988, 1048, 1037, 1047, 1051, 981, 982, 1020, 1022, 1023, 1030, 1026, 1023, 1047, 1060, 1044, 1055, 1042, 998, 972, 1073, 1046, 964, 1067, 1025, 1037, 1052, 1013, 1037, 1004, 1037, 1097, 1038, 1017, 987, 1083, 1047, 1063, 1044, 986, 1008, 996, 1072, 987, 1029, 1022, 1053, 940, 1005, 1034, 1030, 1040, 1003, 1002, 1033, 987, 979, 1043, 998, 1023, 975, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1025, 1009, 1013, 1044, 1004, 1024, 1057, 1068, 1016, 1028, 984, 1034, 998, 1078, 1010, 1019, 995, 1049, 1045, 1045, 1033, 1032, 1031, 1035, 997, 1027, 1026, 1016, 1024, 1005, 1032, 1011, 1038, 1015, 967, 1034, 1013, 1015, 1009, 989, 1018, 1054, 1010, 1050, 1034, 1035, 989, 1017, 988, 1056, 1032, 1046, 1012, 999, 1010, 1037, 1060, 1018, 1056, 1005, 1019, 1027, 1032, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1018, 1054, 1071, 1061, 1024, 1063, 1006, 1037, 1012, 987, 1011, 1050, 1063, 1015, 1017, 1009, 1061, 1007, 1070, 1064, 993, 1085, 985, 1034, 982, 974, 1037, 1032, 1030, 1067, 1054, 1006, 1008, 981, 1034, 994, 986, 1063, 1063, 1023, 1050, 1040, 987, 1008, 1017, 1031, 1004, 1014, 1017, 1050, 1035, 979, 984, 1005, 993, 994, 1003, 1020, 1017, 990, 1104, 994, 1015, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1016, 1051, 986, 1033, 1005, 1031, 1054, 1023, 1030, 1074, 1026, 1010, 1011, 1029, 1049, 1026, 1021, 987, 998, 1050, 1034, 1011, 1027, 1061, 1065, 993, 1020, 988, 1025, 1028, 1059, 1012, 1029, 1008, 1036, 993, 1004, 1000, 1025, 1055, 1060, 1037, 977, 1047, 972, 1035, 1009, 1035, 1012, 1023, 1075, 984, 998, 1030, 1024, 1034, 1014, 1015, 1052, 980, 1057, 1020, 988, 1075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[979, 1007, 1047, 1023, 1027, 1006, 1084, 1060, 974, 1060, 999, 1053, 1075, 1063, 1008, 1030, 1032, 1009, 996, 1034, 1039, 1046, 1006, 1069, 1014, 1030, 1064, 1001, 996, 973, 1077, 995, 1026, 1027, 982, 979, 1035, 991, 1020, 1039, 994, 1028, 1066, 1042, 1060, 995, 1036, 984, 1012, 980, 980, 1020, 1035, 1054, 1036, 1001, 1031, 1061, 1004, 1013, 999, 1102, 1019, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[973, 1056, 1016, 999, 1025, 1008, 1080, 1050, 1048, 954, 1052, 951, 1033, 1019, 987, 1017, 979, 1041, 1027, 1047, 1017, 1020, 981, 1039, 1107, 1002, 1027, 1040, 987, 1043, 1014, 994, 1021, 1009, 964, 982, 1010, 1067, 1051, 966, 1016, 1037, 1015, 1074, 1045, 1087, 1052, 1020, 1001, 1054, 1074, 984, 1085, 1031, 1036, 1022, 1029, 1030, 987, 1009, 1057, 1028, 1058, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1001, 1016, 1016, 1038, 975, 1000, 1014, 1014, 975, 976, 1030, 1016, 1020, 997, 1029, 1015, 1073, 1068, 1015, 1057, 1015, 1035, 1038, 1032, 1043, 1004, 1018, 1071, 1060, 1016, 1033, 1057, 1050, 1037, 1001, 1058, 1040, 1045, 1019, 999, 987, 1008, 1000, 1037, 1013, 1039, 1019, 1004, 1062, 1017, 1033, 1022, 1035, 1024, 1039, 987, 986, 1030, 1045, 1004, 1037, 1038, 1021, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1079, 1064, 991, 992, 1036, 1035, 1032, 1023, 1010, 1024, 1022, 1022, 970, 1079, 1060, 1051, 1008, 1047, 1007, 1046, 1070, 1013, 1013, 1021, 1028, 1030, 1010, 989, 1040, 1004, 984, 1023, 990, 1031, 1054, 1062, 1013, 1015, 1005, 1011, 1018, 1017, 988, 985, 1025, 1050, 1056, 1002, 1002, 1015, 1016, 1039, 992, 978, 1048, 996, 1031, 1032, 1028, 1030, 1056, 1035, 1010, 1083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1042, 1012, 988, 1048, 1037, 1047, 1026, 981, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1020, 1022, 1023, 1030, 1026, 1023, 1047, 19, 0, 0, 0, 0, 0, 0, 0, 1060, 1044, 1055, 987, 998, 972, 1073, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 964, 1067, 1025, 1037, 1052, 1013, 1037, 1004, 6, 0, 0, 0, 0, 0, 0, 0, 1037, 1097, 1038, 1017, 987, 1083, 1047, 1063, 18, 5, 0, 0, 0, 0, 0, 0, 1044, 986, 1008, 996, 1072, 987, 1029, 1022, 5, 1, 0, 0, 0, 0, 0, 0, 1053, 940, 1005, 1034, 1030, 1040, 1003, 1002, 37, 0, 0, 0, 0, 0, 0, 0, 1033, 987, 979, 1043, 987, 1023, 975, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8192, 8235, 8205, 8392, 8150, 8144, 8037] avg=8192.0000 max=8392 max_rank=4 min=8037 min_rank=7 imbalance_ratio=1.024414 max_load_reduction=57144 max_load_reduction_pct=0.871948 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 1009, 1013, 1044, 1004, 1024, 1032, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1028, 984, 1034, 998, 1078, 1010, 1019, 19, 0, 0, 0, 0, 0, 0, 0, 995, 1049, 1045, 990, 1033, 1032, 1031, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1027, 1026, 1016, 1024, 1005, 1032, 1011, 6, 0, 0, 0, 0, 0, 0, 0, 1038, 1015, 967, 1034, 1013, 1015, 1009, 989, 18, 5, 0, 0, 0, 0, 0, 0, 1018, 1054, 1010, 1050, 1034, 1035, 989, 1017, 5, 1, 0, 0, 0, 0, 0, 0, 988, 1056, 1032, 1046, 1012, 999, 1010, 1037, 37, 0, 0, 0, 0, 0, 0, 0, 1060, 1018, 1056, 1005, 1008, 1027, 1032, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8186, 8210, 8144, 8103, 8213, 8217, 8244] avg=8192.0000 max=8244 max_rank=7 min=8103 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1018, 1054, 1071, 1061, 1024, 1063, 983, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 987, 1011, 1050, 1063, 1015, 1017, 1009, 18, 0, 0, 0, 0, 0, 0, 0, 1061, 1007, 1070, 1008, 993, 1085, 985, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 982, 974, 1037, 1032, 1030, 1067, 1054, 1006, 7, 0, 0, 0, 0, 0, 0, 0, 1008, 981, 1034, 994, 986, 1063, 1063, 1023, 18, 4, 0, 0, 0, 0, 0, 0, 1050, 1040, 987, 1008, 1017, 1031, 1004, 1014, 6, 1, 0, 0, 0, 0, 0, 0, 1017, 1050, 1035, 979, 984, 1005, 993, 994, 38, 0, 0, 0, 0, 0, 0, 0, 1003, 1020, 1017, 990, 1091, 994, 1015, 1054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8182, 8243, 8189, 8174, 8158, 8095, 8184] avg=8192.0000 max=8311 max_rank=0 min=8095 min_rank=6 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32611 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1016, 1051, 986, 1033, 1005, 1031, 1029, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1074, 1026, 1010, 1011, 1029, 1049, 1026, 19, 0, 0, 0, 0, 0, 0, 0, 1021, 987, 998, 995, 1034, 1011, 1027, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 993, 1020, 988, 1025, 1028, 1059, 1012, 6, 0, 0, 0, 0, 0, 0, 0, 1029, 1008, 1036, 993, 1004, 1000, 1025, 1055, 18, 5, 0, 0, 0, 0, 0, 0, 1060, 1037, 977, 1047, 972, 1035, 1009, 1035, 6, 1, 0, 0, 0, 0, 0, 0, 1012, 1023, 1075, 984, 998, 1030, 1024, 1034, 37, 0, 0, 0, 0, 0, 0, 0, 1014, 1015, 1052, 980, 1045, 1020, 988, 1075, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8274, 8134, 8196, 8173, 8179, 8217, 8189] avg=8192.0000 max=8274 max_rank=1 min=8134 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[973, 1056, 1016, 999, 1025, 1008, 1054, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 954, 1052, 951, 1033, 1019, 987, 1017, 20, 0, 0, 0, 0, 0, 0, 0, 979, 1041, 1027, 992, 1017, 1020, 981, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 1002, 1027, 1040, 987, 1043, 1014, 994, 6, 0, 0, 0, 0, 0, 0, 0, 1021, 1009, 964, 982, 1010, 1067, 1051, 966, 18, 5, 0, 0, 0, 0, 0, 0, 1016, 1037, 1015, 1074, 1045, 1087, 1052, 1020, 6, 1, 0, 0, 0, 0, 0, 0, 1001, 1054, 1074, 984, 1085, 1031, 1036, 1022, 37, 0, 0, 0, 0, 0, 0, 0, 1029, 1030, 987, 1009, 1045, 1028, 1058, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8081, 8096, 8220, 8093, 8353, 8324, 8188] avg=8192.0000 max=8353 max_rank=5 min=8081 min_rank=1 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32958 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[979, 1007, 1047, 1023, 1027, 1006, 1058, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1060, 999, 1053, 1075, 1063, 1008, 1030, 20, 0, 0, 0, 0, 0, 0, 0, 1032, 1009, 996, 980, 1039, 1046, 1006, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1030, 1064, 1001, 996, 973, 1077, 995, 6, 0, 0, 0, 0, 0, 0, 0, 1026, 1027, 982, 979, 1035, 991, 1020, 1039, 17, 5, 0, 0, 0, 0, 0, 0, 994, 1028, 1066, 1042, 1060, 995, 1036, 984, 5, 1, 0, 0, 0, 0, 0, 0, 1012, 980, 980, 1020, 1035, 1054, 1036, 1001, 37, 0, 0, 0, 0, 0, 0, 0, 1031, 1061, 1004, 1013, 988, 1102, 1019, 1009, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8282, 8177, 8156, 8121, 8211, 8155, 8227] avg=8192.0000 max=8282 max_rank=1 min=8121 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1001, 1016, 1016, 1038, 975, 1000, 990, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 975, 976, 1030, 1016, 1020, 997, 1029, 1015, 19, 0, 0, 0, 0, 0, 0, 0, 1073, 1068, 1015, 1001, 1015, 1035, 1038, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1004, 1018, 1071, 1060, 1016, 1033, 1057, 6, 0, 0, 0, 0, 0, 0, 0, 1050, 1037, 1001, 1058, 1040, 1045, 1019, 999, 18, 4, 0, 0, 0, 0, 0, 0, 987, 1008, 1000, 1037, 1013, 1039, 1019, 1004, 6, 1, 0, 0, 0, 0, 0, 0, 1062, 1017, 1033, 1022, 1035, 1024, 1039, 987, 38, 0, 0, 0, 0, 0, 0, 0, 986, 1030, 1045, 1004, 1025, 1038, 1021, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8050, 8077, 8277, 8308, 8271, 8114, 8257, 8182] avg=8192.0000 max=8308 max_rank=3 min=8050 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1079, 1064, 991, 992, 1036, 1035, 993, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1024, 1022, 1022, 970, 1079, 1060, 1051, 24, 0, 0, 0, 0, 0, 0, 0, 1008, 1047, 1007, 985, 1070, 1013, 1013, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1030, 1010, 989, 1040, 1004, 984, 1023, 10, 0, 0, 0, 0, 0, 0, 0, 990, 1031, 1054, 1062, 1013, 1015, 1005, 1011, 20, 8, 0, 0, 0, 0, 0, 0, 1018, 1017, 988, 985, 1025, 1050, 1056, 1002, 10, 7, 0, 0, 0, 0, 0, 0, 1002, 1015, 1016, 1039, 992, 978, 1048, 996, 41, 0, 0, 0, 0, 0, 0, 0, 1031, 1032, 1028, 1030, 1036, 1035, 1010, 1083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8262, 8164, 8118, 8209, 8158, 8127, 8285] avg=8192.0000 max=8285 max_rank=7 min=8118 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8064, 8320, 8320, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8064, 8320, 8320, 8192, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[976, 1061, 1048, 1007, 1002, 1003, 1009, 1031, 984, 1020, 983, 1003, 1038, 1047, 1031, 1071, 963, 1007, 1042, 1055, 1015, 1043, 1016, 1006, 985, 993, 983, 1009, 1051, 1032, 1017, 1016, 1001, 1023, 981, 1023, 1097, 1061, 1025, 1065, 1047, 1069, 1040, 992, 1034, 1059, 994, 1044, 984, 1037, 1041, 1040, 1065, 1019, 1018, 1030, 1033, 1004, 1005, 1068, 989, 1057, 1046, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1023, 994, 1014, 1042, 1067, 1001, 1002, 1044, 1054, 1029, 1046, 1041, 1010, 1035, 953, 1012, 1035, 1057, 1038, 1019, 1025, 1001, 1015, 984, 1044, 990, 996, 1045, 1016, 1022, 1028, 1061, 990, 1010, 1031, 1015, 1058, 1068, 1031, 999, 1034, 1040, 1044, 995, 1036, 1049, 1018, 1037, 1028, 1038, 989, 1003, 1049, 995, 999, 1045, 1019, 1007, 998, 1007, 1022, 1072, 1010, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[991, 1029, 1077, 1056, 988, 1010, 1017, 1024, 1053, 1009, 994, 1009, 1005, 1026, 1018, 1022, 1060, 1007, 1035, 999, 1001, 1067, 1028, 1013, 1056, 1043, 1011, 1016, 1007, 1007, 1017, 1023, 1014, 1068, 1042, 1027, 1041, 1062, 1028, 1051, 982, 1028, 1028, 995, 1011, 1041, 978, 1018, 1041, 1015, 1051, 1013, 1018, 1041, 1014, 1050, 1034, 981, 1021, 1055, 989, 1053, 1022, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1033, 1049, 1036, 1045, 1016, 1042, 1034, 1042, 1032, 1025, 991, 1000, 1050, 1007, 1027, 1032, 1010, 986, 972, 1039, 1062, 1023, 1020, 1054, 1025, 1051, 1022, 992, 1039, 1035, 1005, 1025, 1049, 1001, 1005, 1023, 1031, 1001, 995, 991, 1032, 1022, 1053, 1007, 1033, 1057, 1018, 1013, 1048, 1083, 1074, 1015, 983, 1026, 980, 1015, 1034, 1048, 991, 1004, 1030, 1042, 976, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1001, 1022, 1014, 1016, 1004, 1046, 1019, 1017, 1000, 968, 1028, 1032, 1029, 1007, 992, 1057, 1043, 1033, 1033, 1062, 1061, 980, 1005, 980, 1007, 1029, 1006, 1059, 1005, 974, 1049, 1022, 996, 988, 1069, 1048, 1032, 1016, 1056, 1021, 1067, 1063, 1070, 975, 1042, 1073, 1021, 985, 1071, 1012, 986, 999, 1011, 989, 1119, 1025, 1017, 1001, 1050, 1004, 1034, 1032, 1031, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[999, 1021, 975, 1012, 1001, 1036, 1000, 1048, 953, 1064, 1034, 1051, 995, 1065, 1018, 1007, 1023, 1001, 1045, 1051, 1005, 1018, 1048, 1043, 1060, 1001, 1032, 995, 1030, 1046, 974, 1043, 1038, 970, 1033, 1033, 1043, 1013, 1055, 1013, 1016, 1003, 1051, 1024, 1046, 1014, 1012, 1048, 1012, 1038, 1008, 1052, 1006, 1033, 1043, 1019, 1040, 1003, 1047, 1048, 997, 1034, 1005, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1002, 1034, 999, 1016, 1005, 964, 1076, 1036, 1010, 997, 1012, 1075, 988, 1020, 977, 990, 1044, 1010, 1014, 1046, 1022, 990, 1056, 1044, 1017, 1107, 1033, 1053, 1068, 1018, 992, 1027, 1069, 990, 1037, 1062, 1022, 977, 1016, 1053, 1027, 1072, 1026, 1021, 1022, 990, 1074, 1063, 1095, 1034, 1035, 1001, 1020, 1001, 1050, 1022, 970, 987, 978, 1001, 1023, 1005, 1037, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1002, 981, 1045, 996, 985, 1028, 1005, 956, 1006, 1045, 1029, 1008, 990, 985, 1051, 1047, 1037, 972, 1007, 989, 1066, 1036, 1019, 1019, 1018, 1043, 1015, 1031, 1048, 1030, 1068, 986, 1040, 983, 1009, 1050, 1038, 1030, 1014, 1078, 1022, 1004, 1005, 1004, 1042, 1030, 999, 1038, 1016, 1026, 1048, 1010, 1067, 1006, 982, 1052, 1042, 1037, 1000, 1102, 1075, 978, 1074, 1092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[976, 1061, 1048, 1007, 1002, 1003, 1009, 1031, 34, 12, 0, 0, 0, 0, 0, 0, 984, 1020, 983, 1003, 1038, 1047, 1031, 1071, 22, 20, 0, 0, 0, 0, 0, 0, 963, 1007, 1042, 1055, 1015, 1043, 1016, 1006, 1, 1, 1, 0, 0, 0, 0, 0, 985, 993, 983, 1009, 1050, 1032, 1017, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1023, 981, 1023, 1063, 1061, 1025, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1069, 1008, 992, 1034, 1059, 994, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 961, 1037, 1041, 1040, 1065, 1019, 1018, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1004, 1005, 1067, 989, 1057, 1046, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8219, 8150, 8085, 8242, 8247, 8211, 8199] avg=8192.0000 max=8247 max_rank=5 min=8085 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1033, 1049, 1036, 1045, 1016, 1042, 1034, 1042, 32, 12, 0, 0, 0, 0, 0, 0, 1032, 1025, 991, 1000, 1050, 1007, 1027, 1032, 23, 20, 0, 0, 0, 0, 0, 0, 1010, 986, 972, 1039, 1062, 1023, 1020, 1054, 1, 1, 1, 0, 0, 0, 0, 0, 1025, 1051, 1022, 992, 1038, 1035, 1005, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1001, 1005, 1023, 999, 1001, 995, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1022, 1021, 1007, 1033, 1057, 1018, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1083, 1074, 1015, 983, 1026, 980, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1048, 991, 1003, 1030, 1042, 976, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8341, 8207, 8169, 8193, 8064, 8203, 8200, 8159] avg=8192.0000 max=8341 max_rank=0 min=8064 min_rank=4 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1001, 1022, 1014, 1016, 1004, 1046, 1019, 1017, 32, 12, 0, 0, 0, 0, 0, 0, 1000, 968, 1028, 1032, 1029, 1007, 992, 1057, 24, 20, 0, 0, 0, 0, 0, 0, 1043, 1033, 1033, 1062, 1061, 980, 1005, 980, 1, 1, 1, 0, 0, 0, 0, 0, 1007, 1029, 1006, 1059, 1004, 974, 1049, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 996, 988, 1069, 1048, 1000, 1016, 1056, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1063, 1038, 975, 1042, 1073, 1021, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1012, 986, 999, 1011, 989, 1119, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1001, 1050, 1003, 1034, 1032, 1031, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8157, 8200, 8150, 8194, 8264, 8187, 8201] avg=8192.0000 max=8264 max_rank=5 min=8150 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1023, 994, 1014, 1042, 1067, 1001, 1002, 1044, 33, 12, 0, 0, 0, 0, 0, 0, 1054, 1029, 1046, 1041, 1010, 1035, 953, 1012, 23, 20, 0, 0, 0, 0, 0, 0, 1035, 1057, 1038, 1019, 1025, 1001, 1015, 984, 1, 1, 1, 0, 0, 0, 0, 0, 1044, 990, 996, 1045, 1015, 1022, 1028, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1010, 1031, 1015, 1025, 1068, 1031, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1040, 1012, 995, 1036, 1049, 1018, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1038, 989, 1003, 1049, 995, 999, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1007, 998, 1006, 1022, 1072, 1010, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8223, 8177, 8201, 8169, 8221, 8122, 8191] avg=8192.0000 max=8232 max_rank=0 min=8122 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[991, 1029, 1077, 1056, 988, 1010, 1017, 1024, 32, 12, 0, 0, 0, 0, 0, 0, 1053, 1009, 994, 1009, 1005, 1026, 1018, 1022, 23, 20, 0, 0, 0, 0, 0, 0, 1060, 1007, 1035, 999, 1001, 1067, 1028, 1013, 1, 1, 1, 0, 0, 0, 0, 0, 1056, 1043, 1011, 1016, 1006, 1007, 1017, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1068, 1042, 1027, 1009, 1062, 1028, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1028, 996, 995, 1011, 1041, 978, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1015, 1051, 1013, 1018, 1041, 1014, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 981, 1021, 1054, 989, 1053, 1022, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8179, 8213, 8179, 8301, 8049, 8219, 8160] avg=8192.0000 max=8301 max_rank=4 min=8049 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[999, 1021, 975, 1012, 1001, 1036, 1000, 1048, 32, 12, 0, 0, 0, 0, 0, 0, 953, 1064, 1034, 1051, 995, 1065, 1018, 1007, 22, 20, 0, 0, 0, 0, 0, 0, 1023, 1001, 1045, 1051, 1005, 1018, 1048, 1043, 1, 1, 1, 0, 0, 0, 0, 0, 1060, 1001, 1032, 995, 1029, 1046, 974, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 970, 1033, 1033, 1011, 1013, 1055, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1003, 1019, 1024, 1046, 1014, 1012, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1038, 1008, 1052, 1006, 1033, 1043, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1003, 1047, 1047, 997, 1034, 1005, 1045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8229, 8237, 8180, 8166, 8182, 8188, 8218] avg=8192.0000 max=8237 max_rank=2 min=8136 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1002, 1034, 999, 1016, 1005, 964, 1076, 1036, 36, 16, 0, 0, 0, 0, 0, 0, 1010, 997, 1012, 1075, 988, 1020, 977, 990, 28, 23, 0, 0, 0, 0, 0, 0, 1044, 1010, 1014, 1046, 1022, 990, 1056, 1044, 6, 5, 5, 0, 0, 0, 0, 0, 1017, 1107, 1033, 1053, 1063, 1018, 992, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 990, 1037, 1062, 986, 977, 1016, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1072, 987, 1021, 1022, 990, 1074, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1034, 1035, 1001, 1020, 1001, 1050, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 970, 987, 978, 995, 1023, 1005, 1037, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8120, 8242, 8310, 8190, 8256, 8225, 8009] avg=8192.0000 max=8310 max_rank=3 min=8009 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1002, 981, 1045, 996, 985, 1028, 1005, 956, 32, 11, 0, 0, 0, 0, 0, 0, 1006, 1045, 1029, 1008, 990, 985, 1051, 1047, 22, 19, 0, 0, 0, 0, 0, 0, 1037, 972, 1007, 989, 1066, 1036, 1019, 1019, 1, 1, 1, 0, 0, 0, 0, 0, 1018, 1043, 1015, 1031, 1047, 1030, 1068, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 983, 1009, 1050, 1006, 1030, 1014, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1004, 975, 1004, 1042, 1030, 999, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1026, 1048, 1010, 1067, 1006, 982, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1037, 1000, 1101, 1075, 978, 1074, 1092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8041, 8202, 8148, 8238, 8210, 8114, 8184, 8399] avg=8192.0000 max=8399 max_rank=7 min=8041 min_rank=0 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8064, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[998, 955, 996, 964, 1078, 1019, 1022, 1001, 1085, 1093, 1043, 1022, 1070, 1046, 1042, 1013, 998, 1076, 1063, 1030, 992, 1016, 1062, 1024, 1026, 1008, 1035, 1025, 1008, 1076, 1019, 1083, 1009, 983, 1058, 1034, 1012, 1004, 1022, 1001, 989, 1040, 1017, 1008, 999, 1001, 986, 1097, 995, 1011, 1052, 1004, 1038, 1046, 1047, 999, 1028, 1026, 987, 1041, 994, 1004, 1010, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1012, 1002, 992, 1017, 1017, 1005, 1035, 1017, 1011, 1018, 1058, 967, 1002, 1014, 1012, 1033, 1006, 1037, 981, 1083, 1056, 1043, 1038, 978, 966, 989, 1045, 963, 1065, 1022, 1041, 1034, 998, 1049, 1006, 1072, 1062, 1036, 1052, 1100, 1013, 969, 988, 982, 1021, 1011, 1032, 1062, 1047, 1007, 1070, 1015, 1030, 1048, 1012, 1026, 1011, 1025, 1017, 1078, 1012, 1062, 995, 1069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[993, 1034, 1000, 990, 1063, 1039, 992, 994, 1039, 1009, 991, 1030, 1039, 998, 1043, 1052, 1049, 1057, 1050, 1059, 1001, 1022, 1008, 1033, 1033, 993, 1035, 1081, 1019, 978, 975, 1113, 1012, 1023, 1043, 1025, 1000, 999, 1051, 1073, 1013, 999, 980, 1021, 1014, 1026, 1055, 1046, 996, 1019, 989, 999, 1074, 1004, 1046, 1044, 1029, 998, 1020, 1002, 1010, 1027, 1061, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1011, 1050, 1037, 1031, 1041, 991, 1022, 1040, 1004, 1049, 1066, 1019, 1012, 1014, 967, 1024, 1033, 1024, 1058, 995, 1054, 1037, 1043, 1002, 1011, 966, 1047, 1060, 1049, 1018, 1013, 1063, 1015, 1034, 1023, 1011, 1023, 1051, 983, 993, 1023, 1045, 1003, 990, 1015, 1019, 1039, 1015, 1024, 978, 1020, 1005, 1048, 1034, 983, 1035, 1032, 1056, 998, 1012, 1036, 1012, 1058, 1072] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1020, 1040, 1065, 1011, 1034, 1023, 1029, 991, 1053, 1067, 1060, 1009, 1038, 1024, 1010, 1007, 1025, 964, 1088, 1025, 1001, 1080, 1056, 1043, 1033, 1033, 1037, 1012, 978, 1042, 1014, 1002, 1059, 993, 998, 982, 1009, 1012, 977, 1056, 1017, 1013, 1011, 1010, 1048, 1033, 1058, 1024, 1049, 1028, 1043, 1027, 1012, 1049, 961, 1039, 1014, 964, 1004, 1020, 1042, 1024, 1050, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1048, 1037, 1046, 1076, 1000, 1041, 1015, 1016, 1015, 1051, 1034, 1032, 981, 1000, 998, 1029, 992, 1010, 1035, 1074, 1034, 1023, 1002, 1126, 997, 1012, 1034, 1080, 1032, 1025, 1006, 1033, 998, 1055, 1070, 1008, 1021, 1061, 989, 1038, 1072, 974, 971, 946, 983, 1060, 1012, 1015, 1031, 1049, 988, 990, 982, 981, 1023, 1004, 1003, 1083, 1013, 991, 1028, 1035, 1084, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1007, 1033, 1076, 997, 973, 1013, 993, 1048, 993, 1023, 1018, 1038, 1064, 978, 1033, 1000, 1055, 998, 1048, 1004, 1036, 976, 1053, 1060, 1013, 1019, 1016, 1033, 985, 971, 1011, 1067, 1034, 979, 1025, 1036, 1007, 994, 1012, 1039, 1039, 1029, 1019, 1089, 972, 1047, 998, 987, 1058, 1020, 1045, 997, 1057, 1004, 1094, 1024, 1041, 1056, 1044, 1029, 1053, 1050, 989, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1000, 1059, 1036, 1082, 1040, 1081, 1008, 1062, 1007, 1000, 1046, 1006, 1010, 1061, 1009, 996, 1028, 1020, 991, 1055, 1000, 980, 1037, 978, 1030, 1023, 1094, 1075, 989, 1061, 1040, 962, 1033, 985, 1044, 1028, 989, 996, 1020, 1053, 1054, 983, 1009, 984, 1024, 1024, 1066, 1055, 1113, 1045, 1028, 1034, 990, 1014, 1003, 1020, 985, 1027, 1028, 998, 1034, 988, 1015, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[998, 955, 996, 964, 1078, 1019, 1022, 1001, 4, 3, 1, 0, 0, 0, 0, 0, 1085, 1093, 1039, 1022, 1070, 1046, 1042, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1076, 1063, 985, 992, 1016, 1062, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1008, 1035, 1025, 1008, 1076, 1019, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 983, 1058, 1034, 1012, 1004, 1022, 1001, 7, 2, 0, 0, 0, 0, 0, 0, 989, 1040, 1017, 1008, 999, 1001, 986, 1097, 45, 2, 0, 0, 0, 0, 0, 0, 994, 1011, 1052, 1004, 1038, 1046, 1047, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1026, 987, 1041, 994, 1004, 1005, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8041, 8410, 8216, 8271, 8132, 8184, 8191, 8091] avg=8192.0000 max=8410 max_rank=1 min=8041 min_rank=0 imbalance_ratio=1.026611 max_load_reduction=57126 max_load_reduction_pct=0.871674 moved_to_echo=32598 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[993, 1034, 1000, 990, 1063, 1039, 992, 994, 4, 3, 1, 0, 0, 0, 0, 0, 1039, 1009, 987, 1030, 1039, 998, 1043, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1057, 1050, 1013, 1001, 1022, 1008, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 993, 1035, 1081, 1019, 978, 975, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1023, 1043, 1025, 1000, 999, 1051, 1073, 7, 2, 0, 0, 0, 0, 0, 0, 1013, 999, 980, 1021, 1014, 1026, 1055, 1046, 46, 3, 0, 0, 0, 0, 0, 0, 995, 1019, 989, 999, 1074, 1004, 1046, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 998, 1020, 1002, 1010, 1027, 1056, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8197, 8233, 8217, 8235, 8203, 8170, 8168] avg=8192.0000 max=8235 max_rank=4 min=8113 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1012, 1002, 992, 1017, 1017, 1005, 1035, 1017, 4, 3, 1, 0, 0, 0, 0, 0, 1011, 1018, 1054, 967, 1002, 1014, 1012, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1037, 981, 1035, 1056, 1043, 1038, 978, 0, 0, 0, 0, 0, 0, 0, 0, 966, 989, 1045, 963, 1065, 1022, 1041, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1049, 1006, 1072, 1062, 1036, 1052, 1100, 7, 2, 0, 0, 0, 0, 0, 0, 1013, 969, 988, 982, 1021, 1011, 1032, 1062, 48, 2, 0, 0, 0, 0, 0, 0, 1046, 1007, 1070, 1015, 1030, 1048, 1012, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1025, 1017, 1078, 1012, 1062, 990, 1069, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8111, 8174, 8116, 8384, 8128, 8254, 8264] avg=8192.0000 max=8384 max_rank=4 min=8105 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=33030 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1007, 1033, 1076, 997, 973, 1013, 993, 1048, 4, 3, 1, 0, 0, 0, 0, 0, 993, 1023, 1014, 1038, 1064, 978, 1033, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 998, 1048, 960, 1036, 976, 1053, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1019, 1016, 1033, 985, 971, 1011, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 979, 1025, 1036, 1007, 994, 1012, 1039, 7, 2, 0, 0, 0, 0, 0, 0, 1039, 1029, 1019, 1089, 972, 1047, 998, 987, 44, 2, 0, 0, 0, 0, 0, 0, 1057, 1020, 1045, 997, 1057, 1004, 1094, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1056, 1044, 1029, 1053, 1050, 984, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8143, 8186, 8106, 8135, 8226, 8298, 8294] avg=8192.0000 max=8298 max_rank=6 min=8106 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32953 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1020, 1040, 1065, 1011, 1034, 1023, 1029, 991, 4, 3, 1, 0, 0, 0, 0, 0, 1053, 1067, 1056, 1009, 1038, 1024, 1010, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 964, 1088, 980, 1001, 1080, 1056, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1033, 1037, 1012, 978, 1042, 1014, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 993, 998, 982, 1009, 1012, 977, 1056, 7, 2, 0, 0, 0, 0, 0, 0, 1017, 1013, 1011, 1010, 1048, 1033, 1058, 1024, 45, 2, 0, 0, 0, 0, 0, 0, 1048, 1028, 1043, 1027, 1012, 1049, 961, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 964, 1004, 1020, 1042, 1024, 1045, 996, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8264, 8237, 8142, 8095, 8261, 8207, 8109] avg=8192.0000 max=8264 max_rank=1 min=8095 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1011, 1050, 1037, 1031, 1041, 991, 1022, 1040, 11, 7, 4, 0, 0, 0, 0, 0, 1004, 1049, 1055, 1019, 1012, 1014, 967, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1024, 1058, 947, 1054, 1037, 1043, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 966, 1047, 1060, 1049, 1018, 1013, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1034, 1023, 1011, 1023, 1051, 983, 993, 11, 6, 0, 0, 0, 0, 0, 0, 1023, 1045, 1003, 990, 1015, 1019, 1039, 1015, 48, 8, 0, 0, 0, 0, 0, 0, 1020, 978, 1020, 1005, 1048, 1034, 983, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1056, 998, 1012, 1036, 1012, 1045, 1072, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8144, 8198, 8208, 8150, 8205, 8123, 8263] avg=8192.0000 max=8263 max_rank=7 min=8123 min_rank=6 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1000, 1059, 1036, 1082, 1040, 1081, 1008, 1062, 4, 3, 1, 0, 0, 0, 0, 0, 1007, 1000, 1042, 1006, 1010, 1061, 1009, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1020, 991, 1009, 1000, 980, 1037, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1023, 1094, 1075, 989, 1061, 1040, 954, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 985, 1044, 1028, 989, 996, 1020, 1053, 6, 2, 0, 0, 0, 0, 0, 0, 1054, 983, 1009, 984, 1024, 1024, 1066, 1055, 46, 2, 0, 0, 0, 0, 0, 0, 1112, 1045, 1028, 1034, 990, 1014, 1003, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1027, 1028, 998, 1034, 988, 1010, 1001, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8376, 8131, 8043, 8266, 8156, 8247, 8246, 8071] avg=8192.0000 max=8376 max_rank=0 min=8043 min_rank=2 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1048, 1037, 1046, 1076, 1000, 1041, 1015, 1016, 4, 3, 1, 0, 0, 0, 0, 0, 1015, 1051, 1030, 1032, 981, 1000, 998, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1010, 1035, 1027, 1034, 1023, 1002, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1012, 1034, 1080, 1032, 1025, 1006, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1055, 1070, 1008, 1021, 1061, 989, 1038, 7, 2, 0, 0, 0, 0, 0, 0, 1072, 974, 971, 946, 983, 1060, 1012, 1015, 47, 2, 0, 0, 0, 0, 0, 0, 1030, 1049, 988, 990, 982, 981, 1023, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1083, 1013, 991, 1028, 1035, 1079, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8136, 8249, 8210, 8249, 8082, 8047, 8276] avg=8192.0000 max=8287 max_rank=0 min=8047 min_rank=6 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32654 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8064, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8064, 8064, 8192, 8320, 8320, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8448, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8448, 8448, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1098, 1041, 1059, 1047, 1024, 1006, 984, 1007, 1026, 971, 1015, 1040, 1018, 1014, 1070, 1046, 1043, 1054, 1037, 1055, 1077, 1050, 1080, 980, 987, 997, 1047, 1034, 1039, 1030, 994, 983, 1021, 1061, 1000, 1046, 1029, 996, 1027, 1089, 986, 1019, 1020, 1001, 990, 1083, 977, 1007, 1036, 1050, 984, 1032, 1001, 1007, 1048, 1034, 1016, 1021, 969, 1017, 991, 1004, 1034, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1006, 1036, 1012, 1091, 1062, 1042, 1023, 963, 1033, 1026, 1010, 1023, 1023, 1036, 1026, 1034, 984, 1046, 1026, 1014, 1029, 1047, 1055, 1015, 1069, 1017, 965, 1036, 1048, 997, 1028, 1036, 977, 998, 1005, 995, 1051, 1021, 1000, 1020, 987, 1060, 1056, 1003, 1008, 1033, 1014, 1068, 1073, 1006, 1033, 1040, 1032, 993, 1022, 1068, 969, 1008, 993, 1030, 991, 1054, 1057, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[986, 1021, 1004, 1032, 1057, 1012, 1040, 1008, 970, 1082, 1058, 998, 1045, 1017, 1020, 1033, 1048, 1026, 1046, 991, 1034, 996, 1002, 1013, 1052, 1012, 1084, 1039, 1019, 1035, 1068, 1022, 1031, 990, 1019, 1046, 1053, 1043, 1020, 990, 994, 1020, 1017, 1000, 985, 1047, 972, 1014, 1021, 1020, 1032, 988, 1027, 1033, 1007, 1042, 1029, 1052, 1020, 999, 1084, 1028, 1035, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[978, 1031, 980, 1016, 992, 1051, 1034, 1048, 1042, 1006, 1007, 1011, 1025, 1038, 1043, 1023, 972, 1020, 1007, 1024, 1057, 1020, 1011, 1071, 1031, 1093, 1065, 1005, 994, 1030, 1045, 1055, 1011, 999, 1004, 1055, 1056, 1053, 1050, 1021, 1043, 998, 1049, 1050, 1004, 971, 1016, 1025, 981, 1023, 1040, 1044, 1040, 1027, 986, 1047, 1023, 1048, 1015, 1029, 1027, 1010, 983, 983] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[992, 983, 1057, 986, 994, 995, 1062, 1042, 1036, 1008, 1060, 1028, 1064, 984, 1018, 999, 1003, 1036, 1013, 1057, 1047, 1037, 1014, 1042, 1051, 1018, 1061, 1012, 1012, 1013, 1036, 1035, 990, 1021, 1040, 1029, 1058, 987, 1040, 1033, 1042, 1019, 1039, 1016, 1041, 958, 1021, 1057, 981, 1029, 1034, 976, 1045, 1026, 993, 1027, 1028, 1051, 1025, 1018, 991, 1055, 1018, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[990, 1036, 998, 1066, 1005, 994, 1038, 1024, 1053, 1020, 1039, 1099, 1025, 1044, 1042, 1008, 992, 1003, 1011, 1001, 1054, 1063, 1027, 1019, 1001, 1020, 1014, 1048, 980, 1050, 1001, 1028, 1038, 1029, 1031, 1043, 1005, 994, 1014, 1034, 1071, 1037, 962, 1008, 993, 966, 1019, 1014, 1053, 997, 1048, 1010, 992, 1030, 1052, 1017, 1049, 1039, 1041, 1068, 1045, 1026, 1023, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1056, 1033, 1025, 1021, 1007, 995, 999, 962, 1062, 1038, 996, 1034, 1024, 1064, 1033, 1064, 1022, 1072, 1016, 1042, 1004, 1001, 990, 1048, 1017, 1013, 1019, 982, 1031, 1038, 1068, 1039, 1016, 1010, 1046, 1067, 997, 974, 988, 947, 973, 1061, 1027, 1075, 995, 989, 1068, 1049, 1096, 1038, 1062, 1045, 1056, 1012, 1045, 1005, 1020, 1051, 1055, 944, 999, 1035, 967, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1054, 1008, 1034, 1014, 1046, 997, 1062, 1025, 993, 1006, 1011, 1008, 1070, 1033, 1055, 991, 1054, 1011, 994, 1048, 1040, 1008, 1005, 1012, 1011, 1007, 1012, 1025, 1042, 986, 1034, 1035, 1058, 1023, 997, 1068, 1073, 1014, 1003, 982, 1048, 965, 1023, 939, 975, 1008, 1057, 1038, 1056, 1013, 1050, 1039, 1021, 1053, 955, 1064, 1039, 1041, 1053, 1038, 1064, 1015, 1036, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1098, 1041, 1059, 1047, 1024, 1006, 984, 1007, 13, 4, 0, 0, 0, 0, 0, 0, 1026, 971, 1015, 1040, 1018, 1014, 1032, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1054, 1037, 1055, 1055, 1050, 1080, 980, 0, 0, 0, 0, 0, 0, 0, 0, 987, 997, 1047, 1034, 1039, 1030, 976, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1061, 1000, 1046, 1029, 996, 1027, 1089, 13, 0, 0, 0, 0, 0, 0, 0, 986, 1019, 1020, 1001, 990, 1083, 977, 1007, 38, 20, 0, 0, 0, 0, 0, 0, 1036, 1050, 984, 1032, 1001, 1007, 1048, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1021, 969, 1017, 991, 1004, 1034, 987, 9, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8162, 8354, 8093, 8282, 8141, 8168, 8053] avg=8192.0000 max=8354 max_rank=2 min=8053 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1006, 1036, 1012, 1091, 1062, 1042, 1023, 963, 13, 4, 0, 0, 0, 0, 0, 0, 1033, 1026, 1010, 1023, 1023, 1036, 989, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1046, 1026, 1014, 1008, 1047, 1055, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1017, 965, 1036, 1048, 997, 1009, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 977, 998, 1005, 995, 1051, 1021, 1000, 1020, 13, 0, 0, 0, 0, 0, 0, 0, 987, 1060, 1056, 1003, 1008, 1033, 1014, 1068, 37, 21, 0, 0, 0, 0, 0, 0, 1073, 1006, 1033, 1040, 1032, 993, 1022, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1008, 993, 1030, 991, 1054, 1057, 1013, 8, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8174, 8195, 8177, 8080, 8287, 8242, 8129] avg=8192.0000 max=8287 max_rank=5 min=8080 min_rank=4 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[986, 1021, 1004, 1032, 1057, 1012, 1040, 1008, 13, 4, 0, 0, 0, 0, 0, 0, 970, 1082, 1058, 998, 1045, 1017, 983, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1026, 1046, 991, 1013, 996, 1002, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1012, 1084, 1039, 1019, 1035, 1048, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 990, 1019, 1046, 1053, 1043, 1020, 990, 14, 0, 0, 0, 0, 0, 0, 0, 994, 1020, 1017, 1000, 985, 1047, 972, 1014, 37, 20, 0, 0, 0, 0, 0, 0, 1021, 1020, 1032, 988, 1027, 1033, 1007, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1052, 1020, 999, 1084, 1028, 1035, 1008, 8, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8186, 8135, 8311, 8206, 8106, 8146, 8269] avg=8192.0000 max=8311 max_rank=3 min=8106 min_rank=5 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[992, 983, 1057, 986, 994, 995, 1062, 1042, 13, 4, 0, 0, 0, 0, 0, 0, 1036, 1008, 1060, 1028, 1064, 984, 981, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1036, 1013, 1057, 1025, 1037, 1014, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1018, 1061, 1012, 1012, 1013, 1017, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1021, 1040, 1029, 1058, 987, 1040, 1033, 13, 0, 0, 0, 0, 0, 0, 0, 1042, 1019, 1039, 1016, 1041, 958, 1021, 1057, 37, 20, 0, 0, 0, 0, 0, 0, 981, 1029, 1034, 976, 1045, 1026, 993, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1051, 1025, 1018, 991, 1055, 1018, 1053, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8160, 8227, 8219, 8211, 8250, 8087, 8254] avg=8192.0000 max=8254 max_rank=7 min=8087 min_rank=6 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[978, 1031, 980, 1016, 992, 1051, 1034, 1048, 13, 4, 0, 0, 0, 0, 0, 0, 1042, 1006, 1007, 1011, 1025, 1038, 1006, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1020, 1007, 1024, 1035, 1020, 1011, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1093, 1065, 1005, 994, 1030, 1026, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 999, 1004, 1055, 1056, 1053, 1050, 1021, 13, 0, 0, 0, 0, 0, 0, 0, 1043, 998, 1049, 1050, 1004, 971, 1016, 1025, 37, 20, 0, 0, 0, 0, 0, 0, 981, 1023, 1040, 1044, 1040, 1027, 986, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1048, 1015, 1029, 1027, 1010, 983, 983, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8158, 8160, 8299, 8262, 8213, 8164, 8133] avg=8192.0000 max=8299 max_rank=3 min=8133 min_rank=7 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[990, 1036, 998, 1066, 1005, 994, 1038, 1024, 13, 4, 0, 0, 0, 0, 0, 0, 1053, 1020, 1039, 1099, 1025, 1044, 1005, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1003, 1011, 1001, 1032, 1063, 1027, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1020, 1014, 1048, 980, 1050, 983, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1029, 1031, 1043, 1005, 994, 1014, 1034, 13, 0, 0, 0, 0, 0, 0, 0, 1071, 1037, 962, 1008, 993, 966, 1019, 1014, 37, 20, 0, 0, 0, 0, 0, 0, 1053, 997, 1048, 1010, 992, 1030, 1052, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1039, 1041, 1068, 1045, 1026, 1023, 995, 9, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8293, 8148, 8124, 8201, 8127, 8175, 8300] avg=8192.0000 max=8300 max_rank=7 min=8124 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1056, 1033, 1025, 1021, 1007, 995, 999, 962, 12, 4, 0, 0, 0, 0, 0, 0, 1062, 1038, 996, 1034, 1024, 1064, 996, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1072, 1016, 1042, 984, 1001, 990, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1013, 1019, 982, 1031, 1038, 1048, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1010, 1046, 1067, 997, 974, 988, 947, 14, 0, 0, 0, 0, 0, 0, 0, 973, 1061, 1027, 1075, 995, 989, 1068, 1049, 37, 19, 0, 0, 0, 0, 0, 0, 1096, 1038, 1062, 1045, 1056, 1012, 1045, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1051, 1055, 944, 999, 1035, 967, 1009, 8, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8278, 8175, 8187, 8059, 8293, 8336, 8094] avg=8192.0000 max=8336 max_rank=6 min=8059 min_rank=4 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1054, 1008, 1034, 1014, 1046, 997, 1062, 1025, 18, 9, 0, 0, 0, 0, 0, 0, 993, 1006, 1011, 1008, 1070, 1033, 1013, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1011, 994, 1048, 1010, 1008, 1005, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1007, 1012, 1025, 1042, 986, 1008, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1023, 997, 1068, 1073, 1014, 1003, 982, 17, 0, 0, 0, 0, 0, 0, 0, 1048, 965, 1023, 939, 975, 1008, 1057, 1038, 42, 24, 0, 0, 0, 0, 0, 0, 1056, 1013, 1050, 1039, 1021, 1053, 955, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1041, 1053, 1038, 1064, 1015, 1036, 997, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8125, 8142, 8126, 8235, 8119, 8218, 8304] avg=8192.0000 max=8304 max_rank=7 min=8119 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8064, 8064, 8192, 8320, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8448, 8448, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8192, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1055, 1040, 1007, 1030, 1017, 1004, 1026, 1026, 1014, 1034, 1037, 1035, 1012, 1061, 1051, 1019, 972, 1038, 1058, 1043, 986, 1063, 1040, 1055, 1074, 1014, 1016, 1015, 1054, 987, 985, 986, 1040, 1032, 978, 1020, 996, 1041, 1015, 1045, 974, 1036, 1007, 1018, 1065, 1073, 1045, 1025, 1053, 1040, 1028, 1019, 1006, 1033, 1035, 1030, 995, 1023, 957, 1020, 982, 1012, 1002, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1019, 1032, 1014, 1061, 1021, 1025, 1013, 1025, 1022, 1002, 993, 1020, 1039, 970, 972, 1000, 1020, 1064, 1011, 1066, 1052, 973, 1052, 1037, 1032, 1045, 1057, 1034, 991, 994, 1007, 1000, 1078, 1006, 1005, 998, 1005, 1036, 994, 1077, 1056, 1093, 1022, 1053, 1038, 1043, 1048, 1008, 997, 1045, 994, 966, 1056, 991, 1002, 1002, 1019, 1058, 1016, 1042, 1042, 995, 1027, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1024, 1011, 998, 1037, 1013, 1012, 1027, 1036, 1045, 1012, 1019, 1000, 1025, 997, 1019, 1061, 1023, 1045, 1021, 1070, 1044, 1033, 1044, 1011, 999, 1017, 1032, 1067, 960, 1038, 1007, 1036, 998, 1020, 1024, 1021, 1025, 1004, 994, 1031, 1017, 1032, 1042, 991, 1052, 1057, 1038, 1023, 1031, 989, 977, 1035, 1003, 1004, 994, 1035, 1063, 989, 1075, 1060, 1040, 1032, 1054, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[999, 1040, 1033, 1029, 1034, 1026, 1002, 1008, 1035, 989, 1000, 1002, 1024, 1093, 1035, 1018, 1028, 1072, 1040, 1029, 1020, 1004, 1051, 1037, 1014, 1029, 1017, 1016, 1044, 971, 987, 1038, 1028, 1030, 993, 1010, 1006, 1041, 1023, 993, 1067, 983, 1027, 1015, 1031, 984, 1014, 1014, 1030, 1056, 1030, 1059, 1025, 1005, 1015, 1058, 1036, 975, 994, 1029, 1075, 1054, 1009, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1027, 1027, 1033, 1016, 1007, 994, 982, 1062, 1040, 990, 1022, 1081, 988, 1049, 1030, 1034, 1030, 1038, 988, 1006, 1038, 984, 1013, 1013, 970, 1014, 1019, 1003, 1014, 1051, 1056, 998, 1036, 1045, 1030, 1010, 1001, 1086, 1010, 1035, 966, 1029, 1044, 1007, 996, 1046, 1013, 985, 1034, 1001, 1028, 1011, 1046, 1029, 1058, 1005, 1051, 1095, 1027, 1054, 1029, 1040, 1031, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1072, 1030, 1054, 1043, 1038, 1008, 1034, 1026, 999, 977, 1017, 1076, 1010, 1028, 1046, 1017, 1023, 1025, 1071, 1015, 1020, 1056, 1051, 1020, 1022, 999, 996, 1006, 1057, 1042, 1021, 1008, 1054, 1056, 1020, 998, 985, 1055, 986, 1046, 987, 972, 1012, 1038, 1074, 1024, 1050, 1017, 1011, 1054, 1027, 1011, 959, 1034, 1044, 998, 1070, 990, 1005, 1019, 995, 1020, 975, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1042, 957, 1000, 1024, 1039, 991, 1084, 986, 1032, 1031, 1052, 1071, 1024, 1060, 980, 1068, 1025, 1013, 980, 1070, 1012, 1031, 996, 960, 1011, 1043, 1037, 1054, 1009, 1049, 993, 1024, 1071, 974, 1013, 1025, 1017, 1025, 982, 1005, 1079, 1011, 1004, 994, 988, 1042, 1011, 1102, 969, 1025, 1012, 1014, 1042, 1021, 1045, 1057, 1036, 1074, 1034, 1043, 1039, 1019, 990, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1030, 988, 1031, 1027, 985, 1035, 1051, 1060, 1011, 999, 1055, 1032, 997, 1026, 1031, 1066, 1008, 962, 1006, 1041, 1093, 994, 988, 1033, 995, 993, 1026, 1035, 1005, 1041, 1036, 1078, 1004, 1016, 1017, 1001, 1029, 994, 1031, 1019, 1063, 1008, 992, 1047, 1047, 995, 1012, 1024, 1039, 984, 1028, 1059, 1027, 1053, 995, 1055, 1027, 1007, 1028, 1030, 1045, 1059, 1017, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1019, 1032, 1014, 1061, 1021, 1025, 1013, 1025, 1, 0, 0, 0, 0, 0, 0, 0, 1022, 1002, 993, 1013, 1039, 970, 972, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1064, 1011, 1045, 1052, 973, 1052, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1045, 1057, 1034, 991, 994, 1007, 1000, 34, 0, 0, 0, 0, 0, 0, 0, 1078, 1006, 1005, 998, 1005, 1036, 994, 1077, 21, 8, 2, 0, 0, 0, 0, 0, 1056, 1093, 1022, 1053, 1022, 1043, 1048, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1045, 994, 966, 1056, 991, 1002, 1002, 8, 6, 0, 0, 0, 0, 0, 0, 1019, 1058, 1016, 1042, 1042, 995, 1027, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8011, 8254, 8194, 8230, 8345, 8067, 8224] avg=8192.0000 max=8345 max_rank=5 min=8011 min_rank=1 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32866 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1055, 1040, 1007, 1030, 1017, 1004, 1026, 1026, 1, 0, 0, 0, 0, 0, 0, 0, 1014, 1034, 1037, 1028, 1012, 1061, 1051, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1038, 1058, 1022, 986, 1063, 1040, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1014, 1016, 1015, 1054, 987, 985, 986, 33, 0, 0, 0, 0, 0, 0, 0, 1040, 1032, 978, 1020, 996, 1041, 1015, 1045, 21, 8, 2, 0, 0, 0, 0, 0, 974, 1036, 1007, 1018, 1049, 1073, 1045, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1040, 1028, 1019, 1006, 1033, 1035, 1030, 8, 6, 0, 0, 0, 0, 0, 0, 995, 1023, 957, 1020, 982, 1012, 1002, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8256, 8234, 8164, 8198, 8227, 8258, 7993] avg=8192.0000 max=8258 max_rank=6 min=7993 min_rank=7 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1072, 1030, 1054, 1043, 1038, 1008, 1034, 1026, 1, 0, 0, 0, 0, 0, 0, 0, 999, 977, 1017, 1069, 1010, 1028, 1046, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1025, 1071, 995, 1020, 1056, 1051, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 999, 996, 1006, 1057, 1042, 1021, 1008, 33, 0, 0, 0, 0, 0, 0, 0, 1054, 1056, 1020, 998, 985, 1055, 986, 1046, 20, 8, 2, 0, 0, 0, 0, 0, 987, 972, 1012, 1038, 1058, 1024, 1050, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1054, 1027, 1011, 959, 1034, 1044, 998, 8, 6, 0, 0, 0, 0, 0, 0, 1070, 990, 1005, 1019, 995, 1020, 975, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 8163, 8261, 8184, 8230, 8158, 8152, 8082] avg=8192.0000 max=8306 max_rank=0 min=8082 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[999, 1040, 1033, 1029, 1034, 1026, 1002, 1008, 1, 0, 0, 0, 0, 0, 0, 0, 1035, 989, 1000, 996, 1024, 1093, 1035, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1072, 1040, 1009, 1020, 1004, 1051, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1029, 1017, 1016, 1044, 971, 987, 1038, 34, 0, 0, 0, 0, 0, 0, 0, 1028, 1030, 993, 1010, 1006, 1041, 1023, 993, 20, 8, 2, 0, 0, 0, 0, 0, 1067, 983, 1027, 1015, 1015, 984, 1014, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1056, 1030, 1059, 1025, 1005, 1015, 1058, 8, 5, 0, 0, 0, 0, 0, 0, 1036, 975, 994, 1029, 1075, 1054, 1009, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8190, 8261, 8150, 8154, 8119, 8291, 8199] avg=8192.0000 max=8291 max_rank=6 min=8119 min_rank=5 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1027, 1027, 1033, 1016, 1007, 994, 982, 1062, 1, 0, 0, 0, 0, 0, 0, 0, 1040, 990, 1022, 1074, 988, 1049, 1030, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1038, 988, 986, 1038, 984, 1013, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1014, 1019, 1003, 1014, 1051, 1056, 998, 33, 0, 0, 0, 0, 0, 0, 0, 1036, 1045, 1030, 1010, 1001, 1086, 1010, 1035, 20, 7, 2, 0, 0, 0, 0, 0, 966, 1029, 1044, 1007, 981, 1046, 1013, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1001, 1028, 1011, 1046, 1029, 1058, 1005, 8, 6, 0, 0, 0, 0, 0, 0, 1051, 1095, 1027, 1054, 1029, 1040, 1031, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8227, 8090, 8158, 8282, 8071, 8226, 8333] avg=8192.0000 max=8333 max_rank=7 min=8071 min_rank=5 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1024, 1011, 998, 1037, 1013, 1012, 1027, 1036, 1, 0, 0, 0, 0, 0, 0, 0, 1045, 1012, 1019, 994, 1025, 997, 1019, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1045, 1021, 1049, 1044, 1033, 1044, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1017, 1032, 1067, 960, 1038, 1007, 1036, 32, 0, 0, 0, 0, 0, 0, 0, 998, 1020, 1024, 1021, 1025, 1004, 994, 1031, 21, 8, 2, 0, 0, 0, 0, 0, 1017, 1032, 1042, 991, 1036, 1057, 1038, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 989, 977, 1035, 1003, 1004, 994, 1035, 8, 5, 0, 0, 0, 0, 0, 0, 1063, 989, 1075, 1060, 1040, 1032, 1054, 969, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8172, 8270, 8188, 8148, 8236, 8081, 8282] avg=8192.0000 max=8282 max_rank=7 min=8081 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1042, 957, 1000, 1024, 1039, 991, 1084, 986, 1, 0, 0, 0, 0, 0, 0, 0, 1032, 1031, 1052, 1064, 1024, 1060, 980, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1013, 980, 1049, 1012, 1031, 996, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1043, 1037, 1054, 1009, 1049, 993, 1024, 33, 0, 0, 0, 0, 0, 0, 0, 1071, 974, 1013, 1025, 1017, 1025, 982, 1005, 21, 7, 2, 0, 0, 0, 0, 0, 1079, 1011, 1004, 994, 973, 1042, 1011, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1025, 1012, 1014, 1042, 1021, 1045, 1057, 8, 6, 0, 0, 0, 0, 0, 0, 1036, 1074, 1034, 1043, 1039, 1019, 990, 990, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8311, 8066, 8253, 8142, 8216, 8199, 8225] avg=8192.0000 max=8311 max_rank=1 min=8066 min_rank=2 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1030, 988, 1031, 1027, 985, 1035, 1051, 1060, 2, 0, 0, 0, 0, 0, 0, 0, 1011, 999, 1055, 1021, 997, 1026, 1031, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 962, 1006, 1016, 1093, 994, 988, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 995, 993, 1026, 1035, 1005, 1041, 1036, 1078, 36, 0, 0, 0, 0, 0, 0, 0, 1004, 1016, 1017, 1001, 1029, 994, 1031, 1019, 25, 11, 5, 0, 0, 0, 0, 0, 1063, 1008, 992, 1047, 1023, 995, 1012, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 984, 1028, 1059, 1027, 1053, 995, 1055, 13, 9, 0, 0, 0, 0, 0, 0, 1027, 1007, 1028, 1030, 1045, 1059, 1017, 985, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8206, 8100, 8245, 8152, 8164, 8262, 8198] avg=8192.0000 max=8262 max_rank=6 min=8100 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8320, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8064, 8320, 8064, 8320, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1065, 1011, 1004, 1060, 1041, 1010, 1003, 1040, 1034, 1036, 1027, 1007, 1038, 1008, 978, 1041, 1010, 1038, 1019, 1033, 966, 1058, 1049, 1023, 1036, 1056, 1071, 1040, 1041, 1054, 1016, 1031, 985, 1029, 1014, 1000, 1010, 1071, 1027, 999, 1016, 987, 1015, 1011, 1019, 1102, 1006, 1022, 995, 1010, 974, 1015, 983, 1027, 1010, 1046, 1006, 1071, 989, 1000, 1032, 1077, 1040, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1012, 996, 973, 1057, 971, 1037, 1032, 1046, 1030, 1080, 975, 1016, 1064, 1019, 991, 1003, 1050, 1031, 1066, 999, 1013, 1024, 1049, 1003, 1017, 1011, 974, 1001, 1026, 1043, 1030, 999, 1047, 1017, 984, 1015, 988, 1039, 1022, 1055, 1072, 1045, 1034, 1022, 1061, 979, 1039, 1051, 975, 1061, 1064, 1018, 1005, 1012, 1044, 1023, 1052, 1018, 1033, 1004, 1021, 1068, 1022, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1046, 1072, 998, 1047, 976, 1078, 1016, 1009, 1010, 1058, 985, 1028, 987, 1070, 1014, 1054, 1013, 1027, 1004, 1021, 1011, 992, 989, 1006, 1042, 1019, 1045, 1048, 993, 1022, 1077, 996, 968, 1020, 1042, 1021, 1023, 1051, 1023, 1052, 978, 971, 1044, 971, 986, 1048, 1006, 1024, 946, 1030, 1029, 1053, 1009, 1091, 1044, 1051, 1072, 1050, 1047, 1010, 1060, 1011, 1028, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1019, 1037, 1045, 1003, 1011, 1075, 994, 1008, 1012, 1048, 970, 961, 970, 1038, 1025, 1051, 1027, 1011, 1019, 1003, 998, 1003, 1000, 1039, 1024, 1011, 1016, 1063, 1059, 1032, 1027, 1075, 1063, 1011, 980, 991, 1053, 988, 1013, 1018, 1045, 1054, 1028, 1041, 1036, 1067, 990, 995, 1046, 1006, 975, 1056, 1027, 1040, 1040, 1068, 1057, 1045, 993, 1044, 989, 1075, 1011, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1015, 1056, 1008, 1032, 1027, 976, 1030, 1067, 1039, 1042, 1052, 979, 1006, 991, 996, 1023, 1046, 1033, 989, 1018, 1032, 1047, 1022, 1001, 1018, 1007, 1033, 1025, 1044, 1029, 992, 1028, 1055, 998, 1046, 1076, 987, 1021, 998, 1019, 1058, 1036, 1041, 1051, 1024, 1029, 996, 1021, 1003, 1024, 1058, 987, 1019, 1053, 1018, 1048, 992, 1018, 999, 983, 1048, 1031, 1040, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[980, 1058, 1004, 1061, 1056, 1016, 1040, 1005, 976, 1017, 1003, 1027, 1044, 1009, 1030, 1050, 1026, 1064, 962, 1026, 1023, 1020, 1003, 989, 1015, 1021, 960, 1054, 1013, 1050, 1018, 1026, 1019, 1041, 1040, 1073, 965, 1051, 1029, 1005, 1053, 1062, 1031, 1062, 994, 973, 1060, 1002, 994, 995, 1038, 1040, 1027, 1014, 1060, 1047, 995, 999, 1026, 1021, 1007, 1039, 1068, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[983, 1000, 1064, 1056, 978, 1033, 1029, 1063, 1035, 1020, 1026, 954, 1007, 1043, 1010, 1034, 1000, 1004, 1056, 1005, 1016, 1045, 1059, 1009, 957, 1037, 990, 1021, 990, 967, 1023, 1035, 1010, 1036, 1012, 1043, 995, 1036, 1053, 1016, 998, 1023, 1085, 1026, 1006, 989, 1034, 989, 1073, 1047, 1076, 1033, 1022, 1071, 1068, 1034, 995, 1032, 1047, 1053, 1056, 1000, 1016, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1036, 1029, 1007, 1013, 997, 999, 1015, 998, 1040, 1023, 1071, 1073, 1033, 1026, 981, 1047, 1016, 1045, 1037, 1076, 1019, 982, 1009, 987, 1090, 1024, 991, 1064, 1040, 1016, 1023, 1002, 965, 1062, 1032, 1008, 1062, 1052, 1031, 988, 1007, 1053, 1004, 1012, 1006, 1000, 1037, 1036, 1020, 994, 1054, 1073, 1064, 1008, 1078, 953, 1010, 1026, 961, 1026, 1091, 997, 975, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1012, 996, 973, 1057, 971, 1037, 1032, 1046, 1, 0, 0, 0, 0, 0, 0, 0, 1030, 1080, 975, 1016, 1064, 1019, 991, 1003, 18, 6, 0, 0, 0, 0, 0, 0, 1050, 1031, 1066, 999, 1013, 1024, 1049, 1003, 34, 0, 0, 0, 0, 0, 0, 0, 1017, 1011, 974, 994, 1026, 1043, 1030, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1017, 984, 1015, 988, 1039, 1022, 1055, 7, 2, 1, 0, 0, 0, 0, 0, 1072, 1045, 1032, 1022, 1061, 979, 1039, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1061, 1064, 1018, 1005, 1012, 1004, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1018, 1033, 1004, 1001, 1068, 1022, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8202, 8269, 8094, 8177, 8301, 8162, 8206] avg=8192.0000 max=8301 max_rank=5 min=8094 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1065, 1011, 1004, 1060, 1041, 1010, 1003, 1040, 1, 0, 0, 0, 0, 0, 0, 0, 1034, 1036, 1027, 1007, 1038, 1008, 978, 1041, 18, 6, 0, 0, 0, 0, 0, 0, 1010, 1038, 1019, 1033, 966, 1058, 1049, 1023, 33, 0, 0, 0, 0, 0, 0, 0, 1036, 1056, 1071, 1033, 1041, 1054, 1016, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1029, 1014, 1000, 1010, 1071, 1027, 999, 7, 2, 1, 0, 0, 0, 0, 0, 1016, 987, 1013, 1011, 1019, 1102, 1006, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1010, 974, 1015, 983, 1027, 971, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1071, 989, 1000, 1012, 1077, 1040, 1004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8193, 8229, 8338, 8145, 8176, 8021, 8199] avg=8192.0000 max=8338 max_rank=3 min=8021 min_rank=6 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32541 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1015, 1056, 1008, 1032, 1027, 976, 1030, 1067, 1, 0, 0, 0, 0, 0, 0, 0, 1039, 1042, 1052, 979, 1006, 991, 996, 1023, 18, 6, 0, 0, 0, 0, 0, 0, 1046, 1033, 989, 1018, 1032, 1047, 1022, 1001, 33, 0, 0, 0, 0, 0, 0, 0, 1018, 1007, 1033, 1018, 1044, 1029, 992, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 998, 1046, 1076, 987, 1021, 998, 1019, 7, 2, 1, 0, 0, 0, 0, 0, 1058, 1036, 1039, 1051, 1024, 1029, 996, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1024, 1058, 987, 1019, 1053, 979, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1018, 999, 983, 1028, 1031, 1040, 1056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8152, 8221, 8169, 8210, 8254, 8171, 8147] avg=8192.0000 max=8254 max_rank=5 min=8147 min_rank=7 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1046, 1072, 998, 1047, 976, 1078, 1016, 1009, 1, 0, 0, 0, 0, 0, 0, 0, 1010, 1058, 985, 1028, 987, 1070, 1014, 1054, 18, 6, 0, 0, 0, 0, 0, 0, 1013, 1027, 1004, 1021, 1011, 992, 989, 1006, 34, 0, 0, 0, 0, 0, 0, 0, 1042, 1019, 1045, 1041, 993, 1022, 1077, 996, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1020, 1042, 1021, 1023, 1051, 1023, 1052, 7, 2, 1, 0, 0, 0, 0, 0, 978, 971, 1042, 971, 986, 1048, 1006, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 946, 1030, 1029, 1053, 1009, 1091, 1004, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1050, 1047, 1010, 1040, 1011, 1028, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8230, 8097, 8235, 8210, 8026, 8213, 8282] avg=8192.0000 max=8282 max_rank=7 min=8026 min_rank=5 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1019, 1037, 1045, 1003, 1011, 1075, 994, 1008, 1, 0, 0, 0, 0, 0, 0, 0, 1012, 1048, 970, 961, 970, 1038, 1025, 1051, 17, 6, 0, 0, 0, 0, 0, 0, 1027, 1011, 1019, 1003, 998, 1003, 1000, 1039, 34, 0, 0, 0, 0, 0, 0, 0, 1024, 1011, 1016, 1056, 1059, 1032, 1027, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1011, 980, 991, 1053, 988, 1013, 1018, 7, 2, 1, 0, 0, 0, 0, 0, 1045, 1054, 1026, 1041, 1036, 1067, 990, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1006, 975, 1056, 1027, 1040, 1000, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1045, 993, 1044, 970, 1075, 1011, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8098, 8134, 8300, 8127, 8254, 8218, 8212] avg=8192.0000 max=8300 max_rank=3 min=8098 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[980, 1058, 1004, 1061, 1056, 1016, 1040, 1005, 1, 0, 0, 0, 0, 0, 0, 0, 976, 1017, 1003, 1027, 1044, 1009, 1030, 1050, 17, 6, 0, 0, 0, 0, 0, 0, 1026, 1064, 962, 1026, 1023, 1020, 1003, 989, 34, 0, 0, 0, 0, 0, 0, 0, 1015, 1021, 960, 1047, 1013, 1050, 1018, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1041, 1040, 1073, 965, 1051, 1029, 1005, 7, 2, 1, 0, 0, 0, 0, 0, 1053, 1062, 1029, 1062, 994, 973, 1060, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 994, 995, 1038, 1040, 1027, 1014, 1020, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 995, 999, 1026, 1021, 988, 1039, 1068, 1060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8179, 8147, 8150, 8233, 8235, 8175, 8196] avg=8192.0000 max=8235 max_rank=5 min=8147 min_rank=2 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[983, 1000, 1064, 1056, 978, 1033, 1029, 1063, 1, 0, 0, 0, 0, 0, 0, 0, 1035, 1020, 1026, 954, 1007, 1043, 1010, 1034, 18, 6, 0, 0, 0, 0, 0, 0, 1000, 1004, 1056, 1005, 1016, 1045, 1059, 1009, 35, 0, 0, 0, 0, 0, 0, 0, 957, 1037, 990, 1014, 990, 967, 1023, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1036, 1012, 1043, 995, 1036, 1053, 1016, 7, 2, 1, 0, 0, 0, 0, 0, 998, 1023, 1083, 1026, 1006, 989, 1034, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1047, 1076, 1033, 1022, 1071, 1027, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1032, 1047, 1053, 1036, 1000, 1016, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8153, 8229, 8013, 8211, 8148, 8383, 8192] avg=8192.0000 max=8383 max_rank=6 min=8013 min_rank=3 imbalance_ratio=1.023315 max_load_reduction=57153 max_load_reduction_pct=0.872086 moved_to_echo=32934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1036, 1029, 1007, 1013, 997, 999, 1015, 998, 6, 0, 0, 0, 0, 0, 0, 0, 1040, 1023, 1071, 1073, 1033, 1026, 981, 1047, 23, 12, 0, 0, 0, 0, 0, 0, 1016, 1045, 1037, 1076, 1019, 982, 1009, 987, 39, 0, 0, 0, 0, 0, 0, 0, 1090, 1024, 991, 1051, 1040, 1016, 1023, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 965, 1062, 1032, 1008, 1062, 1052, 1031, 988, 13, 3, 7, 0, 0, 0, 0, 0, 1007, 1053, 991, 1012, 1006, 1000, 1037, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 994, 1054, 1073, 1064, 1008, 1027, 953, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1026, 961, 1026, 1065, 997, 975, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8329, 8210, 8237, 8223, 8142, 8193, 8102] avg=8192.0000 max=8329 max_rank=1 min=8100 min_rank=0 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32660 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8192, 8064, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8064, 8192, 8320, 8320, 8192, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8192, 8064, 8192, 8320, 8064, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8448, 8064, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1059, 965, 1053, 1053, 1040, 1039, 1076, 1024, 1015, 1028, 999, 1023, 1011, 990, 1007, 998, 968, 1041, 1025, 1029, 1048, 1025, 993, 1010, 998, 1027, 1034, 1027, 1062, 1037, 1040, 999, 1018, 1024, 999, 998, 1062, 1025, 1046, 1004, 945, 1028, 1070, 1014, 1026, 1041, 1027, 1037, 1001, 1016, 1044, 1039, 1014, 1046, 1046, 997, 1020, 1059, 1017, 1037, 1023, 1032, 1039, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1030, 971, 1040, 991, 1044, 994, 1017, 1032, 1034, 1022, 1041, 1018, 1014, 992, 1023, 1022, 990, 1052, 1079, 1013, 1046, 1059, 1095, 1007, 1005, 1002, 985, 1033, 1074, 1020, 1018, 965, 1048, 1017, 977, 978, 1008, 1043, 1038, 1018, 1035, 1011, 1079, 1006, 990, 1053, 1053, 1078, 1023, 1050, 1031, 1013, 1061, 993, 995, 1050, 1025, 988, 1028, 1033, 1061, 1005, 1005, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1051, 1032, 998, 1029, 1051, 1009, 1051, 1032, 1067, 964, 1003, 1085, 1034, 1010, 1041, 1042, 973, 1034, 1017, 995, 1035, 1014, 980, 1030, 1030, 1036, 993, 993, 1120, 988, 1060, 1035, 993, 1035, 1036, 980, 1027, 1014, 980, 1034, 1019, 1032, 1018, 1091, 981, 1011, 1036, 1014, 990, 1033, 1059, 1053, 1015, 1037, 1037, 1012, 1010, 1035, 1025, 1039, 978, 1000, 1036, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1049, 990, 1029, 1060, 1039, 995, 970, 1026, 1015, 1019, 995, 927, 1025, 996, 1063, 1026, 1033, 990, 1094, 1054, 1023, 1055, 1028, 1059, 1051, 1009, 1009, 1047, 1014, 1040, 1050, 992, 1025, 962, 1030, 1052, 1022, 994, 1056, 992, 1041, 997, 1050, 1010, 1029, 1059, 1047, 1030, 994, 1011, 995, 1035, 998, 999, 1071, 1043, 1037, 1036, 1003, 1052, 995, 1035, 988, 1076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1063, 981, 1047, 1070, 1015, 967, 1031, 1003, 1062, 1049, 1030, 1013, 1026, 959, 1036, 1042, 1049, 1036, 1000, 1062, 994, 1003, 1055, 1048, 985, 1039, 1063, 1008, 1008, 975, 978, 1021, 1003, 1003, 1046, 1020, 1055, 1037, 1005, 1036, 1021, 1012, 1045, 1043, 1013, 1011, 1013, 989, 1104, 1028, 1015, 988, 1023, 1009, 1013, 1009, 1053, 1006, 1064, 1043, 1024, 1021, 1004, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[992, 1033, 1035, 1082, 1018, 1014, 1032, 990, 1022, 1027, 1026, 986, 1027, 1023, 1021, 1066, 1022, 1007, 1051, 1018, 1032, 1010, 1065, 985, 999, 1047, 974, 1009, 1061, 1017, 1029, 1019, 1026, 999, 1029, 1036, 1053, 986, 1072, 1001, 1009, 1000, 1008, 1014, 1035, 1003, 1016, 1031, 1030, 1047, 1069, 1032, 1033, 984, 1038, 1007, 1058, 1012, 1009, 1029, 1052, 1020, 1028, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1010, 1068, 1013, 1006, 1019, 1015, 1011, 997, 1045, 1015, 1011, 1016, 1037, 1006, 993, 1054, 1056, 994, 1044, 991, 1036, 1046, 997, 1027, 994, 1062, 986, 1056, 1053, 1020, 1007, 1018, 1028, 994, 1006, 1034, 975, 1021, 1001, 1092, 1012, 1051, 1027, 1046, 1007, 1052, 1059, 1003, 987, 1051, 1025, 1009, 1030, 1044, 1058, 988, 1001, 1041, 1059, 1023, 1007, 1049, 1027, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1021, 983, 1039, 1013, 1029, 1013, 1048, 993, 1034, 1003, 1066, 1064, 1009, 1055, 1037, 1029, 1049, 1021, 994, 995, 1034, 1026, 1006, 1008, 999, 1000, 1003, 1049, 1061, 1016, 1013, 1014, 1001, 1027, 1002, 1030, 1036, 1037, 1014, 1052, 977, 1043, 1042, 1051, 1017, 950, 1004, 1038, 1056, 1020, 1009, 975, 1084, 1037, 1032, 1011, 1052, 1014, 1068, 1008, 1007, 1046, 1016, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1030, 971, 1040, 991, 1044, 994, 1017, 1032, 5, 0, 0, 0, 0, 0, 0, 0, 1034, 1022, 1041, 1018, 1014, 992, 1023, 1022, 9, 2, 0, 0, 0, 0, 0, 0, 990, 1052, 1064, 1013, 1046, 1059, 1095, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1002, 985, 1033, 1074, 1020, 1018, 965, 15, 3, 1, 0, 0, 0, 0, 0, 1048, 1017, 977, 978, 1008, 1043, 1038, 1018, 29, 0, 0, 0, 0, 0, 0, 0, 1035, 1011, 1072, 1006, 990, 1053, 1053, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1050, 1031, 1013, 1061, 993, 983, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 988, 1028, 1033, 1061, 1005, 1005, 985, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8177, 8326, 8121, 8156, 8298, 8204, 8130] avg=8192.0000 max=8326 max_rank=2 min=8121 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1051, 1032, 998, 1029, 1051, 1009, 1051, 1032, 5, 0, 0, 0, 0, 0, 0, 0, 1067, 964, 1003, 1085, 1034, 1010, 1041, 1042, 10, 2, 0, 0, 0, 0, 0, 0, 973, 1034, 1003, 995, 1035, 1014, 980, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1036, 993, 993, 1120, 988, 1060, 1035, 14, 3, 1, 0, 0, 0, 0, 0, 993, 1035, 1036, 980, 1027, 1014, 980, 1034, 30, 0, 0, 0, 0, 0, 0, 0, 1019, 1032, 1011, 1091, 981, 1011, 1036, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1033, 1059, 1053, 1015, 1037, 1024, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1035, 1025, 1039, 978, 1000, 1036, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8258, 8064, 8273, 8129, 8195, 8223, 8136] avg=8192.0000 max=8273 max_rank=3 min=8064 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1059, 965, 1053, 1053, 1040, 1039, 1076, 1024, 5, 0, 0, 0, 0, 0, 0, 0, 1015, 1028, 999, 1023, 1011, 990, 1007, 998, 10, 2, 0, 0, 0, 0, 0, 0, 968, 1041, 1011, 1029, 1048, 1025, 993, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1027, 1034, 1027, 1062, 1037, 1040, 999, 14, 3, 1, 0, 0, 0, 0, 0, 1018, 1024, 999, 998, 1062, 1025, 1046, 1004, 29, 0, 0, 0, 0, 0, 0, 0, 945, 1028, 1063, 1014, 1026, 1041, 1027, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1016, 1044, 1039, 1014, 1046, 1033, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1059, 1017, 1037, 1023, 1032, 1039, 969, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8314, 8083, 8125, 8242, 8205, 8181, 8190, 8196] avg=8192.0000 max=8314 max_rank=0 min=8083 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1063, 981, 1047, 1070, 1015, 967, 1031, 1003, 5, 0, 0, 0, 0, 0, 0, 0, 1062, 1049, 1030, 1013, 1026, 959, 1036, 1042, 9, 2, 0, 0, 0, 0, 0, 0, 1049, 1036, 986, 1062, 994, 1003, 1055, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1039, 1063, 1008, 1008, 975, 978, 1021, 14, 3, 1, 0, 0, 0, 0, 0, 1003, 1003, 1046, 1020, 1055, 1037, 1005, 1036, 31, 0, 0, 0, 0, 0, 0, 0, 1021, 1012, 1038, 1043, 1013, 1011, 1013, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 1028, 1015, 988, 1023, 1009, 1001, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1006, 1064, 1043, 1024, 1021, 1004, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8228, 8233, 8095, 8236, 8140, 8177, 8245] avg=8192.0000 max=8245 max_rank=7 min=8095 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1049, 990, 1029, 1060, 1039, 995, 970, 1026, 5, 0, 0, 0, 0, 0, 0, 0, 1015, 1019, 995, 927, 1025, 996, 1063, 1026, 10, 2, 0, 0, 0, 0, 0, 0, 1033, 990, 1079, 1054, 1023, 1055, 1028, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1009, 1009, 1047, 1014, 1040, 1050, 992, 15, 3, 1, 0, 0, 0, 0, 0, 1025, 962, 1030, 1052, 1022, 994, 1056, 992, 31, 0, 0, 0, 0, 0, 0, 0, 1041, 997, 1043, 1010, 1029, 1059, 1047, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1011, 995, 1035, 998, 999, 1058, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1036, 1003, 1052, 995, 1035, 988, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8078, 8321, 8231, 8164, 8256, 8133, 8190] avg=8192.0000 max=8321 max_rank=2 min=8078 min_rank=1 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[992, 1033, 1035, 1082, 1018, 1014, 1032, 990, 5, 0, 0, 0, 0, 0, 0, 0, 1022, 1027, 1026, 986, 1027, 1023, 1021, 1066, 10, 2, 0, 0, 0, 0, 0, 0, 1022, 1007, 1036, 1018, 1032, 1010, 1065, 985, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1047, 974, 1009, 1061, 1017, 1029, 1019, 15, 3, 1, 0, 0, 0, 0, 0, 1026, 999, 1029, 1036, 1053, 986, 1072, 1001, 30, 0, 0, 0, 0, 0, 0, 0, 1009, 1000, 1001, 1014, 1035, 1003, 1016, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1047, 1069, 1032, 1033, 984, 1025, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1012, 1009, 1029, 1052, 1020, 1028, 1000, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8210, 8175, 8174, 8232, 8109, 8227, 8208] avg=8192.0000 max=8232 max_rank=4 min=8109 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1021, 983, 1039, 1013, 1029, 1013, 1048, 993, 11, 0, 0, 0, 0, 0, 0, 0, 1034, 1003, 1066, 1064, 1009, 1055, 1037, 1029, 12, 4, 0, 0, 0, 0, 0, 0, 1049, 1021, 976, 995, 1034, 1026, 1006, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1000, 1003, 1049, 1061, 1016, 1013, 1014, 18, 9, 4, 0, 0, 0, 0, 0, 1001, 1027, 1002, 1030, 1036, 1037, 1014, 1052, 34, 0, 0, 0, 0, 0, 0, 0, 977, 1043, 1027, 1051, 1017, 950, 1004, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1020, 1009, 975, 1084, 1037, 1011, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1014, 1068, 1008, 1007, 1046, 1016, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8313, 8115, 8186, 8233, 8107, 8203, 8229] avg=8192.0000 max=8313 max_rank=1 min=8107 min_rank=5 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1010, 1068, 1013, 1006, 1019, 1015, 1011, 997, 5, 0, 0, 0, 0, 0, 0, 0, 1045, 1015, 1011, 1016, 1037, 1006, 993, 1054, 10, 2, 0, 0, 0, 0, 0, 0, 1056, 994, 1030, 991, 1036, 1046, 997, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1062, 986, 1056, 1053, 1020, 1007, 1018, 14, 3, 1, 0, 0, 0, 0, 0, 1028, 994, 1006, 1034, 975, 1021, 1001, 1092, 30, 0, 0, 0, 0, 0, 0, 0, 1012, 1051, 1020, 1046, 1007, 1052, 1059, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1051, 1025, 1009, 1030, 1044, 1045, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1041, 1059, 1023, 1007, 1049, 1027, 995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8189, 8177, 8214, 8181, 8250, 8179, 8202] avg=8192.0000 max=8250 max_rank=5 min=8144 min_rank=0 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8064, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8064, 8320, 8576, 8192, 8320, 8064, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8320, 8320, 8064, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8192, 8320, 8192, 8320, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[993, 1014, 1052, 1053, 1035, 987, 1012, 1034, 1003, 1041, 1023, 1049, 1035, 994, 1014, 1024, 976, 998, 1037, 1027, 1053, 1010, 1067, 1008, 1032, 992, 1029, 1008, 1052, 1016, 981, 1024, 986, 983, 1073, 1004, 1029, 1038, 1085, 1036, 1074, 1036, 1013, 1003, 1041, 1004, 1050, 1035, 1061, 1003, 1044, 1020, 1005, 1014, 1015, 991, 1044, 1065, 1037, 983, 1040, 1006, 999, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1027, 1053, 990, 993, 1000, 992, 1031, 1045, 1031, 1025, 1024, 1004, 1000, 1066, 1038, 1067, 1017, 1057, 1040, 1009, 1005, 965, 1079, 1057, 1016, 1013, 1019, 1067, 1006, 1036, 1005, 1031, 1014, 1024, 1014, 1006, 1036, 1047, 1071, 1040, 1049, 1070, 1005, 1012, 1051, 1005, 1015, 948, 1039, 1048, 1019, 1012, 1021, 1025, 1047, 1040, 970, 1027, 1005, 1024, 1018, 1008, 979, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1017, 1005, 959, 1033, 1025, 1002, 1053, 992, 1062, 954, 1044, 990, 1030, 1064, 1007, 1049, 997, 1022, 1011, 1036, 1031, 967, 1033, 1043, 959, 1068, 1065, 1030, 1002, 1007, 1009, 1037, 997, 1018, 1053, 1040, 1020, 1053, 1048, 1009, 1008, 981, 1041, 1005, 1053, 1046, 1040, 1057, 1029, 1064, 1019, 1013, 987, 993, 1024, 1064, 1073, 1033, 1004, 1019, 1074, 1029, 1057, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1028, 1015, 1008, 1028, 1005, 991, 1006, 983, 986, 976, 1023, 1012, 1006, 1017, 995, 1022, 1025, 1039, 1040, 1011, 1057, 1029, 1039, 1019, 1059, 1018, 1034, 1038, 1033, 1027, 1060, 1018, 1020, 1017, 1064, 1008, 1028, 1014, 1010, 1035, 977, 999, 1026, 1007, 1026, 1034, 1027, 1016, 1020, 1023, 1023, 1029, 1033, 1006, 1022, 1051, 1057, 1008, 1067, 996, 1064, 1074, 1009, 1099] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1022, 1019, 1009, 997, 1017, 1014, 1049, 1017, 1037, 1074, 1058, 983, 1008, 1033, 1073, 1015, 1068, 1021, 988, 984, 1054, 988, 1004, 1044, 966, 1039, 1114, 1004, 1038, 1034, 1015, 1051, 1015, 972, 969, 1000, 1002, 1026, 1034, 1097, 1002, 1012, 1026, 1020, 1023, 1024, 1000, 989, 1013, 1051, 1051, 1086, 1057, 988, 1054, 1034, 987, 1034, 1036, 1049, 965, 1017, 1002, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1006, 1069, 1009, 1037, 991, 1012, 1008, 1079, 1005, 996, 1011, 1018, 1071, 1034, 1031, 1000, 1039, 1022, 1037, 1026, 1000, 1071, 974, 1021, 1012, 1080, 1011, 1009, 1041, 1044, 961, 1021, 1001, 1018, 1022, 1058, 1051, 1037, 1006, 1039, 1034, 1045, 1072, 1034, 1005, 1002, 993, 1045, 1041, 959, 1020, 993, 978, 1000, 1087, 1028, 997, 1025, 1062, 1028, 1030, 1045, 1042, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[979, 1035, 1044, 945, 1041, 1020, 1040, 1028, 1039, 1007, 1082, 1033, 1002, 1036, 985, 1070, 965, 1034, 1039, 1005, 1036, 1051, 1018, 1024, 989, 988, 1025, 1044, 1033, 1022, 1089, 1029, 1049, 1000, 1024, 1027, 1019, 1006, 1045, 1063, 999, 1057, 1052, 1055, 1076, 976, 987, 1069, 1024, 1075, 1021, 989, 1047, 992, 976, 1018, 1032, 1016, 996, 1063, 986, 1010, 1019, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1014, 1052, 1031, 1016, 1033, 1046, 1041, 1036, 1025, 1050, 1030, 1056, 1010, 1061, 1006, 1040, 1011, 1046, 967, 1029, 1010, 1050, 1015, 1059, 1000, 991, 1049, 1000, 998, 1008, 1032, 1019, 1009, 1021, 1008, 995, 968, 1026, 1020, 992, 1030, 1051, 1041, 1000, 985, 1053, 1050, 1020, 1040, 1029, 1006, 1061, 1020, 1048, 1002, 1029, 1007, 1045, 1056, 977, 1046, 1037, 1026, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1027, 1053, 990, 993, 1000, 992, 1031, 1045, 15, 10, 10, 3, 0, 0, 0, 0, 1031, 1025, 1024, 1004, 1000, 1051, 1038, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1057, 1040, 1009, 1005, 965, 1079, 1057, 2, 1, 0, 0, 0, 0, 0, 0, 1016, 1013, 1018, 1067, 1006, 1036, 1005, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1024, 1014, 1006, 1036, 1047, 1071, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1070, 1000, 1012, 1051, 1005, 1015, 948, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1048, 1019, 1012, 1021, 1025, 1047, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1027, 995, 1024, 1018, 1008, 979, 1039, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8240, 8232, 8192, 8252, 8150, 8241, 8060] avg=8192.0000 max=8252 max_rank=4 min=8060 min_rank=7 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[993, 1014, 1052, 1053, 1035, 987, 1012, 1034, 14, 11, 10, 3, 0, 0, 0, 0, 1003, 1041, 1023, 1049, 1035, 980, 1014, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 976, 998, 1037, 1027, 1053, 1010, 1067, 1008, 2, 1, 0, 0, 0, 0, 0, 0, 1032, 992, 1028, 1008, 1052, 1016, 981, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 986, 983, 1073, 1004, 1029, 1038, 1085, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1036, 1008, 1003, 1041, 1004, 1050, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1003, 1044, 1020, 1005, 1014, 1015, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1065, 1026, 983, 1040, 1006, 999, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8169, 8179, 8133, 8234, 8251, 8143, 8209] avg=8192.0000 max=8251 max_rank=5 min=8133 min_rank=3 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1028, 1015, 1008, 1028, 1005, 991, 1006, 983, 14, 11, 10, 3, 0, 0, 0, 0, 986, 976, 1023, 1012, 1006, 1003, 995, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1039, 1040, 1011, 1057, 1029, 1039, 1019, 2, 1, 0, 0, 0, 0, 0, 0, 1059, 1018, 1033, 1038, 1033, 1027, 1060, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1017, 1064, 1008, 1028, 1014, 1010, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 977, 999, 1021, 1007, 1026, 1034, 1027, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1023, 1023, 1029, 1033, 1006, 1022, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1008, 1056, 996, 1064, 1074, 1009, 1099, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8023, 8262, 8286, 8196, 8107, 8197, 8363] avg=8192.0000 max=8363 max_rank=7 min=8023 min_rank=1 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32863 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1017, 1005, 959, 1033, 1025, 1002, 1053, 992, 15, 10, 10, 3, 0, 0, 0, 0, 1062, 954, 1044, 990, 1030, 1049, 1007, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1022, 1011, 1036, 1031, 967, 1033, 1043, 2, 1, 0, 0, 0, 0, 0, 0, 959, 1068, 1064, 1030, 1002, 1007, 1009, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1018, 1053, 1040, 1020, 1053, 1048, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 981, 1036, 1005, 1053, 1046, 1040, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1064, 1019, 1013, 987, 993, 1024, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1033, 994, 1019, 1074, 1029, 1057, 982, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8185, 8143, 8176, 8238, 8226, 8183, 8261] avg=8192.0000 max=8261 max_rank=7 min=8124 min_rank=0 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32908 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1022, 1019, 1009, 997, 1017, 1014, 1049, 1017, 14, 11, 10, 3, 0, 0, 0, 0, 1037, 1074, 1058, 983, 1008, 1019, 1073, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1021, 988, 984, 1054, 988, 1004, 1044, 2, 1, 0, 0, 0, 0, 0, 0, 966, 1039, 1113, 1004, 1038, 1034, 1015, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 972, 969, 1000, 1002, 1026, 1034, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1012, 1021, 1020, 1023, 1024, 1000, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1051, 1051, 1086, 1057, 988, 1054, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1034, 1025, 1049, 965, 1017, 1002, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8267, 8154, 8260, 8115, 8091, 8324, 8143] avg=8192.0000 max=8324 max_rank=6 min=8091 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1006, 1069, 1009, 1037, 991, 1012, 1008, 1079, 14, 11, 10, 3, 0, 0, 0, 0, 1005, 996, 1011, 1018, 1071, 1020, 1031, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1022, 1037, 1026, 1000, 1071, 974, 1021, 2, 1, 0, 0, 0, 0, 0, 0, 1012, 1080, 1010, 1009, 1041, 1044, 961, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1018, 1022, 1058, 1051, 1037, 1006, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1045, 1067, 1034, 1005, 1002, 993, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 959, 1020, 993, 978, 1000, 1087, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1025, 1051, 1028, 1030, 1045, 1042, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8152, 8193, 8178, 8232, 8225, 8096, 8211] avg=8192.0000 max=8249 max_rank=0 min=8096 min_rank=6 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1014, 1052, 1031, 1016, 1033, 1046, 1041, 1036, 18, 15, 15, 6, 0, 0, 0, 0, 1025, 1050, 1030, 1056, 1010, 1043, 1006, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1046, 967, 1029, 1010, 1050, 1015, 1059, 4, 4, 3, 0, 0, 0, 0, 0, 1000, 991, 1045, 1000, 998, 1008, 1032, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1021, 1008, 995, 968, 1026, 1017, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1051, 1031, 1000, 985, 1053, 1050, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1029, 1006, 1061, 1020, 1048, 1002, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1045, 1041, 977, 1046, 1037, 1026, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8323, 8260, 8198, 8093, 8036, 8220, 8220, 8186] avg=8192.0000 max=8323 max_rank=0 min=8036 min_rank=4 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[979, 1035, 1044, 945, 1041, 1020, 1040, 1028, 14, 10, 10, 3, 0, 0, 0, 0, 1039, 1007, 1082, 1033, 1002, 1022, 985, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 965, 1034, 1039, 1005, 1036, 1051, 1018, 1024, 2, 1, 0, 0, 0, 0, 0, 0, 989, 988, 1024, 1044, 1033, 1022, 1089, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1000, 1024, 1027, 1019, 1006, 1045, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1057, 1047, 1055, 1076, 976, 987, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1075, 1021, 989, 1047, 992, 976, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1016, 986, 1063, 986, 1010, 1019, 991, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8240, 8175, 8218, 8233, 8266, 8132, 8103] avg=8192.0000 max=8266 max_rank=5 min=8103 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8064, 8320, 8320, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8448, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1027, 1007, 1013, 1040, 1010, 997, 1049, 972, 982, 995, 1021, 1036, 1022, 1011, 1008, 1097, 1056, 1023, 1057, 1017, 1028, 1014, 983, 1061, 1044, 1020, 976, 1106, 1003, 1014, 972, 1046, 995, 984, 1028, 1000, 1057, 1064, 1034, 1042, 1024, 1011, 1004, 1010, 995, 1011, 1008, 1035, 986, 1013, 1053, 1055, 1015, 1053, 997, 992, 1062, 1043, 1063, 1017, 1075, 1036, 1034, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1033, 994, 986, 1101, 1045, 1030, 1058, 1025, 1035, 1001, 1076, 1079, 1015, 1042, 994, 1086, 987, 990, 1020, 1022, 1046, 1073, 1041, 1040, 1052, 1021, 1037, 1032, 1038, 966, 976, 1018, 1001, 994, 1076, 989, 1029, 1032, 1002, 998, 1022, 1060, 1039, 1014, 988, 1030, 1042, 1005, 977, 1043, 1039, 1054, 972, 989, 1009, 1019, 1020, 1025, 1031, 1010, 974, 1070, 1040, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1026, 1014, 1056, 981, 1076, 992, 1011, 1085, 1004, 993, 1007, 1007, 998, 1007, 1040, 1020, 994, 988, 996, 996, 1064, 1065, 1020, 1047, 1064, 1007, 1022, 1047, 945, 1023, 982, 1056, 1036, 1028, 1001, 1052, 1020, 1019, 1017, 987, 1032, 1013, 1046, 1036, 1027, 1061, 1067, 992, 1001, 1048, 1030, 1021, 1044, 1027, 1018, 1012, 1015, 1028, 1072, 1051, 1041, 1041, 1027, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1012, 1001, 1029, 1054, 1038, 1037, 1031, 1063, 1008, 1008, 1046, 1000, 1049, 1042, 1045, 1023, 1043, 1052, 995, 1043, 1021, 1001, 1018, 1018, 1033, 988, 1002, 1020, 1020, 994, 1067, 1003, 1007, 1021, 1059, 1038, 1020, 985, 1033, 1044, 1004, 1071, 1039, 986, 1036, 1024, 1021, 1025, 1050, 1049, 1006, 1040, 1037, 997, 1027, 1003, 1023, 1009, 1019, 1024, 954, 1038, 1030, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1001, 1045, 1014, 1058, 1007, 1015, 1086, 1028, 1011, 1008, 1023, 973, 1040, 1017, 971, 988, 1081, 1040, 1033, 1014, 1027, 1052, 1015, 1029, 991, 945, 1000, 1032, 1019, 1077, 1062, 1021, 1021, 1005, 1009, 1029, 989, 1045, 1006, 1090, 1028, 1008, 1002, 1054, 1008, 1085, 1054, 1032, 1065, 1023, 1035, 1051, 1032, 1087, 1033, 1006, 1003, 958, 1006, 999, 991, 1030, 980, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1019, 1021, 1044, 996, 992, 1059, 987, 992, 1015, 996, 1096, 1046, 1047, 1005, 1018, 1002, 1022, 1017, 1012, 997, 1030, 1042, 1038, 1032, 1038, 1028, 973, 983, 1007, 985, 1002, 1047, 1045, 978, 1077, 1039, 1039, 1011, 1034, 1000, 1073, 983, 1022, 1025, 1059, 1058, 1060, 997, 1030, 1025, 1033, 1011, 1043, 1007, 996, 1020, 1028, 1018, 1033, 1036, 1040, 1083, 1038, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1019, 1031, 1026, 1036, 1069, 1011, 1025, 1062, 1006, 1026, 1051, 1052, 1026, 1046, 1044, 1052, 1029, 1014, 976, 1104, 1025, 1060, 1038, 979, 1027, 1057, 1005, 1008, 1070, 1035, 1023, 1041, 1039, 1050, 1005, 1016, 1044, 1027, 984, 1036, 1021, 980, 1068, 973, 1009, 1008, 1022, 1048, 1019, 958, 1022, 1020, 990, 987, 1003, 1026, 1051, 985, 1014, 1040, 968, 1019, 999, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1051, 995, 1034, 1035, 998, 1053, 1030, 979, 1044, 984, 1009, 1002, 995, 1033, 1021, 998, 1073, 1027, 1079, 1009, 1062, 1046, 1027, 1061, 1077, 1052, 1013, 1013, 1038, 1018, 997, 1022, 983, 1011, 965, 1013, 1035, 1033, 1041, 1006, 1062, 1022, 1090, 1015, 1024, 1030, 1008, 990, 940, 990, 1033, 1003, 1040, 1002, 1035, 1028, 1031, 1015, 997, 1059, 1056, 1042, 1060, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1027, 1007, 1013, 1019, 1010, 997, 1049, 972, 0, 0, 0, 0, 0, 0, 0, 0, 982, 995, 1021, 1036, 1022, 1011, 1008, 1097, 7, 3, 0, 0, 0, 0, 0, 0, 1056, 1023, 1057, 1017, 1028, 970, 983, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1020, 976, 1106, 1003, 1014, 972, 1046, 16, 12, 0, 0, 0, 0, 0, 0, 995, 984, 1028, 1000, 1057, 1064, 1034, 1042, 11, 5, 0, 0, 0, 0, 0, 0, 1024, 1011, 986, 1010, 995, 1011, 1008, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1013, 1053, 1055, 1015, 1053, 997, 992, 32, 0, 0, 0, 0, 0, 0, 0, 1062, 1043, 1063, 1017, 1075, 1033, 1034, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8094, 8182, 8195, 8209, 8220, 8080, 8196, 8360] avg=8192.0000 max=8360 max_rank=7 min=8080 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1033, 994, 986, 1079, 1045, 1030, 1058, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1001, 1076, 1079, 1015, 1042, 994, 1086, 8, 3, 0, 0, 0, 0, 0, 0, 987, 990, 1020, 1022, 1046, 1026, 1041, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1021, 1037, 1032, 1038, 966, 976, 1018, 17, 13, 0, 0, 0, 0, 0, 0, 1001, 994, 1076, 989, 1029, 1032, 1002, 998, 11, 5, 0, 0, 0, 0, 0, 0, 1022, 1060, 1020, 1014, 988, 1030, 1042, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1043, 1039, 1054, 972, 989, 1009, 1019, 34, 0, 0, 0, 0, 0, 0, 0, 1020, 1025, 1031, 1010, 974, 1067, 1040, 984, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8339, 8172, 8170, 8137, 8181, 8136, 8151] avg=8192.0000 max=8339 max_rank=1 min=8136 min_rank=6 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32605 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1019, 1021, 1044, 976, 992, 1059, 987, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 996, 1096, 1046, 1047, 1005, 1018, 1002, 8, 3, 0, 0, 0, 0, 0, 0, 1022, 1017, 1012, 997, 1030, 996, 1038, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1028, 973, 983, 1007, 985, 1002, 1047, 15, 13, 0, 0, 0, 0, 0, 0, 1045, 978, 1077, 1039, 1039, 1011, 1034, 1000, 11, 5, 0, 0, 0, 0, 0, 0, 1073, 983, 1003, 1025, 1059, 1058, 1060, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1025, 1033, 1011, 1043, 1007, 996, 1020, 33, 0, 0, 0, 0, 0, 0, 0, 1028, 1018, 1033, 1036, 1040, 1080, 1038, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8236, 8144, 8091, 8239, 8258, 8198, 8280] avg=8192.0000 max=8280 max_rank=7 min=8090 min_rank=0 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32975 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1026, 1014, 1056, 961, 1076, 992, 1011, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 993, 1007, 1007, 998, 1007, 1040, 1020, 8, 3, 0, 0, 0, 0, 0, 0, 994, 988, 996, 996, 1064, 1018, 1020, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1007, 1022, 1047, 945, 1023, 982, 1056, 15, 13, 0, 0, 0, 0, 0, 0, 1036, 1028, 1001, 1052, 1020, 1019, 1017, 987, 11, 5, 0, 0, 0, 0, 0, 0, 1032, 1013, 1027, 1036, 1027, 1061, 1067, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1048, 1030, 1021, 1044, 1027, 1018, 1012, 34, 0, 0, 0, 0, 0, 0, 0, 1015, 1028, 1072, 1051, 1041, 1038, 1027, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8087, 8123, 8174, 8176, 8255, 8235, 8265] avg=8192.0000 max=8265 max_rank=7 min=8087 min_rank=1 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32931 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1001, 1045, 1014, 1037, 1007, 1015, 1086, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1008, 1023, 973, 1040, 1017, 971, 988, 7, 3, 0, 0, 0, 0, 0, 0, 1081, 1040, 1033, 1014, 1027, 1006, 1015, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 991, 945, 1000, 1032, 1019, 1077, 1062, 1021, 16, 13, 0, 0, 0, 0, 0, 0, 1021, 1005, 1009, 1029, 989, 1045, 1006, 1090, 11, 5, 0, 0, 0, 0, 0, 0, 1028, 1008, 984, 1054, 1008, 1085, 1054, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1023, 1035, 1051, 1032, 1087, 1033, 1006, 33, 0, 0, 0, 0, 0, 0, 0, 1003, 958, 1006, 999, 991, 1027, 980, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8041, 8245, 8176, 8210, 8253, 8365, 8013] avg=8192.0000 max=8365 max_rank=6 min=8013 min_rank=7 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1012, 1001, 1029, 1033, 1038, 1037, 1031, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1008, 1046, 1000, 1049, 1042, 1045, 1023, 8, 3, 0, 0, 0, 0, 0, 0, 1043, 1052, 995, 1043, 1021, 958, 1018, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 988, 1002, 1020, 1020, 994, 1067, 1003, 16, 12, 0, 0, 0, 0, 0, 0, 1007, 1021, 1059, 1038, 1020, 985, 1033, 1044, 11, 5, 0, 0, 0, 0, 0, 0, 1004, 1071, 1020, 986, 1036, 1024, 1021, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1049, 1006, 1040, 1037, 997, 1027, 1003, 31, 0, 0, 0, 0, 0, 0, 0, 1023, 1009, 1019, 1024, 954, 1035, 1030, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8232, 8148, 8155, 8223, 8187, 8240, 8107] avg=8192.0000 max=8244 max_rank=0 min=8107 min_rank=7 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1019, 1031, 1026, 1007, 1069, 1011, 1025, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1026, 1051, 1052, 1026, 1046, 1044, 1052, 12, 7, 0, 0, 0, 0, 0, 0, 1029, 1014, 976, 1104, 1025, 1006, 1038, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1057, 1005, 1008, 1070, 1035, 1023, 1041, 19, 17, 0, 0, 0, 0, 0, 0, 1039, 1050, 1005, 1016, 1044, 1027, 984, 1036, 16, 10, 0, 0, 0, 0, 0, 0, 1021, 980, 1040, 973, 1009, 1008, 1022, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 958, 1022, 1020, 990, 987, 1003, 1026, 37, 0, 0, 0, 0, 0, 0, 0, 1051, 985, 1014, 1040, 968, 1012, 999, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8322, 8171, 8302, 8227, 8101, 8062, 8101] avg=8192.0000 max=8322 max_rank=1 min=8062 min_rank=6 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1051, 995, 1034, 1014, 998, 1053, 1030, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 984, 1009, 1002, 995, 1033, 1021, 998, 8, 3, 0, 0, 0, 0, 0, 0, 1073, 1027, 1079, 1009, 1062, 1000, 1027, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 1052, 1013, 1013, 1038, 1018, 997, 1022, 16, 13, 0, 0, 0, 0, 0, 0, 983, 1011, 965, 1013, 1035, 1033, 1041, 1006, 12, 5, 0, 0, 0, 0, 0, 0, 1062, 1022, 1070, 1015, 1024, 1030, 1008, 990, 0, 0, 0, 0, 0, 0, 0, 0, 940, 990, 1033, 1003, 1040, 1002, 1035, 1028, 33, 0, 0, 0, 0, 0, 0, 0, 1031, 1015, 997, 1059, 1056, 1039, 1060, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8097, 8338, 8259, 8104, 8221, 8104, 8259] avg=8192.0000 max=8338 max_rank=2 min=8097 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8064, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8064, 8448, 8320, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8448, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8448, 8192, 8064, 8320, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1059, 1043, 986, 1010, 1002, 1019, 1008, 1061, 1090, 1021, 1024, 1013, 982, 1034, 1029, 966, 1052, 1057, 1038, 1019, 1093, 1003, 1071, 1067, 986, 1004, 1065, 966, 1032, 1030, 1061, 1022, 1051, 1039, 998, 1057, 1043, 1014, 1007, 1004, 1015, 999, 1094, 948, 1040, 967, 985, 1009, 1029, 1012, 1048, 1065, 1032, 995, 1027, 958, 1013, 984, 1048, 1031, 1041, 1044, 1014, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[979, 1049, 1021, 1007, 1016, 1032, 987, 1016, 1033, 1036, 1062, 1027, 1046, 1058, 1000, 1094, 984, 982, 956, 971, 1035, 1010, 994, 1012, 1052, 1050, 1021, 1005, 1036, 955, 1046, 1055, 1031, 1034, 1074, 1058, 1013, 1002, 992, 988, 1068, 1046, 1018, 1024, 1046, 1020, 1063, 1028, 1037, 1049, 1050, 1023, 1063, 1012, 1023, 1022, 1024, 996, 986, 1013, 995, 1039, 1036, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[984, 1022, 1029, 1037, 1029, 978, 1044, 1025, 1005, 1019, 1014, 996, 1045, 997, 1031, 1058, 974, 1071, 949, 1065, 1002, 980, 1047, 1061, 1014, 1034, 1027, 1020, 1017, 1048, 988, 1068, 1029, 1043, 1012, 1069, 1060, 1041, 1038, 1047, 1034, 1001, 1043, 1043, 1013, 946, 1001, 998, 947, 1053, 1035, 990, 1010, 1020, 987, 1036, 1093, 1046, 1057, 1050, 1036, 1034, 1030, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[993, 1021, 1022, 984, 1063, 1078, 1070, 1061, 1001, 1036, 1022, 1064, 995, 1066, 1036, 1041, 1008, 1052, 1061, 1052, 1041, 1049, 968, 1018, 1032, 1035, 1017, 1060, 1050, 1000, 965, 1071, 1013, 1038, 998, 1025, 1026, 1052, 961, 998, 929, 990, 1018, 985, 1012, 985, 1027, 1008, 975, 1027, 986, 1048, 1070, 1051, 1043, 1052, 1030, 1010, 1033, 1017, 1023, 1019, 1012, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1037, 1030, 1009, 1025, 1056, 969, 1027, 1064, 1045, 977, 1021, 1024, 1033, 1037, 999, 1004, 1051, 1031, 1020, 999, 1040, 990, 1041, 1004, 1039, 1056, 1009, 1072, 1033, 1086, 999, 1023, 986, 1065, 1029, 1009, 1055, 1024, 980, 1035, 1031, 1027, 961, 995, 972, 979, 1023, 1006, 1002, 1064, 1063, 1090, 1017, 982, 1007, 1073, 963, 1029, 1017, 1025, 1057, 1053, 1019, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[994, 1010, 1002, 1051, 1003, 985, 1016, 1032, 1035, 1086, 993, 999, 1002, 1063, 1019, 1041, 1008, 1050, 1018, 974, 1027, 1012, 1032, 955, 1027, 1010, 1080, 1034, 1015, 1000, 1035, 1009, 1040, 993, 1033, 964, 1052, 1041, 1006, 1025, 998, 1084, 1069, 1008, 1055, 1003, 1082, 1011, 1027, 980, 1029, 1045, 1000, 1045, 1041, 1031, 1034, 1024, 1031, 995, 1051, 1012, 1091, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[986, 1020, 1127, 1028, 1066, 955, 1029, 1058, 968, 1051, 1005, 1030, 1015, 1048, 1000, 1077, 995, 1030, 1033, 1064, 1026, 1043, 1043, 985, 1015, 1033, 1018, 1019, 1065, 1008, 1026, 1008, 1056, 1016, 1033, 1058, 1016, 1012, 1001, 998, 1007, 1022, 1012, 1072, 1033, 1004, 1066, 1010, 1004, 1040, 1026, 1032, 1044, 1025, 991, 1044, 1006, 977, 991, 1062, 1015, 973, 1020, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1052, 1011, 1047, 1062, 1015, 1008, 1002, 1009, 1027, 1001, 1022, 1052, 947, 978, 973, 1067, 1012, 1028, 1042, 1048, 1013, 1027, 991, 1049, 1031, 1076, 970, 1026, 980, 975, 1069, 1023, 1062, 994, 978, 1043, 1046, 1011, 1060, 1032, 966, 1038, 1085, 1082, 1045, 1047, 1075, 989, 1033, 1010, 995, 1012, 1019, 1037, 1064, 994, 1054, 1003, 1069, 992, 1061, 1023, 979, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[993, 1021, 1022, 984, 1063, 1078, 1070, 1061, 1, 0, 0, 0, 0, 0, 0, 0, 1001, 1036, 1022, 1064, 995, 1066, 1036, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1052, 1061, 1052, 1041, 1049, 968, 1018, 8, 5, 0, 0, 0, 0, 0, 0, 1032, 1015, 1017, 1060, 1050, 1000, 965, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1038, 998, 1025, 1018, 1052, 961, 998, 0, 0, 0, 0, 0, 0, 0, 0, 929, 990, 1018, 985, 1012, 985, 1027, 1008, 20, 13, 0, 0, 0, 0, 0, 0, 975, 1027, 986, 1035, 1070, 1051, 1043, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1010, 1033, 1017, 1022, 1019, 1012, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8256, 8262, 8210, 8103, 7987, 8239, 8186] avg=8192.0000 max=8293 max_rank=0 min=7987 min_rank=5 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1059, 1043, 986, 1010, 1002, 1019, 1008, 1061, 2, 0, 0, 0, 0, 0, 0, 0, 1090, 1021, 1024, 1013, 982, 1034, 1029, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1057, 1038, 1019, 1093, 1003, 1071, 1067, 8, 5, 0, 0, 0, 0, 0, 0, 986, 985, 1065, 966, 1032, 1030, 1061, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1039, 998, 1057, 1035, 1014, 1007, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 999, 1094, 948, 1040, 967, 985, 1009, 19, 13, 0, 0, 0, 0, 0, 0, 1029, 1012, 1048, 1052, 1032, 995, 1027, 958, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 984, 1048, 1031, 1039, 1044, 1014, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8154, 8413, 8147, 8205, 8089, 8153, 8185] avg=8192.0000 max=8413 max_rank=2 min=8089 min_rank=5 imbalance_ratio=1.026978 max_load_reduction=57123 max_load_reduction_pct=0.871628 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1037, 1030, 1009, 1025, 1056, 969, 1027, 1064, 2, 0, 0, 0, 0, 0, 0, 0, 1045, 977, 1021, 1024, 1033, 1037, 999, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1031, 1020, 999, 1040, 990, 1041, 1004, 8, 5, 0, 0, 0, 0, 0, 0, 1039, 1036, 1009, 1072, 1033, 1086, 999, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1065, 1029, 1009, 1047, 1024, 980, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1027, 961, 995, 972, 979, 1023, 1006, 20, 13, 0, 0, 0, 0, 0, 0, 1002, 1064, 1063, 1077, 1017, 982, 1007, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1029, 1017, 1025, 1055, 1053, 1019, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8135, 8189, 8297, 8175, 8027, 8285, 8209] avg=8192.0000 max=8297 max_rank=3 min=8027 min_rank=5 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[979, 1049, 1021, 1007, 1016, 1032, 987, 1016, 1, 0, 0, 0, 0, 0, 0, 0, 1033, 1036, 1062, 1027, 1046, 1058, 1000, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 984, 982, 956, 971, 1035, 1010, 994, 1012, 8, 5, 0, 0, 0, 0, 0, 0, 1052, 1030, 1021, 1005, 1036, 955, 1046, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1034, 1074, 1058, 1005, 1002, 992, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1046, 1018, 1024, 1046, 1020, 1063, 1028, 20, 12, 0, 0, 0, 0, 0, 0, 1037, 1049, 1050, 1011, 1063, 1012, 1023, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 996, 986, 1013, 994, 1039, 1036, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8351, 7957, 8200, 8184, 8345, 8267, 8124] avg=8192.0000 max=8351 max_rank=1 min=7957 min_rank=2 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[994, 1010, 1002, 1051, 1003, 985, 1016, 1032, 2, 0, 0, 0, 0, 0, 0, 0, 1035, 1086, 993, 999, 1002, 1063, 1019, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1050, 1018, 974, 1027, 1012, 1032, 955, 8, 5, 0, 0, 0, 0, 0, 0, 1027, 990, 1080, 1034, 1015, 1000, 1035, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 993, 1033, 964, 1044, 1041, 1006, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1084, 1069, 1008, 1055, 1003, 1082, 1011, 20, 13, 0, 0, 0, 0, 0, 0, 1027, 980, 1029, 1032, 1000, 1045, 1041, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1024, 1031, 995, 1049, 1012, 1091, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8233, 8089, 8190, 8146, 8343, 8185, 8255] avg=8192.0000 max=8343 max_rank=5 min=8089 min_rank=2 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[984, 1022, 1029, 1037, 1029, 978, 1044, 1025, 2, 0, 0, 0, 0, 0, 0, 0, 1005, 1019, 1014, 996, 1045, 997, 1031, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1071, 949, 1065, 1002, 980, 1047, 1061, 8, 5, 0, 0, 0, 0, 0, 0, 1014, 1014, 1027, 1020, 1017, 1048, 988, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1043, 1012, 1069, 1052, 1041, 1038, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1001, 1043, 1043, 1013, 946, 1001, 998, 20, 12, 0, 0, 0, 0, 0, 0, 947, 1053, 1035, 978, 1010, 1020, 987, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 1046, 1057, 1050, 1034, 1034, 1030, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8160, 8162, 8196, 8331, 8111, 8066, 8360] avg=8192.0000 max=8360 max_rank=7 min=8066 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32868 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[986, 1020, 1127, 1028, 1066, 955, 1029, 1058, 4, 0, 0, 0, 0, 0, 0, 0, 968, 1051, 1005, 1030, 1015, 1048, 1000, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1030, 1033, 1064, 1026, 1043, 1043, 985, 10, 9, 3, 0, 0, 0, 0, 0, 1015, 1008, 1018, 1019, 1065, 1008, 1026, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1016, 1033, 1058, 1000, 1012, 1001, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1022, 1012, 1072, 1033, 1004, 1066, 1010, 25, 17, 6, 0, 0, 0, 0, 0, 1004, 1040, 1026, 1015, 1044, 1025, 991, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 977, 991, 1062, 1008, 973, 1020, 996, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8185, 8241, 8167, 8174, 8274, 8189, 8033] avg=8192.0000 max=8274 max_rank=5 min=8033 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1052, 1011, 1047, 1062, 1015, 1008, 1002, 1009, 2, 0, 0, 0, 0, 0, 0, 0, 1027, 1001, 1022, 1052, 947, 978, 973, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1028, 1042, 1048, 1013, 1027, 991, 1049, 8, 5, 0, 0, 0, 0, 0, 0, 1031, 1055, 970, 1026, 980, 975, 1069, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 994, 978, 1043, 1038, 1011, 1060, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1038, 1085, 1082, 1045, 1047, 1075, 989, 21, 12, 0, 0, 0, 0, 0, 0, 1033, 1010, 995, 1000, 1019, 1037, 1064, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1003, 1069, 992, 1059, 1023, 979, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8062, 8223, 8129, 8218, 8360, 8152, 8184] avg=8192.0000 max=8360 max_rank=5 min=8062 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8064, 8192, 8448, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8320, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8320, 8192, 8320, 8064, 8448, 8064, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1026, 1057, 1050, 1023, 1007, 1024, 1043, 1030, 981, 996, 1023, 1043, 974, 1030, 1054, 1024, 1015, 1021, 1037, 1024, 1013, 1063, 1008, 1067, 1010, 1004, 959, 1005, 1053, 995, 1064, 969, 990, 983, 1019, 1007, 1022, 1044, 1007, 1025, 1008, 1041, 991, 1028, 1032, 1019, 1032, 1088, 1059, 1083, 1003, 1022, 1002, 1050, 1018, 1007, 1034, 1081, 999, 989, 1045, 1012, 1060, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1043, 1020, 991, 1070, 1063, 1055, 985, 1048, 995, 1035, 1013, 1044, 977, 1031, 1036, 993, 1078, 1051, 1062, 1014, 1020, 1045, 1025, 1049, 998, 995, 1021, 988, 1004, 1008, 1007, 1055, 1033, 1017, 1027, 1014, 1015, 1048, 1010, 1039, 982, 1083, 1017, 1057, 1004, 1022, 1016, 1021, 1020, 1034, 1059, 993, 983, 1040, 1034, 997, 1002, 1016, 1037, 960, 1033, 1067, 989, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1017, 1086, 1007, 1031, 962, 1043, 1022, 968, 992, 1039, 1013, 1056, 1079, 1037, 1013, 1029, 1033, 1006, 1089, 1086, 998, 1070, 1019, 1060, 966, 1022, 1004, 948, 1025, 1052, 995, 1037, 995, 1026, 1023, 992, 1009, 1041, 957, 983, 1063, 974, 1029, 1075, 1090, 1053, 1000, 996, 989, 1029, 1052, 1037, 1020, 1009, 1079, 1001, 1052, 1038, 1012, 1044, 1042, 1014, 1013, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1007, 1009, 1018, 1042, 1033, 1072, 990, 1007, 1037, 1044, 1044, 1012, 1019, 1044, 1007, 1008, 991, 1032, 1055, 971, 1032, 1025, 1020, 1046, 1014, 1051, 991, 1039, 1026, 1040, 1010, 998, 977, 1012, 1044, 1064, 1064, 1075, 1014, 1008, 1007, 997, 1027, 1017, 990, 1017, 1050, 1041, 1007, 1025, 1036, 954, 1035, 1031, 1028, 1023, 1057, 1037, 986, 1028, 1024, 1039, 1054, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1031, 995, 1014, 1054, 987, 1008, 1022, 1010, 1016, 1047, 1019, 1049, 1090, 1003, 1047, 1056, 1027, 1015, 1035, 1048, 968, 984, 1032, 973, 976, 1052, 1039, 1067, 979, 1027, 997, 1015, 1072, 985, 1041, 996, 1048, 1023, 1025, 1051, 1004, 1051, 1015, 1037, 1040, 1017, 1032, 1049, 1085, 1046, 1007, 1006, 995, 1031, 998, 998, 1026, 1028, 1077, 971, 1037, 980, 1015, 1068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[997, 1014, 1004, 1008, 1032, 1018, 1035, 1008, 982, 1049, 1028, 1063, 1023, 1013, 1038, 993, 1045, 1018, 996, 1064, 1039, 998, 999, 1041, 1048, 1047, 1032, 1030, 988, 1081, 1060, 1019, 1002, 991, 1002, 1030, 1002, 1022, 1009, 1012, 1000, 1045, 1018, 1020, 1006, 1018, 1083, 998, 1026, 1011, 989, 1034, 1023, 1062, 1079, 1072, 1051, 1042, 1014, 982, 1039, 988, 1034, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1062, 1007, 976, 998, 999, 980, 958, 1068, 1052, 1074, 1051, 1019, 1061, 1046, 1029, 1012, 991, 1002, 1054, 1053, 1018, 1003, 996, 1038, 1051, 1028, 1019, 1072, 1054, 1049, 1019, 1009, 1027, 996, 968, 1031, 1026, 1098, 1011, 1012, 1005, 995, 985, 1036, 1020, 1027, 985, 997, 1043, 1026, 1040, 1026, 1016, 978, 1035, 1077, 986, 1084, 1016, 997, 1057, 1025, 1039, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[961, 957, 1031, 1053, 1027, 1061, 1009, 1061, 1021, 1035, 1078, 1021, 1045, 1026, 1006, 1077, 1056, 1016, 1026, 1027, 958, 1034, 999, 1042, 980, 1010, 1001, 1049, 990, 986, 1010, 999, 1055, 998, 1058, 1009, 1016, 1017, 1031, 976, 998, 1032, 986, 990, 1008, 1073, 1051, 1015, 1043, 1014, 1043, 1049, 1005, 975, 1075, 1014, 1059, 1041, 1058, 1075, 1023, 1054, 1029, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 1057, 1050, 1023, 1007, 1024, 1043, 1030, 16, 13, 0, 0, 0, 0, 0, 0, 981, 954, 1023, 1043, 974, 1030, 1054, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1021, 1015, 1024, 1013, 1063, 1008, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1004, 959, 1005, 1053, 995, 1064, 969, 42, 5, 0, 0, 0, 0, 0, 0, 990, 983, 1019, 1007, 1022, 1044, 1007, 1025, 33, 6, 0, 0, 0, 0, 0, 0, 1008, 1041, 991, 1028, 1032, 1019, 1032, 1088, 4, 0, 0, 0, 0, 0, 0, 0, 1059, 1083, 1003, 1022, 1002, 1050, 1001, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1043, 999, 989, 1045, 1012, 1060, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8289, 8083, 8226, 8106, 8136, 8243, 8227, 8226] avg=8192.0000 max=8289 max_rank=0 min=8083 min_rank=1 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1017, 1086, 1007, 1031, 962, 1043, 1022, 968, 17, 14, 0, 0, 0, 0, 0, 0, 992, 995, 1013, 1056, 1079, 1037, 1013, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1006, 1066, 1086, 998, 1070, 1019, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1022, 1004, 948, 1025, 1052, 995, 1037, 44, 5, 0, 0, 0, 0, 0, 0, 995, 1026, 1023, 992, 1009, 1041, 957, 983, 32, 6, 0, 0, 0, 0, 0, 0, 1063, 974, 1029, 1075, 1090, 1053, 1000, 996, 4, 0, 0, 0, 0, 0, 0, 0, 989, 1029, 1052, 1037, 1020, 1009, 1061, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1001, 1012, 1044, 1042, 1014, 1013, 995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8214, 8338, 8098, 8064, 8284, 8198, 8173] avg=8192.0000 max=8338 max_rank=2 min=8064 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1007, 1009, 1018, 1042, 1033, 1072, 990, 1007, 16, 13, 0, 0, 0, 0, 0, 0, 1037, 1000, 1044, 1012, 1019, 1044, 1007, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1032, 1033, 971, 1032, 1025, 1020, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1051, 991, 1039, 1026, 1040, 1010, 998, 44, 5, 0, 0, 0, 0, 0, 0, 977, 1012, 1044, 1064, 1064, 1075, 1014, 1008, 32, 6, 0, 0, 0, 0, 0, 0, 1007, 997, 1027, 1017, 990, 1017, 1050, 1041, 4, 0, 0, 0, 0, 0, 0, 0, 1007, 1025, 1036, 954, 1035, 1031, 1011, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1000, 986, 1028, 1024, 1039, 1054, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8171, 8150, 8218, 8296, 8150, 8122, 8222] avg=8192.0000 max=8296 max_rank=4 min=8122 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1043, 1020, 991, 1070, 1063, 1055, 985, 1048, 16, 13, 0, 0, 0, 0, 0, 0, 995, 991, 1013, 1044, 977, 1031, 1036, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1051, 1040, 1014, 1020, 1045, 1025, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 998, 995, 1021, 988, 1004, 1008, 1007, 1055, 44, 5, 0, 0, 0, 0, 0, 0, 1033, 1017, 1027, 1014, 1015, 1048, 1010, 1039, 31, 6, 0, 0, 0, 0, 0, 0, 982, 1083, 1017, 1057, 1004, 1022, 1016, 1021, 4, 0, 0, 0, 0, 0, 0, 0, 1020, 1034, 1059, 993, 983, 1040, 1017, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 980, 1037, 960, 1033, 1067, 989, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8304, 8080, 8322, 8125, 8240, 8206, 8143, 8116] avg=8192.0000 max=8322 max_rank=2 min=8080 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[997, 1014, 1004, 1008, 1032, 1018, 1035, 1008, 15, 14, 0, 0, 0, 0, 0, 0, 982, 1005, 1028, 1063, 1023, 1013, 1038, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1018, 975, 1064, 1039, 998, 999, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1047, 1032, 1030, 988, 1081, 1060, 1019, 44, 5, 0, 0, 0, 0, 0, 0, 1002, 991, 1002, 1030, 1002, 1022, 1009, 1012, 32, 6, 0, 0, 0, 0, 0, 0, 1000, 1045, 1018, 1020, 1006, 1018, 1083, 998, 4, 0, 0, 0, 0, 0, 0, 0, 1026, 1011, 989, 1034, 1023, 1062, 1061, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1005, 1014, 982, 1039, 988, 1034, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8145, 8179, 8354, 8108, 8192, 8278, 8135] avg=8192.0000 max=8354 max_rank=3 min=8108 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1031, 995, 1014, 1054, 987, 1008, 1022, 1010, 16, 13, 0, 0, 0, 0, 0, 0, 1016, 1003, 1019, 1049, 1090, 1003, 1047, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1015, 1013, 1048, 968, 984, 1032, 973, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1052, 1039, 1067, 979, 1027, 997, 1015, 44, 5, 0, 0, 0, 0, 0, 0, 1072, 985, 1041, 996, 1048, 1023, 1025, 1051, 31, 6, 0, 0, 0, 0, 0, 0, 1004, 1051, 1015, 1037, 1040, 1017, 1032, 1049, 3, 0, 0, 0, 0, 0, 0, 0, 1085, 1046, 1007, 1006, 995, 1031, 982, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 992, 1077, 971, 1037, 980, 1015, 1068, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8283, 8060, 8201, 8278, 8248, 8150, 8166] avg=8192.0000 max=8283 max_rank=1 min=8060 min_rank=2 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[961, 957, 1031, 1053, 1027, 1061, 1009, 1061, 19, 18, 0, 0, 0, 0, 0, 0, 1021, 987, 1078, 1021, 1045, 1026, 1006, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1016, 996, 1027, 958, 1034, 999, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1010, 1001, 1049, 990, 986, 1010, 999, 48, 10, 0, 0, 0, 0, 0, 0, 1055, 998, 1058, 1009, 1016, 1017, 1031, 976, 35, 11, 0, 0, 0, 0, 0, 0, 998, 1032, 986, 990, 1008, 1073, 1051, 1015, 6, 0, 0, 0, 0, 0, 0, 0, 1043, 1014, 1043, 1049, 1005, 975, 1051, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 996, 1058, 1075, 1023, 1054, 1029, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8261, 8128, 8083, 8206, 8159, 8194, 8308] avg=8192.0000 max=8308 max_rank=7 min=8083 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1062, 1007, 976, 998, 999, 980, 958, 1068, 16, 13, 0, 0, 0, 0, 0, 0, 1052, 1029, 1051, 1019, 1061, 1046, 1029, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1002, 1032, 1053, 1018, 1003, 996, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1028, 1019, 1072, 1054, 1049, 1019, 1009, 45, 5, 0, 0, 0, 0, 0, 0, 1027, 996, 968, 1031, 1026, 1098, 1011, 1012, 33, 6, 0, 0, 0, 0, 0, 0, 1005, 995, 985, 1036, 1020, 1027, 985, 997, 4, 0, 0, 0, 0, 0, 0, 0, 1043, 1026, 1040, 1026, 1016, 978, 1018, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1046, 1016, 997, 1057, 1025, 1039, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8077, 8299, 8133, 8351, 8208, 8054, 8224, 8190] avg=8192.0000 max=8351 max_rank=3 min=8054 min_rank=5 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8192, 8192, 8320, 8448, 8064, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8064, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8192, 8320, 8192, 8320, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8064, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8064, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1012, 1054, 1023, 1031, 1037, 1021, 1011, 1011, 1044, 1015, 990, 1012, 933, 1012, 1033, 987, 966, 1023, 1076, 1021, 1052, 1015, 1047, 997, 987, 996, 1025, 1081, 1032, 1049, 1058, 1028, 992, 1037, 1045, 1031, 1032, 1008, 1034, 981, 1023, 1024, 1041, 1008, 985, 1036, 1059, 1035, 978, 1007, 1058, 1044, 1075, 1069, 1008, 1009, 1005, 1013, 989, 1036, 1087, 1064, 1008, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1030, 1017, 1053, 1070, 1052, 1020, 974, 1024, 1009, 1036, 984, 1043, 975, 987, 1013, 980, 1033, 997, 1058, 1029, 998, 1034, 1047, 1044, 1003, 1072, 999, 1001, 966, 996, 1012, 1040, 999, 1048, 924, 1025, 1020, 1052, 1043, 1055, 1078, 1020, 1031, 1002, 978, 1051, 1002, 1030, 1019, 1015, 1049, 1038, 1047, 1047, 1004, 1064, 1050, 1035, 1060, 989, 1018, 1002, 1087, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1027, 1006, 1036, 1021, 1025, 1066, 1021, 1013, 1071, 1064, 1047, 1012, 1017, 1029, 1014, 1046, 1028, 1069, 1057, 1016, 989, 1004, 1004, 1027, 1018, 1028, 1019, 1036, 1032, 989, 1016, 988, 963, 1081, 984, 1014, 1026, 976, 1043, 1028, 1038, 1038, 1032, 1012, 1008, 1011, 1043, 1025, 997, 1027, 997, 1042, 988, 1032, 1005, 1059, 1005, 1046, 1044, 979, 1063, 1070, 1014, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1077, 1048, 1021, 1043, 1044, 1014, 1023, 984, 1013, 1061, 1008, 1042, 1032, 1017, 1032, 1028, 985, 1014, 1085, 1049, 1065, 1009, 1027, 1110, 1040, 1021, 998, 996, 1046, 996, 989, 1042, 1010, 1043, 1006, 970, 1018, 1039, 1047, 1007, 1026, 1043, 1015, 1031, 996, 1086, 1013, 992, 1003, 1022, 1045, 1029, 1029, 949, 1011, 964, 1052, 1047, 1010, 1052, 1051, 1013, 979, 979] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1054, 990, 1022, 1014, 1061, 1000, 1039, 992, 1039, 1041, 1034, 1064, 1063, 1015, 1034, 996, 1077, 1036, 1005, 990, 993, 985, 1090, 1033, 1022, 1053, 1060, 983, 1021, 1044, 1006, 1017, 1030, 1044, 1055, 1013, 1008, 964, 1043, 1069, 998, 994, 1006, 1011, 1032, 1028, 1020, 1080, 1009, 966, 1032, 993, 1039, 1030, 1007, 1019, 1019, 994, 994, 1029, 1024, 1037, 1028, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1037, 1015, 1080, 994, 1090, 1012, 986, 1042, 1017, 1086, 991, 1012, 1044, 1027, 988, 987, 1037, 1046, 1014, 1037, 1002, 1033, 1075, 976, 1076, 1030, 1023, 1008, 1045, 1007, 1002, 1015, 1062, 1026, 1033, 1008, 1022, 1069, 1044, 1036, 1011, 999, 1027, 1001, 1010, 1008, 1020, 1036, 957, 1019, 1012, 1014, 1014, 1024, 1039, 1097, 1013, 1015, 1055, 1024, 975, 1009, 1003, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1038, 1012, 1033, 1074, 1018, 1041, 1019, 1047, 1047, 1007, 993, 1032, 1037, 1067, 1046, 1016, 1041, 991, 1043, 1012, 1008, 1011, 1014, 1046, 976, 1011, 1014, 1004, 1026, 1012, 1019, 1002, 1001, 1035, 1018, 1038, 1019, 987, 1002, 1066, 998, 997, 1014, 1033, 1015, 1051, 1022, 1029, 987, 1026, 1062, 1013, 1069, 1047, 1009, 1016, 1022, 1010, 1021, 1037, 1049, 1030, 999, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1011, 1006, 1049, 1063, 1019, 1022, 1025, 1038, 1019, 991, 1011, 1018, 1021, 1088, 1004, 1011, 1021, 1012, 1014, 1024, 1000, 1026, 1013, 1022, 1030, 971, 1022, 1028, 1011, 1020, 1034, 1044, 1052, 1005, 1027, 1039, 1064, 1008, 1034, 1012, 1037, 1032, 991, 1022, 1040, 1023, 1031, 1052, 994, 1026, 1039, 987, 988, 1069, 1034, 1016, 1098, 1024, 1014, 1021, 1019, 1015, 1043, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1012, 1054, 1023, 1031, 999, 1021, 1011, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1015, 990, 1012, 933, 1012, 1033, 987, 9, 3, 0, 0, 0, 0, 0, 0, 966, 1023, 1056, 1021, 1052, 1015, 1047, 997, 0, 0, 0, 0, 0, 0, 0, 0, 987, 996, 1025, 1081, 1032, 1049, 1058, 1028, 37, 0, 0, 0, 0, 0, 0, 0, 992, 1037, 1045, 1031, 1032, 1008, 1034, 981, 2, 0, 0, 0, 0, 0, 0, 0, 1023, 1024, 1041, 1008, 985, 1036, 1059, 1035, 7, 0, 0, 0, 0, 0, 0, 0, 978, 1007, 1058, 1044, 1075, 1069, 1008, 1009, 17, 1, 0, 0, 0, 0, 0, 0, 1005, 1013, 989, 1036, 1069, 1064, 1008, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8038, 8177, 8293, 8162, 8218, 8266, 8220] avg=8192.0000 max=8293 max_rank=3 min=8038 min_rank=1 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32866 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1030, 1017, 1053, 1070, 1014, 1020, 974, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1036, 984, 1043, 975, 987, 1013, 980, 9, 3, 0, 0, 0, 0, 0, 0, 1033, 997, 1038, 1029, 998, 1034, 1047, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1072, 999, 1001, 966, 996, 1012, 1040, 37, 0, 0, 0, 0, 0, 0, 0, 999, 1048, 924, 1025, 1020, 1052, 1043, 1055, 2, 0, 0, 0, 0, 0, 0, 0, 1078, 1020, 1031, 1002, 978, 1051, 1002, 1030, 6, 0, 0, 0, 0, 0, 0, 0, 1019, 1015, 1049, 1038, 1047, 1047, 1004, 1064, 17, 1, 0, 0, 0, 0, 0, 0, 1050, 1035, 1060, 989, 1001, 1002, 1087, 1058, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8039, 8220, 8126, 8168, 8198, 8301, 8282] avg=8192.0000 max=8301 max_rank=6 min=8039 min_rank=1 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32949 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1054, 990, 1022, 1014, 1022, 1000, 1039, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1041, 1034, 1064, 1063, 1015, 1034, 996, 9, 3, 0, 0, 0, 0, 0, 0, 1077, 1036, 986, 990, 993, 985, 1090, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1053, 1060, 983, 1021, 1044, 1006, 1017, 38, 0, 0, 0, 0, 0, 0, 0, 1030, 1044, 1055, 1013, 1008, 964, 1043, 1069, 2, 0, 0, 0, 0, 0, 0, 0, 998, 994, 1006, 1011, 1032, 1028, 1020, 1080, 6, 0, 0, 0, 0, 0, 0, 0, 1009, 966, 1032, 993, 1039, 1030, 1007, 1019, 16, 1, 0, 0, 0, 0, 0, 0, 1019, 994, 994, 1029, 1007, 1037, 1028, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8298, 8190, 8244, 8228, 8175, 8112, 8156] avg=8192.0000 max=8298 max_rank=1 min=8112 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1077, 1048, 1021, 1043, 1006, 1014, 1023, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1061, 1008, 1042, 1032, 1017, 1032, 1028, 9, 3, 0, 0, 0, 0, 0, 0, 985, 1014, 1065, 1049, 1065, 1009, 1027, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1021, 998, 996, 1046, 996, 989, 1042, 37, 0, 0, 0, 0, 0, 0, 0, 1010, 1043, 1006, 970, 1018, 1039, 1047, 1007, 2, 0, 0, 0, 0, 0, 0, 0, 1026, 1043, 1015, 1031, 996, 1086, 1013, 992, 7, 0, 0, 0, 0, 0, 0, 0, 1003, 1022, 1045, 1029, 1029, 949, 1011, 964, 17, 1, 0, 0, 0, 0, 0, 0, 1052, 1047, 1010, 1052, 1033, 1013, 979, 979, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8245, 8324, 8165, 8142, 8209, 8070, 8165] avg=8192.0000 max=8324 max_rank=2 min=8070 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32586 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1027, 1006, 1036, 1021, 988, 1066, 1021, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1064, 1047, 1012, 1017, 1029, 1014, 1046, 9, 3, 0, 0, 0, 0, 0, 0, 1028, 1069, 1037, 1016, 989, 1004, 1004, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1028, 1019, 1036, 1032, 989, 1016, 988, 36, 0, 0, 0, 0, 0, 0, 0, 963, 1081, 984, 1014, 1026, 976, 1043, 1028, 2, 0, 0, 0, 0, 0, 0, 0, 1038, 1038, 1032, 1012, 1008, 1011, 1043, 1025, 7, 0, 0, 0, 0, 0, 0, 0, 997, 1027, 997, 1042, 988, 1032, 1005, 1059, 17, 1, 0, 0, 0, 0, 0, 0, 1005, 1046, 1044, 979, 1045, 1070, 1014, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8312, 8174, 8162, 8117, 8214, 8165, 8214] avg=8192.0000 max=8312 max_rank=1 min=8117 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1037, 1015, 1080, 994, 1050, 1012, 986, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1086, 991, 1012, 1044, 1027, 988, 987, 8, 3, 0, 0, 0, 0, 0, 0, 1037, 1046, 995, 1037, 1002, 1033, 1075, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1030, 1023, 1008, 1045, 1007, 1002, 1015, 39, 0, 0, 0, 0, 0, 0, 0, 1062, 1026, 1033, 1008, 1022, 1069, 1044, 1036, 2, 0, 0, 0, 0, 0, 0, 0, 1011, 999, 1027, 1001, 1010, 1008, 1020, 1036, 6, 0, 0, 0, 0, 0, 0, 0, 957, 1019, 1012, 1014, 1014, 1024, 1039, 1097, 16, 1, 0, 0, 0, 0, 0, 0, 1013, 1015, 1055, 1024, 959, 1009, 1003, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8163, 8201, 8245, 8302, 8118, 8193, 8098] avg=8192.0000 max=8302 max_rank=4 min=8098 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1038, 1012, 1033, 1074, 969, 1041, 1019, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1007, 993, 1032, 1037, 1067, 1046, 1016, 14, 7, 0, 0, 0, 0, 0, 0, 1041, 991, 1014, 1012, 1008, 1011, 1014, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1011, 1014, 1004, 1026, 1012, 1019, 1002, 41, 0, 0, 0, 0, 0, 0, 0, 1001, 1035, 1018, 1038, 1019, 987, 1002, 1066, 8, 0, 0, 0, 0, 0, 0, 0, 998, 997, 1014, 1033, 1015, 1051, 1022, 1029, 11, 0, 0, 0, 0, 0, 0, 0, 987, 1026, 1062, 1013, 1069, 1047, 1009, 1016, 22, 8, 0, 0, 0, 0, 0, 0, 1022, 1010, 1021, 1037, 1016, 1030, 999, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8266, 8137, 8105, 8174, 8170, 8259, 8192] avg=8192.0000 max=8266 max_rank=1 min=8105 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1011, 1006, 1049, 1063, 982, 1022, 1025, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 991, 1011, 1018, 1021, 1088, 1004, 1011, 9, 3, 0, 0, 0, 0, 0, 0, 1021, 1012, 995, 1024, 1000, 1026, 1013, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 971, 1022, 1028, 1011, 1020, 1034, 1044, 36, 0, 0, 0, 0, 0, 0, 0, 1052, 1005, 1027, 1039, 1064, 1008, 1034, 1012, 2, 0, 0, 0, 0, 0, 0, 0, 1037, 1032, 991, 1022, 1040, 1023, 1031, 1052, 6, 0, 0, 0, 0, 0, 0, 0, 994, 1026, 1039, 987, 988, 1069, 1034, 1016, 16, 1, 0, 0, 0, 0, 0, 0, 1098, 1024, 1014, 1021, 1002, 1015, 1043, 992, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8175, 8113, 8196, 8243, 8234, 8170, 8209] avg=8192.0000 max=8243 max_rank=4 min=8113 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8064, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8320, 8192, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1079, 1083, 1028, 1010, 1023, 1021, 991, 1053, 1063, 1028, 988, 1010, 1033, 1032, 1029, 987, 1073, 1034, 1024, 997, 1008, 996, 1018, 1038, 1023, 979, 1027, 1014, 1062, 1031, 1058, 974, 1041, 1042, 1040, 1017, 988, 1014, 999, 1023, 1074, 1057, 999, 1058, 1035, 926, 1030, 1030, 1047, 1013, 1045, 1022, 1010, 980, 998, 1048, 977, 994, 984, 1056, 1062, 1044, 1019, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1018, 1032, 1053, 1015, 1069, 1037, 1069, 1028, 1077, 1029, 1020, 1022, 991, 1016, 1043, 1052, 1015, 1043, 983, 999, 1011, 1026, 1030, 988, 967, 997, 982, 995, 1062, 1010, 1021, 993, 1002, 1056, 1044, 1033, 1026, 1048, 1004, 1001, 989, 1049, 1048, 1017, 1088, 1024, 995, 1061, 1007, 1044, 1000, 1027, 1069, 1014, 1034, 1024, 1026, 972, 1039, 992, 1027, 1040, 1021, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1045, 1025, 1029, 1037, 1030, 993, 1028, 1043, 1030, 1003, 1075, 998, 1036, 1001, 994, 1018, 1026, 982, 1037, 1041, 1004, 1046, 985, 1088, 1037, 973, 1071, 1050, 1049, 1010, 1008, 1031, 992, 1005, 998, 1053, 1025, 1016, 1027, 1028, 989, 1035, 1061, 974, 1062, 1018, 1030, 1059, 1029, 992, 1013, 1095, 974, 1052, 1047, 1032, 1016, 1009, 1029, 1023, 1020, 1011, 979, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1035, 1005, 994, 1032, 983, 1058, 1035, 990, 1020, 1013, 1027, 1068, 992, 1042, 1051, 1021, 1013, 996, 1088, 985, 1026, 1026, 1000, 1051, 1056, 994, 983, 1085, 1050, 1071, 1034, 1008, 1020, 1003, 1030, 1040, 975, 1018, 1041, 993, 983, 1016, 1039, 1000, 1021, 1049, 982, 1048, 1024, 1048, 1013, 1036, 1040, 1056, 1008, 1017, 1001, 1034, 1056, 1000, 1040, 1001, 1046, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1032, 1036, 1056, 1040, 1044, 1015, 978, 993, 998, 1037, 1033, 1038, 1018, 997, 982, 1065, 1027, 1026, 988, 1040, 1024, 1014, 1036, 1079, 1008, 1037, 1015, 1050, 991, 1016, 1024, 1011, 1054, 1003, 973, 978, 1060, 1014, 974, 1011, 1010, 1053, 1035, 1029, 1019, 998, 1037, 1050, 1064, 1000, 998, 1026, 1000, 1091, 1127, 989, 1000, 999, 1066, 1034, 1020, 1020, 1013, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1081, 1048, 1007, 1080, 998, 1051, 989, 1013, 962, 962, 1027, 1027, 1041, 1016, 1048, 1056, 1038, 1026, 1017, 1033, 1036, 994, 1034, 1098, 1084, 978, 1022, 1049, 980, 1068, 1022, 1005, 1001, 1034, 1046, 1063, 995, 1070, 967, 1059, 1000, 974, 1013, 1043, 994, 1003, 1046, 1022, 1030, 1024, 1040, 1000, 1020, 1053, 1053, 1049, 973, 1018, 1039, 1046, 981, 1006, 986, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1043, 1083, 969, 1031, 1093, 1076, 1043, 1050, 1024, 1005, 1021, 961, 1042, 1014, 972, 990, 993, 987, 975, 976, 1024, 1029, 988, 1068, 1065, 1026, 1025, 1008, 1052, 1012, 1026, 1046, 1040, 1052, 963, 1037, 1021, 990, 1061, 1004, 1029, 1059, 1017, 998, 1007, 1019, 1047, 1026, 1043, 1003, 1034, 1063, 1030, 1061, 1050, 1021, 1034, 1037, 1012, 986, 1014, 992, 1039, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1031, 1013, 1015, 996, 1038, 1009, 996, 1054, 1075, 1051, 1069, 1000, 984, 976, 1030, 1051, 986, 995, 1074, 1003, 983, 1030, 1027, 940, 998, 1017, 1043, 1046, 1019, 1010, 1038, 1014, 1037, 1056, 1047, 1071, 1003, 1012, 1079, 980, 1062, 1041, 1012, 1046, 985, 1047, 984, 1033, 994, 1048, 998, 1030, 1032, 1022, 1012, 1026, 1042, 1030, 1044, 998, 996, 1027, 1059, 1072] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1025, 1048, 1007, 1080, 998, 1051, 989, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 962, 962, 1027, 1027, 1041, 1016, 1048, 1056, 19, 0, 0, 0, 0, 0, 0, 0, 1038, 1026, 1017, 1033, 1036, 994, 1034, 1098, 35, 0, 0, 0, 0, 0, 0, 0, 1084, 978, 1022, 1049, 980, 1068, 1022, 1005, 3, 0, 0, 0, 0, 0, 0, 0, 1001, 1034, 1046, 1063, 995, 1070, 967, 1059, 14, 2, 0, 0, 0, 0, 0, 0, 1000, 974, 1013, 1043, 994, 1003, 1046, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1024, 1040, 1000, 1020, 1053, 1010, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1018, 1039, 1046, 981, 1006, 986, 998, 21, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8158, 8311, 8211, 8251, 8090, 8226, 8078] avg=8192.0000 max=8311 max_rank=2 min=8078 min_rank=7 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[966, 1032, 1053, 1015, 1069, 1037, 1069, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 1029, 1020, 1022, 991, 1016, 1043, 1052, 19, 0, 0, 0, 0, 0, 0, 0, 1015, 1043, 983, 999, 1011, 1026, 1030, 988, 32, 0, 0, 0, 0, 0, 0, 0, 967, 997, 982, 995, 1062, 1010, 1021, 993, 3, 0, 0, 0, 0, 0, 0, 0, 1002, 1056, 1044, 1033, 1026, 1048, 1004, 1001, 14, 2, 0, 0, 0, 0, 0, 0, 989, 1049, 1048, 1017, 1088, 1024, 995, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1044, 1000, 1027, 1069, 1014, 991, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 972, 1039, 992, 1027, 1040, 1021, 1022, 20, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8269, 8127, 8030, 8230, 8266, 8176, 8169] avg=8192.0000 max=8269 max_rank=0 min=8030 min_rank=3 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[979, 1036, 1056, 1040, 1044, 1015, 978, 993, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1037, 1033, 1038, 1018, 997, 982, 1065, 20, 0, 0, 0, 0, 0, 0, 0, 1027, 1026, 988, 1040, 1024, 1014, 1036, 1079, 33, 0, 0, 0, 0, 0, 0, 0, 1008, 1037, 1015, 1050, 991, 1016, 1024, 1011, 3, 0, 0, 0, 0, 0, 0, 0, 1054, 1003, 973, 978, 1060, 1014, 974, 1011, 15, 2, 0, 0, 0, 0, 0, 0, 1010, 1053, 1035, 1029, 1019, 998, 1037, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1000, 998, 1026, 1000, 1091, 1082, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 999, 1066, 1034, 1020, 1020, 1013, 1043, 20, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8188, 8267, 8155, 8084, 8226, 8250, 8225] avg=8192.0000 max=8267 max_rank=2 min=8084 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1024, 1083, 1028, 1010, 1023, 1021, 991, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1028, 988, 1010, 1033, 1032, 1029, 987, 18, 0, 0, 0, 0, 0, 0, 0, 1073, 1034, 1024, 997, 1008, 996, 1018, 1038, 34, 0, 0, 0, 0, 0, 0, 0, 1023, 979, 1027, 1014, 1062, 1031, 1058, 974, 3, 0, 0, 0, 0, 0, 0, 0, 1041, 1042, 1040, 1017, 988, 1014, 999, 1023, 14, 2, 0, 0, 0, 0, 0, 0, 1074, 1057, 999, 1058, 1035, 926, 1030, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1013, 1045, 1022, 1010, 980, 957, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 977, 994, 984, 1056, 1062, 1044, 1019, 1050, 21, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8188, 8222, 8171, 8180, 8204, 8122, 8216] avg=8192.0000 max=8233 max_rank=0 min=8122 min_rank=6 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[992, 1025, 1029, 1037, 1030, 993, 1028, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1003, 1075, 998, 1036, 1001, 994, 1018, 19, 0, 0, 0, 0, 0, 0, 0, 1026, 982, 1037, 1041, 1004, 1046, 985, 1088, 33, 0, 0, 0, 0, 0, 0, 0, 1037, 973, 1071, 1050, 1049, 1010, 1008, 1031, 3, 0, 0, 0, 0, 0, 0, 0, 992, 1005, 998, 1053, 1025, 1016, 1027, 1028, 14, 2, 0, 0, 0, 0, 0, 0, 989, 1035, 1061, 974, 1062, 1018, 1030, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 992, 1013, 1095, 974, 1052, 1004, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1009, 1029, 1023, 1020, 1011, 979, 1020, 20, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8174, 8242, 8232, 8160, 8223, 8191, 8137] avg=8192.0000 max=8242 max_rank=2 min=8137 min_rank=7 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[974, 1005, 994, 1032, 983, 1058, 1035, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1013, 1027, 1068, 992, 1042, 1051, 1021, 23, 0, 0, 0, 0, 0, 0, 0, 1013, 996, 1088, 985, 1026, 1026, 1000, 1051, 38, 0, 0, 0, 0, 0, 0, 0, 1056, 994, 983, 1085, 1050, 1071, 1034, 1008, 5, 0, 0, 0, 0, 0, 0, 0, 1020, 1003, 1030, 1040, 975, 1018, 1041, 993, 18, 8, 0, 0, 0, 0, 0, 0, 983, 1016, 1039, 1000, 1021, 1049, 982, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1048, 1013, 1036, 1040, 1056, 954, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1034, 1056, 1000, 1040, 1001, 1046, 1026, 23, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8257, 8223, 8286, 8146, 8125, 8188, 8240] avg=8192.0000 max=8286 max_rank=3 min=8071 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[990, 1083, 969, 1031, 1093, 1076, 1043, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1005, 1021, 961, 1042, 1014, 972, 990, 19, 0, 0, 0, 0, 0, 0, 0, 993, 987, 975, 976, 1024, 1029, 988, 1068, 33, 0, 0, 0, 0, 0, 0, 0, 1065, 1026, 1025, 1008, 1052, 1012, 1026, 1046, 3, 0, 0, 0, 0, 0, 0, 0, 1040, 1052, 963, 1037, 1021, 990, 1061, 1004, 14, 2, 0, 0, 0, 0, 0, 0, 1029, 1059, 1017, 998, 1007, 1019, 1047, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1003, 1034, 1063, 1030, 1061, 1007, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1037, 1012, 986, 1014, 992, 1039, 1030, 20, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8335, 8048, 8073, 8263, 8184, 8197, 8262, 8174] avg=8192.0000 max=8335 max_rank=0 min=8048 min_rank=1 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[978, 1013, 1015, 996, 1038, 1009, 996, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1051, 1069, 1000, 984, 976, 1030, 1051, 18, 0, 0, 0, 0, 0, 0, 0, 986, 995, 1074, 1003, 983, 1030, 1027, 940, 33, 0, 0, 0, 0, 0, 0, 0, 998, 1017, 1043, 1046, 1019, 1010, 1038, 1014, 3, 0, 0, 0, 0, 0, 0, 0, 1037, 1056, 1047, 1071, 1003, 1012, 1079, 980, 14, 2, 0, 0, 0, 0, 0, 0, 1062, 1041, 1012, 1046, 985, 1047, 984, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1048, 998, 1030, 1032, 1022, 971, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1030, 1044, 998, 996, 1027, 1059, 1072, 20, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8254, 8071, 8188, 8301, 8205, 8121, 8297] avg=8192.0000 max=8301 max_rank=4 min=8071 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32924 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8448, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7936, 8448, 8192, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8448, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8192, 8320, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1008, 1000, 1059, 1049, 997, 1037, 1013, 1018, 1014, 1034, 1036, 954, 1012, 1040, 1020, 989, 1053, 1051, 1049, 998, 1037, 1025, 1042, 998, 1012, 1036, 1030, 1033, 1032, 1062, 1034, 1045, 981, 958, 1018, 1011, 1011, 1051, 1066, 1036, 1060, 972, 1060, 956, 1045, 1022, 1030, 1052, 1015, 1004, 986, 1016, 1053, 1019, 998, 1075, 987, 1018, 1016, 1042, 1097, 1007, 1045, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1059, 1024, 1069, 1005, 1025, 1009, 1082, 1046, 1023, 1030, 1041, 1055, 1035, 1016, 976, 1045, 1009, 1022, 1066, 994, 985, 1050, 990, 1045, 1020, 948, 1034, 1065, 1022, 1002, 1018, 1057, 995, 999, 1079, 998, 1043, 1030, 1034, 1025, 1065, 1025, 959, 1022, 1056, 925, 1003, 983, 1006, 1003, 1004, 1063, 1023, 958, 1069, 1027, 1081, 994, 998, 1016, 1072, 1037, 1033, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1037, 1033, 1003, 1095, 1022, 1054, 1030, 1018, 1032, 1026, 1038, 1020, 993, 1015, 1018, 1047, 1032, 1040, 1001, 1032, 1001, 1022, 1067, 1011, 1045, 1024, 1039, 996, 1043, 1037, 999, 1017, 1034, 1038, 1011, 1015, 987, 1055, 1005, 1015, 1048, 994, 1058, 1037, 964, 1044, 1022, 973, 986, 1007, 1032, 1038, 1034, 988, 1024, 1013, 1037, 1053, 1039, 1033, 1009, 980, 1015, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[982, 1090, 1016, 1007, 999, 1002, 1071, 975, 1045, 1071, 1030, 1007, 1013, 1039, 997, 1029, 1052, 1019, 1051, 1051, 1040, 993, 1029, 1011, 1017, 1004, 1055, 1012, 1035, 1031, 1004, 1050, 1046, 1032, 1055, 1000, 1052, 974, 1018, 1009, 1003, 1031, 1062, 993, 999, 994, 995, 1029, 993, 1078, 1057, 1052, 1002, 1100, 963, 970, 1015, 1049, 1019, 1051, 1035, 995, 998, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1071, 1044, 1045, 1025, 997, 1054, 994, 1036, 1066, 1043, 1027, 998, 984, 972, 1040, 965, 1045, 1084, 1021, 1029, 1027, 1016, 1079, 1032, 1005, 1046, 973, 1052, 1002, 1021, 1017, 985, 1022, 973, 960, 1021, 1114, 1033, 1038, 1076, 1038, 1012, 1024, 1028, 985, 998, 1004, 1012, 1000, 961, 1080, 991, 1024, 1065, 1046, 1055, 1030, 1015, 1018, 1023, 1046, 1045, 1006, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1024, 1017, 1036, 977, 990, 984, 1034, 1046, 1006, 1014, 1028, 1023, 1013, 1059, 1032, 1009, 1032, 1019, 983, 1024, 1036, 1017, 1048, 1005, 1081, 1020, 1023, 997, 1039, 1036, 990, 1004, 1048, 1059, 1035, 1036, 1002, 1023, 1026, 1049, 1035, 977, 1017, 983, 1026, 1034, 1004, 1047, 1033, 1011, 1091, 1024, 998, 1033, 1008, 1056, 1079, 1041, 1022, 1001, 1032, 1037, 1013, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1046, 1017, 1032, 1024, 1037, 990, 973, 990, 1038, 1058, 1072, 1061, 1083, 1002, 1019, 1064, 1020, 997, 970, 1022, 1027, 1001, 1038, 1040, 1022, 1005, 1011, 1071, 1001, 993, 1014, 1036, 996, 1046, 1056, 1006, 1030, 1003, 1040, 1013, 1037, 1018, 1021, 1011, 1016, 981, 1104, 1042, 1024, 984, 1034, 1020, 963, 1039, 1071, 999, 1053, 1058, 1007, 1019, 990, 1021, 1024, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[977, 1010, 1046, 1021, 1012, 1016, 1060, 990, 1028, 993, 1032, 1014, 1005, 953, 1030, 1056, 1008, 1020, 1071, 1029, 1001, 1072, 1135, 991, 1014, 1076, 1018, 1005, 1029, 1011, 1022, 1046, 1040, 969, 1055, 987, 1020, 984, 1032, 1015, 1000, 982, 1023, 1053, 1041, 962, 1000, 1021, 1005, 1054, 1038, 1062, 1039, 1014, 1053, 1015, 1014, 1056, 1003, 1010, 1049, 1049, 1020, 1080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1008, 1000, 1058, 1049, 997, 1037, 1013, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1034, 1036, 954, 1012, 1040, 1020, 989, 1, 0, 0, 0, 0, 0, 0, 0, 1053, 1051, 1049, 998, 1037, 1025, 1010, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1036, 1030, 1033, 1032, 1062, 1034, 1045, 1, 0, 0, 0, 0, 0, 0, 0, 981, 958, 1018, 1011, 1011, 1051, 1066, 1036, 5, 0, 0, 0, 0, 0, 0, 0, 1060, 972, 1060, 956, 1045, 1022, 1030, 1052, 39, 27, 0, 0, 0, 0, 0, 0, 1015, 1004, 985, 1016, 1053, 1019, 998, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1018, 1016, 1042, 1058, 1007, 1045, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8100, 8221, 8285, 8137, 8263, 8165, 8185] avg=8192.0000 max=8285 max_rank=3 min=8100 min_rank=1 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1024, 1017, 1035, 977, 990, 984, 1034, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1014, 1028, 1023, 1013, 1059, 1032, 1009, 1, 0, 0, 0, 0, 0, 0, 0, 1032, 1019, 983, 1024, 1036, 1017, 1016, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1020, 1023, 997, 1039, 1036, 990, 1004, 1, 0, 0, 0, 0, 0, 0, 0, 1048, 1059, 1035, 1036, 1002, 1023, 1026, 1049, 5, 0, 0, 0, 0, 0, 0, 0, 1035, 977, 1017, 983, 1026, 1034, 1004, 1047, 36, 27, 0, 0, 0, 0, 0, 0, 1033, 1011, 1090, 1024, 998, 1033, 1008, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1041, 1022, 1001, 996, 1037, 1013, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8185, 8132, 8191, 8283, 8186, 8253, 8199] avg=8192.0000 max=8283 max_rank=4 min=8107 min_rank=0 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1059, 1024, 1068, 1005, 1025, 1009, 1082, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1030, 1041, 1055, 1035, 1016, 976, 1045, 1, 0, 0, 0, 0, 0, 0, 0, 1009, 1022, 1066, 994, 985, 1050, 959, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 948, 1034, 1065, 1022, 1002, 1018, 1057, 1, 0, 0, 0, 0, 0, 0, 0, 995, 999, 1079, 998, 1043, 1030, 1034, 1025, 5, 0, 0, 0, 0, 0, 0, 0, 1065, 1025, 959, 1022, 1056, 925, 1003, 983, 38, 26, 0, 0, 0, 0, 0, 0, 1006, 1003, 1003, 1063, 1023, 958, 1069, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 994, 998, 1016, 1034, 1037, 1033, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8318, 8222, 8130, 8167, 8208, 8102, 8152, 8237] avg=8192.0000 max=8318 max_rank=0 min=8102 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[982, 1090, 1015, 1007, 999, 1002, 1071, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1071, 1030, 1007, 1013, 1039, 997, 1029, 1, 0, 0, 0, 0, 0, 0, 0, 1052, 1019, 1051, 1051, 1040, 993, 997, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1004, 1055, 1012, 1035, 1031, 1004, 1050, 1, 0, 0, 0, 0, 0, 0, 0, 1046, 1032, 1055, 1000, 1052, 974, 1018, 1009, 5, 0, 0, 0, 0, 0, 0, 0, 1003, 1031, 1062, 993, 999, 994, 995, 1029, 36, 27, 0, 0, 0, 0, 0, 0, 993, 1078, 1056, 1052, 1002, 1100, 963, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1049, 1019, 1051, 999, 995, 998, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8232, 8214, 8209, 8191, 8169, 8214, 8166] avg=8192.0000 max=8232 max_rank=1 min=8141 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1037, 1033, 1002, 1095, 1022, 1054, 1030, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1026, 1038, 1020, 993, 1015, 1018, 1047, 1, 0, 0, 0, 0, 0, 0, 0, 1032, 1040, 1001, 1032, 1001, 1022, 1034, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1024, 1039, 996, 1043, 1037, 999, 1017, 1, 0, 0, 0, 0, 0, 0, 0, 1034, 1038, 1011, 1015, 987, 1055, 1005, 1015, 5, 0, 0, 0, 0, 0, 0, 0, 1048, 994, 1058, 1037, 964, 1044, 1022, 973, 35, 28, 0, 0, 0, 0, 0, 0, 986, 1007, 1031, 1038, 1034, 988, 1024, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1053, 1039, 1033, 974, 980, 1015, 1061, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8190, 8173, 8201, 8165, 8203, 8121, 8192] avg=8192.0000 max=8291 max_rank=0 min=8121 min_rank=6 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1071, 1044, 1044, 1025, 997, 1054, 994, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1043, 1027, 998, 984, 972, 1040, 965, 1, 0, 0, 0, 0, 0, 0, 0, 1045, 1084, 1021, 1029, 1027, 1016, 1045, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1046, 973, 1052, 1002, 1021, 1017, 985, 1, 0, 0, 0, 0, 0, 0, 0, 1022, 973, 960, 1021, 1114, 1033, 1038, 1076, 6, 0, 0, 0, 0, 0, 0, 0, 1038, 1012, 1024, 1028, 985, 998, 1004, 1012, 37, 28, 0, 0, 0, 0, 0, 0, 1000, 961, 1079, 991, 1024, 1065, 1046, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1015, 1018, 1023, 1009, 1045, 1006, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8096, 8299, 8102, 8243, 8166, 8221, 8144] avg=8192.0000 max=8299 max_rank=2 min=8096 min_rank=1 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1046, 1017, 1026, 1024, 1037, 990, 973, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1058, 1072, 1061, 1083, 1002, 1019, 1064, 2, 0, 0, 0, 0, 0, 0, 0, 1020, 997, 970, 1022, 1027, 1001, 998, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1005, 1011, 1071, 1001, 993, 1014, 1036, 5, 1, 0, 0, 0, 0, 0, 0, 996, 1046, 1056, 1006, 1030, 1003, 1040, 1013, 10, 1, 0, 0, 0, 0, 0, 0, 1037, 1018, 1021, 1011, 1016, 981, 1104, 1042, 39, 30, 0, 0, 0, 0, 0, 0, 1024, 984, 1031, 1020, 963, 1039, 1071, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1058, 1007, 1019, 951, 1021, 1024, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8399, 8075, 8159, 8201, 8299, 8131, 8169] avg=8192.0000 max=8399 max_rank=1 min=8075 min_rank=2 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[977, 1010, 1045, 1021, 1012, 1016, 1060, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 993, 1032, 1014, 1005, 953, 1030, 1056, 1, 0, 0, 0, 0, 0, 0, 0, 1008, 1020, 1071, 1029, 1001, 1072, 1100, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1076, 1018, 1005, 1029, 1011, 1022, 1046, 1, 0, 0, 0, 0, 0, 0, 0, 1040, 969, 1055, 987, 1020, 984, 1032, 1015, 6, 0, 0, 0, 0, 0, 0, 0, 1000, 982, 1023, 1053, 1041, 962, 1000, 1021, 37, 29, 0, 0, 0, 0, 0, 0, 1005, 1054, 1037, 1062, 1039, 1014, 1053, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1056, 1003, 1010, 1012, 1049, 1020, 1080, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8112, 8292, 8222, 8108, 8148, 8279, 8244] avg=8192.0000 max=8292 max_rank=2 min=8108 min_rank=4 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 8064, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8192, 8192, 8064, 8192, 8192, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1031, 1034, 1038, 1049, 1057, 988, 1043, 1024, 1059, 1019, 1062, 1068, 1062, 1004, 1022, 1026, 1097, 996, 1011, 964, 1034, 1029, 993, 1025, 1039, 1027, 1063, 1027, 999, 1031, 1047, 1011, 1035, 994, 998, 1004, 1012, 970, 1001, 1030, 1067, 1018, 1010, 1025, 998, 1041, 1010, 1024, 1027, 1063, 1045, 997, 983, 1059, 999, 975, 1033, 1021, 1025, 988, 1039, 1038, 1012, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1018, 1009, 1034, 1019, 1020, 1039, 1050, 1050, 1012, 1042, 993, 1096, 1003, 1036, 1003, 1045, 1044, 1001, 1012, 1036, 1016, 1040, 1040, 1022, 980, 1023, 1029, 993, 1054, 1057, 1105, 999, 1042, 963, 1025, 1030, 1005, 999, 1037, 1070, 1007, 1049, 980, 1015, 1003, 1036, 1002, 1026, 1024, 990, 1008, 1040, 1004, 986, 1035, 1052, 1029, 1024, 984, 1054, 1026, 1021, 1043, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[973, 984, 1002, 993, 1011, 998, 1017, 1016, 1027, 988, 1059, 1037, 1039, 1021, 1019, 1037, 1085, 1045, 1024, 1032, 1058, 1002, 1006, 1023, 1062, 1029, 1026, 1018, 1022, 1029, 988, 1033, 999, 1015, 988, 1050, 1001, 1035, 996, 1025, 1044, 1040, 1021, 1031, 1022, 1028, 1033, 1023, 1003, 1036, 1025, 1011, 1079, 1017, 998, 1023, 1037, 1030, 1051, 1049, 1074, 1009, 1031, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1009, 943, 1034, 970, 1053, 1036, 1047, 1004, 1006, 1019, 1015, 1034, 980, 1027, 988, 1075, 1035, 1025, 1007, 1056, 994, 1045, 1013, 1039, 1034, 1038, 1019, 1043, 1008, 1021, 1078, 1030, 1008, 1039, 1067, 978, 1007, 1050, 1000, 1039, 1050, 1011, 997, 1055, 998, 998, 1008, 1016, 1045, 1003, 1022, 1038, 1044, 1035, 1049, 1064, 1047, 1028, 1005, 1033, 988, 1059, 1004, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1000, 1050, 978, 1015, 1076, 1062, 1042, 1014, 1048, 969, 1007, 1080, 1040, 1055, 1005, 1041, 956, 1036, 1026, 1039, 1003, 1014, 975, 1100, 1045, 1079, 1002, 1052, 1023, 960, 1063, 1011, 984, 1011, 982, 1064, 1007, 1045, 1063, 1077, 1010, 1005, 1008, 1066, 999, 1013, 1020, 1005, 963, 972, 1000, 980, 1031, 1013, 1069, 1032, 1047, 1014, 1081, 1030, 1023, 1033, 1055, 968] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[981, 1004, 1012, 1000, 1038, 1046, 1075, 1044, 1036, 991, 981, 1042, 1012, 1015, 978, 1022, 980, 1081, 1059, 1049, 1001, 1031, 1034, 1076, 987, 1021, 1002, 995, 1025, 1033, 1048, 1025, 1018, 1020, 1072, 1017, 1038, 986, 1030, 1019, 994, 1010, 991, 1060, 1012, 1076, 995, 1001, 1017, 960, 1069, 984, 1029, 1021, 1070, 1035, 1004, 1011, 1043, 1004, 1067, 1112, 982, 1065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[996, 1033, 1056, 1038, 1011, 1008, 989, 1011, 1015, 1034, 1062, 1022, 1014, 1018, 1014, 1035, 1023, 1074, 991, 1043, 1062, 1017, 993, 986, 1007, 998, 999, 1023, 1003, 1035, 1002, 1046, 1070, 1067, 1025, 997, 1042, 1043, 999, 972, 1038, 1003, 1029, 1008, 1036, 1034, 1021, 1040, 1044, 981, 1106, 952, 1011, 1051, 1032, 1026, 1023, 1035, 1042, 1042, 1026, 1063, 1010, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1025, 1007, 1020, 1037, 1053, 1009, 1021, 1041, 989, 1049, 1033, 1095, 991, 1088, 1015, 970, 1005, 1007, 986, 999, 1107, 979, 992, 1085, 1028, 1021, 1036, 1028, 1021, 1060, 981, 1023, 968, 1036, 1049, 998, 1016, 1025, 992, 1086, 1028, 1032, 1018, 1016, 1036, 1040, 1039, 1051, 1047, 1047, 1040, 1013, 1023, 1001, 1021, 1027, 1026, 1042, 1048, 1010, 974, 974, 975, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[973, 984, 1002, 993, 1011, 998, 1017, 1016, 15, 2, 0, 0, 0, 0, 0, 0, 1027, 988, 1059, 1019, 1039, 1021, 1019, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 1045, 1024, 1032, 1058, 1002, 1006, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1029, 1026, 1018, 1022, 1029, 976, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1015, 988, 1050, 1001, 1035, 996, 1025, 21, 3, 0, 0, 0, 0, 0, 0, 1044, 1040, 1021, 1031, 1022, 1028, 1033, 1023, 11, 2, 0, 0, 0, 0, 0, 0, 1003, 1036, 1025, 1011, 1079, 1017, 998, 1023, 10, 0, 0, 0, 0, 0, 0, 0, 1037, 1030, 1051, 1049, 1074, 988, 1031, 1009, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8011, 8209, 8262, 8195, 8133, 8255, 8202, 8269] avg=8192.0000 max=8269 max_rank=7 min=8011 min_rank=0 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1018, 1009, 1034, 1019, 1020, 1039, 1050, 1050, 16, 2, 0, 0, 0, 0, 0, 0, 1012, 1042, 993, 1077, 1003, 1036, 1003, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1001, 1012, 1036, 1016, 1040, 1040, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1023, 1029, 993, 1054, 1057, 1092, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 963, 1025, 1030, 1005, 999, 1037, 1070, 22, 3, 0, 0, 0, 0, 0, 0, 1007, 1049, 980, 1015, 1003, 1036, 1002, 1026, 11, 2, 0, 0, 0, 0, 0, 0, 1024, 990, 1008, 1040, 1004, 986, 1035, 1052, 11, 0, 0, 0, 0, 0, 0, 0, 1029, 1024, 984, 1054, 1026, 999, 1043, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8211, 8198, 8227, 8196, 8131, 8150, 8166] avg=8192.0000 max=8257 max_rank=0 min=8131 min_rank=5 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1009, 943, 1034, 970, 1053, 1036, 1047, 1004, 15, 2, 0, 0, 0, 0, 0, 0, 1006, 1019, 1015, 1016, 980, 1027, 988, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1025, 1007, 1056, 994, 1045, 1013, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1038, 1019, 1043, 1008, 1021, 1065, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1039, 1067, 978, 1007, 1050, 1000, 1039, 22, 3, 0, 0, 0, 0, 0, 0, 1050, 1011, 997, 1055, 998, 998, 1008, 1016, 12, 2, 0, 0, 0, 0, 0, 0, 1045, 1003, 1022, 1038, 1044, 1035, 1049, 1064, 11, 0, 0, 0, 0, 0, 0, 0, 1047, 1028, 1005, 1033, 988, 1037, 1004, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8126, 8200, 8258, 8213, 8147, 8311, 8168] avg=8192.0000 max=8311 max_rank=6 min=8113 min_rank=0 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1031, 1034, 1038, 1049, 1057, 988, 1043, 1024, 16, 2, 0, 0, 0, 0, 0, 0, 1059, 1019, 1062, 1049, 1062, 1004, 1022, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 996, 1011, 964, 1034, 1029, 993, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1027, 1063, 1027, 999, 1031, 1034, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 994, 998, 1004, 1012, 970, 1001, 1030, 22, 3, 0, 0, 0, 0, 0, 0, 1067, 1018, 1010, 1025, 998, 1041, 1010, 1024, 12, 2, 0, 0, 0, 0, 0, 0, 1027, 1063, 1045, 997, 983, 1059, 999, 975, 11, 0, 0, 0, 0, 0, 0, 0, 1033, 1021, 1025, 988, 1039, 1016, 1012, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8303, 8135, 8231, 8069, 8207, 8159, 8150] avg=8192.0000 max=8303 max_rank=1 min=8069 min_rank=4 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1000, 1050, 978, 1015, 1076, 1062, 1042, 1014, 16, 3, 0, 0, 0, 0, 0, 0, 1048, 969, 1007, 1061, 1040, 1055, 1005, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1036, 1026, 1039, 1003, 1014, 975, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1079, 1002, 1052, 1023, 960, 1050, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1011, 982, 1064, 1007, 1045, 1063, 1077, 22, 3, 0, 0, 0, 0, 0, 0, 1010, 1005, 1008, 1066, 999, 1013, 1020, 1005, 12, 2, 0, 0, 0, 0, 0, 0, 963, 972, 1000, 980, 1031, 1013, 1069, 1032, 11, 0, 0, 0, 0, 0, 0, 0, 1047, 1014, 1081, 1030, 1023, 1011, 1055, 968, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8226, 8134, 8222, 8258, 8140, 8071, 8229] avg=8192.0000 max=8258 max_rank=4 min=8071 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[981, 1004, 1012, 1000, 1038, 1046, 1075, 1044, 15, 3, 0, 0, 0, 0, 0, 0, 1036, 991, 981, 1024, 1012, 1015, 978, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1081, 1059, 1049, 1001, 1031, 1034, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1021, 1002, 995, 1025, 1033, 1035, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1020, 1072, 1017, 1038, 986, 1030, 1019, 24, 3, 0, 0, 0, 0, 0, 0, 994, 1010, 991, 1060, 1012, 1076, 995, 1001, 12, 2, 0, 0, 0, 0, 0, 0, 1017, 960, 1069, 984, 1029, 1021, 1070, 1035, 11, 0, 0, 0, 0, 0, 0, 0, 1004, 1011, 1043, 1004, 1067, 1088, 982, 1065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8059, 8296, 8123, 8227, 8153, 8196, 8264] avg=8192.0000 max=8296 max_rank=2 min=8059 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[996, 1033, 1056, 1038, 1011, 1008, 989, 1011, 18, 7, 0, 0, 0, 0, 0, 0, 1015, 1034, 1062, 999, 1014, 1018, 1014, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1074, 991, 1043, 1062, 1017, 993, 963, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 998, 999, 1023, 1003, 1035, 983, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1067, 1025, 997, 1042, 1043, 999, 972, 26, 5, 0, 0, 0, 0, 0, 0, 1038, 1003, 1029, 1008, 1036, 1034, 1021, 1040, 16, 4, 0, 0, 0, 0, 0, 0, 1044, 981, 1106, 952, 1011, 1051, 1032, 1026, 15, 0, 0, 0, 0, 0, 0, 0, 1023, 1035, 1042, 1042, 1026, 1037, 1010, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8191, 8166, 8094, 8246, 8229, 8218, 8225] avg=8192.0000 max=8246 max_rank=4 min=8094 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 1007, 1020, 1037, 1053, 1009, 1021, 1041, 16, 3, 0, 0, 0, 0, 0, 0, 989, 1049, 1033, 1076, 991, 1088, 1015, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1007, 986, 999, 1107, 979, 992, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1021, 1036, 1028, 1021, 1060, 969, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1036, 1049, 998, 1016, 1025, 992, 1086, 21, 3, 0, 0, 0, 0, 0, 0, 1028, 1032, 1018, 1016, 1036, 1040, 1039, 1051, 12, 2, 0, 0, 0, 0, 0, 0, 1047, 1047, 1040, 1013, 1023, 1001, 1021, 1027, 10, 0, 0, 0, 0, 0, 0, 0, 1026, 1042, 1048, 1010, 974, 953, 975, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8211, 8145, 8186, 8194, 8274, 8229, 8065] avg=8192.0000 max=8274 max_rank=5 min=8065 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8064, 8192, 8192, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8064, 8320, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8448, 8192, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8320, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1038, 1066, 996, 1041, 1066, 1031, 1024, 1047, 1033, 1066, 1050, 1008, 1018, 1025, 1040, 1034, 987, 1019, 1041, 950, 1037, 1045, 1054, 998, 1015, 1038, 998, 1038, 1085, 1050, 1036, 992, 994, 995, 982, 997, 1022, 1067, 1051, 1075, 1001, 1034, 1021, 1019, 1028, 1051, 1000, 1027, 1024, 989, 1000, 1020, 991, 1030, 1005, 1028, 995, 1027, 1034, 1028, 1053, 1017, 1016, 949] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1036, 1019, 1016, 1014, 1040, 1006, 1003, 1031, 1026, 1017, 1020, 1017, 1008, 991, 1060, 990, 1046, 1068, 1053, 1024, 1022, 1042, 1049, 1064, 1027, 1009, 1020, 1020, 1010, 1028, 1092, 1024, 1042, 999, 1024, 935, 1015, 1018, 1032, 1064, 1008, 1026, 1050, 1015, 1032, 1004, 1050, 1029, 1053, 1041, 1004, 1041, 966, 1042, 988, 1004, 1055, 972, 1020, 1061, 1018, 1009, 1014, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1001, 1025, 1063, 1022, 1036, 1023, 1008, 1030, 1088, 1023, 969, 978, 1032, 1050, 1086, 1038, 1041, 1042, 1005, 1058, 1058, 977, 1048, 1019, 1021, 1022, 1018, 1011, 1028, 998, 1041, 984, 1012, 1013, 997, 960, 1045, 1102, 1079, 985, 1035, 1024, 1000, 1009, 1035, 979, 1003, 984, 1044, 1065, 1037, 1030, 1038, 1012, 996, 1029, 1000, 1033, 1070, 1086, 1019, 994, 965, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1037, 1011, 1019, 1022, 985, 1032, 1065, 1045, 1009, 999, 1020, 1068, 1052, 1068, 1015, 1024, 993, 1058, 1011, 992, 1045, 1064, 1033, 1036, 1013, 1012, 1042, 1037, 990, 1040, 998, 1081, 1039, 1005, 1054, 1008, 1018, 1026, 1015, 1033, 987, 1009, 1038, 1038, 1005, 1012, 1036, 1045, 1027, 993, 1026, 1028, 1033, 961, 1004, 1019, 1015, 1021, 1035, 956, 1076, 1022, 979, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1027, 1014, 1023, 993, 1043, 1048, 1000, 1008, 1032, 1031, 1051, 1023, 1040, 1033, 1025, 1026, 988, 1044, 1031, 973, 990, 1029, 1063, 1019, 1010, 1032, 991, 1027, 1021, 991, 1072, 1042, 1038, 982, 1028, 1032, 999, 1023, 999, 1061, 1083, 1025, 1034, 1003, 1081, 1043, 1021, 989, 1005, 1046, 1011, 989, 1028, 1000, 1032, 1026, 1059, 1034, 1043, 976, 1000, 1019, 1043, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1029, 1055, 949, 1017, 990, 988, 1038, 1033, 1050, 1020, 1052, 1032, 1029, 1052, 1016, 1050, 970, 1041, 1064, 1046, 1126, 1026, 1040, 1039, 1003, 1027, 1033, 995, 1015, 1012, 1025, 1025, 1030, 1028, 1010, 1003, 1007, 1029, 1037, 1026, 1055, 962, 1040, 1045, 955, 1033, 996, 947, 997, 1038, 1080, 1064, 1019, 1053, 1034, 977, 1002, 1062, 1042, 1002, 992, 1047, 1004, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[993, 998, 1026, 1053, 1023, 1019, 979, 1031, 1043, 1037, 1059, 978, 1026, 997, 1057, 1061, 1043, 1037, 1005, 1030, 1039, 1048, 1053, 996, 995, 1001, 1044, 1019, 1071, 999, 1009, 1022, 1040, 1013, 1048, 1039, 1017, 991, 1052, 1012, 1027, 1054, 1026, 1030, 1040, 1026, 980, 1017, 1014, 1008, 989, 1009, 1057, 1010, 1034, 1059, 994, 980, 1016, 1018, 1042, 1042, 1024, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1009, 1006, 1017, 1038, 1012, 994, 1019, 1020, 1048, 1029, 1081, 1038, 1068, 1063, 1031, 1033, 973, 1039, 987, 1068, 1007, 1015, 1033, 1048, 1027, 1011, 1003, 1019, 1007, 1046, 1014, 1015, 1020, 1016, 1017, 1027, 1060, 982, 997, 1022, 1006, 1001, 1007, 1035, 1005, 1047, 1083, 1007, 1040, 1019, 1017, 1017, 1042, 1011, 1032, 1004, 1035, 1008, 1035, 1020, 1017, 1032, 1047, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1038, 1066, 996, 1041, 1066, 1031, 1024, 1047, 18, 0, 0, 0, 0, 0, 0, 0, 1033, 1066, 1050, 1008, 1018, 1025, 968, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1019, 1041, 950, 1037, 1045, 1011, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1038, 998, 1038, 1085, 1050, 1036, 992, 11, 0, 0, 0, 0, 0, 0, 0, 994, 995, 982, 997, 1022, 1067, 1051, 1075, 14, 4, 0, 0, 0, 0, 0, 0, 1001, 1034, 1021, 1019, 1028, 1051, 1000, 1027, 24, 0, 0, 0, 0, 0, 0, 0, 1024, 989, 1000, 1020, 991, 1030, 1005, 1028, 25, 0, 0, 0, 0, 0, 0, 0, 995, 1027, 1034, 1028, 1053, 1017, 1016, 949, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8327, 8202, 8088, 8263, 8201, 8205, 8112, 8138] avg=8192.0000 max=8327 max_rank=0 min=8088 min_rank=2 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1036, 1019, 1016, 1014, 1040, 1006, 1003, 1031, 18, 0, 0, 0, 0, 0, 0, 0, 1026, 1017, 1020, 1017, 1008, 991, 986, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1068, 1053, 1024, 1022, 1042, 1006, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1009, 1020, 1020, 1010, 1028, 1092, 1024, 11, 0, 0, 0, 0, 0, 0, 0, 1042, 999, 1024, 935, 1015, 1018, 1032, 1064, 14, 4, 0, 0, 0, 0, 0, 0, 1008, 1026, 1050, 1015, 1032, 1004, 1050, 1029, 25, 0, 0, 0, 0, 0, 0, 0, 1053, 1041, 1004, 1041, 966, 1042, 988, 1004, 26, 0, 0, 0, 0, 0, 0, 0, 1055, 972, 1020, 1061, 1018, 1009, 1014, 1013, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8055, 8325, 8241, 8147, 8239, 8165, 8181] avg=8192.0000 max=8325 max_rank=2 min=8055 min_rank=1 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1001, 1025, 1063, 1022, 1036, 1023, 1008, 1030, 18, 0, 0, 0, 0, 0, 0, 0, 1088, 1023, 969, 978, 1032, 1050, 1011, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1042, 1005, 1058, 1058, 977, 1005, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1022, 1018, 1011, 1028, 998, 1041, 984, 11, 0, 0, 0, 0, 0, 0, 0, 1012, 1013, 997, 960, 1045, 1102, 1079, 985, 14, 4, 0, 0, 0, 0, 0, 0, 1035, 1024, 1000, 1009, 1035, 979, 1003, 984, 25, 0, 0, 0, 0, 0, 0, 0, 1044, 1065, 1037, 1030, 1038, 1012, 996, 1029, 26, 0, 0, 0, 0, 0, 0, 0, 1000, 1033, 1070, 1086, 1019, 994, 965, 1013, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8189, 8205, 8134, 8211, 8094, 8277, 8200] avg=8192.0000 max=8277 max_rank=6 min=8094 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1027, 1014, 1023, 993, 1043, 1048, 1000, 1008, 18, 0, 0, 0, 0, 0, 0, 0, 1032, 1031, 1051, 1023, 1040, 1033, 953, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1044, 1031, 973, 990, 1029, 1019, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1032, 991, 1027, 1021, 991, 1072, 1042, 12, 0, 0, 0, 0, 0, 0, 0, 1038, 982, 1028, 1032, 999, 1023, 999, 1061, 14, 4, 0, 0, 0, 0, 0, 0, 1083, 1025, 1034, 1003, 1081, 1043, 1021, 989, 24, 0, 0, 0, 0, 0, 0, 0, 1005, 1046, 1011, 989, 1028, 1000, 1032, 1026, 25, 0, 0, 0, 0, 0, 0, 0, 1059, 1034, 1043, 976, 1000, 1019, 1043, 1044, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8189, 8093, 8198, 8180, 8303, 8162, 8237] avg=8192.0000 max=8303 max_rank=5 min=8093 min_rank=2 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1037, 1011, 1019, 1022, 985, 1032, 1065, 1045, 18, 0, 0, 0, 0, 0, 0, 0, 1009, 999, 1020, 1068, 1052, 1068, 944, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1058, 1011, 992, 1045, 1064, 991, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1012, 1042, 1037, 990, 1040, 998, 1081, 11, 0, 0, 0, 0, 0, 0, 0, 1039, 1005, 1054, 1008, 1018, 1026, 1015, 1033, 13, 4, 0, 0, 0, 0, 0, 0, 987, 1009, 1038, 1038, 1005, 1012, 1036, 1045, 24, 0, 0, 0, 0, 0, 0, 0, 1027, 993, 1026, 1028, 1033, 961, 1004, 1019, 25, 0, 0, 0, 0, 0, 0, 0, 1015, 1021, 1035, 956, 1076, 1022, 979, 1057, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8184, 8190, 8224, 8215, 8194, 8116, 8179] avg=8192.0000 max=8234 max_rank=0 min=8116 min_rank=6 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[993, 998, 1026, 1053, 1023, 1019, 979, 1031, 18, 0, 0, 0, 0, 0, 0, 0, 1043, 1037, 1059, 978, 1026, 997, 983, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1037, 1005, 1030, 1039, 1048, 1010, 996, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1001, 1044, 1019, 1071, 999, 1009, 1022, 11, 0, 0, 0, 0, 0, 0, 0, 1040, 1013, 1048, 1039, 1017, 991, 1052, 1012, 14, 4, 0, 0, 0, 0, 0, 0, 1027, 1054, 1026, 1030, 1040, 1026, 980, 1017, 25, 0, 0, 0, 0, 0, 0, 0, 1014, 1008, 989, 1009, 1057, 1010, 1034, 1059, 26, 0, 0, 0, 0, 0, 0, 0, 994, 980, 1016, 1018, 1042, 1042, 1024, 1037, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8184, 8208, 8171, 8230, 8225, 8206, 8172] avg=8192.0000 max=8230 max_rank=4 min=8140 min_rank=0 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1029, 1055, 949, 1017, 990, 988, 1038, 1033, 20, 0, 0, 0, 0, 0, 0, 0, 1050, 1020, 1052, 1032, 1029, 1052, 929, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1041, 1064, 1046, 1126, 1026, 987, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1027, 1033, 995, 1015, 1012, 1025, 1025, 17, 0, 0, 0, 0, 0, 0, 0, 1030, 1028, 1010, 1003, 1007, 1029, 1037, 1026, 16, 8, 0, 0, 0, 0, 0, 0, 1055, 962, 1040, 1045, 955, 1033, 996, 947, 28, 0, 0, 0, 0, 0, 0, 0, 997, 1038, 1080, 1064, 1019, 1053, 1034, 977, 29, 0, 0, 0, 0, 0, 0, 0, 1002, 1062, 1042, 1002, 992, 1047, 1004, 1033, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8214, 8299, 8152, 8194, 8061, 8291, 8206] avg=8192.0000 max=8299 max_rank=2 min=8061 min_rank=5 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1009, 1006, 1017, 1038, 1012, 994, 1019, 1020, 18, 0, 0, 0, 0, 0, 0, 0, 1048, 1029, 1081, 1038, 1068, 1063, 959, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1039, 987, 1068, 1007, 1015, 991, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1011, 1003, 1019, 1007, 1046, 1014, 1015, 11, 0, 0, 0, 0, 0, 0, 0, 1020, 1016, 1017, 1027, 1060, 982, 997, 1022, 13, 4, 0, 0, 0, 0, 0, 0, 1006, 1001, 1007, 1035, 1005, 1047, 1083, 1007, 24, 0, 0, 0, 0, 0, 0, 0, 1040, 1019, 1017, 1017, 1042, 1011, 1032, 1004, 25, 0, 0, 0, 0, 0, 0, 0, 1035, 1008, 1035, 1020, 1017, 1032, 1047, 1010, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8319, 8128, 8153, 8158, 8215, 8207, 8223] avg=8192.0000 max=8319 max_rank=1 min=8128 min_rank=2 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8448, 8320, 8320, 8192, 8320, 8320, 7808, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8192, 8064, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8448, 8320, 8192, 8448, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8064, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[977, 1055, 1042, 1062, 1061, 1043, 1008, 1033, 1031, 1018, 1082, 987, 992, 1071, 1029, 1029, 1060, 1026, 981, 987, 979, 1031, 997, 1014, 998, 1035, 1032, 983, 1021, 1059, 1048, 1018, 1034, 1078, 1062, 1010, 1020, 1032, 971, 1073, 1022, 1011, 1040, 1034, 1028, 1053, 1054, 998, 1052, 971, 989, 1012, 1032, 1018, 1013, 1007, 1006, 1014, 1034, 1057, 1004, 996, 1023, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1050, 1002, 1003, 1022, 1103, 1067, 997, 1025, 984, 1027, 1089, 1040, 1021, 1015, 1032, 979, 1049, 1011, 1021, 1057, 1072, 987, 1001, 1031, 1040, 984, 1054, 1004, 1016, 1026, 1043, 1021, 1025, 992, 1031, 1005, 1033, 1022, 1068, 1054, 999, 1001, 993, 1010, 1062, 1013, 1031, 998, 983, 957, 1040, 1036, 1043, 1026, 1016, 1007, 1037, 1030, 1024, 1024, 1014, 1068, 1002, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1002, 997, 970, 1057, 1008, 1023, 1043, 1037, 1036, 1019, 987, 1037, 1040, 977, 1069, 1020, 1065, 1012, 1039, 1019, 1012, 1039, 1021, 1025, 989, 1039, 1016, 948, 1029, 1033, 1051, 999, 959, 1019, 1074, 1018, 1031, 1049, 1039, 970, 1020, 1019, 1037, 1026, 1037, 1021, 1037, 1050, 1046, 1014, 1019, 1005, 1015, 1060, 994, 1001, 1085, 1056, 1045, 1022, 1019, 1008, 1055, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[993, 1068, 1048, 1039, 1009, 1005, 1030, 1074, 996, 1016, 1002, 1060, 1006, 1064, 1009, 1020, 1032, 1037, 987, 1028, 991, 1057, 969, 1021, 1046, 984, 1007, 991, 1049, 1067, 1030, 1006, 1041, 1002, 1010, 1022, 1010, 1034, 1050, 1006, 979, 1042, 1025, 1030, 1079, 1050, 1008, 1053, 1035, 1037, 1004, 1037, 1004, 1025, 1029, 982, 1036, 1010, 1002, 1008, 1056, 1040, 996, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1034, 1066, 994, 1063, 1021, 1031, 1028, 1012, 1027, 1005, 1022, 1008, 998, 1009, 1038, 1004, 1005, 1022, 1008, 1024, 1030, 1038, 992, 1030, 1045, 1012, 1065, 1049, 1011, 982, 1012, 1030, 1005, 969, 983, 1081, 1011, 1036, 1000, 1039, 1075, 1021, 1022, 1065, 1021, 1000, 1026, 997, 1013, 980, 1007, 1044, 1033, 1002, 1061, 1011, 1011, 1048, 1016, 1063, 1124, 1018, 1047, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1044, 1063, 987, 1043, 1032, 999, 999, 1028, 1042, 1062, 1035, 1038, 1019, 998, 1003, 1007, 1024, 1021, 1018, 1015, 1041, 1075, 1075, 1015, 1036, 1064, 1016, 1016, 980, 1027, 992, 1011, 1007, 1021, 996, 1030, 1067, 1055, 1053, 991, 1004, 1040, 1067, 1042, 1041, 1031, 1011, 1038, 1027, 1063, 1047, 1025, 987, 977, 1008, 1012, 943, 1041, 940, 1045, 1017, 1016, 1020, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1020, 964, 1031, 1053, 1054, 1069, 1001, 997, 1053, 1075, 1087, 1037, 980, 1027, 999, 993, 982, 992, 1049, 1064, 1054, 1004, 1039, 1049, 1021, 1048, 1043, 987, 1003, 1038, 1053, 1007, 1066, 1031, 1068, 1053, 1024, 1052, 1028, 992, 1005, 1039, 1023, 1041, 970, 977, 1015, 1017, 1042, 1032, 986, 1006, 970, 1042, 996, 1029, 999, 1028, 1033, 957, 1062, 1027, 1038, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1007, 1045, 999, 996, 992, 1026, 1042, 1017, 985, 1001, 1052, 1022, 998, 1078, 998, 1015, 1039, 1035, 1014, 1020, 996, 1019, 1056, 1031, 997, 999, 1028, 982, 1009, 1048, 1036, 1023, 1008, 979, 1019, 1030, 1059, 1055, 1017, 1038, 1047, 1007, 1024, 1006, 1044, 1032, 1053, 1014, 1026, 1015, 1023, 1038, 1048, 1039, 1031, 1018, 1066, 1025, 1035, 1047, 999, 1026, 1060, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[977, 1055, 1042, 1040, 1061, 1043, 1008, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1018, 1082, 987, 992, 1071, 1029, 1029, 4, 0, 0, 0, 0, 0, 0, 0, 1060, 1026, 981, 987, 979, 1031, 997, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1035, 1032, 983, 1021, 1059, 1048, 1018, 12, 12, 0, 0, 0, 0, 0, 0, 1034, 1078, 1062, 1010, 1020, 1011, 971, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1011, 1040, 1034, 1012, 1053, 1054, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 971, 989, 1012, 1032, 1018, 1013, 1007, 22, 21, 5, 0, 0, 0, 0, 0, 1006, 1014, 1034, 1057, 987, 996, 1023, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8243, 8075, 8218, 8259, 8224, 8142, 8116] avg=8192.0000 max=8259 max_rank=0 min=8075 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1050, 1002, 1003, 1001, 1103, 1067, 997, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1027, 1089, 1040, 1021, 1015, 1032, 979, 4, 0, 0, 0, 0, 0, 0, 0, 1049, 1011, 1021, 1057, 1072, 987, 1001, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 984, 1054, 1004, 1016, 1026, 1043, 1021, 12, 12, 0, 0, 0, 0, 0, 0, 1025, 992, 1031, 1005, 1033, 1002, 1068, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1001, 993, 1010, 1046, 1013, 1031, 998, 0, 0, 0, 0, 0, 0, 0, 0, 983, 957, 1040, 1036, 1043, 1026, 1016, 1007, 21, 20, 5, 0, 0, 0, 0, 0, 1037, 1030, 1024, 1024, 997, 1068, 1002, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8191, 8229, 8212, 8210, 8091, 8154, 8201] avg=8192.0000 max=8248 max_rank=0 min=8091 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[993, 1068, 1048, 1018, 1009, 1005, 1030, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1016, 1002, 1060, 1006, 1064, 1009, 1020, 4, 0, 0, 0, 0, 0, 0, 0, 1032, 1037, 987, 1028, 991, 1057, 969, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 984, 1007, 991, 1049, 1067, 1030, 1006, 13, 12, 0, 0, 0, 0, 0, 0, 1041, 1002, 1010, 1022, 1010, 1013, 1050, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1042, 1025, 1030, 1062, 1050, 1008, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1037, 1004, 1037, 1004, 1025, 1029, 982, 21, 21, 5, 0, 0, 0, 0, 0, 1036, 1010, 1002, 1008, 1039, 1040, 996, 1053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8177, 8122, 8205, 8154, 8249, 8200, 8184] avg=8192.0000 max=8249 max_rank=5 min=8122 min_rank=2 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1002, 997, 970, 1035, 1008, 1023, 1043, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1019, 987, 1037, 1040, 977, 1069, 1020, 4, 0, 0, 0, 0, 0, 0, 0, 1065, 1012, 1039, 1019, 1012, 1039, 1021, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1039, 1016, 948, 1029, 1033, 1051, 999, 12, 12, 0, 0, 0, 0, 0, 0, 959, 1019, 1074, 1018, 1031, 1028, 1039, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1019, 1037, 1026, 1021, 1021, 1037, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1014, 1019, 1005, 1015, 1060, 994, 1001, 22, 21, 5, 0, 0, 0, 0, 0, 1085, 1056, 1045, 1022, 1002, 1008, 1055, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8189, 8232, 8128, 8138, 8231, 8202, 8301] avg=8192.0000 max=8301 max_rank=7 min=8115 min_rank=0 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1044, 1063, 987, 1022, 1032, 999, 999, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1062, 1035, 1038, 1019, 998, 1003, 1007, 4, 0, 0, 0, 0, 0, 0, 0, 1024, 1021, 1018, 1015, 1041, 1075, 1075, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1064, 1016, 1016, 980, 1027, 992, 1011, 12, 12, 0, 0, 0, 0, 0, 0, 1007, 1021, 996, 1030, 1067, 1034, 1053, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1040, 1067, 1042, 1025, 1031, 1011, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1063, 1047, 1025, 987, 977, 1008, 1012, 21, 21, 5, 0, 0, 0, 0, 0, 943, 1041, 940, 1045, 1000, 1016, 1020, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8208, 8284, 8166, 8199, 8258, 8193, 8054] avg=8192.0000 max=8284 max_rank=2 min=8054 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1034, 1066, 994, 1041, 1021, 1031, 1028, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1005, 1022, 1008, 998, 1009, 1038, 1004, 4, 0, 0, 0, 0, 0, 0, 0, 1005, 1022, 1008, 1024, 1030, 1038, 992, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1012, 1065, 1049, 1011, 982, 1012, 1030, 12, 13, 0, 0, 0, 0, 0, 0, 1005, 969, 983, 1081, 1011, 1015, 1000, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1021, 1022, 1065, 1005, 1000, 1026, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 980, 1007, 1044, 1033, 1002, 1061, 1011, 22, 21, 5, 0, 0, 0, 0, 0, 1011, 1048, 1016, 1063, 1106, 1018, 1047, 992, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8115, 8149, 8231, 8103, 8211, 8199, 8301] avg=8192.0000 max=8301 max_rank=7 min=8103 min_rank=4 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1007, 1045, 999, 972, 992, 1026, 1042, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1001, 1052, 1022, 998, 1078, 998, 1015, 9, 0, 0, 0, 0, 0, 0, 0, 1039, 1035, 1014, 1020, 996, 1019, 1056, 1031, 2, 0, 0, 0, 0, 0, 0, 0, 997, 999, 1028, 982, 1009, 1048, 1036, 1023, 16, 15, 0, 0, 0, 0, 0, 0, 1008, 979, 1019, 1030, 1059, 1030, 1017, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1007, 1024, 1006, 1017, 1032, 1053, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1015, 1023, 1038, 1048, 1039, 1031, 1018, 24, 25, 9, 0, 0, 0, 0, 0, 1066, 1025, 1035, 1047, 975, 1026, 1060, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8158, 8212, 8153, 8180, 8200, 8296, 8237] avg=8192.0000 max=8296 max_rank=6 min=8100 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1020, 964, 1031, 1032, 1054, 1069, 1001, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1075, 1087, 1037, 980, 1027, 999, 993, 4, 0, 0, 0, 0, 0, 0, 0, 982, 992, 1049, 1064, 1054, 1004, 1039, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1048, 1043, 987, 1003, 1038, 1053, 1007, 11, 12, 0, 0, 0, 0, 0, 0, 1066, 1031, 1068, 1053, 1024, 1031, 1028, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1039, 1023, 1041, 955, 977, 1015, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1032, 986, 1006, 970, 1042, 996, 1029, 21, 21, 5, 0, 0, 0, 0, 0, 999, 1028, 1033, 957, 1045, 1027, 1038, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8255, 8233, 8223, 8293, 8072, 8150, 8142] avg=8192.0000 max=8293 max_rank=4 min=8072 min_rank=5 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8448, 8320, 8064, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8064, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8192, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1057, 1036, 1069, 1004, 1050, 1001, 963, 1002, 1026, 1018, 1008, 1079, 985, 1046, 1043, 1034, 1063, 1069, 988, 1056, 1062, 988, 1002, 1073, 1003, 1053, 998, 1042, 967, 1060, 1011, 1030, 1017, 1024, 1032, 1026, 982, 1016, 1055, 1022, 1013, 1093, 994, 1007, 984, 996, 992, 1062, 1020, 1043, 1024, 1029, 1026, 1009, 1004, 1017, 1025, 1032, 1043, 1001, 1041, 995, 1000, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1020, 994, 1024, 1037, 1007, 1025, 1047, 1002, 1041, 1028, 1037, 972, 998, 1067, 1020, 1004, 1021, 996, 1009, 1068, 991, 1020, 1037, 1016, 1051, 995, 1024, 981, 1003, 1028, 1075, 971, 1074, 1013, 1011, 1030, 1046, 1072, 1007, 1024, 1003, 1015, 1060, 1014, 1018, 969, 977, 1062, 1016, 1055, 1047, 1077, 1056, 1041, 1027, 1008, 1053, 1030, 1017, 1038, 1015, 1033, 1000, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1062, 1040, 1042, 1062, 1010, 1029, 1037, 1020, 1026, 998, 1017, 1051, 1014, 1050, 1047, 1014, 1068, 978, 1037, 989, 1023, 1032, 1041, 1026, 998, 1006, 1009, 1026, 983, 1040, 1022, 1026, 1028, 1079, 1042, 1067, 1032, 986, 1017, 1040, 1023, 1021, 1056, 1033, 1027, 1000, 1031, 1026, 1029, 982, 959, 1015, 1034, 1015, 1024, 1008, 1019, 1003, 989, 1020, 1059, 1010, 1015, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1048, 1052, 1021, 1044, 1064, 1043, 1018, 1021, 997, 1015, 1054, 1072, 1026, 999, 1008, 1052, 1052, 993, 1038, 993, 1007, 997, 1041, 1033, 1008, 986, 953, 1023, 1004, 996, 1001, 996, 1000, 1022, 992, 1050, 1024, 999, 1009, 1023, 1012, 1067, 1026, 1058, 1011, 997, 1068, 1056, 974, 1010, 1047, 1083, 973, 1013, 980, 1056, 1004, 1039, 1029, 1032, 1027, 1045, 1068, 1087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[956, 1026, 1052, 1043, 1074, 990, 1022, 1077, 1009, 1017, 998, 1042, 994, 1004, 1027, 1014, 1020, 1059, 1004, 1023, 971, 1073, 973, 1038, 1051, 1056, 1021, 1060, 1011, 1049, 1017, 1045, 1019, 1026, 1099, 1028, 1036, 993, 1034, 1087, 996, 1051, 1008, 998, 1011, 983, 1010, 948, 1038, 1017, 1063, 990, 1095, 1054, 1012, 995, 1002, 993, 1035, 1039, 1011, 1044, 1021, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1052, 1042, 1016, 1012, 1079, 1049, 1002, 998, 1057, 984, 1012, 1007, 1094, 1073, 979, 1018, 956, 1010, 1028, 1037, 983, 1046, 1042, 1038, 950, 1026, 1026, 1012, 1042, 978, 1046, 1001, 1004, 1054, 1048, 1006, 1020, 1037, 1042, 983, 1006, 1029, 1055, 1028, 1045, 1027, 1055, 1056, 1011, 1015, 1035, 1027, 1024, 1064, 993, 1041, 1015, 1036, 993, 1015, 1016, 985, 1047, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1026, 1055, 1006, 1018, 1044, 1015, 1022, 1006, 1027, 999, 998, 1026, 1020, 1013, 1039, 1097, 1039, 1011, 1023, 1061, 1025, 1023, 1031, 1014, 1031, 1066, 1029, 1048, 1070, 996, 1085, 1093, 999, 1039, 1048, 991, 1075, 983, 984, 1080, 1001, 1017, 1028, 1004, 1020, 1001, 1023, 975, 1046, 970, 1016, 1057, 988, 980, 970, 1035, 986, 1006, 1074, 1044, 964, 1031, 1052, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[992, 999, 971, 998, 1021, 1022, 1032, 1006, 1063, 1007, 1029, 1025, 1038, 1037, 1082, 1060, 1018, 996, 1007, 1022, 1037, 1052, 1017, 1006, 1003, 1002, 1025, 1038, 1012, 1001, 1066, 1069, 1032, 1028, 1015, 1045, 1016, 1009, 1078, 1060, 1066, 1014, 1030, 1003, 1019, 1016, 1008, 1011, 1012, 1018, 998, 980, 978, 988, 1048, 1022, 994, 1045, 1063, 1055, 1028, 1073, 1015, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1020, 994, 1024, 1037, 992, 1025, 1047, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1028, 1037, 972, 998, 1067, 1020, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 996, 1009, 1068, 991, 1020, 1037, 1016, 4, 0, 0, 0, 0, 0, 0, 0, 1051, 995, 1024, 981, 1003, 1028, 1075, 971, 11, 5, 0, 0, 0, 0, 0, 0, 1074, 1013, 1011, 1030, 1046, 1072, 1007, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1015, 1060, 1014, 1018, 969, 977, 1062, 18, 8, 0, 0, 0, 0, 0, 0, 1016, 1055, 1047, 1077, 1056, 1041, 1027, 1008, 31, 0, 0, 0, 0, 0, 0, 0, 1053, 1030, 1017, 1038, 1015, 1033, 1000, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8144, 8162, 8144, 8238, 8144, 8358, 8205] avg=8192.0000 max=8358 max_rank=6 min=8141 min_rank=0 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32945 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1048, 1052, 1021, 1044, 1047, 1043, 1018, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1015, 1054, 1072, 1026, 999, 1008, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 993, 1038, 993, 1007, 997, 1041, 1033, 5, 0, 0, 0, 0, 0, 0, 0, 1008, 986, 953, 1023, 1004, 996, 1001, 996, 12, 5, 0, 0, 0, 0, 0, 0, 1000, 1022, 992, 1050, 1024, 999, 1009, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1067, 1026, 1058, 1011, 997, 1068, 1056, 19, 8, 0, 0, 0, 0, 0, 0, 974, 1010, 1047, 1083, 973, 1013, 980, 1056, 31, 0, 0, 0, 0, 0, 0, 0, 1004, 1039, 1029, 1032, 1027, 1045, 1068, 1087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 8199, 8159, 7984, 8080, 8322, 8167, 8331] avg=8192.0000 max=8331 max_rank=7 min=7984 min_rank=3 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32900 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1057, 1036, 1069, 1004, 1033, 1001, 963, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1018, 1008, 1079, 985, 1046, 1043, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1069, 988, 1056, 1062, 988, 1002, 1073, 5, 0, 0, 0, 0, 0, 0, 0, 1003, 1053, 998, 1042, 967, 1060, 1011, 1030, 12, 5, 0, 0, 0, 0, 0, 0, 1017, 1024, 1032, 1026, 982, 1016, 1055, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1093, 994, 1007, 984, 996, 992, 1062, 19, 8, 0, 0, 0, 0, 0, 0, 1020, 1043, 1024, 1029, 1026, 1009, 1004, 1017, 31, 0, 0, 0, 0, 0, 0, 0, 1025, 1032, 1043, 1001, 1041, 995, 1000, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8215, 8306, 8181, 8135, 8168, 8203, 8163] avg=8192.0000 max=8306 max_rank=2 min=8135 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1062, 1040, 1042, 1062, 995, 1029, 1037, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 998, 1017, 1051, 1014, 1050, 1047, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 978, 1037, 989, 1023, 1032, 1041, 1026, 4, 0, 0, 0, 0, 0, 0, 0, 998, 1006, 1009, 1026, 983, 1040, 1022, 1026, 11, 5, 0, 0, 0, 0, 0, 0, 1028, 1079, 1042, 1067, 1032, 986, 1017, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1021, 1056, 1033, 1027, 1000, 1031, 1026, 18, 8, 0, 0, 0, 0, 0, 0, 1029, 982, 959, 1015, 1034, 1015, 1024, 1008, 31, 0, 0, 0, 0, 0, 0, 0, 1019, 1003, 989, 1020, 1059, 1010, 1015, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8194, 8198, 8126, 8252, 8243, 8097, 8139] avg=8192.0000 max=8287 max_rank=0 min=8097 min_rank=6 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1026, 1055, 1006, 1018, 1027, 1015, 1022, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 999, 998, 1026, 1020, 1013, 1039, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1011, 1023, 1061, 1025, 1023, 1031, 1014, 5, 0, 0, 0, 0, 0, 0, 0, 1031, 1066, 1029, 1048, 1070, 996, 1085, 1093, 12, 5, 0, 0, 0, 0, 0, 0, 999, 1039, 1048, 991, 1075, 983, 984, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1017, 1028, 1004, 1020, 1001, 1023, 975, 20, 8, 0, 0, 0, 0, 0, 0, 1046, 970, 1016, 1057, 988, 980, 970, 1035, 33, 0, 0, 0, 0, 0, 0, 0, 986, 1006, 1074, 1044, 964, 1031, 1052, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8194, 8232, 8435, 8158, 8097, 8095, 8150] avg=8192.0000 max=8435 max_rank=3 min=8095 min_rank=6 imbalance_ratio=1.029663 max_load_reduction=57101 max_load_reduction_pct=0.871292 moved_to_echo=32500 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[956, 1026, 1052, 1043, 1057, 990, 1022, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1017, 998, 1042, 994, 1004, 1027, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1059, 1004, 1023, 971, 1073, 973, 1038, 5, 0, 0, 0, 0, 0, 0, 0, 1051, 1056, 1021, 1060, 1011, 1049, 1017, 1045, 12, 5, 0, 0, 0, 0, 0, 0, 1019, 1026, 1099, 1028, 1036, 993, 1034, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1051, 1008, 998, 1011, 983, 1010, 948, 18, 8, 0, 0, 0, 0, 0, 0, 1038, 1017, 1063, 990, 1095, 1054, 1012, 995, 33, 0, 0, 0, 0, 0, 0, 0, 1002, 993, 1035, 1039, 1011, 1044, 1021, 984, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8082, 8166, 8327, 8281, 8031, 8297, 8129] avg=8192.0000 max=8327 max_rank=3 min=8031 min_rank=5 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1052, 1042, 1016, 1012, 1054, 1049, 1002, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 984, 1012, 1007, 1094, 1073, 979, 985, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1010, 1028, 1037, 983, 1046, 1042, 1038, 8, 6, 0, 0, 0, 0, 0, 0, 950, 1026, 1026, 1012, 1042, 978, 1046, 1001, 17, 9, 0, 0, 0, 0, 0, 0, 1004, 1054, 1048, 1006, 1020, 1037, 1042, 938, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1029, 1055, 1028, 1045, 1027, 1055, 1056, 24, 12, 0, 0, 0, 0, 0, 0, 1011, 1015, 1035, 1027, 1024, 1064, 993, 1041, 33, 0, 0, 0, 0, 0, 0, 0, 1015, 1036, 993, 1009, 1016, 985, 1047, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8191, 8154, 8107, 8149, 8337, 8243, 8130] avg=8192.0000 max=8337 max_rank=5 min=8107 min_rank=3 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[992, 999, 971, 998, 1006, 1022, 1032, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1007, 1029, 1025, 1038, 1037, 1082, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 996, 1007, 1022, 1037, 1052, 1017, 1006, 4, 0, 0, 0, 0, 0, 0, 0, 1003, 1002, 1025, 1038, 1012, 1001, 1066, 1069, 11, 5, 0, 0, 0, 0, 0, 0, 1032, 1028, 1015, 1045, 1016, 1009, 1078, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1014, 1030, 1003, 1019, 1016, 1008, 1011, 19, 8, 0, 0, 0, 0, 0, 0, 1012, 1018, 998, 980, 978, 988, 1048, 1022, 32, 0, 0, 0, 0, 0, 0, 0, 994, 1045, 1063, 1055, 1028, 1073, 1015, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8026, 8317, 8159, 8232, 8243, 8194, 8076, 8289] avg=8192.0000 max=8317 max_rank=1 min=8026 min_rank=0 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8064, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8448, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8064, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1033, 1052, 1013, 1016, 999, 999, 987, 1051, 1034, 971, 1000, 1059, 1036, 1031, 1034, 1009, 1060, 1078, 966, 993, 1011, 1041, 1048, 999, 1045, 1029, 1049, 1033, 927, 1051, 1015, 1051, 1035, 1022, 974, 1010, 1035, 1010, 1008, 1029, 1025, 1074, 960, 1010, 1026, 1024, 1014, 1068, 1075, 1043, 1029, 986, 984, 1004, 1049, 1094, 1019, 995, 1077, 1054, 1021, 973, 1078, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1027, 1019, 996, 1012, 1055, 1005, 1032, 1043, 1016, 1015, 1023, 955, 1030, 1035, 1044, 1022, 1074, 1023, 1029, 1067, 1007, 1065, 966, 998, 945, 1040, 1038, 1031, 975, 1015, 1010, 1035, 1066, 1024, 1029, 1072, 1048, 1020, 988, 1063, 1036, 994, 1011, 1034, 1022, 1035, 1015, 1050, 980, 1025, 1041, 1022, 1090, 1055, 1030, 998, 1054, 1038, 989, 982, 1020, 1036, 1013, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[985, 1019, 1054, 1032, 1032, 1031, 1011, 1007, 1065, 1048, 1074, 1100, 1020, 1003, 1034, 983, 1000, 1015, 1027, 976, 1063, 991, 1028, 960, 1010, 1040, 1006, 1016, 958, 1037, 966, 984, 1009, 1006, 1031, 1021, 1000, 1037, 1031, 996, 1046, 1077, 1053, 1040, 985, 1039, 1053, 1010, 1030, 1091, 982, 1004, 1023, 1033, 1008, 1046, 1006, 1029, 1076, 1057, 1013, 1079, 1025, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[971, 1035, 1023, 1067, 1010, 1004, 996, 1060, 1034, 1068, 966, 1014, 1038, 1016, 1068, 1008, 1005, 1077, 977, 1081, 1017, 1022, 1050, 1005, 1029, 987, 1022, 981, 1042, 1041, 1015, 1014, 1052, 1008, 1006, 1050, 1023, 981, 1007, 1017, 1013, 1009, 1024, 1021, 989, 1018, 1054, 1027, 1046, 1030, 1031, 988, 1092, 1023, 1031, 1018, 1086, 1001, 1040, 1038, 1038, 1023, 1004, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1024, 1024, 1018, 1002, 1015, 1032, 1045, 1030, 1070, 1048, 1007, 1044, 1015, 1058, 1036, 1032, 1004, 995, 977, 993, 1011, 1044, 1035, 1060, 1021, 1046, 1026, 1035, 1032, 997, 1018, 1033, 1018, 1012, 1028, 948, 1037, 1000, 1087, 1017, 1026, 1039, 1088, 993, 1072, 1029, 1016, 1082, 1056, 962, 965, 1000, 1019, 1027, 952, 990, 997, 995, 1060, 1002, 1056, 1074, 1011, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[975, 1017, 983, 1028, 1020, 1076, 1020, 1038, 1008, 1021, 1004, 1019, 1003, 1001, 1024, 1053, 1060, 991, 1050, 995, 1017, 1045, 1072, 1041, 1021, 1021, 1043, 986, 1018, 1038, 965, 1071, 1009, 1068, 1000, 1050, 1014, 1072, 1070, 973, 1015, 1015, 1041, 979, 1028, 1056, 1057, 996, 1020, 982, 1071, 1026, 981, 1013, 1036, 1021, 1006, 1002, 1073, 974, 1065, 1024, 1000, 1075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1032, 1009, 954, 983, 990, 1035, 1035, 1042, 990, 1044, 1021, 1063, 1039, 1048, 1048, 1002, 1026, 1020, 1037, 1034, 1049, 1037, 1018, 1021, 1044, 1062, 989, 998, 1051, 997, 1002, 1025, 1056, 1036, 1023, 1051, 1070, 1060, 1011, 1067, 1020, 973, 1002, 1011, 1012, 1016, 1082, 999, 1006, 984, 1028, 1046, 1025, 1040, 1027, 1013, 1042, 989, 993, 1014, 1024, 1032, 1012, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[997, 1069, 1018, 1000, 1051, 1004, 1003, 1010, 1053, 1055, 1055, 981, 1000, 1031, 969, 1026, 1031, 1011, 1022, 1026, 1038, 1009, 1008, 1000, 1025, 1025, 1015, 1011, 1017, 1050, 995, 997, 1056, 992, 1060, 1019, 1046, 1047, 1049, 996, 1003, 1054, 1032, 1024, 1042, 1052, 988, 1022, 1007, 1067, 1069, 985, 1045, 1037, 1010, 1026, 1040, 1003, 1005, 1032, 1007, 1032, 1045, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[971, 1035, 1023, 1067, 1010, 1004, 996, 1060, 20, 15, 0, 0, 0, 0, 0, 0, 1011, 1068, 966, 1014, 1038, 1016, 1068, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1077, 977, 1081, 1017, 1022, 1050, 1005, 5, 0, 0, 0, 0, 0, 0, 0, 1029, 987, 1022, 981, 1042, 1041, 1015, 1014, 27, 23, 3, 0, 0, 0, 0, 0, 1025, 1008, 1006, 1050, 1023, 981, 1007, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1009, 1024, 1021, 989, 1018, 1031, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1030, 1031, 988, 1092, 1023, 1031, 1018, 2, 0, 0, 0, 0, 0, 0, 0, 1086, 1001, 1018, 1038, 1038, 1023, 1004, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8189, 8239, 8184, 8117, 8132, 8261, 8213] avg=8192.0000 max=8261 max_rank=6 min=8117 min_rank=4 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1024, 1024, 1018, 1002, 1015, 1032, 1045, 1030, 19, 15, 0, 0, 0, 0, 0, 0, 1047, 1048, 1007, 1044, 1015, 1058, 1036, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 995, 977, 993, 1011, 1044, 1035, 1060, 5, 0, 0, 0, 0, 0, 0, 0, 1021, 1046, 1026, 1035, 1032, 997, 1018, 1033, 26, 23, 3, 0, 0, 0, 0, 0, 992, 1012, 1028, 948, 1037, 1000, 1087, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1039, 1088, 993, 1072, 1029, 994, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 962, 965, 1000, 1019, 1027, 952, 990, 2, 0, 0, 0, 0, 0, 0, 0, 997, 995, 1038, 1002, 1056, 1074, 1011, 1051, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8287, 8124, 8260, 8121, 8323, 7973, 8224] avg=8192.0000 max=8323 max_rank=5 min=7973 min_rank=6 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1027, 1019, 996, 1012, 1055, 1005, 1032, 1043, 19, 14, 0, 0, 0, 0, 0, 0, 994, 1015, 1023, 955, 1030, 1035, 1044, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1023, 1029, 1067, 1007, 1065, 966, 998, 4, 0, 0, 0, 0, 0, 0, 0, 945, 1040, 1038, 1031, 975, 1015, 1010, 1035, 27, 22, 3, 0, 0, 0, 0, 0, 1039, 1024, 1029, 1072, 1048, 1020, 988, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 994, 1011, 1034, 1022, 1035, 993, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1025, 1041, 1022, 1090, 1055, 1030, 998, 2, 0, 0, 0, 0, 0, 0, 0, 1054, 1038, 969, 982, 1020, 1036, 1013, 1009, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8118, 8233, 8141, 8283, 8175, 8243, 8121] avg=8192.0000 max=8283 max_rank=4 min=8118 min_rank=1 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1033, 1052, 1013, 1016, 999, 999, 987, 1051, 19, 15, 0, 0, 0, 0, 0, 0, 1011, 971, 1000, 1059, 1036, 1031, 1034, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1078, 966, 993, 1011, 1041, 1048, 999, 5, 0, 0, 0, 0, 0, 0, 0, 1045, 1029, 1049, 1033, 927, 1051, 1015, 1051, 26, 23, 3, 0, 0, 0, 0, 0, 1009, 1022, 974, 1010, 1035, 1010, 1008, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1074, 960, 1010, 1026, 1024, 992, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1043, 1029, 986, 984, 1004, 1049, 1094, 2, 0, 0, 0, 0, 0, 0, 0, 1019, 995, 1055, 1054, 1021, 973, 1078, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8151, 8201, 8252, 8097, 8179, 8266, 8206] avg=8192.0000 max=8266 max_rank=6 min=8097 min_rank=4 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[985, 1019, 1054, 1032, 1032, 1031, 1011, 1007, 20, 15, 0, 0, 0, 0, 0, 0, 1042, 1048, 1074, 1100, 1020, 1003, 1034, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1015, 1027, 976, 1063, 991, 1028, 960, 5, 0, 0, 0, 0, 0, 0, 0, 1010, 1040, 1006, 1016, 958, 1037, 966, 984, 26, 23, 3, 0, 0, 0, 0, 0, 983, 1006, 1031, 1021, 1000, 1037, 1031, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1077, 1053, 1040, 985, 1039, 1030, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1091, 982, 1004, 1023, 1033, 1008, 1046, 2, 0, 0, 0, 0, 0, 0, 0, 1006, 1029, 1054, 1057, 1013, 1079, 1025, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8304, 8065, 8069, 8105, 8280, 8219, 8288] avg=8192.0000 max=8304 max_rank=1 min=8065 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32892 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1032, 1009, 954, 983, 990, 1035, 1035, 1042, 23, 20, 0, 0, 0, 0, 0, 0, 964, 1044, 1021, 1063, 1039, 1048, 1048, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1020, 1037, 1034, 1049, 1037, 1018, 1021, 7, 0, 0, 0, 0, 0, 0, 0, 1044, 1062, 989, 998, 1051, 997, 1002, 1025, 29, 26, 5, 0, 0, 0, 0, 0, 1027, 1036, 1023, 1051, 1070, 1060, 1011, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 973, 1002, 1011, 1012, 1016, 1054, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 984, 1028, 1046, 1025, 1040, 1027, 1013, 5, 0, 0, 0, 0, 0, 0, 0, 1042, 989, 961, 1014, 1024, 1032, 1012, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8229, 8249, 8228, 8345, 8087, 8174, 8101] avg=8192.0000 max=8345 max_rank=4 min=8087 min_rank=5 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[975, 1017, 983, 1028, 1020, 1076, 1020, 1038, 20, 15, 0, 0, 0, 0, 0, 0, 986, 1021, 1004, 1019, 1003, 1001, 1024, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 991, 1050, 995, 1017, 1045, 1072, 1041, 5, 0, 0, 0, 0, 0, 0, 0, 1021, 1021, 1043, 986, 1018, 1038, 965, 1071, 26, 22, 3, 0, 0, 0, 0, 0, 983, 1068, 1000, 1050, 1014, 1072, 1070, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1015, 1041, 979, 1028, 1056, 1034, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 982, 1071, 1026, 981, 1013, 1036, 1021, 2, 0, 0, 0, 0, 0, 0, 0, 1006, 1002, 1051, 974, 1065, 1024, 1000, 1075, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8111, 8276, 8214, 8230, 8164, 8152, 8197] avg=8192.0000 max=8276 max_rank=2 min=8111 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[997, 1069, 1018, 1000, 1051, 1004, 1003, 1010, 18, 14, 0, 0, 0, 0, 0, 0, 1030, 1055, 1055, 981, 1000, 1031, 969, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1011, 1022, 1026, 1038, 1009, 1008, 1000, 4, 0, 0, 0, 0, 0, 0, 0, 1025, 1025, 1015, 1011, 1017, 1050, 995, 997, 27, 23, 3, 0, 0, 0, 0, 0, 1029, 992, 1060, 1019, 1046, 1047, 1049, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1054, 1032, 1024, 1042, 1052, 967, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1067, 1069, 985, 1045, 1037, 1010, 1026, 2, 0, 0, 0, 0, 0, 0, 0, 1040, 1003, 985, 1032, 1007, 1032, 1045, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8147, 8149, 8188, 8238, 8196, 8248, 8186] avg=8192.0000 max=8248 max_rank=6 min=8147 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32868 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8320, 8064, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8064, 8320, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8064, 8320, 8064, 8320, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True + [2026-08-30 05:02:10] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 667.9 | throughput per GPU (TFLOP/s/GPU): 499.1 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.058020E+01 | loss scale: 1.0 | grad norm: 217.585 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1089, 1023, 966, 999, 1016, 1037, 1019, 1028, 1059, 967, 1001, 1038, 946, 1009, 1021, 996, 1059, 1026, 1033, 988, 987, 950, 1074, 1003, 1031, 1016, 1037, 1066, 1035, 1054, 1046, 1041, 1061, 1017, 1036, 994, 1027, 1004, 1026, 1023, 1058, 1070, 1035, 1023, 1045, 978, 1057, 1002, 1039, 1017, 1045, 1007, 1034, 985, 1029, 980, 1018, 1023, 998, 1035, 1027, 1048, 1069, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[995, 1029, 1064, 1030, 1024, 1004, 1020, 963, 1068, 1020, 986, 1016, 1012, 959, 990, 1010, 1040, 1024, 1038, 1008, 1023, 1047, 1042, 1063, 1049, 1034, 1067, 1036, 1054, 1055, 1041, 1009, 1048, 1019, 995, 1030, 1034, 1008, 1025, 1051, 1080, 1015, 1006, 991, 1048, 1031, 1033, 1023, 1031, 1032, 1017, 988, 1041, 1012, 1038, 1000, 979, 1042, 974, 1034, 1014, 1033, 1032, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1023, 1053, 1010, 963, 1002, 998, 1006, 1054, 1029, 1021, 1082, 1026, 1006, 1015, 1031, 1034, 1034, 1060, 1030, 985, 1018, 1009, 1011, 1044, 972, 1042, 1030, 1056, 1046, 1043, 1050, 1016, 1023, 986, 1034, 1015, 982, 1038, 1008, 1041, 1022, 1025, 986, 1029, 1034, 1013, 1026, 1025, 1002, 1014, 991, 987, 1045, 1013, 1017, 1012, 1047, 1081, 1036, 1092, 1053, 1013, 1010, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1019, 1028, 992, 1019, 1050, 1026, 1002, 1063, 1055, 1014, 1006, 1022, 1050, 976, 1054, 1053, 1014, 1015, 980, 1023, 980, 1043, 1026, 1013, 1038, 1022, 1008, 1030, 1051, 1055, 1024, 975, 1075, 1003, 1004, 1035, 1005, 1058, 1002, 1013, 1001, 1028, 1001, 985, 1004, 995, 1034, 1015, 1017, 1049, 1053, 1065, 1036, 1033, 1049, 943, 1040, 992, 989, 1033, 1037, 1127, 1015, 1074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1039, 1034, 995, 1081, 981, 1044, 1016, 1007, 1006, 999, 978, 996, 1016, 1053, 1016, 1047, 1027, 1034, 1033, 1020, 1037, 1011, 1027, 1034, 1030, 987, 1028, 1016, 1054, 993, 1011, 1050, 1003, 1008, 1043, 1025, 1021, 1068, 1044, 1023, 971, 981, 1079, 1009, 999, 1054, 1020, 1025, 1063, 1000, 1044, 1046, 1039, 999, 1064, 1056, 1010, 1034, 1007, 1013, 1039, 1056, 980, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1035, 966, 1018, 982, 1044, 1028, 1053, 1021, 1031, 995, 1068, 1015, 1024, 992, 1013, 1048, 1067, 1044, 1061, 1016, 1046, 1032, 1074, 1008, 1039, 1041, 1050, 989, 1020, 1042, 981, 980, 999, 987, 1020, 1043, 986, 1004, 1006, 1027, 1035, 1020, 1003, 1007, 1033, 1053, 1047, 1042, 1012, 1024, 1044, 1045, 1049, 975, 1022, 1054, 991, 1019, 998, 1030, 1060, 994, 1057, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1001, 1030, 1028, 1036, 1036, 1034, 1027, 1006, 1046, 997, 1049, 1034, 1057, 1002, 967, 1007, 1018, 1061, 1034, 1050, 1004, 1066, 1043, 987, 1015, 989, 1045, 955, 1020, 1047, 951, 1020, 1036, 1013, 1036, 1007, 1045, 1022, 1031, 1030, 1024, 1015, 1014, 1061, 1030, 1028, 996, 1013, 1037, 1067, 989, 1061, 1057, 1007, 1055, 990, 1075, 1036, 1015, 1039, 1029, 1007, 1032, 977] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1045, 965, 1011, 1037, 1023, 1062, 1065, 1022, 1023, 996, 1006, 981, 1067, 1008, 1058, 1037, 1023, 1027, 1035, 1039, 1067, 1042, 1034, 983, 1061, 1015, 985, 1051, 986, 1018, 1026, 980, 1010, 1078, 1018, 980, 1025, 1047, 1022, 965, 1051, 1028, 1032, 1041, 1061, 1016, 1047, 987, 996, 1074, 1018, 1026, 1022, 1009, 1027, 1019, 1035, 1016, 1028, 995, 1020, 1056, 998, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1089, 1023, 966, 999, 1016, 1037, 1019, 1028, 18, 0, 0, 0, 0, 0, 0, 0, 1059, 967, 1001, 1038, 946, 1009, 1021, 996, 30, 11, 0, 0, 0, 0, 0, 0, 1059, 1026, 1033, 988, 987, 950, 1045, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1016, 1037, 1066, 1035, 1046, 1046, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1017, 1036, 994, 1027, 1004, 1026, 1023, 8, 8, 0, 0, 0, 0, 0, 0, 1058, 1070, 1035, 1023, 1045, 978, 1057, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1017, 1045, 1007, 1026, 985, 1029, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1023, 998, 1035, 1027, 1018, 1069, 1066, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8078, 8091, 8318, 8204, 8268, 8128, 8254] avg=8192.0000 max=8318 max_rank=3 min=8078 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[995, 1029, 1064, 1030, 1024, 1004, 1020, 963, 18, 0, 0, 0, 0, 0, 0, 0, 1068, 1020, 986, 1016, 1012, 959, 990, 1010, 29, 11, 0, 0, 0, 0, 0, 0, 1040, 1024, 1038, 1008, 1023, 1047, 1013, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1034, 1067, 1036, 1054, 1047, 1041, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1019, 995, 1030, 1034, 1008, 1025, 1051, 9, 8, 0, 0, 0, 0, 0, 0, 1080, 1015, 1006, 991, 1048, 1031, 1033, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1032, 1017, 988, 1032, 1012, 1038, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1042, 974, 1034, 1014, 1004, 1032, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8101, 8256, 8337, 8227, 8227, 8150, 8091] avg=8192.0000 max=8337 max_rank=3 min=8091 min_rank=7 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1019, 1028, 992, 1019, 1050, 1026, 1002, 1063, 18, 0, 0, 0, 0, 0, 0, 0, 1055, 1014, 1006, 1022, 1050, 976, 1054, 1053, 32, 11, 0, 0, 0, 0, 0, 0, 1014, 1015, 980, 1023, 980, 1043, 997, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1022, 1008, 1030, 1051, 1047, 1024, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1003, 1004, 1035, 1005, 1058, 1002, 1013, 8, 8, 0, 0, 0, 0, 0, 0, 1001, 1028, 1001, 985, 1004, 995, 1034, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1049, 1053, 1065, 1028, 1033, 1049, 943, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 992, 989, 1033, 1037, 1095, 1015, 1074, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8273, 8065, 8195, 8211, 8063, 8237, 8275] avg=8192.0000 max=8275 max_rank=7 min=8063 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1023, 1053, 1010, 963, 1002, 998, 1006, 1054, 17, 0, 0, 0, 0, 0, 0, 0, 1029, 1021, 1082, 1026, 1006, 1015, 1031, 1034, 29, 11, 0, 0, 0, 0, 0, 0, 1034, 1060, 1030, 985, 1018, 1009, 983, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1042, 1030, 1056, 1046, 1035, 1050, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 986, 1034, 1015, 982, 1038, 1008, 1041, 9, 8, 0, 0, 0, 0, 0, 0, 1022, 1025, 986, 1029, 1034, 1013, 1026, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1014, 991, 987, 1036, 1013, 1017, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1081, 1036, 1092, 1053, 984, 1010, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8284, 8163, 8247, 8144, 8160, 8072, 8340] avg=8192.0000 max=8340 max_rank=7 min=8072 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1039, 1034, 995, 1081, 981, 1044, 1016, 1007, 18, 0, 0, 0, 0, 0, 0, 0, 1006, 999, 978, 996, 1016, 1053, 1016, 1047, 30, 11, 0, 0, 0, 0, 0, 0, 1027, 1034, 1033, 1020, 1037, 1011, 998, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 987, 1028, 1016, 1054, 985, 1011, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1008, 1043, 1025, 1021, 1068, 1044, 1023, 8, 8, 0, 0, 0, 0, 0, 0, 971, 981, 1079, 1009, 999, 1054, 1020, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1000, 1044, 1046, 1031, 999, 1064, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1034, 1007, 1013, 1039, 1026, 980, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8152, 8194, 8161, 8251, 8138, 8303, 8122] avg=8192.0000 max=8303 max_rank=6 min=8122 min_rank=7 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1035, 966, 1018, 982, 1044, 1028, 1053, 1021, 18, 0, 0, 0, 0, 0, 0, 0, 1031, 995, 1068, 1015, 1024, 992, 1013, 1048, 28, 11, 0, 0, 0, 0, 0, 0, 1067, 1044, 1061, 1016, 1046, 1032, 1045, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1041, 1050, 989, 1020, 1034, 981, 980, 0, 0, 0, 0, 0, 0, 0, 0, 999, 987, 1020, 1043, 986, 1004, 1006, 1027, 9, 8, 0, 0, 0, 0, 0, 0, 1035, 1020, 1003, 1007, 1033, 1053, 1047, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1024, 1044, 1045, 1040, 975, 1022, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1019, 998, 1030, 1060, 966, 1057, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8225, 8319, 8134, 8089, 8240, 8216, 8148] avg=8192.0000 max=8319 max_rank=2 min=8089 min_rank=4 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1001, 1030, 1028, 1036, 1036, 1034, 1027, 1006, 18, 0, 0, 0, 0, 0, 0, 0, 1046, 997, 1049, 1034, 1057, 1002, 967, 1007, 29, 11, 0, 0, 0, 0, 0, 0, 1018, 1061, 1034, 1050, 1004, 1066, 1014, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 989, 1045, 955, 1020, 1039, 951, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1013, 1036, 1007, 1045, 1022, 1031, 1030, 9, 8, 0, 0, 0, 0, 0, 0, 1024, 1015, 1014, 1061, 1030, 1028, 996, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1067, 989, 1061, 1048, 1007, 1055, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1036, 1015, 1039, 1029, 978, 1032, 977, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8199, 8234, 8034, 8237, 8181, 8254, 8181] avg=8192.0000 max=8254 max_rank=6 min=8034 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1045, 965, 1011, 1037, 1023, 1062, 1065, 1022, 22, 3, 0, 0, 0, 0, 0, 0, 1023, 996, 1006, 981, 1067, 1008, 1058, 1037, 34, 14, 0, 0, 0, 0, 0, 0, 1023, 1027, 1035, 1039, 1067, 1042, 998, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1015, 985, 1051, 986, 1006, 1026, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1078, 1018, 980, 1025, 1047, 1022, 965, 12, 12, 4, 0, 0, 0, 0, 0, 1051, 1028, 1032, 1041, 1061, 1016, 1043, 987, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1074, 1018, 1026, 1007, 1009, 1027, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1016, 1028, 995, 1020, 1022, 998, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8224, 8214, 8110, 8173, 8259, 8176, 8125] avg=8192.0000 max=8259 max_rank=5 min=8110 min_rank=3 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8192, 8192, 8064, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8064, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[987, 994, 1033, 1028, 957, 1049, 982, 1065, 1055, 1000, 981, 987, 999, 1022, 1060, 1034, 1027, 1008, 1032, 1001, 1064, 958, 1027, 1050, 1053, 987, 1027, 1005, 1041, 989, 987, 1022, 1053, 1050, 1034, 1008, 1107, 1034, 1033, 1011, 1084, 1039, 1035, 1079, 1092, 988, 1042, 1045, 992, 1029, 1000, 1052, 1004, 967, 1062, 1047, 1001, 974, 1020, 1031, 1067, 1012, 1019, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1035, 1025, 1000, 1029, 1039, 1066, 1038, 1010, 1044, 1047, 1008, 1010, 1068, 1053, 1053, 974, 1061, 1009, 1043, 1034, 1004, 1055, 1010, 1022, 1023, 1002, 1024, 1045, 1031, 1038, 1045, 1030, 1072, 1044, 997, 1001, 1013, 955, 1035, 1001, 1008, 1024, 1036, 1020, 992, 977, 1040, 1034, 1058, 1023, 971, 1018, 1040, 1044, 1035, 1001, 990, 989, 998, 1053, 984, 1043, 1025, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[994, 1049, 1012, 1018, 1065, 1056, 1012, 1019, 1036, 1000, 1026, 1045, 1042, 1018, 1024, 1055, 1044, 995, 978, 1035, 1023, 1044, 1074, 1020, 1022, 1009, 999, 1033, 1018, 1059, 1037, 1066, 1036, 1001, 1053, 1004, 1011, 1019, 1053, 1035, 1029, 990, 1076, 1034, 1006, 1034, 1036, 998, 1005, 1020, 1007, 1004, 1043, 1010, 1005, 993, 1000, 1020, 1005, 999, 1039, 992, 1051, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1046, 995, 1009, 1017, 1050, 980, 1018, 1000, 1054, 1049, 1054, 1014, 994, 1020, 1056, 1048, 971, 1070, 1037, 1017, 1023, 1008, 992, 1002, 1058, 1046, 979, 1000, 973, 1030, 999, 1029, 1088, 1006, 1098, 1009, 993, 992, 1057, 1038, 988, 1003, 1033, 1021, 992, 979, 1093, 1059, 1004, 993, 1036, 1044, 1049, 1057, 1067, 1008, 1026, 1020, 1004, 989, 1065, 1065, 1014, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[959, 985, 1041, 1034, 999, 1069, 1029, 1070, 1004, 1041, 1053, 995, 1111, 1057, 1011, 1012, 1046, 1053, 1032, 1074, 1041, 1048, 1018, 1025, 1057, 1042, 1013, 988, 1019, 982, 1007, 1047, 979, 1035, 1051, 1033, 993, 982, 1008, 1013, 986, 991, 1001, 1071, 1054, 1028, 1035, 977, 1001, 1018, 998, 1019, 1041, 1016, 1052, 1033, 999, 994, 1052, 1038, 1048, 1002, 1013, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[986, 979, 1056, 1061, 1017, 994, 1013, 1026, 1011, 1028, 1016, 1055, 1003, 1050, 1024, 1022, 1020, 1044, 1039, 993, 997, 1013, 1032, 1094, 1053, 1016, 1001, 1065, 1005, 1039, 991, 1043, 1042, 974, 1011, 993, 1001, 1022, 1049, 1029, 1013, 1020, 1041, 1083, 1029, 975, 1040, 1027, 1040, 1026, 968, 1016, 1031, 1077, 1056, 1049, 1044, 1018, 1008, 1009, 1018, 1014, 1017, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1052, 999, 1037, 1020, 1032, 1029, 1009, 1028, 996, 1018, 965, 1046, 1011, 1021, 1017, 1026, 1064, 996, 1050, 993, 988, 1015, 1059, 1011, 1016, 1034, 978, 971, 1027, 1043, 991, 1064, 1049, 1037, 1041, 1061, 1068, 1039, 1083, 1013, 964, 1032, 996, 1032, 1050, 1044, 1057, 1049, 1037, 1005, 1077, 1014, 1015, 1005, 1074, 974, 1010, 999, 1006, 1001, 1030, 1009, 1021, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[949, 986, 1021, 1076, 1056, 1009, 997, 1080, 1016, 1008, 1020, 994, 1075, 1018, 995, 986, 1066, 982, 1048, 1015, 1003, 1063, 1031, 1031, 1037, 1025, 1007, 1079, 1023, 1037, 1050, 994, 1016, 1032, 1027, 1060, 1047, 942, 1006, 982, 1026, 1041, 1068, 985, 1026, 1027, 1038, 997, 1051, 1040, 1016, 1000, 1032, 1088, 1056, 1007, 1004, 997, 1019, 1015, 1057, 1027, 1000, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[987, 994, 1033, 1028, 957, 1049, 982, 1065, 11, 8, 0, 0, 0, 0, 0, 0, 1055, 1000, 981, 987, 988, 1022, 1060, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1008, 1032, 1001, 1064, 958, 1027, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 987, 1027, 1005, 1041, 989, 987, 1022, 10, 0, 0, 0, 0, 0, 0, 0, 1038, 1050, 1034, 1008, 1107, 1034, 1033, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 1039, 1035, 1079, 1092, 988, 1021, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1029, 1000, 1052, 1004, 967, 1052, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 974, 1020, 1031, 1067, 1012, 1019, 1014, 23, 21, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8127, 8144, 8121, 8315, 8383, 8143, 8189] avg=8192.0000 max=8383 max_rank=5 min=8114 min_rank=0 imbalance_ratio=1.023315 max_load_reduction=57153 max_load_reduction_pct=0.872086 moved_to_echo=33030 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1035, 1025, 1000, 1029, 1039, 1066, 1038, 1010, 12, 8, 0, 0, 0, 0, 0, 0, 1044, 1047, 1008, 1010, 1056, 1053, 1053, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1009, 1043, 1034, 1004, 1055, 1010, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1002, 1024, 1045, 1031, 1038, 1045, 1030, 10, 0, 0, 0, 0, 0, 0, 0, 1057, 1044, 997, 1001, 1013, 955, 1035, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1024, 1036, 1020, 992, 977, 1019, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1023, 971, 1018, 1040, 1044, 1025, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 990, 989, 998, 1053, 984, 1043, 1025, 1040, 23, 21, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8245, 8215, 8248, 8103, 8110, 8180, 8173] avg=8192.0000 max=8262 max_rank=0 min=8103 min_rank=4 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1046, 995, 1009, 1017, 1050, 980, 1018, 1000, 11, 8, 0, 0, 0, 0, 0, 0, 1054, 1049, 1054, 1014, 983, 1020, 1056, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 950, 1070, 1037, 1017, 1023, 1008, 992, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1046, 979, 1000, 973, 1030, 999, 1029, 10, 0, 0, 0, 0, 0, 0, 0, 1073, 1006, 1098, 1009, 993, 992, 1057, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1003, 1033, 1021, 992, 979, 1071, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 993, 1036, 1044, 1049, 1057, 1057, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1020, 1004, 989, 1065, 1065, 1014, 1008, 21, 22, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8278, 8099, 8124, 8266, 8146, 8248, 8241] avg=8192.0000 max=8278 max_rank=1 min=8099 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32901 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[959, 985, 1041, 1034, 999, 1069, 1029, 1070, 12, 7, 0, 0, 0, 0, 0, 0, 1004, 1041, 1053, 995, 1099, 1057, 1011, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1053, 1032, 1074, 1041, 1048, 1018, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1042, 1013, 988, 1019, 982, 1007, 1047, 10, 0, 0, 0, 0, 0, 0, 0, 966, 1035, 1051, 1033, 993, 982, 1008, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 986, 991, 1001, 1071, 1054, 1028, 1014, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1018, 998, 1019, 1041, 1016, 1042, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 999, 994, 1052, 1038, 1048, 1002, 1013, 1013, 23, 21, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8272, 8314, 8165, 8081, 8122, 8168, 8209] avg=8192.0000 max=8314 max_rank=2 min=8081 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[994, 1049, 1012, 1018, 1065, 1056, 1012, 1019, 17, 10, 0, 0, 0, 0, 0, 0, 1036, 1000, 1026, 1045, 1023, 1018, 1024, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 995, 978, 1035, 1023, 1044, 1074, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1009, 999, 1033, 1018, 1059, 1037, 1066, 14, 2, 0, 0, 0, 0, 0, 0, 1015, 1001, 1053, 1004, 1011, 1019, 1053, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 990, 1076, 1034, 1006, 1034, 1011, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1020, 1007, 1004, 1043, 1010, 991, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1020, 1005, 999, 1039, 992, 1051, 1001, 28, 25, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8227, 8185, 8259, 8191, 8178, 8073, 8171] avg=8192.0000 max=8259 max_rank=3 min=8073 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32613 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[986, 979, 1056, 1061, 1017, 994, 1013, 1026, 11, 8, 0, 0, 0, 0, 0, 0, 1011, 1028, 1016, 1055, 992, 1050, 1024, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1044, 1039, 993, 997, 1013, 1032, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1016, 1001, 1065, 1005, 1039, 991, 1043, 10, 0, 0, 0, 0, 0, 0, 0, 1027, 974, 1011, 993, 1001, 1022, 1049, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1020, 1041, 1083, 1029, 975, 1019, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1026, 968, 1016, 1031, 1077, 1046, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1018, 1008, 1009, 1018, 1014, 1017, 1010, 22, 21, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8198, 8210, 8223, 8106, 8207, 8253, 8188] avg=8192.0000 max=8253 max_rank=6 min=8106 min_rank=4 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1052, 999, 1037, 1020, 1032, 1029, 1009, 1028, 11, 8, 0, 0, 0, 0, 0, 0, 996, 1018, 965, 1046, 1000, 1021, 1017, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 996, 1050, 993, 988, 1015, 1059, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1034, 978, 971, 1027, 1043, 991, 1064, 10, 0, 0, 0, 0, 0, 0, 0, 1034, 1037, 1041, 1061, 1068, 1039, 1083, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 964, 1032, 996, 1032, 1050, 1044, 1036, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1005, 1077, 1014, 1015, 1005, 1064, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 999, 1006, 1001, 1030, 1009, 1021, 1038, 23, 21, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8089, 8153, 8134, 8376, 8203, 8191, 8165] avg=8192.0000 max=8376 max_rank=4 min=8089 min_rank=1 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=32935 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[949, 986, 1021, 1076, 1056, 1009, 997, 1080, 12, 7, 0, 0, 0, 0, 0, 0, 1016, 1008, 1020, 994, 1063, 1018, 995, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 982, 1048, 1015, 1003, 1063, 1031, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1025, 1007, 1079, 1023, 1037, 1050, 994, 10, 0, 0, 0, 0, 0, 0, 0, 1002, 1032, 1027, 1060, 1047, 942, 1006, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1041, 1068, 985, 1026, 1027, 1017, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1040, 1016, 1000, 1032, 1088, 1046, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 997, 1019, 1015, 1057, 1027, 1000, 1030, 23, 21, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8100, 8216, 8262, 8098, 8187, 8280, 8200] avg=8192.0000 max=8280 max_rank=6 min=8098 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8192, 8192, 8320, 8192, 8192, 8192, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8064, 8320, 8320, 8320, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8064, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8448, 8320, 8064, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8192, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1045, 1066, 1043, 1008, 1017, 1068, 975, 975, 995, 994, 1014, 1012, 1038, 1019, 998, 1020, 1060, 1012, 1017, 1042, 1027, 1004, 1065, 1070, 1020, 1031, 1017, 1058, 1086, 1011, 1007, 1058, 1013, 1025, 976, 1024, 1017, 1000, 1027, 1047, 966, 1043, 1056, 1076, 1022, 1024, 1011, 977, 1051, 1073, 1030, 1012, 1050, 1017, 1012, 987, 996, 1033, 1014, 989, 1054, 1019, 1001, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[968, 1058, 1024, 1013, 968, 992, 1038, 1049, 1024, 1091, 1054, 1078, 1022, 1034, 969, 1027, 1038, 1045, 961, 1029, 1003, 992, 1058, 1051, 1086, 1013, 1072, 988, 971, 996, 1051, 1017, 975, 1030, 1023, 1026, 1044, 1046, 1020, 1043, 1041, 1047, 1027, 1026, 1044, 1001, 972, 1022, 1019, 1016, 1007, 1021, 1072, 1025, 982, 1019, 998, 1017, 1011, 1027, 990, 1054, 1061, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1000, 997, 995, 1021, 1057, 1013, 1025, 1031, 1011, 981, 1044, 988, 987, 1037, 1066, 1049, 991, 1011, 952, 1041, 979, 1016, 988, 1046, 1002, 1084, 1057, 1054, 1026, 1021, 1017, 1069, 1002, 1042, 1046, 1011, 1023, 1055, 989, 1040, 1038, 1065, 1046, 969, 1018, 1035, 1040, 1014, 1045, 1022, 1028, 1028, 1015, 1017, 1030, 1058, 994, 1044, 1035, 1073, 981, 1022, 1062, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1004, 1050, 967, 1013, 997, 1035, 1030, 1005, 1062, 991, 1064, 1006, 1007, 1048, 1024, 1049, 986, 1070, 975, 992, 966, 1043, 1012, 1028, 1025, 978, 1027, 1110, 1013, 1054, 1044, 999, 1086, 1107, 1068, 1008, 966, 1026, 996, 1018, 1067, 1040, 1016, 1011, 1024, 1045, 1044, 1029, 1027, 1025, 1039, 1009, 996, 1059, 1020, 1010, 987, 1020, 1014, 1010, 999, 1039, 1032, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1017, 1014, 1010, 1002, 1024, 1025, 1026, 987, 1044, 1033, 1025, 1079, 1013, 1009, 1034, 1031, 1026, 1047, 1039, 1019, 1009, 1003, 982, 1012, 1004, 1053, 1008, 1016, 1003, 1041, 1010, 1055, 1023, 1017, 988, 1009, 1001, 1037, 1070, 1029, 1027, 1030, 1028, 1038, 1038, 1004, 991, 1076, 1008, 994, 1053, 986, 1038, 1029, 997, 1037, 1032, 1033, 1105, 1027, 1034, 1015, 1001, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1016, 1041, 1041, 965, 1036, 1030, 1041, 984, 1031, 1024, 1000, 1009, 964, 1044, 1035, 1027, 990, 978, 1035, 1012, 1058, 1004, 1029, 988, 1041, 1081, 1023, 1004, 1052, 1023, 1022, 1014, 1021, 1014, 1077, 1043, 994, 1023, 1059, 1028, 1020, 1004, 1082, 1015, 1055, 1002, 998, 1063, 1053, 1064, 1000, 1041, 1021, 1004, 1073, 1020, 1007, 1009, 1046, 1065, 1006, 1002, 995, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[994, 990, 996, 1009, 1040, 995, 1038, 1048, 994, 1022, 1031, 964, 1035, 1049, 1058, 999, 1030, 1018, 994, 996, 1036, 1037, 1058, 1026, 1029, 994, 993, 1027, 1024, 1060, 1061, 1000, 1017, 1066, 1049, 1013, 1041, 1025, 1036, 1068, 1010, 1046, 1077, 990, 1078, 1032, 1040, 981, 1027, 1039, 1002, 1031, 1018, 983, 1020, 1038, 1015, 1005, 1028, 1062, 1015, 1045, 1014, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1019, 1018, 1069, 961, 999, 1045, 1033, 1019, 1009, 1031, 1053, 1018, 1026, 1032, 993, 1007, 1041, 1052, 1032, 1000, 1062, 1031, 1040, 1044, 992, 1047, 1029, 1091, 1012, 1027, 1026, 1029, 1047, 1038, 995, 969, 986, 978, 1013, 998, 1067, 1049, 1010, 1050, 1041, 997, 1052, 1054, 1026, 1021, 1024, 1020, 1015, 1003, 997, 996, 1024, 1056, 1011, 1040, 999, 1038, 992, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[968, 1058, 1024, 1013, 968, 992, 1038, 1049, 49, 4, 0, 0, 0, 0, 0, 0, 1024, 1091, 1054, 1078, 1022, 1034, 969, 1027, 1, 0, 0, 0, 0, 0, 0, 0, 1038, 1045, 961, 1029, 1003, 992, 1058, 1051, 29, 0, 0, 0, 0, 0, 0, 0, 1086, 1013, 1072, 939, 971, 996, 1051, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1025, 1023, 1026, 1044, 1046, 1020, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1047, 984, 1026, 1044, 1001, 972, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1016, 1007, 1021, 1072, 1025, 982, 1019, 4, 0, 0, 0, 0, 0, 0, 0, 998, 1017, 1011, 1027, 990, 1054, 1061, 1050, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8300, 8206, 8145, 8202, 8137, 8165, 8218] avg=8192.0000 max=8300 max_rank=1 min=8137 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1045, 1066, 1043, 1008, 1017, 1068, 975, 975, 52, 5, 0, 0, 0, 0, 0, 0, 995, 994, 1014, 1012, 1038, 1019, 998, 1020, 1, 0, 0, 0, 0, 0, 0, 0, 1060, 1012, 1017, 1042, 1027, 1004, 1065, 1070, 30, 0, 0, 0, 0, 0, 0, 0, 1020, 1031, 1017, 1006, 1086, 1011, 1007, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1020, 976, 1024, 1017, 1000, 1027, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1043, 1010, 1076, 1022, 1024, 1011, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1073, 1030, 1012, 1050, 1017, 1012, 987, 4, 0, 0, 0, 0, 0, 0, 0, 996, 1033, 1014, 989, 1054, 1019, 1001, 1022, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8091, 8327, 8236, 8124, 8129, 8236, 8139] avg=8192.0000 max=8327 max_rank=2 min=8091 min_rank=1 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1017, 1014, 1010, 1002, 1024, 1025, 1026, 987, 50, 4, 0, 0, 0, 0, 0, 0, 1044, 1033, 1025, 1079, 1013, 1009, 1034, 1031, 1, 0, 0, 0, 0, 0, 0, 0, 1026, 1047, 1039, 1019, 1009, 1003, 982, 1012, 29, 0, 0, 0, 0, 0, 0, 0, 1004, 1053, 1008, 966, 1003, 1041, 1010, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1012, 988, 1009, 1001, 1037, 1070, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1030, 985, 1038, 1038, 1004, 991, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 994, 1053, 986, 1038, 1029, 997, 1037, 4, 0, 0, 0, 0, 0, 0, 0, 1032, 1033, 1105, 1027, 1034, 1015, 1001, 1041, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8269, 8166, 8140, 8169, 8189, 8146, 8298] avg=8192.0000 max=8298 max_rank=7 min=8140 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1000, 997, 995, 1021, 1057, 1013, 1025, 1031, 52, 5, 0, 0, 0, 0, 0, 0, 1011, 981, 1044, 988, 987, 1037, 1066, 1049, 1, 0, 0, 0, 0, 0, 0, 0, 991, 1011, 952, 1041, 979, 1016, 988, 1046, 29, 0, 0, 0, 0, 0, 0, 0, 1002, 1084, 1057, 1002, 1026, 1021, 1017, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1037, 1046, 1011, 1023, 1055, 989, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1065, 1002, 969, 1018, 1035, 1040, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1022, 1028, 1028, 1015, 1017, 1030, 1058, 4, 0, 0, 0, 0, 0, 0, 0, 994, 1044, 1035, 1073, 981, 1022, 1062, 993, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8164, 8053, 8278, 8203, 8181, 8247, 8214] avg=8192.0000 max=8278 max_rank=3 min=8053 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1019, 1018, 1069, 961, 999, 1045, 1033, 1019, 54, 4, 0, 0, 0, 0, 0, 0, 1009, 1031, 1053, 1018, 1026, 1032, 993, 1007, 1, 0, 0, 0, 0, 0, 0, 0, 1041, 1052, 1032, 1000, 1062, 1031, 1040, 1044, 28, 0, 0, 0, 0, 0, 0, 0, 992, 1047, 1029, 1037, 1012, 1027, 1026, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1033, 995, 969, 986, 978, 1013, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1049, 968, 1050, 1041, 997, 1052, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1021, 1024, 1020, 1015, 1003, 997, 996, 4, 0, 0, 0, 0, 0, 0, 0, 1024, 1056, 1011, 1040, 999, 1038, 992, 1043, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8170, 8330, 8199, 8019, 8278, 8106, 8213] avg=8192.0000 max=8330 max_rank=2 min=8019 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32616 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[994, 990, 996, 1009, 1040, 995, 1038, 1048, 51, 5, 0, 0, 0, 0, 0, 0, 994, 1022, 1031, 964, 1035, 1049, 1058, 999, 1, 0, 0, 0, 0, 0, 0, 0, 1030, 1018, 994, 996, 1036, 1037, 1058, 1026, 30, 0, 0, 0, 0, 0, 0, 0, 1029, 994, 993, 976, 1024, 1060, 1061, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1060, 1049, 1013, 1041, 1025, 1036, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1046, 1031, 990, 1078, 1032, 1040, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1039, 1002, 1031, 1018, 983, 1020, 1038, 4, 0, 0, 0, 0, 0, 0, 0, 1015, 1005, 1028, 1062, 1015, 1045, 1014, 980, 11, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8153, 8225, 8137, 8309, 8208, 8162, 8176] avg=8192.0000 max=8309 max_rank=4 min=8137 min_rank=3 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1004, 1050, 967, 1013, 997, 1035, 1030, 1005, 59, 8, 0, 0, 0, 0, 0, 0, 1062, 991, 1064, 1006, 1007, 1048, 1024, 1049, 3, 0, 0, 0, 0, 0, 0, 0, 986, 1070, 975, 992, 966, 1043, 1012, 1028, 33, 0, 0, 0, 0, 0, 0, 0, 1025, 978, 1027, 1051, 1013, 1054, 1044, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 1088, 1068, 1008, 966, 1026, 996, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1040, 961, 1011, 1024, 1045, 1044, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1025, 1039, 1009, 996, 1059, 1020, 1010, 9, 0, 0, 0, 0, 0, 0, 0, 987, 1020, 1014, 1010, 999, 1039, 1032, 1025, 14, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8254, 8105, 8191, 8256, 8221, 8194, 8147] avg=8192.0000 max=8256 max_rank=4 min=8105 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1016, 1041, 1041, 965, 1036, 1030, 1041, 984, 50, 5, 0, 0, 0, 0, 0, 0, 1031, 1024, 1000, 1009, 964, 1044, 1035, 1027, 1, 0, 0, 0, 0, 0, 0, 0, 990, 978, 1035, 1012, 1058, 1004, 1029, 988, 30, 0, 0, 0, 0, 0, 0, 0, 1041, 1081, 1023, 954, 1052, 1023, 1022, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1009, 1077, 1043, 994, 1023, 1059, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1004, 1036, 1015, 1055, 1002, 998, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1064, 1000, 1041, 1021, 1004, 1073, 1020, 4, 0, 0, 0, 0, 0, 0, 0, 1007, 1009, 1046, 1065, 1006, 1002, 995, 990, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8135, 8124, 8210, 8254, 8193, 8280, 8131] avg=8192.0000 max=8280 max_rank=6 min=8124 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8192, 8064, 8192, 8320, 8320, 8192, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8192, 8192, 8192, 8320, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 7936, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8448, 8064, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[996, 994, 1042, 1037, 1025, 1011, 1011, 1015, 1004, 974, 1010, 1044, 987, 1015, 1047, 1084, 1003, 976, 1021, 1067, 1038, 1008, 1013, 1091, 1024, 979, 1021, 1063, 1053, 1008, 1052, 1030, 1061, 1025, 1069, 1051, 1012, 1021, 1038, 1018, 1031, 1024, 1031, 1024, 1066, 1001, 972, 1075, 1072, 1037, 988, 995, 1015, 1019, 1018, 1027, 981, 1010, 971, 1034, 1034, 981, 1052, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[974, 1015, 994, 1038, 998, 991, 1049, 1089, 979, 1067, 1033, 989, 992, 1021, 1068, 1052, 1024, 1022, 968, 987, 1045, 1035, 987, 985, 1021, 1006, 1025, 1044, 1079, 1006, 1039, 1007, 1011, 967, 1084, 1018, 1025, 1029, 1045, 1063, 992, 1015, 1023, 1064, 1072, 1045, 1100, 1007, 970, 1015, 1002, 984, 1046, 1033, 1060, 976, 1044, 1052, 989, 1003, 1061, 995, 1053, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1026, 993, 1103, 1018, 1046, 1054, 1079, 1031, 1057, 1019, 977, 1012, 1048, 1015, 987, 984, 1052, 1085, 981, 1021, 1047, 1067, 1003, 996, 1039, 1016, 1025, 965, 1035, 968, 1029, 1029, 1059, 1032, 1026, 1026, 1024, 1027, 1077, 991, 1027, 1004, 1015, 1031, 1043, 966, 1022, 1012, 1008, 1052, 979, 1003, 1022, 1037, 1040, 1011, 1042, 992, 1012, 1034, 1029, 986, 1086, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1066, 1058, 968, 1026, 1027, 1014, 1046, 1012, 1035, 979, 1005, 1044, 1057, 1027, 1074, 974, 1034, 1065, 1024, 1047, 1016, 1016, 1046, 1047, 1004, 1079, 984, 1012, 1011, 1010, 982, 1049, 999, 1059, 1020, 1003, 1062, 1070, 1004, 1050, 1019, 1021, 1032, 995, 1051, 1031, 1023, 1043, 972, 1048, 990, 1068, 1015, 1033, 987, 1008, 1000, 1009, 986, 1026, 1001, 1029, 1028, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1006, 983, 1005, 1044, 990, 1097, 1051, 1090, 1060, 1013, 1086, 1042, 1028, 1029, 1050, 1032, 995, 972, 1023, 1035, 1029, 1001, 1037, 1043, 999, 994, 1017, 992, 997, 1050, 1010, 1037, 983, 993, 1051, 1011, 1017, 1016, 1010, 1008, 1027, 1057, 976, 1066, 989, 1052, 1002, 1006, 1015, 1096, 1032, 1042, 1039, 1031, 1038, 1054, 999, 1015, 1010, 1048, 988, 983, 1034, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1004, 962, 1020, 1026, 1050, 974, 1040, 1033, 1036, 1074, 1091, 1059, 981, 1027, 1029, 1011, 1047, 1051, 1026, 1037, 1037, 1019, 1026, 1056, 995, 998, 995, 1017, 1038, 1000, 1019, 973, 1041, 1047, 1037, 1036, 947, 1011, 1030, 1015, 1028, 1064, 983, 973, 1025, 1021, 1069, 1034, 1003, 1021, 1038, 990, 1045, 990, 1051, 1015, 1017, 1062, 1026, 1054, 1037, 1027, 1021, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1012, 1020, 1004, 1051, 1000, 997, 1040, 1006, 987, 1058, 1036, 1025, 1015, 984, 1060, 1003, 998, 1042, 1021, 1045, 1049, 985, 1049, 1005, 1029, 1019, 1006, 1021, 1002, 1050, 1081, 1035, 1031, 1095, 996, 993, 986, 1018, 1072, 1016, 990, 1080, 1030, 1000, 1028, 1002, 1040, 1011, 1000, 1063, 980, 1039, 1025, 1040, 978, 1041, 1004, 998, 1009, 1072, 1030, 1047, 1038, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1014, 1024, 1000, 1070, 1003, 1030, 977, 1023, 1080, 1001, 1018, 1013, 986, 995, 1053, 1047, 1061, 1012, 1070, 989, 1008, 1004, 1027, 1053, 968, 1043, 1052, 1047, 1002, 1057, 1041, 1025, 1001, 1044, 1017, 1022, 989, 985, 980, 1055, 1026, 1012, 989, 1014, 1020, 1062, 1014, 1053, 1026, 1061, 1035, 1005, 950, 1069, 1005, 1127, 1002, 1053, 1002, 1014, 984, 1039, 1048, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1026, 993, 1103, 1018, 1046, 1054, 1079, 1031, 5, 0, 0, 0, 0, 0, 0, 0, 1057, 1019, 977, 1012, 1048, 1015, 972, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1085, 981, 1021, 1047, 1067, 1003, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1016, 1025, 965, 1035, 968, 1029, 1029, 15, 12, 0, 0, 0, 0, 0, 0, 1059, 1032, 1017, 1026, 1024, 1027, 1077, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1004, 1015, 1031, 1034, 966, 1022, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1052, 979, 1003, 1022, 1037, 1040, 1011, 4, 2, 0, 0, 0, 0, 0, 0, 1042, 992, 1012, 1034, 1029, 986, 1086, 1014, 7, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8355, 8084, 8237, 8133, 8253, 8111, 8158, 8205] avg=8192.0000 max=8355 max_rank=0 min=8084 min_rank=1 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[996, 994, 1042, 1037, 1025, 1011, 1011, 1015, 5, 0, 0, 0, 0, 0, 0, 0, 1004, 974, 1010, 1044, 987, 1015, 1031, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 976, 1021, 1067, 1038, 1008, 1013, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 979, 1021, 1063, 1053, 1008, 1052, 1030, 16, 13, 0, 0, 0, 0, 0, 0, 1061, 1025, 1059, 1051, 1012, 1021, 1038, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1024, 1031, 1024, 1057, 1001, 972, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1037, 988, 995, 1015, 1019, 1018, 1027, 5, 2, 0, 0, 0, 0, 0, 0, 981, 1010, 971, 1034, 1034, 981, 1052, 1040, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8149, 8200, 8259, 8285, 8215, 8178, 8114] avg=8192.0000 max=8285 max_rank=4 min=8114 min_rank=7 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1006, 983, 1005, 1044, 990, 1097, 1051, 1090, 5, 0, 0, 0, 0, 0, 0, 0, 1060, 1013, 1086, 1042, 1028, 1029, 1034, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 995, 972, 1023, 1035, 1029, 1001, 1037, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 999, 994, 1017, 992, 997, 1050, 1010, 1037, 16, 12, 0, 0, 0, 0, 0, 0, 983, 993, 1042, 1011, 1017, 1016, 1010, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1057, 976, 1066, 980, 1052, 1002, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1096, 1032, 1042, 1039, 1031, 1038, 1054, 4, 2, 0, 0, 0, 0, 0, 0, 999, 1015, 1010, 1048, 988, 983, 1034, 1011, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8324, 8119, 8124, 8080, 8166, 8353, 8099] avg=8192.0000 max=8353 max_rank=6 min=8080 min_rank=4 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[974, 1015, 994, 1038, 998, 991, 1049, 1089, 5, 0, 0, 0, 0, 0, 0, 0, 979, 1067, 1033, 989, 992, 1021, 1052, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1022, 968, 987, 1045, 1035, 987, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1006, 1025, 1044, 1079, 1006, 1039, 1007, 16, 11, 0, 0, 0, 0, 0, 0, 1011, 967, 1074, 1018, 1025, 1029, 1045, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1015, 1023, 1064, 1063, 1045, 1100, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1015, 1002, 984, 1046, 1033, 1060, 976, 5, 2, 0, 0, 0, 0, 0, 0, 1044, 1052, 989, 1003, 1061, 995, 1053, 1064, 7, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8185, 8039, 8254, 8232, 8309, 8093, 8271] avg=8192.0000 max=8309 max_rank=5 min=8039 min_rank=2 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32905 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1066, 1058, 968, 1026, 1027, 1014, 1046, 1012, 5, 0, 0, 0, 0, 0, 0, 0, 1035, 979, 1005, 1044, 1057, 1027, 1057, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1065, 1024, 1047, 1016, 1016, 1046, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1079, 984, 1012, 1011, 1010, 982, 1049, 17, 12, 0, 0, 0, 0, 0, 0, 999, 1059, 1011, 1003, 1062, 1070, 1004, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1021, 1032, 995, 1042, 1031, 1023, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1048, 990, 1068, 1015, 1033, 987, 1008, 4, 2, 0, 0, 0, 0, 0, 0, 1000, 1009, 986, 1026, 1001, 1029, 1028, 1016, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8178, 8279, 8160, 8258, 8206, 8127, 8106] avg=8192.0000 max=8279 max_rank=2 min=8106 min_rank=7 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1004, 962, 1020, 1026, 1050, 974, 1040, 1033, 5, 0, 0, 0, 0, 0, 0, 0, 1036, 1074, 1091, 1059, 981, 1027, 1013, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1051, 1026, 1037, 1037, 1019, 1026, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 995, 998, 995, 1017, 1038, 1000, 1019, 973, 16, 12, 0, 0, 0, 0, 0, 0, 1041, 1047, 1028, 1036, 947, 1011, 1030, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1064, 983, 973, 1016, 1021, 1069, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1021, 1038, 990, 1045, 990, 1051, 1015, 4, 2, 0, 0, 0, 0, 0, 0, 1017, 1062, 1026, 1054, 1037, 1027, 1021, 1027, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8292, 8283, 8063, 8155, 8188, 8159, 8282] avg=8192.0000 max=8292 max_rank=1 min=8063 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1012, 1020, 1004, 1051, 1000, 997, 1040, 1006, 9, 0, 0, 0, 0, 0, 0, 0, 987, 1058, 1036, 1025, 1015, 984, 1039, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1042, 1021, 1045, 1049, 985, 1049, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1019, 1006, 1021, 1002, 1050, 1081, 1035, 21, 16, 0, 0, 0, 0, 0, 0, 1031, 1095, 978, 993, 986, 1018, 1072, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1080, 1030, 1000, 1007, 1002, 1040, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1063, 980, 1039, 1025, 1040, 978, 1041, 9, 9, 0, 0, 0, 0, 0, 0, 1004, 998, 1009, 1072, 1030, 1047, 1038, 1049, 12, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8147, 8171, 8280, 8189, 8160, 8184, 8266] avg=8192.0000 max=8280 max_rank=3 min=8139 min_rank=0 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1014, 1024, 1000, 1070, 1003, 1030, 977, 1023, 5, 0, 0, 0, 0, 0, 0, 0, 1080, 1001, 1018, 1013, 986, 995, 1037, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1012, 1070, 989, 1008, 1004, 1027, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1043, 1052, 1047, 1002, 1057, 1041, 1025, 16, 12, 0, 0, 0, 0, 0, 0, 1001, 1044, 1008, 1022, 989, 985, 980, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1012, 989, 1014, 1011, 1062, 1014, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1061, 1035, 1005, 950, 1069, 1005, 1127, 4, 2, 0, 0, 0, 0, 0, 0, 1002, 1053, 1002, 1014, 984, 1039, 1048, 1040, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8177, 8208, 8263, 8084, 8181, 8284, 8193] avg=8192.0000 max=8284 max_rank=6 min=8084 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8320, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8064, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8064, 8192, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8064, 8320, 8192, 8192, 8448, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1057, 1007, 1008, 1054, 984, 1031, 989, 1031, 1010, 995, 1046, 981, 1040, 1025, 946, 1058, 1019, 1013, 1032, 1004, 1025, 1065, 1033, 1018, 1042, 1023, 1050, 1013, 992, 951, 1057, 1021, 982, 1038, 1022, 1063, 1037, 1032, 968, 1023, 1048, 1050, 1020, 993, 1050, 1039, 1039, 994, 1002, 991, 1066, 1027, 1087, 1032, 1066, 970, 1023, 1062, 1079, 1084, 997, 966, 1026, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1049, 1001, 1001, 1014, 1106, 1013, 1021, 1054, 1012, 989, 988, 990, 995, 1004, 989, 1069, 1005, 1055, 1048, 1040, 969, 990, 996, 1016, 987, 1059, 1023, 1081, 1014, 1030, 1022, 1050, 1022, 999, 1033, 1048, 1008, 1051, 1008, 1011, 1050, 1020, 989, 1060, 999, 1024, 1094, 1040, 1003, 992, 1078, 1024, 1011, 1032, 1004, 1049, 1061, 980, 973, 1013, 1015, 1055, 1050, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[947, 1000, 1074, 1069, 1009, 1024, 1056, 1005, 1029, 1006, 1007, 1043, 1022, 987, 1027, 1023, 998, 1039, 993, 1006, 1021, 1024, 984, 1047, 1067, 1030, 1022, 1015, 1048, 1067, 1066, 991, 987, 1053, 1019, 1028, 1052, 1056, 1021, 1047, 1013, 1021, 1033, 1031, 1082, 1072, 966, 1034, 1036, 1015, 1013, 995, 1013, 1045, 1013, 1011, 987, 1013, 1030, 1032, 992, 1029, 1040, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1029, 982, 1028, 1029, 1047, 1007, 1043, 992, 1036, 1044, 1055, 1008, 1022, 1002, 1028, 1063, 1025, 1019, 1056, 1004, 1024, 1074, 1011, 1050, 976, 1062, 1012, 1038, 997, 1043, 1027, 1026, 1033, 1036, 1032, 996, 1006, 1048, 1037, 1066, 1011, 1062, 1003, 969, 1045, 1028, 1095, 990, 1017, 1034, 1080, 1015, 1039, 998, 969, 1018, 1021, 1035, 984, 1002, 1002, 1015, 987, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1029, 1089, 1018, 1056, 1008, 979, 1031, 1041, 1023, 1034, 1010, 1048, 994, 1040, 988, 1033, 1023, 1001, 1011, 996, 1025, 1040, 1022, 969, 1022, 1032, 1053, 1026, 1051, 1018, 1011, 1038, 1022, 993, 1017, 1014, 1047, 1064, 1012, 1031, 1022, 1006, 1088, 991, 1049, 1036, 999, 1052, 1024, 1019, 1084, 998, 1078, 1022, 993, 1051, 1009, 1043, 1013, 1025, 1014, 1017, 964, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1017, 1015, 1015, 1023, 1019, 1020, 1029, 1028, 1022, 992, 1021, 1017, 1017, 1053, 1028, 1059, 1005, 996, 1028, 976, 969, 1017, 1044, 1017, 1010, 967, 1026, 981, 1056, 993, 1011, 1017, 1060, 1036, 1007, 1058, 1062, 1028, 1071, 1084, 1027, 1057, 1059, 1010, 1025, 1011, 1019, 1032, 1014, 1005, 1015, 983, 1039, 986, 1006, 1025, 1032, 1048, 1048, 1083, 1080, 1015, 1002, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[997, 1004, 960, 1054, 1020, 991, 1022, 1032, 994, 983, 1047, 1046, 1048, 1039, 1014, 1056, 1083, 1029, 1003, 1033, 1056, 1053, 1052, 1061, 1027, 1021, 1010, 1042, 1056, 1053, 985, 977, 1070, 1021, 1010, 986, 1052, 1067, 1022, 1009, 1018, 1021, 996, 1038, 1031, 1057, 975, 996, 1005, 1012, 1003, 1064, 968, 1032, 1002, 1015, 1038, 1005, 1081, 1015, 1014, 979, 1059, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1002, 991, 976, 1022, 1042, 1038, 1059, 1059, 1039, 1023, 1028, 998, 992, 1055, 957, 1043, 1041, 1055, 1058, 971, 1047, 1037, 1073, 1047, 997, 1037, 1012, 1039, 1013, 975, 1052, 1016, 1030, 1082, 1024, 1044, 1061, 1027, 964, 1003, 1032, 1032, 1052, 1031, 1022, 1011, 997, 991, 1051, 984, 1022, 1017, 1053, 1052, 1027, 1008, 1022, 1032, 992, 989, 1025, 1033, 999, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1049, 1001, 1001, 1014, 1106, 1013, 1021, 1054, 10, 0, 0, 0, 0, 0, 0, 0, 1012, 989, 988, 990, 995, 1004, 989, 1069, 32, 0, 0, 0, 0, 0, 0, 0, 1005, 1055, 1048, 1040, 969, 990, 996, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1059, 1023, 1081, 1014, 1030, 1022, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 999, 1033, 1048, 1008, 1002, 1008, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1020, 989, 1060, 975, 1024, 1094, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 992, 1078, 1024, 1011, 1032, 1004, 1049, 16, 0, 0, 0, 0, 0, 0, 0, 1061, 980, 973, 1013, 1015, 1055, 1050, 1060, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8068, 8119, 8266, 8131, 8252, 8209, 8222] avg=8192.0000 max=8269 max_rank=0 min=8068 min_rank=1 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1029, 1089, 1018, 1056, 1008, 979, 1031, 1041, 11, 0, 0, 0, 0, 0, 0, 0, 1023, 1034, 1010, 1048, 994, 1040, 988, 1033, 32, 0, 0, 0, 0, 0, 0, 0, 1023, 1001, 1011, 996, 1025, 1040, 1022, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1032, 1053, 1026, 1051, 1018, 1011, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 993, 1017, 1014, 1047, 1014, 1012, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1006, 1088, 991, 1024, 1036, 999, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1019, 1084, 998, 1078, 1022, 993, 1051, 17, 0, 0, 0, 0, 0, 0, 0, 1009, 1043, 1013, 1025, 1014, 1017, 964, 980, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8202, 8087, 8251, 8150, 8218, 8286, 8080] avg=8192.0000 max=8286 max_rank=6 min=8080 min_rank=7 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1057, 1007, 1008, 1054, 984, 1031, 989, 1031, 11, 0, 0, 0, 0, 0, 0, 0, 1010, 995, 1046, 981, 1040, 1025, 946, 1058, 31, 0, 0, 0, 0, 0, 0, 0, 1019, 1013, 1032, 1004, 1025, 1065, 1033, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1023, 1050, 1013, 992, 951, 1057, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1038, 1022, 1063, 1037, 984, 968, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1050, 1020, 993, 1025, 1039, 1039, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 991, 1066, 1027, 1087, 1032, 1066, 970, 16, 0, 0, 0, 0, 0, 0, 0, 1023, 1062, 1079, 1084, 997, 966, 1026, 1040, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8132, 8209, 8149, 8117, 8208, 8257, 8292] avg=8192.0000 max=8292 max_rank=7 min=8117 min_rank=4 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[947, 1000, 1074, 1069, 1009, 1024, 1056, 1005, 11, 0, 0, 0, 0, 0, 0, 0, 1029, 1006, 1007, 1043, 1022, 987, 1027, 1023, 32, 0, 0, 0, 0, 0, 0, 0, 998, 1039, 993, 1006, 1021, 1024, 984, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1030, 1022, 1015, 1048, 1067, 1066, 991, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1053, 1019, 1028, 1052, 1007, 1021, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1021, 1033, 1031, 1056, 1072, 966, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1015, 1013, 995, 1013, 1045, 1013, 1011, 16, 0, 0, 0, 0, 0, 0, 0, 987, 1013, 1030, 1032, 992, 1029, 1040, 1011, 15, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8176, 8112, 8306, 8214, 8226, 8157, 8150] avg=8192.0000 max=8306 max_rank=3 min=8112 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1029, 982, 1028, 1029, 1047, 1007, 1043, 992, 11, 0, 0, 0, 0, 0, 0, 0, 1036, 1044, 1055, 1008, 1022, 1002, 1028, 1063, 32, 0, 0, 0, 0, 0, 0, 0, 1025, 1019, 1056, 1004, 1024, 1074, 1011, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1062, 1012, 1038, 997, 1043, 1027, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1036, 1032, 996, 1006, 999, 1037, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1062, 1003, 969, 1020, 1028, 1095, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1034, 1080, 1015, 1039, 998, 969, 1018, 16, 0, 0, 0, 0, 0, 0, 0, 1021, 1035, 984, 1002, 1002, 1015, 987, 1004, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8290, 8263, 8181, 8205, 8178, 8186, 8065] avg=8192.0000 max=8290 max_rank=1 min=8065 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1017, 1015, 1015, 1023, 1019, 1020, 1029, 1028, 14, 0, 0, 0, 0, 0, 0, 0, 1022, 992, 1021, 1017, 1017, 1053, 1028, 1059, 34, 0, 0, 0, 0, 0, 0, 0, 1005, 996, 1028, 976, 969, 1012, 1044, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 967, 1026, 981, 1056, 993, 1011, 1017, 5, 5, 0, 0, 0, 0, 0, 0, 1060, 1036, 1007, 1058, 1062, 967, 1071, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1057, 1059, 1010, 988, 1011, 1019, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1005, 1015, 983, 1039, 986, 1006, 1025, 20, 0, 0, 0, 0, 0, 0, 0, 1032, 1048, 1048, 1083, 1080, 1015, 1002, 1021, 18, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8243, 8047, 8071, 8345, 8203, 8093, 8354] avg=8192.0000 max=8354 max_rank=7 min=8047 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32995 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[997, 1004, 960, 1054, 1020, 991, 1022, 1032, 11, 0, 0, 0, 0, 0, 0, 0, 994, 983, 1047, 1046, 1048, 1039, 1014, 1056, 32, 0, 0, 0, 0, 0, 0, 0, 1083, 1029, 1003, 1033, 1056, 1053, 1052, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1021, 1010, 1042, 1056, 1053, 985, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1021, 1010, 986, 1052, 1017, 1022, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1021, 996, 1038, 1006, 1057, 975, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1012, 1003, 1064, 968, 1032, 1002, 1015, 17, 0, 0, 0, 0, 0, 0, 0, 1038, 1005, 1081, 1015, 1014, 979, 1059, 1027, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8091, 8259, 8370, 8171, 8187, 8107, 8118, 8233] avg=8192.0000 max=8370 max_rank=2 min=8091 min_rank=0 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1002, 991, 976, 1022, 1042, 1038, 1059, 1059, 10, 0, 0, 0, 0, 0, 0, 0, 1039, 1023, 1028, 998, 992, 1055, 957, 1043, 31, 0, 0, 0, 0, 0, 0, 0, 1041, 1055, 1058, 971, 1047, 1037, 1073, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1037, 1012, 1039, 1013, 975, 1052, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1082, 1024, 1044, 1061, 979, 964, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1032, 1052, 1031, 998, 1011, 997, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 984, 1022, 1017, 1053, 1052, 1027, 1008, 16, 0, 0, 0, 0, 0, 0, 0, 1022, 1032, 992, 989, 1025, 1033, 999, 1033, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8166, 8329, 8141, 8187, 8144, 8230, 8140] avg=8192.0000 max=8329 max_rank=2 min=8140 min_rank=7 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8448, 8320, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8448, 8064, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8064, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8448, 8192, 8320, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8064, 8448, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1051, 979, 999, 1045, 1013, 1023, 1030, 1002, 1064, 1057, 994, 1010, 1006, 994, 999, 1011, 1044, 978, 1015, 1033, 1018, 1001, 1092, 1093, 1019, 1043, 1038, 989, 1006, 1000, 1048, 1019, 1039, 1056, 1021, 1019, 1020, 1059, 1032, 1068, 970, 1007, 1010, 970, 1036, 995, 996, 1030, 1065, 1038, 1028, 978, 1026, 1046, 1055, 1015, 1059, 977, 1005, 1036, 1045, 1058, 990, 1074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1019, 943, 979, 984, 1011, 1037, 999, 1004, 1033, 1030, 1074, 1072, 1003, 1021, 1021, 1042, 1017, 984, 1046, 993, 1042, 999, 1042, 974, 1027, 1036, 1046, 1010, 1037, 995, 1045, 1022, 1033, 1023, 1029, 1032, 1062, 1001, 1039, 1016, 1036, 1031, 1046, 1068, 1046, 1011, 1022, 988, 1068, 1025, 971, 1047, 1035, 1018, 1054, 1035, 1044, 999, 1030, 1019, 1033, 1000, 1051, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1017, 1031, 994, 1051, 1021, 1032, 1034, 1023, 1011, 1042, 974, 1027, 993, 1003, 969, 999, 1027, 993, 1008, 1014, 1044, 1071, 1044, 1042, 1059, 1001, 1076, 1025, 1039, 1015, 1101, 1013, 1029, 1013, 1035, 995, 1056, 1013, 1034, 972, 1025, 1018, 965, 1043, 1040, 1014, 1048, 1012, 1017, 1065, 1053, 1026, 1006, 1012, 1030, 1065, 1025, 970, 1089, 1000, 1034, 1028, 1009, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[983, 1069, 977, 1082, 1018, 1056, 990, 1088, 1018, 993, 999, 1001, 1037, 1014, 1050, 991, 1020, 1036, 991, 1052, 991, 1021, 1008, 999, 1029, 1034, 977, 1030, 1010, 1043, 1007, 1063, 1026, 1031, 1098, 1036, 1042, 1038, 1021, 1017, 1035, 1047, 1036, 1049, 1021, 993, 1021, 986, 1001, 1043, 1011, 1074, 1052, 1020, 969, 1004, 1030, 1051, 993, 999, 1007, 1066, 1029, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1027, 1024, 1034, 1028, 1024, 1027, 1052, 992, 1007, 988, 1022, 1086, 1011, 1063, 1004, 1007, 1049, 1007, 996, 1024, 1028, 1006, 1013, 1004, 996, 1040, 1004, 1019, 1008, 1024, 989, 1033, 995, 1018, 963, 1052, 1017, 1058, 1047, 984, 1032, 1063, 1067, 994, 997, 1019, 1084, 1028, 1047, 1044, 972, 1039, 988, 1053, 1073, 995, 1018, 1059, 1025, 1019, 1101, 1039, 963, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[963, 1018, 1009, 1028, 1032, 1057, 1035, 953, 1057, 1029, 1020, 1088, 1043, 1112, 1044, 1038, 1011, 1019, 1071, 971, 1005, 1084, 1047, 1030, 1047, 1032, 992, 1057, 1051, 995, 987, 997, 1040, 991, 1002, 967, 949, 1022, 1001, 957, 1074, 1037, 1027, 1042, 1078, 1003, 1012, 1075, 1057, 1091, 1042, 1038, 947, 1033, 1051, 1020, 982, 983, 1020, 1052, 1009, 996, 1014, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[985, 1030, 1005, 1000, 989, 996, 1032, 1024, 1041, 1045, 1015, 1064, 1053, 984, 1040, 1069, 1005, 1015, 1029, 1076, 1036, 1003, 1098, 1071, 949, 1063, 1039, 998, 1007, 989, 1043, 997, 1007, 1000, 1063, 1018, 1046, 1058, 1080, 1032, 1048, 1001, 1015, 983, 1003, 986, 1035, 1012, 1002, 1042, 991, 1005, 1055, 1014, 1033, 1009, 1011, 1033, 1051, 1046, 1020, 1012, 1004, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1032, 1028, 1020, 1055, 1047, 992, 986, 1059, 1025, 1011, 1036, 1051, 1070, 1061, 1027, 1003, 1020, 1007, 1044, 1037, 992, 1007, 1002, 996, 1024, 1020, 1035, 1043, 1024, 1027, 1013, 1068, 1005, 1031, 1092, 994, 996, 1028, 1065, 1010, 1002, 1041, 1044, 1031, 991, 1027, 993, 1037, 1021, 1020, 1038, 1029, 1007, 1013, 1018, 995, 1036, 1052, 1028, 976, 953, 988, 1077, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1051, 979, 999, 1045, 1013, 1023, 1030, 1002, 27, 17, 0, 0, 0, 0, 0, 0, 1064, 1057, 994, 983, 1006, 994, 999, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 978, 1015, 1033, 1018, 1001, 1092, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1043, 1038, 989, 1006, 1000, 1048, 1019, 2, 0, 0, 0, 0, 0, 0, 0, 1039, 1056, 1021, 1019, 1020, 1059, 1027, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1007, 1010, 970, 1036, 995, 996, 1030, 3, 0, 0, 0, 0, 0, 0, 0, 1065, 1015, 1028, 978, 1026, 1046, 1055, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 977, 1005, 1036, 1045, 1058, 990, 1074, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8108, 8274, 8164, 8309, 8017, 8228, 8250] avg=8192.0000 max=8309 max_rank=4 min=8017 min_rank=5 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[983, 1069, 977, 1082, 1018, 1056, 990, 1088, 27, 17, 0, 0, 0, 0, 0, 0, 1018, 993, 999, 974, 1037, 1014, 1050, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1036, 991, 1052, 991, 1021, 1008, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1034, 977, 1030, 1010, 1043, 1007, 1063, 2, 0, 0, 0, 0, 0, 0, 0, 1026, 1031, 1098, 1036, 1042, 1038, 1016, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1047, 1036, 1049, 1021, 993, 1021, 986, 3, 0, 0, 0, 0, 0, 0, 0, 1001, 1020, 1011, 1074, 1052, 1020, 969, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1051, 993, 999, 1007, 1066, 1029, 1013, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 8076, 8118, 8195, 8304, 8191, 8151, 8194] avg=8192.0000 max=8307 max_rank=0 min=8076 min_rank=1 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[963, 1018, 1009, 1028, 1032, 1057, 1035, 953, 29, 18, 0, 0, 0, 0, 0, 0, 1057, 1029, 1020, 1059, 1043, 1112, 1044, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1019, 1071, 971, 1005, 1084, 1047, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1032, 992, 1057, 1051, 995, 987, 997, 2, 0, 0, 0, 0, 0, 0, 0, 1040, 991, 1002, 967, 949, 1022, 996, 957, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1037, 1027, 1042, 1078, 1003, 1012, 1075, 3, 1, 0, 0, 0, 0, 0, 0, 1057, 1066, 1042, 1038, 947, 1033, 1051, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 982, 983, 1020, 1052, 1009, 996, 1014, 1002, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8402, 8238, 8160, 7924, 8352, 8254, 8064] avg=8192.0000 max=8402 max_rank=1 min=7924 min_rank=4 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1017, 1031, 994, 1051, 1021, 1032, 1034, 1023, 28, 17, 0, 0, 0, 0, 0, 0, 1011, 1042, 974, 999, 993, 1003, 969, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 993, 1008, 1014, 1044, 1071, 1044, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1001, 1076, 1025, 1039, 1015, 1101, 1013, 2, 0, 0, 0, 0, 0, 0, 0, 1029, 1013, 1035, 995, 1056, 1013, 1029, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1018, 965, 1043, 1040, 1014, 1048, 1012, 3, 1, 0, 0, 0, 0, 0, 0, 1017, 1041, 1053, 1026, 1006, 1012, 1030, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 970, 1089, 1000, 1034, 1028, 1009, 1002, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 7990, 8243, 8331, 8142, 8169, 8250, 8163] avg=8192.0000 max=8331 max_rank=3 min=7990 min_rank=1 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1027, 1024, 1034, 1028, 1024, 1027, 1052, 992, 29, 17, 0, 0, 0, 0, 0, 0, 1007, 988, 1022, 1057, 1011, 1063, 1004, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1007, 996, 1024, 1028, 1006, 1013, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1040, 1004, 1019, 1008, 1024, 989, 1033, 3, 0, 0, 0, 0, 0, 0, 0, 995, 1018, 963, 1052, 1017, 1058, 1040, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1063, 1067, 994, 997, 1019, 1084, 1028, 4, 0, 0, 0, 0, 0, 0, 0, 1047, 1021, 972, 1039, 988, 1053, 1073, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1059, 1025, 1019, 1101, 1039, 963, 1047, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8159, 8127, 8116, 8127, 8288, 8188, 8277] avg=8192.0000 max=8288 max_rank=5 min=8116 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1019, 943, 979, 984, 1011, 1037, 999, 1004, 33, 20, 0, 0, 0, 0, 0, 0, 1033, 1030, 1074, 1039, 1003, 1021, 1021, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 984, 1046, 993, 1042, 999, 1042, 974, 1, 0, 0, 0, 0, 0, 0, 0, 1027, 1036, 1046, 1010, 1037, 995, 1045, 1022, 7, 0, 0, 0, 0, 0, 0, 0, 1033, 1023, 1029, 1032, 1062, 1001, 1023, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1031, 1046, 1068, 1046, 1011, 1022, 988, 8, 6, 0, 0, 0, 0, 0, 0, 1068, 990, 971, 1047, 1035, 1018, 1054, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 999, 1030, 1019, 1033, 1000, 1051, 1037, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8029, 8263, 8098, 8225, 8219, 8262, 8218, 8222] avg=8192.0000 max=8263 max_rank=1 min=8029 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[985, 1030, 1005, 1000, 989, 996, 1032, 1024, 29, 17, 0, 0, 0, 0, 0, 0, 1041, 1045, 1015, 1035, 1053, 984, 1040, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1015, 1029, 1076, 1036, 1003, 1098, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 949, 1063, 1039, 998, 1007, 989, 1043, 997, 3, 0, 0, 0, 0, 0, 0, 0, 1007, 1000, 1063, 1018, 1046, 1058, 1073, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1001, 1015, 983, 1003, 986, 1035, 1012, 4, 0, 0, 0, 0, 0, 0, 0, 1002, 1019, 991, 1005, 1055, 1014, 1033, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1033, 1051, 1046, 1020, 1012, 1004, 1031, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8282, 8333, 8088, 8297, 8087, 8128, 8214] avg=8192.0000 max=8333 max_rank=2 min=8087 min_rank=5 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1032, 1028, 1020, 1055, 1047, 992, 986, 1059, 28, 16, 0, 0, 0, 0, 0, 0, 1025, 1011, 1036, 1023, 1070, 1061, 1027, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1007, 1044, 1037, 992, 1007, 1002, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1020, 1035, 1043, 1024, 1027, 1013, 1068, 3, 0, 0, 0, 0, 0, 0, 0, 1005, 1031, 1092, 994, 996, 1028, 1058, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1041, 1044, 1031, 991, 1027, 993, 1037, 4, 0, 0, 0, 0, 0, 0, 0, 1021, 998, 1038, 1029, 1007, 1013, 1018, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1052, 1028, 976, 953, 988, 1077, 1036, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8256, 8105, 8257, 8214, 8170, 8119, 8152] avg=8192.0000 max=8263 max_rank=0 min=8105 min_rank=2 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8064, 8320, 8320, 8192, 8192, 8448, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8064, 8320, 8192, 8320, 8192, 8192, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8064, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1026, 1103, 1025, 1009, 1014, 1017, 987, 1018, 998, 1049, 1024, 1026, 1032, 1029, 1088, 1029, 1031, 1019, 1042, 1020, 1062, 993, 1041, 1015, 1058, 1009, 1002, 1014, 1018, 1017, 1027, 987, 965, 1025, 1029, 1012, 953, 1060, 1062, 1019, 1060, 1025, 992, 968, 1045, 1000, 1017, 1071, 1042, 1058, 1075, 1030, 1041, 1049, 1023, 1009, 977, 1026, 1017, 1059, 1039, 1024, 972, 963] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[975, 993, 1012, 1015, 1042, 1038, 1063, 1036, 1010, 1009, 1030, 1005, 996, 1014, 995, 1034, 1042, 1042, 1043, 984, 1002, 986, 996, 980, 1051, 981, 1046, 1013, 1020, 984, 1075, 994, 1023, 1081, 1016, 1017, 1055, 1026, 1042, 1042, 1043, 1044, 1025, 1053, 1037, 1034, 1025, 973, 1013, 1018, 1039, 1036, 1006, 1020, 1054, 1018, 1030, 978, 1014, 1027, 1071, 1068, 1053, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1037, 1032, 1008, 1013, 999, 1087, 1022, 995, 992, 1037, 1035, 1021, 1036, 1040, 991, 978, 1016, 1048, 1042, 1037, 1038, 971, 1043, 1010, 1030, 1005, 1017, 1039, 1008, 1043, 975, 1009, 998, 1040, 1021, 1077, 998, 1046, 999, 978, 1014, 1050, 983, 1050, 1024, 1070, 1048, 1039, 1033, 1077, 1000, 1053, 1109, 987, 977, 997, 1019, 1056, 1027, 1031, 1009, 1010, 1074, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1013, 1026, 1045, 1013, 1020, 1035, 987, 1036, 1013, 1003, 976, 1001, 975, 1048, 981, 1076, 1000, 1035, 1074, 1005, 1038, 988, 1022, 996, 1031, 1061, 1018, 1013, 1054, 1032, 1009, 1064, 1099, 995, 1057, 1016, 1021, 1036, 1005, 998, 986, 1046, 1032, 1051, 1005, 1021, 1007, 1033, 987, 1002, 1033, 1015, 1025, 997, 1073, 1047, 1024, 1062, 1042, 1018, 1024, 1069, 994, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1019, 987, 1021, 990, 1051, 1017, 1054, 1014, 1071, 1027, 1055, 1025, 1055, 1048, 1025, 1011, 1031, 1058, 1043, 1085, 1022, 1023, 1020, 1059, 998, 1034, 1004, 1030, 992, 1052, 1004, 1018, 1064, 1006, 1036, 1050, 1015, 1049, 969, 1012, 1012, 1034, 1080, 1048, 1013, 1006, 1006, 1015, 1046, 1015, 991, 982, 1038, 1008, 987, 989, 1040, 1042, 991, 1012, 1013, 1032, 1015, 977] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[987, 976, 1032, 1025, 1023, 1015, 979, 1007, 1077, 1047, 1017, 976, 1020, 973, 1059, 1067, 1026, 1036, 1022, 1017, 995, 1010, 1057, 1031, 988, 1027, 1032, 990, 1000, 1008, 1014, 1024, 1025, 1041, 1015, 1055, 1028, 1004, 1018, 1001, 1032, 1030, 1069, 1036, 1055, 1015, 1000, 1048, 1006, 1031, 1046, 1019, 1047, 999, 1030, 1034, 1050, 1054, 1005, 1029, 1087, 1012, 1006, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1014, 980, 963, 1005, 1013, 1030, 997, 1062, 1029, 1020, 1058, 1042, 1012, 1032, 1056, 1013, 1031, 1038, 1030, 1014, 1052, 1034, 1003, 1000, 1032, 1073, 984, 1033, 1012, 1018, 1062, 1016, 1064, 1068, 1022, 944, 1012, 1078, 1039, 1082, 1025, 1020, 1011, 1031, 1010, 1008, 1021, 1094, 1025, 981, 1010, 1019, 1090, 955, 1010, 996, 1019, 1046, 962, 1050, 994, 1027, 1016, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[993, 1055, 1007, 1025, 1008, 997, 1012, 1008, 990, 1032, 1026, 1009, 997, 1026, 1056, 1067, 1021, 994, 1050, 1064, 984, 994, 1009, 980, 1051, 1049, 975, 1020, 1069, 1020, 997, 1048, 1040, 1022, 1034, 1058, 1009, 1030, 1072, 990, 984, 1025, 999, 1062, 1015, 1014, 1032, 1012, 1059, 1017, 1072, 1036, 1007, 1040, 1018, 1033, 1030, 1010, 1028, 1009, 1004, 1007, 1052, 1083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1026, 1103, 1025, 1009, 1014, 1017, 987, 1018, 28, 22, 0, 0, 0, 0, 0, 0, 998, 1049, 1024, 1026, 1032, 1029, 1088, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1019, 1042, 1020, 1062, 993, 1041, 1015, 5, 0, 0, 0, 0, 0, 0, 0, 1058, 1009, 1002, 1014, 1018, 1017, 1027, 987, 12, 6, 0, 0, 0, 0, 0, 0, 965, 1025, 1029, 1012, 953, 1032, 1062, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1025, 992, 946, 1045, 1000, 1017, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1058, 1075, 1030, 1036, 1049, 1023, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1014, 1017, 1059, 1039, 1024, 972, 963, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8269, 8228, 8150, 8097, 8156, 8322, 8065] avg=8192.0000 max=8322 max_rank=6 min=8065 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[975, 993, 1012, 1015, 1042, 1038, 1063, 1036, 27, 24, 0, 0, 0, 0, 0, 0, 1010, 1009, 1030, 1005, 996, 1014, 995, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1042, 1043, 984, 1002, 986, 996, 980, 5, 0, 0, 0, 0, 0, 0, 0, 1051, 981, 1046, 1013, 1020, 984, 1075, 994, 12, 6, 0, 0, 0, 0, 0, 0, 1023, 1081, 1016, 1017, 1055, 999, 1042, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1044, 1025, 1029, 1037, 1034, 1025, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1018, 1039, 1036, 1001, 1020, 1054, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 966, 1014, 1027, 1071, 1068, 1053, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8087, 8080, 8182, 8275, 8210, 8199, 8278] avg=8192.0000 max=8278 max_rank=7 min=8080 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32962 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1019, 987, 1021, 990, 1051, 1017, 1054, 1014, 28, 24, 0, 0, 0, 0, 0, 0, 1071, 1027, 1055, 1025, 1055, 1048, 1025, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1058, 1043, 1085, 1022, 1023, 1020, 1059, 5, 0, 0, 0, 0, 0, 0, 0, 998, 1034, 1004, 1030, 992, 1052, 1004, 1018, 13, 6, 0, 0, 0, 0, 0, 0, 1064, 1006, 1036, 1050, 1015, 1021, 969, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1034, 1080, 1024, 1013, 1006, 1006, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1015, 991, 982, 1033, 1008, 987, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1029, 991, 1012, 1013, 1032, 1015, 977, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8311, 8346, 8151, 8173, 8190, 8051, 8109] avg=8192.0000 max=8346 max_rank=2 min=8051 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32523 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1037, 1032, 1008, 1013, 999, 1087, 1022, 995, 28, 24, 0, 0, 0, 0, 0, 0, 992, 1037, 1035, 1021, 1036, 1040, 991, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1048, 1042, 1037, 1038, 971, 1043, 1010, 5, 0, 0, 0, 0, 0, 0, 0, 1030, 1005, 1017, 1039, 1008, 1043, 975, 1009, 13, 6, 0, 0, 0, 0, 0, 0, 998, 1040, 1021, 1077, 998, 1018, 999, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1050, 983, 1026, 1024, 1070, 1048, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1077, 1000, 1053, 1104, 987, 977, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1043, 1027, 1031, 1009, 1010, 1074, 988, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8124, 8210, 8145, 8129, 8254, 8228, 8201] avg=8192.0000 max=8254 max_rank=5 min=8124 min_rank=1 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1013, 1026, 1045, 1013, 1020, 1035, 987, 1036, 32, 30, 5, 0, 0, 0, 0, 0, 1013, 1003, 976, 1001, 975, 1048, 981, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1035, 1074, 1005, 1038, 988, 1022, 996, 7, 0, 0, 0, 0, 0, 0, 0, 1031, 1061, 1018, 1013, 1054, 1032, 1009, 1064, 16, 11, 1, 0, 0, 0, 0, 0, 1099, 995, 1057, 1016, 1021, 1004, 1005, 998, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1046, 1032, 1021, 1005, 1021, 1007, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1002, 1033, 1015, 1017, 997, 1073, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1041, 1042, 1018, 1024, 1069, 994, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8062, 8165, 8310, 8195, 8151, 8171, 8240] avg=8192.0000 max=8310 max_rank=3 min=8062 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1014, 980, 963, 1005, 1013, 1030, 997, 1062, 29, 24, 0, 0, 0, 0, 0, 0, 1029, 1020, 1058, 1042, 1012, 1032, 1056, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1038, 1030, 1014, 1052, 1034, 1003, 1000, 5, 0, 0, 0, 0, 0, 0, 0, 1032, 1073, 984, 1033, 1012, 1018, 1062, 1016, 13, 6, 0, 0, 0, 0, 0, 0, 1064, 1068, 1022, 944, 1012, 1049, 1039, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1020, 1011, 1007, 1010, 1008, 1021, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 981, 1010, 1019, 1085, 955, 1010, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1033, 962, 1050, 994, 1027, 1016, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8256, 8207, 8249, 8280, 8196, 8081, 8150] avg=8192.0000 max=8280 max_rank=4 min=8081 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[993, 1055, 1007, 1025, 1008, 997, 1012, 1008, 28, 25, 0, 0, 0, 0, 0, 0, 990, 1032, 1026, 1009, 997, 1026, 1056, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 994, 1050, 1064, 984, 994, 1009, 980, 5, 0, 0, 0, 0, 0, 0, 0, 1051, 1049, 975, 1020, 1069, 1020, 997, 1048, 12, 6, 0, 0, 0, 0, 0, 0, 1040, 1022, 1034, 1058, 1009, 1002, 1072, 990, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1025, 999, 1037, 1015, 1014, 1032, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1017, 1072, 1036, 1002, 1040, 1018, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 998, 1028, 1009, 1004, 1007, 1052, 1083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8197, 8101, 8247, 8227, 8118, 8277, 8211] avg=8192.0000 max=8277 max_rank=6 min=8101 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[987, 976, 1032, 1025, 1023, 1015, 979, 1007, 27, 24, 0, 0, 0, 0, 0, 0, 1077, 1047, 1017, 976, 1020, 973, 1059, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1036, 1022, 1017, 995, 1010, 1057, 1031, 5, 0, 0, 0, 0, 0, 0, 0, 988, 1027, 1032, 990, 1000, 1008, 1014, 1024, 13, 6, 0, 0, 0, 0, 0, 0, 1025, 1041, 1015, 1055, 1028, 977, 1018, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1030, 1069, 1012, 1055, 1015, 1000, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1031, 1046, 1019, 1042, 999, 1030, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1041, 1005, 1029, 1087, 1012, 1006, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8230, 8199, 8102, 8160, 8261, 8207, 8282] avg=8192.0000 max=8282 max_rank=7 min=8095 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32910 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8192, 8192, 8192, 8320, 8192, 8192, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8064, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8448, 8320, 8320, 8064, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[994, 998, 1054, 1016, 1017, 1075, 1031, 1014, 1003, 1010, 1030, 1055, 1034, 1000, 1050, 1015, 1019, 1041, 964, 997, 1072, 1028, 1059, 1068, 962, 1018, 986, 1041, 1012, 1089, 1061, 1017, 1022, 988, 998, 999, 1017, 1013, 1007, 1059, 1058, 1025, 1010, 1085, 997, 970, 1050, 1003, 1040, 1073, 1021, 988, 1034, 1006, 1025, 1033, 1000, 1004, 1017, 1001, 1032, 1011, 1040, 1080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1026, 1051, 1012, 967, 1036, 984, 996, 1041, 998, 1048, 1077, 1023, 1058, 995, 1011, 1006, 1001, 1055, 1060, 997, 1047, 1038, 1005, 1013, 998, 985, 1083, 1014, 1022, 1030, 1057, 1050, 1016, 1025, 1010, 1012, 1017, 1009, 1015, 965, 1001, 1027, 1000, 1024, 980, 1056, 1035, 1037, 1032, 1031, 1028, 1050, 1030, 1001, 1045, 997, 1000, 1054, 1067, 1072, 1018, 1078, 1000, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1005, 1046, 1004, 1034, 1029, 1009, 1032, 981, 996, 1049, 1067, 1027, 990, 1001, 1022, 1016, 1034, 1050, 1066, 1024, 1068, 1000, 1067, 1028, 1034, 1035, 1040, 1007, 1049, 1042, 1054, 993, 1020, 1030, 1025, 998, 1044, 991, 1019, 1009, 1031, 1054, 986, 990, 1038, 1027, 990, 999, 1042, 959, 1037, 1060, 1051, 1050, 1014, 1013, 1002, 1045, 1040, 1047, 1028, 1014, 962, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1083, 1023, 979, 966, 1041, 996, 994, 1026, 1048, 1054, 1028, 967, 1001, 1038, 998, 1036, 1034, 1000, 1013, 1022, 1013, 1019, 1016, 1042, 1013, 1096, 1006, 1023, 998, 1025, 1016, 989, 1063, 995, 1014, 1036, 1059, 1011, 1020, 1038, 1036, 1069, 1050, 995, 1010, 1039, 1022, 1062, 1044, 1021, 986, 1056, 1014, 1062, 1029, 1030, 1053, 979, 1034, 1002, 1037, 1001, 1054, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1051, 1000, 1064, 1039, 1047, 991, 1024, 1060, 1028, 1023, 1035, 1009, 1010, 1001, 1027, 1005, 965, 1025, 986, 1024, 1008, 1037, 1000, 1048, 1031, 1071, 982, 1002, 1024, 972, 1045, 1035, 1037, 1019, 1017, 1041, 1036, 1068, 1062, 1034, 971, 1040, 1042, 1064, 1002, 1059, 1017, 1078, 1033, 1008, 982, 993, 1034, 1015, 957, 1046, 1026, 997, 1048, 1017, 1051, 1031, 1007, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1013, 1030, 1030, 1026, 1038, 1027, 1052, 1065, 1018, 1018, 1011, 1070, 1034, 1022, 1039, 1032, 1022, 1002, 1044, 989, 1073, 986, 1030, 1011, 1043, 1055, 1010, 1024, 1029, 1005, 972, 1007, 993, 1012, 1003, 1045, 1072, 993, 1031, 1058, 1074, 1048, 972, 979, 1037, 1000, 1017, 1025, 1046, 952, 1056, 983, 1004, 1048, 1068, 1075, 988, 1008, 998, 995, 1025, 1040, 1038, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1051, 1059, 1023, 1030, 1039, 1042, 999, 963, 1007, 1055, 980, 1058, 1022, 1043, 1004, 985, 1002, 1046, 1026, 1055, 1012, 1038, 1014, 1045, 987, 1027, 1002, 1057, 967, 1071, 1075, 1026, 1040, 1058, 1003, 1028, 1048, 1032, 1000, 979, 980, 1031, 1006, 1015, 1032, 1039, 1052, 1012, 984, 1013, 1025, 1011, 1036, 1034, 1021, 1029, 1043, 1024, 1028, 1030, 1027, 1026, 1008, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1027, 991, 1049, 1025, 1024, 1046, 1011, 991, 1020, 1013, 1035, 1016, 1007, 1040, 1039, 990, 1017, 1066, 999, 1026, 1012, 1011, 1024, 995, 1064, 1036, 992, 1058, 1033, 997, 1048, 1065, 1029, 1005, 992, 980, 1052, 972, 1044, 1046, 1028, 1047, 1012, 1032, 1027, 997, 1019, 1003, 1053, 1024, 1032, 1022, 1087, 998, 1004, 1009, 1099, 1003, 1057, 1045, 1021, 1016, 1026, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[994, 998, 1054, 1016, 1017, 1075, 1031, 1014, 3, 3, 0, 0, 0, 0, 0, 0, 1003, 1010, 1030, 1055, 1034, 1000, 1050, 1015, 6, 4, 0, 0, 0, 0, 0, 0, 1019, 1041, 964, 997, 1065, 1028, 1059, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1018, 986, 1041, 1012, 1089, 1046, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 988, 998, 999, 1017, 1013, 1007, 1059, 15, 5, 0, 0, 0, 0, 0, 0, 1058, 1025, 1010, 1085, 997, 970, 1050, 1003, 2, 0, 0, 0, 0, 0, 0, 0, 1040, 1073, 1021, 988, 1029, 1006, 1025, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1004, 1006, 1001, 1032, 1011, 1040, 1080, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8207, 8241, 8171, 8123, 8200, 8215, 8174] avg=8192.0000 max=8241 max_rank=2 min=8123 min_rank=4 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1005, 1046, 1004, 1034, 1029, 1009, 1032, 981, 3, 3, 0, 0, 0, 0, 0, 0, 996, 1049, 1067, 1027, 990, 1001, 1022, 1016, 6, 4, 0, 0, 0, 0, 0, 0, 1034, 1050, 1066, 1024, 1061, 1000, 1067, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1035, 1040, 1007, 1049, 1042, 1039, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1030, 1025, 998, 1044, 991, 1019, 1009, 15, 5, 0, 0, 0, 0, 0, 0, 1031, 1054, 986, 990, 1038, 1027, 990, 999, 2, 0, 0, 0, 0, 0, 0, 0, 1042, 959, 1037, 1060, 1046, 1050, 1014, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1045, 1029, 1047, 1028, 1014, 962, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8178, 8330, 8239, 8156, 8117, 8221, 8149] avg=8192.0000 max=8330 max_rank=2 min=8117 min_rank=5 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1051, 1000, 1064, 1039, 1047, 991, 1024, 1060, 3, 2, 0, 0, 0, 0, 0, 0, 1028, 1023, 1035, 1009, 1010, 1001, 1027, 1005, 6, 4, 0, 0, 0, 0, 0, 0, 965, 1025, 986, 1024, 1002, 1037, 1000, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1071, 982, 1002, 1024, 972, 1030, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1019, 1017, 1041, 1036, 1068, 1062, 1034, 15, 5, 0, 0, 0, 0, 0, 0, 971, 1040, 1042, 1064, 1002, 1059, 1017, 1078, 2, 0, 0, 0, 0, 0, 0, 0, 1033, 1008, 982, 993, 1029, 1015, 957, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 997, 1037, 1017, 1051, 1031, 1007, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8148, 8087, 8147, 8334, 8275, 8063, 8201] avg=8192.0000 max=8334 max_rank=4 min=8063 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 1051, 1012, 967, 1036, 984, 996, 1041, 3, 3, 0, 0, 0, 0, 0, 0, 998, 1048, 1077, 1023, 1058, 995, 1011, 1006, 6, 4, 0, 0, 0, 0, 0, 0, 1001, 1055, 1060, 997, 1040, 1038, 1005, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 998, 985, 1083, 1014, 1022, 1030, 1042, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1025, 1010, 1012, 1017, 1009, 1015, 965, 15, 5, 0, 0, 0, 0, 0, 0, 1001, 1027, 1000, 1024, 980, 1056, 1035, 1037, 2, 0, 0, 0, 0, 0, 0, 0, 1032, 1031, 1028, 1050, 1025, 1001, 1045, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1054, 1056, 1072, 1018, 1078, 1000, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8226, 8209, 8224, 8089, 8162, 8209, 8298] avg=8192.0000 max=8298 max_rank=7 min=8089 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1083, 1023, 979, 966, 1041, 996, 994, 1026, 4, 6, 0, 0, 0, 0, 0, 0, 1048, 1054, 1028, 967, 1001, 1038, 998, 1036, 9, 10, 0, 0, 0, 0, 0, 0, 1034, 1000, 1013, 1022, 997, 1019, 1016, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1096, 1006, 1023, 998, 1025, 999, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 995, 1014, 1036, 1059, 1011, 1020, 1038, 17, 7, 0, 0, 0, 0, 0, 0, 1036, 1069, 1050, 995, 1010, 1039, 1022, 1062, 8, 0, 0, 0, 0, 0, 0, 0, 1044, 1021, 986, 1056, 1002, 1062, 1029, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 979, 1018, 1002, 1037, 1001, 1054, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8189, 8143, 8149, 8260, 8291, 8230, 8156] avg=8192.0000 max=8291 max_rank=5 min=8118 min_rank=0 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32937 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1013, 1030, 1030, 1026, 1038, 1027, 1052, 1065, 3, 3, 0, 0, 0, 0, 0, 0, 1018, 1018, 1011, 1070, 1034, 1022, 1039, 1032, 6, 4, 0, 0, 0, 0, 0, 0, 1022, 1002, 1044, 989, 1066, 986, 1030, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1055, 1010, 1024, 1029, 1005, 958, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1012, 1003, 1045, 1072, 993, 1031, 1058, 14, 5, 0, 0, 0, 0, 0, 0, 1074, 1048, 972, 979, 1037, 1000, 1017, 1025, 2, 0, 0, 0, 0, 0, 0, 0, 1046, 952, 1056, 983, 999, 1048, 1068, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1008, 987, 995, 1025, 1040, 1038, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8254, 8150, 8131, 8226, 8154, 8227, 8107] avg=8192.0000 max=8287 max_rank=0 min=8107 min_rank=7 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1051, 1059, 1023, 1030, 1039, 1042, 999, 963, 3, 2, 0, 0, 0, 0, 0, 0, 1007, 1055, 980, 1058, 1022, 1043, 1004, 985, 6, 4, 0, 0, 0, 0, 0, 0, 1002, 1046, 1026, 1055, 1006, 1038, 1014, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1027, 1002, 1057, 967, 1071, 1060, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1058, 1003, 1028, 1048, 1032, 1000, 979, 15, 5, 0, 0, 0, 0, 0, 0, 980, 1031, 1006, 1015, 1032, 1039, 1052, 1012, 2, 0, 0, 0, 0, 0, 0, 0, 984, 1013, 1025, 1011, 1031, 1034, 1021, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1024, 1017, 1030, 1027, 1026, 1008, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8164, 8232, 8197, 8208, 8169, 8148, 8207] avg=8192.0000 max=8232 max_rank=2 min=8148 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1027, 991, 1049, 1025, 1024, 1046, 1011, 991, 3, 2, 0, 0, 0, 0, 0, 0, 1020, 1013, 1035, 1016, 1007, 1040, 1039, 990, 6, 4, 0, 0, 0, 0, 0, 0, 1017, 1066, 999, 1026, 1006, 1011, 1024, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1036, 992, 1058, 1033, 997, 1033, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1005, 992, 980, 1052, 972, 1044, 1046, 15, 5, 0, 0, 0, 0, 0, 0, 1028, 1047, 1012, 1032, 1027, 997, 1019, 1003, 3, 0, 0, 0, 0, 0, 0, 0, 1053, 1024, 1032, 1022, 1081, 998, 1004, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 1003, 1046, 1045, 1021, 1016, 1026, 988, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8170, 8144, 8278, 8140, 8168, 8223, 8244] avg=8192.0000 max=8278 max_rank=3 min=8140 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8064, 8192, 8448, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8192, 8192, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1000, 1011, 1016, 1019, 998, 1032, 990, 1036, 1076, 987, 1030, 1034, 1031, 964, 1066, 1042, 990, 1030, 1001, 1010, 1064, 995, 996, 1010, 1038, 982, 1004, 1024, 1021, 1013, 1036, 1097, 1055, 1042, 977, 987, 980, 1029, 1034, 1072, 1017, 1073, 1052, 1029, 1013, 1011, 1005, 1016, 1039, 1012, 1066, 994, 1008, 1034, 1038, 1030, 1009, 1038, 1036, 1033, 1023, 1048, 1032, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[951, 1060, 1023, 1022, 1010, 993, 1047, 1005, 1062, 1062, 992, 950, 1049, 1012, 1010, 980, 1026, 991, 1044, 1034, 1033, 1007, 1038, 997, 1061, 1037, 1067, 1036, 972, 1034, 1013, 1035, 998, 1027, 1017, 1061, 989, 1003, 1031, 1007, 1005, 1024, 1025, 1007, 1037, 1002, 1046, 1067, 1041, 1057, 1048, 1000, 1065, 1028, 1042, 1055, 1030, 1016, 1054, 1011, 1035, 1040, 1048, 967] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1038, 1018, 1007, 1033, 1013, 1044, 1056, 997, 1045, 992, 1014, 1047, 1113, 1048, 1015, 1049, 1047, 1047, 1020, 997, 1031, 1016, 1036, 1017, 1032, 1020, 1004, 1034, 1052, 1010, 1046, 1048, 966, 1001, 1001, 1019, 961, 1026, 1070, 997, 1026, 1010, 983, 1065, 1014, 1003, 1021, 1037, 1034, 1040, 1071, 1026, 1029, 1003, 992, 1029, 1063, 1039, 981, 969, 1010, 1030, 978, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[988, 1036, 997, 1033, 1033, 1025, 1040, 1085, 999, 1001, 1010, 1060, 1015, 1037, 986, 1024, 1032, 1020, 995, 1030, 989, 1029, 1054, 1042, 1040, 1036, 1014, 1024, 1018, 1029, 1026, 1025, 1029, 1028, 1019, 997, 1000, 1010, 955, 1049, 1026, 1014, 994, 992, 1073, 1021, 1001, 984, 1031, 994, 985, 1004, 1054, 1053, 1058, 1035, 1073, 1088, 1010, 1077, 990, 1033, 1026, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1028, 1047, 993, 1069, 992, 1095, 1047, 935, 993, 1016, 998, 1071, 1025, 1050, 1079, 988, 1014, 1008, 1010, 994, 1040, 1028, 1014, 1010, 1072, 973, 1057, 1030, 1045, 982, 1018, 1030, 1031, 1018, 1009, 993, 1091, 1018, 991, 1010, 1031, 1011, 998, 1030, 1066, 1026, 1046, 1042, 1025, 1037, 984, 1026, 1019, 1002, 1076, 1009, 1069, 1012, 1039, 983, 1032, 1030, 1043, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1007, 999, 1034, 1051, 1041, 995, 988, 1050, 1007, 1023, 1014, 1037, 1024, 1066, 1017, 1037, 1001, 1011, 1000, 1065, 1006, 1028, 1046, 998, 1057, 1015, 1050, 1007, 996, 1039, 994, 1043, 1018, 1034, 1036, 1043, 984, 1073, 1047, 1027, 1044, 1006, 983, 1018, 1032, 1022, 1033, 1052, 993, 1026, 1054, 1031, 1046, 1036, 1019, 992, 1030, 1049, 1020, 1005, 968, 1000, 1037, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1021, 1029, 1087, 1026, 1024, 996, 1053, 1059, 989, 1061, 1078, 1009, 1063, 1016, 988, 1078, 1045, 987, 1013, 1006, 1021, 1073, 1024, 1010, 1010, 1004, 1074, 971, 1017, 999, 1035, 1020, 1095, 977, 1040, 1061, 1045, 1037, 965, 999, 965, 1018, 1016, 991, 991, 1005, 1051, 980, 1035, 1026, 982, 1050, 987, 1044, 1024, 987, 1045, 1059, 1027, 1063, 1036, 1004, 1006, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1061, 989, 1017, 985, 980, 1026, 1016, 1102, 1027, 1031, 1039, 973, 1047, 1031, 1024, 1006, 1049, 1041, 1050, 1031, 997, 993, 997, 1037, 1029, 1003, 1002, 1021, 998, 1041, 1026, 1009, 1027, 1042, 1058, 1035, 1048, 990, 987, 1001, 1034, 1011, 1029, 1022, 1020, 1025, 1031, 1028, 994, 1039, 1015, 1029, 1082, 1041, 1027, 1052, 1037, 1025, 991, 1024, 1007, 1054, 1048, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1038, 1018, 1007, 1033, 1013, 1044, 1056, 997, 4, 0, 0, 0, 0, 0, 0, 0, 1045, 992, 1014, 1047, 1092, 1048, 1015, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1047, 1020, 997, 1031, 1016, 1036, 1017, 27, 0, 0, 0, 0, 0, 0, 0, 1026, 1020, 1004, 1034, 1052, 1010, 1046, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1001, 1001, 1019, 961, 1026, 1070, 997, 30, 2, 0, 0, 0, 0, 0, 0, 1026, 1010, 983, 1065, 1014, 1003, 1021, 1037, 21, 2, 0, 0, 0, 0, 0, 0, 1034, 1040, 1071, 1026, 999, 1003, 992, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1039, 981, 969, 1010, 1030, 978, 1056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8302, 8238, 8240, 8073, 8182, 8194, 8097] avg=8192.0000 max=8302 max_rank=1 min=8073 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32546 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1028, 1047, 993, 1069, 992, 1095, 1047, 935, 4, 0, 0, 0, 0, 0, 0, 0, 993, 1016, 998, 1071, 1005, 1050, 1079, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1008, 1010, 994, 1040, 1028, 1014, 1010, 27, 0, 0, 0, 0, 0, 0, 0, 1066, 973, 1057, 1030, 1045, 982, 1018, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1018, 1009, 993, 1091, 1018, 991, 1010, 30, 2, 0, 0, 0, 0, 0, 0, 1031, 1011, 998, 1030, 1066, 1026, 1046, 1042, 20, 2, 0, 0, 0, 0, 0, 0, 1025, 1037, 984, 1026, 989, 1002, 1076, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1012, 1039, 983, 1032, 1030, 1043, 988, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8200, 8145, 8201, 8193, 8272, 8148, 8167] avg=8192.0000 max=8272 max_rank=5 min=8145 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1000, 1011, 1016, 1019, 998, 1032, 990, 1036, 4, 0, 0, 0, 0, 0, 0, 0, 1076, 987, 1030, 1034, 1011, 964, 1066, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1030, 1001, 1010, 1064, 995, 996, 1010, 25, 0, 0, 0, 0, 0, 0, 0, 1032, 982, 1004, 1024, 1021, 1013, 1036, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1042, 977, 987, 980, 1029, 1034, 1072, 30, 2, 0, 0, 0, 0, 0, 0, 1017, 1073, 1052, 1029, 1013, 1011, 1005, 1016, 20, 2, 0, 0, 0, 0, 0, 0, 1039, 1012, 1066, 994, 978, 1034, 1038, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1038, 1036, 1033, 1023, 1048, 1032, 1061, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8210, 8121, 8209, 8208, 8238, 8191, 8253] avg=8192.0000 max=8253 max_rank=7 min=8106 min_rank=0 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1007, 999, 1034, 1051, 1041, 995, 988, 1050, 4, 0, 0, 0, 0, 0, 0, 0, 1007, 1023, 1014, 1037, 1004, 1066, 1017, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1011, 1000, 1065, 1006, 1028, 1046, 998, 26, 0, 0, 0, 0, 0, 0, 0, 1051, 1015, 1050, 1007, 996, 1039, 994, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1034, 1036, 1043, 984, 1073, 1047, 1027, 31, 2, 0, 0, 0, 0, 0, 0, 1044, 1006, 983, 1018, 1032, 1022, 1033, 1052, 20, 2, 0, 0, 0, 0, 0, 0, 993, 1026, 1054, 1031, 1015, 1036, 1019, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1049, 1020, 1005, 968, 1000, 1037, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8205, 8181, 8195, 8295, 8212, 8166, 8113] avg=8192.0000 max=8295 max_rank=4 min=8113 min_rank=7 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[951, 1060, 1023, 1022, 1010, 993, 1047, 1005, 4, 0, 0, 0, 0, 0, 0, 0, 1062, 1062, 992, 950, 1029, 1012, 1010, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 991, 1044, 1034, 1033, 1007, 1038, 997, 26, 0, 0, 0, 0, 0, 0, 0, 1055, 1037, 1067, 1036, 972, 1034, 1013, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1027, 1017, 1061, 989, 1003, 1031, 1007, 31, 2, 0, 0, 0, 0, 0, 0, 1005, 1024, 1025, 1007, 1037, 1002, 1046, 1067, 20, 2, 0, 0, 0, 0, 0, 0, 1041, 1057, 1048, 1000, 1034, 1028, 1042, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1016, 1054, 1011, 1035, 1040, 1048, 967, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8097, 8196, 8249, 8166, 8235, 8305, 8173] avg=8192.0000 max=8305 max_rank=6 min=8097 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32879 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[988, 1036, 997, 1033, 1033, 1025, 1040, 1085, 4, 0, 0, 0, 0, 0, 0, 0, 999, 1001, 1010, 1060, 995, 1037, 986, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1020, 995, 1030, 989, 1029, 1054, 1042, 27, 0, 0, 0, 0, 0, 0, 0, 1034, 1036, 1014, 1024, 1018, 1029, 1026, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1028, 1019, 997, 1000, 1010, 955, 1049, 31, 2, 0, 0, 0, 0, 0, 0, 1026, 1014, 994, 992, 1073, 1021, 1001, 984, 20, 2, 0, 0, 0, 0, 0, 0, 1031, 994, 985, 1004, 1023, 1053, 1058, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1088, 1010, 1077, 990, 1033, 1026, 1061, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8112, 8218, 8206, 8120, 8127, 8183, 8329] avg=8192.0000 max=8329 max_rank=7 min=8112 min_rank=1 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1021, 1029, 1087, 1026, 1024, 996, 1053, 1059, 10, 0, 0, 0, 0, 0, 0, 0, 989, 1061, 1078, 1009, 1033, 1016, 988, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 987, 1013, 1006, 1021, 1073, 1024, 1010, 31, 6, 0, 0, 0, 0, 0, 0, 993, 1004, 1074, 971, 1017, 999, 1035, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 977, 1040, 1061, 1045, 1037, 965, 999, 33, 7, 0, 0, 0, 0, 0, 0, 965, 1018, 1016, 991, 991, 1005, 1051, 980, 24, 7, 0, 0, 0, 0, 0, 0, 1035, 1026, 982, 1050, 954, 1044, 1024, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1059, 1027, 1063, 1036, 1004, 1006, 1039, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 8252, 8216, 8113, 8259, 8048, 8102, 8241] avg=8192.0000 max=8305 max_rank=0 min=8048 min_rank=5 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1061, 989, 1017, 985, 980, 1026, 1016, 1102, 4, 0, 0, 0, 0, 0, 0, 0, 1027, 1031, 1039, 973, 1027, 1031, 1024, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1041, 1050, 1031, 997, 993, 997, 1037, 26, 0, 0, 0, 0, 0, 0, 0, 1023, 1003, 1002, 1021, 998, 1041, 1026, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1042, 1058, 1035, 1048, 990, 987, 1001, 32, 2, 0, 0, 0, 0, 0, 0, 1034, 1011, 1029, 1022, 1020, 1025, 1031, 1028, 20, 2, 0, 0, 0, 0, 0, 0, 994, 1039, 1015, 1029, 1050, 1041, 1027, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1025, 991, 1024, 1007, 1054, 1048, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8158, 8221, 8123, 8222, 8222, 8247, 8163] avg=8192.0000 max=8247 max_rank=6 min=8123 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8064, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1015, 1054, 1021, 1047, 1039, 1066, 928, 1022, 1057, 1007, 1007, 1009, 1020, 1015, 1036, 1059, 1006, 1023, 989, 1000, 1049, 1024, 1056, 1020, 1084, 998, 1055, 1019, 1032, 1075, 1017, 1058, 999, 985, 1057, 1025, 1054, 979, 1048, 1026, 1030, 1011, 1033, 955, 963, 1056, 1010, 1055, 1024, 1035, 1046, 1073, 1061, 1016, 1011, 1035, 1027, 982, 1039, 982, 982, 1041, 981, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1128, 1010, 1011, 1030, 1028, 962, 1077, 994, 1043, 1019, 1039, 1040, 1053, 971, 1054, 1016, 1005, 1008, 1052, 981, 1025, 1046, 1018, 1007, 1072, 1041, 1019, 981, 988, 1024, 968, 992, 1088, 1036, 991, 1046, 1016, 1023, 1024, 1045, 1080, 1038, 1027, 1006, 969, 1031, 1058, 1036, 999, 1041, 1027, 994, 1070, 1023, 1011, 1003, 1026, 1056, 983, 1008, 1028, 1032, 997, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1030, 1028, 1047, 1015, 991, 1012, 996, 1061, 998, 1011, 1033, 1039, 1027, 1059, 1015, 1035, 1032, 1048, 1020, 1014, 1049, 1011, 1016, 962, 1034, 1003, 999, 1030, 999, 1011, 992, 1013, 1065, 1040, 1054, 1011, 1025, 1072, 983, 1009, 1032, 1027, 1048, 1007, 1006, 1036, 1006, 998, 987, 1009, 1079, 1063, 977, 1044, 1044, 1044, 1052, 988, 1033, 980, 1048, 1011, 1072, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1049, 1032, 1004, 1009, 1024, 1031, 1068, 999, 994, 1031, 1079, 997, 1020, 1042, 1022, 1000, 987, 1009, 1043, 1048, 1005, 1001, 989, 974, 1058, 1076, 1025, 1016, 1071, 996, 1018, 1077, 1018, 1031, 988, 1000, 1039, 1085, 1027, 979, 1042, 1028, 1038, 1014, 1006, 1050, 1003, 992, 1029, 1002, 1043, 1034, 1051, 1045, 987, 1035, 1017, 1023, 1018, 1070, 1038, 1024, 950, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1005, 979, 1022, 1007, 1011, 1089, 1021, 1048, 1047, 997, 1049, 1051, 1028, 1042, 1036, 1039, 1039, 1017, 970, 1033, 1026, 1012, 1026, 1008, 1023, 1025, 1007, 968, 1026, 1041, 996, 999, 1047, 1035, 1016, 1010, 1038, 1002, 1032, 1020, 1010, 1099, 1022, 985, 1042, 986, 1041, 1012, 1015, 1021, 1020, 1036, 1007, 1018, 1028, 1036, 1023, 1035, 1038, 1037, 1032, 1041, 1019, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[965, 1037, 1018, 985, 1031, 1038, 1013, 1022, 992, 998, 1027, 977, 1046, 1041, 1036, 990, 995, 1001, 1066, 1025, 1023, 1016, 1024, 1049, 1022, 945, 1016, 1039, 1043, 1009, 1050, 987, 1010, 1037, 1025, 1028, 1037, 1042, 1000, 986, 1102, 1031, 1038, 1011, 1068, 1035, 1013, 1016, 1034, 992, 1061, 998, 1024, 1007, 1061, 1069, 1089, 1046, 1024, 989, 1059, 1002, 1011, 1065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[984, 1044, 992, 1001, 1052, 1004, 1044, 1019, 977, 979, 1047, 998, 1029, 1011, 1085, 980, 987, 1030, 1039, 1023, 1016, 986, 1048, 979, 1052, 1011, 1029, 1014, 1078, 1038, 1002, 1058, 1064, 1030, 1032, 1093, 1000, 1075, 1038, 1048, 1060, 978, 1046, 1042, 1002, 1044, 999, 1054, 1026, 1012, 1032, 1004, 1004, 989, 996, 1018, 1035, 970, 1024, 1021, 1013, 1055, 1022, 1074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1010, 1038, 1027, 1015, 1056, 992, 1005, 995, 1059, 1077, 1027, 1077, 1025, 1018, 1039, 1018, 998, 1034, 1018, 1043, 1009, 1018, 1011, 1040, 1077, 974, 1008, 1046, 1015, 980, 1016, 987, 1001, 999, 1061, 1052, 991, 995, 1042, 1095, 1047, 1024, 1033, 1002, 999, 1027, 993, 1021, 1046, 1042, 1041, 1029, 1032, 995, 1014, 990, 1030, 1050, 1037, 988, 1063, 1049, 1024, 972] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1128, 1010, 1011, 1030, 1028, 962, 1077, 994, 10, 7, 0, 0, 0, 0, 0, 0, 1043, 1019, 1039, 1040, 1053, 971, 1035, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1008, 1052, 981, 1025, 1046, 1018, 1007, 41, 12, 0, 0, 0, 0, 0, 0, 1072, 1041, 1019, 981, 988, 1024, 968, 992, 7, 4, 2, 0, 0, 0, 0, 0, 1047, 1036, 991, 1046, 1016, 1023, 1024, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1038, 1027, 1006, 969, 1031, 1058, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1041, 1015, 994, 1070, 1023, 1011, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1056, 983, 1008, 1028, 1032, 997, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8216, 8195, 8098, 8228, 8241, 8156, 8145] avg=8192.0000 max=8257 max_rank=0 min=8098 min_rank=3 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1049, 1032, 1004, 1009, 1024, 1031, 1068, 999, 10, 6, 0, 0, 0, 0, 0, 0, 994, 1031, 1079, 997, 1020, 1042, 1005, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1009, 1043, 1048, 1005, 1001, 989, 974, 38, 11, 0, 0, 0, 0, 0, 0, 1058, 1076, 1025, 1016, 1071, 996, 1018, 1077, 6, 4, 2, 0, 0, 0, 0, 0, 980, 1031, 988, 1000, 1039, 1085, 1027, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1028, 1038, 1014, 1006, 1050, 1003, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1002, 1031, 1034, 1051, 1045, 987, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1023, 1018, 1070, 1038, 1024, 950, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8168, 8105, 8349, 8129, 8169, 8214, 8170] avg=8192.0000 max=8349 max_rank=3 min=8105 min_rank=2 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1005, 979, 1022, 1007, 1011, 1089, 1021, 1048, 10, 6, 0, 0, 0, 0, 0, 0, 1047, 997, 1049, 1051, 1028, 1042, 1018, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1017, 970, 1033, 1026, 1012, 1026, 1008, 39, 12, 0, 0, 0, 0, 0, 0, 1023, 1025, 1007, 968, 1026, 1041, 996, 999, 7, 4, 2, 0, 0, 0, 0, 0, 1008, 1035, 1016, 1010, 1038, 1002, 1032, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1099, 1022, 985, 1042, 986, 1041, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1021, 1008, 1036, 1007, 1018, 1028, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1035, 1038, 1037, 1032, 1041, 1019, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8271, 8182, 8098, 8161, 8193, 8169, 8264] avg=8192.0000 max=8271 max_rank=1 min=8098 min_rank=3 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1015, 1054, 1021, 1047, 1039, 1066, 928, 1022, 10, 6, 0, 0, 0, 0, 0, 0, 1057, 1007, 1007, 1009, 1020, 1015, 1018, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1023, 989, 1000, 1049, 1024, 1056, 1020, 37, 12, 0, 0, 0, 0, 0, 0, 1084, 998, 1055, 1019, 1032, 1075, 1017, 1058, 7, 4, 2, 0, 0, 0, 0, 0, 962, 985, 1057, 1025, 1054, 979, 1048, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1011, 1033, 955, 963, 1056, 1010, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1035, 1034, 1073, 1061, 1016, 1011, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 982, 1039, 982, 982, 1041, 981, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8192, 8216, 8351, 8136, 8109, 8289, 8035] avg=8192.0000 max=8351 max_rank=3 min=8035 min_rank=7 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1030, 1028, 1047, 1015, 991, 1012, 996, 1061, 13, 12, 0, 0, 0, 0, 0, 0, 998, 1011, 1033, 1039, 1027, 1059, 989, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1048, 1020, 1014, 1049, 1011, 1016, 962, 43, 14, 0, 0, 0, 0, 0, 0, 1034, 1003, 999, 1030, 999, 1011, 992, 1013, 9, 9, 6, 0, 0, 0, 0, 0, 1022, 1040, 1054, 1011, 1025, 1072, 983, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1027, 1048, 1007, 1006, 1036, 1006, 998, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1009, 1060, 1063, 977, 1044, 1044, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 988, 1033, 980, 1048, 1011, 1072, 1056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8191, 8209, 8105, 8216, 8151, 8228, 8231] avg=8192.0000 max=8231 max_rank=7 min=8105 min_rank=3 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[965, 1037, 1018, 985, 1031, 1038, 1013, 1022, 10, 6, 0, 0, 0, 0, 0, 0, 992, 998, 1027, 977, 1046, 1041, 1018, 990, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1001, 1066, 1025, 1023, 1016, 1024, 1049, 38, 12, 0, 0, 0, 0, 0, 0, 1022, 945, 1016, 1039, 1043, 1009, 1050, 987, 7, 4, 2, 0, 0, 0, 0, 0, 972, 1037, 1025, 1028, 1037, 1042, 1000, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1098, 1031, 1038, 1011, 1068, 1035, 1013, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 992, 1049, 998, 1024, 1007, 1061, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 1046, 1024, 989, 1059, 1002, 1011, 1065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8089, 8249, 8124, 8127, 8310, 8234, 8278] avg=8192.0000 max=8310 max_rank=5 min=8089 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32949 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[984, 1044, 992, 1001, 1052, 1004, 1044, 1019, 10, 7, 0, 0, 0, 0, 0, 0, 977, 979, 1047, 998, 1029, 1011, 1066, 980, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1030, 1039, 1023, 1016, 986, 1048, 979, 40, 12, 0, 0, 0, 0, 0, 0, 1052, 1011, 1029, 1014, 1078, 1038, 1002, 1058, 7, 4, 2, 0, 0, 0, 0, 0, 1024, 1030, 1032, 1093, 1000, 1075, 1038, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 978, 1046, 1042, 1002, 1044, 999, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1012, 1020, 1004, 1004, 989, 996, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 970, 1024, 1021, 1013, 1055, 1022, 1074, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8087, 8160, 8295, 8340, 8221, 8069, 8207] avg=8192.0000 max=8340 max_rank=4 min=8069 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1010, 1038, 1027, 1015, 1056, 992, 1005, 995, 10, 6, 0, 0, 0, 0, 0, 0, 1059, 1077, 1027, 1077, 1025, 1018, 1021, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1034, 1018, 1043, 1009, 1018, 1011, 1040, 37, 12, 0, 0, 0, 0, 0, 0, 1077, 974, 1008, 1046, 1015, 980, 1016, 987, 7, 4, 2, 0, 0, 0, 0, 0, 964, 999, 1061, 1052, 991, 995, 1042, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1024, 1033, 1002, 999, 1027, 993, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1042, 1029, 1029, 1032, 995, 1014, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1050, 1037, 988, 1063, 1049, 1024, 972, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8322, 8220, 8116, 8199, 8142, 8177, 8206] avg=8192.0000 max=8322 max_rank=1 min=8116 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8192, 8320, 8192, 8192, 8064, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8320, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8448, 8320, 8320, 8064, 8064, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8192, 8192, 8320, 8192, 8064, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1009, 995, 1002, 1051, 1028, 1010, 1009, 1067, 1005, 1036, 1063, 993, 1011, 1023, 1023, 1036, 1019, 1005, 1050, 963, 1022, 1021, 1035, 990, 1035, 1054, 1006, 1061, 986, 1048, 1014, 1031, 1044, 1020, 1082, 994, 1091, 1040, 985, 1004, 988, 1057, 1005, 982, 1005, 1005, 1021, 1026, 1031, 1014, 1010, 1043, 1052, 1049, 1025, 1005, 1023, 1029, 988, 1050, 1078, 1025, 1043, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1061, 1016, 1033, 1069, 1044, 989, 987, 1031, 1042, 997, 1044, 1007, 993, 1051, 1045, 990, 1046, 1042, 1021, 1041, 1045, 1029, 1037, 1065, 994, 965, 962, 1031, 1009, 1056, 1048, 1055, 1001, 1029, 1039, 1053, 1037, 960, 1051, 1025, 1003, 1041, 1031, 1009, 1017, 1050, 1037, 1014, 996, 1026, 1022, 1021, 1018, 1031, 971, 989, 1046, 1039, 982, 969, 1026, 1093, 1043, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[953, 1030, 1043, 1067, 1062, 985, 1010, 1008, 1031, 1086, 1000, 1027, 1032, 1000, 1002, 1079, 1025, 1018, 1032, 1019, 1041, 1024, 1041, 1001, 1006, 1033, 1067, 1052, 1053, 976, 1025, 1049, 1035, 1062, 1029, 1037, 1041, 1021, 996, 1046, 942, 961, 1096, 1012, 1031, 986, 1003, 985, 1016, 999, 1030, 1035, 1043, 1057, 1005, 1038, 1052, 1016, 1015, 993, 1025, 1013, 1016, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1002, 994, 1062, 1005, 1017, 1031, 970, 1011, 1033, 1031, 1005, 1002, 1057, 1057, 978, 997, 1069, 957, 1046, 1029, 1008, 1066, 1025, 1030, 1040, 1044, 1006, 1006, 1039, 1034, 1069, 996, 1053, 1023, 998, 1024, 1024, 1030, 1053, 1052, 1039, 1015, 1067, 1036, 952, 998, 1007, 993, 1018, 989, 1021, 1042, 1034, 1027, 1027, 1024, 1006, 996, 1042, 1025, 1062, 1054, 1041, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1036, 1033, 1048, 1032, 1025, 972, 1046, 1024, 1069, 1037, 1054, 1006, 1014, 1014, 1009, 969, 1061, 980, 973, 1041, 1033, 1023, 1013, 1040, 991, 1050, 971, 1044, 1034, 975, 1030, 979, 1031, 1024, 996, 1013, 990, 1027, 1034, 1034, 1023, 992, 1031, 989, 1034, 1036, 1049, 1026, 1053, 1053, 1056, 1038, 1018, 1031, 1040, 1052, 1004, 996, 1029, 1102, 1020, 1024, 1003, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1041, 996, 1056, 986, 1008, 1038, 1014, 1011, 1022, 1028, 990, 1033, 1077, 1028, 1032, 1069, 1068, 1009, 997, 1044, 1004, 1002, 985, 1005, 996, 1003, 1024, 1006, 1039, 1053, 975, 1001, 1009, 1002, 1031, 1033, 979, 957, 1064, 1033, 1035, 1021, 1064, 989, 968, 1044, 998, 1007, 1071, 988, 1028, 1048, 990, 1028, 1062, 1020, 1050, 1050, 1056, 1067, 1013, 1082, 1034, 1075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1037, 1054, 1041, 1058, 1047, 1035, 1080, 1014, 1041, 1058, 1019, 1020, 1008, 1012, 1034, 1026, 1037, 1016, 1022, 1030, 1081, 1038, 1017, 1057, 1030, 1035, 1014, 1001, 969, 1058, 1007, 1010, 1013, 1046, 1021, 1008, 1019, 1020, 1050, 981, 1036, 1037, 997, 1035, 977, 1054, 1033, 1036, 949, 1048, 1049, 1027, 1010, 1026, 1014, 1006, 1023, 1013, 961, 1005, 988, 1045, 979, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1039, 1053, 1017, 1004, 1016, 1063, 996, 977, 984, 997, 1010, 1007, 1002, 1056, 1057, 1089, 1069, 966, 1027, 976, 1058, 1023, 1045, 1019, 1031, 1021, 1037, 1029, 1055, 1044, 1049, 1007, 1057, 1048, 1019, 987, 1007, 1029, 1012, 989, 1034, 967, 1034, 1042, 1022, 987, 1046, 1016, 970, 1020, 1112, 1022, 1041, 1035, 1018, 1032, 1037, 966, 1035, 989, 1041, 1014, 1037, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1009, 995, 1001, 1051, 1028, 1010, 1009, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1023, 1063, 993, 1011, 1023, 1023, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1005, 1050, 963, 1022, 1021, 1035, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1054, 1006, 1061, 986, 1048, 1014, 1031, 10, 1, 1, 0, 0, 0, 0, 0, 1044, 1020, 1082, 994, 1091, 1040, 985, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1057, 1005, 982, 1005, 1005, 1021, 1026, 29, 15, 12, 0, 0, 0, 0, 0, 1031, 1014, 995, 1043, 1052, 1049, 1025, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1029, 988, 1050, 1078, 996, 1043, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8177, 8095, 8247, 8260, 8145, 8214, 8228] avg=8192.0000 max=8260 max_rank=4 min=8095 min_rank=2 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1061, 1016, 1032, 1069, 1044, 989, 987, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 985, 1044, 1007, 993, 1051, 1045, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1042, 1021, 1041, 1045, 1029, 1037, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 994, 965, 962, 1031, 1009, 1056, 1048, 1055, 10, 1, 1, 0, 0, 0, 0, 0, 1001, 1029, 1039, 1053, 1037, 960, 1051, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1041, 1031, 1009, 1017, 1050, 1037, 1014, 31, 15, 11, 0, 0, 0, 0, 0, 996, 1026, 1007, 1021, 1018, 1031, 971, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1039, 982, 969, 1026, 1062, 1043, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8157, 8316, 8132, 8195, 8259, 8059, 8189] avg=8192.0000 max=8316 max_rank=2 min=8059 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[953, 1030, 1042, 1067, 1062, 985, 1010, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1073, 1000, 1027, 1032, 1000, 1002, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1018, 1032, 1019, 1041, 1024, 1041, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1033, 1067, 1052, 1053, 976, 1025, 1049, 10, 1, 1, 0, 0, 0, 0, 0, 1035, 1062, 1029, 1037, 1041, 1021, 996, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 942, 961, 1096, 1012, 1031, 986, 1003, 985, 29, 15, 12, 0, 0, 0, 0, 0, 1016, 999, 1015, 1035, 1043, 1057, 1005, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1016, 1015, 993, 1025, 984, 1016, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8244, 8191, 8273, 8267, 8072, 8208, 8124] avg=8192.0000 max=8273 max_rank=3 min=8072 min_rank=5 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1036, 1033, 1047, 1032, 1025, 972, 1046, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1024, 1054, 1006, 1014, 1014, 1009, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 980, 973, 1041, 1033, 1023, 1013, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1050, 971, 1044, 1034, 975, 1030, 979, 10, 1, 1, 0, 0, 0, 0, 0, 1031, 1024, 996, 1013, 990, 1027, 1034, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 992, 1031, 989, 1034, 1036, 1049, 1026, 29, 16, 12, 0, 0, 0, 0, 0, 1053, 1053, 1040, 1038, 1018, 1031, 1040, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 996, 1029, 1102, 1020, 995, 1003, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8159, 8154, 8086, 8149, 8237, 8325, 8211] avg=8192.0000 max=8325 max_rank=6 min=8086 min_rank=3 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32922 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1002, 994, 1061, 1005, 1017, 1031, 970, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1019, 1005, 1002, 1057, 1057, 978, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 957, 1046, 1029, 1008, 1066, 1025, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1044, 1006, 1006, 1039, 1034, 1069, 996, 10, 1, 1, 0, 0, 0, 0, 0, 1053, 1023, 998, 1024, 1024, 1030, 1053, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1015, 1067, 1036, 952, 998, 1007, 993, 30, 15, 11, 0, 0, 0, 0, 0, 1018, 989, 1006, 1042, 1034, 1027, 1027, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 996, 1042, 1025, 1062, 1024, 1041, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8091, 8148, 8220, 8246, 8257, 8163, 8167, 8244] avg=8192.0000 max=8257 max_rank=4 min=8091 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1041, 996, 1055, 986, 1008, 1038, 1014, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1016, 990, 1033, 1077, 1028, 1032, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1009, 997, 1044, 1004, 1002, 985, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1003, 1024, 1006, 1039, 1053, 975, 1001, 10, 1, 1, 0, 0, 0, 0, 0, 1009, 1002, 1031, 1033, 979, 957, 1064, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1021, 1064, 989, 968, 1044, 998, 1007, 31, 15, 11, 0, 0, 0, 0, 0, 1071, 988, 1013, 1048, 990, 1028, 1062, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1050, 1056, 1067, 1013, 1051, 1034, 1075, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8267, 8104, 8109, 8108, 8183, 8220, 8396] avg=8192.0000 max=8396 max_rank=7 min=8104 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=57140 max_load_reduction_pct=0.871887 moved_to_echo=32907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1037, 1054, 1036, 1058, 1047, 1035, 1080, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1034, 1019, 1020, 1008, 1012, 1034, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1016, 1022, 1030, 1081, 1038, 1017, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1035, 1014, 1001, 969, 1058, 1007, 1010, 15, 8, 5, 6, 0, 0, 0, 0, 1013, 1040, 1021, 1008, 1019, 1020, 1050, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1037, 997, 1035, 977, 1054, 1033, 1036, 32, 20, 16, 0, 0, 0, 0, 0, 949, 1048, 1029, 1027, 1010, 1026, 1014, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1013, 961, 1005, 988, 1013, 979, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8361, 8194, 8283, 8158, 8152, 8273, 8109, 8006] avg=8192.0000 max=8361 max_rank=0 min=8006 min_rank=7 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1039, 1053, 1016, 1004, 1016, 1063, 996, 977, 0, 0, 0, 0, 0, 0, 0, 0, 984, 985, 1010, 1007, 1002, 1056, 1057, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 966, 1027, 976, 1058, 1023, 1045, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1021, 1037, 1029, 1055, 1044, 1049, 1007, 10, 1, 1, 0, 0, 0, 0, 0, 1057, 1048, 1019, 987, 1007, 1029, 1012, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 967, 1034, 1042, 1022, 987, 1046, 1016, 29, 16, 11, 0, 0, 0, 0, 0, 970, 1020, 1096, 1022, 1041, 1035, 1018, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 966, 1035, 989, 1041, 985, 1037, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8190, 8173, 8285, 8148, 8204, 8234, 8138] avg=8192.0000 max=8285 max_rank=3 min=8138 min_rank=7 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8448, 8192, 8064, 8192, 8320, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1046, 972, 1023, 1051, 1069, 995, 1055, 1037, 989, 1031, 1025, 1023, 1051, 1034, 1007, 1050, 1015, 1025, 1049, 1022, 1049, 945, 1031, 1036, 1039, 1007, 1012, 1050, 1059, 1018, 1002, 988, 996, 1092, 1014, 1022, 1046, 1046, 1085, 1012, 1002, 1019, 1015, 1001, 1012, 1031, 1046, 1022, 990, 1010, 1067, 1002, 1011, 1020, 1020, 1016, 973, 1010, 1040, 1019, 1031, 1031, 1004, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1026, 996, 1041, 1003, 1042, 1049, 1025, 1026, 1057, 1058, 1042, 995, 977, 1017, 1009, 999, 992, 1015, 1062, 1029, 1014, 1017, 1021, 1012, 1012, 999, 1017, 1000, 1032, 1025, 1029, 982, 1037, 1042, 1043, 1036, 1041, 1028, 1040, 1085, 1025, 1008, 1011, 1034, 1043, 1012, 1012, 1021, 1022, 1006, 1031, 1073, 997, 1021, 1001, 1026, 1046, 1020, 1049, 1042, 1036, 1016, 1009, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1022, 1041, 1045, 1015, 1043, 986, 1049, 1025, 992, 1009, 1010, 1007, 1022, 1063, 1020, 1086, 1033, 1004, 986, 991, 983, 1001, 1025, 1042, 1073, 1023, 1044, 1036, 1034, 1064, 1029, 1025, 1031, 1009, 1020, 1054, 971, 984, 1036, 1038, 1012, 1044, 1025, 1025, 1004, 1019, 1012, 1017, 993, 1039, 1062, 1062, 1023, 1050, 1022, 1017, 1014, 1038, 973, 1004, 1001, 1025, 1050, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[997, 1059, 1065, 1106, 1002, 1051, 1047, 1049, 1006, 1002, 1070, 995, 1006, 1016, 1007, 1038, 985, 1022, 1033, 1042, 1015, 1027, 992, 1043, 1025, 992, 981, 997, 1033, 990, 1027, 1038, 1046, 994, 1044, 1038, 1023, 1020, 1000, 979, 980, 1009, 1004, 1058, 1007, 1033, 1039, 1086, 1047, 1008, 998, 1051, 988, 1023, 1024, 999, 1058, 1012, 1049, 989, 1041, 1022, 1035, 1074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1022, 1037, 1074, 1041, 1020, 1032, 1011, 1049, 1056, 1018, 946, 1003, 1036, 1001, 989, 980, 1020, 1017, 1061, 1064, 990, 1023, 977, 996, 999, 1033, 1050, 1020, 1008, 1028, 1043, 1025, 1013, 1037, 1020, 1057, 1096, 994, 961, 984, 994, 1040, 1015, 1053, 1036, 1073, 1016, 1026, 1010, 1046, 990, 1004, 984, 1022, 1030, 1031, 1032, 1054, 1064, 1060, 1000, 1001, 1056, 1068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[978, 1027, 1019, 1059, 1016, 1010, 1035, 1027, 1020, 961, 1000, 1035, 1067, 956, 1008, 990, 996, 983, 1102, 1049, 1045, 1048, 1070, 1049, 1076, 1084, 962, 1035, 1071, 1041, 1029, 1025, 968, 1023, 974, 1040, 1019, 990, 996, 1027, 1014, 1011, 1050, 993, 1019, 1087, 995, 1047, 962, 997, 1015, 1035, 1054, 982, 1028, 1015, 1000, 1026, 1046, 1058, 1027, 1065, 1030, 1070] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1031, 1034, 1068, 1062, 1040, 1026, 1003, 1033, 996, 1012, 1032, 979, 1015, 1031, 1014, 1008, 1032, 1057, 1011, 993, 1091, 1007, 1003, 1002, 1078, 1048, 1040, 1035, 1010, 1018, 1009, 1010, 1073, 1039, 1035, 1036, 1071, 1029, 1034, 1005, 1048, 1008, 1049, 972, 1060, 1022, 1017, 979, 1033, 970, 1017, 953, 1017, 1039, 1062, 997, 1022, 1021, 1029, 1042, 990, 1043, 1009, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1008, 1039, 1044, 1020, 1052, 997, 989, 994, 996, 1042, 1037, 1022, 994, 1016, 1036, 1025, 1019, 999, 1048, 1067, 1085, 1061, 1035, 1035, 1002, 1030, 1048, 1045, 994, 999, 1091, 987, 1031, 1017, 1044, 1034, 1044, 1048, 982, 1014, 1016, 1066, 1012, 1033, 984, 1044, 1063, 1044, 1035, 1039, 993, 976, 992, 995, 1062, 1057, 1034, 1016, 993, 957, 993, 1015, 1008, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1046, 972, 972, 1051, 1069, 995, 1055, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1031, 1025, 1023, 1051, 1034, 1007, 1050, 51, 10, 0, 0, 0, 0, 0, 0, 1015, 1025, 1042, 1022, 1049, 945, 1031, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1007, 1012, 1050, 1059, 1018, 1002, 988, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1092, 1014, 1004, 1046, 1046, 1085, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1019, 1015, 1001, 1012, 1027, 1046, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1010, 1067, 1002, 1011, 1020, 1020, 1016, 14, 11, 8, 7, 4, 0, 0, 0, 973, 1010, 1040, 1019, 1031, 1031, 1004, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8271, 8165, 8161, 8295, 8144, 8180, 8123] avg=8192.0000 max=8295 max_rank=4 min=8123 min_rank=7 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1022, 1041, 993, 1015, 1043, 986, 1049, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1009, 1010, 1007, 1022, 1063, 1020, 1086, 52, 10, 0, 0, 0, 0, 0, 0, 1033, 1004, 980, 991, 983, 1001, 1025, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1023, 1044, 1036, 1034, 1064, 1029, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1009, 1020, 1036, 971, 984, 1036, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1044, 1025, 1025, 1004, 1015, 1012, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1039, 1062, 1062, 1023, 1050, 1022, 1017, 15, 11, 8, 6, 4, 0, 0, 0, 1014, 1038, 973, 1004, 1001, 1025, 1050, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8271, 8059, 8313, 8125, 8154, 8312, 8128] avg=8192.0000 max=8313 max_rank=3 min=8059 min_rank=2 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[978, 1027, 969, 1059, 1016, 1010, 1035, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 961, 1000, 1035, 1067, 956, 1008, 990, 50, 10, 0, 0, 0, 0, 0, 0, 996, 983, 1095, 1049, 1045, 1048, 1070, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1084, 962, 1035, 1071, 1041, 1029, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1023, 974, 1022, 1019, 990, 996, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1011, 1050, 993, 1019, 1083, 995, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 962, 997, 1015, 1035, 1054, 982, 1028, 1015, 15, 11, 8, 7, 4, 0, 0, 0, 1000, 1026, 1046, 1058, 1027, 1065, 1030, 1059, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8097, 8335, 8308, 8019, 8212, 8133, 8311] avg=8192.0000 max=8335 max_rank=2 min=8019 min_rank=4 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32675 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 996, 989, 1003, 1042, 1049, 1025, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1058, 1042, 995, 977, 1017, 1009, 999, 52, 10, 0, 0, 0, 0, 0, 0, 992, 1015, 1055, 1029, 1014, 1017, 1021, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 998, 999, 1017, 1000, 1032, 1025, 1029, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1042, 1043, 1018, 1041, 1028, 1040, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1008, 1011, 1034, 1043, 1008, 1012, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1006, 1031, 1073, 997, 1021, 1001, 1026, 14, 11, 8, 7, 4, 0, 0, 0, 1046, 1020, 1049, 1042, 1036, 1016, 1009, 992, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8216, 8155, 8082, 8334, 8162, 8221, 8210] avg=8192.0000 max=8334 max_rank=4 min=8082 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32927 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[997, 1059, 1012, 1106, 1002, 1051, 1047, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1002, 1070, 995, 1006, 1016, 1007, 1038, 53, 10, 0, 0, 0, 0, 0, 0, 985, 1022, 1026, 1042, 1015, 1027, 992, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 992, 981, 997, 1033, 990, 1027, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 994, 1044, 1020, 1023, 1020, 1000, 979, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1009, 1004, 1058, 1007, 1029, 1039, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1008, 998, 1051, 988, 1023, 1024, 999, 14, 12, 8, 7, 4, 0, 0, 0, 1058, 1012, 1049, 989, 1041, 1022, 1035, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8323, 8203, 8152, 8069, 8126, 8212, 8183, 8268] avg=8192.0000 max=8323 max_rank=0 min=8069 min_rank=3 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1031, 1034, 1015, 1062, 1040, 1026, 1003, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1012, 1032, 979, 1015, 1031, 1014, 1008, 53, 10, 0, 0, 0, 0, 0, 0, 1032, 1057, 1005, 993, 1091, 1007, 1003, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1048, 1040, 1035, 1010, 1018, 1009, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1039, 1035, 1018, 1071, 1029, 1034, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1008, 1049, 972, 1060, 1018, 1017, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 970, 1017, 953, 1017, 1039, 1062, 997, 15, 11, 8, 6, 4, 0, 0, 0, 1022, 1021, 1029, 1042, 990, 1043, 1009, 976, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8150, 8190, 8233, 8304, 8151, 8132, 8132] avg=8192.0000 max=8304 max_rank=4 min=8132 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1022, 1037, 1018, 1041, 1020, 1032, 1011, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1018, 946, 1003, 1036, 1001, 989, 980, 56, 13, 0, 0, 0, 0, 0, 0, 1020, 1017, 1051, 1064, 990, 1023, 977, 996, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1033, 1050, 1020, 1008, 1028, 1043, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1037, 1020, 1032, 1096, 994, 961, 984, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1040, 1015, 1053, 1036, 1063, 1016, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1046, 990, 1004, 984, 1022, 1030, 1031, 18, 15, 12, 10, 10, 0, 0, 0, 1032, 1054, 1064, 1060, 1000, 1001, 1056, 1053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8098, 8138, 8188, 8137, 8243, 8182, 8320] avg=8192.0000 max=8320 max_rank=7 min=8098 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1008, 1039, 992, 1020, 1052, 997, 989, 994, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1042, 1037, 1022, 994, 1016, 1036, 1025, 52, 10, 0, 0, 0, 0, 0, 0, 1019, 999, 1041, 1067, 1085, 1061, 1035, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1030, 1048, 1045, 994, 999, 1091, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1017, 1044, 1016, 1044, 1048, 982, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1066, 1012, 1033, 984, 1040, 1063, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1039, 993, 976, 992, 995, 1062, 1057, 14, 11, 8, 7, 4, 0, 0, 0, 1034, 1016, 993, 957, 993, 1015, 1008, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8091, 8230, 8342, 8182, 8196, 8258, 8193, 8044] avg=8192.0000 max=8342 max_rank=2 min=8044 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8064, 8448, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8064, 8192, 8192, 8192, 8192, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8320, 8192, 128, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1029, 1019, 965, 1028, 1038, 992, 1025, 1048, 1024, 1026, 1040, 1001, 1032, 1047, 1042, 1014, 1075, 1049, 985, 973, 1050, 1068, 1041, 1038, 1011, 1034, 1033, 1068, 1024, 1031, 1031, 999, 1040, 997, 1067, 1018, 1058, 1010, 992, 1019, 1025, 984, 1003, 1046, 1019, 1007, 1035, 1043, 1057, 1022, 997, 988, 1066, 1022, 961, 1021, 995, 1029, 1046, 1002, 1064, 1003, 1018, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[992, 1023, 1046, 1038, 995, 1006, 1006, 1022, 1009, 1034, 1022, 1039, 1025, 1027, 1078, 1014, 1003, 1012, 1038, 998, 964, 1021, 1080, 1085, 1021, 1033, 991, 1015, 1020, 980, 1057, 1065, 1067, 999, 1044, 999, 1017, 1030, 1027, 1026, 1016, 1060, 1029, 1050, 1003, 996, 1044, 962, 1022, 989, 1002, 967, 1055, 1003, 985, 1024, 1064, 1025, 990, 1026, 1078, 1080, 1034, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1041, 1014, 1048, 1008, 992, 1013, 1001, 1027, 1053, 1044, 1013, 1052, 1025, 1046, 1039, 1069, 1084, 1012, 968, 967, 1011, 1004, 1000, 1041, 1010, 1037, 1004, 1003, 1005, 1015, 997, 1030, 1060, 1042, 1083, 984, 1018, 1055, 1004, 1042, 1057, 1069, 1039, 1023, 1008, 1048, 1005, 1032, 1012, 1033, 1005, 1077, 1015, 999, 980, 1055, 1030, 1028, 971, 1023, 1029, 1020, 1001, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1020, 1033, 1012, 1043, 1054, 1044, 982, 1043, 1006, 995, 1085, 996, 1001, 1030, 1054, 1005, 1026, 1027, 1004, 974, 995, 1061, 963, 1065, 1002, 977, 976, 1011, 1057, 1025, 1002, 978, 1018, 1011, 1072, 1001, 1009, 1012, 1041, 1045, 1050, 1051, 1039, 1003, 1015, 1025, 1048, 1044, 1046, 1062, 1056, 1011, 1022, 1061, 977, 1037, 1022, 1064, 1048, 1024, 1043, 1009, 1032, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1063, 1030, 998, 1047, 1067, 1056, 1018, 1030, 1029, 1046, 994, 1005, 1051, 1032, 1055, 1009, 969, 1019, 1019, 1043, 1020, 964, 990, 1038, 1066, 994, 990, 999, 1081, 1050, 976, 1007, 1033, 998, 1024, 1061, 1043, 1030, 1060, 1027, 1029, 980, 1055, 1041, 1078, 1002, 1042, 1008, 929, 1037, 1036, 1038, 994, 986, 1054, 1008, 1014, 1028, 1038, 978, 1003, 1034, 1064, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[996, 996, 1014, 986, 1008, 1042, 1025, 1033, 1030, 1019, 1005, 1077, 1024, 1009, 1069, 1049, 1014, 1037, 1026, 995, 1044, 998, 1008, 1074, 1032, 1053, 1026, 1030, 986, 1028, 1077, 1038, 1016, 1001, 1019, 1013, 1044, 1039, 1091, 1051, 1023, 1029, 1023, 1049, 1016, 979, 1007, 1016, 1010, 1012, 1044, 1019, 993, 1036, 1047, 1014, 1020, 1063, 994, 1035, 980, 1017, 989, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1022, 1057, 1004, 981, 1011, 1050, 971, 1058, 1001, 1033, 1006, 1013, 1010, 1036, 997, 1081, 1036, 1016, 1011, 1076, 1033, 988, 1055, 1006, 1018, 1008, 1069, 1062, 1009, 996, 1049, 992, 1020, 1008, 1051, 1090, 978, 1035, 1009, 1059, 1042, 979, 1048, 1023, 988, 1017, 996, 985, 1019, 1056, 1035, 1043, 1039, 1052, 1021, 1000, 1027, 1013, 976, 1049, 1019, 991, 1046, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[980, 1058, 1053, 996, 1032, 1028, 1006, 1010, 1031, 1012, 1028, 1076, 961, 1014, 1030, 1014, 1083, 994, 1066, 1017, 1057, 1042, 1044, 1000, 1033, 1026, 981, 1030, 1032, 1040, 1022, 1020, 1047, 1065, 980, 1029, 1070, 1000, 1011, 1023, 968, 961, 1028, 1033, 1028, 1015, 1029, 994, 1034, 1009, 1032, 1054, 1047, 1008, 1041, 1072, 1009, 1008, 995, 1012, 1071, 1016, 1041, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1041, 1014, 1048, 1008, 992, 1013, 1001, 1027, 21, 0, 0, 0, 0, 0, 0, 0, 1053, 1044, 1013, 1052, 1025, 1046, 1005, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 1012, 968, 967, 1011, 1004, 1000, 1041, 8, 0, 0, 0, 0, 0, 0, 0, 1010, 1037, 1004, 1003, 1005, 1015, 997, 1030, 22, 0, 0, 0, 0, 0, 0, 0, 1060, 1042, 1033, 984, 1018, 1055, 1004, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1069, 1039, 1023, 1008, 1048, 1005, 1032, 10, 7, 0, 0, 0, 0, 0, 0, 1012, 1033, 1005, 1077, 1015, 999, 980, 1055, 10, 0, 0, 0, 0, 0, 0, 0, 1030, 1028, 971, 1023, 1029, 1020, 1001, 1016, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8307, 8095, 8123, 8238, 8298, 8186, 8124] avg=8192.0000 max=8307 max_rank=1 min=8095 min_rank=2 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1029, 1019, 965, 1028, 1038, 992, 1025, 1048, 20, 0, 0, 0, 0, 0, 0, 0, 1024, 1026, 1040, 1001, 1032, 1047, 1008, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1049, 985, 973, 1050, 1068, 1041, 1038, 8, 0, 0, 0, 0, 0, 0, 0, 1011, 1034, 1033, 1068, 1024, 1031, 1031, 999, 22, 0, 0, 0, 0, 0, 0, 0, 1040, 997, 1018, 1018, 1058, 1010, 992, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 984, 1003, 1046, 1019, 1007, 1035, 1043, 10, 7, 0, 0, 0, 0, 0, 0, 1057, 1022, 997, 988, 1066, 1022, 961, 1021, 10, 0, 0, 0, 0, 0, 0, 0, 995, 1029, 1046, 1002, 1064, 1003, 1018, 1002, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8192, 8287, 8253, 8152, 8179, 8144, 8165] avg=8192.0000 max=8287 max_rank=2 min=8144 min_rank=6 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1020, 1033, 1012, 1043, 1054, 1044, 982, 1043, 20, 0, 0, 0, 0, 0, 0, 0, 1006, 995, 1085, 996, 1001, 1030, 1017, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1027, 1004, 974, 995, 1061, 963, 1065, 9, 0, 0, 0, 0, 0, 0, 0, 1002, 977, 976, 1011, 1057, 1025, 1002, 978, 22, 0, 0, 0, 0, 0, 0, 0, 1018, 1011, 1023, 1001, 1009, 1012, 1041, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1051, 1039, 1003, 1015, 1025, 1048, 1044, 11, 7, 0, 0, 0, 0, 0, 0, 1046, 1062, 1056, 1011, 1022, 1061, 977, 1037, 11, 0, 0, 0, 0, 0, 0, 0, 1022, 1064, 1048, 1024, 1043, 1009, 1032, 992, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8135, 8124, 8050, 8160, 8293, 8283, 8240] avg=8192.0000 max=8293 max_rank=5 min=8050 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[992, 1023, 1046, 1038, 995, 1006, 1006, 1022, 20, 0, 0, 0, 0, 0, 0, 0, 1009, 1034, 1022, 1039, 1025, 1027, 1041, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1012, 1038, 998, 964, 1021, 1080, 1085, 9, 0, 0, 0, 0, 0, 0, 0, 1021, 1033, 991, 1015, 1020, 980, 1057, 1065, 21, 0, 0, 0, 0, 0, 0, 0, 1067, 999, 996, 999, 1017, 1030, 1027, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1060, 1029, 1050, 1003, 996, 1044, 962, 11, 7, 0, 0, 0, 0, 0, 0, 1022, 989, 1002, 967, 1055, 1003, 985, 1024, 11, 0, 0, 0, 0, 0, 0, 0, 1064, 1025, 990, 1026, 1078, 1080, 1034, 1064, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8211, 8210, 8203, 8161, 8178, 8058, 8367] avg=8192.0000 max=8367 max_rank=7 min=8058 min_rank=6 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1063, 1030, 998, 1047, 1067, 1056, 1018, 1030, 20, 0, 0, 0, 0, 0, 0, 0, 1029, 1046, 994, 1005, 1051, 1032, 1018, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1019, 1019, 1043, 1020, 964, 990, 1038, 9, 0, 0, 0, 0, 0, 0, 0, 1066, 994, 990, 999, 1081, 1050, 976, 1007, 21, 0, 0, 0, 0, 0, 0, 0, 1033, 998, 976, 1061, 1043, 1030, 1060, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 980, 1055, 1041, 1078, 1002, 1042, 1008, 11, 7, 0, 0, 0, 0, 0, 0, 929, 1037, 1036, 1038, 994, 986, 1054, 1008, 11, 0, 0, 0, 0, 0, 0, 0, 1014, 1028, 1038, 978, 1003, 1034, 1064, 1029, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8329, 8184, 8071, 8184, 8228, 8253, 8093, 8194] avg=8192.0000 max=8329 max_rank=0 min=8071 min_rank=2 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[996, 996, 1014, 986, 1008, 1042, 1025, 1033, 23, 0, 0, 0, 0, 0, 0, 0, 1030, 1019, 1005, 1077, 1024, 1009, 1018, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1037, 1026, 995, 1044, 998, 1008, 1074, 13, 0, 0, 0, 0, 0, 0, 0, 1032, 1053, 1026, 1030, 986, 1028, 1077, 1038, 25, 0, 0, 0, 0, 0, 0, 0, 1016, 1001, 960, 1013, 1044, 1039, 1091, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1029, 1023, 1049, 1016, 979, 1007, 1016, 15, 11, 0, 0, 0, 0, 0, 0, 1010, 1012, 1044, 1019, 993, 1036, 1047, 1014, 15, 0, 0, 0, 0, 0, 0, 0, 1020, 1063, 994, 1035, 980, 1017, 989, 999, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8231, 8209, 8295, 8215, 8168, 8190, 8105] avg=8192.0000 max=8295 max_rank=3 min=8105 min_rank=7 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1022, 1057, 1004, 981, 1011, 1050, 971, 1058, 20, 0, 0, 0, 0, 0, 0, 0, 1001, 1033, 1006, 1013, 1010, 1036, 964, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1016, 1011, 1076, 1033, 988, 1055, 1006, 8, 0, 0, 0, 0, 0, 0, 0, 1018, 1008, 1069, 1062, 1009, 996, 1049, 992, 21, 0, 0, 0, 0, 0, 0, 0, 1020, 1008, 1003, 1090, 978, 1035, 1009, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 979, 1048, 1023, 988, 1017, 996, 985, 10, 7, 0, 0, 0, 0, 0, 0, 1019, 1056, 1035, 1043, 1039, 1052, 1021, 1000, 10, 0, 0, 0, 0, 0, 0, 0, 1027, 1013, 976, 1049, 1019, 991, 1046, 1067, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8144, 8229, 8224, 8202, 8095, 8275, 8193] avg=8192.0000 max=8275 max_rank=6 min=8095 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[980, 1058, 1053, 996, 1032, 1028, 1006, 1010, 19, 0, 0, 0, 0, 0, 0, 0, 1031, 1012, 1028, 1076, 961, 1014, 996, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 994, 1066, 1017, 1057, 1042, 1044, 1000, 8, 0, 0, 0, 0, 0, 0, 0, 1033, 1026, 981, 1030, 1032, 1040, 1022, 1020, 20, 0, 0, 0, 0, 0, 0, 0, 1047, 1065, 935, 1029, 1070, 1000, 1011, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 968, 961, 1028, 1033, 1028, 1015, 1029, 994, 10, 6, 0, 0, 0, 0, 0, 0, 1034, 1009, 1032, 1054, 1047, 1008, 1041, 1072, 10, 0, 0, 0, 0, 0, 0, 0, 1009, 1008, 995, 1012, 1071, 1016, 1041, 990, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8132, 8311, 8204, 8180, 8072, 8307, 8148] avg=8192.0000 max=8311 max_rank=2 min=8072 min_rank=5 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8064, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8064, 8192, 8192, 8192, 8448, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8192, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8064, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1079, 1008, 1028, 1048, 971, 990, 983, 1009, 1037, 1023, 993, 1061, 981, 1007, 1040, 1013, 1031, 1073, 1026, 999, 1004, 1034, 1032, 1066, 1027, 999, 1021, 1011, 1013, 1024, 1011, 1022, 1075, 997, 993, 1062, 1011, 1009, 1030, 1007, 1035, 1021, 1062, 1020, 1000, 1003, 1041, 1008, 1047, 986, 1051, 1002, 1048, 1022, 1031, 1056, 1016, 1061, 1026, 1009, 1030, 1038, 1027, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1015, 1055, 992, 1009, 1002, 1024, 1063, 1024, 1041, 1070, 1087, 1052, 1063, 1024, 1014, 984, 1032, 1015, 1030, 1026, 995, 970, 1005, 1058, 1015, 1004, 1026, 1027, 997, 1074, 1038, 1000, 1008, 984, 1048, 1050, 1021, 1037, 995, 996, 1038, 1041, 1000, 1026, 1042, 1025, 1050, 1060, 987, 1043, 1064, 977, 1023, 1057, 991, 1025, 1012, 1060, 1016, 1044, 926, 968, 1050, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1018, 998, 1047, 1008, 998, 1034, 1045, 1031, 1059, 1087, 986, 943, 1032, 1005, 1059, 1024, 1074, 1039, 995, 1049, 1083, 1078, 1019, 1039, 1010, 1027, 1026, 982, 1043, 1018, 1018, 1006, 970, 1045, 1038, 1035, 1082, 986, 1008, 1032, 975, 1015, 988, 970, 1038, 1063, 1071, 1017, 1021, 1020, 1056, 992, 1001, 1040, 988, 1024, 980, 1022, 1028, 1050, 1011, 1040, 1011, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1082, 959, 1012, 1020, 971, 1048, 1069, 1034, 1000, 1015, 1012, 1046, 1044, 1057, 1054, 1014, 1044, 1008, 1038, 1035, 974, 992, 1031, 1021, 1035, 1057, 1024, 1045, 998, 982, 1029, 1048, 1029, 999, 1009, 1029, 1018, 1023, 989, 996, 1025, 990, 1060, 979, 995, 1032, 1005, 1025, 1093, 1022, 1062, 1028, 1032, 1067, 993, 1021, 1000, 999, 993, 1056, 1058, 1089, 1030, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1047, 1014, 973, 988, 1006, 978, 1017, 965, 1033, 1068, 1048, 1059, 1009, 1022, 1012, 993, 1005, 1034, 1001, 987, 1035, 1012, 1025, 1057, 1088, 1053, 958, 1046, 1063, 1039, 1055, 1012, 1034, 1018, 1028, 1013, 1043, 1033, 1064, 1035, 1027, 1025, 1068, 1047, 1052, 1037, 994, 1011, 1032, 998, 1006, 1016, 999, 1020, 1063, 1025, 1027, 1031, 1000, 983, 1008, 1030, 1061, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1053, 1076, 993, 1032, 1035, 1017, 974, 1015, 1044, 1009, 1032, 1054, 1014, 979, 1025, 1079, 999, 1038, 1043, 1004, 986, 1049, 1030, 968, 1041, 986, 1018, 1046, 1001, 1061, 1010, 1001, 1038, 1040, 1017, 1062, 977, 1035, 1027, 968, 993, 1056, 1030, 1032, 1050, 982, 1022, 987, 1042, 1092, 1071, 1023, 1021, 1040, 991, 1034, 1038, 979, 1016, 1007, 1082, 1043, 986, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1012, 1062, 1048, 1069, 993, 1084, 1008, 1006, 1001, 1023, 1075, 1006, 1015, 1025, 1033, 1012, 989, 1026, 1018, 996, 1061, 1030, 999, 1055, 1019, 967, 1040, 1069, 1019, 1047, 1014, 1010, 1020, 1021, 1065, 1014, 1042, 1032, 1039, 991, 1039, 966, 994, 1032, 988, 1039, 1004, 1079, 990, 1038, 982, 1032, 1064, 1012, 1026, 1012, 1018, 987, 1032, 992, 1034, 1018, 1052, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[993, 1062, 1023, 1029, 1070, 1014, 983, 1026, 1004, 1064, 1042, 1037, 1068, 1078, 1028, 1007, 1014, 1083, 992, 999, 1064, 1039, 1015, 973, 1016, 1057, 1029, 1016, 1012, 1028, 1060, 995, 1025, 1036, 1050, 1005, 987, 990, 1092, 1083, 1079, 1035, 1066, 1013, 1013, 1042, 1052, 1007, 1002, 1010, 984, 1037, 985, 1020, 992, 991, 1025, 969, 1025, 1012, 1005, 985, 993, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1079, 1008, 1028, 1048, 971, 990, 983, 1009, 20, 3, 0, 0, 0, 0, 0, 0, 1037, 976, 993, 1061, 981, 1007, 1040, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1073, 1026, 999, 1004, 1034, 1032, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 999, 1021, 1011, 1013, 1024, 1011, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 997, 993, 1062, 1011, 1009, 1030, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1021, 1058, 1020, 1000, 1003, 1041, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 986, 1051, 1002, 1048, 1022, 1031, 1056, 1, 0, 0, 0, 0, 0, 0, 0, 1016, 1061, 1026, 1009, 1030, 1038, 1027, 1048, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8108, 8265, 8128, 8184, 8186, 8244, 8282] avg=8192.0000 max=8282 max_rank=7 min=8108 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1018, 998, 1047, 1008, 998, 1034, 1045, 1031, 21, 3, 0, 0, 0, 0, 0, 0, 1059, 1038, 986, 943, 1032, 1005, 1059, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1039, 995, 1049, 1083, 1078, 1019, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1027, 1026, 982, 1043, 1018, 1018, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1045, 1038, 1035, 1082, 986, 1008, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1015, 984, 970, 1038, 1063, 1071, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1020, 1056, 992, 1001, 1040, 988, 1024, 1, 0, 0, 0, 0, 0, 0, 0, 980, 1022, 1028, 1050, 1011, 1040, 1011, 1039, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8146, 8376, 8130, 8196, 8133, 8143, 8209] avg=8192.0000 max=8376 max_rank=2 min=8130 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1015, 1055, 992, 1009, 1002, 1024, 1063, 1024, 21, 3, 0, 0, 0, 0, 0, 0, 1041, 1021, 1087, 1052, 1063, 1024, 1014, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1015, 1030, 1026, 995, 970, 1005, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1004, 1026, 1027, 997, 1074, 1038, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 984, 1048, 1050, 1021, 1037, 995, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1041, 996, 1026, 1042, 1025, 1050, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1043, 1064, 977, 1023, 1057, 991, 1025, 1, 0, 0, 0, 0, 0, 0, 0, 1012, 1060, 1016, 1044, 926, 968, 1050, 1041, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8286, 8131, 8181, 8139, 8278, 8168, 8145] avg=8192.0000 max=8286 max_rank=1 min=8131 min_rank=2 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1082, 959, 1012, 1020, 971, 1048, 1069, 1034, 20, 3, 0, 0, 0, 0, 0, 0, 1000, 969, 1012, 1046, 1044, 1057, 1054, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1008, 1038, 1035, 974, 992, 1031, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1057, 1024, 1045, 998, 982, 1029, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 999, 1009, 1029, 1018, 1023, 989, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 990, 1056, 979, 995, 1032, 1005, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 1022, 1062, 1028, 1032, 1067, 993, 1021, 1, 0, 0, 0, 0, 0, 0, 0, 1000, 999, 993, 1056, 1058, 1089, 1030, 992, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8196, 8143, 8218, 8092, 8107, 8319, 8243] avg=8192.0000 max=8319 max_rank=6 min=8092 min_rank=4 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1053, 1076, 993, 1032, 1035, 1017, 974, 1015, 20, 3, 0, 0, 0, 0, 0, 0, 1044, 963, 1032, 1054, 1014, 979, 1025, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1038, 1043, 1004, 986, 1049, 1030, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 986, 1018, 1046, 1001, 1061, 1010, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1040, 1017, 1062, 977, 1035, 1027, 968, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1056, 1026, 1032, 1050, 982, 1022, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1092, 1071, 1023, 1021, 1040, 991, 1034, 1, 0, 0, 0, 0, 0, 0, 0, 1038, 979, 1016, 1007, 1082, 1043, 986, 1043, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8190, 8117, 8164, 8164, 8148, 8315, 8220] avg=8192.0000 max=8315 max_rank=6 min=8117 min_rank=2 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1012, 1062, 1048, 1069, 993, 1084, 1008, 1006, 23, 9, 0, 0, 0, 0, 0, 0, 1001, 969, 1075, 1006, 1015, 1025, 1033, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1021, 1018, 996, 1061, 1030, 999, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 967, 1040, 1069, 1019, 1047, 1014, 1010, 3, 0, 0, 0, 0, 0, 0, 0, 1020, 1021, 1065, 1014, 1042, 1032, 1039, 991, 1, 0, 0, 0, 0, 0, 0, 0, 1039, 966, 977, 1032, 988, 1039, 1004, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1038, 982, 1032, 1064, 1012, 1026, 1012, 8, 1, 0, 0, 0, 0, 0, 0, 1018, 987, 1032, 992, 1034, 1018, 1052, 1051, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8314, 8136, 8169, 8188, 8225, 8124, 8165, 8215] avg=8192.0000 max=8314 max_rank=0 min=8124 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1047, 1014, 973, 988, 1006, 978, 1017, 965, 21, 3, 0, 0, 0, 0, 0, 0, 1033, 1019, 1048, 1059, 1009, 1022, 1012, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1034, 1001, 987, 1035, 1012, 1025, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 1053, 958, 1046, 1063, 1039, 1055, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1018, 1028, 1013, 1043, 1033, 1064, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1025, 1064, 1047, 1052, 1037, 994, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 998, 1006, 1016, 999, 1020, 1063, 1025, 1, 0, 0, 0, 0, 0, 0, 0, 1027, 1031, 1000, 983, 1008, 1030, 1061, 1006, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8012, 8195, 8156, 8314, 8268, 8257, 8160, 8174] avg=8192.0000 max=8314 max_rank=3 min=8012 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[993, 1062, 1023, 1029, 1070, 1014, 983, 1026, 21, 3, 0, 0, 0, 0, 0, 0, 1004, 1015, 1042, 1037, 1068, 1078, 1028, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1083, 992, 999, 1064, 1039, 1015, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1057, 1029, 1016, 1012, 1028, 1060, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1036, 1050, 1005, 987, 990, 1092, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1035, 1062, 1013, 1013, 1042, 1052, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1010, 984, 1037, 985, 1020, 992, 991, 1, 0, 0, 0, 0, 0, 0, 0, 1025, 969, 1025, 1012, 1005, 985, 993, 1006, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8279, 8179, 8213, 8268, 8303, 8022, 8048] avg=8192.0000 max=8303 max_rank=5 min=8022 min_rank=6 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8064, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[996, 1015, 1034, 1067, 1028, 984, 1030, 999, 1003, 1013, 1040, 1032, 986, 1054, 1081, 1021, 1021, 989, 1060, 1002, 987, 1047, 1031, 1019, 1004, 1085, 1021, 1006, 1054, 1083, 1038, 1040, 1000, 1009, 1041, 1001, 1021, 1049, 1020, 1017, 1022, 1056, 1089, 1050, 962, 1024, 1052, 995, 1055, 1018, 1046, 1019, 1024, 947, 1019, 977, 1036, 980, 1057, 1023, 1005, 1024, 1026, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1026, 1002, 1033, 1045, 1054, 1037, 1038, 992, 1014, 1038, 1014, 987, 1021, 1048, 1032, 1022, 1050, 1041, 1048, 1043, 1031, 1024, 1013, 991, 1042, 1013, 1024, 1041, 1026, 1015, 995, 1018, 1028, 1031, 989, 1003, 1006, 1019, 1038, 1011, 1047, 1017, 1036, 999, 1035, 1029, 1031, 1056, 979, 1009, 973, 994, 1018, 1008, 1030, 1069, 1013, 993, 1039, 1078, 1003, 1030, 1078, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1042, 1054, 1006, 967, 996, 998, 1018, 1064, 1063, 1011, 1080, 1008, 1012, 1114, 1059, 1019, 1037, 1011, 1035, 1008, 1007, 1054, 1012, 1046, 1025, 1054, 1004, 976, 1080, 1028, 1048, 1076, 969, 991, 1035, 1000, 1024, 966, 1017, 1025, 1020, 1019, 983, 1000, 1044, 995, 995, 1034, 1049, 1043, 991, 1036, 1047, 1075, 1010, 1021, 1027, 1092, 1035, 995, 1026, 1008, 971, 981] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[999, 1003, 984, 1032, 1018, 971, 1067, 1029, 1022, 1050, 1023, 1006, 1065, 1048, 1047, 1043, 1044, 1008, 1043, 1056, 1013, 1035, 1014, 1036, 1062, 1022, 996, 1017, 1088, 993, 1029, 993, 1026, 1022, 1030, 1008, 1067, 1022, 1061, 1015, 1038, 1006, 1017, 967, 1040, 981, 1029, 1070, 995, 1048, 1027, 1020, 1001, 982, 1029, 1019, 1009, 1016, 1020, 1032, 994, 1068, 997, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[991, 1024, 1006, 1017, 1031, 1088, 1020, 1043, 1006, 977, 1075, 1012, 1045, 1018, 1049, 1032, 1015, 1024, 1061, 1054, 968, 1061, 983, 1016, 1028, 1053, 964, 1024, 1045, 1003, 1022, 1025, 1039, 1007, 1001, 1031, 1023, 1009, 1073, 1034, 975, 1041, 1048, 1035, 1015, 987, 1014, 1011, 1058, 997, 1020, 1002, 1009, 1001, 1038, 1033, 1054, 1087, 1023, 1051, 1021, 1017, 988, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1046, 1055, 999, 1027, 1049, 1016, 1005, 1075, 1005, 1038, 1012, 1032, 1007, 1030, 1000, 1016, 1034, 970, 1104, 973, 1006, 1021, 1024, 982, 1016, 1043, 1030, 1072, 1000, 1005, 1033, 1000, 1002, 1048, 1028, 1029, 982, 1003, 1050, 1014, 991, 1020, 1075, 1040, 983, 1067, 996, 971, 1065, 981, 1037, 1025, 1061, 1064, 1040, 1022, 1041, 990, 1043, 1052, 1024, 1033, 986, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1012, 1048, 1022, 1046, 997, 973, 1007, 1004, 1066, 1086, 1031, 1078, 1043, 1032, 1008, 1014, 1030, 989, 966, 1001, 1066, 1003, 1029, 1029, 979, 1062, 1023, 1019, 1012, 989, 1021, 1004, 1046, 990, 1032, 1004, 1052, 1022, 1082, 955, 1028, 1060, 1059, 1024, 1003, 1025, 1037, 1018, 1017, 991, 1009, 1020, 1020, 1037, 1034, 1047, 987, 1073, 1070, 974, 1055, 1056, 1037, 983] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1037, 999, 1033, 1015, 1015, 1034, 1010, 1035, 1008, 1000, 1041, 1073, 1027, 1001, 1039, 1057, 1039, 1008, 980, 1029, 1002, 1020, 1069, 959, 1040, 1002, 995, 1046, 995, 1043, 1017, 998, 1032, 1045, 1032, 1029, 1031, 1056, 1076, 1042, 1046, 1070, 1016, 1024, 1053, 1016, 1042, 1026, 981, 1002, 1004, 1054, 1016, 1010, 1032, 991, 1037, 979, 1009, 1043, 1019, 1036, 1000, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1042, 1054, 1006, 967, 996, 998, 1018, 1064, 12, 4, 0, 0, 0, 0, 0, 0, 1063, 1011, 1080, 1008, 1012, 1048, 1059, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1011, 1035, 1008, 1007, 1054, 1012, 1046, 22, 0, 0, 0, 0, 0, 0, 0, 1025, 1046, 1004, 976, 1080, 1028, 1048, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 969, 991, 1035, 1000, 1024, 966, 1017, 1025, 4, 3, 2, 0, 0, 0, 0, 0, 1020, 1019, 981, 1000, 1044, 995, 995, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1043, 991, 1036, 1047, 1075, 1010, 1021, 32, 0, 0, 0, 0, 0, 0, 0, 1027, 1092, 1032, 995, 1026, 1008, 971, 981, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8300, 8232, 8283, 8036, 8088, 8304, 8132] avg=8192.0000 max=8304 max_rank=6 min=8036 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32560 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[999, 1003, 984, 1032, 1018, 971, 1067, 1029, 11, 4, 0, 0, 0, 0, 0, 0, 1022, 1050, 1023, 1006, 1065, 987, 1047, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1008, 1043, 1056, 1013, 1035, 1014, 1036, 20, 0, 0, 0, 0, 0, 0, 0, 1062, 1015, 996, 1017, 1088, 993, 1029, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1022, 1030, 1008, 1067, 1022, 1061, 1015, 3, 3, 2, 0, 0, 0, 0, 0, 1038, 1006, 1015, 967, 1040, 981, 1029, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1048, 1027, 1020, 1001, 982, 1029, 1019, 30, 0, 0, 0, 0, 0, 0, 0, 1009, 1016, 1017, 1032, 994, 1068, 997, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8243, 8269, 8193, 8259, 8146, 8151, 8157] avg=8192.0000 max=8269 max_rank=2 min=8118 min_rank=0 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[996, 1015, 1034, 1067, 1028, 984, 1030, 999, 11, 4, 0, 0, 0, 0, 0, 0, 1003, 1013, 1040, 1032, 986, 993, 1081, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 989, 1060, 1002, 987, 1047, 1031, 1019, 20, 0, 0, 0, 0, 0, 0, 0, 1004, 1077, 1021, 1006, 1054, 1083, 1038, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1009, 1041, 1001, 1021, 1049, 1020, 1017, 4, 3, 2, 0, 0, 0, 0, 0, 1022, 1056, 1087, 1050, 962, 1024, 1052, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1018, 1046, 1019, 1024, 947, 1019, 977, 30, 0, 0, 0, 0, 0, 0, 0, 1036, 980, 1054, 1023, 1005, 1024, 1026, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8169, 8176, 8323, 8167, 8248, 8135, 8150] avg=8192.0000 max=8323 max_rank=3 min=8135 min_rank=6 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1046, 1055, 999, 1027, 1049, 1016, 1005, 1075, 11, 4, 0, 0, 0, 0, 0, 0, 1005, 1038, 1012, 1032, 1007, 970, 1000, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 970, 1104, 973, 1006, 1021, 1024, 982, 20, 0, 0, 0, 0, 0, 0, 0, 1016, 1035, 1030, 1072, 1000, 1005, 1033, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1048, 1028, 1029, 982, 1003, 1050, 1014, 4, 3, 2, 0, 0, 0, 0, 0, 991, 1020, 1073, 1040, 983, 1067, 996, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 981, 1037, 1025, 1061, 1064, 1040, 1022, 29, 0, 0, 0, 0, 0, 0, 0, 1041, 990, 1040, 1052, 1024, 1033, 986, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8080, 8134, 8191, 8165, 8141, 8324, 8214] avg=8192.0000 max=8324 max_rank=6 min=8080 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1026, 1002, 1033, 1045, 1054, 1037, 1038, 992, 11, 4, 0, 0, 0, 0, 0, 0, 1014, 1038, 1014, 987, 1021, 987, 1032, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1041, 1048, 1043, 1031, 1024, 1013, 991, 20, 0, 0, 0, 0, 0, 0, 0, 1042, 1006, 1024, 1041, 1026, 1015, 995, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1031, 989, 1003, 1006, 1019, 1038, 1011, 3, 3, 2, 0, 0, 0, 0, 0, 1047, 1017, 1034, 999, 1035, 1029, 1031, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1009, 973, 994, 1018, 1008, 1030, 1069, 30, 0, 0, 0, 0, 0, 0, 0, 1013, 993, 1036, 1078, 1003, 1030, 1078, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8115, 8261, 8167, 8133, 8248, 8110, 8260] avg=8192.0000 max=8261 max_rank=2 min=8110 min_rank=6 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[991, 1024, 1006, 1017, 1031, 1088, 1020, 1043, 11, 4, 0, 0, 0, 0, 0, 0, 1006, 977, 1075, 1012, 1045, 958, 1049, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1024, 1061, 1054, 968, 1061, 983, 1016, 20, 0, 0, 0, 0, 0, 0, 0, 1028, 1045, 964, 1024, 1045, 1003, 1022, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1007, 1001, 1031, 1023, 1009, 1073, 1034, 4, 3, 2, 0, 0, 0, 0, 0, 975, 1041, 1046, 1035, 1015, 987, 1014, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 997, 1020, 1002, 1009, 1001, 1038, 1033, 29, 0, 0, 0, 0, 0, 0, 0, 1054, 1087, 1020, 1051, 1021, 1017, 988, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8154, 8202, 8156, 8226, 8124, 8187, 8252] avg=8192.0000 max=8252 max_rank=7 min=8124 min_rank=5 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1012, 1048, 1022, 1046, 997, 973, 1007, 1004, 15, 8, 0, 0, 0, 0, 0, 0, 1066, 1086, 1031, 1078, 1043, 961, 1008, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 989, 966, 1001, 1066, 1003, 1029, 1029, 24, 0, 0, 0, 0, 0, 0, 0, 979, 1047, 1023, 1019, 1012, 989, 1021, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 990, 1032, 1004, 1052, 1022, 1082, 955, 7, 5, 4, 0, 0, 0, 0, 0, 1028, 1060, 1055, 1024, 1003, 1025, 1037, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 991, 1009, 1020, 1020, 1037, 1034, 1047, 32, 0, 0, 0, 0, 0, 0, 0, 987, 1073, 1065, 974, 1055, 1056, 1037, 983, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8287, 8137, 8094, 8199, 8250, 8207, 8230] avg=8192.0000 max=8287 max_rank=1 min=8094 min_rank=3 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32886 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1037, 999, 1033, 1015, 1015, 1034, 1010, 1035, 11, 4, 0, 0, 0, 0, 0, 0, 1008, 1000, 1041, 1073, 1027, 943, 1039, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1008, 980, 1029, 1002, 1020, 1069, 959, 19, 0, 0, 0, 0, 0, 0, 0, 1040, 995, 995, 1046, 995, 1043, 1017, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1045, 1032, 1029, 1031, 1056, 1076, 1042, 3, 3, 2, 0, 0, 0, 0, 0, 1046, 1070, 1014, 1024, 1053, 1016, 1042, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1002, 1004, 1054, 1016, 1010, 1032, 991, 28, 0, 0, 0, 0, 0, 0, 0, 1037, 979, 1006, 1043, 1019, 1036, 1000, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8188, 8125, 8129, 8351, 8291, 8118, 8141] avg=8192.0000 max=8351 max_rank=4 min=8118 min_rank=6 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32901 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 7936, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8448, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8192, 8192, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1008, 1057, 1026, 993, 1034, 1038, 983, 1074, 1032, 1025, 1043, 1003, 1004, 1047, 1060, 1061, 975, 1025, 1027, 975, 1017, 1032, 1045, 1038, 1029, 1005, 1042, 1043, 1044, 1007, 1020, 989, 1073, 1023, 1046, 998, 1018, 1069, 1051, 1020, 1006, 1055, 1033, 1026, 986, 1078, 1066, 1029, 1001, 1073, 1018, 994, 1029, 1031, 994, 1014, 979, 970, 1021, 984, 1070, 1012, 1002, 966] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1003, 1012, 996, 981, 1000, 1050, 1005, 1016, 1020, 1027, 1057, 959, 995, 1013, 1004, 1017, 1052, 1058, 990, 1015, 1066, 1011, 1023, 987, 1082, 1018, 990, 993, 1020, 1040, 1068, 1042, 969, 1038, 1014, 1059, 1052, 1008, 1009, 1027, 990, 1015, 1057, 1058, 1079, 998, 1017, 1037, 1026, 1009, 1068, 1000, 1028, 1044, 1061, 1017, 1066, 1012, 1012, 1026, 1042, 988, 1039, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1021, 1057, 1047, 1013, 1063, 1040, 1024, 1019, 990, 1031, 1059, 1006, 996, 999, 1002, 1011, 1054, 1037, 1038, 999, 1020, 999, 1010, 1035, 1018, 1047, 1018, 1059, 1012, 1011, 1009, 1013, 1047, 1074, 1009, 1052, 1041, 1037, 1013, 1034, 1001, 1043, 1054, 1046, 968, 987, 1002, 1015, 1060, 1018, 1006, 983, 1024, 1019, 1050, 971, 1010, 1071, 1048, 1017, 1001, 982, 1053, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[996, 1024, 969, 1022, 1034, 1037, 1046, 1010, 988, 1052, 1056, 1029, 1045, 992, 1062, 961, 982, 1047, 1002, 1031, 1031, 1012, 1036, 1011, 1035, 1046, 1066, 998, 1033, 1011, 1054, 1059, 1028, 994, 991, 1005, 1037, 1023, 1129, 1055, 1051, 1019, 1009, 1025, 1040, 1033, 1005, 1003, 1006, 1029, 1050, 984, 1015, 1023, 1020, 1034, 1032, 1061, 996, 1003, 983, 1009, 1043, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1038, 1038, 982, 988, 1046, 1088, 1055, 1014, 999, 1006, 1011, 1031, 1064, 972, 1042, 983, 1015, 993, 1013, 990, 1069, 1061, 1020, 989, 1019, 1041, 1007, 1023, 1027, 997, 1016, 1066, 1060, 1076, 1027, 1003, 1088, 1019, 1057, 1013, 1034, 964, 1021, 1034, 1004, 1037, 1009, 1020, 1012, 1009, 1009, 1023, 1003, 1026, 1063, 1040, 1057, 1052, 984, 1009, 967, 1018, 1061, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1037, 1044, 1070, 1031, 1049, 1047, 1020, 1027, 1036, 1020, 1028, 1055, 1055, 1002, 987, 966, 1012, 1063, 1004, 997, 1018, 1022, 1054, 1064, 1013, 1015, 1050, 1029, 1019, 1007, 974, 1016, 994, 1028, 1088, 979, 1089, 1045, 1040, 1001, 1023, 976, 1043, 1002, 1032, 1018, 1024, 968, 1005, 1044, 1048, 993, 1041, 1032, 1017, 985, 1051, 1030, 1027, 1032, 1002, 1057, 972, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1043, 1029, 1054, 970, 1019, 1063, 1006, 1051, 1016, 994, 1005, 1072, 1059, 1036, 1053, 1015, 1045, 1026, 1011, 1043, 1005, 1055, 1016, 1047, 1015, 985, 1028, 1028, 1028, 1011, 921, 1060, 1018, 1003, 1043, 1081, 994, 994, 1006, 1032, 1019, 1061, 997, 1061, 1020, 995, 1044, 1085, 1002, 1010, 1015, 994, 1059, 1015, 1032, 987, 1025, 1005, 1011, 1024, 1045, 1032, 1008, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1037, 1039, 1031, 1023, 1044, 1038, 1042, 1054, 1046, 976, 1037, 1060, 1012, 1056, 986, 1053, 1012, 1032, 975, 1048, 977, 1026, 1022, 1005, 1011, 1065, 975, 1059, 1009, 1009, 1033, 1011, 988, 971, 1008, 1041, 982, 1010, 1086, 1003, 1016, 1024, 1016, 1032, 1069, 981, 1029, 1077, 1079, 995, 984, 1035, 1018, 965, 1013, 1026, 1005, 1041, 1030, 1070, 1026, 1038, 1065, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[996, 1024, 969, 1022, 1034, 1004, 1046, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1052, 1056, 1029, 1045, 992, 1062, 961, 15, 5, 0, 0, 0, 0, 0, 0, 982, 1047, 1002, 1031, 1031, 1012, 1036, 1011, 15, 0, 0, 0, 0, 0, 0, 0, 1035, 1046, 1066, 998, 1033, 1011, 1054, 1059, 3, 2, 0, 0, 0, 0, 0, 0, 1028, 994, 991, 1005, 1037, 1023, 1080, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1019, 1009, 1022, 1040, 1033, 1005, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1029, 1050, 984, 1015, 1023, 1020, 1034, 34, 0, 0, 0, 0, 0, 0, 0, 1032, 1061, 996, 1003, 983, 1009, 1043, 1024, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8205, 8167, 8307, 8213, 8182, 8195, 8162] avg=8192.0000 max=8307 max_rank=3 min=8105 min_rank=0 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1038, 1038, 982, 988, 1046, 1053, 1055, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1006, 1011, 1031, 1064, 972, 1042, 983, 14, 5, 0, 0, 0, 0, 0, 0, 1015, 993, 1013, 990, 1069, 1061, 1020, 989, 16, 0, 0, 0, 0, 0, 0, 0, 1019, 1041, 1007, 1023, 1027, 997, 1016, 1066, 3, 2, 0, 0, 0, 0, 0, 0, 1060, 1076, 1027, 1003, 1088, 1019, 1011, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 964, 1021, 1031, 1004, 1037, 1009, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1009, 1009, 1023, 1003, 1026, 1063, 1040, 32, 0, 0, 0, 0, 0, 0, 0, 1057, 1052, 984, 1009, 967, 1018, 1061, 1034, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8127, 8166, 8201, 8297, 8120, 8217, 8194] avg=8192.0000 max=8297 max_rank=4 min=8120 min_rank=5 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1021, 1057, 1047, 1013, 1063, 1007, 1024, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1031, 1059, 1006, 996, 999, 1002, 1011, 14, 5, 0, 0, 0, 0, 0, 0, 1054, 1037, 1038, 999, 1020, 999, 1010, 1035, 15, 0, 0, 0, 0, 0, 0, 0, 1018, 1047, 1018, 1059, 1012, 1011, 1009, 1013, 3, 2, 0, 0, 0, 0, 0, 0, 1047, 1074, 1009, 1052, 1041, 1037, 968, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1043, 1054, 1043, 968, 987, 1002, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1018, 1006, 983, 1024, 1019, 1050, 971, 31, 0, 0, 0, 0, 0, 0, 0, 1010, 1071, 1048, 1017, 1001, 982, 1053, 1043, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8113, 8207, 8192, 8262, 8113, 8162, 8236] avg=8192.0000 max=8262 max_rank=4 min=8113 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1003, 1012, 996, 981, 1000, 1017, 1005, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1027, 1057, 959, 995, 1013, 1004, 1017, 13, 5, 0, 0, 0, 0, 0, 0, 1052, 1058, 990, 1015, 1066, 1011, 1023, 987, 15, 0, 0, 0, 0, 0, 0, 0, 1082, 1018, 990, 993, 1020, 1040, 1068, 1042, 3, 2, 0, 0, 0, 0, 0, 0, 969, 1038, 1014, 1059, 1052, 1008, 965, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1015, 1057, 1055, 1079, 998, 1017, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1009, 1068, 1000, 1028, 1044, 1061, 1017, 31, 0, 0, 0, 0, 0, 0, 0, 1066, 1012, 1012, 1026, 1042, 988, 1039, 1061, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8030, 8110, 8217, 8258, 8132, 8248, 8284, 8257] avg=8192.0000 max=8284 max_rank=6 min=8030 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1008, 1057, 1026, 993, 1034, 1005, 983, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1025, 1043, 1003, 1004, 1047, 1060, 1061, 14, 5, 0, 0, 0, 0, 0, 0, 975, 1025, 1027, 975, 1017, 1032, 1045, 1038, 15, 0, 0, 0, 0, 0, 0, 0, 1029, 1005, 1042, 1043, 1044, 1007, 1020, 989, 3, 2, 0, 0, 0, 0, 0, 0, 1073, 1023, 1046, 998, 1018, 1069, 1005, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1055, 1033, 1023, 986, 1078, 1066, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1073, 1018, 994, 1029, 1031, 994, 1014, 32, 0, 0, 0, 0, 0, 0, 0, 979, 970, 1021, 984, 1070, 1012, 1002, 966, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8294, 8149, 8184, 8252, 8276, 8186, 8015] avg=8192.0000 max=8294 max_rank=1 min=8015 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1037, 1044, 1070, 1031, 1049, 1014, 1020, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1020, 1028, 1055, 1055, 1002, 987, 966, 14, 5, 0, 0, 0, 0, 0, 0, 1012, 1063, 1004, 997, 1018, 1022, 1054, 1064, 15, 0, 0, 0, 0, 0, 0, 0, 1013, 1015, 1050, 1029, 1019, 1007, 974, 1016, 3, 2, 0, 0, 0, 0, 0, 0, 994, 1028, 1088, 979, 1089, 1045, 995, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 976, 1043, 999, 1032, 1018, 1024, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1044, 1048, 993, 1041, 1032, 1017, 985, 31, 0, 0, 0, 0, 0, 0, 0, 1051, 1030, 1027, 1032, 1002, 1057, 972, 1019, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8292, 8168, 8249, 8128, 8219, 8083, 8196, 8201] avg=8192.0000 max=8292 max_rank=0 min=8083 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1043, 1029, 1054, 970, 1019, 1017, 1006, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 994, 1005, 1072, 1059, 1036, 1053, 1015, 17, 6, 0, 0, 0, 0, 0, 0, 1045, 1026, 1011, 1043, 1005, 1055, 1016, 1047, 21, 0, 0, 0, 0, 0, 0, 0, 1015, 985, 1028, 1028, 1028, 1011, 921, 1060, 9, 4, 0, 0, 0, 0, 0, 0, 1018, 1003, 1043, 1081, 994, 994, 955, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1061, 997, 1052, 1020, 995, 1044, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1010, 1015, 994, 1059, 1015, 1032, 987, 34, 0, 0, 0, 0, 0, 0, 0, 1025, 1005, 1011, 1024, 1045, 1032, 1008, 1010, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8273, 8269, 8089, 8120, 8273, 8148, 8175] avg=8192.0000 max=8273 max_rank=1 min=8089 min_rank=3 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1037, 1039, 1031, 1023, 1044, 1005, 1042, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 976, 1037, 1060, 1012, 1056, 986, 1053, 15, 5, 0, 0, 0, 0, 0, 0, 1012, 1032, 975, 1048, 977, 1026, 1022, 1005, 15, 0, 0, 0, 0, 0, 0, 0, 1011, 1065, 975, 1059, 1009, 1009, 1033, 1011, 3, 2, 0, 0, 0, 0, 0, 0, 988, 971, 1008, 1041, 982, 1010, 1038, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1024, 1016, 1029, 1069, 981, 1029, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 995, 984, 1035, 1018, 965, 1013, 1026, 33, 0, 0, 0, 0, 0, 0, 0, 1005, 1041, 1030, 1070, 1026, 1038, 1065, 1010, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8246, 8112, 8177, 8041, 8241, 8148, 8296] avg=8192.0000 max=8296 max_rank=7 min=8041 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8192, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8064, 8320, 8192, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8064, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8320, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1031, 1024, 1006, 1017, 977, 1048, 1060, 984, 1046, 1018, 1015, 1022, 1017, 1047, 1052, 1053, 1030, 1014, 1020, 1023, 1000, 957, 1058, 1013, 1027, 1030, 1065, 1024, 1085, 1025, 986, 1039, 1021, 1015, 1024, 1031, 1033, 1070, 1063, 1025, 1032, 1015, 1021, 1015, 1008, 1079, 1007, 999, 1016, 1031, 982, 1001, 968, 1032, 1004, 1014, 1011, 1035, 1053, 1023, 1049, 1029, 1010, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1057, 993, 1029, 1058, 976, 970, 1011, 944, 969, 1054, 1041, 1049, 999, 1005, 980, 1020, 1041, 998, 1021, 1043, 1013, 995, 1007, 1070, 1087, 1023, 1023, 1034, 1014, 1012, 1025, 1027, 1025, 1059, 1087, 1086, 1009, 1034, 1000, 1019, 1071, 1057, 1033, 980, 1042, 1089, 1020, 1075, 1011, 1011, 1043, 966, 1054, 969, 1075, 997, 1015, 957, 1041, 983, 1040, 1007, 1031, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1038, 1026, 1054, 1000, 1022, 1008, 1070, 1026, 1040, 1048, 995, 1023, 988, 1022, 980, 1051, 983, 1043, 1034, 1079, 1036, 1009, 1001, 966, 1029, 1017, 1054, 1042, 1069, 1012, 1058, 1051, 990, 984, 1070, 1020, 997, 1008, 1024, 1022, 1020, 1009, 1063, 1052, 1018, 1046, 1000, 1002, 1021, 1024, 975, 1012, 1023, 981, 1036, 1048, 1038, 1024, 1006, 1040, 1060, 1053, 988, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1042, 998, 984, 1061, 1057, 1019, 960, 1021, 1043, 1066, 1075, 990, 1001, 1097, 965, 992, 967, 1046, 1039, 1072, 1007, 1006, 1010, 1050, 1014, 1008, 1073, 1049, 1031, 1064, 1006, 987, 1054, 996, 1046, 1006, 1000, 1030, 1037, 998, 1020, 1001, 1027, 1039, 1036, 988, 1011, 997, 1023, 1043, 993, 1028, 989, 1031, 1030, 1042, 1048, 1035, 1083, 998, 1033, 1014, 1014, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1017, 1019, 995, 1072, 1029, 998, 1023, 972, 1037, 1052, 1015, 1000, 1031, 1033, 1022, 1045, 1049, 1017, 1041, 1012, 995, 998, 968, 1003, 1007, 1031, 1059, 995, 1011, 1021, 996, 1037, 1025, 990, 1036, 1015, 1047, 1060, 1064, 1027, 1092, 1005, 1017, 970, 1030, 1031, 971, 1049, 1021, 1022, 1066, 1047, 1025, 1049, 1023, 1051, 1022, 1024, 1069, 1014, 1013, 1026, 997, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1031, 1005, 1031, 1037, 1066, 1022, 1028, 1022, 994, 1054, 1025, 1035, 1031, 982, 1039, 1008, 1048, 1067, 1027, 1019, 1040, 1036, 1009, 982, 1011, 971, 1014, 1059, 1042, 1014, 1016, 995, 1015, 1011, 1062, 1035, 979, 1087, 1047, 1004, 994, 1044, 1004, 1031, 1006, 998, 1022, 1019, 1027, 1040, 1038, 1046, 1018, 1018, 981, 997, 966, 1070, 1081, 1004, 1001, 1035, 1001, 1095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1034, 1023, 1011, 1064, 1048, 1051, 1024, 1021, 1073, 966, 1051, 1045, 1041, 1004, 1038, 1029, 1007, 1001, 985, 1031, 977, 1021, 1106, 1038, 1021, 995, 1020, 1025, 1066, 1068, 1039, 1047, 991, 997, 993, 987, 1012, 1015, 1022, 1030, 1038, 997, 1004, 974, 1022, 1054, 990, 1017, 1005, 993, 1066, 1034, 1056, 1029, 982, 1033, 1071, 1008, 1026, 1037, 1042, 1007, 1026, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1037, 1035, 1024, 1002, 975, 1013, 1083, 1030, 1027, 1081, 993, 1074, 1033, 992, 1036, 1007, 1007, 1045, 1019, 1000, 1035, 1007, 1036, 1024, 1024, 1025, 1069, 1058, 992, 1043, 1054, 987, 1023, 1059, 1072, 1043, 1026, 1043, 1012, 983, 1031, 1022, 1023, 988, 1014, 1047, 988, 987, 995, 1035, 993, 1002, 1024, 1062, 1006, 1005, 1000, 1018, 1020, 1013, 1004, 1059, 1046, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1031, 1024, 1006, 1017, 977, 1048, 1060, 984, 14, 0, 0, 0, 0, 0, 0, 0, 1046, 1004, 1015, 1022, 1017, 1047, 1052, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1014, 1020, 1023, 1000, 957, 1058, 1013, 17, 11, 0, 0, 0, 0, 0, 0, 1027, 1030, 1020, 1024, 1085, 1025, 986, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1015, 1005, 1031, 1033, 1070, 1063, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1015, 1021, 1015, 1008, 1079, 1007, 999, 19, 3, 0, 0, 0, 0, 0, 0, 1016, 1031, 982, 1001, 968, 1032, 1004, 1014, 34, 0, 0, 0, 0, 0, 0, 0, 1011, 1035, 1033, 1023, 1049, 1029, 1010, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8256, 8143, 8236, 8263, 8198, 8082, 8197] avg=8192.0000 max=8263 max_rank=4 min=8082 min_rank=6 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1038, 1026, 1054, 1000, 1022, 1008, 1070, 1026, 15, 0, 0, 0, 0, 0, 0, 0, 1040, 1033, 995, 1023, 988, 1022, 980, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1043, 1034, 1079, 1036, 1009, 1001, 966, 17, 11, 0, 0, 0, 0, 0, 0, 1029, 1017, 1009, 1042, 1069, 1012, 1058, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 990, 984, 1051, 1020, 997, 1008, 1024, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1009, 1063, 1052, 1018, 1046, 1000, 1002, 19, 3, 0, 0, 0, 0, 0, 0, 1021, 1024, 975, 1012, 1023, 981, 1036, 1048, 34, 0, 0, 0, 0, 0, 0, 0, 1038, 1024, 986, 1040, 1060, 1053, 988, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8132, 8179, 8287, 8096, 8232, 8154, 8197] avg=8192.0000 max=8287 max_rank=3 min=8096 min_rank=4 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1057, 993, 1029, 1058, 976, 970, 1011, 944, 15, 0, 0, 0, 0, 0, 0, 0, 969, 1039, 1041, 1049, 999, 1005, 980, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 998, 1021, 1043, 1013, 995, 1007, 1070, 17, 11, 0, 0, 0, 0, 0, 0, 1087, 1023, 979, 1034, 1014, 1012, 1025, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1059, 1067, 1086, 1009, 1034, 1000, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1057, 1033, 980, 1042, 1089, 1020, 1075, 20, 3, 0, 0, 0, 0, 0, 0, 1011, 1011, 1043, 966, 1054, 969, 1075, 997, 33, 0, 0, 0, 0, 0, 0, 0, 1015, 957, 1021, 983, 1040, 1007, 1031, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8053, 8102, 8216, 8201, 8299, 8390, 8159, 8116] avg=8192.0000 max=8390 max_rank=5 min=8053 min_rank=0 imbalance_ratio=1.024170 max_load_reduction=57146 max_load_reduction_pct=0.871979 moved_to_echo=32964 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1042, 998, 984, 1061, 1057, 1019, 960, 1021, 15, 0, 0, 0, 0, 0, 0, 0, 1043, 1051, 1075, 990, 1001, 1097, 965, 992, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1046, 1039, 1072, 1007, 1006, 1010, 1050, 18, 11, 0, 0, 0, 0, 0, 0, 1014, 1008, 1027, 1049, 1031, 1064, 1006, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 996, 1027, 1006, 1000, 1030, 1037, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1001, 1027, 1039, 1036, 988, 1011, 997, 19, 3, 0, 0, 0, 0, 0, 0, 1023, 1043, 993, 1028, 989, 1031, 1030, 1042, 35, 0, 0, 0, 0, 0, 0, 0, 1048, 1035, 1062, 998, 1033, 1014, 1014, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8214, 8226, 8186, 8148, 8141, 8214, 8250] avg=8192.0000 max=8250 max_rank=7 min=8141 min_rank=5 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1017, 1019, 995, 1072, 1029, 998, 1023, 972, 15, 0, 0, 0, 0, 0, 0, 0, 1037, 1037, 1015, 1000, 1031, 1033, 1022, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1017, 1041, 1012, 995, 998, 968, 1003, 18, 11, 0, 0, 0, 0, 0, 0, 1007, 1031, 1014, 995, 1011, 1021, 996, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 990, 1017, 1015, 1047, 1060, 1064, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 1005, 1017, 970, 1030, 1031, 971, 1049, 19, 3, 0, 0, 0, 0, 0, 0, 1021, 1022, 1066, 1047, 1025, 1049, 1023, 1051, 34, 0, 0, 0, 0, 0, 0, 0, 1022, 1024, 1048, 1014, 1013, 1026, 997, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8220, 8112, 8112, 8245, 8187, 8338, 8182] avg=8192.0000 max=8338 max_rank=6 min=8112 min_rank=2 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32952 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1031, 1005, 1031, 1037, 1066, 1022, 1028, 1022, 15, 0, 0, 0, 0, 0, 0, 0, 994, 1039, 1025, 1035, 1031, 982, 1039, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1067, 1027, 1019, 1040, 1036, 1009, 982, 18, 11, 0, 0, 0, 0, 0, 0, 1011, 971, 970, 1059, 1042, 1014, 1016, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1011, 1043, 1035, 979, 1087, 1047, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1044, 1004, 1031, 1006, 998, 1022, 1019, 19, 3, 0, 0, 0, 0, 0, 0, 1027, 1040, 1038, 1046, 1018, 1018, 981, 997, 33, 0, 0, 0, 0, 0, 0, 0, 966, 1070, 1060, 1004, 1001, 1035, 1001, 1095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8153, 8257, 8078, 8221, 8140, 8198, 8232] avg=8192.0000 max=8257 max_rank=0 min=8078 min_rank=3 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1034, 1023, 1011, 1064, 1048, 1051, 1024, 1021, 13, 0, 0, 0, 0, 0, 0, 0, 1073, 953, 1051, 1045, 1041, 1004, 1038, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1001, 985, 1031, 977, 1021, 1106, 1038, 17, 11, 0, 0, 0, 0, 0, 0, 1021, 995, 976, 1025, 1066, 1068, 1039, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 991, 997, 975, 987, 1012, 1015, 1022, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 997, 1004, 974, 1022, 1054, 990, 1017, 18, 3, 0, 0, 0, 0, 0, 0, 1005, 993, 1066, 1034, 1056, 1029, 982, 1033, 33, 0, 0, 0, 0, 0, 0, 0, 1071, 1008, 1006, 1037, 1042, 1007, 1026, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8289, 8234, 8194, 8237, 8029, 8117, 8231, 8205] avg=8192.0000 max=8289 max_rank=0 min=8029 min_rank=4 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1037, 1035, 1024, 1002, 975, 1013, 1083, 1030, 18, 3, 0, 0, 0, 0, 0, 0, 1027, 1063, 993, 1074, 1033, 992, 1036, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1045, 1019, 1000, 1035, 1007, 1036, 1024, 21, 15, 0, 0, 0, 0, 0, 0, 1024, 1025, 1016, 1058, 992, 1043, 1054, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1059, 1046, 1043, 1026, 1043, 1012, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1022, 1023, 988, 1014, 1047, 988, 987, 23, 8, 0, 0, 0, 0, 0, 0, 995, 1035, 993, 1002, 1024, 1062, 1006, 1005, 38, 0, 0, 0, 0, 0, 0, 0, 1000, 1018, 991, 1013, 1004, 1059, 1046, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8225, 8209, 8199, 8235, 8131, 8160, 8157] avg=8192.0000 max=8235 max_rank=4 min=8131 min_rank=5 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8064, 8192, 8448, 8064, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8064, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8448, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1018, 1011, 1026, 996, 989, 1077, 1014, 983, 1047, 995, 1067, 986, 1016, 1015, 1012, 1025, 1021, 1037, 1007, 1028, 1010, 997, 1060, 1022, 1074, 1068, 1016, 1005, 1003, 1027, 1044, 1014, 1026, 1014, 1006, 1021, 1029, 1012, 1025, 1015, 998, 1012, 998, 1037, 999, 1053, 1038, 1050, 1073, 1071, 997, 1060, 1008, 1003, 1015, 1047, 1025, 1068, 1040, 1034, 1030, 997, 1008, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1010, 1058, 1020, 1044, 1065, 1029, 994, 984, 1004, 976, 1044, 1008, 1013, 1043, 1030, 1029, 1018, 1013, 1009, 1058, 1052, 1014, 971, 1048, 1012, 1043, 1007, 1036, 963, 998, 1033, 1010, 1004, 1017, 977, 1019, 1027, 997, 1014, 1010, 1028, 1041, 1024, 1028, 1024, 1022, 1048, 1082, 1037, 979, 1048, 1045, 1031, 1037, 1044, 1026, 1052, 1051, 1004, 1025, 979, 1066, 1041, 1073] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1031, 1021, 1033, 1006, 1050, 1003, 998, 1019, 1027, 978, 977, 1034, 1046, 1001, 1057, 979, 1035, 979, 994, 1063, 1052, 1059, 1040, 985, 1026, 1005, 1047, 1062, 1031, 1020, 1021, 1024, 1050, 1039, 1039, 987, 1055, 1030, 1017, 1044, 987, 1021, 1006, 1043, 1046, 953, 1037, 1051, 1046, 984, 1013, 1070, 1016, 1026, 998, 1030, 1045, 1023, 1012, 1050, 1017, 1043, 1021, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1018, 993, 973, 1032, 1060, 1041, 1034, 1017, 1051, 996, 996, 1045, 999, 1046, 1047, 1024, 1027, 1008, 1004, 995, 964, 969, 1056, 1038, 998, 1023, 1029, 1020, 1035, 1024, 1018, 1015, 1021, 1041, 1047, 1050, 1060, 1052, 1027, 1072, 983, 993, 1040, 1069, 1050, 1040, 1017, 1018, 992, 1054, 978, 1058, 1013, 1016, 1041, 1053, 1034, 1030, 1016, 970, 996, 1047, 1023, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1107, 1039, 1037, 985, 1013, 1011, 1070, 1057, 976, 1039, 998, 1055, 1026, 969, 1018, 1021, 1014, 1078, 1001, 1012, 1068, 1017, 1059, 991, 1015, 993, 1052, 1027, 1008, 1062, 1033, 1011, 1017, 1061, 1009, 1042, 1017, 1008, 996, 961, 1009, 1006, 1013, 1036, 1022, 1046, 1035, 994, 1059, 1034, 976, 995, 994, 1042, 1060, 1036, 965, 1040, 1027, 1038, 1040, 1003, 1068, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1074, 1031, 1027, 1023, 993, 1034, 1012, 981, 1031, 1047, 997, 1041, 1034, 1012, 1031, 1059, 1069, 979, 1005, 1026, 1001, 1048, 999, 1041, 1010, 1044, 1032, 990, 1037, 1036, 1035, 1016, 996, 1054, 1035, 1038, 1027, 1046, 1039, 1033, 1027, 996, 1033, 1029, 1006, 1039, 1017, 1042, 1061, 1005, 1030, 1037, 972, 1000, 1031, 1029, 988, 1038, 1034, 1027, 1002, 1031, 1010, 989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1033, 1052, 933, 1016, 1027, 1008, 1005, 1005, 1033, 1043, 1050, 1036, 966, 989, 1013, 1050, 999, 1052, 1018, 1035, 1001, 1041, 1011, 1018, 1022, 1002, 1043, 1041, 1010, 1052, 1046, 996, 1033, 1032, 987, 1049, 1053, 1020, 1017, 1035, 1015, 1003, 1045, 1040, 1044, 968, 1053, 1057, 1063, 1027, 1050, 1051, 1018, 1032, 1007, 1025, 1050, 978, 1027, 1049, 1018, 990, 1005, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1015, 1066, 1046, 1022, 1032, 1054, 1019, 1028, 1033, 1000, 1051, 1011, 1038, 1072, 976, 1010, 1028, 1063, 1026, 1000, 999, 1059, 967, 1054, 1020, 1011, 984, 1000, 984, 1037, 1040, 985, 990, 1023, 1013, 1013, 1074, 1025, 1029, 1013, 1023, 971, 1025, 1042, 1052, 1014, 1027, 1034, 989, 1047, 997, 1098, 1049, 1027, 1007, 1033, 1057, 1011, 1040, 1007, 1044, 1003, 1016, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1018, 1011, 1026, 996, 989, 1077, 1014, 983, 4, 0, 0, 0, 0, 0, 0, 0, 1047, 995, 1067, 986, 1016, 1015, 1012, 1025, 24, 0, 0, 0, 0, 0, 0, 0, 1021, 1037, 1007, 1028, 1010, 997, 1060, 1022, 8, 7, 0, 0, 0, 0, 0, 0, 1074, 1068, 1016, 1005, 1003, 1027, 1044, 1014, 7, 3, 2, 0, 0, 0, 0, 0, 1026, 1014, 1006, 1021, 1018, 1012, 1025, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1012, 998, 1037, 999, 1053, 1038, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1071, 997, 1029, 1008, 1003, 1015, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1068, 1040, 1034, 1030, 997, 1008, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8187, 8197, 8263, 8137, 8178, 8243, 8213] avg=8192.0000 max=8263 max_rank=3 min=8118 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1010, 1058, 1020, 1044, 1065, 1029, 994, 984, 4, 0, 0, 0, 0, 0, 0, 0, 1004, 976, 1044, 1008, 1013, 1043, 1030, 1029, 24, 0, 0, 0, 0, 0, 0, 0, 1018, 1013, 1009, 1058, 1052, 1014, 971, 1048, 8, 6, 0, 0, 0, 0, 0, 0, 1012, 1043, 1007, 1036, 963, 998, 1033, 1010, 8, 3, 2, 0, 0, 0, 0, 0, 1004, 1017, 977, 1019, 1016, 997, 1014, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1041, 1024, 1028, 1024, 1022, 1048, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 979, 1048, 1015, 1031, 1037, 1044, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1051, 1004, 1025, 979, 1066, 1041, 1067, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8171, 8197, 8115, 8054, 8289, 8217, 8285] avg=8192.0000 max=8289 max_rank=5 min=8054 min_rank=4 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1031, 1021, 1033, 1006, 1050, 1003, 998, 1019, 4, 0, 0, 0, 0, 0, 0, 0, 1027, 978, 977, 1034, 1046, 1001, 1057, 979, 25, 0, 0, 0, 0, 0, 0, 0, 1035, 979, 994, 1063, 1052, 1059, 1040, 985, 8, 7, 0, 0, 0, 0, 0, 0, 1026, 1005, 1047, 1062, 1031, 1020, 1021, 1024, 7, 3, 2, 0, 0, 0, 0, 0, 1050, 1039, 1039, 987, 1044, 1030, 1017, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1021, 1006, 1043, 1046, 953, 1037, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 984, 1013, 1038, 1016, 1026, 998, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1023, 1012, 1050, 1017, 1043, 1021, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8124, 8222, 8248, 8250, 8137, 8151, 8239] avg=8192.0000 max=8250 max_rank=4 min=8124 min_rank=1 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1074, 1031, 1027, 1023, 993, 1034, 1012, 981, 4, 0, 0, 0, 0, 0, 0, 0, 1031, 1047, 997, 1041, 1034, 1012, 1031, 1059, 24, 0, 0, 0, 0, 0, 0, 0, 1069, 979, 1005, 1026, 1001, 1048, 999, 1041, 8, 6, 0, 0, 0, 0, 0, 0, 1010, 1044, 1032, 990, 1037, 1036, 1035, 1016, 7, 3, 2, 0, 0, 0, 0, 0, 996, 1054, 1035, 1038, 1016, 1046, 1039, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 996, 1033, 1029, 1006, 1039, 1017, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1005, 1030, 1007, 972, 1000, 1031, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1038, 1034, 1027, 1002, 1031, 1010, 983, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8276, 8182, 8212, 8257, 8182, 8135, 8113] avg=8192.0000 max=8276 max_rank=1 min=8113 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1107, 1039, 1037, 985, 1013, 1011, 1070, 1057, 4, 0, 0, 0, 0, 0, 0, 0, 976, 1039, 998, 1055, 1026, 969, 1018, 1021, 23, 0, 0, 0, 0, 0, 0, 0, 1014, 1078, 1001, 1012, 1068, 1017, 1059, 991, 8, 6, 0, 0, 0, 0, 0, 0, 1015, 993, 1052, 1027, 1008, 1062, 1033, 1011, 7, 3, 2, 0, 0, 0, 0, 0, 1017, 1061, 1009, 1042, 1006, 1008, 996, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1006, 1013, 1036, 1022, 1046, 1035, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1034, 976, 966, 994, 1042, 1060, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 965, 1040, 1027, 1038, 1040, 1003, 1068, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8323, 8125, 8254, 8213, 8100, 8154, 8167, 8200] avg=8192.0000 max=8323 max_rank=0 min=8100 min_rank=4 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32621 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1018, 993, 973, 1032, 1060, 1041, 1034, 1017, 4, 0, 0, 0, 0, 0, 0, 0, 1051, 996, 996, 1045, 999, 1046, 1047, 1024, 24, 0, 0, 0, 0, 0, 0, 0, 1027, 1008, 1004, 995, 964, 969, 1056, 1038, 8, 7, 0, 0, 0, 0, 0, 0, 998, 1023, 1029, 1020, 1035, 1024, 1018, 1015, 7, 3, 2, 0, 0, 0, 0, 0, 1021, 1041, 1047, 1050, 1049, 1052, 1027, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 983, 993, 1040, 1069, 1050, 1040, 1017, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1054, 978, 1027, 1013, 1016, 1041, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1030, 1016, 970, 996, 1047, 1023, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8228, 8076, 8174, 8359, 8203, 8174, 8150] avg=8192.0000 max=8359 max_rank=4 min=8076 min_rank=2 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32886 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1033, 1052, 933, 1016, 1027, 1008, 1005, 1005, 4, 0, 0, 0, 0, 0, 0, 0, 1033, 1043, 1050, 1036, 966, 989, 1013, 1050, 24, 0, 0, 0, 0, 0, 0, 0, 999, 1052, 1018, 1035, 1001, 1041, 1011, 1018, 8, 6, 0, 0, 0, 0, 0, 0, 1022, 1002, 1043, 1041, 1010, 1052, 1046, 996, 7, 3, 2, 0, 0, 0, 0, 0, 1033, 1032, 987, 1049, 1042, 1020, 1017, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1003, 1045, 1040, 1044, 968, 1053, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1027, 1050, 1021, 1018, 1032, 1007, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 978, 1027, 1049, 1018, 990, 1005, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8204, 8189, 8224, 8215, 8218, 8243, 8160] avg=8192.0000 max=8243 max_rank=6 min=8083 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1015, 1066, 1046, 1022, 1032, 1054, 1019, 1028, 6, 0, 0, 0, 0, 0, 0, 0, 1033, 1000, 1051, 1011, 1038, 1072, 976, 1010, 30, 0, 0, 0, 0, 0, 0, 0, 1028, 1063, 1026, 1000, 999, 1059, 967, 1054, 12, 11, 0, 0, 0, 0, 0, 0, 1020, 1011, 984, 1000, 984, 1037, 1040, 985, 13, 4, 9, 0, 0, 0, 0, 0, 990, 1023, 1013, 1013, 1058, 1025, 1029, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 971, 1025, 1042, 1052, 1014, 1027, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1047, 997, 1057, 1049, 1027, 1007, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1011, 1040, 1007, 1044, 1003, 1016, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8221, 8219, 8087, 8164, 8175, 8206, 8176] avg=8192.0000 max=8288 max_rank=0 min=8087 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8448, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[984, 1017, 1028, 1065, 1032, 1033, 1032, 1050, 1032, 995, 1036, 1022, 1061, 1040, 995, 991, 1041, 998, 989, 994, 1040, 1006, 1077, 1076, 1014, 1079, 1000, 1003, 1067, 1012, 1017, 1019, 987, 1049, 1029, 1025, 1061, 1009, 1042, 1043, 1004, 1035, 1038, 1011, 1024, 1042, 1077, 1007, 949, 1025, 1039, 964, 984, 1049, 1027, 1023, 967, 1063, 1005, 1010, 1008, 1043, 1054, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1029, 977, 996, 1000, 1050, 1039, 1041, 1015, 1031, 975, 1061, 1048, 1034, 1008, 1072, 1042, 1045, 997, 1024, 1063, 976, 984, 1057, 1039, 1069, 993, 999, 1023, 1077, 1003, 981, 1027, 1042, 1038, 1054, 1003, 985, 994, 1014, 1007, 1044, 1064, 973, 1051, 1010, 1013, 1029, 1021, 990, 1045, 1068, 1018, 1025, 1039, 970, 967, 1006, 1035, 1082, 1046, 1046, 1021, 1050, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1027, 1065, 995, 1026, 1057, 1004, 1060, 1002, 975, 1001, 1056, 1014, 1039, 1012, 990, 1006, 1028, 1016, 984, 1069, 1016, 994, 1072, 1023, 1026, 1016, 1028, 1007, 1035, 999, 1056, 1067, 1044, 1006, 1024, 978, 1015, 1032, 1026, 1057, 987, 1070, 1017, 1030, 1049, 1018, 1027, 993, 1040, 1035, 1027, 1014, 1057, 1003, 1018, 1013, 1026, 998, 1040, 1028, 1007, 1060, 1022, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[976, 1004, 985, 1013, 1083, 1034, 1016, 1045, 1056, 1018, 1071, 1023, 1035, 1040, 1030, 992, 996, 1050, 1029, 1016, 1047, 1044, 982, 1027, 1026, 1039, 1010, 1005, 1014, 1033, 1052, 997, 987, 1014, 1012, 1020, 1028, 1036, 1060, 994, 1024, 1070, 1044, 1005, 1011, 963, 1089, 1011, 1016, 1015, 1007, 1051, 1045, 1038, 1045, 975, 1031, 995, 1031, 1077, 988, 1040, 983, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1087, 1028, 978, 1034, 1034, 1003, 1023, 1023, 1051, 1013, 1045, 1020, 1019, 1099, 1043, 1027, 1003, 1039, 1009, 954, 1022, 1027, 1019, 987, 1068, 999, 1061, 1022, 1103, 1024, 1027, 960, 1036, 1042, 1045, 1049, 1002, 1009, 1034, 1009, 1023, 1006, 1073, 1005, 977, 986, 965, 1043, 1020, 1051, 1010, 1045, 1052, 956, 1013, 1056, 1014, 1046, 1005, 1054, 995, 993, 1044, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1014, 1033, 1020, 1031, 1021, 1083, 1019, 1048, 1037, 1048, 1018, 1054, 1001, 986, 1001, 1038, 1008, 1027, 999, 1062, 1018, 1042, 992, 1002, 1029, 1086, 1049, 994, 1049, 1066, 1027, 986, 1018, 1013, 1052, 998, 1012, 1003, 995, 1042, 1012, 1038, 1064, 987, 1038, 1009, 1019, 1027, 1005, 995, 1033, 996, 1056, 960, 989, 1058, 1024, 1070, 1002, 995, 1032, 1005, 1056, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1043, 1065, 997, 993, 1060, 1040, 1034, 1042, 1024, 959, 1061, 974, 1047, 1013, 968, 1011, 1023, 1006, 984, 1033, 1033, 1008, 999, 989, 992, 1028, 1024, 1085, 1026, 1008, 1030, 998, 1014, 1049, 987, 999, 1069, 1057, 992, 998, 1081, 1004, 1001, 1012, 1015, 1024, 1092, 1044, 1006, 1008, 1030, 1028, 1024, 1035, 1092, 1011, 1065, 999, 1023, 991, 1058, 1045, 1043, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[985, 974, 1058, 1051, 980, 986, 1065, 1034, 1059, 958, 1005, 1034, 1008, 1052, 1036, 1003, 1026, 1013, 1001, 1022, 986, 968, 1067, 1020, 1032, 1014, 993, 1034, 1005, 1046, 1010, 1005, 1038, 1043, 988, 1024, 1047, 1042, 981, 1059, 1016, 1034, 999, 1070, 1042, 1090, 1055, 1043, 1012, 1024, 1064, 982, 1027, 1043, 1064, 1072, 999, 993, 1003, 999, 998, 1079, 1027, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[984, 1017, 1028, 1065, 1017, 1033, 1032, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 995, 1036, 1022, 1061, 1040, 995, 991, 2, 0, 0, 0, 0, 0, 0, 0, 1041, 998, 989, 994, 1040, 1006, 1077, 1076, 26, 17, 0, 0, 0, 0, 0, 0, 1014, 1079, 1000, 1003, 1050, 1012, 1017, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1049, 1029, 1025, 1061, 1009, 1042, 1043, 9, 0, 0, 0, 0, 0, 0, 0, 1004, 1035, 1038, 1011, 1024, 1042, 1051, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 949, 1025, 1039, 964, 984, 1049, 1027, 1023, 15, 1, 0, 0, 0, 0, 0, 0, 967, 1063, 1005, 1010, 1008, 1031, 1054, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8174, 8264, 8194, 8254, 8212, 8076, 8136] avg=8192.0000 max=8264 max_rank=2 min=8076 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1029, 977, 996, 1000, 1034, 1039, 1041, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 975, 1061, 1048, 1034, 1008, 1072, 1042, 2, 0, 0, 0, 0, 0, 0, 0, 1045, 997, 1024, 1063, 976, 984, 1057, 1039, 25, 17, 0, 0, 0, 0, 0, 0, 1069, 993, 999, 1023, 1060, 1003, 981, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1038, 1054, 1003, 985, 994, 1014, 1007, 9, 0, 0, 0, 0, 0, 0, 0, 1044, 1064, 973, 1051, 1010, 1013, 1004, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1045, 1068, 1018, 1025, 1039, 970, 967, 16, 1, 0, 0, 0, 0, 0, 0, 1006, 1035, 1082, 1046, 1046, 1009, 1050, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8273, 8227, 8155, 8146, 8180, 8139, 8285] avg=8192.0000 max=8285 max_rank=7 min=8131 min_rank=0 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[976, 1004, 985, 1013, 1067, 1034, 1016, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1018, 1071, 1023, 1035, 1040, 1030, 992, 2, 0, 0, 0, 0, 0, 0, 0, 996, 1050, 1029, 1016, 1047, 1044, 982, 1027, 27, 16, 0, 0, 0, 0, 0, 0, 1026, 1039, 1010, 1005, 998, 1033, 1052, 997, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1014, 1012, 1020, 1028, 1036, 1060, 994, 9, 0, 0, 0, 0, 0, 0, 0, 1024, 1070, 1044, 1005, 1011, 963, 1062, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1015, 1007, 1051, 1045, 1038, 1045, 975, 16, 1, 0, 0, 0, 0, 0, 0, 1031, 995, 1031, 1077, 988, 1028, 983, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8267, 8234, 8160, 8160, 8190, 8209, 8176] avg=8192.0000 max=8267 max_rank=1 min=8140 min_rank=0 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1087, 1028, 978, 1034, 1019, 1003, 1023, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1013, 1045, 1020, 1019, 1099, 1043, 1027, 2, 0, 0, 0, 0, 0, 0, 0, 1003, 1039, 1009, 954, 1022, 1027, 1019, 987, 24, 18, 0, 0, 0, 0, 0, 0, 1068, 999, 1061, 1022, 1085, 1024, 1027, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1042, 1045, 1049, 1002, 1009, 1034, 1009, 8, 0, 0, 0, 0, 0, 0, 0, 1023, 1006, 1073, 1005, 977, 986, 941, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1051, 1010, 1045, 1052, 956, 1013, 1056, 15, 1, 0, 0, 0, 0, 0, 0, 1014, 1046, 1005, 1054, 995, 982, 1044, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8319, 8102, 8246, 8234, 8054, 8219, 8167] avg=8192.0000 max=8319 max_rank=1 min=8054 min_rank=5 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1014, 1033, 1020, 1031, 1006, 1083, 1019, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1048, 1018, 1054, 1001, 986, 1001, 1038, 2, 0, 0, 0, 0, 0, 0, 0, 1008, 1027, 999, 1062, 1018, 1042, 992, 1002, 25, 17, 0, 0, 0, 0, 0, 0, 1029, 1086, 1049, 994, 1032, 1066, 1027, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1013, 1052, 998, 1012, 1003, 995, 1042, 9, 0, 0, 0, 0, 0, 0, 0, 1012, 1038, 1064, 987, 1038, 1009, 994, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 995, 1033, 996, 1056, 960, 989, 1058, 15, 1, 0, 0, 0, 0, 0, 0, 1024, 1070, 1002, 995, 1032, 993, 1056, 1045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8185, 8192, 8269, 8142, 8169, 8108, 8217] avg=8192.0000 max=8269 max_rank=3 min=8108 min_rank=6 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1027, 1065, 995, 1026, 1034, 1004, 1060, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1001, 1056, 1014, 1039, 1012, 990, 1006, 9, 0, 0, 0, 0, 0, 0, 0, 1028, 1016, 984, 1069, 1016, 994, 1072, 1023, 29, 20, 3, 0, 0, 0, 0, 0, 1026, 1016, 1028, 1007, 1015, 999, 1056, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1006, 1024, 978, 1015, 1032, 1026, 1057, 13, 0, 0, 0, 0, 0, 0, 0, 987, 1070, 1017, 1030, 1049, 1018, 998, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1035, 1027, 1014, 1057, 1003, 1018, 1013, 20, 4, 0, 0, 0, 0, 0, 0, 1026, 998, 1040, 1028, 1007, 1034, 1022, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8102, 8254, 8214, 8195, 8162, 8231, 8165] avg=8192.0000 max=8254 max_rank=2 min=8102 min_rank=1 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1043, 1065, 997, 993, 1044, 1040, 1034, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 959, 1061, 974, 1047, 1013, 968, 1011, 2, 0, 0, 0, 0, 0, 0, 0, 1023, 1006, 984, 1033, 1033, 1008, 999, 989, 27, 16, 0, 0, 0, 0, 0, 0, 992, 1028, 1024, 1085, 1010, 1008, 1030, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1049, 987, 999, 1069, 1057, 992, 998, 9, 0, 0, 0, 0, 0, 0, 0, 1081, 1004, 1001, 1012, 1015, 1024, 1065, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1008, 1030, 1028, 1024, 1035, 1092, 1011, 16, 1, 0, 0, 0, 0, 0, 0, 1065, 999, 1023, 991, 1058, 1033, 1043, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8059, 8118, 8175, 8174, 8246, 8251, 8255] avg=8192.0000 max=8258 max_rank=0 min=8059 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32926 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[985, 974, 1058, 1051, 966, 986, 1065, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 958, 1005, 1034, 1008, 1052, 1036, 1003, 2, 0, 0, 0, 0, 0, 0, 0, 1026, 1013, 1001, 1022, 986, 968, 1067, 1020, 26, 16, 0, 0, 0, 0, 0, 0, 1032, 1014, 993, 1034, 989, 1046, 1010, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1043, 988, 1024, 1047, 1042, 981, 1059, 9, 0, 0, 0, 0, 0, 0, 0, 1016, 1034, 999, 1070, 1042, 1090, 1029, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1024, 1064, 982, 1027, 1043, 1064, 1072, 14, 1, 0, 0, 0, 0, 0, 0, 999, 993, 1003, 999, 998, 1067, 1027, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8157, 8145, 8123, 8231, 8323, 8303, 8135] avg=8192.0000 max=8323 max_rank=5 min=8119 min_rank=0 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32992 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8064, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8448, 8192, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8320, 8192, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8064, 8320, 8192, 8192, 8320, 8192, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8448, 8320, 8192, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[996, 1044, 978, 1043, 1006, 976, 1035, 1011, 1061, 993, 1045, 1045, 1043, 1027, 1054, 1040, 1028, 1066, 995, 1022, 1034, 1087, 1028, 1042, 1050, 1004, 1046, 969, 1014, 1023, 1029, 995, 1034, 999, 1039, 998, 975, 1017, 1036, 1022, 1003, 1054, 1046, 994, 986, 1036, 1069, 1016, 1028, 991, 1035, 1078, 1025, 1064, 1010, 1015, 1022, 1067, 998, 1034, 983, 1019, 994, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1017, 1000, 1049, 998, 1012, 1011, 1065, 1005, 991, 973, 1008, 1061, 1035, 1035, 1086, 977, 1007, 1013, 1034, 1027, 1009, 1027, 993, 1001, 1103, 1028, 1031, 1037, 1024, 1033, 1039, 1016, 1036, 1000, 1032, 1029, 1046, 1021, 1044, 1058, 1075, 996, 1035, 970, 985, 1061, 1009, 986, 1018, 1020, 989, 1035, 1054, 1027, 1010, 1046, 1018, 1066, 1055, 1052, 1015, 1057, 992, 954] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1034, 991, 1019, 1027, 1015, 1019, 1015, 1029, 1059, 1029, 1043, 1060, 1029, 1040, 1096, 1016, 976, 1033, 1026, 1002, 1037, 1014, 987, 1024, 1000, 997, 1059, 1046, 1058, 996, 1026, 1036, 1038, 1053, 1057, 1037, 1051, 1031, 976, 1016, 1004, 994, 1014, 976, 982, 993, 987, 995, 1066, 1012, 1050, 994, 1075, 1034, 1023, 1040, 1000, 1058, 1060, 982, 1025, 1062, 1000, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1017, 1051, 994, 985, 1058, 1046, 1005, 1080, 1046, 1038, 1040, 1003, 996, 1032, 1026, 1005, 1068, 996, 1029, 1007, 1066, 1046, 1016, 1038, 979, 955, 1027, 999, 1061, 1004, 1018, 1019, 1026, 1086, 988, 1013, 1031, 1013, 1016, 1053, 971, 1086, 1046, 1033, 1064, 1071, 995, 1018, 1039, 1026, 988, 1059, 1007, 1057, 1033, 1009, 1041, 1000, 1021, 1005, 990, 1014, 989, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1037, 960, 989, 982, 1103, 1026, 1075, 995, 1009, 1022, 1006, 1026, 1046, 1043, 1080, 1032, 999, 1093, 977, 1089, 1009, 1025, 979, 1031, 1062, 1019, 1011, 1054, 1018, 985, 1033, 1022, 1081, 992, 1028, 983, 1028, 1026, 1019, 1026, 1048, 1019, 1054, 998, 1010, 1044, 986, 1032, 1055, 1047, 999, 981, 1029, 1038, 1038, 1026, 1001, 1036, 1056, 992, 1027, 1011, 991, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1070, 1039, 1048, 1015, 985, 1002, 1041, 1006, 1019, 991, 1023, 999, 1027, 1046, 1028, 1075, 1049, 995, 1019, 1050, 1055, 1050, 1019, 1044, 1013, 1021, 989, 1015, 1027, 1045, 1047, 966, 1036, 1018, 987, 975, 1013, 1037, 1012, 998, 1069, 1035, 1003, 1039, 1018, 1055, 1031, 1028, 1027, 1082, 1062, 1062, 999, 994, 1029, 996, 1021, 971, 1050, 1048, 962, 1029, 1012, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1045, 984, 1013, 1035, 1065, 999, 1014, 1042, 1062, 1025, 966, 1029, 1015, 1031, 1018, 985, 1038, 1002, 1029, 1061, 1075, 1077, 1001, 1017, 1041, 1006, 1014, 1062, 980, 1036, 1035, 992, 1026, 1055, 977, 1009, 994, 1028, 1040, 1041, 1019, 1021, 1029, 1006, 993, 1041, 1045, 999, 965, 1039, 1057, 1073, 1059, 1004, 1051, 995, 1027, 996, 1016, 1079, 983, 999, 1043, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1028, 1035, 1018, 974, 991, 1012, 1003, 980, 1048, 1060, 1018, 1005, 1038, 1049, 1041, 998, 1103, 1019, 1019, 1047, 988, 985, 1056, 1066, 994, 1004, 977, 1026, 1038, 946, 982, 1003, 1011, 1056, 1023, 1075, 968, 1000, 997, 1051, 1074, 1024, 1031, 1005, 1037, 1025, 1040, 1069, 989, 1052, 1066, 1071, 1024, 1028, 1056, 985, 1042, 1053, 1024, 1026, 1014, 961, 1077, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[996, 1044, 978, 1043, 1006, 976, 1035, 1011, 25, 12, 0, 0, 0, 0, 0, 0, 1061, 993, 1045, 1045, 1043, 1027, 1010, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1066, 995, 1022, 1034, 1048, 1028, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1004, 1046, 969, 1014, 1023, 1029, 995, 45, 0, 0, 0, 0, 0, 0, 0, 1034, 999, 1039, 998, 975, 1017, 1036, 1022, 7, 0, 0, 0, 0, 0, 0, 0, 1003, 1054, 1046, 994, 986, 1036, 1069, 1016, 7, 0, 0, 0, 0, 0, 0, 0, 1028, 991, 1035, 1027, 1025, 1064, 1010, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1067, 998, 1034, 983, 1019, 994, 1020, 32, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8264, 8263, 8175, 8127, 8211, 8195, 8175] avg=8192.0000 max=8264 max_rank=1 min=8126 min_rank=0 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1034, 991, 1019, 1027, 1015, 1019, 1015, 1029, 23, 13, 0, 0, 0, 0, 0, 0, 1059, 1029, 1043, 1060, 1029, 1040, 1050, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1033, 1026, 1002, 1037, 978, 987, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 997, 1059, 1046, 1058, 996, 1026, 1036, 41, 0, 0, 0, 0, 0, 0, 0, 1038, 1053, 1057, 1037, 1051, 1031, 976, 1016, 6, 0, 0, 0, 0, 0, 0, 0, 1004, 994, 1014, 976, 982, 993, 987, 995, 7, 0, 0, 0, 0, 0, 0, 0, 1066, 1012, 1050, 948, 1075, 1034, 1023, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1058, 1060, 982, 1025, 1062, 1000, 1013, 33, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8326, 8063, 8259, 8265, 7952, 8248, 8238] avg=8192.0000 max=8326 max_rank=1 min=7952 min_rank=5 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1037, 960, 989, 982, 1103, 1026, 1075, 995, 23, 12, 0, 0, 0, 0, 0, 0, 1009, 1022, 1006, 1026, 1046, 1043, 1036, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1093, 977, 1089, 1009, 989, 979, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1019, 1011, 1054, 1018, 985, 1033, 1022, 41, 0, 0, 0, 0, 0, 0, 0, 1081, 992, 1028, 983, 1028, 1026, 1019, 1026, 6, 0, 0, 0, 0, 0, 0, 0, 1048, 1019, 1054, 998, 1010, 1044, 986, 1032, 7, 0, 0, 0, 0, 0, 0, 0, 1055, 1047, 999, 935, 1029, 1038, 1038, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1036, 1056, 992, 1027, 1011, 991, 998, 32, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8220, 8166, 8245, 8189, 8198, 8167, 8149] avg=8192.0000 max=8245 max_rank=3 min=8149 min_rank=7 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1017, 1000, 1049, 998, 1012, 1011, 1065, 1005, 23, 12, 0, 0, 0, 0, 0, 0, 991, 973, 1008, 1061, 1035, 1035, 1042, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1013, 1034, 1027, 1009, 991, 993, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 1028, 1031, 1037, 1024, 1033, 1039, 1016, 43, 0, 0, 0, 0, 0, 0, 0, 1036, 1000, 1032, 1029, 1046, 1021, 1044, 1058, 6, 0, 0, 0, 0, 0, 0, 0, 1075, 996, 1035, 970, 985, 1061, 1009, 986, 7, 0, 0, 0, 0, 0, 0, 0, 1018, 1020, 989, 987, 1054, 1027, 1010, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1066, 1055, 1052, 1015, 1057, 992, 954, 32, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8122, 8075, 8354, 8272, 8124, 8151, 8246] avg=8192.0000 max=8354 max_rank=3 min=8075 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1017, 1051, 994, 985, 1058, 1046, 1005, 1080, 24, 12, 0, 0, 0, 0, 0, 0, 1046, 1038, 1040, 1003, 996, 1032, 983, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 996, 1029, 1007, 1066, 1009, 1016, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 979, 955, 1027, 999, 1061, 1004, 1018, 1019, 44, 0, 0, 0, 0, 0, 0, 0, 1026, 1086, 988, 1013, 1031, 1013, 1016, 1053, 6, 0, 0, 0, 0, 0, 0, 0, 971, 1086, 1046, 1033, 1064, 1071, 995, 1018, 7, 0, 0, 0, 0, 0, 0, 0, 1039, 1026, 988, 1010, 1007, 1057, 1033, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1000, 1021, 1005, 990, 1014, 989, 998, 31, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8143, 8229, 8106, 8232, 8291, 8169, 8094] avg=8192.0000 max=8291 max_rank=5 min=8094 min_rank=7 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1045, 984, 1013, 1035, 1065, 999, 1014, 1042, 25, 12, 0, 0, 0, 0, 0, 0, 1062, 1025, 966, 1029, 1015, 1031, 976, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1002, 1029, 1061, 1075, 1038, 1001, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1006, 1014, 1062, 980, 1036, 1035, 992, 45, 0, 0, 0, 0, 0, 0, 0, 1026, 1055, 977, 1009, 994, 1028, 1040, 1041, 7, 0, 0, 0, 0, 0, 0, 0, 1019, 1021, 1029, 1006, 993, 1041, 1045, 999, 7, 0, 0, 0, 0, 0, 0, 0, 965, 1039, 1057, 1022, 1059, 1004, 1051, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 996, 1016, 1079, 983, 999, 1043, 1033, 30, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8089, 8261, 8211, 8177, 8160, 8192, 8212] avg=8192.0000 max=8261 max_rank=2 min=8089 min_rank=1 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1028, 1035, 1018, 974, 991, 1012, 1003, 980, 27, 15, 0, 0, 0, 0, 0, 0, 1048, 1060, 1018, 1005, 1038, 1049, 991, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 1019, 1019, 1047, 988, 937, 1056, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1004, 977, 1026, 1038, 946, 982, 1003, 49, 0, 0, 0, 0, 0, 0, 0, 1011, 1056, 1023, 1075, 968, 1000, 997, 1051, 11, 0, 0, 0, 0, 0, 0, 0, 1074, 1024, 1031, 1005, 1037, 1025, 1040, 1069, 10, 0, 0, 0, 0, 0, 0, 0, 989, 1052, 1066, 1012, 1024, 1028, 1056, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1053, 1024, 1026, 1014, 961, 1077, 1031, 35, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8207, 8235, 8019, 8192, 8315, 8212, 8273] avg=8192.0000 max=8315 max_rank=5 min=8019 min_rank=3 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32992 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1070, 1039, 1048, 1015, 985, 1002, 1041, 1006, 24, 12, 0, 0, 0, 0, 0, 0, 1019, 991, 1023, 999, 1027, 1046, 985, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 995, 1019, 1050, 1055, 1013, 1019, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1021, 989, 1015, 1027, 1045, 1047, 966, 44, 0, 0, 0, 0, 0, 0, 0, 1036, 1018, 987, 975, 1013, 1037, 1012, 998, 6, 0, 0, 0, 0, 0, 0, 0, 1069, 1035, 1003, 1039, 1018, 1055, 1031, 1028, 7, 0, 0, 0, 0, 0, 0, 0, 1027, 1082, 1062, 1013, 999, 994, 1029, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 971, 1050, 1048, 962, 1029, 1012, 1020, 31, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8165, 8244, 8167, 8082, 8285, 8202, 8149] avg=8192.0000 max=8285 max_rank=5 min=8082 min_rank=4 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8064, 8320, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8064, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8064, 8192, 8448, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8064, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8192, 8320, 8320, 8192, 8320, 8064, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1022, 966, 1042, 1064, 1076, 1010, 997, 996, 973, 1031, 1030, 1031, 1044, 986, 1009, 1035, 1008, 1042, 1102, 994, 1001, 1009, 1033, 1020, 974, 1021, 1047, 1076, 1063, 1111, 1057, 1029, 971, 1072, 1031, 986, 1002, 1014, 1036, 1033, 995, 967, 1022, 1037, 1016, 1016, 1006, 1020, 1009, 1077, 1001, 1018, 1032, 998, 1027, 1028, 1055, 1014, 1022, 966, 1052, 1014, 1103, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1020, 1037, 1041, 1010, 1030, 1027, 1027, 1060, 1029, 1007, 1044, 997, 969, 1047, 1058, 1006, 982, 963, 1028, 1056, 989, 1059, 993, 1031, 1061, 1099, 1006, 1016, 1039, 987, 1023, 974, 1043, 1057, 1024, 1017, 1026, 1042, 1024, 1023, 1049, 1082, 950, 1017, 1029, 1021, 1024, 1054, 1003, 1033, 976, 988, 1009, 1080, 1072, 1021, 1012, 1009, 1048, 1017, 1029, 1011, 1037, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1008, 997, 1052, 1017, 969, 1051, 1031, 1019, 1040, 1016, 985, 994, 1043, 1039, 980, 1008, 994, 1022, 998, 1056, 1038, 1035, 987, 997, 1080, 1040, 985, 1007, 1031, 1055, 1013, 1085, 994, 1030, 1023, 1019, 999, 1017, 1069, 1059, 996, 1039, 1039, 1025, 1029, 1034, 1009, 1006, 1007, 1068, 996, 1106, 991, 1054, 1011, 1035, 1016, 1036, 1062, 1054, 1029, 1000, 973, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1013, 1018, 1055, 1028, 1042, 985, 944, 1045, 1033, 1028, 1020, 1036, 1027, 1020, 1057, 970, 1038, 1009, 1028, 1049, 1023, 1026, 1024, 992, 1033, 1042, 1041, 1059, 1049, 999, 981, 1032, 1034, 1018, 995, 1038, 1012, 1063, 1025, 1019, 1012, 1008, 996, 999, 963, 1033, 1019, 1009, 1030, 972, 1046, 1042, 1049, 1037, 1050, 1042, 1072, 1022, 1055, 1034, 1005, 1032, 1041, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1048, 999, 1024, 1041, 980, 1029, 977, 1017, 1010, 1007, 1026, 1052, 1026, 1007, 1022, 990, 1019, 1063, 1033, 1033, 1019, 994, 1024, 1050, 1007, 1014, 1048, 994, 1033, 1006, 1029, 997, 1070, 1020, 1073, 1029, 995, 1074, 996, 1040, 998, 1026, 1019, 1043, 1072, 1059, 972, 1014, 1028, 1025, 1090, 1056, 1056, 1002, 994, 1017, 1007, 1017, 1028, 1044, 1047, 1006, 1020, 981] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1013, 1029, 985, 1072, 1002, 986, 1014, 1016, 1074, 972, 1051, 1000, 1013, 1035, 1024, 1079, 995, 1008, 1016, 1029, 1079, 1047, 981, 1057, 1035, 1072, 1010, 1008, 1055, 1019, 1042, 1008, 1063, 997, 1039, 1027, 1007, 1043, 1030, 1011, 1089, 1036, 998, 989, 1007, 972, 963, 1074, 1044, 1023, 985, 1029, 1037, 1035, 1092, 1028, 998, 973, 1020, 1044, 977, 1095, 984, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1075, 1051, 1000, 1040, 1006, 1046, 1042, 1044, 1029, 1027, 994, 1016, 1052, 997, 984, 966, 1011, 1044, 1011, 1020, 1033, 1031, 1001, 989, 1045, 1003, 1038, 1019, 1019, 1008, 1026, 980, 1051, 1041, 1036, 1002, 1075, 991, 1050, 988, 1033, 1008, 1017, 1012, 1002, 1036, 1035, 1044, 1045, 1044, 1042, 1030, 1044, 1009, 1052, 1005, 1061, 991, 1003, 1022, 1096, 992, 1008, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1067, 1028, 1005, 1016, 1107, 1024, 1042, 1042, 1074, 1034, 1036, 972, 1028, 993, 1021, 1030, 1042, 1039, 1051, 1024, 1025, 1008, 1049, 998, 1084, 1009, 991, 950, 1031, 965, 1011, 1018, 997, 1027, 1043, 1092, 1054, 978, 990, 1052, 1027, 1068, 1003, 1050, 1022, 1053, 997, 1013, 1001, 1011, 1000, 999, 1009, 1021, 1014, 1028, 1017, 1045, 1022, 1034, 1080, 968, 970, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1022, 966, 1042, 1061, 1076, 1010, 997, 996, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1031, 1030, 1031, 1044, 986, 1009, 1035, 32, 4, 0, 0, 0, 0, 0, 0, 1008, 1042, 1102, 994, 1001, 1009, 1033, 1020, 11, 0, 0, 0, 0, 0, 0, 0, 974, 1021, 1047, 1076, 1044, 1111, 1057, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1072, 1000, 986, 1002, 1014, 1036, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 995, 967, 1022, 1037, 1016, 1016, 1006, 1020, 27, 5, 0, 0, 0, 0, 0, 0, 1009, 1077, 1001, 1018, 1032, 998, 995, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1014, 1022, 966, 1052, 1014, 1103, 997, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8175, 8220, 8359, 8114, 8111, 8158, 8229] avg=8192.0000 max=8359 max_rank=3 min=8111 min_rank=5 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32612 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1020, 1037, 1041, 1007, 1030, 1027, 1027, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1007, 1044, 997, 969, 1047, 1058, 1006, 34, 4, 0, 0, 0, 0, 0, 0, 982, 963, 1028, 1056, 989, 1059, 993, 1031, 10, 0, 0, 0, 0, 0, 0, 0, 1061, 1099, 1006, 1016, 1022, 987, 1023, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1057, 993, 1017, 1026, 1042, 1024, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1082, 950, 1017, 1029, 1021, 1024, 1054, 27, 4, 0, 0, 0, 0, 0, 0, 1003, 1033, 976, 988, 1009, 1080, 1038, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1009, 1048, 1017, 1029, 1011, 1037, 994, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8195, 8111, 8188, 8225, 8257, 8148, 8163] avg=8192.0000 max=8257 max_rank=5 min=8111 min_rank=2 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1013, 1018, 1055, 1025, 1042, 985, 944, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1028, 1020, 1036, 1027, 1020, 1057, 970, 33, 4, 0, 0, 0, 0, 0, 0, 1038, 1009, 1028, 1049, 1023, 1026, 1024, 992, 10, 0, 0, 0, 0, 0, 0, 0, 1033, 1042, 1041, 1059, 1031, 999, 981, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1018, 965, 1038, 1012, 1063, 1025, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1008, 996, 999, 963, 1033, 1019, 1009, 26, 5, 0, 0, 0, 0, 0, 0, 1030, 972, 1046, 1042, 1049, 1037, 1017, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1022, 1055, 1034, 1005, 1032, 1041, 1018, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8228, 8199, 8218, 8174, 8070, 8235, 8285] avg=8192.0000 max=8285 max_rank=7 min=8070 min_rank=5 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1008, 997, 1052, 1014, 969, 1051, 1031, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1016, 985, 994, 1043, 1039, 980, 1008, 32, 4, 0, 0, 0, 0, 0, 0, 994, 1022, 998, 1056, 1038, 1035, 987, 997, 10, 0, 0, 0, 0, 0, 0, 0, 1080, 1040, 985, 1007, 1014, 1055, 1013, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1030, 992, 1019, 999, 1017, 1069, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1039, 1039, 1025, 1029, 1034, 1009, 1006, 27, 4, 0, 0, 0, 0, 0, 0, 1007, 1068, 996, 1106, 991, 1054, 979, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1036, 1062, 1054, 1029, 1000, 973, 1039, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8141, 8137, 8279, 8179, 8208, 8236, 8215] avg=8192.0000 max=8279 max_rank=3 min=8137 min_rank=2 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1048, 999, 1024, 1038, 980, 1029, 977, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1007, 1026, 1052, 1026, 1007, 1022, 990, 31, 4, 0, 0, 0, 0, 0, 0, 1019, 1063, 1033, 1033, 1019, 994, 1024, 1050, 10, 0, 0, 0, 0, 0, 0, 0, 1007, 1014, 1048, 994, 1016, 1006, 1029, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1020, 1040, 1029, 995, 1074, 996, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1026, 1019, 1043, 1072, 1059, 972, 1014, 29, 4, 0, 0, 0, 0, 0, 0, 1028, 1025, 1090, 1056, 1056, 1002, 963, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1017, 1028, 1044, 1047, 1006, 1020, 981, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8175, 8245, 8111, 8264, 8236, 8237, 8156] avg=8192.0000 max=8264 max_rank=4 min=8111 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1013, 1029, 985, 1069, 1002, 986, 1014, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 972, 1051, 1000, 1013, 1035, 1024, 1079, 35, 4, 0, 0, 0, 0, 0, 0, 995, 1008, 1016, 1029, 1079, 1047, 981, 1057, 11, 0, 0, 0, 0, 0, 0, 0, 1035, 1072, 1010, 1008, 1036, 1019, 1042, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 997, 1007, 1027, 1007, 1043, 1030, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 1036, 998, 989, 1007, 972, 963, 1074, 28, 5, 0, 0, 0, 0, 0, 0, 1044, 1023, 985, 1029, 1037, 1035, 1057, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 998, 973, 1020, 1044, 977, 1095, 984, 1001, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8287, 8223, 8230, 8185, 8161, 8238, 8098] avg=8192.0000 max=8287 max_rank=1 min=8098 min_rank=7 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1075, 1051, 1000, 1031, 1006, 1046, 1042, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1027, 994, 1016, 1052, 997, 984, 966, 38, 8, 0, 0, 0, 0, 0, 0, 1011, 1044, 1011, 1020, 1033, 1031, 1001, 989, 15, 0, 0, 0, 0, 0, 0, 0, 1045, 1003, 1038, 1019, 990, 1008, 1026, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1041, 996, 1002, 1075, 991, 1050, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1008, 1017, 1012, 1002, 1036, 1035, 1044, 32, 9, 0, 0, 0, 0, 0, 0, 1045, 1044, 1042, 1030, 1044, 1009, 1014, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 991, 1003, 1022, 1096, 992, 1008, 1024, 9, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8111, 8155, 8109, 8194, 8228, 8233, 8211] avg=8192.0000 max=8295 max_rank=0 min=8109 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32866 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1067, 1028, 1005, 1013, 1107, 1024, 1042, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1034, 1036, 972, 1028, 993, 1021, 1030, 32, 4, 0, 0, 0, 0, 0, 0, 1042, 1039, 1051, 1024, 1025, 1008, 1049, 998, 10, 0, 0, 0, 0, 0, 0, 0, 1084, 1009, 991, 950, 1014, 965, 1011, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1027, 1011, 1092, 1054, 978, 990, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1068, 1003, 1050, 1022, 1053, 997, 1013, 28, 4, 0, 0, 0, 0, 0, 0, 1001, 1011, 1000, 999, 1009, 1021, 982, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1045, 1022, 1034, 1080, 968, 970, 1037, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8328, 8224, 8246, 8042, 8201, 8265, 8051, 8179] avg=8192.0000 max=8328 max_rank=0 min=8042 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8192, 8192, 8320, 8064, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1012, 1027, 1033, 1027, 969, 997, 1048, 1054, 1024, 991, 1016, 1067, 1032, 1024, 1036, 1027, 947, 1032, 1024, 1069, 1075, 1078, 1073, 988, 1030, 1004, 1004, 1050, 1037, 969, 1043, 982, 996, 1049, 1026, 1017, 1023, 1023, 1030, 1043, 1004, 1036, 1045, 963, 977, 1035, 1042, 1054, 981, 1038, 1066, 942, 989, 1043, 989, 1055, 976, 1055, 1037, 1077, 975, 1073, 1027, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1025, 1018, 1028, 1018, 997, 1033, 999, 1031, 1072, 1052, 1048, 1029, 997, 994, 1033, 1014, 1049, 1032, 1042, 1014, 1068, 1100, 1025, 1048, 999, 1043, 1016, 1010, 993, 1015, 1005, 1073, 1008, 963, 1005, 1039, 1037, 1045, 1004, 979, 1026, 1007, 1042, 946, 1018, 1037, 1060, 1007, 1017, 1032, 1078, 979, 1032, 1043, 984, 1029, 1041, 1018, 1039, 992, 1028, 986, 1078, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1033, 1016, 1063, 1005, 991, 1004, 1038, 1034, 1062, 1076, 1009, 1009, 1006, 1028, 1022, 1049, 1052, 1051, 1014, 1057, 1008, 1052, 1011, 1048, 1031, 1015, 988, 1047, 1017, 1025, 1009, 1006, 1018, 1054, 1048, 1006, 1005, 1049, 1023, 993, 1010, 1041, 1040, 1086, 1026, 1009, 1043, 1015, 1003, 1039, 1069, 961, 965, 1014, 1003, 1040, 1035, 1006, 1012, 974, 984, 1067, 978, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1043, 1016, 996, 1003, 1033, 1070, 995, 984, 1034, 1017, 996, 1027, 1048, 1010, 1075, 991, 1015, 1036, 1029, 1052, 1010, 1006, 1041, 1019, 1038, 1001, 1016, 987, 1016, 1040, 1025, 1052, 998, 1031, 1038, 1006, 1041, 1029, 1031, 1018, 1008, 996, 1054, 1010, 1040, 979, 1080, 1067, 1055, 954, 1014, 1053, 1013, 1046, 1036, 1043, 1040, 1033, 1000, 1017, 1028, 1029, 1023, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1072, 982, 1038, 1029, 1033, 1047, 993, 1010, 1031, 998, 1092, 1037, 1024, 1006, 1021, 1056, 1030, 1030, 1041, 1013, 1017, 1044, 1003, 1059, 1038, 1015, 1031, 1019, 961, 1011, 1042, 1001, 1012, 1033, 1038, 1031, 1023, 1031, 986, 972, 1039, 1041, 961, 1031, 1028, 969, 1050, 1043, 1003, 1023, 1064, 984, 1022, 1035, 1012, 1080, 991, 1026, 975, 1097, 1012, 1041, 1020, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1013, 1058, 1058, 1044, 1021, 1009, 1033, 1019, 1064, 1045, 1018, 1030, 1022, 977, 1016, 1006, 1050, 974, 1003, 996, 1024, 1070, 1072, 1031, 1033, 1046, 988, 995, 1068, 1065, 1043, 1050, 969, 983, 1008, 1008, 1000, 1037, 997, 1057, 1015, 1026, 1049, 1033, 1009, 961, 1057, 1018, 1017, 1050, 1065, 1043, 1013, 1028, 1018, 989, 1018, 1045, 1011, 996, 1028, 983, 972, 1092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1004, 1036, 1059, 1031, 999, 1052, 1041, 977, 1043, 1051, 1032, 1081, 1032, 1011, 1007, 1002, 1024, 1027, 1011, 1003, 987, 1009, 1030, 1064, 1004, 994, 1063, 1054, 1022, 992, 960, 1038, 1005, 1003, 1033, 1030, 1049, 1037, 1009, 1029, 1028, 1053, 1029, 1020, 1022, 1000, 1050, 1034, 1034, 993, 1005, 1029, 1009, 1021, 1031, 1044, 1040, 1058, 1009, 1031, 1031, 991, 1026, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[991, 938, 1018, 998, 1056, 1085, 1046, 1012, 970, 1060, 1040, 1030, 1007, 1016, 1023, 999, 1015, 1062, 1048, 1007, 1058, 1044, 993, 1012, 1056, 1041, 1014, 986, 1078, 1050, 1031, 1008, 1042, 1029, 996, 1047, 1045, 1037, 1020, 1005, 999, 1031, 1047, 1076, 1025, 1076, 985, 1003, 991, 1005, 1066, 1046, 1043, 988, 1007, 1015, 986, 1031, 1043, 1041, 982, 1020, 1031, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1012, 1027, 1033, 1027, 969, 997, 1048, 1054, 9, 2, 0, 0, 0, 0, 0, 0, 1024, 991, 1016, 1040, 1032, 1024, 1036, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 947, 1032, 1024, 1069, 1075, 1018, 1073, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1004, 1004, 1050, 1037, 969, 1043, 982, 11, 7, 0, 0, 0, 0, 0, 0, 996, 1049, 1026, 1017, 1023, 1023, 1030, 1043, 33, 0, 0, 0, 0, 0, 0, 0, 1004, 1036, 1045, 963, 977, 1035, 1033, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1038, 1066, 942, 989, 1043, 989, 1055, 20, 0, 0, 0, 0, 0, 0, 0, 976, 1055, 1037, 1077, 975, 1073, 1027, 1061, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8190, 8226, 8137, 8240, 8147, 8123, 8295] avg=8192.0000 max=8295 max_rank=7 min=8123 min_rank=6 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1033, 1016, 1063, 1005, 991, 1004, 1038, 1034, 9, 2, 0, 0, 0, 0, 0, 0, 1062, 1076, 1009, 982, 1006, 1028, 1022, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1051, 1014, 1057, 1008, 994, 1011, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1015, 988, 1047, 1017, 1025, 1009, 1006, 11, 7, 0, 0, 0, 0, 0, 0, 1018, 1054, 1048, 1006, 1005, 1049, 1023, 993, 32, 0, 0, 0, 0, 0, 0, 0, 1010, 1041, 1040, 1086, 1026, 1009, 1034, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1039, 1069, 961, 965, 1014, 1003, 1040, 19, 0, 0, 0, 0, 0, 0, 0, 1035, 1006, 1012, 974, 984, 1067, 978, 1044, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8234, 8235, 8156, 8228, 8261, 8113, 8114] avg=8192.0000 max=8261 max_rank=5 min=8113 min_rank=6 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1043, 1016, 996, 1003, 1033, 1070, 995, 984, 9, 2, 0, 0, 0, 0, 0, 0, 1034, 1017, 996, 1000, 1048, 1010, 1075, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1036, 1029, 1052, 1010, 950, 1041, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1001, 1016, 987, 1016, 1040, 1025, 1052, 11, 7, 0, 0, 0, 0, 0, 0, 998, 1031, 1038, 1006, 1041, 1029, 1031, 1018, 31, 0, 0, 0, 0, 0, 0, 0, 1008, 996, 1054, 1010, 1040, 979, 1071, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 954, 1014, 1053, 1013, 1046, 1036, 1043, 18, 0, 0, 0, 0, 0, 0, 0, 1040, 1033, 1000, 1017, 1028, 1029, 1023, 1005, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8171, 8152, 8193, 8223, 8225, 8232, 8189] avg=8192.0000 max=8232 max_rank=6 min=8151 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1072, 982, 1038, 1029, 1033, 1047, 993, 1010, 9, 2, 0, 0, 0, 0, 0, 0, 1031, 998, 1092, 1010, 1024, 1006, 1021, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1030, 1041, 1013, 1017, 986, 1003, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1015, 1031, 1019, 961, 1011, 1042, 1001, 11, 7, 0, 0, 0, 0, 0, 0, 1012, 1033, 1038, 1031, 1023, 1031, 986, 972, 32, 0, 0, 0, 0, 0, 0, 0, 1039, 1041, 961, 1031, 1028, 969, 1041, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1023, 1064, 984, 1022, 1035, 1012, 1080, 19, 0, 0, 0, 0, 0, 0, 0, 991, 1026, 975, 1097, 1012, 1041, 1020, 1039, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8238, 8179, 8136, 8158, 8153, 8242, 8215] avg=8192.0000 max=8242 max_rank=6 min=8136 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1013, 1058, 1058, 1044, 1021, 1009, 1033, 1019, 9, 2, 0, 0, 0, 0, 0, 0, 1064, 1045, 1018, 1003, 1022, 977, 1016, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 974, 1003, 996, 1024, 1010, 1072, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1046, 988, 995, 1068, 1065, 1043, 1050, 11, 7, 0, 0, 0, 0, 0, 0, 969, 983, 1008, 1008, 1000, 1037, 997, 1057, 33, 0, 0, 0, 0, 0, 0, 0, 1015, 1026, 1049, 1033, 1009, 961, 1048, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1050, 1065, 1043, 1013, 1028, 1018, 989, 20, 0, 0, 0, 0, 0, 0, 0, 1018, 1045, 1011, 996, 1028, 983, 972, 1092, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8151, 8160, 8306, 8092, 8159, 8243, 8159] avg=8192.0000 max=8306 max_rank=3 min=8092 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32653 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1025, 1018, 1028, 1018, 997, 1033, 999, 1031, 9, 2, 0, 0, 0, 0, 0, 0, 1072, 1052, 1048, 1002, 997, 994, 1033, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1032, 1042, 1014, 1068, 1038, 1025, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1043, 1016, 1010, 993, 1015, 1005, 1073, 11, 8, 0, 0, 0, 0, 0, 0, 1008, 963, 1005, 1039, 1037, 1045, 1004, 979, 34, 0, 0, 0, 0, 0, 0, 0, 1026, 1007, 1042, 946, 1018, 1037, 1051, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1032, 1078, 979, 1032, 1043, 984, 1029, 20, 0, 0, 0, 0, 0, 0, 0, 1041, 1018, 1039, 992, 1028, 986, 1078, 1017, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8212, 8316, 8173, 8114, 8134, 8214, 8213] avg=8192.0000 max=8316 max_rank=2 min=8114 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32675 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[991, 938, 1018, 998, 1056, 1085, 1046, 1012, 12, 5, 0, 0, 0, 0, 0, 0, 970, 1060, 1040, 994, 1007, 1016, 1023, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1062, 1048, 1007, 1058, 974, 993, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1041, 1014, 986, 1078, 1050, 1031, 1008, 14, 12, 0, 0, 0, 0, 0, 0, 1042, 1029, 996, 1047, 1045, 1037, 1020, 1005, 34, 0, 0, 0, 0, 0, 0, 0, 999, 1031, 1047, 1076, 1025, 1076, 973, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1005, 1066, 1046, 1043, 988, 1007, 1015, 24, 0, 0, 0, 0, 0, 0, 0, 986, 1031, 1043, 1041, 982, 1020, 1031, 986, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8109, 8169, 8290, 8255, 8230, 8185, 8137] avg=8192.0000 max=8290 max_rank=3 min=8109 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1004, 1036, 1059, 1031, 999, 1052, 1041, 977, 9, 2, 0, 0, 0, 0, 0, 0, 1043, 1051, 1032, 1053, 1032, 1011, 1007, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1027, 1011, 1003, 987, 953, 1030, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 994, 1063, 1054, 1022, 992, 960, 1038, 11, 7, 0, 0, 0, 0, 0, 0, 1005, 1003, 1033, 1030, 1049, 1037, 1009, 1029, 31, 0, 0, 0, 0, 0, 0, 0, 1028, 1053, 1029, 1020, 1022, 1000, 1041, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 993, 1005, 1029, 1009, 1021, 1031, 1044, 18, 0, 0, 0, 0, 0, 0, 0, 1040, 1058, 1009, 1031, 1031, 991, 1026, 1013, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8231, 8099, 8145, 8226, 8227, 8184, 8214] avg=8192.0000 max=8231 max_rank=1 min=8099 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8192, 8320, 8320, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8448, 8064, 8192, 8320, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 7936, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1041, 1038, 1025, 998, 978, 1014, 1061, 998, 995, 991, 1061, 1033, 1044, 1066, 1026, 967, 1036, 1031, 987, 997, 1027, 994, 994, 1000, 1047, 1039, 1048, 1043, 1012, 1049, 1057, 1035, 999, 993, 1016, 1010, 1027, 1026, 1011, 1041, 1045, 1018, 1006, 1009, 1015, 1047, 1022, 1039, 1020, 1068, 1040, 1041, 1072, 1016, 1005, 1045, 1051, 1042, 1000, 978, 1016, 1007, 1012, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1020, 1012, 994, 1068, 1011, 1039, 1043, 1021, 1030, 1023, 989, 1015, 1023, 994, 1035, 1020, 1029, 954, 1012, 1008, 995, 1058, 1035, 1069, 1052, 1049, 1052, 1053, 1049, 1003, 1054, 1003, 1032, 1041, 1026, 1055, 1032, 1040, 1003, 1013, 976, 1029, 956, 1009, 1061, 1053, 975, 1031, 997, 1020, 1055, 1089, 992, 1032, 1041, 999, 1013, 1064, 1011, 994, 1036, 1021, 1032, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[985, 1052, 1017, 1056, 1040, 1023, 1072, 1034, 1055, 1018, 1026, 1022, 1041, 1022, 1011, 1003, 1029, 1011, 1001, 999, 1006, 992, 1034, 1009, 1014, 961, 983, 1027, 1014, 1072, 1001, 1060, 1029, 1050, 1111, 1034, 951, 1046, 1021, 1054, 1023, 996, 1013, 1051, 1009, 1006, 1063, 1038, 1000, 1053, 1001, 1012, 1016, 1013, 1062, 967, 1072, 1037, 995, 1045, 1011, 1004, 1026, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1070, 1038, 1011, 1063, 1025, 1001, 1029, 1047, 975, 1090, 1005, 1006, 981, 1037, 1057, 1013, 1091, 1084, 1066, 1044, 1008, 1013, 1020, 967, 1011, 955, 991, 1020, 1037, 956, 1055, 1023, 1039, 1006, 1037, 975, 1003, 990, 973, 984, 1074, 1044, 1053, 1052, 1039, 1005, 1071, 1064, 1009, 1039, 1013, 1027, 990, 1017, 1094, 983, 1002, 996, 983, 1026, 1007, 1032, 1054, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1024, 1067, 1021, 1039, 1008, 1034, 984, 1027, 1025, 1024, 1012, 1027, 989, 1057, 1043, 1015, 1050, 1003, 995, 1017, 1030, 1008, 988, 1023, 1028, 1020, 1006, 1034, 1049, 1067, 1056, 1050, 1041, 1034, 1054, 1049, 1044, 1010, 1030, 1030, 994, 1034, 988, 1012, 1055, 1034, 974, 1003, 989, 1051, 1084, 1019, 996, 1022, 1015, 1019, 1026, 1026, 1008, 991, 1026, 1022, 1021, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1010, 975, 1050, 1056, 1045, 1007, 1010, 1010, 1041, 1030, 1012, 993, 996, 1037, 1022, 1022, 1000, 1001, 1024, 1008, 1018, 1010, 1025, 1013, 1053, 1039, 1052, 1000, 1039, 1048, 998, 1070, 1012, 976, 1019, 1062, 1029, 990, 1013, 1073, 1060, 990, 1039, 1076, 1003, 1052, 1050, 1063, 999, 1050, 1019, 1052, 960, 1051, 1000, 1015, 1050, 997, 1000, 975, 1041, 1004, 1039, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1011, 1009, 1015, 991, 1067, 1037, 1012, 1031, 994, 1086, 987, 1043, 1010, 1024, 1037, 1045, 1009, 1035, 1049, 1004, 1042, 1015, 1063, 1016, 1034, 1094, 1025, 1031, 1030, 1076, 1023, 1009, 1030, 1037, 1000, 967, 1061, 1038, 1008, 1016, 1039, 1037, 1002, 1030, 963, 1034, 1047, 1048, 989, 1003, 1006, 1045, 1009, 1009, 970, 1020, 1046, 984, 1039, 1021, 1032, 1008, 1036, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1004, 1022, 1008, 1000, 1014, 974, 1028, 998, 1022, 1023, 1019, 1007, 1034, 1045, 1012, 957, 1041, 993, 1054, 1056, 1031, 1022, 968, 1077, 1032, 1018, 1014, 1022, 1010, 1033, 1016, 1050, 1044, 1047, 993, 1066, 1082, 988, 1063, 1032, 1023, 1047, 988, 1057, 1084, 1015, 1017, 1064, 1009, 980, 1008, 1023, 998, 1072, 1029, 1031, 1050, 1020, 1070, 1004, 982, 1050, 1000, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1041, 1038, 1025, 998, 978, 1014, 1061, 998, 2, 0, 0, 0, 0, 0, 0, 0, 995, 991, 1061, 1033, 1044, 1066, 1026, 967, 21, 0, 0, 0, 0, 0, 0, 0, 1036, 1031, 987, 997, 1027, 994, 994, 1000, 31, 0, 0, 0, 0, 0, 0, 0, 1047, 1039, 1048, 1043, 1012, 997, 1057, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 999, 993, 1009, 1010, 1027, 1026, 1011, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1018, 1006, 1009, 1015, 1047, 1022, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1068, 1040, 1041, 1072, 1016, 1005, 1045, 20, 0, 0, 0, 0, 0, 0, 0, 1051, 1042, 1000, 978, 1016, 1007, 1012, 1067, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8204, 8097, 8278, 8116, 8168, 8327, 8191] avg=8192.0000 max=8327 max_rank=6 min=8097 min_rank=2 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1020, 1012, 994, 1068, 1011, 1039, 1043, 1021, 2, 0, 0, 0, 0, 0, 0, 0, 1030, 1023, 989, 1015, 1023, 994, 1035, 1020, 20, 0, 0, 0, 0, 0, 0, 0, 1029, 954, 1012, 1008, 995, 1058, 1035, 1069, 29, 0, 0, 0, 0, 0, 0, 0, 1052, 1049, 1052, 1053, 1049, 954, 1054, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1041, 1019, 1055, 1032, 1040, 1003, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1029, 956, 1009, 1061, 1053, 975, 998, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1020, 1055, 1089, 992, 1032, 1041, 999, 20, 0, 0, 0, 0, 0, 0, 0, 1013, 1064, 1011, 994, 1036, 1021, 1032, 996, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8149, 8189, 8266, 8235, 8057, 8245, 8185] avg=8192.0000 max=8266 max_rank=3 min=8057 min_rank=5 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1070, 1038, 1011, 1063, 1025, 1001, 1029, 1047, 3, 0, 0, 0, 0, 0, 0, 0, 975, 1090, 1005, 1006, 981, 1037, 1057, 1013, 19, 0, 0, 0, 0, 0, 0, 0, 1091, 1084, 1066, 1044, 1008, 1013, 1020, 967, 28, 0, 0, 0, 0, 0, 0, 0, 1011, 955, 991, 1020, 1037, 909, 1055, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1006, 1029, 975, 1003, 990, 973, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1044, 1053, 1052, 1039, 1005, 1071, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1039, 1013, 1027, 990, 1017, 1094, 983, 21, 0, 0, 0, 0, 0, 0, 0, 1002, 996, 983, 1026, 1007, 1032, 1054, 1066, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8183, 8321, 8001, 7999, 8368, 8193, 8184] avg=8192.0000 max=8368 max_rank=5 min=7999 min_rank=4 imbalance_ratio=1.021484 max_load_reduction=57168 max_load_reduction_pct=0.872314 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1024, 1067, 1021, 1039, 1008, 1034, 984, 1027, 3, 0, 0, 0, 0, 0, 0, 0, 1025, 1024, 1012, 1027, 989, 1057, 1043, 1015, 21, 0, 0, 0, 0, 0, 0, 0, 1050, 1003, 995, 1017, 1030, 1008, 988, 1023, 31, 0, 0, 0, 0, 0, 0, 0, 1028, 1020, 1006, 1034, 1049, 1015, 1056, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1034, 1046, 1049, 1044, 1010, 1030, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1034, 988, 1012, 1055, 1034, 974, 972, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1051, 1084, 1019, 996, 1022, 1015, 1019, 19, 0, 0, 0, 0, 0, 0, 0, 1026, 1026, 1008, 991, 1026, 1022, 1021, 1015, 12, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8213, 8145, 8258, 8284, 8063, 8214, 8152] avg=8192.0000 max=8284 max_rank=4 min=8063 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[985, 1052, 1017, 1056, 1040, 1023, 1072, 1034, 3, 0, 0, 0, 0, 0, 0, 0, 1055, 1018, 1026, 1022, 1041, 1022, 1011, 1003, 21, 0, 0, 0, 0, 0, 0, 0, 1029, 1011, 1001, 999, 1006, 992, 1034, 1009, 32, 0, 0, 0, 0, 0, 0, 0, 1014, 961, 983, 1027, 1014, 1019, 1001, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1050, 1102, 1034, 951, 1046, 1021, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 996, 1013, 1051, 1009, 1006, 1063, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1053, 1001, 1012, 1016, 1013, 1062, 967, 20, 0, 0, 0, 0, 0, 0, 0, 1072, 1037, 995, 1045, 1011, 1004, 1026, 1037, 13, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8219, 8113, 8079, 8287, 8166, 8144, 8246] avg=8192.0000 max=8287 max_rank=4 min=8079 min_rank=3 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1010, 975, 1050, 1056, 1045, 1007, 1010, 1010, 2, 0, 0, 0, 0, 0, 0, 0, 1041, 1030, 1012, 993, 996, 1037, 1022, 1022, 21, 0, 0, 0, 0, 0, 0, 0, 1000, 1001, 1024, 1008, 1018, 1010, 1025, 1013, 31, 0, 0, 0, 0, 0, 0, 0, 1053, 1039, 1052, 1000, 1039, 996, 998, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 976, 1012, 1062, 1029, 990, 1013, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 990, 1039, 1076, 1003, 1052, 1050, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1050, 1019, 1052, 960, 1051, 1000, 1015, 21, 0, 0, 0, 0, 0, 0, 0, 1050, 997, 1000, 975, 1041, 1004, 1039, 1063, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8174, 8130, 8247, 8167, 8299, 8167, 8187] avg=8192.0000 max=8299 max_rank=5 min=8130 min_rank=2 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1011, 1009, 1015, 991, 1067, 1037, 1012, 1031, 2, 0, 0, 0, 0, 0, 0, 0, 994, 1086, 987, 1043, 1010, 1024, 1037, 1045, 21, 0, 0, 0, 0, 0, 0, 0, 1009, 1035, 1049, 1004, 1042, 1015, 1063, 1016, 32, 0, 0, 0, 0, 0, 0, 0, 1034, 1094, 1025, 1031, 1030, 1023, 1023, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1037, 993, 967, 1061, 1038, 1008, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1037, 1002, 1030, 963, 1034, 1047, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1003, 1006, 1045, 1009, 1009, 970, 1020, 20, 0, 0, 0, 0, 0, 0, 0, 1046, 984, 1039, 1021, 1032, 1008, 1036, 1008, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8247, 8265, 8269, 8150, 8167, 8071, 8192] avg=8192.0000 max=8269 max_rank=3 min=8071 min_rank=6 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1004, 1022, 1008, 1000, 1014, 974, 1028, 998, 7, 0, 0, 0, 0, 0, 0, 0, 1022, 1023, 1019, 1007, 1034, 1045, 1012, 957, 23, 5, 0, 0, 0, 0, 0, 0, 1041, 993, 1054, 1056, 1031, 1022, 968, 1077, 34, 0, 0, 0, 0, 0, 0, 0, 1032, 1018, 1014, 1022, 1010, 976, 1016, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1047, 976, 1066, 1082, 988, 1063, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1047, 988, 1057, 1084, 1015, 1017, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 980, 1008, 1023, 998, 1072, 1029, 1031, 25, 0, 0, 0, 0, 0, 0, 0, 1050, 1020, 1070, 1004, 982, 1050, 1000, 996, 17, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8055, 8147, 8276, 8138, 8298, 8248, 8175, 8199] avg=8192.0000 max=8298 max_rank=4 min=8055 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8064, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8320, 8320, 8064, 8320, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 7936, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8064, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1018, 1034, 1012, 1008, 1117, 1031, 1038, 977, 1071, 1056, 1043, 1069, 975, 1021, 1017, 1072, 1032, 1024, 994, 1027, 953, 1016, 1017, 995, 1056, 1010, 989, 1072, 1033, 1004, 1004, 982, 1050, 1034, 1010, 1018, 1047, 1043, 1030, 1020, 1019, 1013, 1025, 975, 1032, 1029, 1000, 1034, 1029, 1029, 1078, 1020, 1003, 1061, 1036, 1034, 1048, 980, 1024, 977, 1036, 1001, 1037, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1056, 1006, 1007, 1024, 1016, 1051, 981, 1019, 1006, 1021, 1018, 1056, 1028, 985, 1052, 1090, 1009, 1024, 1031, 992, 1023, 1049, 1047, 1069, 1023, 1022, 1008, 1019, 1034, 1060, 1038, 1063, 1000, 1023, 993, 1063, 990, 970, 1025, 998, 975, 1048, 1044, 1002, 1043, 1046, 1027, 1033, 1021, 1050, 1042, 1029, 997, 1004, 1083, 988, 1039, 1041, 1017, 1021, 1027, 973, 1024, 973] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1061, 1029, 978, 1002, 990, 1032, 1078, 1025, 1035, 1068, 1011, 1062, 1085, 1031, 1040, 987, 1009, 1007, 996, 1008, 997, 1012, 1012, 1021, 1027, 995, 983, 996, 1045, 989, 1024, 1046, 1057, 1017, 1013, 1056, 1077, 991, 1042, 1015, 1023, 982, 1026, 1025, 996, 989, 1014, 988, 1059, 1046, 1069, 1062, 1024, 1016, 1068, 1031, 1053, 980, 1012, 990, 1034, 1054, 1006, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1017, 1045, 1029, 1027, 1024, 1028, 1008, 1039, 1017, 1029, 1013, 1001, 1047, 983, 1028, 1039, 995, 1053, 1043, 993, 996, 990, 1044, 1030, 1031, 1023, 1020, 1027, 980, 1004, 1032, 961, 1010, 1024, 1039, 1033, 1039, 1093, 998, 1004, 1059, 1023, 1023, 1078, 1026, 1011, 1025, 1010, 1028, 1014, 988, 1016, 1049, 968, 1039, 1079, 1016, 1001, 1070, 1062, 1015, 1093, 1011, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1004, 1009, 974, 974, 1001, 1008, 1038, 1031, 1035, 1056, 1015, 1021, 1075, 1052, 1033, 1030, 1042, 1032, 1024, 1011, 1021, 1049, 1006, 1024, 1035, 1020, 994, 993, 1009, 1098, 1016, 960, 1049, 1035, 1029, 980, 981, 977, 1014, 1041, 985, 1034, 1015, 1025, 1068, 1027, 1004, 1058, 1064, 1031, 1079, 996, 1032, 1000, 1003, 1037, 1055, 1023, 1017, 1010, 1047, 1019, 1050, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1067, 1015, 1042, 1033, 1056, 1015, 1055, 1038, 997, 1042, 1014, 984, 1017, 979, 967, 1055, 997, 969, 1002, 1002, 1020, 988, 1012, 1018, 1074, 1053, 1023, 1021, 1000, 1023, 1050, 1059, 1037, 1025, 1028, 1069, 989, 1041, 1031, 1014, 990, 1049, 1030, 1023, 1038, 1028, 1035, 1033, 1058, 998, 1044, 1013, 1057, 1048, 1028, 1017, 944, 1003, 1015, 1048, 1015, 1061, 1029, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1006, 1026, 1081, 1040, 1018, 987, 1084, 1018, 1004, 985, 999, 1026, 1003, 1060, 1007, 1024, 962, 1012, 1091, 1087, 1057, 973, 1016, 1035, 1042, 1015, 1028, 974, 1027, 1033, 1011, 1008, 1050, 1021, 1072, 1010, 1048, 994, 1029, 1019, 1055, 1022, 1035, 979, 961, 1029, 1002, 1031, 1021, 1026, 1043, 1014, 1044, 1008, 1034, 1047, 1076, 997, 1003, 1012, 1020, 1019, 1013, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1008, 1030, 1020, 998, 1033, 1054, 1069, 1047, 1041, 1047, 965, 981, 1067, 1020, 1023, 1037, 1066, 1065, 985, 1091, 1070, 1049, 1009, 1034, 1016, 1001, 1055, 1032, 1002, 1006, 987, 972, 993, 1016, 1015, 995, 1011, 1029, 1055, 1078, 982, 980, 1000, 1010, 1041, 1008, 1029, 1006, 1008, 985, 990, 1010, 1052, 1011, 1064, 1065, 1031, 975, 1026, 1065, 1056, 1020, 1006, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1018, 1034, 1012, 1008, 1117, 1031, 1020, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1056, 1043, 1069, 975, 1021, 1017, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1024, 994, 1027, 953, 1016, 1017, 995, 18, 6, 0, 0, 0, 0, 0, 0, 1056, 1010, 989, 1072, 1033, 1004, 1004, 982, 37, 0, 0, 0, 0, 0, 0, 0, 1044, 1034, 1010, 1018, 1047, 1043, 1030, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1013, 1025, 975, 1032, 1029, 1000, 1034, 18, 16, 0, 0, 0, 0, 0, 0, 1029, 1029, 1078, 1020, 1003, 1061, 981, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 980, 1024, 977, 1036, 1001, 1037, 997, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8302, 8082, 8187, 8246, 8161, 8235, 8106] avg=8192.0000 max=8302 max_rank=1 min=8082 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1061, 1029, 978, 1002, 990, 1032, 1059, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1068, 1011, 1062, 1085, 1031, 1040, 966, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1007, 996, 1008, 997, 1012, 1012, 1021, 19, 6, 0, 0, 0, 0, 0, 0, 1027, 995, 983, 996, 1045, 989, 1024, 1046, 38, 0, 0, 0, 0, 0, 0, 0, 1051, 1017, 1013, 1056, 1077, 991, 1042, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 982, 1026, 1025, 996, 989, 1014, 988, 19, 15, 0, 0, 0, 0, 0, 0, 1059, 1046, 1069, 1062, 1024, 1016, 1011, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 980, 1012, 990, 1034, 1054, 1006, 1040, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8298, 8087, 8143, 8262, 8077, 8318, 8175] avg=8192.0000 max=8318 max_rank=6 min=8077 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1056, 1006, 1007, 1024, 1016, 1051, 964, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1021, 1018, 1056, 1028, 985, 1052, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1024, 1031, 992, 1023, 1049, 1047, 1069, 17, 6, 0, 0, 0, 0, 0, 0, 1023, 1022, 1008, 1019, 1034, 1060, 1038, 1063, 38, 0, 0, 0, 0, 0, 0, 0, 994, 1023, 993, 1063, 990, 970, 1025, 998, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1048, 1044, 1002, 1043, 1046, 1027, 1033, 19, 16, 0, 0, 0, 0, 0, 0, 1021, 1050, 1042, 1029, 997, 1004, 1026, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1041, 1017, 1021, 1027, 973, 1024, 973, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8234, 8267, 8305, 8056, 8253, 8157, 8121] avg=8192.0000 max=8305 max_rank=3 min=8056 min_rank=4 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32587 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1004, 1009, 974, 974, 1001, 1008, 1020, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1056, 1015, 1021, 1075, 1052, 1033, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1032, 1024, 1011, 1021, 1049, 1006, 1024, 18, 6, 0, 0, 0, 0, 0, 0, 1035, 1020, 994, 993, 1009, 1098, 1016, 960, 35, 0, 0, 0, 0, 0, 0, 0, 1043, 1035, 1029, 980, 981, 977, 1014, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1034, 1015, 1025, 1068, 1027, 1004, 1058, 18, 15, 0, 0, 0, 0, 0, 0, 1064, 1031, 1079, 996, 1032, 1000, 950, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1023, 1017, 1010, 1047, 1019, 1050, 1061, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8021, 8296, 8233, 8160, 8100, 8249, 8189, 8288] avg=8192.0000 max=8296 max_rank=1 min=8021 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1017, 1045, 1029, 1027, 1024, 1028, 990, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1029, 1013, 1001, 1047, 983, 1028, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1053, 1043, 993, 996, 990, 1044, 1030, 18, 6, 0, 0, 0, 0, 0, 0, 1031, 1023, 1020, 1027, 980, 1004, 1032, 961, 37, 0, 0, 0, 0, 0, 0, 0, 1004, 1024, 1039, 1033, 1039, 1093, 998, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1023, 1023, 1078, 1026, 1011, 1025, 1010, 18, 16, 0, 0, 0, 0, 0, 0, 1028, 1014, 988, 1016, 1049, 968, 984, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1001, 1070, 1062, 1015, 1093, 1011, 996, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8135, 8168, 8115, 8234, 8289, 8126, 8270] avg=8192.0000 max=8289 max_rank=5 min=8115 min_rank=3 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1067, 1015, 1042, 1033, 1056, 1015, 1037, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1042, 1014, 984, 1017, 979, 967, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 997, 969, 1002, 1002, 1020, 988, 1012, 1018, 18, 6, 0, 0, 0, 0, 0, 0, 1074, 1053, 1023, 1021, 1000, 1023, 1050, 1059, 36, 0, 0, 0, 0, 0, 0, 0, 1031, 1025, 1028, 1069, 989, 1041, 1031, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1049, 1030, 1023, 1038, 1028, 1035, 1033, 18, 16, 0, 0, 0, 0, 0, 0, 1058, 998, 1044, 1013, 1057, 1048, 974, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 944, 1003, 1015, 1048, 1015, 1061, 1029, 1011, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8033, 8032, 8339, 8228, 8260, 8209, 8132] avg=8192.0000 max=8339 max_rank=3 min=8032 min_rank=2 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1006, 1026, 1081, 1040, 1018, 987, 1065, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 985, 999, 1026, 1003, 1060, 1007, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1012, 1091, 1087, 1057, 973, 1016, 1035, 19, 6, 0, 0, 0, 0, 0, 0, 1042, 1015, 1028, 974, 1027, 1033, 1011, 1008, 37, 0, 0, 0, 0, 0, 0, 0, 1044, 1021, 1072, 1010, 1048, 994, 1029, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1022, 1035, 979, 961, 1029, 1002, 1031, 18, 15, 0, 0, 0, 0, 0, 0, 1021, 1026, 1043, 1014, 1044, 1008, 979, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 997, 1003, 1012, 1020, 1019, 1013, 1063, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8087, 8258, 8175, 8237, 8147, 8182, 8209] avg=8192.0000 max=8258 max_rank=2 min=8087 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1008, 1030, 1020, 998, 1033, 1054, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1047, 965, 981, 1067, 1020, 1023, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1065, 985, 1091, 1070, 1049, 1009, 1034, 23, 10, 7, 0, 0, 0, 0, 0, 1016, 1001, 1055, 1032, 1002, 1006, 987, 972, 41, 0, 0, 0, 0, 0, 0, 0, 974, 1016, 1015, 995, 1011, 1029, 1055, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 982, 980, 1000, 1010, 1041, 1008, 1029, 1006, 24, 20, 0, 0, 0, 0, 0, 0, 1008, 985, 990, 1010, 1052, 1011, 999, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 975, 1026, 1065, 1056, 1020, 1006, 1044, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8151, 8409, 8112, 8173, 8100, 8120, 8235] avg=8192.0000 max=8409 max_rank=2 min=8100 min_rank=5 imbalance_ratio=1.026489 max_load_reduction=57127 max_load_reduction_pct=0.871689 moved_to_echo=32628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8064, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8192, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8448, 8192, 8320, 8192, 7936, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[977, 989, 1049, 998, 944, 989, 1021, 998, 1064, 1000, 999, 1043, 1059, 1038, 998, 989, 1028, 1008, 1046, 1030, 1036, 1052, 1029, 967, 1117, 1066, 1013, 988, 1062, 1049, 1017, 973, 1027, 1004, 1040, 1055, 1045, 1045, 980, 1036, 1050, 1025, 1035, 1062, 1042, 1017, 1021, 1017, 1053, 1030, 1027, 1049, 1034, 1001, 1009, 1025, 1020, 1004, 1005, 1024, 1059, 983, 1028, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1009, 969, 1023, 1052, 1030, 1046, 1078, 1014, 1047, 1000, 1045, 1023, 1036, 995, 1002, 1036, 1046, 1028, 1005, 996, 1049, 1030, 1021, 1064, 1060, 972, 1007, 998, 981, 1034, 1028, 1072, 1067, 982, 987, 1024, 1001, 1021, 989, 1007, 1005, 1004, 1062, 983, 1048, 1041, 961, 1020, 1039, 988, 1055, 1039, 991, 1056, 995, 1103, 1076, 1032, 1005, 1031, 987, 1063, 1076, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1056, 954, 995, 1008, 1053, 990, 1016, 1066, 1008, 1004, 1015, 1018, 1003, 1025, 1047, 1013, 1019, 1006, 1006, 1016, 1041, 1037, 1057, 952, 996, 1022, 1040, 1065, 1026, 1022, 1102, 1057, 1011, 1049, 1014, 1028, 999, 1038, 1038, 1009, 1048, 1041, 1030, 1045, 1004, 1014, 1003, 1019, 1065, 998, 1017, 1004, 994, 1050, 1005, 1045, 1027, 1044, 1081, 1036, 1006, 1014, 955, 1070] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1012, 988, 1055, 1022, 1015, 970, 1054, 1039, 1012, 1020, 1048, 1023, 1050, 994, 998, 1022, 1037, 1042, 1034, 1041, 1007, 1081, 1031, 1023, 1069, 991, 1031, 1044, 1056, 1022, 1003, 992, 1026, 1012, 990, 1001, 1033, 1013, 1009, 1019, 957, 1049, 1042, 1056, 983, 1068, 999, 994, 1034, 1010, 1083, 1065, 1045, 1012, 1014, 1023, 1085, 1050, 966, 1001, 1018, 1022, 1003, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1049, 1029, 1032, 985, 1043, 1043, 1033, 1019, 1043, 1010, 975, 1047, 1017, 1013, 975, 1031, 1039, 1044, 1059, 1015, 1005, 996, 1027, 1018, 1025, 1034, 1003, 1026, 1042, 996, 1024, 1037, 1016, 1046, 1002, 1011, 1056, 1021, 1061, 1045, 961, 999, 1056, 999, 1037, 1030, 1055, 1017, 1013, 1081, 985, 1036, 1000, 1010, 1022, 993, 1003, 1030, 1046, 1053, 1037, 1057, 967, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1044, 1022, 996, 1051, 1098, 1026, 1071, 1006, 1018, 1053, 1005, 1015, 1049, 1012, 1030, 992, 1048, 1049, 1033, 972, 1063, 1000, 1059, 1029, 1003, 997, 977, 1018, 1027, 1062, 1043, 1031, 1035, 1031, 1028, 999, 996, 1064, 1015, 995, 999, 1048, 987, 1065, 1067, 1036, 1056, 976, 995, 1004, 1013, 1043, 1033, 967, 1008, 1032, 987, 998, 1020, 1058, 1039, 1012, 1015, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1013, 1077, 1077, 1025, 1045, 1035, 1039, 1001, 1023, 1023, 1009, 1015, 970, 969, 1048, 1014, 1049, 1038, 1032, 1027, 1015, 1033, 1027, 1012, 1083, 1054, 1026, 1000, 1028, 1011, 1007, 1050, 1022, 1025, 1018, 999, 1024, 1035, 1000, 998, 1051, 1026, 1028, 1020, 1013, 1048, 1053, 1036, 1004, 1031, 1067, 1011, 998, 979, 1010, 1028, 1031, 1046, 988, 1019, 1038, 1006, 964, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1045, 1022, 1020, 1016, 1028, 1059, 998, 1010, 1022, 994, 1037, 1037, 1021, 954, 1023, 1063, 1020, 1057, 1042, 1048, 1028, 1008, 1091, 1066, 1039, 1056, 1048, 1044, 998, 1025, 981, 1004, 994, 973, 1056, 998, 1061, 967, 1031, 1008, 983, 997, 1003, 993, 1047, 1047, 963, 1015, 1035, 1062, 1050, 1060, 1020, 1018, 1031, 997, 1049, 1005, 1034, 1038, 1021, 1021, 1025, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[977, 989, 1049, 998, 944, 989, 1021, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1000, 999, 1043, 1059, 1038, 998, 989, 43, 0, 0, 0, 0, 0, 0, 0, 1028, 1008, 1046, 1030, 1036, 1052, 984, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 1066, 1013, 988, 1062, 1049, 1017, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1004, 1040, 1055, 1045, 1045, 980, 1036, 31, 5, 2, 0, 0, 0, 0, 0, 1050, 1025, 1035, 1062, 1042, 1017, 1021, 1017, 7, 2, 0, 0, 0, 0, 0, 0, 1053, 1030, 1027, 1042, 1034, 1001, 1009, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1004, 1005, 1024, 1059, 983, 1028, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7965, 8233, 8151, 8254, 8270, 8278, 8221, 8164] avg=8192.0000 max=8278 max_rank=5 min=7965 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32933 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1056, 954, 995, 1008, 1053, 990, 1016, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1004, 1015, 1018, 1003, 1025, 1047, 1013, 44, 0, 0, 0, 0, 0, 0, 0, 1019, 1006, 1006, 1016, 1041, 1037, 1011, 952, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1022, 1040, 1065, 1026, 1022, 1102, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1049, 1014, 1028, 999, 1038, 1038, 1009, 28, 5, 2, 0, 0, 0, 0, 0, 1048, 1041, 1030, 1045, 1004, 1014, 1003, 1019, 7, 2, 0, 0, 0, 0, 0, 0, 1065, 998, 1017, 997, 994, 1050, 1005, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1044, 1081, 1036, 1006, 1014, 955, 1063, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8177, 8088, 8302, 8221, 8213, 8171, 8226] avg=8192.0000 max=8302 max_rank=3 min=8088 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32831 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1044, 1022, 996, 1051, 1098, 1026, 1071, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1053, 1005, 1015, 1049, 1012, 1030, 992, 45, 0, 0, 0, 0, 0, 0, 0, 1048, 1049, 1033, 972, 1063, 1000, 1012, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 975, 997, 977, 1018, 1027, 1062, 1043, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1031, 1028, 999, 996, 1064, 1015, 995, 28, 5, 2, 0, 0, 0, 0, 0, 999, 1048, 987, 1065, 1067, 1036, 1056, 976, 7, 2, 0, 0, 0, 0, 0, 0, 995, 1004, 1013, 1036, 1033, 967, 1008, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 987, 998, 1020, 1058, 1039, 1012, 1015, 1009, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8314, 8219, 8206, 8130, 8198, 8243, 8088, 8138] avg=8192.0000 max=8314 max_rank=0 min=8088 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32667 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1012, 988, 1055, 1022, 1015, 970, 1054, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1020, 1048, 1023, 1050, 994, 998, 1022, 43, 0, 0, 0, 0, 0, 0, 0, 1037, 1042, 1034, 1041, 1007, 1081, 986, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 991, 1031, 1044, 1056, 1022, 1003, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1012, 990, 1001, 1033, 1013, 1009, 1019, 30, 5, 2, 0, 0, 0, 0, 0, 957, 1049, 1042, 1056, 983, 1068, 999, 994, 7, 2, 0, 0, 0, 0, 0, 0, 1034, 1010, 1083, 1058, 1045, 1012, 1014, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 1050, 966, 1001, 1018, 1022, 1003, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8210, 8251, 8178, 8140, 8157, 8279, 8166] avg=8192.0000 max=8279 max_rank=6 min=8140 min_rank=4 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32742 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1009, 969, 1023, 1052, 1030, 1046, 1078, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1000, 1045, 1023, 1036, 995, 1002, 1036, 43, 0, 0, 0, 0, 0, 0, 0, 1046, 1028, 1005, 996, 1049, 1030, 976, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 972, 1007, 998, 981, 1034, 1028, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 982, 987, 1024, 1001, 1021, 989, 1007, 30, 5, 2, 0, 0, 0, 0, 0, 1005, 1004, 1062, 983, 1048, 1041, 961, 1020, 7, 2, 0, 0, 0, 0, 0, 0, 1039, 988, 1055, 1032, 991, 1056, 995, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1032, 1005, 1031, 987, 1063, 1076, 995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8227, 8194, 8122, 8115, 8133, 8259, 8265] avg=8192.0000 max=8265 max_rank=7 min=8115 min_rank=4 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32772 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1049, 1029, 1032, 985, 1043, 1043, 1033, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1010, 975, 1047, 1017, 1013, 975, 1031, 43, 0, 0, 0, 0, 0, 0, 0, 1039, 1044, 1059, 1015, 1005, 996, 982, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1034, 1003, 1026, 1042, 996, 1024, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1046, 1002, 1011, 1056, 1021, 1061, 1045, 29, 5, 2, 0, 0, 0, 0, 0, 961, 999, 1056, 999, 1037, 1030, 1055, 1017, 7, 2, 0, 0, 0, 0, 0, 0, 1013, 1081, 985, 1029, 1000, 1010, 1022, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1030, 1046, 1053, 1037, 1057, 967, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8154, 8158, 8158, 8294, 8163, 8133, 8243] avg=8192.0000 max=8294 max_rank=4 min=8133 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32833 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1045, 1022, 1020, 1016, 1028, 1059, 998, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 994, 1037, 1037, 1021, 954, 1023, 1063, 51, 0, 0, 0, 0, 0, 0, 0, 1020, 1057, 1042, 1048, 1028, 1008, 1031, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1056, 1048, 1044, 998, 1025, 981, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 994, 973, 1056, 998, 1061, 967, 1031, 1008, 32, 11, 9, 0, 0, 0, 0, 0, 983, 997, 1003, 993, 1047, 1047, 963, 1015, 9, 8, 0, 0, 0, 0, 0, 0, 1035, 1062, 1050, 1051, 1020, 1018, 1031, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1005, 1034, 1038, 1021, 1021, 1025, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8202, 8300, 8163, 8140, 8065, 8264, 8204] avg=8192.0000 max=8300 max_rank=2 min=8065 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32673 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1013, 1077, 1077, 1025, 1045, 1035, 1039, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1023, 1009, 1015, 970, 969, 1048, 1014, 43, 0, 0, 0, 0, 0, 0, 0, 1049, 1038, 1032, 1027, 1015, 1033, 982, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1054, 1026, 1000, 1028, 1011, 1007, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1025, 1018, 999, 1024, 1035, 1000, 998, 30, 5, 2, 0, 0, 0, 0, 0, 1051, 1026, 1028, 1020, 1013, 1048, 1053, 1036, 7, 2, 0, 0, 0, 0, 0, 0, 1004, 1031, 1067, 1004, 998, 979, 1010, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1046, 988, 1019, 1038, 1006, 964, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8312, 8114, 8188, 8229, 8158, 8284, 8121, 8130] avg=8192.0000 max=8312 max_rank=0 min=8114 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32693 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8320, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8064, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[997, 1042, 1014, 1031, 1073, 986, 1069, 1005, 1039, 1015, 1007, 1095, 1006, 1019, 1070, 980, 1021, 1027, 1041, 988, 1020, 1004, 1079, 993, 1018, 1037, 1015, 1036, 998, 982, 1004, 1024, 1048, 987, 986, 998, 995, 1024, 1055, 1052, 1030, 1031, 1023, 999, 1034, 1034, 1011, 1014, 1035, 1021, 990, 975, 1041, 1035, 1025, 988, 1062, 1045, 1028, 1059, 1019, 1034, 1067, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1041, 1061, 1015, 1050, 1054, 1039, 1032, 1012, 997, 983, 1025, 1018, 1042, 1046, 1022, 1013, 1052, 1038, 982, 1042, 1072, 1009, 1026, 1060, 1015, 1023, 1022, 990, 1027, 1014, 1006, 1003, 1015, 945, 1010, 1066, 1039, 955, 1012, 1095, 1019, 1033, 996, 1031, 1048, 1033, 1021, 981, 998, 1053, 1036, 1045, 1027, 1014, 992, 1011, 1046, 1034, 1104, 992, 980, 1049, 1043, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1051, 1030, 1002, 1050, 1023, 1048, 979, 1015, 993, 997, 996, 1046, 1036, 971, 1055, 1071, 997, 986, 999, 978, 1040, 1044, 1014, 1007, 1026, 1032, 953, 1032, 1033, 1057, 1034, 1022, 1052, 1011, 1046, 1053, 1018, 998, 1060, 985, 994, 1017, 1060, 1073, 1044, 1005, 1043, 976, 1033, 1012, 1016, 1021, 1020, 977, 1035, 1070, 1019, 1054, 1034, 1008, 1063, 1043, 1054, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[991, 1041, 1032, 1046, 1005, 1023, 1007, 1026, 1034, 1014, 1038, 1020, 1022, 1050, 1045, 1018, 1029, 1000, 1055, 997, 1006, 1022, 1049, 1002, 1095, 1056, 998, 1028, 1048, 996, 1052, 1024, 1049, 982, 1067, 994, 1015, 999, 995, 976, 1021, 1013, 1000, 1007, 1035, 1037, 1047, 1074, 1076, 979, 1028, 1013, 1018, 1013, 1032, 1024, 989, 1001, 1013, 1009, 1071, 1058, 1030, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1061, 996, 980, 1035, 1001, 1034, 1025, 1041, 1045, 1036, 1044, 1051, 1033, 1018, 1013, 1041, 947, 1054, 1017, 1029, 1047, 1030, 1036, 1007, 1047, 963, 1068, 1045, 1023, 1029, 1003, 1054, 972, 1042, 1042, 1032, 1051, 1016, 1085, 977, 977, 978, 1069, 1031, 1032, 1035, 1040, 990, 1015, 1027, 1002, 1028, 1020, 1001, 1040, 1021, 1015, 1031, 1023, 1003, 1023, 1019, 1017, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1015, 984, 1037, 992, 984, 1043, 1012, 994, 1037, 979, 1006, 1002, 1022, 1049, 1034, 1024, 1046, 991, 994, 1017, 999, 1020, 1029, 1016, 1038, 1057, 1038, 1011, 990, 972, 1026, 1033, 1005, 1020, 1040, 1004, 1030, 1063, 1060, 1029, 1049, 993, 972, 1037, 1012, 1057, 1094, 1016, 1014, 1022, 1052, 1015, 1049, 1046, 1041, 1017, 1096, 1033, 1010, 1039, 1027, 1059, 1034, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1049, 1006, 1003, 1011, 1020, 1010, 1034, 963, 1003, 1007, 1079, 1031, 1022, 1080, 1003, 1053, 1078, 1032, 1046, 969, 991, 1019, 1001, 1027, 1042, 1037, 1056, 1022, 1036, 1050, 978, 995, 1035, 1026, 1056, 1039, 1019, 976, 993, 996, 1043, 999, 1034, 985, 1046, 1045, 1026, 1037, 1051, 1029, 1021, 990, 1056, 1049, 1032, 1043, 1029, 1027, 1006, 951, 1035, 1034, 1040, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1030, 1032, 1049, 1012, 958, 1000, 1034, 995, 1022, 1008, 1028, 984, 999, 1041, 1017, 1025, 1024, 1046, 953, 1058, 1040, 1034, 1037, 973, 1048, 1072, 1039, 1025, 1005, 1032, 1016, 993, 1029, 1039, 985, 1094, 1016, 999, 1038, 1052, 1043, 1000, 1054, 1029, 1000, 1057, 1014, 1035, 1004, 984, 1031, 1048, 1045, 1038, 1011, 1034, 1034, 1023, 1047, 1010, 1010, 1079, 1028, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1041, 1061, 1015, 1050, 1054, 1039, 1032, 1012, 17, 8, 0, 0, 0, 0, 0, 0, 997, 983, 1025, 1018, 1042, 1036, 1022, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1038, 982, 1042, 1072, 1009, 1026, 1060, 31, 0, 0, 0, 0, 0, 0, 0, 1015, 1023, 1022, 990, 1027, 1014, 1006, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 945, 1010, 1066, 1039, 955, 1012, 1095, 9, 2, 0, 0, 0, 0, 0, 0, 1019, 1033, 996, 1031, 1048, 1024, 1021, 981, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1053, 1036, 1045, 1027, 1014, 992, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1034, 1104, 992, 980, 1001, 1043, 982, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8329, 8136, 8312, 8100, 8148, 8153, 8176, 8182] avg=8192.0000 max=8329 max_rank=0 min=8100 min_rank=3 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32659 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[991, 1041, 1032, 1046, 1005, 1023, 1007, 1026, 17, 8, 0, 0, 0, 0, 0, 0, 1034, 1014, 1038, 1020, 1022, 1040, 1045, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1000, 1055, 997, 1006, 1022, 1049, 1002, 31, 0, 0, 0, 0, 0, 0, 0, 1095, 1056, 998, 1028, 1048, 996, 1052, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 982, 1067, 994, 1015, 999, 995, 976, 9, 2, 0, 0, 0, 0, 0, 0, 1021, 1013, 1000, 1007, 1035, 1028, 1047, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 979, 1028, 1013, 1018, 1013, 1032, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1001, 1013, 1009, 1071, 1010, 1030, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8231, 8191, 8297, 8088, 8225, 8183, 8125] avg=8192.0000 max=8297 max_rank=3 min=8088 min_rank=4 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32621 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1051, 1030, 1002, 1050, 1023, 1048, 979, 1015, 17, 7, 0, 0, 0, 0, 0, 0, 993, 997, 996, 1046, 1036, 963, 1055, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 997, 986, 999, 978, 1040, 1044, 1014, 1007, 31, 0, 0, 0, 0, 0, 0, 0, 1026, 1032, 953, 1032, 1033, 1057, 1034, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1011, 1046, 1053, 1018, 998, 1060, 985, 9, 1, 0, 0, 0, 0, 0, 0, 994, 1017, 1060, 1073, 1044, 996, 1043, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1012, 1016, 1021, 1020, 977, 1035, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1054, 1034, 1008, 1063, 995, 1054, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8157, 8096, 8189, 8233, 8203, 8184, 8252] avg=8192.0000 max=8252 max_rank=7 min=8096 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32872 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[997, 1042, 1014, 1031, 1073, 986, 1069, 1005, 17, 8, 0, 0, 0, 0, 0, 0, 1039, 1015, 1007, 1095, 1006, 1009, 1070, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1027, 1041, 988, 1020, 1004, 1079, 993, 30, 0, 0, 0, 0, 0, 0, 0, 1018, 1037, 1015, 1036, 998, 982, 1004, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 987, 986, 998, 995, 1024, 1055, 1052, 9, 2, 0, 0, 0, 0, 0, 0, 1030, 1031, 1023, 999, 1034, 1025, 1011, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1021, 990, 975, 1041, 1035, 1025, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1045, 1028, 1059, 1019, 987, 1067, 1056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8221, 8203, 8114, 8156, 8167, 8110, 8323] avg=8192.0000 max=8323 max_rank=7 min=8110 min_rank=6 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32756 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1061, 996, 980, 1035, 1001, 1034, 1025, 1041, 17, 8, 0, 0, 0, 0, 0, 0, 1045, 1036, 1044, 1051, 1033, 1008, 1013, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 947, 1054, 1017, 1029, 1047, 1030, 1036, 1007, 30, 0, 0, 0, 0, 0, 0, 0, 1047, 963, 1068, 1045, 1023, 1029, 1003, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1042, 1042, 1032, 1051, 1016, 1085, 977, 9, 2, 0, 0, 0, 0, 0, 0, 977, 978, 1069, 1031, 1032, 1026, 1040, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1027, 1002, 1028, 1020, 1001, 1040, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1031, 1023, 1003, 1023, 972, 1017, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8271, 8197, 8232, 8228, 8143, 8154, 8113] avg=8192.0000 max=8271 max_rank=1 min=8113 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32638 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1015, 984, 1037, 992, 984, 1043, 1012, 994, 17, 8, 0, 0, 0, 0, 0, 0, 1037, 979, 1006, 1002, 1022, 1039, 1034, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 991, 994, 1017, 999, 1020, 1029, 1016, 31, 0, 0, 0, 0, 0, 0, 0, 1038, 1057, 1038, 1011, 990, 972, 1026, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1020, 1040, 1004, 1030, 1063, 1060, 1029, 9, 2, 0, 0, 0, 0, 0, 0, 1049, 993, 972, 1037, 1012, 1048, 1094, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1022, 1052, 1015, 1049, 1046, 1041, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1096, 1033, 1010, 1039, 1027, 1011, 1034, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 8143, 8143, 8165, 8262, 8221, 8256, 8260] avg=8192.0000 max=8262 max_rank=4 min=8086 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32999 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1049, 1006, 1003, 1011, 1020, 1010, 1034, 963, 20, 11, 0, 0, 0, 0, 0, 0, 1003, 1007, 1079, 1031, 1022, 1065, 1003, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1032, 1046, 969, 991, 1019, 1001, 1027, 34, 0, 0, 0, 0, 0, 0, 0, 1035, 1037, 1056, 1022, 1036, 1050, 978, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1026, 1056, 1039, 1019, 976, 993, 996, 14, 4, 0, 0, 0, 0, 0, 0, 1043, 999, 1034, 985, 1046, 1031, 1026, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1029, 1021, 990, 1056, 1049, 1032, 1043, 7, 0, 0, 0, 0, 0, 0, 0, 1029, 1027, 1006, 951, 1035, 980, 1040, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8263, 8197, 8209, 8158, 8201, 8278, 8103] avg=8192.0000 max=8278 max_rank=6 min=8103 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32740 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1030, 1032, 1049, 1012, 958, 1000, 1034, 995, 18, 8, 0, 0, 0, 0, 0, 0, 1022, 1008, 1028, 984, 999, 1031, 1017, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1046, 953, 1058, 1040, 1034, 1037, 973, 32, 0, 0, 0, 0, 0, 0, 0, 1048, 1072, 1039, 1025, 1005, 1032, 1016, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1039, 985, 1094, 1016, 999, 1038, 1052, 9, 2, 0, 0, 0, 0, 0, 0, 1043, 1000, 1054, 1029, 1000, 1048, 1014, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 984, 1031, 1048, 1045, 1038, 1011, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1023, 1047, 1010, 1010, 1029, 1028, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8114, 8197, 8230, 8263, 8223, 8195, 8178] avg=8192.0000 max=8263 max_rank=4 min=8114 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32859 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8064, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8064, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8320, 8192, 8064, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8448, 8320, 8192, 8192, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1079, 1036, 1009, 1017, 1074, 992, 994, 1070, 1014, 1020, 1046, 1012, 999, 986, 1040, 1024, 1072, 1064, 988, 990, 1061, 1000, 1050, 1059, 1024, 1016, 1052, 1018, 1044, 994, 1029, 1002, 990, 999, 1033, 1007, 1043, 1076, 1043, 1012, 976, 1025, 1001, 1033, 1016, 1013, 1047, 1041, 971, 1052, 1016, 1006, 996, 1018, 1003, 1022, 974, 1035, 1017, 1009, 1071, 1019, 1061, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1045, 1055, 1034, 1017, 1010, 1050, 998, 1030, 990, 1046, 1022, 1028, 1080, 1034, 1071, 1035, 1068, 1008, 985, 1026, 966, 1022, 1080, 1032, 1042, 984, 1001, 1018, 1000, 1030, 1040, 1036, 1020, 1068, 998, 994, 1053, 1029, 1016, 960, 986, 1016, 956, 1033, 1003, 1031, 1051, 1005, 1029, 1029, 978, 996, 992, 1055, 1024, 1051, 1054, 1008, 1050, 1050, 987, 1052, 1000, 1079] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1036, 1026, 1025, 1010, 1024, 1089, 1054, 1011, 1002, 1030, 1041, 923, 1039, 1040, 1012, 990, 991, 1046, 1039, 1048, 985, 1069, 1033, 1000, 1074, 1018, 1051, 1055, 1001, 1048, 1003, 1037, 1055, 1048, 1001, 1024, 973, 1000, 1038, 1070, 987, 1007, 1043, 1009, 1009, 1050, 1038, 1005, 1047, 1058, 985, 1044, 1028, 1017, 1068, 972, 1021, 1015, 1034, 1001, 1037, 1007, 1010, 985] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1070, 1007, 1019, 1050, 1032, 1024, 1071, 1041, 997, 1018, 1036, 1058, 1053, 1039, 1020, 1026, 1051, 1037, 1065, 1040, 999, 1013, 1069, 1018, 1038, 986, 972, 1016, 989, 1012, 986, 1010, 1027, 1040, 1030, 1065, 1031, 1070, 1006, 1016, 1045, 1029, 978, 1048, 991, 1022, 1049, 1039, 987, 1000, 1041, 998, 1041, 1028, 1036, 996, 1042, 1022, 1019, 1033, 1006, 972, 1000, 967] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1048, 1043, 1033, 1028, 1001, 997, 1023, 1068, 1046, 1025, 1034, 1060, 1035, 1104, 1023, 979, 1044, 999, 1012, 1034, 1016, 1007, 1059, 1020, 1002, 1027, 965, 1062, 996, 1059, 1022, 1035, 1018, 1050, 1028, 1017, 1018, 1040, 966, 1045, 1003, 1044, 989, 1026, 988, 988, 1039, 1032, 1050, 1030, 1028, 1047, 1018, 960, 1015, 1030, 1022, 1002, 1010, 999, 1036, 1012, 1043, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1047, 1094, 1029, 1039, 1034, 1057, 1012, 1001, 1053, 1031, 1033, 1087, 951, 1020, 1005, 1036, 1038, 1028, 991, 1031, 1069, 1024, 1005, 1009, 1057, 1044, 1011, 1020, 1011, 953, 1035, 1076, 1030, 988, 1017, 1013, 1007, 1025, 1045, 999, 1006, 1008, 1053, 967, 1028, 1059, 970, 1026, 986, 993, 1019, 1074, 1016, 1035, 1063, 1012, 1053, 1032, 1022, 1028, 1034, 1029, 983, 985] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1013, 983, 1083, 1027, 1002, 1041, 1054, 989, 995, 1050, 1013, 1011, 1025, 1055, 996, 968, 1028, 1005, 1040, 1051, 1036, 996, 1028, 1067, 1058, 1014, 1010, 980, 1008, 965, 1006, 1054, 1080, 976, 1069, 1043, 1017, 1054, 1021, 1006, 996, 987, 1021, 1014, 1000, 1014, 1023, 1032, 1051, 1032, 1035, 1028, 1038, 1000, 1054, 1022, 1016, 984, 1009, 1064, 1046, 1074, 1002, 1077] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1050, 1024, 973, 1013, 985, 1037, 976, 1033, 1030, 981, 1015, 986, 1003, 1054, 1079, 996, 1032, 1040, 1036, 1034, 1051, 1057, 1026, 1091, 1061, 996, 997, 984, 1010, 992, 1079, 996, 1044, 1019, 980, 1020, 1046, 986, 1030, 1030, 1037, 1035, 1048, 998, 1032, 1058, 1062, 1026, 1022, 1093, 1014, 1011, 1035, 1048, 1052, 1010, 1069, 1019, 988, 1028, 995, 983, 989, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[996, 1055, 1034, 1017, 1010, 1050, 998, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1046, 1022, 1028, 1080, 1033, 1071, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1008, 985, 1026, 966, 1022, 1024, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 984, 1001, 1018, 1000, 1030, 1040, 1036, 28, 12, 0, 0, 0, 0, 0, 0, 1020, 1068, 998, 994, 1053, 1026, 1016, 960, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1016, 956, 1033, 1003, 1031, 1051, 1005, 44, 0, 0, 0, 0, 0, 0, 0, 1029, 1029, 978, 996, 992, 1055, 1024, 1051, 2, 1, 0, 0, 0, 0, 0, 0, 1054, 1008, 1050, 1050, 987, 1052, 1000, 1079, 21, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8305, 8131, 8191, 8135, 8125, 8157, 8302] avg=8192.0000 max=8305 max_rank=1 min=8125 min_rank=5 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1028, 1036, 1009, 1017, 1074, 992, 994, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1020, 1046, 1012, 999, 985, 1040, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1064, 988, 990, 1061, 1000, 995, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1016, 1052, 1018, 1044, 994, 1029, 1002, 29, 12, 0, 0, 0, 0, 0, 0, 990, 999, 1033, 1007, 1043, 1073, 1043, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1025, 1001, 1033, 1016, 1013, 1047, 1041, 43, 0, 0, 0, 0, 0, 0, 0, 971, 1052, 1016, 1006, 996, 1018, 1003, 1022, 2, 1, 0, 0, 0, 0, 0, 0, 974, 1035, 1017, 1009, 1071, 1019, 1061, 1036, 22, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8140, 8229, 8220, 8200, 8195, 8087, 8245] avg=8192.0000 max=8245 max_rank=7 min=8087 min_rank=6 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1020, 1007, 1019, 1050, 1032, 1024, 1071, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1018, 1036, 1058, 1053, 1038, 1020, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1037, 1065, 1040, 999, 1013, 1013, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 986, 972, 1016, 989, 1012, 986, 1010, 29, 12, 0, 0, 0, 0, 0, 0, 1027, 1040, 1030, 1065, 1031, 1067, 1006, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1029, 978, 1048, 991, 1022, 1049, 1039, 44, 0, 0, 0, 0, 0, 0, 0, 987, 1000, 1041, 998, 1041, 1028, 1036, 996, 2, 1, 0, 0, 0, 0, 0, 0, 1042, 1022, 1019, 1033, 1006, 972, 1000, 967, 21, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8246, 8236, 8050, 8282, 8245, 8130, 8083] avg=8192.0000 max=8282 max_rank=4 min=8050 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[987, 1026, 1025, 1010, 1024, 1089, 1054, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1030, 1041, 923, 1039, 1039, 1012, 990, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1046, 1039, 1048, 985, 1069, 979, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1018, 1051, 1055, 1001, 1048, 1003, 1037, 28, 12, 0, 0, 0, 0, 0, 0, 1055, 1048, 1001, 1024, 973, 997, 1038, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1007, 1043, 1009, 1009, 1050, 1038, 1005, 42, 0, 0, 0, 0, 0, 0, 0, 1047, 1058, 985, 1044, 1028, 1017, 1068, 972, 2, 1, 0, 0, 0, 0, 0, 0, 1021, 1015, 1034, 1001, 1037, 1007, 1010, 985, 21, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8076, 8157, 8327, 8206, 8190, 8222, 8132] avg=8192.0000 max=8327 max_rank=3 min=8076 min_rank=1 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[999, 1043, 1033, 1028, 1001, 997, 1023, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1025, 1034, 1060, 1035, 1103, 1023, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 999, 1012, 1034, 1016, 1007, 1004, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1027, 965, 1062, 996, 1059, 1022, 1035, 28, 12, 0, 0, 0, 0, 0, 0, 1018, 1050, 1028, 1017, 1018, 1037, 966, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1044, 989, 1026, 988, 988, 1039, 1032, 43, 0, 0, 0, 0, 0, 0, 0, 1050, 1030, 1028, 1047, 1018, 960, 1015, 1030, 2, 1, 0, 0, 0, 0, 0, 0, 1022, 1002, 1010, 999, 1036, 1012, 1043, 1037, 21, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8305, 8136, 8208, 8179, 8152, 8181, 8183] avg=8192.0000 max=8305 max_rank=1 min=8136 min_rank=2 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[998, 1094, 1029, 1039, 1034, 1057, 1012, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1031, 1033, 1087, 951, 1019, 1005, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1028, 991, 1031, 1069, 1024, 953, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1044, 1011, 1020, 1011, 953, 1035, 1076, 28, 11, 0, 0, 0, 0, 0, 0, 1030, 988, 1017, 1013, 1007, 1022, 1045, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1008, 1053, 967, 1028, 1059, 970, 1026, 41, 0, 0, 0, 0, 0, 0, 0, 986, 993, 1019, 1074, 1016, 1035, 1063, 1012, 2, 1, 0, 0, 0, 0, 0, 0, 1053, 1032, 1022, 1028, 1034, 1029, 983, 985, 21, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8215, 8143, 8246, 8121, 8158, 8201, 8188] avg=8192.0000 max=8264 max_rank=0 min=8121 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[994, 1024, 973, 1013, 985, 1037, 976, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 981, 1015, 986, 1003, 1047, 1079, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1040, 1036, 1034, 1051, 1057, 965, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 996, 997, 984, 1010, 992, 1079, 996, 31, 15, 0, 0, 0, 0, 0, 0, 1044, 1019, 980, 1020, 1046, 981, 1030, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1035, 1048, 998, 1032, 1058, 1062, 1026, 46, 0, 0, 0, 0, 0, 0, 0, 1022, 1093, 1014, 1011, 1035, 1048, 1052, 1010, 3, 7, 0, 0, 0, 0, 0, 0, 1069, 1019, 988, 1028, 995, 983, 989, 1012, 25, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8035, 8137, 8306, 8161, 8150, 8342, 8295, 8110] avg=8192.0000 max=8342 max_rank=5 min=8035 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[966, 983, 1083, 1027, 1002, 1041, 1054, 989, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1050, 1013, 1011, 1025, 1054, 996, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1005, 1040, 1051, 1036, 996, 975, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1014, 1010, 980, 1008, 965, 1006, 1054, 27, 11, 0, 0, 0, 0, 0, 0, 1080, 976, 1069, 1043, 1017, 1051, 1021, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 996, 987, 1021, 1014, 1000, 1014, 1023, 1032, 42, 0, 0, 0, 0, 0, 0, 0, 1051, 1032, 1035, 1028, 1038, 1000, 1054, 1022, 2, 1, 0, 0, 0, 0, 0, 0, 1016, 984, 1009, 1064, 1046, 1074, 1002, 1077, 20, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8112, 8198, 8133, 8263, 8129, 8263, 8293] avg=8192.0000 max=8293 max_rank=7 min=8112 min_rank=1 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32948 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8448, 8320, 8192, 8320, 8192, 8192, 7936, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8448, 8192, 8064, 8192, 8064, 8064, 8320, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1051, 981, 1056, 1030, 1062, 1054, 1037, 1014, 1045, 1058, 1081, 1039, 1025, 1013, 1028, 1038, 983, 1049, 997, 1044, 1048, 992, 1016, 1064, 1020, 968, 1041, 1013, 1043, 996, 1028, 1066, 1041, 995, 968, 1031, 998, 979, 1002, 1022, 1028, 982, 1048, 1027, 1025, 980, 1023, 1022, 1053, 1042, 997, 1027, 1030, 1014, 1049, 1018, 999, 1024, 993, 1039, 992, 1048, 1028, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1056, 1031, 1012, 1021, 1028, 1020, 965, 1049, 995, 1011, 1002, 979, 1058, 1047, 1024, 1036, 1017, 991, 997, 1000, 1005, 1009, 1055, 1038, 1049, 1021, 1018, 1009, 1037, 991, 1023, 1039, 1019, 1050, 996, 1021, 1079, 977, 1051, 1006, 1022, 1052, 1049, 1018, 1090, 1037, 1038, 1056, 1037, 1015, 1038, 1042, 1001, 1058, 990, 984, 1034, 1022, 1028, 1006, 1012, 1034, 997, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1032, 966, 1082, 986, 1021, 1032, 999, 989, 1043, 1011, 1039, 1048, 1024, 1011, 1035, 1055, 1026, 1071, 1012, 1020, 1014, 1009, 1024, 1010, 1018, 998, 1031, 953, 971, 1067, 1011, 997, 1048, 1010, 1016, 1028, 1044, 1049, 1040, 1006, 1017, 1072, 999, 1012, 1060, 1035, 1009, 1062, 1040, 993, 1067, 994, 1001, 1068, 1003, 991, 1029, 1051, 1016, 1049, 1026, 1002, 1021, 1073] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1042, 1083, 997, 1053, 1032, 1066, 1050, 1062, 1034, 1006, 986, 1030, 1040, 963, 1059, 1041, 996, 1053, 1002, 1015, 1028, 1020, 1007, 1010, 983, 1006, 1042, 1049, 1011, 1063, 998, 1031, 1091, 1018, 1092, 1033, 989, 1000, 1061, 1061, 982, 1030, 988, 1006, 1007, 1027, 1006, 1018, 1062, 1015, 1017, 1019, 1009, 1012, 976, 1036, 1016, 1006, 995, 987, 1039, 1022, 1053, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1007, 1086, 999, 1027, 1024, 996, 1012, 1016, 1040, 1018, 1027, 1019, 1029, 1020, 986, 992, 1021, 1005, 1055, 1009, 1042, 928, 1039, 1054, 1077, 1075, 1098, 1037, 993, 994, 998, 1010, 1006, 1008, 1041, 1025, 1003, 986, 1009, 1028, 1021, 1066, 1025, 1034, 1052, 1074, 1017, 1000, 1020, 988, 1045, 1025, 1047, 1015, 1055, 1029, 1018, 1018, 1041, 976, 1015, 1031, 1055, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1067, 1021, 1000, 1014, 1012, 1053, 1031, 984, 1026, 1073, 975, 1018, 1014, 984, 1051, 1038, 1022, 1018, 1052, 939, 1056, 1001, 1086, 1022, 1043, 1056, 1023, 1007, 1011, 985, 1018, 1051, 989, 1010, 986, 1047, 1014, 1016, 979, 1050, 1109, 1030, 1012, 1029, 1037, 1016, 1045, 1029, 996, 1021, 1052, 1028, 1026, 1045, 1025, 999, 1026, 1067, 1013, 1034, 1014, 971, 1028, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[978, 1009, 962, 1033, 1029, 1056, 1021, 1051, 1023, 1035, 1031, 970, 1079, 1037, 1066, 1022, 992, 993, 1039, 997, 1004, 1092, 1047, 995, 977, 1008, 1036, 1078, 1014, 1001, 1039, 1080, 1048, 1034, 1014, 1005, 1059, 1020, 1015, 1063, 1017, 1051, 1019, 1047, 1006, 1044, 1024, 1054, 1042, 1031, 1011, 964, 1024, 984, 1012, 1045, 1030, 1027, 969, 1035, 1004, 1014, 988, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1038, 990, 994, 1003, 1024, 1031, 1044, 1013, 1057, 1059, 1013, 1013, 1054, 1006, 1021, 971, 1007, 1044, 1031, 1000, 1024, 1035, 996, 1013, 1038, 1015, 1020, 1076, 1026, 1046, 996, 996, 987, 1019, 1017, 1006, 1008, 1021, 1036, 1078, 1023, 1031, 991, 1004, 1017, 1001, 1079, 1007, 1045, 1048, 1073, 1027, 988, 1009, 987, 1007, 1031, 1087, 1020, 1059, 1032, 1060, 1008, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1051, 981, 1056, 1030, 1062, 1048, 1037, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1058, 1081, 1039, 1010, 1013, 1028, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1049, 997, 1044, 1048, 992, 1016, 1064, 30, 0, 0, 0, 0, 0, 0, 0, 1020, 968, 1041, 1013, 1043, 996, 1028, 1066, 2, 0, 0, 0, 0, 0, 0, 0, 1041, 995, 968, 1031, 998, 979, 1002, 1022, 6, 4, 0, 0, 0, 0, 0, 0, 1028, 947, 1048, 1027, 1025, 980, 1023, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1042, 997, 1027, 1030, 1014, 1049, 1018, 9, 5, 0, 0, 0, 0, 0, 0, 999, 1024, 993, 1039, 992, 1048, 1028, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8312, 8223, 8177, 8046, 8100, 8244, 8155] avg=8192.0000 max=8312 max_rank=1 min=8046 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32545 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1032, 966, 1082, 986, 1021, 1028, 999, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1011, 1039, 1048, 1009, 1011, 1035, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1071, 1012, 1020, 1014, 1009, 1024, 1010, 33, 0, 0, 0, 0, 0, 0, 0, 1018, 998, 1031, 953, 971, 1067, 1011, 997, 1, 0, 0, 0, 0, 0, 0, 0, 1048, 1010, 1016, 1028, 1044, 1049, 1040, 1006, 6, 3, 0, 0, 0, 0, 0, 0, 1017, 1034, 999, 1012, 1060, 1035, 1009, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 993, 1067, 994, 1001, 1068, 1003, 991, 9, 5, 0, 0, 0, 0, 0, 0, 1029, 1051, 1016, 1049, 1026, 1002, 1021, 1073, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8251, 8219, 8047, 8250, 8228, 8171, 8267] avg=8192.0000 max=8267 max_rank=7 min=8047 min_rank=3 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32916 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1007, 1086, 999, 1027, 1024, 992, 1012, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1018, 1027, 1019, 1014, 1020, 986, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1005, 1055, 1009, 1042, 928, 1039, 1054, 32, 0, 0, 0, 0, 0, 0, 0, 1077, 1075, 1098, 1037, 993, 994, 998, 1010, 1, 0, 0, 0, 0, 0, 0, 0, 1006, 1008, 1041, 1025, 1003, 986, 1009, 1028, 6, 3, 0, 0, 0, 0, 0, 0, 1021, 1029, 1025, 1034, 1052, 1074, 1017, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 988, 1045, 1025, 1047, 1015, 1055, 1029, 9, 5, 0, 0, 0, 0, 0, 0, 1018, 1018, 1041, 976, 1015, 1031, 1055, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8116, 8185, 8283, 8115, 8252, 8238, 8184] avg=8192.0000 max=8283 max_rank=3 min=8115 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1056, 1031, 1012, 1021, 1028, 1016, 965, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1011, 1002, 979, 1041, 1047, 1024, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 991, 997, 1000, 1005, 1009, 1055, 1038, 32, 0, 0, 0, 0, 0, 0, 0, 1049, 1021, 1018, 1009, 1037, 991, 1023, 1039, 1, 0, 0, 0, 0, 0, 0, 0, 1019, 1050, 996, 1021, 1079, 977, 1051, 1006, 7, 3, 0, 0, 0, 0, 0, 0, 1022, 1015, 1049, 1018, 1090, 1037, 1038, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1015, 1038, 1042, 1001, 1058, 990, 984, 10, 5, 0, 0, 0, 0, 0, 0, 1034, 1022, 1028, 1006, 1012, 1034, 997, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8135, 8144, 8188, 8209, 8325, 8180, 8177] avg=8192.0000 max=8325 max_rank=5 min=8135 min_rank=1 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32891 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1042, 1083, 997, 1053, 1032, 1052, 1050, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1006, 986, 1030, 1016, 963, 1059, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1053, 1002, 1015, 1028, 1020, 1007, 1010, 35, 0, 0, 0, 0, 0, 0, 0, 983, 1006, 1042, 1049, 1011, 1063, 998, 1031, 6, 7, 0, 0, 0, 0, 0, 0, 1091, 1018, 1092, 1033, 989, 1000, 1061, 1061, 11, 8, 0, 0, 0, 0, 0, 0, 982, 984, 988, 1006, 1007, 1027, 1006, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1015, 1017, 1019, 1009, 1012, 976, 1036, 13, 11, 0, 0, 0, 0, 0, 0, 1016, 1006, 995, 987, 1039, 1022, 1053, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8371, 8135, 8166, 8196, 8364, 8018, 8170, 8116] avg=8192.0000 max=8371 max_rank=0 min=8018 min_rank=5 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1067, 1021, 1000, 1014, 1012, 1047, 1031, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1073, 975, 1018, 999, 984, 1051, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1018, 1052, 939, 1056, 1001, 1086, 1022, 31, 0, 0, 0, 0, 0, 0, 0, 1043, 1056, 1023, 1007, 1011, 985, 1018, 1051, 2, 0, 0, 0, 0, 0, 0, 0, 989, 1010, 986, 1047, 1014, 1016, 979, 1050, 6, 4, 0, 0, 0, 0, 0, 0, 1109, 994, 1012, 1029, 1037, 1016, 1045, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1021, 1052, 1028, 1026, 1045, 1025, 999, 9, 5, 0, 0, 0, 0, 0, 0, 1026, 1067, 1013, 1034, 1014, 971, 1028, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8164, 8227, 8196, 8101, 8271, 8206, 8195] avg=8192.0000 max=8271 max_rank=5 min=8101 min_rank=4 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[978, 1009, 962, 1033, 1029, 1050, 1021, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1035, 1031, 970, 1062, 1037, 1066, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 992, 993, 1039, 997, 1004, 1092, 1047, 995, 32, 0, 0, 0, 0, 0, 0, 0, 977, 1008, 1036, 1078, 1014, 1001, 1039, 1080, 2, 0, 0, 0, 0, 0, 0, 0, 1048, 1034, 1014, 1005, 1059, 1020, 1015, 1063, 7, 4, 0, 0, 0, 0, 0, 0, 1017, 1014, 1019, 1047, 1006, 1044, 1024, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1031, 1011, 964, 1024, 984, 1012, 1045, 10, 5, 0, 0, 0, 0, 0, 0, 1030, 1027, 969, 1035, 1004, 1014, 988, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8246, 8191, 8235, 8269, 8225, 8128, 8109] avg=8192.0000 max=8269 max_rank=4 min=8109 min_rank=7 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1038, 990, 994, 1003, 1024, 1027, 1044, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1059, 1013, 1013, 1037, 1006, 1021, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1044, 1031, 1000, 1024, 1035, 996, 1013, 31, 0, 0, 0, 0, 0, 0, 0, 1038, 1015, 1020, 1076, 1026, 1046, 996, 996, 1, 0, 0, 0, 0, 0, 0, 0, 987, 1019, 1017, 1006, 1008, 1021, 1036, 1078, 7, 3, 0, 0, 0, 0, 0, 0, 1023, 995, 991, 1004, 1017, 1001, 1079, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1048, 1073, 1027, 988, 1009, 987, 1007, 10, 5, 0, 0, 0, 0, 0, 0, 1031, 1087, 1020, 1059, 1032, 1060, 1008, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8177, 8181, 8214, 8182, 8117, 8199, 8333] avg=8192.0000 max=8333 max_rank=7 min=8117 min_rank=5 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8064, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8064, 8320, 8192, 8320, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1061, 974, 1025, 1043, 968, 1032, 1018, 1045, 1052, 1027, 1040, 1031, 980, 994, 1053, 1021, 1006, 1029, 1000, 969, 1014, 1015, 1026, 1028, 1002, 1037, 1077, 1011, 1026, 976, 990, 1004, 1000, 984, 1032, 1011, 1020, 1051, 996, 1045, 1045, 1007, 1023, 1026, 1036, 1046, 1033, 1082, 1033, 1047, 1060, 1024, 1046, 1069, 1039, 1083, 1013, 1024, 1013, 983, 1007, 1019, 1021, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1036, 1055, 1044, 984, 1046, 1053, 1044, 1014, 1004, 1031, 1039, 1031, 999, 1085, 1019, 1011, 1001, 1012, 1010, 991, 1012, 1043, 1028, 995, 1011, 1036, 1017, 1063, 992, 1021, 1052, 1039, 1046, 1043, 1047, 1018, 1011, 1087, 1062, 1007, 988, 982, 1045, 1011, 1046, 1005, 1021, 1038, 1041, 977, 989, 1056, 1010, 1014, 995, 1015, 1042, 1005, 1039, 1033, 985, 1021, 1040, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[999, 1079, 983, 1007, 1050, 987, 982, 1032, 1049, 1053, 1025, 1063, 1021, 1013, 984, 1017, 1033, 1018, 1011, 993, 967, 1038, 1007, 1057, 1043, 1043, 1052, 985, 1008, 1030, 1023, 1041, 986, 1114, 1018, 1026, 1049, 1017, 1002, 1009, 1027, 1046, 1080, 1039, 1061, 1046, 1018, 1048, 941, 978, 1004, 1015, 1024, 997, 1054, 998, 1008, 1064, 1012, 1043, 999, 1041, 1043, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1024, 1065, 1016, 1003, 979, 1017, 1021, 970, 977, 985, 1027, 1008, 1017, 1040, 1039, 1072, 1024, 1037, 1052, 1030, 1061, 976, 997, 1051, 1018, 996, 1057, 1001, 1012, 1023, 1057, 1073, 1027, 1027, 1047, 1032, 1048, 1008, 1044, 1076, 1059, 1011, 955, 1082, 1027, 1072, 1001, 1013, 1040, 983, 999, 1007, 985, 963, 1020, 1038, 1077, 1017, 1035, 1017, 1035, 1024, 1025, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1035, 1032, 1047, 1043, 1011, 1028, 1037, 969, 1051, 1047, 1031, 973, 1070, 1036, 1052, 994, 1003, 1040, 1051, 999, 1015, 1060, 1105, 1057, 1026, 1035, 1033, 1004, 954, 1015, 1046, 1049, 1022, 1006, 1055, 1059, 991, 1056, 1043, 995, 983, 1048, 992, 971, 1022, 1005, 1057, 1008, 1013, 1001, 1054, 1014, 1015, 1047, 1055, 983, 1032, 1040, 1008, 970, 1007, 1006, 1024, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[981, 1057, 994, 1054, 953, 1029, 1032, 1010, 1053, 1074, 1047, 997, 1031, 1021, 995, 1020, 1068, 1039, 1052, 1070, 1061, 1020, 1023, 1010, 1017, 1039, 1046, 1055, 977, 1001, 1042, 1005, 1018, 1049, 999, 1006, 1045, 1030, 1042, 1007, 1040, 1031, 1060, 998, 1020, 973, 1005, 983, 1062, 1007, 1024, 1026, 1016, 1014, 951, 987, 1041, 976, 1055, 1011, 1004, 1067, 1059, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1057, 1032, 1013, 1080, 1056, 1033, 1034, 1067, 1020, 1019, 1055, 1010, 1030, 1055, 975, 1021, 1022, 1019, 1050, 1023, 1009, 978, 991, 1031, 988, 1067, 1042, 1056, 1078, 1006, 1020, 939, 1020, 1004, 1010, 1006, 1014, 1059, 986, 1040, 1002, 1051, 983, 1051, 1010, 989, 987, 1053, 957, 1034, 1021, 1024, 1034, 1016, 1039, 1064, 1030, 1070, 1021, 1020, 1009, 965, 1047, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1037, 1034, 1004, 1015, 972, 1016, 1048, 1027, 1024, 1029, 1037, 990, 1035, 1015, 1002, 1013, 1020, 1004, 1017, 1092, 1037, 1104, 1038, 1020, 1022, 988, 1027, 1001, 990, 1016, 1014, 1035, 1066, 1036, 999, 1004, 1024, 1008, 1044, 999, 1031, 1024, 1027, 988, 1088, 977, 1076, 1012, 1030, 1003, 1012, 1045, 1067, 1034, 1029, 1034, 1020, 1026, 964, 1049, 1016, 1052, 1004, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[999, 1079, 983, 1007, 1050, 987, 982, 1032, 5, 0, 0, 0, 0, 0, 0, 0, 1049, 1053, 1015, 1063, 1021, 1013, 984, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1018, 1011, 993, 967, 1038, 1007, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1043, 1052, 985, 1008, 1030, 1023, 1041, 5, 4, 0, 0, 0, 0, 0, 0, 986, 1114, 1018, 1026, 1049, 994, 1002, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1046, 1080, 1039, 1058, 1046, 1018, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 941, 978, 1004, 1015, 1024, 997, 1054, 998, 23, 10, 0, 0, 0, 0, 0, 0, 1008, 1064, 1012, 1043, 999, 1041, 1043, 1036, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8215, 8110, 8234, 8198, 8362, 8044, 8249] avg=8192.0000 max=8362 max_rank=5 min=8044 min_rank=6 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1036, 1055, 1044, 984, 1046, 1053, 1044, 1014, 5, 0, 0, 0, 0, 0, 0, 0, 1004, 1031, 1028, 1031, 999, 1085, 1019, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1012, 1010, 991, 1012, 1043, 1028, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1036, 1017, 1063, 992, 1021, 1052, 1039, 6, 4, 0, 0, 0, 0, 0, 0, 1046, 1043, 1047, 1018, 1011, 1062, 1062, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 988, 982, 1045, 1011, 1043, 1005, 1021, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 977, 989, 1056, 1010, 1014, 995, 1015, 25, 10, 0, 0, 0, 0, 0, 0, 1042, 1005, 1039, 1033, 985, 1021, 1040, 999, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8208, 8078, 8241, 8296, 8133, 8132, 8167] avg=8192.0000 max=8296 max_rank=4 min=8078 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1024, 1065, 1016, 1003, 979, 1017, 1021, 970, 5, 0, 0, 0, 0, 0, 0, 0, 977, 985, 1017, 1008, 1017, 1040, 1039, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1037, 1052, 1030, 1061, 976, 997, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 996, 1057, 1001, 1012, 1023, 1057, 1073, 5, 4, 0, 0, 0, 0, 0, 0, 1027, 1027, 1047, 1032, 1048, 985, 1044, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1011, 955, 1082, 1024, 1072, 1001, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 983, 999, 1007, 985, 963, 1020, 1038, 23, 10, 0, 0, 0, 0, 0, 0, 1077, 1017, 1035, 1017, 1035, 1024, 1025, 1017, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8155, 8214, 8246, 8286, 8217, 8068, 8250] avg=8192.0000 max=8286 max_rank=4 min=8068 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1061, 974, 1025, 1043, 968, 1032, 1018, 1045, 5, 0, 0, 0, 0, 0, 0, 0, 1052, 1027, 1029, 1031, 980, 994, 1053, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1029, 1000, 969, 1014, 1015, 1026, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1037, 1077, 1011, 1026, 976, 990, 1004, 6, 4, 0, 0, 0, 0, 0, 0, 1000, 984, 1032, 1011, 1020, 1027, 996, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1007, 1023, 1026, 1033, 1046, 1033, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1047, 1060, 1024, 1046, 1069, 1039, 1083, 24, 10, 0, 0, 0, 0, 0, 0, 1013, 1024, 1013, 983, 1007, 1019, 1021, 1044, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8187, 8073, 8133, 8115, 8295, 8435, 8127] avg=8192.0000 max=8435 max_rank=6 min=8073 min_rank=2 imbalance_ratio=1.029663 max_load_reduction=57101 max_load_reduction_pct=0.871292 moved_to_echo=32972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1035, 1032, 1047, 1043, 1011, 1028, 1037, 969, 5, 0, 0, 0, 0, 0, 0, 0, 1051, 1047, 1021, 973, 1070, 1036, 1052, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1040, 1051, 999, 1015, 1060, 1105, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1035, 1033, 1004, 954, 1015, 1046, 1049, 5, 4, 0, 0, 0, 0, 0, 0, 1022, 1006, 1055, 1059, 991, 1032, 1043, 995, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1048, 992, 971, 1019, 1005, 1057, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1001, 1054, 1014, 1015, 1047, 1055, 983, 24, 10, 0, 0, 0, 0, 0, 0, 1032, 1040, 1008, 970, 1007, 1006, 1024, 1006, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8244, 8316, 8171, 8203, 8083, 8216, 8096] avg=8192.0000 max=8316 max_rank=2 min=8083 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32598 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[981, 1057, 994, 1054, 953, 1029, 1032, 1010, 5, 0, 0, 0, 0, 0, 0, 0, 1053, 1074, 1036, 997, 1031, 1021, 995, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1039, 1052, 1070, 1061, 1020, 1023, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1039, 1046, 1055, 977, 1001, 1042, 1005, 6, 4, 0, 0, 0, 0, 0, 0, 1018, 1049, 999, 1006, 1045, 1006, 1042, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1031, 1060, 998, 1017, 973, 1005, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1007, 1024, 1026, 1016, 1014, 951, 987, 24, 10, 0, 0, 0, 0, 0, 0, 1041, 976, 1055, 1011, 1004, 1067, 1059, 1057, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8227, 8329, 8192, 8172, 8107, 8121, 8273] avg=8192.0000 max=8329 max_rank=2 min=8107 min_rank=5 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1057, 1032, 1013, 1080, 1056, 1033, 1034, 1067, 8, 0, 0, 0, 0, 0, 0, 0, 1020, 1019, 1035, 1010, 1030, 1055, 975, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1019, 1050, 1023, 1009, 978, 991, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1067, 1042, 1056, 1078, 1006, 1020, 939, 10, 11, 0, 0, 0, 0, 0, 0, 1020, 1004, 1010, 1006, 1014, 1029, 986, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1051, 983, 1051, 1003, 989, 987, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 957, 1034, 1021, 1024, 1034, 1016, 1039, 1064, 30, 14, 0, 0, 0, 0, 0, 0, 1030, 1070, 1021, 1020, 1009, 965, 1047, 1044, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8380, 8165, 8098, 8217, 8109, 8119, 8233, 8215] avg=8192.0000 max=8380 max_rank=0 min=8098 min_rank=2 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1037, 1034, 1004, 1015, 972, 1016, 1048, 1027, 5, 0, 0, 0, 0, 0, 0, 0, 1024, 1029, 1027, 990, 1035, 1015, 1002, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1004, 1017, 1092, 1037, 1104, 1038, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 988, 1027, 1001, 990, 1016, 1014, 1035, 5, 4, 0, 0, 0, 0, 0, 0, 1066, 1036, 999, 1004, 1024, 985, 1044, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1024, 1027, 988, 1085, 977, 1076, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1003, 1012, 1045, 1067, 1034, 1029, 1034, 23, 10, 0, 0, 0, 0, 0, 0, 1020, 1026, 964, 1049, 1016, 1052, 1004, 1025, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8135, 8318, 8102, 8157, 8220, 8287, 8159] avg=8192.0000 max=8318 max_rank=2 min=8102 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8192, 8320, 8320, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8448, 8192, 8064, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8448, 8192, 8320, 8064, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8064, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[998, 1091, 959, 1056, 985, 1070, 1041, 1031, 980, 1022, 1036, 1080, 1078, 970, 1055, 1009, 1001, 1035, 1032, 1012, 1008, 1022, 1049, 1040, 1032, 1038, 1067, 1032, 1024, 1007, 1014, 1022, 972, 1034, 1054, 1037, 1031, 1014, 1032, 1019, 1001, 1003, 1041, 1068, 997, 1012, 1015, 1043, 1038, 1037, 1066, 1006, 1020, 986, 1010, 1051, 1013, 1021, 1037, 991, 1041, 994, 974, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[997, 991, 997, 992, 1022, 1029, 1012, 1037, 1035, 1016, 1026, 1051, 1006, 1003, 1049, 1016, 999, 1104, 1007, 1039, 1044, 1011, 1043, 1017, 1037, 1037, 1031, 1012, 1007, 1030, 1051, 1037, 1060, 1022, 1009, 1024, 1044, 1044, 1010, 1036, 1011, 1028, 1025, 987, 1015, 1004, 981, 1039, 1038, 993, 1076, 1078, 1070, 993, 1010, 965, 1001, 982, 1027, 1064, 992, 1066, 1018, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1028, 1003, 1067, 1005, 984, 1071, 1044, 1030, 1066, 1046, 1001, 1016, 1040, 1030, 1038, 1021, 1053, 1038, 1015, 979, 977, 1061, 1035, 1050, 1048, 997, 1029, 1054, 1037, 1032, 1036, 1013, 1048, 1028, 1041, 1003, 1024, 993, 998, 1059, 1021, 1021, 1017, 1020, 1023, 986, 1049, 1058, 991, 1028, 1018, 1023, 1008, 1033, 1059, 1027, 1033, 985, 994, 955, 1049, 982, 1003, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1009, 1006, 990, 1030, 1021, 1024, 973, 1002, 1063, 979, 1012, 1036, 1021, 1045, 1010, 1039, 1017, 1007, 1012, 1009, 1062, 1034, 1045, 971, 992, 1061, 1060, 973, 1046, 1028, 991, 1041, 1011, 1045, 1043, 1064, 1002, 1062, 1022, 1009, 1021, 1020, 1067, 1043, 1030, 1013, 988, 997, 1036, 1077, 1029, 1024, 1028, 1013, 1030, 1009, 980, 1006, 1101, 1038, 1001, 1024, 1032, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1048, 994, 1034, 1017, 1044, 1053, 1020, 992, 1030, 1016, 1056, 1008, 1036, 1000, 1005, 1027, 1016, 1008, 1027, 1012, 997, 1060, 1066, 1020, 1034, 1003, 1070, 1004, 1051, 1015, 998, 997, 1052, 1069, 1047, 991, 1018, 1000, 1049, 1053, 1054, 974, 976, 1054, 1038, 1016, 1031, 1008, 1032, 997, 984, 1048, 1006, 1009, 1037, 1047, 1023, 1070, 974, 1005, 1045, 990, 1026, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1053, 984, 1038, 1061, 1042, 1016, 1040, 998, 987, 1022, 1045, 1041, 1034, 1083, 1043, 997, 1056, 1038, 1012, 1022, 998, 1028, 1027, 1063, 989, 997, 1003, 1020, 1018, 1008, 1033, 1055, 992, 1008, 1014, 1016, 1042, 1025, 1060, 1013, 1034, 967, 1022, 1074, 1030, 998, 999, 992, 1021, 987, 1053, 1013, 1039, 1007, 1020, 1067, 1062, 988, 1040, 1064, 1006, 1017, 1002, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1016, 1055, 1038, 977, 1010, 1002, 1078, 1017, 1007, 1044, 1036, 1019, 1061, 1020, 992, 1042, 1018, 1047, 1056, 987, 1040, 1006, 990, 1060, 1051, 1012, 1039, 1002, 1027, 1044, 1014, 1030, 1037, 1028, 993, 965, 1033, 1004, 984, 988, 1015, 993, 1024, 1056, 993, 1039, 1061, 1046, 1021, 1048, 1050, 1001, 1013, 1005, 1039, 998, 1055, 1052, 1004, 1063, 1028, 1036, 1064, 963] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1024, 1047, 987, 1023, 1010, 1091, 1049, 1040, 1047, 991, 1041, 1026, 1047, 1054, 1043, 1014, 992, 978, 998, 1025, 991, 1067, 1041, 1055, 1041, 1055, 959, 970, 1003, 1007, 992, 970, 1043, 974, 991, 1018, 1004, 1045, 1024, 1022, 1008, 1050, 1031, 1039, 1006, 1040, 1036, 1028, 986, 1021, 1010, 995, 1063, 1048, 1081, 1030, 1055, 1006, 1036, 1019, 1064, 1085, 1000, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1028, 1003, 1067, 1005, 984, 1071, 1044, 1030, 5, 0, 0, 0, 0, 0, 0, 0, 1066, 1046, 1001, 1016, 1007, 1030, 1038, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1038, 1015, 979, 977, 1061, 1025, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 997, 1029, 1054, 1037, 1032, 1036, 1013, 12, 1, 0, 0, 0, 0, 0, 0, 1048, 1028, 1041, 1003, 1024, 993, 998, 1059, 4, 0, 0, 0, 0, 0, 0, 0, 1021, 1021, 1017, 1020, 1023, 986, 1049, 1058, 19, 0, 0, 0, 0, 0, 0, 0, 991, 1028, 1018, 1023, 1008, 1033, 1046, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 985, 994, 955, 1049, 982, 1003, 1015, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8225, 8198, 8259, 8198, 8214, 8174, 8031] avg=8192.0000 max=8259 max_rank=3 min=8031 min_rank=7 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32617 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[998, 1091, 959, 1056, 985, 1070, 1041, 1031, 5, 0, 0, 0, 0, 0, 0, 0, 980, 1022, 1036, 1080, 1044, 970, 1055, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1035, 1032, 1012, 1008, 1022, 1038, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1038, 1067, 1032, 1024, 1007, 1014, 1022, 12, 1, 0, 0, 0, 0, 0, 0, 972, 1034, 1054, 1037, 1031, 1014, 1032, 1019, 5, 0, 0, 0, 0, 0, 0, 0, 1001, 1003, 1041, 1068, 997, 1012, 1015, 1043, 20, 0, 0, 0, 0, 0, 0, 0, 1038, 1037, 1066, 1006, 1020, 986, 997, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1021, 1037, 991, 1041, 994, 974, 982, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8196, 8188, 8249, 8198, 8200, 8201, 8068] avg=8192.0000 max=8249 max_rank=3 min=8068 min_rank=7 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1053, 984, 1038, 1061, 1042, 1016, 1040, 998, 5, 0, 0, 0, 0, 0, 0, 0, 987, 1022, 1045, 1041, 1001, 1083, 1043, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1038, 1012, 1022, 998, 1028, 1017, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 989, 997, 1003, 1020, 1018, 1008, 1033, 1055, 12, 1, 0, 0, 0, 0, 0, 0, 992, 1008, 1014, 1016, 1042, 1025, 1060, 1013, 4, 0, 0, 0, 0, 0, 0, 0, 1034, 967, 1022, 1074, 1030, 998, 999, 992, 19, 0, 0, 0, 0, 0, 0, 0, 1021, 987, 1053, 1013, 1039, 1007, 1007, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 988, 1040, 1064, 1006, 1017, 1002, 1013, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8219, 8234, 8136, 8174, 8135, 8194, 8207] avg=8192.0000 max=8237 max_rank=0 min=8135 min_rank=5 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1009, 1006, 990, 1030, 1021, 1024, 973, 1002, 5, 0, 0, 0, 0, 0, 0, 0, 1063, 979, 1012, 1036, 989, 1045, 1010, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1007, 1012, 1009, 1062, 1034, 1034, 971, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1061, 1060, 973, 1046, 1028, 991, 1041, 12, 1, 0, 0, 0, 0, 0, 0, 1011, 1045, 1043, 1064, 1002, 1062, 1022, 1009, 5, 0, 0, 0, 0, 0, 0, 0, 1021, 1020, 1067, 1043, 1030, 1013, 988, 997, 19, 0, 0, 0, 0, 0, 0, 0, 1036, 1077, 1029, 1024, 1028, 1013, 1017, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1006, 1101, 1038, 1001, 1024, 1032, 1062, 13, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8060, 8173, 8146, 8205, 8263, 8198, 8233, 8258] avg=8192.0000 max=8263 max_rank=4 min=8060 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32952 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[997, 991, 997, 992, 1022, 1029, 1012, 1037, 5, 0, 0, 0, 0, 0, 0, 0, 1035, 1016, 1026, 1051, 974, 1003, 1049, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1104, 1007, 1039, 1044, 1011, 1032, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1037, 1031, 1012, 1007, 1030, 1051, 1037, 12, 1, 0, 0, 0, 0, 0, 0, 1060, 1022, 1009, 1024, 1044, 1044, 1010, 1036, 5, 0, 0, 0, 0, 0, 0, 0, 1011, 1028, 1025, 987, 1015, 1004, 981, 1039, 19, 0, 0, 0, 0, 0, 0, 0, 1038, 993, 1076, 1078, 1070, 993, 997, 965, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 982, 1027, 1064, 992, 1066, 1018, 1039, 13, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8170, 8253, 8255, 8254, 8109, 8210, 8203] avg=8192.0000 max=8255 max_rank=3 min=8082 min_rank=0 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1048, 994, 1034, 1017, 1044, 1053, 1020, 992, 5, 0, 0, 0, 0, 0, 0, 0, 1030, 1016, 1056, 1008, 1003, 1000, 1005, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1008, 1027, 1012, 997, 1060, 1055, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1003, 1070, 1004, 1051, 1015, 998, 997, 12, 1, 0, 0, 0, 0, 0, 0, 1052, 1069, 1047, 991, 1018, 1000, 1049, 1053, 5, 0, 0, 0, 0, 0, 0, 0, 1054, 974, 976, 1054, 1038, 1016, 1031, 1008, 19, 0, 0, 0, 0, 0, 0, 0, 1032, 997, 984, 1048, 1006, 1009, 1024, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1070, 974, 1005, 1045, 990, 1026, 1055, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8145, 8195, 8185, 8284, 8170, 8147, 8203] avg=8192.0000 max=8284 max_rank=4 min=8145 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1016, 1055, 1038, 977, 1010, 1002, 1078, 1017, 8, 0, 0, 0, 0, 0, 0, 0, 1007, 1044, 1036, 1019, 1019, 1020, 992, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1047, 1056, 987, 1040, 1006, 972, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1012, 1039, 1002, 1027, 1044, 1014, 1030, 15, 3, 0, 0, 0, 0, 0, 0, 1037, 1028, 993, 965, 1033, 1004, 984, 988, 7, 0, 0, 0, 0, 0, 0, 0, 1015, 993, 1024, 1056, 993, 1039, 1061, 1046, 25, 0, 0, 0, 0, 0, 0, 0, 1021, 1048, 1050, 1001, 1013, 1005, 1020, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1052, 1004, 1063, 1028, 1036, 1064, 963, 17, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8179, 8186, 8237, 8039, 8252, 8156, 8286] avg=8192.0000 max=8286 max_rank=7 min=8039 min_rank=4 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1024, 1047, 987, 1023, 1010, 1091, 1049, 1040, 5, 0, 0, 0, 0, 0, 0, 0, 1047, 991, 1041, 1026, 1013, 1054, 1043, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 992, 978, 998, 1025, 991, 1067, 1030, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1055, 959, 970, 1003, 1007, 992, 970, 12, 1, 0, 0, 0, 0, 0, 0, 1043, 974, 991, 1018, 1004, 1045, 1024, 1022, 5, 0, 0, 0, 0, 0, 0, 0, 1008, 1050, 1031, 1039, 1006, 1040, 1036, 1028, 20, 0, 0, 0, 0, 0, 0, 0, 986, 1021, 1010, 995, 1063, 1048, 1068, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1006, 1036, 1019, 1064, 1085, 1000, 1000, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8229, 8136, 8010, 8126, 8258, 8221, 8280] avg=8192.0000 max=8280 max_rank=7 min=8010 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8448, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8320, 8448, 8192, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1036, 960, 1005, 1036, 1058, 1004, 1051, 974, 1077, 975, 1027, 1038, 1016, 1009, 1050, 1061, 1041, 998, 1046, 1061, 1050, 1034, 1010, 1009, 1056, 1020, 1011, 1035, 1026, 998, 1037, 1006, 1014, 1051, 1057, 1017, 1074, 1013, 1009, 998, 1030, 1030, 1005, 1037, 983, 985, 975, 987, 1002, 1033, 1022, 1009, 1096, 975, 1045, 1058, 1006, 1088, 1003, 1026, 1045, 1034, 1004, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1043, 1008, 1023, 1030, 1018, 1013, 982, 1023, 1026, 973, 1033, 1024, 1042, 988, 1008, 1070, 1022, 1067, 1030, 1034, 1034, 1035, 1056, 1007, 1007, 1021, 1006, 1036, 1020, 990, 1042, 1082, 1012, 1053, 1059, 1044, 1016, 1071, 1042, 1006, 1049, 991, 1025, 1079, 1058, 1056, 1009, 1035, 1040, 989, 1036, 1032, 995, 998, 950, 990, 1028, 1030, 1023, 1016, 998, 1001, 1006, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1082, 1016, 1057, 1067, 1016, 985, 1021, 1046, 977, 946, 997, 1046, 1043, 1001, 1015, 982, 1006, 1066, 1015, 983, 1024, 1032, 1051, 1023, 1002, 1046, 1090, 975, 1021, 989, 1066, 1001, 1093, 1079, 1003, 1015, 1035, 1000, 990, 1055, 1069, 1064, 1021, 970, 1035, 1015, 1048, 1043, 976, 1016, 1047, 1070, 1076, 1029, 1058, 1037, 992, 1007, 986, 1005, 1011, 1031, 1003, 970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1014, 1025, 1019, 1035, 1016, 1031, 1032, 1061, 1035, 998, 1043, 1044, 1011, 1016, 1034, 1045, 1012, 1035, 1023, 1025, 1005, 1055, 1029, 991, 1025, 1033, 984, 972, 1020, 1038, 1056, 1051, 1023, 1013, 983, 1059, 1001, 961, 1067, 1061, 1060, 994, 1056, 998, 1004, 1056, 1010, 1029, 1012, 1021, 1017, 1053, 996, 1018, 1016, 1084, 1055, 1014, 1037, 1000, 992, 1003, 992, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1031, 991, 1045, 1047, 1049, 1054, 1026, 1022, 1029, 993, 994, 1033, 1049, 988, 987, 999, 1021, 995, 1073, 1042, 1039, 1029, 1007, 977, 1030, 1058, 1069, 1035, 1013, 1035, 1061, 1018, 1020, 1014, 1046, 1042, 1027, 1077, 1023, 1058, 1012, 1051, 1000, 1005, 1040, 1007, 1008, 1032, 1035, 984, 1085, 1015, 1034, 1022, 977, 981, 1024, 1015, 1039, 1020, 1023, 966, 1005, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[991, 1033, 1005, 1062, 1023, 1016, 1036, 1014, 1019, 1014, 1055, 1057, 1035, 1064, 1012, 998, 1038, 1010, 1024, 1006, 964, 982, 1023, 965, 1001, 1000, 1037, 1008, 1041, 988, 1002, 1033, 963, 1047, 1053, 1013, 1060, 1052, 1025, 1004, 1008, 1047, 1015, 1044, 1048, 1043, 1069, 1030, 1008, 1047, 1023, 1019, 1009, 1012, 1045, 1037, 1024, 1028, 1035, 1039, 994, 1046, 1040, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1028, 1044, 1043, 1032, 1088, 1046, 1020, 1061, 1056, 1044, 998, 957, 1049, 1011, 1031, 971, 1046, 1012, 1011, 1046, 978, 1023, 1051, 1039, 1033, 1045, 1052, 1067, 1004, 1051, 1012, 1126, 1076, 999, 1014, 1010, 982, 1025, 990, 1004, 955, 1009, 1010, 1009, 972, 1024, 1016, 1026, 1003, 1020, 1057, 978, 1035, 1058, 1042, 1046, 1001, 1027, 1035, 989, 1024, 987, 1039, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1017, 995, 1016, 980, 989, 1052, 1013, 1010, 973, 1033, 1008, 1049, 1014, 1007, 993, 1033, 1029, 1025, 1010, 1029, 1072, 1072, 1040, 1029, 1006, 1090, 1059, 1003, 1059, 1024, 982, 1006, 986, 1039, 1021, 1052, 1016, 1055, 1019, 1069, 1005, 1016, 993, 1023, 1019, 1026, 1051, 1057, 1018, 995, 1085, 974, 999, 1040, 974, 1059, 999, 1048, 1022, 972, 1002, 1071, 1040, 1074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1036, 960, 1005, 1021, 1058, 1004, 1051, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 975, 1027, 1038, 1016, 1009, 1050, 1061, 25, 15, 0, 0, 0, 0, 0, 0, 1041, 998, 1046, 1061, 1050, 1034, 1002, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1020, 1011, 1035, 1026, 998, 1037, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1011, 1057, 1017, 1074, 1013, 1009, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1030, 1005, 1037, 983, 985, 975, 987, 6, 0, 0, 0, 0, 0, 0, 0, 1002, 1033, 1022, 1009, 1096, 975, 1045, 1058, 2, 0, 0, 0, 0, 0, 0, 0, 1006, 1088, 1003, 1026, 1045, 1034, 1004, 1010, 40, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8293, 8241, 8156, 8193, 8038, 8242, 8264] avg=8192.0000 max=8293 max_rank=1 min=8038 min_rank=5 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1043, 1008, 1023, 1015, 1018, 1013, 982, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 973, 1033, 1024, 1042, 988, 1008, 1070, 26, 15, 0, 0, 0, 0, 0, 0, 1022, 1067, 1030, 1034, 1034, 1035, 1046, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1021, 1006, 1036, 1020, 990, 1042, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1013, 1059, 1044, 1016, 1071, 1042, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 991, 1025, 1079, 1058, 1056, 1009, 1035, 7, 0, 0, 0, 0, 0, 0, 0, 1040, 989, 1036, 1032, 995, 998, 950, 990, 3, 0, 0, 0, 0, 0, 0, 0, 1028, 1030, 1023, 1016, 998, 1001, 1006, 1006, 40, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8205, 8275, 8170, 8263, 8309, 8033, 8156] avg=8192.0000 max=8309 max_rank=5 min=8033 min_rank=6 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1082, 1016, 1057, 1051, 1016, 985, 1021, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 977, 946, 997, 1046, 1043, 1001, 1015, 982, 24, 16, 0, 0, 0, 0, 0, 0, 1006, 1066, 1015, 983, 1024, 1032, 1041, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1046, 1090, 975, 1021, 989, 1066, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 1038, 1003, 1015, 1035, 1000, 990, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1064, 1021, 970, 1035, 1015, 1048, 1043, 7, 0, 0, 0, 0, 0, 0, 0, 976, 1016, 1047, 1070, 1076, 1029, 1058, 1037, 3, 0, 0, 0, 0, 0, 0, 0, 992, 1007, 986, 1005, 1011, 1031, 1003, 970, 41, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8047, 8190, 8158, 8229, 8272, 8312, 8054] avg=8192.0000 max=8312 max_rank=6 min=8047 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1014, 1025, 1019, 1020, 1016, 1031, 1032, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 998, 1043, 1044, 1011, 1016, 1034, 1045, 26, 15, 0, 0, 0, 0, 0, 0, 1012, 1035, 1023, 1025, 1005, 1055, 1021, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1033, 984, 972, 1020, 1038, 1056, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 975, 983, 1059, 1001, 961, 1067, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 994, 1056, 998, 1004, 1056, 1010, 1029, 6, 0, 0, 0, 0, 0, 0, 0, 1012, 1021, 1017, 1053, 996, 1018, 1016, 1084, 2, 0, 0, 0, 0, 0, 0, 0, 1055, 1014, 1037, 1000, 992, 1003, 992, 1038, 38, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8267, 8167, 8145, 8130, 8213, 8219, 8177] avg=8192.0000 max=8267 max_rank=1 min=8130 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1031, 991, 1045, 1032, 1049, 1054, 1026, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 993, 994, 1033, 1049, 988, 987, 999, 25, 15, 0, 0, 0, 0, 0, 0, 1021, 995, 1073, 1042, 1039, 1029, 999, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1058, 1069, 1035, 1013, 1035, 1061, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 976, 1046, 1042, 1027, 1077, 1023, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1051, 1000, 1005, 1040, 1007, 1008, 1032, 6, 0, 0, 0, 0, 0, 0, 0, 1035, 984, 1085, 1015, 1034, 1022, 977, 981, 2, 0, 0, 0, 0, 0, 0, 0, 1024, 1015, 1039, 1020, 1023, 966, 1005, 1010, 38, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8112, 8175, 8286, 8269, 8161, 8135, 8148] avg=8192.0000 max=8286 max_rank=3 min=8112 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[991, 1033, 1005, 1037, 1023, 1016, 1036, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1014, 1055, 1057, 1035, 1064, 1012, 998, 28, 21, 0, 0, 0, 0, 0, 0, 1038, 1010, 1024, 1006, 964, 982, 1007, 965, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1000, 1037, 1008, 1041, 988, 1002, 995, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1002, 1053, 1013, 1060, 1052, 1025, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1047, 1015, 1044, 1048, 1043, 1069, 1030, 10, 4, 0, 0, 0, 0, 0, 0, 1008, 1047, 1023, 1019, 1009, 1012, 1045, 1037, 6, 0, 0, 0, 0, 0, 0, 0, 1024, 1028, 1035, 1039, 994, 1046, 1040, 1053, 45, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8303, 7996, 8072, 8172, 8318, 8206, 8314] avg=8192.0000 max=8318 max_rank=5 min=7996 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=33010 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1028, 1044, 1043, 1017, 1088, 1046, 1020, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1044, 998, 957, 1049, 1011, 1031, 971, 28, 15, 0, 0, 0, 0, 0, 0, 1046, 1012, 1011, 1046, 978, 1023, 1041, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1045, 1052, 1067, 1004, 1051, 1012, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 961, 1014, 1010, 982, 1025, 990, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 955, 1009, 1010, 1009, 972, 1024, 1016, 1026, 7, 0, 0, 0, 0, 0, 0, 0, 1003, 1020, 1057, 978, 1035, 1058, 1042, 1046, 3, 0, 0, 0, 0, 0, 0, 0, 1001, 1027, 1035, 989, 1024, 987, 1039, 999, 38, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8347, 8160, 8196, 8353, 8062, 8028, 8242, 8148] avg=8192.0000 max=8353 max_rank=3 min=8028 min_rank=5 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1017, 995, 1016, 966, 989, 1052, 1013, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1033, 1008, 1049, 1014, 1007, 993, 1033, 25, 14, 0, 0, 0, 0, 0, 0, 1029, 1025, 1010, 1029, 1072, 1072, 1030, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1090, 1059, 1003, 1059, 1024, 982, 973, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1000, 1021, 1052, 1016, 1055, 1019, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1016, 993, 1023, 1019, 1026, 1051, 1057, 7, 0, 0, 0, 0, 0, 0, 0, 1018, 995, 1085, 974, 999, 1040, 974, 1059, 3, 0, 0, 0, 0, 0, 0, 0, 999, 1048, 1022, 972, 1002, 1071, 1040, 1074, 39, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8058, 8149, 8296, 8196, 8218, 8197, 8147, 8275] avg=8192.0000 max=8296 max_rank=2 min=8058 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8320, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8448, 8192, 8320, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8320, 8320, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1026, 1019, 1042, 1041, 1084, 1117, 1066, 1003, 1030, 1034, 1000, 991, 1082, 1054, 1055, 1004, 1104, 1048, 1006, 1022, 992, 998, 965, 1017, 1048, 1055, 1060, 1016, 959, 998, 1032, 994, 991, 1008, 982, 1015, 1022, 986, 1049, 1016, 995, 1068, 1025, 1036, 1043, 984, 1000, 1025, 943, 1070, 1010, 1082, 984, 1023, 1005, 1029, 1001, 1010, 1024, 1005, 1064, 1012, 1007, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1008, 1070, 992, 1031, 997, 1085, 976, 980, 998, 1049, 1066, 993, 1031, 996, 1016, 968, 1010, 1003, 1004, 1027, 1033, 1018, 1039, 1038, 1052, 1027, 1003, 982, 1039, 1008, 1053, 1031, 1023, 970, 1020, 1054, 978, 1027, 1028, 1033, 1015, 1029, 1066, 1028, 1046, 1020, 1045, 1038, 1060, 1051, 1036, 1035, 1055, 980, 1043, 1067, 979, 1038, 1001, 981, 1033, 1051, 1093, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1013, 1046, 1040, 1008, 1001, 1031, 1007, 1046, 1010, 1016, 1042, 1001, 978, 1028, 1030, 1076, 1045, 1009, 1023, 964, 1039, 1032, 1044, 1042, 1015, 1017, 1052, 1046, 1044, 991, 1047, 1013, 1017, 1018, 978, 1049, 1025, 1002, 1035, 1061, 1005, 983, 1088, 999, 1003, 1035, 1032, 1036, 1015, 1032, 1032, 1017, 1036, 1042, 1016, 1061, 1042, 1023, 1013, 1003, 1030, 1006, 998, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1069, 1008, 1050, 1017, 1022, 1007, 994, 1001, 1044, 1096, 1041, 1030, 1041, 1085, 1050, 1027, 1041, 1018, 980, 1015, 990, 1004, 1036, 1034, 1014, 1042, 1036, 981, 1051, 1031, 1040, 1003, 1058, 1010, 1036, 1034, 1041, 1048, 1015, 1062, 973, 1022, 1008, 1011, 1027, 1007, 987, 1019, 984, 1008, 1021, 956, 1055, 1059, 1011, 998, 1024, 1035, 1039, 981, 1042, 1054, 1010, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1054, 1011, 997, 1081, 1023, 1059, 1012, 1045, 972, 972, 977, 1053, 1031, 1031, 1055, 1020, 991, 970, 1035, 1006, 1022, 1014, 1045, 1011, 987, 1004, 1054, 1024, 1033, 1015, 992, 1071, 1064, 1018, 1033, 993, 1064, 999, 1069, 1011, 993, 1056, 1038, 1003, 986, 1047, 1073, 1019, 1038, 1037, 1038, 1020, 1006, 1035, 1019, 1006, 998, 1066, 1017, 1039, 1009, 1014, 1020, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1026, 1042, 1003, 956, 1008, 1029, 1007, 988, 1019, 995, 1044, 1035, 1052, 1054, 994, 1016, 1044, 1082, 1051, 1033, 996, 1031, 988, 1032, 1033, 1041, 1024, 1001, 969, 1024, 1019, 1018, 1048, 1042, 993, 1001, 992, 1046, 996, 1025, 1000, 1030, 1020, 1052, 985, 1059, 1065, 1032, 1044, 1024, 976, 1050, 1021, 1008, 1065, 1033, 1004, 1096, 996, 1083, 999, 1020, 1034, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1033, 1057, 1041, 1004, 982, 962, 1024, 1019, 992, 1021, 1035, 1040, 1028, 998, 1017, 1006, 1006, 1063, 1004, 1011, 1031, 1053, 1063, 1062, 1016, 1052, 1030, 1017, 1019, 1034, 1021, 1044, 1007, 1000, 1022, 1022, 1031, 1035, 1030, 1041, 1033, 991, 1038, 1039, 1019, 1015, 1062, 1101, 1043, 1024, 1055, 1054, 990, 1046, 1018, 998, 1005, 995, 1008, 932, 1027, 1007, 1010, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1038, 1036, 998, 992, 1042, 1041, 1031, 1032, 1056, 1022, 994, 1002, 1002, 1078, 1008, 1007, 1049, 1044, 1021, 975, 1032, 1020, 1034, 1018, 1025, 1035, 1031, 1002, 1015, 1044, 1076, 998, 996, 999, 1076, 1028, 1003, 1013, 1033, 1039, 1043, 1079, 983, 1021, 1018, 1007, 1049, 1037, 1010, 1035, 1032, 1030, 1054, 1006, 1005, 1030, 1038, 1036, 965, 1036, 989, 1043, 969, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1008, 1070, 992, 1031, 997, 1081, 976, 980, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1049, 1066, 993, 1031, 991, 1016, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1003, 1004, 1027, 1033, 1018, 1039, 1038, 4, 1, 1, 0, 0, 0, 0, 0, 1052, 1027, 1002, 982, 1039, 1008, 1053, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 970, 1020, 1054, 978, 1027, 1028, 1033, 4, 4, 0, 0, 0, 0, 0, 0, 1015, 1029, 1066, 1028, 1046, 1020, 1026, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1036, 1036, 1035, 1055, 980, 1043, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1038, 1001, 981, 1033, 1051, 1093, 990, 19, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8112, 8178, 8194, 8141, 8268, 8312, 8196] avg=8192.0000 max=8312 max_rank=6 min=8112 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1026, 1019, 1042, 1041, 1084, 1113, 1066, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1034, 1000, 991, 1082, 1049, 1055, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 1048, 1006, 1022, 992, 998, 965, 1017, 4, 1, 1, 0, 0, 0, 0, 0, 1048, 1055, 1059, 1016, 959, 998, 1032, 994, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1008, 982, 1015, 1022, 986, 1049, 1016, 4, 4, 0, 0, 0, 0, 0, 0, 995, 1068, 1025, 1036, 1043, 984, 982, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 943, 1054, 1010, 1082, 984, 1023, 1005, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1010, 1024, 1005, 1064, 1012, 1007, 1060, 18, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8394, 8245, 8158, 8161, 8077, 8158, 8130, 8213] avg=8192.0000 max=8394 max_rank=0 min=8077 min_rank=4 imbalance_ratio=1.024658 max_load_reduction=57142 max_load_reduction_pct=0.871918 moved_to_echo=32578 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1013, 1046, 1040, 1008, 1001, 1027, 1007, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1016, 1042, 1001, 978, 1023, 1030, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1009, 1023, 964, 1039, 1032, 1044, 1042, 4, 1, 1, 0, 0, 0, 0, 0, 1015, 1017, 1051, 1046, 1044, 991, 1047, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1018, 978, 1049, 1025, 1002, 1035, 1061, 4, 4, 0, 0, 0, 0, 0, 0, 1005, 983, 1088, 999, 1003, 1035, 1013, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1017, 1032, 1017, 1036, 1042, 1016, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1023, 1013, 1003, 1030, 1006, 998, 1008, 19, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8176, 8204, 8224, 8193, 8162, 8236, 8153] avg=8192.0000 max=8236 max_rank=6 min=8153 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1069, 1008, 1050, 1017, 1022, 1003, 994, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1096, 1041, 1030, 1041, 1079, 1050, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1018, 980, 1015, 990, 1004, 1036, 1034, 4, 1, 1, 0, 0, 0, 0, 0, 1014, 1042, 1035, 981, 1051, 1031, 1040, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1010, 1036, 1034, 1041, 1048, 1015, 1062, 5, 4, 0, 0, 0, 0, 0, 0, 973, 1022, 1008, 1011, 1027, 1007, 969, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 984, 993, 1021, 956, 1055, 1059, 1011, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1035, 1039, 981, 1042, 1054, 1010, 1003, 18, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8408, 8124, 8197, 8313, 8036, 8077, 8217] avg=8192.0000 max=8408 max_rank=1 min=8036 min_rank=5 imbalance_ratio=1.026367 max_load_reduction=57128 max_load_reduction_pct=0.871704 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1054, 1011, 997, 1081, 1023, 1055, 1012, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 972, 972, 977, 1053, 1031, 1026, 1055, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 991, 970, 1035, 1006, 1022, 1014, 1045, 1011, 4, 1, 1, 0, 0, 0, 0, 0, 987, 1004, 1053, 1024, 1033, 1015, 992, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1018, 1033, 993, 1064, 999, 1069, 1011, 4, 4, 0, 0, 0, 0, 0, 0, 993, 1056, 1038, 1003, 986, 1047, 1053, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1022, 1038, 1020, 1006, 1035, 1019, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1066, 1017, 1039, 1009, 1014, 1020, 1041, 20, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8106, 8100, 8179, 8259, 8195, 8184, 8235] avg=8192.0000 max=8278 max_rank=0 min=8100 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1026, 1042, 1003, 956, 1008, 1025, 1007, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 995, 1044, 1035, 1052, 1049, 994, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1082, 1051, 1033, 996, 1031, 988, 1032, 4, 1, 1, 0, 0, 0, 0, 0, 1033, 1041, 1023, 1001, 969, 1024, 1019, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1042, 993, 1001, 992, 1046, 996, 1025, 4, 4, 0, 0, 0, 0, 0, 0, 1000, 1030, 1020, 1052, 985, 1059, 1046, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1009, 976, 1050, 1021, 1008, 1065, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1096, 996, 1083, 999, 1020, 1034, 1043, 19, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8055, 8204, 8263, 8128, 8151, 8224, 8206, 8305] avg=8192.0000 max=8305 max_rank=7 min=8055 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32886 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1033, 1057, 1041, 1004, 982, 956, 1024, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1021, 1035, 1040, 1028, 983, 1017, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1063, 1004, 1011, 1031, 1053, 1063, 1062, 6, 6, 5, 0, 0, 0, 0, 0, 1016, 1052, 1025, 1017, 1019, 1034, 1021, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1000, 1022, 1022, 1031, 1035, 1030, 1041, 9, 9, 0, 0, 0, 0, 0, 0, 1033, 991, 1038, 1039, 1019, 1015, 1039, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1000, 1055, 1054, 990, 1046, 1018, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 995, 1008, 932, 1027, 1007, 1010, 1053, 23, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8122, 8310, 8228, 8206, 8275, 8204, 8075] avg=8192.0000 max=8310 max_rank=2 min=8075 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1038, 1036, 998, 992, 1042, 1037, 1031, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1022, 994, 1002, 1002, 1072, 1008, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1044, 1021, 975, 1032, 1020, 1034, 1018, 4, 1, 1, 0, 0, 0, 0, 0, 1025, 1035, 1030, 1002, 1015, 1044, 1076, 998, 0, 0, 0, 0, 0, 0, 0, 0, 996, 999, 1076, 1028, 1003, 1013, 1033, 1039, 5, 4, 0, 0, 0, 0, 0, 0, 1043, 1079, 983, 1021, 1018, 1007, 1030, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1020, 1032, 1030, 1054, 1006, 1005, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1036, 965, 1036, 989, 1043, 969, 1036, 19, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8163, 8199, 8225, 8196, 8218, 8187, 8142] avg=8192.0000 max=8225 max_rank=3 min=8142 min_rank=7 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8064, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8064, 8192, 8192, 8320, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8064, 8064, 8320, 8320, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True + [2026-08-30 05:02:11] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 679.9 | throughput per GPU (TFLOP/s/GPU): 490.4 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.310929E+01 | loss scale: 1.0 | grad norm: 39.939 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1012, 1042, 1019, 984, 1057, 1063, 1012, 1014, 1063, 1089, 1047, 1032, 1028, 981, 1031, 1003, 1018, 989, 1017, 1040, 1021, 1036, 1060, 1038, 1057, 1032, 1021, 1012, 1017, 1055, 1030, 979, 1004, 1044, 1002, 984, 1054, 1057, 1047, 1031, 1000, 1016, 1007, 1032, 1008, 1059, 1017, 1015, 1038, 1016, 973, 1038, 973, 1003, 1016, 974, 1014, 1015, 1011, 1034, 1054, 1031, 1042, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1014, 1056, 997, 1063, 1012, 1010, 1005, 965, 1022, 1031, 1051, 1055, 1073, 963, 1020, 1062, 1038, 1063, 1057, 1054, 1015, 975, 987, 1003, 1001, 981, 978, 997, 1009, 1017, 1024, 1013, 978, 1009, 1036, 1052, 987, 1006, 1069, 1001, 1042, 1022, 1050, 1056, 1051, 1042, 958, 972, 1019, 1059, 1144, 968, 1028, 1013, 1014, 1047, 1012, 1063, 1056, 1031, 1008, 1041, 1047, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1003, 1006, 1003, 1009, 1044, 1045, 1017, 996, 1033, 1025, 1040, 1012, 1015, 1070, 1021, 1034, 995, 1065, 1005, 1036, 993, 1020, 1031, 1047, 1081, 1028, 1022, 956, 1019, 1019, 971, 1046, 1005, 980, 956, 1015, 1059, 1058, 1012, 1015, 1072, 1078, 988, 1027, 1050, 1041, 1066, 1031, 997, 1049, 1046, 1026, 1042, 976, 993, 1007, 1052, 1051, 985, 983, 1045, 1035, 1029, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1072, 990, 978, 1049, 1035, 988, 1036, 1096, 983, 1018, 1041, 1020, 968, 1048, 1005, 987, 1045, 984, 1012, 1036, 1052, 1053, 1067, 1030, 999, 1046, 997, 993, 1029, 1070, 1027, 996, 1035, 1000, 1057, 1046, 1019, 1041, 1054, 1009, 1038, 1079, 1003, 1027, 964, 1036, 1086, 1049, 1036, 1028, 990, 1019, 943, 1026, 1035, 1059, 986, 987, 1029, 987, 1030, 1054, 1029, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1029, 1016, 1031, 1035, 1031, 1036, 963, 1021, 999, 1037, 1044, 1023, 1064, 984, 1090, 982, 1036, 1010, 1025, 981, 987, 1051, 1015, 1023, 1035, 1023, 1055, 1068, 1020, 1019, 1035, 950, 998, 1065, 1020, 1041, 1021, 1039, 1015, 1034, 1029, 1029, 998, 1046, 1004, 1028, 1003, 1010, 1048, 1052, 1023, 979, 992, 1040, 1003, 1016, 964, 1073, 1053, 1010, 1053, 1026, 1026, 1080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[998, 1024, 980, 1025, 1023, 971, 983, 1042, 1040, 1017, 1041, 997, 997, 1030, 1040, 1003, 1001, 1004, 1015, 1026, 1021, 1008, 982, 1004, 1034, 1048, 999, 981, 1058, 1041, 988, 1022, 1029, 1055, 1033, 1082, 1049, 1010, 1020, 1055, 989, 965, 978, 966, 1044, 1074, 1040, 1048, 1074, 1045, 1025, 1055, 1097, 1021, 1039, 1031, 1034, 1068, 1017, 1049, 1028, 1007, 1018, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1034, 1015, 1004, 1033, 1004, 1037, 996, 1053, 998, 989, 1117, 1001, 1031, 969, 1026, 1028, 985, 1007, 1037, 1045, 992, 1080, 1012, 1052, 1004, 1017, 1001, 1025, 988, 1023, 1046, 1048, 1035, 1027, 1022, 988, 973, 1060, 1043, 1102, 1015, 1012, 1070, 1045, 1053, 977, 1038, 982, 992, 1019, 1049, 1033, 986, 1007, 1021, 1066, 1036, 1024, 1053, 941, 1039, 1018, 1049, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[990, 982, 1022, 1035, 1025, 1071, 961, 983, 991, 1077, 1037, 1067, 1062, 1078, 991, 1042, 1035, 1000, 1077, 1066, 1019, 1022, 1045, 999, 982, 985, 999, 1016, 960, 996, 1039, 998, 1045, 998, 1020, 1046, 1031, 1065, 1027, 982, 1029, 1001, 1003, 1015, 1021, 1024, 1050, 1023, 1062, 1038, 1013, 1036, 1076, 1064, 1048, 1034, 1017, 1033, 1034, 991, 1017, 1021, 1006, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1012, 1042, 1019, 984, 1057, 1063, 1012, 1014, 23, 17, 7, 1, 0, 0, 0, 0, 1063, 1089, 1019, 1032, 1028, 981, 1031, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 989, 1017, 1039, 1021, 1036, 1060, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1032, 1021, 1012, 1017, 1055, 1030, 979, 31, 28, 3, 0, 0, 0, 0, 0, 1004, 1044, 1002, 984, 1054, 1031, 1047, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1016, 1007, 1032, 1008, 1052, 1017, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 999, 973, 1038, 973, 1003, 1016, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1015, 1011, 1034, 1054, 1031, 1042, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8246, 8218, 8265, 8197, 8147, 8014, 8198] avg=8192.0000 max=8265 max_rank=3 min=8014 min_rank=6 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1014, 1056, 997, 1063, 1012, 1010, 1005, 965, 22, 18, 6, 1, 0, 0, 0, 0, 1022, 1031, 1023, 1055, 1073, 963, 1020, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1063, 1057, 1053, 1015, 975, 987, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 981, 978, 997, 1009, 1017, 1024, 1013, 31, 28, 3, 0, 0, 0, 0, 0, 978, 1009, 1036, 1052, 987, 981, 1069, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1022, 1050, 1056, 1051, 1036, 958, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1041, 1144, 968, 1028, 1013, 1014, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1063, 1056, 1031, 1008, 1041, 1047, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8249, 8191, 8082, 8113, 8187, 8274, 8271] avg=8192.0000 max=8274 max_rank=6 min=8082 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1003, 1006, 1003, 1009, 1044, 1045, 1017, 996, 23, 18, 6, 1, 0, 0, 0, 0, 1033, 1025, 1012, 1012, 1015, 1070, 1021, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1065, 1005, 1035, 993, 1020, 1031, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1028, 1022, 956, 1019, 1019, 971, 1046, 32, 28, 3, 0, 0, 0, 0, 0, 1005, 980, 956, 1015, 1059, 1032, 1012, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1078, 988, 1027, 1050, 1035, 1066, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1031, 1046, 1026, 1042, 976, 993, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1051, 985, 983, 1045, 1035, 1029, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8222, 8191, 8205, 8074, 8347, 8118, 8208] avg=8192.0000 max=8347 max_rank=5 min=8074 min_rank=4 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1072, 990, 978, 1049, 1035, 988, 1036, 1096, 23, 17, 6, 1, 0, 0, 0, 0, 983, 1018, 1013, 1020, 968, 1048, 1005, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 984, 1012, 1035, 1052, 1053, 1067, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1046, 997, 993, 1029, 1070, 1027, 996, 30, 28, 3, 0, 0, 0, 0, 0, 1035, 1000, 1057, 1046, 1019, 1015, 1054, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1079, 1003, 1027, 964, 1030, 1086, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1011, 990, 1019, 943, 1026, 1035, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 986, 987, 1029, 987, 1030, 1054, 1029, 975, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8042, 8278, 8218, 8235, 8276, 8119, 8077] avg=8192.0000 max=8291 max_rank=0 min=8042 min_rank=1 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[998, 1024, 980, 1025, 1023, 971, 983, 1042, 25, 21, 12, 2, 0, 0, 0, 0, 1040, 1017, 1011, 997, 997, 1030, 1040, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1004, 1015, 1024, 1021, 1008, 982, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1048, 999, 981, 1058, 1041, 988, 1022, 35, 30, 10, 0, 0, 0, 0, 0, 1029, 1055, 1033, 1082, 1049, 975, 1020, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 989, 965, 978, 966, 1044, 1062, 1040, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1024, 1025, 1055, 1097, 1021, 1039, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1068, 1017, 1049, 1028, 1007, 1018, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8135, 8059, 8246, 8298, 8092, 8366, 8234] avg=8192.0000 max=8366 max_rank=6 min=8059 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32990 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1029, 1016, 1031, 1035, 1031, 1036, 963, 1021, 23, 18, 6, 0, 0, 0, 0, 0, 999, 1037, 1016, 1023, 1064, 984, 1090, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1010, 1025, 981, 987, 1051, 1015, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1023, 1055, 1068, 1020, 1019, 1035, 950, 32, 28, 3, 0, 0, 0, 0, 0, 998, 1065, 1020, 1041, 1021, 1013, 1015, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1029, 998, 1046, 1004, 1022, 1003, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1034, 1023, 979, 992, 1040, 1003, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 964, 1073, 1053, 1010, 1053, 1026, 1026, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8195, 8128, 8268, 8207, 8141, 8135, 8253] avg=8192.0000 max=8268 max_rank=3 min=8128 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1034, 1015, 1004, 1033, 1004, 1037, 996, 1053, 23, 17, 6, 1, 0, 0, 0, 0, 998, 989, 1087, 1001, 1031, 969, 1026, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1007, 1037, 1044, 992, 1080, 1012, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1017, 1001, 1025, 988, 1023, 1046, 1048, 32, 30, 3, 0, 0, 0, 0, 0, 1035, 1027, 1022, 988, 973, 1034, 1043, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1012, 1070, 1045, 1053, 971, 1038, 982, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1002, 1049, 1033, 986, 1007, 1021, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1024, 1053, 941, 1039, 1018, 1049, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8129, 8209, 8217, 8224, 8186, 8156, 8192] avg=8192.0000 max=8224 max_rank=4 min=8129 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[990, 982, 1022, 1035, 1025, 1071, 961, 983, 23, 17, 6, 1, 0, 0, 0, 0, 991, 1077, 1010, 1067, 1062, 1078, 991, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1000, 1077, 1065, 1019, 1022, 1045, 999, 0, 0, 0, 0, 0, 0, 0, 0, 982, 985, 999, 1016, 960, 996, 1039, 998, 30, 27, 3, 0, 0, 0, 0, 0, 1045, 998, 1020, 1046, 1031, 1039, 1027, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1001, 1003, 1015, 1021, 1018, 1050, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1021, 1013, 1036, 1076, 1064, 1048, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1033, 1034, 991, 1017, 1021, 1006, 984, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8318, 8262, 8035, 8188, 8160, 8354, 8103] avg=8192.0000 max=8354 max_rank=6 min=8035 min_rank=3 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8064, 8320, 8320, 8320, 8064, 8192, 256, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8064, 8192, 8320, 8192, 8064, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1012, 1013, 1039, 1012, 1004, 1043, 1034, 1046, 1056, 1006, 1041, 993, 992, 984, 1023, 1045, 981, 1055, 1001, 1023, 1008, 1001, 1036, 1015, 1012, 1041, 1023, 1013, 1058, 1057, 1005, 999, 1033, 1030, 1026, 1029, 1042, 1056, 1022, 983, 1014, 1049, 1051, 1060, 1022, 1000, 1015, 1021, 956, 1083, 1052, 1051, 1043, 984, 1072, 994, 1021, 1001, 1009, 1053, 1036, 1013, 1013, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1003, 1008, 979, 1023, 1043, 1003, 989, 1015, 1030, 1023, 1043, 1030, 1019, 986, 1028, 1083, 1011, 1006, 1042, 1009, 972, 1045, 1025, 1015, 1033, 1019, 970, 1016, 1016, 1014, 1038, 1081, 1007, 1041, 1024, 1037, 965, 1031, 1042, 1043, 1068, 1038, 1066, 1025, 1024, 1019, 1045, 1075, 993, 1063, 1015, 995, 1059, 1016, 991, 1008, 1046, 1004, 1037, 1039, 1050, 1035, 997, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1033, 1032, 1064, 1036, 1012, 1011, 1034, 1014, 1076, 1022, 1023, 964, 977, 1019, 1014, 1044, 1040, 1021, 1040, 1029, 1037, 997, 1024, 1056, 1006, 1061, 1002, 1040, 1047, 1022, 1081, 1020, 1021, 990, 1023, 1011, 1026, 997, 1027, 1030, 957, 995, 1047, 1029, 1048, 972, 998, 1007, 1068, 1069, 969, 1058, 1043, 1054, 1000, 1041, 1011, 1030, 1009, 1015, 1017, 1038, 995, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1044, 1078, 999, 1068, 1006, 1043, 1022, 1043, 984, 1017, 997, 1072, 1033, 1033, 1044, 1009, 1048, 998, 1037, 1016, 1061, 1040, 1013, 1047, 1012, 1015, 1005, 982, 991, 1017, 983, 1052, 1032, 1000, 1045, 958, 1019, 1029, 1015, 989, 1027, 1012, 1037, 991, 1015, 994, 1074, 1031, 997, 1089, 968, 1015, 1060, 1015, 1059, 999, 1038, 1028, 969, 1077, 1016, 1033, 1041, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1021, 1016, 1020, 1056, 1022, 1041, 1037, 1049, 995, 1004, 1035, 1012, 1025, 1030, 1020, 1007, 1040, 1057, 1008, 1016, 1038, 1038, 971, 1053, 1030, 1000, 996, 1008, 988, 1035, 1067, 1041, 1012, 1020, 1034, 1005, 993, 1051, 1022, 1074, 1017, 1034, 1047, 1053, 974, 1010, 1057, 1011, 979, 995, 1036, 1020, 1029, 1058, 1020, 1009, 1041, 1057, 1031, 976, 1016, 1031, 1023, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1004, 1026, 973, 1010, 1042, 1038, 1041, 986, 1012, 999, 1008, 1017, 1031, 998, 1065, 1027, 977, 1037, 983, 1022, 993, 1076, 1035, 1044, 1021, 1035, 998, 1045, 1009, 1023, 1002, 1021, 1071, 1015, 1014, 1032, 1019, 1045, 1015, 1003, 1012, 1031, 1006, 979, 1035, 1059, 1025, 1071, 1025, 1008, 1024, 1039, 1040, 1022, 1003, 1012, 1039, 1076, 1014, 1075, 1010, 1019, 1065, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1029, 1068, 962, 1002, 1010, 1037, 998, 1048, 1021, 1072, 1028, 1004, 1047, 1068, 978, 1005, 1054, 1032, 1087, 1011, 1043, 972, 1017, 1045, 1021, 1012, 941, 1062, 1055, 1044, 996, 1061, 984, 1061, 1015, 1068, 1019, 1000, 1075, 1047, 999, 951, 1000, 1023, 1057, 1026, 1042, 1056, 1001, 1060, 1008, 1051, 1002, 1030, 1051, 963, 1050, 1020, 1024, 967, 1017, 992, 1016, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1038, 1069, 1004, 1041, 1022, 1031, 1028, 1007, 1026, 1004, 1099, 1022, 1002, 1008, 1015, 987, 1035, 1009, 1032, 1043, 990, 1017, 995, 1008, 1054, 999, 993, 1051, 1013, 995, 1044, 992, 1047, 1031, 999, 1061, 1046, 1021, 1022, 973, 1036, 1021, 1048, 1006, 1024, 1014, 1006, 1003, 1015, 1011, 1029, 1084, 1015, 1028, 1079, 1054, 1052, 1015, 1017, 1075, 1028, 1022, 966, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1012, 1009, 1039, 1012, 1004, 1043, 1034, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1006, 1041, 993, 992, 984, 1023, 1045, 18, 1, 0, 0, 0, 0, 0, 0, 981, 1055, 1001, 1023, 1008, 1001, 1036, 1015, 1, 0, 0, 0, 0, 0, 0, 0, 1012, 1041, 1023, 1013, 1058, 1057, 1005, 999, 13, 4, 0, 0, 0, 0, 0, 0, 1033, 1030, 1026, 1029, 1042, 1056, 1022, 983, 1, 0, 0, 0, 0, 0, 0, 0, 1014, 1049, 1049, 1060, 1022, 1000, 1015, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1065, 1052, 1051, 1043, 984, 1072, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1001, 1009, 1053, 1036, 1013, 1013, 1031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8159, 8121, 8225, 8222, 8230, 8217, 8163] avg=8192.0000 max=8230 max_rank=5 min=8121 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1033, 1028, 1064, 1036, 1012, 1011, 1034, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1022, 1023, 964, 977, 1019, 1014, 1044, 18, 1, 0, 0, 0, 0, 0, 0, 1040, 1021, 1040, 1029, 1037, 997, 1024, 1056, 1, 0, 0, 0, 0, 0, 0, 0, 1006, 1061, 1002, 1040, 1047, 1022, 1081, 1020, 13, 4, 0, 0, 0, 0, 0, 0, 1021, 990, 1023, 1011, 1026, 997, 1027, 1030, 1, 0, 0, 0, 0, 0, 0, 0, 957, 995, 1045, 1029, 1048, 972, 998, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1051, 969, 1058, 1043, 1054, 1000, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1030, 1009, 1015, 1017, 1038, 995, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8158, 8245, 8296, 8126, 8051, 8284, 8144] avg=8192.0000 max=8296 max_rank=3 min=8051 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32605 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1021, 1012, 1020, 1056, 1022, 1041, 1037, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1004, 1035, 1012, 1025, 1030, 1020, 1007, 16, 1, 0, 0, 0, 0, 0, 0, 1040, 1057, 1008, 1016, 1038, 1038, 971, 1053, 1, 0, 0, 0, 0, 0, 0, 0, 1030, 1000, 996, 1008, 988, 1035, 1067, 1041, 14, 4, 0, 0, 0, 0, 0, 0, 1012, 1020, 1034, 1005, 993, 1051, 1022, 1074, 1, 0, 0, 0, 0, 0, 0, 0, 1017, 1034, 1045, 1053, 974, 1010, 1057, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 979, 979, 1036, 1020, 1029, 1058, 1020, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1057, 1031, 976, 1016, 1031, 1023, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8145, 8222, 8183, 8212, 8201, 8130, 8185] avg=8192.0000 max=8258 max_rank=0 min=8130 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1029, 1063, 962, 1002, 1010, 1037, 998, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1072, 1028, 1004, 1047, 1068, 978, 1005, 18, 1, 0, 0, 0, 0, 0, 0, 1054, 1032, 1087, 1011, 1043, 972, 1017, 1045, 1, 0, 0, 0, 0, 0, 0, 0, 1021, 1012, 941, 1062, 1055, 1044, 996, 1061, 14, 5, 0, 0, 0, 0, 0, 0, 984, 1061, 1015, 1068, 1019, 1000, 1075, 1047, 1, 0, 0, 0, 0, 0, 0, 0, 999, 951, 998, 1023, 1057, 1026, 1042, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1042, 1008, 1051, 1002, 1030, 1051, 963, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1020, 1024, 967, 1017, 992, 1016, 1031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8242, 8262, 8211, 8270, 8152, 8148, 8102] avg=8192.0000 max=8270 max_rank=4 min=8102 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1003, 1004, 979, 1023, 1043, 1003, 989, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1023, 1043, 1030, 1019, 986, 1028, 1083, 18, 1, 0, 0, 0, 0, 0, 0, 1011, 1006, 1042, 1009, 972, 1045, 1025, 1015, 1, 0, 0, 0, 0, 0, 0, 0, 1033, 1019, 970, 1016, 1016, 1014, 1038, 1081, 14, 4, 0, 0, 0, 0, 0, 0, 1007, 1041, 1024, 1037, 965, 1031, 1042, 1043, 1, 0, 0, 0, 0, 0, 0, 0, 1068, 1038, 1064, 1025, 1024, 1019, 1045, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1045, 1015, 995, 1059, 1016, 991, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1004, 1037, 1039, 1050, 1035, 997, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8059, 8261, 8126, 8205, 8191, 8358, 8122, 8214] avg=8192.0000 max=8358 max_rank=5 min=8059 min_rank=0 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1044, 1073, 999, 1068, 1006, 1043, 1022, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1017, 997, 1072, 1033, 1033, 1044, 1009, 18, 1, 0, 0, 0, 0, 0, 0, 1048, 998, 1037, 1016, 1061, 1040, 1013, 1047, 1, 0, 0, 0, 0, 0, 0, 0, 1012, 1015, 1005, 982, 991, 1017, 983, 1052, 14, 5, 0, 0, 0, 0, 0, 0, 1032, 1000, 1045, 958, 1019, 1029, 1015, 989, 1, 0, 0, 0, 0, 0, 0, 0, 1027, 1012, 1035, 991, 1015, 994, 1074, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1071, 968, 1015, 1060, 1015, 1059, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1028, 969, 1077, 1016, 1033, 1041, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8208, 8261, 8076, 8088, 8179, 8184, 8242] avg=8192.0000 max=8298 max_rank=0 min=8076 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1004, 1012, 973, 1010, 1042, 1038, 1041, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 999, 1008, 1017, 1031, 998, 1065, 1027, 20, 5, 0, 0, 0, 0, 0, 0, 977, 1037, 983, 1022, 993, 1076, 1035, 1044, 2, 0, 0, 0, 0, 0, 0, 0, 1021, 1035, 998, 1045, 1009, 1023, 1002, 1021, 17, 9, 0, 0, 0, 0, 0, 0, 1071, 1015, 1014, 1032, 1019, 1045, 1015, 1003, 7, 5, 0, 0, 0, 0, 0, 0, 1012, 1031, 997, 979, 1035, 1059, 1025, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 988, 1024, 1039, 1040, 1022, 1003, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1076, 1014, 1075, 1010, 1019, 1065, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8182, 8169, 8180, 8226, 8209, 8153, 8311] avg=8192.0000 max=8311 max_rank=7 min=8106 min_rank=0 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1038, 1064, 1004, 1041, 1022, 1031, 1028, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1004, 1099, 1022, 1002, 1008, 1015, 987, 17, 1, 0, 0, 0, 0, 0, 0, 1035, 1009, 1032, 1043, 990, 1017, 995, 1008, 1, 0, 0, 0, 0, 0, 0, 0, 1054, 999, 993, 1051, 1013, 995, 1044, 992, 14, 5, 0, 0, 0, 0, 0, 0, 1047, 1031, 999, 1061, 1046, 1021, 1022, 973, 1, 0, 0, 0, 0, 0, 0, 0, 1036, 1021, 1046, 1006, 1024, 1014, 1006, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 994, 1029, 1084, 1015, 1028, 1079, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1015, 1017, 1075, 1028, 1022, 966, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8181, 8130, 8160, 8201, 8156, 8298, 8175] avg=8192.0000 max=8298 max_rank=6 min=8130 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 7936, 8320, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1041, 1008, 1008, 1077, 1006, 1055, 1029, 1013, 1012, 1017, 1034, 1036, 1041, 984, 1057, 1047, 994, 1036, 1073, 1027, 998, 1015, 1000, 1038, 1008, 1011, 1011, 1039, 992, 1037, 1014, 1080, 1067, 1040, 1018, 1021, 1015, 1017, 1025, 1044, 1042, 1022, 1023, 987, 1002, 1045, 1038, 1013, 990, 1031, 1020, 984, 1060, 1028, 1106, 1033, 1013, 1011, 991, 975, 1035, 992, 998, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1085, 1012, 1069, 1022, 1009, 932, 980, 1006, 1055, 1020, 1043, 1050, 1006, 1048, 1013, 1003, 1037, 1005, 981, 1070, 977, 1006, 1078, 1075, 1026, 1017, 1008, 1002, 1049, 1059, 993, 1034, 1082, 995, 998, 1022, 1028, 1046, 1073, 1034, 1020, 1043, 933, 1023, 1001, 1018, 1021, 1009, 1028, 1032, 996, 1050, 1047, 1013, 973, 993, 1044, 1015, 1069, 1042, 1039, 1013, 1046, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1022, 1074, 1019, 1032, 1032, 1036, 1021, 980, 1025, 1049, 1021, 993, 1003, 1000, 1020, 1061, 1001, 1025, 1027, 1023, 1037, 1025, 1024, 982, 962, 1057, 979, 1041, 1036, 1014, 991, 1046, 1034, 1012, 994, 968, 1041, 1066, 1026, 1030, 981, 1041, 1013, 1081, 1016, 1004, 1031, 1016, 1047, 1021, 1020, 1049, 1030, 1015, 1046, 995, 1049, 1042, 1011, 1063, 1027, 1037, 1045, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1050, 1053, 991, 1043, 1038, 972, 1033, 1047, 1003, 1006, 1024, 1017, 1017, 990, 1052, 1004, 1008, 1015, 1037, 1057, 1008, 1021, 1029, 1029, 1001, 1040, 1082, 1000, 1027, 1027, 1043, 1006, 1075, 1067, 985, 995, 1050, 1046, 988, 1023, 1060, 1001, 1033, 1060, 1055, 994, 1060, 991, 1030, 1026, 985, 1016, 1020, 1085, 1045, 1009, 1008, 1054, 1011, 994, 1048, 967, 952, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1026, 1021, 996, 1092, 990, 984, 1043, 1044, 1076, 1013, 1032, 1051, 1041, 1014, 1045, 984, 1027, 998, 1009, 1012, 1060, 1028, 1054, 1107, 988, 1014, 1017, 979, 1017, 1042, 957, 1026, 1034, 1017, 1015, 1024, 1035, 1049, 1002, 1049, 1024, 1009, 1021, 1055, 1047, 1037, 1058, 960, 1056, 1056, 1039, 981, 1009, 994, 1030, 1000, 1002, 956, 1042, 1001, 1032, 1026, 1058, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1011, 1001, 1084, 1027, 1058, 985, 981, 1010, 990, 1008, 1036, 995, 1011, 1066, 1035, 1047, 1053, 1052, 1024, 1056, 1052, 1054, 992, 988, 1004, 1031, 1014, 1015, 991, 997, 1006, 1079, 1017, 1024, 1069, 1045, 968, 1041, 972, 1048, 1033, 1068, 1059, 1026, 1008, 1030, 1063, 1026, 1011, 1047, 1013, 1038, 946, 992, 1023, 1040, 1048, 988, 1034, 1024, 1048, 1031, 968, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1017, 1003, 1049, 1061, 1054, 1035, 1058, 984, 1030, 1031, 1053, 966, 1011, 1009, 1044, 991, 982, 1051, 1002, 1022, 1021, 1035, 1071, 1017, 988, 1015, 1003, 962, 998, 1071, 1038, 994, 1041, 1050, 1049, 1022, 990, 1021, 999, 1010, 1049, 997, 1038, 1012, 1071, 1014, 1024, 1057, 1105, 1030, 1014, 1037, 1011, 998, 1030, 1013, 1036, 1049, 999, 1011, 1009, 1057, 1008, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1018, 1039, 1050, 1066, 964, 1044, 1060, 1022, 1004, 1005, 1014, 1057, 996, 1013, 1060, 1049, 1009, 1007, 1003, 1043, 965, 1027, 1042, 1016, 1068, 1023, 1009, 994, 1038, 1018, 1027, 1076, 1016, 1007, 1016, 1006, 987, 1067, 1016, 971, 1040, 1008, 1066, 1064, 990, 998, 1029, 1060, 991, 995, 1065, 1030, 983, 1015, 1011, 1071, 1051, 1015, 1059, 1025, 1018, 1043, 1003, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1050, 1053, 991, 1027, 1038, 972, 1033, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1006, 1024, 1017, 1017, 990, 1052, 1004, 1, 0, 0, 0, 0, 0, 0, 0, 1008, 1015, 1037, 1045, 1008, 1021, 1029, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1040, 1082, 1000, 1027, 1027, 1043, 1006, 23, 15, 0, 0, 0, 0, 0, 0, 1067, 1067, 985, 995, 1050, 1046, 988, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1001, 1033, 1060, 1055, 994, 1037, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1026, 985, 1016, 1020, 1085, 1045, 1009, 5, 0, 0, 0, 0, 0, 0, 0, 1008, 1054, 1011, 994, 1048, 967, 952, 1033, 12, 2, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8114, 8192, 8264, 8221, 8231, 8221, 8082] avg=8192.0000 max=8264 max_rank=3 min=8082 min_rank=7 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1041, 1008, 1008, 1061, 1006, 1055, 1029, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1017, 1034, 1036, 1041, 984, 1057, 1047, 1, 0, 0, 0, 0, 0, 0, 0, 994, 1036, 1073, 1015, 998, 1015, 1000, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1011, 1011, 1039, 992, 1037, 1014, 1080, 22, 15, 0, 0, 0, 0, 0, 0, 1060, 1040, 1018, 1021, 1015, 1017, 1025, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1022, 1023, 987, 1002, 1045, 1016, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1031, 1020, 984, 1060, 1028, 1106, 1033, 4, 0, 0, 0, 0, 0, 0, 0, 1013, 1011, 991, 975, 1035, 992, 998, 1012, 12, 2, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8229, 8169, 8229, 8240, 8150, 8256, 8042] avg=8192.0000 max=8256 max_rank=6 min=8042 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1085, 1012, 1069, 1007, 1009, 932, 980, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1020, 1043, 1050, 1006, 1048, 1013, 1003, 1, 0, 0, 0, 0, 0, 0, 0, 1037, 1005, 981, 1057, 977, 1006, 1078, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1017, 1008, 1002, 1049, 1059, 993, 1034, 22, 14, 0, 0, 0, 0, 0, 0, 1074, 995, 998, 1022, 1028, 1046, 1073, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1043, 933, 1023, 1001, 1018, 999, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1032, 996, 1050, 1047, 1013, 973, 993, 5, 0, 0, 0, 0, 0, 0, 0, 1044, 1015, 1069, 1042, 1039, 1013, 1046, 1020, 13, 2, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8239, 8216, 8224, 8270, 8046, 8137, 8304] avg=8192.0000 max=8304 max_rank=7 min=8046 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1022, 1074, 1019, 1016, 1032, 1036, 1021, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1049, 1021, 993, 1003, 1000, 1020, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1025, 1027, 1011, 1037, 1025, 1024, 982, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1057, 979, 1041, 1036, 1014, 991, 1046, 22, 15, 0, 0, 0, 0, 0, 0, 1028, 1012, 994, 968, 1041, 1066, 1026, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1041, 1013, 1081, 1016, 1004, 1009, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1021, 1020, 1049, 1030, 1015, 1046, 995, 4, 0, 0, 0, 0, 0, 0, 0, 1049, 1042, 1011, 1063, 1027, 1037, 1045, 1027, 12, 2, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8172, 8132, 8163, 8165, 8161, 8227, 8316] avg=8192.0000 max=8316 max_rank=7 min=8132 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1017, 1003, 1049, 1045, 1054, 1035, 1058, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1031, 1053, 966, 1011, 1009, 1044, 991, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1051, 1002, 1010, 1021, 1035, 1071, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1015, 1003, 962, 998, 1071, 1038, 994, 22, 15, 0, 0, 0, 0, 0, 0, 1035, 1050, 1049, 1022, 990, 1021, 999, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 997, 1038, 1012, 1071, 1014, 1002, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 1030, 1014, 1037, 1011, 998, 1030, 1013, 4, 0, 0, 0, 0, 0, 0, 0, 1036, 1049, 999, 1011, 1009, 1057, 1008, 1019, 12, 2, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8135, 8189, 8106, 8176, 8240, 8242, 8203] avg=8192.0000 max=8245 max_rank=0 min=8106 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32861 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1026, 1021, 996, 1076, 990, 984, 1043, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1013, 1032, 1051, 1041, 1014, 1045, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 998, 1009, 1000, 1060, 1028, 1054, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1014, 1017, 979, 1017, 1042, 957, 1026, 23, 15, 0, 0, 0, 0, 0, 0, 1028, 1017, 1015, 1024, 1035, 1049, 1002, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1009, 1021, 1055, 1047, 1037, 1035, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1056, 1039, 981, 1009, 994, 1030, 1000, 4, 0, 0, 0, 0, 0, 0, 0, 1002, 956, 1042, 1001, 1032, 1026, 1058, 1031, 12, 2, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8256, 8283, 8078, 8219, 8188, 8169, 8163] avg=8192.0000 max=8283 max_rank=2 min=8078 min_rank=3 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1011, 1001, 1084, 1002, 1058, 985, 981, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1008, 1036, 995, 1011, 1066, 1035, 1047, 5, 0, 0, 0, 0, 0, 0, 0, 1053, 1052, 1024, 1040, 1052, 1054, 992, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1031, 1014, 1015, 991, 997, 1006, 1079, 26, 19, 0, 0, 0, 0, 0, 0, 998, 1024, 1069, 1045, 968, 1041, 972, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1068, 1059, 1026, 1008, 1030, 1037, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1047, 1013, 1038, 946, 992, 1023, 1040, 9, 0, 0, 0, 0, 0, 0, 0, 1048, 988, 1034, 1024, 1048, 1031, 968, 1035, 16, 5, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8193, 8255, 8182, 8165, 8287, 8119, 8203] avg=8192.0000 max=8287 max_rank=5 min=8119 min_rank=6 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1018, 1039, 1050, 1050, 964, 1044, 1060, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1005, 1014, 1057, 996, 1013, 1060, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1007, 1003, 1031, 965, 1027, 1042, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1023, 1009, 994, 1038, 1018, 1027, 1076, 22, 15, 0, 0, 0, 0, 0, 0, 1010, 1007, 1016, 1006, 987, 1067, 1016, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1008, 1066, 1064, 990, 998, 1007, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 991, 995, 1065, 1030, 983, 1015, 1011, 1071, 4, 0, 0, 0, 0, 0, 0, 0, 1051, 1015, 1059, 1025, 1018, 1043, 1003, 994, 12, 2, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8198, 8100, 8290, 8080, 8233, 8165, 8223] avg=8192.0000 max=8290 max_rank=3 min=8080 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8448, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8064, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8064, 8192, 8320, 8192, 8448, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8448, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1022, 988, 1037, 1013, 1015, 1063, 1014, 1020, 1070, 994, 1025, 1010, 1015, 1023, 1027, 1019, 1011, 981, 989, 1031, 1008, 997, 1046, 1003, 1022, 1042, 1044, 1057, 1044, 1019, 1021, 995, 1063, 1066, 1047, 1020, 1023, 992, 1038, 1026, 1014, 1044, 999, 1021, 1025, 1012, 1070, 1052, 1078, 1015, 1020, 1075, 1024, 1000, 1002, 957, 1023, 995, 997, 987, 1033, 1020, 1068, 1065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1031, 1100, 979, 1038, 1044, 1059, 1002, 1039, 988, 959, 1081, 1004, 1006, 1026, 1018, 1019, 1078, 1028, 1005, 1001, 1013, 977, 995, 1004, 1004, 1077, 1033, 1068, 1007, 991, 1033, 1008, 1017, 1047, 1038, 1023, 1029, 1077, 1063, 1031, 1034, 1038, 1041, 1017, 1027, 1038, 997, 1022, 1014, 1066, 1039, 1021, 1032, 973, 1019, 1001, 1007, 1010, 1001, 1026, 1047, 1014, 996, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1012, 1025, 1047, 1037, 1015, 1038, 1008, 1051, 1029, 1044, 1026, 1041, 1022, 1057, 1042, 974, 1047, 1009, 1051, 992, 995, 1038, 1038, 991, 1014, 1040, 1019, 1054, 1000, 1047, 1034, 1066, 1025, 1040, 1040, 1000, 1029, 1034, 1022, 978, 1042, 959, 1031, 1070, 1052, 1034, 1006, 1024, 1009, 997, 1039, 973, 1015, 1017, 979, 1020, 1028, 1034, 1025, 1042, 1058, 1014, 987, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1035, 1029, 1040, 1038, 1004, 1057, 1037, 1044, 1013, 968, 1058, 1007, 1009, 996, 1022, 1029, 993, 1022, 997, 1008, 1038, 1004, 1066, 1007, 1000, 1034, 1008, 1006, 1061, 990, 1020, 1023, 1071, 1077, 1046, 1028, 1018, 1044, 1021, 1002, 1053, 1021, 1015, 1031, 988, 986, 1031, 1004, 1017, 1026, 993, 1031, 1076, 1068, 1025, 1038, 1026, 1019, 980, 1030, 1018, 1059, 1027, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[989, 1049, 994, 1044, 1087, 1007, 951, 961, 1096, 1001, 1039, 1058, 1037, 1020, 1061, 1046, 992, 1039, 1084, 1047, 1017, 1039, 1044, 1007, 1043, 997, 1001, 1041, 1048, 1025, 987, 1056, 1034, 1015, 1048, 995, 980, 1018, 1007, 1088, 985, 1037, 986, 1028, 984, 1030, 1044, 1010, 1066, 1030, 995, 1030, 981, 1041, 997, 1009, 1011, 996, 1049, 1017, 1061, 1037, 1019, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1048, 1017, 984, 995, 1000, 1060, 1036, 1055, 1061, 1009, 1051, 1029, 1009, 982, 1037, 1075, 969, 972, 997, 1078, 989, 973, 957, 970, 1020, 1067, 993, 989, 1026, 1022, 992, 1020, 1030, 1029, 1058, 1032, 1041, 1047, 1055, 1042, 1026, 1015, 1086, 1001, 1005, 982, 1052, 1042, 1018, 990, 992, 1080, 1056, 1062, 1033, 1009, 1035, 1037, 1086, 1009, 1017, 1016, 1052, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1016, 1004, 1045, 1015, 1037, 997, 1042, 1026, 1021, 1005, 1059, 1005, 1043, 996, 1046, 1027, 1023, 971, 1018, 1039, 1017, 996, 1049, 1034, 1006, 1020, 1050, 1032, 1031, 1016, 1024, 994, 1010, 1031, 1030, 1037, 1021, 1034, 1010, 1028, 1040, 979, 987, 1035, 1074, 1027, 1005, 1030, 1062, 1006, 1018, 1018, 1047, 1038, 999, 1035, 1019, 1073, 1047, 1010, 1032, 1039, 1019, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1031, 1023, 1045, 1006, 1025, 981, 1015, 1035, 1027, 1039, 985, 1016, 1017, 1028, 1037, 1045, 1030, 1054, 1016, 1038, 1041, 998, 1042, 1041, 1025, 964, 1050, 1038, 970, 977, 1037, 996, 1053, 1018, 1005, 1025, 989, 1047, 1008, 1000, 1033, 1053, 1068, 979, 991, 993, 1062, 1089, 1024, 1047, 964, 1060, 991, 1015, 1018, 1035, 1027, 1074, 1025, 1036, 1048, 1027, 1015, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1022, 988, 1037, 1013, 1015, 1055, 1014, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 994, 1015, 1010, 1015, 1023, 1027, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 981, 989, 1031, 1008, 997, 1046, 1003, 51, 10, 0, 0, 0, 0, 0, 0, 1022, 1042, 1044, 1057, 1044, 1019, 1021, 995, 8, 2, 1, 0, 0, 0, 0, 0, 1063, 1015, 1047, 1020, 1023, 992, 1038, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1044, 999, 1021, 1025, 1012, 1070, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1015, 1020, 1075, 1024, 1000, 1002, 957, 9, 3, 0, 0, 0, 0, 0, 0, 1023, 995, 997, 987, 1020, 1020, 1068, 1065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8173, 8127, 8255, 8224, 8235, 8183, 8175] avg=8192.0000 max=8255 max_rank=3 min=8127 min_rank=2 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[989, 1049, 994, 1044, 1087, 1000, 951, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1096, 1001, 1029, 1058, 1037, 1020, 1061, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1039, 1084, 1047, 1017, 1039, 1044, 1007, 49, 11, 0, 0, 0, 0, 0, 0, 1043, 997, 1001, 1041, 1048, 1025, 987, 1056, 7, 2, 1, 0, 0, 0, 0, 0, 1034, 966, 1048, 995, 980, 1018, 1007, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1037, 986, 1028, 984, 1030, 1044, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1030, 995, 1030, 981, 1041, 997, 1009, 9, 4, 0, 0, 0, 0, 0, 0, 1011, 996, 1049, 1017, 1046, 1037, 1019, 1001, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8075, 8348, 8329, 8208, 8136, 8102, 8162, 8176] avg=8192.0000 max=8348 max_rank=1 min=8075 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1012, 1025, 1047, 1037, 1015, 1030, 1008, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1044, 1016, 1041, 1022, 1057, 1042, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1009, 1051, 992, 995, 1038, 1038, 991, 50, 11, 0, 0, 0, 0, 0, 0, 1014, 1040, 1019, 1054, 1000, 1047, 1034, 1066, 8, 2, 1, 0, 0, 0, 0, 0, 1025, 990, 1040, 1000, 1029, 1034, 1022, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 959, 1031, 1070, 1052, 1034, 1006, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 997, 1039, 973, 1015, 1017, 979, 1020, 9, 4, 0, 0, 0, 0, 0, 0, 1028, 1034, 1025, 1042, 1043, 1014, 987, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8225, 8222, 8285, 8118, 8216, 8062, 8183] avg=8192.0000 max=8285 max_rank=3 min=8062 min_rank=6 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32579 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1031, 1100, 979, 1038, 1044, 1051, 1002, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 988, 959, 1070, 1004, 1006, 1026, 1018, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1028, 1005, 1001, 1013, 977, 995, 1004, 50, 11, 0, 0, 0, 0, 0, 0, 1004, 1077, 1033, 1068, 1007, 991, 1033, 1008, 8, 2, 1, 0, 0, 0, 0, 0, 1017, 997, 1038, 1023, 1029, 1077, 1063, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1038, 1041, 1017, 1027, 1038, 997, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1066, 1039, 1021, 1032, 973, 1019, 1001, 10, 4, 0, 0, 0, 0, 0, 0, 1007, 1010, 1001, 1026, 1032, 1014, 996, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8090, 8162, 8232, 8275, 8212, 8179, 8102] avg=8192.0000 max=8284 max_rank=0 min=8090 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1035, 1029, 1040, 1038, 1004, 1049, 1037, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 968, 1047, 1007, 1009, 996, 1022, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1022, 997, 1008, 1038, 1004, 1066, 1007, 52, 10, 0, 0, 0, 0, 0, 0, 1000, 1034, 1008, 1006, 1061, 990, 1020, 1023, 8, 2, 1, 0, 0, 0, 0, 0, 1071, 1025, 1046, 1028, 1018, 1044, 1021, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1021, 1015, 1031, 988, 986, 1031, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1026, 993, 1031, 1076, 1068, 1025, 1038, 10, 3, 0, 0, 0, 0, 0, 0, 1026, 1019, 980, 1030, 1005, 1059, 1027, 1004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8091, 8197, 8153, 8255, 8127, 8287, 8150] avg=8192.0000 max=8287 max_rank=6 min=8091 min_rank=1 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1016, 1004, 1045, 1015, 1037, 990, 1042, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1005, 1048, 1005, 1043, 996, 1046, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 971, 1018, 1039, 1017, 996, 1049, 1034, 50, 10, 0, 0, 0, 0, 0, 0, 1006, 1020, 1050, 1032, 1031, 1016, 1024, 994, 7, 2, 1, 0, 0, 0, 0, 0, 1010, 981, 1030, 1037, 1021, 1034, 1010, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 979, 987, 1035, 1074, 1027, 1005, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1006, 1018, 1018, 1047, 1038, 999, 1035, 10, 3, 0, 0, 0, 0, 0, 0, 1019, 1073, 1047, 1010, 1019, 1039, 1019, 992, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8191, 8207, 8183, 8151, 8175, 8236, 8218] avg=8192.0000 max=8236 max_rank=6 min=8151 min_rank=4 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1048, 1017, 984, 995, 1000, 1048, 1036, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1009, 1032, 1029, 1009, 982, 1037, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 969, 972, 997, 1078, 989, 973, 957, 970, 53, 14, 0, 0, 0, 0, 0, 0, 1020, 1067, 993, 989, 1026, 1022, 992, 1020, 12, 6, 6, 0, 0, 0, 0, 0, 1030, 976, 1058, 1032, 1041, 1047, 1055, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1015, 1086, 1001, 1005, 982, 1052, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 990, 992, 1080, 1056, 1062, 1033, 1009, 13, 7, 0, 0, 0, 0, 0, 0, 1035, 1037, 1086, 1009, 996, 1016, 1052, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8234, 7972, 8153, 8281, 8203, 8260, 8250] avg=8192.0000 max=8281 max_rank=4 min=7972 min_rank=2 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32994 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1031, 1023, 1045, 1006, 1025, 974, 1015, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1039, 975, 1016, 1017, 1028, 1037, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1054, 1016, 1038, 1041, 998, 1042, 1041, 49, 11, 0, 0, 0, 0, 0, 0, 1025, 964, 1050, 1038, 970, 977, 1037, 996, 7, 2, 1, 0, 0, 0, 0, 0, 1053, 969, 1005, 1025, 989, 1047, 1008, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1053, 1068, 979, 991, 993, 1062, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1047, 964, 1060, 991, 1015, 1018, 1035, 9, 4, 0, 0, 0, 0, 0, 0, 1027, 1074, 1025, 1036, 1033, 1027, 1015, 1045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8184, 8320, 8067, 8096, 8266, 8167, 8282] avg=8192.0000 max=8320 max_rank=2 min=8067 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8064, 8320, 8064, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 7936, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8064, 8320, 8320, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1019, 1002, 1017, 1031, 1090, 1023, 1015, 1004, 1038, 987, 1096, 1029, 975, 1030, 1039, 1006, 1019, 1043, 1051, 1057, 1020, 1024, 990, 965, 1065, 1052, 1040, 1027, 1008, 993, 1017, 1029, 977, 1047, 1029, 991, 1061, 1026, 1036, 1014, 1039, 1026, 1031, 1035, 1011, 1017, 1051, 997, 1037, 1001, 1048, 985, 1017, 1015, 1019, 1078, 1014, 1006, 1049, 1025, 1026, 1012, 986, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1051, 989, 1017, 1100, 1052, 1049, 1021, 1017, 1053, 1036, 1025, 984, 1067, 1039, 1083, 1043, 1024, 1018, 991, 952, 996, 1045, 980, 1073, 996, 1030, 1038, 1046, 1025, 1036, 1030, 1047, 1030, 1015, 1008, 979, 987, 1004, 1046, 1010, 1018, 1029, 1027, 1004, 1021, 1001, 1051, 1023, 1030, 1025, 979, 1038, 1007, 968, 1027, 1022, 1056, 1044, 1024, 952, 1085, 1019, 1047, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1025, 1031, 969, 1021, 1084, 1052, 1042, 1075, 1049, 987, 979, 994, 1026, 1073, 1005, 1020, 1025, 1035, 1026, 978, 1035, 1030, 981, 1028, 990, 1028, 1003, 1003, 1048, 995, 1014, 977, 1012, 1005, 1043, 1047, 1038, 1034, 1029, 1000, 1024, 1028, 1013, 1031, 1016, 1023, 1010, 1044, 1029, 1001, 1038, 1040, 1005, 1045, 1048, 1030, 1028, 1018, 1078, 1047, 1046, 1022, 1009, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1002, 1043, 1020, 996, 1032, 1017, 1032, 1006, 993, 1084, 1049, 1003, 980, 1003, 1051, 1063, 982, 1036, 1014, 1013, 1060, 1018, 1042, 1049, 1024, 1039, 1047, 1015, 1042, 1027, 1047, 1001, 1044, 1027, 1024, 995, 1036, 1039, 1006, 990, 1056, 1055, 1032, 1014, 1048, 1024, 992, 994, 1019, 1005, 1043, 1048, 1068, 1010, 1061, 984, 1029, 998, 973, 1082, 1004, 1046, 994, 966] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1042, 1002, 1017, 1028, 1008, 1015, 1007, 1004, 1052, 1028, 1000, 1040, 1008, 1032, 1020, 1010, 1043, 988, 999, 987, 1035, 1000, 992, 1019, 1047, 1051, 1018, 1048, 1009, 1014, 1015, 1040, 1040, 1019, 1017, 1008, 1026, 1052, 1009, 984, 1040, 996, 1062, 1028, 1013, 978, 1053, 1014, 1054, 1066, 1059, 1039, 995, 1023, 1064, 1026, 1070, 1014, 1016, 1017, 1075, 1036, 1018, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1024, 1008, 998, 1054, 1063, 1027, 968, 1003, 992, 1042, 959, 1100, 1026, 1000, 1076, 1087, 1044, 996, 992, 1017, 1012, 1000, 1034, 1013, 993, 1076, 966, 1039, 1032, 1028, 1005, 1043, 992, 1015, 1053, 1031, 1006, 1027, 1055, 1032, 988, 1058, 1009, 1009, 1009, 1036, 1035, 1014, 990, 1036, 988, 1038, 1023, 1036, 1042, 1077, 1047, 1065, 1013, 999, 949, 997, 1046, 1104] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[986, 1049, 1034, 1016, 1075, 1024, 1003, 1087, 1020, 1056, 1025, 1035, 995, 1034, 996, 1003, 1071, 1037, 973, 1025, 1062, 1015, 1029, 1050, 1008, 1014, 1073, 1043, 1032, 1083, 1076, 1005, 1050, 1001, 1002, 1004, 1003, 1028, 1017, 1006, 1023, 973, 987, 1008, 1039, 1059, 1016, 1013, 1002, 1004, 1022, 1030, 1028, 1021, 1031, 981, 1028, 1012, 1012, 1043, 1046, 1043, 991, 979] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1008, 1006, 995, 1059, 1029, 972, 1068, 1053, 1048, 986, 1012, 1052, 1052, 1007, 1035, 1005, 1026, 1021, 991, 1050, 949, 968, 1046, 1087, 1041, 1070, 996, 1017, 984, 991, 991, 1038, 1061, 1039, 1041, 1037, 1013, 1029, 1042, 1047, 1054, 1013, 1052, 1037, 1036, 1040, 1030, 988, 991, 1035, 1023, 1012, 1035, 1008, 992, 1009, 999, 1013, 1065, 1025, 1040, 1010, 1026, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1051, 989, 1017, 1100, 1035, 1049, 1021, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1036, 1025, 984, 1067, 1039, 1059, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1018, 991, 952, 996, 1045, 980, 1073, 24, 17, 4, 0, 0, 0, 0, 0, 996, 1015, 1038, 1046, 1025, 1036, 1030, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1015, 1008, 979, 987, 1004, 1046, 1010, 11, 4, 0, 0, 0, 0, 0, 0, 1018, 1029, 1027, 1004, 1021, 1001, 1051, 1023, 3, 1, 0, 0, 0, 0, 0, 0, 1030, 1025, 979, 1038, 1007, 968, 1026, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1044, 1024, 952, 1078, 1019, 1047, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8306, 8124, 8233, 8094, 8178, 8095, 8227] avg=8192.0000 max=8306 max_rank=1 min=8094 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1025, 1031, 969, 1021, 1067, 1052, 1042, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 987, 979, 994, 1026, 1073, 983, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1035, 1026, 978, 1035, 1030, 981, 1028, 22, 17, 4, 0, 0, 0, 0, 0, 990, 1013, 1003, 1003, 1048, 995, 1014, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1005, 1043, 1047, 1038, 1034, 1029, 1000, 11, 3, 0, 0, 0, 0, 0, 0, 1024, 1028, 1013, 1031, 1016, 1023, 1010, 1044, 3, 1, 0, 0, 0, 0, 0, 0, 1029, 1001, 1038, 1040, 1005, 1045, 1047, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1018, 1078, 1047, 1040, 1022, 1009, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8111, 8181, 8043, 8222, 8193, 8235, 8269] avg=8192.0000 max=8282 max_rank=0 min=8043 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1019, 1002, 1017, 1031, 1072, 1023, 1015, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 987, 1096, 1029, 975, 1030, 1016, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1043, 1051, 1057, 1020, 1024, 990, 965, 23, 18, 4, 0, 0, 0, 0, 0, 1065, 1037, 1040, 1027, 1008, 993, 1017, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1047, 1029, 991, 1061, 1026, 1036, 1014, 11, 3, 0, 0, 0, 0, 0, 0, 1039, 1026, 1031, 1035, 1011, 1017, 1051, 997, 3, 1, 0, 0, 0, 0, 0, 0, 1037, 1001, 1048, 985, 1017, 1015, 1018, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1006, 1049, 1025, 1020, 1012, 986, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8177, 8214, 8216, 8195, 8211, 8199, 8141] avg=8192.0000 max=8216 max_rank=3 min=8141 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=57320 max_load_reduction_pct=0.874634 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1002, 1043, 1020, 996, 1015, 1017, 1032, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1084, 1049, 1003, 980, 1003, 1028, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1036, 1014, 1013, 1060, 1018, 1042, 1049, 23, 17, 4, 0, 0, 0, 0, 0, 1024, 1024, 1047, 1015, 1042, 1027, 1047, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1027, 1024, 995, 1036, 1039, 1006, 990, 11, 3, 0, 0, 0, 0, 0, 0, 1056, 1055, 1032, 1014, 1048, 1024, 992, 994, 3, 1, 0, 0, 0, 0, 0, 0, 1019, 1005, 1043, 1048, 1068, 1010, 1060, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 998, 973, 1082, 998, 1046, 994, 966, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8203, 8258, 8227, 8175, 8219, 8237, 8086] avg=8192.0000 max=8258 max_rank=2 min=8086 min_rank=7 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1042, 1002, 1017, 1028, 992, 1015, 1007, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1028, 1000, 1040, 1008, 1032, 998, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 988, 999, 987, 1035, 1000, 992, 1019, 22, 16, 4, 0, 0, 0, 0, 0, 1047, 1036, 1018, 1048, 1009, 1014, 1015, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1019, 1017, 1008, 1026, 1052, 1009, 984, 11, 4, 0, 0, 0, 0, 0, 0, 1040, 996, 1062, 1028, 1013, 978, 1053, 1014, 3, 1, 0, 0, 0, 0, 0, 0, 1054, 1066, 1059, 1039, 995, 1023, 1063, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1014, 1016, 1017, 1068, 1036, 1018, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8168, 8105, 8227, 8170, 8188, 8325, 8246] avg=8192.0000 max=8325 max_rank=6 min=8105 min_rank=2 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1024, 1008, 998, 1054, 1046, 1027, 968, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1042, 959, 1100, 1026, 1000, 1052, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 996, 992, 1017, 1012, 1000, 1034, 1013, 24, 17, 5, 0, 0, 0, 0, 0, 993, 1060, 966, 1039, 1032, 1028, 1005, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1015, 1053, 1031, 1006, 1027, 1055, 1032, 11, 3, 0, 0, 0, 0, 0, 0, 988, 1058, 1009, 1009, 1009, 1036, 1035, 1014, 3, 1, 0, 0, 0, 0, 0, 0, 990, 1036, 988, 1038, 1023, 1036, 1041, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1065, 1013, 999, 943, 997, 1046, 1104, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8258, 8154, 8166, 8225, 8162, 8229, 8214] avg=8192.0000 max=8258 max_rank=1 min=8128 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[986, 1049, 1034, 1016, 1054, 1024, 1003, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1056, 1025, 1035, 995, 1034, 971, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1037, 973, 1025, 1062, 1015, 1029, 1050, 25, 21, 10, 0, 0, 0, 0, 0, 1008, 991, 1073, 1043, 1032, 1083, 1076, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1001, 1002, 1004, 1003, 1028, 1017, 1006, 13, 8, 0, 0, 0, 0, 0, 0, 1023, 973, 987, 1008, 1039, 1059, 1016, 1013, 6, 7, 0, 0, 0, 0, 0, 0, 1002, 1004, 1022, 1030, 1028, 1021, 1024, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1012, 1012, 1043, 1032, 1043, 991, 979, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8139, 8318, 8311, 8132, 8131, 8112, 8140] avg=8192.0000 max=8318 max_rank=2 min=8112 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1008, 1006, 995, 1059, 1012, 972, 1068, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 986, 1012, 1052, 1052, 1007, 1012, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1021, 991, 1050, 949, 968, 1046, 1087, 23, 17, 4, 0, 0, 0, 0, 0, 1041, 1055, 996, 1017, 984, 991, 991, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1039, 1041, 1037, 1013, 1029, 1042, 1047, 11, 3, 0, 0, 0, 0, 0, 0, 1054, 1013, 1052, 1037, 1036, 1040, 1030, 988, 3, 1, 0, 0, 0, 0, 0, 0, 991, 1035, 1023, 1012, 1035, 1008, 991, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1013, 1065, 1025, 1034, 1010, 1026, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8174, 8182, 8113, 8323, 8254, 8104, 8213] avg=8192.0000 max=8323 max_rank=4 min=8104 min_rank=6 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32894 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8064, 8192, 8192, 8192, 8192, 8320, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1054, 1036, 1015, 1043, 1014, 974, 1026, 1047, 1012, 980, 1067, 1018, 1038, 1098, 1040, 1016, 1008, 1012, 1052, 1023, 1017, 994, 972, 999, 1020, 1014, 1022, 1023, 1023, 1015, 1065, 1058, 1057, 1037, 987, 1012, 1047, 967, 1027, 1061, 1016, 967, 1028, 1026, 995, 999, 1011, 997, 1066, 1041, 1017, 1045, 1032, 1021, 1081, 1053, 1043, 1016, 1012, 1050, 998, 992, 1026, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1042, 1007, 968, 1030, 1065, 1052, 1028, 991, 1013, 995, 1010, 1057, 1015, 1054, 1028, 1052, 980, 1007, 981, 1029, 1015, 991, 1035, 1008, 1015, 978, 1006, 1042, 1044, 1012, 1050, 1056, 1027, 1013, 999, 1032, 1023, 973, 998, 1007, 1025, 1023, 1039, 1077, 1041, 1051, 1038, 1056, 998, 1045, 1023, 988, 1055, 1080, 1021, 1041, 1028, 991, 1041, 1068, 1005, 1040, 1019, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[993, 1027, 988, 1045, 1060, 1042, 1057, 1029, 1030, 1046, 1018, 1045, 999, 1032, 1029, 1066, 1005, 1044, 1050, 1027, 996, 1098, 988, 1019, 1015, 1076, 990, 1037, 1018, 1042, 978, 1004, 1050, 1029, 1017, 1023, 1008, 1041, 966, 1043, 1055, 992, 1052, 1048, 1003, 1048, 982, 1010, 977, 1027, 1027, 1046, 1021, 1001, 1017, 1018, 1023, 1000, 1023, 1038, 997, 1007, 1027, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[996, 1023, 1000, 1052, 1028, 1051, 1004, 964, 1045, 975, 1032, 1059, 983, 969, 1026, 993, 1038, 1090, 1030, 1039, 1018, 1043, 1019, 1096, 1036, 1069, 1016, 1016, 986, 1020, 999, 1044, 1008, 1031, 1023, 1002, 1008, 1045, 1038, 1015, 1032, 1002, 1001, 999, 994, 996, 1040, 1005, 1018, 989, 1034, 1073, 1046, 1001, 1078, 989, 1028, 1048, 1042, 1054, 1025, 1023, 1063, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1046, 1026, 1015, 1074, 995, 1028, 1042, 1052, 1055, 989, 1029, 1048, 1007, 1021, 1015, 1037, 1034, 1024, 999, 1022, 1022, 1031, 1029, 1042, 1020, 993, 1001, 1016, 1016, 1025, 1003, 1040, 1029, 1024, 1029, 1038, 1007, 1045, 1018, 1004, 1039, 1004, 966, 980, 1018, 1019, 978, 1008, 1044, 1079, 1060, 1020, 1059, 998, 1008, 1028, 969, 997, 1013, 1039, 1088, 1078, 1039, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1005, 977, 993, 1056, 1026, 979, 1032, 973, 997, 1051, 1035, 997, 967, 1073, 1046, 1031, 987, 1023, 1023, 1018, 1078, 1029, 1022, 1044, 1028, 1037, 1031, 994, 1022, 1021, 1002, 1007, 1037, 998, 1055, 979, 1013, 1022, 1016, 1060, 1064, 991, 1018, 1067, 1041, 1004, 1022, 1007, 1052, 1060, 1079, 1017, 1012, 1054, 1063, 998, 1001, 993, 1056, 1080, 1028, 1031, 1004, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1034, 1043, 1004, 1050, 1031, 1027, 1076, 1030, 1025, 1023, 1018, 1027, 1075, 1011, 1043, 990, 973, 1024, 1037, 1008, 1047, 1023, 984, 1068, 1032, 1059, 981, 1003, 983, 1009, 989, 1050, 1034, 1063, 1055, 1014, 1062, 968, 1051, 1021, 1035, 1075, 1009, 1048, 1029, 1000, 1014, 997, 1055, 1030, 1000, 997, 1009, 1013, 1032, 1042, 996, 1029, 1027, 987, 1013, 1040, 1025, 989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1010, 1015, 1026, 1034, 996, 970, 1026, 1036, 1075, 1070, 1052, 1010, 1041, 999, 1016, 1038, 951, 1026, 1032, 1016, 1043, 989, 1058, 1030, 1031, 1094, 1085, 1014, 980, 1041, 1019, 1037, 1016, 966, 1055, 1005, 973, 1029, 1023, 1018, 1035, 1042, 995, 1039, 1049, 1042, 995, 998, 1006, 1037, 1033, 1051, 1008, 991, 1012, 1038, 1028, 1059, 1010, 969, 1074, 996, 1032, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1042, 1007, 968, 1030, 1065, 1052, 1028, 991, 5, 2, 0, 0, 0, 0, 0, 0, 1013, 995, 1010, 1032, 1015, 1054, 1028, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1007, 981, 1029, 1015, 991, 1035, 1008, 9, 0, 0, 0, 0, 0, 0, 0, 1015, 978, 1006, 1042, 1044, 1012, 1050, 1056, 10, 0, 0, 0, 0, 0, 0, 0, 1027, 1013, 999, 1032, 1023, 973, 998, 1007, 19, 4, 0, 0, 0, 0, 0, 0, 1025, 1023, 1039, 1077, 1041, 1051, 1038, 1056, 31, 0, 0, 0, 0, 0, 0, 0, 998, 1045, 1023, 988, 1055, 1080, 971, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 991, 1041, 1063, 1005, 1040, 1019, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8199, 8055, 8213, 8095, 8381, 8201, 8202] avg=8192.0000 max=8381 max_rank=5 min=8055 min_rank=2 imbalance_ratio=1.023071 max_load_reduction=57155 max_load_reduction_pct=0.872116 moved_to_echo=32879 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1054, 1036, 1015, 1043, 1014, 974, 1026, 1047, 5, 2, 0, 0, 0, 0, 0, 0, 1012, 980, 1067, 993, 1038, 1098, 1040, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1012, 1052, 1023, 1017, 994, 972, 999, 9, 0, 0, 0, 0, 0, 0, 0, 1020, 1014, 1022, 1023, 1023, 1015, 1065, 1058, 10, 0, 0, 0, 0, 0, 0, 0, 1057, 1037, 987, 1012, 1047, 967, 1027, 1061, 20, 4, 0, 0, 0, 0, 0, 0, 1016, 967, 1028, 1026, 995, 999, 1011, 997, 33, 0, 0, 0, 0, 0, 0, 0, 1066, 1041, 1017, 1045, 1032, 1021, 1028, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1016, 1012, 1045, 998, 992, 1026, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8244, 8086, 8250, 8219, 8072, 8303, 8146] avg=8192.0000 max=8303 max_rank=6 min=8072 min_rank=5 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[993, 1027, 988, 1045, 1060, 1042, 1057, 1029, 5, 2, 0, 0, 0, 0, 0, 0, 1030, 1046, 1018, 1020, 999, 1032, 1029, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1044, 1050, 1027, 996, 1098, 988, 1019, 9, 0, 0, 0, 0, 0, 0, 0, 1015, 1076, 990, 1037, 1018, 1042, 978, 1004, 10, 0, 0, 0, 0, 0, 0, 0, 1050, 1029, 1017, 1023, 1008, 1041, 966, 1043, 19, 4, 0, 0, 0, 0, 0, 0, 1055, 992, 1052, 1048, 1003, 1048, 982, 1010, 31, 0, 0, 0, 0, 0, 0, 0, 977, 1027, 1027, 1046, 1021, 1001, 967, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1000, 1023, 1033, 997, 1007, 1027, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8240, 8236, 8170, 8200, 8221, 8084, 8137] avg=8192.0000 max=8248 max_rank=0 min=8084 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32642 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1005, 977, 993, 1056, 1026, 979, 1032, 973, 5, 2, 0, 0, 0, 0, 0, 0, 997, 1051, 1035, 972, 967, 1073, 1046, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1023, 1023, 1018, 1078, 1029, 1022, 1044, 9, 0, 0, 0, 0, 0, 0, 0, 1028, 1037, 1031, 994, 1022, 1021, 1002, 1007, 10, 0, 0, 0, 0, 0, 0, 0, 1037, 998, 1055, 979, 1013, 1022, 1016, 1060, 20, 4, 0, 0, 0, 0, 0, 0, 1064, 991, 1018, 1067, 1041, 1004, 1022, 1007, 33, 0, 0, 0, 0, 0, 0, 0, 1052, 1060, 1079, 1017, 1012, 1054, 1010, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 993, 1056, 1075, 1028, 1031, 1004, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8048, 8172, 8233, 8152, 8204, 8247, 8282, 8198] avg=8192.0000 max=8282 max_rank=6 min=8048 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32931 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[996, 1023, 1000, 1052, 1028, 1051, 1004, 964, 5, 2, 0, 0, 0, 0, 0, 0, 1045, 975, 1032, 1034, 983, 969, 1026, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1090, 1030, 1039, 1018, 1043, 1019, 1096, 9, 0, 0, 0, 0, 0, 0, 0, 1036, 1069, 1016, 1016, 986, 1020, 999, 1044, 10, 0, 0, 0, 0, 0, 0, 0, 1008, 1031, 1023, 1002, 1008, 1045, 1038, 1015, 20, 4, 0, 0, 0, 0, 0, 0, 1032, 1002, 1001, 999, 994, 996, 1040, 1005, 33, 0, 0, 0, 0, 0, 0, 0, 1018, 989, 1034, 1073, 1046, 1001, 1025, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1048, 1042, 1049, 1025, 1023, 1063, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8057, 8382, 8196, 8194, 8102, 8175, 8305] avg=8192.0000 max=8382 max_rank=2 min=8057 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1046, 1026, 1015, 1074, 995, 1028, 1042, 1052, 5, 2, 0, 0, 0, 0, 0, 0, 1055, 989, 1029, 1023, 1007, 1021, 1015, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1024, 999, 1022, 1022, 1031, 1029, 1042, 9, 0, 0, 0, 0, 0, 0, 0, 1020, 993, 1001, 1016, 1016, 1025, 1003, 1040, 10, 0, 0, 0, 0, 0, 0, 0, 1029, 1024, 1029, 1038, 1007, 1045, 1018, 1004, 19, 4, 0, 0, 0, 0, 0, 0, 1039, 1004, 966, 980, 1018, 1019, 978, 1008, 31, 0, 0, 0, 0, 0, 0, 0, 1044, 1079, 1060, 1020, 1059, 998, 958, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 969, 997, 1013, 1034, 1088, 1078, 1039, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8176, 8212, 8124, 8217, 8043, 8246, 8233] avg=8192.0000 max=8285 max_rank=0 min=8043 min_rank=5 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1010, 1015, 1026, 1034, 996, 970, 1026, 1036, 4, 2, 0, 0, 0, 0, 0, 0, 1075, 1070, 1052, 985, 1041, 999, 1016, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1026, 1032, 1016, 1043, 989, 1058, 1030, 9, 0, 0, 0, 0, 0, 0, 0, 1031, 1094, 1085, 1014, 980, 1041, 1019, 1037, 10, 0, 0, 0, 0, 0, 0, 0, 1016, 966, 1055, 1005, 973, 1029, 1023, 1018, 19, 4, 0, 0, 0, 0, 0, 0, 1035, 1042, 995, 1039, 1049, 1042, 995, 998, 31, 0, 0, 0, 0, 0, 0, 0, 1006, 1037, 1033, 1051, 1008, 991, 962, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1059, 1010, 965, 1074, 996, 1032, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8276, 8154, 8311, 8108, 8226, 8126, 8216] avg=8192.0000 max=8311 max_rank=3 min=8108 min_rank=4 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1034, 1043, 1004, 1050, 1031, 1027, 1076, 1030, 7, 3, 0, 0, 0, 0, 0, 0, 1025, 1023, 1018, 987, 1075, 1011, 1043, 990, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1024, 1037, 1008, 1047, 1023, 984, 1068, 14, 0, 0, 0, 0, 0, 0, 0, 1032, 1059, 981, 1003, 983, 1009, 989, 1050, 14, 0, 0, 0, 0, 0, 0, 0, 1034, 1063, 1055, 1014, 1062, 968, 1051, 1021, 22, 9, 0, 0, 0, 0, 0, 0, 1035, 1075, 1009, 1048, 1029, 1000, 1014, 997, 37, 0, 0, 0, 0, 0, 0, 0, 1055, 1030, 1000, 997, 1009, 1013, 973, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1029, 1027, 980, 1013, 1040, 1025, 989, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 8172, 8178, 8120, 8299, 8244, 8119, 8099] avg=8192.0000 max=8305 max_rank=0 min=8099 min_rank=7 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8064, 8448, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8320, 8192, 8320, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8064, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 7936, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1000, 1004, 982, 1029, 1004, 1030, 1030, 998, 1068, 984, 1043, 1067, 995, 1023, 1028, 1060, 1041, 1008, 1018, 1039, 1005, 1018, 1049, 1015, 992, 1025, 1010, 1014, 1002, 1059, 1041, 1042, 1099, 1013, 1067, 974, 1011, 1015, 1021, 1007, 1006, 1023, 985, 1016, 998, 1036, 1035, 1062, 1013, 1023, 1066, 1063, 994, 979, 1059, 998, 1030, 1042, 1022, 1069, 1061, 1014, 984, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1040, 968, 981, 1044, 1053, 1018, 1048, 1036, 1011, 1006, 1038, 1007, 1019, 1053, 1078, 1072, 1014, 1014, 992, 1062, 1055, 1072, 1053, 1003, 1033, 991, 1015, 1053, 987, 954, 1012, 1013, 1016, 1004, 1035, 1009, 1020, 996, 1048, 1053, 1015, 1076, 978, 1014, 1059, 1051, 1073, 1019, 1035, 972, 1027, 987, 1001, 1028, 991, 1022, 1003, 1029, 1045, 1039, 1017, 1054, 1051, 974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1037, 1046, 1013, 998, 1039, 1016, 1054, 1014, 1012, 983, 1009, 1033, 1043, 1013, 1047, 1033, 1031, 1026, 1014, 1021, 1057, 992, 1037, 1024, 1071, 1056, 1044, 1005, 967, 1065, 1045, 1051, 1034, 1042, 985, 1034, 994, 1071, 1035, 1029, 997, 1097, 1043, 1000, 986, 1007, 1020, 1008, 1023, 1022, 981, 1014, 981, 1006, 1017, 1065, 1042, 1030, 1059, 959, 1014, 1024, 1006, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[992, 1031, 999, 1008, 999, 1004, 976, 1028, 1033, 1016, 1046, 1065, 1037, 1012, 1037, 1063, 1042, 1021, 1029, 1078, 1054, 994, 1022, 1046, 978, 1039, 1022, 1024, 1063, 1005, 1018, 1059, 1000, 1070, 1006, 1011, 1001, 1033, 1014, 964, 1047, 1038, 1035, 1019, 1041, 1007, 1005, 979, 1032, 1038, 1068, 1047, 1052, 982, 1027, 1026, 1001, 1029, 1028, 1026, 978, 1061, 1020, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1015, 1028, 996, 1008, 1022, 967, 1052, 1031, 1032, 1019, 975, 1084, 1039, 1035, 1036, 1018, 1036, 980, 1050, 1056, 997, 1043, 1067, 997, 1078, 1004, 985, 1033, 1059, 955, 999, 997, 1077, 984, 1018, 1001, 1042, 1043, 973, 1016, 1023, 1017, 1062, 997, 1029, 990, 1029, 1008, 1049, 1026, 1063, 996, 1033, 1046, 1045, 1066, 1023, 1004, 1057, 1025, 1035, 1041, 1017, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1030, 1018, 1040, 941, 1013, 1021, 1020, 1063, 1044, 1034, 992, 997, 1012, 1031, 1049, 1033, 1013, 1041, 1057, 995, 1088, 1056, 1090, 1020, 1019, 1018, 1047, 1039, 1019, 990, 1054, 1038, 1024, 1028, 987, 1009, 1029, 1009, 1017, 1103, 1024, 953, 1044, 1011, 993, 1007, 978, 1004, 1052, 994, 1023, 1052, 1000, 1011, 1031, 1049, 1029, 1034, 981, 1041, 1003, 1070, 980, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[995, 980, 1058, 1045, 1023, 946, 1013, 1059, 1048, 1035, 1011, 1024, 1071, 1026, 968, 942, 998, 1015, 1027, 1024, 1090, 1011, 1037, 1038, 974, 1067, 1042, 1029, 1042, 987, 1038, 1065, 992, 1028, 1004, 987, 1076, 983, 1057, 998, 1060, 1065, 1009, 1007, 1069, 1025, 993, 992, 1036, 1004, 1067, 994, 1057, 1029, 1050, 1046, 1072, 1011, 1043, 997, 1049, 1033, 975, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1018, 995, 1042, 1019, 1038, 1010, 984, 1037, 1042, 1024, 1015, 1041, 1004, 1017, 1041, 1064, 974, 1037, 1029, 1068, 1009, 1025, 1015, 1024, 1056, 987, 1040, 1019, 995, 1040, 993, 1084, 1005, 1041, 1051, 1048, 1016, 1052, 1051, 1020, 985, 1010, 982, 1036, 1063, 963, 1034, 1000, 1054, 1051, 998, 1023, 1022, 1070, 1030, 1049, 1052, 1003, 1008, 1013, 971, 989, 1026, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1000, 1004, 982, 1029, 1004, 1030, 1030, 998, 52, 9, 0, 0, 0, 0, 0, 0, 1068, 984, 1043, 1029, 995, 1023, 1028, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1008, 1018, 1039, 1005, 1018, 997, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1025, 1010, 1014, 1002, 1059, 1041, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 1013, 1067, 974, 1011, 1015, 1021, 1007, 5, 1, 0, 0, 0, 0, 0, 0, 1006, 1023, 985, 1016, 998, 1036, 1035, 1062, 29, 8, 0, 0, 0, 0, 0, 0, 1013, 1023, 1066, 1063, 994, 979, 1059, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1042, 1022, 1069, 1061, 1014, 984, 1028, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8230, 8141, 8184, 8213, 8198, 8170, 8262] avg=8192.0000 max=8262 max_rank=7 min=8138 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[992, 1031, 999, 1008, 999, 1004, 976, 1028, 50, 9, 0, 0, 0, 0, 0, 0, 1033, 1016, 1046, 1027, 1037, 1012, 1037, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1021, 1029, 1078, 1054, 994, 972, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1039, 1022, 1024, 1063, 1005, 1018, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1070, 1006, 1011, 1001, 1033, 1014, 964, 5, 1, 0, 0, 0, 0, 0, 0, 1047, 1038, 1035, 1019, 1041, 1007, 1005, 979, 29, 8, 0, 0, 0, 0, 0, 0, 1032, 1038, 1068, 1047, 1052, 982, 1027, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1029, 1028, 1026, 978, 1061, 1020, 1011, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8271, 8236, 8207, 8105, 8208, 8247, 8166] avg=8192.0000 max=8271 max_rank=1 min=8096 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1030, 1018, 1040, 941, 1013, 1021, 1020, 1063, 54, 8, 0, 0, 0, 0, 0, 0, 1044, 1034, 992, 962, 1012, 1031, 1049, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1041, 1057, 995, 1088, 1056, 1036, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1018, 1047, 1039, 1019, 990, 1054, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1028, 987, 1009, 1029, 1009, 1017, 1103, 5, 1, 0, 0, 0, 0, 0, 0, 1024, 953, 1044, 1011, 993, 1007, 978, 1004, 27, 8, 0, 0, 0, 0, 0, 0, 1052, 994, 1023, 1052, 1000, 1011, 1031, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1034, 981, 1041, 1003, 1070, 980, 1044, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8157, 8306, 8223, 8212, 8049, 8187, 8194] avg=8192.0000 max=8306 max_rank=2 min=8049 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32642 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1040, 968, 981, 1044, 1053, 1018, 1048, 1036, 52, 8, 0, 0, 0, 0, 0, 0, 1011, 1006, 1038, 972, 1019, 1053, 1078, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1014, 992, 1062, 1055, 1072, 1001, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 991, 1015, 1053, 987, 954, 1012, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1004, 1035, 1009, 1020, 996, 1048, 1053, 5, 1, 0, 0, 0, 0, 0, 0, 1015, 1076, 978, 1014, 1059, 1051, 1073, 1019, 27, 8, 0, 0, 0, 0, 0, 0, 1035, 972, 1027, 987, 1001, 1028, 991, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1029, 1045, 1039, 1017, 1054, 1051, 974, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8249, 8213, 8057, 8187, 8320, 8038, 8224] avg=8192.0000 max=8320 max_rank=5 min=8038 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1037, 1046, 1013, 998, 1039, 1016, 1054, 1014, 51, 9, 0, 0, 0, 0, 0, 0, 1012, 983, 1009, 996, 1043, 1013, 1047, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1026, 1014, 1021, 1057, 992, 986, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1056, 1044, 1005, 967, 1065, 1045, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1042, 985, 1034, 994, 1071, 1035, 1029, 5, 1, 0, 0, 0, 0, 0, 0, 997, 1097, 1043, 1000, 986, 1007, 1020, 1008, 28, 9, 0, 0, 0, 0, 0, 0, 1023, 1022, 981, 1014, 981, 1006, 1017, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1030, 1059, 959, 1014, 1024, 1006, 1015, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8136, 8151, 8303, 8230, 8195, 8082, 8162] avg=8192.0000 max=8303 max_rank=3 min=8082 min_rank=6 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1015, 1028, 996, 1008, 1022, 967, 1052, 1031, 53, 9, 0, 0, 0, 0, 0, 0, 1032, 1019, 975, 1046, 1039, 1035, 1036, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 980, 1050, 1056, 997, 1043, 1014, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1004, 985, 1033, 1059, 955, 999, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 984, 1018, 1001, 1042, 1043, 973, 1016, 5, 1, 0, 0, 0, 0, 0, 0, 1023, 1017, 1062, 997, 1029, 990, 1029, 1008, 29, 9, 0, 0, 0, 0, 0, 0, 1049, 1026, 1063, 996, 1033, 1046, 1045, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1004, 1057, 1025, 1035, 1041, 1017, 1008, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8200, 8173, 8109, 8160, 8193, 8297, 8223] avg=8192.0000 max=8297 max_rank=6 min=8109 min_rank=3 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[995, 980, 1058, 1045, 1023, 946, 1013, 1059, 51, 8, 0, 0, 0, 0, 0, 0, 1048, 1035, 1011, 988, 1071, 1026, 968, 942, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1015, 1027, 1024, 1090, 1011, 986, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1067, 1042, 1029, 1042, 987, 1038, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1028, 1004, 987, 1076, 983, 1057, 998, 5, 1, 0, 0, 0, 0, 0, 0, 1060, 1065, 1009, 1007, 1069, 1025, 993, 992, 28, 8, 0, 0, 0, 0, 0, 0, 1036, 1004, 1067, 994, 1057, 1029, 1050, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1011, 1043, 997, 1049, 1033, 975, 1000, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8089, 8189, 8243, 8131, 8256, 8258, 8192] avg=8192.0000 max=8258 max_rank=6 min=8089 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1018, 995, 1042, 1019, 1038, 1010, 984, 1037, 54, 13, 0, 0, 0, 0, 0, 0, 1042, 1024, 1015, 997, 1004, 1017, 1041, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1037, 1029, 1068, 1009, 1025, 961, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 987, 1040, 1019, 995, 1040, 993, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1041, 1051, 1048, 1016, 1052, 1051, 1020, 10, 4, 0, 0, 0, 0, 0, 0, 985, 1010, 982, 1036, 1063, 963, 1034, 1000, 31, 13, 0, 0, 0, 0, 0, 0, 1054, 1051, 998, 1023, 1022, 1070, 1030, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1003, 1008, 1013, 971, 989, 1026, 1034, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8204, 8127, 8210, 8298, 8117, 8257, 8113] avg=8192.0000 max=8298 max_rank=4 min=8113 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8064, 8192, 8320, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8064, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8064, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8448, 8448, 8320, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8064, 8320, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1049, 985, 1032, 1057, 1036, 1021, 1059, 1037, 1035, 1005, 1020, 1018, 1073, 1001, 1032, 1037, 1049, 973, 1004, 983, 994, 1047, 1029, 1040, 1018, 1034, 1024, 1086, 960, 1036, 1060, 1012, 1008, 966, 990, 987, 1019, 1041, 992, 1091, 1037, 1025, 1044, 1009, 964, 1047, 1004, 1039, 1068, 1054, 1057, 1009, 1012, 1006, 1021, 1052, 991, 1043, 1029, 1049, 1074, 978, 994, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[973, 1022, 991, 1036, 1041, 987, 1048, 1033, 1016, 1010, 1050, 1015, 1052, 998, 1010, 980, 1031, 964, 1012, 1062, 1067, 1069, 1011, 1050, 980, 1021, 1022, 1013, 1057, 1047, 1068, 1022, 1063, 1019, 983, 1018, 1051, 1001, 996, 1032, 1056, 986, 1039, 1013, 1017, 1067, 984, 1023, 1057, 981, 1032, 1028, 1088, 999, 1028, 1037, 1049, 988, 1027, 1027, 1039, 1023, 948, 1079] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[995, 1017, 995, 1049, 984, 1045, 1034, 1029, 1020, 1041, 1033, 1030, 1053, 1004, 1074, 1042, 992, 1008, 1021, 1030, 1063, 1033, 1025, 1049, 1016, 1018, 1030, 1006, 1024, 1048, 985, 1065, 1031, 987, 1023, 1018, 1058, 1036, 967, 1018, 990, 1030, 1046, 1013, 1047, 1009, 1046, 999, 1062, 1043, 987, 1036, 1027, 1024, 1003, 1013, 1062, 1051, 1003, 992, 1010, 992, 1017, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1052, 1006, 1042, 1011, 1028, 1029, 1061, 1060, 1010, 1012, 990, 1036, 984, 1014, 1026, 1059, 1025, 1038, 1015, 1073, 1018, 1010, 975, 988, 1051, 1056, 1037, 1076, 1034, 1022, 998, 1020, 1037, 992, 1065, 937, 1035, 1053, 1040, 1042, 1023, 1056, 982, 1046, 1051, 1005, 991, 1021, 1086, 1033, 970, 998, 1020, 1055, 1033, 1026, 971, 1019, 1004, 1013, 1006, 1002, 1070, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1035, 1005, 1052, 1011, 1052, 1056, 1028, 1034, 1040, 1003, 1007, 1053, 1007, 1034, 1007, 1028, 1081, 1034, 1027, 1039, 1047, 1052, 1043, 1004, 995, 993, 1038, 1033, 965, 977, 1016, 1012, 1042, 995, 993, 1043, 1087, 1042, 1007, 1005, 1018, 1006, 1040, 1011, 1025, 1015, 978, 1051, 1054, 1023, 1031, 1042, 1058, 1035, 1072, 991, 1014, 999, 988, 1017, 970, 1002, 1025, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[997, 1004, 1017, 1038, 1050, 1056, 1010, 1002, 980, 1069, 991, 1034, 1033, 1026, 1028, 1052, 1030, 995, 1021, 1001, 1026, 1028, 1014, 990, 1045, 1048, 1036, 1065, 1044, 1050, 1012, 993, 1022, 1037, 1056, 1049, 988, 1012, 996, 1007, 1035, 996, 1003, 1025, 1077, 1019, 1018, 1029, 1007, 1027, 1037, 1019, 1041, 994, 986, 995, 1063, 1037, 1074, 1035, 1032, 1011, 1036, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1015, 1028, 1007, 1053, 1034, 1008, 1019, 1025, 1030, 1020, 1005, 984, 1016, 1038, 978, 1025, 1003, 1027, 985, 1044, 1004, 1000, 1057, 962, 1054, 1051, 1077, 1046, 980, 1037, 957, 1012, 1076, 1017, 1052, 1011, 996, 1075, 1014, 1063, 1056, 1049, 968, 1081, 1012, 982, 1026, 965, 1054, 1033, 1087, 990, 1031, 979, 1013, 1034, 1072, 1021, 1025, 1059, 988, 1071, 994, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[979, 1049, 1016, 1036, 1066, 1040, 1024, 1081, 1052, 1001, 1030, 1034, 1022, 1027, 1013, 1069, 998, 1050, 970, 1033, 1060, 1034, 1028, 1022, 1050, 977, 1016, 1021, 1017, 987, 1035, 1030, 1061, 974, 1024, 1015, 1036, 1005, 1004, 1033, 969, 1062, 1012, 1043, 1048, 1036, 994, 1017, 1045, 1036, 1015, 994, 1031, 1020, 1028, 1002, 988, 992, 1034, 1062, 1020, 1029, 1029, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1049, 985, 1032, 1057, 1036, 1021, 1059, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1005, 1020, 1018, 1073, 1001, 1032, 1037, 2, 0, 0, 0, 0, 0, 0, 0, 1049, 973, 1004, 983, 994, 1047, 1029, 1040, 7, 2, 0, 0, 0, 0, 0, 0, 1018, 1034, 1024, 1077, 960, 1036, 1060, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 966, 990, 987, 1019, 1041, 992, 1091, 13, 0, 0, 0, 0, 0, 0, 0, 1037, 1025, 1044, 1009, 964, 1047, 1004, 1039, 11, 8, 0, 0, 0, 0, 0, 0, 1042, 1054, 1057, 1009, 1012, 1006, 1021, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1043, 1029, 1049, 1074, 978, 994, 990, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8223, 8128, 8221, 8107, 8188, 8253, 8168] avg=8192.0000 max=8253 max_rank=6 min=8107 min_rank=4 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[973, 1022, 991, 1036, 1041, 987, 1048, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1010, 1050, 1015, 1052, 998, 1010, 980, 2, 0, 0, 0, 0, 0, 0, 0, 1031, 964, 1012, 1062, 1067, 1069, 1011, 1050, 7, 2, 0, 0, 0, 0, 0, 0, 980, 1021, 1022, 1004, 1057, 1047, 1068, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1019, 983, 1018, 1051, 1001, 996, 1032, 12, 0, 0, 0, 0, 0, 0, 0, 1056, 986, 1039, 1013, 1017, 1067, 984, 1023, 11, 8, 0, 0, 0, 0, 0, 0, 1032, 981, 1032, 1028, 1088, 999, 1028, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 988, 1027, 1027, 1039, 1023, 948, 1079, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8133, 8275, 8221, 8175, 8204, 8225, 8200] avg=8192.0000 max=8275 max_rank=2 min=8103 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1035, 1005, 1052, 1011, 1052, 1056, 1028, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1003, 1007, 1053, 1007, 1034, 1007, 1028, 2, 0, 0, 0, 0, 0, 0, 0, 1081, 1034, 1027, 1039, 1047, 1052, 1043, 1004, 7, 2, 0, 0, 0, 0, 0, 0, 995, 993, 1038, 1024, 965, 977, 1016, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 995, 993, 1043, 1087, 1042, 1007, 1005, 12, 0, 0, 0, 0, 0, 0, 0, 1018, 1006, 1040, 1011, 1025, 1015, 978, 1051, 11, 8, 0, 0, 0, 0, 0, 0, 1029, 1023, 1031, 1042, 1058, 1035, 1072, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 999, 988, 1017, 970, 1002, 1025, 1049, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8181, 8336, 8020, 8226, 8163, 8281, 8084] avg=8192.0000 max=8336 max_rank=2 min=8020 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[995, 1017, 995, 1049, 984, 1045, 1034, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1041, 1033, 1030, 1053, 1004, 1074, 1042, 2, 0, 0, 0, 0, 0, 0, 0, 992, 1008, 1021, 1030, 1063, 1033, 1025, 1049, 7, 2, 0, 0, 0, 0, 0, 0, 1016, 1018, 1030, 997, 1024, 1048, 985, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 987, 1023, 1018, 1058, 1036, 967, 1018, 12, 0, 0, 0, 0, 0, 0, 0, 990, 1030, 1046, 1013, 1047, 1009, 1046, 999, 11, 8, 0, 0, 0, 0, 0, 0, 1037, 1043, 987, 1036, 1027, 1024, 1003, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1051, 1003, 992, 1010, 992, 1017, 1038, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8299, 8230, 8183, 8150, 8199, 8170, 8185] avg=8192.0000 max=8299 max_rank=1 min=8120 min_rank=0 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1052, 1006, 1042, 1011, 1028, 1029, 1061, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1012, 990, 1036, 984, 1014, 1026, 1059, 2, 0, 0, 0, 0, 0, 0, 0, 1025, 1038, 1015, 1073, 1018, 1010, 975, 988, 7, 2, 0, 0, 0, 0, 0, 0, 1051, 1056, 1037, 1067, 1034, 1022, 998, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 992, 1065, 937, 1035, 1053, 1040, 1042, 13, 0, 0, 0, 0, 0, 0, 0, 1023, 1056, 982, 1046, 1051, 1005, 991, 1021, 11, 9, 0, 0, 0, 0, 0, 0, 1060, 1033, 970, 998, 1020, 1055, 1033, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1019, 1004, 1013, 1006, 1002, 1070, 998, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8133, 8151, 8285, 8214, 8195, 8195, 8103] avg=8192.0000 max=8285 max_rank=3 min=8103 min_rank=7 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[997, 1004, 1017, 1038, 1050, 1056, 1010, 975, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1069, 991, 1034, 1033, 1026, 1028, 1052, 2, 0, 0, 0, 0, 0, 0, 0, 1030, 995, 1021, 1001, 1026, 1028, 1014, 990, 7, 2, 0, 0, 0, 0, 0, 0, 1045, 1048, 1036, 1056, 1044, 1050, 1012, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1037, 1056, 1049, 988, 1012, 996, 1007, 12, 0, 0, 0, 0, 0, 0, 0, 1035, 996, 1003, 1025, 1077, 1019, 1018, 1029, 10, 8, 0, 0, 0, 0, 0, 0, 983, 1027, 1037, 1019, 1041, 994, 986, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1037, 1074, 1035, 1032, 1011, 1036, 988, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8215, 8114, 8284, 8179, 8220, 8082, 8295] avg=8192.0000 max=8295 max_rank=7 min=8082 min_rank=6 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1015, 1028, 1007, 1053, 1034, 1008, 1019, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1020, 1005, 984, 1016, 1038, 978, 1025, 6, 0, 0, 0, 0, 0, 0, 0, 1003, 1027, 985, 1044, 1004, 1000, 1057, 962, 10, 6, 0, 0, 0, 0, 0, 0, 1054, 1051, 1077, 1030, 980, 1037, 957, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1017, 1052, 1011, 996, 1075, 1014, 1063, 17, 0, 0, 0, 0, 0, 0, 0, 1056, 1049, 968, 1081, 1012, 982, 1026, 965, 14, 13, 0, 0, 0, 0, 0, 0, 1017, 1033, 1087, 990, 1031, 979, 1013, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1021, 1025, 1059, 988, 1071, 994, 1061, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8102, 8098, 8198, 8321, 8166, 8184, 8315] avg=8192.0000 max=8321 max_rank=4 min=8098 min_rank=2 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32986 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[979, 1049, 1016, 1036, 1066, 1040, 1024, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1001, 1030, 1034, 1022, 1027, 1013, 1069, 2, 0, 0, 0, 0, 0, 0, 0, 998, 1050, 970, 1033, 1060, 1034, 1028, 1022, 7, 2, 0, 0, 0, 0, 0, 0, 1050, 977, 1016, 1012, 1017, 987, 1035, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 974, 1024, 1015, 1036, 1005, 1004, 1033, 12, 0, 0, 0, 0, 0, 0, 0, 969, 1062, 1012, 1043, 1048, 1036, 994, 1017, 11, 9, 0, 0, 0, 0, 0, 0, 1020, 1036, 1015, 994, 1031, 1020, 1028, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 988, 992, 1034, 1062, 1020, 1029, 1029, 1011, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8250, 8204, 8124, 8164, 8201, 8146, 8186] avg=8192.0000 max=8261 max_rank=0 min=8124 min_rank=3 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8064, 8320, 8320, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8064, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8448, 8064, 8192, 8192, 8320, 8320, 8064, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1040, 1031, 1041, 1039, 1011, 1026, 1050, 1059, 1032, 1046, 1031, 1028, 1049, 1024, 1036, 973, 1037, 975, 1018, 1057, 1031, 1051, 957, 965, 1083, 1034, 1002, 1028, 1012, 996, 1022, 1000, 1017, 1044, 1032, 1017, 964, 996, 1021, 1018, 1046, 1027, 1008, 1045, 1020, 1005, 1028, 1055, 1021, 1047, 1033, 1062, 1024, 999, 1009, 1015, 1035, 1040, 1052, 1031, 979, 1061, 986, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1117, 1018, 1037, 1046, 1106, 997, 1026, 1014, 1057, 1026, 1041, 1059, 1078, 1026, 1005, 1046, 995, 1016, 993, 984, 1062, 967, 950, 1041, 1049, 1020, 1027, 1020, 1064, 949, 1044, 1049, 1038, 1024, 1038, 998, 1057, 1046, 1018, 1093, 1066, 1003, 1030, 1012, 1025, 998, 935, 1012, 985, 1046, 1006, 971, 1018, 1002, 1009, 1011, 1035, 1015, 1045, 990, 1032, 1031, 1036, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1029, 1045, 994, 982, 1023, 1065, 1018, 976, 1024, 998, 988, 1027, 1000, 985, 978, 1049, 1054, 1002, 1081, 1074, 1024, 1049, 1044, 987, 1020, 1058, 1019, 1003, 1050, 1074, 1012, 1066, 1079, 1030, 1003, 995, 979, 1044, 1020, 1004, 1061, 1046, 1007, 1034, 1028, 1037, 951, 1053, 1050, 996, 1022, 1007, 1058, 995, 1027, 1013, 1027, 1011, 1016, 1005, 1044, 1002, 1055, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1007, 1030, 1080, 1032, 1000, 990, 1068, 1003, 988, 1003, 1009, 1015, 1027, 1019, 1052, 1022, 1002, 1027, 1007, 997, 1016, 1022, 1008, 1073, 1031, 1045, 1021, 1039, 1014, 1040, 1004, 970, 1024, 1013, 1051, 1045, 998, 1061, 1029, 1028, 1006, 1049, 1016, 1042, 1056, 997, 1060, 1010, 1017, 1036, 1048, 1024, 1034, 1005, 1078, 1036, 983, 1002, 1019, 1014, 1016, 1016, 1050, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1052, 965, 1009, 988, 1023, 988, 1039, 1048, 1050, 1004, 1010, 1078, 1019, 984, 1068, 994, 1058, 1034, 1045, 1051, 1037, 1018, 989, 1030, 1016, 996, 1053, 1011, 1028, 986, 1033, 1041, 1084, 1049, 1046, 1032, 1027, 991, 1059, 1032, 1007, 1063, 1008, 1020, 992, 992, 1064, 1009, 1001, 1017, 1051, 993, 1031, 1004, 1016, 985, 1044, 1078, 1020, 1034, 1024, 993, 1024, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1041, 1036, 1023, 1048, 1001, 1018, 1052, 1016, 1023, 1064, 1077, 1013, 1045, 1048, 1020, 1043, 1001, 1051, 1022, 1007, 1005, 1007, 993, 1064, 1029, 1046, 1024, 1020, 1039, 1004, 1063, 999, 984, 1045, 1007, 1002, 1037, 1037, 1017, 1055, 988, 1032, 1010, 999, 998, 1000, 1005, 1021, 1045, 1074, 1035, 1008, 1030, 1024, 1071, 973, 983, 1000, 1008, 993, 1044, 1013, 1040, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1047, 1032, 982, 1022, 1052, 1008, 1047, 1011, 956, 978, 1024, 1024, 1043, 1041, 1015, 1019, 1009, 1022, 1029, 1043, 1039, 1012, 1049, 1063, 1005, 978, 1004, 1017, 1027, 1038, 991, 1051, 1049, 1033, 1011, 1018, 1104, 1067, 995, 1016, 1049, 1003, 1000, 1002, 1040, 1009, 1048, 1025, 1020, 1023, 1020, 1029, 968, 1027, 1036, 1008, 1068, 1008, 1024, 1012, 1039, 1011, 1037, 1059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[971, 1046, 1026, 1019, 1038, 1043, 1024, 1051, 1052, 1046, 1083, 1004, 1088, 1051, 1003, 1011, 1047, 1007, 1000, 1018, 1000, 1009, 1059, 970, 1042, 1002, 1033, 1058, 1029, 990, 1073, 1012, 1015, 1039, 1014, 1051, 1024, 995, 1011, 964, 998, 1008, 1058, 1058, 1020, 995, 1011, 1039, 1009, 1003, 1034, 1017, 1053, 1039, 1060, 1005, 977, 1025, 985, 1025, 1034, 998, 1010, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1007, 1030, 1080, 1032, 1000, 990, 1039, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1003, 1009, 1015, 1006, 1019, 1052, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1027, 1007, 997, 1016, 1022, 1008, 1073, 18, 4, 0, 0, 0, 0, 0, 0, 1023, 1045, 1021, 1039, 1014, 1040, 1004, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1013, 1051, 1045, 998, 1061, 1029, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1049, 1016, 1042, 1056, 997, 1060, 1010, 25, 0, 0, 0, 0, 0, 0, 0, 1017, 1036, 1048, 1024, 1034, 1005, 1078, 1036, 8, 6, 0, 0, 0, 0, 0, 0, 983, 1002, 1019, 1014, 1016, 1016, 1050, 1012, 15, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8114, 8174, 8156, 8225, 8261, 8292, 8133] avg=8192.0000 max=8292 max_rank=6 min=8114 min_rank=1 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32911 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1029, 1045, 994, 982, 1023, 1065, 990, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 998, 988, 1027, 979, 985, 978, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1002, 1081, 1074, 1024, 1049, 1044, 987, 19, 4, 0, 0, 0, 0, 0, 0, 1012, 1058, 1019, 1003, 1050, 1074, 1012, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1030, 1003, 995, 979, 1044, 1020, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1046, 1007, 1034, 1028, 1037, 951, 1053, 24, 0, 0, 0, 0, 0, 0, 0, 1050, 996, 1022, 1007, 1058, 995, 1027, 1013, 8, 6, 0, 0, 0, 0, 0, 0, 1027, 1011, 1016, 1005, 1044, 1002, 1055, 1039, 15, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8028, 8338, 8294, 8129, 8241, 8182, 8220] avg=8192.0000 max=8338 max_rank=2 min=8028 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1052, 965, 1009, 988, 1023, 988, 1011, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1004, 1010, 1078, 998, 984, 1068, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1034, 1045, 1051, 1037, 1018, 989, 1030, 19, 4, 0, 0, 0, 0, 0, 0, 1008, 996, 1053, 1011, 1028, 986, 1033, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1049, 1046, 1032, 1027, 991, 1059, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1063, 1008, 1020, 992, 992, 1064, 1009, 24, 0, 0, 0, 0, 0, 0, 0, 1001, 1017, 1051, 993, 1031, 1004, 1016, 985, 8, 6, 0, 0, 0, 0, 0, 0, 1044, 1078, 1020, 1034, 1024, 993, 1024, 1001, 15, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8084, 8186, 8285, 8156, 8295, 8179, 8112, 8239] avg=8192.0000 max=8295 max_rank=4 min=8084 min_rank=0 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1117, 1018, 1037, 1046, 1106, 997, 998, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1026, 1041, 1059, 1055, 1026, 1005, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1016, 993, 984, 1062, 967, 950, 1041, 18, 4, 0, 0, 0, 0, 0, 0, 1041, 1020, 1027, 1020, 1064, 949, 1044, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1024, 1038, 998, 1057, 1046, 1018, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1003, 1030, 1012, 1025, 998, 935, 1012, 24, 0, 0, 0, 0, 0, 0, 0, 985, 1046, 1006, 971, 1018, 1002, 1009, 1011, 8, 7, 0, 0, 0, 0, 0, 0, 1035, 1015, 1045, 990, 1032, 1031, 1036, 982, 16, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8333, 8315, 8030, 8214, 8288, 8105, 8063, 8188] avg=8192.0000 max=8333 max_rank=0 min=8030 min_rank=2 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1040, 1031, 1041, 1039, 1011, 1026, 1022, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1046, 1031, 1028, 1027, 1024, 1036, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 975, 1018, 1057, 1031, 1051, 957, 965, 18, 4, 0, 0, 0, 0, 0, 0, 1075, 1034, 1002, 1028, 1012, 996, 1022, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1044, 1032, 1017, 964, 996, 1021, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1027, 1008, 1045, 1020, 1005, 1028, 1055, 24, 0, 0, 0, 0, 0, 0, 0, 1021, 1047, 1033, 1062, 1024, 999, 1009, 1015, 8, 7, 0, 0, 0, 0, 0, 0, 1035, 1040, 1052, 1031, 979, 1061, 986, 1015, 15, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8197, 8113, 8169, 8085, 8258, 8225, 8220] avg=8192.0000 max=8269 max_rank=0 min=8085 min_rank=4 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1041, 1036, 1023, 1048, 1001, 1018, 1024, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1064, 1077, 1013, 1023, 1048, 1020, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1051, 1022, 1007, 1005, 1007, 993, 1064, 17, 4, 0, 0, 0, 0, 0, 0, 1021, 1046, 1024, 1020, 1039, 1004, 1063, 999, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1045, 1007, 1002, 1037, 1037, 1017, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1032, 1010, 999, 998, 1000, 1005, 1021, 24, 0, 0, 0, 0, 0, 0, 0, 1045, 1074, 1035, 1008, 1030, 1024, 1071, 973, 8, 7, 0, 0, 0, 0, 0, 0, 983, 1000, 1008, 993, 1044, 1013, 1040, 1016, 15, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8311, 8171, 8216, 8162, 8077, 8275, 8117] avg=8192.0000 max=8311 max_rank=1 min=8077 min_rank=5 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1047, 1032, 982, 1022, 1052, 1008, 1019, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 956, 978, 1024, 1024, 1022, 1041, 1015, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1022, 1029, 1043, 1039, 1012, 1049, 1063, 18, 4, 0, 0, 0, 0, 0, 0, 997, 978, 1004, 1017, 1027, 1038, 991, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1033, 1011, 1018, 1104, 1067, 995, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1003, 1000, 1002, 1040, 1009, 1048, 1025, 24, 0, 0, 0, 0, 0, 0, 0, 1020, 1023, 1020, 1029, 968, 1027, 1036, 1008, 8, 6, 0, 0, 0, 0, 0, 0, 1068, 1008, 1024, 1012, 1039, 1011, 1037, 1059, 15, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8079, 8288, 8103, 8269, 8200, 8145, 8279] avg=8192.0000 max=8288 max_rank=2 min=8079 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[971, 1046, 1026, 1019, 1038, 1043, 991, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1046, 1083, 1004, 1056, 1051, 1003, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1007, 1000, 1018, 1000, 1009, 1059, 970, 22, 5, 0, 0, 0, 0, 0, 0, 1031, 1002, 1033, 1058, 1029, 990, 1073, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1039, 1014, 1051, 1024, 995, 1011, 964, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1008, 1058, 1058, 1020, 995, 1011, 1039, 28, 0, 0, 0, 0, 0, 0, 0, 1009, 1003, 1034, 1017, 1053, 1039, 1060, 1005, 11, 11, 0, 0, 0, 0, 0, 0, 977, 1025, 985, 1025, 1034, 998, 1010, 1057, 21, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8306, 8137, 8228, 8083, 8215, 8242, 8140] avg=8192.0000 max=8306 max_rank=1 min=8083 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8064, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8064, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1018, 1000, 1020, 1037, 1016, 1032, 1052, 1019, 957, 1005, 1014, 1023, 1002, 1072, 1009, 1033, 1038, 1005, 1008, 1067, 997, 1031, 987, 1033, 997, 982, 998, 1025, 1000, 1024, 1064, 977, 1007, 1032, 992, 1000, 1072, 1044, 985, 1047, 1054, 1018, 1060, 1022, 1048, 1015, 1071, 992, 1014, 1008, 1017, 1058, 1053, 1082, 1046, 1052, 1002, 1063, 1017, 1019, 1050, 1025, 980, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1060, 1083, 1039, 1074, 1027, 1060, 1033, 1048, 995, 1028, 1045, 1012, 1060, 982, 1052, 1000, 1026, 998, 1006, 1059, 1004, 1009, 1014, 1075, 1054, 1036, 1018, 1040, 1069, 1030, 1029, 1073, 1024, 1003, 1001, 1009, 949, 990, 1026, 1019, 1036, 979, 1017, 988, 1009, 987, 1049, 1032, 998, 1051, 966, 1039, 1004, 1032, 1033, 1040, 1002, 962, 965, 1052, 1058, 1050, 1020, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[961, 1032, 994, 1030, 1070, 998, 1008, 1023, 1089, 1001, 1009, 991, 1014, 988, 1012, 1051, 1066, 1011, 1040, 1056, 1042, 1024, 1034, 1065, 1034, 1007, 1005, 1015, 1052, 996, 995, 994, 994, 1078, 1032, 998, 1022, 989, 986, 1010, 1059, 1007, 1023, 1069, 1055, 1003, 1058, 1016, 993, 999, 1056, 1091, 1038, 1032, 1029, 1046, 1051, 1005, 1016, 1041, 1004, 1028, 976, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[972, 1049, 1036, 1011, 1089, 978, 1001, 984, 1031, 1019, 1011, 1012, 1021, 1063, 1044, 1052, 1022, 1025, 1045, 1027, 996, 997, 958, 1025, 1032, 1036, 1010, 1083, 1054, 1042, 1014, 995, 1040, 1033, 1014, 1091, 992, 1028, 1031, 1052, 1016, 1049, 1029, 1037, 1034, 1027, 1036, 1019, 1007, 1032, 1003, 1045, 1006, 1035, 1041, 1041, 962, 1039, 1007, 972, 1035, 992, 1058, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[984, 1037, 1052, 1003, 1060, 1089, 1054, 1016, 1008, 1022, 992, 1052, 1007, 1080, 1059, 1014, 985, 1024, 1056, 1012, 1066, 1039, 1019, 1010, 1073, 977, 1007, 963, 1004, 1027, 1007, 1031, 1032, 1081, 990, 1051, 985, 1041, 1031, 1080, 1033, 1015, 1021, 1002, 1008, 1002, 980, 1023, 1021, 1013, 1015, 1023, 1027, 1029, 1006, 1018, 1063, 985, 1016, 996, 1014, 1015, 1024, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1003, 1018, 1022, 984, 1047, 1001, 1055, 993, 1041, 1060, 1030, 988, 1019, 1037, 1044, 971, 1057, 997, 1003, 1041, 1035, 1039, 1020, 1046, 975, 992, 1061, 1044, 1108, 1015, 983, 1021, 1016, 978, 1034, 1033, 1069, 998, 1057, 1052, 1028, 1026, 1017, 1030, 1028, 1047, 1019, 1073, 1046, 1037, 1045, 987, 1015, 1009, 1028, 971, 1012, 1038, 987, 1008, 1032, 1064, 1002, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1036, 1031, 1017, 1006, 1032, 1038, 1086, 1050, 1042, 1037, 970, 1032, 1091, 1038, 1043, 1029, 963, 1034, 1066, 1041, 962, 1031, 992, 1045, 966, 1003, 979, 979, 999, 1073, 1036, 1065, 1031, 1037, 1048, 1024, 1040, 1022, 1011, 1016, 1022, 1039, 1041, 1045, 1032, 1013, 1018, 1068, 1023, 1046, 1030, 1032, 1037, 967, 1022, 1041, 1031, 1034, 1027, 1008, 973, 979, 998, 969] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1046, 1051, 998, 1043, 1021, 1026, 994, 1014, 1017, 995, 1055, 1005, 1044, 1056, 992, 1024, 1025, 1031, 1027, 1011, 975, 1012, 1036, 1041, 985, 1046, 1005, 999, 1036, 1043, 1000, 1023, 1050, 988, 1042, 1023, 998, 1009, 1056, 968, 1091, 1041, 1050, 1014, 1060, 1045, 1018, 1040, 1013, 1043, 1011, 1056, 986, 1068, 991, 1033, 1023, 1001, 1015, 1061, 1018, 1003, 1025, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[961, 1032, 994, 1030, 1042, 998, 1008, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 1001, 1009, 991, 1014, 983, 1012, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1011, 1040, 1056, 1042, 1024, 1034, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1007, 1005, 1015, 1052, 996, 995, 994, 17, 12, 0, 0, 0, 0, 0, 0, 994, 1078, 1032, 998, 1022, 989, 986, 1010, 5, 0, 0, 0, 0, 0, 0, 0, 1013, 1007, 1023, 1069, 1055, 1003, 1058, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 993, 999, 1056, 1074, 1038, 1032, 1029, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1005, 1016, 1041, 1004, 1028, 976, 1025, 46, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8088, 8150, 8338, 8127, 8114, 8244, 8267, 8208] avg=8192.0000 max=8338 max_rank=2 min=8088 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1060, 1083, 1039, 1074, 1001, 1060, 1033, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1028, 1045, 1012, 1060, 977, 1052, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 998, 1006, 1059, 1004, 1009, 1014, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1036, 1018, 1040, 1069, 1030, 1029, 1073, 16, 11, 0, 0, 0, 0, 0, 0, 1024, 1003, 1001, 1009, 949, 990, 1026, 1019, 5, 0, 0, 0, 0, 0, 0, 0, 991, 979, 1017, 988, 1009, 987, 1049, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1051, 966, 1023, 1004, 1032, 1033, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 962, 965, 1052, 1058, 1050, 1020, 1008, 45, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8398, 8169, 8191, 8376, 8026, 8052, 8147, 8177] avg=8192.0000 max=8398 max_rank=0 min=8026 min_rank=4 imbalance_ratio=1.025146 max_load_reduction=57138 max_load_reduction_pct=0.871857 moved_to_echo=32402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1018, 1000, 1020, 1037, 990, 1032, 1052, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 957, 1005, 1014, 1023, 1002, 1066, 1009, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1005, 1008, 1067, 997, 1031, 987, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 997, 982, 998, 1025, 1000, 1024, 1064, 977, 17, 11, 0, 0, 0, 0, 0, 0, 1007, 1032, 992, 1000, 1072, 1044, 985, 1047, 6, 0, 0, 0, 0, 0, 0, 0, 1008, 1018, 1060, 1022, 1048, 1015, 1071, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1008, 1017, 1041, 1053, 1082, 1046, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1063, 1017, 1019, 1050, 1025, 980, 1049, 46, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8109, 8166, 8095, 8185, 8234, 8313, 8266] avg=8192.0000 max=8313 max_rank=6 min=8095 min_rank=3 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32998 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1003, 1018, 1022, 984, 1019, 1001, 1055, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1060, 1030, 988, 1019, 1031, 1044, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 997, 1003, 1041, 1035, 1039, 1020, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 975, 992, 1061, 1044, 1108, 1015, 983, 1021, 15, 12, 0, 0, 0, 0, 0, 0, 1016, 978, 1034, 1033, 1069, 998, 1057, 1052, 6, 0, 0, 0, 0, 0, 0, 0, 983, 1026, 1017, 1030, 1028, 1047, 1019, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1037, 1045, 972, 1015, 1009, 1028, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1038, 987, 1008, 1032, 1064, 1002, 1000, 45, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8184, 8238, 8226, 8243, 8223, 8123, 8204] avg=8192.0000 max=8243 max_rank=4 min=8095 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[984, 1037, 1052, 1003, 1032, 1089, 1054, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1022, 992, 1052, 1007, 1074, 1059, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1024, 1056, 1012, 1066, 1039, 1019, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 977, 1007, 963, 1004, 1027, 1007, 1031, 16, 12, 0, 0, 0, 0, 0, 0, 1032, 1081, 990, 1051, 985, 1041, 1031, 1080, 6, 0, 0, 0, 0, 0, 0, 0, 988, 1015, 1021, 1002, 1008, 1002, 980, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1013, 1015, 1007, 1027, 1029, 1006, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 985, 1016, 996, 1014, 1015, 1024, 1067, 45, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8228, 8211, 8117, 8297, 8039, 8136, 8241] avg=8192.0000 max=8297 max_rank=4 min=8039 min_rank=5 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[972, 1049, 1036, 1011, 1061, 978, 1001, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1019, 1011, 1012, 1021, 1057, 1044, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1025, 1045, 1027, 996, 997, 958, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1036, 1010, 1083, 1054, 1042, 1014, 995, 16, 12, 0, 0, 0, 0, 0, 0, 1040, 1033, 1014, 1091, 992, 1028, 1031, 1052, 6, 0, 0, 0, 0, 0, 0, 0, 972, 1049, 1029, 1037, 1034, 1027, 1036, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1032, 1003, 1029, 1006, 1035, 1041, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1039, 1007, 972, 1035, 992, 1058, 999, 44, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8092, 8247, 8095, 8294, 8287, 8203, 8194, 8124] avg=8192.0000 max=8294 max_rank=3 min=8092 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1036, 1031, 1017, 1006, 997, 1038, 1086, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1037, 970, 1032, 1091, 1023, 1043, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1034, 1066, 1041, 962, 1031, 992, 1045, 5, 0, 0, 0, 0, 0, 0, 0, 966, 1003, 979, 979, 999, 1073, 1036, 1065, 21, 15, 0, 0, 0, 0, 0, 0, 1031, 1037, 1048, 1024, 1040, 1022, 1011, 1016, 10, 5, 0, 0, 0, 0, 0, 0, 974, 1039, 1041, 1045, 1032, 1013, 1018, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1046, 1030, 1006, 1037, 967, 1022, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1034, 1027, 1008, 973, 979, 998, 969, 48, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8267, 8139, 8136, 8244, 8230, 8172, 8087] avg=8192.0000 max=8267 max_rank=1 min=8087 min_rank=7 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1046, 1051, 998, 1043, 995, 1026, 994, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 995, 1055, 1005, 1044, 1050, 992, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1031, 1027, 1011, 975, 1012, 1036, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1046, 1005, 999, 1036, 1043, 1000, 1023, 17, 11, 0, 0, 0, 0, 0, 0, 1050, 988, 1042, 1023, 998, 1009, 1056, 968, 6, 0, 0, 0, 0, 0, 0, 0, 1043, 1041, 1050, 1014, 1060, 1045, 1018, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1043, 1011, 1039, 986, 1068, 991, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1001, 1015, 1061, 1018, 1003, 1025, 1020, 48, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8182, 8158, 8165, 8140, 8311, 8184, 8229] avg=8192.0000 max=8311 max_rank=5 min=8140 min_rank=4 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8064, 8448, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8064, 8192, 8192, 8192, 8192, 8192, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8448, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1032, 1032, 1028, 981, 1002, 1019, 1020, 951, 1038, 1053, 1023, 1035, 1025, 1031, 1034, 1020, 1013, 1013, 964, 1002, 993, 1025, 1055, 1055, 1007, 1056, 1032, 1037, 1050, 1051, 1048, 983, 1071, 1010, 1020, 1046, 1041, 1045, 1022, 991, 1023, 1047, 1057, 1022, 1009, 1000, 1015, 1028, 1037, 978, 990, 1080, 987, 1049, 1035, 1037, 1062, 1085, 974, 1028, 1001, 1021, 1023, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1044, 997, 1014, 1045, 1074, 1021, 1018, 1016, 1053, 1009, 1030, 1021, 1052, 985, 1031, 1014, 1058, 987, 1046, 1017, 993, 1030, 1016, 1013, 1024, 996, 988, 1046, 1010, 1038, 975, 1019, 1059, 1021, 1072, 1027, 1043, 1015, 1004, 1024, 1014, 1041, 1085, 981, 1050, 1039, 1014, 981, 1024, 1038, 1073, 995, 999, 978, 1069, 990, 1003, 1027, 1059, 1040, 990, 1024, 1037, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1026, 1005, 1005, 1011, 1011, 992, 1016, 1053, 1016, 1042, 1005, 1047, 1007, 1066, 1001, 1068, 1041, 1060, 987, 959, 989, 995, 1019, 1066, 1017, 1021, 1009, 1052, 1095, 986, 1059, 1014, 1083, 1083, 1040, 1012, 1036, 1007, 1033, 1064, 994, 1024, 1017, 1003, 968, 990, 995, 1037, 981, 989, 1024, 1032, 1005, 1012, 1014, 1010, 1074, 1044, 1029, 1061, 1041, 1014, 1048, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1042, 1026, 988, 1022, 1004, 982, 1028, 1028, 1055, 1018, 1032, 1058, 1058, 1053, 989, 995, 1012, 1001, 992, 1047, 993, 1014, 1047, 1012, 1026, 972, 1003, 1017, 991, 1045, 1001, 1023, 1026, 1035, 1028, 1038, 1038, 1066, 1056, 1040, 1036, 1023, 1016, 1033, 1000, 1001, 1057, 1041, 1044, 1019, 1015, 1048, 1028, 993, 1042, 1052, 1012, 1032, 1054, 1051, 1005, 1031, 985, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[972, 1001, 1053, 1004, 1044, 996, 1040, 1039, 1041, 960, 1032, 1056, 1037, 1028, 1028, 1049, 1019, 990, 1052, 1025, 1023, 1031, 1025, 1010, 1029, 1039, 1062, 1022, 994, 983, 1100, 973, 1040, 1031, 1022, 1045, 1021, 1039, 1047, 1045, 1010, 1032, 1026, 1008, 1039, 1012, 1026, 1027, 1030, 1004, 994, 1056, 1054, 993, 964, 1042, 1026, 1055, 1022, 1038, 1016, 998, 1020, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1069, 998, 1011, 978, 1030, 998, 989, 1070, 996, 1038, 1005, 1018, 1093, 1029, 985, 996, 1015, 1004, 1056, 995, 1033, 1043, 1076, 1027, 985, 1017, 991, 1047, 1038, 1088, 1008, 983, 1075, 1007, 998, 1009, 1021, 1065, 1030, 1089, 948, 998, 1024, 1065, 991, 1024, 1065, 1006, 1009, 1039, 1026, 996, 1108, 1042, 1057, 1004, 1009, 988, 1039, 1035, 972, 1057, 1020, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1008, 988, 1042, 1000, 1046, 1075, 984, 1026, 1060, 1073, 1006, 1031, 1056, 1035, 1016, 974, 1074, 1063, 1014, 1000, 1042, 1065, 1013, 1038, 1026, 1044, 965, 1032, 983, 1004, 1052, 1053, 1005, 1008, 1028, 1041, 1071, 1037, 1034, 968, 1027, 1054, 992, 1088, 984, 1016, 1039, 987, 1067, 1023, 988, 1025, 1031, 1006, 1007, 957, 1018, 1049, 1011, 1008, 993, 1015, 1039, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[991, 994, 1051, 1076, 1059, 1055, 1036, 1010, 986, 1072, 1030, 986, 1046, 1038, 1023, 1021, 969, 1029, 996, 990, 974, 1013, 1063, 1042, 1027, 1013, 1076, 1015, 1000, 1014, 1001, 1052, 1070, 1045, 1040, 1068, 1043, 1019, 1015, 1011, 1026, 1031, 1031, 980, 1005, 998, 1016, 956, 977, 984, 1053, 1014, 1057, 1065, 1025, 1071, 1024, 997, 1064, 1030, 987, 996, 1043, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1026, 1005, 1005, 1011, 1011, 992, 1016, 1053, 34, 0, 0, 0, 0, 0, 0, 0, 1016, 1042, 1005, 1047, 975, 1066, 1001, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1060, 987, 959, 989, 995, 1019, 1066, 14, 12, 0, 0, 0, 0, 0, 0, 1017, 1021, 1009, 1052, 1095, 986, 1059, 1014, 15, 0, 0, 0, 0, 0, 0, 0, 989, 1083, 1040, 1012, 1036, 1007, 1033, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1024, 1017, 1003, 968, 990, 995, 1037, 46, 0, 0, 0, 0, 0, 0, 0, 981, 989, 1024, 1032, 1005, 1012, 1014, 1010, 7, 5, 0, 0, 0, 0, 0, 0, 1074, 1044, 1029, 1054, 1041, 1014, 1048, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8220, 8142, 8268, 8264, 8074, 8079, 8336] avg=8192.0000 max=8336 max_rank=7 min=8074 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1032, 1032, 1028, 981, 1002, 1019, 1020, 951, 34, 0, 0, 0, 0, 0, 0, 0, 1038, 1053, 1023, 1035, 993, 1031, 1034, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1013, 964, 1002, 993, 1025, 1055, 1055, 14, 12, 0, 0, 0, 0, 0, 0, 1007, 1056, 1032, 1037, 1050, 1051, 1048, 983, 15, 0, 0, 0, 0, 0, 0, 0, 977, 1010, 1020, 1046, 1041, 1045, 1022, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1047, 1057, 1022, 1009, 1000, 1015, 1028, 46, 0, 0, 0, 0, 0, 0, 0, 1037, 978, 990, 1080, 987, 1049, 1035, 1037, 7, 5, 0, 0, 0, 0, 0, 0, 1062, 1085, 974, 1021, 1001, 1021, 1023, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8227, 8146, 8279, 8152, 8247, 8205, 8181] avg=8192.0000 max=8279 max_rank=3 min=8099 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1044, 997, 1014, 1045, 1074, 1021, 1018, 1016, 33, 0, 0, 0, 0, 0, 0, 0, 1053, 1009, 1030, 1021, 1019, 985, 1031, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 987, 1046, 1017, 993, 1030, 1016, 1013, 14, 12, 0, 0, 0, 0, 0, 0, 1024, 996, 988, 1046, 1010, 1038, 975, 1019, 16, 0, 0, 0, 0, 0, 0, 0, 967, 1021, 1072, 1027, 1043, 1015, 1004, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1041, 1085, 981, 1050, 1039, 1014, 981, 45, 0, 0, 0, 0, 0, 0, 0, 1024, 1038, 1073, 995, 999, 978, 1069, 990, 7, 5, 0, 0, 0, 0, 0, 0, 1003, 1027, 1059, 1033, 990, 1024, 1037, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8162, 8186, 8112, 8173, 8250, 8178, 8213] avg=8192.0000 max=8262 max_rank=0 min=8112 min_rank=3 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1042, 1026, 988, 1022, 1004, 982, 1028, 1028, 32, 0, 0, 0, 0, 0, 0, 0, 1055, 1018, 1032, 1058, 1025, 1053, 989, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1001, 992, 1047, 993, 1014, 1047, 1012, 13, 12, 0, 0, 0, 0, 0, 0, 1026, 972, 1003, 1017, 991, 1045, 1001, 1023, 16, 0, 0, 0, 0, 0, 0, 0, 937, 1035, 1028, 1038, 1038, 1066, 1056, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1023, 1016, 1033, 1000, 1001, 1057, 1041, 44, 0, 0, 0, 0, 0, 0, 0, 1044, 1019, 1015, 1048, 1028, 993, 1042, 1052, 7, 5, 0, 0, 0, 0, 0, 0, 1012, 1032, 1054, 1044, 1005, 1031, 985, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8225, 8143, 8094, 8238, 8251, 8253, 8180] avg=8192.0000 max=8253 max_rank=6 min=8094 min_rank=3 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32922 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1069, 998, 1011, 978, 1030, 998, 989, 1070, 34, 0, 0, 0, 0, 0, 0, 0, 996, 1038, 1005, 1018, 1059, 1029, 985, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1004, 1056, 995, 1033, 1043, 1076, 1027, 14, 13, 0, 0, 0, 0, 0, 0, 985, 1017, 991, 1047, 1038, 1088, 1008, 983, 16, 0, 0, 0, 0, 0, 0, 0, 981, 1007, 998, 1009, 1021, 1065, 1030, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 948, 998, 1024, 1065, 991, 1024, 1065, 1006, 46, 0, 0, 0, 0, 0, 0, 0, 1009, 1039, 1026, 996, 1108, 1042, 1057, 1004, 7, 5, 0, 0, 0, 0, 0, 0, 1009, 988, 1039, 1028, 972, 1057, 1020, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8126, 8276, 8173, 8200, 8167, 8293, 8124] avg=8192.0000 max=8293 max_rank=6 min=8124 min_rank=7 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[991, 994, 1051, 1076, 1059, 1055, 1036, 1010, 34, 0, 0, 0, 0, 0, 0, 0, 986, 1072, 1030, 986, 1013, 1038, 1023, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1029, 996, 990, 974, 1013, 1063, 1042, 14, 12, 0, 0, 0, 0, 0, 0, 1027, 1013, 1076, 1015, 1000, 1014, 1001, 1052, 16, 0, 0, 0, 0, 0, 0, 0, 976, 1045, 1040, 1068, 1043, 1019, 1015, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1031, 1031, 980, 1005, 998, 1016, 956, 46, 0, 0, 0, 0, 0, 0, 0, 977, 984, 1053, 1014, 1057, 1065, 1025, 1071, 7, 5, 0, 0, 0, 0, 0, 0, 1024, 997, 1064, 1023, 987, 996, 1043, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 8169, 8102, 8214, 8217, 8089, 8258, 8181] avg=8192.0000 max=8306 max_rank=0 min=8089 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1008, 988, 1042, 1000, 1046, 1075, 984, 1026, 36, 0, 0, 0, 0, 0, 0, 0, 1060, 1073, 1006, 1031, 1013, 1035, 1016, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1063, 1014, 1000, 1042, 1065, 1013, 1038, 17, 15, 0, 0, 0, 0, 0, 0, 1026, 1044, 965, 1032, 983, 1004, 1052, 1053, 20, 0, 0, 0, 0, 0, 0, 0, 905, 1008, 1028, 1041, 1071, 1037, 1034, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1054, 992, 1088, 984, 1016, 1039, 987, 47, 0, 0, 0, 0, 0, 0, 0, 1067, 1023, 988, 1025, 1031, 1006, 1007, 957, 9, 8, 0, 0, 0, 0, 0, 0, 1018, 1049, 1011, 999, 993, 1015, 1039, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8208, 8341, 8179, 8092, 8234, 8121, 8156] avg=8192.0000 max=8341 max_rank=2 min=8092 min_rank=4 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32603 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[972, 1001, 1053, 1004, 1044, 996, 1040, 1039, 33, 0, 0, 0, 0, 0, 0, 0, 1041, 960, 1032, 1056, 1005, 1028, 1028, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 990, 1052, 1025, 1023, 1031, 1025, 1010, 13, 12, 0, 0, 0, 0, 0, 0, 1029, 1039, 1062, 1022, 994, 983, 1100, 973, 15, 0, 0, 0, 0, 0, 0, 0, 950, 1031, 1022, 1045, 1021, 1039, 1047, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1032, 1026, 1008, 1039, 1012, 1026, 1027, 44, 0, 0, 0, 0, 0, 0, 0, 1030, 1004, 994, 1056, 1054, 993, 964, 1042, 7, 5, 0, 0, 0, 0, 0, 0, 1026, 1055, 1022, 1031, 1016, 998, 1020, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8199, 8200, 8217, 8200, 8224, 8149, 8165] avg=8192.0000 max=8224 max_rank=5 min=8149 min_rank=6 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8064, 8192, 8320, 8064, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8064, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7808, 8320, 8320, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8192, 8320, 8192, 8192, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8064, 8064, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[997, 997, 1013, 987, 1019, 1016, 1012, 1007, 1087, 1016, 1025, 1031, 992, 1027, 993, 1009, 1012, 1074, 1031, 1001, 1028, 1088, 1050, 1031, 999, 943, 1017, 1058, 1027, 1045, 1040, 1006, 1051, 1050, 1020, 1043, 1031, 1000, 1091, 1035, 997, 1026, 1012, 1107, 1028, 998, 1006, 995, 1042, 1016, 984, 1058, 1016, 1039, 1054, 986, 1024, 1018, 1038, 1007, 1049, 1010, 1014, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1043, 1061, 1013, 1027, 988, 1015, 999, 1017, 1014, 1020, 999, 1033, 1018, 1047, 1036, 1041, 1024, 1030, 1020, 1067, 1030, 1016, 1029, 1055, 1050, 1067, 996, 989, 995, 1058, 1035, 1021, 1067, 1061, 1031, 1047, 995, 1009, 1053, 1008, 999, 989, 1059, 1035, 1010, 963, 1028, 1017, 1004, 1007, 1010, 1027, 1024, 1021, 1025, 999, 1006, 1032, 1026, 1050, 1015, 988, 1026, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1014, 1032, 1036, 1039, 1075, 1034, 1060, 996, 1055, 1054, 1095, 1029, 1018, 996, 1015, 1027, 1064, 1052, 1026, 1028, 1027, 986, 997, 1029, 1048, 1022, 1017, 1059, 998, 1014, 1059, 973, 1037, 1027, 998, 1036, 1058, 999, 1015, 1001, 1041, 1075, 989, 1004, 1014, 1045, 1044, 1019, 1023, 983, 985, 1035, 987, 1063, 1001, 1018, 987, 986, 999, 1054, 1050, 977, 1013, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[962, 1012, 1023, 1007, 1027, 1057, 1033, 1050, 1038, 1002, 1018, 1029, 1034, 985, 1056, 1037, 980, 1016, 1044, 1022, 1013, 1024, 1021, 1039, 1032, 1040, 1005, 1017, 1049, 1003, 999, 1003, 1002, 1067, 1008, 1045, 990, 1054, 1053, 1076, 1035, 1069, 979, 1046, 1006, 1064, 1025, 1061, 1045, 972, 1040, 1026, 1028, 965, 1006, 1027, 1031, 1020, 998, 1016, 1013, 985, 1036, 1071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[997, 1018, 1032, 1004, 1053, 1029, 1061, 1048, 998, 1034, 1022, 1060, 1005, 1055, 1032, 1038, 1026, 1018, 969, 1042, 1039, 1007, 1033, 1067, 1055, 1021, 1022, 1022, 1045, 1045, 975, 968, 1032, 981, 1035, 1038, 1001, 1031, 1031, 1005, 1059, 1032, 1016, 1064, 976, 1011, 1009, 1068, 1056, 1059, 1006, 1043, 1015, 971, 1019, 1016, 1042, 980, 1017, 1041, 999, 1009, 990, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[960, 1026, 1018, 1067, 1014, 1026, 1049, 998, 1003, 1076, 970, 988, 1053, 1051, 989, 1025, 995, 1032, 1007, 1030, 1033, 1050, 1081, 1024, 1003, 1078, 1053, 1051, 994, 1004, 1029, 1028, 1024, 985, 1053, 998, 1031, 968, 1000, 1048, 999, 1015, 959, 1020, 1113, 1066, 1039, 1003, 1021, 1001, 1035, 1039, 1087, 1072, 1058, 1010, 1019, 1008, 1026, 945, 1064, 1012, 1006, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[992, 1022, 963, 1012, 1006, 1011, 1014, 1000, 1050, 978, 1045, 1021, 1036, 1027, 1063, 977, 1026, 1015, 979, 1017, 981, 1060, 1054, 973, 1073, 1052, 1021, 1044, 1031, 990, 1051, 963, 1045, 1020, 1033, 999, 1066, 1017, 1028, 1015, 1031, 1057, 1037, 1019, 1014, 1005, 1086, 1055, 1004, 1048, 989, 1003, 1039, 1013, 1047, 969, 1060, 1037, 1056, 1084, 1055, 1053, 992, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1067, 1105, 1059, 1021, 1092, 990, 968, 1074, 1029, 1051, 1024, 1026, 1031, 1005, 1015, 1023, 1057, 949, 1039, 1064, 1039, 1035, 1033, 1032, 1059, 1003, 1038, 1007, 984, 977, 1038, 987, 993, 1023, 1026, 1013, 1021, 1020, 996, 948, 1035, 1019, 983, 989, 995, 1018, 1023, 1084, 1041, 1014, 973, 1062, 1058, 1040, 1034, 1024, 1062, 983, 1025, 976, 1044, 1006, 1041, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1043, 1061, 1013, 1027, 988, 1015, 999, 1017, 5, 3, 0, 0, 0, 0, 0, 0, 998, 1020, 999, 1033, 1018, 1047, 1036, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1030, 1020, 1067, 1030, 1016, 1002, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1067, 996, 989, 995, 1058, 1035, 1021, 13, 3, 0, 0, 0, 0, 0, 0, 1067, 1061, 1031, 1047, 995, 1009, 1048, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 999, 989, 1059, 1035, 1010, 963, 1028, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1007, 1010, 1027, 1024, 1021, 1025, 999, 14, 4, 0, 0, 0, 0, 0, 0, 1006, 1032, 1026, 1050, 1015, 988, 1026, 1052, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8192, 8244, 8227, 8266, 8083, 8135, 8218] avg=8192.0000 max=8266 max_rank=4 min=8083 min_rank=5 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1014, 1032, 1036, 1039, 1075, 1034, 1060, 996, 5, 3, 0, 0, 0, 0, 0, 0, 1038, 1054, 1095, 1029, 1018, 996, 1015, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1052, 1026, 1028, 1027, 986, 972, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1022, 1017, 1059, 998, 1014, 1059, 973, 14, 3, 0, 0, 0, 0, 0, 0, 1037, 1027, 998, 1036, 1058, 999, 1010, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1075, 989, 1004, 1014, 1045, 1044, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 983, 985, 1035, 987, 1063, 1001, 1018, 14, 3, 0, 0, 0, 0, 0, 0, 987, 986, 999, 1054, 1050, 977, 1013, 999, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 8272, 8184, 8207, 8166, 8214, 8112, 8087] avg=8192.0000 max=8294 max_rank=0 min=8087 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32579 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[962, 1012, 1023, 1007, 1027, 1057, 1033, 1050, 5, 3, 0, 0, 0, 0, 0, 0, 1021, 1002, 1018, 1029, 1034, 985, 1056, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1016, 1044, 1022, 1013, 1024, 994, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1040, 1005, 1017, 1049, 1003, 999, 1003, 14, 3, 0, 0, 0, 0, 0, 0, 1002, 1067, 1008, 1045, 990, 1054, 1048, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1069, 979, 1046, 1006, 1064, 1025, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 972, 1040, 1026, 1028, 965, 1006, 1027, 15, 4, 0, 0, 0, 0, 0, 0, 1031, 1020, 998, 1016, 1013, 985, 1036, 1071, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8182, 8132, 8165, 8290, 8267, 8128, 8193] avg=8192.0000 max=8290 max_rank=4 min=8128 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[997, 1018, 1032, 1004, 1053, 1029, 1061, 1048, 5, 3, 0, 0, 0, 0, 0, 0, 982, 1034, 1022, 1060, 1005, 1055, 1032, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1018, 969, 1042, 1039, 1007, 1006, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1021, 1022, 1022, 1045, 1045, 975, 968, 13, 3, 0, 0, 0, 0, 0, 0, 1032, 981, 1035, 1038, 1001, 1031, 1026, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1032, 1016, 1064, 976, 1011, 1009, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1059, 1006, 1043, 1015, 971, 1019, 1016, 15, 4, 0, 0, 0, 0, 0, 0, 1042, 980, 1017, 1041, 999, 1009, 990, 1044, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8228, 8174, 8169, 8149, 8217, 8204, 8145] avg=8192.0000 max=8250 max_rank=0 min=8145 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[997, 997, 1013, 987, 1019, 1016, 1012, 1007, 6, 3, 0, 0, 0, 0, 0, 0, 1070, 1016, 1025, 1031, 992, 1027, 993, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1074, 1031, 1001, 1028, 1088, 1022, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 999, 943, 1017, 1058, 1027, 1045, 1040, 1006, 14, 3, 0, 0, 0, 0, 0, 0, 1051, 1050, 1020, 1043, 1031, 1000, 1085, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1026, 1012, 1107, 1028, 998, 1006, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1016, 984, 1058, 1016, 1039, 1054, 986, 14, 4, 0, 0, 0, 0, 0, 0, 1024, 1018, 1038, 1007, 1049, 1010, 1014, 1013, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8057, 8163, 8287, 8152, 8315, 8152, 8213, 8197] avg=8192.0000 max=8315 max_rank=4 min=8057 min_rank=0 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[960, 1026, 1018, 1067, 1014, 1026, 1049, 998, 10, 5, 0, 0, 0, 0, 0, 0, 980, 1076, 970, 988, 1053, 1051, 989, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1032, 1007, 1030, 1033, 1050, 1046, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1078, 1053, 1051, 994, 1004, 1029, 1028, 18, 6, 0, 0, 0, 0, 0, 0, 1024, 985, 1053, 998, 1031, 968, 990, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1015, 959, 1020, 1113, 1066, 1039, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1001, 1035, 1039, 1087, 1072, 1058, 1010, 19, 6, 0, 0, 0, 0, 0, 0, 1019, 1008, 1026, 945, 1064, 1012, 1006, 1007, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8132, 8217, 8264, 8097, 8189, 8348, 8116] avg=8192.0000 max=8348 max_rank=6 min=8097 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1067, 1105, 1059, 1021, 1092, 990, 968, 1074, 5, 3, 0, 0, 0, 0, 0, 0, 1012, 1051, 1024, 1026, 1031, 1005, 1015, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 949, 1039, 1064, 1039, 1035, 1006, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1003, 1038, 1007, 984, 977, 1038, 987, 14, 3, 0, 0, 0, 0, 0, 0, 993, 1023, 1026, 1013, 1021, 1020, 991, 948, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1019, 983, 989, 995, 1018, 1023, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1014, 973, 1062, 1058, 1040, 1034, 1024, 15, 4, 0, 0, 0, 0, 0, 0, 1062, 983, 1025, 976, 1044, 1006, 1041, 1046, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8384, 8187, 8221, 8110, 8035, 8128, 8265, 8206] avg=8192.0000 max=8384 max_rank=0 min=8035 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[992, 1022, 963, 1012, 1006, 1011, 1014, 1000, 5, 3, 0, 0, 0, 0, 0, 0, 1033, 978, 1045, 1021, 1036, 1027, 1063, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1015, 979, 1017, 981, 1060, 1026, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1052, 1021, 1044, 1031, 990, 1051, 963, 14, 3, 0, 0, 0, 0, 0, 0, 1045, 1020, 1033, 999, 1066, 1017, 1023, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1057, 1037, 1019, 1014, 1005, 1086, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1048, 989, 1003, 1039, 1013, 1047, 969, 15, 4, 0, 0, 0, 0, 0, 0, 1060, 1037, 1056, 1084, 1055, 1053, 992, 1013, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8028, 8180, 8077, 8242, 8218, 8286, 8131, 8374] avg=8192.0000 max=8374 max_rank=7 min=8028 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=57162 max_load_reduction_pct=0.872223 moved_to_echo=33009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8320, 8320, 8192, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8192, 8320, 8064, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1015, 981, 1045, 979, 995, 1067, 1012, 1024, 1035, 1032, 1023, 989, 999, 1063, 997, 1034, 1027, 1060, 1006, 1033, 994, 1033, 1044, 1057, 1022, 1018, 1062, 1039, 1020, 1036, 1013, 1022, 989, 1010, 1055, 1026, 1061, 1012, 990, 1016, 1013, 1029, 1023, 1064, 1038, 978, 1024, 1018, 1054, 999, 985, 1049, 1018, 1076, 980, 1035, 1017, 1053, 1027, 1037, 1039, 1039, 1017, 989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[994, 1000, 995, 1044, 1008, 1030, 1039, 1022, 1033, 1029, 957, 1011, 1027, 986, 1007, 996, 973, 1022, 1021, 1073, 986, 1035, 1098, 1045, 1052, 1028, 1025, 1037, 1073, 1028, 972, 1039, 1051, 1078, 1000, 1034, 1052, 1074, 978, 1049, 1034, 1054, 1036, 1016, 1029, 1061, 1030, 967, 989, 1031, 1059, 1034, 1049, 995, 986, 979, 1048, 1004, 1047, 1018, 1045, 995, 1037, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1061, 1034, 1034, 1032, 1075, 1001, 996, 999, 999, 1006, 1005, 1035, 1016, 1001, 1059, 1089, 1021, 1055, 1022, 1014, 988, 1043, 992, 1059, 1045, 1062, 1003, 1008, 1012, 1013, 1014, 1051, 1030, 1066, 1042, 1018, 1091, 1022, 1035, 1059, 1044, 1004, 991, 959, 1041, 1060, 1022, 1004, 988, 1042, 1004, 1036, 1019, 983, 991, 980, 1074, 1001, 1060, 1014, 991, 1034, 1000, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1052, 994, 1030, 993, 1017, 1034, 1025, 1012, 1011, 1021, 1052, 1054, 1020, 1016, 1059, 1017, 1012, 1000, 1007, 998, 1027, 1043, 989, 1009, 1032, 1019, 1014, 1038, 1067, 995, 1053, 1058, 1044, 990, 1030, 1016, 1029, 974, 1017, 1068, 1036, 1026, 1046, 1005, 1018, 966, 1011, 1055, 1065, 1052, 973, 1005, 985, 1018, 1040, 1024, 1021, 1033, 1061, 983, 1114, 1003, 1024, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1014, 983, 1016, 1026, 1043, 1040, 1050, 1019, 991, 992, 1026, 982, 1006, 1056, 988, 1037, 1052, 1033, 1006, 968, 1026, 1096, 1052, 1043, 1043, 1029, 1045, 1046, 981, 1062, 1040, 1037, 1004, 1014, 1019, 1021, 988, 1003, 1005, 1023, 1010, 1028, 1005, 1095, 1044, 1021, 1087, 999, 1052, 1041, 1031, 1015, 1045, 1020, 1004, 1019, 1030, 1007, 1025, 1045, 1014, 990, 1027, 977] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[980, 979, 1072, 1060, 1063, 1065, 995, 1039, 1036, 997, 1001, 1093, 981, 1003, 1015, 1044, 1047, 978, 1096, 1044, 1056, 1026, 1034, 1072, 1040, 1091, 986, 998, 1052, 1001, 1007, 1021, 1013, 989, 1034, 1022, 985, 1050, 1050, 1041, 1055, 1048, 972, 972, 1072, 1032, 1008, 1014, 1009, 946, 996, 1036, 1018, 1084, 998, 1042, 1061, 1021, 986, 1033, 1009, 954, 988, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1013, 1042, 1044, 1027, 1041, 1013, 1027, 991, 1025, 978, 1046, 1024, 1052, 962, 1054, 1050, 994, 1009, 981, 1019, 1068, 1056, 986, 982, 1069, 1005, 1028, 1054, 1012, 1014, 1090, 1060, 1044, 1036, 1010, 993, 1003, 1021, 1043, 1006, 1023, 1050, 1001, 1031, 1054, 996, 1031, 1087, 985, 956, 1054, 1072, 1021, 1088, 960, 1014, 1055, 1006, 1051, 1003, 1021, 1037, 981, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1096, 1048, 1028, 1020, 1037, 961, 1072, 971, 1054, 1042, 947, 958, 1007, 1013, 997, 1050, 1044, 1000, 1090, 1008, 995, 1028, 1050, 1024, 1063, 1038, 992, 993, 1031, 1008, 1044, 1003, 1043, 1049, 999, 975, 1070, 1037, 1033, 982, 1085, 1021, 1015, 1029, 1038, 1005, 1034, 1005, 1026, 1043, 1036, 1010, 1021, 999, 993, 1038, 979, 1032, 1074, 1049, 1088, 1005, 1027, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[980, 979, 1072, 1060, 1063, 1065, 995, 1039, 2, 0, 0, 0, 0, 0, 0, 0, 1036, 997, 1001, 1093, 981, 1003, 1015, 1044, 43, 0, 0, 0, 0, 0, 0, 0, 1047, 978, 1096, 1044, 1056, 1000, 1034, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1091, 986, 998, 1052, 1001, 1007, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 989, 1034, 1022, 977, 1050, 1050, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1048, 972, 972, 1056, 1032, 1008, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 946, 996, 1036, 1018, 1084, 998, 1042, 26, 8, 7, 0, 0, 0, 0, 0, 1061, 1021, 986, 1033, 1009, 954, 988, 1026, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8213, 8327, 8145, 8176, 8157, 8170, 8093] avg=8192.0000 max=8327 max_rank=2 min=8093 min_rank=7 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1015, 981, 1045, 979, 995, 1067, 1012, 1024, 2, 0, 0, 0, 0, 0, 0, 0, 1035, 1032, 1023, 989, 999, 1063, 997, 1034, 42, 0, 0, 0, 0, 0, 0, 0, 1027, 1060, 1006, 1033, 994, 1007, 1044, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1018, 1062, 1039, 1020, 1036, 1013, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1010, 1055, 1026, 1053, 1012, 990, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1029, 1023, 1064, 1023, 978, 1024, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 999, 985, 1049, 1018, 1076, 980, 1035, 26, 8, 7, 0, 0, 0, 0, 0, 1017, 1053, 1027, 1037, 1039, 1039, 1017, 989, 8, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8214, 8228, 8182, 8151, 8172, 8237, 8232] avg=8192.0000 max=8237 max_rank=6 min=8120 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[994, 1000, 995, 1044, 1008, 1030, 1039, 1022, 2, 0, 0, 0, 0, 0, 0, 0, 1033, 1029, 957, 1011, 1027, 986, 1007, 996, 44, 0, 0, 0, 0, 0, 0, 0, 973, 1022, 1021, 1073, 986, 1009, 1098, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1028, 1025, 1037, 1073, 1028, 972, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1078, 1000, 1034, 1044, 1074, 978, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1054, 1036, 1016, 1014, 1061, 1030, 967, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1031, 1059, 1034, 1049, 995, 986, 979, 26, 8, 7, 0, 0, 0, 0, 0, 1048, 1004, 1047, 1018, 1045, 995, 1037, 992, 8, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8090, 8227, 8202, 8308, 8212, 8163, 8200] avg=8192.0000 max=8308 max_rank=4 min=8090 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1014, 983, 1016, 1026, 1043, 1040, 1050, 1019, 2, 0, 0, 0, 0, 0, 0, 0, 991, 992, 1026, 982, 1006, 1056, 988, 1037, 43, 0, 0, 0, 0, 0, 0, 0, 1052, 1033, 1006, 968, 1026, 1069, 1052, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1029, 1045, 1046, 981, 1062, 1040, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1014, 1019, 1021, 980, 1003, 1005, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1028, 1005, 1095, 1029, 1021, 1087, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1041, 1031, 1015, 1045, 1020, 1004, 1019, 27, 8, 7, 0, 0, 0, 0, 0, 1030, 1007, 1025, 1045, 1014, 990, 1027, 977, 8, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8121, 8249, 8232, 8069, 8274, 8269, 8129] avg=8192.0000 max=8274 max_rank=5 min=8069 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1061, 1034, 1034, 1032, 1075, 1001, 996, 999, 2, 0, 0, 0, 0, 0, 0, 0, 999, 1006, 1005, 1035, 1016, 1001, 1059, 1089, 43, 0, 0, 0, 0, 0, 0, 0, 1021, 1055, 1022, 1014, 988, 1017, 992, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1062, 1003, 1008, 1012, 1013, 1014, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1066, 1042, 1018, 1083, 1022, 1035, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1004, 991, 959, 1026, 1060, 1022, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1042, 1004, 1036, 1019, 983, 991, 980, 26, 8, 7, 0, 0, 0, 0, 0, 1074, 1001, 1060, 1014, 991, 1034, 1000, 987, 8, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8253, 8168, 8157, 8355, 8110, 8084, 8175] avg=8192.0000 max=8355 max_rank=4 min=8084 min_rank=6 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1052, 994, 1030, 993, 1017, 1034, 1025, 1012, 8, 0, 0, 0, 0, 0, 0, 0, 1011, 1021, 1052, 1054, 1020, 1016, 1059, 1017, 48, 0, 0, 0, 0, 0, 0, 0, 1012, 1000, 1007, 998, 1027, 1013, 989, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1019, 1014, 1038, 1067, 995, 1053, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 990, 1030, 1016, 1010, 974, 1017, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1026, 1046, 1005, 991, 966, 1011, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1052, 973, 1005, 985, 1018, 1040, 1024, 30, 10, 13, 0, 0, 0, 0, 0, 1021, 1033, 1061, 983, 1114, 1003, 1024, 1036, 14, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8298, 8055, 8218, 8149, 8136, 8215, 8300] avg=8192.0000 max=8300 max_rank=7 min=8055 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1013, 1042, 1044, 1027, 1041, 1013, 1027, 991, 2, 0, 0, 0, 0, 0, 0, 0, 1025, 978, 1046, 1024, 1052, 962, 1054, 1050, 44, 0, 0, 0, 0, 0, 0, 0, 994, 1009, 981, 1019, 1068, 1030, 986, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1005, 1028, 1054, 1012, 1014, 1090, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1036, 1010, 993, 995, 1021, 1043, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1050, 1001, 1031, 1039, 996, 1031, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 985, 956, 1054, 1072, 1021, 1088, 960, 1014, 26, 8, 7, 0, 0, 0, 0, 0, 1055, 1006, 1051, 1003, 1021, 1037, 981, 987, 8, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8235, 8069, 8280, 8148, 8258, 8191, 8155] avg=8192.0000 max=8280 max_rank=3 min=8069 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1096, 1048, 1028, 1020, 1037, 961, 1072, 971, 2, 0, 0, 0, 0, 0, 0, 0, 1054, 1042, 947, 958, 1007, 1013, 997, 1050, 44, 0, 0, 0, 0, 0, 0, 0, 1044, 1000, 1090, 1008, 995, 1002, 1050, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1038, 992, 993, 1031, 1008, 1044, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1049, 999, 975, 1062, 1037, 1033, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 1021, 1015, 1029, 1023, 1005, 1034, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1043, 1036, 1010, 1021, 999, 993, 1038, 26, 8, 7, 0, 0, 0, 0, 0, 979, 1032, 1074, 1049, 1088, 1005, 1027, 984, 8, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8112, 8213, 8120, 8180, 8217, 8207, 8252] avg=8192.0000 max=8252 max_rank=7 min=8112 min_rank=1 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8448, 8192, 8448, 8064, 8192, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8192, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8064, 8192, 8192, 8192, 8192, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8064, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[991, 1010, 1015, 1018, 1030, 1026, 1023, 1077, 979, 1028, 1016, 1036, 1079, 1095, 1039, 1024, 1004, 1039, 1029, 1037, 1002, 1066, 1028, 994, 1028, 1009, 1075, 1053, 1057, 1014, 1012, 979, 1050, 1033, 1012, 1045, 1003, 1021, 983, 1051, 995, 1011, 1029, 1040, 981, 973, 1037, 1046, 1008, 1049, 1028, 1057, 1010, 995, 1014, 981, 1039, 991, 1003, 1054, 1027, 988, 1042, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1034, 1025, 1032, 987, 1019, 1058, 983, 1020, 1022, 1072, 1046, 1061, 1042, 1007, 1045, 999, 1068, 1037, 1019, 1035, 1042, 1042, 1006, 1075, 1055, 1013, 989, 1012, 1040, 1042, 1014, 1011, 1001, 1056, 1006, 1036, 967, 1042, 1034, 1005, 1024, 1024, 1024, 1057, 1019, 1007, 1061, 1006, 1015, 1025, 1020, 1037, 1012, 1000, 1013, 1001, 1033, 1016, 1037, 994, 1015, 1020, 999, 978] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1020, 1070, 1002, 1016, 1043, 1012, 1053, 1009, 1057, 1035, 1046, 1058, 1033, 1032, 988, 991, 1089, 1038, 1039, 1041, 992, 1000, 1037, 1018, 1069, 1033, 1054, 1057, 1071, 1060, 981, 1041, 1008, 1013, 1034, 993, 962, 1040, 1023, 1015, 1047, 1005, 995, 1009, 1001, 1014, 993, 1002, 1032, 988, 1043, 1008, 991, 1047, 1061, 988, 1039, 981, 1041, 1029, 1000, 1027, 1001, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1039, 1024, 1042, 1079, 1015, 1052, 1048, 1034, 1006, 1000, 1058, 1013, 994, 1013, 995, 1044, 1065, 1053, 993, 1038, 1003, 983, 1014, 1022, 991, 1040, 1032, 997, 1034, 1047, 1047, 1074, 1036, 980, 1000, 977, 1020, 1044, 1040, 1025, 1040, 1044, 976, 1065, 1033, 1071, 981, 1028, 1074, 1026, 989, 1003, 1029, 993, 1064, 1015, 1053, 965, 1031, 1025, 1005, 1017, 980, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[990, 1003, 987, 1042, 1044, 987, 1049, 1054, 1015, 1040, 1000, 1050, 987, 1007, 1041, 1046, 1010, 1054, 999, 1053, 1039, 1026, 1020, 1019, 1007, 991, 1038, 992, 1017, 1001, 1038, 1023, 1037, 1058, 1006, 1064, 1017, 1078, 1063, 1004, 1016, 1071, 1019, 1014, 1044, 1055, 998, 1001, 1040, 1011, 988, 1015, 1035, 1063, 1042, 1007, 1037, 1008, 1043, 995, 983, 1018, 1003, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1007, 1033, 1038, 1041, 980, 987, 1021, 1014, 1093, 983, 1031, 1038, 1032, 1046, 1020, 1011, 971, 1014, 1054, 1054, 1055, 1021, 1005, 973, 1012, 973, 1038, 986, 956, 1058, 1002, 1036, 1018, 965, 1053, 1073, 1055, 1001, 1042, 1051, 974, 1047, 990, 1038, 992, 1033, 1028, 1031, 997, 1069, 1059, 1038, 989, 1050, 1078, 1056, 1049, 1033, 1014, 1049, 1012, 1015, 981, 1073] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1037, 1044, 1014, 935, 1072, 1037, 1049, 1025, 1020, 1064, 1028, 1002, 997, 1074, 986, 1005, 1058, 928, 1049, 1004, 1069, 1013, 1057, 1007, 968, 1026, 1014, 1050, 1036, 1066, 1008, 1012, 1007, 1023, 980, 1044, 1004, 1026, 1033, 1069, 1011, 1040, 1063, 1002, 1037, 1071, 1052, 1077, 1010, 1005, 1020, 965, 1057, 995, 1011, 1021, 1043, 1073, 1022, 980, 1005, 1015, 1048, 973] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1020, 1018, 980, 1050, 1021, 1038, 991, 1036, 1009, 998, 1022, 987, 1077, 1044, 1014, 1052, 1001, 1024, 1069, 1005, 1023, 1018, 1024, 1058, 1030, 1003, 1000, 1011, 1032, 1049, 1022, 1040, 1012, 1039, 971, 1005, 1044, 1052, 1074, 1063, 1036, 1048, 1080, 988, 1019, 1028, 1030, 1048, 1032, 1041, 1004, 1030, 976, 1021, 1065, 1006, 1041, 976, 993, 984, 995, 1049, 1043, 977] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1020, 1070, 1002, 1016, 1043, 1012, 1053, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1035, 1046, 1058, 1033, 1003, 988, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 1038, 1039, 1018, 992, 1000, 1037, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1033, 1054, 1057, 1071, 1057, 981, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1013, 1034, 993, 962, 1034, 1023, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 996, 995, 1009, 1001, 1014, 993, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 988, 1043, 1008, 991, 1047, 1061, 988, 6, 3, 3, 1, 0, 0, 0, 0, 1039, 981, 1041, 1029, 1000, 1027, 1001, 1021, 29, 23, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8211, 8231, 8363, 8082, 8057, 8171, 8197] avg=8192.0000 max=8363 max_rank=3 min=8057 min_rank=5 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1039, 1024, 1042, 1079, 1015, 1052, 1048, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1000, 1058, 1013, 994, 985, 995, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1053, 993, 1015, 1003, 983, 1014, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1040, 1032, 997, 1034, 1044, 1047, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 980, 1000, 977, 1020, 1038, 1040, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1035, 976, 1065, 1033, 1071, 981, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1026, 989, 1003, 1029, 993, 1064, 1015, 6, 3, 3, 1, 0, 0, 0, 0, 1053, 965, 1031, 1025, 1005, 1017, 980, 1018, 28, 23, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8332, 8095, 8148, 8259, 8116, 8229, 8206, 8151] avg=8192.0000 max=8332 max_rank=0 min=8095 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[991, 1010, 1015, 1018, 1030, 1026, 1023, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1028, 1016, 1036, 1079, 1064, 1039, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1039, 1029, 1014, 1002, 1066, 1028, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1009, 1075, 1053, 1057, 1011, 1012, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1033, 1012, 1045, 1003, 1015, 983, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1002, 1029, 1040, 981, 973, 1037, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1049, 1028, 1057, 1010, 995, 1014, 981, 6, 3, 3, 1, 0, 0, 0, 0, 1039, 991, 1003, 1054, 1027, 988, 1042, 1028, 31, 23, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8265, 8176, 8224, 8192, 8103, 8155, 8232] avg=8192.0000 max=8265 max_rank=1 min=8103 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[990, 1003, 987, 1042, 1044, 987, 1049, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1040, 1000, 1050, 987, 979, 1041, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1054, 999, 1030, 1039, 1026, 1020, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 991, 1038, 992, 1017, 998, 1038, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1058, 1006, 1064, 1017, 1072, 1063, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1062, 1019, 1014, 1044, 1055, 998, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1011, 988, 1015, 1035, 1063, 1042, 1007, 6, 3, 3, 1, 0, 0, 0, 0, 1037, 1008, 1043, 995, 983, 1018, 1003, 1034, 28, 23, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8158, 8197, 8104, 8321, 8209, 8214, 8178] avg=8192.0000 max=8321 max_rank=4 min=8104 min_rank=3 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32922 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1037, 1044, 1014, 935, 1072, 1037, 1049, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1064, 1028, 1002, 997, 1044, 986, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 928, 1049, 982, 1069, 1013, 1057, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1026, 1014, 1050, 1036, 1063, 1008, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1023, 980, 1044, 1004, 1020, 1033, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1031, 1063, 1002, 1037, 1071, 1052, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1005, 1020, 965, 1057, 995, 1011, 1021, 6, 3, 3, 1, 0, 0, 0, 0, 1043, 1073, 1022, 980, 1005, 1015, 1048, 973, 30, 22, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8146, 8163, 8177, 8180, 8344, 8097, 8217] avg=8192.0000 max=8344 max_rank=5 min=8097 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1007, 1033, 1038, 1041, 980, 987, 1021, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 983, 1031, 1038, 1032, 1013, 1020, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1014, 1054, 1027, 1055, 1021, 1005, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 973, 1038, 986, 956, 1049, 1002, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 965, 1053, 1073, 1055, 993, 1042, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1027, 990, 1038, 992, 1033, 1028, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1069, 1059, 1038, 989, 1050, 1078, 1056, 8, 9, 9, 7, 0, 0, 0, 0, 1049, 1033, 1014, 1049, 1012, 1015, 981, 1073, 33, 27, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8221, 8120, 8052, 8250, 8113, 8369, 8297] avg=8192.0000 max=8369 max_rank=6 min=8052 min_rank=3 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=33029 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1020, 1018, 980, 1050, 1021, 1038, 991, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 998, 1022, 987, 1077, 1015, 1014, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1024, 1069, 983, 1023, 1018, 1024, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1003, 1000, 1011, 1032, 1046, 1022, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1039, 971, 1005, 1044, 1046, 1074, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1039, 1080, 988, 1019, 1028, 1030, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1041, 1004, 1030, 976, 1021, 1065, 1006, 6, 3, 3, 1, 0, 0, 0, 0, 1041, 976, 993, 984, 995, 1049, 1043, 977, 29, 22, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8174, 8200, 8184, 8254, 8268, 8188, 8115] avg=8192.0000 max=8268 max_rank=5 min=8115 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1034, 1025, 1032, 987, 1019, 1058, 983, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1072, 1046, 1061, 1042, 979, 1045, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1037, 1019, 1012, 1042, 1042, 1006, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1013, 989, 1012, 1040, 1039, 1014, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1056, 1006, 1036, 967, 1036, 1034, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1015, 1024, 1057, 1019, 1007, 1061, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1025, 1020, 1037, 1012, 1000, 1013, 1001, 6, 3, 3, 1, 0, 0, 0, 0, 1033, 1016, 1037, 994, 1015, 1020, 999, 978, 28, 23, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8266, 8301, 8173, 8141, 8213, 8136, 8149] avg=8192.0000 max=8301 max_rank=2 min=8136 min_rank=6 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8448, 8192, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8064, 8192, 8192, 8064, 8192, 8192, 8192, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8064, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1008, 1029, 999, 1004, 1024, 1011, 1013, 1049, 1014, 1040, 991, 1072, 1060, 1025, 1024, 1054, 995, 1043, 1040, 1010, 1048, 997, 982, 1032, 1037, 1016, 1042, 996, 1052, 1012, 989, 982, 1088, 1032, 999, 1023, 970, 1038, 1037, 1016, 999, 1078, 1002, 1007, 1044, 1046, 1073, 1029, 1026, 1001, 1043, 992, 1017, 1003, 1036, 1052, 1003, 998, 1033, 1041, 1033, 1014, 1003, 1070] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1057, 997, 1016, 1054, 997, 1036, 1015, 1018, 1003, 1027, 1025, 1053, 1029, 997, 1022, 993, 1041, 1017, 1067, 1012, 1032, 1036, 1015, 999, 940, 1021, 999, 1017, 1041, 1065, 1009, 1021, 1016, 1019, 1087, 1044, 1045, 1045, 977, 967, 1014, 1044, 1028, 1038, 1013, 1033, 1011, 1047, 1026, 1067, 1003, 1068, 1008, 1036, 1025, 1001, 1073, 1056, 994, 1019, 990, 1023, 1028, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1022, 1000, 1072, 1079, 1098, 1032, 1032, 1037, 1035, 1037, 1053, 1031, 1002, 1025, 1045, 1003, 1047, 1046, 1021, 1028, 1023, 1061, 1110, 1014, 1029, 1030, 983, 983, 998, 985, 1057, 1007, 1006, 987, 1006, 1040, 999, 1026, 1009, 1013, 984, 1015, 1039, 985, 1026, 1025, 1033, 1007, 1025, 1046, 1036, 1061, 987, 1019, 1021, 999, 1010, 1016, 1007, 1015, 1029, 998, 1033, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1023, 996, 1010, 988, 1047, 1032, 1020, 1057, 1019, 1019, 1044, 1015, 1047, 992, 930, 1038, 1045, 988, 1024, 979, 1017, 1041, 1059, 1052, 1041, 1048, 1001, 1045, 1032, 1042, 1009, 1020, 1057, 1037, 1035, 1022, 1035, 971, 1021, 999, 1017, 1053, 1022, 1019, 937, 1018, 1089, 1024, 1048, 1004, 1041, 1026, 1013, 1013, 1014, 982, 1022, 1035, 1061, 1084, 1032, 998, 1047, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1003, 1018, 1041, 1019, 1057, 1037, 1033, 1051, 1024, 1043, 1065, 1033, 1047, 1012, 1009, 1052, 998, 1015, 994, 1018, 1033, 1004, 1008, 1032, 1035, 994, 1035, 1030, 1031, 1055, 1036, 1078, 1060, 1075, 1034, 971, 992, 995, 995, 1025, 1026, 992, 1046, 1049, 1019, 1025, 995, 1020, 1037, 1058, 1026, 1005, 986, 956, 989, 1033, 1042, 1005, 1034, 1038, 1023, 1012, 1044, 989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[993, 1096, 1071, 1020, 1037, 1015, 1074, 949, 972, 1027, 1068, 1015, 1029, 1048, 987, 1004, 1044, 1095, 1019, 984, 1023, 1065, 964, 1004, 1058, 1044, 986, 998, 1012, 1057, 1021, 1102, 968, 1004, 1029, 1020, 993, 1001, 1045, 1081, 1036, 1026, 1042, 1038, 988, 1019, 1040, 1028, 1016, 1070, 973, 979, 1056, 999, 1031, 1033, 1053, 993, 1030, 989, 982, 1035, 997, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1003, 1035, 1000, 1015, 1052, 1035, 1016, 1026, 1036, 1046, 983, 1024, 1048, 1009, 1048, 942, 1052, 1058, 1043, 1035, 1050, 1005, 1031, 1002, 1013, 1053, 977, 1051, 1051, 1031, 1006, 1001, 1021, 1009, 1029, 1064, 997, 1063, 1015, 968, 1008, 1038, 996, 1005, 1040, 1036, 987, 1020, 1098, 983, 979, 1082, 1019, 1060, 1019, 1027, 1023, 1074, 983, 1037, 1016, 1020, 994, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1025, 1024, 1052, 1001, 1000, 1047, 1007, 980, 982, 1057, 1059, 1046, 1023, 1019, 1037, 1048, 1006, 1022, 1060, 1055, 1002, 1112, 996, 1028, 1014, 1042, 1036, 1056, 1032, 1014, 1038, 1005, 1047, 1036, 1007, 978, 983, 1010, 1015, 1012, 1021, 1045, 1035, 1025, 1040, 993, 1035, 1037, 1028, 1038, 1008, 996, 988, 969, 1068, 1011, 1001, 1006, 1041, 1036, 1038, 995, 1062, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1008, 1029, 999, 1004, 1004, 1011, 1013, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1032, 991, 1072, 1060, 1025, 1024, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1043, 1040, 1010, 1048, 969, 982, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1016, 1042, 996, 1052, 1012, 989, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 1032, 999, 1023, 970, 1038, 1037, 1016, 28, 10, 0, 0, 0, 0, 0, 0, 999, 1077, 1002, 1007, 1044, 1046, 1073, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1001, 1043, 992, 1017, 1003, 1036, 1052, 10, 8, 1, 0, 0, 0, 0, 0, 1003, 998, 1033, 1041, 1033, 1014, 1003, 1070, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8272, 8119, 8126, 8241, 8277, 8189, 8195] avg=8192.0000 max=8277 max_rank=5 min=8117 min_rank=0 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32902 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1023, 996, 1010, 988, 1027, 1032, 1020, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1011, 1044, 1015, 1047, 992, 930, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 988, 1024, 979, 1017, 1011, 1059, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1048, 1001, 1045, 1032, 1042, 1009, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1037, 1035, 1022, 1035, 971, 1021, 999, 30, 10, 0, 0, 0, 0, 0, 0, 1017, 1052, 1022, 1019, 937, 1018, 1089, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1004, 1041, 1026, 1013, 1013, 1014, 982, 10, 8, 1, 0, 0, 0, 0, 0, 1022, 1035, 1061, 1084, 1032, 998, 1047, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8096, 8175, 8238, 8217, 8178, 8160, 8319] avg=8192.0000 max=8319 max_rank=7 min=8096 min_rank=1 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1057, 997, 1016, 1054, 978, 1036, 1015, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1019, 1025, 1053, 1029, 997, 1022, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1017, 1067, 1012, 1032, 1006, 1015, 999, 0, 0, 0, 0, 0, 0, 0, 0, 940, 1021, 999, 1017, 1041, 1065, 1009, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1019, 1087, 1044, 1045, 1045, 977, 967, 30, 10, 0, 0, 0, 0, 0, 0, 1014, 1043, 1028, 1038, 1013, 1033, 1011, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1067, 1003, 1068, 1008, 1036, 1025, 1001, 9, 8, 1, 0, 0, 0, 0, 0, 1073, 1056, 994, 1019, 990, 1023, 1028, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8141, 8189, 8113, 8240, 8227, 8252, 8203] avg=8192.0000 max=8252 max_rank=6 min=8113 min_rank=3 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32922 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1003, 1018, 1041, 1019, 1037, 1037, 1033, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1035, 1065, 1033, 1047, 1012, 1009, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1015, 994, 1018, 1033, 975, 1008, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 994, 1035, 1030, 1031, 1055, 1036, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1075, 1034, 971, 992, 995, 995, 1025, 29, 10, 0, 0, 0, 0, 0, 0, 1026, 991, 1046, 1049, 1019, 1025, 995, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1058, 1026, 1005, 986, 956, 989, 1033, 10, 8, 1, 0, 0, 0, 0, 0, 1042, 1005, 1034, 1038, 1023, 1012, 1044, 989, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8277, 8073, 8294, 8186, 8171, 8109, 8187] avg=8192.0000 max=8294 max_rank=3 min=8073 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32653 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[993, 1096, 1071, 1020, 1008, 1015, 1074, 949, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1013, 1068, 1015, 1029, 1048, 987, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1095, 1019, 984, 1023, 1031, 964, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1044, 986, 998, 1012, 1052, 1021, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1004, 1029, 1020, 993, 1001, 1045, 1081, 34, 15, 0, 0, 0, 0, 0, 0, 1036, 1020, 1042, 1038, 988, 1019, 1040, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1070, 973, 979, 1056, 999, 1031, 1033, 14, 14, 6, 7, 5, 0, 0, 0, 1053, 993, 1030, 982, 982, 1035, 997, 1061, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8136, 8164, 8273, 8190, 8211, 8203, 8133] avg=8192.0000 max=8273 max_rank=3 min=8133 min_rank=7 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1003, 1035, 1000, 1015, 1032, 1035, 1016, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1038, 983, 1024, 1048, 1009, 1048, 942, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1058, 1043, 1035, 1050, 976, 1031, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1053, 977, 1051, 1051, 1031, 1006, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1009, 1029, 1064, 997, 1063, 1015, 968, 29, 10, 0, 0, 0, 0, 0, 0, 1008, 1037, 996, 1005, 1040, 1036, 987, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1098, 983, 979, 1082, 1019, 1060, 1019, 1027, 10, 8, 1, 0, 0, 0, 0, 0, 1023, 1074, 983, 1037, 1016, 1020, 994, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8128, 8247, 8183, 8205, 8129, 8286, 8196] avg=8192.0000 max=8286 max_rank=6 min=8128 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1025, 1024, 1052, 1001, 981, 1047, 1007, 980, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1049, 1059, 1046, 1023, 1019, 1037, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1022, 1060, 1055, 1002, 1080, 996, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1042, 1036, 1056, 1032, 1014, 1038, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1036, 1007, 978, 983, 1010, 1015, 1012, 32, 10, 0, 0, 0, 0, 0, 0, 1021, 1044, 1035, 1025, 1040, 993, 1035, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1038, 1008, 996, 988, 969, 1068, 1011, 9, 8, 1, 0, 0, 0, 0, 0, 1001, 1006, 1041, 1036, 1038, 995, 1062, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8263, 8249, 8237, 8130, 8230, 8124, 8186] avg=8192.0000 max=8263 max_rank=1 min=8117 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1022, 1000, 1072, 1079, 1077, 1032, 1032, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1029, 1053, 1031, 1002, 1025, 1045, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1046, 1021, 1028, 1023, 1031, 1110, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1030, 983, 983, 998, 985, 1057, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 987, 1006, 1040, 999, 1026, 1009, 1013, 30, 11, 0, 0, 0, 0, 0, 0, 984, 1014, 1039, 985, 1026, 1025, 1033, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1046, 1036, 1061, 987, 1019, 1021, 999, 10, 8, 1, 0, 0, 0, 0, 0, 1010, 1016, 1007, 1015, 1029, 998, 1033, 1009, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8351, 8223, 8320, 8072, 8127, 8113, 8213, 8117] avg=8192.0000 max=8351 max_rank=0 min=8072 min_rank=3 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8064, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8064, 8320, 8192, 128, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[994, 1025, 986, 1038, 1017, 990, 1045, 1022, 1036, 973, 1050, 977, 1097, 1007, 1024, 1010, 1028, 1038, 1035, 1007, 1051, 1030, 1004, 1015, 1019, 1034, 1089, 1000, 989, 1057, 1016, 1062, 1007, 1001, 993, 1038, 1030, 976, 1089, 979, 1055, 973, 1060, 1022, 1066, 1056, 1013, 1017, 1029, 1044, 1049, 987, 1017, 1042, 1048, 1006, 1010, 1004, 1044, 1012, 1005, 1011, 1047, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1049, 1004, 985, 969, 1037, 1021, 997, 1028, 1013, 996, 1081, 1050, 1048, 1036, 993, 979, 1007, 1071, 1019, 1069, 1066, 1024, 1000, 1010, 1015, 1022, 1079, 1031, 1017, 1074, 1052, 1027, 1033, 1012, 1026, 1040, 1016, 991, 1024, 1020, 1016, 1052, 1001, 1032, 1014, 1018, 1054, 995, 998, 1034, 991, 1038, 993, 1063, 1041, 997, 989, 1029, 1006, 988, 1032, 1005, 1056, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1057, 1025, 1012, 998, 1031, 1002, 1057, 980, 1041, 1034, 1057, 1062, 1008, 1018, 1020, 994, 1039, 981, 1083, 1051, 1021, 1033, 983, 1039, 1020, 976, 1037, 1001, 1015, 1083, 1077, 1005, 1034, 980, 1058, 1006, 1033, 1000, 1053, 1000, 978, 1040, 1015, 1064, 1004, 1047, 1000, 1082, 1053, 1035, 1066, 1014, 1075, 975, 996, 1001, 987, 963, 1033, 1009, 1009, 1058, 990, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1011, 1021, 986, 1028, 1043, 1042, 1003, 1114, 972, 992, 1041, 993, 996, 1066, 1014, 1069, 1005, 1001, 957, 1078, 1013, 1031, 1028, 1090, 1038, 1051, 963, 1009, 1058, 985, 1031, 1053, 1038, 1038, 1050, 990, 1034, 1034, 1080, 1008, 1040, 1041, 1020, 1009, 1018, 1000, 1033, 975, 1027, 1038, 1057, 1007, 1011, 1015, 1010, 1022, 1021, 1046, 1044, 1002, 1024, 994, 1023, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1018, 1017, 1054, 1010, 980, 1027, 1014, 1101, 1039, 1050, 1047, 1015, 1004, 1024, 1060, 1050, 952, 1004, 1013, 1075, 995, 1037, 997, 1031, 1024, 1011, 1007, 1064, 1073, 1024, 1007, 1040, 1039, 1033, 997, 1033, 1006, 1026, 1072, 1007, 1022, 1003, 1014, 1019, 993, 1009, 1024, 1018, 1014, 1053, 1042, 997, 1002, 1036, 1009, 1044, 1010, 1025, 1009, 1034, 1008, 1017, 1041, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1040, 991, 1024, 1024, 1019, 1005, 1028, 994, 1062, 1004, 1064, 991, 1025, 1010, 1003, 1058, 1010, 991, 1064, 1029, 1003, 1011, 1002, 1065, 1062, 1049, 1047, 1018, 1003, 994, 1049, 1003, 1052, 1027, 1034, 1034, 1044, 1025, 1034, 1011, 1019, 1031, 1009, 1019, 1029, 1000, 968, 1058, 1041, 1043, 1053, 1012, 1024, 1023, 1000, 1027, 1029, 1058, 958, 1026, 1010, 1022, 1025, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1043, 1058, 1013, 1065, 986, 1021, 971, 1047, 1049, 982, 1005, 1021, 987, 959, 1007, 1076, 1007, 1015, 1021, 1031, 1011, 1040, 1013, 992, 1012, 1057, 1028, 997, 1050, 1024, 1028, 1053, 1011, 1024, 1030, 1042, 1053, 1004, 996, 1042, 993, 1048, 1037, 1074, 993, 1013, 980, 1030, 1010, 1012, 1032, 1008, 1014, 1033, 1068, 1073, 1064, 1051, 1024, 1043, 975, 1032, 1003, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1040, 1043, 1062, 1004, 1032, 998, 1020, 959, 1013, 1025, 983, 1065, 1030, 1025, 1007, 1039, 1059, 1037, 1006, 1002, 989, 1060, 1031, 1023, 1074, 1014, 1019, 1006, 1026, 1052, 991, 973, 1022, 1030, 998, 1038, 962, 1022, 1039, 1013, 980, 1007, 1033, 1050, 1015, 984, 986, 1060, 1017, 1020, 1003, 1043, 1063, 1048, 1040, 1061, 1050, 1036, 1019, 1072, 1011, 1030, 1016, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[994, 1025, 986, 1038, 1017, 990, 1045, 1022, 26, 0, 0, 0, 0, 0, 0, 0, 1036, 973, 1050, 977, 1097, 1007, 1024, 1010, 1, 0, 0, 0, 0, 0, 0, 0, 1028, 1038, 1035, 1007, 1051, 1030, 1004, 1015, 1, 0, 0, 0, 0, 0, 0, 0, 1019, 1034, 1089, 1000, 989, 1017, 1016, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1001, 993, 1038, 1030, 976, 1089, 979, 3, 0, 0, 0, 0, 0, 0, 0, 1055, 973, 1060, 1022, 1066, 1056, 1013, 1017, 14, 11, 0, 0, 0, 0, 0, 0, 1029, 1044, 1025, 987, 1017, 1042, 1048, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1004, 1044, 1012, 1005, 1011, 1047, 1041, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8175, 8209, 8226, 8116, 8287, 8198, 8182] avg=8192.0000 max=8287 max_rank=5 min=8116 min_rank=4 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1049, 1004, 985, 969, 1037, 1021, 997, 1028, 27, 0, 0, 0, 0, 0, 0, 0, 1013, 996, 1081, 1050, 1048, 1036, 993, 979, 1, 0, 0, 0, 0, 0, 0, 0, 1007, 1071, 1019, 1069, 1066, 1024, 1000, 1010, 1, 0, 0, 0, 0, 0, 0, 0, 1015, 1022, 1079, 1031, 1017, 1032, 1052, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1012, 1026, 1040, 1016, 991, 1024, 1020, 2, 0, 0, 0, 0, 0, 0, 0, 1016, 1052, 1001, 1032, 1014, 1018, 1054, 995, 15, 10, 0, 0, 0, 0, 0, 0, 998, 1034, 969, 1038, 993, 1063, 1041, 997, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1029, 1006, 988, 1032, 1005, 1056, 1063, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8197, 8267, 8275, 8164, 8207, 8133, 8176] avg=8192.0000 max=8275 max_rank=3 min=8117 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1040, 991, 1024, 1024, 1019, 1005, 1028, 994, 25, 0, 0, 0, 0, 0, 0, 0, 1062, 1004, 1064, 991, 1025, 1010, 1003, 1058, 1, 0, 0, 0, 0, 0, 0, 0, 1010, 991, 1064, 1029, 1003, 1011, 1002, 1065, 1, 0, 0, 0, 0, 0, 0, 0, 1062, 1049, 1047, 1018, 1003, 955, 1049, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1027, 1034, 1034, 1044, 1025, 1034, 1011, 3, 0, 0, 0, 0, 0, 0, 0, 1019, 1031, 1009, 1019, 1029, 1000, 968, 1058, 14, 11, 0, 0, 0, 0, 0, 0, 1041, 1043, 1029, 1012, 1024, 1023, 1000, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1058, 958, 1026, 1010, 1022, 1025, 1049, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8218, 8176, 8186, 8264, 8158, 8199, 8185] avg=8192.0000 max=8264 max_rank=4 min=8150 min_rank=0 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1057, 1025, 1012, 998, 1031, 1002, 1057, 980, 27, 0, 0, 0, 0, 0, 0, 0, 1041, 1034, 1057, 1062, 1008, 1018, 1020, 994, 1, 0, 0, 0, 0, 0, 0, 0, 1039, 981, 1083, 1051, 1021, 1033, 983, 1039, 1, 0, 0, 0, 0, 0, 0, 0, 1020, 976, 1037, 1001, 1015, 1041, 1077, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 980, 1058, 1006, 1033, 1000, 1053, 1000, 3, 0, 0, 0, 0, 0, 0, 0, 978, 1040, 1015, 1064, 1004, 1047, 1000, 1082, 15, 11, 0, 0, 0, 0, 0, 0, 1053, 1035, 1042, 1014, 1075, 975, 996, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 987, 963, 1033, 1009, 1009, 1058, 990, 1038, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8235, 8231, 8172, 8167, 8256, 8191, 8095] avg=8192.0000 max=8256 max_rank=5 min=8095 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1011, 1021, 986, 1028, 1043, 1042, 1003, 1114, 25, 0, 0, 0, 0, 0, 0, 0, 972, 992, 1041, 993, 996, 1066, 1014, 1069, 1, 0, 0, 0, 0, 0, 0, 0, 1005, 1001, 957, 1078, 1013, 1031, 1028, 1090, 1, 0, 0, 0, 0, 0, 0, 0, 1038, 1051, 963, 1009, 1058, 947, 1031, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1038, 1050, 990, 1034, 1034, 1080, 1008, 3, 0, 0, 0, 0, 0, 0, 0, 1040, 1041, 1020, 1009, 1018, 1000, 1033, 975, 13, 11, 0, 0, 0, 0, 0, 0, 1027, 1038, 1033, 1007, 1011, 1015, 1010, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1046, 1044, 1002, 1024, 994, 1023, 1005, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8144, 8204, 8150, 8275, 8160, 8163, 8167] avg=8192.0000 max=8275 max_rank=4 min=8144 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1018, 1017, 1054, 1010, 980, 1027, 1014, 1101, 26, 0, 0, 0, 0, 0, 0, 0, 1039, 1050, 1047, 1015, 1004, 1024, 1060, 1050, 1, 0, 0, 0, 0, 0, 0, 0, 952, 1004, 1013, 1075, 995, 1037, 997, 1031, 1, 0, 0, 0, 0, 0, 0, 0, 1024, 1011, 1007, 1064, 1073, 984, 1007, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1033, 997, 1033, 1006, 1026, 1072, 1007, 3, 0, 0, 0, 0, 0, 0, 0, 1022, 1003, 1014, 1019, 993, 1009, 1024, 1018, 14, 11, 0, 0, 0, 0, 0, 0, 1014, 1053, 1018, 997, 1002, 1036, 1009, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1025, 1009, 1034, 1008, 1017, 1041, 1016, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8290, 8105, 8210, 8216, 8127, 8173, 8168] avg=8192.0000 max=8290 max_rank=1 min=8105 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1043, 1058, 1013, 1065, 986, 1021, 971, 1047, 29, 0, 0, 0, 0, 0, 0, 0, 1049, 982, 1005, 1021, 987, 959, 1007, 1076, 3, 0, 0, 0, 0, 0, 0, 0, 1007, 1015, 1021, 1031, 1011, 1040, 1013, 992, 6, 0, 0, 0, 0, 0, 0, 0, 1012, 1057, 1028, 997, 1050, 977, 1028, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1024, 1030, 1042, 1053, 1004, 996, 1042, 5, 0, 0, 0, 0, 0, 0, 0, 993, 1048, 1037, 1074, 993, 1013, 980, 1030, 18, 15, 0, 0, 0, 0, 0, 0, 1010, 1012, 990, 1008, 1014, 1033, 1068, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1051, 1024, 1043, 975, 1032, 1003, 1055, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8089, 8136, 8202, 8207, 8201, 8208, 8260] avg=8192.0000 max=8260 max_rank=7 min=8089 min_rank=1 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1040, 1043, 1062, 1004, 1032, 998, 1020, 959, 26, 0, 0, 0, 0, 0, 0, 0, 1013, 1025, 983, 1065, 1030, 1025, 1007, 1039, 1, 0, 0, 0, 0, 0, 0, 0, 1059, 1037, 1006, 1002, 989, 1060, 1031, 1023, 1, 0, 0, 0, 0, 0, 0, 0, 1074, 1014, 1019, 1006, 1026, 1012, 991, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1030, 998, 1038, 962, 1022, 1039, 1013, 3, 0, 0, 0, 0, 0, 0, 0, 980, 1007, 1033, 1050, 1015, 984, 986, 1060, 14, 11, 0, 0, 0, 0, 0, 0, 1017, 1020, 979, 1043, 1063, 1048, 1040, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1036, 1019, 1072, 1011, 1030, 1016, 1061, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8188, 8208, 8115, 8127, 8140, 8271, 8303] avg=8192.0000 max=8303 max_rank=7 min=8115 min_rank=3 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8448, 8192, 8320, 8064, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8448, 8192, 8320, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8320, 8448, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8448, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1054, 1021, 1077, 1078, 975, 1009, 999, 1003, 1034, 1017, 1023, 1009, 1010, 1095, 1002, 995, 989, 1031, 1053, 1051, 977, 1010, 1026, 989, 1041, 1056, 953, 1008, 1007, 983, 1043, 1013, 988, 979, 968, 1000, 1030, 1055, 1086, 1006, 1000, 1052, 1057, 1066, 1067, 1038, 1032, 1010, 1048, 1018, 1068, 1030, 1013, 1013, 1048, 1004, 978, 1037, 986, 1055, 1045, 1034, 1093, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1022, 1022, 1057, 1043, 1067, 1045, 1016, 1030, 1019, 1030, 1039, 1070, 1011, 1052, 1010, 1008, 994, 1049, 1047, 1004, 1049, 1061, 1029, 995, 1016, 996, 1021, 1027, 1015, 1005, 1001, 995, 1059, 1026, 1061, 1018, 1046, 1007, 1033, 1070, 1039, 1000, 1048, 986, 950, 1065, 1047, 1026, 1002, 995, 1060, 1036, 1021, 1032, 948, 1037, 1037, 1014, 974, 1019, 1043, 1038, 970, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[998, 949, 1056, 1031, 1085, 1042, 1003, 979, 1011, 998, 963, 1005, 1051, 1075, 991, 1016, 1027, 1035, 1019, 1061, 1055, 1055, 1019, 1023, 1063, 990, 1047, 959, 1008, 1004, 1032, 998, 1073, 1055, 1023, 1012, 1020, 1041, 1052, 999, 1031, 1051, 1063, 1030, 1013, 1079, 1056, 1023, 1029, 993, 1062, 1024, 1018, 1033, 983, 1054, 998, 1021, 1013, 1026, 1005, 1019, 995, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1050, 1012, 1044, 983, 1081, 1040, 1009, 1080, 1024, 1019, 1008, 1027, 1046, 1052, 1008, 987, 970, 1071, 964, 1031, 1013, 995, 1016, 1016, 1036, 1030, 1065, 1061, 1029, 1019, 1005, 1052, 1076, 1001, 1004, 974, 1011, 1005, 1019, 1030, 988, 1041, 995, 979, 1034, 1074, 1012, 988, 1003, 1065, 1039, 1015, 1061, 1018, 1100, 991, 1027, 1001, 1035, 1053, 1034, 1006, 976, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1055, 1019, 1047, 1021, 1042, 1016, 984, 970, 1031, 1035, 998, 1066, 999, 1049, 993, 1088, 1011, 988, 1012, 1047, 1055, 998, 1033, 1047, 1028, 1000, 984, 1035, 1045, 974, 1086, 1037, 1040, 1115, 1043, 1042, 983, 984, 1009, 1012, 1033, 1073, 941, 987, 1021, 992, 1036, 1049, 1051, 994, 1048, 1007, 1040, 1042, 1004, 1071, 1031, 1020, 1023, 988, 1005, 1010, 994, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1023, 1028, 1016, 1007, 1031, 1017, 1087, 967, 1069, 1045, 1055, 1001, 1006, 1030, 1043, 1036, 974, 1001, 985, 1003, 992, 1041, 1042, 1038, 1003, 1045, 1045, 996, 1002, 1026, 1038, 980, 1033, 1027, 1049, 1020, 1026, 1002, 1045, 965, 1063, 1038, 1004, 1039, 957, 1038, 1045, 1081, 1027, 1017, 1067, 1000, 1008, 1028, 1033, 995, 1007, 1076, 1049, 1036, 1078, 1003, 1011, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1033, 1005, 1077, 1023, 1057, 1036, 999, 1015, 1070, 996, 1001, 1032, 1012, 1045, 1001, 1042, 1015, 1021, 1000, 989, 1040, 985, 1028, 1047, 1039, 1039, 1009, 997, 1045, 1011, 1020, 1016, 1040, 1005, 1053, 1009, 1033, 1003, 1008, 1035, 957, 1052, 1020, 1059, 991, 999, 1054, 1052, 986, 1070, 1084, 1001, 976, 1030, 1044, 1041, 1047, 1043, 1026, 998, 1006, 1047, 982, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1018, 1056, 1049, 1029, 1059, 1031, 1028, 1050, 1061, 951, 977, 1045, 1000, 1002, 1025, 999, 996, 1019, 1044, 1055, 1091, 1001, 1048, 1024, 1012, 987, 1025, 1008, 1014, 989, 1011, 1047, 1052, 1040, 1060, 1004, 1007, 1068, 1058, 1048, 1038, 1019, 1007, 979, 1025, 1016, 987, 1019, 970, 1037, 1033, 1020, 992, 1030, 1020, 1044, 1053, 1029, 1010, 1003, 1043, 1028, 1013, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1022, 1022, 1017, 1043, 1067, 1045, 1016, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1030, 1039, 1070, 1011, 1052, 1010, 1008, 3, 0, 0, 0, 0, 0, 0, 0, 994, 1049, 1047, 1004, 1049, 1061, 1029, 995, 11, 5, 0, 0, 0, 0, 0, 0, 1016, 996, 1021, 1027, 1015, 1005, 1001, 995, 40, 5, 0, 0, 0, 0, 0, 0, 1034, 1026, 1061, 1018, 1046, 1007, 1033, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 992, 1048, 986, 950, 1065, 1047, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 995, 1044, 1036, 1021, 1032, 948, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1014, 974, 1019, 1043, 1038, 970, 984, 20, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8242, 8244, 8121, 8295, 8153, 8115, 8104] avg=8192.0000 max=8295 max_rank=4 min=8104 min_rank=7 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1054, 1021, 1037, 1078, 975, 1009, 999, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1017, 1023, 1009, 1010, 1095, 1002, 995, 3, 0, 0, 0, 0, 0, 0, 0, 989, 1031, 1053, 1051, 977, 1010, 1026, 989, 11, 5, 0, 0, 0, 0, 0, 0, 1041, 1056, 953, 1008, 1007, 983, 1043, 1013, 40, 5, 0, 0, 0, 0, 0, 0, 964, 979, 968, 1000, 1030, 1055, 1086, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1044, 1057, 1066, 1067, 1038, 1032, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1018, 1052, 1030, 1013, 1013, 1048, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1037, 986, 1055, 1045, 1034, 1093, 1001, 19, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8188, 8142, 8149, 8088, 8314, 8226, 8253] avg=8192.0000 max=8314 max_rank=5 min=8088 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[998, 949, 1017, 1031, 1085, 1042, 1003, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 998, 963, 1005, 1051, 1075, 991, 1016, 3, 0, 0, 0, 0, 0, 0, 0, 1027, 1035, 1019, 1061, 1055, 1055, 1019, 1023, 11, 5, 0, 0, 0, 0, 0, 0, 1063, 990, 1047, 959, 1008, 1004, 1032, 998, 39, 5, 0, 0, 0, 0, 0, 0, 1048, 1055, 1023, 1012, 1020, 1041, 1052, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1043, 1063, 1030, 1013, 1079, 1056, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 993, 1046, 1024, 1018, 1033, 983, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1021, 1013, 1026, 1005, 1019, 995, 994, 20, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8113, 8310, 8145, 8250, 8338, 8180, 8096] avg=8192.0000 max=8338 max_rank=5 min=8096 min_rank=7 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1050, 1012, 1005, 983, 1081, 1040, 1009, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1019, 1008, 1027, 1046, 1052, 1008, 987, 3, 0, 0, 0, 0, 0, 0, 0, 970, 1071, 964, 1031, 1013, 995, 1016, 1016, 11, 5, 0, 0, 0, 0, 0, 0, 1036, 1030, 1065, 1061, 1029, 1019, 1005, 1052, 39, 5, 0, 0, 0, 0, 0, 0, 1051, 1001, 1004, 974, 1011, 1005, 1019, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1033, 995, 979, 1034, 1074, 1012, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1065, 1024, 1015, 1061, 1018, 1100, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1001, 1035, 1053, 1034, 1006, 976, 1038, 20, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8174, 8092, 8341, 8095, 8103, 8277, 8194] avg=8192.0000 max=8341 max_rank=3 min=8092 min_rank=2 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1055, 1019, 1008, 1021, 1042, 1016, 984, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1035, 998, 1066, 999, 1049, 993, 1088, 3, 0, 0, 0, 0, 0, 0, 0, 1011, 988, 1012, 1047, 1055, 998, 1033, 1047, 11, 6, 0, 0, 0, 0, 0, 0, 1028, 1000, 984, 1035, 1045, 974, 1086, 1037, 39, 5, 0, 0, 0, 0, 0, 0, 1015, 1115, 1043, 1042, 983, 984, 1009, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1064, 941, 987, 1021, 992, 1036, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 994, 1033, 1007, 1040, 1042, 1004, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1020, 1023, 988, 1005, 1010, 994, 1055, 20, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8262, 8208, 8233, 8203, 8123, 8242, 8150] avg=8192.0000 max=8262 max_rank=1 min=8115 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1023, 1028, 978, 1007, 1031, 1017, 1087, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1045, 1055, 1001, 1006, 1030, 1043, 1036, 3, 0, 0, 0, 0, 0, 0, 0, 974, 1001, 985, 1003, 992, 1041, 1042, 1038, 11, 5, 0, 0, 0, 0, 0, 0, 1003, 1045, 1045, 996, 1002, 1026, 1038, 980, 38, 5, 0, 0, 0, 0, 0, 0, 1008, 1027, 1049, 1020, 1026, 1002, 1045, 965, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1030, 1004, 1039, 957, 1038, 1045, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1017, 1051, 1000, 1008, 1028, 1033, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1076, 1049, 1036, 1078, 1003, 1011, 997, 20, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8288, 8092, 8178, 8142, 8257, 8159, 8282] avg=8192.0000 max=8288 max_rank=1 min=8092 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1033, 1005, 1037, 1023, 1057, 1036, 999, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 996, 1001, 1032, 1012, 1045, 1001, 1042, 3, 0, 0, 0, 0, 0, 0, 0, 1015, 1021, 1000, 989, 1040, 985, 1028, 1047, 12, 5, 0, 0, 0, 0, 0, 0, 1039, 1039, 1009, 997, 1045, 1011, 1020, 1016, 40, 5, 0, 0, 0, 0, 0, 0, 1015, 1005, 1053, 1009, 1033, 1003, 1008, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 957, 1044, 1020, 1059, 991, 999, 1054, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1070, 1067, 1001, 976, 1030, 1044, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1043, 1026, 998, 1006, 1047, 982, 1040, 20, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8202, 8142, 8221, 8161, 8176, 8215, 8214] avg=8192.0000 max=8221 max_rank=3 min=8142 min_rank=2 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1018, 1056, 1005, 1029, 1059, 1031, 1028, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 951, 977, 1045, 1000, 1002, 1025, 999, 7, 0, 0, 0, 0, 0, 0, 0, 996, 1019, 1044, 1055, 1091, 1001, 1048, 1024, 17, 11, 0, 0, 0, 0, 0, 0, 1012, 987, 1025, 1008, 1014, 989, 1011, 1047, 44, 11, 0, 0, 0, 0, 0, 0, 1017, 1040, 1060, 1004, 1007, 1068, 1058, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1001, 1007, 979, 1025, 1016, 987, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1037, 1009, 1020, 992, 1030, 1020, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1029, 1010, 1003, 1043, 1028, 1013, 1033, 24, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8067, 8306, 8148, 8302, 8072, 8122, 8243] avg=8192.0000 max=8306 max_rank=2 min=8067 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8064, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8064, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8448, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8448, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8064, 8320, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8320, 8192, 8448, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1060, 977, 990, 1056, 1034, 1035, 1002, 1056, 1083, 1001, 1035, 1013, 1052, 1078, 1027, 1053, 1058, 1032, 1063, 1014, 1050, 1039, 1012, 961, 1003, 1027, 1015, 976, 1029, 1004, 1025, 1014, 1006, 1021, 1035, 1000, 1017, 982, 1034, 1001, 1001, 1027, 1043, 1005, 1046, 1043, 994, 973, 992, 1033, 1029, 1011, 1038, 1035, 1004, 1104, 1032, 1053, 1019, 1013, 1069, 989, 978, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1026, 1091, 1056, 1028, 1012, 1024, 981, 962, 1009, 995, 1037, 1033, 1036, 1000, 1017, 991, 995, 1021, 1046, 1027, 1004, 995, 1054, 1017, 1011, 960, 1042, 1019, 1045, 1070, 1032, 1064, 996, 1066, 1048, 1004, 999, 1030, 1008, 1062, 1108, 1003, 1079, 1035, 990, 1030, 1014, 1023, 1017, 989, 1009, 1041, 987, 1051, 993, 1045, 1023, 1027, 981, 1017, 1041, 1019, 1069, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1050, 990, 1093, 1013, 1046, 1023, 952, 1072, 985, 990, 988, 1044, 1032, 1026, 1036, 1036, 1042, 1018, 997, 976, 1067, 1074, 1016, 1028, 1015, 1028, 992, 1014, 1056, 996, 1083, 970, 1058, 984, 1028, 1036, 1014, 990, 1044, 1054, 1038, 1021, 1000, 1015, 1030, 1028, 1034, 1016, 1017, 1015, 997, 1034, 1018, 1033, 1017, 1040, 1009, 1023, 1011, 1043, 1020, 1051, 1032, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1054, 1006, 1062, 1060, 973, 991, 1048, 1036, 1063, 1015, 1040, 993, 992, 1063, 1033, 1057, 1021, 1026, 1041, 1079, 1012, 1035, 1043, 1026, 1030, 1025, 1004, 1030, 1009, 980, 1006, 1077, 1030, 1010, 956, 993, 1038, 982, 1008, 1012, 1035, 1081, 1039, 1016, 1030, 985, 1019, 1013, 1029, 1001, 1016, 1007, 976, 1033, 1065, 1036, 1016, 1007, 1028, 1004, 1002, 1075, 1037, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1036, 1046, 1035, 1002, 1058, 997, 1063, 1038, 1047, 999, 1013, 999, 1025, 978, 1069, 1049, 1058, 1015, 998, 1009, 1036, 997, 1035, 1037, 999, 1023, 1048, 1030, 1042, 1050, 1009, 1021, 1011, 1036, 1019, 1039, 1002, 974, 1018, 1071, 1025, 1011, 1041, 1047, 1021, 1025, 1031, 972, 1007, 1000, 1069, 953, 1044, 1103, 1061, 995, 1011, 1007, 1005, 1064, 1025, 1013, 970, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1053, 1041, 1036, 1071, 1011, 1047, 1006, 1017, 991, 1043, 1002, 1054, 1056, 1002, 971, 1066, 1023, 1014, 1027, 1002, 987, 1003, 1031, 1022, 991, 993, 1039, 1074, 1056, 1078, 1041, 1042, 979, 952, 1069, 1039, 1040, 1007, 1034, 1026, 969, 1067, 1014, 1047, 1019, 1055, 990, 1015, 1014, 1011, 1056, 998, 1017, 1018, 1004, 1011, 977, 1078, 1018, 1058, 980, 1024, 998, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1025, 1020, 1051, 1009, 1007, 1017, 1023, 1064, 1040, 1007, 1003, 1017, 1006, 1018, 1015, 1023, 1043, 1023, 999, 1039, 985, 1068, 1013, 1003, 1018, 1019, 1087, 1004, 1035, 1014, 1043, 1002, 1024, 1032, 1058, 999, 988, 1053, 1032, 1057, 1060, 1001, 997, 1040, 1042, 1032, 1031, 1000, 1057, 1024, 990, 1040, 1053, 996, 1053, 1057, 1028, 1032, 996, 1011, 1028, 1008, 963, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1050, 967, 1070, 1057, 1008, 993, 1055, 1017, 1058, 998, 1025, 964, 991, 1019, 994, 1025, 978, 995, 1078, 1055, 1032, 1046, 1076, 1023, 1009, 987, 1023, 1080, 1039, 1040, 1000, 1065, 992, 1024, 996, 1095, 1001, 1043, 1056, 985, 1096, 1012, 998, 988, 1016, 996, 1032, 990, 965, 1023, 1078, 1002, 1039, 1079, 1047, 996, 1030, 960, 1045, 1018, 1057, 1046, 1015, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 1091, 1017, 1028, 1012, 1024, 981, 962, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 995, 1037, 1033, 1036, 1000, 1017, 991, 12, 2, 0, 0, 0, 0, 0, 0, 995, 1021, 1046, 1027, 1004, 995, 1042, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 960, 1042, 1019, 1031, 1070, 1032, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1066, 1048, 1004, 999, 1030, 1008, 1062, 30, 0, 0, 0, 0, 0, 0, 0, 1108, 1003, 1079, 1035, 990, 1030, 1014, 1023, 4, 0, 0, 0, 0, 0, 0, 0, 1017, 989, 1009, 1041, 987, 1047, 993, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1027, 981, 1017, 1041, 1019, 1069, 1032, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8132, 8147, 8229, 8243, 8286, 8128, 8230] avg=8192.0000 max=8286 max_rank=5 min=8128 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32887 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1054, 1006, 1023, 1060, 973, 991, 1048, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1015, 1040, 993, 992, 1063, 1033, 1057, 12, 2, 0, 0, 0, 0, 0, 0, 1021, 1026, 1041, 1079, 1012, 1035, 1031, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1025, 1004, 1030, 996, 980, 1006, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1010, 956, 993, 1038, 982, 1008, 1012, 30, 0, 0, 0, 0, 0, 0, 0, 1035, 1081, 1039, 1016, 1030, 985, 1019, 1013, 4, 0, 0, 0, 0, 0, 0, 0, 1029, 1001, 1016, 1007, 976, 1029, 1065, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1007, 1028, 1004, 1002, 1075, 1037, 1027, 11, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8270, 8271, 8148, 8059, 8222, 8159, 8216] avg=8192.0000 max=8271 max_rank=2 min=8059 min_rank=4 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1050, 990, 1053, 1013, 1046, 1023, 952, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 985, 990, 988, 1044, 1032, 1026, 1036, 1036, 11, 2, 0, 0, 0, 0, 0, 0, 1042, 1018, 997, 976, 1067, 1074, 1005, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1028, 992, 1014, 1042, 996, 1083, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 984, 1028, 1036, 1014, 990, 1044, 1054, 31, 0, 0, 0, 0, 0, 0, 0, 1038, 1021, 1000, 1015, 1030, 1028, 1034, 1016, 4, 0, 0, 0, 0, 0, 0, 0, 1017, 1015, 997, 1034, 1018, 1029, 1017, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1023, 1011, 1043, 1020, 1051, 1032, 1038, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8150, 8207, 8140, 8239, 8186, 8167, 8248] avg=8192.0000 max=8248 max_rank=7 min=8140 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1036, 1046, 997, 1002, 1058, 997, 1063, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 999, 1013, 999, 1025, 978, 1069, 1049, 12, 2, 0, 0, 0, 0, 0, 0, 1058, 1015, 998, 1009, 1036, 997, 1023, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1023, 1048, 1030, 1028, 1050, 1009, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1036, 1019, 1039, 1002, 974, 1018, 1071, 29, 0, 0, 0, 0, 0, 0, 0, 1025, 1011, 1041, 1047, 1021, 1025, 1031, 972, 4, 0, 0, 0, 0, 0, 0, 0, 1007, 1000, 1069, 953, 1044, 1099, 1061, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1007, 1005, 1064, 1025, 1013, 970, 1005, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8193, 8173, 8208, 8199, 8177, 8228, 8121] avg=8192.0000 max=8237 max_rank=0 min=8121 min_rank=7 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1060, 977, 954, 1056, 1034, 1035, 1002, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 1001, 1035, 1013, 1052, 1078, 1027, 1053, 11, 2, 0, 0, 0, 0, 0, 0, 1058, 1032, 1063, 1014, 1050, 1039, 1001, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1027, 1015, 976, 1015, 1004, 1025, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1021, 1035, 1000, 1017, 982, 1034, 1001, 28, 0, 0, 0, 0, 0, 0, 0, 1001, 1027, 1043, 1005, 1046, 1043, 994, 973, 4, 0, 0, 0, 0, 0, 0, 0, 992, 1033, 1029, 1011, 1038, 1031, 1004, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1053, 1019, 1013, 1069, 989, 978, 1035, 12, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8355, 8218, 8079, 8124, 8136, 8242, 8208] avg=8192.0000 max=8355 max_rank=1 min=8079 min_rank=3 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1053, 1041, 998, 1071, 1011, 1047, 1006, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1043, 1002, 1054, 1056, 1002, 971, 1066, 12, 2, 0, 0, 0, 0, 0, 0, 1023, 1014, 1027, 1002, 987, 1003, 1019, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 991, 993, 1039, 1074, 1042, 1078, 1041, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 979, 952, 1069, 1039, 1040, 1007, 1034, 1026, 29, 0, 0, 0, 0, 0, 0, 0, 969, 1067, 1014, 1047, 1019, 1055, 990, 1015, 4, 0, 0, 0, 0, 0, 0, 0, 1014, 1011, 1056, 998, 1017, 1014, 1004, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1078, 1018, 1058, 980, 1024, 998, 1062, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8199, 8097, 8300, 8175, 8180, 8125, 8216] avg=8192.0000 max=8300 max_rank=3 min=8097 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 1020, 1013, 1009, 1007, 1017, 1023, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1007, 1003, 1017, 1006, 1018, 1015, 1023, 11, 2, 0, 0, 0, 0, 0, 0, 1043, 1023, 999, 1039, 985, 1068, 1002, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1019, 1087, 1004, 1021, 1014, 1043, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1032, 1058, 999, 988, 1053, 1032, 1057, 29, 0, 0, 0, 0, 0, 0, 0, 1060, 1001, 997, 1040, 1042, 1032, 1031, 1000, 4, 0, 0, 0, 0, 0, 0, 0, 1057, 1024, 990, 1040, 1053, 992, 1053, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1032, 996, 1011, 1028, 1008, 963, 1014, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8142, 8162, 8208, 8272, 8207, 8266, 8101] avg=8192.0000 max=8272 max_rank=4 min=8101 min_rank=7 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1050, 967, 1025, 1057, 1008, 993, 1055, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 998, 1025, 964, 991, 1019, 994, 1025, 16, 5, 0, 0, 0, 0, 0, 0, 978, 995, 1078, 1055, 1032, 1046, 1054, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 987, 1023, 1080, 1020, 1040, 1000, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1024, 996, 1095, 1001, 1043, 1056, 985, 33, 0, 0, 0, 0, 0, 0, 0, 1096, 1012, 998, 988, 1016, 996, 1032, 990, 8, 6, 0, 0, 0, 0, 0, 0, 965, 1023, 1078, 1002, 1039, 1071, 1047, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 960, 1045, 1018, 1057, 1046, 1015, 999, 14, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8095, 8261, 8224, 8225, 8142, 8221, 8196] avg=8192.0000 max=8261 max_rank=2 min=8095 min_rank=1 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8448, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8448, 8320, 8320, 8320, 8320, 8320, 8192, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8064, 8320, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8320, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1039, 1021, 997, 1043, 1052, 1062, 1040, 1011, 1054, 1046, 1012, 1048, 1000, 1036, 978, 1061, 1062, 975, 1072, 1007, 1013, 965, 1040, 1036, 995, 1005, 1034, 1055, 1047, 1001, 1000, 973, 1010, 1019, 1028, 958, 987, 1016, 1052, 1053, 1023, 1056, 1020, 1046, 1002, 1001, 1008, 1048, 1017, 1011, 1042, 1019, 1051, 1014, 1044, 1023, 1014, 1054, 1021, 975, 1075, 979, 1053, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1043, 1025, 1058, 994, 1062, 1026, 1043, 1033, 1005, 1038, 988, 1047, 1040, 1005, 1028, 1010, 1011, 1003, 980, 982, 1078, 1006, 1029, 1046, 1034, 981, 1033, 1062, 1068, 1027, 1064, 1044, 1012, 986, 1018, 1062, 1034, 1001, 1043, 1027, 1016, 1005, 1002, 1002, 1022, 1011, 1051, 1045, 1058, 1008, 1008, 971, 1049, 1061, 1042, 1013, 1024, 1016, 977, 1020, 1027, 997, 1049, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1045, 1060, 999, 1001, 1061, 1003, 992, 993, 1031, 1020, 1003, 1044, 1015, 1017, 1056, 974, 991, 1038, 1001, 1000, 994, 1029, 1067, 1045, 1014, 1054, 1005, 1051, 1029, 1029, 1015, 1003, 1060, 992, 996, 1036, 1048, 1029, 984, 1013, 998, 999, 1024, 1003, 999, 1043, 1018, 1080, 1083, 1010, 1056, 1022, 1105, 1037, 1055, 1011, 1058, 1012, 960, 1008, 1027, 1053, 1035, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[991, 1068, 997, 1029, 1078, 1004, 1013, 1029, 1016, 1007, 1040, 1052, 1017, 1029, 972, 1019, 1015, 1013, 1026, 1030, 975, 1074, 957, 1080, 1028, 1046, 1058, 1043, 1034, 987, 1018, 1042, 1044, 1009, 988, 1025, 1055, 1068, 1007, 1041, 1027, 1024, 1034, 1004, 1039, 1002, 1033, 1034, 1026, 1033, 1013, 1014, 1045, 995, 1025, 1003, 1084, 1027, 1045, 972, 984, 1021, 1005, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1018, 1003, 1009, 983, 1040, 1058, 1020, 1041, 1006, 1015, 995, 1037, 1070, 1065, 1029, 1047, 1072, 987, 1063, 1024, 1047, 1018, 1002, 992, 1074, 1047, 1007, 1023, 995, 1035, 1040, 1029, 1005, 1037, 982, 1013, 994, 1024, 1038, 1031, 1027, 1032, 1036, 1075, 1026, 1031, 1031, 1007, 1053, 1027, 992, 1051, 1001, 978, 1015, 1023, 1066, 1014, 983, 1027, 1023, 1030, 975, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[988, 1021, 1027, 1036, 992, 1049, 1012, 1023, 1036, 1083, 1025, 1015, 1007, 1050, 1008, 1025, 1035, 1054, 1037, 967, 1031, 1028, 1036, 1052, 1019, 1029, 1031, 1042, 1010, 1024, 1057, 999, 1014, 999, 982, 1034, 1055, 1031, 1017, 1027, 1001, 999, 1010, 987, 1002, 1016, 1045, 1076, 1051, 1020, 1034, 1021, 1058, 997, 1011, 1008, 1016, 1051, 1023, 999, 1024, 1015, 1041, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1019, 1017, 1075, 1046, 1014, 1044, 972, 1022, 1022, 1063, 1010, 1018, 1037, 1042, 1034, 1030, 1059, 1065, 977, 1054, 976, 1053, 1018, 1030, 1038, 970, 1036, 1013, 1025, 1027, 1047, 1047, 1042, 1046, 1058, 1055, 979, 1007, 968, 1015, 989, 987, 1015, 1010, 993, 1067, 1047, 1006, 1070, 1000, 1035, 1017, 988, 991, 1033, 1040, 1061, 1017, 1010, 1052, 973, 973, 1051, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1050, 1036, 989, 1042, 1048, 1016, 1056, 1057, 1037, 1005, 1040, 1049, 1031, 986, 1070, 1062, 1020, 999, 1007, 1012, 1016, 1009, 1008, 997, 962, 981, 1056, 1010, 1051, 1052, 998, 1013, 1054, 1034, 978, 992, 989, 1021, 1034, 1000, 1028, 1099, 1058, 988, 1048, 1005, 1053, 994, 1012, 1013, 1026, 1023, 959, 1026, 1050, 1021, 997, 1016, 1037, 1036, 1051, 1035, 1022, 1072] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1039, 1021, 997, 1043, 1028, 1062, 1040, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1046, 1012, 1021, 1000, 1036, 978, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 975, 1072, 1007, 1013, 965, 1040, 1036, 13, 5, 0, 0, 0, 0, 0, 0, 995, 1005, 1034, 1040, 1047, 1001, 1000, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1019, 1028, 958, 987, 1016, 1052, 1053, 27, 7, 0, 0, 0, 0, 0, 0, 1023, 1056, 1020, 1046, 1002, 1001, 1008, 1048, 3, 0, 0, 0, 0, 0, 0, 0, 1009, 1011, 1042, 1019, 1051, 1014, 1044, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1054, 1021, 975, 1075, 979, 1053, 1037, 17, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8208, 8188, 8095, 8157, 8207, 8213, 8227] avg=8192.0000 max=8241 max_rank=0 min=8095 min_rank=3 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1043, 1025, 1058, 994, 1038, 1026, 1043, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1038, 988, 1020, 1040, 1005, 1028, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1003, 980, 982, 1078, 1006, 1029, 1046, 13, 5, 0, 0, 0, 0, 0, 0, 1034, 981, 1033, 1047, 1068, 1027, 1064, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 986, 1018, 1062, 1034, 1001, 1043, 1027, 27, 7, 0, 0, 0, 0, 0, 0, 1016, 1005, 1002, 1002, 1022, 1011, 1051, 1045, 3, 0, 0, 0, 0, 0, 0, 0, 1050, 1008, 1008, 971, 1049, 1061, 1042, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1016, 977, 1020, 1027, 997, 1049, 986, 17, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8134, 8153, 8298, 8217, 8157, 8202, 8115] avg=8192.0000 max=8298 max_rank=3 min=8115 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1018, 1003, 1009, 983, 1016, 1058, 1020, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1015, 995, 1010, 1070, 1065, 1029, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 987, 1063, 1024, 1047, 1018, 1002, 992, 13, 5, 0, 0, 0, 0, 0, 0, 1074, 1047, 1007, 1008, 995, 1035, 1040, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1037, 982, 1013, 994, 1024, 1038, 1031, 27, 7, 0, 0, 0, 0, 0, 0, 1027, 1032, 1036, 1075, 1026, 1031, 1031, 1007, 3, 0, 0, 0, 0, 0, 0, 0, 1045, 1027, 992, 1051, 1001, 978, 1015, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1014, 983, 1027, 1023, 1030, 975, 998, 17, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8237, 8223, 8235, 8158, 8268, 8132, 8135] avg=8192.0000 max=8268 max_rank=5 min=8132 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1045, 1060, 999, 1001, 1037, 1003, 992, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1020, 1003, 1017, 1015, 1017, 1056, 974, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1038, 1001, 1000, 994, 1029, 1067, 1045, 13, 5, 0, 0, 0, 0, 0, 0, 1014, 1054, 1005, 1036, 1029, 1029, 1015, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 992, 996, 1036, 1048, 1029, 984, 1013, 27, 7, 0, 0, 0, 0, 0, 0, 998, 999, 1024, 1003, 999, 1043, 1018, 1080, 3, 0, 0, 0, 0, 0, 0, 0, 1075, 1010, 1056, 1022, 1105, 1037, 1055, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1012, 960, 1008, 1027, 1053, 1035, 1003, 17, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8133, 8183, 8185, 8192, 8167, 8371, 8175] avg=8192.0000 max=8371 max_rank=6 min=8130 min_rank=0 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32905 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[988, 1021, 1027, 1036, 969, 1049, 1012, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1083, 1025, 989, 1007, 1050, 1008, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1054, 1037, 967, 1031, 1028, 1036, 1052, 13, 5, 0, 0, 0, 0, 0, 0, 1019, 1029, 1031, 1027, 1010, 1024, 1057, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 999, 982, 1034, 1055, 1031, 1017, 1027, 26, 7, 0, 0, 0, 0, 0, 0, 1001, 999, 1010, 987, 1002, 1016, 1045, 1076, 3, 0, 0, 0, 0, 0, 0, 0, 1043, 1020, 1034, 1021, 1058, 997, 1011, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1051, 1023, 999, 1024, 1015, 1041, 1024, 16, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8223, 8258, 8196, 8192, 8139, 8192, 8211] avg=8192.0000 max=8258 max_rank=2 min=8125 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[991, 1068, 997, 1029, 1053, 1004, 1013, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1007, 1040, 1025, 1017, 1029, 972, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1013, 1026, 1030, 975, 1074, 957, 1080, 13, 5, 0, 0, 0, 0, 0, 0, 1028, 1046, 1058, 1028, 1034, 987, 1018, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1009, 988, 1025, 1055, 1068, 1007, 1041, 27, 7, 0, 0, 0, 0, 0, 0, 1027, 1024, 1034, 1004, 1039, 1002, 1033, 1034, 3, 0, 0, 0, 0, 0, 0, 0, 1018, 1033, 1013, 1014, 1045, 995, 1025, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 1027, 1045, 972, 984, 1021, 1005, 1023, 18, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8125, 8188, 8241, 8271, 8200, 8146, 8181] avg=8192.0000 max=8271 max_rank=4 min=8125 min_rank=1 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1019, 1017, 1075, 1046, 990, 1044, 972, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1063, 1010, 991, 1037, 1042, 1034, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1065, 977, 1054, 976, 1053, 1018, 1030, 13, 5, 0, 0, 0, 0, 0, 0, 1038, 970, 1036, 998, 1025, 1027, 1047, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1046, 1058, 1055, 979, 1007, 968, 1015, 27, 7, 0, 0, 0, 0, 0, 0, 989, 987, 1015, 1010, 993, 1067, 1047, 1006, 3, 0, 0, 0, 0, 0, 0, 0, 1062, 1000, 1035, 1017, 988, 991, 1033, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1017, 1010, 1052, 973, 973, 1051, 1041, 17, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8229, 8250, 8188, 8204, 8117, 8166, 8197] avg=8192.0000 max=8250 max_rank=2 min=8117 min_rank=5 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1050, 1036, 989, 1042, 1017, 1016, 1056, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1005, 1040, 1016, 1031, 986, 1070, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 999, 1007, 1012, 1016, 1009, 1008, 997, 17, 8, 0, 0, 0, 0, 0, 0, 962, 981, 1056, 985, 1051, 1052, 998, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1034, 978, 992, 989, 1021, 1034, 1000, 33, 10, 0, 0, 0, 0, 0, 0, 1028, 1099, 1058, 988, 1048, 1005, 1053, 994, 8, 0, 0, 0, 0, 0, 0, 0, 996, 1013, 1026, 1023, 959, 1026, 1050, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1016, 1037, 1036, 1051, 1035, 1022, 1072, 21, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8247, 8093, 8098, 8145, 8281, 8114, 8295] avg=8192.0000 max=8295 max_rank=7 min=8093 min_rank=2 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8192, 8192, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8064, 8192, 8192, 8320, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1005, 1084, 995, 1048, 1043, 1010, 1014, 1027, 1004, 970, 999, 1019, 1059, 1040, 1018, 1067, 987, 999, 1006, 1024, 1006, 1070, 1023, 1000, 973, 1027, 1017, 1028, 1064, 1026, 1041, 1073, 1034, 985, 1105, 1039, 932, 1078, 1025, 1029, 1031, 991, 1012, 1077, 996, 1020, 990, 1036, 981, 980, 1053, 1016, 1028, 1038, 1022, 982, 1049, 1030, 1056, 1052, 1039, 1023, 1036, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1020, 986, 1066, 1005, 1052, 1081, 1067, 1013, 1037, 994, 1051, 1064, 1008, 1026, 1032, 1059, 1023, 1016, 1030, 1020, 1004, 1046, 964, 1002, 1053, 1062, 1034, 1002, 1013, 1002, 1019, 987, 972, 1026, 1002, 1058, 997, 1031, 1001, 970, 1055, 1048, 983, 985, 1029, 1045, 1021, 1015, 993, 1014, 1028, 1081, 1010, 989, 1026, 1048, 1040, 1022, 1018, 1033, 1003, 1101, 1000, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1069, 1014, 1028, 1057, 1061, 1000, 1014, 1008, 995, 1032, 1009, 1023, 983, 1076, 1005, 1041, 966, 973, 1045, 980, 1061, 1067, 1013, 1008, 1052, 1033, 1007, 1039, 1066, 952, 1028, 1054, 1061, 1087, 1024, 1071, 978, 987, 1061, 991, 1024, 1020, 1006, 986, 1051, 977, 1057, 1020, 993, 1039, 1096, 1003, 947, 1015, 1072, 1041, 1027, 990, 997, 1039, 996, 1003, 1066, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1054, 1040, 1036, 1058, 1030, 1009, 1037, 1007, 985, 1072, 1007, 1055, 1026, 1002, 1039, 1024, 1033, 1070, 1021, 1048, 994, 1005, 1003, 990, 1041, 1047, 996, 1001, 972, 1017, 994, 1060, 989, 1037, 1071, 1038, 1039, 1043, 1013, 1016, 1029, 1056, 1008, 1003, 990, 995, 1028, 1042, 1046, 1055, 1028, 1044, 1004, 1024, 1041, 1034, 1049, 1025, 1013, 978, 986, 1001, 995, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1015, 999, 999, 1053, 1003, 1035, 1024, 1041, 1036, 1005, 1024, 1069, 965, 988, 1001, 1070, 1006, 969, 1041, 1030, 1023, 1054, 1083, 1053, 1031, 1024, 982, 982, 951, 1026, 1019, 995, 951, 1060, 1014, 987, 1040, 1040, 1038, 1026, 1052, 1009, 1065, 1027, 1007, 1041, 978, 1034, 1002, 1023, 1053, 1007, 1028, 1061, 1060, 1029, 1069, 1055, 1021, 994, 1043, 1039, 1052, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1030, 998, 1059, 1052, 1016, 996, 1072, 1027, 984, 961, 1058, 1009, 1010, 999, 1082, 1002, 1026, 1018, 1087, 1011, 1057, 1050, 1023, 1047, 1008, 1063, 981, 1006, 995, 1091, 1025, 1002, 1004, 985, 1021, 1050, 1008, 1049, 1010, 1009, 1042, 1035, 1049, 1012, 1026, 1017, 1016, 1048, 1033, 1057, 1035, 1033, 1043, 1050, 995, 1040, 1015, 1010, 1021, 985, 997, 1054, 973, 969] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1016, 1007, 992, 1021, 1038, 1017, 1064, 1040, 1042, 1042, 1001, 1040, 1058, 1034, 990, 1020, 1049, 1035, 1057, 1004, 1006, 1032, 1036, 1028, 1055, 1036, 1044, 1034, 1006, 1033, 1004, 1032, 1040, 1037, 1030, 1045, 977, 1031, 1016, 1043, 1005, 988, 1012, 1042, 1000, 1011, 1021, 1051, 1036, 1023, 1060, 969, 1013, 1044, 1000, 1022, 1009, 1066, 1029, 954, 1054, 974, 970, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1018, 1020, 989, 1002, 1024, 1007, 1046, 990, 1007, 1004, 999, 1004, 996, 980, 987, 1006, 1016, 1080, 990, 1037, 980, 1027, 1034, 1036, 1012, 1024, 1016, 1091, 1034, 1024, 993, 1064, 1028, 1058, 1034, 1065, 1028, 1005, 1030, 1010, 1025, 1040, 1033, 1027, 1029, 1004, 1005, 984, 994, 1015, 1040, 1018, 1059, 1059, 979, 1038, 1057, 1068, 1069, 977, 1047, 1062, 1079, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1005, 1084, 995, 1048, 1043, 1010, 989, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 970, 999, 1019, 1059, 1040, 1018, 1067, 25, 3, 0, 0, 0, 0, 0, 0, 987, 999, 1006, 1024, 1006, 1070, 1023, 1000, 1, 0, 0, 0, 0, 0, 0, 0, 973, 1027, 1017, 1028, 1064, 1026, 1041, 1073, 8, 1, 0, 0, 0, 0, 0, 0, 1034, 985, 1105, 1037, 932, 1078, 1025, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 991, 1012, 1077, 996, 1020, 990, 1036, 15, 6, 0, 0, 0, 0, 0, 0, 981, 980, 1035, 1016, 1028, 1038, 1022, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1030, 1056, 1052, 1039, 1023, 1036, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8204, 8116, 8258, 8225, 8174, 8082, 8276] avg=8192.0000 max=8276 max_rank=7 min=8082 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1069, 1014, 1028, 1057, 1061, 1000, 989, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1032, 1009, 1023, 983, 1076, 1005, 1041, 25, 3, 0, 0, 0, 0, 0, 0, 966, 973, 1045, 980, 1061, 1067, 1013, 1008, 1, 0, 0, 0, 0, 0, 0, 0, 1052, 1033, 1007, 1039, 1066, 952, 1028, 1054, 7, 1, 0, 0, 0, 0, 0, 0, 1061, 1087, 1024, 1069, 978, 987, 1061, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1020, 1006, 986, 1051, 977, 1057, 1020, 16, 6, 0, 0, 0, 0, 0, 0, 993, 1039, 1077, 1003, 947, 1015, 1072, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 990, 997, 1039, 996, 1003, 1066, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8192, 8114, 8239, 8258, 8163, 8187, 8157] avg=8192.0000 max=8258 max_rank=4 min=8114 min_rank=2 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1015, 999, 999, 1053, 1003, 1035, 998, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1005, 1024, 1069, 965, 988, 1001, 1070, 26, 3, 0, 0, 0, 0, 0, 0, 1006, 969, 1041, 1030, 1023, 1054, 1083, 1053, 1, 0, 0, 0, 0, 0, 0, 0, 1031, 1024, 982, 982, 951, 1026, 1019, 995, 8, 1, 0, 0, 0, 0, 0, 0, 951, 1060, 1014, 985, 1040, 1040, 1038, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1009, 1065, 1027, 1007, 1041, 978, 1034, 15, 6, 0, 0, 0, 0, 0, 0, 1002, 1023, 1035, 1007, 1028, 1061, 1060, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1055, 1021, 994, 1043, 1039, 1052, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8187, 8260, 8019, 8154, 8234, 8245, 8294] avg=8192.0000 max=8294 max_rank=7 min=8019 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32927 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1016, 1007, 992, 1021, 1038, 1017, 1037, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1042, 1001, 1040, 1058, 1034, 990, 1020, 27, 3, 0, 0, 0, 0, 0, 0, 1049, 1035, 1057, 1004, 1006, 1032, 1036, 1028, 1, 0, 0, 0, 0, 0, 0, 0, 1055, 1036, 1044, 1034, 1006, 1033, 1004, 1032, 7, 1, 0, 0, 0, 0, 0, 0, 1040, 1037, 1030, 1043, 977, 1031, 1016, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 988, 1012, 1042, 1000, 1011, 1021, 1051, 15, 6, 0, 0, 0, 0, 0, 0, 1036, 1023, 1042, 969, 1013, 1044, 1000, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1066, 1029, 954, 1054, 974, 970, 1051, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8257, 8248, 8252, 8217, 8151, 8149, 8094] avg=8192.0000 max=8257 max_rank=1 min=8094 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32611 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1030, 998, 1059, 1052, 1016, 996, 1045, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 984, 961, 1058, 1009, 1010, 999, 1082, 1002, 27, 3, 0, 0, 0, 0, 0, 0, 1026, 1018, 1087, 1011, 1057, 1050, 1023, 1047, 1, 0, 0, 0, 0, 0, 0, 0, 1008, 1063, 981, 1006, 995, 1091, 1025, 1002, 7, 1, 0, 0, 0, 0, 0, 0, 1004, 985, 1021, 1048, 1008, 1049, 1010, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1035, 1049, 1012, 1026, 1017, 1016, 1048, 15, 6, 0, 0, 0, 0, 0, 0, 1033, 1057, 1017, 1033, 1043, 1050, 995, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1010, 1021, 985, 997, 1054, 973, 969, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8135, 8320, 8179, 8134, 8266, 8268, 8011] avg=8192.0000 max=8320 max_rank=2 min=8011 min_rank=7 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1054, 1040, 1036, 1058, 1030, 1009, 1011, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1072, 1007, 1055, 1026, 1002, 1039, 1024, 26, 3, 0, 0, 0, 0, 0, 0, 1033, 1070, 1021, 1048, 994, 1005, 1003, 990, 1, 0, 0, 0, 0, 0, 0, 0, 1041, 1047, 996, 1001, 972, 1017, 994, 1060, 8, 1, 0, 0, 0, 0, 0, 0, 989, 1037, 1071, 1036, 1039, 1043, 1013, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1056, 1008, 1003, 990, 995, 1028, 1042, 15, 6, 0, 0, 0, 0, 0, 0, 1046, 1055, 1010, 1044, 1004, 1024, 1041, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1025, 1013, 978, 986, 1001, 995, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8239, 8165, 8137, 8244, 8172, 8258, 8076] avg=8192.0000 max=8258 max_rank=6 min=8076 min_rank=7 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1020, 986, 1066, 1005, 1052, 1081, 1040, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 994, 1051, 1064, 1008, 1026, 1032, 1059, 27, 3, 0, 0, 0, 0, 0, 0, 1023, 1016, 1030, 1020, 1004, 1046, 964, 1002, 1, 0, 0, 0, 0, 0, 0, 0, 1053, 1062, 1034, 1002, 1013, 1002, 1019, 987, 8, 1, 0, 0, 0, 0, 0, 0, 972, 1026, 1002, 1056, 997, 1031, 1001, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1048, 983, 985, 1029, 1045, 1021, 1015, 15, 6, 0, 0, 0, 0, 0, 0, 993, 1014, 1010, 1081, 1010, 989, 1026, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1022, 1018, 1033, 1003, 1101, 1000, 1054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8301, 8106, 8181, 8055, 8202, 8171, 8257] avg=8192.0000 max=8301 max_rank=1 min=8055 min_rank=4 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1018, 1020, 989, 1002, 1024, 1007, 1017, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1004, 999, 1004, 996, 980, 987, 1006, 29, 9, 0, 0, 0, 0, 0, 0, 1016, 1080, 990, 1037, 980, 1027, 1034, 1036, 7, 0, 0, 0, 0, 0, 0, 0, 1012, 1024, 1016, 1091, 1034, 1024, 993, 1064, 11, 2, 0, 0, 0, 0, 0, 0, 1028, 1058, 1034, 1056, 1028, 1005, 1030, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1040, 1033, 1027, 1029, 1004, 1005, 984, 17, 10, 0, 0, 0, 0, 0, 0, 994, 1015, 1014, 1018, 1059, 1059, 979, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1068, 1069, 977, 1047, 1062, 1079, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8067, 8021, 8207, 8271, 8249, 8174, 8176, 8371] avg=8192.0000 max=8371 max_rank=7 min=8021 min_rank=1 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32970 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8448, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8064, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8448, 8064, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[989, 1028, 991, 944, 1041, 965, 1035, 988, 1004, 1035, 1031, 1007, 1058, 1022, 1009, 1029, 1053, 1081, 1023, 1029, 1029, 1005, 1062, 979, 1006, 1055, 1015, 1064, 1052, 1036, 972, 1029, 1002, 1055, 982, 1010, 1022, 1082, 1052, 1088, 986, 1069, 1007, 1021, 1044, 1072, 1054, 985, 1023, 1010, 1025, 995, 1032, 1003, 1015, 1027, 1079, 1003, 1025, 1051, 1055, 1032, 992, 972] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[959, 1024, 1035, 997, 1018, 1017, 1011, 999, 990, 1046, 1051, 1020, 1032, 1062, 1046, 1039, 1057, 1004, 1070, 1039, 1007, 1006, 1068, 978, 1020, 1033, 979, 1054, 1043, 991, 1009, 1004, 1041, 1025, 1063, 1065, 1003, 988, 982, 1035, 1046, 940, 1040, 1008, 1055, 985, 1005, 1024, 1060, 983, 1083, 992, 976, 1062, 1060, 1056, 1026, 1053, 1000, 1019, 1082, 995, 1056, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1079, 1064, 1008, 975, 1001, 1099, 1016, 1076, 1002, 1010, 1075, 1063, 999, 1012, 1015, 1031, 1034, 1056, 1006, 1020, 1002, 1010, 1033, 1031, 1014, 1014, 1001, 1019, 984, 1036, 1067, 1058, 982, 1013, 1016, 982, 995, 996, 1003, 1012, 1066, 983, 971, 1040, 1026, 1054, 994, 1004, 1026, 1065, 984, 1068, 1035, 1025, 1027, 1025, 998, 1027, 1078, 1034, 1038, 997, 1041, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1021, 1043, 1014, 992, 1025, 1003, 1016, 1020, 1039, 1032, 1028, 1028, 1054, 1002, 1009, 985, 1047, 1024, 992, 1058, 1056, 1007, 1058, 1039, 984, 1025, 997, 1003, 1016, 977, 1076, 1026, 1032, 1052, 1036, 1014, 1050, 1032, 1044, 1052, 1009, 1016, 993, 1027, 1005, 1025, 1021, 1021, 1025, 1035, 1045, 1057, 991, 1028, 974, 1039, 1040, 1037, 1016, 1034, 1052, 1015, 1013, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1021, 996, 1031, 988, 1056, 1045, 1026, 1031, 1012, 1053, 1019, 1013, 958, 1034, 1009, 1007, 1025, 1036, 1025, 1025, 1028, 1003, 982, 1077, 1023, 976, 1055, 1006, 1072, 1070, 1063, 1030, 1055, 1026, 1047, 1024, 1020, 990, 1017, 1055, 1058, 992, 1018, 996, 1017, 1017, 1020, 1063, 1021, 1029, 1029, 967, 995, 1069, 1014, 1013, 1050, 1009, 1010, 1055, 1035, 991, 1009, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1040, 1056, 1042, 1001, 990, 1017, 1066, 1001, 1025, 999, 986, 995, 1053, 1015, 1054, 1028, 1052, 1039, 1003, 1028, 1026, 993, 1019, 1043, 1066, 1034, 1043, 1000, 1000, 1056, 1027, 1010, 1042, 1044, 1043, 1042, 997, 1085, 1006, 1026, 1045, 1040, 1010, 1027, 1018, 1001, 1039, 1009, 1002, 1004, 993, 993, 993, 1007, 1029, 1007, 1008, 1037, 1063, 1020, 1012, 1025, 1044, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1035, 989, 1024, 1040, 976, 1016, 1009, 1057, 975, 979, 1021, 1025, 1053, 1020, 1014, 1005, 1003, 1021, 1018, 1047, 1043, 1070, 1013, 1011, 1029, 998, 1065, 1004, 1032, 1007, 1022, 994, 1029, 1048, 1011, 1037, 1057, 1033, 1042, 1060, 1026, 1022, 1023, 991, 1026, 999, 987, 1073, 1041, 1026, 1086, 1003, 1032, 1022, 1020, 1016, 1055, 1027, 1057, 1016, 1058, 996, 977, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[994, 1077, 1024, 1033, 1075, 1067, 1063, 1025, 1015, 1004, 977, 1033, 1061, 1082, 1029, 1018, 1013, 998, 1043, 1047, 1014, 993, 1037, 987, 1001, 1043, 1032, 1029, 1020, 1036, 990, 1066, 999, 1011, 986, 992, 1096, 1032, 998, 1027, 1003, 1044, 1027, 1023, 995, 1003, 1024, 1049, 1004, 950, 1048, 1068, 1033, 1003, 1024, 991, 1026, 1038, 1042, 993, 990, 1024, 1053, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1079, 1064, 1008, 975, 1001, 1099, 1016, 1076, 12, 3, 0, 0, 0, 0, 0, 0, 1002, 1010, 1075, 1063, 999, 1012, 1015, 1031, 9, 2, 0, 0, 0, 0, 0, 0, 1034, 1056, 1006, 999, 1002, 1010, 1033, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1014, 1001, 1019, 984, 1036, 1065, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1013, 1016, 982, 995, 996, 1003, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 983, 971, 1040, 1026, 1054, 994, 1004, 29, 0, 0, 0, 0, 0, 0, 0, 1026, 1065, 984, 1068, 1035, 1025, 1027, 1025, 18, 0, 0, 0, 0, 0, 0, 0, 998, 1027, 1078, 1034, 1017, 997, 1041, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8333, 8218, 8171, 8191, 7970, 8167, 8273, 8213] avg=8192.0000 max=8333 max_rank=0 min=7970 min_rank=4 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[989, 1028, 991, 944, 1041, 965, 1035, 988, 13, 3, 0, 0, 0, 0, 0, 0, 1004, 1035, 1031, 1007, 1058, 1022, 1009, 1029, 9, 2, 0, 0, 0, 0, 0, 0, 1053, 1081, 1023, 1008, 1029, 1005, 1062, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1055, 1015, 1064, 1052, 1036, 970, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1055, 982, 1010, 1022, 1082, 1052, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1069, 1007, 1021, 1044, 1072, 1054, 985, 32, 0, 0, 0, 0, 0, 0, 0, 1023, 1010, 1025, 995, 1032, 1003, 1015, 1027, 18, 0, 0, 0, 0, 0, 0, 0, 1079, 1003, 1025, 1051, 1033, 1032, 992, 972, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7997, 8206, 8240, 8227, 8261, 8270, 8148, 8187] avg=8192.0000 max=8270 max_rank=5 min=7997 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32866 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1021, 1043, 1014, 992, 1025, 1003, 1016, 1020, 13, 3, 0, 0, 0, 0, 0, 0, 1039, 1032, 1028, 1028, 1054, 1002, 1009, 985, 9, 2, 0, 0, 0, 0, 0, 0, 1047, 1024, 992, 1035, 1056, 1007, 1058, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1025, 997, 1003, 1016, 977, 1074, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1052, 1036, 1014, 1050, 1032, 1044, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1016, 993, 1027, 1005, 1025, 1021, 1021, 31, 1, 0, 0, 0, 0, 0, 0, 1025, 1035, 1045, 1057, 991, 1028, 974, 1039, 19, 0, 0, 0, 0, 0, 0, 0, 1040, 1037, 1016, 1034, 1030, 1015, 1013, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8188, 8258, 8102, 8281, 8149, 8213, 8195] avg=8192.0000 max=8281 max_rank=4 min=8102 min_rank=3 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1021, 996, 1031, 988, 1056, 1045, 1026, 1031, 12, 3, 0, 0, 0, 0, 0, 0, 1012, 1053, 1019, 1013, 958, 1034, 1009, 1007, 9, 2, 0, 0, 0, 0, 0, 0, 1025, 1036, 1025, 1004, 1028, 1003, 982, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 976, 1055, 1006, 1072, 1070, 1061, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1026, 1047, 1024, 1020, 990, 1017, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 992, 1018, 996, 1017, 1017, 1020, 1063, 31, 0, 0, 0, 0, 0, 0, 0, 1021, 1029, 1029, 967, 995, 1069, 1014, 1013, 18, 0, 0, 0, 0, 0, 0, 0, 1050, 1009, 1010, 1055, 1014, 991, 1009, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8116, 8180, 8293, 8203, 8212, 8155, 8168] avg=8192.0000 max=8293 max_rank=3 min=8116 min_rank=1 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1040, 1056, 1042, 1001, 990, 1017, 1066, 1001, 12, 3, 0, 0, 0, 0, 0, 0, 1025, 999, 986, 995, 1053, 1015, 1054, 1028, 9, 2, 0, 0, 0, 0, 0, 0, 1052, 1039, 1003, 1007, 1026, 993, 1019, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1034, 1043, 1000, 1000, 1056, 1025, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1044, 1043, 1042, 997, 1085, 1006, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1040, 1010, 1027, 1018, 1001, 1039, 1009, 30, 0, 0, 0, 0, 0, 0, 0, 1002, 1004, 993, 993, 993, 1007, 1029, 1007, 18, 0, 0, 0, 0, 0, 0, 0, 1008, 1037, 1063, 1020, 991, 1025, 1044, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8166, 8182, 8234, 8255, 8219, 8046, 8206] avg=8192.0000 max=8255 max_rank=4 min=8046 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1035, 989, 1024, 1040, 976, 1016, 1009, 1057, 13, 3, 0, 0, 0, 0, 0, 0, 975, 979, 1021, 1025, 1053, 1020, 1014, 1005, 9, 2, 0, 0, 0, 0, 0, 0, 1003, 1021, 1018, 1024, 1043, 1070, 1013, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 998, 1065, 1004, 1032, 1007, 1020, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1048, 1011, 1037, 1057, 1033, 1042, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1022, 1023, 991, 1026, 999, 987, 1073, 31, 1, 0, 0, 0, 0, 0, 0, 1041, 1026, 1086, 1003, 1032, 1022, 1020, 1016, 19, 0, 0, 0, 0, 0, 0, 0, 1055, 1027, 1057, 1016, 1036, 996, 977, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8103, 8203, 8149, 8286, 8179, 8265, 8189] avg=8192.0000 max=8286 max_rank=4 min=8103 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[994, 1077, 1024, 1033, 1075, 1067, 1063, 1025, 16, 7, 0, 0, 0, 0, 0, 0, 1015, 1004, 977, 1033, 1061, 1082, 1029, 1018, 14, 8, 0, 0, 0, 0, 0, 0, 1013, 998, 1043, 1011, 1014, 993, 1037, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1043, 1032, 1029, 1020, 1036, 982, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1011, 986, 992, 1096, 1032, 998, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1044, 1027, 1023, 995, 1003, 1024, 1049, 33, 5, 0, 0, 0, 0, 0, 0, 1004, 950, 1048, 1068, 1033, 1003, 1024, 991, 24, 0, 0, 0, 0, 0, 0, 0, 1026, 1038, 1042, 993, 960, 1024, 1053, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8381, 8241, 8096, 8209, 8108, 8206, 8145, 8150] avg=8192.0000 max=8381 max_rank=0 min=8096 min_rank=2 imbalance_ratio=1.023071 max_load_reduction=57155 max_load_reduction_pct=0.872116 moved_to_echo=32609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[959, 1024, 1035, 997, 1018, 1017, 1011, 999, 13, 3, 0, 0, 0, 0, 0, 0, 990, 1046, 1051, 1020, 1032, 1062, 1046, 1039, 10, 2, 0, 0, 0, 0, 0, 0, 1057, 1004, 1070, 1016, 1007, 1006, 1068, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1033, 979, 1054, 1043, 991, 1007, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1025, 1063, 1065, 1003, 988, 982, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 940, 1040, 1008, 1055, 985, 1005, 1024, 30, 1, 0, 0, 0, 0, 0, 0, 1060, 983, 1083, 992, 976, 1062, 1060, 1056, 19, 0, 0, 0, 0, 0, 0, 0, 1026, 1053, 1000, 1019, 1059, 995, 1056, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8076, 8298, 8206, 8131, 8172, 8134, 8291, 8228] avg=8192.0000 max=8298 max_rank=1 min=8076 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8064, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8192, 8320, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1023, 1019, 1019, 1037, 1010, 990, 1064, 999, 1011, 1026, 1045, 1041, 1011, 1018, 986, 1036, 989, 1020, 1003, 1063, 1032, 1001, 1025, 1017, 993, 1041, 1009, 1039, 993, 1008, 1039, 1055, 986, 1025, 1022, 1039, 1042, 1023, 1022, 1035, 1029, 1003, 1009, 1071, 989, 1007, 1046, 986, 989, 1061, 1043, 1046, 1027, 1019, 1044, 1023, 1054, 1063, 981, 1040, 1042, 1009, 1072, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1001, 1020, 939, 1026, 986, 1027, 991, 985, 1048, 1063, 977, 1083, 1037, 1022, 1006, 1023, 1026, 1041, 1076, 1007, 1002, 1021, 1002, 1016, 1025, 1040, 1020, 999, 1011, 1024, 1014, 1010, 1024, 1027, 987, 1086, 1031, 1005, 982, 1028, 1060, 1056, 992, 1018, 1065, 991, 1072, 1028, 993, 1009, 1002, 1041, 1057, 1071, 1014, 1046, 1067, 1050, 993, 998, 1060, 1053, 1025, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1026, 1018, 996, 1014, 973, 1041, 1057, 1046, 1058, 1002, 1008, 1028, 1019, 1067, 988, 1004, 970, 1032, 1041, 993, 1011, 1017, 1053, 1000, 1007, 1047, 1003, 998, 1042, 1001, 1019, 1045, 1062, 1036, 1023, 995, 1008, 1021, 972, 1057, 1022, 1042, 1034, 1010, 1016, 1016, 1023, 1010, 995, 1019, 1027, 1034, 994, 1027, 1082, 1027, 1048, 1004, 1086, 1059, 1071, 1059, 995, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1041, 1045, 1017, 999, 1021, 1018, 1027, 1039, 966, 994, 1008, 1044, 1008, 972, 1067, 1053, 1006, 1007, 1046, 995, 1050, 1037, 963, 975, 1030, 1017, 1075, 1014, 1032, 1037, 1031, 994, 1029, 996, 1049, 1053, 1039, 1081, 1055, 1064, 1064, 1057, 972, 966, 996, 1044, 1003, 952, 1016, 1021, 1049, 1027, 997, 1020, 995, 1012, 1099, 988, 1020, 1007, 1109, 1046, 1066, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1077, 1013, 1005, 1027, 1034, 1034, 1003, 1049, 1014, 1033, 974, 1049, 966, 1041, 1047, 1026, 1025, 1037, 1007, 1029, 1064, 1008, 1030, 974, 991, 1012, 992, 999, 997, 1043, 1017, 976, 1019, 1033, 1050, 993, 1041, 1028, 1056, 1068, 967, 1101, 1004, 1056, 1042, 1029, 1028, 1029, 1020, 954, 1024, 1063, 1010, 1020, 1002, 1048, 1064, 1048, 1071, 999, 1024, 993, 1013, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1025, 995, 1053, 1008, 1063, 1056, 1028, 1047, 1001, 1020, 1022, 994, 1052, 1004, 1006, 1098, 1047, 1009, 1035, 1020, 1010, 1060, 1006, 996, 1088, 1044, 1030, 984, 989, 1048, 1034, 1009, 982, 1021, 1015, 1093, 1033, 1052, 1034, 1076, 968, 1016, 1044, 1051, 1009, 1014, 1031, 996, 1061, 982, 1032, 996, 1021, 958, 985, 1004, 1014, 1011, 1068, 1015, 1003, 1023, 1013, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1017, 1032, 1098, 1051, 1009, 1048, 1034, 1007, 982, 1009, 1036, 1047, 1004, 1004, 988, 1007, 1013, 1030, 1036, 1051, 983, 1068, 1033, 979, 988, 1034, 1076, 996, 1027, 1022, 1008, 1065, 1002, 1046, 993, 1039, 1055, 1002, 1008, 1082, 1022, 999, 1038, 948, 1049, 1027, 1008, 976, 977, 1002, 984, 1016, 1084, 1020, 992, 1029, 1062, 1061, 1003, 1055, 1040, 1027, 1051, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[985, 1004, 1024, 970, 991, 1029, 1044, 987, 1029, 1008, 961, 1044, 1032, 1051, 1007, 1025, 1025, 1060, 1002, 1071, 1019, 1041, 1032, 1042, 960, 1041, 1037, 1027, 1022, 1045, 1033, 1063, 972, 1051, 1009, 1010, 998, 1025, 1029, 1072, 1007, 1038, 1051, 1035, 1046, 1042, 1038, 1047, 1040, 1010, 1045, 1009, 1032, 997, 1021, 1001, 1045, 1059, 961, 1017, 1029, 1037, 1000, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1001, 1020, 939, 1026, 986, 1027, 991, 985, 22, 0, 0, 0, 0, 0, 0, 0, 1048, 1063, 977, 1083, 1037, 1022, 1006, 1023, 29, 0, 0, 0, 0, 0, 0, 0, 1026, 1041, 1076, 1007, 1002, 1021, 1002, 1016, 11, 7, 0, 0, 0, 0, 0, 0, 1025, 1040, 1020, 999, 1011, 1024, 1014, 1010, 15, 0, 0, 0, 0, 0, 0, 0, 1024, 1027, 987, 1086, 1031, 1005, 982, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1056, 992, 1018, 1065, 991, 1072, 1028, 15, 0, 0, 0, 0, 0, 0, 0, 993, 1009, 1002, 1041, 1057, 1071, 1014, 1046, 34, 0, 0, 0, 0, 0, 0, 0, 975, 1050, 993, 998, 1060, 1053, 1025, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7997, 8288, 8209, 8158, 8129, 8297, 8267, 8191] avg=8192.0000 max=8297 max_rank=5 min=7997 min_rank=0 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1023, 1019, 1019, 1037, 1010, 990, 1064, 999, 21, 0, 0, 0, 0, 0, 0, 0, 1011, 1026, 1045, 1041, 1011, 1018, 986, 1036, 29, 0, 0, 0, 0, 0, 0, 0, 989, 1020, 1003, 1063, 1032, 1001, 1025, 1017, 11, 7, 0, 0, 0, 0, 0, 0, 993, 1041, 1009, 1039, 993, 1008, 1039, 1055, 15, 0, 0, 0, 0, 0, 0, 0, 986, 1025, 1022, 1039, 1042, 1023, 1022, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1003, 1009, 1071, 989, 1007, 1046, 986, 15, 0, 0, 0, 0, 0, 0, 0, 989, 1061, 1043, 1046, 1027, 1019, 1044, 1023, 33, 0, 0, 0, 0, 0, 0, 0, 964, 1063, 981, 1040, 1042, 1009, 1072, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8203, 8168, 8192, 8153, 8155, 8285, 8198] avg=8192.0000 max=8285 max_rank=6 min=8153 min_rank=4 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1025, 995, 1053, 1008, 1063, 1056, 1028, 1047, 20, 0, 0, 0, 0, 0, 0, 0, 1001, 1020, 1022, 994, 1052, 1004, 1006, 1098, 28, 0, 0, 0, 0, 0, 0, 0, 1047, 1009, 1035, 1020, 1010, 1060, 1006, 996, 12, 7, 0, 0, 0, 0, 0, 0, 1088, 1044, 1030, 984, 989, 1048, 1034, 1009, 16, 0, 0, 0, 0, 0, 0, 0, 982, 1021, 1015, 1093, 1033, 1052, 1034, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1016, 1044, 1051, 1009, 1014, 1031, 996, 16, 0, 0, 0, 0, 0, 0, 0, 1061, 982, 1032, 996, 1021, 958, 985, 1004, 32, 0, 0, 0, 0, 0, 0, 0, 927, 1011, 1068, 1015, 1003, 1023, 1013, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8225, 8202, 8242, 8262, 8145, 8071, 8094] avg=8192.0000 max=8295 max_rank=0 min=8071 min_rank=6 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1026, 1018, 996, 1014, 973, 1041, 1057, 1046, 21, 0, 0, 0, 0, 0, 0, 0, 1058, 1002, 1008, 1028, 1019, 1067, 988, 1004, 29, 0, 0, 0, 0, 0, 0, 0, 970, 1032, 1041, 993, 1011, 1017, 1053, 1000, 12, 7, 0, 0, 0, 0, 0, 0, 1007, 1047, 1003, 998, 1042, 1001, 1019, 1045, 15, 0, 0, 0, 0, 0, 0, 0, 1062, 1036, 1023, 995, 1008, 1021, 972, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1042, 1034, 1010, 1016, 1016, 1023, 1010, 16, 0, 0, 0, 0, 0, 0, 0, 995, 1019, 1027, 1034, 994, 1027, 1082, 1027, 33, 0, 0, 0, 0, 0, 0, 0, 958, 1004, 1086, 1059, 1071, 1059, 995, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8203, 8136, 8177, 8131, 8189, 8238, 8270] avg=8192.0000 max=8270 max_rank=7 min=8131 min_rank=4 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1017, 1032, 1098, 1051, 1009, 1048, 1034, 1007, 21, 0, 0, 0, 0, 0, 0, 0, 982, 1009, 1036, 1047, 1004, 1004, 988, 1007, 29, 0, 0, 0, 0, 0, 0, 0, 1013, 1030, 1036, 1051, 983, 1068, 1033, 979, 12, 7, 0, 0, 0, 0, 0, 0, 988, 1034, 1076, 996, 1027, 1022, 1008, 1065, 16, 0, 0, 0, 0, 0, 0, 0, 1002, 1046, 993, 1039, 1055, 1002, 1008, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 999, 1038, 948, 1049, 1027, 1008, 976, 16, 0, 0, 0, 0, 0, 0, 0, 977, 1002, 984, 1016, 1084, 1020, 992, 1029, 33, 0, 0, 0, 0, 0, 0, 0, 972, 1061, 1003, 1055, 1040, 1027, 1051, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 8106, 8212, 8232, 8183, 8083, 8137, 8266] avg=8192.0000 max=8317 max_rank=0 min=8083 min_rank=5 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1077, 1013, 1005, 1027, 1034, 1034, 1003, 1049, 22, 0, 0, 0, 0, 0, 0, 0, 1014, 1033, 974, 1049, 966, 1041, 1047, 1026, 29, 0, 0, 0, 0, 0, 0, 0, 1025, 1037, 1007, 1029, 1064, 1008, 1030, 974, 12, 7, 0, 0, 0, 0, 0, 0, 991, 1012, 992, 999, 997, 1043, 1017, 976, 15, 0, 0, 0, 0, 0, 0, 0, 1019, 1033, 1050, 993, 1041, 1028, 1056, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1101, 1004, 1056, 1042, 1029, 1028, 1029, 16, 0, 0, 0, 0, 0, 0, 0, 1020, 954, 1024, 1063, 1010, 1020, 1002, 1048, 33, 0, 0, 0, 0, 0, 0, 0, 973, 1048, 1071, 999, 1024, 993, 1013, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8179, 8193, 8042, 8245, 8272, 8174, 8167] avg=8192.0000 max=8272 max_rank=5 min=8042 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1041, 1045, 1017, 999, 1021, 1018, 1027, 1039, 27, 0, 0, 0, 0, 0, 0, 0, 966, 994, 1008, 1044, 1008, 972, 1067, 1053, 34, 0, 0, 0, 0, 0, 0, 0, 1006, 1007, 1046, 995, 1050, 1037, 963, 975, 15, 11, 0, 0, 0, 0, 0, 0, 1030, 1017, 1075, 1014, 1032, 1037, 1031, 994, 19, 0, 0, 0, 0, 0, 0, 0, 1029, 996, 1049, 1053, 1039, 1081, 1055, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1057, 972, 966, 996, 1044, 1003, 952, 21, 0, 0, 0, 0, 0, 0, 0, 1016, 1021, 1049, 1027, 997, 1020, 995, 1012, 39, 0, 0, 0, 0, 0, 0, 0, 988, 988, 1020, 1007, 1109, 1046, 1066, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8146, 8105, 8249, 8311, 8075, 8176, 8240] avg=8192.0000 max=8311 max_rank=4 min=8075 min_rank=5 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[985, 1004, 1024, 970, 991, 1029, 1044, 987, 21, 0, 0, 0, 0, 0, 0, 0, 1029, 1008, 961, 1044, 1032, 1051, 1007, 1025, 29, 0, 0, 0, 0, 0, 0, 0, 1025, 1060, 1002, 1071, 1019, 1041, 1032, 1042, 12, 7, 0, 0, 0, 0, 0, 0, 960, 1041, 1037, 1027, 1022, 1045, 1033, 1063, 16, 0, 0, 0, 0, 0, 0, 0, 972, 1051, 1009, 1010, 998, 1025, 1029, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1038, 1051, 1035, 1046, 1042, 1038, 1047, 16, 0, 0, 0, 0, 0, 0, 0, 1040, 1010, 1045, 1009, 1032, 997, 1021, 1001, 33, 0, 0, 0, 0, 0, 0, 0, 955, 1059, 961, 1017, 1029, 1037, 1000, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8055, 8186, 8311, 8244, 8122, 8320, 8188, 8110] avg=8192.0000 max=8320 max_rank=5 min=8055 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7808, 8320, 8192, 8192, 8448, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8064, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8320, 8320, 8320, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8192, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8064, 8448, 8192, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8064, 8192, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1006, 1051, 1073, 1009, 985, 1023, 1023, 1021, 1020, 1066, 1024, 1031, 1039, 1015, 1057, 998, 1033, 1029, 1024, 996, 1026, 1032, 1053, 1061, 971, 978, 970, 1033, 1014, 1039, 1029, 1012, 988, 1002, 1009, 1008, 1074, 1024, 1027, 1008, 1011, 1059, 1018, 1056, 1072, 1042, 968, 1014, 1027, 992, 989, 1014, 1034, 1047, 1044, 1023, 988, 1034, 1032, 1042, 1004, 1011, 1082, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1061, 1003, 1067, 1002, 1005, 1069, 1030, 1033, 1012, 996, 1016, 1021, 1047, 1010, 1015, 1023, 999, 1004, 1078, 1041, 983, 1093, 1004, 1041, 992, 991, 1042, 991, 1033, 1022, 1021, 1043, 1005, 1033, 1001, 1034, 1013, 1033, 1056, 1064, 988, 1000, 1043, 981, 1026, 1025, 1033, 1030, 986, 1020, 1041, 992, 1033, 1065, 1038, 1023, 1049, 1020, 1020, 1065, 1011, 987, 982, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1028, 998, 950, 1027, 1029, 1029, 1037, 998, 1017, 1003, 1029, 1055, 1095, 1056, 1009, 996, 985, 962, 1012, 1015, 1011, 1043, 1004, 1044, 985, 1003, 1051, 1035, 1051, 1005, 1056, 1003, 1014, 1024, 1094, 1021, 1047, 1039, 1074, 1053, 1068, 978, 1028, 1046, 1044, 1024, 982, 1043, 1012, 1024, 989, 1051, 1053, 1069, 1017, 992, 1058, 1002, 969, 1051, 1052, 989, 1047, 961] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[986, 989, 1041, 1011, 986, 1050, 1014, 1035, 986, 1021, 1088, 1006, 1026, 1036, 1004, 978, 985, 1073, 1112, 1001, 1013, 1030, 1009, 991, 1055, 1016, 1009, 975, 1023, 1081, 1003, 1013, 984, 1047, 985, 1033, 1038, 1016, 999, 1033, 1077, 1042, 1009, 1008, 1022, 1041, 1016, 1100, 1029, 1009, 1034, 1031, 1059, 1035, 986, 989, 1078, 1012, 1031, 1038, 1034, 1005, 1018, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[978, 1010, 1058, 1025, 1031, 1044, 1046, 1089, 1032, 1045, 1032, 1008, 1045, 1029, 1012, 1063, 1024, 1009, 1051, 1026, 1016, 1030, 1024, 1009, 997, 1051, 1060, 1012, 994, 1018, 1019, 1022, 1027, 1042, 1017, 1074, 1050, 1020, 1011, 1075, 1049, 976, 1027, 1033, 1017, 1039, 998, 1077, 1011, 936, 1029, 1012, 998, 1022, 991, 925, 1017, 1060, 980, 1037, 1045, 1004, 1022, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1053, 1002, 1063, 1024, 1024, 979, 1031, 1030, 958, 1044, 1031, 1009, 974, 989, 1008, 1007, 1040, 1088, 968, 1030, 1047, 1040, 1008, 1025, 1028, 1080, 1000, 1090, 999, 1013, 1001, 1043, 1028, 1020, 1009, 1056, 1003, 1012, 1003, 1041, 1029, 1070, 1023, 1059, 1027, 1007, 1009, 1039, 1029, 984, 995, 1061, 1018, 1060, 1025, 1056, 984, 1011, 1070, 1003, 994, 1037, 1006, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1035, 1077, 1047, 1053, 1022, 1019, 995, 1015, 995, 995, 1044, 1070, 1013, 1019, 1088, 1018, 1086, 1008, 1024, 993, 1072, 1011, 1013, 957, 1025, 1019, 1000, 1012, 1050, 1014, 1041, 1032, 962, 1040, 1018, 1070, 1019, 1025, 1052, 1055, 1010, 980, 999, 1055, 1011, 1009, 1015, 974, 1066, 975, 995, 1061, 992, 1016, 1005, 1069, 992, 1030, 999, 999, 1033, 1061, 1050, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1000, 1001, 1024, 998, 1006, 1011, 1026, 1001, 973, 991, 1023, 1022, 967, 980, 1022, 1034, 1017, 983, 1035, 1075, 1033, 1040, 1054, 991, 1053, 1052, 1032, 1015, 1056, 1013, 1046, 1029, 980, 1033, 1023, 1020, 1087, 1003, 1027, 1027, 1041, 1051, 1028, 1011, 1057, 1040, 1060, 1023, 1048, 1014, 1036, 1028, 1028, 1044, 1042, 1000, 987, 985, 1072, 1069, 1011, 991, 1005, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1006, 1051, 1073, 1009, 985, 1023, 1023, 1021, 6, 0, 0, 0, 0, 0, 0, 0, 1020, 1066, 1024, 1031, 1039, 1015, 1057, 998, 17, 6, 0, 0, 0, 0, 0, 0, 1033, 1029, 1024, 996, 1026, 1024, 1053, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 971, 978, 970, 1033, 1014, 1039, 1029, 1012, 6, 2, 0, 0, 0, 0, 0, 0, 988, 1002, 1009, 1008, 1074, 1024, 1027, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1059, 1018, 1056, 1072, 1042, 968, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 992, 989, 1014, 1034, 1047, 1044, 1023, 26, 0, 0, 0, 0, 0, 0, 0, 988, 1034, 1032, 1042, 1004, 1011, 1082, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8273, 8246, 8054, 8108, 8217, 8196, 8245] avg=8192.0000 max=8273 max_rank=1 min=8054 min_rank=3 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1061, 1003, 1067, 1002, 1005, 1069, 1030, 1033, 6, 0, 0, 0, 0, 0, 0, 0, 1012, 996, 1016, 1021, 1047, 1010, 1015, 1023, 18, 7, 0, 0, 0, 0, 0, 0, 999, 1004, 1078, 1041, 983, 1084, 1004, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 992, 991, 1042, 991, 1033, 1022, 1021, 1043, 6, 2, 0, 0, 0, 0, 0, 0, 1005, 1033, 1001, 1034, 1013, 1033, 1056, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1000, 1043, 981, 1026, 1025, 1033, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1020, 1041, 992, 1033, 1065, 1038, 1023, 27, 0, 0, 0, 0, 0, 0, 0, 1049, 1020, 1020, 1065, 1011, 987, 982, 1051, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8165, 8234, 8143, 8205, 8102, 8225, 8186] avg=8192.0000 max=8276 max_rank=0 min=8102 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1028, 998, 950, 1027, 1029, 1029, 1037, 998, 6, 0, 0, 0, 0, 0, 0, 0, 1017, 1003, 1029, 1055, 1095, 1056, 1009, 996, 18, 6, 0, 0, 0, 0, 0, 0, 985, 962, 1012, 1015, 1011, 1034, 1004, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1003, 1051, 1035, 1051, 1005, 1056, 1003, 6, 2, 0, 0, 0, 0, 0, 0, 1014, 1024, 1094, 1021, 1047, 1039, 1074, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 978, 1028, 1046, 1044, 1024, 982, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1024, 989, 1051, 1053, 1069, 1017, 992, 27, 0, 0, 0, 0, 0, 0, 0, 1058, 1002, 969, 1051, 1052, 989, 1047, 961, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8284, 8067, 8197, 8333, 8189, 8234, 8130] avg=8192.0000 max=8333 max_rank=4 min=8067 min_rank=2 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32886 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[986, 989, 1041, 1011, 986, 1050, 1014, 1035, 6, 0, 0, 0, 0, 0, 0, 0, 986, 1021, 1088, 1006, 1026, 1036, 1004, 978, 19, 6, 0, 0, 0, 0, 0, 0, 985, 1073, 1112, 1001, 1013, 1022, 1009, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1016, 1009, 975, 1023, 1081, 1003, 1013, 7, 2, 0, 0, 0, 0, 0, 0, 984, 1047, 985, 1033, 1038, 1016, 999, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 1042, 1009, 1008, 1022, 1041, 1016, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1009, 1034, 1031, 1059, 1035, 986, 989, 26, 0, 0, 0, 0, 0, 0, 0, 1078, 1012, 1031, 1038, 1034, 1005, 1018, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8170, 8206, 8184, 8103, 8289, 8198, 8268] avg=8192.0000 max=8289 max_rank=5 min=8103 min_rank=4 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1053, 1002, 1063, 1024, 1024, 979, 1031, 1030, 6, 0, 0, 0, 0, 0, 0, 0, 958, 1044, 1031, 1009, 974, 989, 1008, 1007, 18, 6, 0, 0, 0, 0, 0, 0, 1040, 1088, 968, 1030, 1047, 1031, 1008, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1080, 1000, 1090, 999, 1013, 1001, 1043, 6, 2, 0, 0, 0, 0, 0, 0, 1028, 1020, 1009, 1056, 1003, 1012, 1003, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1070, 1023, 1059, 1027, 1007, 1009, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 984, 995, 1061, 1018, 1060, 1025, 1056, 27, 0, 0, 0, 0, 0, 0, 0, 984, 1011, 1070, 1003, 994, 1037, 1006, 1042, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8044, 8237, 8262, 8139, 8239, 8255, 8148] avg=8192.0000 max=8262 max_rank=3 min=8044 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[978, 1010, 1058, 1025, 1031, 1044, 1046, 1089, 6, 0, 0, 0, 0, 0, 0, 0, 1032, 1045, 1032, 1008, 1045, 1029, 1012, 1063, 18, 7, 0, 0, 0, 0, 0, 0, 1024, 1009, 1051, 1026, 1016, 1022, 1024, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1051, 1060, 1012, 994, 1018, 1019, 1022, 7, 2, 0, 0, 0, 0, 0, 0, 1027, 1042, 1017, 1074, 1050, 1020, 1011, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 976, 1027, 1033, 1017, 1039, 998, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 936, 1029, 1012, 998, 1022, 991, 925, 28, 0, 0, 0, 0, 0, 0, 0, 1017, 1060, 980, 1037, 1045, 1004, 1022, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8291, 8181, 8182, 8281, 8191, 7952, 8171] avg=8192.0000 max=8291 max_rank=1 min=7952 min_rank=6 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32595 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1000, 1001, 1024, 998, 1006, 1011, 1026, 1001, 8, 0, 0, 0, 0, 0, 0, 0, 973, 991, 1023, 1022, 967, 980, 1022, 1034, 21, 11, 0, 0, 0, 0, 0, 0, 1017, 983, 1035, 1075, 1033, 1021, 1054, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1052, 1032, 1015, 1056, 1013, 1046, 1029, 11, 6, 0, 0, 0, 0, 0, 0, 980, 1033, 1023, 1020, 1087, 1003, 1027, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1051, 1028, 1011, 1057, 1040, 1060, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1014, 1036, 1028, 1028, 1044, 1042, 1000, 30, 0, 0, 0, 0, 0, 0, 0, 987, 985, 1072, 1069, 1011, 991, 1005, 1062, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8075, 8044, 8209, 8313, 8159, 8279, 8270, 8187] avg=8192.0000 max=8313 max_rank=3 min=8044 min_rank=1 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1035, 1077, 1047, 1053, 1022, 1019, 995, 1015, 6, 0, 0, 0, 0, 0, 0, 0, 995, 995, 1044, 1070, 1013, 1019, 1088, 1018, 17, 6, 0, 0, 0, 0, 0, 0, 1086, 1008, 1024, 993, 1072, 1003, 1013, 957, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1019, 1000, 1012, 1050, 1014, 1041, 1032, 6, 2, 0, 0, 0, 0, 0, 0, 962, 1040, 1018, 1070, 1019, 1025, 1052, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 980, 999, 1055, 1011, 1009, 1015, 951, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 975, 995, 1061, 992, 1016, 1005, 1069, 27, 0, 0, 0, 0, 0, 0, 0, 992, 1030, 999, 999, 1033, 1061, 1050, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8265, 8156, 8201, 8208, 8030, 8206, 8201] avg=8192.0000 max=8269 max_rank=0 min=8030 min_rank=5 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8320, 8448, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8448, 8192, 8192, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8064, 8192, 8320, 8320, 8448, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1017, 995, 1078, 1016, 1046, 996, 1042, 1027, 1071, 985, 1019, 975, 982, 1000, 1029, 1003, 1040, 1045, 1063, 1029, 1030, 1018, 1033, 1064, 1039, 1017, 1051, 999, 1017, 1080, 1034, 1047, 1005, 1056, 1041, 1008, 1018, 975, 990, 1044, 1013, 1028, 1050, 1037, 1046, 1057, 994, 1025, 1021, 1029, 996, 997, 1042, 1058, 993, 997, 1032, 1031, 986, 1039, 1028, 1031, 994, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1001, 1044, 1073, 994, 1037, 956, 985, 1029, 978, 1045, 1036, 1062, 1100, 1004, 990, 1065, 1079, 1065, 959, 1026, 987, 1040, 977, 1029, 1047, 1041, 1004, 1051, 996, 1015, 997, 1023, 1054, 999, 1032, 1000, 1020, 1001, 1032, 1053, 1011, 1022, 999, 1024, 1000, 1038, 1024, 1003, 1015, 1025, 1074, 996, 969, 1082, 1032, 990, 1044, 1104, 985, 1029, 1069, 1002, 1030, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1002, 1043, 1030, 1029, 1070, 1044, 1048, 1057, 982, 1030, 1080, 996, 1032, 1008, 1010, 1049, 966, 1058, 1030, 1031, 1048, 994, 1001, 1045, 1038, 936, 1029, 1021, 1024, 1072, 994, 1030, 1031, 968, 1050, 1012, 1020, 1029, 1025, 1015, 1037, 1035, 1036, 1071, 1063, 981, 1051, 1060, 980, 1030, 972, 1024, 980, 985, 1011, 1013, 1041, 1062, 1024, 1019, 1017, 1026, 1040, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1004, 1029, 1035, 1009, 1031, 967, 1069, 1026, 1011, 999, 1019, 1010, 1050, 1010, 1045, 987, 1036, 1009, 1030, 1054, 993, 1011, 996, 998, 1057, 1003, 1020, 1013, 1052, 1035, 1012, 996, 1052, 1066, 1025, 981, 1064, 1014, 1033, 980, 1041, 1066, 1043, 1027, 1044, 1032, 1048, 1037, 1003, 1053, 1020, 992, 1034, 1085, 1005, 1013, 1039, 1036, 1005, 1022, 1002, 1007, 1030, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1018, 1028, 1056, 967, 996, 1007, 1028, 993, 1022, 1011, 1019, 1023, 1011, 995, 1055, 1034, 1073, 1032, 1041, 1039, 1031, 1013, 1023, 1011, 1033, 1037, 1035, 1045, 985, 1024, 1052, 1038, 986, 967, 995, 1053, 1009, 998, 1093, 987, 1061, 1104, 1042, 1027, 993, 1048, 1033, 1016, 977, 1008, 1004, 1053, 1056, 994, 1026, 1052, 1002, 1072, 1045, 1004, 1050, 1015, 998, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1058, 1022, 1014, 986, 1045, 1083, 1064, 1016, 1073, 1031, 1088, 988, 1023, 1060, 1019, 1040, 1016, 1010, 996, 1026, 1056, 1001, 1014, 1052, 1054, 1008, 1033, 977, 1003, 1012, 1033, 974, 1025, 1029, 1018, 996, 1058, 1046, 1027, 948, 1032, 1052, 1004, 977, 995, 988, 1044, 981, 1025, 1003, 1008, 1036, 1017, 1079, 997, 978, 1035, 980, 1051, 1041, 1057, 1010, 1030, 1094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1038, 997, 1039, 1058, 1002, 1077, 1021, 997, 1006, 986, 960, 1044, 998, 1015, 1016, 1000, 1041, 1030, 1006, 1030, 982, 997, 1021, 1006, 1042, 982, 1047, 998, 1122, 1015, 1078, 1026, 1023, 1000, 1037, 985, 1018, 1031, 949, 1021, 1025, 959, 1095, 1053, 1038, 1028, 1004, 1031, 999, 1073, 1053, 1064, 1018, 1041, 1013, 1068, 1047, 1015, 979, 1018, 1073, 1044, 1044, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1036, 1018, 1022, 1044, 1012, 1068, 1004, 1060, 1030, 1090, 1009, 1016, 1051, 992, 965, 984, 1055, 987, 1018, 1024, 1029, 1019, 996, 1062, 1005, 1033, 1008, 1066, 1048, 1077, 1034, 1017, 993, 993, 1013, 1056, 1080, 1078, 974, 1053, 997, 1026, 1056, 1090, 1001, 997, 1082, 1009, 974, 1011, 1047, 1008, 1017, 1011, 1015, 974, 1021, 1004, 1001, 1012, 1017, 999, 1031, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1017, 995, 1057, 1016, 1046, 996, 1042, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 985, 1019, 975, 982, 1000, 1029, 1003, 17, 10, 0, 0, 0, 0, 0, 0, 1040, 1045, 1063, 1029, 1030, 1018, 1033, 1064, 1, 0, 0, 0, 0, 0, 0, 0, 1039, 1017, 1051, 999, 1017, 1055, 1034, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1056, 1041, 1008, 1018, 975, 990, 1044, 25, 9, 4, 0, 0, 0, 0, 0, 1013, 1028, 1001, 1037, 1046, 1057, 994, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1029, 996, 997, 1042, 1058, 993, 997, 40, 0, 0, 0, 0, 0, 0, 0, 1032, 1031, 986, 1039, 1017, 1031, 994, 988, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8091, 8323, 8259, 8175, 8201, 8173, 8118] avg=8192.0000 max=8323 max_rank=2 min=8091 min_rank=1 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1001, 1044, 1053, 994, 1037, 956, 985, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1045, 1036, 1062, 1100, 1004, 990, 1065, 16, 11, 0, 0, 0, 0, 0, 0, 1079, 1065, 959, 1026, 987, 1040, 977, 1029, 1, 0, 0, 0, 0, 0, 0, 0, 1047, 1041, 1004, 1051, 996, 992, 997, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 999, 1032, 1000, 1020, 1001, 1032, 1053, 23, 8, 4, 0, 0, 0, 0, 0, 1011, 1022, 953, 1024, 1000, 1038, 1024, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1025, 1074, 996, 969, 1082, 1032, 990, 38, 0, 0, 0, 0, 0, 0, 0, 1044, 1104, 985, 1029, 1057, 1002, 1030, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8307, 8163, 8151, 8226, 8075, 8221, 8294] avg=8192.0000 max=8307 max_rank=1 min=8075 min_rank=5 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1038, 997, 1019, 1058, 1002, 1077, 1021, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 986, 960, 1044, 998, 1015, 1016, 1000, 16, 11, 0, 0, 0, 0, 0, 0, 1041, 1030, 1006, 1030, 982, 997, 1021, 1006, 1, 0, 0, 0, 0, 0, 0, 0, 1042, 982, 1047, 998, 1122, 992, 1078, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1000, 1037, 985, 1018, 1031, 949, 1021, 23, 9, 4, 0, 0, 0, 0, 0, 1025, 959, 1044, 1053, 1038, 1028, 1004, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1073, 1053, 1064, 1018, 1041, 1013, 1068, 42, 0, 0, 0, 0, 0, 0, 0, 1047, 1015, 979, 1018, 1061, 1044, 1044, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8052, 8114, 8287, 8100, 8182, 8371, 8221] avg=8192.0000 max=8371 max_rank=6 min=8052 min_rank=1 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1004, 1029, 1015, 1009, 1031, 967, 1069, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 999, 1019, 1010, 1050, 1010, 1045, 987, 16, 10, 0, 0, 0, 0, 0, 0, 1036, 1009, 1030, 1054, 993, 1011, 996, 998, 1, 0, 0, 0, 0, 0, 0, 0, 1057, 1003, 1020, 1013, 1052, 1011, 1012, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1066, 1025, 981, 1064, 1014, 1033, 980, 24, 9, 4, 0, 0, 0, 0, 0, 1041, 1066, 994, 1027, 1044, 1032, 1048, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1053, 1020, 992, 1034, 1085, 1005, 1013, 40, 0, 0, 0, 0, 0, 0, 0, 1039, 1036, 1005, 1022, 991, 1007, 1030, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8157, 8128, 8164, 8252, 8289, 8245, 8151] avg=8192.0000 max=8289 max_rank=5 min=8128 min_rank=2 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32937 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1018, 1028, 1036, 967, 996, 1007, 1028, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1011, 1019, 1023, 1011, 995, 1055, 1034, 16, 11, 0, 0, 0, 0, 0, 0, 1073, 1032, 1041, 1039, 1031, 1013, 1023, 1011, 1, 0, 0, 0, 0, 0, 0, 0, 1033, 1037, 1035, 1045, 985, 1001, 1052, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 986, 967, 995, 1053, 1009, 998, 1093, 987, 23, 9, 4, 0, 0, 0, 0, 0, 1061, 1104, 993, 1027, 993, 1048, 1033, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1008, 1004, 1053, 1056, 994, 1026, 1052, 40, 0, 0, 0, 0, 0, 0, 0, 1002, 1072, 1045, 1004, 1038, 1015, 998, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8073, 8197, 8264, 8226, 8124, 8275, 8210, 8167] avg=8192.0000 max=8275 max_rank=5 min=8073 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1002, 1043, 1010, 1029, 1070, 1044, 1048, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1030, 1080, 996, 1032, 1008, 1010, 1049, 16, 10, 0, 0, 0, 0, 0, 0, 966, 1058, 1030, 1031, 1048, 994, 1001, 1045, 1, 0, 0, 0, 0, 0, 0, 0, 1038, 936, 1029, 1021, 1024, 1047, 994, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 968, 1050, 1012, 1020, 1029, 1025, 1015, 25, 9, 4, 0, 0, 0, 0, 0, 1037, 1035, 988, 1071, 1063, 981, 1051, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1030, 972, 1024, 980, 985, 1011, 1013, 39, 0, 0, 0, 0, 0, 0, 0, 1041, 1062, 1024, 1019, 1006, 1026, 1040, 1001, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8213, 8174, 8119, 8188, 8286, 8034, 8219] avg=8192.0000 max=8303 max_rank=0 min=8034 min_rank=6 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1058, 1022, 994, 986, 1045, 1083, 1064, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1031, 1088, 988, 1023, 1060, 1019, 1040, 16, 11, 0, 0, 0, 0, 0, 0, 1016, 1010, 996, 1026, 1056, 1001, 1014, 1052, 1, 0, 0, 0, 0, 0, 0, 0, 1054, 1008, 1033, 977, 1003, 989, 1033, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1029, 1018, 996, 1058, 1046, 1027, 948, 23, 8, 4, 0, 0, 0, 0, 0, 1032, 1052, 958, 977, 995, 988, 1044, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1003, 1008, 1036, 1017, 1079, 997, 978, 38, 0, 0, 0, 0, 0, 0, 0, 1035, 980, 1051, 1041, 1045, 1010, 1030, 1094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8349, 8172, 8071, 8182, 8027, 8181, 8286] avg=8192.0000 max=8349 max_rank=1 min=8027 min_rank=5 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1036, 1018, 996, 1044, 1012, 1068, 1004, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1090, 1009, 1016, 1051, 992, 965, 984, 19, 14, 0, 0, 0, 0, 0, 0, 1055, 987, 1018, 1024, 1029, 1019, 996, 1062, 8, 0, 0, 0, 0, 0, 0, 0, 1005, 1033, 1008, 1066, 1048, 1048, 1034, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 993, 993, 1013, 1056, 1080, 1078, 974, 1053, 29, 13, 7, 0, 0, 0, 0, 0, 997, 1026, 999, 1090, 1001, 997, 1082, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1011, 1047, 1008, 1017, 1011, 1015, 974, 44, 0, 0, 0, 0, 0, 0, 0, 1021, 1004, 1001, 1012, 995, 999, 1031, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8170, 8198, 8259, 8289, 8201, 8101, 8080] avg=8192.0000 max=8289 max_rank=4 min=8080 min_rank=7 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8064, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8320, 7936, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8192, 8192, 8448, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1041, 987, 1023, 1034, 1044, 976, 1033, 1063, 1011, 978, 1003, 1019, 1045, 1017, 998, 1019, 1032, 1030, 1034, 995, 1024, 1011, 1038, 1018, 1031, 1035, 1007, 1020, 999, 1085, 1002, 1043, 1025, 1011, 1041, 1020, 1015, 1020, 1017, 1003, 1044, 1048, 1004, 1021, 1032, 1045, 1002, 1085, 1003, 1032, 1013, 1009, 1023, 1052, 1043, 1060, 976, 1032, 977, 1061, 1029, 1031, 1033, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1039, 1001, 1106, 1059, 984, 1023, 1064, 1025, 1051, 1052, 1059, 1040, 985, 1001, 986, 1049, 1005, 1002, 1013, 1033, 1012, 1074, 1057, 993, 1019, 1004, 1012, 1008, 1045, 1022, 985, 1033, 1090, 1065, 1033, 988, 1006, 1004, 1003, 1056, 1017, 989, 965, 1058, 1000, 1046, 1022, 971, 998, 1041, 1097, 975, 1010, 968, 1020, 1071, 1029, 1049, 1023, 1047, 1022, 1050, 989, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1040, 1034, 1068, 1027, 1023, 1036, 1004, 995, 1004, 1060, 1061, 1035, 1017, 1014, 1035, 1038, 976, 1060, 983, 1095, 965, 1055, 1011, 1034, 1033, 1004, 1016, 1009, 1054, 1052, 1022, 1020, 1042, 957, 978, 981, 1016, 1032, 1034, 995, 1013, 1021, 1050, 1073, 1037, 1011, 1056, 1004, 1015, 1051, 1071, 1013, 1032, 1021, 1022, 1015, 1025, 1042, 1004, 1027, 1030, 1008, 989, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1095, 1049, 1000, 1028, 1014, 1003, 1024, 970, 1017, 1016, 984, 1074, 1030, 1041, 1061, 1010, 1042, 1024, 1008, 1077, 1027, 988, 989, 1022, 1021, 998, 985, 1039, 1021, 985, 1006, 1053, 1013, 1066, 1045, 1023, 1036, 1094, 1037, 982, 1005, 1000, 1022, 1051, 1029, 1002, 1066, 1036, 1001, 1028, 1019, 1060, 969, 1044, 1020, 989, 1025, 1005, 998, 1053, 1058, 959, 1045, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1028, 1002, 1034, 1017, 1053, 1002, 1066, 1006, 1065, 1021, 981, 1044, 1068, 1001, 1076, 1021, 1013, 1001, 982, 1036, 1047, 1003, 1026, 1009, 974, 1007, 1030, 1026, 1026, 1057, 1009, 1020, 1037, 1020, 1003, 1008, 1037, 1066, 1048, 1039, 1046, 1044, 1005, 979, 1031, 1065, 998, 1039, 973, 1026, 1011, 1066, 971, 1022, 954, 1035, 1043, 1004, 1052, 1014, 1018, 1043, 1056, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[996, 1018, 1036, 1022, 1071, 1030, 1005, 996, 1019, 1044, 1012, 1014, 1005, 1017, 1047, 1065, 1046, 1042, 1004, 1032, 1041, 990, 1055, 1029, 1053, 1071, 1020, 987, 988, 1010, 1025, 981, 1094, 985, 1012, 1043, 1014, 1057, 1026, 1056, 1057, 1042, 1077, 1069, 972, 996, 1012, 1041, 997, 1042, 1023, 1025, 1007, 1040, 1019, 1007, 982, 1022, 1009, 1053, 1031, 989, 972, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[995, 1027, 1010, 1011, 1021, 1028, 999, 1057, 1049, 989, 1036, 1057, 1027, 1016, 1026, 977, 1045, 979, 1070, 1017, 1050, 990, 1028, 1018, 1054, 1066, 989, 1032, 1000, 973, 1032, 989, 1065, 1044, 1018, 1020, 1013, 979, 994, 1018, 983, 977, 1008, 1020, 1063, 1093, 1030, 1024, 1072, 1078, 946, 1066, 996, 1033, 1082, 1022, 1061, 1000, 1021, 1031, 1026, 1062, 1001, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1036, 1040, 984, 1024, 1028, 1068, 1002, 1032, 1026, 990, 1006, 961, 1012, 1039, 1008, 1024, 1044, 995, 1047, 1057, 992, 1051, 993, 1071, 1006, 988, 1042, 1069, 1033, 1040, 1013, 1044, 1037, 985, 1077, 1041, 1024, 1021, 1026, 1007, 1066, 1058, 1007, 997, 1028, 1018, 1003, 1008, 1032, 1019, 1047, 1057, 996, 972, 1050, 1030, 1041, 1012, 1056, 985, 987, 1028, 1044, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1027, 987, 1023, 1034, 1044, 976, 1033, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 978, 1003, 1018, 1045, 1017, 998, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1030, 1034, 995, 1024, 1011, 1038, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1035, 1007, 1020, 999, 1085, 1002, 1043, 18, 8, 0, 0, 0, 0, 0, 0, 1013, 1011, 1041, 1020, 1015, 1020, 1017, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1048, 1004, 1021, 1032, 1027, 1002, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1032, 1013, 1009, 1023, 1052, 1043, 1060, 2, 1, 0, 0, 0, 0, 0, 0, 976, 1032, 977, 1061, 1029, 1031, 1033, 1034, 10, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8089, 8182, 8248, 8140, 8263, 8238, 8189] avg=8192.0000 max=8263 max_rank=5 min=8089 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1026, 1034, 1068, 1027, 1023, 1036, 1004, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1060, 1061, 1034, 1017, 1014, 1035, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1060, 983, 1095, 965, 1055, 1011, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1004, 1016, 1009, 1054, 1052, 1022, 1020, 17, 8, 0, 0, 0, 0, 0, 0, 1030, 957, 978, 981, 1016, 1032, 1034, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1021, 1050, 1073, 1037, 994, 1056, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1051, 1071, 1013, 1032, 1021, 1022, 1015, 2, 1, 0, 0, 0, 0, 0, 0, 1025, 1042, 1004, 1027, 1030, 1008, 989, 991, 10, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8263, 8179, 8235, 8023, 8248, 8243, 8132] avg=8192.0000 max=8263 max_rank=1 min=8023 min_rank=4 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1025, 1001, 1106, 1059, 984, 1023, 1064, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1052, 1059, 1039, 985, 1001, 986, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1002, 1013, 1033, 1012, 1074, 1057, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1004, 1012, 1008, 1045, 1022, 985, 1033, 18, 8, 0, 0, 0, 0, 0, 0, 1077, 1065, 1033, 988, 1006, 1004, 1003, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 989, 965, 1058, 1000, 1028, 1022, 971, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1041, 1097, 975, 1010, 968, 1020, 1071, 3, 1, 0, 0, 0, 0, 0, 0, 1029, 1049, 1023, 1047, 1022, 1050, 989, 993, 10, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8222, 8189, 8154, 8232, 8050, 8184, 8218] avg=8192.0000 max=8287 max_rank=0 min=8050 min_rank=5 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[982, 1018, 1036, 1022, 1071, 1030, 1005, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1044, 1012, 1013, 1005, 1017, 1047, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1042, 1004, 1032, 1041, 990, 1055, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1071, 1020, 987, 988, 1010, 1025, 981, 17, 8, 0, 0, 0, 0, 0, 0, 1081, 985, 1012, 1043, 1014, 1057, 1026, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1042, 1077, 1069, 972, 979, 1012, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1042, 1023, 1025, 1007, 1040, 1019, 1007, 3, 1, 0, 0, 0, 0, 0, 0, 982, 1022, 1009, 1053, 1031, 989, 972, 994, 10, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8222, 8239, 8160, 8274, 8249, 8164, 8068] avg=8192.0000 max=8274 max_rank=4 min=8068 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1014, 1002, 1034, 1017, 1053, 1002, 1066, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1021, 981, 1043, 1068, 1001, 1076, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1001, 982, 1036, 1047, 1003, 1026, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1007, 1030, 1026, 1026, 1057, 1009, 1020, 18, 8, 0, 0, 0, 0, 0, 0, 1025, 1020, 1003, 1008, 1037, 1066, 1048, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1044, 1005, 979, 1031, 1047, 998, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1026, 1011, 1066, 971, 1022, 954, 1035, 2, 1, 0, 0, 0, 0, 0, 0, 1043, 1004, 1052, 1014, 1018, 1043, 1056, 1032, 10, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8276, 8117, 8175, 8246, 8189, 8061, 8278] avg=8192.0000 max=8278 max_rank=7 min=8061 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1080, 1049, 1000, 1028, 1014, 1003, 1024, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1016, 984, 1073, 1030, 1041, 1061, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1024, 1008, 1077, 1027, 988, 989, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 998, 985, 1039, 1021, 985, 1006, 1053, 17, 9, 0, 0, 0, 0, 0, 0, 1002, 1066, 1045, 1023, 1036, 1094, 1037, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1000, 1022, 1051, 1029, 985, 1066, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1028, 1019, 1060, 969, 1044, 1020, 989, 2, 1, 0, 0, 0, 0, 0, 0, 1025, 1005, 998, 1053, 1058, 959, 1045, 1055, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8232, 8177, 8134, 8285, 8194, 8133, 8213] avg=8192.0000 max=8285 max_rank=4 min=8133 min_rank=6 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[981, 1027, 1010, 1011, 1021, 1028, 999, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 989, 1036, 1056, 1027, 1016, 1026, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 979, 1070, 1017, 1050, 990, 1028, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1066, 989, 1032, 1000, 973, 1032, 989, 19, 8, 0, 0, 0, 0, 0, 0, 1052, 1044, 1018, 1020, 1013, 979, 994, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 983, 977, 1008, 1020, 1063, 1074, 1030, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1078, 946, 1066, 996, 1033, 1082, 1022, 3, 1, 0, 0, 0, 0, 0, 0, 1061, 1000, 1021, 1031, 1026, 1062, 1001, 1033, 10, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8176, 8197, 8162, 8138, 8179, 8299, 8251] avg=8192.0000 max=8299 max_rank=6 min=8134 min_rank=0 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1015, 1040, 984, 1024, 1028, 1068, 1002, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 990, 1006, 951, 1012, 1039, 1008, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 995, 1047, 1057, 992, 1051, 993, 1071, 6, 0, 0, 0, 0, 0, 0, 0, 1006, 988, 1042, 1069, 1033, 1040, 1013, 1044, 21, 12, 0, 0, 0, 0, 0, 0, 1017, 985, 1077, 1041, 1024, 1021, 1026, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1058, 1007, 997, 1028, 997, 1003, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1019, 1047, 1057, 996, 972, 1050, 1030, 7, 4, 0, 0, 0, 0, 0, 0, 1041, 1012, 1056, 985, 987, 1028, 1044, 1012, 13, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8056, 8256, 8268, 8198, 8164, 8214, 8187] avg=8192.0000 max=8268 max_rank=3 min=8056 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8064, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8448, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1043, 995, 1031, 985, 981, 1077, 1025, 1011, 1031, 1052, 1002, 1019, 1009, 995, 1021, 1011, 1019, 1081, 1033, 990, 1032, 1008, 1043, 1007, 1022, 1010, 1002, 998, 1021, 1019, 1042, 1062, 1022, 1048, 1020, 999, 1078, 999, 1023, 1055, 1027, 1014, 1015, 997, 1041, 1021, 1014, 973, 977, 1066, 1018, 1017, 1049, 1022, 1047, 1045, 1021, 1027, 1063, 1051, 1004, 1040, 987, 1079] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1014, 1060, 1022, 1042, 1016, 1033, 1007, 1031, 1068, 1056, 1022, 999, 1052, 1025, 991, 1014, 1042, 998, 1027, 1026, 994, 965, 1013, 1038, 1032, 1028, 1012, 1033, 1030, 1011, 1039, 1059, 1039, 1061, 1047, 1029, 993, 1029, 1038, 993, 1052, 1068, 984, 998, 972, 1040, 1064, 1012, 1024, 1057, 1026, 1000, 1010, 988, 1050, 990, 1011, 968, 1005, 1040, 1030, 1050, 1019, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1009, 1027, 1051, 1015, 1023, 1010, 963, 1014, 1057, 1026, 1074, 1039, 1000, 1055, 1032, 969, 1006, 1024, 1033, 1000, 1051, 1036, 1042, 1035, 1047, 1015, 1098, 1062, 1045, 1034, 993, 1075, 973, 1049, 1017, 1004, 977, 1023, 1005, 1075, 1029, 1046, 1020, 1029, 1027, 1018, 997, 1006, 939, 1078, 1029, 1081, 996, 1039, 998, 1016, 1035, 959, 1056, 982, 1044, 985, 1008, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1054, 1093, 980, 1011, 1002, 1034, 1031, 1062, 1030, 1090, 993, 1010, 1040, 1041, 1045, 1020, 985, 1011, 1000, 1005, 1018, 1008, 1017, 1057, 1082, 1014, 1035, 1009, 1055, 1020, 1036, 1047, 1018, 1034, 1034, 1061, 972, 1075, 1026, 1052, 1005, 1061, 989, 1012, 995, 996, 1009, 1018, 1014, 1024, 963, 1030, 1025, 1051, 1017, 991, 1013, 998, 1003, 1015, 1005, 1004, 1068, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1006, 1039, 1036, 1035, 1000, 1068, 1028, 998, 1031, 1018, 1020, 1052, 1054, 1060, 1041, 985, 1021, 1013, 1011, 975, 975, 1001, 1040, 1040, 1027, 981, 1007, 1056, 1026, 1003, 1065, 1042, 1006, 1042, 1029, 1032, 998, 1025, 1035, 1029, 1062, 1002, 1033, 1037, 1032, 1038, 1009, 1016, 1035, 1007, 1065, 976, 1042, 1002, 1046, 987, 1040, 1007, 1049, 1006, 1041, 1032, 1010, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1028, 1079, 1011, 1101, 1053, 1051, 985, 1027, 1004, 1007, 999, 977, 1047, 987, 1006, 1010, 1066, 995, 1003, 987, 1018, 1040, 1021, 1004, 1034, 1050, 1004, 985, 1070, 1037, 1081, 1016, 972, 1003, 1001, 1048, 1025, 979, 1031, 995, 1002, 1060, 1075, 1020, 1056, 1001, 1016, 1004, 1010, 1004, 975, 982, 1017, 1051, 1055, 1019, 1021, 1067, 998, 1018, 1085, 1042, 1041, 1080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[969, 1042, 1007, 1029, 1053, 1012, 1044, 991, 1065, 1056, 992, 1017, 996, 1058, 978, 1004, 1057, 1016, 1079, 1023, 1052, 964, 993, 1007, 1011, 1007, 1051, 1007, 1031, 1029, 1066, 1030, 1035, 985, 1065, 1005, 1016, 1005, 990, 1049, 1038, 1006, 1026, 1004, 1051, 1071, 1027, 1058, 1050, 996, 1039, 981, 1036, 1025, 1068, 1045, 980, 975, 987, 1025, 1056, 1028, 1010, 1068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1007, 1021, 1017, 1020, 1028, 1041, 1024, 1045, 1082, 1046, 1013, 1038, 964, 1049, 1036, 1028, 1056, 994, 987, 1046, 1018, 988, 1016, 1024, 1002, 1040, 1008, 1031, 998, 984, 933, 1041, 1042, 1041, 1017, 1066, 1053, 1048, 1020, 1019, 1003, 1010, 1039, 1048, 1035, 1056, 994, 1034, 1098, 1021, 1005, 988, 989, 1004, 976, 1016, 1004, 1073, 1054, 1035, 1043, 1018, 1028, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1043, 983, 1031, 985, 981, 1077, 1025, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1052, 1002, 1019, 1009, 995, 1021, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1081, 1033, 990, 1032, 1008, 1043, 1007, 38, 6, 0, 0, 0, 0, 0, 0, 1022, 1010, 1002, 998, 1021, 1019, 1042, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1048, 1020, 999, 1078, 999, 1023, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1014, 1015, 997, 1041, 1021, 1014, 973, 3, 0, 0, 0, 0, 0, 0, 0, 977, 1066, 1018, 1017, 1049, 1022, 1047, 1045, 14, 8, 6, 1, 0, 0, 0, 0, 1021, 1027, 1063, 1051, 1004, 1040, 987, 1065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8132, 8257, 8138, 8240, 8105, 8270, 8258] avg=8192.0000 max=8270 max_rank=6 min=8105 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1014, 1047, 1022, 1042, 1016, 1033, 1007, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1056, 1022, 999, 1052, 1025, 991, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 998, 1027, 1026, 994, 965, 1013, 1038, 38, 7, 0, 0, 0, 0, 0, 0, 1032, 1028, 1012, 1033, 1030, 1011, 1039, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1061, 1047, 1029, 993, 1029, 1038, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1068, 984, 998, 972, 1040, 1064, 1012, 2, 0, 0, 0, 0, 0, 0, 0, 1024, 1057, 1026, 1000, 1010, 988, 1050, 990, 13, 9, 6, 1, 0, 0, 0, 0, 1011, 968, 1005, 1040, 1030, 1050, 1019, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8218, 8148, 8206, 8226, 8192, 8174, 8160] avg=8192.0000 max=8226 max_rank=4 min=8148 min_rank=2 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1006, 1026, 1036, 1035, 1000, 1068, 1028, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1018, 1020, 1052, 1054, 1060, 1041, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1013, 1011, 975, 975, 1001, 1040, 1040, 37, 7, 0, 0, 0, 0, 0, 0, 1027, 981, 1007, 1056, 1026, 1003, 1065, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1042, 1029, 1032, 998, 1025, 1035, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1002, 1033, 1037, 1032, 1038, 1009, 1016, 2, 0, 0, 0, 0, 0, 0, 0, 1035, 1007, 1065, 976, 1042, 1002, 1046, 987, 13, 8, 6, 1, 0, 0, 0, 0, 1040, 1007, 1049, 1006, 1041, 1032, 1010, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8253, 8120, 8170, 8193, 8231, 8188, 8184] avg=8192.0000 max=8253 max_rank=1 min=8120 min_rank=2 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1009, 1014, 1051, 1015, 1023, 1010, 963, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1026, 1074, 1039, 1000, 1055, 1032, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1024, 1033, 1000, 1051, 1036, 1042, 1035, 39, 7, 0, 0, 0, 0, 0, 0, 1047, 1015, 1098, 1062, 1045, 1034, 993, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1049, 1017, 1004, 977, 1023, 1005, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1046, 1020, 1029, 1027, 1018, 997, 1006, 3, 0, 0, 0, 0, 0, 0, 0, 939, 1078, 1029, 1081, 996, 1039, 998, 1016, 13, 9, 6, 1, 0, 0, 0, 0, 1035, 959, 1056, 982, 1044, 985, 1008, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8243, 8273, 8330, 8119, 8175, 8205, 8092] avg=8192.0000 max=8330 max_rank=3 min=8092 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1054, 1080, 980, 1011, 1002, 1034, 1031, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1090, 993, 1010, 1040, 1041, 1045, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1011, 1000, 1005, 1018, 1008, 1017, 1057, 38, 7, 0, 0, 0, 0, 0, 0, 1082, 1014, 1035, 1009, 1055, 1020, 1036, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1034, 1034, 1061, 972, 1075, 1026, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1061, 989, 1012, 995, 996, 1009, 1018, 3, 0, 0, 0, 0, 0, 0, 0, 1014, 1024, 963, 1030, 1025, 1051, 1017, 991, 13, 8, 6, 1, 0, 0, 0, 0, 1013, 998, 1003, 1015, 1005, 1004, 1068, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8261, 8146, 8260, 8268, 8088, 8143, 8116] avg=8192.0000 max=8268 max_rank=4 min=8088 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1028, 1066, 1011, 1101, 1053, 1051, 985, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1007, 999, 977, 1047, 987, 1006, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 995, 1003, 987, 1018, 1040, 1021, 1004, 36, 7, 0, 0, 0, 0, 0, 0, 1034, 1050, 1004, 985, 1070, 1037, 1081, 980, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1003, 1001, 1048, 1025, 979, 1031, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1060, 1075, 1020, 1056, 1001, 1016, 1004, 2, 0, 0, 0, 0, 0, 0, 0, 1010, 1004, 975, 982, 1017, 1051, 1055, 1019, 14, 8, 6, 1, 0, 0, 0, 0, 1021, 1067, 998, 1018, 1085, 1042, 1041, 1066, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8322, 8029, 8177, 8241, 8051, 8236, 8142, 8338] avg=8192.0000 max=8338 max_rank=7 min=8029 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[969, 1029, 1007, 1029, 1053, 1012, 1044, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1056, 992, 1017, 996, 1058, 978, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1016, 1079, 1023, 1052, 964, 993, 1007, 37, 7, 0, 0, 0, 0, 0, 0, 1011, 1007, 1051, 1007, 1031, 1029, 1066, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 985, 1065, 1005, 1016, 1005, 990, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1006, 1026, 1004, 1051, 1071, 1027, 1058, 3, 0, 0, 0, 0, 0, 0, 0, 1050, 996, 1039, 981, 1036, 1025, 1068, 1045, 14, 9, 6, 1, 0, 0, 0, 0, 980, 975, 987, 1025, 1056, 1028, 1010, 1054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8157, 8235, 8195, 8146, 8284, 8270, 8115] avg=8192.0000 max=8284 max_rank=5 min=8115 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1007, 1000, 1017, 1020, 1028, 1041, 1024, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1046, 1013, 1038, 964, 1049, 1036, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 994, 987, 1046, 1018, 988, 1016, 1024, 41, 10, 0, 0, 0, 0, 0, 0, 1002, 1040, 1008, 1031, 998, 984, 933, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1041, 1017, 1066, 1053, 1048, 1020, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1010, 1039, 1048, 1035, 1056, 994, 1034, 6, 0, 0, 0, 0, 0, 0, 0, 1098, 1021, 1005, 988, 989, 1004, 976, 1016, 16, 13, 11, 7, 0, 0, 0, 0, 1004, 1073, 1054, 1035, 1043, 1018, 1028, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8243, 8180, 7996, 8293, 8225, 8144, 8273] avg=8192.0000 max=8293 max_rank=4 min=7996 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32935 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8064, 8192, 8192, 8320, 8192, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8448, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8448, 8192, 8192, 8192, 8320, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8064, 8192, 8064, 8192, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1007, 1033, 1051, 1044, 1005, 1031, 1055, 986, 1020, 1006, 1007, 1019, 1032, 1039, 1045, 1035, 1027, 1035, 988, 1026, 966, 984, 1036, 1004, 1003, 1002, 1045, 1032, 1011, 1042, 1045, 1037, 988, 997, 1003, 1043, 1060, 1023, 1016, 1031, 1020, 1028, 1029, 1031, 1030, 1031, 1036, 1037, 1030, 1023, 1005, 1028, 1071, 1008, 1049, 1024, 1030, 1032, 1052, 1057, 1036, 1040, 940, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1053, 999, 1034, 1014, 1037, 1030, 1043, 1070, 1086, 997, 1057, 1004, 1005, 1036, 1056, 1037, 1061, 1048, 1022, 993, 976, 978, 1059, 1035, 1011, 994, 1039, 1008, 1012, 997, 1007, 1050, 987, 1045, 1013, 1025, 1005, 1046, 986, 1017, 984, 1038, 1006, 1036, 976, 1063, 1059, 1032, 1028, 1012, 1025, 1015, 1067, 1034, 1062, 978, 1039, 969, 1052, 1033, 978, 1021, 1026, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1013, 1046, 1036, 1004, 1043, 1022, 1011, 1056, 974, 1043, 1016, 1024, 1033, 1029, 1002, 1056, 1025, 967, 1050, 975, 975, 1042, 1020, 1053, 1070, 1011, 1034, 1044, 1046, 1046, 1042, 992, 1023, 1025, 998, 1002, 1035, 1023, 1064, 1023, 1037, 1069, 1098, 988, 1061, 1026, 1049, 992, 992, 1059, 1022, 960, 965, 1036, 1036, 994, 1021, 983, 968, 1077, 1010, 1033, 1032, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1040, 1050, 995, 1065, 1013, 1000, 1008, 1042, 1065, 1003, 1009, 1000, 1059, 967, 1022, 990, 1044, 962, 1024, 949, 1008, 1056, 1016, 1052, 1037, 1008, 1071, 1028, 1051, 971, 1044, 989, 1026, 1001, 1042, 1027, 1055, 1009, 1027, 973, 1000, 1024, 1071, 1077, 990, 1009, 1055, 1013, 1097, 1014, 1038, 1034, 1005, 1043, 1042, 1033, 1020, 1074, 1005, 1008, 1016, 991, 1061, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1063, 1007, 985, 1010, 974, 1004, 1018, 1013, 1025, 1073, 1050, 1000, 1057, 1003, 1004, 1025, 1026, 970, 1025, 1005, 1069, 1050, 994, 1027, 1029, 1009, 1085, 1034, 1025, 1030, 1058, 1051, 995, 1027, 1021, 1000, 1036, 1067, 1033, 1070, 1036, 1030, 1057, 960, 1017, 1022, 1043, 1009, 1018, 985, 1028, 1001, 1046, 1037, 981, 994, 1030, 1062, 1047, 1009, 1027, 1028, 1060, 962] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1030, 1038, 1042, 991, 1030, 1011, 1070, 987, 1058, 1032, 997, 1034, 1003, 978, 1029, 1009, 1070, 988, 1017, 1033, 1030, 1047, 1011, 1004, 1046, 1001, 1003, 982, 1039, 1042, 1006, 1012, 1038, 1023, 1042, 1066, 1075, 1038, 1033, 995, 1014, 1033, 1031, 1051, 1021, 1013, 1035, 1015, 1014, 1045, 1040, 1048, 978, 1010, 1022, 1035, 983, 976, 1031, 1064, 1017, 1067, 1001, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1050, 988, 1015, 1029, 1037, 1043, 959, 981, 1036, 1022, 1017, 1023, 1027, 1012, 1033, 1040, 1030, 1008, 1044, 975, 1025, 1012, 1060, 1012, 1038, 1040, 1076, 1022, 997, 1027, 1011, 1033, 948, 994, 1016, 1051, 995, 1060, 1051, 1050, 1100, 1027, 1042, 1002, 1064, 985, 1006, 1017, 1028, 1040, 1019, 1045, 996, 1076, 1036, 1025, 1053, 979, 1030, 1044, 998, 1000, 1037, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1007, 1020, 1057, 972, 1056, 1024, 1013, 1049, 1009, 1055, 993, 1017, 981, 1018, 1039, 1021, 1003, 995, 1027, 1017, 1004, 1004, 1020, 1015, 968, 1065, 1012, 1063, 1027, 1047, 1002, 984, 1061, 1036, 1030, 977, 1007, 1003, 1049, 1034, 1014, 1042, 1037, 1001, 1010, 1036, 1063, 1036, 1015, 1047, 1061, 1010, 1039, 1078, 1058, 1041, 1001, 1024, 1017, 1021, 997, 1050, 1033, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1053, 999, 1034, 1014, 1037, 1030, 1043, 1070, 2, 0, 0, 0, 0, 0, 0, 0, 1086, 997, 1057, 1004, 1005, 1036, 1056, 1037, 5, 0, 0, 0, 0, 0, 0, 0, 1061, 1048, 1022, 993, 976, 978, 1059, 1035, 39, 16, 0, 0, 0, 0, 0, 0, 1011, 994, 1022, 1008, 1012, 997, 1007, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1045, 1013, 1025, 1005, 1044, 986, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1038, 967, 1036, 976, 1063, 1059, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1012, 1025, 1015, 1067, 1012, 1062, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 969, 1052, 1033, 978, 1021, 1026, 1031, 12, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8283, 8227, 8101, 8122, 8155, 8199, 8167] avg=8192.0000 max=8283 max_rank=1 min=8101 min_rank=3 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1007, 1033, 1051, 1044, 1005, 1031, 1055, 986, 2, 0, 0, 0, 0, 0, 0, 0, 1020, 1006, 1007, 1019, 1032, 1039, 1045, 1035, 5, 0, 0, 0, 0, 0, 0, 0, 1027, 1035, 988, 1026, 966, 984, 1036, 1004, 40, 16, 0, 0, 0, 0, 0, 0, 1003, 1002, 1028, 1032, 1011, 1042, 1045, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 988, 997, 1003, 1043, 1060, 1021, 1016, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1028, 989, 1031, 1030, 1031, 1036, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1023, 1005, 1028, 1071, 986, 1049, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1032, 1052, 1057, 1036, 1040, 940, 1010, 12, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8208, 8122, 8200, 8159, 8202, 8216, 8215] avg=8192.0000 max=8216 max_rank=6 min=8122 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=57320 max_load_reduction_pct=0.874634 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1013, 1046, 1036, 1004, 1043, 1022, 1011, 1056, 2, 0, 0, 0, 0, 0, 0, 0, 974, 1043, 1016, 1024, 1033, 1029, 1002, 1056, 5, 0, 0, 0, 0, 0, 0, 0, 1025, 967, 1050, 975, 975, 1042, 1020, 1053, 43, 16, 0, 0, 0, 0, 0, 0, 1070, 1011, 1017, 1044, 1046, 1046, 1042, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1025, 998, 1002, 1035, 1021, 1064, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1069, 1055, 988, 1061, 1026, 1049, 992, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1059, 1022, 960, 965, 1014, 1036, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 983, 968, 1077, 1010, 1033, 1032, 1035, 12, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8182, 8166, 8268, 8191, 8277, 8042, 8177] avg=8192.0000 max=8277 max_rank=5 min=8042 min_rank=6 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1040, 1050, 995, 1065, 1013, 1000, 1008, 1042, 2, 0, 0, 0, 0, 0, 0, 0, 1065, 1003, 1009, 1000, 1059, 967, 1022, 990, 5, 0, 0, 0, 0, 0, 0, 0, 1044, 962, 1024, 949, 1008, 1056, 1016, 1052, 41, 16, 0, 0, 0, 0, 0, 0, 1037, 1008, 1054, 1028, 1051, 971, 1044, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1001, 1042, 1027, 1055, 1007, 1027, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1024, 1030, 1077, 990, 1009, 1055, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 1014, 1038, 1034, 1005, 1021, 1042, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1074, 1005, 1008, 1016, 991, 1061, 1018, 12, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8120, 8168, 8182, 8158, 8198, 8284, 8211] avg=8192.0000 max=8284 max_rank=6 min=8120 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1063, 1007, 985, 1010, 974, 1004, 1018, 1013, 2, 0, 0, 0, 0, 0, 0, 0, 1025, 1073, 1050, 1000, 1057, 1003, 1004, 1025, 5, 0, 0, 0, 0, 0, 0, 0, 1026, 970, 1025, 1005, 1069, 1050, 994, 1027, 41, 16, 0, 0, 0, 0, 0, 0, 1029, 1009, 1068, 1034, 1025, 1030, 1058, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1027, 1021, 1000, 1036, 1065, 1033, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1030, 1016, 960, 1017, 1022, 1043, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 985, 1028, 1001, 1046, 1015, 981, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1062, 1047, 1009, 1027, 1028, 1060, 962, 12, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8076, 8242, 8223, 8304, 8247, 8133, 8068, 8243] avg=8192.0000 max=8304 max_rank=3 min=8068 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1030, 1038, 1042, 991, 1030, 1011, 1070, 987, 2, 0, 0, 0, 0, 0, 0, 0, 1058, 1032, 997, 1034, 1003, 978, 1029, 1009, 5, 0, 0, 0, 0, 0, 0, 0, 1070, 988, 1017, 1033, 1030, 1047, 1011, 1004, 40, 16, 0, 0, 0, 0, 0, 0, 1046, 1001, 987, 982, 1039, 1042, 1006, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1023, 1042, 1066, 1075, 1036, 1033, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1033, 991, 1051, 1021, 1013, 1035, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1045, 1040, 1048, 978, 988, 1022, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 983, 976, 1031, 1064, 1017, 1067, 1001, 1012, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8145, 8256, 8115, 8308, 8173, 8170, 8168] avg=8192.0000 max=8308 max_rank=4 min=8115 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1050, 988, 1015, 1029, 1037, 1043, 959, 981, 2, 0, 0, 0, 0, 0, 0, 0, 1036, 1022, 1017, 1023, 1027, 1012, 1033, 1040, 5, 0, 0, 0, 0, 0, 0, 0, 1030, 1008, 1044, 975, 1025, 1012, 1060, 1012, 40, 17, 0, 0, 0, 0, 0, 0, 1038, 1040, 1059, 1022, 997, 1027, 1011, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 948, 994, 1016, 1051, 995, 1058, 1051, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 1027, 1002, 1002, 1064, 985, 1006, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1040, 1019, 1045, 996, 1053, 1036, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 979, 1030, 1044, 998, 1000, 1037, 1000, 12, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8215, 8223, 8227, 8163, 8203, 8242, 8159] avg=8192.0000 max=8242 max_rank=6 min=8104 min_rank=0 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1007, 1020, 1057, 972, 1056, 1024, 1013, 1049, 9, 4, 0, 0, 0, 0, 0, 0, 1009, 1055, 993, 1017, 981, 1018, 1039, 1021, 8, 0, 0, 0, 0, 0, 0, 0, 1003, 995, 1027, 1017, 1004, 1004, 1020, 1015, 44, 22, 0, 0, 0, 0, 0, 0, 968, 1065, 983, 1063, 1027, 1047, 1002, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1036, 1030, 977, 1007, 994, 1049, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1042, 993, 1001, 1010, 1036, 1063, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1047, 1061, 1010, 1039, 1044, 1058, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1024, 1017, 1021, 997, 1050, 1033, 1024, 17, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8141, 8151, 8139, 8188, 8195, 8315, 8196] avg=8192.0000 max=8315 max_rank=6 min=8139 min_rank=3 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32894 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8064, 8320, 8064, 8064, 8192, 8320, 8320, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8192, 8192, 8192, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1009, 1037, 1053, 1018, 972, 997, 1054, 1010, 1018, 1042, 1029, 1014, 968, 1046, 1055, 1085, 1017, 1037, 1001, 982, 1020, 1002, 1056, 1043, 997, 1032, 1020, 997, 1002, 993, 1019, 1003, 1071, 1047, 1007, 1081, 983, 1051, 1068, 1032, 992, 1019, 1023, 1028, 1041, 1009, 977, 1015, 987, 1008, 1018, 1022, 1026, 1067, 1006, 1034, 1033, 999, 1067, 1022, 1039, 1047, 1041, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1076, 1057, 971, 991, 1011, 1039, 991, 1045, 1037, 1046, 1038, 976, 1011, 1010, 1006, 1022, 976, 979, 1064, 1011, 986, 1048, 1035, 1007, 1045, 1013, 1031, 1004, 1016, 991, 1035, 1038, 1065, 1016, 988, 1051, 983, 1056, 1057, 1058, 1050, 1060, 1021, 1032, 1059, 1027, 1028, 1024, 1031, 1000, 1052, 1026, 1013, 995, 1017, 992, 1045, 1024, 1014, 994, 1024, 1028, 1040, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1052, 1034, 980, 1033, 986, 1053, 1033, 1036, 1035, 1024, 1001, 987, 1031, 1048, 993, 1017, 994, 1042, 1054, 1019, 1038, 1032, 990, 1009, 1015, 1063, 988, 1046, 1022, 984, 1058, 1017, 1054, 1076, 988, 996, 1031, 1008, 1042, 1031, 1046, 1045, 1043, 1037, 1008, 1041, 1009, 1034, 1015, 1001, 1025, 979, 979, 1048, 1017, 1029, 982, 1018, 1063, 1024, 1077, 1055, 1035, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1065, 971, 1048, 993, 1042, 995, 1006, 1004, 1022, 989, 1030, 1029, 1027, 1024, 1021, 996, 964, 986, 988, 1031, 1003, 1063, 1050, 1044, 1009, 1020, 1040, 1036, 1033, 1014, 1023, 1041, 982, 983, 1031, 1034, 1057, 1048, 1032, 978, 1035, 1038, 1062, 1028, 1005, 1027, 1011, 1032, 1057, 964, 1032, 1032, 1042, 1048, 1031, 1031, 1051, 1034, 1101, 1037, 980, 1018, 1034, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1053, 1062, 1030, 1015, 988, 1043, 1055, 1023, 1008, 1031, 1039, 994, 1053, 1053, 1024, 1001, 1047, 1030, 1033, 982, 1037, 970, 992, 999, 1036, 1009, 965, 1029, 1006, 979, 1071, 1005, 1073, 1059, 1020, 1024, 1030, 1051, 1048, 1052, 1032, 1033, 1022, 994, 1054, 1001, 1023, 1011, 1011, 1015, 1057, 1020, 1010, 1022, 1000, 1037, 1018, 995, 1000, 1068, 990, 1038, 1016, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1003, 1016, 1051, 1057, 974, 1064, 1039, 1041, 985, 1043, 1001, 1010, 1028, 1022, 1046, 1040, 1008, 1040, 1006, 1048, 1000, 1038, 1067, 1012, 1020, 1044, 1029, 987, 1065, 1024, 1075, 998, 1001, 1058, 1034, 948, 1007, 1031, 1053, 1007, 1013, 1015, 1041, 1010, 1059, 1039, 965, 1063, 996, 1097, 1018, 1031, 1012, 995, 1011, 998, 981, 1031, 1025, 1023, 1042, 1021, 1031, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[985, 990, 993, 1003, 1025, 1051, 1062, 982, 993, 1033, 1027, 989, 1035, 1016, 1041, 1089, 977, 1029, 1006, 1000, 988, 1031, 1068, 1051, 1042, 1010, 993, 1081, 1031, 1044, 1033, 987, 1021, 1019, 1038, 1051, 998, 984, 1005, 1018, 1046, 1007, 1044, 1042, 1114, 975, 1048, 1023, 999, 988, 997, 964, 1033, 1027, 1062, 1030, 1052, 985, 1052, 1037, 1091, 1025, 1029, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1018, 1032, 1025, 1017, 1042, 1002, 1026, 990, 1043, 1023, 1054, 1026, 1033, 1013, 1009, 1017, 1043, 1040, 974, 995, 1090, 1010, 1028, 1022, 1039, 1021, 991, 1058, 996, 1055, 1104, 987, 1003, 1061, 1019, 1013, 1057, 1035, 1057, 983, 1035, 1010, 1036, 1006, 1028, 1009, 1026, 1003, 1049, 1027, 1031, 978, 1050, 1010, 1010, 1028, 1018, 1022, 1012, 1075, 1003, 1053, 984, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1009, 1037, 1053, 1018, 972, 997, 1054, 1010, 9, 4, 0, 0, 0, 0, 0, 0, 1018, 1042, 1029, 1014, 968, 1046, 1055, 1085, 4, 0, 0, 0, 0, 0, 0, 0, 1017, 1037, 1001, 982, 1020, 1002, 1056, 1043, 38, 0, 0, 0, 0, 0, 0, 0, 997, 1032, 1020, 997, 1002, 993, 1019, 1003, 9, 0, 0, 0, 0, 0, 0, 0, 1071, 1047, 1007, 1081, 983, 1051, 1030, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1019, 1023, 1028, 1019, 1009, 977, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1008, 1018, 1022, 1026, 1067, 1006, 1034, 34, 3, 0, 0, 0, 0, 0, 0, 1033, 999, 1026, 1022, 1039, 1047, 1041, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8261, 8196, 8072, 8302, 8082, 8205, 8255] avg=8192.0000 max=8302 max_rank=4 min=8072 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1076, 1057, 971, 991, 1011, 1039, 991, 1045, 10, 4, 0, 0, 0, 0, 0, 0, 1037, 1046, 1038, 976, 1011, 1010, 1006, 1022, 5, 0, 0, 0, 0, 0, 0, 0, 976, 979, 1064, 1011, 986, 1048, 1035, 1007, 36, 0, 0, 0, 0, 0, 0, 0, 1045, 1013, 1031, 1004, 1016, 991, 1035, 1038, 9, 0, 0, 0, 0, 0, 0, 0, 1065, 1016, 988, 1051, 983, 1056, 1019, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1060, 1021, 1032, 1035, 1027, 1028, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1000, 1052, 1026, 1013, 995, 1017, 992, 34, 3, 0, 0, 0, 0, 0, 0, 1045, 1024, 975, 994, 1024, 1028, 1040, 1060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8151, 8142, 8182, 8236, 8277, 8163, 8190] avg=8192.0000 max=8277 max_rank=5 min=8142 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32866 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1003, 1016, 1051, 1057, 974, 1064, 1039, 1041, 10, 4, 0, 0, 0, 0, 0, 0, 985, 1043, 1001, 1010, 1028, 1022, 1046, 1040, 5, 0, 0, 0, 0, 0, 0, 0, 1008, 1040, 1006, 1048, 1000, 1038, 1067, 1012, 37, 0, 0, 0, 0, 0, 0, 0, 1020, 1044, 1029, 987, 1065, 1024, 1075, 998, 9, 0, 0, 0, 0, 0, 0, 0, 1001, 1058, 1034, 948, 1007, 1031, 1015, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1015, 1041, 1010, 1035, 1039, 965, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1097, 1018, 1031, 1012, 995, 1011, 998, 34, 3, 0, 0, 0, 0, 0, 0, 981, 1031, 985, 1023, 1042, 1021, 1031, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8180, 8256, 8251, 8101, 8181, 8195, 8113] avg=8192.0000 max=8259 max_rank=0 min=8101 min_rank=4 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32590 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1065, 971, 1048, 993, 1042, 995, 1006, 1004, 9, 4, 0, 0, 0, 0, 0, 0, 1022, 989, 1030, 1029, 1027, 1024, 1021, 996, 4, 0, 0, 0, 0, 0, 0, 0, 964, 986, 988, 1031, 1003, 1063, 1050, 1044, 40, 0, 0, 0, 0, 0, 0, 0, 1009, 1020, 1040, 1036, 1033, 1014, 1023, 1041, 9, 0, 0, 0, 0, 0, 0, 0, 982, 983, 1031, 1034, 1057, 1048, 995, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1038, 1062, 1028, 983, 1027, 1011, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 964, 1032, 1032, 1042, 1048, 1031, 1031, 33, 3, 0, 0, 0, 0, 0, 0, 1051, 1034, 1058, 1037, 980, 1018, 1034, 1054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8142, 8169, 8225, 8108, 8216, 8273, 8266] avg=8192.0000 max=8273 max_rank=6 min=8108 min_rank=4 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32863 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1052, 1034, 980, 1033, 986, 1053, 1033, 1036, 9, 4, 0, 0, 0, 0, 0, 0, 1035, 1024, 1001, 987, 1031, 1048, 993, 1017, 4, 0, 0, 0, 0, 0, 0, 0, 994, 1042, 1054, 1019, 1038, 1032, 990, 1009, 38, 0, 0, 0, 0, 0, 0, 0, 1015, 1063, 988, 1046, 1022, 984, 1058, 1017, 9, 0, 0, 0, 0, 0, 0, 0, 1054, 1076, 988, 996, 1031, 1008, 1005, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1045, 1043, 1037, 986, 1041, 1009, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1001, 1025, 979, 979, 1048, 1017, 1029, 33, 3, 0, 0, 0, 0, 0, 0, 982, 1018, 1022, 1024, 1077, 1055, 1035, 986, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8140, 8216, 8202, 8189, 8241, 8129, 8199] avg=8192.0000 max=8241 max_rank=5 min=8129 min_rank=6 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1053, 1062, 1030, 1015, 988, 1043, 1055, 1023, 10, 4, 0, 0, 0, 0, 0, 0, 1008, 1031, 1039, 994, 1053, 1053, 1024, 1001, 5, 0, 0, 0, 0, 0, 0, 0, 1047, 1030, 1033, 982, 1037, 970, 992, 999, 36, 0, 0, 0, 0, 0, 0, 0, 1036, 1009, 965, 1029, 1006, 979, 1071, 1005, 9, 0, 0, 0, 0, 0, 0, 0, 1073, 1059, 1020, 1024, 1030, 1051, 1011, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1033, 1022, 994, 1030, 1001, 1023, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1015, 1057, 1020, 1010, 1022, 1000, 1037, 33, 3, 0, 0, 0, 0, 0, 0, 1018, 995, 961, 1068, 990, 1038, 1016, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8208, 8126, 8109, 8320, 8146, 8208, 8136] avg=8192.0000 max=8320 max_rank=4 min=8109 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[985, 990, 993, 1003, 1025, 1051, 1062, 982, 14, 9, 0, 0, 0, 0, 0, 0, 993, 1033, 1027, 989, 1035, 1016, 1041, 1089, 9, 0, 0, 0, 0, 0, 0, 0, 977, 1029, 1006, 1000, 988, 1031, 1068, 1051, 43, 0, 0, 0, 0, 0, 0, 0, 1042, 1010, 993, 1081, 1031, 1044, 1033, 987, 14, 0, 0, 0, 0, 0, 0, 0, 1021, 1019, 1038, 1051, 998, 984, 961, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1007, 1044, 1042, 1077, 975, 1048, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 999, 988, 997, 964, 1033, 1027, 1062, 1030, 35, 8, 0, 0, 0, 0, 0, 0, 1052, 985, 1001, 1037, 1091, 1025, 1029, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8232, 8193, 8235, 8090, 8262, 8143, 8267] avg=8192.0000 max=8267 max_rank=7 min=8090 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1018, 1032, 1025, 1017, 1042, 1002, 1026, 990, 9, 4, 0, 0, 0, 0, 0, 0, 1043, 1023, 1054, 1026, 1033, 1013, 1009, 1017, 4, 0, 0, 0, 0, 0, 0, 0, 1043, 1040, 974, 995, 1090, 1010, 1028, 1022, 36, 0, 0, 0, 0, 0, 0, 0, 1039, 1021, 991, 1058, 996, 1055, 1104, 987, 9, 0, 0, 0, 0, 0, 0, 0, 1003, 1061, 1019, 1013, 1057, 1035, 1019, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1010, 1036, 1006, 1006, 1009, 1026, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1027, 1031, 978, 1050, 1010, 1010, 1028, 34, 3, 0, 0, 0, 0, 0, 0, 1018, 1022, 973, 1075, 1003, 1053, 984, 982, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8222, 8238, 8260, 8190, 8131, 8220, 8110] avg=8192.0000 max=8260 max_rank=3 min=8110 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8064, 8192, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8064, 8192, 8320, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8064, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8192, 8448, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8064, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1064, 1017, 988, 1007, 1032, 1027, 1025, 1044, 989, 1022, 1044, 1040, 1010, 1024, 1030, 972, 958, 1030, 1006, 1058, 1058, 1030, 1009, 1063, 1042, 1082, 1034, 1052, 996, 1034, 987, 1032, 1009, 1033, 1005, 1001, 1021, 1007, 1008, 1031, 1028, 1058, 1043, 1001, 1014, 1018, 1049, 1016, 1031, 1010, 1003, 1055, 1015, 1068, 1038, 1063, 981, 1021, 978, 997, 1041, 1003, 1017, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1029, 993, 1042, 1068, 1023, 1031, 1067, 1045, 1051, 1038, 1009, 985, 1008, 1004, 999, 1059, 978, 1021, 1016, 997, 1033, 1041, 1041, 1031, 1065, 1008, 978, 1057, 1005, 988, 986, 999, 1050, 943, 1061, 996, 980, 1024, 1062, 1032, 1114, 1004, 1033, 1057, 1026, 988, 947, 1007, 1076, 1041, 991, 1027, 1010, 1033, 1024, 1012, 1012, 1014, 1019, 1066, 989, 1026, 1081, 1096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1050, 1054, 1023, 1029, 990, 1047, 1062, 1023, 1008, 1040, 1022, 1010, 1062, 1058, 1013, 983, 1046, 1030, 1042, 963, 1034, 1005, 1044, 1031, 1068, 986, 969, 1035, 1028, 1027, 1011, 1048, 1020, 1056, 1073, 981, 1008, 1045, 987, 1089, 1002, 998, 1018, 991, 1073, 1059, 994, 1052, 987, 962, 999, 1002, 1006, 1024, 999, 1029, 1020, 1057, 1034, 1050, 947, 1031, 1046, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1021, 992, 1024, 1027, 1006, 970, 1040, 1048, 1020, 988, 1053, 972, 1022, 1022, 987, 1003, 1016, 1017, 1049, 988, 987, 1003, 1051, 1028, 983, 994, 1046, 1018, 1068, 1057, 1068, 1028, 1017, 1001, 1049, 999, 1032, 1055, 1015, 1042, 1018, 1029, 1094, 1097, 993, 1000, 1018, 999, 1036, 1014, 1034, 1048, 1051, 1016, 1061, 1061, 972, 1089, 1009, 1016, 1019, 1030, 1012, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1025, 1017, 1036, 997, 1042, 993, 1059, 1009, 1017, 1029, 1023, 1059, 998, 1048, 1050, 1018, 1010, 974, 1029, 991, 1010, 973, 1024, 1076, 1085, 1031, 1027, 1020, 1037, 1037, 1022, 1008, 1018, 979, 1047, 1012, 991, 967, 1024, 1083, 1040, 1031, 1055, 1012, 1054, 1030, 1042, 1006, 1024, 1028, 1039, 1055, 1019, 1018, 1040, 1019, 963, 1036, 1027, 1059, 1013, 1017, 1004, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1086, 998, 991, 1000, 1021, 1069, 1021, 1044, 1043, 1069, 1015, 1047, 1055, 1014, 1022, 1043, 1043, 981, 992, 965, 983, 1037, 1016, 1013, 1010, 1019, 1071, 1052, 1044, 1086, 992, 967, 1000, 1010, 1036, 1012, 1016, 1047, 1064, 1058, 1006, 982, 996, 1062, 1011, 1055, 1022, 1025, 956, 975, 983, 993, 1009, 1033, 1021, 1059, 1077, 1064, 1006, 1049, 1061, 999, 1037, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1035, 1050, 1059, 996, 1034, 1018, 1047, 1022, 981, 1033, 1017, 1059, 1017, 1074, 1026, 1044, 1025, 1034, 1037, 1013, 1053, 1013, 1075, 1033, 992, 1027, 1059, 1052, 1046, 982, 1025, 1041, 983, 1037, 1053, 1053, 990, 939, 1063, 1051, 1037, 1005, 1055, 1028, 997, 993, 966, 993, 1041, 1010, 1033, 1074, 983, 1023, 972, 1018, 1015, 1001, 1038, 963, 1002, 1030, 1000, 1071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[979, 968, 1007, 999, 1041, 1039, 999, 1037, 1016, 1077, 1050, 982, 1043, 990, 1013, 1024, 1064, 1003, 1025, 968, 1001, 1026, 1065, 1040, 1050, 1036, 1016, 1036, 998, 1026, 1018, 1044, 1022, 1022, 1009, 1060, 1041, 994, 1037, 1023, 1055, 1036, 1020, 1020, 1006, 1045, 1022, 1047, 1017, 1018, 997, 1027, 1000, 1057, 1000, 1072, 1031, 1033, 1028, 1023, 992, 1029, 1035, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1064, 1017, 988, 1007, 1032, 1027, 1012, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1022, 1044, 1040, 1010, 1024, 1030, 972, 0, 0, 0, 0, 0, 0, 0, 0, 958, 1030, 1006, 1058, 1058, 1030, 1009, 1063, 25, 5, 0, 0, 0, 0, 0, 0, 1042, 1082, 1034, 1027, 996, 1034, 987, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1033, 1005, 1001, 1021, 1007, 1008, 1031, 3, 3, 0, 0, 0, 0, 0, 0, 1028, 1058, 1037, 1001, 1014, 1018, 1049, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1010, 1003, 1055, 1015, 1068, 1038, 1063, 8, 3, 0, 0, 0, 0, 0, 0, 981, 1021, 978, 997, 1041, 1003, 1017, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8131, 8242, 8234, 8121, 8221, 8294, 8102] avg=8192.0000 max=8294 max_rank=6 min=8102 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1050, 1054, 1023, 1029, 990, 1047, 1049, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1040, 1022, 1010, 1062, 1058, 1013, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1030, 1042, 963, 1034, 1005, 1044, 1031, 24, 5, 0, 0, 0, 0, 0, 0, 1068, 986, 969, 1011, 1028, 1027, 1011, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1056, 1073, 981, 1008, 1045, 987, 1089, 3, 3, 0, 0, 0, 0, 0, 0, 1002, 998, 1012, 991, 1073, 1059, 994, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 987, 962, 999, 1002, 1006, 1024, 999, 1029, 8, 3, 0, 0, 0, 0, 0, 0, 1020, 1057, 1034, 1050, 947, 1031, 1046, 1053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8196, 8224, 8148, 8265, 8181, 8019, 8238] avg=8192.0000 max=8265 max_rank=0 min=8019 min_rank=6 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1025, 1017, 1036, 997, 1042, 993, 1046, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1029, 1023, 1059, 998, 1048, 1050, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 974, 1029, 991, 1010, 973, 1024, 1076, 24, 5, 0, 0, 0, 0, 0, 0, 1085, 1031, 1027, 996, 1037, 1037, 1022, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 979, 1047, 1012, 991, 967, 1024, 1083, 3, 3, 0, 0, 0, 0, 0, 0, 1040, 1031, 1049, 1012, 1054, 1030, 1042, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1028, 1039, 1055, 1019, 1018, 1040, 1019, 8, 3, 0, 0, 0, 0, 0, 0, 963, 1036, 1027, 1059, 1013, 1017, 1004, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8242, 8116, 8243, 8127, 8264, 8253, 8126] avg=8192.0000 max=8264 max_rank=5 min=8116 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1029, 993, 1042, 1068, 1023, 1031, 1054, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1038, 1009, 985, 1008, 1004, 999, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1021, 1016, 997, 1033, 1041, 1041, 1031, 25, 5, 0, 0, 0, 0, 0, 0, 1065, 1008, 978, 1032, 1005, 988, 986, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 943, 1061, 996, 980, 1024, 1062, 1032, 3, 3, 0, 0, 0, 0, 0, 0, 1114, 1004, 1027, 1057, 1026, 988, 947, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1041, 991, 1027, 1010, 1033, 1024, 1012, 8, 3, 0, 0, 0, 0, 0, 0, 1012, 1014, 1019, 1066, 989, 1026, 1081, 1093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8153, 8188, 8061, 8154, 8170, 8225, 8300] avg=8192.0000 max=8300 max_rank=7 min=8061 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1021, 992, 1024, 1027, 1006, 970, 1027, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 988, 1053, 972, 1022, 1022, 987, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1017, 1049, 988, 987, 1003, 1051, 1028, 24, 5, 0, 0, 0, 0, 0, 0, 983, 994, 1046, 994, 1068, 1057, 1068, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1001, 1049, 999, 1032, 1055, 1015, 1042, 3, 3, 0, 0, 0, 0, 0, 0, 1018, 1029, 1088, 1097, 993, 1000, 1018, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1014, 1034, 1048, 1051, 1016, 1061, 1061, 8, 3, 0, 0, 0, 0, 0, 0, 972, 1089, 1009, 1016, 1019, 1030, 1012, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8067, 8168, 8238, 8216, 8242, 8332, 8158] avg=8192.0000 max=8332 max_rank=6 min=8067 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32948 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1086, 998, 991, 1000, 1021, 1069, 1008, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1069, 1015, 1047, 1055, 1014, 1022, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 981, 992, 965, 983, 1037, 1016, 1013, 25, 5, 0, 0, 0, 0, 0, 0, 1010, 1019, 1071, 1027, 1044, 1086, 992, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1010, 1036, 1012, 1016, 1047, 1064, 1058, 3, 3, 0, 0, 0, 0, 0, 0, 1006, 982, 990, 1062, 1011, 1055, 1022, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 956, 975, 983, 993, 1009, 1033, 1021, 1059, 8, 3, 0, 0, 0, 0, 0, 0, 1077, 1064, 1006, 1049, 1061, 999, 1037, 1000, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8308, 8060, 8216, 8249, 8153, 8040, 8293] avg=8192.0000 max=8308 max_rank=1 min=8040 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[979, 968, 1007, 999, 1041, 1039, 980, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1070, 1050, 982, 1043, 990, 1013, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1003, 1025, 968, 1001, 1026, 1065, 1040, 27, 6, 0, 0, 0, 0, 0, 0, 1050, 1036, 1016, 1009, 998, 1026, 1018, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1022, 1009, 1060, 1041, 994, 1037, 1023, 7, 7, 7, 0, 0, 0, 0, 0, 1055, 1036, 1006, 1020, 1006, 1045, 1022, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1018, 997, 1027, 1000, 1057, 1000, 1072, 13, 7, 0, 0, 0, 0, 0, 0, 1031, 1033, 1028, 1023, 992, 1029, 1035, 1031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8050, 8188, 8225, 8197, 8229, 8237, 8208, 8202] avg=8192.0000 max=8237 max_rank=5 min=8050 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1035, 1050, 1059, 996, 1034, 1018, 1034, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1033, 1017, 1059, 1017, 1074, 1026, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1034, 1037, 1013, 1053, 1013, 1075, 1033, 25, 5, 0, 0, 0, 0, 0, 0, 992, 1027, 1059, 1027, 1046, 982, 1025, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1037, 1053, 1053, 990, 939, 1063, 1051, 3, 3, 0, 0, 0, 0, 0, 0, 1037, 1005, 1049, 1028, 997, 993, 966, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1010, 1033, 1074, 983, 1023, 972, 1018, 8, 3, 0, 0, 0, 0, 0, 0, 1015, 1001, 1038, 963, 1002, 1030, 1000, 1068, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8251, 8313, 8199, 8175, 8068, 8165, 8117] avg=8192.0000 max=8313 max_rank=2 min=8068 min_rank=5 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32525 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8064, 8192, 8192, 8448, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8064, 8192, 8320, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8192, 8192, 8448, 8192, 8192, 8192, 8320, 8448, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8192, 8320, 8192, 8320, 8192, 8448, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[996, 984, 1013, 1015, 1002, 1013, 1018, 1027, 1024, 1074, 1007, 972, 1066, 1098, 1032, 996, 1029, 1013, 1092, 1009, 963, 994, 1053, 1081, 1024, 1044, 986, 1015, 978, 1026, 1043, 1001, 998, 1041, 1015, 1048, 1022, 1013, 1038, 1058, 1025, 1061, 1016, 976, 1065, 1043, 1037, 1050, 1038, 1012, 967, 1039, 977, 1072, 1071, 1028, 1014, 1042, 1045, 1045, 972, 1007, 1026, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[996, 964, 1105, 1025, 1035, 1047, 1000, 1039, 999, 1024, 1074, 1028, 1056, 981, 990, 1040, 1066, 1058, 1014, 1008, 1058, 1075, 1055, 1036, 1031, 1028, 1013, 1022, 1025, 1042, 998, 1036, 984, 1035, 1024, 1037, 1049, 1001, 1040, 1065, 1042, 966, 1018, 968, 1057, 1036, 1014, 986, 1043, 983, 1028, 1028, 1015, 1014, 999, 1047, 992, 971, 999, 1027, 1037, 1017, 1020, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1064, 1051, 1035, 1047, 1012, 1021, 992, 1046, 1032, 1013, 1034, 955, 1071, 1004, 1004, 1032, 1053, 1035, 1005, 1048, 1004, 1047, 984, 974, 1005, 989, 1060, 1029, 1012, 1055, 1045, 1020, 1039, 1017, 1003, 1011, 1023, 1032, 1032, 1023, 1053, 924, 1052, 1039, 1096, 997, 1046, 1031, 1032, 1003, 1019, 1074, 1054, 985, 1059, 1049, 984, 980, 1005, 968, 1064, 1034, 1040, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1035, 996, 997, 1011, 1061, 1034, 1047, 984, 974, 976, 1030, 974, 1039, 1029, 1071, 1041, 1085, 1029, 1056, 981, 1072, 1053, 997, 1032, 1005, 1015, 1031, 1004, 1030, 1028, 1022, 1059, 1065, 1007, 1021, 1074, 1031, 1026, 1050, 1034, 989, 1006, 1057, 1010, 1034, 1050, 1020, 1007, 1012, 1037, 1026, 953, 1026, 1037, 988, 1086, 1023, 1037, 985, 995, 1017, 978, 1024, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[993, 1098, 1069, 1008, 1001, 1000, 975, 1001, 1057, 992, 987, 1015, 1061, 1028, 1028, 1038, 996, 980, 999, 1067, 1033, 987, 1017, 1077, 1062, 961, 992, 1050, 1003, 1014, 1012, 976, 1037, 1041, 983, 1087, 1038, 1053, 1045, 1034, 1021, 1010, 991, 1030, 1027, 1056, 1028, 994, 991, 1024, 1011, 1024, 1006, 998, 1047, 1059, 991, 1053, 1043, 1061, 1067, 1051, 1017, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1013, 1050, 1023, 990, 1020, 997, 1031, 1070, 1012, 1049, 1025, 1006, 984, 995, 999, 1038, 1032, 1017, 1036, 1031, 1067, 1013, 1067, 1010, 1018, 1055, 1006, 1003, 995, 1025, 1013, 1068, 981, 1027, 996, 1092, 1009, 1030, 1008, 1095, 1016, 1057, 1040, 1024, 1021, 997, 1021, 980, 1030, 1015, 1031, 1006, 1026, 1037, 1028, 1020, 1043, 1002, 1012, 992, 1039, 1061, 1006, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1019, 1056, 1007, 1007, 1004, 1046, 1082, 1039, 1023, 1068, 1050, 1045, 1047, 1026, 1080, 1015, 997, 1002, 971, 1010, 1031, 1030, 984, 987, 989, 1018, 1061, 1010, 1023, 1069, 963, 1046, 1109, 1033, 1042, 1023, 1018, 1009, 1032, 997, 1003, 995, 1025, 1055, 1033, 1007, 1076, 1035, 1032, 983, 959, 1037, 987, 1002, 1023, 1042, 1019, 1042, 1003, 975, 1055, 1033, 994, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1046, 998, 1031, 1027, 1074, 986, 1037, 975, 981, 1005, 1057, 1044, 1017, 996, 1080, 1060, 1039, 984, 1049, 1009, 1001, 1037, 1070, 1021, 982, 1001, 1045, 1087, 1058, 1004, 1026, 1008, 1013, 997, 1001, 1013, 1064, 988, 1028, 1003, 1038, 1020, 1029, 1050, 1064, 995, 1004, 970, 1050, 1027, 1031, 1008, 1000, 1064, 1030, 986, 1015, 1020, 1060, 1024, 989, 1067, 1051, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1064, 1051, 1035, 1047, 1012, 1021, 992, 1046, 6, 0, 0, 0, 0, 0, 0, 0, 1032, 1013, 1034, 955, 1058, 1004, 1004, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1035, 1005, 1048, 1004, 1047, 984, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 989, 1060, 1029, 1012, 1055, 1045, 1020, 13, 7, 0, 0, 0, 0, 0, 0, 1039, 1017, 1003, 970, 1023, 1032, 1032, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 924, 1052, 1039, 1096, 997, 1046, 1031, 6, 0, 0, 0, 0, 0, 0, 0, 1032, 1003, 1019, 1074, 1054, 985, 1059, 1049, 14, 1, 0, 0, 0, 0, 0, 0, 984, 980, 1005, 968, 1064, 1034, 1040, 990, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8132, 8129, 8235, 8139, 8244, 8290, 8093] avg=8192.0000 max=8290 max_rank=6 min=8093 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[996, 964, 1105, 1025, 1035, 1047, 1000, 1039, 6, 0, 0, 0, 0, 0, 0, 0, 999, 1024, 1074, 1028, 1043, 981, 990, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1058, 1014, 1008, 1058, 1075, 1055, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1028, 1013, 1022, 1025, 1042, 998, 1036, 13, 7, 0, 0, 0, 0, 0, 0, 984, 1035, 1024, 995, 1049, 1001, 1040, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 966, 1018, 968, 1057, 1036, 1014, 986, 6, 0, 0, 0, 0, 0, 0, 0, 1043, 983, 1028, 1028, 1015, 1014, 999, 1047, 14, 1, 0, 0, 0, 0, 0, 0, 992, 971, 999, 1027, 1037, 1017, 1020, 1026, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8179, 8349, 8215, 8193, 8093, 8172, 8118] avg=8192.0000 max=8349 max_rank=2 min=8093 min_rank=5 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[996, 984, 1013, 1015, 1002, 1013, 1018, 1027, 6, 0, 0, 0, 0, 0, 0, 0, 1024, 1074, 1007, 972, 1053, 1098, 1032, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1013, 1092, 1009, 963, 994, 1053, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1044, 986, 1015, 978, 1026, 1043, 1001, 13, 7, 0, 0, 0, 0, 0, 0, 998, 1041, 1015, 1006, 1022, 1013, 1038, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1061, 1016, 976, 1065, 1043, 1037, 1050, 6, 0, 0, 0, 0, 0, 0, 0, 1038, 1012, 967, 1039, 977, 1072, 1071, 1028, 14, 1, 0, 0, 0, 0, 0, 0, 1014, 1042, 1045, 1045, 972, 1007, 1026, 987, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8074, 8256, 8213, 8137, 8191, 8279, 8219, 8167] avg=8192.0000 max=8279 max_rank=5 min=8074 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1035, 996, 997, 1011, 1061, 1034, 1047, 984, 6, 0, 0, 0, 0, 0, 0, 0, 974, 976, 1030, 974, 1027, 1029, 1071, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1029, 1056, 981, 1072, 1053, 997, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1015, 1031, 1004, 1030, 1028, 1022, 1059, 14, 7, 0, 0, 0, 0, 0, 0, 1065, 1007, 1021, 1030, 1031, 1026, 1050, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1006, 1057, 1010, 1034, 1050, 1020, 1007, 6, 0, 0, 0, 0, 0, 0, 0, 1012, 1037, 1026, 953, 1026, 1037, 988, 1086, 14, 0, 0, 0, 0, 0, 0, 0, 1023, 1037, 985, 995, 1017, 978, 1024, 1033, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8122, 8284, 8215, 8264, 8179, 8179, 8122] avg=8192.0000 max=8284 max_rank=2 min=8122 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1013, 1050, 1023, 990, 1020, 997, 1031, 1070, 6, 0, 0, 0, 0, 0, 0, 0, 1012, 1049, 1025, 1006, 972, 995, 999, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1017, 1036, 1031, 1067, 1013, 1067, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1055, 1006, 1003, 995, 1025, 1013, 1068, 14, 7, 0, 0, 0, 0, 0, 0, 981, 1027, 996, 1048, 1009, 1030, 1008, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1057, 1040, 1024, 1021, 997, 1021, 980, 6, 0, 0, 0, 0, 0, 0, 0, 1030, 1015, 1031, 1006, 1026, 1037, 1028, 1020, 14, 0, 0, 0, 0, 0, 0, 0, 1043, 1002, 1012, 992, 1039, 1061, 1006, 1036, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8096, 8252, 8204, 8194, 8162, 8207, 8221] avg=8192.0000 max=8252 max_rank=2 min=8096 min_rank=1 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[993, 1098, 1069, 1008, 1001, 1000, 975, 1001, 6, 0, 0, 0, 0, 0, 0, 0, 1057, 992, 987, 1015, 1048, 1028, 1028, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 976, 980, 999, 1067, 1033, 987, 1017, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 961, 992, 1050, 1003, 1014, 1012, 976, 14, 7, 0, 0, 0, 0, 0, 0, 1037, 1041, 983, 1043, 1038, 1053, 1045, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1010, 991, 1030, 1027, 1056, 1028, 994, 6, 0, 0, 0, 0, 0, 0, 0, 991, 1024, 1011, 1024, 1006, 998, 1047, 1059, 13, 1, 0, 0, 0, 0, 0, 0, 991, 1053, 1043, 1061, 1067, 1051, 1017, 1041, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8193, 8136, 8091, 8274, 8163, 8174, 8354] avg=8192.0000 max=8354 max_rank=7 min=8091 min_rank=3 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32965 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1019, 1056, 1007, 1007, 1004, 1046, 1082, 1039, 9, 0, 0, 0, 0, 0, 0, 0, 1023, 1068, 1050, 1045, 1023, 1026, 1080, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1002, 971, 1010, 1031, 1030, 984, 987, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1018, 1061, 1010, 1023, 1069, 963, 1046, 17, 12, 0, 0, 0, 0, 0, 0, 1109, 1033, 1042, 975, 1018, 1009, 1032, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 995, 1025, 1055, 1033, 1007, 1076, 1035, 11, 0, 0, 0, 0, 0, 0, 0, 1032, 983, 959, 1037, 987, 1002, 1023, 1042, 16, 4, 0, 0, 0, 0, 0, 0, 1019, 1042, 1003, 975, 1055, 1033, 994, 1053, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8330, 7984, 8208, 8215, 8240, 8085, 8205] avg=8192.0000 max=8330 max_rank=1 min=7984 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1046, 998, 1031, 1027, 1074, 986, 1037, 975, 6, 0, 0, 0, 0, 0, 0, 0, 981, 1005, 1057, 1044, 1005, 996, 1080, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 984, 1049, 1009, 1001, 1037, 1070, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1001, 1045, 1087, 1058, 1004, 1026, 1008, 13, 7, 0, 0, 0, 0, 0, 0, 1013, 997, 1001, 972, 1064, 988, 1028, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1020, 1029, 1050, 1064, 995, 1004, 970, 6, 0, 0, 0, 0, 0, 0, 0, 1050, 1027, 1031, 1008, 1000, 1064, 1030, 986, 14, 0, 0, 0, 0, 0, 0, 0, 1015, 1020, 1060, 1024, 989, 1067, 1051, 1002, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8228, 8189, 8231, 8066, 8176, 8210, 8256] avg=8192.0000 max=8256 max_rank=7 min=8066 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8064, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8192, 8448, 8192, 8320, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1072, 990, 971, 1010, 1008, 1025, 1068, 1002, 999, 1052, 1010, 1042, 990, 972, 1016, 1038, 1036, 1075, 1010, 1040, 1067, 1046, 1031, 1009, 1060, 1101, 1033, 1011, 998, 1028, 998, 1031, 1052, 1018, 985, 1038, 1045, 983, 1022, 956, 1016, 1006, 1038, 987, 1042, 1007, 1022, 976, 1043, 1018, 1059, 1082, 1080, 998, 1029, 1003, 1002, 1006, 1036, 1030, 1022, 1010, 1035, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[987, 1000, 1065, 972, 1048, 1049, 1035, 1021, 1000, 977, 951, 1021, 1031, 1038, 998, 1038, 1051, 1079, 1011, 1037, 1028, 1037, 1054, 997, 985, 1065, 1031, 1046, 972, 1003, 1018, 1035, 1003, 1030, 1055, 1055, 1061, 992, 1025, 1031, 966, 1057, 986, 1045, 1060, 1073, 1040, 997, 1047, 1008, 1016, 1038, 1049, 1059, 1015, 1004, 1028, 1038, 1042, 993, 1019, 1011, 993, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1055, 1068, 1073, 1035, 1039, 1065, 1055, 987, 1062, 1010, 1015, 1103, 986, 1000, 1023, 996, 1000, 976, 1022, 1038, 1030, 1020, 1019, 998, 1030, 998, 1008, 1095, 1059, 1019, 1028, 1023, 1003, 1012, 994, 964, 1001, 1010, 1017, 1031, 1038, 990, 1075, 1011, 1095, 1066, 998, 1049, 1050, 1028, 996, 1015, 1018, 1057, 964, 1000, 1002, 995, 1018, 1098, 989, 1011, 991, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1051, 1020, 1093, 971, 998, 1077, 1013, 1004, 1000, 1020, 1025, 1026, 1059, 1013, 999, 1041, 1042, 1055, 998, 1018, 1017, 1036, 1050, 1012, 1065, 994, 1012, 1028, 1030, 1051, 1056, 1027, 1070, 1020, 966, 1023, 1029, 1013, 1021, 951, 1017, 1023, 1032, 1043, 989, 1034, 1022, 1052, 997, 1012, 1009, 1027, 1021, 993, 1023, 1084, 1010, 1061, 1037, 997, 1025, 1023, 985, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1010, 978, 1072, 1024, 987, 997, 983, 1034, 1023, 1046, 994, 1022, 1028, 1018, 1047, 1044, 1041, 1047, 1071, 1045, 1027, 1030, 1025, 1021, 1073, 1051, 1035, 1034, 1003, 993, 969, 1023, 955, 1024, 1055, 1027, 991, 976, 1034, 1054, 1051, 996, 1056, 1047, 999, 987, 1032, 1002, 1030, 1046, 1014, 1059, 1048, 1077, 999, 1056, 1022, 1025, 1007, 1069, 991, 994, 994, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1005, 1056, 974, 1036, 990, 1039, 1002, 1024, 1035, 1002, 1071, 987, 1021, 973, 998, 1018, 1032, 1034, 1047, 1008, 976, 1011, 1046, 1017, 1041, 1039, 1028, 1025, 1039, 1045, 1018, 1059, 1047, 1046, 996, 999, 1066, 1000, 998, 1022, 1038, 1034, 1007, 1063, 1073, 1036, 1013, 988, 1012, 1031, 1063, 1018, 1011, 1026, 1003, 1015, 979, 1034, 1035, 1023, 1028, 1036, 1019, 1081] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1054, 1056, 962, 993, 1041, 1032, 1037, 1007, 1038, 1074, 1017, 991, 1001, 1023, 992, 1039, 1044, 1036, 1009, 1051, 984, 1055, 992, 1031, 1043, 1017, 1051, 1059, 1054, 1045, 1003, 1044, 1037, 1015, 1028, 983, 1009, 1051, 1006, 1014, 1031, 1051, 1024, 1015, 1006, 1034, 1016, 1036, 1054, 1021, 1037, 988, 1003, 1043, 1020, 1049, 1018, 990, 962, 1009, 1013, 1027, 1058, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1131, 988, 1022, 1037, 1020, 1009, 986, 1021, 1032, 1048, 985, 1030, 1019, 1028, 988, 1074, 1045, 981, 1035, 1020, 1005, 1021, 1050, 990, 1035, 1034, 1023, 1011, 1045, 1020, 1012, 1040, 1009, 1009, 1048, 1008, 1089, 1022, 991, 1012, 1014, 987, 997, 1009, 1047, 1040, 1057, 1020, 969, 1040, 1007, 1014, 1043, 1036, 1040, 1007, 982, 1057, 1015, 1026, 1014, 1027, 1080, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1072, 990, 971, 1010, 1008, 1025, 1068, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1052, 1010, 1042, 990, 972, 1016, 1038, 25, 9, 0, 0, 0, 0, 0, 0, 1008, 1075, 1010, 1040, 1067, 1046, 1031, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1101, 1033, 1011, 998, 1028, 998, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1018, 985, 1038, 1045, 983, 1022, 956, 52, 3, 0, 0, 0, 0, 0, 0, 1016, 1006, 1038, 987, 1027, 1007, 1022, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1018, 1059, 1082, 1080, 976, 1029, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1006, 1036, 1030, 1022, 1010, 1035, 1051, 15, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8153, 8286, 8208, 8154, 8079, 8290, 8220] avg=8192.0000 max=8290 max_rank=6 min=8079 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[987, 1000, 1065, 972, 1048, 1049, 1035, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 977, 951, 1021, 1031, 1038, 998, 1038, 25, 9, 0, 0, 0, 0, 0, 0, 1023, 1079, 1011, 1037, 1028, 1037, 1054, 997, 0, 0, 0, 0, 0, 0, 0, 0, 937, 1065, 1031, 1046, 972, 1003, 1018, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1030, 1055, 1055, 1061, 992, 1025, 1031, 48, 3, 0, 0, 0, 0, 0, 0, 966, 1057, 986, 1045, 1045, 1073, 1040, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1008, 1016, 1038, 1049, 1036, 1015, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1038, 1042, 993, 1019, 1011, 993, 1020, 15, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8088, 8266, 8107, 8303, 8209, 8213, 8173] avg=8192.0000 max=8303 max_rank=4 min=8088 min_rank=1 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1010, 978, 1072, 1024, 987, 997, 983, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1046, 994, 1022, 1028, 1018, 1047, 1044, 25, 9, 0, 0, 0, 0, 0, 0, 1013, 1047, 1071, 1045, 1027, 1030, 1025, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1051, 1035, 1034, 1003, 993, 969, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 955, 1024, 1055, 1027, 991, 976, 1034, 1054, 53, 3, 0, 0, 0, 0, 0, 0, 1051, 996, 1056, 1047, 985, 987, 1032, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1046, 1014, 1059, 1048, 1054, 999, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1025, 1007, 1069, 991, 994, 994, 1024, 14, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8085, 8256, 8279, 8128, 8172, 8156, 8306, 8154] avg=8192.0000 max=8306 max_rank=6 min=8085 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1055, 1068, 1073, 1035, 1039, 1065, 1055, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1010, 1015, 1103, 986, 1000, 1023, 996, 24, 9, 0, 0, 0, 0, 0, 0, 973, 976, 1022, 1038, 1030, 1020, 1019, 998, 0, 0, 0, 0, 0, 0, 0, 0, 979, 998, 1008, 1095, 1059, 1019, 1028, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1012, 994, 964, 1001, 1010, 1017, 1031, 51, 3, 0, 0, 0, 0, 0, 0, 1038, 990, 1075, 1011, 1080, 1066, 998, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1028, 996, 1015, 1018, 1034, 964, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 995, 1018, 1098, 989, 1011, 991, 1015, 15, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8377, 8228, 8076, 8209, 8086, 8307, 8105, 8148] avg=8192.0000 max=8377 max_rank=0 min=8076 min_rank=2 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1005, 1056, 974, 1036, 990, 1039, 1002, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1002, 1071, 987, 1021, 973, 998, 1018, 25, 9, 0, 0, 0, 0, 0, 0, 1004, 1034, 1047, 1008, 976, 1011, 1046, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1039, 1028, 1025, 1039, 1045, 1018, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1046, 996, 999, 1066, 1000, 998, 1022, 51, 3, 0, 0, 0, 0, 0, 0, 1038, 1034, 1007, 1063, 1058, 1036, 1013, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1031, 1063, 1018, 1011, 1004, 1003, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1034, 1035, 1023, 1028, 1036, 1019, 1081, 15, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8139, 8143, 8243, 8228, 8237, 8157, 8263] avg=8192.0000 max=8263 max_rank=7 min=8126 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1051, 1020, 1093, 971, 998, 1077, 1013, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1020, 1025, 1026, 1059, 1013, 999, 1041, 25, 8, 0, 0, 0, 0, 0, 0, 1014, 1055, 998, 1018, 1017, 1036, 1050, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 994, 1012, 1028, 1030, 1051, 1056, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1020, 966, 1023, 1029, 1013, 1021, 951, 52, 3, 0, 0, 0, 0, 0, 0, 1017, 1023, 1032, 1043, 975, 1034, 1022, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1012, 1009, 1027, 1021, 972, 1023, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1061, 1037, 997, 1025, 1023, 985, 1026, 14, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8216, 8200, 8211, 8148, 8198, 8145, 8191] avg=8192.0000 max=8227 max_rank=0 min=8145 min_rank=6 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1054, 1056, 962, 993, 1041, 1032, 1037, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1074, 1017, 991, 1001, 1023, 992, 1039, 25, 9, 0, 0, 0, 0, 0, 0, 1016, 1036, 1009, 1051, 984, 1055, 992, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1017, 1051, 1059, 1054, 1045, 1003, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1015, 1028, 983, 1009, 1051, 1006, 1014, 51, 3, 0, 0, 0, 0, 0, 0, 1031, 1051, 1024, 1015, 992, 1034, 1016, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1021, 1037, 988, 1003, 1021, 1020, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 990, 962, 1009, 1013, 1027, 1058, 1013, 14, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8209, 8174, 8265, 8197, 8199, 8193, 8117] avg=8192.0000 max=8265 max_rank=3 min=8117 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1131, 988, 1022, 1037, 1020, 1009, 986, 1021, 2, 0, 0, 0, 0, 0, 0, 0, 1032, 1048, 985, 1030, 1019, 1028, 988, 1074, 30, 13, 0, 0, 0, 0, 0, 0, 1010, 981, 1035, 1020, 1005, 1021, 1050, 990, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1034, 1023, 1011, 1045, 1020, 1012, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1009, 1048, 1008, 1089, 1022, 991, 1012, 55, 5, 0, 0, 0, 0, 0, 0, 1014, 987, 997, 1009, 1027, 1040, 1057, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1040, 1007, 1014, 1043, 1007, 1040, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1057, 1015, 1026, 1014, 1027, 1080, 1035, 18, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8247, 8112, 8165, 8248, 8151, 8127, 8270] avg=8192.0000 max=8270 max_rank=7 min=8112 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8320, 8192, 8192, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7936, 8320, 8320, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8064, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8064, 8192, 8192, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[980, 1055, 1059, 1056, 993, 1019, 999, 988, 970, 1072, 980, 1061, 1034, 996, 1036, 1015, 985, 1051, 1035, 1077, 1057, 1025, 1008, 1036, 1004, 993, 1033, 990, 1016, 1060, 1022, 1059, 1081, 1026, 998, 1046, 1011, 1038, 1021, 1054, 1041, 1043, 977, 1038, 1042, 1003, 1035, 1011, 1009, 1086, 1035, 1041, 1011, 991, 1038, 992, 1035, 1020, 1008, 985, 1038, 970, 1004, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[973, 1008, 1057, 987, 1036, 1072, 1032, 993, 1056, 1036, 1014, 1011, 1022, 1018, 1071, 1078, 1008, 1036, 1054, 1046, 1042, 1013, 1000, 1026, 1050, 977, 1042, 1053, 1020, 1018, 1058, 1046, 994, 960, 1003, 1040, 1040, 1052, 1032, 992, 1003, 1035, 1013, 1051, 1002, 1033, 983, 1091, 992, 1064, 1015, 1008, 1010, 939, 1014, 1032, 1003, 1011, 1026, 983, 1001, 1089, 1044, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[997, 1018, 978, 1055, 1034, 1072, 963, 967, 1022, 996, 975, 1035, 1010, 1034, 1016, 1048, 1037, 1035, 999, 1009, 1017, 1035, 970, 1044, 1030, 1049, 1043, 1079, 1074, 1043, 1017, 1071, 1035, 1027, 1006, 1033, 1037, 997, 1030, 1007, 1017, 1070, 1052, 1041, 1059, 1007, 1008, 1003, 1066, 995, 1012, 1054, 1001, 981, 974, 1042, 1019, 1024, 999, 1027, 1095, 999, 1001, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1009, 963, 996, 1012, 998, 1042, 1025, 1002, 1090, 1041, 1015, 1032, 990, 1024, 980, 1020, 1009, 1008, 1009, 1014, 1062, 1059, 1043, 1041, 1041, 990, 1017, 1025, 1043, 1011, 1051, 1047, 996, 1009, 1008, 1045, 1028, 1071, 984, 1023, 1000, 1048, 1028, 1012, 978, 1019, 1029, 1014, 1026, 1046, 1018, 1028, 1064, 1040, 994, 1056, 1062, 1024, 960, 1083, 1069, 1039, 1007, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1023, 1024, 1020, 980, 1072, 973, 1028, 1030, 1052, 1028, 1030, 1002, 1046, 1009, 1015, 999, 1033, 1002, 1015, 1038, 1004, 1045, 1032, 994, 995, 1047, 1020, 1040, 991, 983, 1036, 1056, 1054, 1030, 1022, 1086, 1038, 1007, 989, 982, 938, 989, 1059, 1062, 1063, 1031, 1063, 1052, 990, 1028, 1014, 994, 1043, 1023, 1056, 1092, 1046, 1000, 1036, 1008, 1015, 1035, 1012, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1009, 1033, 1041, 1053, 1051, 987, 1055, 996, 1018, 1040, 1007, 1007, 1040, 1043, 1022, 1020, 1006, 1055, 989, 982, 1076, 1030, 1005, 1042, 1033, 1033, 1032, 1003, 1010, 1018, 1025, 1069, 959, 1045, 1037, 1036, 1042, 1056, 1071, 993, 1006, 1018, 995, 981, 1029, 1048, 1036, 1011, 1062, 1043, 1044, 1049, 1034, 961, 992, 1029, 1006, 1017, 981, 997, 1037, 999, 1043, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1040, 1051, 1025, 1031, 1029, 1036, 1027, 1060, 1014, 1012, 1023, 1047, 1002, 1004, 1047, 994, 1012, 1023, 1064, 994, 989, 1016, 1017, 1042, 1064, 997, 1008, 1032, 977, 1020, 986, 986, 1011, 971, 993, 1081, 1017, 1016, 1082, 1010, 1058, 1048, 1035, 1023, 1050, 1009, 1022, 1011, 1030, 1036, 1030, 1038, 1017, 1003, 1093, 1037, 1032, 986, 1021, 1038, 1046, 1021, 968, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[969, 1040, 1017, 1058, 1033, 1028, 1025, 1068, 1027, 990, 1047, 1011, 1043, 1013, 1057, 1068, 1003, 1086, 1003, 986, 1008, 1018, 1029, 1032, 1019, 1028, 1017, 1050, 1049, 1042, 1023, 1021, 1032, 1013, 1073, 1055, 998, 1018, 980, 1006, 1065, 1007, 1016, 1022, 1024, 1025, 1002, 978, 1007, 1003, 992, 1033, 1029, 1032, 1013, 982, 1012, 1034, 992, 1045, 1007, 1052, 1036, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[980, 1055, 1059, 1056, 993, 1019, 999, 988, 23, 0, 0, 0, 0, 0, 0, 0, 966, 1072, 980, 1061, 1034, 996, 1036, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1049, 1035, 1077, 1057, 1025, 1008, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 993, 1033, 990, 1016, 1060, 1022, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1026, 998, 1046, 1011, 1038, 1021, 1054, 1, 0, 0, 0, 0, 0, 0, 0, 1041, 1040, 977, 1038, 1042, 1003, 1035, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1086, 1035, 1041, 1011, 991, 1038, 992, 2, 1, 0, 0, 0, 0, 0, 0, 1035, 1020, 1008, 985, 1038, 970, 1004, 1044, 7, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8160, 8272, 8147, 8276, 8187, 8206, 8116] avg=8192.0000 max=8276 max_rank=4 min=8116 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[997, 1018, 978, 1055, 1034, 1072, 963, 967, 23, 0, 0, 0, 0, 0, 0, 0, 1018, 996, 975, 1035, 1010, 1034, 1016, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1033, 999, 1009, 1017, 1035, 970, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1049, 1043, 1079, 1074, 1043, 1017, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1027, 1006, 1033, 1037, 997, 1030, 1007, 1, 0, 0, 0, 0, 0, 0, 0, 1017, 1067, 1052, 1041, 1059, 1007, 1008, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 995, 1012, 1054, 1001, 981, 974, 1042, 2, 1, 0, 0, 0, 0, 0, 0, 1019, 1024, 999, 1027, 1095, 999, 1001, 1046, 7, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8132, 8144, 8376, 8173, 8254, 8128, 8222] avg=8192.0000 max=8376 max_rank=3 min=8107 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1009, 1033, 1041, 1053, 1051, 987, 1055, 996, 23, 0, 0, 0, 0, 0, 0, 0, 1014, 1040, 1007, 1007, 1040, 1043, 1022, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1053, 989, 982, 1076, 1030, 1005, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1033, 1032, 1003, 1010, 1018, 1025, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 959, 1045, 1037, 1036, 1042, 1056, 1071, 993, 1, 0, 0, 0, 0, 0, 0, 0, 1006, 1016, 995, 981, 1029, 1048, 1036, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1043, 1044, 1049, 1034, 961, 992, 1029, 2, 1, 0, 0, 0, 0, 0, 0, 1006, 1017, 981, 997, 1037, 999, 1043, 1049, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8193, 8183, 8193, 8240, 8122, 8217, 8140] avg=8192.0000 max=8248 max_rank=0 min=8122 min_rank=5 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1023, 1024, 1020, 980, 1072, 973, 1028, 1030, 23, 0, 0, 0, 0, 0, 0, 0, 1048, 1028, 1030, 1002, 1046, 1009, 1015, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1000, 1015, 1038, 1004, 1045, 1032, 994, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1047, 1020, 1040, 991, 983, 1036, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1030, 1022, 1086, 1038, 1007, 989, 982, 1, 0, 0, 0, 0, 0, 0, 0, 938, 987, 1059, 1062, 1063, 1031, 1063, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1028, 1014, 994, 1043, 1023, 1056, 1092, 2, 1, 0, 0, 0, 0, 0, 0, 1046, 1000, 1036, 1008, 1015, 1035, 1012, 1017, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8177, 8161, 8138, 8209, 8255, 8243, 8180] avg=8192.0000 max=8255 max_rank=5 min=8138 min_rank=3 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32887 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[973, 1008, 1057, 987, 1036, 1072, 1032, 993, 23, 0, 0, 0, 0, 0, 0, 0, 1052, 1036, 1014, 1011, 1022, 1018, 1071, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1034, 1054, 1046, 1042, 1013, 1000, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 977, 1042, 1053, 1020, 1018, 1058, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 994, 960, 1003, 1040, 1040, 1052, 1032, 992, 1, 0, 0, 0, 0, 0, 0, 0, 1003, 1032, 1013, 1051, 1002, 1033, 983, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1064, 1015, 1008, 1010, 939, 1014, 1032, 2, 1, 0, 0, 0, 0, 0, 0, 1003, 1011, 1026, 983, 1001, 1089, 1044, 1028, 7, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8302, 8223, 8234, 8114, 8208, 8077, 8197] avg=8192.0000 max=8302 max_rank=1 min=8077 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1009, 963, 996, 1012, 998, 1042, 1025, 1002, 23, 0, 0, 0, 0, 0, 0, 0, 1085, 1041, 1015, 1032, 990, 1024, 980, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1006, 1009, 1014, 1062, 1059, 1043, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 990, 1017, 1025, 1043, 1011, 1051, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1009, 1008, 1045, 1028, 1071, 984, 1023, 1, 0, 0, 0, 0, 0, 0, 0, 1000, 1045, 1028, 1012, 978, 1019, 1029, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1046, 1018, 1028, 1064, 1040, 994, 1056, 2, 1, 0, 0, 0, 0, 0, 0, 1062, 1024, 960, 1083, 1069, 1039, 1007, 1019, 7, 5, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 8187, 8243, 8195, 8165, 8125, 8275, 8276] avg=8192.0000 max=8276 max_rank=7 min=8070 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1040, 1051, 1025, 1031, 1029, 1036, 1027, 1060, 26, 0, 0, 0, 0, 0, 0, 0, 1004, 1012, 1023, 1047, 1002, 1004, 1047, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1013, 1064, 994, 989, 1016, 1017, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 997, 1008, 1032, 977, 1020, 986, 949, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 971, 993, 1081, 1017, 1016, 1082, 1010, 2, 0, 0, 0, 0, 0, 0, 0, 1058, 1040, 1035, 1023, 1050, 1009, 1022, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1036, 1030, 1038, 1017, 1003, 1093, 1037, 4, 8, 0, 0, 0, 0, 0, 0, 1032, 986, 1021, 1038, 1046, 1021, 968, 1034, 11, 10, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8325, 8133, 8147, 8033, 8183, 8248, 8296, 8171] avg=8192.0000 max=8325 max_rank=0 min=8033 min_rank=3 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[969, 1040, 1017, 1058, 1033, 1028, 1025, 1068, 22, 0, 0, 0, 0, 0, 0, 0, 1023, 990, 1047, 1011, 1043, 1013, 1057, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1084, 1003, 986, 1008, 1018, 1029, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1028, 1017, 1050, 1049, 1042, 1023, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1013, 1073, 1055, 998, 1018, 980, 1006, 1, 0, 0, 0, 0, 0, 0, 0, 1065, 1005, 1016, 1022, 1024, 1025, 1002, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1003, 992, 1033, 1029, 1032, 1013, 982, 2, 1, 0, 0, 0, 0, 0, 0, 1012, 1034, 992, 1045, 1007, 1052, 1036, 1045, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8252, 8163, 8220, 8176, 8137, 8094, 8234] avg=8192.0000 max=8260 max_rank=0 min=8094 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8064, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8448, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8064, 8192, 8320, 8320, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True + [2026-08-30 05:02:11] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 674.6 | throughput per GPU (TFLOP/s/GPU): 494.2 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.035547E+01 | loss scale: 1.0 | grad norm: 15.304 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1015, 1040, 937, 1037, 1019, 1004, 1045, 995, 962, 992, 1080, 1005, 1043, 1013, 1072, 1059, 1044, 991, 1001, 1029, 1069, 1033, 1011, 996, 978, 1035, 1003, 1021, 1013, 1002, 1006, 1048, 1034, 1000, 1033, 1038, 1002, 1025, 1020, 1015, 1068, 1067, 1073, 996, 1069, 1004, 1000, 1061, 988, 1005, 1000, 1059, 1026, 982, 994, 1079, 1049, 1087, 1032, 1065, 1046, 1023, 990, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1039, 1011, 1033, 988, 1050, 1042, 1015, 988, 939, 1069, 1036, 1017, 1062, 1090, 991, 1063, 1050, 982, 1040, 1111, 1012, 1015, 1106, 1060, 1041, 1030, 1037, 962, 1012, 1049, 1025, 1072, 1051, 1032, 998, 992, 1039, 960, 1004, 1018, 1017, 1011, 1034, 978, 1007, 1011, 1021, 1004, 985, 1019, 1005, 1055, 1014, 1073, 1015, 1045, 1009, 1022, 1018, 1013, 1031, 1005, 999, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1047, 993, 1023, 1048, 1044, 1052, 1003, 997, 966, 1030, 1031, 1011, 1060, 949, 1042, 1033, 1002, 1034, 1058, 1052, 1025, 1054, 969, 991, 1007, 1028, 1000, 1021, 1049, 1017, 1036, 1006, 994, 1045, 1042, 981, 998, 1059, 1077, 1017, 1095, 1059, 980, 1028, 1053, 991, 1043, 1051, 1038, 1018, 1008, 1047, 1050, 962, 1014, 989, 980, 1002, 1025, 1036, 1035, 1029, 1095, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1036, 1023, 1035, 1001, 999, 1027, 1059, 1059, 1057, 1018, 999, 1012, 964, 1022, 1018, 1034, 990, 1069, 1077, 1025, 1008, 1022, 998, 1021, 1010, 1048, 1028, 1015, 1032, 1003, 1031, 1028, 993, 1023, 1023, 1063, 1003, 1001, 1049, 1042, 1014, 1032, 1017, 1048, 1006, 955, 1065, 1070, 1007, 1002, 1027, 1040, 1040, 1046, 991, 996, 1003, 1023, 1007, 1031, 1015, 1076, 1018, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1023, 1022, 1079, 1058, 1037, 1036, 1036, 1024, 994, 1056, 1095, 1020, 1054, 1052, 970, 1102, 1015, 938, 999, 976, 1050, 1013, 1068, 1004, 1012, 967, 1043, 1046, 1024, 1018, 1050, 973, 1023, 1023, 1030, 1008, 973, 1031, 1021, 1039, 1048, 1013, 1024, 1038, 1051, 1018, 1015, 1041, 1000, 999, 1010, 1049, 1000, 1024, 1022, 1020, 980, 1031, 1029, 1015, 1037, 1027, 1004, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1078, 1024, 1051, 1055, 1008, 987, 984, 1076, 1030, 1041, 979, 1036, 970, 1046, 974, 1050, 1011, 1039, 993, 1030, 1024, 1035, 1014, 1008, 995, 969, 1020, 1099, 1054, 1012, 994, 1038, 999, 1084, 980, 1012, 1013, 1008, 1004, 1091, 1015, 1010, 999, 1015, 1057, 1051, 940, 1041, 1014, 1028, 1041, 1107, 999, 1053, 1042, 1027, 1022, 1018, 1062, 1048, 1026, 998, 1000, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1007, 1039, 1047, 962, 1029, 1026, 1020, 1041, 1009, 964, 979, 1042, 1027, 1017, 1045, 1018, 1039, 1038, 1033, 1044, 1070, 1030, 1045, 1015, 1073, 1038, 987, 1020, 1052, 997, 1062, 1015, 1042, 1019, 997, 992, 995, 975, 1008, 1037, 981, 1038, 1072, 1063, 1029, 988, 1044, 1055, 1047, 1025, 1035, 986, 1039, 1013, 993, 971, 1037, 1020, 1083, 1024, 1014, 1078, 1002, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1014, 1019, 1064, 1003, 1054, 1011, 1004, 1016, 1060, 1054, 993, 987, 1026, 1029, 1017, 1059, 1008, 1027, 963, 1044, 1067, 1015, 975, 1016, 1054, 1026, 1024, 1035, 1045, 1048, 1051, 1049, 1064, 975, 1038, 1017, 996, 1024, 1017, 997, 1064, 1011, 1027, 1052, 1026, 951, 1033, 1016, 1009, 1078, 1032, 1048, 1026, 1035, 1099, 983, 988, 991, 1000, 1036, 1058, 1007, 1002, 979] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1039, 1011, 1021, 988, 1050, 1042, 1015, 988, 0, 0, 0, 0, 0, 0, 0, 0, 939, 1069, 1036, 1017, 1062, 1090, 991, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 982, 1040, 1111, 1006, 1015, 1106, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1030, 1037, 962, 1008, 1049, 1025, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1032, 998, 992, 1039, 960, 1004, 1018, 26, 12, 1, 0, 0, 0, 0, 0, 1017, 1011, 1034, 978, 1007, 1011, 1021, 978, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1019, 1005, 1055, 1014, 1073, 1015, 1045, 5, 2, 0, 0, 0, 0, 0, 0, 1009, 1022, 1018, 1013, 1031, 1005, 999, 1014, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8267, 8370, 8224, 8133, 8057, 8218, 8113] avg=8192.0000 max=8370 max_rank=2 min=8057 min_rank=5 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32521 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1015, 1040, 926, 1037, 1019, 1004, 1045, 995, 0, 0, 0, 0, 0, 0, 0, 0, 962, 992, 1080, 1005, 1043, 1013, 1072, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 991, 1001, 1029, 1063, 1033, 1011, 996, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1035, 1003, 1021, 1009, 1002, 1006, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1000, 1033, 1038, 1002, 1025, 1020, 1015, 27, 11, 1, 0, 0, 0, 0, 0, 1068, 1067, 1073, 996, 1069, 1004, 1000, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1005, 1000, 1059, 1026, 982, 994, 1079, 5, 2, 0, 0, 0, 0, 0, 0, 1049, 1087, 1032, 1065, 1046, 1023, 990, 1008, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8081, 8226, 8168, 8102, 8206, 8311, 8140, 8302] avg=8192.0000 max=8311 max_rank=5 min=8081 min_rank=0 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32959 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1047, 993, 1011, 1048, 1044, 1052, 1003, 997, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1030, 1031, 1011, 1060, 949, 1042, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1034, 1058, 1052, 1019, 1054, 969, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1028, 1000, 1021, 1044, 1017, 1036, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1045, 1042, 981, 998, 1059, 1077, 1017, 27, 12, 1, 0, 0, 0, 0, 0, 1095, 1059, 980, 1028, 1053, 991, 1043, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1018, 1008, 1047, 1050, 962, 1014, 989, 5, 2, 0, 0, 0, 0, 0, 0, 980, 1002, 1025, 1036, 1035, 1029, 1095, 1017, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8122, 8179, 8159, 8253, 8273, 8133, 8222] avg=8192.0000 max=8273 max_rank=5 min=8122 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1078, 1024, 1039, 1055, 1008, 987, 984, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1041, 979, 1036, 970, 1046, 974, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1039, 993, 1030, 1018, 1035, 1014, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 995, 969, 1020, 1099, 1049, 1012, 994, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1084, 980, 1012, 1013, 1008, 1004, 1091, 27, 12, 1, 0, 0, 0, 0, 0, 1015, 1010, 999, 1015, 1057, 1051, 940, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1028, 1041, 1107, 999, 1053, 1042, 1027, 5, 2, 0, 0, 0, 0, 0, 0, 1022, 1018, 1062, 1048, 1026, 998, 1000, 1008, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8126, 8148, 8176, 8231, 8101, 8318, 8185] avg=8192.0000 max=8318 max_rank=6 min=8101 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1036, 1023, 1023, 1001, 999, 1027, 1059, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1018, 999, 1012, 964, 1022, 1018, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1069, 1077, 1025, 1002, 1022, 998, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1048, 1028, 1015, 1028, 1003, 1031, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1023, 1023, 1063, 1003, 1001, 1049, 1042, 27, 12, 1, 0, 0, 0, 0, 0, 1014, 1032, 1017, 1048, 1006, 955, 1065, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1002, 1027, 1040, 1040, 1046, 991, 996, 5, 2, 0, 0, 0, 0, 0, 0, 1003, 1023, 1007, 1031, 1015, 1076, 1018, 1042, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8124, 8204, 8191, 8237, 8180, 8156, 8217] avg=8192.0000 max=8237 max_rank=4 min=8124 min_rank=1 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1023, 1022, 1066, 1058, 1037, 1036, 1036, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1056, 1095, 1020, 1054, 1052, 970, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 938, 999, 976, 1044, 1013, 1068, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 967, 1043, 1046, 1020, 1018, 1050, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1023, 1030, 1008, 973, 1031, 1021, 1039, 27, 13, 1, 0, 0, 0, 0, 0, 1048, 1013, 1024, 1038, 1051, 1018, 1015, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 999, 1010, 1049, 1000, 1024, 1022, 1020, 5, 2, 0, 0, 0, 0, 0, 0, 980, 1031, 1029, 1015, 1037, 1027, 1004, 1039, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8302, 8343, 8057, 8129, 8189, 8221, 8131, 8164] avg=8192.0000 max=8343 max_rank=1 min=8057 min_rank=2 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1007, 1039, 1030, 962, 1029, 1026, 1020, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 964, 979, 1042, 1027, 1017, 1045, 1018, 2, 0, 0, 0, 0, 0, 0, 0, 1039, 1038, 1033, 1044, 1057, 1030, 1045, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1038, 987, 1020, 1036, 997, 1062, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1019, 997, 992, 995, 975, 1008, 1037, 31, 17, 6, 0, 0, 0, 0, 0, 981, 1038, 1072, 1063, 1029, 988, 1044, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1025, 1035, 986, 1039, 1013, 993, 971, 7, 7, 0, 0, 0, 0, 0, 0, 1037, 1020, 1083, 1024, 1014, 1078, 1002, 1004, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8103, 8301, 8228, 8119, 8239, 8123, 8269] avg=8192.0000 max=8301 max_rank=2 min=8103 min_rank=1 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1014, 1019, 1051, 1003, 1054, 1011, 1004, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1054, 993, 987, 1026, 1029, 1017, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1027, 963, 1044, 1061, 1015, 975, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1026, 1024, 1035, 1040, 1048, 1051, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 975, 1038, 1017, 996, 1024, 1017, 997, 26, 13, 1, 0, 0, 0, 0, 0, 1064, 1011, 1027, 1052, 1026, 951, 1033, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1078, 1032, 1048, 1026, 1035, 1099, 983, 5, 2, 0, 0, 0, 0, 0, 0, 988, 991, 1000, 1036, 1058, 1007, 1002, 979, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8225, 8109, 8327, 8168, 8154, 8317, 8064] avg=8192.0000 max=8327 max_rank=3 min=8064 min_rank=7 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8192, 8192, 8064, 8192, 8320, 8320, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8448, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8064, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8192, 8320, 8320, 8192, 8448, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[990, 1020, 1020, 994, 1000, 1035, 1044, 1035, 1014, 1056, 1034, 1012, 1027, 1027, 1012, 1006, 1030, 1016, 978, 1016, 999, 1016, 1057, 1044, 1042, 1035, 1006, 1014, 1041, 989, 1052, 1057, 1027, 993, 1032, 1035, 1012, 991, 998, 1006, 1068, 1019, 1021, 1012, 1050, 965, 1047, 1033, 1012, 1038, 1035, 999, 1041, 1050, 1033, 1031, 1004, 1086, 1038, 1038, 1063, 969, 1033, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1012, 1034, 1041, 1062, 1021, 1000, 1025, 986, 1025, 1098, 1035, 1018, 1042, 1007, 1002, 1026, 988, 1026, 988, 1040, 1049, 1017, 1050, 1037, 1021, 1008, 1032, 1039, 1006, 1034, 1031, 1056, 1009, 1036, 1051, 1017, 1002, 1011, 1002, 990, 1014, 1008, 1006, 1035, 1051, 1018, 1030, 992, 1037, 1011, 996, 1012, 1050, 987, 1041, 1039, 1041, 983, 1015, 1042, 1026, 1024, 1093, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1028, 986, 1082, 1080, 1067, 1071, 1027, 1037, 1011, 1046, 1005, 1038, 1012, 1035, 967, 1028, 980, 998, 1005, 1010, 969, 1033, 1035, 1018, 1091, 1019, 1077, 1019, 1013, 999, 1021, 1046, 1031, 1000, 1015, 1057, 1018, 1014, 1025, 1012, 971, 1023, 1059, 1039, 1046, 998, 1026, 1051, 976, 1005, 1010, 1077, 1053, 969, 1006, 1058, 1058, 1038, 994, 1057, 1007, 1033, 973, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1020, 1029, 1046, 1063, 1022, 1058, 1012, 1024, 1017, 1006, 1033, 951, 1008, 975, 1033, 1019, 1012, 1023, 1006, 1070, 1085, 1006, 1066, 1003, 1023, 1045, 1025, 1023, 1048, 994, 1018, 1027, 1045, 953, 1065, 1028, 1041, 982, 1010, 1035, 1037, 1043, 1002, 1053, 1007, 1026, 1064, 1049, 1023, 990, 1022, 1011, 1004, 996, 999, 1049, 1004, 1016, 1052, 1007, 1026, 1037, 1043, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1044, 1022, 1035, 1020, 996, 998, 1025, 1008, 968, 1036, 1035, 1048, 982, 1033, 1068, 1026, 1024, 1025, 1038, 1012, 1061, 998, 1052, 1040, 982, 994, 1007, 1042, 991, 975, 1010, 1041, 1009, 982, 1015, 1022, 1033, 1054, 998, 1029, 1039, 1037, 1032, 1026, 1043, 1052, 999, 1007, 1051, 1106, 992, 1018, 993, 1033, 1002, 1070, 1044, 1001, 1029, 1051, 1048, 1039, 1054, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1028, 1037, 1047, 980, 1065, 1048, 1011, 978, 979, 1013, 1025, 1041, 985, 1028, 1043, 1010, 1059, 993, 1020, 1036, 987, 1049, 1069, 1037, 1024, 1063, 1032, 1021, 966, 1048, 1083, 1029, 1017, 1047, 1019, 1053, 977, 1047, 993, 1007, 1042, 1014, 1008, 1000, 1033, 994, 1067, 1045, 961, 1021, 1020, 1019, 1027, 1036, 1056, 990, 1008, 1016, 1004, 1094, 1025, 1011, 1043, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1089, 1010, 1043, 1000, 1022, 1033, 1009, 1020, 962, 1073, 1006, 1037, 1059, 1017, 1013, 1002, 1003, 1058, 1047, 1019, 1040, 1049, 1035, 1048, 1050, 1026, 977, 1076, 1003, 1027, 1072, 993, 1021, 996, 1036, 1065, 1051, 1024, 1018, 1045, 1010, 1045, 1033, 1014, 983, 1025, 962, 1030, 1005, 1062, 1000, 1018, 992, 1012, 984, 1031, 1007, 991, 1033, 1011, 1057, 1018, 1052, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1026, 992, 999, 1001, 1061, 1049, 989, 1054, 1021, 1043, 1000, 1026, 1003, 1054, 934, 1072, 1011, 994, 997, 1042, 1048, 1068, 1018, 1027, 1033, 977, 1010, 992, 1031, 1019, 1032, 1003, 989, 997, 1043, 1071, 1019, 1030, 1016, 1024, 1032, 1047, 1042, 1008, 995, 1075, 1046, 1063, 1013, 1060, 1041, 1017, 1019, 1061, 1010, 1050, 1015, 999, 1024, 1038, 997, 1031, 1033, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1012, 1034, 1020, 1062, 1021, 1000, 1025, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1098, 1035, 1018, 1042, 1007, 1002, 1026, 20, 11, 0, 0, 0, 0, 0, 0, 988, 1026, 988, 1040, 1049, 1017, 1033, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1008, 1032, 1039, 1006, 1034, 1026, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1036, 1051, 1017, 1002, 1011, 1002, 990, 17, 10, 3, 0, 0, 0, 0, 0, 1014, 1008, 1006, 1035, 1051, 1018, 1030, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1011, 996, 1012, 1050, 987, 1041, 1039, 7, 5, 0, 0, 0, 0, 0, 0, 1041, 983, 1015, 1032, 1026, 1024, 1093, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8284, 8178, 8222, 8148, 8134, 8185, 8225] avg=8192.0000 max=8284 max_rank=1 min=8134 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1020, 1029, 1025, 1063, 1022, 1058, 1012, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1006, 1033, 951, 1008, 975, 1033, 1019, 22, 11, 0, 0, 0, 0, 0, 0, 1012, 1023, 1006, 1070, 1085, 1006, 1049, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1045, 1025, 1023, 1048, 994, 1013, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 953, 1065, 1028, 1041, 982, 1010, 1035, 17, 10, 3, 0, 0, 0, 0, 0, 1037, 1043, 1002, 1053, 1007, 1026, 1064, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 990, 1022, 1011, 1004, 996, 999, 1049, 7, 5, 0, 0, 0, 0, 0, 0, 1004, 1016, 1052, 997, 1026, 1037, 1043, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8075, 8254, 8198, 8189, 8259, 8106, 8202] avg=8192.0000 max=8259 max_rank=5 min=8075 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1044, 1022, 1015, 1020, 996, 998, 1025, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1036, 1035, 1048, 982, 1033, 1068, 1026, 21, 11, 0, 0, 0, 0, 0, 0, 1024, 1025, 1038, 1012, 1061, 998, 1035, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 982, 994, 1007, 1042, 991, 975, 1005, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 982, 1015, 1022, 1033, 1054, 998, 1029, 17, 9, 3, 0, 0, 0, 0, 0, 1039, 1037, 1032, 1026, 1043, 1052, 999, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1106, 992, 1018, 993, 1033, 1002, 1070, 7, 5, 0, 0, 0, 0, 0, 0, 1044, 1001, 1029, 1041, 1048, 1039, 1054, 992, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8228, 8233, 8037, 8171, 8214, 8277, 8248] avg=8192.0000 max=8277 max_rank=6 min=8037 min_rank=3 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32910 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1028, 986, 1060, 1080, 1067, 1071, 1027, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1046, 1005, 1038, 1012, 1035, 967, 1028, 22, 12, 0, 0, 0, 0, 0, 0, 980, 998, 1005, 1010, 969, 1033, 1018, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 1019, 1077, 1019, 1013, 999, 1016, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1000, 1015, 1057, 1018, 1014, 1025, 1012, 17, 10, 3, 0, 0, 0, 0, 0, 971, 1023, 1059, 1039, 1046, 998, 1026, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1005, 1010, 1077, 1053, 969, 1006, 1058, 7, 5, 0, 0, 0, 0, 0, 0, 1058, 1038, 994, 1047, 1007, 1033, 973, 984, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8356, 8176, 8031, 8280, 8202, 8191, 8166, 8134] avg=8192.0000 max=8356 max_rank=0 min=8031 min_rank=2 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 992, 979, 1001, 1061, 1049, 989, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1043, 1000, 1026, 1003, 1054, 934, 1072, 22, 11, 0, 0, 0, 0, 0, 0, 1011, 994, 997, 1042, 1048, 1068, 1002, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 977, 1010, 992, 1031, 1019, 1027, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 989, 997, 1043, 1071, 1019, 1030, 1016, 1024, 16, 9, 3, 0, 0, 0, 0, 0, 1032, 1047, 1042, 1008, 995, 1075, 1046, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1060, 1041, 1017, 1019, 1061, 1010, 1050, 7, 5, 0, 0, 0, 0, 0, 0, 1015, 999, 1024, 1028, 997, 1031, 1033, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8186, 8189, 8092, 8217, 8286, 8283, 8132] avg=8192.0000 max=8286 max_rank=5 min=8092 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1089, 1010, 1022, 1000, 1022, 1033, 1009, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1073, 1006, 1037, 1059, 1017, 1013, 1002, 21, 11, 0, 0, 0, 0, 0, 0, 1003, 1058, 1047, 1019, 1040, 1049, 1018, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1026, 977, 1076, 1003, 1027, 1067, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 996, 1036, 1065, 1051, 1024, 1018, 1045, 17, 10, 3, 0, 0, 0, 0, 0, 1010, 1045, 1033, 1014, 983, 1025, 962, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1062, 1000, 1018, 992, 1012, 984, 1031, 7, 5, 0, 0, 0, 0, 0, 0, 1007, 991, 1033, 1001, 1057, 1018, 1052, 987, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8201, 8282, 8219, 8286, 8081, 8116, 8146] avg=8192.0000 max=8286 max_rank=4 min=8081 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1028, 1037, 1018, 980, 1065, 1048, 1011, 978, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1013, 1025, 1041, 985, 1028, 1043, 1010, 26, 16, 0, 0, 0, 0, 0, 0, 1059, 993, 1020, 1036, 987, 1049, 1049, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1063, 1032, 1021, 966, 1048, 1074, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1047, 1019, 1053, 977, 1047, 993, 1007, 20, 13, 7, 0, 0, 0, 0, 0, 1042, 1014, 1008, 1000, 1033, 994, 1067, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 961, 1021, 1020, 1019, 1027, 1036, 1056, 990, 13, 9, 0, 0, 0, 0, 0, 0, 1008, 1016, 1004, 1074, 1025, 1011, 1043, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8166, 8230, 8257, 8200, 8177, 8152, 8189] avg=8192.0000 max=8257 max_rank=3 min=8152 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[990, 1020, 1000, 994, 1000, 1035, 1044, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1056, 1034, 1012, 1027, 1027, 1012, 1006, 21, 11, 0, 0, 0, 0, 0, 0, 1030, 1016, 978, 1016, 999, 1016, 1040, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1035, 1006, 1014, 1041, 989, 1047, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 993, 1032, 1035, 1012, 991, 998, 1006, 17, 9, 3, 0, 0, 0, 0, 0, 1068, 1019, 1021, 1012, 1050, 965, 1047, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1038, 1035, 999, 1041, 1050, 1033, 1031, 7, 5, 0, 0, 0, 0, 0, 0, 1004, 1086, 1038, 1028, 1063, 969, 1033, 1039, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8220, 8139, 8231, 8123, 8194, 8251, 8260] avg=8192.0000 max=8260 max_rank=7 min=8118 min_rank=0 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8448, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8320, 8448, 8192, 8192, 8064, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8448, 8192, 8192, 8192, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1005, 1012, 993, 1026, 1022, 986, 1014, 1037, 961, 1069, 999, 1020, 1040, 1041, 1034, 1019, 1062, 1104, 996, 993, 978, 1078, 1057, 1106, 983, 1056, 1063, 1001, 995, 1038, 1042, 991, 1006, 1036, 1031, 1048, 991, 1011, 1003, 1012, 1032, 1011, 1024, 1009, 1022, 1020, 1043, 1013, 976, 1022, 1031, 1011, 1004, 988, 1076, 1001, 1037, 1009, 1085, 1047, 1036, 1053, 1017, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1011, 1008, 1030, 1021, 1048, 1030, 1038, 1015, 1018, 1063, 999, 1024, 965, 974, 1068, 1059, 1060, 1015, 1051, 1042, 1014, 1004, 1040, 1040, 1048, 1050, 1023, 1035, 983, 988, 994, 1044, 992, 1012, 975, 1028, 1031, 995, 1052, 1046, 1024, 950, 1037, 1003, 1027, 1081, 1045, 995, 1045, 1014, 1028, 1074, 1024, 1059, 970, 1015, 1076, 1009, 994, 1016, 1013, 1031, 1032, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1018, 1072, 1000, 1009, 1060, 1024, 1017, 992, 1021, 970, 1044, 969, 1075, 1010, 1015, 1103, 1005, 1046, 1031, 1028, 1061, 982, 1026, 1035, 990, 1034, 1019, 1012, 1012, 984, 984, 1045, 1033, 1033, 1034, 987, 997, 1016, 1032, 1038, 1020, 1010, 1000, 1029, 1029, 1015, 1061, 1018, 1001, 1047, 989, 1026, 1010, 1040, 1020, 1057, 1040, 1014, 1053, 1010, 1092, 1040, 1032, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1080, 1029, 982, 1030, 1001, 999, 1056, 1049, 1055, 1009, 1023, 997, 1004, 1030, 1013, 1012, 1100, 1013, 1037, 992, 1053, 1024, 985, 991, 1125, 1022, 1010, 1035, 1012, 1046, 1005, 1036, 984, 1046, 1069, 1027, 977, 1002, 959, 992, 1018, 1024, 1023, 1026, 1008, 1005, 1024, 1060, 1051, 991, 1009, 1047, 1081, 985, 1034, 1033, 1060, 1046, 994, 985, 1058, 981, 1033, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1023, 1054, 1078, 1041, 1030, 1023, 1024, 995, 1041, 1026, 1015, 1031, 1014, 1038, 998, 998, 1057, 1030, 1024, 1055, 996, 1002, 1021, 1039, 998, 1029, 1053, 1043, 1004, 1001, 1015, 1009, 977, 984, 1005, 1032, 1040, 997, 1025, 1041, 1008, 1023, 1031, 1057, 1039, 1009, 1021, 1037, 1020, 1063, 1009, 1056, 1000, 1050, 1008, 1032, 972, 987, 1015, 1009, 1030, 1051, 1046, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1007, 1022, 1020, 1027, 1064, 1013, 1038, 991, 1026, 1003, 1017, 1007, 1047, 1037, 1078, 1050, 1023, 1026, 1033, 1001, 1024, 1034, 1020, 1045, 1047, 1051, 1028, 1008, 978, 1048, 1019, 999, 1005, 1025, 1020, 1002, 1030, 1046, 1044, 1037, 976, 986, 1023, 1070, 1007, 997, 993, 1015, 1013, 1005, 1018, 996, 995, 1061, 979, 1013, 983, 1054, 1020, 1059, 1056, 1042, 1080, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1004, 1045, 1040, 1020, 1009, 1016, 991, 1079, 1020, 1082, 1028, 1013, 1010, 1009, 1013, 996, 1013, 995, 984, 974, 1029, 1048, 1032, 1033, 996, 1038, 1057, 1021, 1096, 1046, 1033, 1043, 1044, 1030, 1019, 1025, 1046, 1003, 985, 1057, 1014, 995, 1014, 977, 1037, 1029, 1030, 1019, 1047, 1014, 1020, 1021, 1040, 1069, 1010, 1055, 1048, 1014, 1051, 1036, 999, 990, 1013, 972] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1067, 1046, 969, 1029, 982, 1022, 1046, 1022, 1001, 1032, 1030, 1036, 1040, 1013, 1003, 1044, 1039, 1034, 1042, 1016, 1002, 1033, 1046, 976, 1041, 1066, 1033, 995, 995, 984, 1030, 993, 1035, 1051, 1048, 1018, 1012, 977, 1038, 1093, 1050, 1028, 1064, 1031, 1031, 1025, 1014, 988, 1040, 1001, 1010, 1003, 1049, 1004, 976, 1026, 1062, 1077, 1024, 1049, 1001, 999, 1027, 978] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1005, 1011, 993, 1026, 1022, 986, 1014, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 961, 1069, 999, 1020, 1040, 1041, 1034, 1019, 4, 0, 0, 0, 0, 0, 0, 0, 1034, 1104, 996, 993, 978, 1078, 1057, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1056, 1063, 1001, 995, 1038, 1042, 991, 2, 1, 0, 0, 0, 0, 0, 0, 1006, 1036, 1031, 1048, 991, 1011, 1003, 1012, 31, 0, 0, 0, 0, 0, 0, 0, 1032, 1011, 1024, 1009, 1022, 1020, 1043, 1013, 13, 10, 0, 0, 0, 0, 0, 0, 976, 1022, 1031, 1011, 1004, 988, 1076, 1001, 9, 0, 0, 0, 0, 0, 0, 0, 1037, 1009, 1085, 1047, 995, 1053, 1017, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8094, 8187, 8346, 8172, 8169, 8197, 8118, 8253] avg=8192.0000 max=8346 max_rank=2 min=8094 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1007, 1021, 1020, 1027, 1064, 1013, 1038, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1003, 1017, 1007, 1047, 1037, 1078, 1050, 4, 0, 0, 0, 0, 0, 0, 0, 995, 1026, 1033, 1001, 1024, 1034, 1020, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1051, 1028, 1008, 978, 1048, 1019, 999, 2, 1, 0, 0, 0, 0, 0, 0, 1005, 1025, 1020, 1002, 1030, 1046, 1044, 1037, 31, 0, 0, 0, 0, 0, 0, 0, 976, 986, 1023, 1070, 1007, 997, 993, 1015, 13, 10, 0, 0, 0, 0, 0, 0, 1013, 1005, 1018, 996, 995, 1061, 979, 1013, 9, 0, 0, 0, 0, 0, 0, 0, 983, 1054, 1020, 1059, 1015, 1042, 1080, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8269, 8178, 8181, 8240, 8090, 8089, 8308] avg=8192.0000 max=8308 max_rank=7 min=8089 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1067, 1045, 969, 1029, 982, 1022, 1046, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1032, 1030, 1036, 1040, 1013, 1003, 1044, 4, 0, 0, 0, 0, 0, 0, 0, 1011, 1034, 1042, 1016, 1002, 1033, 1046, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1066, 1033, 995, 995, 984, 1030, 993, 2, 1, 0, 0, 0, 0, 0, 0, 1035, 1051, 1048, 1018, 1012, 977, 1038, 1093, 30, 0, 0, 0, 0, 0, 0, 0, 1050, 1028, 1064, 1031, 1031, 1025, 1014, 988, 13, 10, 0, 0, 0, 0, 0, 0, 1040, 1001, 1010, 1003, 1049, 1004, 976, 1026, 9, 0, 0, 0, 0, 0, 0, 0, 1062, 1077, 1024, 1049, 961, 999, 1027, 978, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8203, 8160, 8140, 8302, 8254, 8118, 8177] avg=8192.0000 max=8302 max_rank=4 min=8118 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1018, 1071, 1000, 1009, 1060, 1024, 1017, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 970, 1044, 969, 1075, 1010, 1015, 1103, 4, 0, 0, 0, 0, 0, 0, 0, 978, 1046, 1031, 1028, 1061, 982, 1026, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1034, 1019, 1012, 1012, 984, 984, 1045, 2, 1, 0, 0, 0, 0, 0, 0, 1033, 1033, 1034, 987, 997, 1016, 1032, 1038, 32, 0, 0, 0, 0, 0, 0, 0, 1020, 1010, 1000, 1029, 1029, 1015, 1061, 1018, 13, 10, 0, 0, 0, 0, 0, 0, 1001, 1047, 989, 1026, 1010, 1040, 1020, 1057, 8, 0, 0, 0, 0, 0, 0, 0, 1040, 1014, 1053, 1010, 1050, 1040, 1032, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8211, 8187, 8083, 8202, 8205, 8198, 8259] avg=8192.0000 max=8259 max_rank=7 min=8083 min_rank=3 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1011, 1007, 1030, 1021, 1048, 1030, 1038, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1063, 999, 1024, 965, 974, 1068, 1059, 4, 0, 0, 0, 0, 0, 0, 0, 1032, 1015, 1051, 1042, 1014, 1004, 1040, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1050, 1023, 1035, 983, 988, 994, 1044, 2, 1, 0, 0, 0, 0, 0, 0, 992, 1012, 975, 1028, 1031, 995, 1052, 1046, 30, 0, 0, 0, 0, 0, 0, 0, 1024, 950, 1037, 1003, 1027, 1081, 1045, 995, 13, 10, 0, 0, 0, 0, 0, 0, 1045, 1014, 1028, 1074, 1024, 1059, 970, 1015, 9, 0, 0, 0, 0, 0, 0, 0, 1076, 1009, 994, 1016, 973, 1031, 1032, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8174, 8238, 8168, 8161, 8185, 8238, 8172] avg=8192.0000 max=8238 max_rank=2 min=8161 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1080, 1028, 982, 1030, 1001, 999, 1056, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1009, 1023, 997, 1004, 1030, 1013, 1012, 4, 0, 0, 0, 0, 0, 0, 0, 1071, 1013, 1037, 992, 1053, 1024, 985, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 1022, 1010, 1035, 1012, 1046, 1005, 1036, 2, 1, 0, 0, 0, 0, 0, 0, 984, 1046, 1069, 1027, 977, 1002, 959, 992, 31, 0, 0, 0, 0, 0, 0, 0, 1018, 1024, 1023, 1026, 1008, 1005, 1024, 1060, 14, 10, 0, 0, 0, 0, 0, 0, 1051, 991, 1009, 1047, 1081, 985, 1034, 1033, 9, 0, 0, 0, 0, 0, 0, 0, 1060, 1046, 994, 985, 1017, 981, 1033, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8147, 8166, 8294, 8087, 8212, 8240, 8165] avg=8192.0000 max=8294 max_rank=3 min=8087 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1023, 1053, 1078, 1041, 1030, 1023, 1024, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1026, 1015, 1031, 1014, 1038, 998, 998, 4, 0, 0, 0, 0, 0, 0, 0, 1029, 1030, 1024, 1055, 996, 1002, 1021, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1029, 1053, 1043, 1004, 1001, 1015, 1009, 2, 1, 0, 0, 0, 0, 0, 0, 977, 984, 1005, 1032, 1040, 997, 1025, 1041, 31, 0, 0, 0, 0, 0, 0, 0, 1008, 1023, 1031, 1057, 1039, 1009, 1021, 1037, 13, 10, 0, 0, 0, 0, 0, 0, 1020, 1063, 1009, 1056, 1000, 1050, 1008, 1032, 9, 0, 0, 0, 0, 0, 0, 0, 972, 987, 1015, 1009, 989, 1051, 1046, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8165, 8196, 8155, 8132, 8248, 8247, 8126] avg=8192.0000 max=8267 max_rank=0 min=8126 min_rank=7 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1004, 1037, 1040, 1020, 1009, 1016, 991, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1082, 1028, 1013, 1010, 1009, 1013, 996, 9, 0, 0, 0, 0, 0, 0, 0, 970, 995, 984, 974, 1029, 1048, 1032, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1038, 1057, 1021, 1096, 1046, 1033, 1043, 5, 8, 0, 0, 0, 0, 0, 0, 1044, 1030, 1019, 1025, 1046, 1003, 985, 1057, 34, 0, 0, 0, 0, 0, 0, 0, 1014, 995, 1014, 977, 1037, 1029, 1030, 1019, 17, 13, 0, 0, 0, 0, 0, 0, 1047, 1014, 1020, 1021, 1040, 1069, 1010, 1055, 12, 0, 0, 0, 0, 0, 0, 0, 1048, 1014, 1051, 1036, 952, 990, 1013, 972, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8180, 8065, 8343, 8243, 8145, 8288, 8076] avg=8192.0000 max=8343 max_rank=3 min=8065 min_rank=2 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8064, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8448, 8320, 8192, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8320, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8064, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1095, 1041, 1066, 952, 1014, 990, 1093, 1013, 1018, 988, 1020, 1008, 1057, 1043, 1056, 1008, 1023, 1078, 985, 1009, 1061, 985, 992, 1022, 1021, 1051, 1037, 988, 988, 983, 1037, 1004, 1053, 978, 1028, 1009, 1049, 1005, 1004, 1014, 1040, 998, 1018, 1048, 1024, 1054, 1028, 993, 1029, 998, 988, 1074, 966, 1007, 1043, 982, 999, 1039, 1042, 1065, 1098, 1048, 1023, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1073, 974, 1076, 1022, 1043, 1005, 1017, 1074, 1045, 1030, 1022, 982, 1006, 1022, 1042, 1045, 1016, 1025, 1027, 1041, 1054, 1041, 1015, 1011, 1046, 958, 980, 978, 1053, 1065, 969, 1048, 1030, 1004, 1029, 1012, 1019, 1031, 1076, 1009, 1010, 1058, 1058, 1015, 1039, 1020, 1051, 1001, 987, 1022, 975, 1019, 1072, 969, 981, 1096, 991, 1012, 1005, 1068, 1038, 1000, 1013, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1018, 1015, 1012, 1014, 1066, 994, 1000, 1032, 1008, 1046, 984, 1008, 1031, 1079, 1037, 1037, 1021, 1030, 1063, 1056, 1003, 1039, 979, 1074, 1000, 1042, 1042, 982, 1030, 1025, 1033, 1061, 1028, 1027, 1033, 989, 1010, 985, 1031, 1020, 1032, 1057, 1010, 1001, 1070, 1021, 1048, 1052, 995, 991, 1062, 979, 1063, 985, 1079, 994, 1012, 1025, 987, 1031, 1018, 1017, 987, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1030, 1056, 1022, 1023, 1055, 1033, 1078, 1020, 1018, 1046, 1075, 1044, 992, 1017, 1057, 1022, 1000, 1003, 1031, 1017, 1054, 996, 994, 992, 1041, 1037, 983, 1054, 1011, 1054, 1053, 1023, 972, 1009, 1072, 987, 1028, 1022, 1017, 1069, 1025, 1027, 984, 1017, 1060, 1083, 1033, 1021, 1075, 1016, 1007, 1000, 1043, 1021, 1015, 1031, 970, 976, 987, 1002, 1001, 987, 1001, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1009, 1009, 1021, 1022, 1049, 988, 1044, 1090, 1016, 987, 1038, 970, 1021, 1023, 996, 1053, 981, 1018, 977, 1058, 995, 995, 1006, 1024, 1036, 1056, 1067, 1043, 993, 1066, 1043, 996, 1024, 990, 1068, 1048, 998, 1038, 984, 1074, 1038, 1006, 1042, 1037, 1065, 1020, 1020, 1030, 986, 1053, 1032, 1010, 1066, 1000, 1038, 1018, 1020, 986, 1031, 1047, 1007, 1026, 1020, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1071, 1031, 991, 1064, 1091, 986, 1026, 1037, 1044, 1042, 1060, 1045, 1004, 1041, 1042, 1014, 1051, 1042, 1005, 1044, 976, 1008, 995, 1004, 1045, 1024, 1027, 1019, 1099, 1004, 1029, 956, 1038, 1006, 1020, 1000, 1073, 1033, 973, 1004, 1011, 1006, 1031, 1010, 1022, 1033, 1015, 1050, 969, 985, 1043, 1000, 1002, 1044, 1002, 1029, 1064, 1009, 974, 1042, 1054, 1059, 977, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1036, 1014, 998, 1028, 1043, 1021, 1025, 1014, 1056, 1082, 1028, 1028, 1088, 1012, 975, 997, 1012, 1032, 1023, 1023, 977, 1064, 1003, 1036, 1001, 994, 981, 1027, 1029, 989, 986, 1081, 983, 1002, 1011, 993, 1061, 1022, 1058, 1036, 1064, 1050, 1006, 989, 979, 1017, 990, 1045, 1038, 1034, 1031, 991, 1003, 1011, 1047, 1043, 1052, 1041, 1029, 1036, 1006, 1067, 1034, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1029, 980, 1001, 992, 1112, 1015, 1029, 963, 1039, 1029, 1011, 966, 1036, 1020, 1027, 1057, 1026, 995, 1018, 1034, 1063, 1016, 1042, 1033, 1061, 1049, 1012, 1031, 967, 1018, 1010, 1054, 1054, 1038, 1020, 978, 988, 1065, 983, 997, 1008, 1034, 1005, 1061, 1049, 1026, 1033, 1020, 1034, 1061, 1040, 1017, 1047, 1006, 1064, 1020, 974, 963, 1071, 1071, 1032, 1040, 996, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1018, 1015, 1012, 1014, 1028, 994, 1000, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1046, 984, 1008, 1031, 1054, 1037, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1030, 1063, 1056, 1003, 1039, 979, 1074, 18, 10, 0, 0, 0, 0, 0, 0, 1000, 1042, 1042, 982, 1030, 1025, 1033, 1061, 8, 0, 0, 0, 0, 0, 0, 0, 1028, 1027, 1033, 989, 1010, 985, 1031, 1020, 28, 0, 0, 0, 0, 0, 0, 0, 1032, 1057, 1010, 1001, 1036, 1021, 1048, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 995, 991, 1062, 979, 1063, 985, 1079, 994, 16, 10, 0, 0, 0, 0, 0, 0, 1012, 1025, 987, 1031, 1018, 1017, 987, 1036, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8205, 8293, 8223, 8151, 8257, 8174, 8120] avg=8192.0000 max=8293 max_rank=2 min=8113 min_rank=0 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1030, 1056, 1022, 1023, 1017, 1033, 1078, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1046, 1075, 1044, 992, 993, 1057, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1003, 1031, 1017, 1054, 996, 994, 992, 17, 10, 0, 0, 0, 0, 0, 0, 1041, 1037, 983, 1054, 1011, 1054, 1053, 1023, 8, 0, 0, 0, 0, 0, 0, 0, 972, 1009, 1072, 987, 1028, 1022, 1017, 1069, 28, 0, 0, 0, 0, 0, 0, 0, 1025, 1027, 984, 1017, 1027, 1083, 1033, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1016, 1007, 1000, 1043, 1021, 1015, 1031, 16, 9, 0, 0, 0, 0, 0, 0, 970, 976, 987, 1002, 1001, 987, 1001, 1047, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8247, 8114, 8264, 8204, 8217, 8233, 7978] avg=8192.0000 max=8279 max_rank=0 min=7978 min_rank=7 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1073, 974, 1076, 1022, 1006, 1005, 1017, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1030, 1022, 982, 1006, 998, 1042, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1025, 1027, 1041, 1054, 1041, 1015, 1011, 17, 10, 0, 0, 0, 0, 0, 0, 1046, 958, 980, 978, 1053, 1065, 969, 1048, 8, 0, 0, 0, 0, 0, 0, 0, 1030, 1004, 1029, 1012, 1019, 1031, 1076, 1009, 27, 0, 0, 0, 0, 0, 0, 0, 1010, 1058, 1058, 1015, 1006, 1020, 1051, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1022, 975, 1019, 1072, 969, 981, 1096, 16, 9, 0, 0, 0, 0, 0, 0, 991, 1012, 1005, 1068, 1038, 1000, 1013, 1021, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8170, 8257, 8105, 8237, 8219, 8146, 8155] avg=8192.0000 max=8257 max_rank=2 min=8105 min_rank=3 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1095, 1041, 1066, 952, 978, 990, 1093, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 988, 1020, 1008, 1057, 1019, 1056, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1078, 985, 1009, 1061, 985, 992, 1022, 17, 9, 0, 0, 0, 0, 0, 0, 1021, 1051, 1037, 988, 988, 983, 1037, 1004, 8, 0, 0, 0, 0, 0, 0, 0, 1053, 978, 1028, 1009, 1049, 1005, 1004, 1014, 27, 0, 0, 0, 0, 0, 0, 0, 1040, 998, 1018, 1048, 991, 1054, 1028, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 998, 988, 1074, 966, 1007, 1043, 982, 16, 9, 0, 0, 0, 0, 0, 0, 999, 1039, 1042, 1065, 1098, 1048, 1023, 1066, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8174, 8181, 8117, 8167, 8170, 8112, 8387] avg=8192.0000 max=8387 max_rank=7 min=8112 min_rank=6 imbalance_ratio=1.023804 max_load_reduction=57149 max_load_reduction_pct=0.872025 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1009, 1009, 1021, 1022, 1011, 988, 1044, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 987, 1038, 970, 1021, 999, 996, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1018, 977, 1058, 995, 995, 1006, 1024, 18, 10, 0, 0, 0, 0, 0, 0, 1036, 1056, 1067, 1043, 993, 1066, 1043, 996, 8, 0, 0, 0, 0, 0, 0, 0, 1024, 990, 1068, 1048, 998, 1038, 984, 1074, 28, 0, 0, 0, 0, 0, 0, 0, 1038, 1006, 1042, 1037, 1031, 1020, 1020, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1053, 1032, 1010, 1066, 1000, 1038, 1018, 16, 9, 0, 0, 0, 0, 0, 0, 1020, 986, 1031, 1047, 1007, 1026, 1020, 1024, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8080, 8082, 8308, 8252, 8224, 8228, 8168] avg=8192.0000 max=8308 max_rank=3 min=8080 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1071, 1031, 991, 1064, 1052, 986, 1026, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1042, 1060, 1045, 1004, 1017, 1042, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1042, 1005, 1044, 976, 1008, 995, 1004, 17, 10, 0, 0, 0, 0, 0, 0, 1045, 1024, 1027, 1019, 1099, 1004, 1029, 956, 8, 0, 0, 0, 0, 0, 0, 0, 1038, 1006, 1020, 1000, 1073, 1033, 973, 1004, 29, 0, 0, 0, 0, 0, 0, 0, 1011, 1006, 1031, 1010, 989, 1033, 1015, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 969, 985, 1043, 1000, 1002, 1044, 1002, 1029, 16, 9, 0, 0, 0, 0, 0, 0, 1064, 1009, 974, 1042, 1054, 1059, 977, 1041, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8268, 8152, 8211, 8176, 8145, 8099, 8227] avg=8192.0000 max=8268 max_rank=1 min=8099 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1029, 980, 1001, 992, 1073, 1015, 1029, 963, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1029, 1011, 966, 1036, 996, 1027, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 995, 1018, 1034, 1063, 1016, 1042, 1033, 17, 10, 0, 0, 0, 0, 0, 0, 1061, 1049, 1012, 1031, 967, 1018, 1010, 1054, 8, 0, 0, 0, 0, 0, 0, 0, 1054, 1038, 1020, 978, 988, 1065, 983, 997, 29, 0, 0, 0, 0, 0, 0, 0, 1008, 1034, 1005, 1061, 1016, 1026, 1033, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1061, 1040, 1017, 1047, 1006, 1064, 1020, 16, 9, 0, 0, 0, 0, 0, 0, 974, 963, 1071, 1071, 1032, 1040, 996, 1006, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8161, 8254, 8210, 8152, 8203, 8314, 8160] avg=8192.0000 max=8314 max_rank=6 min=8082 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1036, 1014, 998, 1028, 999, 1021, 1025, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1082, 1028, 1028, 1088, 977, 975, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1032, 1023, 1023, 977, 1064, 1003, 1036, 20, 13, 0, 0, 0, 0, 0, 0, 1001, 994, 981, 1027, 1029, 989, 986, 1081, 10, 0, 0, 0, 0, 0, 0, 0, 983, 1002, 1011, 993, 1061, 1022, 1058, 1036, 31, 0, 0, 0, 0, 0, 0, 0, 1064, 1050, 1006, 989, 940, 1017, 990, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1034, 1031, 991, 1003, 1011, 1047, 1043, 19, 13, 0, 0, 0, 0, 0, 0, 1052, 1041, 1029, 1036, 1006, 1067, 1034, 1064, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8231, 8203, 8098, 8197, 8101, 8230, 8341] avg=8192.0000 max=8341 max_rank=7 min=8098 min_rank=3 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8064, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8064, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8064, 8320, 8320, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8448, 8192, 8192, 8192, 8192, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8064, 8320, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8448, 8320, 8320, 8064, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8064, 8320, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1045, 1062, 984, 1034, 1045, 1001, 1051, 1036, 1102, 1016, 1000, 996, 996, 1011, 1055, 1007, 1044, 1045, 1074, 1073, 1006, 1056, 1048, 1029, 995, 999, 1019, 1024, 985, 1008, 1053, 995, 1003, 1021, 1032, 1017, 1033, 991, 1000, 997, 995, 1046, 996, 1063, 1006, 1026, 1036, 982, 1041, 1072, 1037, 1022, 1005, 1056, 1000, 1089, 951, 1021, 999, 1049, 1003, 1005, 1020, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1004, 995, 1005, 994, 1011, 1056, 989, 997, 1000, 986, 1082, 1050, 1068, 1015, 999, 965, 1044, 990, 989, 1036, 1021, 1029, 1015, 1021, 1078, 1034, 1050, 1021, 1014, 983, 999, 1058, 1011, 1010, 1064, 1025, 1003, 1035, 1032, 1048, 997, 996, 1042, 1058, 963, 1017, 1057, 1025, 1065, 1031, 1037, 1035, 1052, 1052, 1065, 1012, 1035, 1035, 1015, 1015, 1035, 1059, 1002, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1052, 1059, 1018, 1039, 1053, 1008, 1007, 1011, 988, 1035, 998, 1046, 1058, 1002, 1029, 1016, 1061, 1043, 1017, 978, 1020, 1020, 1048, 1018, 996, 1017, 1004, 1025, 1049, 986, 1011, 1003, 1002, 1026, 1023, 1062, 1026, 1000, 1088, 1091, 1014, 1018, 996, 1066, 1040, 958, 1073, 972, 1062, 1067, 1004, 1017, 996, 1053, 995, 1028, 988, 996, 977, 1037, 1071, 1070, 976, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1032, 994, 1060, 1020, 1039, 1014, 1024, 1052, 1030, 1010, 1045, 1031, 1006, 1056, 1022, 1041, 1035, 1013, 1018, 1019, 1015, 1002, 993, 1017, 1022, 981, 1086, 1048, 981, 1070, 1062, 948, 1034, 1014, 1046, 1016, 1055, 986, 1006, 1009, 997, 1008, 978, 1048, 1022, 961, 1009, 1052, 1004, 1033, 1051, 969, 1005, 1100, 1054, 1035, 1006, 1078, 1059, 1024, 1052, 1046, 979, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1045, 1014, 1008, 1008, 996, 1011, 1030, 1051, 1035, 1021, 993, 1033, 1027, 999, 1062, 1037, 1046, 1005, 988, 1044, 1023, 1061, 1054, 1042, 1055, 1028, 994, 973, 1052, 1011, 984, 1008, 1025, 1015, 983, 1011, 1033, 1019, 1049, 1002, 1091, 1002, 1020, 984, 1060, 1006, 1030, 1052, 1026, 1007, 1032, 1038, 1020, 1017, 1030, 1035, 1032, 1019, 1020, 1057, 1014, 1034, 1015, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1004, 1033, 1014, 989, 1020, 1002, 1021, 1018, 1002, 1038, 1019, 1007, 1044, 983, 1054, 1002, 1024, 1008, 985, 1018, 1025, 1044, 1005, 1001, 1005, 1047, 995, 1032, 1025, 1032, 1007, 1003, 1039, 1029, 1030, 1038, 1016, 1057, 1060, 1026, 1080, 1041, 995, 1004, 1045, 1017, 1059, 999, 990, 1033, 1055, 1086, 1055, 996, 1056, 988, 1017, 1038, 1049, 982, 1052, 1035, 1065, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1071, 1058, 1069, 997, 1035, 997, 1025, 1003, 992, 996, 1041, 1025, 987, 982, 1014, 1003, 1021, 1047, 1017, 1075, 1000, 1022, 1046, 1047, 1010, 1060, 1027, 1008, 1008, 1032, 1045, 1020, 1002, 1049, 1055, 1029, 1066, 992, 1033, 1024, 1015, 997, 978, 1045, 984, 1008, 1053, 1003, 1028, 1013, 1042, 981, 1002, 1054, 1012, 1044, 1054, 1069, 1019, 1042, 1011, 1062, 991, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1017, 1080, 1001, 1000, 1045, 1039, 1011, 1025, 1007, 1052, 1057, 1042, 1036, 1042, 1036, 1070, 1001, 1040, 1103, 1011, 1025, 978, 942, 1062, 1040, 1029, 1021, 1065, 1079, 990, 1047, 1025, 1044, 1016, 1041, 1077, 1018, 1048, 1002, 967, 1060, 981, 1004, 1026, 974, 985, 998, 1011, 1055, 1020, 1032, 1056, 1024, 1021, 1032, 1012, 1067, 1054, 991, 996, 956, 1004, 949, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1045, 1062, 984, 1034, 1045, 1001, 1051, 1036, 1, 0, 0, 0, 0, 0, 0, 0, 1102, 1016, 1000, 996, 996, 1011, 1055, 1007, 4, 0, 0, 0, 0, 0, 0, 0, 1031, 1045, 1074, 1073, 1006, 1056, 1048, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 995, 999, 1019, 1024, 985, 1008, 1053, 995, 11, 10, 0, 0, 0, 0, 0, 0, 1003, 1021, 1032, 998, 1033, 991, 1000, 997, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1046, 996, 1063, 1006, 1026, 1036, 982, 52, 0, 0, 0, 0, 0, 0, 0, 1041, 1072, 1037, 1022, 1005, 993, 1000, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1021, 999, 1049, 1003, 1005, 1020, 1028, 9, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8187, 8362, 8099, 8075, 8202, 8259, 8093] avg=8192.0000 max=8362 max_rank=2 min=8075 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1004, 995, 1005, 994, 1011, 1056, 989, 997, 1, 0, 0, 0, 0, 0, 0, 0, 1000, 986, 1082, 1050, 1068, 1015, 999, 965, 4, 0, 0, 0, 0, 0, 0, 0, 1031, 990, 989, 1036, 1021, 1029, 1015, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1034, 1050, 1021, 1014, 983, 999, 1058, 11, 10, 0, 0, 0, 0, 0, 0, 1011, 1010, 1064, 1006, 1003, 1035, 1032, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 997, 996, 1042, 1058, 963, 1017, 1057, 1025, 51, 0, 0, 0, 0, 0, 0, 0, 1065, 1031, 1037, 1035, 1052, 990, 1065, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1035, 1015, 1015, 1035, 1059, 1002, 1010, 9, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8052, 8169, 8132, 8258, 8209, 8206, 8287, 8223] avg=8192.0000 max=8287 max_rank=6 min=8052 min_rank=0 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32925 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1004, 1033, 1014, 989, 1020, 1002, 1021, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1038, 1019, 1007, 1044, 983, 1054, 1002, 4, 0, 0, 0, 0, 0, 0, 0, 1012, 1008, 985, 1018, 1025, 1044, 1005, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1047, 995, 1032, 1025, 1032, 1007, 1003, 10, 10, 0, 0, 0, 0, 0, 0, 1039, 1029, 1030, 1019, 1016, 1057, 1060, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 1041, 995, 1004, 1045, 1017, 1059, 999, 49, 0, 0, 0, 0, 0, 0, 0, 990, 1033, 1055, 1086, 1055, 937, 1056, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1038, 1049, 982, 1052, 1035, 1065, 998, 9, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8153, 8098, 8166, 8276, 8289, 8200, 8253] avg=8192.0000 max=8289 max_rank=5 min=8098 min_rank=2 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=33018 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1052, 1059, 1018, 1039, 1053, 1008, 1007, 1011, 1, 0, 0, 0, 0, 0, 0, 0, 988, 1035, 998, 1046, 1058, 1002, 1029, 1016, 4, 0, 0, 0, 0, 0, 0, 0, 1048, 1043, 1017, 978, 1020, 1020, 1048, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1017, 1004, 1025, 1049, 986, 1011, 1003, 11, 11, 0, 0, 0, 0, 0, 0, 1002, 1026, 1023, 1041, 1026, 1000, 1088, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1018, 996, 1066, 1040, 958, 1073, 972, 51, 0, 0, 0, 0, 0, 0, 0, 1062, 1067, 1004, 1017, 996, 991, 995, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 988, 996, 977, 1037, 1071, 1070, 976, 1029, 10, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8176, 8192, 8113, 8297, 8188, 8160, 8162] avg=8192.0000 max=8297 max_rank=4 min=8113 min_rank=3 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1045, 1014, 1008, 1008, 996, 1011, 1030, 1051, 1, 0, 0, 0, 0, 0, 0, 0, 1035, 1021, 993, 1033, 1027, 999, 1062, 1037, 4, 0, 0, 0, 0, 0, 0, 0, 1033, 1005, 988, 1044, 1023, 1061, 1054, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1028, 994, 973, 1052, 1011, 984, 1008, 10, 10, 0, 0, 0, 0, 0, 0, 1025, 1015, 983, 992, 1033, 1019, 1049, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 1002, 1020, 984, 1060, 1006, 1030, 1052, 50, 0, 0, 0, 0, 0, 0, 0, 1026, 1007, 1032, 1038, 1020, 957, 1030, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1019, 1020, 1057, 1014, 1034, 1015, 1020, 9, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8211, 8250, 8125, 8118, 8295, 8145, 8228] avg=8192.0000 max=8295 max_rank=5 min=8118 min_rank=4 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1032, 994, 1060, 1020, 1039, 1014, 1024, 1052, 1, 0, 0, 0, 0, 0, 0, 0, 1030, 1010, 1045, 1031, 1006, 1056, 1022, 1041, 4, 0, 0, 0, 0, 0, 0, 0, 1022, 1013, 1018, 1019, 1015, 1002, 993, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 981, 1086, 1048, 981, 1070, 1062, 948, 11, 10, 0, 0, 0, 0, 0, 0, 1034, 1014, 1046, 997, 1055, 986, 1006, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1008, 978, 1048, 1022, 961, 1009, 1052, 54, 0, 0, 0, 0, 0, 0, 0, 1004, 1033, 1051, 969, 1005, 1035, 1054, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1078, 1059, 1024, 1052, 1046, 979, 1014, 9, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8245, 8099, 8219, 8147, 8129, 8186, 8275] avg=8192.0000 max=8275 max_rank=7 min=8099 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1071, 1058, 1069, 997, 1035, 997, 1025, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 992, 996, 1041, 1025, 987, 982, 1014, 1003, 4, 0, 0, 0, 0, 0, 0, 0, 1009, 1047, 1017, 1075, 1000, 1022, 1046, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1060, 1027, 1008, 1008, 1032, 1045, 1020, 11, 10, 0, 0, 0, 0, 0, 0, 1002, 1049, 1055, 1010, 1066, 992, 1033, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 997, 978, 1045, 984, 1008, 1053, 1003, 52, 0, 0, 0, 0, 0, 0, 0, 1028, 1013, 1042, 981, 1002, 991, 1012, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1069, 1019, 1042, 1011, 1062, 991, 999, 9, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8044, 8263, 8231, 8231, 8135, 8113, 8264] avg=8192.0000 max=8264 max_rank=7 min=8044 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1017, 1080, 1001, 1000, 1045, 1039, 1011, 1025, 3, 0, 0, 0, 0, 0, 0, 0, 1007, 1052, 1057, 1042, 1036, 1042, 1036, 1070, 9, 0, 0, 0, 0, 0, 0, 0, 979, 1040, 1103, 1011, 1025, 978, 942, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1029, 1021, 1065, 1079, 990, 1047, 1025, 13, 16, 0, 0, 0, 0, 0, 0, 1044, 1016, 1041, 1047, 1018, 1048, 1002, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 981, 1004, 1026, 974, 985, 998, 1011, 53, 0, 0, 0, 0, 0, 0, 0, 1055, 1020, 1032, 1056, 1024, 955, 1032, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1054, 991, 996, 956, 1004, 949, 997, 14, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8351, 8140, 8325, 8183, 8092, 8186, 8038] avg=8192.0000 max=8351 max_rank=1 min=8038 min_rank=7 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 7936, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8064, 8320, 8192, 8064, 8320, 8192, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8320, 8064, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[988, 977, 989, 1040, 1017, 1010, 970, 1061, 1017, 1078, 1008, 1037, 1049, 1073, 1027, 1009, 1104, 1017, 1043, 1007, 1021, 1032, 1026, 1046, 1032, 1018, 1001, 1018, 1006, 1034, 1053, 1031, 1039, 1048, 1043, 1051, 969, 1025, 1052, 1014, 973, 1054, 987, 1060, 1020, 1013, 986, 1059, 986, 1009, 974, 1026, 1006, 1025, 1006, 985, 1007, 1062, 998, 1009, 1088, 1023, 1070, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1022, 1001, 1056, 995, 1001, 1052, 1039, 1048, 1040, 1002, 1054, 1001, 1002, 995, 1021, 1061, 989, 1020, 1026, 1057, 1021, 1069, 1057, 1022, 1011, 954, 1075, 1048, 966, 1051, 993, 1001, 1080, 1019, 1023, 1020, 1079, 1042, 1025, 1003, 1057, 986, 1041, 1017, 995, 1064, 996, 970, 1010, 1054, 1010, 1067, 1049, 975, 990, 1044, 1035, 1001, 1073, 981, 1006, 1017, 1018, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1018, 1037, 1031, 1021, 1013, 1019, 1028, 1045, 1057, 1004, 1036, 1036, 1021, 1025, 1046, 1027, 1030, 1045, 1034, 1056, 1026, 1044, 1020, 1014, 1055, 1012, 974, 1061, 1021, 1003, 1113, 1013, 1039, 977, 1039, 989, 1026, 985, 1035, 1079, 1017, 1068, 1038, 1059, 978, 1005, 995, 973, 1055, 1007, 967, 1007, 956, 986, 984, 1072, 958, 1060, 989, 1035, 1071, 1049, 1036, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1038, 1079, 1035, 1061, 1016, 1034, 990, 1028, 1013, 961, 1005, 1025, 1045, 993, 998, 1031, 1028, 1008, 1052, 1002, 1025, 1051, 1023, 1039, 1026, 1020, 1020, 1039, 1003, 1018, 1041, 1003, 1005, 1062, 1023, 1004, 1007, 1109, 1058, 999, 1023, 1066, 1067, 1057, 1010, 1093, 1057, 1012, 1049, 978, 1009, 1022, 999, 1031, 1009, 984, 966, 987, 995, 1047, 1027, 1012, 1024, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[999, 974, 1023, 1050, 990, 1109, 1011, 983, 1038, 974, 1036, 1011, 1041, 1069, 988, 970, 972, 1015, 1041, 1056, 1028, 997, 1071, 1047, 1078, 1051, 979, 1065, 998, 1042, 1034, 1005, 1030, 1016, 1073, 1046, 1028, 1021, 1026, 1028, 1023, 1043, 1031, 1065, 980, 1020, 1017, 1009, 1058, 1014, 1025, 1058, 1027, 1058, 1044, 1026, 989, 1014, 1021, 976, 1029, 965, 1006, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1016, 1002, 976, 979, 1026, 1009, 1036, 1001, 1005, 988, 1008, 1036, 1004, 1037, 1103, 1044, 1047, 1023, 1042, 1020, 1018, 1038, 1052, 1017, 1030, 1031, 1009, 1061, 1052, 1059, 1026, 1040, 1070, 1016, 1052, 1034, 1001, 983, 1006, 986, 1004, 999, 1051, 1050, 1019, 1053, 988, 1012, 1051, 1034, 1017, 1074, 993, 1031, 1065, 1040, 997, 1011, 1049, 1020, 988, 1003, 992, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1002, 1054, 1013, 1046, 1042, 1009, 1076, 971, 1037, 1094, 1020, 1013, 1011, 945, 986, 1015, 980, 1005, 997, 1045, 1003, 1028, 1022, 1007, 1011, 1040, 991, 1063, 1005, 1067, 1040, 1030, 1039, 1066, 1011, 1048, 1032, 993, 1030, 1048, 993, 1034, 1013, 1043, 1046, 1051, 1038, 1022, 1032, 998, 1009, 1034, 1051, 1036, 996, 1052, 1000, 1007, 1042, 1007, 1006, 1029, 1015, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[994, 1001, 1000, 1003, 1039, 1044, 1013, 1012, 984, 1068, 1036, 994, 1031, 1013, 966, 1001, 999, 1001, 1003, 1014, 1032, 1005, 1009, 1036, 1021, 1033, 1029, 1039, 1023, 1049, 1023, 1002, 1015, 1024, 1063, 1045, 1065, 963, 1030, 983, 1027, 1018, 1072, 1022, 1021, 1032, 1044, 1082, 1052, 1055, 1027, 1014, 1059, 1085, 1024, 977, 1037, 1013, 1079, 1017, 1016, 965, 1047, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[988, 977, 989, 1040, 1017, 1010, 970, 1061, 20, 13, 0, 0, 0, 0, 0, 0, 1017, 1078, 1008, 1037, 1049, 1073, 1027, 1009, 11, 9, 0, 0, 0, 0, 0, 0, 1104, 1017, 1043, 1007, 1021, 1032, 1004, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1018, 1001, 994, 1006, 1034, 1053, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1048, 1006, 1051, 969, 1025, 1052, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1054, 987, 1038, 1020, 1013, 986, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1009, 974, 1026, 1006, 1025, 1006, 985, 11, 0, 0, 0, 0, 0, 0, 0, 1007, 1062, 998, 1009, 1088, 1023, 1070, 1030, 37, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8085, 8318, 8274, 8169, 8204, 8130, 8028, 8328] avg=8192.0000 max=8328 max_rank=7 min=8028 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1002, 1054, 1013, 1046, 1042, 1009, 1076, 971, 21, 13, 0, 0, 0, 0, 0, 0, 1037, 1094, 1020, 1013, 1011, 945, 986, 1015, 11, 9, 0, 0, 0, 0, 0, 0, 980, 1005, 997, 1045, 1003, 1028, 1000, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1040, 991, 1038, 1005, 1067, 1040, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1066, 975, 1048, 1032, 993, 1030, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1034, 1013, 1021, 1046, 1051, 1038, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 998, 1009, 1034, 1051, 1036, 996, 1052, 11, 0, 0, 0, 0, 0, 0, 0, 1000, 1007, 1042, 1007, 1006, 1029, 1015, 1047, 36, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8141, 8065, 8222, 8231, 8218, 8219, 8193] avg=8192.0000 max=8247 max_rank=0 min=8065 min_rank=2 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32861 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[999, 974, 1023, 1050, 990, 1109, 1011, 983, 21, 13, 0, 0, 0, 0, 0, 0, 1038, 974, 1036, 1011, 1041, 1069, 988, 970, 11, 10, 0, 0, 0, 0, 0, 0, 972, 1015, 1041, 1056, 1028, 997, 1048, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1051, 979, 1040, 998, 1042, 1034, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1016, 1034, 1046, 1028, 1021, 1026, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1043, 1031, 1043, 980, 1020, 1017, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1014, 1025, 1058, 1027, 1058, 1044, 1026, 11, 0, 0, 0, 0, 0, 0, 0, 989, 1014, 1021, 976, 1029, 965, 1006, 1025, 39, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8148, 8204, 8227, 8229, 8166, 8321, 8068] avg=8192.0000 max=8321 max_rank=6 min=8068 min_rank=7 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1022, 1001, 1056, 995, 1001, 1052, 1039, 1048, 20, 13, 0, 0, 0, 0, 0, 0, 1040, 1002, 1054, 1001, 1002, 995, 1021, 1061, 11, 10, 0, 0, 0, 0, 0, 0, 989, 1020, 1026, 1057, 1021, 1069, 1034, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 954, 1075, 1024, 966, 1051, 993, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 1019, 986, 1020, 1079, 1042, 1025, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 986, 1041, 996, 995, 1064, 996, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1054, 1010, 1067, 1049, 975, 990, 1044, 10, 0, 0, 0, 0, 0, 0, 0, 1035, 1001, 1073, 981, 1006, 1017, 1018, 1039, 37, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8197, 8238, 8075, 8254, 8105, 8209, 8211] avg=8192.0000 max=8254 max_rank=4 min=8075 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1018, 1037, 1031, 1021, 1013, 1019, 1028, 1045, 20, 13, 0, 0, 0, 0, 0, 0, 1057, 1004, 1036, 1036, 1021, 1025, 1046, 1027, 11, 9, 0, 0, 0, 0, 0, 0, 1030, 1045, 1034, 1056, 1026, 1044, 998, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1012, 974, 1037, 1021, 1003, 1113, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 977, 1002, 989, 1026, 985, 1035, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1068, 1038, 1037, 978, 1005, 995, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1007, 967, 1007, 956, 986, 984, 1072, 11, 0, 0, 0, 0, 0, 0, 0, 958, 1060, 989, 1035, 1071, 1049, 1036, 1017, 37, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8272, 8247, 8228, 8132, 8111, 8045, 8256] avg=8192.0000 max=8272 max_rank=1 min=8045 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32544 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1038, 1079, 1035, 1061, 1016, 1034, 990, 1028, 20, 13, 0, 0, 0, 0, 0, 0, 1013, 961, 1005, 1025, 1045, 993, 998, 1031, 11, 9, 0, 0, 0, 0, 0, 0, 1028, 1008, 1052, 1002, 1025, 1051, 1001, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1020, 1020, 1015, 1003, 1018, 1041, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1062, 986, 1004, 1007, 1109, 1058, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1066, 1067, 1035, 1010, 1093, 1057, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 978, 1009, 1022, 999, 1031, 1009, 984, 11, 0, 0, 0, 0, 0, 0, 0, 966, 987, 995, 1047, 1027, 1012, 1024, 995, 37, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8314, 8091, 8206, 8146, 8230, 8363, 8092, 8094] avg=8192.0000 max=8363 max_rank=5 min=8091 min_rank=1 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1016, 1002, 976, 979, 1026, 1009, 1036, 1001, 20, 13, 0, 0, 0, 0, 0, 0, 1005, 988, 1008, 1036, 1004, 1037, 1103, 1044, 11, 10, 0, 0, 0, 0, 0, 0, 1047, 1023, 1042, 1020, 1018, 1038, 1029, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1031, 1009, 1037, 1052, 1059, 1026, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1016, 1014, 1034, 1001, 983, 1006, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 999, 1051, 1028, 1019, 1053, 988, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1034, 1017, 1074, 993, 1031, 1065, 1040, 11, 0, 0, 0, 0, 0, 0, 0, 997, 1011, 1049, 1020, 988, 1003, 992, 1012, 38, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8078, 8246, 8234, 8284, 8110, 8154, 8316, 8114] avg=8192.0000 max=8316 max_rank=6 min=8078 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[994, 1001, 1000, 1003, 1039, 1044, 1013, 1012, 24, 17, 0, 0, 0, 0, 0, 0, 984, 1068, 1036, 994, 1031, 1013, 966, 1001, 16, 14, 0, 0, 0, 0, 0, 0, 999, 1001, 1003, 1014, 1032, 1005, 978, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1033, 1029, 1005, 1023, 1049, 1023, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1024, 1021, 1045, 1065, 963, 1030, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1018, 1072, 993, 1021, 1032, 1044, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1055, 1027, 1014, 1059, 1085, 1024, 977, 13, 0, 0, 0, 0, 0, 0, 0, 1037, 1013, 1079, 1017, 1016, 965, 1047, 1046, 42, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8123, 8068, 8185, 8146, 8289, 8306, 8272] avg=8192.0000 max=8306 max_rank=6 min=8068 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=33013 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8320, 8192, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8448, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8448, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8192, 8320, 8192, 8320, 8064, 8320, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1042, 1080, 1042, 1017, 1063, 1020, 1022, 1068, 1027, 989, 1009, 1011, 1032, 1032, 1030, 977, 1035, 1023, 1032, 1065, 1010, 1032, 1028, 1052, 1044, 978, 1032, 998, 1011, 1021, 1036, 997, 1065, 1019, 1019, 1002, 1052, 1091, 989, 1023, 1014, 1010, 1034, 982, 989, 1045, 1014, 1015, 996, 1018, 1012, 1013, 1010, 1025, 995, 1054, 1019, 1062, 1011, 1052, 1003, 1019, 984, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1036, 1038, 1022, 1006, 965, 1050, 1014, 973, 1002, 1040, 1020, 1049, 1016, 1004, 990, 1030, 1043, 1006, 1038, 1018, 1024, 1032, 961, 1050, 1055, 1008, 1059, 1006, 1044, 1018, 1021, 1017, 1047, 1023, 1086, 959, 1074, 1002, 1008, 1048, 995, 1041, 1047, 1038, 964, 1034, 1039, 1025, 1000, 1056, 1049, 999, 1013, 1033, 1037, 1027, 1067, 982, 1083, 1022, 1023, 989, 1034, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1007, 996, 1050, 1043, 1049, 1031, 1018, 985, 999, 1015, 1041, 1018, 1014, 1007, 990, 955, 1050, 1036, 1075, 1022, 1010, 1054, 1030, 1088, 1018, 1010, 1008, 1031, 997, 1032, 995, 995, 1005, 1025, 1055, 1057, 996, 1054, 1029, 1042, 1021, 1082, 1031, 1060, 1035, 1013, 1031, 1000, 1004, 1045, 1018, 1015, 1031, 1081, 1012, 1000, 988, 1000, 1015, 1019, 1010, 1040, 1019, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1019, 1007, 1016, 1028, 1066, 1017, 991, 1036, 947, 1043, 1016, 1006, 1043, 1055, 1018, 980, 1012, 994, 991, 1024, 1029, 1045, 996, 1049, 1039, 1026, 1002, 1089, 1033, 1047, 1037, 1030, 1047, 1032, 1020, 1005, 1015, 1042, 1070, 1055, 986, 1040, 1054, 1030, 997, 945, 1008, 1049, 1022, 1046, 1021, 1011, 1024, 1054, 1031, 1048, 1012, 1048, 1041, 1062, 1040, 961, 985, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1015, 993, 1025, 1052, 1030, 941, 1040, 1087, 1052, 1029, 1064, 1041, 1032, 1015, 1047, 1006, 1017, 1018, 1032, 1076, 978, 1056, 992, 985, 1065, 1036, 1025, 1029, 1062, 999, 1012, 1011, 1040, 1005, 1012, 990, 988, 1017, 986, 1047, 1025, 1042, 1044, 1022, 1023, 1026, 1054, 1009, 1032, 1021, 1057, 980, 959, 1016, 1028, 1055, 1026, 987, 1045, 1056, 1020, 1031, 1004, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1042, 972, 1026, 1051, 1005, 976, 1012, 1018, 1019, 994, 998, 1032, 1027, 1024, 1033, 1053, 985, 1022, 1059, 1048, 975, 1049, 985, 1051, 967, 979, 1011, 982, 1003, 998, 1024, 1049, 1004, 989, 988, 1052, 1078, 1041, 1007, 1044, 1057, 1091, 969, 1062, 984, 1046, 1007, 1052, 1098, 1018, 1033, 1031, 1082, 996, 1018, 1056, 1059, 1011, 1082, 1023, 1037, 1020, 995, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1069, 1009, 1016, 1042, 991, 1099, 1040, 999, 1022, 975, 992, 1003, 1027, 1038, 1036, 1090, 1070, 1037, 1068, 972, 987, 1057, 985, 971, 1010, 1009, 1035, 1012, 1055, 994, 996, 1048, 1038, 1067, 982, 1018, 1056, 1031, 1044, 1050, 998, 1012, 1030, 1092, 1013, 1008, 1072, 996, 1020, 1013, 1069, 1004, 1029, 1006, 987, 994, 998, 1011, 995, 1025, 1049, 1038, 1011, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1045, 1027, 999, 1012, 1021, 1000, 1017, 1069, 1006, 1051, 1047, 1005, 1023, 990, 987, 1034, 1046, 1029, 1059, 1023, 1029, 975, 1015, 998, 1030, 995, 1052, 1049, 1053, 989, 1039, 1077, 1049, 1033, 1029, 1076, 1028, 1036, 1062, 1011, 1050, 1069, 1046, 986, 962, 1014, 1030, 1034, 970, 1028, 1024, 1028, 1025, 1077, 1008, 986, 999, 1015, 1002, 971, 1051, 986, 998, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1034, 1038, 1022, 1006, 965, 1050, 1014, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1040, 1020, 1049, 1016, 1004, 990, 1030, 42, 0, 0, 0, 0, 0, 0, 0, 1043, 1006, 1030, 1018, 1024, 1032, 961, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1008, 1059, 1006, 1044, 1018, 1021, 1017, 7, 2, 2, 1, 0, 0, 0, 0, 1047, 1023, 1086, 959, 1074, 1002, 1008, 997, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1034, 1047, 1038, 964, 1034, 1039, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1056, 1049, 999, 1013, 1032, 1037, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 982, 1083, 1022, 1023, 989, 1034, 1037, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8193, 8164, 8240, 8196, 8176, 8213, 8252] avg=8192.0000 max=8252 max_rank=7 min=8102 min_rank=0 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1017, 1007, 1016, 1028, 1066, 1017, 991, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 947, 1043, 1016, 1006, 1043, 1055, 1018, 980, 42, 0, 0, 0, 0, 0, 0, 0, 1012, 994, 984, 1024, 1029, 1045, 996, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1026, 1002, 1089, 1033, 1047, 1037, 1030, 7, 2, 2, 1, 0, 0, 0, 0, 1047, 1032, 1020, 1005, 1015, 1042, 1070, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1033, 1054, 1030, 997, 945, 1008, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1046, 1021, 1011, 1024, 1053, 1031, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1048, 1041, 1062, 1040, 961, 985, 1004, 10, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8150, 8133, 8315, 8234, 8102, 8256, 8168] avg=8192.0000 max=8315 max_rank=3 min=8102 min_rank=5 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1005, 996, 1050, 1043, 1049, 1031, 1018, 985, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1015, 1041, 1018, 1014, 1007, 990, 955, 42, 0, 0, 0, 0, 0, 0, 0, 1050, 1036, 1067, 1022, 1010, 1054, 1030, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1010, 1008, 1031, 997, 1032, 995, 995, 7, 2, 2, 1, 0, 0, 0, 0, 1005, 1025, 1055, 1057, 996, 1054, 1029, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1075, 1031, 1060, 1035, 1013, 1031, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1045, 1018, 1015, 1031, 1080, 1012, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1000, 1015, 1019, 1010, 1040, 1019, 1034, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8081, 8357, 8098, 8212, 8266, 8205, 8140] avg=8192.0000 max=8357 max_rank=2 min=8081 min_rank=1 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1040, 972, 1026, 1051, 1005, 976, 1012, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 994, 998, 1032, 1027, 1024, 1033, 1053, 42, 0, 0, 0, 0, 0, 0, 0, 985, 1022, 1051, 1048, 975, 1049, 985, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 967, 979, 1011, 982, 1003, 998, 1024, 1049, 7, 2, 2, 1, 0, 0, 0, 0, 1004, 989, 988, 1052, 1078, 1041, 1007, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1084, 969, 1062, 984, 1046, 1007, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1098, 1018, 1033, 1031, 1082, 995, 1018, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1011, 1082, 1023, 1037, 1020, 995, 1037, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8222, 8166, 8025, 8152, 8261, 8331, 8279] avg=8192.0000 max=8331 max_rank=6 min=8025 min_rank=3 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=33023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1067, 1009, 1016, 1042, 991, 1099, 1040, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 975, 992, 1003, 1027, 1038, 1036, 1090, 42, 0, 0, 0, 0, 0, 0, 0, 1070, 1037, 1060, 972, 987, 1057, 985, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1009, 1035, 1012, 1055, 994, 996, 1048, 6, 2, 2, 1, 0, 0, 0, 0, 1038, 1067, 982, 1018, 1056, 1031, 1044, 999, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1006, 1030, 1092, 1013, 1008, 1072, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1013, 1069, 1004, 1029, 1005, 987, 994, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1011, 995, 1025, 1049, 1038, 1011, 1026, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8225, 8139, 8170, 8235, 8215, 8121, 8168] avg=8192.0000 max=8263 max_rank=0 min=8121 min_rank=6 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1013, 993, 1025, 1052, 1030, 941, 1040, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1029, 1064, 1041, 1032, 1015, 1047, 1006, 42, 0, 0, 0, 0, 0, 0, 0, 1017, 1018, 1024, 1076, 978, 1056, 992, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1036, 1025, 1029, 1062, 999, 1012, 1011, 7, 2, 2, 1, 0, 0, 0, 0, 1040, 1005, 1012, 990, 988, 1017, 986, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1035, 1044, 1022, 1023, 1026, 1054, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1021, 1057, 980, 959, 1015, 1028, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 987, 1045, 1056, 1020, 1031, 1004, 1027, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8328, 8146, 8251, 8034, 8238, 8147, 8211] avg=8192.0000 max=8328 max_rank=1 min=8034 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32630 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1040, 1080, 1042, 1017, 1063, 1020, 1022, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 989, 1009, 1011, 1032, 1032, 1030, 977, 41, 0, 0, 0, 0, 0, 0, 0, 1035, 1023, 1024, 1065, 1010, 1032, 1028, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 978, 1032, 998, 1011, 1021, 1036, 997, 6, 2, 2, 1, 0, 0, 0, 0, 1065, 1019, 1019, 1002, 1052, 1091, 989, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1004, 1034, 982, 989, 1045, 1014, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1018, 1012, 1013, 1010, 1024, 995, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1062, 1011, 1052, 1003, 1019, 984, 1045, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8352, 8148, 8269, 8128, 8210, 8097, 8122, 8210] avg=8192.0000 max=8352 max_rank=0 min=8097 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32639 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1038, 1027, 999, 1012, 1021, 1000, 1017, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1051, 1047, 1005, 1023, 990, 987, 1034, 46, 0, 0, 0, 0, 0, 0, 0, 1046, 1029, 1047, 1023, 1029, 975, 1015, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 995, 1052, 1049, 1053, 989, 1039, 1077, 10, 7, 3, 5, 0, 0, 0, 0, 1049, 1033, 1029, 1076, 1028, 1036, 1062, 950, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1059, 1046, 986, 962, 1014, 1030, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1028, 1024, 1028, 1025, 1072, 1008, 986, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1015, 1002, 971, 1051, 986, 998, 1062, 15, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8189, 8162, 8309, 8263, 8181, 8141, 8108] avg=8192.0000 max=8309 max_rank=3 min=8108 min_rank=7 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8448, 8320, 8320, 8064, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8320, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8320, 8064, 8320, 8320, 8320, 8192, 8192, 8320, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1034, 1018, 1017, 1047, 1034, 1040, 1038, 1011, 985, 1002, 1061, 1042, 1005, 1022, 1041, 1028, 1019, 1066, 1061, 1039, 1001, 1019, 1007, 1067, 1021, 1049, 1034, 1053, 1001, 1038, 1027, 1070, 998, 1000, 1022, 1014, 1028, 1036, 977, 947, 1038, 1010, 1051, 990, 1004, 1042, 1021, 1060, 984, 971, 1026, 981, 1048, 1017, 1017, 1027, 1031, 1043, 1004, 1013, 1036, 995, 1057, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1022, 1061, 1029, 1032, 1028, 1027, 976, 1033, 1032, 1058, 1035, 1076, 1015, 1032, 1018, 979, 1005, 1078, 1013, 1039, 1013, 1004, 1018, 1012, 1045, 1056, 1051, 1023, 1037, 970, 1031, 990, 1022, 1017, 1057, 989, 1028, 1057, 1073, 1008, 1060, 1026, 1052, 986, 1000, 1030, 1058, 1005, 976, 1012, 1011, 1034, 992, 1033, 976, 996, 1025, 1002, 1016, 1026, 1057, 1042, 1004, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[987, 1022, 1005, 1043, 980, 1076, 1045, 1020, 1014, 994, 1051, 1016, 1060, 1023, 1010, 1003, 1040, 1002, 1028, 1014, 1050, 983, 1024, 1011, 1044, 1033, 1030, 1035, 1065, 1050, 1028, 987, 1014, 1012, 1010, 1021, 995, 1039, 1043, 1070, 1020, 987, 997, 1052, 1024, 1029, 1042, 1018, 1048, 1026, 1000, 1031, 984, 1027, 1012, 1008, 1030, 1062, 1024, 1025, 1062, 1041, 1013, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1002, 1010, 1006, 1024, 1014, 1047, 1045, 1000, 977, 1048, 1024, 1071, 1026, 1039, 1060, 998, 1044, 1006, 1043, 996, 973, 1065, 1013, 1045, 987, 983, 985, 1055, 1023, 1018, 985, 1036, 1106, 1011, 1016, 1030, 1050, 1059, 1024, 1000, 985, 1074, 1023, 1053, 1045, 1012, 1007, 997, 1023, 1016, 1036, 1049, 1006, 1038, 1059, 1039, 1016, 1059, 1039, 982, 1022, 1056, 948, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1066, 1023, 1012, 1039, 1023, 1083, 1048, 1039, 977, 968, 1032, 1005, 986, 1030, 1036, 1096, 989, 995, 988, 993, 1080, 1005, 1004, 997, 1029, 1038, 981, 1005, 985, 1016, 1050, 1057, 1036, 1052, 1022, 1047, 1059, 1040, 966, 1047, 1039, 1069, 970, 1009, 985, 1002, 1035, 1047, 1095, 1026, 1061, 1039, 1009, 1015, 1003, 1036, 1019, 1052, 1054, 1052, 997, 986, 1006, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1029, 1016, 1036, 1087, 999, 1049, 1013, 1068, 1024, 1033, 1004, 995, 1080, 1017, 1016, 1032, 1019, 1043, 975, 1065, 1026, 1024, 1016, 1012, 1021, 973, 1037, 1012, 1009, 1028, 1035, 1009, 1011, 1010, 998, 1048, 1055, 1047, 1068, 1035, 1020, 1019, 973, 1049, 1040, 1011, 1041, 1062, 953, 1017, 1034, 1036, 1003, 1009, 1007, 989, 1011, 1074, 1028, 1023, 993, 1035, 1013, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1036, 1046, 1009, 1056, 1026, 1041, 1018, 979, 1042, 1026, 1042, 974, 982, 1037, 1018, 999, 1084, 1034, 1038, 1068, 1017, 995, 1020, 1045, 1003, 994, 1015, 1028, 1006, 1000, 1083, 1023, 1017, 1022, 1036, 1049, 1059, 1023, 1013, 973, 1044, 1062, 956, 1000, 1012, 1071, 1063, 986, 1014, 1001, 1048, 1053, 1035, 1003, 1072, 1047, 964, 1014, 1013, 1005, 949, 1008, 1085, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1049, 1016, 1007, 990, 992, 1041, 1049, 1035, 1045, 1080, 998, 992, 1083, 1021, 1041, 1012, 1027, 962, 984, 995, 1028, 1036, 1032, 971, 982, 1058, 1038, 1042, 1026, 1041, 1072, 1047, 1027, 1052, 988, 1021, 1041, 1070, 1055, 1009, 1090, 1006, 963, 986, 985, 1038, 1052, 984, 1064, 1018, 1033, 1030, 1000, 1047, 1019, 1060, 978, 977, 1084, 1028, 1015, 1007, 1018, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1002, 1010, 1006, 1024, 1014, 1016, 1045, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1048, 1021, 1071, 1026, 1039, 1060, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1006, 1043, 996, 973, 1065, 1013, 1045, 10, 10, 0, 0, 0, 0, 0, 0, 987, 983, 985, 1055, 1023, 1018, 980, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1106, 1011, 1016, 1030, 1050, 1031, 1024, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1074, 1023, 1053, 1045, 1012, 1007, 997, 4, 3, 0, 0, 0, 0, 0, 0, 1023, 1016, 1036, 1049, 1006, 1038, 1059, 1039, 18, 1, 0, 0, 0, 0, 0, 0, 1016, 1059, 1039, 982, 1022, 1056, 948, 1008, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8240, 8205, 8067, 8268, 8203, 8285, 8151] avg=8192.0000 max=8285 max_rank=6 min=8067 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[987, 1022, 1005, 1043, 980, 1044, 1045, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 994, 1047, 1016, 1060, 1023, 1010, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1002, 1028, 1014, 1050, 983, 1024, 1011, 10, 10, 0, 0, 0, 0, 0, 0, 1044, 1033, 1030, 1035, 1065, 1050, 1023, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1012, 1010, 1021, 995, 1011, 1043, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 987, 997, 1052, 1024, 1029, 1042, 1018, 4, 4, 0, 0, 0, 0, 0, 0, 1048, 1026, 1000, 1031, 984, 1027, 1012, 1008, 18, 1, 0, 0, 0, 0, 0, 0, 1030, 1062, 1024, 1025, 1062, 1041, 1013, 997, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8167, 8172, 8267, 8176, 8177, 8155, 8276] avg=8192.0000 max=8276 max_rank=7 min=8146 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1022, 1061, 1029, 1032, 1028, 996, 976, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1058, 1031, 1076, 1015, 1032, 1018, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1078, 1013, 1039, 1013, 1004, 1018, 1012, 10, 10, 0, 0, 0, 0, 0, 0, 1045, 1056, 1051, 1023, 1037, 970, 1026, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1017, 1057, 989, 1028, 1029, 1073, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1026, 1052, 986, 1000, 1030, 1058, 1005, 4, 4, 0, 0, 0, 0, 0, 0, 976, 1012, 1011, 1034, 992, 1033, 976, 996, 18, 1, 0, 0, 0, 0, 0, 0, 1025, 1002, 1016, 1026, 1057, 1042, 1004, 1028, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8241, 8202, 8198, 8223, 8225, 8049, 8221] avg=8192.0000 max=8241 max_rank=1 min=8049 min_rank=6 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1034, 1018, 1017, 1047, 1034, 1009, 1038, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1002, 1057, 1042, 1005, 1022, 1041, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1066, 1061, 1039, 1001, 1019, 1007, 1067, 10, 10, 0, 0, 0, 0, 0, 0, 1021, 1049, 1034, 1053, 1001, 1038, 1022, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1000, 1022, 1014, 1028, 1008, 977, 947, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1010, 1051, 990, 1004, 1042, 1021, 1060, 4, 4, 0, 0, 0, 0, 0, 0, 984, 971, 1026, 981, 1048, 1017, 1017, 1027, 18, 1, 0, 0, 0, 0, 0, 0, 1031, 1043, 1004, 1013, 1036, 995, 1057, 1051, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8182, 8299, 8288, 7994, 8224, 8090, 8251] avg=8192.0000 max=8299 max_rank=2 min=7994 min_rank=4 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1029, 1016, 1036, 1087, 999, 1018, 1013, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1033, 1001, 995, 1080, 1017, 1016, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1043, 975, 1065, 1026, 1024, 1016, 1012, 10, 10, 0, 0, 0, 0, 0, 0, 1021, 973, 1037, 1012, 1009, 1028, 1030, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1010, 998, 1048, 1055, 1019, 1068, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1019, 973, 1049, 1040, 1011, 1041, 1062, 4, 3, 0, 0, 0, 0, 0, 0, 953, 1017, 1034, 1036, 1003, 1009, 1007, 989, 18, 1, 0, 0, 0, 0, 0, 0, 1011, 1074, 1028, 1023, 993, 1035, 1013, 1022, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8198, 8200, 8119, 8244, 8222, 8067, 8220] avg=8192.0000 max=8266 max_rank=0 min=8067 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1066, 1023, 1012, 1039, 1023, 1051, 1048, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 977, 968, 1028, 1005, 986, 1030, 1036, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 989, 995, 988, 993, 1080, 1005, 1004, 997, 10, 10, 0, 0, 0, 0, 0, 0, 1029, 1038, 981, 1005, 985, 1016, 1045, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1052, 1022, 1047, 1059, 1012, 966, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1069, 970, 1009, 985, 1002, 1035, 1047, 4, 4, 0, 0, 0, 0, 0, 0, 1095, 1026, 1061, 1039, 1009, 1015, 1003, 1036, 18, 1, 0, 0, 0, 0, 0, 0, 1019, 1052, 1054, 1052, 997, 986, 1006, 986, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8126, 8071, 8156, 8241, 8164, 8303, 8174] avg=8192.0000 max=8303 max_rank=6 min=8071 min_rank=2 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1036, 1046, 1009, 1056, 1026, 1010, 1018, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1026, 1038, 974, 982, 1037, 1018, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 1034, 1038, 1068, 1017, 995, 1020, 1045, 10, 10, 0, 0, 0, 0, 0, 0, 1003, 994, 1015, 1028, 1006, 1000, 1078, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1022, 1036, 1049, 1059, 996, 1013, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1062, 956, 1000, 1012, 1071, 1063, 986, 4, 4, 0, 0, 0, 0, 0, 0, 1014, 1001, 1048, 1053, 1035, 1003, 1072, 1047, 17, 1, 0, 0, 0, 0, 0, 0, 964, 1014, 1013, 1005, 949, 1008, 1085, 1055, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8116, 8321, 8147, 8165, 8202, 8291, 8114] avg=8192.0000 max=8321 max_rank=2 min=8114 min_rank=7 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1049, 1016, 1007, 990, 992, 1003, 1049, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1080, 992, 992, 1083, 1021, 1041, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 962, 984, 995, 1028, 1036, 1032, 971, 16, 15, 0, 0, 0, 0, 0, 0, 982, 1058, 1038, 1042, 1026, 1041, 1060, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1052, 988, 1021, 1041, 1032, 1055, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 1006, 963, 986, 985, 1038, 1052, 984, 9, 6, 0, 0, 0, 0, 0, 0, 1064, 1018, 1033, 1030, 1000, 1047, 1019, 1060, 22, 3, 0, 0, 0, 0, 0, 0, 978, 977, 1084, 1028, 1015, 1007, 1018, 999, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8266, 8066, 8294, 8225, 8119, 8296, 8129] avg=8192.0000 max=8296 max_rank=6 min=8066 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1030, 1056, 1023, 1005, 1048, 965, 1007, 1062, 1032, 1030, 1056, 1053, 1040, 1000, 1002, 975, 1070, 1022, 1023, 1057, 1042, 1033, 1024, 996, 1078, 1010, 988, 983, 1028, 1010, 1050, 993, 993, 1042, 1034, 1019, 984, 1019, 964, 1050, 1010, 994, 1031, 1040, 1071, 1049, 1052, 980, 1011, 1011, 1036, 1028, 1025, 1010, 978, 1034, 1032, 1061, 1047, 1079, 1003, 1017, 998, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[992, 1011, 1005, 1051, 1030, 1007, 996, 1052, 991, 1051, 1052, 1007, 1028, 998, 1067, 1036, 1063, 995, 1020, 1030, 1008, 976, 993, 1087, 1044, 1026, 1015, 1009, 1025, 970, 1004, 985, 1026, 1003, 1017, 1043, 965, 1081, 1020, 1037, 1048, 978, 999, 1006, 1091, 1075, 1017, 1017, 1048, 1026, 1035, 998, 1043, 1052, 1015, 1070, 1008, 1024, 991, 1037, 1039, 1006, 1089, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1030, 1038, 1013, 989, 1005, 1052, 1057, 1076, 1023, 975, 1091, 1022, 997, 1025, 1035, 996, 1002, 1018, 1059, 1018, 1021, 997, 1055, 1035, 985, 1002, 1045, 1062, 1049, 1012, 1027, 1005, 1043, 1001, 1009, 1038, 1011, 965, 1039, 1045, 1071, 1035, 1027, 1018, 1011, 1011, 1018, 980, 1030, 978, 1021, 994, 1049, 1048, 1052, 1009, 1020, 1060, 1032, 998, 1023, 985, 1036, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1007, 1029, 1046, 1073, 1036, 982, 983, 1033, 1028, 1011, 1012, 1029, 1078, 1041, 994, 1014, 1053, 1012, 1075, 1009, 1062, 993, 1035, 1007, 1068, 980, 1035, 1007, 1024, 1026, 1047, 1055, 1030, 1011, 1044, 1041, 998, 1046, 1013, 983, 986, 1012, 1021, 981, 995, 994, 976, 1004, 989, 977, 1018, 1017, 1069, 1037, 1030, 1060, 1029, 1007, 1042, 1065, 983, 1040, 1065, 1089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1001, 1003, 1075, 1042, 1014, 1058, 994, 1023, 1048, 1063, 1044, 1037, 989, 1034, 1056, 1049, 1035, 1027, 1039, 977, 1067, 996, 996, 1039, 1056, 1066, 971, 991, 998, 985, 1020, 1048, 1007, 994, 1032, 1043, 1017, 1052, 980, 1007, 986, 1038, 1028, 1076, 1028, 984, 1021, 1047, 1057, 1043, 1033, 994, 1037, 1039, 1034, 1003, 997, 1003, 1007, 982, 1054, 1038, 984, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1034, 1045, 1022, 1040, 1035, 993, 964, 998, 1044, 1047, 1025, 997, 980, 1083, 976, 1032, 1030, 990, 1026, 1035, 1004, 1051, 1073, 1073, 1026, 1006, 1002, 1021, 1018, 1041, 1040, 990, 1045, 973, 985, 1002, 1040, 1078, 1021, 963, 1052, 1044, 1051, 1009, 1042, 1035, 986, 990, 1019, 1007, 1040, 1025, 1048, 1066, 1080, 1008, 1063, 994, 1015, 1038, 1047, 977, 1025, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[997, 1028, 981, 985, 1005, 1058, 990, 1073, 993, 1013, 1089, 1105, 985, 1033, 1019, 984, 1052, 1085, 1058, 1027, 1004, 1042, 1052, 987, 1027, 1059, 1036, 963, 984, 1034, 1063, 993, 991, 1010, 1019, 1011, 1045, 1108, 1026, 1002, 1028, 949, 1026, 1053, 1021, 998, 1049, 1045, 1041, 1013, 1056, 1006, 1043, 1011, 1042, 1016, 999, 1033, 1008, 1039, 984, 978, 1066, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[995, 974, 1051, 1034, 999, 1034, 1064, 1054, 1025, 961, 995, 993, 1025, 961, 1066, 1010, 1064, 1059, 1022, 1076, 1001, 1020, 1039, 1028, 1008, 1036, 1041, 1023, 1057, 1043, 1032, 1041, 1028, 1053, 1023, 1061, 1011, 1025, 1017, 1045, 976, 1022, 1021, 1092, 1023, 980, 995, 1000, 1118, 1005, 1030, 1016, 1032, 989, 1038, 976, 970, 1022, 996, 1018, 1030, 1078, 1015, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1030, 1056, 1023, 1005, 1048, 965, 1007, 1062, 4, 4, 1, 0, 0, 0, 0, 0, 1032, 1030, 1055, 1053, 1040, 1000, 1002, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1022, 1023, 1057, 1042, 1033, 1024, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1010, 988, 983, 1028, 1010, 1050, 993, 20, 0, 0, 0, 0, 0, 0, 0, 993, 1042, 1034, 1019, 984, 1019, 964, 1050, 23, 2, 0, 0, 0, 0, 0, 0, 1010, 994, 1031, 1040, 1071, 1049, 1052, 980, 31, 0, 0, 0, 0, 0, 0, 0, 1011, 1011, 1036, 1028, 999, 1010, 978, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1061, 1047, 1079, 1003, 1017, 998, 1039, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8187, 8213, 8160, 8130, 8258, 8107, 8276] avg=8192.0000 max=8276 max_rank=7 min=8107 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1030, 1038, 1013, 989, 1005, 1052, 1057, 1076, 4, 4, 1, 0, 0, 0, 0, 0, 1023, 975, 1090, 1022, 997, 1025, 1035, 996, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1018, 1059, 1018, 1021, 997, 1055, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1002, 1045, 1062, 1049, 1012, 1027, 1005, 21, 0, 0, 0, 0, 0, 0, 0, 1043, 1001, 1009, 1038, 1011, 965, 1039, 1045, 22, 2, 0, 0, 0, 0, 0, 0, 1071, 1035, 1027, 1018, 1011, 1011, 1018, 980, 29, 0, 0, 0, 0, 0, 0, 0, 1030, 978, 1021, 994, 1022, 1048, 1052, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1060, 1032, 998, 1023, 985, 1036, 1059, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8163, 8154, 8208, 8175, 8200, 8154, 8213] avg=8192.0000 max=8269 max_rank=0 min=8154 min_rank=2 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[992, 1011, 1005, 1051, 1030, 1007, 996, 1052, 4, 4, 1, 0, 0, 0, 0, 0, 991, 1051, 1051, 1007, 1028, 998, 1067, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 995, 1020, 1030, 1008, 976, 993, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1026, 1015, 1009, 1025, 970, 1004, 985, 21, 0, 0, 0, 0, 0, 0, 0, 1026, 1003, 1017, 1043, 965, 1081, 1020, 1037, 23, 2, 0, 0, 0, 0, 0, 0, 1048, 978, 999, 1006, 1091, 1075, 1017, 1017, 30, 0, 0, 0, 0, 0, 0, 0, 1048, 1026, 1035, 998, 1016, 1052, 1015, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1024, 991, 1037, 1039, 1006, 1089, 1004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8229, 8119, 8099, 8217, 8261, 8260, 8198] avg=8192.0000 max=8261 max_rank=5 min=8099 min_rank=3 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1001, 1003, 1075, 1042, 1014, 1058, 994, 1023, 4, 4, 1, 0, 0, 0, 0, 0, 1048, 1063, 1043, 1037, 989, 1034, 1056, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1027, 1039, 977, 1067, 996, 996, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1066, 971, 991, 998, 985, 1020, 1048, 21, 0, 0, 0, 0, 0, 0, 0, 1007, 994, 1032, 1043, 1017, 1052, 980, 1007, 22, 2, 0, 0, 0, 0, 0, 0, 986, 1038, 1028, 1076, 1028, 984, 1021, 1047, 30, 0, 0, 0, 0, 0, 0, 0, 1057, 1043, 1033, 994, 1010, 1039, 1034, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1003, 1007, 982, 1054, 1038, 984, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8319, 8124, 8156, 8156, 8238, 8213, 8111] avg=8192.0000 max=8319 max_rank=1 min=8111 min_rank=7 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1034, 1045, 1022, 1040, 1035, 993, 964, 998, 4, 4, 1, 0, 0, 0, 0, 0, 1044, 1047, 1024, 997, 980, 1083, 976, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 979, 990, 1026, 1035, 1004, 1051, 1073, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1006, 1002, 1021, 1018, 1041, 1040, 990, 21, 0, 0, 0, 0, 0, 0, 0, 1045, 973, 985, 1002, 1040, 1078, 1021, 963, 22, 2, 0, 0, 0, 0, 0, 0, 1052, 1044, 1051, 1009, 1042, 1035, 986, 990, 29, 0, 0, 0, 0, 0, 0, 0, 1019, 1007, 1040, 1025, 1021, 1066, 1080, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 994, 1015, 1038, 1047, 977, 1025, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8183, 8231, 8165, 8131, 8238, 8266, 8182] avg=8192.0000 max=8266 max_rank=6 min=8131 min_rank=4 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[997, 1028, 981, 985, 1005, 1058, 990, 1073, 4, 4, 1, 0, 0, 0, 0, 0, 993, 1013, 1088, 1105, 985, 1033, 1019, 984, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1085, 1058, 1027, 1004, 1042, 1052, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1059, 1036, 963, 984, 1034, 1063, 993, 21, 0, 0, 0, 0, 0, 0, 0, 991, 1010, 1019, 1011, 1045, 1108, 1026, 1002, 23, 2, 0, 0, 0, 0, 0, 0, 1028, 949, 1026, 1053, 1021, 998, 1049, 1045, 30, 0, 0, 0, 0, 0, 0, 0, 1041, 1013, 1056, 1006, 1016, 1011, 1042, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1033, 1008, 1039, 984, 978, 1066, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8220, 8254, 8180, 8237, 8199, 8201, 8119] avg=8192.0000 max=8254 max_rank=2 min=8119 min_rank=7 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1007, 1029, 1046, 1073, 1036, 982, 983, 1033, 4, 4, 1, 0, 0, 0, 0, 0, 1028, 1011, 1011, 1029, 1078, 1041, 994, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1012, 1075, 1009, 1062, 993, 1035, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 980, 1035, 1007, 1024, 1026, 1047, 1055, 21, 0, 0, 0, 0, 0, 0, 0, 1030, 1011, 1044, 1041, 998, 1046, 1013, 983, 23, 2, 0, 0, 0, 0, 0, 0, 986, 1012, 1021, 981, 995, 994, 976, 1004, 30, 0, 0, 0, 0, 0, 0, 0, 989, 977, 1018, 1017, 1042, 1037, 1030, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1007, 1042, 1065, 983, 1040, 1065, 1085, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8206, 8193, 8263, 8191, 7999, 8170, 8316] avg=8192.0000 max=8316 max_rank=7 min=7999 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[995, 974, 1051, 1034, 999, 1034, 1064, 1054, 8, 6, 7, 0, 0, 0, 0, 0, 1025, 961, 988, 993, 1025, 961, 1066, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1059, 1022, 1076, 1001, 1020, 1039, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1036, 1041, 1023, 1057, 1043, 1032, 1041, 24, 0, 0, 0, 0, 0, 0, 0, 1028, 1053, 1023, 1061, 1011, 1025, 1017, 1045, 27, 9, 0, 0, 0, 0, 0, 0, 976, 1022, 1021, 1092, 1023, 980, 995, 1000, 34, 0, 0, 0, 0, 0, 0, 0, 1118, 1005, 1030, 1016, 993, 989, 1038, 976, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1022, 996, 1018, 1030, 1078, 1015, 992, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8029, 8248, 8305, 8299, 8143, 8165, 8121] avg=8192.0000 max=8305 max_rank=3 min=8029 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8064, 8192, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8064, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8448, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8064, 8448, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8448, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8448, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1029, 1041, 995, 973, 996, 1002, 1056, 1014, 976, 989, 968, 995, 1059, 996, 1031, 1027, 1014, 1003, 995, 1049, 991, 989, 1016, 1020, 1019, 1032, 983, 1013, 1064, 1020, 1041, 1071, 1008, 1037, 1005, 1040, 1033, 1036, 1010, 1045, 1041, 1059, 1035, 1052, 1005, 1068, 1019, 1001, 1043, 1024, 1060, 1057, 1046, 1036, 1050, 1034, 1064, 1009, 1011, 1046, 1007, 1005, 1020, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[977, 1005, 992, 1036, 989, 1044, 1058, 1081, 948, 1032, 1014, 1009, 1025, 971, 1016, 1077, 1052, 1008, 982, 1008, 994, 995, 1083, 1051, 1046, 1012, 1002, 953, 1030, 988, 1031, 1115, 1047, 1019, 990, 1027, 1026, 1052, 1006, 1057, 977, 1053, 1062, 1053, 1041, 1029, 960, 1021, 1011, 1041, 1028, 1050, 1054, 985, 998, 1050, 1025, 1025, 1054, 1043, 1060, 1054, 1002, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1061, 1018, 1075, 1039, 1000, 1047, 980, 1042, 972, 1044, 992, 987, 1034, 1012, 1032, 1045, 1026, 1031, 1007, 1050, 1045, 1039, 980, 1025, 1039, 1097, 1044, 973, 1072, 1029, 997, 1063, 1000, 1001, 1038, 986, 1048, 1023, 979, 1068, 1007, 997, 994, 1018, 1024, 984, 991, 1022, 1010, 1040, 997, 1030, 1071, 1014, 1010, 1026, 1108, 1024, 1015, 1015, 988, 1019, 1019, 1073] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1065, 977, 1019, 1034, 1004, 1067, 1102, 1015, 1025, 1061, 1008, 974, 1004, 1039, 1018, 981, 1028, 1042, 1039, 1043, 1058, 980, 999, 1054, 1035, 1095, 1024, 1065, 985, 1014, 1038, 1002, 977, 1006, 977, 1008, 1052, 1060, 1081, 1036, 975, 1036, 1050, 1013, 1044, 1015, 1039, 983, 989, 1019, 1038, 979, 1015, 1040, 1036, 1022, 1000, 984, 1011, 1086, 1033, 990, 1032, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1022, 1011, 1075, 1006, 990, 1008, 1019, 958, 1057, 1032, 1039, 1050, 1066, 1016, 1024, 1027, 1032, 1063, 982, 978, 1024, 1059, 1000, 1042, 1063, 1009, 1024, 1019, 1053, 1057, 1009, 1022, 999, 1057, 1075, 1057, 988, 1013, 1002, 1061, 1073, 1008, 1033, 1036, 1029, 1020, 998, 1047, 1013, 1004, 949, 1014, 993, 1008, 1002, 1038, 1058, 1038, 1057, 1043, 1012, 984, 1005, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1028, 1013, 1050, 1017, 1007, 1004, 1067, 963, 1020, 1021, 970, 1020, 1016, 1035, 1024, 1051, 1055, 1040, 1033, 1008, 1043, 1061, 1014, 1086, 986, 1080, 1027, 1044, 1039, 997, 1025, 1012, 979, 1075, 1036, 1014, 1008, 1034, 967, 1028, 1013, 1030, 1048, 1064, 1044, 984, 1006, 1039, 1037, 1012, 1001, 1016, 999, 1046, 1023, 990, 1017, 1021, 1061, 1026, 1033, 1020, 991, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1017, 1051, 1059, 1073, 1002, 1015, 1062, 1008, 1042, 1091, 1061, 1013, 1022, 1036, 1069, 1021, 1014, 1026, 1034, 1043, 1019, 1032, 1049, 951, 1033, 1031, 1054, 1095, 1012, 1016, 999, 1038, 985, 981, 1021, 1018, 1044, 1013, 971, 983, 1021, 995, 981, 1005, 1041, 1028, 982, 1007, 1059, 1044, 1029, 1083, 1001, 1031, 992, 1036, 1023, 969, 962, 1024, 1058, 965, 1052, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1032, 986, 1047, 1047, 1058, 1000, 1035, 1035, 1040, 1027, 1068, 974, 1017, 1028, 1082, 1027, 1030, 1057, 1033, 996, 1047, 1010, 1005, 1014, 1030, 1026, 990, 1010, 1016, 1037, 1042, 1006, 1067, 1017, 1024, 1018, 1000, 1029, 989, 1017, 1046, 941, 1025, 1012, 1052, 976, 1018, 1029, 1045, 993, 1049, 1040, 1009, 1035, 991, 1030, 1019, 1070, 1009, 1022, 1012, 1035, 1064, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1029, 1041, 995, 973, 996, 1002, 1049, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 976, 989, 968, 995, 1059, 996, 1031, 1027, 7, 2, 1, 0, 0, 0, 0, 0, 1014, 999, 995, 1049, 991, 989, 1016, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 989, 983, 1013, 1064, 1020, 1041, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1037, 1005, 1040, 1033, 1036, 1010, 1045, 14, 8, 0, 0, 0, 0, 0, 0, 1041, 1059, 1035, 1052, 1005, 1068, 1019, 1001, 29, 0, 0, 0, 0, 0, 0, 0, 1043, 1024, 1060, 1057, 1046, 1036, 1050, 1034, 2, 0, 0, 0, 0, 0, 0, 0, 1055, 1009, 1011, 1046, 1007, 1005, 1020, 1063, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8051, 8073, 8200, 8236, 8309, 8352, 8216] avg=8192.0000 max=8352 max_rank=6 min=8051 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=33113 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1065, 977, 1019, 1034, 1004, 1067, 1094, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1061, 1008, 974, 1004, 1039, 1018, 981, 8, 2, 0, 0, 0, 0, 0, 0, 1028, 1038, 1039, 1043, 1058, 980, 999, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1050, 1024, 1065, 985, 1014, 1038, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1006, 977, 1008, 1052, 1060, 1081, 1036, 15, 8, 0, 0, 0, 0, 0, 0, 975, 1036, 1050, 1013, 1044, 1015, 1039, 983, 30, 0, 0, 0, 0, 0, 0, 0, 989, 1019, 1038, 979, 1015, 1040, 1036, 1022, 2, 0, 0, 0, 0, 0, 0, 0, 992, 984, 1011, 1086, 1033, 990, 1032, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8120, 8239, 8213, 8220, 8185, 8140, 8144] avg=8192.0000 max=8275 max_rank=0 min=8120 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[977, 1005, 992, 1036, 989, 1044, 1051, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 948, 1032, 1014, 1009, 1025, 971, 1016, 1077, 7, 2, 0, 0, 0, 0, 0, 0, 1052, 1004, 982, 1008, 994, 995, 1083, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 970, 1002, 953, 1030, 988, 1031, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1019, 990, 1027, 1026, 1052, 1006, 1057, 14, 8, 0, 0, 0, 0, 0, 0, 977, 1053, 1062, 1053, 1041, 1029, 960, 1021, 28, 0, 0, 0, 0, 0, 0, 0, 1011, 1041, 1028, 1050, 1054, 985, 998, 1050, 2, 0, 0, 0, 0, 0, 0, 0, 1017, 1025, 1054, 1043, 1060, 1054, 1002, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8101, 8169, 8135, 8246, 8224, 8219, 8267] avg=8192.0000 max=8267 max_rank=7 min=8101 min_rank=1 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32956 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1061, 1018, 1075, 1039, 1000, 1047, 973, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1044, 992, 987, 1034, 1012, 1032, 1045, 7, 2, 1, 0, 0, 0, 0, 0, 1026, 1027, 1007, 1050, 1045, 1039, 980, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1051, 1044, 973, 1072, 1029, 997, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1001, 1038, 986, 1048, 1023, 979, 1068, 15, 9, 0, 0, 0, 0, 0, 0, 1007, 997, 994, 1018, 1024, 984, 991, 1022, 31, 0, 0, 0, 0, 0, 0, 0, 1010, 1040, 997, 1030, 1071, 1014, 1010, 1026, 2, 0, 0, 0, 0, 0, 0, 0, 1098, 1024, 1015, 1015, 988, 1019, 1019, 1073, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8128, 8199, 8268, 8167, 8068, 8200, 8251] avg=8192.0000 max=8268 max_rank=3 min=8068 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1022, 1011, 1075, 1006, 990, 1008, 1012, 958, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1032, 1039, 1050, 1066, 1016, 1024, 1027, 7, 2, 1, 0, 0, 0, 0, 0, 1032, 1059, 982, 978, 1024, 1059, 1000, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 967, 1024, 1019, 1053, 1057, 1009, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1057, 1075, 1057, 988, 1013, 1002, 1061, 14, 8, 0, 0, 0, 0, 0, 0, 1073, 1008, 1033, 1036, 1029, 1020, 998, 1047, 28, 0, 0, 0, 0, 0, 0, 0, 1013, 1004, 949, 1014, 993, 1008, 1002, 1038, 2, 0, 0, 0, 0, 0, 0, 0, 1049, 1038, 1057, 1043, 1012, 984, 1005, 986, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8321, 8176, 8214, 8274, 8272, 8023, 8174] avg=8192.0000 max=8321 max_rank=1 min=8023 min_rank=6 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1028, 1013, 1050, 1017, 1007, 1004, 1060, 963, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1021, 970, 1020, 1016, 1035, 1024, 1051, 7, 2, 0, 0, 0, 0, 0, 0, 1055, 1036, 1033, 1008, 1043, 1061, 1014, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1035, 1027, 1044, 1039, 997, 1025, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1075, 1036, 1014, 1008, 1034, 967, 1028, 15, 8, 0, 0, 0, 0, 0, 0, 1013, 1030, 1048, 1064, 1044, 984, 1006, 1039, 30, 0, 0, 0, 0, 0, 0, 0, 1037, 1012, 1001, 1016, 999, 1046, 1023, 990, 2, 0, 0, 0, 0, 0, 0, 0, 1009, 1021, 1061, 1026, 1033, 1020, 991, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8166, 8336, 8165, 8164, 8258, 8126, 8179] avg=8192.0000 max=8336 max_rank=2 min=8126 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1017, 1051, 1059, 1073, 1002, 1015, 1055, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1091, 1061, 1013, 1022, 1036, 1069, 1021, 7, 2, 0, 0, 0, 0, 0, 0, 1014, 1022, 1034, 1043, 1019, 1032, 1049, 951, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 988, 1054, 1095, 1012, 1016, 999, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 985, 981, 1021, 1018, 1044, 1013, 971, 983, 14, 8, 0, 0, 0, 0, 0, 0, 1021, 995, 981, 1005, 1041, 1028, 982, 1007, 29, 0, 0, 0, 0, 0, 0, 0, 1059, 1044, 1029, 1083, 1001, 1031, 992, 1036, 2, 0, 0, 0, 0, 0, 0, 0, 1015, 969, 962, 1024, 1058, 965, 1052, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8280, 8364, 8164, 8235, 8038, 8089, 8277, 8089] avg=8192.0000 max=8364 max_rank=1 min=8038 min_rank=4 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1032, 986, 1047, 1047, 1058, 1000, 1023, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1027, 1068, 974, 1017, 1028, 1082, 1027, 12, 5, 5, 0, 0, 0, 0, 0, 1030, 1045, 1033, 996, 1047, 1010, 1005, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 975, 990, 1010, 1016, 1037, 1042, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1017, 1024, 1018, 1000, 1029, 989, 1017, 19, 11, 0, 0, 0, 0, 0, 0, 1046, 941, 1025, 1012, 1052, 976, 1018, 1029, 32, 0, 0, 0, 0, 0, 0, 0, 1045, 993, 1049, 1040, 1009, 1035, 991, 1030, 7, 0, 0, 0, 0, 0, 0, 0, 1003, 1070, 1009, 1022, 1012, 1035, 1064, 1001, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8285, 8180, 8106, 8191, 8131, 8199, 8216] avg=8192.0000 max=8285 max_rank=1 min=8106 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8192, 8320, 8320, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8064, 8320, 8192, 8320, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8064, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8192, 8320, 8192, 8192, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1070, 1026, 1052, 1021, 1017, 1021, 988, 1030, 1048, 1004, 1017, 998, 969, 978, 1032, 978, 1014, 1022, 1008, 1045, 1015, 1060, 1016, 1032, 1017, 1014, 981, 997, 1035, 1082, 1034, 1059, 1042, 1063, 1023, 1023, 1061, 983, 989, 981, 1041, 1069, 1062, 974, 978, 1019, 994, 1079, 972, 1044, 1027, 1044, 1016, 1002, 1017, 1083, 1041, 1036, 1017, 1076, 1053, 1014, 1011, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1046, 1041, 1015, 1048, 1000, 1012, 1012, 1032, 974, 984, 1007, 1024, 1008, 983, 1043, 1008, 1044, 1037, 994, 1062, 1027, 1002, 1031, 1038, 1031, 1057, 1053, 1055, 1017, 1079, 977, 1030, 1021, 1079, 1006, 992, 1004, 1027, 1069, 1041, 1059, 1030, 1015, 1021, 1053, 1035, 1059, 1005, 1016, 1005, 1007, 1054, 1012, 1040, 1013, 1021, 1012, 994, 1012, 1007, 991, 1028, 1039, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1049, 1056, 984, 994, 1047, 1007, 1022, 958, 1030, 967, 1042, 988, 1035, 1021, 1043, 1030, 1000, 1090, 1007, 1080, 1049, 1049, 1018, 1061, 1053, 1026, 1046, 1045, 1071, 1101, 1035, 1041, 1026, 1006, 1020, 1022, 1090, 1027, 1042, 961, 1095, 1047, 1024, 1002, 1033, 1010, 990, 1017, 996, 1032, 1078, 927, 1019, 988, 999, 1004, 1008, 1031, 1011, 1022, 999, 996, 989, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1042, 1052, 1033, 1058, 1023, 1022, 1056, 1024, 1002, 1039, 1066, 989, 1033, 1006, 997, 1022, 1017, 1026, 1015, 1006, 1017, 1055, 1044, 999, 1002, 1011, 1005, 1031, 1037, 1034, 1054, 1017, 1016, 995, 1056, 1041, 1056, 982, 1005, 1034, 1060, 1021, 1047, 1032, 1061, 1045, 1006, 999, 1030, 1001, 1001, 997, 1052, 996, 1006, 975, 942, 1068, 1012, 1028, 1037, 1036, 1014, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1029, 1059, 1004, 1028, 1017, 995, 1010, 1051, 1025, 1029, 952, 985, 1028, 1003, 1033, 1000, 1061, 1040, 1042, 1043, 1029, 1051, 1025, 1027, 1036, 1063, 981, 1007, 1055, 996, 1022, 1042, 1016, 989, 1015, 1057, 1032, 1024, 1014, 1040, 1063, 1079, 988, 1008, 1043, 1073, 1029, 998, 1011, 1060, 1007, 1052, 1032, 1000, 977, 1007, 1023, 991, 1038, 1056, 1065, 967, 987, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[976, 1014, 1060, 1021, 985, 956, 1008, 1024, 1036, 1018, 1005, 1044, 1013, 1081, 1020, 1020, 952, 1016, 1039, 1021, 1032, 1066, 1009, 1029, 1063, 1021, 1028, 1039, 985, 1043, 1035, 1031, 1011, 1025, 1029, 1023, 1001, 1042, 1059, 1016, 1056, 1033, 1016, 1031, 1021, 1040, 978, 1087, 1034, 1046, 1005, 1015, 1061, 1048, 1025, 1007, 979, 1069, 987, 1006, 1044, 1014, 1017, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1033, 1025, 1028, 1022, 1017, 1042, 1003, 1042, 1031, 968, 988, 1004, 1005, 1050, 992, 1046, 1049, 1084, 1022, 1038, 1046, 1014, 1004, 1007, 1050, 1037, 992, 1050, 991, 1028, 1022, 992, 1040, 1019, 977, 1034, 1051, 1055, 1040, 1001, 1053, 1019, 989, 1047, 1010, 1052, 1017, 1050, 1027, 1007, 989, 1022, 996, 1039, 985, 1054, 1035, 1007, 1036, 1072, 1011, 1003, 1062, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1031, 1027, 1031, 1009, 1044, 1039, 983, 1046, 1032, 986, 1040, 1003, 1013, 989, 1047, 1033, 1013, 1023, 1035, 984, 1062, 998, 1009, 1018, 1037, 1013, 1055, 1037, 1054, 1011, 1020, 1040, 1036, 1031, 1013, 1002, 1035, 1036, 1001, 1037, 998, 1038, 1015, 1102, 983, 961, 1026, 1018, 1088, 1033, 1017, 1040, 1018, 992, 1013, 1044, 1081, 995, 986, 1031, 1024, 1024, 1037, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1046, 1041, 1015, 1048, 1000, 1012, 1012, 1032, 2, 0, 0, 0, 0, 0, 0, 0, 974, 984, 1007, 1024, 1008, 983, 1043, 1008, 60, 23, 0, 0, 0, 0, 0, 0, 1044, 996, 994, 1062, 1027, 1002, 1031, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1057, 1053, 1055, 1017, 1019, 977, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1079, 1006, 992, 999, 1027, 1069, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1030, 1015, 1021, 1053, 1035, 1059, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1005, 1007, 1054, 1012, 1040, 1013, 1021, 22, 16, 0, 0, 0, 0, 0, 0, 1012, 994, 1012, 1007, 991, 1028, 1039, 998, 25, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8114, 8194, 8239, 8234, 8232, 8206, 8109] avg=8192.0000 max=8239 max_rank=3 min=8109 min_rank=7 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1070, 1026, 1052, 1021, 1017, 1021, 988, 1030, 2, 0, 0, 0, 0, 0, 0, 0, 1048, 1004, 1017, 998, 969, 978, 1032, 978, 60, 22, 0, 0, 0, 0, 0, 0, 1014, 983, 1008, 1045, 1015, 1060, 1016, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1014, 981, 997, 1035, 1022, 1034, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1063, 1023, 1023, 1056, 983, 989, 981, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1069, 1062, 974, 978, 1019, 994, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1044, 1027, 1044, 1016, 1002, 1017, 1083, 22, 15, 0, 0, 0, 0, 0, 0, 1041, 1036, 1017, 1076, 1053, 1014, 1011, 1022, 24, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8106, 8173, 8159, 8160, 8172, 8242, 8297] avg=8192.0000 max=8297 max_rank=7 min=8106 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1049, 1056, 984, 994, 1047, 1007, 1022, 958, 2, 0, 0, 0, 0, 0, 0, 0, 1030, 967, 1042, 988, 1035, 1021, 1043, 1030, 61, 24, 0, 0, 0, 0, 0, 0, 1000, 1048, 1007, 1080, 1049, 1049, 1018, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1026, 1046, 1045, 1071, 1040, 1035, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1006, 1020, 1022, 1085, 1027, 1042, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1047, 1024, 1002, 1033, 1010, 990, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1032, 1078, 927, 1019, 988, 999, 1004, 23, 16, 0, 0, 0, 0, 0, 0, 1008, 1031, 1011, 1022, 999, 996, 989, 980, 26, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8241, 8312, 8357, 8189, 8171, 8082, 8065] avg=8192.0000 max=8357 max_rank=3 min=8065 min_rank=7 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1029, 1059, 1004, 1028, 1017, 995, 1010, 1051, 2, 0, 0, 0, 0, 0, 0, 0, 1025, 1029, 952, 985, 1028, 1003, 1033, 1000, 55, 23, 0, 0, 0, 0, 0, 0, 1061, 999, 1042, 1043, 1029, 1051, 1025, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1063, 981, 1007, 1055, 941, 1022, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 989, 1015, 1057, 1027, 1024, 1014, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1079, 988, 1008, 1043, 1073, 1029, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1060, 1007, 1052, 1032, 1000, 977, 1007, 22, 16, 0, 0, 0, 0, 0, 0, 1023, 991, 1038, 1056, 1065, 967, 987, 1027, 25, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8133, 8277, 8147, 8182, 8236, 8184, 8182] avg=8192.0000 max=8277 max_rank=2 min=8133 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[976, 1014, 1060, 1021, 985, 956, 1008, 1024, 2, 0, 0, 0, 0, 0, 0, 0, 1036, 1018, 1005, 1044, 1013, 1081, 1020, 1020, 58, 23, 0, 0, 0, 0, 0, 0, 952, 977, 1039, 1021, 1032, 1066, 1009, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1021, 1028, 1039, 985, 985, 1035, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1025, 1029, 1023, 996, 1042, 1059, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1033, 1016, 1031, 1021, 1040, 978, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1046, 1005, 1015, 1061, 1048, 1025, 1007, 22, 15, 0, 0, 0, 0, 0, 0, 979, 1069, 987, 1006, 1044, 1014, 1017, 1021, 24, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 8318, 8125, 8187, 8201, 8217, 8278, 8164] avg=8192.0000 max=8318 max_rank=1 min=8046 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1033, 1025, 1028, 1022, 1017, 1042, 1003, 1042, 2, 0, 0, 0, 0, 0, 0, 0, 1031, 968, 988, 1004, 1005, 1050, 992, 1046, 57, 23, 0, 0, 0, 0, 0, 0, 1049, 1042, 1022, 1038, 1046, 1014, 1004, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1037, 992, 1050, 991, 971, 1022, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1019, 977, 1034, 1046, 1055, 1040, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1019, 989, 1047, 1010, 1052, 1017, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1007, 989, 1022, 996, 1039, 985, 1054, 22, 16, 0, 0, 0, 0, 0, 0, 1035, 1007, 1036, 1072, 1011, 1003, 1062, 1015, 26, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8164, 8222, 8105, 8212, 8192, 8157, 8270] avg=8192.0000 max=8270 max_rank=7 min=8105 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1042, 1052, 1033, 1058, 1023, 1022, 1056, 1024, 2, 0, 0, 0, 0, 0, 0, 0, 1002, 1039, 1066, 989, 1033, 1006, 997, 1022, 57, 23, 0, 0, 0, 0, 0, 0, 1017, 987, 1015, 1006, 1017, 1055, 1044, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1011, 1005, 1031, 1037, 977, 1054, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 995, 1056, 1041, 1051, 982, 1005, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1021, 1047, 1032, 1061, 1045, 1006, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1001, 1001, 997, 1052, 996, 1006, 975, 22, 15, 0, 0, 0, 0, 0, 0, 942, 1068, 1012, 1028, 1037, 1036, 1014, 1051, 24, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8312, 8234, 8140, 8134, 8180, 8226, 8095, 8215] avg=8192.0000 max=8312 max_rank=0 min=8095 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1031, 1027, 1031, 1009, 1044, 1039, 983, 1046, 5, 0, 0, 0, 0, 0, 0, 0, 1032, 986, 1040, 1003, 1013, 989, 1047, 1033, 59, 24, 0, 0, 0, 0, 0, 0, 1013, 974, 1035, 984, 1062, 998, 1009, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1013, 1055, 1037, 1054, 952, 1020, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1031, 1013, 1002, 1022, 1036, 1001, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 947, 1038, 1015, 1102, 983, 961, 1026, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 1033, 1017, 1040, 1018, 992, 1013, 1044, 27, 20, 0, 0, 0, 0, 0, 0, 1081, 995, 986, 1031, 1024, 1024, 1037, 1019, 29, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8226, 8093, 8208, 8178, 8090, 8292, 8234] avg=8192.0000 max=8292 max_rank=6 min=8090 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8192, 8064, 8192, 8192, 8320, 8192, 512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8448, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8064, 8320, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 7936, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1042, 978, 1021, 995, 1044, 1045, 1035, 1010, 1040, 988, 1079, 1009, 1022, 1073, 1050, 1018, 1014, 1020, 1036, 1020, 1003, 1050, 1015, 1013, 1061, 977, 1024, 993, 1013, 1045, 1099, 957, 989, 1012, 1031, 1013, 995, 1063, 958, 1022, 1032, 1067, 1005, 1062, 1024, 1006, 998, 1052, 1042, 1053, 1012, 1029, 1002, 1013, 1064, 967, 1008, 1040, 1046, 1000, 1072, 1044, 971, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1020, 1068, 1036, 1072, 1061, 1084, 1059, 1028, 1047, 1116, 1058, 1012, 990, 988, 1004, 1002, 1004, 995, 953, 1015, 954, 1068, 944, 1083, 1032, 975, 1024, 1020, 988, 1050, 1047, 1043, 1009, 1047, 1036, 1029, 1020, 956, 1050, 1011, 977, 1049, 1035, 1011, 983, 1020, 1025, 1074, 1038, 1001, 1039, 1024, 1007, 1028, 1014, 1031, 1056, 983, 1034, 1062, 1028, 992, 1011, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[963, 1034, 963, 1016, 1011, 1013, 991, 1005, 1042, 992, 1013, 1016, 1031, 1047, 1033, 1024, 987, 1009, 1029, 989, 1069, 1056, 1022, 1024, 1029, 1045, 1036, 1018, 1019, 1034, 989, 1025, 1038, 1111, 1055, 1034, 1017, 1005, 1046, 1010, 1011, 1065, 1055, 1034, 1032, 992, 1007, 1069, 1027, 1022, 1002, 1062, 1023, 1054, 1009, 1006, 1000, 995, 1052, 1032, 974, 1016, 1011, 1096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[992, 1036, 1029, 987, 1023, 1015, 1014, 1036, 1061, 1045, 1033, 1043, 1024, 1032, 1067, 1022, 1039, 1037, 1066, 995, 1010, 1000, 1007, 1026, 1059, 1020, 1069, 1050, 1039, 1020, 979, 1025, 987, 986, 1010, 1049, 983, 1031, 1004, 1013, 1029, 1071, 1013, 1015, 1030, 978, 1003, 1007, 1012, 986, 1031, 1016, 1006, 1020, 1027, 1083, 1006, 1044, 1071, 1031, 1006, 1057, 1043, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1013, 985, 1041, 997, 1044, 1064, 1017, 1040, 1009, 1055, 1023, 1012, 1044, 1007, 1092, 1048, 1048, 998, 1029, 1036, 1053, 1048, 1065, 1064, 1049, 1044, 1018, 1014, 1052, 1014, 1083, 1033, 985, 1020, 1048, 1022, 996, 998, 1031, 972, 988, 1031, 1000, 1041, 1034, 1060, 1033, 1020, 1021, 974, 992, 1013, 1018, 1014, 1057, 996, 996, 988, 1027, 1034, 992, 1023, 982, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1009, 996, 1074, 1034, 1017, 1061, 1008, 1022, 1054, 1055, 1068, 1022, 1009, 1032, 978, 987, 1043, 1036, 1039, 1022, 1042, 975, 1000, 1090, 1033, 1032, 987, 1082, 1044, 1017, 1030, 1024, 998, 1025, 1007, 1047, 995, 1010, 1038, 1014, 1088, 1068, 998, 1010, 1042, 1024, 1005, 1027, 1021, 997, 978, 999, 975, 986, 1069, 1017, 1034, 977, 1027, 1046, 1000, 1015, 1067, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1012, 1065, 995, 1017, 1014, 1027, 1005, 1013, 1043, 1005, 1016, 1033, 1036, 968, 995, 1026, 1028, 1017, 1017, 1022, 991, 1023, 1020, 1030, 1008, 1058, 1020, 1058, 1042, 1036, 1022, 999, 994, 997, 1015, 1082, 1063, 1014, 1060, 999, 1050, 1004, 1061, 1030, 999, 1047, 1016, 1049, 997, 1029, 1040, 1033, 1002, 1006, 1054, 1020, 1026, 1045, 1031, 1003, 1071, 1040, 982, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1039, 1034, 1052, 1045, 970, 982, 1058, 1047, 1038, 1048, 1040, 1019, 1020, 1020, 1039, 1036, 1064, 1014, 1069, 1048, 1018, 1047, 1012, 1021, 1002, 978, 1015, 1038, 1019, 1006, 1014, 1033, 977, 983, 1024, 1035, 1022, 1018, 1037, 1037, 1048, 1031, 1026, 989, 1009, 979, 1060, 1051, 995, 1041, 1022, 1044, 996, 1008, 996, 998, 1044, 1056, 1036, 1014, 1045, 1022, 1012, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1042, 978, 1021, 995, 1044, 1045, 1035, 1010, 1, 0, 0, 0, 0, 0, 0, 0, 995, 988, 1079, 1009, 1022, 1073, 1050, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1020, 1036, 1020, 1003, 1050, 1015, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 977, 1024, 993, 1013, 1045, 1099, 957, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1012, 1031, 1013, 995, 1063, 958, 1022, 25, 20, 0, 0, 0, 0, 0, 0, 1032, 1040, 1005, 1062, 1024, 1006, 998, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1053, 1012, 1029, 1002, 1013, 1064, 967, 45, 2, 0, 0, 0, 0, 0, 0, 1008, 1040, 1046, 1000, 1072, 1044, 971, 1055, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8234, 8167, 8147, 8128, 8219, 8229, 8241] avg=8192.0000 max=8241 max_rank=7 min=8128 min_rank=4 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[963, 1034, 963, 1016, 1011, 1013, 991, 1005, 1, 0, 0, 0, 0, 0, 0, 0, 997, 992, 1013, 1016, 1031, 1047, 1033, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1009, 1029, 989, 1069, 1056, 1022, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1045, 1036, 1018, 1019, 1034, 989, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1111, 1055, 1034, 1017, 1005, 1046, 1010, 25, 19, 0, 0, 0, 0, 0, 0, 1011, 1038, 1055, 1034, 1032, 992, 1007, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1022, 1002, 1062, 1023, 1054, 1009, 1006, 45, 2, 0, 0, 0, 0, 0, 0, 1000, 995, 1052, 1032, 974, 1016, 1011, 1096, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7997, 8153, 8181, 8175, 8360, 8238, 8252, 8180] avg=8192.0000 max=8360 max_rank=4 min=7997 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=33030 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1020, 1068, 1036, 1072, 1061, 1084, 1059, 1028, 1, 0, 0, 0, 0, 0, 0, 0, 1002, 1116, 1058, 1012, 990, 988, 1004, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 995, 953, 1015, 954, 1068, 944, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 975, 1024, 1020, 988, 1050, 1047, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1047, 1036, 1029, 1020, 956, 1050, 1011, 24, 19, 0, 0, 0, 0, 0, 0, 977, 1023, 1035, 1011, 983, 1020, 1025, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1001, 1039, 1024, 1007, 1028, 1014, 1031, 45, 2, 0, 0, 0, 0, 0, 0, 1056, 983, 1034, 1062, 1028, 992, 1011, 1016, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8429, 8172, 8011, 8159, 8201, 8148, 8229, 8187] avg=8192.0000 max=8429 max_rank=0 min=8011 min_rank=2 imbalance_ratio=1.028931 max_load_reduction=57107 max_load_reduction_pct=0.871384 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[992, 1036, 1029, 987, 1023, 1015, 1014, 1036, 1, 0, 0, 0, 0, 0, 0, 0, 1015, 1045, 1033, 1043, 1024, 1032, 1067, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1037, 1066, 995, 1010, 1000, 1007, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1020, 1069, 1050, 1039, 1020, 979, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 987, 986, 1010, 1049, 983, 1031, 1004, 1013, 25, 19, 0, 0, 0, 0, 0, 0, 1029, 1044, 1013, 1015, 1030, 978, 1003, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 986, 1031, 1016, 1006, 1020, 1027, 1083, 46, 2, 0, 0, 0, 0, 0, 0, 1006, 1044, 1071, 1031, 1006, 1057, 1043, 988, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8281, 8176, 8240, 8107, 8119, 8229, 8251] avg=8192.0000 max=8281 max_rank=1 min=8107 min_rank=4 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1013, 985, 1041, 997, 1044, 1064, 1017, 1040, 1, 0, 0, 0, 0, 0, 0, 0, 966, 1055, 1023, 1012, 1044, 1007, 1092, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 998, 1029, 1036, 1053, 1048, 1065, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1044, 1018, 1014, 1052, 1014, 1083, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1020, 1048, 1022, 996, 998, 1031, 972, 24, 19, 0, 0, 0, 0, 0, 0, 988, 1006, 1000, 1041, 1034, 1060, 1033, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 974, 992, 1013, 1018, 1014, 1057, 996, 43, 1, 0, 0, 0, 0, 0, 0, 996, 988, 1027, 1034, 992, 1023, 982, 991, 4, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8247, 8336, 8286, 8115, 8182, 8129, 8039] avg=8192.0000 max=8336 max_rank=2 min=8039 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1009, 996, 1074, 1034, 1017, 1061, 1008, 1022, 1, 0, 0, 0, 0, 0, 0, 0, 1009, 1055, 1068, 1022, 1009, 1032, 978, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1036, 1039, 1022, 1042, 975, 1000, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1032, 987, 1082, 1044, 1017, 1030, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1025, 1007, 1047, 995, 1010, 1038, 1014, 25, 19, 0, 0, 0, 0, 0, 0, 1088, 1041, 998, 1010, 1042, 1024, 1005, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 997, 978, 999, 975, 986, 1069, 1017, 45, 2, 0, 0, 0, 0, 0, 0, 1034, 977, 1027, 1046, 1000, 1015, 1067, 1010, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8160, 8242, 8229, 8178, 8235, 8089, 8181] avg=8192.0000 max=8242 max_rank=2 min=8089 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1012, 1065, 995, 1017, 1014, 1027, 1005, 1013, 6, 0, 0, 0, 0, 0, 0, 0, 995, 1005, 1016, 1033, 1036, 968, 995, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1017, 1017, 1022, 991, 1023, 1020, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1058, 1020, 1058, 1042, 1036, 1022, 999, 0, 0, 0, 0, 0, 0, 0, 0, 994, 997, 1015, 1082, 1063, 1014, 1060, 999, 25, 23, 0, 0, 0, 0, 0, 0, 1050, 975, 1061, 1030, 999, 1047, 1016, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1029, 1040, 1033, 1002, 1006, 1054, 1020, 48, 4, 0, 0, 0, 0, 0, 0, 1026, 1045, 1031, 1003, 1071, 1040, 982, 1016, 8, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8074, 8134, 8214, 8272, 8227, 8233, 8228] avg=8192.0000 max=8272 max_rank=4 min=8074 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32960 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1039, 1034, 1052, 1045, 970, 982, 1058, 1047, 1, 0, 0, 0, 0, 0, 0, 0, 993, 1048, 1040, 1019, 1020, 1020, 1039, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1014, 1069, 1048, 1018, 1047, 1012, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 983, 978, 1015, 1038, 1019, 1006, 1014, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 977, 983, 1024, 1035, 1022, 1018, 1037, 1037, 24, 18, 0, 0, 0, 0, 0, 0, 1048, 1006, 1026, 989, 1009, 979, 1060, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1041, 1022, 1044, 996, 1008, 996, 998, 45, 1, 0, 0, 0, 0, 0, 0, 1044, 1056, 1036, 1014, 1045, 1022, 1012, 996, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8215, 8289, 8086, 8175, 8168, 8146, 8229] avg=8192.0000 max=8289 max_rank=2 min=8086 min_rank=3 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8192, 8320, 8320, 8192, 8192, 8320, 8192, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8448, 8320, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8320, 8448, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8064, 8192, 8320, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1016, 994, 984, 1086, 1036, 1041, 1034, 1016, 1045, 982, 978, 1040, 1013, 1010, 1071, 1011, 1011, 1068, 1044, 1001, 1017, 1103, 1034, 1067, 1086, 1002, 1023, 1026, 1040, 980, 998, 997, 1025, 1020, 990, 971, 1035, 1049, 982, 987, 1040, 1027, 996, 1066, 1048, 1041, 1013, 1031, 1016, 1054, 1048, 1012, 998, 1014, 1027, 1080, 1050, 982, 999, 1021, 1005, 1049, 1017, 989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1037, 1011, 1037, 986, 1015, 984, 1009, 1076, 972, 1032, 1026, 1008, 1049, 1010, 1005, 1022, 1058, 1024, 1002, 1007, 1082, 979, 1010, 965, 1044, 999, 1004, 1045, 1027, 1019, 1060, 998, 1056, 1041, 1012, 990, 1030, 1044, 1016, 979, 1056, 1025, 1050, 1037, 1022, 1033, 1009, 1011, 1047, 1068, 1041, 987, 962, 1084, 1021, 1077, 1037, 1031, 1005, 1064, 1012, 1060, 1017, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1077, 1021, 991, 992, 1016, 1016, 1064, 1021, 1023, 997, 1028, 1047, 1052, 971, 1019, 1050, 1033, 1082, 979, 1017, 1028, 1023, 974, 1046, 1010, 1063, 1027, 1067, 1015, 1007, 1006, 1002, 1028, 1054, 1020, 1040, 1002, 1058, 1000, 1000, 1013, 1006, 983, 1033, 1023, 1049, 1038, 1020, 1024, 1025, 1098, 1010, 1033, 1050, 1027, 994, 1053, 1007, 993, 1019, 1031, 1070, 1006, 965] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1069, 1067, 1044, 1029, 998, 1045, 1030, 1029, 985, 1008, 1037, 1004, 1015, 994, 1012, 1023, 1024, 1044, 1005, 1034, 1026, 1012, 949, 1075, 955, 992, 1044, 1015, 1021, 1022, 1042, 971, 1050, 1023, 1022, 1090, 1022, 1037, 970, 1077, 1031, 1000, 1029, 1019, 1030, 970, 984, 1020, 1026, 1021, 1011, 1014, 1075, 1015, 1041, 1049, 1050, 1033, 1100, 1027, 1020, 1015, 1047, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1018, 974, 999, 1010, 1019, 971, 1035, 954, 1003, 995, 1028, 1028, 1001, 1064, 1005, 1035, 1055, 1025, 1003, 1106, 1069, 1021, 1030, 994, 1018, 1088, 1072, 1064, 1032, 1017, 1041, 1018, 1035, 1085, 1029, 984, 1022, 1065, 1030, 999, 1004, 999, 1007, 1032, 1063, 962, 1025, 1031, 999, 977, 1066, 1028, 1002, 1006, 1059, 1026, 1063, 1036, 1034, 980, 1024, 1016, 1049, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1088, 1009, 984, 972, 1051, 1049, 1021, 1036, 1016, 1018, 1067, 993, 1093, 1032, 1022, 1023, 1048, 973, 1001, 1049, 1010, 1008, 1054, 986, 1050, 994, 994, 988, 989, 1029, 1029, 1057, 1023, 1023, 1015, 1014, 1040, 1024, 985, 1025, 1046, 1052, 1035, 1057, 1089, 1027, 954, 1013, 1051, 1036, 981, 996, 970, 1047, 1004, 1051, 1055, 1048, 1064, 1023, 1031, 1033, 978, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1037, 993, 1043, 1034, 1052, 1003, 1005, 1028, 1031, 992, 1004, 1020, 1059, 992, 1023, 1019, 999, 1037, 1021, 1006, 1037, 1067, 1030, 1036, 1021, 1023, 1015, 1042, 989, 1073, 1047, 1022, 1020, 1074, 1069, 1019, 1019, 1003, 1028, 1014, 1045, 1042, 1027, 1034, 1061, 1006, 971, 975, 1041, 1008, 1031, 1049, 1051, 968, 1033, 1001, 1010, 1013, 1034, 997, 1086, 999, 985, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[981, 997, 1070, 1026, 1045, 1088, 1018, 1022, 1036, 1065, 1033, 1016, 1045, 1034, 995, 1076, 1049, 1028, 980, 1046, 1064, 1004, 997, 993, 1028, 1004, 1085, 1022, 1028, 1039, 1061, 1054, 1007, 1051, 1046, 1009, 1078, 1015, 996, 1025, 1058, 1025, 988, 1010, 1008, 998, 1003, 986, 997, 1054, 1023, 1063, 984, 999, 1043, 1012, 965, 977, 1039, 953, 1021, 998, 1047, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1016, 994, 984, 1086, 1036, 1041, 1034, 1016, 4, 0, 0, 0, 0, 0, 0, 0, 1045, 982, 978, 1040, 1013, 1010, 1071, 1011, 13, 2, 0, 0, 0, 0, 0, 0, 1011, 1068, 1044, 1001, 1004, 1103, 1034, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 1002, 1023, 1026, 1040, 980, 986, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1014, 990, 971, 1035, 1049, 982, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1027, 996, 1066, 1048, 1041, 1013, 1031, 18, 0, 0, 0, 0, 0, 0, 0, 1016, 1054, 1028, 1012, 998, 1014, 1027, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 982, 999, 1021, 1005, 1049, 1017, 989, 12, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8165, 8332, 8140, 8053, 8280, 8229, 8126] avg=8192.0000 max=8332 max_rank=2 min=8053 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1018, 974, 999, 1010, 1019, 971, 1035, 954, 4, 0, 0, 0, 0, 0, 0, 0, 1003, 995, 1028, 1028, 1001, 1064, 1005, 1035, 14, 2, 0, 0, 0, 0, 0, 0, 1055, 1025, 1003, 1106, 1055, 1021, 1030, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1088, 1072, 1064, 1032, 1017, 1029, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1079, 1029, 984, 1022, 1065, 1030, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 999, 1007, 1032, 1063, 962, 1025, 1031, 19, 0, 0, 0, 0, 0, 0, 0, 999, 977, 1045, 1028, 1002, 1006, 1059, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1036, 1034, 980, 1024, 1016, 1049, 1007, 12, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7984, 8175, 8289, 8338, 8243, 8142, 8142, 8223] avg=8192.0000 max=8338 max_rank=3 min=7984 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1037, 1011, 1037, 986, 1015, 984, 1009, 1076, 4, 0, 0, 0, 0, 0, 0, 0, 972, 1032, 1026, 1008, 1049, 1010, 1005, 1022, 14, 2, 0, 0, 0, 0, 0, 0, 1058, 1024, 1002, 1007, 1068, 979, 1010, 965, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 999, 1004, 1045, 1027, 1019, 1047, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1035, 1012, 990, 1030, 1044, 1016, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1025, 1050, 1037, 1022, 1033, 1009, 1011, 18, 0, 0, 0, 0, 0, 0, 0, 1047, 1068, 1021, 987, 962, 1084, 1021, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1031, 1005, 1064, 1012, 1060, 1017, 1010, 13, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8140, 8113, 8183, 8162, 8261, 8267, 8251] avg=8192.0000 max=8267 max_rank=6 min=8113 min_rank=2 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32941 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1077, 1021, 991, 992, 1016, 1016, 1064, 1021, 4, 0, 0, 0, 0, 0, 0, 0, 1023, 997, 1028, 1047, 1052, 971, 1019, 1050, 13, 2, 0, 0, 0, 0, 0, 0, 1033, 1082, 979, 1017, 1015, 1023, 974, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1063, 1027, 1067, 1015, 1007, 994, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1048, 1020, 1040, 1002, 1058, 1000, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1006, 983, 1033, 1023, 1049, 1038, 1020, 19, 0, 0, 0, 0, 0, 0, 0, 1024, 1025, 1077, 1010, 1033, 1050, 1027, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1007, 993, 1019, 1031, 1070, 1006, 965, 12, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8202, 8169, 8185, 8196, 8184, 8240, 8158] avg=8192.0000 max=8240 max_rank=6 min=8158 min_rank=7 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1069, 1067, 1044, 1029, 998, 1045, 1030, 1029, 4, 0, 0, 0, 0, 0, 0, 0, 985, 1008, 1037, 1004, 1015, 994, 1012, 1023, 13, 2, 0, 0, 0, 0, 0, 0, 1024, 1044, 1005, 1034, 1013, 1012, 949, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 955, 992, 1044, 1015, 1021, 1022, 1030, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1017, 1022, 1090, 1022, 1037, 970, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1000, 1029, 1019, 1030, 970, 984, 1020, 18, 0, 0, 0, 0, 0, 0, 0, 1026, 1021, 991, 1014, 1075, 1015, 1041, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1033, 1100, 1027, 1020, 1015, 1047, 998, 12, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8315, 8093, 8156, 8050, 8285, 8101, 8232, 8304] avg=8192.0000 max=8315 max_rank=0 min=8050 min_rank=3 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32922 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[981, 997, 1070, 1026, 1045, 1088, 1018, 1022, 4, 0, 0, 0, 0, 0, 0, 0, 1036, 1065, 1033, 1016, 1045, 1034, 995, 1076, 14, 2, 0, 0, 0, 0, 0, 0, 1049, 1028, 980, 1046, 1050, 1004, 997, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1004, 1085, 1022, 1028, 1039, 1048, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1045, 1046, 1009, 1078, 1015, 996, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1025, 988, 1010, 1008, 998, 1003, 986, 18, 0, 0, 0, 0, 0, 0, 0, 997, 1054, 1003, 1063, 984, 999, 1043, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 965, 977, 1039, 953, 1021, 998, 1047, 1029, 13, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8316, 8147, 8308, 8221, 8094, 8155, 8044] avg=8192.0000 max=8316 max_rank=1 min=8044 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32514 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1088, 1009, 984, 972, 1051, 1049, 1021, 1036, 5, 0, 0, 0, 0, 0, 0, 0, 1016, 1018, 1067, 993, 1093, 1032, 1022, 1023, 18, 5, 2, 0, 0, 0, 0, 0, 1048, 973, 1001, 1049, 992, 1008, 1054, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 994, 994, 988, 989, 1029, 1011, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1010, 1015, 1014, 1040, 1024, 985, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1052, 1035, 1057, 1089, 1027, 954, 1013, 22, 0, 0, 0, 0, 0, 0, 0, 1051, 1036, 954, 996, 970, 1047, 1004, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1048, 1064, 1023, 1031, 1033, 978, 1013, 16, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8289, 8111, 8112, 8136, 8295, 8109, 8269] avg=8192.0000 max=8295 max_rank=5 min=8109 min_rank=6 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1037, 993, 1043, 1034, 1052, 1003, 1005, 1028, 4, 0, 0, 0, 0, 0, 0, 0, 1031, 992, 1004, 1020, 1059, 992, 1023, 1019, 14, 2, 0, 0, 0, 0, 0, 0, 999, 1037, 1021, 1006, 1023, 1067, 1030, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1023, 1015, 1042, 989, 1073, 1035, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1068, 1069, 1019, 1019, 1003, 1028, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1042, 1027, 1034, 1061, 1006, 971, 975, 18, 0, 0, 0, 0, 0, 0, 0, 1041, 1008, 1011, 1049, 1051, 968, 1033, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1013, 1034, 997, 1086, 999, 985, 1023, 12, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8156, 8219, 8220, 8240, 8179, 8162, 8161] avg=8192.0000 max=8240 max_rank=4 min=8156 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8448, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8320, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8320, 8192, 8448, 8192, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8448, 8192, 8064, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8192, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1023, 1053, 1034, 1029, 1067, 1061, 1011, 981, 990, 1048, 1053, 1014, 986, 1044, 977, 995, 1045, 1061, 1046, 993, 1032, 1053, 1019, 999, 984, 1091, 1011, 995, 1078, 996, 1014, 999, 1016, 1063, 1046, 1006, 995, 1009, 988, 1043, 1056, 1034, 1007, 1060, 983, 1017, 1029, 978, 1003, 1025, 1032, 1060, 1044, 1036, 1020, 1017, 998, 1032, 1013, 989, 1072, 997, 1017, 1069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[996, 1008, 980, 975, 1032, 1003, 1089, 989, 1039, 1064, 999, 1019, 1010, 984, 1018, 1065, 1022, 1042, 1064, 1048, 1062, 1017, 1004, 1032, 1051, 984, 1052, 974, 992, 1016, 1024, 988, 1008, 963, 1025, 1039, 1035, 1008, 990, 1028, 1001, 1083, 1044, 1018, 1039, 1029, 1039, 1046, 1021, 1035, 1044, 1038, 1057, 995, 1044, 1013, 1011, 1029, 1002, 1011, 1050, 1048, 1033, 1068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1027, 1013, 976, 1037, 996, 1054, 1004, 982, 1005, 1028, 1059, 1076, 1070, 992, 1027, 1025, 1012, 975, 1004, 1005, 1018, 1064, 1041, 1023, 1022, 996, 1009, 1033, 1020, 1004, 1032, 1015, 1054, 1006, 1007, 1058, 1027, 993, 1041, 1000, 1026, 1046, 981, 1030, 1020, 1055, 989, 1071, 1050, 1011, 1040, 1057, 1047, 998, 1013, 1031, 1023, 1026, 1030, 1041, 1014, 1011, 1045, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1056, 1046, 1049, 1049, 1011, 1084, 1019, 946, 990, 991, 1050, 979, 1071, 992, 1026, 1039, 963, 1075, 1010, 1014, 983, 1005, 1032, 1043, 1054, 1007, 1050, 1057, 1039, 1057, 981, 1080, 1022, 1089, 978, 1027, 1067, 1064, 1002, 1006, 1034, 1011, 1001, 1033, 1004, 1085, 1003, 1000, 1094, 948, 1037, 1043, 1003, 1055, 992, 1026, 996, 986, 998, 985, 1007, 1006, 1083, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[999, 1010, 976, 1025, 1041, 941, 1021, 980, 1021, 966, 1007, 1089, 1009, 996, 1010, 996, 1071, 1059, 1045, 1099, 1042, 1011, 1012, 1012, 1069, 1071, 1018, 1029, 1038, 1000, 1100, 1019, 1036, 1026, 1035, 1035, 1013, 972, 1052, 988, 1053, 1045, 1025, 986, 1025, 1015, 1001, 1035, 1004, 1033, 1005, 1033, 1052, 1054, 1010, 1035, 1026, 1041, 1038, 1000, 1033, 990, 1043, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1085, 1024, 1005, 938, 1048, 1051, 1055, 989, 1035, 1052, 1051, 1055, 1000, 1042, 1015, 1031, 1007, 1027, 1007, 1056, 1026, 1042, 1032, 1078, 1006, 1023, 1026, 1020, 1028, 979, 1028, 1026, 978, 979, 1074, 1001, 1023, 1006, 1038, 1065, 1059, 1039, 1042, 1005, 991, 997, 1030, 956, 1056, 1019, 1037, 1042, 1045, 1017, 982, 991, 1021, 1069, 995, 1013, 1015, 1001, 1003, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[988, 1038, 1042, 988, 1045, 981, 1004, 1043, 1034, 1045, 997, 1051, 1042, 1028, 1035, 1045, 982, 1004, 1026, 1051, 1044, 1040, 961, 1014, 1101, 1004, 954, 1059, 1026, 1033, 1023, 1061, 1048, 1059, 1005, 1013, 1045, 1005, 1068, 1012, 1070, 1007, 1049, 1003, 1028, 1017, 1010, 1026, 1012, 1002, 1002, 1026, 996, 1023, 974, 1045, 986, 1022, 994, 1050, 1014, 1050, 1072, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1018, 1060, 987, 988, 991, 1036, 1013, 997, 980, 998, 1042, 1044, 994, 1053, 1054, 1036, 1037, 1036, 1044, 1012, 1037, 1034, 1020, 988, 1046, 1025, 1034, 1057, 1008, 953, 1059, 992, 1031, 1050, 980, 994, 1045, 1055, 1010, 1005, 1040, 1019, 1033, 1093, 1006, 972, 1023, 1077, 1048, 1023, 1077, 1030, 1019, 1017, 1058, 968, 995, 1000, 1033, 1018, 1099, 1028, 1035, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1023, 1053, 1034, 1029, 1067, 1061, 1011, 981, 28, 16, 11, 0, 0, 0, 0, 0, 990, 1048, 1053, 1010, 986, 1044, 977, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1033, 1046, 993, 1032, 1053, 1019, 999, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1091, 1011, 995, 1078, 996, 1014, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1063, 1046, 1006, 995, 1009, 988, 1043, 9, 0, 0, 0, 0, 0, 0, 0, 1045, 1034, 1007, 1060, 983, 1017, 1029, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1025, 1032, 1044, 1044, 1036, 1020, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1032, 1013, 989, 1072, 997, 1017, 1069, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8314, 8103, 8220, 8159, 8175, 8153, 8221, 8191] avg=8192.0000 max=8314 max_rank=0 min=8103 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1027, 1013, 976, 1037, 996, 1054, 1004, 982, 26, 16, 11, 0, 0, 0, 0, 0, 1005, 1028, 1059, 1072, 1070, 992, 1027, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 949, 1004, 1005, 1018, 1064, 1041, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 996, 1009, 1033, 1020, 1004, 1032, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1006, 1007, 1058, 1027, 993, 1041, 1000, 10, 0, 0, 0, 0, 0, 0, 0, 1015, 1046, 981, 1030, 1020, 1055, 989, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1011, 1040, 1041, 1047, 998, 1013, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1026, 1030, 1041, 1014, 1011, 1045, 1051, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8278, 8116, 8121, 8196, 8207, 8231, 8245] avg=8192.0000 max=8278 max_rank=1 min=8116 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32879 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[996, 1008, 980, 975, 1032, 1003, 1089, 989, 28, 15, 11, 0, 0, 0, 0, 0, 1039, 1064, 999, 1015, 1010, 984, 1018, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1014, 1064, 1048, 1062, 1017, 1004, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 984, 1052, 974, 992, 1016, 1024, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 963, 1025, 1039, 1035, 1008, 990, 1028, 10, 0, 0, 0, 0, 0, 0, 0, 990, 1083, 1044, 1018, 1039, 1029, 1039, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1035, 1044, 1023, 1057, 995, 1044, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1029, 1002, 1011, 1050, 1048, 1033, 1068, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8194, 8263, 8071, 8106, 8288, 8232, 8256] avg=8192.0000 max=8288 max_rank=5 min=8071 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1056, 1046, 1049, 1049, 1011, 1084, 1019, 946, 29, 16, 11, 0, 0, 0, 0, 0, 990, 991, 1050, 975, 1071, 992, 1026, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1046, 1010, 1014, 983, 1005, 1032, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1007, 1050, 1057, 1039, 1057, 981, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1089, 978, 1027, 1067, 1064, 1002, 1006, 10, 0, 0, 0, 0, 0, 0, 0, 1023, 1011, 1001, 1033, 1004, 1085, 1003, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 948, 1037, 1027, 1003, 1055, 992, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 996, 986, 998, 985, 1007, 1006, 1083, 1003, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8316, 8134, 8096, 8315, 8265, 8160, 8182, 8068] avg=8192.0000 max=8316 max_rank=0 min=8068 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32675 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1018, 1060, 987, 988, 991, 1036, 1013, 997, 28, 15, 11, 0, 0, 0, 0, 0, 980, 998, 1042, 1040, 994, 1053, 1054, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1008, 1044, 1012, 1037, 1034, 1020, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1025, 1034, 1057, 1008, 953, 1059, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1050, 980, 994, 1045, 1055, 1010, 1005, 10, 0, 0, 0, 0, 0, 0, 0, 1029, 1019, 1033, 1093, 1006, 972, 1023, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1023, 1077, 1015, 1019, 1017, 1058, 968, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1000, 1033, 1018, 1099, 1028, 1035, 982, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8197, 8180, 8164, 8180, 8252, 8225, 8194] avg=8192.0000 max=8252 max_rank=5 min=8144 min_rank=0 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1085, 1024, 1005, 938, 1048, 1051, 1055, 989, 28, 16, 11, 0, 0, 0, 0, 0, 1035, 1052, 1051, 1051, 1000, 1042, 1015, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 999, 1007, 1056, 1026, 1042, 1032, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1023, 1026, 1020, 1028, 979, 1028, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 978, 979, 1074, 1001, 1023, 1006, 1038, 1065, 9, 0, 0, 0, 0, 0, 0, 0, 1048, 1039, 1042, 1005, 991, 997, 1030, 956, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1019, 1037, 1026, 1045, 1017, 982, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1069, 995, 1013, 1015, 1001, 1003, 1060, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8277, 8247, 8127, 8173, 8108, 8173, 8181] avg=8192.0000 max=8277 max_rank=1 min=8108 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[999, 1010, 976, 1025, 1041, 941, 1021, 980, 32, 19, 16, 2, 0, 0, 0, 0, 1021, 966, 1007, 1075, 1009, 996, 1010, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1027, 1045, 1099, 1042, 1011, 1012, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1071, 1018, 1029, 1038, 1000, 1100, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1026, 1035, 1035, 1013, 972, 1052, 988, 14, 5, 0, 0, 0, 0, 0, 0, 1037, 1045, 1025, 986, 1025, 1015, 1001, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1033, 1005, 1014, 1052, 1054, 1010, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1041, 1038, 1000, 1033, 990, 1043, 1015, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8062, 8080, 8319, 8328, 8176, 8169, 8207, 8195] avg=8192.0000 max=8328 max_rank=3 min=8062 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[988, 1038, 1042, 988, 1045, 981, 1004, 1043, 27, 15, 11, 0, 0, 0, 0, 0, 1034, 1045, 997, 1047, 1042, 1028, 1035, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 982, 977, 1026, 1051, 1044, 1040, 961, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 1004, 954, 1059, 1026, 1033, 1023, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1059, 1005, 1013, 1045, 1005, 1068, 1012, 10, 0, 0, 0, 0, 0, 0, 0, 1059, 1007, 1049, 1003, 1028, 1017, 1010, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1002, 1002, 1011, 996, 1023, 974, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1022, 994, 1050, 1014, 1050, 1072, 1014, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8273, 8095, 8251, 8265, 8199, 8065, 8206] avg=8192.0000 max=8273 max_rank=1 min=8065 min_rank=6 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8064, 8064, 8320, 8320, 8320, 7936, 256, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8064, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8192, 8320, 8192, 8448, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1019, 1010, 1062, 976, 1028, 1016, 997, 1032, 948, 1064, 1013, 1090, 1002, 1021, 995, 1040, 960, 998, 1017, 1028, 1024, 1057, 1003, 1033, 1060, 1042, 975, 1047, 997, 1059, 1033, 1006, 1012, 1080, 1096, 987, 995, 1008, 1058, 1082, 1020, 1040, 1073, 1041, 1013, 1040, 999, 997, 1018, 1005, 980, 1047, 1036, 1034, 1031, 1054, 1009, 1022, 1027, 1023, 1074, 994, 999, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1034, 998, 1050, 1073, 1027, 1005, 1007, 1037, 1031, 986, 999, 1053, 967, 970, 1029, 1042, 1036, 989, 1035, 1019, 1004, 1000, 1017, 1011, 1006, 1018, 990, 1043, 1005, 1066, 1068, 1053, 1008, 1005, 1033, 1013, 1025, 986, 996, 1103, 1025, 1054, 1025, 1064, 1027, 1041, 1064, 961, 1054, 999, 1001, 1012, 1078, 1016, 1019, 1031, 1092, 1034, 1018, 1049, 1036, 1012, 986, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1035, 993, 1035, 991, 1009, 990, 1038, 1004, 1038, 1053, 1015, 1051, 1018, 1002, 990, 1080, 1036, 1026, 1049, 1021, 1032, 1004, 1053, 1025, 1023, 1064, 1043, 1015, 1024, 1042, 996, 1077, 1033, 992, 995, 1048, 1013, 1018, 1029, 1085, 1039, 1013, 1051, 1002, 1045, 1015, 984, 1024, 1006, 1075, 1002, 1056, 992, 1017, 975, 1013, 1055, 1019, 984, 1007, 1043, 985, 1008, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1089, 1038, 1042, 1016, 1048, 994, 1015, 1013, 980, 995, 999, 1044, 1023, 1013, 1020, 1022, 1024, 1037, 1040, 1003, 1058, 1014, 1001, 1017, 1052, 1061, 1071, 1072, 1025, 1045, 1042, 1010, 997, 1006, 1032, 993, 1045, 1001, 975, 1050, 996, 996, 1079, 1045, 1035, 983, 964, 1025, 1001, 1012, 1044, 1042, 1020, 1047, 1088, 984, 1008, 1022, 1077, 1014, 996, 1041, 991, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1027, 1051, 1019, 1040, 1035, 998, 1045, 981, 1021, 1013, 972, 1013, 1064, 1047, 1000, 1007, 1044, 1000, 1025, 1014, 1022, 1021, 1004, 999, 1053, 1027, 1010, 1029, 1055, 1016, 1017, 1024, 993, 1016, 1019, 993, 1028, 1032, 1107, 1008, 1020, 983, 1104, 1036, 1011, 1069, 1025, 1010, 990, 1019, 1063, 1017, 1025, 1026, 1013, 1045, 953, 1002, 1033, 1024, 1066, 1064, 1000, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1027, 1058, 994, 1028, 988, 982, 1001, 996, 1036, 986, 1036, 1021, 1061, 1030, 1017, 1045, 1076, 1011, 1045, 979, 951, 1049, 987, 1027, 1011, 1033, 1037, 1060, 1033, 1032, 1038, 1011, 1067, 1027, 1014, 1044, 1007, 1017, 1010, 1071, 1050, 985, 1025, 1041, 1041, 1034, 990, 1068, 1056, 996, 978, 1012, 1007, 993, 1049, 1039, 1036, 999, 1004, 1013, 1042, 1036, 1067, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1060, 1012, 988, 1021, 1081, 1008, 1002, 1023, 1037, 996, 1014, 1059, 1024, 999, 1025, 1021, 984, 1016, 1078, 1076, 1042, 1010, 1012, 1044, 1000, 1074, 1011, 996, 1053, 996, 1020, 1022, 1051, 1056, 1083, 994, 1036, 1016, 1046, 964, 1051, 1010, 1061, 1039, 1022, 1009, 1069, 1025, 994, 1008, 1030, 1012, 1007, 1030, 1009, 1023, 1012, 1052, 1033, 995, 1024, 996, 985, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1061, 1024, 1059, 1044, 1006, 1011, 1023, 1054, 1030, 999, 1012, 966, 1009, 1049, 1019, 1082, 1019, 1023, 1090, 1025, 1022, 1021, 1014, 989, 1040, 1058, 1037, 1001, 1045, 1039, 1019, 1039, 1034, 1016, 1026, 1034, 1040, 927, 1001, 1054, 1044, 1032, 1050, 1064, 1008, 1025, 1027, 1003, 1081, 989, 1033, 973, 1002, 1012, 999, 1054, 996, 1014, 1022, 998, 1032, 997, 1035, 985] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1019, 1010, 1062, 976, 1028, 1016, 997, 1032, 8, 0, 0, 0, 0, 0, 0, 0, 948, 1064, 1013, 1090, 1002, 1021, 995, 1040, 28, 0, 0, 0, 0, 0, 0, 0, 960, 998, 1017, 1028, 1024, 1057, 1003, 1033, 15, 3, 0, 0, 0, 0, 0, 0, 1060, 979, 975, 1047, 997, 1059, 1033, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1080, 1096, 987, 995, 1008, 1058, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1040, 1039, 1041, 1013, 1040, 999, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1005, 980, 1047, 1036, 1034, 1031, 1054, 19, 0, 0, 0, 0, 0, 0, 0, 1009, 1022, 1027, 1023, 1074, 994, 999, 990, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8201, 8138, 8156, 8307, 8189, 8224, 8173] avg=8192.0000 max=8307 max_rank=4 min=8138 min_rank=2 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1089, 1038, 1042, 1016, 1048, 994, 1015, 1013, 8, 0, 0, 0, 0, 0, 0, 0, 980, 995, 999, 1044, 1023, 1013, 1020, 1022, 29, 0, 0, 0, 0, 0, 0, 0, 1024, 1037, 1040, 1003, 1058, 1014, 1001, 1017, 15, 3, 0, 0, 0, 0, 0, 0, 1052, 995, 1071, 1072, 1025, 1045, 1042, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1006, 1032, 993, 1045, 1001, 975, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 996, 996, 1045, 1045, 1035, 983, 964, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1012, 1044, 1042, 1020, 1047, 1088, 984, 19, 1, 0, 0, 0, 0, 0, 0, 1008, 1022, 1077, 1014, 996, 1041, 991, 1004, 36, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8125, 8212, 8312, 8088, 8089, 8258, 8189] avg=8192.0000 max=8312 max_rank=3 min=8088 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1034, 998, 1050, 1073, 1027, 1005, 1007, 1037, 8, 0, 0, 0, 0, 0, 0, 0, 1031, 986, 999, 1053, 967, 970, 1029, 1042, 28, 0, 0, 0, 0, 0, 0, 0, 1036, 989, 1035, 1019, 1004, 1000, 1017, 1011, 14, 3, 0, 0, 0, 0, 0, 0, 1006, 955, 990, 1043, 1005, 1066, 1068, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1005, 1033, 1013, 1025, 986, 996, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1054, 993, 1064, 1027, 1041, 1064, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 999, 1001, 1012, 1078, 1016, 1019, 1031, 18, 0, 0, 0, 0, 0, 0, 0, 1092, 1034, 1018, 1049, 1036, 1012, 986, 1001, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8105, 8128, 8186, 8158, 8229, 8228, 8263] avg=8192.0000 max=8263 max_rank=7 min=8105 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1035, 993, 1035, 991, 1009, 990, 1038, 1004, 8, 0, 0, 0, 0, 0, 0, 0, 1038, 1053, 1015, 1051, 1018, 1002, 990, 1080, 29, 0, 0, 0, 0, 0, 0, 0, 1036, 1026, 1049, 1021, 1032, 1004, 1053, 1025, 14, 3, 0, 0, 0, 0, 0, 0, 1023, 998, 1043, 1015, 1024, 1042, 996, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 992, 995, 1048, 1013, 1018, 1029, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1013, 1018, 1002, 1045, 1015, 984, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1075, 1002, 1056, 992, 1017, 975, 1013, 19, 1, 0, 0, 0, 0, 0, 0, 1055, 1019, 984, 1007, 1043, 985, 1008, 1041, 36, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8276, 8263, 8218, 8202, 8140, 8156, 8178] avg=8192.0000 max=8276 max_rank=1 min=8103 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1027, 1058, 994, 1028, 988, 982, 1001, 996, 8, 0, 0, 0, 0, 0, 0, 0, 1036, 986, 1036, 1021, 1061, 1030, 1017, 1045, 28, 0, 0, 0, 0, 0, 0, 0, 1076, 1011, 1045, 979, 951, 1049, 987, 1027, 14, 3, 0, 0, 0, 0, 0, 0, 1011, 970, 1037, 1060, 1033, 1032, 1038, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1027, 1014, 1044, 1007, 1017, 1010, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 985, 993, 1041, 1041, 1034, 990, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 996, 978, 1012, 1007, 993, 1049, 1039, 18, 0, 0, 0, 0, 0, 0, 0, 1036, 999, 1004, 1013, 1042, 1036, 1067, 1032, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8260, 8142, 8192, 8246, 8202, 8148, 8264] avg=8192.0000 max=8264 max_rank=7 min=8082 min_rank=0 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1027, 1051, 1019, 1040, 1035, 998, 1045, 981, 8, 0, 0, 0, 0, 0, 0, 0, 1021, 1013, 972, 1013, 1064, 1047, 1000, 1007, 28, 0, 0, 0, 0, 0, 0, 0, 1044, 1000, 1025, 1014, 1022, 1021, 1004, 999, 15, 3, 0, 0, 0, 0, 0, 0, 1053, 964, 1010, 1029, 1055, 1016, 1017, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1016, 1019, 993, 1028, 1032, 1107, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 983, 1069, 1036, 1011, 1069, 1025, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1019, 1063, 1017, 1025, 1026, 1013, 1045, 20, 0, 0, 0, 0, 0, 0, 0, 953, 1002, 1033, 1024, 1066, 1064, 1000, 1049, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8165, 8147, 8168, 8185, 8223, 8218, 8226] avg=8192.0000 max=8226 max_rank=7 min=8147 min_rank=2 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32852 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1061, 1024, 1059, 1044, 1006, 1011, 1023, 1054, 8, 0, 0, 0, 0, 0, 0, 0, 1030, 999, 1012, 966, 1009, 1049, 1019, 1082, 29, 0, 0, 0, 0, 0, 0, 0, 1019, 1023, 1090, 1025, 1022, 1021, 1014, 989, 14, 3, 0, 0, 0, 0, 0, 0, 1040, 992, 1037, 1001, 1045, 1039, 1019, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1016, 1026, 1034, 1040, 927, 1001, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1032, 1017, 1064, 1008, 1025, 1027, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 989, 1033, 973, 1002, 1012, 999, 1054, 19, 1, 0, 0, 0, 0, 0, 0, 996, 1014, 1022, 998, 1032, 997, 1035, 985, 36, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 8195, 8220, 8212, 8121, 8220, 8163, 8115] avg=8192.0000 max=8290 max_rank=0 min=8115 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32619 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1060, 1012, 988, 1021, 1081, 1008, 1002, 1023, 12, 0, 0, 0, 0, 0, 0, 0, 1037, 996, 1014, 1059, 1024, 999, 1025, 1021, 34, 0, 0, 0, 0, 0, 0, 0, 984, 1016, 1078, 1076, 1042, 1010, 1012, 1044, 19, 5, 0, 0, 0, 0, 0, 0, 1000, 994, 1011, 996, 1053, 996, 1020, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1056, 1083, 994, 1036, 1016, 1046, 947, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1010, 1019, 1039, 1022, 1009, 1069, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1008, 1030, 1012, 1007, 1030, 1009, 1023, 23, 5, 0, 0, 0, 0, 0, 0, 1012, 1052, 1033, 995, 1024, 996, 985, 990, 41, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8209, 8286, 8092, 8229, 8244, 8141, 8128] avg=8192.0000 max=8286 max_rank=2 min=8092 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8448, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 7936, 8192, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8064, 8320, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8448, 8192, 8320, 8192, 8320, 8064, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8064, 8320, 8192, 8192, 8192, 8448, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8448, 8192, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1038, 1025, 1039, 1075, 1049, 1035, 1019, 1069, 1000, 1006, 1045, 985, 1040, 977, 1006, 1009, 980, 985, 1027, 1031, 958, 1057, 1047, 1018, 1030, 1055, 1044, 1045, 1027, 958, 1005, 1013, 1051, 1099, 988, 1046, 1036, 1018, 1043, 997, 1051, 990, 1039, 1011, 1006, 988, 1085, 1044, 1033, 1023, 1000, 1055, 989, 1033, 1065, 998, 995, 1018, 1036, 1041, 980, 1004, 1035, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1029, 999, 1044, 1040, 1052, 1020, 993, 998, 1009, 1042, 1028, 996, 1003, 1040, 1026, 1026, 1042, 996, 1032, 1023, 1020, 987, 1018, 1009, 1066, 1028, 1024, 985, 1006, 1041, 1052, 1039, 1073, 1034, 1045, 1017, 1043, 1058, 1005, 1021, 1014, 1005, 1077, 1053, 1001, 1014, 1034, 1012, 1021, 1009, 1012, 1075, 1005, 1009, 1027, 1025, 978, 983, 1099, 967, 1000, 990, 1043, 1074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[986, 985, 1036, 1046, 1049, 1022, 1028, 1029, 1023, 1067, 1013, 1005, 1027, 983, 1041, 1070, 994, 1032, 1009, 1055, 1038, 1015, 1030, 1042, 1006, 1054, 993, 990, 1001, 1020, 1038, 1027, 1024, 1042, 1033, 1019, 1072, 1054, 999, 1045, 1019, 1026, 1036, 981, 1032, 1016, 1009, 1061, 997, 1025, 1029, 1069, 977, 1045, 1061, 976, 1066, 956, 1020, 1044, 1031, 1000, 987, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[980, 1022, 1025, 1013, 993, 998, 1051, 1000, 1083, 1018, 1033, 1048, 984, 1018, 970, 980, 1063, 1008, 1045, 1030, 1023, 1053, 1002, 1038, 1027, 1043, 1044, 1039, 987, 1017, 1031, 994, 1033, 968, 1050, 1059, 999, 1058, 1069, 1048, 1018, 1007, 996, 963, 971, 1034, 997, 1016, 1024, 1047, 1039, 1061, 1015, 1049, 1033, 1024, 1047, 1019, 1024, 1060, 1047, 991, 1053, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[969, 1017, 1077, 1016, 1046, 977, 1037, 1017, 998, 1025, 1045, 1022, 1025, 990, 1062, 1040, 1026, 1072, 997, 1031, 1063, 1019, 1059, 1046, 991, 1049, 1037, 1027, 1046, 1017, 1004, 1031, 1048, 1011, 984, 1043, 1059, 964, 1024, 950, 992, 988, 969, 993, 996, 1030, 1042, 971, 1027, 1034, 1035, 1040, 1062, 1017, 1051, 1051, 982, 1044, 1028, 1037, 1049, 1047, 1093, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1056, 1032, 1001, 1052, 1022, 1021, 1029, 1027, 1049, 1021, 1052, 1038, 996, 993, 1008, 1046, 963, 1056, 1038, 1009, 1030, 1021, 987, 1045, 981, 1034, 1020, 1054, 1027, 991, 992, 1039, 988, 1054, 1004, 1029, 1007, 1077, 1029, 1057, 1024, 1050, 1027, 1061, 983, 1036, 1017, 1009, 1008, 1008, 990, 993, 1036, 995, 1054, 1006, 1034, 1023, 1064, 996, 1058, 1024, 1049, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1003, 1009, 1015, 1012, 1025, 957, 1038, 1052, 1002, 1029, 1021, 1016, 1064, 975, 1004, 1041, 1021, 1009, 1003, 1005, 1037, 1064, 1060, 1022, 1015, 1052, 1050, 986, 1048, 1020, 994, 1040, 988, 1024, 1041, 1014, 1045, 1036, 1010, 1056, 1007, 1019, 1023, 1024, 979, 1025, 1022, 1031, 1059, 1041, 1034, 1042, 1050, 1005, 1015, 1048, 1014, 1010, 1024, 1029, 1001, 1010, 1050, 1071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[989, 986, 1027, 1013, 1069, 1047, 1052, 1021, 1034, 1023, 1046, 1004, 1022, 1046, 1037, 1015, 1055, 1034, 986, 995, 988, 1016, 1006, 1042, 1001, 1012, 1056, 1041, 1034, 1006, 972, 1032, 1047, 1057, 1059, 1013, 1001, 992, 1023, 1051, 977, 1061, 1011, 1061, 1041, 968, 1027, 1042, 1028, 1033, 1026, 1014, 1039, 1046, 1008, 1014, 999, 1024, 1028, 1015, 1040, 1028, 1035, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1038, 1025, 1039, 1075, 1049, 1035, 1019, 1069, 4, 0, 0, 0, 0, 0, 0, 0, 1000, 1006, 1045, 985, 1040, 977, 1006, 1009, 18, 0, 0, 0, 0, 0, 0, 0, 980, 985, 1027, 1031, 958, 1057, 1047, 1018, 2, 0, 0, 0, 0, 0, 0, 0, 1030, 1055, 1044, 1045, 1027, 958, 1005, 1013, 7, 6, 0, 0, 0, 0, 0, 0, 1051, 1047, 988, 1046, 1036, 1018, 1043, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 990, 1039, 1011, 1006, 988, 1085, 1044, 52, 3, 0, 0, 0, 0, 0, 0, 1033, 1023, 1000, 1028, 989, 1033, 1065, 998, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1018, 1036, 1041, 980, 1004, 1022, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8353, 8086, 8105, 8190, 8226, 8269, 8169, 8138] avg=8192.0000 max=8353 max_rank=0 min=8086 min_rank=1 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[980, 1022, 1025, 1013, 993, 998, 1051, 1000, 4, 0, 0, 0, 0, 0, 0, 0, 1083, 1018, 1033, 1048, 984, 1018, 970, 980, 18, 0, 0, 0, 0, 0, 0, 0, 1063, 1008, 1045, 1030, 1023, 1053, 1002, 1038, 3, 0, 0, 0, 0, 0, 0, 0, 1027, 1043, 1044, 1039, 987, 1017, 1031, 994, 7, 6, 0, 0, 0, 0, 0, 0, 1033, 922, 1050, 1059, 999, 1058, 1069, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1007, 996, 963, 971, 1034, 997, 1016, 46, 3, 0, 0, 0, 0, 0, 0, 1024, 1047, 1039, 1034, 1015, 1049, 1033, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1019, 1024, 1060, 1047, 991, 1039, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 8152, 8265, 8195, 8238, 8051, 8265, 8284] avg=8192.0000 max=8284 max_rank=7 min=8051 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1029, 999, 1044, 1040, 1052, 1020, 993, 998, 4, 0, 0, 0, 0, 0, 0, 0, 1009, 1042, 1028, 996, 1003, 1040, 1026, 1026, 18, 0, 0, 0, 0, 0, 0, 0, 1042, 996, 1032, 1023, 1020, 987, 1018, 1009, 2, 0, 0, 0, 0, 0, 0, 0, 1066, 1028, 1024, 985, 1006, 1041, 1052, 1039, 7, 6, 0, 0, 0, 0, 0, 0, 1073, 985, 1045, 1017, 1043, 1058, 1005, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1005, 1077, 1053, 1001, 1014, 1034, 1012, 49, 3, 0, 0, 0, 0, 0, 0, 1021, 1009, 1012, 1048, 1005, 1009, 1027, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 978, 983, 1099, 967, 1000, 990, 1030, 1074, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8188, 8129, 8254, 8247, 8262, 8156, 8121] avg=8192.0000 max=8262 max_rank=5 min=8121 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[986, 985, 1036, 1046, 1049, 1022, 1028, 1029, 4, 0, 0, 0, 0, 0, 0, 0, 1023, 1067, 1013, 1005, 1027, 983, 1041, 1070, 18, 0, 0, 0, 0, 0, 0, 0, 994, 1032, 1009, 1055, 1038, 1015, 1030, 1042, 2, 0, 0, 0, 0, 0, 0, 0, 1006, 1054, 993, 990, 1001, 1020, 1038, 1027, 7, 6, 0, 0, 0, 0, 0, 0, 1024, 993, 1033, 1019, 1072, 1054, 999, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1026, 1036, 981, 1032, 1016, 1009, 1061, 49, 3, 0, 0, 0, 0, 0, 0, 997, 1025, 1029, 1042, 977, 1045, 1061, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 956, 1020, 1044, 1031, 1000, 974, 1031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8247, 8217, 8142, 8239, 8232, 8152, 8122] avg=8192.0000 max=8247 max_rank=1 min=8122 min_rank=7 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1056, 1032, 1001, 1052, 1022, 1021, 1029, 1027, 4, 0, 0, 0, 0, 0, 0, 0, 1049, 1021, 1052, 1038, 996, 993, 1008, 1046, 17, 0, 0, 0, 0, 0, 0, 0, 963, 1056, 1038, 1009, 1030, 1021, 987, 1045, 3, 0, 0, 0, 0, 0, 0, 0, 981, 1034, 1020, 1054, 1027, 991, 992, 1039, 7, 5, 0, 0, 0, 0, 0, 0, 988, 1004, 1004, 1029, 1007, 1077, 1029, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1050, 1027, 1061, 983, 1036, 1017, 1009, 50, 3, 0, 0, 0, 0, 0, 0, 1008, 1008, 990, 968, 1036, 995, 1054, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1023, 1064, 996, 1058, 1024, 1035, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8220, 8152, 8150, 8195, 8260, 8065, 8250] avg=8192.0000 max=8260 max_rank=5 min=8065 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[969, 1017, 1077, 1016, 1046, 977, 1037, 1017, 4, 0, 0, 0, 0, 0, 0, 0, 998, 1025, 1045, 1022, 1025, 990, 1062, 1040, 18, 0, 0, 0, 0, 0, 0, 0, 1026, 1072, 997, 1031, 1063, 1019, 1059, 1046, 3, 0, 0, 0, 0, 0, 0, 0, 991, 1049, 1037, 1027, 1046, 1017, 1004, 1031, 7, 5, 0, 0, 0, 0, 0, 0, 1048, 963, 984, 1043, 1059, 964, 1024, 950, 0, 0, 0, 0, 0, 0, 0, 0, 992, 988, 969, 993, 996, 1030, 1042, 971, 48, 3, 0, 0, 0, 0, 0, 0, 1027, 1034, 1035, 1014, 1062, 1017, 1051, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1044, 1028, 1037, 1049, 1047, 1079, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8225, 8316, 8214, 8035, 8032, 8291, 8263] avg=8192.0000 max=8316 max_rank=2 min=8032 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32621 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[989, 986, 1027, 1013, 1069, 1047, 1052, 1021, 4, 0, 0, 0, 0, 0, 0, 0, 1034, 1023, 1046, 1004, 1022, 1046, 1037, 1015, 17, 0, 0, 0, 0, 0, 0, 0, 1055, 1034, 986, 995, 988, 1016, 1006, 1042, 2, 0, 0, 0, 0, 0, 0, 0, 1001, 1012, 1056, 1041, 1034, 1006, 972, 1032, 7, 5, 0, 0, 0, 0, 0, 0, 1047, 1007, 1059, 1013, 1001, 992, 1023, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1061, 1011, 1061, 1041, 968, 1027, 1042, 50, 3, 0, 0, 0, 0, 0, 0, 1028, 1033, 1026, 989, 1039, 1046, 1008, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1024, 1028, 1015, 1040, 1028, 1022, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8244, 8124, 8166, 8193, 8241, 8183, 8177] avg=8192.0000 max=8244 max_rank=1 min=8124 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1003, 1009, 1015, 1012, 1025, 957, 1038, 1052, 10, 0, 0, 0, 0, 0, 0, 0, 1002, 1029, 1021, 1016, 1064, 975, 1004, 1041, 22, 0, 0, 0, 0, 0, 0, 0, 1021, 1009, 1003, 1005, 1037, 1064, 1060, 1022, 7, 0, 0, 0, 0, 0, 0, 0, 1015, 1052, 1050, 986, 1048, 1020, 994, 1040, 11, 9, 0, 0, 0, 0, 0, 0, 988, 973, 1041, 1014, 1045, 1036, 1010, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1019, 1023, 1024, 979, 1025, 1022, 1031, 51, 8, 0, 0, 0, 0, 0, 0, 1059, 1041, 1034, 1003, 1050, 1005, 1015, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1010, 1024, 1029, 1001, 1010, 1022, 1071, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8174, 8228, 8225, 8163, 8189, 8255, 8181] avg=8192.0000 max=8255 max_rank=6 min=8121 min_rank=0 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8448, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8064, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 7936, 8320, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8064, 8192, 8320, 8192, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8448, 8320, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1044, 1023, 1048, 958, 1018, 1021, 1014, 1043, 1017, 1026, 1064, 1049, 1049, 1063, 1099, 1053, 995, 1034, 1010, 1016, 1021, 965, 1000, 981, 982, 1018, 1009, 1044, 998, 994, 995, 1010, 1052, 1052, 1002, 1035, 1054, 1018, 984, 989, 1023, 995, 1034, 1087, 1035, 1041, 988, 1024, 1036, 1026, 1074, 1032, 1012, 1028, 1009, 1008, 1072, 1027, 1026, 1020, 1047, 990, 1000, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1005, 1018, 1027, 1024, 1043, 992, 1003, 1010, 1012, 1032, 1076, 1006, 1033, 1020, 1016, 1074, 1062, 1048, 1034, 1017, 1027, 997, 1063, 1006, 1041, 1024, 1070, 1006, 1009, 962, 1046, 1034, 983, 1020, 1004, 990, 1064, 1023, 1010, 1007, 1056, 1004, 967, 976, 1006, 1060, 1057, 1033, 1026, 987, 1051, 1033, 1027, 1060, 1025, 1025, 1009, 1018, 1004, 1016, 1059, 1034, 1010, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1055, 1071, 989, 1005, 1008, 988, 1043, 1000, 1017, 1044, 1000, 1038, 1047, 1053, 1012, 993, 1081, 1010, 1060, 1038, 1005, 978, 1038, 1009, 1080, 976, 1029, 1019, 1038, 1027, 994, 1029, 983, 1016, 1098, 1000, 1039, 1022, 1082, 1005, 1050, 1004, 1062, 1051, 1049, 1005, 978, 993, 1093, 984, 1022, 984, 1069, 1022, 976, 1031, 1031, 978, 1046, 1033, 1003, 1035, 1013, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1013, 1026, 1023, 1019, 1006, 1012, 978, 978, 994, 1025, 1010, 1030, 1041, 1031, 1085, 1015, 992, 1031, 975, 1005, 1037, 1023, 1049, 994, 1022, 1062, 1075, 966, 976, 1068, 1058, 1019, 1020, 1009, 938, 1047, 982, 1079, 1106, 1094, 1029, 1082, 1023, 1008, 991, 979, 996, 1000, 977, 994, 1064, 1027, 1029, 1069, 990, 1046, 1043, 1015, 999, 1026, 1064, 1052, 1092, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1004, 999, 995, 1034, 1089, 1004, 1023, 1020, 1008, 1075, 979, 991, 1025, 1050, 1014, 937, 1010, 1023, 1041, 1040, 1013, 1031, 1042, 1052, 1025, 1033, 1055, 1064, 1006, 993, 1031, 1034, 1007, 1002, 1025, 975, 1026, 1057, 1060, 1012, 984, 1017, 1039, 1043, 988, 1023, 1057, 1027, 991, 1005, 1023, 1068, 991, 1037, 1035, 991, 1067, 1027, 1033, 1033, 1054, 987, 1076, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1055, 1033, 1060, 1037, 989, 1033, 1051, 985, 984, 995, 1009, 1011, 1004, 1030, 1029, 998, 1046, 1031, 1042, 1085, 1053, 1065, 1028, 1029, 1002, 1001, 985, 975, 989, 1060, 1027, 1027, 1029, 957, 1014, 985, 994, 982, 1046, 1078, 1002, 1046, 983, 1046, 1018, 1008, 1008, 1049, 1025, 1064, 1068, 1041, 1065, 1018, 1025, 980, 1017, 989, 1040, 1042, 1029, 1097, 1033, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[997, 1068, 1084, 1046, 1025, 1044, 1037, 1046, 1030, 1073, 994, 978, 971, 1025, 1062, 985, 1047, 1006, 1036, 1001, 1041, 1024, 1054, 1002, 1052, 984, 1024, 1009, 1023, 1017, 1033, 1083, 1020, 1019, 1042, 1069, 1001, 1046, 986, 1003, 987, 1059, 985, 992, 1002, 1040, 1064, 1034, 1002, 1010, 977, 1062, 1021, 1008, 1049, 978, 1020, 1051, 995, 987, 1047, 1046, 1012, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1075, 1002, 1005, 1017, 1015, 1030, 1005, 1019, 1000, 1024, 981, 1014, 1043, 1056, 1023, 1057, 1037, 1051, 1053, 1018, 1015, 1023, 1075, 977, 1062, 990, 1014, 1052, 1045, 1013, 1062, 1061, 1029, 1004, 1038, 1004, 1039, 1039, 1023, 1037, 998, 1059, 983, 1045, 1012, 997, 1019, 980, 1023, 1043, 962, 1052, 1018, 1021, 1044, 999, 1070, 1007, 1011, 1013, 1016, 1019, 1037, 981] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1005, 1018, 1027, 1024, 1043, 992, 1003, 1010, 11, 1, 0, 0, 0, 0, 0, 0, 1012, 1032, 1076, 1006, 1033, 1020, 1012, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1048, 1034, 1017, 1027, 997, 1044, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1024, 1070, 1006, 1009, 962, 1046, 1034, 3, 0, 0, 0, 0, 0, 0, 0, 983, 1020, 1004, 990, 1064, 1023, 1010, 1007, 8, 1, 0, 0, 0, 0, 0, 0, 1056, 1004, 967, 976, 1006, 1060, 1057, 1033, 31, 0, 0, 0, 0, 0, 0, 0, 1026, 987, 1051, 1033, 1027, 1060, 1025, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1018, 1004, 1016, 1059, 1034, 1010, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8265, 8235, 8195, 8110, 8190, 8234, 8173] avg=8192.0000 max=8265 max_rank=1 min=8110 min_rank=4 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1044, 1023, 1048, 958, 1018, 1021, 1014, 1043, 10, 2, 0, 0, 0, 0, 0, 0, 1017, 1026, 1064, 1049, 1049, 1063, 1095, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1034, 1010, 1016, 1021, 965, 982, 981, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1018, 1009, 1044, 998, 994, 995, 1010, 3, 0, 0, 0, 0, 0, 0, 0, 1052, 1052, 1002, 1035, 1054, 1018, 984, 989, 8, 1, 0, 0, 0, 0, 0, 0, 1023, 995, 1034, 1087, 1035, 1041, 988, 1024, 32, 0, 0, 0, 0, 0, 0, 0, 1036, 1026, 1074, 1032, 1012, 1028, 1009, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1027, 1026, 1020, 1047, 990, 1000, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8416, 8004, 8053, 8195, 8259, 8225, 8203] avg=8192.0000 max=8416 max_rank=1 min=8004 min_rank=2 imbalance_ratio=1.027344 max_load_reduction=57120 max_load_reduction_pct=0.871582 moved_to_echo=32882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1055, 1071, 989, 1005, 1008, 988, 1043, 1000, 11, 1, 0, 0, 0, 0, 0, 0, 1017, 1044, 1000, 1038, 1047, 1053, 1008, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1010, 1060, 1038, 1005, 978, 1019, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 976, 1029, 1019, 1038, 1027, 994, 1029, 3, 0, 0, 0, 0, 0, 0, 0, 983, 1016, 1098, 1000, 1039, 1022, 1082, 1005, 8, 1, 0, 0, 0, 0, 0, 0, 1050, 1004, 1062, 1051, 1049, 1005, 978, 993, 31, 0, 0, 0, 0, 0, 0, 0, 1093, 984, 1022, 984, 1069, 1022, 976, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 999, 978, 1046, 1033, 1003, 1035, 1013, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8200, 8200, 8195, 8254, 8223, 8181, 8112] avg=8192.0000 max=8254 max_rank=4 min=8112 min_rank=7 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1013, 1026, 1023, 1019, 1006, 1012, 978, 978, 11, 2, 0, 0, 0, 0, 0, 0, 994, 1025, 1010, 1030, 1041, 1031, 1081, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1031, 975, 1005, 1037, 1023, 1030, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1062, 1075, 966, 976, 1068, 1058, 1019, 3, 0, 0, 0, 0, 0, 0, 0, 1020, 1009, 938, 1047, 982, 1079, 1106, 1094, 8, 1, 0, 0, 0, 0, 0, 0, 1029, 1082, 1023, 1008, 991, 979, 996, 1000, 32, 0, 0, 0, 0, 0, 0, 0, 977, 994, 1064, 1027, 1029, 1069, 990, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1015, 999, 1026, 1064, 1052, 1092, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 8227, 8087, 8249, 8284, 8140, 8196, 8285] avg=8192.0000 max=8285 max_rank=7 min=8068 min_rank=0 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32905 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1055, 1033, 1060, 1037, 989, 1033, 1051, 985, 10, 1, 0, 0, 0, 0, 0, 0, 984, 995, 1009, 1011, 1004, 1030, 1025, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1031, 1042, 1085, 1053, 1065, 1010, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1001, 985, 975, 989, 1060, 1027, 1027, 3, 0, 0, 0, 0, 0, 0, 0, 1029, 957, 1014, 985, 994, 982, 1046, 1078, 8, 1, 0, 0, 0, 0, 0, 0, 1002, 1046, 983, 1046, 1018, 1008, 1008, 1049, 31, 0, 0, 0, 0, 0, 0, 0, 1025, 1064, 1068, 1041, 1065, 1018, 1025, 980, 0, 0, 0, 0, 0, 0, 0, 0, 985, 989, 1040, 1042, 1029, 1097, 1033, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8056, 8361, 8069, 8094, 8191, 8286, 8225] avg=8192.0000 max=8361 max_rank=2 min=8056 min_rank=1 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[997, 1068, 1084, 1046, 1025, 1044, 1037, 1046, 11, 1, 0, 0, 0, 0, 0, 0, 1030, 1073, 994, 978, 971, 1025, 1058, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1006, 1036, 1001, 1041, 1024, 1035, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 984, 1024, 1009, 1023, 1017, 1033, 1083, 3, 0, 0, 0, 0, 0, 0, 0, 1020, 1019, 1042, 1069, 1001, 1046, 986, 1003, 8, 1, 0, 0, 0, 0, 0, 0, 987, 1059, 985, 992, 1002, 1040, 1064, 1034, 31, 0, 0, 0, 0, 0, 0, 0, 1002, 1010, 977, 1062, 1021, 1008, 1049, 978, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1051, 995, 987, 1047, 1046, 1012, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8359, 8114, 8192, 8228, 8195, 8194, 8107, 8147] avg=8192.0000 max=8359 max_rank=0 min=8107 min_rank=6 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1004, 999, 995, 1034, 1089, 1004, 1023, 1020, 11, 2, 0, 0, 0, 0, 0, 0, 1008, 1075, 979, 991, 1025, 1050, 1010, 937, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1023, 1041, 1040, 1013, 1031, 1023, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1033, 1055, 1064, 1006, 993, 1031, 1034, 3, 0, 0, 0, 0, 0, 0, 0, 1007, 1002, 1025, 975, 1026, 1057, 1060, 1012, 8, 1, 0, 0, 0, 0, 0, 0, 984, 1017, 1039, 1043, 988, 1023, 1057, 1027, 32, 0, 0, 0, 0, 0, 0, 0, 991, 1005, 1023, 1068, 991, 1037, 1035, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1027, 1033, 1033, 1054, 987, 1076, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8075, 8233, 8244, 8173, 8210, 8141, 8279] avg=8192.0000 max=8279 max_rank=7 min=8075 min_rank=1 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1075, 1002, 1005, 1017, 1015, 1030, 1005, 1019, 14, 6, 0, 0, 0, 0, 0, 0, 1000, 1024, 981, 1014, 1043, 1056, 1008, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1051, 1053, 1018, 1015, 1023, 1050, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 990, 1014, 1052, 1045, 1013, 1062, 1061, 4, 0, 0, 0, 0, 0, 0, 0, 1029, 1004, 1038, 1004, 1039, 1039, 1023, 1037, 11, 7, 0, 0, 0, 0, 0, 0, 998, 1059, 983, 1045, 1012, 997, 1019, 980, 36, 0, 0, 0, 0, 0, 0, 0, 1023, 1043, 962, 1052, 1018, 1021, 1044, 999, 4, 0, 0, 0, 0, 0, 0, 0, 1028, 1007, 1011, 1013, 1016, 1019, 1037, 981, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8183, 8224, 8303, 8231, 8129, 8166, 8112] avg=8192.0000 max=8303 max_rank=3 min=8112 min_rank=7 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32638 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8192, 8320, 8448, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8192, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1048, 990, 1045, 995, 1047, 973, 1029, 1002, 1048, 1026, 1042, 1031, 1054, 1025, 1024, 1058, 1048, 1000, 1021, 1025, 1044, 1028, 1013, 1061, 1053, 997, 1025, 985, 1003, 1007, 964, 1056, 1012, 1011, 1010, 1006, 1061, 1040, 1034, 1041, 1069, 1025, 1003, 1031, 1018, 983, 1006, 1020, 1027, 1027, 1037, 1048, 983, 1036, 1022, 973, 1032, 1036, 1023, 998, 1047, 1013, 1073, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1011, 1025, 1025, 986, 1034, 1016, 1064, 1066, 1053, 1060, 1009, 1040, 980, 1053, 998, 1047, 1044, 1004, 1040, 1036, 1043, 995, 1021, 991, 1021, 1071, 1048, 969, 965, 1025, 1051, 991, 989, 1013, 1063, 1031, 1029, 1050, 1029, 1001, 1050, 994, 986, 1056, 1046, 997, 1039, 1073, 1002, 1043, 1053, 1025, 1019, 990, 1042, 1017, 996, 1060, 995, 1013, 1036, 1024, 988, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1018, 985, 1030, 1057, 1055, 1076, 1037, 1030, 1076, 1057, 1041, 982, 995, 1074, 1018, 1028, 1046, 1023, 978, 1009, 959, 1014, 1008, 1029, 1042, 977, 1060, 998, 1061, 1018, 1029, 999, 1033, 1047, 1025, 1001, 1024, 1092, 1001, 1042, 1047, 1034, 1019, 1027, 1055, 1054, 1044, 991, 984, 1029, 1093, 1012, 952, 1023, 1044, 1014, 978, 976, 1020, 973, 1043, 1013, 1030, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1013, 1025, 1037, 1027, 999, 999, 1046, 1015, 1013, 997, 1052, 1004, 996, 1009, 989, 1005, 990, 1017, 1038, 959, 1017, 1049, 1075, 1039, 1031, 1035, 1074, 999, 1009, 1038, 1014, 1078, 1026, 997, 1036, 1086, 1050, 1049, 1001, 1001, 1015, 995, 1061, 1060, 1045, 1029, 976, 1017, 1046, 988, 1072, 1038, 1045, 1018, 1030, 1047, 1009, 1037, 994, 1004, 1027, 994, 1077, 978] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1060, 1039, 1084, 1039, 1033, 1036, 1051, 975, 1039, 1017, 975, 1058, 1002, 1014, 1018, 1000, 1033, 1074, 1031, 1027, 1024, 1053, 1083, 994, 977, 992, 1018, 1017, 1017, 990, 1008, 1019, 1042, 1013, 1021, 993, 1005, 1033, 1009, 1015, 1018, 1013, 1026, 1051, 1008, 1049, 974, 1030, 1009, 1007, 995, 1006, 999, 1103, 1068, 1041, 1023, 1035, 1074, 1024, 1020, 1026, 1028, 981] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[985, 989, 973, 1019, 1035, 1021, 1041, 1023, 987, 1028, 998, 1015, 994, 1019, 1000, 1018, 1060, 1102, 1087, 1072, 1009, 1031, 986, 1063, 972, 1013, 1035, 1012, 1037, 1024, 1047, 1045, 1047, 1064, 989, 995, 1017, 1014, 1048, 1039, 1068, 1040, 1008, 1018, 995, 1048, 1034, 1043, 1025, 999, 955, 1038, 1000, 1014, 1007, 1016, 1064, 1074, 1000, 988, 1044, 1069, 1026, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1050, 996, 1041, 1016, 1051, 1032, 1032, 1018, 996, 1050, 977, 1054, 1054, 1048, 1028, 1033, 1007, 1042, 1038, 1034, 1008, 1010, 1017, 982, 1023, 1037, 968, 998, 1011, 979, 1082, 1038, 987, 1026, 983, 1049, 1023, 983, 1044, 1054, 1090, 1047, 1064, 990, 1020, 1041, 1039, 1002, 1032, 1019, 1020, 991, 1049, 1018, 995, 1034, 1025, 1064, 1040, 1005, 992, 1011, 1031, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[994, 1043, 1036, 1040, 1056, 1012, 1083, 1015, 988, 1046, 1036, 990, 1023, 1013, 991, 1032, 1054, 1001, 1031, 1005, 1084, 975, 997, 1026, 981, 1019, 1002, 1016, 1067, 1057, 1019, 1023, 1037, 1006, 1086, 1007, 1037, 1056, 1000, 1011, 1035, 996, 994, 985, 1032, 990, 992, 1039, 1007, 1001, 1018, 1007, 1038, 1013, 1046, 1052, 1069, 1002, 1059, 1040, 988, 1009, 1046, 1083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1011, 1025, 1025, 986, 1034, 1016, 1041, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1060, 1009, 1040, 980, 1053, 998, 1047, 7, 6, 0, 0, 0, 0, 0, 0, 1024, 1004, 1040, 1036, 1043, 995, 1021, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1071, 1048, 969, 965, 1025, 1051, 991, 23, 14, 0, 0, 0, 0, 0, 0, 989, 1013, 1063, 1031, 1029, 1034, 1029, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 994, 986, 1056, 1046, 997, 1039, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1043, 1053, 1025, 1019, 990, 1042, 1017, 10, 6, 0, 0, 0, 0, 0, 0, 996, 1060, 995, 1013, 1036, 1024, 988, 1005, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8253, 8154, 8178, 8189, 8229, 8207, 8122] avg=8192.0000 max=8253 max_rank=1 min=8122 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1060, 1039, 1084, 1039, 1033, 1036, 1028, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1017, 975, 1058, 1002, 1014, 1018, 1000, 7, 5, 0, 0, 0, 0, 0, 0, 1013, 1074, 1031, 1027, 1024, 1053, 1083, 994, 0, 0, 0, 0, 0, 0, 0, 0, 977, 992, 1018, 1017, 1017, 990, 1008, 1019, 23, 14, 0, 0, 0, 0, 0, 0, 1042, 1013, 1021, 993, 1005, 1019, 1009, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1013, 1026, 1051, 1008, 1049, 974, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1007, 995, 1006, 999, 1103, 1068, 1041, 9, 6, 0, 0, 0, 0, 0, 0, 1023, 1035, 1074, 1024, 1020, 1026, 1028, 981, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 8135, 8299, 8075, 8117, 8157, 8243, 8216] avg=8192.0000 max=8299 max_rank=2 min=8075 min_rank=3 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1048, 990, 1045, 995, 1047, 973, 1007, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1026, 1042, 1031, 1054, 1025, 1024, 1058, 8, 6, 0, 0, 0, 0, 0, 0, 1028, 1000, 1021, 1025, 1044, 1028, 1013, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 997, 1025, 985, 1003, 1007, 964, 1056, 22, 14, 0, 0, 0, 0, 0, 0, 1012, 1011, 1010, 1006, 1061, 1024, 1034, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1025, 1003, 1031, 1018, 983, 1006, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1027, 1037, 1048, 983, 1036, 1022, 973, 10, 6, 0, 0, 0, 0, 0, 0, 1032, 1036, 1023, 998, 1047, 1013, 1073, 1024, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8322, 8220, 8126, 8199, 8142, 8169, 8251] avg=8192.0000 max=8322 max_rank=1 min=8107 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1018, 985, 1030, 1057, 1055, 1076, 1014, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1057, 1041, 982, 995, 1074, 1018, 1028, 7, 6, 0, 0, 0, 0, 0, 0, 1026, 1023, 978, 1009, 959, 1014, 1008, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 977, 1060, 998, 1061, 1018, 1029, 999, 23, 14, 0, 0, 0, 0, 0, 0, 1033, 1047, 1025, 1001, 1024, 1076, 1001, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1034, 1019, 1027, 1055, 1054, 1044, 991, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1029, 1093, 1012, 952, 1023, 1044, 1014, 10, 6, 0, 0, 0, 0, 0, 0, 978, 976, 1020, 973, 1043, 1013, 1030, 1007, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8284, 8046, 8221, 8249, 8259, 8167, 8045] avg=8192.0000 max=8284 max_rank=1 min=8045 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1013, 1025, 1037, 1027, 999, 999, 1023, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 997, 1052, 1004, 996, 1009, 989, 1005, 7, 6, 0, 0, 0, 0, 0, 0, 971, 1017, 1038, 959, 1017, 1049, 1075, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1035, 1074, 999, 1009, 1038, 1014, 1078, 23, 13, 0, 0, 0, 0, 0, 0, 1026, 997, 1036, 1086, 1050, 1033, 1001, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 995, 1061, 1060, 1045, 1029, 976, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 988, 1072, 1038, 1045, 1018, 1030, 1047, 10, 6, 0, 0, 0, 0, 0, 0, 1009, 1037, 994, 1004, 1027, 994, 1077, 978, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8078, 8165, 8314, 8230, 8186, 8300, 8125] avg=8192.0000 max=8314 max_rank=3 min=8078 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[985, 989, 973, 1019, 1035, 1021, 1018, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1028, 998, 1015, 994, 1019, 1000, 1018, 8, 5, 0, 0, 0, 0, 0, 0, 1039, 1102, 1087, 1072, 1009, 1031, 986, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1013, 1035, 1012, 1037, 1024, 1047, 1045, 23, 14, 0, 0, 0, 0, 0, 0, 1047, 1064, 989, 995, 1017, 1000, 1048, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1040, 1008, 1018, 995, 1048, 1034, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 999, 955, 1038, 1000, 1014, 1007, 1016, 9, 7, 0, 0, 0, 0, 0, 0, 1064, 1074, 1000, 988, 1044, 1069, 1026, 1010, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8063, 8072, 8389, 8222, 8199, 8241, 8070, 8280] avg=8192.0000 max=8389 max_rank=2 min=8063 min_rank=0 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1050, 996, 1041, 1016, 1051, 1032, 1009, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1050, 977, 1054, 1054, 1048, 1028, 1033, 8, 5, 0, 0, 0, 0, 0, 0, 988, 1042, 1038, 1034, 1008, 1010, 1017, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1037, 968, 998, 1011, 979, 1082, 1038, 23, 13, 0, 0, 0, 0, 0, 0, 987, 1026, 983, 1049, 1023, 969, 1044, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 1047, 1064, 990, 1020, 1041, 1039, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1019, 1020, 991, 1049, 1018, 995, 1034, 9, 6, 0, 0, 0, 0, 0, 0, 1025, 1064, 1040, 1005, 992, 1011, 1031, 1018, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8253, 8119, 8172, 8135, 8280, 8173, 8191] avg=8192.0000 max=8280 max_rank=5 min=8119 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[994, 1043, 1036, 1040, 1056, 1012, 1056, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1046, 1036, 990, 1023, 1013, 991, 1032, 11, 9, 0, 0, 0, 0, 0, 0, 1025, 1001, 1031, 1005, 1084, 975, 997, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1019, 1002, 1016, 1067, 1057, 1019, 1023, 27, 17, 0, 0, 0, 0, 0, 0, 1037, 1006, 1086, 1007, 1037, 1034, 1000, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 996, 994, 985, 1032, 990, 992, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1001, 1018, 1007, 1038, 1013, 1046, 1052, 13, 12, 0, 0, 0, 0, 0, 0, 1069, 1002, 1059, 1040, 988, 1009, 1046, 1083, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8139, 8144, 8228, 8218, 8042, 8207, 8306] avg=8192.0000 max=8306 max_rank=7 min=8042 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8064, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8064, 8192, 8192, 8320, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1008, 1039, 1010, 1024, 1028, 1078, 1035, 992, 977, 1011, 996, 1055, 1031, 1037, 986, 1013, 1019, 996, 1031, 1050, 1038, 1010, 1089, 1032, 1018, 1069, 1043, 1014, 948, 999, 1007, 1002, 1010, 1037, 1030, 1038, 983, 982, 1054, 973, 993, 1024, 1014, 1081, 1014, 1010, 1045, 1048, 1047, 1004, 1002, 1067, 1066, 1027, 1047, 1027, 1009, 1027, 1080, 1067, 1004, 1044, 984, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1006, 1050, 987, 1040, 1035, 990, 987, 1000, 1032, 1006, 1010, 1013, 1047, 1057, 1043, 1069, 1027, 1030, 978, 1026, 1054, 981, 996, 989, 1010, 998, 1053, 1035, 1070, 1067, 954, 1074, 1029, 1073, 1016, 997, 971, 1027, 1061, 1050, 1004, 1041, 1072, 1027, 1070, 1027, 1003, 1008, 1017, 1027, 1024, 1037, 974, 1024, 1038, 1027, 1068, 1004, 1001, 1048, 1042, 981, 1019, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1025, 1033, 1027, 1019, 1018, 1045, 1045, 1051, 1078, 1038, 1069, 1004, 1012, 1018, 1026, 1028, 1049, 1021, 1018, 1007, 1020, 1030, 984, 1033, 1015, 1033, 1000, 1026, 1019, 1018, 1027, 986, 991, 1052, 1074, 1035, 993, 1032, 1018, 1015, 1088, 999, 1075, 1032, 968, 1051, 1054, 998, 978, 971, 1026, 1002, 1071, 1038, 1043, 1048, 1036, 1023, 991, 1008, 993, 976, 1027, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1016, 989, 1056, 955, 999, 1013, 1028, 996, 994, 1071, 1030, 1011, 985, 1066, 989, 1049, 1032, 1091, 977, 1040, 1006, 969, 1007, 1007, 1010, 997, 1033, 1032, 1029, 985, 1001, 983, 1023, 1016, 1009, 1027, 1007, 1029, 1045, 984, 1038, 1029, 1046, 996, 1057, 1062, 1025, 1086, 988, 1075, 1066, 1018, 1032, 1021, 1063, 1036, 1037, 1054, 1029, 1043, 1031, 1065, 1015, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1022, 979, 992, 1058, 1030, 992, 1050, 992, 1036, 1083, 1069, 1010, 1019, 1060, 1025, 1039, 1041, 1017, 1026, 995, 1061, 1022, 983, 1041, 984, 1000, 1002, 1025, 985, 1041, 1003, 991, 1008, 1056, 1043, 1048, 1012, 994, 1044, 986, 1037, 1002, 1035, 988, 1031, 1001, 1073, 1101, 1049, 1063, 1017, 1027, 983, 1023, 1073, 995, 1089, 1002, 1005, 1028, 1055, 962, 1020, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1034, 970, 1076, 1017, 1042, 1048, 1031, 994, 1099, 1037, 994, 998, 996, 988, 1003, 1023, 1046, 1021, 1053, 1025, 1026, 1008, 1007, 1012, 1048, 990, 1040, 1004, 1020, 1050, 1026, 987, 981, 969, 1027, 1028, 970, 1048, 1007, 1000, 1072, 1057, 996, 1020, 956, 1087, 1055, 1049, 991, 999, 1038, 1025, 995, 1029, 1047, 1064, 1053, 1045, 1019, 1043, 1044, 1013, 1074, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1027, 1005, 1023, 1046, 1023, 994, 1018, 1004, 998, 1013, 1022, 1029, 1055, 1014, 1008, 1010, 979, 1037, 1020, 1004, 998, 1002, 991, 998, 1017, 1015, 1048, 1061, 987, 1064, 1046, 992, 1023, 1029, 985, 984, 1086, 1031, 1027, 1042, 1072, 1019, 995, 1014, 999, 1051, 1067, 991, 1003, 1047, 1076, 1005, 1025, 1018, 1070, 1032, 1016, 1070, 1017, 1041, 1001, 1094, 1062, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1040, 1038, 987, 1051, 1075, 1056, 1001, 984, 1044, 1037, 1044, 1037, 1011, 1021, 1020, 1020, 1020, 1054, 989, 997, 1020, 1037, 968, 1023, 1041, 1026, 1006, 1048, 992, 1005, 989, 1021, 1021, 1058, 1036, 1046, 1011, 1033, 1074, 1050, 1047, 983, 1016, 1018, 1026, 946, 968, 1034, 952, 1016, 1036, 1053, 1037, 1030, 1052, 1063, 1045, 1055, 1027, 998, 979, 1065, 1044, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1008, 1039, 1010, 1024, 1028, 1078, 1035, 992, 16, 9, 0, 0, 0, 0, 0, 0, 977, 991, 996, 1055, 1031, 1037, 986, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 996, 1031, 1050, 1038, 1010, 1089, 1032, 31, 14, 0, 0, 0, 0, 0, 0, 1018, 1069, 1043, 1014, 948, 999, 1007, 1002, 42, 8, 0, 0, 0, 0, 0, 0, 1010, 1037, 1030, 1038, 983, 982, 1054, 973, 11, 0, 0, 0, 0, 0, 0, 0, 951, 1024, 1014, 1081, 1014, 1010, 1045, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1004, 1002, 1067, 1066, 1027, 1008, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1027, 1080, 1067, 1004, 1044, 984, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8086, 8310, 8150, 8118, 8187, 8248, 8198] avg=8192.0000 max=8310 max_rank=2 min=8086 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1025, 1033, 1027, 1019, 1018, 1045, 1045, 1051, 16, 9, 0, 0, 0, 0, 0, 0, 1078, 1017, 1069, 1004, 1012, 1018, 1026, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1021, 1018, 1007, 1020, 1030, 984, 1033, 31, 15, 0, 0, 0, 0, 0, 0, 1015, 1033, 1000, 1026, 1019, 1018, 1027, 986, 46, 8, 0, 0, 0, 0, 0, 0, 991, 1052, 1074, 1035, 993, 1032, 1018, 1015, 12, 0, 0, 0, 0, 0, 0, 0, 1042, 999, 1075, 1032, 968, 1051, 1054, 998, 0, 0, 0, 0, 0, 0, 0, 0, 978, 971, 1026, 1002, 1071, 1038, 1004, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1023, 991, 1008, 993, 976, 1027, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8252, 8208, 8178, 8222, 8219, 8138, 8031] avg=8192.0000 max=8288 max_rank=0 min=8031 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32610 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1022, 979, 992, 1058, 1030, 992, 1050, 992, 17, 10, 0, 0, 0, 0, 0, 0, 1036, 1061, 1069, 1010, 1019, 1060, 1025, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1017, 1026, 995, 1061, 1022, 983, 1041, 32, 15, 0, 0, 0, 0, 0, 0, 984, 1000, 1002, 1025, 985, 1041, 1003, 991, 44, 9, 0, 0, 0, 0, 0, 0, 1008, 1056, 1043, 1048, 1012, 994, 1044, 986, 12, 0, 0, 0, 0, 0, 0, 0, 993, 1002, 1035, 988, 1031, 1001, 1073, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1063, 1017, 1027, 983, 1023, 1032, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1002, 1005, 1028, 1055, 962, 1020, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8319, 8233, 8084, 8203, 8224, 8189, 8142] avg=8192.0000 max=8319 max_rank=1 min=8084 min_rank=3 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1016, 989, 1056, 955, 999, 1013, 1028, 996, 16, 10, 0, 0, 0, 0, 0, 0, 994, 1049, 1030, 1011, 985, 1066, 989, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1091, 977, 1040, 1006, 969, 1007, 1007, 32, 15, 0, 0, 0, 0, 0, 0, 1010, 997, 1033, 1032, 1029, 985, 1001, 983, 44, 9, 0, 0, 0, 0, 0, 0, 1023, 1016, 1009, 1027, 1007, 1029, 1045, 984, 12, 0, 0, 0, 0, 0, 0, 0, 994, 1029, 1046, 996, 1057, 1062, 1025, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1075, 1066, 1018, 1032, 1021, 1022, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1054, 1029, 1043, 1031, 1065, 1015, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8078, 8173, 8176, 8123, 8152, 8295, 8258, 8281] avg=8192.0000 max=8295 max_rank=5 min=8078 min_rank=0 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32986 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1006, 1050, 987, 1040, 1035, 990, 987, 1000, 17, 9, 0, 0, 0, 0, 0, 0, 1032, 986, 1010, 1013, 1047, 1057, 1043, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1030, 978, 1026, 1054, 981, 996, 989, 31, 15, 0, 0, 0, 0, 0, 0, 1010, 998, 1053, 1035, 1070, 1067, 954, 1074, 42, 8, 0, 0, 0, 0, 0, 0, 1029, 1073, 1016, 997, 971, 1027, 1061, 1050, 11, 0, 0, 0, 0, 0, 0, 0, 962, 1041, 1072, 1027, 1070, 1027, 1003, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1027, 1024, 1037, 974, 1024, 999, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1004, 1001, 1048, 1042, 981, 1019, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8257, 8127, 8311, 8235, 8210, 8129, 8146] avg=8192.0000 max=8311 max_rank=3 min=8121 min_rank=0 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1034, 970, 1076, 1017, 1042, 1048, 1031, 994, 16, 9, 0, 0, 0, 0, 0, 0, 1099, 1016, 994, 998, 996, 988, 1003, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1021, 1053, 1025, 1026, 1008, 1007, 1012, 31, 15, 0, 0, 0, 0, 0, 0, 1048, 990, 1040, 1004, 1020, 1050, 1026, 987, 45, 8, 0, 0, 0, 0, 0, 0, 981, 969, 1027, 1028, 970, 1048, 1007, 1000, 12, 0, 0, 0, 0, 0, 0, 0, 1027, 1057, 996, 1020, 956, 1087, 1055, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 991, 999, 1038, 1025, 995, 1029, 1008, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1045, 1019, 1043, 1044, 1013, 1074, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8117, 8244, 8218, 8042, 8247, 8149, 8282] avg=8192.0000 max=8282 max_rank=7 min=8042 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1027, 1005, 1023, 1046, 1023, 994, 1018, 1004, 16, 9, 0, 0, 0, 0, 0, 0, 998, 993, 1022, 1029, 1055, 1014, 1008, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1037, 1020, 1004, 998, 1002, 991, 998, 32, 14, 0, 0, 0, 0, 0, 0, 1017, 1015, 1048, 1061, 987, 1064, 1046, 992, 45, 9, 0, 0, 0, 0, 0, 0, 1023, 1029, 985, 984, 1086, 1031, 1027, 1042, 11, 0, 0, 0, 0, 0, 0, 0, 1027, 1019, 995, 1014, 999, 1051, 1067, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1047, 1076, 1005, 1025, 1018, 1029, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1070, 1017, 1041, 1001, 1094, 1062, 996, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8129, 8075, 8284, 8218, 8163, 8235, 8267] avg=8192.0000 max=8284 max_rank=3 min=8075 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1040, 1038, 987, 1051, 1075, 1056, 1001, 984, 20, 14, 0, 0, 0, 0, 0, 0, 1044, 1006, 1044, 1037, 1011, 1021, 1020, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1054, 989, 997, 1020, 1037, 968, 1023, 36, 19, 0, 0, 0, 0, 0, 0, 1041, 1026, 1006, 1048, 992, 1005, 989, 1021, 47, 13, 0, 0, 0, 0, 0, 0, 1021, 1058, 1036, 1046, 1011, 1033, 1074, 1050, 17, 0, 0, 0, 0, 0, 0, 0, 1000, 983, 1016, 1018, 1026, 946, 968, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 952, 1016, 1036, 1053, 1037, 1030, 1003, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1055, 1027, 998, 979, 1065, 1044, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8203, 8163, 8188, 8346, 7991, 8190, 8189] avg=8192.0000 max=8346 max_rank=4 min=7991 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8064, 8064, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8064, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8064, 8192, 8448, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8064, 8320, 8064, 8064, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1056, 990, 1026, 1064, 1056, 1021, 1022, 1020, 1066, 1083, 1025, 995, 1023, 1005, 983, 1023, 1040, 1022, 1058, 1018, 1021, 1018, 1001, 991, 970, 1008, 1024, 979, 1040, 963, 1058, 990, 1040, 1042, 1064, 1011, 1031, 1053, 1075, 1017, 1007, 959, 1049, 1016, 1054, 1010, 1036, 1047, 1032, 1043, 1046, 1034, 1006, 1072, 1015, 1026, 1003, 1058, 1031, 1029, 996, 987, 1018, 970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1030, 977, 998, 1070, 1003, 1067, 1030, 1023, 1000, 1018, 981, 1032, 1010, 1010, 1037, 1064, 1026, 981, 974, 994, 1012, 1009, 1011, 1016, 1005, 1068, 1010, 1082, 1032, 1073, 1032, 978, 1057, 1011, 980, 1019, 1023, 1030, 1058, 1020, 1038, 1052, 975, 1015, 1028, 1024, 1016, 1020, 1038, 1003, 1054, 1040, 1002, 1026, 1034, 1020, 1030, 1023, 1022, 1060, 972, 1042, 1062, 1089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[990, 1052, 1027, 1050, 1005, 988, 1052, 1009, 1072, 1039, 1025, 1024, 1068, 1010, 1042, 1070, 988, 1044, 1026, 1030, 1066, 1027, 1033, 1001, 1024, 1075, 1055, 1004, 974, 1017, 1031, 1030, 1005, 1022, 1024, 1019, 1044, 1029, 970, 1040, 1001, 1037, 1002, 998, 1019, 1009, 1020, 993, 1041, 1022, 1015, 979, 1032, 989, 988, 1031, 1035, 1021, 990, 1049, 1078, 1049, 1044, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1015, 1030, 1044, 1032, 1037, 1077, 1038, 999, 1064, 1047, 1034, 1036, 1046, 1027, 978, 1024, 1038, 1050, 1016, 993, 1048, 979, 1070, 1005, 964, 1007, 1055, 1004, 999, 969, 979, 1026, 1025, 954, 1032, 1039, 1034, 1054, 1027, 1038, 1037, 1020, 1011, 1092, 1028, 971, 1012, 1007, 1037, 1035, 1023, 1033, 1028, 1004, 1025, 1031, 1029, 1010, 1007, 1052, 1024, 1043, 1045, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1014, 1033, 1071, 986, 1005, 1008, 1017, 1046, 1002, 979, 1025, 1016, 1014, 1018, 1037, 987, 1036, 968, 1035, 1022, 1064, 1023, 1036, 975, 1017, 986, 1017, 1044, 1022, 1004, 1036, 1063, 1000, 974, 1052, 998, 994, 1024, 1041, 1070, 1018, 1038, 1041, 1038, 1064, 1034, 1023, 1055, 1000, 1046, 1029, 1059, 995, 1046, 1044, 1027, 996, 1032, 1030, 1035, 1043, 1013, 1017, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1026, 1071, 987, 1092, 1020, 1046, 1102, 1010, 1026, 1004, 1020, 1007, 1046, 1029, 1037, 1035, 982, 1017, 1037, 1027, 1067, 1008, 1020, 1102, 1029, 975, 1027, 1018, 966, 1015, 997, 1016, 986, 1036, 1012, 1021, 1018, 1061, 972, 1013, 1065, 1054, 1023, 962, 1062, 992, 1041, 1108, 1063, 958, 1021, 1004, 981, 1014, 1023, 1050, 1023, 1028, 988, 1054, 1041, 1020, 970, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1008, 1016, 1039, 1033, 1041, 1049, 980, 1019, 1070, 1041, 1046, 1062, 987, 1010, 1011, 1012, 988, 1051, 1045, 1045, 1031, 981, 999, 1058, 1005, 965, 983, 1041, 1036, 1043, 1018, 1009, 1012, 965, 1014, 1047, 1005, 1022, 992, 1027, 1092, 1069, 1023, 1034, 1054, 1063, 1056, 1069, 1013, 964, 1036, 983, 984, 989, 1047, 1005, 1062, 984, 1042, 1039, 1005, 1052, 1055, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1069, 993, 1000, 1039, 1024, 1095, 1023, 995, 992, 1028, 1067, 1043, 1031, 976, 1017, 980, 1002, 1012, 1013, 1013, 1072, 1001, 1029, 1049, 1002, 1026, 1025, 1006, 1015, 1016, 1060, 1016, 1005, 1005, 978, 1031, 961, 1003, 1023, 1033, 1064, 1017, 1020, 1040, 1007, 1025, 1029, 1006, 1032, 1057, 1069, 1056, 1001, 1023, 1010, 1018, 1056, 1072, 1078, 999, 997, 1003, 1034, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1056, 990, 1026, 1024, 1056, 1021, 1022, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1083, 1025, 995, 1023, 1005, 983, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1022, 1058, 1018, 1021, 1018, 1001, 991, 15, 0, 0, 0, 0, 0, 0, 0, 970, 1008, 1024, 979, 1040, 963, 1058, 990, 42, 20, 0, 0, 0, 0, 0, 0, 1040, 1042, 1064, 1011, 1031, 1053, 1075, 1017, 20, 11, 0, 0, 0, 0, 0, 0, 965, 959, 1049, 1016, 1054, 1010, 1036, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1043, 1046, 1034, 1006, 1072, 1015, 1026, 10, 0, 0, 0, 0, 0, 0, 0, 1003, 1058, 1031, 1003, 996, 987, 1018, 970, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8193, 8184, 8094, 8364, 8136, 8284, 8066] avg=8192.0000 max=8364 max_rank=4 min=8066 min_rank=7 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1030, 977, 998, 1030, 1003, 1067, 1030, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1018, 981, 1032, 1010, 1010, 1037, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 981, 974, 994, 1012, 1009, 1011, 1016, 15, 0, 0, 0, 0, 0, 0, 0, 1005, 1068, 1010, 1082, 1032, 1073, 1032, 978, 44, 20, 0, 0, 0, 0, 0, 0, 1057, 1011, 980, 1019, 1023, 1030, 1058, 1020, 20, 12, 0, 0, 0, 0, 0, 0, 994, 1052, 975, 1015, 1028, 1024, 1016, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1003, 1054, 1040, 1002, 1026, 1034, 1020, 9, 0, 0, 0, 0, 0, 0, 0, 1030, 1023, 1022, 1033, 972, 1042, 1062, 1089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8143, 8038, 8344, 8230, 8124, 8226, 8273] avg=8192.0000 max=8344 max_rank=3 min=8038 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[990, 1052, 1027, 1011, 1005, 988, 1052, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1039, 1025, 1024, 1068, 1010, 1042, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1044, 1026, 1030, 1066, 1027, 1033, 1001, 15, 0, 0, 0, 0, 0, 0, 0, 1024, 1075, 1055, 1004, 974, 1017, 1031, 1030, 42, 20, 0, 0, 0, 0, 0, 0, 1005, 1022, 1024, 1019, 1044, 1029, 970, 1040, 19, 11, 0, 0, 0, 0, 0, 0, 959, 1037, 1002, 998, 1019, 1009, 1020, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1022, 1015, 979, 1032, 989, 988, 1031, 10, 0, 0, 0, 0, 0, 0, 0, 1035, 1021, 990, 1023, 1078, 1049, 1044, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8340, 8230, 8272, 8183, 8037, 8107, 8233] avg=8192.0000 max=8340 max_rank=1 min=8037 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32560 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1015, 1030, 1044, 994, 1037, 1077, 1038, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1047, 1034, 1036, 1046, 1027, 978, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1050, 1016, 993, 1048, 979, 1070, 1005, 15, 0, 0, 0, 0, 0, 0, 0, 964, 1007, 1055, 1004, 999, 969, 979, 1026, 43, 19, 0, 0, 0, 0, 0, 0, 1025, 954, 1032, 1039, 1034, 1054, 1027, 1038, 19, 12, 0, 0, 0, 0, 0, 0, 994, 1020, 1011, 1092, 1028, 971, 1012, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1035, 1023, 1033, 1028, 1004, 1025, 1031, 10, 0, 0, 0, 0, 0, 0, 0, 1029, 1010, 1007, 1025, 1024, 1043, 1045, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8246, 8214, 8065, 8234, 8135, 8226, 8182] avg=8192.0000 max=8246 max_rank=1 min=8065 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1008, 1016, 1039, 995, 1041, 1049, 980, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1041, 1046, 1062, 987, 1010, 1011, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1051, 1045, 1045, 1031, 981, 999, 1058, 15, 0, 0, 0, 0, 0, 0, 0, 1005, 965, 983, 1041, 1036, 1043, 1018, 1009, 46, 19, 0, 0, 0, 0, 0, 0, 1012, 965, 1014, 1047, 1005, 1022, 992, 1027, 19, 11, 0, 0, 0, 0, 0, 0, 1046, 1069, 1023, 1034, 1054, 1063, 1056, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 964, 1036, 983, 984, 989, 1047, 1005, 10, 0, 0, 0, 0, 0, 0, 0, 1062, 984, 1042, 1013, 1005, 1052, 1055, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8229, 8213, 8165, 8114, 8414, 8031, 8223] avg=8192.0000 max=8414 max_rank=5 min=8031 min_rank=6 imbalance_ratio=1.027100 max_load_reduction=57122 max_load_reduction_pct=0.871613 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1014, 1033, 1071, 950, 1005, 1008, 1017, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 993, 979, 1025, 1016, 1014, 1018, 1037, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 968, 1035, 1022, 1064, 1023, 1036, 975, 15, 0, 0, 0, 0, 0, 0, 0, 1017, 986, 1017, 1044, 1022, 1004, 1036, 1063, 43, 18, 0, 0, 0, 0, 0, 0, 1000, 974, 1052, 998, 994, 1024, 1041, 1070, 18, 11, 0, 0, 0, 0, 0, 0, 975, 1038, 1041, 1038, 1064, 1034, 1023, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1046, 1029, 1059, 995, 1046, 1044, 1027, 9, 0, 0, 0, 0, 0, 0, 0, 996, 1032, 1030, 1009, 1043, 1013, 1017, 1054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8069, 8174, 8250, 8182, 8268, 8255, 8194] avg=8192.0000 max=8268 max_rank=5 min=8069 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1026, 1071, 987, 1052, 1020, 1046, 1102, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1004, 1020, 1007, 1046, 1029, 1037, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1017, 1037, 1027, 1067, 1008, 1020, 1102, 15, 0, 0, 0, 0, 0, 0, 0, 1029, 975, 1027, 1018, 966, 1015, 997, 1016, 45, 20, 0, 0, 0, 0, 0, 0, 986, 1036, 1012, 1021, 1018, 1061, 972, 1013, 20, 12, 0, 0, 0, 0, 0, 0, 1020, 1054, 1023, 962, 1062, 992, 1041, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 958, 1021, 1004, 981, 1014, 1023, 1050, 9, 0, 0, 0, 0, 0, 0, 0, 1023, 1028, 988, 1027, 1041, 1020, 970, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8314, 8195, 8275, 8108, 8151, 8262, 8123, 8108] avg=8192.0000 max=8314 max_rank=0 min=8108 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1069, 993, 1000, 991, 1024, 1095, 1023, 995, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1028, 1067, 1043, 1031, 976, 1017, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1012, 1013, 1013, 1072, 1001, 1029, 1049, 17, 0, 0, 0, 0, 0, 0, 0, 1002, 1026, 1025, 1006, 1015, 1016, 1060, 1016, 48, 24, 0, 0, 0, 0, 0, 0, 1005, 1005, 978, 1031, 961, 1003, 1023, 1033, 24, 15, 0, 0, 0, 0, 0, 0, 1016, 1017, 1020, 1040, 1007, 1025, 1029, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1057, 1069, 1056, 1001, 1023, 1010, 1018, 13, 5, 0, 0, 0, 0, 0, 0, 1056, 1072, 1078, 962, 997, 1003, 1034, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8121, 8208, 8238, 8078, 8160, 8284, 8257] avg=8192.0000 max=8284 max_rank=6 min=8078 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8064, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8192, 8192, 8320, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8448, 8064, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8064, 8192, 8448, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8192, 8320, 8192, 8192, 8192, 8320, 8192, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[987, 1032, 1054, 1033, 1046, 996, 1008, 1005, 987, 1056, 1045, 1043, 1031, 1056, 1020, 1039, 1019, 1025, 980, 983, 1051, 1025, 1054, 1016, 1026, 998, 1012, 1028, 1046, 1015, 1050, 1027, 1033, 980, 1034, 1024, 1022, 1025, 1012, 1057, 1066, 1047, 1011, 1022, 1058, 993, 1008, 1028, 976, 990, 1022, 994, 1072, 1028, 1026, 1049, 1031, 1067, 991, 1022, 1009, 1042, 1045, 959] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1031, 966, 1040, 1032, 1061, 1029, 1027, 1029, 1007, 999, 1052, 987, 990, 1021, 1032, 994, 1058, 980, 1041, 1083, 1031, 1060, 1037, 1031, 1043, 1026, 1000, 1071, 1010, 1063, 1008, 987, 1033, 1088, 1034, 1025, 1025, 1024, 1034, 1035, 990, 979, 998, 989, 1032, 1023, 1005, 1013, 1037, 1034, 1031, 974, 1034, 1070, 1049, 1040, 1009, 1044, 1020, 1012, 1047, 997, 1003, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[977, 1021, 1012, 1076, 1035, 1013, 1022, 1038, 1017, 1013, 991, 1046, 1006, 999, 1016, 1044, 987, 1088, 1049, 1006, 1037, 1040, 1011, 1009, 1027, 1030, 1041, 1007, 1039, 1024, 993, 1030, 1020, 971, 1040, 977, 980, 1015, 1013, 1062, 1037, 1032, 1035, 1007, 972, 1072, 1022, 1061, 1034, 1027, 1031, 1022, 1032, 1077, 1047, 1003, 1059, 1049, 992, 1040, 998, 1050, 989, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1031, 1047, 993, 1042, 1044, 1005, 1015, 1052, 1012, 1060, 1007, 1003, 969, 1032, 1040, 1020, 1026, 1022, 1024, 1014, 1025, 1043, 1000, 1005, 1000, 1015, 1042, 1011, 1046, 998, 1016, 1006, 939, 1065, 1044, 997, 1018, 1004, 1045, 1007, 1029, 1032, 1065, 1022, 1002, 1008, 1020, 1038, 1030, 1049, 988, 1006, 1047, 1075, 1028, 967, 1022, 1026, 1037, 1110, 1085, 988, 1036, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1004, 1052, 1006, 1009, 1049, 1017, 1022, 996, 962, 1011, 983, 990, 1068, 1070, 996, 1017, 998, 988, 1018, 1058, 1010, 1003, 1043, 1011, 1033, 1039, 1046, 1014, 1050, 1048, 1011, 1008, 1075, 954, 1074, 1027, 1016, 1066, 994, 998, 1086, 1037, 1035, 1043, 1013, 1079, 1011, 1015, 1017, 1036, 1057, 1021, 985, 1015, 991, 1009, 1041, 1069, 1050, 1029, 1034, 982, 1003, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1022, 1032, 988, 1058, 1033, 1004, 1078, 1036, 1036, 1007, 1036, 966, 1023, 989, 1008, 991, 1070, 1029, 1068, 982, 1045, 1083, 1019, 1056, 981, 1045, 1023, 1101, 1087, 990, 1020, 984, 1040, 1043, 1044, 1005, 1039, 1007, 987, 952, 1059, 985, 1011, 1030, 1015, 1025, 1000, 995, 1015, 1051, 1070, 1018, 1085, 1010, 1034, 1023, 1012, 990, 1049, 995, 1025, 986, 1044, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1000, 1041, 1027, 1033, 1011, 1013, 1024, 1023, 1013, 1026, 1012, 983, 1047, 1047, 1052, 1071, 1004, 1014, 1007, 1030, 1009, 1063, 1032, 1066, 1043, 1028, 1055, 1032, 1023, 1034, 998, 978, 1010, 1037, 1000, 1042, 1059, 1039, 960, 1015, 1014, 991, 1008, 999, 986, 1041, 1017, 1009, 1055, 995, 1035, 1013, 1037, 992, 1075, 1065, 1010, 1064, 1011, 1005, 1032, 999, 1029, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1035, 1036, 975, 1010, 1007, 998, 1029, 1004, 958, 1036, 1009, 997, 1043, 1080, 1024, 1060, 1023, 1073, 1022, 1066, 1056, 992, 1010, 1019, 1050, 1054, 1013, 1050, 1034, 1055, 1028, 1055, 1035, 970, 1055, 1011, 1034, 976, 1044, 1084, 1035, 980, 1009, 1027, 1040, 1072, 1015, 964, 1041, 1082, 975, 986, 1043, 1027, 982, 960, 1009, 1083, 1010, 969, 1023, 1038, 989, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[987, 1032, 1054, 1033, 1046, 996, 1008, 1005, 4, 3, 0, 0, 0, 0, 0, 0, 987, 1056, 1045, 1043, 1031, 1056, 1020, 1039, 35, 0, 0, 0, 0, 0, 0, 0, 1019, 1025, 980, 983, 1051, 990, 1054, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 998, 1012, 1028, 1021, 1015, 1050, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 980, 1034, 1024, 1022, 1025, 1012, 1057, 16, 4, 0, 0, 0, 0, 0, 0, 1066, 1047, 1011, 1022, 1058, 993, 1008, 1028, 21, 0, 0, 0, 0, 0, 0, 0, 976, 990, 1022, 994, 1053, 1028, 1026, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1063, 991, 1022, 1009, 1042, 1045, 959, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8312, 8118, 8177, 8207, 8254, 8138, 8162] avg=8192.0000 max=8312 max_rank=1 min=8118 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32761 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[977, 1021, 1012, 1076, 1035, 1013, 1022, 1038, 4, 3, 0, 0, 0, 0, 0, 0, 1017, 1013, 991, 1046, 1006, 999, 1016, 1044, 36, 0, 0, 0, 0, 0, 0, 0, 987, 1088, 1049, 1006, 1037, 1004, 1011, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1030, 1041, 1007, 1014, 1024, 993, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 971, 1040, 977, 980, 1015, 1013, 1062, 15, 4, 0, 0, 0, 0, 0, 0, 1037, 1032, 1035, 1007, 972, 1072, 1022, 1061, 21, 0, 0, 0, 0, 0, 0, 0, 1034, 1027, 1031, 1022, 1014, 1077, 1047, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1045, 992, 1040, 998, 1050, 989, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8168, 8191, 8166, 8097, 8259, 8255, 8199] avg=8192.0000 max=8259 max_rank=5 min=8097 min_rank=4 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32810 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1031, 966, 1040, 1032, 1061, 1029, 1027, 1029, 4, 3, 0, 0, 0, 0, 0, 0, 1007, 999, 1052, 987, 990, 1021, 1032, 994, 37, 0, 0, 0, 0, 0, 0, 0, 1058, 980, 1041, 1083, 1031, 1023, 1037, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1026, 1000, 1071, 985, 1063, 1008, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1088, 1034, 1025, 1025, 1024, 1034, 1035, 15, 4, 0, 0, 0, 0, 0, 0, 990, 979, 998, 989, 1032, 1023, 1005, 1013, 21, 0, 0, 0, 0, 0, 0, 0, 1037, 1034, 1031, 974, 1016, 1070, 1049, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1040, 1020, 1012, 1047, 997, 1003, 982, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8119, 8284, 8183, 8317, 8050, 8251, 8110] avg=8192.0000 max=8317 max_rank=4 min=8050 min_rank=5 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32728 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1031, 1047, 993, 1042, 1044, 1005, 1015, 1052, 4, 3, 0, 0, 0, 0, 0, 0, 1012, 1060, 1007, 1003, 969, 1032, 1040, 1020, 36, 0, 0, 0, 0, 0, 0, 0, 1026, 1022, 1024, 1014, 1025, 1007, 1000, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1015, 1042, 1011, 1021, 998, 1016, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 939, 1065, 1044, 997, 1018, 1004, 1045, 1007, 15, 4, 0, 0, 0, 0, 0, 0, 1029, 1032, 1065, 1022, 1002, 1008, 1020, 1038, 21, 0, 0, 0, 0, 0, 0, 0, 1030, 1049, 988, 1006, 1029, 1075, 1028, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1022, 1037, 1110, 1085, 988, 1036, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8179, 8123, 8109, 8138, 8237, 8172, 8342] avg=8192.0000 max=8342 max_rank=7 min=8109 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32889 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1004, 1052, 1006, 1009, 1049, 1017, 1022, 996, 4, 3, 0, 0, 0, 0, 0, 0, 962, 1011, 983, 990, 1068, 1070, 996, 1017, 35, 0, 0, 0, 0, 0, 0, 0, 998, 988, 1018, 1058, 1010, 968, 1043, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1039, 1046, 1014, 1025, 1048, 1011, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 954, 1074, 1027, 1016, 1066, 994, 998, 15, 4, 0, 0, 0, 0, 0, 0, 1086, 1037, 1035, 1043, 1013, 1079, 1011, 1015, 21, 0, 0, 0, 0, 0, 0, 0, 1017, 1036, 1057, 1021, 967, 1015, 991, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1065, 1050, 1029, 1034, 982, 1003, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8132, 8094, 8224, 8223, 8340, 8113, 8248] avg=8192.0000 max=8340 max_rank=5 min=8094 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32924 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1022, 1032, 988, 1058, 1033, 1004, 1078, 1036, 4, 3, 0, 0, 0, 0, 0, 0, 1036, 1007, 1036, 966, 1023, 989, 1008, 991, 37, 0, 0, 0, 0, 0, 0, 0, 1070, 1029, 1068, 982, 1045, 1046, 1019, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1045, 1023, 1101, 1061, 990, 1020, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1043, 1044, 1005, 1039, 1007, 987, 952, 16, 4, 0, 0, 0, 0, 0, 0, 1059, 985, 1011, 1030, 1015, 1025, 1000, 995, 22, 0, 0, 0, 0, 0, 0, 0, 1015, 1051, 1070, 1018, 1066, 1010, 1034, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 986, 1049, 995, 1025, 986, 1044, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8093, 8315, 8205, 8137, 8142, 8287, 8099] avg=8192.0000 max=8315 max_rank=2 min=8093 min_rank=1 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32665 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1000, 1041, 1027, 1033, 1011, 1013, 1024, 1023, 4, 3, 0, 0, 0, 0, 0, 0, 1013, 1026, 1012, 983, 1047, 1047, 1052, 1071, 37, 0, 0, 0, 0, 0, 0, 0, 1004, 1014, 1007, 1030, 1009, 1026, 1032, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1028, 1055, 1032, 998, 1034, 998, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1037, 1000, 1042, 1059, 1039, 960, 1015, 15, 4, 0, 0, 0, 0, 0, 0, 1014, 991, 1008, 999, 986, 1041, 1017, 1009, 21, 0, 0, 0, 0, 0, 0, 0, 1055, 995, 1035, 1013, 1019, 992, 1075, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1060, 1011, 1005, 1032, 999, 1029, 1053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8288, 8188, 8166, 8181, 8086, 8249, 8199] avg=8192.0000 max=8288 max_rank=1 min=8086 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32715 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1035, 1036, 975, 1010, 1007, 998, 1029, 1004, 11, 5, 0, 0, 0, 0, 0, 0, 958, 1036, 1009, 997, 1043, 1080, 1024, 1060, 38, 0, 0, 0, 0, 0, 0, 0, 1023, 1073, 1022, 1066, 1056, 954, 1010, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1054, 1013, 1050, 1001, 1055, 1028, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 970, 1055, 1011, 1034, 976, 1044, 1084, 20, 7, 0, 0, 0, 0, 0, 0, 1035, 980, 1009, 1027, 1040, 1072, 1015, 964, 26, 0, 0, 0, 0, 0, 0, 0, 1041, 1082, 975, 986, 1018, 1027, 982, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1072, 1010, 969, 1023, 1038, 989, 1067, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8245, 8223, 8306, 8236, 8168, 8071, 8177] avg=8192.0000 max=8306 max_rank=3 min=8071 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32652 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8064, 8192, 8320, 8192, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8064, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8448, 8192, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8448, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1017, 1056, 1017, 975, 1081, 1057, 1045, 1085, 984, 1030, 1025, 1043, 1076, 1027, 1020, 985, 1014, 1028, 1003, 1015, 994, 1032, 989, 987, 1004, 1021, 1006, 1078, 1033, 1013, 986, 1126, 1021, 988, 954, 1080, 1035, 1079, 1008, 1053, 1001, 1028, 1015, 1040, 930, 1029, 1068, 1032, 994, 1048, 1031, 1045, 1010, 981, 1035, 1034, 1017, 1049, 1060, 977, 1036, 1009, 1010, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1035, 1031, 1012, 973, 1048, 991, 1001, 1056, 974, 1026, 996, 1002, 984, 1038, 1022, 1041, 1030, 1026, 1055, 1085, 1005, 1065, 1041, 1015, 995, 1064, 1029, 1040, 1081, 998, 992, 1028, 968, 1031, 1005, 1021, 983, 1059, 1065, 1038, 1018, 1057, 1061, 1004, 1048, 1046, 998, 1017, 1074, 1009, 1022, 971, 1047, 1042, 1017, 1009, 1019, 1031, 1003, 1025, 939, 1053, 1017, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1031, 1073, 1028, 987, 1049, 1093, 1023, 1010, 1050, 1066, 1014, 1070, 1016, 1063, 1017, 1007, 1009, 992, 1007, 968, 1006, 1028, 987, 990, 1022, 1061, 1078, 1032, 1027, 1000, 980, 1019, 1005, 1010, 1051, 1037, 1034, 1002, 1079, 1045, 1032, 1012, 1079, 1043, 1017, 1066, 991, 983, 1038, 1028, 985, 1008, 1005, 1023, 1013, 1022, 1023, 1020, 1038, 1015, 1029, 1011, 1003, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1031, 1043, 1030, 1027, 1039, 1052, 1018, 1004, 1017, 1078, 1025, 996, 1024, 1036, 1014, 1048, 1015, 995, 1042, 996, 1036, 1055, 993, 991, 1017, 975, 1051, 1003, 976, 1011, 1046, 1043, 1039, 990, 1006, 997, 1002, 1035, 1046, 1022, 1015, 1040, 1031, 989, 1024, 1015, 996, 969, 1024, 1066, 1035, 1062, 1006, 1065, 1002, 1009, 1054, 1042, 1042, 1019, 1034, 1047, 1035, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1058, 1000, 1043, 1028, 1027, 1027, 1021, 922, 1049, 1040, 1057, 1049, 1004, 1010, 1013, 1036, 1016, 989, 989, 974, 1031, 1030, 1048, 1026, 1037, 1039, 1020, 992, 991, 1027, 1064, 963, 977, 1052, 1050, 1030, 1058, 1021, 979, 1056, 1082, 1056, 989, 1027, 1001, 990, 1047, 1030, 1014, 1061, 1019, 1009, 1055, 1068, 1040, 1014, 990, 1038, 983, 1015, 1008, 1054, 1055, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1047, 1028, 979, 1006, 1020, 973, 1005, 1001, 1028, 1022, 1029, 1004, 1042, 1040, 1039, 965, 1009, 1057, 985, 1025, 1028, 1032, 1034, 995, 1006, 1065, 999, 1050, 1032, 1031, 1028, 988, 1027, 1051, 993, 1038, 1002, 1077, 1015, 1047, 1091, 1028, 1057, 1030, 1043, 1007, 1047, 968, 1017, 1005, 1042, 1062, 997, 1034, 1036, 1034, 1083, 1023, 1018, 1002, 1011, 1022, 991, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1008, 1057, 1010, 1038, 1018, 1054, 1019, 1035, 1056, 1058, 1034, 1013, 995, 1010, 1010, 1044, 1002, 1001, 1042, 1101, 1028, 1019, 989, 999, 1044, 999, 1000, 1026, 1029, 1032, 995, 1037, 994, 1040, 1032, 1022, 1089, 1049, 1012, 1020, 1029, 1037, 1037, 999, 1043, 1027, 1007, 992, 1020, 1038, 993, 972, 1077, 1017, 1049, 1011, 1031, 973, 981, 1010, 977, 1059, 1085, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1010, 1040, 1037, 1008, 1053, 1044, 1035, 1063, 1025, 1036, 988, 1026, 1044, 1011, 992, 1021, 1089, 1072, 992, 980, 1004, 1063, 1041, 1021, 999, 1071, 1032, 1020, 1005, 1055, 995, 1022, 1001, 1042, 983, 973, 1014, 993, 1011, 1057, 993, 997, 1060, 978, 1010, 1017, 1055, 1045, 1020, 993, 1032, 1059, 1013, 1043, 990, 1031, 1052, 1026, 992, 993, 1079, 1030, 1056, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1017, 1056, 1017, 975, 1056, 1057, 1045, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1023, 1025, 1043, 1076, 1027, 1020, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1028, 1003, 1015, 994, 1032, 989, 987, 25, 10, 5, 0, 0, 0, 0, 0, 1004, 1021, 1006, 1078, 1033, 1013, 986, 1126, 1, 0, 0, 0, 0, 0, 0, 0, 1021, 988, 954, 1080, 1035, 1079, 1008, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1028, 1015, 1040, 930, 1029, 1068, 1032, 2, 0, 0, 0, 0, 0, 0, 0, 994, 1048, 1031, 1045, 1010, 971, 1035, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1049, 1060, 977, 1036, 1009, 1010, 987, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8183, 8102, 8268, 8212, 8145, 8168, 8150] avg=8192.0000 max=8308 max_rank=0 min=8102 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32675 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1031, 1043, 1030, 1027, 1015, 1052, 1018, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1071, 1025, 996, 1024, 1036, 1014, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 995, 1042, 996, 1036, 1055, 993, 991, 24, 11, 5, 0, 0, 0, 0, 0, 1017, 975, 1051, 1003, 976, 1011, 1046, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 990, 1006, 997, 1002, 1035, 1046, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1040, 1031, 989, 1024, 1015, 996, 969, 2, 0, 0, 0, 0, 0, 0, 0, 1024, 1066, 1035, 1062, 1006, 1054, 1002, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1042, 1042, 1019, 1034, 1047, 1035, 1051, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8231, 8163, 8122, 8132, 8081, 8258, 8329] avg=8192.0000 max=8329 max_rank=7 min=8081 min_rank=5 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1035, 1031, 1012, 973, 1024, 991, 1001, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1019, 996, 1002, 984, 1038, 1022, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1026, 1055, 1085, 1005, 1065, 1041, 1015, 24, 11, 5, 0, 0, 0, 0, 0, 995, 1064, 1029, 1040, 1081, 998, 992, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1031, 1005, 1021, 983, 1059, 1065, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1057, 1061, 1004, 1048, 1046, 998, 1017, 2, 0, 0, 0, 0, 0, 0, 0, 1074, 1009, 1022, 971, 1047, 1031, 1017, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1031, 1003, 1025, 939, 1053, 1017, 1060, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8076, 8362, 8227, 8165, 8251, 8180, 8152] avg=8192.0000 max=8362 max_rank=2 min=8076 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1031, 1073, 1028, 987, 1025, 1093, 1023, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1059, 1014, 1070, 1016, 1063, 1017, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 992, 1007, 968, 1006, 1028, 987, 990, 24, 11, 5, 0, 0, 0, 0, 0, 1022, 1061, 1078, 1032, 1027, 1000, 980, 1019, 1, 0, 0, 0, 0, 0, 0, 0, 1005, 1010, 1051, 1037, 1034, 1002, 1079, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1012, 1079, 1043, 1017, 1066, 991, 983, 2, 0, 0, 0, 0, 0, 0, 0, 1038, 1028, 985, 1008, 1005, 1012, 1013, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1020, 1038, 1015, 1029, 1011, 1003, 986, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8296, 8027, 8220, 8257, 8225, 8111, 8130] avg=8192.0000 max=8296 max_rank=1 min=8027 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1058, 1000, 1043, 1028, 1003, 1027, 1021, 922, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1033, 1057, 1049, 1004, 1010, 1013, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 989, 989, 974, 1031, 1030, 1048, 1026, 24, 11, 5, 0, 0, 0, 0, 0, 1037, 1039, 1020, 992, 991, 1027, 1064, 963, 1, 0, 0, 0, 0, 0, 0, 0, 977, 1052, 1050, 1030, 1058, 1021, 979, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 1056, 989, 1027, 1001, 990, 1047, 1030, 2, 0, 0, 0, 0, 0, 0, 0, 1014, 1061, 1019, 1009, 1055, 1057, 1040, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1038, 983, 1015, 1008, 1054, 1055, 1048, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8251, 8143, 8134, 8217, 8224, 8269, 8196] avg=8192.0000 max=8269 max_rank=6 min=8102 min_rank=0 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1008, 1057, 1010, 1038, 995, 1054, 1019, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1051, 1034, 1013, 995, 1010, 1010, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1001, 1042, 1101, 1028, 1019, 989, 999, 23, 10, 5, 0, 0, 0, 0, 0, 1044, 999, 1000, 1026, 1029, 1032, 995, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1040, 1032, 1022, 1089, 1049, 1012, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1037, 1037, 999, 1043, 1027, 1007, 992, 2, 0, 0, 0, 0, 0, 0, 0, 1020, 1038, 993, 972, 1077, 1007, 1049, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 973, 981, 1010, 977, 1059, 1085, 1012, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8213, 8219, 8162, 8253, 8173, 8167, 8133] avg=8192.0000 max=8253 max_rank=4 min=8133 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1047, 1028, 979, 1006, 997, 973, 1005, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1015, 1029, 1004, 1042, 1040, 1039, 965, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1057, 985, 1025, 1028, 1032, 1034, 995, 23, 11, 5, 0, 0, 0, 0, 0, 1006, 1065, 999, 1050, 1032, 1031, 1028, 988, 1, 0, 0, 0, 0, 0, 0, 0, 1027, 1051, 993, 1038, 1002, 1077, 1015, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 1028, 1057, 1030, 1043, 1007, 1047, 968, 2, 0, 0, 0, 0, 0, 0, 0, 1017, 1005, 1042, 1062, 997, 1023, 1036, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 1023, 1018, 1002, 1011, 1022, 991, 1046, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8036, 8162, 8204, 8200, 8244, 8273, 8216, 8201] avg=8192.0000 max=8273 max_rank=5 min=8036 min_rank=0 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1010, 1040, 1037, 1008, 1024, 1044, 1035, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1017, 988, 1026, 1044, 1011, 992, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 1072, 992, 980, 1004, 1063, 1041, 1021, 29, 14, 11, 0, 0, 0, 0, 0, 999, 1071, 1032, 1020, 1005, 1055, 995, 1022, 4, 0, 0, 0, 0, 0, 0, 0, 1001, 1042, 983, 973, 1014, 993, 1011, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 993, 997, 1060, 978, 1010, 1017, 1055, 1045, 9, 0, 0, 0, 0, 0, 0, 0, 1020, 993, 1032, 1059, 1013, 1029, 990, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1026, 992, 993, 1079, 1030, 1056, 1004, 5, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8124, 8316, 8203, 8056, 8164, 8167, 8245] avg=8192.0000 max=8316 max_rank=2 min=8056 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8448, 8192, 8064, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8448, 8192, 8064, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8448, 8192, 8192, 8320, 8320, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8064, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1036, 1013, 1048, 1017, 1006, 1008, 982, 990, 990, 1029, 1045, 1058, 1034, 1051, 993, 1023, 996, 1029, 1071, 996, 1060, 1022, 1011, 1033, 998, 1005, 1026, 1055, 991, 1029, 1015, 1040, 1079, 970, 1042, 1073, 992, 1016, 977, 1054, 1022, 1035, 1002, 1047, 996, 1049, 1049, 954, 1058, 999, 1059, 1020, 1029, 1068, 999, 995, 1017, 1060, 1023, 1070, 1029, 1026, 1003, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1075, 1023, 1011, 1004, 1023, 1006, 1048, 984, 1034, 978, 1016, 1005, 1069, 1012, 1032, 1014, 1029, 1038, 998, 1056, 1024, 1024, 960, 1039, 994, 998, 1072, 992, 999, 991, 1044, 1045, 1065, 988, 1053, 1042, 1065, 1042, 1019, 1026, 1028, 1029, 1000, 1059, 1048, 1030, 1005, 990, 1054, 1050, 1048, 1022, 1051, 1009, 1036, 1061, 1008, 1052, 1006, 1014, 1017, 1013, 992, 977] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1064, 982, 1061, 992, 996, 1023, 1033, 1048, 977, 1008, 1030, 1028, 1014, 1011, 1018, 1075, 999, 991, 1026, 1013, 1004, 1020, 1019, 1002, 1041, 1063, 1016, 1070, 1028, 1044, 1009, 981, 1056, 979, 1078, 999, 1071, 1030, 1009, 995, 1052, 1023, 1031, 979, 1027, 1043, 1021, 1027, 1064, 1068, 1046, 1055, 993, 1006, 1009, 1017, 1056, 1008, 1014, 988, 1026, 1033, 1005, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1004, 1019, 1058, 1018, 1030, 999, 1008, 1016, 1042, 974, 1022, 1006, 965, 1043, 1065, 1030, 1024, 1041, 1061, 1017, 1058, 1020, 1051, 995, 1069, 1041, 1021, 1005, 1064, 1028, 993, 973, 982, 1009, 1046, 1015, 1021, 968, 997, 1064, 1049, 1020, 1018, 1064, 1018, 1034, 1012, 1030, 985, 1023, 1057, 1028, 1037, 1038, 1013, 1011, 1016, 1057, 1020, 1087, 1060, 998, 954, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[993, 1050, 1044, 995, 1006, 1033, 1041, 994, 1036, 1015, 1012, 987, 1002, 1033, 1043, 1009, 1016, 1004, 1014, 1080, 1005, 997, 1065, 1009, 1004, 1020, 986, 1016, 1038, 1005, 1024, 1029, 1049, 1032, 1043, 1032, 1044, 1015, 1015, 1012, 1067, 1034, 1056, 1031, 1003, 1026, 995, 1045, 1062, 1031, 999, 1053, 993, 1000, 1038, 1041, 1056, 1023, 1062, 983, 1018, 1042, 1011, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1006, 1014, 1037, 1035, 1034, 988, 1010, 991, 1022, 1018, 1011, 1007, 1036, 1038, 1030, 1080, 1028, 1010, 1022, 1060, 1045, 1010, 1007, 1008, 1065, 962, 1036, 988, 994, 1007, 1018, 1048, 1016, 1015, 1074, 1049, 986, 1038, 1042, 1018, 1026, 1022, 978, 1006, 1001, 1049, 990, 981, 1017, 1064, 1019, 990, 993, 1077, 995, 1035, 1054, 1055, 1018, 1036, 1035, 1050, 1049, 1093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1033, 1019, 1002, 1049, 1015, 1021, 987, 1018, 1016, 1053, 1001, 1005, 1053, 1030, 1044, 1001, 943, 1038, 1048, 1013, 1053, 1030, 1018, 991, 1020, 967, 951, 1075, 1003, 1017, 995, 1011, 985, 1000, 1049, 1030, 1045, 1026, 1023, 1014, 1080, 1050, 994, 986, 1032, 1072, 1099, 1082, 1014, 1013, 1008, 1013, 1043, 1037, 1046, 1034, 1056, 990, 1049, 1006, 1022, 1017, 1026, 1075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1039, 1074, 1064, 1000, 1018, 1027, 1037, 1032, 1058, 1027, 1042, 985, 1019, 1067, 1001, 998, 1035, 1031, 1063, 1038, 1062, 964, 1015, 1016, 1055, 991, 986, 1002, 1033, 1009, 1005, 971, 977, 1059, 1011, 997, 1037, 1000, 1061, 1058, 1027, 1048, 1028, 1049, 1080, 1008, 1051, 993, 989, 992, 1037, 1028, 1048, 969, 1065, 983, 980, 1010, 1047, 1045, 1048, 1015, 1036, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1075, 1023, 1011, 1004, 1023, 1006, 1048, 984, 25, 0, 0, 0, 0, 0, 0, 0, 1034, 978, 1016, 1005, 1069, 1012, 1032, 1014, 8, 0, 0, 0, 0, 0, 0, 0, 1029, 1038, 998, 1056, 1024, 1024, 960, 1039, 8, 0, 0, 0, 0, 0, 0, 0, 994, 998, 1072, 992, 999, 991, 1044, 1045, 30, 26, 0, 0, 0, 0, 0, 0, 1065, 988, 1037, 1042, 1065, 1042, 1019, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1029, 1000, 1059, 1048, 1030, 1005, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1050, 1023, 1022, 1051, 1009, 1036, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1052, 1006, 1014, 1017, 1013, 992, 951, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8168, 8176, 8191, 8284, 8159, 8306, 8053] avg=8192.0000 max=8306 max_rank=6 min=8053 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1064, 982, 1061, 992, 996, 1023, 1033, 1048, 25, 0, 0, 0, 0, 0, 0, 0, 977, 1008, 1030, 1028, 1014, 1011, 1018, 1075, 8, 0, 0, 0, 0, 0, 0, 0, 999, 991, 1026, 1013, 1004, 1020, 1019, 1002, 9, 0, 0, 0, 0, 0, 0, 0, 1041, 1063, 1016, 1070, 1028, 1044, 1009, 981, 30, 27, 0, 0, 0, 0, 0, 0, 1056, 979, 1061, 999, 1071, 1030, 1009, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1023, 1031, 979, 1027, 1043, 1021, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1068, 1021, 1055, 993, 1006, 1009, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1008, 1014, 988, 1026, 1033, 1005, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8169, 8083, 8309, 8200, 8173, 8233, 8145] avg=8192.0000 max=8309 max_rank=3 min=8083 min_rank=2 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1006, 1014, 1037, 1035, 1034, 988, 1010, 991, 24, 0, 0, 0, 0, 0, 0, 0, 1022, 1018, 1011, 1007, 1036, 1038, 1030, 1080, 8, 0, 0, 0, 0, 0, 0, 0, 1028, 1010, 1022, 1060, 1045, 1010, 1007, 1008, 9, 0, 0, 0, 0, 0, 0, 0, 1065, 962, 1036, 988, 994, 1007, 1018, 1048, 29, 29, 0, 0, 0, 0, 0, 0, 1016, 1015, 1057, 1049, 986, 1038, 1042, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1022, 978, 1006, 1001, 1049, 990, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1064, 995, 990, 993, 1077, 995, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1055, 1018, 1036, 1035, 1050, 1049, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8250, 8199, 8176, 8221, 8024, 8166, 8361] avg=8192.0000 max=8361 max_rank=7 min=8024 min_rank=5 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1036, 1013, 1048, 1017, 1006, 1008, 982, 990, 25, 0, 0, 0, 0, 0, 0, 0, 990, 1029, 1045, 1058, 1034, 1051, 993, 1023, 8, 0, 0, 0, 0, 0, 0, 0, 996, 1029, 1071, 996, 1060, 1022, 1011, 1033, 8, 0, 0, 0, 0, 0, 0, 0, 998, 1005, 1026, 1055, 991, 1029, 1015, 1040, 29, 27, 0, 0, 0, 0, 0, 0, 1079, 970, 1026, 1073, 992, 1016, 977, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1035, 1002, 1047, 996, 1049, 1049, 954, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 999, 1034, 1020, 1029, 1068, 999, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1060, 1023, 1070, 1029, 1026, 1003, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8231, 8226, 8215, 8187, 8125, 8202, 8225] avg=8192.0000 max=8231 max_rank=1 min=8125 min_rank=0 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[993, 1050, 1044, 995, 1006, 1033, 1041, 994, 23, 0, 0, 0, 0, 0, 0, 0, 1036, 1015, 1012, 987, 1002, 1033, 1043, 1009, 8, 0, 0, 0, 0, 0, 0, 0, 1016, 1004, 1014, 1080, 1005, 997, 1065, 1009, 8, 0, 0, 0, 0, 0, 0, 0, 1004, 1020, 986, 1016, 1038, 1005, 1024, 1029, 31, 27, 0, 0, 0, 0, 0, 0, 1049, 1032, 1027, 1032, 1044, 1015, 1015, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1034, 1056, 1031, 1003, 1026, 995, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1031, 976, 1053, 993, 1000, 1038, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1023, 1062, 983, 1018, 1042, 1011, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8145, 8198, 8180, 8226, 8226, 8194, 8188] avg=8192.0000 max=8226 max_rank=4 min=8145 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1004, 1019, 1058, 1018, 1030, 999, 1008, 1016, 25, 0, 0, 0, 0, 0, 0, 0, 1042, 974, 1022, 1006, 965, 1043, 1065, 1030, 8, 0, 0, 0, 0, 0, 0, 0, 1024, 1041, 1061, 1017, 1058, 1020, 1051, 995, 8, 0, 0, 0, 0, 0, 0, 0, 1069, 1041, 1021, 1005, 1064, 1028, 993, 973, 30, 27, 0, 0, 0, 0, 0, 0, 982, 1009, 1030, 1015, 1021, 968, 997, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1020, 1018, 1064, 1018, 1034, 1012, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1023, 1032, 1028, 1037, 1038, 1013, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1057, 1020, 1087, 1060, 998, 954, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8155, 8275, 8251, 8086, 8215, 8167, 8210] avg=8192.0000 max=8275 max_rank=2 min=8086 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1033, 1019, 1002, 1049, 1015, 1021, 987, 1018, 24, 0, 0, 0, 0, 0, 0, 0, 1016, 1053, 1001, 1005, 1053, 1030, 1044, 1001, 8, 0, 0, 0, 0, 0, 0, 0, 943, 1038, 1048, 1013, 1053, 1030, 1018, 991, 8, 0, 0, 0, 0, 0, 0, 0, 1020, 967, 951, 1075, 1003, 1017, 995, 1011, 31, 28, 0, 0, 0, 0, 0, 0, 985, 1000, 1033, 1030, 1045, 1026, 1023, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1050, 994, 986, 1032, 1072, 1099, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1013, 984, 1013, 1043, 1037, 1046, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 990, 1049, 1006, 1022, 1017, 1026, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8211, 8142, 8098, 8156, 8364, 8184, 8213] avg=8192.0000 max=8364 max_rank=5 min=8098 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1039, 1074, 1064, 1000, 1018, 1027, 1037, 1032, 27, 6, 1, 0, 0, 0, 0, 0, 1058, 1027, 1042, 985, 1019, 1067, 1001, 998, 10, 0, 0, 0, 0, 0, 0, 0, 1035, 1031, 1063, 1038, 1062, 964, 1015, 1016, 13, 0, 0, 0, 0, 0, 0, 0, 1055, 991, 986, 1002, 1033, 1009, 1005, 971, 34, 30, 0, 0, 0, 0, 0, 0, 977, 1059, 987, 997, 1037, 1000, 1061, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1048, 1028, 1049, 1080, 1008, 1051, 993, 0, 0, 0, 0, 0, 0, 0, 0, 989, 992, 1010, 1028, 1048, 969, 1065, 983, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1010, 1047, 1045, 1048, 1015, 1036, 960, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8325, 8207, 8237, 8116, 8176, 8250, 8084, 8141] avg=8192.0000 max=8325 max_rank=0 min=8084 min_rank=6 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8320, 8320, 8448, 8192, 8192, 8192, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8320, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8192, 8320, 8192, 8192, 8192, 8192, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[952, 1018, 1056, 1032, 1000, 1042, 972, 998, 992, 1018, 1022, 993, 1016, 987, 1020, 1059, 989, 984, 1095, 1038, 1014, 1070, 1076, 1026, 1015, 955, 1002, 1105, 1058, 1022, 1077, 1043, 979, 975, 1055, 1028, 996, 1013, 1042, 1069, 1027, 947, 1034, 1036, 1051, 1032, 1027, 1019, 983, 1031, 1035, 993, 1003, 1005, 1002, 1021, 1075, 1023, 1014, 1024, 1082, 1095, 1039, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1008, 1020, 1035, 1023, 1010, 1036, 1012, 1030, 1021, 1025, 1049, 985, 1055, 999, 1010, 1014, 1057, 1060, 1100, 1063, 972, 1023, 991, 1035, 969, 1000, 963, 990, 1008, 1022, 1067, 972, 1011, 1027, 1038, 1061, 974, 1011, 963, 1080, 1047, 1055, 1066, 1049, 1048, 1034, 1025, 1031, 1036, 1000, 1080, 1042, 989, 985, 1011, 1070, 1003, 1059, 1023, 1041, 1009, 1036, 1022, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1005, 1039, 1029, 1062, 1008, 997, 993, 1031, 1062, 1016, 1055, 965, 1010, 1046, 1029, 983, 1057, 1046, 1015, 1053, 994, 1056, 1049, 1042, 1010, 1069, 1040, 1082, 1004, 998, 1052, 1001, 986, 956, 1039, 987, 1038, 1047, 1022, 1058, 1049, 1043, 1023, 1007, 985, 1002, 1001, 1020, 1071, 943, 960, 1070, 1062, 1037, 1022, 1015, 1003, 1009, 1047, 1014, 1051, 1056, 1016, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1015, 1049, 1035, 1032, 1045, 1056, 1048, 1022, 1031, 1065, 1021, 1051, 1052, 1040, 1030, 989, 1018, 980, 1012, 1002, 1037, 1011, 1014, 1052, 1029, 1036, 1046, 1067, 1025, 1043, 1031, 1012, 1055, 1020, 1016, 1016, 1022, 1026, 998, 1029, 997, 999, 1030, 1042, 962, 1037, 1025, 1043, 1011, 1013, 997, 1032, 992, 1018, 1041, 1042, 1026, 1014, 955, 1014, 984, 1059, 1006, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1029, 997, 1029, 1042, 1082, 1053, 1016, 1063, 1076, 1014, 1007, 1023, 1028, 985, 1064, 969, 981, 1012, 1051, 1018, 1022, 1024, 1041, 1015, 1028, 1018, 1034, 1071, 1019, 991, 1019, 1040, 1013, 977, 975, 1011, 980, 1003, 1046, 1013, 1021, 1062, 1005, 1042, 1049, 1029, 1029, 1049, 1084, 1024, 1018, 1030, 1043, 1027, 1009, 1052, 988, 1021, 986, 1016, 977, 1022, 1003, 1071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1060, 978, 1010, 1012, 1053, 1048, 1054, 1006, 1030, 994, 1002, 1042, 1057, 1058, 1016, 1011, 1051, 975, 1026, 1029, 1008, 1020, 974, 1015, 1002, 1012, 1020, 1037, 995, 1030, 1032, 990, 1054, 1015, 1037, 1035, 1044, 1020, 1022, 1010, 1010, 1042, 1042, 963, 1033, 998, 1025, 1045, 1030, 1018, 1010, 1042, 1053, 976, 1057, 1048, 1019, 1053, 1035, 1043, 1051, 978, 1012, 1069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1001, 991, 998, 1034, 1027, 1039, 1032, 999, 1076, 1046, 1008, 1053, 985, 998, 1032, 997, 1061, 1061, 1034, 987, 1021, 1023, 1039, 1069, 994, 1042, 999, 993, 1086, 1022, 1046, 1026, 1057, 1058, 1011, 1023, 1039, 1058, 1053, 1013, 1003, 1026, 1045, 1007, 1020, 1005, 997, 1039, 1009, 1043, 984, 1027, 1031, 993, 996, 1018, 1035, 1049, 1074, 985, 1005, 1004, 1021, 989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1010, 1070, 1044, 989, 1049, 1034, 1076, 1012, 977, 1045, 1038, 965, 976, 1060, 1067, 1013, 1000, 1003, 1024, 986, 1035, 1092, 1087, 1019, 1057, 984, 1062, 997, 1057, 976, 996, 985, 1027, 1012, 1005, 1012, 1047, 1032, 1018, 1063, 1046, 1039, 1029, 1041, 1052, 988, 1020, 1021, 982, 1003, 1018, 1069, 1027, 1059, 1004, 988, 1017, 1029, 1030, 1015, 990, 1019, 1022, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1005, 1039, 1029, 1062, 1008, 984, 993, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1016, 1055, 965, 1010, 1046, 1029, 983, 10, 3, 0, 0, 0, 0, 0, 0, 1057, 1046, 980, 1053, 994, 1056, 1049, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1069, 1040, 1082, 1004, 998, 1052, 1001, 4, 3, 0, 0, 0, 0, 0, 0, 986, 956, 1039, 987, 1038, 1047, 1022, 1058, 14, 0, 0, 0, 0, 0, 0, 0, 1049, 1043, 1014, 1007, 985, 1002, 1001, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 943, 960, 1070, 1062, 1037, 1022, 1015, 18, 0, 0, 0, 0, 0, 0, 0, 1003, 1009, 1047, 1014, 1051, 1056, 1016, 999, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8179, 8277, 8263, 8147, 8121, 8198, 8200] avg=8192.0000 max=8277 max_rank=2 min=8121 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1008, 1020, 1035, 1023, 1010, 1023, 1012, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1025, 1049, 985, 1055, 999, 1010, 1014, 10, 3, 0, 0, 0, 0, 0, 0, 1057, 1060, 1062, 1063, 972, 1023, 991, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1000, 963, 990, 1008, 1022, 1067, 972, 4, 3, 0, 0, 0, 0, 0, 0, 1011, 1027, 1038, 1061, 974, 1011, 963, 1080, 15, 0, 0, 0, 0, 0, 0, 0, 1047, 1055, 1057, 1049, 1048, 1034, 1025, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1000, 1080, 1042, 989, 985, 1011, 1070, 20, 0, 0, 0, 0, 0, 0, 0, 1003, 1059, 1023, 1041, 1009, 1036, 1022, 986, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8171, 8263, 7998, 8180, 8346, 8233, 8184] avg=8192.0000 max=8346 max_rank=5 min=7998 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32943 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1001, 991, 998, 1034, 1027, 1026, 1032, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1046, 1008, 1053, 985, 998, 1032, 997, 10, 3, 0, 0, 0, 0, 0, 0, 1061, 1061, 999, 987, 1021, 1023, 1039, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1042, 999, 993, 1086, 1022, 1046, 1026, 4, 3, 0, 0, 0, 0, 0, 0, 1057, 1058, 1011, 1023, 1039, 1058, 1053, 1013, 14, 0, 0, 0, 0, 0, 0, 0, 1003, 1026, 1036, 1007, 1020, 1005, 997, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1043, 984, 1027, 1031, 993, 996, 1018, 18, 0, 0, 0, 0, 0, 0, 0, 1035, 1049, 1074, 985, 1005, 1004, 1021, 989, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8208, 8260, 8215, 8326, 8133, 8119, 8167] avg=8192.0000 max=8326 max_rank=4 min=8108 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1029, 997, 1029, 1042, 1082, 1040, 1016, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1014, 1007, 1023, 1028, 985, 1064, 969, 10, 3, 0, 0, 0, 0, 0, 0, 981, 1012, 1015, 1018, 1022, 1024, 1041, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1018, 1034, 1071, 1019, 991, 1019, 1040, 4, 3, 0, 0, 0, 0, 0, 0, 1013, 977, 975, 1011, 980, 1003, 1046, 1013, 14, 0, 0, 0, 0, 0, 0, 0, 1021, 1062, 996, 1042, 1049, 1029, 1029, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 1024, 1018, 1030, 1043, 1027, 1009, 1052, 19, 0, 0, 0, 0, 0, 0, 0, 988, 1021, 986, 1016, 977, 1022, 1003, 1071, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8179, 8128, 8227, 8032, 8277, 8306, 8089] avg=8192.0000 max=8306 max_rank=6 min=8032 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[952, 1018, 1056, 1032, 1000, 1029, 972, 998, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1018, 1022, 993, 1016, 987, 1020, 1059, 10, 3, 0, 0, 0, 0, 0, 0, 989, 984, 1058, 1038, 1014, 1070, 1076, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 955, 1002, 1105, 1058, 1022, 1077, 1043, 4, 3, 0, 0, 0, 0, 0, 0, 979, 975, 1055, 1028, 996, 1013, 1042, 1069, 15, 0, 0, 0, 0, 0, 0, 0, 1027, 947, 1025, 1036, 1051, 1032, 1027, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1031, 1035, 993, 1003, 1005, 1002, 1021, 19, 0, 0, 0, 0, 0, 0, 0, 1075, 1023, 1014, 1024, 1082, 1095, 1039, 1035, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8057, 8120, 8255, 8284, 8172, 8164, 8092, 8392] avg=8192.0000 max=8392 max_rank=7 min=8057 min_rank=0 imbalance_ratio=1.024414 max_load_reduction=57144 max_load_reduction_pct=0.871948 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1060, 978, 1010, 1012, 1053, 1035, 1054, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 994, 1002, 1042, 1057, 1058, 1016, 1011, 10, 3, 0, 0, 0, 0, 0, 0, 1051, 975, 991, 1029, 1008, 1020, 974, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1012, 1020, 1037, 995, 1030, 1032, 990, 4, 3, 0, 0, 0, 0, 0, 0, 1054, 1015, 1037, 1035, 1044, 1020, 1022, 1010, 14, 0, 0, 0, 0, 0, 0, 0, 1010, 1042, 1033, 963, 1033, 998, 1025, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1018, 1010, 1042, 1053, 976, 1057, 1048, 18, 0, 0, 0, 0, 0, 0, 0, 1019, 1053, 1035, 1043, 1051, 978, 1012, 1069, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8223, 8063, 8125, 8251, 8149, 8252, 8265] avg=8192.0000 max=8265 max_rank=7 min=8063 min_rank=2 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1015, 1049, 1035, 1032, 1045, 1043, 1048, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1065, 1021, 1051, 1052, 1040, 1030, 989, 10, 3, 0, 0, 0, 0, 0, 0, 1018, 980, 977, 1002, 1037, 1011, 1014, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1036, 1046, 1067, 1025, 1043, 1031, 1012, 4, 3, 0, 0, 0, 0, 0, 0, 1055, 1020, 1016, 1016, 1022, 1026, 998, 1029, 14, 0, 0, 0, 0, 0, 0, 0, 997, 999, 1021, 1042, 962, 1037, 1025, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1013, 997, 1032, 992, 1018, 1041, 1042, 18, 0, 0, 0, 0, 0, 0, 0, 1026, 1014, 955, 1014, 984, 1059, 1006, 1019, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8289, 8292, 8091, 8296, 8196, 8126, 8164, 8082] avg=8192.0000 max=8296 max_rank=3 min=8082 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1010, 1070, 1044, 989, 1049, 1014, 1076, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1045, 1038, 965, 976, 1060, 1067, 1013, 14, 9, 0, 0, 0, 0, 0, 0, 1000, 1003, 977, 986, 1035, 1092, 1087, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 984, 1062, 997, 1057, 976, 996, 985, 8, 6, 0, 0, 0, 0, 0, 0, 1027, 1012, 1005, 1012, 1047, 1032, 1018, 1063, 16, 0, 0, 0, 0, 0, 0, 0, 1046, 1039, 1013, 1041, 1052, 988, 1020, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1003, 1018, 1069, 1027, 1059, 1004, 988, 22, 0, 0, 0, 0, 0, 0, 0, 1017, 1029, 1030, 1015, 990, 1019, 1022, 1027, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8164, 8199, 8128, 8232, 8220, 8172, 8157] avg=8192.0000 max=8264 max_rank=0 min=8128 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8448, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8192, 8192, 8320, 8192, 8448, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1039, 1020, 992, 1038, 1014, 995, 1018, 1043, 1023, 1041, 1018, 1034, 1038, 1074, 1046, 996, 1001, 1011, 1050, 1004, 1068, 1031, 1030, 1072, 1002, 1005, 986, 993, 1035, 1022, 975, 1040, 959, 1057, 1038, 1028, 1034, 1054, 974, 1051, 1004, 1091, 1094, 1014, 1016, 1002, 1023, 1022, 1002, 998, 1008, 994, 1025, 1059, 1063, 1006, 976, 1027, 979, 1058, 1036, 1060, 1043, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1005, 1107, 1009, 1043, 1036, 989, 1020, 1040, 1006, 1023, 1003, 1007, 998, 1027, 998, 1015, 1055, 1017, 1042, 1025, 1011, 1081, 996, 1041, 1009, 1032, 996, 1038, 1052, 1054, 1007, 1007, 1012, 986, 1011, 1020, 1061, 1066, 973, 1009, 1025, 1028, 1029, 1017, 1027, 997, 1033, 981, 1025, 1080, 1005, 1031, 1038, 1042, 1026, 1015, 969, 1057, 1021, 1072, 1002, 985, 1027, 1077] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[988, 1000, 1028, 1078, 996, 1002, 987, 999, 1027, 1043, 1029, 988, 1037, 1074, 1002, 1081, 1024, 971, 1002, 1015, 1040, 986, 991, 1015, 1058, 1049, 1072, 1078, 1074, 1017, 1019, 1027, 1021, 1022, 986, 1020, 1018, 1013, 1005, 1043, 981, 1026, 1056, 998, 1033, 1040, 1057, 1027, 1029, 1038, 1015, 1018, 1026, 1004, 1035, 995, 1026, 1078, 991, 1015, 1056, 984, 1030, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1060, 1019, 1063, 1024, 995, 976, 1018, 1070, 1008, 1021, 1056, 1013, 1030, 983, 1024, 1031, 1040, 1033, 1046, 1033, 1035, 1063, 1047, 1033, 1062, 967, 1012, 1047, 1024, 1023, 1002, 1051, 1016, 1004, 1008, 1053, 1056, 1032, 1027, 1019, 1002, 1060, 1025, 1028, 1020, 1035, 970, 1036, 1052, 1035, 1006, 1015, 1074, 1003, 977, 990, 972, 1027, 1020, 1003, 1055, 1008, 1017, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1035, 1051, 1039, 1026, 1038, 1014, 985, 1069, 1059, 1073, 1030, 1028, 1089, 1007, 964, 1022, 1001, 1047, 978, 1018, 1064, 991, 1031, 1030, 1047, 1022, 1008, 1082, 976, 993, 1024, 1027, 1001, 993, 1013, 1069, 999, 989, 977, 1053, 947, 1064, 996, 1042, 1029, 1022, 973, 1015, 1039, 1028, 1049, 1046, 963, 1037, 1060, 1009, 1009, 1029, 1022, 1007, 1021, 1060, 1019, 1088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1011, 1037, 971, 989, 1075, 1036, 1016, 1042, 1079, 1037, 995, 1006, 953, 1031, 1083, 1017, 981, 1020, 1006, 997, 995, 1048, 1030, 1047, 1019, 1028, 1094, 1023, 1043, 1017, 1002, 986, 1037, 1024, 1003, 989, 1032, 1039, 1095, 1060, 1008, 1002, 1015, 1029, 1005, 1039, 1022, 986, 1019, 1044, 1023, 1077, 1052, 1046, 984, 1044, 1017, 1082, 1018, 988, 990, 1030, 1000, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[983, 1010, 991, 1064, 991, 1027, 1020, 1026, 1046, 1029, 1002, 1025, 1039, 1038, 977, 1005, 1027, 1006, 1021, 1016, 1018, 1055, 1044, 1027, 1046, 1092, 963, 994, 1017, 1058, 994, 1051, 1054, 1028, 1032, 1023, 1022, 1023, 1014, 1005, 1009, 1067, 999, 1048, 987, 1052, 1002, 985, 1029, 1042, 1051, 986, 983, 1029, 1011, 1040, 1024, 1074, 995, 988, 1034, 1098, 1057, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1034, 985, 1012, 1015, 1035, 1006, 1034, 1070, 1055, 1010, 966, 953, 1005, 1028, 987, 1044, 1006, 992, 1035, 1051, 1057, 1004, 1000, 981, 1018, 1038, 1093, 1112, 1075, 990, 1019, 1004, 1040, 1060, 1070, 994, 1050, 1065, 999, 995, 1019, 1041, 974, 1007, 976, 1099, 1005, 1063, 1003, 1051, 1049, 1021, 1004, 1000, 1041, 1056, 1022, 997, 1040, 1031, 1004, 1028, 1025, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1005, 1107, 1009, 1043, 1036, 989, 1020, 1040, 5, 5, 0, 0, 0, 0, 0, 0, 1006, 1023, 1003, 1007, 998, 1027, 998, 1015, 7, 0, 0, 0, 0, 0, 0, 0, 1055, 1017, 1042, 1025, 1011, 1081, 996, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1032, 996, 1007, 1052, 1054, 1007, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 986, 1011, 1020, 1061, 1066, 973, 1009, 1, 0, 0, 0, 0, 0, 0, 0, 1025, 1028, 1029, 1017, 1027, 997, 1033, 981, 26, 0, 0, 0, 0, 0, 0, 0, 1025, 1068, 1005, 1031, 1038, 1042, 1026, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1056, 1021, 1072, 1002, 985, 1027, 1077, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8084, 8268, 8164, 8139, 8163, 8250, 8209] avg=8192.0000 max=8268 max_rank=2 min=8084 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1011, 1037, 971, 989, 1075, 1036, 1016, 1042, 5, 5, 0, 0, 0, 0, 0, 0, 1079, 1037, 995, 1006, 953, 1031, 1083, 1017, 7, 0, 0, 0, 0, 0, 0, 0, 981, 1020, 1006, 997, 995, 1048, 1030, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1028, 1094, 993, 1043, 1017, 1002, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1024, 1003, 989, 1032, 1039, 1095, 1060, 1, 0, 0, 0, 0, 0, 0, 0, 1008, 1002, 1015, 1029, 1005, 1039, 1022, 986, 25, 0, 0, 0, 0, 0, 0, 0, 1019, 1032, 1023, 1077, 1052, 1046, 984, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1081, 1018, 988, 990, 1030, 1000, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8208, 8124, 8182, 8280, 8131, 8277, 8147] avg=8192.0000 max=8280 max_rank=4 min=8124 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1060, 1019, 1063, 1024, 995, 976, 1018, 1070, 5, 5, 0, 0, 0, 0, 0, 0, 1008, 1021, 1056, 1013, 1030, 983, 1024, 1031, 7, 0, 0, 0, 0, 0, 0, 0, 1040, 1033, 1046, 1033, 1035, 1063, 1047, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 967, 1012, 1016, 1024, 1023, 1002, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1004, 1008, 1053, 1056, 1032, 1027, 1019, 1, 0, 0, 0, 0, 0, 0, 0, 1002, 1060, 1025, 1028, 1020, 1035, 970, 1036, 26, 0, 0, 0, 0, 0, 0, 0, 1052, 1023, 1006, 1015, 1074, 1003, 977, 990, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1026, 1020, 1003, 1055, 1008, 1017, 982, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8173, 8330, 8157, 8216, 8202, 8140, 8083] avg=8192.0000 max=8330 max_rank=2 min=8083 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1039, 1020, 992, 1038, 1014, 995, 1018, 1043, 5, 4, 0, 0, 0, 0, 0, 0, 1023, 1041, 1018, 1034, 1038, 1074, 1046, 996, 7, 0, 0, 0, 0, 0, 0, 0, 1001, 1011, 1050, 1004, 1068, 1031, 1030, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1005, 986, 964, 1035, 1022, 975, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 959, 1057, 1038, 1028, 1034, 1054, 974, 1051, 1, 0, 0, 0, 0, 0, 0, 0, 1004, 1091, 1094, 1014, 1016, 1002, 1023, 1022, 25, 0, 0, 0, 0, 0, 0, 0, 1002, 986, 1008, 994, 1025, 1059, 1063, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1026, 979, 1058, 1036, 1060, 1043, 987, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8277, 8267, 8029, 8196, 8291, 8143, 8165] avg=8192.0000 max=8291 max_rank=5 min=8029 min_rank=3 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[988, 1000, 1028, 1078, 996, 1002, 987, 999, 5, 5, 0, 0, 0, 0, 0, 0, 1027, 1043, 1029, 988, 1037, 1074, 1002, 1081, 7, 0, 0, 0, 0, 0, 0, 0, 1024, 971, 1002, 1015, 1040, 986, 991, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1049, 1072, 1046, 1074, 1017, 1019, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1022, 986, 1020, 1018, 1013, 1005, 1043, 1, 0, 0, 0, 0, 0, 0, 0, 981, 1026, 1056, 998, 1033, 1040, 1057, 1027, 27, 0, 0, 0, 0, 0, 0, 0, 1029, 1026, 1015, 1018, 1026, 1004, 1035, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1077, 991, 1015, 1056, 984, 1030, 1053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8088, 8288, 8044, 8362, 8129, 8245, 8148, 8232] avg=8192.0000 max=8362 max_rank=3 min=8044 min_rank=2 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1035, 1051, 1039, 1026, 1038, 1014, 985, 1069, 5, 5, 0, 0, 0, 0, 0, 0, 1059, 1073, 1030, 1028, 1089, 1007, 964, 1022, 7, 0, 0, 0, 0, 0, 0, 0, 1001, 1047, 978, 1018, 1064, 991, 1031, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1022, 1008, 1050, 976, 993, 1024, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 993, 1013, 1069, 999, 989, 977, 1053, 1, 0, 0, 0, 0, 0, 0, 0, 947, 1064, 996, 1042, 1029, 1022, 973, 1015, 27, 0, 0, 0, 0, 0, 0, 0, 1039, 1016, 1049, 1046, 963, 1037, 1060, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1028, 1022, 1007, 1021, 1060, 1019, 1088, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8279, 8160, 8147, 8095, 8115, 8219, 8254] avg=8192.0000 max=8279 max_rank=1 min=8095 min_rank=4 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[983, 1010, 991, 1064, 991, 1027, 1020, 1026, 5, 4, 0, 0, 0, 0, 0, 0, 1046, 1029, 1002, 1025, 1039, 1038, 977, 1005, 7, 0, 0, 0, 0, 0, 0, 0, 1027, 1006, 1021, 1016, 1018, 1055, 1044, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1092, 963, 965, 1017, 1058, 994, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1028, 1032, 1023, 1022, 1023, 1014, 1005, 1, 0, 0, 0, 0, 0, 0, 0, 1009, 1067, 999, 1048, 987, 1052, 1002, 985, 25, 0, 0, 0, 0, 0, 0, 0, 1029, 1030, 1051, 986, 983, 1029, 1011, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1073, 995, 988, 1034, 1098, 1057, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8168, 8214, 8186, 8202, 8174, 8159, 8312] avg=8192.0000 max=8312 max_rank=7 min=8121 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1034, 985, 1012, 1015, 1035, 1006, 1034, 1070, 11, 9, 0, 0, 0, 0, 0, 0, 1055, 1010, 966, 953, 1005, 1028, 987, 1044, 11, 0, 0, 0, 0, 0, 0, 0, 1006, 992, 1035, 1051, 1057, 1004, 1000, 981, 3, 0, 0, 0, 0, 0, 0, 0, 1018, 1038, 1093, 1072, 1075, 990, 1019, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1060, 1070, 994, 1050, 1065, 999, 995, 3, 3, 0, 0, 0, 0, 0, 0, 1019, 1041, 974, 1007, 976, 1099, 1005, 1063, 31, 0, 0, 0, 0, 0, 0, 0, 1003, 1026, 1049, 1021, 1004, 1000, 1041, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 991, 1040, 1031, 1004, 1028, 1025, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8059, 8129, 8309, 8279, 8215, 8200, 8134] avg=8192.0000 max=8309 max_rank=3 min=8059 min_rank=1 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8448, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8448, 8192, 8192, 8192, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8064, 8192, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8064, 8192, 8448, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1030, 1018, 1062, 971, 970, 999, 1018, 1054, 1051, 1003, 1021, 1028, 1055, 1027, 1023, 1026, 1013, 987, 1034, 1049, 1012, 1017, 1024, 1050, 1025, 1009, 1037, 1082, 1017, 1030, 983, 1064, 1047, 1036, 1048, 1013, 1043, 1032, 1084, 1025, 1077, 994, 992, 1011, 1000, 975, 969, 1000, 1044, 1014, 976, 1036, 1004, 1010, 1007, 1049, 1004, 1049, 1062, 1012, 1018, 1007, 1049, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1015, 1017, 1028, 1078, 999, 1001, 1028, 1050, 1034, 1004, 1057, 1088, 993, 1019, 1023, 998, 990, 1045, 969, 1005, 997, 974, 1028, 1000, 1034, 969, 1008, 989, 1025, 952, 1029, 1038, 1005, 1063, 1072, 1056, 973, 1029, 1025, 1013, 1027, 1058, 1070, 1044, 1018, 971, 1042, 1024, 1026, 1018, 1029, 1019, 1080, 1054, 1008, 1034, 1040, 1026, 1055, 1064, 1033, 1053, 980, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1024, 1025, 1004, 1085, 1054, 1029, 1062, 1025, 1023, 966, 1003, 1010, 1043, 1065, 1010, 995, 1020, 1025, 1036, 1049, 1014, 1029, 1016, 1003, 1013, 1041, 1021, 994, 1025, 1100, 1056, 993, 1009, 1027, 1020, 1049, 1023, 991, 1000, 1035, 1036, 1044, 1019, 1066, 981, 975, 1009, 1033, 997, 1020, 1045, 1040, 1007, 1030, 1061, 1036, 1052, 1022, 1020, 972, 1008, 1041, 966, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[995, 989, 1049, 1039, 1064, 1042, 1017, 1030, 1034, 1010, 975, 1058, 1029, 989, 997, 1058, 1033, 1041, 1045, 1022, 1012, 1062, 1021, 1063, 996, 1051, 999, 992, 1087, 997, 982, 1091, 1034, 1065, 988, 1040, 1063, 1017, 1030, 1035, 1017, 1096, 1002, 1014, 1026, 992, 956, 1070, 1057, 967, 1028, 1006, 985, 1019, 1016, 965, 1015, 1014, 1071, 1061, 1011, 963, 1024, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[979, 1084, 982, 1061, 995, 985, 1004, 1002, 1016, 1055, 1014, 1047, 1027, 960, 1013, 1059, 1012, 1017, 1021, 1015, 995, 1012, 1038, 1047, 1023, 1026, 1017, 994, 1012, 1019, 1025, 1022, 1095, 1025, 985, 993, 999, 1059, 1000, 1002, 1043, 1025, 1042, 1008, 1044, 1001, 1050, 1033, 1025, 1008, 1064, 1094, 1045, 1025, 1044, 1032, 1065, 1062, 1063, 992, 1051, 1013, 1022, 949] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1048, 1017, 994, 1039, 1027, 1059, 979, 1037, 1017, 1059, 1057, 1071, 1061, 1000, 1028, 1049, 1033, 1049, 1009, 1051, 1037, 1053, 1034, 1007, 987, 1014, 991, 997, 1004, 1031, 1013, 1024, 1028, 1018, 973, 1019, 986, 998, 1029, 1054, 1027, 1004, 1023, 1017, 1012, 1057, 1028, 1006, 1031, 1015, 999, 1060, 1055, 1021, 1021, 992, 979, 995, 1040, 1035, 1056, 1064, 975, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1001, 1040, 1051, 1019, 1012, 1022, 1011, 1019, 1078, 1068, 1058, 1010, 1007, 1001, 1037, 1020, 1020, 1053, 979, 1020, 1077, 1068, 997, 1065, 1024, 1013, 1021, 1078, 1025, 1003, 996, 1016, 981, 942, 1003, 1047, 1021, 1030, 1014, 1003, 1025, 1002, 1047, 1040, 1049, 1011, 1057, 1075, 1026, 999, 1023, 1027, 1036, 1023, 992, 1013, 1010, 1030, 1051, 1022, 972, 1000, 1021, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1032, 987, 976, 1023, 1037, 1002, 1007, 1028, 1012, 1009, 1026, 992, 1044, 1012, 1071, 980, 1066, 1014, 1045, 1017, 1048, 1005, 980, 1013, 1031, 1017, 1034, 1047, 979, 1029, 1030, 1042, 1035, 1001, 1047, 1021, 1058, 1005, 1021, 1032, 1026, 1053, 1024, 1025, 1031, 1018, 1016, 1043, 1019, 1005, 1043, 1023, 1018, 1007, 1021, 1039, 1029, 993, 982, 1063, 1018, 1055, 1076, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1030, 1018, 1062, 971, 970, 999, 1018, 1054, 6, 5, 0, 0, 0, 0, 0, 0, 1051, 1003, 1021, 1012, 1055, 1027, 1023, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 987, 1034, 1049, 1012, 1017, 1024, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1009, 1037, 1082, 1017, 1030, 983, 1064, 17, 10, 0, 0, 0, 0, 0, 0, 1047, 1036, 1048, 1013, 1043, 1032, 1084, 1025, 2, 0, 0, 0, 0, 0, 0, 0, 1077, 994, 992, 1011, 1000, 975, 969, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1014, 976, 1036, 1004, 1010, 1007, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1049, 1045, 1012, 1018, 1007, 1049, 1060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8218, 8181, 8274, 8330, 8016, 8140, 8244] avg=8192.0000 max=8330 max_rank=4 min=8016 min_rank=5 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1015, 1017, 1028, 1078, 999, 1001, 1028, 1050, 6, 5, 0, 0, 0, 0, 0, 0, 1034, 1004, 1057, 1071, 993, 1019, 1023, 998, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1045, 969, 1005, 997, 974, 1028, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 969, 1008, 989, 1025, 952, 1029, 1038, 17, 11, 0, 0, 0, 0, 0, 0, 1005, 1063, 1072, 1056, 973, 1029, 1025, 1013, 2, 0, 0, 0, 0, 0, 0, 0, 1027, 1058, 1070, 1044, 1018, 971, 1042, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1018, 1029, 1019, 1080, 1054, 1008, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1026, 1038, 1064, 1033, 1053, 980, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8199, 8003, 8072, 8238, 8252, 8268, 8277] avg=8192.0000 max=8277 max_rank=7 min=8003 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=33035 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[995, 989, 1049, 1039, 1064, 1042, 1017, 1030, 6, 5, 1, 0, 0, 0, 0, 0, 1034, 1010, 975, 1042, 1029, 989, 997, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1041, 1045, 1022, 1012, 1062, 1021, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1051, 999, 992, 1087, 997, 982, 1091, 17, 10, 0, 0, 0, 0, 0, 0, 1034, 1065, 988, 1040, 1063, 1017, 1030, 1035, 2, 0, 0, 0, 0, 0, 0, 0, 1017, 1096, 1002, 1014, 1026, 992, 956, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 967, 1028, 1006, 985, 1019, 1016, 965, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1014, 1054, 1061, 1011, 963, 1024, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8134, 8294, 8222, 8274, 8170, 8043, 8162] avg=8192.0000 max=8294 max_rank=2 min=8043 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1024, 1025, 1004, 1085, 1054, 1029, 1062, 1025, 6, 5, 0, 0, 0, 0, 0, 0, 1023, 966, 1003, 994, 1043, 1065, 1010, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1025, 1036, 1049, 1014, 1029, 1016, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1041, 1021, 994, 1025, 1100, 1056, 993, 16, 10, 0, 0, 0, 0, 0, 0, 1009, 1027, 1020, 1049, 1023, 991, 1000, 1035, 2, 0, 0, 0, 0, 0, 0, 0, 1036, 1044, 1019, 1066, 981, 975, 1009, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1020, 1045, 1040, 1007, 1030, 1061, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1022, 1004, 972, 1008, 1041, 966, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8319, 8099, 8187, 8269, 8156, 8161, 8236, 8109] avg=8192.0000 max=8319 max_rank=0 min=8099 min_rank=1 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[979, 1084, 982, 1061, 995, 985, 1004, 1002, 6, 5, 0, 0, 0, 0, 0, 0, 1016, 1055, 1014, 1031, 1027, 960, 1013, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1017, 1021, 1015, 995, 1012, 1038, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1026, 1017, 994, 1012, 1019, 1025, 1022, 17, 10, 0, 0, 0, 0, 0, 0, 1095, 1025, 985, 993, 999, 1059, 1000, 1002, 2, 0, 0, 0, 0, 0, 0, 0, 1043, 1025, 1042, 1008, 1044, 1001, 1050, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1008, 1064, 1094, 1045, 1025, 1044, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1062, 1046, 992, 1051, 1013, 1022, 949, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8175, 8152, 8165, 8160, 8244, 8337, 8200] avg=8192.0000 max=8337 max_rank=6 min=8103 min_rank=0 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32941 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1048, 1017, 994, 1039, 1027, 1059, 979, 1037, 6, 5, 0, 0, 0, 0, 0, 0, 1017, 1059, 1057, 1055, 1061, 1000, 1028, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1049, 1009, 1051, 1037, 1053, 1034, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1014, 991, 997, 1004, 1031, 1013, 1024, 17, 10, 0, 0, 0, 0, 0, 0, 1028, 1018, 973, 1019, 986, 998, 1029, 1054, 2, 0, 0, 0, 0, 0, 0, 0, 1027, 1004, 1023, 1017, 1012, 1057, 1028, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1015, 999, 1060, 1055, 1021, 1021, 992, 0, 0, 0, 0, 0, 0, 0, 0, 979, 995, 1023, 1035, 1056, 1064, 975, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8326, 8268, 8088, 8107, 8172, 8194, 8170] avg=8192.0000 max=8326 max_rank=1 min=8088 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1001, 1040, 1051, 1019, 1012, 1022, 1011, 1019, 6, 5, 1, 0, 0, 0, 0, 0, 1078, 1068, 1058, 994, 1007, 1001, 1037, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1053, 979, 1020, 1077, 1068, 997, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1013, 1021, 1078, 1025, 1003, 996, 1016, 17, 10, 0, 0, 0, 0, 0, 0, 981, 942, 1003, 1047, 1021, 1030, 1014, 1003, 2, 0, 0, 0, 0, 0, 0, 0, 1025, 1002, 1047, 1040, 1049, 1011, 1057, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 999, 1023, 1027, 1036, 1023, 992, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1030, 1034, 1022, 972, 1000, 1021, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8263, 8274, 8203, 8043, 8303, 8139, 8124] avg=8192.0000 max=8303 max_rank=5 min=8043 min_rank=4 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1032, 987, 976, 1023, 1037, 1002, 1007, 1028, 10, 11, 6, 0, 0, 0, 0, 0, 1012, 1009, 1026, 968, 1044, 1012, 1071, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1014, 1045, 1017, 1048, 1005, 980, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1017, 1034, 1047, 979, 1029, 1030, 1042, 20, 14, 0, 0, 0, 0, 0, 0, 1035, 1001, 1047, 1021, 1058, 1005, 1021, 1032, 8, 0, 0, 0, 0, 0, 0, 0, 1026, 1053, 1024, 1025, 1031, 1018, 1016, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1005, 1043, 1023, 1018, 1007, 1021, 1039, 4, 0, 0, 0, 0, 0, 0, 0, 1029, 993, 962, 1063, 1018, 1055, 1076, 1054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8122, 8177, 8243, 8228, 8218, 8179, 8250] avg=8192.0000 max=8250 max_rank=7 min=8119 min_rank=0 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8320, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8064, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[955, 1059, 1038, 1019, 1049, 968, 999, 1060, 988, 1034, 1009, 1046, 983, 1015, 1064, 1008, 990, 1012, 1062, 1045, 1059, 987, 1005, 1048, 1016, 1023, 1003, 1025, 1057, 1025, 1029, 1033, 1033, 1013, 1078, 1072, 971, 1033, 1043, 1003, 1038, 998, 1052, 1020, 1029, 1058, 1014, 1022, 958, 1051, 1066, 983, 1061, 983, 1058, 1048, 970, 1025, 961, 1052, 1012, 1030, 1080, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[979, 1003, 1107, 981, 1033, 992, 1030, 1032, 1016, 1008, 1058, 1023, 967, 978, 1022, 1042, 1069, 1006, 1008, 1009, 1001, 1039, 1003, 1040, 1050, 1036, 1095, 1053, 1001, 1058, 1019, 1035, 1002, 1050, 1015, 948, 1058, 1040, 1026, 996, 1047, 1040, 1027, 1026, 1072, 998, 971, 1021, 1045, 977, 997, 996, 1044, 1022, 1062, 1022, 973, 1070, 1027, 1013, 1035, 1054, 1011, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1042, 1054, 992, 988, 1029, 1017, 1033, 996, 1006, 1018, 1016, 988, 1025, 1038, 1035, 1048, 1057, 1010, 1082, 971, 992, 1026, 1059, 1047, 1012, 1035, 1074, 968, 1028, 981, 994, 1053, 995, 1027, 1021, 1007, 1028, 1036, 1019, 1024, 1011, 1033, 1022, 1082, 1019, 1037, 1024, 1004, 1045, 1023, 1072, 1021, 949, 1080, 1033, 988, 1040, 1036, 1026, 1017, 1060, 1049, 990, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1019, 1030, 1065, 1032, 1079, 1027, 1048, 1021, 1098, 1058, 1041, 1051, 1036, 1044, 993, 1031, 1042, 983, 1046, 1073, 1044, 1062, 1011, 1020, 1038, 1041, 995, 984, 1026, 987, 1065, 1007, 1025, 972, 1025, 1020, 1007, 996, 1048, 1048, 1080, 1017, 1008, 1001, 996, 971, 1060, 1038, 993, 1011, 993, 1028, 955, 1021, 992, 996, 1093, 987, 1011, 1011, 1012, 1007, 1019, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[961, 1047, 1041, 1020, 1038, 1000, 1016, 971, 1004, 1023, 1029, 1034, 1000, 1039, 986, 1048, 1011, 1012, 1013, 1022, 1015, 979, 1003, 1090, 1009, 1015, 1041, 1009, 1044, 1012, 1037, 1069, 1072, 996, 1012, 989, 1027, 988, 1092, 1072, 1028, 1021, 1028, 1046, 1069, 976, 1036, 1003, 1045, 1006, 1062, 1027, 1057, 1012, 1000, 1006, 1049, 1051, 966, 1035, 1027, 1032, 1037, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1033, 983, 1017, 1089, 964, 1049, 1032, 1020, 1042, 1059, 1049, 991, 1005, 1029, 1043, 1015, 1050, 1052, 987, 1029, 998, 1026, 1005, 1039, 1010, 1032, 1024, 1046, 1051, 1040, 1044, 1045, 1015, 1044, 995, 1035, 1032, 1013, 1025, 1074, 1008, 1031, 1076, 974, 1036, 1010, 1063, 997, 1041, 1060, 1030, 998, 1036, 980, 964, 1019, 966, 1023, 970, 1038, 1036, 1034, 1007, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1016, 1073, 990, 967, 1065, 1045, 1052, 1031, 996, 1006, 1016, 1034, 1046, 1018, 1002, 1021, 1028, 1035, 997, 1057, 1014, 1037, 1050, 1005, 1063, 965, 993, 1008, 1092, 1060, 1016, 1009, 1025, 1053, 999, 1048, 1036, 1016, 1027, 1046, 1031, 1050, 976, 1071, 1009, 1037, 1038, 1014, 1019, 1052, 971, 994, 1028, 1061, 994, 1016, 1038, 1017, 1007, 984, 1002, 1014, 1033, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[999, 961, 988, 995, 1024, 1021, 996, 1002, 991, 996, 1048, 1025, 1046, 995, 1019, 1018, 1039, 1068, 1038, 1000, 1056, 999, 1033, 1027, 1022, 1010, 1027, 1056, 1010, 973, 1032, 1053, 1053, 1013, 1003, 1024, 1048, 1065, 1064, 1050, 1014, 996, 1028, 1036, 1054, 969, 1068, 1054, 1024, 1057, 1053, 992, 1038, 1039, 1044, 1039, 1016, 966, 1075, 1020, 1007, 992, 1040, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1042, 1054, 992, 988, 1029, 1017, 1033, 996, 28, 3, 0, 0, 0, 0, 0, 0, 1006, 1018, 1016, 988, 1025, 1038, 1035, 1048, 9, 0, 0, 0, 0, 0, 0, 0, 1057, 1010, 1082, 971, 992, 1026, 1059, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1035, 1074, 968, 1000, 981, 994, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1027, 1021, 1007, 1028, 1036, 996, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1033, 1022, 1082, 1002, 1037, 1024, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1023, 1072, 1021, 949, 1080, 1033, 988, 16, 8, 0, 0, 0, 0, 0, 0, 1040, 1036, 1026, 1017, 1060, 1049, 990, 1004, 20, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8183, 8221, 8117, 8134, 8215, 8235, 8249] avg=8192.0000 max=8249 max_rank=7 min=8117 min_rank=3 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[979, 1003, 1107, 981, 1033, 992, 1030, 1032, 27, 3, 0, 0, 0, 0, 0, 0, 1016, 1008, 1058, 1023, 967, 978, 1022, 1042, 9, 0, 0, 0, 0, 0, 0, 0, 1069, 1006, 1008, 1009, 1001, 1039, 1003, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1036, 1095, 1053, 974, 1058, 1019, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1050, 1015, 948, 1058, 1040, 1003, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1040, 1027, 1026, 1054, 998, 971, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 977, 997, 996, 1044, 1022, 1062, 1022, 16, 9, 0, 0, 0, 0, 0, 0, 973, 1070, 1027, 1013, 1035, 1054, 1011, 1058, 20, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8123, 8152, 8320, 8112, 8184, 8190, 8268] avg=8192.0000 max=8320 max_rank=3 min=8112 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[955, 1059, 1038, 1019, 1049, 968, 999, 1060, 28, 3, 0, 0, 0, 0, 0, 0, 988, 1034, 1009, 1046, 983, 1015, 1064, 1008, 9, 0, 0, 0, 0, 0, 0, 0, 990, 1012, 1062, 1045, 1059, 987, 1005, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1023, 1003, 1025, 1029, 1025, 1029, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1013, 1078, 1072, 971, 1033, 1020, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 998, 1052, 1020, 1012, 1058, 1014, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 958, 1051, 1066, 983, 1061, 983, 1058, 1048, 16, 8, 0, 0, 0, 0, 0, 0, 970, 1025, 961, 1052, 1012, 1030, 1080, 1008, 20, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8156, 8185, 8183, 8223, 8214, 8232, 8165] avg=8192.0000 max=8232 max_rank=6 min=8156 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1019, 1030, 1065, 1032, 1079, 1027, 1048, 1021, 28, 3, 0, 0, 0, 0, 0, 0, 1098, 1058, 1041, 1051, 1036, 1044, 993, 1031, 9, 0, 0, 0, 0, 0, 0, 0, 1042, 983, 1046, 1073, 1044, 1062, 1011, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1041, 995, 984, 998, 987, 1065, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 972, 1025, 1020, 1007, 996, 1025, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 1017, 1008, 1001, 979, 971, 1060, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1011, 993, 1028, 955, 1021, 992, 996, 15, 8, 0, 0, 0, 0, 0, 0, 1093, 987, 1011, 1011, 1012, 1007, 1019, 998, 20, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8352, 8361, 8259, 8115, 8118, 8154, 8012, 8165] avg=8192.0000 max=8361 max_rank=1 min=8012 min_rank=6 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[961, 1047, 1041, 1020, 1038, 1000, 1016, 971, 28, 3, 0, 0, 0, 0, 0, 0, 1004, 1023, 1029, 1034, 1000, 1039, 986, 1048, 9, 0, 0, 0, 0, 0, 0, 0, 1011, 1012, 1013, 1022, 1015, 979, 1003, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1015, 1041, 1009, 1016, 1012, 1037, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 996, 1012, 989, 1027, 988, 1068, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1021, 1028, 1046, 1051, 976, 1036, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1006, 1062, 1027, 1057, 1012, 1000, 1006, 16, 9, 0, 0, 0, 0, 0, 0, 1049, 1051, 966, 1035, 1027, 1032, 1037, 1031, 21, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8172, 8122, 8208, 8224, 8189, 8240, 8256] avg=8192.0000 max=8256 max_rank=7 min=8122 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[999, 961, 988, 995, 1024, 1021, 996, 1002, 27, 3, 0, 0, 0, 0, 0, 0, 991, 996, 1048, 1025, 1046, 995, 1019, 1018, 9, 0, 0, 0, 0, 0, 0, 0, 1039, 1068, 1038, 1000, 1056, 999, 1033, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1010, 1027, 1056, 983, 973, 1032, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1013, 1003, 1024, 1048, 1065, 1040, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 996, 1028, 1036, 1036, 969, 1068, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1057, 1053, 992, 1038, 1039, 1044, 1039, 15, 9, 0, 0, 0, 0, 0, 0, 1016, 966, 1075, 1020, 1007, 992, 1040, 1028, 21, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8016, 8147, 8238, 8156, 8296, 8201, 8310, 8172] avg=8192.0000 max=8310 max_rank=6 min=8016 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32979 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1033, 983, 1017, 1089, 964, 1049, 1032, 1020, 28, 3, 0, 0, 0, 0, 0, 0, 1042, 1059, 1049, 991, 1005, 1029, 1043, 1015, 9, 0, 0, 0, 0, 0, 0, 0, 1050, 1052, 987, 1029, 998, 1026, 1005, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1032, 1024, 1046, 1023, 1040, 1044, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1044, 995, 1035, 1032, 1013, 1002, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1031, 1076, 974, 1019, 1010, 1063, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1060, 1030, 998, 1036, 980, 964, 1019, 15, 8, 0, 0, 0, 0, 0, 0, 966, 1023, 970, 1038, 1036, 1034, 1007, 1008, 20, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8242, 8164, 8264, 8210, 8178, 8151, 8109] avg=8192.0000 max=8264 max_rank=3 min=8109 min_rank=7 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1016, 1073, 990, 967, 1065, 1045, 1052, 1031, 33, 6, 0, 0, 0, 0, 0, 0, 996, 1006, 1016, 1034, 1046, 1018, 1002, 1021, 13, 0, 0, 0, 0, 0, 0, 0, 1028, 1035, 997, 1057, 1014, 1037, 1050, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 965, 993, 1008, 1059, 1060, 1016, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1053, 999, 1048, 1036, 1016, 996, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1050, 976, 1071, 984, 1037, 1038, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1052, 971, 994, 1028, 1061, 994, 1016, 19, 12, 0, 0, 0, 0, 0, 0, 1038, 1017, 1007, 984, 1002, 1014, 1033, 1023, 25, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8152, 8195, 8173, 8219, 8201, 8166, 8152] avg=8192.0000 max=8278 max_rank=0 min=8152 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8320, 8192, 8320, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8064, 8192, 8192, 8320, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8064, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1062, 1024, 986, 1026, 1036, 1056, 1001, 1022, 1001, 1013, 1062, 1021, 999, 1064, 1033, 1041, 1043, 1043, 1003, 1023, 1052, 1063, 966, 1006, 950, 1023, 1046, 995, 1018, 1006, 1016, 1074, 1050, 1051, 1035, 1118, 1005, 1011, 1031, 1000, 1056, 1013, 1033, 1035, 992, 1015, 997, 1034, 1037, 1039, 1024, 1018, 1025, 998, 1004, 1026, 1021, 1000, 1021, 1015, 1026, 1007, 1050, 975] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1031, 1007, 1047, 1066, 1004, 1056, 997, 1003, 1017, 989, 1021, 1023, 1047, 1052, 988, 1011, 1061, 991, 1062, 1063, 1048, 1049, 1027, 1055, 1010, 1020, 1020, 977, 1023, 986, 1033, 984, 1062, 1054, 1004, 1089, 998, 1021, 1036, 993, 991, 996, 992, 991, 1016, 1017, 1027, 1017, 1012, 1028, 1026, 1040, 994, 1017, 1029, 1000, 1043, 1068, 1056, 991, 1061, 1036, 1023, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1062, 1013, 1042, 1016, 1012, 1029, 1020, 1061, 996, 1018, 1042, 1010, 1041, 1040, 1010, 993, 1004, 1045, 1045, 1050, 1027, 969, 1097, 1023, 954, 983, 976, 1073, 1068, 1005, 1003, 1001, 1049, 989, 1034, 996, 1023, 1025, 1026, 1042, 1010, 1087, 1042, 1025, 1070, 993, 1039, 1019, 1018, 1021, 981, 1000, 1021, 1000, 1028, 1011, 1025, 1073, 1015, 1069, 1050, 1006, 1031, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1022, 1043, 1034, 1020, 1071, 1076, 1008, 973, 1041, 1035, 1022, 1039, 1015, 1033, 980, 1029, 1032, 1009, 1041, 1033, 1045, 1038, 1005, 1005, 1026, 1009, 1014, 1016, 1002, 1031, 1042, 1058, 964, 1052, 1010, 1056, 1001, 995, 1012, 1054, 1001, 1044, 989, 1047, 1003, 1013, 1030, 1037, 1043, 1016, 1074, 1007, 1008, 1011, 1019, 1088, 1011, 1014, 1012, 1005, 1054, 985, 1031, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[985, 1048, 997, 994, 1017, 1052, 1002, 994, 1004, 1002, 1052, 1031, 1028, 1006, 1029, 1014, 1011, 1005, 1026, 1029, 1018, 1041, 989, 983, 997, 1000, 1015, 1008, 1052, 981, 1011, 1023, 1048, 1042, 1033, 1051, 1035, 1067, 992, 1047, 1012, 990, 1028, 1027, 1038, 1001, 1079, 1041, 1022, 1022, 1079, 1002, 1011, 1011, 1058, 996, 1057, 1030, 1079, 1043, 1012, 1023, 1058, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[995, 1035, 992, 1025, 1047, 979, 1005, 1033, 1030, 1034, 1058, 1032, 1028, 996, 1046, 1007, 1062, 1039, 1079, 1068, 989, 1028, 994, 981, 1010, 1070, 1045, 1044, 1034, 989, 971, 1034, 1092, 1082, 990, 1032, 1011, 1032, 1021, 974, 1016, 1000, 1027, 1087, 1001, 981, 1034, 1042, 1007, 1033, 1005, 979, 986, 1018, 1062, 1044, 1001, 1032, 1022, 1033, 1049, 1049, 1012, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1011, 1033, 1035, 1003, 1010, 1003, 1031, 1027, 1006, 1082, 999, 1041, 1050, 986, 1024, 1054, 972, 1036, 1020, 1059, 1015, 991, 1038, 1017, 1026, 991, 1025, 1046, 1028, 988, 1104, 1059, 1009, 1024, 1026, 1056, 1063, 1031, 970, 1001, 995, 1010, 1027, 1047, 1025, 1016, 1072, 1021, 985, 1036, 1055, 1064, 1000, 1019, 1036, 990, 1018, 994, 1000, 1034, 1051, 1056, 1019, 976] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1044, 982, 1011, 1043, 1028, 1027, 1008, 1053, 1011, 1044, 1027, 1004, 1083, 999, 1047, 1034, 1030, 1029, 1034, 1052, 1071, 1029, 1030, 1039, 1009, 1007, 1045, 1018, 1047, 997, 1037, 1038, 1012, 1023, 1027, 1001, 997, 1040, 1017, 984, 1047, 976, 1014, 1048, 1003, 1018, 1015, 1009, 1012, 1039, 1055, 1014, 1024, 1063, 1014, 1015, 1050, 1032, 1001, 1015, 992, 987, 987, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1062, 1024, 986, 1026, 1036, 1056, 1001, 1022, 7, 0, 0, 0, 0, 0, 0, 0, 1001, 1013, 1062, 1021, 990, 1064, 1033, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1043, 1003, 988, 1052, 1063, 966, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 950, 1023, 1046, 995, 1018, 1006, 1016, 1074, 35, 7, 0, 0, 0, 0, 0, 0, 1050, 1051, 1035, 1095, 1005, 1011, 1031, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1013, 1033, 1035, 992, 1015, 997, 1034, 15, 0, 0, 0, 0, 0, 0, 0, 1037, 1039, 1024, 1018, 1025, 998, 1004, 1026, 11, 2, 1, 0, 0, 0, 0, 0, 1021, 1000, 1021, 1015, 1015, 1007, 1050, 975, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8225, 8164, 8170, 8278, 8190, 8185, 8104] avg=8192.0000 max=8278 max_rank=4 min=8104 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1062, 1013, 1042, 1016, 1012, 1029, 1020, 1061, 7, 0, 0, 0, 0, 0, 0, 0, 996, 1018, 1042, 1010, 1032, 1040, 1010, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1045, 1045, 1014, 1027, 969, 1097, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 954, 983, 976, 1073, 1068, 1005, 1003, 1001, 36, 6, 0, 0, 0, 0, 0, 0, 1049, 989, 1034, 977, 1023, 1025, 1026, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1087, 1042, 1025, 1070, 993, 1039, 1019, 13, 0, 0, 0, 0, 0, 0, 0, 1018, 1021, 981, 1000, 1021, 1000, 1028, 1011, 11, 2, 0, 0, 0, 0, 0, 0, 1025, 1073, 1015, 1069, 1039, 1006, 1031, 990, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8141, 8224, 8105, 8165, 8298, 8093, 8248] avg=8192.0000 max=8298 max_rank=5 min=8093 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1031, 1007, 1047, 1066, 1004, 1056, 997, 1003, 7, 0, 0, 0, 0, 0, 0, 0, 1017, 989, 1021, 1023, 1038, 1052, 988, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 991, 1062, 1027, 1048, 1049, 1027, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1020, 1020, 977, 1023, 986, 1033, 984, 36, 7, 0, 0, 0, 0, 0, 0, 1062, 1054, 1004, 1066, 998, 1021, 1036, 993, 0, 0, 0, 0, 0, 0, 0, 0, 991, 996, 992, 991, 1016, 1017, 1027, 1017, 15, 0, 0, 0, 0, 0, 0, 0, 1012, 1028, 1026, 1040, 994, 1017, 1029, 1000, 11, 2, 1, 0, 0, 0, 0, 0, 1043, 1068, 1056, 991, 1050, 1036, 1023, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8139, 8320, 8096, 8234, 8062, 8160, 8307] avg=8192.0000 max=8320 max_rank=2 min=8062 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[985, 1048, 997, 994, 1017, 1052, 1002, 994, 7, 0, 0, 0, 0, 0, 0, 0, 1004, 1002, 1052, 1031, 1019, 1006, 1029, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1005, 1026, 994, 1018, 1041, 989, 983, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1000, 1015, 1008, 1052, 981, 1011, 1023, 35, 6, 0, 0, 0, 0, 0, 0, 1048, 1042, 1033, 1030, 1035, 1067, 992, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 990, 1028, 1027, 1038, 1001, 1079, 1041, 14, 0, 0, 0, 0, 0, 0, 0, 1022, 1022, 1079, 1002, 1011, 1011, 1058, 996, 11, 2, 1, 0, 0, 0, 0, 0, 1057, 1030, 1079, 1043, 1001, 1023, 1058, 1058, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8157, 8067, 8128, 8294, 8230, 8215, 8349] avg=8192.0000 max=8349 max_rank=7 min=8067 min_rank=2 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=33088 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1044, 982, 1011, 1043, 1028, 1027, 1008, 1053, 7, 0, 0, 0, 0, 0, 0, 0, 1011, 1044, 1027, 1004, 1074, 999, 1047, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1029, 1034, 1016, 1071, 1029, 1030, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1007, 1045, 1018, 1047, 997, 1037, 1038, 36, 6, 0, 0, 0, 0, 0, 0, 1012, 1023, 1027, 981, 997, 1040, 1017, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 976, 1014, 1048, 1003, 1018, 1015, 1009, 14, 0, 0, 0, 0, 0, 0, 0, 1012, 1039, 1055, 1014, 1024, 1063, 1014, 1015, 11, 2, 0, 0, 0, 0, 0, 0, 1050, 1032, 1001, 1015, 981, 987, 987, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8240, 8278, 8240, 8081, 8144, 8249, 8101] avg=8192.0000 max=8278 max_rank=2 min=8081 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1022, 1043, 1034, 1020, 1071, 1076, 1008, 973, 7, 0, 0, 0, 0, 0, 0, 0, 1041, 1035, 1022, 1039, 1006, 1033, 980, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1009, 1041, 998, 1045, 1038, 1005, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1009, 1014, 1016, 1002, 1031, 1042, 1058, 35, 6, 0, 0, 0, 0, 0, 0, 964, 1052, 1010, 1035, 1001, 995, 1012, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1044, 989, 1047, 1003, 1013, 1030, 1037, 14, 0, 0, 0, 0, 0, 0, 0, 1043, 1016, 1074, 1007, 1008, 1011, 1019, 1088, 11, 2, 1, 0, 0, 0, 0, 0, 1011, 1014, 1012, 1005, 1043, 985, 1031, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8185, 8173, 8239, 8123, 8178, 8280, 8104] avg=8192.0000 max=8280 max_rank=6 min=8104 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1011, 1033, 1035, 1003, 1010, 1003, 1031, 1027, 7, 0, 0, 0, 0, 0, 0, 0, 1006, 1082, 999, 1041, 1041, 986, 1024, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1036, 1020, 1023, 1015, 991, 1038, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 991, 1025, 1046, 1028, 988, 1104, 1059, 36, 6, 0, 0, 0, 0, 0, 0, 1009, 1024, 1026, 1035, 1063, 1031, 970, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1010, 1027, 1047, 1025, 1016, 1072, 1021, 14, 0, 0, 0, 0, 0, 0, 0, 985, 1036, 1055, 1064, 1000, 1019, 1036, 990, 11, 2, 1, 0, 0, 0, 0, 0, 1018, 994, 1000, 1034, 1040, 1056, 1019, 976, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8233, 8112, 8309, 8159, 8227, 8199, 8137] avg=8192.0000 max=8309 max_rank=3 min=8112 min_rank=2 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[995, 1035, 992, 1025, 1047, 979, 1005, 1033, 12, 0, 0, 0, 0, 0, 0, 0, 1030, 1034, 1058, 1032, 1013, 996, 1046, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1039, 1079, 1026, 989, 1028, 994, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1070, 1045, 1044, 1034, 989, 971, 1034, 42, 10, 0, 0, 0, 0, 0, 0, 1092, 1082, 990, 1000, 1011, 1032, 1021, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1000, 1027, 1087, 1001, 981, 1034, 1042, 19, 0, 0, 0, 0, 0, 0, 0, 1007, 1033, 1005, 979, 986, 1018, 1062, 1044, 15, 3, 3, 0, 0, 0, 0, 0, 1001, 1032, 1022, 1033, 1034, 1049, 1012, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8216, 8198, 8249, 8202, 8207, 8155, 8186] avg=8192.0000 max=8249 max_rank=3 min=8123 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8064, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8192, 8192, 8320, 8064, 8320, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1044, 1016, 996, 1027, 1069, 1062, 940, 1057, 1048, 1055, 982, 1046, 1065, 1030, 1082, 1042, 1057, 1068, 980, 1066, 960, 996, 1012, 1067, 1076, 985, 1031, 1032, 1037, 997, 1000, 1011, 996, 1005, 989, 1016, 989, 996, 1024, 979, 1013, 1034, 993, 1048, 1009, 1036, 993, 1052, 1030, 996, 1036, 1037, 1068, 976, 1021, 1039, 1024, 1027, 1014, 1034, 985, 1038, 1057, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1022, 1008, 1035, 1003, 1065, 1019, 1001, 1020, 1029, 1035, 1025, 1038, 1064, 986, 996, 1029, 1028, 1050, 1005, 1053, 1008, 1044, 1049, 1021, 1026, 1018, 1019, 1050, 1015, 998, 1018, 1080, 1060, 1033, 1018, 1031, 1006, 1038, 1013, 980, 1016, 1033, 1001, 976, 1045, 1068, 1013, 1025, 1007, 1009, 1012, 967, 1012, 1048, 1021, 1048, 1023, 1011, 1030, 962, 1018, 1067, 1056, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1059, 986, 1003, 1069, 1050, 1023, 999, 1058, 981, 1007, 996, 967, 1031, 992, 1045, 987, 1034, 1025, 1074, 1011, 1035, 1022, 1048, 976, 1011, 1036, 1042, 1028, 995, 1056, 985, 1011, 1000, 1018, 1005, 1020, 968, 1054, 1054, 1026, 1022, 1069, 996, 1050, 1030, 1065, 1032, 1061, 1014, 1048, 1028, 979, 1012, 1047, 1020, 1047, 1017, 1004, 1031, 1036, 1079, 1007, 1027, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1044, 1081, 1007, 1025, 1072, 1028, 1023, 1052, 1016, 1023, 1006, 1028, 1058, 1043, 1029, 1023, 1042, 1064, 1034, 1003, 968, 982, 1017, 1013, 1006, 1054, 1014, 984, 1042, 1003, 980, 1005, 1030, 993, 1009, 1020, 996, 1035, 1020, 1032, 1013, 1060, 1050, 1036, 1031, 1001, 1028, 1014, 1034, 997, 1013, 1018, 1007, 998, 1022, 1008, 1021, 1047, 1055, 1000, 1074, 1003, 1061, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1030, 1065, 1022, 978, 982, 1004, 1038, 1006, 1005, 1044, 988, 993, 1037, 1006, 1044, 1001, 1026, 1001, 1007, 988, 1023, 992, 1051, 1012, 1061, 1051, 1029, 1057, 983, 1033, 985, 1031, 1013, 1072, 981, 1060, 1052, 1017, 1030, 1036, 1029, 985, 1003, 1018, 1050, 1016, 988, 1061, 1042, 1021, 1000, 1057, 1024, 1035, 1006, 1061, 1032, 1035, 1053, 1073, 1077, 1016, 969, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1021, 997, 1072, 1037, 1010, 989, 1016, 1045, 1011, 1055, 1052, 976, 1036, 1050, 1007, 1053, 1016, 1015, 999, 1009, 1017, 1013, 1058, 970, 1012, 1058, 1003, 1057, 1024, 1012, 1023, 1028, 1023, 1002, 1012, 979, 993, 1027, 989, 1034, 1015, 1016, 1009, 965, 1056, 1048, 1039, 1073, 1012, 986, 1057, 1037, 1035, 1037, 991, 1070, 1015, 1028, 1055, 1054, 1015, 1037, 1078, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[984, 1058, 1027, 977, 1051, 1020, 1003, 1001, 1052, 998, 1043, 1006, 1011, 1037, 1014, 1055, 1063, 1020, 1032, 961, 1054, 1053, 1019, 1025, 1040, 1018, 1077, 1045, 1006, 1011, 1013, 1000, 1033, 1064, 1036, 1014, 1020, 1038, 1002, 987, 1005, 1029, 1027, 1019, 1014, 1065, 1038, 1044, 981, 1004, 999, 1034, 1019, 1038, 1043, 1088, 1056, 1061, 1040, 979, 1011, 970, 994, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1063, 1020, 1012, 996, 1008, 1013, 1017, 1033, 998, 988, 1037, 1008, 1055, 1005, 1028, 1011, 1031, 998, 1014, 1030, 1016, 1037, 1035, 994, 1025, 1034, 989, 981, 1058, 1015, 1027, 1068, 1014, 1019, 1036, 1052, 1056, 1030, 1024, 1008, 1057, 1061, 1058, 1047, 1040, 1050, 978, 1043, 1013, 1019, 1022, 1023, 1052, 978, 1004, 1035, 1068, 1019, 1045, 999, 979, 1008, 1006, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1044, 1081, 1007, 1025, 1070, 1028, 1023, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1023, 1006, 1028, 1058, 1043, 1029, 1023, 6, 0, 0, 0, 0, 0, 0, 0, 1042, 1064, 1034, 1003, 968, 982, 1017, 1013, 18, 0, 0, 0, 0, 0, 0, 0, 1006, 1054, 1014, 984, 1042, 1003, 980, 1005, 2, 1, 0, 0, 0, 0, 0, 0, 1030, 993, 1009, 1020, 996, 1035, 1020, 1032, 39, 0, 0, 0, 0, 0, 0, 0, 1013, 1060, 1050, 1036, 1031, 1001, 1028, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 997, 1013, 1018, 1007, 998, 1022, 1008, 11, 0, 0, 0, 0, 0, 0, 0, 1021, 1047, 1019, 1000, 1074, 1003, 1061, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8330, 8232, 8141, 8091, 8174, 8194, 8108, 8266] avg=8192.0000 max=8330 max_rank=0 min=8091 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1044, 1016, 996, 1027, 1067, 1062, 940, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1055, 982, 1046, 1065, 1030, 1082, 1042, 5, 0, 0, 0, 0, 0, 0, 0, 1057, 1068, 980, 1066, 960, 996, 1012, 1067, 17, 0, 0, 0, 0, 0, 0, 0, 1076, 985, 1031, 1032, 1037, 997, 1000, 1011, 2, 1, 0, 0, 0, 0, 0, 0, 996, 1005, 989, 1016, 989, 996, 1024, 979, 40, 0, 0, 0, 0, 0, 0, 0, 1013, 1034, 993, 1048, 1009, 1036, 993, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 996, 1036, 1037, 1068, 976, 1021, 1039, 11, 0, 0, 0, 0, 0, 0, 0, 1024, 1027, 980, 1034, 985, 1038, 1057, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8355, 8223, 8172, 8034, 8138, 8214, 8191] avg=8192.0000 max=8355 max_rank=1 min=8034 min_rank=4 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32577 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1021, 997, 1072, 1037, 1008, 989, 1016, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1055, 1052, 976, 1036, 1050, 1007, 1053, 6, 0, 0, 0, 0, 0, 0, 0, 1016, 1015, 999, 1009, 1017, 1013, 1058, 970, 18, 0, 0, 0, 0, 0, 0, 0, 1012, 1058, 1003, 1057, 1024, 1012, 1023, 1028, 2, 1, 0, 0, 0, 0, 0, 0, 1023, 1002, 1012, 979, 993, 1027, 989, 1034, 41, 0, 0, 0, 0, 0, 0, 0, 1015, 1016, 1009, 965, 1056, 1048, 1039, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 986, 1057, 1037, 1035, 1037, 991, 1070, 11, 0, 0, 0, 0, 0, 0, 0, 1015, 1028, 1019, 1054, 1015, 1037, 1078, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8246, 8115, 8220, 8100, 8180, 8236, 8254] avg=8192.0000 max=8254 max_rank=7 min=8100 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1022, 1008, 1035, 1003, 1063, 1019, 1001, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1035, 1025, 1038, 1064, 986, 996, 1029, 5, 0, 0, 0, 0, 0, 0, 0, 1028, 1050, 1005, 1053, 1008, 1044, 1049, 1021, 17, 0, 0, 0, 0, 0, 0, 0, 1026, 1018, 1019, 1050, 1015, 998, 1018, 1080, 2, 1, 0, 0, 0, 0, 0, 0, 1060, 1033, 1018, 1031, 1006, 1038, 1013, 980, 39, 0, 0, 0, 0, 0, 0, 0, 1016, 1033, 1001, 976, 1045, 1068, 1013, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1009, 1012, 967, 1012, 1048, 1021, 1048, 11, 0, 0, 0, 0, 0, 0, 0, 1023, 1011, 996, 962, 1018, 1067, 1056, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8207, 8275, 8227, 8218, 8138, 8135, 8165] avg=8192.0000 max=8275 max_rank=2 min=8135 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1030, 1065, 1022, 978, 980, 1004, 1038, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1044, 988, 993, 1037, 1006, 1044, 1001, 6, 0, 0, 0, 0, 0, 0, 0, 1026, 1001, 1007, 988, 1023, 992, 1051, 1012, 18, 0, 0, 0, 0, 0, 0, 0, 1061, 1051, 1029, 1057, 983, 1033, 985, 1031, 2, 1, 0, 0, 0, 0, 0, 0, 1013, 1072, 981, 1060, 1052, 1017, 1030, 1036, 40, 0, 0, 0, 0, 0, 0, 0, 1029, 985, 1003, 1018, 1050, 1016, 988, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1021, 1000, 1057, 1024, 1035, 1006, 1061, 11, 0, 0, 0, 0, 0, 0, 0, 1032, 1035, 1017, 1073, 1077, 1016, 969, 1051, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8124, 8118, 8233, 8301, 8110, 8257, 8270] avg=8192.0000 max=8301 max_rank=4 min=8110 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32938 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1059, 986, 1003, 1069, 1048, 1023, 999, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1007, 996, 967, 1031, 992, 1045, 987, 5, 0, 0, 0, 0, 0, 0, 0, 1034, 1025, 1074, 1011, 1035, 1022, 1048, 976, 17, 0, 0, 0, 0, 0, 0, 0, 1011, 1036, 1042, 1028, 995, 1056, 985, 1011, 2, 1, 0, 0, 0, 0, 0, 0, 1000, 1018, 1005, 1020, 968, 1054, 1054, 1026, 40, 0, 0, 0, 0, 0, 0, 0, 1022, 1069, 996, 1050, 1030, 1065, 1032, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1048, 1028, 979, 1012, 1047, 1020, 1047, 11, 0, 0, 0, 0, 0, 0, 0, 1017, 1004, 997, 1036, 1079, 1007, 1027, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8011, 8242, 8167, 8185, 8285, 8206, 8195] avg=8192.0000 max=8285 max_rank=5 min=8011 min_rank=1 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[984, 1058, 1027, 977, 1043, 1020, 1003, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 998, 1043, 1006, 1011, 1037, 1014, 1055, 9, 0, 0, 0, 0, 0, 0, 0, 1063, 1020, 1032, 961, 1054, 1053, 1019, 1025, 22, 0, 0, 0, 0, 0, 0, 0, 1040, 1018, 1077, 1045, 1006, 1011, 1013, 1000, 8, 8, 0, 0, 0, 0, 0, 0, 1033, 1064, 1036, 1014, 1020, 1038, 1002, 987, 44, 7, 0, 0, 0, 0, 0, 0, 1005, 1029, 1027, 1019, 1014, 1065, 1038, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1004, 999, 1034, 1019, 1038, 1043, 1088, 17, 0, 0, 0, 0, 0, 0, 0, 1056, 1061, 977, 979, 1011, 970, 994, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8225, 8249, 8226, 8245, 8197, 8223, 8058] avg=8192.0000 max=8249 max_rank=2 min=8058 min_rank=7 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1063, 1020, 1012, 996, 1006, 1013, 1017, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 998, 988, 1037, 1008, 1055, 1005, 1028, 1011, 6, 0, 0, 0, 0, 0, 0, 0, 1031, 998, 1014, 1030, 1016, 1037, 1035, 994, 18, 0, 0, 0, 0, 0, 0, 0, 1025, 1034, 989, 981, 1058, 1015, 1027, 1068, 2, 1, 0, 0, 0, 0, 0, 0, 1014, 1019, 1036, 1052, 1056, 1030, 1024, 1008, 40, 0, 0, 0, 0, 0, 0, 0, 1057, 1061, 1058, 1047, 1040, 1050, 978, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1019, 1022, 1023, 1052, 978, 1004, 1035, 11, 0, 0, 0, 0, 0, 0, 0, 1068, 1019, 1009, 999, 979, 1008, 1006, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8136, 8173, 8200, 8279, 8294, 8157, 8137] avg=8192.0000 max=8294 max_rank=5 min=8136 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8448, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8448, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8064, 8448, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1034, 1020, 1021, 1093, 986, 995, 1018, 992, 1038, 999, 1008, 1056, 999, 954, 1015, 985, 997, 1065, 1033, 993, 1021, 1008, 1030, 1059, 1024, 1003, 1066, 1028, 1025, 1054, 1043, 1048, 992, 1059, 1036, 1014, 1004, 1048, 1073, 1032, 1040, 1023, 1019, 1000, 1010, 1011, 1048, 1005, 1018, 1015, 987, 1051, 983, 1048, 1024, 972, 1093, 1083, 1095, 1008, 983, 1041, 1030, 981] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[971, 1014, 1004, 1031, 1044, 1057, 1031, 1043, 1046, 1025, 1053, 1014, 1022, 1031, 1013, 1030, 985, 1040, 1013, 992, 1060, 1004, 1005, 1076, 1010, 1076, 1003, 1006, 991, 1060, 1005, 967, 1019, 994, 979, 1042, 1010, 966, 1087, 1092, 1019, 1040, 1023, 1025, 1050, 952, 1113, 1060, 994, 1037, 1037, 1006, 1000, 979, 1019, 1013, 1062, 1049, 1014, 1011, 1031, 1019, 1034, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1036, 978, 1042, 1094, 1030, 1039, 1020, 1054, 1017, 980, 1001, 1016, 957, 1038, 1071, 986, 1083, 1001, 1035, 1013, 955, 1018, 1000, 981, 1035, 971, 1015, 1078, 1058, 1053, 1023, 1059, 1033, 1006, 1035, 1031, 1023, 1029, 1005, 1042, 1030, 1032, 1018, 1055, 1015, 1019, 1059, 1037, 1056, 1058, 1031, 1043, 1001, 989, 1003, 1023, 1042, 1046, 1028, 990, 991, 1042, 980, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1035, 1028, 1001, 988, 1038, 1009, 1031, 1008, 1097, 1065, 1022, 1034, 1006, 1018, 1017, 1018, 993, 978, 1011, 1069, 1054, 1006, 993, 1046, 1064, 996, 1019, 949, 1034, 1069, 990, 990, 1038, 1035, 1050, 1012, 997, 1021, 1002, 1014, 1009, 1036, 1073, 1018, 990, 1115, 979, 1050, 1012, 1006, 985, 996, 1031, 1017, 1051, 1046, 1023, 1052, 1044, 1061, 995, 1029, 1021, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1064, 1043, 1040, 1024, 1003, 1044, 1018, 1094, 1050, 1037, 993, 996, 1056, 1049, 1039, 1030, 1038, 1045, 1019, 1042, 1035, 1039, 1085, 1045, 955, 990, 1023, 1087, 1020, 1032, 1026, 1021, 1030, 960, 1051, 1035, 1014, 1037, 1032, 1009, 978, 1026, 1042, 998, 988, 1038, 996, 1024, 986, 1025, 1000, 1030, 1016, 994, 1012, 995, 1000, 1049, 987, 1035, 1049, 997, 1013, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1043, 999, 1040, 1004, 1011, 1029, 1048, 1004, 1025, 1068, 1035, 991, 1035, 1042, 977, 1054, 1026, 976, 989, 1018, 1028, 1050, 1016, 992, 995, 990, 988, 1028, 1059, 1017, 1039, 1011, 1033, 1026, 1044, 991, 1040, 1007, 1034, 1009, 1029, 1009, 1000, 1048, 1039, 1020, 1037, 1024, 1041, 1043, 1060, 992, 1032, 1043, 1043, 1033, 1043, 1061, 1011, 1007, 1043, 995, 1001, 1071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1041, 994, 1066, 1019, 957, 976, 1020, 1004, 1014, 1054, 967, 1070, 1000, 979, 977, 1066, 1075, 1017, 973, 1024, 990, 1037, 1011, 1019, 1030, 993, 1030, 1021, 999, 977, 1042, 1084, 1033, 993, 1013, 1034, 1024, 1073, 1039, 1028, 1052, 1037, 1062, 1048, 1003, 1061, 1043, 1030, 1036, 1017, 1047, 1031, 1016, 1027, 1023, 1047, 979, 1067, 976, 1066, 985, 1043, 1013, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1018, 1035, 1073, 1022, 1013, 1003, 1060, 1046, 1016, 1048, 1015, 1019, 1030, 975, 1047, 1002, 1003, 1012, 1012, 1062, 1070, 959, 1006, 995, 1071, 1050, 958, 1012, 1060, 993, 1024, 1049, 1043, 1032, 1055, 967, 983, 1039, 1017, 1007, 1046, 1002, 1036, 983, 1081, 1054, 1008, 1042, 977, 1002, 999, 1048, 1055, 962, 1047, 986, 1064, 1054, 1038, 996, 1030, 1021, 1013, 1091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1036, 978, 1030, 1094, 1030, 1039, 1020, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 980, 1001, 1016, 957, 1038, 1071, 986, 9, 8, 0, 0, 0, 0, 0, 0, 1083, 1001, 1035, 1013, 955, 1018, 1000, 981, 26, 0, 0, 0, 0, 0, 0, 0, 1035, 971, 1015, 1078, 1058, 1053, 1023, 1059, 4, 1, 0, 0, 0, 0, 0, 0, 1033, 1006, 1035, 1031, 1023, 1029, 1004, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1032, 1018, 1055, 1015, 1019, 1018, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1058, 1031, 1043, 1001, 989, 1003, 1023, 41, 2, 0, 0, 0, 0, 0, 0, 1042, 1009, 1028, 990, 991, 1042, 980, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8083, 8112, 8297, 8203, 8224, 8247, 8089] avg=8192.0000 max=8297 max_rank=3 min=8083 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[971, 1014, 992, 1031, 1044, 1057, 1031, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1025, 1053, 1014, 1022, 1031, 1013, 1030, 9, 8, 0, 0, 0, 0, 0, 0, 985, 1040, 1013, 992, 1060, 1004, 1005, 1076, 26, 0, 0, 0, 0, 0, 0, 0, 1010, 1076, 1003, 1006, 991, 1060, 1005, 967, 4, 2, 0, 0, 0, 0, 0, 0, 1019, 994, 979, 1042, 1010, 966, 1085, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1040, 1023, 1025, 1050, 952, 1070, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1037, 1037, 1006, 1000, 979, 1019, 1013, 43, 2, 0, 0, 0, 0, 0, 0, 1062, 1012, 1014, 1011, 1031, 1019, 1034, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8251, 8201, 8124, 8187, 8239, 8130, 8221] avg=8192.0000 max=8251 max_rank=1 min=8124 min_rank=3 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1034, 1020, 1009, 1093, 986, 995, 1018, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 999, 1008, 1056, 999, 954, 1015, 985, 10, 8, 0, 0, 0, 0, 0, 0, 997, 1065, 1033, 993, 1021, 1008, 1030, 1059, 27, 0, 0, 0, 0, 0, 0, 0, 1024, 1003, 1066, 1028, 1025, 1054, 1043, 1048, 4, 2, 0, 0, 0, 0, 0, 0, 992, 1059, 1036, 1014, 1004, 1048, 1071, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1023, 1019, 1000, 1010, 1011, 1008, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1015, 987, 1051, 983, 1048, 1024, 972, 40, 2, 0, 0, 0, 0, 0, 0, 1093, 1044, 1095, 1008, 983, 1041, 1030, 981, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8072, 8233, 8297, 8256, 8116, 8140, 8275] avg=8192.0000 max=8297 max_rank=3 min=8072 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1041, 994, 1053, 1019, 957, 976, 1020, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1054, 967, 1070, 1000, 979, 977, 1066, 9, 9, 0, 0, 0, 0, 0, 0, 1075, 1017, 973, 1024, 990, 1037, 1011, 1019, 26, 0, 0, 0, 0, 0, 0, 0, 1030, 993, 1030, 1021, 999, 977, 1042, 1084, 4, 2, 0, 0, 0, 0, 0, 0, 1033, 993, 1013, 1034, 1024, 1073, 1037, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1037, 1062, 1048, 1003, 1061, 1003, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1017, 1047, 1031, 1016, 1027, 1023, 1047, 40, 2, 0, 0, 0, 0, 0, 0, 979, 1030, 976, 1066, 985, 1043, 1013, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8064, 8145, 8172, 8182, 8235, 8296, 8286, 8156] avg=8192.0000 max=8296 max_rank=5 min=8064 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32973 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1064, 1043, 1028, 1024, 1003, 1044, 1018, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1037, 993, 996, 1056, 1049, 1039, 1030, 9, 8, 0, 0, 0, 0, 0, 0, 1038, 1045, 1019, 1042, 1035, 1039, 1085, 1045, 26, 0, 0, 0, 0, 0, 0, 0, 955, 990, 1023, 1087, 1020, 1032, 1026, 1021, 4, 1, 0, 0, 0, 0, 0, 0, 1030, 960, 1051, 1035, 1014, 1037, 1031, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1026, 1042, 998, 988, 1038, 958, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1025, 1000, 1030, 1016, 994, 1012, 995, 38, 2, 0, 0, 0, 0, 0, 0, 1000, 1012, 987, 1035, 1049, 997, 1013, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8318, 8267, 8374, 8159, 8167, 8052, 8098, 8101] avg=8192.0000 max=8374 max_rank=2 min=8052 min_rank=5 imbalance_ratio=1.022217 max_load_reduction=57162 max_load_reduction_pct=0.872223 moved_to_echo=32418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1035, 1028, 989, 988, 1038, 1009, 1031, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 1065, 1022, 1034, 1006, 1018, 1017, 1018, 9, 8, 0, 0, 0, 0, 0, 0, 993, 978, 1011, 1069, 1054, 1006, 993, 1046, 26, 0, 0, 0, 0, 0, 0, 0, 1064, 996, 1019, 949, 1034, 1069, 990, 990, 4, 1, 0, 0, 0, 0, 0, 0, 1038, 1035, 1050, 1012, 997, 1021, 1001, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1036, 1073, 1018, 990, 1115, 942, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1006, 985, 996, 1031, 1017, 1051, 1046, 37, 2, 0, 0, 0, 0, 0, 0, 1023, 1015, 1044, 1061, 995, 1029, 1021, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8294, 8176, 8116, 8168, 8233, 8183, 8240] avg=8192.0000 max=8294 max_rank=1 min=8116 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1043, 999, 1028, 1004, 1011, 1029, 1048, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1068, 1035, 991, 1035, 1042, 977, 1054, 9, 8, 0, 0, 0, 0, 0, 0, 1026, 976, 989, 1018, 1028, 1050, 1016, 992, 26, 0, 0, 0, 0, 0, 0, 0, 995, 990, 988, 1028, 1059, 1017, 1039, 1011, 4, 1, 0, 0, 0, 0, 0, 0, 1033, 1026, 1044, 991, 1040, 1007, 1033, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1009, 1000, 1048, 1039, 1020, 997, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1043, 1060, 992, 1032, 1043, 1043, 1033, 40, 2, 0, 0, 0, 0, 0, 0, 1043, 1024, 1011, 1007, 1043, 995, 1001, 1071, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8244, 8121, 8132, 8183, 8166, 8329, 8195] avg=8192.0000 max=8329 max_rank=6 min=8121 min_rank=2 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1018, 1035, 1054, 1022, 1013, 1003, 1060, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1048, 1015, 1019, 1030, 975, 1047, 1002, 15, 13, 0, 0, 0, 0, 0, 0, 1003, 1012, 1012, 1062, 1070, 959, 1006, 995, 28, 0, 0, 0, 0, 0, 0, 0, 1071, 1050, 958, 1012, 1060, 993, 1024, 1049, 6, 6, 0, 0, 0, 0, 0, 0, 1043, 1032, 1055, 967, 983, 1039, 1011, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1002, 1036, 983, 1081, 1054, 966, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1002, 999, 1048, 1055, 962, 1047, 986, 42, 5, 0, 0, 0, 0, 0, 0, 1064, 1006, 1038, 996, 1030, 1021, 1013, 1091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8180, 8147, 8229, 8137, 8210, 8123, 8259] avg=8192.0000 max=8259 max_rank=7 min=8123 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8064, 8320, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1018, 1029, 996, 1090, 995, 979, 1014, 976, 1028, 1026, 992, 992, 1009, 1062, 1044, 1021, 1050, 1043, 1012, 1016, 994, 1088, 1045, 1011, 1009, 1017, 1015, 1020, 1033, 1062, 998, 1070, 1066, 1010, 1049, 982, 1000, 1030, 986, 982, 990, 1029, 1024, 1044, 1055, 996, 1026, 1019, 1036, 1047, 1006, 1027, 1052, 1045, 1014, 998, 1011, 1051, 1043, 1029, 1033, 1043, 1031, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1059, 977, 1029, 1022, 977, 1085, 1002, 1078, 1036, 975, 1001, 1085, 995, 996, 1080, 1087, 1038, 1024, 1020, 992, 1052, 1046, 1013, 1009, 1012, 1000, 1016, 1063, 982, 980, 987, 1017, 999, 998, 1031, 1050, 1051, 1028, 999, 999, 1079, 1026, 1019, 1012, 979, 1070, 1074, 1006, 979, 1010, 1037, 988, 1014, 1019, 1081, 1046, 1048, 1002, 1013, 1018, 1033, 1055, 1039, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1027, 996, 1037, 1023, 1017, 995, 1065, 1063, 985, 997, 997, 1103, 1041, 1016, 1054, 981, 1042, 955, 1063, 998, 1050, 1024, 1036, 1038, 1084, 1014, 1024, 1010, 1029, 1055, 1029, 1037, 1021, 955, 1027, 1038, 1065, 981, 984, 1033, 1062, 1063, 997, 1028, 1039, 1008, 1009, 1024, 1022, 1010, 1000, 994, 1040, 1040, 1018, 1053, 976, 1045, 1016, 1012, 1030, 1004, 1008, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1021, 1028, 1005, 1032, 1014, 1030, 1065, 1001, 1036, 1028, 1082, 1018, 1047, 1039, 1014, 1042, 1029, 1060, 1013, 1012, 1016, 993, 1035, 1060, 972, 1017, 1020, 1022, 1037, 1051, 995, 997, 1014, 1006, 1001, 993, 1041, 1046, 1010, 1055, 993, 1029, 1032, 1007, 979, 1010, 1002, 1029, 1014, 1070, 1027, 998, 1014, 1055, 1059, 1025, 980, 999, 997, 1072, 1022, 1059, 1012, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1015, 1010, 989, 1006, 968, 1026, 1057, 998, 1010, 1069, 1003, 1027, 1021, 1040, 1004, 1092, 1035, 997, 1040, 969, 1035, 1058, 1077, 1006, 1072, 1052, 1008, 1003, 1042, 1042, 1010, 1035, 1023, 1021, 1033, 984, 982, 1038, 1029, 1032, 981, 1028, 1028, 1033, 978, 997, 1037, 998, 1018, 1012, 1056, 990, 1020, 993, 1027, 1061, 1048, 1054, 1037, 1041, 1026, 1045, 1053, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1056, 1006, 1026, 1008, 1003, 1005, 1028, 1001, 1028, 1065, 972, 993, 1064, 1031, 1069, 1021, 1013, 979, 1045, 1059, 971, 1032, 1023, 1006, 1012, 1067, 1006, 996, 1056, 1044, 981, 1035, 992, 1059, 1043, 1016, 1070, 1041, 998, 1037, 1012, 1016, 980, 1044, 1034, 1072, 1010, 992, 1018, 995, 1052, 1050, 994, 1028, 994, 1005, 1015, 1063, 1011, 1024, 1013, 1104, 1030, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1019, 1007, 1055, 998, 1065, 1013, 1007, 1083, 971, 965, 1041, 1067, 1036, 1007, 1008, 1022, 1000, 1010, 1048, 1042, 998, 996, 1049, 1040, 1047, 1017, 1005, 1024, 989, 1042, 1069, 1004, 1033, 1057, 1015, 1054, 1005, 1018, 1026, 1045, 1036, 1000, 1053, 1018, 984, 995, 1078, 1018, 1003, 1025, 1028, 1047, 949, 1006, 1034, 1012, 1007, 1067, 1031, 1041, 1050, 997, 1049, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1040, 1030, 973, 1011, 1080, 1041, 1019, 1037, 1011, 1006, 985, 1068, 979, 999, 1046, 1029, 1045, 1019, 972, 982, 1009, 1022, 1027, 1037, 1007, 1036, 1032, 1009, 1017, 1001, 1102, 1051, 980, 1036, 1068, 997, 1041, 1026, 952, 1081, 1024, 1040, 1016, 1002, 1007, 981, 1006, 986, 1066, 1024, 1051, 996, 1025, 1096, 1050, 1015, 1046, 1001, 1020, 1026, 1070, 997, 1054, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1021, 1028, 1005, 1032, 1014, 1030, 1065, 1001, 6, 4, 3, 0, 0, 0, 0, 0, 1036, 1028, 1082, 1004, 1047, 1039, 1014, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1060, 1013, 1012, 1016, 993, 1035, 1060, 2, 0, 0, 0, 0, 0, 0, 0, 972, 1017, 1020, 1022, 1037, 1045, 995, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1006, 1001, 993, 1041, 1046, 1010, 1055, 11, 10, 0, 0, 0, 0, 0, 0, 993, 1029, 1032, 1007, 979, 1010, 1002, 1029, 37, 0, 0, 0, 0, 0, 0, 0, 1014, 1070, 1027, 998, 1014, 1050, 1059, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 980, 999, 997, 1072, 1022, 1011, 1012, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8292, 8220, 8105, 8187, 8118, 8257, 8148] avg=8192.0000 max=8292 max_rank=1 min=8105 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1027, 996, 1037, 1023, 1017, 995, 1065, 1063, 6, 4, 3, 0, 0, 0, 0, 0, 985, 997, 997, 1088, 1041, 1016, 1054, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 955, 1063, 998, 1050, 1024, 1036, 1038, 2, 0, 0, 0, 0, 0, 0, 0, 1084, 1014, 1024, 1010, 1029, 1049, 1029, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 955, 1027, 1038, 1065, 981, 984, 1033, 10, 11, 0, 0, 0, 0, 0, 0, 1062, 1063, 997, 1028, 1039, 1008, 1009, 1024, 35, 0, 0, 0, 0, 0, 0, 0, 1022, 1010, 1000, 994, 1040, 1035, 1018, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1045, 1016, 1012, 1030, 959, 1008, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8159, 8208, 8276, 8125, 8265, 8172, 8095] avg=8192.0000 max=8276 max_rank=3 min=8095 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1018, 1029, 996, 1090, 995, 979, 1014, 976, 6, 4, 3, 0, 0, 0, 0, 0, 1028, 1026, 992, 978, 1009, 1062, 1044, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1043, 1012, 1016, 994, 1088, 1045, 1011, 2, 0, 0, 0, 0, 0, 0, 0, 1009, 1017, 1015, 1020, 1033, 1056, 998, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1010, 1049, 982, 1000, 1030, 986, 982, 11, 10, 0, 0, 0, 0, 0, 0, 990, 1029, 1024, 1044, 1055, 996, 1026, 1019, 36, 0, 0, 0, 0, 0, 0, 0, 1036, 1047, 1006, 1027, 1052, 1040, 1014, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1051, 1043, 1029, 1033, 996, 1031, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8160, 8261, 8218, 8126, 8219, 8220, 8222] avg=8192.0000 max=8261 max_rank=2 min=8110 min_rank=0 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1059, 977, 1029, 1022, 977, 1085, 1002, 1078, 6, 4, 2, 0, 0, 0, 0, 0, 1036, 975, 1001, 1070, 995, 996, 1080, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1024, 1020, 992, 1052, 1046, 1013, 1009, 2, 0, 0, 0, 0, 0, 0, 0, 1012, 1000, 1016, 1063, 982, 974, 987, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 999, 998, 1031, 1050, 1051, 1028, 999, 999, 11, 11, 0, 0, 0, 0, 0, 0, 1079, 1026, 1019, 1012, 979, 1070, 1074, 1006, 37, 0, 0, 0, 0, 0, 0, 0, 979, 1010, 1037, 988, 1014, 1015, 1081, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1002, 1013, 1018, 1033, 1007, 1039, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8240, 8196, 8051, 8177, 8302, 8170, 8159] avg=8192.0000 max=8302 max_rank=5 min=8051 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1056, 1006, 1026, 1008, 1003, 1005, 1028, 1001, 6, 4, 2, 0, 0, 0, 0, 0, 1028, 1065, 972, 979, 1064, 1031, 1069, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 979, 1045, 1059, 971, 1032, 1023, 1006, 2, 0, 0, 0, 0, 0, 0, 0, 1012, 1067, 1006, 996, 1056, 1038, 981, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1059, 1043, 1016, 1070, 1041, 998, 1037, 11, 10, 0, 0, 0, 0, 0, 0, 1012, 1016, 980, 1044, 1034, 1072, 1010, 992, 38, 0, 0, 0, 0, 0, 0, 0, 1018, 995, 1052, 1050, 994, 1024, 994, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1063, 1011, 1024, 1013, 1055, 1030, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8229, 8130, 8191, 8277, 8198, 8132, 8234] avg=8192.0000 max=8277 max_rank=4 min=8130 min_rank=2 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1015, 1010, 989, 1006, 968, 1026, 1057, 998, 6, 4, 2, 0, 0, 0, 0, 0, 1010, 1069, 1003, 1013, 1021, 1040, 1004, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 997, 1040, 969, 1035, 1058, 1077, 1006, 2, 0, 0, 0, 0, 0, 0, 0, 1072, 1052, 1008, 1003, 1042, 1036, 1010, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1021, 1033, 984, 982, 1038, 1029, 1032, 11, 10, 0, 0, 0, 0, 0, 0, 981, 1028, 1028, 1033, 978, 997, 1037, 998, 36, 0, 0, 0, 0, 0, 0, 0, 1018, 1012, 1056, 990, 1020, 989, 1027, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1054, 1037, 1041, 1026, 998, 1053, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8081, 8252, 8219, 8258, 8163, 8116, 8173, 8274] avg=8192.0000 max=8274 max_rank=7 min=8081 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1040, 1030, 973, 1011, 1080, 1041, 1019, 1037, 6, 4, 3, 0, 0, 0, 0, 0, 1011, 1006, 985, 1054, 979, 999, 1046, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1019, 972, 982, 1009, 1022, 1027, 1037, 2, 0, 0, 0, 0, 0, 0, 0, 1007, 1036, 1032, 1009, 1017, 995, 1102, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1036, 1068, 997, 1041, 1026, 952, 1081, 10, 10, 0, 0, 0, 0, 0, 0, 1024, 1040, 1016, 1002, 1007, 981, 1006, 986, 35, 0, 0, 0, 0, 0, 0, 0, 1066, 1024, 1051, 996, 1025, 1091, 1050, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1001, 1020, 1026, 1070, 952, 1054, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8109, 8115, 8249, 8201, 8097, 8318, 8203] avg=8192.0000 max=8318 max_rank=6 min=8097 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1019, 1007, 1055, 998, 1065, 1013, 1007, 1083, 9, 8, 6, 0, 0, 0, 0, 0, 971, 965, 1041, 1045, 1036, 1007, 1008, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1010, 1048, 1042, 998, 996, 1049, 1040, 4, 0, 0, 0, 0, 0, 0, 0, 1047, 1017, 1005, 1024, 989, 1033, 1069, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1057, 1015, 1054, 1005, 1018, 1026, 1045, 13, 14, 0, 0, 0, 0, 0, 0, 1036, 1000, 1053, 1018, 984, 995, 1078, 1018, 39, 0, 0, 0, 0, 0, 0, 0, 1003, 1025, 1028, 1047, 949, 996, 1034, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1067, 1031, 1041, 1050, 945, 1049, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8095, 8187, 8188, 8280, 8221, 8094, 8201] avg=8192.0000 max=8280 max_rank=4 min=8094 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 7936, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8064, 8192, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1001, 1016, 1023, 1005, 973, 1007, 1033, 973, 1010, 1036, 1003, 1061, 1017, 1066, 1030, 1065, 1019, 1040, 981, 1057, 1011, 1031, 1035, 1020, 1039, 1035, 1004, 971, 1044, 1100, 983, 1025, 1079, 1019, 1036, 1056, 1037, 1050, 1051, 1010, 975, 1058, 1000, 1020, 1022, 1011, 1025, 1049, 985, 1016, 1036, 1047, 996, 1042, 1006, 979, 1027, 1027, 993, 1046, 1013, 1039, 1052, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1030, 1040, 1039, 1002, 1084, 993, 1030, 1028, 1015, 1041, 1049, 1098, 996, 1010, 1005, 981, 998, 1040, 1006, 1020, 1037, 938, 1027, 1030, 1062, 1059, 1029, 1109, 1009, 1023, 1071, 993, 1027, 996, 1017, 1049, 1009, 1036, 1005, 1013, 1020, 1050, 1018, 977, 1025, 1056, 990, 1017, 1062, 1049, 1084, 1072, 961, 1004, 1041, 1019, 967, 1022, 999, 1018, 1019, 998, 1006, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1030, 1013, 1009, 976, 986, 1006, 997, 1049, 978, 1002, 1075, 1007, 1049, 1019, 1013, 1008, 1013, 977, 995, 1001, 1027, 1025, 1053, 1017, 1026, 1024, 1010, 1047, 1018, 1013, 1041, 1006, 1009, 999, 1023, 1038, 1014, 1045, 1016, 1041, 1022, 1006, 992, 995, 1046, 1035, 1038, 1045, 1054, 1031, 1069, 1027, 1002, 1029, 1033, 1078, 1021, 1041, 1047, 1053, 1069, 1015, 1009, 1084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1053, 1035, 1050, 997, 1036, 1031, 1040, 1017, 1012, 996, 1063, 991, 1069, 1060, 967, 998, 995, 1044, 1041, 1045, 1049, 1008, 1003, 1041, 1031, 1016, 1052, 1049, 1022, 985, 1046, 1021, 1005, 1048, 1038, 1099, 1034, 984, 1014, 1002, 1035, 1045, 1034, 1027, 1055, 1009, 994, 1081, 1057, 1020, 977, 1022, 1027, 963, 1007, 1054, 998, 1009, 1039, 989, 1017, 1012, 981, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1027, 1043, 988, 1048, 1036, 1031, 1054, 1042, 998, 1011, 1041, 993, 1003, 1052, 994, 1001, 1023, 1022, 1000, 1029, 1035, 1015, 1005, 988, 1059, 993, 1029, 1024, 994, 1011, 1011, 1004, 1025, 1038, 1027, 1016, 1025, 1004, 1034, 1042, 1101, 973, 1007, 988, 961, 1050, 1061, 996, 1002, 1050, 1029, 1048, 1030, 1049, 1081, 1019, 1033, 1050, 1027, 1042, 1106, 1008, 1037, 973] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1038, 977, 1075, 952, 1006, 1004, 995, 976, 1007, 1047, 1049, 1035, 1029, 1085, 1029, 1007, 1036, 1039, 1005, 1018, 1005, 1010, 1027, 1035, 1043, 1030, 997, 1031, 1052, 1040, 1048, 1013, 1025, 1041, 1030, 1019, 977, 976, 1024, 1043, 1030, 1054, 998, 1005, 1014, 1018, 1024, 1049, 975, 1039, 978, 978, 1064, 1027, 1044, 1012, 1027, 1057, 1089, 1008, 1040, 1067, 1022, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1078, 1002, 970, 1020, 1054, 1001, 1035, 1036, 1016, 1090, 1036, 1015, 1048, 1001, 1069, 1021, 1082, 1025, 1038, 1029, 1034, 995, 1004, 1021, 1020, 1031, 1017, 970, 977, 1047, 1069, 1066, 1057, 1027, 1050, 1022, 998, 975, 992, 1051, 1035, 1021, 1022, 1004, 1054, 1035, 1002, 1016, 1001, 1017, 1002, 1008, 1071, 948, 990, 1024, 1071, 992, 1003, 1038, 1037, 1030, 990, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1014, 1017, 1003, 1002, 1050, 1027, 1041, 1034, 1027, 1029, 1002, 1001, 1046, 1045, 1015, 973, 1005, 1024, 1062, 1045, 1018, 1016, 1041, 1021, 1087, 1040, 979, 950, 988, 1045, 1025, 1007, 1039, 1032, 1067, 1038, 992, 980, 1002, 1016, 1003, 1035, 1021, 993, 1073, 1044, 1047, 1066, 1023, 1044, 1022, 1027, 1029, 1079, 978, 1044, 1029, 998, 1061, 1015, 1006, 946, 1044, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1038, 977, 1075, 952, 1006, 1004, 995, 976, 15, 12, 4, 0, 0, 0, 0, 0, 1007, 1047, 1049, 1035, 1029, 1077, 1029, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1039, 1005, 1018, 1005, 1010, 1027, 1035, 5, 5, 4, 4, 0, 0, 0, 0, 1028, 1030, 997, 1031, 1052, 1040, 1048, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1041, 1030, 1014, 977, 976, 1024, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1054, 998, 1005, 1014, 1018, 1024, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1034, 978, 978, 1064, 1027, 1044, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1057, 1089, 1008, 1028, 1067, 1022, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8054, 8280, 8193, 8239, 8130, 8188, 8112, 8340] avg=8192.0000 max=8340 max_rank=7 min=8054 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1001, 1016, 1023, 1005, 973, 1007, 1033, 973, 15, 11, 4, 0, 0, 0, 0, 0, 1010, 1036, 1003, 1061, 1017, 1058, 1030, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1040, 981, 1057, 1011, 1031, 1035, 1020, 5, 5, 4, 4, 0, 0, 0, 0, 1024, 1035, 1004, 971, 1044, 1100, 983, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1019, 1036, 1051, 1037, 1050, 1051, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1058, 1000, 1020, 1022, 1011, 1025, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1011, 1036, 1047, 996, 1042, 1006, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1027, 993, 1046, 1002, 1039, 1052, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8061, 8280, 8212, 8186, 8333, 8156, 8102, 8206] avg=8192.0000 max=8333 max_rank=4 min=8061 min_rank=0 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1027, 1043, 988, 1048, 1036, 1031, 1054, 1042, 15, 12, 4, 0, 0, 0, 0, 0, 998, 1011, 1041, 993, 1003, 1044, 994, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1022, 1000, 1029, 1035, 1015, 1005, 988, 5, 5, 4, 4, 0, 0, 0, 0, 1044, 993, 1029, 1024, 994, 1011, 1011, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1038, 1027, 1011, 1025, 1004, 1034, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 973, 1007, 988, 961, 1050, 1061, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1045, 1029, 1048, 1030, 1049, 1081, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1050, 1027, 1042, 1094, 1008, 1037, 973, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8085, 8135, 8110, 8206, 8133, 8303, 8264] avg=8192.0000 max=8303 max_rank=6 min=8085 min_rank=1 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1030, 1013, 1009, 976, 986, 1006, 997, 1049, 15, 12, 4, 0, 0, 0, 0, 0, 978, 1002, 1075, 1007, 1049, 1012, 1013, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 977, 995, 1001, 1027, 1025, 1053, 1017, 5, 5, 4, 3, 0, 0, 0, 0, 1011, 1024, 1010, 1047, 1018, 1013, 1041, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 999, 1023, 1033, 1014, 1045, 1016, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1006, 992, 995, 1046, 1035, 1038, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1026, 1069, 1027, 1002, 1029, 1033, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1041, 1047, 1053, 1057, 1015, 1009, 1084, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8144, 8125, 8170, 8180, 8175, 8318, 8327] avg=8192.0000 max=8327 max_rank=7 min=8097 min_rank=0 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=33000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1030, 1040, 1039, 1002, 1084, 993, 1030, 1028, 15, 11, 4, 0, 0, 0, 0, 0, 1015, 1041, 1049, 1098, 996, 1003, 1005, 981, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1040, 1006, 1020, 1037, 938, 1027, 1030, 5, 5, 4, 3, 0, 0, 0, 0, 1047, 1059, 1029, 1109, 1009, 1023, 1071, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 996, 1017, 1044, 1009, 1036, 1005, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1050, 1018, 977, 1025, 1056, 990, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1044, 1084, 1072, 961, 1004, 1041, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1022, 999, 1018, 1008, 998, 1006, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8188, 8113, 8340, 8147, 8149, 8287, 8036] avg=8192.0000 max=8340 max_rank=3 min=8036 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32619 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1053, 1035, 1050, 997, 1036, 1031, 1040, 1017, 15, 11, 4, 0, 0, 0, 0, 0, 1012, 996, 1063, 991, 1069, 1052, 967, 998, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1044, 1041, 1045, 1049, 1008, 1003, 1041, 6, 5, 4, 4, 0, 0, 0, 0, 1016, 1016, 1052, 1049, 1022, 985, 1046, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1048, 1038, 1093, 1034, 984, 1014, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1045, 1034, 1027, 1055, 1009, 994, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1015, 977, 1022, 1027, 963, 1007, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1009, 1039, 989, 1006, 1012, 981, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8289, 8148, 8245, 8207, 8218, 8276, 8122, 8031] avg=8192.0000 max=8289 max_rank=0 min=8031 min_rank=7 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1078, 1002, 970, 1020, 1054, 1001, 1035, 1036, 15, 12, 4, 0, 0, 0, 0, 0, 1016, 1090, 1036, 1015, 1048, 994, 1069, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 1025, 1038, 1029, 1034, 995, 1004, 1021, 5, 5, 4, 3, 0, 0, 0, 0, 1005, 1031, 1017, 970, 977, 1047, 1069, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1027, 1050, 1017, 998, 975, 992, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1021, 1022, 1004, 1054, 1035, 1002, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1012, 1002, 1008, 1071, 948, 990, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 992, 1003, 1038, 1025, 1030, 990, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8289, 8245, 8182, 8167, 8185, 8056, 8185] avg=8192.0000 max=8289 max_rank=1 min=8056 min_rank=6 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32593 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1014, 1017, 1003, 1002, 1050, 1027, 1041, 1034, 19, 16, 9, 0, 0, 0, 0, 0, 1027, 1029, 1002, 1001, 1046, 1029, 1015, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1024, 1062, 1045, 1018, 1016, 1041, 1021, 11, 10, 8, 7, 0, 0, 0, 0, 1068, 1040, 979, 950, 988, 1045, 1025, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1032, 1067, 1027, 992, 980, 1002, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1035, 1021, 993, 1073, 1044, 1047, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1034, 1022, 1027, 1029, 1079, 978, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 998, 1061, 1015, 990, 946, 1044, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8122, 8268, 8102, 8155, 8274, 8236, 8147] avg=8192.0000 max=8274 max_rank=5 min=8102 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8064, 8320, 8192, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8320, 8320, 8320, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8064, 8320, 8192, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8064, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8064, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True + [2026-08-30 05:02:12] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 666.3 | throughput per GPU (TFLOP/s/GPU): 500.4 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.030058E+01 | loss scale: 1.0 | grad norm: 28.689 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[985, 1001, 968, 1035, 1031, 1026, 1069, 1038, 1014, 1038, 1053, 1034, 1090, 1034, 1033, 1031, 959, 1066, 1086, 1034, 1037, 1031, 1047, 1011, 1050, 1032, 1061, 987, 1001, 1061, 992, 976, 1005, 1002, 1042, 1021, 1015, 1046, 985, 1019, 974, 1028, 1056, 1012, 1032, 1038, 1018, 1028, 1036, 1031, 1043, 960, 992, 996, 1058, 995, 1046, 1049, 998, 942, 1049, 1052, 1023, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1035, 1041, 1035, 991, 996, 1041, 1034, 1049, 1065, 1038, 1038, 1057, 1052, 993, 1050, 1050, 1017, 1024, 1027, 1009, 1007, 1057, 1039, 1037, 1013, 988, 1030, 1007, 988, 1018, 1027, 990, 1016, 1040, 987, 1017, 1075, 1071, 979, 973, 1004, 1026, 1037, 1013, 1053, 990, 1011, 1020, 1057, 1100, 994, 1023, 1030, 994, 1043, 988, 981, 985, 987, 984, 1051, 1087, 1014, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1016, 1013, 1020, 1032, 1023, 1066, 1039, 979, 982, 1005, 1011, 980, 1008, 1000, 1026, 1041, 1050, 1028, 1046, 1030, 988, 1037, 1024, 1019, 1047, 1004, 1051, 1009, 1068, 995, 1045, 1024, 1046, 1027, 1000, 1019, 1030, 1010, 1017, 1045, 1032, 1022, 986, 1012, 1068, 1024, 1037, 1018, 1090, 1077, 1074, 1026, 972, 1014, 1034, 1068, 1044, 987, 999, 1029, 1083, 940, 1007, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1001, 1040, 1049, 984, 1029, 1025, 1041, 1031, 1022, 1050, 1011, 1009, 1067, 1012, 1034, 1016, 1021, 1019, 1002, 1047, 1042, 1010, 1062, 1007, 1022, 1005, 975, 983, 1037, 1024, 1000, 1045, 1057, 977, 953, 990, 1039, 1037, 1046, 1054, 1055, 993, 1018, 1011, 994, 1078, 1100, 1021, 1044, 1078, 978, 1037, 1005, 1069, 1022, 1036, 1018, 1021, 1024, 1024, 1000, 1034, 971, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1034, 1006, 978, 1064, 1040, 1007, 1068, 1005, 1036, 1013, 1005, 1010, 1053, 1028, 1053, 976, 1008, 1021, 1005, 1058, 1040, 976, 1028, 1028, 1020, 1037, 1005, 996, 1055, 966, 1026, 1007, 1033, 1065, 981, 1025, 1009, 1017, 1028, 1064, 1020, 1006, 1079, 1023, 1041, 1047, 1041, 1059, 1014, 1002, 1021, 1001, 1035, 997, 1049, 1087, 998, 998, 1015, 1032, 1031, 1039, 1010, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1074, 1021, 1029, 1010, 1018, 1012, 1001, 1005, 1079, 1020, 986, 999, 1071, 1025, 999, 1046, 1028, 996, 1046, 1046, 999, 1036, 1011, 986, 1008, 1087, 998, 970, 1043, 995, 1056, 991, 1030, 1061, 994, 1058, 1055, 1014, 1018, 1033, 1014, 1012, 985, 1018, 1025, 1030, 1032, 1093, 1033, 1030, 1043, 1009, 1021, 1009, 993, 990, 1028, 1037, 989, 1059, 1048, 1025, 1041, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1054, 1034, 965, 1033, 1044, 1015, 1041, 1006, 989, 1067, 1001, 1034, 967, 1004, 1000, 1042, 1073, 1054, 1054, 1061, 1075, 1034, 1028, 1028, 1043, 961, 1030, 1062, 1014, 1030, 1052, 1038, 1002, 1034, 991, 987, 1049, 1022, 1011, 1028, 1023, 1048, 994, 1005, 1022, 1035, 996, 999, 973, 1001, 1056, 1070, 1018, 1021, 1027, 1022, 1030, 1000, 1017, 1027, 1033, 1064, 989, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1024, 983, 1030, 1061, 1060, 1001, 1044, 994, 1065, 1054, 998, 1020, 1011, 1009, 1018, 1025, 1018, 1036, 1030, 1039, 1025, 1086, 1064, 1017, 1027, 996, 1016, 1050, 999, 1029, 1047, 1007, 1027, 1039, 1010, 999, 1046, 1006, 1039, 1059, 1034, 1083, 1033, 1033, 1007, 975, 984, 1083, 972, 1062, 1013, 1018, 1016, 985, 1033, 1039, 970, 1022, 1026, 972, 996, 1022, 1043, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[985, 1001, 968, 1035, 1031, 1026, 1069, 1038, 1, 0, 0, 0, 0, 0, 0, 0, 1014, 1038, 1053, 1034, 1076, 1034, 1033, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 959, 1066, 1086, 983, 1037, 1031, 1047, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1032, 1061, 987, 1001, 1061, 992, 976, 18, 14, 5, 1, 0, 0, 0, 0, 1005, 1002, 1042, 1021, 1015, 1046, 985, 1019, 7, 0, 0, 0, 0, 0, 0, 0, 974, 1028, 1056, 1012, 1032, 1038, 1018, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1022, 1043, 960, 992, 996, 1058, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1049, 998, 942, 1049, 1052, 1023, 1034, 46, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8313, 8220, 8198, 8142, 8168, 8102, 8239] avg=8192.0000 max=8313 max_rank=1 min=8102 min_rank=6 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1035, 1041, 1035, 991, 996, 1041, 1034, 1049, 1, 0, 0, 0, 0, 0, 0, 0, 1065, 1038, 1038, 1057, 1038, 993, 1050, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1024, 1027, 959, 1007, 1057, 1039, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 988, 1030, 1007, 988, 1018, 1027, 990, 18, 14, 5, 1, 0, 0, 0, 0, 1016, 1040, 987, 1017, 1075, 1071, 979, 973, 8, 0, 0, 0, 0, 0, 0, 0, 1004, 1026, 1037, 1013, 1053, 990, 1011, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1090, 994, 1023, 1030, 994, 1043, 988, 0, 0, 0, 0, 0, 0, 0, 0, 981, 985, 987, 984, 1051, 1087, 1014, 1063, 45, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8329, 8167, 8099, 8166, 8136, 8219, 8197] avg=8192.0000 max=8329 max_rank=1 min=8099 min_rank=3 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1016, 1013, 1020, 1032, 1023, 1066, 1039, 979, 1, 0, 0, 0, 0, 0, 0, 0, 982, 1005, 1011, 980, 995, 1000, 1026, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1028, 1046, 979, 988, 1037, 1024, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1004, 1051, 1009, 1068, 995, 1045, 1024, 18, 13, 5, 1, 0, 0, 0, 0, 1046, 1027, 1000, 1019, 1030, 1010, 1017, 1045, 7, 0, 0, 0, 0, 0, 0, 0, 1032, 1022, 986, 1012, 1068, 1024, 1037, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 1068, 1074, 1026, 972, 1014, 1034, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 987, 999, 1029, 1083, 940, 1007, 993, 46, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8040, 8171, 8280, 8201, 8181, 8346, 8128] avg=8192.0000 max=8346 max_rank=6 min=8040 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1054, 1034, 965, 1033, 1044, 1015, 1041, 1006, 1, 0, 0, 0, 0, 0, 0, 0, 989, 1067, 1001, 1034, 955, 1004, 1000, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1054, 1054, 1009, 1075, 1034, 1028, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 961, 1030, 1062, 1014, 1030, 1052, 1038, 18, 12, 5, 1, 0, 0, 0, 0, 1002, 1034, 991, 987, 1049, 1022, 1011, 1028, 7, 0, 0, 0, 0, 0, 0, 0, 1023, 1048, 994, 1005, 1022, 1035, 996, 981, 0, 0, 0, 0, 0, 0, 0, 0, 973, 992, 1056, 1070, 1018, 1021, 1027, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1000, 1017, 1027, 1033, 1064, 989, 1009, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8092, 8355, 8266, 8131, 8104, 8179, 8216] avg=8192.0000 max=8355 max_rank=2 min=8092 min_rank=1 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32630 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1001, 1040, 1049, 984, 1029, 1025, 1041, 1031, 1, 0, 0, 0, 0, 0, 0, 0, 1022, 1050, 1011, 1009, 1053, 1012, 1034, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1019, 1002, 995, 1042, 1010, 1062, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1005, 975, 983, 1037, 1024, 1000, 1045, 18, 14, 5, 1, 0, 0, 0, 0, 1057, 977, 953, 990, 1039, 1037, 1046, 1054, 7, 0, 0, 0, 0, 0, 0, 0, 1055, 993, 1018, 1011, 994, 1078, 1100, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1069, 978, 1037, 1005, 1069, 1022, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1021, 1024, 1024, 1000, 1034, 971, 1030, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8207, 8158, 8129, 8160, 8252, 8260, 8169] avg=8192.0000 max=8260 max_rank=6 min=8129 min_rank=3 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1074, 1021, 1029, 1010, 1018, 1012, 1001, 1005, 1, 0, 0, 0, 0, 0, 0, 0, 1079, 1020, 986, 999, 1057, 1025, 999, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 996, 1046, 994, 999, 1036, 1011, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1087, 998, 970, 1043, 995, 1056, 991, 19, 14, 5, 1, 0, 0, 0, 0, 1030, 1061, 994, 1058, 1055, 1014, 1018, 1033, 7, 0, 0, 0, 0, 0, 0, 0, 1014, 1012, 985, 1018, 1025, 1030, 1032, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1021, 1043, 1009, 1021, 1009, 993, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1037, 989, 1059, 1048, 1025, 1041, 1018, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8211, 8096, 8187, 8270, 8190, 8119, 8292] avg=8192.0000 max=8292 max_rank=7 min=8096 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1034, 1006, 978, 1064, 1040, 1007, 1068, 1005, 1, 0, 0, 0, 0, 0, 0, 0, 1036, 1013, 1005, 1010, 1039, 1028, 1053, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1021, 1005, 1006, 1040, 976, 1028, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1037, 1005, 996, 1055, 966, 1026, 1007, 19, 14, 5, 1, 0, 0, 0, 0, 1033, 1065, 981, 1025, 1009, 1017, 1028, 1064, 7, 0, 0, 0, 0, 0, 0, 0, 1020, 1006, 1079, 1023, 1041, 1047, 1041, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 993, 1021, 1001, 1035, 997, 1049, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 998, 998, 1015, 1032, 1031, 1039, 1010, 1017, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8160, 8112, 8151, 8229, 8297, 8197, 8187] avg=8192.0000 max=8297 max_rank=5 min=8112 min_rank=2 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32910 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1024, 983, 1030, 1061, 1060, 1001, 1044, 994, 5, 0, 0, 0, 0, 0, 0, 0, 1065, 1054, 998, 1020, 995, 1009, 1018, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1036, 1030, 981, 1025, 1086, 1064, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 996, 1016, 1050, 999, 1029, 1047, 1007, 24, 16, 8, 7, 0, 0, 0, 0, 1027, 1039, 1010, 999, 1046, 1006, 1039, 1059, 12, 0, 0, 0, 0, 0, 0, 0, 1034, 1083, 1033, 1033, 1007, 975, 984, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1038, 1013, 1018, 1016, 985, 1033, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1022, 1026, 972, 996, 1022, 1043, 1007, 50, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8184, 8257, 8226, 8237, 8208, 8114, 8108] avg=8192.0000 max=8257 max_rank=2 min=8108 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8448, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8064, 8192, 8320, 8320, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 7936, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8192, 8320, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8192, 8064, 8320, 8192, 8320, 8192, 256, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1046, 1016, 1084, 1028, 1001, 1039, 1015, 1002, 1020, 1021, 1029, 965, 1044, 1043, 1059, 1053, 1026, 995, 993, 1033, 968, 1001, 1061, 1023, 1088, 1004, 972, 1026, 1024, 1011, 1078, 1016, 1015, 1012, 980, 1006, 1055, 1015, 994, 1082, 994, 1010, 1041, 1027, 1004, 1073, 1010, 1037, 978, 1047, 1015, 1058, 1039, 1026, 1020, 1022, 1038, 1051, 1043, 961, 1021, 1000, 1026, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[972, 1054, 1037, 993, 1027, 1019, 1035, 1021, 994, 1005, 985, 1052, 1046, 1060, 1069, 1059, 1038, 976, 1057, 1066, 1062, 1012, 980, 1061, 979, 1019, 1032, 998, 986, 1038, 1081, 992, 972, 982, 1022, 1047, 1056, 1020, 1086, 1017, 1021, 958, 992, 1051, 1009, 1023, 1018, 1032, 1029, 987, 1057, 1063, 1023, 1037, 1031, 1031, 1006, 983, 1038, 1040, 1027, 1033, 1022, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[990, 1044, 1053, 1065, 1057, 1029, 1025, 1050, 998, 1028, 1035, 975, 959, 1001, 994, 1005, 1052, 1024, 1027, 1044, 1000, 1026, 1017, 1024, 1011, 1067, 1026, 986, 1048, 1015, 1035, 1038, 1022, 990, 1028, 1050, 1001, 1006, 996, 1036, 1078, 988, 1045, 1033, 1020, 1036, 1045, 994, 1041, 1012, 995, 1034, 1060, 998, 1010, 1051, 1004, 1042, 1028, 1063, 1021, 1000, 1007, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1020, 974, 1054, 1013, 1019, 1054, 1078, 999, 1046, 1031, 985, 1044, 1009, 1044, 1039, 1031, 1049, 1079, 1060, 1052, 1047, 996, 1032, 1024, 998, 1051, 1063, 1049, 1043, 1019, 978, 1005, 988, 1025, 930, 1047, 1088, 995, 1007, 1011, 1058, 991, 1012, 1022, 1013, 1000, 984, 974, 1026, 1055, 998, 1024, 973, 1042, 1057, 986, 964, 1056, 984, 1015, 1058, 1043, 1082, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1020, 1016, 1047, 1057, 1024, 1001, 1052, 1090, 974, 982, 1049, 1019, 1028, 1024, 1004, 988, 1005, 1042, 1030, 1074, 1004, 1066, 1049, 1042, 1022, 1033, 1008, 1056, 1029, 1032, 986, 1019, 1041, 1030, 994, 1003, 1008, 1012, 1014, 1021, 1037, 1035, 997, 1036, 987, 1061, 1017, 1022, 1072, 1003, 1042, 1028, 994, 1040, 1015, 996, 1032, 1025, 1026, 1031, 1021, 1003, 1001, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[998, 999, 1036, 1035, 1010, 989, 1069, 1018, 1037, 1026, 1030, 978, 999, 1056, 1028, 992, 1016, 1069, 1015, 1037, 1028, 1033, 1035, 1040, 1052, 1011, 1022, 1009, 1047, 1003, 1027, 1024, 1046, 1033, 1092, 995, 987, 1095, 1033, 986, 999, 969, 1043, 1019, 1029, 1019, 1061, 1020, 1007, 1070, 1005, 1018, 1003, 1006, 979, 1017, 959, 1061, 1040, 1031, 1073, 1081, 1009, 983] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[986, 1013, 998, 988, 985, 1040, 1061, 1039, 1083, 1054, 1009, 1031, 1046, 1023, 996, 1074, 1016, 1017, 985, 1043, 980, 989, 982, 1041, 1048, 1001, 1015, 1018, 1025, 981, 1002, 1041, 1048, 1036, 1032, 1002, 1008, 1052, 968, 1009, 1033, 1015, 1037, 1006, 1020, 1084, 992, 1002, 1056, 1037, 1073, 1066, 1001, 1035, 1065, 1031, 1016, 1058, 1010, 1058, 988, 1042, 1005, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1056, 1030, 1057, 1030, 1002, 1029, 1033, 1023, 1021, 992, 1050, 987, 1020, 1000, 995, 1055, 986, 1057, 1046, 1027, 1026, 1024, 1010, 971, 1019, 1032, 996, 1011, 1020, 1041, 1045, 1043, 1011, 1035, 1039, 1055, 1036, 1049, 1077, 992, 1047, 990, 1039, 1025, 1008, 1011, 1019, 1018, 1044, 1025, 1029, 1029, 1039, 1007, 1062, 1015, 1026, 987, 1037, 1000, 1030, 1018, 976, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1046, 1016, 1084, 1028, 1001, 1039, 990, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1021, 1029, 965, 1044, 1043, 1059, 1053, 20, 0, 0, 0, 0, 0, 0, 0, 1026, 995, 993, 1015, 968, 1001, 1061, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 1004, 972, 1026, 1024, 1011, 1078, 1016, 4, 0, 0, 0, 0, 0, 0, 0, 1015, 1012, 980, 1006, 1055, 1015, 994, 1082, 14, 0, 0, 0, 0, 0, 0, 0, 994, 1010, 1041, 1027, 1004, 1073, 1010, 1037, 25, 4, 0, 0, 0, 0, 0, 0, 978, 1047, 1015, 1034, 1039, 1026, 1020, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1051, 1043, 961, 1021, 1000, 1026, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8254, 8082, 8223, 8173, 8225, 8181, 8192] avg=8192.0000 max=8254 max_rank=1 min=8082 min_rank=2 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[972, 1054, 1037, 993, 1027, 1019, 1010, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1005, 985, 1052, 1046, 1060, 1069, 1059, 20, 0, 0, 0, 0, 0, 0, 0, 1038, 976, 1057, 1047, 1062, 1012, 980, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1019, 1032, 998, 986, 1038, 1081, 992, 5, 0, 0, 0, 0, 0, 0, 0, 972, 982, 1022, 1047, 1056, 1020, 1086, 1017, 14, 0, 0, 0, 0, 0, 0, 0, 1021, 958, 992, 1051, 1009, 1023, 1018, 1032, 25, 4, 0, 0, 0, 0, 0, 0, 1029, 987, 1057, 1039, 1023, 1037, 1031, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 983, 1038, 1040, 1027, 1033, 1022, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8290, 8233, 8130, 8216, 8133, 8234, 8167] avg=8192.0000 max=8290 max_rank=1 min=8130 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1020, 974, 1054, 1013, 1019, 1054, 1052, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1031, 985, 1044, 1009, 1044, 1039, 1031, 19, 0, 0, 0, 0, 0, 0, 0, 1049, 1079, 1060, 1033, 1047, 996, 1032, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1051, 1063, 1049, 1043, 1019, 978, 1005, 5, 0, 0, 0, 0, 0, 0, 0, 988, 1025, 930, 1047, 1088, 995, 1007, 1011, 14, 0, 0, 0, 0, 0, 0, 0, 1058, 991, 1012, 1022, 1013, 1000, 984, 974, 26, 4, 0, 0, 0, 0, 0, 0, 1026, 1055, 998, 1001, 973, 1042, 1057, 986, 0, 0, 0, 0, 0, 0, 0, 0, 964, 1056, 984, 1015, 1058, 1043, 1082, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8248, 8320, 8211, 8105, 8084, 8138, 8245] avg=8192.0000 max=8320 max_rank=2 min=8084 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[990, 1044, 1053, 1065, 1057, 1029, 1000, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1028, 1035, 975, 959, 1001, 994, 1005, 19, 0, 0, 0, 0, 0, 0, 0, 1052, 1024, 1027, 1026, 1000, 1026, 1017, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1067, 1026, 986, 1048, 1015, 1035, 1038, 4, 0, 0, 0, 0, 0, 0, 0, 1022, 990, 1028, 1050, 1001, 1006, 996, 1036, 14, 0, 0, 0, 0, 0, 0, 0, 1078, 988, 1045, 1033, 1020, 1036, 1045, 994, 25, 4, 0, 0, 0, 0, 0, 0, 1041, 1012, 995, 1011, 1060, 998, 1010, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1042, 1028, 1063, 1021, 1000, 1007, 1054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8014, 8196, 8230, 8143, 8268, 8178, 8219] avg=8192.0000 max=8288 max_rank=0 min=8014 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[998, 999, 1036, 1035, 1010, 989, 1043, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1026, 1030, 978, 999, 1056, 1028, 992, 19, 0, 0, 0, 0, 0, 0, 0, 1016, 1069, 1015, 1019, 1028, 1033, 1035, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1011, 1022, 1009, 1047, 1003, 1027, 1024, 4, 0, 0, 0, 0, 0, 0, 0, 1046, 1033, 1092, 995, 987, 1095, 1033, 986, 14, 0, 0, 0, 0, 0, 0, 0, 999, 969, 1043, 1019, 1029, 1019, 1061, 1020, 26, 4, 0, 0, 0, 0, 0, 0, 1007, 1070, 1005, 995, 1003, 1006, 979, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 959, 1061, 1040, 1031, 1073, 1081, 1009, 983, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8165, 8255, 8199, 8281, 8189, 8082, 8237] avg=8192.0000 max=8281 max_rank=4 min=8082 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1020, 1016, 1047, 1057, 1024, 1001, 1022, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 974, 982, 1049, 1019, 1028, 1024, 1004, 988, 22, 0, 0, 0, 0, 0, 0, 0, 1005, 1042, 1030, 1047, 1004, 1066, 1049, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1033, 1008, 1056, 1029, 1032, 986, 1019, 10, 2, 0, 0, 0, 0, 0, 0, 1041, 1030, 994, 1003, 1008, 1012, 1014, 1021, 17, 2, 0, 0, 0, 0, 0, 0, 1037, 1035, 997, 1036, 987, 1061, 1017, 1022, 30, 10, 0, 0, 0, 0, 0, 0, 1072, 1003, 1042, 994, 994, 1040, 1015, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1023, 1026, 1031, 1021, 1003, 1001, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8090, 8285, 8197, 8142, 8232, 8156, 8157] avg=8192.0000 max=8285 max_rank=2 min=8090 min_rank=1 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1056, 1030, 1057, 1030, 1002, 1029, 1008, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 992, 1050, 987, 1020, 1000, 995, 1055, 19, 0, 0, 0, 0, 0, 0, 0, 986, 1057, 1046, 1010, 1026, 1024, 1010, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1032, 996, 1011, 1020, 1041, 1045, 1043, 4, 0, 0, 0, 0, 0, 0, 0, 1011, 1035, 1039, 1055, 1036, 1049, 1077, 992, 13, 0, 0, 0, 0, 0, 0, 0, 1047, 990, 1039, 1025, 1008, 1011, 1019, 1018, 25, 4, 0, 0, 0, 0, 0, 0, 1044, 1025, 1029, 1006, 1039, 1007, 1062, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 987, 1037, 1000, 1030, 1018, 976, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8139, 8130, 8211, 8307, 8186, 8227, 8101] avg=8192.0000 max=8307 max_rank=4 min=8101 min_rank=7 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[986, 1013, 998, 988, 985, 1040, 1035, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 1054, 1009, 1031, 1046, 1023, 996, 1074, 20, 0, 0, 0, 0, 0, 0, 0, 1016, 1017, 985, 1025, 980, 989, 982, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1001, 1015, 1018, 1025, 981, 1002, 1041, 4, 0, 0, 0, 0, 0, 0, 0, 1048, 1036, 1032, 1002, 1008, 1052, 968, 1009, 14, 0, 0, 0, 0, 0, 0, 0, 1033, 1015, 1037, 1006, 1020, 1084, 992, 1002, 26, 4, 0, 0, 0, 0, 0, 0, 1056, 1037, 1073, 1042, 1001, 1035, 1065, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1058, 1010, 1058, 988, 1042, 1005, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8084, 8336, 8035, 8135, 8169, 8219, 8340, 8218] avg=8192.0000 max=8340 max_rank=6 min=8035 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32946 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8064, 8192, 8320, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8448, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8064, 8320, 8320, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8320, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1027, 1040, 998, 996, 1039, 996, 1036, 1011, 1001, 990, 1074, 1061, 1043, 1056, 1090, 1016, 958, 999, 1001, 1075, 1067, 1024, 1015, 1011, 1025, 1040, 993, 1027, 1051, 989, 1006, 1018, 1010, 1017, 1021, 1048, 990, 1036, 1008, 1049, 1040, 1027, 1063, 1006, 985, 1040, 1043, 1059, 999, 995, 1066, 987, 1045, 1021, 1049, 1003, 1015, 1002, 1002, 1020, 1041, 1018, 1026, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1026, 1002, 998, 1006, 1024, 981, 1008, 968, 1023, 1089, 1038, 1044, 1018, 1007, 1030, 1013, 1048, 1021, 1083, 1058, 989, 1019, 1102, 1028, 1023, 972, 1068, 1038, 975, 975, 1069, 1000, 1063, 1054, 1052, 1044, 1026, 1020, 1008, 1060, 990, 1051, 1031, 979, 988, 1039, 1037, 1024, 1019, 1042, 1002, 971, 1055, 1004, 997, 1045, 1026, 964, 1066, 1004, 1005, 1057, 992, 1078] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1024, 990, 1026, 1030, 1009, 1025, 1005, 1049, 1020, 988, 1028, 1025, 1030, 976, 1036, 1038, 1013, 1056, 1060, 996, 992, 1014, 1010, 1032, 1021, 1002, 1025, 1020, 1010, 997, 1023, 1025, 1043, 1040, 1021, 984, 1075, 1006, 1062, 1060, 1024, 978, 1022, 1055, 1031, 1031, 1022, 1076, 1012, 1017, 1015, 981, 1049, 1042, 1079, 1050, 1011, 971, 1038, 1021, 1056, 1039, 989, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[964, 1019, 1010, 1060, 1045, 1029, 1068, 1006, 1007, 1064, 1038, 1029, 1005, 1002, 1063, 991, 1047, 1028, 1018, 1061, 1017, 1010, 1110, 1083, 1056, 1029, 1032, 1032, 1019, 1021, 1009, 992, 974, 987, 1010, 1039, 975, 991, 1046, 1001, 1027, 1024, 970, 1056, 1000, 1027, 1025, 977, 1018, 1016, 1035, 1020, 1008, 1017, 1015, 1064, 1021, 982, 1047, 1021, 1097, 1040, 1019, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1020, 999, 1045, 1093, 1036, 999, 1055, 996, 1050, 988, 1039, 1006, 1089, 1060, 991, 1007, 1001, 1007, 1049, 986, 992, 1023, 1043, 1033, 1021, 1046, 1014, 1059, 995, 1028, 1027, 1004, 971, 1041, 1020, 1025, 1066, 1002, 981, 1049, 1051, 1008, 1044, 1007, 995, 1036, 1002, 1040, 1015, 1051, 1003, 1004, 1026, 1006, 1018, 1094, 1064, 1025, 1016, 1029, 1005, 1025, 975, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1037, 1005, 1030, 981, 1055, 975, 996, 1022, 1052, 996, 1005, 979, 1006, 1038, 1008, 1059, 1002, 1008, 979, 1046, 1070, 1027, 1029, 1047, 1029, 944, 1063, 1074, 1054, 1003, 1002, 1037, 1011, 1062, 1030, 1059, 1000, 1057, 1044, 1025, 985, 991, 1034, 1049, 1065, 1016, 1051, 1020, 1008, 1041, 1054, 1066, 1036, 978, 1017, 1002, 1050, 1042, 1047, 1014, 976, 983, 1038, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1028, 996, 1060, 1049, 1038, 1058, 1027, 1029, 1011, 1004, 1057, 1041, 1028, 1037, 991, 1029, 1021, 1010, 1055, 959, 1063, 995, 1020, 1013, 1078, 1002, 963, 995, 996, 1004, 1054, 993, 1042, 1062, 1060, 1067, 987, 1056, 1025, 1023, 972, 1011, 1091, 1002, 997, 960, 1042, 1017, 1016, 1029, 1076, 1052, 997, 1029, 1073, 1067, 957, 1030, 987, 984, 1033, 997, 1029, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1061, 1064, 1058, 1054, 1049, 1025, 1036, 1018, 1028, 1064, 1041, 1007, 1044, 1044, 1057, 980, 996, 977, 992, 1085, 1038, 1032, 1050, 1027, 999, 996, 1013, 1005, 1017, 1023, 1005, 943, 1057, 1034, 1024, 968, 1034, 1010, 1057, 1014, 1014, 1077, 1076, 1027, 1066, 978, 1030, 1037, 1002, 1048, 1037, 1011, 1041, 977, 998, 1025, 1054, 1029, 1039, 944, 988, 1047, 989, 976] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1027, 1040, 998, 996, 1039, 996, 1036, 1011, 3, 0, 0, 0, 0, 0, 0, 0, 1001, 990, 1045, 1061, 1043, 1056, 1090, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 958, 999, 1001, 1075, 1067, 1024, 993, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1040, 993, 1027, 1051, 989, 1006, 1018, 30, 28, 0, 0, 0, 0, 0, 0, 1010, 987, 1021, 1048, 990, 1036, 1008, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1027, 1063, 1006, 985, 1040, 1043, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 999, 995, 1066, 987, 1045, 1021, 1049, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1002, 1002, 1020, 1041, 1018, 1026, 1032, 22, 8, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8302, 8128, 8207, 8149, 8263, 8154, 8187] avg=8192.0000 max=8302 max_rank=1 min=8128 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1026, 1002, 998, 1006, 1024, 981, 1008, 968, 3, 0, 0, 0, 0, 0, 0, 0, 1023, 1089, 1010, 1044, 1018, 1007, 1030, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1021, 1083, 1058, 989, 1019, 1078, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 972, 1068, 1038, 975, 975, 1069, 1000, 31, 27, 0, 0, 0, 0, 0, 0, 1063, 1023, 1052, 1044, 1026, 1020, 1008, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1051, 1031, 979, 988, 1039, 1037, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1042, 1002, 971, 1055, 1004, 997, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 964, 1066, 1004, 1005, 1057, 992, 1078, 24, 9, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8016, 8234, 8324, 8178, 8296, 8139, 8123, 8226] avg=8192.0000 max=8324 max_rank=2 min=8016 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1024, 990, 1026, 1030, 1009, 1025, 1005, 1049, 3, 0, 0, 0, 0, 0, 0, 0, 1020, 988, 1000, 1025, 1030, 976, 1036, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1056, 1060, 996, 992, 1014, 988, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1002, 1025, 1020, 1010, 997, 1023, 1025, 30, 27, 0, 0, 0, 0, 0, 0, 1043, 1010, 1021, 984, 1075, 1006, 1062, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 978, 1022, 1055, 1031, 1031, 1022, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1017, 1015, 981, 1049, 1042, 1079, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 971, 1038, 1021, 1056, 1039, 989, 1041, 22, 9, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8113, 8151, 8180, 8261, 8239, 8233, 8198] avg=8192.0000 max=8261 max_rank=4 min=8113 min_rank=1 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32931 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[964, 1019, 1010, 1060, 1045, 1029, 1068, 1006, 3, 0, 0, 0, 0, 0, 0, 0, 1007, 1064, 1010, 1029, 1005, 1002, 1063, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1028, 1018, 1061, 1017, 1010, 1086, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1029, 1032, 1032, 1019, 1021, 1009, 992, 29, 27, 0, 0, 0, 0, 0, 0, 974, 958, 1010, 1039, 975, 991, 1046, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1024, 970, 1056, 1000, 1027, 1025, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1016, 1035, 1020, 1008, 1017, 1015, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 982, 1047, 1021, 1097, 1040, 1019, 1023, 24, 9, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8171, 8350, 8246, 7994, 8106, 8181, 8284] avg=8192.0000 max=8350 max_rank=2 min=7994 min_rank=4 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32565 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1037, 1005, 1030, 981, 1055, 975, 996, 1022, 3, 0, 0, 0, 0, 0, 0, 0, 1052, 996, 978, 979, 1006, 1038, 1008, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1008, 979, 1046, 1070, 1027, 1007, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 944, 1063, 1074, 1054, 1003, 1002, 1037, 31, 26, 0, 0, 0, 0, 0, 0, 1011, 1031, 1030, 1059, 1000, 1057, 1044, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 985, 991, 1034, 1049, 1065, 1016, 1051, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1041, 1054, 1066, 1036, 978, 1017, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1042, 1047, 1014, 976, 983, 1038, 1027, 22, 8, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8116, 8186, 8263, 8257, 8211, 8191, 8208] avg=8192.0000 max=8263 max_rank=3 min=8104 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1020, 999, 1045, 1093, 1036, 999, 1055, 996, 3, 0, 0, 0, 0, 0, 0, 0, 1050, 988, 1011, 1006, 1089, 1060, 991, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1007, 1049, 986, 992, 1023, 1021, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1046, 1014, 1059, 995, 1028, 1027, 1004, 30, 27, 0, 0, 0, 0, 0, 0, 971, 1011, 1020, 1025, 1066, 1002, 981, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1008, 1044, 1007, 995, 1036, 1002, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1051, 1003, 1004, 1026, 1006, 1018, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1025, 1016, 1029, 1005, 1025, 975, 1041, 22, 9, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8202, 8112, 8251, 8125, 8183, 8205, 8212] avg=8192.0000 max=8251 max_rank=3 min=8112 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1061, 1064, 1058, 1054, 1049, 1025, 1036, 1018, 3, 0, 0, 0, 0, 0, 0, 0, 1028, 1064, 1013, 1007, 1044, 1044, 1057, 980, 0, 0, 0, 0, 0, 0, 0, 0, 996, 977, 992, 1085, 1038, 1032, 1027, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 999, 996, 1013, 1005, 1017, 1023, 1005, 943, 30, 27, 0, 0, 0, 0, 0, 0, 1057, 1004, 1024, 968, 1034, 1010, 1057, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1077, 1076, 1027, 1066, 978, 1030, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1048, 1037, 1011, 1041, 977, 998, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1029, 1039, 944, 988, 1047, 989, 976, 23, 9, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8368, 8237, 8174, 8058, 8168, 8305, 8127, 8099] avg=8192.0000 max=8368 max_rank=0 min=8058 min_rank=3 imbalance_ratio=1.021484 max_load_reduction=57168 max_load_reduction_pct=0.872314 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1028, 996, 1060, 1049, 1038, 1058, 1027, 1029, 4, 2, 0, 0, 0, 0, 0, 0, 1011, 1004, 1020, 1041, 1028, 1037, 991, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1010, 1055, 959, 1063, 995, 995, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1002, 963, 995, 996, 1004, 1054, 993, 36, 32, 0, 0, 0, 0, 0, 0, 1042, 1026, 1060, 1067, 987, 1056, 1025, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1011, 1089, 1002, 997, 960, 1042, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1029, 1076, 1052, 997, 1029, 1073, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 957, 1030, 987, 984, 1033, 997, 1029, 1062, 25, 13, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8161, 8111, 8153, 8286, 8090, 8322, 8122] avg=8192.0000 max=8322 max_rank=6 min=8090 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8448, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8064, 8320, 8064, 8320, 8320, 8064, 8320, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8320, 8192, 8064, 8320, 8064, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1032, 1073, 1038, 1008, 1029, 1035, 1007, 1025, 1031, 1003, 1011, 993, 1042, 1039, 973, 1006, 1025, 1044, 1003, 1021, 1016, 1025, 1027, 1033, 1042, 1038, 992, 998, 1015, 1002, 1015, 1041, 1036, 969, 1057, 1065, 990, 1016, 1015, 1035, 1003, 1017, 1003, 1045, 1046, 1025, 1034, 1081, 1014, 1068, 1081, 1045, 993, 1015, 1009, 1009, 1001, 1027, 1064, 1064, 1025, 1008, 1007, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1041, 1011, 1013, 1027, 1054, 1021, 1015, 1028, 1018, 1049, 991, 1015, 989, 1013, 1032, 1067, 1031, 1022, 1092, 1008, 988, 1030, 1015, 1033, 1009, 1063, 1000, 1040, 997, 979, 1039, 987, 994, 1030, 1055, 1065, 1022, 1007, 1036, 988, 1045, 1043, 986, 1018, 1037, 1028, 1008, 988, 1039, 1069, 1033, 1035, 1024, 1040, 1029, 1060, 1014, 1013, 1038, 1019, 999, 999, 1038, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[989, 1020, 1020, 1044, 1007, 1004, 1011, 1022, 1006, 1006, 1010, 1047, 1030, 1012, 1036, 1036, 1053, 1053, 1025, 1018, 1038, 975, 1019, 992, 1005, 1027, 982, 1031, 1032, 1039, 1031, 1030, 1025, 1033, 1066, 1015, 991, 974, 1004, 1030, 1000, 1041, 1011, 1044, 1016, 1078, 990, 1062, 1069, 1027, 1057, 1029, 1019, 1070, 1022, 971, 1038, 1025, 1017, 1044, 1007, 1071, 1022, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1002, 971, 1027, 1033, 1016, 1065, 1045, 1011, 1029, 1046, 1057, 1054, 1053, 1002, 1017, 1062, 997, 1059, 1036, 980, 955, 1019, 1017, 1033, 1005, 1034, 998, 1025, 1040, 1004, 1040, 1008, 1014, 1044, 1010, 1005, 1027, 1017, 1038, 999, 947, 1068, 1062, 987, 1002, 1052, 1020, 1005, 1020, 1005, 1025, 1002, 1056, 1004, 1081, 1050, 1012, 1028, 1032, 1029, 1040, 1026, 1059, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1014, 988, 997, 999, 1037, 998, 984, 1021, 1023, 1021, 1003, 1083, 1052, 1073, 1025, 1051, 1081, 977, 1027, 996, 1007, 1059, 1044, 975, 998, 1025, 1001, 1064, 1028, 1019, 977, 1004, 1025, 1030, 1024, 996, 970, 1066, 1043, 1076, 1017, 1049, 1005, 999, 1044, 1028, 1050, 996, 1052, 1002, 1030, 1027, 1052, 1038, 1007, 978, 990, 1038, 1071, 1029, 1056, 1046, 1033, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1014, 1011, 1050, 1040, 1018, 1015, 1020, 997, 1024, 1026, 986, 1024, 1055, 1054, 1089, 1018, 932, 1050, 1026, 1023, 1070, 1058, 997, 1016, 998, 1027, 993, 1010, 1053, 1035, 1029, 1044, 1037, 1013, 1020, 1039, 1057, 1068, 1011, 1035, 1012, 1011, 1017, 1027, 1017, 1003, 994, 1018, 964, 1037, 1020, 1000, 1040, 1020, 985, 1073, 1005, 1010, 1014, 1077, 1023, 1028, 1034, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[990, 981, 1047, 1011, 1064, 1081, 1026, 1028, 1021, 1031, 1010, 1036, 1007, 953, 1079, 1015, 1027, 1045, 1050, 1022, 991, 980, 1052, 1049, 1061, 1032, 1021, 1015, 1023, 1010, 1055, 1052, 1005, 1021, 1074, 1032, 1022, 995, 980, 1006, 1020, 1057, 1051, 1041, 993, 1014, 1004, 1014, 1040, 1054, 1055, 1054, 1026, 1018, 1033, 999, 1007, 971, 992, 982, 1062, 1034, 1012, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1019, 982, 997, 1041, 1035, 1010, 1024, 1009, 1034, 1030, 1036, 1004, 1043, 1032, 1015, 1013, 967, 1020, 1020, 994, 1048, 1065, 1009, 1073, 1005, 1025, 1021, 974, 1059, 1013, 1031, 1009, 1052, 1042, 1085, 1016, 1022, 1032, 1047, 1065, 1001, 1039, 996, 1037, 1014, 1021, 1022, 1020, 1016, 1083, 1032, 1034, 1048, 1019, 1008, 998, 1002, 1026, 1035, 1027, 987, 1031, 1003, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1041, 1011, 1013, 1027, 1054, 1021, 1015, 1028, 31, 0, 0, 0, 0, 0, 0, 0, 1018, 1049, 991, 1015, 966, 1013, 1032, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1022, 1092, 1008, 988, 1030, 1015, 1033, 13, 0, 0, 0, 0, 0, 0, 0, 1009, 1063, 1000, 1040, 997, 979, 1039, 987, 16, 12, 0, 0, 0, 0, 0, 0, 994, 1030, 1038, 1065, 1022, 1007, 1036, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1043, 986, 1018, 1037, 1028, 1008, 988, 11, 1, 0, 0, 0, 0, 0, 0, 1039, 1022, 1033, 1035, 1024, 1040, 1029, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1013, 1038, 1019, 999, 999, 1038, 1020, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8151, 8232, 8142, 8180, 8165, 8282, 8143] avg=8192.0000 max=8282 max_rank=6 min=8142 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1032, 1073, 1038, 1008, 1029, 1035, 1007, 1025, 31, 0, 0, 0, 0, 0, 0, 0, 1031, 1003, 1011, 993, 1017, 1039, 973, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1044, 1003, 1021, 1016, 1025, 1027, 1033, 13, 0, 0, 0, 0, 0, 0, 0, 1042, 1038, 992, 998, 1015, 1002, 1015, 1041, 16, 13, 0, 0, 0, 0, 0, 0, 1036, 969, 1040, 1065, 990, 1016, 1015, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1017, 1003, 1045, 1046, 1025, 1034, 1081, 12, 1, 0, 0, 0, 0, 0, 0, 1014, 1021, 1081, 1045, 993, 1015, 1009, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1027, 1064, 1064, 1025, 1008, 1007, 987, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8073, 8207, 8172, 8166, 8267, 8187, 8186] avg=8192.0000 max=8278 max_rank=0 min=8073 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1002, 971, 1027, 1033, 1016, 1065, 1045, 1011, 29, 0, 0, 0, 0, 0, 0, 0, 1029, 1046, 1057, 1054, 1028, 1002, 1017, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1059, 1036, 980, 955, 1019, 1017, 1033, 12, 0, 0, 0, 0, 0, 0, 0, 1005, 1034, 998, 1025, 1040, 1004, 1040, 1008, 15, 13, 0, 0, 0, 0, 0, 0, 1014, 1044, 994, 1005, 1027, 1017, 1038, 999, 0, 0, 0, 0, 0, 0, 0, 0, 947, 1068, 1062, 987, 1002, 1052, 1020, 1005, 12, 1, 0, 0, 0, 0, 0, 0, 1020, 961, 1025, 1002, 1056, 1004, 1081, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1028, 1032, 1029, 1040, 1026, 1059, 1030, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8295, 8108, 8182, 8138, 8156, 8199, 8259] avg=8192.0000 max=8295 max_rank=1 min=8108 min_rank=2 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[989, 1020, 1020, 1044, 1007, 1004, 1011, 1022, 30, 0, 0, 0, 0, 0, 0, 0, 1006, 1006, 1010, 1047, 1005, 1012, 1036, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1053, 1025, 1018, 1038, 975, 1019, 992, 13, 0, 0, 0, 0, 0, 0, 0, 1005, 1027, 982, 1031, 1032, 1039, 1031, 1030, 16, 13, 0, 0, 0, 0, 0, 0, 1025, 1033, 1049, 1015, 991, 974, 1004, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1041, 1011, 1044, 1016, 1078, 990, 1062, 12, 1, 0, 0, 0, 0, 0, 0, 1069, 981, 1057, 1029, 1019, 1070, 1022, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1025, 1017, 1044, 1007, 1071, 1022, 1018, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8158, 8186, 8206, 8121, 8255, 8218, 8245] avg=8192.0000 max=8255 max_rank=5 min=8121 min_rank=4 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1014, 1011, 1050, 1040, 1018, 1015, 1020, 997, 30, 0, 0, 0, 0, 0, 0, 0, 1024, 1026, 986, 1024, 1030, 1054, 1089, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 932, 1050, 1026, 1023, 1070, 1058, 997, 1016, 12, 0, 0, 0, 0, 0, 0, 0, 998, 1027, 993, 1010, 1053, 1035, 1029, 1044, 16, 13, 0, 0, 0, 0, 0, 0, 1037, 1013, 1004, 1039, 1057, 1068, 1011, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1011, 1017, 1027, 1017, 1003, 994, 1018, 12, 1, 0, 0, 0, 0, 0, 0, 964, 991, 1020, 1000, 1040, 1020, 985, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1010, 1014, 1077, 1023, 1028, 1034, 1025, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8251, 8184, 8218, 8264, 8112, 8093, 8219] avg=8192.0000 max=8264 max_rank=4 min=8093 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1014, 988, 997, 999, 1037, 998, 984, 1021, 29, 0, 0, 0, 0, 0, 0, 0, 1023, 1021, 1003, 1083, 1027, 1073, 1025, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 977, 1027, 996, 1007, 1059, 1044, 975, 12, 0, 0, 0, 0, 0, 0, 0, 998, 1025, 1001, 1064, 1028, 1019, 977, 1004, 15, 13, 0, 0, 0, 0, 0, 0, 1025, 1030, 1008, 996, 970, 1066, 1043, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1049, 1005, 999, 1044, 1028, 1050, 996, 12, 1, 0, 0, 0, 0, 0, 0, 1052, 958, 1030, 1027, 1052, 1038, 1007, 978, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1038, 1071, 1029, 1056, 1046, 1033, 1018, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8067, 8306, 8178, 8144, 8214, 8201, 8142, 8284] avg=8192.0000 max=8306 max_rank=1 min=8067 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[990, 981, 1047, 1011, 1064, 1081, 1026, 1028, 30, 0, 0, 0, 0, 0, 0, 0, 1021, 1031, 1010, 1036, 983, 953, 1079, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1045, 1050, 1022, 991, 980, 1052, 1049, 13, 0, 0, 0, 0, 0, 0, 0, 1061, 1032, 1021, 1015, 1023, 1010, 1055, 1052, 16, 12, 0, 0, 0, 0, 0, 0, 1005, 1021, 1057, 1032, 1022, 995, 980, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1057, 1051, 1041, 993, 1014, 1004, 1014, 12, 1, 0, 0, 0, 0, 0, 0, 1040, 1008, 1055, 1054, 1026, 1018, 1033, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 971, 992, 982, 1062, 1034, 1012, 1003, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8128, 8229, 8297, 8118, 8207, 8233, 8066] avg=8192.0000 max=8297 max_rank=3 min=8066 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1019, 982, 997, 1041, 1035, 1010, 1024, 1009, 34, 0, 0, 0, 0, 0, 0, 0, 1034, 1030, 1036, 1004, 1010, 1032, 1015, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1020, 1020, 994, 1048, 1065, 1009, 1073, 16, 0, 0, 0, 0, 0, 0, 0, 1005, 1025, 1021, 974, 1059, 1013, 1031, 1009, 22, 16, 0, 0, 0, 0, 0, 0, 1052, 1042, 1059, 1016, 1022, 1032, 1047, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1039, 996, 1037, 1014, 1021, 1022, 1020, 17, 6, 0, 0, 0, 0, 0, 0, 1016, 1027, 1032, 1034, 1048, 1019, 1008, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1026, 1035, 1027, 987, 1031, 1003, 1019, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8174, 8212, 8175, 8335, 8173, 8182, 8134] avg=8192.0000 max=8335 max_rank=4 min=8134 min_rank=7 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8320, 8320, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8064, 8192, 8320, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8448, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8448, 8192, 8320, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 8192, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1035, 1065, 1026, 1007, 1034, 990, 1035, 1016, 1016, 1025, 1026, 1037, 1044, 1028, 957, 1006, 1034, 991, 1032, 1004, 1026, 1051, 1023, 1034, 1007, 1033, 992, 988, 1057, 1037, 1036, 1016, 1047, 1051, 1042, 1025, 1069, 1010, 984, 1023, 1023, 1033, 992, 1042, 1057, 1048, 978, 1045, 1000, 1045, 1064, 987, 1025, 1007, 1050, 1038, 1005, 1033, 1004, 1023, 1005, 1034, 1014, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1017, 1006, 1011, 1080, 1052, 1013, 1015, 1051, 975, 1021, 1020, 1027, 1046, 992, 1020, 1068, 1067, 1041, 1069, 1007, 1014, 999, 1065, 1019, 1030, 1014, 1010, 1055, 1021, 981, 1042, 1059, 987, 1003, 1037, 1026, 1043, 997, 1036, 1004, 1018, 1012, 1046, 1040, 974, 1020, 975, 1015, 1003, 1060, 1056, 1010, 983, 1080, 1079, 1048, 1008, 1019, 1032, 1027, 1017, 975, 950, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1046, 1061, 1055, 1012, 1017, 1006, 1036, 1008, 1023, 1039, 1042, 1040, 1045, 1063, 1017, 1003, 981, 1019, 1009, 1041, 1009, 1054, 1008, 992, 1075, 1072, 1039, 1021, 990, 954, 992, 962, 994, 1039, 1005, 1035, 996, 1070, 1009, 1008, 1018, 987, 1091, 1047, 1063, 1014, 1004, 1012, 1032, 1042, 1007, 1033, 1068, 967, 1057, 1067, 1004, 1006, 1030, 1028, 988, 1032, 1019, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[997, 1015, 993, 1057, 1053, 1039, 1008, 1030, 1037, 1019, 1020, 1021, 1036, 1047, 1041, 1012, 1017, 1002, 1022, 1013, 996, 1028, 1000, 1043, 1018, 982, 977, 1068, 1013, 1014, 1025, 1070, 1023, 1012, 1009, 1011, 956, 1047, 964, 1056, 1036, 1029, 995, 981, 1013, 1007, 1049, 1041, 1065, 1030, 1056, 1017, 1058, 1020, 1016, 1016, 1008, 1030, 1067, 1062, 1021, 1064, 1044, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[989, 1012, 1091, 1042, 1050, 1008, 1002, 1040, 1017, 1031, 1057, 1054, 1018, 1030, 1016, 1037, 1009, 1009, 1009, 982, 1015, 1005, 1016, 1032, 1059, 1050, 1026, 1016, 1040, 977, 1017, 1041, 1044, 1040, 1000, 1048, 1064, 1035, 981, 989, 968, 1010, 1014, 1052, 959, 1043, 1017, 1043, 986, 1038, 988, 989, 1050, 1032, 981, 1034, 1032, 1032, 1013, 1054, 1066, 1043, 1062, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1031, 1034, 968, 1063, 1028, 963, 1020, 1081, 1035, 1024, 1031, 1063, 991, 1008, 999, 989, 1047, 1046, 1069, 985, 974, 974, 1066, 1009, 1004, 1068, 989, 1056, 1010, 1049, 1009, 971, 1048, 1038, 1044, 1067, 1045, 1018, 1019, 1014, 1108, 1014, 1035, 1031, 1001, 1018, 965, 1086, 1027, 1017, 1039, 1060, 1012, 1049, 998, 1044, 1004, 1002, 998, 1001, 1016, 1043, 1025, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1047, 1006, 1054, 1001, 1036, 1018, 1086, 1063, 1008, 1013, 983, 1043, 1027, 1052, 1045, 1046, 1026, 1024, 1020, 1029, 1046, 1059, 1065, 1043, 996, 1021, 985, 1059, 1018, 1036, 1031, 1014, 1037, 973, 992, 1021, 1065, 1027, 964, 1053, 1000, 994, 1043, 1040, 1011, 1080, 1024, 1017, 1022, 990, 1021, 984, 987, 1028, 1005, 1031, 979, 979, 1034, 1026, 1032, 1006, 1039, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1056, 1061, 1009, 980, 973, 1059, 1002, 1014, 1035, 1011, 1019, 1006, 1052, 1031, 1006, 1063, 1031, 1033, 1050, 992, 1018, 969, 997, 1055, 1030, 979, 975, 1027, 1023, 1026, 1051, 1009, 1084, 1060, 1027, 1012, 1055, 1018, 1025, 972, 1064, 965, 1011, 1074, 1000, 941, 1004, 1035, 1003, 1087, 1019, 1079, 1145, 1011, 1043, 1050, 1007, 981, 1019, 1021, 1060, 997, 992, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1017, 1006, 1011, 1080, 1052, 1013, 1015, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1021, 1020, 1014, 1046, 992, 1020, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1041, 1069, 1007, 1014, 999, 1065, 1019, 9, 5, 0, 0, 0, 0, 0, 0, 1030, 1014, 1010, 1055, 1021, 981, 1042, 1059, 28, 0, 0, 0, 0, 0, 0, 0, 987, 1003, 1037, 1026, 1043, 997, 1036, 1004, 4, 0, 0, 0, 0, 0, 0, 0, 1018, 1012, 1046, 1040, 974, 1020, 975, 1015, 20, 0, 0, 0, 0, 0, 0, 0, 1003, 1060, 1056, 1010, 983, 1080, 1079, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1019, 1032, 1027, 1017, 975, 950, 1049, 22, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8156, 8295, 8240, 8137, 8120, 8269, 8102] avg=8192.0000 max=8295 max_rank=2 min=8102 min_rank=7 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1035, 1065, 1026, 1007, 1034, 990, 1035, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1025, 1026, 1024, 1044, 1028, 957, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 991, 1032, 1004, 1026, 1051, 1023, 1034, 9, 5, 0, 0, 0, 0, 0, 0, 1007, 1033, 992, 988, 1057, 1037, 1036, 1016, 27, 0, 0, 0, 0, 0, 0, 0, 1047, 1051, 1042, 1025, 1069, 1010, 984, 1023, 4, 0, 0, 0, 0, 0, 0, 0, 1023, 1033, 992, 1042, 1057, 1048, 978, 1045, 20, 0, 0, 0, 0, 0, 0, 0, 1000, 1045, 1064, 987, 1025, 1007, 1050, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1033, 1004, 1023, 1005, 1034, 1014, 1025, 21, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8126, 8209, 8193, 8255, 8238, 8168, 8167] avg=8192.0000 max=8255 max_rank=4 min=8126 min_rank=1 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1047, 1006, 1054, 1001, 1036, 1018, 1086, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1013, 983, 1030, 1027, 1052, 1045, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1024, 1020, 1029, 1046, 1059, 1065, 1043, 9, 5, 0, 0, 0, 0, 0, 0, 996, 1021, 985, 1059, 1018, 1036, 1031, 1014, 27, 0, 0, 0, 0, 0, 0, 0, 1037, 973, 992, 1021, 1065, 1027, 964, 1053, 4, 0, 0, 0, 0, 0, 0, 0, 1000, 994, 1043, 1040, 1011, 1080, 1024, 1017, 20, 0, 0, 0, 0, 0, 0, 0, 1022, 990, 1021, 984, 987, 1028, 1005, 983, 0, 0, 0, 0, 0, 0, 0, 0, 979, 979, 1034, 1026, 1032, 1006, 1039, 1032, 21, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8204, 8326, 8187, 8136, 8229, 8020, 8151] avg=8192.0000 max=8326 max_rank=2 min=8020 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32536 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1046, 1061, 1055, 1012, 1017, 1006, 1036, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1039, 1042, 1027, 1045, 1063, 1017, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1019, 1009, 1041, 1009, 1054, 1008, 992, 9, 5, 0, 0, 0, 0, 0, 0, 1075, 1072, 1039, 1021, 990, 954, 992, 962, 28, 0, 0, 0, 0, 0, 0, 0, 994, 1039, 1005, 1035, 996, 1070, 1009, 1008, 4, 0, 0, 0, 0, 0, 0, 0, 1018, 987, 1091, 1047, 1063, 1014, 1004, 1012, 19, 0, 0, 0, 0, 0, 0, 0, 1032, 1042, 1007, 1033, 1068, 967, 1057, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1006, 1030, 1028, 988, 1032, 1019, 1033, 22, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8259, 8127, 8133, 8160, 8255, 8223, 8165] avg=8192.0000 max=8259 max_rank=1 min=8127 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[989, 1012, 1091, 1042, 1050, 1008, 1002, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1031, 1057, 1041, 1018, 1030, 1016, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1009, 1009, 982, 1015, 1005, 1016, 1032, 9, 5, 0, 0, 0, 0, 0, 0, 1059, 1050, 1026, 1016, 1040, 977, 1017, 1041, 27, 0, 0, 0, 0, 0, 0, 0, 1044, 1040, 1000, 1048, 1064, 1035, 981, 989, 4, 0, 0, 0, 0, 0, 0, 0, 968, 1010, 1014, 1052, 959, 1043, 1017, 1043, 20, 0, 0, 0, 0, 0, 0, 0, 986, 1038, 988, 989, 1050, 1032, 981, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1032, 1013, 1054, 1066, 1043, 1062, 1032, 21, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8247, 8091, 8253, 8205, 8126, 8050, 8358] avg=8192.0000 max=8358 max_rank=7 min=8050 min_rank=6 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1031, 1034, 968, 1063, 1028, 963, 1020, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1024, 1031, 1048, 991, 1008, 999, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1046, 1069, 985, 974, 974, 1066, 1009, 10, 5, 0, 0, 0, 0, 0, 0, 1004, 1068, 989, 1056, 1010, 1049, 1009, 971, 28, 0, 0, 0, 0, 0, 0, 0, 1048, 1038, 1044, 1067, 1045, 1018, 1019, 1014, 5, 0, 0, 0, 0, 0, 0, 0, 1108, 1014, 1035, 1031, 1001, 1018, 965, 1086, 21, 0, 0, 0, 0, 0, 0, 0, 1027, 1017, 1039, 1060, 1012, 1049, 998, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1002, 998, 1001, 1016, 1043, 1025, 996, 21, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8125, 8185, 8184, 8298, 8279, 8197, 8109] avg=8192.0000 max=8298 max_rank=4 min=8109 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[997, 1015, 993, 1057, 1053, 1039, 1008, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1019, 1020, 1008, 1036, 1047, 1041, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1002, 1022, 1013, 996, 1028, 1000, 1043, 9, 5, 0, 0, 0, 0, 0, 0, 1018, 982, 977, 1068, 1013, 1014, 1025, 1070, 27, 0, 0, 0, 0, 0, 0, 0, 1023, 1012, 1009, 1011, 956, 1047, 964, 1056, 4, 0, 0, 0, 0, 0, 0, 0, 1036, 1029, 995, 981, 1013, 1007, 1049, 1041, 20, 0, 0, 0, 0, 0, 0, 0, 1065, 1030, 1056, 1017, 1058, 1020, 1016, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1030, 1067, 1062, 1021, 1064, 1044, 1020, 21, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8220, 8135, 8194, 8082, 8171, 8230, 8340] avg=8192.0000 max=8340 max_rank=7 min=8082 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1056, 1061, 1009, 980, 973, 1059, 1002, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1011, 1019, 982, 1052, 1031, 1006, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1033, 1050, 992, 1018, 969, 997, 1055, 14, 9, 0, 0, 0, 0, 0, 0, 1030, 979, 975, 1027, 1023, 1026, 1051, 1009, 32, 0, 0, 0, 0, 0, 0, 0, 1084, 1060, 1027, 1012, 1055, 1018, 1025, 972, 10, 0, 0, 0, 0, 0, 0, 0, 1064, 965, 1011, 1074, 1000, 941, 1004, 1035, 24, 0, 0, 0, 0, 0, 0, 0, 1003, 1087, 1019, 1079, 1145, 1011, 1043, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 981, 1019, 1021, 1060, 997, 992, 1033, 26, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8199, 8168, 8152, 8263, 8118, 8379, 8144] avg=8192.0000 max=8379 max_rank=6 min=8113 min_rank=0 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8448, 8320, 8320, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8320, 8192, 8192, 8064, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8320, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8064, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8320, 8064, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1011, 1054, 1050, 1062, 949, 1031, 1020, 996, 1100, 1027, 1012, 1029, 1001, 1034, 1076, 1017, 1045, 988, 1037, 982, 1036, 1021, 1000, 1073, 971, 1053, 1040, 1032, 992, 1041, 975, 1037, 1026, 1024, 1041, 1009, 1012, 1048, 1057, 1029, 1004, 1007, 1034, 1007, 973, 994, 978, 1010, 1018, 1044, 990, 1059, 1028, 1119, 1028, 1027, 1003, 1013, 1024, 1043, 1015, 1011, 1050, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[991, 978, 1079, 976, 1034, 1001, 1025, 1020, 1056, 984, 1010, 1007, 1049, 989, 1017, 1005, 1021, 1064, 1018, 988, 1015, 1016, 1013, 1027, 1076, 1034, 1104, 1052, 1061, 1010, 1003, 1057, 1025, 1020, 1003, 1015, 1060, 1004, 987, 1031, 1034, 1042, 1035, 1034, 1010, 1021, 997, 1006, 1055, 1035, 1002, 1000, 993, 1010, 1014, 1060, 1083, 1009, 1036, 1008, 1061, 1012, 969, 1085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1030, 999, 1018, 1021, 1072, 1015, 1076, 1009, 1066, 995, 982, 1032, 1013, 1075, 1006, 1021, 1013, 988, 1053, 1036, 1058, 1035, 1003, 1007, 1047, 981, 1033, 1002, 1025, 1021, 962, 1017, 1043, 1036, 1033, 1050, 1005, 997, 969, 1014, 1005, 1034, 968, 1062, 1086, 1073, 1048, 993, 1051, 1039, 1004, 1049, 1035, 1051, 1001, 977, 984, 1026, 1067, 1043, 1017, 970, 1064, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1057, 1039, 1051, 1056, 1013, 1007, 1027, 960, 1005, 990, 1053, 1070, 1029, 1065, 1033, 1050, 1012, 1019, 991, 989, 1068, 1040, 1054, 1014, 1039, 1052, 1036, 1010, 1016, 1055, 1057, 1059, 1066, 1009, 1020, 1008, 1028, 1029, 1007, 1042, 1022, 994, 1005, 1043, 1049, 987, 1041, 986, 1003, 1016, 1040, 954, 1021, 1039, 1031, 994, 985, 1021, 1045, 1002, 1086, 961, 992, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1050, 988, 1068, 1006, 980, 985, 976, 1032, 992, 1059, 961, 1012, 1038, 1009, 1010, 992, 994, 1030, 1051, 1033, 1063, 1039, 1046, 983, 1042, 1037, 1032, 1001, 995, 991, 1052, 982, 1017, 1021, 1013, 1058, 983, 1032, 1052, 990, 1025, 1051, 997, 1013, 1034, 1029, 1057, 1045, 1043, 1093, 1076, 1054, 1018, 1024, 1048, 1000, 1029, 1061, 1042, 1037, 986, 1014, 1045, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1078, 1009, 1044, 984, 1033, 989, 1027, 1016, 1033, 1031, 1021, 1005, 1092, 1002, 1025, 968, 1084, 997, 1069, 1033, 1007, 979, 997, 1067, 1015, 1060, 977, 1025, 1013, 1058, 985, 1037, 1000, 991, 1017, 1036, 1034, 1050, 1044, 1057, 1015, 1009, 986, 999, 1034, 1021, 1037, 1000, 1012, 999, 1041, 1067, 972, 1020, 1018, 1030, 1037, 1061, 1025, 1073, 1078, 1004, 1029, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[980, 987, 1032, 1063, 1023, 984, 1024, 1075, 1031, 1067, 1022, 1027, 1011, 1058, 1025, 994, 1068, 994, 1034, 1107, 1033, 1029, 1014, 1043, 1029, 1017, 1004, 1017, 995, 1027, 1057, 1035, 1038, 1046, 1031, 997, 991, 990, 1061, 1001, 1038, 997, 1023, 1010, 1011, 1001, 991, 1053, 1031, 1025, 1040, 1010, 1020, 1053, 991, 1008, 1048, 1001, 1002, 1067, 1026, 1005, 1019, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1032, 1014, 1021, 982, 1046, 1042, 992, 1044, 992, 1083, 1022, 1038, 1044, 1024, 1050, 1021, 1060, 1038, 1036, 1018, 991, 1016, 991, 1040, 1054, 1001, 994, 1024, 1008, 1075, 1021, 1004, 1052, 1036, 1065, 1027, 1003, 954, 1022, 1018, 1032, 1010, 1062, 1053, 992, 1056, 1005, 1037, 1024, 1019, 1066, 989, 987, 1046, 1033, 1059, 1008, 998, 993, 1019, 950, 1034, 1029, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1011, 1054, 1050, 1062, 949, 1031, 1020, 996, 14, 11, 0, 0, 0, 0, 0, 0, 1100, 1027, 1012, 1029, 987, 1034, 1076, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 988, 1037, 982, 1036, 1021, 1000, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1053, 1040, 1032, 992, 1032, 975, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1024, 1041, 1009, 1012, 1048, 1057, 1029, 4, 3, 0, 0, 0, 0, 0, 0, 1004, 1007, 1034, 1007, 973, 994, 978, 1010, 21, 7, 0, 0, 0, 0, 0, 0, 1018, 1044, 990, 1059, 1028, 1098, 1028, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1013, 1024, 1043, 1015, 1011, 1050, 1019, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8282, 8161, 8132, 8253, 8035, 8292, 8183] avg=8192.0000 max=8292 max_rank=6 min=8035 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[991, 978, 1079, 976, 1034, 1001, 1025, 1020, 13, 12, 0, 0, 0, 0, 0, 0, 1056, 984, 1010, 1007, 1034, 989, 1017, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1064, 1018, 988, 1015, 1016, 1013, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1034, 1104, 1052, 1061, 1001, 1003, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1020, 1003, 1015, 1060, 1004, 987, 1031, 4, 3, 0, 0, 0, 0, 0, 0, 1034, 1042, 1035, 1034, 1010, 1021, 997, 1006, 21, 7, 0, 0, 0, 0, 0, 0, 1055, 1035, 1002, 1000, 993, 990, 1014, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 1009, 1036, 1008, 1061, 1012, 969, 1085, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8102, 8141, 8388, 8152, 8207, 8149, 8268] avg=8192.0000 max=8388 max_rank=3 min=8102 min_rank=1 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[980, 987, 1032, 1063, 1023, 984, 1024, 1075, 13, 11, 0, 0, 0, 0, 0, 0, 1031, 1067, 1022, 1027, 997, 1058, 1025, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 994, 1034, 1107, 1033, 1029, 1014, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1017, 1004, 1017, 995, 1018, 1057, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1046, 1031, 997, 991, 990, 1061, 1001, 4, 3, 0, 0, 0, 0, 0, 0, 1038, 997, 1023, 1010, 1011, 1001, 991, 1053, 22, 7, 0, 0, 0, 0, 0, 0, 1031, 1025, 1040, 1010, 1020, 1033, 991, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1001, 1002, 1067, 1026, 1005, 1019, 1005, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8221, 8300, 8172, 8162, 8153, 8158, 8178] avg=8192.0000 max=8300 max_rank=2 min=8153 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1050, 988, 1068, 1006, 980, 985, 976, 1032, 13, 11, 0, 0, 0, 0, 0, 0, 992, 1059, 961, 1012, 1024, 1009, 1010, 992, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1030, 1051, 1033, 1063, 1039, 1046, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1037, 1032, 1001, 995, 982, 1052, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1021, 1013, 1058, 983, 1032, 1052, 990, 4, 3, 0, 0, 0, 0, 0, 0, 1025, 1051, 997, 1013, 1034, 1029, 1057, 1045, 20, 7, 0, 0, 0, 0, 0, 0, 1043, 1093, 1076, 1054, 1018, 1004, 1048, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1061, 1042, 1037, 986, 1014, 1045, 1020, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8059, 8219, 8123, 8173, 8278, 8336, 8239] avg=8192.0000 max=8336 max_rank=6 min=8059 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=33026 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1078, 1009, 1044, 984, 1033, 989, 1027, 1016, 13, 12, 0, 0, 0, 0, 0, 0, 1033, 1031, 1021, 1005, 1077, 1002, 1025, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 997, 1069, 1033, 1007, 979, 997, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1060, 977, 1025, 1013, 1049, 985, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 991, 1017, 1036, 1034, 1050, 1044, 1057, 4, 3, 0, 0, 0, 0, 0, 0, 1015, 1009, 986, 999, 1034, 1021, 1037, 1000, 22, 7, 0, 0, 0, 0, 0, 0, 1012, 999, 1041, 1067, 972, 1000, 1018, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1061, 1025, 1073, 1078, 1004, 1029, 980, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8162, 8211, 8161, 8236, 8130, 8139, 8292] avg=8192.0000 max=8292 max_rank=7 min=8130 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1057, 1039, 1051, 1056, 1013, 1007, 1027, 960, 13, 11, 0, 0, 0, 0, 0, 0, 1005, 990, 1053, 1070, 1015, 1065, 1033, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1019, 991, 989, 1068, 1040, 1054, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1052, 1036, 1010, 1016, 1046, 1057, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1009, 1020, 1008, 1028, 1029, 1007, 1042, 4, 3, 0, 0, 0, 0, 0, 0, 1022, 994, 1005, 1043, 1049, 987, 1041, 986, 20, 7, 0, 0, 0, 0, 0, 0, 1003, 1016, 1040, 954, 1021, 1019, 1031, 994, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1021, 1045, 1002, 1086, 961, 992, 994, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8281, 8167, 8315, 8216, 8154, 8078, 8091] avg=8192.0000 max=8315 max_rank=3 min=8078 min_rank=6 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1030, 999, 1018, 1021, 1072, 1015, 1076, 1009, 13, 11, 0, 0, 0, 0, 0, 0, 1066, 995, 982, 1032, 999, 1075, 1006, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 993, 988, 1053, 1036, 1058, 1035, 1003, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 981, 1033, 1002, 1025, 1012, 962, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1036, 1033, 1050, 1005, 997, 969, 1014, 4, 3, 0, 0, 0, 0, 0, 0, 1005, 1034, 968, 1062, 1086, 1073, 1048, 993, 20, 7, 0, 0, 0, 0, 0, 0, 1051, 1039, 1004, 1049, 1035, 1031, 1001, 977, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1026, 1067, 1043, 1017, 970, 1064, 1031, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8176, 8173, 8079, 8154, 8296, 8187, 8207] avg=8192.0000 max=8296 max_rank=5 min=8079 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1032, 1014, 1021, 982, 1046, 1042, 992, 1044, 16, 16, 0, 0, 0, 0, 0, 0, 992, 1083, 1022, 1038, 1023, 1024, 1050, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1038, 1036, 1018, 991, 1016, 991, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1001, 994, 1024, 1008, 1060, 1021, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1036, 1065, 1027, 1003, 954, 1022, 1018, 8, 5, 0, 0, 0, 0, 0, 0, 1032, 1010, 1062, 1053, 992, 1056, 1005, 1037, 26, 10, 0, 0, 0, 0, 0, 0, 1024, 1019, 1066, 989, 987, 1020, 1033, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 998, 993, 1019, 950, 1034, 1029, 1040, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8253, 8164, 8166, 8190, 8283, 8197, 8078] avg=8192.0000 max=8283 max_rank=5 min=8078 min_rank=7 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8448, 8192, 8192, 8064, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8064, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1013, 1031, 1065, 1020, 976, 998, 1011, 1008, 1042, 1074, 1009, 994, 1021, 994, 1108, 1021, 994, 1016, 1032, 1010, 1025, 1003, 1027, 1056, 1047, 1013, 1038, 1017, 1031, 1018, 1007, 1019, 990, 1045, 1009, 1035, 1047, 1023, 1024, 1049, 1018, 1025, 1071, 1034, 1045, 1063, 1001, 1030, 973, 999, 1022, 1005, 1002, 1026, 1024, 1051, 1019, 1056, 990, 1019, 1045, 1022, 996, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1036, 1041, 1083, 990, 1048, 1008, 1017, 1012, 999, 1067, 1016, 1045, 1044, 1052, 989, 1012, 999, 1051, 985, 999, 1040, 1007, 981, 1032, 1030, 1037, 973, 1047, 1031, 999, 1025, 1016, 1070, 997, 1040, 1006, 995, 1007, 1038, 1022, 1064, 1045, 1004, 1029, 1050, 1051, 1027, 993, 1051, 1028, 1021, 1002, 1028, 1026, 1018, 1031, 1046, 1007, 1050, 1017, 1007, 1056, 966, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1044, 1029, 1080, 1010, 989, 1060, 1034, 982, 1031, 1033, 1011, 998, 1018, 1001, 1075, 1011, 1006, 1048, 1064, 1008, 1035, 963, 1044, 981, 992, 988, 1023, 1049, 1041, 1016, 1016, 975, 985, 990, 1031, 1026, 1080, 1014, 1002, 1055, 962, 1019, 1054, 1033, 985, 1076, 1044, 1032, 1004, 1045, 1040, 1040, 1046, 1032, 1002, 1075, 1031, 1053, 991, 1016, 1057, 1026, 1001, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1102, 1061, 1008, 1028, 1043, 1056, 1020, 1031, 1013, 1046, 1048, 1024, 968, 1017, 1035, 994, 1060, 1009, 1062, 1004, 1029, 1011, 1017, 1046, 1036, 1049, 1034, 1036, 1059, 961, 1034, 1083, 1028, 986, 1039, 1008, 975, 1025, 1008, 1012, 989, 1030, 1070, 1041, 1017, 1022, 1012, 979, 988, 1027, 1027, 1005, 939, 1011, 1068, 1043, 1033, 1018, 1019, 1009, 1022, 1003, 1013, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1036, 1053, 1053, 1028, 1030, 1039, 1058, 1010, 1017, 977, 1037, 1033, 1030, 969, 1029, 1052, 1050, 1039, 991, 1016, 1008, 982, 1086, 1048, 1029, 1001, 1000, 1026, 1003, 1023, 1022, 1060, 1007, 987, 1035, 1080, 1019, 1022, 1026, 1000, 1004, 1010, 984, 1055, 982, 1065, 1042, 1013, 1019, 1055, 1013, 1018, 998, 1072, 1020, 1049, 992, 1031, 1021, 1011, 1019, 1027, 1006, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1039, 1007, 1065, 1000, 988, 1039, 1054, 1007, 985, 1063, 984, 1012, 1037, 971, 1057, 1057, 994, 1006, 1036, 1041, 1021, 972, 1031, 1062, 1075, 993, 1048, 997, 1036, 984, 1033, 1031, 1012, 1000, 1037, 985, 1020, 1020, 1034, 1054, 1044, 1040, 1009, 1003, 1031, 1039, 1049, 1024, 1016, 1006, 1039, 1063, 1042, 1031, 1029, 1044, 989, 1025, 1044, 1036, 1014, 1004, 1010, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[999, 1009, 1022, 994, 1038, 1034, 969, 1029, 1028, 1034, 1074, 1035, 1047, 1060, 995, 1013, 995, 1047, 1025, 994, 991, 996, 984, 1019, 1030, 1017, 996, 1041, 1053, 1050, 1044, 994, 1045, 1040, 1025, 1032, 1095, 1070, 1005, 1059, 1014, 1085, 961, 1064, 989, 1009, 981, 1024, 1043, 998, 1042, 1033, 1022, 1032, 988, 1089, 1045, 992, 1065, 981, 993, 988, 1015, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1003, 1007, 999, 995, 1040, 1039, 1063, 1037, 989, 946, 1027, 1043, 1050, 1027, 1066, 1028, 1056, 994, 1009, 996, 1053, 1001, 1005, 1075, 1035, 1068, 1035, 1059, 1041, 1031, 982, 1039, 1073, 983, 1019, 1055, 1033, 1008, 1015, 1056, 1006, 1030, 1027, 1036, 1038, 1046, 1014, 1006, 1011, 1000, 1027, 1068, 1007, 1004, 1065, 995, 1026, 1004, 1018, 1005, 1041, 980, 976, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1013, 1031, 1042, 1020, 976, 998, 1011, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1074, 1009, 994, 1021, 994, 1102, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1016, 1032, 1010, 1025, 1003, 1027, 1056, 9, 9, 8, 6, 0, 0, 0, 0, 1047, 1013, 1038, 1017, 1022, 1018, 1007, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1045, 1009, 1035, 1047, 1023, 1024, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1025, 1071, 1034, 1045, 1050, 1001, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 973, 999, 1022, 1005, 1002, 1026, 1024, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1056, 990, 1019, 1045, 1022, 996, 1040, 23, 13, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8257, 8195, 8181, 8213, 8274, 8090, 8227] avg=8192.0000 max=8274 max_rank=5 min=8090 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1102, 1061, 987, 1028, 1043, 1056, 1020, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1046, 1048, 1024, 968, 1017, 1030, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1009, 1062, 1004, 1029, 1011, 1017, 1046, 10, 9, 8, 5, 0, 0, 0, 0, 1036, 1049, 1034, 1036, 1049, 961, 1034, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 986, 1039, 1008, 975, 1025, 1008, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1030, 1070, 1041, 1017, 1010, 1012, 979, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1027, 1027, 1005, 939, 1011, 1068, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1018, 1019, 1009, 1022, 1003, 1013, 1046, 21, 12, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8328, 8140, 8270, 8282, 8072, 8148, 8096, 8200] avg=8192.0000 max=8328 max_rank=0 min=8072 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1039, 1007, 1042, 1000, 988, 1039, 1054, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1063, 984, 1012, 1037, 971, 1052, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1006, 1036, 1041, 1021, 972, 1031, 1062, 9, 9, 8, 5, 0, 0, 0, 0, 1075, 993, 1048, 997, 1027, 984, 1033, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1000, 1037, 985, 1020, 1020, 1034, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1040, 1009, 1003, 1031, 1027, 1049, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1006, 1039, 1063, 1042, 1031, 1029, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1025, 1044, 1036, 1014, 1004, 1010, 1018, 23, 12, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8161, 8194, 8188, 8153, 8227, 8258, 8179] avg=8192.0000 max=8258 max_rank=6 min=8153 min_rank=4 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1036, 1041, 1060, 990, 1048, 1008, 1017, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1067, 1016, 1045, 1044, 1052, 984, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1051, 985, 999, 1040, 1007, 981, 1032, 9, 9, 8, 5, 0, 0, 0, 0, 1030, 1037, 973, 1047, 1022, 999, 1025, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 997, 1040, 1006, 995, 1007, 1038, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1045, 1004, 1029, 1050, 1039, 1027, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1028, 1021, 1002, 1028, 1026, 1018, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1007, 1050, 1017, 1007, 1056, 966, 1033, 23, 12, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8219, 8125, 8149, 8166, 8251, 8193, 8221] avg=8192.0000 max=8251 max_rank=5 min=8125 min_rank=2 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1044, 1029, 1057, 1010, 989, 1060, 1034, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1033, 1011, 998, 1018, 1001, 1070, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1048, 1064, 1008, 1035, 963, 1044, 981, 10, 9, 8, 5, 0, 0, 0, 0, 992, 988, 1023, 1049, 1031, 1016, 1016, 975, 0, 0, 0, 0, 0, 0, 0, 0, 985, 990, 1031, 1026, 1080, 1014, 1002, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1019, 1054, 1033, 985, 1063, 1044, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1045, 1040, 1040, 1046, 1032, 1002, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1053, 991, 1016, 1057, 1026, 1001, 1034, 23, 13, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8173, 8181, 8090, 8174, 8192, 8272, 8249] avg=8192.0000 max=8272 max_rank=6 min=8090 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32887 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[999, 1009, 1000, 994, 1038, 1034, 969, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1034, 1074, 1035, 1047, 1060, 990, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1047, 1025, 994, 991, 996, 984, 1019, 10, 9, 8, 5, 0, 0, 0, 0, 1030, 1017, 996, 1041, 1043, 1050, 1044, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1040, 1025, 1032, 1095, 1070, 1005, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1085, 961, 1064, 989, 997, 981, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 998, 1042, 1033, 1022, 1032, 988, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 992, 1065, 981, 993, 988, 1015, 1056, 22, 12, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8072, 8281, 8083, 8215, 8362, 8115, 8235, 8173] avg=8192.0000 max=8362 max_rank=4 min=8072 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1036, 1053, 1031, 1028, 1030, 1039, 1058, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 977, 1037, 1033, 1030, 969, 1024, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1039, 991, 1016, 1008, 982, 1086, 1048, 9, 9, 8, 5, 0, 0, 0, 0, 1029, 1001, 1000, 1026, 994, 1023, 1022, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 987, 1035, 1080, 1019, 1022, 1026, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1010, 984, 1055, 982, 1052, 1042, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1055, 1013, 1018, 998, 1072, 1020, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1031, 1021, 1011, 1019, 1027, 1006, 1019, 22, 13, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8139, 8251, 8155, 8167, 8142, 8232, 8165] avg=8192.0000 max=8285 max_rank=0 min=8139 min_rank=1 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1003, 1007, 975, 995, 1040, 1039, 1063, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 989, 946, 1027, 1043, 1050, 1027, 1056, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 994, 1009, 996, 1053, 1001, 1005, 1075, 14, 13, 11, 10, 0, 0, 0, 0, 1035, 1068, 1035, 1059, 1027, 1031, 982, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 983, 1019, 1055, 1033, 1008, 1015, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1030, 1027, 1036, 1038, 1030, 1014, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1000, 1027, 1068, 1007, 1004, 1065, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1004, 1018, 1005, 1041, 980, 976, 1026, 24, 16, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8166, 8237, 8276, 8229, 8187, 8160, 8122] avg=8192.0000 max=8276 max_rank=3 min=8122 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 7936, 8192, 8320, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8064, 8320, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1046, 1080, 1027, 1042, 1034, 971, 1023, 999, 1045, 1028, 1061, 1063, 979, 1018, 1066, 1092, 1018, 1032, 1021, 1047, 1044, 1067, 991, 979, 1000, 1033, 1033, 1016, 1064, 1039, 989, 1070, 1004, 1017, 1030, 986, 997, 957, 1017, 983, 1023, 1044, 1051, 1025, 1009, 1037, 984, 1044, 1017, 971, 1057, 1026, 1027, 1043, 1004, 1042, 1030, 1027, 975, 998, 1018, 1026, 1027, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1031, 1058, 1031, 1038, 1021, 992, 1067, 1061, 977, 994, 1049, 1009, 1025, 1024, 1058, 1043, 1033, 1035, 1054, 1034, 1048, 1042, 1028, 1034, 1007, 1052, 972, 1019, 1061, 1011, 987, 985, 1068, 998, 1014, 1046, 1056, 1029, 1033, 1010, 1038, 1020, 973, 1051, 999, 1012, 1054, 1001, 1014, 1071, 1032, 1052, 997, 1017, 1009, 995, 1017, 1036, 991, 1011, 990, 1037, 1002, 983] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1009, 1014, 968, 1064, 1002, 1045, 1037, 1006, 1015, 984, 1015, 1059, 1040, 1032, 1022, 1033, 1041, 1005, 1034, 1036, 991, 1062, 1006, 1047, 1002, 1004, 998, 1074, 1042, 1010, 1036, 1009, 1086, 1006, 983, 1007, 1038, 1033, 1009, 1003, 1032, 1046, 1033, 1041, 1013, 986, 991, 1057, 1012, 1032, 1039, 1036, 1025, 1032, 1033, 1022, 976, 1037, 1019, 1021, 1043, 1054, 1034, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1024, 979, 1005, 1032, 1012, 990, 1005, 1011, 984, 1074, 979, 1053, 998, 1039, 1015, 1017, 1043, 1044, 957, 1006, 1038, 1002, 1073, 1043, 993, 1024, 1075, 1016, 1014, 1065, 1053, 1074, 979, 1007, 1029, 1036, 1017, 1037, 1045, 985, 1041, 1032, 1002, 1057, 1025, 1077, 1009, 939, 1053, 1044, 1048, 1004, 993, 1041, 1094, 1004, 991, 1008, 1046, 1018, 1039, 1043, 1035, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1045, 1077, 980, 1010, 995, 1006, 1043, 1003, 1016, 998, 991, 981, 1030, 981, 1025, 1032, 973, 1084, 1042, 983, 1015, 977, 991, 1052, 1028, 1011, 1023, 1046, 1108, 1003, 1027, 1048, 999, 1029, 1058, 1013, 1070, 1035, 1042, 1004, 1015, 1049, 1010, 1026, 1030, 1070, 1085, 990, 1046, 1015, 1007, 998, 1044, 1014, 1054, 1008, 1024, 1028, 1002, 1027, 1066, 1026, 997, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1014, 989, 1025, 1015, 988, 1048, 1048, 1054, 975, 1040, 1000, 1005, 1037, 1074, 988, 1069, 1069, 1005, 1048, 949, 1034, 1052, 994, 1040, 1037, 1092, 969, 1021, 1022, 1011, 1062, 1004, 1040, 1012, 979, 1037, 1067, 1033, 1065, 968, 1057, 1016, 1026, 1064, 991, 1033, 1053, 1006, 975, 1044, 1038, 995, 1044, 1017, 1033, 1007, 1036, 973, 1033, 1016, 1005, 1045, 1007, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1007, 1052, 1005, 1038, 994, 1104, 1010, 1041, 1062, 997, 987, 1035, 1014, 1022, 980, 1022, 997, 1049, 1057, 1020, 1006, 1027, 980, 1053, 1028, 1055, 1035, 1047, 997, 989, 981, 1015, 1004, 1020, 1009, 1002, 998, 1036, 1030, 1060, 1033, 1026, 1008, 1030, 1035, 1073, 1018, 1019, 998, 1050, 1050, 980, 1018, 1065, 1043, 958, 1012, 1074, 1051, 996, 1040, 1081, 1006, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1010, 997, 1009, 1023, 994, 1054, 991, 1016, 1075, 1024, 1072, 1023, 1038, 992, 1038, 1040, 1035, 990, 1044, 993, 1060, 1003, 1026, 1030, 995, 1050, 1006, 993, 1047, 1043, 1052, 1021, 1022, 1005, 1052, 1017, 1021, 1070, 1070, 1066, 1007, 1001, 1044, 1021, 1016, 1021, 1023, 1021, 1041, 994, 974, 1043, 1013, 1023, 1018, 999, 1012, 1009, 1006, 1046, 1042, 1011, 1027, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1046, 1080, 1027, 1042, 1034, 971, 1023, 999, 14, 1, 0, 0, 0, 0, 0, 0, 1045, 1028, 1061, 1063, 979, 1018, 1066, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1032, 1021, 1047, 1044, 1067, 991, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1033, 1033, 1016, 1045, 1039, 989, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1017, 1030, 986, 997, 957, 1017, 983, 5, 0, 0, 0, 0, 0, 0, 0, 1023, 1044, 1051, 1011, 1009, 1037, 984, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 971, 1057, 1026, 1027, 1043, 1004, 1042, 3, 2, 0, 0, 0, 0, 0, 0, 1030, 1027, 975, 998, 1018, 1026, 1027, 1023, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8350, 8191, 8225, 7996, 8203, 8192, 8142] avg=8192.0000 max=8350 max_rank=1 min=7996 min_rank=4 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1009, 1014, 968, 1064, 1002, 1045, 1037, 1006, 14, 0, 0, 0, 0, 0, 0, 0, 1015, 984, 1015, 1059, 1040, 1032, 1022, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1005, 1034, 1036, 991, 1062, 1006, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1004, 998, 1074, 1024, 1010, 1036, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 1006, 983, 1007, 1038, 1033, 1009, 1003, 6, 1, 0, 0, 0, 0, 0, 0, 1032, 1046, 1033, 1026, 1013, 986, 991, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1032, 1039, 1036, 1025, 1032, 1033, 1022, 3, 2, 0, 0, 0, 0, 0, 0, 976, 1037, 1019, 1021, 1043, 1054, 1034, 1015, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8198, 8213, 8157, 8172, 8184, 8236, 8217] avg=8192.0000 max=8236 max_rank=6 min=8157 min_rank=3 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1031, 1058, 1031, 1038, 1021, 992, 1067, 1061, 15, 1, 0, 0, 0, 0, 0, 0, 977, 994, 1049, 1009, 1025, 1024, 1058, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1035, 1054, 1034, 1048, 1042, 1028, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1052, 972, 1019, 1042, 1011, 987, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 998, 1014, 1046, 1056, 1029, 1033, 1010, 6, 1, 0, 0, 0, 0, 0, 0, 1038, 1020, 973, 1035, 999, 1012, 1054, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1071, 1032, 1052, 997, 1017, 1009, 995, 3, 2, 0, 0, 0, 0, 0, 0, 1017, 1036, 991, 1011, 990, 1037, 1002, 983, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8315, 8177, 8299, 8075, 8261, 8132, 8192, 8085] avg=8192.0000 max=8315 max_rank=0 min=8075 min_rank=3 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1045, 1077, 980, 1010, 995, 1006, 1043, 1003, 14, 1, 0, 0, 0, 0, 0, 0, 1016, 998, 991, 981, 1030, 981, 1025, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1084, 1042, 983, 1015, 977, 991, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1011, 1023, 1046, 1088, 1003, 1027, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1029, 1058, 1013, 1070, 1035, 1042, 1004, 6, 0, 0, 0, 0, 0, 0, 0, 1015, 1049, 1010, 1012, 1030, 1070, 1085, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1015, 1007, 998, 1044, 1014, 1054, 1008, 3, 2, 0, 0, 0, 0, 0, 0, 1024, 1028, 1002, 1027, 1066, 1026, 997, 1031, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8052, 8108, 8274, 8256, 8261, 8191, 8220] avg=8192.0000 max=8274 max_rank=3 min=8052 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32928 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1010, 997, 1009, 1023, 994, 1054, 991, 1016, 14, 1, 0, 0, 0, 0, 0, 0, 1075, 1024, 1072, 1023, 1038, 992, 1038, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 990, 1044, 993, 1060, 1003, 1026, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1050, 1006, 993, 1028, 1043, 1052, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1005, 1052, 1017, 1021, 1070, 1070, 1066, 6, 0, 0, 0, 0, 0, 0, 0, 1007, 1001, 1044, 1007, 1016, 1021, 1023, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 994, 974, 1043, 1013, 1023, 1018, 999, 3, 2, 0, 0, 0, 0, 0, 0, 1012, 1009, 1006, 1046, 1042, 1011, 1027, 1017, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8300, 8172, 8188, 8329, 8140, 8110, 8188] avg=8192.0000 max=8329 max_rank=4 min=8109 min_rank=0 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1024, 979, 1005, 1032, 1012, 990, 1005, 1011, 15, 0, 0, 0, 0, 0, 0, 0, 984, 1074, 979, 1053, 998, 1039, 1015, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1044, 957, 1006, 1038, 1002, 1073, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1024, 1075, 1016, 996, 1065, 1053, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1007, 1029, 1036, 1017, 1037, 1045, 985, 6, 1, 0, 0, 0, 0, 0, 0, 1041, 1032, 1002, 1041, 1025, 1077, 1009, 939, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1044, 1048, 1004, 993, 1041, 1094, 1004, 3, 2, 0, 0, 0, 0, 0, 0, 991, 1008, 1046, 1018, 1039, 1043, 1035, 1021, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8073, 8157, 8197, 8296, 8142, 8166, 8286, 8219] avg=8192.0000 max=8296 max_rank=3 min=8073 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1014, 989, 1025, 1015, 988, 1048, 1048, 1054, 15, 0, 0, 0, 0, 0, 0, 0, 975, 1040, 1000, 1005, 1037, 1074, 988, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1005, 1048, 949, 1034, 1052, 994, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1092, 969, 1021, 1004, 1011, 1062, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1012, 979, 1037, 1067, 1033, 1065, 968, 6, 1, 0, 0, 0, 0, 0, 0, 1057, 1016, 1026, 1048, 991, 1033, 1053, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1044, 1038, 995, 1044, 1017, 1033, 1007, 3, 2, 0, 0, 0, 0, 0, 0, 1036, 973, 1033, 1016, 1005, 1045, 1007, 1043, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8186, 8182, 8200, 8208, 8230, 8158, 8176] avg=8192.0000 max=8230 max_rank=5 min=8158 min_rank=6 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1007, 1052, 1005, 1038, 994, 1104, 1010, 1041, 18, 4, 0, 0, 0, 0, 0, 0, 1062, 997, 987, 1035, 1014, 1022, 980, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1049, 1057, 1020, 1006, 1027, 980, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1055, 1035, 1047, 971, 989, 981, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1020, 1009, 1002, 998, 1036, 1030, 1060, 9, 4, 0, 0, 0, 0, 0, 0, 1033, 1026, 1008, 1008, 1035, 1073, 1018, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1050, 1050, 980, 1018, 1065, 1043, 958, 6, 3, 0, 0, 0, 0, 0, 0, 1012, 1074, 1051, 996, 1040, 1081, 1006, 1007, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8116, 8174, 8121, 8172, 8220, 8171, 8289] avg=8192.0000 max=8289 max_rank=7 min=8116 min_rank=1 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32852 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8320, 8064, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8192, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8448, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1079, 1032, 1039, 1052, 1014, 1017, 979, 1072, 1022, 1019, 1005, 1068, 1047, 992, 1056, 1051, 1008, 1012, 1043, 999, 1029, 1047, 1060, 1058, 959, 998, 1002, 1056, 948, 985, 988, 1038, 1010, 1030, 1056, 1032, 1060, 1074, 1006, 1015, 1011, 1020, 1063, 1059, 998, 1044, 1019, 1035, 1019, 954, 1036, 1010, 1005, 989, 1034, 989, 1008, 999, 1029, 972, 1039, 1035, 1054, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[993, 1001, 1005, 1024, 1024, 1007, 1023, 1075, 1017, 1036, 1027, 1010, 1047, 1011, 1015, 1043, 1075, 988, 1085, 1028, 1030, 1036, 1044, 970, 1026, 1019, 1011, 1001, 1050, 1043, 959, 993, 1064, 1022, 1007, 1046, 1001, 1054, 1007, 1008, 991, 1045, 1005, 1063, 1004, 1017, 983, 1023, 1008, 999, 1017, 1055, 1028, 1060, 1052, 1069, 1046, 1052, 1020, 1001, 1004, 1053, 1012, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1023, 1059, 999, 1040, 1065, 1017, 1074, 1016, 1036, 1035, 1011, 1084, 994, 997, 1010, 1026, 996, 999, 1061, 986, 988, 1013, 995, 1034, 1063, 986, 1058, 1032, 1028, 1031, 1055, 1043, 1034, 1001, 1022, 1039, 1002, 1004, 1050, 1032, 1023, 1026, 989, 996, 1059, 985, 1018, 1019, 1019, 1002, 1047, 1042, 1082, 1025, 999, 1010, 1032, 986, 993, 1006, 1031, 1053, 997, 1059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1059, 993, 1038, 1042, 1039, 1037, 998, 1004, 1009, 1008, 1005, 1052, 1058, 1034, 1055, 1001, 1090, 1025, 1035, 1006, 1008, 1001, 1041, 1037, 1039, 1022, 1048, 1038, 995, 1035, 998, 1038, 985, 1047, 1026, 1019, 1016, 1039, 1025, 992, 1049, 977, 1003, 1013, 1009, 1006, 1019, 991, 1029, 1016, 986, 993, 1041, 998, 1063, 1027, 1061, 1022, 1001, 1003, 1007, 1038, 1118, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1015, 1041, 1080, 999, 1000, 1032, 1021, 1037, 1034, 996, 973, 1032, 1023, 1035, 1022, 981, 1026, 1024, 1033, 1019, 1056, 1011, 982, 1015, 1018, 1061, 1056, 987, 1056, 1009, 1041, 1083, 1050, 1006, 1012, 1042, 1045, 986, 1038, 1032, 1029, 1018, 1023, 1042, 1030, 1001, 1061, 1010, 1069, 1049, 1017, 1022, 983, 1022, 984, 1036, 1048, 1027, 1017, 1002, 999, 1021, 999, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1002, 1038, 1005, 998, 1040, 1015, 1040, 1032, 1051, 1003, 1015, 1052, 1021, 1023, 963, 1003, 1021, 1003, 1084, 1007, 1004, 1011, 1025, 1064, 1053, 1026, 1081, 1069, 1061, 987, 1016, 1011, 987, 1051, 1013, 1033, 1047, 1076, 1035, 1005, 995, 1012, 1008, 1038, 1009, 1002, 995, 1031, 1013, 1008, 1002, 1006, 1042, 1045, 1057, 996, 1032, 1034, 978, 1001, 1054, 1042, 1026, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1061, 1029, 1047, 995, 1036, 986, 1066, 1024, 1005, 994, 1013, 954, 1010, 1016, 1044, 1032, 999, 1049, 1042, 1008, 959, 1051, 995, 1048, 1007, 1036, 1026, 1021, 1017, 1058, 1042, 1010, 991, 1080, 1006, 1024, 1036, 1064, 1031, 1021, 1030, 1001, 1035, 964, 1055, 1033, 979, 1031, 1044, 1002, 1003, 1018, 1020, 1000, 1023, 1072, 1046, 998, 1046, 1037, 1043, 1010, 1042, 1071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1030, 1024, 1012, 1056, 1057, 1023, 1004, 1044, 1010, 1037, 1016, 999, 1088, 1023, 1029, 1036, 1008, 980, 1051, 1044, 1059, 1005, 1002, 1019, 1032, 1038, 1042, 993, 996, 985, 1006, 1037, 1031, 1072, 1051, 1017, 1015, 1017, 1039, 996, 1022, 998, 1032, 1028, 1045, 1030, 977, 983, 1048, 1003, 1094, 1023, 1039, 1011, 1002, 1014, 1011, 1040, 1031, 1002, 961, 1028, 1046, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1079, 1032, 1039, 1052, 1014, 1017, 979, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1019, 1005, 1068, 1047, 992, 1056, 1051, 13, 2, 0, 0, 0, 0, 0, 0, 1008, 1012, 1043, 999, 1029, 1047, 1060, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 959, 998, 1002, 1056, 948, 985, 988, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1030, 1056, 1032, 1060, 1044, 1006, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1020, 1063, 1059, 998, 1044, 1019, 1035, 37, 17, 0, 0, 0, 0, 0, 0, 1019, 954, 1036, 1010, 1005, 989, 1034, 989, 7, 0, 0, 0, 0, 0, 0, 0, 1008, 999, 1029, 972, 1039, 1035, 1054, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8275, 8256, 7974, 8253, 8303, 8043, 8185] avg=8192.0000 max=8303 max_rank=5 min=7974 min_rank=3 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[993, 1001, 1005, 1024, 1024, 1007, 1023, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1036, 1027, 1010, 1047, 1011, 1015, 1043, 13, 2, 0, 0, 0, 0, 0, 0, 1075, 988, 1085, 1028, 1030, 1036, 1044, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1019, 1011, 1001, 1050, 1043, 959, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1022, 1007, 1046, 1001, 1024, 1007, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1045, 1005, 1063, 1004, 1017, 983, 1023, 37, 17, 0, 0, 0, 0, 0, 0, 1008, 999, 1017, 1055, 1028, 1060, 1052, 1069, 7, 0, 0, 0, 0, 0, 0, 0, 1046, 1052, 1020, 1001, 1004, 1053, 1012, 995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8221, 8256, 8102, 8179, 8185, 8295, 8183] avg=8192.0000 max=8295 max_rank=6 min=8102 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1015, 1041, 1080, 999, 1000, 1032, 1021, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 996, 973, 1032, 1023, 1035, 1022, 981, 12, 2, 0, 0, 0, 0, 0, 0, 1026, 1024, 1033, 1019, 1056, 1011, 982, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1061, 1056, 987, 1056, 1009, 1041, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1006, 1012, 1042, 1045, 958, 1038, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1018, 1023, 1042, 1030, 1001, 1061, 1010, 36, 16, 0, 0, 0, 0, 0, 0, 1069, 1049, 1017, 1022, 983, 1022, 984, 1036, 7, 0, 0, 0, 0, 0, 0, 0, 1048, 1027, 1017, 1002, 999, 1021, 999, 1009, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8110, 8166, 8311, 8183, 8266, 8189, 8122] avg=8192.0000 max=8311 max_rank=3 min=8110 min_rank=1 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1023, 1059, 999, 1040, 1065, 1017, 1074, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1035, 1011, 1084, 994, 997, 1010, 1026, 12, 2, 0, 0, 0, 0, 0, 0, 996, 999, 1061, 986, 988, 1013, 995, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 986, 1058, 1032, 1028, 1031, 1055, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1001, 1022, 1039, 1002, 976, 1050, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1026, 989, 996, 1059, 985, 1018, 1019, 35, 16, 0, 0, 0, 0, 0, 0, 1019, 1002, 1047, 1042, 1082, 1025, 999, 1010, 7, 0, 0, 0, 0, 0, 0, 0, 1032, 986, 993, 1006, 1031, 1053, 997, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8207, 8072, 8296, 8156, 8166, 8233, 8148] avg=8192.0000 max=8296 max_rank=3 min=8072 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1002, 1038, 1005, 998, 1040, 1015, 1040, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1003, 1015, 1052, 1021, 1023, 963, 1003, 13, 2, 0, 0, 0, 0, 0, 0, 1021, 1003, 1084, 1007, 1004, 1011, 1025, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1026, 1081, 1069, 1061, 987, 1016, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1051, 1013, 1033, 1047, 1046, 1035, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1012, 1008, 1038, 1009, 1002, 995, 1031, 36, 17, 0, 0, 0, 0, 0, 0, 1013, 1008, 1002, 1006, 1042, 1045, 1057, 996, 7, 0, 0, 0, 0, 0, 0, 0, 1032, 1034, 978, 1001, 1054, 1042, 1026, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8146, 8219, 8304, 8217, 8143, 8176, 8197] avg=8192.0000 max=8304 max_rank=3 min=8134 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1059, 993, 1038, 1042, 1039, 1037, 998, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1008, 1005, 1052, 1058, 1034, 1055, 1001, 12, 2, 0, 0, 0, 0, 0, 0, 1090, 1025, 1035, 1006, 1008, 1001, 1041, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1022, 1048, 1038, 995, 1035, 998, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1047, 1026, 1019, 1016, 1011, 1025, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 977, 1003, 1013, 1009, 1006, 1019, 991, 35, 16, 0, 0, 0, 0, 0, 0, 1029, 1016, 986, 993, 1041, 998, 1063, 1027, 7, 0, 0, 0, 0, 0, 0, 0, 1061, 1022, 1001, 1003, 1007, 1038, 1118, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8236, 8243, 8213, 8121, 8118, 8160, 8270] avg=8192.0000 max=8270 max_rank=7 min=8118 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1061, 1029, 1047, 995, 1036, 986, 1066, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 994, 1013, 954, 1010, 1016, 1044, 1032, 13, 2, 0, 0, 0, 0, 0, 0, 999, 1049, 1042, 1008, 959, 1051, 995, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1036, 1026, 1021, 1017, 1058, 1042, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1080, 1006, 1024, 1036, 1034, 1031, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1001, 1035, 964, 1055, 1033, 979, 1031, 36, 17, 0, 0, 0, 0, 0, 0, 1044, 1002, 1003, 1018, 1020, 1000, 1023, 1072, 7, 0, 0, 0, 0, 0, 0, 0, 1046, 998, 1046, 1037, 1043, 1010, 1042, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8083, 8151, 8217, 8223, 8181, 8189, 8284] avg=8192.0000 max=8284 max_rank=7 min=8083 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1030, 1024, 1012, 1056, 1057, 1023, 1004, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1037, 1016, 999, 1088, 1023, 1029, 1036, 15, 5, 0, 0, 0, 0, 0, 0, 1008, 980, 1051, 1044, 1059, 1005, 1002, 1019, 5, 0, 0, 0, 0, 0, 0, 0, 1032, 1038, 1032, 993, 996, 985, 1006, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1072, 1051, 1017, 1015, 983, 1039, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 998, 1032, 1028, 1045, 1030, 977, 983, 40, 19, 0, 0, 0, 0, 0, 0, 1048, 1003, 1094, 1023, 1039, 1011, 1002, 1014, 12, 5, 0, 0, 0, 0, 0, 0, 1011, 1040, 1031, 1002, 961, 1028, 1046, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8258, 8173, 8119, 8204, 8174, 8251, 8147] avg=8192.0000 max=8258 max_rank=1 min=8119 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8064, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8064, 8192, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8064, 8320, 8192, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8448, 8192, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[993, 1003, 1073, 1039, 1007, 1017, 1038, 1051, 998, 1006, 1034, 982, 1064, 1051, 988, 1005, 1050, 1034, 1025, 1039, 969, 973, 1050, 1011, 1052, 1021, 1041, 1043, 1035, 1044, 1012, 1017, 1021, 1037, 1055, 1023, 1007, 1034, 1053, 1030, 1031, 1025, 997, 987, 1036, 1011, 987, 1002, 1053, 1004, 1030, 1001, 981, 1086, 1001, 1006, 1011, 1054, 1074, 1068, 997, 1065, 992, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1032, 1037, 1079, 1018, 1034, 1086, 1003, 1016, 982, 1020, 1020, 1078, 1013, 1032, 985, 1042, 1095, 1026, 1004, 989, 1069, 1025, 1024, 999, 1019, 1014, 996, 998, 1018, 1043, 997, 1050, 1041, 1033, 998, 1033, 1037, 1011, 1004, 1021, 1030, 1015, 1048, 1034, 1012, 995, 978, 1038, 1004, 1015, 996, 1037, 1050, 996, 1020, 1070, 1022, 1045, 994, 1035, 1029, 1042, 1005, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1020, 996, 1015, 1012, 1012, 1038, 1028, 992, 1029, 1030, 1051, 1022, 987, 1025, 1057, 1068, 970, 1023, 1029, 1020, 987, 1078, 990, 1075, 1012, 1023, 975, 1002, 1014, 1054, 1066, 1071, 1023, 1026, 1007, 1035, 965, 1045, 1016, 1053, 1041, 1029, 1014, 1034, 1046, 1073, 1007, 1040, 993, 998, 991, 1017, 1074, 978, 1042, 1029, 1054, 987, 1014, 1008, 1035, 1008, 1049, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1041, 1031, 1068, 1028, 1093, 1052, 978, 987, 991, 1065, 1062, 1023, 1024, 1043, 981, 1023, 1052, 1022, 986, 1081, 1011, 1029, 1015, 980, 971, 1004, 1006, 996, 997, 971, 1017, 1018, 1043, 1030, 1035, 1055, 989, 1029, 1020, 1018, 1015, 1002, 1034, 978, 1009, 1036, 1052, 1042, 1029, 1003, 1031, 1078, 1077, 1057, 1064, 994, 990, 982, 1026, 1043, 992, 1029, 1064, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1026, 1032, 1018, 1008, 1014, 1001, 1025, 1023, 1025, 1007, 1089, 1044, 1046, 1050, 1056, 1028, 995, 1041, 981, 968, 1035, 982, 1019, 1013, 1015, 1059, 996, 1013, 1032, 1047, 1052, 1041, 1022, 1043, 1074, 1042, 994, 1052, 1019, 1015, 1039, 1014, 990, 934, 1035, 1058, 1014, 1010, 1012, 1051, 1077, 1017, 1023, 1004, 1048, 995, 988, 1021, 1013, 1062, 1026, 1021, 1039, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1030, 1022, 999, 1043, 997, 995, 1041, 996, 1058, 1020, 1005, 1019, 1003, 1025, 1015, 1015, 1032, 1085, 1017, 1036, 997, 1027, 1042, 1003, 1032, 1028, 1012, 1048, 1071, 987, 1074, 1019, 1008, 1014, 1035, 1046, 1004, 1002, 1041, 1022, 987, 1060, 1034, 1020, 1004, 1040, 1022, 1013, 1040, 1013, 1018, 999, 1042, 993, 1077, 1010, 1052, 1011, 1025, 1021, 1027, 1012, 1031, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1047, 984, 1013, 1016, 1024, 986, 1012, 1039, 987, 1057, 1018, 1008, 1042, 1039, 1025, 1063, 1016, 1026, 979, 1016, 1026, 1042, 1049, 1022, 1071, 1005, 1033, 1042, 1039, 1019, 1014, 1035, 979, 968, 996, 1012, 1019, 1093, 1086, 1013, 989, 1046, 1027, 995, 976, 1063, 1058, 1037, 1013, 1046, 983, 998, 1010, 985, 1012, 1028, 1001, 1041, 1031, 1060, 1014, 1046, 1047, 1070] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[988, 1018, 1029, 1043, 1008, 1115, 1000, 1050, 969, 1041, 1076, 1007, 1019, 1066, 1011, 1007, 1036, 1026, 1003, 999, 1039, 1003, 990, 984, 984, 977, 1014, 998, 987, 1060, 970, 1034, 1038, 1009, 1101, 1007, 1041, 1022, 1037, 1066, 1074, 1039, 1038, 1039, 983, 982, 1057, 1099, 1049, 1060, 1010, 960, 1079, 1033, 1016, 1055, 974, 1049, 1003, 984, 1016, 1020, 1007, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1020, 996, 1013, 1012, 1012, 1038, 1028, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1030, 1029, 1022, 987, 1025, 1057, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1023, 1029, 1020, 987, 1078, 990, 1075, 25, 8, 0, 0, 0, 0, 0, 0, 1012, 1023, 975, 1002, 1014, 1054, 1066, 1071, 14, 1, 0, 0, 0, 0, 0, 0, 1023, 1026, 982, 1035, 965, 1045, 1016, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1029, 1014, 1034, 1046, 1073, 1007, 1040, 4, 3, 2, 0, 0, 0, 0, 0, 993, 998, 991, 1017, 1071, 978, 1042, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 987, 1014, 1003, 1035, 1008, 1049, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8247, 8205, 8232, 8145, 8293, 8119, 8184] avg=8192.0000 max=8293 max_rank=5 min=8111 min_rank=0 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[993, 1003, 1071, 1039, 1007, 1017, 1038, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1006, 1012, 982, 1064, 1051, 988, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1034, 1025, 1039, 969, 973, 1050, 1011, 26, 8, 0, 0, 0, 0, 0, 0, 1052, 1021, 1041, 1043, 1035, 1044, 1012, 1017, 14, 1, 0, 0, 0, 0, 0, 0, 1021, 1037, 1029, 1023, 1007, 1034, 1053, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1025, 997, 987, 1036, 1011, 987, 1002, 4, 2, 2, 0, 0, 0, 0, 0, 1053, 1004, 1030, 1001, 979, 1086, 1001, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1054, 1074, 1063, 997, 1065, 992, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8106, 8185, 8280, 8234, 8084, 8160, 8268] avg=8192.0000 max=8280 max_rank=3 min=8084 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1032, 1037, 1077, 1018, 1034, 1086, 1003, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1020, 998, 1078, 1013, 1032, 985, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 1026, 1004, 989, 1069, 1025, 1024, 999, 25, 8, 0, 0, 0, 0, 0, 0, 1019, 1014, 996, 998, 1018, 1043, 997, 1050, 14, 1, 0, 0, 0, 0, 0, 0, 1041, 1033, 973, 1033, 1037, 1011, 1004, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1015, 1048, 1034, 1012, 995, 978, 1038, 4, 3, 2, 0, 0, 0, 0, 0, 1004, 1015, 996, 1037, 1047, 996, 1020, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1045, 994, 1030, 1029, 1042, 1005, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8150, 8264, 8150, 8153, 8159, 8185, 8172] avg=8192.0000 max=8303 max_rank=0 min=8150 min_rank=1 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1041, 1031, 1066, 1028, 1093, 1052, 978, 987, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1065, 1039, 1023, 1024, 1043, 981, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1022, 986, 1081, 1011, 1029, 1015, 980, 26, 8, 0, 0, 0, 0, 0, 0, 971, 1004, 1006, 996, 997, 971, 1017, 1018, 15, 1, 0, 0, 0, 0, 0, 0, 1043, 1030, 1009, 1055, 989, 1029, 1020, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1002, 1034, 978, 1009, 1036, 1052, 1042, 4, 3, 2, 0, 0, 0, 0, 0, 1029, 1003, 1031, 1078, 1074, 1057, 1064, 994, 0, 0, 0, 0, 0, 0, 0, 0, 990, 982, 1026, 1038, 992, 1029, 1064, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8189, 8210, 7996, 8193, 8177, 8330, 8165] avg=8192.0000 max=8330 max_rank=6 min=7996 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1026, 1032, 1016, 1008, 1014, 1001, 1025, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1007, 1066, 1044, 1046, 1050, 1056, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1041, 981, 968, 1035, 982, 1019, 1013, 27, 8, 0, 0, 0, 0, 0, 0, 1015, 1059, 996, 1013, 1032, 1047, 1052, 1041, 15, 1, 0, 0, 0, 0, 0, 0, 1022, 1043, 1047, 1042, 994, 1052, 1019, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1014, 990, 934, 1035, 1058, 1014, 1010, 4, 3, 2, 0, 0, 0, 0, 0, 1012, 1051, 1077, 1017, 1020, 1004, 1048, 995, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1021, 1013, 1057, 1026, 1021, 1039, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8322, 8069, 8271, 8234, 8103, 8224, 8168] avg=8192.0000 max=8322 max_rank=1 min=8069 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1030, 1022, 997, 1043, 997, 995, 1041, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1020, 984, 1019, 1003, 1025, 1015, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1085, 1017, 1036, 997, 1027, 1042, 1003, 26, 7, 0, 0, 0, 0, 0, 0, 1032, 1028, 1012, 1048, 1071, 987, 1074, 1019, 14, 1, 0, 0, 0, 0, 0, 0, 1008, 1014, 1009, 1046, 1004, 1002, 1041, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1060, 1034, 1020, 1004, 1040, 1022, 1013, 4, 3, 2, 0, 0, 0, 0, 0, 1040, 1013, 1018, 999, 1039, 993, 1077, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1011, 1025, 1016, 1027, 1012, 1031, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8139, 8272, 8286, 8146, 8189, 8189, 8194] avg=8192.0000 max=8286 max_rank=3 min=8121 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1047, 984, 1011, 1016, 1024, 986, 1012, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1057, 996, 1008, 1042, 1039, 1025, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1026, 979, 1016, 1026, 1042, 1049, 1022, 25, 8, 0, 0, 0, 0, 0, 0, 1071, 1005, 1033, 1042, 1039, 1019, 1014, 1035, 14, 1, 0, 0, 0, 0, 0, 0, 979, 968, 971, 1012, 1019, 1093, 1086, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1046, 1027, 995, 976, 1063, 1058, 1037, 4, 3, 2, 0, 0, 0, 0, 0, 1013, 1046, 983, 998, 1007, 985, 1012, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1041, 1031, 1055, 1014, 1046, 1047, 1070, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8217, 8209, 8273, 8141, 8200, 8072, 8305] avg=8192.0000 max=8305 max_rank=7 min=8072 min_rank=6 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[988, 1018, 1020, 1043, 1008, 1115, 1000, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1041, 1046, 1007, 1019, 1066, 1011, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1026, 1003, 999, 1039, 1003, 990, 984, 31, 11, 0, 0, 0, 0, 0, 0, 984, 977, 1014, 998, 987, 1060, 970, 1034, 19, 5, 0, 0, 0, 0, 0, 0, 1038, 1009, 1070, 1007, 1041, 1022, 1037, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1039, 1038, 1039, 983, 982, 1057, 1099, 6, 5, 9, 0, 0, 0, 0, 0, 1049, 1060, 1010, 960, 1074, 1033, 1016, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1049, 1003, 973, 1016, 1020, 1007, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8166, 8122, 8048, 8290, 8331, 8257, 8080] avg=8192.0000 max=8331 max_rank=5 min=8048 min_rank=3 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32958 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8064, 8192, 8320, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8192, 8192, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8064, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8192, 8320, 8448, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1000, 1044, 1055, 1001, 1024, 1003, 1046, 1035, 1004, 1044, 1013, 989, 953, 999, 976, 1016, 1020, 1025, 1076, 1010, 1030, 1050, 1028, 1070, 1029, 994, 1025, 1027, 1030, 996, 1087, 1005, 1010, 1016, 1052, 1095, 1005, 1009, 1062, 1004, 1039, 1078, 1034, 1062, 1024, 1016, 1023, 1051, 1011, 1002, 1049, 1057, 1018, 993, 967, 1006, 971, 1014, 994, 1061, 1036, 1015, 1016, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1021, 1045, 1033, 1026, 1027, 1046, 1006, 1020, 1026, 1046, 1080, 990, 1025, 975, 1048, 962, 1032, 996, 980, 1032, 985, 1019, 1010, 1033, 1025, 1037, 1021, 980, 1023, 1031, 1045, 1027, 1032, 1097, 1045, 1037, 1003, 1046, 996, 1066, 1005, 1021, 996, 1054, 1011, 982, 978, 1002, 1029, 1005, 1068, 1003, 1038, 1073, 954, 1052, 1071, 1019, 1027, 1062, 1018, 1035, 1000, 1059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1022, 988, 988, 1063, 1019, 996, 1028, 1024, 956, 972, 1020, 973, 1065, 1063, 1058, 1031, 1013, 1019, 1026, 1049, 958, 1032, 1035, 1036, 1014, 982, 1057, 1049, 1004, 1007, 1076, 1005, 1053, 1013, 1059, 1045, 1069, 1012, 980, 1041, 1039, 1070, 1028, 1041, 1086, 1043, 1008, 1053, 1000, 1000, 1041, 1014, 1043, 1016, 1042, 1059, 956, 1065, 1028, 1053, 996, 979, 1011, 965] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1040, 1068, 1053, 1052, 1073, 1036, 1025, 995, 982, 1026, 1010, 978, 1031, 1055, 1040, 1029, 991, 997, 1025, 1036, 1039, 1085, 987, 1065, 961, 1031, 995, 1019, 990, 1044, 1066, 1011, 1093, 1018, 1077, 1032, 1010, 1006, 951, 1075, 1040, 1035, 968, 1035, 1021, 1012, 1050, 1046, 1043, 1008, 1054, 1027, 1005, 957, 1037, 980, 1039, 1020, 978, 1043, 1055, 987, 1022, 977] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1009, 1014, 992, 1008, 1035, 1003, 1036, 1012, 1048, 1098, 1043, 1017, 1005, 1022, 1018, 986, 996, 1004, 975, 1002, 1001, 1021, 1003, 987, 1063, 1008, 1037, 1047, 1000, 1015, 1067, 992, 1061, 997, 1094, 1022, 1031, 1066, 1016, 1050, 1033, 1032, 1001, 1005, 1015, 996, 1029, 1020, 1003, 1016, 1059, 1066, 1003, 1033, 1081, 1009, 987, 995, 1006, 1080, 1074, 1054, 1048, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1045, 1065, 1026, 982, 1041, 1057, 1051, 1041, 1025, 983, 990, 1042, 1047, 1069, 1058, 1034, 982, 962, 1023, 1046, 1002, 1042, 1088, 1055, 1029, 1010, 1011, 1012, 1017, 984, 1033, 1012, 1079, 1017, 1011, 1018, 996, 1016, 1043, 1013, 1037, 1018, 1016, 1056, 976, 1007, 1050, 1015, 977, 1031, 1009, 1047, 1021, 1058, 1003, 997, 1028, 1021, 1039, 1014, 1036, 987, 1004, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1048, 1006, 1032, 1053, 1030, 1065, 1048, 1015, 1032, 998, 1024, 1028, 1020, 1056, 1036, 1020, 1044, 1077, 1068, 1019, 986, 1088, 1026, 1030, 1011, 1012, 990, 996, 1020, 1029, 1015, 1030, 992, 1051, 1037, 1035, 1006, 998, 1015, 1025, 1014, 989, 1008, 1039, 1043, 1014, 1014, 963, 951, 998, 1083, 1028, 996, 1042, 995, 999, 1070, 1024, 1031, 997, 1041, 1034, 1033, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1014, 1020, 1047, 995, 1029, 1030, 1035, 1005, 1032, 984, 1020, 1055, 1073, 1054, 1004, 977, 1036, 1010, 1063, 1001, 1062, 1065, 1016, 1028, 1031, 1036, 1027, 1026, 1010, 1004, 993, 993, 1026, 1064, 992, 988, 1014, 1009, 1059, 1037, 1018, 1032, 1020, 1019, 1011, 1031, 1024, 966, 994, 1069, 999, 1010, 1024, 1026, 1044, 1045, 1037, 1019, 1033, 1029, 1065, 998, 1059, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1000, 1044, 1055, 1001, 994, 1003, 1046, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1044, 1013, 989, 953, 999, 976, 1016, 21, 4, 0, 0, 0, 0, 0, 0, 1020, 1025, 1076, 1010, 1030, 1044, 1028, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 994, 1025, 1027, 1030, 996, 1087, 1005, 35, 0, 0, 0, 0, 0, 0, 0, 1010, 1016, 996, 1095, 1005, 1009, 1062, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1078, 1034, 1062, 1024, 1016, 1023, 1051, 6, 2, 0, 0, 0, 0, 0, 0, 1011, 1002, 1049, 1057, 1018, 993, 967, 1006, 20, 0, 0, 0, 0, 0, 0, 0, 971, 1014, 994, 1061, 1036, 1015, 1016, 1042, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8019, 8303, 8228, 8197, 8335, 8123, 8153] avg=8192.0000 max=8335 max_rank=5 min=8019 min_rank=1 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1045, 1065, 1026, 982, 1010, 1057, 1051, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 983, 990, 1042, 1047, 1069, 1058, 1034, 20, 4, 0, 0, 0, 0, 0, 0, 982, 962, 1023, 1046, 1002, 1036, 1088, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1010, 1011, 1012, 1017, 984, 1033, 1012, 34, 0, 0, 0, 0, 0, 0, 0, 1079, 1017, 957, 1018, 996, 1016, 1043, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1018, 1016, 1056, 976, 1007, 1050, 1015, 6, 2, 0, 0, 0, 0, 0, 0, 977, 1031, 1009, 1047, 1021, 1058, 1003, 997, 21, 0, 0, 0, 0, 0, 0, 0, 1028, 1021, 1039, 1014, 1036, 987, 1004, 1032, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8272, 8194, 8142, 8139, 8183, 8164, 8165] avg=8192.0000 max=8277 max_rank=0 min=8139 min_rank=4 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1022, 988, 988, 1063, 989, 996, 1028, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 956, 972, 1020, 973, 1065, 1063, 1058, 1031, 21, 4, 0, 0, 0, 0, 0, 0, 1013, 1019, 1026, 1049, 958, 1026, 1035, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 982, 1057, 1049, 1004, 1007, 1076, 1005, 35, 0, 0, 0, 0, 0, 0, 0, 1053, 1013, 1003, 1045, 1069, 1012, 980, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1070, 1028, 1041, 1086, 1043, 1008, 1053, 6, 2, 0, 0, 0, 0, 0, 0, 1000, 1000, 1041, 1014, 1043, 1016, 1042, 1059, 20, 0, 0, 0, 0, 0, 0, 0, 956, 1065, 1028, 1053, 996, 979, 1011, 965, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8163, 8162, 8229, 8216, 8376, 8235, 8057] avg=8192.0000 max=8376 max_rank=5 min=8057 min_rank=7 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=32884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1021, 1045, 1033, 1026, 997, 1046, 1006, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1046, 1080, 990, 1025, 975, 1048, 962, 20, 4, 0, 0, 0, 0, 0, 0, 1032, 996, 980, 1032, 985, 1013, 1010, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1037, 1021, 980, 1023, 1031, 1045, 1027, 35, 0, 0, 0, 0, 0, 0, 0, 1032, 1097, 990, 1037, 1003, 1046, 996, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1021, 996, 1054, 1011, 982, 978, 1002, 6, 2, 0, 0, 0, 0, 0, 0, 1029, 1005, 1068, 1003, 1038, 1073, 954, 1052, 20, 0, 0, 0, 0, 0, 0, 0, 1071, 1019, 1027, 1062, 1018, 1035, 1000, 1059, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8176, 8081, 8224, 8267, 8057, 8242, 8295] avg=8192.0000 max=8295 max_rank=7 min=8057 min_rank=5 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32861 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1009, 1014, 992, 1008, 1004, 1003, 1036, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1098, 1043, 1017, 1005, 1022, 1018, 986, 21, 4, 0, 0, 0, 0, 0, 0, 996, 1004, 975, 1002, 1001, 1015, 1003, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1008, 1037, 1047, 1000, 1015, 1067, 992, 37, 0, 0, 0, 0, 0, 0, 0, 1061, 997, 1036, 1022, 1031, 1066, 1016, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1032, 1001, 1005, 1015, 996, 1029, 1020, 6, 2, 0, 0, 0, 0, 0, 0, 1003, 1016, 1059, 1066, 1003, 1033, 1081, 1009, 21, 0, 0, 0, 0, 0, 0, 0, 987, 995, 1006, 1080, 1074, 1054, 1048, 990, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8078, 8262, 7983, 8266, 8279, 8139, 8291, 8238] avg=8192.0000 max=8291 max_rank=6 min=7983 min_rank=2 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32947 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1040, 1068, 1053, 1052, 1042, 1036, 1025, 995, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1026, 1010, 978, 1031, 1055, 1040, 1029, 21, 4, 0, 0, 0, 0, 0, 0, 991, 997, 1025, 1036, 1039, 1079, 987, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 961, 1031, 995, 1019, 990, 1044, 1066, 1011, 36, 0, 0, 0, 0, 0, 0, 0, 1093, 1018, 1020, 1032, 1010, 1006, 951, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1035, 968, 1035, 1021, 1012, 1050, 1046, 6, 2, 0, 0, 0, 0, 0, 0, 1043, 1008, 1054, 1027, 1005, 957, 1037, 980, 21, 0, 0, 0, 0, 0, 0, 0, 1039, 1020, 978, 1043, 1055, 987, 1022, 977, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8176, 8219, 8153, 8205, 8215, 8132, 8125] avg=8192.0000 max=8311 max_rank=0 min=8125 min_rank=7 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1048, 1006, 1032, 1053, 999, 1065, 1048, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 998, 1024, 1028, 1020, 1056, 1036, 1020, 20, 4, 0, 0, 0, 0, 0, 0, 1044, 1077, 1068, 1019, 986, 1082, 1026, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1012, 990, 996, 1020, 1029, 1015, 1030, 35, 0, 0, 0, 0, 0, 0, 0, 992, 1051, 982, 1035, 1006, 998, 1015, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 989, 1008, 1039, 1043, 1014, 1014, 963, 6, 2, 0, 0, 0, 0, 0, 0, 951, 998, 1083, 1028, 996, 1042, 995, 999, 21, 0, 0, 0, 0, 0, 0, 0, 1070, 1024, 1031, 997, 1041, 1034, 1033, 1019, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8238, 8332, 8138, 8104, 8092, 8113, 8253] avg=8192.0000 max=8332 max_rank=2 min=8092 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32562 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1014, 1020, 1047, 995, 988, 1030, 1035, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 984, 1020, 1055, 1073, 1054, 1004, 977, 24, 7, 0, 0, 0, 0, 0, 0, 1036, 1010, 1063, 1001, 1062, 1046, 1016, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1036, 1027, 1026, 1010, 1004, 993, 993, 36, 0, 0, 0, 0, 0, 0, 0, 1026, 1064, 932, 988, 1014, 1009, 1059, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1032, 1020, 1019, 1011, 1031, 1024, 966, 9, 9, 0, 0, 0, 0, 0, 0, 994, 1069, 999, 1010, 1024, 1026, 1044, 1045, 25, 0, 0, 0, 0, 0, 0, 0, 1037, 1019, 1033, 1029, 1065, 998, 1059, 1000, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8230, 8262, 8156, 8129, 8139, 8236, 8250] avg=8192.0000 max=8262 max_rank=2 min=8129 min_rank=4 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8064, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8448, 8320, 7936, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8064, 8448, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8448, 8448, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8320, 8064, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8448, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8448, 8320, 8192, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[983, 1011, 1046, 1069, 1035, 1036, 1036, 1031, 1008, 985, 1022, 1010, 1027, 1020, 1026, 1019, 1043, 983, 986, 1065, 1023, 1039, 1056, 1045, 1022, 984, 1035, 1018, 1079, 998, 1030, 1022, 1007, 984, 1029, 1029, 995, 1032, 1019, 1026, 985, 1086, 1006, 1021, 1053, 1029, 979, 1026, 1004, 1034, 1034, 1076, 988, 1017, 995, 1032, 1024, 1046, 1076, 1006, 1013, 1029, 1046, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1057, 1009, 1016, 1016, 1050, 1033, 1044, 979, 1003, 1065, 1079, 1058, 1049, 999, 1074, 1016, 1041, 997, 1050, 1023, 986, 1024, 1055, 1017, 1052, 989, 1034, 997, 1049, 1053, 978, 1017, 963, 1008, 1014, 984, 1010, 1009, 1000, 1046, 1016, 1057, 1044, 1036, 1057, 1061, 1018, 975, 1002, 999, 1001, 1026, 1037, 1018, 972, 1035, 1045, 1024, 974, 998, 1034, 1072, 1042, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[957, 1010, 1039, 1009, 1017, 1034, 1031, 1035, 1021, 1010, 1043, 1013, 1012, 988, 1057, 1019, 968, 1027, 1054, 1056, 1017, 1046, 1042, 1047, 1019, 1046, 1001, 1065, 1024, 995, 1014, 1016, 1050, 1008, 1023, 1064, 1028, 992, 1074, 1050, 1056, 1016, 1033, 1044, 968, 1032, 1026, 1018, 1040, 974, 1027, 1021, 997, 1051, 1019, 1009, 1011, 1027, 995, 1050, 1008, 1043, 1042, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[982, 1085, 963, 1007, 1018, 1008, 1050, 1005, 1059, 1014, 974, 1038, 1079, 1042, 1049, 1019, 1033, 1024, 1034, 1028, 1037, 987, 1019, 1024, 1027, 1039, 1049, 1040, 1030, 1006, 1026, 1049, 988, 1009, 1036, 1018, 1060, 982, 1003, 1058, 1051, 1008, 1027, 1018, 1022, 1065, 1001, 1024, 1033, 1045, 1036, 1046, 1047, 1068, 1064, 998, 992, 1026, 1001, 979, 1015, 1010, 993, 969] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1027, 1008, 1037, 1006, 1018, 1054, 1011, 1026, 971, 1053, 988, 1031, 1046, 1057, 1034, 1022, 1035, 1003, 1001, 1023, 987, 999, 1043, 1017, 968, 1034, 1050, 1030, 1024, 1010, 998, 1012, 1011, 1012, 1086, 1056, 1008, 1044, 1013, 991, 1063, 973, 1019, 1088, 1014, 1063, 1040, 1017, 1012, 1018, 1012, 1005, 1039, 1066, 1041, 1037, 1055, 986, 1038, 1029, 1029, 1050, 1025, 973] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1070, 1059, 1004, 968, 1029, 1068, 1020, 1003, 1075, 1004, 1039, 1062, 1023, 988, 1021, 1001, 1023, 1017, 989, 1044, 1013, 992, 1002, 1046, 1017, 1047, 1028, 1007, 1087, 1041, 1025, 1019, 994, 1039, 1041, 1049, 1002, 1049, 985, 1042, 1004, 1000, 1056, 1050, 976, 1034, 1061, 986, 977, 1023, 1011, 1035, 1028, 1054, 1009, 1011, 1019, 1042, 998, 1005, 1031, 1047, 1049, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1023, 1031, 998, 993, 1027, 1001, 984, 1060, 985, 1080, 1018, 1057, 991, 1025, 1072, 1041, 1063, 1044, 1041, 1041, 1019, 1025, 1021, 1022, 1027, 1011, 1040, 1001, 1060, 1015, 1063, 1046, 1042, 1064, 963, 998, 1001, 982, 994, 1075, 1014, 1050, 1003, 1002, 1003, 1047, 1086, 1065, 1010, 995, 990, 1039, 1056, 988, 1061, 996, 1020, 1042, 1019, 1035, 1014, 942, 1015, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1009, 972, 1054, 1033, 1003, 1035, 1031, 1003, 1030, 1052, 1022, 1064, 981, 1069, 1018, 1027, 1036, 1035, 1020, 1035, 1037, 1032, 1003, 1051, 1054, 964, 984, 1016, 1054, 1014, 1083, 1012, 1012, 1002, 1053, 1020, 1016, 1019, 1010, 1032, 1011, 1028, 1000, 1078, 1044, 1037, 1000, 1047, 953, 988, 1063, 959, 1027, 945, 1016, 1068, 996, 1051, 1011, 1029, 1054, 1041, 1050, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1057, 1009, 1016, 1016, 1050, 1033, 1044, 979, 12, 9, 0, 0, 0, 0, 0, 0, 1003, 1065, 1079, 1058, 1049, 999, 1035, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 997, 1050, 1011, 986, 1024, 1055, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 989, 1034, 997, 1036, 1053, 978, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1008, 1014, 984, 1010, 1009, 1000, 1046, 23, 6, 0, 0, 0, 0, 0, 0, 1016, 1057, 1044, 1036, 1057, 1026, 1018, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 999, 1001, 1026, 1037, 1018, 972, 1035, 33, 0, 0, 0, 0, 0, 0, 0, 1045, 1024, 974, 998, 1034, 1072, 1042, 1050, 12, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8304, 8181, 8156, 8063, 8229, 8123, 8255] avg=8192.0000 max=8304 max_rank=1 min=8063 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[957, 1010, 1039, 1009, 1017, 1034, 1031, 1035, 11, 8, 0, 0, 0, 0, 0, 0, 1021, 1010, 1043, 1013, 1012, 988, 1019, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1027, 1054, 1044, 1017, 1046, 1042, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1046, 1001, 1065, 1012, 995, 1014, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1008, 1023, 1064, 1028, 992, 1074, 1050, 22, 6, 0, 0, 0, 0, 0, 0, 1056, 1016, 1033, 1044, 968, 999, 1026, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 974, 1027, 1021, 997, 1051, 1019, 1009, 32, 0, 0, 0, 0, 0, 0, 0, 1011, 1027, 995, 1050, 1008, 1043, 1042, 1008, 12, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8125, 8245, 8168, 8317, 8160, 8170, 8200] avg=8192.0000 max=8317 max_rank=4 min=8125 min_rank=1 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[983, 1011, 1046, 1069, 1035, 1036, 1036, 1031, 11, 9, 0, 0, 0, 0, 0, 0, 1008, 985, 1022, 1010, 1027, 1020, 989, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 983, 986, 1053, 1023, 1039, 1056, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 984, 1035, 1018, 1066, 998, 1030, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 984, 1029, 1029, 995, 1032, 1019, 1026, 22, 6, 0, 0, 0, 0, 0, 0, 985, 1086, 1006, 1021, 1053, 996, 979, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1034, 1034, 1076, 988, 1017, 995, 1032, 31, 0, 0, 0, 0, 0, 0, 0, 1024, 1046, 1076, 1006, 1013, 1029, 1046, 1018, 12, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8080, 8228, 8175, 8149, 8152, 8211, 8274] avg=8192.0000 max=8274 max_rank=7 min=8080 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1027, 1008, 1037, 1006, 1018, 1054, 1011, 1026, 12, 8, 0, 0, 0, 0, 0, 0, 971, 1053, 988, 1031, 1046, 1057, 996, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1003, 1001, 1011, 987, 999, 1043, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1034, 1050, 1030, 1012, 1010, 998, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1012, 1086, 1056, 1008, 1044, 1013, 991, 23, 6, 0, 0, 0, 0, 0, 0, 1063, 973, 1019, 1088, 1014, 1028, 1040, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1018, 1012, 1005, 1039, 1066, 1041, 1037, 32, 0, 0, 0, 0, 0, 0, 0, 1055, 986, 1038, 1029, 1029, 1050, 1025, 973, 12, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8164, 8096, 8114, 8250, 8242, 8262, 8201] avg=8192.0000 max=8262 max_rank=6 min=8096 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32955 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[982, 1085, 963, 1007, 1018, 1008, 1050, 1005, 12, 8, 0, 0, 0, 0, 0, 0, 1059, 1014, 974, 1038, 1079, 1042, 1011, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1024, 1034, 1016, 1037, 987, 1019, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1039, 1049, 1040, 1018, 1006, 1026, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1009, 1036, 1018, 1060, 982, 1003, 1058, 23, 6, 0, 0, 0, 0, 0, 0, 1051, 1008, 1027, 1018, 1022, 1030, 1001, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1045, 1036, 1046, 1047, 1068, 1064, 998, 32, 0, 0, 0, 0, 0, 0, 0, 992, 1026, 1001, 979, 1015, 1010, 993, 969, 12, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8236, 8174, 8254, 8183, 8181, 8369, 8001] avg=8192.0000 max=8369 max_rank=6 min=8001 min_rank=7 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1023, 1031, 998, 993, 1027, 1001, 984, 1060, 12, 9, 0, 0, 0, 0, 0, 0, 985, 1080, 1018, 1057, 991, 1025, 1033, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1044, 1041, 1029, 1019, 1025, 1021, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1011, 1040, 1001, 1047, 1015, 1063, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1064, 963, 998, 1001, 982, 994, 1075, 23, 6, 0, 0, 0, 0, 0, 0, 1014, 1050, 1003, 1002, 1003, 1012, 1086, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 995, 990, 1039, 1056, 988, 1061, 996, 33, 0, 0, 0, 0, 0, 0, 0, 1020, 1042, 1019, 1035, 1014, 942, 1015, 1000, 12, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8230, 8264, 8250, 8148, 8235, 8168, 8103] avg=8192.0000 max=8264 max_rank=2 min=8103 min_rank=7 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32654 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1070, 1059, 1004, 968, 1029, 1068, 1020, 1003, 11, 9, 0, 0, 0, 0, 0, 0, 1075, 1004, 1039, 1062, 1023, 988, 985, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1017, 989, 1032, 1013, 992, 1002, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1047, 1028, 1007, 1074, 1041, 1025, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1039, 1041, 1049, 1002, 1049, 985, 1042, 22, 5, 0, 0, 0, 0, 0, 0, 1004, 1000, 1056, 1050, 976, 1001, 1061, 986, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1023, 1011, 1035, 1028, 1054, 1009, 1011, 31, 0, 0, 0, 0, 0, 0, 0, 1019, 1042, 998, 1005, 1031, 1047, 1049, 998, 12, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8177, 8114, 8258, 8228, 8134, 8179, 8205] avg=8192.0000 max=8258 max_rank=3 min=8114 min_rank=2 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1009, 972, 1054, 1033, 1003, 1035, 1031, 1003, 16, 13, 0, 0, 0, 0, 0, 0, 1030, 1052, 1022, 1064, 981, 1069, 975, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1035, 1020, 1020, 1037, 1032, 1003, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 964, 984, 1016, 1034, 1014, 1083, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1002, 1053, 1020, 1016, 1019, 1010, 1032, 26, 8, 0, 0, 0, 0, 0, 0, 1011, 1028, 1000, 1078, 1044, 995, 1000, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 953, 988, 1063, 959, 1027, 945, 1016, 1068, 35, 0, 0, 0, 0, 0, 0, 0, 996, 1051, 1011, 1029, 1054, 1041, 1050, 1043, 15, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8220, 8234, 8161, 8198, 8203, 8054, 8297] avg=8192.0000 max=8297 max_rank=7 min=8054 min_rank=6 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8448, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8448, 8320, 8192, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1001, 1030, 1009, 1016, 1005, 1053, 1034, 1010, 1035, 1095, 968, 1044, 1032, 1043, 1069, 1032, 1049, 1019, 990, 1000, 964, 1067, 998, 1048, 964, 1015, 1038, 1019, 1051, 1019, 1078, 1034, 1072, 1052, 1006, 1009, 1001, 1012, 979, 1035, 1036, 1003, 1005, 1060, 993, 1038, 988, 993, 1056, 1026, 1036, 1039, 1013, 1025, 1095, 997, 1070, 988, 991, 1004, 1015, 1011, 1059, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1010, 996, 1052, 1018, 983, 1025, 1053, 1027, 1043, 1023, 1073, 1009, 963, 1012, 977, 978, 1008, 1065, 1019, 1034, 1018, 1041, 994, 1078, 1029, 991, 1020, 1051, 1022, 1012, 1021, 1040, 1031, 1036, 993, 965, 1030, 1068, 1015, 981, 1064, 1033, 1049, 1022, 1023, 1035, 1021, 1083, 1033, 1034, 989, 998, 1031, 1037, 1089, 1041, 982, 1011, 1043, 1071, 1005, 992, 1021, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1008, 1063, 1034, 998, 1021, 1106, 1045, 1067, 1029, 1016, 1040, 1010, 999, 1025, 1014, 1000, 956, 1016, 996, 1027, 1060, 1028, 1021, 978, 1039, 1012, 999, 1046, 982, 1024, 1046, 980, 1032, 1028, 1032, 993, 1022, 1032, 1014, 1009, 1014, 1025, 1049, 1016, 1001, 1010, 1013, 1002, 1092, 1033, 1022, 1041, 1020, 1057, 1038, 1032, 1072, 1021, 1075, 1007, 982, 974, 1047, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1024, 1024, 1030, 1014, 957, 1034, 1041, 1005, 1065, 1086, 1016, 998, 1004, 1016, 1041, 1036, 1003, 995, 1023, 1043, 1064, 1009, 1001, 1071, 1034, 1093, 1015, 1093, 979, 1017, 1035, 1039, 1069, 1004, 1047, 1009, 1024, 1010, 1012, 1027, 982, 1037, 1009, 1009, 1009, 1009, 1076, 1049, 1017, 1012, 1027, 1019, 1041, 1022, 1022, 1021, 1034, 1023, 1055, 995, 1016, 972, 982, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1025, 1028, 1051, 1010, 1007, 1036, 1043, 1004, 1085, 1037, 1011, 1009, 993, 988, 1079, 1035, 1016, 1048, 985, 1041, 998, 1020, 1021, 1003, 1027, 1000, 1023, 1016, 1044, 1004, 1005, 1002, 1025, 975, 1044, 987, 1059, 1033, 983, 1023, 1058, 1032, 1055, 1042, 1008, 1048, 1013, 1022, 997, 987, 1006, 1038, 989, 1014, 1057, 1043, 1059, 1068, 1001, 1082, 1041, 1026, 998, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1014, 980, 971, 1019, 1020, 1017, 1033, 1063, 1024, 1017, 1015, 1002, 1025, 1052, 1015, 1019, 981, 1038, 1029, 1028, 1075, 1037, 1078, 1002, 1014, 985, 1018, 1068, 1046, 1012, 1045, 1056, 1011, 1058, 970, 1039, 1024, 999, 1035, 1037, 1005, 989, 1048, 1008, 969, 1024, 1009, 1003, 987, 1071, 960, 1056, 1025, 1007, 1037, 989, 1081, 1012, 1098, 1040, 1044, 1007, 1039, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1046, 1035, 1010, 961, 1024, 988, 1073, 985, 1078, 1049, 1026, 994, 990, 982, 1015, 1001, 1030, 980, 1002, 998, 1013, 991, 1026, 1026, 1030, 1017, 968, 1053, 1015, 1000, 1060, 1024, 1022, 1018, 1082, 1009, 1041, 1026, 1038, 1034, 1023, 1012, 1037, 1016, 1049, 1005, 1050, 1027, 1022, 1053, 1028, 1033, 1023, 1032, 1045, 1001, 1049, 1039, 1015, 1063, 1001, 1041, 1065, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[991, 1070, 1027, 1006, 1011, 999, 1021, 1036, 1043, 1020, 1004, 1052, 967, 1040, 1030, 1010, 1077, 1052, 1013, 1010, 1036, 1011, 993, 1094, 1012, 996, 1040, 1034, 987, 1012, 1024, 1035, 998, 1002, 1015, 1008, 1015, 1040, 1025, 1058, 1035, 994, 1065, 1042, 1025, 1059, 1008, 996, 1018, 1058, 1023, 1013, 1039, 1035, 1004, 1031, 997, 1007, 1036, 1063, 998, 955, 1082, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1010, 996, 1052, 1018, 983, 1025, 1053, 1027, 11, 6, 0, 0, 0, 0, 0, 0, 1043, 1023, 1073, 1009, 963, 1012, 977, 978, 4, 0, 0, 0, 0, 0, 0, 0, 1008, 1065, 1019, 1034, 1018, 1041, 994, 1078, 11, 0, 0, 0, 0, 0, 0, 0, 1029, 991, 1020, 1051, 1022, 1012, 1021, 1040, 3, 0, 0, 0, 0, 0, 0, 0, 1031, 1036, 993, 965, 1030, 1068, 1015, 981, 19, 0, 0, 0, 0, 0, 0, 0, 1064, 1033, 1049, 1022, 1023, 1035, 1021, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1034, 989, 998, 1031, 1037, 1059, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 958, 1011, 1043, 1071, 1005, 992, 1021, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8082, 8268, 8189, 8138, 8330, 8222, 8126] avg=8192.0000 max=8330 max_rank=5 min=8082 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1001, 1030, 1009, 1016, 1005, 1053, 1034, 1010, 11, 6, 0, 0, 0, 0, 0, 0, 1035, 1095, 968, 1044, 1032, 1043, 1069, 1032, 4, 0, 0, 0, 0, 0, 0, 0, 1049, 1019, 990, 1000, 964, 1067, 998, 1048, 12, 0, 0, 0, 0, 0, 0, 0, 964, 1015, 1038, 1019, 1051, 1019, 1078, 1034, 3, 0, 0, 0, 0, 0, 0, 0, 1072, 1052, 1006, 1009, 1001, 1012, 979, 1035, 20, 0, 0, 0, 0, 0, 0, 0, 1036, 1003, 1005, 1060, 993, 1038, 988, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1026, 1036, 1039, 1013, 1025, 1064, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 988, 991, 1004, 1015, 1011, 1059, 1000, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8322, 8147, 8221, 8186, 8116, 8256, 8113] avg=8192.0000 max=8322 max_rank=1 min=8113 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1014, 980, 971, 1019, 1020, 1017, 1033, 1063, 10, 6, 0, 0, 0, 0, 0, 0, 1024, 1017, 1015, 1002, 1025, 1052, 1015, 1019, 4, 0, 0, 0, 0, 0, 0, 0, 981, 1038, 1029, 1028, 1075, 1037, 1078, 1002, 13, 0, 0, 0, 0, 0, 0, 0, 1014, 985, 1018, 1068, 1046, 1012, 1045, 1056, 3, 0, 0, 0, 0, 0, 0, 0, 1011, 1058, 970, 1039, 1024, 999, 1035, 1037, 19, 0, 0, 0, 0, 0, 0, 0, 1005, 989, 1048, 1008, 969, 1024, 1009, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1071, 960, 1056, 1025, 1007, 1008, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1012, 1098, 1040, 1044, 1007, 1039, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8173, 8281, 8247, 8192, 8055, 8103, 8352] avg=8192.0000 max=8352 max_rank=7 min=8055 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1008, 1063, 1034, 998, 1021, 1106, 1045, 1067, 10, 6, 0, 0, 0, 0, 0, 0, 1029, 1016, 1040, 1010, 999, 1025, 1014, 1000, 4, 0, 0, 0, 0, 0, 0, 0, 956, 1016, 996, 1027, 1060, 1028, 1021, 978, 12, 0, 0, 0, 0, 0, 0, 0, 1039, 1012, 999, 1046, 982, 1024, 1046, 980, 3, 0, 0, 0, 0, 0, 0, 0, 1032, 1028, 1032, 993, 1022, 1032, 1014, 1009, 19, 0, 0, 0, 0, 0, 0, 0, 1014, 1025, 1049, 1016, 1001, 1010, 1013, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 1033, 1022, 1041, 1020, 1057, 1009, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1021, 1075, 1007, 982, 974, 1047, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8358, 8137, 8094, 8131, 8181, 8130, 8306, 8199] avg=8192.0000 max=8358 max_rank=0 min=8094 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1024, 1024, 1030, 1014, 957, 1034, 1041, 1005, 10, 6, 0, 0, 0, 0, 0, 0, 1065, 1086, 1016, 998, 1004, 1016, 1041, 1036, 4, 0, 0, 0, 0, 0, 0, 0, 1003, 995, 1023, 1043, 1064, 1009, 1001, 1071, 12, 0, 0, 0, 0, 0, 0, 0, 1034, 1093, 1015, 1093, 979, 1017, 1035, 1039, 3, 0, 0, 0, 0, 0, 0, 0, 1069, 1004, 1047, 1009, 1024, 1010, 1012, 1027, 18, 0, 0, 0, 0, 0, 0, 0, 982, 1037, 1009, 1009, 1009, 1009, 1076, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1012, 1027, 1019, 1041, 1022, 994, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1023, 1055, 995, 1016, 972, 982, 991, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8266, 8221, 8308, 8220, 8180, 8153, 8043] avg=8192.0000 max=8308 max_rank=3 min=8043 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1025, 1028, 1051, 1010, 1007, 1036, 1043, 1004, 10, 6, 0, 0, 0, 0, 0, 0, 1085, 1037, 1011, 1009, 993, 988, 1079, 1035, 4, 0, 0, 0, 0, 0, 0, 0, 1016, 1048, 985, 1041, 998, 1020, 1021, 1003, 12, 0, 0, 0, 0, 0, 0, 0, 1027, 1000, 1023, 1016, 1044, 1004, 1005, 1002, 3, 0, 0, 0, 0, 0, 0, 0, 1025, 975, 1044, 987, 1059, 1033, 983, 1023, 19, 0, 0, 0, 0, 0, 0, 0, 1058, 1032, 1055, 1042, 1008, 1048, 1013, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 997, 987, 1006, 1038, 989, 1014, 1028, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1068, 1001, 1082, 1041, 1026, 998, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8241, 8144, 8124, 8148, 8278, 8102, 8279] avg=8192.0000 max=8279 max_rank=7 min=8102 min_rank=6 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1046, 1035, 1010, 961, 1024, 988, 1073, 985, 10, 6, 0, 0, 0, 0, 0, 0, 1078, 1049, 1026, 994, 990, 982, 1015, 1001, 4, 0, 0, 0, 0, 0, 0, 0, 1030, 980, 1002, 998, 1013, 991, 1026, 1026, 12, 0, 0, 0, 0, 0, 0, 0, 1030, 1017, 968, 1053, 1015, 1000, 1060, 1024, 3, 0, 0, 0, 0, 0, 0, 0, 1022, 1018, 1082, 1009, 1041, 1026, 1038, 1034, 19, 0, 0, 0, 0, 0, 0, 0, 1023, 1012, 1037, 1016, 1049, 1005, 1050, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1053, 1028, 1033, 1023, 1032, 1016, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1039, 1015, 1063, 1001, 1041, 1065, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8139, 8078, 8170, 8289, 8219, 8208, 8295] avg=8192.0000 max=8295 max_rank=7 min=8078 min_rank=2 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=33011 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[991, 1070, 1027, 1006, 1011, 999, 1021, 1036, 14, 11, 0, 0, 0, 0, 0, 0, 1043, 1020, 1004, 1052, 967, 1040, 1030, 1010, 10, 0, 0, 0, 0, 0, 0, 0, 1077, 1052, 1013, 1010, 1036, 1011, 993, 1094, 17, 0, 0, 0, 0, 0, 0, 0, 1012, 996, 1040, 1034, 987, 1012, 1024, 1035, 6, 0, 0, 0, 0, 0, 0, 0, 998, 1002, 1015, 1008, 1015, 1040, 1025, 1058, 21, 0, 0, 0, 0, 0, 0, 0, 1035, 994, 1065, 1042, 1025, 1059, 1008, 996, 4, 0, 0, 0, 0, 0, 0, 0, 1018, 1058, 1023, 1013, 1039, 1035, 969, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 949, 1007, 1036, 1063, 998, 955, 1082, 1039, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8176, 8303, 8146, 8182, 8228, 8186, 8129] avg=8192.0000 max=8303 max_rank=2 min=8129 min_rank=7 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8064, 8064, 8320, 8448, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8448, 8192, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8448, 8192, 8192, 8064, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8448, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8064, 8320, 8320, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8192, 8320, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[995, 1030, 1049, 1009, 1037, 1053, 1051, 1048, 1011, 995, 1031, 1020, 1020, 997, 1074, 1039, 1007, 1025, 995, 1051, 1042, 1001, 1051, 1017, 1093, 1008, 1070, 1059, 960, 1026, 1030, 1028, 995, 973, 1006, 1029, 988, 1061, 1031, 979, 1066, 1023, 1021, 1033, 1049, 1027, 1045, 1008, 1000, 986, 958, 1012, 1013, 1025, 998, 1058, 1034, 1067, 1028, 1021, 1053, 1019, 969, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1014, 1020, 1005, 970, 951, 985, 1060, 1074, 990, 1009, 1023, 1067, 1015, 1035, 1001, 995, 1026, 1024, 1043, 1073, 1017, 993, 1049, 996, 1034, 1046, 1070, 1033, 985, 1034, 1055, 962, 1023, 1073, 1037, 1047, 1005, 1033, 1002, 1040, 1016, 1031, 1080, 1042, 1022, 1028, 991, 1033, 1036, 1008, 1025, 1010, 1014, 983, 1076, 1018, 998, 1034, 1062, 1060, 1002, 1039, 1023, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1015, 1036, 1025, 1011, 1025, 1059, 1000, 1020, 1063, 1025, 1007, 1023, 1056, 1034, 1082, 1031, 1045, 1001, 1019, 1072, 1045, 1058, 1012, 1007, 1020, 1019, 1007, 983, 986, 1018, 1015, 1037, 1085, 1028, 1016, 1046, 1017, 1051, 1094, 1008, 1017, 1042, 1064, 1008, 1006, 1037, 1003, 1041, 1044, 1015, 1024, 1018, 969, 1025, 978, 980, 1021, 977, 1013, 1034, 1014, 981, 1015, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1022, 988, 1038, 1007, 1040, 1032, 1022, 977, 1022, 1016, 977, 1000, 1041, 1006, 1022, 1042, 1035, 974, 1015, 1046, 1010, 1009, 1073, 1055, 1032, 1010, 1022, 1054, 1027, 1020, 984, 1019, 1021, 1021, 1020, 1015, 1014, 1026, 1037, 1049, 988, 1020, 1021, 1022, 1036, 1048, 1049, 1057, 1037, 1046, 1004, 1037, 1015, 1009, 1033, 1068, 1062, 993, 1028, 1001, 1059, 1007, 1013, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[989, 996, 1037, 1033, 959, 1006, 1027, 1030, 1002, 998, 1070, 1012, 1012, 1017, 1013, 1004, 1004, 957, 1003, 1044, 1004, 1021, 1079, 1010, 1020, 1050, 1029, 1085, 1007, 1068, 997, 1091, 1022, 1005, 999, 1030, 997, 1005, 1024, 1007, 1012, 1053, 1075, 1050, 1047, 1053, 996, 1017, 1017, 1047, 1051, 1075, 1090, 1020, 953, 1046, 1018, 1030, 955, 1041, 1075, 1034, 996, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1016, 1023, 1028, 1029, 1013, 1043, 1016, 1009, 1031, 1051, 1043, 970, 1002, 1084, 1009, 1049, 1007, 988, 1032, 1005, 1027, 1011, 1040, 1025, 1017, 1027, 1085, 1003, 1017, 1069, 1034, 1033, 1065, 983, 994, 1026, 1075, 1040, 1005, 955, 995, 1042, 1042, 1048, 991, 1001, 1014, 1005, 1052, 1058, 1041, 966, 1034, 1050, 983, 1008, 1045, 1006, 1061, 1020, 972, 1057, 1032, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[986, 1024, 975, 1019, 1046, 999, 1017, 1016, 1036, 1043, 1005, 1021, 1012, 1002, 1041, 1018, 1046, 1027, 1048, 1062, 982, 1032, 1047, 1042, 1027, 997, 1057, 981, 1069, 1028, 1012, 983, 1045, 1034, 1006, 1051, 1024, 1006, 1009, 1002, 1025, 1063, 1032, 1030, 1039, 1023, 1050, 1030, 1086, 1029, 1022, 1017, 1068, 979, 998, 1059, 1049, 968, 1022, 1044, 1026, 994, 1001, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[982, 1014, 1068, 1023, 1038, 1021, 999, 1045, 1024, 980, 1010, 1018, 1015, 1036, 1053, 990, 1022, 1027, 1048, 1006, 1068, 1011, 1028, 1004, 1064, 1025, 979, 1043, 1004, 1003, 1024, 1016, 1006, 1053, 995, 1058, 1052, 1058, 1045, 1066, 1028, 1074, 1084, 1049, 1019, 987, 1013, 1048, 982, 1057, 1046, 1022, 1001, 994, 989, 994, 1029, 964, 1038, 1032, 961, 1007, 1074, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1014, 1020, 1005, 970, 951, 985, 1060, 1074, 43, 0, 0, 0, 0, 0, 0, 0, 990, 1009, 1023, 1067, 1015, 1035, 1001, 995, 9, 2, 0, 0, 0, 0, 0, 0, 1026, 1024, 1043, 1073, 1017, 993, 1037, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1046, 1052, 1033, 985, 1034, 1055, 962, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1073, 1037, 1038, 1005, 1033, 1002, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1031, 1021, 1042, 1022, 1028, 991, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1008, 1025, 1010, 1014, 983, 1076, 1018, 11, 10, 0, 0, 0, 0, 0, 0, 998, 1034, 1062, 1060, 1002, 1039, 1023, 991, 16, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8146, 8209, 8201, 8251, 8184, 8191, 8232] avg=8192.0000 max=8251 max_rank=4 min=8122 min_rank=0 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[986, 1024, 975, 1019, 1046, 999, 1017, 1016, 41, 0, 0, 0, 0, 0, 0, 0, 1036, 1043, 1005, 1021, 1012, 1002, 1041, 1018, 9, 2, 0, 0, 0, 0, 0, 0, 1046, 1027, 1048, 1062, 982, 1032, 1035, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 997, 1039, 981, 1069, 1028, 1012, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1034, 1006, 1042, 1024, 1006, 1009, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1063, 975, 1030, 1039, 1023, 1050, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 1029, 1022, 1017, 1068, 979, 998, 1059, 11, 10, 0, 0, 0, 0, 0, 0, 1049, 968, 1022, 1044, 1026, 994, 1001, 1005, 16, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8189, 8274, 8136, 8168, 8235, 8279, 8132] avg=8192.0000 max=8279 max_rank=6 min=8123 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1022, 988, 1038, 1007, 1040, 1032, 1022, 977, 41, 0, 0, 0, 0, 0, 0, 0, 1022, 1016, 977, 1000, 1041, 1006, 1022, 1042, 9, 2, 0, 0, 0, 0, 0, 0, 1035, 974, 1015, 1046, 1010, 1009, 1060, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1010, 1004, 1054, 1027, 1020, 984, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1021, 1020, 1006, 1014, 1026, 1037, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1020, 965, 1022, 1036, 1048, 1049, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1046, 1004, 1037, 1015, 1009, 1033, 1068, 11, 11, 0, 0, 0, 0, 0, 0, 1062, 993, 1028, 1001, 1059, 1007, 1013, 1043, 15, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8137, 8204, 8150, 8194, 8185, 8271, 8228] avg=8192.0000 max=8271 max_rank=6 min=8137 min_rank=1 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[989, 996, 1037, 1033, 959, 1006, 1027, 1030, 43, 0, 0, 0, 0, 0, 0, 0, 1002, 998, 1070, 1012, 1012, 1017, 1013, 1004, 9, 2, 0, 0, 0, 0, 0, 0, 1004, 957, 1003, 1044, 1004, 1021, 1066, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1050, 1011, 1085, 1007, 1068, 997, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1005, 999, 1021, 997, 1005, 1024, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1053, 1016, 1050, 1047, 1053, 996, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1047, 1051, 1075, 1090, 1020, 953, 1046, 11, 11, 0, 0, 0, 0, 0, 0, 1018, 1030, 955, 1041, 1075, 1034, 996, 1022, 16, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8139, 8109, 8329, 8080, 8244, 8321, 8194] avg=8192.0000 max=8329 max_rank=3 min=8080 min_rank=4 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[995, 1030, 1049, 1009, 1037, 1053, 1051, 1048, 41, 0, 0, 0, 0, 0, 0, 0, 1011, 995, 1031, 1020, 1020, 997, 1074, 1039, 9, 2, 0, 0, 0, 0, 0, 0, 1007, 1025, 995, 1051, 1042, 1001, 1039, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 1008, 1052, 1059, 960, 1026, 1030, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 995, 973, 1006, 1020, 988, 1061, 1031, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1023, 965, 1033, 1049, 1027, 1045, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 986, 958, 1012, 1013, 1025, 998, 1058, 11, 10, 0, 0, 0, 0, 0, 0, 1034, 1067, 1028, 1021, 1053, 1019, 969, 1039, 15, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8313, 8198, 8177, 8256, 8053, 8216, 8071, 8252] avg=8192.0000 max=8313 max_rank=0 min=8053 min_rank=4 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32592 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1015, 1036, 1025, 1011, 1025, 1059, 1000, 1020, 43, 0, 0, 0, 0, 0, 0, 0, 1063, 1025, 1007, 1023, 1056, 1034, 1082, 1031, 9, 2, 0, 0, 0, 0, 0, 0, 1045, 1001, 1019, 1072, 1045, 1058, 1000, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1019, 989, 983, 986, 1018, 1015, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 1028, 1016, 1037, 1017, 1051, 1094, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1042, 1005, 1008, 1006, 1037, 1003, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1015, 1024, 1018, 969, 1025, 978, 980, 11, 10, 0, 0, 0, 0, 0, 0, 1021, 977, 1013, 1034, 1014, 981, 1015, 1009, 16, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8332, 8247, 8067, 8336, 8159, 8074, 8087] avg=8192.0000 max=8336 max_rank=4 min=8067 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1016, 1023, 1028, 1029, 1013, 1043, 1016, 1009, 42, 0, 0, 0, 0, 0, 0, 0, 1031, 1051, 1043, 970, 1002, 1084, 1009, 1049, 9, 2, 0, 0, 0, 0, 0, 0, 1007, 988, 1032, 1005, 1027, 1011, 1028, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1027, 1066, 1003, 1017, 1069, 1034, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 983, 994, 1017, 1075, 1040, 1005, 955, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1042, 984, 1048, 991, 1001, 1014, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1058, 1041, 966, 1034, 1050, 983, 1008, 12, 10, 0, 0, 0, 0, 0, 0, 1045, 1006, 1061, 1020, 972, 1057, 1032, 1034, 16, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8250, 8123, 8266, 8134, 8080, 8214, 8250] avg=8192.0000 max=8266 max_rank=3 min=8080 min_rank=5 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[982, 1014, 1068, 1023, 1038, 1021, 999, 1045, 48, 0, 0, 0, 0, 0, 0, 0, 1024, 980, 1010, 1018, 1015, 1036, 1053, 990, 13, 6, 0, 0, 0, 0, 0, 0, 1022, 1027, 1048, 1006, 1068, 1011, 1007, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1025, 952, 1043, 1004, 1003, 1024, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1053, 995, 1045, 1052, 1058, 1045, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1074, 1015, 1049, 1019, 987, 1013, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1057, 1046, 1022, 1001, 994, 989, 994, 15, 15, 0, 0, 0, 0, 0, 0, 1029, 964, 1038, 1032, 961, 1007, 1074, 1023, 21, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8145, 8193, 8131, 8320, 8233, 8115, 8161] avg=8192.0000 max=8320 max_rank=4 min=8115 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8064, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 8064, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8448, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8320, 8192, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8320, 8192, 8192, 8320, 8192, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1066, 984, 1061, 1002, 991, 1031, 1002, 1057, 1059, 1066, 1016, 1036, 1062, 1011, 996, 970, 995, 1012, 980, 1010, 1030, 1014, 1007, 1049, 1053, 1033, 1021, 1025, 1033, 1038, 1019, 1075, 1032, 988, 1008, 1015, 1017, 1024, 987, 1032, 1054, 1011, 1032, 1009, 1076, 988, 1024, 994, 1051, 995, 1050, 1009, 992, 1021, 1054, 1038, 1006, 1008, 1050, 1003, 1019, 1061, 1028, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1043, 988, 1036, 997, 1073, 1037, 975, 1003, 1017, 1027, 1028, 984, 1033, 1036, 1035, 1029, 1024, 1067, 974, 1042, 1050, 1034, 993, 997, 995, 1001, 1022, 1041, 1090, 1062, 1036, 1031, 1015, 1036, 1049, 1057, 1083, 955, 994, 979, 1021, 1025, 1003, 1031, 1013, 1056, 997, 984, 1031, 1009, 1040, 1001, 975, 1000, 1022, 1062, 1052, 1029, 1025, 1047, 1004, 1040, 1048, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1039, 1027, 1009, 994, 1035, 1044, 1044, 1051, 1067, 1043, 988, 1010, 1052, 1013, 1037, 1042, 1004, 1018, 1016, 1069, 1027, 1040, 1007, 955, 1045, 1046, 1053, 1055, 1015, 1006, 1035, 1076, 1017, 1021, 1077, 1028, 951, 1003, 959, 1031, 1004, 1043, 1017, 1063, 1051, 1048, 1033, 1007, 987, 993, 1023, 1017, 1046, 1066, 996, 978, 1032, 1005, 1000, 968, 1055, 1026, 1038, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[976, 1052, 1028, 1075, 1014, 1005, 1018, 1012, 1009, 1046, 1025, 1041, 1003, 1002, 1045, 1028, 976, 1044, 991, 1012, 1009, 995, 1018, 1006, 1024, 1033, 1025, 1037, 1016, 1051, 1055, 1036, 1012, 1006, 1001, 1013, 1011, 1027, 998, 1024, 1028, 1064, 999, 1057, 1059, 1047, 1011, 1085, 998, 1025, 1028, 1013, 1072, 999, 1043, 997, 991, 1023, 1040, 999, 1075, 1026, 1025, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1037, 1069, 1051, 1036, 985, 1032, 1008, 1015, 997, 1032, 1013, 984, 988, 1043, 1029, 1061, 1024, 1043, 952, 993, 1018, 1039, 1037, 1049, 1018, 1053, 1014, 1035, 1046, 998, 1003, 1000, 982, 1014, 1026, 1043, 1047, 1016, 1004, 996, 1034, 1027, 1063, 990, 1031, 1020, 1006, 1044, 1014, 1056, 1042, 993, 1014, 1023, 1050, 1022, 1041, 1002, 1055, 1048, 1020, 1018, 1062, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1065, 1004, 1021, 1022, 1037, 1023, 1005, 1073, 1007, 1005, 996, 940, 955, 1017, 1035, 1034, 1063, 1034, 965, 1046, 1033, 1006, 1027, 1025, 1050, 1012, 1018, 1050, 1006, 1055, 1001, 1032, 1037, 1018, 1033, 990, 1020, 1028, 992, 1004, 1015, 1023, 1016, 1023, 1043, 1015, 1059, 1052, 1002, 1049, 1023, 1046, 1047, 1028, 1046, 1032, 1052, 986, 1035, 1035, 1053, 993, 1069, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1036, 1021, 1012, 1050, 1000, 998, 1056, 983, 1017, 1034, 1039, 1084, 1027, 1007, 1064, 1006, 993, 1070, 1068, 974, 1054, 1034, 988, 1008, 1004, 1034, 992, 1017, 1046, 1048, 974, 1008, 1039, 1031, 1011, 1044, 1009, 998, 1070, 1033, 988, 1016, 1059, 983, 1029, 1050, 1000, 1035, 1046, 1006, 1056, 1048, 1058, 1042, 1021, 961, 1044, 1009, 1022, 1009, 1024, 1019, 1001, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1049, 1029, 961, 1016, 1050, 995, 993, 1010, 1006, 1053, 1058, 1053, 987, 1000, 1044, 1036, 997, 1043, 1023, 1013, 1095, 1015, 1006, 1039, 1033, 1025, 1010, 985, 1044, 1024, 1057, 1008, 1004, 1035, 1034, 1063, 980, 1015, 1019, 1036, 1029, 1012, 1031, 1000, 1020, 1050, 1033, 1011, 1048, 1081, 1014, 1021, 1078, 1026, 1023, 1009, 1058, 998, 963, 1065, 1020, 1012, 993, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1043, 988, 1036, 997, 1073, 1037, 975, 1003, 3, 0, 0, 0, 0, 0, 0, 0, 1017, 1027, 1028, 984, 1033, 1036, 1035, 1029, 3, 0, 0, 0, 0, 0, 0, 0, 1024, 1067, 974, 1042, 1050, 1034, 993, 997, 16, 14, 5, 0, 0, 0, 0, 0, 995, 1001, 1022, 1041, 1045, 1062, 1036, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1036, 1049, 1057, 1083, 955, 994, 979, 45, 8, 0, 0, 0, 0, 0, 0, 1021, 1025, 1003, 1031, 989, 1056, 997, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1009, 1040, 1001, 961, 1000, 1022, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1029, 1025, 1047, 1004, 1040, 1048, 1053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8192, 8216, 8233, 8221, 8106, 8126, 8287] avg=8192.0000 max=8287 max_rank=7 min=8106 min_rank=5 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1039, 1027, 1009, 994, 1035, 1044, 1044, 1051, 3, 0, 0, 0, 0, 0, 0, 0, 1067, 1043, 988, 1010, 1052, 1013, 1037, 1042, 3, 0, 0, 0, 0, 0, 0, 0, 1004, 1018, 1016, 1069, 1027, 1040, 1007, 955, 17, 15, 4, 0, 0, 0, 0, 0, 1045, 1046, 1053, 1055, 973, 1006, 1035, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1021, 1077, 1028, 951, 1003, 959, 1031, 42, 8, 0, 0, 0, 0, 0, 0, 1004, 1043, 1017, 1063, 1026, 1048, 1033, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 987, 993, 1023, 1017, 1031, 1066, 996, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1005, 1000, 968, 1055, 1026, 1038, 991, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8255, 8172, 8289, 8137, 8241, 8091, 8105] avg=8192.0000 max=8289 max_rank=3 min=8091 min_rank=6 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32574 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1066, 984, 1061, 1002, 991, 1031, 1002, 1057, 3, 0, 0, 0, 0, 0, 0, 0, 1059, 1066, 1016, 1036, 1062, 1011, 996, 970, 3, 0, 0, 0, 0, 0, 0, 0, 995, 1012, 980, 1010, 1030, 1014, 1007, 1049, 17, 14, 4, 0, 0, 0, 0, 0, 1053, 1033, 1021, 1025, 990, 1038, 1019, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 988, 1008, 1015, 1017, 1024, 987, 1032, 43, 8, 0, 0, 0, 0, 0, 0, 1054, 1011, 1032, 1009, 1051, 988, 1024, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 995, 1050, 1009, 978, 1021, 1054, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1008, 1050, 1003, 1019, 1061, 1028, 1056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8219, 8132, 8254, 8154, 8163, 8196, 8221] avg=8192.0000 max=8254 max_rank=3 min=8132 min_rank=2 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1037, 1069, 1051, 1036, 985, 1032, 1008, 1015, 3, 0, 0, 0, 0, 0, 0, 0, 997, 1032, 1013, 984, 988, 1043, 1029, 1061, 3, 0, 0, 0, 0, 0, 0, 0, 1024, 1043, 952, 993, 1018, 1039, 1037, 1049, 16, 14, 5, 0, 0, 0, 0, 0, 1018, 1053, 1014, 1035, 1003, 998, 1003, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1014, 1026, 1043, 1047, 1016, 1004, 996, 43, 8, 0, 0, 0, 0, 0, 0, 1034, 1027, 1063, 990, 1007, 1020, 1006, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1056, 1042, 993, 1000, 1023, 1050, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1002, 1055, 1048, 1020, 1018, 1062, 1031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8150, 8190, 8124, 8179, 8191, 8200, 8266] avg=8192.0000 max=8266 max_rank=7 min=8124 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1065, 1004, 1021, 1022, 1037, 1023, 1005, 1073, 3, 0, 0, 0, 0, 0, 0, 0, 1007, 1005, 996, 940, 955, 1017, 1035, 1034, 3, 0, 0, 0, 0, 0, 0, 0, 1063, 1034, 965, 1046, 1033, 1006, 1027, 1025, 17, 15, 5, 0, 0, 0, 0, 0, 1050, 1012, 1018, 1050, 964, 1055, 1001, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1018, 1033, 990, 1020, 1028, 992, 1004, 42, 8, 0, 0, 0, 0, 0, 0, 1015, 1023, 1016, 1023, 1018, 1015, 1059, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1049, 1023, 1046, 1032, 1028, 1046, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 986, 1035, 1035, 1053, 993, 1069, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 7992, 8236, 8182, 8172, 8221, 8258, 8222] avg=8192.0000 max=8258 max_rank=6 min=7992 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[976, 1052, 1028, 1075, 1014, 1005, 1018, 1012, 2, 0, 0, 0, 0, 0, 0, 0, 1009, 1046, 1025, 1041, 1003, 1002, 1045, 1028, 3, 0, 0, 0, 0, 0, 0, 0, 976, 1044, 991, 1012, 1009, 995, 1018, 1006, 17, 15, 4, 0, 0, 0, 0, 0, 1024, 1033, 1025, 1037, 974, 1051, 1055, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1006, 1001, 1013, 1011, 1027, 998, 1024, 42, 8, 0, 0, 0, 0, 0, 0, 1028, 1064, 999, 1057, 1034, 1047, 1011, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1025, 1028, 1013, 1057, 999, 1043, 997, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1023, 1040, 999, 1075, 1026, 1025, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8202, 8087, 8235, 8142, 8325, 8160, 8203] avg=8192.0000 max=8325 max_rank=5 min=8087 min_rank=2 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1049, 1029, 961, 1016, 1050, 995, 993, 1010, 5, 0, 0, 0, 0, 0, 0, 0, 1006, 1053, 1058, 1053, 987, 1000, 1044, 1036, 8, 0, 0, 0, 0, 0, 0, 0, 997, 1043, 1023, 1013, 1095, 1015, 1006, 1039, 21, 18, 8, 0, 0, 0, 0, 0, 1033, 1025, 1010, 985, 997, 1024, 1057, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1035, 1034, 1063, 980, 1015, 1019, 1036, 47, 12, 0, 0, 0, 0, 0, 0, 1029, 1012, 1031, 1000, 987, 1050, 1033, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1081, 1014, 1021, 1060, 1026, 1023, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 998, 963, 1065, 1020, 1012, 993, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8245, 8278, 8139, 8245, 8153, 8282, 8086] avg=8192.0000 max=8282 max_rank=6 min=8086 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1036, 1021, 1012, 1050, 1000, 998, 1056, 983, 3, 0, 0, 0, 0, 0, 0, 0, 1017, 1034, 1039, 1084, 1027, 1007, 1064, 1006, 3, 0, 0, 0, 0, 0, 0, 0, 993, 1070, 1068, 974, 1054, 1034, 988, 1008, 16, 15, 5, 0, 0, 0, 0, 0, 1004, 1034, 992, 1017, 1003, 1048, 974, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1031, 1011, 1044, 1009, 998, 1070, 1033, 43, 8, 0, 0, 0, 0, 0, 0, 988, 1016, 1059, 983, 1005, 1050, 1000, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1006, 1056, 1048, 1043, 1042, 1021, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1009, 1022, 1009, 1024, 1019, 1001, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8281, 8225, 8080, 8286, 8136, 8223, 8146] avg=8192.0000 max=8286 max_rank=4 min=8080 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8064, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8064, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8448, 8064, 8192, 8320, 8192, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1037, 1019, 1051, 1030, 1062, 1055, 1014, 1015, 1000, 1049, 1072, 1079, 1034, 1003, 1035, 1038, 1038, 1060, 976, 1025, 1023, 1040, 1013, 1030, 1043, 1023, 990, 1022, 1014, 1020, 1002, 1026, 1048, 1001, 1014, 998, 1077, 970, 1074, 1015, 977, 1038, 1055, 1017, 1079, 990, 1021, 999, 1028, 1007, 1074, 964, 1018, 988, 1003, 999, 987, 1020, 965, 1023, 1053, 1048, 1011, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[982, 1035, 1047, 1045, 1009, 1008, 1041, 1008, 1029, 1051, 1063, 979, 1031, 1051, 948, 1039, 984, 1062, 1029, 1001, 1008, 1016, 1055, 1040, 1025, 1011, 1026, 1043, 981, 1075, 1030, 949, 1038, 1031, 1051, 1031, 999, 1037, 1023, 1011, 997, 1062, 1043, 1041, 996, 1026, 1030, 1010, 1000, 1063, 1063, 1016, 1060, 1037, 1020, 967, 1039, 1019, 1050, 1010, 1000, 972, 1043, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1025, 978, 1014, 979, 1000, 1004, 1027, 1034, 1042, 1016, 1071, 1005, 983, 1048, 993, 1069, 1033, 1047, 1049, 1076, 987, 1024, 1069, 1001, 1023, 1057, 1055, 1030, 1011, 1057, 1029, 1011, 1013, 1024, 1061, 980, 988, 1021, 1019, 1054, 1031, 986, 1061, 965, 1044, 996, 1037, 1018, 1015, 977, 1075, 1010, 1039, 1029, 1090, 1042, 984, 974, 986, 1037, 1017, 1012, 1058, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1030, 1030, 1010, 1028, 1031, 1041, 1033, 1020, 983, 1031, 967, 1072, 1009, 996, 1059, 1056, 1040, 1023, 997, 1022, 1033, 1041, 1037, 1022, 956, 999, 1009, 1085, 1070, 1012, 1055, 1002, 1041, 960, 1037, 1021, 1021, 1077, 1037, 1001, 1016, 1017, 935, 992, 1068, 1033, 1014, 977, 996, 1033, 1031, 1026, 992, 1072, 1056, 1029, 965, 1067, 1050, 1017, 1068, 1047, 1007, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1016, 1032, 1021, 1016, 1016, 1049, 1011, 976, 997, 1066, 1051, 1046, 1026, 1039, 1071, 1020, 994, 1083, 1003, 1055, 965, 1017, 1049, 1033, 1052, 1045, 1034, 1038, 966, 1034, 1038, 1045, 1055, 999, 1033, 1011, 1064, 987, 996, 1003, 1018, 984, 1013, 1033, 1020, 1039, 1000, 1065, 1011, 1038, 1054, 1038, 1019, 1030, 924, 1026, 1060, 1005, 1006, 1005, 949, 1053, 1043, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1019, 989, 1050, 1009, 1005, 1014, 1044, 1053, 1037, 1043, 1034, 1043, 1043, 1037, 1001, 1041, 998, 1021, 1055, 996, 1008, 1029, 1031, 1017, 1036, 1026, 1046, 1030, 993, 1040, 1025, 1028, 1037, 1021, 1062, 1015, 1050, 979, 1049, 1043, 991, 989, 1000, 1033, 995, 981, 1049, 1085, 1070, 1040, 1058, 1008, 1021, 1005, 1021, 986, 1018, 1059, 989, 1030, 980, 1011, 1004, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1025, 998, 1011, 1071, 1037, 1021, 1028, 1058, 1043, 1027, 1055, 1022, 1019, 1029, 1022, 1007, 1028, 1044, 970, 1059, 1006, 1008, 1065, 1028, 974, 1040, 1047, 1016, 1016, 994, 1002, 1073, 992, 1023, 1065, 1038, 989, 1042, 1057, 1013, 981, 995, 1028, 999, 1007, 1012, 973, 1041, 1023, 1023, 1015, 1040, 1048, 1030, 1038, 1062, 1010, 1013, 981, 1047, 1030, 1031, 1077, 970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1041, 1023, 991, 1009, 1061, 1047, 1013, 1000, 1024, 1003, 1028, 976, 1045, 979, 1029, 1072, 1060, 1020, 978, 1060, 962, 975, 962, 1039, 1038, 994, 1051, 1029, 1025, 1042, 1012, 1020, 1037, 1030, 1061, 1016, 1010, 1069, 1025, 1007, 1016, 1023, 1011, 1017, 1024, 1052, 1010, 1004, 1002, 1033, 1036, 1016, 1060, 1055, 1026, 1004, 1003, 983, 1052, 1061, 1016, 1113, 1021, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[982, 1035, 1047, 1045, 1009, 1008, 1041, 1008, 4, 0, 0, 0, 0, 0, 0, 0, 1029, 1051, 1063, 979, 1031, 1051, 948, 997, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1062, 1029, 1001, 1008, 1016, 1055, 1040, 2, 0, 0, 0, 0, 0, 0, 0, 1025, 1011, 1026, 1037, 981, 1075, 1030, 949, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1031, 1037, 1031, 999, 1037, 1023, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1062, 1043, 1041, 996, 1026, 1030, 1010, 42, 17, 0, 0, 0, 0, 0, 0, 1000, 1063, 1046, 1016, 1060, 1037, 1020, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1019, 1050, 1010, 1000, 972, 1043, 1050, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8149, 8197, 8134, 8207, 8264, 8209, 8197] avg=8192.0000 max=8264 max_rank=5 min=8134 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1037, 1019, 1051, 1030, 1062, 1055, 1014, 1015, 4, 0, 0, 0, 0, 0, 0, 0, 1000, 1049, 1072, 1079, 1034, 1003, 1035, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1060, 976, 1025, 1023, 1040, 1013, 1030, 2, 0, 0, 0, 0, 0, 0, 0, 1043, 1023, 990, 1016, 1014, 1020, 1002, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1001, 1001, 998, 1077, 970, 1074, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1038, 1055, 1017, 1079, 990, 1021, 999, 42, 17, 0, 0, 0, 0, 0, 0, 1028, 1007, 1057, 964, 1018, 988, 1003, 999, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1020, 965, 1023, 1053, 1048, 1011, 1037, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8268, 8207, 8134, 8184, 8235, 8064, 8157] avg=8192.0000 max=8287 max_rank=0 min=8064 min_rank=6 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 978, 1014, 979, 1000, 1004, 1027, 1034, 4, 0, 0, 0, 0, 0, 0, 0, 1042, 1016, 1071, 1005, 983, 1048, 993, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1047, 1049, 1076, 987, 1024, 1069, 1001, 2, 0, 0, 0, 0, 0, 0, 0, 1023, 1057, 1055, 1024, 1011, 1057, 1029, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1024, 1047, 980, 988, 1021, 1019, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 986, 1061, 965, 1044, 996, 1037, 1018, 43, 17, 0, 0, 0, 0, 0, 0, 1015, 977, 1058, 1010, 1039, 1029, 1090, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 984, 974, 986, 1037, 1017, 1012, 1058, 1046, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8065, 8184, 8288, 8267, 8146, 8198, 8260, 8128] avg=8192.0000 max=8288 max_rank=2 min=8065 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1030, 1030, 1010, 1028, 1031, 1041, 1033, 1020, 4, 0, 0, 0, 0, 0, 0, 0, 983, 1031, 967, 1072, 1009, 996, 1059, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1023, 997, 1022, 1033, 1041, 1037, 1022, 2, 0, 0, 0, 0, 0, 0, 0, 956, 999, 1009, 1079, 1070, 1012, 1055, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 960, 1024, 1021, 1021, 1077, 1037, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1017, 935, 992, 1068, 1033, 1014, 977, 43, 16, 0, 0, 0, 0, 0, 0, 996, 1033, 1015, 1026, 992, 1072, 1056, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 965, 1067, 1050, 1017, 1068, 1047, 1007, 1034, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8130, 8217, 8182, 8182, 8111, 8219, 8268] avg=8192.0000 max=8268 max_rank=7 min=8111 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1016, 1032, 1021, 1016, 1016, 1049, 1011, 976, 4, 0, 0, 0, 0, 0, 0, 0, 997, 1066, 1051, 1046, 1026, 1039, 1071, 979, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1083, 1003, 1055, 965, 1017, 1049, 1033, 2, 0, 0, 0, 0, 0, 0, 0, 1052, 1045, 1034, 1032, 966, 1034, 1038, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 999, 1020, 1011, 1064, 987, 996, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 984, 1013, 1033, 1020, 1039, 1000, 1065, 41, 17, 0, 0, 0, 0, 0, 0, 1011, 1038, 1037, 1038, 1019, 1030, 924, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1005, 1006, 1005, 949, 1053, 1043, 1051, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8275, 8201, 8246, 8135, 8230, 8123, 8185] avg=8192.0000 max=8275 max_rank=1 min=8123 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1019, 989, 1050, 1009, 1005, 1014, 1044, 1053, 4, 0, 0, 0, 0, 0, 0, 0, 1037, 1043, 1034, 1043, 1043, 1037, 1001, 999, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1021, 1055, 996, 1008, 1029, 1031, 1017, 2, 0, 0, 0, 0, 0, 0, 0, 1036, 1026, 1046, 1024, 993, 1040, 1025, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1021, 1048, 1015, 1050, 979, 1049, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 991, 989, 1000, 1033, 995, 981, 1049, 1085, 42, 17, 0, 0, 0, 0, 0, 0, 1070, 1040, 1041, 1008, 1021, 1005, 1021, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1059, 989, 1030, 980, 1011, 1004, 1016, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8237, 8157, 8218, 8242, 8182, 8192, 8121] avg=8192.0000 max=8242 max_rank=4 min=8121 min_rank=7 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1041, 1023, 991, 1009, 1061, 1047, 1013, 1000, 4, 0, 0, 0, 0, 0, 0, 0, 1024, 1003, 1028, 976, 1045, 979, 1029, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1020, 978, 1060, 962, 975, 962, 1039, 2, 0, 0, 0, 0, 0, 0, 0, 1038, 994, 1051, 1023, 1025, 1042, 1012, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1030, 1047, 1016, 1010, 1069, 1025, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1023, 1011, 1017, 1024, 1052, 1010, 1004, 43, 16, 0, 0, 0, 0, 0, 0, 1002, 1033, 1020, 1016, 1060, 1055, 1026, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 983, 1052, 1061, 1016, 1113, 1021, 1035, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8113, 8058, 8205, 8241, 8216, 8216, 8298] avg=8192.0000 max=8298 max_rank=7 min=8058 min_rank=2 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 998, 1011, 1071, 1037, 1021, 1028, 1058, 9, 3, 0, 0, 0, 0, 0, 0, 1043, 1027, 1055, 1022, 1019, 1029, 1022, 963, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1044, 970, 1059, 1006, 1008, 1065, 1028, 3, 0, 0, 0, 0, 0, 0, 0, 974, 1040, 1047, 1004, 1016, 994, 1002, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1023, 1045, 1038, 989, 1042, 1057, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 981, 995, 1028, 999, 1007, 1012, 973, 1041, 44, 20, 0, 0, 0, 0, 0, 0, 1023, 1023, 989, 1040, 1048, 1030, 1038, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1013, 981, 1047, 1030, 1031, 1077, 970, 17, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8180, 8211, 8150, 8199, 8100, 8253, 8182] avg=8192.0000 max=8261 max_rank=0 min=8100 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8448, 8064, 8320, 8064, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8448, 8320, 8192, 8192, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1044, 1009, 995, 1032, 983, 1020, 1064, 1039, 969, 1007, 1013, 998, 1002, 1080, 1039, 1020, 1031, 1018, 1036, 1001, 979, 1042, 1008, 1023, 1075, 978, 1012, 1047, 1045, 1047, 994, 1012, 986, 1023, 1012, 1036, 1061, 1040, 953, 979, 1028, 1023, 1032, 976, 1005, 1016, 1042, 1035, 1023, 1068, 1074, 1089, 1013, 1073, 1027, 1013, 1053, 1042, 1015, 1104, 988, 1013, 977, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1007, 1020, 963, 1037, 1028, 1018, 1043, 1064, 1020, 995, 1040, 1015, 1017, 1021, 1007, 1032, 1030, 1036, 1018, 1100, 1026, 1026, 1034, 1074, 983, 1034, 1022, 1018, 1085, 1030, 1072, 1083, 1020, 1044, 990, 1031, 982, 1006, 1053, 1017, 1008, 1022, 1009, 1012, 985, 1046, 1024, 1017, 1031, 1019, 1029, 980, 1030, 1056, 1014, 1065, 1034, 989, 985, 965, 987, 1066, 1003, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1001, 1053, 1026, 974, 1032, 1040, 977, 981, 1051, 1016, 1038, 1011, 1022, 1037, 1044, 1014, 1011, 1047, 1048, 1019, 1038, 1046, 1055, 1071, 1020, 1013, 1022, 986, 1018, 1016, 1017, 998, 995, 1061, 1019, 962, 1006, 987, 972, 1038, 1083, 1098, 1028, 1048, 1049, 1092, 994, 988, 1019, 1012, 1019, 1116, 1014, 1023, 1015, 1020, 1013, 1014, 1004, 1055, 1003, 1012, 1044, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1033, 1052, 1022, 1030, 1004, 1010, 1003, 1038, 1049, 1055, 994, 1026, 1009, 998, 1036, 1035, 1061, 1021, 1015, 1043, 1021, 1087, 971, 969, 1078, 993, 1022, 1024, 993, 1087, 1035, 1011, 1018, 1008, 1027, 1045, 1025, 1040, 1002, 1002, 1047, 980, 998, 1026, 1035, 967, 1026, 1000, 1007, 1035, 1018, 1032, 1040, 1039, 1063, 1003, 989, 988, 1016, 985, 1006, 1082, 1096, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1022, 1047, 1062, 1027, 1009, 1071, 1041, 1002, 1035, 1037, 1040, 1054, 1032, 979, 1053, 1016, 1020, 1022, 1064, 1030, 1002, 1037, 1049, 1054, 1020, 994, 1081, 1007, 1024, 1022, 970, 1089, 1033, 1009, 1045, 995, 1052, 1069, 984, 1021, 986, 980, 1052, 997, 1026, 1080, 1014, 999, 1001, 1006, 991, 1011, 1020, 1020, 982, 1018, 1045, 966, 1001, 1013, 998, 1022, 1047, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1053, 1029, 999, 1078, 1034, 1014, 1018, 1030, 1052, 1072, 1052, 986, 990, 1015, 1024, 1017, 1059, 1023, 1072, 1007, 1031, 1018, 1028, 998, 1008, 1046, 974, 975, 1052, 999, 1034, 1007, 989, 985, 1003, 1021, 953, 1049, 1096, 986, 1030, 1007, 1034, 1033, 1068, 982, 1030, 999, 1031, 983, 1031, 1027, 1023, 1002, 1043, 1051, 1033, 1040, 1047, 1054, 998, 1052, 1049, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1039, 1057, 986, 1030, 1033, 980, 1063, 1041, 981, 1070, 1022, 981, 1042, 1033, 1020, 1087, 1032, 1036, 1026, 1014, 985, 998, 999, 1029, 1030, 1054, 1023, 1016, 1032, 1008, 965, 993, 1016, 1000, 1029, 1020, 1034, 1059, 1044, 1022, 1040, 1051, 1004, 1028, 1031, 1049, 1008, 998, 1017, 1015, 1043, 1030, 1005, 1013, 1050, 1012, 1025, 1030, 1016, 1044, 1037, 1048, 1034, 979] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1028, 1070, 986, 973, 1034, 980, 1043, 1002, 1018, 1016, 1022, 1014, 997, 975, 1019, 1009, 1034, 1044, 1062, 1036, 1064, 1031, 981, 977, 1026, 999, 979, 1019, 980, 1014, 1043, 1042, 1044, 1021, 1017, 1055, 1012, 998, 1016, 1047, 1015, 1037, 1051, 1062, 1048, 1009, 1048, 1054, 1003, 1052, 1074, 1089, 1083, 1041, 1009, 1099, 1060, 977, 1032, 950, 989, 1005, 1035, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1007, 1020, 963, 1037, 1028, 1018, 1043, 1064, 1, 0, 0, 0, 0, 0, 0, 0, 1020, 995, 1040, 1015, 1017, 1021, 1007, 1032, 4, 3, 0, 0, 0, 0, 0, 0, 1030, 1036, 978, 1100, 1026, 1026, 1034, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1034, 1022, 1018, 1085, 1030, 1072, 1083, 17, 0, 0, 0, 0, 0, 0, 0, 1020, 1044, 990, 1031, 982, 1006, 1053, 1017, 43, 0, 0, 0, 0, 0, 0, 0, 1008, 1022, 1009, 1012, 980, 1046, 1024, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1019, 1029, 932, 1030, 1056, 1014, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 989, 985, 965, 987, 1066, 1003, 1019, 20, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8154, 8304, 8344, 8186, 8118, 8176, 8073] avg=8192.0000 max=8344 max_rank=3 min=8073 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1044, 1009, 995, 1032, 983, 1020, 1064, 1039, 1, 0, 0, 0, 0, 0, 0, 0, 969, 1007, 1013, 998, 1002, 1080, 1039, 1020, 4, 3, 0, 0, 0, 0, 0, 0, 1031, 1018, 996, 1001, 979, 1042, 1008, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 978, 1012, 1047, 1045, 1047, 994, 1012, 17, 0, 0, 0, 0, 0, 0, 0, 986, 1023, 1012, 1036, 1061, 1040, 953, 979, 48, 0, 0, 0, 0, 0, 0, 0, 1028, 1023, 1032, 976, 1000, 1016, 1042, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1068, 1074, 1036, 1013, 1073, 1027, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1042, 1015, 1104, 988, 1013, 977, 1055, 20, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8135, 8098, 8227, 8138, 8152, 8327, 8272] avg=8192.0000 max=8327 max_rank=6 min=8098 min_rank=2 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32889 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1033, 1052, 1022, 1030, 1004, 1010, 1003, 1038, 1, 0, 0, 0, 0, 0, 0, 0, 1049, 1055, 994, 1026, 1009, 998, 1036, 1035, 5, 3, 0, 0, 0, 0, 0, 0, 1061, 1021, 976, 1043, 1021, 1087, 971, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 993, 1022, 1024, 993, 1087, 1035, 1011, 17, 0, 0, 0, 0, 0, 0, 0, 1018, 1008, 1027, 1045, 1025, 1040, 1002, 1002, 45, 0, 0, 0, 0, 0, 0, 0, 1047, 980, 998, 1026, 1029, 967, 1026, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1035, 1018, 982, 1040, 1039, 1063, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 989, 988, 1016, 985, 1006, 1082, 1096, 1066, 19, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8210, 8149, 8260, 8212, 8073, 8187, 8252] avg=8192.0000 max=8260 max_rank=3 min=8073 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1001, 1053, 1026, 974, 1032, 1040, 977, 981, 1, 0, 0, 0, 0, 0, 0, 0, 1051, 1016, 1038, 1011, 1022, 1037, 1044, 1014, 5, 3, 0, 0, 0, 0, 0, 0, 1011, 1047, 1008, 1019, 1038, 1046, 1055, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1013, 1022, 986, 1018, 1016, 1017, 998, 17, 0, 0, 0, 0, 0, 0, 0, 995, 1061, 1019, 962, 1006, 987, 972, 1038, 49, 0, 0, 0, 0, 0, 0, 0, 1083, 1098, 1028, 1048, 1043, 1092, 994, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1012, 1019, 1061, 1014, 1023, 1015, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1014, 1004, 1055, 1003, 1012, 1044, 991, 20, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8085, 8241, 8295, 8107, 8089, 8374, 8183, 8162] avg=8192.0000 max=8374 max_rank=5 min=8085 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=57162 max_load_reduction_pct=0.872223 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1053, 1029, 999, 1078, 1034, 1014, 1018, 1030, 1, 0, 0, 0, 0, 0, 0, 0, 1052, 1072, 1052, 986, 990, 1015, 1024, 1017, 5, 3, 0, 0, 0, 0, 0, 0, 1059, 1023, 1030, 1007, 1031, 1018, 1028, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1046, 974, 975, 1052, 999, 1034, 1007, 18, 0, 0, 0, 0, 0, 0, 0, 989, 985, 1003, 1021, 953, 1049, 1096, 986, 45, 0, 0, 0, 0, 0, 0, 0, 1030, 1007, 1034, 1033, 1062, 982, 1030, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 983, 1031, 977, 1023, 1002, 1043, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1040, 1047, 1054, 998, 1052, 1049, 1013, 21, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8216, 8194, 8113, 8127, 8177, 8141, 8312] avg=8192.0000 max=8312 max_rank=7 min=8113 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1022, 1047, 1062, 1027, 1009, 1071, 1041, 1002, 1, 0, 0, 0, 0, 0, 0, 0, 1035, 1037, 1040, 1054, 1032, 979, 1053, 1016, 4, 3, 0, 0, 0, 0, 0, 0, 1020, 1022, 1024, 1030, 1002, 1037, 1049, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 994, 1081, 1007, 1024, 1022, 970, 1089, 17, 0, 0, 0, 0, 0, 0, 0, 1033, 1009, 1045, 995, 1052, 1069, 984, 1021, 44, 0, 0, 0, 0, 0, 0, 0, 986, 980, 1052, 997, 1021, 1080, 1014, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1006, 991, 962, 1020, 1020, 982, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 966, 1001, 1013, 998, 1022, 1047, 1041, 20, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8253, 8238, 8224, 8252, 8129, 8000, 8158] avg=8192.0000 max=8282 max_rank=0 min=8000 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1039, 1057, 986, 1030, 1033, 980, 1063, 1041, 1, 0, 0, 0, 0, 0, 0, 0, 981, 1070, 1022, 981, 1042, 1033, 1020, 1087, 5, 3, 0, 0, 0, 0, 0, 0, 1032, 1036, 986, 1014, 985, 998, 999, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1054, 1023, 1016, 1032, 1008, 965, 993, 17, 0, 0, 0, 0, 0, 0, 0, 1016, 1000, 1029, 1020, 1034, 1059, 1044, 1022, 45, 0, 0, 0, 0, 0, 0, 0, 1040, 1051, 1004, 1028, 1025, 1049, 1008, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1015, 1043, 980, 1005, 1013, 1050, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1030, 1016, 1044, 1037, 1048, 1034, 979, 20, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8244, 8079, 8138, 8269, 8203, 8135, 8238] avg=8192.0000 max=8269 max_rank=4 min=8079 min_rank=2 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1028, 1070, 986, 973, 1034, 980, 1043, 1002, 6, 0, 0, 0, 0, 0, 0, 0, 1018, 1016, 1022, 1014, 997, 975, 1019, 1009, 8, 5, 0, 0, 0, 0, 0, 0, 1034, 1044, 1012, 1036, 1064, 1031, 981, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 999, 979, 1019, 980, 1014, 1043, 1042, 21, 0, 0, 0, 0, 0, 0, 0, 1044, 1021, 1017, 1055, 1012, 998, 1016, 1047, 53, 0, 0, 0, 0, 0, 0, 0, 1015, 1037, 1051, 1062, 1034, 1009, 1048, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1052, 1074, 1026, 1083, 1041, 1009, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 977, 1032, 950, 989, 1005, 1035, 987, 24, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8083, 8179, 8123, 8263, 8310, 8387, 8069] avg=8192.0000 max=8387 max_rank=6 min=8069 min_rank=7 imbalance_ratio=1.023804 max_load_reduction=57149 max_load_reduction_pct=0.872025 moved_to_echo=33029 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8192, 8064, 8320, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8448, 8064, 8192, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8064, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[977, 1062, 1029, 1026, 965, 1026, 1027, 1000, 1034, 1110, 1026, 1045, 1056, 1049, 1020, 1057, 996, 1055, 1026, 1101, 1020, 1002, 1031, 1052, 1017, 1016, 1013, 1002, 1022, 1070, 1012, 1043, 1025, 1021, 1012, 998, 966, 1077, 1059, 1038, 1007, 977, 1007, 994, 1008, 1028, 1036, 1030, 983, 1011, 1044, 999, 1022, 973, 1053, 1011, 1036, 1066, 1029, 1043, 972, 1029, 986, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1051, 1030, 1007, 1001, 1050, 1017, 995, 1103, 1038, 1053, 996, 995, 996, 1049, 1018, 1063, 1025, 997, 1001, 1030, 997, 1003, 1030, 1048, 1057, 997, 1026, 1046, 950, 1023, 1019, 1031, 983, 1003, 1036, 1030, 1000, 987, 1022, 1030, 1029, 1007, 1005, 1036, 1085, 1081, 1014, 1035, 1012, 1008, 1074, 1042, 1033, 1014, 992, 1059, 1005, 1016, 1004, 1039, 1015, 1016, 1052, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1023, 1061, 1035, 966, 993, 1042, 1005, 1050, 1036, 997, 1049, 1022, 1083, 1038, 1037, 982, 1099, 1007, 1022, 1026, 964, 1034, 1055, 1013, 1036, 975, 1011, 1066, 1009, 994, 973, 1035, 1016, 995, 1003, 1043, 1047, 997, 1030, 952, 1093, 1007, 1090, 1021, 1014, 1020, 965, 1001, 1032, 1066, 1082, 989, 1013, 1015, 1035, 1045, 1027, 1066, 1025, 974, 1005, 1014, 1071, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1047, 1009, 1040, 983, 1010, 1053, 1005, 1003, 1047, 1031, 996, 1050, 1046, 1050, 1048, 1011, 1008, 1047, 994, 1040, 1052, 997, 1037, 1004, 979, 970, 1031, 1014, 1003, 1057, 1002, 1009, 990, 994, 1049, 1064, 1032, 1018, 1057, 1065, 1043, 999, 994, 994, 1023, 1059, 1002, 1037, 1039, 1033, 1051, 1044, 1012, 1004, 1032, 1072, 1029, 992, 1020, 1034, 1023, 1011, 1016, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1010, 1068, 1001, 1041, 1039, 1007, 1085, 1041, 1008, 1026, 1005, 1058, 1004, 999, 1018, 1058, 988, 1040, 1043, 1033, 1045, 1022, 1024, 1052, 987, 1034, 1010, 1061, 995, 1007, 993, 1083, 1005, 1020, 987, 1045, 1027, 1028, 1038, 1008, 1036, 1024, 962, 1024, 994, 1044, 1070, 1007, 1023, 1052, 1008, 1082, 960, 1040, 973, 1027, 1027, 998, 1080, 996, 1019, 1015, 1055, 977] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1016, 1014, 1016, 1002, 995, 1058, 961, 1029, 1047, 992, 1054, 1012, 1058, 1011, 1055, 1053, 1034, 1020, 1036, 1044, 1065, 1015, 999, 1019, 1023, 1054, 1081, 1011, 1030, 986, 1038, 990, 1041, 1015, 1066, 968, 1069, 1021, 1018, 979, 1045, 999, 994, 1001, 1056, 1055, 1004, 989, 1005, 987, 1018, 1087, 1030, 1023, 976, 1047, 1051, 1021, 1038, 1003, 1043, 996, 1052, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1029, 1034, 1057, 994, 1028, 1050, 997, 1016, 1039, 1019, 1037, 1087, 1038, 1027, 1037, 1029, 1030, 1025, 998, 1012, 1050, 1085, 1038, 982, 1046, 1009, 1016, 1087, 1019, 1049, 1029, 1016, 1033, 984, 991, 1008, 1077, 1048, 1031, 1000, 1029, 983, 1035, 1054, 988, 972, 1025, 1007, 1023, 1004, 1008, 997, 1004, 1009, 1004, 1018, 1036, 1009, 1023, 1036, 1057, 987, 1012, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1038, 1083, 981, 1016, 1049, 997, 1018, 1013, 1036, 1063, 1036, 1021, 1048, 945, 1005, 1073, 1011, 1048, 1018, 1033, 983, 1052, 976, 1011, 968, 1022, 1017, 1006, 1062, 1049, 1041, 1052, 1048, 1028, 1071, 1029, 1038, 960, 990, 1014, 1012, 1021, 995, 1019, 987, 957, 1061, 1030, 1047, 1070, 1054, 1011, 1023, 1115, 992, 995, 1002, 1016, 1022, 1025, 1030, 1013, 1058, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1023, 1061, 1035, 966, 993, 1042, 1005, 1050, 6, 1, 0, 0, 0, 0, 0, 0, 1036, 997, 1049, 1022, 1007, 1038, 1037, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 1007, 1022, 1014, 964, 1034, 1055, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 975, 1011, 1066, 1009, 994, 973, 1035, 11, 8, 0, 0, 0, 0, 0, 0, 1016, 995, 1003, 1043, 1047, 997, 1030, 952, 27, 0, 0, 0, 0, 0, 0, 0, 1093, 1007, 1090, 1021, 1014, 1020, 965, 1001, 41, 0, 0, 0, 0, 0, 0, 0, 1032, 1066, 1074, 989, 1013, 1015, 1035, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1066, 1025, 974, 1005, 1014, 1071, 1045, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8168, 8208, 8118, 8110, 8252, 8269, 8229] avg=8192.0000 max=8269 max_rank=6 min=8110 min_rank=4 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[977, 1062, 1029, 1026, 965, 1026, 1027, 1000, 6, 1, 0, 0, 0, 0, 0, 0, 1034, 1110, 1026, 1045, 983, 1049, 1020, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1055, 1026, 1088, 1020, 1002, 1031, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1016, 1013, 1002, 1022, 1070, 1012, 1043, 12, 7, 0, 0, 0, 0, 0, 0, 1025, 1021, 1012, 998, 966, 1077, 1059, 1038, 26, 0, 0, 0, 0, 0, 0, 0, 1007, 977, 1007, 994, 1008, 1028, 1036, 1030, 40, 0, 0, 0, 0, 0, 0, 0, 983, 1011, 1036, 999, 1022, 973, 1053, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1066, 1029, 1043, 972, 1029, 986, 1009, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8324, 8270, 8214, 8222, 8127, 8088, 8172] avg=8192.0000 max=8324 max_rank=1 min=8088 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1051, 1030, 1007, 1001, 1050, 1017, 995, 1103, 6, 1, 0, 0, 0, 0, 0, 0, 1038, 1053, 996, 995, 927, 1049, 1018, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 997, 1001, 1018, 997, 1003, 1030, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 997, 1026, 1046, 950, 1023, 1019, 1031, 11, 7, 0, 0, 0, 0, 0, 0, 983, 1003, 1036, 1030, 1000, 987, 1022, 1030, 25, 0, 0, 0, 0, 0, 0, 0, 1029, 1007, 1005, 1036, 1085, 1081, 1014, 1035, 37, 0, 0, 0, 0, 0, 0, 0, 1012, 1008, 1066, 1042, 1033, 1014, 992, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1016, 1004, 1039, 1015, 1016, 1052, 1030, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8139, 8119, 8167, 8116, 8329, 8226, 8179] avg=8192.0000 max=8329 max_rank=5 min=8116 min_rank=4 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1029, 1034, 1057, 994, 1028, 1050, 997, 1016, 5, 1, 0, 0, 0, 0, 0, 0, 1039, 1019, 1037, 1087, 966, 1027, 1037, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1025, 998, 1000, 1050, 1085, 1038, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1009, 1016, 1087, 1019, 1049, 1029, 1016, 11, 7, 0, 0, 0, 0, 0, 0, 1033, 984, 991, 1008, 1077, 1048, 1031, 1000, 26, 0, 0, 0, 0, 0, 0, 0, 1029, 983, 1035, 1054, 988, 972, 1025, 1007, 39, 0, 0, 0, 0, 0, 0, 0, 1023, 1004, 1001, 997, 1004, 1009, 1004, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1009, 1023, 1036, 1057, 987, 1012, 1035, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8241, 8208, 8289, 8198, 8132, 8060, 8197] avg=8192.0000 max=8289 max_rank=3 min=8060 min_rank=6 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32587 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1010, 1068, 1001, 1041, 1039, 1007, 1085, 1041, 5, 1, 0, 0, 0, 0, 0, 0, 1008, 1026, 1005, 1058, 934, 999, 1018, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1040, 1043, 1021, 1045, 1022, 1024, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1034, 1010, 1061, 995, 1007, 993, 1083, 11, 7, 0, 0, 0, 0, 0, 0, 1005, 1020, 987, 1045, 1027, 1028, 1038, 1008, 25, 0, 0, 0, 0, 0, 0, 0, 1036, 1024, 962, 1024, 994, 1044, 1070, 1007, 38, 0, 0, 0, 0, 0, 0, 0, 1023, 1052, 1001, 1082, 960, 1040, 973, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 998, 1080, 996, 1019, 1015, 1055, 977, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8106, 8235, 8188, 8183, 8199, 8158, 8169] avg=8192.0000 max=8298 max_rank=0 min=8106 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1016, 1014, 1016, 1002, 995, 1058, 961, 1029, 5, 1, 0, 0, 0, 0, 0, 0, 1047, 992, 1054, 1012, 985, 1011, 1055, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1020, 1036, 1032, 1065, 1015, 999, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1054, 1081, 1011, 1030, 986, 1038, 990, 11, 7, 0, 0, 0, 0, 0, 0, 1041, 1015, 1066, 968, 1069, 1021, 1018, 979, 26, 0, 0, 0, 0, 0, 0, 0, 1045, 999, 994, 1001, 1056, 1055, 1004, 989, 40, 0, 0, 0, 0, 0, 0, 0, 1005, 987, 1011, 1087, 1030, 1023, 976, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1021, 1038, 1003, 1043, 996, 1052, 1021, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8209, 8220, 8231, 8203, 8183, 8166, 8227] avg=8192.0000 max=8231 max_rank=3 min=8097 min_rank=0 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1047, 1009, 1040, 983, 1010, 1053, 1005, 1003, 6, 1, 0, 0, 0, 0, 0, 0, 1047, 1031, 996, 1050, 974, 1050, 1048, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1047, 994, 1028, 1052, 997, 1037, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 979, 970, 1031, 1014, 1003, 1057, 1002, 1009, 11, 7, 0, 0, 0, 0, 0, 0, 990, 994, 1049, 1064, 1032, 1018, 1057, 1065, 26, 0, 0, 0, 0, 0, 0, 0, 1043, 999, 994, 994, 1023, 1059, 1002, 1037, 39, 0, 0, 0, 0, 0, 0, 0, 1039, 1033, 1043, 1044, 1012, 1004, 1032, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 992, 1020, 1034, 1023, 1011, 1016, 1031, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8207, 8167, 8083, 8295, 8190, 8279, 8158] avg=8192.0000 max=8295 max_rank=4 min=8083 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32922 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1038, 1083, 981, 1016, 1049, 997, 1018, 1013, 10, 6, 0, 0, 0, 0, 0, 0, 1036, 1063, 1036, 1021, 963, 945, 1005, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1048, 1018, 1010, 983, 1052, 976, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1022, 1017, 1006, 1062, 1049, 1041, 1052, 17, 12, 0, 0, 0, 0, 0, 0, 1048, 1028, 1071, 1029, 1038, 960, 990, 1014, 31, 0, 0, 0, 0, 0, 0, 0, 1012, 1021, 995, 1019, 987, 957, 1061, 1030, 42, 0, 0, 0, 0, 0, 0, 0, 1047, 1070, 1037, 1011, 1023, 1115, 992, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1016, 1022, 1025, 1030, 1013, 1058, 1032, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8142, 8109, 8246, 8209, 8124, 8290, 8205] avg=8192.0000 max=8290 max_rank=6 min=8109 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8192, 8064, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8192, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8192, 8192, 8320, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 7808, 8192, 8320, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1013, 1041, 993, 1004, 1048, 1027, 1060, 1059, 1046, 1015, 1006, 963, 999, 989, 1054, 1006, 1008, 983, 1013, 1042, 1052, 1075, 1004, 1043, 1045, 1035, 1005, 1069, 1041, 1035, 984, 1022, 987, 1040, 1011, 975, 990, 1026, 1054, 1013, 1011, 994, 1062, 1007, 1016, 1034, 972, 1027, 1020, 998, 1036, 1017, 1089, 1036, 1011, 1021, 1042, 990, 1069, 1048, 1050, 1065, 1018, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1008, 983, 999, 986, 980, 1045, 1026, 1057, 1035, 1058, 1008, 1032, 1036, 1041, 1052, 1019, 1021, 1017, 991, 1026, 1111, 1027, 1010, 1049, 1040, 972, 1002, 1048, 976, 1000, 1032, 1034, 1006, 1045, 1092, 1055, 1009, 1031, 1016, 996, 1026, 1071, 1008, 1050, 1065, 1034, 1013, 999, 1030, 1011, 1050, 1063, 984, 1018, 1007, 1044, 1000, 1005, 1051, 1015, 981, 1020, 997, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1042, 992, 1019, 997, 988, 1043, 1034, 1036, 1027, 1046, 989, 1017, 1039, 1023, 1000, 1068, 1057, 981, 1056, 974, 1058, 1061, 984, 1046, 1044, 997, 1006, 1020, 1038, 1043, 1030, 1034, 1079, 954, 1049, 990, 1036, 997, 1060, 995, 1063, 1008, 1038, 1056, 983, 1039, 1028, 1018, 1011, 1020, 1016, 1040, 1022, 1082, 1055, 1013, 1030, 993, 1036, 1016, 985, 1023, 1032, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1025, 1103, 1034, 1007, 1008, 1024, 1092, 1000, 1012, 1019, 1041, 1010, 1000, 1019, 1031, 1029, 1051, 1010, 1021, 987, 1034, 1031, 994, 1009, 1011, 1013, 1013, 998, 1059, 1034, 1051, 996, 1012, 995, 1023, 1015, 1037, 996, 1066, 1055, 1017, 970, 1044, 1025, 1035, 1021, 1000, 984, 1053, 1042, 1024, 1016, 1024, 1071, 1020, 998, 1025, 1036, 997, 1003, 1035, 1075, 1018, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1019, 1020, 1045, 1052, 989, 1076, 1038, 988, 1040, 1020, 1032, 1052, 1034, 1029, 1064, 1041, 1015, 1052, 1036, 1054, 1022, 1008, 980, 1080, 1068, 1036, 1049, 992, 1003, 1057, 1009, 990, 1019, 1081, 1073, 1026, 1011, 997, 1002, 994, 1019, 1048, 1031, 1005, 1035, 979, 1031, 1033, 951, 1007, 1022, 1034, 1025, 1005, 1034, 1000, 963, 1008, 985, 1042, 1006, 1037, 1025, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[984, 998, 1022, 1051, 1077, 1024, 1049, 1054, 1004, 1002, 1018, 1040, 1034, 1022, 1014, 1049, 1034, 1026, 1033, 1037, 1045, 1029, 1032, 985, 1054, 971, 1007, 1035, 1038, 1062, 986, 965, 997, 1025, 1036, 998, 1017, 1031, 1023, 1033, 1049, 995, 983, 983, 1038, 1053, 958, 1061, 971, 1018, 1037, 1024, 1041, 1036, 1039, 1043, 1028, 1017, 1019, 1071, 1050, 1007, 1017, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1007, 1044, 1012, 1045, 966, 1052, 1037, 1053, 1068, 1039, 1022, 1010, 1061, 1006, 1003, 986, 1025, 1011, 1059, 996, 1052, 1052, 1004, 1029, 1026, 1014, 994, 991, 998, 1028, 1029, 1046, 1000, 1057, 1014, 1041, 1060, 952, 1068, 1046, 1014, 977, 1036, 1060, 1062, 1059, 1057, 920, 1089, 995, 1084, 987, 964, 1033, 1028, 1018, 1031, 1023, 997, 1016, 1006, 985, 1045, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[993, 1013, 1037, 1055, 1065, 998, 1012, 1025, 1031, 1016, 1022, 1031, 1047, 1024, 1042, 1018, 990, 1024, 1019, 1048, 962, 961, 1062, 983, 1061, 1004, 1012, 1005, 1052, 1047, 1027, 1043, 1054, 1005, 1006, 1032, 1026, 982, 1034, 997, 1055, 1061, 1063, 1029, 1058, 990, 1057, 1065, 1043, 1052, 1011, 1021, 980, 1031, 998, 1006, 1024, 977, 1034, 1058, 1030, 985, 1004, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1042, 992, 1019, 997, 988, 1043, 1018, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1046, 989, 1017, 1039, 1023, 1000, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 981, 1056, 974, 1054, 1061, 984, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 997, 1006, 1020, 1038, 1043, 1030, 1034, 11, 2, 0, 0, 0, 0, 0, 0, 1079, 954, 1049, 990, 1036, 997, 1060, 995, 4, 4, 3, 0, 0, 0, 0, 0, 1063, 1008, 1038, 1056, 978, 1039, 1028, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1020, 1016, 1040, 1022, 1078, 1055, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 993, 1036, 1016, 985, 1023, 1032, 980, 16, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8196, 8213, 8225, 8171, 8228, 8255, 8113] avg=8192.0000 max=8255 max_rank=6 min=8113 min_rank=7 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1025, 1103, 1034, 1007, 1008, 1024, 1075, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1019, 1041, 1010, 1000, 1019, 1031, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1010, 1021, 987, 1030, 1031, 994, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1013, 1013, 998, 1059, 1034, 1051, 996, 11, 2, 0, 0, 0, 0, 0, 0, 1012, 995, 1023, 1015, 1037, 996, 1066, 1055, 4, 4, 3, 0, 0, 0, 0, 0, 1017, 970, 1044, 1025, 1030, 1021, 1000, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1042, 1024, 1016, 1024, 1067, 1020, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1036, 997, 1003, 1035, 1075, 1018, 1038, 17, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8148, 8133, 8188, 8210, 8091, 8244, 8246] avg=8192.0000 max=8276 max_rank=0 min=8091 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1019, 1020, 1045, 1052, 989, 1076, 1021, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1020, 1032, 1052, 1034, 1029, 1064, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1052, 1036, 1054, 1018, 1008, 980, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1036, 1049, 992, 1003, 1057, 1009, 990, 11, 2, 0, 0, 0, 0, 0, 0, 1019, 1081, 1073, 1026, 1011, 997, 1002, 994, 4, 3, 3, 0, 0, 0, 0, 0, 1019, 1048, 1031, 1005, 1030, 979, 1031, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1007, 1022, 1034, 1025, 1002, 1034, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1008, 985, 1042, 1006, 1037, 1025, 1018, 17, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8298, 8243, 8217, 8213, 8176, 8075, 8104] avg=8192.0000 max=8298 max_rank=1 min=8075 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1013, 1041, 993, 1004, 1048, 1027, 1043, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1015, 1006, 963, 999, 989, 1054, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 983, 1013, 1042, 1048, 1075, 1004, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1035, 1005, 1069, 1041, 1035, 984, 1022, 11, 2, 0, 0, 0, 0, 0, 0, 987, 1040, 1011, 975, 990, 1026, 1054, 1013, 4, 4, 3, 0, 0, 0, 0, 0, 1011, 994, 1062, 1007, 1011, 1034, 972, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 998, 1036, 1017, 1089, 1032, 1011, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 990, 1069, 1048, 1050, 1065, 1018, 1028, 17, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8064, 8216, 8249, 8107, 8118, 8224, 8330] avg=8192.0000 max=8330 max_rank=7 min=8064 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1008, 983, 999, 986, 980, 1045, 1010, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1058, 1008, 1032, 1036, 1041, 1052, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1017, 991, 1026, 1107, 1027, 1010, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 972, 1002, 1048, 976, 1000, 1032, 1034, 11, 2, 0, 0, 0, 0, 0, 0, 1006, 1045, 1092, 1055, 1009, 1031, 1016, 996, 4, 4, 3, 0, 0, 0, 0, 0, 1026, 1071, 1008, 1050, 1060, 1034, 1013, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1011, 1050, 1063, 984, 1014, 1007, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1005, 1051, 1015, 981, 1020, 997, 1023, 16, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 8267, 8248, 8117, 8261, 8261, 8203, 8111] avg=8192.0000 max=8267 max_rank=1 min=8068 min_rank=0 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[984, 998, 1022, 1051, 1077, 1024, 1032, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1002, 1018, 1040, 1034, 1022, 1014, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1026, 1033, 1037, 1041, 1029, 1032, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 971, 1007, 1035, 1038, 1062, 986, 965, 10, 2, 0, 0, 0, 0, 0, 0, 997, 1025, 1036, 998, 1017, 1031, 1023, 1033, 4, 4, 3, 0, 0, 0, 0, 0, 1049, 995, 983, 983, 1033, 1053, 958, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1018, 1037, 1024, 1041, 1032, 1039, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1017, 1019, 1071, 1050, 1007, 1017, 1057, 17, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8171, 8217, 8130, 8171, 8115, 8205, 8285] avg=8192.0000 max=8285 max_rank=7 min=8115 min_rank=5 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1007, 1044, 1012, 1045, 966, 1052, 1020, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1039, 1022, 1010, 1061, 1006, 1003, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1011, 1059, 996, 1048, 1052, 1004, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1014, 994, 991, 998, 1028, 1029, 1046, 11, 2, 0, 0, 0, 0, 0, 0, 1000, 1057, 1014, 1041, 1060, 952, 1068, 1046, 4, 4, 3, 0, 0, 0, 0, 0, 1014, 977, 1036, 1060, 1057, 1059, 1057, 920, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 995, 1084, 987, 964, 1029, 1028, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1023, 997, 1016, 1006, 985, 1045, 1047, 17, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8181, 8224, 8139, 8249, 8180, 8194, 8170] avg=8192.0000 max=8249 max_rank=4 min=8139 min_rank=3 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[993, 1013, 1037, 1055, 1065, 998, 992, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1016, 1022, 1031, 1047, 1024, 1042, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1024, 1019, 1048, 955, 961, 1062, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1004, 1012, 1005, 1052, 1047, 1027, 1043, 14, 6, 0, 0, 0, 0, 0, 0, 1054, 1005, 1006, 1032, 1026, 982, 1034, 997, 7, 6, 5, 0, 0, 0, 0, 0, 1055, 1061, 1063, 1029, 1047, 990, 1057, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1052, 1011, 1021, 980, 1025, 998, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 977, 1034, 1058, 1030, 985, 1004, 1039, 20, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8211, 8042, 8271, 8154, 8367, 8136, 8177] avg=8192.0000 max=8367 max_rank=5 min=8042 min_rank=2 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8064, 8448, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8064, 8192, 8192, 8320, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8192, 8320, 8192, 8320, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1033, 1053, 996, 1015, 1056, 995, 1046, 1046, 1030, 1035, 1029, 1019, 1028, 1078, 1073, 1044, 1029, 1076, 1038, 998, 1059, 1027, 1046, 1058, 1017, 989, 985, 1017, 1011, 1041, 1048, 1015, 1024, 1030, 1059, 1005, 1030, 990, 989, 1036, 979, 1057, 1046, 1037, 990, 1040, 975, 1040, 1032, 1010, 993, 1035, 1023, 1016, 952, 978, 997, 1054, 994, 1000, 1010, 1053, 1003, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1009, 1046, 1051, 1011, 1053, 1046, 1028, 989, 985, 1025, 1013, 1027, 1034, 1002, 1040, 1051, 1033, 1032, 1026, 1014, 982, 1018, 1010, 1089, 978, 1018, 1003, 1007, 1038, 968, 1031, 1024, 1016, 1024, 1010, 1064, 1001, 1004, 1036, 1025, 1004, 982, 1041, 1050, 1026, 1056, 1004, 988, 1013, 1048, 1034, 1076, 1049, 1062, 1023, 987, 1029, 1061, 1048, 968, 1074, 1031, 1031, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[980, 1006, 1024, 1074, 1031, 1011, 1036, 1058, 1001, 1012, 981, 1019, 1065, 1055, 1064, 1024, 1026, 1026, 1005, 990, 1027, 1016, 1011, 1034, 1010, 1029, 1024, 1010, 1028, 1016, 1029, 1060, 1004, 1024, 994, 1034, 1010, 1016, 1028, 1022, 1011, 1007, 1006, 1023, 1016, 1063, 1030, 1023, 1034, 1087, 1033, 1051, 1015, 1016, 1002, 1014, 1018, 1008, 1018, 1026, 1059, 1012, 1047, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[981, 1038, 986, 983, 1034, 1029, 1051, 1040, 999, 1054, 1009, 990, 1011, 1079, 1057, 1002, 1059, 1019, 976, 997, 1025, 983, 1045, 1034, 1040, 1039, 1007, 1037, 1019, 1102, 1022, 1026, 990, 1033, 1032, 1013, 992, 1031, 1056, 1037, 1015, 1022, 1040, 1037, 1007, 1059, 1058, 1029, 1056, 993, 999, 1004, 973, 1008, 1082, 1017, 1034, 1048, 993, 998, 1004, 1026, 1045, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1041, 1023, 1056, 1033, 1037, 1033, 1001, 1026, 1000, 1031, 995, 1040, 1038, 973, 1073, 979, 1055, 1035, 1047, 1031, 1011, 1069, 974, 996, 1007, 1028, 999, 1041, 1040, 1068, 1030, 1058, 1030, 971, 971, 1043, 1043, 998, 1043, 996, 1035, 997, 954, 1066, 1056, 996, 1075, 1006, 1022, 1044, 1042, 974, 1038, 1004, 1040, 1006, 1056, 1009, 1026, 1024, 1028, 1025, 1023, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1029, 1026, 1041, 1052, 1037, 1054, 1045, 1030, 1031, 1004, 1015, 1030, 1041, 1051, 999, 998, 992, 1018, 1096, 961, 1006, 1021, 1066, 1028, 1038, 1045, 1028, 1071, 996, 969, 1004, 1029, 1027, 985, 990, 1017, 1020, 1019, 1069, 1037, 1044, 1011, 1003, 973, 1000, 1024, 1055, 1021, 1009, 1058, 1048, 1005, 1016, 1017, 1010, 997, 1022, 1018, 1035, 1032, 1024, 1064, 999, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1006, 1006, 1050, 1060, 1033, 1045, 1040, 1033, 987, 1028, 1031, 1030, 1034, 986, 1023, 1061, 1018, 1029, 1022, 1019, 1067, 972, 972, 1089, 1005, 1012, 1057, 1054, 1031, 1024, 1027, 988, 992, 1031, 1012, 1036, 1049, 986, 1032, 1019, 1015, 1006, 1013, 1093, 999, 1017, 1010, 1030, 1023, 1034, 1056, 1066, 994, 1032, 987, 979, 1010, 1001, 1034, 1050, 1025, 1043, 986, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1035, 997, 1023, 1035, 1073, 1030, 1014, 984, 1055, 981, 991, 984, 1015, 983, 1004, 1056, 1038, 1019, 1040, 1030, 1019, 996, 1003, 1004, 1055, 986, 1011, 1051, 1019, 1078, 1071, 1019, 1062, 950, 1003, 1016, 982, 1051, 1036, 1011, 1062, 1047, 1065, 1048, 1078, 1015, 983, 1012, 1034, 1065, 999, 1037, 1014, 1078, 1035, 994, 1062, 985, 997, 1016, 1069, 985, 1016, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1033, 1053, 996, 1015, 1015, 995, 1046, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1035, 1029, 1019, 1028, 1078, 1073, 1044, 4, 1, 0, 0, 0, 0, 0, 0, 1029, 1076, 1038, 998, 1059, 1027, 1046, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 989, 985, 1008, 1011, 1041, 1048, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1030, 1059, 1005, 1030, 990, 989, 1036, 41, 2, 0, 0, 0, 0, 0, 0, 979, 1057, 1046, 1033, 990, 1040, 975, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1010, 993, 1035, 1023, 1016, 952, 978, 7, 0, 0, 0, 0, 0, 0, 0, 997, 1054, 994, 1000, 1010, 1053, 1003, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8341, 8330, 8114, 8206, 8160, 8046, 8140] avg=8192.0000 max=8341 max_rank=1 min=8046 min_rank=6 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32552 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1009, 1046, 1051, 1011, 1012, 1046, 1028, 989, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1025, 1013, 1027, 1034, 1002, 1040, 1051, 4, 1, 0, 0, 0, 0, 0, 0, 1033, 1032, 1026, 1014, 982, 1018, 1010, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1018, 1003, 998, 1038, 968, 1031, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1024, 1010, 1064, 1001, 1004, 1036, 1025, 41, 2, 0, 0, 0, 0, 0, 0, 1004, 982, 1041, 1046, 1026, 1056, 1004, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1048, 1034, 1076, 1049, 1062, 1023, 987, 7, 0, 0, 0, 0, 0, 0, 0, 1029, 1061, 1048, 968, 1074, 1031, 1031, 990, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8182, 8203, 8058, 8223, 8147, 8299, 8232] avg=8192.0000 max=8299 max_rank=6 min=8058 min_rank=3 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32901 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[980, 1006, 1024, 1074, 991, 1011, 1036, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1012, 981, 1019, 1065, 1055, 1064, 1024, 4, 1, 0, 0, 0, 0, 0, 0, 1026, 1026, 1005, 990, 1027, 1016, 1011, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1029, 1024, 1001, 1028, 1016, 1029, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1024, 994, 1034, 1010, 1016, 1028, 1022, 40, 2, 0, 0, 0, 0, 0, 0, 1011, 1007, 1006, 1019, 1016, 1063, 1030, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1087, 1033, 1051, 1015, 1016, 1002, 1014, 7, 0, 0, 0, 0, 0, 0, 0, 1018, 1008, 1018, 1026, 1059, 1012, 1047, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8226, 8134, 8197, 8174, 8175, 8259, 8191] avg=8192.0000 max=8259 max_rank=6 min=8134 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[981, 1038, 986, 983, 994, 1029, 1051, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1054, 1009, 990, 1011, 1079, 1057, 1002, 4, 1, 0, 0, 0, 0, 0, 0, 1059, 1019, 976, 997, 1025, 983, 1045, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1039, 1007, 1027, 1019, 1102, 1022, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1033, 1032, 1013, 992, 1031, 1056, 1037, 40, 2, 0, 0, 0, 0, 0, 0, 1015, 1022, 1040, 1033, 1007, 1059, 1058, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 993, 999, 1004, 973, 1008, 1082, 1017, 8, 0, 0, 0, 0, 0, 0, 0, 1034, 1048, 993, 998, 1004, 1026, 1045, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8206, 8137, 8282, 8226, 8263, 8140, 8180] avg=8192.0000 max=8282 max_rank=3 min=8102 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1029, 1026, 1041, 1052, 997, 1054, 1045, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1004, 1015, 1030, 1041, 1051, 999, 998, 3, 1, 0, 0, 0, 0, 0, 0, 992, 1018, 1096, 961, 1006, 1021, 1066, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1045, 1028, 1061, 996, 969, 1004, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 985, 990, 1017, 1020, 1019, 1069, 1037, 40, 2, 0, 0, 0, 0, 0, 0, 1044, 1011, 1003, 970, 1000, 1024, 1055, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1058, 1048, 1005, 1016, 1017, 1010, 997, 8, 0, 0, 0, 0, 0, 0, 0, 1022, 1018, 1035, 1032, 1024, 1064, 999, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8173, 8187, 8170, 8206, 8128, 8168, 8230] avg=8192.0000 max=8274 max_rank=0 min=8128 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1006, 1006, 1050, 1060, 993, 1045, 1040, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1028, 1031, 1030, 1034, 986, 1023, 1061, 4, 1, 0, 0, 0, 0, 0, 0, 1018, 1029, 1022, 1019, 1067, 972, 972, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1012, 1057, 1044, 1031, 1024, 1027, 988, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1031, 1012, 1036, 1049, 986, 1032, 1019, 40, 2, 0, 0, 0, 0, 0, 0, 1015, 1006, 1013, 1089, 999, 1017, 1010, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1034, 1056, 1066, 994, 1032, 987, 979, 8, 0, 0, 0, 0, 0, 0, 0, 1010, 1001, 1034, 1050, 1025, 1043, 986, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8185, 8187, 8188, 8199, 8179, 8179, 8186] avg=8192.0000 max=8233 max_rank=0 min=8179 min_rank=5 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1041, 1023, 1056, 1033, 997, 1033, 1001, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1031, 995, 1040, 1038, 973, 1073, 979, 4, 1, 0, 0, 0, 0, 0, 0, 1055, 1035, 1047, 1031, 1011, 1069, 974, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1028, 999, 1031, 1040, 1068, 1030, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 971, 971, 1043, 1043, 998, 1043, 996, 40, 2, 0, 0, 0, 0, 0, 0, 1035, 997, 954, 1062, 1056, 996, 1075, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1044, 1042, 974, 1038, 1004, 1040, 1006, 8, 0, 0, 0, 0, 0, 0, 0, 1056, 1009, 1026, 1024, 1028, 1025, 1023, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8134, 8217, 8261, 8137, 8181, 8178, 8218] avg=8192.0000 max=8261 max_rank=3 min=8134 min_rank=1 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1035, 997, 1023, 1035, 1028, 1030, 1014, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 981, 991, 984, 1015, 983, 1004, 1056, 7, 8, 4, 1, 0, 0, 0, 0, 1038, 1019, 1040, 1030, 1019, 996, 1003, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 986, 1011, 1027, 1019, 1078, 1071, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 950, 1003, 1016, 982, 1051, 1036, 1011, 45, 9, 0, 0, 0, 0, 0, 0, 1062, 1047, 1065, 1041, 1078, 1015, 983, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1065, 999, 1037, 1014, 1078, 1035, 994, 11, 0, 0, 0, 0, 0, 0, 0, 1062, 985, 997, 1016, 1068, 985, 1016, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8089, 8141, 8266, 8165, 8303, 8267, 8159] avg=8192.0000 max=8303 max_rank=5 min=8089 min_rank=1 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32894 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8064, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8192, 8192, 8064, 8192, 8320, 8320, 8448, 8320, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8064, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8448, 8320, 8320, 8192, 8320, 8192, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8192, 8320, 8192, 8192, 8320, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1038, 1045, 1053, 1045, 987, 1018, 1006, 1049, 999, 1006, 1055, 1009, 1029, 996, 1016, 1057, 1026, 1015, 1068, 991, 1008, 1009, 1050, 985, 1029, 1033, 1004, 1028, 1030, 1076, 1014, 1003, 1012, 1014, 1019, 1037, 1022, 1020, 974, 1019, 1077, 1030, 1022, 1028, 1056, 1028, 1031, 1041, 1030, 1046, 1038, 1048, 1036, 1002, 1048, 968, 1000, 986, 994, 1016, 1030, 1025, 1054, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[968, 1030, 1018, 1025, 1019, 1006, 1086, 1029, 1072, 1036, 1012, 1014, 1050, 986, 994, 1031, 1029, 1070, 1011, 1039, 1030, 1060, 1036, 1000, 1003, 1049, 1009, 1043, 1028, 1013, 1014, 1037, 1005, 1057, 1043, 1014, 1033, 1005, 1031, 974, 1007, 1000, 998, 1008, 1042, 1053, 1023, 1040, 1001, 1040, 1027, 986, 1098, 1078, 1000, 1038, 994, 939, 1015, 1008, 1012, 1024, 1064, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1030, 1011, 1046, 1050, 1045, 1039, 1027, 1041, 1015, 1012, 1032, 1050, 951, 1041, 1034, 1000, 999, 980, 1029, 1027, 1018, 987, 1072, 1042, 1034, 1026, 1015, 1036, 964, 1083, 1061, 992, 1042, 1027, 1043, 1004, 1000, 1024, 1060, 989, 975, 1024, 1014, 1032, 1064, 1035, 1009, 1045, 1004, 986, 980, 997, 1064, 973, 965, 1058, 1078, 1068, 1018, 1043, 1022, 1036, 1030, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[974, 1003, 1086, 1015, 1026, 1011, 1033, 1087, 992, 989, 1036, 1028, 1047, 1019, 985, 1060, 1031, 1066, 994, 1043, 1045, 1022, 1029, 1013, 1006, 1039, 1038, 1006, 997, 1043, 1025, 998, 1081, 1042, 989, 1014, 1011, 1052, 991, 1039, 1033, 1059, 1013, 965, 1011, 1053, 1020, 1041, 1041, 1028, 982, 1042, 1028, 1032, 1036, 1040, 992, 1023, 1024, 1011, 1052, 1038, 973, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[991, 1055, 1059, 1001, 1047, 1006, 970, 1021, 1037, 1021, 1051, 1023, 1024, 1019, 1060, 964, 1028, 1012, 996, 1030, 986, 1029, 1061, 1019, 1032, 987, 1028, 1047, 1018, 1037, 1038, 1003, 1009, 1032, 1052, 1043, 1045, 1025, 1035, 1024, 970, 991, 1019, 1058, 1020, 1029, 1030, 1019, 999, 1029, 990, 988, 1014, 1018, 1046, 1009, 1072, 1019, 1000, 1073, 1053, 1030, 1040, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1039, 992, 1049, 1078, 1030, 1055, 1038, 991, 1018, 1033, 1000, 976, 1051, 1024, 1027, 1018, 1069, 1031, 1045, 1004, 1027, 1018, 988, 1032, 1040, 961, 1048, 1001, 1006, 1025, 1041, 1006, 1036, 1029, 1028, 1035, 1057, 1031, 1049, 1015, 1035, 1040, 988, 1039, 996, 1001, 1024, 1006, 993, 1012, 990, 1004, 1011, 1021, 1059, 1013, 992, 1080, 1061, 1049, 1021, 1003, 1044, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1083, 1034, 1002, 1056, 1056, 1003, 1027, 1055, 1016, 995, 1010, 994, 1025, 1008, 1015, 1043, 1003, 1034, 1029, 1006, 1065, 1057, 1029, 1062, 1019, 1012, 1021, 947, 1081, 1053, 994, 1022, 1030, 979, 1057, 986, 1068, 1074, 961, 1020, 1008, 1035, 1026, 1079, 993, 965, 1036, 1075, 986, 979, 1000, 1009, 1002, 992, 1018, 1033, 1021, 1046, 1037, 1043, 1051, 1004, 1065, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1031, 1014, 1022, 1085, 976, 994, 995, 1008, 1015, 1044, 1017, 993, 1074, 1003, 1025, 995, 999, 1007, 1036, 977, 981, 1035, 1065, 1027, 1012, 1032, 1009, 986, 1045, 1012, 1091, 1011, 1034, 1054, 1013, 1040, 1008, 1033, 1051, 1074, 1041, 1062, 1011, 1083, 1067, 997, 1057, 1024, 1053, 1082, 1005, 984, 981, 1030, 987, 1020, 1008, 990, 967, 1040, 1045, 1045, 1003, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1038, 1045, 1053, 1011, 987, 1018, 1006, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1006, 1055, 1009, 1029, 996, 1016, 1057, 20, 10, 1, 0, 0, 0, 0, 0, 1026, 1015, 1068, 991, 1008, 1009, 1041, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1033, 1004, 1028, 1030, 1076, 1014, 1003, 9, 2, 0, 0, 0, 0, 0, 0, 1012, 1014, 1019, 1037, 1022, 999, 974, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 1030, 1022, 1008, 1056, 1028, 1031, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1046, 1038, 1048, 1036, 1002, 1048, 968, 34, 20, 0, 0, 0, 0, 0, 0, 1000, 986, 994, 1016, 1018, 1025, 1054, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8198, 8143, 8228, 8096, 8293, 8270, 8101] avg=8192.0000 max=8293 max_rank=5 min=8096 min_rank=4 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[968, 1030, 1018, 992, 1019, 1006, 1086, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1036, 1012, 1014, 1050, 986, 994, 1031, 20, 10, 1, 0, 0, 0, 0, 0, 1029, 1070, 1011, 1039, 1030, 1060, 1027, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1049, 1009, 1043, 1028, 1013, 1014, 1037, 9, 2, 0, 0, 0, 0, 0, 0, 1005, 1057, 1043, 1014, 1033, 984, 1031, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1000, 998, 988, 1042, 1053, 1023, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1040, 1027, 986, 1098, 1078, 1000, 1038, 33, 20, 0, 0, 0, 0, 0, 0, 994, 939, 1015, 1008, 1000, 1024, 1064, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8226, 8266, 8207, 8141, 8151, 8321, 8076] avg=8192.0000 max=8321 max_rank=6 min=8076 min_rank=7 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1083, 1034, 1002, 1022, 1056, 1003, 1027, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 995, 1010, 994, 1025, 1008, 1015, 1043, 21, 11, 1, 0, 0, 0, 0, 0, 1003, 1034, 1029, 1006, 1065, 1057, 1020, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1012, 1021, 947, 1081, 1053, 994, 1022, 9, 2, 0, 0, 0, 0, 0, 0, 1030, 979, 1057, 986, 1068, 1052, 961, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1035, 1026, 1058, 993, 965, 1036, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 986, 979, 1000, 1009, 1002, 992, 1018, 1033, 34, 21, 0, 0, 0, 0, 0, 0, 1021, 1046, 1037, 1043, 1038, 1004, 1065, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8139, 8276, 8160, 8153, 8196, 8074, 8256] avg=8192.0000 max=8282 max_rank=0 min=8074 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[991, 1055, 1059, 969, 1047, 1006, 970, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1021, 1051, 1023, 1024, 1019, 1060, 964, 21, 11, 1, 0, 0, 0, 0, 0, 1028, 1012, 996, 1030, 986, 1029, 1052, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 987, 1028, 1047, 1018, 1037, 1038, 1003, 9, 2, 0, 0, 0, 0, 0, 0, 1009, 1032, 1052, 1043, 1045, 1004, 1035, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 970, 991, 1019, 1037, 1020, 1029, 1030, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1029, 990, 988, 1014, 1018, 1046, 1009, 32, 20, 0, 0, 0, 0, 0, 0, 1072, 1019, 1000, 1073, 1040, 1030, 1040, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8232, 8152, 8201, 8244, 8115, 8145, 8329] avg=8192.0000 max=8329 max_rank=7 min=8115 min_rank=5 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32833 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1030, 1011, 1046, 1016, 1045, 1039, 1027, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1012, 1032, 1050, 951, 1041, 1034, 1000, 20, 10, 1, 0, 0, 0, 0, 0, 999, 980, 1029, 1027, 1018, 987, 1063, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1026, 1015, 1036, 964, 1083, 1061, 992, 9, 2, 0, 0, 0, 0, 0, 0, 1042, 1027, 1043, 1004, 1000, 1003, 1060, 989, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1024, 1014, 1012, 1064, 1035, 1009, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 986, 980, 997, 1064, 973, 965, 1058, 34, 20, 0, 0, 0, 0, 0, 0, 1078, 1068, 1018, 1043, 1010, 1036, 1030, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8166, 8145, 8222, 8168, 8178, 8081, 8321] avg=8192.0000 max=8321 max_rank=7 min=8081 min_rank=6 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[974, 1003, 1086, 982, 1026, 1011, 1033, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 992, 989, 1036, 1028, 1047, 1019, 985, 1060, 19, 11, 1, 0, 0, 0, 0, 0, 1031, 1066, 994, 1043, 1045, 1022, 1020, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1039, 1038, 1006, 997, 1043, 1025, 998, 9, 2, 0, 0, 0, 0, 0, 0, 1081, 1042, 989, 1014, 1011, 1030, 991, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1059, 1013, 946, 1011, 1053, 1020, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1028, 982, 1042, 1028, 1032, 1036, 1040, 33, 21, 0, 0, 0, 0, 0, 0, 992, 1023, 1024, 1011, 1039, 1038, 973, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8187, 8234, 8163, 8197, 8176, 8283, 8094] avg=8192.0000 max=8283 max_rank=6 min=8094 min_rank=7 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1039, 992, 1049, 1040, 1030, 1055, 1038, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1033, 1000, 976, 1051, 1024, 1027, 1018, 23, 14, 5, 0, 0, 0, 0, 0, 1069, 1031, 1045, 1004, 1027, 1018, 976, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 961, 1048, 1001, 1006, 1025, 1041, 1006, 12, 6, 0, 0, 0, 0, 0, 0, 1036, 1029, 1028, 1035, 1057, 1002, 1049, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1040, 988, 1016, 996, 1001, 1024, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1012, 990, 1004, 1011, 1021, 1059, 1013, 38, 24, 0, 0, 0, 0, 0, 0, 992, 1080, 1061, 1049, 1001, 1003, 1044, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8189, 8202, 8146, 8251, 8106, 8165, 8243] avg=8192.0000 max=8251 max_rank=4 min=8106 min_rank=5 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1031, 1014, 1022, 1050, 976, 994, 995, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1044, 1017, 993, 1074, 1003, 1025, 995, 21, 11, 1, 0, 0, 0, 0, 0, 999, 1007, 1036, 977, 981, 1035, 1056, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1032, 1009, 986, 1045, 1012, 1091, 1011, 9, 2, 0, 0, 0, 0, 0, 0, 1034, 1054, 1013, 1040, 1008, 1012, 1051, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1062, 1011, 1062, 1067, 997, 1057, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1082, 1005, 984, 981, 1030, 987, 1020, 35, 20, 0, 0, 0, 0, 0, 0, 1008, 990, 967, 1040, 1032, 1045, 1003, 1031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8199, 8118, 8209, 8286, 8321, 8197, 8116] avg=8192.0000 max=8321 max_rank=5 min=8090 min_rank=0 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8448, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8064, 8064, 8320, 8192, 8192, 8192, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8448, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[999, 1033, 1059, 1026, 979, 1023, 1014, 1007, 977, 1011, 1038, 992, 986, 972, 1043, 1026, 1027, 1000, 1028, 1059, 1037, 1074, 1048, 1022, 1022, 1049, 1068, 1053, 978, 1014, 995, 1048, 1019, 966, 1036, 1035, 1048, 1071, 1040, 1034, 1052, 1062, 1009, 994, 1014, 1030, 1028, 1029, 1007, 1018, 1025, 1025, 977, 1022, 989, 1008, 1053, 1011, 1051, 1005, 1058, 1065, 993, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1094, 1067, 959, 1044, 1044, 1034, 1047, 1023, 1004, 1105, 1053, 1069, 1018, 1058, 1035, 1020, 1008, 1021, 1033, 1016, 1020, 1011, 990, 1069, 1041, 1046, 1050, 1026, 1017, 1034, 1041, 1066, 952, 999, 1017, 1021, 990, 1049, 971, 981, 1025, 1012, 996, 1040, 1036, 950, 1009, 1029, 963, 1055, 1046, 975, 1094, 1010, 1045, 1004, 1041, 986, 1006, 1018, 995, 994, 1063, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[987, 1022, 1003, 1042, 1018, 1017, 1059, 1060, 1014, 1057, 997, 1010, 1017, 1038, 1032, 1107, 1038, 1024, 1020, 1088, 972, 1043, 1043, 1014, 1058, 998, 1005, 1031, 1046, 985, 1047, 1039, 1001, 1034, 1019, 998, 1018, 1046, 999, 1035, 1009, 981, 1011, 1036, 998, 990, 1036, 1009, 1015, 1007, 1087, 1041, 1003, 1008, 1056, 1057, 1028, 1012, 1017, 1013, 1006, 1020, 1036, 979] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1002, 986, 970, 1041, 1077, 1006, 1058, 981, 1034, 1025, 1075, 1034, 1022, 1010, 1023, 1023, 1032, 973, 985, 1056, 1047, 1019, 1015, 1015, 1022, 1004, 1008, 1012, 1024, 1021, 1025, 1037, 1034, 1039, 994, 1029, 976, 1006, 1039, 1018, 1039, 1065, 1053, 1010, 1036, 1059, 984, 1050, 1009, 1033, 1049, 989, 1028, 1010, 1004, 1042, 1055, 1046, 1048, 999, 1032, 1046, 1067, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1011, 985, 1046, 1049, 1026, 1040, 989, 1041, 1075, 1016, 967, 1090, 995, 1000, 1037, 1077, 1025, 1006, 1018, 1049, 1046, 1019, 1035, 1013, 1005, 1002, 998, 1041, 1073, 1020, 1027, 1036, 1022, 1048, 1040, 1008, 1032, 987, 1022, 992, 1035, 1038, 980, 995, 997, 1043, 1019, 1010, 1015, 1066, 1030, 1021, 1031, 1018, 1055, 1022, 998, 1032, 1046, 1018, 1020, 1005, 1017, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[992, 1029, 1059, 1016, 1005, 1069, 1033, 1001, 1011, 1035, 1056, 1021, 1036, 1016, 1044, 1027, 1013, 1055, 1046, 977, 1018, 1003, 979, 1026, 1029, 1058, 1024, 1019, 1005, 990, 1033, 1063, 1053, 1039, 1052, 1006, 995, 1007, 991, 1031, 1058, 1062, 1067, 1060, 1005, 1039, 1024, 1028, 1039, 996, 1003, 996, 1014, 991, 1001, 1040, 1029, 1019, 1006, 977, 1029, 1047, 1029, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[976, 1049, 1056, 1073, 1031, 1084, 1040, 978, 1068, 1015, 1076, 999, 1047, 1058, 983, 1049, 977, 1031, 1020, 1002, 991, 965, 1034, 1091, 1036, 1014, 1009, 1030, 1041, 1019, 1067, 1035, 1059, 995, 981, 1014, 1030, 1006, 1040, 1033, 1027, 1067, 1026, 1031, 1023, 979, 978, 995, 1065, 1034, 1015, 1056, 992, 1000, 998, 1029, 1054, 1011, 999, 1028, 977, 985, 1022, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1060, 1017, 994, 992, 1016, 1066, 1029, 1037, 1029, 1028, 1036, 973, 1000, 1048, 992, 1010, 1034, 1056, 1015, 1005, 980, 1020, 991, 989, 1044, 1033, 974, 1013, 1010, 1049, 1022, 1033, 1050, 1039, 1099, 993, 1069, 1023, 1030, 1032, 1028, 1024, 1021, 1009, 1007, 1005, 1042, 1001, 996, 1011, 1018, 1050, 1014, 989, 1070, 1029, 1056, 1043, 1045, 1059, 1027, 1002, 1050, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[999, 1033, 1059, 1026, 979, 1007, 1014, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1011, 1038, 992, 986, 972, 1043, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1000, 1028, 1059, 1037, 1074, 1048, 1022, 12, 7, 0, 0, 0, 0, 0, 0, 1022, 1049, 1068, 1053, 978, 1014, 995, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 966, 1036, 1035, 1048, 1071, 1040, 1034, 25, 0, 0, 0, 0, 0, 0, 0, 1052, 1062, 1009, 994, 1014, 1030, 1028, 1029, 16, 0, 0, 0, 0, 0, 0, 0, 1007, 1018, 1025, 1025, 977, 1022, 989, 1008, 12, 4, 0, 0, 0, 0, 0, 0, 1053, 1011, 1051, 1005, 1058, 1065, 993, 1055, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8008, 8314, 8200, 8274, 8234, 8087, 8295] avg=8192.0000 max=8314 max_rank=2 min=8008 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1002, 986, 970, 1041, 1077, 991, 1058, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1025, 1075, 1034, 1022, 1010, 1023, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 973, 985, 1056, 1047, 1019, 1015, 1015, 12, 7, 0, 0, 0, 0, 0, 0, 1022, 1004, 1008, 1012, 1024, 1021, 1025, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1039, 994, 1029, 976, 1006, 1039, 1018, 25, 0, 0, 0, 0, 0, 0, 0, 1039, 1065, 1053, 1010, 1036, 1059, 984, 1050, 16, 0, 0, 0, 0, 0, 0, 0, 1009, 1033, 1049, 989, 1028, 1010, 1004, 1042, 11, 4, 0, 0, 0, 0, 0, 0, 1055, 1046, 1048, 999, 1032, 1046, 1067, 986, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 8209, 8161, 8126, 8160, 8312, 8179, 8283] avg=8192.0000 max=8312 max_rank=5 min=8106 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1011, 985, 1046, 1049, 1026, 1024, 989, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1016, 967, 1090, 995, 1000, 1037, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1006, 1018, 1049, 1046, 1019, 1035, 1013, 12, 7, 0, 0, 0, 0, 0, 0, 1005, 1002, 998, 1041, 1073, 1020, 1027, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1048, 1040, 1008, 1032, 987, 1022, 992, 26, 0, 0, 0, 0, 0, 0, 0, 1035, 1038, 980, 995, 997, 1043, 1019, 1010, 16, 0, 0, 0, 0, 0, 0, 0, 1015, 1066, 1030, 1021, 1031, 1018, 1055, 1022, 12, 4, 0, 0, 0, 0, 0, 0, 998, 1032, 1046, 1018, 1020, 1005, 1017, 1017, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8219, 8230, 8175, 8177, 8133, 8274, 8157] avg=8192.0000 max=8274 max_rank=6 min=8133 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1094, 1067, 959, 1044, 1044, 1018, 1047, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1105, 1053, 1069, 1018, 1058, 1035, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1021, 1033, 1016, 1020, 1011, 990, 1069, 12, 7, 0, 0, 0, 0, 0, 0, 1041, 1046, 1050, 1026, 1017, 1034, 1041, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 952, 999, 1017, 1021, 990, 1049, 971, 981, 24, 0, 0, 0, 0, 0, 0, 0, 1025, 1012, 996, 1040, 1036, 950, 1009, 1029, 16, 0, 0, 0, 0, 0, 0, 0, 963, 1055, 1046, 975, 1094, 1010, 1045, 1004, 12, 4, 0, 0, 0, 0, 0, 0, 1041, 986, 1006, 1018, 995, 994, 1063, 1001, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8326, 8187, 8294, 8004, 8113, 8208, 8108] avg=8192.0000 max=8326 max_rank=1 min=8004 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[992, 1029, 1059, 1016, 1005, 1053, 1033, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1035, 1056, 1021, 1036, 1016, 1044, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1055, 1046, 977, 1018, 1003, 979, 1026, 12, 7, 0, 0, 0, 0, 0, 0, 1029, 1058, 1024, 1019, 1005, 990, 1033, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1039, 1052, 1006, 995, 1007, 991, 1031, 25, 0, 0, 0, 0, 0, 0, 0, 1058, 1062, 1067, 1060, 1005, 1039, 1024, 1028, 16, 0, 0, 0, 0, 0, 0, 0, 1039, 996, 1003, 996, 1014, 991, 1001, 1040, 12, 4, 0, 0, 0, 0, 0, 0, 1029, 1019, 1006, 977, 1029, 1047, 1029, 1015, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8209, 8136, 8194, 8199, 8359, 8096, 8155] avg=8192.0000 max=8359 max_rank=5 min=8096 min_rank=6 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[987, 1022, 1003, 1042, 1018, 1002, 1059, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1057, 997, 1010, 1017, 1038, 1032, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1024, 1020, 1088, 972, 1043, 1043, 1014, 13, 7, 0, 0, 0, 0, 0, 0, 1058, 998, 1005, 1031, 1046, 985, 1047, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1034, 1019, 998, 1018, 1046, 999, 1035, 27, 0, 0, 0, 0, 0, 0, 0, 1009, 981, 1011, 1036, 998, 990, 1036, 1009, 16, 0, 0, 0, 0, 0, 0, 0, 1015, 1007, 1087, 1041, 1003, 1008, 1056, 1057, 11, 4, 0, 0, 0, 0, 0, 0, 1028, 1012, 1017, 1013, 1006, 1020, 1036, 979, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8232, 8262, 8182, 8177, 8086, 8289, 8115] avg=8192.0000 max=8289 max_rank=6 min=8086 min_rank=5 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[976, 1049, 1056, 1073, 1031, 1068, 1040, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1015, 1076, 999, 1047, 1058, 983, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1031, 1020, 1002, 991, 965, 1034, 1091, 12, 7, 0, 0, 0, 0, 0, 0, 1036, 1014, 1009, 1030, 1041, 1019, 1067, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 995, 981, 1014, 1030, 1006, 1040, 1033, 25, 0, 0, 0, 0, 0, 0, 0, 1027, 1067, 1026, 1031, 1023, 979, 978, 995, 16, 0, 0, 0, 0, 0, 0, 0, 1065, 1034, 1015, 1056, 992, 1000, 998, 1029, 12, 4, 0, 0, 0, 0, 0, 0, 1054, 1011, 999, 1028, 977, 985, 1022, 1043, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8258, 8130, 8224, 8183, 8142, 8205, 8123] avg=8192.0000 max=8271 max_rank=0 min=8123 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32653 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1060, 1017, 994, 992, 1016, 1042, 1029, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1028, 1036, 973, 1000, 1048, 992, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1056, 1015, 1005, 980, 1020, 991, 989, 14, 12, 0, 0, 0, 0, 0, 0, 1044, 1033, 974, 1013, 1010, 1049, 1022, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1039, 1099, 993, 1069, 1023, 1030, 1032, 27, 0, 0, 0, 0, 0, 0, 0, 1028, 1024, 1021, 1009, 1007, 1005, 1042, 1001, 20, 0, 0, 0, 0, 0, 0, 0, 996, 1011, 1018, 1050, 1014, 989, 1070, 1029, 16, 5, 0, 0, 0, 0, 0, 0, 1056, 1043, 1045, 1059, 1027, 1002, 1050, 1010, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8075, 8116, 8141, 8362, 8157, 8198, 8300] avg=8192.0000 max=8362 max_rank=4 min=8075 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=33017 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8064, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1048, 1013, 1060, 1016, 1034, 975, 961, 1007, 1039, 1036, 995, 994, 1042, 1047, 990, 1031, 1014, 1032, 1020, 1010, 1004, 1003, 1067, 1033, 1052, 1064, 1059, 1013, 1023, 1019, 1033, 1045, 967, 1014, 1046, 972, 1023, 980, 1080, 1017, 1044, 1055, 1015, 1054, 1010, 971, 1005, 1042, 1006, 1033, 980, 991, 1060, 1020, 979, 1043, 1089, 1061, 1010, 997, 988, 1084, 1094, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1013, 991, 1019, 1010, 1052, 1025, 1029, 1034, 1049, 1052, 1031, 995, 1041, 1031, 1008, 1038, 1049, 1006, 1008, 1067, 1017, 1045, 1010, 1046, 990, 1009, 1069, 1042, 1050, 1026, 1056, 996, 1044, 1010, 1044, 1010, 1007, 1040, 1048, 1014, 995, 1003, 1035, 992, 982, 998, 1032, 1017, 1032, 972, 996, 1019, 1006, 1039, 1002, 1022, 1011, 1020, 1014, 1063, 991, 1073, 1039, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1040, 1011, 1066, 1001, 1016, 1062, 1001, 1005, 1046, 994, 982, 1031, 1024, 1056, 1082, 1088, 1052, 1002, 997, 1012, 982, 1021, 1058, 1023, 997, 1035, 1050, 976, 1012, 981, 1032, 1021, 1011, 1032, 1047, 1019, 1015, 1045, 1077, 984, 1026, 1031, 1044, 984, 1030, 1050, 1033, 1072, 1036, 1015, 1030, 1011, 996, 1032, 1020, 1001, 1006, 986, 1012, 1008, 1078, 1006, 1012, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1030, 1048, 1060, 1007, 1006, 1083, 971, 1100, 1032, 1041, 1071, 997, 1047, 1007, 988, 1030, 1031, 1044, 997, 1015, 1041, 980, 1050, 1070, 994, 976, 989, 1019, 1032, 981, 962, 1010, 1069, 1030, 1015, 1036, 1040, 1001, 1039, 1044, 1042, 1037, 1064, 1069, 984, 1040, 976, 1005, 1047, 1000, 1018, 1019, 1007, 1010, 974, 998, 1044, 1042, 1008, 1039, 1016, 1004, 1072, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1052, 1059, 1057, 1047, 1023, 1039, 1040, 984, 1059, 997, 1014, 950, 1035, 1020, 1025, 1040, 987, 952, 1052, 1019, 980, 1041, 1009, 1050, 1063, 1043, 989, 1027, 1020, 1040, 1034, 1033, 1052, 1037, 1063, 1042, 1043, 1022, 1001, 1032, 1036, 1000, 1032, 1037, 1017, 1052, 1022, 1047, 1017, 1023, 963, 1003, 1068, 1017, 984, 1055, 1048, 982, 1013, 986, 1003, 1013, 1021, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[986, 1037, 1023, 1020, 1022, 1023, 1030, 1009, 1009, 1042, 1020, 1073, 1014, 972, 1056, 991, 1048, 1023, 1040, 997, 1014, 1021, 1047, 1029, 1032, 1032, 1016, 1088, 973, 1062, 1051, 1012, 1003, 1012, 1017, 1045, 980, 1015, 999, 997, 1059, 1008, 1013, 1026, 1010, 1034, 1000, 1085, 1076, 1037, 1012, 949, 996, 1022, 1010, 1033, 1083, 1046, 1005, 1014, 1006, 1058, 1046, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1008, 1027, 1030, 993, 1044, 1022, 1028, 1049, 1054, 1021, 1080, 1031, 1037, 1061, 1044, 996, 1027, 1033, 1037, 1007, 1034, 968, 1045, 1049, 1031, 993, 991, 940, 1003, 1023, 1033, 1043, 986, 1020, 1074, 1011, 1044, 1017, 955, 1061, 1024, 1069, 980, 1039, 1058, 1007, 1041, 1047, 1024, 1032, 1074, 1009, 1017, 1044, 1007, 991, 987, 1013, 1046, 1000, 987, 1071, 1025, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1022, 1035, 1005, 1003, 1031, 1018, 1066, 994, 1055, 1056, 1030, 1034, 1085, 983, 991, 1010, 1045, 995, 951, 1034, 1016, 1000, 1013, 1026, 996, 1046, 986, 1053, 1054, 1066, 975, 1035, 1055, 967, 1016, 1051, 1021, 1074, 1012, 1029, 1021, 973, 1015, 1041, 990, 1033, 1004, 1010, 1026, 1025, 1046, 995, 1033, 1074, 1077, 1063, 1028, 979, 1056, 1037, 1014, 1030, 1009, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1040, 1011, 1056, 1001, 1016, 1062, 1001, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 994, 982, 1031, 1024, 1056, 1082, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1002, 997, 1012, 982, 1021, 1058, 1023, 15, 1, 0, 0, 0, 0, 0, 0, 997, 1035, 1050, 976, 1012, 981, 1032, 1021, 9, 4, 3, 0, 0, 0, 0, 0, 1011, 1032, 1043, 1019, 1015, 1045, 1077, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1031, 1044, 984, 1030, 1050, 1033, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1015, 1030, 1011, 996, 1032, 1020, 1001, 35, 4, 0, 0, 0, 0, 0, 0, 1006, 986, 1012, 1008, 1078, 987, 1012, 1031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8268, 8163, 8120, 8226, 8267, 8180, 8120] avg=8192.0000 max=8268 max_rank=1 min=8120 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1048, 1013, 1050, 1016, 1034, 975, 961, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1036, 995, 994, 1042, 1047, 990, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1032, 1020, 1010, 1004, 1003, 1067, 1033, 16, 1, 0, 0, 0, 0, 0, 0, 1052, 1064, 1059, 1013, 1023, 1019, 1033, 1045, 9, 4, 3, 0, 0, 0, 0, 0, 967, 1014, 1042, 972, 1023, 980, 1080, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1055, 1015, 1054, 1010, 971, 1005, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1033, 980, 991, 1060, 1020, 979, 1043, 35, 4, 0, 0, 0, 0, 0, 0, 1089, 1061, 1010, 997, 988, 1064, 1094, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8139, 8200, 8324, 8095, 8193, 8151, 8330] avg=8192.0000 max=8330 max_rank=7 min=8095 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1013, 991, 1010, 1010, 1052, 1025, 1029, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1052, 1031, 995, 1041, 1031, 1008, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1006, 1008, 1067, 1017, 1045, 1010, 1046, 16, 1, 0, 0, 0, 0, 0, 0, 990, 1009, 1069, 1042, 1050, 1026, 1056, 996, 8, 4, 3, 0, 0, 0, 0, 0, 1044, 1010, 1040, 1010, 1007, 1040, 1048, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1003, 1035, 992, 982, 998, 1032, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 972, 996, 1019, 1006, 1039, 1002, 1022, 35, 4, 0, 0, 0, 0, 0, 0, 1011, 1020, 1014, 1063, 991, 1053, 1039, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8210, 8265, 8253, 8213, 8051, 8127, 8253] avg=8192.0000 max=8265 max_rank=2 min=8051 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1052, 1059, 1047, 1047, 1023, 1039, 1040, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 997, 1014, 950, 1035, 1020, 1025, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 987, 952, 1052, 1019, 980, 1041, 1009, 1050, 15, 1, 0, 0, 0, 0, 0, 0, 1063, 1043, 989, 1027, 1020, 1040, 1034, 1033, 9, 4, 3, 0, 0, 0, 0, 0, 1052, 1037, 1059, 1042, 1043, 1022, 1001, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1000, 1032, 1037, 1017, 1052, 1022, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1023, 963, 1003, 1068, 1017, 984, 1055, 36, 4, 0, 0, 0, 0, 0, 0, 1048, 982, 1013, 986, 1003, 994, 1021, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8104, 8106, 8265, 8288, 8240, 8170, 8072] avg=8192.0000 max=8291 max_rank=0 min=8072 min_rank=7 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1008, 1027, 1021, 993, 1044, 1022, 1028, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1021, 1080, 1031, 1037, 1061, 1044, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1033, 1037, 1007, 1034, 968, 1045, 1049, 16, 1, 0, 0, 0, 0, 0, 0, 1031, 993, 991, 940, 1003, 1023, 1033, 1043, 8, 4, 3, 0, 0, 0, 0, 0, 986, 1020, 1070, 1011, 1044, 1017, 955, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1069, 980, 1039, 1058, 1007, 1041, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1032, 1074, 1009, 1017, 1044, 1007, 991, 35, 4, 0, 0, 0, 0, 0, 0, 987, 1013, 1046, 1000, 987, 1051, 1025, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8289, 8217, 8072, 8164, 8262, 8237, 8103] avg=8192.0000 max=8289 max_rank=1 min=8072 min_rank=3 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1030, 1048, 1050, 1007, 1006, 1083, 971, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1041, 1071, 997, 1047, 1007, 988, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1044, 997, 1015, 1041, 980, 1050, 1070, 15, 1, 0, 0, 0, 0, 0, 0, 994, 976, 989, 1019, 1032, 981, 962, 1010, 9, 4, 3, 0, 0, 0, 0, 0, 1069, 1030, 1011, 1036, 1040, 1001, 1039, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1037, 1064, 1069, 984, 1040, 976, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1000, 1018, 1019, 1007, 1010, 974, 998, 35, 4, 0, 0, 0, 0, 0, 0, 1044, 1042, 1008, 1039, 1016, 985, 1072, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8178, 8244, 7979, 8270, 8214, 8112, 8244] avg=8192.0000 max=8295 max_rank=0 min=7979 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[986, 1037, 1014, 1020, 1022, 1023, 1030, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1042, 1020, 1073, 1014, 972, 1056, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1023, 1040, 997, 1014, 1021, 1047, 1029, 16, 1, 0, 0, 0, 0, 0, 0, 1032, 1032, 1016, 1088, 973, 1062, 1051, 1012, 8, 4, 4, 0, 0, 0, 0, 0, 1003, 1012, 1013, 1045, 980, 1015, 999, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1008, 1013, 1026, 1010, 1034, 1000, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1037, 1012, 949, 996, 1022, 1010, 1033, 34, 4, 0, 0, 0, 0, 0, 0, 1083, 1046, 1005, 1014, 1006, 1038, 1046, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8143, 8236, 8282, 8064, 8231, 8173, 8266] avg=8192.0000 max=8282 max_rank=3 min=8064 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1022, 1035, 988, 1003, 1031, 1018, 1066, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1056, 1030, 1034, 1085, 983, 991, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 995, 951, 1034, 1016, 1000, 1013, 1026, 20, 5, 0, 0, 0, 0, 0, 0, 996, 1046, 986, 1053, 1054, 1066, 975, 1035, 12, 9, 9, 0, 0, 0, 0, 0, 1055, 967, 1007, 1051, 1021, 1074, 1012, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 973, 1015, 1041, 990, 1033, 1004, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1025, 1046, 995, 1033, 1074, 1077, 1063, 39, 8, 0, 0, 0, 0, 0, 0, 1028, 979, 1056, 1037, 1014, 1002, 1009, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8205, 8105, 8241, 8216, 8078, 8386, 8148] avg=8192.0000 max=8386 max_rank=6 min=8078 min_rank=5 imbalance_ratio=1.023682 max_load_reduction=57150 max_load_reduction_pct=0.872040 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8064, 8192, 8320, 8064, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8320, 8448, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8192, 8448, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1019, 1060, 1038, 1011, 1020, 1051, 1020, 1051, 998, 1052, 1038, 969, 1028, 990, 1040, 1032, 1016, 977, 984, 1029, 1010, 1002, 1012, 1017, 1043, 1019, 1082, 1004, 1021, 1027, 1048, 1047, 1013, 1015, 1029, 985, 1031, 1022, 1010, 984, 1018, 1000, 992, 1034, 1039, 1043, 1020, 1021, 1007, 1013, 1041, 1039, 1040, 1017, 1055, 1054, 1004, 1002, 1017, 1072, 1016, 1007, 1091, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1029, 1060, 1023, 1005, 1021, 1019, 1062, 977, 1015, 1039, 997, 1020, 993, 1046, 1026, 1015, 1004, 1048, 1051, 1030, 1052, 1014, 1003, 1023, 990, 1007, 1035, 1015, 1056, 1049, 937, 1071, 1013, 1022, 988, 1045, 1012, 996, 993, 1048, 1071, 992, 1041, 1008, 1066, 1063, 1060, 1016, 971, 1015, 1017, 1062, 1062, 1005, 1001, 1053, 1031, 1025, 1023, 997, 1050, 1019, 1022, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1057, 1034, 1045, 1001, 996, 971, 1050, 1000, 1022, 1027, 1023, 1016, 1007, 1028, 1069, 1031, 1008, 1061, 1036, 1037, 994, 1006, 1028, 978, 1051, 1047, 1043, 1041, 1035, 1036, 1027, 1005, 1033, 988, 952, 1012, 992, 1034, 999, 1029, 1046, 1009, 1071, 997, 1058, 1029, 1034, 1028, 993, 1036, 1026, 1036, 1000, 1025, 1035, 996, 1039, 1036, 1030, 1090, 1024, 1064, 974, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1025, 1047, 1016, 1049, 981, 1079, 1032, 1019, 1018, 1078, 1022, 1025, 1074, 1002, 1013, 975, 1002, 1055, 1072, 981, 1037, 1019, 1023, 995, 1014, 1025, 1025, 1012, 1014, 1075, 1049, 995, 1006, 1024, 1017, 1021, 1023, 1036, 1029, 1020, 1006, 1025, 980, 1026, 1026, 1009, 1034, 1058, 1001, 1028, 999, 977, 1052, 1028, 992, 969, 1050, 1037, 1022, 1068, 1016, 992, 1077, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1043, 1002, 1026, 1049, 1021, 1047, 1018, 1007, 1045, 967, 1011, 1014, 1044, 1059, 978, 1081, 1002, 1042, 1032, 1071, 1058, 1029, 1054, 996, 1060, 1030, 1007, 1019, 1031, 1056, 990, 1015, 1004, 965, 973, 1025, 1056, 1004, 1049, 979, 1008, 1046, 1011, 1011, 1012, 1025, 1046, 1034, 1036, 1061, 981, 1028, 1011, 1080, 1012, 997, 1006, 1028, 1020, 1073, 1015, 1023, 981, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1003, 1040, 1077, 1006, 1036, 1016, 1037, 1020, 1013, 992, 1008, 1057, 1036, 1005, 1046, 1041, 968, 1012, 1081, 1068, 965, 1046, 1013, 1050, 1007, 999, 1029, 1007, 1041, 1024, 1041, 1071, 1094, 1003, 1004, 1034, 1063, 1006, 1020, 1019, 980, 1029, 1026, 1005, 1020, 1032, 1046, 1018, 1003, 1039, 1041, 1032, 1037, 977, 997, 1015, 1000, 1015, 1030, 1029, 1056, 999, 1007, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[998, 985, 1060, 981, 1086, 1005, 1006, 1041, 1004, 1017, 1067, 1010, 1044, 1078, 1009, 1018, 1010, 1079, 1062, 1020, 1023, 984, 1018, 1027, 1029, 1016, 1013, 1026, 1005, 986, 1029, 986, 1013, 1029, 1058, 1020, 1044, 996, 1012, 1025, 1012, 1018, 989, 1013, 1019, 965, 1045, 1102, 992, 986, 1042, 1055, 1035, 1012, 1060, 1076, 1021, 999, 1029, 1053, 1008, 996, 1049, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1041, 1025, 997, 1046, 982, 1007, 991, 1083, 1110, 1039, 971, 1010, 997, 989, 1066, 1022, 1002, 1035, 1065, 1032, 984, 985, 1009, 1001, 1047, 1013, 1060, 994, 1002, 1033, 1015, 1052, 1007, 997, 1037, 1025, 1033, 1014, 1032, 1076, 1027, 1067, 1031, 1045, 1013, 998, 955, 1005, 1043, 1038, 1028, 1040, 1042, 1025, 945, 1003, 1063, 1053, 988, 1079, 1009, 1028, 1045, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1029, 1060, 1010, 1005, 1021, 1019, 1062, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1039, 997, 1020, 993, 1046, 1022, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1048, 1051, 1030, 1052, 1014, 1003, 1023, 4, 4, 3, 0, 0, 0, 0, 0, 990, 1007, 1019, 1015, 1056, 1049, 937, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1022, 988, 1045, 1012, 996, 993, 1048, 34, 12, 0, 0, 0, 0, 0, 0, 1071, 992, 1041, 1008, 1066, 1063, 1060, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1015, 1017, 1062, 1062, 1005, 1001, 1053, 10, 4, 0, 0, 0, 0, 0, 0, 1031, 1025, 1023, 963, 1050, 1019, 1022, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8147, 8236, 8144, 8163, 8313, 8200, 8150] avg=8192.0000 max=8313 max_rank=5 min=8144 min_rank=3 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 1047, 1003, 1049, 981, 1079, 1032, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1078, 1022, 1025, 1074, 1002, 1009, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1055, 1072, 981, 1037, 1019, 1023, 995, 4, 4, 3, 0, 0, 0, 0, 0, 1014, 1025, 1009, 1012, 1014, 1075, 1049, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1024, 1017, 1021, 1023, 1036, 1029, 1020, 37, 12, 0, 0, 0, 0, 0, 0, 1006, 1025, 980, 1026, 1026, 1009, 1034, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1028, 999, 977, 1052, 1028, 992, 969, 10, 4, 0, 0, 0, 0, 0, 0, 1050, 1037, 1022, 1031, 1016, 992, 1077, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8203, 8195, 8193, 8225, 8160, 8060, 8265] avg=8192.0000 max=8265 max_rank=7 min=8060 min_rank=6 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1043, 1002, 1013, 1049, 1021, 1047, 1018, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 967, 1011, 1014, 1044, 1059, 974, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1042, 1032, 1071, 1058, 1029, 1054, 996, 4, 4, 3, 0, 0, 0, 0, 0, 1060, 1030, 991, 1019, 1031, 1056, 990, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 965, 973, 1025, 1056, 1004, 1049, 979, 37, 12, 0, 0, 0, 0, 0, 0, 1008, 1046, 1011, 1011, 1012, 1025, 1046, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1061, 981, 1028, 1011, 1080, 1012, 997, 10, 4, 0, 0, 0, 0, 0, 0, 1006, 1028, 1020, 1036, 1015, 1023, 981, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8195, 8295, 8192, 8104, 8189, 8220, 8141] avg=8192.0000 max=8295 max_rank=2 min=8104 min_rank=4 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32654 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1019, 1060, 1024, 1011, 1020, 1051, 1020, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1052, 1038, 969, 1028, 990, 1035, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 977, 984, 1029, 1010, 1002, 1012, 1017, 5, 4, 4, 0, 0, 0, 0, 0, 1043, 1019, 1064, 1004, 1021, 1027, 1048, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1015, 1029, 985, 1031, 1022, 1010, 984, 37, 13, 0, 0, 0, 0, 0, 0, 1018, 1000, 992, 1034, 1039, 1043, 1020, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1013, 1041, 1039, 1040, 1017, 1055, 1054, 10, 5, 0, 0, 0, 0, 0, 0, 1004, 1002, 1017, 1035, 1016, 1007, 1091, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8142, 8060, 8273, 8139, 8163, 8281, 8222] avg=8192.0000 max=8281 max_rank=6 min=8060 min_rank=2 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1041, 1025, 985, 1046, 982, 1007, 991, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1110, 1039, 971, 1010, 997, 989, 1061, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1035, 1065, 1032, 984, 985, 1009, 1001, 5, 4, 3, 0, 0, 0, 0, 0, 1047, 1013, 1042, 994, 1002, 1033, 1015, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 997, 1037, 1025, 1033, 1014, 1032, 1076, 37, 13, 0, 0, 0, 0, 0, 0, 1027, 1067, 1031, 1045, 1013, 998, 955, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1038, 1028, 1040, 1042, 1025, 945, 1003, 9, 5, 0, 0, 0, 0, 0, 0, 1063, 1053, 988, 1042, 1009, 1028, 1045, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8199, 8125, 8198, 8271, 8137, 8178, 8268] avg=8192.0000 max=8271 max_rank=4 min=8125 min_rank=2 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1057, 1034, 1031, 1001, 996, 971, 1050, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1027, 1023, 1016, 1007, 1028, 1064, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1061, 1036, 1037, 994, 1006, 1028, 978, 5, 4, 4, 0, 0, 0, 0, 0, 1051, 1047, 1026, 1041, 1035, 1036, 1027, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 988, 952, 1012, 992, 1034, 999, 1029, 38, 12, 0, 0, 0, 0, 0, 0, 1046, 1009, 1071, 997, 1058, 1029, 1034, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1036, 1026, 1036, 1000, 1025, 1035, 996, 10, 5, 0, 0, 0, 0, 0, 0, 1039, 1036, 1030, 1052, 1024, 1064, 974, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8218, 8161, 8268, 8089, 8268, 8162, 8230] avg=8192.0000 max=8268 max_rank=3 min=8089 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1003, 1040, 1063, 1006, 1036, 1016, 1037, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 992, 1008, 1057, 1036, 1005, 1041, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1012, 1081, 1068, 965, 1046, 1013, 1050, 5, 4, 4, 0, 0, 0, 0, 0, 1007, 999, 1013, 1007, 1041, 1024, 1041, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 1003, 1004, 1034, 1063, 1006, 1020, 1019, 36, 12, 0, 0, 0, 0, 0, 0, 980, 1029, 1026, 1005, 1020, 1032, 1046, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1039, 1041, 1032, 1037, 977, 997, 1015, 10, 4, 0, 0, 0, 0, 0, 0, 1000, 1015, 1030, 993, 1056, 999, 1007, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8193, 8216, 8203, 8291, 8152, 8155, 8105] avg=8192.0000 max=8291 max_rank=4 min=8105 min_rank=7 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[998, 985, 1039, 981, 1086, 1005, 1006, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1017, 1067, 1010, 1044, 1078, 1001, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1079, 1062, 1020, 1023, 984, 1018, 1027, 8, 9, 8, 0, 0, 0, 0, 0, 1029, 1016, 988, 1026, 1005, 986, 1029, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1029, 1058, 1020, 1044, 996, 1012, 1025, 41, 16, 0, 0, 0, 0, 0, 0, 1012, 1018, 989, 1013, 1019, 965, 1045, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 992, 986, 1042, 1055, 1035, 1012, 1060, 1076, 13, 9, 0, 0, 0, 0, 0, 0, 1021, 999, 1029, 1012, 1008, 996, 1049, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8239, 8248, 8065, 8254, 8154, 8280, 8155] avg=8192.0000 max=8280 max_rank=6 min=8065 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8064, 8192, 8320, 8192, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8448, 8320, 8192, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1058, 1028, 1021, 999, 1034, 1035, 1001, 999, 1042, 1013, 976, 1003, 1049, 1057, 1009, 992, 1013, 1044, 998, 1027, 1004, 1010, 1034, 981, 1056, 1043, 990, 1023, 1029, 1025, 1071, 1040, 1004, 1017, 1045, 1004, 1006, 1014, 1029, 1020, 995, 1025, 1073, 1055, 1008, 1021, 986, 1048, 1033, 997, 1020, 1044, 1009, 1038, 1003, 1032, 1051, 1103, 1000, 1036, 1024, 1018, 1010, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1039, 1001, 1030, 1059, 1014, 1000, 1062, 1026, 1035, 1082, 955, 1022, 1003, 1009, 1075, 1033, 1033, 978, 1013, 1031, 985, 1037, 1023, 1047, 998, 1042, 1072, 985, 970, 974, 1029, 1026, 976, 997, 1027, 1075, 1030, 1045, 1040, 1016, 1017, 1035, 1040, 1026, 1029, 1004, 1042, 1047, 1034, 957, 984, 1011, 1030, 1075, 1014, 1051, 1093, 1017, 1125, 1046, 952, 1013, 1009, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1074, 1049, 1001, 1052, 1011, 1039, 1026, 1030, 991, 1014, 1026, 1016, 1089, 1045, 995, 1027, 1036, 1031, 1019, 1007, 1050, 1019, 1041, 1058, 1047, 1023, 972, 995, 1063, 1023, 1050, 994, 1000, 1023, 1048, 1064, 1011, 1041, 1020, 998, 1044, 1033, 1009, 1026, 1007, 982, 999, 998, 1009, 1025, 969, 1022, 1005, 978, 1024, 1019, 1069, 1088, 1004, 1044, 1058, 1051, 994, 961] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1040, 1041, 1035, 1013, 1010, 1006, 946, 1076, 987, 1096, 988, 1048, 984, 991, 1055, 1037, 1039, 998, 1045, 1049, 1016, 1006, 1080, 1037, 992, 1035, 989, 991, 1043, 999, 963, 1037, 1042, 998, 984, 1005, 1002, 1031, 1017, 1021, 1043, 1042, 1006, 1014, 1015, 1059, 997, 1021, 1036, 1077, 1052, 1056, 991, 1019, 1069, 1072, 1086, 1031, 992, 992, 1009, 1026, 1070, 989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[993, 1023, 995, 996, 1030, 1032, 1039, 995, 1068, 1022, 1044, 1025, 1031, 1035, 1051, 1045, 1045, 1050, 978, 1034, 1038, 1029, 1020, 992, 1037, 1043, 1075, 1030, 1027, 1012, 978, 1035, 985, 1011, 970, 980, 1057, 1028, 1062, 1019, 997, 987, 1035, 1089, 998, 969, 1011, 978, 1055, 1008, 1038, 1002, 1070, 1058, 1048, 1003, 1006, 1044, 1024, 1042, 1016, 1021, 1049, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1009, 1019, 988, 1016, 1057, 1018, 999, 1057, 1027, 1082, 1011, 1050, 1016, 1023, 1057, 1019, 1049, 1011, 1018, 1027, 1038, 1029, 1033, 1033, 993, 990, 1026, 985, 1071, 1061, 1033, 1025, 971, 1031, 1021, 999, 953, 1006, 995, 1027, 1007, 1049, 978, 1050, 1010, 1036, 1027, 1019, 1025, 1009, 1026, 1074, 1046, 1045, 961, 1048, 1026, 1045, 1055, 1063, 969, 1017, 1009, 1069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[991, 1010, 953, 989, 1015, 1019, 991, 1055, 991, 1021, 1040, 1057, 1037, 1000, 1064, 1026, 1043, 1068, 1048, 1030, 986, 1051, 971, 1053, 1012, 1018, 1072, 1033, 1049, 1013, 1006, 994, 1063, 1080, 1000, 1072, 1038, 978, 1021, 1012, 1007, 1032, 1004, 987, 1045, 1016, 1047, 1030, 1075, 1049, 1061, 1007, 1073, 1071, 1013, 969, 999, 987, 983, 1019, 1046, 1024, 994, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1011, 984, 1017, 1011, 1041, 983, 1023, 1012, 1019, 1025, 1036, 1008, 984, 1009, 1036, 1014, 1060, 1052, 1000, 1033, 1002, 1002, 996, 1022, 1042, 968, 1015, 961, 1051, 1018, 1024, 1094, 1042, 1019, 990, 1022, 1033, 1040, 1049, 987, 1070, 1057, 980, 1039, 1025, 1031, 1006, 1066, 1047, 1010, 1020, 1027, 1052, 1020, 1032, 1088, 1038, 984, 1042, 1019, 1017, 1085, 1039, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1074, 1049, 1001, 1052, 1011, 1039, 1026, 1030, 35, 2, 1, 0, 0, 0, 0, 0, 991, 994, 1026, 1016, 1089, 1045, 995, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1031, 1019, 1007, 1050, 1019, 1041, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1023, 972, 995, 1063, 1023, 1050, 994, 18, 0, 0, 0, 0, 0, 0, 0, 1000, 1023, 1048, 1064, 1011, 1041, 1020, 998, 39, 0, 0, 0, 0, 0, 0, 0, 1044, 1033, 1009, 1026, 1007, 982, 999, 998, 11, 1, 0, 0, 0, 0, 0, 0, 968, 1025, 969, 1022, 1005, 978, 1024, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1088, 1004, 1044, 1058, 1051, 994, 961, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8183, 8250, 8185, 8244, 8110, 8010, 8234] avg=8192.0000 max=8320 max_rank=0 min=8010 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32598 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1058, 1028, 1021, 999, 1034, 1035, 1001, 999, 34, 2, 1, 0, 0, 0, 0, 0, 1042, 993, 976, 1003, 1049, 1057, 1009, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1044, 998, 1027, 1004, 1010, 1034, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1043, 990, 1023, 1029, 1025, 1071, 1040, 18, 0, 0, 0, 0, 0, 0, 0, 1004, 1017, 1045, 1004, 1006, 1014, 1029, 1020, 40, 0, 0, 0, 0, 0, 0, 0, 995, 1025, 1073, 1055, 1008, 1021, 986, 1048, 11, 1, 0, 0, 0, 0, 0, 0, 991, 997, 1020, 1044, 1009, 1038, 1003, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1103, 1000, 1036, 1024, 1018, 1010, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8121, 8100, 8295, 8179, 8223, 8134, 8272] avg=8192.0000 max=8295 max_rank=3 min=8100 min_rank=2 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[991, 1010, 953, 989, 1015, 1019, 991, 1055, 32, 3, 1, 0, 0, 0, 0, 0, 991, 1000, 1040, 1057, 1037, 1000, 1064, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1068, 1048, 1030, 986, 1051, 971, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1018, 1072, 1033, 1049, 1013, 1006, 994, 19, 0, 0, 0, 0, 0, 0, 0, 1063, 1080, 1000, 1072, 1038, 978, 1021, 1012, 42, 0, 0, 0, 0, 0, 0, 0, 1007, 1032, 1004, 987, 1045, 1016, 1047, 1030, 11, 1, 0, 0, 0, 0, 0, 0, 1030, 1049, 1061, 1007, 1073, 1071, 1013, 969, 0, 0, 0, 0, 0, 0, 0, 0, 967, 987, 983, 1019, 1046, 1024, 994, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8059, 8215, 8239, 8216, 8306, 8180, 8273, 8048] avg=8192.0000 max=8306 max_rank=4 min=8048 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1039, 1001, 1030, 1059, 1014, 1000, 1062, 1026, 35, 2, 1, 0, 0, 0, 0, 0, 1035, 1060, 955, 1022, 1003, 1009, 1075, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 978, 1013, 1031, 985, 1037, 1023, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1042, 1072, 985, 970, 974, 1029, 1026, 20, 0, 0, 0, 0, 0, 0, 0, 976, 997, 1027, 1075, 1030, 1045, 1040, 1016, 40, 0, 0, 0, 0, 0, 0, 0, 1017, 1035, 1040, 1026, 1029, 1004, 1042, 1047, 11, 1, 0, 0, 0, 0, 0, 0, 992, 957, 984, 1011, 1030, 1075, 1014, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1017, 1125, 1046, 952, 1013, 1009, 991, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8192, 8136, 8116, 8246, 8252, 8114, 8211] avg=8192.0000 max=8269 max_rank=0 min=8114 min_rank=6 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[993, 1023, 995, 996, 1030, 1032, 1039, 995, 33, 3, 1, 0, 0, 0, 0, 0, 1068, 1001, 1044, 1025, 1031, 1035, 1051, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1050, 978, 1034, 1038, 1029, 1020, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1043, 1075, 1030, 1027, 1012, 978, 1035, 19, 0, 0, 0, 0, 0, 0, 0, 985, 1011, 970, 980, 1057, 1028, 1062, 1019, 41, 0, 0, 0, 0, 0, 0, 0, 997, 987, 1035, 1089, 998, 969, 1011, 978, 11, 1, 0, 0, 0, 0, 0, 0, 1011, 1008, 1038, 1002, 1070, 1058, 1048, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1044, 1024, 1042, 1016, 1021, 1049, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8300, 8175, 8256, 8153, 8076, 8238, 8198] avg=8192.0000 max=8300 max_rank=1 min=8076 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1040, 1041, 1035, 1013, 1010, 1006, 946, 1076, 35, 2, 1, 0, 0, 0, 0, 0, 987, 1074, 988, 1048, 984, 991, 1055, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 998, 1045, 1049, 1016, 1006, 1080, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1035, 989, 991, 1043, 999, 963, 1037, 20, 0, 0, 0, 0, 0, 0, 0, 1042, 998, 984, 1005, 1002, 1031, 1017, 1021, 40, 0, 0, 0, 0, 0, 0, 0, 1043, 1042, 1006, 1014, 1015, 1059, 997, 1021, 11, 1, 0, 0, 0, 0, 0, 0, 994, 1077, 1052, 1056, 991, 1019, 1069, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1031, 992, 992, 1009, 1026, 1070, 989, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8164, 8259, 8069, 8140, 8209, 8330, 8160] avg=8192.0000 max=8330 max_rank=6 min=8069 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1009, 1019, 988, 1016, 1057, 1018, 999, 1057, 33, 2, 1, 0, 0, 0, 0, 0, 1027, 1060, 1011, 1050, 1016, 1023, 1057, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1011, 1018, 1027, 1038, 1029, 1033, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 993, 990, 1026, 985, 1071, 1061, 1033, 1025, 20, 0, 0, 0, 0, 0, 0, 0, 971, 1031, 1021, 999, 953, 1006, 995, 1027, 40, 0, 0, 0, 0, 0, 0, 0, 1007, 1049, 978, 1050, 1010, 1036, 1027, 1019, 11, 1, 0, 0, 0, 0, 0, 0, 983, 1009, 1026, 1074, 1046, 1045, 961, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1045, 1055, 1063, 969, 1017, 1009, 1069, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8263, 8227, 8204, 8043, 8188, 8192, 8220] avg=8192.0000 max=8263 max_rank=1 min=8043 min_rank=4 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1011, 984, 1017, 1011, 1041, 983, 1023, 1012, 38, 8, 4, 0, 0, 0, 0, 0, 1019, 992, 1036, 1008, 984, 1009, 1036, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1052, 1000, 1033, 1002, 1002, 996, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 968, 1015, 961, 1051, 1018, 1024, 1094, 22, 0, 0, 0, 0, 0, 0, 0, 1042, 1019, 990, 1022, 1033, 1040, 1049, 987, 43, 0, 0, 0, 0, 0, 0, 0, 1070, 1057, 980, 1039, 1025, 1031, 1006, 1066, 17, 7, 0, 0, 0, 0, 0, 0, 996, 1010, 1020, 1027, 1052, 1020, 1032, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 984, 1042, 1019, 1017, 1085, 1039, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8098, 8150, 8195, 8225, 8298, 8245, 8193] avg=8192.0000 max=8298 max_rank=5 min=8098 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8064, 8192, 8320, 8192, 8192, 8320, 384, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8064, 8320, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8448, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1002, 1024, 1020, 988, 1090, 1030, 1021, 1032, 1020, 1061, 1024, 995, 1025, 1012, 1058, 979, 1015, 1053, 973, 1034, 1084, 1056, 1058, 1035, 981, 1016, 1045, 956, 963, 1001, 1016, 952, 1024, 1014, 1064, 1114, 1018, 1035, 1050, 1035, 992, 1035, 989, 1021, 1055, 1018, 1057, 987, 1033, 987, 1017, 1043, 973, 1058, 1048, 1027, 1047, 1017, 1065, 975, 1043, 1002, 1043, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1038, 1108, 1056, 1017, 1022, 996, 1020, 986, 1038, 1052, 1004, 1031, 1032, 1024, 997, 982, 1004, 1033, 1033, 1007, 990, 1030, 1031, 1040, 1017, 1049, 1022, 1026, 1055, 1050, 1051, 1065, 1026, 968, 1030, 977, 1004, 1066, 1052, 1050, 1008, 993, 1034, 1025, 1017, 1005, 992, 1007, 1013, 983, 1040, 1010, 1041, 1000, 1029, 1086, 995, 984, 1046, 1057, 1058, 980, 1035, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1059, 1009, 999, 990, 1072, 1061, 1039, 1016, 1023, 1031, 1018, 1022, 1002, 1005, 977, 1027, 1071, 1056, 1064, 1043, 982, 1045, 999, 1097, 1037, 1019, 1010, 1058, 1017, 1036, 1005, 981, 1037, 1069, 1019, 980, 1045, 1002, 1106, 1033, 1012, 1016, 1036, 1025, 1030, 981, 1031, 1034, 992, 992, 940, 1045, 1029, 1066, 985, 1006, 989, 995, 1052, 993, 1026, 989, 1057, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1038, 1004, 965, 1034, 1039, 1028, 1059, 1035, 1011, 1015, 1021, 996, 1067, 1023, 1050, 1021, 1012, 966, 1020, 1007, 1071, 1023, 1029, 1046, 999, 966, 1000, 1011, 1043, 1030, 1021, 1044, 1056, 1026, 989, 1011, 1005, 1063, 1013, 1040, 1044, 989, 1079, 1017, 1017, 1052, 1040, 1049, 1038, 1037, 977, 1005, 1029, 1025, 1022, 1009, 1031, 980, 1080, 1018, 1051, 1016, 1015, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1040, 1024, 1036, 1064, 1016, 1034, 1037, 1020, 987, 1014, 984, 1012, 1015, 1055, 1039, 1068, 1057, 1049, 1005, 1015, 1038, 937, 1013, 1028, 1001, 1036, 1034, 1049, 1049, 1053, 954, 992, 1042, 998, 1012, 1018, 1023, 999, 1046, 1033, 1003, 1024, 1018, 1062, 1063, 1027, 1016, 1005, 1039, 985, 1050, 1077, 1037, 1062, 1013, 1016, 1060, 1002, 1020, 995, 1017, 992, 1017, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1016, 1039, 1082, 1065, 1044, 986, 1047, 1058, 1010, 1018, 1018, 991, 1000, 1014, 1015, 996, 1025, 1019, 1015, 995, 996, 1051, 1008, 1043, 966, 1027, 1007, 1013, 1032, 991, 1004, 1063, 1046, 1036, 1020, 1041, 1005, 1030, 991, 1049, 1041, 987, 1008, 1038, 1048, 1057, 1046, 1033, 1003, 987, 1067, 1043, 979, 1063, 1022, 1013, 1049, 1016, 1052, 1035, 1017, 994, 1045, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1053, 1088, 952, 1059, 1050, 1052, 1039, 976, 998, 1009, 1020, 1011, 1072, 970, 997, 1047, 1008, 1044, 1101, 999, 986, 1002, 1012, 994, 1031, 1034, 1029, 1018, 1025, 1046, 1076, 1045, 1024, 1018, 974, 990, 1064, 1019, 1028, 1032, 1001, 1048, 1062, 1018, 1024, 1032, 1017, 1033, 1088, 1036, 1021, 1001, 979, 1032, 1044, 998, 1059, 964, 1035, 999, 1033, 1005, 1022, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1070, 1065, 1020, 1030, 1027, 954, 1016, 939, 1057, 1039, 1005, 1009, 1025, 1027, 1014, 1059, 1034, 1011, 1073, 1024, 1026, 1008, 1058, 1025, 984, 1050, 1031, 1065, 1011, 1044, 1010, 1016, 1008, 1067, 1003, 1018, 1024, 1020, 988, 1054, 1039, 1082, 979, 1048, 1028, 992, 974, 1069, 1010, 973, 1045, 1070, 1012, 1048, 1060, 1033, 993, 998, 1018, 982, 1011, 1040, 997, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1002, 981, 1020, 988, 1090, 1030, 1021, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1061, 1024, 995, 1025, 1012, 1058, 979, 36, 0, 0, 0, 0, 0, 0, 0, 1015, 1053, 973, 1034, 1084, 1056, 1058, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1016, 1045, 956, 963, 1001, 1016, 952, 21, 3, 1, 0, 0, 0, 0, 0, 1024, 1014, 1064, 1114, 1018, 1035, 1050, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1035, 989, 1021, 1047, 1018, 1057, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 987, 1017, 1043, 973, 1058, 1048, 1027, 5, 0, 0, 0, 0, 0, 0, 0, 1047, 1017, 1065, 975, 1043, 1002, 1043, 1031, 24, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8210, 8287, 7955, 8329, 8146, 8191, 8254] avg=8192.0000 max=8329 max_rank=4 min=7955 min_rank=3 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1059, 967, 999, 990, 1072, 1061, 1039, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1031, 1018, 1022, 1002, 1005, 977, 1027, 35, 0, 0, 0, 0, 0, 0, 0, 1071, 1056, 1064, 1043, 982, 1045, 999, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1019, 1010, 1058, 1017, 1036, 1005, 981, 22, 3, 1, 0, 0, 0, 0, 0, 1037, 1069, 1019, 980, 1045, 1002, 1106, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1016, 1036, 1025, 1022, 981, 1031, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 992, 992, 940, 1045, 1029, 1066, 985, 1006, 5, 0, 0, 0, 0, 0, 0, 0, 989, 995, 1052, 993, 1026, 989, 1057, 1054, 24, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8140, 8335, 8189, 8266, 8157, 8060, 8186] avg=8192.0000 max=8335 max_rank=2 min=8060 min_rank=6 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1053, 1043, 952, 1059, 1050, 1052, 1039, 976, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1009, 1020, 1011, 1072, 970, 997, 1047, 38, 0, 0, 0, 0, 0, 0, 0, 1008, 1044, 1101, 999, 986, 1002, 1012, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1034, 1029, 1018, 1025, 1046, 1076, 1045, 20, 3, 1, 0, 0, 0, 0, 0, 1024, 1018, 974, 990, 1064, 1019, 1028, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1048, 1062, 1018, 1016, 1032, 1017, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 1036, 1021, 1001, 979, 1032, 1044, 998, 5, 0, 0, 0, 0, 0, 0, 0, 1059, 964, 1035, 999, 1033, 1005, 1022, 993, 24, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8162, 8126, 8328, 8124, 8227, 8204, 8141] avg=8192.0000 max=8328 max_rank=3 min=8124 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1038, 1061, 1056, 1017, 1022, 996, 1020, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1052, 1004, 1031, 1032, 1024, 997, 982, 39, 0, 0, 0, 0, 0, 0, 0, 1004, 1033, 1033, 1007, 990, 1030, 1031, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1049, 1022, 1026, 1055, 1050, 1051, 1065, 21, 3, 1, 0, 0, 0, 0, 0, 1026, 968, 1030, 977, 1004, 1066, 1052, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 993, 1034, 1025, 1009, 1005, 992, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 983, 1040, 1010, 1041, 1000, 1029, 1086, 5, 0, 0, 0, 0, 0, 0, 0, 995, 984, 1046, 1057, 1058, 980, 1035, 1019, 24, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8199, 8147, 8360, 8148, 8073, 8207, 8206] avg=8192.0000 max=8360 max_rank=3 min=8073 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1038, 962, 965, 1034, 1039, 1028, 1059, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1015, 1021, 996, 1067, 1023, 1050, 1021, 35, 0, 0, 0, 0, 0, 0, 0, 1012, 966, 1020, 1007, 1071, 1023, 1029, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 999, 966, 1000, 1011, 1043, 1030, 1021, 1044, 21, 3, 1, 0, 0, 0, 0, 0, 1056, 1026, 989, 1011, 1005, 1063, 1013, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 989, 1079, 1017, 1009, 1052, 1040, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1037, 977, 1005, 1029, 1025, 1022, 1009, 5, 0, 0, 0, 0, 0, 0, 0, 1031, 980, 1080, 1018, 1051, 1016, 1015, 1019, 24, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8239, 8153, 8139, 8178, 8279, 8147, 8241] avg=8192.0000 max=8279 max_rank=5 min=8139 min_rank=3 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1040, 981, 1036, 1064, 1016, 1034, 1037, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1014, 984, 1012, 1015, 1055, 1039, 1068, 36, 0, 0, 0, 0, 0, 0, 0, 1057, 1049, 1005, 1015, 1038, 937, 1013, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1036, 1034, 1049, 1049, 1053, 954, 992, 21, 3, 1, 0, 0, 0, 0, 0, 1042, 998, 1012, 1018, 1023, 999, 1046, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1024, 1018, 1062, 1055, 1027, 1016, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 985, 1050, 1077, 1037, 1062, 1013, 1016, 5, 0, 0, 0, 0, 0, 0, 0, 1060, 1002, 1020, 995, 1017, 992, 1017, 1010, 24, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8210, 8121, 8193, 8146, 8210, 8284, 8144] avg=8192.0000 max=8284 max_rank=6 min=8121 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1016, 995, 1082, 1065, 1044, 986, 1047, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1018, 1018, 991, 1000, 1014, 1015, 996, 37, 0, 0, 0, 0, 0, 0, 0, 1025, 1019, 1015, 995, 996, 1051, 1008, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1027, 1007, 1013, 1032, 991, 1004, 1063, 21, 3, 1, 0, 0, 0, 0, 0, 1046, 1036, 1020, 1041, 1005, 1030, 991, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 987, 1008, 1038, 1040, 1057, 1046, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 987, 1067, 1043, 979, 1063, 1022, 1013, 5, 0, 0, 0, 0, 0, 0, 0, 1049, 1016, 1052, 1035, 1017, 994, 1045, 1021, 24, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8099, 8131, 8128, 8193, 8250, 8182, 8260] avg=8192.0000 max=8293 max_rank=0 min=8099 min_rank=1 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1070, 1012, 1020, 1030, 1027, 954, 1016, 939, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1039, 1005, 1009, 1025, 1027, 1014, 1059, 42, 0, 0, 0, 0, 0, 0, 0, 1034, 1011, 1073, 1024, 1026, 1008, 1058, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1050, 1031, 1065, 1011, 1044, 1010, 1016, 23, 7, 3, 0, 0, 0, 0, 0, 1008, 1067, 1003, 1018, 1024, 1020, 988, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1082, 979, 1048, 1011, 992, 974, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 973, 1045, 1070, 1012, 1048, 1060, 1033, 10, 0, 0, 0, 0, 0, 0, 0, 993, 998, 1018, 982, 1011, 1040, 997, 1027, 27, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 8277, 8236, 8244, 8152, 8194, 8261, 8104] avg=8192.0000 max=8277 max_rank=1 min=8068 min_rank=0 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8320, 8448, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8448, 8064, 8320, 8064, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8320, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8192, 8320, 8192, 8448, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[981, 984, 1033, 1028, 990, 1072, 1043, 1015, 1040, 1033, 1061, 1003, 967, 1007, 1057, 1045, 1016, 1049, 1031, 1024, 1051, 1030, 1016, 1033, 1007, 1056, 1019, 1037, 949, 993, 1020, 1074, 993, 1026, 1019, 1009, 1034, 1065, 1039, 1032, 1051, 1050, 1001, 1040, 1010, 1057, 1024, 995, 1029, 1008, 992, 1032, 1013, 1029, 1001, 1052, 1032, 1074, 1043, 986, 1005, 961, 1028, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1009, 1028, 978, 1024, 998, 1039, 1064, 1041, 1077, 970, 1003, 1017, 1060, 1066, 1031, 1037, 1062, 1015, 1080, 1038, 998, 998, 1036, 1022, 1027, 997, 1003, 1047, 1015, 1000, 1016, 1015, 1056, 1056, 1000, 1033, 1042, 994, 1022, 992, 1027, 1052, 1010, 996, 994, 999, 1047, 1071, 1042, 1025, 977, 1038, 1012, 992, 1006, 1042, 989, 990, 1039, 1030, 1037, 1042, 1044, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1045, 990, 984, 1018, 1053, 1028, 1036, 1082, 1026, 1007, 997, 1047, 1064, 1029, 1001, 1006, 1018, 1053, 1081, 1004, 974, 977, 1025, 1020, 984, 1013, 1016, 1028, 1083, 1041, 982, 1012, 974, 1060, 1034, 1043, 961, 1030, 985, 975, 1014, 1096, 1080, 991, 998, 989, 1009, 1064, 1062, 1045, 1034, 995, 975, 1005, 1045, 1051, 1060, 1030, 1068, 1028, 1023, 1032, 1022, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[998, 1071, 1039, 1008, 1010, 1032, 1023, 1009, 1033, 1023, 1062, 1037, 1004, 1042, 1019, 988, 989, 1057, 982, 990, 1051, 1046, 1032, 1034, 1016, 1015, 1028, 1072, 1005, 1002, 1034, 1018, 1073, 991, 1020, 1017, 981, 1016, 1017, 1029, 990, 1025, 998, 1079, 1001, 1027, 1029, 967, 1003, 1037, 984, 987, 1094, 1029, 1038, 1023, 1003, 1058, 1033, 1032, 1037, 1043, 1044, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[999, 1063, 1089, 1035, 1035, 1025, 1025, 966, 989, 1017, 1007, 1011, 1019, 1043, 1042, 1036, 1035, 1007, 1033, 1036, 1048, 1001, 1004, 1071, 1037, 1029, 991, 1028, 1010, 1038, 1006, 1009, 992, 1018, 1047, 1053, 1059, 1018, 1069, 1016, 1044, 1002, 1023, 989, 1006, 1038, 1038, 1046, 1030, 1007, 975, 1048, 1075, 1017, 1002, 1046, 1026, 984, 1036, 1015, 995, 1016, 984, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1016, 1043, 1014, 1045, 1028, 1030, 1049, 1055, 992, 1052, 1041, 959, 967, 1013, 1039, 976, 1040, 1016, 1042, 1081, 1014, 1058, 1013, 1018, 1060, 1037, 963, 1009, 1018, 1033, 1013, 1013, 1038, 1023, 1010, 1070, 1034, 1051, 1101, 996, 1037, 1027, 1067, 992, 1007, 991, 1035, 1024, 992, 990, 1025, 1036, 1006, 1036, 1015, 1003, 1048, 1015, 1048, 1027, 1009, 1024, 987, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1008, 1013, 1049, 1003, 1051, 1035, 1046, 976, 1011, 1028, 1054, 1042, 1021, 1046, 1010, 1003, 1041, 1008, 998, 992, 1009, 1040, 1069, 991, 1010, 1040, 1012, 1015, 1044, 1034, 1020, 1002, 1055, 1030, 1010, 1062, 1037, 1050, 1021, 1072, 1024, 1004, 1065, 1029, 1057, 1008, 1048, 1036, 1001, 978, 1040, 1015, 1005, 995, 1063, 1010, 1061, 1010, 994, 975, 1015, 1002, 1033, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[958, 1015, 1038, 1045, 1010, 1025, 1018, 1025, 1003, 1015, 1001, 1005, 1087, 1033, 1013, 1040, 1024, 1036, 1042, 989, 1012, 1051, 1035, 1040, 1019, 1024, 1038, 1013, 1064, 988, 1015, 1081, 1026, 1024, 998, 997, 1043, 966, 1011, 1029, 1023, 1067, 1048, 1058, 1029, 1084, 1003, 1096, 1007, 1018, 1046, 1010, 978, 1034, 1025, 1013, 1011, 1067, 1011, 1004, 1019, 999, 1049, 941] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1009, 1028, 978, 1024, 998, 1039, 1058, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 970, 1003, 1017, 1060, 1066, 1031, 1037, 6, 1, 0, 0, 0, 0, 0, 0, 1062, 1015, 1056, 1038, 998, 998, 1036, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 997, 1003, 1047, 1015, 1000, 1016, 1015, 23, 2, 0, 0, 0, 0, 0, 0, 1056, 1056, 1000, 1021, 1042, 994, 1022, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1016, 1010, 996, 994, 999, 1047, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1025, 977, 1038, 1012, 992, 1006, 1042, 36, 1, 0, 0, 0, 0, 0, 0, 989, 990, 1039, 1030, 1037, 1042, 1044, 1029, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8268, 8225, 8145, 8183, 8160, 8171, 8209] avg=8192.0000 max=8268 max_rank=1 min=8145 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1016, 1043, 1014, 1045, 1028, 1030, 1043, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1052, 1041, 959, 967, 1013, 1039, 976, 6, 1, 0, 0, 0, 0, 0, 0, 1040, 1016, 1019, 1081, 1014, 1058, 1013, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1037, 963, 1009, 1018, 1033, 1013, 1013, 22, 2, 0, 0, 0, 0, 0, 0, 1038, 1023, 1010, 1057, 1034, 1051, 1101, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 992, 1067, 992, 1007, 991, 1035, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 992, 990, 1025, 1036, 1006, 1036, 1015, 1003, 35, 1, 0, 0, 0, 0, 0, 0, 1048, 1015, 1048, 1027, 1009, 1024, 987, 1025, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8046, 8259, 8170, 8310, 8145, 8139, 8193] avg=8192.0000 max=8310 max_rank=4 min=8046 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[981, 984, 1033, 1028, 990, 1072, 1037, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1033, 1061, 1003, 967, 1007, 1057, 1045, 6, 1, 0, 0, 0, 0, 0, 0, 1016, 1049, 1009, 1024, 1051, 1030, 1016, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1056, 1019, 1037, 949, 993, 1020, 1074, 22, 2, 0, 0, 0, 0, 0, 0, 993, 1026, 1019, 997, 1034, 1065, 1039, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1014, 1001, 1040, 1010, 1057, 1024, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1008, 992, 1032, 1013, 1029, 1001, 1052, 36, 0, 0, 0, 0, 0, 0, 0, 1032, 1074, 1043, 986, 1005, 961, 1028, 1042, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8220, 8228, 8179, 8205, 8192, 8192, 8180] avg=8192.0000 max=8228 max_rank=2 min=8140 min_rank=0 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[998, 1071, 1039, 1008, 1010, 1032, 1017, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1023, 1062, 1037, 1004, 1042, 1019, 988, 6, 1, 0, 0, 0, 0, 0, 0, 989, 1057, 961, 990, 1051, 1046, 1032, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1015, 1028, 1072, 1005, 1002, 1034, 1018, 21, 2, 0, 0, 0, 0, 0, 0, 1073, 991, 1020, 1005, 981, 1016, 1017, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 990, 990, 998, 1079, 1001, 1027, 1029, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1037, 984, 987, 1094, 1029, 1038, 1023, 35, 0, 0, 0, 0, 0, 0, 0, 1003, 1058, 1033, 1032, 1037, 1043, 1044, 1062, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8215, 8160, 8213, 8132, 8081, 8230, 8321] avg=8192.0000 max=8321 max_rank=7 min=8081 min_rank=5 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1008, 1013, 1049, 1003, 1051, 1035, 1040, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1028, 1054, 1042, 1021, 1046, 1010, 1003, 6, 1, 0, 0, 0, 0, 0, 0, 1041, 1008, 977, 992, 1009, 1040, 1069, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1040, 1012, 1015, 1044, 1034, 1020, 1002, 21, 2, 0, 0, 0, 0, 0, 0, 1055, 1030, 1010, 1050, 1037, 1050, 1021, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 970, 1065, 1029, 1057, 1008, 1048, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 978, 1040, 1015, 1005, 995, 1063, 1010, 34, 0, 0, 0, 0, 0, 0, 0, 1061, 1010, 994, 975, 1015, 1002, 1033, 1010, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8222, 8127, 8200, 8325, 8237, 8141, 8109] avg=8192.0000 max=8325 max_rank=4 min=8109 min_rank=7 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[999, 1063, 1089, 1035, 1035, 1025, 1019, 966, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1017, 1007, 1011, 1019, 1043, 1042, 1036, 6, 1, 0, 0, 0, 0, 0, 0, 1035, 1007, 1011, 1036, 1048, 1001, 1004, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1029, 991, 1028, 1010, 1038, 1006, 1009, 22, 2, 0, 0, 0, 0, 0, 0, 992, 1018, 1047, 1041, 1059, 1018, 1069, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 968, 1023, 989, 1006, 1038, 1038, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1007, 975, 1048, 1075, 1017, 1002, 1046, 34, 0, 0, 0, 0, 0, 0, 0, 1026, 984, 1036, 1015, 995, 1016, 984, 1038, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8171, 8213, 8172, 8260, 8152, 8234, 8103] avg=8192.0000 max=8260 max_rank=4 min=8103 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1045, 990, 984, 1018, 1053, 1028, 1030, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1007, 997, 1047, 1064, 1029, 1001, 1006, 6, 1, 0, 0, 0, 0, 0, 0, 1018, 1053, 1057, 1004, 974, 977, 1025, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1013, 1016, 1028, 1083, 1041, 982, 1012, 23, 2, 0, 0, 0, 0, 0, 0, 974, 1060, 1034, 1031, 961, 1030, 985, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1058, 1080, 991, 998, 989, 1009, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1045, 1034, 995, 975, 1005, 1045, 1051, 38, 1, 0, 0, 0, 0, 0, 0, 1060, 1030, 1068, 1028, 1023, 1032, 1022, 1034, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8184, 8128, 8184, 8050, 8203, 8251, 8306] avg=8192.0000 max=8306 max_rank=7 min=8050 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[958, 1015, 1038, 1045, 1010, 1025, 1011, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1015, 1001, 1005, 1087, 1033, 1013, 1040, 7, 6, 0, 0, 0, 0, 0, 0, 1024, 1036, 1009, 989, 1012, 1051, 1035, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1024, 1038, 1013, 1064, 988, 1015, 1081, 28, 3, 0, 0, 0, 0, 0, 0, 1026, 1024, 998, 974, 1043, 966, 1011, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1025, 1048, 1058, 1029, 1084, 1003, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1018, 1046, 1010, 978, 1034, 1025, 1013, 42, 5, 0, 0, 0, 0, 0, 0, 1011, 1067, 1011, 1004, 1019, 999, 1049, 941, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8210, 8196, 8273, 8071, 8366, 8178, 8115] avg=8192.0000 max=8366 max_rank=5 min=8071 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1037, 995, 994, 1080, 999, 1008, 976, 1059, 1073, 1033, 1020, 1014, 1022, 1032, 1009, 1018, 1008, 1033, 1056, 1060, 956, 1034, 982, 1024, 1069, 1006, 1005, 1026, 1057, 1038, 1025, 1030, 1025, 1022, 1046, 1087, 1023, 988, 1009, 1048, 1004, 1071, 1069, 1030, 938, 1025, 989, 1026, 1051, 1019, 984, 978, 1032, 1009, 1076, 999, 1021, 1005, 1031, 1071, 1027, 1017, 1021, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1071, 1059, 1003, 1025, 1077, 992, 1028, 1051, 1028, 1005, 1052, 1045, 1016, 1075, 1028, 1043, 1021, 1028, 1015, 1046, 1018, 992, 1002, 982, 977, 1024, 1006, 1025, 1020, 1021, 1033, 1026, 1023, 1044, 984, 1014, 997, 994, 982, 1062, 1053, 1064, 1066, 1009, 1035, 1041, 1003, 1029, 1056, 1019, 1026, 1047, 972, 1010, 1050, 999, 1054, 1042, 995, 992, 977, 1018, 1039, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[984, 1044, 1042, 996, 1039, 985, 979, 1006, 1004, 1012, 1078, 1031, 1023, 1016, 1065, 1047, 1011, 986, 1029, 1035, 1035, 1015, 1043, 1008, 1001, 987, 1015, 1042, 1033, 1048, 1036, 1022, 1041, 1031, 1034, 1070, 1013, 1027, 1003, 1023, 979, 1025, 1019, 1013, 986, 1023, 1045, 1025, 1019, 1024, 1039, 1018, 1030, 1003, 975, 1058, 1039, 1015, 1052, 1019, 1016, 1073, 1052, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1037, 1016, 1018, 997, 982, 1009, 1071, 1044, 970, 1011, 976, 1061, 1041, 1028, 1014, 1016, 1004, 1015, 1002, 1018, 1047, 1050, 1050, 1067, 1022, 1001, 1039, 1036, 1070, 1031, 1042, 987, 1038, 1012, 1049, 1041, 989, 1033, 1039, 999, 1068, 1035, 1019, 1007, 1034, 1053, 997, 1049, 1009, 1020, 1042, 980, 1008, 1032, 1009, 1052, 986, 1032, 1039, 1054, 1002, 999, 1014, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1048, 1029, 1004, 1045, 1010, 992, 999, 1057, 1024, 1028, 1088, 973, 1006, 998, 1040, 1030, 1041, 1046, 1057, 1005, 973, 1005, 995, 1034, 1018, 1007, 1035, 1024, 1006, 990, 1005, 1018, 976, 1023, 1030, 1003, 1055, 975, 988, 1053, 1068, 1065, 953, 1023, 1043, 1034, 1027, 1069, 995, 1061, 1006, 1014, 1064, 1002, 1050, 1029, 1058, 1022, 1045, 1041, 1042, 991, 1049, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1003, 1018, 1070, 1034, 987, 1024, 1042, 1041, 1055, 1023, 1049, 1012, 1010, 989, 1033, 1032, 1019, 1025, 997, 1028, 1039, 965, 1027, 1035, 1022, 1043, 1028, 1039, 978, 1047, 1014, 1042, 990, 1020, 1029, 1021, 1041, 1021, 1019, 1009, 1046, 1023, 1032, 993, 1070, 1052, 1006, 1053, 981, 1056, 994, 1034, 1049, 1016, 1039, 1027, 1036, 1050, 986, 995, 1027, 1010, 1011, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1041, 1036, 1031, 1056, 982, 994, 999, 1007, 994, 1022, 1024, 1022, 1040, 1061, 1054, 1029, 1063, 1045, 1022, 996, 1064, 1036, 1047, 1033, 1043, 1042, 989, 993, 988, 999, 994, 1048, 1027, 1025, 1043, 1025, 1035, 1017, 1045, 978, 1027, 1012, 1037, 1044, 1026, 991, 1036, 1048, 1022, 995, 1035, 1047, 1068, 992, 1049, 1018, 996, 1003, 1008, 1011, 1023, 999, 1033, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[970, 1049, 1035, 1092, 1036, 996, 1063, 1039, 960, 1031, 1020, 1001, 1026, 997, 1053, 978, 1062, 1044, 1012, 1037, 1040, 1058, 1033, 994, 1059, 987, 1011, 1022, 1020, 1061, 1006, 1014, 979, 992, 1002, 1051, 1027, 1010, 1031, 999, 996, 1041, 966, 981, 1059, 1014, 1035, 1020, 1041, 1039, 1019, 1017, 1038, 1071, 1058, 1071, 1019, 1046, 1013, 1029, 1014, 1024, 1012, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1037, 995, 994, 1080, 999, 1008, 976, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1033, 1012, 1014, 1022, 1032, 1009, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1033, 1056, 1060, 956, 1034, 982, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1006, 1005, 1026, 1057, 1038, 1025, 1030, 10, 7, 0, 0, 0, 0, 0, 0, 1025, 1022, 1046, 1087, 1023, 988, 1009, 1048, 23, 3, 0, 0, 0, 0, 0, 0, 1004, 1048, 1069, 1030, 938, 1025, 989, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1019, 984, 978, 1032, 1009, 1063, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1005, 1031, 1071, 1027, 1017, 1021, 1017, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8213, 8152, 8273, 8274, 8129, 8135, 8212] avg=8192.0000 max=8274 max_rank=4 min=8129 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1037, 1016, 1018, 997, 982, 1009, 1071, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1011, 968, 1061, 1041, 1028, 1014, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1015, 1002, 1018, 1047, 1050, 1050, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1001, 1039, 1036, 1070, 1031, 1042, 987, 9, 7, 0, 0, 0, 0, 0, 0, 1038, 1012, 1049, 1041, 989, 1033, 1039, 999, 22, 2, 0, 0, 0, 0, 0, 0, 1068, 1013, 1019, 1007, 1034, 1053, 997, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1020, 1042, 980, 1008, 1032, 998, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1032, 1039, 1054, 1002, 999, 1014, 1024, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8109, 8252, 8244, 8224, 8240, 8141, 8152] avg=8192.0000 max=8252 max_rank=2 min=8109 min_rank=1 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1071, 1059, 1003, 1025, 1077, 992, 1028, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1005, 1044, 1045, 1016, 1075, 1028, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1028, 1015, 1046, 1018, 992, 1002, 982, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1024, 1006, 1025, 1020, 1021, 1033, 1026, 10, 7, 0, 0, 0, 0, 0, 0, 1023, 1044, 984, 1014, 997, 994, 982, 1062, 23, 3, 0, 0, 0, 0, 0, 0, 1053, 1041, 1066, 1009, 1035, 1041, 1003, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1019, 1026, 1047, 972, 1010, 1037, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1042, 995, 992, 977, 1018, 1039, 1006, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 8284, 8103, 8149, 8126, 8277, 8166, 8125] avg=8192.0000 max=8306 max_rank=0 min=8103 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[984, 1044, 1042, 996, 1039, 985, 979, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1012, 1069, 1031, 1023, 1016, 1065, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 986, 1029, 1035, 1035, 1015, 1043, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 987, 1015, 1042, 1033, 1048, 1036, 1022, 9, 8, 0, 0, 0, 0, 0, 0, 1041, 1031, 1034, 1070, 1013, 1027, 1003, 1023, 22, 2, 0, 0, 0, 0, 0, 0, 979, 1003, 1019, 1013, 986, 1023, 1045, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1024, 1039, 1018, 1030, 1003, 964, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1015, 1052, 1019, 1016, 1073, 1052, 1050, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8075, 8267, 8161, 8201, 8266, 8093, 8155, 8318] avg=8192.0000 max=8318 max_rank=7 min=8075 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1003, 1018, 1070, 1034, 987, 1024, 1042, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1023, 1041, 1012, 1010, 989, 1033, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1025, 997, 1028, 1039, 965, 1027, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1043, 1028, 1039, 978, 1047, 1014, 1042, 10, 7, 0, 0, 0, 0, 0, 0, 990, 1020, 1029, 1021, 1041, 1021, 1019, 1009, 22, 3, 0, 0, 0, 0, 0, 0, 1046, 1001, 1032, 993, 1070, 1052, 1006, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1056, 994, 1034, 1049, 1016, 1026, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1050, 986, 995, 1027, 1010, 1011, 1030, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8195, 8134, 8230, 8175, 8253, 8183, 8147] avg=8192.0000 max=8253 max_rank=5 min=8134 min_rank=2 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1041, 1036, 1031, 1056, 982, 994, 999, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1022, 1016, 1022, 1040, 1061, 1054, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1045, 1022, 996, 1064, 1036, 1047, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1042, 989, 993, 988, 999, 994, 1048, 10, 7, 0, 0, 0, 0, 0, 0, 1027, 1025, 1043, 1025, 1035, 1017, 1045, 978, 22, 3, 0, 0, 0, 0, 0, 0, 1027, 990, 1037, 1044, 1026, 991, 1036, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 995, 1035, 1047, 1068, 992, 1036, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1003, 1008, 1011, 1023, 999, 1033, 1027, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8238, 8305, 8113, 8220, 8199, 8213, 8102] avg=8192.0000 max=8305 max_rank=2 min=8102 min_rank=7 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1048, 1029, 1004, 1045, 1010, 992, 999, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1028, 1079, 973, 1006, 998, 1040, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1046, 1057, 1005, 973, 1005, 995, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1007, 1035, 1024, 1006, 990, 1005, 1018, 10, 8, 0, 0, 0, 0, 0, 0, 976, 1023, 1030, 1003, 1055, 975, 988, 1053, 23, 3, 0, 0, 0, 0, 0, 0, 1068, 1042, 953, 1023, 1043, 1034, 1027, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1061, 1006, 1014, 1064, 1002, 1037, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1022, 1045, 1041, 1042, 991, 1049, 1052, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8178, 8155, 8121, 8129, 8259, 8208, 8302] avg=8192.0000 max=8302 max_rank=7 min=8121 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[970, 1049, 1035, 1092, 1036, 996, 1063, 1039, 4, 0, 0, 0, 0, 0, 0, 0, 960, 1031, 1006, 1001, 1026, 997, 1053, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1044, 1012, 1037, 1040, 1058, 1033, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 987, 1011, 1022, 1020, 1061, 1006, 1014, 14, 11, 0, 0, 0, 0, 0, 0, 979, 992, 1002, 1051, 1027, 1010, 1031, 999, 26, 5, 0, 0, 0, 0, 0, 0, 996, 1015, 966, 981, 1059, 1014, 1035, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1039, 1019, 1017, 1038, 1071, 1039, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1046, 1013, 1029, 1014, 1024, 1012, 1016, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8052, 8274, 8205, 8122, 8086, 8335, 8178] avg=8192.0000 max=8335 max_rank=6 min=8052 min_rank=1 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8448, 8192, 8064, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1000, 1022, 1024, 1015, 1011, 1021, 985, 1058, 1031, 1074, 982, 1031, 1056, 1001, 1014, 966, 1005, 970, 1034, 1013, 1040, 1018, 981, 1039, 1044, 1056, 1006, 1055, 1014, 1036, 966, 1048, 1052, 1018, 1056, 988, 1011, 1038, 1022, 1071, 1027, 1014, 1028, 995, 1102, 999, 1042, 1042, 1030, 1019, 1040, 1032, 1055, 994, 1048, 996, 1022, 1012, 1005, 1055, 996, 1045, 995, 1071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1001, 1009, 1037, 976, 1028, 1078, 1014, 1047, 1028, 1021, 980, 1006, 1018, 1033, 990, 1003, 1002, 984, 984, 999, 1011, 1012, 1000, 1004, 1033, 1043, 1011, 987, 1053, 1021, 1050, 1010, 1056, 1060, 1024, 1025, 1025, 1027, 1051, 1074, 1003, 1051, 1031, 1040, 1026, 988, 1043, 1020, 1020, 1028, 1022, 1009, 1037, 1036, 1049, 1034, 1016, 1011, 1034, 1046, 1049, 1015, 1051, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1048, 978, 994, 1080, 1001, 1005, 992, 992, 1056, 1016, 1011, 1048, 1031, 989, 1036, 998, 1018, 1009, 1018, 1035, 958, 1086, 1093, 1066, 983, 1004, 1021, 1048, 1024, 1000, 1042, 1045, 1030, 1056, 1020, 1025, 1009, 1019, 1074, 1043, 1028, 1059, 1024, 1042, 998, 966, 1060, 1013, 1066, 1024, 1027, 981, 977, 987, 1030, 1019, 988, 1057, 1026, 1043, 1034, 1078, 993, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1028, 1039, 1024, 1007, 983, 1076, 1029, 971, 1015, 1006, 1069, 970, 1044, 1006, 1057, 1005, 1049, 995, 1016, 1000, 1042, 1051, 998, 1014, 1076, 1022, 1049, 1074, 1031, 1050, 1038, 1030, 1029, 1011, 1005, 1016, 1002, 1004, 1007, 1031, 1049, 991, 1025, 1017, 1000, 1013, 1029, 994, 990, 1001, 1012, 1034, 1050, 1029, 1017, 1033, 1029, 1097, 1092, 1012, 1043, 1009, 984, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1034, 953, 1060, 1020, 1009, 1068, 1015, 1000, 1018, 996, 1018, 990, 1052, 1030, 988, 1014, 1023, 1038, 1003, 986, 1043, 1000, 1093, 981, 1056, 1010, 1061, 1034, 1025, 982, 1060, 1048, 1012, 1054, 1027, 1019, 990, 1008, 1051, 991, 1019, 1082, 1046, 1040, 1017, 1034, 1006, 1014, 1045, 1031, 1015, 1029, 1056, 1036, 1045, 1026, 990, 1011, 1072, 1011, 983, 1018, 1016, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1065, 1022, 1016, 1057, 1053, 1063, 1049, 1021, 1051, 1001, 1013, 1011, 1062, 990, 1020, 1052, 1023, 1057, 1069, 993, 985, 1037, 994, 989, 994, 971, 1027, 1053, 1044, 1049, 1060, 1035, 1015, 1076, 1013, 1015, 996, 1036, 1039, 1000, 1009, 1059, 1030, 1038, 985, 1051, 1028, 1031, 1015, 1024, 974, 1055, 990, 987, 1015, 1016, 1029, 1003, 1045, 991, 992, 969, 1048, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1011, 1060, 996, 1059, 1027, 963, 1032, 1031, 1046, 1001, 1011, 1084, 1000, 1008, 1053, 1039, 1018, 1015, 1023, 1025, 1044, 1078, 1057, 1023, 1007, 1037, 1059, 1003, 1013, 983, 1053, 1012, 1003, 1058, 978, 1014, 1031, 1035, 1032, 1015, 1064, 1041, 1013, 1022, 1021, 974, 977, 1012, 1010, 1064, 992, 1011, 1004, 995, 1003, 994, 1064, 1043, 1031, 1012, 1055, 1059, 1020, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1017, 1011, 1026, 1016, 967, 1007, 998, 1051, 1029, 1045, 1032, 1026, 1067, 1012, 997, 975, 1044, 1030, 1064, 1030, 1026, 1007, 1027, 989, 1041, 1003, 1030, 1009, 1035, 982, 1026, 1033, 1044, 989, 1048, 1000, 1024, 1046, 1003, 1028, 1031, 1039, 977, 1030, 1025, 1017, 1017, 1026, 1078, 1024, 1014, 1037, 1012, 1039, 1004, 1022, 1028, 1039, 1083, 1025, 1054, 1011, 1048, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1000, 1022, 1024, 1015, 1011, 1021, 985, 1058, 13, 8, 0, 0, 0, 0, 0, 0, 1031, 1074, 982, 1031, 1056, 1001, 1014, 966, 25, 0, 0, 0, 0, 0, 0, 0, 1005, 970, 1034, 1013, 1040, 1018, 981, 1039, 20, 0, 0, 0, 0, 0, 0, 0, 1044, 1056, 1006, 1055, 1014, 1036, 937, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1002, 1056, 988, 1011, 1038, 1022, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1014, 1028, 995, 1102, 999, 1042, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1019, 1040, 1032, 1055, 994, 1048, 996, 16, 1, 0, 0, 0, 0, 0, 0, 1022, 1012, 967, 1055, 996, 1045, 995, 1071, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8180, 8120, 8196, 8240, 8249, 8231, 8163] avg=8192.0000 max=8249 max_rank=5 min=8120 min_rank=2 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1048, 978, 994, 1080, 1001, 1005, 992, 992, 13, 9, 0, 0, 0, 0, 0, 0, 1056, 1016, 1011, 1048, 1031, 989, 1036, 998, 26, 0, 0, 0, 0, 0, 0, 0, 1018, 1009, 1018, 1035, 958, 1086, 1093, 1066, 22, 0, 0, 0, 0, 0, 0, 0, 983, 1004, 1021, 1048, 1024, 1000, 1009, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1040, 1020, 1025, 1009, 1019, 1074, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1059, 1024, 1042, 998, 966, 1060, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1024, 1027, 981, 977, 987, 1030, 1019, 16, 2, 0, 0, 0, 0, 0, 0, 988, 1057, 987, 1043, 1034, 1078, 993, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8211, 8305, 8134, 8260, 8190, 8129, 8195] avg=8192.0000 max=8305 max_rank=2 min=8112 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1001, 1009, 1037, 976, 1028, 1078, 1014, 1047, 13, 9, 0, 0, 0, 0, 0, 0, 1028, 1021, 980, 1006, 1018, 1033, 990, 1003, 26, 0, 0, 0, 0, 0, 0, 0, 1002, 984, 984, 999, 1011, 1012, 1000, 1004, 22, 0, 0, 0, 0, 0, 0, 0, 1033, 1043, 1011, 987, 1053, 1021, 1017, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1044, 1024, 1025, 1025, 1027, 1051, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1051, 1031, 1040, 1026, 988, 1043, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1028, 1022, 1009, 1037, 1036, 1049, 1034, 16, 2, 0, 0, 0, 0, 0, 0, 1016, 1011, 995, 1046, 1049, 1015, 1051, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8105, 8018, 8175, 8326, 8202, 8253, 8245] avg=8192.0000 max=8326 max_rank=4 min=8018 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=33026 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1065, 1022, 1016, 1057, 1053, 1063, 1049, 1021, 13, 9, 0, 0, 0, 0, 0, 0, 1051, 1001, 1013, 1011, 1062, 990, 1020, 1052, 26, 0, 0, 0, 0, 0, 0, 0, 1023, 1057, 1069, 993, 985, 1037, 994, 989, 22, 0, 0, 0, 0, 0, 0, 0, 994, 971, 1027, 1053, 1044, 1049, 1027, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1060, 1013, 1015, 996, 1036, 1039, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1059, 1030, 1038, 985, 1051, 1028, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1024, 974, 1055, 990, 987, 1015, 1016, 16, 2, 0, 0, 0, 0, 0, 0, 1029, 1003, 1006, 991, 992, 969, 1048, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8368, 8226, 8169, 8200, 8174, 8231, 8094, 8074] avg=8192.0000 max=8368 max_rank=0 min=8074 min_rank=7 imbalance_ratio=1.021484 max_load_reduction=57168 max_load_reduction_pct=0.872314 moved_to_echo=32573 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1011, 1060, 996, 1059, 1027, 963, 1032, 1031, 13, 9, 0, 0, 0, 0, 0, 0, 1046, 1001, 1011, 1084, 1000, 1008, 1053, 1039, 26, 0, 0, 0, 0, 0, 0, 0, 1018, 1015, 1023, 1025, 1044, 1078, 1057, 1023, 22, 0, 0, 0, 0, 0, 0, 0, 1007, 1037, 1059, 1003, 1013, 983, 1020, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1042, 978, 1014, 1031, 1035, 1032, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1041, 1013, 1022, 1021, 974, 977, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1064, 992, 1011, 1004, 995, 1003, 994, 16, 2, 0, 0, 0, 0, 0, 0, 1064, 1043, 992, 1012, 1055, 1059, 1020, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8268, 8305, 8134, 8150, 8124, 8091, 8263] avg=8192.0000 max=8305 max_rank=2 min=8091 min_rank=6 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1034, 953, 1060, 1020, 1009, 1068, 1015, 1000, 14, 9, 0, 0, 0, 0, 0, 0, 1018, 996, 1018, 990, 1052, 1030, 988, 1014, 27, 0, 0, 0, 0, 0, 0, 0, 1023, 1038, 1003, 986, 1043, 1000, 1093, 981, 22, 0, 0, 0, 0, 0, 0, 0, 1056, 1010, 1061, 1034, 1025, 982, 1027, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1038, 1027, 1019, 990, 1008, 1051, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1082, 1046, 1040, 1017, 1034, 1006, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1031, 1015, 1029, 1056, 1036, 1045, 1026, 16, 2, 0, 0, 0, 0, 0, 0, 990, 1011, 1031, 1011, 983, 1018, 1016, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8133, 8189, 8243, 8136, 8258, 8301, 8094] avg=8192.0000 max=8301 max_rank=6 min=8094 min_rank=7 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1028, 1039, 1024, 1007, 983, 1076, 1029, 971, 14, 9, 0, 0, 0, 0, 0, 0, 1015, 1006, 1069, 970, 1044, 1006, 1057, 1005, 27, 0, 0, 0, 0, 0, 0, 0, 1049, 995, 1016, 1000, 1042, 1051, 998, 1014, 22, 0, 0, 0, 0, 0, 0, 0, 1076, 1022, 1049, 1074, 1031, 1050, 1005, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 996, 1005, 1016, 1002, 1004, 1007, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 991, 1025, 1017, 1000, 1013, 1029, 994, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1001, 1012, 1034, 1050, 1029, 1017, 1033, 15, 2, 0, 0, 0, 0, 0, 0, 1029, 1097, 1051, 1012, 1043, 1009, 984, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8199, 8187, 8337, 8090, 8118, 8183, 8242] avg=8192.0000 max=8337 max_rank=3 min=8090 min_rank=4 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1017, 1011, 1026, 1016, 967, 1007, 998, 1051, 19, 12, 0, 0, 0, 0, 0, 0, 1029, 1045, 1032, 1026, 1067, 1012, 997, 975, 31, 0, 0, 0, 0, 0, 0, 0, 1044, 1030, 1064, 1030, 1026, 1007, 1027, 989, 26, 0, 0, 0, 0, 0, 0, 0, 1041, 1003, 1030, 1009, 1035, 982, 984, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 967, 1048, 1000, 1024, 1046, 1003, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1039, 977, 1030, 1025, 1017, 1017, 1026, 2, 0, 0, 0, 0, 0, 0, 0, 1078, 1024, 1014, 1037, 1012, 1039, 1004, 1022, 20, 4, 0, 0, 0, 0, 0, 0, 1028, 1039, 1033, 1025, 1054, 1011, 1048, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8214, 8243, 8117, 8160, 8164, 8254, 8260] avg=8192.0000 max=8260 max_rank=7 min=8117 min_rank=3 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8448, 8192, 8320, 8192, 8064, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8448, 8192, 8192, 8064, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1015, 1021, 986, 1027, 1018, 1075, 1034, 1085, 1097, 1040, 1024, 1074, 1021, 998, 981, 991, 1026, 987, 1000, 1017, 1005, 1040, 1006, 1000, 1010, 1021, 1035, 1000, 1072, 1040, 1018, 1029, 989, 1008, 1001, 1059, 1021, 1045, 1007, 992, 1006, 1013, 1014, 1039, 1015, 1015, 1051, 1031, 1004, 1002, 1050, 1006, 1015, 1003, 1018, 1082, 1020, 1051, 1058, 961, 1050, 988, 1084, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1012, 1032, 1067, 1001, 1038, 1026, 995, 1067, 1021, 1040, 1098, 994, 994, 1053, 1037, 1030, 1017, 1048, 1034, 969, 1005, 1068, 1085, 977, 1015, 1024, 985, 1002, 1072, 1000, 1078, 1007, 1057, 977, 1008, 965, 1034, 1066, 1047, 1049, 1051, 959, 1044, 1047, 1002, 1005, 1047, 1019, 1047, 1028, 999, 1009, 994, 1012, 1030, 1039, 983, 986, 1011, 1000, 982, 1040, 1055, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1058, 995, 1015, 1012, 1017, 1001, 1056, 1030, 1039, 1094, 1021, 1010, 1024, 1006, 1033, 1015, 1000, 1038, 1022, 989, 1042, 1038, 1037, 1007, 978, 974, 1009, 1080, 1001, 1055, 1036, 1005, 1073, 1007, 994, 1008, 1066, 1002, 1068, 1025, 1053, 986, 1011, 994, 1005, 1031, 1057, 1003, 1039, 1035, 1021, 1048, 1030, 1047, 1005, 1007, 989, 1005, 1041, 1001, 1048, 1023, 1036, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[996, 1038, 992, 989, 1021, 966, 1022, 987, 1036, 1039, 984, 1020, 1007, 1010, 1081, 1039, 1062, 1021, 1045, 1026, 1010, 1021, 1045, 1030, 1041, 1014, 1059, 1013, 992, 1014, 1053, 996, 1086, 1028, 1067, 1082, 975, 1026, 1008, 1012, 955, 1056, 1030, 1000, 1032, 1041, 1041, 1060, 991, 1016, 1051, 1052, 1058, 1042, 1040, 990, 1027, 1028, 1068, 1048, 972, 1047, 981, 957] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1034, 1048, 1031, 1061, 1051, 1032, 1039, 994, 1014, 1006, 1037, 1060, 1029, 1001, 988, 989, 1018, 972, 1012, 1011, 1076, 1059, 1043, 1074, 1052, 997, 1055, 990, 1065, 1009, 1013, 1016, 1056, 1010, 1057, 1043, 963, 1014, 1015, 1003, 1034, 1004, 1021, 1008, 1022, 1018, 1028, 996, 988, 1078, 1046, 1023, 1009, 1037, 1014, 1051, 1013, 1060, 1020, 981, 1013, 1034, 1021, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1015, 1001, 1039, 1012, 1026, 1080, 1010, 1035, 1015, 1073, 955, 1016, 1014, 1010, 1018, 986, 1027, 1030, 1047, 1047, 1039, 1044, 1053, 1007, 986, 1028, 1089, 1031, 988, 1002, 1072, 1020, 1038, 1023, 1045, 1072, 993, 1040, 969, 1035, 977, 1039, 1003, 1027, 1044, 1056, 1003, 1020, 997, 1021, 1033, 1031, 1041, 1015, 1064, 1021, 1034, 1013, 999, 961, 998, 1093, 1016, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1059, 1084, 1045, 1044, 1031, 1012, 1010, 1020, 998, 1003, 1052, 967, 1009, 993, 975, 1001, 978, 1024, 1043, 1025, 1030, 1028, 1031, 1060, 1006, 1014, 1004, 1054, 1042, 952, 1019, 1018, 1043, 1046, 985, 956, 974, 1044, 1015, 1059, 1019, 1029, 1007, 1031, 999, 1016, 1041, 1100, 1020, 1070, 1021, 1041, 1008, 1017, 1069, 1066, 1013, 1048, 1036, 999, 1040, 1060, 1023, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1024, 1004, 1030, 1085, 1040, 1062, 1054, 1059, 1032, 1030, 999, 1092, 1054, 1005, 1027, 1021, 1049, 1004, 994, 1037, 980, 1021, 1077, 1041, 1001, 1023, 1018, 999, 1021, 1058, 1023, 1021, 1030, 990, 1024, 1034, 1032, 1055, 1025, 1028, 1055, 967, 1024, 1050, 1019, 1027, 1044, 989, 1002, 1037, 1052, 1014, 976, 1003, 998, 984, 1008, 1004, 1019, 987, 1009, 1038, 1020, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1012, 1032, 1067, 1001, 1038, 1026, 995, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1040, 1098, 994, 994, 1053, 1037, 1030, 8, 5, 0, 0, 0, 0, 0, 0, 1017, 1048, 1034, 969, 1005, 1068, 1065, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1024, 985, 1002, 1072, 1000, 1078, 1007, 12, 3, 0, 0, 0, 0, 0, 0, 1053, 977, 1008, 965, 1034, 1066, 1047, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 959, 1044, 1047, 1002, 1005, 1047, 1019, 9, 4, 0, 0, 0, 0, 0, 0, 1047, 1014, 999, 1009, 994, 1012, 1030, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 983, 986, 1011, 1000, 982, 1040, 1055, 1053, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8280, 8183, 8198, 8199, 8187, 8144, 8148] avg=8192.0000 max=8280 max_rank=1 min=8144 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1015, 1021, 986, 1027, 1018, 1075, 1034, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 1040, 1024, 1074, 1021, 998, 981, 991, 8, 5, 0, 0, 0, 0, 0, 0, 1026, 987, 1000, 1017, 1005, 1040, 987, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1021, 1035, 1000, 1072, 1040, 1018, 1029, 11, 3, 0, 0, 0, 0, 0, 0, 986, 1008, 1001, 1059, 1021, 1045, 1007, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1013, 1014, 1039, 1015, 1015, 1051, 1031, 8, 3, 0, 0, 0, 0, 0, 0, 1004, 989, 1050, 1006, 1015, 1003, 1018, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1051, 1058, 961, 1050, 988, 1084, 1045, 39, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8239, 8062, 8239, 8119, 8195, 8167, 8296] avg=8192.0000 max=8296 max_rank=7 min=8062 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[996, 1038, 992, 989, 1021, 966, 1022, 949, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1039, 984, 1020, 1007, 1010, 1081, 1039, 8, 5, 0, 0, 0, 0, 0, 0, 1062, 1021, 1045, 1026, 1010, 1021, 1026, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1014, 1059, 1013, 992, 1014, 1053, 996, 11, 3, 0, 0, 0, 0, 0, 0, 1082, 1028, 1067, 1082, 975, 1026, 1008, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 955, 1056, 1030, 1000, 1032, 1041, 1041, 1060, 9, 4, 0, 0, 0, 0, 0, 0, 991, 1002, 1051, 1052, 1058, 1042, 1040, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1028, 1068, 1048, 972, 1047, 981, 957, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7973, 8229, 8241, 8196, 8280, 8228, 8226, 8163] avg=8192.0000 max=8280 max_rank=4 min=7973 min_rank=0 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1015, 1001, 1039, 1012, 1026, 1080, 1010, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1073, 955, 1016, 1014, 1010, 1018, 986, 8, 5, 0, 0, 0, 0, 0, 0, 1027, 1030, 1047, 1047, 1039, 1044, 1034, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1028, 1089, 1031, 988, 1002, 1072, 1020, 11, 3, 0, 0, 0, 0, 0, 0, 1035, 1023, 1045, 1072, 993, 1040, 969, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1039, 1003, 1027, 1044, 1056, 1003, 1020, 9, 3, 0, 0, 0, 0, 0, 0, 997, 1007, 1033, 1031, 1041, 1015, 1064, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1013, 999, 961, 998, 1093, 1016, 1000, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8100, 8275, 8230, 8212, 8181, 8209, 8151] avg=8192.0000 max=8275 max_rank=2 min=8100 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1058, 995, 1015, 1012, 1017, 1001, 1056, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1094, 1021, 1010, 1024, 1006, 1033, 1015, 8, 5, 0, 0, 0, 0, 0, 0, 1000, 1038, 1022, 989, 1042, 1038, 1018, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 978, 974, 1009, 1080, 1001, 1055, 1036, 1005, 11, 3, 0, 0, 0, 0, 0, 0, 1069, 1007, 994, 1008, 1066, 1002, 1068, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 986, 1011, 994, 1005, 1031, 1057, 1003, 9, 4, 0, 0, 0, 0, 0, 0, 1039, 1021, 1021, 1048, 1030, 1047, 1005, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1005, 1041, 1001, 1048, 1023, 1036, 1041, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8255, 8154, 8152, 8239, 8153, 8218, 8221] avg=8192.0000 max=8255 max_rank=1 min=8144 min_rank=0 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1034, 1048, 1031, 1061, 1051, 1032, 1039, 955, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1006, 1037, 1060, 1029, 1001, 988, 989, 8, 6, 0, 0, 0, 0, 0, 0, 1018, 972, 1012, 1011, 1076, 1059, 1024, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 997, 1055, 990, 1065, 1009, 1013, 1016, 11, 3, 0, 0, 0, 0, 0, 0, 1052, 1010, 1057, 1043, 963, 1014, 1015, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1004, 1021, 1008, 1022, 1018, 1028, 996, 9, 4, 0, 0, 0, 0, 0, 0, 988, 1063, 1046, 1023, 1009, 1037, 1014, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1060, 1020, 981, 1013, 1034, 1021, 980, 36, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8138, 8246, 8211, 8157, 8144, 8231, 8158] avg=8192.0000 max=8251 max_rank=0 min=8138 min_rank=1 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1059, 1084, 1045, 1044, 1031, 1012, 1010, 981, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1003, 1052, 967, 1009, 993, 975, 1001, 8, 6, 0, 0, 0, 0, 0, 0, 978, 1024, 1043, 1025, 1030, 1028, 1012, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1014, 1004, 1054, 1042, 952, 1019, 1018, 11, 3, 0, 0, 0, 0, 0, 0, 1040, 1046, 985, 956, 974, 1044, 1015, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1029, 1007, 1031, 999, 1016, 1041, 1100, 9, 3, 0, 0, 0, 0, 0, 0, 1020, 1055, 1021, 1041, 1008, 1017, 1069, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1048, 1036, 999, 1040, 1060, 1023, 1010, 36, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8012, 8200, 8123, 8119, 8254, 8297, 8265] avg=8192.0000 max=8297 max_rank=6 min=8012 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32935 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1024, 1004, 1030, 1085, 1040, 1062, 1054, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1030, 999, 1092, 1054, 1005, 1027, 1021, 13, 10, 0, 0, 0, 0, 0, 0, 1049, 1004, 994, 1037, 980, 1021, 1049, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1023, 1018, 999, 1021, 1058, 1023, 1021, 15, 8, 0, 0, 0, 0, 0, 0, 1023, 990, 1024, 1034, 1032, 1055, 1025, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 967, 1024, 1050, 1019, 1027, 1044, 989, 12, 7, 0, 0, 0, 0, 0, 0, 1002, 1015, 1052, 1014, 976, 1003, 998, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1004, 1019, 987, 1009, 1038, 1020, 1007, 42, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8283, 8175, 8187, 8211, 8194, 8044, 8134] avg=8192.0000 max=8308 max_rank=0 min=8044 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32583 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8448, 8192, 8192, 8320, 8064, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8192, 8320, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8320, 8064, 8192, 8448, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8448, 8192, 8320, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1042, 1070, 1064, 1022, 1063, 1057, 1004, 1056, 966, 1053, 997, 1020, 992, 1019, 995, 998, 979, 1053, 1054, 1037, 1038, 1014, 1024, 985, 1022, 1062, 1026, 969, 1011, 1019, 1012, 1021, 1046, 987, 1002, 917, 1024, 1017, 1005, 1067, 998, 1059, 1010, 1033, 1032, 1031, 1060, 983, 1085, 1058, 984, 1043, 1029, 1047, 1005, 984, 986, 1042, 1034, 1049, 1044, 1038, 1042, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1050, 1009, 1060, 1056, 984, 1059, 959, 991, 1032, 996, 982, 1067, 1057, 1024, 1011, 1002, 1001, 991, 991, 1037, 1026, 1046, 1049, 1011, 994, 1060, 1008, 1060, 1005, 1041, 1021, 1040, 1032, 1066, 1020, 1004, 1027, 1052, 1008, 1014, 966, 1033, 1019, 1005, 1008, 998, 1046, 1020, 1017, 1016, 995, 1057, 1041, 1023, 1068, 1044, 1003, 1053, 1062, 1032, 1005, 1033, 1057, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1033, 1063, 1019, 1001, 1031, 1012, 1040, 987, 1008, 1004, 1008, 1033, 1008, 1041, 1038, 991, 1055, 997, 1061, 1014, 1025, 1024, 1002, 997, 1002, 1032, 970, 1016, 992, 1036, 1017, 1012, 1035, 1005, 1052, 1014, 1053, 1039, 1037, 996, 1018, 992, 1069, 1070, 1051, 1033, 1036, 1005, 1016, 1050, 998, 986, 1029, 1069, 1068, 1017, 995, 1019, 1055, 1022, 1049, 1054, 991, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1023, 990, 996, 1048, 1017, 1026, 1012, 1019, 1012, 1046, 1015, 1033, 948, 955, 1047, 1026, 1035, 981, 1032, 967, 1045, 1070, 994, 1031, 1037, 1027, 1059, 1063, 1007, 1031, 1059, 1060, 1046, 1051, 1016, 1026, 1010, 1035, 1054, 1005, 985, 1024, 1025, 985, 1057, 1009, 1055, 1009, 1012, 998, 1042, 995, 1051, 1031, 1033, 1014, 1059, 1048, 1019, 1045, 1024, 1033, 1021, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1053, 964, 1009, 1082, 1045, 997, 1052, 1029, 1019, 1061, 1015, 1035, 1003, 1035, 1066, 994, 1040, 1012, 1053, 977, 1003, 1014, 1018, 1102, 1012, 1044, 1023, 1051, 1060, 1026, 1034, 1001, 1009, 1010, 1021, 1037, 1055, 1006, 1001, 1019, 1044, 991, 1038, 1032, 1029, 1006, 1009, 1015, 1053, 1015, 1063, 1032, 978, 1000, 1029, 994, 1048, 1047, 1006, 972, 1004, 1025, 1021, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1016, 1029, 1006, 1035, 1068, 1028, 975, 1030, 1018, 1008, 1004, 1025, 1084, 1025, 1012, 969, 1016, 1019, 1069, 1053, 1036, 1008, 1032, 1014, 1030, 982, 1025, 1001, 1030, 1038, 982, 1032, 1038, 1036, 1038, 992, 1039, 1010, 981, 1011, 1037, 1011, 1078, 1039, 1014, 1055, 1080, 1014, 996, 980, 951, 1043, 995, 1090, 1024, 1027, 1032, 1014, 1036, 1088, 993, 1021, 1013, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1028, 1025, 1026, 1011, 1055, 1060, 1028, 995, 1027, 970, 1030, 1008, 1046, 1011, 999, 1026, 1077, 1005, 1034, 1035, 1054, 1040, 1029, 1016, 1023, 1020, 1073, 1042, 1020, 1028, 1042, 1023, 960, 1057, 1057, 1017, 1003, 1014, 982, 994, 958, 992, 1060, 1043, 1065, 1031, 1054, 1022, 986, 1017, 1067, 979, 1056, 1001, 1051, 1029, 1004, 990, 1049, 1047, 1007, 1068, 944, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1012, 1020, 1007, 1030, 1015, 1009, 1052, 1006, 1031, 1042, 1000, 1046, 1039, 1037, 1021, 1119, 992, 1050, 1051, 1049, 1066, 993, 1050, 1024, 1004, 990, 998, 1017, 1031, 1039, 1051, 1036, 992, 1016, 989, 1037, 1033, 1007, 1051, 1044, 1072, 1004, 1039, 1022, 1027, 988, 995, 1030, 1001, 1011, 981, 977, 1069, 987, 1028, 1095, 1035, 975, 987, 1023, 1013, 1041, 989, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1033, 1063, 1019, 987, 1031, 1012, 1040, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1004, 1008, 1033, 1008, 1041, 1038, 991, 20, 14, 0, 0, 0, 0, 0, 0, 1055, 997, 1041, 1014, 1025, 1024, 1002, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1032, 970, 1016, 992, 1029, 1017, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1005, 1052, 1014, 1053, 1039, 1037, 996, 12, 10, 4, 0, 0, 0, 0, 0, 1018, 992, 1059, 1070, 1051, 1033, 1036, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1050, 998, 986, 1029, 1069, 1068, 1017, 3, 0, 0, 0, 0, 0, 0, 0, 995, 1019, 1055, 1022, 1049, 1042, 991, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8165, 8155, 8070, 8257, 8264, 8236, 8217] avg=8192.0000 max=8264 max_rank=5 min=8070 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32974 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1053, 964, 1009, 1066, 1045, 997, 1052, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1061, 1015, 1035, 1003, 1035, 1066, 994, 20, 16, 0, 0, 0, 0, 0, 0, 1040, 1012, 1033, 977, 1003, 1014, 1018, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1044, 1023, 1051, 1060, 1019, 1034, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1010, 1021, 1037, 1055, 1006, 1001, 1019, 12, 10, 4, 0, 0, 0, 0, 0, 1044, 991, 1028, 1032, 1029, 1006, 1009, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1015, 1063, 1032, 978, 1000, 1029, 994, 3, 0, 0, 0, 0, 0, 0, 0, 1048, 1047, 1006, 972, 1004, 1013, 1021, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8264, 8199, 8244, 8184, 8154, 8167, 8109] avg=8192.0000 max=8264 max_rank=1 min=8109 min_rank=7 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1042, 1070, 1064, 1007, 1063, 1057, 1004, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1053, 997, 1020, 992, 1019, 995, 998, 20, 15, 0, 0, 0, 0, 0, 0, 979, 1053, 1034, 1037, 1038, 1014, 1024, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1062, 1026, 969, 1011, 1012, 1012, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 987, 1002, 917, 1024, 1017, 1005, 1067, 12, 9, 4, 0, 0, 0, 0, 0, 998, 1059, 1001, 1033, 1032, 1031, 1060, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 1058, 984, 1043, 1029, 1047, 1005, 984, 3, 0, 0, 0, 0, 0, 0, 0, 986, 1042, 1034, 1049, 1044, 1026, 1042, 1051, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8363, 8075, 8164, 8135, 8090, 8197, 8238, 8274] avg=8192.0000 max=8363 max_rank=0 min=8075 min_rank=1 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1023, 990, 996, 1033, 1017, 1026, 1012, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1046, 1015, 1033, 948, 955, 1047, 1026, 19, 15, 0, 0, 0, 0, 0, 0, 1035, 981, 1013, 967, 1045, 1070, 994, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1027, 1059, 1063, 1007, 1024, 1059, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1051, 1016, 1026, 1010, 1035, 1054, 1005, 12, 10, 4, 0, 0, 0, 0, 0, 985, 1024, 1015, 985, 1057, 1009, 1055, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 998, 1042, 995, 1051, 1031, 1033, 1014, 3, 0, 0, 0, 0, 0, 0, 0, 1059, 1048, 1019, 1045, 1024, 1021, 1021, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8116, 8136, 8336, 8269, 8139, 8179, 8245] avg=8192.0000 max=8336 max_rank=3 min=8116 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1050, 1009, 1060, 1041, 984, 1059, 959, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 996, 982, 1067, 1057, 1024, 1011, 1002, 19, 15, 0, 0, 0, 0, 0, 0, 1001, 991, 972, 1037, 1026, 1046, 1049, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1060, 1008, 1060, 1005, 1034, 1021, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1066, 1020, 1004, 1027, 1052, 1008, 1014, 12, 10, 4, 0, 0, 0, 0, 0, 966, 1033, 1009, 1005, 1008, 998, 1046, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1016, 995, 1057, 1041, 1023, 1068, 1044, 3, 0, 0, 0, 0, 0, 0, 0, 1003, 1053, 1062, 1032, 1005, 1021, 1057, 992, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8205, 8133, 8222, 8249, 8085, 8264, 8225] avg=8192.0000 max=8264 max_rank=6 min=8085 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1016, 1029, 1006, 1020, 1068, 1028, 975, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1008, 1004, 1025, 1084, 1025, 1012, 969, 20, 15, 0, 0, 0, 0, 0, 0, 1016, 1019, 1049, 1053, 1036, 1008, 1032, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 982, 1025, 1001, 1030, 1031, 982, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1036, 1038, 992, 1039, 1010, 981, 1011, 12, 10, 4, 0, 0, 0, 0, 0, 1037, 1011, 1068, 1039, 1014, 1055, 1080, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 996, 980, 951, 1043, 995, 1090, 1024, 1027, 3, 0, 0, 0, 0, 0, 0, 0, 1032, 1014, 1036, 1088, 993, 1009, 1013, 1061, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8180, 8227, 8113, 8171, 8318, 8109, 8246] avg=8192.0000 max=8318 max_rank=5 min=8109 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1028, 1025, 1026, 996, 1055, 1060, 1028, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 970, 1030, 1008, 1046, 1011, 999, 1026, 19, 15, 0, 0, 0, 0, 0, 0, 1077, 1005, 1015, 1035, 1054, 1040, 1029, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1020, 1073, 1042, 1020, 1021, 1042, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 960, 1057, 1057, 1017, 1003, 1014, 982, 994, 12, 10, 4, 0, 0, 0, 0, 0, 958, 992, 1050, 1043, 1065, 1031, 1054, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1017, 1067, 979, 1056, 1001, 1051, 1029, 3, 0, 0, 0, 0, 0, 0, 0, 1004, 990, 1049, 1047, 1007, 1056, 944, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8151, 8271, 8264, 8110, 8215, 8189, 8123] avg=8192.0000 max=8271 max_rank=2 min=8110 min_rank=4 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1012, 1020, 1007, 1011, 1015, 1009, 1052, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1042, 1000, 1046, 1039, 1037, 1021, 1119, 24, 19, 2, 0, 0, 0, 0, 0, 992, 1050, 1027, 1049, 1066, 993, 1050, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 990, 998, 1017, 1031, 1025, 1051, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1016, 989, 1037, 1033, 1007, 1051, 1044, 15, 13, 9, 0, 0, 0, 0, 0, 1072, 1004, 1026, 1022, 1027, 988, 995, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1011, 981, 977, 1069, 987, 1028, 1095, 5, 0, 0, 0, 0, 0, 0, 0, 1035, 975, 987, 1023, 1013, 1024, 989, 1051, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8380, 8251, 8152, 8206, 8164, 8154, 8097] avg=8192.0000 max=8380 max_rank=1 min=8097 min_rank=7 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32621 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8064, 8320, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8064, 8320, 8192, 8192, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1009, 1027, 994, 1043, 1018, 992, 1034, 1039, 1041, 1030, 1006, 1000, 1018, 1056, 1035, 1030, 1000, 996, 1045, 1054, 1055, 1038, 1049, 1027, 973, 995, 1029, 958, 1001, 1034, 992, 933, 1061, 990, 1011, 974, 1038, 996, 1004, 1024, 1081, 1014, 1026, 1055, 1055, 1024, 1016, 984, 1031, 982, 1069, 1081, 955, 1067, 1032, 1034, 1030, 1054, 1050, 1075, 1045, 1032, 1047, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1006, 975, 1011, 1023, 1000, 1022, 1068, 1015, 1025, 1027, 1055, 1019, 1000, 1057, 992, 1063, 1026, 1030, 992, 1008, 1015, 1020, 977, 1029, 1047, 1025, 1023, 1020, 1021, 1001, 1076, 1037, 1038, 1014, 993, 1053, 1017, 1077, 1040, 1051, 1031, 975, 944, 1017, 1013, 1003, 1047, 1075, 1050, 1064, 1012, 1011, 957, 1092, 1029, 996, 1043, 1055, 1026, 1028, 923, 1004, 1078, 1075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1016, 1045, 1060, 1014, 982, 1005, 1038, 1054, 1045, 974, 1048, 1018, 985, 1056, 998, 973, 994, 1075, 1028, 1019, 1069, 964, 1032, 1080, 1038, 1047, 967, 1015, 1057, 1000, 965, 964, 1106, 1032, 1023, 1053, 999, 1027, 1040, 1070, 1023, 1011, 982, 1021, 1033, 1049, 1002, 1014, 1054, 1002, 1014, 1018, 1008, 1076, 1040, 1003, 1024, 1010, 1070, 1054, 1026, 1052, 978, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1044, 1027, 1020, 1018, 1060, 1040, 1042, 1035, 995, 1026, 1036, 1031, 1035, 1039, 1077, 1009, 1005, 1019, 1026, 1053, 956, 1023, 1056, 1037, 1069, 992, 1052, 1046, 1024, 1035, 982, 966, 1039, 1020, 1102, 1014, 1055, 1044, 992, 979, 1016, 1007, 1017, 1014, 1037, 1027, 1065, 1091, 1045, 1024, 1019, 1017, 993, 995, 984, 1039, 1014, 982, 1001, 1004, 986, 1008, 1019, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1006, 1030, 1007, 1028, 1034, 1025, 1027, 1064, 1026, 966, 992, 1006, 1007, 1049, 996, 1011, 1035, 992, 1057, 1016, 1025, 1035, 1083, 1022, 1021, 1032, 1031, 1039, 1044, 1003, 1002, 1004, 1027, 1007, 1032, 1066, 1031, 1037, 1001, 1004, 997, 1060, 1018, 1009, 1038, 1003, 1013, 1040, 1021, 1029, 1042, 1054, 1033, 1000, 1020, 1039, 1017, 1021, 1056, 1017, 1033, 1040, 975, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1022, 1015, 996, 1028, 1025, 1013, 999, 982, 1042, 1020, 1044, 1027, 1011, 987, 1030, 1054, 1042, 1015, 1023, 1053, 982, 1056, 1062, 977, 1012, 1057, 985, 1022, 1031, 1006, 1054, 1048, 989, 1048, 999, 1032, 1026, 990, 1038, 1037, 983, 1028, 1117, 1044, 1009, 1013, 1033, 983, 983, 1044, 1040, 1018, 1050, 987, 1038, 1042, 1026, 1036, 1002, 1048, 1077, 1050, 1001, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1010, 1055, 1039, 1034, 1008, 1045, 987, 1086, 1051, 1059, 1044, 1080, 1050, 1009, 984, 1010, 1003, 1020, 1025, 1021, 1007, 1016, 1015, 1072, 1059, 1045, 1010, 1029, 1019, 1012, 1051, 1026, 1053, 1060, 1056, 1011, 1066, 1005, 957, 1056, 997, 1067, 990, 1015, 965, 1009, 960, 1015, 985, 955, 1066, 1063, 969, 1053, 1010, 1039, 1027, 1006, 1030, 963, 992, 1031, 1001, 1083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1032, 1046, 1066, 1018, 984, 1065, 1024, 1001, 1000, 1000, 1014, 1018, 994, 999, 987, 1023, 1015, 1028, 1020, 1002, 1032, 1051, 997, 1037, 1015, 1047, 1035, 1080, 1019, 1035, 1056, 1031, 1039, 1046, 987, 1018, 1023, 998, 982, 1040, 1048, 1041, 1016, 1051, 1025, 1008, 1011, 1037, 1003, 1019, 1006, 1033, 1012, 1018, 1037, 1010, 1021, 1009, 1086, 1047, 1054, 1019, 993, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1009, 1027, 994, 1043, 1018, 992, 1034, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1030, 1006, 1000, 1018, 1056, 1035, 1030, 17, 1, 0, 0, 0, 0, 0, 0, 1000, 996, 1045, 1054, 1055, 1038, 1049, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 973, 995, 1029, 958, 1001, 1034, 992, 933, 24, 0, 0, 0, 0, 0, 0, 0, 1036, 990, 1011, 974, 1038, 996, 1004, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1014, 1026, 1055, 1055, 1024, 1016, 984, 12, 2, 1, 0, 0, 0, 0, 0, 1031, 982, 1069, 1081, 955, 1067, 1032, 1034, 3, 0, 0, 0, 0, 0, 0, 0, 1030, 1054, 1032, 1075, 1045, 1032, 1047, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8234, 8252, 7939, 8073, 8270, 8254, 8363] avg=8192.0000 max=8363 max_rank=7 min=7939 min_rank=3 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32960 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1044, 1027, 1020, 1018, 1060, 1040, 1042, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1026, 1036, 1031, 1035, 1039, 1077, 1009, 16, 1, 0, 0, 0, 0, 0, 0, 1005, 1019, 1026, 1053, 956, 1023, 1056, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 992, 1052, 1046, 1024, 1035, 982, 966, 23, 0, 0, 0, 0, 0, 0, 0, 1015, 1020, 1102, 1014, 1055, 1044, 992, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1007, 1017, 1014, 1037, 1027, 1065, 1091, 12, 2, 1, 0, 0, 0, 0, 0, 1045, 1024, 1019, 1017, 993, 995, 984, 1039, 3, 0, 0, 0, 0, 0, 0, 0, 1014, 982, 984, 1004, 986, 1008, 1019, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8265, 8163, 8189, 8221, 8289, 8119, 8009] avg=8192.0000 max=8289 max_rank=5 min=8009 min_rank=7 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32638 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1006, 1030, 1007, 1028, 1034, 1025, 1027, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 966, 992, 1006, 1007, 1049, 996, 1011, 17, 1, 0, 0, 0, 0, 0, 0, 1035, 992, 1057, 1016, 1025, 1035, 1083, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1032, 1031, 1039, 1044, 1003, 1002, 1004, 23, 0, 0, 0, 0, 0, 0, 0, 1003, 1007, 1032, 1066, 1031, 1037, 1001, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1060, 1018, 1009, 1038, 1003, 1013, 1040, 11, 2, 1, 0, 0, 0, 0, 0, 1021, 1029, 1042, 1054, 1033, 1000, 1020, 1039, 3, 0, 0, 0, 0, 0, 0, 0, 1017, 1021, 1038, 1017, 1033, 1040, 975, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8071, 8254, 8199, 8181, 8192, 8241, 8182] avg=8192.0000 max=8254 max_rank=2 min=8071 min_rank=1 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1016, 1045, 1060, 1014, 982, 1005, 1038, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 974, 1048, 1018, 985, 1056, 998, 973, 17, 1, 0, 0, 0, 0, 0, 0, 994, 1075, 1028, 1019, 1069, 964, 1032, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1047, 967, 1015, 1057, 1000, 965, 964, 25, 0, 0, 0, 0, 0, 0, 0, 1080, 1032, 1023, 1053, 999, 1027, 1040, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1011, 982, 1021, 1033, 1049, 1002, 1014, 12, 2, 1, 0, 0, 0, 0, 0, 1054, 1002, 1014, 1018, 1008, 1076, 1040, 1003, 3, 0, 0, 0, 0, 0, 0, 0, 1024, 1010, 1052, 1054, 1026, 1052, 978, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8115, 8249, 8078, 8324, 8150, 8218, 8193] avg=8192.0000 max=8324 max_rank=4 min=8078 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1006, 975, 1011, 1023, 1000, 1022, 1068, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1027, 1055, 1019, 1000, 1057, 992, 1063, 17, 1, 0, 0, 0, 0, 0, 0, 1026, 1030, 992, 1008, 1015, 1020, 977, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1025, 1023, 1020, 1021, 1001, 1076, 1037, 23, 0, 0, 0, 0, 0, 0, 0, 1014, 1014, 993, 1053, 1017, 1077, 1040, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 975, 944, 1017, 1013, 1003, 1047, 1075, 12, 1, 1, 0, 0, 0, 0, 0, 1050, 1064, 1012, 1011, 957, 1092, 1029, 996, 3, 0, 0, 0, 0, 0, 0, 0, 1043, 1055, 1008, 1028, 923, 1004, 1078, 1075, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8256, 8085, 8273, 8259, 8119, 8214, 8214] avg=8192.0000 max=8273 max_rank=3 min=8085 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1010, 1055, 1039, 1034, 1008, 1045, 987, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1059, 1044, 1080, 1050, 1009, 984, 1010, 17, 1, 0, 0, 0, 0, 0, 0, 1003, 1020, 1025, 1021, 1007, 1016, 1015, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1045, 1010, 1029, 1019, 1012, 1051, 1026, 24, 0, 0, 0, 0, 0, 0, 0, 1028, 1060, 1056, 1011, 1066, 1005, 957, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1067, 990, 1015, 965, 1009, 960, 1015, 12, 2, 1, 0, 0, 0, 0, 0, 985, 955, 1066, 1063, 969, 1053, 1010, 1039, 3, 0, 0, 0, 0, 0, 0, 0, 1027, 1006, 1012, 963, 992, 1031, 1001, 1083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8305, 8167, 8275, 8239, 8033, 8143, 8115] avg=8192.0000 max=8305 max_rank=1 min=8033 min_rank=5 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32530 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1022, 1015, 996, 1028, 1025, 1013, 999, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1020, 1044, 1027, 1011, 987, 1030, 1054, 16, 1, 0, 0, 0, 0, 0, 0, 1042, 1015, 1023, 1053, 982, 1056, 1062, 966, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1057, 985, 1022, 1031, 1006, 1054, 1048, 22, 0, 0, 0, 0, 0, 0, 0, 966, 1048, 999, 1032, 1026, 990, 1038, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1028, 1117, 1044, 1009, 1013, 1033, 983, 11, 1, 1, 0, 0, 0, 0, 0, 983, 1044, 1040, 1018, 1050, 987, 1038, 1042, 2, 0, 0, 0, 0, 0, 0, 0, 1026, 1036, 985, 1048, 1077, 1050, 1001, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8077, 8232, 8199, 8237, 8136, 8223, 8204, 8228] avg=8192.0000 max=8237 max_rank=3 min=8077 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1032, 1046, 1066, 1018, 984, 1065, 1024, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1000, 1014, 1018, 994, 999, 987, 1023, 21, 2, 0, 0, 0, 0, 0, 0, 1015, 1028, 1020, 1002, 1032, 1051, 997, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1047, 1035, 1080, 1019, 1035, 1056, 1031, 28, 0, 0, 0, 0, 0, 0, 0, 1009, 1046, 987, 1018, 1023, 998, 982, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1041, 1016, 1051, 1025, 1008, 1011, 1037, 15, 4, 4, 0, 0, 0, 0, 0, 1003, 1019, 1006, 1033, 1012, 1018, 1037, 1010, 5, 0, 0, 0, 0, 0, 0, 0, 1021, 1009, 1061, 1047, 1054, 1019, 993, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8058, 8167, 8346, 8103, 8260, 8143, 8232] avg=8192.0000 max=8346 max_rank=3 min=8058 min_rank=1 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8192, 8064, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8064, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True + [2026-08-30 05:02:13] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 672.5 | throughput per GPU (TFLOP/s/GPU): 495.7 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.990947E+00 | loss scale: 1.0 | grad norm: 6.883 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1034, 1085, 1043, 1033, 975, 1057, 1042, 1041, 1096, 1035, 1007, 960, 1045, 999, 1021, 989, 1058, 1020, 1008, 1006, 1003, 974, 985, 975, 1026, 1044, 975, 1044, 1026, 1068, 993, 1042, 995, 1055, 1067, 1031, 991, 1026, 1019, 1015, 1067, 1029, 1058, 975, 1025, 1041, 1003, 1001, 1004, 1009, 1032, 1002, 1027, 1017, 1043, 1050, 1046, 1038, 1031, 1032, 1030, 1061, 1048, 959] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1023, 1013, 1065, 1014, 1036, 1022, 1053, 1021, 1000, 1024, 1015, 1074, 1052, 1035, 1034, 979, 1020, 1030, 1031, 1038, 993, 1057, 1028, 1001, 1008, 982, 995, 1008, 988, 1067, 988, 1033, 1042, 1025, 979, 1020, 1011, 1043, 1013, 1004, 997, 977, 1047, 1023, 991, 980, 1038, 950, 1056, 1048, 1063, 1002, 1045, 1068, 1006, 995, 1068, 1101, 1073, 1003, 1086, 1006, 1011, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1003, 1021, 1048, 1035, 1021, 1062, 1065, 997, 1024, 1034, 1017, 1033, 1007, 1060, 994, 999, 996, 1034, 1011, 1036, 987, 1035, 1032, 1016, 1000, 1016, 973, 1023, 997, 1013, 1048, 1051, 1037, 1056, 1031, 1052, 1041, 1038, 991, 1006, 995, 1067, 985, 1071, 1015, 1067, 1000, 1031, 1048, 1082, 1033, 1013, 1006, 986, 1034, 1003, 1028, 973, 1009, 1082, 1035, 998, 1024, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[996, 974, 1041, 1034, 1090, 1047, 1017, 1061, 1010, 1034, 1055, 1042, 1008, 1042, 982, 1007, 1055, 1035, 1015, 994, 1051, 1037, 1015, 1044, 1048, 980, 1056, 1017, 978, 1060, 1065, 993, 1063, 970, 1053, 1030, 1014, 979, 1014, 1006, 993, 1025, 1017, 997, 934, 993, 996, 1043, 1013, 1017, 1035, 1028, 1014, 1055, 1046, 996, 1021, 1072, 1020, 1001, 1064, 1034, 1102, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1010, 975, 1048, 995, 1098, 1071, 1057, 1025, 1049, 1045, 1052, 1002, 1054, 991, 1038, 1015, 1041, 1019, 1003, 1026, 1016, 1016, 1039, 979, 1015, 1044, 1030, 1019, 1045, 965, 1061, 1043, 1016, 1010, 1033, 983, 1002, 1014, 997, 1038, 1049, 1033, 1039, 1051, 1035, 1015, 1073, 1025, 1044, 1004, 1025, 1007, 983, 993, 1044, 1053, 963, 1052, 1010, 1016, 1024, 987, 1014, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1054, 1030, 1013, 982, 1005, 1059, 997, 1069, 998, 1003, 1023, 1016, 1051, 1001, 1013, 995, 1042, 1053, 1039, 1038, 981, 993, 1042, 1014, 1037, 1039, 1058, 995, 1044, 1048, 971, 964, 976, 1018, 1101, 1034, 1051, 1084, 994, 1007, 1018, 1034, 1014, 997, 1014, 1080, 1021, 1049, 1041, 1060, 1031, 995, 1002, 1002, 1007, 1010, 1023, 998, 1058, 1047, 1031, 1077, 997, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1056, 998, 986, 1022, 1001, 1091, 991, 951, 1054, 1014, 1032, 1081, 1032, 1017, 1031, 1038, 1042, 941, 1012, 1030, 1025, 1047, 1016, 1067, 1020, 1031, 1020, 1049, 1079, 1064, 998, 1031, 1045, 1016, 1018, 1031, 987, 1008, 1026, 1061, 991, 1038, 1023, 1026, 1033, 1039, 1016, 1020, 1026, 1038, 994, 1031, 1013, 1029, 1015, 1032, 970, 1014, 1025, 1000, 1017, 1011, 1039, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1020, 985, 1065, 999, 1035, 991, 1082, 1033, 1086, 986, 1038, 1014, 1039, 1026, 1016, 984, 991, 1014, 1040, 1025, 1020, 1031, 1014, 1075, 1053, 1037, 1016, 1062, 939, 1050, 1012, 1027, 1065, 1017, 985, 1021, 1053, 1023, 996, 1018, 989, 1032, 1029, 1066, 1055, 1012, 1014, 1015, 1026, 993, 983, 1030, 1030, 1028, 1057, 1024, 1040, 1027, 997, 990, 1007, 1044, 1046, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1003, 1021, 1048, 1035, 1021, 1022, 1065, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1034, 1017, 1033, 1007, 1060, 994, 999, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1034, 1011, 1036, 987, 1035, 1032, 1016, 23, 0, 0, 0, 0, 0, 0, 0, 1000, 1016, 973, 1023, 997, 1013, 1048, 1051, 7, 0, 0, 0, 0, 0, 0, 0, 1037, 1056, 1031, 1052, 1041, 1038, 991, 1006, 11, 0, 0, 0, 0, 0, 0, 0, 995, 1067, 985, 1071, 1015, 1067, 1000, 1031, 17, 2, 0, 0, 0, 0, 0, 0, 1048, 1082, 1033, 1013, 1006, 986, 1034, 1003, 1, 0, 0, 0, 0, 0, 0, 0, 1028, 973, 1009, 1082, 1015, 998, 1024, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8167, 8170, 8128, 8263, 8250, 8206, 8140] avg=8192.0000 max=8263 max_rank=4 min=8128 min_rank=3 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1034, 1085, 1043, 1033, 975, 1017, 1042, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 1035, 1007, 960, 1045, 999, 1021, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1020, 1008, 1006, 1003, 974, 985, 975, 23, 0, 0, 0, 0, 0, 0, 0, 1026, 1044, 975, 1044, 1026, 1068, 993, 1042, 7, 0, 0, 0, 0, 0, 0, 0, 995, 1055, 1067, 1031, 991, 1026, 1019, 1015, 11, 0, 0, 0, 0, 0, 0, 0, 1067, 1029, 1058, 975, 1025, 1041, 1003, 1001, 17, 2, 0, 0, 0, 0, 0, 0, 1004, 1009, 1032, 1002, 1027, 1017, 1043, 1050, 1, 0, 0, 0, 0, 0, 0, 0, 1046, 1038, 1031, 1032, 1010, 1061, 1048, 959, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8151, 8052, 8225, 8210, 8218, 8185, 8225] avg=8192.0000 max=8270 max_rank=0 min=8052 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[996, 974, 1041, 1034, 1090, 1007, 1017, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1034, 1055, 1042, 1008, 1042, 982, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1035, 1015, 994, 1051, 1037, 1015, 1044, 23, 0, 0, 0, 0, 0, 0, 0, 1048, 980, 1056, 1017, 978, 1060, 1065, 993, 8, 0, 0, 0, 0, 0, 0, 0, 1063, 970, 1053, 1030, 1014, 979, 1014, 1006, 11, 0, 0, 0, 0, 0, 0, 0, 993, 1025, 1017, 997, 934, 993, 996, 1043, 17, 2, 0, 0, 0, 0, 0, 0, 1013, 1017, 1035, 1028, 1014, 1055, 1046, 996, 1, 0, 0, 0, 0, 0, 0, 0, 1021, 1072, 1020, 1001, 1043, 1034, 1102, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8179, 8269, 8205, 8140, 8017, 8205, 8301] avg=8192.0000 max=8301 max_rank=7 min=8017 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32663 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1010, 975, 1048, 995, 1098, 1030, 1057, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1045, 1052, 1002, 1054, 991, 1038, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1019, 1003, 1026, 1016, 1016, 1039, 979, 23, 0, 0, 0, 0, 0, 0, 0, 1015, 1044, 1030, 1019, 1045, 965, 1061, 1043, 7, 0, 0, 0, 0, 0, 0, 0, 1016, 1010, 1033, 983, 1002, 1014, 997, 1038, 11, 0, 0, 0, 0, 0, 0, 0, 1049, 1033, 1039, 1051, 1035, 1015, 1073, 1025, 18, 2, 0, 0, 0, 0, 0, 0, 1044, 1004, 1025, 1007, 983, 993, 1044, 1053, 1, 0, 0, 0, 0, 0, 0, 0, 963, 1052, 1010, 1016, 1004, 987, 1014, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8245, 8162, 8229, 8104, 8340, 8154, 8064] avg=8192.0000 max=8340 max_rank=5 min=8064 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1054, 1030, 1013, 982, 1005, 1019, 997, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1003, 1023, 1016, 1051, 1001, 1013, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1053, 1039, 1038, 981, 993, 1042, 1014, 23, 0, 0, 0, 0, 0, 0, 0, 1037, 1039, 1058, 995, 1044, 1048, 971, 964, 7, 0, 0, 0, 0, 0, 0, 0, 976, 1018, 1101, 1034, 1051, 1084, 994, 1007, 11, 0, 0, 0, 0, 0, 0, 0, 1018, 1034, 1014, 997, 1014, 1080, 1021, 1049, 17, 2, 0, 0, 0, 0, 0, 0, 1041, 1060, 1031, 995, 1002, 1002, 1007, 1010, 1, 0, 0, 0, 0, 0, 0, 0, 1023, 998, 1058, 1047, 1011, 1077, 997, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8099, 8225, 8163, 8276, 8246, 8149, 8209] avg=8192.0000 max=8276 max_rank=4 min=8099 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1023, 1013, 1065, 1014, 1036, 983, 1053, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1024, 1015, 1074, 1052, 1035, 1034, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1030, 1031, 1038, 993, 1057, 1028, 1001, 22, 0, 0, 0, 0, 0, 0, 0, 1008, 982, 995, 1008, 988, 1067, 988, 1033, 8, 0, 0, 0, 0, 0, 0, 0, 1042, 1025, 979, 1020, 1011, 1043, 1013, 1004, 11, 0, 0, 0, 0, 0, 0, 0, 997, 977, 1047, 1023, 991, 980, 1038, 950, 17, 2, 0, 0, 0, 0, 0, 0, 1056, 1048, 1063, 1002, 1045, 1068, 1006, 995, 1, 0, 0, 0, 0, 0, 0, 0, 1068, 1101, 1073, 1003, 1065, 1006, 1011, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8212, 8220, 8077, 8148, 8022, 8284, 8365] avg=8192.0000 max=8365 max_rank=7 min=8022 min_rank=5 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1056, 998, 986, 1022, 1001, 1041, 991, 951, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1014, 1032, 1081, 1032, 1017, 1031, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 941, 1012, 1030, 1025, 1047, 1016, 1067, 27, 0, 0, 0, 0, 0, 0, 0, 1020, 1031, 1020, 1049, 1079, 1064, 998, 1031, 13, 1, 0, 0, 0, 0, 0, 0, 1045, 1016, 1018, 1031, 987, 1008, 1026, 1061, 14, 0, 0, 0, 0, 0, 0, 0, 991, 1038, 1023, 1026, 1033, 1039, 1016, 1020, 23, 4, 0, 0, 0, 0, 0, 0, 1026, 1038, 994, 1031, 1013, 1029, 1015, 1032, 3, 0, 0, 0, 0, 0, 0, 0, 970, 1014, 1025, 1000, 986, 1011, 1039, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 8295, 8207, 8306, 8206, 8213, 8181, 8082] avg=8192.0000 max=8306 max_rank=3 min=8046 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1020, 985, 1065, 999, 1035, 954, 1082, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 986, 1038, 1014, 1039, 1026, 1016, 984, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1014, 1040, 1025, 1020, 1031, 1014, 1075, 21, 0, 0, 0, 0, 0, 0, 0, 1053, 1037, 1016, 1062, 939, 1050, 1012, 1027, 7, 0, 0, 0, 0, 0, 0, 0, 1065, 1017, 985, 1021, 1053, 1023, 996, 1018, 11, 0, 0, 0, 0, 0, 0, 0, 989, 1032, 1029, 1066, 1055, 1012, 1014, 1015, 16, 2, 0, 0, 0, 0, 0, 0, 1026, 993, 983, 1030, 1030, 1028, 1057, 1024, 1, 0, 0, 0, 0, 0, 0, 0, 1040, 1027, 997, 990, 987, 1044, 1046, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8188, 8231, 8203, 8189, 8230, 8172, 8150] avg=8192.0000 max=8231 max_rank=2 min=8150 min_rank=7 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8064, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8448, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1039, 1004, 1009, 986, 1040, 994, 1062, 963, 987, 1039, 1027, 1045, 1044, 1031, 1041, 1004, 1002, 987, 1005, 1014, 1047, 1044, 1049, 1023, 1098, 1032, 1021, 1020, 1012, 1031, 981, 998, 989, 1043, 1019, 1022, 995, 1002, 1020, 991, 1056, 1006, 1012, 1066, 1019, 1044, 1110, 1025, 1035, 1005, 1026, 1023, 1012, 1071, 1026, 1022, 1009, 1009, 1029, 1024, 1063, 1064, 1016, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1015, 1028, 1021, 1073, 1025, 1020, 1030, 1049, 996, 996, 962, 1004, 1019, 1012, 1051, 1046, 993, 1014, 1049, 1023, 1030, 1015, 994, 1018, 1018, 1054, 994, 1008, 1028, 1082, 1026, 1049, 1060, 1019, 1012, 997, 1031, 1001, 968, 1082, 1047, 998, 998, 991, 1004, 1025, 995, 1067, 1045, 1002, 1047, 1046, 1014, 1052, 1060, 1013, 990, 998, 1006, 1014, 1057, 1080, 1019, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1017, 1012, 1039, 1043, 1012, 1026, 1006, 1039, 979, 1014, 1077, 1042, 1045, 1038, 1005, 1002, 971, 1027, 982, 1008, 988, 1016, 982, 1052, 1028, 1044, 979, 1072, 1016, 1026, 1036, 983, 1042, 1049, 991, 1071, 1044, 1044, 1032, 1053, 1015, 1056, 965, 1032, 1045, 1059, 1024, 1085, 992, 1026, 1047, 1019, 1051, 1058, 1046, 958, 1003, 999, 1071, 1011, 1031, 996, 1024, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[969, 1029, 1025, 1024, 1046, 1032, 1010, 991, 987, 1034, 1083, 1052, 1013, 1031, 1029, 1069, 1036, 1034, 996, 1067, 1015, 1055, 1033, 1041, 1007, 1010, 1054, 1051, 1016, 1009, 1020, 1046, 1070, 1026, 1011, 1019, 1029, 1001, 1016, 964, 1040, 1021, 1028, 1032, 1034, 1009, 989, 999, 1047, 1001, 1066, 976, 994, 1066, 1014, 1020, 994, 1046, 1017, 1028, 1011, 1033, 1005, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1031, 1048, 978, 998, 1025, 1057, 1037, 1045, 969, 1012, 998, 1035, 1035, 1011, 1004, 976, 983, 1047, 1015, 950, 1003, 1093, 1047, 1000, 1013, 1058, 1058, 1035, 1081, 1073, 1002, 984, 985, 955, 1092, 1035, 1006, 1015, 1036, 1050, 1026, 1014, 978, 1093, 1008, 1066, 1018, 1016, 1058, 1013, 988, 1000, 1035, 1064, 989, 1033, 1030, 1005, 1014, 1030, 1014, 1036, 1082, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1051, 1011, 1061, 1057, 990, 1069, 1064, 942, 1066, 1017, 1061, 995, 1042, 1023, 1019, 995, 981, 1025, 1049, 993, 1040, 1010, 1011, 995, 1045, 993, 1044, 1072, 978, 1016, 986, 1045, 1034, 991, 1012, 1019, 985, 1028, 1008, 1052, 977, 1044, 1041, 1038, 1042, 1077, 1059, 988, 1026, 1029, 1033, 996, 1046, 1036, 979, 1074, 1022, 1009, 1050, 983, 1049, 1036, 989, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1046, 1080, 1021, 1046, 1019, 998, 1015, 1038, 1038, 1055, 983, 993, 1046, 1020, 1024, 997, 1018, 1012, 1018, 1019, 1008, 1054, 1039, 1040, 1034, 1008, 1002, 1010, 1031, 1092, 1006, 1011, 1032, 1065, 1052, 996, 1056, 991, 1000, 1041, 1057, 1003, 1008, 1015, 1011, 972, 988, 996, 1036, 1066, 1046, 1035, 1049, 1004, 999, 1041, 1010, 1004, 1006, 997, 1077, 1002, 1051, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[996, 1017, 1045, 1079, 1043, 1005, 982, 1011, 1025, 1025, 1002, 1002, 1032, 1002, 1027, 1074, 1066, 1003, 1066, 1043, 1025, 1015, 1019, 1027, 942, 1043, 1049, 1057, 1024, 1092, 997, 1038, 1019, 1037, 1006, 1056, 990, 1046, 1013, 1068, 1009, 1039, 1004, 950, 1036, 993, 974, 994, 1046, 1020, 989, 1016, 1092, 1028, 1026, 1035, 1025, 1013, 1038, 996, 1025, 985, 1066, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1017, 1012, 1039, 1038, 1012, 1026, 1006, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1014, 1077, 1042, 1045, 1038, 1005, 1002, 8, 7, 0, 0, 0, 0, 0, 0, 971, 1027, 982, 1008, 988, 1016, 982, 1052, 26, 0, 0, 0, 0, 0, 0, 0, 1028, 1044, 979, 1072, 1016, 1000, 1036, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1049, 991, 1071, 1044, 1044, 1032, 1053, 6, 0, 0, 0, 0, 0, 0, 0, 1015, 1056, 965, 1032, 1045, 1059, 1024, 1085, 12, 0, 0, 0, 0, 0, 0, 0, 992, 1026, 1047, 1019, 1051, 1025, 1046, 958, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 999, 1071, 1011, 1031, 996, 1024, 991, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8217, 8052, 8158, 8332, 8293, 8164, 8131] avg=8192.0000 max=8332 max_rank=4 min=8052 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1031, 1048, 978, 993, 1025, 1057, 1037, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1012, 998, 1035, 1035, 1011, 1004, 976, 8, 7, 0, 0, 0, 0, 0, 0, 983, 1047, 1015, 950, 1003, 1093, 1047, 1000, 26, 0, 0, 0, 0, 0, 0, 0, 1013, 1058, 1058, 1035, 1081, 1046, 1002, 984, 0, 0, 0, 0, 0, 0, 0, 0, 985, 955, 1092, 1035, 1006, 1015, 1036, 1050, 6, 0, 0, 0, 0, 0, 0, 0, 1026, 1014, 978, 1093, 1008, 1066, 1018, 1016, 13, 0, 0, 0, 0, 0, 0, 0, 1058, 1013, 988, 1000, 1035, 1031, 989, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1005, 1014, 1030, 1014, 1036, 1082, 1051, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8055, 8164, 8277, 8180, 8232, 8147, 8267] avg=8192.0000 max=8277 max_rank=3 min=8055 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[969, 1029, 1025, 1019, 1046, 1032, 1010, 991, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1034, 1083, 1052, 1013, 1031, 1029, 1069, 8, 7, 0, 0, 0, 0, 0, 0, 1036, 1034, 996, 1067, 1015, 1055, 1033, 1041, 26, 0, 0, 0, 0, 0, 0, 0, 1007, 1010, 1054, 1051, 1016, 983, 1020, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1026, 1011, 1019, 1029, 1001, 1016, 964, 6, 0, 0, 0, 0, 0, 0, 0, 1040, 1021, 1028, 1032, 1034, 1009, 989, 999, 12, 0, 0, 0, 0, 0, 0, 0, 1047, 1001, 1066, 976, 994, 1033, 1014, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1046, 1017, 1028, 1011, 1033, 1005, 1016, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8313, 8303, 8187, 8142, 8164, 8151, 8155] avg=8192.0000 max=8313 max_rank=1 min=8121 min_rank=0 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32612 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1039, 1004, 1009, 981, 1040, 994, 1062, 963, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1039, 1027, 1045, 1044, 1031, 1041, 1004, 8, 7, 0, 0, 0, 0, 0, 0, 1002, 987, 1005, 1014, 1047, 1044, 1049, 1023, 27, 0, 0, 0, 0, 0, 0, 0, 1098, 1032, 1021, 1020, 1012, 1005, 981, 998, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1043, 1019, 1022, 995, 1002, 1020, 991, 6, 0, 0, 0, 0, 0, 0, 0, 1056, 1006, 1012, 1066, 1019, 1044, 1110, 1025, 12, 0, 0, 0, 0, 0, 0, 0, 1035, 1005, 1026, 1023, 1012, 1037, 1026, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1009, 1029, 1024, 1063, 1064, 1016, 1004, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8092, 8233, 8198, 8167, 8087, 8350, 8186, 8223] avg=8192.0000 max=8350 max_rank=5 min=8087 min_rank=4 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1015, 1028, 1021, 1067, 1025, 1020, 1030, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 996, 996, 962, 1004, 1019, 1012, 1051, 1046, 8, 7, 0, 0, 0, 0, 0, 0, 993, 1014, 1049, 1023, 1030, 1015, 994, 1018, 26, 0, 0, 0, 0, 0, 0, 0, 1018, 1054, 994, 1008, 1028, 1055, 1026, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1019, 1012, 997, 1031, 1001, 968, 1082, 6, 0, 0, 0, 0, 0, 0, 0, 1047, 998, 998, 991, 1004, 1025, 995, 1067, 13, 0, 0, 0, 0, 0, 0, 0, 1045, 1002, 1047, 1046, 1014, 1019, 1060, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 990, 998, 1006, 1014, 1057, 1080, 1019, 1056, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8101, 8162, 8232, 8176, 8138, 8246, 8226] avg=8192.0000 max=8255 max_rank=0 min=8101 min_rank=1 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1051, 1011, 1061, 1052, 990, 1069, 1064, 942, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1017, 1061, 995, 1042, 1023, 1019, 995, 8, 7, 0, 0, 0, 0, 0, 0, 981, 1025, 1049, 993, 1040, 1010, 1011, 995, 26, 0, 0, 0, 0, 0, 0, 0, 1045, 993, 1044, 1072, 978, 990, 986, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 991, 1012, 1019, 985, 1028, 1008, 1052, 6, 0, 0, 0, 0, 0, 0, 0, 977, 1044, 1041, 1038, 1042, 1077, 1059, 988, 12, 0, 0, 0, 0, 0, 0, 0, 1026, 1029, 1033, 996, 1046, 1003, 979, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1009, 1050, 983, 1049, 1036, 989, 1038, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8233, 8130, 8153, 8135, 8278, 8186, 8181] avg=8192.0000 max=8278 max_rank=5 min=8130 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1046, 1080, 1021, 1036, 1019, 998, 1015, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1055, 983, 993, 1046, 1020, 1024, 997, 13, 10, 0, 0, 0, 0, 0, 0, 1018, 1012, 1018, 1019, 1008, 1054, 1039, 1040, 29, 0, 0, 0, 0, 0, 0, 0, 1034, 1008, 1002, 1010, 1031, 1046, 1006, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1065, 1052, 996, 1056, 991, 1000, 1041, 10, 0, 0, 0, 0, 0, 0, 0, 1057, 1003, 1008, 1015, 1011, 972, 988, 996, 19, 0, 0, 0, 0, 0, 0, 0, 1036, 1066, 1046, 1035, 1049, 965, 999, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1004, 1006, 997, 1077, 1002, 1051, 1009, 10, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8179, 8237, 8148, 8243, 8069, 8237, 8170] avg=8192.0000 max=8253 max_rank=0 min=8069 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[996, 1017, 1045, 1073, 1043, 1005, 982, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1025, 1002, 1002, 1032, 1002, 1027, 1074, 9, 7, 0, 0, 0, 0, 0, 0, 1066, 1003, 1066, 1043, 1025, 1015, 1019, 1027, 26, 0, 0, 0, 0, 0, 0, 0, 942, 1043, 1049, 1057, 1024, 1064, 997, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1037, 1006, 1056, 990, 1046, 1013, 1068, 6, 0, 0, 0, 0, 0, 0, 0, 1009, 1039, 1004, 950, 1036, 993, 974, 994, 13, 0, 0, 0, 0, 0, 0, 0, 1046, 1020, 989, 1016, 1092, 995, 1026, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1013, 1038, 996, 1025, 985, 1066, 1029, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8205, 8290, 8214, 8241, 8012, 8219, 8183] avg=8192.0000 max=8290 max_rank=2 min=8012 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8192, 8192, 8320, 8192, 8448, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8320, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8192, 8192, 8320, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8448, 8192, 8192, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1008, 1006, 1009, 1057, 1011, 1020, 1074, 1021, 1024, 1043, 1001, 1039, 1051, 1004, 1071, 1028, 1006, 1087, 1059, 1018, 1065, 1006, 991, 1024, 1010, 979, 1050, 1013, 1019, 1053, 1036, 981, 1033, 1051, 1003, 981, 1010, 1017, 1019, 1027, 1102, 1075, 1014, 1007, 1000, 1050, 1017, 1037, 1027, 1001, 1013, 987, 986, 1012, 1056, 1052, 973, 990, 1010, 996, 997, 1001, 1037, 1091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1007, 1020, 1028, 1046, 984, 1034, 1029, 965, 996, 996, 1047, 987, 1014, 1035, 1036, 989, 1043, 1029, 1029, 1062, 1021, 1052, 1008, 1021, 1036, 1040, 1056, 1042, 1035, 1031, 1054, 1002, 1033, 1032, 1026, 983, 994, 1035, 1081, 1049, 1014, 1043, 963, 1035, 1053, 1058, 990, 1005, 1060, 996, 987, 1005, 991, 1014, 1057, 1013, 978, 1066, 1032, 1021, 1079, 1036, 1013, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1034, 1049, 1017, 1028, 1031, 1066, 1046, 1076, 1046, 1051, 991, 987, 1053, 992, 999, 1038, 1065, 1042, 999, 1035, 1024, 1012, 989, 1045, 1000, 1049, 1052, 1015, 1006, 1032, 1005, 1019, 955, 1026, 1044, 1004, 1023, 1004, 1033, 1009, 1073, 1014, 995, 991, 1000, 992, 1060, 1050, 1056, 1040, 1056, 1029, 981, 1048, 984, 1048, 1009, 1040, 1035, 1014, 965, 983, 1057, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[956, 1026, 1029, 1034, 1072, 1008, 1009, 1012, 1052, 1029, 1026, 1035, 1022, 1020, 1042, 1024, 1013, 1012, 993, 1026, 1003, 1046, 1009, 975, 1063, 1006, 1010, 1022, 1016, 1011, 1035, 1032, 1013, 1008, 1057, 1028, 1025, 994, 1060, 1044, 1016, 1017, 1031, 1021, 1034, 1072, 1000, 1025, 1015, 1044, 1071, 1062, 1045, 1010, 1032, 995, 1044, 1016, 962, 1001, 1044, 1023, 1041, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1023, 997, 1037, 1058, 1075, 1065, 991, 1089, 1073, 1002, 995, 1040, 1062, 1017, 1027, 1023, 996, 1016, 1018, 993, 1019, 983, 1044, 1050, 1011, 1067, 1049, 987, 1038, 1033, 1014, 1051, 1031, 1076, 1017, 1023, 1017, 1035, 997, 1021, 1014, 1019, 1005, 1011, 1025, 1061, 1008, 1021, 1001, 984, 1001, 1003, 1021, 1004, 1045, 1056, 1042, 1052, 972, 1011, 1025, 982, 983, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1007, 1025, 1023, 995, 1013, 1037, 1066, 1007, 997, 1033, 1002, 985, 1010, 1087, 1049, 1012, 1012, 1042, 976, 1000, 1055, 1030, 1020, 1015, 1022, 1016, 1031, 1033, 1030, 1058, 1033, 1037, 1046, 1062, 1020, 995, 1039, 979, 1033, 1026, 1006, 1010, 1053, 1042, 976, 1037, 1050, 1038, 999, 1018, 1005, 1040, 1076, 1059, 1030, 1039, 958, 1047, 1023, 997, 1008, 1038, 1028, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1002, 1008, 1011, 1030, 1001, 1012, 1040, 1007, 1027, 1057, 1008, 1079, 998, 1037, 1025, 1079, 1045, 1058, 992, 1009, 1047, 989, 975, 1002, 1063, 1002, 975, 1033, 1047, 1034, 1025, 1088, 1037, 1045, 1019, 1011, 1072, 1047, 1013, 993, 1055, 1016, 973, 999, 1004, 1000, 976, 1038, 1025, 1027, 1028, 1009, 1038, 1048, 1052, 1096, 992, 1021, 1009, 1064, 1012, 1007, 1014, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[991, 1025, 977, 1000, 1038, 1014, 1029, 1029, 1031, 1012, 1058, 1096, 994, 1028, 991, 1041, 1080, 1025, 1057, 989, 1033, 1029, 1016, 1031, 1012, 1076, 1013, 1041, 1055, 1004, 992, 1027, 988, 1069, 1042, 1041, 990, 1031, 1051, 1037, 1033, 1064, 1019, 1049, 1052, 996, 1041, 976, 1027, 1021, 1030, 1027, 969, 1044, 1018, 1000, 1046, 973, 1014, 1023, 992, 985, 1043, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1007, 1020, 1028, 1046, 984, 1034, 1029, 965, 0, 0, 0, 0, 0, 0, 0, 0, 996, 996, 1047, 963, 1014, 1035, 1036, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1029, 1029, 1062, 1021, 1052, 1008, 1021, 6, 3, 0, 0, 0, 0, 0, 0, 1036, 1040, 1056, 1042, 1035, 998, 1054, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1025, 1026, 983, 994, 1035, 1081, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1043, 963, 1035, 1053, 1058, 990, 1005, 1, 0, 0, 0, 0, 0, 0, 0, 1060, 996, 987, 1005, 991, 1014, 1057, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1066, 1032, 1021, 1079, 1036, 1013, 1020, 33, 24, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8076, 8274, 8263, 8226, 8162, 8115, 8307] avg=8192.0000 max=8307 max_rank=7 min=8076 min_rank=1 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1023, 997, 1037, 1058, 1075, 1065, 991, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1002, 995, 1014, 1062, 1017, 1027, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1016, 1018, 993, 1019, 983, 1044, 1050, 6, 3, 0, 0, 0, 0, 0, 0, 1011, 1067, 1049, 987, 1038, 1000, 1014, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1069, 1017, 1023, 1017, 1035, 997, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1019, 1005, 1011, 1025, 1061, 1008, 1021, 1, 0, 0, 0, 0, 0, 0, 0, 1001, 984, 1001, 1003, 1021, 1004, 1045, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1052, 972, 1011, 1025, 982, 983, 1030, 33, 26, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8335, 8213, 8128, 8217, 8210, 8165, 8107, 8161] avg=8192.0000 max=8335 max_rank=0 min=8107 min_rank=6 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1008, 1006, 1009, 1057, 1011, 1020, 1074, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1043, 1001, 1013, 1051, 1004, 1071, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1087, 1059, 1018, 1065, 1006, 991, 1024, 6, 3, 0, 0, 0, 0, 0, 0, 1010, 979, 1050, 1013, 1019, 1019, 1036, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1044, 1003, 981, 1010, 1017, 1019, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 1075, 1014, 1007, 1000, 1050, 1017, 1037, 1, 0, 0, 0, 0, 0, 0, 0, 1027, 1001, 1013, 987, 986, 1012, 1056, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 973, 990, 1010, 996, 997, 1001, 1037, 1091, 34, 26, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8235, 8265, 8107, 8134, 8303, 8126, 8160] avg=8192.0000 max=8303 max_rank=5 min=8107 min_rank=3 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1034, 1049, 1017, 1028, 1031, 1066, 1046, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1051, 991, 963, 1053, 992, 999, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1042, 999, 1035, 1024, 1012, 989, 1045, 6, 3, 0, 0, 0, 0, 0, 0, 1000, 1049, 1052, 1015, 1006, 999, 1005, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 955, 1019, 1044, 1004, 1023, 1004, 1033, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1014, 995, 991, 1000, 992, 1060, 1050, 1, 0, 0, 0, 0, 0, 0, 0, 1056, 1040, 1056, 1029, 981, 1048, 984, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1040, 1035, 1014, 965, 983, 1057, 1025, 33, 24, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8347, 8133, 8220, 8145, 8091, 8176, 8234, 8190] avg=8192.0000 max=8347 max_rank=0 min=8091 min_rank=4 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1007, 1025, 1023, 995, 1013, 1037, 1066, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1033, 1002, 961, 1010, 1087, 1049, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1042, 976, 1000, 1055, 1030, 1020, 1015, 6, 3, 0, 0, 0, 0, 0, 0, 1022, 1016, 1031, 1033, 1030, 1024, 1033, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1055, 1020, 995, 1039, 979, 1033, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1010, 1053, 1042, 976, 1037, 1050, 1038, 1, 0, 0, 0, 0, 0, 0, 0, 999, 1018, 1005, 1040, 1076, 1059, 1030, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 958, 1047, 1023, 997, 1008, 1038, 1028, 1001, 34, 24, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8151, 8159, 8226, 8193, 8213, 8258, 8163] avg=8192.0000 max=8258 max_rank=6 min=8151 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[956, 1026, 1029, 1034, 1072, 1008, 1009, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1029, 1026, 1010, 1022, 1020, 1042, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1012, 993, 1026, 1003, 1046, 1009, 975, 6, 3, 0, 0, 0, 0, 0, 0, 1063, 1006, 1010, 1022, 1016, 978, 1035, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1001, 1057, 1028, 1025, 994, 1060, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1017, 1031, 1021, 1034, 1072, 1000, 1025, 1, 0, 0, 0, 0, 0, 0, 0, 1015, 1044, 1071, 1062, 1045, 1010, 1032, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1016, 962, 1001, 1044, 1023, 1041, 1018, 33, 25, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8225, 8086, 8162, 8222, 8217, 8266, 8212] avg=8192.0000 max=8266 max_rank=6 min=8086 min_rank=2 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1002, 1008, 1011, 1030, 1001, 1012, 1040, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1057, 1008, 1052, 998, 1037, 1025, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1058, 992, 1009, 1047, 989, 975, 1002, 6, 3, 0, 0, 0, 0, 0, 0, 1063, 1002, 975, 1033, 1047, 1001, 1025, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1038, 1019, 1011, 1072, 1047, 1013, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1016, 973, 999, 1004, 1000, 976, 1038, 1, 0, 0, 0, 0, 0, 0, 0, 1025, 1027, 1028, 1009, 1038, 1048, 1052, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1021, 1009, 1064, 1012, 1007, 1014, 991, 33, 27, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8283, 8126, 8234, 8230, 8062, 8314, 8176] avg=8192.0000 max=8314 max_rank=6 min=8062 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[991, 1025, 977, 1000, 1038, 1014, 1029, 1029, 2, 0, 0, 0, 0, 0, 0, 0, 1031, 1012, 1058, 1065, 994, 1028, 991, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 1025, 1057, 989, 1033, 1029, 1016, 1031, 9, 9, 0, 0, 0, 0, 0, 0, 1012, 1076, 1013, 1041, 1055, 966, 992, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1050, 1042, 1041, 990, 1031, 1051, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1064, 1019, 1049, 1052, 996, 1041, 976, 8, 0, 0, 0, 0, 0, 0, 0, 1027, 1021, 1030, 1027, 969, 1044, 1018, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 973, 1014, 1023, 992, 985, 1043, 1011, 38, 31, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8220, 8278, 8182, 8230, 8238, 8116, 8167] avg=8192.0000 max=8278 max_rank=2 min=8105 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8064, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8192, 8192, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8064, 8192, 8192, 8064, 8320, 8192, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8064, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8192, 8320, 8320, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1013, 1037, 963, 980, 1054, 1034, 1017, 965, 1013, 1020, 1020, 1023, 1060, 1024, 1012, 1037, 1047, 1020, 1042, 1004, 982, 1013, 1042, 1057, 1031, 1011, 1031, 1037, 1019, 1034, 1020, 1027, 1065, 1064, 1019, 1012, 1005, 1018, 1030, 1018, 972, 1069, 988, 995, 1026, 1009, 1038, 1034, 1037, 1052, 1045, 1056, 1070, 1033, 1035, 1004, 1001, 1004, 1056, 1069, 1013, 1022, 967, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1032, 1003, 1002, 1027, 998, 1070, 1065, 1035, 994, 1015, 1083, 1044, 944, 1018, 1025, 1036, 1009, 1082, 1038, 997, 997, 1074, 980, 1031, 1017, 1038, 1044, 1063, 1078, 997, 1037, 1029, 1009, 988, 1017, 1032, 1041, 968, 961, 1049, 1079, 977, 1099, 1065, 1016, 1059, 1044, 967, 1000, 1029, 1011, 1016, 1037, 1047, 1016, 1029, 1002, 1027, 1010, 976, 996, 1022, 1015, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[984, 1031, 995, 1019, 956, 1027, 1003, 1058, 994, 1007, 1016, 1063, 1022, 1040, 1050, 969, 1005, 992, 1018, 1059, 1072, 1033, 1021, 1024, 968, 1033, 1036, 1070, 1043, 1009, 1112, 994, 1023, 1046, 974, 1009, 1068, 1026, 1022, 972, 1061, 1045, 1061, 1027, 1038, 1018, 1019, 1061, 957, 1048, 1027, 982, 1025, 1008, 1027, 1001, 1043, 1090, 994, 1074, 999, 990, 1034, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1049, 1011, 1073, 1004, 981, 1062, 994, 991, 971, 1015, 1053, 952, 992, 1000, 1006, 1034, 1051, 989, 1037, 1045, 1003, 1062, 1034, 996, 1058, 1070, 1042, 1042, 1009, 1021, 969, 1040, 1049, 1010, 1073, 1040, 1043, 1012, 1016, 1017, 1064, 1028, 1014, 1025, 1052, 1002, 1054, 987, 1060, 1011, 1014, 1026, 1026, 1048, 997, 1041, 1015, 958, 1053, 1069, 1034, 1034, 1015, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1021, 1037, 1077, 997, 1046, 1042, 1029, 986, 1065, 982, 1047, 1051, 1065, 1019, 1016, 1019, 1028, 1043, 1023, 1069, 1063, 1039, 989, 1045, 1038, 975, 1031, 998, 1008, 1058, 1046, 1018, 1006, 1095, 1066, 974, 1067, 1020, 1000, 1026, 1019, 1023, 1013, 999, 1025, 1040, 1014, 1006, 1013, 1006, 914, 1066, 1019, 1009, 988, 1011, 981, 1005, 962, 1074, 1003, 1044, 1056, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1043, 990, 973, 1023, 1037, 1037, 1048, 1045, 1040, 1060, 1019, 1007, 1019, 1011, 990, 1037, 1034, 1045, 1034, 1003, 1027, 1016, 1069, 988, 1005, 984, 1060, 1073, 1060, 1015, 985, 1027, 1044, 982, 1005, 1062, 1053, 1040, 990, 1029, 999, 1006, 1043, 1042, 1002, 997, 1063, 1021, 989, 989, 1016, 993, 1034, 1054, 1027, 999, 1025, 1020, 1027, 1022, 1097, 1029, 997, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[985, 1034, 1047, 1045, 1007, 1003, 1054, 1046, 990, 1062, 1011, 998, 1027, 1043, 987, 999, 1009, 1035, 1002, 987, 1007, 1074, 981, 1049, 1057, 1021, 1015, 1023, 1052, 1043, 1020, 1004, 1043, 1004, 997, 1007, 1061, 1055, 1025, 1038, 1016, 975, 1029, 1059, 1016, 1019, 1017, 1014, 1021, 1049, 1017, 1062, 1004, 1050, 973, 1012, 1024, 1031, 1007, 1081, 1018, 1004, 1013, 1078] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1010, 1033, 1018, 1041, 1076, 1030, 1026, 999, 996, 997, 1028, 1003, 1037, 994, 1044, 1014, 1046, 1018, 1016, 993, 1012, 1067, 1031, 975, 962, 1058, 1038, 1024, 1038, 991, 1043, 1048, 1026, 1014, 1016, 1067, 1050, 1039, 979, 1048, 1064, 1026, 1021, 1063, 1024, 1083, 1007, 1013, 1011, 967, 1036, 1069, 1086, 985, 1059, 1011, 951, 1004, 1000, 1015, 1019, 1018, 1026, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1013, 1037, 963, 980, 1054, 1034, 1017, 965, 10, 3, 0, 0, 0, 0, 0, 0, 1013, 1020, 1020, 1023, 1060, 1024, 1012, 1037, 38, 0, 0, 0, 0, 0, 0, 0, 1047, 1020, 1042, 1004, 982, 998, 1042, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1011, 1031, 999, 1019, 1034, 1020, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1064, 1019, 1012, 995, 1018, 1030, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1069, 988, 970, 1026, 1009, 1038, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1052, 1045, 1056, 1070, 1033, 1035, 1004, 21, 0, 0, 0, 0, 0, 0, 0, 1001, 1004, 1056, 1069, 1013, 1022, 967, 1021, 12, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8076, 8247, 8192, 8172, 8221, 8106, 8353, 8169] avg=8192.0000 max=8353 max_rank=6 min=8076 min_rank=0 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1049, 1011, 1073, 1004, 981, 1062, 994, 991, 10, 3, 0, 0, 0, 0, 0, 0, 971, 1015, 1053, 952, 992, 1000, 1006, 1034, 38, 0, 0, 0, 0, 0, 0, 0, 1051, 989, 1037, 1045, 1003, 1046, 1034, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1070, 1042, 1004, 1009, 1021, 969, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1010, 1073, 1040, 1033, 1012, 1016, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1028, 1014, 999, 1052, 1002, 1054, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1011, 1014, 1026, 1026, 1048, 997, 1041, 22, 0, 0, 0, 0, 0, 0, 0, 1015, 958, 1053, 1069, 1034, 1034, 1015, 993, 13, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8061, 8201, 8213, 8250, 8200, 8245, 8188] avg=8192.0000 max=8250 max_rank=4 min=8061 min_rank=1 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1032, 1003, 1002, 1027, 998, 1070, 1065, 1035, 10, 3, 0, 0, 0, 0, 0, 0, 994, 1015, 1083, 1044, 944, 1018, 1025, 1036, 39, 0, 0, 0, 0, 0, 0, 0, 1009, 1082, 1038, 997, 997, 1058, 980, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1038, 1044, 1024, 1078, 997, 1037, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 988, 1017, 1032, 1031, 968, 961, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 977, 1099, 1039, 1016, 1059, 1044, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1029, 1011, 1016, 1037, 1047, 1016, 1029, 22, 0, 0, 0, 0, 0, 0, 0, 1002, 1027, 1010, 976, 996, 1022, 1015, 1030, 13, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8198, 8192, 8264, 8055, 8280, 8207, 8095] avg=8192.0000 max=8280 max_rank=5 min=8055 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1021, 1037, 1077, 997, 1046, 1042, 1029, 986, 11, 3, 0, 0, 0, 0, 0, 0, 1065, 982, 1047, 1051, 1065, 1019, 1016, 1019, 36, 0, 0, 0, 0, 0, 0, 0, 1028, 1043, 1023, 1069, 1063, 1023, 989, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 975, 1031, 962, 1008, 1058, 1046, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1095, 1066, 974, 1056, 1020, 1000, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1023, 1013, 974, 1025, 1040, 1014, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1006, 914, 1066, 1019, 1009, 988, 1011, 21, 0, 0, 0, 0, 0, 0, 0, 981, 1005, 962, 1074, 1003, 1044, 1056, 1022, 13, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8300, 8283, 8136, 8243, 8114, 8047, 8164] avg=8192.0000 max=8300 max_rank=1 min=8047 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[984, 1031, 995, 1019, 956, 1027, 1003, 1058, 11, 3, 0, 0, 0, 0, 0, 0, 994, 1007, 1016, 1063, 1022, 1040, 1050, 969, 39, 0, 0, 0, 0, 0, 0, 0, 1005, 992, 1018, 1059, 1072, 1017, 1021, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1033, 1036, 1031, 1043, 1009, 1112, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1046, 974, 1009, 1057, 1026, 1022, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1045, 1061, 1001, 1038, 1018, 1019, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 957, 1048, 1027, 982, 1025, 1008, 1027, 1001, 22, 0, 0, 0, 0, 0, 0, 0, 1043, 1090, 994, 1074, 999, 990, 1034, 1044, 13, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8200, 8208, 8226, 8129, 8304, 8097, 8285] avg=8192.0000 max=8304 max_rank=5 min=8087 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1043, 990, 973, 1023, 1037, 1037, 1048, 1045, 11, 3, 0, 0, 0, 0, 0, 0, 1040, 1060, 1019, 1007, 1019, 1011, 990, 1037, 39, 0, 0, 0, 0, 0, 0, 0, 1034, 1045, 1034, 1003, 1027, 1001, 1069, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 984, 1060, 1034, 1060, 1015, 985, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 982, 1005, 1062, 1042, 1040, 990, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1006, 1043, 1016, 1002, 997, 1063, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 989, 989, 1016, 993, 1034, 1054, 1027, 999, 22, 0, 0, 0, 0, 0, 0, 0, 1025, 1020, 1027, 1022, 1097, 1029, 997, 1036, 12, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8222, 8201, 8170, 8194, 8147, 8123, 8269] avg=8192.0000 max=8269 max_rank=7 min=8123 min_rank=6 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[985, 1034, 1047, 1045, 1007, 1003, 1054, 1046, 11, 3, 0, 0, 0, 0, 0, 0, 990, 1062, 1011, 998, 1027, 1043, 987, 999, 37, 0, 0, 0, 0, 0, 0, 0, 1009, 1035, 1002, 987, 1007, 1058, 981, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1021, 1015, 986, 1052, 1043, 1020, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1004, 997, 1007, 1050, 1055, 1025, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 975, 1029, 1033, 1016, 1019, 1017, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1049, 1017, 1062, 1004, 1050, 973, 1012, 22, 0, 0, 0, 0, 0, 0, 0, 1024, 1031, 1007, 1081, 1018, 1004, 1013, 1078, 13, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8154, 8128, 8198, 8219, 8119, 8210, 8273] avg=8192.0000 max=8273 max_rank=7 min=8119 min_rank=5 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1010, 1033, 1018, 1041, 1076, 1030, 1026, 999, 14, 9, 0, 0, 0, 0, 0, 0, 996, 997, 1028, 1003, 1037, 994, 1044, 1014, 41, 0, 0, 0, 0, 0, 0, 0, 1046, 1018, 1016, 993, 1012, 1040, 1031, 975, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1058, 1038, 979, 1038, 991, 1043, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1014, 1016, 1067, 1036, 1039, 979, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1026, 1021, 1028, 1024, 1083, 1007, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 967, 1036, 1069, 1086, 985, 1059, 1011, 26, 4, 0, 0, 0, 0, 0, 0, 951, 1004, 1000, 1015, 1019, 1018, 1026, 1033, 18, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8154, 8131, 8157, 8225, 8266, 8254, 8093] avg=8192.0000 max=8266 max_rank=5 min=8093 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8192, 8192, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8448, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8064, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8192, 8320, 8064, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1026, 1037, 986, 1031, 971, 1025, 947, 1008, 1011, 1000, 1032, 1035, 1057, 997, 1022, 984, 1056, 1091, 979, 995, 1057, 1001, 1032, 1010, 1053, 1043, 990, 1032, 1050, 987, 1022, 1031, 1074, 1037, 1028, 1041, 1017, 1015, 1046, 946, 1062, 1058, 1003, 1009, 1062, 1078, 1053, 1059, 1052, 1034, 1050, 1047, 1006, 992, 1007, 1037, 1036, 1069, 1009, 992, 1016, 990, 1007, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1007, 1009, 971, 1049, 999, 1027, 1035, 1041, 1040, 987, 989, 1052, 1007, 994, 1009, 1059, 1036, 1056, 1050, 1019, 1011, 990, 1017, 1062, 1020, 1050, 1045, 1018, 1033, 1044, 1017, 1050, 1028, 1023, 1020, 1057, 1019, 995, 967, 1057, 968, 1035, 970, 1093, 1030, 992, 1072, 1032, 1035, 1056, 1024, 1043, 989, 1030, 1046, 1013, 1017, 963, 1025, 1077, 1041, 1030, 979, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[998, 1028, 1076, 999, 1004, 1014, 1017, 992, 1025, 995, 1017, 986, 1028, 1036, 1058, 1026, 1018, 1064, 1029, 1042, 1016, 1013, 1051, 1027, 1048, 996, 1029, 1036, 1019, 1041, 1035, 1066, 1065, 1009, 1023, 1068, 1024, 1021, 986, 1010, 1031, 1022, 981, 1002, 1083, 1053, 1051, 964, 1012, 1024, 992, 1033, 1055, 1021, 1003, 1052, 1042, 1008, 1021, 1017, 991, 1043, 977, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1012, 1029, 1015, 962, 1005, 1001, 989, 1053, 1005, 1009, 1025, 1031, 1051, 1042, 1025, 996, 1027, 1017, 1023, 992, 1089, 1000, 1058, 996, 1059, 1068, 1010, 1022, 1062, 964, 1027, 1012, 1052, 1001, 1035, 1045, 993, 1074, 1026, 1043, 1050, 1012, 1017, 1000, 1055, 982, 1084, 1007, 1030, 1051, 1043, 1044, 1071, 997, 1033, 1012, 1049, 999, 1018, 942, 1064, 1057, 969, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1010, 1028, 1002, 1076, 1006, 1026, 994, 973, 1038, 1018, 1089, 1021, 1041, 1030, 986, 1052, 1058, 1038, 1028, 1025, 1047, 1000, 1003, 1004, 1006, 1007, 1000, 1001, 1021, 981, 1036, 1010, 1036, 1034, 1038, 995, 1011, 1065, 1020, 1008, 1047, 1032, 1016, 1008, 1023, 986, 1078, 1038, 1009, 983, 1080, 1038, 1011, 1037, 1048, 1007, 1040, 1027, 1078, 970, 1038, 1029, 1014, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1003, 999, 1096, 1031, 1054, 1030, 1059, 1021, 1049, 1004, 1031, 1049, 1005, 1028, 1038, 1000, 1026, 987, 1033, 1035, 1004, 1023, 1001, 1077, 1009, 1046, 1054, 1054, 1020, 1041, 1009, 1035, 1027, 978, 1009, 1016, 1002, 985, 1024, 996, 1085, 1031, 978, 1010, 1018, 997, 1098, 1009, 1012, 1016, 1004, 1032, 1025, 1020, 1000, 1040, 1002, 1060, 1061, 1055, 997, 987, 1007, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1022, 1024, 1040, 1037, 1013, 939, 992, 1007, 1031, 976, 983, 1002, 987, 1062, 1035, 1047, 1038, 1053, 1052, 1003, 1036, 1040, 1019, 989, 1032, 993, 1046, 1029, 1033, 1007, 1035, 994, 999, 1034, 1086, 977, 1017, 1015, 1009, 1054, 1006, 1026, 1022, 1051, 1057, 1020, 1060, 1059, 1012, 1004, 1015, 1015, 1076, 1053, 1001, 1019, 1058, 1016, 1055, 1026, 1046, 989, 1021, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1066, 1019, 993, 968, 1038, 989, 1010, 1026, 1049, 1031, 1044, 1012, 965, 1063, 1034, 1059, 1055, 1043, 1017, 1036, 1029, 1038, 1032, 1034, 949, 1028, 1025, 1057, 1012, 1003, 995, 1031, 1061, 1014, 1016, 1020, 1061, 986, 1011, 1005, 1060, 1014, 1014, 1048, 1008, 1044, 1064, 1039, 1063, 1032, 1015, 990, 1035, 1043, 1070, 966, 1008, 980, 1035, 1009, 1009, 1039, 1009, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[998, 1028, 1076, 999, 1004, 1014, 1017, 992, 58, 12, 0, 0, 0, 0, 0, 0, 1025, 995, 1017, 986, 1028, 1036, 1058, 1026, 5, 0, 0, 0, 0, 0, 0, 0, 1018, 1028, 1029, 1042, 1016, 1013, 1051, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 996, 1029, 1036, 1014, 1041, 1035, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1009, 1023, 1068, 1024, 1021, 986, 1010, 10, 0, 0, 0, 0, 0, 0, 0, 1031, 1022, 981, 1002, 1083, 1053, 993, 964, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1024, 992, 1033, 1035, 1021, 1003, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1008, 1021, 1017, 991, 1043, 977, 1023, 24, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8176, 8224, 8265, 8216, 8129, 8172, 8156] avg=8192.0000 max=8265 max_rank=3 min=8129 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1026, 1037, 986, 1031, 971, 1025, 947, 1008, 59, 13, 0, 0, 0, 0, 0, 0, 1011, 1000, 1032, 1035, 1057, 997, 1022, 984, 5, 0, 0, 0, 0, 0, 0, 0, 1056, 1054, 979, 995, 1057, 1001, 1032, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1043, 990, 1032, 1045, 987, 1022, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1037, 1028, 1041, 1017, 1015, 1046, 946, 9, 0, 0, 0, 0, 0, 0, 0, 1062, 1058, 1003, 1009, 1062, 1078, 994, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1034, 1050, 1047, 988, 992, 1007, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1069, 1009, 992, 1016, 990, 1007, 1006, 24, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8143, 8184, 8203, 8213, 8325, 8207, 8158] avg=8192.0000 max=8325 max_rank=5 min=8103 min_rank=0 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1012, 1029, 1015, 962, 1005, 1001, 989, 1053, 60, 12, 0, 0, 0, 0, 0, 0, 1005, 1009, 1025, 1031, 1051, 1042, 1025, 996, 5, 0, 0, 0, 0, 0, 0, 0, 1027, 982, 1023, 992, 1089, 1000, 1058, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1068, 1010, 1022, 1057, 964, 1027, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1001, 1035, 1045, 993, 1074, 1026, 1043, 10, 0, 0, 0, 0, 0, 0, 0, 1050, 1012, 1017, 1000, 1055, 982, 1024, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1051, 1043, 1044, 1051, 997, 1033, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 999, 1018, 942, 1064, 1057, 969, 1005, 23, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8189, 8167, 8219, 8279, 8147, 8261, 8136] avg=8192.0000 max=8279 max_rank=4 min=8136 min_rank=7 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1007, 1009, 971, 1049, 999, 1027, 1035, 1041, 60, 12, 0, 0, 0, 0, 0, 0, 1040, 987, 989, 1052, 1007, 994, 1009, 1059, 5, 0, 0, 0, 0, 0, 0, 0, 1036, 1021, 1050, 1019, 1011, 990, 1017, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1050, 1045, 1018, 1028, 1044, 1017, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1023, 1020, 1057, 1019, 995, 967, 1057, 9, 0, 0, 0, 0, 0, 0, 0, 968, 1035, 970, 1093, 1030, 992, 1012, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1056, 1024, 1043, 971, 1030, 1046, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 963, 1025, 1077, 1041, 1030, 979, 1017, 23, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8142, 8206, 8272, 8175, 8132, 8218, 8181] avg=8192.0000 max=8272 max_rank=3 min=8132 min_rank=5 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1010, 1028, 1002, 1076, 1006, 1026, 994, 973, 60, 12, 0, 0, 0, 0, 0, 0, 1038, 1018, 1089, 1021, 1041, 1030, 986, 1052, 5, 0, 0, 0, 0, 0, 0, 0, 1058, 1003, 1028, 1025, 1047, 1000, 1003, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1007, 1000, 1001, 1016, 981, 1036, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1034, 1038, 995, 1011, 1065, 1020, 1008, 10, 0, 0, 0, 0, 0, 0, 0, 1047, 1032, 1016, 1008, 1023, 986, 1018, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 983, 1080, 1038, 991, 1037, 1048, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1027, 1078, 970, 1038, 1029, 1014, 1037, 23, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8280, 8168, 8057, 8217, 8168, 8193, 8266] avg=8192.0000 max=8280 max_rank=1 min=8057 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1003, 999, 1096, 1031, 1054, 1030, 1059, 1021, 61, 12, 0, 0, 0, 0, 0, 0, 1049, 1004, 1031, 1049, 1005, 1028, 1038, 1000, 5, 0, 0, 0, 0, 0, 0, 0, 1026, 953, 1033, 1035, 1004, 1023, 1001, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1046, 1054, 1054, 1015, 1041, 1009, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 978, 1009, 1016, 1002, 985, 1024, 996, 10, 0, 0, 0, 0, 0, 0, 0, 1085, 1031, 978, 1010, 1018, 997, 1037, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1016, 1004, 1032, 1005, 1020, 1000, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1060, 1061, 1055, 997, 987, 1007, 1004, 22, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8366, 8209, 8152, 8263, 8047, 8165, 8129, 8205] avg=8192.0000 max=8366 max_rank=0 min=8047 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32546 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1022, 1024, 1040, 1037, 1013, 939, 992, 1007, 59, 12, 0, 0, 0, 0, 0, 0, 1031, 976, 983, 1002, 987, 1062, 1035, 1047, 5, 0, 0, 0, 0, 0, 0, 0, 1038, 1018, 1052, 1003, 1036, 1040, 1019, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 993, 1046, 1029, 1028, 1007, 1035, 994, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1034, 1086, 977, 1017, 1015, 1009, 1054, 10, 0, 0, 0, 0, 0, 0, 0, 1006, 1026, 1022, 1051, 1057, 1020, 1001, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1004, 1015, 1015, 1056, 1053, 1001, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1016, 1055, 1026, 1046, 989, 1021, 1042, 23, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8128, 8195, 8164, 8201, 8242, 8175, 8286] avg=8192.0000 max=8286 max_rank=7 min=8128 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1066, 1019, 993, 968, 1038, 989, 1010, 1026, 63, 17, 0, 0, 0, 0, 0, 0, 1049, 1031, 1044, 1012, 965, 1063, 1034, 1059, 7, 5, 0, 0, 0, 0, 0, 0, 1055, 999, 1017, 1036, 1029, 1038, 1032, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 949, 1028, 1025, 1057, 1005, 1003, 995, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1014, 1016, 1020, 1061, 986, 1011, 1005, 14, 0, 0, 0, 0, 0, 0, 0, 1060, 1014, 1014, 1048, 1008, 1044, 1001, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1032, 1015, 990, 1002, 1043, 1070, 966, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 980, 1035, 1009, 1009, 1039, 1009, 1018, 27, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8269, 8240, 8093, 8188, 8228, 8181, 8148] avg=8192.0000 max=8269 max_rank=1 min=8093 min_rank=3 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8064, 8320, 8192, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8064, 8064, 8192, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8064, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8320, 8448, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[970, 1008, 1043, 1059, 1041, 995, 982, 1031, 1087, 1045, 1057, 1053, 992, 1040, 1067, 1022, 1000, 1080, 1016, 1034, 1054, 1028, 1024, 1038, 1013, 969, 1013, 993, 1072, 1032, 989, 947, 1031, 1014, 1018, 1002, 1096, 1049, 1005, 1007, 1024, 1027, 1042, 984, 1026, 996, 1007, 996, 1048, 995, 1049, 1015, 1019, 1090, 984, 991, 969, 1107, 999, 1007, 1048, 1033, 1034, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1029, 1050, 1028, 1034, 995, 1016, 1047, 1010, 1040, 1020, 1032, 975, 1055, 1048, 1036, 1031, 1059, 1008, 1056, 1019, 1031, 1075, 1047, 991, 994, 1053, 1001, 1035, 1049, 1031, 1008, 990, 1022, 981, 1016, 1029, 1002, 1000, 1057, 1025, 1061, 1063, 982, 1039, 1037, 1018, 1047, 1055, 996, 1027, 1006, 1016, 1060, 1013, 1005, 1045, 979, 1030, 1036, 1042, 969, 1024, 992, 969] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1054, 1013, 1079, 974, 1045, 1026, 1050, 1012, 1040, 1102, 1047, 1021, 994, 955, 1068, 1073, 1044, 1075, 987, 994, 1034, 1015, 999, 1012, 1013, 971, 1019, 1005, 1017, 1012, 1004, 1041, 1023, 1061, 1016, 1033, 1028, 1007, 1053, 1087, 1012, 1000, 1027, 1010, 977, 1045, 980, 1000, 1001, 1003, 1044, 1004, 974, 1016, 1041, 1031, 1031, 1027, 1044, 1037, 1027, 1054, 1033, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[989, 1012, 1060, 1032, 1017, 987, 1039, 1046, 985, 1094, 1047, 1050, 1052, 1050, 1041, 1045, 990, 984, 990, 1058, 1041, 1005, 1009, 1009, 1009, 1013, 1016, 1029, 1042, 1026, 1064, 1053, 1002, 1011, 1008, 1049, 984, 1025, 995, 1022, 996, 1056, 1007, 1033, 1012, 1056, 1032, 1026, 1011, 1017, 1040, 1032, 1069, 978, 1015, 1046, 1037, 1009, 975, 1021, 1036, 1038, 980, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[980, 1011, 1034, 1010, 997, 1004, 1028, 996, 1001, 1033, 1055, 1037, 1029, 1014, 1002, 1002, 1016, 1047, 1026, 1033, 1027, 1038, 995, 1012, 1027, 1032, 1032, 1021, 990, 1043, 1039, 1034, 1028, 938, 1019, 1010, 1029, 1047, 983, 1070, 1043, 1044, 1045, 1010, 1021, 1013, 1028, 1079, 1033, 1025, 1018, 1048, 1033, 1056, 1050, 1014, 1044, 1036, 966, 999, 1043, 1025, 1055, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1001, 1016, 1078, 1065, 1037, 1014, 985, 1006, 1057, 994, 1014, 1019, 1050, 976, 1050, 1056, 1016, 1019, 1053, 1014, 1019, 985, 1032, 1051, 1044, 1000, 1020, 1000, 1006, 1014, 972, 1026, 1095, 1010, 991, 1033, 1028, 1034, 1005, 1041, 1031, 1034, 1062, 977, 986, 1052, 1022, 1025, 978, 1020, 1014, 1028, 984, 1053, 1066, 1062, 1020, 1076, 1011, 1029, 988, 1036, 1009, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1029, 1029, 1013, 1021, 1044, 1053, 1064, 1071, 1036, 977, 1024, 1043, 1024, 1031, 1031, 978, 1018, 1053, 1030, 1025, 1005, 1052, 1008, 976, 1030, 1036, 1041, 982, 1020, 995, 990, 994, 1038, 1014, 1043, 1040, 1034, 1031, 1021, 1052, 1026, 983, 959, 1063, 991, 1079, 1059, 992, 1049, 1044, 987, 1033, 1005, 1024, 996, 1074, 1016, 1044, 1059, 994, 1051, 954, 1051, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1043, 1064, 1020, 1024, 1059, 952, 1042, 1035, 997, 1009, 1076, 1078, 1013, 1043, 953, 978, 1023, 1020, 1023, 1033, 1032, 1045, 1013, 1037, 1042, 1055, 983, 1028, 1035, 1007, 1013, 1028, 1037, 986, 983, 1044, 996, 1036, 1011, 1033, 1078, 1021, 1037, 994, 1059, 1034, 996, 1029, 1060, 996, 998, 988, 991, 975, 1065, 1006, 1013, 1090, 1035, 1013, 1083, 1011, 1048, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[970, 1008, 1036, 1059, 1041, 995, 982, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 1045, 1006, 1053, 992, 1040, 1067, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1075, 1016, 1034, 1054, 1028, 1024, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 969, 1013, 993, 1072, 1032, 989, 947, 51, 3, 0, 0, 0, 0, 0, 0, 1031, 1014, 1018, 1002, 1096, 1049, 1005, 1007, 1, 1, 0, 0, 0, 0, 0, 0, 1024, 1027, 1042, 984, 1026, 995, 1007, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 995, 1049, 1015, 1019, 1090, 984, 991, 5, 4, 0, 0, 0, 0, 0, 0, 969, 1106, 999, 1007, 1048, 1033, 1034, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8312, 8269, 8082, 8224, 8101, 8200, 8226] avg=8192.0000 max=8312 max_rank=1 min=8082 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32751 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1001, 1016, 1071, 1065, 1037, 1014, 985, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 994, 965, 1019, 1050, 976, 1050, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1014, 1053, 1014, 1019, 985, 1032, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1000, 1020, 1000, 1006, 1014, 972, 1026, 49, 3, 0, 0, 0, 0, 0, 0, 1095, 1010, 991, 1033, 1028, 1034, 1005, 1041, 1, 1, 0, 0, 0, 0, 0, 0, 1031, 1034, 1062, 977, 986, 1051, 1022, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1020, 1014, 1028, 984, 1053, 1066, 1062, 5, 4, 0, 0, 0, 0, 0, 0, 1020, 1075, 1011, 1029, 988, 1036, 1009, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8167, 8184, 8134, 8239, 8188, 8214, 8215] avg=8192.0000 max=8239 max_rank=4 min=8134 min_rank=3 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32856 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1054, 1013, 1072, 974, 1045, 1026, 1050, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1102, 996, 1021, 994, 955, 1068, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1070, 987, 994, 1034, 1015, 999, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 971, 1019, 1005, 1017, 1012, 1004, 1041, 51, 3, 0, 0, 0, 0, 0, 0, 1023, 1061, 1016, 1033, 1028, 1007, 1053, 1087, 1, 1, 0, 0, 0, 0, 0, 0, 1012, 1000, 1027, 1010, 977, 1044, 980, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1003, 1044, 1004, 974, 1016, 1041, 1031, 5, 4, 0, 0, 0, 0, 0, 0, 1031, 1026, 1044, 1037, 1027, 1054, 1033, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8249, 8155, 8136, 8310, 8050, 8123, 8267] avg=8192.0000 max=8310 max_rank=4 min=8050 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32750 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[989, 1012, 1053, 1032, 1017, 987, 1039, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1094, 996, 1050, 1052, 1050, 1041, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 990, 979, 990, 1058, 1041, 1005, 1009, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1013, 1016, 1029, 1042, 1026, 1064, 1053, 51, 3, 0, 0, 0, 0, 0, 0, 1002, 1011, 1008, 1049, 984, 1025, 995, 1022, 1, 1, 0, 0, 0, 0, 0, 0, 996, 1056, 1007, 1033, 1012, 1055, 1032, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1017, 1040, 1032, 1069, 978, 1015, 1046, 5, 4, 0, 0, 0, 0, 0, 0, 1037, 1008, 975, 1021, 1036, 1038, 980, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8313, 8081, 8306, 8098, 8217, 8217, 8129] avg=8192.0000 max=8313 max_rank=1 min=8081 min_rank=2 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32661 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1029, 1050, 1021, 1034, 995, 1016, 1047, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1020, 982, 975, 1055, 1048, 1036, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1003, 1056, 1019, 1031, 1075, 1047, 991, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1053, 1001, 1035, 1049, 1031, 1008, 990, 50, 3, 0, 0, 0, 0, 0, 0, 1022, 981, 1016, 1029, 1002, 1000, 1057, 1025, 1, 1, 0, 0, 0, 0, 0, 0, 1061, 1063, 982, 1039, 1037, 1017, 1047, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1027, 1006, 1016, 1060, 1013, 1005, 1045, 5, 4, 0, 0, 0, 0, 0, 0, 979, 1029, 1036, 1042, 969, 1024, 992, 969, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8187, 8281, 8214, 8134, 8301, 8177, 8040] avg=8192.0000 max=8301 max_rank=5 min=8040 min_rank=7 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32652 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[980, 1011, 1027, 1010, 997, 1004, 1028, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1033, 1004, 1037, 1029, 1014, 1002, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1042, 1026, 1033, 1027, 1038, 995, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1032, 1032, 1021, 990, 1043, 1039, 1034, 51, 3, 0, 0, 0, 0, 0, 0, 1028, 938, 1019, 1010, 1029, 1047, 983, 1070, 1, 1, 0, 0, 0, 0, 0, 0, 1043, 1044, 1045, 1010, 1021, 1012, 1028, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1025, 1018, 1048, 1033, 1056, 1050, 1014, 5, 4, 0, 0, 0, 0, 0, 0, 1044, 1035, 966, 999, 1043, 1025, 1055, 1039, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8053, 8122, 8189, 8272, 8126, 8282, 8286, 8206] avg=8192.0000 max=8286 max_rank=6 min=8053 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32900 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1029, 1029, 1006, 1021, 1044, 1053, 1064, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 977, 974, 1043, 1024, 1031, 1031, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1048, 1030, 1025, 1005, 1052, 1008, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1036, 1041, 982, 1020, 995, 990, 994, 50, 3, 0, 0, 0, 0, 0, 0, 1038, 1014, 1043, 1040, 1034, 1031, 1021, 1052, 1, 1, 0, 0, 0, 0, 0, 0, 1026, 983, 959, 1063, 991, 1078, 1059, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1044, 987, 1033, 1005, 1024, 996, 1074, 5, 4, 0, 0, 0, 0, 0, 0, 1016, 1043, 1059, 994, 1051, 954, 1051, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 8094, 8162, 8141, 8275, 8151, 8221, 8175] avg=8192.0000 max=8317 max_rank=0 min=8094 min_rank=1 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32822 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1043, 1064, 1007, 1024, 1059, 952, 1042, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1009, 1021, 1078, 1013, 1043, 953, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1009, 1023, 1033, 1032, 1045, 1013, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1055, 983, 1028, 1035, 1007, 1013, 1028, 55, 5, 0, 0, 0, 0, 0, 0, 1037, 986, 983, 1044, 996, 1036, 1011, 1033, 2, 2, 0, 0, 0, 0, 0, 0, 1078, 1021, 1037, 994, 1059, 1032, 996, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 996, 998, 988, 991, 975, 1065, 1006, 11, 8, 0, 0, 0, 0, 0, 0, 1013, 1088, 1035, 1013, 1083, 1011, 1048, 987, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8092, 8215, 8251, 8130, 8246, 8098, 8278] avg=8192.0000 max=8278 max_rank=7 min=8092 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32752 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8448, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8064, 8192, 8320, 8320, 8320, 8192, 8448, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8192, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[982, 977, 1010, 1016, 1032, 1022, 1100, 1034, 1019, 1047, 1032, 1025, 1044, 1004, 1015, 1018, 1068, 1044, 1027, 1037, 1046, 1014, 1086, 989, 1018, 1057, 1030, 1041, 1018, 1010, 1050, 1076, 1027, 1023, 1043, 956, 1019, 1027, 1036, 957, 1028, 1032, 1034, 1006, 1028, 1034, 1041, 956, 1026, 1059, 1002, 1025, 993, 1029, 1019, 1028, 1008, 985, 1069, 1020, 1028, 977, 1005, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1067, 1018, 1025, 999, 1011, 1038, 1000, 996, 974, 1054, 988, 1042, 1030, 998, 1041, 1033, 1061, 1014, 1014, 1031, 1002, 1024, 1080, 1013, 1024, 1031, 1029, 1045, 1012, 997, 1015, 1016, 1043, 1023, 1035, 1017, 1036, 1058, 999, 1058, 996, 1009, 1039, 1012, 1013, 1021, 1031, 1008, 1024, 1054, 1055, 972, 1073, 1000, 1049, 1031, 1066, 1032, 996, 1035, 1009, 992, 989, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1022, 979, 987, 998, 1033, 1005, 1020, 1060, 1071, 1042, 983, 1050, 1077, 1032, 1058, 995, 1037, 987, 962, 1028, 1035, 1025, 996, 1026, 1007, 1012, 979, 1046, 1005, 1044, 1027, 1003, 1052, 1001, 1031, 1014, 1012, 1071, 987, 1005, 1040, 1041, 1013, 1025, 983, 1039, 1040, 1048, 1047, 1055, 1039, 1033, 1044, 1014, 1097, 1015, 1077, 1024, 998, 1019, 972, 1039, 1026, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1028, 1009, 983, 1043, 987, 1034, 1002, 1027, 1086, 988, 1016, 1036, 1032, 1019, 1023, 997, 989, 1057, 1029, 1029, 993, 1042, 1045, 1036, 1093, 1039, 974, 1022, 990, 1016, 1026, 1025, 1063, 1033, 1015, 1020, 975, 1007, 993, 1026, 1058, 1012, 1043, 1012, 1011, 1005, 1009, 1060, 1069, 1021, 1025, 1000, 1023, 1033, 966, 1021, 1083, 1013, 1047, 1034, 1011, 1085, 1034, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1049, 1023, 1030, 1039, 1031, 1005, 1012, 1029, 1047, 1031, 998, 1001, 1021, 1007, 1018, 1032, 999, 1015, 1049, 1033, 969, 1041, 1058, 1002, 1037, 1025, 1031, 998, 998, 1072, 1035, 992, 997, 1030, 1010, 1025, 1008, 1021, 1061, 1022, 1047, 1017, 1030, 1030, 1044, 993, 1031, 1016, 1034, 1027, 975, 1081, 1073, 1006, 1009, 1053, 1025, 1004, 1023, 1005, 1037, 1054, 1039, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1052, 1024, 1042, 1042, 1030, 994, 1052, 1056, 1067, 1014, 1021, 1036, 1005, 1002, 1015, 1022, 970, 984, 1047, 1072, 1046, 1024, 1040, 1040, 1018, 1028, 1047, 1007, 1035, 1031, 1037, 1034, 1033, 1017, 1007, 1047, 1002, 1025, 1017, 1033, 985, 1015, 977, 1035, 1031, 992, 1032, 1027, 1049, 993, 1009, 1024, 1064, 1048, 1002, 1022, 1044, 997, 964, 1046, 1033, 1009, 1026, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1019, 1058, 1001, 1026, 978, 1060, 1002, 1017, 1008, 1051, 1054, 1039, 985, 1040, 1095, 1024, 1035, 1001, 1025, 1048, 985, 1030, 1033, 1030, 1047, 1015, 1015, 1031, 1007, 1044, 1017, 999, 1094, 986, 1041, 1050, 996, 1008, 1000, 1021, 1042, 1019, 1002, 984, 1031, 1085, 1004, 1013, 1024, 1014, 973, 1034, 1061, 993, 1052, 1008, 1051, 1041, 1028, 1050, 982, 1014, 1003, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1022, 1077, 1025, 1027, 1064, 995, 1026, 1059, 1032, 1005, 1030, 994, 999, 999, 1039, 1011, 1038, 1010, 1034, 978, 1012, 1006, 1059, 1004, 1044, 983, 991, 989, 1005, 1037, 1038, 1056, 1028, 1019, 999, 1025, 1032, 1000, 1011, 1027, 1030, 1041, 1003, 1018, 1044, 1045, 1026, 995, 1062, 1084, 968, 1031, 1019, 1063, 1025, 1004, 1035, 1043, 1067, 1033, 1045, 1044, 1009, 973] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[982, 977, 1010, 1016, 1032, 1022, 1100, 1034, 3, 0, 0, 0, 0, 0, 0, 0, 1019, 1047, 1032, 1025, 1044, 1004, 1007, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1044, 1027, 1037, 1046, 1014, 1081, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1057, 1030, 1041, 1018, 1010, 1050, 1076, 2, 0, 0, 0, 0, 0, 0, 0, 1027, 1023, 1043, 956, 1019, 1027, 1036, 957, 21, 0, 0, 0, 0, 0, 0, 0, 1028, 1032, 1034, 1006, 1028, 1034, 1041, 956, 12, 2, 0, 0, 0, 0, 0, 0, 1026, 1059, 1002, 1025, 960, 1029, 1019, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 985, 1069, 1020, 1028, 977, 1005, 1028, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8196, 8306, 8302, 8109, 8173, 8148, 8126] avg=8192.0000 max=8306 max_rank=2 min=8109 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1022, 979, 987, 998, 1033, 1005, 1020, 1060, 3, 0, 0, 0, 0, 0, 0, 0, 1071, 1042, 983, 1050, 1077, 1032, 1049, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 987, 962, 1028, 1035, 1025, 992, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1012, 979, 1046, 1005, 1044, 1027, 1003, 1, 0, 0, 0, 0, 0, 0, 0, 1052, 1001, 1031, 1014, 1012, 1071, 987, 1005, 22, 0, 0, 0, 0, 0, 0, 0, 1040, 1041, 1013, 1025, 983, 1039, 1040, 1048, 13, 2, 0, 0, 0, 0, 0, 0, 1047, 1055, 1039, 1033, 1009, 1014, 1097, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 1024, 998, 1019, 972, 1039, 1026, 1004, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8299, 8092, 8124, 8195, 8244, 8309, 8166] avg=8192.0000 max=8309 max_rank=6 min=8092 min_rank=2 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1052, 1024, 1042, 1042, 1030, 994, 1052, 1056, 3, 0, 0, 0, 0, 0, 0, 0, 1067, 1014, 1021, 1036, 1005, 1002, 1007, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 970, 984, 1047, 1072, 1046, 1024, 1036, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1028, 1047, 1007, 1035, 1031, 1037, 1034, 1, 0, 0, 0, 0, 0, 0, 0, 1033, 1017, 1007, 1047, 1002, 1025, 1017, 1033, 23, 0, 0, 0, 0, 0, 0, 0, 985, 1015, 977, 1035, 1031, 992, 1032, 1027, 13, 2, 0, 0, 0, 0, 0, 0, 1049, 993, 1009, 1024, 1028, 1048, 1002, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 997, 964, 1046, 1033, 1009, 1026, 997, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8174, 8219, 8238, 8204, 8109, 8175, 8122] avg=8192.0000 max=8295 max_rank=0 min=8109 min_rank=5 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32610 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1067, 1018, 1025, 999, 1011, 1038, 1000, 996, 3, 0, 0, 0, 0, 0, 0, 0, 974, 1054, 988, 1042, 1030, 998, 1032, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1014, 1014, 1031, 1002, 1024, 1075, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1031, 1029, 1045, 1012, 997, 1015, 1016, 2, 0, 0, 0, 0, 0, 0, 0, 1043, 1023, 1035, 1017, 1036, 1058, 999, 1058, 23, 0, 0, 0, 0, 0, 0, 0, 996, 1009, 1039, 1012, 1013, 1021, 1031, 1008, 13, 2, 0, 0, 0, 0, 0, 0, 1024, 1054, 1055, 972, 1037, 1000, 1049, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1032, 996, 1035, 1009, 992, 989, 1039, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8151, 8234, 8171, 8292, 8144, 8222, 8165] avg=8192.0000 max=8292 max_rank=4 min=8144 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1028, 1009, 983, 1043, 987, 1034, 1002, 1027, 3, 0, 0, 0, 0, 0, 0, 0, 1086, 988, 1016, 1036, 1032, 1019, 1014, 997, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1057, 1029, 1029, 993, 1042, 1041, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 1039, 974, 1022, 990, 1016, 1026, 1025, 1, 0, 0, 0, 0, 0, 0, 0, 1063, 1033, 1015, 1020, 975, 1007, 993, 1026, 22, 0, 0, 0, 0, 0, 0, 0, 1058, 1012, 1043, 1012, 1011, 1005, 1009, 1060, 13, 2, 0, 0, 0, 0, 0, 0, 1069, 1021, 1025, 1000, 988, 1033, 966, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 1013, 1047, 1034, 1011, 1085, 1034, 1014, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8188, 8216, 8186, 8154, 8225, 8123, 8328] avg=8192.0000 max=8328 max_rank=7 min=8116 min_rank=0 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1049, 1023, 1030, 1039, 1031, 1005, 1012, 1029, 3, 0, 0, 0, 0, 0, 0, 0, 1047, 1031, 998, 1001, 1021, 1007, 1010, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1015, 1049, 1033, 969, 1041, 1053, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1025, 1031, 998, 998, 1072, 1035, 992, 2, 0, 0, 0, 0, 0, 0, 0, 997, 1030, 1010, 1025, 1008, 1021, 1061, 1022, 23, 0, 0, 0, 0, 0, 0, 0, 1047, 1017, 1030, 1030, 1044, 993, 1031, 1016, 13, 2, 0, 0, 0, 0, 0, 0, 1034, 1027, 975, 1081, 1037, 1006, 1009, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1004, 1023, 1005, 1037, 1054, 1039, 982, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8147, 8161, 8190, 8197, 8223, 8222, 8175] avg=8192.0000 max=8223 max_rank=5 min=8147 min_rank=1 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1019, 1058, 1001, 1026, 978, 1060, 1002, 1017, 3, 0, 0, 0, 0, 0, 0, 0, 1008, 1051, 1054, 1039, 985, 1040, 1086, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1001, 1025, 1048, 985, 1030, 1029, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1015, 1015, 1031, 1007, 1044, 1017, 999, 1, 0, 0, 0, 0, 0, 0, 0, 1094, 986, 1041, 1050, 996, 1008, 1000, 1021, 23, 0, 0, 0, 0, 0, 0, 0, 1042, 1019, 1002, 984, 1031, 1085, 1004, 1013, 13, 2, 0, 0, 0, 0, 0, 0, 1024, 1014, 973, 1034, 1025, 993, 1052, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1041, 1028, 1050, 982, 1014, 1003, 1013, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8287, 8183, 8176, 8219, 8195, 8123, 8189] avg=8192.0000 max=8287 max_rank=1 min=8123 min_rank=6 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1022, 1077, 1025, 1027, 1064, 995, 1026, 1059, 5, 0, 0, 0, 0, 0, 0, 0, 1032, 1005, 1030, 994, 999, 999, 1024, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1010, 1034, 978, 1012, 1006, 1043, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 983, 991, 989, 1005, 1037, 1038, 1056, 6, 0, 0, 0, 0, 0, 0, 0, 1028, 1019, 999, 1025, 1032, 1000, 1011, 1027, 25, 0, 0, 0, 0, 0, 0, 0, 1030, 1041, 1003, 1018, 1044, 1045, 1026, 995, 17, 4, 0, 0, 0, 0, 0, 0, 1062, 1084, 968, 1031, 977, 1063, 1025, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1043, 1067, 1033, 1045, 1044, 1009, 973, 11, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8094, 8125, 8149, 8166, 8223, 8214, 8265] avg=8192.0000 max=8300 max_rank=0 min=8094 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32868 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8448, 8320, 8064, 8320, 8064, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8192, 8320, 8192, 8320, 8192, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8192, 8192, 8192, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1008, 1037, 1053, 1039, 1039, 1034, 1023, 1018, 1030, 1007, 986, 1018, 1027, 947, 1056, 1092, 988, 1030, 989, 1020, 1030, 1023, 997, 1012, 981, 1084, 1008, 1028, 992, 980, 1010, 970, 1044, 1019, 970, 1061, 1053, 1017, 1035, 961, 1021, 1035, 1007, 1071, 1065, 1021, 981, 1041, 1051, 1019, 1036, 1029, 1065, 1022, 1044, 1050, 1040, 1052, 998, 1001, 1033, 1047, 1014, 1077] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[997, 1012, 998, 1026, 996, 1067, 976, 1045, 1034, 992, 995, 987, 1012, 991, 1042, 1010, 1030, 1020, 1008, 1096, 1016, 980, 1024, 1031, 1038, 1043, 1025, 1019, 1025, 996, 998, 1051, 1059, 1004, 1051, 1030, 1036, 990, 1037, 977, 1005, 988, 987, 1024, 1033, 1000, 1027, 1059, 1018, 1022, 1047, 1081, 1092, 1037, 1049, 1021, 1024, 1015, 1079, 1028, 1025, 1029, 1040, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1027, 1010, 1016, 1008, 1016, 1018, 1032, 1039, 987, 991, 1053, 1014, 1067, 1057, 1076, 1073, 1059, 1078, 1065, 972, 1014, 1021, 1003, 1045, 1068, 1007, 1039, 1018, 979, 976, 1001, 1035, 1065, 1005, 1029, 1040, 1017, 1053, 1053, 1035, 1013, 1017, 1025, 1009, 1011, 1032, 962, 1057, 1044, 962, 1033, 1007, 1048, 991, 1002, 1034, 1015, 1026, 1010, 989, 1006, 994, 1054, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1006, 1033, 1030, 1028, 1064, 1030, 1024, 1014, 1062, 1049, 1044, 1015, 999, 1080, 979, 980, 1070, 999, 1044, 1004, 1043, 1017, 1047, 1025, 1002, 1062, 1060, 994, 1036, 996, 984, 1037, 1051, 1033, 1044, 1027, 1056, 1033, 1026, 988, 1004, 975, 1083, 1076, 1002, 957, 998, 1016, 1022, 998, 1062, 1014, 1003, 1022, 1017, 1031, 978, 997, 1008, 1027, 1058, 1019, 1011, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1066, 1010, 1045, 1054, 1004, 998, 1017, 1027, 1034, 1012, 978, 1019, 1086, 1001, 1008, 963, 1014, 1034, 1004, 983, 1021, 1009, 1000, 1010, 1009, 1017, 1044, 1063, 978, 1051, 1035, 1050, 999, 1081, 1004, 1038, 1003, 1002, 1016, 1032, 1054, 1075, 1037, 1038, 1042, 1010, 1005, 1023, 1044, 1008, 1033, 998, 1038, 1046, 1026, 1025, 974, 1035, 993, 1019, 1054, 1010, 1019, 1111] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1013, 1004, 1051, 1036, 1026, 1024, 1021, 993, 1016, 1037, 1006, 1016, 1036, 992, 1039, 1031, 1044, 1014, 1049, 1024, 1053, 1051, 1037, 1047, 1033, 1022, 1040, 1029, 992, 988, 1023, 1014, 986, 1054, 1040, 990, 988, 984, 1037, 1016, 1056, 1036, 1041, 1028, 1026, 1035, 1022, 980, 1015, 1049, 1059, 1028, 1032, 1016, 1004, 1069, 1042, 1043, 1017, 1003, 1030, 987, 999, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[972, 1027, 1040, 1036, 1011, 1035, 1031, 1061, 1048, 1047, 1005, 1005, 1018, 1029, 1013, 1041, 1005, 1057, 1057, 1030, 1051, 982, 1073, 1082, 1023, 1070, 1055, 978, 1018, 1023, 959, 994, 1046, 1047, 1071, 1009, 985, 1048, 1034, 1093, 1044, 1023, 945, 983, 1044, 1000, 1004, 1044, 1017, 1061, 1015, 1017, 1011, 1044, 994, 1087, 1014, 981, 934, 1027, 971, 1011, 1031, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1012, 1026, 1021, 1030, 1030, 1050, 1037, 986, 1011, 1000, 1004, 1026, 1061, 1004, 1031, 1032, 1071, 1019, 965, 1038, 999, 1000, 1041, 1012, 1015, 1056, 1052, 1046, 995, 1020, 988, 1044, 1001, 1006, 999, 1006, 1005, 1035, 1047, 1014, 1030, 958, 1010, 1038, 1021, 1029, 1038, 1032, 1064, 1014, 1024, 1016, 1009, 1030, 1097, 990, 1023, 1029, 1035, 1003, 1041, 1019, 1043, 1108] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1008, 1037, 1053, 1037, 1039, 1034, 1023, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1007, 986, 1018, 1027, 947, 1056, 1092, 12, 3, 0, 0, 0, 0, 0, 0, 972, 1030, 989, 1020, 1030, 1023, 997, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1084, 1008, 1028, 992, 980, 1010, 970, 33, 0, 0, 0, 0, 0, 0, 0, 1044, 1019, 970, 1061, 1053, 1017, 1027, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1035, 1007, 1071, 1065, 1021, 981, 1041, 18, 4, 0, 0, 0, 0, 0, 0, 1051, 1019, 985, 1029, 1065, 1022, 1044, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1052, 998, 1001, 1033, 1047, 1014, 1077, 5, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8178, 8073, 8086, 8152, 8264, 8265, 8269] avg=8192.0000 max=8269 max_rank=7 min=8073 min_rank=2 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32950 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[997, 1012, 998, 1024, 996, 1067, 976, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 992, 995, 987, 1012, 991, 1042, 1010, 12, 3, 0, 0, 0, 0, 0, 0, 1014, 1020, 1008, 1096, 1016, 980, 1024, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1043, 1025, 1019, 1025, 996, 998, 1051, 34, 0, 0, 0, 0, 0, 0, 0, 1059, 1004, 1051, 1030, 1036, 990, 1028, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 988, 987, 1024, 1033, 1000, 1027, 1059, 18, 4, 0, 0, 0, 0, 0, 0, 1018, 1022, 995, 1081, 1092, 1037, 1049, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1015, 1079, 1028, 1025, 1029, 1040, 1042, 6, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8078, 8189, 8229, 8175, 8145, 8315, 8290] avg=8192.0000 max=8315 max_rank=6 min=8078 min_rank=1 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32925 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1006, 1033, 1030, 1026, 1064, 1030, 1024, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1049, 1044, 1015, 999, 1080, 979, 980, 13, 3, 0, 0, 0, 0, 0, 0, 1053, 999, 1044, 1004, 1043, 1017, 1047, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1062, 1060, 994, 1036, 996, 984, 1037, 34, 0, 0, 0, 0, 0, 0, 0, 1051, 1033, 1044, 1027, 1056, 1033, 1018, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 975, 1083, 1076, 1002, 957, 998, 1016, 18, 4, 0, 0, 0, 0, 0, 0, 1022, 998, 1010, 1014, 1003, 1022, 1017, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 978, 997, 1008, 1027, 1058, 1019, 1011, 1043, 5, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8224, 8232, 8205, 8250, 8133, 8117, 8148] avg=8192.0000 max=8250 max_rank=4 min=8117 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1027, 1010, 1016, 1006, 1016, 1018, 1032, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 987, 991, 1053, 1014, 1067, 1057, 1076, 1073, 13, 3, 0, 0, 0, 0, 0, 0, 1042, 1078, 1065, 972, 1014, 1021, 1003, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1007, 1039, 1018, 979, 976, 1001, 1035, 33, 0, 0, 0, 0, 0, 0, 0, 1065, 1005, 1029, 1040, 1017, 1053, 1044, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1017, 1025, 1009, 1011, 1032, 962, 1057, 18, 4, 0, 0, 0, 0, 0, 0, 1044, 962, 982, 1007, 1048, 991, 1002, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1026, 1010, 989, 1006, 994, 1054, 1034, 6, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8334, 8240, 8156, 8288, 8148, 8070, 8136] avg=8192.0000 max=8334 max_rank=1 min=8070 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32642 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1066, 1010, 1045, 1052, 1004, 998, 1017, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1012, 978, 1019, 1086, 1001, 1008, 963, 12, 3, 0, 0, 0, 0, 0, 0, 998, 1034, 1004, 983, 1021, 1009, 1000, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1017, 1044, 1063, 978, 1051, 1035, 1050, 33, 0, 0, 0, 0, 0, 0, 0, 999, 1081, 1004, 1038, 1003, 1002, 1008, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1075, 1037, 1038, 1042, 1010, 1005, 1023, 18, 4, 0, 0, 0, 0, 0, 0, 1044, 1008, 982, 998, 1038, 1046, 1026, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1035, 993, 1019, 1054, 1010, 1019, 1111, 5, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8116, 8059, 8280, 8167, 8306, 8167, 8222] avg=8192.0000 max=8306 max_rank=5 min=8059 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32862 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1013, 1004, 1051, 1034, 1026, 1024, 1021, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1037, 1006, 1016, 1036, 992, 1039, 1031, 13, 3, 0, 0, 0, 0, 0, 0, 1027, 1014, 1049, 1024, 1053, 1051, 1037, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1022, 1040, 1029, 992, 988, 1023, 1014, 34, 0, 0, 0, 0, 0, 0, 0, 986, 1054, 1040, 990, 988, 984, 1028, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1036, 1041, 1028, 1026, 1035, 1022, 980, 18, 4, 0, 0, 0, 0, 0, 0, 1015, 1049, 1007, 1028, 1032, 1016, 1004, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1043, 1017, 1003, 1030, 987, 999, 1023, 6, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8189, 8302, 8175, 8086, 8246, 8220, 8152] avg=8192.0000 max=8302 max_rank=2 min=8086 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[972, 1027, 1040, 1034, 1011, 1035, 1031, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1047, 1005, 1005, 1018, 1029, 1013, 1041, 12, 3, 0, 0, 0, 0, 0, 0, 989, 1057, 1057, 1030, 1051, 982, 1073, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1070, 1055, 978, 1018, 1023, 959, 994, 32, 0, 0, 0, 0, 0, 0, 0, 1046, 1047, 1071, 1009, 985, 1048, 1026, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1023, 945, 983, 1044, 1000, 1004, 1044, 17, 4, 0, 0, 0, 0, 0, 0, 1017, 1061, 966, 1017, 1011, 1044, 994, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 981, 934, 1027, 971, 1011, 1031, 1025, 5, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8221, 8321, 8152, 8325, 8108, 8197, 8001] avg=8192.0000 max=8325 max_rank=4 min=8001 min_rank=7 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1012, 1026, 1021, 1023, 1030, 1050, 1037, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1000, 1004, 1026, 1061, 1004, 1031, 1032, 17, 10, 0, 0, 0, 0, 0, 0, 1046, 1019, 965, 1038, 999, 1000, 1041, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1056, 1052, 1046, 995, 1020, 988, 1044, 37, 0, 0, 0, 0, 0, 0, 0, 1001, 1006, 999, 1006, 1005, 1035, 1027, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 958, 1010, 1038, 1021, 1029, 1038, 1032, 22, 8, 0, 0, 0, 0, 0, 0, 1064, 1014, 965, 1016, 1009, 1030, 1097, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1029, 1035, 1003, 1041, 1019, 1043, 1108, 10, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8196, 8120, 8253, 8093, 8186, 8185, 8318] avg=8192.0000 max=8318 max_rank=7 min=8093 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8064, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 7936, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8448, 8192, 8064, 8064, 8064, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8576, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1019, 1037, 1052, 1033, 1035, 1014, 1047, 1050, 1061, 1029, 1033, 1024, 992, 1032, 998, 1040, 1014, 1038, 1045, 985, 1054, 972, 1024, 1027, 1063, 1013, 1030, 992, 1035, 1033, 1024, 1055, 1005, 1023, 1006, 1079, 944, 1007, 1049, 1069, 1017, 1021, 1042, 1023, 977, 1026, 971, 992, 1024, 983, 1008, 1000, 1025, 1048, 1046, 1043, 1030, 1003, 1048, 1057, 1024, 1000, 1016, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1031, 1015, 1066, 1037, 1028, 1062, 1015, 1039, 995, 1009, 1029, 1023, 974, 1065, 1006, 1039, 978, 1038, 1031, 1017, 1040, 985, 1012, 1059, 1002, 1047, 1020, 1040, 1073, 1006, 1009, 1010, 1007, 1031, 1024, 1016, 994, 1024, 1011, 1005, 1021, 1079, 1049, 989, 956, 1027, 1057, 1034, 1063, 1041, 1019, 989, 1050, 1032, 1018, 1027, 1008, 1004, 1044, 1038, 1046, 1008, 1043, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1021, 1032, 965, 974, 1009, 1053, 1053, 1006, 1018, 1036, 1038, 993, 1047, 1030, 1018, 1050, 1005, 1018, 1075, 1058, 977, 985, 1048, 996, 1004, 1018, 1051, 1055, 1021, 1026, 1053, 1008, 980, 1016, 1017, 1062, 1022, 1049, 1011, 1030, 1001, 1043, 1089, 1018, 1070, 992, 1014, 1032, 1058, 972, 1001, 1043, 1050, 999, 1041, 1032, 1019, 1060, 1000, 1039, 1012, 1076, 973, 974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1051, 1059, 1042, 1035, 1046, 1018, 1025, 1019, 1056, 1004, 1058, 1045, 957, 995, 1021, 952, 1039, 1010, 969, 1026, 1009, 1005, 1005, 1045, 1036, 989, 1045, 1043, 1054, 1026, 1067, 1070, 1023, 1017, 1016, 1018, 1039, 1042, 1051, 990, 1035, 1046, 1005, 1102, 1018, 1009, 1046, 1041, 991, 1020, 1015, 1016, 994, 962, 996, 1104, 1026, 1001, 1027, 1021, 1009, 996, 1026, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1019, 1044, 1000, 1004, 991, 976, 1035, 1008, 1036, 1027, 1010, 1055, 1049, 979, 1002, 1012, 1009, 1037, 1044, 1061, 964, 1089, 986, 1052, 1013, 1048, 1047, 1026, 1070, 1022, 999, 1022, 1018, 1060, 1037, 1019, 1072, 1022, 995, 1013, 1073, 1074, 1043, 1019, 1016, 1015, 1041, 1002, 1011, 1041, 974, 1028, 1001, 1010, 1013, 1020, 1053, 1016, 1003, 1040, 1016, 1001, 1062, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1045, 1010, 1061, 1021, 1038, 988, 1036, 1073, 1028, 1056, 1018, 1036, 1014, 992, 1008, 1082, 1041, 1001, 1047, 1057, 1005, 1012, 1026, 1028, 1031, 1017, 1005, 1040, 1041, 985, 987, 1074, 980, 1047, 984, 1027, 1080, 990, 1034, 1003, 968, 1057, 1038, 1016, 1073, 992, 1030, 1056, 1033, 960, 1013, 981, 1040, 1029, 1034, 1007, 1054, 1003, 1019, 1040, 1003, 1047, 1060, 935] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1013, 1048, 1002, 1062, 979, 1008, 1026, 999, 1017, 1021, 1009, 1081, 1040, 1021, 972, 1001, 1099, 982, 1021, 1042, 1059, 1020, 1050, 999, 1012, 1077, 998, 1005, 1033, 980, 995, 1025, 1021, 1052, 1020, 1055, 993, 1048, 1035, 1019, 1076, 1030, 1036, 1010, 1035, 1005, 1058, 1016, 1038, 1036, 990, 974, 1051, 1051, 1049, 1004, 979, 1020, 1005, 1050, 1054, 1017, 997, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1011, 1068, 1029, 1042, 1005, 1042, 988, 1064, 1012, 1011, 1003, 1010, 935, 1018, 1004, 1022, 1026, 1048, 1027, 1014, 1033, 1028, 1037, 1063, 988, 1032, 1007, 1072, 1021, 1007, 1009, 1007, 1011, 1058, 1048, 1053, 1050, 1024, 1016, 990, 1022, 991, 1045, 1013, 1068, 1037, 1007, 1057, 1018, 973, 1052, 1077, 1014, 1028, 1023, 991, 1066, 1024, 1061, 1012, 1021, 1009, 1045, 949] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1031, 993, 1066, 1037, 1028, 1062, 1015, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1009, 1029, 1023, 974, 1065, 1006, 1039, 36, 0, 0, 0, 0, 0, 0, 0, 978, 1038, 1031, 1017, 1040, 985, 1012, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1047, 1020, 1040, 1052, 1006, 1009, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1031, 1024, 1015, 994, 1024, 1011, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1079, 1009, 989, 956, 1027, 1057, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1041, 1019, 989, 1050, 1032, 1018, 1027, 22, 4, 4, 0, 0, 0, 0, 0, 1008, 1004, 1044, 1038, 1046, 1008, 1043, 982, 17, 7, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8176, 8153, 8186, 8111, 8172, 8269, 8198] avg=8192.0000 max=8271 max_rank=0 min=8111 min_rank=4 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1045, 988, 1061, 1021, 1038, 988, 1036, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1056, 1018, 1036, 1014, 992, 1008, 1082, 35, 0, 0, 0, 0, 0, 0, 0, 1041, 1001, 1047, 1057, 1005, 1012, 1026, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1017, 1005, 1040, 1020, 985, 987, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1047, 984, 1026, 1080, 990, 1034, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1057, 999, 1016, 1073, 992, 1030, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 960, 1013, 981, 1040, 1029, 1034, 1007, 22, 4, 4, 0, 0, 0, 0, 0, 1054, 1003, 1019, 1040, 1003, 1047, 1060, 935, 17, 7, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8269, 8210, 8159, 8144, 8191, 8127, 8186] avg=8192.0000 max=8269 max_rank=1 min=8127 min_rank=6 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1019, 1014, 1052, 1033, 1035, 1014, 1047, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1029, 1033, 1024, 992, 1032, 998, 1040, 35, 0, 0, 0, 0, 0, 0, 0, 1014, 1038, 1045, 985, 1054, 972, 1024, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1013, 1030, 992, 1014, 1033, 1024, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1023, 1006, 1078, 944, 1007, 1049, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1021, 1003, 1023, 977, 1026, 971, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 983, 1008, 1000, 1025, 1048, 1046, 1043, 23, 4, 4, 0, 0, 0, 0, 0, 1030, 1003, 1048, 1057, 1024, 1000, 1016, 1030, 17, 7, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8244, 8152, 8224, 8181, 8030, 8208, 8233] avg=8192.0000 max=8264 max_rank=0 min=8030 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1021, 1009, 965, 974, 1009, 1053, 1053, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1036, 1038, 993, 1047, 1030, 1018, 1050, 37, 0, 0, 0, 0, 0, 0, 0, 1005, 1018, 1075, 1058, 977, 985, 1048, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1018, 1051, 1055, 1000, 1026, 1053, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1016, 1017, 1061, 1022, 1049, 1011, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1043, 1048, 1018, 1070, 992, 1014, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 972, 1001, 1043, 1050, 999, 1041, 1032, 23, 4, 4, 0, 0, 0, 0, 0, 1019, 1060, 1000, 1039, 1012, 1076, 973, 974, 17, 7, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8267, 8155, 8215, 8186, 8218, 8227, 8178] avg=8192.0000 max=8267 max_rank=1 min=8090 min_rank=0 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1019, 1021, 1000, 1004, 991, 976, 1035, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1027, 1010, 1055, 1049, 979, 1002, 1012, 35, 0, 0, 0, 0, 0, 0, 0, 1009, 1037, 1044, 1061, 964, 1089, 986, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1048, 1047, 1026, 1049, 1022, 999, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1060, 1037, 1018, 1072, 1022, 995, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1074, 1004, 1019, 1016, 1015, 1041, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1041, 974, 1028, 1001, 1010, 1013, 1020, 23, 4, 4, 0, 0, 0, 0, 0, 1053, 1016, 1003, 1040, 1016, 1001, 1062, 992, 17, 7, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8054, 8205, 8235, 8226, 8235, 8244, 8129, 8208] avg=8192.0000 max=8244 max_rank=5 min=8054 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1051, 1036, 1042, 1035, 1046, 1018, 1025, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1004, 1058, 1045, 957, 995, 1021, 952, 34, 0, 0, 0, 0, 0, 0, 0, 1039, 1010, 969, 1026, 1009, 1005, 1005, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 989, 1045, 1043, 1033, 1026, 1067, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1017, 1016, 1017, 1039, 1042, 1051, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1046, 967, 1102, 1018, 1009, 1046, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1020, 1015, 1016, 994, 962, 996, 1104, 23, 4, 4, 0, 0, 0, 0, 0, 1026, 1001, 1027, 1021, 1009, 996, 1026, 1013, 17, 7, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8122, 8101, 8309, 8195, 8264, 8129, 8144] avg=8192.0000 max=8309 max_rank=3 min=8101 min_rank=2 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1013, 1025, 1002, 1062, 979, 1008, 1026, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1021, 1009, 1081, 1040, 1021, 972, 1001, 35, 0, 0, 0, 0, 0, 0, 0, 1099, 982, 1021, 1042, 1059, 1020, 1050, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1077, 998, 1005, 1012, 980, 995, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1052, 1020, 1054, 993, 1048, 1035, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1030, 997, 1010, 1035, 1005, 1058, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1036, 990, 974, 1051, 1051, 1049, 1004, 23, 4, 4, 0, 0, 0, 0, 0, 979, 1020, 1005, 1050, 1054, 1017, 997, 1016, 17, 7, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8197, 8265, 8104, 8242, 8227, 8224, 8163] avg=8192.0000 max=8265 max_rank=2 min=8104 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1011, 1040, 1029, 1042, 1005, 1042, 988, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1011, 1003, 1010, 935, 1018, 1004, 1022, 41, 0, 0, 0, 0, 0, 0, 0, 1026, 1048, 1027, 1014, 1033, 1028, 1037, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1032, 1007, 1072, 991, 1007, 1009, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1058, 1048, 1045, 1050, 1024, 1016, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 991, 995, 1013, 1068, 1037, 1007, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 973, 1052, 1077, 1014, 1028, 1023, 991, 28, 10, 9, 0, 0, 0, 0, 0, 1066, 1024, 1061, 1012, 1021, 1009, 1045, 949, 20, 11, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8056, 8265, 8113, 8242, 8190, 8223, 8226] avg=8192.0000 max=8265 max_rank=2 min=8056 min_rank=1 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8064, 8192, 8192, 8320, 8192, 8320, 8320, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8064, 8192, 8064, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8320, 7936, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8448, 8064, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[957, 981, 1023, 1010, 1023, 1022, 1036, 1001, 1033, 1019, 1028, 990, 1003, 996, 1046, 1000, 1040, 1067, 1059, 1022, 1032, 994, 1011, 1017, 1050, 1025, 1005, 1034, 966, 993, 1037, 1040, 985, 1023, 1037, 1030, 1038, 1032, 1029, 1068, 1021, 1026, 1052, 1031, 1010, 1046, 1015, 1061, 991, 1014, 1046, 1034, 1056, 1032, 1015, 1049, 1070, 1016, 995, 1012, 1074, 1010, 1028, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[989, 995, 1031, 991, 988, 1058, 974, 994, 1050, 1059, 1017, 1007, 1003, 1059, 1039, 1023, 995, 1025, 1068, 1006, 1074, 1002, 1034, 1045, 1061, 1027, 1051, 1013, 1030, 1018, 1082, 1069, 1013, 1010, 1001, 1029, 1047, 1048, 955, 1032, 1016, 972, 1002, 1049, 1010, 998, 1096, 988, 996, 1031, 1016, 1018, 1053, 1033, 1045, 1016, 993, 992, 1008, 1011, 1011, 1085, 1037, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1033, 1013, 1039, 1030, 994, 1008, 1060, 1021, 1044, 1001, 1055, 997, 1003, 1006, 1024, 1047, 1030, 1026, 1006, 1018, 1055, 988, 1019, 1034, 1016, 997, 1020, 1024, 1032, 1032, 1009, 1047, 1054, 1020, 1028, 1019, 1053, 1005, 1027, 1021, 998, 1016, 1063, 1034, 1078, 1039, 971, 1008, 974, 1024, 1042, 1038, 1014, 1038, 1036, 984, 985, 1081, 1062, 1062, 997, 1015, 1010, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1025, 1029, 1035, 1035, 1019, 987, 1016, 1006, 1025, 1055, 1063, 1029, 1091, 1013, 1056, 981, 1000, 998, 1055, 1060, 1022, 1056, 1015, 1061, 1042, 982, 1017, 1008, 1041, 980, 1030, 1004, 1065, 1043, 1007, 1018, 1053, 1005, 999, 956, 983, 982, 1054, 1020, 1033, 1000, 1048, 1057, 988, 1027, 1012, 1032, 1022, 1016, 1070, 960, 1013, 1022, 1054, 1040, 991, 1067, 1012, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1018, 1062, 1040, 1056, 1030, 1036, 1000, 1035, 995, 1077, 998, 1009, 983, 1073, 953, 1066, 1043, 1050, 1034, 1002, 1012, 1033, 962, 1002, 1081, 1006, 1060, 980, 1071, 954, 1024, 1075, 978, 1002, 1087, 1010, 988, 993, 978, 1013, 1018, 1078, 1053, 1081, 995, 1016, 1055, 1002, 987, 1026, 1026, 1076, 1063, 1002, 1004, 1045, 1005, 956, 1026, 986, 1073, 1037, 1002, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1057, 1015, 991, 1013, 997, 1043, 1019, 1015, 922, 985, 1053, 969, 1042, 996, 1020, 1013, 1029, 1051, 995, 1054, 1003, 1039, 1010, 1038, 1032, 1050, 1046, 1039, 1030, 995, 1058, 1023, 1013, 1020, 1027, 1057, 984, 1005, 1002, 1010, 1029, 1000, 1033, 1004, 1001, 1016, 1076, 1020, 1051, 1022, 1035, 1042, 1041, 1065, 995, 1078, 1034, 1022, 1020, 1044, 1000, 1028, 1051, 1069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1004, 1009, 1007, 1039, 1010, 1043, 1041, 992, 1037, 1002, 1003, 1032, 1047, 1028, 1013, 1080, 1001, 1097, 994, 998, 1016, 1019, 986, 1048, 1019, 1018, 1024, 1027, 997, 1057, 1038, 1015, 1059, 993, 984, 1038, 1008, 970, 1065, 1051, 990, 1043, 991, 1036, 987, 1045, 1026, 1037, 1040, 1035, 1035, 1080, 1060, 983, 1022, 1041, 1018, 1006, 1029, 971, 1022, 1036, 1032, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1007, 1073, 1009, 1018, 1033, 1016, 1078, 1032, 1027, 982, 1060, 1066, 997, 1100, 1043, 1028, 1034, 1023, 1003, 974, 999, 1060, 1066, 1056, 995, 1034, 1033, 1036, 1023, 999, 1041, 991, 1009, 1051, 1054, 1012, 1001, 1030, 1016, 1020, 1060, 967, 988, 1073, 1028, 1023, 1051, 1001, 994, 1011, 1020, 1001, 1045, 1011, 1028, 1045, 998, 997, 986, 1001, 1014, 1037, 1008, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1033, 1013, 1039, 1030, 994, 1008, 1060, 1021, 23, 9, 0, 0, 0, 0, 0, 0, 1044, 1001, 1052, 997, 1003, 1006, 1024, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1011, 1006, 1018, 1055, 988, 1019, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 997, 1020, 1024, 1032, 1032, 995, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1020, 1028, 1019, 1053, 1005, 1027, 1021, 14, 7, 6, 3, 0, 0, 0, 0, 998, 1016, 1063, 1034, 1078, 1039, 964, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1024, 1042, 1038, 991, 1038, 1036, 984, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1081, 1062, 1062, 997, 1015, 1010, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8174, 8161, 8163, 8257, 8200, 8127, 8224] avg=8192.0000 max=8257 max_rank=4 min=8127 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1025, 1029, 1035, 1035, 1019, 987, 1016, 1006, 23, 9, 0, 0, 0, 0, 0, 0, 1025, 1055, 1060, 1029, 1091, 1013, 1056, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 984, 1055, 1060, 1022, 1056, 1015, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 982, 1017, 1008, 1041, 980, 1016, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1043, 1007, 1018, 1053, 1005, 999, 956, 14, 8, 5, 3, 0, 0, 0, 0, 983, 982, 1054, 1020, 1033, 1000, 1040, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1027, 1012, 1032, 999, 1016, 1070, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1022, 1054, 1040, 991, 1067, 1012, 1051, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8310, 8253, 8090, 8176, 8169, 8104, 8250] avg=8192.0000 max=8310 max_rank=1 min=8090 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1057, 1015, 991, 1013, 997, 1043, 1019, 1015, 24, 10, 0, 0, 0, 0, 0, 0, 922, 985, 1050, 969, 1042, 996, 1020, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1035, 995, 1054, 1003, 1039, 1010, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1050, 1046, 1039, 1030, 995, 1044, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1020, 1027, 1057, 984, 1005, 1002, 1010, 14, 8, 6, 3, 0, 0, 0, 0, 1029, 1000, 1033, 1004, 1001, 1016, 1068, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1022, 1035, 1042, 1017, 1065, 995, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1022, 1020, 1044, 1000, 1028, 1051, 1069, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 7997, 8203, 8259, 8149, 8171, 8305, 8268] avg=8192.0000 max=8305 max_rank=6 min=7997 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32893 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[957, 981, 1023, 1010, 1023, 1022, 1036, 1001, 24, 10, 0, 0, 0, 0, 0, 0, 1033, 1019, 1025, 990, 1003, 996, 1046, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1051, 1059, 1022, 1032, 994, 1011, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1025, 1005, 1034, 966, 993, 1023, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1023, 1037, 1030, 1038, 1032, 1029, 1068, 14, 7, 6, 3, 0, 0, 0, 0, 1021, 1026, 1052, 1031, 1010, 1046, 1008, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1014, 1046, 1034, 1032, 1032, 1015, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1016, 995, 1012, 1074, 1010, 1028, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8112, 8226, 8136, 8272, 8255, 8213, 8235] avg=8192.0000 max=8272 max_rank=4 min=8087 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32975 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[989, 995, 1031, 991, 988, 1058, 974, 994, 24, 9, 0, 0, 0, 0, 0, 0, 1050, 1059, 1014, 1007, 1003, 1059, 1039, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1010, 1068, 1006, 1074, 1002, 1034, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1027, 1051, 1013, 1030, 1018, 1067, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1010, 1001, 1029, 1047, 1048, 955, 1032, 15, 8, 6, 3, 0, 0, 0, 0, 1016, 972, 1002, 1049, 1010, 998, 1088, 988, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1031, 1016, 1018, 1029, 1033, 1045, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 993, 992, 1008, 1011, 1011, 1085, 1037, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8053, 8254, 8234, 8336, 8167, 8123, 8184, 8185] avg=8192.0000 max=8336 max_rank=3 min=8053 min_rank=0 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1018, 1062, 1040, 1056, 1030, 1036, 1000, 1035, 24, 10, 0, 0, 0, 0, 0, 0, 995, 1077, 995, 1009, 983, 1073, 953, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1034, 1034, 1002, 1012, 1033, 962, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1006, 1060, 980, 1071, 954, 1010, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1002, 1087, 1010, 988, 993, 978, 1013, 14, 8, 6, 3, 0, 0, 0, 0, 1018, 1078, 1053, 1081, 995, 1016, 1047, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1026, 1026, 1076, 1039, 1002, 1004, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 956, 1026, 986, 1073, 1037, 1002, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8151, 8122, 8237, 8080, 8290, 8205, 8140] avg=8192.0000 max=8311 max_rank=0 min=8080 min_rank=4 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1004, 1009, 1007, 1039, 1010, 1043, 1041, 992, 24, 10, 0, 0, 0, 0, 0, 0, 1037, 1002, 1000, 1032, 1047, 1028, 1013, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1081, 994, 998, 1016, 1019, 986, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1018, 1024, 1027, 997, 1057, 1024, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 993, 984, 1038, 1008, 970, 1065, 1051, 14, 7, 6, 3, 0, 0, 0, 0, 990, 1043, 991, 1036, 987, 1045, 1019, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1035, 1035, 1080, 1036, 983, 1022, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1006, 1029, 971, 1022, 1036, 1032, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8239, 8143, 8181, 8198, 8148, 8272, 8176] avg=8192.0000 max=8272 max_rank=6 min=8143 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1007, 1073, 1009, 1018, 1033, 1016, 1078, 1032, 29, 13, 0, 0, 0, 0, 0, 0, 1027, 982, 1056, 1066, 997, 1100, 1043, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1002, 1003, 974, 999, 1060, 1066, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1034, 1033, 1036, 1023, 999, 1023, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1051, 1054, 1012, 1001, 1030, 1016, 1020, 18, 11, 8, 4, 3, 0, 0, 0, 1060, 967, 988, 1073, 1028, 1023, 1040, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1011, 1020, 1001, 1016, 1011, 1028, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 998, 997, 986, 1001, 1014, 1037, 1008, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8299, 8194, 8134, 8237, 8180, 8126, 8058] avg=8192.0000 max=8308 max_rank=0 min=8058 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8448, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8448, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8192, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 128, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1043, 1044, 1023, 1030, 1029, 1014, 1013, 1002, 1028, 1041, 1057, 964, 1051, 1003, 1086, 1003, 1025, 1066, 1035, 1036, 1006, 998, 1015, 1048, 1037, 1008, 1042, 1009, 1051, 1031, 1041, 1055, 999, 1009, 993, 1032, 979, 1015, 1026, 1007, 1027, 1023, 1014, 1016, 1054, 1029, 1005, 1015, 1000, 1011, 1057, 1033, 1047, 1039, 1000, 1041, 1027, 1013, 1030, 969, 982, 1062, 1030, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1012, 1006, 1016, 1034, 1022, 1067, 1037, 1041, 1007, 1037, 1012, 1007, 1020, 1065, 1046, 1044, 1026, 984, 1022, 1014, 989, 995, 988, 1040, 1003, 1095, 1047, 992, 1036, 1044, 998, 1008, 1072, 1012, 1022, 1009, 1090, 1019, 1026, 1000, 1004, 993, 1018, 1008, 1094, 1058, 996, 1028, 996, 1008, 994, 1024, 1041, 981, 1012, 1045, 1006, 983, 1024, 1088, 986, 1002, 1071, 1072] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1014, 980, 1008, 1010, 1008, 1026, 1021, 1003, 1046, 980, 1062, 1013, 1024, 995, 998, 1036, 1008, 1036, 1030, 1081, 1018, 1039, 1051, 1041, 1035, 998, 1074, 989, 1057, 1042, 1036, 1038, 1016, 980, 1074, 1023, 1023, 1027, 970, 1028, 1001, 1003, 1019, 1035, 1039, 1035, 1000, 1045, 1026, 975, 1003, 1098, 1016, 1006, 998, 1034, 1011, 1029, 1017, 1021, 1071, 1013, 1084, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1023, 1060, 1066, 1026, 992, 1066, 1055, 1033, 965, 1019, 1059, 1041, 1008, 1016, 1018, 1025, 1057, 1069, 1026, 1013, 1078, 1051, 988, 1017, 990, 1024, 1040, 1039, 999, 1057, 940, 984, 1020, 1007, 1046, 1048, 1060, 973, 1038, 1019, 976, 1014, 1020, 1034, 1012, 974, 996, 1010, 1030, 997, 1063, 1035, 1038, 1004, 1015, 1068, 1020, 1042, 992, 1047, 1055, 1018, 1053, 968] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1041, 1017, 1031, 1062, 1046, 1028, 958, 1029, 1025, 1032, 1046, 1032, 1069, 990, 1025, 1044, 1031, 1066, 1026, 1003, 1035, 984, 1048, 991, 1010, 1030, 1030, 1021, 1021, 1053, 1037, 1017, 1010, 992, 1000, 998, 1029, 1030, 997, 1091, 1037, 1066, 1031, 1029, 1023, 1057, 1003, 1003, 1004, 1056, 988, 996, 1008, 1004, 1012, 1042, 996, 1000, 1035, 1038, 1002, 1003, 1026, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1052, 1021, 1035, 1007, 1024, 1044, 1017, 1003, 1053, 1012, 988, 1049, 987, 1016, 982, 998, 1062, 1024, 1013, 1005, 1046, 1014, 1006, 979, 1008, 1031, 1006, 1035, 1019, 1039, 1041, 1031, 1025, 1011, 1112, 1001, 1004, 1006, 1029, 1001, 1025, 1025, 1062, 1053, 999, 980, 1039, 1050, 1009, 1068, 1051, 1012, 1012, 1006, 1035, 1006, 1036, 1082, 1043, 1002, 1025, 992, 1009, 1079] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1061, 980, 1019, 1027, 1056, 981, 1016, 973, 1022, 993, 1029, 993, 1052, 1037, 1039, 993, 1029, 1064, 1049, 1002, 1037, 1030, 1033, 1001, 997, 1009, 959, 966, 1060, 1001, 995, 1027, 1013, 1005, 1033, 1050, 1025, 1031, 1069, 1045, 995, 1019, 1029, 1051, 1077, 973, 1013, 1014, 1084, 1012, 1006, 1019, 1023, 1076, 951, 994, 1047, 1070, 1059, 1021, 1047, 1070, 1049, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1074, 1031, 1002, 1029, 996, 1085, 1033, 999, 1020, 1043, 1002, 975, 1027, 1018, 1078, 1037, 984, 1037, 1002, 1020, 1035, 1030, 1046, 1032, 999, 1033, 1026, 1051, 1059, 1022, 1045, 1063, 1019, 1007, 1005, 1000, 1005, 1031, 1069, 1075, 1027, 979, 1030, 1026, 1041, 1044, 1019, 1056, 1011, 1053, 1006, 1018, 1012, 982, 1028, 1027, 1043, 987, 996, 1032, 1005, 988, 999, 983] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1007, 980, 1008, 1010, 1008, 1026, 1021, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 980, 1062, 1013, 1024, 995, 998, 1036, 5, 1, 0, 0, 0, 0, 0, 0, 1008, 1022, 1030, 1081, 1018, 1039, 1051, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 998, 1074, 989, 1052, 1042, 1036, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 980, 1074, 1023, 1023, 1027, 970, 1028, 6, 0, 0, 0, 0, 0, 0, 0, 1001, 1003, 1019, 1035, 1039, 1035, 1000, 1045, 8, 0, 0, 0, 0, 0, 0, 0, 1026, 975, 1003, 1098, 1016, 1006, 998, 1034, 18, 6, 0, 0, 0, 0, 0, 0, 1011, 1029, 1017, 1021, 1071, 1013, 1066, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8063, 8160, 8290, 8264, 8147, 8185, 8180, 8247] avg=8192.0000 max=8290 max_rank=2 min=8063 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1036, 1044, 1023, 1030, 1029, 1014, 1013, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1041, 1057, 964, 1051, 1003, 1086, 1003, 5, 1, 0, 0, 0, 0, 0, 0, 1025, 1052, 1035, 1036, 1006, 998, 1015, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1008, 1042, 1009, 1046, 1031, 1041, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1009, 993, 1032, 979, 1015, 1026, 1007, 6, 0, 0, 0, 0, 0, 0, 0, 1027, 1023, 1014, 1016, 1054, 1029, 1005, 1015, 8, 0, 0, 0, 0, 0, 0, 0, 1000, 1011, 1057, 1033, 1047, 1039, 1000, 1041, 17, 6, 0, 0, 0, 0, 0, 0, 1027, 1013, 1030, 969, 982, 1062, 1013, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8239, 8215, 8269, 8066, 8191, 8251, 8114] avg=8192.0000 max=8269 max_rank=3 min=8066 min_rank=4 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1005, 1006, 1016, 1034, 1022, 1067, 1037, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1037, 1012, 1007, 1020, 1065, 1046, 1044, 5, 1, 0, 0, 0, 0, 0, 0, 1026, 971, 1022, 1014, 989, 995, 988, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1095, 1047, 992, 1031, 1044, 998, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1012, 1022, 1009, 1090, 1019, 1026, 1000, 6, 0, 0, 0, 0, 0, 0, 0, 1004, 993, 1018, 1008, 1094, 1058, 996, 1028, 7, 0, 0, 0, 0, 0, 0, 0, 996, 1008, 994, 1024, 1041, 981, 1012, 1045, 17, 6, 0, 0, 0, 0, 0, 0, 1006, 983, 1024, 1088, 986, 1002, 1054, 1072, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8244, 8045, 8218, 8256, 8206, 8124, 8215] avg=8192.0000 max=8256 max_rank=4 min=8045 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1066, 1031, 1002, 1029, 996, 1085, 1033, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1043, 1002, 975, 1027, 1018, 1078, 1037, 5, 1, 0, 0, 0, 0, 0, 0, 984, 1023, 1002, 1020, 1035, 1030, 1046, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1033, 1026, 1051, 1054, 1022, 1045, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1007, 1005, 1000, 1005, 1031, 1069, 1075, 7, 0, 0, 0, 0, 0, 0, 0, 1027, 979, 1030, 1026, 1041, 1044, 1019, 1056, 8, 0, 0, 0, 0, 0, 0, 0, 1011, 1053, 1006, 1018, 1012, 982, 1028, 1027, 16, 6, 0, 0, 0, 0, 0, 0, 1043, 987, 996, 1032, 1005, 988, 983, 983, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8206, 8172, 8293, 8218, 8230, 8159, 8017] avg=8192.0000 max=8293 max_rank=3 min=8017 min_rank=7 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1034, 1017, 1031, 1062, 1046, 1028, 958, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1032, 1046, 1032, 1069, 990, 1025, 1044, 5, 1, 0, 0, 0, 0, 0, 0, 1031, 1052, 1026, 1003, 1035, 984, 1048, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1030, 1030, 1021, 1016, 1053, 1037, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 992, 1000, 998, 1029, 1030, 997, 1091, 6, 0, 0, 0, 0, 0, 0, 0, 1037, 1066, 1031, 1029, 1023, 1057, 1003, 1003, 8, 0, 0, 0, 0, 0, 0, 0, 1004, 1056, 988, 996, 1008, 1004, 1012, 1042, 17, 6, 0, 0, 0, 0, 0, 0, 996, 1000, 1035, 1038, 1002, 1003, 1009, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8269, 8170, 8214, 8153, 8257, 8133, 8135] avg=8192.0000 max=8269 max_rank=1 min=8133 min_rank=6 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1045, 1021, 1035, 1007, 1024, 1044, 1017, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1012, 988, 1049, 987, 1016, 982, 998, 5, 1, 0, 0, 0, 0, 0, 0, 1062, 1010, 1013, 1005, 1046, 1014, 1006, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1031, 1006, 1035, 1014, 1039, 1041, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1011, 1112, 1001, 1004, 1006, 1029, 1001, 6, 0, 0, 0, 0, 0, 0, 0, 1025, 1025, 1062, 1053, 999, 980, 1039, 1050, 8, 0, 0, 0, 0, 0, 0, 0, 1009, 1068, 1051, 1012, 1012, 1006, 1035, 1006, 16, 6, 0, 0, 0, 0, 0, 0, 1036, 1082, 1043, 1002, 1025, 992, 993, 1079, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8091, 8135, 8205, 8195, 8241, 8221, 8252] avg=8192.0000 max=8252 max_rank=7 min=8091 min_rank=1 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1053, 980, 1019, 1027, 1056, 981, 1016, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 993, 1029, 993, 1052, 1037, 1039, 993, 5, 1, 0, 0, 0, 0, 0, 0, 1029, 1050, 1049, 1002, 1037, 1030, 1033, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1009, 959, 966, 1055, 1001, 995, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1005, 1033, 1050, 1025, 1031, 1069, 1045, 7, 0, 0, 0, 0, 0, 0, 0, 995, 1019, 1029, 1051, 1077, 973, 1013, 1014, 8, 0, 0, 0, 0, 0, 0, 0, 1084, 1012, 1006, 1019, 1023, 1076, 951, 994, 17, 6, 0, 0, 0, 0, 0, 0, 1047, 1070, 1059, 1021, 1047, 1070, 1032, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8164, 8231, 8009, 8278, 8179, 8188, 8382] avg=8192.0000 max=8382 max_rank=7 min=8009 min_rank=3 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=33027 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1009, 1060, 1066, 1026, 992, 1066, 1055, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 965, 1019, 1059, 1041, 1008, 1016, 1018, 1025, 9, 3, 0, 0, 0, 0, 0, 0, 1057, 1048, 1026, 1013, 1078, 1051, 988, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1024, 1040, 1039, 990, 1057, 940, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1007, 1046, 1048, 1060, 973, 1038, 1019, 11, 1, 0, 0, 0, 0, 0, 0, 976, 1014, 1020, 1034, 1012, 974, 996, 1010, 11, 0, 0, 0, 0, 0, 0, 0, 1030, 997, 1063, 1035, 1038, 1004, 1015, 1068, 21, 9, 0, 0, 0, 0, 0, 0, 1020, 1042, 992, 1047, 1055, 1018, 1032, 968, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 8163, 8278, 8064, 8223, 8047, 8280, 8174] avg=8192.0000 max=8307 max_rank=0 min=8047 min_rank=5 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8064, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8320, 8192, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8448, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1111, 1053, 1057, 995, 1019, 1030, 934, 1025, 1015, 1032, 989, 1009, 1005, 1075, 1044, 969, 1010, 1090, 1014, 1013, 994, 1053, 1033, 1024, 1012, 1051, 1023, 1026, 1040, 1021, 1050, 1049, 1017, 1000, 997, 1027, 1038, 1016, 977, 1043, 1031, 1035, 1013, 1021, 998, 1001, 1010, 1008, 1019, 1028, 1026, 1031, 994, 1014, 1047, 1065, 1048, 1023, 1053, 1046, 997, 994, 1021, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[999, 1005, 1012, 1046, 1009, 1031, 1039, 979, 1019, 1019, 1009, 991, 1028, 1001, 1044, 1026, 1025, 995, 1004, 1059, 1042, 1020, 1007, 1047, 971, 1033, 1017, 1033, 1052, 1049, 1010, 1027, 1029, 1018, 1019, 992, 1005, 1036, 1031, 1024, 1047, 998, 1009, 1025, 1012, 1019, 1093, 992, 1033, 999, 1019, 1050, 1079, 963, 1019, 1025, 975, 1060, 1065, 1023, 1044, 1095, 1037, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1048, 1026, 1014, 978, 975, 1001, 1066, 999, 1053, 1003, 1049, 1010, 1010, 1038, 1007, 1052, 1052, 996, 1013, 1029, 1073, 991, 1029, 1028, 1016, 1029, 1057, 1043, 1071, 1021, 1009, 975, 996, 1022, 1009, 980, 1019, 1004, 1004, 1008, 984, 1018, 1046, 1010, 1014, 1041, 1040, 1013, 1067, 1011, 1052, 996, 1057, 1051, 1078, 992, 1062, 1043, 1062, 1040, 992, 979, 1048, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1091, 1041, 1047, 1052, 1032, 1026, 996, 984, 996, 1013, 1009, 984, 1067, 1014, 964, 1020, 1046, 1013, 1021, 975, 1010, 1033, 1007, 987, 1050, 1033, 1022, 992, 1079, 1026, 1022, 1003, 1013, 1042, 999, 1015, 1037, 1004, 1011, 1060, 1048, 1022, 976, 1027, 1002, 1016, 1015, 1066, 1058, 1033, 1037, 1026, 1024, 1018, 1066, 1048, 1007, 972, 1036, 1035, 1061, 1045, 1002, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1014, 1005, 985, 996, 990, 1023, 1002, 1003, 1020, 1059, 980, 1093, 1037, 1021, 1030, 1007, 1055, 1035, 994, 1038, 1030, 1053, 1019, 1068, 1014, 1046, 1027, 1021, 1004, 1059, 1086, 1000, 997, 1020, 1002, 1008, 1013, 955, 1031, 1037, 1010, 1037, 1052, 1090, 1028, 1059, 996, 991, 990, 1061, 1059, 994, 1033, 1030, 995, 1017, 1032, 1041, 1025, 1020, 988, 1047, 1017, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[974, 1042, 1064, 1039, 1017, 1016, 1000, 1014, 1008, 1055, 1053, 1015, 1055, 1037, 984, 1007, 994, 1047, 1023, 1067, 992, 1001, 1031, 1067, 1011, 1071, 1030, 1003, 1025, 1001, 1034, 1007, 967, 986, 1029, 1035, 1009, 975, 1029, 1052, 1079, 1021, 1001, 1038, 993, 1036, 1030, 981, 1008, 985, 1033, 1045, 986, 1044, 1072, 1007, 1060, 1081, 1050, 1067, 1036, 1022, 997, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[958, 989, 1017, 1040, 1055, 1031, 1017, 1047, 1040, 1026, 1012, 1014, 1034, 1045, 1040, 1014, 977, 1025, 1044, 993, 1024, 1041, 996, 1029, 1059, 1015, 1067, 984, 997, 1030, 986, 1011, 1062, 1025, 1096, 1000, 1004, 1019, 1050, 1035, 963, 1040, 1062, 1024, 1041, 1030, 1056, 1036, 947, 1045, 1008, 1024, 1005, 1026, 1066, 1010, 1052, 1008, 1028, 1000, 1021, 1043, 1017, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1027, 1012, 1060, 967, 1036, 1048, 1048, 1065, 997, 1049, 1030, 1008, 1028, 1024, 1048, 1053, 1008, 1091, 999, 1004, 1008, 1024, 1029, 1003, 1017, 1037, 1000, 1043, 1053, 967, 1024, 1086, 1045, 1044, 950, 1067, 1046, 982, 962, 1012, 1054, 988, 1075, 1013, 1016, 1048, 995, 1026, 954, 1049, 992, 1024, 1023, 1018, 1032, 976, 1051, 1042, 1055, 1054, 1003, 1015, 1073, 959] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1111, 1053, 1057, 995, 1019, 1030, 934, 1025, 23, 3, 0, 0, 0, 0, 0, 0, 1015, 1032, 989, 1009, 1005, 1075, 1044, 969, 2, 2, 0, 0, 0, 0, 0, 0, 1010, 1090, 1014, 1013, 994, 1053, 1033, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1051, 1023, 1026, 1017, 1021, 1050, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1000, 997, 1027, 1038, 1016, 977, 1043, 62, 0, 0, 0, 0, 0, 0, 0, 1031, 1035, 1013, 1021, 998, 999, 1010, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1028, 1026, 1031, 994, 1014, 1042, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1023, 991, 1046, 997, 994, 1021, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8142, 8231, 8249, 8177, 8115, 8219, 8153] avg=8192.0000 max=8250 max_rank=0 min=8115 min_rank=5 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[999, 1005, 1012, 1046, 1009, 1031, 1039, 979, 23, 3, 0, 0, 0, 0, 0, 0, 1019, 1019, 1009, 991, 1028, 1001, 1044, 1026, 2, 2, 0, 0, 0, 0, 0, 0, 1025, 995, 1004, 1059, 1042, 1020, 1007, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1033, 1017, 1033, 1029, 1049, 1010, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1018, 1019, 992, 1005, 1036, 1031, 1024, 63, 0, 0, 0, 0, 0, 0, 0, 1047, 998, 1009, 1025, 1012, 1017, 1093, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 999, 1019, 1050, 1079, 963, 1014, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1060, 1002, 1023, 1044, 1095, 1037, 1053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8141, 8199, 8169, 8217, 8193, 8182, 8289] avg=8192.0000 max=8289 max_rank=7 min=8141 min_rank=1 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1048, 1026, 1014, 978, 975, 1001, 1066, 999, 24, 3, 0, 0, 0, 0, 0, 0, 1053, 1003, 1049, 1010, 1010, 1038, 1007, 1052, 2, 2, 0, 0, 0, 0, 0, 0, 1052, 996, 1013, 1029, 1073, 991, 1029, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1029, 1057, 1043, 1047, 1021, 1009, 975, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1022, 1009, 980, 1019, 1004, 1004, 1008, 63, 0, 0, 0, 0, 0, 0, 0, 984, 1018, 1046, 1010, 1014, 1039, 1040, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1011, 1052, 996, 1057, 1051, 1073, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1043, 999, 1040, 992, 979, 1048, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8226, 8211, 8197, 8105, 8164, 8299, 8200] avg=8192.0000 max=8299 max_rank=6 min=8105 min_rank=4 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1091, 1041, 1047, 1052, 1032, 1026, 996, 984, 24, 3, 0, 0, 0, 0, 0, 0, 996, 1013, 1009, 984, 1067, 1014, 964, 1020, 2, 2, 0, 0, 0, 0, 0, 0, 1046, 1013, 1021, 975, 1010, 1033, 1007, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1033, 1022, 992, 1055, 1026, 1022, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1042, 999, 1015, 1037, 1004, 1011, 1060, 61, 0, 0, 0, 0, 0, 0, 0, 1048, 1022, 976, 1027, 1002, 1014, 1015, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1033, 1037, 1026, 1024, 1018, 1061, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 972, 975, 1035, 1061, 1045, 1002, 1060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8071, 8092, 8203, 8242, 8170, 8305, 8157] avg=8192.0000 max=8305 max_rank=6 min=8071 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[958, 989, 1017, 1040, 1055, 1031, 1017, 1047, 22, 3, 0, 0, 0, 0, 0, 0, 1040, 1026, 1012, 1014, 1034, 1045, 1040, 1014, 2, 2, 0, 0, 0, 0, 0, 0, 977, 1025, 1044, 993, 1024, 1041, 996, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1015, 1067, 984, 975, 1030, 986, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1025, 1096, 1000, 1004, 1019, 1050, 1035, 61, 0, 0, 0, 0, 0, 0, 0, 963, 1040, 1062, 1024, 1041, 1028, 1056, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 947, 1045, 1008, 1024, 1005, 1026, 1061, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1008, 967, 1000, 1021, 1043, 1017, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8229, 8129, 8127, 8352, 8250, 8126, 8144] avg=8192.0000 max=8352 max_rank=4 min=8126 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[974, 1042, 1064, 1039, 1017, 1016, 1000, 1014, 23, 3, 0, 0, 0, 0, 0, 0, 1008, 1055, 1053, 1015, 1055, 1037, 984, 1007, 2, 2, 0, 0, 0, 0, 0, 0, 994, 1047, 1023, 1067, 992, 1001, 1031, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1071, 1030, 1003, 1002, 1001, 1034, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 967, 986, 1029, 1035, 1009, 975, 1029, 1052, 62, 0, 0, 0, 0, 0, 0, 0, 1079, 1021, 1001, 1038, 993, 1034, 1030, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 985, 1033, 1045, 986, 1044, 1067, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1081, 988, 1067, 1036, 1022, 997, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8218, 8222, 8159, 8144, 8177, 8175, 8249] avg=8192.0000 max=8249 max_rank=7 min=8144 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1027, 1012, 1060, 967, 1036, 1048, 1048, 1065, 27, 6, 0, 0, 0, 0, 0, 0, 997, 1049, 1030, 1008, 1028, 1024, 1048, 1053, 9, 6, 6, 0, 0, 0, 0, 0, 1008, 1085, 999, 1004, 1008, 1024, 1029, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1037, 1000, 1043, 1023, 967, 1024, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1044, 950, 1067, 1046, 982, 962, 1012, 65, 3, 0, 0, 0, 0, 0, 0, 1054, 988, 1075, 1013, 1016, 1039, 995, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1049, 992, 1024, 1023, 1018, 1020, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1042, 990, 1054, 1003, 1015, 1073, 959, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8258, 8160, 8197, 8176, 8206, 8056, 8187] avg=8192.0000 max=8296 max_rank=0 min=8056 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32625 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1014, 1005, 985, 996, 990, 1023, 1002, 1003, 22, 3, 0, 0, 0, 0, 0, 0, 1020, 1059, 980, 1093, 1037, 1021, 1030, 1007, 2, 2, 0, 0, 0, 0, 0, 0, 1055, 1035, 994, 1038, 1030, 1053, 1019, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1046, 1027, 1021, 982, 1059, 1086, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1020, 1002, 1008, 1013, 955, 1031, 1037, 60, 0, 0, 0, 0, 0, 0, 0, 1010, 1037, 1052, 1090, 1028, 1057, 996, 991, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1061, 1059, 994, 1033, 1030, 990, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1041, 965, 1020, 988, 1047, 1017, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8043, 8251, 8292, 8235, 8123, 8261, 8174, 8157] avg=8192.0000 max=8292 max_rank=2 min=8043 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8064, 8192, 8320, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 7936, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[984, 1032, 1039, 1001, 985, 1024, 1043, 969, 999, 1014, 997, 1042, 1032, 999, 1043, 1050, 1008, 1077, 1076, 1034, 1043, 1039, 1009, 1013, 1056, 1016, 1037, 955, 1012, 1010, 1023, 969, 1067, 1068, 1048, 1053, 1040, 1021, 992, 1021, 1011, 1010, 1032, 1075, 1002, 1001, 1026, 1024, 1019, 1020, 1067, 991, 1010, 1034, 1043, 1033, 1051, 971, 1049, 1036, 961, 1107, 989, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1016, 976, 1035, 1002, 1050, 1007, 1003, 1079, 1016, 999, 1023, 1004, 1029, 1002, 1009, 1005, 1034, 1026, 1032, 1024, 1054, 1053, 996, 1061, 973, 1064, 1048, 1023, 1028, 1056, 1007, 1036, 1020, 1047, 1052, 1011, 999, 966, 1005, 1019, 1041, 1030, 1056, 1043, 982, 1045, 1008, 1029, 1016, 1051, 1042, 990, 1005, 988, 1000, 1027, 1078, 1044, 1067, 1061, 962, 1029, 1024, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1010, 1008, 982, 994, 1013, 1043, 997, 1032, 986, 1068, 1013, 1033, 1031, 1026, 986, 1027, 1030, 1016, 956, 1035, 997, 980, 1016, 1005, 1045, 1001, 990, 990, 1040, 1027, 1030, 1006, 1023, 1087, 1020, 1041, 1029, 990, 1044, 1049, 1053, 1054, 1035, 1060, 1040, 1029, 1019, 1065, 1028, 1067, 1046, 1025, 1052, 1006, 1041, 1026, 1027, 1003, 1015, 1031, 1005, 1078, 1037, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1012, 1017, 1024, 1025, 1018, 1054, 1030, 998, 1043, 1024, 1041, 990, 1033, 977, 1038, 1019, 1012, 1030, 1005, 1061, 979, 1077, 1033, 1046, 1061, 1057, 1046, 1029, 1017, 1052, 1042, 1032, 1046, 1025, 997, 1008, 1031, 1023, 1033, 1010, 963, 1043, 1027, 1045, 1028, 1023, 1024, 1015, 1021, 1022, 1009, 1030, 1038, 992, 981, 1083, 1015, 1000, 1009, 1009, 1009, 1004, 1018, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1026, 1054, 1023, 1006, 987, 1035, 1023, 947, 1034, 1040, 974, 1031, 1030, 1003, 1022, 973, 1047, 1042, 998, 1020, 996, 1034, 1043, 1047, 1031, 1027, 982, 1076, 1002, 1012, 1029, 1024, 1024, 1028, 1010, 1013, 1026, 1003, 1047, 1024, 1063, 1053, 1046, 1004, 965, 1011, 1000, 992, 1047, 1071, 1061, 1036, 1031, 1001, 1057, 1042, 1048, 1058, 1034, 996, 1037, 1037, 1017, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1021, 963, 934, 1056, 976, 1018, 1056, 1044, 1009, 1038, 1007, 1031, 992, 1062, 1021, 1025, 1056, 1020, 1007, 1063, 1028, 1043, 1021, 970, 1027, 1029, 1016, 1064, 1042, 1042, 1016, 1010, 1048, 1041, 1017, 1041, 1037, 916, 964, 1029, 1075, 1024, 1052, 1055, 1033, 977, 1029, 1046, 1035, 1031, 1006, 1009, 995, 1046, 1052, 1061, 1019, 999, 1007, 1023, 1081, 1061, 1006, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1046, 1026, 1056, 1040, 1063, 1010, 1053, 1026, 1023, 1019, 1043, 1019, 1030, 1063, 1000, 999, 1013, 1010, 1032, 981, 1034, 984, 956, 1020, 1046, 1014, 1019, 1087, 989, 1028, 1059, 1015, 1034, 1020, 1052, 1009, 1041, 1038, 1026, 1051, 1021, 1022, 1072, 1054, 1045, 1015, 1010, 1000, 980, 1019, 1003, 1016, 1053, 1039, 1019, 996, 1037, 1043, 1040, 986, 1059, 971, 1000, 962] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1037, 1032, 1054, 1024, 1048, 969, 1079, 1032, 995, 1031, 1052, 999, 1085, 1036, 986, 1018, 1033, 1031, 1005, 1045, 1029, 1003, 1074, 1037, 1042, 969, 1041, 988, 1001, 1046, 972, 1031, 1041, 1044, 997, 1042, 994, 984, 992, 1044, 1049, 1023, 1063, 997, 1042, 1054, 1039, 1048, 1039, 1032, 1019, 970, 1012, 999, 1000, 1012, 1055, 969, 989, 994, 994, 1056, 1062, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[984, 1032, 1039, 1001, 985, 1024, 1043, 969, 33, 0, 0, 0, 0, 0, 0, 0, 999, 1014, 997, 1042, 1032, 999, 1043, 1050, 13, 6, 5, 3, 1, 0, 0, 0, 1008, 1077, 1076, 1029, 1043, 1039, 1009, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1016, 1037, 955, 1012, 1010, 1023, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1065, 1048, 1053, 1040, 1021, 992, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1010, 986, 1075, 1002, 1001, 1026, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1014, 1067, 991, 1010, 1034, 1043, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 971, 1049, 1036, 961, 1107, 989, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8204, 8294, 8077, 8307, 8135, 8211, 8198] avg=8192.0000 max=8307 max_rank=4 min=8077 min_rank=3 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1016, 976, 1035, 1002, 1050, 1007, 1003, 1079, 34, 0, 0, 0, 0, 0, 0, 0, 1016, 999, 1023, 1004, 1029, 1002, 1009, 1005, 13, 7, 5, 3, 1, 0, 0, 0, 1034, 1026, 1032, 1019, 1054, 1053, 996, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1064, 1048, 1023, 1028, 1056, 1007, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1044, 1052, 1011, 999, 966, 1005, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1030, 1009, 1043, 982, 1045, 1008, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1044, 1042, 990, 1005, 988, 1000, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1044, 1067, 1061, 962, 1029, 1024, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8116, 8275, 8234, 8116, 8187, 8112, 8294] avg=8192.0000 max=8294 max_rank=7 min=8112 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1026, 1054, 1023, 1006, 987, 1035, 1023, 947, 33, 0, 0, 0, 0, 0, 0, 0, 1034, 1040, 974, 1031, 1030, 1003, 1022, 973, 13, 7, 5, 3, 1, 0, 0, 0, 1047, 1042, 998, 1015, 996, 1034, 1043, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1027, 982, 1076, 1002, 1012, 1029, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1025, 1010, 1013, 1026, 1003, 1047, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1053, 1000, 1004, 965, 1011, 1000, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1064, 1061, 1036, 1031, 1001, 1057, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1058, 1034, 996, 1037, 1037, 1017, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8136, 8222, 8182, 8172, 8088, 8339, 8263] avg=8192.0000 max=8339 max_rank=6 min=8088 min_rank=5 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32862 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1012, 1017, 1024, 1025, 1018, 1054, 1030, 998, 33, 0, 0, 0, 0, 0, 0, 0, 1043, 1024, 1041, 990, 1033, 977, 1038, 1019, 13, 6, 5, 3, 1, 0, 0, 0, 1012, 1030, 1005, 1056, 979, 1077, 1033, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1057, 1046, 1029, 1017, 1052, 1042, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1022, 997, 1008, 1031, 1023, 1033, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1043, 981, 1045, 1028, 1023, 1024, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1016, 1009, 1030, 1038, 992, 981, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1000, 1009, 1009, 1009, 1004, 1018, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8193, 8238, 8335, 8170, 8122, 8170, 8097] avg=8192.0000 max=8335 max_rank=3 min=8097 min_rank=7 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32559 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1010, 1008, 982, 994, 1013, 1043, 997, 1032, 36, 0, 0, 0, 0, 0, 0, 0, 986, 1068, 1013, 1033, 1031, 1026, 986, 1027, 18, 12, 8, 5, 5, 0, 0, 0, 1030, 1016, 956, 1027, 997, 980, 1016, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1001, 990, 990, 1040, 1027, 1030, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1082, 1020, 1041, 1029, 990, 1044, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1054, 981, 1060, 1040, 1029, 1019, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1055, 1046, 1025, 1052, 1006, 1041, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1003, 1015, 1031, 1005, 1078, 1037, 998, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8218, 8027, 8118, 8278, 8301, 8279, 8200] avg=8192.0000 max=8301 max_rank=5 min=8027 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=33058 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1021, 963, 934, 1056, 976, 1018, 1056, 1044, 33, 0, 0, 0, 0, 0, 0, 0, 1009, 1038, 1007, 1031, 992, 1062, 1021, 1025, 13, 6, 5, 3, 1, 0, 0, 0, 1056, 1020, 1007, 1058, 1028, 1043, 1021, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1029, 1016, 1064, 1042, 1042, 1016, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1038, 1017, 1041, 1037, 916, 964, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1024, 1006, 1055, 1033, 977, 1029, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1025, 1006, 1009, 995, 1046, 1052, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 999, 1007, 1023, 1081, 1061, 1006, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8213, 8203, 8245, 8090, 8245, 8229, 8210] avg=8192.0000 max=8245 max_rank=3 min=8090 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1046, 1026, 1056, 1040, 1063, 1010, 1053, 1026, 34, 0, 0, 0, 0, 0, 0, 0, 1023, 1019, 1043, 1019, 1030, 1063, 1000, 999, 14, 6, 5, 3, 1, 0, 0, 0, 1013, 1010, 1032, 976, 1034, 984, 956, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1014, 1019, 1087, 989, 1028, 1059, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1017, 1052, 1009, 1041, 1038, 1026, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1022, 1024, 1054, 1045, 1015, 1010, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1013, 1003, 1016, 1053, 1039, 1019, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1043, 1040, 986, 1059, 971, 1000, 962, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8354, 8225, 8025, 8256, 8268, 8191, 8119, 8098] avg=8192.0000 max=8354 max_rank=0 min=8025 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1037, 1032, 1054, 1024, 1048, 969, 1079, 1032, 34, 0, 0, 0, 0, 0, 0, 0, 995, 1031, 1052, 999, 1085, 1036, 986, 1018, 14, 6, 5, 3, 1, 0, 0, 0, 1033, 1031, 1005, 1040, 1029, 1003, 1074, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 969, 1041, 988, 1001, 1046, 972, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1041, 997, 1042, 994, 984, 992, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1023, 1015, 997, 1042, 1054, 1039, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1026, 1019, 970, 1012, 999, 1000, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 969, 989, 994, 994, 1056, 1062, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8309, 8231, 8252, 8089, 8135, 8267, 8077, 8176] avg=8192.0000 max=8309 max_rank=0 min=8077 min_rank=6 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8448, 8320, 8320, 8320, 8064, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8448, 8192, 8320, 8192, 8192, 8448, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8448, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 128, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1023, 985, 994, 1071, 975, 1029, 1050, 1022, 1047, 1032, 1001, 1000, 1020, 1003, 1005, 1042, 1075, 1023, 1066, 1019, 1032, 1013, 1044, 1027, 1009, 978, 1028, 998, 1036, 1078, 1012, 983, 1027, 1005, 1016, 990, 979, 993, 1004, 984, 1045, 1002, 1044, 1033, 1093, 1051, 1055, 1021, 1004, 1010, 1051, 1060, 1022, 1004, 1020, 1017, 976, 1005, 1039, 1051, 1048, 1028, 1066, 1073] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1028, 989, 1050, 993, 1022, 1033, 1048, 1042, 1079, 1050, 1026, 991, 1002, 1017, 973, 1000, 973, 959, 1063, 1069, 1013, 1028, 1057, 1049, 1050, 995, 1011, 1054, 1007, 989, 1070, 965, 1024, 1019, 1001, 1028, 1015, 1053, 1075, 1016, 1026, 978, 1014, 990, 1039, 990, 1074, 992, 1060, 1056, 1049, 1030, 995, 1021, 1037, 1080, 1029, 1021, 1028, 1041, 1010, 1024, 1001, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1022, 990, 976, 1056, 1023, 1024, 1046, 1039, 1064, 988, 1041, 1032, 1003, 1051, 971, 1014, 1031, 1036, 1037, 1005, 1020, 1035, 983, 1031, 1018, 1029, 1027, 1015, 1047, 992, 1008, 1024, 1008, 1046, 1047, 1032, 1050, 1027, 973, 1023, 1057, 1061, 966, 998, 1035, 993, 1017, 1041, 1040, 1103, 1038, 1005, 1048, 1017, 1009, 1092, 1006, 984, 1057, 1020, 1029, 985, 1013, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1044, 999, 972, 1018, 1038, 1043, 1041, 958, 1039, 1054, 956, 1028, 993, 1026, 1016, 1019, 1016, 1061, 1042, 1027, 1015, 1018, 1010, 1020, 1026, 1044, 992, 1035, 1040, 1019, 1007, 1042, 1035, 1029, 1004, 1047, 1018, 1068, 1049, 1022, 1015, 1002, 1053, 1014, 1046, 1019, 969, 1047, 1010, 1017, 1031, 1048, 1038, 999, 1001, 1014, 1034, 1020, 1064, 1129, 996, 996, 1010, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1026, 1030, 1064, 1038, 1025, 1020, 1031, 1048, 1040, 992, 1068, 1043, 1024, 983, 1017, 1014, 1028, 1026, 1061, 1009, 1004, 1076, 1040, 1010, 1023, 1036, 1025, 1009, 1057, 954, 1045, 1044, 1035, 1014, 1077, 1030, 995, 997, 1067, 1031, 993, 1032, 976, 1033, 1023, 1033, 970, 1059, 1034, 1045, 1016, 1030, 1024, 978, 1035, 995, 982, 1007, 1036, 951, 1032, 1002, 1042, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[998, 1014, 1030, 1027, 1051, 1021, 1006, 996, 1035, 1076, 1010, 1013, 1023, 1018, 1010, 1062, 1044, 984, 1032, 994, 1028, 1016, 1024, 1026, 985, 1044, 1021, 998, 986, 1038, 1011, 987, 1007, 1027, 1052, 1002, 1034, 1038, 1047, 1031, 994, 1046, 995, 1016, 1063, 1040, 1012, 1089, 1089, 1043, 1034, 1033, 1011, 1019, 1019, 1017, 1009, 1048, 987, 1008, 1019, 1048, 1034, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1008, 1061, 1004, 990, 1018, 1012, 1009, 1002, 1053, 1057, 1027, 1058, 1030, 1061, 1011, 1006, 1022, 1061, 1041, 998, 1090, 1025, 1035, 991, 1066, 1052, 1010, 1012, 1035, 1052, 1052, 981, 1042, 1009, 974, 1032, 1053, 1029, 1005, 1021, 1010, 1010, 1057, 1036, 1010, 1051, 1039, 989, 1060, 1024, 1014, 1059, 1034, 970, 1012, 1018, 1032, 973, 1001, 1005, 1002, 991, 1032, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1010, 1033, 1054, 1069, 1038, 1024, 1046, 993, 993, 1033, 1011, 1068, 1021, 1014, 1027, 1000, 1046, 1033, 1032, 997, 1025, 1031, 1045, 994, 1008, 1042, 999, 1024, 988, 1063, 1058, 960, 1040, 1023, 1065, 1010, 985, 984, 1053, 1076, 994, 1019, 996, 1007, 1074, 1001, 995, 1047, 1000, 1000, 998, 1042, 1045, 1007, 1032, 1024, 1012, 1050, 1071, 993, 1048, 1031, 1023, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1022, 990, 976, 1056, 1023, 1024, 1046, 1039, 11, 0, 0, 0, 0, 0, 0, 0, 1064, 988, 1041, 1032, 1003, 1051, 971, 1014, 5, 1, 0, 0, 0, 0, 0, 0, 1031, 1036, 1010, 1005, 1020, 1035, 983, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1029, 1027, 1015, 1047, 992, 1008, 1024, 29, 0, 0, 0, 0, 0, 0, 0, 1008, 1046, 1047, 1032, 1050, 1027, 967, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1061, 966, 998, 1035, 993, 1017, 1041, 5, 0, 0, 0, 0, 0, 0, 0, 1040, 1103, 1038, 976, 1048, 1017, 1009, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 984, 1057, 1020, 1029, 985, 1013, 1038, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8170, 8151, 8189, 8200, 8173, 8323, 8143] avg=8192.0000 max=8323 max_rank=6 min=8143 min_rank=7 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1028, 989, 1050, 993, 1022, 1033, 1048, 1042, 11, 0, 0, 0, 0, 0, 0, 0, 1079, 1050, 1026, 991, 1002, 1017, 973, 1000, 5, 1, 0, 0, 0, 0, 0, 0, 973, 959, 1036, 1069, 1013, 1028, 1057, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 995, 1011, 1054, 1007, 989, 1070, 965, 30, 0, 0, 0, 0, 0, 0, 0, 1024, 1019, 1001, 1028, 1015, 1053, 1068, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 978, 1014, 990, 1039, 990, 1074, 992, 6, 0, 0, 0, 0, 0, 0, 0, 1060, 1056, 1049, 1000, 995, 1021, 1037, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1021, 1028, 1041, 1010, 1024, 1001, 1025, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8144, 8184, 8171, 8224, 8109, 8298, 8190] avg=8192.0000 max=8298 max_rank=6 min=8109 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 1030, 1064, 1038, 1025, 1020, 1031, 1048, 11, 0, 0, 0, 0, 0, 0, 0, 1040, 992, 1068, 1043, 1024, 983, 1017, 1014, 5, 1, 0, 0, 0, 0, 0, 0, 1028, 1026, 1034, 1009, 1004, 1076, 1040, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1036, 1025, 1009, 1057, 954, 1045, 1044, 30, 0, 0, 0, 0, 0, 0, 0, 1035, 1014, 1077, 1030, 995, 997, 1060, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1032, 976, 1033, 1023, 1033, 970, 1059, 6, 0, 0, 0, 0, 0, 0, 0, 1034, 1045, 1016, 1000, 1024, 978, 1035, 995, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1007, 1036, 951, 1032, 1002, 1042, 1052, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8187, 8227, 8223, 8239, 8125, 8127, 8115] avg=8192.0000 max=8293 max_rank=0 min=8115 min_rank=7 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32606 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1023, 985, 994, 1071, 975, 1029, 1050, 1022, 11, 1, 0, 0, 0, 0, 0, 0, 1047, 1032, 1001, 1000, 1020, 1003, 1005, 1042, 5, 1, 0, 0, 0, 0, 0, 0, 1075, 1023, 1039, 1019, 1032, 1013, 1044, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 978, 1028, 998, 1036, 1078, 1012, 983, 31, 0, 0, 0, 0, 0, 0, 0, 1027, 1005, 1016, 990, 979, 993, 998, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1002, 1044, 1033, 1093, 1051, 1055, 1021, 5, 0, 0, 0, 0, 0, 0, 0, 1004, 1010, 1051, 1028, 1022, 1004, 1020, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1005, 1039, 1051, 1048, 1028, 1066, 1073, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8156, 8272, 8153, 7992, 8349, 8156, 8297] avg=8192.0000 max=8349 max_rank=5 min=7992 min_rank=4 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1044, 999, 972, 1018, 1038, 1043, 1041, 958, 11, 1, 0, 0, 0, 0, 0, 0, 1039, 1054, 956, 1028, 993, 1026, 1016, 1019, 5, 1, 0, 0, 0, 0, 0, 0, 1016, 1061, 1015, 1027, 1015, 1018, 1010, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1044, 992, 1035, 1040, 1019, 1007, 1042, 30, 0, 0, 0, 0, 0, 0, 0, 1035, 1029, 1004, 1047, 1018, 1068, 1043, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1002, 1053, 1014, 1046, 1019, 969, 1047, 5, 0, 0, 0, 0, 0, 0, 0, 1010, 1017, 1031, 1017, 1038, 999, 1001, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1020, 1064, 1129, 996, 996, 1010, 1034, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8137, 8182, 8235, 8266, 8170, 8127, 8294] avg=8192.0000 max=8294 max_rank=7 min=8125 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[998, 1014, 1030, 1027, 1051, 1021, 1006, 996, 13, 4, 0, 0, 0, 0, 0, 0, 1035, 1076, 1010, 1013, 1023, 1018, 1010, 1062, 11, 2, 0, 0, 0, 0, 0, 0, 1044, 984, 991, 994, 1028, 1016, 1024, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1044, 1021, 998, 986, 1038, 1011, 987, 33, 0, 0, 0, 0, 0, 0, 0, 1007, 1027, 1052, 1002, 1034, 1038, 1035, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1046, 995, 1016, 1063, 1040, 1012, 1089, 10, 0, 0, 0, 0, 0, 0, 0, 1089, 1043, 1034, 996, 1011, 1019, 1019, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1048, 987, 1008, 1019, 1048, 1034, 1017, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8260, 8107, 8103, 8226, 8265, 8228, 8187] avg=8192.0000 max=8265 max_rank=5 min=8103 min_rank=3 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1008, 1061, 1004, 990, 1018, 1012, 1009, 1002, 11, 1, 0, 0, 0, 0, 0, 0, 1053, 1057, 1027, 1058, 1030, 1061, 1011, 1006, 5, 1, 0, 0, 0, 0, 0, 0, 1022, 1061, 1014, 998, 1090, 1025, 1035, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1052, 1010, 1012, 1035, 1052, 1052, 981, 30, 0, 0, 0, 0, 0, 0, 0, 1042, 1009, 974, 1032, 1053, 1029, 999, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1010, 1057, 1036, 1010, 1051, 1039, 989, 5, 0, 0, 0, 0, 0, 0, 0, 1060, 1024, 1014, 1028, 1034, 970, 1012, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 973, 1001, 1005, 1002, 991, 1032, 1012, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8309, 8236, 8290, 8159, 8207, 8160, 8059] avg=8192.0000 max=8309 max_rank=1 min=8059 min_rank=7 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1010, 1033, 1054, 1069, 1038, 1024, 1046, 993, 10, 1, 0, 0, 0, 0, 0, 0, 993, 1033, 1011, 1068, 1021, 1014, 1027, 1000, 5, 1, 0, 0, 0, 0, 0, 0, 1046, 1033, 1006, 997, 1025, 1031, 1045, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1042, 999, 1024, 988, 1063, 1058, 960, 30, 0, 0, 0, 0, 0, 0, 0, 1040, 1023, 1065, 1010, 985, 984, 1047, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1019, 996, 1007, 1074, 1001, 995, 1047, 5, 0, 0, 0, 0, 0, 0, 0, 1000, 1000, 998, 1011, 1045, 1007, 1032, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1050, 1071, 993, 1048, 1031, 1023, 1012, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8173, 8177, 8172, 8230, 8138, 8117, 8251] avg=8192.0000 max=8278 max_rank=0 min=8117 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8320, 8192, 8320, 8192, 8192, 8064, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8064, 8320, 8064, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8448, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8064, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1050, 1041, 1058, 1036, 1012, 1004, 1062, 1040, 983, 1010, 1004, 1039, 1010, 1036, 932, 989, 1004, 1026, 987, 1005, 985, 998, 995, 1044, 984, 1029, 1012, 1006, 1002, 1041, 1005, 1106, 1043, 1046, 988, 1077, 1045, 1002, 1006, 1045, 1084, 999, 1019, 1047, 1094, 1008, 1012, 1007, 1029, 1035, 1052, 1053, 1063, 1043, 1029, 1018, 1072, 982, 1012, 1031, 999, 995, 1048, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1002, 923, 1042, 1010, 993, 1012, 1046, 1020, 1054, 985, 1034, 1024, 1035, 1020, 1033, 1032, 1034, 1015, 1033, 1128, 1054, 1071, 970, 1024, 1013, 999, 1061, 956, 1031, 1017, 1019, 1004, 1031, 1080, 1046, 1038, 978, 1043, 994, 1044, 1037, 1018, 1012, 994, 975, 1014, 1004, 1049, 1037, 1015, 1089, 1074, 992, 1056, 1031, 1037, 998, 1016, 1028, 1042, 988, 1044, 1016, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[994, 1050, 995, 979, 1044, 1065, 1012, 1002, 1018, 1012, 1094, 1006, 1039, 1036, 1044, 1053, 1010, 1079, 981, 1058, 1015, 1037, 1000, 965, 1031, 1007, 1009, 1022, 1014, 1034, 995, 1027, 1031, 1043, 1030, 1039, 1018, 1067, 1005, 1013, 1027, 1084, 1000, 1008, 1001, 985, 1077, 971, 1037, 980, 1040, 1008, 994, 998, 997, 1029, 1068, 1082, 1045, 1009, 1037, 1041, 1027, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1025, 1043, 987, 1064, 1010, 1037, 982, 1038, 986, 984, 1019, 1053, 1028, 1021, 1021, 1043, 1058, 1040, 1065, 1066, 1046, 997, 1016, 1017, 1009, 998, 1018, 1028, 990, 1033, 1046, 1009, 1006, 1004, 973, 998, 1052, 1007, 1018, 985, 1074, 1055, 1026, 995, 1020, 1073, 1006, 1018, 1004, 1023, 1077, 980, 1098, 1021, 1022, 1002, 1079, 1007, 1051, 1062, 1020, 1000, 963, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[998, 998, 1010, 1049, 1056, 984, 999, 979, 1079, 1034, 1038, 999, 1021, 1023, 1017, 1009, 989, 1032, 1004, 1057, 1008, 1014, 1009, 1002, 1014, 1004, 1043, 1019, 1036, 1054, 1024, 1055, 1016, 1039, 989, 1007, 1058, 1020, 1046, 988, 1038, 1038, 1018, 1065, 999, 1061, 1012, 984, 1032, 1012, 1056, 1004, 1032, 1023, 1066, 993, 1050, 1069, 1056, 1057, 1018, 988, 1046, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1019, 1030, 1056, 1030, 1032, 1054, 1046, 1010, 1072, 1027, 1026, 1041, 1038, 1031, 986, 1038, 1057, 1034, 1004, 1016, 1013, 1046, 1007, 1005, 1028, 987, 1019, 1043, 1019, 1031, 983, 1071, 1016, 1037, 954, 1019, 994, 987, 1001, 1058, 1000, 1001, 1039, 1028, 1008, 1012, 1035, 1031, 1018, 1068, 1038, 1001, 1041, 1027, 1031, 1031, 1054, 1032, 995, 986, 1059, 994, 1025, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1073, 980, 1001, 1035, 978, 1001, 1023, 1018, 1012, 1035, 1001, 986, 1019, 1048, 1030, 1013, 1048, 1027, 989, 1024, 1046, 1043, 1058, 1006, 1018, 1031, 1002, 1039, 1005, 1029, 990, 1000, 1013, 1039, 1018, 1019, 1004, 1026, 991, 1017, 1009, 1004, 1072, 1049, 1039, 1053, 1027, 990, 1057, 1030, 1044, 1037, 991, 1081, 1001, 1031, 1017, 1015, 1041, 1063, 1033, 1051, 990, 1076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1040, 1027, 1051, 1008, 1012, 1005, 1027, 1072, 1018, 1011, 1014, 993, 1017, 1016, 1013, 1059, 1025, 1023, 1046, 992, 1005, 1055, 1051, 1055, 1031, 977, 1019, 958, 1064, 1011, 1066, 1021, 1037, 1011, 1003, 998, 994, 1018, 1020, 1075, 995, 1084, 1007, 1092, 975, 1043, 1060, 971, 1026, 1020, 1068, 1024, 980, 977, 1023, 1011, 1067, 992, 1037, 1015, 1081, 1043, 1012, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[994, 1050, 995, 979, 1044, 1065, 1012, 1002, 10, 10, 0, 0, 0, 0, 0, 0, 1018, 1012, 1094, 1006, 1039, 1036, 1044, 1053, 9, 2, 0, 0, 0, 0, 0, 0, 1010, 1079, 981, 1049, 1015, 1037, 1000, 965, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1007, 1009, 1022, 1014, 1034, 995, 1027, 35, 0, 0, 0, 0, 0, 0, 0, 1031, 1043, 1030, 1039, 1018, 1067, 1005, 1013, 27, 1, 0, 0, 0, 0, 0, 0, 1027, 1072, 1000, 1008, 1001, 985, 1077, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 980, 1004, 1008, 994, 998, 997, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1082, 1045, 1009, 1037, 1041, 1027, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8313, 8136, 8174, 8274, 8141, 8047, 8290] avg=8192.0000 max=8313 max_rank=1 min=8047 min_rank=6 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1002, 923, 1042, 1010, 993, 1012, 1046, 1020, 9, 9, 0, 0, 0, 0, 0, 0, 1054, 985, 1034, 1024, 1035, 1020, 1033, 1032, 10, 2, 0, 0, 0, 0, 0, 0, 1034, 1015, 1033, 1118, 1054, 1071, 970, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 999, 1061, 956, 1031, 1017, 1019, 1004, 37, 0, 0, 0, 0, 0, 0, 0, 1031, 1080, 1046, 1038, 978, 1043, 994, 1044, 25, 1, 0, 0, 0, 0, 0, 0, 1037, 1007, 1012, 994, 975, 1014, 1004, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1015, 1051, 1074, 992, 1056, 1031, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 964, 1016, 1028, 1042, 988, 1044, 1016, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8066, 8229, 8319, 8137, 8280, 8092, 8293, 8120] avg=8192.0000 max=8319 max_rank=2 min=8066 min_rank=0 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1050, 1041, 1058, 1036, 1012, 1004, 1062, 1040, 10, 9, 0, 0, 0, 0, 0, 0, 983, 1010, 1004, 1039, 1010, 1036, 932, 989, 9, 2, 0, 0, 0, 0, 0, 0, 1004, 1026, 987, 996, 985, 998, 995, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1029, 1012, 1006, 1002, 1041, 1005, 1106, 36, 0, 0, 0, 0, 0, 0, 0, 1043, 1046, 988, 1077, 1045, 1002, 1006, 1045, 27, 1, 0, 0, 0, 0, 0, 0, 1084, 988, 1019, 1047, 1094, 1008, 1012, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1035, 1015, 1053, 1063, 1043, 1029, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 982, 1012, 1031, 999, 995, 1048, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8322, 8014, 8035, 8221, 8280, 8259, 8285, 8120] avg=8192.0000 max=8322 max_rank=0 min=8014 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32944 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[998, 998, 1010, 1049, 1056, 984, 999, 979, 9, 9, 0, 0, 0, 0, 0, 0, 1079, 1034, 1038, 999, 1021, 1023, 1017, 1009, 9, 2, 0, 0, 0, 0, 0, 0, 989, 1032, 1004, 1048, 1008, 1014, 1009, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1004, 1043, 1019, 1036, 1054, 1024, 1055, 36, 0, 0, 0, 0, 0, 0, 0, 1016, 1039, 989, 1007, 1058, 1020, 1046, 988, 27, 1, 0, 0, 0, 0, 0, 0, 1038, 1027, 1018, 1065, 999, 1061, 1012, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1012, 1019, 1004, 1032, 1023, 1066, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1069, 1056, 1057, 1018, 988, 1046, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8091, 8231, 8106, 8285, 8191, 8204, 8181, 8247] avg=8192.0000 max=8285 max_rank=3 min=8091 min_rank=0 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1025, 1043, 987, 1064, 1010, 1037, 982, 1038, 10, 9, 0, 0, 0, 0, 0, 0, 986, 984, 1019, 1053, 1028, 1021, 1021, 1043, 9, 2, 0, 0, 0, 0, 0, 0, 1058, 1040, 1065, 1057, 1046, 997, 1016, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 998, 1018, 1028, 990, 1033, 1046, 1009, 36, 0, 0, 0, 0, 0, 0, 0, 1006, 1004, 973, 998, 1052, 1007, 1018, 985, 27, 1, 0, 0, 0, 0, 0, 0, 1074, 1044, 1026, 995, 1020, 1073, 1006, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1023, 1040, 980, 1098, 1021, 1022, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1007, 1051, 1062, 1020, 1000, 963, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8166, 8296, 8167, 8071, 8256, 8190, 8185] avg=8192.0000 max=8296 max_rank=2 min=8071 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1019, 1030, 1056, 1030, 1032, 1054, 1046, 1010, 9, 9, 0, 0, 0, 0, 0, 0, 1072, 1027, 1026, 1041, 1038, 1031, 986, 1038, 9, 2, 0, 0, 0, 0, 0, 0, 1057, 1034, 1004, 1007, 1013, 1046, 1007, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 987, 1019, 1043, 1019, 1031, 983, 1071, 35, 0, 0, 0, 0, 0, 0, 0, 1016, 1037, 954, 1019, 994, 987, 1001, 1058, 27, 1, 0, 0, 0, 0, 0, 0, 1000, 990, 1039, 1028, 1008, 1012, 1035, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1068, 1002, 1001, 1041, 1027, 1031, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1032, 995, 986, 1059, 994, 1025, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8270, 8173, 8216, 8094, 8143, 8219, 8126] avg=8192.0000 max=8295 max_rank=0 min=8094 min_rank=4 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1073, 980, 1001, 1035, 978, 1001, 1023, 1018, 12, 13, 0, 0, 0, 0, 0, 0, 1012, 1035, 1001, 986, 1019, 1048, 1030, 1013, 13, 4, 0, 0, 0, 0, 0, 0, 1048, 1027, 989, 1011, 1046, 1043, 1058, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1031, 1002, 1039, 1005, 1029, 990, 1000, 39, 0, 0, 0, 0, 0, 0, 0, 1013, 1039, 1018, 1019, 1004, 1026, 991, 1017, 31, 4, 0, 0, 0, 0, 0, 0, 1009, 987, 1072, 1049, 1039, 1053, 1027, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1030, 1001, 1037, 991, 1081, 1001, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1015, 1041, 1063, 1033, 1051, 990, 1076, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8161, 8228, 8153, 8162, 8226, 8229, 8243] avg=8192.0000 max=8243 max_rank=7 min=8134 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1040, 1027, 1051, 1008, 1012, 1005, 1027, 1072, 10, 10, 0, 0, 0, 0, 0, 0, 1018, 1011, 1014, 993, 1017, 1016, 1013, 1059, 9, 2, 0, 0, 0, 0, 0, 0, 1025, 1023, 1046, 983, 1005, 1055, 1051, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 977, 1019, 958, 1064, 1011, 1066, 1021, 36, 0, 0, 0, 0, 0, 0, 0, 1037, 1011, 1003, 998, 994, 1018, 1020, 1075, 27, 1, 0, 0, 0, 0, 0, 0, 995, 1072, 1007, 1092, 975, 1043, 1060, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1020, 1031, 1024, 980, 977, 1023, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 992, 1037, 1015, 1081, 1043, 1012, 995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8152, 8243, 8183, 8184, 8215, 8092, 8205] avg=8192.0000 max=8262 max_rank=0 min=8092 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8192, 8192, 8320, 8192, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1054, 1071, 1060, 1028, 1046, 1027, 989, 977, 1037, 1021, 1028, 1012, 1009, 1044, 1018, 1036, 992, 1018, 1052, 1004, 1073, 1049, 1007, 1039, 1046, 1021, 1021, 1041, 1045, 983, 1024, 987, 1073, 999, 1033, 988, 953, 1034, 1012, 1077, 1034, 1026, 990, 1021, 979, 1039, 1024, 1009, 1022, 1027, 1035, 1041, 1018, 962, 1006, 1065, 1039, 1041, 1039, 1020, 1067, 959, 1029, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1034, 1039, 1027, 1013, 983, 1021, 1021, 1006, 1013, 986, 1056, 995, 1018, 1044, 1010, 1046, 1036, 1025, 1036, 1017, 1059, 923, 1023, 1052, 1036, 1065, 1030, 1033, 1021, 1040, 1050, 1036, 982, 1017, 1022, 1072, 995, 1006, 984, 974, 1005, 972, 1038, 1042, 1059, 993, 1036, 1045, 1014, 1052, 1066, 1050, 1036, 1048, 1002, 1011, 1004, 1044, 1026, 1020, 1034, 1041, 1085, 967] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1047, 1030, 986, 1037, 993, 1046, 1032, 1065, 1014, 1052, 1024, 996, 957, 1001, 1037, 1040, 997, 1026, 1049, 1075, 1037, 1055, 1046, 1031, 1020, 1024, 1043, 1023, 1054, 1058, 1053, 976, 964, 1001, 992, 1006, 1048, 1012, 1026, 1061, 1020, 966, 995, 1065, 1065, 984, 1038, 1000, 1055, 997, 1025, 1028, 1003, 1077, 1020, 1099, 983, 1008, 1048, 999, 991, 1047, 1025, 964] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1002, 1039, 1007, 1006, 992, 1046, 1031, 1085, 1048, 1014, 1034, 1079, 1042, 1048, 1004, 1043, 1000, 983, 974, 984, 1012, 976, 1011, 1005, 1025, 1064, 1068, 1062, 1013, 1000, 1093, 1043, 1029, 1002, 1005, 1014, 1052, 1057, 1016, 1043, 1004, 1012, 1045, 1023, 1034, 981, 979, 1043, 978, 1031, 1019, 1033, 993, 1023, 1010, 999, 1022, 1062, 1014, 1036, 1037, 1060, 1017, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1036, 1076, 1032, 1009, 1025, 1049, 1011, 1019, 1027, 1017, 1043, 1000, 1004, 1028, 993, 984, 1033, 1006, 973, 976, 988, 1043, 1010, 1025, 1028, 985, 1002, 1089, 1058, 1049, 1005, 1050, 983, 1016, 979, 1029, 1065, 1032, 1092, 1018, 1019, 1076, 967, 1014, 1008, 1020, 966, 1028, 1057, 964, 1014, 1005, 1067, 1040, 1005, 1055, 1074, 1048, 1033, 1049, 1047, 1023, 1032, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1013, 1046, 1014, 1037, 993, 1074, 1069, 1032, 1003, 1034, 1005, 993, 1042, 976, 1030, 1028, 1034, 1031, 1026, 1021, 989, 1061, 1012, 1049, 1035, 952, 1038, 1038, 1018, 1043, 1041, 987, 991, 1036, 1066, 1018, 1035, 989, 1029, 1024, 1046, 986, 1054, 1057, 973, 1008, 1051, 1011, 993, 1003, 1045, 1046, 1064, 1069, 1084, 977, 978, 1029, 987, 1027, 1029, 1010, 1030, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1007, 1019, 966, 998, 1050, 1060, 1056, 1085, 978, 1012, 1003, 1027, 1099, 964, 974, 1019, 1029, 1033, 1022, 1040, 1004, 1038, 992, 1045, 1052, 1025, 1013, 1056, 1053, 1035, 1016, 1058, 1027, 1011, 1014, 1033, 1078, 1055, 1074, 1029, 1046, 1016, 1022, 1012, 969, 1023, 1043, 1025, 1016, 953, 1036, 986, 1059, 991, 1008, 979, 971, 1100, 1010, 1044, 1059, 1012, 989, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1008, 982, 1037, 1079, 967, 1065, 1005, 1025, 1040, 1037, 1021, 1021, 1067, 1050, 988, 986, 1033, 1033, 990, 1013, 1035, 1051, 972, 1053, 1053, 987, 1000, 1012, 1017, 1059, 989, 1011, 1029, 1014, 1054, 1053, 979, 1052, 1014, 1069, 1034, 993, 1040, 1060, 968, 1016, 1046, 1026, 1024, 999, 1043, 1051, 1034, 1015, 1030, 1089, 989, 1006, 1044, 1022, 1032, 994, 1011, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1054, 1071, 1060, 1028, 1046, 998, 989, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1021, 1028, 1012, 1009, 1044, 1018, 1036, 20, 8, 0, 0, 0, 0, 0, 0, 992, 1018, 1052, 1004, 1073, 1049, 1007, 1039, 13, 9, 2, 0, 0, 0, 0, 0, 1046, 1021, 1021, 990, 1045, 983, 1024, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 999, 1033, 988, 953, 1034, 1012, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1026, 990, 1021, 979, 1039, 1024, 1009, 43, 0, 0, 0, 0, 0, 0, 0, 1022, 1027, 1022, 1041, 1018, 962, 1006, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1039, 1039, 1020, 1067, 959, 1029, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8233, 8258, 8117, 8169, 8165, 8163, 8208] avg=8192.0000 max=8258 max_rank=2 min=8117 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32705 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1013, 1046, 1014, 1037, 993, 1043, 1069, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1034, 1005, 993, 1042, 976, 1030, 1028, 21, 8, 0, 0, 0, 0, 0, 0, 1034, 1031, 1026, 1021, 989, 1061, 1012, 1049, 13, 10, 2, 0, 0, 0, 0, 0, 1035, 952, 1038, 987, 1018, 1043, 1041, 987, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1036, 1066, 1018, 1035, 989, 1029, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 986, 1054, 1057, 973, 1008, 1051, 1011, 43, 0, 0, 0, 0, 0, 0, 0, 993, 1003, 1032, 1046, 1064, 1069, 1084, 977, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1027, 987, 1027, 1029, 1010, 1030, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8140, 8248, 8101, 8188, 8229, 8268, 8115] avg=8192.0000 max=8268 max_rank=6 min=8101 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32800 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1047, 1030, 986, 1037, 993, 1017, 1032, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1052, 1024, 996, 957, 1001, 1037, 1040, 20, 8, 0, 0, 0, 0, 0, 0, 997, 1026, 1049, 1075, 1037, 1055, 1046, 1031, 13, 9, 2, 0, 0, 0, 0, 0, 1020, 1024, 1043, 973, 1054, 1058, 1053, 976, 0, 0, 0, 0, 0, 0, 0, 0, 964, 1001, 992, 1006, 1048, 1012, 1026, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 966, 995, 1065, 1065, 984, 1038, 1000, 42, 0, 0, 0, 0, 0, 0, 0, 1055, 997, 1012, 1028, 1003, 1077, 1020, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1006, 1048, 999, 991, 1047, 1025, 964, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8149, 8340, 8201, 8110, 8175, 8291, 8063] avg=8192.0000 max=8340 max_rank=2 min=8063 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32639 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1034, 1039, 1027, 1013, 983, 992, 1021, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 986, 1056, 995, 1018, 1044, 1010, 1046, 20, 8, 0, 0, 0, 0, 0, 0, 1036, 1025, 1036, 1017, 1059, 923, 1023, 1052, 14, 9, 2, 0, 0, 0, 0, 0, 1036, 1065, 1030, 983, 1021, 1040, 1050, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1017, 1022, 1072, 995, 1006, 984, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 972, 1038, 1042, 1059, 993, 1036, 1045, 42, 0, 0, 0, 0, 0, 0, 0, 1014, 1052, 1052, 1050, 1036, 1048, 1002, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1042, 1026, 1020, 1034, 1041, 1085, 967, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8196, 8196, 8261, 8052, 8232, 8265, 8219] avg=8192.0000 max=8265 max_rank=6 min=8052 min_rank=4 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32768 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1002, 1039, 1007, 1006, 992, 1017, 1031, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1014, 1034, 1079, 1042, 1048, 1004, 1043, 20, 8, 0, 0, 0, 0, 0, 0, 1000, 983, 974, 984, 1012, 976, 1011, 1005, 13, 9, 2, 0, 0, 0, 0, 0, 1025, 1064, 1068, 1010, 1013, 1000, 1093, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1002, 1005, 1014, 1052, 1057, 1016, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1012, 1045, 1023, 1034, 981, 979, 1043, 44, 0, 0, 0, 0, 0, 0, 0, 978, 1031, 1006, 1033, 993, 1023, 1010, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1060, 1014, 1036, 1037, 1060, 1017, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8340, 7969, 8316, 8218, 8165, 8073, 8276] avg=8192.0000 max=8340 max_rank=1 min=7969 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32732 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1007, 1019, 966, 998, 1050, 1029, 1056, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1012, 1003, 1027, 1099, 964, 974, 1019, 21, 8, 0, 0, 0, 0, 0, 0, 1029, 1033, 1022, 1040, 1004, 1038, 992, 1045, 13, 10, 2, 0, 0, 0, 0, 0, 1052, 1025, 1013, 1005, 1053, 1035, 1016, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1011, 1014, 1033, 1078, 1055, 1074, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1016, 1022, 1012, 969, 1023, 1043, 1025, 43, 0, 0, 0, 0, 0, 0, 0, 1016, 953, 1023, 986, 1059, 991, 1008, 979, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1098, 1010, 1044, 1059, 1012, 989, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8105, 8228, 8257, 8321, 8199, 8015, 8201] avg=8192.0000 max=8321 max_rank=4 min=8015 min_rank=6 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32736 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1036, 1076, 1032, 1009, 1025, 1010, 1011, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1017, 1043, 1000, 1004, 1028, 993, 984, 25, 13, 0, 0, 0, 0, 0, 0, 1033, 1006, 973, 976, 988, 1043, 1010, 1025, 18, 14, 6, 0, 0, 0, 0, 0, 1028, 985, 1002, 1028, 1058, 1049, 1005, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1016, 979, 1029, 1065, 1032, 1092, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1076, 967, 1014, 1008, 1020, 966, 1028, 48, 0, 0, 0, 0, 0, 0, 0, 1057, 964, 996, 1005, 1067, 1040, 1005, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1042, 1033, 1049, 1047, 1023, 1032, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8134, 8092, 8205, 8214, 8146, 8189, 8338] avg=8192.0000 max=8338 max_rank=7 min=8092 min_rank=2 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32887 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1008, 982, 1037, 1079, 967, 1034, 1005, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1037, 1021, 1021, 1067, 1050, 988, 986, 21, 8, 0, 0, 0, 0, 0, 0, 1033, 1033, 990, 1013, 1035, 1051, 972, 1053, 13, 10, 2, 0, 0, 0, 0, 0, 1053, 987, 1000, 962, 1017, 1059, 989, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1014, 1054, 1053, 979, 1052, 1014, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 993, 1040, 1060, 968, 1016, 1046, 1026, 42, 0, 0, 0, 0, 0, 0, 0, 1024, 999, 1030, 1051, 1034, 1015, 1030, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1004, 1044, 1022, 1032, 994, 1011, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8239, 8205, 8078, 8264, 8225, 8272, 8116] avg=8192.0000 max=8272 max_rank=6 min=8078 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32877 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8064, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8064, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8064, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8064, 8192, 8320, 8064, 8064, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1071, 1026, 1029, 1019, 1011, 1036, 1042, 1054, 1056, 1041, 1021, 989, 1001, 1047, 1094, 1010, 1045, 1015, 1046, 1020, 997, 1035, 972, 1010, 1013, 1024, 1020, 1028, 1001, 1034, 969, 1021, 1009, 999, 1040, 997, 1000, 1023, 969, 1022, 1010, 1029, 1019, 1031, 1049, 1007, 1054, 1038, 1045, 999, 1074, 1008, 1045, 1036, 1048, 1038, 989, 1057, 1031, 1036, 1014, 993, 1010, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[981, 1012, 986, 1067, 1030, 1014, 1020, 1072, 1023, 1033, 1016, 1006, 1024, 1010, 1034, 981, 989, 1062, 1018, 1043, 956, 1072, 997, 972, 1032, 1048, 1005, 1034, 1057, 1020, 1011, 1032, 1025, 1024, 1049, 1076, 1028, 953, 1018, 1038, 1004, 1061, 1058, 1060, 1009, 1040, 1000, 1054, 1043, 1054, 986, 993, 1017, 1067, 1021, 1063, 1009, 1090, 982, 996, 978, 1052, 1006, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1010, 1033, 1063, 993, 1045, 1017, 1079, 1017, 1006, 1031, 993, 983, 1030, 1015, 1027, 1033, 1048, 1013, 980, 1070, 1012, 1019, 1017, 1038, 1044, 1071, 1014, 1012, 1066, 1030, 1019, 1066, 1019, 1006, 1034, 1033, 1059, 1038, 1066, 1047, 1048, 982, 1000, 1044, 1015, 1020, 972, 993, 1053, 1017, 998, 999, 993, 1016, 1014, 994, 1037, 1036, 1073, 1035, 1018, 992, 972, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1044, 991, 1069, 1005, 998, 996, 1010, 1044, 1050, 1004, 1032, 982, 1030, 995, 1040, 996, 1089, 1053, 1003, 971, 1073, 1019, 1013, 1011, 1051, 1022, 1018, 1027, 1028, 1028, 1020, 1069, 972, 1009, 1022, 1042, 1031, 1016, 1017, 1032, 1025, 1054, 999, 1026, 1025, 1013, 1007, 999, 1039, 1037, 1033, 1042, 1081, 1015, 1005, 980, 1073, 1026, 1048, 998, 1040, 1016, 1041, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1048, 1017, 1083, 1046, 971, 995, 997, 979, 994, 1040, 1007, 1034, 1002, 1032, 994, 1027, 1013, 1072, 1019, 994, 1030, 995, 1087, 1033, 1012, 1033, 1026, 1015, 995, 979, 1031, 1009, 1106, 1045, 1031, 975, 1003, 1033, 1070, 1070, 1042, 1027, 1055, 1009, 979, 1064, 1040, 1039, 993, 1049, 1025, 1033, 1007, 1038, 1046, 1049, 1012, 1028, 984, 1029, 1058, 1029, 1006, 953] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1041, 1030, 1016, 1013, 976, 1017, 992, 1020, 1024, 1027, 981, 1040, 1029, 1028, 998, 1024, 1000, 981, 964, 1082, 1013, 1055, 992, 1058, 1044, 1066, 1062, 1026, 1046, 1037, 1013, 1052, 1006, 1069, 1013, 1057, 1077, 1033, 1005, 1002, 1035, 994, 1102, 1005, 1000, 1050, 1011, 1050, 1015, 981, 988, 1014, 1029, 1088, 1022, 990, 966, 1000, 1055, 1022, 1054, 1071, 1002, 983] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[970, 1030, 960, 1004, 1040, 1045, 1031, 1053, 1018, 1057, 1009, 1016, 1001, 1054, 1042, 1000, 1043, 1018, 1027, 1019, 1015, 1019, 1018, 976, 1012, 1027, 1063, 1032, 973, 1000, 1040, 1078, 1026, 1005, 1062, 1032, 1043, 1059, 1045, 1017, 1020, 1009, 1055, 1037, 1047, 959, 1047, 1003, 1016, 1014, 1017, 1003, 1058, 1014, 1053, 1026, 1024, 1015, 999, 1019, 1019, 1026, 1079, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[990, 994, 997, 973, 1023, 985, 983, 1006, 989, 1061, 981, 1037, 989, 1026, 997, 986, 1012, 1081, 999, 962, 1124, 1028, 1060, 979, 1071, 1053, 1092, 1054, 1027, 1045, 1031, 1052, 1030, 1014, 1038, 1069, 975, 1048, 1083, 1000, 1056, 1026, 1003, 1115, 966, 1019, 959, 1067, 1038, 1006, 1057, 991, 1009, 1072, 1002, 1044, 1020, 1011, 982, 1051, 1006, 1051, 1010, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1071, 1026, 1029, 1019, 1011, 1036, 1042, 1054, 24, 16, 0, 0, 0, 0, 0, 0, 1056, 1041, 1021, 989, 1001, 1047, 1094, 1010, 43, 0, 0, 0, 0, 0, 0, 0, 1045, 1015, 1046, 1020, 997, 1035, 972, 1010, 11, 0, 0, 0, 0, 0, 0, 0, 1013, 1024, 1020, 1028, 1001, 1034, 969, 962, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 999, 1001, 997, 1000, 1023, 969, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1029, 1019, 1019, 1049, 1007, 1054, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 999, 1074, 1008, 1045, 1024, 1048, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1057, 1031, 1036, 1014, 993, 1010, 1020, 15, 12, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8328, 8302, 8151, 8051, 8020, 8225, 8281, 8178] avg=8192.0000 max=8328 max_rank=0 min=8020 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1041, 1030, 1016, 1013, 976, 1017, 992, 1020, 23, 16, 0, 0, 0, 0, 0, 0, 1024, 1027, 981, 1040, 1029, 1028, 998, 1024, 45, 0, 0, 0, 0, 0, 0, 0, 1000, 981, 964, 1082, 1013, 1055, 992, 1058, 10, 0, 0, 0, 0, 0, 0, 0, 1044, 1066, 1062, 1026, 1046, 1037, 1013, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1069, 975, 1057, 1077, 1033, 1005, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 994, 1102, 994, 1000, 1050, 1011, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 981, 988, 1014, 1029, 1075, 1022, 990, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1000, 1055, 1022, 1054, 1071, 1002, 983, 15, 13, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8196, 8155, 8285, 8224, 8236, 8114, 8182] avg=8192.0000 max=8285 max_rank=3 min=8114 min_rank=6 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1010, 1033, 1063, 993, 1045, 1017, 1079, 1017, 23, 17, 0, 0, 0, 0, 0, 0, 1006, 1031, 993, 983, 1030, 1015, 1027, 1033, 45, 0, 0, 0, 0, 0, 0, 0, 1048, 1013, 980, 1070, 1012, 1019, 1017, 1038, 11, 0, 0, 0, 0, 0, 0, 0, 1044, 1071, 1014, 1012, 1066, 1030, 1019, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1006, 996, 1033, 1059, 1038, 1066, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 982, 1000, 1032, 1015, 1020, 972, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1017, 998, 999, 993, 1004, 1014, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1036, 1073, 1035, 1018, 992, 972, 1019, 15, 12, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8163, 8208, 8260, 8264, 8062, 8072, 8210] avg=8192.0000 max=8297 max_rank=0 min=8062 min_rank=5 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[981, 1012, 986, 1067, 1030, 1014, 1020, 1072, 24, 16, 0, 0, 0, 0, 0, 0, 1023, 1033, 1016, 1006, 1024, 1010, 1034, 981, 44, 0, 0, 0, 0, 0, 0, 0, 989, 1062, 1018, 1043, 956, 1072, 997, 972, 11, 0, 0, 0, 0, 0, 0, 0, 1032, 1048, 1005, 1034, 1057, 1020, 1011, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1024, 1010, 1076, 1028, 953, 1018, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1061, 1058, 1048, 1009, 1040, 1000, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1054, 986, 993, 1017, 1054, 1021, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1090, 982, 996, 978, 1052, 1006, 1025, 15, 13, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8171, 8120, 8179, 8172, 8274, 8231, 8167] avg=8192.0000 max=8274 max_rank=5 min=8120 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1048, 1017, 1083, 1046, 971, 995, 997, 979, 23, 16, 0, 0, 0, 0, 0, 0, 994, 1040, 1007, 1034, 1002, 1032, 994, 1027, 43, 0, 0, 0, 0, 0, 0, 0, 1013, 1072, 1019, 994, 1030, 995, 1087, 1033, 10, 0, 0, 0, 0, 0, 0, 0, 1012, 1033, 1026, 1015, 995, 979, 1031, 950, 0, 0, 0, 0, 0, 0, 0, 0, 1106, 1045, 993, 975, 1003, 1033, 1070, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1027, 1055, 998, 979, 1064, 1040, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1049, 1025, 1033, 1007, 1026, 1046, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1028, 984, 1029, 1058, 1029, 1006, 953, 15, 12, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8173, 8253, 8041, 8295, 8244, 8228, 8127] avg=8192.0000 max=8295 max_rank=4 min=8041 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32894 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1044, 991, 1069, 1005, 998, 996, 1010, 1044, 23, 17, 0, 0, 0, 0, 0, 0, 1050, 1004, 1032, 982, 1030, 995, 1040, 996, 45, 0, 0, 0, 0, 0, 0, 0, 1089, 1053, 1003, 971, 1073, 1019, 1013, 1011, 11, 0, 0, 0, 0, 0, 0, 0, 1051, 1022, 1018, 1027, 1028, 1028, 1020, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1009, 984, 1042, 1031, 1016, 1017, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1054, 999, 1014, 1025, 1013, 1007, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1037, 1033, 1042, 1081, 1003, 1005, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1026, 1048, 998, 1040, 1016, 1041, 992, 15, 12, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8174, 8243, 8201, 8103, 8136, 8220, 8262] avg=8192.0000 max=8262 max_rank=7 min=8103 min_rank=4 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[970, 1030, 960, 1004, 1040, 1045, 1031, 1053, 24, 17, 0, 0, 0, 0, 0, 0, 1018, 1057, 1009, 1016, 1001, 1054, 1042, 1000, 46, 0, 0, 0, 0, 0, 0, 0, 1043, 1018, 1027, 1019, 1015, 1019, 1018, 976, 11, 0, 0, 0, 0, 0, 0, 0, 1012, 1027, 1063, 1032, 973, 1000, 1040, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1005, 1022, 1032, 1043, 1059, 1045, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1009, 1055, 1025, 1047, 959, 1047, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1014, 1017, 1003, 1058, 1002, 1053, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1015, 999, 1019, 1019, 1026, 1079, 998, 16, 12, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8243, 8146, 8162, 8249, 8165, 8189, 8208] avg=8192.0000 max=8249 max_rank=4 min=8146 min_rank=2 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[990, 994, 997, 973, 1023, 985, 983, 1006, 28, 20, 0, 0, 0, 0, 0, 0, 989, 1061, 981, 1037, 989, 1026, 997, 986, 48, 0, 0, 0, 0, 0, 0, 0, 1012, 1081, 999, 962, 1124, 1028, 1060, 979, 15, 0, 0, 0, 0, 0, 0, 0, 1071, 1053, 1092, 1054, 1027, 1045, 1031, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1014, 990, 1069, 975, 1048, 1083, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1026, 1003, 1098, 966, 1019, 959, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1006, 1057, 991, 1009, 1054, 1002, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1011, 982, 1051, 1006, 1051, 1010, 1031, 20, 18, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7999, 8114, 8260, 8357, 8209, 8194, 8201, 8202] avg=8192.0000 max=8357 max_rank=3 min=7999 min_rank=0 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8192, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8192, 8064, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8448, 8320, 8320, 8320, 8192, 8192, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8192, 8192, 8320, 8320, 8064, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8064, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[990, 1009, 1044, 1025, 1023, 991, 1038, 1064, 987, 1020, 1015, 1027, 1032, 1041, 1012, 1009, 1032, 992, 1064, 1048, 1077, 976, 1003, 978, 1053, 1095, 1042, 1045, 1031, 1027, 1030, 1041, 1014, 1041, 1002, 1031, 1018, 985, 993, 1011, 1067, 1021, 1043, 972, 1028, 1087, 1016, 1034, 1021, 997, 1003, 1019, 1002, 1030, 1019, 1018, 1083, 1048, 991, 1025, 1027, 996, 995, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1063, 1062, 1008, 995, 1032, 1002, 1021, 1010, 1054, 1054, 998, 1031, 1023, 1073, 965, 1019, 945, 1043, 1005, 1054, 1004, 1003, 1000, 1054, 1025, 1039, 974, 1030, 1031, 1030, 1034, 1036, 984, 983, 1013, 1042, 984, 1043, 1047, 1025, 1073, 1063, 1064, 1015, 1011, 1035, 1054, 1004, 1023, 1006, 1004, 991, 1064, 1027, 1027, 991, 1037, 1091, 1001, 993, 1048, 1022, 1027, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1011, 1060, 1046, 982, 998, 1020, 1035, 1021, 962, 1006, 1019, 1061, 1035, 1039, 1008, 983, 1005, 1018, 992, 1024, 1003, 1069, 996, 1050, 1056, 948, 1011, 1059, 1017, 1048, 1020, 1007, 1002, 1039, 1084, 998, 1017, 996, 1001, 1019, 1044, 1040, 1038, 1034, 985, 1015, 1013, 1070, 1019, 1052, 1048, 1045, 957, 1050, 1070, 1055, 1062, 1045, 1032, 1035, 992, 1041, 1047, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1034, 1025, 1023, 982, 1020, 1021, 1029, 1040, 1002, 1041, 1007, 1049, 1101, 1027, 1044, 1016, 972, 1033, 1013, 1024, 1045, 1004, 1019, 1038, 1038, 1002, 1001, 1014, 979, 1065, 1079, 1036, 1007, 1059, 1104, 1058, 998, 1039, 996, 1074, 1033, 993, 1071, 1003, 1022, 1002, 1032, 987, 992, 992, 1036, 1001, 1034, 1038, 1015, 999, 1057, 937, 1054, 1028, 1005, 1038, 1009, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1040, 1070, 1042, 1031, 1035, 1010, 991, 1028, 1061, 1043, 1025, 1022, 1044, 1047, 990, 1005, 1016, 1027, 1039, 1023, 1044, 998, 992, 980, 1020, 1039, 1081, 1017, 1039, 1024, 1008, 1006, 1035, 1033, 1037, 1011, 1069, 1019, 1035, 1010, 999, 1018, 1037, 1041, 1014, 1015, 964, 981, 1035, 1048, 1032, 1072, 970, 1021, 1018, 1012, 992, 1037, 1001, 1045, 1008, 1001, 1035, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1025, 1038, 1050, 1029, 1015, 1056, 1066, 1008, 984, 1005, 985, 1037, 997, 1042, 997, 1029, 1022, 998, 1041, 1001, 1021, 1071, 982, 1047, 1015, 1029, 1032, 1056, 1021, 1026, 1041, 1041, 937, 1031, 1016, 1005, 1007, 1026, 1021, 957, 977, 988, 1080, 993, 1037, 1057, 1055, 1028, 1043, 1062, 1052, 974, 1043, 1039, 978, 1033, 1059, 1021, 1043, 1026, 1015, 1024, 1043, 1059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1055, 977, 1027, 1001, 1025, 1052, 1009, 1037, 1017, 1014, 1022, 1027, 1036, 1019, 1034, 1006, 1056, 1036, 1048, 1044, 1044, 990, 1047, 1007, 998, 1034, 1028, 1008, 981, 1015, 1020, 1036, 1002, 1050, 1012, 1049, 1041, 1021, 1016, 1009, 1019, 1002, 1035, 1021, 1022, 1075, 1012, 1023, 1037, 1039, 1039, 1027, 1056, 1043, 1028, 1027, 983, 984, 1055, 1023, 1026, 1021, 1013, 976] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1004, 1038, 1023, 1077, 1010, 963, 986, 1038, 1024, 977, 1018, 1032, 1028, 977, 988, 1058, 1009, 1013, 1090, 1083, 1054, 1025, 1058, 960, 1005, 1012, 1068, 1016, 1021, 1030, 1005, 1053, 1070, 1003, 1016, 1042, 1051, 1050, 1019, 1041, 1035, 1011, 981, 1061, 973, 1080, 1013, 1074, 1038, 1012, 1002, 1037, 1026, 1014, 1039, 940, 990, 1054, 1040, 1025, 1017, 1015, 1023, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1040, 1066, 1042, 1031, 1035, 1010, 991, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1043, 1025, 1022, 1044, 1047, 990, 1005, 23, 0, 0, 0, 0, 0, 0, 0, 1016, 1027, 1039, 1023, 1044, 998, 992, 980, 10, 0, 0, 0, 0, 0, 0, 0, 1020, 1039, 1081, 1017, 1039, 996, 1008, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1033, 1037, 1011, 1069, 1019, 1035, 1010, 5, 5, 0, 0, 0, 0, 0, 0, 999, 1018, 1037, 1041, 1014, 997, 964, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1048, 1032, 1072, 970, 1021, 1018, 1012, 3, 3, 0, 0, 0, 0, 0, 0, 992, 1037, 1001, 1045, 1008, 1001, 1035, 1054, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8260, 8129, 8206, 8259, 8051, 8214, 8174] avg=8192.0000 max=8260 max_rank=1 min=8051 min_rank=5 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[990, 1005, 1044, 1025, 1023, 991, 1038, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1020, 1015, 1027, 1032, 1041, 1012, 1009, 23, 0, 0, 0, 0, 0, 0, 0, 1032, 992, 1064, 1048, 1077, 976, 1003, 978, 11, 0, 0, 0, 0, 0, 0, 0, 1053, 1095, 1042, 1045, 1031, 999, 1030, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1041, 1002, 1031, 1018, 985, 993, 1011, 5, 5, 0, 0, 0, 0, 0, 0, 1067, 1021, 1043, 972, 1028, 1068, 1016, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 997, 1003, 1019, 1002, 1030, 1019, 1018, 3, 3, 0, 0, 0, 0, 0, 0, 1083, 1048, 991, 1025, 1027, 996, 995, 1038, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8166, 8181, 8336, 8105, 8249, 8115, 8204] avg=8192.0000 max=8336 max_rank=3 min=8105 min_rank=4 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1011, 1056, 1046, 982, 998, 1020, 1035, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1006, 1019, 1061, 1035, 1039, 1008, 983, 23, 0, 0, 0, 0, 0, 0, 0, 1005, 1018, 992, 1024, 1003, 1069, 996, 1050, 10, 0, 0, 0, 0, 0, 0, 0, 1056, 948, 1011, 1059, 1017, 1020, 1020, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1039, 1084, 998, 1017, 996, 1001, 1019, 5, 5, 0, 0, 0, 0, 0, 0, 1044, 1040, 1038, 1034, 985, 997, 1013, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1052, 1048, 1045, 957, 1050, 1070, 1055, 3, 3, 0, 0, 0, 0, 0, 0, 1062, 1045, 1032, 1035, 992, 1041, 1047, 982, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8136, 8167, 8138, 8166, 8221, 8302, 8237] avg=8192.0000 max=8302 max_rank=6 min=8136 min_rank=1 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32926 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1063, 1058, 1008, 995, 1032, 1002, 1021, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1054, 998, 1031, 1023, 1073, 965, 1019, 23, 0, 0, 0, 0, 0, 0, 0, 945, 1043, 1005, 1054, 1004, 1003, 1000, 1054, 10, 0, 0, 0, 0, 0, 0, 0, 1025, 1039, 974, 1030, 1031, 1002, 1034, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 984, 983, 1013, 1042, 984, 1043, 1047, 1025, 5, 5, 0, 0, 0, 0, 0, 0, 1073, 1063, 1064, 1015, 1011, 1017, 1054, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1006, 1004, 991, 1064, 1027, 1027, 991, 3, 3, 0, 0, 0, 0, 0, 0, 1037, 1091, 1001, 993, 1048, 1022, 1027, 1027, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8240, 8118, 8171, 8131, 8301, 8139, 8247] avg=8192.0000 max=8301 max_rank=5 min=8118 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1055, 973, 1027, 1001, 1025, 1052, 1009, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1014, 1022, 1027, 1036, 1019, 1034, 1006, 22, 0, 0, 0, 0, 0, 0, 0, 1056, 1036, 1048, 1044, 1044, 990, 1047, 1007, 11, 0, 0, 0, 0, 0, 0, 0, 998, 1034, 1028, 1008, 981, 988, 1020, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1050, 1012, 1049, 1041, 1021, 1016, 1009, 5, 5, 0, 0, 0, 0, 0, 0, 1019, 1002, 1035, 1021, 1022, 1056, 1012, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1039, 1039, 1027, 1056, 1043, 1028, 1027, 3, 3, 0, 0, 0, 0, 0, 0, 983, 984, 1055, 1023, 1026, 1021, 1013, 976, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8197, 8283, 8093, 8210, 8190, 8302, 8082] avg=8192.0000 max=8302 max_rank=6 min=8082 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1034, 1021, 1023, 982, 1020, 1021, 1029, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1041, 1007, 1049, 1101, 1027, 1044, 1016, 23, 0, 0, 0, 0, 0, 0, 0, 972, 1033, 1013, 1024, 1045, 1004, 1019, 1038, 10, 0, 0, 0, 0, 0, 0, 0, 1038, 1002, 1001, 1014, 979, 1037, 1079, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1059, 1104, 1058, 998, 1039, 996, 1074, 5, 5, 0, 0, 0, 0, 0, 0, 1033, 993, 1071, 1003, 1022, 985, 1032, 987, 0, 0, 0, 0, 0, 0, 0, 0, 992, 992, 1036, 1001, 1034, 1038, 1015, 999, 3, 2, 0, 0, 0, 0, 0, 0, 1057, 937, 1054, 1028, 1005, 1038, 1009, 1000, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8310, 8158, 8186, 8345, 8126, 8112, 8129] avg=8192.0000 max=8345 max_rank=4 min=8112 min_rank=6 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1025, 1034, 1050, 1029, 1015, 1056, 1066, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1005, 985, 1037, 997, 1042, 997, 1029, 23, 0, 0, 0, 0, 0, 0, 0, 1022, 998, 1041, 1001, 1021, 1071, 982, 1047, 10, 0, 0, 0, 0, 0, 0, 0, 1015, 1029, 1032, 1056, 1021, 998, 1041, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 937, 1031, 1016, 1005, 1007, 1026, 1021, 957, 5, 5, 0, 0, 0, 0, 0, 0, 977, 988, 1080, 993, 1037, 1039, 1055, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1062, 1052, 974, 1043, 1039, 978, 1033, 3, 3, 0, 0, 0, 0, 0, 0, 1059, 1021, 1043, 1026, 1015, 1024, 1043, 1059, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8099, 8193, 8233, 8010, 8197, 8230, 8291] avg=8192.0000 max=8291 max_rank=7 min=8010 min_rank=4 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1004, 1022, 1023, 1077, 1010, 963, 986, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 977, 1018, 1032, 1028, 977, 988, 1058, 26, 0, 0, 0, 0, 0, 0, 0, 1009, 1013, 1090, 1083, 1054, 1025, 1058, 960, 15, 0, 0, 0, 0, 0, 0, 0, 1005, 1012, 1068, 1016, 1021, 993, 1005, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1003, 1016, 1042, 1051, 1050, 1019, 1041, 11, 7, 0, 0, 0, 0, 0, 0, 1035, 1011, 981, 1061, 973, 1053, 1013, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1012, 1002, 1037, 1026, 1014, 1039, 940, 9, 5, 0, 0, 0, 0, 0, 0, 990, 1054, 1040, 1025, 1017, 1015, 1023, 1001, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8128, 8307, 8173, 8310, 8201, 8122, 8172] avg=8192.0000 max=8310 max_rank=4 min=8122 min_rank=6 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8448, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8064, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1077, 1034, 1026, 984, 985, 1017, 1019, 1045, 1030, 1010, 1076, 1009, 1064, 1007, 1011, 1001, 1025, 985, 1038, 1071, 1012, 1031, 1036, 978, 1051, 995, 1037, 1024, 1088, 1032, 1018, 956, 1003, 992, 1021, 1017, 1023, 979, 983, 1046, 993, 1008, 1049, 1032, 1079, 1041, 1057, 986, 1005, 1051, 1018, 1006, 1054, 1057, 1046, 1028, 1004, 1022, 1083, 988, 1059, 1003, 1012, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1065, 1030, 1026, 1006, 1035, 1021, 1026, 1023, 981, 1032, 1016, 1069, 1069, 1035, 1038, 1072, 1053, 960, 1022, 1025, 981, 1045, 1018, 1038, 1044, 1015, 1034, 973, 1012, 1011, 1030, 1023, 1029, 1067, 999, 1032, 1024, 1012, 1016, 1047, 1032, 995, 1017, 995, 1008, 1011, 1016, 1038, 1007, 1055, 1010, 1038, 1019, 1034, 1016, 993, 1033, 1020, 1075, 976, 1028, 1005, 1032, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1027, 1022, 1051, 1024, 1002, 1009, 1041, 1013, 988, 1030, 1051, 1018, 1010, 1045, 1069, 996, 1030, 963, 1037, 1009, 1018, 1042, 1005, 1014, 1041, 1046, 1029, 1020, 1030, 1076, 1001, 1067, 1007, 1055, 974, 1035, 1022, 994, 1064, 1044, 1026, 1052, 1023, 1055, 1013, 1036, 1055, 996, 1020, 1000, 1009, 1027, 996, 1010, 1030, 976, 1014, 1003, 1058, 997, 1016, 1013, 1056, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[979, 996, 1058, 999, 1052, 1042, 1027, 996, 1032, 1040, 1004, 1017, 1001, 1018, 1030, 1040, 1034, 1025, 1029, 1029, 974, 1017, 997, 1045, 1011, 1016, 1011, 1002, 1057, 1037, 1015, 1009, 1058, 1006, 1006, 1002, 1021, 1044, 1028, 1021, 986, 1015, 996, 1079, 1052, 1042, 1050, 1041, 1054, 1084, 1030, 1045, 1037, 1029, 1027, 1056, 1039, 992, 1034, 1031, 992, 1051, 991, 958] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1037, 1027, 1034, 1013, 1049, 1018, 999, 1038, 1008, 984, 1006, 1062, 1028, 1042, 1010, 1029, 1011, 1067, 1018, 988, 1049, 1096, 1030, 1011, 1024, 996, 999, 1026, 1036, 1019, 1004, 1001, 1077, 993, 957, 972, 1027, 1006, 995, 1051, 1025, 1043, 1011, 1060, 1039, 999, 1051, 1005, 1050, 971, 1039, 1068, 1081, 1027, 1043, 1070, 983, 1032, 1041, 1008, 989, 1046, 1003, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1033, 1054, 972, 1008, 1035, 996, 1010, 1000, 1022, 1008, 1049, 1023, 987, 1038, 990, 1018, 1042, 1053, 1066, 1044, 1041, 1011, 990, 1039, 1019, 1027, 1018, 1059, 989, 1003, 1036, 1038, 964, 1046, 1041, 1013, 1014, 1065, 1027, 1024, 1020, 1027, 1049, 1040, 1051, 1073, 1014, 971, 1034, 1041, 1047, 1000, 1008, 1001, 982, 1068, 1028, 1001, 1021, 1119, 979, 1006, 996, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1041, 963, 992, 965, 1054, 1006, 1024, 1081, 999, 1030, 1034, 997, 1028, 1025, 1023, 1108, 1028, 1026, 1059, 995, 1030, 1046, 1008, 1052, 1069, 1000, 988, 1069, 962, 993, 1007, 1013, 1020, 1044, 997, 1045, 995, 1036, 1004, 1085, 1041, 1061, 1053, 1077, 1032, 1047, 1031, 1008, 1038, 1011, 1043, 1000, 996, 1027, 973, 1041, 1063, 998, 952, 1006, 1002, 1025, 1048, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[999, 1035, 1004, 1081, 1003, 992, 1024, 1022, 1048, 1047, 1002, 985, 967, 1095, 1021, 1006, 993, 1016, 1062, 1013, 1029, 1038, 1019, 1075, 989, 993, 1028, 1023, 986, 1036, 1015, 1016, 1069, 1027, 1010, 982, 996, 991, 1021, 984, 1022, 1006, 1027, 1061, 1012, 1049, 1055, 996, 1038, 1016, 1056, 1030, 1019, 1055, 1042, 1017, 1031, 998, 1030, 1057, 1063, 1073, 1054, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1027, 1022, 1051, 1024, 1002, 1009, 1041, 1013, 15, 5, 0, 0, 0, 0, 0, 0, 988, 1030, 1051, 1018, 1010, 1034, 1069, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 963, 1026, 1009, 1018, 1042, 1005, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1046, 1029, 1020, 1030, 1076, 1001, 1067, 16, 13, 0, 0, 0, 0, 0, 0, 1007, 1055, 974, 1035, 1022, 994, 1064, 1044, 36, 0, 0, 0, 0, 0, 0, 0, 1026, 1052, 1023, 1006, 1013, 1036, 1055, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1000, 978, 1027, 996, 1010, 1030, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1003, 1058, 997, 1016, 1013, 1056, 1036, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8196, 8107, 8339, 8231, 8207, 8037, 8210] avg=8192.0000 max=8339 max_rank=3 min=8037 min_rank=6 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1077, 1034, 1026, 984, 985, 1017, 1019, 1045, 15, 5, 0, 0, 0, 0, 0, 0, 1030, 1010, 1076, 1009, 1064, 996, 1011, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 985, 1027, 1071, 1012, 1031, 1036, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 995, 1037, 1024, 1088, 1032, 1018, 956, 16, 13, 0, 0, 0, 0, 0, 0, 1003, 992, 1021, 1017, 1023, 979, 983, 1046, 35, 0, 0, 0, 0, 0, 0, 0, 993, 1008, 1049, 984, 1079, 1041, 1057, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1051, 987, 1006, 1054, 1057, 1046, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1022, 1083, 988, 1059, 1003, 1012, 1019, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8197, 8165, 8230, 8099, 8197, 8234, 8207] avg=8192.0000 max=8234 max_rank=6 min=8099 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1065, 1030, 1026, 1006, 1035, 1021, 1026, 1023, 15, 5, 0, 0, 0, 0, 0, 0, 981, 1032, 1016, 1069, 1069, 1024, 1038, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 960, 1011, 1025, 981, 1045, 1018, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1015, 1034, 973, 1012, 1011, 1030, 1023, 16, 12, 0, 0, 0, 0, 0, 0, 1029, 1067, 999, 1032, 1024, 1012, 1016, 1047, 33, 0, 0, 0, 0, 0, 0, 0, 1032, 995, 1017, 950, 1008, 1011, 1016, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1055, 979, 1038, 1019, 1034, 1016, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1020, 1075, 976, 1028, 1005, 1032, 1029, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8301, 8131, 8170, 8259, 8067, 8141, 8215] avg=8192.0000 max=8301 max_rank=1 min=8067 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[979, 996, 1058, 999, 1052, 1042, 1027, 996, 15, 5, 0, 0, 0, 0, 0, 0, 1032, 1040, 1004, 1017, 1001, 1007, 1030, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1025, 1018, 1029, 974, 1017, 997, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1016, 1011, 1002, 1057, 1037, 1015, 1009, 17, 13, 0, 0, 0, 0, 0, 0, 1058, 1006, 1006, 1002, 1021, 1044, 1028, 1021, 36, 0, 0, 0, 0, 0, 0, 0, 986, 1015, 996, 1030, 1052, 1042, 1050, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1084, 998, 1045, 1037, 1029, 1027, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 992, 1034, 1031, 992, 1051, 991, 958, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8171, 8139, 8188, 8222, 8212, 8330, 8105] avg=8192.0000 max=8330 max_rank=6 min=8105 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1037, 1027, 1034, 1013, 1049, 1018, 999, 1038, 15, 5, 0, 0, 0, 0, 0, 0, 1008, 984, 1006, 1062, 1028, 1031, 1010, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1067, 1007, 988, 1049, 1096, 1030, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 996, 999, 1026, 1036, 1019, 1004, 1001, 17, 13, 0, 0, 0, 0, 0, 0, 1077, 993, 957, 972, 1027, 1006, 995, 1051, 36, 0, 0, 0, 0, 0, 0, 0, 1025, 1043, 1011, 1011, 1039, 999, 1051, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 971, 1007, 1068, 1081, 1027, 1043, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1032, 1041, 1008, 989, 1046, 1003, 1015, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8158, 8259, 8135, 8114, 8184, 8317, 8134] avg=8192.0000 max=8317 max_rank=6 min=8114 min_rank=4 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1033, 1054, 972, 1008, 1035, 996, 1010, 1000, 15, 5, 0, 0, 0, 0, 0, 0, 1022, 1008, 1049, 1023, 987, 1027, 990, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1053, 1055, 1044, 1041, 1011, 990, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1027, 1018, 1059, 989, 1003, 1036, 1038, 17, 13, 0, 0, 0, 0, 0, 0, 964, 1046, 1041, 1013, 1014, 1065, 1027, 1024, 35, 0, 0, 0, 0, 0, 0, 0, 1020, 1027, 1049, 992, 1051, 1073, 1014, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1041, 1015, 1000, 1008, 1001, 982, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1001, 1021, 1119, 979, 1006, 996, 1048, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8124, 8275, 8219, 8229, 8197, 8149, 8215] avg=8192.0000 max=8275 max_rank=2 min=8124 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1041, 963, 992, 965, 1054, 1006, 1024, 1081, 15, 5, 0, 0, 0, 0, 0, 0, 999, 1030, 1034, 997, 1028, 1014, 1023, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1026, 1048, 995, 1030, 1046, 1008, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1000, 988, 1069, 962, 993, 1007, 1013, 17, 13, 0, 0, 0, 0, 0, 0, 1020, 1044, 997, 1045, 995, 1036, 1004, 1085, 36, 0, 0, 0, 0, 0, 0, 0, 1041, 1061, 1053, 1028, 1032, 1047, 1031, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1011, 1011, 1000, 996, 1027, 973, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 998, 952, 1006, 1002, 1025, 1048, 1022, 11, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8233, 8233, 8131, 8262, 8301, 8097, 8133] avg=8192.0000 max=8301 max_rank=5 min=8097 min_rank=6 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[999, 1035, 1004, 1081, 1003, 992, 1024, 1022, 21, 9, 0, 0, 0, 0, 0, 0, 1048, 1047, 1002, 985, 967, 1080, 1021, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1016, 1044, 1013, 1029, 1038, 1019, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 989, 993, 1028, 1023, 986, 1036, 1015, 1016, 21, 17, 0, 0, 0, 0, 0, 0, 1069, 1027, 1010, 982, 996, 991, 1021, 984, 40, 0, 0, 0, 0, 0, 0, 0, 1022, 1006, 1027, 1004, 1012, 1049, 1055, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1016, 1014, 1030, 1019, 1055, 1042, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 998, 1030, 1057, 1063, 1073, 1054, 987, 15, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8156, 8227, 8124, 8120, 8171, 8231, 8317] avg=8192.0000 max=8317 max_rank=7 min=8120 min_rank=4 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 8320, 8320, 8448, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8192, 8192, 8192, 8192, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8448, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[978, 1015, 1030, 1046, 977, 1024, 998, 1028, 1063, 1020, 1029, 1026, 989, 1020, 1050, 1041, 1052, 1044, 1062, 1011, 997, 1023, 1028, 1038, 1059, 1018, 1016, 1070, 1067, 949, 1003, 995, 983, 1023, 1006, 1063, 1024, 1074, 980, 1034, 1016, 1007, 1055, 982, 1060, 1019, 1108, 1027, 1013, 1051, 1047, 1009, 1031, 967, 1005, 1049, 1002, 1038, 1061, 1041, 989, 994, 984, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1000, 1011, 1002, 996, 1014, 953, 1022, 1005, 1029, 970, 998, 1053, 1091, 996, 1035, 1049, 1035, 1006, 1050, 1030, 1024, 1066, 994, 1037, 1002, 1054, 1054, 1018, 1061, 1038, 966, 989, 1017, 979, 1023, 1074, 1033, 1051, 1021, 1029, 1009, 1009, 1068, 1017, 977, 1015, 1064, 1043, 1049, 1019, 1008, 1068, 998, 1037, 996, 979, 1062, 1013, 1015, 1022, 1027, 1080, 1045, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1040, 997, 1002, 1032, 1084, 1023, 1080, 1044, 1053, 1066, 1004, 1052, 1072, 1005, 1100, 1020, 1003, 1053, 937, 1049, 1040, 1058, 1066, 1006, 1063, 1028, 1016, 953, 1002, 1025, 1017, 1014, 989, 1026, 1020, 1016, 1024, 1033, 973, 1007, 1049, 1016, 1035, 1040, 1029, 1029, 1031, 974, 1050, 1007, 1041, 975, 1075, 955, 1073, 984, 1011, 997, 1000, 1022, 1034, 1043, 1007, 967] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1066, 976, 1034, 1032, 1043, 1017, 1008, 1048, 954, 1027, 1046, 1006, 1023, 1051, 1038, 1048, 1016, 979, 1019, 1045, 1015, 1004, 1026, 1018, 1006, 1057, 1017, 1035, 1023, 1079, 1015, 1021, 1050, 998, 1037, 1011, 1032, 1009, 992, 1017, 1028, 1062, 1035, 974, 1067, 1047, 983, 1014, 1069, 996, 1021, 990, 1058, 1027, 1009, 1038, 1058, 1025, 971, 1053, 1086, 999, 964, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[998, 995, 1064, 1047, 966, 1039, 1061, 1048, 1071, 1052, 1038, 1020, 1040, 1039, 980, 999, 1046, 1024, 1052, 1035, 1047, 1053, 1031, 1035, 1041, 974, 1009, 1015, 1022, 987, 1049, 1023, 962, 1003, 967, 1029, 1026, 1040, 1040, 978, 1017, 1061, 1034, 1042, 1014, 1030, 1068, 991, 962, 1003, 995, 1015, 1033, 1007, 1031, 1065, 1045, 1053, 993, 1017, 1039, 995, 1009, 1072] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1045, 1014, 1010, 951, 1020, 1011, 1023, 1022, 1012, 1052, 1016, 1026, 1036, 1034, 1034, 1015, 1022, 1067, 1066, 1030, 1014, 1022, 1041, 1000, 1013, 1007, 1043, 1033, 1027, 1034, 1083, 1039, 1015, 1014, 1006, 1045, 1024, 1055, 1063, 1042, 1086, 1076, 1020, 1030, 1064, 974, 996, 1012, 1034, 1027, 1005, 999, 1003, 1036, 1003, 992, 1021, 1004, 1001, 1037, 988, 992, 1050, 960] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1031, 1062, 996, 977, 1006, 1055, 1000, 1069, 1032, 1060, 979, 1014, 1041, 1076, 1024, 1032, 952, 995, 1075, 1025, 979, 1018, 1064, 990, 1055, 984, 1032, 974, 1040, 1037, 1027, 1008, 1025, 1025, 1002, 1033, 1049, 1019, 1029, 1016, 1021, 1074, 986, 1000, 1023, 1070, 1010, 1022, 1007, 1018, 1034, 1049, 1008, 1044, 1034, 961, 1099, 1021, 1044, 1011, 1025, 1029, 991, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[989, 1004, 1036, 1010, 1007, 1072, 1037, 1046, 1006, 1031, 987, 983, 975, 1042, 1036, 1003, 1056, 1003, 1045, 1052, 1025, 1021, 1028, 992, 995, 1024, 1063, 1003, 1004, 1008, 960, 1068, 1057, 1054, 1079, 1042, 1038, 1015, 1026, 1038, 973, 1007, 1039, 1015, 991, 1054, 986, 1101, 1061, 1068, 1014, 1071, 960, 1003, 1046, 1045, 1047, 1064, 996, 1021, 1029, 970, 985, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1000, 1011, 1002, 996, 1014, 953, 1022, 1005, 16, 7, 0, 0, 0, 0, 0, 0, 1029, 970, 998, 1053, 1091, 996, 1002, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1006, 1026, 1030, 1024, 1066, 994, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1054, 1054, 1018, 1061, 1038, 966, 989, 15, 0, 0, 0, 0, 0, 0, 0, 1017, 979, 1023, 1074, 1033, 1051, 1021, 1029, 4, 0, 0, 0, 0, 0, 0, 0, 1009, 981, 1068, 1017, 977, 1015, 1064, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1019, 1008, 1068, 998, 1037, 996, 979, 26, 0, 0, 0, 0, 0, 0, 0, 1062, 1013, 1015, 1022, 1027, 1080, 1045, 1041, 12, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8026, 8188, 8218, 8197, 8231, 8174, 8180, 8322] avg=8192.0000 max=8322 max_rank=7 min=8026 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[978, 1015, 1030, 1046, 977, 1024, 998, 1028, 16, 8, 0, 0, 0, 0, 0, 0, 1063, 1020, 1029, 1026, 989, 1020, 1016, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1044, 1038, 1011, 997, 1023, 1028, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1018, 1016, 1070, 1067, 949, 1003, 995, 15, 0, 0, 0, 0, 0, 0, 0, 983, 1023, 1006, 1063, 1024, 1074, 980, 1034, 4, 0, 0, 0, 0, 0, 0, 0, 1016, 979, 1055, 982, 1060, 1019, 1108, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1051, 1047, 1009, 1031, 967, 1005, 1049, 26, 0, 0, 0, 0, 0, 0, 0, 1002, 1038, 1061, 1041, 989, 994, 984, 1028, 12, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8204, 8231, 8192, 8191, 8246, 8198, 8154] avg=8192.0000 max=8246 max_rank=5 min=8120 min_rank=0 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1040, 997, 1002, 1032, 1084, 1023, 1080, 1044, 16, 8, 0, 0, 0, 0, 0, 0, 1053, 1066, 1004, 1052, 1072, 1005, 1065, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1053, 916, 1049, 1040, 1058, 1066, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1028, 1016, 953, 1002, 1025, 1017, 1014, 14, 0, 0, 0, 0, 0, 0, 0, 989, 1026, 1020, 1016, 1024, 1033, 973, 1007, 3, 0, 0, 0, 0, 0, 0, 0, 1049, 988, 1035, 1040, 1029, 1029, 1031, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1007, 1041, 975, 1075, 955, 1073, 984, 27, 0, 0, 0, 0, 0, 0, 0, 1011, 997, 1000, 1022, 1034, 1043, 1007, 967, 12, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8326, 8337, 8191, 8132, 8091, 8175, 8187, 8097] avg=8192.0000 max=8337 max_rank=1 min=8091 min_rank=4 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32550 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1066, 976, 1034, 1032, 1043, 1017, 1008, 1048, 17, 8, 0, 0, 0, 0, 0, 0, 954, 1027, 1046, 1006, 1023, 1051, 1004, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 979, 996, 1045, 1015, 1004, 1026, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1057, 1017, 1035, 1023, 1079, 1015, 1021, 15, 0, 0, 0, 0, 0, 0, 0, 1050, 998, 1037, 1011, 1032, 1009, 992, 1017, 3, 0, 0, 0, 0, 0, 0, 0, 1028, 1032, 1035, 974, 1067, 1047, 983, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 996, 1021, 990, 1058, 1027, 1009, 1038, 26, 0, 0, 0, 0, 0, 0, 0, 1058, 1025, 971, 1053, 1086, 999, 964, 1024, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8159, 8099, 8268, 8149, 8180, 8234, 8198] avg=8192.0000 max=8268 max_rank=3 min=8099 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1045, 1014, 1010, 951, 1020, 1011, 1023, 1022, 17, 7, 0, 0, 0, 0, 0, 0, 1012, 1052, 1016, 1026, 1036, 1034, 1001, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1067, 1041, 1030, 1014, 1022, 1041, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1007, 1043, 1033, 1027, 1034, 1083, 1039, 16, 0, 0, 0, 0, 0, 0, 0, 1015, 1014, 1006, 1045, 1024, 1055, 1063, 1042, 4, 0, 0, 0, 0, 0, 0, 0, 1086, 1046, 1020, 1030, 1064, 974, 996, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1027, 1005, 999, 1003, 1036, 1003, 992, 26, 0, 0, 0, 0, 0, 0, 0, 1021, 1004, 1001, 1037, 988, 992, 1050, 960, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8192, 8237, 8295, 8268, 8228, 8125, 8071] avg=8192.0000 max=8295 max_rank=3 min=8071 min_rank=7 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[998, 995, 1064, 1047, 966, 1039, 1061, 1048, 17, 7, 0, 0, 0, 0, 0, 0, 1071, 1052, 1038, 1020, 1040, 1039, 949, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1024, 1028, 1035, 1047, 1053, 1031, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 974, 1009, 1015, 1022, 987, 1049, 1023, 15, 0, 0, 0, 0, 0, 0, 0, 962, 1003, 967, 1029, 1026, 1040, 1040, 978, 4, 0, 0, 0, 0, 0, 0, 0, 1017, 1031, 1034, 1042, 1014, 1030, 1068, 991, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1003, 995, 1015, 1033, 1007, 1031, 1065, 24, 0, 0, 0, 0, 0, 0, 0, 1045, 1053, 993, 1017, 1039, 995, 1009, 1072, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8208, 8299, 8135, 8049, 8227, 8135, 8241] avg=8192.0000 max=8299 max_rank=2 min=8049 min_rank=4 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1031, 1062, 996, 977, 1006, 1055, 1000, 1069, 17, 7, 0, 0, 0, 0, 0, 0, 1032, 1060, 979, 1014, 1041, 1076, 992, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 952, 995, 1050, 1025, 979, 1018, 1064, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 984, 1032, 974, 1040, 1037, 1027, 1008, 16, 0, 0, 0, 0, 0, 0, 0, 1025, 1025, 1002, 1033, 1049, 1019, 1029, 1016, 4, 0, 0, 0, 0, 0, 0, 0, 1021, 1044, 986, 1000, 1023, 1070, 1010, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1018, 1034, 1049, 1008, 1044, 1034, 961, 25, 0, 0, 0, 0, 0, 0, 0, 1099, 1021, 1044, 1011, 1025, 1029, 991, 1048, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8226, 8073, 8173, 8202, 8176, 8180, 8286] avg=8192.0000 max=8286 max_rank=7 min=8073 min_rank=2 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[989, 1004, 1036, 1010, 1007, 1072, 1037, 1046, 20, 12, 0, 0, 0, 0, 0, 0, 1006, 1031, 987, 983, 975, 1042, 995, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1003, 1011, 1052, 1025, 1021, 1028, 992, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1024, 1063, 1003, 1004, 1008, 960, 1068, 19, 0, 0, 0, 0, 0, 0, 0, 1057, 1054, 1079, 1042, 1038, 1015, 1026, 1038, 6, 0, 0, 0, 0, 0, 0, 0, 973, 971, 1039, 1015, 991, 1054, 986, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1068, 1014, 1071, 960, 1003, 1046, 1045, 29, 0, 0, 0, 0, 0, 0, 0, 1047, 1064, 996, 1021, 1029, 970, 985, 1030, 16, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8022, 8188, 8144, 8355, 8130, 8297, 8167] avg=8192.0000 max=8355 max_rank=4 min=8022 min_rank=1 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32949 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8448, 8320, 8192, 8320, 8320, 8192, 8064, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1036, 996, 989, 1032, 1025, 1029, 1063, 999, 1028, 1003, 1024, 999, 990, 1045, 1021, 1064, 1016, 1011, 1016, 1043, 980, 1025, 1024, 1035, 980, 1049, 1019, 1005, 1022, 996, 980, 1021, 1056, 1015, 1095, 1023, 1005, 1006, 1022, 1079, 984, 1039, 1045, 1006, 1047, 1043, 1046, 985, 1037, 995, 1014, 1029, 1047, 1008, 1006, 1056, 1041, 1020, 1050, 1040, 1024, 1033, 1053, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1047, 1074, 968, 1033, 999, 1037, 1028, 1028, 1015, 1054, 1011, 1040, 991, 984, 1016, 1007, 1020, 1025, 970, 1005, 1060, 1042, 1016, 1104, 976, 1067, 1112, 1054, 972, 1007, 1023, 1058, 999, 1013, 1041, 964, 1028, 1026, 1036, 1044, 1046, 1018, 1020, 1035, 1043, 996, 979, 999, 1002, 1043, 1031, 1035, 1030, 977, 1025, 1079, 1018, 1024, 1056, 979, 965, 1073, 1063, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1078, 1008, 965, 997, 1026, 1022, 1003, 1027, 956, 1010, 1033, 1061, 1037, 1049, 1022, 1015, 1052, 1029, 981, 1037, 1002, 1028, 1040, 1051, 1034, 1024, 1020, 1033, 1067, 991, 1045, 994, 1043, 1003, 1027, 1023, 1003, 1043, 1014, 1036, 1012, 1034, 1070, 1034, 963, 1086, 1058, 1016, 1031, 1056, 1031, 1006, 1053, 989, 1026, 981, 982, 1014, 1022, 1028, 1052, 1009, 1023, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1007, 984, 1009, 1008, 998, 1034, 952, 1003, 977, 1051, 1007, 1016, 1055, 1059, 1040, 1030, 1065, 1005, 1027, 1031, 1006, 1055, 1055, 1078, 979, 993, 1060, 1044, 1021, 1010, 988, 1044, 1032, 1049, 992, 1028, 1020, 1059, 987, 982, 1060, 1026, 1064, 1001, 1054, 1047, 998, 1033, 1026, 1045, 1039, 1006, 1079, 1012, 1020, 1049, 1020, 1068, 990, 1009, 999, 1020, 980, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1026, 1032, 1053, 1008, 1039, 1034, 1025, 1042, 1000, 977, 1056, 1034, 989, 1009, 986, 1008, 1041, 1037, 1070, 1052, 1050, 1052, 1020, 1043, 1050, 1018, 1023, 1010, 1059, 1001, 1031, 1060, 1028, 1051, 998, 997, 971, 1042, 1017, 987, 983, 982, 1007, 1034, 1068, 1060, 1074, 1043, 1002, 1018, 1010, 1055, 965, 1002, 1026, 1037, 1036, 1004, 981, 988, 1053, 1042, 1014, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[987, 1009, 952, 1015, 1039, 1025, 1046, 999, 1045, 991, 1046, 1039, 1079, 1033, 1000, 1038, 998, 1033, 1023, 979, 1043, 1004, 1026, 1037, 1002, 997, 1005, 1027, 1046, 1083, 1059, 1065, 1036, 1019, 1060, 1056, 1001, 1031, 990, 1069, 1022, 996, 980, 1017, 987, 1002, 1048, 1048, 999, 1036, 1005, 991, 1055, 1010, 1045, 1040, 1033, 1012, 1038, 1028, 1075, 1003, 1017, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1034, 1017, 1058, 1001, 1060, 991, 1035, 976, 1017, 1029, 972, 1011, 1012, 1059, 1058, 1066, 1017, 993, 1041, 1037, 1008, 1040, 1037, 1008, 1051, 992, 1037, 979, 1015, 1037, 1026, 1018, 1038, 1044, 1012, 1033, 1000, 1024, 1020, 1036, 982, 1056, 1042, 1031, 1018, 1056, 1023, 1046, 996, 1047, 992, 1086, 1003, 984, 1054, 998, 1008, 1033, 983, 1086, 1069, 982, 1024, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1063, 1049, 1052, 1003, 1040, 1042, 1030, 1038, 1045, 1009, 1062, 1024, 1019, 1032, 1006, 1027, 1018, 1025, 1054, 1053, 1058, 971, 1047, 1010, 968, 1014, 1011, 1004, 1001, 1062, 1041, 1014, 1029, 1012, 1008, 1015, 1003, 1010, 946, 1023, 1021, 1065, 1023, 1022, 1021, 1041, 1055, 1001, 1006, 1063, 975, 1039, 1009, 999, 1004, 1022, 1018, 1038, 1026, 1040, 1004, 977, 1056, 1073] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1078, 1008, 965, 997, 1026, 1022, 1003, 1027, 26, 0, 0, 0, 0, 0, 0, 0, 956, 1010, 1033, 1061, 1037, 1049, 1022, 1015, 6, 0, 0, 0, 0, 0, 0, 0, 1052, 1029, 981, 1037, 1002, 1028, 1040, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1024, 1020, 1033, 1067, 991, 1045, 994, 4, 1, 0, 0, 0, 0, 0, 0, 1043, 1003, 1027, 1023, 1003, 1043, 1014, 1036, 13, 3, 0, 0, 0, 0, 0, 0, 1012, 1034, 1070, 1034, 963, 1060, 1058, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1056, 1031, 1006, 1053, 989, 1026, 981, 13, 0, 0, 0, 0, 0, 0, 0, 982, 1014, 1022, 1028, 1051, 1009, 1023, 1031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8189, 8181, 8213, 8208, 8247, 8186, 8160] avg=8192.0000 max=8247 max_rank=5 min=8152 min_rank=0 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[987, 1009, 952, 1015, 1039, 1025, 1046, 999, 26, 0, 0, 0, 0, 0, 0, 0, 1045, 991, 1046, 1039, 1079, 1033, 1000, 1038, 5, 0, 0, 0, 0, 0, 0, 0, 998, 1033, 1023, 979, 1043, 1004, 1026, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 997, 1005, 1027, 1046, 1083, 1059, 1065, 3, 1, 0, 0, 0, 0, 0, 0, 1036, 1019, 1060, 1056, 1001, 1031, 990, 1069, 13, 3, 0, 0, 0, 0, 0, 0, 1022, 996, 980, 1017, 987, 979, 1048, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1036, 1005, 991, 1055, 1010, 1045, 1040, 12, 0, 0, 0, 0, 0, 0, 0, 1033, 1012, 1038, 1028, 1074, 1003, 1017, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8276, 8104, 8288, 8278, 8077, 8193, 8222] avg=8192.0000 max=8288 max_rank=3 min=8077 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1047, 1074, 968, 1033, 999, 1037, 1028, 1028, 27, 0, 0, 0, 0, 0, 0, 0, 1015, 1054, 1011, 1040, 991, 984, 1016, 1007, 5, 0, 0, 0, 0, 0, 0, 0, 1020, 1025, 970, 1005, 1060, 1042, 1016, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1067, 1112, 1054, 972, 1007, 1023, 1058, 3, 1, 0, 0, 0, 0, 0, 0, 999, 1013, 1041, 964, 1028, 1026, 1036, 1044, 14, 3, 0, 0, 0, 0, 0, 0, 1046, 1018, 1020, 1035, 1043, 974, 979, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1043, 1031, 1035, 1030, 977, 1025, 1079, 11, 0, 0, 0, 0, 0, 0, 0, 1018, 1024, 1056, 979, 964, 1073, 1063, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8123, 8201, 8273, 8168, 8114, 8233, 8183] avg=8192.0000 max=8273 max_rank=3 min=8114 min_rank=5 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1036, 996, 989, 1032, 1025, 1029, 1063, 999, 26, 0, 0, 0, 0, 0, 0, 0, 1028, 1003, 1024, 999, 990, 1045, 1021, 1064, 6, 0, 0, 0, 0, 0, 0, 0, 1016, 1011, 1016, 1043, 980, 1025, 1024, 996, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1049, 1019, 1005, 1022, 996, 980, 1021, 3, 1, 0, 0, 0, 0, 0, 0, 1056, 1015, 1095, 1023, 1005, 1006, 1022, 1079, 13, 3, 0, 0, 0, 0, 0, 0, 984, 1039, 1045, 1006, 1047, 1019, 1046, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 995, 1014, 1029, 1047, 1008, 1006, 1056, 12, 0, 0, 0, 0, 0, 0, 0, 1041, 1020, 1050, 1040, 1023, 1033, 1053, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8180, 8111, 8076, 8317, 8171, 8204, 8282] avg=8192.0000 max=8317 max_rank=4 min=8076 min_rank=3 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32974 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1026, 1032, 1053, 1008, 1039, 1034, 1025, 1042, 26, 0, 0, 0, 0, 0, 0, 0, 1000, 977, 1056, 1034, 989, 1009, 986, 1008, 6, 0, 0, 0, 0, 0, 0, 0, 1041, 1037, 1070, 1052, 1050, 1052, 1020, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1018, 1023, 1010, 1059, 1001, 1031, 1060, 3, 1, 0, 0, 0, 0, 0, 0, 1028, 1051, 998, 997, 971, 1042, 1017, 987, 13, 3, 0, 0, 0, 0, 0, 0, 983, 982, 1007, 1034, 1068, 1036, 1074, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1018, 1010, 1055, 965, 1002, 1026, 1037, 12, 0, 0, 0, 0, 0, 0, 0, 1036, 1004, 981, 988, 1052, 1042, 1014, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8065, 8326, 8256, 8107, 8227, 8127, 8143] avg=8192.0000 max=8326 max_rank=2 min=8065 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32604 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1007, 984, 1009, 1008, 998, 1034, 952, 1003, 27, 0, 0, 0, 0, 0, 0, 0, 977, 1051, 1007, 1016, 1055, 1059, 1040, 1030, 6, 0, 0, 0, 0, 0, 0, 0, 1065, 1005, 1027, 1031, 1006, 1055, 1055, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 979, 993, 1060, 1044, 1021, 1010, 988, 1044, 3, 1, 0, 0, 0, 0, 0, 0, 1032, 1049, 992, 1028, 1020, 1059, 987, 982, 14, 3, 0, 0, 0, 0, 0, 0, 1060, 1026, 1064, 1001, 1054, 1023, 998, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1045, 1039, 1006, 1079, 1012, 1020, 1049, 12, 0, 0, 0, 0, 0, 0, 0, 1020, 1068, 990, 1009, 998, 1020, 980, 1051, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8022, 8241, 8281, 8143, 8166, 8259, 8288, 8136] avg=8192.0000 max=8288 max_rank=6 min=8022 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1063, 1049, 1052, 1003, 1040, 1042, 1030, 1038, 25, 0, 0, 0, 0, 0, 0, 0, 1045, 1009, 1062, 1024, 1019, 1032, 1006, 1027, 5, 0, 0, 0, 0, 0, 0, 0, 1018, 1025, 1054, 1053, 1058, 971, 1047, 972, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1014, 1011, 1004, 1001, 1062, 1041, 1014, 3, 1, 0, 0, 0, 0, 0, 0, 1029, 1012, 1008, 1015, 1003, 1010, 946, 1023, 13, 3, 0, 0, 0, 0, 0, 0, 1021, 1065, 1023, 1022, 1021, 1018, 1055, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1063, 975, 1039, 1009, 999, 1004, 1022, 12, 0, 0, 0, 0, 0, 0, 0, 1018, 1038, 1026, 1040, 1003, 977, 1056, 1073, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8342, 8229, 8198, 8119, 8062, 8226, 8129, 8231] avg=8192.0000 max=8342 max_rank=0 min=8062 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1034, 1017, 1058, 1001, 1060, 991, 1035, 976, 29, 0, 0, 0, 0, 0, 0, 0, 1017, 1029, 972, 1011, 1012, 1059, 1058, 1066, 9, 0, 0, 0, 0, 0, 0, 0, 1017, 993, 1041, 1037, 1008, 1040, 1037, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 992, 1037, 979, 1015, 1037, 1026, 1018, 9, 4, 0, 0, 0, 0, 0, 0, 1038, 1044, 1012, 1033, 1000, 1024, 1020, 1036, 18, 5, 0, 0, 0, 0, 0, 0, 982, 1056, 1042, 1031, 1018, 1017, 1023, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1047, 992, 1086, 1003, 984, 1054, 998, 16, 0, 0, 0, 0, 0, 0, 0, 1008, 1033, 983, 1086, 1065, 982, 1024, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8233, 8134, 8168, 8230, 8215, 8176, 8179] avg=8192.0000 max=8233 max_rank=1 min=8134 min_rank=2 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8192, 8320, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8064, 8320, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8064, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1047, 1050, 1081, 1016, 998, 1044, 1063, 1012, 1042, 989, 1002, 1018, 1029, 1036, 995, 981, 1031, 995, 1030, 1049, 1029, 1035, 998, 1002, 972, 1076, 981, 1074, 1036, 1003, 1047, 1068, 1033, 1007, 1048, 1017, 1022, 1065, 1020, 1048, 992, 1035, 1054, 986, 1052, 1016, 1037, 987, 999, 1027, 1031, 965, 978, 1054, 1007, 1014, 1018, 1009, 996, 1050, 1066, 1038, 1037, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1003, 1031, 1017, 1042, 1024, 974, 1000, 974, 1001, 1026, 1025, 1033, 1059, 1035, 1043, 1006, 994, 1019, 1054, 1015, 1036, 1072, 957, 990, 993, 999, 1051, 1040, 1021, 995, 1020, 1031, 1030, 979, 1028, 1059, 988, 1042, 1003, 1029, 1036, 1016, 1024, 1066, 1024, 1048, 1050, 1012, 1028, 1070, 1044, 1055, 1012, 991, 1051, 1044, 1022, 1013, 982, 1041, 1026, 1046, 1064, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1062, 1022, 1018, 1010, 1032, 1004, 1035, 1018, 1025, 1023, 1031, 1021, 1047, 1004, 992, 1014, 1023, 1027, 986, 1021, 1047, 1007, 1028, 975, 1003, 999, 1059, 1053, 990, 1036, 985, 1059, 1081, 1032, 1023, 979, 1087, 1032, 981, 1032, 982, 978, 1009, 999, 994, 1062, 1019, 1021, 1050, 1022, 1028, 1002, 1104, 1049, 1004, 1055, 1064, 1003, 1031, 1031, 1014, 1050, 1059, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[974, 1052, 1002, 1031, 972, 981, 1013, 992, 1037, 1041, 1030, 1062, 1008, 1028, 1057, 1040, 1042, 1012, 1024, 1050, 1000, 1030, 1019, 1049, 1064, 1008, 995, 1046, 1001, 1037, 1000, 1013, 1055, 1041, 1038, 1042, 1016, 1011, 1017, 1057, 1032, 1026, 1024, 1033, 1065, 986, 1042, 989, 1006, 993, 1038, 1045, 1040, 1011, 1040, 1043, 958, 985, 1019, 1051, 1059, 1044, 996, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1056, 1017, 1048, 1048, 1009, 953, 1036, 1055, 1009, 1024, 1033, 1029, 1019, 1064, 999, 1042, 1025, 1035, 1018, 1029, 1016, 1013, 1030, 1029, 1055, 965, 1020, 1053, 1010, 1033, 1043, 975, 1029, 1068, 1044, 1010, 980, 1021, 1051, 1013, 1017, 1023, 995, 1018, 1025, 1031, 1025, 1014, 1032, 1051, 973, 1002, 1004, 993, 1042, 1025, 1078, 1036, 1013, 1061, 1000, 1016, 1033, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1027, 1060, 1019, 1028, 1014, 994, 1046, 1046, 1061, 1026, 1022, 1046, 1056, 1031, 1038, 1019, 1013, 1019, 1012, 1040, 1073, 992, 1018, 994, 1028, 1012, 1016, 1022, 1032, 1029, 981, 999, 1048, 1041, 1022, 1069, 1008, 1015, 1092, 1038, 1001, 1021, 1009, 1009, 1015, 1039, 985, 1025, 973, 1038, 1046, 981, 970, 1018, 1017, 1040, 1012, 1022, 1047, 1053, 991, 1034, 1039, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1024, 1039, 1018, 1044, 1009, 975, 1043, 1045, 1021, 1034, 1043, 988, 1047, 1003, 1040, 1004, 1017, 1028, 1076, 983, 1010, 1003, 987, 1045, 1011, 1031, 1037, 1035, 1037, 1046, 990, 1014, 1043, 1020, 1017, 994, 1037, 1013, 1035, 1017, 1008, 1008, 998, 1047, 1057, 1075, 982, 1046, 1062, 1054, 1049, 1084, 1016, 997, 987, 1004, 1023, 1046, 998, 1040, 1020, 998, 1049, 985] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1016, 1003, 1018, 1039, 1011, 994, 1002, 1065, 1038, 995, 1050, 1080, 1014, 1009, 1096, 995, 996, 1044, 1079, 1030, 1022, 1083, 1001, 1009, 1038, 1041, 1008, 1019, 1010, 1031, 1071, 1001, 1017, 1039, 1078, 1033, 986, 1009, 1022, 1026, 981, 1013, 1038, 980, 1069, 993, 1012, 1057, 1026, 1040, 1030, 1039, 1043, 1002, 1005, 1026, 1028, 1043, 1005, 1035, 987, 968, 964, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1003, 1031, 1017, 1042, 1024, 974, 1000, 974, 14, 3, 0, 0, 0, 0, 0, 0, 1001, 1026, 1025, 1033, 1032, 1035, 1043, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1019, 1054, 1015, 1036, 1072, 957, 990, 15, 0, 0, 0, 0, 0, 0, 0, 993, 999, 1051, 1040, 1021, 995, 1020, 1031, 9, 1, 0, 0, 0, 0, 0, 0, 989, 979, 1028, 1059, 988, 1042, 1003, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1016, 1024, 1066, 1024, 1048, 1050, 1012, 27, 0, 0, 0, 0, 0, 0, 0, 1028, 1067, 1044, 1055, 1012, 991, 1051, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1013, 982, 1041, 1026, 1046, 1064, 1033, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8201, 8152, 8160, 8117, 8303, 8292, 8229] avg=8192.0000 max=8303 max_rank=5 min=8082 min_rank=0 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32941 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1047, 1050, 1081, 1016, 998, 1044, 1063, 1012, 14, 2, 0, 0, 0, 0, 0, 0, 1042, 989, 1002, 1018, 1003, 1036, 995, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 995, 1030, 1049, 1029, 1035, 998, 1002, 15, 0, 0, 0, 0, 0, 0, 0, 972, 1076, 981, 1074, 1036, 1003, 1047, 1068, 9, 1, 0, 0, 0, 0, 0, 0, 992, 1007, 1048, 1017, 1022, 1065, 1020, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1035, 1054, 986, 1052, 1016, 1037, 987, 27, 0, 0, 0, 0, 0, 0, 0, 999, 1024, 1031, 965, 978, 1054, 1007, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1009, 996, 1050, 1066, 1038, 1037, 999, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8327, 8066, 8184, 8267, 8219, 8186, 8072, 8215] avg=8192.0000 max=8327 max_rank=0 min=8066 min_rank=1 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1062, 1022, 1018, 1010, 1032, 1004, 1035, 1018, 15, 3, 0, 0, 0, 0, 0, 0, 1025, 1023, 1031, 1021, 1020, 1004, 992, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1027, 986, 1021, 1047, 1007, 1028, 975, 15, 0, 0, 0, 0, 0, 0, 0, 1003, 999, 1059, 1053, 990, 1036, 985, 1059, 9, 1, 0, 0, 0, 0, 0, 0, 1037, 1032, 1023, 979, 1087, 1032, 981, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 982, 978, 1009, 999, 994, 1062, 1019, 1021, 29, 0, 0, 0, 0, 0, 0, 0, 1050, 1019, 1028, 1002, 1104, 1049, 1004, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1003, 1031, 1031, 1014, 1050, 1059, 1003, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8130, 8129, 8194, 8203, 8093, 8311, 8257] avg=8192.0000 max=8311 max_rank=6 min=8093 min_rank=5 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1056, 1017, 1048, 1048, 1009, 953, 1036, 1055, 14, 2, 0, 0, 0, 0, 0, 0, 1009, 1024, 1033, 1029, 994, 1064, 999, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1035, 1018, 1029, 1016, 1013, 1030, 1029, 14, 0, 0, 0, 0, 0, 0, 0, 1055, 965, 1020, 1053, 1010, 1033, 1043, 975, 9, 1, 0, 0, 0, 0, 0, 0, 988, 1068, 1044, 1010, 980, 1021, 1051, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1023, 995, 1018, 1025, 1031, 1025, 1014, 27, 0, 0, 0, 0, 0, 0, 0, 1032, 1048, 973, 1002, 1004, 993, 1042, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1036, 1013, 1061, 1000, 1016, 1033, 1023, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8194, 8209, 8164, 8175, 8175, 8119, 8262] avg=8192.0000 max=8262 max_rank=7 min=8119 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[974, 1052, 1002, 1031, 972, 981, 1013, 992, 14, 2, 0, 0, 0, 0, 0, 0, 1037, 1041, 1030, 1062, 983, 1028, 1057, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1012, 1024, 1050, 1000, 1030, 1019, 1049, 14, 0, 0, 0, 0, 0, 0, 0, 1064, 1008, 995, 1046, 1001, 1037, 1000, 1013, 9, 1, 0, 0, 0, 0, 0, 0, 1013, 1041, 1038, 1042, 1016, 1011, 1017, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1026, 1024, 1033, 1065, 986, 1042, 989, 28, 0, 0, 0, 0, 0, 0, 0, 1006, 990, 1038, 1045, 1040, 1011, 1040, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 958, 985, 1019, 1051, 1059, 1044, 996, 1024, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8033, 8278, 8240, 8174, 8235, 8225, 8213, 8138] avg=8192.0000 max=8278 max_rank=1 min=8033 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1024, 1039, 1018, 1044, 1009, 975, 1043, 1045, 14, 3, 0, 0, 0, 0, 0, 0, 1021, 1034, 1043, 988, 1020, 1003, 1040, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1028, 1076, 983, 1010, 1003, 987, 1045, 15, 0, 0, 0, 0, 0, 0, 0, 1011, 1031, 1037, 1035, 1037, 1046, 990, 1014, 9, 1, 0, 0, 0, 0, 0, 0, 1001, 1020, 1017, 994, 1037, 1013, 1035, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1008, 998, 1047, 1057, 1075, 982, 1046, 28, 0, 0, 0, 0, 0, 0, 0, 1062, 1051, 1049, 1084, 1016, 997, 987, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1046, 998, 1040, 1020, 998, 1049, 985, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8153, 8164, 8211, 8134, 8249, 8250, 8161] avg=8192.0000 max=8250 max_rank=6 min=8134 min_rank=4 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1027, 1060, 1019, 1028, 1014, 994, 1046, 1046, 14, 3, 0, 0, 0, 0, 0, 0, 1061, 1026, 1022, 1046, 1029, 1031, 1038, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1019, 1012, 1040, 1073, 992, 1018, 994, 15, 0, 0, 0, 0, 0, 0, 0, 1028, 1012, 1016, 1022, 1032, 1029, 981, 999, 9, 1, 0, 0, 0, 0, 0, 0, 1006, 1041, 1022, 1069, 1008, 1015, 1092, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1021, 1009, 1009, 1015, 1039, 985, 1025, 28, 0, 0, 0, 0, 0, 0, 0, 973, 1035, 1046, 981, 970, 1018, 1017, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1022, 1047, 1053, 991, 1034, 1039, 1005, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8272, 8176, 8129, 8291, 8132, 8080, 8205] avg=8192.0000 max=8291 max_rank=4 min=8080 min_rank=6 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1016, 1003, 1018, 1039, 1011, 994, 1002, 1065, 18, 6, 0, 0, 0, 0, 0, 0, 1038, 995, 1050, 1080, 979, 1009, 1096, 995, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1044, 1079, 1030, 1022, 1083, 1001, 1009, 18, 0, 0, 0, 0, 0, 0, 0, 1038, 1041, 1008, 1019, 1010, 1031, 1071, 1001, 11, 7, 0, 0, 0, 0, 0, 0, 969, 1039, 1078, 1033, 986, 1009, 1022, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1013, 1038, 980, 1069, 993, 1012, 1057, 30, 0, 0, 0, 0, 0, 0, 0, 1026, 1028, 1030, 1039, 1043, 1002, 1005, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1043, 1005, 1035, 987, 968, 964, 1034, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8242, 8282, 8237, 8162, 8173, 8199, 8069] avg=8192.0000 max=8282 max_rank=2 min=8069 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32603 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8448, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 7936, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8064, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8192, 8320, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8448, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1014, 1002, 977, 1041, 1045, 1038, 1034, 1035, 1029, 1036, 1032, 1005, 1036, 974, 1061, 1011, 1018, 1006, 1022, 998, 1069, 1048, 992, 1079, 1029, 1021, 1015, 1035, 1093, 974, 999, 1010, 1046, 1041, 1061, 987, 1035, 985, 1036, 1002, 978, 999, 1052, 1091, 1077, 996, 1038, 1044, 1032, 982, 1008, 1001, 983, 1104, 1011, 993, 1052, 1030, 1048, 1034, 992, 1023, 1032, 965] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1052, 1068, 1019, 999, 1023, 994, 1058, 990, 1017, 1043, 1021, 1050, 1044, 1054, 990, 1005, 1033, 1037, 1001, 1021, 1012, 1001, 993, 996, 1076, 1041, 1086, 1067, 993, 997, 1020, 1060, 1021, 1066, 1023, 992, 994, 1007, 991, 1018, 1020, 1007, 1024, 1012, 1031, 1007, 1053, 1081, 1010, 1049, 1033, 1017, 1080, 1001, 1024, 1008, 1028, 998, 1014, 984, 1091, 1025, 968, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1024, 1028, 1003, 1005, 1048, 1050, 996, 997, 1024, 1015, 994, 1036, 988, 1017, 1026, 1010, 1083, 1012, 989, 1049, 978, 1057, 1057, 1056, 1029, 1035, 1031, 1002, 1019, 1016, 993, 1019, 1004, 1036, 1047, 962, 1035, 971, 1080, 962, 1039, 1013, 991, 1063, 1111, 998, 1013, 1037, 1059, 994, 1013, 1031, 996, 1068, 1018, 1008, 1063, 1066, 1019, 1019, 1023, 1049, 1025, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1023, 1084, 1011, 1030, 1064, 1067, 1079, 1010, 1047, 1042, 1040, 988, 1033, 1014, 996, 977, 1072, 992, 1015, 1017, 1025, 1036, 986, 1004, 977, 1012, 1032, 1002, 978, 991, 1020, 1035, 1086, 1044, 1006, 994, 1035, 1025, 1049, 1062, 1023, 1028, 993, 1021, 999, 1008, 1022, 1089, 1020, 1010, 1013, 1034, 1074, 1018, 1041, 1020, 1025, 1017, 987, 1037, 1002, 1027, 1003, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[993, 1004, 1054, 1000, 1016, 1082, 983, 1079, 1067, 1012, 996, 1036, 1045, 1027, 1068, 994, 1006, 996, 1004, 985, 1005, 999, 1059, 1007, 1040, 1035, 1029, 1037, 1025, 1038, 1009, 983, 1030, 1052, 1030, 1033, 1037, 999, 997, 1053, 1025, 995, 1009, 1053, 999, 1028, 989, 1021, 1049, 1038, 982, 1020, 1072, 1031, 1004, 981, 1055, 1041, 1001, 1036, 1058, 1041, 1030, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[999, 1018, 1026, 1030, 1029, 998, 1009, 1008, 1014, 1042, 1044, 1023, 1022, 1039, 1011, 991, 1061, 987, 1025, 1008, 1038, 971, 998, 1059, 1024, 1008, 998, 1098, 1014, 1025, 1043, 1044, 985, 1063, 1053, 1043, 1025, 987, 1067, 1037, 1019, 1023, 1013, 1034, 1037, 1051, 1070, 1056, 1010, 988, 1060, 1052, 953, 985, 1053, 1004, 1016, 1041, 990, 1041, 949, 1001, 1042, 1084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1025, 1038, 1016, 979, 1057, 1012, 1064, 1036, 1050, 1027, 1066, 1054, 1074, 1026, 991, 1002, 1029, 1018, 1003, 1016, 1107, 989, 984, 1066, 1009, 1010, 991, 1035, 980, 964, 1024, 1001, 998, 976, 999, 1020, 1006, 1063, 1016, 1053, 1088, 1033, 1062, 1009, 1031, 1045, 977, 1067, 1019, 1050, 1022, 1016, 1015, 1015, 1048, 1007, 987, 1045, 1054, 976, 1042, 1028, 1000, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[999, 1030, 1037, 978, 980, 1003, 1057, 1009, 956, 1010, 1026, 1061, 1031, 1030, 1038, 1012, 1059, 1022, 1033, 1058, 1005, 979, 1017, 1033, 1037, 1013, 1042, 1028, 1004, 1011, 1035, 1020, 1056, 1046, 947, 1040, 992, 995, 1065, 1031, 1071, 997, 1004, 999, 1030, 1004, 1028, 1027, 1016, 1001, 1054, 1038, 1055, 1024, 1047, 1029, 988, 1049, 1011, 1022, 1013, 1073, 1075, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1023, 1084, 1011, 1030, 1064, 1067, 1077, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1042, 1040, 988, 1033, 1014, 996, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 992, 1015, 1017, 1025, 1036, 986, 1004, 18, 1, 0, 0, 0, 0, 0, 0, 977, 1012, 1032, 1002, 978, 991, 1020, 1035, 22, 0, 0, 0, 0, 0, 0, 0, 1086, 1044, 1006, 994, 1035, 1025, 1049, 1062, 2, 1, 0, 0, 0, 0, 0, 0, 1023, 1028, 993, 1021, 999, 1008, 1022, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1010, 1013, 1034, 1074, 1018, 1041, 1020, 5, 0, 0, 0, 0, 0, 0, 0, 1025, 1010, 987, 1037, 1002, 1027, 1003, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8366, 8137, 8166, 8069, 8304, 8143, 8235, 8116] avg=8192.0000 max=8366 max_rank=0 min=8069 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1024, 1028, 1003, 1005, 1048, 1050, 994, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1015, 994, 1036, 988, 1017, 1026, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 1012, 989, 1049, 978, 1057, 1057, 1056, 17, 1, 0, 0, 0, 0, 0, 0, 1029, 1035, 1031, 1002, 1019, 1016, 993, 1019, 21, 0, 0, 0, 0, 0, 0, 0, 1004, 1036, 1047, 962, 1035, 971, 1080, 962, 2, 1, 0, 0, 0, 0, 0, 0, 1039, 1013, 991, 1063, 1111, 998, 1013, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 994, 1013, 1031, 996, 1068, 1018, 1008, 5, 0, 0, 0, 0, 0, 0, 0, 1063, 1059, 1019, 1019, 1023, 1049, 1025, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8110, 8299, 8165, 8100, 8227, 8192, 8294] avg=8192.0000 max=8299 max_rank=2 min=8100 min_rank=4 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1014, 1002, 977, 1041, 1045, 1038, 1032, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1036, 1032, 1005, 1036, 974, 1061, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1006, 1022, 998, 1069, 1048, 992, 1079, 17, 1, 0, 0, 0, 0, 0, 0, 1029, 1021, 1015, 1035, 1093, 974, 999, 1010, 21, 0, 0, 0, 0, 0, 0, 0, 1046, 1041, 1061, 987, 1035, 985, 1036, 1002, 2, 1, 0, 0, 0, 0, 0, 0, 978, 999, 1052, 1091, 1077, 996, 1038, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 982, 1008, 1001, 983, 1104, 1011, 993, 5, 0, 0, 0, 0, 0, 0, 0, 1052, 1023, 1048, 1034, 992, 1023, 1032, 965, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8184, 8250, 8197, 8196, 8237, 8119, 8169] avg=8192.0000 max=8250 max_rank=2 min=8119 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[993, 1004, 1054, 1000, 1016, 1082, 981, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1012, 996, 1036, 1045, 1027, 1068, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 996, 1004, 985, 1005, 999, 1059, 1007, 17, 1, 0, 0, 0, 0, 0, 0, 1040, 1035, 1029, 1037, 1025, 1038, 1009, 983, 21, 0, 0, 0, 0, 0, 0, 0, 1030, 1052, 1030, 1033, 1037, 999, 997, 1053, 2, 1, 0, 0, 0, 0, 0, 0, 1025, 995, 1009, 1053, 999, 1028, 989, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1038, 982, 1020, 1072, 1031, 1004, 981, 5, 0, 0, 0, 0, 0, 0, 0, 1055, 1034, 1001, 1036, 1058, 1041, 1030, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8245, 8079, 8217, 8234, 8081, 8182, 8289] avg=8192.0000 max=8289 max_rank=7 min=8079 min_rank=2 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1052, 1068, 1019, 999, 1023, 994, 1056, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1043, 1021, 1050, 1044, 1054, 990, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1037, 1001, 1021, 1012, 1001, 993, 996, 18, 1, 0, 0, 0, 0, 0, 0, 1076, 1041, 1086, 1067, 993, 997, 1020, 1060, 22, 0, 0, 0, 0, 0, 0, 0, 1021, 1066, 1023, 992, 994, 1007, 991, 1018, 2, 1, 0, 0, 0, 0, 0, 0, 1020, 1007, 1024, 1012, 1031, 1007, 1053, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1049, 1033, 1017, 1080, 1001, 1024, 1008, 5, 0, 0, 0, 0, 0, 0, 0, 1028, 991, 1014, 984, 1091, 1025, 968, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8224, 8113, 8362, 8115, 8195, 8227, 8099] avg=8192.0000 max=8362 max_rank=3 min=8099 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1025, 1038, 1016, 979, 1057, 1012, 1062, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1027, 1066, 1054, 1074, 1026, 991, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1018, 1003, 1016, 1107, 989, 984, 1066, 17, 1, 0, 0, 0, 0, 0, 0, 1009, 1010, 991, 1035, 980, 964, 1024, 1001, 22, 0, 0, 0, 0, 0, 0, 0, 998, 976, 999, 1020, 1006, 1063, 1016, 1053, 2, 1, 0, 0, 0, 0, 0, 0, 1088, 1033, 1062, 1009, 1031, 1045, 977, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1050, 1022, 1016, 1015, 1015, 1048, 1007, 5, 0, 0, 0, 0, 0, 0, 0, 987, 1038, 1054, 976, 1042, 1028, 1000, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8290, 8230, 8036, 8134, 8273, 8197, 8151] avg=8192.0000 max=8290 max_rank=1 min=8036 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[999, 1018, 1026, 1030, 1029, 998, 1007, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1042, 1044, 1023, 1022, 1039, 1011, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 987, 1025, 1008, 1038, 971, 998, 1059, 17, 1, 0, 0, 0, 0, 0, 0, 1024, 1008, 998, 1098, 1014, 1025, 1043, 1044, 21, 0, 0, 0, 0, 0, 0, 0, 985, 1063, 1053, 1043, 1025, 987, 1067, 1037, 2, 1, 0, 0, 0, 0, 0, 0, 1019, 1023, 1013, 1034, 1037, 1051, 1070, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 988, 1060, 1052, 953, 985, 1053, 1004, 5, 0, 0, 0, 0, 0, 0, 0, 1016, 1034, 990, 1041, 949, 1001, 1042, 1084, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8186, 8165, 8275, 8263, 8265, 8110, 8157] avg=8192.0000 max=8275 max_rank=3 min=8110 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[999, 1030, 1037, 978, 980, 1003, 1051, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1010, 1026, 1061, 1027, 1030, 1038, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1022, 1033, 1058, 1005, 979, 1017, 1033, 20, 8, 0, 0, 0, 0, 0, 0, 1037, 1013, 1042, 1028, 1004, 1011, 1035, 1020, 25, 0, 0, 0, 0, 0, 0, 0, 1056, 1046, 947, 1040, 992, 995, 1065, 1031, 6, 8, 4, 0, 0, 0, 0, 0, 1071, 997, 1004, 999, 1030, 1004, 1028, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1001, 1054, 1038, 1055, 1024, 1047, 1029, 10, 0, 0, 0, 0, 0, 0, 0, 988, 1023, 1011, 1022, 1013, 1073, 1075, 1056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8160, 8234, 8215, 8190, 8115, 8274, 8261] avg=8192.0000 max=8274 max_rank=6 min=8087 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8448, 8192, 8192, 8192, 8064, 8320, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8320, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8064, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8448, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1031, 981, 1017, 1006, 1065, 1001, 1008, 1019, 1043, 1025, 990, 1014, 1020, 1025, 998, 979, 996, 1073, 1001, 1040, 983, 1061, 1066, 1056, 1051, 1006, 1018, 1048, 1017, 1038, 1023, 1020, 1037, 1002, 1051, 997, 1020, 1066, 1005, 997, 1059, 995, 1079, 1051, 1014, 1031, 1016, 1006, 1058, 1012, 1047, 993, 1052, 1003, 1025, 1043, 959, 1049, 1027, 1034, 1061, 1020, 1014, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[966, 1028, 1101, 1031, 968, 1008, 1033, 995, 1005, 1047, 1041, 1016, 1028, 1085, 1050, 1039, 1036, 1059, 992, 1062, 994, 1014, 1008, 973, 1057, 1036, 1007, 1025, 1032, 1034, 1039, 1006, 980, 1015, 1054, 1025, 1059, 1038, 975, 1009, 1030, 1048, 987, 1001, 1040, 989, 1013, 1002, 978, 997, 1013, 1032, 1054, 1023, 1047, 1035, 1046, 996, 1068, 1060, 1010, 1017, 1061, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1028, 1009, 997, 1038, 1084, 1020, 988, 1012, 986, 1021, 993, 1002, 1044, 1032, 1003, 995, 1063, 992, 1008, 987, 1029, 1071, 1068, 1034, 1062, 1019, 1028, 1000, 970, 1058, 1031, 1036, 1084, 1024, 1051, 971, 1023, 1064, 966, 1029, 1038, 1041, 993, 1044, 1022, 1055, 1051, 976, 1005, 1024, 1068, 1002, 1083, 1009, 1043, 994, 1008, 1001, 1048, 1044, 1001, 1010, 1046, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1042, 1017, 1048, 998, 1037, 1012, 968, 1028, 1013, 1033, 1046, 1005, 1008, 1021, 1033, 1012, 1034, 1031, 987, 1060, 991, 1025, 1034, 1019, 983, 996, 1047, 1025, 1063, 1008, 1052, 1076, 996, 1034, 1028, 1017, 1034, 1029, 1016, 1043, 1034, 1018, 1026, 1032, 1009, 1014, 1000, 1031, 997, 1041, 1010, 1053, 1066, 1012, 958, 1068, 1040, 1048, 1033, 1042, 1004, 1011, 1036, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1008, 1057, 1020, 1070, 1025, 970, 1031, 1060, 1047, 1033, 983, 993, 1041, 1015, 1032, 1023, 1053, 1032, 1035, 1003, 1045, 1030, 1077, 1019, 1014, 1045, 1043, 1011, 1025, 996, 1030, 992, 1028, 1034, 1037, 995, 1050, 1032, 1021, 1039, 1018, 982, 1006, 1055, 1049, 993, 1040, 1015, 1049, 999, 1056, 962, 1048, 960, 978, 1071, 966, 1036, 1075, 1016, 1034, 1017, 1027, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[966, 989, 1054, 1039, 1015, 976, 1010, 1049, 1034, 1007, 1021, 1030, 986, 1092, 1000, 1025, 1015, 1006, 1045, 971, 1030, 1038, 1029, 991, 1055, 1005, 995, 993, 1056, 995, 1062, 1049, 1012, 1001, 1026, 1008, 966, 1029, 1084, 1010, 1018, 997, 1036, 1010, 1079, 1018, 1081, 1029, 1049, 1042, 1042, 1052, 993, 1024, 1030, 1038, 1026, 1012, 1014, 1059, 1081, 1020, 1022, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1018, 1022, 1058, 1021, 1014, 998, 1028, 1013, 1040, 1000, 1051, 1042, 995, 1027, 1060, 1046, 1047, 1000, 1001, 1044, 1059, 1008, 1002, 1011, 995, 988, 980, 1010, 1039, 1044, 980, 1044, 1018, 1055, 1019, 1027, 1001, 999, 1025, 1024, 1018, 1045, 983, 1033, 1028, 994, 1033, 1014, 1051, 1014, 1001, 1044, 1044, 1058, 1034, 1094, 1047, 1032, 1001, 1008, 1064, 1025, 999, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1021, 1061, 1067, 1025, 1011, 989, 1001, 1024, 1025, 1037, 1043, 965, 1009, 996, 1025, 1046, 1043, 1035, 1026, 1035, 966, 1011, 972, 1029, 999, 1016, 1062, 1010, 1040, 1010, 1086, 1069, 1045, 1053, 981, 1006, 1016, 1031, 1022, 1035, 1060, 1004, 1022, 1022, 1016, 1029, 988, 1039, 1062, 1015, 1045, 1010, 1008, 1004, 1041, 1020, 1021, 1042, 956, 1042, 1039, 1013, 1019, 1076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1031, 981, 1017, 1006, 1065, 1001, 1008, 1019, 30, 0, 0, 0, 0, 0, 0, 0, 1043, 1025, 990, 1014, 1020, 1025, 998, 979, 13, 0, 0, 0, 0, 0, 0, 0, 994, 1073, 1001, 1040, 983, 1061, 1066, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1006, 1018, 1048, 1017, 1038, 1010, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1002, 1051, 997, 1020, 1066, 1005, 997, 8, 0, 0, 0, 0, 0, 0, 0, 1059, 995, 1079, 1051, 1014, 1031, 1016, 1006, 2, 2, 0, 0, 0, 0, 0, 0, 1058, 1012, 1047, 993, 1052, 1003, 1025, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 959, 1049, 1027, 1024, 1061, 1020, 1014, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8107, 8274, 8208, 8183, 8255, 8203, 8148] avg=8192.0000 max=8274 max_rank=2 min=8107 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[966, 1028, 1101, 1031, 968, 1008, 1033, 995, 29, 0, 0, 0, 0, 0, 0, 0, 1005, 1047, 1041, 1016, 1028, 1085, 1050, 1039, 13, 0, 0, 0, 0, 0, 0, 0, 1034, 1059, 992, 1062, 994, 1014, 1008, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1036, 1007, 1025, 1032, 1034, 1026, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1015, 1054, 1025, 1059, 1038, 975, 1009, 8, 0, 0, 0, 0, 0, 0, 0, 1030, 1048, 987, 1001, 1040, 989, 1013, 1002, 2, 2, 0, 0, 0, 0, 0, 0, 978, 997, 1013, 1032, 1054, 1023, 1047, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 996, 1068, 1050, 1010, 1017, 1061, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8324, 8136, 8223, 8163, 8114, 8150, 8267] avg=8192.0000 max=8324 max_rank=1 min=8114 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[966, 989, 1054, 1039, 1015, 976, 1010, 1049, 30, 0, 0, 0, 0, 0, 0, 0, 1034, 1007, 1021, 1030, 986, 1092, 1000, 1025, 14, 0, 0, 0, 0, 0, 0, 0, 1013, 1006, 1045, 971, 1030, 1038, 1029, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1005, 995, 993, 1056, 995, 1048, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1001, 1026, 1008, 966, 1029, 1084, 1010, 8, 0, 0, 0, 0, 0, 0, 0, 1018, 997, 1036, 1010, 1079, 1018, 1081, 1029, 2, 2, 0, 0, 0, 0, 0, 0, 1049, 1042, 1042, 1052, 993, 1024, 1030, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1012, 1014, 1049, 1081, 1020, 1022, 1000, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8209, 8123, 8196, 8144, 8272, 8240, 8224] avg=8192.0000 max=8272 max_rank=5 min=8123 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1042, 1017, 1048, 998, 1037, 1012, 968, 1028, 30, 0, 0, 0, 0, 0, 0, 0, 1013, 1033, 1046, 1005, 1008, 1021, 1033, 1012, 13, 0, 0, 0, 0, 0, 0, 0, 1032, 1031, 987, 1060, 991, 1025, 1034, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 983, 996, 1047, 1025, 1063, 1008, 1039, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1034, 1028, 1017, 1034, 1029, 1016, 1043, 8, 0, 0, 0, 0, 0, 0, 0, 1034, 1018, 1026, 1032, 1009, 1014, 1000, 1031, 2, 2, 0, 0, 0, 0, 0, 0, 997, 1041, 1010, 1053, 1066, 1012, 958, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1048, 1033, 1032, 1004, 1011, 1036, 1004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8184, 8179, 8237, 8205, 8168, 8175, 8208] avg=8192.0000 max=8237 max_rank=3 min=8168 min_rank=5 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1028, 1009, 997, 1038, 1084, 1020, 988, 1012, 28, 0, 0, 0, 0, 0, 0, 0, 986, 1021, 993, 1002, 1044, 1032, 1003, 995, 13, 0, 0, 0, 0, 0, 0, 0, 1061, 992, 1008, 987, 1029, 1071, 1068, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1019, 1028, 1000, 970, 1058, 1018, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 1024, 1051, 971, 1023, 1064, 966, 1029, 8, 0, 0, 0, 0, 0, 0, 0, 1038, 1041, 993, 1044, 1022, 1055, 1051, 976, 2, 2, 0, 0, 0, 0, 0, 0, 1005, 1024, 1068, 1002, 1083, 1009, 1043, 966, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1001, 1048, 1034, 1001, 1010, 1046, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8089, 8250, 8191, 8220, 8224, 8200, 8158] avg=8192.0000 max=8250 max_rank=2 min=8089 min_rank=1 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1008, 1057, 1020, 1070, 1025, 970, 1031, 1060, 30, 0, 0, 0, 0, 0, 0, 0, 1047, 1033, 983, 993, 1041, 1015, 1032, 1023, 13, 0, 0, 0, 0, 0, 0, 0, 1051, 1032, 1035, 1003, 1045, 1030, 1077, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1045, 1043, 1011, 1025, 996, 1017, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1034, 1037, 995, 1050, 1032, 1021, 1039, 7, 0, 0, 0, 0, 0, 0, 0, 1018, 982, 1006, 1055, 1049, 993, 1040, 1015, 2, 2, 0, 0, 0, 0, 0, 0, 1049, 999, 1056, 962, 1048, 960, 978, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1036, 1075, 1007, 1034, 1017, 1027, 990, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8180, 8292, 8143, 8243, 8162, 8093, 8152] avg=8192.0000 max=8292 max_rank=2 min=8093 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1018, 1022, 1058, 1021, 1014, 998, 1028, 1013, 31, 0, 0, 0, 0, 0, 0, 0, 1040, 1000, 1051, 1042, 995, 1027, 1060, 1046, 12, 0, 0, 0, 0, 0, 0, 0, 1045, 1000, 1001, 1044, 1059, 1008, 1002, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 995, 988, 980, 1010, 1039, 1044, 968, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1055, 1019, 1027, 1001, 999, 1025, 1024, 7, 0, 0, 0, 0, 0, 0, 0, 1018, 1045, 983, 1033, 1028, 994, 1033, 1014, 2, 2, 0, 0, 0, 0, 0, 0, 1051, 1014, 1001, 1044, 1044, 1058, 1034, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1032, 1001, 999, 1064, 1025, 999, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8273, 8170, 8068, 8175, 8152, 8309, 8186] avg=8192.0000 max=8309 max_rank=6 min=8068 min_rank=3 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1021, 1061, 1067, 1025, 1011, 989, 1001, 1024, 34, 0, 0, 0, 0, 0, 0, 0, 1025, 1037, 1043, 965, 1009, 996, 1025, 1046, 19, 5, 0, 0, 0, 0, 0, 0, 1038, 1035, 1026, 1035, 966, 1011, 972, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1016, 1062, 1010, 1040, 1010, 1064, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1053, 981, 1006, 1016, 1031, 1022, 1035, 11, 3, 0, 0, 0, 0, 0, 0, 1060, 1004, 1022, 1022, 1016, 1029, 988, 1039, 5, 4, 0, 0, 0, 0, 0, 0, 1062, 1015, 1045, 1010, 1008, 1004, 1041, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1042, 956, 1027, 1039, 1013, 1019, 1076, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8170, 8112, 8270, 8203, 8189, 8166, 8193] avg=8192.0000 max=8270 max_rank=3 min=8112 min_rank=2 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8448, 8320, 8320, 8064, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8064, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8448, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[994, 987, 1034, 1055, 986, 1035, 1056, 1048, 1045, 1018, 1051, 980, 999, 1071, 1070, 988, 1018, 937, 1001, 1061, 987, 1008, 1082, 1005, 1047, 1009, 1044, 1012, 953, 1029, 954, 1048, 1028, 999, 1039, 1039, 1016, 1021, 1026, 1075, 1026, 1005, 1022, 982, 1052, 1041, 1050, 1015, 1041, 1069, 1046, 1064, 1016, 987, 994, 1025, 1039, 1027, 1018, 1038, 1054, 1037, 1044, 989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1038, 962, 982, 1023, 1062, 1043, 1020, 1041, 1006, 1009, 1035, 1009, 1047, 1016, 1044, 1006, 1067, 1025, 988, 1030, 1036, 988, 998, 1074, 1034, 1033, 992, 995, 1082, 1069, 1034, 1081, 1047, 1042, 1056, 1037, 1005, 992, 1045, 1054, 1036, 985, 1016, 1024, 992, 1009, 1022, 1040, 1044, 1017, 1023, 980, 980, 1006, 996, 1040, 1012, 1006, 1026, 1036, 1007, 1017, 1028, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1003, 1016, 987, 1023, 1010, 1011, 1035, 1040, 963, 1020, 987, 1044, 973, 975, 1054, 1001, 1041, 1052, 1034, 1055, 1017, 1057, 992, 1011, 1031, 1005, 990, 1039, 1044, 1003, 1031, 1070, 1000, 1059, 1047, 1018, 1007, 1024, 1053, 1017, 996, 1029, 1000, 1001, 1028, 1039, 1032, 1003, 1044, 1023, 1021, 1018, 1061, 1020, 1043, 1067, 1017, 1041, 1076, 982, 1008, 1081, 1045, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[988, 1004, 1051, 984, 1050, 1018, 981, 970, 1044, 981, 1055, 1006, 996, 1019, 1025, 1041, 1051, 1012, 1027, 1027, 1044, 1032, 1060, 1058, 1038, 983, 979, 974, 1025, 1083, 983, 1002, 1018, 1078, 1028, 998, 1040, 1006, 1050, 1056, 992, 1025, 1021, 1076, 1037, 1047, 1027, 1069, 1067, 1008, 1020, 1027, 1020, 1029, 1002, 1018, 968, 1021, 1025, 1030, 1062, 1021, 1065, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[916, 1046, 1013, 1015, 1043, 1047, 1015, 1017, 1038, 1059, 1029, 1058, 996, 994, 988, 995, 998, 993, 1056, 1037, 1030, 1005, 1055, 1012, 1057, 1034, 1005, 1038, 996, 1003, 1011, 1028, 1045, 1031, 1044, 1072, 1038, 1024, 1050, 1055, 1043, 1018, 1071, 1010, 999, 1021, 986, 999, 1043, 1001, 1000, 1042, 1022, 1043, 1003, 1008, 974, 1045, 1029, 1036, 1029, 1027, 1055, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1037, 1008, 1013, 1038, 1055, 1046, 1050, 1072, 1068, 958, 1023, 996, 980, 1054, 1030, 1023, 1020, 1050, 993, 1050, 1052, 983, 1014, 1029, 1060, 1002, 1000, 1032, 1047, 982, 1067, 1032, 1045, 1064, 1019, 1054, 998, 1048, 1001, 1039, 967, 1051, 1032, 1022, 1039, 1015, 990, 1004, 957, 1041, 999, 950, 999, 1064, 1041, 1062, 1038, 1005, 1023, 1031, 1056, 997, 979, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1020, 974, 993, 1014, 971, 992, 1052, 979, 991, 1038, 1000, 1044, 1014, 1053, 1025, 1053, 1029, 1000, 1028, 952, 1000, 1026, 1028, 1036, 1036, 1016, 1015, 1068, 1012, 1082, 1054, 1015, 1033, 994, 1001, 1018, 999, 1019, 1039, 1047, 1041, 1092, 1004, 1014, 1006, 1000, 1030, 994, 998, 1010, 1060, 1030, 1037, 1093, 1024, 1062, 1039, 994, 1064, 1012, 1025, 1036, 1074, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1059, 1031, 1011, 1019, 1021, 1017, 1021, 1001, 1047, 1034, 1024, 1012, 1064, 1023, 1044, 1046, 1053, 975, 970, 994, 992, 1046, 1034, 1056, 1000, 1047, 1080, 984, 976, 1050, 988, 1084, 988, 1059, 1032, 1047, 1035, 1043, 992, 1033, 1021, 1014, 1015, 1040, 1035, 1046, 1064, 1038, 1001, 1042, 1039, 1004, 1043, 1042, 1036, 1054, 1019, 942, 1001, 1027, 1004, 985, 1018, 974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[994, 987, 1034, 1055, 986, 1035, 1056, 1048, 50, 0, 0, 0, 0, 0, 0, 0, 1045, 1018, 1051, 980, 999, 1071, 1070, 988, 12, 7, 0, 0, 0, 0, 0, 0, 1018, 937, 1001, 1061, 987, 1008, 1082, 1005, 6, 1, 0, 0, 0, 0, 0, 0, 1047, 1009, 1044, 1012, 953, 1029, 954, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 999, 1039, 1039, 1016, 1021, 1026, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1005, 1022, 982, 1052, 1041, 1050, 1015, 8, 0, 0, 0, 0, 0, 0, 0, 1041, 1069, 1046, 1064, 1016, 987, 994, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1027, 1018, 1038, 1054, 1037, 1043, 989, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8241, 8106, 8090, 8181, 8201, 8227, 8245] avg=8192.0000 max=8245 max_rank=0 min=8090 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1003, 1016, 987, 1023, 1010, 1011, 1035, 1040, 47, 0, 0, 0, 0, 0, 0, 0, 963, 1020, 987, 1044, 973, 975, 1054, 1001, 11, 7, 0, 0, 0, 0, 0, 0, 1041, 1052, 1034, 1055, 1017, 1057, 992, 1011, 7, 1, 0, 0, 0, 0, 0, 0, 1031, 1005, 990, 1039, 1044, 1003, 1031, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1059, 1047, 1018, 1007, 1024, 1053, 959, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1029, 1000, 1001, 1028, 1039, 1032, 1003, 8, 0, 0, 0, 0, 0, 0, 0, 1044, 1023, 1021, 1018, 1061, 1020, 1043, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1041, 1076, 982, 1008, 1081, 1044, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8035, 8267, 8206, 8167, 8136, 8282, 8271] avg=8192.0000 max=8282 max_rank=6 min=8035 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1038, 962, 982, 1023, 1062, 1043, 1020, 1041, 49, 0, 0, 0, 0, 0, 0, 0, 1006, 1009, 1035, 1009, 1047, 1016, 1044, 1006, 12, 7, 0, 0, 0, 0, 0, 0, 1067, 1025, 988, 1030, 1036, 988, 998, 1074, 7, 1, 0, 0, 0, 0, 0, 0, 1034, 1033, 992, 995, 1082, 1069, 1034, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1042, 1056, 1037, 1005, 992, 1045, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 985, 1016, 1024, 992, 1009, 1022, 1040, 8, 0, 0, 0, 0, 0, 0, 0, 1044, 1017, 1023, 980, 980, 1006, 996, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1006, 1026, 1036, 1007, 1017, 1027, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8191, 8214, 8313, 8217, 8132, 8071, 8178] avg=8192.0000 max=8313 max_rank=3 min=8071 min_rank=6 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32598 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[988, 1004, 1051, 984, 1050, 1018, 981, 970, 49, 0, 0, 0, 0, 0, 0, 0, 1044, 981, 1055, 1006, 996, 1019, 1025, 1041, 12, 7, 0, 0, 0, 0, 0, 0, 1051, 1012, 1027, 1027, 1044, 1032, 1060, 1058, 6, 1, 0, 0, 0, 0, 0, 0, 1038, 983, 979, 974, 1025, 1083, 983, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1078, 1028, 998, 1040, 1006, 1050, 995, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1025, 1021, 1076, 1037, 1047, 1027, 1069, 8, 0, 0, 0, 0, 0, 0, 0, 1067, 1008, 1020, 1027, 1020, 1029, 1002, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1021, 1025, 1030, 1062, 1021, 1064, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8186, 8318, 8061, 8213, 8302, 8176, 8185] avg=8192.0000 max=8318 max_rank=2 min=8061 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1020, 974, 993, 1014, 971, 992, 1052, 979, 49, 0, 0, 0, 0, 0, 0, 0, 991, 1038, 1000, 1044, 1014, 1053, 1025, 1053, 12, 7, 0, 0, 0, 0, 0, 0, 1029, 1000, 1028, 952, 1000, 1026, 1028, 1036, 6, 1, 0, 0, 0, 0, 0, 0, 1036, 1016, 1015, 1068, 1012, 1082, 1054, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 994, 1001, 1018, 999, 1019, 1039, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1092, 1004, 1014, 1006, 1000, 1030, 994, 8, 0, 0, 0, 0, 0, 0, 0, 998, 1010, 1060, 1030, 1037, 1093, 1024, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 994, 1064, 1012, 1025, 1036, 1073, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8044, 8237, 8106, 8292, 8089, 8189, 8299, 8280] avg=8192.0000 max=8299 max_rank=6 min=8044 min_rank=0 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[916, 1046, 1013, 1015, 1043, 1047, 1015, 1017, 49, 0, 0, 0, 0, 0, 0, 0, 1038, 1059, 1029, 1058, 996, 994, 988, 995, 12, 7, 0, 0, 0, 0, 0, 0, 998, 993, 1056, 1037, 1030, 1005, 1055, 1012, 6, 1, 0, 0, 0, 0, 0, 0, 1057, 1034, 1005, 1038, 996, 1003, 1011, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1031, 1044, 1072, 1038, 1024, 1050, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1018, 1071, 1010, 999, 1021, 986, 999, 8, 0, 0, 0, 0, 0, 0, 0, 1043, 1001, 1000, 1042, 1022, 1043, 1003, 993, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1045, 1029, 1036, 1029, 1027, 1054, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8176, 8193, 8166, 8298, 8155, 8147, 8240] avg=8192.0000 max=8298 max_rank=4 min=8147 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1037, 1008, 1013, 1038, 1055, 1046, 1050, 1072, 48, 0, 0, 0, 0, 0, 0, 0, 1068, 958, 1023, 996, 980, 1054, 1030, 1023, 12, 7, 0, 0, 0, 0, 0, 0, 1020, 1050, 993, 1050, 1052, 983, 1014, 1029, 6, 1, 0, 0, 0, 0, 0, 0, 1060, 1002, 1000, 1032, 1047, 982, 1067, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1064, 1019, 1054, 998, 1048, 1001, 979, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1051, 1032, 1022, 1039, 1015, 990, 1004, 8, 0, 0, 0, 0, 0, 0, 0, 957, 1041, 999, 950, 999, 1064, 1041, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1005, 1023, 1031, 1056, 997, 978, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8367, 8151, 8198, 8216, 8208, 8128, 8098, 8170] avg=8192.0000 max=8367 max_rank=0 min=8098 min_rank=6 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32604 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1059, 1031, 1011, 1019, 1021, 1017, 1021, 1001, 52, 0, 0, 0, 0, 0, 0, 0, 1047, 1034, 1024, 1012, 1064, 1023, 1044, 1046, 14, 11, 0, 0, 0, 0, 0, 0, 1053, 975, 970, 994, 992, 1046, 1034, 1056, 11, 3, 0, 0, 0, 0, 0, 0, 1000, 1047, 1080, 984, 976, 1050, 988, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1059, 1032, 1047, 1035, 1043, 992, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1014, 1015, 1040, 1035, 1046, 1064, 1038, 14, 6, 0, 0, 0, 0, 0, 0, 1001, 1042, 1039, 1004, 1043, 1042, 1036, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 942, 1001, 1027, 1004, 985, 1015, 974, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8319, 8134, 8192, 8163, 8293, 8236, 7967] avg=8192.0000 max=8319 max_rank=1 min=7967 min_rank=7 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8448, 8320, 8320, 8192, 8192, 8320, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8064, 8192, 8192, 8320, 8320, 8320, 8192, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8192, 8320, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1003, 998, 1004, 1058, 1047, 1001, 1031, 1002, 1043, 1026, 1005, 1009, 1028, 1096, 998, 1056, 1022, 1002, 1021, 1022, 1003, 1006, 953, 980, 1018, 1057, 1007, 1019, 1022, 992, 1029, 1017, 1023, 1004, 987, 1061, 982, 1018, 1029, 1012, 1088, 1106, 1064, 1032, 1022, 1022, 1050, 991, 1050, 1025, 998, 1019, 1008, 1070, 1041, 993, 1057, 1069, 1024, 1030, 1036, 980, 1049, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1009, 1002, 1019, 1040, 974, 1055, 1005, 1019, 981, 984, 1035, 1004, 1022, 1039, 1060, 1020, 1011, 989, 1036, 1083, 1016, 981, 1052, 993, 1038, 1022, 1026, 1046, 1019, 999, 1064, 1037, 1015, 1036, 1015, 1026, 1051, 996, 1017, 1030, 1074, 970, 1062, 1003, 991, 1014, 1041, 1033, 1049, 1054, 1013, 992, 1072, 1041, 1032, 968, 1044, 1001, 996, 1043, 1024, 1016, 1068, 1069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[975, 967, 1036, 996, 1003, 1004, 1034, 1077, 1021, 1073, 1007, 1052, 1012, 1043, 1032, 1040, 1006, 1006, 1074, 1019, 1042, 995, 1020, 1006, 1000, 990, 1091, 997, 1005, 1040, 1064, 1040, 1042, 1047, 1020, 978, 1019, 1018, 1020, 1022, 1050, 1042, 1066, 1004, 1003, 988, 958, 1044, 1039, 1057, 1036, 1010, 1052, 1004, 1014, 1012, 1028, 995, 995, 1027, 1074, 1024, 1015, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1016, 1017, 994, 1007, 1000, 973, 1067, 1031, 984, 1042, 1039, 1044, 1106, 1011, 1010, 1018, 1061, 1011, 1052, 1054, 1038, 1007, 999, 1061, 1021, 1016, 1066, 1080, 982, 1009, 1012, 1051, 1015, 1016, 1006, 1018, 1012, 1065, 990, 1034, 1062, 1015, 1036, 1017, 1017, 1011, 999, 1012, 1013, 1031, 988, 1033, 1038, 1032, 995, 1086, 984, 984, 1023, 1049, 1011, 970, 1027, 1068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1039, 1050, 1028, 1012, 1018, 1045, 1014, 994, 1053, 1057, 1060, 1027, 953, 974, 994, 984, 1018, 1043, 1074, 1081, 968, 995, 1053, 1025, 1047, 1014, 1026, 1038, 986, 1046, 1038, 1041, 990, 1011, 1064, 1036, 1047, 1035, 1035, 1043, 1020, 968, 1020, 975, 985, 1066, 1014, 1017, 1042, 1028, 1011, 1018, 1036, 1048, 978, 1016, 1023, 1034, 1052, 1038, 1014, 1028, 1004, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1022, 1009, 1051, 1014, 1010, 1004, 1036, 1028, 1035, 1002, 969, 1007, 1008, 1001, 973, 1011, 1034, 1017, 1047, 951, 1018, 1034, 1015, 1051, 1067, 1034, 1008, 992, 997, 1061, 1057, 1008, 1026, 1088, 993, 1039, 1022, 1059, 999, 1069, 1048, 992, 1061, 1083, 988, 1029, 1012, 1016, 1045, 1067, 1035, 1050, 999, 1058, 1020, 1050, 1057, 979, 1002, 1054, 1041, 1034, 980, 970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1050, 1009, 1015, 1015, 1032, 1013, 973, 999, 1049, 985, 1030, 1007, 973, 1026, 1035, 991, 1039, 1055, 1025, 1047, 1025, 1041, 1001, 1015, 1041, 1023, 1025, 1016, 1030, 1018, 1051, 1046, 1062, 1000, 1108, 1039, 1010, 1033, 1057, 989, 1026, 1003, 1013, 1056, 985, 1053, 982, 1032, 1042, 1026, 1016, 1021, 1024, 1008, 1030, 983, 1031, 1059, 1046, 1004, 1024, 1002, 1014, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1072, 1046, 1017, 1060, 1035, 1032, 972, 1051, 1055, 1067, 1077, 1062, 1041, 1012, 1053, 1021, 983, 1004, 1014, 1047, 981, 1073, 963, 1097, 1019, 992, 989, 1004, 982, 988, 984, 1024, 1027, 1005, 997, 1011, 992, 1017, 1094, 994, 1025, 1018, 1032, 1070, 1027, 962, 1005, 1024, 1002, 1019, 1012, 1040, 1042, 1021, 1037, 1035, 1060, 1008, 1008, 979, 1057, 1007, 1077, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1009, 1002, 1019, 1040, 974, 1055, 1005, 1019, 23, 15, 0, 0, 0, 0, 0, 0, 981, 984, 1035, 1004, 1022, 1039, 1060, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 989, 1036, 1083, 1016, 981, 1052, 993, 8, 2, 0, 0, 0, 0, 0, 0, 1038, 1022, 1026, 1046, 1019, 999, 1061, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1036, 1015, 1026, 1051, 996, 1008, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 970, 1062, 1003, 991, 1014, 1041, 1033, 3, 1, 0, 0, 0, 0, 0, 0, 1049, 1031, 1013, 992, 1072, 1041, 1032, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1001, 996, 1043, 1024, 1016, 1068, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8145, 8171, 8248, 8177, 8192, 8198, 8244] avg=8192.0000 max=8248 max_rank=3 min=8145 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1003, 998, 1004, 1058, 1047, 1001, 1031, 1002, 23, 14, 0, 0, 0, 0, 0, 0, 1043, 1026, 1005, 1009, 1028, 1096, 998, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1002, 1021, 1022, 1003, 1006, 953, 980, 8, 1, 0, 0, 0, 0, 0, 0, 1018, 1057, 1007, 1019, 1022, 992, 1026, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1004, 987, 1061, 982, 1018, 1020, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 1106, 1064, 1032, 1022, 1022, 1050, 991, 3, 1, 0, 0, 0, 0, 0, 0, 1050, 1002, 998, 1019, 1008, 1070, 1041, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1069, 1024, 1030, 1036, 980, 1049, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8261, 8018, 8158, 8107, 8379, 8181, 8251] avg=8192.0000 max=8379 max_rank=5 min=8018 min_rank=2 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1039, 1050, 1028, 1012, 1018, 1045, 1014, 994, 23, 14, 0, 0, 0, 0, 0, 0, 1053, 1057, 1060, 1027, 953, 974, 994, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1043, 1074, 1081, 968, 995, 1053, 1025, 8, 2, 0, 0, 0, 0, 0, 0, 1047, 1014, 1026, 1038, 986, 1046, 1035, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1011, 1064, 1036, 1047, 1035, 1026, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 968, 1020, 975, 985, 1066, 1014, 1017, 3, 1, 0, 0, 0, 0, 0, 0, 1042, 1005, 1011, 1018, 1036, 1048, 978, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1034, 1052, 1038, 1014, 1028, 1004, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8102, 8267, 8233, 8252, 8069, 8154, 8222] avg=8192.0000 max=8267 max_rank=2 min=8069 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32697 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[975, 967, 1036, 996, 1003, 1004, 1034, 1077, 23, 15, 0, 0, 0, 0, 0, 0, 1021, 1073, 1007, 1052, 1012, 1043, 1032, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1006, 1074, 1019, 1042, 995, 1020, 1006, 8, 2, 0, 0, 0, 0, 0, 0, 1000, 990, 1091, 997, 1005, 1040, 1061, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1047, 1020, 978, 1019, 1018, 1011, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1042, 1066, 1004, 1003, 988, 958, 1044, 3, 1, 0, 0, 0, 0, 0, 0, 1039, 1034, 1036, 1010, 1052, 1004, 1014, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 995, 995, 1027, 1074, 1024, 1015, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8280, 8178, 8224, 8157, 8159, 8201, 8207] avg=8192.0000 max=8280 max_rank=1 min=8130 min_rank=0 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1016, 1017, 994, 1007, 1000, 973, 1067, 1031, 23, 15, 0, 0, 0, 0, 0, 0, 984, 1042, 1039, 1044, 1106, 1011, 1010, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1011, 1052, 1054, 1038, 1007, 999, 1061, 7, 2, 0, 0, 0, 0, 0, 0, 1021, 1016, 1066, 1080, 982, 1009, 1009, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1016, 1006, 1018, 1012, 1065, 982, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1015, 1036, 1017, 1017, 1011, 999, 1012, 3, 1, 0, 0, 0, 0, 0, 0, 1013, 1008, 988, 1033, 1038, 1032, 995, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 984, 984, 1023, 1049, 1011, 970, 1027, 1051, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8254, 8292, 8234, 8148, 8173, 8193, 8099] avg=8192.0000 max=8292 max_rank=2 min=8099 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32613 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1050, 1009, 1015, 1015, 1032, 1013, 973, 999, 23, 15, 0, 0, 0, 0, 0, 0, 1049, 985, 1030, 1007, 973, 1026, 1035, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1055, 1025, 1047, 1025, 1041, 1001, 1015, 8, 2, 0, 0, 0, 0, 0, 0, 1041, 1023, 1025, 1016, 1030, 1018, 1048, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1000, 1108, 1039, 1010, 1033, 1048, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1003, 1013, 1056, 985, 1053, 982, 1032, 3, 1, 0, 0, 0, 0, 0, 0, 1042, 1003, 1016, 1021, 1024, 1008, 1030, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1059, 1046, 1004, 1024, 1002, 1014, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8096, 8258, 8247, 8289, 8154, 8127, 8221] avg=8192.0000 max=8289 max_rank=4 min=8096 min_rank=1 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1022, 1009, 1051, 1014, 1010, 1004, 1036, 1028, 24, 13, 0, 0, 0, 0, 0, 0, 1035, 1002, 969, 1007, 1008, 1001, 973, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1017, 1047, 951, 1018, 1034, 1015, 1051, 7, 1, 0, 0, 0, 0, 0, 0, 1067, 1034, 1008, 992, 997, 1061, 1054, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1088, 993, 1039, 1022, 1059, 991, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 992, 1061, 1083, 988, 1029, 1012, 1016, 3, 1, 0, 0, 0, 0, 0, 0, 1045, 1043, 1035, 1050, 999, 1058, 1020, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 979, 1002, 1054, 1041, 1034, 980, 956, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8006, 8175, 8221, 8287, 8233, 8300, 8103] avg=8192.0000 max=8300 max_rank=6 min=8006 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32923 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1072, 1046, 1017, 1060, 1035, 1032, 972, 1051, 26, 18, 0, 0, 0, 0, 0, 0, 1055, 1067, 1077, 1062, 1041, 1012, 1053, 1021, 4, 0, 0, 0, 0, 0, 0, 0, 983, 1004, 1014, 1047, 981, 1073, 963, 1097, 11, 4, 0, 0, 0, 0, 0, 0, 1019, 992, 989, 1004, 982, 988, 973, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1005, 997, 1011, 992, 1017, 1076, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1018, 1032, 1070, 1027, 962, 1005, 1024, 7, 7, 0, 0, 0, 0, 0, 0, 1002, 993, 1012, 1040, 1042, 1021, 1037, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1008, 1008, 979, 1057, 1007, 1077, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8329, 8392, 8177, 7971, 8119, 8177, 8182, 8189] avg=8192.0000 max=8392 max_rank=1 min=7971 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=57144 max_load_reduction_pct=0.871948 moved_to_echo=32667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8064, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8448, 8320, 8064, 8192, 8064, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8448, 8320, 8192, 8192, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8064, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[989, 976, 998, 996, 1035, 1058, 1016, 974, 1022, 1058, 1041, 1051, 1007, 1023, 1068, 1018, 1067, 1072, 1075, 1027, 1009, 1010, 1017, 988, 1041, 1065, 979, 989, 1020, 983, 1016, 969, 1006, 1014, 987, 996, 1047, 1020, 1023, 1022, 1016, 1023, 1057, 978, 1043, 1022, 1006, 1003, 1076, 1001, 1013, 1006, 1069, 1036, 1029, 1048, 1050, 1029, 1030, 971, 1060, 1085, 1048, 1065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1010, 988, 1068, 995, 1029, 1009, 1015, 1013, 1022, 998, 1071, 1035, 1066, 1028, 1006, 1028, 1043, 1043, 992, 1059, 1025, 1022, 1058, 1049, 1036, 1082, 1025, 977, 1024, 1014, 1016, 1060, 1029, 1011, 968, 1055, 981, 998, 1009, 928, 1011, 1028, 1037, 1020, 985, 1078, 1026, 1009, 1019, 1021, 1097, 967, 1015, 1095, 992, 1020, 1009, 1060, 1085, 1006, 981, 1049, 1000, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[996, 1013, 1023, 1025, 1035, 1041, 1030, 1010, 1044, 1003, 989, 1054, 990, 1039, 1030, 1049, 1058, 1063, 976, 1015, 1063, 1040, 1056, 1036, 1019, 1015, 1059, 1041, 1028, 1027, 1029, 1046, 1045, 1051, 1036, 1003, 1025, 1001, 1037, 1042, 994, 1005, 1065, 1044, 1036, 1013, 997, 1042, 1034, 974, 960, 1066, 1056, 997, 1022, 963, 1014, 1026, 1055, 1000, 950, 1015, 1015, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1010, 1036, 1048, 1063, 1054, 958, 1068, 1032, 1009, 1013, 1046, 950, 1067, 1054, 1029, 1019, 1025, 1041, 983, 1057, 995, 1036, 1045, 1047, 1027, 971, 1026, 1073, 1027, 1033, 1011, 1044, 1010, 993, 1005, 1022, 1010, 959, 1014, 1091, 1044, 1031, 992, 991, 1002, 1042, 1004, 1052, 1032, 980, 1036, 1002, 1053, 1029, 1036, 1029, 1033, 997, 1026, 1051, 997, 1007, 1066, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1021, 1012, 977, 1007, 995, 1020, 1017, 1039, 1044, 1026, 1024, 1057, 1067, 986, 1061, 1008, 1009, 1050, 993, 1002, 998, 1029, 1022, 962, 1052, 1041, 1027, 1030, 1057, 1006, 1027, 1036, 1053, 1010, 1043, 1021, 1046, 1068, 1033, 1038, 1014, 963, 1040, 1003, 1030, 990, 1002, 1097, 1000, 985, 1085, 1048, 1058, 1075, 1023, 1021, 993, 1052, 1007, 1023, 1004, 1001, 1014, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[993, 1095, 1030, 1029, 1036, 1010, 1027, 1007, 1046, 1068, 1039, 978, 959, 1029, 1041, 1001, 997, 1105, 1030, 1034, 1055, 1029, 976, 1073, 1029, 1075, 1067, 1067, 995, 1001, 961, 1079, 1021, 993, 1047, 1002, 1059, 1048, 988, 1041, 1011, 1031, 986, 1002, 1023, 956, 1045, 1028, 1008, 1025, 1026, 1025, 1005, 1056, 1014, 1038, 1041, 1010, 968, 1015, 974, 1006, 1042, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1033, 1013, 1007, 1024, 1013, 1029, 1012, 1060, 1008, 1050, 978, 1063, 1017, 1002, 1056, 1047, 1060, 1035, 1054, 1025, 1011, 1012, 1026, 1013, 1014, 1058, 1003, 1047, 1004, 1019, 1012, 1011, 1023, 981, 986, 1012, 982, 1031, 1011, 1079, 1040, 1007, 1048, 994, 1060, 1018, 995, 1078, 1019, 1069, 1002, 1039, 1073, 1017, 1030, 1053, 1031, 983, 1008, 987, 1003, 987, 1041, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1032, 983, 1016, 1019, 971, 1052, 1059, 1030, 1029, 1024, 982, 1026, 1081, 1016, 994, 1037, 958, 1043, 1014, 1026, 1055, 1054, 983, 984, 1025, 974, 1065, 1015, 1058, 1041, 969, 1076, 1034, 1053, 1032, 1049, 1037, 1047, 1053, 1004, 1004, 1008, 1012, 1001, 985, 1044, 1007, 1060, 1029, 1073, 992, 982, 1012, 1092, 1061, 998, 1010, 1021, 1027, 1029, 1026, 1022, 1020, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[989, 976, 998, 996, 1035, 1058, 1016, 974, 30, 2, 0, 0, 0, 0, 0, 0, 1022, 1058, 1041, 1051, 1007, 1023, 1038, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1039, 1075, 1027, 1009, 1010, 1017, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1065, 979, 989, 1020, 983, 1016, 949, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1014, 987, 996, 1047, 1020, 1023, 1022, 20, 0, 0, 0, 0, 0, 0, 0, 1016, 1023, 1057, 978, 1043, 1022, 1006, 1003, 31, 0, 0, 0, 0, 0, 0, 0, 1076, 1001, 1013, 1006, 1069, 1003, 1029, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1029, 1030, 971, 1060, 1085, 1048, 1065, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8074, 8258, 8232, 8042, 8135, 8179, 8245, 8371] avg=8192.0000 max=8371 max_rank=7 min=8042 min_rank=3 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32930 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1010, 988, 1068, 995, 1029, 1009, 1015, 1013, 28, 2, 0, 0, 0, 0, 0, 0, 1022, 998, 1071, 1035, 1066, 1028, 978, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1011, 992, 1059, 1025, 1022, 1058, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1082, 1025, 977, 1024, 1014, 1016, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1011, 968, 1055, 981, 998, 1009, 928, 22, 0, 0, 0, 0, 0, 0, 0, 1011, 1028, 1037, 1020, 985, 1078, 1026, 1009, 32, 0, 0, 0, 0, 0, 0, 0, 1019, 1021, 1097, 967, 1015, 1061, 992, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1060, 1085, 1006, 981, 1049, 1000, 1041, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8226, 8259, 8212, 8001, 8226, 8192, 8263] avg=8192.0000 max=8263 max_rank=7 min=8001 min_rank=4 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[996, 1013, 1023, 1025, 1035, 1041, 1030, 1010, 29, 2, 0, 0, 0, 0, 0, 0, 1044, 1003, 989, 1054, 990, 1039, 1001, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1030, 976, 1015, 1063, 1040, 1056, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1015, 1059, 1041, 1028, 1027, 1029, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1051, 1036, 1003, 1025, 1001, 1037, 1042, 21, 0, 0, 0, 0, 0, 0, 0, 994, 1005, 1065, 1044, 1036, 1013, 997, 1042, 29, 0, 0, 0, 0, 0, 0, 0, 1034, 974, 960, 1066, 1056, 966, 1022, 963, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1026, 1055, 1000, 950, 1015, 1015, 1011, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8169, 8274, 8243, 8261, 8225, 8041, 8119] avg=8192.0000 max=8274 max_rank=2 min=8041 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1032, 983, 1016, 1019, 971, 1052, 1059, 1030, 28, 2, 0, 0, 0, 0, 0, 0, 1029, 1024, 982, 1026, 1081, 1016, 966, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 958, 1011, 1014, 1026, 1055, 1054, 983, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 974, 1065, 1015, 1058, 1041, 969, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1053, 1032, 1049, 1037, 1047, 1053, 1004, 22, 0, 0, 0, 0, 0, 0, 0, 1004, 1008, 1012, 1001, 985, 1044, 1007, 1060, 32, 0, 0, 0, 0, 0, 0, 0, 1029, 1073, 992, 982, 1012, 1058, 1061, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1021, 1027, 1029, 1026, 1022, 1020, 1021, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8161, 8085, 8201, 8331, 8153, 8205, 8208] avg=8192.0000 max=8331 max_rank=4 min=8085 min_rank=2 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1010, 1036, 1048, 1063, 1054, 958, 1068, 1032, 29, 2, 0, 0, 0, 0, 0, 0, 1009, 1013, 1046, 950, 1067, 1054, 1000, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1009, 983, 1057, 995, 1036, 1045, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 971, 1026, 1073, 1027, 1033, 1011, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 993, 1005, 1022, 1010, 959, 1014, 1091, 21, 0, 0, 0, 0, 0, 0, 0, 1044, 1031, 992, 991, 1002, 1042, 1004, 1052, 30, 0, 0, 0, 0, 0, 0, 0, 1032, 980, 1036, 1002, 1053, 997, 1036, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 997, 1026, 1051, 997, 1007, 1066, 1003, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8158, 8197, 8191, 8125, 8188, 8165, 8212] avg=8192.0000 max=8300 max_rank=0 min=8125 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[993, 1095, 1030, 1029, 1036, 1010, 1027, 1007, 29, 2, 0, 0, 0, 0, 0, 0, 1046, 1068, 1039, 978, 959, 1029, 1012, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1071, 1030, 1034, 1055, 1029, 976, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1075, 1067, 1067, 995, 1001, 961, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 993, 1047, 1002, 1059, 1048, 988, 1041, 22, 0, 0, 0, 0, 0, 0, 0, 1011, 1031, 986, 1002, 1023, 956, 1045, 1028, 31, 0, 0, 0, 0, 0, 0, 0, 1008, 1025, 1026, 1025, 1005, 1023, 1014, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1010, 968, 1015, 974, 1006, 1042, 1041, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8132, 8265, 8252, 8221, 8113, 8164, 8131] avg=8192.0000 max=8265 max_rank=2 min=8113 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1033, 1013, 1007, 1024, 1013, 1029, 1012, 1060, 32, 4, 4, 0, 0, 0, 0, 0, 1008, 1050, 978, 1063, 1017, 1002, 1024, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 991, 1054, 1025, 1011, 1012, 1026, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1058, 1003, 1047, 1004, 1019, 1012, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 981, 986, 1012, 982, 1031, 1011, 1079, 24, 0, 0, 0, 0, 0, 0, 0, 1040, 1007, 1048, 994, 1060, 1018, 995, 1078, 35, 6, 0, 0, 0, 0, 0, 0, 1019, 1069, 1002, 1039, 1073, 978, 1030, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 983, 1008, 987, 1003, 987, 1041, 1033, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8189, 8192, 8140, 8129, 8281, 8263, 8111] avg=8192.0000 max=8281 max_rank=5 min=8111 min_rank=7 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1021, 1012, 977, 1007, 995, 1020, 1017, 1039, 30, 2, 0, 0, 0, 0, 0, 0, 1044, 1026, 1024, 1057, 1067, 986, 1031, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1017, 993, 1002, 998, 1029, 1022, 962, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1041, 1027, 1030, 1057, 1006, 1027, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1010, 1043, 1021, 1046, 1068, 1033, 1038, 21, 0, 0, 0, 0, 0, 0, 0, 1014, 963, 1040, 1003, 1030, 990, 1002, 1097, 32, 0, 0, 0, 0, 0, 0, 0, 1000, 985, 1085, 1048, 1058, 1041, 1023, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1052, 1007, 1023, 1004, 1001, 1014, 994, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8243, 8032, 8255, 8333, 8171, 8261, 8121] avg=8192.0000 max=8333 max_rank=4 min=8032 min_rank=2 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32886 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8064, 8192, 8320, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8064, 8192, 8192, 8192, 8448, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8320, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8448, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1049, 1038, 1035, 994, 1032, 1032, 997, 1037, 1037, 999, 956, 996, 1004, 1027, 1028, 1027, 1054, 1034, 1010, 1089, 1025, 1006, 994, 1013, 1078, 1036, 1008, 1000, 1039, 1015, 1024, 1054, 961, 1061, 1010, 1056, 1011, 1027, 997, 990, 988, 1020, 1045, 992, 1044, 1040, 1045, 1069, 1058, 1030, 1010, 1043, 1026, 1003, 1025, 1016, 1069, 1036, 994, 1030, 1000, 1075, 1013, 985] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1030, 1047, 972, 1007, 1039, 1047, 1029, 1058, 991, 993, 1023, 1047, 1037, 1031, 1034, 1039, 1029, 1001, 1022, 1049, 1060, 989, 1052, 1050, 998, 1082, 1019, 974, 1049, 1014, 1002, 1038, 1002, 1004, 1035, 1052, 1028, 1019, 983, 996, 1035, 1020, 1019, 977, 1048, 1040, 991, 987, 978, 1018, 1009, 1025, 1050, 1065, 1036, 1021, 993, 1014, 1052, 1027, 1019, 1041, 1038, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1003, 1039, 1012, 977, 966, 971, 1038, 1026, 1061, 1021, 1019, 1018, 1016, 1035, 1005, 1049, 1057, 1019, 1058, 997, 1069, 986, 987, 996, 1045, 1040, 1022, 1025, 991, 982, 1046, 1010, 1031, 1013, 1051, 1055, 1060, 1028, 1070, 1066, 973, 991, 1026, 1024, 1052, 1038, 993, 1049, 1030, 1047, 1037, 993, 1015, 1032, 1045, 1006, 1053, 1011, 1025, 1015, 1018, 995, 1072, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[981, 991, 1022, 1003, 1046, 1027, 1032, 1038, 1056, 996, 1040, 1003, 996, 1035, 1018, 999, 1043, 1052, 973, 983, 997, 1057, 1084, 1042, 1045, 1027, 1039, 986, 1050, 1059, 1022, 1038, 1018, 1053, 1013, 1044, 996, 1030, 1067, 996, 1021, 1004, 1013, 1036, 1008, 1073, 979, 1038, 1043, 1033, 983, 1013, 1053, 1018, 1028, 1052, 1022, 1044, 1024, 995, 1013, 994, 1032, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1030, 1016, 1024, 1013, 1026, 1034, 1044, 1030, 1002, 998, 989, 960, 1016, 1012, 993, 1034, 978, 998, 1069, 1043, 1046, 1054, 1042, 1033, 999, 1025, 1039, 965, 1059, 1028, 983, 1029, 1010, 990, 1044, 1041, 1007, 1039, 999, 1097, 1013, 1038, 1063, 981, 1031, 1028, 1086, 1039, 1002, 1046, 1005, 1070, 1052, 958, 1013, 1059, 1061, 1051, 1039, 1056, 958, 1067, 1021, 961] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1006, 990, 1009, 1049, 1041, 1036, 1039, 993, 1035, 1021, 1016, 1056, 1060, 1009, 1028, 1082, 1012, 1001, 1059, 1011, 1028, 998, 1017, 1040, 1008, 1019, 1004, 1045, 1035, 1014, 992, 1031, 1019, 1026, 1000, 962, 968, 1060, 1030, 1033, 1026, 1024, 998, 1034, 1043, 1044, 1025, 1058, 1023, 1044, 1041, 1049, 1024, 1026, 1003, 1027, 1028, 1010, 1050, 1020, 1022, 1014, 986, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1014, 1007, 1050, 1031, 1001, 966, 984, 1001, 1017, 994, 983, 1023, 1031, 1002, 1042, 1013, 1050, 993, 933, 995, 1041, 1041, 1045, 1122, 1020, 1027, 1090, 1039, 1024, 1060, 1006, 1012, 1022, 1048, 993, 1023, 1009, 1068, 1053, 1038, 997, 1102, 1044, 1001, 1022, 1009, 1026, 1001, 1055, 999, 1040, 1010, 982, 1025, 1005, 1060, 1025, 1025, 1034, 1013, 1087, 1026, 1005, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[984, 1064, 1018, 1028, 1049, 998, 1042, 1049, 996, 1023, 1016, 1011, 1078, 1024, 1025, 1021, 979, 1057, 1051, 963, 961, 1027, 1034, 1035, 1069, 1041, 960, 1033, 1009, 1073, 960, 1034, 1014, 1059, 974, 1076, 1050, 1048, 996, 1044, 1016, 1025, 1053, 980, 982, 972, 1039, 1018, 1028, 1026, 1059, 1038, 1019, 1041, 1029, 1040, 1004, 1024, 1058, 1011, 996, 1025, 1011, 1069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[981, 991, 1022, 1003, 1046, 1027, 1032, 1038, 12, 12, 4, 1, 0, 0, 0, 0, 1056, 996, 1040, 1003, 996, 1035, 1018, 999, 25, 10, 0, 0, 0, 0, 0, 0, 1043, 1052, 973, 983, 997, 1057, 1084, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1022, 1039, 986, 1050, 1059, 1022, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1053, 1013, 1044, 996, 1016, 1067, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1004, 1013, 1036, 1008, 1073, 979, 1038, 4, 0, 0, 0, 0, 0, 0, 0, 1043, 1033, 983, 1013, 1053, 1018, 1028, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1044, 1012, 995, 1013, 994, 1032, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8178, 8219, 8261, 8203, 8176, 8198, 8132] avg=8192.0000 max=8261 max_rank=3 min=8132 min_rank=7 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1049, 1038, 1035, 994, 1032, 1032, 997, 1037, 12, 11, 4, 1, 0, 0, 0, 0, 1037, 999, 956, 996, 1004, 1027, 1028, 1027, 24, 10, 0, 0, 0, 0, 0, 0, 1054, 1034, 1010, 1089, 1025, 1006, 994, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1031, 1008, 1000, 1039, 1015, 1024, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 961, 1061, 1010, 1056, 1011, 1013, 997, 990, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1020, 1045, 992, 1044, 1040, 1045, 1069, 4, 0, 0, 0, 0, 0, 0, 0, 1058, 1030, 1010, 1043, 1026, 1003, 1025, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1036, 982, 1030, 1000, 1075, 1013, 985, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8108, 8214, 8249, 8099, 8247, 8187, 8190] avg=8192.0000 max=8249 max_rank=3 min=8099 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1030, 1047, 972, 1007, 1039, 1047, 1029, 1058, 13, 12, 4, 1, 0, 0, 0, 0, 991, 993, 1023, 1047, 1037, 1031, 1034, 1039, 25, 10, 0, 0, 0, 0, 0, 0, 1029, 1001, 1022, 1049, 1060, 989, 1052, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1076, 1019, 974, 1049, 1014, 1002, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1004, 1035, 1052, 1028, 1005, 983, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1020, 1019, 977, 1048, 1040, 991, 987, 5, 0, 0, 0, 0, 0, 0, 0, 978, 1018, 1009, 1025, 1050, 1065, 1036, 996, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1014, 1039, 1027, 1019, 1041, 1038, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8230, 8240, 8170, 8105, 8122, 8177, 8233] avg=8192.0000 max=8259 max_rank=0 min=8105 min_rank=4 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1003, 1039, 1012, 977, 966, 971, 1038, 1026, 13, 11, 4, 1, 0, 0, 0, 0, 1061, 1021, 1019, 1018, 1016, 1035, 1005, 1049, 24, 10, 0, 0, 0, 0, 0, 0, 1057, 1019, 1058, 997, 1069, 986, 987, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1034, 1022, 1025, 991, 982, 1046, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1013, 1051, 1055, 1060, 1014, 1070, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 973, 991, 1026, 1024, 1052, 1038, 993, 1049, 5, 0, 0, 0, 0, 0, 0, 0, 1030, 1047, 1037, 993, 1015, 1032, 1045, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1011, 1012, 1015, 1018, 995, 1072, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8061, 8258, 8158, 8155, 8360, 8151, 8181, 8212] avg=8192.0000 max=8360 max_rank=4 min=8061 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1030, 1016, 1024, 1013, 1026, 1034, 1044, 1030, 16, 14, 9, 6, 0, 0, 0, 0, 1002, 998, 989, 960, 1016, 1012, 993, 1034, 30, 15, 0, 0, 0, 0, 0, 0, 978, 998, 1069, 1043, 1046, 1054, 1042, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1010, 1039, 965, 1059, 1028, 983, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 990, 1044, 1041, 1007, 1015, 999, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1038, 1063, 981, 1031, 1028, 1086, 1039, 9, 0, 0, 0, 0, 0, 0, 0, 1002, 1046, 1005, 1070, 1052, 958, 1013, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1051, 1023, 1056, 958, 1067, 1021, 961, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8049, 8249, 8112, 8203, 8288, 8175, 8198] avg=8192.0000 max=8288 max_rank=5 min=8049 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1006, 990, 1009, 1049, 1041, 1036, 1039, 993, 13, 12, 5, 1, 0, 0, 0, 0, 1035, 1021, 1016, 1056, 1060, 1009, 1028, 1082, 25, 11, 0, 0, 0, 0, 0, 0, 1012, 1001, 1059, 1011, 1028, 998, 1017, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1014, 1004, 1045, 1035, 1014, 992, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1026, 1000, 962, 968, 1044, 1030, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1024, 998, 1034, 1043, 1044, 1025, 1058, 4, 0, 0, 0, 0, 0, 0, 0, 1023, 1044, 1041, 1049, 1024, 1026, 1003, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1010, 1037, 1020, 1022, 1014, 986, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8343, 8154, 8143, 8082, 8256, 8212, 8152] avg=8192.0000 max=8343 max_rank=1 min=8082 min_rank=4 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1014, 1007, 1050, 1031, 1001, 966, 984, 1001, 13, 13, 5, 1, 0, 0, 0, 0, 1017, 994, 983, 1023, 1031, 1002, 1042, 1013, 25, 11, 0, 0, 0, 0, 0, 0, 1050, 993, 933, 995, 1041, 1041, 1045, 1109, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1022, 1090, 1039, 1024, 1060, 1006, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1048, 993, 1023, 1009, 1052, 1053, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1102, 1044, 1001, 1022, 1009, 1026, 1001, 4, 0, 0, 0, 0, 0, 0, 0, 1055, 999, 1040, 1010, 982, 1025, 1005, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1025, 1021, 1013, 1087, 1026, 1005, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 8141, 8207, 8273, 8238, 8206, 8151, 8234] avg=8192.0000 max=8273 max_rank=3 min=8086 min_rank=0 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[984, 1064, 1018, 1028, 1049, 998, 1042, 1049, 13, 12, 5, 1, 0, 0, 0, 0, 996, 1023, 1016, 1011, 1078, 1024, 1025, 1021, 25, 10, 0, 0, 0, 0, 0, 0, 979, 1057, 1051, 963, 961, 1027, 1034, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1035, 960, 1033, 1009, 1073, 960, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1059, 974, 1076, 1050, 1033, 996, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1025, 1053, 980, 982, 972, 1039, 1018, 5, 0, 0, 0, 0, 0, 0, 0, 1028, 1026, 1059, 1038, 1019, 1041, 1029, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1024, 1045, 1011, 996, 1025, 1011, 1069, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8229, 8095, 8173, 8246, 8090, 8255, 8185] avg=8192.0000 max=8263 max_rank=0 min=8090 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8064, 8320, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8064, 8192, 8320, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1014, 1035, 1020, 1007, 1069, 1005, 1072, 971, 1015, 1048, 985, 986, 1035, 1005, 1086, 999, 1000, 1034, 1041, 1001, 1003, 1040, 1039, 1063, 1035, 1011, 1039, 1070, 1039, 1078, 1007, 1010, 1021, 1059, 1011, 995, 1027, 1010, 1039, 1017, 1032, 1014, 1068, 985, 1028, 1027, 1021, 976, 1018, 974, 1038, 1032, 1029, 1030, 1047, 1039, 1036, 1010, 1012, 1038, 1014, 1020, 996, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1059, 1067, 1005, 987, 982, 985, 1066, 1023, 1046, 1079, 1015, 996, 982, 975, 1019, 1004, 1018, 1081, 1015, 1008, 1082, 1061, 1082, 983, 974, 1025, 1051, 1049, 1008, 1026, 1038, 1042, 1057, 1018, 1022, 1033, 996, 1038, 949, 1023, 1016, 1004, 994, 1018, 1078, 1037, 1011, 1026, 996, 1003, 1071, 1035, 1031, 1019, 1003, 1048, 1025, 1007, 1017, 999, 1013, 1040, 1062, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1020, 962, 1036, 1006, 1004, 1026, 997, 1008, 1073, 1090, 1033, 963, 1012, 1006, 1038, 1010, 1026, 980, 1043, 1039, 1014, 985, 1035, 1010, 1022, 1037, 1005, 1033, 1025, 1015, 1046, 1048, 1066, 985, 951, 1032, 1013, 1027, 1010, 1053, 1060, 1000, 1009, 1032, 990, 1080, 981, 1010, 1046, 1124, 1023, 1011, 1021, 1035, 983, 1067, 1040, 1038, 1022, 1015, 1065, 1057, 1006, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1048, 1013, 1027, 1032, 1029, 1027, 1015, 1046, 1023, 1041, 987, 1018, 1047, 1034, 1030, 1055, 986, 1046, 994, 1007, 959, 1014, 997, 999, 1030, 1051, 1029, 1055, 1024, 1017, 1017, 1003, 1031, 1037, 994, 1025, 1011, 1025, 1024, 977, 1010, 1025, 1036, 1065, 1015, 984, 1030, 1049, 1038, 1014, 1061, 1010, 1062, 1043, 998, 1008, 1015, 980, 1029, 1094, 1093, 991, 1027, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1026, 992, 1032, 1105, 1036, 1038, 1010, 1017, 1043, 1030, 1034, 1071, 1045, 973, 1058, 1040, 1034, 1056, 1064, 1000, 997, 990, 1001, 1005, 974, 1022, 1016, 1038, 1017, 1009, 1013, 1057, 1043, 1014, 1002, 990, 1001, 1032, 1042, 1021, 1002, 1001, 1056, 1051, 968, 1068, 1010, 992, 1093, 1032, 1001, 1041, 1035, 983, 1025, 1031, 1016, 1046, 1044, 1015, 999, 1004, 994, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[995, 1000, 1015, 1017, 1027, 1035, 1025, 1001, 1005, 1035, 1012, 1053, 990, 1004, 1014, 1050, 1001, 1044, 1012, 1110, 1034, 1028, 995, 1022, 1060, 1041, 999, 1077, 1055, 1077, 1044, 958, 1006, 1004, 1027, 1009, 1051, 984, 1022, 999, 968, 1026, 1069, 1030, 1042, 1041, 1013, 1020, 983, 1046, 1006, 1030, 1051, 1020, 1045, 1050, 1034, 1010, 1002, 1016, 1018, 1034, 1038, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1069, 993, 1026, 1024, 1042, 1021, 1016, 1032, 1023, 1013, 1039, 1020, 1050, 995, 1045, 1026, 1046, 1011, 1009, 997, 1044, 1004, 965, 1032, 1056, 1003, 994, 1006, 1011, 999, 1029, 1023, 1029, 1040, 1036, 1054, 1044, 1055, 1040, 1015, 1051, 973, 1010, 1054, 939, 1071, 1047, 1034, 1018, 997, 1044, 996, 1000, 1006, 1007, 1058, 1017, 1044, 1059, 1037, 1049, 1030, 1025, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[994, 1030, 1038, 1032, 1091, 985, 1036, 1032, 977, 1007, 1067, 996, 1003, 1017, 1001, 1034, 1079, 995, 1062, 992, 1006, 1017, 999, 993, 1040, 1049, 998, 1037, 1024, 1076, 1077, 1053, 1053, 1005, 997, 1003, 1043, 1006, 1038, 1015, 1044, 1041, 1002, 1072, 1013, 1056, 997, 1021, 1033, 1017, 990, 1055, 1031, 1032, 1003, 972, 1061, 997, 1030, 1049, 998, 972, 991, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[995, 1000, 1015, 1017, 1027, 1035, 1025, 1001, 2, 2, 0, 0, 0, 0, 0, 0, 1005, 1035, 1012, 1053, 990, 1004, 1014, 1050, 3, 0, 0, 0, 0, 0, 0, 0, 1001, 1044, 1012, 1110, 1034, 1028, 995, 1022, 14, 11, 0, 0, 0, 0, 0, 0, 1060, 1041, 999, 1033, 1055, 1077, 1044, 958, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1004, 1027, 1009, 1051, 984, 1022, 999, 30, 0, 0, 0, 0, 0, 0, 0, 968, 1026, 1069, 1030, 1042, 1041, 1013, 1020, 5, 0, 0, 0, 0, 0, 0, 0, 983, 1046, 1006, 1030, 1051, 1020, 1045, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1010, 1002, 1011, 1018, 1034, 1038, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8166, 8271, 8267, 8132, 8214, 8213, 8154] avg=8192.0000 max=8271 max_rank=2 min=8119 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1048, 1013, 1027, 1032, 1029, 1027, 1015, 1046, 3, 2, 0, 0, 0, 0, 0, 0, 1023, 1041, 987, 1018, 1047, 1034, 1030, 1055, 4, 0, 0, 0, 0, 0, 0, 0, 986, 1046, 994, 1007, 959, 1014, 997, 999, 14, 11, 0, 0, 0, 0, 0, 0, 1030, 1051, 1029, 1011, 1024, 1017, 1017, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1037, 994, 1025, 1011, 1025, 1024, 977, 30, 0, 0, 0, 0, 0, 0, 0, 1010, 1025, 1036, 1065, 1015, 984, 1030, 1049, 5, 0, 0, 0, 0, 0, 0, 0, 1038, 1014, 1061, 1010, 1062, 1043, 998, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 980, 1029, 1087, 1093, 991, 1027, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8239, 8027, 8182, 8154, 8219, 8216, 8257] avg=8192.0000 max=8257 max_rank=7 min=8027 min_rank=2 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1026, 992, 1032, 1105, 1036, 1038, 1010, 1017, 2, 2, 0, 0, 0, 0, 0, 0, 1043, 1030, 1034, 1071, 1045, 973, 1058, 1040, 3, 0, 0, 0, 0, 0, 0, 0, 1034, 1056, 1064, 1000, 997, 990, 1001, 1005, 14, 11, 0, 0, 0, 0, 0, 0, 974, 1022, 1016, 995, 1017, 1009, 1013, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1014, 1002, 990, 1001, 1032, 1042, 1021, 29, 0, 0, 0, 0, 0, 0, 0, 1002, 1001, 1056, 1051, 968, 1068, 1010, 992, 5, 0, 0, 0, 0, 0, 0, 0, 1093, 1032, 1001, 1041, 1035, 983, 1025, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1046, 1044, 1010, 999, 1004, 994, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8297, 8172, 8103, 8174, 8153, 8223, 8154] avg=8192.0000 max=8297 max_rank=1 min=8103 min_rank=3 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1014, 1035, 1020, 1007, 1069, 1005, 1072, 971, 3, 2, 0, 0, 0, 0, 0, 0, 1015, 1048, 985, 986, 1035, 1005, 1086, 999, 3, 0, 0, 0, 0, 0, 0, 0, 1000, 1034, 1041, 1001, 1003, 1040, 1039, 1063, 14, 11, 0, 0, 0, 0, 0, 0, 1035, 1011, 1039, 1026, 1039, 1078, 1007, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1059, 1011, 995, 1027, 1010, 1039, 1017, 30, 0, 0, 0, 0, 0, 0, 0, 1032, 1014, 1068, 985, 1028, 1027, 1021, 976, 5, 0, 0, 0, 0, 0, 0, 0, 1018, 974, 1038, 1032, 1029, 1030, 1047, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1010, 1012, 1032, 1014, 1020, 996, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8162, 8246, 8245, 8209, 8156, 8189, 8131] avg=8192.0000 max=8246 max_rank=2 min=8131 min_rank=7 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1059, 1067, 1005, 987, 982, 985, 1066, 1023, 2, 2, 0, 0, 0, 0, 0, 0, 1046, 1079, 1015, 996, 982, 975, 1019, 1004, 3, 0, 0, 0, 0, 0, 0, 0, 1018, 1081, 1015, 1008, 1082, 1061, 1082, 983, 14, 11, 0, 0, 0, 0, 0, 0, 974, 1025, 1051, 1005, 1008, 1026, 1038, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1018, 1022, 1033, 996, 1038, 949, 1023, 30, 0, 0, 0, 0, 0, 0, 0, 1016, 1004, 994, 1018, 1078, 1037, 1011, 1026, 5, 0, 0, 0, 0, 0, 0, 0, 996, 1003, 1071, 1035, 1031, 1019, 1003, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1007, 1017, 994, 1013, 1040, 1062, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8119, 8355, 8169, 8166, 8189, 8188, 8172] avg=8192.0000 max=8355 max_rank=2 min=8119 min_rank=1 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1020, 962, 1036, 1006, 1004, 1026, 997, 1008, 6, 4, 0, 0, 0, 0, 0, 0, 1073, 1090, 1033, 963, 1012, 1006, 1038, 1010, 9, 0, 0, 0, 0, 0, 0, 0, 1026, 980, 1043, 1039, 1014, 985, 1035, 1010, 18, 16, 0, 0, 0, 0, 0, 0, 1022, 1037, 1005, 981, 1025, 1015, 1046, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 985, 951, 1032, 1013, 1027, 1010, 1053, 34, 0, 0, 0, 0, 0, 0, 0, 1060, 1000, 1009, 1032, 990, 1080, 981, 1010, 8, 0, 0, 0, 0, 0, 0, 0, 1046, 1124, 1023, 1011, 1021, 1035, 983, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1038, 1022, 1000, 1065, 1057, 1006, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8069, 8234, 8166, 8179, 8171, 8170, 8282, 8265] avg=8192.0000 max=8282 max_rank=6 min=8069 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1069, 993, 1026, 1024, 1042, 1021, 1016, 1032, 3, 2, 0, 0, 0, 0, 0, 0, 1023, 1013, 1039, 1020, 1050, 995, 1045, 1026, 3, 0, 0, 0, 0, 0, 0, 0, 1046, 1011, 1009, 997, 1044, 1004, 965, 1032, 13, 11, 0, 0, 0, 0, 0, 0, 1056, 1003, 994, 965, 1011, 999, 1029, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1040, 1036, 1054, 1044, 1055, 1040, 1015, 28, 0, 0, 0, 0, 0, 0, 0, 1051, 973, 1010, 1054, 939, 1071, 1047, 1034, 5, 0, 0, 0, 0, 0, 0, 0, 1018, 997, 1044, 996, 1000, 1006, 1007, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1044, 1059, 1031, 1049, 1030, 1025, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8214, 8132, 8080, 8341, 8184, 8108, 8249] avg=8192.0000 max=8341 max_rank=4 min=8080 min_rank=3 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[994, 1030, 1038, 1032, 1091, 985, 1036, 1032, 3, 1, 0, 0, 0, 0, 0, 0, 977, 1007, 1067, 996, 1003, 1017, 1001, 1034, 3, 0, 0, 0, 0, 0, 0, 0, 1079, 995, 1062, 992, 1006, 1017, 999, 993, 14, 10, 0, 0, 0, 0, 0, 0, 1040, 1049, 998, 994, 1024, 1076, 1077, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1005, 997, 1003, 1043, 1006, 1038, 1015, 29, 0, 0, 0, 0, 0, 0, 0, 1044, 1041, 1002, 1072, 1013, 1056, 997, 1021, 5, 0, 0, 0, 0, 0, 0, 0, 1033, 1017, 990, 1055, 1031, 1032, 1003, 956, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 997, 1030, 1043, 998, 972, 991, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8105, 8167, 8311, 8189, 8251, 8117, 8154] avg=8192.0000 max=8311 max_rank=3 min=8105 min_rank=1 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8448, 8192, 8192, 8192, 8064, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8192, 8064, 8192, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8064, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8448, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[998, 1017, 1066, 1005, 1037, 1031, 1058, 1076, 1060, 1031, 1065, 930, 1035, 1070, 1042, 1011, 1008, 994, 988, 1040, 959, 1026, 1054, 1032, 1023, 1038, 1018, 1040, 998, 982, 1055, 991, 1022, 1013, 1041, 988, 997, 1060, 1067, 1048, 1013, 1028, 1054, 989, 1030, 1026, 1050, 1060, 996, 1004, 1060, 1047, 979, 1042, 1041, 982, 968, 979, 988, 1033, 1056, 1033, 1013, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1092, 1031, 958, 979, 1020, 1047, 1032, 1048, 1008, 1043, 1072, 979, 1023, 1056, 1025, 1013, 1064, 1017, 989, 1061, 1036, 1027, 1039, 1010, 1033, 1018, 989, 1012, 1011, 1029, 1046, 1041, 994, 1054, 1077, 963, 1013, 1016, 1075, 978, 1053, 1073, 990, 1012, 1005, 1035, 991, 1050, 975, 1036, 996, 1012, 1065, 1040, 1008, 985, 1043, 1011, 1008, 1035, 1029, 1056, 1010, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1012, 1000, 1014, 966, 1034, 1051, 1046, 1039, 1014, 987, 994, 1009, 1043, 1060, 997, 1006, 993, 1018, 1036, 1032, 1009, 1061, 1005, 1022, 1019, 953, 1016, 1060, 1054, 1033, 1015, 1027, 1067, 1028, 997, 1016, 988, 1028, 1038, 1031, 996, 1030, 1037, 1029, 1011, 1003, 1041, 1057, 1026, 1055, 1063, 1005, 975, 1029, 1082, 1044, 1029, 1021, 1064, 998, 1054, 1050, 1043, 976] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1018, 1044, 1009, 1046, 1001, 1059, 1026, 1026, 977, 1018, 1039, 1059, 1029, 1001, 990, 978, 994, 1005, 1056, 1017, 1013, 984, 1052, 996, 1018, 1045, 1031, 1034, 1066, 1007, 977, 996, 1006, 1004, 1017, 995, 1048, 1056, 1068, 1034, 1029, 1043, 1053, 1035, 1051, 1046, 1009, 970, 963, 1045, 1008, 1058, 1015, 1039, 1073, 1024, 1016, 1047, 1035, 1074, 1006, 1028, 997, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[991, 1064, 1040, 1043, 990, 1069, 994, 1025, 1052, 1068, 1074, 976, 1061, 1060, 1069, 1027, 1077, 1052, 1000, 1001, 1057, 1009, 1000, 1039, 1038, 1040, 1028, 1034, 1069, 1038, 1033, 1028, 1003, 1008, 973, 1040, 1061, 1014, 1000, 978, 956, 1002, 1032, 1026, 990, 1038, 1039, 1013, 1033, 1002, 1011, 1049, 1026, 1008, 1017, 1055, 988, 1001, 1025, 1001, 1001, 1013, 991, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1065, 973, 1091, 1013, 982, 1004, 975, 1021, 1017, 991, 1007, 1064, 1020, 1006, 1034, 1027, 1025, 995, 1006, 1034, 1038, 933, 1014, 1006, 1024, 1051, 970, 1051, 1068, 985, 1048, 1053, 1032, 1024, 1040, 1041, 1067, 1037, 1035, 1026, 1020, 1015, 1100, 993, 1042, 1042, 1046, 978, 1026, 980, 999, 1068, 982, 1053, 1048, 994, 1010, 1032, 1055, 1052, 1020, 991, 1067, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1058, 1005, 1044, 1056, 977, 989, 1022, 1036, 1036, 1011, 1019, 1058, 988, 1018, 995, 1023, 1032, 1089, 1019, 1003, 1054, 1040, 1022, 1018, 1044, 993, 1033, 1029, 1010, 1053, 1032, 1008, 1030, 1006, 1031, 1029, 1087, 1009, 1066, 968, 1004, 1005, 981, 1025, 1007, 1042, 1008, 991, 1047, 1081, 1041, 997, 1028, 1022, 996, 990, 967, 1006, 1014, 1012, 1057, 1045, 1054, 1076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1005, 999, 1037, 1088, 1051, 1031, 1039, 1004, 1028, 1046, 1047, 998, 1022, 1004, 1022, 1048, 1054, 974, 1025, 1016, 1033, 1045, 990, 990, 1038, 1050, 1049, 1038, 1017, 1008, 1011, 1009, 1070, 1052, 1055, 1002, 1017, 1026, 1035, 993, 965, 1071, 991, 1003, 1027, 1032, 1013, 1006, 1026, 1023, 1007, 1019, 1014, 993, 997, 1066, 1030, 1030, 1031, 1044, 1033, 1046, 983, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1012, 1000, 1014, 966, 1034, 1036, 1046, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 987, 990, 1009, 1043, 1060, 997, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1018, 1036, 1032, 1009, 1061, 1005, 1022, 18, 7, 0, 0, 0, 0, 0, 0, 1019, 953, 1016, 1046, 1054, 1033, 1015, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1028, 997, 1016, 988, 1028, 1020, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1030, 1037, 1029, 1011, 1003, 1041, 1057, 8, 8, 0, 0, 0, 0, 0, 0, 1026, 1055, 1063, 1005, 975, 1029, 1082, 1044, 6, 1, 0, 0, 0, 0, 0, 0, 1029, 1021, 1064, 998, 1054, 1050, 1043, 976, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8106, 8201, 8163, 8175, 8220, 8286, 8238] avg=8192.0000 max=8286 max_rank=6 min=8106 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32919 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1092, 1031, 958, 979, 1020, 1032, 1032, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1043, 1068, 979, 1023, 1056, 1025, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1017, 989, 1061, 1036, 1027, 1039, 1010, 18, 7, 0, 0, 0, 0, 0, 0, 1033, 1018, 989, 998, 1011, 1029, 1046, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1054, 1077, 963, 1013, 1016, 1057, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1073, 990, 1012, 1005, 1035, 991, 1050, 8, 8, 0, 0, 0, 0, 0, 0, 975, 1036, 996, 1012, 1065, 1040, 1008, 985, 6, 1, 0, 0, 0, 0, 0, 0, 1043, 1011, 1008, 1035, 1029, 1056, 1010, 1000, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8215, 8268, 8165, 8152, 8225, 8124, 8195] avg=8192.0000 max=8268 max_rank=2 min=8124 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[998, 1017, 1066, 1005, 1037, 1016, 1058, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1031, 1061, 930, 1035, 1070, 1042, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 994, 988, 1040, 959, 1026, 1054, 1032, 18, 7, 0, 0, 0, 0, 0, 0, 1023, 1038, 1018, 1026, 998, 982, 1055, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1013, 1041, 988, 997, 1060, 1049, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1028, 1054, 989, 1030, 1026, 1050, 1060, 8, 8, 0, 0, 0, 0, 0, 0, 996, 1004, 1060, 1047, 979, 1042, 1041, 982, 6, 1, 0, 0, 0, 0, 0, 0, 968, 979, 988, 1033, 1056, 1033, 1013, 1051, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8240, 8126, 8131, 8218, 8266, 8158, 8124] avg=8192.0000 max=8273 max_rank=0 min=8124 min_rank=7 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1018, 1044, 1009, 1046, 1001, 1043, 1026, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1018, 1035, 1059, 1029, 1001, 990, 978, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1005, 1056, 1017, 1013, 984, 1052, 996, 18, 8, 0, 0, 0, 0, 0, 0, 1018, 1045, 1031, 1020, 1066, 1007, 977, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1004, 1017, 995, 1048, 1056, 1050, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1043, 1053, 1035, 1051, 1046, 1009, 970, 8, 8, 0, 0, 0, 0, 0, 0, 963, 1045, 1008, 1058, 1015, 1039, 1073, 1024, 6, 1, 0, 0, 0, 0, 0, 0, 1016, 1047, 1035, 1074, 1006, 1028, 997, 1033, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8087, 8143, 8160, 8210, 8252, 8232, 8239] avg=8192.0000 max=8252 max_rank=5 min=8087 min_rank=1 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[991, 1064, 1040, 1043, 990, 1053, 994, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1068, 1069, 976, 1061, 1060, 1069, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 1052, 1000, 1001, 1057, 1009, 1000, 1039, 17, 8, 0, 0, 0, 0, 0, 0, 1038, 1040, 1028, 1020, 1069, 1038, 1033, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1008, 973, 1040, 1061, 1014, 983, 978, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1002, 1032, 1026, 990, 1038, 1039, 1013, 8, 8, 0, 0, 0, 0, 0, 0, 1033, 1002, 1011, 1049, 1026, 1008, 1017, 1055, 6, 1, 0, 0, 0, 0, 0, 0, 988, 1001, 1025, 1001, 1001, 1013, 991, 996, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8382, 8260, 8294, 8060, 8112, 8208, 8020] avg=8192.0000 max=8382 max_rank=1 min=8020 min_rank=7 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1065, 973, 1091, 1013, 982, 980, 975, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 991, 992, 1064, 1020, 1006, 1034, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 995, 1006, 1034, 1038, 933, 1014, 1006, 21, 12, 0, 0, 0, 0, 0, 0, 1024, 1051, 970, 1030, 1068, 985, 1048, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1024, 1040, 1041, 1067, 1037, 1014, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1015, 1100, 993, 1042, 1042, 1046, 978, 12, 10, 0, 0, 0, 0, 0, 0, 1026, 980, 999, 1068, 982, 1053, 1048, 994, 11, 6, 0, 0, 0, 0, 0, 0, 1010, 1032, 1055, 1052, 1020, 991, 1067, 1030, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8151, 8084, 8229, 8281, 8258, 8167, 8266] avg=8192.0000 max=8281 max_rank=4 min=8084 min_rank=2 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1058, 1005, 1044, 1056, 977, 974, 1022, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1011, 1015, 1058, 988, 1018, 995, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1089, 1019, 1003, 1054, 1040, 1022, 1018, 18, 7, 0, 0, 0, 0, 0, 0, 1044, 993, 1033, 1015, 1010, 1053, 1032, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1006, 1031, 1029, 1087, 1009, 1048, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1005, 981, 1025, 1007, 1042, 1008, 991, 8, 8, 0, 0, 0, 0, 0, 0, 1047, 1081, 1041, 997, 1028, 1022, 996, 990, 6, 1, 0, 0, 0, 0, 0, 0, 967, 1006, 1014, 1012, 1057, 1045, 1054, 1076, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8144, 8302, 8188, 8208, 8079, 8209, 8234] avg=8192.0000 max=8302 max_rank=2 min=8079 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1005, 999, 1037, 1088, 1051, 1016, 1039, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1046, 1043, 998, 1022, 1004, 1022, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 974, 1025, 1016, 1033, 1045, 990, 990, 18, 7, 0, 0, 0, 0, 0, 0, 1038, 1050, 1049, 1024, 1017, 1008, 1011, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1052, 1055, 1002, 1017, 1026, 1017, 993, 0, 0, 0, 0, 0, 0, 0, 0, 965, 1071, 991, 1003, 1027, 1032, 1013, 1006, 8, 8, 0, 0, 0, 0, 0, 0, 1026, 1023, 1007, 1019, 1014, 993, 997, 1066, 6, 1, 0, 0, 0, 0, 0, 0, 1030, 1030, 1031, 1044, 1033, 1046, 983, 1020, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8211, 8152, 8206, 8232, 8124, 8152, 8220] avg=8192.0000 max=8239 max_rank=0 min=8124 min_rank=5 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1021, 990, 1012, 1030, 1042, 1065, 1009, 1071, 1079, 1052, 1028, 1016, 1038, 1020, 1008, 1018, 957, 1039, 1055, 1006, 964, 1072, 1051, 1002, 1040, 1025, 961, 1051, 1012, 1032, 1039, 1054, 1005, 1001, 983, 1019, 1008, 1031, 1009, 997, 1038, 1081, 1048, 1093, 1005, 997, 1005, 988, 1056, 988, 1045, 1000, 979, 1033, 1014, 983, 1036, 1060, 1033, 1036, 1049, 1070, 960, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1062, 998, 1036, 1021, 1021, 987, 1037, 994, 992, 1021, 1030, 1035, 1006, 990, 1065, 1000, 1005, 1037, 1038, 1011, 1027, 1029, 1007, 1031, 1034, 1043, 1045, 1016, 1031, 986, 1033, 1000, 1049, 962, 1029, 1041, 1042, 1052, 1009, 1045, 1051, 1020, 1038, 1045, 1008, 1091, 1034, 1035, 1018, 957, 1007, 1023, 998, 1078, 1014, 1020, 1050, 960, 1000, 1046, 1018, 1068, 1008, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1042, 1043, 1035, 1012, 1040, 1042, 1022, 1008, 1057, 1045, 1041, 1027, 994, 1012, 1039, 1032, 1026, 975, 1045, 1039, 1035, 1024, 1087, 1012, 1014, 971, 990, 1025, 1055, 1056, 1013, 1025, 1015, 1021, 1040, 953, 1055, 1012, 1046, 1080, 1054, 1043, 977, 1001, 1090, 987, 1031, 997, 1012, 1037, 1065, 1042, 1011, 993, 1002, 1004, 972, 991, 1085, 1043, 983, 1051, 990, 970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1041, 1034, 1037, 1038, 1002, 1035, 994, 1006, 1030, 1019, 1041, 1021, 1057, 1012, 1035, 1055, 1042, 1056, 1061, 1003, 1048, 1037, 1017, 1042, 1030, 985, 1018, 1008, 1002, 1035, 1007, 1030, 1006, 1059, 1024, 999, 1042, 995, 969, 1003, 1006, 995, 1017, 972, 989, 1070, 1057, 1053, 1065, 1001, 1020, 1060, 1071, 992, 1040, 1035, 990, 967, 1040, 1059, 1022, 1011, 1008, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1015, 1029, 985, 1030, 997, 1008, 1043, 1057, 997, 1038, 1020, 1067, 1030, 1017, 1058, 1052, 1005, 1015, 1016, 1091, 978, 1016, 1041, 981, 1009, 1034, 1007, 1030, 1004, 1036, 1016, 995, 999, 1048, 1008, 1032, 1056, 972, 980, 1014, 1015, 1007, 1026, 1063, 1022, 1076, 1019, 1001, 973, 1060, 1069, 1091, 1025, 1066, 1033, 1027, 1004, 1021, 1018, 1021, 989, 1025, 1016, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1024, 1018, 1051, 989, 988, 1019, 1024, 1052, 1054, 1013, 1015, 1051, 980, 1012, 1013, 1026, 1031, 1020, 1074, 1061, 979, 1054, 1054, 1022, 986, 1015, 1012, 998, 1009, 1015, 1022, 1024, 1009, 1059, 1031, 1054, 1065, 994, 1033, 1039, 1042, 997, 1037, 1034, 1034, 972, 999, 1054, 1002, 987, 1037, 1039, 1003, 1028, 1021, 1079, 1027, 1039, 995, 1012, 1000, 1030, 1042, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1034, 1009, 1036, 997, 1048, 1055, 986, 982, 1019, 1031, 1027, 996, 1025, 1067, 996, 1044, 1031, 1039, 1090, 1001, 1030, 991, 1006, 993, 1034, 998, 1002, 1022, 1043, 1043, 1021, 1013, 1054, 1009, 1068, 1020, 1042, 982, 1042, 1032, 1018, 1025, 1019, 993, 977, 1079, 1031, 982, 994, 1048, 1000, 1017, 1011, 1024, 1073, 991, 1040, 998, 1037, 1021, 1033, 1089, 1053, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1035, 962, 1009, 1014, 1043, 984, 1017, 1068, 1004, 1026, 1016, 1046, 993, 1007, 1041, 997, 1031, 1044, 1015, 1006, 994, 1020, 1086, 1015, 1062, 968, 1024, 1084, 1034, 1076, 1056, 1007, 1052, 1025, 1040, 1037, 1020, 992, 1024, 1002, 992, 976, 1056, 1029, 1004, 1045, 1051, 1062, 1026, 1071, 1010, 1055, 1017, 1023, 979, 1056, 1046, 999, 1060, 983, 966, 1024, 1032, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1062, 998, 1036, 1021, 1021, 987, 1037, 994, 10, 2, 0, 0, 0, 0, 0, 0, 992, 1021, 1030, 1017, 1006, 990, 1065, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1037, 1017, 1011, 1027, 1029, 1007, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1043, 1045, 1016, 1031, 986, 1033, 1000, 21, 0, 0, 0, 0, 0, 0, 0, 1049, 962, 1029, 1041, 1042, 1052, 1009, 1045, 7, 5, 0, 0, 0, 0, 0, 0, 1051, 1020, 1038, 1045, 1008, 1076, 1034, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 957, 1007, 1016, 998, 1078, 1014, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 960, 1000, 1046, 1018, 1068, 1008, 1052, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8121, 8164, 8209, 8241, 8307, 8108, 8218] avg=8192.0000 max=8307 max_rank=5 min=8108 min_rank=6 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1042, 1043, 1035, 1012, 1040, 1042, 1022, 1008, 9, 2, 0, 0, 0, 0, 0, 0, 1057, 1045, 1041, 1009, 994, 1012, 1039, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 975, 1024, 1039, 1035, 1024, 1087, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 971, 990, 1025, 1055, 1056, 1013, 1025, 21, 0, 0, 0, 0, 0, 0, 0, 1015, 1021, 1040, 953, 1055, 1012, 1046, 1080, 7, 4, 0, 0, 0, 0, 0, 0, 1054, 1043, 977, 1001, 1090, 974, 1031, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1037, 1065, 1035, 1011, 993, 1002, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 972, 991, 1085, 1043, 983, 1051, 990, 970, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8229, 8222, 8170, 8233, 8167, 8159, 8101] avg=8192.0000 max=8255 max_rank=0 min=8101 min_rank=7 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32660 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1021, 990, 1012, 1030, 1042, 1065, 1009, 1071, 9, 2, 0, 0, 0, 0, 0, 0, 1079, 1052, 1028, 998, 1038, 1020, 1008, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 957, 1039, 1034, 1006, 964, 1072, 1051, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1025, 961, 1051, 1012, 1032, 1039, 1054, 21, 0, 0, 0, 0, 0, 0, 0, 1005, 1001, 983, 1019, 1008, 1031, 1009, 997, 7, 4, 0, 0, 0, 0, 0, 0, 1038, 1081, 1048, 1093, 1005, 984, 1005, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 988, 1045, 993, 979, 1033, 1014, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1060, 1033, 1036, 1049, 1070, 960, 1027, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8241, 8125, 8235, 8064, 8242, 8091, 8287] avg=8192.0000 max=8287 max_rank=7 min=8064 min_rank=4 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1041, 1034, 1037, 1038, 1002, 1035, 994, 1006, 10, 2, 0, 0, 0, 0, 0, 0, 1030, 1019, 1041, 1003, 1057, 1012, 1035, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1056, 1040, 1003, 1048, 1037, 1017, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 985, 1018, 1008, 1002, 1035, 1007, 1030, 21, 0, 0, 0, 0, 0, 0, 0, 1006, 1059, 1024, 999, 1042, 995, 969, 1003, 7, 5, 0, 0, 0, 0, 0, 0, 1006, 995, 1017, 972, 989, 1055, 1057, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1001, 1020, 1053, 1071, 992, 1040, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 990, 967, 1040, 1059, 1022, 1011, 1008, 1021, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8252, 8285, 8136, 8109, 8144, 8277, 8134] avg=8192.0000 max=8285 max_rank=2 min=8109 min_rank=4 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1024, 1018, 1051, 989, 988, 1019, 1024, 1052, 9, 2, 0, 0, 0, 0, 0, 0, 1054, 1013, 1015, 1032, 980, 1012, 1013, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1020, 1053, 1061, 979, 1054, 1054, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1015, 1012, 998, 1009, 1015, 1022, 1024, 21, 0, 0, 0, 0, 0, 0, 0, 1009, 1059, 1031, 1054, 1065, 994, 1033, 1039, 7, 4, 0, 0, 0, 0, 0, 0, 1042, 997, 1037, 1034, 1034, 959, 999, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 987, 1037, 1032, 1003, 1028, 1021, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1039, 995, 1012, 1000, 1030, 1042, 1037, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8145, 8274, 8102, 8295, 8156, 8189, 8199] avg=8192.0000 max=8295 max_rank=4 min=8102 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1015, 1029, 985, 1030, 997, 1008, 1043, 1057, 10, 2, 0, 0, 0, 0, 0, 0, 997, 1038, 1020, 1048, 1030, 1017, 1058, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1015, 996, 1091, 978, 1016, 1041, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1034, 1007, 1030, 1004, 1036, 1016, 995, 20, 0, 0, 0, 0, 0, 0, 0, 999, 1048, 1008, 1032, 1056, 972, 980, 1014, 8, 5, 0, 0, 0, 0, 0, 0, 1015, 1007, 1026, 1063, 1022, 1061, 1019, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1060, 1069, 1083, 1025, 1066, 1033, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1021, 1018, 1021, 989, 1025, 1016, 1043, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8260, 8123, 8151, 8122, 8214, 8336, 8154] avg=8192.0000 max=8336 max_rank=6 min=8122 min_rank=4 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1034, 1009, 1036, 997, 1048, 1055, 986, 982, 12, 9, 0, 0, 0, 0, 0, 0, 1019, 1031, 1027, 967, 1025, 1067, 996, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1039, 1061, 1001, 1030, 991, 1006, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 998, 1002, 1022, 1043, 1043, 1021, 1013, 26, 0, 0, 0, 0, 0, 0, 0, 1054, 1009, 1068, 1020, 1042, 982, 1042, 1032, 12, 8, 0, 0, 0, 0, 0, 0, 1018, 1025, 1019, 993, 977, 1059, 1031, 982, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1048, 1000, 1005, 1011, 1024, 1073, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 998, 1037, 1021, 1033, 1089, 1053, 1025, 20, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8176, 8152, 8202, 8269, 8104, 8146, 8319] avg=8192.0000 max=8319 max_rank=7 min=8104 min_rank=5 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1035, 962, 1009, 1014, 1043, 984, 1017, 1068, 9, 2, 0, 0, 0, 0, 0, 0, 1004, 1026, 1016, 1028, 993, 1007, 1041, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1044, 995, 1006, 994, 1020, 1086, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 968, 1024, 1084, 1034, 1076, 1056, 1007, 20, 0, 0, 0, 0, 0, 0, 0, 1052, 1025, 1040, 1037, 1020, 992, 1024, 1002, 7, 4, 0, 0, 0, 0, 0, 0, 992, 976, 1056, 1029, 1004, 1032, 1051, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1071, 1010, 1048, 1017, 1023, 979, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 999, 1060, 983, 966, 1024, 1032, 998, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8112, 8191, 8331, 8203, 8202, 8230, 8124] avg=8192.0000 max=8331 max_rank=3 min=8112 min_rank=1 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8064, 8320, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8448, 8064, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8064, 8320, 8192, 8320, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8320, 8320, 8064, 8448, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1050, 1009, 1033, 1053, 1002, 1016, 1009, 1020, 1015, 1024, 1066, 995, 988, 1045, 1050, 986, 988, 970, 1072, 1072, 1043, 978, 1002, 1041, 967, 1018, 1073, 1068, 1038, 1043, 1006, 1003, 1029, 1051, 976, 1035, 1008, 1010, 1079, 1005, 1063, 1009, 1028, 1026, 1019, 1002, 1031, 982, 1054, 1049, 1039, 1028, 1013, 1020, 1011, 1021, 990, 1043, 1014, 1057, 1010, 1006, 1047, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1026, 1022, 1001, 1041, 1021, 1062, 1042, 1034, 1018, 1036, 1009, 1059, 1013, 1022, 998, 1011, 1002, 1023, 1067, 995, 1007, 1040, 1010, 1052, 1030, 1038, 1049, 980, 1031, 998, 1021, 1061, 1065, 1017, 1059, 921, 1008, 1008, 993, 1037, 998, 1055, 1041, 1033, 1038, 1023, 978, 1027, 1062, 1035, 1003, 996, 1046, 981, 1021, 1021, 1032, 1013, 1034, 1063, 1018, 1043, 1033, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1064, 1060, 1054, 1028, 959, 993, 1059, 1054, 990, 1028, 1026, 999, 1004, 1034, 1054, 1005, 1026, 1045, 1040, 1002, 1002, 1050, 993, 1013, 1009, 992, 1053, 1002, 1055, 1020, 1027, 1004, 1013, 1067, 1035, 979, 1005, 1031, 1050, 1010, 1024, 996, 969, 1008, 991, 1012, 1010, 1028, 1004, 1018, 1040, 1045, 1038, 1032, 989, 1014, 1030, 1056, 1038, 998, 1050, 1090, 1093, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1056, 983, 1010, 1002, 1007, 1025, 1019, 1044, 999, 1043, 981, 1010, 989, 1054, 1013, 989, 1006, 999, 1039, 1008, 1068, 1044, 1031, 1003, 1035, 1030, 1008, 1018, 1034, 1062, 1046, 1052, 1070, 1021, 1043, 1033, 931, 991, 1062, 984, 1051, 1003, 1006, 1044, 1053, 991, 1044, 1032, 998, 1058, 1010, 1031, 1050, 1055, 1044, 1007, 1033, 1057, 1045, 1039, 966, 1006, 1066, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1078, 1008, 1064, 1025, 1021, 1001, 1015, 1058, 1104, 1012, 971, 989, 1047, 1011, 1083, 967, 1039, 984, 1023, 1032, 1012, 994, 976, 1043, 992, 1033, 1050, 1103, 1019, 1008, 1029, 1012, 1039, 1018, 1005, 1007, 996, 1044, 1052, 1050, 981, 1010, 1052, 975, 1014, 1014, 1020, 1065, 1057, 1005, 1023, 1030, 1043, 997, 993, 1038, 993, 1053, 1030, 990, 1034, 1043, 1000, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1005, 967, 1007, 1021, 1009, 1047, 1041, 1043, 1044, 1019, 1027, 1050, 1054, 1052, 1004, 1041, 1021, 1075, 1005, 1021, 1015, 1010, 1019, 1046, 953, 1048, 1048, 1050, 1043, 1045, 1047, 994, 984, 1001, 1049, 1017, 998, 1071, 978, 1050, 965, 990, 1059, 1031, 1010, 1020, 1054, 1011, 996, 1043, 1043, 1044, 1070, 1066, 997, 993, 997, 1025, 1034, 1034, 1012, 1019, 995, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1035, 1010, 1000, 1057, 1067, 1047, 1031, 1064, 975, 992, 1028, 970, 1009, 1029, 1018, 1017, 1015, 1008, 1037, 1054, 992, 1026, 1037, 1029, 1108, 1047, 1078, 1022, 1014, 1051, 1035, 1011, 985, 1017, 1001, 1023, 1010, 1037, 995, 1015, 1012, 1037, 1055, 1036, 1005, 990, 1070, 1027, 999, 951, 1013, 1022, 1032, 1032, 1043, 1061, 980, 1023, 1050, 989, 1023, 1003, 1031, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1078, 1021, 1039, 1068, 1014, 1002, 989, 1058, 1016, 1037, 982, 1043, 978, 1007, 1031, 1045, 1010, 952, 1006, 1007, 1025, 1041, 1034, 1071, 1045, 995, 1036, 1036, 1016, 1005, 1004, 1014, 978, 1054, 1007, 1023, 1079, 1101, 1078, 1027, 1021, 1040, 1036, 1052, 1003, 1051, 960, 1063, 997, 1008, 1037, 1052, 1023, 1048, 1020, 984, 1012, 1077, 970, 969, 1012, 1014, 1029, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1012, 1009, 1033, 1053, 1002, 1016, 1009, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1024, 1066, 995, 988, 1045, 1050, 986, 41, 0, 0, 0, 0, 0, 0, 0, 988, 970, 1072, 1072, 1043, 978, 1002, 1041, 14, 3, 0, 0, 0, 0, 0, 0, 967, 1018, 1032, 1068, 1038, 1043, 1006, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1051, 976, 1035, 1008, 1010, 1079, 1005, 7, 6, 0, 0, 0, 0, 0, 0, 1063, 1009, 1028, 1026, 1019, 1002, 1031, 982, 24, 0, 0, 0, 0, 0, 0, 0, 1054, 1049, 1039, 1028, 1007, 1020, 1011, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1033, 1014, 1057, 1010, 1006, 1047, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8210, 8183, 8175, 8206, 8184, 8229, 8195] avg=8192.0000 max=8229 max_rank=6 min=8154 min_rank=0 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[970, 967, 1007, 1021, 1009, 1047, 1041, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1019, 1027, 1050, 1054, 1052, 1004, 1041, 40, 0, 0, 0, 0, 0, 0, 0, 1021, 1075, 1005, 1021, 1015, 1010, 1019, 1046, 13, 3, 0, 0, 0, 0, 0, 0, 953, 1048, 1008, 1050, 1043, 1045, 1047, 994, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1001, 1049, 1017, 998, 1071, 978, 1050, 7, 7, 0, 0, 0, 0, 0, 0, 965, 990, 1059, 1031, 1010, 1020, 1054, 1011, 22, 0, 0, 0, 0, 0, 0, 0, 996, 1043, 1043, 1044, 1063, 1066, 997, 993, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1015, 1034, 1034, 1012, 1019, 995, 1009, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8331, 8228, 8188, 8162, 8162, 8245, 8115] avg=8192.0000 max=8331 max_rank=1 min=8105 min_rank=0 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1026, 1060, 1054, 1028, 959, 993, 1059, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1028, 1026, 999, 1004, 1034, 1054, 1005, 40, 0, 0, 0, 0, 0, 0, 0, 1026, 1045, 1040, 1002, 1002, 1050, 993, 1013, 14, 3, 0, 0, 0, 0, 0, 0, 1009, 992, 1013, 1002, 1055, 1020, 1027, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1067, 1035, 979, 1005, 1031, 1050, 1010, 8, 7, 0, 0, 0, 0, 0, 0, 1024, 996, 969, 1008, 991, 1012, 1010, 1028, 24, 0, 0, 0, 0, 0, 0, 0, 1004, 1018, 1040, 1045, 1031, 1032, 989, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1045, 1038, 998, 1050, 1090, 1093, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8180, 8188, 8122, 8205, 8062, 8173, 8373] avg=8192.0000 max=8373 max_rank=7 min=8062 min_rank=5 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[976, 1022, 1001, 1041, 1021, 1062, 1042, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1036, 1009, 1059, 1013, 1022, 998, 1011, 45, 5, 0, 0, 0, 0, 0, 0, 1002, 1023, 1067, 995, 1007, 1040, 1010, 1052, 18, 10, 0, 0, 0, 0, 0, 0, 1030, 1038, 1004, 980, 1031, 998, 1021, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1017, 1059, 921, 1008, 1008, 993, 1037, 11, 9, 0, 0, 0, 0, 0, 0, 998, 1055, 1041, 1033, 1038, 1023, 978, 1027, 27, 0, 0, 0, 0, 0, 0, 0, 1062, 1035, 1003, 996, 1037, 981, 1021, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 992, 1034, 1063, 1018, 1043, 1033, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8216, 8224, 8163, 8128, 8220, 8156, 8230] avg=8192.0000 max=8230 max_rank=7 min=8128 min_rank=4 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1018, 983, 1010, 1002, 1007, 1025, 1019, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1043, 981, 1010, 989, 1054, 1013, 989, 39, 0, 0, 0, 0, 0, 0, 0, 1006, 999, 1039, 1008, 1068, 1044, 1031, 1003, 14, 3, 0, 0, 0, 0, 0, 0, 1035, 1030, 969, 1018, 1034, 1062, 1046, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1021, 1043, 1033, 931, 991, 1062, 984, 8, 7, 0, 0, 0, 0, 0, 0, 1051, 1003, 1006, 1044, 1053, 991, 1044, 1032, 24, 0, 0, 0, 0, 0, 0, 0, 998, 1058, 1010, 1031, 1043, 1055, 1044, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1046, 1045, 1039, 966, 1006, 1066, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8117, 8215, 8246, 8150, 8248, 8246, 8206] avg=8192.0000 max=8248 max_rank=5 min=8108 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1040, 1008, 1064, 1025, 1021, 1001, 1015, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 1012, 971, 989, 1047, 1011, 1083, 967, 40, 0, 0, 0, 0, 0, 0, 0, 1039, 984, 1023, 1032, 1012, 994, 976, 1043, 14, 3, 0, 0, 0, 0, 0, 0, 992, 1033, 1010, 1103, 1019, 1008, 1029, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1018, 1005, 1007, 996, 1044, 1052, 1050, 8, 7, 0, 0, 0, 0, 0, 0, 981, 1010, 1052, 975, 1014, 1014, 1020, 1065, 24, 0, 0, 0, 0, 0, 0, 0, 1057, 1005, 1023, 1030, 1036, 997, 993, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1042, 1030, 990, 1034, 1043, 1000, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8224, 8120, 8206, 8226, 8155, 8179, 8194] avg=8192.0000 max=8232 max_rank=0 min=8120 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1040, 1021, 1039, 1068, 1014, 1002, 989, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1037, 982, 1043, 978, 1007, 1031, 1045, 40, 0, 0, 0, 0, 0, 0, 0, 1010, 952, 1006, 1007, 1025, 1041, 1034, 1071, 14, 3, 0, 0, 0, 0, 0, 0, 1045, 995, 996, 1036, 1016, 1005, 1004, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1054, 1007, 1023, 1079, 1101, 1078, 1027, 8, 7, 0, 0, 0, 0, 0, 0, 1021, 1040, 1036, 1052, 1003, 1051, 960, 1063, 24, 0, 0, 0, 0, 0, 0, 0, 997, 1008, 1037, 1052, 1016, 1048, 1020, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1066, 970, 969, 1012, 1014, 1029, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8179, 8163, 8111, 8362, 8250, 8162, 8078] avg=8192.0000 max=8362 max_rank=4 min=8078 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32852 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[998, 1010, 1000, 1057, 1067, 1047, 1031, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 975, 992, 1028, 970, 1009, 1029, 1018, 1017, 41, 0, 0, 0, 0, 0, 0, 0, 1015, 1008, 1037, 1054, 992, 1026, 1037, 1029, 14, 3, 0, 0, 0, 0, 0, 0, 1108, 1047, 1037, 1022, 1014, 1051, 1035, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1017, 1001, 1023, 1010, 1037, 995, 1015, 7, 7, 0, 0, 0, 0, 0, 0, 1012, 1037, 1055, 1036, 1005, 990, 1070, 1027, 23, 0, 0, 0, 0, 0, 0, 0, 999, 951, 1013, 1022, 1025, 1032, 1043, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1013, 1050, 989, 1023, 1003, 1031, 1056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8079, 8215, 8325, 8097, 8255, 8146, 8145] avg=8192.0000 max=8325 max_rank=3 min=8079 min_rank=1 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8064, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 8064, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8320, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True + [2026-08-30 05:02:13] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 670.5 | throughput per GPU (TFLOP/s/GPU): 497.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.485067E+00 | loss scale: 1.0 | grad norm: 5.584 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1013, 1034, 1066, 1042, 1027, 1013, 1054, 988, 1022, 1012, 1046, 1004, 1046, 1051, 1046, 1043, 1015, 1033, 1074, 1063, 1062, 1020, 1037, 972, 991, 1053, 1032, 985, 1000, 1079, 994, 1042, 1045, 1057, 975, 1063, 1014, 1012, 1046, 977, 987, 1029, 1009, 1034, 1015, 1025, 1006, 1012, 976, 1023, 1061, 977, 1039, 1017, 1019, 1024, 1033, 1026, 1002, 1061, 998, 995, 1040, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1012, 1024, 1067, 997, 996, 961, 1008, 1039, 1008, 982, 1016, 963, 1015, 1037, 1101, 1030, 1091, 1014, 1019, 1023, 1050, 989, 1043, 986, 1020, 1009, 1022, 1026, 1030, 1000, 1061, 1042, 1061, 1063, 1004, 1068, 1014, 1019, 998, 1039, 971, 1056, 1030, 1074, 1063, 968, 1025, 1053, 978, 1013, 1038, 1034, 1031, 1037, 1015, 1062, 1056, 994, 997, 1023, 986, 1055, 1021, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1036, 1028, 1042, 1051, 1013, 1083, 1027, 1026, 1033, 1034, 1075, 1039, 1011, 1051, 1035, 1009, 1021, 1045, 1016, 1025, 1035, 1042, 1022, 999, 1039, 1032, 984, 1016, 1042, 1033, 1015, 1040, 996, 1013, 969, 1028, 1040, 1033, 963, 1006, 1042, 1051, 981, 1010, 1003, 1114, 981, 1021, 1042, 1028, 1050, 1003, 962, 1034, 1053, 982, 1046, 998, 1025, 986, 1060, 1008, 1005, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1048, 998, 1035, 1036, 1060, 1021, 1024, 1004, 1046, 996, 984, 1026, 1009, 1037, 1035, 1006, 1027, 991, 1003, 1036, 1017, 1026, 1026, 1005, 1040, 1012, 1038, 1062, 1036, 1075, 1020, 1009, 987, 1033, 1032, 1091, 1029, 1011, 1018, 1065, 1025, 1042, 1014, 1002, 1010, 1035, 1058, 1068, 1031, 998, 1015, 1012, 1059, 1108, 1012, 999, 1026, 953, 1023, 989, 1016, 999, 989, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[978, 1057, 985, 999, 1004, 1043, 1052, 992, 986, 1025, 1017, 1015, 1014, 1029, 1050, 1022, 1045, 1026, 1067, 1017, 965, 1049, 1000, 994, 1019, 977, 1008, 1014, 1015, 987, 1055, 1023, 1017, 1016, 990, 992, 1025, 1003, 1036, 1068, 1033, 1042, 1061, 1081, 1047, 1026, 999, 1039, 1044, 1058, 958, 1013, 1031, 1074, 1052, 964, 1045, 1033, 1066, 1060, 1063, 1018, 1041, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1009, 1029, 986, 1049, 1019, 1016, 1039, 1040, 1018, 1100, 1020, 1025, 1030, 990, 990, 1027, 1082, 1082, 984, 1028, 1008, 1014, 1017, 1038, 980, 1036, 1028, 1031, 998, 981, 1046, 1023, 994, 1033, 985, 1022, 1045, 1040, 1061, 981, 1029, 998, 997, 1021, 1013, 1034, 1028, 974, 996, 1033, 1033, 1053, 1040, 1018, 1034, 1029, 1015, 1026, 1049, 1061, 1079, 1008, 1054, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1006, 1051, 1012, 1043, 1071, 1042, 1046, 1012, 1046, 993, 1067, 1032, 1080, 1039, 1055, 1028, 1018, 1062, 991, 1057, 991, 1030, 982, 1015, 1017, 991, 1044, 996, 1060, 1055, 986, 1035, 1028, 1034, 1040, 1052, 983, 999, 1038, 1008, 1024, 1030, 977, 996, 971, 1015, 1015, 1065, 959, 1071, 978, 1022, 1047, 1026, 1001, 1010, 1004, 1010, 1029, 1059, 1011, 1006, 1040, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1043, 1036, 969, 1027, 1005, 1083, 1084, 1069, 1017, 1053, 1009, 1041, 966, 1035, 1053, 1049, 1017, 1000, 1045, 1042, 1022, 1000, 1023, 1042, 1011, 1031, 987, 1074, 1025, 1036, 992, 1089, 1033, 1014, 1022, 979, 1037, 1004, 1031, 1038, 960, 1030, 992, 1056, 998, 1006, 997, 1055, 1009, 1023, 1028, 996, 1023, 1073, 1038, 1039, 1009, 1034, 992, 1023, 1048, 1010, 965, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1013, 1034, 1066, 1042, 1027, 1013, 1025, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1012, 1046, 1004, 1046, 1051, 1019, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1033, 1074, 1063, 1062, 1020, 1037, 972, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1053, 1032, 985, 1000, 1079, 994, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1057, 975, 1063, 1014, 1012, 1046, 977, 14, 0, 0, 0, 0, 0, 0, 0, 987, 1029, 1009, 1034, 1015, 1025, 1006, 1012, 13, 0, 0, 0, 0, 0, 0, 0, 976, 1023, 1061, 977, 1039, 1017, 1019, 1024, 3, 0, 0, 0, 0, 0, 0, 0, 1033, 1026, 1002, 1061, 998, 995, 1040, 980, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8243, 8273, 8176, 8203, 8130, 8139, 8164] avg=8192.0000 max=8273 max_rank=2 min=8130 min_rank=5 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1012, 1024, 1067, 997, 996, 961, 980, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 982, 1016, 963, 1015, 1037, 1072, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 1014, 1019, 1023, 1050, 989, 1043, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1009, 1022, 1026, 1030, 1000, 1061, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1063, 1004, 1068, 1014, 1019, 998, 1039, 15, 0, 0, 0, 0, 0, 0, 0, 971, 1056, 1030, 1074, 1063, 968, 1025, 1053, 14, 0, 0, 0, 0, 0, 0, 0, 978, 1013, 1038, 1034, 1031, 1037, 1015, 1062, 4, 0, 0, 0, 0, 0, 0, 0, 1056, 994, 997, 1023, 986, 1055, 1021, 1009, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8076, 8123, 8211, 8210, 8281, 8254, 8212, 8169] avg=8192.0000 max=8281 max_rank=4 min=8076 min_rank=0 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32916 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1048, 998, 1035, 1036, 1060, 1021, 996, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 996, 984, 1026, 1009, 1037, 1008, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 991, 1003, 1036, 1017, 1026, 1026, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1012, 1038, 1062, 1036, 1075, 1020, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1033, 1032, 1091, 1029, 1011, 1018, 1065, 14, 0, 0, 0, 0, 0, 0, 0, 1025, 1042, 1014, 1002, 1010, 1035, 1058, 1068, 13, 0, 0, 0, 0, 0, 0, 0, 1031, 998, 1015, 1012, 1059, 1108, 1012, 999, 3, 0, 0, 0, 0, 0, 0, 0, 1026, 953, 1023, 989, 1016, 999, 989, 999, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8112, 8128, 8292, 8280, 8267, 8237, 8022] avg=8192.0000 max=8292 max_rank=3 min=8022 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1036, 1028, 1042, 1051, 1013, 1083, 999, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1034, 1075, 1039, 1011, 1051, 1008, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1045, 1016, 1025, 1035, 1042, 1022, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1032, 984, 1016, 1042, 1033, 1015, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1013, 969, 1028, 1040, 1033, 963, 1006, 14, 0, 0, 0, 0, 0, 0, 0, 1042, 1051, 981, 1010, 1003, 1114, 981, 1021, 13, 0, 0, 0, 0, 0, 0, 0, 1042, 1028, 1050, 1003, 962, 1034, 1053, 982, 3, 0, 0, 0, 0, 0, 0, 0, 1046, 998, 1025, 986, 1060, 1008, 1005, 1004, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8260, 8202, 8201, 8062, 8216, 8157, 8160] avg=8192.0000 max=8278 max_rank=0 min=8062 min_rank=4 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32595 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[978, 1057, 985, 999, 1004, 1043, 1023, 992, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1025, 1017, 1015, 1014, 1029, 1023, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1026, 1067, 1017, 965, 1049, 1000, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 977, 1008, 1014, 1015, 987, 1055, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1016, 990, 992, 1025, 1003, 1036, 1068, 14, 0, 0, 0, 0, 0, 0, 0, 1033, 1042, 1061, 1081, 1047, 1026, 999, 1039, 13, 0, 0, 0, 0, 0, 0, 0, 1044, 1058, 958, 1013, 1031, 1074, 1052, 964, 3, 0, 0, 0, 0, 0, 0, 0, 1045, 1033, 1066, 1060, 1063, 1018, 1041, 1012, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8081, 8131, 8160, 8098, 8161, 8341, 8197, 8367] avg=8192.0000 max=8367 max_rank=7 min=8081 min_rank=0 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=33066 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1006, 1051, 1012, 1043, 1071, 1042, 1017, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 993, 1067, 1032, 1080, 1039, 1027, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1062, 991, 1057, 991, 1030, 982, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 991, 1044, 996, 1060, 1055, 986, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1034, 1040, 1052, 983, 999, 1038, 1008, 15, 0, 0, 0, 0, 0, 0, 0, 1024, 1030, 977, 996, 971, 1015, 1015, 1065, 13, 0, 0, 0, 0, 0, 0, 0, 959, 1071, 978, 1022, 1047, 1026, 1001, 1010, 3, 0, 0, 0, 0, 0, 0, 0, 1004, 1010, 1029, 1059, 1011, 1006, 1040, 1035, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8312, 8143, 8184, 8197, 8106, 8117, 8223] avg=8192.0000 max=8312 max_rank=1 min=8106 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1009, 1029, 986, 1049, 1019, 1016, 1010, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1100, 1020, 1025, 1030, 990, 964, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1082, 984, 1028, 1008, 1014, 1017, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1036, 1028, 1031, 998, 981, 1046, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1033, 985, 1022, 1045, 1040, 1061, 981, 14, 0, 0, 0, 0, 0, 0, 0, 1029, 998, 997, 1021, 1013, 1034, 1028, 974, 12, 0, 0, 0, 0, 0, 0, 0, 996, 1033, 1033, 1053, 1040, 1018, 1034, 1029, 4, 0, 0, 0, 0, 0, 0, 0, 1015, 1026, 1049, 1061, 1079, 1008, 1054, 990, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8174, 8249, 8123, 8175, 8106, 8240, 8311] avg=8192.0000 max=8311 max_rank=7 min=8106 min_rank=5 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1043, 1036, 969, 1027, 1005, 1083, 1051, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1053, 1009, 1041, 966, 1035, 1011, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1000, 1045, 1042, 1022, 1000, 1023, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1031, 987, 1074, 1025, 1036, 992, 1089, 7, 0, 0, 0, 0, 0, 0, 0, 1033, 1014, 1022, 979, 1037, 1004, 1031, 1038, 19, 0, 0, 0, 0, 0, 0, 0, 960, 1030, 992, 1056, 998, 1006, 997, 1055, 16, 6, 0, 0, 0, 0, 0, 0, 1009, 1023, 1028, 996, 1023, 1073, 1038, 1039, 8, 0, 0, 0, 0, 0, 0, 0, 1009, 1034, 992, 1023, 1048, 1010, 965, 999, 33, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8181, 8170, 8252, 8177, 8116, 8237, 8120] avg=8192.0000 max=8283 max_rank=0 min=8116 min_rank=5 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8064, 8320, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8320, 8064, 8320, 8192, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8064, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8192, 8320, 8448, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1011, 1060, 992, 1044, 1020, 998, 1082, 1020, 1037, 1015, 1027, 1038, 1032, 1046, 1038, 1022, 974, 1001, 1019, 1035, 1044, 992, 1036, 1016, 1021, 1046, 1012, 1016, 1010, 1012, 1004, 1036, 979, 1066, 1062, 1040, 984, 1031, 1042, 1025, 1053, 1015, 1036, 1031, 1012, 1017, 1000, 1052, 975, 1022, 1019, 1018, 1022, 1051, 986, 1010, 1031, 1005, 1005, 1041, 1046, 1045, 1021, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[969, 1002, 1023, 1016, 1047, 1054, 1042, 1056, 1050, 1033, 1024, 1039, 998, 1054, 998, 1065, 1005, 1009, 1014, 1012, 1005, 1029, 1070, 1041, 988, 1031, 1057, 1047, 988, 1013, 1020, 1032, 1040, 985, 1054, 1086, 1040, 1057, 1086, 1033, 1015, 1036, 985, 1002, 961, 994, 980, 1027, 979, 1082, 1044, 1040, 998, 1057, 1052, 1025, 940, 1060, 989, 1009, 1040, 1030, 978, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1060, 1014, 1045, 1022, 1031, 1064, 1048, 1022, 1009, 997, 1047, 1052, 1031, 1043, 998, 1054, 980, 1028, 1061, 1023, 978, 1069, 1027, 1024, 1013, 1004, 974, 1028, 1024, 1036, 1049, 1036, 1016, 991, 1000, 1030, 1014, 1019, 1050, 985, 1016, 977, 1037, 999, 1016, 1022, 1051, 1014, 1028, 1041, 986, 992, 1034, 1074, 1035, 1013, 1047, 1042, 1040, 965, 1062, 1054, 1004, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1036, 1093, 1044, 1031, 992, 1024, 1051, 1028, 1031, 1017, 1012, 1062, 1042, 996, 990, 1031, 1073, 1010, 1028, 1028, 1006, 1071, 994, 1013, 1025, 1068, 1001, 1052, 1041, 1039, 965, 1003, 1005, 974, 1019, 984, 1001, 1036, 1055, 1020, 1014, 1006, 1023, 1019, 1014, 1021, 1059, 1029, 1011, 966, 1029, 1008, 1044, 1054, 1008, 1053, 1043, 1029, 996, 1040, 994, 1030, 1057, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1045, 1032, 1041, 1002, 1001, 1022, 1026, 1026, 1054, 1064, 1068, 1043, 1046, 1006, 1054, 1063, 1066, 1031, 1032, 1038, 992, 984, 1041, 990, 1048, 1076, 992, 1016, 985, 1044, 996, 1006, 996, 1015, 995, 952, 1028, 1018, 970, 1006, 1064, 1022, 999, 1001, 1066, 1039, 1013, 1002, 1035, 1007, 994, 1022, 1064, 1046, 980, 1027, 1056, 1024, 1007, 1058, 1019, 1037, 1021, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1072, 1034, 1025, 1004, 988, 967, 1049, 1000, 1010, 1009, 1004, 1035, 1012, 1076, 997, 1010, 976, 985, 1012, 998, 1028, 1041, 1051, 1016, 1050, 1020, 1035, 992, 1056, 1045, 990, 964, 1030, 984, 1012, 1013, 1069, 1046, 1072, 1056, 1001, 1052, 1023, 1077, 1066, 1048, 1046, 1015, 1041, 1027, 1033, 1032, 1011, 1017, 1033, 1037, 1028, 1007, 1047, 1009, 1001, 1055, 1028, 969] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1035, 992, 1045, 1013, 1021, 1037, 1016, 953, 985, 1009, 977, 1031, 1013, 1048, 1018, 1029, 1042, 1024, 1081, 1032, 1037, 1053, 1028, 981, 1030, 1028, 1035, 1018, 1035, 1040, 1054, 972, 1010, 1045, 1039, 1039, 1022, 1041, 1029, 1040, 1005, 995, 990, 1046, 1049, 1051, 1037, 990, 1019, 1016, 1034, 972, 1072, 1018, 1001, 1033, 1040, 1002, 1027, 1048, 1024, 1051, 1016, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1018, 1003, 1051, 980, 990, 952, 1043, 1040, 982, 1032, 1044, 1096, 1014, 1075, 992, 997, 1028, 1035, 1056, 1050, 1028, 1011, 1020, 1013, 1026, 1008, 1040, 1083, 1044, 975, 1029, 1067, 1056, 1021, 1006, 992, 998, 1022, 1003, 1013, 1069, 1028, 1015, 1083, 1057, 982, 1040, 1070, 987, 996, 1041, 1037, 1012, 1017, 1025, 1046, 1014, 1045, 979, 988, 1019, 1007, 1038, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1011, 1060, 992, 1044, 1020, 998, 1080, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1015, 1027, 1004, 1032, 1046, 1038, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1001, 1019, 1035, 1044, 992, 1036, 1016, 2, 0, 0, 0, 0, 0, 0, 0, 1021, 1046, 1012, 1016, 1010, 1012, 1004, 1036, 4, 0, 0, 0, 0, 0, 0, 0, 979, 1066, 1062, 1040, 984, 1031, 1042, 1025, 11, 0, 0, 0, 0, 0, 0, 0, 1053, 1015, 1036, 1027, 1012, 1017, 1000, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1022, 1019, 1018, 1022, 1051, 986, 1010, 5, 0, 0, 0, 0, 0, 0, 0, 1031, 1005, 1005, 1041, 1046, 1045, 1021, 1038, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8221, 8119, 8161, 8240, 8212, 8108, 8250] avg=8192.0000 max=8250 max_rank=7 min=8108 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1045, 1032, 1041, 1002, 1001, 1022, 1024, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1064, 1068, 1009, 1046, 1006, 1054, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1031, 1032, 1038, 992, 984, 1041, 990, 2, 0, 0, 0, 0, 0, 0, 0, 1048, 1076, 992, 1016, 985, 1044, 996, 1006, 4, 0, 0, 0, 0, 0, 0, 0, 996, 1015, 995, 952, 1028, 1018, 970, 1006, 11, 0, 0, 0, 0, 0, 0, 0, 1064, 1022, 999, 997, 1066, 1039, 1013, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1007, 994, 1022, 1064, 1046, 980, 1027, 5, 0, 0, 0, 0, 0, 0, 0, 1056, 1024, 1007, 1058, 1019, 1037, 1021, 1023, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8364, 8176, 8167, 7991, 8202, 8180, 8263] avg=8192.0000 max=8364 max_rank=1 min=7991 min_rank=4 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1036, 1093, 1044, 1031, 992, 1024, 1049, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1017, 1012, 1027, 1042, 996, 990, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1010, 1028, 1028, 1006, 1071, 994, 1013, 2, 0, 0, 0, 0, 0, 0, 0, 1025, 1068, 1001, 1052, 1041, 1039, 965, 1003, 4, 0, 0, 0, 0, 0, 0, 0, 1005, 974, 1019, 984, 1001, 1036, 1055, 1020, 11, 0, 0, 0, 0, 0, 0, 0, 1014, 1006, 1023, 1015, 1014, 1021, 1059, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 966, 1029, 1008, 1044, 1054, 1008, 1053, 6, 0, 0, 0, 0, 0, 0, 0, 1043, 1029, 996, 1040, 994, 1030, 1057, 998, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8146, 8225, 8198, 8105, 8181, 8179, 8205] avg=8192.0000 max=8297 max_rank=0 min=8105 min_rank=4 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[969, 1002, 1023, 1016, 1047, 1054, 1040, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1033, 1024, 1005, 998, 1054, 998, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1009, 1014, 1012, 1005, 1029, 1070, 1041, 2, 0, 0, 0, 0, 0, 0, 0, 988, 1031, 1057, 1047, 988, 1013, 1020, 1032, 4, 0, 0, 0, 0, 0, 0, 0, 1040, 985, 1054, 1086, 1040, 1057, 1086, 1033, 11, 0, 0, 0, 0, 0, 0, 0, 1015, 1036, 985, 998, 961, 994, 980, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1082, 1044, 1040, 998, 1057, 1052, 1025, 5, 0, 0, 0, 0, 0, 0, 0, 940, 1060, 989, 1009, 1040, 1030, 978, 1001, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8227, 8187, 8180, 8392, 7996, 8282, 8065] avg=8192.0000 max=8392 max_rank=4 min=7996 min_rank=5 imbalance_ratio=1.024414 max_load_reduction=57144 max_load_reduction_pct=0.871948 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1060, 1014, 1045, 1022, 1031, 1064, 1046, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 997, 1047, 1017, 1031, 1043, 998, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1028, 1061, 1023, 978, 1069, 1027, 1024, 2, 0, 0, 0, 0, 0, 0, 0, 1013, 1004, 974, 1028, 1024, 1036, 1049, 1036, 4, 0, 0, 0, 0, 0, 0, 0, 1016, 991, 1000, 1030, 1014, 1019, 1050, 985, 11, 0, 0, 0, 0, 0, 0, 0, 1016, 977, 1037, 995, 1016, 1022, 1051, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1041, 986, 992, 1034, 1074, 1035, 1013, 6, 0, 0, 0, 0, 0, 0, 0, 1047, 1042, 1040, 965, 1062, 1054, 1004, 991, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8304, 8196, 8192, 8168, 8116, 8128, 8209, 8223] avg=8192.0000 max=8304 max_rank=0 min=8116 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1072, 1034, 1025, 1004, 988, 967, 1047, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1009, 1004, 1002, 1012, 1076, 997, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 976, 985, 1012, 998, 1028, 1041, 1051, 1016, 2, 0, 0, 0, 0, 0, 0, 0, 1050, 1020, 1035, 992, 1056, 1045, 990, 964, 4, 0, 0, 0, 0, 0, 0, 0, 1030, 984, 1012, 1013, 1069, 1046, 1072, 1056, 10, 0, 0, 0, 0, 0, 0, 0, 1001, 1052, 1023, 1073, 1066, 1048, 1046, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1027, 1033, 1032, 1011, 1017, 1033, 1037, 5, 0, 0, 0, 0, 0, 0, 0, 1028, 1007, 1047, 1009, 1001, 1055, 1028, 969, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8120, 8109, 8156, 8292, 8324, 8236, 8162] avg=8192.0000 max=8324 max_rank=5 min=8109 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=33014 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1035, 992, 1045, 1013, 1021, 1037, 1002, 953, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1009, 977, 987, 1013, 1048, 1018, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1024, 1081, 1032, 1037, 1053, 1028, 981, 7, 0, 0, 0, 0, 0, 0, 0, 1030, 1028, 1035, 1018, 1035, 1040, 1054, 972, 10, 7, 1, 0, 0, 0, 0, 0, 1010, 1045, 1039, 1039, 1022, 1041, 1029, 1040, 13, 0, 0, 0, 0, 0, 0, 0, 1005, 995, 990, 1036, 1049, 1051, 1037, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1016, 1034, 972, 1072, 1018, 1001, 1033, 10, 0, 0, 0, 0, 0, 0, 0, 1040, 1002, 1027, 1048, 1024, 1051, 1016, 1023, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8066, 8285, 8230, 8278, 8153, 8175, 8251] avg=8192.0000 max=8285 max_rank=2 min=8066 min_rank=1 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1018, 1003, 1051, 980, 990, 952, 1041, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1032, 1044, 1060, 1014, 1075, 992, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1035, 1056, 1050, 1028, 1011, 1020, 1013, 2, 0, 0, 0, 0, 0, 0, 0, 1026, 1008, 1040, 1083, 1044, 975, 1029, 1067, 4, 0, 0, 0, 0, 0, 0, 0, 1056, 1021, 1006, 992, 998, 1022, 1003, 1013, 11, 0, 0, 0, 0, 0, 0, 0, 1069, 1028, 1015, 1079, 1057, 982, 1040, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 987, 996, 1041, 1037, 1012, 1017, 1025, 1046, 6, 0, 0, 0, 0, 0, 0, 0, 1014, 1045, 979, 988, 1019, 1007, 1038, 1008, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8075, 8196, 8243, 8276, 8122, 8340, 8167, 8117] avg=8192.0000 max=8340 max_rank=5 min=8075 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8064, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8448, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8448, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1008, 1068, 1017, 1042, 985, 1036, 1018, 1016, 968, 1007, 1076, 1063, 1067, 978, 1011, 1045, 1028, 1011, 1038, 1001, 1000, 1009, 1019, 1000, 1022, 1056, 1026, 1008, 1050, 1044, 1059, 1030, 1027, 1052, 1050, 1031, 1005, 983, 1037, 1016, 1045, 1042, 1062, 1030, 992, 1052, 1038, 986, 1007, 1019, 968, 1031, 1062, 984, 1010, 1041, 1034, 962, 1051, 1033, 1001, 1029, 1015, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1050, 1027, 1048, 1023, 1044, 1023, 1031, 1021, 989, 1016, 1038, 1014, 1025, 998, 1041, 1061, 1043, 1046, 1025, 1101, 1028, 998, 1029, 1028, 1028, 1029, 1015, 1021, 1035, 1039, 1030, 1036, 1037, 993, 981, 1007, 1026, 1017, 1030, 1117, 1024, 1036, 1005, 1042, 1005, 967, 1009, 1067, 1033, 1018, 1022, 1055, 993, 998, 1002, 992, 1043, 1028, 993, 1046, 967, 1014, 1010, 979] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1067, 1064, 982, 1036, 1016, 961, 1063, 1058, 1021, 1028, 1039, 1003, 1028, 1010, 1066, 1037, 1014, 1034, 1054, 1028, 1056, 978, 967, 1025, 1000, 1019, 1034, 1042, 1032, 993, 1059, 980, 1029, 1052, 1020, 964, 1005, 1026, 1050, 1023, 997, 1051, 1058, 1015, 1024, 1069, 1032, 1069, 974, 1056, 1033, 1018, 998, 990, 991, 985, 1080, 996, 1027, 1036, 1021, 1002, 1017, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1024, 1052, 1058, 1033, 1080, 1021, 1031, 1000, 996, 965, 1040, 999, 1059, 1038, 1029, 1025, 1006, 1073, 1073, 998, 1030, 1087, 1009, 978, 1036, 956, 1013, 1084, 1004, 1007, 1020, 1024, 962, 1031, 1011, 1020, 1020, 1021, 1004, 1036, 1011, 1012, 1022, 1005, 1042, 1062, 1019, 976, 1045, 1043, 1008, 1012, 1053, 1049, 1006, 973, 1029, 1081, 997, 1007, 1044, 1027, 1035, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1042, 1023, 1022, 1043, 1063, 1013, 991, 1013, 1055, 984, 1045, 1004, 1071, 1025, 981, 1000, 1049, 1027, 994, 1021, 1002, 951, 1061, 1030, 981, 1020, 1014, 1034, 1038, 1005, 1019, 1010, 1037, 969, 1039, 1040, 1049, 975, 1068, 964, 1014, 1033, 1045, 1065, 1044, 1003, 1031, 1040, 1020, 1018, 1022, 1074, 1060, 1035, 1028, 1058, 1094, 1028, 1016, 978, 1036, 1034, 1017, 971] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1073, 1066, 1047, 1021, 990, 984, 1013, 1070, 1051, 987, 1020, 1012, 1004, 1021, 1051, 1072, 990, 1022, 981, 1013, 1002, 1042, 1047, 992, 1027, 1014, 1026, 1026, 1011, 982, 1042, 1021, 1035, 1013, 1043, 1059, 1056, 995, 976, 1012, 1042, 1004, 1036, 1064, 1009, 1082, 1038, 1063, 1003, 1058, 1022, 1029, 1031, 975, 1002, 1010, 1020, 1001, 1019, 1015, 1031, 1002, 1050, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1072, 978, 1011, 1057, 1018, 1047, 1070, 1018, 1006, 1065, 1026, 1054, 1014, 1066, 1061, 1041, 1079, 1042, 957, 989, 1024, 1009, 1069, 988, 1023, 1051, 1018, 1009, 1037, 1018, 1048, 1039, 1021, 1056, 1000, 1014, 1006, 1052, 1029, 1037, 989, 1037, 1030, 1008, 1044, 1053, 1009, 1019, 1026, 1021, 1006, 1018, 989, 1038, 997, 946, 1022, 1009, 1036, 965, 998, 973, 1027, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1030, 1010, 1033, 1039, 1084, 1033, 1011, 1039, 1040, 1020, 1044, 976, 1079, 1079, 1036, 1032, 945, 1003, 1061, 1029, 1025, 1024, 1054, 1071, 1013, 995, 1037, 1022, 991, 1003, 980, 1013, 1017, 1052, 995, 1068, 1049, 1069, 1053, 1031, 1012, 1045, 1034, 1072, 1012, 1027, 995, 1045, 975, 992, 1014, 996, 1045, 964, 1006, 1040, 1001, 1002, 1030, 1001, 1002, 1019, 1023, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[949, 1068, 1017, 1042, 985, 1036, 1018, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1007, 1076, 1063, 1034, 978, 1011, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1011, 1038, 1001, 1000, 1009, 1019, 1000, 13, 3, 0, 0, 0, 0, 0, 0, 1022, 1056, 1026, 1008, 1050, 1044, 1059, 1030, 17, 0, 0, 0, 0, 0, 0, 0, 1027, 1052, 1050, 1031, 1005, 983, 1034, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1042, 1062, 1030, 992, 1006, 1038, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1019, 968, 1031, 1062, 984, 1010, 1041, 59, 7, 0, 0, 0, 0, 0, 0, 1034, 962, 1051, 1033, 1001, 1029, 1015, 1035, 39, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8182, 8122, 8312, 8198, 8201, 8188, 8202] avg=8192.0000 max=8312 max_rank=3 min=8122 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[989, 1027, 1048, 1023, 1044, 1023, 1031, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1016, 1038, 1014, 994, 998, 1041, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1046, 1025, 1101, 1028, 998, 1029, 1028, 12, 3, 0, 0, 0, 0, 0, 0, 1028, 1029, 1015, 1021, 1035, 1039, 1030, 1036, 16, 0, 0, 0, 0, 0, 0, 0, 1037, 993, 981, 1007, 1026, 1017, 1027, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1036, 1005, 1042, 1005, 925, 1009, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1018, 1022, 1055, 993, 998, 1002, 992, 61, 6, 0, 0, 0, 0, 0, 0, 1043, 1028, 993, 1046, 967, 1014, 1010, 979, 36, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8151, 8313, 8249, 8205, 8113, 8180, 8119] avg=8192.0000 max=8313 max_rank=2 min=8113 min_rank=5 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32617 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[964, 1052, 1058, 1033, 1080, 1021, 1031, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 996, 965, 1040, 999, 1026, 1038, 1029, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1073, 1073, 998, 1030, 1087, 1009, 978, 13, 3, 0, 0, 0, 0, 0, 0, 1036, 956, 1013, 1084, 1004, 1007, 1020, 1024, 17, 0, 0, 0, 0, 0, 0, 0, 962, 1031, 1011, 1020, 1020, 1021, 1001, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1012, 1022, 1005, 1042, 1015, 1019, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1043, 1008, 1012, 1053, 1049, 1006, 973, 60, 7, 0, 0, 0, 0, 0, 0, 1029, 1081, 997, 1007, 1044, 1027, 1035, 1025, 40, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8118, 8270, 8161, 8102, 8102, 8256, 8288] avg=8192.0000 max=8288 max_rank=7 min=8102 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[981, 1023, 1022, 1043, 1063, 1013, 991, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 984, 1045, 1004, 1038, 1025, 981, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1027, 994, 1021, 1002, 951, 1061, 1030, 13, 3, 0, 0, 0, 0, 0, 0, 981, 1020, 1014, 1034, 1038, 1005, 1019, 1010, 17, 0, 0, 0, 0, 0, 0, 0, 1037, 969, 1039, 1040, 1049, 975, 1065, 964, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1033, 1045, 1065, 1044, 959, 1031, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1018, 1022, 1074, 1060, 1035, 1028, 1058, 61, 6, 0, 0, 0, 0, 0, 0, 1094, 1028, 1016, 978, 1036, 1034, 1017, 971, 38, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8132, 8151, 8138, 8138, 8231, 8382, 8215] avg=8192.0000 max=8382 max_rank=6 min=8132 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32966 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1005, 1064, 982, 1036, 1016, 961, 1063, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1028, 1039, 1003, 997, 1010, 1066, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1034, 1054, 1028, 1056, 978, 967, 1025, 12, 3, 0, 0, 0, 0, 0, 0, 1000, 1019, 1034, 1042, 1032, 993, 1059, 980, 16, 0, 0, 0, 0, 0, 0, 0, 1029, 1052, 1020, 964, 1005, 1026, 1047, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1051, 1058, 1015, 1024, 1022, 1032, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1056, 1033, 1018, 998, 990, 991, 985, 62, 7, 0, 0, 0, 0, 0, 0, 1080, 996, 1027, 1036, 1021, 1002, 1017, 1034, 40, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8201, 8171, 8175, 8166, 8268, 8114, 8256] avg=8192.0000 max=8268 max_rank=5 min=8114 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1011, 1066, 1047, 1021, 990, 984, 1013, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 987, 1020, 1012, 973, 1021, 1051, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1022, 981, 1013, 1002, 1042, 1047, 992, 12, 3, 0, 0, 0, 0, 0, 0, 1027, 1014, 1026, 1026, 1011, 982, 1042, 1021, 16, 0, 0, 0, 0, 0, 0, 0, 1035, 1013, 1043, 1059, 1056, 995, 973, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1004, 1036, 1064, 1009, 1034, 1038, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1058, 1022, 1029, 1031, 975, 1002, 1010, 62, 7, 0, 0, 0, 0, 0, 0, 1020, 1001, 1019, 1015, 1031, 1002, 1050, 1021, 41, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8187, 8104, 8165, 8186, 8290, 8199, 8203] avg=8192.0000 max=8290 max_rank=5 min=8104 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[970, 1010, 1033, 1039, 1084, 1033, 1011, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1020, 1044, 976, 1046, 1079, 1036, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 945, 1003, 1061, 1029, 1025, 1024, 1054, 1071, 13, 3, 0, 0, 0, 0, 0, 0, 1013, 995, 1037, 1022, 991, 1003, 980, 1013, 17, 0, 0, 0, 0, 0, 0, 0, 1017, 1052, 995, 1068, 1049, 1069, 1050, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1045, 1034, 1072, 1012, 981, 995, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 975, 992, 1014, 996, 1045, 964, 1006, 1040, 60, 7, 0, 0, 0, 0, 0, 0, 1001, 1002, 1030, 1001, 1002, 1019, 1023, 999, 39, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8273, 8228, 8071, 8331, 8196, 8099, 8119] avg=8192.0000 max=8331 max_rank=4 min=8071 min_rank=3 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1006, 978, 1011, 1057, 1018, 1047, 1070, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1065, 1026, 1054, 973, 1066, 1061, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1042, 957, 989, 1024, 1009, 1069, 988, 15, 5, 0, 0, 0, 0, 0, 0, 1023, 1051, 1018, 1009, 1037, 1018, 1048, 1039, 22, 0, 0, 0, 0, 0, 0, 0, 1021, 1056, 1000, 1014, 1006, 1052, 1024, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1037, 1030, 1008, 1044, 999, 1009, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1021, 1006, 1018, 989, 1038, 997, 946, 66, 11, 0, 0, 0, 0, 0, 0, 1022, 1009, 1036, 965, 998, 973, 1027, 1057, 43, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8292, 8177, 8265, 8210, 8135, 8118, 8134] avg=8192.0000 max=8292 max_rank=1 min=8118 min_rank=6 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8064, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8448, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8448, 8192, 8192, 8192, 8192, 8192, 8320, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8064, 8064, 8064, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7936, 8320, 8320, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1024, 1004, 1048, 1022, 1005, 1036, 1061, 994, 1026, 1025, 1049, 983, 1051, 933, 1009, 1005, 997, 1024, 1036, 996, 998, 1037, 1036, 1018, 1068, 1058, 1028, 1006, 1044, 1016, 1060, 1032, 1006, 1010, 1019, 1013, 1042, 1032, 1010, 1020, 1045, 1049, 1057, 1009, 1039, 1030, 1008, 1008, 1003, 1034, 1031, 1044, 1020, 1068, 1009, 1003, 1051, 1033, 1017, 1042, 1021, 985, 1041, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1030, 1016, 1046, 1045, 964, 1020, 1017, 1068, 1009, 1036, 1022, 1038, 1016, 1067, 1015, 992, 1052, 1097, 1044, 986, 1023, 1069, 983, 1068, 1068, 1063, 1021, 986, 982, 991, 1005, 968, 1020, 991, 989, 1012, 984, 1061, 1008, 1019, 1051, 1070, 1035, 970, 1006, 1040, 1003, 1012, 972, 1024, 1054, 1025, 1028, 1036, 999, 993, 1074, 1066, 1019, 1052, 1039, 1020, 1011, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1015, 1010, 1014, 1021, 1036, 1049, 1000, 1060, 1054, 1073, 1010, 992, 972, 1010, 1022, 1024, 1040, 1057, 1024, 1042, 1031, 982, 1033, 1028, 1032, 1076, 996, 1016, 997, 1065, 1037, 1086, 1013, 955, 1021, 999, 1015, 1001, 1086, 980, 1034, 1008, 985, 1046, 1035, 1018, 984, 1043, 1040, 1034, 1006, 1051, 1056, 995, 1073, 1059, 976, 995, 1015, 1035, 1033, 995, 1038, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1019, 1062, 1006, 986, 1012, 1034, 1035, 1026, 993, 998, 986, 1054, 998, 1047, 1052, 1006, 1027, 1045, 1043, 969, 1073, 1058, 1034, 1016, 1011, 973, 1003, 1006, 1093, 995, 1019, 1039, 1033, 997, 1033, 1026, 1012, 1012, 1069, 1014, 1052, 1029, 1003, 1005, 1014, 1056, 1017, 1046, 982, 1059, 1040, 1021, 1035, 969, 1026, 995, 1004, 1056, 1036, 1064, 1015, 1034, 1056, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1030, 982, 1067, 1078, 1013, 991, 1024, 1062, 1026, 991, 1095, 1066, 972, 986, 1050, 1011, 964, 991, 993, 1015, 1020, 1034, 1069, 1028, 1027, 1029, 1011, 1064, 1064, 1017, 1012, 994, 1010, 968, 957, 1037, 1007, 1035, 1009, 1021, 1021, 1042, 1073, 1005, 985, 985, 1038, 1046, 1055, 1020, 1022, 1086, 997, 992, 1023, 1051, 976, 1022, 1002, 1020, 1098, 1040, 1067, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[983, 1029, 1013, 1058, 1057, 1037, 1057, 978, 998, 1044, 1026, 991, 1041, 1069, 993, 1035, 1012, 981, 1003, 1010, 964, 1005, 1001, 1018, 1011, 1011, 1030, 1017, 1002, 1012, 962, 1069, 1039, 1052, 1004, 1053, 1048, 1047, 993, 1036, 1045, 1029, 1041, 1027, 996, 1069, 1017, 962, 1039, 1001, 1066, 1024, 1057, 1126, 1071, 982, 1033, 1022, 1049, 994, 1011, 1017, 1054, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1033, 1030, 1011, 1019, 1048, 1032, 1005, 1064, 1007, 1000, 986, 996, 1013, 1015, 1045, 1031, 1030, 1010, 1018, 1027, 1022, 1071, 1006, 986, 1026, 1054, 1011, 1042, 1022, 1012, 996, 1000, 1023, 1031, 1013, 1017, 1016, 1031, 1036, 1012, 1007, 959, 1026, 1002, 1020, 1016, 977, 1087, 1010, 1005, 1030, 1117, 1016, 1040, 1046, 1046, 1009, 1047, 1002, 1052, 1074, 1050, 1021, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1031, 1065, 1028, 1003, 967, 1047, 1025, 1049, 1020, 1026, 1011, 1036, 1051, 1005, 1059, 1012, 1022, 1040, 1032, 1001, 1009, 998, 1029, 1051, 1032, 1040, 1029, 1018, 1024, 1031, 1014, 1017, 1058, 1022, 1005, 1025, 1009, 989, 1007, 1002, 1071, 992, 982, 1056, 1067, 987, 1001, 1058, 1021, 1020, 1049, 1008, 999, 1008, 1074, 1024, 1009, 1034, 1024, 1036, 1037, 988, 1031, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1024, 1004, 1048, 1022, 1005, 1036, 1061, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1025, 1049, 983, 1051, 933, 1009, 1005, 29, 2, 0, 0, 0, 0, 0, 0, 997, 1024, 1036, 996, 998, 1037, 1036, 1018, 13, 0, 0, 0, 0, 0, 0, 0, 1068, 1058, 1028, 1006, 1044, 1016, 1060, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1010, 1019, 1013, 1042, 1032, 1010, 1020, 46, 6, 0, 0, 0, 0, 0, 0, 1045, 1049, 1057, 1009, 1039, 1030, 1008, 1008, 4, 0, 0, 0, 0, 0, 0, 0, 1003, 1034, 1031, 998, 1020, 1068, 1009, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1033, 1017, 1042, 986, 985, 1041, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8112, 8155, 8312, 8204, 8249, 8166, 8163] avg=8192.0000 max=8312 max_rank=3 min=8112 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1030, 1016, 1046, 1045, 964, 1020, 1017, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1036, 1022, 1038, 1016, 1067, 1015, 992, 30, 2, 0, 0, 0, 0, 0, 0, 1052, 1097, 1044, 986, 1023, 1069, 983, 1068, 14, 0, 0, 0, 0, 0, 0, 0, 1068, 1063, 1021, 986, 982, 991, 1005, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 991, 989, 1012, 984, 1061, 1008, 1019, 45, 6, 0, 0, 0, 0, 0, 0, 1051, 1070, 1035, 970, 1006, 1040, 1003, 1012, 4, 0, 0, 0, 0, 0, 0, 0, 972, 1024, 1054, 980, 1028, 1036, 999, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1066, 1019, 1052, 1003, 1020, 1011, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8227, 8336, 8084, 8135, 8191, 8086, 8291] avg=8192.0000 max=8336 max_rank=2 min=8084 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[983, 1029, 1013, 1058, 1057, 1037, 1057, 960, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1044, 1026, 991, 1041, 1069, 993, 1035, 29, 2, 0, 0, 0, 0, 0, 0, 1012, 981, 1003, 1010, 964, 1005, 1001, 1018, 12, 0, 0, 0, 0, 0, 0, 0, 1011, 1011, 1030, 1017, 1002, 1012, 962, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1052, 1004, 1053, 1048, 1047, 993, 1036, 45, 5, 0, 0, 0, 0, 0, 0, 1045, 1029, 1041, 1027, 996, 1069, 1017, 962, 4, 0, 0, 0, 0, 0, 0, 0, 1039, 1001, 1066, 979, 1057, 1126, 1071, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1022, 1049, 994, 977, 1017, 1054, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8228, 8006, 8114, 8322, 8190, 8321, 8161] avg=8192.0000 max=8322 max_rank=4 min=8006 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32994 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1019, 1062, 1006, 986, 1012, 1034, 1035, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 993, 998, 986, 1054, 998, 1047, 1052, 1006, 29, 2, 0, 0, 0, 0, 0, 0, 1027, 1045, 1043, 969, 1073, 1058, 1034, 1016, 13, 0, 0, 0, 0, 0, 0, 0, 1011, 973, 1003, 1006, 1093, 995, 1019, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 997, 1033, 1026, 1012, 1012, 1069, 1014, 45, 5, 0, 0, 0, 0, 0, 0, 1052, 1029, 1003, 1005, 1014, 1056, 1017, 1046, 4, 0, 0, 0, 0, 0, 0, 0, 982, 1059, 1040, 976, 1035, 969, 1026, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1056, 1036, 1064, 981, 1034, 1056, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8165, 8278, 8139, 8246, 8226, 8082, 8239] avg=8192.0000 max=8278 max_rank=2 min=8082 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1030, 982, 1067, 1078, 1013, 991, 1024, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 991, 1095, 1066, 972, 986, 1050, 1011, 32, 2, 0, 0, 0, 0, 0, 0, 964, 991, 993, 1015, 1020, 1034, 1069, 1028, 14, 0, 0, 0, 0, 0, 0, 0, 1027, 1029, 1011, 1064, 1064, 1017, 1012, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 968, 957, 1037, 1007, 1035, 1009, 1021, 48, 6, 0, 0, 0, 0, 0, 0, 1021, 1042, 1073, 1005, 985, 985, 1038, 1046, 4, 0, 0, 0, 0, 0, 0, 0, 1055, 1020, 1022, 1038, 997, 992, 1023, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1022, 1002, 1020, 1060, 1040, 1067, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8231, 8128, 8218, 8098, 8199, 8198, 8237] avg=8192.0000 max=8237 max_rank=7 min=8098 min_rank=4 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1015, 1010, 1014, 1021, 1036, 1049, 1000, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1073, 1010, 992, 972, 1010, 1022, 1024, 30, 2, 0, 0, 0, 0, 0, 0, 1040, 1057, 1024, 1042, 1031, 982, 1033, 1028, 14, 0, 0, 0, 0, 0, 0, 0, 1032, 1076, 996, 1016, 997, 1065, 1037, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 955, 1021, 999, 1015, 1001, 1086, 980, 46, 6, 0, 0, 0, 0, 0, 0, 1034, 1008, 985, 1046, 1035, 1018, 984, 1043, 4, 0, 0, 0, 0, 0, 0, 0, 1040, 1034, 1006, 1005, 1056, 995, 1073, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 976, 995, 1015, 1035, 997, 995, 1038, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8189, 8251, 8305, 8122, 8157, 8268, 8059] avg=8192.0000 max=8305 max_rank=3 min=8059 min_rank=7 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32606 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1033, 1030, 1011, 1019, 1048, 1032, 1005, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1000, 986, 996, 1013, 1015, 1045, 1031, 31, 2, 0, 0, 0, 0, 0, 0, 1030, 1010, 1018, 1027, 1022, 1071, 1006, 986, 14, 0, 0, 0, 0, 0, 0, 0, 1026, 1054, 1011, 1042, 1022, 1012, 996, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1031, 1013, 1017, 1016, 1031, 1036, 1012, 49, 6, 0, 0, 0, 0, 0, 0, 1007, 959, 1026, 1002, 1020, 1016, 977, 1087, 4, 0, 0, 0, 0, 0, 0, 0, 1010, 1005, 1030, 1068, 1016, 1040, 1046, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1047, 1002, 1052, 1037, 1050, 1021, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8126, 8184, 8163, 8234, 8098, 8261, 8248] avg=8192.0000 max=8261 max_rank=6 min=8098 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1031, 1065, 1028, 1003, 967, 1047, 1025, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1026, 1011, 1036, 1051, 1005, 1059, 1012, 33, 5, 0, 0, 0, 0, 0, 0, 1022, 1040, 1032, 1001, 1009, 998, 1029, 1051, 16, 0, 0, 0, 0, 0, 0, 0, 1032, 1036, 1029, 1018, 1024, 1031, 1014, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1022, 1005, 1025, 1009, 989, 1007, 1002, 49, 9, 0, 0, 0, 0, 0, 0, 1071, 992, 982, 1056, 1067, 987, 1001, 1058, 8, 4, 0, 0, 0, 0, 0, 0, 1021, 1020, 1049, 959, 999, 1008, 1074, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1034, 1024, 1036, 995, 988, 1031, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8258, 8198, 8201, 8175, 8226, 8154, 8138] avg=8192.0000 max=8258 max_rank=1 min=8138 min_rank=7 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8320, 8064, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8448, 8192, 8320, 8192, 8192, 8320, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8064, 8320, 8320, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 8192, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8064, 8192, 8192, 8320, 8320, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1044, 1026, 1061, 1024, 1020, 984, 987, 1046, 975, 1022, 1035, 1025, 1064, 1039, 1047, 1037, 1076, 1023, 1068, 1034, 1029, 1038, 994, 1061, 1000, 994, 990, 1006, 1031, 1037, 1029, 1024, 1063, 1011, 995, 1070, 1010, 1032, 1036, 1012, 1024, 998, 978, 1030, 1052, 1009, 1039, 1027, 1029, 1014, 1019, 970, 1019, 1073, 1007, 1032, 1048, 997, 1018, 1030, 1005, 1004, 986, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1011, 1048, 1002, 1096, 1032, 1034, 996, 1040, 1004, 1037, 1018, 1047, 1050, 1048, 1070, 1058, 1046, 1011, 1038, 1045, 1019, 1009, 978, 1060, 995, 1027, 1029, 1019, 1021, 1043, 1039, 993, 1025, 1005, 1047, 1042, 996, 1032, 974, 1039, 1000, 1045, 1027, 972, 1008, 1012, 984, 1038, 986, 1016, 1009, 1018, 982, 974, 1047, 1031, 1000, 1018, 1011, 1049, 1085, 1046, 1024, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1060, 1051, 1001, 1012, 982, 1049, 964, 1030, 1002, 1001, 1054, 1044, 988, 1100, 1000, 1037, 1007, 996, 1023, 1031, 996, 1042, 983, 1046, 1030, 1006, 1017, 1031, 1007, 1000, 1042, 1046, 988, 980, 1027, 1041, 1056, 1030, 1055, 995, 1030, 1102, 1042, 983, 1029, 1029, 1019, 1089, 1052, 984, 993, 1043, 1023, 1041, 1028, 1051, 1037, 981, 1016, 1019, 1025, 1038, 1014, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1000, 1033, 1019, 994, 1006, 1028, 1072, 1053, 1053, 1030, 993, 953, 1024, 1028, 1064, 1017, 1056, 978, 1022, 1015, 1047, 1021, 1000, 1002, 1049, 1042, 1043, 1042, 1010, 1028, 1003, 1097, 982, 1046, 1031, 1007, 1054, 1053, 1024, 1025, 1011, 970, 1026, 1026, 1002, 1049, 1005, 985, 1012, 1035, 1037, 1007, 1039, 987, 1015, 999, 1027, 998, 1076, 1051, 1029, 1051, 1031, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1024, 1002, 1025, 1061, 1007, 1045, 1007, 1008, 1043, 1064, 992, 1028, 1016, 1028, 1048, 1056, 1012, 1042, 1060, 1025, 1014, 999, 1026, 1040, 981, 986, 979, 1035, 1020, 987, 997, 1008, 1038, 1024, 1044, 1014, 1003, 1072, 1090, 989, 1033, 1034, 1028, 1015, 1023, 1020, 1043, 997, 1061, 1019, 1030, 1059, 1002, 1002, 998, 981, 999, 1002, 1036, 1020, 1048, 1034, 1037, 1076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1068, 1026, 1033, 1013, 1018, 1006, 1046, 1002, 1000, 988, 1009, 1032, 996, 1045, 1001, 992, 1009, 1026, 996, 1045, 1053, 1034, 1024, 1063, 1029, 1020, 1018, 1000, 1081, 1023, 1055, 1032, 976, 1035, 1008, 1066, 1051, 1037, 1050, 957, 965, 981, 1022, 1021, 1045, 1090, 992, 1058, 1037, 1023, 1053, 1036, 1041, 1040, 1026, 1045, 1023, 1004, 1056, 988, 1051, 1002, 981, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[971, 1046, 1058, 1038, 977, 962, 1024, 1009, 1022, 1065, 1081, 1027, 1032, 1018, 1018, 1033, 996, 1049, 1011, 1048, 1009, 1051, 1034, 1058, 993, 1051, 1026, 983, 1063, 1034, 1020, 1087, 1025, 1053, 1010, 1008, 1028, 1049, 1040, 1039, 1009, 1034, 1053, 1037, 989, 1005, 999, 989, 1025, 1018, 980, 1037, 999, 1047, 1040, 1063, 1045, 1015, 1016, 999, 1006, 1032, 1005, 948] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1058, 988, 1045, 1028, 1001, 1031, 975, 1041, 1032, 1073, 1074, 983, 1062, 1065, 996, 1037, 1046, 1058, 1025, 1021, 996, 996, 1002, 1076, 1001, 1012, 1018, 970, 1024, 1016, 971, 1055, 948, 946, 1052, 1071, 1042, 1018, 1024, 1015, 1022, 1021, 1017, 1047, 995, 1032, 998, 1027, 1062, 1014, 1053, 1060, 1049, 1055, 1032, 989, 1021, 1031, 1020, 1022, 1000, 1024, 1027, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1060, 1051, 1001, 1012, 982, 1049, 964, 1030, 8, 1, 0, 0, 0, 0, 0, 0, 1002, 1001, 1054, 1044, 988, 1051, 1000, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 996, 1023, 1031, 996, 1042, 983, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1006, 1017, 1031, 1007, 1000, 1042, 1046, 16, 7, 0, 0, 0, 0, 0, 0, 988, 980, 1027, 1041, 1056, 1022, 1055, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1102, 1042, 983, 1029, 1029, 1019, 1089, 33, 0, 0, 0, 0, 0, 0, 0, 1051, 984, 993, 1043, 1023, 1041, 1028, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 981, 1016, 1019, 1025, 1038, 1014, 1018, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8177, 8101, 8202, 8164, 8356, 8214, 8164] avg=8192.0000 max=8356 max_rank=5 min=8101 min_rank=2 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1044, 1026, 1061, 1024, 1020, 984, 987, 1046, 8, 1, 0, 0, 0, 0, 0, 0, 975, 1022, 1035, 1025, 1064, 992, 1047, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1023, 1068, 1034, 1029, 1038, 994, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 994, 990, 1006, 1031, 1037, 1029, 1024, 16, 7, 0, 0, 0, 0, 0, 0, 1063, 1011, 995, 1070, 1010, 1024, 1036, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 998, 978, 1030, 1052, 1009, 1039, 1027, 31, 0, 0, 0, 0, 0, 0, 0, 1028, 1014, 1019, 970, 1019, 1073, 1007, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 997, 1018, 1030, 1005, 1004, 986, 1029, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8197, 8300, 8134, 8221, 8188, 8162, 8133] avg=8192.0000 max=8300 max_rank=2 min=8133 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[971, 1046, 1058, 1038, 977, 962, 1024, 1009, 8, 1, 0, 0, 0, 0, 0, 0, 1022, 1065, 1081, 1027, 1032, 972, 1018, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1049, 1011, 1048, 1009, 1051, 1034, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1051, 1026, 983, 1063, 1034, 1020, 1087, 15, 7, 0, 0, 0, 0, 0, 0, 1025, 1053, 1010, 1008, 1028, 1041, 1040, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1034, 1053, 1037, 989, 1005, 999, 989, 31, 0, 0, 0, 0, 0, 0, 0, 1024, 1018, 980, 1037, 999, 1047, 1040, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1015, 1016, 999, 1006, 1032, 1005, 948, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8094, 8250, 8233, 8279, 8244, 8146, 8208, 8082] avg=8192.0000 max=8279 max_rank=3 min=8082 min_rank=7 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1011, 1048, 1002, 1096, 1032, 1034, 996, 1040, 8, 1, 0, 0, 0, 0, 0, 0, 1004, 1037, 1018, 1047, 1050, 1001, 1070, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1011, 1038, 1045, 1019, 1009, 978, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1027, 1029, 1019, 1021, 1043, 1039, 993, 16, 7, 0, 0, 0, 0, 0, 0, 1025, 1005, 1047, 1042, 996, 1024, 974, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1045, 1027, 972, 1008, 1012, 984, 1038, 31, 0, 0, 0, 0, 0, 0, 0, 985, 1016, 1009, 1018, 982, 974, 1047, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1018, 1011, 1049, 1085, 1046, 1024, 1031, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8285, 8183, 8189, 8152, 8117, 8062, 8280] avg=8192.0000 max=8285 max_rank=1 min=8062 min_rank=6 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32611 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1000, 1033, 1019, 994, 1006, 1028, 1072, 1053, 8, 1, 0, 0, 0, 0, 0, 0, 1053, 1030, 993, 953, 1024, 982, 1064, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 978, 1022, 1015, 1047, 1021, 1000, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1042, 1043, 1042, 1010, 1028, 1003, 1097, 15, 7, 0, 0, 0, 0, 0, 0, 982, 1046, 1031, 1007, 1054, 1045, 1024, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 970, 1026, 1026, 1002, 1049, 1005, 985, 31, 0, 0, 0, 0, 0, 0, 0, 1011, 1035, 1037, 1007, 1039, 987, 1015, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 998, 1076, 1051, 1029, 1051, 1031, 1024, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8116, 8119, 8336, 8214, 8105, 8130, 8302] avg=8192.0000 max=8336 max_rank=3 min=8105 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1024, 1002, 1025, 1061, 1007, 1045, 1007, 1008, 8, 1, 0, 0, 0, 0, 0, 0, 1043, 1064, 992, 1028, 1016, 982, 1048, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1042, 1060, 1025, 1014, 999, 1026, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 981, 986, 979, 1035, 1020, 987, 997, 1008, 15, 7, 0, 0, 0, 0, 0, 0, 1038, 1024, 1044, 1014, 1003, 1064, 1090, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1034, 1028, 1015, 1023, 1020, 1043, 997, 31, 0, 0, 0, 0, 0, 0, 0, 1060, 1019, 1030, 1059, 1002, 1002, 998, 981, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1002, 1036, 1020, 1048, 1034, 1037, 1076, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8229, 8195, 8015, 8266, 8224, 8151, 8268] avg=8192.0000 max=8268 max_rank=7 min=8015 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1068, 1026, 1033, 1013, 1018, 1006, 1046, 1002, 8, 1, 0, 0, 0, 0, 0, 0, 1000, 988, 1009, 1032, 996, 998, 1001, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1026, 996, 1045, 1053, 1034, 1024, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1020, 1018, 1000, 1081, 1023, 1055, 1032, 16, 7, 0, 0, 0, 0, 0, 0, 976, 1035, 1008, 1066, 1051, 1029, 1050, 957, 0, 0, 0, 0, 0, 0, 0, 0, 965, 981, 1022, 1021, 1045, 1090, 992, 1058, 31, 0, 0, 0, 0, 0, 0, 0, 1036, 1023, 1053, 1036, 1041, 1040, 1026, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1004, 1056, 988, 1051, 1002, 981, 993, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8016, 8227, 8281, 8172, 8205, 8300, 8114] avg=8192.0000 max=8300 max_rank=6 min=8016 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1058, 988, 1045, 1028, 1001, 1031, 975, 1041, 13, 5, 7, 0, 0, 0, 0, 0, 1032, 1073, 1074, 983, 1062, 1009, 996, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1058, 1025, 1021, 996, 996, 1002, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1012, 1018, 970, 1024, 1016, 971, 1055, 20, 13, 0, 0, 0, 0, 0, 0, 948, 946, 1052, 1071, 1042, 1005, 1024, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1021, 1017, 1047, 995, 1032, 998, 1027, 36, 0, 0, 0, 0, 0, 0, 0, 1057, 1014, 1053, 1060, 1049, 1055, 1032, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1031, 1020, 1022, 1000, 1024, 1027, 1026, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8266, 8178, 8100, 8103, 8195, 8309, 8193] avg=8192.0000 max=8309 max_rank=6 min=8100 min_rank=3 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8320, 8320, 8064, 8192, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8192, 8320, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8448, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1086, 1040, 1006, 1024, 1045, 1027, 1043, 1046, 1058, 1070, 1037, 993, 992, 999, 1073, 1009, 1028, 1015, 1043, 976, 1004, 1027, 1064, 979, 1049, 1065, 1017, 1012, 1028, 991, 1063, 981, 1023, 1049, 982, 1031, 1058, 1035, 943, 996, 1022, 1008, 1037, 1028, 1035, 1046, 987, 976, 1034, 1006, 958, 1019, 1060, 1076, 1058, 1012, 1016, 1015, 1029, 1082, 1017, 978, 1003, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1021, 1061, 1037, 1116, 1070, 1019, 1052, 1042, 997, 944, 971, 1013, 971, 956, 1042, 1076, 1038, 1076, 1038, 984, 1016, 1032, 1009, 1011, 1048, 1001, 1046, 1014, 1045, 1016, 969, 1031, 995, 1002, 1015, 1030, 1006, 1087, 1051, 1002, 975, 1051, 1038, 980, 1025, 1042, 1004, 1077, 1045, 978, 1032, 1022, 1050, 1083, 1036, 1003, 1030, 1034, 984, 1038, 1009, 1060, 973, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1087, 1044, 1010, 1034, 1006, 1006, 1083, 984, 1001, 1039, 1001, 1040, 1028, 1009, 996, 1055, 997, 992, 974, 993, 1047, 1034, 982, 1044, 1013, 1047, 1025, 1029, 956, 1060, 1043, 1004, 1035, 1060, 1047, 1064, 987, 1042, 1022, 1021, 988, 1025, 1053, 1059, 1012, 1010, 978, 1050, 1004, 1054, 1008, 985, 1032, 1008, 1030, 1002, 1049, 1041, 1045, 1038, 1031, 1043, 1009, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1011, 1029, 978, 1036, 991, 1009, 951, 1061, 986, 1039, 987, 1117, 973, 1059, 1046, 1026, 1058, 1055, 994, 1062, 1062, 1023, 1057, 1015, 1004, 1033, 1025, 987, 1027, 1057, 993, 1057, 1012, 1002, 994, 1012, 1026, 1064, 1024, 976, 1052, 1059, 1013, 1038, 1045, 1011, 1021, 985, 995, 1014, 1035, 997, 1064, 1041, 1094, 993, 998, 1005, 1017, 1044, 1009, 994, 1063, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1016, 1004, 1005, 1002, 1035, 1004, 1035, 1065, 1010, 1053, 1059, 1037, 1033, 1043, 1019, 1009, 1029, 1023, 1023, 993, 1006, 1040, 1016, 942, 1065, 1028, 1064, 1051, 994, 1043, 966, 1037, 1041, 1004, 1085, 1042, 1009, 1044, 993, 1020, 1012, 1001, 1011, 1043, 1025, 1043, 1018, 994, 1074, 1072, 1017, 1054, 1016, 1013, 1043, 996, 1032, 991, 1051, 1059, 948, 992, 1001, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[991, 1074, 1092, 989, 999, 1007, 992, 1006, 1035, 1036, 1002, 1003, 1018, 987, 1026, 1030, 1054, 1043, 1011, 1029, 1007, 999, 996, 1011, 962, 1033, 1037, 979, 1019, 1052, 1007, 1026, 1029, 992, 1019, 1064, 1058, 1004, 1008, 1051, 1043, 1008, 1010, 994, 1035, 1009, 1049, 1037, 1029, 1046, 1048, 1003, 1052, 1004, 1049, 1063, 1066, 1011, 1016, 1010, 1037, 1044, 1081, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1043, 1002, 1008, 1019, 989, 991, 989, 1048, 1036, 991, 1038, 1049, 1074, 1008, 1044, 1064, 1070, 979, 1056, 996, 1062, 1045, 965, 1025, 993, 1072, 1073, 997, 994, 991, 1039, 1081, 967, 1020, 1049, 1025, 1049, 1038, 1003, 1049, 1090, 997, 987, 1015, 1039, 1000, 998, 997, 1044, 995, 1015, 991, 1075, 1025, 1021, 1051, 1028, 1023, 991, 1055, 1027, 1011, 999, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1028, 1022, 1070, 1076, 1007, 984, 1004, 1016, 1040, 992, 1009, 1036, 1010, 1061, 1041, 1028, 1000, 998, 1014, 1014, 1038, 1007, 1061, 1039, 1009, 1077, 1013, 987, 1035, 1049, 1007, 1022, 1039, 1021, 992, 986, 1032, 979, 957, 1002, 1046, 1056, 1036, 1061, 1014, 1029, 1048, 998, 1022, 1021, 1056, 1040, 1077, 988, 994, 1068, 955, 1041, 1030, 1019, 1024, 1045, 1008, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1087, 1044, 1010, 1020, 1006, 1006, 1083, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1039, 1001, 1040, 1028, 1009, 996, 1055, 1, 0, 0, 0, 0, 0, 0, 0, 997, 992, 974, 993, 1047, 1034, 982, 1044, 26, 0, 0, 0, 0, 0, 0, 0, 1013, 1047, 1025, 1029, 956, 1060, 1043, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1060, 1047, 1064, 987, 1042, 1022, 1021, 16, 4, 0, 0, 0, 0, 0, 0, 988, 1025, 1053, 1059, 1012, 1010, 978, 1050, 7, 0, 0, 0, 0, 0, 0, 0, 1004, 1054, 1008, 985, 990, 1008, 1030, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1041, 1045, 1038, 1031, 1043, 1009, 1041, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8170, 8089, 8177, 8298, 8182, 8081, 8299] avg=8192.0000 max=8299 max_rank=7 min=8081 min_rank=6 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1021, 1061, 1037, 1100, 1070, 1019, 1052, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 997, 944, 971, 1013, 971, 956, 1042, 1076, 1, 0, 0, 0, 0, 0, 0, 0, 1038, 1076, 1038, 984, 1016, 1032, 1009, 1011, 26, 0, 0, 0, 0, 0, 0, 0, 1048, 1001, 1046, 1014, 1045, 1016, 969, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1002, 1015, 1030, 1006, 1087, 1051, 1002, 17, 4, 0, 0, 0, 0, 0, 0, 975, 1051, 1038, 980, 1025, 1042, 1004, 1077, 8, 0, 0, 0, 0, 0, 0, 0, 1045, 978, 1032, 1022, 1007, 1083, 1036, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1034, 984, 1038, 1009, 1060, 973, 1017, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8402, 7971, 8230, 8170, 8209, 8200, 8206, 8148] avg=8192.0000 max=8402 max_rank=0 min=7971 min_rank=1 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1086, 1040, 1006, 1010, 1045, 1027, 1043, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1070, 1037, 993, 992, 999, 1073, 1009, 1, 0, 0, 0, 0, 0, 0, 0, 1028, 1015, 1043, 976, 1004, 1027, 1064, 979, 27, 0, 0, 0, 0, 0, 0, 0, 1049, 1065, 1017, 1012, 1028, 991, 1063, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1049, 982, 1031, 1058, 1035, 943, 996, 17, 4, 0, 0, 0, 0, 0, 0, 1022, 1008, 1037, 1028, 1035, 1046, 987, 976, 7, 0, 0, 0, 0, 0, 0, 0, 1034, 1006, 958, 1019, 1016, 1076, 1058, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1015, 1029, 1082, 1017, 978, 1003, 1027, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8232, 8163, 8206, 8138, 8146, 8179, 8169] avg=8192.0000 max=8303 max_rank=0 min=8138 min_rank=4 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[991, 1074, 1092, 975, 999, 1007, 992, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1036, 1002, 1003, 1018, 987, 1026, 1030, 1, 0, 0, 0, 0, 0, 0, 0, 1054, 1043, 1011, 1029, 1007, 999, 996, 1011, 26, 0, 0, 0, 0, 0, 0, 0, 962, 1033, 1037, 979, 1019, 1052, 1007, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 992, 1019, 1064, 1058, 1004, 1008, 1051, 17, 4, 0, 0, 0, 0, 0, 0, 1043, 1008, 1010, 994, 1035, 1009, 1049, 1037, 7, 0, 0, 0, 0, 0, 0, 0, 1029, 1046, 1048, 1003, 1009, 1004, 1049, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1011, 1016, 1010, 1037, 1044, 1081, 1015, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8138, 8176, 8115, 8246, 8192, 8251, 8282] avg=8192.0000 max=8282 max_rank=7 min=8115 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1016, 1004, 1005, 988, 1035, 1004, 1035, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1053, 1059, 1037, 1033, 1043, 1019, 1009, 1, 0, 0, 0, 0, 0, 0, 0, 1029, 1023, 1023, 993, 1006, 1040, 1016, 942, 26, 0, 0, 0, 0, 0, 0, 0, 1065, 1028, 1064, 1051, 994, 1043, 966, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1004, 1085, 1042, 1009, 1044, 993, 1020, 16, 4, 0, 0, 0, 0, 0, 0, 1012, 1001, 1011, 1043, 1025, 1043, 1018, 994, 7, 0, 0, 0, 0, 0, 0, 0, 1074, 1072, 1017, 1054, 974, 1013, 1043, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 991, 1051, 1059, 948, 992, 1001, 1043, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8264, 8098, 8248, 8258, 8154, 8243, 8119] avg=8192.0000 max=8264 max_rank=1 min=8098 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1011, 1029, 978, 1022, 991, 1009, 951, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1039, 987, 1117, 973, 1059, 1046, 1026, 1, 0, 0, 0, 0, 0, 0, 0, 1058, 1055, 994, 1062, 1062, 1023, 1057, 1015, 27, 0, 0, 0, 0, 0, 0, 0, 1004, 1033, 1025, 987, 1027, 1057, 993, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1002, 994, 1012, 1026, 1064, 1024, 976, 17, 4, 0, 0, 0, 0, 0, 0, 1052, 1059, 1013, 1038, 1045, 1011, 1021, 985, 7, 0, 0, 0, 0, 0, 0, 0, 995, 1014, 1035, 997, 1020, 1041, 1094, 993, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1005, 1017, 1044, 1009, 994, 1063, 1031, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8052, 8234, 8353, 8183, 8131, 8231, 8189, 8163] avg=8192.0000 max=8353 max_rank=2 min=8052 min_rank=0 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1043, 1002, 1008, 1005, 989, 991, 989, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 991, 1038, 1049, 1074, 1008, 1044, 1064, 1, 0, 0, 0, 0, 0, 0, 0, 1070, 979, 1056, 996, 1062, 1045, 965, 1025, 27, 0, 0, 0, 0, 0, 0, 0, 993, 1072, 1073, 997, 994, 991, 1039, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1020, 1049, 1025, 1049, 1038, 1003, 1049, 17, 4, 0, 0, 0, 0, 0, 0, 1090, 997, 987, 1015, 1039, 1000, 998, 997, 7, 0, 0, 0, 0, 0, 0, 0, 1044, 995, 1015, 991, 1031, 1025, 1021, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1023, 991, 1055, 1027, 1011, 999, 1031, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8075, 8305, 8225, 8240, 8221, 8130, 8173, 8167] avg=8192.0000 max=8305 max_rank=1 min=8075 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1028, 1022, 1070, 1045, 1007, 984, 1004, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 992, 1009, 1036, 1010, 1061, 1041, 1028, 3, 2, 0, 0, 0, 0, 0, 0, 1000, 998, 1014, 1014, 1038, 1007, 1061, 1039, 31, 0, 0, 0, 0, 0, 0, 0, 1009, 1075, 1013, 987, 1035, 1049, 1007, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1021, 992, 986, 1032, 979, 957, 1002, 21, 6, 0, 0, 0, 0, 0, 0, 1046, 1056, 1036, 1061, 1014, 1029, 1048, 998, 13, 0, 0, 0, 0, 0, 0, 0, 1022, 1021, 1056, 1040, 1025, 988, 994, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 955, 1041, 1030, 1019, 1024, 1045, 1008, 1058, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8222, 8202, 8197, 8035, 8301, 8214, 8189] avg=8192.0000 max=8301 max_rank=5 min=8035 min_rank=4 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8448, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8064, 8320, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8064, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8064, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8448, 8320, 8064, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1048, 998, 1080, 1052, 1013, 1002, 990, 1024, 1030, 1037, 1005, 1021, 1016, 961, 1028, 1110, 989, 1020, 1036, 989, 1064, 980, 1061, 1037, 1027, 992, 1070, 1008, 997, 983, 1015, 1019, 1003, 1014, 967, 1007, 1017, 1069, 1054, 1091, 1053, 1064, 1020, 1041, 1007, 1042, 1040, 1050, 1047, 1044, 1077, 1060, 1031, 1026, 1016, 1095, 977, 977, 995, 1018, 969, 975, 995, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1004, 1044, 1008, 1032, 1056, 1045, 1053, 1008, 1034, 1001, 1025, 1014, 1049, 1014, 1014, 988, 1073, 974, 1013, 1053, 982, 1017, 1010, 1097, 974, 1018, 1018, 993, 1001, 1057, 1017, 1025, 1045, 1004, 1041, 1039, 998, 1059, 1014, 1024, 1048, 984, 1032, 1018, 1035, 1012, 1018, 1026, 1005, 1024, 1009, 1032, 989, 1061, 1035, 1032, 1080, 1082, 989, 1022, 1027, 1028, 1016, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1050, 1035, 1061, 989, 969, 993, 999, 1079, 1042, 1018, 1001, 1025, 1044, 1019, 1084, 1000, 1096, 1026, 1035, 1038, 1077, 1012, 1024, 1035, 985, 1026, 1034, 1023, 995, 1039, 1015, 1041, 987, 1025, 1025, 999, 1042, 1046, 995, 979, 1040, 1007, 991, 979, 1039, 1013, 1069, 984, 1058, 1042, 1014, 1045, 980, 988, 1045, 1028, 1078, 1024, 998, 1050, 996, 1006, 1026, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1020, 1030, 1046, 998, 1029, 1029, 983, 1011, 1020, 1009, 1053, 954, 1038, 1023, 1058, 1001, 1001, 1023, 1063, 1053, 1056, 996, 1032, 1045, 1021, 1028, 1042, 1028, 1054, 1027, 1046, 1010, 1038, 1047, 1030, 1001, 1048, 1019, 1038, 996, 1060, 1010, 1005, 1004, 981, 1004, 1034, 1037, 1013, 1007, 1029, 1018, 1000, 1045, 994, 1000, 1016, 988, 1043, 1048, 999, 1020, 1103, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1006, 989, 1036, 993, 1022, 1028, 1039, 1014, 1063, 1077, 1013, 1023, 1020, 1073, 1053, 1035, 1009, 1040, 1042, 1072, 994, 1065, 1022, 1037, 1063, 1017, 1022, 1019, 1020, 1094, 1006, 980, 995, 1014, 1001, 1075, 1040, 1035, 1013, 961, 1017, 1025, 1023, 1033, 1052, 1030, 992, 977, 998, 996, 1076, 1010, 1031, 1036, 1043, 994, 1005, 1043, 984, 988, 1014, 1030, 997, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[978, 993, 987, 1034, 1053, 1027, 1022, 1055, 1040, 1012, 1024, 1000, 1010, 1018, 1036, 980, 1068, 1010, 1021, 1032, 981, 993, 1040, 1037, 1059, 1040, 1031, 1008, 1019, 1011, 999, 1022, 1057, 1015, 1068, 1039, 1022, 1013, 1029, 1010, 1068, 1036, 1046, 1016, 1019, 1049, 1055, 980, 1044, 990, 1037, 1049, 986, 1050, 995, 1019, 1032, 1029, 997, 1042, 1060, 1034, 979, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1044, 1018, 1025, 1075, 1068, 1022, 1017, 1083, 1010, 1039, 1023, 1010, 1055, 1006, 1028, 1033, 1043, 1048, 1019, 1008, 1021, 987, 1006, 995, 1104, 987, 1012, 1006, 1073, 1012, 1068, 1018, 977, 1025, 1031, 1000, 993, 964, 1000, 1089, 1015, 1000, 1046, 1027, 1024, 1033, 1043, 1003, 1006, 1015, 966, 1004, 1017, 1035, 1008, 1045, 1017, 1026, 1036, 1053, 982, 1033, 1041, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[942, 1027, 1013, 984, 1022, 1007, 1034, 1092, 1076, 1048, 1030, 1052, 1060, 1003, 1021, 1054, 974, 996, 1064, 1041, 1028, 1065, 958, 1000, 979, 1002, 1029, 1032, 1035, 995, 1064, 1002, 1047, 1008, 1009, 1033, 1021, 950, 1009, 1031, 1010, 1021, 1008, 1047, 1034, 1051, 999, 1013, 1069, 986, 1051, 1045, 1028, 1034, 1036, 1021, 1032, 1028, 988, 1048, 1057, 1029, 1050, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1004, 1044, 1008, 1032, 1056, 1045, 1053, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1001, 1025, 1014, 1049, 1014, 988, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 974, 991, 1053, 982, 1017, 1010, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1018, 1018, 993, 1001, 1057, 1017, 1025, 8, 1, 0, 0, 0, 0, 0, 0, 1045, 1004, 1041, 1039, 998, 1059, 1014, 1024, 24, 0, 0, 0, 0, 0, 0, 0, 1048, 984, 1032, 1018, 1035, 1012, 1018, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1024, 1009, 1032, 989, 1053, 1035, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 1082, 989, 1022, 1027, 1028, 1016, 997, 21, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8113, 8197, 8112, 8248, 8173, 8179, 8264] avg=8192.0000 max=8264 max_rank=7 min=8112 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1048, 998, 1080, 1052, 1013, 1002, 990, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1037, 1005, 1021, 1016, 961, 1002, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1020, 1014, 989, 1064, 980, 1061, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 992, 1070, 1008, 997, 983, 1015, 1019, 8, 1, 0, 0, 0, 0, 0, 0, 1003, 1014, 967, 1007, 1017, 1069, 1054, 1091, 24, 0, 0, 0, 0, 0, 0, 0, 1053, 1064, 1020, 1041, 1007, 1042, 1040, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1044, 1077, 1060, 1031, 1018, 1016, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 977, 977, 995, 1018, 969, 975, 995, 993, 21, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8182, 8154, 8120, 8246, 8317, 8388, 7922] avg=8192.0000 max=8388 max_rank=6 min=7922 min_rank=7 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1020, 1030, 1046, 998, 1029, 1029, 983, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1009, 1053, 954, 1038, 1023, 1030, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1023, 1040, 1053, 1056, 996, 1032, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1028, 1042, 1028, 1054, 1027, 1046, 1010, 8, 1, 0, 0, 0, 0, 0, 0, 1038, 1047, 1030, 1001, 1048, 1019, 1038, 996, 25, 0, 0, 0, 0, 0, 0, 0, 1060, 1010, 1005, 1004, 981, 1004, 1034, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1007, 1029, 1018, 1000, 1037, 994, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 988, 1043, 1048, 999, 1020, 1103, 1034, 22, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8128, 8246, 8265, 8242, 8135, 8098, 8276] avg=8192.0000 max=8276 max_rank=7 min=8098 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1044, 1018, 1025, 1075, 1068, 1022, 1017, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1039, 1023, 1010, 1055, 1006, 1002, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1048, 997, 1008, 1021, 987, 1006, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 987, 1012, 1006, 1073, 1012, 1068, 1018, 8, 1, 0, 0, 0, 0, 0, 0, 977, 1025, 1031, 1000, 993, 964, 1000, 1089, 24, 0, 0, 0, 0, 0, 0, 0, 1015, 1000, 1046, 1027, 1024, 1033, 1043, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1015, 966, 1004, 1017, 1027, 1008, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1026, 1036, 1053, 982, 1033, 1041, 1019, 21, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8352, 8178, 8105, 8289, 8103, 8191, 8088, 8230] avg=8192.0000 max=8352 max_rank=0 min=8088 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32612 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[978, 993, 987, 1034, 1053, 1027, 1022, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1012, 1024, 1000, 1010, 1018, 1009, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1010, 999, 1032, 981, 993, 1040, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1040, 1031, 1008, 1019, 1011, 999, 1022, 8, 1, 0, 0, 0, 0, 0, 0, 1057, 1015, 1068, 1039, 1022, 1013, 1029, 1010, 25, 0, 0, 0, 0, 0, 0, 0, 1068, 1036, 1046, 1016, 1019, 1049, 1055, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 990, 1037, 1049, 986, 1042, 995, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1029, 997, 1042, 1060, 1034, 979, 1031, 21, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8093, 8160, 8198, 8278, 8269, 8162, 8227] avg=8192.0000 max=8278 max_rank=4 min=8093 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1050, 1035, 1061, 989, 969, 993, 999, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1018, 1001, 1025, 1044, 1019, 1055, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1096, 1026, 1013, 1038, 1077, 1012, 1024, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1026, 1034, 1023, 995, 1039, 1015, 1041, 7, 1, 0, 0, 0, 0, 0, 0, 987, 1025, 1025, 999, 1042, 1046, 995, 979, 26, 0, 0, 0, 0, 0, 0, 0, 1040, 1007, 991, 979, 1039, 1013, 1069, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1042, 1014, 1045, 980, 981, 1045, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1024, 998, 1050, 996, 1006, 1026, 1029, 21, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8204, 8321, 8166, 8124, 8122, 8193, 8231] avg=8192.0000 max=8321 max_rank=2 min=8122 min_rank=5 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1006, 989, 1036, 993, 1022, 1028, 1039, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1077, 1013, 1023, 1020, 1073, 1025, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1040, 1020, 1072, 994, 1065, 1022, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1017, 1022, 1019, 1020, 1094, 1006, 980, 8, 1, 0, 0, 0, 0, 0, 0, 995, 1014, 1001, 1075, 1040, 1035, 1013, 961, 25, 0, 0, 0, 0, 0, 0, 0, 1017, 1025, 1023, 1033, 1052, 1030, 992, 977, 0, 0, 0, 0, 0, 0, 0, 0, 998, 996, 1076, 1010, 1031, 1028, 1043, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1043, 984, 988, 1014, 1030, 997, 1022, 21, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8329, 8259, 8230, 8159, 8149, 8176, 8107] avg=8192.0000 max=8329 max_rank=1 min=8107 min_rank=7 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[942, 1027, 1013, 984, 1022, 1007, 1034, 1092, 6, 3, 0, 0, 0, 0, 0, 0, 1076, 1048, 1030, 1052, 1060, 1003, 986, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 974, 996, 1032, 1041, 1028, 1065, 958, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1002, 1029, 1032, 1035, 995, 1064, 1002, 12, 6, 0, 0, 0, 0, 0, 0, 1047, 1008, 1009, 1033, 1021, 950, 1009, 1031, 28, 0, 0, 0, 0, 0, 0, 0, 1007, 1021, 1008, 1047, 1034, 1051, 999, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 986, 1051, 1045, 1028, 1016, 1036, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1028, 988, 1048, 1057, 1029, 1050, 1014, 26, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8309, 8094, 8156, 8136, 8180, 8252, 8279] avg=8192.0000 max=8309 max_rank=1 min=8094 min_rank=2 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8448, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[976, 1023, 991, 1038, 1069, 1054, 1030, 1047, 1017, 1018, 1051, 1033, 1008, 1045, 996, 1051, 1034, 1005, 1027, 1012, 981, 1070, 997, 1036, 1029, 1047, 1006, 1044, 1000, 1016, 1019, 1026, 1017, 1009, 1012, 1020, 969, 1074, 1037, 1012, 995, 998, 995, 1047, 1046, 986, 1057, 1041, 1069, 1065, 1041, 1039, 1024, 1032, 1022, 1059, 996, 1009, 1053, 1024, 1027, 974, 1021, 970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1053, 1039, 1000, 1064, 993, 1073, 1009, 965, 1025, 1030, 1023, 1010, 1036, 1032, 1004, 971, 1038, 1005, 1030, 1019, 1059, 1037, 1089, 1012, 1037, 982, 1057, 1029, 1019, 951, 974, 1014, 992, 1013, 1022, 998, 1022, 1056, 991, 1063, 986, 1015, 1013, 1043, 1057, 1067, 1052, 1045, 1024, 1027, 1040, 1038, 1009, 1018, 1005, 1005, 1015, 1005, 1007, 1032, 1026, 1079, 1059, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1018, 1002, 1045, 1044, 1021, 985, 1028, 977, 1042, 1060, 1043, 1091, 1029, 1029, 1046, 1016, 1034, 1050, 994, 987, 996, 1031, 1006, 1007, 1042, 1000, 960, 1029, 1022, 1009, 1008, 1024, 1034, 998, 1067, 999, 1024, 1048, 1020, 1031, 1057, 1048, 1041, 1028, 1005, 1035, 1060, 1035, 1011, 984, 992, 1008, 1001, 1084, 1013, 1011, 1009, 1042, 1010, 1007, 1052, 1039, 1064, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1018, 1040, 1031, 1006, 1043, 1034, 1043, 1061, 1047, 1056, 1003, 1001, 1042, 1019, 1054, 1050, 1041, 1009, 1051, 1021, 1014, 991, 1023, 1004, 1014, 1038, 1059, 1021, 1041, 1013, 995, 1015, 1016, 1033, 1031, 1032, 1027, 1005, 997, 985, 1034, 1060, 1024, 997, 1058, 989, 1011, 1058, 996, 1013, 988, 1037, 1048, 1062, 997, 997, 1000, 964, 1009, 1015, 1062, 1056, 1022, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1082, 1042, 1015, 1042, 1012, 1023, 1048, 1042, 1040, 1029, 979, 1025, 958, 1012, 1042, 1046, 1050, 1032, 995, 1030, 1005, 1036, 1049, 1028, 1041, 978, 980, 1029, 1013, 1057, 1014, 1046, 1033, 1029, 964, 981, 1024, 1040, 976, 1088, 1029, 1031, 1114, 1012, 1045, 1055, 982, 1031, 1047, 1079, 1017, 1058, 1025, 1005, 977, 1041, 985, 976, 1016, 1073, 1023, 949, 1005, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[994, 1034, 1008, 1064, 1026, 1037, 1056, 1064, 1029, 1000, 1054, 1034, 981, 1026, 1033, 1031, 1053, 976, 1103, 985, 1079, 968, 988, 1063, 992, 1088, 1056, 1036, 939, 1056, 980, 998, 1036, 1054, 1063, 1055, 1028, 1014, 927, 999, 1038, 1010, 995, 1046, 1024, 1015, 1045, 1033, 982, 1014, 990, 930, 1018, 1086, 1037, 1053, 1020, 1010, 1033, 1058, 1027, 1030, 1004, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1005, 1007, 1022, 1076, 1004, 1060, 1042, 1053, 1014, 984, 990, 993, 1080, 1006, 1073, 1061, 1022, 1072, 1014, 979, 1047, 1026, 1055, 1034, 990, 993, 1043, 1002, 997, 966, 1017, 990, 1041, 1065, 1038, 1040, 1025, 1051, 985, 1055, 973, 997, 1010, 1000, 1045, 1042, 1009, 1052, 970, 1063, 1022, 1034, 1028, 972, 997, 1074, 1044, 1042, 1022, 1055, 1000, 1023, 1037, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1040, 979, 1030, 1037, 1007, 1012, 1021, 1028, 1015, 1011, 1028, 1023, 1064, 1013, 987, 1059, 1007, 1043, 1016, 993, 1022, 1073, 1063, 1030, 1008, 1019, 1019, 1023, 995, 981, 1065, 1072, 1034, 1016, 1058, 1036, 1023, 992, 981, 1055, 994, 1008, 994, 1005, 1015, 1046, 1075, 1003, 1021, 1017, 1026, 1022, 1032, 1023, 1084, 1020, 1006, 1026, 1050, 1012, 993, 1060, 996, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[976, 1023, 991, 1002, 1069, 1054, 1030, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1018, 1051, 1033, 1008, 1045, 996, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1005, 1027, 1012, 981, 1070, 997, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1047, 1006, 1044, 1000, 1016, 1019, 1026, 36, 27, 0, 0, 0, 0, 0, 0, 1017, 1009, 1012, 1020, 969, 1074, 1037, 1012, 9, 0, 0, 0, 0, 0, 0, 0, 995, 998, 995, 1047, 1046, 986, 1057, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1065, 1041, 1039, 1024, 1032, 1022, 1059, 1, 0, 0, 0, 0, 0, 0, 0, 996, 1009, 1053, 1024, 1027, 974, 1021, 970, 20, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8199, 8150, 8250, 8159, 8138, 8352, 8096] avg=8192.0000 max=8352 max_rank=6 min=8096 min_rank=7 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1053, 1039, 1000, 1027, 993, 1073, 1009, 965, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1030, 1023, 1010, 1036, 1032, 1004, 953, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1005, 1030, 1019, 1059, 1037, 1089, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 982, 1057, 1029, 1019, 951, 974, 1014, 37, 27, 0, 0, 0, 0, 0, 0, 992, 1013, 1022, 998, 1022, 1056, 991, 1063, 9, 0, 0, 0, 0, 0, 0, 0, 986, 1015, 1013, 1043, 1057, 1067, 1052, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1027, 1040, 1038, 1009, 1018, 1005, 1005, 1, 0, 0, 0, 0, 0, 0, 0, 1015, 1005, 1007, 1032, 1026, 1079, 1059, 1033, 18, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8113, 8277, 8127, 8166, 8251, 8167, 8276] avg=8192.0000 max=8277 max_rank=2 min=8113 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[994, 1034, 1008, 1027, 1026, 1037, 1056, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1000, 1054, 1034, 981, 1026, 1033, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 976, 1103, 985, 1079, 968, 988, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1088, 1056, 1036, 939, 1056, 980, 998, 37, 27, 0, 0, 0, 0, 0, 0, 1036, 1054, 1063, 1055, 1028, 1014, 927, 999, 9, 0, 0, 0, 0, 0, 0, 0, 1038, 1010, 995, 1046, 1024, 1015, 1045, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1014, 990, 930, 1018, 1086, 1037, 1053, 1, 0, 0, 0, 0, 0, 0, 0, 1020, 1010, 1033, 1058, 1027, 1030, 1004, 1031, 20, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8168, 8203, 8209, 8185, 8179, 8111, 8235] avg=8192.0000 max=8246 max_rank=0 min=8111 min_rank=6 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1082, 1042, 1015, 1006, 1012, 1023, 1048, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1029, 979, 1025, 958, 1012, 1042, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1032, 995, 1030, 1005, 1036, 1049, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 978, 980, 1029, 1013, 1057, 1014, 1046, 36, 26, 0, 0, 0, 0, 0, 0, 1033, 1029, 964, 981, 1024, 1040, 976, 1088, 9, 0, 0, 0, 0, 0, 0, 0, 1029, 1031, 1114, 1012, 1045, 1055, 982, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1079, 1017, 1058, 1025, 1005, 977, 1041, 1, 0, 0, 0, 0, 0, 0, 0, 985, 976, 1016, 1073, 1023, 949, 1005, 1006, 20, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8111, 8213, 8220, 8144, 8273, 8250, 8055] avg=8192.0000 max=8273 max_rank=5 min=8055 min_rank=7 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1018, 1002, 1045, 1008, 1021, 985, 1028, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1060, 1043, 1091, 1029, 1029, 1046, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1050, 994, 987, 996, 1031, 1006, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1000, 960, 1029, 1022, 1009, 1008, 1024, 36, 27, 0, 0, 0, 0, 0, 0, 1034, 998, 1067, 999, 1024, 1048, 1020, 1031, 9, 0, 0, 0, 0, 0, 0, 0, 1057, 1048, 1041, 1028, 1005, 1035, 1060, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 984, 992, 1008, 1001, 1084, 1013, 1011, 1, 0, 0, 0, 0, 0, 0, 0, 1009, 1042, 1010, 1007, 1052, 1039, 1064, 1004, 19, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8084, 8337, 8093, 8157, 8230, 8282, 8105, 8248] avg=8192.0000 max=8337 max_rank=1 min=8084 min_rank=0 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1005, 1007, 1022, 1038, 1004, 1060, 1042, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 984, 990, 993, 1080, 1006, 1073, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1072, 1014, 979, 1047, 1026, 1055, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 990, 993, 1043, 1002, 997, 966, 1017, 990, 38, 27, 0, 0, 0, 0, 0, 0, 1041, 1065, 1038, 1040, 1025, 1051, 985, 1055, 9, 0, 0, 0, 0, 0, 0, 0, 973, 997, 1010, 1000, 1045, 1042, 1009, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1063, 1022, 1034, 1028, 972, 997, 1074, 1, 0, 0, 0, 0, 0, 0, 0, 1044, 1042, 1022, 1055, 1000, 1023, 1037, 1008, 20, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8181, 8237, 8063, 8309, 8101, 8161, 8253] avg=8192.0000 max=8309 max_rank=4 min=8063 min_rank=3 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1018, 1040, 1031, 971, 1043, 1034, 1043, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1056, 1003, 1001, 1042, 1019, 1054, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1009, 1051, 1021, 1014, 991, 1023, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1038, 1059, 1021, 1041, 1013, 995, 1015, 35, 27, 0, 0, 0, 0, 0, 0, 1016, 1033, 1031, 1032, 1027, 1005, 997, 985, 9, 0, 0, 0, 0, 0, 0, 0, 1034, 1060, 1024, 997, 1058, 989, 1011, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1013, 988, 1037, 1048, 1062, 997, 997, 1, 0, 0, 0, 0, 0, 0, 0, 1000, 964, 1009, 1015, 1062, 1056, 1022, 1015, 20, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8252, 8142, 8258, 8135, 8204, 8139, 8165] avg=8192.0000 max=8258 max_rank=3 min=8135 min_rank=4 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1040, 979, 1030, 996, 1007, 1012, 1021, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1011, 1028, 1023, 1064, 1013, 987, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1043, 1016, 993, 1022, 1073, 1063, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1019, 1019, 1023, 995, 981, 1065, 1072, 41, 29, 0, 0, 0, 0, 0, 0, 1034, 1016, 1058, 1036, 1023, 992, 981, 1055, 13, 0, 0, 0, 0, 0, 0, 0, 994, 1008, 994, 1005, 1015, 1046, 1075, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1017, 1026, 1022, 1032, 1023, 1084, 1020, 6, 0, 0, 0, 0, 0, 0, 0, 1006, 1026, 1050, 1012, 993, 1060, 996, 1030, 25, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8175, 8221, 8252, 8208, 8108, 8251, 8208] avg=8192.0000 max=8252 max_rank=3 min=8108 min_rank=5 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8064, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 7936, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8064, 8064, 8192, 8192, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8320, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66688 slack_slots=3584 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1101, 1004, 1044, 1001, 1035, 1048, 1031, 1026, 1004, 1007, 1031, 995, 1059, 1046, 1044, 1043, 1006, 1037, 1000, 987, 1001, 1030, 967, 1052, 1006, 1028, 1057, 1039, 1002, 1019, 1027, 1016, 994, 1016, 1009, 1027, 1048, 1019, 1071, 1054, 996, 1025, 1045, 1032, 995, 972, 1058, 989, 979, 1078, 1061, 1016, 1033, 1062, 1014, 1047, 1000, 1063, 985, 1027, 1024, 1038, 982, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1043, 1050, 1032, 1073, 1035, 1041, 977, 1057, 1016, 1050, 1019, 1056, 1028, 961, 1052, 1035, 1054, 1040, 1024, 999, 1017, 1016, 996, 1022, 969, 953, 1023, 1020, 1088, 995, 1043, 1009, 987, 1001, 1018, 1085, 1021, 1024, 1019, 1065, 1024, 1001, 992, 1033, 1055, 1003, 1029, 1028, 1037, 1039, 1056, 1058, 1012, 975, 1025, 1040, 1013, 1042, 1013, 1011, 956, 1014, 1058, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1050, 1046, 1039, 1036, 1041, 1043, 989, 1023, 1016, 1018, 1019, 1001, 1057, 1042, 1030, 996, 966, 984, 990, 1059, 974, 1046, 1025, 1024, 1039, 1048, 1032, 1034, 1007, 1006, 1032, 1020, 1011, 1015, 1030, 1068, 1072, 1042, 1053, 987, 1050, 1063, 997, 1007, 994, 1032, 1011, 986, 994, 969, 1066, 1019, 1017, 1021, 1018, 974, 1058, 1072, 1003, 1040, 1010, 1071, 1048, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1061, 979, 1072, 1022, 993, 1063, 1002, 1058, 1041, 1000, 1023, 968, 1000, 976, 1047, 1051, 1042, 1040, 1032, 1025, 1072, 1022, 1050, 1010, 1016, 984, 1039, 1027, 1009, 1023, 971, 990, 1025, 999, 1008, 984, 1048, 1021, 1061, 1035, 1021, 1000, 1008, 1013, 1023, 1044, 1000, 980, 1052, 1012, 984, 1037, 1050, 1040, 1033, 1070, 1038, 1050, 1079, 1000, 1074, 1036, 971, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1052, 1050, 1039, 1003, 1016, 1046, 1014, 987, 1028, 1007, 1010, 1063, 937, 1016, 1074, 1041, 1051, 1015, 1014, 1004, 1011, 1017, 1012, 1053, 1080, 1000, 1016, 1035, 1047, 1015, 989, 977, 996, 992, 1021, 1021, 1005, 996, 1010, 1014, 1029, 1060, 1030, 1033, 1028, 993, 1037, 996, 1067, 1003, 1025, 1068, 1016, 1043, 1068, 1000, 1076, 1036, 1035, 1007, 1036, 1043, 1030, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1039, 1052, 1027, 1036, 1010, 988, 1050, 1028, 1029, 1052, 1054, 1022, 1009, 984, 1005, 1022, 1030, 1017, 1015, 990, 1033, 1027, 1001, 1078, 1021, 1017, 1002, 1044, 1021, 1029, 1046, 1034, 1044, 979, 1038, 1023, 1013, 1008, 1004, 1003, 1049, 1024, 1060, 1012, 1029, 976, 999, 975, 1055, 989, 1044, 1015, 1012, 1082, 1028, 1042, 995, 1056, 1031, 994, 1051, 1030, 1018, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1024, 1014, 1019, 1079, 1016, 1047, 1007, 1005, 1027, 1010, 985, 1081, 1067, 1046, 1040, 1020, 1016, 985, 1027, 1027, 973, 1031, 1017, 995, 1055, 1050, 1069, 1053, 1083, 995, 1034, 1035, 1029, 1048, 1034, 1000, 1025, 1000, 1025, 1052, 973, 1018, 1056, 1060, 1024, 1012, 997, 1034, 1014, 994, 1042, 1008, 1013, 1001, 1014, 1016, 1016, 1029, 1026, 964, 1032, 1023, 1002, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[982, 1006, 1007, 1049, 1043, 1010, 1019, 974, 1005, 1052, 1009, 1027, 1037, 1021, 1046, 1005, 1049, 993, 1057, 995, 1027, 1044, 1049, 992, 1022, 1047, 1047, 1057, 1063, 1025, 1078, 988, 1032, 1003, 1017, 957, 1058, 975, 1003, 1018, 1025, 990, 1026, 1008, 1067, 1003, 997, 1027, 1080, 1011, 1029, 1032, 1019, 965, 1025, 1015, 1080, 1072, 1015, 1082, 1020, 1015, 1015, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1004, 1050, 1032, 1073, 1035, 1041, 977, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1050, 1019, 1056, 1028, 961, 1049, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1040, 1024, 999, 1017, 1016, 996, 1022, 18, 16, 0, 0, 0, 0, 0, 0, 969, 953, 1023, 1020, 1074, 995, 1043, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1001, 1018, 1085, 1021, 1024, 1019, 1065, 14, 5, 3, 0, 0, 0, 0, 0, 1024, 1001, 992, 1033, 1055, 1003, 1029, 1028, 39, 8, 0, 0, 0, 0, 0, 0, 1037, 1039, 1033, 1058, 1012, 975, 1025, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1018, 1013, 1011, 956, 1014, 1058, 1009, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8214, 8202, 8086, 8242, 8212, 8219, 8092] avg=8192.0000 max=8269 max_rank=0 min=8086 min_rank=3 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1060, 1004, 1044, 1001, 1035, 1048, 1031, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1007, 1031, 995, 1059, 1046, 1041, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1037, 1000, 987, 1001, 1030, 967, 1052, 18, 17, 0, 0, 0, 0, 0, 0, 1006, 1028, 1057, 1039, 989, 1019, 1027, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1016, 1009, 1027, 1048, 1019, 1071, 1054, 13, 5, 3, 0, 0, 0, 0, 0, 996, 1025, 1045, 1032, 995, 972, 1058, 989, 41, 8, 0, 0, 0, 0, 0, 0, 979, 1078, 1038, 1016, 1033, 1062, 1014, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1038, 985, 1027, 1024, 1038, 982, 984, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8226, 8115, 8181, 8259, 8161, 8267, 8078] avg=8192.0000 max=8267 max_rank=6 min=8078 min_rank=7 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1013, 1050, 1039, 1003, 1016, 1046, 1014, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1007, 1010, 1063, 937, 1016, 1071, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1015, 1014, 1004, 1011, 1017, 1012, 1053, 18, 16, 0, 0, 0, 0, 0, 0, 1080, 1000, 1016, 1035, 1034, 1015, 989, 977, 0, 0, 0, 0, 0, 0, 0, 0, 996, 992, 1021, 1021, 1005, 996, 1010, 1014, 13, 5, 3, 0, 0, 0, 0, 0, 1029, 1060, 1030, 1033, 1028, 993, 1037, 996, 39, 8, 0, 0, 0, 0, 0, 0, 1067, 1003, 1002, 1068, 1016, 1043, 1068, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1012, 1035, 1007, 1036, 1043, 1030, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8173, 8211, 8146, 8076, 8253, 8267, 8242] avg=8192.0000 max=8267 max_rank=6 min=8076 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1011, 1046, 1039, 1036, 1041, 1043, 989, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1018, 1019, 1001, 1057, 1042, 1027, 996, 0, 0, 0, 0, 0, 0, 0, 0, 966, 984, 990, 1059, 974, 1046, 1025, 1024, 18, 17, 0, 0, 0, 0, 0, 0, 1039, 1048, 1032, 1034, 994, 1006, 1032, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1015, 1030, 1068, 1072, 1042, 1053, 987, 13, 5, 3, 0, 0, 0, 0, 0, 1050, 1063, 997, 1007, 994, 1032, 1011, 986, 39, 8, 0, 0, 0, 0, 0, 0, 994, 969, 1043, 1019, 1017, 1021, 1018, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1047, 1003, 1040, 1010, 1071, 1048, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8176, 8103, 8205, 8299, 8187, 8055, 8283] avg=8192.0000 max=8299 max_rank=4 min=8055 min_rank=6 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1021, 979, 1072, 1022, 993, 1063, 1002, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1000, 1023, 968, 1000, 976, 1044, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1040, 1032, 1025, 1072, 1022, 1050, 1010, 17, 16, 0, 0, 0, 0, 0, 0, 1016, 984, 1039, 1027, 996, 1023, 971, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 999, 1008, 984, 1048, 1021, 1061, 1035, 13, 4, 3, 0, 0, 0, 0, 0, 1021, 1000, 1008, 1013, 1023, 1044, 1000, 980, 40, 8, 0, 0, 0, 0, 0, 0, 1052, 1012, 963, 1037, 1050, 1040, 1033, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1026, 1079, 1000, 1074, 1036, 971, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8103, 8326, 8046, 8201, 8137, 8257, 8256] avg=8192.0000 max=8326 max_rank=2 min=8046 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1000, 1052, 1027, 1036, 1010, 988, 1050, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1052, 1054, 1022, 1009, 984, 1002, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1017, 1015, 990, 1033, 1027, 1001, 1078, 18, 17, 0, 0, 0, 0, 0, 0, 1021, 1017, 1002, 1044, 1008, 1029, 1046, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 979, 1038, 1023, 1013, 1008, 1004, 1003, 13, 5, 3, 0, 0, 0, 0, 0, 1049, 1024, 1060, 1012, 1029, 976, 999, 975, 39, 8, 0, 0, 0, 0, 0, 0, 1055, 989, 1021, 1015, 1012, 1082, 1028, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1031, 1031, 994, 1051, 1030, 1018, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8174, 8226, 8201, 8133, 8171, 8244, 8196] avg=8192.0000 max=8244 max_rank=6 min=8133 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[986, 1014, 1019, 1079, 1016, 1047, 1007, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1010, 985, 1081, 1067, 1046, 1037, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 985, 1027, 1027, 973, 1031, 1017, 995, 18, 16, 0, 0, 0, 0, 0, 0, 1055, 1050, 1069, 1053, 1069, 995, 1034, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1048, 1034, 1000, 1025, 1000, 1025, 1052, 14, 5, 3, 0, 0, 0, 0, 0, 973, 1018, 1056, 1060, 1024, 1012, 997, 1034, 38, 8, 0, 0, 0, 0, 0, 0, 1014, 994, 1019, 1008, 1013, 1001, 1014, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1005, 1026, 964, 1032, 1023, 1002, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8273, 8105, 8360, 8235, 8220, 8079, 8091] avg=8192.0000 max=8360 max_rank=3 min=8079 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32625 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[940, 1006, 1007, 1049, 1043, 1010, 1019, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1052, 1009, 1027, 1037, 1021, 1041, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 993, 1057, 995, 1027, 1044, 1049, 992, 21, 21, 0, 0, 0, 0, 0, 0, 1022, 1047, 1047, 1057, 1047, 1025, 1078, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1003, 1017, 957, 1058, 975, 1003, 1018, 16, 7, 5, 0, 0, 0, 0, 0, 1025, 990, 1026, 1008, 1067, 1003, 997, 1027, 42, 10, 0, 0, 0, 0, 0, 0, 1080, 1011, 1001, 1032, 1019, 965, 1025, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 1041, 1015, 1082, 1020, 1015, 1015, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8048, 8197, 8248, 8311, 8091, 8195, 8148, 8298] avg=8192.0000 max=8311 max_rank=3 min=8048 min_rank=0 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8192, 8192, 8320, 8192, 8320, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8320, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8064, 8192, 8064, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1040, 1013, 1022, 1077, 993, 1038, 1031, 956, 1015, 1101, 988, 1061, 1003, 996, 1046, 1059, 1058, 1019, 1017, 1019, 1020, 1013, 1037, 1060, 1043, 1003, 1001, 1020, 1033, 1058, 1001, 1035, 1020, 1010, 951, 1023, 1057, 1017, 1046, 1045, 1033, 1040, 985, 1026, 1097, 996, 1051, 1001, 1036, 1040, 1025, 1019, 1027, 997, 1043, 1061, 997, 978, 1027, 1008, 992, 979, 1039, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1017, 1036, 1051, 1045, 1006, 1011, 973, 1060, 1028, 1012, 994, 1032, 984, 993, 961, 1027, 1063, 1042, 991, 1014, 1023, 1035, 1018, 1016, 982, 1030, 1016, 1023, 997, 1022, 1018, 1038, 1013, 1053, 1061, 1008, 1026, 1062, 979, 1014, 1022, 1010, 1003, 1013, 1027, 1015, 1020, 1071, 1029, 999, 1036, 1055, 1049, 1032, 1064, 1009, 1043, 1016, 1106, 1009, 1092, 1019, 1032, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1013, 1010, 1015, 983, 1029, 1036, 1004, 1012, 1023, 1027, 1014, 1013, 1001, 1069, 1000, 1058, 1027, 968, 1049, 1015, 1036, 991, 984, 1021, 1001, 1024, 1118, 982, 1026, 1000, 1036, 959, 1008, 1046, 1006, 1000, 1053, 1014, 996, 1014, 1053, 1018, 1046, 1040, 1009, 1058, 1038, 989, 1104, 1049, 1032, 1033, 1054, 994, 1004, 1054, 1005, 1079, 1020, 1030, 1003, 1034, 1061, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1066, 983, 1035, 995, 1045, 996, 1010, 1021, 1042, 1067, 989, 1051, 1030, 1032, 995, 1049, 990, 1057, 1024, 1038, 1011, 1053, 1031, 1055, 1028, 1027, 1057, 1018, 1003, 1045, 1061, 973, 1012, 1038, 984, 1061, 1050, 1006, 984, 1001, 1034, 987, 996, 1023, 1045, 1006, 1038, 1016, 1066, 1004, 996, 1036, 1026, 959, 1034, 1032, 1064, 979, 1059, 1005, 1021, 998, 1057, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1055, 1014, 1020, 1071, 989, 1041, 1008, 1018, 1062, 1025, 1035, 1040, 967, 1044, 1054, 1021, 1023, 1041, 1064, 1047, 978, 1037, 994, 1048, 1004, 998, 1041, 1015, 1040, 1015, 1047, 1023, 993, 1025, 1020, 1044, 1050, 969, 1006, 1022, 1045, 1034, 993, 1042, 1018, 1023, 985, 1013, 1072, 1042, 1019, 1032, 1030, 1000, 1035, 992, 1028, 1032, 1046, 993, 1011, 1024, 1041, 978] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[989, 1021, 1028, 989, 1001, 1024, 1008, 976, 1008, 1008, 1003, 1032, 991, 1085, 1043, 1048, 1038, 1054, 1030, 1041, 992, 1017, 1015, 1092, 1024, 1020, 1014, 1052, 1000, 1026, 1031, 1049, 970, 1080, 991, 971, 1031, 986, 1016, 1046, 1009, 1025, 977, 1018, 1023, 1003, 1059, 1079, 1034, 990, 1036, 1018, 1062, 1031, 1044, 1019, 1013, 1077, 1029, 1020, 1062, 1033, 1001, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1032, 1020, 1012, 1079, 1025, 1013, 988, 972, 983, 1052, 994, 1095, 1004, 1004, 1023, 1014, 1033, 999, 1028, 1022, 1103, 1024, 1011, 989, 1026, 1015, 1033, 1000, 1040, 1002, 997, 1033, 961, 1037, 1042, 1066, 1006, 1044, 1007, 1089, 1005, 1023, 985, 1012, 1031, 1019, 1026, 1054, 1073, 1023, 1010, 983, 997, 984, 1063, 1032, 1110, 1031, 1043, 1013, 1050, 993, 989, 1070] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1014, 1068, 1033, 993, 1042, 992, 1052, 1045, 1020, 1023, 1068, 1045, 1062, 1016, 1045, 1001, 1037, 1003, 1033, 1044, 954, 1021, 1011, 1086, 1018, 1027, 1022, 995, 1040, 991, 1000, 1051, 1009, 1000, 1034, 1024, 1000, 1034, 983, 975, 970, 1025, 973, 992, 1044, 982, 1067, 995, 1012, 1024, 1008, 1053, 1038, 1018, 1060, 1078, 1029, 1041, 1042, 1024, 1014, 1049, 1009, 1078] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1017, 1036, 1051, 1045, 1006, 1011, 973, 1060, 33, 3, 0, 0, 0, 0, 0, 0, 1028, 1012, 994, 1015, 984, 993, 961, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1042, 991, 1014, 1023, 1035, 1018, 997, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1030, 1016, 1023, 997, 1022, 1018, 1038, 17, 3, 0, 0, 0, 0, 0, 0, 1013, 1053, 1061, 1008, 1026, 1062, 979, 1014, 42, 0, 0, 0, 0, 0, 0, 0, 1022, 1010, 1003, 1013, 1027, 1015, 1020, 1071, 16, 5, 0, 0, 0, 0, 0, 0, 984, 999, 1036, 1055, 1049, 1032, 1064, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1016, 1068, 1009, 1092, 1019, 1032, 991, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8014, 8183, 8146, 8258, 8202, 8228, 8270] avg=8192.0000 max=8270 max_rank=7 min=8014 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32958 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1040, 1013, 1022, 1077, 993, 1038, 1031, 956, 31, 3, 0, 0, 0, 0, 0, 0, 1015, 1101, 988, 1044, 1003, 996, 1046, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1019, 1017, 1019, 1020, 1013, 1037, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1003, 1001, 1020, 1033, 1058, 1001, 1035, 17, 3, 0, 0, 0, 0, 0, 0, 1020, 1010, 951, 1023, 1057, 1017, 1046, 1045, 42, 0, 0, 0, 0, 0, 0, 0, 1033, 1040, 985, 1026, 1097, 996, 1051, 1001, 17, 5, 0, 0, 0, 0, 0, 0, 991, 1040, 1025, 1019, 1027, 997, 1043, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 997, 978, 991, 1008, 992, 979, 1039, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8252, 8223, 8214, 8211, 8251, 8203, 7978] avg=8192.0000 max=8252 max_rank=1 min=7978 min_rank=7 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1066, 983, 1035, 995, 1045, 996, 1010, 1021, 32, 3, 0, 0, 0, 0, 0, 0, 1042, 1067, 989, 1034, 1030, 1032, 995, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1057, 1024, 1038, 1011, 1053, 1031, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1027, 1057, 1018, 1003, 1045, 1061, 973, 17, 3, 0, 0, 0, 0, 0, 0, 1012, 1038, 984, 1061, 1050, 1006, 984, 1001, 43, 0, 0, 0, 0, 0, 0, 0, 1034, 987, 996, 1023, 1045, 1006, 1038, 1016, 17, 5, 0, 0, 0, 0, 0, 0, 1020, 1004, 996, 1036, 1026, 959, 1034, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 979, 1022, 1005, 1021, 998, 1057, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8238, 8239, 8232, 8179, 8167, 8107, 8188] avg=8192.0000 max=8239 max_rank=2 min=8107 min_rank=6 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1013, 1010, 1015, 983, 1029, 1036, 1004, 1012, 31, 3, 0, 0, 0, 0, 0, 0, 1023, 1027, 1014, 997, 1001, 1069, 1000, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 968, 1049, 1015, 1036, 991, 984, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1024, 1118, 982, 1026, 1000, 1036, 959, 16, 3, 0, 0, 0, 0, 0, 0, 1008, 1046, 1006, 1000, 1053, 1014, 996, 1014, 45, 0, 0, 0, 0, 0, 0, 0, 1053, 1018, 1046, 1040, 1009, 1058, 1038, 989, 16, 5, 0, 0, 0, 0, 0, 0, 1056, 1049, 1032, 1033, 1054, 994, 1004, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1079, 984, 1030, 1003, 1034, 1061, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8189, 8072, 8165, 8182, 8272, 8276, 8244] avg=8192.0000 max=8276 max_rank=6 min=8072 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32974 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1055, 1014, 1020, 1071, 989, 1041, 1008, 1018, 31, 3, 0, 0, 0, 0, 0, 0, 1062, 1025, 1035, 1023, 967, 1044, 1054, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1041, 1064, 1047, 978, 1037, 994, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 998, 1041, 1015, 1040, 1015, 1047, 1023, 17, 3, 0, 0, 0, 0, 0, 0, 993, 1025, 1020, 1044, 1050, 969, 1006, 1022, 44, 0, 0, 0, 0, 0, 0, 0, 1045, 1034, 993, 1042, 1018, 1023, 985, 1013, 17, 5, 0, 0, 0, 0, 0, 0, 1025, 1042, 1019, 1032, 1030, 1000, 1035, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1032, 1010, 993, 1011, 1024, 1041, 978, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8231, 8212, 8203, 8173, 8175, 8175, 8117] avg=8192.0000 max=8250 max_rank=0 min=8117 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[989, 1021, 1028, 989, 1001, 1024, 1008, 976, 31, 3, 0, 0, 0, 0, 0, 0, 1008, 1008, 1003, 1015, 991, 1085, 1043, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1054, 1030, 1041, 992, 1017, 1015, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1020, 1014, 1052, 1000, 1026, 1031, 1049, 17, 3, 0, 0, 0, 0, 0, 0, 970, 1080, 991, 971, 1031, 986, 1016, 1046, 42, 0, 0, 0, 0, 0, 0, 0, 1009, 1025, 977, 1018, 1023, 1003, 1059, 1079, 18, 5, 0, 0, 0, 0, 0, 0, 989, 990, 1036, 1018, 1062, 1031, 1044, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1077, 993, 1020, 1062, 1033, 1001, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8070, 8201, 8258, 8236, 8133, 8216, 8189, 8233] avg=8192.0000 max=8258 max_rank=2 min=8070 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1032, 1020, 1012, 1079, 1025, 1013, 988, 972, 35, 6, 0, 0, 0, 0, 0, 0, 983, 1052, 994, 1074, 1004, 1004, 1023, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 999, 1028, 1022, 1103, 1024, 1011, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1015, 1033, 1000, 1040, 1002, 997, 1033, 21, 9, 0, 0, 0, 0, 0, 0, 961, 1037, 1042, 1066, 1006, 1044, 1007, 1089, 48, 0, 0, 0, 0, 0, 0, 0, 1005, 1023, 985, 1012, 1031, 1019, 1026, 1054, 20, 8, 0, 0, 0, 0, 0, 0, 1019, 1023, 1010, 983, 997, 984, 1063, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1110, 1031, 1000, 1013, 1050, 993, 989, 1070, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8148, 8180, 8176, 8300, 8183, 8111, 8256] avg=8192.0000 max=8300 max_rank=4 min=8111 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1014, 1068, 1033, 993, 1042, 992, 1052, 1045, 31, 3, 0, 0, 0, 0, 0, 0, 1020, 1023, 1068, 1028, 1062, 1016, 1045, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1003, 1033, 1044, 954, 1021, 1011, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1027, 1022, 995, 1040, 991, 1000, 1051, 17, 3, 0, 0, 0, 0, 0, 0, 1009, 1000, 1034, 1024, 1000, 1034, 983, 975, 41, 0, 0, 0, 0, 0, 0, 0, 970, 1025, 973, 992, 1044, 982, 1067, 995, 17, 5, 0, 0, 0, 0, 0, 0, 968, 1024, 1008, 1053, 1038, 1018, 1060, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1041, 1006, 1024, 1014, 1049, 1009, 1078, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8263, 8169, 8164, 8100, 8070, 8247, 8250] avg=8192.0000 max=8273 max_rank=0 min=8070 min_rank=5 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8064, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8320, 8320, 8064, 8448, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8064, 8192, 8320, 8192, 8320, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8192, 8064, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 8064, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1015, 1001, 1015, 1054, 1052, 1036, 1024, 1042, 1054, 1012, 1015, 1008, 1008, 1059, 1066, 989, 1044, 985, 1031, 988, 1062, 996, 989, 1043, 1024, 1035, 1030, 1035, 1001, 1033, 1049, 1042, 1021, 1038, 1028, 1063, 1052, 1048, 998, 1023, 972, 1041, 1025, 992, 1013, 1028, 1035, 1020, 1001, 1021, 1059, 1028, 1013, 1030, 1013, 1028, 1055, 1017, 1004, 987, 1007, 1006, 1033, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[996, 1030, 1022, 1032, 1026, 1039, 1035, 1042, 1010, 1024, 1068, 997, 1001, 1011, 1014, 1040, 1020, 978, 1013, 997, 1060, 1025, 1076, 1000, 992, 1009, 1008, 964, 1006, 993, 1047, 1001, 1057, 995, 1019, 1008, 962, 1057, 1044, 1037, 987, 1055, 1026, 979, 1062, 1056, 1073, 1036, 1089, 1018, 1012, 984, 1069, 1069, 1007, 1029, 1015, 1029, 1036, 1006, 1032, 1028, 1026, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1007, 972, 1035, 1047, 1015, 985, 992, 1088, 1003, 971, 1017, 1055, 1019, 1029, 966, 1039, 1006, 1026, 1119, 1031, 1008, 1071, 973, 1014, 988, 980, 1038, 1017, 1009, 1014, 1066, 1078, 1043, 1040, 1039, 1046, 1008, 1034, 1058, 1078, 1062, 1007, 1043, 989, 1000, 989, 1032, 1014, 997, 1043, 934, 1023, 1013, 1034, 1024, 998, 1023, 1043, 1036, 1065, 1055, 1076, 1036, 976] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1066, 1024, 1036, 990, 991, 1024, 1006, 1037, 984, 1036, 1007, 986, 979, 1067, 1093, 1037, 1058, 1026, 1072, 1024, 1033, 1023, 1020, 990, 1017, 1039, 995, 1046, 1049, 1050, 926, 1003, 1058, 1051, 1011, 1079, 1031, 994, 1008, 1003, 1034, 1044, 1037, 1002, 1060, 1049, 1013, 1026, 1019, 1031, 1079, 993, 1042, 1020, 1011, 1006, 1013, 945, 1001, 1061, 1081, 1031, 1018, 951] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[988, 996, 1029, 1058, 1014, 1002, 1039, 1086, 1012, 1020, 1011, 1008, 1091, 1051, 992, 1034, 1022, 1101, 974, 1064, 981, 1027, 968, 988, 1055, 1039, 997, 1015, 1064, 1027, 1023, 1005, 1051, 983, 1055, 997, 1006, 1031, 1004, 1016, 1002, 1030, 1034, 1032, 1032, 1021, 1030, 1026, 1023, 1023, 1108, 1022, 1025, 990, 1018, 1002, 1010, 1016, 1014, 1008, 1049, 1023, 1057, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[981, 1023, 1032, 1009, 990, 1045, 1037, 1006, 1023, 1052, 1070, 1045, 982, 1054, 1054, 1003, 1019, 1054, 1045, 1003, 988, 1033, 1046, 1046, 1023, 1000, 1002, 1021, 1028, 1033, 979, 960, 1035, 1007, 1021, 1056, 1033, 1010, 1022, 1052, 1084, 1041, 1043, 1042, 1031, 1007, 1013, 1015, 997, 995, 1055, 994, 1023, 1043, 972, 1062, 1038, 1004, 1042, 1010, 1024, 1005, 1037, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1028, 1026, 1017, 1041, 1029, 1029, 1092, 1039, 1087, 1032, 989, 976, 1033, 1063, 1050, 1054, 1007, 989, 1009, 1034, 995, 1004, 1022, 1002, 981, 1017, 1018, 997, 1026, 1001, 987, 1045, 1000, 1016, 1024, 1033, 1015, 1047, 1012, 1012, 1047, 1014, 1020, 1042, 1010, 1057, 1039, 1064, 1014, 996, 1040, 1003, 1064, 1041, 1023, 988, 995, 1045, 1035, 1065, 994, 987, 1042, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[935, 1010, 1082, 1038, 1053, 1055, 1062, 1003, 1053, 1027, 1003, 1041, 1035, 1029, 1029, 964, 1015, 955, 1021, 1079, 1040, 1019, 1054, 1047, 1030, 1074, 1053, 1004, 1076, 1050, 1029, 999, 1054, 1038, 1022, 1016, 998, 1033, 1061, 1042, 1013, 1004, 1009, 1058, 986, 999, 1024, 1007, 1003, 1005, 1039, 997, 1004, 1023, 1051, 999, 983, 1035, 1000, 1043, 972, 1045, 973, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1015, 1001, 1015, 1054, 1052, 1036, 1024, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1012, 1015, 1008, 1008, 1048, 1066, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 985, 1031, 988, 1062, 996, 989, 1043, 10, 0, 0, 0, 0, 0, 0, 0, 1024, 1035, 1030, 1035, 1001, 1033, 1049, 1042, 36, 0, 0, 0, 0, 0, 0, 0, 985, 1038, 1028, 1063, 1052, 1048, 998, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1041, 1025, 992, 1013, 1028, 1018, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1021, 1059, 1028, 1013, 1030, 1013, 1028, 17, 1, 0, 0, 0, 0, 0, 0, 1055, 1017, 1004, 987, 1007, 1006, 1033, 1000, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8200, 8148, 8285, 8235, 8109, 8211, 8122] avg=8192.0000 max=8285 max_rank=3 min=8109 min_rank=5 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1007, 972, 1035, 1047, 1015, 985, 992, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 971, 1017, 1055, 1019, 1018, 966, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1026, 1119, 1031, 1008, 1071, 973, 1014, 10, 0, 0, 0, 0, 0, 0, 0, 988, 980, 1038, 1017, 1009, 1014, 1066, 1078, 36, 0, 0, 0, 0, 0, 0, 0, 1007, 1040, 1039, 1046, 1008, 1034, 1058, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1007, 1043, 989, 1000, 989, 1015, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1043, 934, 1023, 1013, 1034, 1024, 998, 17, 1, 0, 0, 0, 0, 0, 0, 1023, 1043, 1036, 1065, 1055, 1076, 1036, 976, 13, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8088, 8258, 8226, 8310, 8119, 8084, 8324] avg=8192.0000 max=8324 max_rank=7 min=8084 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[996, 1030, 1022, 1032, 1026, 1039, 1035, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1024, 1068, 997, 1001, 1000, 1014, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 978, 1013, 997, 1060, 1025, 1076, 1000, 10, 0, 0, 0, 0, 0, 0, 0, 992, 1009, 1008, 964, 1006, 993, 1047, 1001, 37, 0, 0, 0, 0, 0, 0, 0, 1020, 995, 1019, 1008, 962, 1057, 1044, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1055, 1026, 979, 1062, 1056, 1055, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 1018, 1012, 984, 1069, 1069, 1007, 1029, 18, 1, 0, 0, 0, 0, 0, 0, 1015, 1029, 1036, 1006, 1032, 1028, 1026, 1058, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8154, 8179, 8057, 8142, 8256, 8296, 8243] avg=8192.0000 max=8296 max_rank=6 min=8057 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32937 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1066, 1024, 1036, 990, 991, 1024, 1006, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1036, 1007, 986, 979, 1056, 1093, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1026, 1072, 1024, 1033, 1023, 1020, 990, 10, 0, 0, 0, 0, 0, 0, 0, 1017, 1039, 995, 1046, 1049, 1050, 926, 1003, 37, 0, 0, 0, 0, 0, 0, 0, 1021, 1051, 1011, 1079, 1031, 994, 1008, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1044, 1037, 1002, 1060, 1049, 996, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1031, 1079, 993, 1042, 1020, 1011, 1006, 17, 1, 0, 0, 0, 0, 0, 0, 1013, 945, 1001, 1061, 1081, 1031, 1018, 951, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8178, 8256, 8162, 8198, 8248, 8219, 8114] avg=8192.0000 max=8256 max_rank=2 min=8114 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[988, 996, 1029, 1058, 1014, 1002, 1039, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1020, 1011, 1008, 1091, 1040, 992, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1101, 974, 1064, 981, 1027, 968, 988, 10, 0, 0, 0, 0, 0, 0, 0, 1055, 1039, 997, 1015, 1064, 1027, 1023, 1005, 37, 0, 0, 0, 0, 0, 0, 0, 1014, 983, 1055, 997, 1006, 1031, 1004, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1030, 1034, 1032, 1032, 1021, 1013, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1023, 1108, 1022, 1025, 990, 1018, 1002, 17, 1, 0, 0, 0, 0, 0, 0, 1010, 1016, 1014, 1008, 1049, 1023, 1057, 1017, 13, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8208, 8135, 8262, 8106, 8190, 8229, 8208] avg=8192.0000 max=8262 max_rank=3 min=8106 min_rank=4 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[981, 1023, 1032, 1009, 990, 1045, 1037, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1052, 1070, 1045, 982, 1043, 1054, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1054, 1045, 1003, 988, 1033, 1046, 1046, 10, 0, 0, 0, 0, 0, 0, 0, 1023, 1000, 1002, 1021, 1028, 1033, 979, 960, 36, 0, 0, 0, 0, 0, 0, 0, 999, 1007, 1021, 1056, 1033, 1010, 1022, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 1041, 1043, 1042, 1031, 1007, 996, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 997, 995, 1055, 994, 1023, 1043, 972, 1062, 17, 1, 0, 0, 0, 0, 0, 0, 1038, 1004, 1042, 1010, 1024, 1005, 1037, 1037, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8272, 8244, 8082, 8200, 8259, 8159, 8210] avg=8192.0000 max=8272 max_rank=1 min=8082 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1028, 1026, 1017, 1041, 1029, 1029, 1092, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 1032, 989, 976, 1033, 1052, 1050, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 989, 1009, 1034, 995, 1004, 1022, 1002, 10, 0, 0, 0, 0, 0, 0, 0, 981, 1017, 1018, 997, 1026, 1001, 987, 1045, 35, 0, 0, 0, 0, 0, 0, 0, 965, 1016, 1024, 1033, 1015, 1047, 1012, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1014, 1020, 1042, 1010, 1057, 1022, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 996, 1040, 1003, 1064, 1041, 1023, 988, 17, 1, 0, 0, 0, 0, 0, 0, 995, 1045, 1035, 1065, 994, 987, 1042, 1033, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8273, 8072, 8107, 8124, 8276, 8187, 8209] avg=8192.0000 max=8288 max_rank=0 min=8072 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[935, 1010, 1082, 1038, 1053, 1055, 1062, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1027, 1003, 1041, 1035, 1011, 1029, 964, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 955, 1021, 1079, 1040, 1019, 1054, 1047, 14, 0, 0, 0, 0, 0, 0, 0, 1030, 1074, 1053, 1004, 1076, 1050, 1029, 999, 40, 0, 0, 0, 0, 0, 0, 0, 1011, 1038, 1022, 1016, 998, 1033, 1061, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1004, 1009, 1058, 986, 999, 1003, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1005, 1039, 997, 1004, 1023, 1051, 999, 21, 6, 3, 0, 0, 0, 0, 0, 983, 1035, 1000, 1043, 972, 1045, 973, 1036, 15, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8163, 8244, 8355, 8221, 8079, 8151, 8106] avg=8192.0000 max=8355 max_rank=3 min=8079 min_rank=5 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32557 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8448, 8064, 8320, 8320, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[997, 1037, 1065, 999, 1020, 1041, 1073, 989, 1033, 1024, 1048, 988, 1009, 992, 1084, 994, 1036, 1041, 1039, 958, 999, 997, 1022, 994, 1033, 1010, 1017, 1021, 996, 1062, 1057, 1036, 982, 1109, 1039, 994, 1019, 1000, 972, 1033, 1044, 1058, 1074, 1015, 997, 1014, 1022, 992, 1022, 1034, 1059, 981, 1055, 1077, 984, 1051, 1028, 1068, 1030, 975, 1040, 1027, 995, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1001, 996, 1070, 1020, 1021, 1039, 1046, 1052, 1013, 1054, 1029, 1042, 1055, 990, 1022, 984, 1065, 1029, 1034, 1029, 1052, 1009, 997, 1000, 1054, 1028, 1012, 1027, 994, 1020, 1072, 1060, 1008, 1023, 1035, 998, 1022, 993, 1008, 985, 1022, 1000, 1063, 1048, 1040, 991, 1023, 955, 1022, 1011, 1036, 978, 1032, 1038, 1027, 1051, 1005, 1060, 1010, 1034, 1018, 1020, 1052, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[979, 1033, 1009, 1004, 994, 1058, 1050, 1097, 1009, 1014, 1016, 1034, 1006, 1042, 1020, 1026, 1046, 1023, 1015, 1005, 1021, 1010, 970, 1022, 1049, 1000, 1057, 1071, 1021, 1014, 1052, 1040, 1054, 1013, 1019, 1043, 989, 1012, 1003, 997, 1029, 984, 1008, 1026, 1014, 1052, 953, 1019, 1070, 1066, 1070, 1017, 1003, 1013, 1013, 1030, 1007, 997, 1030, 1057, 1010, 1051, 1057, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1014, 1000, 1006, 1022, 976, 1059, 1058, 997, 956, 1089, 1019, 999, 997, 1035, 1026, 991, 1025, 1036, 1012, 994, 1033, 976, 1031, 1022, 1076, 992, 1035, 1034, 994, 1020, 1013, 1020, 1043, 1023, 1014, 984, 1012, 1030, 1032, 1044, 993, 1005, 1028, 1051, 1025, 1030, 1019, 1122, 1051, 1036, 1029, 1046, 1027, 1024, 1053, 1008, 1019, 1019, 1028, 1004, 1012, 1048, 1025, 1095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1026, 1041, 1031, 990, 1076, 997, 1018, 1015, 993, 1049, 1007, 988, 1030, 993, 1035, 1061, 1031, 950, 992, 1055, 1059, 983, 967, 1005, 1010, 1047, 1054, 998, 1007, 1043, 1098, 1047, 1025, 1018, 1034, 984, 1005, 1026, 1003, 984, 1037, 1046, 1028, 1047, 1016, 1034, 1066, 1059, 1020, 1033, 1016, 1026, 1056, 1050, 1015, 996, 1043, 1063, 988, 1057, 999, 1019, 1040, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1043, 1058, 1048, 1030, 936, 1033, 1029, 1029, 1010, 1019, 971, 1036, 1071, 1029, 1033, 996, 1029, 975, 987, 1031, 1044, 1018, 990, 1017, 1042, 1057, 1063, 1004, 1030, 1028, 1021, 1016, 1050, 998, 1047, 1062, 1011, 1053, 1023, 986, 1006, 986, 1049, 1019, 986, 1028, 1013, 1014, 1049, 1007, 985, 1043, 1014, 1012, 1041, 1028, 979, 1058, 1037, 1015, 1092, 1030, 1042, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1040, 1046, 1002, 1023, 1028, 997, 1011, 1050, 1037, 1035, 1019, 1005, 1053, 1028, 1007, 1001, 981, 1008, 1023, 1046, 1004, 994, 1093, 1019, 1042, 1055, 1047, 1062, 1041, 1018, 1002, 1036, 1060, 1043, 1035, 973, 1004, 1032, 1007, 1011, 1054, 1035, 1027, 1038, 1045, 1018, 1021, 1037, 1052, 1035, 999, 996, 1015, 992, 1035, 1039, 1022, 965, 1026, 1029, 1044, 982, 1004, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1015, 998, 1028, 1019, 1088, 989, 1060, 985, 1026, 1025, 1048, 1065, 1042, 1017, 1055, 1001, 966, 1001, 1043, 1019, 1022, 1003, 976, 1045, 1007, 1060, 1022, 970, 1008, 1035, 1041, 1026, 997, 999, 1001, 1033, 1041, 1028, 1017, 1031, 1042, 1047, 996, 1014, 1018, 1026, 1010, 1041, 1058, 1077, 1030, 1071, 1026, 1037, 1045, 1043, 1018, 1013, 986, 1024, 1023, 1016, 1036, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[997, 1037, 1065, 999, 1020, 1041, 1066, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1024, 1048, 988, 1009, 992, 1084, 994, 6, 6, 0, 0, 0, 0, 0, 0, 1036, 1041, 1039, 958, 999, 997, 1022, 994, 61, 15, 0, 0, 0, 0, 0, 0, 1033, 1010, 1017, 1021, 996, 1062, 996, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1109, 1039, 994, 1019, 1000, 972, 1033, 35, 10, 1, 0, 0, 0, 0, 0, 1044, 1058, 1064, 1015, 997, 1014, 1022, 992, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1034, 1059, 981, 1055, 1077, 984, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1068, 1030, 975, 1040, 1027, 989, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8184, 8162, 8171, 8194, 8206, 8213, 8192] avg=8192.0000 max=8214 max_rank=0 min=8162 min_rank=2 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1043, 1058, 1048, 1030, 936, 1033, 1022, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1019, 971, 1036, 1071, 1029, 1033, 996, 7, 6, 0, 0, 0, 0, 0, 0, 1029, 975, 987, 1031, 1044, 1018, 990, 1017, 59, 16, 0, 0, 0, 0, 0, 0, 1042, 1057, 1063, 1004, 1030, 1028, 962, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 998, 1047, 1062, 1011, 1053, 1023, 986, 36, 10, 1, 0, 0, 0, 0, 0, 1006, 986, 1039, 1019, 986, 1028, 1013, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1007, 985, 1043, 1014, 1012, 1041, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1058, 1037, 1015, 1092, 1030, 1035, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8178, 8166, 8202, 8277, 8091, 8127, 8296] avg=8192.0000 max=8296 max_rank=7 min=8091 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1014, 1000, 1006, 1022, 976, 1059, 1051, 997, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1089, 1019, 999, 997, 1035, 1026, 991, 7, 6, 0, 0, 0, 0, 0, 0, 1025, 1036, 1012, 994, 1033, 976, 1031, 1022, 59, 16, 0, 0, 0, 0, 0, 0, 1076, 992, 1035, 1034, 994, 1020, 954, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1023, 1014, 984, 1012, 1030, 1032, 1044, 36, 10, 1, 0, 0, 0, 0, 0, 993, 1005, 1018, 1051, 1025, 1030, 1019, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1036, 1029, 1046, 1027, 1024, 1053, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1019, 1028, 1004, 1012, 1048, 1018, 1095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8125, 8204, 8125, 8229, 8263, 8222, 8243] avg=8192.0000 max=8263 max_rank=5 min=8125 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32957 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1001, 996, 1070, 1020, 1021, 1039, 1039, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1054, 1029, 1042, 1055, 990, 1022, 984, 7, 6, 0, 0, 0, 0, 0, 0, 1065, 1029, 1034, 1029, 1052, 1009, 997, 1000, 62, 15, 0, 0, 0, 0, 0, 0, 1054, 1028, 1012, 1027, 994, 1020, 1010, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1023, 1035, 998, 1022, 993, 1008, 985, 35, 10, 1, 0, 0, 0, 0, 0, 1022, 1000, 1053, 1048, 1040, 991, 1023, 955, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1011, 1036, 978, 1032, 1038, 1027, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1060, 1010, 1034, 1018, 1020, 1045, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8202, 8292, 8205, 8118, 8132, 8145, 8204] avg=8192.0000 max=8292 max_rank=2 min=8118 min_rank=4 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32599 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[979, 1033, 1009, 1004, 994, 1058, 1043, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1014, 1016, 1034, 1006, 1042, 1020, 1026, 7, 6, 0, 0, 0, 0, 0, 0, 1046, 1023, 1015, 1005, 1021, 1010, 970, 1022, 61, 16, 0, 0, 0, 0, 0, 0, 1049, 1000, 1057, 1071, 1021, 1014, 991, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1013, 1019, 1043, 989, 1012, 1003, 997, 37, 10, 1, 0, 0, 0, 0, 0, 1029, 984, 998, 1026, 1014, 1052, 953, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1066, 1070, 1017, 1003, 1013, 1013, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 997, 1030, 1057, 1010, 1051, 1050, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8180, 8189, 8243, 8178, 8075, 8229, 8225] avg=8192.0000 max=8243 max_rank=3 min=8075 min_rank=5 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1026, 1041, 1031, 990, 1076, 997, 1011, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1049, 1007, 988, 1030, 993, 1035, 1061, 7, 6, 0, 0, 0, 0, 0, 0, 1031, 950, 992, 1055, 1059, 983, 967, 1005, 64, 15, 0, 0, 0, 0, 0, 0, 1010, 1047, 1054, 998, 1007, 1043, 1034, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1018, 1034, 984, 1005, 1026, 1003, 984, 35, 10, 1, 0, 0, 0, 0, 0, 1037, 1046, 1018, 1047, 1016, 1034, 1066, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1033, 1016, 1026, 1056, 1050, 1015, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1063, 988, 1057, 999, 1019, 1033, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8169, 8121, 8240, 8125, 8323, 8162, 8209] avg=8192.0000 max=8323 max_rank=5 min=8121 min_rank=2 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1040, 1046, 1002, 1023, 1028, 997, 1004, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1035, 1019, 1005, 1053, 1028, 1007, 1001, 6, 6, 0, 0, 0, 0, 0, 0, 981, 1008, 1023, 1046, 1004, 994, 1093, 1019, 58, 16, 0, 0, 0, 0, 0, 0, 1042, 1055, 1047, 1062, 1041, 1018, 944, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1043, 1035, 973, 1004, 1032, 1007, 1011, 36, 10, 1, 0, 0, 0, 0, 0, 1054, 1035, 1017, 1038, 1045, 1018, 1021, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1035, 999, 996, 1015, 992, 1035, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 965, 1026, 1029, 1044, 982, 998, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8197, 8242, 8245, 8212, 8265, 8111, 8074] avg=8192.0000 max=8265 max_rank=5 min=8074 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1015, 998, 1028, 1019, 1088, 989, 1044, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1025, 1048, 1065, 1042, 1017, 1055, 1001, 10, 12, 0, 0, 0, 0, 0, 0, 966, 1001, 1043, 1019, 1022, 1003, 976, 1045, 64, 21, 0, 0, 0, 0, 0, 0, 1007, 1060, 1022, 970, 1008, 1035, 977, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 997, 999, 1001, 1033, 1041, 1028, 1017, 1031, 39, 13, 4, 0, 0, 0, 0, 0, 1042, 1047, 983, 1014, 1018, 1026, 1010, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1077, 1030, 1071, 1026, 1037, 1045, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1013, 986, 1024, 1023, 1016, 1026, 987, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8301, 8160, 8105, 8203, 8181, 8327, 8093] avg=8192.0000 max=8327 max_rank=6 min=8093 min_rank=7 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8192, 8320, 8064, 8064, 8192, 512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 7936, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8192, 384, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7936, 8320, 8320, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1034, 1009, 1037, 1028, 1051, 980, 1033, 1028, 1002, 1005, 1020, 1033, 982, 1007, 1023, 978, 1010, 1052, 1063, 1036, 997, 1038, 1063, 982, 1034, 1040, 1037, 967, 1029, 1034, 1021, 997, 1030, 1056, 1049, 1046, 1053, 1022, 992, 1070, 1034, 1020, 1022, 1011, 1032, 1057, 1014, 1006, 1059, 996, 994, 1047, 1059, 1038, 1036, 957, 1039, 1060, 1001, 991, 1054, 977, 1058, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1060, 1019, 1011, 1075, 1043, 1028, 1043, 1007, 1031, 1044, 1025, 1034, 1030, 1001, 1008, 1013, 1008, 1021, 1051, 1038, 1041, 1013, 1029, 1056, 991, 1037, 963, 1030, 1055, 973, 1056, 1010, 1015, 969, 1006, 1081, 1019, 1021, 1019, 1029, 1001, 1030, 1046, 1043, 1049, 1058, 1017, 1019, 998, 972, 1010, 1043, 1031, 1032, 983, 996, 1004, 1031, 1070, 1001, 981, 984, 1082, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[998, 1010, 1020, 1066, 976, 966, 1024, 1070, 929, 1003, 1026, 1051, 1048, 1053, 1030, 1025, 1060, 1031, 1017, 1062, 1010, 978, 1067, 1041, 1021, 1005, 1010, 1039, 1023, 996, 1007, 1038, 1005, 1083, 996, 991, 988, 1056, 1039, 1075, 1016, 1013, 1014, 991, 1039, 1037, 1052, 1036, 1028, 1057, 1006, 997, 1024, 1047, 997, 1038, 992, 1025, 1033, 1027, 1035, 1023, 1009, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[975, 1016, 1067, 973, 1060, 1002, 1068, 1003, 1033, 1007, 983, 1059, 1039, 1037, 1029, 1035, 983, 1007, 1036, 996, 1021, 1000, 1025, 1008, 1069, 1113, 1019, 1029, 980, 1013, 999, 1017, 1001, 1010, 1018, 1014, 1052, 1040, 1030, 1049, 1014, 1050, 1028, 1023, 1054, 1002, 1054, 1029, 956, 1054, 1016, 1048, 1028, 977, 1031, 1008, 1040, 996, 1067, 1028, 1052, 1015, 1014, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1009, 1019, 1043, 996, 963, 971, 1008, 1046, 1035, 1021, 981, 1069, 1059, 1070, 1021, 1026, 1049, 995, 1025, 991, 1007, 984, 962, 1050, 1036, 1055, 1020, 1024, 1021, 1004, 1024, 1036, 1053, 1047, 999, 1041, 1048, 1038, 1035, 1050, 1035, 1025, 976, 1010, 1033, 1043, 1037, 1039, 1040, 1066, 1070, 1005, 968, 998, 999, 1050, 981, 1035, 1018, 1009, 1037, 1039, 1012, 1080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1028, 1000, 996, 1004, 987, 1035, 1030, 991, 1040, 1037, 1012, 986, 1026, 1033, 1016, 1062, 1012, 1029, 1020, 997, 1070, 1031, 980, 1010, 1041, 1014, 1073, 1049, 1076, 993, 1010, 1056, 1061, 986, 1068, 1045, 1062, 1049, 976, 988, 996, 1018, 1023, 1006, 1029, 1009, 1034, 1063, 1052, 1031, 972, 997, 1032, 1078, 1028, 1013, 1043, 962, 1061, 1005, 1022, 1024, 1031, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1025, 1008, 998, 1036, 1016, 990, 1029, 1012, 1036, 1031, 998, 1010, 1019, 1002, 1045, 1012, 1038, 1007, 1042, 1000, 1106, 1021, 992, 1041, 953, 1053, 1057, 1034, 992, 984, 988, 1050, 1023, 1023, 1024, 1010, 1051, 1057, 1021, 1034, 1015, 1095, 1065, 986, 990, 1000, 1023, 1036, 1001, 1011, 1062, 996, 1047, 1036, 1038, 1015, 1037, 1055, 1055, 1069, 1036, 1010, 1012, 978] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1012, 1018, 1054, 1026, 990, 1004, 1055, 1014, 1020, 947, 1073, 1004, 1004, 1035, 1073, 1013, 1036, 1008, 1032, 1060, 1004, 1073, 1019, 1001, 1053, 1033, 977, 994, 989, 1068, 1012, 1002, 945, 1024, 1034, 1069, 1023, 966, 1022, 1027, 1055, 1068, 1040, 1027, 991, 1003, 1012, 1056, 1057, 1035, 968, 1044, 993, 1048, 1081, 1037, 1071, 1016, 1064, 1046, 1009, 1019, 973, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1060, 1019, 1011, 1075, 1043, 1028, 1043, 1007, 35, 6, 0, 0, 0, 0, 0, 0, 1031, 1044, 1025, 1034, 1030, 1001, 1008, 1013, 12, 0, 0, 0, 0, 0, 0, 0, 1008, 1021, 1051, 1038, 1041, 1013, 1029, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1037, 963, 1030, 1055, 973, 1056, 1010, 13, 0, 0, 0, 0, 0, 0, 0, 1015, 969, 1006, 1081, 1019, 1021, 1019, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1006, 1046, 1043, 1049, 1058, 1017, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 998, 972, 1010, 1043, 1031, 1032, 983, 996, 7, 5, 0, 0, 0, 0, 0, 0, 1004, 1031, 1051, 1001, 981, 984, 1082, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8327, 8198, 8257, 8128, 8124, 8239, 8077, 8186] avg=8192.0000 max=8327 max_rank=0 min=8077 min_rank=6 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1034, 1009, 1037, 1028, 1051, 980, 1033, 1028, 36, 6, 0, 0, 0, 0, 0, 0, 1002, 1005, 1020, 1033, 982, 1007, 1023, 978, 11, 0, 0, 0, 0, 0, 0, 0, 1010, 1052, 1063, 1036, 997, 1038, 1063, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1040, 1037, 967, 1029, 1034, 1021, 997, 13, 0, 0, 0, 0, 0, 0, 0, 1030, 1056, 1049, 1046, 1053, 1022, 992, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 996, 1022, 1011, 1032, 1057, 1014, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 996, 994, 1047, 1059, 1038, 1036, 957, 7, 5, 0, 0, 0, 0, 0, 0, 1039, 1060, 983, 991, 1054, 977, 1058, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8061, 8241, 8172, 8282, 8172, 8198, 8168] avg=8192.0000 max=8282 max_rank=4 min=8061 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1028, 1000, 996, 1004, 987, 1035, 1030, 991, 34, 6, 0, 0, 0, 0, 0, 0, 1040, 1037, 1012, 986, 1026, 1033, 1016, 1062, 12, 0, 0, 0, 0, 0, 0, 0, 1012, 1029, 1020, 997, 1070, 1031, 980, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1014, 1073, 1049, 1076, 993, 1010, 1056, 13, 0, 0, 0, 0, 0, 0, 0, 1061, 986, 1068, 1045, 1062, 1049, 976, 954, 0, 0, 0, 0, 0, 0, 0, 0, 996, 994, 1023, 1006, 1029, 1009, 1034, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1031, 972, 997, 1032, 1078, 1028, 1013, 7, 5, 0, 0, 0, 0, 0, 0, 1043, 962, 1042, 1005, 1022, 1024, 1031, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8224, 8149, 8325, 8201, 8154, 8215, 8157] avg=8192.0000 max=8325 max_rank=3 min=8111 min_rank=0 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[998, 1010, 1020, 1066, 976, 966, 1024, 1070, 37, 6, 0, 0, 0, 0, 0, 0, 929, 1003, 1026, 1051, 1048, 1053, 1030, 1025, 11, 0, 0, 0, 0, 0, 0, 0, 1060, 1031, 1017, 1062, 1010, 978, 1067, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1005, 1010, 1039, 1023, 996, 1007, 1038, 13, 0, 0, 0, 0, 0, 0, 0, 1005, 1083, 996, 991, 988, 1056, 1039, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 989, 1014, 991, 1039, 1037, 1052, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1057, 1006, 997, 1024, 1047, 997, 1038, 7, 5, 0, 0, 0, 0, 0, 0, 992, 1025, 1015, 1027, 1035, 1023, 1009, 1067, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8176, 8266, 8152, 8196, 8174, 8206, 8193] avg=8192.0000 max=8266 max_rank=2 min=8152 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[975, 1016, 1067, 973, 1060, 1002, 1068, 1003, 36, 7, 0, 0, 0, 0, 0, 0, 1033, 1007, 983, 1059, 1039, 1037, 1029, 1035, 12, 0, 0, 0, 0, 0, 0, 0, 983, 1007, 1036, 996, 1021, 1000, 1025, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1113, 1019, 1029, 980, 1013, 999, 1017, 14, 0, 0, 0, 0, 0, 0, 0, 1001, 1010, 1018, 1014, 1052, 1040, 1030, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1024, 1028, 1023, 1054, 1002, 1054, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1054, 1016, 1048, 1028, 977, 1031, 1008, 7, 5, 0, 0, 0, 0, 0, 0, 1040, 996, 1048, 1028, 1052, 1015, 1014, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8234, 8076, 8253, 8178, 8228, 8130, 8230] avg=8192.0000 max=8253 max_rank=3 min=8076 min_rank=2 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1009, 1019, 1043, 996, 963, 971, 1008, 1046, 36, 6, 0, 0, 0, 0, 0, 0, 1035, 1021, 981, 1069, 1059, 1070, 1021, 1026, 11, 0, 0, 0, 0, 0, 0, 0, 1049, 995, 1025, 991, 1007, 984, 962, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1055, 1020, 1024, 1021, 1004, 1024, 1036, 13, 0, 0, 0, 0, 0, 0, 0, 1053, 1047, 999, 1041, 1048, 1038, 1035, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1001, 976, 1010, 1033, 1043, 1037, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1066, 1070, 1005, 968, 998, 999, 1050, 7, 5, 0, 0, 0, 0, 0, 0, 981, 1035, 1000, 1009, 1037, 1039, 1012, 1080, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8293, 8063, 8233, 8275, 8174, 8208, 8193] avg=8192.0000 max=8293 max_rank=1 min=8063 min_rank=2 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32850 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1025, 1008, 998, 1036, 1016, 990, 1029, 1012, 35, 7, 0, 0, 0, 0, 0, 0, 1036, 1031, 998, 1010, 1019, 1002, 1045, 1012, 12, 0, 0, 0, 0, 0, 0, 0, 1038, 1007, 1042, 1000, 1106, 1021, 992, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 953, 1053, 1057, 1034, 992, 984, 988, 1050, 14, 0, 0, 0, 0, 0, 0, 0, 1023, 1023, 1024, 1010, 1051, 1057, 1021, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1069, 1065, 986, 990, 1000, 1023, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1011, 1062, 996, 1047, 1036, 1038, 1015, 7, 5, 0, 0, 0, 0, 0, 0, 1037, 1055, 1036, 1069, 1036, 1010, 1012, 978, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8165, 8247, 8125, 8208, 8184, 8218, 8233] avg=8192.0000 max=8247 max_rank=2 min=8125 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1012, 1018, 1054, 1026, 990, 1004, 1055, 1014, 38, 12, 0, 0, 0, 0, 0, 0, 1020, 947, 1073, 1004, 1004, 1035, 1073, 1013, 16, 0, 0, 0, 0, 0, 0, 0, 1036, 1008, 1032, 1060, 1004, 1073, 1019, 1001, 4, 0, 0, 0, 0, 0, 0, 0, 1053, 1033, 977, 994, 989, 1068, 1012, 1002, 20, 0, 0, 0, 0, 0, 0, 0, 945, 1024, 1034, 1069, 1023, 966, 1022, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1027, 1040, 1027, 991, 1003, 1012, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1035, 968, 1044, 993, 1048, 1081, 1037, 12, 9, 0, 0, 0, 0, 0, 0, 1071, 1016, 1032, 1046, 1009, 1019, 973, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8185, 8237, 8148, 8072, 8211, 8284, 8176] avg=8192.0000 max=8284 max_rank=6 min=8072 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8064, 8320, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8448, 8192, 8192, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8320, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1029, 961, 1037, 980, 1022, 1011, 1093, 1055, 1014, 1069, 1053, 1066, 1046, 983, 1038, 1014, 1012, 1053, 1018, 1068, 1067, 1031, 1022, 990, 1001, 1042, 1043, 1021, 999, 1017, 1037, 993, 1038, 1016, 1006, 1073, 1005, 1054, 1008, 1013, 1006, 992, 1055, 1029, 1037, 983, 985, 1010, 988, 1025, 1008, 1012, 1021, 1030, 1013, 1028, 1028, 1031, 1049, 988, 1001, 1055, 1020, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1032, 1023, 1080, 1010, 1072, 1030, 1003, 1009, 1025, 1053, 994, 1048, 1040, 1029, 1093, 1025, 1021, 981, 1003, 1032, 1108, 998, 1031, 1059, 1018, 1051, 1050, 1038, 1057, 1058, 1029, 1081, 1034, 987, 1048, 1030, 1000, 1025, 1023, 1023, 1042, 1014, 991, 986, 1025, 1026, 1037, 957, 1005, 1027, 982, 959, 994, 1010, 1052, 1009, 984, 976, 1029, 971, 1061, 1012, 1074, 962] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1031, 1014, 1031, 1033, 1011, 1019, 1054, 1009, 1016, 990, 1029, 1051, 1008, 1017, 1057, 992, 1054, 1027, 1039, 1010, 1053, 1042, 998, 1042, 1020, 1061, 965, 969, 1028, 995, 991, 1063, 1065, 979, 999, 1009, 1030, 1010, 1061, 1024, 993, 1058, 1051, 1076, 1047, 1005, 1045, 1035, 1040, 1031, 1012, 1045, 1013, 989, 1032, 1009, 1021, 1058, 1013, 1033, 1018, 1015, 1008, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1010, 1028, 987, 1003, 991, 1028, 1005, 961, 1012, 1033, 1029, 1018, 1015, 1068, 1010, 997, 1050, 1000, 1033, 1038, 1086, 975, 1034, 1058, 996, 1031, 993, 964, 1047, 1065, 1018, 1051, 1046, 1055, 1002, 1032, 1028, 1029, 1062, 1030, 1049, 1018, 990, 1008, 1027, 1052, 1050, 1021, 1048, 1097, 1002, 1037, 1045, 1062, 1039, 1035, 976, 1035, 1059, 982, 1007, 961, 1007, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1058, 1033, 1038, 1007, 1065, 959, 1051, 1011, 1060, 1032, 1014, 1005, 1006, 1000, 1014, 997, 1012, 1044, 1042, 1029, 1086, 996, 1005, 986, 1034, 989, 1009, 1062, 1002, 1026, 1029, 1060, 1110, 1027, 1013, 1053, 1060, 980, 1021, 1049, 1050, 1023, 1045, 993, 1011, 1044, 1048, 1047, 1018, 1058, 1000, 987, 1020, 1052, 992, 957, 1034, 1073, 1058, 979, 995, 967, 982, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1048, 1016, 1055, 1010, 1034, 986, 1027, 1020, 1077, 996, 984, 1000, 1008, 1011, 984, 1067, 1035, 1007, 1073, 991, 1026, 1006, 1070, 1048, 1069, 1011, 1009, 1005, 998, 1048, 1049, 1048, 1027, 1058, 1013, 1035, 999, 1009, 1007, 1046, 1062, 1035, 1048, 1053, 1039, 983, 996, 962, 998, 1080, 1015, 977, 1037, 1035, 1025, 1043, 1003, 1008, 1037, 1004, 1006, 1047, 996, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1058, 990, 1011, 1063, 1025, 1047, 1042, 1014, 1062, 965, 1011, 1041, 1029, 1065, 1050, 1039, 970, 1044, 1048, 1012, 1009, 1031, 974, 1018, 1011, 1001, 1020, 1043, 1028, 1020, 1029, 996, 1055, 1007, 1020, 1095, 1037, 1002, 1055, 982, 940, 1018, 1081, 963, 1026, 1032, 1034, 1038, 1071, 992, 1006, 1080, 1066, 1055, 1009, 983, 1017, 1044, 1041, 1008, 991, 975, 1015, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1028, 996, 1063, 1027, 1027, 1013, 976, 1063, 991, 986, 977, 1002, 1002, 1059, 1007, 996, 1051, 1038, 1018, 1013, 1030, 1003, 1032, 995, 1049, 986, 989, 1088, 991, 1057, 1010, 1004, 1018, 1049, 1059, 1064, 1030, 1030, 1044, 1020, 1031, 980, 1033, 1040, 1053, 1035, 1029, 1005, 1032, 1049, 979, 1031, 1015, 1041, 1033, 1047, 1002, 1025, 1038, 1004, 1038, 1051, 1047, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1031, 1014, 1031, 1033, 1011, 1019, 1054, 1009, 1, 0, 0, 0, 0, 0, 0, 0, 1016, 990, 1029, 1051, 1008, 1017, 1057, 992, 8, 0, 0, 0, 0, 0, 0, 0, 1054, 1027, 1039, 1010, 1025, 1042, 998, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1061, 965, 969, 1028, 995, 991, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 979, 999, 1009, 1030, 1010, 1061, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1058, 1051, 1076, 1047, 1005, 1045, 1035, 7, 0, 0, 0, 0, 0, 0, 0, 1040, 1031, 1012, 1045, 1013, 989, 1032, 1009, 5, 1, 0, 0, 0, 0, 0, 0, 1021, 1058, 1013, 1033, 1018, 1015, 1008, 993, 48, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8168, 8237, 8090, 8129, 8317, 8177, 8215] avg=8192.0000 max=8317 max_rank=5 min=8090 min_rank=3 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1029, 961, 1037, 980, 1022, 1011, 1093, 1055, 1, 0, 0, 0, 0, 0, 0, 0, 1014, 1069, 1053, 1066, 1046, 983, 1038, 1014, 8, 0, 0, 0, 0, 0, 0, 0, 1012, 1053, 1018, 1068, 1038, 1031, 1022, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1042, 1043, 1021, 999, 1017, 1037, 991, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1016, 1006, 1073, 1005, 1054, 1008, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 992, 1055, 1029, 1037, 983, 985, 1010, 7, 0, 0, 0, 0, 0, 0, 0, 988, 1025, 1008, 1012, 1021, 1030, 1013, 1028, 5, 1, 0, 0, 0, 0, 0, 0, 1028, 1031, 1049, 988, 1001, 1055, 1020, 1044, 47, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8291, 8232, 8151, 8166, 8104, 8131, 8272] avg=8192.0000 max=8291 max_rank=1 min=8104 min_rank=5 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1032, 1023, 1080, 1010, 1072, 1030, 1003, 1009, 1, 0, 0, 0, 0, 0, 0, 0, 1025, 1053, 994, 1048, 1040, 1029, 1093, 1025, 8, 0, 0, 0, 0, 0, 0, 0, 1021, 981, 1003, 1032, 1079, 998, 1031, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1051, 1050, 1038, 1057, 1058, 1029, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 987, 987, 1048, 1030, 1000, 1025, 1023, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1014, 991, 986, 1025, 1026, 1037, 957, 7, 0, 0, 0, 0, 0, 0, 0, 1005, 1027, 982, 959, 994, 1010, 1052, 1009, 5, 1, 0, 0, 0, 0, 0, 0, 984, 976, 1029, 971, 1061, 1012, 1074, 962, 47, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8315, 8204, 8380, 8123, 8085, 8044, 8125] avg=8192.0000 max=8380 max_rank=3 min=8044 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1058, 990, 1011, 1063, 1025, 1047, 1042, 1014, 1, 0, 0, 0, 0, 0, 0, 0, 1062, 965, 1011, 1041, 1029, 1065, 1050, 1039, 7, 0, 0, 0, 0, 0, 0, 0, 970, 1044, 1048, 1012, 983, 1031, 974, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1001, 1020, 1043, 1028, 1020, 1029, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1007, 1020, 1095, 1037, 1002, 1055, 982, 0, 0, 0, 0, 0, 0, 0, 0, 940, 1018, 1081, 963, 1026, 1032, 1034, 1038, 7, 0, 0, 0, 0, 0, 0, 0, 1071, 992, 1006, 1080, 1066, 1055, 1009, 983, 4, 1, 0, 0, 0, 0, 0, 0, 1017, 1044, 1041, 1008, 991, 975, 1015, 1032, 48, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8269, 8080, 8146, 8205, 8139, 8267, 8179] avg=8192.0000 max=8269 max_rank=1 min=8080 min_rank=2 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1058, 1033, 1038, 1007, 1065, 959, 1051, 1011, 1, 0, 0, 0, 0, 0, 0, 0, 1060, 1032, 1014, 1005, 1006, 1000, 1014, 997, 8, 0, 0, 0, 0, 0, 0, 0, 1012, 1044, 1042, 1029, 1057, 996, 1005, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 989, 1009, 1062, 1002, 1026, 1029, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1027, 1013, 1053, 1060, 980, 1021, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1023, 1045, 993, 1011, 1044, 1048, 1047, 7, 0, 0, 0, 0, 0, 0, 0, 1018, 1058, 1000, 987, 1020, 1052, 992, 957, 5, 1, 0, 0, 0, 0, 0, 0, 1034, 1073, 1058, 979, 995, 967, 982, 1029, 50, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8136, 8171, 8209, 8263, 8268, 8090, 8176] avg=8192.0000 max=8268 max_rank=5 min=8090 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1048, 1016, 1055, 1010, 1034, 986, 1027, 1020, 1, 0, 0, 0, 0, 0, 0, 0, 1077, 996, 984, 1000, 1008, 1011, 984, 1067, 8, 0, 0, 0, 0, 0, 0, 0, 1035, 1007, 1073, 991, 999, 1006, 1070, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1011, 1009, 1005, 998, 1048, 1049, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1058, 1013, 1035, 999, 1009, 1007, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1035, 1048, 1053, 1039, 983, 996, 962, 7, 0, 0, 0, 0, 0, 0, 0, 998, 1080, 1015, 977, 1037, 1035, 1025, 1043, 4, 1, 0, 0, 0, 0, 0, 0, 1003, 1008, 1037, 1004, 1006, 1047, 996, 1037, 46, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8135, 8229, 8235, 8148, 8185, 8215, 8192] avg=8192.0000 max=8235 max_rank=3 min=8135 min_rank=1 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1010, 1028, 987, 1003, 991, 1028, 1005, 961, 1, 0, 0, 0, 0, 0, 0, 0, 1012, 1033, 1029, 1018, 1015, 1068, 1010, 997, 8, 0, 0, 0, 0, 0, 0, 0, 1050, 1000, 1033, 1038, 1057, 975, 1034, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1031, 993, 964, 1047, 1065, 1018, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1055, 1002, 1032, 1028, 1029, 1062, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1018, 990, 1008, 1027, 1052, 1050, 1021, 7, 0, 0, 0, 0, 0, 0, 0, 1048, 1097, 1002, 1037, 1045, 1062, 1039, 1035, 5, 1, 0, 0, 0, 0, 0, 0, 976, 1035, 1059, 982, 1007, 961, 1007, 1011, 47, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8014, 8190, 8245, 8163, 8237, 8222, 8371, 8094] avg=8192.0000 max=8371 max_rank=6 min=8014 min_rank=0 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32924 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1028, 996, 1063, 1027, 1027, 1013, 976, 1063, 6, 0, 0, 0, 0, 0, 0, 0, 991, 986, 977, 1002, 1002, 1059, 1007, 996, 12, 0, 0, 0, 0, 0, 0, 0, 1051, 1038, 1018, 1013, 988, 1003, 1032, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 986, 989, 1088, 991, 1057, 1010, 992, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1049, 1059, 1064, 1030, 1030, 1044, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 980, 1033, 1040, 1053, 1035, 1029, 1005, 10, 0, 0, 0, 0, 0, 0, 0, 1032, 1049, 979, 1031, 1015, 1041, 1033, 1047, 8, 6, 0, 0, 0, 0, 0, 0, 1002, 1025, 1038, 1004, 1038, 1051, 1047, 1017, 49, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8032, 8138, 8162, 8265, 8216, 8241, 8283] avg=8192.0000 max=8283 max_rank=7 min=8032 min_rank=1 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=33005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8192, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8448, 8064, 8192, 8192, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8448, 8064, 8192, 8320, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8448, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1027, 1034, 1018, 1020, 1030, 1030, 1027, 1006, 1034, 1007, 1006, 1036, 1078, 1020, 1071, 1053, 1011, 1033, 1009, 1041, 1037, 996, 1001, 1023, 1011, 1036, 990, 1017, 974, 1038, 1011, 1036, 1047, 1006, 1017, 1041, 1004, 1050, 1082, 962, 988, 1000, 986, 1033, 1050, 1001, 1006, 1015, 1050, 1009, 1088, 1031, 997, 1016, 1004, 1019, 1032, 1001, 1005, 1046, 1060, 1066, 1036, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1028, 1029, 953, 1022, 1054, 1021, 1041, 1024, 980, 995, 1077, 1047, 1002, 1056, 1064, 988, 996, 1027, 997, 1017, 1019, 971, 1010, 986, 1029, 1046, 1034, 984, 1052, 1073, 997, 1030, 1008, 1090, 1015, 1050, 982, 1090, 1064, 1036, 1001, 1070, 997, 1042, 1021, 1011, 1024, 1024, 1015, 1069, 1004, 1044, 1044, 1002, 1005, 1079, 982, 1002, 1015, 1030, 1009, 1017, 1006, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[975, 1042, 1021, 994, 1053, 1039, 1018, 997, 1052, 1017, 1028, 1107, 1021, 1047, 997, 976, 1055, 1015, 1051, 1043, 1028, 1055, 1007, 992, 998, 1077, 948, 1015, 1002, 975, 1003, 1028, 1024, 993, 1047, 1024, 1019, 1048, 1014, 1061, 984, 1022, 1008, 1049, 1029, 1061, 1050, 989, 1036, 1022, 1038, 1016, 951, 1049, 1072, 1030, 1051, 1053, 1024, 1012, 994, 1005, 1039, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1014, 1038, 1026, 1000, 1022, 1020, 985, 1032, 1008, 1020, 1076, 1070, 1035, 1075, 1024, 1002, 1056, 1027, 1040, 1059, 990, 1017, 1025, 1057, 1028, 1034, 1049, 1013, 1007, 1016, 985, 1036, 1028, 1071, 967, 1064, 1031, 1048, 1019, 1025, 1001, 982, 1022, 1014, 1001, 1009, 992, 1057, 1002, 1034, 1042, 1029, 1014, 1011, 999, 1052, 1019, 1024, 1012, 990, 1045, 1009, 1032, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1036, 1026, 1049, 1033, 1053, 1028, 1069, 1011, 1000, 1026, 989, 1076, 1012, 1042, 1013, 973, 1031, 1013, 1013, 1029, 1007, 1026, 1048, 1027, 1059, 1086, 991, 1001, 1017, 1055, 997, 1051, 1056, 1008, 1026, 1007, 987, 1036, 1075, 984, 1032, 1028, 1044, 1000, 1037, 1034, 980, 1074, 1028, 1048, 962, 991, 991, 1032, 1026, 1067, 1025, 1039, 1028, 950, 982, 1028, 1043, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1024, 1001, 1043, 1050, 1063, 1031, 1060, 1069, 1050, 1019, 1058, 1019, 979, 987, 1047, 1008, 1066, 1028, 1019, 1036, 1087, 1031, 1013, 1015, 996, 1034, 973, 1053, 1053, 973, 999, 1043, 1050, 1011, 987, 1033, 1024, 1084, 1051, 1015, 1008, 1021, 1047, 972, 1026, 1013, 1004, 1041, 993, 1010, 943, 992, 985, 1042, 1048, 1048, 972, 1057, 1035, 1040, 1013, 994, 1014, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1018, 1027, 1001, 1061, 996, 1065, 1045, 992, 1040, 1086, 1014, 1044, 1052, 1042, 1010, 1044, 1031, 979, 1030, 998, 1023, 996, 1017, 983, 1034, 1046, 1039, 1023, 1003, 1004, 1037, 1051, 1019, 1020, 1075, 981, 1014, 1004, 1053, 983, 1038, 1009, 983, 1036, 1030, 1035, 1008, 976, 1033, 1010, 1047, 1052, 1062, 1040, 1018, 1026, 1039, 993, 1027, 976, 1046, 1027, 1040, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1071, 1049, 1038, 1023, 969, 1027, 1059, 1008, 1043, 972, 959, 1054, 985, 1070, 1033, 1034, 1018, 1029, 977, 1060, 1049, 1084, 1031, 991, 1003, 1008, 1063, 1006, 1001, 1040, 1039, 1014, 1071, 1036, 1048, 1047, 1008, 982, 1018, 981, 1030, 1033, 1048, 1034, 1035, 1005, 1067, 1012, 1052, 1002, 1053, 972, 1012, 1058, 1031, 1045, 1007, 980, 1001, 1030, 994, 1052, 1014, 971] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1027, 1034, 1018, 1020, 1030, 1030, 1004, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1007, 1006, 998, 1078, 1020, 1071, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1033, 1009, 1041, 1037, 996, 1001, 1023, 6, 0, 0, 0, 0, 0, 0, 0, 1011, 1036, 990, 1017, 974, 1038, 1011, 1036, 21, 0, 0, 0, 0, 0, 0, 0, 1047, 1006, 1017, 1041, 1004, 1050, 1048, 962, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1000, 986, 1033, 1050, 1001, 1006, 1015, 30, 2, 0, 0, 0, 0, 0, 0, 1050, 1009, 1088, 1031, 997, 1016, 1004, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1001, 1005, 1046, 1060, 1066, 1036, 1027, 38, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8267, 8157, 8134, 8175, 8111, 8208, 8315] avg=8192.0000 max=8315 max_rank=7 min=8111 min_rank=5 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1014, 1038, 1026, 1000, 1022, 1020, 963, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1020, 1076, 1031, 1035, 1075, 1024, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1027, 1040, 1059, 990, 1017, 1025, 1057, 7, 0, 0, 0, 0, 0, 0, 0, 1028, 1034, 1049, 1013, 1007, 1016, 985, 1036, 20, 0, 0, 0, 0, 0, 0, 0, 1028, 1071, 967, 1064, 1031, 1048, 988, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 982, 1022, 1014, 1001, 1009, 992, 1057, 28, 2, 0, 0, 0, 0, 0, 0, 1002, 1034, 1042, 1029, 1014, 1011, 999, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1024, 1012, 990, 1045, 1009, 1032, 1005, 39, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8271, 8278, 8188, 8222, 8108, 8176, 8178] avg=8192.0000 max=8278 max_rank=2 min=8108 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1028, 1029, 953, 1022, 1054, 1021, 1018, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 980, 995, 1077, 1009, 1002, 1056, 1064, 988, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1027, 997, 1017, 1019, 971, 1010, 986, 7, 0, 0, 0, 0, 0, 0, 0, 1029, 1046, 1034, 984, 1052, 1073, 997, 1030, 21, 0, 0, 0, 0, 0, 0, 0, 1008, 1090, 1015, 1050, 982, 1090, 1032, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1070, 997, 1042, 1021, 1011, 1024, 1024, 29, 2, 0, 0, 0, 0, 0, 0, 1015, 1069, 1004, 1044, 1044, 1002, 1005, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1002, 1015, 1030, 1009, 1017, 1006, 1039, 38, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8171, 8030, 8266, 8303, 8221, 8255, 8141] avg=8192.0000 max=8303 max_rank=4 min=8030 min_rank=2 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[975, 1042, 1021, 994, 1053, 1039, 995, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1017, 1028, 1067, 1021, 1047, 997, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1015, 1051, 1043, 1028, 1055, 1007, 992, 6, 0, 0, 0, 0, 0, 0, 0, 998, 1077, 948, 1015, 1002, 975, 1003, 1028, 21, 0, 0, 0, 0, 0, 0, 0, 1024, 993, 1047, 1024, 1019, 1048, 983, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1022, 1008, 1049, 1029, 1061, 1050, 989, 28, 2, 0, 0, 0, 0, 0, 0, 1036, 1022, 1038, 1016, 951, 1049, 1072, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1053, 1024, 1012, 994, 1005, 1039, 1046, 40, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8205, 8252, 8067, 8199, 8222, 8208, 8267] avg=8192.0000 max=8267 max_rank=7 min=8067 min_rank=3 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1036, 1026, 1049, 1033, 1053, 1028, 1045, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1026, 989, 1037, 1012, 1042, 1013, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1013, 1013, 1029, 1007, 1026, 1048, 1027, 7, 0, 0, 0, 0, 0, 0, 0, 1059, 1086, 991, 1001, 1017, 1055, 997, 1051, 22, 0, 0, 0, 0, 0, 0, 0, 1056, 1008, 1026, 1007, 987, 1036, 1042, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1028, 1044, 1000, 1037, 1034, 980, 1074, 30, 2, 0, 0, 0, 0, 0, 0, 1028, 1048, 962, 991, 991, 1032, 1026, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1039, 1028, 950, 982, 1028, 1043, 1001, 39, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8092, 8201, 8279, 8146, 8261, 8138, 8138] avg=8192.0000 max=8281 max_rank=0 min=8092 min_rank=1 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1018, 1027, 1001, 1061, 996, 1065, 1022, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1086, 1014, 1006, 1052, 1042, 1010, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 979, 1030, 998, 1023, 996, 1017, 983, 6, 0, 0, 0, 0, 0, 0, 0, 1034, 1046, 1039, 1023, 1003, 1004, 1037, 1051, 21, 0, 0, 0, 0, 0, 0, 0, 1019, 1020, 1075, 981, 1014, 1004, 1021, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1009, 983, 1036, 1030, 1035, 1008, 976, 29, 2, 0, 0, 0, 0, 0, 0, 1033, 1010, 1047, 1052, 1062, 1040, 1018, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 993, 1027, 976, 1046, 1027, 1040, 1005, 38, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8294, 8063, 8258, 8117, 8146, 8282, 8194] avg=8192.0000 max=8294 max_rank=1 min=8063 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1024, 1001, 1043, 1050, 1063, 1031, 1037, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1019, 1058, 982, 979, 987, 1047, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1028, 1019, 1036, 1087, 1031, 1013, 1015, 7, 0, 0, 0, 0, 0, 0, 0, 996, 1034, 973, 1053, 1053, 973, 999, 1043, 21, 0, 0, 0, 0, 0, 0, 0, 1050, 1011, 987, 1033, 1024, 1084, 1019, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1021, 1047, 972, 1026, 1013, 1004, 1041, 29, 2, 0, 0, 0, 0, 0, 0, 993, 1010, 943, 992, 985, 1042, 1048, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1057, 1035, 1040, 1013, 994, 1014, 1036, 37, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8318, 8130, 8302, 8145, 8223, 8163, 8054, 8201] avg=8192.0000 max=8318 max_rank=0 min=8054 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1071, 1049, 1038, 1023, 969, 1027, 1021, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 972, 959, 1010, 985, 1070, 1033, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1029, 977, 1060, 1049, 1084, 1031, 991, 10, 4, 0, 0, 0, 0, 0, 0, 1003, 1008, 1063, 1006, 1001, 1040, 1039, 1014, 25, 0, 0, 0, 0, 0, 0, 0, 1071, 1036, 1048, 1047, 1008, 982, 978, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1033, 1048, 1034, 1035, 1005, 1067, 1012, 31, 9, 0, 0, 0, 0, 0, 0, 1052, 1002, 1053, 972, 1012, 1058, 1031, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 980, 1001, 1030, 994, 1052, 1014, 971, 44, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8106, 8253, 8199, 8151, 8304, 8215, 8102] avg=8192.0000 max=8304 max_rank=5 min=8102 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8064, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8192, 8192, 8192, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8448, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8448, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1073, 1052, 1005, 1056, 1064, 1030, 924, 1018, 1019, 1014, 1055, 1014, 1037, 999, 968, 1002, 1009, 1036, 1049, 1038, 964, 1072, 1041, 997, 1016, 1019, 1053, 1008, 1015, 1006, 1030, 1018, 1059, 1059, 1025, 979, 1039, 1087, 1050, 1028, 984, 989, 1021, 1075, 1013, 999, 1067, 1037, 1020, 1011, 1014, 989, 1030, 997, 1019, 1039, 1048, 1024, 1016, 1018, 1035, 1027, 1037, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1033, 1026, 1003, 1018, 1095, 1018, 1001, 1047, 1013, 1019, 1018, 991, 959, 980, 1029, 1053, 1027, 1026, 1017, 1032, 1083, 1015, 967, 1001, 982, 995, 1000, 1005, 1015, 1027, 1077, 1019, 1034, 1064, 1045, 1057, 977, 1005, 1028, 988, 1048, 1051, 979, 1028, 1063, 1049, 997, 1062, 1061, 1015, 1044, 1034, 1028, 1012, 1041, 1003, 1108, 983, 1022, 999, 1040, 1037, 1027, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1059, 1015, 1024, 1029, 1001, 1046, 996, 979, 993, 1010, 1040, 1019, 1015, 1037, 1055, 1026, 1038, 1006, 1024, 1095, 1040, 992, 1044, 1040, 1021, 1056, 1043, 1036, 1040, 1023, 1002, 1024, 996, 1042, 1000, 1008, 1013, 1015, 1007, 1030, 1003, 966, 1111, 1030, 1051, 992, 1017, 1022, 1018, 1029, 1000, 1064, 1004, 1018, 993, 1033, 1012, 1013, 991, 1052, 1007, 1019, 1092, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1047, 1026, 1024, 1012, 1077, 1049, 1054, 1012, 965, 1002, 989, 1004, 1015, 1033, 986, 1003, 1015, 1057, 987, 1042, 1045, 1008, 1011, 1029, 1017, 1011, 983, 1039, 1040, 1019, 987, 1001, 1004, 1035, 1061, 1018, 1003, 1025, 986, 1084, 1015, 1038, 1026, 1057, 1055, 1038, 1027, 1038, 1004, 985, 1004, 1019, 995, 1018, 1071, 1031, 1066, 1051, 1021, 1047, 1061, 1009, 1058, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[977, 1024, 968, 1022, 1027, 1001, 1036, 1027, 999, 1017, 1025, 1028, 1037, 1009, 1018, 1046, 1026, 1046, 1006, 1041, 1023, 1034, 1037, 1049, 956, 1050, 1005, 979, 1050, 1013, 1079, 1064, 1017, 1062, 979, 1008, 1004, 973, 991, 1044, 1030, 1066, 1004, 1024, 970, 1056, 1028, 1012, 1056, 1003, 1108, 1059, 1021, 1003, 1064, 1038, 1034, 1031, 1058, 1002, 1037, 1012, 1010, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1033, 1061, 1014, 1035, 1005, 1064, 1036, 1030, 1032, 1045, 1041, 950, 1068, 998, 1078, 1001, 1060, 1001, 1047, 1029, 1023, 1024, 1019, 1039, 1039, 1025, 987, 1021, 1021, 1027, 987, 1047, 1041, 978, 1003, 972, 1024, 972, 1013, 1021, 1039, 1041, 1060, 989, 957, 1006, 1090, 1050, 1072, 1006, 1054, 1002, 1044, 1018, 1026, 1016, 1048, 971, 1002, 999, 1029, 1091, 973, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1025, 1053, 1059, 1032, 999, 1046, 1046, 969, 1001, 1023, 1033, 1026, 1014, 1078, 1025, 1037, 1048, 992, 1016, 1031, 1056, 1019, 986, 1009, 998, 1001, 1035, 1033, 1002, 1025, 1048, 1057, 1050, 1029, 997, 980, 1047, 1040, 1034, 1003, 1008, 1033, 1034, 1015, 1002, 996, 1027, 1018, 1011, 1044, 1039, 1036, 1047, 1010, 1031, 1054, 1013, 1061, 1019, 993, 1037, 986, 1036, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1013, 1018, 1054, 1002, 976, 1081, 1005, 1036, 1068, 1017, 1028, 1001, 1012, 1039, 1034, 1031, 1013, 1039, 1049, 1081, 1019, 1016, 1051, 1023, 994, 1042, 988, 1012, 1033, 1050, 1025, 999, 985, 1026, 990, 1081, 987, 1033, 1055, 1017, 1085, 1018, 1036, 1037, 1034, 992, 1007, 1011, 1026, 1041, 1062, 1007, 1008, 1020, 1041, 999, 972, 1014, 1024, 1030, 1031, 1002, 1029, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1059, 1015, 1024, 1029, 1001, 1028, 996, 979, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1010, 1040, 1019, 1015, 1037, 1055, 1026, 34, 6, 0, 0, 0, 0, 0, 0, 1038, 1006, 1024, 1061, 1040, 992, 1044, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1056, 1043, 1036, 1040, 1023, 1002, 1024, 17, 9, 0, 0, 0, 0, 0, 0, 996, 1042, 1000, 1008, 1013, 1015, 1007, 1030, 18, 11, 0, 0, 0, 0, 0, 0, 1003, 966, 1087, 1030, 1051, 992, 1017, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1029, 980, 1064, 1004, 1018, 993, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1013, 991, 1052, 1007, 1019, 1092, 1020, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8235, 8245, 8271, 8140, 8168, 8139, 8207] avg=8192.0000 max=8271 max_rank=3 min=8131 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32654 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1033, 1026, 1003, 1018, 1095, 1001, 1001, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1019, 1018, 991, 959, 980, 1029, 1053, 32, 6, 0, 0, 0, 0, 0, 0, 1027, 1026, 1017, 1000, 1083, 1015, 967, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 982, 995, 1000, 1005, 1015, 1027, 1077, 1019, 16, 9, 0, 0, 0, 0, 0, 0, 1034, 1064, 1045, 1057, 977, 1005, 1028, 988, 15, 11, 0, 0, 0, 0, 0, 0, 1048, 1051, 958, 1028, 1063, 1049, 997, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1015, 1024, 1034, 1028, 1012, 1041, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 983, 1022, 999, 1040, 1037, 1027, 1016, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8100, 8136, 8145, 8224, 8256, 8218, 8233] avg=8192.0000 max=8256 max_rank=5 min=8100 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32931 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1073, 1052, 1005, 1056, 1064, 1012, 924, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1014, 1055, 1014, 1037, 999, 968, 1002, 32, 6, 0, 0, 0, 0, 0, 0, 1009, 1036, 1049, 1006, 964, 1072, 1041, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1019, 1053, 1008, 1015, 1006, 1030, 1018, 17, 9, 0, 0, 0, 0, 0, 0, 1059, 1059, 1025, 979, 1039, 1087, 1050, 1028, 16, 11, 0, 0, 0, 0, 0, 0, 984, 989, 999, 1075, 1013, 999, 1067, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1011, 994, 989, 1030, 997, 1019, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1024, 1016, 1018, 1035, 1027, 1037, 1000, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8146, 8174, 8191, 8353, 8163, 8099, 8206] avg=8192.0000 max=8353 max_rank=4 min=8099 min_rank=6 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1047, 1026, 1024, 1012, 1077, 1031, 1054, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 965, 1002, 989, 1004, 1015, 1033, 986, 1003, 32, 6, 0, 0, 0, 0, 0, 0, 1015, 1057, 987, 1010, 1045, 1008, 1011, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1011, 983, 1039, 1040, 1019, 987, 1001, 17, 9, 0, 0, 0, 0, 0, 0, 1004, 1035, 1061, 1018, 1003, 1025, 986, 1084, 16, 11, 0, 0, 0, 0, 0, 0, 1015, 1038, 1004, 1057, 1055, 1038, 1027, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 985, 984, 1019, 995, 1018, 1071, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1051, 1021, 1047, 1061, 1009, 1058, 997, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8035, 8162, 8123, 8243, 8272, 8107, 8311] avg=8192.0000 max=8311 max_rank=7 min=8035 min_rank=1 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[977, 1024, 968, 1022, 1027, 984, 1036, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1017, 1025, 1028, 1037, 1009, 1018, 1046, 32, 6, 0, 0, 0, 0, 0, 0, 1026, 1046, 1006, 1009, 1023, 1034, 1037, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1050, 1005, 979, 1050, 1013, 1079, 1064, 16, 10, 0, 0, 0, 0, 0, 0, 1017, 1062, 979, 1008, 1004, 973, 991, 1044, 16, 12, 0, 0, 0, 0, 0, 0, 1030, 1066, 982, 1024, 970, 1056, 1028, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1003, 1086, 1059, 1021, 1003, 1064, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1031, 1058, 1002, 1037, 1012, 1010, 1013, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8065, 8217, 8230, 8222, 8106, 8168, 8330, 8198] avg=8192.0000 max=8330 max_rank=6 min=8065 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1025, 1053, 1059, 1032, 999, 1028, 1046, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1023, 1033, 1026, 1014, 1078, 1025, 1037, 32, 6, 0, 0, 0, 0, 0, 0, 1048, 992, 1016, 999, 1056, 1019, 986, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1001, 1035, 1033, 1002, 1025, 1048, 1057, 17, 9, 0, 0, 0, 0, 0, 0, 1050, 1029, 997, 980, 1047, 1040, 1034, 1003, 16, 11, 0, 0, 0, 0, 0, 0, 1008, 1033, 1012, 1015, 1002, 996, 1027, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1044, 1019, 1036, 1047, 1010, 1031, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1061, 1019, 993, 1037, 986, 1036, 984, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8275, 8125, 8225, 8207, 8111, 8252, 8130] avg=8192.0000 max=8275 max_rank=1 min=8111 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1033, 1061, 1014, 1035, 1005, 1046, 1036, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1045, 1041, 950, 1068, 998, 1078, 1001, 32, 6, 0, 0, 0, 0, 0, 0, 1060, 1001, 1047, 997, 1023, 1024, 1019, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1025, 987, 1021, 1021, 1027, 987, 1047, 17, 10, 0, 0, 0, 0, 0, 0, 1041, 978, 1003, 972, 1024, 972, 1013, 1021, 17, 11, 0, 0, 0, 0, 0, 0, 1039, 1041, 1037, 989, 957, 1006, 1090, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1006, 1033, 1002, 1044, 1018, 1026, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 971, 1002, 999, 1029, 1091, 973, 1042, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8251, 8210, 8181, 8052, 8209, 8217, 8156] avg=8192.0000 max=8260 max_rank=0 min=8052 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1013, 1018, 1054, 1002, 976, 1054, 1005, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1017, 1028, 1001, 1012, 1039, 1034, 1031, 37, 10, 0, 0, 0, 0, 0, 0, 1013, 1039, 1049, 1044, 1019, 1016, 1051, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1042, 988, 1012, 1033, 1050, 1025, 999, 21, 14, 0, 0, 0, 0, 0, 0, 985, 1026, 990, 1081, 987, 1033, 1055, 1017, 21, 16, 0, 0, 0, 0, 0, 0, 1085, 1018, 1005, 1037, 1034, 992, 1007, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1041, 1032, 1007, 1008, 1020, 1041, 999, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1014, 1024, 1030, 1031, 1002, 1029, 987, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8277, 8254, 8178, 8211, 8189, 8174, 8095] avg=8192.0000 max=8277 max_rank=1 min=8095 min_rank=7 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8064, 8192, 8192, 8320, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[984, 1047, 1048, 1016, 1062, 1005, 1015, 1042, 972, 1014, 1057, 1053, 1063, 1037, 1045, 1028, 997, 1001, 1031, 1027, 1044, 1005, 1017, 1042, 1056, 1048, 971, 1010, 982, 1068, 1027, 1094, 984, 1089, 995, 1016, 1008, 1053, 1010, 1038, 1008, 995, 1025, 1024, 1002, 1048, 1029, 1036, 1046, 1037, 1017, 1025, 1029, 988, 1026, 1043, 999, 1022, 997, 1024, 1012, 988, 1003, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1016, 1022, 1014, 1056, 1048, 994, 1049, 1019, 997, 1014, 1016, 1030, 1060, 1031, 1051, 1042, 1037, 1009, 1034, 1032, 1002, 1014, 1015, 996, 1024, 1015, 996, 1033, 1019, 1026, 1068, 999, 1055, 1052, 1000, 994, 1042, 1078, 1057, 1015, 1031, 1057, 1017, 1036, 1067, 1034, 1038, 973, 1003, 1019, 1017, 1042, 1036, 988, 995, 1019, 1056, 1011, 1006, 991, 1000, 959, 1037, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[993, 1090, 1028, 1013, 1023, 1071, 964, 1015, 1033, 1040, 1079, 1023, 1072, 963, 930, 989, 1045, 1029, 1036, 1031, 1012, 1082, 1010, 1021, 995, 1003, 1020, 958, 1045, 973, 1037, 1084, 1035, 999, 1029, 1023, 1028, 1096, 1034, 1011, 997, 960, 1089, 1027, 1069, 1032, 966, 1008, 1070, 1036, 1036, 1030, 1060, 1029, 1064, 987, 1016, 1058, 981, 992, 1018, 991, 1047, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1015, 1088, 973, 984, 1007, 972, 1063, 1033, 1044, 1015, 1022, 1082, 1037, 1012, 1061, 1026, 994, 994, 1024, 1020, 1019, 1040, 1002, 988, 1040, 1007, 1079, 1000, 1021, 1084, 1009, 1042, 1036, 988, 1017, 1033, 1048, 1085, 1048, 1045, 991, 1054, 1054, 1027, 1049, 989, 975, 1059, 1015, 1000, 1057, 1023, 965, 956, 1021, 1041, 1012, 1015, 1021, 1003, 1012, 1034, 1047, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1021, 1031, 990, 1015, 990, 1027, 1036, 1075, 1027, 987, 1048, 1028, 989, 1016, 1067, 1039, 992, 991, 1035, 1028, 1024, 980, 1047, 1017, 1067, 1087, 985, 995, 1009, 1090, 1000, 1020, 1015, 1039, 1029, 1026, 999, 1010, 1014, 1010, 1011, 1070, 1059, 1005, 1025, 1038, 1021, 1008, 1018, 1007, 1046, 1042, 995, 1012, 1011, 1050, 1037, 1049, 1045, 1027, 995, 1036, 995, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1041, 1029, 1045, 1021, 977, 1047, 1031, 986, 1043, 1053, 976, 1018, 1029, 993, 1014, 1002, 1022, 1039, 1003, 1046, 1018, 1003, 1041, 993, 1026, 1046, 975, 984, 1012, 1020, 1052, 1037, 1004, 1005, 1089, 1071, 1016, 1056, 1013, 1032, 1023, 1027, 1006, 1016, 1007, 1037, 1032, 1070, 1056, 1005, 958, 987, 983, 1055, 1035, 1002, 1106, 1081, 1002, 1040, 1020, 1025, 1060, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1041, 1013, 1015, 1016, 997, 990, 1055, 1015, 1026, 1031, 939, 1082, 1010, 1008, 1034, 1033, 1032, 941, 1093, 1002, 1025, 1038, 1005, 1008, 1043, 1045, 1024, 1056, 1035, 1049, 1002, 1063, 1006, 1004, 1026, 997, 984, 1012, 1072, 983, 1012, 1071, 1034, 1030, 1026, 1013, 1040, 1019, 1060, 996, 990, 1032, 1036, 1015, 1048, 1064, 1074, 1039, 1033, 1008, 1036, 978, 992, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1033, 990, 1008, 960, 1016, 1032, 1037, 1042, 981, 1047, 1017, 1053, 1016, 1001, 1054, 974, 1041, 998, 1068, 1017, 1031, 984, 1029, 1048, 1001, 1038, 997, 1041, 1045, 999, 1042, 963, 1098, 1042, 1004, 982, 1032, 1028, 1015, 1058, 1020, 1043, 1040, 1004, 1022, 1087, 1028, 1015, 1033, 1010, 1024, 1057, 1012, 1051, 1042, 1050, 1059, 1024, 967, 1035, 1012, 997, 1005, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[984, 1047, 1048, 1016, 1062, 1005, 1015, 1042, 18, 0, 0, 0, 0, 0, 0, 0, 972, 1014, 1057, 1049, 1063, 1037, 1045, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1001, 1031, 1027, 1044, 1005, 1017, 1042, 30, 0, 0, 0, 0, 0, 0, 0, 1056, 1048, 971, 1010, 982, 1050, 1027, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1089, 995, 1016, 1008, 1020, 1010, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 995, 1002, 1024, 1002, 1048, 1029, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1037, 1017, 1025, 1029, 988, 1026, 1043, 4, 0, 0, 0, 0, 0, 0, 0, 999, 1022, 997, 1024, 1012, 988, 1003, 1012, 23, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8265, 8194, 8238, 8160, 8144, 8215, 8083] avg=8192.0000 max=8265 max_rank=1 min=8083 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32602 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1016, 1022, 1014, 1056, 1048, 994, 1049, 1019, 17, 0, 0, 0, 0, 0, 0, 0, 997, 1014, 1016, 1026, 1060, 1031, 1051, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1009, 1034, 1032, 1002, 1014, 1015, 996, 31, 0, 0, 0, 0, 0, 0, 0, 1024, 1015, 996, 1033, 1019, 1009, 1068, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1052, 1000, 994, 1042, 1044, 1057, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1057, 994, 1036, 1067, 1034, 1038, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1019, 1017, 1042, 1036, 988, 995, 1019, 4, 0, 0, 0, 0, 0, 0, 0, 1056, 1011, 1006, 991, 1000, 959, 1037, 1033, 23, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8237, 8170, 8163, 8259, 8230, 8123, 8119] avg=8192.0000 max=8259 max_rank=4 min=8119 min_rank=7 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1015, 1088, 973, 984, 1007, 972, 1063, 1033, 18, 0, 0, 0, 0, 0, 0, 0, 1044, 1015, 1022, 1078, 1037, 1012, 1061, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 994, 994, 1024, 1020, 1019, 1040, 1002, 988, 31, 0, 0, 0, 0, 0, 0, 0, 1040, 1007, 1079, 1000, 1021, 1066, 1009, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 988, 1017, 1033, 1048, 1051, 1048, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1054, 1031, 1027, 1049, 989, 975, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1000, 1057, 1023, 965, 956, 1021, 1041, 4, 0, 0, 0, 0, 0, 0, 0, 1012, 1015, 1021, 1003, 1012, 1034, 1047, 1019, 23, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8295, 8112, 8264, 8266, 8175, 8082, 8189] avg=8192.0000 max=8295 max_rank=1 min=8082 min_rank=6 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1041, 1029, 1045, 1021, 977, 1047, 1031, 986, 17, 0, 0, 0, 0, 0, 0, 0, 1043, 1053, 976, 1014, 1029, 993, 1014, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1039, 1003, 1046, 1018, 1003, 1041, 993, 30, 0, 0, 0, 0, 0, 0, 0, 1026, 1046, 975, 984, 1012, 1003, 1052, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1005, 1089, 1071, 1016, 1023, 1013, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1027, 984, 1016, 1007, 1037, 1032, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1005, 958, 987, 983, 1055, 1035, 1002, 4, 0, 0, 0, 0, 0, 0, 0, 1106, 1081, 1002, 1040, 1020, 1025, 1060, 995, 22, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8124, 8195, 8135, 8253, 8196, 8085, 8354] avg=8192.0000 max=8354 max_rank=7 min=8085 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1021, 1031, 990, 1015, 990, 1027, 1036, 1075, 18, 0, 0, 0, 0, 0, 0, 0, 1027, 987, 1048, 1024, 989, 1016, 1067, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 992, 991, 1035, 1028, 1024, 980, 1047, 1017, 29, 0, 0, 0, 0, 0, 0, 0, 1067, 1087, 985, 995, 1009, 1072, 1000, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1039, 1029, 1026, 999, 978, 1014, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1070, 1035, 1005, 1025, 1038, 1021, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1007, 1046, 1042, 995, 1012, 1011, 1050, 4, 0, 0, 0, 0, 0, 0, 0, 1037, 1049, 1045, 1027, 995, 1036, 995, 1039, 24, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8197, 8143, 8235, 8110, 8213, 8185, 8250] avg=8192.0000 max=8250 max_rank=7 min=8110 min_rank=4 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[993, 1090, 1028, 1013, 1023, 1071, 964, 1015, 16, 0, 0, 0, 0, 0, 0, 0, 1033, 1040, 1079, 1019, 1072, 963, 930, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1029, 1036, 1031, 1012, 1082, 1010, 1021, 32, 0, 0, 0, 0, 0, 0, 0, 995, 1003, 1020, 958, 1045, 957, 1037, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 999, 1029, 1023, 1028, 1061, 1034, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 997, 960, 1065, 1027, 1069, 1032, 966, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1036, 1036, 1030, 1060, 1029, 1064, 987, 4, 0, 0, 0, 0, 0, 0, 0, 1016, 1058, 981, 992, 1018, 991, 1047, 1011, 24, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8125, 8298, 8099, 8220, 8124, 8316, 8141] avg=8192.0000 max=8316 max_rank=6 min=8099 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1041, 1013, 1015, 1016, 997, 990, 1055, 1015, 17, 0, 0, 0, 0, 0, 0, 0, 1026, 1031, 939, 1078, 1010, 1008, 1034, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 941, 1093, 1002, 1025, 1038, 1005, 1008, 29, 0, 0, 0, 0, 0, 0, 0, 1043, 1045, 1024, 1056, 1035, 1032, 1002, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1004, 1026, 997, 984, 980, 1072, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1071, 1011, 1030, 1026, 1013, 1040, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 996, 990, 1032, 1036, 1015, 1048, 1064, 4, 0, 0, 0, 0, 0, 0, 0, 1074, 1039, 1033, 1008, 1036, 978, 992, 1040, 23, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8159, 8173, 8300, 8052, 8222, 8245, 8226] avg=8192.0000 max=8300 max_rank=3 min=8052 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1033, 990, 1008, 960, 1016, 1032, 1037, 1042, 21, 3, 0, 0, 0, 0, 0, 0, 981, 1047, 1017, 1044, 1016, 1001, 1054, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 998, 1068, 1017, 1031, 984, 1029, 1048, 35, 0, 0, 0, 0, 0, 0, 0, 1001, 1038, 997, 1041, 1045, 975, 1042, 963, 0, 0, 0, 0, 0, 0, 0, 0, 1098, 1042, 1004, 982, 1032, 985, 1015, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1043, 1013, 1004, 1022, 1087, 1028, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1010, 1024, 1057, 1012, 1051, 1042, 1050, 6, 0, 0, 0, 0, 0, 0, 0, 1059, 1024, 967, 1035, 1012, 997, 1005, 1037, 27, 8, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8134, 8251, 8102, 8216, 8232, 8285, 8174] avg=8192.0000 max=8285 max_rank=6 min=8102 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8448, 8320, 8064, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8448, 8320, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8320, 8064, 8192, 8192, 8064, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8320, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1028, 976, 1047, 1014, 1025, 1033, 998, 1031, 1038, 998, 1019, 1054, 1043, 1067, 993, 1065, 964, 1027, 1039, 1027, 1082, 993, 1039, 1030, 1044, 1033, 1023, 1012, 1044, 993, 986, 1070, 1047, 1042, 1040, 1006, 1072, 1023, 1027, 989, 1004, 1025, 1023, 1013, 1021, 1014, 1056, 1024, 1032, 1057, 1005, 1008, 1009, 1020, 1073, 1023, 959, 1025, 995, 1045, 1017, 1021, 1012, 974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1051, 977, 1037, 1039, 1032, 1069, 1070, 979, 1012, 1015, 1070, 1085, 1022, 1055, 987, 1034, 1058, 1014, 1031, 1009, 1039, 1034, 1057, 1036, 1012, 991, 1011, 1026, 1004, 1042, 1032, 1011, 1000, 1027, 1037, 1051, 1030, 1039, 982, 1058, 1006, 1004, 1045, 1022, 1045, 1021, 1027, 1028, 1010, 985, 1033, 986, 1021, 1013, 1035, 1010, 1006, 1014, 1019, 1057, 996, 1025, 966, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1034, 1046, 1046, 1012, 1008, 1007, 1028, 1056, 1045, 1014, 990, 991, 981, 1044, 1053, 1044, 1013, 1030, 1094, 1037, 985, 1041, 1037, 1063, 991, 1098, 1021, 1025, 1013, 1028, 1038, 996, 989, 1020, 1032, 1017, 1028, 1037, 1072, 1030, 1046, 1000, 1008, 1016, 1047, 1021, 967, 1027, 1055, 1041, 1033, 1037, 1028, 1020, 1009, 1028, 983, 964, 997, 1046, 979, 1000, 1055, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1048, 1026, 1045, 1050, 993, 1069, 968, 1019, 1023, 1017, 1018, 1017, 1020, 994, 1023, 992, 1049, 1063, 1018, 999, 1012, 985, 1023, 1049, 1080, 999, 1029, 1009, 1026, 1024, 1073, 1059, 986, 1011, 1007, 1016, 1054, 999, 974, 991, 1015, 997, 1037, 1039, 1025, 1051, 1018, 995, 984, 1017, 1027, 1056, 1016, 1032, 1022, 1065, 1088, 990, 1049, 1016, 1016, 1009, 1067, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[972, 1026, 1027, 1033, 988, 1062, 965, 1025, 983, 977, 1041, 1010, 1009, 976, 1000, 1051, 1049, 993, 1081, 1025, 997, 1047, 1033, 1029, 1042, 1028, 1064, 977, 1046, 1052, 1046, 1024, 995, 1022, 992, 1053, 1087, 1018, 1058, 1039, 983, 1058, 1011, 1031, 1007, 1085, 1022, 1047, 1027, 1058, 1022, 1020, 1028, 1054, 1016, 1070, 998, 1015, 969, 1053, 982, 976, 1027, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[993, 1004, 1009, 1043, 1064, 1040, 1036, 1059, 1040, 1029, 1032, 1049, 966, 991, 1014, 1058, 1039, 1052, 1068, 1084, 993, 1031, 996, 1077, 1010, 1017, 992, 992, 1022, 1054, 1012, 1016, 1001, 1037, 975, 1015, 1020, 977, 1021, 992, 1059, 1015, 1056, 1035, 1020, 1031, 1004, 1072, 1022, 1035, 994, 1044, 999, 1035, 1015, 1074, 1015, 1062, 956, 1008, 995, 1008, 1043, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1070, 1043, 986, 1013, 1023, 1030, 1030, 976, 1048, 1056, 1025, 1016, 993, 1010, 1065, 1047, 1050, 1052, 1024, 1037, 1026, 1031, 971, 1058, 995, 1014, 989, 1035, 994, 1008, 1046, 1019, 1074, 1027, 1068, 962, 1028, 1072, 994, 1016, 1038, 1015, 1015, 1030, 1016, 993, 1049, 1026, 1003, 991, 1014, 990, 1022, 1037, 1032, 946, 1029, 1038, 1023, 1066, 1056, 1020, 1047, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[999, 1072, 1043, 1034, 1011, 1026, 988, 969, 966, 1008, 990, 992, 1005, 1038, 1051, 1062, 1000, 1027, 995, 1053, 1039, 1035, 1038, 1011, 1054, 1008, 1025, 1045, 1034, 1014, 982, 1049, 1064, 1084, 1029, 1040, 1030, 998, 1027, 993, 1063, 1025, 1051, 1056, 1035, 1067, 1016, 1027, 986, 1010, 1018, 1020, 1033, 1016, 1006, 1012, 1011, 1077, 1002, 1025, 985, 1040, 1008, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1051, 977, 1037, 1039, 1032, 1069, 1070, 979, 1, 0, 0, 0, 0, 0, 0, 0, 1012, 1015, 1070, 1085, 1022, 1055, 987, 1034, 13, 0, 0, 0, 0, 0, 0, 0, 1058, 1014, 1031, 1009, 1039, 1034, 1057, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 991, 1011, 1026, 1004, 1042, 1032, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1027, 1037, 1051, 1030, 1039, 982, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1004, 1045, 1022, 1045, 997, 1027, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 985, 1033, 986, 1021, 1013, 1035, 1010, 11, 0, 0, 0, 0, 0, 0, 0, 1006, 1014, 1019, 1057, 996, 1025, 966, 997, 59, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8293, 8219, 8128, 8224, 8174, 8104, 8139] avg=8192.0000 max=8293 max_rank=1 min=8104 min_rank=6 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1034, 1046, 1046, 1012, 1008, 1007, 1028, 1056, 1, 0, 0, 0, 0, 0, 0, 0, 1045, 1014, 990, 991, 981, 1044, 1053, 1044, 13, 0, 0, 0, 0, 0, 0, 0, 1013, 1030, 1094, 1037, 985, 1041, 1037, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1098, 1021, 1025, 1013, 1028, 1038, 995, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1020, 1032, 1017, 1028, 1037, 1072, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1000, 1008, 1016, 1047, 997, 967, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1041, 1033, 1037, 1028, 1020, 1009, 1028, 11, 0, 0, 0, 0, 0, 0, 0, 983, 964, 997, 1046, 979, 1000, 1055, 995, 61, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8175, 8239, 8209, 8225, 8108, 8262, 8080] avg=8192.0000 max=8262 max_rank=6 min=8080 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32675 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[972, 1026, 1027, 1033, 988, 1062, 965, 1025, 1, 0, 0, 0, 0, 0, 0, 0, 983, 977, 1041, 1010, 1009, 976, 1000, 1051, 14, 0, 0, 0, 0, 0, 0, 0, 1049, 993, 1081, 1025, 997, 1047, 1033, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1028, 1064, 977, 1046, 1052, 1046, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1022, 992, 1053, 1087, 1018, 1058, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1058, 1011, 1031, 1007, 1059, 1022, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1058, 1022, 1020, 1028, 1054, 1016, 1070, 12, 0, 0, 0, 0, 0, 0, 0, 998, 1015, 969, 1053, 982, 976, 1027, 1035, 59, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8061, 8195, 8278, 8264, 8218, 8307, 8114] avg=8192.0000 max=8307 max_rank=6 min=8061 min_rank=1 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1028, 976, 1047, 1014, 1025, 1033, 998, 1031, 1, 0, 0, 0, 0, 0, 0, 0, 1038, 998, 1019, 1054, 1043, 1067, 993, 1065, 13, 0, 0, 0, 0, 0, 0, 0, 964, 1027, 1039, 1027, 1082, 993, 1039, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1033, 1023, 1012, 1044, 993, 986, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1042, 1040, 1006, 1072, 1023, 1027, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1025, 1023, 1013, 1021, 990, 1056, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1057, 1005, 1008, 1009, 1020, 1073, 1023, 11, 0, 0, 0, 0, 0, 0, 0, 959, 1025, 995, 1045, 1017, 1021, 1012, 974, 59, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8290, 8142, 8204, 8246, 8156, 8238, 8107] avg=8192.0000 max=8290 max_rank=1 min=8107 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1048, 1026, 1045, 1050, 993, 1069, 968, 1019, 1, 0, 0, 0, 0, 0, 0, 0, 1023, 1017, 1018, 1017, 1020, 994, 1023, 992, 14, 0, 0, 0, 0, 0, 0, 0, 1049, 1063, 1018, 999, 1012, 985, 1023, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 999, 1029, 1009, 1026, 1024, 1073, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1011, 1007, 1016, 1054, 999, 974, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 997, 1037, 1039, 1025, 1025, 1018, 995, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1017, 1027, 1056, 1016, 1032, 1022, 1065, 12, 0, 0, 0, 0, 0, 0, 0, 1088, 990, 1049, 1016, 1016, 1009, 1067, 1048, 60, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8118, 8138, 8298, 8038, 8151, 8231, 8343] avg=8192.0000 max=8343 max_rank=7 min=8038 min_rank=4 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1070, 1043, 986, 1013, 1023, 1030, 1030, 976, 1, 0, 0, 0, 0, 0, 0, 0, 1048, 1056, 1025, 1016, 993, 1010, 1065, 1047, 13, 0, 0, 0, 0, 0, 0, 0, 1050, 1052, 1024, 1037, 1026, 1031, 971, 998, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1014, 989, 1035, 994, 1008, 1046, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1027, 1068, 962, 1028, 1072, 994, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1015, 1015, 1030, 1016, 969, 1049, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 991, 1014, 990, 1022, 1037, 1032, 946, 11, 0, 0, 0, 0, 0, 0, 0, 1029, 1038, 1023, 1066, 1056, 1020, 1047, 1019, 60, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8273, 8189, 8099, 8241, 8158, 8046, 8358] avg=8192.0000 max=8358 max_rank=7 min=8046 min_rank=6 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[993, 1004, 1009, 1043, 1064, 1040, 1036, 1059, 1, 0, 0, 0, 0, 0, 0, 0, 1040, 1029, 1032, 1049, 966, 991, 1014, 1058, 14, 0, 0, 0, 0, 0, 0, 0, 1039, 1052, 1068, 1084, 993, 1031, 996, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1017, 992, 992, 1022, 1054, 1012, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1037, 975, 1015, 1020, 977, 1021, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1015, 1056, 1035, 1020, 1006, 1004, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1035, 994, 1044, 999, 1035, 1015, 1074, 11, 0, 0, 0, 0, 0, 0, 0, 1015, 1062, 956, 1008, 995, 1008, 1043, 1019, 61, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8193, 8279, 8114, 8038, 8267, 8229, 8167] avg=8192.0000 max=8279 max_rank=2 min=8038 min_rank=4 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[999, 1072, 1043, 1034, 1011, 1026, 988, 969, 4, 5, 0, 0, 0, 0, 0, 0, 966, 1008, 990, 992, 1005, 1038, 1051, 1062, 19, 2, 0, 0, 0, 0, 0, 0, 1000, 1027, 995, 1053, 1039, 1035, 1038, 948, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1008, 1025, 1045, 1034, 1014, 982, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1084, 1029, 1040, 1025, 998, 1027, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1025, 1051, 1056, 1035, 1031, 1016, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1010, 1018, 1020, 1033, 1016, 1006, 1012, 17, 1, 0, 0, 0, 0, 0, 0, 1011, 1077, 1002, 1025, 985, 1040, 1008, 1019, 61, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8133, 8135, 8206, 8260, 8304, 8119, 8228] avg=8192.0000 max=8304 max_rank=5 min=8119 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32911 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8448, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8448, 8320, 8192, 8320, 8320, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8448, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8064, 8192, 8192, 8448, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8064, 8320, 8064, 8192, 8320, 8192, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1066, 1019, 1031, 1057, 1001, 1002, 1011, 1042, 1036, 1043, 1033, 1000, 1032, 1017, 1016, 988, 1044, 1070, 1006, 1024, 1037, 986, 1023, 1039, 1013, 1003, 1019, 1000, 974, 1025, 1000, 1027, 997, 1049, 1022, 1048, 1010, 1000, 1062, 1018, 998, 1018, 1016, 1013, 1004, 1047, 978, 1072, 1005, 1016, 1041, 1005, 1056, 1062, 1067, 1015, 1000, 1015, 1030, 1012, 1045, 1048, 1043, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[961, 1007, 1016, 1035, 1068, 1031, 1040, 987, 994, 1039, 1009, 1014, 1010, 1032, 1061, 1029, 1081, 1003, 1040, 1044, 1037, 993, 1006, 998, 1062, 1049, 1020, 1024, 1026, 1005, 998, 1023, 1064, 1044, 1035, 973, 1005, 1008, 1066, 992, 1020, 1034, 982, 1001, 1040, 1046, 1039, 990, 997, 1066, 1031, 1020, 982, 996, 1081, 1032, 1060, 986, 996, 1068, 1075, 999, 1032, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1013, 1038, 1056, 1012, 1041, 1015, 1072, 987, 1008, 1030, 1035, 1015, 1041, 1037, 1019, 973, 1016, 1003, 1044, 1027, 997, 973, 1012, 1026, 1019, 1006, 1001, 1031, 1078, 1057, 1050, 1031, 1042, 1035, 1022, 995, 1003, 1012, 1061, 1015, 1003, 978, 1055, 1058, 1030, 1018, 999, 1083, 1029, 987, 1031, 1033, 999, 1072, 1003, 980, 1022, 1039, 1013, 1019, 1014, 1031, 1036, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[985, 1008, 956, 1023, 1047, 1058, 1038, 1060, 1010, 1016, 1041, 1023, 1022, 1005, 1068, 1061, 1044, 990, 1027, 1054, 1000, 1028, 993, 1041, 1016, 1043, 1055, 1014, 1026, 1026, 1038, 1050, 1043, 1039, 1008, 1042, 1041, 1012, 1016, 1016, 1010, 1029, 994, 1033, 1021, 1007, 1087, 1003, 1011, 1024, 1038, 1010, 1109, 1029, 968, 1000, 1017, 1009, 1021, 1015, 989, 1033, 994, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1052, 1047, 989, 1024, 1060, 988, 1074, 1067, 1012, 1065, 1012, 937, 1041, 1024, 974, 975, 1038, 1104, 1022, 974, 1003, 1021, 1020, 1032, 1057, 1005, 1017, 981, 1042, 1004, 1023, 1034, 1027, 1031, 1032, 1056, 1012, 1050, 1022, 1013, 1004, 1018, 957, 1050, 1000, 1041, 1057, 1035, 1001, 1027, 1046, 1010, 1036, 1040, 1038, 1021, 1035, 1015, 955, 1015, 1151, 994, 1016, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1050, 1034, 1009, 992, 1035, 1042, 1015, 1033, 1082, 1054, 1037, 1022, 994, 1017, 1003, 1005, 972, 986, 1042, 1030, 1002, 1092, 1029, 1026, 1019, 1004, 1037, 1018, 1027, 1040, 1031, 1038, 1012, 1019, 1017, 1033, 999, 999, 1040, 1061, 977, 1010, 1009, 1041, 1083, 986, 1071, 1015, 1025, 1005, 1000, 1026, 1034, 1038, 1036, 991, 1074, 1028, 1040, 1005, 1016, 1008, 996, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1013, 970, 1021, 1024, 1034, 1021, 1037, 965, 1001, 1015, 1035, 1060, 990, 1029, 1026, 1077, 1006, 1031, 1039, 1069, 999, 1044, 1053, 1070, 1027, 997, 988, 1030, 983, 993, 1044, 996, 1037, 952, 1026, 1005, 999, 1090, 1018, 1075, 1082, 1015, 1037, 999, 1073, 1019, 1071, 1038, 1033, 997, 974, 1043, 1007, 1026, 1036, 1071, 1039, 1058, 989, 968, 981, 1041, 1018, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1007, 1018, 992, 1017, 979, 993, 1017, 1040, 1033, 1019, 1002, 1026, 1008, 961, 1065, 1054, 1011, 1001, 999, 1048, 1022, 1050, 1040, 1032, 968, 1122, 1003, 1056, 1054, 963, 1006, 1003, 998, 1032, 1120, 1015, 1006, 1073, 1064, 1030, 1012, 1040, 1056, 1004, 1022, 1059, 1040, 986, 1023, 999, 1006, 976, 1047, 995, 1044, 1008, 1054, 1034, 1006, 1012, 1095, 1020, 1048, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1066, 1019, 1031, 1057, 1001, 1002, 1011, 1042, 7, 4, 0, 0, 0, 0, 0, 0, 1036, 1043, 1033, 1000, 1032, 1017, 1016, 988, 15, 0, 0, 0, 0, 0, 0, 0, 1044, 1070, 1006, 1015, 1037, 986, 1023, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1003, 1019, 1000, 974, 1025, 1000, 1027, 12, 2, 0, 0, 0, 0, 0, 0, 997, 1049, 1022, 1048, 1010, 1000, 1035, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1018, 1016, 1013, 1004, 1047, 970, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1016, 1041, 1005, 1056, 1062, 1067, 1015, 5, 4, 0, 0, 0, 0, 0, 0, 1000, 1015, 1030, 1012, 1040, 1048, 1043, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8180, 8220, 8075, 8179, 8138, 8276, 8228] avg=8192.0000 max=8276 max_rank=6 min=8075 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[961, 1007, 1016, 1035, 1068, 1031, 1040, 987, 7, 4, 0, 0, 0, 0, 0, 0, 994, 1039, 1009, 1014, 1010, 1032, 1061, 1029, 15, 0, 0, 0, 0, 0, 0, 0, 1081, 1003, 1040, 1035, 1037, 993, 1006, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1049, 1020, 1024, 1026, 1005, 998, 1023, 12, 2, 0, 0, 0, 0, 0, 0, 1064, 1044, 1035, 973, 1005, 1008, 1039, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1034, 982, 1001, 1040, 1046, 1030, 990, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1066, 1031, 1020, 982, 996, 1081, 1032, 5, 5, 0, 0, 0, 0, 0, 0, 1060, 986, 996, 1068, 1070, 999, 1032, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8203, 8193, 8221, 8160, 8143, 8215, 8245] avg=8192.0000 max=8245 max_rank=7 min=8143 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1050, 1034, 1009, 992, 1035, 1042, 1015, 1033, 7, 4, 0, 0, 0, 0, 0, 0, 1082, 1054, 1037, 1022, 994, 1017, 1003, 1005, 15, 0, 0, 0, 0, 0, 0, 0, 972, 986, 1042, 1021, 1002, 1092, 1029, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1004, 1037, 1018, 1027, 1040, 1031, 1038, 12, 2, 0, 0, 0, 0, 0, 0, 1012, 1019, 1017, 1033, 999, 999, 1013, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1010, 1009, 1041, 1083, 986, 1062, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1005, 1000, 1026, 1034, 1038, 1036, 991, 4, 5, 0, 0, 0, 0, 0, 0, 1074, 1028, 1040, 1005, 1012, 1008, 996, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8229, 8170, 8228, 8153, 8183, 8164, 8188] avg=8192.0000 max=8229 max_rank=1 min=8153 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1013, 1038, 1056, 1012, 1041, 1015, 1072, 987, 7, 4, 0, 0, 0, 0, 0, 0, 1008, 1030, 1035, 1015, 1041, 1037, 1019, 973, 15, 0, 0, 0, 0, 0, 0, 0, 1016, 1003, 1044, 1018, 997, 973, 1012, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1006, 1001, 1031, 1078, 1057, 1050, 1031, 12, 2, 0, 0, 0, 0, 0, 0, 1042, 1035, 1022, 995, 1003, 1012, 1034, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 978, 1055, 1058, 1030, 1018, 991, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 987, 1031, 1033, 999, 1072, 1003, 980, 4, 4, 0, 0, 0, 0, 0, 0, 1022, 1039, 1013, 1019, 1010, 1031, 1036, 1056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8173, 8089, 8287, 8158, 8216, 8142, 8226] avg=8192.0000 max=8287 max_rank=3 min=8089 min_rank=2 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[985, 1008, 956, 1023, 1047, 1058, 1038, 1060, 7, 4, 0, 0, 0, 0, 0, 0, 1010, 1016, 1041, 1023, 1022, 1005, 1068, 1061, 15, 0, 0, 0, 0, 0, 0, 0, 1044, 990, 1027, 1045, 1000, 1028, 993, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1043, 1055, 1014, 1026, 1026, 1038, 1050, 11, 2, 0, 0, 0, 0, 0, 0, 1043, 1039, 1008, 1042, 1041, 1012, 990, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1029, 994, 1033, 1021, 1007, 1078, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1024, 1038, 1010, 1109, 1029, 968, 1000, 4, 5, 0, 0, 0, 0, 0, 0, 1017, 1009, 1021, 1015, 985, 1033, 994, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8261, 8168, 8281, 8191, 8175, 8198, 8076] avg=8192.0000 max=8281 max_rank=3 min=8076 min_rank=7 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1013, 970, 1021, 1024, 1034, 1021, 1037, 965, 7, 4, 0, 0, 0, 0, 0, 0, 1001, 1015, 1035, 1060, 990, 1029, 1026, 1077, 15, 0, 0, 0, 0, 0, 0, 0, 1006, 1031, 1039, 1060, 999, 1044, 1053, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 997, 988, 1030, 983, 993, 1044, 996, 11, 2, 0, 0, 0, 0, 0, 0, 1037, 952, 1026, 1005, 999, 1090, 992, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 1015, 1037, 999, 1073, 1019, 1062, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 997, 974, 1043, 1007, 1026, 1036, 1071, 4, 5, 0, 0, 0, 0, 0, 0, 1039, 1058, 989, 968, 977, 1041, 1018, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8248, 8302, 8071, 8176, 8325, 8196, 8122] avg=8192.0000 max=8325 max_rank=5 min=8071 min_rank=3 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1052, 1047, 989, 1024, 1060, 988, 1074, 1067, 6, 4, 0, 0, 0, 0, 0, 0, 1012, 1065, 1012, 937, 1041, 1024, 974, 975, 15, 0, 0, 0, 0, 0, 0, 0, 1038, 1104, 1022, 966, 1003, 1021, 1020, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1005, 1017, 981, 1042, 1004, 1023, 1034, 11, 2, 0, 0, 0, 0, 0, 0, 1027, 1031, 1032, 1056, 1012, 1050, 996, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1018, 957, 1050, 1000, 1041, 1048, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1027, 1046, 1010, 1036, 1040, 1038, 1021, 5, 5, 0, 0, 0, 0, 0, 0, 1035, 1015, 955, 1015, 1146, 994, 1016, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8055, 8206, 8176, 8217, 8153, 8229, 8189] avg=8192.0000 max=8311 max_rank=0 min=8055 min_rank=1 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1007, 1018, 992, 1017, 979, 993, 1017, 1040, 10, 8, 0, 0, 0, 0, 0, 0, 1033, 1019, 1002, 1026, 1008, 961, 1065, 1054, 19, 0, 0, 0, 0, 0, 0, 0, 1011, 1001, 999, 1033, 1022, 1050, 1040, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1122, 1003, 1056, 1054, 963, 1006, 1003, 17, 5, 0, 0, 0, 0, 0, 0, 998, 1032, 1120, 1015, 1006, 1073, 1028, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1040, 1056, 1004, 1022, 1059, 1024, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 999, 1006, 976, 1047, 995, 1044, 1008, 10, 8, 0, 0, 0, 0, 0, 0, 1054, 1034, 1006, 1012, 1085, 1020, 1048, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8081, 8187, 8188, 8197, 8302, 8203, 8116, 8262] avg=8192.0000 max=8302 max_rank=4 min=8081 min_rank=0 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8192, 8448, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1042, 1026, 1064, 1008, 1030, 998, 1032, 989, 1083, 1015, 1064, 1075, 1048, 1015, 1012, 996, 1072, 1039, 1053, 1090, 1020, 1012, 975, 983, 1023, 1032, 1021, 1060, 982, 984, 1032, 1019, 1029, 1029, 1016, 1054, 1045, 989, 1015, 1007, 1055, 1013, 1004, 1014, 999, 992, 1038, 962, 1015, 1061, 987, 993, 1037, 1062, 992, 1039, 958, 1045, 1087, 1056, 1029, 989, 1039, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[985, 967, 1008, 1027, 957, 982, 954, 1044, 1038, 1034, 1000, 1078, 1072, 1002, 1059, 1051, 1028, 1014, 1052, 1033, 992, 979, 1019, 1027, 1014, 1059, 1012, 1041, 995, 971, 1029, 1072, 1034, 1023, 1032, 1047, 996, 1045, 1015, 1032, 1063, 1057, 1039, 1023, 1007, 1022, 1001, 1022, 1050, 970, 1012, 1061, 1038, 996, 1108, 1042, 1081, 1034, 1003, 1066, 990, 1008, 1038, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1012, 981, 1062, 1037, 1005, 966, 1017, 996, 1075, 1049, 1022, 999, 1008, 1024, 1025, 1014, 1020, 1028, 972, 1026, 1027, 997, 1057, 1033, 1049, 994, 1049, 1048, 1018, 1047, 1078, 1036, 991, 1031, 1061, 1031, 1024, 1005, 992, 1021, 1040, 1048, 986, 981, 1039, 1013, 999, 1046, 1013, 952, 1037, 1028, 1026, 1043, 1018, 1009, 1037, 1061, 1063, 1057, 1023, 1041, 1037, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[999, 1015, 1035, 994, 989, 1038, 998, 1036, 1057, 1013, 1057, 1048, 1021, 1031, 977, 997, 996, 1048, 1004, 1025, 1010, 1014, 1030, 1054, 1055, 1036, 1010, 1055, 1043, 1015, 995, 1063, 985, 1034, 1021, 1018, 1045, 1082, 1048, 1040, 1040, 1055, 993, 1008, 1031, 1020, 1046, 1022, 963, 1035, 1049, 1024, 1029, 1009, 1014, 986, 1037, 1043, 1047, 976, 997, 1015, 1016, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1050, 1039, 1010, 1000, 1030, 1045, 1008, 1021, 1004, 988, 1019, 1023, 1060, 994, 1106, 1066, 1031, 1017, 1055, 1019, 1042, 1059, 990, 1053, 1009, 1022, 1029, 1015, 998, 1046, 1007, 1003, 1001, 995, 1027, 1033, 1046, 990, 1032, 1057, 1002, 1083, 1019, 983, 1055, 993, 1060, 1000, 1016, 1009, 983, 999, 1054, 984, 1049, 1020, 1056, 1020, 1010, 1014, 1016, 1041, 1016, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1021, 991, 1051, 1007, 973, 1002, 1024, 1022, 1087, 961, 985, 1047, 1005, 1040, 1037, 1055, 1008, 1048, 973, 1053, 1024, 1039, 1001, 1061, 1003, 1030, 1038, 1010, 1006, 1013, 985, 1022, 1047, 1031, 1082, 1008, 1009, 1056, 1031, 1014, 1025, 1062, 1011, 1062, 1065, 1036, 1062, 1014, 1027, 1033, 1011, 1061, 988, 1040, 1027, 1004, 1016, 993, 1009, 1043, 998, 1011, 995, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1072, 1035, 1052, 1017, 1021, 1030, 1021, 1047, 1016, 988, 998, 992, 1009, 1014, 989, 986, 1049, 1015, 999, 1098, 1012, 1017, 1071, 991, 1009, 1019, 1026, 1024, 1033, 1051, 1064, 998, 989, 1059, 988, 1026, 1033, 1053, 1011, 1016, 1030, 992, 1022, 996, 1047, 1043, 1023, 990, 1032, 1013, 1055, 1011, 1047, 1004, 1031, 1097, 1001, 1055, 1000, 1005, 1055, 1010, 1011, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1041, 1045, 1001, 1018, 1019, 1009, 1011, 1083, 1017, 1045, 1040, 989, 1028, 1048, 1018, 970, 1035, 1057, 1007, 1067, 1006, 1002, 1020, 1038, 1013, 1011, 1011, 992, 1022, 1027, 1037, 1019, 1070, 1014, 999, 999, 1034, 1017, 1043, 1040, 1002, 1071, 1012, 1061, 1060, 1026, 978, 985, 1095, 979, 1041, 1027, 1024, 1023, 1003, 1026, 989, 1000, 995, 1062, 1028, 1017, 1042, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1042, 1026, 1064, 1008, 1030, 998, 1032, 989, 18, 19, 16, 0, 0, 0, 0, 0, 1065, 1015, 1064, 1075, 1048, 1015, 1012, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1039, 1053, 1071, 1020, 1012, 975, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1032, 1021, 1060, 982, 984, 1032, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1029, 1016, 1054, 1045, 971, 1015, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1013, 1004, 1014, 999, 992, 1038, 962, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1061, 987, 993, 1037, 1062, 992, 1039, 2, 0, 0, 0, 0, 0, 0, 0, 958, 1045, 1087, 1056, 1029, 989, 1039, 992, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8290, 8225, 8153, 8166, 8077, 8188, 8195] avg=8192.0000 max=8290 max_rank=1 min=8077 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1050, 1039, 1010, 1000, 1030, 1045, 1008, 1021, 18, 18, 15, 0, 0, 0, 0, 0, 987, 988, 1019, 1023, 1060, 994, 1106, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1017, 1055, 1001, 1042, 1059, 990, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1022, 1029, 1015, 998, 1046, 1007, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 995, 1027, 1033, 1046, 972, 1032, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1083, 1019, 983, 1055, 993, 1060, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1009, 983, 999, 1054, 984, 1049, 1020, 2, 0, 0, 0, 0, 0, 0, 0, 1056, 1020, 1010, 1014, 1016, 1041, 1016, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8243, 8248, 8129, 8163, 8195, 8116, 8188] avg=8192.0000 max=8254 max_rank=0 min=8116 min_rank=6 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[985, 967, 1008, 1027, 957, 982, 954, 1044, 19, 18, 15, 0, 0, 0, 0, 0, 1021, 1034, 1000, 1078, 1072, 1002, 1059, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1014, 1052, 1015, 992, 979, 1019, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1059, 1012, 1041, 995, 971, 1029, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1023, 1032, 1047, 996, 1026, 1015, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1057, 1039, 1023, 1007, 1022, 1001, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 970, 1012, 1061, 1038, 996, 1108, 1042, 2, 0, 0, 0, 0, 0, 0, 0, 1081, 1034, 1003, 1066, 990, 1008, 1038, 986, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7976, 8317, 8126, 8193, 8205, 8234, 8279, 8206] avg=8192.0000 max=8317 max_rank=1 min=7976 min_rank=0 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32924 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1012, 981, 1062, 1037, 1005, 966, 1017, 996, 18, 18, 16, 0, 0, 0, 0, 0, 1057, 1049, 1022, 999, 1008, 1024, 1025, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1028, 972, 1008, 1027, 997, 1057, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 994, 1049, 1048, 1018, 1047, 1078, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1031, 1061, 1031, 1024, 987, 992, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1048, 986, 981, 1039, 1013, 999, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 952, 1037, 1028, 1026, 1043, 1018, 1009, 2, 0, 0, 0, 0, 0, 0, 0, 1037, 1061, 1063, 1057, 1023, 1041, 1037, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8198, 8142, 8319, 8138, 8152, 8128, 8331] avg=8192.0000 max=8331 max_rank=7 min=8128 min_rank=0 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[999, 1015, 1035, 994, 989, 1038, 998, 1036, 19, 18, 15, 0, 0, 0, 0, 0, 1040, 1013, 1057, 1048, 1021, 1031, 977, 997, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1048, 1004, 1007, 1010, 1014, 1030, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1036, 1010, 1055, 1043, 1015, 995, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1034, 1021, 1018, 1045, 1063, 1048, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1055, 993, 1008, 1031, 1020, 1046, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1035, 1049, 1024, 1029, 1009, 1014, 986, 2, 0, 0, 0, 0, 0, 0, 0, 1037, 1043, 1047, 976, 997, 1015, 1016, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8184, 8163, 8272, 8254, 8215, 8111, 8181] avg=8192.0000 max=8272 max_rank=3 min=8111 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1021, 991, 1051, 1007, 973, 1002, 1024, 1022, 19, 18, 16, 0, 0, 0, 0, 0, 1069, 961, 985, 1047, 1005, 1040, 1037, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1048, 973, 1035, 1024, 1039, 1001, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1030, 1038, 1010, 1006, 1013, 985, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1031, 1082, 1008, 1009, 1037, 1031, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1062, 1011, 1062, 1065, 1036, 1062, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1033, 1011, 1061, 988, 1040, 1027, 1004, 2, 0, 0, 0, 0, 0, 0, 0, 1016, 993, 1009, 1043, 998, 1011, 995, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8199, 8189, 8107, 8259, 8337, 8193, 8108] avg=8192.0000 max=8337 max_rank=5 min=8107 min_rank=3 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1072, 1035, 1052, 1017, 1021, 1030, 1021, 1047, 19, 19, 15, 0, 0, 0, 0, 0, 999, 988, 998, 992, 1009, 1014, 989, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1015, 999, 1079, 1012, 1017, 1071, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1019, 1026, 1024, 1033, 1051, 1064, 998, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1059, 988, 1026, 1033, 1034, 1011, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 992, 1022, 996, 1047, 1043, 1023, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1013, 1055, 1011, 1047, 1004, 1031, 1097, 2, 0, 0, 0, 0, 0, 0, 0, 1001, 1055, 1000, 1005, 1055, 1010, 1011, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8348, 7975, 8233, 8224, 8156, 8143, 8292, 8165] avg=8192.0000 max=8348 max_rank=0 min=7975 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1041, 1045, 1001, 1018, 1019, 1009, 1011, 1083, 21, 22, 18, 0, 0, 0, 0, 0, 992, 1045, 1040, 989, 1028, 1048, 1018, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1057, 1007, 1045, 1006, 1002, 1020, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1011, 1011, 992, 1022, 1027, 1037, 1019, 7, 0, 0, 0, 0, 0, 0, 0, 1070, 1014, 999, 999, 1034, 996, 1043, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1059, 1012, 1061, 1060, 1026, 978, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 979, 1041, 1027, 1024, 1023, 1003, 1026, 7, 4, 0, 0, 0, 0, 0, 0, 989, 1000, 995, 1062, 1028, 1017, 1042, 1028, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8130, 8210, 8139, 8195, 8183, 8229, 8162] avg=8192.0000 max=8288 max_rank=0 min=8130 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8320, 8192, 8064, 8192, 8192, 8320, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8448, 8192, 8192, 8320, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8064, 8192, 8320, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1005, 1036, 1025, 1044, 1025, 1004, 1021, 1045, 1044, 989, 1004, 1055, 1047, 1038, 1018, 1029, 1038, 1008, 1007, 1068, 1009, 1003, 1006, 1099, 1030, 1024, 993, 1030, 1026, 1015, 1039, 1027, 1020, 965, 1069, 1044, 1050, 1032, 1017, 1040, 993, 1051, 1078, 995, 1006, 1010, 979, 1059, 1007, 1046, 1040, 1036, 1012, 997, 961, 1015, 976, 1048, 1039, 951, 1022, 1049, 1019, 1059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[987, 1023, 1046, 1004, 938, 1016, 1026, 1028, 1054, 1068, 1065, 1037, 1031, 1025, 1017, 1051, 1003, 1050, 1080, 1007, 1010, 1041, 1029, 1002, 1078, 1081, 1016, 1096, 980, 1057, 996, 1013, 1013, 1005, 1027, 1052, 1002, 1048, 990, 1018, 1009, 1064, 1035, 1039, 987, 1025, 1001, 1008, 959, 1015, 1023, 1044, 1002, 1073, 1046, 1027, 1025, 1001, 1023, 972, 989, 1007, 1030, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1050, 1047, 996, 992, 1036, 976, 1015, 1063, 1029, 1060, 1036, 1016, 1062, 1023, 1008, 1046, 1017, 1015, 1010, 960, 1041, 1014, 1030, 1009, 1040, 1029, 977, 1062, 1009, 1024, 1018, 1030, 960, 1001, 1029, 1016, 1002, 1018, 1033, 973, 1024, 990, 1074, 1025, 1042, 1028, 1016, 1013, 1015, 1053, 1050, 962, 1055, 1008, 1041, 1034, 1073, 1018, 1052, 1071, 1022, 1023, 1042, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1002, 974, 992, 1040, 1004, 1054, 1044, 1019, 995, 1000, 1001, 978, 981, 1014, 1004, 1032, 1039, 1007, 1030, 1040, 1031, 1029, 1084, 1019, 1065, 1024, 1075, 1022, 1006, 1033, 1028, 1003, 1019, 1041, 1042, 1048, 1020, 987, 1065, 1040, 1034, 1017, 1036, 1013, 1034, 1022, 1018, 1030, 1061, 1022, 1032, 1029, 1053, 1012, 1059, 990, 1014, 1018, 1041, 1051, 1011, 990, 1022, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1015, 1022, 1007, 985, 1046, 1052, 1017, 1043, 1023, 1037, 988, 1006, 1030, 1000, 1049, 1015, 1011, 984, 1035, 988, 991, 1021, 1067, 1013, 1026, 1024, 1005, 1065, 1059, 1030, 1007, 1041, 1009, 1025, 1038, 1065, 1066, 1013, 1016, 1013, 998, 1014, 1007, 1024, 1019, 958, 1027, 1007, 1007, 1012, 1023, 1013, 1056, 1020, 1062, 1033, 1009, 1073, 1004, 1058, 1023, 1037, 1044, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1036, 1008, 1037, 1035, 1014, 987, 1031, 1022, 1058, 1055, 1024, 1017, 1017, 1028, 1031, 1006, 1053, 1009, 1008, 1023, 1069, 1005, 989, 1004, 1027, 1019, 1034, 1013, 1021, 1026, 1038, 1003, 1021, 1063, 1010, 1050, 1010, 1026, 1047, 1020, 1024, 1016, 1015, 973, 957, 1045, 1043, 992, 1007, 1020, 1040, 1053, 1047, 1057, 1023, 1031, 1050, 1038, 1045, 1010, 1019, 1004, 1015, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[983, 1024, 1010, 1023, 1058, 1018, 1032, 1030, 1078, 971, 1039, 1009, 1018, 1023, 1031, 1021, 1035, 1090, 1011, 1045, 1004, 1015, 1035, 1033, 1042, 1036, 992, 1012, 987, 1061, 1002, 1052, 1017, 1030, 1015, 1052, 944, 1029, 998, 969, 1029, 1027, 1003, 1029, 991, 1026, 1054, 1050, 1025, 1024, 1032, 1049, 993, 1035, 996, 1055, 1010, 1059, 1050, 1027, 1053, 1021, 1039, 985] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1005, 1023, 1034, 1022, 971, 995, 1009, 1045, 1068, 1044, 1082, 1110, 1060, 998, 1043, 980, 1013, 1043, 998, 1019, 1008, 1033, 998, 987, 1029, 972, 1055, 1067, 987, 1027, 1044, 1002, 1084, 1009, 1036, 1033, 997, 989, 1030, 1034, 1063, 1004, 1006, 1007, 1052, 1020, 993, 1027, 990, 1060, 1006, 1012, 1030, 1000, 1029, 991, 1051, 1075, 1041, 1036, 1023, 990, 1025, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[987, 1023, 1046, 1004, 938, 1016, 1026, 1028, 35, 5, 0, 0, 0, 0, 0, 0, 1019, 1068, 1065, 1037, 1031, 1025, 1017, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1050, 1080, 1007, 1010, 1041, 1029, 1002, 7, 0, 0, 0, 0, 0, 0, 0, 1078, 1081, 1016, 1069, 980, 1057, 996, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1005, 1027, 1052, 1002, 1048, 990, 1018, 9, 1, 0, 0, 0, 0, 0, 0, 1009, 1064, 1035, 1039, 987, 1025, 1001, 1008, 22, 12, 0, 0, 0, 0, 0, 0, 959, 1007, 1023, 1044, 1002, 1073, 1046, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 980, 1023, 972, 989, 1007, 1030, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8313, 8229, 8290, 8165, 8202, 8181, 8048] avg=8192.0000 max=8313 max_rank=1 min=8048 min_rank=7 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1005, 1036, 1025, 1044, 1025, 1004, 1021, 1045, 35, 4, 0, 0, 0, 0, 0, 0, 1009, 989, 1004, 1055, 1047, 1038, 1018, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1008, 1007, 1068, 1009, 1003, 1006, 1099, 8, 0, 0, 0, 0, 0, 0, 0, 1030, 1024, 993, 1005, 1026, 1015, 1039, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 965, 1069, 1044, 1050, 1032, 1017, 1040, 10, 1, 0, 0, 0, 0, 0, 0, 993, 1051, 1078, 995, 1006, 1010, 979, 1059, 21, 13, 0, 0, 0, 0, 0, 0, 1007, 1037, 1040, 1036, 1012, 997, 961, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1025, 1039, 951, 1022, 1049, 1019, 1059, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8189, 8246, 8159, 8248, 8205, 8105, 8140] avg=8192.0000 max=8248 max_rank=4 min=8105 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1002, 974, 992, 1040, 1004, 1054, 1044, 1019, 33, 4, 0, 0, 0, 0, 0, 0, 962, 1000, 1001, 978, 981, 1014, 1004, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1007, 1030, 1040, 1031, 1029, 1084, 1019, 7, 0, 0, 0, 0, 0, 0, 0, 1065, 1024, 1075, 997, 1006, 1033, 1028, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1041, 1042, 1048, 1020, 987, 1065, 1040, 10, 1, 0, 0, 0, 0, 0, 0, 1034, 1017, 1036, 1013, 1034, 1022, 1018, 1030, 21, 12, 0, 0, 0, 0, 0, 0, 1061, 1014, 1032, 1029, 1053, 1012, 1059, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 996, 1041, 1051, 1011, 990, 1022, 996, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 7972, 8286, 8231, 8273, 8237, 8250, 8121] avg=8192.0000 max=8286 max_rank=2 min=7972 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1050, 1047, 996, 992, 1036, 976, 1015, 1063, 34, 5, 0, 0, 0, 0, 0, 0, 995, 1060, 1036, 1016, 1062, 1023, 1008, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1015, 1010, 960, 1041, 1014, 1030, 1009, 8, 0, 0, 0, 0, 0, 0, 0, 1040, 1029, 977, 1035, 1009, 1024, 1018, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 960, 1001, 1029, 1016, 1002, 1018, 1033, 973, 10, 1, 0, 0, 0, 0, 0, 0, 1024, 990, 1074, 1025, 1042, 1028, 1016, 1013, 22, 12, 0, 0, 0, 0, 0, 0, 1015, 1044, 1050, 962, 1055, 1008, 1041, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 996, 1052, 1071, 1022, 1023, 1042, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8246, 8104, 8162, 8043, 8246, 8209, 8312] avg=8192.0000 max=8312 max_rank=7 min=8043 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[983, 1024, 1010, 1023, 1058, 1018, 1032, 1030, 36, 4, 0, 0, 0, 0, 0, 0, 1042, 971, 1039, 1009, 1018, 1023, 1031, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1090, 1011, 1045, 1004, 1015, 1035, 1033, 7, 0, 0, 0, 0, 0, 0, 0, 1042, 1036, 992, 987, 987, 1061, 1002, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1030, 1015, 1052, 944, 1029, 998, 969, 10, 1, 0, 0, 0, 0, 0, 0, 1029, 1027, 1003, 1029, 991, 1026, 1054, 1050, 21, 13, 0, 0, 0, 0, 0, 0, 1025, 1016, 1032, 1049, 993, 1035, 996, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1036, 1050, 1027, 1053, 1021, 1039, 985, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8154, 8275, 8159, 8065, 8243, 8201, 8221] avg=8192.0000 max=8275 max_rank=2 min=8065 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1015, 1022, 1007, 985, 1046, 1052, 1017, 1043, 34, 5, 0, 0, 0, 0, 0, 0, 989, 1037, 988, 1006, 1030, 1000, 1049, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 984, 1035, 988, 991, 1021, 1067, 1013, 7, 0, 0, 0, 0, 0, 0, 0, 1026, 1024, 1005, 1038, 1059, 1030, 1007, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1025, 1038, 1065, 1066, 1013, 1016, 1013, 10, 1, 0, 0, 0, 0, 0, 0, 998, 1014, 1007, 1024, 1019, 958, 1027, 1007, 22, 13, 0, 0, 0, 0, 0, 0, 1007, 1004, 1023, 1013, 1056, 1020, 1062, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1050, 1004, 1058, 1023, 1037, 1044, 1061, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8114, 8117, 8230, 8256, 8089, 8218, 8286] avg=8192.0000 max=8286 max_rank=7 min=8089 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1036, 1008, 1037, 1035, 1014, 987, 1031, 1022, 35, 4, 0, 0, 0, 0, 0, 0, 1023, 1055, 1024, 1017, 1017, 1028, 1031, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1009, 1008, 1023, 1069, 1005, 989, 1004, 7, 0, 0, 0, 0, 0, 0, 0, 1027, 1019, 1034, 988, 1021, 1026, 1038, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1063, 1010, 1050, 1010, 1026, 1047, 1020, 10, 1, 0, 0, 0, 0, 0, 0, 1024, 1016, 1015, 973, 957, 1045, 1043, 992, 21, 13, 0, 0, 0, 0, 0, 0, 1007, 1012, 1040, 1053, 1047, 1057, 1023, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1015, 1045, 1010, 1019, 1004, 1015, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8201, 8167, 8156, 8258, 8099, 8270, 8176] avg=8192.0000 max=8270 max_rank=6 min=8099 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1005, 1023, 1034, 1022, 971, 995, 1009, 1045, 38, 9, 0, 0, 0, 0, 0, 0, 1030, 1044, 1082, 1110, 1060, 998, 1043, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1043, 998, 1019, 1008, 1033, 998, 987, 13, 0, 0, 0, 0, 0, 0, 0, 1029, 972, 1055, 1033, 987, 1027, 1044, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 1009, 1036, 1033, 997, 989, 1030, 1034, 13, 3, 0, 0, 0, 0, 0, 0, 1063, 1004, 1006, 1007, 1052, 1020, 993, 1027, 25, 18, 0, 0, 0, 0, 0, 0, 990, 1044, 1006, 1012, 1030, 1000, 1029, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1044, 1041, 1036, 1023, 990, 1025, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8347, 8112, 8149, 8228, 8215, 8102, 8232] avg=8192.0000 max=8347 max_rank=1 min=8102 min_rank=6 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8448, 8320, 8192, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8448, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[987, 987, 1022, 1052, 981, 1049, 1033, 1015, 998, 1055, 962, 1078, 1003, 1000, 1023, 1035, 1033, 1018, 978, 996, 1015, 1003, 1061, 1041, 1076, 1057, 1017, 1050, 1043, 1074, 989, 1054, 968, 1005, 1051, 973, 1001, 1041, 1015, 962, 1089, 1037, 1035, 1031, 1059, 997, 1031, 1051, 1010, 975, 1044, 1069, 1014, 1043, 1037, 1031, 1081, 1007, 995, 993, 1030, 1021, 1051, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1002, 1063, 1003, 980, 998, 1014, 1021, 1077, 1033, 1006, 1060, 1014, 1052, 1010, 1072, 1022, 1065, 1041, 1041, 1010, 961, 1045, 1033, 1057, 1012, 1035, 1033, 1030, 1054, 1023, 1053, 990, 1025, 993, 968, 1054, 962, 1012, 1007, 1029, 1029, 1062, 1047, 1037, 1022, 1090, 1043, 993, 993, 1034, 1000, 963, 1017, 1050, 983, 1059, 1048, 1057, 1010, 1059, 996, 1005, 998, 981] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1025, 1090, 966, 1022, 1017, 1016, 1034, 1028, 986, 1009, 1060, 991, 1029, 1005, 1039, 1030, 985, 1035, 1064, 1015, 1000, 979, 1024, 1020, 1018, 1018, 1005, 1020, 1001, 1039, 1035, 1028, 1021, 1051, 1027, 1024, 1020, 984, 1062, 1017, 1022, 1056, 1024, 965, 1034, 1018, 1039, 1001, 995, 1083, 1036, 1025, 1001, 1027, 1007, 1033, 1022, 1050, 1015, 1093, 1049, 1084, 1014, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[997, 1021, 1036, 1004, 1035, 994, 1043, 1008, 1038, 1024, 1048, 987, 1067, 1003, 1021, 1022, 1067, 1008, 1027, 993, 990, 1000, 993, 1002, 1000, 1072, 1023, 1039, 1036, 1086, 992, 964, 1014, 1040, 1034, 1005, 969, 1057, 1008, 933, 1021, 1023, 1059, 1025, 1041, 1044, 1062, 1057, 1024, 1040, 1027, 1006, 1048, 1031, 1064, 1037, 1058, 994, 1025, 1055, 1073, 983, 1019, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1082, 989, 1027, 997, 1111, 1055, 984, 1093, 940, 1007, 1003, 1024, 1008, 1006, 1063, 1011, 1011, 1049, 1067, 1007, 1063, 1036, 1034, 1018, 988, 1012, 1017, 1079, 1005, 1018, 999, 1015, 994, 1028, 1053, 998, 1053, 1013, 1041, 990, 1048, 1040, 954, 1004, 1017, 1044, 1009, 1020, 1031, 1004, 1016, 1015, 1005, 1034, 1061, 1035, 1058, 992, 1022, 1004, 1020, 1072, 1013, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[988, 998, 1003, 1036, 1027, 1006, 1057, 1037, 1007, 1048, 1046, 1028, 1069, 1009, 1046, 1053, 1053, 1006, 1012, 1007, 1038, 1030, 1022, 994, 991, 1019, 1037, 1047, 1014, 996, 978, 1048, 972, 1017, 1016, 1078, 1032, 1033, 1005, 1014, 992, 985, 1011, 1069, 1014, 987, 1065, 1063, 1010, 964, 1046, 1025, 1001, 1027, 1025, 1054, 1032, 1047, 1039, 1045, 1049, 1022, 1029, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1065, 1005, 991, 1059, 1054, 1011, 1023, 1028, 1042, 983, 1090, 1011, 1012, 1043, 1034, 1046, 968, 1022, 1038, 1013, 1039, 991, 988, 1025, 1030, 1023, 1007, 1065, 1059, 1036, 1010, 972, 1022, 973, 996, 1022, 984, 1054, 1043, 1043, 990, 1070, 1030, 1011, 992, 1007, 1027, 1023, 991, 1014, 999, 1038, 1040, 1033, 1048, 1047, 1036, 1052, 1038, 1059, 997, 1013, 1056, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1014, 993, 1049, 1015, 1042, 1015, 1049, 1035, 1016, 1062, 1017, 1020, 1049, 1031, 995, 1014, 1049, 1051, 1016, 995, 1013, 1019, 1047, 1003, 973, 1021, 1056, 1041, 1087, 1030, 1101, 978, 1013, 1038, 1036, 984, 1020, 1077, 1035, 1013, 982, 1014, 1016, 955, 1021, 1032, 1029, 1069, 1010, 1015, 1006, 1028, 1049, 999, 1043, 1024, 1019, 967, 1006, 1049, 1005, 1015, 1067, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1002, 1063, 1003, 980, 998, 1014, 1021, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1006, 1060, 1014, 1052, 1010, 1062, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1041, 1041, 1010, 961, 1045, 1033, 1057, 12, 10, 2, 0, 0, 0, 0, 0, 1012, 1035, 1033, 1009, 1054, 1023, 1053, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 993, 968, 1054, 962, 1012, 1007, 1029, 30, 24, 9, 0, 0, 0, 0, 0, 1029, 1062, 1047, 1037, 1022, 1090, 1019, 993, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1034, 1000, 963, 1017, 1050, 983, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1057, 1010, 1029, 996, 1005, 998, 981, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8259, 8277, 8209, 8113, 8299, 8099, 8124] avg=8192.0000 max=8299 max_rank=5 min=8099 min_rank=6 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[987, 987, 1022, 1052, 981, 1049, 1033, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1055, 962, 1078, 1003, 1000, 1014, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1018, 978, 996, 1015, 1003, 1061, 1041, 13, 9, 2, 0, 0, 0, 0, 0, 1076, 1057, 1017, 1028, 1043, 1074, 989, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1005, 1051, 973, 1001, 1041, 1015, 962, 28, 24, 9, 0, 0, 0, 0, 0, 1089, 1037, 1035, 1031, 1059, 997, 1007, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 975, 1044, 1069, 1014, 1043, 1037, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1007, 995, 965, 1030, 1021, 1051, 1004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8145, 8169, 8338, 8077, 8306, 8223, 8154] avg=8192.0000 max=8338 max_rank=3 min=8077 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[997, 1021, 1036, 1004, 1035, 994, 1043, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1024, 1048, 987, 1067, 1003, 1012, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1008, 1027, 993, 990, 1000, 993, 1002, 12, 9, 2, 0, 0, 0, 0, 0, 1000, 1072, 1023, 1018, 1036, 1086, 992, 964, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1040, 1034, 1005, 969, 1057, 1008, 933, 30, 25, 9, 0, 0, 0, 0, 0, 1021, 1023, 1059, 1025, 1041, 1044, 1037, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1040, 1027, 1006, 1048, 1031, 1064, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 994, 1025, 1025, 1073, 983, 1019, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8201, 8103, 8191, 8124, 8307, 8277, 8197] avg=8192.0000 max=8307 max_rank=5 min=8103 min_rank=2 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32905 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1082, 989, 1027, 997, 1111, 1055, 984, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 940, 1007, 1003, 1024, 1008, 1006, 1053, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1049, 1067, 1007, 1063, 1036, 1034, 1018, 13, 10, 2, 0, 0, 0, 0, 0, 988, 1012, 1017, 1056, 1005, 1018, 999, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1028, 1053, 998, 1053, 1013, 1041, 990, 28, 24, 10, 0, 0, 0, 0, 0, 1048, 1040, 954, 1004, 1017, 1044, 985, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1004, 1016, 1015, 1005, 1034, 1061, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 992, 1022, 976, 1020, 1072, 1013, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8336, 8052, 8310, 8110, 8232, 8112, 8201, 8183] avg=8192.0000 max=8336 max_rank=0 min=8052 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1025, 1090, 966, 1022, 1017, 1016, 1034, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1009, 1060, 991, 1029, 1005, 1030, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1035, 1064, 1015, 1000, 979, 1024, 1020, 12, 9, 2, 0, 0, 0, 0, 0, 1018, 1018, 1005, 999, 1001, 1039, 1035, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1051, 1027, 1024, 1020, 984, 1062, 1017, 31, 24, 9, 0, 0, 0, 0, 0, 1022, 1056, 1024, 965, 1034, 1018, 1015, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1083, 1036, 1025, 1001, 1027, 1007, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1050, 1015, 1062, 1049, 1084, 1014, 1004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8140, 8145, 8143, 8270, 8135, 8207, 8300] avg=8192.0000 max=8300 max_rank=7 min=8135 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[988, 998, 1003, 1036, 1027, 1006, 1057, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1048, 1046, 1028, 1069, 1009, 1037, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1006, 1012, 1007, 1038, 1030, 1022, 994, 13, 9, 2, 0, 0, 0, 0, 0, 991, 1019, 1037, 1025, 1014, 996, 978, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1017, 1016, 1078, 1032, 1033, 1005, 1014, 29, 25, 9, 0, 0, 0, 0, 0, 992, 985, 1011, 1069, 1014, 987, 1040, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 964, 1046, 1025, 1001, 1027, 1025, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1047, 1039, 1016, 1049, 1022, 1029, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8297, 8186, 8108, 8230, 8161, 8152, 8252] avg=8192.0000 max=8297 max_rank=1 min=8108 min_rank=3 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1065, 1005, 991, 1059, 1054, 1011, 1023, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 983, 1090, 1011, 1012, 1043, 1025, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1022, 1038, 1013, 1039, 991, 988, 1025, 13, 9, 2, 0, 0, 0, 0, 0, 1030, 1023, 1007, 1043, 1059, 1036, 1010, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 973, 996, 1022, 984, 1054, 1043, 1043, 30, 24, 9, 0, 0, 0, 0, 0, 990, 1070, 1030, 1011, 992, 1007, 1003, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1014, 999, 1038, 1040, 1033, 1048, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1052, 1038, 1029, 997, 1013, 1056, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8252, 8108, 8180, 8200, 8126, 8210, 8226] avg=8192.0000 max=8252 max_rank=1 min=8108 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1014, 993, 1049, 1015, 1042, 1015, 1049, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1062, 1017, 1020, 1049, 1031, 981, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1051, 1016, 995, 1013, 1019, 1047, 1003, 16, 14, 8, 7, 0, 0, 0, 0, 973, 1021, 1056, 1011, 1087, 1030, 1101, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1038, 1036, 984, 1020, 1077, 1035, 1013, 32, 28, 14, 0, 0, 0, 0, 0, 982, 1014, 1016, 955, 1021, 1032, 1001, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1015, 1006, 1028, 1049, 999, 1043, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 967, 1006, 1017, 1005, 1015, 1067, 1004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8190, 8238, 8257, 8290, 8090, 8167, 8100] avg=8192.0000 max=8290 max_rank=4 min=8090 min_rank=5 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32647 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8064, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8192, 8064, 8320, 8320, 8064, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8320, 8064, 8192, 8192, 8320, 8192, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1024, 1059, 1001, 1036, 1020, 1046, 1022, 986, 1007, 981, 1040, 998, 1030, 1059, 1006, 1014, 1058, 1037, 1037, 1014, 1027, 1048, 982, 1020, 980, 1068, 1008, 1034, 1030, 1026, 1013, 1040, 1021, 999, 1047, 1018, 1097, 1042, 1008, 1018, 1029, 1016, 1049, 1021, 969, 1030, 1035, 1030, 1030, 1030, 1012, 1057, 1029, 1024, 955, 1069, 999, 1001, 1010, 995, 1051, 1026, 1022, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1005, 965, 999, 1043, 959, 1036, 1012, 1039, 1010, 1031, 1071, 1067, 1057, 1003, 1085, 987, 1014, 1049, 1053, 993, 1030, 1038, 1019, 1085, 1019, 1035, 1032, 1011, 1036, 974, 1083, 1019, 1061, 1025, 1016, 986, 991, 1007, 1056, 1058, 1008, 990, 1034, 1040, 1030, 1062, 1010, 995, 1003, 1015, 1028, 1046, 1021, 997, 988, 1024, 1078, 1074, 1022, 1028, 1004, 1043, 994, 943] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1022, 1045, 1028, 1029, 1009, 1062, 1028, 983, 1042, 1057, 1033, 1059, 1027, 1050, 1062, 1015, 1004, 1037, 1016, 1040, 1021, 1005, 1003, 1046, 1004, 999, 997, 1018, 998, 1026, 1067, 1035, 1024, 1008, 1026, 1024, 987, 1042, 1035, 1059, 1052, 1004, 1010, 1030, 1035, 1017, 1027, 1025, 1021, 992, 1022, 1014, 1023, 1041, 976, 1022, 1020, 1036, 996, 1018, 1018, 1034, 1031, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1041, 1033, 1063, 992, 1058, 1006, 1029, 1036, 1017, 1031, 934, 1097, 1014, 1027, 984, 987, 1030, 1050, 947, 1068, 1022, 1053, 1035, 1018, 1009, 1017, 1063, 1025, 1051, 948, 1041, 993, 993, 984, 1030, 984, 1112, 1037, 1014, 978, 1018, 1017, 1056, 989, 1072, 1032, 1051, 1049, 1027, 988, 1054, 1012, 1031, 992, 1037, 1020, 1043, 990, 1047, 1016, 1060, 1042, 1037, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1036, 1036, 1008, 985, 1039, 1001, 1036, 1028, 1060, 1051, 1000, 990, 998, 1008, 1055, 1083, 1035, 1012, 996, 1029, 987, 967, 1016, 1022, 1006, 1088, 1040, 1047, 1009, 1007, 1010, 999, 1040, 990, 1031, 1068, 1038, 1024, 1052, 1050, 974, 1014, 1016, 1029, 1043, 1045, 995, 1015, 1038, 1018, 996, 1054, 1015, 1006, 994, 1023, 1050, 1055, 1024, 1057, 1032, 1006, 1032, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1019, 979, 1049, 1034, 1035, 1065, 1021, 1000, 986, 1047, 1097, 1013, 1009, 1020, 1044, 1038, 1001, 1040, 1050, 1008, 1016, 1052, 1037, 981, 1000, 1031, 982, 1081, 1034, 1019, 986, 965, 1063, 983, 1079, 1043, 996, 1023, 1035, 1019, 1046, 1049, 1020, 1002, 984, 993, 1013, 1028, 1038, 999, 1067, 1010, 1052, 1042, 1069, 949, 1013, 1044, 1002, 993, 1031, 1043, 1029, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1037, 956, 1059, 1033, 1038, 1028, 1042, 1056, 1025, 1028, 1014, 1017, 1019, 1051, 1020, 1009, 997, 998, 1029, 1061, 997, 1015, 1027, 1010, 1052, 1002, 1021, 994, 1015, 1062, 986, 1059, 1017, 1012, 1022, 1064, 1014, 1069, 1018, 1004, 1004, 1055, 992, 1046, 1025, 1003, 971, 1044, 1045, 1044, 1068, 994, 981, 973, 1023, 1018, 1046, 1036, 1030, 1043, 1018, 1050, 1021, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1034, 1064, 1003, 1006, 996, 1046, 1041, 971, 1011, 1041, 1042, 1017, 1083, 1064, 1086, 1038, 1020, 1107, 1050, 1034, 1012, 969, 999, 1029, 1033, 1009, 1045, 988, 1017, 1010, 1032, 1067, 1024, 992, 1083, 983, 1048, 1002, 1005, 1029, 1028, 1007, 1016, 1047, 999, 1033, 1002, 1050, 991, 1028, 1075, 1014, 1022, 1030, 1027, 977, 1030, 1015, 1034, 974, 997, 1004, 1009, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1005, 965, 999, 1043, 959, 1036, 1012, 1039, 4, 0, 0, 0, 0, 0, 0, 0, 1010, 1031, 1071, 1067, 1057, 1003, 1033, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1049, 1053, 993, 1030, 1038, 1019, 1085, 3, 0, 0, 0, 0, 0, 0, 0, 1019, 1035, 1032, 1011, 1036, 974, 1083, 1019, 18, 0, 0, 0, 0, 0, 0, 0, 1061, 1025, 997, 986, 991, 1007, 1056, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 990, 1034, 1040, 1030, 1062, 1010, 995, 12, 1, 0, 0, 0, 0, 0, 0, 1003, 1015, 1028, 1046, 1021, 997, 988, 1024, 33, 0, 0, 0, 0, 0, 0, 0, 1078, 1074, 1022, 1028, 1004, 1043, 994, 943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8062, 8259, 8284, 8227, 8181, 8182, 8155, 8186] avg=8192.0000 max=8284 max_rank=2 min=8062 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1024, 1059, 1001, 1036, 1020, 1046, 1022, 986, 4, 0, 0, 0, 0, 0, 0, 0, 1007, 981, 1040, 998, 1030, 1059, 958, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1037, 1037, 1014, 1027, 1048, 982, 1020, 3, 0, 0, 0, 0, 0, 0, 0, 980, 1068, 1008, 1034, 1030, 1026, 1013, 1040, 17, 0, 0, 0, 0, 0, 0, 0, 1021, 999, 1028, 1018, 1097, 1042, 1008, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1016, 1049, 1021, 969, 1030, 1035, 1030, 12, 1, 0, 0, 0, 0, 0, 0, 1030, 1030, 1012, 1057, 1029, 1024, 955, 1069, 30, 0, 0, 0, 0, 0, 0, 0, 999, 1001, 1010, 995, 1051, 1026, 1022, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8087, 8226, 8216, 8231, 8192, 8236, 8150] avg=8192.0000 max=8236 max_rank=6 min=8087 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1022, 1045, 1028, 1029, 1009, 1062, 1028, 983, 4, 0, 0, 0, 0, 0, 0, 0, 1042, 1057, 1033, 1059, 1027, 1050, 1012, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1037, 1016, 1040, 1021, 1005, 1003, 1046, 3, 0, 0, 0, 0, 0, 0, 0, 1004, 999, 997, 1018, 998, 1026, 1067, 1035, 17, 0, 0, 0, 0, 0, 0, 0, 1024, 1008, 1007, 1024, 987, 1042, 1035, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1004, 1010, 1030, 1035, 1017, 1027, 1025, 12, 1, 0, 0, 0, 0, 0, 0, 1021, 992, 1022, 1014, 1023, 1041, 976, 1022, 32, 0, 0, 0, 0, 0, 0, 0, 1020, 1036, 996, 1018, 1018, 1034, 1031, 1000, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8295, 8175, 8161, 8186, 8213, 8143, 8153] avg=8192.0000 max=8295 max_rank=1 min=8143 min_rank=6 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1041, 1033, 1063, 992, 1058, 1006, 1029, 1036, 4, 0, 0, 0, 0, 0, 0, 0, 1017, 1031, 934, 1097, 1014, 1027, 937, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1050, 947, 1068, 1022, 1053, 1035, 1018, 3, 0, 0, 0, 0, 0, 0, 0, 1009, 1017, 1063, 1025, 1051, 948, 1041, 993, 16, 0, 0, 0, 0, 0, 0, 0, 993, 984, 1011, 984, 1112, 1037, 1014, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1017, 1056, 989, 1072, 1032, 1051, 1049, 12, 1, 0, 0, 0, 0, 0, 0, 1027, 988, 1054, 1012, 1031, 992, 1037, 1020, 30, 0, 0, 0, 0, 0, 0, 0, 1043, 990, 1047, 1016, 1060, 1042, 1037, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 8044, 8226, 8163, 8113, 8297, 8191, 8240] avg=8192.0000 max=8297 max_rank=5 min=8044 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1036, 1036, 1008, 985, 1039, 1001, 1036, 1028, 4, 0, 0, 0, 0, 0, 0, 0, 1060, 1051, 1000, 990, 998, 1008, 1005, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1012, 996, 1029, 987, 967, 1016, 1022, 3, 0, 0, 0, 0, 0, 0, 0, 1006, 1088, 1040, 1047, 1009, 1007, 1010, 999, 17, 0, 0, 0, 0, 0, 0, 0, 1040, 990, 1012, 1068, 1038, 1024, 1052, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1014, 1016, 1029, 1043, 1045, 995, 1015, 12, 1, 0, 0, 0, 0, 0, 0, 1038, 1018, 996, 1054, 1015, 1006, 994, 1023, 32, 0, 0, 0, 0, 0, 0, 0, 1050, 1055, 1024, 1057, 1032, 1006, 1032, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8195, 8067, 8223, 8274, 8144, 8176, 8284] avg=8192.0000 max=8284 max_rank=7 min=8067 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1019, 979, 1049, 1034, 1035, 1065, 1021, 1000, 5, 0, 0, 0, 0, 0, 0, 0, 986, 1047, 1097, 1013, 1009, 1020, 994, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1040, 1050, 1008, 1016, 1052, 1037, 981, 3, 0, 0, 0, 0, 0, 0, 0, 1000, 1031, 982, 1081, 1034, 1019, 986, 965, 17, 0, 0, 0, 0, 0, 0, 0, 1063, 983, 1058, 1043, 996, 1023, 1035, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1049, 1020, 1002, 984, 993, 1013, 1028, 13, 1, 0, 0, 0, 0, 0, 0, 1038, 999, 1067, 1010, 1052, 1042, 1069, 949, 32, 0, 0, 0, 0, 0, 0, 0, 1013, 1044, 1002, 993, 1031, 1043, 1029, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8204, 8188, 8115, 8220, 8149, 8258, 8195] avg=8192.0000 max=8258 max_rank=6 min=8115 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1037, 956, 1059, 1033, 1038, 1028, 1042, 1056, 4, 0, 0, 0, 0, 0, 0, 0, 1025, 1028, 1014, 1017, 1019, 1051, 971, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 997, 998, 1029, 1061, 997, 1015, 1027, 1010, 3, 0, 0, 0, 0, 0, 0, 0, 1052, 1002, 1021, 994, 1015, 1062, 986, 1059, 17, 0, 0, 0, 0, 0, 0, 0, 1017, 1012, 1003, 1064, 1014, 1069, 1018, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1055, 992, 1046, 1025, 1003, 971, 1044, 12, 1, 0, 0, 0, 0, 0, 0, 1045, 1044, 1068, 994, 981, 973, 1023, 1018, 31, 0, 0, 0, 0, 0, 0, 0, 1046, 1036, 1030, 1043, 1018, 1050, 1021, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8134, 8137, 8208, 8201, 8153, 8177, 8273] avg=8192.0000 max=8273 max_rank=7 min=8134 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1034, 1064, 1003, 1006, 996, 1046, 1041, 971, 10, 0, 0, 0, 0, 0, 0, 0, 1011, 1041, 1042, 1017, 1083, 1064, 1022, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1107, 1050, 1034, 1012, 969, 999, 1029, 8, 5, 0, 0, 0, 0, 0, 0, 1033, 1009, 1045, 988, 1017, 1010, 1032, 1067, 22, 0, 0, 0, 0, 0, 0, 0, 1024, 992, 1047, 983, 1048, 1002, 1005, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1007, 1016, 1047, 999, 1033, 1002, 1050, 18, 6, 0, 0, 0, 0, 0, 0, 991, 1028, 1075, 1014, 1022, 1030, 1027, 977, 36, 0, 0, 0, 0, 0, 0, 0, 1025, 1015, 1034, 974, 997, 1004, 1009, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8318, 8233, 8223, 8130, 8206, 8200, 8055] avg=8192.0000 max=8318 max_rank=1 min=8055 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8448, 8192, 8320, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8320, 7936, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8448, 8320, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1010, 1022, 984, 1063, 1026, 1067, 1022, 1057, 996, 956, 1014, 959, 1076, 993, 1016, 1008, 1003, 1025, 1022, 984, 1053, 1005, 981, 1054, 1034, 1041, 1012, 1038, 1011, 995, 1007, 1019, 1034, 974, 1065, 1036, 1056, 1027, 1078, 1015, 1061, 1055, 1051, 979, 1032, 1002, 1073, 1034, 1036, 1002, 1071, 1038, 990, 991, 970, 1031, 980, 1048, 1051, 1050, 1039, 1055, 1037, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[999, 1044, 987, 1072, 1058, 964, 1044, 1009, 1046, 1057, 1008, 1007, 1001, 1021, 994, 1032, 1026, 979, 1029, 1004, 1016, 1002, 1012, 1061, 1066, 1061, 967, 1004, 1008, 995, 1062, 1039, 1097, 1040, 1005, 1001, 1044, 1013, 1002, 1030, 1017, 984, 1020, 1041, 1018, 985, 1059, 1083, 1023, 1011, 1066, 1016, 992, 1060, 1006, 1022, 1031, 986, 1020, 1055, 1003, 1025, 1041, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1048, 1007, 1051, 1024, 1034, 993, 1030, 1062, 1031, 1031, 1004, 961, 1041, 1025, 938, 1026, 986, 1042, 1050, 1094, 978, 997, 1016, 1031, 1058, 1071, 992, 1071, 1071, 1005, 1023, 995, 1023, 1010, 1028, 1039, 1011, 1023, 999, 937, 1077, 1032, 1004, 1059, 1052, 1029, 1020, 1058, 988, 987, 1044, 1059, 1009, 1025, 1040, 1006, 1013, 1038, 1015, 1030, 999, 1042, 1012, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1073, 1075, 1034, 1063, 1010, 1047, 1026, 1065, 1039, 1013, 1056, 996, 1005, 1003, 1020, 1052, 1002, 1038, 1032, 1026, 971, 1044, 1034, 998, 1024, 1029, 1029, 1020, 977, 985, 1027, 1032, 1032, 1049, 979, 1081, 1041, 1004, 987, 1058, 970, 1024, 995, 1009, 1028, 1057, 1027, 995, 1056, 1008, 986, 1047, 981, 1039, 1027, 991, 1049, 1094, 1037, 1023, 1008, 1003, 1003, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1007, 1018, 1055, 1044, 971, 1009, 1085, 998, 1012, 991, 1010, 1005, 987, 1031, 1041, 1032, 1037, 996, 997, 1011, 1032, 1071, 1047, 1026, 993, 1004, 1047, 994, 1005, 1028, 994, 1036, 1023, 962, 1009, 1025, 998, 1068, 1035, 1054, 988, 1010, 1037, 1057, 1022, 1077, 1015, 1012, 995, 1016, 1069, 1067, 1052, 1050, 990, 1040, 1043, 1039, 1009, 1049, 1051, 1057, 976, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[965, 979, 964, 1076, 1027, 1069, 1028, 1037, 997, 1000, 975, 985, 1042, 1058, 1013, 1025, 1023, 1067, 994, 985, 1025, 999, 1036, 1030, 1065, 997, 1021, 1019, 1055, 1045, 1026, 998, 1060, 1002, 1044, 1056, 1008, 987, 1015, 1080, 1060, 1003, 1051, 1010, 1049, 1060, 997, 1044, 1024, 1026, 1043, 1056, 1025, 1011, 1051, 1017, 1025, 985, 999, 1021, 1006, 1048, 997, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1035, 1046, 1003, 1008, 1004, 1003, 1033, 1035, 965, 1093, 1026, 1023, 997, 991, 981, 985, 954, 1082, 1042, 991, 998, 1063, 1008, 984, 1045, 1004, 1056, 1041, 1019, 1029, 1026, 1021, 1030, 1059, 1027, 1032, 1024, 1015, 1015, 1048, 1049, 990, 1004, 999, 1023, 1066, 1047, 1052, 1039, 1064, 1045, 1043, 995, 1009, 1012, 1024, 1039, 1046, 1041, 1070, 1014, 1020, 1024, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[972, 1011, 1023, 1018, 996, 979, 1011, 1042, 1043, 1039, 1011, 1006, 1020, 1000, 1035, 1029, 975, 999, 1012, 1030, 1042, 1005, 1054, 1047, 1036, 985, 1019, 1053, 1011, 1047, 1018, 1050, 1064, 1007, 1011, 1026, 1048, 1048, 1014, 1013, 1001, 1013, 1083, 1005, 1018, 1019, 1018, 1028, 1031, 1039, 1080, 1026, 1054, 996, 1018, 977, 1001, 1034, 1048, 1044, 1019, 1011, 1089, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1010, 1022, 984, 1053, 1026, 1067, 1022, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 996, 956, 1014, 959, 1076, 993, 1016, 1008, 37, 35, 9, 0, 0, 0, 0, 0, 1003, 1025, 1022, 984, 1053, 1005, 981, 1054, 13, 10, 8, 2, 0, 0, 0, 0, 1032, 1041, 1012, 1038, 1011, 995, 1007, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 974, 1065, 1036, 1056, 1027, 1078, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1055, 1051, 979, 1032, 1002, 1073, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1002, 1063, 1038, 990, 991, 970, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1048, 1051, 1015, 1039, 1055, 1037, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8099, 8160, 8155, 8263, 8250, 8121, 8247] avg=8192.0000 max=8263 max_rank=4 min=8099 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[999, 1044, 987, 1062, 1058, 964, 1044, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1057, 1008, 1007, 1001, 1021, 994, 1032, 39, 35, 10, 0, 0, 0, 0, 0, 1026, 979, 1029, 1004, 1016, 1002, 1012, 1061, 14, 10, 8, 2, 0, 0, 0, 0, 1064, 1061, 967, 1004, 1008, 995, 1062, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1040, 1005, 1001, 1044, 1013, 1002, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 984, 1020, 1041, 1018, 985, 1059, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1011, 1058, 1016, 992, 1060, 1006, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 986, 1020, 1020, 1003, 1025, 1041, 1066, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8250, 8163, 8200, 8208, 8168, 8188, 8192] avg=8192.0000 max=8250 max_rank=1 min=8163 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1035, 1046, 1003, 998, 1004, 1003, 1033, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 965, 1093, 1026, 1023, 997, 991, 981, 985, 38, 36, 9, 0, 0, 0, 0, 0, 954, 1082, 1042, 991, 998, 1063, 1008, 984, 13, 10, 8, 2, 0, 0, 0, 0, 1043, 1004, 1056, 1041, 1019, 1029, 1026, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1059, 1027, 1032, 1024, 1015, 1015, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 990, 1004, 999, 1023, 1066, 1047, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1064, 1037, 1043, 995, 1009, 1012, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1046, 1041, 1034, 1014, 1020, 1024, 980, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8144, 8155, 8239, 8228, 8192, 8223, 8198] avg=8192.0000 max=8239 max_rank=3 min=8144 min_rank=1 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1073, 1075, 1034, 1053, 1010, 1047, 1026, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1013, 1056, 996, 1005, 1003, 1020, 1052, 36, 34, 9, 0, 0, 0, 0, 0, 1002, 1038, 1032, 1026, 971, 1044, 1034, 998, 13, 10, 7, 2, 0, 0, 0, 0, 1022, 1029, 1029, 1020, 977, 985, 1027, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1049, 979, 1081, 1041, 1004, 987, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1024, 995, 1009, 1028, 1057, 1027, 959, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1008, 979, 1047, 981, 1039, 1027, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1094, 1037, 989, 1008, 1003, 1003, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8383, 8263, 8177, 8121, 8209, 8069, 8128, 8186] avg=8192.0000 max=8383 max_rank=0 min=8069 min_rank=5 imbalance_ratio=1.023315 max_load_reduction=57153 max_load_reduction_pct=0.872086 moved_to_echo=32592 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1048, 1007, 1051, 1014, 1034, 993, 1030, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1031, 1004, 961, 1041, 1025, 938, 1026, 38, 34, 9, 0, 0, 0, 0, 0, 986, 1042, 1050, 1094, 978, 997, 1016, 1031, 13, 10, 8, 2, 0, 0, 0, 0, 1056, 1071, 992, 1071, 1071, 1005, 1023, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1010, 1028, 1039, 1011, 1023, 999, 937, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 1032, 1004, 1059, 1052, 1029, 1020, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 988, 987, 1036, 1059, 1009, 1025, 1040, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1038, 1015, 996, 999, 1042, 1012, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8138, 8227, 8284, 8048, 8293, 8150, 8157] avg=8192.0000 max=8293 max_rank=5 min=8048 min_rank=4 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[965, 979, 964, 1066, 1027, 1069, 1028, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1000, 975, 985, 1042, 1058, 1013, 1025, 37, 34, 10, 0, 0, 0, 0, 0, 1023, 1067, 994, 985, 1025, 999, 1036, 1030, 13, 10, 8, 2, 0, 0, 0, 0, 1063, 997, 1021, 1019, 1055, 1045, 1026, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1002, 1044, 1056, 1008, 987, 1015, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1003, 1051, 1010, 1049, 1060, 997, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1026, 1035, 1056, 1025, 1011, 1051, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 985, 999, 987, 1006, 1048, 997, 1051, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8176, 8192, 8224, 8229, 8237, 8245, 8098] avg=8192.0000 max=8245 max_rank=6 min=8098 min_rank=7 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1007, 1018, 1055, 1034, 971, 1009, 1085, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 991, 1010, 1005, 987, 1031, 1041, 1032, 36, 35, 9, 0, 0, 0, 0, 0, 1037, 996, 997, 1011, 1032, 1071, 1047, 1026, 13, 10, 8, 2, 0, 0, 0, 0, 991, 1004, 1047, 994, 1005, 1028, 994, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 962, 1009, 1025, 998, 1068, 1035, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1010, 1037, 1057, 1022, 1077, 1015, 976, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1016, 1061, 1067, 1052, 1050, 990, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1039, 1009, 1014, 1051, 1057, 976, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8189, 8250, 8099, 8152, 8182, 8271, 8216] avg=8192.0000 max=8271 max_rank=6 min=8099 min_rank=3 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[972, 1011, 1023, 1003, 996, 979, 1011, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1039, 1011, 1006, 1020, 1000, 1035, 1029, 40, 39, 15, 0, 0, 0, 0, 0, 975, 999, 1012, 1030, 1042, 1005, 1054, 1047, 17, 15, 11, 5, 0, 0, 0, 0, 1031, 985, 1019, 1053, 1011, 1047, 1018, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1007, 1011, 1026, 1048, 1048, 1014, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1013, 1083, 1005, 1018, 1019, 1018, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1039, 1069, 1026, 1054, 996, 1018, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1034, 1048, 1005, 1019, 1011, 1089, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8037, 8277, 8212, 8214, 8199, 8145, 8210, 8242] avg=8192.0000 max=8277 max_rank=1 min=8037 min_rank=0 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8448, 8448, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8064, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8192, 8192, 8192, 8192, 8320, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8064, 8192, 8192, 8064, 8192, 384, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[999, 1050, 1032, 995, 1052, 1035, 974, 1016, 1042, 1046, 968, 1048, 1003, 993, 1018, 1021, 978, 1010, 1050, 1022, 1029, 1027, 1008, 1023, 1048, 1049, 1050, 1034, 996, 978, 1019, 1027, 1073, 1028, 1062, 959, 988, 999, 1036, 1026, 1011, 1012, 1029, 1006, 1057, 1031, 990, 1038, 1022, 1063, 1017, 1029, 1023, 1105, 1002, 1002, 1059, 1028, 1030, 1074, 1006, 1068, 1000, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1023, 1033, 999, 997, 1062, 1036, 1082, 1009, 1080, 1018, 1027, 1048, 1029, 992, 1021, 1021, 997, 1051, 1031, 1000, 1013, 1006, 988, 991, 1062, 1049, 1056, 1021, 1013, 1021, 996, 1083, 989, 990, 962, 1004, 1035, 975, 1075, 1024, 1030, 1022, 1072, 1001, 990, 1000, 1057, 1018, 1056, 1033, 985, 1079, 1024, 1050, 1001, 989, 1059, 967, 1065, 1053, 1023, 1017, 1042, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1034, 998, 942, 1047, 992, 1014, 1003, 1022, 986, 1003, 1047, 1047, 1005, 1037, 1045, 1085, 1049, 988, 1041, 1001, 1016, 1053, 1083, 1009, 1011, 1025, 1021, 1041, 1049, 1064, 1025, 1013, 1019, 1053, 994, 1019, 1086, 985, 1029, 1015, 944, 1021, 1009, 999, 1013, 1019, 1053, 1017, 994, 1045, 1015, 1008, 1079, 1046, 1017, 1080, 1031, 1015, 1041, 1021, 1069, 1012, 1012, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[978, 1033, 1055, 1025, 1054, 1036, 1031, 1009, 1009, 1011, 1016, 1050, 1037, 1015, 981, 993, 1044, 1055, 968, 995, 997, 1017, 1049, 969, 1042, 1046, 1008, 1013, 1096, 1033, 1021, 1037, 1028, 1090, 1001, 1022, 1006, 1054, 994, 1000, 1062, 1021, 1024, 966, 1049, 1037, 1057, 1070, 990, 1035, 1025, 955, 1020, 1042, 1020, 1066, 981, 999, 1017, 1026, 1013, 1038, 1057, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[990, 1033, 985, 965, 1063, 1004, 1000, 1012, 992, 1078, 1015, 1063, 1045, 1016, 993, 1119, 1016, 1012, 1028, 945, 1044, 978, 1036, 1045, 1025, 1020, 1003, 1010, 1026, 989, 1029, 1031, 1037, 1030, 1061, 1028, 1028, 1016, 1048, 970, 1035, 1002, 1047, 1035, 1069, 1073, 1062, 1030, 1039, 1020, 1049, 1031, 1061, 984, 1022, 990, 1036, 1001, 963, 1038, 1047, 1035, 1039, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1022, 1053, 975, 990, 1014, 979, 1035, 1036, 1015, 1002, 1054, 1037, 1007, 1002, 1000, 1011, 1066, 1063, 1002, 1041, 1065, 1013, 1059, 1070, 1040, 1053, 993, 1046, 1074, 1016, 1049, 1009, 1003, 1020, 1043, 1001, 1002, 981, 1046, 1000, 1035, 986, 993, 1060, 1040, 967, 969, 1033, 1045, 1036, 1077, 1009, 982, 1043, 1002, 983, 1026, 1042, 983, 1080, 1047, 1066, 1032, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[993, 1054, 1027, 1047, 995, 1027, 984, 1026, 1018, 1087, 1050, 1003, 1027, 1031, 972, 1073, 1032, 1040, 1026, 994, 971, 990, 1056, 1033, 1022, 980, 996, 1008, 1046, 1021, 1054, 1015, 1055, 1078, 1036, 999, 983, 1000, 1033, 1060, 1057, 1028, 984, 1021, 1044, 1009, 1021, 1018, 1047, 1006, 1037, 1018, 1036, 1070, 1013, 1072, 1007, 1000, 1034, 1005, 999, 1008, 1039, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1054, 983, 1057, 1023, 1069, 1008, 987, 1026, 995, 1058, 1029, 968, 994, 997, 992, 1017, 1050, 1055, 1065, 1045, 1061, 1009, 1009, 988, 983, 1020, 1055, 1048, 1053, 1017, 1012, 1047, 992, 1048, 1030, 1026, 988, 995, 1042, 1013, 1016, 1068, 1026, 1022, 1000, 998, 1022, 1065, 1036, 1036, 1012, 1009, 1018, 1033, 1052, 1023, 1006, 999, 1036, 993, 1018, 1063, 1060, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[999, 1050, 1032, 995, 1052, 1035, 974, 1016, 35, 6, 0, 0, 0, 0, 0, 0, 1042, 1046, 968, 1048, 1003, 993, 1018, 1021, 3, 1, 0, 0, 0, 0, 0, 0, 978, 1010, 1050, 1022, 1029, 1027, 1008, 1023, 9, 0, 0, 0, 0, 0, 0, 0, 1048, 1049, 1050, 1034, 961, 978, 1019, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1028, 1062, 959, 988, 999, 1036, 1026, 29, 0, 0, 0, 0, 0, 0, 0, 1011, 1012, 1029, 1006, 1057, 1031, 990, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1063, 1017, 1029, 1023, 1070, 1002, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1028, 1030, 1074, 1006, 1058, 1000, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8143, 8156, 8166, 8200, 8171, 8228, 8278] avg=8192.0000 max=8278 max_rank=7 min=8143 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1023, 1033, 999, 997, 1062, 1036, 1082, 1009, 36, 6, 0, 0, 0, 0, 0, 0, 1080, 1018, 1027, 1048, 1029, 992, 1021, 1021, 2, 1, 0, 0, 0, 0, 0, 0, 997, 1051, 1031, 1000, 1013, 1006, 988, 991, 8, 0, 0, 0, 0, 0, 0, 0, 1062, 1049, 1056, 1021, 977, 1021, 996, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 989, 990, 962, 1004, 1035, 975, 1075, 1024, 27, 0, 0, 0, 0, 0, 0, 0, 1030, 1022, 1072, 1001, 990, 1000, 1057, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1033, 985, 1079, 1024, 1017, 1001, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 967, 1065, 1053, 1023, 1008, 1042, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8239, 8085, 8265, 8081, 8188, 8184, 8211] avg=8192.0000 max=8283 max_rank=0 min=8081 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[990, 1033, 985, 965, 1063, 1004, 1000, 1012, 36, 6, 0, 0, 0, 0, 0, 0, 992, 1078, 1015, 1063, 1045, 1016, 993, 1119, 2, 1, 0, 0, 0, 0, 0, 0, 1016, 1012, 1028, 945, 1044, 978, 1036, 1045, 8, 0, 0, 0, 0, 0, 0, 0, 1025, 1020, 1003, 1010, 990, 989, 1029, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1030, 1061, 1028, 1028, 1016, 1048, 970, 25, 0, 0, 0, 0, 0, 0, 0, 1035, 1002, 1047, 1035, 1069, 1073, 1062, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1020, 1049, 1031, 1061, 953, 1022, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1001, 963, 1038, 1047, 1026, 1039, 1000, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8094, 8324, 8112, 8097, 8243, 8351, 8165, 8150] avg=8192.0000 max=8351 max_rank=5 min=8094 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[978, 1033, 1055, 1025, 1054, 1036, 1031, 1009, 39, 6, 0, 0, 0, 0, 0, 0, 1009, 1011, 1016, 1050, 1037, 1015, 981, 993, 3, 1, 0, 0, 0, 0, 0, 0, 1044, 1055, 968, 995, 997, 1017, 1049, 969, 8, 0, 0, 0, 0, 0, 0, 0, 1042, 1046, 1008, 1013, 1057, 1033, 1021, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1090, 1001, 1022, 1006, 1054, 994, 1000, 27, 0, 0, 0, 0, 0, 0, 0, 1062, 1021, 1024, 966, 1049, 1037, 1057, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1035, 1025, 955, 1020, 1009, 1020, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 981, 999, 1017, 1026, 1013, 1029, 1057, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8116, 8102, 8257, 8222, 8283, 8120, 8170] avg=8192.0000 max=8283 max_rank=5 min=8102 min_rank=2 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1034, 998, 942, 1047, 992, 1014, 1003, 1022, 37, 6, 0, 0, 0, 0, 0, 0, 986, 1003, 1047, 1047, 1005, 1037, 1045, 1085, 2, 1, 0, 0, 0, 0, 0, 0, 1049, 988, 1041, 1001, 1016, 1053, 1083, 1009, 8, 0, 0, 0, 0, 0, 0, 0, 1011, 1025, 1021, 1041, 1012, 1064, 1025, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1053, 994, 1019, 1086, 985, 1029, 1015, 27, 0, 0, 0, 0, 0, 0, 0, 944, 1021, 1009, 999, 1013, 1019, 1053, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1045, 1015, 1008, 1079, 1013, 1017, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1015, 1041, 1021, 1069, 1003, 1012, 980, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8258, 8248, 8212, 8227, 8073, 8251, 8172] avg=8192.0000 max=8258 max_rank=1 min=8073 min_rank=5 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1022, 1053, 975, 990, 1014, 979, 1035, 1036, 38, 6, 0, 0, 0, 0, 0, 0, 1015, 1002, 1054, 1037, 1007, 1002, 1000, 1011, 2, 1, 0, 0, 0, 0, 0, 0, 1066, 1063, 1002, 1041, 1065, 1013, 1059, 1070, 9, 0, 0, 0, 0, 0, 0, 0, 1040, 1053, 993, 1046, 1036, 1016, 1049, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1020, 1043, 1001, 1002, 981, 1046, 1000, 27, 0, 0, 0, 0, 0, 0, 0, 1035, 986, 993, 1060, 1040, 967, 969, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1036, 1077, 1009, 982, 1010, 1002, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1042, 983, 1080, 1047, 1056, 1032, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8131, 8388, 8242, 8123, 8081, 8144, 8279] avg=8192.0000 max=8388 max_rank=2 min=8081 min_rank=5 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[993, 1054, 1027, 1047, 995, 1027, 984, 1026, 37, 6, 0, 0, 0, 0, 0, 0, 1018, 1087, 1050, 1003, 1027, 1031, 972, 1073, 2, 1, 0, 0, 0, 0, 0, 0, 1032, 1040, 1026, 994, 971, 990, 1056, 1033, 8, 0, 0, 0, 0, 0, 0, 0, 1022, 980, 996, 1008, 1009, 1021, 1054, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1078, 1036, 999, 983, 1000, 1033, 1060, 28, 0, 0, 0, 0, 0, 0, 0, 1057, 1028, 984, 1021, 1044, 1009, 1021, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1006, 1037, 1018, 1036, 1036, 1013, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1000, 1034, 1005, 999, 999, 1039, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8264, 8150, 8105, 8272, 8180, 8265, 8104] avg=8192.0000 max=8272 max_rank=4 min=8104 min_rank=7 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1054, 983, 1057, 1023, 1069, 1008, 987, 1026, 43, 10, 0, 0, 0, 0, 0, 0, 995, 1058, 1029, 968, 994, 997, 992, 1017, 8, 3, 0, 0, 0, 0, 0, 0, 1050, 1055, 1065, 1045, 1061, 1009, 1009, 988, 13, 0, 0, 0, 0, 0, 0, 0, 983, 1020, 1055, 1048, 1010, 1017, 1012, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1048, 1030, 1026, 988, 995, 1042, 1013, 30, 4, 0, 0, 0, 0, 0, 0, 1016, 1068, 1026, 1022, 1000, 998, 1022, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1036, 1012, 1009, 1018, 993, 1052, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 999, 1036, 993, 1018, 1043, 1060, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8061, 8295, 8192, 8168, 8209, 8179, 8172] avg=8192.0000 max=8295 max_rank=2 min=8061 min_rank=1 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8064, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8448, 8192, 8064, 8192, 8064, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8064, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8064, 8448, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1065, 1060, 1025, 1052, 955, 1010, 1013, 980, 1032, 1082, 1025, 987, 1008, 987, 1050, 1006, 1075, 1009, 1058, 1010, 1059, 983, 1006, 1026, 1023, 1071, 996, 977, 1054, 1013, 989, 1013, 1001, 1039, 1045, 960, 962, 1052, 1034, 1014, 1024, 1033, 998, 1045, 1054, 1042, 1036, 1031, 1022, 1059, 998, 997, 1038, 998, 1031, 991, 1065, 1010, 1055, 1019, 1036, 1082, 1065, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1039, 996, 1044, 1016, 1026, 992, 1023, 1064, 990, 1003, 1075, 1041, 993, 1070, 1060, 1031, 1046, 1010, 1122, 935, 1008, 1022, 963, 999, 1032, 1083, 1011, 1042, 994, 1053, 1009, 972, 1000, 1066, 1048, 1084, 989, 1018, 1010, 997, 1010, 1092, 1038, 1069, 977, 1018, 1048, 1022, 1013, 1061, 988, 968, 1036, 1017, 1037, 971, 1036, 1004, 993, 1031, 1024, 1031, 1019, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1013, 1039, 1004, 1013, 1035, 1016, 1043, 1053, 1083, 991, 1024, 1018, 1079, 1033, 1020, 1113, 1051, 977, 1063, 1060, 1067, 1074, 1014, 1019, 1044, 1044, 1001, 984, 993, 1031, 1037, 1018, 1045, 1021, 1023, 1022, 1009, 1040, 997, 1019, 976, 1041, 1068, 944, 1032, 1015, 985, 969, 1032, 1033, 1021, 1012, 1032, 1032, 1068, 998, 994, 1005, 1050, 987, 1043, 994, 986, 989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1004, 1029, 972, 994, 1028, 995, 981, 1007, 1024, 1057, 1027, 1015, 1078, 1004, 986, 1055, 1029, 1014, 1031, 1043, 1014, 991, 1031, 1067, 1007, 988, 1072, 951, 1103, 1072, 1038, 995, 1012, 1032, 998, 1020, 1050, 1018, 1052, 996, 1032, 1019, 1038, 1049, 996, 1044, 1023, 981, 1041, 1056, 1046, 1015, 1037, 986, 983, 990, 1091, 1068, 1000, 999, 1095, 1028, 1026, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1062, 1045, 1002, 977, 992, 1027, 1028, 993, 969, 1016, 1012, 1073, 1073, 1021, 1031, 1039, 978, 1040, 1016, 961, 989, 1013, 1000, 1039, 1022, 1026, 1041, 1018, 1060, 1018, 1073, 1008, 1036, 1049, 1083, 1020, 971, 1014, 1049, 1011, 988, 1055, 1020, 996, 1034, 1015, 990, 1047, 1079, 1067, 1018, 1057, 1042, 1040, 1021, 1012, 997, 1029, 1012, 1001, 1038, 1014, 1009, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1055, 1010, 966, 1008, 1045, 1012, 1023, 1022, 974, 1056, 1041, 967, 1014, 1008, 990, 1075, 1016, 1000, 1002, 1003, 1012, 1065, 1055, 973, 1014, 1022, 1082, 1009, 1047, 1052, 1070, 1029, 1053, 1013, 1039, 1052, 1009, 1008, 1046, 1050, 1039, 991, 1044, 1031, 1027, 982, 1005, 1080, 1010, 1010, 976, 1008, 1039, 994, 1063, 1024, 1036, 1050, 1032, 1033, 1005, 1011, 1009, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1046, 1000, 1036, 1043, 1060, 988, 1049, 1049, 1000, 1049, 1071, 1016, 1017, 1046, 1009, 1008, 985, 1018, 1024, 1039, 1087, 957, 980, 983, 1036, 1050, 1003, 1002, 1044, 1060, 1030, 1008, 975, 1005, 1072, 1033, 1020, 1001, 1035, 1031, 989, 1045, 1040, 1061, 1000, 993, 1019, 1067, 1039, 1036, 978, 1023, 990, 1027, 1058, 968, 1055, 1000, 1052, 1046, 1100, 948, 1021, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1024, 1023, 1036, 1070, 1057, 1038, 1006, 1000, 1059, 1030, 1030, 1022, 1037, 1053, 1057, 1017, 1024, 1033, 1021, 996, 1019, 953, 1003, 1082, 1006, 964, 1032, 996, 992, 997, 955, 1039, 1045, 1021, 1008, 1027, 992, 1063, 1014, 983, 997, 973, 1011, 1056, 999, 1057, 1085, 1039, 1062, 1008, 1013, 1042, 1007, 999, 1022, 1080, 1048, 1009, 1050, 1057, 1005, 1011, 1062, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1065, 1060, 1025, 1052, 955, 1010, 1013, 980, 11, 7, 0, 0, 0, 0, 0, 0, 1032, 1082, 1025, 987, 1008, 987, 1050, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1009, 1058, 1010, 1059, 983, 1006, 1026, 35, 0, 0, 0, 0, 0, 0, 0, 1023, 1071, 996, 977, 1054, 1013, 989, 1013, 2, 0, 0, 0, 0, 0, 0, 0, 1001, 1039, 1045, 960, 962, 1052, 1034, 1014, 4, 0, 0, 0, 0, 0, 0, 0, 1024, 1033, 998, 1045, 1054, 1042, 1036, 1031, 1, 0, 0, 0, 0, 0, 0, 0, 1022, 1059, 998, 997, 1038, 998, 1031, 991, 14, 0, 0, 0, 0, 0, 0, 0, 1065, 1010, 1055, 1019, 1008, 1082, 1065, 1001, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8131, 8261, 8138, 8111, 8264, 8148, 8305] avg=8192.0000 max=8305 max_rank=7 min=8111 min_rank=4 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1013, 1039, 1004, 1013, 1035, 1016, 1043, 1053, 12, 7, 0, 0, 0, 0, 0, 0, 1083, 991, 1024, 1018, 1079, 1033, 1020, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 977, 1063, 1060, 1067, 1074, 1014, 1019, 39, 0, 0, 0, 0, 0, 0, 0, 1044, 1044, 1001, 984, 993, 1031, 1037, 1018, 2, 0, 0, 0, 0, 0, 0, 0, 1045, 1021, 1023, 1022, 1009, 1040, 997, 1019, 4, 0, 0, 0, 0, 0, 0, 0, 976, 1041, 1068, 944, 1032, 1015, 985, 969, 1, 0, 0, 0, 0, 0, 0, 0, 1032, 1033, 1021, 1012, 1032, 1032, 1068, 998, 14, 0, 0, 0, 0, 0, 0, 0, 994, 1005, 1050, 987, 1015, 994, 986, 989, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8310, 8364, 8154, 8180, 8031, 8242, 8020] avg=8192.0000 max=8364 max_rank=2 min=8020 min_rank=7 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1046, 1000, 1036, 1043, 1060, 988, 1049, 1049, 11, 8, 0, 0, 0, 0, 0, 0, 1000, 1049, 1071, 1016, 1017, 1046, 1009, 962, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1018, 1024, 1039, 1087, 957, 980, 983, 35, 0, 0, 0, 0, 0, 0, 0, 1036, 1050, 1003, 1002, 1044, 1060, 1030, 1008, 2, 0, 0, 0, 0, 0, 0, 0, 975, 1005, 1072, 1033, 1020, 1001, 1035, 1031, 4, 0, 0, 0, 0, 0, 0, 0, 989, 1045, 1040, 1061, 1000, 993, 1019, 1067, 1, 0, 0, 0, 0, 0, 0, 0, 1039, 1036, 978, 1023, 990, 1027, 1058, 968, 15, 0, 0, 0, 0, 0, 0, 0, 1055, 1000, 1052, 1046, 1070, 948, 1021, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 8170, 8108, 8235, 8176, 8215, 8134, 8208] avg=8192.0000 max=8290 max_rank=0 min=8108 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1039, 996, 1044, 1016, 1026, 992, 1023, 1064, 11, 7, 0, 0, 0, 0, 0, 0, 990, 1003, 1075, 1041, 993, 1070, 1060, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1010, 1122, 935, 1008, 1022, 963, 999, 36, 0, 0, 0, 0, 0, 0, 0, 1032, 1083, 1011, 1042, 994, 1053, 1009, 972, 2, 0, 0, 0, 0, 0, 0, 0, 1000, 1066, 1048, 1084, 989, 1018, 1010, 997, 4, 0, 0, 0, 0, 0, 0, 0, 1010, 1092, 1038, 1069, 977, 1018, 1048, 1022, 1, 0, 0, 0, 0, 0, 0, 0, 1013, 1061, 988, 968, 1036, 1017, 1037, 971, 14, 0, 0, 0, 0, 0, 0, 0, 1036, 1004, 993, 1031, 996, 1031, 1019, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8216, 8141, 8198, 8216, 8275, 8105, 8167] avg=8192.0000 max=8275 max_rank=5 min=8105 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1004, 1029, 972, 994, 1028, 995, 981, 1007, 12, 8, 0, 0, 0, 0, 0, 0, 1024, 1057, 1027, 1015, 1078, 1004, 986, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1014, 1031, 1043, 1014, 991, 1031, 1067, 37, 0, 0, 0, 0, 0, 0, 0, 1007, 988, 1072, 951, 1103, 1072, 1038, 995, 2, 0, 0, 0, 0, 0, 0, 0, 1012, 1032, 998, 1020, 1050, 1018, 1052, 996, 4, 0, 0, 0, 0, 0, 0, 0, 1032, 1019, 1038, 1049, 996, 1044, 1023, 981, 1, 0, 0, 0, 0, 0, 0, 0, 1041, 1056, 1046, 1015, 1037, 986, 983, 990, 15, 0, 0, 0, 0, 0, 0, 0, 1091, 1068, 1000, 999, 1065, 1028, 1026, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8030, 8197, 8257, 8228, 8182, 8183, 8169, 8290] avg=8192.0000 max=8290 max_rank=7 min=8030 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1062, 1045, 1002, 977, 992, 1027, 1028, 993, 12, 7, 0, 0, 0, 0, 0, 0, 969, 1016, 1012, 1073, 1073, 1021, 1031, 991, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1040, 1016, 961, 989, 1013, 1000, 1039, 36, 0, 0, 0, 0, 0, 0, 0, 1022, 1026, 1041, 1018, 1060, 1018, 1073, 1008, 2, 0, 0, 0, 0, 0, 0, 0, 1036, 1049, 1083, 1020, 971, 1014, 1049, 1011, 4, 0, 0, 0, 0, 0, 0, 0, 988, 1055, 1020, 996, 1034, 1015, 990, 1047, 1, 0, 0, 0, 0, 0, 0, 0, 1079, 1067, 1018, 1057, 1042, 1040, 1021, 1012, 14, 0, 0, 0, 0, 0, 0, 0, 997, 1029, 1012, 1001, 1010, 1014, 1009, 1060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8186, 8072, 8268, 8237, 8146, 8350, 8132] avg=8192.0000 max=8350 max_rank=6 min=8072 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1055, 1010, 966, 1008, 1045, 1012, 1023, 1022, 12, 7, 0, 0, 0, 0, 0, 0, 974, 1056, 1041, 967, 1014, 1008, 990, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1000, 1002, 1003, 1012, 1065, 1055, 973, 37, 0, 0, 0, 0, 0, 0, 0, 1014, 1022, 1082, 1009, 1047, 1052, 1070, 1029, 2, 0, 0, 0, 0, 0, 0, 0, 1053, 1013, 1039, 1052, 1009, 1008, 1046, 1050, 4, 0, 0, 0, 0, 0, 0, 0, 1039, 991, 1044, 1031, 1027, 982, 1005, 1080, 1, 0, 0, 0, 0, 0, 0, 0, 1010, 1010, 976, 1008, 1039, 994, 1063, 1024, 14, 0, 0, 0, 0, 0, 0, 0, 1036, 1050, 1032, 1033, 977, 1011, 1009, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8076, 8163, 8327, 8274, 8200, 8138, 8198] avg=8192.0000 max=8327 max_rank=3 min=8076 min_rank=1 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1024, 1023, 1036, 1070, 1057, 1038, 1006, 1000, 16, 10, 0, 0, 0, 0, 0, 0, 1059, 1030, 1030, 1022, 1037, 1053, 1057, 962, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1033, 1021, 996, 1019, 953, 1003, 1082, 39, 0, 0, 0, 0, 0, 0, 0, 1006, 964, 1032, 996, 992, 997, 955, 1039, 7, 0, 0, 0, 0, 0, 0, 0, 1045, 1021, 1008, 1027, 992, 1063, 1014, 983, 7, 0, 0, 0, 0, 0, 0, 0, 997, 973, 1011, 1056, 999, 1057, 1085, 1039, 5, 0, 0, 0, 0, 0, 0, 0, 1062, 1008, 1013, 1042, 1007, 999, 1022, 1080, 17, 0, 0, 0, 0, 0, 0, 0, 1048, 1009, 1050, 1057, 959, 1011, 1062, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8280, 8250, 8170, 7988, 8160, 8222, 8250, 8216] avg=8192.0000 max=8280 max_rank=0 min=7988 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8448, 8064, 8320, 8064, 8064, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8448, 8064, 8192, 8320, 8192, 8320, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8064, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8064, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[994, 972, 1040, 1001, 968, 1034, 989, 1027, 1012, 1032, 982, 1018, 1060, 982, 1052, 998, 988, 1074, 1057, 997, 972, 1002, 990, 1047, 1010, 1029, 1003, 959, 1029, 1019, 1018, 1064, 1046, 1038, 1097, 1062, 1062, 1068, 1053, 1009, 1045, 1020, 1086, 988, 969, 1023, 1016, 963, 1032, 1074, 1008, 1038, 994, 1011, 1048, 1004, 1016, 1085, 1026, 1044, 1121, 1033, 1035, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[998, 1013, 1017, 1062, 1065, 975, 963, 1070, 1026, 1042, 1049, 1017, 1015, 1017, 998, 1039, 993, 1009, 1004, 1010, 1065, 1046, 1008, 997, 1049, 975, 1034, 999, 1035, 1036, 1034, 1056, 1019, 1055, 1011, 1009, 1068, 1030, 1039, 1074, 1023, 1004, 1043, 1019, 1046, 1061, 1021, 1056, 1008, 993, 1044, 1040, 988, 972, 978, 1068, 1045, 1025, 1001, 1026, 1032, 1004, 993, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1035, 1037, 1017, 1080, 1024, 1021, 1005, 1058, 1047, 999, 1031, 994, 1023, 983, 1027, 1011, 1038, 1011, 1068, 1015, 1019, 1013, 1022, 1031, 1003, 1038, 1049, 1012, 1015, 999, 1013, 1050, 1055, 997, 1080, 1050, 1044, 1001, 976, 1052, 1048, 1019, 977, 1036, 1024, 1019, 1013, 1070, 1061, 1072, 1065, 1015, 969, 1013, 1003, 1051, 1054, 998, 996, 951, 1004, 1029, 985, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[971, 1013, 1034, 1018, 1034, 1026, 1033, 1025, 1007, 1023, 1025, 1083, 1036, 984, 1035, 1026, 994, 1018, 1029, 1042, 1038, 1010, 1042, 1065, 1040, 1038, 1022, 1048, 1065, 958, 1008, 1031, 1001, 1002, 1044, 1035, 1028, 999, 1012, 1016, 1045, 1027, 1021, 1076, 1046, 1028, 1017, 1040, 991, 986, 973, 1029, 1002, 996, 1019, 1022, 1044, 1019, 1021, 1028, 1046, 1004, 1047, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[987, 1025, 1042, 1024, 1055, 1017, 1049, 1066, 1040, 1049, 1023, 997, 1045, 1039, 1013, 1035, 1020, 999, 1007, 991, 1005, 958, 1038, 1055, 1022, 1046, 1021, 1051, 987, 997, 1035, 1029, 1059, 1025, 1015, 1029, 1090, 1071, 1066, 1028, 995, 997, 1006, 1050, 1066, 1055, 1027, 981, 1008, 981, 983, 1054, 1029, 1032, 1038, 1064, 975, 1012, 1002, 1050, 952, 1004, 1056, 969] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1037, 1071, 1032, 975, 1047, 1051, 980, 1018, 1043, 1003, 1031, 1005, 1080, 986, 1037, 1057, 1034, 1024, 989, 997, 974, 1045, 1015, 1036, 1019, 1000, 1007, 1036, 1071, 1049, 1035, 1038, 1038, 1035, 1036, 1018, 1018, 1013, 1012, 1000, 991, 985, 1034, 1046, 1070, 1046, 1015, 982, 1024, 1041, 1039, 1011, 993, 998, 1008, 1000, 1059, 1003, 973, 1019, 1093, 1083, 990, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1044, 1008, 1016, 1013, 1039, 980, 1088, 1040, 1015, 1014, 1026, 984, 1033, 1001, 1060, 1020, 1027, 1012, 1016, 1086, 1040, 1000, 1016, 1052, 1068, 1025, 1040, 1020, 1049, 992, 1037, 1002, 1000, 1005, 1019, 1002, 989, 1003, 1048, 997, 1048, 1042, 973, 1006, 1010, 1007, 1034, 1095, 983, 1037, 1006, 1048, 1050, 1023, 1014, 1030, 1040, 1038, 1023, 985, 1005, 1075, 1018, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1003, 1003, 1062, 996, 1042, 1022, 1023, 1007, 1044, 1069, 1011, 1015, 1005, 1014, 1042, 1013, 1002, 1015, 994, 1003, 1008, 1100, 1067, 1025, 990, 1016, 1036, 1032, 1009, 1026, 1045, 1018, 1073, 1024, 1060, 1020, 1033, 1076, 1044, 968, 1020, 1020, 1001, 1040, 1046, 977, 1040, 1054, 1003, 1038, 1072, 996, 1006, 1006, 1040, 984, 1066, 1031, 968, 1004, 1046, 1015, 982, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[998, 1013, 1017, 1062, 1065, 975, 963, 1070, 4, 2, 0, 0, 0, 0, 0, 0, 1026, 1042, 1049, 1017, 1015, 1017, 998, 1039, 1, 0, 0, 0, 0, 0, 0, 0, 993, 1009, 1004, 1010, 1065, 1046, 1008, 997, 20, 0, 0, 0, 0, 0, 0, 0, 1049, 975, 1034, 999, 1035, 1036, 1034, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1055, 952, 1009, 1068, 1030, 1039, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1004, 1043, 1019, 1033, 1061, 1021, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 993, 1044, 1040, 988, 972, 978, 1068, 39, 0, 0, 0, 0, 0, 0, 0, 1045, 1025, 1001, 1026, 1032, 1004, 993, 1025, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8204, 8152, 8213, 8246, 8260, 8130, 8162] avg=8192.0000 max=8260 max_rank=5 min=8130 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[994, 972, 1040, 1001, 968, 1034, 989, 1027, 4, 2, 0, 0, 0, 0, 0, 0, 1012, 1032, 982, 1018, 1060, 982, 1052, 998, 1, 0, 0, 0, 0, 0, 0, 0, 988, 1074, 1057, 997, 972, 1002, 990, 1047, 22, 0, 0, 0, 0, 0, 0, 0, 1010, 1029, 1003, 959, 1029, 1019, 1018, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1038, 1031, 1062, 1062, 1068, 1053, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1020, 1086, 988, 957, 1023, 1016, 963, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1074, 1008, 1038, 994, 1011, 1048, 1004, 43, 0, 0, 0, 0, 0, 0, 0, 1016, 1085, 1026, 1044, 1121, 1033, 1035, 1003, 10, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8031, 8137, 8149, 8126, 8369, 8098, 8252, 8374] avg=8192.0000 max=8374 max_rank=7 min=8031 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=57162 max_load_reduction_pct=0.872223 moved_to_echo=33093 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[971, 1013, 1034, 1018, 1034, 1026, 1033, 1025, 4, 2, 0, 0, 0, 0, 0, 0, 1007, 1023, 1025, 1083, 1036, 984, 1035, 1026, 1, 0, 0, 0, 0, 0, 0, 0, 994, 1018, 1029, 1042, 1038, 1010, 1042, 1065, 21, 0, 0, 0, 0, 0, 0, 0, 1040, 1038, 1022, 1048, 1065, 958, 1008, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1002, 982, 1035, 1028, 999, 1012, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1027, 1021, 1076, 1033, 1028, 1017, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 991, 986, 973, 1029, 1002, 996, 1019, 1022, 41, 0, 0, 0, 0, 0, 0, 0, 1044, 1019, 1021, 1028, 1046, 1004, 1047, 1051, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8220, 8259, 8205, 8075, 8287, 8059, 8271] avg=8192.0000 max=8287 max_rank=5 min=8059 min_rank=6 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[987, 1025, 1042, 1024, 1055, 1017, 1049, 1066, 4, 2, 0, 0, 0, 0, 0, 0, 1040, 1049, 1023, 997, 1045, 1039, 1013, 1035, 1, 0, 0, 0, 0, 0, 0, 0, 1020, 999, 1007, 991, 1005, 958, 1038, 1055, 20, 0, 0, 0, 0, 0, 0, 0, 1022, 1046, 1021, 1051, 987, 997, 1035, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1025, 955, 1029, 1090, 1071, 1066, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 995, 997, 1006, 1050, 1053, 1055, 1027, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 981, 983, 1054, 1029, 1032, 1038, 1064, 40, 0, 0, 0, 0, 0, 0, 0, 975, 1012, 1002, 1050, 952, 1004, 1056, 969, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8242, 8093, 8183, 8323, 8164, 8229, 8031] avg=8192.0000 max=8323 max_rank=4 min=8031 min_rank=7 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1035, 1037, 1017, 1080, 1024, 1021, 1005, 1058, 4, 2, 0, 0, 0, 0, 0, 0, 1047, 999, 1031, 994, 1023, 983, 1027, 1011, 1, 0, 0, 0, 0, 0, 0, 0, 1038, 1011, 1068, 1015, 1019, 1013, 1022, 1031, 22, 0, 0, 0, 0, 0, 0, 0, 1003, 1038, 1049, 1012, 1015, 999, 1013, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 997, 1015, 1050, 1044, 1001, 976, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1019, 977, 1036, 1011, 1019, 1013, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1072, 1065, 1015, 969, 1013, 1003, 1051, 42, 0, 0, 0, 0, 0, 0, 0, 1054, 998, 996, 951, 1004, 1029, 985, 1021, 11, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8116, 8239, 8174, 8190, 8193, 8291, 8050] avg=8192.0000 max=8291 max_rank=6 min=8050 min_rank=7 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1044, 1008, 1016, 1013, 1039, 980, 1088, 1040, 4, 2, 0, 0, 0, 0, 0, 0, 1015, 1014, 1026, 984, 1033, 1001, 1060, 1020, 1, 0, 0, 0, 0, 0, 0, 0, 1027, 1012, 1016, 1086, 1040, 1000, 1016, 1052, 20, 0, 0, 0, 0, 0, 0, 0, 1068, 1025, 1040, 1020, 1049, 992, 1037, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1005, 959, 1002, 989, 1003, 1048, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1042, 973, 1006, 997, 1007, 1034, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1037, 1006, 1048, 1050, 1023, 1014, 1030, 40, 0, 0, 0, 0, 0, 0, 0, 1040, 1038, 1023, 985, 1005, 1075, 1018, 1020, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8154, 8269, 8228, 8003, 8202, 8231, 8215] avg=8192.0000 max=8269 max_rank=2 min=8003 min_rank=4 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1037, 1071, 1032, 975, 1047, 1051, 980, 1018, 4, 2, 0, 0, 0, 0, 0, 0, 1043, 1003, 1031, 1005, 1080, 986, 1037, 1057, 1, 0, 0, 0, 0, 0, 0, 0, 1034, 1024, 989, 997, 974, 1045, 1015, 1036, 21, 0, 0, 0, 0, 0, 0, 0, 1019, 1000, 1007, 1036, 1071, 1049, 1035, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1035, 975, 1018, 1018, 1013, 1012, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 991, 985, 1034, 1046, 1056, 1046, 1015, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1041, 1039, 1011, 993, 998, 1008, 1000, 40, 0, 0, 0, 0, 0, 0, 0, 1059, 1003, 973, 1019, 1093, 1083, 990, 1041, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8243, 8135, 8250, 8109, 8155, 8154, 8273] avg=8192.0000 max=8273 max_rank=7 min=8109 min_rank=4 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1003, 1003, 1062, 996, 1042, 1022, 1023, 1007, 8, 5, 0, 0, 0, 0, 0, 0, 1044, 1069, 1011, 1015, 1005, 1014, 1042, 1013, 7, 0, 0, 0, 0, 0, 0, 0, 1002, 1015, 994, 1003, 1008, 1100, 1067, 1025, 26, 0, 0, 0, 0, 0, 0, 0, 990, 1016, 1036, 1032, 1009, 1026, 1045, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1024, 983, 1020, 1033, 1076, 1044, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1020, 1001, 1040, 1025, 977, 1040, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1038, 1072, 996, 1006, 1006, 1040, 984, 45, 0, 0, 0, 0, 0, 0, 0, 1066, 1031, 968, 1004, 1046, 1015, 982, 1026, 16, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8220, 8240, 8157, 8221, 8177, 8190, 8160] avg=8192.0000 max=8240 max_rank=2 min=8157 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 7936, 8320, 8448, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8064, 8064, 8192, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8192, 8320, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8064, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1013, 1000, 1023, 1004, 1054, 958, 1003, 970, 1068, 982, 1009, 1014, 1067, 985, 999, 1021, 1015, 975, 998, 1091, 1011, 1067, 1046, 1023, 1052, 990, 1013, 1060, 959, 962, 1012, 1044, 1006, 1020, 996, 1046, 1038, 1039, 1022, 1053, 1043, 1034, 1046, 1026, 998, 1041, 1032, 1029, 1068, 1041, 1048, 990, 1052, 956, 1043, 1028, 995, 1035, 1098, 1066, 1051, 1069, 1026, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1067, 1016, 1005, 1032, 1024, 1023, 986, 979, 1060, 1055, 998, 1022, 1002, 1044, 1000, 1023, 1010, 1011, 1035, 1061, 1033, 1035, 994, 1027, 1055, 1057, 1056, 994, 994, 1022, 1015, 992, 1039, 991, 1034, 1019, 1041, 1055, 1017, 1046, 1013, 1032, 1009, 1008, 996, 1023, 1049, 1004, 1001, 1033, 1002, 1054, 1033, 1013, 1039, 1032, 1041, 1015, 1038, 1029, 1035, 1037, 1024, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1018, 1036, 1011, 1070, 1072, 1060, 990, 1026, 1016, 977, 1013, 1009, 1034, 1014, 990, 1021, 1038, 997, 1000, 1024, 982, 1030, 1039, 1054, 1018, 1062, 995, 1040, 1011, 1059, 992, 1077, 990, 1001, 1022, 1037, 984, 1021, 1018, 1024, 999, 971, 1032, 987, 1057, 995, 1015, 1038, 1041, 998, 1030, 1065, 1057, 1044, 1073, 1038, 1043, 1008, 1026, 1009, 1014, 1012, 1030, 1082] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[992, 1023, 1064, 1061, 987, 1023, 1006, 1062, 969, 980, 1030, 973, 1015, 1045, 1072, 1012, 1028, 1070, 1010, 1065, 1015, 1045, 1039, 998, 1015, 1027, 1022, 1036, 1083, 973, 1028, 1021, 986, 1015, 1081, 1074, 1032, 999, 983, 1026, 1030, 1053, 1018, 1046, 1055, 1045, 1061, 1047, 1011, 1016, 1048, 1036, 1005, 989, 1012, 985, 1045, 991, 942, 1001, 1043, 979, 1052, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1067, 1031, 1045, 1026, 1080, 1080, 1023, 970, 1013, 1045, 1027, 1020, 1007, 1029, 1037, 1007, 1019, 1065, 1015, 1035, 1012, 986, 1030, 1013, 1043, 1017, 1073, 1006, 1012, 1015, 1020, 1007, 1020, 995, 1006, 1022, 1067, 992, 1001, 1019, 992, 991, 1058, 980, 1041, 1016, 1021, 1003, 1042, 1003, 994, 1035, 1007, 1003, 1074, 957, 1005, 1038, 1064, 1076, 1003, 1040, 1036, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[996, 1048, 1047, 1071, 1036, 1048, 1011, 1009, 1025, 1023, 1052, 1023, 1084, 1015, 987, 1046, 1025, 999, 1060, 1033, 1021, 1007, 1016, 1057, 976, 993, 1047, 951, 1001, 1083, 1006, 1054, 1037, 1082, 1038, 1058, 1026, 978, 978, 1012, 979, 1002, 1001, 999, 1000, 1017, 1026, 1028, 1070, 1051, 1021, 1015, 1067, 998, 1030, 1027, 1003, 1037, 1032, 994, 1056, 969, 1058, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1018, 1083, 1065, 1020, 1023, 1063, 1026, 1017, 979, 1056, 1004, 1070, 1036, 975, 1057, 1068, 1011, 1039, 1058, 949, 995, 1084, 989, 1032, 1004, 1043, 1048, 1026, 1051, 1053, 943, 1027, 1003, 1015, 1030, 1002, 1006, 1039, 1026, 1028, 1050, 1012, 1014, 1000, 1061, 987, 1029, 1018, 1003, 1000, 1028, 1060, 1004, 998, 1022, 983, 1016, 1063, 1016, 1029, 987, 1043, 1016, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1039, 1019, 994, 998, 986, 1062, 1046, 1012, 1009, 1056, 1042, 1037, 1027, 1048, 1005, 1025, 1076, 1026, 1021, 1043, 1022, 1037, 1061, 1017, 1022, 1039, 1032, 1022, 999, 1015, 954, 1000, 1036, 1038, 962, 1030, 994, 1012, 1023, 990, 1024, 982, 1023, 1005, 1088, 1040, 984, 1010, 1006, 1022, 1049, 1081, 1070, 1010, 1021, 1031, 1005, 982, 1046, 1071, 1015, 1025, 1027, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1067, 1016, 1005, 1032, 1024, 1001, 986, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1055, 998, 1022, 1002, 1044, 1000, 1023, 10, 0, 0, 0, 0, 0, 0, 0, 1010, 1011, 1035, 1038, 1033, 1035, 994, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1057, 1056, 994, 994, 1022, 1015, 992, 23, 3, 1, 0, 0, 0, 0, 0, 1039, 991, 1034, 1019, 1041, 1055, 1017, 1046, 21, 5, 0, 0, 0, 0, 0, 0, 1013, 1032, 1009, 1008, 996, 1023, 1049, 1004, 27, 0, 0, 0, 0, 0, 0, 0, 1001, 1033, 1002, 1039, 1033, 1013, 1039, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1015, 1038, 1029, 1035, 1037, 1024, 977, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8214, 8183, 8212, 8268, 8161, 8192, 8196] avg=8192.0000 max=8268 max_rank=4 min=8110 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1018, 1036, 1011, 1070, 1072, 1038, 990, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 977, 1013, 1009, 1034, 1014, 990, 1021, 10, 0, 0, 0, 0, 0, 0, 0, 1038, 997, 1000, 1002, 982, 1030, 1039, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1062, 995, 1040, 1011, 1059, 992, 1077, 22, 3, 1, 0, 0, 0, 0, 0, 990, 1001, 1022, 1037, 984, 1021, 1018, 1024, 21, 5, 0, 0, 0, 0, 0, 0, 999, 971, 1032, 987, 1057, 995, 1015, 1038, 29, 0, 0, 0, 0, 0, 0, 0, 1041, 998, 1030, 1050, 1057, 1044, 1073, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1008, 1026, 1009, 1014, 1012, 1030, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8084, 8142, 8280, 8123, 8123, 8331, 8192] avg=8192.0000 max=8331 max_rank=6 min=8084 min_rank=1 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1013, 1000, 1023, 1004, 1054, 938, 1003, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 982, 1009, 1014, 1067, 985, 999, 1021, 9, 0, 0, 0, 0, 0, 0, 0, 1015, 975, 998, 1067, 1011, 1067, 1046, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 990, 1013, 1060, 959, 962, 1012, 1044, 24, 3, 1, 0, 0, 0, 0, 0, 1006, 1020, 996, 1046, 1038, 1039, 1022, 1053, 19, 5, 0, 0, 0, 0, 0, 0, 1043, 1034, 1046, 1026, 998, 1041, 1032, 1029, 27, 0, 0, 0, 0, 0, 0, 0, 1068, 1041, 1048, 976, 1052, 956, 1043, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1035, 1098, 1066, 1051, 1069, 1026, 983, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8005, 8154, 8202, 8120, 8244, 8276, 8212, 8323] avg=8192.0000 max=8323 max_rank=7 min=8005 min_rank=0 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=33055 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1067, 1031, 1045, 1026, 1080, 1057, 1023, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1045, 1027, 1020, 1007, 1029, 1037, 1007, 10, 0, 0, 0, 0, 0, 0, 0, 1019, 1065, 1015, 1013, 1012, 986, 1030, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1017, 1073, 1006, 1012, 1015, 1020, 1007, 22, 3, 1, 0, 0, 0, 0, 0, 1020, 995, 1006, 1022, 1067, 992, 1001, 1019, 22, 5, 0, 0, 0, 0, 0, 0, 992, 991, 1058, 980, 1041, 1016, 1021, 1003, 28, 0, 0, 0, 0, 0, 0, 0, 1042, 1003, 994, 1020, 1007, 1003, 1074, 957, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1038, 1064, 1076, 1003, 1040, 1036, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8299, 8195, 8153, 8219, 8149, 8130, 8100, 8291] avg=8192.0000 max=8299 max_rank=0 min=8100 min_rank=6 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[992, 1023, 1064, 1061, 987, 1001, 1006, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 969, 980, 1030, 973, 1015, 1045, 1072, 1012, 10, 0, 0, 0, 0, 0, 0, 0, 1028, 1070, 1010, 1042, 1015, 1045, 1039, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1027, 1022, 1036, 1083, 973, 1028, 1021, 23, 3, 1, 0, 0, 0, 0, 0, 986, 1015, 1081, 1074, 1032, 999, 983, 1026, 21, 5, 0, 0, 0, 0, 0, 0, 1030, 1053, 1018, 1046, 1055, 1045, 1061, 1047, 28, 0, 0, 0, 0, 0, 0, 0, 1011, 1016, 1048, 1021, 1005, 989, 1012, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 991, 942, 1001, 1043, 979, 1052, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8106, 8247, 8232, 8222, 8383, 8087, 8063] avg=8192.0000 max=8383 max_rank=5 min=8063 min_rank=7 imbalance_ratio=1.023315 max_load_reduction=57153 max_load_reduction_pct=0.872086 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1018, 1083, 1065, 1020, 1023, 1041, 1026, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1056, 1004, 1070, 1036, 975, 1057, 1068, 10, 0, 0, 0, 0, 0, 0, 0, 1011, 1039, 1058, 929, 995, 1084, 989, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1043, 1048, 1026, 1051, 1053, 943, 1027, 20, 3, 1, 0, 0, 0, 0, 0, 1003, 1015, 1030, 1002, 1006, 1039, 1026, 1028, 21, 5, 0, 0, 0, 0, 0, 0, 1050, 1012, 1014, 1000, 1061, 987, 1029, 1018, 27, 0, 0, 0, 0, 0, 0, 0, 1003, 1000, 1028, 1045, 1004, 998, 1022, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1063, 1016, 1029, 987, 1043, 1016, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8255, 8137, 8219, 8175, 8198, 8083, 8176] avg=8192.0000 max=8293 max_rank=0 min=8083 min_rank=6 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[996, 1048, 1047, 1071, 1036, 1026, 1011, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1023, 1052, 1023, 1084, 1015, 987, 1046, 10, 0, 0, 0, 0, 0, 0, 0, 1025, 999, 1060, 1011, 1021, 1007, 1016, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 976, 993, 1047, 951, 1001, 1083, 1006, 1054, 22, 3, 1, 0, 0, 0, 0, 0, 1037, 1082, 1038, 1058, 1026, 978, 978, 1012, 21, 5, 0, 0, 0, 0, 0, 0, 979, 1002, 1001, 999, 1000, 1017, 1026, 1028, 26, 0, 0, 0, 0, 0, 0, 0, 1070, 1051, 1021, 1000, 1067, 998, 1030, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1037, 1032, 994, 1056, 969, 1058, 968, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8265, 8196, 8137, 8235, 8078, 8264, 8117] avg=8192.0000 max=8265 max_rank=1 min=8078 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1039, 1019, 994, 998, 986, 1034, 1046, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1056, 1042, 1037, 1027, 1048, 1005, 1025, 14, 0, 0, 0, 0, 0, 0, 0, 1076, 1026, 1021, 1016, 1022, 1037, 1061, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1039, 1032, 1022, 999, 1015, 954, 1000, 27, 5, 2, 0, 0, 0, 0, 0, 1036, 1038, 962, 1030, 994, 1012, 1023, 990, 26, 9, 0, 0, 0, 0, 0, 0, 1024, 982, 1023, 1005, 1088, 1040, 984, 1010, 31, 0, 0, 0, 0, 0, 0, 0, 1006, 1022, 1049, 1058, 1070, 1010, 1021, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 982, 1046, 1071, 1015, 1025, 1027, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8263, 8276, 8117, 8120, 8187, 8267, 8178] avg=8192.0000 max=8276 max_rank=2 min=8117 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8064, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8064, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8064, 8320, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1039, 1012, 1038, 1040, 990, 1031, 1011, 1028, 1053, 1025, 1048, 1007, 1038, 995, 1020, 1027, 1047, 1055, 981, 1047, 1073, 1016, 1002, 1030, 996, 1054, 1006, 1001, 1020, 1015, 1017, 977, 1038, 1014, 1034, 1052, 1016, 1022, 1010, 986, 1073, 1005, 1033, 1001, 993, 1007, 1042, 1030, 1036, 1055, 1071, 1039, 979, 1018, 1003, 998, 1069, 1030, 967, 1055, 1033, 1060, 999, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1007, 1050, 987, 1105, 1014, 988, 1050, 993, 1063, 987, 1011, 1011, 1056, 1022, 1065, 979, 1027, 1011, 1013, 987, 1076, 1006, 1067, 1028, 1025, 1073, 1019, 1031, 1060, 958, 990, 1026, 1046, 1089, 1007, 1012, 1077, 1022, 1047, 996, 1027, 1031, 1058, 1001, 1027, 1007, 981, 1043, 989, 1038, 1017, 986, 1031, 996, 1006, 1014, 1060, 1014, 999, 1028, 988, 1022, 1059, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1024, 1041, 1037, 998, 1006, 1041, 990, 1018, 1014, 1051, 1028, 1030, 1075, 1021, 996, 1044, 1001, 1008, 1028, 1033, 1051, 988, 1023, 1033, 984, 976, 1022, 1038, 1011, 1018, 1071, 1052, 1034, 1032, 1007, 1045, 1023, 1055, 990, 955, 1004, 973, 1013, 1007, 1003, 1007, 1004, 998, 1013, 1032, 1017, 1049, 1024, 1041, 1068, 1039, 1086, 1050, 1051, 1033, 1038, 1041, 1046, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1018, 1033, 981, 1010, 1052, 1009, 1016, 1078, 1014, 998, 1070, 1036, 1034, 1016, 1053, 1038, 981, 1034, 1008, 1051, 1028, 989, 1017, 1027, 991, 995, 1014, 1019, 1069, 1024, 1011, 1019, 1008, 1026, 1017, 1068, 1023, 1059, 1014, 1049, 1003, 1047, 1023, 1026, 1050, 996, 987, 973, 1081, 979, 1075, 1008, 989, 1041, 1011, 989, 1043, 1035, 1047, 1006, 1032, 1019, 1028, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1031, 1026, 997, 1102, 1045, 988, 1029, 1017, 1002, 1037, 1010, 1013, 1044, 1049, 1004, 1024, 1051, 1031, 1004, 1024, 1006, 1026, 1021, 1018, 1022, 982, 1021, 1029, 1018, 1012, 1028, 1037, 1027, 995, 1047, 1032, 1049, 1080, 1032, 1011, 1068, 1019, 981, 1008, 1069, 1014, 1040, 1044, 996, 1030, 1049, 1027, 1022, 979, 982, 967, 1007, 1008, 1032, 1061, 1094, 1007, 982, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1045, 971, 1041, 1009, 990, 1057, 989, 1001, 1060, 1005, 1063, 1012, 1006, 985, 1007, 1026, 1038, 1090, 1047, 1017, 1031, 1023, 1070, 993, 1029, 1042, 999, 1067, 1042, 1041, 1035, 1041, 1034, 1062, 1045, 1050, 1000, 1079, 1013, 1011, 1024, 1000, 1043, 1039, 1025, 1027, 1005, 990, 1015, 1044, 986, 1042, 975, 952, 1026, 1014, 1051, 994, 1002, 1013, 1071, 1011, 1016, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1034, 976, 1055, 1016, 1010, 1022, 1088, 1031, 1037, 997, 1058, 1022, 1035, 1004, 1010, 1027, 1053, 1016, 1003, 995, 1054, 1044, 1035, 991, 1058, 1008, 1040, 1011, 1016, 1037, 1037, 1035, 1014, 1028, 1028, 1037, 977, 1069, 1043, 1016, 982, 1015, 1040, 1008, 1001, 1045, 1027, 1031, 1046, 1011, 992, 970, 954, 1031, 1005, 1021, 1075, 1004, 1040, 1027, 1013, 1045, 987, 1099] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1041, 1046, 1033, 1040, 1057, 1018, 1029, 1016, 1061, 1009, 1027, 1056, 1009, 1026, 1057, 1039, 1015, 1018, 1025, 988, 1017, 1008, 1035, 1086, 1020, 1037, 1055, 1024, 982, 1009, 1076, 955, 969, 1004, 1059, 1013, 1000, 978, 1018, 1050, 1038, 1033, 1050, 1067, 1026, 1023, 1022, 1032, 1012, 983, 1010, 1022, 1014, 1072, 1003, 1041, 1016, 1006, 977, 981, 1017, 1057, 1000, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1039, 1012, 1038, 1034, 990, 1031, 1011, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1025, 1022, 1007, 1038, 995, 1020, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1055, 981, 1047, 1061, 1016, 1002, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1054, 1006, 1001, 1020, 1015, 1017, 977, 7, 6, 0, 0, 0, 0, 0, 0, 1038, 1014, 1034, 1052, 1016, 998, 1010, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1005, 1033, 1001, 993, 1007, 1042, 1030, 22, 5, 0, 0, 0, 0, 0, 0, 1036, 1055, 1071, 1039, 979, 1018, 1003, 998, 35, 26, 2, 0, 0, 0, 0, 0, 1034, 1030, 967, 1055, 1033, 1060, 999, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8187, 8239, 8099, 8148, 8211, 8262, 8207] avg=8192.0000 max=8262 max_rank=6 min=8099 min_rank=3 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1031, 1026, 997, 1096, 1045, 988, 1029, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1037, 985, 1013, 1044, 1049, 1004, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1031, 1004, 1024, 994, 1026, 1021, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 982, 1021, 1029, 1018, 1012, 1028, 1037, 7, 6, 0, 0, 0, 0, 0, 0, 1027, 995, 1047, 1032, 1049, 1055, 1032, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1019, 981, 1008, 1069, 1014, 1040, 1044, 23, 5, 0, 0, 0, 0, 0, 0, 996, 1030, 1049, 1027, 1022, 979, 982, 967, 33, 25, 2, 0, 0, 0, 0, 0, 974, 1008, 1032, 1061, 1094, 1007, 982, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8158, 8169, 8162, 8248, 8271, 8112, 8187] avg=8192.0000 max=8271 max_rank=5 min=8112 min_rank=6 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1007, 1050, 987, 1099, 1014, 988, 1050, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 987, 986, 1011, 1056, 1022, 1065, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1011, 1013, 987, 1064, 1006, 1067, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1073, 1019, 1031, 1060, 958, 990, 1026, 7, 6, 0, 0, 0, 0, 0, 0, 1046, 1089, 1007, 1012, 1077, 998, 1047, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1031, 1058, 1001, 1027, 1007, 981, 1043, 22, 5, 0, 0, 0, 0, 0, 0, 989, 1038, 1017, 986, 1031, 996, 1006, 1014, 35, 25, 2, 0, 0, 0, 0, 0, 1025, 1014, 999, 1028, 988, 1022, 1059, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8169, 8203, 8195, 8272, 8202, 8139, 8168] avg=8192.0000 max=8272 max_rank=4 min=8139 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1034, 976, 1055, 1011, 1010, 1022, 1088, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 997, 1032, 1022, 1035, 1004, 1010, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1016, 1003, 995, 1042, 1044, 1035, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1008, 1040, 1011, 1016, 1037, 1037, 1035, 7, 5, 0, 0, 0, 0, 0, 0, 1014, 1028, 1028, 1037, 977, 1044, 1043, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1015, 1040, 1008, 1001, 1045, 1027, 1031, 23, 5, 0, 0, 0, 0, 0, 0, 1046, 1011, 992, 970, 954, 1031, 1005, 1021, 35, 26, 2, 0, 0, 0, 0, 0, 1040, 1004, 1040, 1027, 1013, 1045, 987, 1099, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8164, 8179, 8254, 8187, 8177, 8093, 8255] avg=8192.0000 max=8255 max_rank=7 min=8093 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1024, 1041, 1037, 993, 1006, 1041, 990, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1051, 1003, 1030, 1075, 1021, 996, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1008, 1028, 1033, 1039, 988, 1023, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 984, 976, 1022, 1038, 1011, 1018, 1071, 1052, 7, 5, 0, 0, 0, 0, 0, 0, 1034, 1032, 1007, 1045, 1023, 1030, 990, 955, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 973, 1013, 1007, 1003, 1007, 1004, 998, 23, 5, 0, 0, 0, 0, 0, 0, 1013, 1032, 1017, 1049, 1024, 1041, 1068, 1039, 35, 25, 2, 0, 0, 0, 0, 0, 1051, 1050, 1051, 1033, 1038, 1041, 1046, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8234, 8153, 8184, 8116, 8037, 8345, 8317] avg=8192.0000 max=8345 max_rank=6 min=8037 min_rank=5 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1018, 1033, 981, 1005, 1052, 1009, 1016, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 998, 1043, 1036, 1034, 1016, 1053, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1034, 1008, 1051, 1016, 989, 1017, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 991, 995, 1014, 1019, 1069, 1024, 1011, 1019, 7, 5, 0, 0, 0, 0, 0, 0, 1008, 1026, 1017, 1068, 1023, 1034, 1014, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1047, 1023, 1026, 1050, 996, 987, 973, 23, 5, 0, 0, 0, 0, 0, 0, 1081, 979, 1075, 1008, 989, 1041, 1011, 989, 34, 27, 2, 0, 0, 0, 0, 0, 1009, 1035, 1047, 1006, 1032, 1019, 1028, 1051, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8232, 8123, 8154, 8239, 8133, 8236, 8227] avg=8192.0000 max=8239 max_rank=4 min=8123 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1045, 971, 1041, 1004, 990, 1057, 989, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1005, 1037, 1012, 1006, 985, 1007, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1090, 1047, 1017, 1019, 1023, 1070, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1042, 999, 1067, 1042, 1041, 1035, 1041, 7, 5, 0, 0, 0, 0, 0, 0, 1034, 1062, 1045, 1050, 1000, 1054, 1013, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1000, 1043, 1039, 1025, 1027, 1005, 990, 23, 5, 0, 0, 0, 0, 0, 0, 1015, 1044, 986, 1042, 975, 952, 1026, 1014, 34, 26, 2, 0, 0, 0, 0, 0, 1017, 994, 1002, 1013, 1071, 1011, 1016, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8138, 8297, 8308, 8269, 8181, 8116, 8129] avg=8192.0000 max=8308 max_rank=3 min=8098 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1041, 1046, 1033, 1029, 1057, 1018, 1029, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1009, 997, 1056, 1009, 1026, 1057, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1018, 1025, 988, 998, 1008, 1035, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1037, 1055, 1024, 982, 1009, 1076, 955, 11, 11, 0, 0, 0, 0, 0, 0, 969, 1004, 1059, 1013, 1000, 944, 1018, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1033, 1050, 1067, 1026, 1023, 1022, 1032, 25, 8, 0, 0, 0, 0, 0, 0, 1012, 983, 1010, 1022, 1014, 1072, 1003, 1041, 37, 30, 9, 0, 0, 0, 0, 0, 979, 1006, 977, 981, 1017, 1057, 1000, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8254, 8173, 8180, 8057, 8324, 8233, 8046] avg=8192.0000 max=8324 max_rank=5 min=8046 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32660 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8064, 8192, 8192, 8192, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[976, 1022, 1056, 1045, 1016, 1008, 1007, 1000, 1008, 1032, 1015, 987, 1010, 1018, 1021, 1027, 1031, 1045, 1038, 1026, 1019, 1014, 1058, 1040, 1027, 1033, 1006, 1024, 1005, 1041, 1036, 1007, 1031, 1027, 1048, 989, 1064, 994, 1032, 1020, 1020, 1085, 1016, 1001, 1022, 1067, 1001, 1004, 1035, 1068, 1002, 1010, 1054, 1027, 983, 1015, 1049, 1060, 1062, 967, 1011, 996, 1039, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[965, 1053, 1068, 1023, 1010, 1034, 1018, 1012, 1052, 995, 1043, 1007, 993, 979, 1020, 988, 1055, 1044, 1037, 1008, 1009, 1040, 1011, 1002, 1023, 1059, 1028, 940, 1070, 1041, 1023, 974, 1019, 1019, 1061, 1016, 994, 1037, 1020, 965, 994, 1039, 1037, 1044, 1055, 1049, 987, 1025, 1040, 1026, 1015, 1008, 1043, 1003, 969, 1063, 1047, 1049, 1051, 1055, 1049, 1054, 1019, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1035, 1085, 1011, 1033, 963, 1006, 1029, 1047, 1039, 1077, 1042, 992, 1043, 1090, 1051, 1025, 1011, 1038, 1038, 973, 1040, 975, 1001, 1031, 991, 1079, 985, 1042, 1051, 1024, 1046, 1010, 1038, 1063, 1010, 1009, 1047, 1001, 1115, 1016, 1029, 1038, 1002, 971, 1060, 968, 997, 1010, 1030, 1035, 1028, 1037, 1013, 1071, 1018, 1009, 1005, 1016, 981, 966, 1019, 983, 1078, 970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[998, 986, 971, 1043, 959, 982, 981, 1003, 1030, 1015, 1004, 1015, 992, 985, 1077, 1014, 1064, 1031, 1030, 990, 983, 1025, 1030, 1016, 1020, 1059, 1056, 1025, 1036, 1039, 999, 1002, 1032, 1054, 1043, 1059, 1026, 1041, 1045, 1032, 1044, 1024, 1030, 1017, 1059, 1048, 1027, 984, 1039, 1049, 1037, 1011, 1035, 976, 1047, 966, 1023, 1071, 1015, 1065, 1022, 1018, 1040, 1097] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[965, 1008, 994, 1048, 993, 1004, 1014, 1050, 1024, 1039, 1047, 1015, 1013, 1044, 1064, 1030, 1015, 1035, 987, 1007, 1046, 1025, 1025, 1017, 1024, 1011, 1018, 990, 1003, 1041, 1071, 1028, 1042, 1023, 1037, 1063, 1013, 1026, 1037, 985, 996, 1031, 1039, 1033, 1084, 1057, 1019, 1007, 1006, 1068, 1035, 1021, 1026, 998, 1038, 1038, 1007, 999, 974, 1069, 1002, 1012, 1037, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1034, 1036, 1007, 1017, 996, 1055, 1083, 1017, 947, 1056, 1063, 1023, 1054, 1083, 1020, 1020, 1015, 1053, 974, 1021, 1015, 1057, 1079, 1052, 1045, 1073, 978, 1035, 1019, 1015, 1016, 1001, 998, 1041, 1012, 1003, 1099, 1001, 1086, 983, 1052, 1013, 983, 1025, 987, 993, 1051, 1045, 1005, 995, 995, 1019, 964, 1042, 1007, 998, 1000, 995, 1000, 1081, 1038, 1020, 970, 1076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1035, 1062, 1035, 1035, 1042, 1019, 959, 1019, 1030, 1018, 1045, 1016, 1005, 1029, 1069, 1031, 1027, 1077, 1018, 1047, 1062, 980, 991, 967, 1060, 1053, 978, 977, 1036, 998, 977, 1041, 981, 1019, 1097, 1040, 1064, 1014, 1041, 1006, 1025, 1005, 1029, 1032, 1006, 1029, 1005, 1066, 961, 980, 1006, 1041, 979, 1043, 1015, 1051, 1011, 1097, 1028, 1017, 1026, 1046, 1064, 974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1042, 1004, 970, 1002, 1006, 1030, 1024, 1030, 1028, 1038, 1002, 1046, 1002, 1046, 1020, 1022, 1018, 1035, 998, 1027, 1073, 1015, 1039, 1086, 1052, 964, 1067, 1067, 1012, 1091, 1066, 1027, 976, 979, 1091, 1052, 1077, 996, 983, 1011, 1009, 1013, 1034, 998, 1031, 1035, 981, 1030, 996, 1023, 1028, 1028, 1065, 1030, 1006, 1024, 1053, 983, 1003, 1044, 1004, 1019, 990, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[976, 1022, 1056, 1045, 1016, 1008, 1007, 1000, 38, 17, 1, 0, 0, 0, 0, 0, 1008, 1032, 1015, 987, 1010, 1018, 1004, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1033, 1038, 1026, 1019, 1014, 1058, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1021, 1006, 1024, 1005, 1041, 1036, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1027, 1010, 989, 1064, 994, 1032, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1085, 1016, 1001, 1022, 1067, 1001, 1004, 4, 0, 0, 0, 0, 0, 0, 0, 1035, 1068, 1002, 1010, 1054, 1027, 983, 1015, 12, 11, 6, 0, 0, 0, 0, 0, 1049, 1050, 1062, 967, 1011, 996, 1039, 1039, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8101, 8259, 8167, 8167, 8220, 8223, 8213] avg=8192.0000 max=8259 max_rank=2 min=8101 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[965, 1053, 1068, 1023, 1010, 1034, 1018, 1012, 38, 16, 1, 0, 0, 0, 0, 0, 1052, 995, 1043, 1007, 993, 979, 1004, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1032, 1037, 1008, 1009, 1040, 1011, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1047, 1028, 940, 1070, 1041, 1023, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1019, 1023, 1016, 994, 1037, 1020, 965, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1039, 1037, 1044, 1055, 1049, 987, 1025, 4, 0, 0, 0, 0, 0, 0, 0, 1040, 1026, 1015, 1008, 1043, 1003, 969, 1063, 12, 11, 6, 0, 0, 0, 0, 0, 1047, 1039, 1051, 1055, 1049, 1054, 1019, 1060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8061, 8194, 8146, 8093, 8234, 8196, 8374] avg=8192.0000 max=8374 max_rank=7 min=8061 min_rank=1 imbalance_ratio=1.022217 max_load_reduction=57162 max_load_reduction_pct=0.872223 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1035, 1062, 1035, 1035, 1042, 1019, 959, 1019, 40, 17, 1, 0, 0, 0, 0, 0, 1030, 1018, 1045, 1016, 1005, 1029, 1052, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1065, 1018, 1047, 1062, 980, 991, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1041, 978, 977, 1036, 998, 977, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1019, 1057, 1040, 1064, 1014, 1041, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1005, 1029, 1032, 1006, 1029, 1005, 1066, 5, 0, 0, 0, 0, 0, 0, 0, 961, 980, 1006, 1041, 979, 1043, 1015, 1051, 12, 11, 7, 0, 0, 0, 0, 0, 1011, 1085, 1028, 1017, 1026, 1046, 1064, 974, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8226, 8157, 8108, 8222, 8202, 8106, 8251] avg=8192.0000 max=8264 max_rank=0 min=8106 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[998, 986, 971, 1043, 959, 982, 981, 1003, 38, 17, 1, 0, 0, 0, 0, 0, 1030, 1015, 1004, 1015, 992, 985, 1060, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1019, 1030, 990, 983, 1025, 1030, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1047, 1056, 1025, 1036, 1039, 999, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1054, 1005, 1059, 1026, 1041, 1045, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1024, 1030, 1017, 1059, 1048, 1027, 984, 5, 0, 0, 0, 0, 0, 0, 0, 1039, 1049, 1037, 1011, 1035, 976, 1047, 966, 12, 11, 6, 0, 0, 0, 0, 0, 1023, 1060, 1015, 1065, 1022, 1018, 1040, 1097, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7979, 8115, 8157, 8224, 8294, 8238, 8189, 8340] avg=8192.0000 max=8340 max_rank=7 min=7979 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=33061 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[965, 1008, 994, 1048, 993, 1004, 1014, 1050, 37, 17, 1, 0, 0, 0, 0, 0, 1024, 1039, 1047, 1015, 1013, 1044, 1047, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1023, 987, 1007, 1046, 1025, 1025, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 999, 1018, 990, 1003, 1041, 1071, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1023, 1000, 1063, 1013, 1026, 1037, 985, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1031, 1039, 1033, 1084, 1057, 1019, 1007, 4, 0, 0, 0, 0, 0, 0, 0, 1006, 1068, 1035, 1021, 1026, 998, 1038, 1038, 12, 11, 6, 0, 0, 0, 0, 0, 1007, 989, 974, 1069, 1002, 1012, 1037, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8259, 8145, 8174, 8189, 8270, 8259, 8109] avg=8192.0000 max=8270 max_rank=5 min=8109 min_rank=7 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1034, 1036, 1007, 1017, 996, 1055, 1083, 1017, 36, 16, 1, 0, 0, 0, 0, 0, 947, 1056, 1063, 1023, 1054, 1083, 1004, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1041, 974, 1021, 1015, 1057, 1079, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1061, 978, 1035, 1019, 1015, 1016, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1041, 976, 1003, 1099, 1001, 1086, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1013, 983, 1025, 987, 993, 1051, 1045, 4, 0, 0, 0, 0, 0, 0, 0, 1005, 995, 995, 1019, 964, 1042, 1007, 998, 12, 11, 6, 0, 0, 0, 0, 0, 1000, 985, 1000, 1081, 1038, 1020, 970, 1076, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8250, 8254, 8170, 8187, 8153, 8054, 8170] avg=8192.0000 max=8298 max_rank=0 min=8054 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32564 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1035, 1085, 1011, 1033, 963, 1006, 1029, 1047, 36, 17, 1, 0, 0, 0, 0, 0, 1039, 1077, 1042, 992, 1043, 1090, 1034, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1026, 1038, 973, 1040, 975, 1001, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1067, 985, 1042, 1051, 1024, 1046, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1063, 974, 1009, 1047, 1001, 1115, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1038, 1002, 971, 1060, 968, 997, 1010, 4, 0, 0, 0, 0, 0, 0, 0, 1030, 1035, 1028, 1037, 1013, 1071, 1018, 1009, 12, 11, 6, 0, 0, 0, 0, 0, 1005, 1006, 981, 966, 1019, 983, 1078, 970, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8342, 8095, 8216, 8263, 8079, 8270, 8008] avg=8192.0000 max=8342 max_rank=1 min=8008 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1042, 1004, 970, 1002, 1006, 1030, 1024, 1030, 44, 22, 3, 0, 0, 0, 0, 0, 1028, 1038, 1002, 1046, 1002, 1046, 998, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1019, 998, 1027, 1073, 1015, 1039, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 949, 1067, 1067, 1012, 1091, 1066, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 976, 979, 1047, 1052, 1077, 996, 983, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1013, 1034, 998, 1031, 1035, 981, 1030, 9, 0, 0, 0, 0, 0, 0, 0, 996, 1023, 1028, 1028, 1065, 1030, 1006, 1024, 15, 13, 11, 0, 0, 0, 0, 0, 1053, 963, 1003, 1044, 1004, 1019, 990, 995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8182, 8275, 8331, 8121, 8140, 8239, 8071] avg=8192.0000 max=8331 max_rank=3 min=8071 min_rank=7 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32571 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8192, 8320, 8064, 8192, 8192, 8192, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8448, 8192, 8448, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1006, 981, 1006, 1016, 1026, 1052, 1088, 1012, 1040, 1025, 1016, 999, 986, 1055, 1008, 1036, 1052, 1002, 1045, 1067, 1046, 1014, 1068, 962, 1005, 991, 1010, 991, 1013, 1018, 1044, 1013, 1053, 1016, 1000, 1070, 1026, 1043, 1094, 1022, 1013, 1032, 1047, 1015, 1030, 1045, 1056, 1028, 1041, 951, 986, 1010, 1040, 989, 1064, 1037, 1033, 1013, 1039, 989, 1016, 1033, 1002, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1004, 1086, 1023, 1012, 1008, 1032, 1068, 1053, 1006, 1026, 1070, 1071, 1091, 1072, 983, 1077, 989, 1048, 946, 1033, 989, 971, 1010, 1006, 1008, 995, 1004, 1069, 1027, 969, 1060, 1049, 997, 994, 1052, 1032, 1044, 994, 973, 1047, 1005, 1001, 1005, 1047, 1012, 1058, 1036, 1038, 1031, 1052, 967, 986, 1049, 998, 1024, 1030, 1012, 1063, 995, 1014, 997, 1045, 1021, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1036, 1018, 1023, 1033, 1016, 1043, 1025, 1022, 989, 1053, 973, 995, 1074, 1017, 1045, 1023, 960, 1001, 1005, 1049, 1062, 1010, 1052, 1031, 1038, 1017, 1023, 1053, 1034, 1015, 1060, 1057, 1046, 986, 996, 972, 1030, 1021, 1037, 1045, 1014, 992, 1106, 996, 1038, 987, 1013, 1000, 1035, 1028, 1009, 1059, 995, 1065, 999, 980, 1078, 964, 1042, 1011, 1022, 1030, 1033, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1078, 1003, 982, 1027, 1017, 1046, 1041, 994, 1017, 1037, 1028, 974, 1033, 1034, 1028, 979, 1063, 1067, 1056, 1083, 1010, 1022, 1064, 1037, 1028, 1017, 1002, 1033, 1078, 1004, 1036, 1057, 1001, 1022, 984, 1008, 995, 1029, 986, 1037, 1035, 944, 1014, 994, 1014, 1037, 1058, 1048, 993, 1003, 1053, 954, 1037, 995, 1047, 1084, 976, 1007, 1031, 1051, 1045, 1057, 1023, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1058, 1020, 1050, 1029, 1022, 1000, 1077, 971, 1000, 1062, 1050, 1028, 1052, 987, 1025, 1042, 983, 953, 1039, 1053, 999, 1036, 1029, 1012, 1021, 1003, 1031, 1031, 1025, 1034, 1051, 1033, 1024, 1055, 999, 1028, 976, 1024, 999, 1048, 1016, 1021, 1053, 1039, 1059, 1038, 1025, 986, 1046, 1031, 1044, 1003, 980, 1068, 996, 1030, 969, 1026, 1006, 1066, 1005, 1035, 1008, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[993, 1066, 1028, 1040, 1048, 1040, 1053, 1019, 1095, 1056, 1000, 1041, 985, 1022, 1039, 1060, 1000, 1058, 980, 934, 1010, 1033, 1006, 1030, 1002, 983, 1004, 1025, 1033, 1007, 1050, 1028, 1035, 1034, 1051, 1005, 1011, 1008, 1000, 952, 1058, 973, 1037, 1022, 996, 1007, 977, 1066, 1040, 1031, 1063, 1076, 1055, 1039, 1008, 999, 1071, 980, 1059, 997, 1031, 990, 1043, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1082, 979, 1056, 1021, 1033, 1054, 990, 1016, 1016, 1011, 999, 1041, 982, 1030, 1021, 997, 1034, 963, 977, 1052, 1001, 1002, 1042, 1038, 979, 1031, 1023, 1002, 1047, 999, 1037, 995, 1039, 1018, 1028, 1055, 1013, 1039, 1045, 1061, 1046, 1013, 1052, 1027, 1070, 1037, 1010, 1004, 997, 967, 1035, 1017, 1032, 1043, 1024, 1010, 1012, 1053, 1048, 1085, 1002, 1018, 1037, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1012, 1022, 1052, 1014, 1039, 1016, 1012, 1001, 1011, 1064, 1021, 1018, 1034, 1087, 1002, 1042, 1044, 1008, 1010, 1015, 1044, 1085, 1070, 1037, 983, 1048, 1004, 1019, 1022, 1073, 989, 981, 1031, 1034, 1027, 1000, 1026, 1010, 986, 1011, 974, 1004, 998, 1041, 990, 994, 1065, 1057, 980, 1027, 1037, 1083, 980, 1021, 1051, 1088, 1032, 1007, 1006, 987, 1025, 1010, 1029, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1004, 1086, 1023, 1012, 1008, 1032, 1036, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 998, 1070, 1071, 1091, 1072, 983, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1048, 946, 1033, 989, 971, 1010, 1006, 11, 5, 0, 0, 0, 0, 0, 0, 1008, 995, 1004, 1069, 1027, 969, 1060, 1049, 15, 0, 0, 0, 0, 0, 0, 0, 997, 994, 1052, 1032, 1044, 994, 973, 1047, 27, 0, 0, 0, 0, 0, 0, 0, 1005, 1001, 1005, 1047, 1012, 1058, 1036, 1038, 2, 0, 0, 0, 0, 0, 0, 0, 1031, 1052, 967, 986, 1049, 998, 1024, 1030, 2, 2, 0, 0, 0, 0, 0, 0, 1012, 1063, 995, 1014, 997, 1045, 1021, 1058, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8368, 8008, 8196, 8160, 8204, 8141, 8205] avg=8192.0000 max=8368 max_rank=1 min=8008 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=57168 max_load_reduction_pct=0.872314 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1036, 1018, 1023, 1033, 1016, 1043, 994, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1024, 973, 995, 1074, 1017, 1045, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 960, 1001, 1005, 1049, 1062, 1010, 1052, 1031, 12, 5, 0, 0, 0, 0, 0, 0, 1038, 1017, 1023, 1053, 1034, 1015, 1060, 1057, 15, 0, 0, 0, 0, 0, 0, 0, 1046, 986, 996, 972, 1030, 1021, 1037, 1045, 26, 0, 0, 0, 0, 0, 0, 0, 1014, 992, 1106, 996, 1038, 987, 1013, 1000, 2, 0, 0, 0, 0, 0, 0, 0, 1035, 1028, 1009, 1059, 995, 1065, 999, 980, 2, 2, 0, 0, 0, 0, 0, 0, 1078, 964, 1042, 1011, 1022, 1030, 1033, 1051, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8140, 8187, 8312, 8159, 8148, 8174, 8231] avg=8192.0000 max=8312 max_rank=3 min=8140 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1078, 1003, 982, 1027, 1017, 1046, 1010, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1008, 1028, 974, 1033, 1034, 1028, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1067, 1056, 1083, 1010, 1022, 1064, 1037, 12, 5, 0, 0, 0, 0, 0, 0, 1028, 1017, 1002, 1033, 1078, 1004, 1036, 1057, 15, 0, 0, 0, 0, 0, 0, 0, 1001, 1022, 984, 1008, 995, 1029, 986, 1037, 26, 0, 0, 0, 0, 0, 0, 0, 1035, 944, 1014, 994, 1014, 1037, 1058, 1048, 2, 0, 0, 0, 0, 0, 0, 0, 993, 1003, 1053, 954, 1037, 995, 1047, 1084, 2, 2, 0, 0, 0, 0, 0, 0, 976, 1007, 1031, 1051, 1045, 1057, 1023, 995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8101, 8419, 8270, 8088, 8146, 8170, 8185] avg=8192.0000 max=8419 max_rank=2 min=8088 min_rank=4 imbalance_ratio=1.027710 max_load_reduction=57117 max_load_reduction_pct=0.871536 moved_to_echo=32589 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1006, 981, 1006, 1016, 1026, 1052, 1056, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 997, 1016, 999, 986, 1055, 1008, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1002, 1045, 1067, 1046, 1014, 1068, 962, 11, 5, 0, 0, 0, 0, 0, 0, 1005, 991, 1010, 991, 1013, 1018, 1044, 1013, 15, 0, 0, 0, 0, 0, 0, 0, 1053, 1016, 1000, 1070, 1026, 1043, 1094, 1022, 27, 0, 0, 0, 0, 0, 0, 0, 1013, 1032, 1047, 1015, 1030, 1045, 1056, 1028, 2, 0, 0, 0, 0, 0, 0, 0, 1041, 951, 986, 1010, 1040, 989, 1064, 1037, 2, 2, 0, 0, 0, 0, 0, 0, 1033, 1013, 1039, 989, 1016, 1033, 1002, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8137, 8272, 8100, 8351, 8268, 8122, 8131] avg=8192.0000 max=8351 max_rank=4 min=8100 min_rank=3 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1058, 1020, 1050, 1029, 1022, 1000, 1045, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1033, 1050, 1028, 1052, 987, 1025, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 983, 953, 1039, 1053, 999, 1036, 1029, 1012, 12, 5, 0, 0, 0, 0, 0, 0, 1021, 1003, 1031, 1031, 1025, 1034, 1051, 1033, 15, 0, 0, 0, 0, 0, 0, 0, 1024, 1055, 999, 1028, 976, 1024, 999, 1048, 27, 0, 0, 0, 0, 0, 0, 0, 1016, 1021, 1053, 1039, 1059, 1038, 1025, 986, 2, 0, 0, 0, 0, 0, 0, 0, 1046, 1031, 1044, 1003, 980, 1068, 996, 1030, 2, 2, 0, 0, 0, 0, 0, 0, 969, 1026, 1006, 1066, 1005, 1035, 1008, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8217, 8121, 8244, 8180, 8239, 8202, 8138] avg=8192.0000 max=8244 max_rank=3 min=8121 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[993, 1066, 1028, 1040, 1048, 1040, 1022, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 1027, 1000, 1041, 985, 1022, 1039, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1058, 980, 934, 1010, 1033, 1006, 1030, 12, 5, 0, 0, 0, 0, 0, 0, 1002, 983, 1004, 1025, 1033, 1007, 1050, 1028, 15, 0, 0, 0, 0, 0, 0, 0, 1035, 1034, 1051, 1005, 1011, 1008, 1000, 952, 26, 0, 0, 0, 0, 0, 0, 0, 1058, 973, 1037, 1022, 996, 1007, 977, 1066, 2, 0, 0, 0, 0, 0, 0, 0, 1040, 1031, 1063, 1076, 1055, 1039, 1008, 999, 2, 2, 0, 0, 0, 0, 0, 0, 1071, 980, 1059, 997, 1031, 990, 1043, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8269, 8068, 8147, 8122, 8138, 8315, 8221] avg=8192.0000 max=8315 max_rank=6 min=8068 min_rank=2 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1082, 979, 1056, 1021, 1033, 1054, 961, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 983, 999, 1041, 982, 1030, 1021, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 963, 977, 1052, 1001, 1002, 1042, 1038, 11, 4, 0, 0, 0, 0, 0, 0, 979, 1031, 1023, 1002, 1047, 999, 1037, 995, 15, 0, 0, 0, 0, 0, 0, 0, 1039, 1018, 1028, 1055, 1013, 1039, 1045, 1061, 25, 0, 0, 0, 0, 0, 0, 0, 1046, 1013, 1052, 1027, 1070, 1037, 1010, 1004, 2, 0, 0, 0, 0, 0, 0, 0, 997, 967, 1035, 1017, 1032, 1043, 1024, 1010, 2, 2, 0, 0, 0, 0, 0, 0, 1012, 1053, 1048, 1085, 1002, 1018, 1037, 1045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8069, 8124, 8128, 8323, 8261, 8129, 8300] avg=8192.0000 max=8323 max_rank=4 min=8069 min_rank=1 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=33013 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1012, 1022, 1052, 1014, 1039, 1016, 976, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1020, 1021, 1018, 1034, 1087, 1002, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1008, 1010, 1015, 1044, 1085, 1070, 1037, 16, 8, 0, 0, 0, 0, 0, 0, 983, 1048, 1004, 1019, 1022, 1073, 989, 981, 20, 0, 0, 0, 0, 0, 0, 0, 1031, 1034, 1027, 1000, 1026, 1010, 986, 1011, 28, 0, 0, 0, 0, 0, 0, 0, 974, 1004, 998, 1041, 990, 994, 1065, 1057, 9, 0, 0, 0, 0, 0, 0, 0, 980, 1027, 1037, 1083, 980, 1021, 1051, 1088, 8, 8, 0, 0, 0, 0, 0, 0, 1032, 1007, 1006, 987, 1025, 1010, 1029, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8235, 8337, 8139, 8153, 8132, 8283, 8125] avg=8192.0000 max=8337 max_rank=2 min=8125 min_rank=7 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8064, 8320, 8192, 8192, 8448, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8320, 8192, 8320, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8192, 8320, 8320, 8192, 8448, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1035, 1085, 1070, 1024, 960, 997, 1025, 1023, 1029, 1077, 999, 1022, 1029, 1013, 1038, 1017, 972, 1073, 1076, 1043, 987, 1047, 1039, 998, 1025, 1002, 1015, 1000, 973, 1024, 1062, 1039, 1018, 1072, 1000, 1024, 1039, 1002, 1049, 1040, 990, 1013, 1049, 1030, 1019, 1006, 1009, 968, 1041, 1091, 1015, 1032, 1016, 1051, 1031, 980, 1028, 1063, 993, 983, 1000, 1024, 1001, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1045, 977, 1026, 987, 1052, 1013, 1071, 1070, 1026, 1017, 1023, 1047, 1058, 1042, 1019, 996, 1000, 965, 1003, 1037, 1079, 1011, 1029, 1023, 977, 999, 990, 993, 993, 1034, 1020, 1050, 1038, 974, 1048, 1039, 1049, 1032, 994, 1068, 983, 1025, 1057, 999, 1022, 1058, 1056, 1045, 999, 999, 997, 1005, 1016, 1015, 1027, 1028, 1036, 1079, 1006, 1094, 1074, 992, 1007, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1087, 976, 1003, 993, 1054, 1004, 981, 1022, 1022, 973, 1030, 1091, 1010, 1035, 1026, 1030, 980, 1002, 1059, 1038, 1011, 959, 1049, 1004, 1042, 1061, 1020, 1041, 1088, 1045, 1013, 1079, 969, 1044, 1071, 1018, 1037, 1038, 1011, 1030, 1031, 980, 1031, 984, 1054, 1015, 1050, 1053, 1045, 1015, 976, 980, 1007, 1028, 1042, 979, 1061, 1013, 985, 1007, 1067, 1011, 1019, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[998, 1003, 996, 1006, 970, 1018, 1023, 1034, 1021, 1012, 1033, 1050, 1046, 975, 1047, 1020, 1056, 1027, 1051, 1070, 1029, 999, 1070, 1027, 1012, 1007, 1011, 1054, 1044, 1078, 1031, 1080, 1021, 1056, 998, 1029, 1021, 1024, 1009, 1042, 978, 1007, 986, 1045, 1056, 1042, 1004, 1027, 1012, 1014, 1031, 1018, 1009, 996, 1062, 1016, 1081, 981, 1022, 1023, 1050, 964, 1047, 967] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1026, 958, 1007, 1029, 1016, 1010, 1019, 1015, 1024, 1025, 1036, 997, 1023, 1027, 1046, 983, 987, 1015, 1021, 1023, 1049, 1015, 1024, 1018, 1018, 1020, 996, 999, 1070, 1005, 1078, 1005, 1027, 1045, 980, 1001, 1024, 996, 1076, 993, 1026, 1018, 1053, 1022, 1055, 1050, 1049, 1055, 1040, 1041, 977, 1039, 1029, 1053, 1030, 1039, 1054, 1020, 1063, 1006, 1012, 1006, 1031, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1048, 1013, 1000, 997, 991, 980, 1003, 1001, 1021, 1015, 1016, 1010, 992, 1079, 1039, 1023, 1064, 994, 1069, 982, 1025, 1022, 1023, 1064, 1033, 1054, 1054, 1040, 1054, 984, 1014, 1011, 996, 1044, 1056, 1014, 1010, 988, 1043, 1019, 999, 1037, 1046, 1033, 1047, 1066, 1053, 1050, 1045, 1003, 1043, 1037, 959, 1065, 999, 1006, 1018, 1045, 994, 1046, 1003, 1015, 1040, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1009, 1043, 1036, 1034, 1015, 1042, 1020, 1053, 1002, 1013, 1032, 1046, 1030, 1064, 1034, 980, 1017, 1019, 1063, 1019, 1003, 1046, 990, 986, 991, 1000, 986, 1040, 1038, 1079, 1029, 1033, 987, 1047, 1059, 1018, 998, 1080, 1035, 1042, 976, 1071, 1046, 997, 1000, 1009, 1056, 964, 1034, 1011, 1036, 1071, 1071, 1000, 966, 1032, 1032, 1021, 1065, 976, 1056, 1026, 956, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[980, 992, 1005, 1023, 1038, 1028, 1040, 1060, 1047, 1066, 985, 1040, 966, 1019, 1004, 1048, 1030, 1041, 1003, 1014, 1008, 1028, 1037, 1018, 1027, 1046, 995, 1033, 985, 1030, 1048, 1026, 988, 1023, 1026, 1020, 1043, 994, 1071, 1046, 1035, 1006, 991, 1002, 1009, 1000, 1060, 1067, 1025, 984, 1042, 1053, 1056, 1015, 1063, 1002, 1064, 990, 1023, 1060, 1013, 1032, 1007, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1035, 1085, 1070, 1024, 960, 997, 1025, 1023, 23, 20, 3, 0, 0, 0, 0, 0, 1029, 1077, 999, 1015, 1029, 1013, 1038, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1073, 1076, 1043, 987, 1047, 1039, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1002, 1015, 1000, 973, 1024, 1062, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1052, 1000, 1024, 1039, 1002, 1049, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1013, 1049, 1030, 1019, 1006, 1000, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1091, 1015, 1032, 1016, 1051, 1031, 980, 6, 5, 0, 0, 0, 0, 0, 0, 1028, 1063, 993, 983, 1000, 1024, 1001, 1041, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8217, 8235, 8117, 8224, 8075, 8268, 8135] avg=8192.0000 max=8268 max_rank=6 min=8075 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1087, 976, 1003, 993, 1054, 1004, 981, 1022, 24, 19, 4, 0, 0, 0, 0, 0, 1022, 973, 1030, 1083, 1010, 1035, 1026, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1002, 1059, 1038, 1011, 959, 1049, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1061, 1020, 1041, 1088, 1045, 1013, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1025, 1071, 1018, 1037, 1038, 1011, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 980, 1031, 984, 1054, 1015, 1040, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1015, 976, 980, 1007, 1028, 1042, 979, 6, 6, 0, 0, 0, 0, 0, 0, 1061, 1013, 985, 1007, 1067, 1011, 1019, 1057, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8209, 8102, 8365, 8199, 8188, 8084, 8222] avg=8192.0000 max=8365 max_rank=3 min=8084 min_rank=6 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1048, 1013, 1000, 997, 991, 980, 1003, 1001, 22, 19, 4, 0, 0, 0, 0, 0, 1021, 1015, 1016, 1003, 992, 1079, 1039, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 994, 1069, 982, 1025, 1022, 1023, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1054, 1054, 1040, 1054, 984, 1014, 989, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1025, 1056, 1014, 1010, 988, 1043, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1037, 1046, 1033, 1047, 1066, 1043, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1003, 1043, 1037, 959, 1065, 999, 1006, 6, 5, 0, 0, 0, 0, 0, 0, 1018, 1045, 994, 1046, 1003, 1015, 1040, 1002, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8078, 8188, 8243, 8222, 8151, 8321, 8168, 8165] avg=8192.0000 max=8321 max_rank=5 min=8078 min_rank=0 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1045, 977, 1026, 987, 1052, 1013, 1071, 1070, 23, 18, 4, 0, 0, 0, 0, 0, 1026, 1017, 1023, 1040, 1058, 1042, 1019, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 965, 1003, 1037, 1079, 1011, 1029, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 977, 999, 990, 993, 993, 1034, 1020, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 956, 1048, 1039, 1049, 1032, 994, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1025, 1057, 999, 1022, 1058, 1046, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 999, 999, 997, 1005, 1016, 1015, 1027, 1028, 6, 5, 0, 0, 0, 0, 0, 0, 1036, 1079, 1006, 1094, 1074, 992, 1007, 1003, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8221, 8147, 8033, 8224, 8235, 8097, 8293] avg=8192.0000 max=8293 max_rank=7 min=8033 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[998, 1003, 996, 1006, 970, 1018, 1023, 1034, 24, 19, 3, 0, 0, 0, 0, 0, 1021, 1012, 1033, 1043, 1046, 975, 1047, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1027, 1051, 1070, 1029, 999, 1070, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1007, 1011, 1054, 1044, 1078, 1031, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1037, 998, 1029, 1021, 1024, 1009, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1007, 986, 1045, 1056, 1042, 995, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1014, 1031, 1018, 1009, 996, 1062, 1016, 6, 5, 0, 0, 0, 0, 0, 0, 1081, 981, 1022, 1023, 1050, 964, 1047, 967, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8094, 8197, 8329, 8293, 8181, 8136, 8169, 8137] avg=8192.0000 max=8329 max_rank=2 min=8094 min_rank=0 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1026, 958, 1007, 1029, 1016, 1010, 1019, 1015, 22, 19, 4, 0, 0, 0, 0, 0, 1024, 1025, 1036, 990, 1023, 1027, 1046, 983, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1015, 1021, 1023, 1049, 1015, 1024, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1020, 996, 999, 1070, 1005, 1078, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1026, 980, 1001, 1024, 996, 1076, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1018, 1053, 1022, 1055, 1050, 1039, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1041, 977, 1039, 1029, 1053, 1030, 1039, 6, 5, 0, 0, 0, 0, 0, 0, 1054, 1020, 1063, 1006, 1012, 1006, 1031, 1042, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8154, 8152, 8169, 8123, 8318, 8259, 8236] avg=8192.0000 max=8318 max_rank=5 min=8123 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1009, 1043, 1036, 1034, 1015, 1042, 1020, 1053, 23, 19, 4, 0, 0, 0, 0, 0, 1002, 1013, 1032, 1039, 1030, 1064, 1034, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1019, 1063, 1019, 1003, 1046, 990, 986, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1000, 986, 1040, 1038, 1079, 1029, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1028, 1059, 1018, 998, 1080, 1035, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1071, 1046, 997, 1000, 1009, 1046, 964, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1011, 1036, 1071, 1071, 1000, 966, 1032, 6, 5, 0, 0, 0, 0, 0, 0, 1032, 1021, 1065, 976, 1056, 1026, 956, 1006, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8194, 8143, 8173, 8247, 8109, 8232, 8140] avg=8192.0000 max=8298 max_rank=0 min=8109 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[980, 992, 1005, 1023, 1038, 1028, 1040, 1060, 26, 24, 7, 0, 0, 0, 0, 0, 1047, 1066, 985, 1021, 966, 1019, 1004, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1041, 1003, 1014, 1008, 1028, 1037, 1018, 6, 0, 0, 0, 0, 0, 0, 0, 1027, 1046, 995, 1033, 985, 1030, 1048, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 988, 999, 1026, 1020, 1043, 994, 1071, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1006, 991, 1002, 1009, 1000, 1044, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 984, 1042, 1053, 1056, 1015, 1063, 1002, 9, 10, 0, 0, 0, 0, 0, 0, 1064, 990, 1023, 1060, 1013, 1032, 1007, 1016, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8156, 8185, 8164, 8187, 8154, 8259, 8208] avg=8192.0000 max=8259 max_rank=6 min=8154 min_rank=5 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8064, 8192, 8192, 8192, 8192, 8192, 8320, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8448, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8448, 8320, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True + [2026-08-30 05:02:14] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 659.1 | throughput per GPU (TFLOP/s/GPU): 505.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.251086E+00 | loss scale: 1.0 | grad norm: 5.500 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[968, 996, 1020, 1028, 1025, 1015, 1013, 1009, 999, 978, 984, 1063, 1035, 1008, 1077, 1042, 1022, 1004, 1001, 1049, 1060, 1064, 1052, 992, 1026, 1017, 1015, 1051, 1055, 1065, 1041, 1029, 986, 996, 1016, 1054, 1012, 987, 1012, 1068, 1028, 1047, 1024, 991, 1018, 1056, 1057, 1054, 1008, 1030, 1028, 1013, 1011, 1039, 1035, 1059, 1014, 1055, 998, 1024, 974, 1013, 994, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1003, 998, 1020, 1045, 1017, 1046, 976, 1016, 1045, 1027, 1026, 1018, 1018, 1057, 1016, 988, 985, 1061, 982, 1019, 1051, 1017, 1053, 1016, 1120, 996, 991, 1063, 992, 1038, 997, 976, 976, 1041, 1016, 1026, 1013, 1036, 1022, 1040, 1043, 1064, 1019, 1044, 1018, 1080, 1020, 980, 1000, 1064, 1032, 1025, 1025, 1057, 1006, 1026, 1037, 1012, 1009, 1017, 1041, 1049, 983, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[969, 1011, 1037, 1041, 1033, 1014, 1007, 1025, 1049, 1021, 1019, 1062, 989, 989, 1097, 1029, 1013, 1006, 1056, 1012, 996, 1053, 1010, 1000, 1044, 1044, 1006, 1003, 1004, 1037, 1045, 995, 1004, 1022, 1011, 1019, 996, 1026, 1054, 1004, 1006, 1037, 1024, 1062, 1080, 1007, 999, 1046, 1041, 995, 1012, 1014, 1074, 998, 1030, 1066, 1025, 1019, 986, 1063, 1037, 1016, 1043, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1042, 981, 1024, 948, 991, 1035, 1036, 979, 990, 1011, 1035, 1051, 1016, 1049, 1041, 997, 1028, 1010, 976, 1061, 1068, 1003, 1042, 1037, 988, 982, 1079, 1001, 1012, 1019, 1052, 1100, 1038, 1014, 1016, 1010, 1056, 1008, 966, 1045, 1022, 1043, 1008, 1031, 1046, 986, 1036, 1011, 1047, 1035, 1077, 1016, 1063, 1013, 1076, 1006, 1037, 1006, 1037, 1007, 984, 1042, 1035, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1076, 1024, 991, 1023, 995, 1094, 1011, 997, 1071, 1047, 1030, 1019, 1003, 991, 1034, 1068, 1031, 1005, 1013, 1022, 1072, 1054, 1023, 1043, 1073, 1008, 995, 1067, 1003, 974, 1027, 990, 1037, 1027, 1024, 971, 1080, 1049, 999, 1044, 1008, 997, 991, 1026, 995, 1015, 1024, 991, 1044, 1004, 964, 1022, 1020, 1015, 1029, 1008, 1020, 1019, 1097, 1016, 1019, 1022, 1018, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1028, 1026, 1040, 1038, 1049, 1023, 1018, 1009, 1041, 1041, 1092, 978, 1022, 1014, 1031, 1040, 1025, 1043, 1016, 1023, 1097, 1012, 988, 1049, 1036, 1035, 994, 1037, 1030, 1029, 995, 974, 1016, 1050, 1020, 1003, 984, 1006, 1060, 1078, 1073, 1021, 1060, 1006, 1012, 1022, 1039, 986, 981, 988, 1016, 1016, 1015, 1047, 1015, 1043, 1025, 979, 1026, 1030, 1012, 998, 1030, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1098, 1052, 1029, 1048, 1061, 1014, 969, 1050, 977, 1031, 1031, 981, 1059, 1063, 1005, 1020, 1031, 1013, 1035, 1010, 1054, 1039, 1025, 1027, 1045, 1063, 968, 1057, 1055, 996, 979, 1053, 983, 995, 1030, 1043, 1011, 1036, 1027, 971, 1033, 978, 1012, 997, 1039, 993, 1060, 1020, 1018, 1027, 1077, 975, 1025, 992, 1022, 1050, 1018, 1019, 1008, 1027, 1000, 1042, 1039, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1003, 1015, 1003, 1042, 1010, 1045, 1010, 1086, 987, 1003, 1026, 1047, 1048, 1044, 985, 1001, 990, 1069, 945, 1044, 1051, 986, 1017, 1057, 1002, 990, 1004, 1034, 1093, 1025, 1024, 1033, 1000, 1044, 1029, 966, 1005, 1010, 1024, 1032, 1073, 1076, 1010, 1052, 1076, 1032, 1046, 1018, 1015, 1005, 1016, 993, 1053, 1016, 1064, 1001, 1034, 1020, 1053, 997, 945, 1031, 1020, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[968, 996, 1020, 1028, 1025, 1015, 1013, 1009, 14, 10, 0, 0, 0, 0, 0, 0, 999, 978, 984, 1063, 1035, 1008, 1062, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1004, 1001, 1049, 1039, 1064, 1052, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1017, 1015, 1051, 1055, 1065, 1041, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 986, 996, 1016, 1054, 1012, 987, 1012, 1068, 28, 7, 0, 0, 0, 0, 0, 0, 1000, 1047, 1024, 991, 1018, 1056, 1057, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1030, 1028, 1013, 1004, 1039, 1035, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1055, 998, 1024, 974, 1013, 994, 1032, 7, 5, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8171, 8223, 8295, 8166, 8247, 8216, 8120] avg=8192.0000 max=8295 max_rank=3 min=8098 min_rank=0 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1003, 998, 1020, 1045, 1017, 1046, 976, 1016, 13, 9, 0, 0, 0, 0, 0, 0, 1045, 1027, 1026, 1018, 1018, 1057, 1002, 988, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1061, 982, 1019, 1031, 1017, 1053, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 996, 991, 1063, 992, 1038, 997, 976, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1041, 1016, 1026, 1013, 1036, 1022, 1040, 29, 7, 0, 0, 0, 0, 0, 0, 1014, 1064, 1019, 1044, 1018, 1080, 1020, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1064, 1032, 1025, 1018, 1057, 1006, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1012, 1009, 1017, 1041, 1049, 983, 1042, 7, 5, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8181, 8164, 8169, 8206, 8239, 8228, 8206] avg=8192.0000 max=8239 max_rank=5 min=8143 min_rank=0 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32879 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1042, 981, 1024, 948, 991, 1035, 1036, 979, 14, 9, 0, 0, 0, 0, 0, 0, 990, 1011, 1035, 1051, 1016, 1049, 1027, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1010, 976, 1061, 1047, 1003, 1042, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 984, 982, 1079, 1001, 1012, 1019, 1052, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1014, 1016, 1010, 1056, 1008, 966, 1045, 28, 7, 0, 0, 0, 0, 0, 0, 994, 1043, 1008, 1031, 1046, 986, 1036, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1035, 1077, 1016, 1056, 1013, 1076, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1006, 1037, 1007, 984, 1042, 1035, 1035, 7, 5, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8059, 8176, 8204, 8229, 8188, 8155, 8326, 8199] avg=8192.0000 max=8326 max_rank=6 min=8059 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32868 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1076, 1024, 991, 1023, 995, 1094, 1011, 997, 14, 9, 0, 0, 0, 0, 0, 0, 1071, 1047, 1030, 1019, 1003, 991, 1020, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1005, 1013, 1022, 1051, 1054, 1023, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1008, 995, 1067, 1003, 974, 1027, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1027, 1024, 971, 1080, 1049, 999, 1044, 28, 7, 0, 0, 0, 0, 0, 0, 980, 997, 991, 1026, 995, 1015, 1024, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1004, 964, 1022, 1013, 1015, 1029, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1019, 1097, 1016, 1019, 1022, 1018, 1067, 7, 5, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8249, 8242, 8133, 8266, 8019, 8099, 8294] avg=8192.0000 max=8294 max_rank=7 min=8019 min_rank=5 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[969, 1011, 1037, 1041, 1033, 1014, 1007, 1025, 13, 10, 0, 0, 0, 0, 0, 0, 1049, 1021, 1019, 1062, 989, 989, 1082, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1006, 1056, 1012, 976, 1053, 1010, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1044, 1006, 1003, 1004, 1037, 1045, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1022, 1011, 1019, 996, 1026, 1054, 1004, 28, 7, 0, 0, 0, 0, 0, 0, 978, 1037, 1024, 1062, 1080, 1007, 999, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 995, 1012, 1014, 1067, 998, 1030, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1019, 986, 1063, 1037, 1016, 1043, 1004, 7, 5, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8240, 8126, 8174, 8171, 8233, 8223, 8209] avg=8192.0000 max=8240 max_rank=1 min=8126 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1098, 1052, 1029, 1048, 1061, 1014, 969, 1050, 13, 9, 0, 0, 0, 0, 0, 0, 977, 1031, 1031, 981, 1059, 1063, 992, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1013, 1035, 1010, 1034, 1039, 1025, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1063, 968, 1057, 1055, 996, 979, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 983, 995, 1030, 1043, 1011, 1036, 1027, 971, 28, 7, 0, 0, 0, 0, 0, 0, 1005, 978, 1012, 997, 1039, 993, 1060, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1027, 1077, 975, 1018, 992, 1022, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1019, 1008, 1027, 1000, 1042, 1039, 1031, 7, 4, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8343, 8154, 8214, 8212, 8131, 8104, 8179, 8199] avg=8192.0000 max=8343 max_rank=0 min=8104 min_rank=5 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32613 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1028, 1026, 1040, 1038, 1049, 1023, 1018, 1009, 14, 9, 0, 0, 0, 0, 0, 0, 1041, 1041, 1092, 978, 1022, 1014, 1017, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1043, 1016, 1023, 1076, 1012, 988, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1035, 994, 1037, 1030, 1029, 995, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1050, 1020, 1003, 984, 1006, 1060, 1078, 30, 7, 0, 0, 0, 0, 0, 0, 1043, 1021, 1060, 1006, 1012, 1022, 1039, 986, 0, 0, 0, 0, 0, 0, 0, 0, 981, 988, 1016, 1016, 1008, 1047, 1015, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 979, 1026, 1030, 1012, 998, 1030, 1006, 7, 5, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8245, 8232, 8126, 8254, 8189, 8114, 8122] avg=8192.0000 max=8254 max_rank=0 min=8114 min_rank=6 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1003, 1015, 1003, 1042, 1010, 1045, 1010, 1086, 17, 14, 0, 0, 0, 0, 0, 0, 987, 1003, 1026, 1047, 1048, 1044, 964, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1069, 945, 1044, 1023, 986, 1017, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 995, 990, 1004, 1034, 1093, 1025, 1024, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1044, 1029, 966, 1005, 1010, 1024, 1032, 33, 11, 0, 0, 0, 0, 0, 0, 1040, 1076, 1010, 1052, 1076, 1032, 1046, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1005, 1016, 993, 1041, 1016, 1064, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1020, 1053, 997, 945, 1031, 1020, 1061, 12, 7, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8120, 8131, 8198, 8154, 8350, 8151, 8187] avg=8192.0000 max=8350 max_rank=5 min=8120 min_rank=1 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8192, 8064, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8064, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8064, 8320, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8064, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1045, 1002, 1042, 1054, 1061, 997, 1008, 1028, 1029, 984, 1028, 1039, 1052, 988, 1038, 1023, 1076, 1070, 1071, 1015, 1016, 1016, 1018, 1020, 1037, 1056, 995, 1000, 1025, 1024, 1036, 1062, 1048, 1065, 1008, 1044, 982, 1021, 996, 1019, 1025, 1050, 992, 1067, 1005, 1024, 1007, 1038, 1027, 1013, 1018, 1019, 1013, 991, 1014, 1042, 1017, 999, 1021, 999, 981, 1011, 1048, 977] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1034, 1007, 1035, 1022, 1044, 1019, 1068, 1059, 1058, 1031, 1042, 1020, 1084, 1018, 1011, 1019, 1023, 1055, 1016, 1025, 1008, 981, 1018, 1062, 1041, 999, 1042, 1037, 1054, 1020, 1000, 1023, 1004, 994, 1016, 998, 1003, 1039, 1025, 989, 1048, 1014, 1030, 1024, 1021, 1001, 974, 983, 1036, 1047, 992, 1050, 1007, 945, 1023, 1052, 1000, 1055, 1048, 1048, 1028, 1022, 1031, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[982, 1042, 1046, 1039, 1050, 965, 1028, 1003, 984, 1064, 1064, 1050, 1039, 1032, 988, 1048, 1022, 1038, 985, 1017, 1062, 970, 1056, 1068, 988, 1015, 1005, 1057, 1020, 1048, 1073, 1003, 1066, 1019, 1059, 1058, 1011, 1085, 1002, 1024, 990, 1037, 1030, 1049, 1001, 1022, 952, 978, 1035, 1009, 1023, 950, 1024, 1014, 1028, 1044, 1010, 1043, 1005, 1047, 1034, 1008, 988, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1053, 979, 1010, 983, 1024, 1038, 1011, 978, 1018, 1005, 1040, 1013, 1041, 1029, 1031, 1032, 1017, 1021, 1003, 1047, 1040, 1042, 1049, 1007, 1029, 1052, 1030, 1028, 1023, 1017, 1036, 991, 1064, 1067, 1070, 986, 995, 1070, 1040, 1038, 1005, 1041, 966, 1028, 1013, 1019, 1017, 1063, 1055, 1021, 997, 1030, 1086, 1034, 1023, 958, 1002, 995, 1037, 1040, 1071, 983, 1002, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1056, 1051, 1079, 1064, 999, 1007, 1004, 1105, 1011, 1021, 1034, 1041, 993, 1024, 972, 1035, 987, 1001, 1040, 1020, 974, 1053, 1030, 1043, 1030, 980, 1075, 1044, 1020, 1031, 1013, 996, 1014, 995, 1055, 1080, 1061, 962, 1059, 1016, 994, 1038, 1034, 1002, 968, 1053, 992, 1037, 1036, 1058, 980, 1007, 984, 998, 1077, 1030, 1051, 1040, 1027, 1006, 1028, 1016, 1029, 976] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1030, 1001, 1009, 1005, 1001, 1095, 986, 1048, 1016, 969, 1020, 1056, 1013, 1029, 1001, 1069, 1028, 1008, 970, 1023, 979, 1040, 989, 1060, 1028, 981, 1056, 991, 1056, 1040, 1042, 987, 1031, 1064, 963, 1041, 1000, 1067, 1056, 1024, 1054, 1036, 1070, 1032, 1006, 990, 995, 943, 1055, 1027, 985, 1038, 1029, 1032, 1008, 1033, 1029, 1014, 1075, 1041, 1029, 1006, 1072, 1065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1011, 1066, 1047, 1030, 1017, 1006, 1046, 1062, 1018, 982, 1048, 1022, 1048, 999, 1010, 1024, 1053, 1087, 1039, 991, 1015, 994, 1027, 1028, 1006, 1009, 1066, 1005, 987, 1039, 983, 1002, 1050, 1023, 1004, 1038, 967, 1059, 1021, 1042, 1001, 990, 994, 998, 1036, 1029, 999, 1063, 978, 1022, 1033, 982, 1024, 1066, 1060, 1042, 1036, 1051, 1050, 1042, 1050, 1038, 1006, 975] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1037, 1013, 1070, 1031, 977, 1028, 1036, 1033, 1011, 1009, 1042, 1007, 1045, 1003, 1013, 1014, 1009, 1060, 1000, 1032, 1036, 1059, 1001, 955, 1040, 1045, 1047, 981, 1017, 1042, 1015, 1018, 1032, 1006, 986, 1049, 996, 1069, 1029, 1052, 1010, 1014, 1090, 1033, 1000, 991, 1017, 1016, 1020, 1053, 1008, 1021, 988, 1017, 1046, 1045, 1074, 1038, 1029, 1032, 1030, 1018, 989, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1034, 1007, 1003, 1022, 1044, 1019, 1068, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1031, 1042, 1020, 1084, 1018, 1011, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1052, 1016, 1025, 1008, 981, 1018, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 999, 1042, 1037, 1054, 1020, 1000, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 994, 1016, 998, 1003, 1005, 1025, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1014, 1030, 1024, 1021, 1001, 974, 983, 34, 26, 0, 0, 0, 0, 0, 0, 1036, 1047, 992, 1050, 1007, 945, 1023, 1052, 6, 5, 3, 0, 0, 0, 0, 0, 1000, 1055, 1043, 1048, 1028, 1022, 1031, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8283, 8185, 8216, 8034, 8155, 8166, 8241] avg=8192.0000 max=8283 max_rank=1 min=8034 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[982, 1042, 1013, 1039, 1050, 965, 1028, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1064, 1064, 1050, 1039, 1032, 988, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1035, 985, 1017, 1062, 970, 1056, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1015, 1005, 1057, 1020, 1048, 1073, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1019, 1059, 1058, 1011, 1049, 1002, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1037, 1030, 1049, 1001, 1022, 952, 978, 36, 27, 0, 0, 0, 0, 0, 0, 1035, 1009, 1023, 950, 1024, 1014, 1028, 1044, 6, 5, 3, 0, 0, 0, 0, 0, 1010, 1043, 1000, 1047, 1034, 1008, 988, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8269, 8215, 8209, 8288, 8122, 8141, 8170] avg=8192.0000 max=8288 max_rank=4 min=8122 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1053, 979, 978, 983, 1024, 1038, 1011, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1005, 1040, 1013, 1041, 1029, 1031, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1018, 1003, 1047, 1040, 1042, 1049, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1052, 1030, 1028, 1023, 1017, 1036, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1067, 1070, 986, 995, 1035, 1040, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1041, 966, 1028, 1013, 1019, 1017, 1063, 35, 26, 0, 0, 0, 0, 0, 0, 1055, 1021, 997, 1030, 1086, 1034, 1023, 958, 6, 5, 3, 0, 0, 0, 0, 0, 1002, 995, 1032, 1040, 1071, 983, 1002, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8044, 8209, 8223, 8206, 8295, 8213, 8218, 8128] avg=8192.0000 max=8295 max_rank=4 min=8044 min_rank=0 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1045, 1002, 1009, 1054, 1061, 997, 1008, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 984, 1028, 1039, 1052, 988, 1038, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1067, 1071, 1015, 1016, 1016, 1018, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1056, 995, 1000, 1025, 1024, 1036, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1065, 1008, 1044, 982, 987, 996, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1050, 992, 1067, 1005, 1024, 1007, 1038, 34, 27, 0, 0, 0, 0, 0, 0, 1027, 1013, 1018, 1019, 1013, 991, 1014, 1042, 6, 5, 3, 0, 0, 0, 0, 0, 1017, 999, 1016, 999, 981, 1011, 1048, 977, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8181, 8299, 8235, 8149, 8269, 8151, 8048] avg=8192.0000 max=8299 max_rank=2 min=8048 min_rank=7 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32617 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1056, 1051, 1045, 1064, 999, 1007, 1004, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1021, 1034, 1041, 993, 1024, 972, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 987, 998, 1040, 1020, 974, 1053, 1030, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 980, 1075, 1044, 1020, 1031, 1013, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 995, 1055, 1080, 1061, 930, 1059, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1038, 1034, 1002, 968, 1053, 992, 1037, 32, 28, 0, 0, 0, 0, 0, 0, 1036, 1058, 980, 1007, 984, 998, 1077, 1030, 6, 5, 3, 0, 0, 0, 0, 0, 1051, 1040, 1022, 1006, 1028, 1016, 1029, 976, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8331, 8131, 8145, 8189, 8210, 8178, 8184, 8168] avg=8192.0000 max=8331 max_rank=0 min=8131 min_rank=1 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1030, 1001, 977, 1005, 1001, 1095, 986, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 969, 1020, 1056, 1013, 1029, 1001, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1005, 970, 1023, 979, 1040, 989, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 981, 1056, 991, 1056, 1040, 1042, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1064, 963, 1041, 1000, 1032, 1056, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1036, 1070, 1032, 1006, 990, 995, 943, 35, 26, 0, 0, 0, 0, 0, 0, 1055, 1027, 985, 1038, 1029, 1032, 1008, 1033, 6, 5, 3, 0, 0, 0, 0, 0, 1029, 1014, 1070, 1041, 1029, 1006, 1072, 1065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8173, 8094, 8181, 8211, 8187, 8221, 8326] avg=8192.0000 max=8326 max_rank=7 min=8094 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32945 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1011, 1066, 1014, 1030, 1017, 1006, 1046, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 982, 1048, 1022, 1048, 999, 1010, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1084, 1039, 991, 1015, 994, 1027, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1009, 1066, 1005, 987, 1039, 983, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1023, 1004, 1038, 967, 1024, 1021, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 990, 994, 998, 1036, 1029, 999, 1063, 35, 27, 0, 0, 0, 0, 0, 0, 978, 1022, 1033, 982, 1024, 1066, 1060, 1042, 6, 5, 3, 0, 0, 0, 0, 0, 1036, 1051, 1045, 1042, 1050, 1038, 1006, 975, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8151, 8231, 8097, 8169, 8172, 8221, 8243] avg=8192.0000 max=8252 max_rank=0 min=8097 min_rank=3 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1037, 1013, 1029, 1031, 977, 1028, 1036, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1009, 1037, 1007, 1045, 1003, 1013, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1052, 1000, 1032, 1036, 1059, 1001, 955, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1045, 1045, 981, 1017, 1042, 1015, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1006, 986, 1049, 996, 1030, 1029, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1014, 1090, 1033, 1000, 991, 1017, 1016, 39, 30, 0, 0, 0, 0, 0, 0, 1020, 1053, 1008, 1021, 988, 1017, 1046, 1045, 11, 10, 8, 5, 2, 0, 0, 0, 1074, 1038, 1019, 1032, 1030, 1018, 989, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8139, 8144, 8203, 8180, 8240, 8234, 8212] avg=8192.0000 max=8240 max_rank=5 min=8139 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32866 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8064, 8192, 8064, 8192, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8064, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8320, 8320, 8064, 8192, 8192, 8192, 8320, 8320, 128, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1038, 991, 971, 984, 1056, 990, 1021, 1045, 1044, 1029, 1014, 998, 1021, 1046, 1045, 1028, 983, 1018, 1069, 1008, 1017, 1028, 1046, 1031, 988, 1076, 1045, 1029, 1003, 988, 1013, 998, 1031, 1009, 1048, 1004, 1070, 989, 1013, 1006, 1005, 1006, 1019, 1055, 1077, 1038, 1052, 1042, 1017, 1026, 1038, 1053, 1032, 997, 972, 1030, 1086, 992, 1014, 1042, 1017, 1045, 1040, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1009, 1000, 1040, 1038, 1031, 1003, 1045, 1058, 981, 1056, 985, 1002, 1048, 1052, 969, 1015, 1010, 1006, 1019, 1038, 1017, 1014, 995, 1022, 1025, 1021, 1051, 1065, 990, 1053, 1008, 1083, 1034, 1071, 1018, 1073, 986, 1061, 1026, 975, 1003, 1028, 1049, 964, 1028, 1003, 1016, 1008, 991, 1041, 1061, 1063, 1035, 1008, 1012, 1044, 1022, 1016, 1001, 994, 1040, 1052, 1012, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1032, 974, 1014, 1030, 959, 1030, 1023, 1027, 1048, 1034, 980, 1005, 983, 1010, 1078, 1067, 1013, 1032, 1016, 1049, 1047, 1038, 1046, 1040, 1029, 1048, 1044, 1050, 1015, 1037, 1053, 1032, 993, 1038, 995, 1018, 1042, 996, 1092, 1036, 1026, 1042, 1039, 1012, 1018, 958, 1028, 1035, 1022, 955, 976, 976, 1037, 1034, 1034, 1001, 994, 1086, 1052, 1019, 1041, 1048, 992, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1020, 1002, 1015, 1073, 1037, 1056, 1003, 1001, 989, 1039, 1022, 1038, 1049, 1061, 1071, 1014, 1041, 994, 995, 987, 1037, 1044, 1023, 1046, 1021, 1039, 990, 1065, 1059, 962, 1000, 1036, 1005, 1015, 974, 1041, 1024, 1041, 1051, 1016, 1003, 991, 1057, 1019, 1026, 1008, 1047, 1041, 1003, 989, 1010, 946, 991, 1022, 1042, 1117, 1054, 997, 1043, 1015, 1044, 1016, 1007, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1013, 1037, 1035, 1017, 1007, 1003, 1020, 1047, 1079, 1004, 1058, 1014, 1061, 1015, 1052, 1019, 1048, 1001, 1040, 1052, 976, 969, 1064, 1025, 1059, 1001, 972, 1061, 1044, 1064, 1039, 1013, 1009, 992, 983, 1008, 987, 1014, 1050, 1047, 1007, 1020, 1018, 1031, 1039, 1047, 1005, 1043, 988, 1041, 1040, 1014, 1002, 1015, 1020, 1001, 1039, 1051, 1013, 1014, 1043, 984, 1051, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1051, 986, 1006, 986, 1016, 1007, 1004, 1006, 978, 975, 1004, 1037, 977, 998, 994, 1000, 1020, 1006, 1081, 1040, 991, 1019, 1065, 1029, 1024, 1060, 1059, 1057, 1030, 1006, 1104, 1051, 1002, 1065, 1095, 1014, 1051, 991, 1047, 1049, 1021, 1020, 1025, 1054, 952, 1034, 1040, 1037, 1066, 1012, 980, 1074, 997, 1025, 999, 1011, 998, 1062, 1022, 1018, 1004, 1017, 1001, 1086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1024, 1015, 1022, 1004, 1022, 1019, 1063, 1002, 991, 1023, 1035, 1020, 1055, 995, 1028, 996, 1026, 1026, 1004, 1032, 1015, 1049, 990, 1086, 1019, 992, 1054, 1065, 1028, 968, 1049, 1005, 1035, 1048, 1036, 1018, 1024, 1015, 1012, 1078, 1000, 1016, 971, 1019, 1047, 1052, 987, 1030, 1036, 1058, 994, 1020, 1020, 1013, 1002, 1000, 1022, 1027, 1053, 1034, 1074, 1043, 1016, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[990, 1045, 1017, 1023, 1051, 1065, 1053, 994, 1081, 962, 1077, 1070, 1007, 1010, 1019, 1006, 1039, 1013, 1011, 1002, 1054, 1000, 995, 1090, 1032, 999, 1012, 999, 999, 1055, 997, 1049, 997, 1025, 1065, 1012, 1017, 1011, 1026, 1016, 1005, 1068, 978, 1051, 990, 1009, 1024, 1029, 1005, 1005, 1040, 1029, 1013, 1002, 1053, 1029, 1006, 994, 1008, 1056, 1072, 1019, 1025, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1009, 1000, 1040, 1038, 1031, 1003, 1045, 1058, 33, 0, 0, 0, 0, 0, 0, 0, 981, 1056, 985, 1002, 1048, 1052, 969, 1015, 9, 0, 0, 0, 0, 0, 0, 0, 1010, 1006, 1019, 1038, 1017, 1014, 995, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1021, 1051, 1022, 990, 1053, 1008, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1071, 1018, 1073, 986, 1061, 1015, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1028, 1049, 964, 1028, 1003, 1016, 1008, 11, 1, 1, 0, 0, 0, 0, 0, 991, 1041, 1061, 1063, 1035, 1008, 1012, 1044, 43, 1, 0, 0, 0, 0, 0, 0, 1022, 1016, 1001, 994, 1005, 1052, 1012, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8117, 8111, 8253, 8233, 8112, 8299, 8154] avg=8192.0000 max=8299 max_rank=6 min=8111 min_rank=2 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1038, 991, 971, 984, 1056, 990, 1021, 1045, 32, 0, 0, 0, 0, 0, 0, 0, 1044, 1029, 1014, 998, 1021, 1046, 1045, 1028, 9, 0, 0, 0, 0, 0, 0, 0, 983, 1018, 1069, 1008, 1017, 1028, 1046, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1076, 1045, 987, 1003, 988, 1013, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1009, 1048, 1004, 1070, 989, 1002, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1006, 1019, 1055, 1077, 1038, 1052, 1042, 11, 1, 1, 0, 0, 0, 0, 0, 1017, 1026, 1038, 1053, 1032, 997, 972, 1030, 42, 1, 0, 0, 0, 0, 0, 0, 1086, 992, 1014, 1042, 983, 1045, 1040, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8234, 8190, 8098, 8159, 8307, 8208, 8212] avg=8192.0000 max=8307 max_rank=5 min=8098 min_rank=3 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32886 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1032, 974, 1014, 1030, 959, 1030, 1023, 1027, 33, 0, 0, 0, 0, 0, 0, 0, 1048, 1034, 980, 1005, 983, 1010, 1078, 1067, 9, 0, 0, 0, 0, 0, 0, 0, 1013, 1032, 1016, 1049, 1047, 1038, 1046, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1048, 1044, 1007, 1015, 1037, 1053, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1038, 995, 1018, 1042, 996, 1080, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1042, 1039, 1012, 1018, 958, 1028, 1035, 12, 1, 1, 0, 0, 0, 0, 0, 1022, 955, 976, 976, 1037, 1034, 1034, 1001, 43, 1, 0, 0, 0, 0, 0, 0, 994, 1086, 1052, 1019, 1006, 1048, 992, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8214, 8271, 8265, 8198, 8172, 8079, 8215] avg=8192.0000 max=8271 max_rank=2 min=8079 min_rank=6 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1013, 1037, 1035, 1017, 1007, 1003, 1020, 1047, 33, 0, 0, 0, 0, 0, 0, 0, 1079, 1004, 1058, 1014, 1061, 1015, 1052, 1019, 9, 0, 0, 0, 0, 0, 0, 0, 1048, 1001, 1040, 1052, 976, 969, 1064, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1001, 972, 1018, 1044, 1064, 1039, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 992, 983, 1008, 987, 1014, 1039, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1020, 1018, 1031, 1039, 1047, 1005, 1043, 11, 1, 1, 0, 0, 0, 0, 0, 988, 1041, 1040, 1014, 1002, 1015, 1020, 1001, 43, 1, 0, 0, 0, 0, 0, 0, 1039, 1051, 1013, 1014, 1008, 984, 1051, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8311, 8165, 8210, 8079, 8223, 8165, 8171] avg=8192.0000 max=8311 max_rank=1 min=8079 min_rank=4 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32638 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1051, 986, 1006, 986, 1016, 1007, 1004, 1006, 32, 0, 0, 0, 0, 0, 0, 0, 978, 975, 1004, 1037, 977, 998, 994, 1000, 9, 0, 0, 0, 0, 0, 0, 0, 1020, 1006, 1081, 1040, 991, 1019, 1065, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1060, 1059, 1014, 1030, 1006, 1104, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1065, 1095, 1014, 1051, 991, 1036, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1020, 1025, 1054, 952, 1034, 1040, 1037, 11, 1, 1, 0, 0, 0, 0, 0, 1066, 1012, 980, 1074, 997, 1025, 999, 1011, 43, 1, 0, 0, 0, 0, 0, 0, 998, 1062, 1022, 1018, 970, 1017, 1001, 1086, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8094, 7972, 8241, 8348, 8303, 8196, 8208, 8174] avg=8192.0000 max=8348 max_rank=3 min=7972 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1020, 1002, 1015, 1073, 1037, 1056, 1003, 1001, 33, 0, 0, 0, 0, 0, 0, 0, 989, 1039, 1022, 1038, 1049, 1061, 1071, 1014, 9, 0, 0, 0, 0, 0, 0, 0, 1041, 994, 995, 987, 1037, 1044, 1023, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1039, 990, 1022, 1059, 962, 1000, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1015, 974, 1041, 1024, 1041, 1040, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 991, 1057, 1019, 1026, 1008, 1047, 1041, 11, 1, 1, 0, 0, 0, 0, 0, 1003, 989, 1010, 946, 991, 1022, 1042, 1117, 43, 1, 0, 0, 0, 0, 0, 0, 1054, 997, 1043, 1015, 1009, 1016, 1007, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8292, 8157, 8129, 8156, 8205, 8164, 8193] avg=8192.0000 max=8292 max_rank=1 min=8129 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1024, 1015, 1022, 1004, 1022, 1019, 1063, 1002, 34, 0, 0, 0, 0, 0, 0, 0, 991, 1023, 1035, 1020, 1055, 995, 1028, 996, 9, 0, 0, 0, 0, 0, 0, 0, 1026, 1026, 1004, 1032, 1015, 1049, 990, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 992, 1054, 1022, 1028, 968, 1049, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1048, 1036, 1018, 1024, 1015, 1001, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1016, 971, 1019, 1047, 1052, 987, 1030, 11, 2, 1, 0, 0, 0, 0, 0, 1036, 1058, 994, 1020, 1020, 1013, 1002, 1000, 43, 1, 0, 0, 0, 0, 0, 0, 1022, 1027, 1053, 1034, 1038, 1043, 1016, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8152, 8217, 8137, 8255, 8136, 8187, 8247] avg=8192.0000 max=8255 max_rank=4 min=8136 min_rank=5 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[990, 1045, 1017, 1023, 1051, 1065, 1053, 994, 40, 0, 0, 0, 0, 0, 0, 0, 1081, 962, 1077, 1070, 1007, 1010, 1019, 1006, 12, 0, 0, 0, 0, 0, 0, 0, 1039, 1013, 1011, 1002, 1054, 1000, 995, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 999, 1012, 953, 999, 1055, 997, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1025, 1065, 1012, 1017, 1011, 1010, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1068, 978, 1051, 990, 1009, 1024, 1029, 16, 8, 7, 0, 0, 0, 0, 0, 1005, 1005, 1040, 1029, 1013, 1002, 1053, 1029, 46, 4, 0, 0, 0, 0, 0, 0, 1006, 994, 1008, 1056, 1021, 1019, 1025, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8244, 8184, 8096, 8153, 8185, 8226, 8170] avg=8192.0000 max=8278 max_rank=0 min=8096 min_rank=3 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8064, 8192, 8192, 8192, 8192, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8064, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8064, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1021, 1025, 1042, 988, 1024, 1020, 1015, 1038, 1000, 1040, 1025, 1011, 1018, 1005, 1034, 1061, 1023, 977, 1001, 1071, 1019, 1054, 1054, 998, 1113, 1048, 1029, 1025, 981, 1018, 1021, 1039, 1015, 960, 1002, 1012, 1004, 1066, 1009, 1026, 1025, 1034, 1059, 997, 1029, 1030, 1020, 1055, 946, 1069, 1028, 1052, 1068, 1024, 1002, 985, 1063, 1013, 998, 991, 1009, 1062, 1030, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1001, 1042, 1050, 1048, 1017, 1027, 1049, 1024, 1014, 1036, 1038, 1029, 1055, 1015, 978, 990, 1006, 1039, 1014, 1017, 1043, 994, 1083, 1000, 1001, 1048, 1004, 1050, 1019, 1093, 999, 1020, 1014, 979, 987, 1015, 1011, 993, 1023, 1045, 1026, 1022, 1014, 999, 1048, 1020, 1044, 1005, 996, 1056, 1043, 1046, 1056, 1052, 990, 1017, 1003, 1071, 1001, 1074, 1029, 1002, 1006, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1041, 995, 1037, 1060, 968, 1089, 1027, 1031, 1063, 1000, 1033, 1011, 1035, 1026, 1007, 1018, 985, 976, 1029, 1067, 982, 990, 1000, 1028, 1038, 996, 1037, 1034, 1033, 988, 1042, 1045, 994, 1051, 1034, 957, 1011, 1004, 1067, 1053, 1019, 983, 1000, 1024, 1027, 1045, 991, 1043, 1025, 1022, 1064, 1045, 1028, 1027, 1062, 1024, 1042, 1029, 1004, 998, 1015, 1053, 1028, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1002, 1056, 1043, 1030, 962, 1047, 984, 1016, 1028, 1032, 1038, 983, 1044, 1021, 1008, 1014, 1030, 1044, 973, 990, 1009, 1021, 1030, 1040, 1027, 1009, 1066, 1018, 1038, 1025, 1027, 1035, 1066, 1031, 1046, 980, 1009, 996, 984, 1042, 1072, 998, 1043, 1009, 1036, 1066, 1026, 1031, 996, 980, 1010, 1048, 1053, 1033, 1056, 1007, 1041, 1073, 1078, 986, 1026, 1014, 1009, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1069, 1019, 1029, 1017, 1042, 1064, 977, 990, 1011, 987, 990, 1030, 1033, 1041, 1010, 1034, 1019, 1055, 1049, 995, 1030, 995, 1013, 1059, 1109, 1014, 1045, 956, 1011, 999, 1016, 1062, 1059, 1027, 1022, 988, 1052, 1007, 1008, 996, 1035, 1055, 1016, 972, 1012, 985, 1015, 982, 1055, 1033, 979, 1026, 1067, 1035, 1028, 1015, 1047, 991, 1072, 1048, 1053, 1035, 1051, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1023, 1019, 1054, 1028, 1054, 1027, 1050, 1017, 1023, 964, 1000, 1017, 1034, 992, 1027, 997, 996, 1016, 1016, 1025, 1007, 963, 1036, 1050, 1028, 1008, 1043, 1035, 1030, 1057, 1046, 1027, 1034, 984, 1028, 1045, 983, 998, 1028, 1022, 1072, 1030, 1044, 1017, 1028, 1055, 980, 1012, 1038, 1053, 957, 1029, 1045, 1040, 970, 1058, 1071, 965, 1038, 1021, 1101, 1007, 1038, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1033, 1024, 1051, 1040, 1022, 1020, 1005, 1064, 985, 1039, 1021, 995, 1017, 1000, 1014, 1014, 993, 1004, 1028, 1050, 955, 1022, 1049, 1057, 1008, 1052, 1030, 998, 1064, 977, 1025, 1016, 1027, 1012, 1026, 1050, 1001, 1056, 995, 1075, 1027, 1015, 1073, 1055, 1016, 986, 1016, 1038, 993, 1030, 1027, 1004, 1050, 995, 1013, 978, 1052, 1020, 1081, 1045, 1062, 1015, 1010, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[991, 987, 1029, 1057, 1009, 1032, 1035, 1045, 1028, 1035, 1004, 1004, 1032, 1011, 1053, 1022, 981, 1043, 996, 1012, 1019, 1022, 1029, 1002, 1007, 1045, 1007, 1094, 997, 1071, 1027, 1028, 1013, 1041, 1082, 1016, 1085, 1034, 1018, 1038, 1036, 1020, 1057, 1028, 1020, 1004, 1050, 1013, 1032, 997, 1060, 994, 1007, 1049, 1003, 988, 1010, 1022, 977, 1065, 1033, 1002, 946, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1021, 1025, 1017, 988, 1024, 1020, 1015, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1040, 1025, 1011, 1018, 1005, 1034, 1061, 33, 12, 0, 0, 0, 0, 0, 0, 1023, 977, 1001, 1071, 1019, 1054, 1054, 998, 48, 0, 0, 0, 0, 0, 0, 0, 1065, 1048, 1029, 1025, 981, 1018, 1021, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 960, 1002, 1012, 1004, 1066, 1009, 1026, 13, 8, 2, 0, 0, 0, 0, 0, 1017, 1034, 1059, 997, 1029, 1030, 1020, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 946, 1069, 1028, 1052, 1066, 1024, 1002, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1013, 998, 991, 1009, 1062, 1030, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8239, 8245, 8226, 8117, 8241, 8172, 8148] avg=8192.0000 max=8245 max_rank=2 min=8117 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1001, 1042, 1025, 1048, 1017, 1027, 1049, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1036, 1038, 1029, 1055, 1015, 978, 990, 32, 12, 0, 0, 0, 0, 0, 0, 1006, 1039, 1014, 1017, 1043, 994, 1083, 1000, 43, 0, 0, 0, 0, 0, 0, 0, 958, 1048, 1004, 1050, 1019, 1093, 999, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 979, 987, 1015, 1011, 993, 1023, 1045, 13, 8, 2, 0, 0, 0, 0, 0, 1018, 1022, 1014, 999, 1048, 1020, 1044, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1056, 1043, 1046, 1054, 1052, 990, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1071, 1001, 1074, 1029, 1002, 1006, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8199, 8239, 8191, 8090, 8170, 8254, 8160] avg=8192.0000 max=8254 max_rank=6 min=8090 min_rank=4 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1041, 995, 1013, 1060, 968, 1089, 1027, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1000, 1033, 1011, 1035, 1026, 1007, 1018, 33, 11, 0, 0, 0, 0, 0, 0, 985, 976, 1029, 1067, 982, 990, 1000, 1028, 45, 0, 0, 0, 0, 0, 0, 0, 993, 996, 1037, 1034, 1033, 988, 1042, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1051, 1034, 957, 1011, 1004, 1067, 1053, 13, 8, 2, 0, 0, 0, 0, 0, 1011, 983, 1000, 1024, 1027, 1045, 991, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1022, 1064, 1045, 1026, 1027, 1062, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1029, 1004, 998, 1015, 1053, 1028, 1056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8237, 8102, 8168, 8194, 8124, 8295, 8192] avg=8192.0000 max=8295 max_rank=6 min=8102 min_rank=2 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32805 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1033, 1024, 1026, 1040, 1022, 1020, 1005, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1039, 1021, 995, 1017, 1000, 1014, 1014, 33, 12, 0, 0, 0, 0, 0, 0, 993, 1004, 1028, 1050, 955, 1022, 1049, 1057, 43, 0, 0, 0, 0, 0, 0, 0, 965, 1052, 1030, 998, 1064, 977, 1025, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1012, 1026, 1050, 1001, 1056, 995, 1075, 13, 8, 2, 0, 0, 0, 0, 0, 1019, 1015, 1073, 1055, 1016, 986, 1016, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1030, 1027, 1004, 1048, 995, 1013, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1020, 1081, 1045, 1062, 1015, 1010, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8130, 8201, 8127, 8265, 8218, 8088, 8273] avg=8192.0000 max=8273 max_rank=7 min=8088 min_rank=6 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1069, 1019, 1005, 1017, 1042, 1064, 977, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 987, 990, 1030, 1033, 1041, 1010, 1034, 33, 11, 0, 0, 0, 0, 0, 0, 1019, 1055, 1049, 995, 1030, 995, 1013, 1059, 48, 0, 0, 0, 0, 0, 0, 0, 1061, 1014, 1045, 956, 1011, 999, 1016, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1027, 1022, 988, 1052, 1007, 1008, 996, 13, 8, 2, 0, 0, 0, 0, 0, 1027, 1055, 1016, 972, 1012, 985, 1015, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1033, 979, 1026, 1065, 1035, 1028, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 991, 1072, 1048, 1053, 1035, 1051, 1000, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8180, 8263, 8164, 8182, 8064, 8236, 8264] avg=8192.0000 max=8264 max_rank=7 min=8064 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1002, 1056, 1018, 1030, 962, 1047, 984, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1032, 1038, 983, 1044, 1021, 1008, 1014, 33, 12, 0, 0, 0, 0, 0, 0, 1030, 1044, 973, 990, 1009, 1021, 1030, 1040, 44, 0, 0, 0, 0, 0, 0, 0, 983, 1009, 1066, 1018, 1038, 1025, 1027, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1031, 1046, 980, 1009, 996, 984, 1042, 13, 8, 2, 0, 0, 0, 0, 0, 1064, 998, 1043, 1009, 1036, 1066, 1026, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 996, 980, 1010, 1048, 1051, 1033, 1056, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1073, 1078, 986, 1026, 1014, 1009, 1001, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8213, 8181, 8201, 8177, 8273, 8181, 8195] avg=8192.0000 max=8273 max_rank=5 min=8115 min_rank=0 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1023, 1019, 1029, 1028, 1054, 1027, 1050, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 964, 1000, 1017, 1034, 992, 1027, 997, 34, 12, 0, 0, 0, 0, 0, 0, 996, 1016, 1016, 1025, 1007, 963, 1036, 1050, 44, 0, 0, 0, 0, 0, 0, 0, 984, 1008, 1043, 1035, 1030, 1057, 1046, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 984, 1028, 1045, 983, 998, 1028, 1022, 13, 8, 2, 0, 0, 0, 0, 0, 1064, 1030, 1044, 1017, 1028, 1055, 980, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1053, 957, 1029, 1043, 1040, 970, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 965, 1038, 1021, 1101, 1007, 1038, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8100, 8153, 8230, 8145, 8230, 8188, 8243] avg=8192.0000 max=8247 max_rank=0 min=8100 min_rank=1 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[991, 987, 996, 1057, 1009, 1032, 1035, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1035, 1004, 1004, 1032, 1011, 1053, 1022, 35, 14, 0, 0, 0, 0, 0, 0, 981, 1043, 996, 1012, 1019, 1022, 1029, 1002, 47, 1, 0, 0, 0, 0, 0, 0, 960, 1045, 1007, 1094, 997, 1071, 1027, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1041, 1082, 1016, 1085, 1034, 1018, 1038, 19, 12, 8, 0, 0, 0, 0, 0, 1024, 1020, 1057, 1028, 1020, 1004, 1050, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 997, 1060, 994, 999, 1049, 1003, 988, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1022, 977, 1065, 1033, 1002, 946, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8238, 8152, 8229, 8366, 8216, 8122, 8061] avg=8192.0000 max=8366 max_rank=4 min=8061 min_rank=7 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8448, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8320, 8064, 8064, 8320, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8064, 8192, 8192, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8320, 8448, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8448, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[990, 970, 1019, 995, 1061, 1036, 1012, 1003, 1013, 1017, 1036, 1033, 1063, 998, 997, 1001, 1068, 1028, 1048, 1027, 1032, 1038, 1010, 1079, 1054, 1020, 987, 1040, 1039, 1048, 1006, 1046, 1015, 1057, 1009, 1000, 1029, 1035, 1024, 991, 1027, 1025, 994, 1042, 1005, 999, 1024, 1064, 1010, 1039, 1015, 999, 1046, 1033, 988, 1021, 1030, 1010, 1071, 1021, 1005, 1021, 1023, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1018, 1045, 984, 997, 1077, 1047, 1002, 998, 1058, 954, 1004, 1052, 1028, 979, 1021, 1059, 1002, 1030, 1054, 1099, 1006, 1013, 1034, 1038, 1047, 1004, 1007, 990, 1008, 1031, 1022, 1029, 1088, 1085, 1013, 996, 988, 1030, 1026, 1040, 1041, 1030, 1020, 1034, 1053, 984, 1013, 994, 1031, 1017, 1071, 1048, 1012, 1060, 1029, 1035, 970, 1022, 1006, 998, 1037, 1021, 1021, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1050, 1047, 1053, 1021, 1046, 995, 1002, 1054, 1028, 1017, 1033, 1011, 1002, 1006, 998, 1041, 1019, 1055, 1019, 1078, 1041, 1002, 983, 1037, 1006, 1047, 1043, 1027, 1033, 1014, 1063, 1012, 1067, 1027, 992, 1010, 1054, 996, 1045, 1065, 1066, 1004, 1011, 1015, 1065, 994, 987, 992, 1000, 1001, 1025, 1018, 999, 1017, 1037, 1022, 1048, 1012, 961, 1006, 1038, 1042, 1017, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1016, 1083, 1024, 1018, 1023, 1031, 1058, 1056, 946, 986, 1003, 981, 1038, 1002, 993, 1003, 1036, 1045, 1011, 985, 1017, 1035, 1083, 1015, 1042, 1000, 1043, 1017, 1025, 1067, 994, 1046, 1055, 977, 982, 1005, 983, 1049, 1022, 1007, 1044, 1028, 977, 1022, 1010, 970, 1046, 1038, 1036, 1024, 1026, 1087, 997, 1032, 1044, 1016, 1039, 1052, 1030, 1043, 1074, 1038, 1029, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1043, 1030, 1037, 1001, 1017, 1043, 986, 1016, 1028, 1061, 978, 1052, 997, 1048, 1016, 1027, 993, 1078, 1051, 1001, 1051, 1069, 1014, 1044, 1030, 1067, 1057, 1032, 1008, 1000, 1013, 1028, 1015, 1027, 1026, 1000, 1015, 1041, 1000, 1017, 1069, 1017, 1024, 989, 1001, 1012, 1038, 1016, 1034, 1057, 1052, 982, 970, 995, 1007, 989, 1027, 978, 1059, 1046, 1014, 1040, 1028, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[994, 988, 1054, 1024, 1017, 1026, 1011, 1039, 1049, 986, 1034, 1012, 1037, 1021, 1031, 1017, 1042, 961, 952, 1024, 1003, 976, 1020, 1056, 1041, 1006, 995, 1009, 1011, 1032, 1035, 947, 1035, 1054, 1088, 1033, 1000, 1017, 966, 1034, 981, 1050, 1057, 1024, 1071, 1019, 1031, 1043, 1034, 1031, 1030, 1021, 1016, 1018, 1035, 996, 1074, 1015, 1014, 1073, 1030, 1080, 1071, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1028, 983, 1037, 1008, 1030, 970, 1034, 1080, 1068, 990, 1051, 1080, 1005, 1006, 980, 988, 1055, 1016, 1073, 1011, 1052, 1040, 1065, 1059, 1016, 1052, 1027, 991, 1040, 1001, 986, 1077, 1022, 1045, 1003, 1058, 1004, 1031, 1010, 1046, 964, 1012, 1060, 1003, 1029, 1062, 997, 1050, 974, 983, 1064, 1010, 990, 981, 1104, 1004, 1036, 1017, 1029, 1000, 991, 1036, 1016, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1041, 1042, 1073, 1031, 1032, 1021, 1028, 985, 1036, 1070, 1045, 1028, 1009, 1053, 1083, 1009, 1072, 1034, 1008, 1061, 980, 1045, 1028, 1013, 993, 1003, 1005, 943, 990, 1013, 1028, 1034, 1035, 1073, 985, 996, 1034, 967, 1036, 1052, 1018, 1005, 1029, 1003, 1036, 1034, 978, 1003, 1058, 1039, 979, 991, 1048, 999, 1081, 1072, 991, 1051, 1000, 1067, 1016, 982, 1035, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1018, 1045, 984, 997, 1072, 1047, 1002, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 954, 1004, 1052, 1028, 979, 1021, 1059, 29, 0, 0, 0, 0, 0, 0, 0, 1002, 1030, 1054, 1099, 1006, 1013, 1034, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1004, 1007, 990, 1008, 1031, 1022, 1029, 15, 5, 0, 0, 0, 0, 0, 0, 1088, 1085, 1013, 996, 988, 1030, 1026, 1040, 1, 0, 0, 0, 0, 0, 0, 0, 1041, 1030, 1020, 1034, 1053, 984, 1013, 994, 24, 0, 0, 0, 0, 0, 0, 0, 1031, 1017, 1071, 1048, 1012, 1060, 1029, 1035, 4, 2, 0, 0, 0, 0, 0, 0, 970, 1022, 1006, 998, 1037, 1004, 1021, 986, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8184, 8218, 8158, 8267, 8193, 8309, 8044] avg=8192.0000 max=8309 max_rank=6 min=8044 min_rank=7 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1043, 1030, 1037, 1001, 1012, 1043, 986, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1061, 978, 1052, 997, 1048, 1016, 1027, 30, 0, 0, 0, 0, 0, 0, 0, 993, 1078, 1051, 1001, 1051, 1069, 1014, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1067, 1057, 1032, 1008, 1000, 1013, 1028, 15, 5, 0, 0, 0, 0, 0, 0, 1015, 1027, 1026, 1000, 1015, 1041, 1000, 1017, 1, 0, 0, 0, 0, 0, 0, 0, 1069, 1017, 1024, 989, 1001, 1012, 1038, 1016, 24, 0, 0, 0, 0, 0, 0, 0, 1034, 1057, 1052, 982, 970, 995, 1007, 989, 4, 2, 0, 0, 0, 0, 0, 0, 1027, 978, 1059, 1046, 1014, 1023, 1028, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8237, 8242, 8255, 8142, 8190, 8092, 8210] avg=8192.0000 max=8255 max_rank=3 min=8092 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1050, 1047, 1053, 1021, 1041, 995, 1002, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1017, 1033, 1011, 1002, 1006, 998, 1041, 29, 0, 0, 0, 0, 0, 0, 0, 1019, 1055, 1019, 1078, 1041, 1002, 983, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1047, 1043, 1027, 1033, 1014, 1063, 1012, 15, 5, 0, 0, 0, 0, 0, 0, 1067, 1027, 992, 1010, 1054, 996, 1045, 1065, 1, 0, 0, 0, 0, 0, 0, 0, 1066, 1004, 1011, 1015, 1065, 994, 987, 992, 23, 0, 0, 0, 0, 0, 0, 0, 1000, 1001, 1025, 1018, 999, 1017, 1037, 1022, 4, 2, 0, 0, 0, 0, 0, 0, 1048, 1012, 961, 1006, 1038, 1025, 1017, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8165, 8177, 8265, 8257, 8157, 8125, 8127] avg=8192.0000 max=8265 max_rank=3 min=8125 min_rank=6 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[990, 970, 1019, 995, 1056, 1036, 1012, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1017, 1036, 1033, 1063, 998, 997, 1001, 31, 0, 0, 0, 0, 0, 0, 0, 1068, 1028, 1048, 1027, 1032, 1038, 1010, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1020, 987, 1040, 1039, 1048, 1006, 1046, 15, 5, 0, 0, 0, 0, 0, 0, 1015, 1057, 1009, 1000, 1029, 1035, 1024, 991, 1, 0, 0, 0, 0, 0, 0, 0, 1027, 1025, 994, 1042, 1005, 999, 1024, 1064, 24, 0, 0, 0, 0, 0, 0, 0, 1010, 1039, 1015, 999, 1046, 1033, 988, 1021, 4, 2, 0, 0, 0, 0, 0, 0, 1030, 1010, 1071, 1021, 1005, 1004, 1023, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8081, 8189, 8270, 8260, 8161, 8204, 8157, 8214] avg=8192.0000 max=8270 max_rank=2 min=8081 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1028, 983, 1037, 1008, 1025, 970, 1034, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 990, 1051, 1080, 1005, 1006, 980, 988, 30, 0, 0, 0, 0, 0, 0, 0, 1055, 1016, 1073, 1011, 1052, 1040, 1065, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1052, 1027, 991, 1040, 1001, 986, 1077, 15, 5, 0, 0, 0, 0, 0, 0, 1022, 1045, 1003, 1058, 1004, 1031, 1010, 1046, 1, 0, 0, 0, 0, 0, 0, 0, 964, 1012, 1060, 1003, 1029, 1062, 997, 1050, 24, 0, 0, 0, 0, 0, 0, 0, 974, 983, 1064, 1010, 990, 981, 1104, 1004, 4, 2, 0, 0, 0, 0, 0, 0, 1036, 1017, 1029, 1000, 991, 1019, 1016, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8198, 8312, 8210, 8220, 8201, 8116, 8114] avg=8192.0000 max=8312 max_rank=2 min=8114 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1016, 1083, 1024, 1018, 1018, 1031, 1058, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 946, 986, 1003, 981, 1038, 1002, 993, 1003, 29, 0, 0, 0, 0, 0, 0, 0, 1036, 1045, 1011, 985, 1017, 1035, 1083, 958, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1000, 1043, 1017, 1025, 1067, 994, 1046, 15, 5, 0, 0, 0, 0, 0, 0, 1055, 977, 982, 1005, 983, 1049, 1022, 1007, 1, 0, 0, 0, 0, 0, 0, 0, 1044, 1028, 977, 1022, 1010, 970, 1046, 1038, 23, 0, 0, 0, 0, 0, 0, 0, 1036, 1024, 1026, 1087, 997, 1032, 1044, 1016, 4, 2, 0, 0, 0, 0, 0, 0, 1039, 1052, 1030, 1043, 1074, 1021, 1029, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8304, 7981, 8170, 8254, 8081, 8158, 8268, 8320] avg=8192.0000 max=8320 max_rank=7 min=7981 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[994, 988, 1054, 1024, 1012, 1026, 1011, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 986, 1034, 1012, 1037, 1021, 1031, 1017, 30, 0, 0, 0, 0, 0, 0, 0, 1042, 961, 952, 1024, 1003, 976, 1020, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1006, 995, 1009, 1011, 1032, 1035, 947, 16, 5, 0, 0, 0, 0, 0, 0, 1035, 1054, 1088, 1033, 1000, 1017, 966, 1034, 1, 0, 0, 0, 0, 0, 0, 0, 981, 1050, 1057, 1024, 1071, 1019, 1031, 1043, 24, 0, 0, 0, 0, 0, 0, 0, 1034, 1031, 1030, 1021, 1016, 1018, 1035, 996, 4, 2, 0, 0, 0, 0, 0, 0, 1074, 1015, 1014, 1073, 1030, 1062, 1071, 1045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8217, 7975, 8097, 8228, 8300, 8187, 8384] avg=8192.0000 max=8384 max_rank=7 min=7975 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=33099 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1041, 1042, 1073, 1031, 1023, 1021, 1028, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1070, 1045, 1028, 1009, 1053, 1083, 1009, 32, 0, 0, 0, 0, 0, 0, 0, 1072, 1034, 1008, 1061, 980, 1045, 1028, 944, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1003, 1005, 943, 990, 1013, 1028, 1034, 18, 10, 0, 0, 0, 0, 0, 0, 1035, 1073, 985, 996, 1034, 967, 1036, 1052, 2, 0, 0, 0, 0, 0, 0, 0, 1018, 1005, 1029, 1003, 1036, 1034, 978, 1003, 27, 0, 0, 0, 0, 0, 0, 0, 1058, 1039, 979, 991, 1048, 999, 1081, 1072, 7, 8, 0, 0, 0, 0, 0, 0, 991, 1051, 1000, 1067, 1016, 956, 1035, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8365, 8172, 8037, 8180, 8133, 8282, 8123] avg=8192.0000 max=8365 max_rank=1 min=8037 min_rank=3 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8320, 8320, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8448, 8448, 8192, 8192, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8064, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8448, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1055, 1033, 997, 1053, 1011, 1009, 1077, 1002, 1053, 1073, 978, 1015, 1006, 975, 1071, 965, 1012, 1040, 1006, 1088, 1046, 1063, 993, 1021, 988, 1017, 1054, 1061, 1039, 981, 991, 1040, 996, 1048, 965, 998, 999, 1045, 989, 991, 1037, 1038, 1019, 1014, 1078, 997, 1061, 1069, 1006, 1015, 1027, 1013, 1049, 1058, 1018, 1032, 1052, 1053, 1007, 1007, 1013, 1011, 998, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1037, 1049, 996, 982, 989, 974, 1079, 1052, 1024, 1005, 1025, 1063, 1024, 984, 1044, 1006, 998, 975, 1040, 1064, 979, 1036, 1046, 1028, 1031, 1046, 967, 964, 1038, 1036, 991, 997, 1042, 1075, 1042, 1017, 1002, 1041, 1061, 998, 1027, 1041, 968, 981, 1017, 1014, 1033, 1021, 1045, 1040, 1023, 990, 1024, 1041, 1030, 1049, 1045, 1076, 1051, 1052, 1091, 998, 1013, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1083, 1036, 1060, 987, 966, 1037, 1042, 1048, 998, 1002, 1003, 1004, 974, 1013, 1003, 1051, 1037, 1038, 984, 1058, 1004, 1074, 1046, 1052, 1029, 1036, 1026, 1026, 1034, 991, 1019, 1024, 1064, 1067, 1035, 963, 1009, 1033, 1064, 1032, 990, 1072, 1011, 1022, 1042, 1009, 1048, 979, 1023, 1024, 1007, 996, 1011, 1014, 1033, 1045, 992, 1037, 1014, 1015, 1048, 1049, 943, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1002, 1033, 1056, 988, 1049, 999, 997, 987, 1058, 1030, 1020, 1027, 969, 1016, 1009, 1033, 965, 1050, 1022, 983, 1035, 998, 1054, 990, 1025, 1006, 1035, 1031, 925, 1043, 1026, 1058, 1067, 1045, 1025, 1019, 962, 1008, 1041, 1047, 1046, 1059, 961, 1000, 1014, 1071, 1097, 987, 1066, 1048, 1011, 995, 993, 1056, 1042, 1022, 1049, 1007, 1065, 1039, 1065, 1026, 1035, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1017, 1024, 1000, 1017, 1071, 1012, 1010, 1081, 1020, 1072, 1032, 1012, 1026, 1004, 1058, 1069, 1033, 1037, 1065, 933, 1082, 993, 975, 1043, 1092, 1023, 965, 1020, 1006, 1004, 1083, 1001, 1037, 1037, 1020, 1020, 1006, 1015, 1016, 987, 952, 1001, 1019, 988, 1046, 1019, 1022, 1012, 1024, 981, 1027, 1035, 1025, 1060, 1054, 996, 1018, 1038, 1027, 1069, 1021, 1037, 1023, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1001, 1064, 1058, 1028, 953, 994, 1068, 1024, 1032, 1019, 971, 1047, 1006, 1010, 1027, 1027, 1038, 1021, 1023, 983, 967, 1013, 1075, 1042, 1010, 1023, 1005, 1038, 1021, 1039, 1080, 1050, 1027, 1000, 1056, 1041, 1011, 1025, 976, 1042, 1014, 1088, 1038, 1013, 1087, 1016, 999, 994, 1028, 1018, 992, 995, 1039, 1018, 1068, 1024, 991, 1081, 1029, 981, 1009, 982, 1063, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1034, 1062, 1089, 1030, 1001, 1015, 988, 1033, 1010, 1017, 970, 999, 983, 979, 1031, 1040, 1039, 1006, 1004, 1032, 1013, 1029, 1013, 989, 1044, 1057, 1073, 1016, 1017, 1051, 984, 1027, 1069, 1043, 1037, 1023, 1022, 991, 1012, 1033, 1050, 1019, 1001, 1036, 1046, 986, 1056, 1058, 979, 1044, 1005, 1058, 991, 1009, 1031, 1042, 1008, 1054, 1034, 998, 987, 1080, 1024, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1019, 1086, 1043, 1012, 1002, 982, 1065, 1012, 1042, 1013, 981, 1006, 1029, 1024, 1002, 1020, 998, 1024, 1025, 1063, 1007, 1033, 1063, 1006, 1035, 996, 1031, 1019, 1053, 1003, 1052, 1048, 1027, 1016, 968, 1008, 1040, 1041, 1075, 1050, 1020, 1016, 1089, 1016, 1062, 1008, 994, 1001, 1063, 1050, 1005, 1024, 1026, 1044, 1021, 1007, 999, 965, 1039, 997, 1055, 1014, 1006, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1002, 1018, 1056, 988, 1049, 999, 997, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1030, 1020, 1027, 969, 1016, 1009, 1033, 21, 15, 11, 4, 0, 0, 0, 0, 965, 1050, 1022, 983, 1035, 998, 1054, 990, 3, 1, 0, 0, 0, 0, 0, 0, 1025, 1006, 1035, 1031, 925, 1043, 1026, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1042, 1025, 1019, 962, 1008, 1041, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1059, 961, 1000, 1009, 1071, 1097, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1048, 1011, 995, 993, 1045, 1042, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 986, 1065, 1039, 1065, 1026, 1035, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8213, 8101, 8149, 8211, 8230, 8222, 8314] avg=8192.0000 max=8314 max_rank=7 min=8096 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32977 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1083, 1021, 1060, 987, 966, 1037, 1042, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1002, 1003, 1004, 974, 1013, 1003, 1051, 22, 15, 11, 4, 0, 0, 0, 0, 1037, 1038, 984, 1058, 1004, 1074, 1046, 1052, 3, 1, 0, 0, 0, 0, 0, 0, 1029, 1036, 1026, 1026, 1034, 991, 1019, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1064, 1035, 963, 1009, 1033, 1064, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1072, 1011, 1022, 1037, 1009, 1048, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1024, 1007, 996, 1011, 1003, 1033, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1015, 1014, 1015, 1048, 1049, 943, 1060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8100, 8297, 8185, 8264, 8168, 8142, 8136] avg=8192.0000 max=8297 max_rank=2 min=8100 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1001, 1049, 1058, 1028, 953, 994, 1068, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1019, 971, 1047, 1006, 1010, 1027, 1027, 23, 15, 11, 5, 0, 0, 0, 0, 1038, 1021, 1023, 983, 967, 1013, 1075, 1042, 3, 1, 0, 0, 0, 0, 0, 0, 1010, 1023, 1005, 1038, 1021, 1039, 1080, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 997, 1056, 1041, 1011, 1025, 976, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1088, 1038, 1013, 1081, 1016, 999, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1018, 992, 995, 1039, 1007, 1068, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1058, 1029, 981, 1009, 982, 1063, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8193, 8166, 8266, 8175, 8243, 8171, 8147] avg=8192.0000 max=8266 max_rank=3 min=8147 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1055, 1018, 997, 1053, 1011, 1009, 1077, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1073, 978, 1015, 1006, 975, 1071, 965, 22, 15, 11, 5, 0, 0, 0, 0, 1012, 1040, 1006, 1088, 1046, 1063, 993, 1021, 3, 1, 0, 0, 0, 0, 0, 0, 988, 1017, 1054, 1061, 1039, 981, 991, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1045, 965, 998, 999, 1045, 989, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1038, 1019, 1014, 1072, 997, 1061, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1015, 1027, 1013, 1049, 1047, 1018, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1031, 1007, 1007, 1013, 1011, 998, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8189, 8273, 8171, 8028, 8307, 8207, 8139] avg=8192.0000 max=8307 max_rank=5 min=8028 min_rank=4 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1017, 1009, 1000, 1017, 1071, 1012, 1010, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1072, 1032, 1012, 1026, 1004, 1058, 1069, 22, 15, 11, 4, 0, 0, 0, 0, 1033, 1037, 1065, 933, 1082, 993, 975, 1043, 3, 1, 0, 0, 0, 0, 0, 0, 1092, 1023, 965, 1020, 1006, 1004, 1083, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1034, 1020, 1020, 1006, 1015, 1016, 987, 0, 0, 0, 0, 0, 0, 0, 0, 952, 1001, 1019, 988, 1041, 1019, 1022, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 981, 1027, 1035, 1025, 1049, 1054, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1016, 1027, 1069, 1021, 1037, 1023, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8345, 8165, 8194, 8135, 8054, 8191, 8235] avg=8192.0000 max=8345 max_rank=1 min=8054 min_rank=5 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1037, 1034, 996, 982, 989, 974, 1079, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1005, 1025, 1063, 1024, 984, 1044, 1006, 23, 15, 11, 4, 0, 0, 0, 0, 998, 975, 1040, 1064, 979, 1036, 1046, 1028, 3, 1, 0, 0, 0, 0, 0, 0, 1031, 1046, 967, 964, 1038, 1036, 991, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1072, 1042, 1017, 1002, 1041, 1061, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1041, 968, 981, 1012, 1014, 1033, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1040, 1023, 990, 1024, 1030, 1030, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1053, 1051, 1052, 1091, 998, 1013, 1019, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8228, 8170, 8070, 8275, 8097, 8231, 8322] avg=8192.0000 max=8322 max_rank=7 min=8070 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32925 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1034, 1047, 1089, 1030, 1001, 1015, 988, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1017, 970, 999, 983, 979, 1031, 1040, 22, 15, 11, 4, 0, 0, 0, 0, 1039, 1006, 1004, 1032, 1013, 1029, 1013, 989, 3, 1, 0, 0, 0, 0, 0, 0, 1044, 1057, 1073, 1016, 1017, 1051, 984, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1040, 1037, 1023, 1022, 991, 1012, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1019, 1001, 1036, 1041, 986, 1056, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1044, 1005, 1058, 991, 998, 1031, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1032, 1034, 998, 987, 1080, 1024, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8081, 8129, 8269, 8227, 8247, 8148, 8198] avg=8192.0000 max=8269 max_rank=3 min=8081 min_rank=1 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1019, 1067, 1043, 1012, 1002, 982, 1065, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1013, 981, 1006, 1029, 1024, 1002, 1020, 26, 19, 16, 9, 0, 0, 0, 0, 998, 1024, 1025, 1063, 1007, 1033, 1063, 1006, 4, 7, 5, 0, 0, 0, 0, 0, 1030, 996, 1031, 1019, 1053, 1003, 1052, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1012, 968, 1008, 1040, 1041, 1075, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1016, 1089, 1016, 1046, 1008, 994, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1050, 1005, 1024, 1026, 1028, 1021, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 999, 939, 1039, 997, 1055, 1014, 1006, 996, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8187, 8235, 8232, 8221, 8190, 8224, 8045] avg=8192.0000 max=8235 max_rank=2 min=8045 min_rank=7 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8448, 8320, 8192, 8192, 8064, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8064, 8064, 8448, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8064, 8192, 8064, 8064, 8320, 8320, 256, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8192, 8192, 8448, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1052, 990, 990, 1067, 1076, 1024, 1050, 1038, 1054, 1046, 985, 1056, 989, 986, 1025, 991, 968, 1044, 1029, 1033, 1054, 1053, 1060, 1040, 1036, 999, 1078, 1061, 971, 1011, 1035, 1035, 1007, 1037, 984, 1057, 989, 1033, 967, 1065, 1028, 1015, 976, 1005, 1027, 974, 981, 1001, 1035, 1035, 1017, 1009, 1059, 1101, 1073, 995, 1001, 960, 988, 1064, 1041, 1001, 1017, 1068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1062, 1036, 1032, 1094, 1028, 1060, 1040, 999, 1013, 1022, 1053, 1000, 1043, 1011, 1058, 1035, 959, 1018, 1053, 1024, 1009, 1019, 1023, 1027, 1102, 987, 1061, 1000, 1036, 986, 973, 1019, 1013, 1053, 978, 989, 995, 1049, 1074, 994, 981, 1021, 1005, 999, 976, 1037, 1001, 990, 1002, 999, 1028, 1024, 1036, 1024, 1014, 1011, 1111, 994, 1014, 1071, 1093, 1032, 1031, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1017, 1038, 997, 972, 1038, 1034, 1099, 1014, 1010, 1009, 1030, 1069, 988, 1100, 999, 1016, 993, 1026, 994, 1026, 1047, 1039, 1025, 1005, 1033, 1037, 1009, 1002, 979, 1045, 1006, 1051, 1035, 961, 1031, 1035, 996, 1030, 1055, 1029, 1011, 1025, 1040, 1026, 1013, 1017, 1019, 1045, 1037, 1019, 1022, 998, 1008, 1001, 1067, 1003, 1033, 1047, 994, 1031, 1037, 1022, 1036, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[973, 1067, 1035, 1003, 1047, 1008, 1028, 980, 1056, 993, 1013, 1004, 1025, 1036, 1030, 1036, 1067, 996, 974, 999, 1013, 1044, 1055, 1008, 995, 1021, 1068, 1019, 1018, 1078, 984, 999, 999, 1068, 1025, 1056, 992, 1055, 1036, 1045, 989, 1028, 1012, 1012, 1041, 1061, 1010, 1031, 1063, 1027, 960, 1025, 1060, 1040, 1001, 1036, 1044, 1039, 1057, 959, 1025, 1033, 986, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1031, 1050, 1074, 1002, 960, 1000, 989, 1008, 1037, 997, 1038, 1059, 1005, 1010, 1045, 1023, 1090, 1050, 1043, 1007, 1034, 1022, 1065, 1019, 986, 1051, 1016, 955, 1016, 1050, 999, 1068, 1025, 1031, 1002, 991, 1055, 996, 1008, 1026, 1008, 1019, 1014, 1032, 1011, 1000, 1056, 1071, 1028, 1026, 990, 1048, 1024, 1038, 1005, 997, 1054, 1039, 1011, 1021, 1066, 1033, 1024, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1003, 1012, 1033, 1057, 996, 996, 1019, 1024, 1060, 1024, 1085, 999, 1014, 1020, 1036, 1028, 1012, 1020, 1046, 998, 1032, 1046, 1047, 1060, 1002, 1060, 995, 994, 1053, 986, 1058, 1018, 1050, 998, 1024, 1015, 1005, 1034, 1055, 1058, 1029, 1008, 1026, 1070, 975, 1034, 1020, 1023, 1011, 1017, 1032, 974, 988, 1047, 1058, 1012, 1003, 1029, 989, 1050, 1071, 1010, 982, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1040, 1049, 1039, 1045, 1001, 1004, 1018, 1022, 1007, 1062, 1061, 993, 1004, 1075, 1055, 1019, 1026, 1016, 1019, 962, 1021, 1040, 1067, 1022, 986, 1013, 1067, 1004, 951, 1014, 1053, 1018, 1032, 1013, 1036, 1021, 959, 1014, 1026, 1007, 1068, 1033, 998, 997, 1040, 1037, 1005, 994, 1023, 1040, 1008, 1042, 1007, 1047, 1041, 1034, 1008, 1030, 988, 1031, 1090, 1050, 1055, 989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1046, 1033, 964, 1016, 1022, 1002, 1021, 981, 1003, 1030, 1005, 1050, 1034, 1063, 1041, 1056, 994, 1064, 1052, 1006, 993, 1004, 1008, 1000, 953, 1001, 1082, 1013, 1014, 1009, 1067, 1005, 1037, 1020, 1035, 1054, 979, 1014, 1006, 1072, 1038, 1031, 1028, 1041, 962, 1041, 1047, 1035, 1055, 1036, 1053, 1035, 1031, 1038, 1021, 1059, 1024, 1032, 1037, 1032, 999, 1013, 1009, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1017, 1037, 997, 972, 1038, 1034, 1099, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1009, 1030, 1069, 988, 1063, 999, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1026, 994, 1026, 1047, 1039, 1017, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1037, 1009, 1002, 979, 1045, 1006, 1051, 16, 13, 0, 0, 0, 0, 0, 0, 1035, 961, 1031, 1035, 996, 1030, 1055, 1029, 8, 2, 1, 0, 0, 0, 0, 0, 1011, 1025, 1040, 1026, 1013, 1017, 1019, 1045, 37, 8, 0, 0, 0, 0, 0, 0, 1037, 1019, 1022, 998, 1008, 983, 1067, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1047, 994, 1031, 1016, 1022, 1036, 1066, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8184, 8147, 8191, 8183, 8241, 8137, 8245] avg=8192.0000 max=8245 max_rank=7 min=8137 min_rank=6 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1062, 1035, 1032, 1094, 1028, 1060, 1040, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1022, 1053, 1000, 1043, 977, 1058, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 959, 1018, 1053, 1024, 1009, 1019, 1015, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 987, 1061, 1000, 1036, 986, 973, 1019, 16, 14, 0, 0, 0, 0, 0, 0, 1013, 1053, 978, 989, 995, 1049, 1074, 994, 8, 2, 1, 0, 0, 0, 0, 0, 981, 1021, 1005, 999, 976, 1037, 1001, 990, 34, 8, 0, 0, 0, 0, 0, 0, 1002, 999, 1028, 1024, 1036, 1006, 1014, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 994, 1014, 1071, 1071, 1032, 1031, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8350, 8201, 8124, 8194, 8156, 8052, 8120, 8339] avg=8192.0000 max=8350 max_rank=0 min=8052 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[973, 1066, 1035, 1003, 1047, 1008, 1028, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 993, 1013, 1004, 1025, 1001, 1030, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 996, 974, 999, 1013, 1044, 1046, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1021, 1068, 1019, 1018, 1078, 984, 999, 16, 13, 0, 0, 0, 0, 0, 0, 999, 1068, 1025, 1056, 992, 1055, 1036, 1045, 9, 2, 1, 0, 0, 0, 0, 0, 989, 1028, 1012, 1012, 1041, 1061, 1010, 1031, 35, 7, 0, 0, 0, 0, 0, 0, 1063, 1027, 960, 1025, 1060, 1022, 1001, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1039, 1057, 959, 1005, 1033, 986, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8158, 8147, 8211, 8288, 8226, 8194, 8172] avg=8192.0000 max=8288 max_rank=4 min=8140 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1052, 989, 990, 1067, 1076, 1024, 1050, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1046, 985, 1056, 989, 953, 1025, 991, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1044, 1029, 1033, 1054, 1053, 1051, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 999, 1078, 1061, 971, 1011, 1035, 1035, 17, 13, 0, 0, 0, 0, 0, 0, 1007, 1037, 984, 1057, 989, 1033, 967, 1065, 9, 3, 1, 0, 0, 0, 0, 0, 1028, 1015, 976, 1005, 1027, 974, 981, 1001, 33, 8, 0, 0, 0, 0, 0, 0, 1035, 1035, 1017, 1009, 1059, 1081, 1073, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 960, 988, 1064, 1020, 1001, 1017, 1068, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8099, 8272, 8256, 8152, 8048, 8304, 8119] avg=8192.0000 max=8304 max_rank=6 min=8048 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1031, 1049, 1074, 1002, 960, 1000, 989, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 997, 1038, 1059, 1005, 976, 1045, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 1050, 1043, 1007, 1034, 1022, 1056, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1051, 1016, 955, 1016, 1050, 999, 1068, 16, 14, 0, 0, 0, 0, 0, 0, 1025, 1031, 1002, 991, 1055, 996, 1008, 1026, 9, 2, 1, 0, 0, 0, 0, 0, 1008, 1019, 1014, 1032, 1011, 1000, 1056, 1071, 34, 8, 0, 0, 0, 0, 0, 0, 1028, 1026, 990, 1048, 1024, 1020, 1005, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1039, 1011, 1021, 1044, 1033, 1024, 988, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8180, 8321, 8171, 8146, 8253, 8138, 8214] avg=8192.0000 max=8321 max_rank=2 min=8113 min_rank=0 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1003, 1011, 1033, 1057, 996, 996, 1019, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1024, 1085, 999, 1014, 986, 1036, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1020, 1046, 998, 1032, 1046, 1038, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1060, 995, 994, 1053, 986, 1058, 1018, 16, 14, 0, 0, 0, 0, 0, 0, 1050, 998, 1024, 1015, 1005, 1034, 1055, 1058, 9, 3, 1, 0, 0, 0, 0, 0, 1029, 1008, 1026, 1070, 975, 1034, 1020, 1023, 34, 8, 0, 0, 0, 0, 0, 0, 1011, 1017, 1032, 974, 988, 1028, 1058, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1029, 989, 1050, 1049, 1010, 982, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8232, 8252, 8196, 8252, 8227, 8120, 8118] avg=8192.0000 max=8252 max_rank=2 min=8118 min_rank=7 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1040, 1048, 1039, 1045, 1001, 1004, 1018, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1062, 1061, 993, 1004, 1039, 1055, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1016, 1019, 962, 1021, 1040, 1058, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1013, 1067, 1004, 951, 1014, 1053, 1018, 16, 14, 0, 0, 0, 0, 0, 0, 1032, 1013, 1036, 1021, 959, 1014, 1026, 1007, 9, 3, 1, 0, 0, 0, 0, 0, 1068, 1033, 998, 997, 1040, 1037, 1005, 994, 36, 8, 0, 0, 0, 0, 0, 0, 1023, 1040, 1008, 1042, 1007, 1028, 1041, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1030, 988, 1031, 1068, 1050, 1055, 989, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8240, 8164, 8136, 8121, 8216, 8223, 8219] avg=8192.0000 max=8240 max_rank=1 min=8121 min_rank=4 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1046, 1031, 964, 1016, 1022, 1002, 1021, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1030, 1005, 1050, 1034, 1023, 1041, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1064, 1052, 1006, 993, 1004, 996, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 953, 1001, 1082, 1013, 1014, 1009, 1067, 1005, 21, 16, 0, 0, 0, 0, 0, 0, 1037, 1020, 1035, 1054, 979, 1014, 1006, 1072, 12, 7, 2, 0, 0, 0, 0, 0, 1038, 1031, 1028, 1041, 962, 1041, 1047, 1035, 40, 10, 0, 0, 0, 0, 0, 0, 1055, 1036, 1053, 1035, 1031, 1010, 1021, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1032, 1037, 1032, 973, 1013, 1009, 990, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8242, 8109, 8181, 8238, 8273, 8300, 8110] avg=8192.0000 max=8300 max_rank=6 min=8083 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8064, 8320, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8448, 8064, 8064, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8192, 8320, 8064, 8320, 8320, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1023, 1004, 1000, 1011, 991, 995, 1020, 1024, 1041, 1030, 1013, 1010, 996, 1028, 1019, 992, 1043, 1016, 1010, 979, 1020, 963, 1039, 1009, 1037, 1025, 1036, 999, 1048, 1051, 1054, 1020, 972, 1020, 1042, 1020, 1007, 1062, 1047, 1115, 1036, 1008, 1041, 1040, 1062, 1044, 1002, 1032, 1082, 1040, 969, 988, 1014, 1025, 1035, 1041, 988, 1048, 1056, 1073, 971, 1053, 1011, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1050, 1023, 1039, 1041, 1011, 1026, 1037, 1011, 1027, 1025, 1021, 1013, 1045, 1035, 1015, 1047, 1045, 1021, 1020, 1075, 1021, 1010, 1059, 1041, 1011, 991, 1062, 983, 1031, 1059, 1041, 990, 1012, 1063, 1020, 1040, 994, 1048, 985, 1045, 966, 1036, 1051, 1010, 998, 1061, 1071, 988, 1037, 1010, 1035, 1022, 975, 1020, 1007, 1047, 990, 1029, 1001, 1043, 1001, 996, 1005, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1020, 1025, 1053, 985, 964, 1006, 999, 1053, 1013, 1041, 1002, 1020, 1038, 996, 1039, 1043, 1028, 1049, 972, 1012, 978, 1058, 1016, 976, 1019, 1044, 1031, 1004, 1028, 1019, 1048, 975, 1043, 1012, 1050, 1068, 1052, 1007, 993, 1034, 998, 985, 1003, 1015, 1041, 1053, 1051, 1093, 1036, 1015, 1011, 1025, 986, 1071, 1064, 1016, 1010, 1043, 1021, 1009, 1079, 990, 1048, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1026, 996, 1053, 1046, 1014, 992, 1091, 1036, 1011, 1030, 1003, 1081, 1020, 1076, 1068, 989, 991, 1031, 1018, 1039, 1023, 1047, 982, 1051, 977, 1008, 1013, 1017, 990, 998, 986, 1039, 1028, 1083, 1031, 958, 1047, 1041, 992, 1036, 972, 1015, 1099, 1025, 1059, 1028, 1005, 1043, 1015, 1079, 975, 1019, 975, 993, 1043, 1021, 1021, 1048, 1048, 1013, 1046, 996, 1017, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[993, 1010, 1027, 1046, 1021, 986, 1032, 1028, 1064, 1028, 1006, 1039, 998, 1005, 1021, 1059, 1004, 1028, 1041, 1044, 992, 1017, 1012, 988, 990, 1046, 1034, 985, 1013, 992, 985, 1032, 977, 1023, 974, 1057, 1010, 1061, 1079, 1072, 1003, 1043, 1046, 1033, 1026, 1027, 1021, 1039, 1004, 1049, 1063, 1038, 1020, 1036, 1037, 1063, 1001, 1001, 1036, 1038, 981, 1016, 1042, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[986, 1064, 994, 1001, 1025, 1073, 1071, 997, 1007, 972, 1019, 1018, 1017, 1048, 1030, 1030, 1006, 986, 984, 1014, 1065, 995, 1019, 1019, 1039, 996, 1032, 1010, 1045, 974, 998, 1080, 994, 1046, 1057, 1034, 996, 1026, 1014, 1056, 1047, 1002, 1093, 1011, 1007, 1036, 1067, 1011, 1016, 1014, 952, 1016, 1044, 1007, 1025, 1030, 1070, 1052, 989, 1060, 1070, 1046, 1013, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1028, 1031, 1014, 1038, 1041, 1081, 1037, 1012, 1040, 1052, 1031, 1048, 1050, 994, 1044, 1031, 1023, 1041, 991, 1022, 1052, 972, 988, 1043, 1033, 1002, 1050, 979, 982, 1027, 1027, 1014, 970, 1033, 1045, 1006, 1094, 946, 1020, 1002, 1013, 1031, 1008, 1017, 984, 1036, 1028, 1024, 1001, 1009, 1061, 999, 1008, 1064, 1034, 1042, 1035, 1018, 1055, 1026, 1069, 995, 1057, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[994, 1020, 995, 986, 1028, 1034, 1032, 1001, 1034, 1025, 1007, 1001, 1035, 1013, 1033, 1010, 1007, 1039, 983, 1107, 1062, 1058, 1043, 1052, 1024, 1033, 1016, 982, 1052, 998, 1030, 1014, 985, 1013, 1024, 1032, 1017, 1051, 1024, 1057, 1033, 978, 980, 1023, 1031, 1065, 1012, 1055, 1018, 1080, 1020, 1016, 995, 1024, 1003, 1009, 1032, 1043, 1027, 1040, 1027, 1025, 1043, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1023, 1004, 1000, 1011, 991, 995, 1020, 1024, 11, 6, 0, 0, 0, 0, 0, 0, 1041, 1030, 1013, 1010, 996, 1028, 1008, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1016, 1010, 979, 1020, 963, 1039, 1009, 16, 8, 0, 0, 0, 0, 0, 0, 1037, 1025, 1036, 999, 1048, 1051, 1054, 1020, 29, 21, 0, 0, 0, 0, 0, 0, 972, 1020, 1042, 1020, 1007, 1062, 1047, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1008, 1041, 1040, 1062, 1017, 1002, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 1040, 969, 988, 1014, 1025, 1035, 1041, 5, 0, 0, 0, 0, 0, 0, 0, 988, 1048, 1056, 1044, 971, 1053, 1011, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8085, 8118, 8103, 8320, 8256, 8238, 8199, 8217] avg=8192.0000 max=8320 max_rank=3 min=8085 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32910 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1026, 996, 1053, 1046, 1014, 992, 1091, 1036, 11, 6, 0, 0, 0, 0, 0, 0, 1011, 1030, 1003, 1081, 1020, 1076, 1056, 989, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1031, 1018, 1039, 1023, 1047, 982, 1051, 16, 8, 0, 0, 0, 0, 0, 0, 977, 1008, 1013, 1017, 990, 998, 986, 1039, 27, 19, 0, 0, 0, 0, 0, 0, 1028, 1083, 1031, 958, 1047, 1041, 992, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1015, 1099, 1025, 1059, 1001, 1005, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1079, 975, 1019, 975, 993, 1043, 1021, 6, 0, 0, 0, 0, 0, 0, 0, 1021, 1048, 1048, 986, 1046, 996, 1017, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8266, 8206, 8074, 8189, 8219, 8126, 8185] avg=8192.0000 max=8271 max_rank=0 min=8074 min_rank=3 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1050, 1023, 1039, 1041, 1011, 1026, 1037, 1011, 11, 6, 0, 0, 0, 0, 0, 0, 1027, 1025, 1021, 1013, 1045, 1035, 1004, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1021, 1020, 1075, 1021, 1010, 1059, 1041, 16, 8, 0, 0, 0, 0, 0, 0, 1011, 991, 1062, 983, 1031, 1059, 1041, 990, 28, 19, 0, 0, 0, 0, 0, 0, 1012, 1063, 1020, 1040, 994, 1048, 985, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1036, 1051, 1010, 998, 1034, 1071, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1010, 1035, 1022, 975, 1020, 1007, 1047, 5, 0, 0, 0, 0, 0, 0, 0, 990, 1029, 1001, 1015, 1001, 996, 1005, 1004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8217, 8316, 8215, 8180, 8154, 8158, 8041] avg=8192.0000 max=8316 max_rank=2 min=8041 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[986, 1064, 994, 1001, 1025, 1073, 1071, 997, 11, 6, 0, 0, 0, 0, 0, 0, 1007, 972, 1019, 1018, 1017, 1048, 1019, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 986, 984, 1014, 1065, 995, 1019, 1019, 16, 8, 0, 0, 0, 0, 0, 0, 1039, 996, 1032, 1010, 1045, 974, 998, 1080, 28, 20, 0, 0, 0, 0, 0, 0, 994, 1046, 1057, 1034, 996, 1026, 1014, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1002, 1093, 1011, 1007, 1009, 1067, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1014, 952, 1016, 1044, 1007, 1025, 1030, 5, 0, 0, 0, 0, 0, 0, 0, 1070, 1052, 989, 1032, 1070, 1046, 1013, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8130, 8112, 8222, 8195, 8247, 8109, 8293] avg=8192.0000 max=8293 max_rank=7 min=8109 min_rank=6 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32844 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1020, 1025, 1053, 985, 964, 1006, 999, 1053, 11, 6, 0, 0, 0, 0, 0, 0, 1013, 1041, 1002, 1020, 1038, 996, 1027, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1049, 972, 1012, 978, 1058, 1016, 976, 16, 7, 0, 0, 0, 0, 0, 0, 1019, 1044, 1031, 1004, 1028, 1019, 1048, 975, 27, 19, 0, 0, 0, 0, 0, 0, 1043, 1012, 1050, 1068, 1052, 1007, 993, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 998, 985, 1003, 1015, 1041, 1026, 1051, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1015, 1011, 1025, 986, 1071, 1064, 1016, 6, 0, 0, 0, 0, 0, 0, 0, 1010, 1043, 1021, 982, 1079, 990, 1048, 1060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8180, 8112, 8214, 8233, 8212, 8230, 8233] avg=8192.0000 max=8233 max_rank=4 min=8112 min_rank=2 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32908 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[994, 1020, 995, 986, 1028, 1034, 1032, 1001, 11, 6, 0, 0, 0, 0, 0, 0, 1034, 1025, 1007, 1001, 1035, 1013, 1022, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1039, 983, 1107, 1062, 1058, 1043, 1052, 16, 8, 0, 0, 0, 0, 0, 0, 1024, 1033, 1016, 982, 1052, 998, 1030, 1014, 28, 20, 0, 0, 0, 0, 0, 0, 985, 1013, 1024, 1032, 1017, 1051, 1024, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 978, 980, 1023, 1031, 1038, 1012, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1080, 1020, 1016, 995, 1024, 1003, 1009, 5, 0, 0, 0, 0, 0, 0, 0, 1032, 1043, 1027, 1012, 1027, 1025, 1043, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8147, 8375, 8197, 8175, 8150, 8170, 8215] avg=8192.0000 max=8375 max_rank=2 min=8107 min_rank=0 imbalance_ratio=1.022339 max_load_reduction=57161 max_load_reduction_pct=0.872208 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[993, 1010, 1027, 1046, 1021, 986, 1032, 1028, 11, 6, 0, 0, 0, 0, 0, 0, 1064, 1028, 1006, 1039, 998, 1005, 1010, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1028, 1041, 1044, 992, 1017, 1012, 988, 16, 8, 0, 0, 0, 0, 0, 0, 990, 1046, 1034, 985, 1013, 992, 985, 1032, 28, 20, 0, 0, 0, 0, 0, 0, 977, 1023, 974, 1057, 1010, 1061, 1079, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1043, 1046, 1033, 1026, 1000, 1021, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1049, 1063, 1038, 1020, 1036, 1037, 1063, 5, 0, 0, 0, 0, 0, 0, 0, 1001, 1001, 1036, 1010, 981, 1016, 1042, 1054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8209, 8150, 8125, 8225, 8211, 8315, 8141] avg=8192.0000 max=8315 max_rank=6 min=8125 min_rank=3 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32892 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1028, 1031, 1014, 1038, 1041, 1081, 1037, 1012, 16, 10, 0, 0, 0, 0, 0, 0, 1040, 1052, 1031, 1048, 1050, 994, 1023, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1041, 991, 1022, 1052, 972, 988, 1043, 20, 10, 0, 0, 0, 0, 0, 0, 1033, 1002, 1050, 979, 982, 1027, 1027, 1014, 32, 23, 0, 0, 0, 0, 0, 0, 970, 1033, 1045, 1006, 1094, 946, 1020, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1031, 1008, 1017, 984, 1000, 1028, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1009, 1061, 999, 1008, 1064, 1034, 1042, 11, 0, 0, 0, 0, 0, 0, 0, 1035, 1018, 1055, 994, 1069, 995, 1057, 988, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8269, 8162, 8169, 8083, 8105, 8229, 8211] avg=8192.0000 max=8308 max_rank=0 min=8083 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32628 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8064, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8448, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8064, 8320, 8320, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8064, 8192, 8192, 8192, 8192, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1000, 1043, 1054, 1031, 1029, 1064, 996, 1010, 964, 984, 1024, 1053, 977, 1017, 1072, 1033, 1025, 1032, 992, 1020, 1024, 1036, 1036, 1072, 1052, 1011, 1004, 1034, 1007, 1022, 1021, 1022, 1057, 1070, 1040, 970, 977, 1012, 1000, 1012, 1024, 965, 1096, 1001, 1029, 1013, 1033, 1005, 1028, 1066, 1035, 1058, 1021, 1073, 1001, 1041, 987, 1062, 1060, 1040, 991, 998, 956, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[987, 1036, 1027, 1032, 1029, 1045, 1038, 1054, 1016, 1095, 1013, 1064, 1039, 1022, 1043, 1033, 1065, 1048, 975, 1012, 1006, 993, 1012, 1025, 1032, 988, 1040, 970, 1033, 1005, 1030, 1022, 973, 987, 1037, 981, 1029, 1082, 1017, 1023, 1045, 1025, 1041, 1023, 1018, 1039, 1069, 1050, 1049, 997, 1036, 1024, 999, 1043, 1027, 1029, 1048, 952, 1026, 1030, 988, 1029, 1004, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1003, 1009, 1078, 1089, 979, 1019, 1051, 1023, 1043, 1029, 1046, 1001, 1008, 1009, 1037, 1015, 1018, 1009, 1034, 1018, 1058, 1009, 1058, 1038, 1032, 1018, 979, 1011, 1011, 1059, 1078, 1012, 985, 979, 1013, 1048, 990, 1052, 1017, 1023, 969, 982, 1000, 1061, 1039, 998, 1014, 1023, 1052, 1035, 992, 1043, 1013, 1009, 986, 1040, 1031, 1065, 1036, 976, 997, 1035, 1061, 1091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1037, 1006, 985, 1032, 1022, 1020, 993, 1056, 1032, 1020, 1026, 1052, 1043, 959, 1024, 1066, 1034, 1049, 1013, 1013, 995, 997, 1074, 1047, 986, 1029, 1022, 1020, 1069, 1019, 1030, 1018, 1002, 1016, 1081, 1091, 1025, 964, 1056, 1017, 1064, 1006, 1076, 1034, 1050, 1053, 1004, 1053, 971, 1033, 1016, 987, 985, 1034, 1024, 999, 1018, 1020, 982, 1046, 1015, 1039, 978, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1021, 1055, 1010, 1027, 1033, 1048, 1037, 1001, 1037, 1003, 993, 982, 1023, 1071, 1022, 1021, 1035, 1016, 1024, 987, 1009, 1029, 1007, 1040, 1033, 1068, 1052, 999, 1038, 1043, 1017, 1009, 985, 976, 1057, 1002, 1019, 1046, 1023, 1012, 1045, 1034, 1038, 984, 1007, 990, 1032, 1037, 1069, 990, 1077, 1012, 1085, 1018, 984, 993, 1041, 1023, 1054, 1034, 1020, 1007, 1040, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1002, 1007, 1026, 994, 1084, 1039, 1060, 1037, 1012, 1011, 1012, 1033, 1002, 1048, 1033, 984, 1037, 1004, 1026, 1109, 1035, 995, 1008, 1054, 1007, 1004, 1079, 1036, 1025, 1004, 1014, 992, 1007, 1032, 1022, 1013, 978, 1025, 1032, 1030, 1009, 996, 1051, 1065, 1033, 1074, 1052, 969, 1043, 1085, 1012, 1034, 1017, 992, 1024, 1034, 1001, 1004, 992, 1040, 975, 997, 1063, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1071, 1048, 1055, 1081, 1005, 1049, 1000, 1071, 1032, 1051, 1069, 1007, 1031, 1028, 1009, 1030, 1000, 1016, 1013, 1010, 989, 984, 1024, 1042, 1015, 1012, 1060, 995, 1042, 1002, 1051, 964, 982, 1019, 1070, 1026, 1095, 1015, 993, 1036, 1037, 994, 1027, 1025, 1017, 984, 1058, 949, 1028, 995, 1039, 1057, 1006, 994, 1023, 973, 1056, 982, 1080, 1003, 1040, 1015, 1034, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1009, 1028, 1035, 1049, 1020, 1033, 1070, 1050, 979, 1050, 958, 1023, 1088, 989, 1061, 1016, 1063, 1023, 1063, 1004, 1018, 1052, 1037, 1015, 1009, 1007, 1000, 981, 1021, 1004, 1055, 1000, 1025, 1043, 1056, 999, 1029, 1006, 991, 992, 1033, 1017, 1011, 1005, 1055, 1029, 1048, 1041, 977, 999, 994, 1057, 1021, 1026, 1060, 1009, 1006, 1009, 1043, 1025, 1041, 1028, 1006, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1021, 1055, 1010, 967, 1033, 1048, 1037, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1003, 993, 982, 1023, 1071, 1019, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1016, 1024, 987, 1009, 1029, 1007, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1068, 1052, 999, 1038, 1043, 1017, 1009, 26, 0, 0, 0, 0, 0, 0, 0, 985, 976, 1057, 1002, 1019, 1046, 1023, 1012, 33, 0, 0, 0, 0, 0, 0, 0, 1045, 1034, 1025, 984, 1007, 990, 1032, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 990, 1077, 1012, 1085, 1018, 984, 993, 3, 3, 0, 0, 0, 0, 0, 0, 1041, 1023, 1054, 1034, 1020, 1007, 1040, 1012, 13, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8149, 8140, 8285, 8153, 8154, 8234, 8249] avg=8192.0000 max=8285 max_rank=3 min=8140 min_rank=2 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[987, 1036, 1027, 972, 1029, 1045, 1038, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1095, 1013, 1064, 1039, 1022, 1040, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1048, 975, 1012, 1006, 993, 1012, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 988, 1040, 970, 1033, 1005, 1030, 1022, 26, 0, 0, 0, 0, 0, 0, 0, 973, 987, 1037, 981, 1029, 1082, 1017, 1023, 33, 0, 0, 0, 0, 0, 0, 0, 1045, 1025, 1028, 1023, 1018, 1039, 1069, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 997, 1036, 1024, 999, 1043, 1027, 1029, 3, 3, 0, 0, 0, 0, 0, 0, 1048, 952, 1026, 1030, 988, 1029, 1004, 987, 13, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8322, 8129, 8146, 8162, 8297, 8210, 8082] avg=8192.0000 max=8322 max_rank=1 min=8082 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1003, 1009, 1078, 1026, 979, 1019, 1051, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1029, 1046, 1001, 1008, 1009, 1034, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1009, 1034, 1018, 1058, 1009, 1058, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1018, 979, 1011, 1011, 1059, 1078, 1012, 27, 0, 0, 0, 0, 0, 0, 0, 985, 979, 1013, 1048, 990, 1052, 1017, 1023, 35, 0, 0, 0, 0, 0, 0, 0, 969, 982, 987, 1061, 1039, 998, 1014, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1035, 992, 1043, 1013, 1009, 986, 1040, 3, 3, 0, 0, 0, 0, 0, 0, 1031, 1065, 1036, 976, 997, 1035, 1061, 1091, 13, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8185, 8235, 8227, 8142, 8073, 8176, 8310] avg=8192.0000 max=8310 max_rank=7 min=8073 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1000, 1043, 1054, 971, 1029, 1064, 996, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 964, 984, 1024, 1053, 977, 1017, 1068, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1032, 992, 1020, 1024, 1036, 1036, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1011, 1004, 1034, 1007, 1022, 1021, 1022, 26, 0, 0, 0, 0, 0, 0, 0, 1057, 1070, 1040, 970, 977, 1012, 1000, 1012, 33, 0, 0, 0, 0, 0, 0, 0, 1024, 965, 1082, 1001, 1029, 1013, 1033, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1066, 1035, 1058, 1021, 1073, 1001, 1041, 4, 4, 0, 0, 0, 0, 0, 0, 987, 1062, 1060, 1040, 991, 998, 956, 1054, 14, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8120, 8229, 8199, 8171, 8152, 8331, 8167] avg=8192.0000 max=8331 max_rank=6 min=8120 min_rank=1 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1037, 1006, 985, 972, 1022, 1020, 993, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1020, 1026, 1052, 1043, 959, 1021, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1049, 1013, 1013, 995, 997, 1074, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1029, 1022, 1020, 1069, 1019, 1030, 1018, 26, 0, 0, 0, 0, 0, 0, 0, 1002, 1016, 1081, 1091, 1025, 964, 1056, 1017, 33, 0, 0, 0, 0, 0, 0, 0, 1064, 1006, 1062, 1034, 1050, 1053, 1004, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1033, 1016, 987, 985, 1034, 1024, 999, 4, 3, 0, 0, 0, 0, 0, 0, 1018, 1020, 982, 1046, 1015, 1039, 978, 1009, 14, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8091, 8219, 8214, 8219, 8285, 8326, 8056, 8126] avg=8192.0000 max=8326 max_rank=5 min=8056 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32793 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1002, 1007, 1026, 936, 1084, 1039, 1060, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1011, 1012, 1033, 1002, 1048, 1030, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1004, 1026, 1109, 1035, 995, 1008, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1004, 1079, 1036, 1025, 1004, 1014, 992, 25, 0, 0, 0, 0, 0, 0, 0, 1007, 1032, 1022, 1013, 978, 1025, 1032, 1030, 32, 0, 0, 0, 0, 0, 0, 0, 1009, 996, 1037, 1065, 1033, 1074, 1052, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1085, 1012, 1034, 1017, 992, 1024, 1034, 4, 3, 0, 0, 0, 0, 0, 0, 1001, 1004, 992, 1040, 975, 997, 1063, 1022, 14, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8132, 8260, 8186, 8171, 8235, 8248, 8113] avg=8192.0000 max=8260 max_rank=2 min=8113 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1071, 1048, 1055, 1018, 1005, 1049, 1000, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1051, 1069, 1007, 1031, 1028, 1006, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1016, 1013, 1010, 989, 984, 1024, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1012, 1060, 995, 1042, 1002, 1051, 964, 27, 0, 0, 0, 0, 0, 0, 0, 982, 1019, 1070, 1026, 1095, 1015, 993, 1036, 35, 0, 0, 0, 0, 0, 0, 0, 1037, 994, 1014, 1025, 1017, 984, 1058, 949, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 995, 1039, 1057, 1006, 994, 1023, 973, 4, 3, 0, 0, 0, 0, 0, 0, 1056, 982, 1080, 1003, 1040, 1015, 1034, 1028, 13, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 8254, 8070, 8168, 8271, 8078, 8122, 8256] avg=8192.0000 max=8317 max_rank=0 min=8070 min_rank=2 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1009, 1028, 1035, 977, 1020, 1033, 1070, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1050, 958, 1023, 1088, 989, 1052, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1023, 1063, 1004, 1018, 1052, 1037, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1007, 1000, 981, 1021, 1004, 1055, 1000, 29, 0, 0, 0, 0, 0, 0, 0, 1025, 1043, 1056, 999, 1029, 1006, 991, 992, 40, 0, 0, 0, 0, 0, 0, 0, 1033, 1017, 993, 1005, 1055, 1029, 1048, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 977, 999, 994, 1057, 1021, 1026, 1060, 1009, 7, 9, 0, 0, 0, 0, 0, 0, 1006, 1009, 1043, 1025, 1041, 1028, 1006, 1045, 18, 9, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8155, 8259, 8106, 8181, 8221, 8159, 8233] avg=8192.0000 max=8259 max_rank=2 min=8106 min_rank=3 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 7936, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8448, 8192, 8192, 8320, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8064, 8320, 8192, 8320, 8064, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1037, 992, 1036, 996, 1053, 996, 1057, 1046, 1017, 1050, 979, 1055, 1028, 1055, 1009, 1020, 1035, 1007, 1031, 1038, 1042, 1006, 1035, 989, 1074, 991, 999, 1046, 1025, 1006, 1064, 1002, 1034, 1051, 1036, 1017, 981, 1062, 1009, 1054, 1076, 1032, 1004, 1048, 1045, 1018, 1023, 1002, 975, 1029, 1036, 1007, 1004, 1053, 1014, 1015, 1002, 995, 1036, 1048, 1005, 977, 1016, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[995, 991, 1069, 978, 986, 1087, 1016, 1034, 1035, 1060, 1003, 1051, 1013, 1032, 975, 1009, 1046, 999, 1043, 986, 1031, 987, 1007, 1073, 1040, 1075, 1066, 1009, 1053, 1003, 1027, 1029, 1002, 974, 1027, 945, 1036, 1006, 1029, 1015, 1047, 989, 1043, 991, 980, 1060, 1010, 1057, 1016, 1065, 1038, 1021, 1030, 1028, 1023, 1041, 986, 1038, 1007, 1027, 1039, 1053, 1058, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[995, 1044, 981, 1050, 1048, 971, 1011, 1041, 992, 1054, 1052, 1013, 951, 1067, 989, 1042, 1072, 992, 1001, 1031, 1053, 1026, 1007, 1038, 1026, 1047, 1009, 1030, 1035, 1022, 1038, 1020, 976, 1030, 1005, 997, 1049, 1030, 1012, 1047, 997, 1038, 1007, 1017, 1026, 1036, 1043, 1042, 1069, 948, 1001, 1053, 1026, 1033, 1006, 974, 1059, 982, 1055, 1019, 1038, 1076, 1023, 1074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1015, 985, 1044, 1058, 1027, 1010, 993, 1025, 1004, 997, 1035, 982, 1006, 1030, 988, 995, 989, 1035, 966, 981, 1041, 1001, 1089, 1053, 1024, 950, 1048, 1048, 1055, 1051, 997, 1050, 1083, 1025, 1024, 1032, 1004, 978, 985, 1036, 1018, 1023, 1044, 1032, 1040, 1066, 1019, 1070, 1037, 1044, 1015, 1010, 1029, 1004, 1014, 957, 1035, 1070, 1071, 1003, 1029, 1062, 1068, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1037, 1069, 1020, 1006, 1019, 1062, 995, 1018, 1031, 1052, 999, 1041, 986, 1062, 996, 959, 1019, 1036, 1046, 995, 988, 1034, 1061, 989, 1029, 1008, 1048, 1044, 985, 1008, 1023, 1035, 992, 1030, 1055, 1048, 993, 1063, 1033, 1009, 1011, 1049, 1013, 970, 1076, 1035, 1022, 1046, 1040, 981, 1024, 1051, 1041, 1047, 1020, 1037, 1008, 1047, 1004, 1008, 995, 1038, 1016, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[977, 1013, 1001, 1048, 1026, 1007, 1016, 1036, 993, 1060, 1045, 987, 1012, 1004, 982, 997, 1045, 1004, 1044, 975, 1021, 1069, 1023, 1042, 1086, 1064, 1006, 975, 1010, 982, 1044, 992, 1053, 1067, 989, 1039, 1051, 1034, 995, 1041, 1002, 1040, 1039, 1000, 1028, 1010, 1056, 1030, 1012, 1026, 1029, 1020, 1036, 1045, 1011, 1010, 994, 1065, 1059, 998, 1047, 1076, 1037, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[968, 1033, 996, 1024, 1022, 1048, 1053, 1028, 1035, 1026, 1057, 1056, 1005, 1012, 1022, 1001, 1061, 1087, 993, 983, 998, 1033, 959, 1040, 1060, 1015, 990, 1054, 988, 1041, 1074, 1008, 1034, 1024, 1008, 1024, 993, 1011, 1026, 1015, 1044, 1007, 1003, 1032, 1035, 1022, 1060, 1046, 950, 1068, 1027, 1039, 1001, 975, 1060, 1008, 1030, 1067, 1057, 991, 1055, 1018, 1026, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1052, 1037, 1042, 1024, 1008, 1021, 1075, 1001, 978, 1022, 1061, 1027, 1025, 1003, 1048, 1025, 1059, 1060, 982, 983, 999, 1016, 1009, 1031, 991, 1013, 1023, 1057, 1053, 994, 1035, 1029, 981, 1022, 1017, 1000, 1000, 1005, 1018, 997, 1004, 1009, 994, 1070, 1099, 992, 1075, 1066, 1100, 1069, 1043, 998, 1059, 1001, 1049, 1037, 1004, 1013, 1014, 1027, 1001, 954, 980, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[995, 991, 1069, 978, 986, 1087, 1016, 1034, 10, 0, 0, 0, 0, 0, 0, 0, 1035, 1060, 1003, 1051, 1013, 1032, 975, 1009, 42, 0, 0, 0, 0, 0, 0, 0, 1046, 999, 1043, 986, 1031, 987, 1007, 1073, 11, 3, 0, 0, 0, 0, 0, 0, 1019, 1075, 1066, 1009, 1053, 1003, 1027, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 974, 1027, 945, 1036, 1006, 1029, 1015, 27, 6, 0, 0, 0, 0, 0, 0, 1047, 989, 1043, 991, 980, 1060, 1010, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1065, 1038, 1021, 1030, 1028, 1023, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1038, 977, 1027, 1039, 1053, 1058, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8220, 8186, 8281, 8067, 8129, 8262, 8225] avg=8192.0000 max=8281 max_rank=3 min=8067 min_rank=4 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1037, 992, 1036, 996, 1053, 996, 1057, 1046, 11, 0, 0, 0, 0, 0, 0, 0, 1017, 1050, 979, 1055, 1028, 1055, 1009, 1020, 40, 0, 0, 0, 0, 0, 0, 0, 1035, 1007, 1031, 1038, 1042, 1006, 1035, 989, 11, 3, 0, 0, 0, 0, 0, 0, 1052, 991, 999, 1046, 1025, 1006, 1064, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1051, 1036, 1017, 981, 1062, 1009, 1054, 28, 6, 0, 0, 0, 0, 0, 0, 1076, 1032, 1004, 1048, 1045, 1018, 1023, 956, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1029, 1036, 1007, 1004, 1053, 1014, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 995, 1005, 1048, 1005, 977, 1016, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8253, 8197, 8185, 8278, 8202, 8133, 8064] avg=8192.0000 max=8278 max_rank=4 min=8064 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1015, 985, 1044, 1058, 1027, 1010, 993, 1025, 10, 0, 0, 0, 0, 0, 0, 0, 1004, 997, 1035, 982, 1006, 1030, 988, 995, 43, 0, 0, 0, 0, 0, 0, 0, 989, 1035, 966, 981, 1041, 1001, 1089, 1053, 11, 3, 0, 0, 0, 0, 0, 0, 1003, 950, 1048, 1048, 1055, 1051, 997, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 1025, 1024, 1032, 1004, 978, 985, 1036, 29, 6, 0, 0, 0, 0, 0, 0, 1018, 1023, 1044, 1032, 1040, 1066, 1019, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1044, 1015, 1010, 1029, 1004, 1014, 957, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1070, 1039, 1003, 1029, 1062, 1068, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8080, 8169, 8202, 8202, 8263, 8110, 8343] avg=8192.0000 max=8343 max_rank=7 min=8080 min_rank=1 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[995, 1044, 981, 1050, 1048, 971, 1011, 1041, 10, 0, 0, 0, 0, 0, 0, 0, 992, 1054, 1052, 1013, 951, 1067, 989, 1042, 42, 0, 0, 0, 0, 0, 0, 0, 1072, 992, 1001, 1031, 1053, 1026, 1007, 1038, 11, 3, 0, 0, 0, 0, 0, 0, 1005, 1047, 1009, 1030, 1035, 1022, 1038, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1030, 1005, 997, 1049, 1030, 1012, 1047, 28, 6, 0, 0, 0, 0, 0, 0, 997, 1038, 1007, 1017, 1026, 1036, 1043, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 948, 1001, 1053, 1026, 1033, 1006, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 982, 1024, 1019, 1038, 1076, 1023, 1074, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8202, 8234, 8206, 8180, 8158, 8110, 8295] avg=8192.0000 max=8295 max_rank=7 min=8110 min_rank=6 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1037, 1069, 1020, 1006, 1019, 1062, 995, 1018, 10, 0, 0, 0, 0, 0, 0, 0, 1031, 1052, 999, 1041, 986, 1062, 996, 959, 42, 0, 0, 0, 0, 0, 0, 0, 1019, 1036, 1046, 995, 988, 1034, 1061, 989, 11, 3, 0, 0, 0, 0, 0, 0, 1008, 1008, 1048, 1044, 985, 1008, 1023, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1030, 1055, 1048, 993, 1063, 1033, 1009, 27, 6, 0, 0, 0, 0, 0, 0, 1011, 1049, 1013, 970, 1076, 1035, 1022, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 981, 1024, 1051, 1041, 1047, 1020, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1047, 974, 1008, 995, 1038, 1016, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8168, 8182, 8159, 8256, 8174, 8241, 8120] avg=8192.0000 max=8256 max_rank=4 min=8120 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[968, 1033, 996, 1024, 1022, 1048, 1053, 1028, 11, 0, 0, 0, 0, 0, 0, 0, 1035, 1026, 1057, 1056, 1005, 1012, 1022, 1001, 42, 0, 0, 0, 0, 0, 0, 0, 1061, 1087, 993, 983, 998, 1033, 959, 1040, 11, 3, 0, 0, 0, 0, 0, 0, 1038, 1015, 990, 1054, 988, 1041, 1074, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1024, 1008, 1024, 993, 1011, 1026, 1015, 28, 6, 0, 0, 0, 0, 0, 0, 1044, 1007, 1003, 1032, 1035, 1022, 1060, 998, 0, 0, 0, 0, 0, 0, 0, 0, 950, 1068, 1027, 1039, 1001, 975, 1060, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1067, 1026, 991, 1055, 1018, 1026, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8256, 8168, 8208, 8169, 8201, 8128, 8223] avg=8192.0000 max=8256 max_rank=1 min=8128 min_rank=6 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[977, 1013, 1001, 1048, 1026, 1007, 1016, 1036, 11, 0, 0, 0, 0, 0, 0, 0, 993, 1060, 1045, 987, 1012, 1004, 982, 997, 41, 0, 0, 0, 0, 0, 0, 0, 1045, 1004, 1044, 975, 1021, 1069, 1023, 1042, 12, 3, 0, 0, 0, 0, 0, 0, 1063, 1064, 1006, 975, 1010, 982, 1044, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1067, 989, 1039, 1051, 1034, 995, 1041, 28, 6, 0, 0, 0, 0, 0, 0, 1002, 1040, 1039, 1000, 1028, 1010, 1056, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1026, 1029, 1020, 1036, 1045, 1011, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1065, 1028, 998, 1047, 1076, 1037, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8121, 8238, 8136, 8303, 8158, 8189, 8256] avg=8192.0000 max=8303 max_rank=4 min=8121 min_rank=1 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1052, 1037, 1042, 1024, 1008, 1021, 1075, 1001, 14, 0, 0, 0, 0, 0, 0, 0, 978, 1022, 1061, 1027, 1025, 1003, 1048, 1025, 47, 0, 0, 0, 0, 0, 0, 0, 1059, 1060, 982, 983, 999, 1016, 1009, 1031, 15, 8, 0, 0, 0, 0, 0, 0, 955, 1013, 1023, 1057, 1053, 994, 1035, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1022, 1017, 1000, 1000, 1005, 1018, 997, 30, 11, 0, 0, 0, 0, 0, 0, 1004, 1009, 994, 1070, 1099, 992, 1075, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 1069, 1043, 998, 1059, 1001, 1049, 1037, 7, 0, 0, 0, 0, 0, 0, 0, 1004, 1013, 976, 1027, 1001, 954, 980, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8236, 8162, 8159, 8081, 8251, 8363, 8010] avg=8192.0000 max=8363 max_rank=6 min=8010 min_rank=7 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8320, 8192, 8064, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8320, 8320, 8064, 8320, 8064, 8064, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8448, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1074, 1021, 982, 1029, 972, 1084, 1000, 1014, 1045, 1021, 1036, 1062, 1030, 1027, 1012, 981, 978, 1043, 1042, 1010, 999, 1090, 1052, 1021, 1053, 1019, 1016, 1044, 1067, 922, 1051, 1028, 1069, 1058, 998, 987, 1032, 1028, 1069, 1005, 986, 1057, 1058, 986, 1016, 999, 1052, 1029, 1002, 1034, 996, 1040, 1028, 1024, 992, 989, 998, 1024, 1012, 1052, 1036, 1002, 1038, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1074, 1035, 995, 979, 1027, 1023, 1015, 968, 1030, 986, 1080, 977, 1039, 1021, 1082, 1028, 1028, 1033, 998, 1069, 1008, 1081, 1003, 1005, 1035, 1033, 1017, 1011, 1012, 979, 1034, 1043, 1010, 1045, 1062, 1005, 1034, 1049, 1023, 1020, 1042, 1020, 1061, 1014, 990, 1027, 1015, 1042, 1063, 1042, 1017, 1005, 977, 997, 1031, 1053, 1004, 1012, 976, 1078, 1071, 1007, 978, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1002, 1061, 1021, 1060, 1001, 1048, 1042, 1025, 931, 1033, 1025, 942, 1039, 1031, 1046, 1013, 1004, 1024, 1005, 986, 1067, 943, 1012, 1041, 978, 1065, 1004, 1007, 1035, 979, 1033, 1078, 1050, 1031, 1018, 1025, 1026, 1033, 991, 1006, 1019, 1042, 1028, 997, 1030, 1077, 1062, 1023, 1008, 1025, 1037, 1001, 1014, 1089, 1021, 1027, 1028, 1060, 1032, 998, 1063, 999, 1058, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1009, 1026, 1004, 1034, 1015, 1067, 1011, 1037, 1010, 999, 1025, 1014, 1018, 1009, 1048, 1015, 1021, 1036, 976, 1044, 1038, 1034, 995, 1051, 1083, 1042, 985, 1021, 1040, 1032, 1003, 1005, 1039, 1021, 1029, 1069, 1020, 1054, 1035, 1036, 972, 1021, 1002, 1051, 1002, 1053, 1026, 1034, 1038, 1042, 1004, 1045, 1063, 1020, 1014, 1002, 986, 1004, 1024, 1004, 997, 1038, 1020, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1035, 1006, 1024, 1010, 1047, 1072, 1023, 1058, 1046, 1072, 1064, 1028, 950, 1064, 1019, 1046, 998, 1012, 1006, 1028, 1065, 997, 1065, 990, 1012, 991, 1081, 1041, 1017, 1029, 995, 991, 993, 966, 1011, 1001, 1009, 1015, 1006, 1035, 1029, 1009, 1026, 1001, 1022, 1046, 1026, 1067, 972, 961, 1076, 1009, 987, 1012, 1023, 1047, 1049, 1076, 1009, 1016, 1050, 1038, 1034, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1052, 1034, 1014, 1068, 966, 1031, 1010, 1068, 1083, 1014, 1017, 1035, 1033, 998, 1070, 1048, 992, 995, 1020, 1039, 1047, 1000, 1019, 1032, 1065, 995, 1059, 963, 1020, 1019, 1030, 1054, 998, 986, 1079, 973, 1036, 1065, 1049, 1019, 1002, 993, 983, 1026, 989, 1009, 1080, 1054, 973, 1070, 989, 1004, 1067, 1053, 1009, 993, 1013, 1023, 1001, 1062, 1019, 997, 1030, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1023, 1060, 1064, 1035, 1016, 1020, 1010, 984, 1007, 1005, 1006, 1074, 1016, 1009, 1046, 1010, 1032, 1054, 1018, 1069, 1022, 991, 1057, 1033, 1029, 1020, 1020, 1043, 1027, 1088, 1052, 1012, 994, 1062, 1029, 1059, 1004, 995, 1051, 1031, 1022, 1025, 986, 1017, 1010, 1003, 1032, 997, 960, 1026, 970, 1080, 1015, 1037, 996, 1073, 989, 1006, 988, 1021, 1064, 1019, 1023, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[996, 1009, 1003, 1014, 973, 1012, 1020, 1057, 984, 1033, 1003, 1013, 1029, 1069, 1020, 1053, 1058, 1030, 1018, 1009, 1002, 1017, 967, 1008, 1017, 1023, 1054, 1027, 1039, 1031, 1031, 1045, 1006, 1046, 1020, 1014, 995, 1003, 1073, 997, 1069, 1002, 1063, 1105, 990, 1025, 1025, 1028, 1087, 978, 987, 1000, 1012, 1042, 1033, 1039, 983, 1022, 1075, 1077, 1052, 977, 1030, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1074, 1035, 995, 979, 1027, 1019, 1015, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 986, 1080, 977, 1039, 1021, 1072, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1033, 998, 1069, 1008, 1081, 1003, 1005, 7, 5, 0, 0, 0, 0, 0, 0, 1018, 1033, 1017, 1011, 1012, 979, 1034, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1045, 1062, 1005, 1034, 1049, 1016, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1020, 1061, 1014, 990, 1027, 1009, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1042, 1017, 1005, 977, 997, 1031, 1053, 17, 10, 0, 0, 0, 0, 0, 0, 1004, 1012, 976, 1078, 1071, 1007, 978, 1018, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8233, 8237, 8147, 8241, 8205, 8212, 8149] avg=8192.0000 max=8241 max_rank=4 min=8112 min_rank=0 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1052, 1034, 1014, 1068, 966, 1027, 1010, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 1014, 1017, 1035, 1033, 998, 1060, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 992, 995, 1020, 1039, 1047, 1000, 1019, 1032, 7, 5, 0, 0, 0, 0, 0, 0, 1048, 995, 1059, 963, 1020, 1019, 1030, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 998, 986, 1079, 973, 1036, 1065, 1042, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 993, 983, 1026, 989, 1009, 1074, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1070, 989, 1004, 1067, 1053, 1009, 993, 17, 10, 0, 0, 0, 0, 0, 0, 1013, 1023, 1001, 1062, 1019, 997, 1030, 1002, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8288, 8156, 8188, 8198, 8130, 8185, 8152] avg=8192.0000 max=8288 max_rank=1 min=8130 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1074, 1021, 982, 1029, 972, 1080, 1000, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1021, 1036, 1062, 1030, 1027, 1003, 981, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1043, 1042, 1010, 999, 1090, 1052, 1021, 7, 5, 0, 0, 0, 0, 0, 0, 1036, 1019, 1016, 1044, 1067, 922, 1051, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1058, 998, 987, 1032, 1028, 1062, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1057, 1058, 986, 1016, 999, 1046, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1034, 996, 1040, 1028, 1024, 992, 989, 17, 9, 0, 0, 0, 0, 0, 0, 998, 1024, 1012, 1052, 1036, 1002, 1038, 1015, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8205, 8247, 8183, 8239, 8177, 8131, 8182] avg=8192.0000 max=8247 max_rank=2 min=8131 min_rank=6 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1035, 1006, 1024, 1010, 1047, 1068, 1023, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1072, 1064, 1028, 950, 1064, 1010, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1012, 1006, 1028, 1065, 997, 1065, 990, 7, 5, 0, 0, 0, 0, 0, 0, 996, 991, 1081, 1041, 1017, 1029, 995, 991, 0, 0, 0, 0, 0, 0, 0, 0, 993, 966, 1011, 1001, 1009, 1015, 999, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1009, 1026, 1001, 1022, 1046, 1020, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 972, 961, 1076, 1009, 987, 1012, 1023, 1047, 16, 9, 0, 0, 0, 0, 0, 0, 1049, 1076, 1009, 1016, 1050, 1038, 1034, 1033, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8280, 8173, 8141, 8029, 8220, 8112, 8310] avg=8192.0000 max=8310 max_rank=7 min=8029 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1009, 1026, 1004, 1034, 1015, 1063, 1011, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 999, 1025, 1014, 1018, 1009, 1039, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1036, 976, 1044, 1038, 1034, 995, 1051, 7, 5, 0, 0, 0, 0, 0, 0, 1065, 1042, 985, 1021, 1040, 1032, 1003, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1021, 1029, 1069, 1020, 1054, 1028, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1021, 1002, 1051, 1002, 1053, 1020, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1042, 1004, 1045, 1063, 1020, 1014, 1002, 18, 9, 0, 0, 0, 0, 0, 0, 986, 1004, 1024, 1004, 997, 1038, 1020, 1024, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8129, 8207, 8193, 8296, 8155, 8255, 8102] avg=8192.0000 max=8296 max_rank=4 min=8102 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1002, 1061, 1021, 1060, 1001, 1044, 1042, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 931, 1033, 1025, 942, 1039, 1031, 1037, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1024, 1005, 986, 1067, 943, 1012, 1041, 7, 5, 0, 0, 0, 0, 0, 0, 962, 1065, 1004, 1007, 1035, 979, 1033, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1031, 1018, 1025, 1026, 1033, 984, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1042, 1028, 997, 1030, 1077, 1056, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1025, 1037, 1001, 1014, 1089, 1021, 1027, 16, 9, 0, 0, 0, 0, 0, 0, 1028, 1060, 1032, 998, 1063, 999, 1058, 1037, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8051, 8094, 8163, 8173, 8272, 8247, 8280] avg=8192.0000 max=8280 max_rank=7 min=8051 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1023, 1060, 1064, 1035, 1016, 1016, 1010, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1005, 1006, 1074, 1016, 1009, 1037, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1054, 1018, 1069, 1022, 991, 1057, 1033, 7, 5, 0, 0, 0, 0, 0, 0, 1012, 1020, 1020, 1043, 1027, 1088, 1052, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1062, 1029, 1059, 1004, 995, 1044, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1025, 986, 1017, 1010, 1003, 1026, 997, 0, 0, 0, 0, 0, 0, 0, 0, 960, 1026, 970, 1080, 1015, 1037, 996, 1073, 17, 9, 0, 0, 0, 0, 0, 0, 989, 1006, 988, 1021, 1064, 1019, 1023, 1000, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8164, 8288, 8274, 8218, 8086, 8183, 8115] avg=8192.0000 max=8288 max_rank=2 min=8086 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32602 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[996, 1009, 1003, 1014, 973, 1007, 1020, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1033, 1003, 1013, 1029, 1069, 1002, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1030, 1018, 1009, 1002, 1017, 967, 1008, 12, 8, 5, 0, 0, 0, 0, 0, 997, 1023, 1054, 1027, 1039, 1031, 1031, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1046, 1020, 1014, 995, 1003, 1061, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1002, 1063, 1105, 990, 1025, 1009, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 978, 987, 1000, 1012, 1042, 1033, 1039, 20, 13, 0, 0, 0, 0, 0, 0, 983, 1022, 1075, 1077, 1052, 977, 1030, 1017, 5, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8079, 8186, 8134, 8247, 8142, 8291, 8211, 8246] avg=8192.0000 max=8291 max_rank=5 min=8079 min_rank=0 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8064, 8448, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8064, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8192, 8320, 8448, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1052, 1005, 1029, 1043, 1044, 1018, 1024, 1007, 986, 995, 1021, 1053, 1025, 1001, 1048, 1065, 1046, 1022, 1017, 1022, 1020, 1006, 985, 1027, 1043, 1022, 1011, 1028, 1008, 1006, 1047, 996, 989, 1008, 1022, 1046, 1073, 988, 974, 992, 989, 1111, 1017, 991, 988, 1040, 1029, 1009, 1037, 1021, 1077, 1047, 1009, 1039, 1032, 1049, 994, 1036, 1033, 1045, 1053, 1058, 1001, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1016, 1072, 991, 999, 1021, 1037, 1062, 989, 1002, 1066, 1035, 1031, 998, 1017, 998, 977, 1029, 1030, 1007, 977, 1036, 997, 1010, 1061, 1050, 989, 1046, 1036, 1075, 1007, 975, 1033, 987, 1025, 1014, 1024, 1051, 1021, 1080, 1021, 1036, 1051, 1039, 1016, 1008, 1065, 1052, 1020, 1005, 1058, 1027, 1050, 1017, 971, 1008, 1021, 1070, 983, 1041, 1054, 1046, 1010, 995, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1004, 1065, 982, 1074, 992, 1026, 1058, 995, 1015, 1004, 1060, 1046, 1028, 994, 1078, 1020, 1046, 1051, 960, 1032, 999, 1043, 1059, 985, 1009, 1053, 1003, 1019, 998, 1075, 1042, 1005, 1059, 1040, 1033, 1017, 1002, 1052, 1056, 1067, 1033, 1002, 1007, 997, 1025, 1018, 1028, 1004, 997, 1029, 1016, 1013, 989, 1012, 1005, 1021, 1053, 1077, 976, 1005, 998, 1046, 1021, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[999, 1107, 1032, 1095, 1007, 1013, 1067, 1031, 1042, 1001, 1023, 1005, 1024, 983, 986, 1028, 1016, 1046, 1027, 1042, 1046, 1013, 1040, 981, 1012, 1033, 990, 1029, 1023, 1038, 994, 1066, 1043, 1039, 1077, 997, 1016, 1013, 994, 998, 1066, 998, 989, 1016, 1031, 1002, 977, 1003, 1047, 1062, 1050, 1038, 973, 1016, 1039, 998, 995, 1019, 1024, 994, 1021, 1072, 1051, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1036, 1017, 998, 1019, 1028, 951, 1038, 1001, 1013, 1022, 1027, 1058, 1058, 1014, 1012, 1021, 1013, 1060, 970, 1005, 1002, 1103, 1038, 1021, 1070, 994, 1046, 1027, 1006, 1008, 956, 996, 1052, 1009, 1018, 1054, 1008, 1028, 1000, 1018, 1074, 1059, 987, 1027, 1061, 1033, 1051, 1025, 1047, 1020, 1089, 1031, 1029, 997, 1047, 1055, 973, 1010, 995, 1007, 985, 1040, 1055, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1018, 1102, 1067, 1043, 996, 1065, 1054, 1064, 1010, 1005, 1004, 1047, 1000, 970, 1048, 970, 1025, 967, 1052, 1052, 1020, 1003, 986, 1040, 1053, 1010, 1059, 1012, 1048, 1069, 1065, 1022, 1025, 991, 1080, 975, 1029, 1023, 984, 1015, 1053, 958, 1024, 1022, 989, 1063, 974, 945, 1025, 1031, 1007, 994, 1011, 1070, 1065, 1031, 1012, 1018, 1046, 1002, 1040, 1017, 1043, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1021, 1051, 998, 994, 1026, 1013, 1056, 1030, 966, 1038, 1016, 1034, 1029, 1003, 1077, 1031, 1052, 1017, 984, 1018, 1020, 1020, 1068, 1038, 975, 1097, 1017, 983, 1031, 1024, 1016, 1052, 1055, 1013, 998, 1011, 1022, 1065, 1016, 1065, 1056, 1055, 1008, 1021, 1011, 979, 1058, 1070, 1001, 1028, 971, 1053, 1057, 1056, 1062, 1057, 993, 986, 1000, 1017, 973, 983, 980, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1005, 1035, 1015, 1013, 988, 1047, 989, 1020, 1019, 1031, 1022, 983, 1052, 1062, 1037, 1031, 1058, 1036, 1014, 1046, 1009, 1034, 1066, 1025, 1036, 1024, 1059, 1056, 1015, 1054, 943, 967, 1017, 1037, 1040, 1003, 1042, 1038, 1017, 1013, 1028, 1065, 1054, 1041, 983, 999, 1008, 1015, 1052, 1052, 1006, 1014, 1101, 1063, 951, 998, 1071, 1000, 996, 1013, 1010, 1031, 987, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1052, 980, 1029, 1043, 1044, 1018, 1024, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 986, 995, 1021, 1053, 1025, 1001, 1048, 1065, 20, 0, 0, 0, 0, 0, 0, 0, 1046, 1022, 1017, 1022, 1020, 1006, 985, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1022, 1011, 1028, 1008, 1005, 1047, 996, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1008, 1017, 1046, 1073, 988, 974, 992, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1111, 1017, 991, 988, 1040, 1029, 1009, 5, 2, 1, 0, 0, 0, 0, 0, 1037, 984, 1077, 1047, 1009, 1039, 1032, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1036, 1033, 1045, 1053, 1058, 1001, 1017, 37, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8214, 8145, 8160, 8087, 8182, 8274, 8277] avg=8192.0000 max=8277 max_rank=7 min=8087 min_rank=4 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1016, 1046, 991, 999, 1021, 1037, 1062, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1066, 1035, 1031, 998, 1017, 998, 977, 21, 0, 0, 0, 0, 0, 0, 0, 1029, 1030, 1007, 977, 1036, 997, 1010, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 989, 1046, 1036, 1075, 1006, 975, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1025, 1009, 1024, 1051, 1021, 1080, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1051, 1039, 1016, 1008, 1065, 1052, 1020, 5, 2, 1, 0, 0, 0, 0, 0, 1005, 1019, 1027, 1050, 1017, 971, 1008, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 983, 1041, 1054, 1046, 1010, 995, 1001, 39, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8145, 8147, 8210, 8218, 8295, 8118, 8242] avg=8192.0000 max=8295 max_rank=5 min=8118 min_rank=6 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[999, 1080, 1032, 1095, 1007, 1013, 1067, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1001, 1023, 1005, 1024, 983, 986, 1028, 22, 0, 0, 0, 0, 0, 0, 0, 1016, 1046, 1027, 1042, 1046, 1013, 1040, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1033, 990, 1029, 1023, 1037, 994, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1039, 1072, 997, 1016, 1013, 994, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 998, 989, 1016, 1031, 1002, 977, 1003, 5, 2, 1, 0, 0, 0, 0, 0, 1047, 1023, 1050, 1038, 973, 1016, 1039, 998, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1019, 1024, 994, 1021, 1072, 1051, 1039, 39, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 8114, 8211, 8184, 8172, 8090, 8184, 8257] avg=8192.0000 max=8324 max_rank=0 min=8090 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1036, 992, 998, 1019, 1028, 951, 1038, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1022, 1027, 1058, 1058, 1014, 1012, 1021, 20, 0, 0, 0, 0, 0, 0, 0, 1013, 1060, 970, 1005, 1002, 1103, 1038, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 994, 1046, 1027, 1006, 1007, 956, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1009, 1013, 1054, 1008, 1028, 1000, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1059, 987, 1027, 1061, 1033, 1051, 1025, 5, 2, 1, 0, 0, 0, 0, 0, 1047, 983, 1089, 1031, 1029, 997, 1047, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1010, 995, 1007, 985, 1040, 1055, 1024, 37, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8063, 8245, 8212, 8102, 8182, 8325, 8278, 8129] avg=8192.0000 max=8325 max_rank=5 min=8063 min_rank=0 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1004, 1039, 982, 1074, 992, 1026, 1058, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1004, 1060, 1046, 1028, 994, 1078, 1020, 21, 0, 0, 0, 0, 0, 0, 0, 1046, 1051, 960, 1032, 999, 1043, 1059, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1053, 1003, 1019, 998, 1074, 1042, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1040, 1028, 1017, 1002, 1052, 1056, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1002, 1007, 997, 1025, 1018, 1028, 1004, 5, 2, 1, 0, 0, 0, 0, 0, 997, 991, 1016, 1013, 989, 1012, 1005, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1077, 976, 1005, 998, 1046, 1021, 1018, 38, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8266, 8175, 8203, 8321, 8122, 8044, 8235] avg=8192.0000 max=8321 max_rank=4 min=8044 min_rank=6 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1021, 1025, 998, 994, 1026, 1013, 1056, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1038, 1016, 1034, 1029, 1003, 1077, 1031, 21, 0, 0, 0, 0, 0, 0, 0, 1052, 1017, 984, 1018, 1020, 1020, 1068, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1097, 1017, 983, 1031, 1023, 1016, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1013, 993, 1011, 1022, 1065, 1016, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1055, 1008, 1021, 1011, 979, 1058, 1070, 5, 2, 1, 0, 0, 0, 0, 0, 1001, 990, 971, 1053, 1057, 1056, 1062, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 993, 986, 1000, 1017, 973, 983, 980, 1021, 38, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8215, 8217, 8194, 8240, 8266, 8247, 7994] avg=8192.0000 max=8266 max_rank=5 min=7994 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1018, 1075, 1067, 1043, 996, 1065, 1054, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1005, 1004, 1047, 1000, 970, 1048, 970, 22, 0, 0, 0, 0, 0, 0, 0, 1025, 967, 1052, 1052, 1020, 1003, 986, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1010, 1059, 1012, 1048, 1068, 1065, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 991, 1075, 975, 1029, 1023, 984, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 958, 1024, 1022, 989, 1063, 974, 945, 5, 2, 1, 0, 0, 0, 0, 0, 1025, 993, 1007, 994, 1011, 1070, 1065, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1018, 1046, 1002, 1040, 1017, 1043, 1028, 38, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8382, 8076, 8145, 8337, 8117, 8036, 8196, 8247] avg=8192.0000 max=8382 max_rank=0 min=8036 min_rank=5 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1005, 999, 1015, 1013, 988, 1047, 989, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1031, 1022, 983, 1052, 1062, 1037, 1031, 24, 0, 0, 0, 0, 0, 0, 0, 1054, 1036, 1014, 1046, 1009, 1034, 1066, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1024, 1059, 1056, 1015, 1046, 943, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1037, 1032, 1003, 1042, 1038, 1017, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1065, 1054, 1041, 983, 999, 1008, 1015, 8, 7, 8, 4, 0, 0, 0, 0, 1052, 1010, 1006, 1014, 1101, 1063, 951, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1000, 996, 1013, 1010, 1031, 987, 1000, 42, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8076, 8261, 8284, 8146, 8199, 8220, 8195, 8155] avg=8192.0000 max=8284 max_rank=2 min=8076 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[8448, 8320, 8192, 8192, 8192, 8320, 8192, 8192, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8320, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8064, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66560 slack_slots=3712 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1011, 972, 1003, 1023, 986, 1055, 1064, 1027, 1031, 1055, 1048, 1014, 1069, 1034, 1053, 996, 1045, 1031, 1041, 1008, 1029, 1026, 992, 995, 1011, 1074, 1086, 1041, 1023, 1065, 1049, 1060, 952, 1030, 946, 1041, 1031, 955, 999, 1051, 1006, 1060, 1018, 1014, 1017, 1005, 985, 1041, 1088, 996, 1033, 985, 1028, 1013, 1067, 1036, 995, 1028, 1007, 1016, 994, 1067, 1004, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1039, 1043, 1040, 1079, 1006, 1021, 988, 1010, 1019, 1035, 1100, 1053, 1015, 1028, 1008, 1042, 1014, 974, 1025, 1010, 1065, 962, 1000, 1042, 1030, 1032, 1006, 1061, 997, 1002, 1056, 1038, 972, 1012, 1038, 994, 1030, 1086, 1033, 1060, 1031, 1028, 1047, 1080, 979, 987, 1022, 999, 1043, 1042, 980, 1044, 1034, 1061, 967, 1014, 995, 1018, 982, 1005, 1009, 1031, 1040, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1028, 1032, 1033, 994, 1066, 1052, 1035, 1084, 1029, 1042, 986, 1042, 1012, 1021, 997, 1006, 1054, 1000, 989, 976, 1091, 1030, 988, 1026, 1013, 1069, 1034, 1044, 1046, 967, 1018, 1040, 1017, 1053, 1076, 1010, 1084, 1089, 1036, 1027, 996, 1030, 1069, 1021, 984, 1014, 1021, 993, 1014, 1033, 1007, 988, 1024, 1021, 1018, 1009, 1021, 1003, 1031, 988, 1000, 1031, 1015, 969] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1016, 954, 976, 1029, 1032, 1060, 950, 1024, 936, 1040, 1029, 1007, 997, 1021, 1028, 1049, 1003, 1031, 1038, 1048, 1025, 1076, 1044, 1075, 987, 995, 1047, 994, 989, 1056, 1063, 1050, 1038, 999, 1016, 1009, 1019, 1031, 1041, 1024, 1009, 990, 1004, 1037, 1012, 1096, 1040, 1028, 1036, 1033, 973, 961, 1041, 1009, 1037, 1037, 1032, 1073, 1079, 987, 1067, 1045, 1018, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[995, 1061, 1018, 1016, 1063, 1007, 1068, 1012, 1014, 1047, 1055, 1047, 998, 1051, 1007, 1006, 997, 1044, 1024, 982, 1064, 1010, 1042, 1054, 1026, 1022, 1042, 1002, 1035, 1007, 1010, 1016, 1061, 1046, 1045, 1047, 975, 1023, 1034, 947, 1003, 987, 1083, 989, 1063, 974, 1057, 997, 995, 1011, 1000, 1008, 1016, 1016, 1059, 1027, 1031, 1087, 1046, 1054, 970, 1026, 1000, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1018, 1055, 1001, 1032, 1082, 1010, 987, 1088, 1054, 1028, 1013, 1030, 1025, 1043, 988, 1039, 1017, 1036, 1017, 1037, 1028, 995, 1005, 1066, 1018, 988, 999, 1031, 1023, 1067, 1029, 972, 1067, 1040, 1060, 1018, 1023, 1074, 971, 1021, 1106, 1031, 1051, 1047, 1014, 1006, 1001, 1003, 997, 974, 1100, 977, 1024, 1000, 1004, 1016, 992, 979, 1001, 1011, 1052, 1009, 1043, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1029, 1023, 1032, 1006, 1018, 1050, 996, 1052, 1015, 1045, 1002, 1021, 1007, 1010, 979, 1061, 991, 1055, 1062, 1042, 1026, 964, 994, 1019, 1031, 1014, 1045, 1062, 1043, 1033, 1030, 984, 992, 1026, 1069, 1040, 1034, 1003, 1016, 1022, 996, 1044, 1009, 1007, 996, 992, 961, 1065, 1061, 1045, 1032, 1032, 1056, 1023, 1045, 1032, 1032, 1025, 1001, 991, 1028, 1077, 1053, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1034, 1039, 1015, 999, 1049, 1055, 977, 1055, 1012, 999, 959, 983, 1029, 1058, 932, 1024, 1025, 1063, 1098, 1021, 980, 1052, 1036, 1035, 1061, 1018, 1032, 1008, 980, 1008, 987, 986, 1069, 1003, 1067, 979, 1006, 1050, 998, 1032, 1045, 1032, 1044, 1034, 1022, 1102, 1022, 1023, 1046, 976, 979, 1039, 1014, 1045, 1048, 1041, 972, 1055, 996, 990, 1046, 1039, 1038, 1075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[995, 1061, 1018, 1016, 1063, 1007, 1068, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1047, 1055, 1047, 998, 1051, 1007, 1006, 10, 3, 0, 0, 0, 0, 0, 0, 997, 1044, 1024, 982, 1064, 1010, 1042, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1022, 1028, 1002, 1035, 1007, 1010, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1046, 1031, 1047, 975, 1023, 1034, 947, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 987, 1083, 989, 1063, 974, 1057, 997, 8, 0, 0, 0, 0, 0, 0, 0, 995, 1011, 1000, 1008, 1016, 1016, 1059, 1027, 11, 4, 0, 0, 0, 0, 0, 0, 1031, 1087, 1046, 1054, 970, 1026, 1000, 1017, 16, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8238, 8206, 8146, 8164, 8161, 8147, 8250] avg=8192.0000 max=8250 max_rank=7 min=8146 min_rank=3 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1011, 972, 1003, 1023, 986, 1055, 1064, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1055, 1048, 1014, 1069, 1034, 1053, 996, 10, 3, 0, 0, 0, 0, 0, 0, 1045, 1031, 1041, 1008, 1029, 1026, 992, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1074, 1072, 1041, 1023, 1065, 1049, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 952, 1030, 933, 1041, 1031, 955, 999, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1060, 1018, 1014, 1017, 1005, 985, 1041, 8, 0, 0, 0, 0, 0, 0, 0, 1088, 996, 1033, 985, 1028, 1013, 1067, 1036, 10, 4, 0, 0, 0, 0, 0, 0, 995, 1028, 1007, 1016, 994, 1067, 1004, 1011, 16, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 8313, 8156, 8395, 7992, 8154, 8260, 8141] avg=8192.0000 max=8395 max_rank=3 min=7992 min_rank=4 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=32547 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1016, 954, 976, 1029, 1032, 1060, 950, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 936, 1040, 1029, 1007, 997, 1021, 1028, 1049, 10, 4, 0, 0, 0, 0, 0, 0, 1003, 1031, 1038, 1048, 1025, 1076, 1044, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 987, 995, 1033, 994, 989, 1056, 1063, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 999, 1002, 1009, 1019, 1031, 1041, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 990, 1004, 1037, 1012, 1096, 1040, 1028, 8, 0, 0, 0, 0, 0, 0, 0, 1036, 1033, 973, 961, 1041, 1009, 1037, 1037, 11, 4, 0, 0, 0, 0, 0, 0, 1032, 1073, 1079, 987, 1067, 1045, 1018, 1046, 16, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8025, 8121, 8328, 8167, 8163, 8224, 8142, 8366] avg=8192.0000 max=8366 max_rank=7 min=8025 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1029, 1023, 1032, 1006, 1018, 1050, 996, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1045, 1002, 1021, 1007, 1010, 979, 1061, 10, 3, 0, 0, 0, 0, 0, 0, 991, 1055, 1062, 1042, 1026, 964, 994, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1014, 1031, 1062, 1043, 1033, 1030, 984, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1026, 1055, 1040, 1034, 1003, 1016, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1044, 1009, 1007, 996, 992, 961, 1065, 8, 0, 0, 0, 0, 0, 0, 0, 1061, 1045, 1032, 1032, 1056, 1023, 1045, 1032, 11, 4, 0, 0, 0, 0, 0, 0, 1032, 1025, 1001, 991, 1028, 1077, 1053, 990, 17, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8153, 8142, 8228, 8188, 8078, 8341, 8217] avg=8192.0000 max=8341 max_rank=6 min=8078 min_rank=5 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1039, 1043, 1040, 1079, 1006, 1021, 988, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1035, 1100, 1053, 1015, 1028, 1008, 1042, 9, 3, 0, 0, 0, 0, 0, 0, 1014, 974, 1025, 1010, 1065, 962, 1000, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1032, 993, 1061, 997, 1002, 1056, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1012, 1024, 994, 1030, 1086, 1033, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1028, 1047, 1080, 979, 987, 1022, 999, 8, 0, 0, 0, 0, 0, 0, 0, 1043, 1042, 980, 1044, 1034, 1061, 967, 1014, 11, 4, 0, 0, 0, 0, 0, 0, 995, 1018, 982, 1005, 1009, 1031, 1040, 1033, 16, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8312, 8081, 8209, 8211, 8181, 8200, 8132] avg=8192.0000 max=8312 max_rank=1 min=8081 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1028, 1032, 1033, 994, 1066, 1052, 1035, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1042, 986, 1042, 1012, 1021, 997, 1006, 10, 3, 0, 0, 0, 0, 0, 0, 1054, 1000, 989, 976, 1091, 1030, 988, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1069, 1020, 1044, 1046, 967, 1018, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1053, 1062, 1010, 1084, 1089, 1036, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1030, 1069, 1021, 984, 1014, 1021, 993, 8, 0, 0, 0, 0, 0, 0, 0, 1014, 1033, 1007, 988, 1024, 1021, 1018, 1009, 11, 4, 0, 0, 0, 0, 0, 0, 1021, 1003, 1031, 988, 1000, 1031, 1015, 969, 17, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 8148, 8143, 8217, 8378, 8136, 8129, 8078] avg=8192.0000 max=8378 max_rank=4 min=8078 min_rank=7 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1018, 1055, 1001, 1032, 1082, 1010, 987, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1028, 1013, 1030, 1025, 1043, 988, 1039, 9, 3, 0, 0, 0, 0, 0, 0, 1017, 1036, 1017, 1037, 1028, 995, 1005, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 988, 986, 1031, 1023, 1067, 1029, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1040, 1046, 1018, 1023, 1074, 971, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1106, 1031, 1051, 1047, 1014, 1006, 1001, 1003, 8, 0, 0, 0, 0, 0, 0, 0, 997, 974, 1100, 977, 1024, 1000, 1004, 1016, 11, 4, 0, 0, 0, 0, 0, 0, 992, 979, 1001, 1011, 1052, 1009, 1043, 1003, 17, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8232, 8190, 8114, 8260, 8267, 8107, 8110] avg=8192.0000 max=8267 max_rank=5 min=8107 min_rank=6 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1034, 1039, 1015, 999, 1049, 1055, 977, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 999, 959, 983, 1029, 1058, 932, 1024, 14, 9, 0, 0, 0, 0, 0, 0, 1025, 1063, 1098, 1021, 980, 1052, 1036, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1018, 1012, 1008, 980, 1008, 987, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1003, 1043, 979, 1006, 1050, 998, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1032, 1044, 1034, 1022, 1102, 1022, 1023, 11, 0, 0, 0, 0, 0, 0, 0, 1046, 976, 979, 1039, 1014, 1045, 1048, 1041, 16, 6, 0, 0, 0, 0, 0, 0, 972, 1055, 996, 990, 1046, 1039, 1038, 1075, 23, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8019, 8290, 8060, 8180, 8335, 8210, 8242] avg=8192.0000 max=8335 max_rank=5 min=8019 min_rank=1 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32967 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8448, 8320, 8192, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8064, 8192, 8448, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8064, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1020, 992, 1015, 1049, 1063, 999, 1018, 1064, 1055, 1040, 1029, 1055, 1011, 1075, 1048, 1020, 1018, 1038, 1014, 1002, 1012, 1049, 1037, 1015, 1022, 1056, 1052, 1001, 1011, 1076, 1010, 1003, 985, 1033, 1010, 1015, 1031, 1008, 1022, 947, 994, 978, 992, 1038, 1057, 1071, 993, 1035, 1021, 1006, 999, 1023, 1044, 1007, 1020, 1060, 1004, 992, 1034, 1061, 1058, 1038, 992, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1009, 1004, 1005, 1043, 1012, 987, 973, 1044, 1000, 998, 1022, 990, 1064, 970, 994, 1018, 1031, 1000, 970, 1050, 1035, 1031, 1007, 1032, 1069, 1056, 1043, 1003, 1040, 1041, 1057, 1040, 997, 1050, 1023, 1030, 1002, 1045, 1046, 1029, 1003, 1004, 1018, 1056, 1016, 1024, 1010, 1037, 1014, 1049, 1042, 977, 1092, 1039, 997, 1066, 1022, 1070, 956, 1010, 1029, 1001, 1079, 1065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1046, 1047, 1009, 1035, 1029, 1034, 980, 1062, 1023, 1032, 998, 1002, 1028, 1018, 1031, 1051, 1026, 1028, 1030, 1027, 1039, 1053, 1013, 1005, 990, 1008, 1010, 1024, 967, 977, 1016, 999, 1024, 1003, 1043, 1030, 1026, 1041, 1057, 1084, 1039, 1032, 1042, 1039, 1033, 990, 1021, 1004, 1006, 1035, 1041, 1026, 1022, 1050, 987, 1026, 1031, 1043, 1063, 999, 1000, 1043, 1032, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1036, 1049, 1018, 1021, 994, 964, 1075, 1038, 1028, 1044, 1049, 1002, 1017, 1052, 1011, 1011, 988, 1006, 1000, 1020, 1005, 1043, 1023, 1022, 1043, 1025, 1038, 1039, 1011, 1081, 1027, 1056, 1008, 1039, 999, 1011, 986, 993, 1065, 1036, 1035, 1046, 1058, 1014, 1081, 1011, 992, 1044, 973, 1013, 973, 998, 1024, 1048, 1030, 1020, 1025, 1044, 1035, 1006, 986, 1045, 1032, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1025, 1050, 1016, 999, 1058, 971, 984, 1009, 1018, 1026, 1028, 1061, 1009, 1015, 1100, 1037, 1000, 1042, 1047, 1050, 983, 1056, 1041, 1015, 1062, 1026, 993, 1039, 1077, 1050, 1020, 966, 1082, 1023, 1044, 1027, 1023, 1001, 1024, 1043, 1005, 1025, 1029, 1023, 1013, 1010, 999, 1001, 1003, 1065, 1019, 1025, 991, 1075, 971, 995, 1047, 1031, 987, 1020, 1049, 1033, 1003, 977] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1003, 1000, 1001, 1030, 1037, 1053, 1037, 1043, 1034, 1046, 1040, 984, 1048, 1019, 971, 989, 964, 1011, 1007, 1058, 1040, 1056, 1046, 1001, 1016, 1052, 1030, 1058, 1020, 996, 1024, 1046, 1034, 1023, 1017, 1064, 1084, 1063, 1049, 1072, 1013, 971, 996, 975, 1063, 1060, 1025, 1035, 1019, 1027, 1042, 1003, 1034, 972, 1009, 1018, 1043, 1016, 1068, 1002, 963, 1009, 1024, 983] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1004, 1064, 1030, 1017, 1002, 1070, 1003, 1005, 1070, 1002, 1032, 1002, 1040, 1050, 1088, 1013, 1029, 998, 995, 1030, 985, 1024, 1018, 1046, 1012, 1008, 1072, 1090, 1037, 988, 1022, 1006, 983, 985, 1029, 991, 1059, 984, 1047, 1031, 1037, 1034, 1022, 1015, 1043, 1034, 1018, 990, 1026, 1038, 1016, 1052, 1057, 1004, 994, 1008, 1082, 1019, 1033, 1010, 1049, 959, 973, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1060, 1044, 1006, 1051, 970, 1010, 1012, 1046, 1055, 1053, 1018, 1025, 1047, 1043, 1025, 1026, 995, 1046, 995, 1017, 1054, 1010, 1021, 992, 1005, 997, 1024, 1014, 1054, 1018, 1021, 1040, 1021, 1038, 1049, 999, 1014, 1013, 1031, 1042, 999, 963, 989, 995, 1013, 1041, 1008, 1095, 1046, 1031, 1032, 1012, 1063, 1021, 1075, 1011, 984, 1040, 987, 1028, 986, 1027, 1028, 1061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1020, 992, 1015, 1049, 1063, 999, 1018, 1064, 11, 0, 0, 0, 0, 0, 0, 0, 1023, 1040, 1029, 1055, 1011, 1075, 1048, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1038, 1014, 1002, 1012, 1049, 1037, 1015, 24, 0, 0, 0, 0, 0, 0, 0, 1022, 1056, 1052, 973, 1011, 1076, 1010, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1033, 1010, 1015, 1031, 1008, 1002, 947, 0, 0, 0, 0, 0, 0, 0, 0, 994, 978, 992, 1038, 1057, 1071, 993, 1035, 14, 8, 0, 0, 0, 0, 0, 0, 1021, 1006, 999, 1023, 1044, 1007, 1020, 1060, 6, 0, 0, 0, 0, 0, 0, 0, 1004, 992, 1034, 1061, 1058, 1038, 992, 999, 14, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8301, 8209, 8203, 8031, 8180, 8186, 8195] avg=8192.0000 max=8301 max_rank=1 min=8031 min_rank=4 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32592 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1009, 1004, 1005, 1043, 1012, 987, 973, 1044, 11, 0, 0, 0, 0, 0, 0, 0, 969, 998, 1022, 990, 1064, 970, 994, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1000, 970, 1050, 1035, 1031, 1007, 1032, 23, 0, 0, 0, 0, 0, 0, 0, 1069, 1056, 1043, 975, 1040, 1041, 1057, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1050, 1023, 1030, 1002, 1045, 1026, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1004, 1018, 1056, 1016, 1024, 1010, 1037, 14, 8, 0, 0, 0, 0, 0, 0, 1014, 1049, 1042, 977, 1092, 1039, 997, 1066, 6, 0, 0, 0, 0, 0, 0, 0, 1022, 1070, 956, 1010, 1029, 1001, 1079, 1065, 14, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8088, 8025, 8179, 8321, 8202, 8190, 8282, 8249] avg=8192.0000 max=8321 max_rank=3 min=8025 min_rank=1 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32923 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1036, 1049, 1018, 1021, 994, 964, 1075, 1038, 11, 0, 0, 0, 0, 0, 0, 0, 996, 1044, 1049, 1002, 1017, 1052, 1011, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1006, 1000, 1020, 1005, 1043, 1023, 1022, 24, 0, 0, 0, 0, 0, 0, 0, 1043, 1025, 1038, 1010, 1011, 1081, 1027, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1039, 999, 1011, 986, 993, 1045, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1046, 1058, 1014, 1081, 1011, 992, 1044, 15, 8, 0, 0, 0, 0, 0, 0, 973, 1013, 973, 998, 1024, 1048, 1030, 1020, 6, 0, 0, 0, 0, 0, 0, 0, 1025, 1044, 1035, 1006, 986, 1045, 1032, 1030, 14, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8182, 8131, 8291, 8117, 8304, 8085, 8220] avg=8192.0000 max=8304 max_rank=5 min=8085 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1046, 1047, 1009, 1035, 1029, 1034, 980, 1062, 11, 0, 0, 0, 0, 0, 0, 0, 991, 1032, 998, 1002, 1028, 1018, 1031, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1028, 1030, 1027, 1039, 1053, 1013, 1005, 24, 0, 0, 0, 0, 0, 0, 0, 990, 1008, 1010, 996, 967, 977, 1016, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1003, 1043, 1030, 1026, 1041, 1037, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1032, 1042, 1039, 1033, 990, 1021, 1004, 14, 8, 0, 0, 0, 0, 0, 0, 1006, 1035, 1041, 1026, 1022, 1050, 987, 1026, 6, 0, 0, 0, 0, 0, 0, 0, 1031, 1043, 1063, 999, 1000, 1043, 1032, 987, 14, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8151, 8245, 7963, 8288, 8222, 8199, 8215] avg=8192.0000 max=8288 max_rank=4 min=7963 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32924 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1003, 1000, 1001, 1030, 1037, 1053, 1037, 1043, 11, 0, 0, 0, 0, 0, 0, 0, 1002, 1046, 1040, 984, 1048, 1019, 971, 989, 0, 0, 0, 0, 0, 0, 0, 0, 964, 1011, 1007, 1058, 1040, 1056, 1046, 1001, 24, 0, 0, 0, 0, 0, 0, 0, 1016, 1052, 1030, 1029, 1020, 996, 1024, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1023, 1017, 1064, 1084, 1063, 1029, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 971, 996, 975, 1063, 1060, 1025, 1035, 15, 8, 0, 0, 0, 0, 0, 0, 1019, 1027, 1042, 1003, 1034, 972, 1009, 1018, 6, 0, 0, 0, 0, 0, 0, 0, 1043, 1016, 1068, 1002, 963, 1009, 1024, 983, 14, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8099, 8207, 8213, 8386, 8161, 8130, 8125] avg=8192.0000 max=8386 max_rank=4 min=8099 min_rank=1 imbalance_ratio=1.023682 max_load_reduction=57150 max_load_reduction_pct=0.872040 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1025, 1050, 1016, 999, 1058, 971, 984, 1009, 11, 0, 0, 0, 0, 0, 0, 0, 987, 1026, 1028, 1061, 1009, 1015, 1100, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1042, 1047, 1050, 983, 1056, 1041, 1015, 23, 0, 0, 0, 0, 0, 0, 0, 1062, 1026, 993, 1010, 1077, 1050, 1020, 966, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 1023, 1044, 1027, 1023, 1001, 1004, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1025, 1029, 1023, 1013, 1010, 999, 1001, 15, 8, 0, 0, 0, 0, 0, 0, 1003, 1065, 1019, 1025, 991, 1075, 971, 995, 6, 0, 0, 0, 0, 0, 0, 0, 1047, 1031, 987, 1020, 1049, 1033, 1003, 977, 14, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8263, 8257, 8204, 8247, 8128, 8150, 8164] avg=8192.0000 max=8263 max_rank=1 min=8123 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1004, 1064, 1030, 1017, 1002, 1070, 1003, 1005, 11, 0, 0, 0, 0, 0, 0, 0, 1037, 1002, 1032, 1002, 1040, 1050, 1088, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 998, 995, 1030, 985, 1024, 1018, 1046, 25, 0, 0, 0, 0, 0, 0, 0, 1012, 1008, 1072, 1060, 1037, 988, 1022, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 983, 985, 1029, 991, 1059, 984, 1027, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1034, 1022, 1015, 1043, 1034, 1018, 990, 15, 8, 0, 0, 0, 0, 0, 0, 1026, 1038, 1016, 1052, 1057, 1004, 994, 1008, 6, 0, 0, 0, 0, 0, 0, 0, 1082, 1019, 1033, 1010, 1049, 959, 973, 1062, 15, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8264, 8150, 8205, 8089, 8216, 8201, 8205] avg=8192.0000 max=8264 max_rank=1 min=8089 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1060, 1044, 1006, 1051, 970, 1010, 1012, 1046, 15, 0, 0, 0, 0, 0, 0, 0, 1014, 1053, 1018, 1025, 1047, 1043, 1025, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1046, 995, 1017, 1054, 1010, 1021, 992, 28, 0, 0, 0, 0, 0, 0, 0, 1005, 997, 1024, 977, 1054, 1018, 1021, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1038, 1049, 999, 1014, 1013, 1000, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 999, 963, 989, 995, 1013, 1041, 1008, 1095, 19, 13, 0, 0, 0, 0, 0, 0, 1046, 1031, 1032, 1012, 1063, 1021, 1075, 1011, 12, 0, 0, 0, 0, 0, 0, 0, 984, 1040, 987, 1028, 986, 1027, 1028, 1061, 18, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8251, 8158, 8136, 8176, 8135, 8303, 8163] avg=8192.0000 max=8303 max_rank=6 min=8135 min_rank=5 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8064, 8320, 8192, 8448, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8192, 8320, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8448, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8064, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1027, 1034, 1044, 1025, 1061, 1040, 1030, 987, 1034, 1039, 1043, 979, 1016, 1010, 1009, 1055, 1003, 1022, 990, 1015, 1028, 1079, 1006, 1042, 1056, 998, 1028, 1013, 1028, 988, 1045, 996, 993, 1052, 1045, 961, 1012, 1034, 1037, 1022, 1049, 1047, 1035, 1041, 1043, 1049, 1020, 1031, 1056, 1011, 1002, 1012, 1022, 1038, 1076, 1013, 1021, 1007, 1025, 1014, 1007, 936, 1015, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1024, 1034, 1028, 1003, 1031, 1010, 1050, 1056, 976, 1045, 1002, 1039, 1025, 1030, 1068, 1006, 1014, 999, 1104, 1003, 1003, 1071, 1019, 999, 1016, 1086, 1007, 992, 969, 998, 1003, 1037, 1010, 1071, 991, 1026, 1008, 1046, 1062, 1040, 1075, 1038, 1064, 997, 993, 1017, 989, 1001, 1042, 1013, 1013, 1062, 1033, 975, 1073, 1008, 983, 1060, 1015, 1037, 1000, 1017, 1042, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[978, 1024, 1002, 1009, 1080, 1039, 1018, 1022, 1057, 1005, 984, 993, 1047, 1010, 1043, 1057, 1072, 1029, 1074, 1033, 1043, 1039, 1030, 975, 1039, 982, 1052, 1012, 1000, 1077, 971, 1025, 1035, 1032, 1023, 983, 1063, 996, 1059, 1055, 1022, 1004, 1008, 1096, 1022, 1065, 1017, 1013, 1014, 1013, 994, 996, 1002, 1037, 1005, 978, 1010, 1017, 1070, 999, 1016, 1043, 1025, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[990, 1039, 1030, 1002, 1047, 1031, 995, 1062, 1003, 1014, 998, 1030, 985, 1032, 1031, 979, 946, 1027, 1059, 1006, 1027, 1001, 1066, 1069, 1065, 1001, 974, 1000, 1041, 1027, 1045, 974, 989, 1016, 1005, 1038, 1073, 1037, 1046, 1030, 1035, 980, 1066, 983, 1032, 983, 1015, 1035, 1052, 1042, 1023, 1034, 1006, 1059, 1035, 1027, 1039, 1021, 1110, 1017, 1075, 1012, 996, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1010, 1072, 996, 1024, 1033, 1004, 1029, 1002, 1050, 1049, 1047, 952, 1026, 1014, 991, 1004, 1060, 1008, 1021, 1004, 1061, 1001, 1082, 1032, 1033, 1054, 1058, 1019, 1029, 1039, 1033, 1030, 1032, 1070, 1017, 1001, 990, 978, 979, 1009, 1011, 1015, 1031, 1048, 987, 995, 1031, 1015, 1037, 1009, 1024, 1000, 1050, 1004, 1081, 1049, 1079, 1010, 1084, 1005, 1029, 980, 1003, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1049, 1029, 1015, 1029, 1009, 1025, 999, 999, 979, 989, 979, 1026, 1000, 1023, 987, 1044, 1059, 1033, 1018, 1058, 1065, 1013, 966, 1011, 1045, 1018, 1074, 1075, 995, 1035, 1028, 1030, 1000, 1026, 1029, 1009, 1039, 1069, 1053, 1040, 1076, 1001, 1023, 1017, 1034, 992, 1009, 967, 992, 1041, 1077, 1015, 1025, 985, 1074, 1026, 1043, 1006, 1006, 1027, 1028, 1035, 1038, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1034, 1027, 1038, 1007, 1075, 1060, 1056, 972, 993, 1022, 1010, 1032, 1009, 1068, 1059, 1003, 1021, 1037, 1044, 1039, 996, 1025, 1010, 1005, 1011, 1048, 1038, 1058, 995, 1004, 1062, 1044, 981, 1047, 1012, 1070, 1008, 983, 1046, 1020, 992, 1057, 978, 1025, 1003, 1040, 1037, 983, 1078, 1010, 991, 982, 1032, 1029, 1040, 1039, 1011, 1031, 1008, 1003, 1035, 1034, 1008, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1011, 1025, 1036, 1011, 1006, 1065, 1014, 1029, 1031, 1033, 1005, 1004, 1055, 1009, 1012, 1035, 1063, 1009, 1063, 1019, 1027, 1052, 1080, 1079, 1018, 1050, 1050, 1023, 1030, 1014, 978, 1031, 985, 983, 1039, 1019, 1041, 1036, 1067, 1010, 1022, 1031, 1014, 1007, 1036, 1009, 1003, 1003, 994, 1036, 1054, 1043, 993, 1001, 1000, 982, 1039, 1026, 1012, 1039, 1014, 1021, 1047, 963] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1027, 1034, 1044, 1025, 1048, 1040, 1030, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1039, 1043, 979, 1016, 1010, 1009, 1055, 41, 0, 0, 0, 0, 0, 0, 0, 1003, 1022, 942, 1015, 1028, 1079, 1006, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 998, 1028, 1013, 1028, 988, 1045, 996, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1052, 1045, 961, 1012, 1034, 1032, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1047, 1035, 1041, 1043, 1049, 1020, 1031, 13, 7, 0, 0, 0, 0, 0, 0, 1056, 1011, 1002, 1012, 1022, 1038, 1074, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1007, 1025, 1014, 1007, 936, 1015, 1040, 6, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8226, 8137, 8146, 8151, 8335, 8228, 8078] avg=8192.0000 max=8335 max_rank=5 min=8078 min_rank=7 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[978, 1024, 1002, 1009, 1067, 1039, 1018, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1005, 984, 993, 1047, 1010, 1043, 1057, 45, 0, 0, 0, 0, 0, 0, 0, 1072, 1029, 1021, 1033, 1043, 1039, 1030, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 982, 1052, 1012, 1000, 1077, 971, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1032, 1023, 983, 1063, 996, 1054, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1004, 1008, 1096, 1022, 1065, 1017, 1013, 13, 8, 0, 0, 0, 0, 0, 0, 1014, 1013, 994, 996, 1002, 1037, 1003, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1017, 1070, 999, 1016, 1043, 1025, 1003, 6, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8241, 8242, 8152, 8241, 8268, 8037, 8196] avg=8192.0000 max=8268 max_rank=5 min=8037 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1034, 1027, 1038, 1007, 1062, 1060, 1056, 972, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1022, 1010, 1032, 1009, 1068, 1059, 1003, 43, 0, 0, 0, 0, 0, 0, 0, 1021, 1037, 993, 1039, 996, 1025, 1010, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1048, 1038, 1058, 995, 1004, 1062, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1047, 1012, 1070, 1008, 983, 1041, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1057, 978, 1025, 1003, 1040, 1037, 983, 13, 8, 0, 0, 0, 0, 0, 0, 1078, 1010, 991, 982, 1032, 1029, 1038, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1031, 1008, 1003, 1035, 1034, 1008, 1021, 6, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8239, 8126, 8254, 8162, 8136, 8199, 8164] avg=8192.0000 max=8256 max_rank=0 min=8126 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1024, 1034, 1028, 1003, 1018, 1010, 1050, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1045, 1002, 1039, 1025, 1030, 1068, 1006, 46, 0, 0, 0, 0, 0, 0, 0, 1014, 999, 1050, 1003, 1003, 1071, 1019, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1086, 1007, 992, 969, 998, 1003, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1071, 991, 1026, 1008, 1046, 1057, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1038, 1064, 997, 993, 1017, 989, 1001, 13, 8, 0, 0, 0, 0, 0, 0, 1042, 1013, 1013, 1062, 1033, 975, 1071, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1060, 1015, 1037, 1000, 1017, 1042, 988, 6, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8237, 8158, 8102, 8249, 8195, 8217, 8155] avg=8192.0000 max=8249 max_rank=4 min=8102 min_rank=3 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[990, 1039, 1030, 1002, 1034, 1031, 995, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1014, 998, 1030, 985, 1032, 1031, 979, 44, 0, 0, 0, 0, 0, 0, 0, 946, 1027, 1007, 1006, 1027, 1001, 1066, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1001, 974, 1000, 1041, 1027, 1045, 974, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1016, 1005, 1038, 1073, 1037, 1041, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 980, 1066, 983, 1032, 983, 1015, 1035, 13, 8, 0, 0, 0, 0, 0, 0, 1052, 1042, 1023, 1034, 1006, 1059, 1033, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1021, 1110, 1017, 1075, 1012, 996, 1029, 6, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8116, 8149, 8121, 8229, 8150, 8276, 8312] avg=8192.0000 max=8312 max_rank=7 min=8116 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32967 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1010, 1072, 996, 1024, 1020, 1004, 1029, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1049, 1047, 952, 1026, 1014, 991, 1004, 42, 0, 0, 0, 0, 0, 0, 0, 1060, 1008, 971, 1004, 1061, 1001, 1082, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1054, 1058, 1019, 1029, 1039, 1033, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1070, 1017, 1001, 990, 978, 975, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1015, 1031, 1048, 987, 995, 1031, 1015, 13, 8, 0, 0, 0, 0, 0, 0, 1037, 1009, 1024, 1000, 1050, 1004, 1079, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1010, 1084, 1005, 1029, 980, 1003, 1016, 6, 4, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8175, 8219, 8289, 8072, 8154, 8252, 8218] avg=8192.0000 max=8289 max_rank=3 min=8072 min_rank=4 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32696 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1049, 1029, 1015, 1029, 997, 1025, 999, 999, 0, 0, 0, 0, 0, 0, 0, 0, 979, 989, 979, 1026, 1000, 1023, 987, 1044, 42, 0, 0, 0, 0, 0, 0, 0, 1059, 1033, 968, 1058, 1065, 1013, 966, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1018, 1074, 1075, 995, 1035, 1028, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1026, 1029, 1009, 1039, 1069, 1048, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1001, 1023, 1017, 1034, 992, 1009, 967, 12, 8, 0, 0, 0, 0, 0, 0, 992, 1041, 1077, 1015, 1025, 985, 1072, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1006, 1006, 1027, 1028, 1035, 1038, 1030, 6, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8069, 8173, 8294, 8260, 8139, 8233, 8226] avg=8192.0000 max=8294 max_rank=3 min=8069 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1011, 1025, 1036, 1011, 990, 1065, 1014, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1033, 1005, 1004, 1055, 1009, 1012, 1035, 49, 0, 0, 0, 0, 0, 0, 0, 1063, 1009, 1003, 1019, 1027, 1052, 1080, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1050, 1050, 1023, 1030, 1014, 978, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 985, 983, 1039, 1019, 1041, 1036, 1059, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1031, 1014, 1007, 1036, 1009, 1003, 1003, 16, 11, 7, 0, 0, 0, 0, 0, 994, 1036, 1054, 1043, 993, 1001, 991, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1026, 1012, 1039, 1014, 1021, 1047, 963, 9, 8, 9, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8233, 8332, 8178, 8172, 8159, 8094, 8187] avg=8192.0000 max=8332 max_rank=2 min=8094 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32612 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8064, 8192, 8320, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8064, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8448, 8192, 8320, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1024, 1028, 1059, 986, 1029, 1002, 1000, 1025, 1028, 1037, 997, 1011, 1069, 1004, 999, 994, 1032, 1016, 1045, 996, 1072, 1010, 989, 992, 1008, 1018, 989, 1094, 1010, 1005, 1002, 1014, 1046, 1073, 1003, 1026, 1040, 1059, 979, 1008, 985, 1016, 1013, 1053, 1092, 1008, 1041, 1058, 1044, 1006, 1044, 1075, 1048, 1042, 1048, 981, 1001, 1014, 1064, 1010, 1011, 1028, 1012, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[993, 1002, 1013, 997, 1048, 1066, 1039, 992, 1022, 1014, 1031, 1018, 1056, 1078, 1046, 1027, 1015, 1023, 1029, 998, 971, 1024, 1033, 1015, 1066, 1048, 1051, 1071, 982, 1041, 1034, 1011, 1069, 1003, 1037, 964, 1026, 1043, 1023, 1032, 1008, 1025, 1035, 1035, 979, 1023, 1066, 1052, 1057, 975, 970, 1002, 1040, 1022, 980, 1041, 1003, 1032, 1006, 1009, 1037, 1020, 1010, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1036, 1045, 1044, 1034, 1026, 1026, 1026, 1070, 1047, 999, 1068, 969, 1021, 1023, 985, 1071, 1028, 996, 1005, 1031, 988, 981, 1048, 1038, 1027, 1062, 1009, 1033, 1025, 1046, 1035, 1026, 1062, 975, 1052, 1038, 964, 1006, 1033, 1054, 1054, 991, 1050, 951, 953, 1002, 1076, 1036, 982, 1051, 991, 998, 1049, 995, 1039, 996, 1016, 1043, 1002, 1029, 1025, 1048, 1050, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1027, 1041, 1051, 1046, 1006, 1036, 1033, 1029, 1096, 1047, 973, 990, 1015, 977, 1086, 1042, 1035, 1016, 1039, 1035, 1045, 1038, 1031, 990, 1000, 1091, 1031, 978, 1037, 1019, 1043, 1011, 1016, 1068, 1014, 1028, 975, 1018, 1030, 968, 1027, 996, 1051, 1057, 1017, 1031, 996, 1007, 1027, 987, 1018, 1014, 1015, 1000, 990, 1081, 1023, 1001, 1046, 1035, 1002, 1010, 1019, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1075, 1065, 1068, 1077, 997, 922, 1066, 999, 992, 1044, 987, 1093, 996, 1032, 1012, 1035, 1048, 1044, 1016, 1018, 1057, 1013, 986, 1004, 997, 996, 1018, 1041, 1026, 1042, 1067, 1047, 1007, 1010, 1006, 985, 1092, 1010, 1075, 1007, 1088, 1005, 1014, 1014, 1023, 1012, 981, 1017, 1014, 1056, 1031, 964, 962, 1001, 974, 1035, 1105, 1040, 980, 1039, 1018, 1019, 1036, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[976, 1059, 1014, 1048, 1046, 997, 1029, 1065, 1043, 1036, 1046, 1064, 1003, 1054, 978, 958, 1036, 1014, 1076, 972, 1011, 1083, 1017, 1050, 1039, 1038, 994, 1009, 1017, 1045, 1060, 1038, 1081, 981, 1035, 1029, 988, 997, 953, 1025, 1073, 991, 975, 1054, 975, 1015, 1000, 1019, 1028, 1043, 995, 1005, 1018, 1085, 1031, 1002, 1057, 1052, 1000, 1053, 1015, 1040, 989, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1008, 1087, 996, 1022, 953, 1043, 999, 1000, 1040, 1042, 1056, 1025, 983, 1025, 1055, 1021, 1031, 1016, 991, 1049, 1036, 991, 1015, 998, 1023, 1055, 1006, 1058, 1075, 1022, 1039, 1040, 1052, 1040, 996, 1052, 1029, 1011, 1022, 1027, 1031, 1001, 1011, 1016, 1005, 1006, 1049, 1051, 989, 951, 979, 1039, 1079, 1082, 1022, 996, 1006, 1074, 1077, 990, 1050, 1009, 1023, 971] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[995, 1045, 1050, 991, 1003, 1093, 1031, 1028, 1025, 985, 1033, 1045, 1060, 1054, 1048, 1006, 1059, 1001, 1028, 996, 1028, 975, 1042, 1043, 1003, 986, 1049, 971, 999, 1029, 992, 995, 1089, 1036, 945, 1005, 991, 1051, 1018, 1034, 1063, 1026, 1063, 988, 1032, 1027, 1051, 1026, 1046, 1033, 1039, 1033, 1026, 1053, 990, 1014, 1040, 1012, 1023, 1019, 997, 1043, 998, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1027, 1018, 1051, 1046, 1006, 1036, 1033, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1047, 973, 990, 1015, 977, 1086, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1016, 1039, 1035, 1045, 1038, 1031, 990, 15, 9, 0, 0, 0, 0, 0, 0, 1000, 1066, 1031, 978, 1037, 1019, 1043, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1068, 1014, 1028, 975, 1018, 1030, 968, 8, 7, 0, 0, 0, 0, 0, 0, 1027, 996, 1051, 1057, 1017, 1031, 996, 1007, 6, 0, 0, 0, 0, 0, 0, 0, 1027, 987, 1018, 1014, 1015, 1000, 990, 1081, 25, 14, 0, 0, 0, 0, 0, 0, 1023, 988, 1046, 1035, 1002, 1010, 1019, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8203, 8253, 8185, 8132, 8188, 8171, 8158] avg=8192.0000 max=8253 max_rank=2 min=8132 min_rank=4 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1024, 1005, 1059, 986, 1029, 1002, 1000, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1037, 997, 1011, 1069, 1004, 999, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1016, 1045, 996, 1072, 1010, 989, 992, 14, 9, 0, 0, 0, 0, 0, 0, 1008, 994, 989, 1094, 1010, 1005, 1002, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1073, 1003, 1026, 1040, 1059, 979, 1008, 8, 7, 0, 0, 0, 0, 0, 0, 985, 1016, 1013, 1053, 1092, 1008, 1041, 1058, 6, 0, 0, 0, 0, 0, 0, 0, 1044, 1006, 1044, 1075, 1048, 1042, 1048, 981, 24, 14, 0, 0, 0, 0, 0, 0, 1001, 1001, 1064, 1010, 1011, 1028, 1012, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8117, 8175, 8116, 8249, 8272, 8326, 8151] avg=8192.0000 max=8326 max_rank=6 min=8116 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32998 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1075, 1042, 1068, 1077, 997, 922, 1066, 999, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1044, 987, 1093, 996, 1032, 1012, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1044, 1016, 1018, 1057, 1013, 986, 1004, 13, 9, 0, 0, 0, 0, 0, 0, 997, 973, 1018, 1041, 1026, 1042, 1067, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1010, 1006, 985, 1092, 1010, 1075, 1007, 7, 7, 0, 0, 0, 0, 0, 0, 1088, 1005, 1014, 1014, 1023, 1012, 981, 1017, 6, 0, 0, 0, 0, 0, 0, 0, 1014, 1056, 1031, 964, 962, 1001, 974, 1035, 23, 14, 0, 0, 0, 0, 0, 0, 1105, 1027, 980, 1039, 1018, 1019, 1036, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8171, 8208, 8211, 8206, 8160, 8074, 8260] avg=8192.0000 max=8260 max_rank=7 min=8074 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[993, 981, 1013, 997, 1048, 1066, 1039, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1014, 1031, 1018, 1056, 1078, 1046, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1023, 1029, 998, 971, 1024, 1033, 1015, 14, 8, 0, 0, 0, 0, 0, 0, 1066, 1024, 1051, 1071, 982, 1041, 1034, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1003, 1037, 964, 1026, 1043, 1023, 1032, 8, 7, 0, 0, 0, 0, 0, 0, 1008, 1025, 1035, 1035, 979, 1023, 1066, 1052, 6, 0, 0, 0, 0, 0, 0, 0, 1057, 975, 970, 1002, 1040, 1022, 980, 1041, 24, 13, 0, 0, 0, 0, 0, 0, 1003, 1019, 1006, 1009, 1037, 1020, 1010, 1058, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8270, 8130, 8280, 8212, 8229, 8124, 8162] avg=8192.0000 max=8280 max_rank=3 min=8124 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1008, 1063, 996, 1022, 953, 1043, 999, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1042, 1056, 1025, 983, 1025, 1055, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1016, 991, 1049, 1036, 991, 1015, 998, 14, 9, 0, 0, 0, 0, 0, 0, 1023, 1030, 1006, 1058, 1075, 1022, 1039, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1040, 996, 1052, 1029, 1011, 1022, 1027, 8, 7, 0, 0, 0, 0, 0, 0, 1031, 1001, 1011, 1016, 1005, 1006, 1049, 1051, 6, 0, 0, 0, 0, 0, 0, 0, 989, 951, 979, 1039, 1079, 1082, 1022, 996, 25, 15, 0, 0, 0, 0, 0, 0, 1006, 1061, 1077, 990, 1050, 1009, 1023, 971, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8084, 8225, 8150, 8293, 8244, 8176, 8177, 8187] avg=8192.0000 max=8293 max_rank=3 min=8084 min_rank=0 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1036, 1022, 1044, 1034, 1026, 1026, 1026, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 999, 1068, 969, 1021, 1023, 985, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 996, 1005, 1031, 988, 981, 1048, 1038, 14, 9, 0, 0, 0, 0, 0, 0, 1027, 1037, 1009, 1033, 1025, 1046, 1035, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 975, 1052, 1038, 964, 1006, 1033, 1054, 8, 7, 0, 0, 0, 0, 0, 0, 1054, 991, 1050, 951, 953, 1002, 1076, 1036, 6, 0, 0, 0, 0, 0, 0, 0, 982, 1051, 991, 998, 1049, 995, 1039, 996, 25, 14, 0, 0, 0, 0, 0, 0, 1016, 1030, 1002, 1029, 1025, 1048, 1050, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8161, 8138, 8238, 8199, 8119, 8140, 8257] avg=8192.0000 max=8284 max_rank=0 min=8119 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[976, 1036, 1014, 1048, 1046, 997, 1029, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1036, 1046, 1064, 1003, 1054, 978, 958, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1014, 1076, 972, 1011, 1083, 1017, 1050, 14, 9, 0, 0, 0, 0, 0, 0, 1039, 1014, 994, 1009, 1017, 1045, 1060, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 981, 1035, 1029, 988, 997, 953, 1025, 8, 7, 0, 0, 0, 0, 0, 0, 1073, 991, 975, 1054, 975, 1015, 1000, 1019, 6, 0, 0, 0, 0, 0, 0, 0, 1028, 1043, 995, 1005, 1018, 1085, 1031, 1002, 24, 14, 0, 0, 0, 0, 0, 0, 1057, 1039, 1000, 1053, 1015, 1040, 989, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8160, 8282, 8216, 8104, 8108, 8245, 8210] avg=8192.0000 max=8282 max_rank=2 min=8104 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[995, 1015, 1050, 991, 1003, 1093, 1031, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 998, 985, 1033, 1045, 1060, 1054, 1048, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1001, 1028, 996, 1028, 975, 1042, 1043, 16, 12, 0, 0, 0, 0, 0, 0, 1003, 959, 1049, 971, 999, 1029, 992, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 1036, 945, 1005, 991, 1051, 1018, 1034, 11, 10, 0, 0, 0, 0, 0, 0, 1063, 1026, 1063, 988, 1032, 1027, 1051, 1026, 8, 0, 0, 0, 0, 0, 0, 0, 1046, 1033, 1039, 1033, 1026, 1053, 990, 1014, 27, 18, 0, 0, 0, 0, 0, 0, 1040, 994, 1023, 1019, 997, 1043, 998, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8229, 8200, 7997, 8190, 8284, 8279, 8151] avg=8192.0000 max=8284 max_rank=5 min=7997 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8192, 8192, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8064, 8320, 8192, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[989, 1042, 1029, 994, 988, 991, 1023, 1058, 1030, 987, 1067, 1036, 1025, 1033, 1027, 1067, 1030, 1044, 1050, 998, 1032, 1002, 1008, 1010, 1044, 973, 1043, 1076, 1010, 1040, 1066, 1022, 989, 1005, 1060, 1008, 1011, 1043, 1007, 1058, 1043, 997, 1046, 946, 1056, 959, 1080, 1024, 963, 1018, 1045, 1018, 1060, 1012, 1053, 1045, 959, 1048, 1066, 1024, 1019, 995, 1033, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1066, 1004, 1039, 1068, 995, 1031, 1053, 971, 1009, 1067, 1038, 1001, 992, 1001, 1062, 1043, 1046, 1046, 998, 1034, 1045, 980, 968, 976, 1073, 991, 976, 1049, 952, 1017, 984, 1040, 1000, 1043, 1021, 1051, 1046, 1091, 1005, 1053, 1036, 999, 1051, 1021, 1013, 993, 1023, 995, 1000, 1077, 1022, 1000, 1079, 1065, 1039, 1007, 1023, 1025, 1016, 1019, 1027, 1026, 1022, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1032, 995, 987, 1061, 1020, 1006, 1003, 996, 964, 1001, 1042, 1037, 1058, 974, 1002, 1022, 1054, 1016, 1029, 1049, 988, 1069, 1097, 972, 1036, 1028, 1059, 1027, 1004, 1039, 1014, 1084, 1023, 1039, 985, 991, 1019, 1035, 1049, 1084, 1059, 994, 1004, 1026, 996, 1011, 958, 1043, 990, 1078, 1006, 1021, 1025, 1010, 1012, 1023, 994, 1016, 1043, 1039, 1065, 1060, 1049, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[998, 1056, 1015, 1025, 1023, 1035, 1028, 1047, 1039, 988, 976, 1021, 1060, 1048, 1040, 1050, 1043, 970, 1037, 1040, 1053, 1021, 1038, 988, 1032, 1038, 1026, 990, 1040, 985, 1057, 1032, 989, 1034, 998, 1046, 1068, 1022, 1005, 1043, 1047, 1034, 1003, 1024, 1057, 1091, 1015, 1037, 1022, 1052, 989, 1052, 1004, 990, 1006, 969, 1028, 1002, 1079, 1030, 933, 1035, 1015, 978] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1033, 1066, 1016, 1029, 1060, 1079, 1027, 971, 989, 1034, 1005, 1008, 1075, 1027, 1018, 1024, 1001, 1003, 1010, 1006, 1040, 1028, 1058, 1028, 981, 1001, 1023, 993, 1021, 990, 1033, 1023, 1025, 1032, 1056, 981, 1038, 1008, 1015, 1038, 1015, 1052, 1019, 1013, 1058, 967, 985, 1030, 1062, 1037, 1085, 1040, 1029, 983, 1069, 974, 1030, 1007, 1003, 1048, 1062, 1009, 1030, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1000, 1009, 1017, 1065, 1018, 1074, 1026, 998, 1002, 1045, 983, 1001, 1092, 1049, 1024, 1035, 1004, 1028, 1057, 1040, 1031, 1027, 1014, 1023, 1063, 969, 1050, 1035, 984, 1020, 971, 1028, 998, 1035, 995, 999, 1015, 979, 1050, 997, 1051, 998, 1048, 1024, 1016, 1058, 1019, 1070, 1019, 991, 1024, 1063, 1067, 970, 1004, 1050, 1023, 1032, 1042, 1021, 1021, 1024, 1049, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1022, 1059, 962, 1009, 987, 1031, 1040, 1014, 1063, 1042, 1036, 1027, 1017, 1044, 1006, 1094, 1004, 1029, 1017, 1005, 1014, 1015, 1041, 1036, 1022, 1027, 1059, 1011, 1033, 1073, 1018, 1011, 1023, 1020, 1004, 1032, 1001, 957, 1026, 1029, 1060, 1022, 1021, 1027, 1013, 976, 978, 1067, 1012, 1026, 1027, 1065, 1019, 1018, 1000, 1067, 1033, 997, 1003, 1031, 1020, 1066, 1042, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1003, 1039, 1051, 1005, 1030, 1035, 1005, 1056, 1034, 1039, 996, 1036, 1033, 1040, 1039, 977, 1055, 994, 1017, 1057, 1047, 1042, 1035, 1028, 1032, 1003, 1036, 1034, 1003, 1018, 1016, 1014, 1005, 1070, 1019, 1001, 1094, 1039, 1010, 1037, 1023, 1016, 983, 1037, 998, 998, 1040, 1020, 1040, 956, 1017, 982, 976, 1030, 1058, 1055, 983, 1032, 983, 1004, 1010, 1044, 1034, 1093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[989, 1042, 1029, 994, 988, 991, 1023, 1058, 2, 1, 0, 0, 0, 0, 0, 0, 1030, 987, 1067, 1036, 1001, 1033, 1027, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1044, 1050, 998, 1032, 1002, 1005, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 973, 1043, 1076, 1010, 1040, 1066, 1022, 6, 3, 1, 0, 0, 0, 0, 0, 989, 1005, 1060, 1008, 1011, 1043, 1007, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 997, 1046, 946, 1056, 959, 1080, 1024, 18, 0, 0, 0, 0, 0, 0, 0, 963, 1018, 1045, 1018, 1057, 1012, 1053, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 959, 1048, 1066, 1024, 1019, 995, 1033, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8248, 8171, 8284, 8180, 8169, 8211, 8156] avg=8192.0000 max=8284 max_rank=3 min=8117 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1066, 1004, 1039, 1068, 995, 1031, 1053, 971, 2, 1, 0, 0, 0, 0, 0, 0, 1009, 1067, 1038, 1001, 968, 1001, 1062, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1046, 998, 1034, 1045, 980, 965, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 991, 976, 1049, 952, 1017, 984, 1040, 6, 4, 1, 0, 0, 0, 0, 0, 1000, 1043, 1021, 1051, 1046, 1091, 1005, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 999, 1051, 1021, 1013, 993, 1023, 995, 18, 0, 0, 0, 0, 0, 0, 0, 1000, 1077, 1022, 1000, 1075, 1065, 1039, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1025, 1016, 1019, 1027, 1026, 1022, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8189, 8090, 8093, 8309, 8149, 8285, 8191] avg=8192.0000 max=8309 max_rank=4 min=8090 min_rank=2 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[998, 1056, 1015, 1025, 1023, 1035, 1028, 1047, 2, 1, 0, 0, 0, 0, 0, 0, 1039, 988, 976, 1021, 1035, 1048, 1040, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 970, 1037, 1040, 1053, 1021, 1035, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1038, 1026, 990, 1040, 985, 1057, 1032, 6, 3, 1, 0, 0, 0, 0, 0, 989, 1034, 998, 1046, 1068, 1022, 1005, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1034, 1003, 1024, 1057, 1091, 1015, 1037, 19, 0, 0, 0, 0, 0, 0, 0, 1022, 1052, 989, 1052, 1001, 990, 1006, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1002, 1079, 1030, 933, 1035, 1015, 978, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8197, 8187, 8210, 8204, 8327, 8081, 8100] avg=8192.0000 max=8327 max_rank=5 min=8081 min_rank=6 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1000, 1009, 1017, 1065, 1018, 1074, 1026, 998, 2, 1, 0, 0, 0, 0, 0, 0, 1002, 1045, 983, 1001, 1066, 1049, 1024, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1028, 1057, 1040, 1031, 1027, 1011, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 969, 1050, 1035, 984, 1020, 971, 1028, 6, 4, 1, 0, 0, 0, 0, 0, 998, 1035, 995, 999, 1015, 979, 1050, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 998, 1048, 1024, 1016, 1058, 1019, 1070, 20, 0, 0, 0, 0, 0, 0, 0, 1019, 991, 1024, 1063, 1063, 970, 1004, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1032, 1042, 1021, 1021, 1024, 1049, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8205, 8221, 8131, 8067, 8304, 8184, 8214] avg=8192.0000 max=8304 max_rank=5 min=8067 min_rank=4 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1032, 995, 987, 1061, 1020, 1006, 1003, 996, 2, 1, 0, 0, 0, 0, 0, 0, 964, 1001, 1042, 1037, 1033, 974, 1002, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1016, 1029, 1049, 988, 1069, 1094, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1028, 1059, 1027, 1004, 1039, 1014, 1084, 6, 3, 1, 0, 0, 0, 0, 0, 1023, 1039, 985, 991, 1019, 1035, 1049, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 994, 1004, 1026, 996, 1011, 958, 1043, 19, 0, 0, 0, 0, 0, 0, 0, 990, 1078, 1006, 1021, 1022, 1010, 1012, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1016, 1043, 1039, 1065, 1060, 1049, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8075, 8271, 8301, 8224, 8110, 8162, 8290] avg=8192.0000 max=8301 max_rank=3 min=8075 min_rank=1 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1033, 1066, 1016, 1029, 1060, 1079, 1027, 971, 2, 1, 0, 0, 0, 0, 0, 0, 989, 1034, 1005, 1008, 1050, 1027, 1018, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1003, 1010, 1006, 1040, 1028, 1055, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1001, 1023, 993, 1021, 990, 1033, 1023, 6, 3, 1, 0, 0, 0, 0, 0, 1025, 1032, 1056, 981, 1038, 1008, 1015, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1052, 1019, 1013, 1058, 967, 985, 1030, 19, 0, 0, 0, 0, 0, 0, 0, 1062, 1037, 1085, 1040, 1026, 983, 1069, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1007, 1003, 1048, 1062, 1009, 1030, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 8155, 8171, 8075, 8192, 8158, 8276, 8225] avg=8192.0000 max=8284 max_rank=0 min=8075 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1022, 1059, 962, 1009, 987, 1031, 1040, 1014, 2, 1, 0, 0, 0, 0, 0, 0, 1063, 1042, 1036, 1027, 993, 1044, 1006, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1029, 1017, 1005, 1014, 1015, 1038, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1027, 1059, 1011, 1033, 1073, 1018, 1011, 6, 3, 1, 0, 0, 0, 0, 0, 1023, 1020, 1004, 1032, 1001, 957, 1026, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1022, 1021, 1027, 1013, 976, 978, 1067, 18, 0, 0, 0, 0, 0, 0, 0, 1012, 1026, 1027, 1065, 1016, 1018, 1000, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 997, 1003, 1031, 1020, 1066, 1042, 986, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8305, 8158, 8264, 8091, 8182, 8231, 8178] avg=8192.0000 max=8305 max_rank=1 min=8091 min_rank=4 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1003, 1039, 1051, 1005, 1030, 1035, 1005, 1056, 4, 6, 1, 0, 0, 0, 0, 0, 1034, 1039, 996, 1036, 1001, 1040, 1039, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 994, 1017, 1057, 1047, 1042, 1027, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1003, 1036, 1034, 1003, 1018, 1016, 1014, 10, 8, 4, 0, 0, 0, 0, 0, 1005, 1070, 1019, 1001, 1094, 1039, 1010, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1016, 983, 1037, 998, 998, 1040, 1020, 22, 0, 0, 0, 0, 0, 0, 0, 1040, 956, 1017, 982, 968, 1030, 1058, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1032, 983, 1004, 1010, 1044, 1033, 1093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8162, 8267, 8178, 8269, 8137, 8106, 8182] avg=8192.0000 max=8269 max_rank=4 min=8106 min_rank=6 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8320, 8064, 8192, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8192, 8320, 8192, 8192, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8192, 8192, 8320, 8064, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[977, 986, 1004, 1072, 1004, 1051, 1028, 1001, 1015, 1033, 1016, 1011, 1014, 979, 1012, 996, 1000, 1048, 1025, 995, 1044, 1010, 993, 1035, 1022, 1031, 1070, 1048, 1035, 1089, 1052, 986, 1018, 1026, 1018, 1039, 1044, 1039, 998, 1012, 1019, 1039, 1032, 1044, 1021, 1051, 1020, 1015, 996, 1056, 993, 988, 999, 1009, 1059, 1027, 1071, 1017, 1066, 1056, 983, 1065, 1018, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1051, 1036, 1051, 1066, 1013, 1036, 1033, 1080, 1019, 1011, 1011, 1032, 979, 1008, 1003, 1015, 1036, 1006, 1039, 1032, 1020, 1042, 999, 992, 1041, 1015, 1019, 1019, 1028, 1001, 1067, 1009, 1051, 1002, 1081, 992, 1040, 1029, 1007, 981, 1009, 1020, 1025, 1014, 1049, 1018, 1033, 1017, 1070, 1037, 985, 1045, 1013, 1024, 984, 998, 1075, 1014, 1057, 974, 1031, 999, 1047, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1023, 996, 1058, 1043, 1039, 998, 1077, 989, 1038, 999, 1005, 1041, 1006, 1032, 1016, 999, 992, 1076, 1055, 1074, 1051, 1036, 1018, 1062, 1044, 987, 1043, 979, 1022, 1031, 980, 1052, 1023, 1009, 1014, 983, 971, 1023, 1040, 1039, 1021, 1037, 1020, 1025, 1067, 1067, 1016, 1060, 1036, 1016, 956, 1068, 972, 1058, 990, 1031, 983, 1058, 1068, 1007, 1025, 993, 997, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[971, 1005, 1014, 953, 1014, 1021, 1002, 1026, 1040, 987, 978, 1053, 1036, 1054, 1030, 1039, 1052, 991, 1022, 1064, 1054, 968, 1084, 1039, 1105, 1023, 1010, 987, 1014, 1037, 965, 1010, 992, 1004, 1003, 1018, 1003, 1011, 1071, 1084, 1031, 1032, 1043, 1054, 1043, 1048, 1006, 1036, 1024, 1012, 1024, 1090, 1035, 997, 1036, 1019, 1105, 1030, 1024, 1028, 1020, 1008, 966, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1068, 1018, 1014, 1013, 1023, 1034, 1022, 1038, 1051, 1043, 1034, 1054, 1001, 1020, 1072, 1023, 1044, 1026, 1051, 1007, 1006, 1076, 1032, 1033, 998, 1017, 1005, 1043, 1028, 1059, 1013, 998, 1055, 1009, 977, 1025, 991, 999, 999, 1036, 1063, 1031, 1001, 1010, 1030, 1007, 1054, 1023, 1015, 995, 1006, 1063, 1032, 1043, 1009, 985, 1008, 995, 1000, 1047, 976, 1036, 1020, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1063, 1006, 1010, 982, 971, 1020, 988, 1032, 1023, 994, 1023, 1053, 1056, 1020, 1011, 1017, 1010, 1028, 1025, 1052, 965, 995, 1049, 1007, 1011, 1050, 1028, 1054, 1018, 1044, 1051, 1015, 1050, 1048, 1004, 1057, 1038, 985, 1046, 1011, 1003, 1002, 1018, 1007, 1019, 1077, 1056, 1045, 1033, 1024, 1010, 1063, 996, 1098, 1054, 1025, 1052, 968, 1008, 999, 1050, 1015, 963, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1019, 1011, 1015, 1021, 1007, 1003, 1004, 1035, 1011, 1028, 1056, 1059, 1038, 1028, 1049, 1049, 1086, 1035, 976, 1027, 1042, 1072, 1006, 1017, 1036, 982, 1034, 1049, 1029, 993, 968, 1032, 1019, 1041, 1020, 989, 1069, 1008, 963, 1048, 1020, 1037, 1033, 1059, 992, 1027, 1033, 1023, 1013, 1046, 1029, 1028, 1024, 1014, 1006, 1037, 1011, 1010, 985, 1062, 1037, 1037, 999, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1069, 1028, 1013, 1026, 1045, 1049, 999, 1062, 1000, 986, 1008, 989, 968, 966, 1025, 1007, 1032, 1086, 1033, 1011, 1056, 1042, 1006, 1009, 978, 999, 1041, 1015, 1024, 1001, 1057, 990, 987, 1057, 1024, 1010, 993, 1032, 1050, 1019, 997, 1034, 1001, 1001, 1035, 979, 1050, 1030, 997, 1058, 1062, 1077, 1065, 1031, 1074, 972, 1028, 1046, 1092, 1024, 1002, 1046, 1001, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[977, 986, 1004, 1072, 1004, 1051, 1028, 1001, 13, 0, 0, 0, 0, 0, 0, 0, 1015, 1033, 1016, 1011, 1014, 979, 1012, 996, 33, 0, 0, 0, 0, 0, 0, 0, 1000, 1004, 1025, 995, 1044, 1010, 993, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1031, 1070, 1048, 1035, 1089, 1052, 986, 6, 0, 0, 0, 0, 0, 0, 0, 1018, 1026, 1018, 1039, 1044, 1039, 998, 1012, 15, 11, 0, 0, 0, 0, 0, 0, 1019, 1039, 1032, 1044, 1021, 1015, 1020, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1056, 993, 978, 999, 1009, 1059, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1017, 1066, 1056, 983, 1065, 1018, 1016, 8, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8109, 8106, 8339, 8220, 8205, 8117, 8304] avg=8192.0000 max=8339 max_rank=3 min=8106 min_rank=2 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1051, 1036, 1051, 1066, 1013, 1036, 1033, 1080, 13, 0, 0, 0, 0, 0, 0, 0, 1019, 1011, 1011, 1032, 979, 1008, 1003, 1015, 32, 0, 0, 0, 0, 0, 0, 0, 1036, 963, 1039, 1032, 1020, 1042, 999, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1015, 1019, 1019, 1028, 1001, 1067, 1009, 6, 0, 0, 0, 0, 0, 0, 0, 1051, 1002, 1081, 992, 1040, 1029, 1007, 981, 14, 11, 0, 0, 0, 0, 0, 0, 1009, 1020, 1025, 1014, 1049, 984, 1033, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1037, 985, 1035, 1013, 1024, 984, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1014, 1057, 974, 1031, 999, 1047, 1006, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8379, 8110, 8123, 8205, 8208, 8151, 8146, 8214] avg=8192.0000 max=8379 max_rank=0 min=8110 min_rank=1 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[971, 1005, 1014, 953, 1014, 1021, 1002, 1026, 13, 0, 0, 0, 0, 0, 0, 0, 1040, 987, 978, 1053, 1036, 1054, 1030, 1039, 31, 0, 0, 0, 0, 0, 0, 0, 1052, 949, 1022, 1064, 1054, 968, 1084, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 1023, 1010, 987, 1014, 1037, 965, 1010, 7, 0, 0, 0, 0, 0, 0, 0, 992, 1004, 1003, 1018, 1003, 1011, 1071, 1084, 15, 11, 0, 0, 0, 0, 0, 0, 1031, 1032, 1043, 1054, 1043, 1012, 1006, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1012, 1024, 1078, 1035, 997, 1036, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 1030, 1024, 1028, 1020, 1008, 966, 991, 8, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8019, 8248, 8232, 8158, 8212, 8257, 8225, 8185] avg=8192.0000 max=8257 max_rank=5 min=8019 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32879 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1063, 1006, 1010, 982, 971, 1020, 988, 1032, 14, 0, 0, 0, 0, 0, 0, 0, 1023, 994, 1023, 1053, 1056, 1020, 1011, 1017, 33, 0, 0, 0, 0, 0, 0, 0, 1010, 984, 1025, 1052, 965, 995, 1049, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1050, 1028, 1054, 1018, 1044, 1051, 1015, 6, 0, 0, 0, 0, 0, 0, 0, 1050, 1048, 1004, 1057, 1038, 985, 1046, 1011, 15, 11, 0, 0, 0, 0, 0, 0, 1003, 1002, 1018, 1007, 1019, 1040, 1056, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1024, 1010, 1052, 996, 1098, 1054, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 968, 1008, 999, 1050, 1015, 963, 1041, 8, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 8230, 8087, 8277, 8265, 8190, 8292, 8109] avg=8192.0000 max=8292 max_rank=6 min=8086 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1068, 1018, 1014, 1013, 1023, 1034, 1022, 1038, 13, 0, 0, 0, 0, 0, 0, 0, 1051, 1043, 1034, 1054, 1001, 1020, 1072, 1023, 33, 0, 0, 0, 0, 0, 0, 0, 1044, 982, 1051, 1007, 1006, 1076, 1032, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1017, 1005, 1043, 1028, 1059, 1013, 998, 6, 0, 0, 0, 0, 0, 0, 0, 1055, 1009, 977, 1025, 991, 999, 999, 1036, 14, 11, 0, 0, 0, 0, 0, 0, 1063, 1031, 1001, 1010, 1030, 973, 1054, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 995, 1006, 1052, 1032, 1043, 1009, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 995, 1000, 1047, 976, 1036, 1020, 1032, 7, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8331, 8231, 8167, 8116, 8185, 8137, 8126] avg=8192.0000 max=8331 max_rank=1 min=8116 min_rank=4 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32564 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1019, 1011, 1015, 1021, 1007, 1003, 1004, 1035, 13, 0, 0, 0, 0, 0, 0, 0, 1011, 1028, 1056, 1059, 1038, 1028, 1049, 1049, 33, 0, 0, 0, 0, 0, 0, 0, 1086, 991, 976, 1027, 1042, 1072, 1006, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 982, 1034, 1049, 1029, 993, 968, 1032, 6, 0, 0, 0, 0, 0, 0, 0, 1019, 1041, 1020, 989, 1069, 1008, 963, 1048, 14, 11, 0, 0, 0, 0, 0, 0, 1020, 1037, 1033, 1059, 992, 993, 1033, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1046, 1029, 1018, 1024, 1014, 1006, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1010, 985, 1062, 1037, 1037, 999, 1000, 7, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8128, 8351, 8217, 8129, 8182, 8190, 8187, 8152] avg=8192.0000 max=8351 max_rank=1 min=8128 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1023, 996, 1058, 1043, 1039, 998, 1077, 989, 14, 0, 0, 0, 0, 0, 0, 0, 1038, 999, 1005, 1041, 1006, 1032, 1016, 999, 34, 0, 0, 0, 0, 0, 0, 0, 992, 1030, 1055, 1074, 1051, 1036, 1018, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 987, 1043, 979, 1022, 1031, 980, 1052, 6, 0, 0, 0, 0, 0, 0, 0, 1023, 1009, 1014, 983, 971, 1023, 1040, 1039, 15, 12, 0, 0, 0, 0, 0, 0, 1021, 1037, 1020, 1025, 1067, 1030, 1016, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1016, 956, 1057, 972, 1058, 990, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1058, 1068, 1007, 1025, 993, 997, 1002, 8, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8170, 8318, 8144, 8129, 8276, 8116, 8146] avg=8192.0000 max=8318 max_rank=2 min=8116 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1069, 1028, 1013, 1026, 1045, 1049, 999, 1062, 17, 0, 0, 0, 0, 0, 0, 0, 1000, 986, 1008, 989, 968, 966, 1025, 1007, 38, 0, 0, 0, 0, 0, 0, 0, 1032, 1033, 1033, 1011, 1056, 1042, 1006, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 978, 999, 1041, 1015, 1024, 1001, 1057, 990, 12, 0, 0, 0, 0, 0, 0, 0, 987, 1057, 1024, 1010, 993, 1032, 1050, 1019, 17, 15, 0, 0, 0, 0, 0, 0, 997, 1034, 1001, 1001, 1035, 934, 1050, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1058, 1062, 1057, 1065, 1031, 1074, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1046, 1092, 1024, 1002, 1046, 1001, 1042, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 7987, 8222, 8117, 8204, 8082, 8316, 8300] avg=8192.0000 max=8316 max_rank=6 min=7987 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32902 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 7936, 8320, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8448, 8192, 8320, 8320, 8192, 8320, 8064, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8448, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1027, 988, 995, 1014, 1056, 1031, 1037, 1095, 1059, 1063, 994, 969, 1012, 1015, 1091, 978, 1061, 997, 998, 1042, 1029, 998, 1022, 1023, 1002, 1029, 993, 1064, 1029, 986, 1002, 1037, 1017, 1075, 1013, 1009, 1056, 1020, 989, 998, 1000, 1019, 1066, 1074, 1022, 1070, 1030, 1014, 981, 1055, 1045, 1017, 1012, 1053, 995, 1015, 998, 1029, 1008, 1009, 1059, 1020, 1039, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1001, 1030, 1011, 1020, 990, 1026, 1000, 986, 998, 1057, 1024, 1018, 1032, 1021, 1030, 1041, 1007, 1039, 1078, 1024, 1057, 1017, 1039, 996, 1002, 1025, 1022, 1030, 996, 959, 1040, 1032, 1066, 1073, 992, 1029, 1030, 1007, 1036, 981, 1027, 1053, 1035, 1060, 1081, 984, 998, 1066, 1037, 1030, 1047, 1011, 1027, 1013, 1016, 1026, 1038, 1008, 1000, 1028, 1011, 999, 1039, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1021, 1001, 1040, 968, 1016, 1038, 1048, 1027, 1034, 998, 999, 1026, 1006, 1060, 1057, 1027, 996, 1007, 992, 993, 1032, 1030, 1010, 1015, 1007, 1011, 944, 1016, 1048, 1081, 1052, 1026, 1041, 1074, 1096, 1006, 1029, 996, 973, 1027, 1038, 1037, 1057, 1040, 1028, 1039, 1014, 1021, 1036, 1021, 1074, 1027, 971, 961, 1024, 1017, 1021, 1000, 1074, 1041, 1033, 1024, 1064, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1001, 1026, 1010, 1060, 1048, 1029, 991, 990, 1033, 1049, 1025, 1023, 1034, 1015, 1043, 1023, 1012, 1060, 1028, 1050, 1016, 991, 1023, 982, 1001, 1010, 1013, 1050, 1037, 1027, 1000, 1021, 1075, 994, 1018, 1055, 1054, 1033, 1037, 1027, 1007, 987, 1058, 1024, 1013, 997, 1035, 1059, 1028, 978, 1060, 971, 1051, 1011, 1003, 1078, 1003, 1027, 1012, 1038, 1004, 1068, 1034, 976] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[994, 990, 1018, 994, 1030, 1039, 1053, 1056, 1004, 985, 1078, 1039, 1020, 1059, 989, 1022, 997, 1009, 1035, 1032, 1028, 1035, 1051, 1017, 1022, 1036, 1057, 995, 999, 993, 998, 1058, 989, 1035, 1029, 1051, 995, 1042, 1027, 1022, 1045, 978, 989, 1030, 1014, 1023, 1106, 1042, 1053, 1033, 1033, 973, 1016, 1018, 1027, 1045, 1006, 998, 1042, 1037, 1054, 985, 1026, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1003, 1045, 1016, 1034, 996, 1022, 1078, 1017, 976, 1064, 980, 1029, 1027, 1018, 1021, 1036, 1010, 1011, 1060, 1039, 1028, 991, 1037, 995, 1003, 1021, 1022, 1007, 998, 989, 1024, 1032, 1054, 1016, 1027, 1037, 1018, 1022, 1047, 1011, 984, 987, 1059, 1051, 1009, 1028, 1022, 1009, 1031, 1059, 1010, 1038, 1057, 1030, 1003, 1003, 1010, 1022, 1078, 1030, 1028, 1043, 1070, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1057, 1044, 1002, 998, 1069, 1036, 1048, 1032, 1000, 1018, 1004, 1010, 1028, 1057, 980, 987, 1036, 933, 1089, 1073, 990, 1053, 1054, 1080, 976, 1022, 1063, 1052, 1015, 1009, 1011, 1055, 1041, 992, 1011, 990, 1019, 1039, 1034, 991, 1010, 1005, 1011, 1053, 1043, 1007, 1010, 994, 1020, 1099, 1054, 1026, 1042, 1024, 1020, 1034, 1005, 1041, 974, 1014, 996, 1033, 998, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[993, 1106, 990, 1057, 993, 990, 1017, 1032, 1042, 1029, 1043, 1019, 1024, 978, 1052, 1026, 1026, 1036, 1047, 938, 1034, 1016, 1012, 1112, 1068, 1033, 1003, 1037, 1001, 1068, 1009, 1055, 1010, 1038, 1011, 1043, 1030, 1034, 984, 1049, 1037, 1042, 1045, 962, 1030, 970, 1062, 996, 1027, 1008, 973, 1023, 999, 1076, 1007, 989, 1033, 1041, 1048, 1014, 1056, 1048, 978, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1001, 1030, 1011, 1020, 990, 1026, 1000, 986, 8, 0, 0, 0, 0, 0, 0, 0, 998, 1057, 1024, 1018, 1032, 1021, 1030, 1041, 2, 2, 0, 0, 0, 0, 0, 0, 1007, 1039, 1075, 1024, 1057, 1017, 1039, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1025, 1022, 1030, 996, 959, 1040, 1032, 21, 5, 0, 0, 0, 0, 0, 0, 1066, 1058, 992, 1029, 1030, 1007, 1036, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1053, 1014, 1060, 1081, 984, 998, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1030, 1047, 1011, 1027, 1013, 1016, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1008, 1000, 1028, 1011, 999, 1039, 1040, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8072, 8225, 8254, 8132, 8199, 8283, 8207, 8164] avg=8192.0000 max=8283 max_rank=5 min=8072 min_rank=0 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1001, 1026, 1010, 1060, 1048, 1029, 991, 990, 7, 0, 0, 0, 0, 0, 0, 0, 1033, 1049, 1025, 1023, 1034, 1015, 1043, 1023, 2, 2, 0, 0, 0, 0, 0, 0, 1012, 1060, 1025, 1050, 1016, 991, 1023, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1010, 1013, 1050, 1037, 1027, 1000, 1021, 21, 5, 0, 0, 0, 0, 0, 0, 1075, 980, 1018, 1055, 1054, 1033, 1037, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 987, 1037, 1024, 1013, 997, 1035, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 978, 1060, 971, 1051, 1011, 1003, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1027, 1012, 1038, 1004, 1068, 1034, 976, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8249, 8159, 8185, 8279, 8159, 8180, 8163] avg=8192.0000 max=8279 max_rank=4 min=8159 min_rank=2 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1027, 988, 995, 1014, 1056, 1031, 1037, 1095, 8, 0, 0, 0, 0, 0, 0, 0, 1059, 1063, 994, 969, 1012, 1015, 1091, 978, 2, 2, 0, 0, 0, 0, 0, 0, 1061, 997, 995, 1042, 1029, 998, 1022, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1029, 993, 1064, 1029, 986, 1002, 1037, 21, 5, 0, 0, 0, 0, 0, 0, 1017, 1060, 1013, 1009, 1056, 1020, 989, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1019, 1045, 1074, 1022, 1070, 1030, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1055, 1045, 1017, 1012, 1053, 995, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1029, 1008, 1009, 1059, 1020, 1039, 993, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8185, 8167, 8168, 8162, 8274, 8173, 8156] avg=8192.0000 max=8274 max_rank=5 min=8156 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1021, 1001, 1040, 968, 1016, 1038, 1048, 1027, 8, 0, 0, 0, 0, 0, 0, 0, 1034, 998, 999, 1026, 1006, 1060, 1057, 1027, 2, 2, 0, 0, 0, 0, 0, 0, 996, 1007, 989, 993, 1032, 1030, 1010, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1011, 944, 1016, 1048, 1081, 1052, 1026, 21, 5, 0, 0, 0, 0, 0, 0, 1041, 1059, 1096, 1006, 1029, 996, 973, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1037, 1036, 1040, 1028, 1039, 1014, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1021, 1074, 1027, 971, 961, 1024, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1000, 1074, 1041, 1033, 1024, 1064, 1006, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8211, 8072, 8211, 8227, 8253, 8131, 8264] avg=8192.0000 max=8264 max_rank=7 min=8072 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[994, 990, 1018, 994, 1030, 1039, 1053, 1056, 8, 0, 0, 0, 0, 0, 0, 0, 1004, 985, 1078, 1039, 1020, 1059, 989, 1022, 2, 2, 0, 0, 0, 0, 0, 0, 997, 1009, 1032, 1032, 1028, 1035, 1051, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1036, 1057, 995, 999, 993, 998, 1058, 20, 5, 0, 0, 0, 0, 0, 0, 989, 1020, 1029, 1051, 995, 1042, 1027, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 978, 969, 1030, 1014, 1023, 1106, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1033, 1033, 973, 1016, 1018, 1027, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 998, 1042, 1037, 1054, 985, 1026, 1041, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8200, 8201, 8183, 8175, 8207, 8198, 8190] avg=8192.0000 max=8207 max_rank=5 min=8175 min_rank=4 imbalance_ratio=1.001831 max_load_reduction=57329 max_load_reduction_pct=0.874771 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1003, 1045, 1016, 1034, 996, 1022, 1078, 1017, 8, 0, 0, 0, 0, 0, 0, 0, 976, 1064, 980, 1029, 1027, 1018, 1021, 1036, 2, 2, 0, 0, 0, 0, 0, 0, 1010, 1011, 1057, 1039, 1028, 991, 1037, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1021, 1022, 1007, 998, 989, 1024, 1032, 21, 5, 0, 0, 0, 0, 0, 0, 1054, 1001, 1027, 1037, 1018, 1022, 1047, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 984, 987, 1038, 1051, 1009, 1028, 1022, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1059, 1010, 1038, 1057, 1030, 1003, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1022, 1078, 1030, 1028, 1043, 1070, 1014, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8155, 8168, 8122, 8217, 8128, 8231, 8296] avg=8192.0000 max=8296 max_rank=7 min=8122 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1057, 1044, 1002, 998, 1069, 1036, 1048, 1032, 7, 0, 0, 0, 0, 0, 0, 0, 1000, 1018, 1004, 1010, 1028, 1057, 980, 987, 2, 2, 0, 0, 0, 0, 0, 0, 1036, 933, 1086, 1073, 990, 1053, 1054, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1022, 1063, 1052, 1015, 1009, 1011, 1055, 20, 5, 0, 0, 0, 0, 0, 0, 1041, 978, 1011, 990, 1019, 1039, 1034, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1005, 991, 1053, 1043, 1007, 1010, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1099, 1054, 1026, 1042, 1024, 1020, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1041, 974, 1014, 996, 1033, 998, 1025, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8088, 8305, 8228, 8103, 8113, 8319, 8087] avg=8192.0000 max=8319 max_rank=6 min=8087 min_rank=7 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[993, 1106, 990, 1057, 993, 990, 1017, 1032, 12, 0, 0, 0, 0, 0, 0, 0, 1042, 1029, 1043, 1019, 1024, 978, 1052, 1026, 5, 5, 0, 0, 0, 0, 0, 0, 1026, 1036, 1036, 938, 1034, 1016, 1012, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1033, 1003, 1037, 1001, 1068, 1009, 1055, 25, 8, 0, 0, 0, 0, 0, 0, 1010, 1013, 1011, 1043, 1030, 1034, 984, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1042, 1020, 962, 1030, 970, 1062, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1008, 968, 1023, 999, 1076, 1007, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1041, 1048, 1014, 1056, 1048, 978, 987, 6, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8223, 8210, 8307, 8174, 8119, 8097, 8216] avg=8192.0000 max=8307 max_rank=3 min=8097 min_rank=6 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32606 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1005, 1037, 975, 1032, 1065, 1018, 1050, 957, 1077, 1016, 1031, 1050, 975, 1087, 1025, 999, 971, 1055, 1019, 1048, 1018, 1017, 1028, 1019, 1011, 1041, 1004, 1003, 1006, 1004, 1030, 1092, 1077, 1009, 1016, 1009, 1029, 960, 1055, 1033, 935, 1045, 1038, 1083, 1034, 1025, 1030, 1052, 1049, 1070, 1005, 1007, 1012, 987, 990, 1039, 1009, 1034, 1053, 972, 1014, 1030, 1057, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[986, 1018, 976, 1044, 1065, 1046, 1053, 1019, 1036, 1005, 995, 1017, 1064, 1050, 1052, 1039, 999, 1024, 1052, 963, 985, 1062, 1008, 1031, 1039, 1016, 995, 1037, 1002, 1023, 1070, 1065, 1004, 1028, 1018, 1025, 997, 1006, 999, 1021, 1065, 1057, 1071, 1013, 1001, 1056, 1053, 1022, 1028, 1044, 1000, 1000, 1023, 1015, 965, 977, 1009, 1007, 1048, 1039, 983, 1079, 1012, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[999, 981, 994, 1056, 977, 999, 1023, 978, 1091, 1024, 1013, 1026, 1068, 1010, 1063, 1008, 1003, 1025, 973, 1055, 1066, 1029, 1017, 1054, 1022, 1050, 1044, 992, 996, 1060, 990, 1008, 952, 1000, 1076, 1061, 1064, 1038, 1011, 1037, 976, 1053, 1054, 1043, 1022, 1032, 1015, 1077, 1048, 1022, 1004, 1010, 989, 1027, 1074, 984, 1054, 991, 1017, 1013, 1071, 951, 1028, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[987, 1069, 1008, 1019, 1023, 1045, 1008, 1053, 990, 1027, 1064, 1001, 987, 1036, 1043, 1051, 994, 1037, 1030, 1036, 1008, 992, 1009, 1044, 1014, 999, 1054, 1014, 998, 986, 1023, 1043, 1048, 1011, 1099, 1018, 984, 1020, 1015, 1011, 1028, 1100, 985, 1009, 1073, 1061, 1006, 1018, 998, 1014, 994, 996, 1037, 1016, 983, 1046, 1046, 1033, 1070, 1060, 1025, 1035, 1031, 974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1029, 1085, 1000, 1052, 1030, 993, 967, 987, 1020, 1068, 1035, 1062, 957, 1040, 1052, 1058, 1013, 1003, 1019, 1023, 1022, 990, 982, 992, 1003, 1009, 983, 1049, 1077, 968, 991, 1035, 1007, 1052, 1041, 1054, 1024, 1060, 1047, 1040, 990, 1011, 1064, 1067, 1077, 1046, 1061, 987, 1065, 998, 971, 1051, 1026, 1021, 1044, 1022, 998, 1033, 1000, 1025, 1057, 999, 974, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1023, 1011, 1012, 1025, 1010, 1006, 1034, 1035, 980, 1003, 1030, 1014, 1007, 1079, 1024, 1047, 1006, 1033, 983, 1008, 1058, 1016, 1066, 1027, 1047, 1086, 1028, 988, 985, 1044, 1059, 1041, 981, 1017, 1033, 1032, 1012, 1007, 1103, 953, 1052, 1039, 983, 1063, 1050, 1024, 1040, 1060, 971, 1016, 1064, 993, 1026, 1018, 1022, 1037, 998, 1049, 985, 963, 1059, 1029, 1016, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1014, 1010, 1033, 1027, 1008, 1031, 1041, 1005, 1024, 990, 1030, 1036, 1037, 1053, 1047, 1032, 1036, 1043, 1027, 1006, 1048, 1034, 1045, 1046, 996, 969, 970, 1028, 1001, 1049, 984, 1058, 963, 1050, 1062, 1051, 947, 1005, 1019, 1070, 1053, 1073, 1029, 1034, 1041, 1023, 1044, 997, 1043, 986, 1088, 1020, 1004, 1026, 1014, 1007, 946, 1058, 1010, 1015, 1028, 1018, 1018, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1019, 1057, 1002, 981, 1042, 1005, 1028, 1025, 1041, 1031, 1025, 1055, 1026, 1033, 1005, 1045, 1020, 998, 1046, 1058, 1007, 1032, 1004, 1044, 1012, 1085, 1038, 987, 991, 1007, 999, 999, 1049, 1017, 1050, 1079, 1018, 1041, 1030, 1010, 994, 980, 1027, 1035, 987, 1042, 1042, 1002, 1045, 1034, 1007, 1007, 1036, 1035, 1014, 1027, 1032, 997, 981, 997, 1017, 1052, 1055, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[986, 1018, 976, 1044, 1065, 1046, 1053, 1019, 33, 9, 0, 0, 0, 0, 0, 0, 1036, 1005, 995, 1017, 1064, 993, 1052, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1024, 1052, 963, 985, 1062, 1008, 1031, 14, 1, 0, 0, 0, 0, 0, 0, 1039, 1016, 995, 1037, 1002, 1023, 1070, 1065, 29, 0, 0, 0, 0, 0, 0, 0, 1004, 1028, 1004, 1025, 997, 1006, 999, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 981, 1071, 1013, 1001, 1056, 1053, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1044, 1000, 1000, 1023, 1015, 965, 977, 43, 0, 0, 0, 0, 0, 0, 0, 1009, 1007, 1048, 1039, 983, 1079, 1012, 1035, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8201, 8139, 8276, 8084, 8262, 8095, 8230] avg=8192.0000 max=8276 max_rank=3 min=8084 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[999, 981, 994, 1056, 977, 999, 1023, 978, 33, 9, 0, 0, 0, 0, 0, 0, 1091, 1024, 1013, 1026, 1068, 955, 1063, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1025, 973, 1055, 1066, 1029, 1017, 1054, 15, 1, 0, 0, 0, 0, 0, 0, 1022, 1050, 1044, 992, 996, 1060, 990, 1008, 28, 0, 0, 0, 0, 0, 0, 0, 952, 1000, 1061, 1061, 1064, 1038, 1011, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 976, 977, 1054, 1043, 1022, 1032, 1015, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1022, 1004, 1010, 989, 1027, 1074, 984, 43, 0, 0, 0, 0, 0, 0, 0, 1054, 991, 1017, 1013, 1071, 951, 1028, 1048, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8049, 8248, 8238, 8190, 8224, 8196, 8201, 8190] avg=8192.0000 max=8248 max_rank=1 min=8049 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1005, 1037, 975, 1032, 1065, 1018, 1050, 957, 33, 9, 0, 0, 0, 0, 0, 0, 1077, 1016, 1031, 1050, 975, 1029, 1025, 999, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1055, 1019, 1048, 1018, 1017, 1028, 1019, 14, 1, 0, 0, 0, 0, 0, 0, 1011, 1041, 1004, 1003, 1006, 1004, 1030, 1092, 30, 0, 0, 0, 0, 0, 0, 0, 1077, 1009, 1002, 1009, 1029, 960, 1055, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 935, 969, 1038, 1083, 1034, 1025, 1030, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1070, 1005, 1007, 1012, 987, 990, 1039, 43, 0, 0, 0, 0, 0, 0, 0, 1009, 1034, 1053, 972, 1014, 1030, 1057, 1013, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8202, 8190, 8221, 8174, 8166, 8202, 8200] avg=8192.0000 max=8221 max_rank=3 min=8166 min_rank=5 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[987, 1069, 1008, 1019, 1023, 1045, 1008, 1053, 35, 9, 0, 0, 0, 0, 0, 0, 990, 1027, 1064, 1001, 987, 980, 1043, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1037, 1030, 1036, 1008, 992, 1009, 1044, 15, 1, 0, 0, 0, 0, 0, 0, 1014, 999, 1054, 1014, 998, 986, 1023, 1043, 29, 0, 0, 0, 0, 0, 0, 0, 1048, 1011, 1084, 1018, 984, 1020, 1015, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1020, 985, 1009, 1073, 1061, 1006, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1014, 994, 996, 1037, 1016, 983, 1046, 45, 0, 0, 0, 0, 0, 0, 0, 1046, 1033, 1070, 1060, 1025, 1035, 1031, 974, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8143, 8166, 8160, 8191, 8200, 8129, 8291] avg=8192.0000 max=8291 max_rank=7 min=8129 min_rank=6 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1014, 1010, 1033, 1027, 1008, 1031, 1041, 1005, 34, 9, 0, 0, 0, 0, 0, 0, 1024, 990, 1030, 1036, 1037, 995, 1047, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1043, 1027, 1006, 1048, 1034, 1045, 1046, 15, 1, 0, 0, 0, 0, 0, 0, 996, 969, 970, 1028, 1001, 1049, 984, 1058, 30, 0, 0, 0, 0, 0, 0, 0, 963, 1050, 1047, 1051, 947, 1005, 1019, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 995, 1029, 1034, 1041, 1023, 1044, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 986, 1088, 1020, 1004, 1026, 1014, 1007, 44, 0, 0, 0, 0, 0, 0, 0, 946, 1058, 1010, 1015, 1028, 1018, 1018, 1036, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8191, 8301, 8085, 8152, 8216, 8232, 8147] avg=8192.0000 max=8301 max_rank=2 min=8085 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1023, 1011, 1012, 1025, 1010, 1006, 1034, 1035, 33, 9, 0, 0, 0, 0, 0, 0, 980, 1003, 1030, 1014, 1007, 1021, 1024, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1033, 983, 1008, 1058, 1016, 1066, 1027, 14, 1, 0, 0, 0, 0, 0, 0, 1047, 1086, 1028, 988, 985, 1044, 1059, 1041, 30, 0, 0, 0, 0, 0, 0, 0, 981, 1017, 1019, 1032, 1012, 1007, 1103, 953, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 964, 983, 1063, 1050, 1024, 1040, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1016, 1064, 993, 1026, 1018, 1022, 1037, 42, 0, 0, 0, 0, 0, 0, 0, 998, 1049, 985, 963, 1059, 1029, 1016, 1026, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8126, 8212, 8308, 8124, 8236, 8189, 8143] avg=8192.0000 max=8308 max_rank=3 min=8124 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1029, 1085, 1000, 1052, 1030, 993, 967, 987, 32, 9, 0, 0, 0, 0, 0, 0, 1020, 1068, 1035, 1062, 957, 984, 1052, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1003, 1019, 1023, 1022, 990, 982, 992, 14, 1, 0, 0, 0, 0, 0, 0, 1003, 1009, 983, 1049, 1077, 968, 991, 1035, 29, 0, 0, 0, 0, 0, 0, 0, 1007, 1052, 1027, 1054, 1024, 1060, 1047, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 990, 938, 1064, 1067, 1077, 1046, 1061, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 998, 971, 1051, 1026, 1021, 1044, 1022, 41, 0, 0, 0, 0, 0, 0, 0, 998, 1033, 1000, 1025, 1057, 999, 974, 1030, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8236, 8059, 8144, 8311, 8230, 8239, 8133] avg=8192.0000 max=8311 max_rank=4 min=8059 min_rank=2 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1019, 1057, 1002, 981, 1042, 1005, 1028, 1025, 35, 13, 0, 0, 0, 0, 0, 0, 1041, 1031, 1025, 1055, 1026, 961, 1005, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 998, 1046, 1058, 1007, 1032, 1004, 1044, 18, 4, 0, 0, 0, 0, 0, 0, 1012, 1085, 1038, 987, 991, 1007, 999, 999, 34, 0, 0, 0, 0, 0, 0, 0, 1049, 1017, 1032, 1079, 1018, 1041, 1030, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 994, 901, 1027, 1035, 987, 1042, 1042, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1034, 1007, 1007, 1036, 1035, 1014, 1027, 44, 0, 0, 0, 0, 0, 0, 0, 1032, 997, 981, 997, 1017, 1052, 1055, 1050, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8189, 8231, 8152, 8276, 8030, 8249, 8202] avg=8192.0000 max=8276 max_rank=4 min=8030 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32757 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8064, 8320, 8320, 8192, 8320, 8064, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 7936, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8064, 8192, 8192, 8448, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 7808, 8320, 8448, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8448, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1057, 1064, 1021, 1013, 1024, 984, 986, 1029, 1044, 1014, 1065, 1066, 1001, 1032, 1029, 1036, 1006, 968, 1030, 1004, 1086, 1008, 1035, 1016, 1033, 1008, 1025, 986, 1006, 1011, 984, 1051, 1025, 1054, 1017, 1016, 1051, 1066, 1036, 1056, 1021, 1068, 1032, 1036, 1026, 986, 1063, 1004, 1038, 965, 1024, 1057, 971, 1014, 1012, 995, 994, 1022, 1012, 1037, 973, 1065, 1052, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1016, 983, 1067, 1006, 1078, 988, 1004, 1060, 1081, 1026, 1006, 1009, 1056, 995, 1037, 1041, 1008, 1037, 947, 1020, 1037, 1027, 1028, 1030, 1025, 1013, 1046, 1002, 1013, 1057, 1032, 1007, 1003, 1002, 1059, 1006, 1033, 1042, 1037, 1024, 1015, 1046, 1021, 1012, 992, 1072, 1020, 1071, 1010, 1016, 1021, 1013, 988, 1023, 1002, 1006, 1062, 1024, 1018, 1017, 1012, 1039, 1033, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1057, 1037, 1049, 1027, 1040, 999, 1033, 973, 1020, 1020, 1026, 1014, 996, 1012, 1095, 1031, 1029, 1001, 1020, 989, 995, 1035, 1060, 1043, 1012, 1072, 1016, 1044, 1012, 994, 1027, 1022, 991, 1030, 1043, 1020, 1003, 1056, 1027, 1015, 1042, 1084, 1032, 989, 1029, 1011, 1019, 1044, 1034, 1037, 1000, 1011, 1009, 1031, 984, 1052, 1084, 1027, 1008, 957, 1045, 1032, 980, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1050, 1029, 1005, 1004, 1016, 977, 1053, 1040, 1035, 1006, 1032, 1045, 972, 1019, 1036, 1031, 1028, 1059, 1059, 1011, 1030, 953, 988, 1042, 1032, 1013, 1004, 956, 1009, 1013, 1006, 1055, 1054, 989, 1048, 1016, 1026, 1053, 999, 1027, 1052, 1051, 1028, 1027, 1008, 1044, 1053, 1038, 1024, 983, 1036, 1011, 1001, 1073, 1028, 1033, 968, 1005, 1062, 1037, 1059, 1049, 1026, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1031, 1025, 1031, 1063, 985, 1041, 984, 980, 1002, 1067, 997, 967, 1083, 1025, 1012, 979, 1026, 1065, 1018, 1012, 1002, 1057, 1046, 994, 1037, 984, 1013, 1035, 988, 1011, 1042, 1020, 983, 1083, 1034, 1044, 1059, 1010, 1038, 1040, 1020, 990, 1024, 1024, 1022, 1031, 1026, 1049, 1062, 1030, 1012, 1019, 1039, 1046, 1037, 1042, 1031, 1011, 949, 1006, 978, 1057, 1096, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[974, 1070, 1044, 1015, 955, 1045, 1010, 1044, 1023, 1033, 1017, 1010, 1020, 1020, 1024, 1025, 1017, 1028, 998, 1027, 1015, 1035, 1005, 1035, 1017, 1060, 1041, 1045, 1034, 1004, 1013, 1028, 1071, 1030, 1051, 1011, 974, 1103, 1045, 997, 990, 1017, 1010, 1087, 1015, 1032, 1037, 985, 1000, 1044, 1024, 1057, 1010, 1036, 997, 1042, 1054, 991, 1022, 1040, 1003, 1032, 1021, 977] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[973, 1021, 997, 1000, 977, 1010, 984, 1046, 1017, 1063, 977, 989, 1007, 980, 1053, 1023, 965, 984, 1053, 1034, 1028, 1013, 1027, 1018, 1035, 1000, 1014, 1037, 997, 988, 1102, 1039, 1036, 1044, 1050, 978, 1026, 1053, 1053, 1080, 1010, 1036, 1045, 1038, 1027, 1043, 1003, 1018, 1049, 1046, 1063, 963, 1039, 1055, 1037, 1009, 1109, 1034, 1037, 983, 1039, 1054, 1024, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1009, 1024, 1060, 1012, 1031, 1032, 1015, 1027, 1035, 1012, 1038, 983, 994, 1048, 1025, 977, 1007, 1069, 1031, 1031, 998, 1021, 1077, 1032, 997, 1010, 1034, 1065, 944, 980, 1020, 995, 1007, 1037, 1009, 1057, 1013, 1027, 1024, 1045, 1059, 1073, 1007, 1008, 1014, 953, 1045, 975, 1023, 1018, 1061, 1068, 1074, 1077, 1035, 1013, 1074, 1011, 976, 1030, 1031, 1013, 1049, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1057, 1064, 1021, 1013, 1024, 984, 986, 1029, 22, 9, 0, 0, 0, 0, 0, 0, 1044, 1014, 1065, 1066, 1001, 1032, 1029, 1036, 9, 0, 0, 0, 0, 0, 0, 0, 1006, 968, 1030, 1004, 1086, 1008, 1035, 1016, 17, 0, 0, 0, 0, 0, 0, 0, 1033, 1008, 1025, 986, 1006, 1011, 984, 1051, 40, 0, 0, 0, 0, 0, 0, 0, 1025, 1054, 1017, 1016, 1051, 1004, 1036, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1042, 1032, 1036, 1026, 986, 1063, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 965, 1024, 1057, 971, 1004, 1012, 995, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1022, 1012, 1037, 973, 1065, 1052, 1026, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8296, 8170, 8144, 8259, 8210, 8066, 8182] avg=8192.0000 max=8296 max_rank=1 min=8066 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1016, 983, 1067, 1006, 1078, 988, 1004, 1060, 22, 9, 0, 0, 0, 0, 0, 0, 1081, 1026, 1006, 1009, 1056, 995, 1037, 1041, 10, 0, 0, 0, 0, 0, 0, 0, 1008, 1037, 947, 1020, 1037, 1027, 1028, 1030, 17, 0, 0, 0, 0, 0, 0, 0, 1025, 1013, 1046, 1002, 1013, 1057, 1032, 1007, 39, 0, 0, 0, 0, 0, 0, 0, 1003, 1002, 1059, 1006, 1033, 981, 1037, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1020, 1021, 1012, 992, 1072, 1020, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1016, 1021, 1013, 988, 1012, 1002, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1024, 1018, 1017, 1012, 1039, 1033, 1015, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8261, 8151, 8234, 8145, 8223, 8068, 8221] avg=8192.0000 max=8261 max_rank=1 min=8068 min_rank=6 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[973, 1021, 997, 1000, 977, 1010, 984, 1046, 22, 9, 0, 0, 0, 0, 0, 0, 1017, 1063, 977, 989, 1007, 980, 1053, 1023, 10, 0, 0, 0, 0, 0, 0, 0, 965, 984, 1053, 1034, 1028, 1013, 1027, 1018, 17, 0, 0, 0, 0, 0, 0, 0, 1035, 1000, 1014, 1037, 997, 988, 1102, 1039, 40, 0, 0, 0, 0, 0, 0, 0, 1036, 1044, 1050, 978, 1026, 991, 1053, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1010, 1045, 1038, 1027, 1043, 1003, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1046, 1063, 963, 1039, 1044, 1037, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 1034, 1037, 983, 1039, 1054, 1024, 1004, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8039, 8119, 8139, 8252, 8258, 8194, 8250, 8285] avg=8192.0000 max=8285 max_rank=7 min=8039 min_rank=0 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32987 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1031, 1025, 1031, 1063, 985, 1041, 984, 980, 21, 8, 0, 0, 0, 0, 0, 0, 1002, 1067, 997, 967, 1083, 1025, 1012, 979, 10, 0, 0, 0, 0, 0, 0, 0, 1026, 1065, 1018, 1012, 1002, 1057, 1046, 994, 16, 0, 0, 0, 0, 0, 0, 0, 1037, 984, 1013, 1035, 988, 1011, 1042, 1020, 38, 0, 0, 0, 0, 0, 0, 0, 983, 1083, 1034, 1044, 1059, 951, 1038, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 966, 1024, 1024, 1022, 1031, 1026, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1030, 1012, 1019, 1039, 1035, 1037, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1011, 949, 1006, 978, 1057, 1096, 1022, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8142, 8236, 8168, 8232, 8162, 8276, 8151] avg=8192.0000 max=8276 max_rank=6 min=8142 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1050, 1029, 1005, 1004, 1016, 977, 1053, 1040, 22, 9, 0, 0, 0, 0, 0, 0, 1035, 1006, 1032, 1045, 972, 1019, 1036, 1031, 10, 0, 0, 0, 0, 0, 0, 0, 1028, 1059, 1059, 1011, 1030, 953, 988, 1042, 17, 0, 0, 0, 0, 0, 0, 0, 1032, 1013, 1004, 956, 1009, 1013, 1006, 1055, 40, 0, 0, 0, 0, 0, 0, 0, 1054, 989, 1048, 1016, 1026, 991, 999, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1025, 1028, 1027, 1008, 1044, 1053, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 983, 1036, 1011, 1001, 1062, 1028, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1005, 1062, 1037, 1059, 1049, 1026, 1020, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8186, 8187, 8128, 8150, 8275, 8178, 8227] avg=8192.0000 max=8275 max_rank=5 min=8128 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[974, 1070, 1044, 1015, 955, 1045, 1010, 1044, 23, 8, 0, 0, 0, 0, 0, 0, 1023, 1033, 1017, 1010, 1020, 1020, 1024, 1025, 10, 0, 0, 0, 0, 0, 0, 0, 1017, 1028, 998, 1027, 1015, 1035, 1005, 1035, 16, 0, 0, 0, 0, 0, 0, 0, 1017, 1060, 1041, 1045, 1034, 1004, 1013, 1028, 42, 0, 0, 0, 0, 0, 0, 0, 1071, 1030, 1051, 1011, 974, 1038, 1045, 997, 0, 0, 0, 0, 0, 0, 0, 0, 990, 993, 1010, 1087, 1015, 1032, 1037, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1044, 1024, 1057, 1010, 1025, 997, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 991, 1022, 1040, 1003, 1032, 1021, 977, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8182, 8176, 8284, 8217, 8149, 8199, 8141] avg=8192.0000 max=8284 max_rank=3 min=8141 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1057, 1037, 1049, 1027, 1040, 999, 1033, 973, 22, 9, 0, 0, 0, 0, 0, 0, 1020, 1020, 1026, 1014, 996, 1012, 1095, 1031, 10, 0, 0, 0, 0, 0, 0, 0, 1029, 1001, 1020, 989, 995, 1035, 1060, 1043, 18, 0, 0, 0, 0, 0, 0, 0, 1012, 1072, 1016, 1044, 1012, 994, 1027, 1022, 40, 0, 0, 0, 0, 0, 0, 0, 991, 1030, 1043, 1020, 1003, 994, 1027, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1057, 1032, 989, 1029, 1011, 1019, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1037, 1000, 1011, 1009, 1020, 984, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 1027, 1008, 957, 1045, 1032, 980, 1010, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8224, 8190, 8239, 8123, 8223, 8147, 8144] avg=8192.0000 max=8246 max_rank=0 min=8123 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1009, 1024, 1060, 1012, 1031, 1032, 1015, 1027, 25, 12, 0, 0, 0, 0, 0, 0, 1035, 1012, 1038, 983, 994, 1048, 1025, 977, 13, 1, 0, 0, 0, 0, 0, 0, 1007, 1069, 1031, 1031, 998, 1021, 1077, 1032, 21, 0, 0, 0, 0, 0, 0, 0, 997, 1010, 1034, 1065, 944, 980, 1020, 995, 42, 0, 0, 0, 0, 0, 0, 0, 1007, 1037, 1009, 1057, 1013, 960, 1024, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1039, 1007, 1008, 1014, 953, 1045, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1018, 1061, 1068, 1074, 1060, 1035, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1011, 976, 1030, 1031, 1013, 1049, 997, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8126, 8287, 8087, 8152, 8100, 8352, 8185] avg=8192.0000 max=8352 max_rank=6 min=8087 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8320, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 7936, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8448, 8192, 8192, 8192, 8192, 8448, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8064, 8064, 8320, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1041, 974, 979, 1006, 1017, 1014, 1068, 1035, 1025, 1045, 993, 1047, 1080, 987, 1046, 1026, 1007, 1046, 1032, 1035, 1001, 995, 997, 1030, 1047, 1004, 1035, 1009, 1042, 993, 982, 1048, 1051, 1001, 1047, 1032, 1002, 1002, 1021, 1009, 1067, 1045, 994, 1070, 1087, 1044, 1040, 1014, 1033, 968, 1007, 1026, 994, 1008, 1045, 1023, 991, 1045, 1025, 1018, 999, 1034, 1006, 1102] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1015, 1013, 1067, 1007, 978, 980, 980, 1007, 1018, 1062, 999, 1036, 1054, 1033, 1065, 1044, 1070, 1018, 1004, 992, 1099, 1023, 1050, 1070, 1034, 998, 965, 1011, 1017, 994, 1040, 1009, 1031, 1024, 996, 1018, 1023, 1087, 1024, 1017, 1007, 1028, 1000, 1051, 1022, 1036, 1065, 1039, 1016, 985, 1028, 983, 1043, 1029, 1062, 1032, 1003, 1030, 1016, 1002, 1032, 1049, 996, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1050, 1037, 1038, 1023, 1022, 1013, 1023, 1005, 1004, 1046, 1013, 1061, 1034, 1037, 1012, 1005, 1007, 1048, 1036, 1000, 1009, 1025, 995, 1040, 1024, 1042, 985, 1021, 1067, 1023, 1037, 1020, 1058, 1049, 985, 978, 977, 1024, 1048, 1031, 1045, 1027, 1090, 1003, 1041, 1004, 1013, 959, 1038, 998, 1008, 1074, 1021, 1036, 1032, 1077, 977, 1020, 1063, 1004, 1060, 933, 1038, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[995, 1030, 1050, 981, 1011, 1005, 998, 992, 1015, 1027, 1047, 1027, 1039, 997, 991, 1022, 1029, 1028, 1046, 980, 1059, 1039, 1027, 1074, 975, 1014, 1068, 999, 1042, 994, 981, 1034, 1046, 1003, 993, 1048, 1029, 990, 988, 1053, 978, 1036, 1069, 1051, 1020, 1017, 1033, 1051, 1040, 1018, 1012, 1015, 1057, 1000, 1044, 1050, 1061, 1046, 1044, 1006, 1005, 1051, 997, 1069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1024, 983, 1032, 1029, 986, 1039, 1116, 1016, 1018, 1004, 979, 1036, 985, 1060, 1001, 993, 1017, 1100, 1035, 981, 1052, 1029, 1000, 1043, 998, 1025, 1000, 1044, 1003, 1007, 1029, 997, 1000, 1058, 975, 994, 1055, 1043, 1045, 1035, 1025, 1034, 1083, 1003, 1024, 1003, 1001, 1032, 943, 1010, 1015, 1055, 990, 1025, 1047, 1020, 1048, 1041, 1016, 982, 1040, 1107, 1062, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[986, 1029, 1002, 1031, 1011, 1091, 992, 1011, 1016, 1046, 1020, 1054, 1022, 1038, 994, 962, 1023, 1036, 980, 1026, 1021, 1039, 1075, 1000, 1046, 1037, 1034, 1020, 1065, 1053, 1042, 1002, 1013, 1061, 1001, 1030, 969, 1063, 1061, 1032, 1010, 1071, 1008, 1047, 992, 1008, 1024, 1076, 1055, 1013, 997, 1003, 1040, 1003, 985, 1003, 989, 978, 1039, 1053, 1027, 1045, 1035, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1064, 968, 1019, 1024, 1020, 1014, 1061, 1065, 1002, 1006, 1049, 1024, 1056, 1034, 1040, 1081, 1027, 1022, 960, 1020, 1028, 1002, 1012, 978, 1087, 1054, 996, 1043, 1002, 981, 1016, 979, 1056, 1026, 1044, 969, 1001, 1014, 1010, 1035, 1032, 1078, 1067, 1022, 982, 996, 1018, 1016, 1004, 1073, 1026, 1067, 991, 1069, 1075, 987, 1022, 1003, 986, 1047, 1035, 1006, 1021, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1025, 986, 1020, 1053, 1045, 1030, 1053, 1027, 1035, 992, 1033, 1020, 975, 962, 1023, 1002, 1101, 988, 1025, 1015, 1003, 997, 1011, 1041, 1049, 992, 1050, 1027, 1039, 1047, 1032, 1054, 1028, 1046, 984, 1042, 1008, 1018, 1047, 1016, 994, 985, 1019, 1015, 977, 988, 1008, 1051, 1043, 1078, 1063, 1015, 1045, 1135, 1027, 1013, 1034, 1040, 1043, 1049, 1021, 963, 993, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1041, 974, 979, 1006, 1017, 1014, 1068, 1035, 23, 3, 0, 0, 0, 0, 0, 0, 1025, 1045, 993, 1047, 1080, 987, 1046, 1026, 3, 1, 0, 0, 0, 0, 0, 0, 1007, 1039, 1032, 1035, 1001, 995, 997, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1004, 1035, 1009, 1042, 993, 982, 1048, 16, 0, 0, 0, 0, 0, 0, 0, 1051, 1001, 1047, 1032, 1002, 1002, 1021, 1009, 6, 2, 0, 0, 0, 0, 0, 0, 1067, 1045, 971, 1070, 1087, 1044, 1040, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 968, 1007, 1026, 994, 1008, 1024, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1045, 1025, 1018, 999, 1034, 1006, 1099, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8253, 8136, 8176, 8173, 8338, 8083, 8217] avg=8192.0000 max=8338 max_rank=5 min=8083 min_rank=6 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1015, 1013, 1067, 1007, 978, 980, 980, 1007, 23, 3, 0, 0, 0, 0, 0, 0, 1018, 1062, 999, 1036, 1054, 1033, 1065, 1044, 3, 0, 0, 0, 0, 0, 0, 0, 1070, 1012, 1004, 992, 1099, 1023, 1050, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 998, 965, 1011, 1017, 994, 1040, 1009, 16, 0, 0, 0, 0, 0, 0, 0, 1031, 1024, 996, 1018, 1023, 1087, 1024, 1017, 6, 2, 0, 0, 0, 0, 0, 0, 1007, 1028, 977, 1051, 1022, 1036, 1065, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 985, 1028, 983, 1043, 1029, 1041, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1030, 1016, 1002, 1032, 1049, 996, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8073, 8314, 8320, 8084, 8228, 8225, 8157, 8135] avg=8192.0000 max=8320 max_rank=2 min=8073 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[995, 1030, 1050, 981, 1011, 1005, 998, 992, 25, 3, 0, 0, 0, 0, 0, 0, 1015, 1027, 1047, 1027, 1039, 997, 991, 1022, 3, 0, 0, 0, 0, 0, 0, 0, 1029, 1022, 1046, 980, 1059, 1039, 1027, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1014, 1068, 999, 1042, 994, 981, 1034, 16, 0, 0, 0, 0, 0, 0, 0, 1046, 1003, 993, 1048, 1029, 990, 988, 1053, 6, 2, 0, 0, 0, 0, 0, 0, 978, 1036, 1044, 1051, 1020, 1017, 1033, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1018, 1012, 1015, 1057, 1000, 1023, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1046, 1044, 1006, 1005, 1051, 997, 1066, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8168, 8276, 8123, 8158, 8230, 8215, 8276] avg=8192.0000 max=8276 max_rank=2 min=8090 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32879 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1050, 1037, 1038, 1023, 1022, 1013, 1023, 1005, 26, 3, 0, 0, 0, 0, 0, 0, 1004, 1046, 1013, 1061, 1034, 1037, 1012, 1005, 3, 1, 0, 0, 0, 0, 0, 0, 1007, 1041, 1036, 1000, 1009, 1025, 995, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1042, 985, 1021, 1067, 1023, 1037, 1020, 16, 0, 0, 0, 0, 0, 0, 0, 1058, 1049, 985, 978, 977, 1024, 1048, 1031, 6, 2, 0, 0, 0, 0, 0, 0, 1045, 1027, 1064, 1003, 1041, 1004, 1013, 959, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 998, 1008, 1074, 1021, 1036, 1011, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1020, 1063, 1004, 1060, 933, 1038, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8216, 8153, 8235, 8158, 8156, 8263, 8115] avg=8192.0000 max=8263 max_rank=6 min=8115 min_rank=7 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1064, 968, 1019, 1024, 1020, 1014, 1061, 1065, 25, 4, 0, 0, 0, 0, 0, 0, 1002, 1006, 1049, 1024, 1056, 1034, 1040, 1081, 3, 0, 0, 0, 0, 0, 0, 0, 1027, 1016, 960, 1020, 1028, 1002, 1012, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 1054, 996, 1043, 1002, 981, 1016, 979, 17, 0, 0, 0, 0, 0, 0, 0, 1056, 1026, 1044, 969, 1001, 1014, 1010, 1035, 6, 2, 0, 0, 0, 0, 0, 0, 1032, 1078, 1042, 1022, 982, 996, 1018, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1073, 1026, 1067, 991, 1069, 1052, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1003, 986, 1047, 1035, 1006, 1021, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8295, 8043, 8175, 8163, 8186, 8269, 8141] avg=8192.0000 max=8295 max_rank=1 min=8043 min_rank=2 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1024, 983, 1032, 1029, 986, 1039, 1116, 1016, 25, 3, 0, 0, 0, 0, 0, 0, 1018, 1004, 979, 1036, 985, 1060, 1001, 993, 3, 1, 0, 0, 0, 0, 0, 0, 1017, 1092, 1035, 981, 1052, 1029, 1000, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1025, 1000, 1044, 1003, 1007, 1029, 997, 16, 0, 0, 0, 0, 0, 0, 0, 1000, 1058, 975, 994, 1055, 1043, 1045, 1035, 7, 2, 0, 0, 0, 0, 0, 0, 1025, 1034, 1058, 1003, 1024, 1003, 1001, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 943, 1010, 1015, 1055, 990, 1025, 1026, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1041, 1016, 982, 1040, 1107, 1062, 1061, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8080, 8249, 8119, 8214, 8180, 8084, 8357] avg=8192.0000 max=8357 max_rank=7 min=8080 min_rank=1 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[986, 1029, 1002, 1031, 1011, 1091, 992, 1011, 24, 3, 0, 0, 0, 0, 0, 0, 1016, 1046, 1020, 1054, 1022, 1038, 994, 962, 3, 1, 0, 0, 0, 0, 0, 0, 1023, 1029, 980, 1026, 1021, 1039, 1075, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1037, 1034, 1020, 1065, 1053, 1042, 1002, 15, 0, 0, 0, 0, 0, 0, 0, 1013, 1061, 1001, 1030, 969, 1063, 1061, 1032, 6, 2, 0, 0, 0, 0, 0, 0, 1010, 1071, 984, 1047, 992, 1008, 1024, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1013, 997, 1003, 1040, 1003, 965, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 989, 978, 1039, 1053, 1027, 1045, 1035, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8156, 8193, 8314, 8238, 8212, 8079, 8164] avg=8192.0000 max=8314 max_rank=3 min=8079 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1025, 986, 1020, 1053, 1045, 1030, 1053, 1027, 28, 9, 0, 0, 0, 0, 0, 0, 1035, 992, 1033, 1020, 975, 962, 1023, 1002, 8, 4, 0, 0, 0, 0, 0, 0, 1101, 973, 1025, 1015, 1003, 997, 1011, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 992, 1050, 1027, 1039, 1047, 1032, 1054, 20, 0, 0, 0, 0, 0, 0, 0, 1028, 1046, 984, 1042, 1008, 1018, 1047, 1016, 11, 4, 0, 0, 0, 0, 0, 0, 994, 985, 991, 1015, 977, 988, 1008, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1078, 1063, 1015, 1045, 1135, 994, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1040, 1043, 1049, 1021, 963, 993, 988, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8054, 8166, 8310, 8204, 8009, 8386, 8131] avg=8192.0000 max=8386 max_rank=6 min=8009 min_rank=5 imbalance_ratio=1.023682 max_load_reduction=57150 max_load_reduction_pct=0.872040 moved_to_echo=32730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8192, 8192, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8064, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8192, 8064, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1030, 1028, 1031, 1047, 1014, 1037, 989, 1055, 1059, 973, 1012, 1011, 1009, 1023, 1060, 1007, 1015, 1008, 996, 1047, 1018, 1031, 1022, 1040, 1061, 1031, 1054, 1004, 977, 1025, 985, 1038, 1021, 1042, 1048, 1028, 1018, 1015, 971, 1014, 993, 1030, 1038, 1065, 1018, 974, 1014, 1019, 1080, 985, 1059, 1046, 1055, 977, 980, 1021, 1002, 1054, 1056, 993, 1065, 1056, 1032, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1031, 994, 1008, 1035, 989, 982, 1035, 1032, 994, 1071, 996, 1092, 1053, 1027, 1049, 1044, 999, 1016, 977, 1013, 1052, 1062, 1024, 1066, 1032, 1013, 1011, 1031, 1050, 997, 1023, 1026, 1040, 1049, 1030, 1042, 1040, 1014, 1030, 1048, 989, 1038, 1021, 1058, 1046, 1017, 1052, 1056, 1017, 1015, 992, 1017, 1020, 1049, 1004, 986, 1012, 1005, 1042, 936, 1048, 974, 982, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1044, 966, 1023, 987, 1067, 1023, 1012, 1035, 1064, 1041, 1018, 1035, 1003, 1049, 986, 1011, 1025, 982, 1064, 1024, 1038, 994, 991, 1028, 1021, 1001, 1043, 1060, 1048, 966, 1016, 971, 1059, 1024, 1100, 1032, 1028, 1072, 1025, 1050, 1015, 973, 1031, 1064, 1036, 1038, 1044, 1024, 1092, 1005, 1048, 958, 993, 1079, 992, 973, 1045, 996, 989, 1031, 1023, 1043, 1025, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[937, 1056, 1018, 1017, 1027, 1026, 1077, 1007, 1023, 1059, 1036, 1088, 1018, 1012, 964, 1043, 1062, 1031, 984, 1031, 1009, 1024, 1014, 1034, 1060, 1037, 1030, 1001, 1036, 1048, 991, 1051, 1027, 1048, 1012, 1065, 1030, 1028, 1015, 1026, 977, 985, 1014, 1011, 981, 1005, 984, 1030, 1034, 1037, 997, 997, 1055, 1017, 1018, 984, 1079, 1024, 1034, 1021, 1020, 1048, 1077, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1034, 997, 1015, 1012, 976, 1054, 1032, 1039, 1013, 1047, 1023, 1037, 1040, 1002, 1026, 1093, 984, 1027, 1028, 1012, 1019, 1034, 1049, 1027, 999, 1055, 1004, 1006, 1033, 990, 1047, 965, 1051, 1057, 1016, 998, 1060, 1019, 1030, 1015, 1036, 1010, 987, 1052, 986, 1040, 997, 1012, 1016, 1030, 1075, 988, 1026, 1050, 1073, 965, 1042, 1044, 1028, 995, 1054, 1017, 1020, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1013, 991, 1027, 1003, 979, 973, 1065, 1060, 1034, 1049, 1048, 994, 1041, 1056, 1015, 1041, 1022, 1036, 991, 971, 1009, 1010, 1064, 1050, 996, 1025, 1037, 1071, 1007, 985, 1036, 1057, 1041, 1051, 1038, 997, 956, 1030, 1086, 1008, 975, 1005, 1003, 1056, 1017, 1007, 990, 1070, 1046, 1093, 1010, 1034, 1006, 1015, 979, 1025, 1033, 1002, 1022, 1046, 1032, 1055, 1008, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1036, 1022, 983, 977, 1077, 1006, 1010, 1027, 1010, 1007, 1025, 972, 987, 1035, 1064, 1012, 1031, 1025, 1041, 1017, 1000, 963, 1013, 1041, 1004, 1045, 1039, 1101, 1001, 1031, 982, 1060, 1050, 1034, 1043, 1049, 1017, 1036, 1076, 1014, 1063, 985, 1025, 1032, 1036, 1020, 1016, 1031, 1013, 1042, 971, 1066, 1057, 1020, 1002, 1031, 1007, 1039, 1020, 949, 1031, 1057, 996, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1034, 1041, 983, 993, 1031, 1050, 1033, 1072, 1045, 1042, 1004, 1071, 1011, 1048, 1008, 1000, 1034, 1007, 1038, 1032, 974, 1045, 1016, 1017, 999, 1048, 1025, 1025, 994, 1053, 1023, 1051, 1042, 951, 1004, 1002, 1021, 992, 1061, 1048, 1002, 1049, 1002, 1033, 999, 988, 1019, 1036, 1016, 1016, 1005, 1007, 1001, 1013, 1002, 1060, 1014, 1029, 1058, 1026, 1004, 1088, 1076, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1044, 966, 1023, 987, 1067, 1023, 1012, 1035, 16, 12, 0, 0, 0, 0, 0, 0, 1064, 1041, 1018, 999, 1003, 1049, 986, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 982, 1064, 1024, 1038, 994, 991, 1028, 23, 0, 0, 0, 0, 0, 0, 0, 1021, 1001, 1043, 1060, 1048, 966, 1016, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1024, 1100, 1032, 1028, 1072, 1025, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 973, 1031, 1064, 1036, 1038, 1044, 1024, 40, 0, 0, 0, 0, 0, 0, 0, 1092, 1005, 1048, 958, 993, 1079, 992, 973, 24, 1, 0, 0, 0, 0, 0, 0, 1045, 996, 989, 1031, 1023, 1019, 1025, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8171, 8169, 8126, 8334, 8265, 8165, 8121] avg=8192.0000 max=8334 max_rank=4 min=8121 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1031, 994, 1008, 1035, 989, 982, 1035, 1032, 16, 13, 0, 0, 0, 0, 0, 0, 994, 1071, 996, 1054, 1053, 1027, 1049, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1016, 977, 1013, 1052, 1062, 1024, 1066, 21, 0, 0, 0, 0, 0, 0, 0, 1032, 1013, 1011, 1031, 1050, 997, 1023, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1049, 1030, 1042, 1040, 1014, 1030, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1038, 1021, 1058, 1046, 1017, 1052, 1056, 39, 0, 0, 0, 0, 0, 0, 0, 1017, 1015, 992, 1017, 1020, 1049, 1004, 986, 25, 0, 0, 0, 0, 0, 0, 0, 1012, 1005, 1042, 936, 1048, 953, 982, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8288, 8230, 8183, 8238, 8316, 8125, 8021] avg=8192.0000 max=8316 max_rank=5 min=8021 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1034, 997, 1015, 1012, 976, 1054, 1032, 1039, 16, 12, 0, 0, 0, 0, 0, 0, 1013, 1047, 1023, 1001, 1040, 1002, 1026, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1027, 1028, 1012, 1019, 1034, 1049, 1027, 22, 0, 0, 0, 0, 0, 0, 0, 999, 1055, 1004, 1006, 1033, 990, 1047, 965, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1057, 1016, 998, 1060, 1019, 1030, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1010, 987, 1052, 986, 1040, 997, 1012, 39, 0, 0, 0, 0, 0, 0, 0, 1016, 1030, 1075, 988, 1026, 1050, 1073, 965, 24, 0, 0, 0, 0, 0, 0, 0, 1042, 1044, 1028, 995, 1054, 995, 1020, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8245, 8202, 8099, 8191, 8159, 8247, 8206] avg=8192.0000 max=8247 max_rank=6 min=8099 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[937, 1056, 1018, 1017, 1027, 1026, 1077, 1007, 16, 13, 0, 0, 0, 0, 0, 0, 1023, 1059, 1036, 1050, 1018, 1012, 964, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1031, 984, 1031, 1009, 1024, 1014, 1034, 23, 0, 0, 0, 0, 0, 0, 0, 1060, 1037, 1030, 1001, 1036, 1048, 991, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1048, 1012, 1065, 1030, 1028, 1015, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 977, 985, 1014, 1011, 981, 1005, 984, 1030, 39, 0, 0, 0, 0, 0, 0, 0, 1034, 1037, 997, 997, 1055, 1017, 1018, 984, 25, 1, 0, 0, 0, 0, 0, 0, 1079, 1024, 1034, 1021, 1020, 1024, 1077, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8205, 8212, 8254, 8196, 8026, 8165, 8284] avg=8192.0000 max=8284 max_rank=7 min=8026 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1030, 1028, 1031, 1047, 1014, 1037, 989, 1055, 16, 12, 0, 0, 0, 0, 0, 0, 1059, 973, 1012, 976, 1009, 1023, 1060, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1008, 996, 1047, 1018, 1031, 1022, 1040, 23, 0, 0, 0, 0, 0, 0, 0, 1061, 1031, 1054, 1004, 977, 1025, 985, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1042, 1048, 1028, 1018, 1015, 971, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1030, 1038, 1065, 1018, 974, 1014, 1019, 38, 0, 0, 0, 0, 0, 0, 0, 1080, 985, 1059, 1046, 1055, 977, 980, 1021, 23, 1, 0, 0, 0, 0, 0, 0, 1002, 1054, 1056, 993, 1065, 1032, 1032, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8119, 8200, 8175, 8103, 8189, 8227, 8264] avg=8192.0000 max=8264 max_rank=7 min=8103 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1036, 1022, 983, 977, 1077, 1006, 1010, 1027, 16, 11, 0, 0, 0, 0, 0, 0, 1010, 1007, 1025, 939, 987, 1035, 1064, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1025, 1041, 1017, 1000, 963, 1013, 1041, 23, 0, 0, 0, 0, 0, 0, 0, 1004, 1045, 1039, 1101, 1001, 1031, 982, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1034, 1043, 1049, 1017, 1036, 1076, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 985, 1025, 1032, 1036, 1020, 1016, 1031, 39, 0, 0, 0, 0, 0, 0, 0, 1013, 1042, 971, 1066, 1057, 1020, 1002, 1031, 22, 1, 0, 0, 0, 0, 0, 0, 1007, 1039, 1020, 949, 1031, 1033, 996, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8079, 8154, 8263, 8264, 8247, 8225, 8139] avg=8192.0000 max=8264 max_rank=4 min=8079 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32875 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1013, 991, 1027, 1003, 979, 973, 1065, 1060, 16, 12, 0, 0, 0, 0, 0, 0, 1034, 1049, 1048, 959, 1041, 1056, 1015, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1036, 991, 971, 1009, 1010, 1064, 1050, 23, 0, 0, 0, 0, 0, 0, 0, 996, 1025, 1037, 1071, 1007, 985, 1036, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1051, 1038, 997, 956, 1030, 1086, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1005, 1003, 1056, 1017, 1007, 990, 1070, 39, 0, 0, 0, 0, 0, 0, 0, 1046, 1093, 1010, 1034, 1006, 1015, 979, 1025, 23, 1, 0, 0, 0, 0, 0, 0, 1033, 1002, 1022, 1046, 1032, 1031, 1008, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8243, 8176, 8214, 8152, 8162, 8232, 8218] avg=8192.0000 max=8243 max_rank=1 min=8139 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1034, 1041, 983, 993, 1031, 1050, 1033, 1072, 19, 16, 0, 0, 0, 0, 0, 0, 1045, 1042, 1004, 1028, 1011, 1048, 1008, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1007, 1038, 1032, 974, 1045, 1016, 1017, 30, 0, 0, 0, 0, 0, 0, 0, 999, 1048, 1025, 1025, 994, 1053, 1023, 1051, 4, 0, 0, 0, 0, 0, 0, 0, 979, 951, 1004, 1002, 1021, 992, 1061, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1049, 1002, 1033, 999, 988, 1019, 1036, 44, 0, 0, 0, 0, 0, 0, 0, 1016, 1016, 1005, 1007, 1001, 1013, 1002, 1060, 27, 3, 0, 0, 0, 0, 0, 0, 1014, 1029, 1058, 1026, 1004, 1051, 1076, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8272, 8186, 8193, 8222, 8058, 8172, 8150, 8283] avg=8192.0000 max=8283 max_rank=7 min=8058 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32663 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8064, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7936, 8320, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8064, 8064, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8192, 8448, 8192, 8192, 8192, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8064, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8320, 8448, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1043, 1030, 1017, 1081, 1000, 1053, 1024, 1087, 1029, 1055, 1026, 1034, 980, 1048, 1052, 1059, 1058, 1043, 1005, 1033, 996, 1074, 998, 1026, 998, 1023, 1024, 1046, 1047, 1012, 1044, 1016, 1026, 1009, 1045, 1002, 1005, 1002, 1057, 1027, 1069, 922, 1027, 1043, 1037, 998, 999, 1038, 1023, 1040, 1045, 983, 913, 987, 1037, 977, 1022, 1037, 1043, 1017, 997, 1020, 974, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[989, 1045, 1039, 1017, 1051, 1069, 1040, 1006, 1055, 999, 1017, 1031, 1023, 1016, 1043, 1050, 1057, 1018, 1048, 1024, 1002, 1010, 1053, 1032, 974, 992, 991, 1004, 979, 1065, 1025, 1045, 1040, 1029, 1044, 1054, 1002, 1061, 1080, 970, 994, 1000, 1073, 1045, 995, 1036, 1050, 1025, 1025, 999, 1027, 1023, 978, 1035, 1045, 1018, 1003, 1047, 1000, 994, 1044, 994, 1023, 974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1081, 1004, 1033, 1037, 964, 1011, 1007, 1005, 1029, 1000, 1030, 1033, 994, 1011, 1017, 1025, 1048, 1019, 1043, 1034, 1047, 995, 1007, 993, 999, 979, 1000, 1001, 978, 1065, 1049, 981, 1022, 1076, 1023, 1049, 1051, 1060, 1075, 1039, 1030, 1049, 1056, 995, 1009, 1006, 1017, 991, 1076, 1036, 1014, 1017, 985, 1049, 1026, 1055, 1069, 1001, 1014, 1040, 1026, 1046, 1005, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1059, 1069, 1020, 1050, 1070, 1033, 984, 1053, 1042, 1061, 1028, 1025, 1027, 979, 1015, 1030, 1066, 1038, 1055, 1008, 1017, 1017, 1051, 993, 1020, 999, 998, 1023, 1043, 1005, 1032, 1039, 1002, 1038, 1021, 1055, 1009, 1041, 1023, 1017, 1019, 1030, 1022, 1003, 1051, 1017, 1048, 1032, 1041, 990, 997, 1045, 1045, 1001, 1029, 1011, 975, 949, 1022, 1004, 983, 1067, 994, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1019, 1027, 1025, 1047, 1020, 1026, 1064, 1082, 1031, 1019, 1018, 995, 1041, 1006, 1084, 1006, 1009, 1020, 964, 985, 1062, 1035, 1020, 956, 1058, 1012, 982, 965, 1040, 1063, 983, 1069, 971, 1017, 1065, 1061, 1012, 1017, 977, 1044, 981, 989, 1056, 1033, 1034, 1093, 1038, 1034, 996, 1019, 1030, 1028, 1051, 1027, 1042, 1016, 1024, 1053, 963, 1039, 1041, 1022, 1013, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1063, 996, 1028, 1077, 956, 1005, 1014, 1000, 988, 1009, 1042, 1047, 1047, 1025, 1016, 1040, 982, 1001, 1035, 986, 1021, 1022, 1029, 986, 1001, 1050, 1106, 1066, 1030, 1008, 1067, 1047, 1023, 1066, 999, 1063, 1052, 995, 1033, 1001, 1019, 1052, 1025, 1002, 967, 995, 1005, 1048, 1055, 1029, 1043, 1004, 1047, 1022, 987, 1064, 1046, 996, 1034, 989, 1009, 1019, 1037, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1045, 1005, 983, 1023, 1047, 1011, 1073, 1008, 1028, 998, 1003, 1022, 1031, 1034, 1002, 1047, 1049, 980, 1025, 1044, 1017, 992, 1057, 1062, 1033, 1066, 1027, 1032, 1054, 1037, 1051, 993, 1062, 1037, 1053, 1030, 1046, 1045, 999, 1014, 1015, 1046, 1046, 1014, 990, 961, 1037, 1054, 1004, 997, 1028, 978, 1034, 974, 964, 990, 1019, 1051, 1029, 1069, 988, 1038, 1039, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1048, 1062, 1028, 1025, 1052, 987, 1061, 996, 999, 1020, 992, 1004, 1031, 1044, 970, 1038, 1060, 1025, 1011, 988, 1081, 1021, 1032, 998, 1033, 1038, 974, 1029, 1021, 1006, 1081, 965, 1034, 1008, 1061, 1027, 1029, 1004, 1038, 1076, 1013, 1045, 1036, 1018, 1012, 1024, 1005, 996, 987, 1027, 1011, 1055, 1031, 1034, 1027, 980, 1011, 1040, 1046, 1006, 1030, 1053, 1010, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[989, 1045, 1039, 966, 1051, 1069, 1040, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 999, 1017, 1031, 1023, 1016, 1043, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1018, 1048, 1024, 1002, 1010, 1053, 1032, 8, 0, 0, 0, 0, 0, 0, 0, 974, 992, 991, 1004, 979, 1065, 1025, 1045, 5, 0, 0, 0, 0, 0, 0, 0, 1040, 1029, 1044, 995, 1002, 1061, 1080, 970, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1000, 1073, 1045, 995, 1036, 1050, 1025, 15, 0, 0, 0, 0, 0, 0, 0, 1025, 999, 1027, 1023, 978, 1035, 1045, 1018, 48, 0, 0, 0, 0, 0, 0, 0, 1003, 1047, 1000, 994, 1044, 994, 1023, 974, 23, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8234, 8252, 8080, 8221, 8233, 8198, 8113] avg=8192.0000 max=8252 max_rank=2 min=8080 min_rank=3 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1043, 1030, 1017, 1026, 1000, 1053, 1024, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1055, 1026, 1034, 980, 1048, 1052, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1043, 1005, 1033, 996, 1074, 998, 1026, 9, 0, 0, 0, 0, 0, 0, 0, 998, 1023, 1024, 1046, 1047, 1012, 1044, 1016, 6, 0, 0, 0, 0, 0, 0, 0, 1026, 1009, 1045, 945, 1005, 1002, 1057, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 922, 1027, 1043, 1037, 998, 999, 1038, 16, 0, 0, 0, 0, 0, 0, 0, 1023, 1040, 1045, 983, 913, 987, 1037, 977, 46, 0, 0, 0, 0, 0, 0, 0, 1022, 1037, 1043, 1017, 997, 1020, 974, 1054, 24, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8280, 8283, 8242, 8216, 8116, 8149, 8051, 8199] avg=8192.0000 max=8283 max_rank=1 min=8051 min_rank=6 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1081, 1004, 1033, 984, 964, 1011, 1007, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1000, 1030, 1033, 994, 1011, 1017, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1019, 1043, 1034, 1047, 995, 1007, 993, 9, 0, 0, 0, 0, 0, 0, 0, 999, 979, 1000, 1001, 978, 1065, 1049, 981, 6, 0, 0, 0, 0, 0, 0, 0, 1022, 1076, 1023, 990, 1051, 1060, 1075, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1049, 1056, 995, 1009, 1006, 1017, 991, 15, 0, 0, 0, 0, 0, 0, 0, 1076, 1036, 1014, 1017, 985, 1049, 1026, 1055, 48, 0, 0, 0, 0, 0, 0, 0, 1069, 1001, 1014, 1040, 1026, 1046, 1005, 1010, 23, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 8139, 8195, 8058, 8336, 8168, 8306, 8245] avg=8192.0000 max=8336 max_rank=4 min=8058 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=33055 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1059, 1069, 1020, 997, 1070, 1033, 984, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1061, 1028, 1025, 1027, 979, 1015, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1038, 1055, 1008, 1017, 1017, 1051, 993, 9, 0, 0, 0, 0, 0, 0, 0, 1020, 999, 998, 1023, 1043, 1005, 1032, 1039, 6, 0, 0, 0, 0, 0, 0, 0, 1002, 1038, 1021, 996, 1009, 1041, 1023, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1030, 1022, 1003, 1051, 1017, 1048, 1032, 15, 0, 0, 0, 0, 0, 0, 0, 1041, 990, 997, 1045, 1045, 1001, 1029, 1011, 48, 0, 0, 0, 0, 0, 0, 0, 975, 949, 1022, 1004, 983, 1067, 994, 1006, 23, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8285, 8207, 8254, 8165, 8147, 8237, 8207, 8034] avg=8192.0000 max=8285 max_rank=0 min=8034 min_rank=7 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32625 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1019, 1027, 1025, 994, 1020, 1026, 1064, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1019, 1018, 995, 1041, 1006, 1084, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1020, 964, 985, 1062, 1035, 1020, 956, 9, 0, 0, 0, 0, 0, 0, 0, 1058, 1012, 982, 965, 1040, 1063, 983, 1069, 6, 0, 0, 0, 0, 0, 0, 0, 971, 1017, 1065, 1002, 1012, 1017, 977, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 981, 989, 1056, 1033, 1034, 1093, 1038, 1034, 15, 0, 0, 0, 0, 0, 0, 0, 996, 1019, 1030, 1028, 1051, 1027, 1042, 1016, 48, 0, 0, 0, 0, 0, 0, 0, 1024, 1053, 963, 1039, 1041, 1022, 1013, 1017, 23, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8200, 8060, 8178, 8105, 8273, 8257, 8206] avg=8192.0000 max=8273 max_rank=5 min=8060 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1045, 1005, 983, 972, 1047, 1011, 1073, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 998, 1003, 1022, 1031, 1034, 1002, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 980, 1025, 1044, 1017, 992, 1057, 1062, 8, 0, 0, 0, 0, 0, 0, 0, 1033, 1066, 1027, 1032, 1054, 1037, 1051, 993, 5, 0, 0, 0, 0, 0, 0, 0, 1062, 1037, 1053, 972, 1046, 1045, 999, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1046, 1046, 1014, 990, 961, 1037, 1054, 15, 0, 0, 0, 0, 0, 0, 0, 1004, 997, 1028, 978, 1034, 974, 964, 990, 47, 0, 0, 0, 0, 0, 0, 0, 1019, 1051, 1029, 1069, 988, 1038, 1039, 1006, 23, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8165, 8234, 8298, 8228, 8178, 8016, 8273] avg=8192.0000 max=8298 max_rank=3 min=8016 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1063, 996, 1028, 1007, 956, 1005, 1014, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1009, 1042, 1047, 1047, 1025, 1016, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1001, 1035, 986, 1021, 1022, 1029, 986, 13, 0, 0, 0, 0, 0, 0, 0, 1001, 1050, 1106, 1066, 1030, 1008, 1067, 1047, 9, 4, 0, 0, 0, 0, 0, 0, 1023, 1066, 999, 995, 1052, 995, 1033, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1052, 1025, 1002, 967, 995, 1005, 1048, 21, 0, 0, 0, 0, 0, 0, 0, 1055, 1029, 1043, 1004, 1047, 1022, 987, 1064, 51, 0, 0, 0, 0, 0, 0, 0, 1046, 996, 1034, 989, 1009, 1019, 1037, 1020, 27, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8069, 8210, 8075, 8388, 8164, 8134, 8302, 8194] avg=8192.0000 max=8388 max_rank=3 min=8069 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1048, 1062, 1028, 973, 1052, 987, 1061, 996, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1020, 992, 1004, 1031, 1044, 970, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1025, 1011, 988, 1081, 1021, 1032, 998, 8, 0, 0, 0, 0, 0, 0, 0, 1033, 1038, 974, 1029, 1021, 1006, 1081, 965, 6, 0, 0, 0, 0, 0, 0, 0, 1034, 1008, 1061, 969, 1029, 1004, 1038, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1045, 1036, 1018, 1012, 1024, 1005, 996, 15, 0, 0, 0, 0, 0, 0, 0, 987, 1027, 1011, 1055, 1031, 1034, 1027, 980, 47, 0, 0, 0, 0, 0, 0, 0, 1011, 1040, 1046, 1006, 1030, 1053, 1010, 1042, 23, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8098, 8224, 8153, 8219, 8164, 8199, 8272] avg=8192.0000 max=8272 max_rank=7 min=8098 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 7936, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8448, 8320, 8192, 7936, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8448, 8192, 8192, 8192, 8320, 8192, 8320, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8448, 8192, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8448, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1037, 980, 973, 1034, 1002, 1015, 1035, 1008, 1045, 1044, 999, 990, 1007, 1026, 1082, 980, 1014, 1015, 1024, 1024, 989, 1088, 1057, 1024, 1056, 1065, 1037, 1034, 1069, 982, 1044, 1003, 1028, 1044, 1030, 1030, 1047, 1030, 1016, 1000, 1027, 1019, 1067, 1039, 1078, 1019, 982, 1048, 990, 1037, 1052, 1015, 955, 977, 1007, 1026, 1056, 1004, 1025, 1018, 994, 1012, 1065, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1035, 984, 1020, 1022, 1040, 1007, 986, 1020, 1052, 997, 969, 1013, 1018, 1029, 1007, 1040, 1014, 1013, 1097, 1099, 999, 998, 1048, 973, 1089, 1111, 1093, 1075, 1008, 992, 988, 1079, 981, 1069, 1019, 1023, 1051, 1040, 981, 1003, 1057, 1005, 1005, 996, 1073, 1031, 1033, 1033, 1020, 994, 1038, 1015, 1018, 1045, 1009, 1001, 970, 1026, 1018, 1001, 1039, 1028, 1040, 959] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[986, 1031, 1010, 985, 1060, 1024, 1024, 1000, 1033, 1052, 989, 1029, 1036, 1048, 1067, 1011, 934, 1035, 1060, 990, 1029, 1038, 1075, 1041, 1017, 999, 983, 990, 1056, 1050, 1020, 1005, 1019, 1039, 1032, 1051, 1043, 1024, 1009, 1050, 1043, 1015, 1056, 1026, 995, 1091, 1063, 986, 1009, 1018, 990, 1024, 1035, 999, 997, 1034, 1066, 1072, 1027, 1017, 989, 1021, 991, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1023, 1019, 1039, 1025, 1019, 1020, 1000, 1023, 1002, 1003, 1044, 1015, 993, 1073, 1037, 1064, 1022, 1009, 1049, 1046, 1046, 1047, 1007, 1011, 1035, 1061, 1038, 1032, 1033, 1038, 1038, 1018, 1013, 1015, 995, 1031, 990, 1019, 1044, 1016, 1013, 1067, 1022, 1032, 1042, 1008, 984, 1025, 1021, 997, 1033, 999, 1042, 1026, 1033, 1041, 1029, 1058, 1026, 1000, 989, 1000, 1006, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[992, 1064, 1034, 1010, 1055, 1040, 1016, 1026, 1059, 1064, 1034, 992, 1057, 1045, 996, 1063, 1048, 1027, 1073, 1018, 990, 1034, 1006, 1070, 1050, 1025, 1008, 998, 991, 1021, 1057, 997, 1059, 1017, 1030, 1018, 1033, 988, 1011, 1058, 1045, 989, 1020, 1000, 1033, 967, 1091, 1022, 1037, 1011, 975, 1054, 1033, 1008, 1027, 1019, 1021, 1054, 1012, 967, 980, 1021, 976, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1094, 1050, 972, 1006, 1027, 1005, 1082, 989, 1024, 1049, 992, 994, 1060, 1030, 1011, 1034, 1014, 1041, 1028, 1043, 1022, 1098, 1066, 990, 1014, 987, 1047, 1039, 1048, 1021, 1023, 1032, 1029, 1011, 1033, 1059, 1022, 1056, 1016, 987, 1045, 1031, 1059, 1048, 993, 1007, 1055, 1067, 1019, 979, 996, 1011, 982, 1016, 981, 1081, 1007, 997, 983, 1016, 984, 1018, 1033, 983] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1052, 1064, 1053, 968, 958, 1019, 1078, 1062, 1014, 1037, 1015, 1052, 992, 1021, 1005, 1034, 1038, 1082, 998, 1010, 1021, 1001, 1057, 1024, 1065, 1039, 1019, 1018, 1017, 1047, 1028, 1023, 1041, 977, 1062, 1060, 1010, 998, 1037, 986, 989, 993, 995, 1029, 983, 994, 1043, 1037, 1011, 1022, 1050, 1011, 1028, 1010, 1026, 1023, 994, 1022, 1026, 987, 1077, 1005, 1066, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1011, 999, 1010, 1000, 994, 1006, 1007, 1017, 1095, 1034, 1049, 1041, 1048, 1052, 1035, 1050, 1055, 977, 1006, 1037, 976, 1020, 1013, 999, 1031, 1061, 1018, 1063, 1072, 1023, 1018, 998, 1057, 1039, 1004, 1040, 998, 1080, 989, 1056, 1029, 1015, 1004, 1061, 984, 1038, 1056, 1049, 1010, 1030, 984, 1007, 1004, 1001, 1016, 1033, 1033, 1026, 1006, 1004, 1040, 1037, 986, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1035, 984, 1020, 1022, 1040, 1007, 986, 1020, 22, 13, 0, 0, 0, 0, 0, 0, 1052, 997, 969, 1013, 1018, 988, 1007, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1013, 1063, 1099, 999, 998, 1048, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1111, 1093, 1075, 1008, 992, 988, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1069, 1019, 1010, 1051, 1040, 981, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1005, 1005, 996, 1073, 1031, 1008, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 994, 1038, 1015, 1018, 1045, 1009, 1001, 34, 31, 3, 0, 0, 0, 0, 0, 970, 1026, 1018, 1001, 1039, 1028, 1040, 959, 65, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8084, 8207, 8370, 8154, 8208, 8208, 8156] avg=8192.0000 max=8370 max_rank=3 min=8084 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[986, 1031, 1010, 985, 1060, 1024, 1024, 1000, 23, 13, 0, 0, 0, 0, 0, 0, 1033, 1052, 989, 1029, 1036, 1007, 1067, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 934, 1035, 1027, 990, 1029, 1038, 1075, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 957, 999, 983, 990, 1056, 1050, 1020, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1039, 1032, 1038, 1043, 1024, 1009, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1015, 1056, 1026, 995, 1091, 1036, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1018, 990, 1024, 1035, 999, 997, 1034, 33, 31, 4, 0, 0, 0, 0, 0, 1066, 1072, 1027, 1017, 989, 1021, 991, 998, 60, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8224, 8169, 8060, 8254, 8248, 8174, 8251] avg=8192.0000 max=8254 max_rank=4 min=8060 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32927 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1037, 980, 973, 1034, 1002, 1015, 1035, 1008, 21, 13, 0, 0, 0, 0, 0, 0, 1045, 1044, 999, 990, 1007, 986, 1082, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1015, 992, 1024, 989, 1088, 1057, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1065, 1037, 1034, 1069, 982, 1044, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1044, 1030, 1017, 1047, 1030, 1016, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1019, 1067, 1039, 1078, 1019, 958, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1037, 1052, 1015, 955, 977, 1007, 1026, 32, 30, 3, 0, 0, 0, 0, 0, 1056, 1004, 1025, 1018, 994, 1012, 1065, 1017, 63, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8133, 8203, 8227, 8212, 8255, 8124, 8264] avg=8192.0000 max=8264 max_rank=7 min=8118 min_rank=0 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1023, 1019, 1039, 1025, 1019, 1020, 1000, 1023, 21, 13, 0, 0, 0, 0, 0, 0, 1002, 1003, 1044, 1015, 993, 1031, 1037, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1009, 1017, 1046, 1046, 1047, 1007, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1061, 1038, 1032, 1033, 1038, 1038, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1015, 995, 1018, 990, 1019, 1044, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1067, 1022, 1032, 1042, 1008, 960, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 997, 1033, 999, 1042, 1026, 1033, 1041, 32, 32, 3, 0, 0, 0, 0, 0, 1029, 1058, 1026, 1000, 989, 1000, 1006, 991, 61, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8189, 8205, 8232, 8110, 8169, 8259, 8170] avg=8192.0000 max=8259 max_rank=6 min=8110 min_rank=4 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1094, 1050, 972, 1006, 1027, 1005, 1082, 989, 23, 13, 0, 0, 0, 0, 0, 0, 1024, 1049, 992, 994, 1060, 989, 1011, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1041, 996, 1043, 1022, 1098, 1066, 990, 0, 0, 0, 0, 0, 0, 0, 0, 954, 987, 1047, 1039, 1048, 1021, 1023, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1011, 1033, 1046, 1022, 1056, 1016, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1031, 1059, 1048, 993, 1007, 1028, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 979, 996, 1011, 982, 1016, 981, 1081, 32, 31, 4, 0, 0, 0, 0, 0, 1007, 997, 983, 1016, 984, 1018, 1033, 983, 60, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8153, 8270, 8151, 8200, 8278, 8132, 8091] avg=8192.0000 max=8278 max_rank=5 min=8091 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1011, 999, 1010, 1000, 994, 1006, 1007, 1017, 23, 13, 0, 0, 0, 0, 0, 0, 1095, 1034, 1049, 1041, 1048, 1011, 1035, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 977, 975, 1037, 976, 1020, 1013, 999, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1061, 1018, 1063, 1072, 1023, 1018, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1039, 1004, 1027, 998, 1080, 989, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1015, 1004, 1061, 984, 1038, 1029, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1030, 984, 1007, 1004, 1001, 1016, 1033, 31, 31, 4, 0, 0, 0, 0, 0, 1033, 1026, 1006, 1004, 1040, 1037, 986, 1005, 61, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8080, 8363, 8052, 8223, 8250, 8209, 8151, 8208] avg=8192.0000 max=8363 max_rank=1 min=8052 min_rank=2 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1052, 1064, 1053, 968, 958, 1019, 1078, 1062, 22, 13, 0, 0, 0, 0, 0, 0, 1014, 1037, 1015, 1052, 992, 981, 1005, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1082, 967, 1010, 1021, 1001, 1057, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1039, 1019, 1018, 1017, 1047, 1028, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 977, 1062, 1047, 1010, 998, 1037, 986, 0, 0, 0, 0, 0, 0, 0, 0, 989, 993, 995, 1029, 983, 994, 1017, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1022, 1050, 1011, 1028, 1010, 1026, 1023, 31, 30, 4, 0, 0, 0, 0, 0, 994, 1022, 1026, 987, 1077, 1005, 1066, 1033, 63, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8289, 8130, 8200, 8193, 8158, 8037, 8246, 8283] avg=8192.0000 max=8289 max_rank=0 min=8037 min_rank=5 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[992, 1064, 1034, 1010, 1055, 1040, 1016, 1026, 28, 16, 0, 0, 0, 0, 0, 0, 1059, 1064, 1034, 992, 1057, 995, 996, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1027, 1037, 1018, 990, 1034, 1006, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1025, 1008, 998, 991, 1021, 1057, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1017, 1030, 1002, 1033, 988, 1011, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 989, 1020, 1000, 1033, 967, 1056, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1011, 975, 1054, 1033, 1008, 1027, 1019, 36, 35, 7, 0, 0, 0, 0, 0, 1021, 1054, 1012, 967, 980, 1021, 976, 1000, 67, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8260, 8230, 8080, 8198, 8132, 8242, 8113] avg=8192.0000 max=8281 max_rank=0 min=8080 min_rank=3 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8320, 8192, 8192, 8320, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8448, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8064, 8192, 8192, 8192, 8064, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8064, 8192, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 384, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7936, 8448, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1031, 971, 1034, 1059, 1033, 989, 1016, 1008, 1035, 997, 1004, 1048, 977, 1026, 988, 997, 1034, 1061, 1030, 1057, 1007, 1010, 1003, 1039, 1002, 1036, 1008, 1051, 1018, 995, 999, 1028, 1007, 1013, 966, 1047, 1080, 1048, 996, 1020, 1029, 1045, 1047, 1060, 1088, 1025, 1020, 1025, 1039, 1036, 1011, 1062, 1027, 987, 1046, 1005, 1010, 1027, 1011, 1045, 1009, 1031, 1052, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1033, 1027, 1027, 1022, 1049, 1096, 1089, 1053, 1053, 993, 997, 941, 1049, 1006, 997, 1073, 1034, 1001, 1009, 1026, 1041, 1019, 1028, 1027, 1067, 1035, 1016, 1036, 1025, 1048, 1046, 1031, 1008, 1010, 1026, 987, 1038, 978, 1020, 986, 1056, 1015, 1008, 1037, 1016, 1011, 1051, 1029, 985, 1049, 1018, 994, 995, 1024, 1033, 1021, 1018, 1042, 1041, 1003, 1057, 981, 1001, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1026, 986, 1040, 1004, 1058, 964, 1022, 1058, 1051, 1071, 1007, 1026, 1002, 1049, 1027, 993, 1072, 1032, 1077, 1069, 1015, 1041, 1063, 1001, 1003, 1026, 1026, 1043, 1009, 1042, 1045, 994, 1021, 1013, 1037, 1062, 1009, 994, 1035, 1022, 1014, 1006, 1003, 1055, 1054, 1064, 1006, 992, 1016, 1014, 1056, 1019, 1037, 1022, 1038, 1003, 997, 1046, 1023, 1005, 983, 968, 988, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1026, 1031, 1021, 1029, 1030, 1083, 1004, 1071, 1040, 1004, 1036, 991, 987, 1044, 1008, 1062, 1040, 1017, 1077, 995, 986, 1000, 1069, 1018, 992, 1064, 1044, 1058, 999, 977, 1020, 983, 1024, 1048, 992, 995, 1035, 1026, 1005, 1012, 1062, 1009, 996, 1030, 1024, 986, 1043, 974, 1017, 1024, 1048, 983, 1002, 1034, 1018, 1044, 1002, 1069, 1043, 1024, 1010, 1044, 1021, 1086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1017, 1027, 1040, 1021, 996, 1028, 1050, 1009, 993, 1016, 1013, 1063, 999, 1054, 1062, 1072, 1018, 1057, 988, 1033, 1043, 1073, 1013, 1024, 1059, 1012, 1015, 1018, 1015, 1063, 1011, 1019, 993, 1015, 1049, 966, 1035, 993, 1043, 1035, 1024, 1020, 998, 1024, 1016, 1039, 1007, 1037, 1031, 1005, 1021, 1032, 1047, 983, 990, 1024, 1038, 1052, 1007, 987, 1065, 985, 1031, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1059, 1056, 1006, 1043, 1061, 1035, 1018, 1036, 1024, 952, 989, 1044, 991, 1072, 1003, 1023, 987, 1001, 1084, 1078, 1046, 1032, 987, 1044, 1018, 1011, 938, 1027, 970, 1073, 1023, 1045, 1036, 1037, 967, 1041, 1031, 1025, 969, 1035, 1018, 1031, 1005, 1027, 1023, 1066, 1004, 1035, 1038, 1055, 989, 1040, 1002, 1035, 997, 996, 1048, 1065, 1019, 1013, 1023, 1018, 1016, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[980, 970, 1082, 993, 1030, 1058, 1044, 1023, 1005, 1039, 1045, 1053, 1025, 995, 995, 1018, 1018, 1010, 1024, 1015, 1025, 1004, 1059, 991, 988, 1034, 1047, 1058, 1044, 1084, 987, 989, 1054, 1001, 1030, 997, 1008, 1013, 1071, 1010, 1052, 1021, 1038, 1054, 1066, 1078, 999, 1040, 1049, 1000, 1055, 1051, 1043, 992, 1010, 1006, 1014, 1004, 978, 1011, 1030, 988, 1032, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1009, 1031, 1052, 1000, 996, 1045, 1012, 1043, 1061, 1025, 1009, 1068, 1011, 1007, 1004, 1071, 1015, 1055, 1043, 1045, 1055, 1006, 1017, 992, 1028, 998, 996, 1014, 1031, 1041, 967, 1028, 1036, 1030, 1033, 1038, 975, 1026, 1068, 1067, 1008, 987, 1060, 1031, 998, 996, 1016, 1008, 1022, 1014, 1011, 995, 1052, 991, 1041, 996, 1016, 1048, 1011, 1007, 1047, 1046, 1028, 1059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1031, 971, 995, 1059, 1033, 989, 1016, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 997, 1004, 1048, 977, 1026, 988, 997, 19, 0, 0, 0, 0, 0, 0, 0, 1034, 1061, 989, 1057, 1007, 1010, 1003, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1036, 1008, 1051, 1018, 995, 999, 1028, 9, 5, 0, 0, 0, 0, 0, 0, 1007, 1013, 966, 1047, 1080, 1048, 996, 1020, 30, 0, 0, 0, 0, 0, 0, 0, 1029, 1045, 1047, 1039, 1088, 1025, 1020, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1036, 1011, 1062, 1027, 987, 1046, 1005, 15, 11, 0, 0, 0, 0, 0, 0, 1010, 1027, 1011, 1045, 1009, 1031, 1052, 1031, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 8091, 8200, 8151, 8207, 8318, 8239, 8228] avg=8192.0000 max=8318 max_rank=5 min=8091 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32992 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1033, 1027, 989, 1022, 1049, 1096, 1089, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 993, 997, 941, 1049, 1006, 997, 1073, 19, 0, 0, 0, 0, 0, 0, 0, 1034, 1001, 968, 1026, 1041, 1019, 1028, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1035, 1016, 1036, 1025, 1048, 1046, 1031, 8, 5, 0, 0, 0, 0, 0, 0, 1008, 1010, 1026, 987, 1038, 978, 1020, 986, 30, 0, 0, 0, 0, 0, 0, 0, 1056, 1015, 1008, 1017, 1016, 1011, 1051, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1049, 1018, 994, 995, 1024, 1033, 1021, 14, 11, 0, 0, 0, 0, 0, 0, 1018, 1042, 1041, 1003, 1057, 981, 1001, 1004, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8358, 8128, 8144, 8317, 8083, 8203, 8144, 8159] avg=8192.0000 max=8358 max_rank=0 min=8083 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32589 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1026, 986, 1001, 1004, 1058, 964, 1022, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1071, 1007, 1026, 1002, 1049, 1027, 993, 19, 0, 0, 0, 0, 0, 0, 0, 1072, 1032, 1033, 1069, 1015, 1041, 1063, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1026, 1026, 1043, 1009, 1042, 1045, 994, 9, 5, 0, 0, 0, 0, 0, 0, 1021, 1013, 1037, 1062, 1009, 994, 1035, 1022, 32, 0, 0, 0, 0, 0, 0, 0, 1014, 1006, 1003, 1034, 1054, 1064, 1006, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1014, 1056, 1019, 1037, 1022, 1038, 1003, 15, 12, 0, 0, 0, 0, 0, 0, 997, 1046, 1023, 1005, 983, 968, 988, 992, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8119, 8245, 8326, 8202, 8225, 8173, 8232, 8014] avg=8192.0000 max=8326 max_rank=2 min=8014 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1017, 1027, 1001, 1021, 996, 1028, 1050, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1016, 1013, 1063, 999, 1054, 1062, 1072, 19, 0, 0, 0, 0, 0, 0, 0, 1018, 1057, 948, 1033, 1043, 1073, 1013, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1012, 1015, 1018, 1015, 1063, 1011, 1019, 8, 5, 0, 0, 0, 0, 0, 0, 993, 1015, 1049, 966, 1035, 993, 1043, 1035, 29, 0, 0, 0, 0, 0, 0, 0, 1024, 1020, 998, 1004, 1016, 1039, 1007, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1005, 1021, 1032, 1047, 983, 990, 1024, 15, 11, 0, 0, 0, 0, 0, 0, 1038, 1052, 1007, 987, 1065, 985, 1031, 1023, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8291, 8209, 8225, 8158, 8145, 8159, 8200] avg=8192.0000 max=8291 max_rank=1 min=8145 min_rank=5 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1059, 1056, 969, 1043, 1061, 1035, 1018, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 952, 989, 1044, 991, 1072, 1003, 1023, 18, 0, 0, 0, 0, 0, 0, 0, 987, 1001, 1040, 1078, 1046, 1032, 987, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1011, 938, 1027, 970, 1073, 1023, 1045, 8, 5, 0, 0, 0, 0, 0, 0, 1036, 1037, 967, 1041, 1031, 1025, 969, 1035, 32, 0, 0, 0, 0, 0, 0, 0, 1018, 1031, 1005, 1007, 1023, 1066, 1004, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1055, 989, 1040, 1002, 1035, 997, 996, 14, 12, 0, 0, 0, 0, 0, 0, 1048, 1065, 1019, 1013, 1023, 1018, 1016, 1056, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8116, 8215, 8118, 8173, 8189, 8178, 8270] avg=8192.0000 max=8277 max_rank=0 min=8116 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1026, 1031, 983, 1029, 1030, 1083, 1004, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1004, 1036, 991, 987, 1044, 1008, 1062, 19, 0, 0, 0, 0, 0, 0, 0, 1040, 1017, 1033, 995, 986, 1000, 1069, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1064, 1044, 1058, 999, 977, 1020, 983, 8, 5, 0, 0, 0, 0, 0, 0, 1024, 1048, 992, 995, 1035, 1026, 1005, 1012, 32, 0, 0, 0, 0, 0, 0, 0, 1062, 1009, 996, 1010, 1024, 986, 1043, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1024, 1048, 983, 1002, 1034, 1018, 1044, 14, 12, 0, 0, 0, 0, 0, 0, 1002, 1069, 1043, 1024, 1010, 1044, 1021, 1086, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8191, 8158, 8150, 8169, 8104, 8196, 8311] avg=8192.0000 max=8311 max_rank=7 min=8104 min_rank=5 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[980, 970, 1041, 993, 1030, 1058, 1044, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1039, 1045, 1053, 1025, 995, 995, 1018, 20, 0, 0, 0, 0, 0, 0, 0, 1018, 1010, 983, 1015, 1025, 1004, 1059, 991, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1034, 1047, 1058, 1044, 1084, 987, 989, 9, 6, 0, 0, 0, 0, 0, 0, 1054, 1001, 1030, 997, 1008, 1013, 1071, 1010, 30, 0, 0, 0, 0, 0, 0, 0, 1052, 1021, 1038, 1033, 1066, 1078, 999, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1000, 1055, 1051, 1043, 992, 1010, 1006, 15, 11, 0, 0, 0, 0, 0, 0, 1014, 1004, 978, 1011, 1030, 988, 1032, 1009, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8195, 8105, 8246, 8214, 8327, 8232, 8078] avg=8192.0000 max=8327 max_rank=5 min=8078 min_rank=7 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1009, 1031, 999, 1000, 996, 1045, 1012, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1025, 1009, 1068, 1011, 1007, 1004, 1071, 23, 0, 0, 0, 0, 0, 0, 0, 1015, 1055, 994, 1045, 1055, 1006, 1017, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 998, 996, 1014, 1031, 1041, 967, 1028, 13, 11, 0, 0, 0, 0, 0, 0, 1036, 1030, 1033, 1038, 975, 1026, 1068, 1067, 34, 0, 0, 0, 0, 0, 0, 0, 1008, 987, 1060, 1004, 998, 996, 1016, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1014, 1011, 995, 1052, 991, 1041, 996, 19, 15, 0, 0, 0, 0, 0, 0, 1016, 1048, 1011, 1007, 1047, 1046, 1028, 1059, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8279, 8179, 8127, 8307, 8077, 8156, 8276] avg=8192.0000 max=8307 max_rank=4 min=8077 min_rank=5 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8448, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8064, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8448, 8192, 8192, 8320, 8064, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8064, 8320, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1003, 1019, 1042, 1021, 1050, 1036, 1046, 1091, 1028, 945, 974, 973, 1022, 1028, 974, 1038, 1025, 1002, 1021, 1034, 1010, 1075, 1057, 1002, 1005, 1026, 1060, 1022, 997, 1045, 1040, 1001, 1063, 1021, 948, 1030, 983, 1075, 1072, 989, 1011, 1068, 997, 1021, 1040, 980, 1006, 984, 1041, 972, 994, 1030, 1032, 1047, 1083, 975, 1035, 1043, 1059, 1027, 1069, 1046, 1018, 1065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[995, 1043, 1054, 962, 1028, 992, 1009, 1012, 1030, 1018, 982, 1024, 990, 1014, 1037, 1026, 1043, 988, 1056, 1033, 1014, 1058, 1049, 1039, 1045, 1038, 1035, 1051, 1040, 974, 1099, 1048, 991, 1055, 1061, 1049, 994, 969, 1022, 1070, 1011, 1023, 1007, 1014, 1044, 1039, 1025, 1034, 1075, 1022, 1012, 962, 1032, 1038, 1014, 1016, 1003, 1019, 1000, 1043, 986, 1061, 1016, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1025, 1013, 968, 992, 1012, 1052, 1009, 1052, 1051, 1016, 976, 1035, 1018, 1058, 1051, 1002, 1070, 1038, 998, 1010, 1067, 1070, 1018, 1036, 997, 1049, 1050, 995, 1016, 1018, 996, 1041, 1080, 1036, 1076, 1059, 977, 997, 1020, 1009, 993, 1017, 1043, 970, 1027, 992, 1038, 1094, 1035, 1044, 1023, 987, 1051, 1043, 1043, 972, 1019, 1052, 986, 1007, 1012, 1023, 988, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1042, 1002, 1042, 1038, 978, 1015, 1050, 1032, 1017, 1010, 1014, 1035, 1042, 1040, 1037, 1013, 1011, 1046, 979, 1028, 1005, 1009, 997, 1008, 1004, 1043, 1040, 1032, 993, 1020, 1013, 1031, 985, 1030, 1054, 1008, 1056, 1014, 1027, 1074, 1016, 1009, 983, 1076, 1044, 1013, 1035, 1018, 1031, 1022, 1064, 1012, 1003, 1048, 994, 1043, 1018, 1026, 1040, 971, 1048, 1047, 1051, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[994, 964, 1033, 1012, 1059, 1024, 990, 1060, 1054, 1022, 1043, 1025, 1058, 1057, 1002, 1061, 954, 1057, 1076, 1045, 1046, 1034, 993, 1021, 1013, 1031, 993, 985, 996, 1021, 1015, 1018, 1043, 1017, 1074, 1007, 1013, 1003, 983, 1022, 1019, 1026, 1040, 1006, 1056, 1000, 1043, 1015, 1009, 1025, 1011, 1039, 1059, 1042, 1010, 1019, 1026, 984, 1010, 1045, 1013, 1073, 1016, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[996, 1030, 1057, 990, 1026, 1071, 1047, 1034, 999, 1035, 1038, 1017, 1040, 1018, 1026, 978, 1043, 974, 1025, 1028, 1029, 1019, 1029, 1018, 1008, 1008, 1025, 1009, 1051, 1030, 1048, 981, 1030, 1029, 1004, 997, 1051, 1057, 1014, 1059, 994, 999, 1014, 1051, 1018, 1003, 1016, 1086, 947, 1024, 968, 1019, 1060, 1008, 1036, 1055, 1053, 985, 1043, 1037, 1030, 1034, 1006, 1082] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1061, 1034, 1025, 973, 1007, 1010, 1022, 1046, 1004, 1038, 1074, 1037, 1078, 1010, 1034, 1080, 984, 1058, 1005, 1064, 1039, 1004, 1033, 1019, 1004, 996, 1054, 1021, 1003, 994, 1028, 1007, 1009, 1022, 957, 1068, 1005, 1004, 1030, 1012, 1059, 983, 986, 1046, 999, 1022, 1069, 1023, 1015, 1030, 1002, 969, 1071, 1058, 1025, 1005, 1024, 1032, 1025, 1080, 1015, 1028, 1030, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1019, 958, 1024, 981, 1068, 1012, 1008, 971, 1004, 1007, 1026, 1025, 996, 1009, 1042, 1013, 1062, 1000, 1016, 1020, 1046, 1013, 1048, 1001, 1036, 1045, 1029, 1051, 1017, 1010, 1055, 1000, 1066, 1045, 1053, 1053, 1078, 1046, 955, 1001, 1044, 1053, 1010, 1027, 991, 1037, 1055, 1016, 1016, 1065, 999, 1013, 1027, 1017, 986, 1049, 1030, 1006, 1078, 1017, 1021, 1047, 988, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1003, 1019, 1042, 1021, 1050, 1036, 1046, 1091, 23, 1, 0, 0, 0, 0, 0, 0, 1028, 945, 974, 973, 1022, 1028, 974, 1038, 8, 0, 0, 0, 0, 0, 0, 0, 1025, 1002, 1021, 1034, 1010, 1056, 1057, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1026, 1060, 1022, 997, 1045, 1040, 1001, 6, 4, 0, 0, 0, 0, 0, 0, 1063, 1021, 948, 1011, 983, 1075, 1072, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1068, 997, 1021, 1040, 980, 1006, 984, 7, 0, 0, 0, 0, 0, 0, 0, 1041, 972, 994, 1030, 1032, 1047, 1083, 975, 12, 0, 0, 0, 0, 0, 0, 0, 1035, 1043, 1059, 1027, 1069, 1023, 1018, 1065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8332, 7990, 8207, 8206, 8162, 8114, 8186, 8339] avg=8192.0000 max=8339 max_rank=7 min=7990 min_rank=1 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[995, 1043, 1054, 962, 1028, 992, 1009, 1012, 24, 1, 0, 0, 0, 0, 0, 0, 1030, 1018, 982, 1024, 990, 1014, 1037, 1026, 8, 0, 0, 0, 0, 0, 0, 0, 1043, 988, 1056, 1033, 1014, 1039, 1049, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1038, 1035, 1051, 1040, 974, 1099, 1048, 6, 4, 0, 0, 0, 0, 0, 0, 991, 1055, 1061, 1030, 994, 969, 1022, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1023, 1007, 1014, 1044, 1039, 1025, 1034, 7, 0, 0, 0, 0, 0, 0, 0, 1075, 1022, 1012, 962, 1032, 1038, 1014, 1016, 12, 0, 0, 0, 0, 0, 0, 0, 1003, 1019, 1000, 1043, 986, 1037, 1016, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 8129, 8261, 8340, 8192, 8204, 8183, 8107] avg=8192.0000 max=8340 max_rank=3 min=8107 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1042, 1002, 1042, 1038, 978, 1015, 1050, 1032, 23, 1, 0, 0, 0, 0, 0, 0, 1017, 1010, 1014, 1035, 1042, 1040, 1037, 1013, 8, 0, 0, 0, 0, 0, 0, 0, 1011, 1046, 979, 1028, 1005, 990, 997, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1043, 1040, 1032, 993, 1020, 1013, 1031, 6, 4, 0, 0, 0, 0, 0, 0, 985, 1030, 1054, 990, 1056, 1014, 1027, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1009, 983, 1076, 1044, 1013, 1035, 1018, 7, 0, 0, 0, 0, 0, 0, 0, 1031, 1022, 1064, 1012, 1003, 1048, 994, 1043, 11, 0, 0, 0, 0, 0, 0, 0, 1018, 1026, 1040, 971, 1048, 1024, 1051, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8216, 8064, 8186, 8230, 8201, 8228, 8188] avg=8192.0000 max=8230 max_rank=4 min=8064 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32847 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[994, 964, 1033, 1012, 1059, 1024, 990, 1060, 24, 1, 0, 0, 0, 0, 0, 0, 1054, 1022, 1043, 1025, 1058, 1057, 1002, 1061, 8, 0, 0, 0, 0, 0, 0, 0, 954, 1057, 1076, 1045, 1046, 1015, 993, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1031, 993, 985, 996, 1021, 1015, 1018, 6, 4, 0, 0, 0, 0, 0, 0, 1043, 1017, 1074, 989, 1013, 1003, 983, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1026, 1040, 1006, 1056, 1000, 1043, 1015, 7, 0, 0, 0, 0, 0, 0, 0, 1009, 1025, 1011, 1039, 1059, 1042, 1010, 1019, 11, 0, 0, 0, 0, 0, 0, 0, 1026, 984, 1010, 1045, 1013, 1049, 1016, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8330, 8207, 8082, 8144, 8212, 8225, 8175] avg=8192.0000 max=8330 max_rank=1 min=8082 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1025, 1013, 968, 992, 1012, 1052, 1009, 1052, 23, 1, 0, 0, 0, 0, 0, 0, 1051, 1016, 976, 1035, 1018, 1058, 1051, 1002, 8, 0, 0, 0, 0, 0, 0, 0, 1070, 1038, 998, 1010, 1067, 1051, 1018, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1049, 1050, 995, 1016, 1018, 996, 1041, 6, 4, 0, 0, 0, 0, 0, 0, 1080, 1036, 1076, 1040, 977, 997, 1020, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1017, 1043, 970, 1027, 992, 1038, 1094, 7, 0, 0, 0, 0, 0, 0, 0, 1035, 1044, 1023, 987, 1051, 1043, 1043, 972, 12, 0, 0, 0, 0, 0, 0, 0, 1019, 1052, 986, 1007, 1012, 1000, 988, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8215, 8288, 8172, 8235, 8181, 8210, 8088] avg=8192.0000 max=8288 max_rank=2 min=8088 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[996, 1030, 1057, 990, 1026, 1071, 1047, 1034, 23, 1, 0, 0, 0, 0, 0, 0, 999, 1035, 1038, 1017, 1040, 1018, 1026, 978, 8, 0, 0, 0, 0, 0, 0, 0, 1043, 974, 1025, 1028, 1029, 1000, 1029, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1008, 1025, 1009, 1051, 1030, 1048, 981, 6, 4, 0, 0, 0, 0, 0, 0, 1030, 1029, 1004, 979, 1051, 1057, 1014, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 994, 999, 1014, 1051, 1018, 1003, 1016, 1086, 7, 0, 0, 0, 0, 0, 0, 0, 947, 1024, 968, 1019, 1060, 1008, 1036, 1055, 11, 0, 0, 0, 0, 0, 0, 0, 1053, 985, 1043, 1037, 1030, 1011, 1006, 1082, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8159, 8146, 8170, 8223, 8188, 8128, 8247] avg=8192.0000 max=8275 max_rank=0 min=8128 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1061, 1034, 1025, 973, 1007, 1010, 1022, 1046, 23, 1, 0, 0, 0, 0, 0, 0, 1004, 1038, 1074, 1037, 1078, 1010, 1034, 1080, 8, 0, 0, 0, 0, 0, 0, 0, 984, 1058, 1005, 1064, 1039, 985, 1033, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 996, 1054, 1021, 1003, 994, 1028, 1007, 6, 4, 0, 0, 0, 0, 0, 0, 1009, 1022, 957, 1049, 1005, 1004, 1030, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 983, 986, 1046, 999, 1022, 1069, 1023, 7, 0, 0, 0, 0, 0, 0, 0, 1015, 1030, 1002, 969, 1071, 1058, 1025, 1005, 12, 0, 0, 0, 0, 0, 0, 0, 1024, 1032, 1025, 1080, 1015, 1005, 1030, 987, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8363, 8187, 8117, 8088, 8194, 8187, 8198] avg=8192.0000 max=8363 max_rank=1 min=8088 min_rank=4 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32667 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1019, 958, 1024, 981, 1068, 1012, 1008, 971, 28, 7, 0, 0, 0, 0, 0, 0, 1004, 1007, 1026, 1025, 996, 1009, 1042, 1013, 12, 0, 0, 0, 0, 0, 0, 0, 1062, 1000, 1016, 1020, 1046, 983, 1048, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1045, 1029, 1051, 1017, 1010, 1055, 1000, 11, 9, 0, 0, 0, 0, 0, 0, 1066, 1045, 1053, 1018, 1078, 1046, 955, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1053, 1010, 1027, 991, 1037, 1055, 1016, 9, 0, 0, 0, 0, 0, 0, 0, 1016, 1065, 999, 1013, 1027, 1017, 986, 1049, 17, 0, 0, 0, 0, 0, 0, 0, 1030, 1006, 1078, 1017, 1021, 1019, 988, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8076, 8134, 8176, 8263, 8262, 8242, 8189, 8194] avg=8192.0000 max=8263 max_rank=3 min=8076 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32887 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8320, 8064, 8320, 8320, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8064, 8448, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1006, 1006, 1000, 999, 1069, 1049, 1009, 1120, 1054, 983, 1047, 1004, 1001, 1042, 1045, 1027, 1018, 994, 1027, 1010, 1048, 1061, 1007, 1060, 1062, 1041, 1003, 1031, 1079, 1027, 1000, 1071, 1024, 1043, 993, 1040, 1042, 1003, 1046, 1025, 983, 969, 1020, 1057, 960, 1047, 1010, 993, 1032, 1012, 995, 1057, 1029, 1027, 1026, 982, 993, 1018, 1041, 1017, 988, 994, 1046, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1060, 1012, 1053, 1053, 1039, 981, 1033, 1005, 970, 1005, 1030, 1030, 1011, 1003, 1026, 1030, 1031, 1012, 1030, 1051, 1050, 1008, 1008, 1032, 1051, 1030, 1022, 1052, 982, 1041, 1036, 1053, 1027, 981, 1018, 1021, 1017, 967, 1054, 1046, 1077, 989, 1037, 1037, 1093, 1029, 1040, 987, 972, 1015, 1056, 1019, 1026, 1006, 991, 1008, 1032, 998, 993, 1059, 1049, 1015, 1047, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[995, 983, 1079, 1044, 971, 1024, 1021, 1022, 989, 1046, 995, 1001, 975, 1066, 1037, 1042, 996, 1051, 987, 1044, 1009, 1011, 1008, 1043, 1057, 1037, 1020, 1043, 1008, 1059, 1004, 1018, 1012, 1044, 964, 1042, 1057, 1043, 1062, 1028, 1021, 992, 1074, 1028, 1035, 940, 1003, 1023, 1049, 1050, 1037, 1020, 1050, 1024, 1040, 1055, 1016, 957, 1026, 1037, 1000, 1031, 1047, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1042, 1045, 1032, 1027, 1037, 1062, 990, 1014, 1032, 1015, 1011, 996, 998, 985, 1019, 1075, 988, 965, 1028, 1065, 1010, 984, 1065, 1019, 1017, 995, 1014, 996, 984, 1010, 1026, 1079, 1008, 1016, 1031, 1018, 1085, 1021, 1012, 1001, 1011, 1039, 1030, 1062, 1052, 993, 1028, 981, 1070, 1012, 1071, 1030, 986, 1017, 1031, 1033, 1096, 1043, 1026, 1038, 1007, 1026, 1011, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[991, 1044, 1052, 1053, 1009, 1017, 1067, 1044, 1040, 1030, 1035, 995, 1034, 989, 1055, 1053, 1034, 1030, 1029, 1026, 989, 1009, 1030, 1008, 1034, 1049, 972, 1032, 964, 1010, 1009, 983, 1034, 1031, 1045, 1000, 1000, 1084, 1044, 1002, 1041, 1008, 1065, 1020, 991, 1028, 1081, 1017, 1073, 1017, 1042, 1033, 1020, 985, 964, 1065, 1051, 994, 983, 1018, 1003, 1027, 1046, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1003, 1032, 981, 1081, 1028, 992, 994, 1025, 1025, 1088, 997, 1006, 1050, 1051, 1091, 1010, 997, 1047, 1045, 1033, 976, 997, 1043, 1013, 1080, 1076, 1079, 1048, 997, 1046, 983, 1058, 1042, 1008, 1005, 1034, 1028, 1005, 996, 1065, 1048, 1036, 986, 1035, 989, 1029, 1032, 1027, 985, 1033, 1021, 986, 1019, 1058, 1008, 1000, 1045, 974, 1036, 969, 1012, 1002, 1020, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1012, 1047, 985, 1059, 1033, 1033, 995, 1061, 1043, 995, 1008, 1042, 1000, 1080, 1014, 1065, 1011, 1020, 967, 1066, 1004, 1012, 1044, 1033, 1065, 1066, 960, 1029, 990, 1013, 1003, 1031, 1055, 978, 1027, 1018, 1023, 1000, 1000, 1031, 1032, 1046, 1006, 1041, 978, 1051, 1038, 1025, 1023, 996, 1042, 986, 1074, 1046, 1029, 1015, 1017, 983, 1035, 1009, 1066, 1067, 996, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1020, 988, 1022, 1022, 973, 1025, 1031, 1050, 1046, 999, 1068, 1015, 1021, 1015, 1007, 988, 1055, 1045, 1009, 1040, 998, 969, 1024, 994, 1002, 1035, 1061, 1029, 1022, 1017, 988, 986, 1025, 1002, 1047, 1047, 1035, 984, 1065, 1032, 959, 1057, 977, 1050, 1036, 1035, 1040, 1027, 1047, 996, 1026, 1056, 1034, 1047, 1059, 1056, 1041, 1048, 1003, 1061, 1034, 1015, 1074, 957] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1006, 1006, 1000, 999, 1069, 1049, 1009, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 983, 1047, 1004, 1001, 1042, 1044, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 994, 1027, 1010, 1048, 1061, 1007, 1060, 15, 10, 0, 0, 0, 0, 0, 0, 1047, 1041, 1003, 1031, 1079, 1027, 1000, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1043, 993, 1040, 1037, 1003, 1046, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 983, 969, 1020, 1057, 960, 1047, 1010, 993, 5, 1, 1, 0, 0, 0, 0, 0, 1032, 1012, 980, 1057, 1029, 1027, 1026, 982, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1018, 1041, 1017, 988, 994, 1046, 1024, 14, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8202, 8250, 8299, 8211, 8046, 8145, 8140] avg=8192.0000 max=8299 max_rank=3 min=8046 min_rank=5 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1003, 1032, 981, 1081, 1028, 992, 994, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1088, 997, 1006, 1050, 1051, 1090, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1047, 1045, 1033, 976, 997, 1043, 1013, 16, 11, 0, 0, 0, 0, 0, 0, 1064, 1076, 1079, 1048, 997, 1046, 983, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1008, 1005, 1034, 1023, 1005, 996, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1036, 986, 1035, 989, 1029, 1032, 1027, 5, 1, 1, 0, 0, 0, 0, 0, 985, 1033, 1005, 986, 1019, 1058, 1008, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 974, 1036, 969, 1012, 1002, 1020, 1031, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8317, 8178, 8351, 8178, 8189, 8094, 8107] avg=8192.0000 max=8351 max_rank=3 min=8094 min_rank=6 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1012, 1047, 985, 1059, 1033, 1033, 995, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 995, 1008, 1042, 1000, 1080, 1013, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1020, 967, 1066, 1004, 1012, 1044, 1033, 16, 10, 0, 0, 0, 0, 0, 0, 1050, 1066, 960, 1029, 990, 1013, 1003, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 978, 1027, 1018, 1018, 1000, 1000, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1046, 1006, 1041, 978, 1051, 1038, 1025, 5, 1, 1, 0, 0, 0, 0, 0, 1023, 996, 1026, 986, 1074, 1046, 1029, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 983, 1035, 1009, 1066, 1067, 996, 1017, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8246, 8183, 8142, 8127, 8224, 8195, 8208] avg=8192.0000 max=8246 max_rank=1 min=8127 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32754 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1060, 1012, 1053, 1053, 1039, 981, 1033, 992, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1005, 1030, 1030, 1011, 1003, 1025, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1012, 1030, 1051, 1050, 1008, 1008, 1032, 16, 10, 0, 0, 0, 0, 0, 0, 1036, 1030, 1022, 1052, 982, 1041, 1036, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 981, 1018, 1021, 1012, 967, 1054, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 989, 1037, 1037, 1093, 1029, 1040, 987, 5, 1, 1, 0, 0, 0, 0, 0, 972, 1015, 1040, 1019, 1026, 1006, 991, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 998, 993, 1059, 1049, 1015, 1047, 1000, 12, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8104, 8248, 8252, 8126, 8296, 8077, 8210] avg=8192.0000 max=8296 max_rank=5 min=8077 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[995, 983, 1079, 1044, 971, 1024, 1021, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1046, 995, 1001, 975, 1066, 1036, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1051, 987, 1044, 1009, 1011, 1008, 1043, 16, 10, 0, 0, 0, 0, 0, 0, 1042, 1037, 1020, 1043, 1008, 1059, 1004, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1044, 964, 1042, 1052, 1043, 1062, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 992, 1074, 1028, 1035, 940, 1003, 1023, 5, 1, 1, 0, 0, 0, 0, 0, 1049, 1050, 1021, 1020, 1050, 1024, 1040, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 957, 1026, 1037, 1000, 1031, 1047, 1044, 12, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8150, 8175, 8231, 8247, 8123, 8309, 8175] avg=8192.0000 max=8309 max_rank=6 min=8123 min_rank=5 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1042, 1045, 1032, 1027, 1037, 1062, 990, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1015, 1011, 996, 998, 985, 1018, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 988, 965, 1028, 1065, 1010, 984, 1065, 1019, 17, 10, 0, 0, 0, 0, 0, 0, 1002, 995, 1014, 996, 984, 1010, 1026, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1016, 1031, 1018, 1079, 1021, 1012, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1039, 1030, 1062, 1052, 993, 1028, 981, 6, 1, 1, 0, 0, 0, 0, 0, 1070, 1012, 1054, 1030, 986, 1017, 1031, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1096, 1043, 1026, 1038, 1007, 1026, 1011, 1026, 12, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8130, 8151, 8106, 8186, 8204, 8233, 8290] avg=8192.0000 max=8290 max_rank=7 min=8106 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[991, 1044, 1052, 1053, 1009, 1017, 1067, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1030, 1035, 995, 1034, 989, 1054, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1030, 1029, 1026, 989, 1009, 1030, 1008, 16, 10, 0, 0, 0, 0, 0, 0, 1019, 1049, 972, 1032, 964, 1010, 1009, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1031, 1045, 1000, 995, 1084, 1044, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1008, 1065, 1020, 991, 1028, 1081, 1017, 5, 1, 1, 0, 0, 0, 0, 0, 1073, 1017, 1026, 1033, 1020, 985, 964, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 994, 983, 1018, 1003, 1027, 1046, 1008, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8230, 8181, 8038, 8235, 8258, 8183, 8148] avg=8192.0000 max=8263 max_rank=0 min=8038 min_rank=3 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1020, 988, 1022, 1022, 973, 1025, 1031, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 999, 1068, 1015, 1021, 1015, 1005, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1045, 1009, 1040, 998, 969, 1024, 994, 21, 15, 0, 0, 0, 0, 0, 0, 979, 1035, 1061, 1029, 1022, 1017, 988, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1002, 1047, 1047, 1024, 984, 1065, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 959, 1057, 977, 1050, 1036, 1035, 1040, 1027, 11, 2, 2, 0, 0, 0, 0, 0, 1047, 996, 1005, 1056, 1034, 1047, 1059, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1048, 1003, 1061, 1034, 1015, 1074, 957, 17, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8157, 8170, 8117, 8226, 8196, 8300, 8258] avg=8192.0000 max=8300 max_rank=6 min=8112 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32980 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8448, 8192, 8320, 8064, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8448, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8320, 8448, 8192, 8192, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8448, 8192, 8064, 8320, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8320, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1017, 1033, 1031, 1031, 1036, 1017, 1026, 1018, 1062, 1072, 1003, 1064, 1049, 1051, 1063, 1043, 1000, 1009, 1048, 1047, 950, 997, 1040, 1047, 996, 1065, 1029, 1013, 1000, 1003, 1072, 1009, 1021, 997, 1053, 1004, 1027, 1047, 997, 1023, 998, 1008, 1035, 1051, 978, 1012, 1040, 1053, 1030, 981, 1029, 1024, 1043, 1027, 981, 1002, 1009, 1038, 1058, 1025, 1007, 976, 1033, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1050, 1025, 1014, 1037, 1012, 1032, 991, 1027, 1011, 1051, 1057, 982, 1007, 989, 1001, 1039, 1004, 984, 1014, 1048, 1060, 1052, 1024, 985, 1031, 1030, 1076, 1012, 1004, 1022, 999, 1047, 994, 1031, 1026, 1035, 1008, 1048, 1020, 968, 1041, 1027, 1005, 1049, 1000, 1030, 1062, 1006, 1014, 993, 1036, 1019, 1054, 1064, 1021, 1081, 1040, 1063, 1036, 1016, 986, 1012, 1004, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1001, 1049, 1019, 1018, 1008, 1033, 1014, 1015, 1034, 1054, 1045, 996, 977, 1019, 1019, 1022, 939, 1088, 996, 1035, 988, 1003, 1000, 1071, 1016, 996, 1037, 1062, 1007, 999, 1037, 1040, 1050, 1070, 1016, 1035, 1042, 1005, 1050, 952, 1073, 1013, 1018, 1035, 1072, 1043, 1010, 1047, 978, 978, 1023, 1040, 999, 994, 1049, 1054, 1035, 995, 1064, 1016, 1021, 1055, 1035, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1010, 1042, 1064, 1014, 1029, 1001, 1054, 1054, 1009, 937, 1007, 1036, 1060, 1068, 1028, 1025, 1050, 1063, 996, 1001, 1079, 1001, 997, 987, 1024, 1022, 1063, 1041, 997, 1049, 1037, 999, 1079, 1050, 1028, 1067, 1004, 986, 1031, 1006, 1016, 975, 1049, 1074, 1050, 1031, 1035, 1025, 1044, 1011, 1036, 910, 992, 1011, 1003, 1046, 963, 1013, 1021, 1003, 1025, 1034, 1037, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1042, 1006, 1009, 1005, 1036, 986, 1046, 1039, 1009, 1063, 1023, 1007, 980, 1025, 1025, 1042, 1004, 1022, 1021, 1042, 1011, 973, 986, 1030, 1001, 1023, 1017, 1009, 1018, 1023, 1042, 1090, 997, 1013, 1058, 1051, 1040, 1013, 1013, 1029, 1036, 1017, 1036, 1014, 1034, 1032, 1039, 1014, 1036, 1062, 1041, 1061, 1056, 1048, 988, 1015, 1005, 1029, 1025, 998, 1023, 1015, 1018, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1003, 972, 1032, 1021, 1011, 999, 1048, 1068, 1033, 1020, 999, 1045, 1059, 1025, 1045, 984, 1022, 1016, 996, 1039, 1053, 1062, 1029, 957, 1048, 1014, 1016, 1017, 1026, 996, 1036, 1020, 925, 991, 1054, 960, 1012, 1015, 1039, 1073, 1058, 1052, 1043, 1029, 1040, 1021, 1058, 1060, 1065, 1002, 1041, 1072, 1015, 1034, 1043, 1070, 1031, 1016, 993, 990, 1003, 1012, 1013, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1016, 1028, 1013, 998, 1021, 1021, 960, 991, 1044, 1005, 1042, 995, 1034, 1004, 994, 979, 1031, 1041, 986, 995, 1033, 1017, 1050, 1022, 1059, 1032, 1062, 1041, 1056, 1037, 1001, 1103, 1004, 996, 1003, 1066, 1039, 957, 1089, 1006, 1002, 1051, 1039, 985, 976, 1042, 1054, 1072, 1043, 1020, 1039, 991, 994, 1007, 1034, 1061, 988, 1094, 978, 1048, 1017, 1076, 1044, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[985, 959, 1037, 997, 1006, 1039, 1019, 1015, 1087, 1039, 1049, 1010, 993, 1034, 1029, 1054, 990, 1054, 995, 1075, 969, 1034, 1021, 1030, 1066, 1100, 1021, 1026, 1015, 1029, 1020, 1049, 1050, 1034, 1050, 984, 1037, 1010, 1035, 1022, 1047, 979, 1011, 1057, 1025, 1039, 995, 1003, 1072, 1000, 1013, 1049, 1020, 1023, 995, 996, 1040, 1011, 1002, 999, 1016, 1067, 1008, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1017, 1033, 1031, 1031, 1036, 1017, 1026, 1018, 25, 10, 0, 0, 0, 0, 0, 0, 1047, 1072, 1003, 1064, 1049, 1051, 1063, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1009, 1048, 1047, 950, 997, 1040, 1047, 39, 0, 0, 0, 0, 0, 0, 0, 996, 1065, 1029, 1013, 1000, 1003, 1072, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 997, 1053, 1004, 1027, 1047, 997, 1023, 8, 4, 0, 0, 0, 0, 0, 0, 998, 1008, 1035, 1003, 978, 1012, 1040, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 981, 1029, 1024, 1043, 1027, 981, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1038, 1058, 1025, 1007, 976, 1033, 988, 23, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8392, 8177, 8138, 8181, 8127, 8113, 8164] avg=8192.0000 max=8392 max_rank=1 min=8113 min_rank=6 imbalance_ratio=1.024414 max_load_reduction=57144 max_load_reduction_pct=0.871948 moved_to_echo=32585 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1001, 1049, 1019, 1018, 1008, 1033, 1014, 1015, 25, 10, 0, 0, 0, 0, 0, 0, 1021, 1054, 1045, 996, 977, 1019, 1019, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 939, 1088, 996, 1035, 988, 1003, 1000, 1071, 40, 0, 0, 0, 0, 0, 0, 0, 1016, 996, 1037, 1062, 1007, 999, 1037, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1070, 1016, 1035, 1042, 1005, 1050, 952, 7, 4, 0, 0, 0, 0, 0, 0, 1073, 1013, 1018, 988, 1072, 1043, 1010, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 978, 978, 1023, 1040, 999, 994, 1049, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 995, 1064, 1016, 1021, 1055, 1035, 1032, 22, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8153, 8160, 8144, 8231, 8264, 8111, 8281] avg=8192.0000 max=8281 max_rank=7 min=8111 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32887 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1042, 1006, 1009, 1005, 1036, 986, 1046, 1039, 24, 10, 0, 0, 0, 0, 0, 0, 996, 1063, 1023, 1007, 980, 1025, 1025, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1022, 1021, 1042, 1011, 973, 986, 1030, 42, 0, 0, 0, 0, 0, 0, 0, 1001, 1023, 1017, 1009, 1018, 1023, 1042, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1013, 1058, 1051, 1040, 1013, 1013, 1029, 7, 4, 0, 0, 0, 0, 0, 0, 1036, 1017, 1036, 968, 1034, 1032, 1039, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1062, 1041, 1061, 1056, 1048, 988, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1029, 1025, 998, 1023, 1015, 1018, 1025, 22, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8161, 8131, 8171, 8225, 8176, 8303, 8166] avg=8192.0000 max=8303 max_rank=6 min=8131 min_rank=2 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1050, 1025, 1014, 1037, 1012, 1032, 991, 1027, 25, 10, 0, 0, 0, 0, 0, 0, 998, 1051, 1057, 982, 1007, 989, 1001, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 984, 1014, 1048, 1060, 1052, 1024, 985, 40, 0, 0, 0, 0, 0, 0, 0, 1031, 1030, 1076, 1012, 1004, 1022, 999, 997, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1031, 1026, 1035, 1008, 1048, 1020, 968, 7, 4, 0, 0, 0, 0, 0, 0, 1041, 1027, 1005, 1001, 1000, 1030, 1062, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 993, 1036, 1019, 1054, 1064, 1021, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1063, 1036, 1016, 986, 1012, 1004, 1030, 23, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8124, 8211, 8171, 8141, 8172, 8278, 8216] avg=8192.0000 max=8278 max_rank=6 min=8124 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1016, 1028, 1013, 998, 1021, 1021, 960, 991, 23, 11, 0, 0, 0, 0, 0, 0, 1029, 1005, 1042, 995, 1034, 1004, 994, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1041, 986, 995, 1033, 1017, 1050, 1022, 43, 0, 0, 0, 0, 0, 0, 0, 1059, 1032, 1062, 1041, 1056, 1037, 1001, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 996, 1003, 1066, 1039, 957, 1089, 1006, 8, 4, 0, 0, 0, 0, 0, 0, 1002, 1051, 1039, 941, 976, 1042, 1054, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1020, 1039, 991, 994, 1007, 1034, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1094, 978, 1048, 1017, 1076, 1044, 1010, 21, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8082, 8218, 8337, 8172, 8177, 8185, 8283] avg=8192.0000 max=8337 max_rank=3 min=8082 min_rank=0 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1003, 972, 1032, 1021, 1011, 999, 1048, 1068, 25, 10, 0, 0, 0, 0, 0, 0, 1020, 1020, 999, 1045, 1059, 1025, 1045, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1016, 996, 1039, 1053, 1062, 1029, 957, 39, 0, 0, 0, 0, 0, 0, 0, 1048, 1014, 1016, 1017, 1026, 996, 1036, 971, 0, 0, 0, 0, 0, 0, 0, 0, 925, 991, 1054, 960, 1012, 1015, 1039, 1073, 7, 4, 0, 0, 0, 0, 0, 0, 1058, 1052, 1043, 982, 1040, 1021, 1058, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1002, 1041, 1072, 1015, 1034, 1043, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1016, 993, 990, 1003, 1012, 1013, 995, 22, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8197, 8213, 8124, 8080, 8314, 8338, 8081] avg=8192.0000 max=8338 max_rank=6 min=8080 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1010, 1042, 1064, 1014, 1029, 1001, 1054, 1054, 26, 10, 0, 0, 0, 0, 0, 0, 996, 937, 1007, 1036, 1060, 1068, 1028, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1063, 996, 1001, 1079, 1001, 997, 987, 39, 0, 0, 0, 0, 0, 0, 0, 1024, 1022, 1063, 1041, 997, 1049, 1037, 950, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1050, 1028, 1067, 1004, 986, 1031, 1006, 7, 4, 0, 0, 0, 0, 0, 0, 1016, 975, 1049, 1025, 1050, 1031, 1035, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1011, 1036, 910, 992, 1011, 1003, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1013, 1021, 1003, 1025, 1034, 1037, 1037, 23, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8304, 8157, 8213, 8183, 8262, 8206, 8049, 8162] avg=8192.0000 max=8304 max_rank=0 min=8049 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[985, 959, 1037, 997, 1006, 1039, 1019, 1015, 29, 13, 0, 0, 0, 0, 0, 0, 1062, 1039, 1049, 1010, 993, 1034, 1029, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1054, 995, 1075, 969, 1034, 1021, 1030, 45, 0, 0, 0, 0, 0, 0, 0, 1066, 1100, 1021, 1026, 1015, 1029, 1020, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1034, 1050, 984, 1037, 1010, 1035, 1022, 13, 9, 0, 0, 0, 0, 0, 0, 1047, 979, 1011, 1001, 1025, 1039, 995, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1000, 1013, 1049, 1020, 1023, 995, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1011, 1002, 999, 1016, 1067, 1008, 1001, 27, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8270, 8213, 8268, 8244, 8100, 8159, 8183] avg=8192.0000 max=8270 max_rank=1 min=8099 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8320, 8064, 8320, 8192, 8192, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 7936, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8448, 8320, 8192, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1006, 1010, 1044, 1032, 1012, 1009, 1040, 1038, 1008, 1039, 1040, 971, 1003, 1047, 1069, 1022, 1012, 1036, 977, 983, 1021, 1022, 992, 1071, 1043, 1030, 1042, 1046, 990, 1055, 1072, 1032, 990, 1043, 1008, 1083, 999, 966, 985, 1002, 1058, 1044, 1033, 1039, 1074, 1026, 1027, 1033, 1005, 1066, 1062, 978, 1018, 1004, 1048, 1005, 1020, 1027, 988, 960, 1028, 1036, 1051, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1067, 1015, 986, 1043, 1017, 980, 1038, 1029, 1061, 1024, 1043, 1026, 1044, 1019, 1033, 1043, 1032, 1041, 985, 1020, 1005, 1076, 1023, 1024, 1006, 1016, 948, 1015, 1021, 1016, 1030, 984, 1027, 1045, 997, 1008, 1017, 1043, 1012, 1025, 998, 1011, 998, 1031, 1099, 1055, 1018, 1038, 1002, 1040, 1034, 1030, 982, 1050, 1047, 1002, 1052, 1043, 1000, 990, 1075, 1013, 1028, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[989, 1019, 1059, 1014, 1057, 1026, 981, 992, 976, 1015, 1021, 1023, 1019, 1028, 1053, 1023, 1047, 1007, 994, 1043, 1019, 930, 1032, 1046, 1031, 998, 1012, 984, 1038, 1021, 1052, 976, 1024, 1045, 1015, 1019, 1020, 1045, 1048, 1026, 1068, 1025, 1051, 1034, 1035, 1044, 992, 1034, 988, 1035, 1040, 1004, 1072, 1055, 1036, 1064, 983, 1022, 1028, 1033, 1069, 1003, 987, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1044, 1043, 1071, 1038, 1006, 1034, 1015, 1004, 1074, 1033, 1046, 1028, 1057, 1003, 1049, 984, 1040, 1072, 1016, 989, 1032, 1029, 996, 1028, 990, 1010, 1025, 1006, 1063, 1033, 1039, 1003, 1005, 1011, 1025, 1012, 1044, 1020, 1013, 1022, 996, 1079, 1052, 1014, 966, 999, 1052, 994, 981, 1024, 1003, 967, 1061, 994, 1007, 1028, 999, 1021, 1039, 1009, 1047, 1060, 1029, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1059, 1043, 1065, 1028, 1038, 982, 982, 1000, 1019, 1053, 1059, 1005, 1049, 1007, 1052, 1013, 1031, 1043, 1033, 1004, 1078, 1046, 992, 987, 989, 1038, 1014, 1030, 966, 1021, 1052, 981, 1019, 1015, 989, 1010, 973, 1049, 997, 1071, 1056, 1024, 1000, 1043, 1028, 1004, 1006, 1030, 987, 1067, 1013, 1001, 1044, 1083, 1069, 1046, 1046, 1046, 1024, 995, 992, 1058, 1012, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1075, 977, 986, 1018, 1047, 1015, 1032, 1010, 949, 1015, 988, 1020, 1005, 998, 1038, 1047, 1018, 1068, 1001, 1067, 1082, 1047, 983, 964, 1001, 1066, 1008, 1029, 1015, 1049, 1011, 1057, 1058, 1020, 1050, 987, 1044, 1022, 1020, 1054, 978, 1035, 1025, 1054, 1030, 971, 1038, 1039, 1006, 1065, 1043, 1046, 993, 1007, 1066, 1026, 1037, 1003, 1023, 1022, 1028, 1044, 987, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[992, 1019, 1009, 1035, 978, 1008, 1021, 1068, 1059, 1025, 1071, 1013, 1005, 1063, 1109, 1043, 1035, 1049, 1035, 1030, 1048, 1032, 1006, 1031, 1005, 1033, 1044, 996, 1025, 1038, 961, 1057, 1036, 1014, 1027, 1009, 1000, 1015, 972, 1003, 1072, 1029, 1020, 1028, 1030, 1028, 991, 975, 1020, 1030, 1000, 984, 1009, 1004, 1023, 995, 1013, 1044, 1055, 1063, 1009, 980, 1078, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1023, 1032, 1043, 1015, 1002, 1014, 1036, 1045, 998, 1092, 1039, 1024, 1029, 1045, 1067, 960, 984, 1012, 1044, 971, 960, 1005, 1010, 964, 1011, 1064, 1014, 1038, 986, 1015, 1041, 1033, 1033, 1019, 1089, 1038, 995, 1007, 1060, 1037, 1026, 1038, 1041, 1023, 1028, 1018, 1034, 1010, 997, 1030, 1043, 1022, 1036, 1027, 1010, 1038, 1054, 1022, 1035, 1063, 1018, 1004, 1072, 953] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[989, 1019, 1059, 1014, 1057, 1026, 981, 992, 7, 3, 0, 0, 0, 0, 0, 0, 976, 1015, 1021, 1023, 1019, 1028, 1011, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1007, 994, 1043, 1019, 930, 1032, 1046, 29, 0, 0, 0, 0, 0, 0, 0, 1031, 998, 1012, 984, 1038, 1021, 1052, 976, 13, 13, 0, 0, 0, 0, 0, 0, 1024, 1045, 1015, 1019, 1020, 1045, 1048, 1026, 15, 4, 0, 0, 0, 0, 0, 0, 1068, 1025, 1051, 1034, 1007, 1044, 992, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1032, 1040, 1004, 1072, 1055, 1036, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1022, 1028, 1033, 1058, 1003, 987, 1067, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8116, 8147, 8138, 8261, 8255, 8291, 8181] avg=8192.0000 max=8291 max_rank=6 min=8116 min_rank=1 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32988 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1006, 1010, 1044, 1032, 1012, 1009, 1040, 1038, 6, 3, 0, 0, 0, 0, 0, 0, 1008, 1039, 1040, 971, 1003, 1047, 1026, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1036, 977, 983, 1021, 1022, 992, 1071, 29, 0, 0, 0, 0, 0, 0, 0, 1043, 1030, 1042, 1046, 990, 1055, 1072, 1032, 14, 14, 0, 0, 0, 0, 0, 0, 990, 1043, 1008, 1083, 999, 966, 985, 1002, 15, 4, 0, 0, 0, 0, 0, 0, 1058, 1044, 1033, 1039, 1045, 1026, 1027, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1063, 1062, 978, 1018, 1004, 1048, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1027, 988, 960, 1018, 1036, 1051, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8156, 8143, 8338, 8095, 8305, 8183, 8116] avg=8192.0000 max=8338 max_rank=3 min=8095 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1067, 1015, 986, 1043, 1017, 980, 1038, 1029, 7, 3, 0, 0, 0, 0, 0, 0, 1061, 1024, 1043, 1026, 1044, 1019, 992, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1041, 985, 1020, 1005, 1076, 1023, 1024, 28, 0, 0, 0, 0, 0, 0, 0, 1006, 1016, 948, 1015, 1021, 1016, 1030, 984, 13, 14, 0, 0, 0, 0, 0, 0, 1027, 1045, 997, 1008, 1017, 1043, 1012, 1025, 16, 4, 0, 0, 0, 0, 0, 0, 998, 1011, 998, 1031, 1069, 1055, 1018, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1037, 1034, 1030, 982, 1050, 1047, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1043, 1000, 990, 1064, 1013, 1028, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8252, 8234, 8063, 8194, 8218, 8184, 8206] avg=8192.0000 max=8252 max_rank=1 min=8063 min_rank=3 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[992, 1019, 1009, 1035, 978, 1008, 1021, 1068, 6, 3, 0, 0, 0, 0, 0, 0, 1059, 1025, 1071, 1013, 1005, 1063, 1065, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1049, 1035, 1030, 1048, 1032, 1006, 1031, 30, 0, 0, 0, 0, 0, 0, 0, 1005, 1033, 1044, 996, 1025, 1038, 961, 1057, 14, 13, 0, 0, 0, 0, 0, 0, 1036, 1014, 1027, 1009, 1000, 1015, 972, 1003, 15, 4, 0, 0, 0, 0, 0, 0, 1072, 1029, 1020, 1028, 1002, 1028, 991, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1027, 1000, 984, 1009, 1004, 1023, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1044, 1055, 1063, 999, 980, 1078, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8344, 8296, 8186, 8095, 8145, 8062, 8269] avg=8192.0000 max=8344 max_rank=1 min=8062 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32571 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1044, 1043, 1071, 1038, 1006, 1034, 1015, 1004, 6, 3, 0, 0, 0, 0, 0, 0, 1074, 1033, 1046, 1028, 1057, 1003, 1007, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1072, 1016, 989, 1032, 1029, 996, 1028, 29, 0, 0, 0, 0, 0, 0, 0, 990, 1010, 1025, 1006, 1063, 1033, 1039, 1003, 13, 12, 0, 0, 0, 0, 0, 0, 1005, 1011, 1025, 1012, 1044, 1020, 1013, 1022, 14, 4, 0, 0, 0, 0, 0, 0, 996, 1079, 1052, 1014, 940, 999, 1052, 994, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1021, 1003, 967, 1061, 994, 1007, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1021, 1039, 1009, 1037, 1060, 1029, 1063, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8232, 8231, 8194, 8170, 8126, 8062, 8257] avg=8192.0000 max=8264 max_rank=0 min=8062 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1075, 977, 986, 1018, 1047, 1015, 1032, 1010, 6, 3, 0, 0, 0, 0, 0, 0, 949, 1015, 988, 1020, 1005, 998, 997, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1068, 1001, 1067, 1082, 1047, 983, 964, 28, 0, 0, 0, 0, 0, 0, 0, 1001, 1066, 1008, 1029, 1015, 1049, 1011, 1057, 13, 13, 0, 0, 0, 0, 0, 0, 1058, 1020, 1050, 987, 1044, 1022, 1020, 1054, 15, 4, 0, 0, 0, 0, 0, 0, 978, 1035, 1025, 1054, 1002, 971, 1038, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1062, 1043, 1046, 993, 1007, 1066, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1003, 1023, 1022, 1018, 1044, 987, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8019, 8258, 8262, 8274, 8142, 8249, 8163] avg=8192.0000 max=8274 max_rank=4 min=8019 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1059, 1043, 1065, 1028, 1038, 982, 982, 1000, 6, 3, 0, 0, 0, 0, 0, 0, 1019, 1053, 1059, 1005, 1049, 1007, 1010, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1043, 1033, 1004, 1078, 1046, 992, 987, 29, 0, 0, 0, 0, 0, 0, 0, 989, 1038, 1014, 1030, 966, 1021, 1052, 981, 13, 13, 0, 0, 0, 0, 0, 0, 1019, 1015, 989, 1010, 973, 1049, 997, 1071, 15, 4, 0, 0, 0, 0, 0, 0, 1056, 1024, 1000, 1043, 1000, 1004, 1006, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1064, 1013, 1001, 1044, 1083, 1069, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1046, 1024, 995, 982, 1058, 1012, 980, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8215, 8243, 8117, 8142, 8163, 8307, 8143] avg=8192.0000 max=8307 max_rank=6 min=8117 min_rank=3 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1023, 1032, 1043, 1015, 1002, 1014, 1036, 1045, 11, 5, 0, 0, 0, 0, 0, 0, 998, 1092, 1039, 1024, 1029, 1045, 1015, 960, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1012, 1044, 971, 960, 1005, 1010, 964, 34, 0, 0, 0, 0, 0, 0, 0, 1011, 1064, 1014, 1038, 986, 1015, 1041, 1033, 18, 18, 0, 0, 0, 0, 0, 0, 1033, 1019, 1089, 1038, 995, 1007, 1060, 1037, 18, 9, 0, 0, 0, 0, 0, 0, 1026, 1038, 1041, 1023, 992, 1018, 1034, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1025, 1043, 1022, 1036, 1027, 1010, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1022, 1035, 1063, 998, 1004, 1072, 953, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8202, 7984, 8238, 8305, 8182, 8198, 8201] avg=8192.0000 max=8305 max_rank=4 min=7984 min_rank=2 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32886 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8320, 8064, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8448, 8192, 8192, 8320, 8192, 8064, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8448, 8320, 8064, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1037, 1056, 1031, 1025, 1073, 1048, 1031, 1094, 1017, 1008, 1028, 1039, 986, 960, 1005, 1007, 1018, 1012, 1041, 1035, 1086, 1029, 994, 1043, 1051, 996, 1026, 1013, 1030, 1000, 1086, 988, 1031, 1004, 1011, 971, 1034, 1068, 1011, 1039, 1019, 1043, 1006, 1019, 1054, 1002, 1014, 1037, 1015, 1008, 1014, 1012, 1052, 1007, 1069, 996, 1034, 966, 1007, 1042, 1033, 1007, 1015, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1082, 1009, 1029, 1026, 1050, 1006, 1038, 1004, 1018, 1057, 991, 1045, 1009, 1019, 1025, 1007, 1012, 1034, 1043, 1015, 1046, 980, 1014, 1091, 1018, 1026, 1022, 981, 1092, 965, 1048, 1045, 992, 1045, 1028, 1051, 1045, 999, 989, 1036, 971, 1075, 1025, 1052, 997, 1026, 1010, 993, 1021, 1029, 1023, 1028, 1047, 987, 1020, 1066, 1002, 1038, 994, 1015, 1010, 1048, 1041, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[982, 1016, 1021, 1041, 1037, 1062, 1011, 1063, 981, 1020, 1036, 993, 1015, 1018, 1037, 1064, 1005, 1036, 1049, 999, 1049, 1001, 1032, 1016, 1030, 1020, 1024, 1028, 1030, 1079, 1038, 1006, 981, 1029, 1004, 1051, 1034, 1078, 1017, 968, 1004, 1075, 1029, 1025, 1006, 1062, 1015, 1043, 1017, 1067, 978, 1022, 987, 1031, 1026, 1025, 1049, 1025, 1014, 996, 985, 996, 1028, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1067, 1005, 1004, 1007, 989, 1054, 1000, 1038, 965, 1037, 1029, 1002, 1069, 1009, 1018, 997, 1058, 1005, 1016, 1027, 998, 1044, 1054, 966, 1062, 999, 1037, 1003, 1031, 1025, 992, 1071, 1010, 995, 990, 1021, 1039, 1046, 1045, 1015, 1035, 1015, 1002, 1035, 1029, 1029, 1061, 1033, 969, 1025, 1059, 1047, 1032, 985, 1003, 1042, 1019, 1058, 1047, 1010, 1055, 1090, 1004, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1016, 1042, 1049, 1016, 1029, 1001, 980, 1018, 1021, 983, 1057, 983, 1021, 998, 968, 1033, 1013, 971, 1001, 1043, 941, 1041, 1060, 1041, 1021, 1026, 1062, 1037, 1025, 1020, 1072, 999, 1042, 1036, 1035, 1039, 1033, 1046, 1050, 1006, 1007, 1023, 1036, 1005, 1012, 1049, 1019, 998, 1063, 1018, 1026, 992, 983, 1019, 1034, 996, 1044, 1037, 1024, 1047, 1047, 1033, 1053, 1096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1009, 1024, 1046, 1001, 1035, 1017, 1057, 981, 1058, 1009, 1031, 997, 1048, 1035, 1063, 975, 1091, 1043, 1009, 999, 1015, 1021, 1024, 1019, 991, 1028, 996, 1012, 1020, 1004, 1044, 1081, 999, 1018, 1062, 995, 1041, 1032, 1001, 1018, 1054, 994, 1025, 1008, 1030, 1052, 977, 1015, 1044, 1040, 1018, 1044, 1067, 1026, 1027, 1019, 990, 1022, 1009, 1019, 1031, 1018, 1036, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1034, 1039, 1066, 1011, 1007, 1053, 980, 1013, 1023, 1014, 1019, 1000, 1028, 1004, 997, 993, 988, 1007, 1026, 1036, 1012, 1071, 973, 1024, 1038, 991, 1011, 1046, 1041, 1103, 1036, 1006, 1025, 1025, 1014, 1030, 1014, 1081, 982, 1044, 1028, 1112, 1010, 1023, 1031, 1023, 1049, 984, 1031, 1011, 1064, 1019, 1080, 1026, 1030, 1005, 1003, 952, 1033, 1036, 1021, 1035, 1038, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1006, 1008, 1012, 1015, 1036, 1040, 1011, 1023, 1009, 1005, 1021, 979, 1016, 1003, 1080, 1049, 1050, 1004, 1056, 1091, 1070, 1073, 1043, 1028, 1052, 1017, 1035, 995, 1044, 1029, 1036, 991, 1034, 1022, 1025, 1029, 1043, 1027, 1025, 1015, 999, 1035, 999, 1051, 993, 989, 1005, 1019, 996, 968, 1070, 1083, 983, 1003, 1025, 974, 1019, 1057, 1006, 1006, 1039, 1004, 1024, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1037, 1056, 1031, 1025, 1073, 1028, 1031, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1008, 1028, 1039, 986, 960, 1005, 1007, 30, 24, 9, 0, 0, 0, 0, 0, 1018, 1012, 1041, 1035, 1086, 1005, 994, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 996, 1026, 1013, 1030, 1000, 1056, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1004, 1011, 971, 1034, 1065, 1011, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1043, 1006, 1019, 1054, 1002, 1014, 1037, 3, 1, 0, 0, 0, 0, 0, 0, 1015, 1008, 1014, 1012, 1052, 1007, 1069, 996, 5, 0, 0, 0, 0, 0, 0, 0, 1034, 966, 1007, 1042, 1033, 1007, 1015, 1003, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8375, 8113, 8234, 8160, 8166, 8198, 8178, 8112] avg=8192.0000 max=8375 max_rank=0 min=8112 min_rank=7 imbalance_ratio=1.022339 max_load_reduction=57161 max_load_reduction_pct=0.872208 moved_to_echo=32654 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1016, 1042, 1049, 1016, 1029, 982, 980, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 983, 1057, 983, 1021, 998, 968, 1033, 30, 24, 8, 0, 0, 0, 0, 0, 1013, 971, 1001, 1043, 941, 1017, 1060, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1026, 1062, 1037, 1025, 1020, 1042, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1036, 1035, 1039, 1033, 1043, 1050, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1023, 1036, 1005, 1012, 1049, 1019, 998, 3, 1, 0, 0, 0, 0, 0, 0, 1063, 1018, 1026, 992, 983, 1019, 1034, 996, 5, 0, 0, 0, 0, 0, 0, 0, 1044, 1037, 1024, 1047, 1047, 1033, 1053, 1096, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8126, 8087, 8232, 8284, 8153, 8136, 8386] avg=8192.0000 max=8386 max_rank=7 min=8087 min_rank=2 imbalance_ratio=1.023682 max_load_reduction=57150 max_load_reduction_pct=0.872040 moved_to_echo=32959 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1082, 1009, 1029, 1026, 1050, 987, 1038, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1057, 991, 1045, 1009, 1019, 1025, 1007, 29, 23, 8, 0, 0, 0, 0, 0, 1012, 1034, 1043, 1015, 1046, 957, 1014, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1026, 1022, 981, 1092, 965, 1019, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1045, 1028, 1051, 1045, 996, 989, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1075, 1025, 1052, 997, 1026, 1010, 993, 3, 1, 0, 0, 0, 0, 0, 0, 1021, 1029, 1023, 1028, 1047, 987, 1020, 1066, 5, 0, 0, 0, 0, 0, 0, 0, 1002, 1038, 994, 1015, 1010, 1048, 1041, 986, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8231, 8212, 8168, 8182, 8153, 8226, 8139] avg=8192.0000 max=8231 max_rank=1 min=8139 min_rank=7 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1009, 1024, 1046, 1001, 1035, 997, 1057, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1009, 1031, 997, 1048, 1035, 1063, 975, 29, 24, 9, 0, 0, 0, 0, 0, 1091, 1043, 1009, 999, 1015, 997, 1024, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1028, 996, 1012, 1020, 1004, 1015, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1018, 1062, 995, 1041, 1029, 1001, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 994, 1025, 1008, 1030, 1052, 977, 1015, 3, 1, 0, 0, 0, 0, 0, 0, 1044, 1040, 1018, 1044, 1067, 1026, 1027, 1019, 5, 0, 0, 0, 0, 0, 0, 0, 990, 1022, 1009, 1019, 1031, 1018, 1036, 1022, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8278, 8197, 8147, 8163, 8159, 8290, 8152] avg=8192.0000 max=8290 max_rank=6 min=8147 min_rank=3 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[982, 1016, 1021, 1041, 1037, 1042, 1011, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1020, 1036, 993, 1015, 1018, 1037, 1064, 29, 23, 9, 0, 0, 0, 0, 0, 1005, 1036, 1049, 999, 1049, 978, 1032, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1020, 1024, 1028, 1030, 1079, 1009, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1029, 1004, 1051, 1034, 1075, 1017, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1075, 1029, 1025, 1006, 1062, 1015, 1043, 3, 1, 0, 0, 0, 0, 0, 0, 1017, 1067, 978, 1022, 987, 1031, 1026, 1025, 5, 0, 0, 0, 0, 0, 0, 0, 1049, 1025, 1014, 996, 985, 996, 1028, 1030, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8225, 8164, 8226, 8159, 8263, 8158, 8128] avg=8192.0000 max=8263 max_rank=5 min=8128 min_rank=7 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32708 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1067, 1005, 1004, 1007, 989, 1034, 1000, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 965, 1037, 1029, 1002, 1069, 1009, 1018, 997, 27, 24, 9, 0, 0, 0, 0, 0, 1058, 1005, 1016, 1027, 998, 1020, 1054, 966, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 999, 1037, 1003, 1031, 1025, 965, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 995, 990, 1021, 1039, 1043, 1045, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1015, 1002, 1035, 1029, 1029, 1061, 1033, 3, 1, 0, 0, 0, 0, 0, 0, 969, 1025, 1059, 1047, 1032, 985, 1003, 1042, 5, 0, 0, 0, 0, 0, 0, 0, 1019, 1058, 1047, 1010, 1055, 1090, 1004, 1013, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8186, 8144, 8193, 8158, 8243, 8167, 8301] avg=8192.0000 max=8301 max_rank=7 min=8144 min_rank=0 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32869 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1034, 1039, 1066, 1011, 1007, 1033, 980, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1014, 1019, 1000, 1028, 1004, 997, 993, 29, 25, 9, 0, 0, 0, 0, 0, 988, 1007, 1026, 1036, 1012, 1046, 973, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 991, 1011, 1046, 1041, 1103, 1007, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1025, 1014, 1030, 1014, 1078, 982, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1112, 1010, 1023, 1031, 1023, 1049, 984, 3, 1, 0, 0, 0, 0, 0, 0, 1031, 1011, 1064, 1019, 1080, 1026, 1030, 1005, 5, 0, 0, 0, 0, 0, 0, 0, 1003, 952, 1033, 1036, 1021, 1035, 1038, 987, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8141, 8112, 8243, 8212, 8264, 8271, 8110] avg=8192.0000 max=8271 max_rank=6 min=8110 min_rank=7 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1006, 1008, 1012, 1015, 1036, 1003, 1011, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1005, 1021, 979, 1016, 1003, 1080, 1049, 32, 29, 13, 0, 0, 0, 0, 0, 1050, 1004, 1056, 1091, 1070, 1044, 1043, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1017, 1035, 995, 1044, 1029, 1004, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1022, 1025, 1029, 1043, 1019, 1025, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1035, 999, 1051, 993, 989, 1005, 1019, 8, 5, 0, 0, 0, 0, 0, 0, 996, 968, 1070, 1083, 983, 1003, 1025, 974, 8, 0, 0, 0, 0, 0, 0, 0, 1019, 1057, 1006, 1006, 1039, 1004, 1024, 1042, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8236, 8386, 8167, 8212, 8103, 8110, 8208] avg=8192.0000 max=8386 max_rank=2 min=8103 min_rank=5 imbalance_ratio=1.023682 max_load_reduction=57150 max_load_reduction_pct=0.872040 moved_to_echo=32633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8448, 8192, 8320, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8064, 8192, 8064, 8320, 8192, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8448, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1086, 1020, 1024, 985, 1047, 1022, 1007, 1094, 991, 1061, 1015, 1017, 1025, 1039, 1036, 981, 1056, 1040, 968, 972, 1018, 1001, 992, 1082, 967, 1044, 1067, 1011, 1060, 1032, 1065, 1010, 1027, 1010, 1011, 1025, 1064, 1040, 1022, 1044, 1090, 1053, 1022, 1015, 1040, 961, 1034, 992, 1033, 1045, 1039, 1011, 1017, 1001, 994, 1006, 985, 1023, 1009, 1006, 1019, 1044, 1029, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1034, 996, 1038, 1045, 1007, 1069, 982, 1046, 994, 1059, 996, 988, 1056, 1004, 1014, 1001, 984, 1073, 1064, 1016, 1060, 994, 1001, 1009, 1030, 1032, 1002, 989, 1013, 993, 999, 1053, 1036, 1011, 1062, 1025, 1037, 1062, 1015, 1010, 1012, 986, 1019, 1043, 1039, 1046, 1043, 1028, 1027, 991, 1037, 975, 1031, 1037, 1015, 1015, 1023, 1042, 1060, 1037, 1067, 1020, 1038, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1039, 970, 966, 1036, 1020, 1007, 1038, 1049, 991, 980, 969, 1025, 1041, 1001, 1041, 1044, 1004, 1050, 1030, 1038, 1109, 1037, 1007, 1056, 1059, 1051, 1048, 1039, 978, 1091, 1000, 1042, 1057, 1042, 1006, 1040, 1042, 1073, 1061, 1001, 1018, 985, 997, 995, 1029, 1014, 981, 1032, 1002, 1021, 1034, 958, 1019, 1023, 1046, 1027, 999, 1067, 1024, 996, 1012, 1029, 1057, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[975, 1002, 1028, 1014, 1087, 1021, 1032, 979, 984, 1029, 1048, 1000, 995, 1075, 1018, 1029, 1014, 1017, 1034, 1072, 1053, 996, 917, 1073, 1002, 1001, 992, 1015, 1046, 1018, 1005, 1028, 1037, 1057, 1059, 1042, 1027, 1006, 1051, 1016, 1022, 1063, 993, 1067, 1017, 1011, 1029, 1069, 947, 1041, 1022, 1066, 1016, 983, 992, 1027, 1005, 1005, 1074, 1044, 1064, 1004, 1011, 1070] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[988, 1081, 1021, 1053, 1084, 1053, 1028, 1104, 1028, 1015, 1072, 1017, 1016, 1023, 984, 1030, 1054, 1042, 1041, 1045, 1003, 983, 1048, 1062, 1002, 1002, 1038, 952, 1054, 994, 1009, 983, 1032, 1045, 998, 1005, 973, 1030, 1055, 1014, 985, 1051, 1059, 1029, 1007, 1044, 1029, 1012, 1016, 1022, 997, 1050, 1017, 976, 992, 1011, 1038, 997, 1035, 1043, 1086, 986, 1009, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1036, 1025, 1082, 1000, 1006, 1036, 1025, 986, 1019, 1001, 996, 996, 1026, 1028, 1025, 1004, 1033, 1020, 1054, 1014, 1022, 990, 1004, 960, 1035, 1012, 1099, 1025, 990, 1064, 1009, 1044, 1002, 1027, 1050, 1032, 1059, 1056, 1016, 1052, 1000, 1041, 999, 1039, 1067, 1038, 1039, 987, 1046, 1052, 1007, 961, 1030, 1080, 1032, 1027, 1059, 1035, 1007, 1019, 1014, 1016, 1007, 974] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1042, 1066, 1025, 1022, 1042, 1004, 1014, 1015, 1034, 1056, 1047, 1003, 1055, 1006, 1067, 1044, 983, 969, 1032, 1012, 1040, 1029, 1068, 1041, 966, 1074, 984, 997, 1012, 1009, 1049, 1032, 1008, 977, 1005, 1029, 1086, 1030, 1004, 1072, 1023, 1011, 984, 1055, 1009, 1076, 1022, 1018, 1010, 1008, 1045, 999, 981, 1053, 1034, 965, 1008, 1114, 1007, 1057, 1017, 954, 1020, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1037, 998, 1042, 1007, 1032, 1026, 1031, 1021, 1054, 1026, 1059, 1038, 984, 1053, 1025, 1007, 954, 968, 1023, 1060, 1023, 1051, 1006, 1015, 965, 1017, 995, 1041, 986, 1057, 1011, 989, 1065, 1023, 1037, 1019, 1033, 1011, 1030, 979, 1034, 1027, 1012, 975, 1035, 999, 1048, 1065, 1057, 1004, 1071, 1030, 1024, 1085, 1024, 996, 1047, 1011, 1075, 980, 1013, 1059, 1014, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1086, 1020, 1024, 985, 1016, 1022, 1007, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1061, 1015, 1017, 1025, 1039, 1036, 981, 12, 5, 0, 0, 0, 0, 0, 0, 1056, 1040, 968, 972, 1018, 1001, 992, 1082, 3, 2, 0, 0, 0, 0, 0, 0, 967, 1044, 1067, 1011, 1060, 1032, 1065, 1010, 19, 12, 0, 0, 0, 0, 0, 0, 1027, 1010, 1011, 1025, 1010, 1040, 1022, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 1053, 1022, 1015, 1037, 961, 1034, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1045, 1039, 1011, 1017, 1001, 994, 1006, 42, 0, 0, 0, 0, 0, 0, 0, 985, 1016, 1009, 1006, 1019, 1044, 1029, 990, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8182, 8134, 8287, 8189, 8204, 8188, 8098] avg=8192.0000 max=8287 max_rank=3 min=8098 min_rank=7 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[975, 1002, 1028, 1014, 1054, 1021, 1032, 979, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1029, 1048, 1000, 995, 1075, 1018, 1029, 13, 5, 0, 0, 0, 0, 0, 0, 1014, 1017, 1034, 1072, 1053, 996, 917, 1073, 3, 2, 0, 0, 0, 0, 0, 0, 1002, 1001, 992, 1015, 1046, 1018, 1005, 1028, 20, 12, 0, 0, 0, 0, 0, 0, 1037, 1057, 1059, 1042, 974, 1006, 1051, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1063, 993, 1067, 1014, 1011, 1029, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 947, 1041, 1022, 1066, 1016, 983, 992, 1027, 41, 0, 0, 0, 0, 0, 0, 0, 1005, 998, 1074, 1044, 1064, 1004, 1011, 1070, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8196, 8181, 8139, 8242, 8268, 8135, 8270] avg=8192.0000 max=8270 max_rank=7 min=8105 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32915 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1034, 996, 1038, 1045, 976, 1069, 982, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1059, 996, 988, 1056, 1004, 1014, 1001, 12, 6, 0, 0, 0, 0, 0, 0, 984, 1073, 1064, 1016, 1060, 994, 1001, 1009, 3, 2, 0, 0, 0, 0, 0, 0, 1030, 1032, 1002, 989, 1013, 993, 999, 1053, 19, 12, 0, 0, 0, 0, 0, 0, 1036, 1011, 1062, 1025, 984, 1062, 1015, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 986, 1019, 1043, 1036, 1046, 1043, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 991, 1037, 975, 1031, 1037, 1015, 1015, 41, 0, 0, 0, 0, 0, 0, 0, 1023, 1034, 1060, 1037, 1067, 1020, 1038, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8130, 8206, 8142, 8205, 8213, 8169, 8285] avg=8192.0000 max=8285 max_rank=7 min=8130 min_rank=1 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1042, 1066, 1025, 1022, 1011, 1004, 1014, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1056, 1047, 1003, 1055, 1006, 1067, 1044, 12, 6, 0, 0, 0, 0, 0, 0, 983, 969, 1032, 1012, 1040, 1029, 1068, 1041, 3, 2, 0, 0, 0, 0, 0, 0, 966, 1074, 984, 997, 1012, 1009, 1049, 1032, 19, 13, 0, 0, 0, 0, 0, 0, 1008, 977, 1005, 1029, 1030, 1030, 1004, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1011, 984, 1055, 1006, 1076, 1022, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1008, 1045, 999, 981, 1053, 1034, 965, 43, 0, 0, 0, 0, 0, 0, 0, 1008, 1106, 1007, 1057, 1017, 954, 1020, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8330, 8179, 8155, 8155, 8195, 8138, 8185] avg=8192.0000 max=8330 max_rank=1 min=8138 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1039, 970, 966, 1036, 989, 1007, 1038, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 991, 980, 969, 1025, 1041, 1001, 1041, 1044, 12, 6, 0, 0, 0, 0, 0, 0, 1004, 1050, 1030, 1038, 1109, 1037, 1007, 1056, 3, 2, 0, 0, 0, 0, 0, 0, 1059, 1051, 1048, 1039, 978, 1091, 1000, 1042, 19, 12, 0, 0, 0, 0, 0, 0, 1057, 1042, 1006, 1040, 988, 1073, 1061, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 985, 997, 995, 1026, 1014, 981, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1021, 1034, 958, 1019, 1023, 1046, 1027, 42, 0, 0, 0, 0, 0, 0, 0, 999, 1059, 1024, 996, 1012, 1029, 1057, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8094, 8110, 8336, 8339, 8268, 8048, 8172, 8169] avg=8192.0000 max=8339 max_rank=3 min=8048 min_rank=5 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1036, 1025, 1082, 1000, 975, 1036, 1025, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1001, 996, 996, 1026, 1028, 1025, 1004, 12, 5, 0, 0, 0, 0, 0, 0, 1033, 1020, 1054, 1014, 1022, 990, 1004, 960, 3, 2, 0, 0, 0, 0, 0, 0, 1035, 1012, 1099, 1025, 990, 1064, 1009, 1044, 19, 12, 0, 0, 0, 0, 0, 0, 1002, 1027, 1050, 1032, 1005, 1056, 1016, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1041, 999, 1039, 1064, 1038, 1039, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1052, 1007, 961, 1030, 1080, 1032, 1027, 42, 0, 0, 0, 0, 0, 0, 0, 1059, 1028, 1007, 1019, 1014, 1016, 1007, 974, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8112, 8102, 8309, 8240, 8207, 8277, 8124] avg=8192.0000 max=8309 max_rank=3 min=8102 min_rank=2 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1037, 998, 1042, 1007, 992, 1026, 1031, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1026, 1059, 1038, 984, 1053, 1025, 1007, 17, 10, 0, 0, 0, 0, 0, 0, 954, 968, 1023, 1060, 1023, 1051, 1006, 1015, 7, 8, 0, 0, 0, 0, 0, 0, 965, 1017, 995, 1041, 986, 1057, 1011, 989, 23, 16, 0, 0, 0, 0, 0, 0, 1065, 1023, 1037, 1019, 971, 1011, 1030, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1027, 1012, 975, 1028, 999, 1048, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1004, 1071, 1030, 1024, 1085, 1024, 996, 46, 0, 0, 0, 0, 0, 0, 0, 1047, 993, 1075, 980, 1013, 1059, 1014, 1053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8273, 8115, 8100, 8135, 8188, 8337, 8234] avg=8192.0000 max=8337 max_rank=6 min=8100 min_rank=3 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32894 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[988, 1081, 1021, 1053, 1051, 1053, 1028, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1015, 1072, 1017, 1016, 1023, 984, 1030, 13, 5, 0, 0, 0, 0, 0, 0, 1054, 1042, 1041, 1045, 1003, 983, 1048, 1062, 3, 2, 0, 0, 0, 0, 0, 0, 1002, 1002, 1038, 952, 1054, 994, 1009, 983, 20, 11, 0, 0, 0, 0, 0, 0, 1032, 1045, 998, 1005, 923, 1030, 1055, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1051, 1059, 1029, 1004, 1044, 1029, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1022, 997, 1050, 1017, 976, 992, 1011, 39, 0, 0, 0, 0, 0, 0, 0, 1038, 990, 1035, 1043, 1086, 986, 1009, 984, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8379, 8203, 8283, 8065, 8102, 8213, 8120, 8171] avg=8192.0000 max=8379 max_rank=0 min=8065 min_rank=3 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32606 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8320, 7936, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8448, 8192, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8064, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8064, 8192, 8320, 8192, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True + [2026-08-30 05:02:15] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 659.4 | throughput per GPU (TFLOP/s/GPU): 505.6 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.096926E+00 | loss scale: 1.0 | grad norm: 5.317 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1045, 1010, 1042, 1041, 977, 1017, 1008, 1060, 972, 1061, 1050, 1010, 1047, 1029, 1022, 1031, 1033, 1073, 1004, 1047, 1011, 1034, 1020, 1009, 1025, 1036, 1015, 980, 1049, 1052, 1034, 1010, 1002, 1002, 1042, 1044, 993, 992, 1008, 1070, 982, 1067, 1016, 1052, 1030, 992, 998, 1012, 1028, 1029, 1013, 954, 1052, 1035, 982, 1021, 1042, 992, 1017, 1074, 1067, 1014, 1050, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1021, 1032, 1006, 1003, 1017, 1013, 1010, 1037, 1033, 1017, 1085, 1041, 984, 980, 1059, 1008, 986, 1034, 1033, 1033, 990, 1062, 1016, 1043, 1027, 1035, 1011, 1010, 1002, 1056, 1000, 1015, 1059, 1091, 1015, 990, 1049, 997, 979, 1052, 1035, 1015, 1016, 1034, 1022, 1026, 1046, 1015, 983, 995, 1031, 1022, 1082, 1085, 1018, 1008, 1008, 1054, 1021, 984, 1004, 1023, 1030, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1012, 1029, 944, 1024, 1039, 1022, 1065, 988, 1053, 1000, 1053, 1009, 971, 1047, 1008, 1031, 1024, 1008, 1042, 1068, 1025, 1031, 1074, 1028, 1002, 1027, 979, 1025, 1036, 994, 1061, 1044, 995, 957, 979, 1058, 1061, 1001, 1054, 1004, 1030, 1072, 1037, 1024, 1054, 1052, 999, 984, 981, 1043, 1028, 1012, 1059, 1008, 1066, 1004, 1037, 1011, 982, 1031, 1045, 1065, 999, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1038, 1016, 1036, 1033, 990, 1063, 1011, 1063, 965, 996, 1065, 1069, 1032, 1017, 991, 1035, 1037, 992, 1067, 1025, 1018, 989, 974, 1039, 1039, 997, 1039, 1009, 1001, 1065, 1042, 999, 1032, 993, 1029, 993, 1010, 1040, 1028, 1007, 979, 1035, 1067, 1052, 1033, 1031, 1016, 1028, 981, 1016, 1073, 1038, 1029, 1002, 1064, 1015, 1021, 1021, 1030, 1030, 1017, 979, 1035, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1013, 1067, 1026, 1017, 1029, 1042, 1015, 1069, 1001, 1010, 1031, 1054, 1064, 978, 1011, 1048, 1030, 990, 1028, 1030, 1033, 963, 1024, 1025, 1008, 1036, 1007, 1046, 1035, 1035, 1023, 989, 1017, 1018, 1048, 1049, 1068, 1013, 1034, 1052, 1009, 1026, 1061, 954, 1051, 1034, 1022, 1029, 981, 996, 995, 1030, 1023, 1002, 997, 1018, 1040, 1027, 1036, 1012, 1051, 1023, 1046, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1013, 987, 1018, 1038, 1058, 1063, 1080, 1020, 1038, 1030, 1044, 1021, 999, 1062, 992, 1027, 1095, 1031, 995, 961, 1005, 1079, 960, 1017, 1019, 1103, 1039, 1023, 1029, 997, 966, 1037, 1017, 1014, 1046, 1000, 979, 1016, 1027, 1006, 968, 1026, 1001, 1037, 1008, 1012, 1023, 1010, 1043, 1016, 1001, 1088, 1073, 1017, 1000, 1047, 973, 1055, 1042, 1018, 1050, 1051, 1048, 978] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[972, 1000, 1060, 1028, 1068, 1018, 1037, 1031, 1054, 1004, 1036, 970, 1012, 1070, 1077, 1049, 1019, 1000, 1051, 1018, 1066, 990, 996, 1053, 993, 1061, 965, 989, 1094, 1068, 990, 1020, 1028, 1022, 1026, 1042, 1001, 1033, 1067, 971, 1044, 963, 1049, 1052, 1011, 1036, 1008, 1063, 986, 1031, 1000, 1083, 1023, 1033, 1020, 955, 1042, 961, 1022, 1002, 1022, 1074, 1008, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1027, 987, 1036, 1026, 1024, 1038, 1045, 1010, 1032, 1042, 1019, 1075, 1062, 1016, 1042, 986, 1029, 1067, 1015, 1026, 1015, 980, 1053, 1031, 1015, 1080, 975, 1005, 1016, 1063, 973, 1017, 1026, 1034, 1016, 1024, 1004, 1044, 1008, 1034, 991, 1046, 1019, 1017, 1017, 1007, 1023, 1025, 1003, 1008, 975, 1005, 1031, 1008, 1004, 1024, 1018, 1062, 1035, 1007, 1028, 1043, 1106, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1045, 1010, 1042, 1041, 977, 1017, 1008, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1061, 1027, 1010, 1047, 1029, 1022, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1073, 1004, 1047, 1011, 1034, 1020, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1036, 1015, 980, 1049, 1052, 1034, 1010, 13, 0, 0, 0, 0, 0, 0, 0, 1002, 1002, 1042, 1044, 993, 992, 1008, 1070, 15, 0, 0, 0, 0, 0, 0, 0, 982, 1067, 1016, 1052, 1030, 992, 998, 1012, 4, 1, 0, 0, 0, 0, 0, 0, 1028, 1029, 1013, 954, 1052, 1035, 982, 1021, 25, 8, 0, 0, 0, 0, 0, 0, 1042, 992, 1017, 1074, 1067, 1014, 1025, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8199, 8230, 8214, 8168, 8154, 8147, 8241] avg=8192.0000 max=8241 max_rank=7 min=8147 min_rank=6 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1021, 1032, 1006, 1003, 1017, 1013, 1010, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1017, 1061, 1041, 984, 980, 1059, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1034, 1033, 1033, 990, 1062, 1016, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1035, 1011, 1010, 1002, 1056, 1000, 1015, 12, 0, 0, 0, 0, 0, 0, 0, 1059, 1091, 1015, 990, 1049, 997, 979, 1052, 16, 0, 0, 0, 0, 0, 0, 0, 1035, 1015, 1016, 1034, 1022, 1026, 1046, 1015, 4, 0, 0, 0, 0, 0, 0, 0, 983, 995, 1031, 1022, 1082, 1085, 1018, 1008, 24, 8, 0, 0, 0, 0, 0, 0, 1008, 1054, 1021, 984, 1004, 1023, 1006, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8183, 8197, 8168, 8248, 8213, 8256, 8148] avg=8192.0000 max=8256 max_rank=6 min=8123 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1038, 1016, 1036, 1033, 990, 1063, 1011, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 965, 996, 1041, 1069, 1032, 1017, 991, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 992, 1067, 1025, 1018, 989, 974, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 997, 1039, 1009, 1001, 1065, 1042, 999, 13, 0, 0, 0, 0, 0, 0, 0, 1032, 993, 1029, 993, 1010, 1040, 1028, 1007, 16, 0, 0, 0, 0, 0, 0, 0, 979, 1035, 1067, 1052, 1033, 1031, 1016, 1028, 4, 1, 0, 0, 0, 0, 0, 0, 981, 1016, 1073, 1038, 1029, 1002, 1064, 1015, 24, 8, 0, 0, 0, 0, 0, 0, 1021, 1021, 1030, 1030, 1017, 979, 1011, 1060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8146, 8140, 8204, 8148, 8246, 8250, 8169] avg=8192.0000 max=8250 max_rank=6 min=8140 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1012, 1029, 944, 1024, 1039, 1022, 1065, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1000, 1030, 1009, 971, 1047, 1008, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1008, 1042, 1068, 1025, 1031, 1074, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1027, 979, 1025, 1036, 994, 1061, 1044, 12, 0, 0, 0, 0, 0, 0, 0, 995, 957, 979, 1058, 1061, 1001, 1054, 1004, 15, 0, 0, 0, 0, 0, 0, 0, 1030, 1072, 1037, 1024, 1054, 1052, 999, 984, 4, 0, 0, 0, 0, 0, 0, 0, 981, 1043, 1028, 1012, 1059, 1008, 1066, 1004, 23, 8, 0, 0, 0, 0, 0, 0, 1037, 1011, 982, 1031, 1045, 1065, 976, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8149, 8300, 8180, 8124, 8256, 8232, 8188] avg=8192.0000 max=8300 max_rank=2 min=8107 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1013, 1067, 1026, 1017, 1029, 1042, 1015, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1010, 1009, 1054, 1064, 978, 1011, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 990, 1028, 1030, 1033, 963, 1024, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1036, 1007, 1046, 1035, 1035, 1023, 989, 13, 0, 0, 0, 0, 0, 0, 0, 1017, 1018, 1048, 1049, 1068, 1013, 1034, 1052, 15, 0, 0, 0, 0, 0, 0, 0, 1009, 1026, 1061, 954, 1051, 1034, 1022, 1029, 4, 0, 0, 0, 0, 0, 0, 0, 981, 996, 995, 1030, 1023, 1002, 997, 1018, 25, 7, 0, 0, 0, 0, 0, 0, 1040, 1027, 1036, 1012, 1051, 1023, 1021, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8175, 8123, 8192, 8314, 8190, 8074, 8207] avg=8192.0000 max=8314 max_rank=4 min=8074 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[972, 1000, 1060, 1028, 1068, 1018, 1037, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1004, 1014, 970, 1012, 1070, 1077, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1000, 1051, 1018, 1066, 990, 996, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1061, 965, 989, 1094, 1068, 990, 1020, 12, 0, 0, 0, 0, 0, 0, 0, 1028, 1022, 1026, 1042, 1001, 1033, 1067, 971, 15, 0, 0, 0, 0, 0, 0, 0, 1044, 963, 1049, 1052, 1011, 1036, 1008, 1063, 4, 0, 0, 0, 0, 0, 0, 0, 986, 1031, 1000, 1083, 1023, 1033, 1020, 955, 24, 7, 0, 0, 0, 0, 0, 0, 1042, 961, 1022, 1002, 1022, 1074, 984, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8250, 8193, 8192, 8205, 8230, 8162, 8106] avg=8192.0000 max=8250 max_rank=1 min=8106 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1013, 987, 1018, 1038, 1058, 1063, 1080, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1030, 1022, 1021, 999, 1062, 992, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 1031, 995, 961, 1005, 1079, 960, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1103, 1039, 1023, 1029, 997, 966, 1037, 12, 0, 0, 0, 0, 0, 0, 0, 1017, 1014, 1046, 1000, 979, 1016, 1027, 1006, 15, 0, 0, 0, 0, 0, 0, 0, 968, 1026, 1001, 1037, 1008, 1012, 1023, 1010, 4, 1, 0, 0, 0, 0, 0, 0, 1043, 1016, 1001, 1088, 1073, 1017, 1000, 1047, 25, 7, 0, 0, 0, 0, 0, 0, 973, 1055, 1042, 1018, 1050, 1051, 1023, 978, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8191, 8142, 8225, 8120, 8090, 8317, 8190] avg=8192.0000 max=8317 max_rank=6 min=8090 min_rank=5 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1027, 987, 1036, 1026, 1024, 1038, 1045, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1042, 988, 1075, 1062, 1016, 1042, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1067, 1015, 1026, 1015, 980, 1053, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1080, 975, 1005, 1016, 1063, 973, 1017, 16, 1, 0, 0, 0, 0, 0, 0, 1026, 1034, 1016, 1024, 1004, 1044, 1008, 1034, 19, 0, 0, 0, 0, 0, 0, 0, 991, 1046, 1019, 1017, 1017, 1007, 1023, 1025, 7, 5, 0, 0, 0, 0, 0, 0, 1003, 1008, 975, 1005, 1031, 1008, 1004, 1024, 29, 11, 0, 0, 0, 0, 0, 0, 1018, 1062, 1035, 1007, 1028, 1043, 1077, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8243, 8211, 8161, 8209, 8157, 8098, 8287] avg=8192.0000 max=8287 max_rank=7 min=8098 min_rank=6 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8320, 8320, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8448, 8064, 8192, 8320, 8448, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8192, 8320, 8448, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1043, 1066, 1054, 1020, 1004, 1037, 1011, 1051, 1045, 1061, 1004, 1012, 1062, 1017, 1031, 1045, 952, 1037, 984, 1036, 1007, 1005, 972, 1010, 1041, 1022, 1035, 1010, 1039, 1065, 1048, 1006, 1022, 1027, 1066, 1005, 1023, 1034, 973, 1028, 1028, 1033, 1002, 1007, 1030, 999, 972, 1015, 1005, 993, 1044, 1014, 1088, 1046, 1018, 1011, 1032, 1032, 1027, 1023, 1084, 992, 992, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1024, 997, 994, 1005, 1078, 988, 1080, 1060, 1061, 1028, 1006, 1057, 975, 938, 1015, 1023, 1060, 956, 978, 1048, 1056, 1025, 1029, 1029, 996, 988, 1006, 994, 966, 1028, 1027, 1068, 1019, 1044, 1037, 1069, 1019, 1063, 1026, 1001, 1030, 1016, 1036, 1002, 1063, 1006, 1031, 993, 1037, 1026, 1071, 977, 1052, 1064, 1038, 1045, 1015, 1028, 1014, 1012, 1019, 1031, 1037, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1005, 1047, 1012, 1018, 1053, 1006, 989, 1034, 1019, 1012, 1059, 953, 1058, 1032, 1056, 1008, 1009, 1035, 1028, 1053, 1043, 1008, 1029, 1033, 1024, 1024, 963, 1052, 990, 983, 1080, 1018, 1030, 1061, 1007, 1018, 1034, 1045, 1015, 1047, 1049, 983, 1068, 1061, 1014, 1067, 1012, 1028, 1044, 971, 988, 1009, 978, 1003, 1042, 1016, 1082, 1018, 1025, 994, 1040, 1033, 998, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1053, 1022, 996, 1022, 1018, 983, 1046, 1042, 1018, 994, 973, 1013, 987, 1049, 1053, 1029, 1014, 1034, 996, 1028, 994, 1007, 1056, 1015, 1023, 1040, 1106, 1083, 986, 1027, 990, 1026, 1045, 1000, 991, 1026, 972, 1037, 1008, 1034, 1031, 1041, 1052, 1038, 1068, 1000, 977, 1044, 1043, 996, 1073, 998, 1034, 1013, 1034, 1061, 1046, 1037, 968, 1018, 1020, 1062, 1009, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1002, 1073, 1010, 1010, 1016, 1057, 1029, 993, 1059, 1087, 1029, 1008, 993, 1057, 988, 1006, 1011, 969, 1008, 1005, 1038, 1023, 1013, 1016, 962, 1034, 1067, 1008, 1028, 1038, 976, 1014, 1028, 1025, 1045, 1010, 999, 1012, 997, 1006, 1050, 1020, 1045, 1051, 1013, 989, 995, 1077, 1035, 1061, 997, 1056, 1082, 1007, 1044, 1028, 1042, 1055, 1054, 961, 1003, 1060, 1044, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1036, 1016, 988, 1036, 1001, 1025, 995, 969, 1040, 1022, 1072, 1016, 1019, 1032, 1040, 1025, 1042, 1036, 1047, 1016, 984, 1041, 968, 1029, 1022, 988, 1034, 1038, 1047, 1061, 1023, 1007, 1025, 992, 1025, 1057, 1052, 987, 1003, 1063, 1016, 1061, 1065, 1007, 1024, 1047, 995, 1017, 1034, 1025, 1031, 981, 1034, 1012, 988, 1004, 1078, 1000, 1036, 1032, 1066, 1019, 1010, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[995, 1051, 1024, 1012, 996, 1055, 1035, 1033, 1031, 1045, 1039, 1006, 1032, 1045, 1001, 1007, 1042, 996, 1011, 1081, 1008, 1033, 964, 1105, 993, 992, 1035, 1016, 1031, 992, 1065, 1006, 1013, 1043, 1045, 1028, 1071, 983, 1052, 1023, 1034, 1013, 1069, 986, 1090, 988, 976, 1017, 1052, 1053, 1005, 1056, 1011, 1001, 1025, 997, 1026, 975, 1042, 1030, 1013, 992, 1033, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1028, 1032, 999, 1004, 1049, 1022, 975, 970, 1047, 995, 989, 992, 1030, 1031, 1007, 1021, 1030, 1001, 1011, 1048, 1054, 1011, 978, 997, 1018, 987, 1019, 1030, 1064, 1042, 1002, 993, 990, 1020, 1051, 1048, 1019, 1018, 1051, 1093, 1036, 1023, 1052, 1045, 1034, 1027, 1044, 1034, 1036, 1015, 1035, 1073, 1049, 1023, 1033, 1050, 1041, 1055, 1036, 1018, 1020, 991, 1022, 978] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1043, 1066, 1054, 1020, 1004, 1037, 1011, 1051, 11, 6, 0, 0, 0, 0, 0, 0, 1045, 1061, 1004, 1012, 1062, 1017, 1031, 1045, 3, 0, 0, 0, 0, 0, 0, 0, 952, 1037, 984, 1036, 1007, 1005, 972, 1010, 29, 14, 0, 0, 0, 0, 0, 0, 1041, 1022, 1035, 1010, 1039, 1065, 1048, 1006, 9, 8, 0, 0, 0, 0, 0, 0, 1022, 1027, 1066, 1005, 1023, 1034, 973, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1033, 980, 1007, 1030, 999, 972, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 993, 1044, 1014, 1059, 1046, 1018, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1032, 1027, 1023, 1084, 992, 992, 1039, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8280, 8046, 8283, 8158, 8064, 8190, 8212] avg=8192.0000 max=8303 max_rank=0 min=8046 min_rank=2 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1024, 997, 994, 1005, 1078, 988, 1080, 1060, 10, 6, 0, 0, 0, 0, 0, 0, 1061, 1028, 1006, 1057, 975, 938, 1015, 1023, 3, 0, 0, 0, 0, 0, 0, 0, 1060, 956, 978, 1048, 1056, 1025, 1029, 1029, 28, 15, 0, 0, 0, 0, 0, 0, 996, 988, 1006, 994, 966, 1028, 1027, 1068, 9, 9, 0, 0, 0, 0, 0, 0, 1019, 1044, 1037, 1069, 1019, 1063, 1026, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1016, 1012, 1002, 1063, 1006, 1031, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1026, 1071, 977, 1024, 1064, 1038, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1028, 1014, 1012, 1019, 1031, 1037, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8106, 8224, 8091, 8259, 8153, 8282, 8179] avg=8192.0000 max=8282 max_rank=6 min=8091 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1036, 1016, 988, 1036, 1001, 1025, 995, 969, 11, 6, 0, 0, 0, 0, 0, 0, 1040, 1022, 1072, 1016, 1019, 1032, 1040, 1025, 4, 0, 0, 0, 0, 0, 0, 0, 1042, 1036, 1047, 1016, 984, 1041, 968, 1029, 28, 15, 0, 0, 0, 0, 0, 0, 1022, 988, 1034, 1038, 1047, 1061, 1023, 1007, 9, 9, 0, 0, 0, 0, 0, 0, 1025, 992, 1025, 1057, 1052, 987, 1003, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1061, 1041, 1007, 1024, 1047, 995, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1025, 1031, 981, 1006, 1012, 988, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1000, 1036, 1032, 1066, 1019, 1010, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8270, 8206, 8238, 8184, 8208, 8081, 8266] avg=8192.0000 max=8270 max_rank=1 min=8081 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32739 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1053, 1022, 996, 1022, 1018, 983, 1046, 1042, 11, 6, 0, 0, 0, 0, 0, 0, 1018, 994, 973, 1013, 987, 1049, 1053, 1029, 4, 0, 0, 0, 0, 0, 0, 0, 1014, 1034, 996, 1028, 994, 1007, 1056, 1015, 28, 15, 0, 0, 0, 0, 0, 0, 1023, 1040, 1106, 1083, 986, 1027, 990, 1026, 9, 9, 0, 0, 0, 0, 0, 0, 1045, 1000, 991, 1026, 972, 1037, 1008, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1041, 1028, 1038, 1068, 1000, 977, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 996, 1073, 998, 1006, 1013, 1034, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1037, 968, 1018, 1020, 1062, 1009, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8120, 8187, 8299, 8093, 8227, 8224, 8187] avg=8192.0000 max=8299 max_rank=3 min=8093 min_rank=4 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1002, 1073, 1010, 1010, 1016, 1057, 1029, 993, 10, 6, 0, 0, 0, 0, 0, 0, 1059, 1087, 1029, 1008, 993, 1057, 988, 1006, 3, 0, 0, 0, 0, 0, 0, 0, 1011, 969, 1008, 1005, 1038, 1023, 1013, 1016, 29, 15, 0, 0, 0, 0, 0, 0, 962, 1034, 1067, 1008, 1028, 1038, 976, 1014, 9, 9, 0, 0, 0, 0, 0, 0, 1028, 1025, 1045, 1010, 999, 1012, 997, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1020, 1021, 1051, 1013, 989, 995, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1061, 997, 1056, 1053, 1007, 1044, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1055, 1054, 961, 1003, 1060, 1044, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8230, 8127, 8145, 8103, 8216, 8281, 8228] avg=8192.0000 max=8281 max_rank=6 min=8103 min_rank=4 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1005, 1047, 1012, 1018, 1053, 1006, 989, 1034, 15, 11, 0, 0, 0, 0, 0, 0, 1019, 1012, 1059, 953, 1058, 1032, 1056, 1008, 9, 0, 0, 0, 0, 0, 0, 0, 1009, 1035, 1028, 1053, 1043, 1008, 1029, 1033, 32, 21, 0, 0, 0, 0, 0, 0, 1024, 1024, 963, 1052, 990, 983, 1080, 1018, 12, 13, 0, 0, 0, 0, 0, 0, 1030, 1061, 1007, 1018, 1034, 1045, 1015, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 983, 1034, 1061, 1014, 1067, 1012, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 971, 988, 1009, 946, 1003, 1042, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1018, 1025, 994, 1040, 1033, 998, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8206, 8291, 8159, 8230, 8248, 8019, 8193] avg=8192.0000 max=8291 max_rank=2 min=8019 min_rank=6 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[995, 1051, 1024, 1012, 996, 1055, 1035, 1033, 10, 6, 0, 0, 0, 0, 0, 0, 1031, 1045, 1039, 1006, 1032, 1045, 1001, 1007, 3, 0, 0, 0, 0, 0, 0, 0, 1042, 996, 1011, 1081, 1008, 1033, 964, 1105, 27, 15, 0, 0, 0, 0, 0, 0, 993, 992, 1035, 1016, 1031, 992, 1065, 1006, 9, 9, 0, 0, 0, 0, 0, 0, 1013, 1043, 1045, 1028, 1071, 983, 1052, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1013, 1045, 986, 1090, 988, 976, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1053, 1005, 1056, 984, 1001, 1025, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 975, 1042, 1030, 1013, 992, 1033, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8209, 8282, 8148, 8239, 8149, 8173, 8119] avg=8192.0000 max=8282 max_rank=2 min=8119 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1028, 1032, 999, 1004, 1049, 1022, 975, 970, 11, 6, 0, 0, 0, 0, 0, 0, 1047, 995, 989, 992, 1030, 1031, 1007, 1021, 3, 0, 0, 0, 0, 0, 0, 0, 1030, 1001, 1011, 1048, 1054, 1011, 978, 997, 28, 15, 0, 0, 0, 0, 0, 0, 1018, 987, 1019, 1030, 1064, 1042, 1002, 993, 9, 9, 0, 0, 0, 0, 0, 0, 990, 1020, 1051, 1048, 1019, 1018, 1051, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1023, 1028, 1045, 1034, 1027, 1044, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1015, 1035, 1073, 1021, 1023, 1033, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1055, 1036, 1018, 1020, 991, 1022, 978, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8115, 8173, 8173, 8270, 8271, 8286, 8152] avg=8192.0000 max=8286 max_rank=6 min=8096 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32979 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8064, 8064, 8320, 8192, 8192, 8064, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8448, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8064, 8192, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1054, 995, 1008, 1019, 1020, 1026, 1072, 1011, 1043, 1057, 1024, 1033, 1017, 1012, 948, 1034, 1006, 1004, 1016, 1028, 1038, 999, 1020, 1065, 999, 1046, 1016, 1069, 1060, 1005, 982, 1059, 1049, 1058, 1016, 1014, 976, 1000, 1010, 1048, 1032, 1045, 1061, 977, 1053, 1023, 1066, 977, 956, 1006, 1014, 1045, 1030, 1023, 1019, 1029, 1093, 990, 988, 997, 1020, 1052, 991, 1093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1021, 1007, 1030, 1015, 1009, 1020, 1038, 1049, 1018, 1011, 1027, 1006, 973, 1047, 1056, 991, 1063, 974, 1027, 1035, 1045, 1011, 998, 1063, 1043, 996, 1021, 1021, 1042, 1028, 1054, 1069, 1022, 999, 1016, 1035, 983, 1040, 1004, 1043, 1054, 1049, 1045, 1036, 982, 1034, 1055, 1043, 979, 1044, 984, 1019, 968, 1019, 1043, 1057, 999, 994, 1019, 1067, 1010, 1004, 1038, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[990, 1027, 1071, 1020, 983, 1003, 1036, 979, 973, 1042, 1017, 982, 1055, 976, 1068, 1023, 1048, 1019, 1045, 1039, 998, 1055, 1020, 986, 1043, 1025, 996, 991, 956, 1046, 1013, 1051, 1024, 986, 968, 1086, 1012, 1065, 976, 1049, 1036, 1010, 1067, 1036, 1040, 1016, 1030, 1047, 1007, 1060, 1056, 1052, 1041, 984, 1009, 1060, 1060, 1008, 1030, 1048, 1009, 1038, 1022, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1011, 1029, 1069, 1050, 1072, 1010, 976, 1083, 1020, 1082, 991, 1002, 985, 988, 1053, 1056, 990, 1075, 986, 987, 1009, 1070, 1071, 1005, 1020, 1024, 1014, 1040, 1075, 1039, 1029, 1034, 959, 1027, 1024, 990, 997, 988, 1016, 1070, 1074, 1023, 1031, 1010, 984, 1012, 1022, 1035, 1022, 1017, 991, 1023, 1025, 1055, 1047, 1001, 1024, 1018, 1083, 1008, 1014, 1022, 971, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1039, 999, 1036, 1045, 998, 1077, 999, 1058, 1056, 1063, 1083, 1057, 1018, 998, 1007, 997, 1031, 1004, 1019, 993, 1031, 980, 996, 999, 1028, 1030, 1047, 1000, 1006, 1025, 1035, 1039, 1060, 988, 1007, 1033, 1010, 1014, 1017, 1010, 944, 1032, 965, 1050, 1047, 1034, 965, 1039, 1012, 1076, 1083, 1023, 1054, 1040, 1008, 1021, 1031, 994, 1063, 1002, 1006, 1051, 1048, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1026, 1022, 1013, 1018, 996, 1027, 1046, 1052, 1036, 1013, 995, 1010, 1016, 1020, 992, 1025, 1026, 1015, 974, 1028, 1017, 982, 1046, 1070, 1028, 994, 1046, 968, 1010, 1049, 1037, 1034, 1058, 1000, 994, 1064, 1039, 1039, 963, 1001, 1010, 1021, 1011, 1028, 995, 1060, 1009, 1030, 1020, 1036, 1003, 1004, 1015, 983, 1034, 1029, 1058, 1040, 1074, 1060, 1062, 1045, 1026, 1094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1025, 1019, 1013, 1041, 1007, 1049, 980, 1090, 1039, 1012, 1038, 1017, 1027, 1025, 1041, 1019, 957, 1051, 1059, 1032, 1022, 1046, 1005, 1032, 1051, 1005, 1027, 1025, 1045, 1047, 1037, 1080, 1011, 1002, 1011, 1013, 1036, 1035, 1002, 1026, 1012, 1057, 1000, 1024, 1029, 1019, 986, 1009, 1103, 1007, 979, 1040, 1009, 1010, 962, 1024, 1061, 1054, 965, 1024, 1022, 981, 1017, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1009, 975, 1020, 1030, 1009, 1032, 1029, 1014, 1018, 1077, 1008, 1047, 1033, 1002, 1015, 1045, 1036, 1027, 1004, 987, 1006, 1038, 988, 1024, 1020, 1013, 1050, 1038, 1011, 1040, 1032, 1032, 1023, 1008, 963, 1026, 1053, 1042, 1031, 1015, 1019, 1043, 998, 1057, 1040, 995, 1091, 1054, 1010, 1050, 1017, 1056, 992, 1043, 1051, 1012, 1004, 1055, 1041, 955, 1045, 1006, 1020, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1054, 995, 1008, 1019, 1020, 1026, 1072, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1057, 1024, 1033, 1017, 1012, 948, 1034, 7, 1, 0, 0, 0, 0, 0, 0, 1006, 1004, 1016, 1028, 1038, 999, 1020, 1065, 17, 10, 0, 0, 0, 0, 0, 0, 999, 1046, 1016, 1069, 1060, 1005, 982, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1058, 1016, 1014, 976, 1000, 1010, 1048, 37, 12, 0, 0, 0, 0, 0, 0, 1032, 1028, 1061, 977, 1053, 1023, 1066, 977, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1006, 1014, 1045, 1030, 1023, 1019, 1029, 5, 0, 0, 0, 0, 0, 0, 0, 1093, 990, 988, 997, 1020, 1052, 991, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8176, 8203, 8199, 8220, 8217, 8127, 8195] avg=8192.0000 max=8220 max_rank=4 min=8127 min_rank=6 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1021, 1007, 1030, 1015, 1009, 1020, 1038, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1011, 1027, 1006, 973, 1047, 1056, 991, 7, 1, 0, 0, 0, 0, 0, 0, 1063, 974, 1027, 1035, 1045, 1011, 998, 1063, 17, 10, 0, 0, 0, 0, 0, 0, 1043, 996, 1021, 1021, 1042, 1028, 1054, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 999, 1016, 1035, 983, 1040, 1004, 1043, 38, 11, 0, 0, 0, 0, 0, 0, 1054, 1032, 1045, 1036, 982, 1034, 1055, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1044, 984, 1019, 968, 1019, 1043, 1057, 5, 0, 0, 0, 0, 0, 0, 0, 999, 994, 1019, 1067, 1010, 1004, 1038, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8137, 8243, 8236, 8191, 8281, 8118, 8147] avg=8192.0000 max=8281 max_rank=5 min=8118 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1025, 1019, 1013, 1041, 1007, 1049, 980, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1012, 1038, 1017, 1027, 1025, 1041, 1019, 7, 1, 0, 0, 0, 0, 0, 0, 957, 1051, 1059, 1032, 1022, 1046, 1005, 1032, 17, 10, 0, 0, 0, 0, 0, 0, 1051, 1005, 1027, 1025, 1045, 1047, 1037, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1002, 1011, 1013, 1036, 1035, 1002, 1026, 38, 11, 0, 0, 0, 0, 0, 0, 1012, 1040, 1000, 1024, 1029, 1019, 986, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 1007, 979, 1040, 1009, 1010, 962, 1024, 5, 0, 0, 0, 0, 0, 0, 0, 1061, 1054, 965, 1024, 1022, 981, 1017, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8226, 8231, 8279, 8185, 8119, 8139, 8139] avg=8192.0000 max=8279 max_rank=3 min=8119 min_rank=5 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32582 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[990, 1027, 1071, 1020, 983, 1003, 1036, 973, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1042, 1017, 982, 1055, 976, 1068, 1023, 7, 1, 0, 0, 0, 0, 0, 0, 1048, 1019, 1045, 1039, 998, 1055, 1020, 986, 16, 9, 0, 0, 0, 0, 0, 0, 1043, 1025, 996, 991, 956, 1046, 1013, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 986, 968, 1086, 1012, 1065, 976, 1049, 37, 11, 0, 0, 0, 0, 0, 0, 1036, 994, 1067, 1036, 1040, 1016, 1030, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1060, 1056, 1052, 1041, 984, 1009, 1060, 5, 0, 0, 0, 0, 0, 0, 0, 1060, 1008, 1030, 1048, 1009, 1038, 1022, 1001, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8144, 8235, 8084, 8214, 8266, 8274, 8216] avg=8192.0000 max=8274 max_rank=6 min=8084 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32970 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1026, 1022, 1013, 1018, 996, 1027, 1046, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1013, 995, 1010, 1016, 1020, 992, 1025, 7, 1, 0, 0, 0, 0, 0, 0, 1026, 1015, 974, 1028, 1017, 982, 1046, 1070, 17, 9, 0, 0, 0, 0, 0, 0, 1028, 994, 1046, 968, 1010, 1049, 1037, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1000, 994, 1064, 1039, 1039, 963, 1001, 36, 12, 0, 0, 0, 0, 0, 0, 1010, 1005, 1011, 1028, 995, 1060, 1009, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1036, 1003, 1004, 1015, 983, 1034, 1029, 5, 0, 0, 0, 0, 0, 0, 0, 1058, 1040, 1074, 1060, 1062, 1045, 1026, 1065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8115, 8184, 8130, 8206, 8148, 8129, 8430] avg=8192.0000 max=8430 max_rank=7 min=8115 min_rank=1 imbalance_ratio=1.029053 max_load_reduction=57106 max_load_reduction_pct=0.871368 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1039, 999, 1036, 1045, 998, 1077, 999, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1063, 1083, 1057, 1018, 998, 1007, 997, 13, 8, 0, 0, 0, 0, 0, 0, 1031, 1004, 1019, 993, 1031, 980, 996, 999, 20, 13, 0, 0, 0, 0, 0, 0, 1028, 1030, 1047, 1000, 1006, 1025, 1035, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 988, 1007, 1033, 1010, 1014, 1017, 1010, 41, 14, 0, 0, 0, 0, 0, 0, 944, 1006, 965, 1050, 1047, 1034, 965, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1076, 1083, 1023, 1054, 1040, 1008, 1021, 10, 0, 0, 0, 0, 0, 0, 0, 1031, 994, 1063, 1002, 1006, 1051, 1048, 982, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8300, 8086, 8169, 8194, 8050, 8327, 8177] avg=8192.0000 max=8327 max_rank=6 min=8050 min_rank=5 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1009, 975, 1020, 1030, 1009, 1032, 1029, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1077, 1008, 1047, 1033, 1002, 1015, 1045, 7, 1, 0, 0, 0, 0, 0, 0, 1036, 1027, 1004, 987, 1006, 1038, 988, 1024, 16, 10, 0, 0, 0, 0, 0, 0, 1020, 1013, 1050, 1038, 1011, 1040, 1032, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1008, 963, 1026, 1053, 1042, 1031, 1015, 36, 11, 0, 0, 0, 0, 0, 0, 1019, 1026, 998, 1057, 1040, 995, 1091, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1050, 1017, 1056, 992, 1043, 1051, 1012, 5, 0, 0, 0, 0, 0, 0, 0, 1004, 1055, 1041, 955, 1045, 1006, 1020, 985, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8253, 8136, 8200, 8208, 8280, 8236, 8111] avg=8192.0000 max=8280 max_rank=5 min=8111 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1011, 1029, 1069, 1050, 1072, 1010, 976, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1082, 991, 1002, 985, 988, 1053, 1056, 7, 1, 0, 0, 0, 0, 0, 0, 990, 1075, 986, 987, 1009, 1070, 1071, 1005, 16, 9, 0, 0, 0, 0, 0, 0, 1020, 1024, 1014, 1040, 1075, 1039, 1029, 998, 0, 0, 0, 0, 0, 0, 0, 0, 959, 1027, 1024, 990, 997, 988, 1016, 1070, 36, 11, 0, 0, 0, 0, 0, 0, 1074, 1007, 1031, 1010, 984, 1012, 1022, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1017, 991, 1023, 1025, 1055, 1047, 1001, 5, 0, 0, 0, 0, 0, 0, 0, 1024, 1018, 1083, 1008, 1014, 1022, 971, 981, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 8185, 8218, 8239, 8118, 8175, 8186, 8121] avg=8192.0000 max=8294 max_rank=0 min=8118 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32600 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8448, 8192, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8064, 8320, 8192, 8320, 8064, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8448, 8192, 8192, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1023, 991, 1020, 1025, 1032, 982, 1081, 1017, 1057, 997, 1011, 1027, 983, 1009, 1016, 1023, 1010, 996, 975, 1052, 1000, 1035, 1021, 1069, 1016, 1000, 1060, 1020, 1034, 952, 1066, 1040, 987, 1004, 1005, 991, 1000, 1027, 1050, 981, 1009, 1060, 1058, 1030, 1060, 985, 1101, 1023, 1021, 1029, 1036, 1011, 1049, 1034, 986, 1059, 1057, 1004, 1067, 1011, 1039, 1052, 1037, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1007, 1031, 1057, 1028, 1066, 1049, 1046, 990, 1013, 984, 1066, 1056, 1028, 1008, 1022, 1013, 1003, 1002, 1013, 1021, 994, 1057, 1015, 1052, 1021, 990, 1030, 1012, 1055, 997, 1046, 1048, 1010, 991, 1010, 1002, 1002, 1058, 1067, 993, 1036, 973, 998, 1002, 1061, 1026, 1075, 1035, 1015, 1012, 1015, 1023, 1043, 1041, 1053, 984, 1030, 1035, 1011, 964, 1046, 1039, 1017, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1079, 1015, 1028, 1010, 1025, 1003, 975, 1049, 1078, 1019, 1085, 1009, 1061, 994, 1061, 985, 958, 1014, 1018, 1040, 977, 1010, 1043, 1049, 1069, 1031, 1022, 1035, 1010, 1020, 941, 1015, 975, 1054, 1025, 980, 1005, 1048, 1042, 1012, 1057, 997, 1038, 1053, 1047, 999, 1016, 1070, 1022, 996, 1032, 989, 1050, 1027, 1061, 989, 1050, 1032, 973, 1040, 1081, 1020, 1006, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1046, 1010, 1016, 992, 1021, 1005, 1032, 1032, 985, 1011, 1002, 1000, 1069, 1079, 1040, 1019, 1022, 999, 987, 1035, 1013, 1052, 1073, 1010, 1050, 1042, 1034, 1023, 981, 1020, 1048, 999, 1012, 1019, 1032, 1025, 1029, 1003, 1049, 986, 1025, 1031, 988, 1074, 1014, 987, 1043, 1023, 1063, 1016, 1050, 987, 1014, 1045, 1035, 996, 1018, 1048, 1016, 1052, 1044, 1019, 1010, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1033, 1032, 1033, 961, 1029, 1026, 1048, 1006, 1019, 1057, 1039, 1035, 1027, 975, 1011, 983, 1067, 1010, 1056, 1039, 1050, 993, 1050, 996, 1009, 1032, 993, 970, 1061, 1028, 998, 1054, 987, 1039, 1044, 1024, 1062, 1008, 1008, 1025, 1033, 1043, 1029, 1057, 1013, 1016, 1061, 1019, 1071, 1026, 1012, 1005, 981, 1045, 1099, 1067, 985, 1016, 1040, 1002, 1010, 1015, 1008, 966] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1057, 1021, 1086, 1060, 997, 1030, 1020, 1024, 1023, 998, 1035, 1027, 1066, 994, 1071, 1037, 1018, 987, 1012, 969, 1031, 1054, 991, 986, 1036, 1056, 1054, 1035, 1048, 1041, 983, 1008, 1020, 991, 1003, 1031, 1041, 1006, 973, 1038, 1032, 1049, 990, 1059, 979, 1029, 991, 996, 1059, 1061, 1022, 1008, 1014, 1025, 1000, 1022, 1038, 1027, 1015, 1024, 1069, 1017, 1046, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1029, 1028, 1039, 994, 1045, 993, 998, 1018, 1004, 1005, 996, 1043, 1032, 1011, 1036, 987, 1031, 1072, 992, 989, 1032, 1065, 1031, 1030, 1004, 1049, 1037, 991, 1049, 992, 1077, 1019, 1033, 1035, 1051, 974, 1055, 1003, 1001, 1015, 1069, 1058, 1045, 1029, 1039, 984, 996, 1063, 1044, 983, 1071, 1005, 1030, 1047, 1004, 1006, 974, 1056, 1036, 1009, 979, 1053, 1048, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1054, 988, 1031, 1015, 1056, 979, 1017, 1009, 1059, 995, 1016, 1005, 1079, 1001, 1058, 967, 976, 1018, 1043, 1076, 1057, 978, 1039, 955, 970, 979, 999, 1063, 1033, 1037, 1021, 1029, 1018, 1019, 1049, 1020, 1044, 988, 1020, 1019, 999, 1040, 1006, 1073, 1058, 1045, 1028, 1026, 1048, 1037, 1029, 1005, 987, 1020, 1023, 1006, 1033, 1031, 1050, 1038, 1016, 1057, 1031, 1071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1023, 991, 1020, 1025, 1032, 982, 1081, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 997, 1011, 1027, 983, 1009, 1016, 1023, 3, 0, 0, 0, 0, 0, 0, 0, 1010, 996, 975, 1052, 1000, 1035, 1021, 1069, 25, 2, 0, 0, 0, 0, 0, 0, 1016, 1000, 1060, 1020, 1034, 952, 1066, 1040, 6, 0, 0, 0, 0, 0, 0, 0, 987, 1004, 1005, 991, 1000, 1027, 1050, 981, 50, 0, 0, 0, 0, 0, 0, 0, 1009, 1060, 1058, 980, 1060, 985, 1101, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1029, 1036, 1011, 1049, 1034, 986, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1004, 1067, 1011, 1014, 1052, 1037, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8126, 8185, 8194, 8095, 8276, 8214, 8275] avg=8192.0000 max=8276 max_rank=5 min=8095 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1029, 1028, 1039, 994, 1045, 993, 998, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1005, 996, 1043, 1032, 1011, 1036, 987, 4, 0, 0, 0, 0, 0, 0, 0, 1031, 1072, 992, 989, 1032, 1065, 1031, 1030, 24, 2, 0, 0, 0, 0, 0, 0, 1004, 1049, 1037, 991, 1049, 992, 1077, 1019, 6, 0, 0, 0, 0, 0, 0, 0, 1033, 1035, 1051, 974, 1055, 1003, 1001, 1015, 50, 0, 0, 0, 0, 0, 0, 0, 1069, 1058, 1045, 979, 1039, 984, 996, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 983, 1071, 1005, 1030, 1047, 1004, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1056, 1036, 1009, 955, 1053, 1048, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8118, 8268, 8224, 8217, 8233, 8178, 8154] avg=8192.0000 max=8268 max_rank=2 min=8118 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1079, 1015, 1028, 1010, 1025, 1003, 975, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 1019, 1085, 1009, 1061, 994, 1061, 985, 3, 0, 0, 0, 0, 0, 0, 0, 958, 1014, 1018, 1040, 977, 1010, 1043, 1049, 26, 2, 0, 0, 0, 0, 0, 0, 1069, 1031, 1022, 1035, 1010, 1020, 941, 1015, 6, 0, 0, 0, 0, 0, 0, 0, 975, 1054, 1025, 980, 1005, 1048, 1042, 1012, 51, 0, 0, 0, 0, 0, 0, 0, 1057, 997, 1038, 1002, 1047, 999, 1016, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 996, 1032, 989, 1050, 1027, 1061, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1032, 973, 1040, 1055, 1020, 1006, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8295, 8137, 8149, 8192, 8226, 8155, 8198] avg=8192.0000 max=8295 max_rank=1 min=8137 min_rank=2 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1033, 1032, 1033, 961, 1029, 1026, 1048, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1057, 1039, 1035, 1027, 975, 1011, 983, 4, 0, 0, 0, 0, 0, 0, 0, 1067, 1010, 1056, 1039, 1050, 993, 1050, 996, 24, 2, 0, 0, 0, 0, 0, 0, 1009, 1032, 993, 970, 1061, 1028, 998, 1054, 6, 0, 0, 0, 0, 0, 0, 0, 987, 1039, 1044, 1024, 1062, 1008, 1008, 1025, 51, 0, 0, 0, 0, 0, 0, 0, 1033, 1043, 1029, 1006, 1013, 1016, 1061, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1026, 1012, 1005, 981, 1045, 1099, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1016, 1040, 1002, 986, 1015, 1008, 966, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8150, 8287, 8151, 8248, 8220, 8294, 8018] avg=8192.0000 max=8294 max_rank=6 min=8018 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1057, 1021, 1086, 1060, 997, 1030, 1020, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 998, 1035, 1027, 1066, 994, 1071, 1037, 4, 0, 0, 0, 0, 0, 0, 0, 1018, 987, 1012, 969, 1031, 1054, 991, 986, 26, 2, 0, 0, 0, 0, 0, 0, 1036, 1056, 1054, 1035, 1048, 1041, 983, 1008, 6, 0, 0, 0, 0, 0, 0, 0, 1020, 991, 1003, 1031, 1041, 1006, 973, 1038, 52, 0, 0, 0, 0, 0, 0, 0, 1032, 1049, 990, 1007, 979, 1029, 991, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1061, 1022, 1008, 1014, 1025, 1000, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1027, 1015, 1024, 1043, 1017, 1046, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8255, 8076, 8267, 8155, 8073, 8199, 8216] avg=8192.0000 max=8295 max_rank=0 min=8073 min_rank=5 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32643 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1007, 1031, 1057, 1028, 1066, 1049, 1046, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 984, 1066, 1056, 1028, 1008, 1022, 1013, 3, 0, 0, 0, 0, 0, 0, 0, 1003, 1002, 1013, 1021, 994, 1057, 1015, 1052, 25, 2, 0, 0, 0, 0, 0, 0, 1021, 990, 1030, 1012, 1055, 997, 1046, 1048, 6, 0, 0, 0, 0, 0, 0, 0, 1010, 991, 1010, 1002, 1002, 1058, 1067, 993, 49, 0, 0, 0, 0, 0, 0, 0, 1036, 973, 998, 953, 1061, 1026, 1075, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1012, 1015, 1023, 1043, 1041, 1053, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1035, 1011, 964, 1021, 1039, 1017, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8193, 8184, 8205, 8182, 8157, 8175, 8166] avg=8192.0000 max=8274 max_rank=0 min=8157 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1046, 1010, 1016, 992, 1021, 1005, 1032, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1011, 1002, 1000, 1069, 1079, 1040, 1019, 4, 0, 0, 0, 0, 0, 0, 0, 1022, 999, 987, 1035, 1013, 1052, 1073, 1010, 25, 2, 0, 0, 0, 0, 0, 0, 1050, 1042, 1034, 1023, 981, 1020, 1048, 999, 6, 0, 0, 0, 0, 0, 0, 0, 1012, 1019, 1032, 1025, 1029, 1003, 1049, 986, 52, 0, 0, 0, 0, 0, 0, 0, 1025, 1031, 988, 1022, 1014, 987, 1043, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1016, 1050, 987, 1014, 1045, 1035, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1048, 1016, 1052, 1019, 1019, 1010, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8209, 8218, 8203, 8207, 8133, 8194, 8218] avg=8192.0000 max=8218 max_rank=2 min=8133 min_rank=5 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1051, 988, 1031, 1015, 1056, 979, 1017, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 995, 1016, 1005, 1079, 1001, 1058, 967, 7, 3, 0, 0, 0, 0, 0, 0, 976, 1018, 1043, 1076, 1057, 978, 1039, 955, 30, 9, 0, 0, 0, 0, 0, 0, 970, 979, 999, 1063, 1033, 1037, 1021, 1029, 12, 0, 0, 0, 0, 0, 0, 0, 1018, 1019, 1049, 1020, 1044, 988, 1020, 1019, 57, 6, 0, 0, 0, 0, 0, 0, 999, 1040, 1006, 1016, 1058, 1045, 1028, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1037, 1029, 1005, 987, 1020, 1023, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1031, 1050, 1038, 980, 1057, 1031, 1071, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8190, 8181, 8143, 8240, 8218, 8127, 8291] avg=8192.0000 max=8291 max_rank=7 min=8127 min_rank=6 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8192, 8320, 8064, 8320, 8192, 8320, 8192, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8320, 8320, 8448, 8192, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8064, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8064, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1006, 1021, 1052, 1021, 1046, 1022, 987, 1048, 1014, 1028, 990, 1041, 1055, 996, 1064, 990, 1026, 1060, 1031, 1002, 1043, 991, 1030, 1055, 1005, 1017, 1031, 1002, 1077, 1005, 1009, 1003, 1012, 1017, 995, 1010, 1023, 1018, 1023, 1031, 1025, 1044, 1006, 1037, 999, 995, 994, 994, 1047, 1047, 1069, 1076, 1004, 1065, 1004, 1061, 1033, 996, 1010, 1031, 1087, 962, 1019, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1026, 1034, 1083, 1056, 1001, 986, 1040, 1025, 1039, 1017, 994, 1033, 996, 996, 1047, 1090, 1030, 975, 989, 992, 987, 1046, 989, 994, 1072, 988, 1040, 1037, 1015, 1031, 1049, 1035, 1036, 1064, 1005, 1060, 1035, 1007, 1022, 1050, 1032, 995, 1026, 1002, 1018, 1062, 1027, 988, 1033, 1040, 984, 1036, 1035, 1029, 1028, 1057, 1044, 1029, 986, 1062, 1035, 989, 981, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[997, 1083, 1010, 1046, 1021, 1037, 990, 1037, 1036, 1045, 1010, 1028, 976, 1039, 1045, 1019, 1089, 1031, 1045, 1034, 1024, 1003, 1024, 1060, 1043, 1045, 1051, 1035, 1007, 990, 1023, 1016, 1037, 1076, 1069, 993, 992, 1063, 1025, 1016, 1045, 1030, 987, 980, 943, 999, 999, 1021, 1027, 1038, 1022, 1032, 1061, 1006, 1033, 1028, 1039, 1002, 1029, 971, 990, 1045, 1019, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1040, 1034, 1060, 1032, 1025, 1008, 1066, 1028, 1071, 1035, 1045, 1024, 1042, 971, 1047, 1010, 1015, 1019, 1010, 1013, 997, 1038, 996, 1035, 1050, 1047, 1032, 985, 991, 1034, 972, 996, 1019, 1055, 989, 1049, 978, 1061, 1056, 1016, 1065, 1064, 1003, 1030, 1013, 996, 1027, 1011, 1014, 1026, 1047, 1026, 1026, 1054, 1026, 1032, 1016, 1025, 999, 1007, 995, 1034, 1005, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1065, 1022, 1047, 1062, 997, 1027, 1005, 1058, 1058, 1018, 1029, 1036, 1055, 1040, 1006, 1019, 1011, 1011, 1017, 1012, 992, 1027, 982, 984, 1027, 1044, 1079, 1008, 1029, 1018, 1023, 1001, 987, 1066, 1052, 983, 981, 1042, 999, 1007, 991, 1012, 1021, 1022, 1070, 1008, 1001, 1027, 995, 996, 1071, 994, 1061, 1006, 1045, 1039, 1059, 1036, 1035, 1003, 1038, 1049, 1014, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1014, 1016, 1027, 1018, 1019, 1007, 1007, 1060, 1089, 1058, 1066, 969, 969, 976, 992, 1032, 995, 1033, 1004, 1050, 1064, 984, 1019, 1057, 1032, 1008, 994, 1027, 1039, 990, 1019, 999, 1051, 1044, 1030, 1040, 1056, 1024, 1033, 1017, 1038, 1005, 1021, 1035, 1027, 1021, 1016, 1013, 1059, 1059, 1029, 1049, 1058, 1032, 1009, 1034, 990, 1000, 1027, 1034, 1036, 1034, 972, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1054, 1040, 1057, 1054, 1019, 1035, 1008, 1060, 986, 1029, 1023, 939, 1008, 990, 1049, 996, 982, 986, 1051, 1011, 1024, 1019, 1038, 1063, 1010, 991, 1067, 995, 1039, 985, 1007, 1041, 1065, 1071, 1048, 1052, 962, 1017, 1034, 1015, 1052, 1040, 1001, 1018, 1023, 995, 1062, 1022, 1034, 1050, 979, 1032, 1020, 1008, 970, 1009, 1094, 994, 1081, 1051, 1030, 1061, 989, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1049, 997, 1064, 1012, 1017, 1034, 1011, 1064, 1057, 1015, 1008, 1021, 954, 1058, 1001, 1070, 1005, 1014, 1057, 1056, 986, 990, 1000, 1059, 1046, 1063, 1066, 1035, 1038, 1106, 1024, 1007, 1009, 1058, 1033, 1053, 1022, 1012, 1036, 1020, 1009, 953, 1001, 1038, 1054, 988, 1042, 1031, 998, 969, 999, 1029, 1013, 997, 1030, 1009, 1017, 1051, 1003, 1045, 978, 1023, 991, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1006, 1021, 996, 1021, 1046, 1022, 987, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1028, 990, 1041, 1055, 996, 1064, 990, 6, 4, 0, 0, 0, 0, 0, 0, 1026, 1060, 1031, 1002, 1043, 991, 1030, 1055, 29, 4, 0, 0, 0, 0, 0, 0, 1005, 1017, 1025, 1002, 1077, 1005, 1009, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 984, 995, 1010, 1023, 1018, 1023, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1044, 1006, 1037, 999, 995, 994, 994, 52, 0, 0, 0, 0, 0, 0, 0, 1047, 1047, 1069, 1076, 973, 1065, 1004, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 996, 1010, 1031, 1087, 962, 1019, 1034, 27, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8188, 8271, 8143, 8096, 8146, 8342, 8203] avg=8192.0000 max=8342 max_rank=6 min=8096 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1026, 1034, 1025, 1056, 1001, 986, 1040, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1017, 994, 1033, 996, 996, 1047, 1090, 6, 4, 0, 0, 0, 0, 0, 0, 1030, 975, 989, 992, 987, 1046, 989, 994, 30, 4, 0, 0, 0, 0, 0, 0, 1072, 988, 1034, 1037, 1015, 1031, 1049, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1029, 1005, 1060, 1035, 1007, 1022, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 995, 1026, 1002, 1018, 1062, 1027, 988, 54, 0, 0, 0, 0, 0, 0, 0, 1033, 1040, 984, 1036, 1003, 1029, 1028, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1029, 986, 1062, 1035, 989, 981, 1007, 28, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8222, 8036, 8261, 8244, 8204, 8210, 8166] avg=8192.0000 max=8261 max_rank=3 min=8036 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1040, 1034, 1003, 1032, 1025, 1008, 1066, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1035, 1045, 1024, 1042, 971, 1047, 1010, 6, 4, 0, 0, 0, 0, 0, 0, 1015, 1019, 1010, 1013, 997, 1038, 996, 1035, 30, 4, 0, 0, 0, 0, 0, 0, 1050, 1047, 1026, 985, 991, 1034, 972, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1020, 989, 1049, 978, 1061, 1056, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1064, 1003, 1030, 1013, 996, 1027, 1011, 53, 0, 0, 0, 0, 0, 0, 0, 1014, 1026, 1047, 1026, 994, 1054, 1026, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1025, 999, 1007, 995, 1034, 1005, 1004, 28, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8255, 8157, 8101, 8188, 8262, 8219, 8118] avg=8192.0000 max=8262 max_rank=5 min=8101 min_rank=3 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[997, 1083, 956, 1046, 1021, 1037, 990, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1045, 1010, 1028, 976, 1039, 1045, 1019, 6, 4, 0, 0, 0, 0, 0, 0, 1089, 1031, 1045, 1034, 1024, 1003, 1024, 1060, 31, 4, 0, 0, 0, 0, 0, 0, 1043, 1045, 1045, 1035, 1007, 990, 1023, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1040, 1069, 993, 992, 1063, 1025, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1030, 987, 980, 943, 999, 999, 1021, 50, 0, 0, 0, 0, 0, 0, 0, 1027, 1038, 1022, 1032, 1029, 1006, 1033, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1002, 1029, 971, 990, 1045, 1019, 980, 28, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8208, 8345, 8204, 8235, 8054, 8215, 8108] avg=8192.0000 max=8345 max_rank=2 min=8054 min_rank=5 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32612 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1054, 1040, 1000, 1054, 1019, 1035, 1008, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1029, 1023, 939, 1008, 990, 1049, 996, 6, 4, 0, 0, 0, 0, 0, 0, 982, 986, 1051, 1011, 1024, 1019, 1038, 1063, 30, 4, 0, 0, 0, 0, 0, 0, 1010, 991, 1061, 995, 1039, 985, 1007, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1036, 1048, 1052, 962, 1017, 1034, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1040, 1001, 1018, 1023, 995, 1062, 1022, 53, 0, 0, 0, 0, 0, 0, 0, 1034, 1050, 979, 1032, 989, 1008, 970, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 994, 1081, 1051, 1030, 1061, 989, 1001, 27, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8030, 8208, 8129, 8229, 8266, 8071, 8333] avg=8192.0000 max=8333 max_rank=7 min=8030 min_rank=1 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1065, 1022, 991, 1062, 997, 1027, 1005, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1018, 1029, 1036, 1055, 1040, 1006, 1019, 6, 4, 0, 0, 0, 0, 0, 0, 1011, 1011, 1017, 1012, 992, 1027, 982, 984, 30, 4, 0, 0, 0, 0, 0, 0, 1027, 1044, 1073, 1008, 1029, 1018, 1023, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1031, 1052, 983, 981, 1042, 999, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1012, 1021, 1022, 1070, 1008, 1001, 1027, 52, 0, 0, 0, 0, 0, 0, 0, 995, 996, 1071, 994, 1029, 1006, 1045, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1036, 1035, 1003, 1038, 1049, 1014, 1017, 28, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8271, 8070, 8223, 8082, 8204, 8175, 8284] avg=8192.0000 max=8284 max_rank=7 min=8070 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1014, 1016, 972, 1018, 1019, 1007, 1007, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 1058, 1066, 969, 969, 976, 992, 1032, 6, 4, 0, 0, 0, 0, 0, 0, 995, 1033, 1004, 1050, 1064, 984, 1019, 1057, 30, 4, 0, 0, 0, 0, 0, 0, 1032, 1008, 988, 1027, 1039, 990, 1019, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1009, 1030, 1040, 1056, 1024, 1033, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1005, 1021, 1035, 1027, 1021, 1016, 1013, 51, 0, 0, 0, 0, 0, 0, 0, 1059, 1059, 1029, 1049, 1026, 1032, 1009, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1000, 1027, 1034, 1036, 1034, 972, 1010, 28, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8161, 8240, 8102, 8260, 8227, 8297, 8136] avg=8192.0000 max=8297 max_rank=6 min=8102 min_rank=3 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1049, 997, 999, 1012, 1017, 1034, 1011, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1015, 1008, 1021, 954, 1058, 1001, 1070, 12, 5, 0, 0, 0, 0, 0, 0, 1005, 1014, 1057, 1056, 986, 990, 1000, 1059, 34, 8, 0, 0, 0, 0, 0, 0, 1046, 1063, 1054, 1035, 1038, 1106, 1024, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1017, 1033, 1053, 1022, 1012, 1036, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 953, 1001, 1038, 1054, 988, 1042, 1031, 57, 0, 0, 0, 0, 0, 0, 0, 998, 969, 999, 1029, 976, 997, 1030, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1051, 1003, 1045, 978, 1023, 991, 1041, 32, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8201, 8209, 8373, 8202, 8173, 8007, 8188] avg=8192.0000 max=8373 max_rank=3 min=8007 min_rank=6 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8064, 8192, 8192, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8192, 8192, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8448, 8320, 8192, 8192, 8064, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8064, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8064, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[987, 1051, 1000, 1077, 1038, 1078, 1044, 1031, 991, 1056, 1017, 1019, 1007, 1042, 1021, 1060, 1028, 1019, 989, 1030, 1025, 1030, 1029, 1056, 1007, 996, 1038, 1027, 1004, 1034, 1004, 1044, 1019, 1018, 980, 1017, 1029, 987, 1064, 1009, 1056, 1059, 997, 1022, 994, 1041, 976, 1021, 1003, 1078, 1009, 1005, 1059, 1013, 987, 1038, 1037, 1024, 989, 992, 1057, 1060, 999, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1029, 1057, 1051, 1069, 1028, 1004, 1056, 1070, 990, 1039, 1052, 1020, 940, 1053, 1046, 957, 1054, 1069, 967, 1048, 1082, 1032, 1001, 1007, 1055, 1061, 1043, 1064, 1045, 977, 1037, 1004, 1040, 1025, 1007, 1070, 1079, 1027, 1007, 984, 1038, 1012, 1001, 1009, 984, 1006, 969, 1015, 998, 1025, 1009, 993, 977, 1013, 1043, 1034, 1013, 1008, 992, 1038, 1026, 1083, 956, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1042, 986, 1062, 1034, 980, 1038, 1022, 1003, 1021, 985, 984, 981, 968, 1040, 965, 998, 994, 1037, 1061, 1036, 1037, 1013, 1004, 1051, 1055, 1068, 986, 1088, 1036, 1012, 1040, 971, 1046, 1023, 1006, 1025, 1019, 1090, 989, 973, 1021, 998, 1035, 1016, 1040, 1042, 1069, 1035, 1051, 1056, 1005, 998, 1058, 1030, 1051, 992, 1079, 1048, 1034, 970, 1011, 1050, 1021, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1008, 986, 1055, 1000, 992, 999, 989, 1028, 1035, 1015, 1043, 1008, 1069, 1033, 1003, 1079, 1079, 960, 1006, 1019, 1075, 1018, 988, 1013, 1000, 1052, 1002, 1020, 1019, 1043, 1000, 1036, 1085, 976, 1023, 1069, 1049, 1038, 1057, 1002, 994, 1009, 1065, 1001, 1019, 1042, 1061, 1011, 1037, 1013, 1011, 1039, 1036, 1067, 1007, 1014, 994, 1017, 1004, 1068, 1049, 1003, 999, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1028, 1072, 1032, 1016, 1008, 959, 1018, 1062, 1028, 1011, 1047, 1004, 1003, 992, 1030, 1035, 1064, 1011, 988, 992, 998, 1021, 1055, 1036, 1014, 1056, 1014, 978, 966, 1043, 1004, 995, 1086, 1024, 1062, 1036, 1017, 991, 1044, 1083, 1026, 1003, 1061, 981, 1033, 970, 997, 1046, 1025, 1004, 1035, 1053, 1024, 1045, 1071, 1054, 959, 1021, 1067, 1023, 1024, 1032, 997, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1011, 1047, 1033, 1058, 1043, 971, 1004, 1013, 1035, 1048, 1072, 1029, 1038, 1041, 1034, 1033, 1046, 1032, 1035, 1019, 994, 1003, 967, 1020, 1028, 1034, 1041, 1049, 1007, 1064, 988, 1019, 1047, 1018, 1033, 1101, 1007, 1021, 1034, 1017, 1010, 1031, 991, 1011, 1063, 1029, 1059, 974, 994, 1049, 1048, 985, 1026, 1027, 946, 1035, 999, 1034, 996, 1071, 1005, 1018, 1041, 960] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1034, 1019, 1042, 1045, 1024, 996, 992, 1069, 1020, 1010, 1019, 975, 1042, 1067, 1059, 1060, 1030, 1041, 1017, 1012, 1026, 1006, 1004, 1017, 1027, 971, 1024, 1047, 1013, 1043, 1053, 1022, 1083, 999, 1037, 1024, 1025, 998, 1000, 1007, 1019, 1005, 1023, 1049, 975, 1001, 983, 1028, 1041, 1014, 1033, 1034, 1051, 1074, 1032, 1037, 961, 1070, 1024, 1039, 1036, 1012, 1015, 981] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1012, 1029, 1057, 1005, 1104, 1020, 1023, 1040, 1026, 1052, 996, 1035, 1058, 994, 1045, 986, 1007, 1008, 1073, 1027, 1006, 1035, 1019, 1001, 1042, 1001, 1006, 1068, 1017, 1041, 1083, 1028, 1019, 1012, 1008, 1018, 1048, 1011, 1021, 990, 1054, 1002, 1023, 1002, 1074, 1000, 1010, 1000, 987, 1034, 1007, 1032, 998, 1042, 1024, 1053, 972, 1008, 988, 987, 1009, 1041, 1032, 1086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[987, 1051, 971, 1077, 1038, 1078, 1044, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1056, 1017, 1019, 1007, 1042, 1021, 1060, 6, 3, 0, 0, 0, 0, 0, 0, 1028, 1019, 989, 1030, 1025, 1030, 1029, 1056, 8, 0, 0, 0, 0, 0, 0, 0, 1007, 996, 1038, 1013, 1004, 1034, 1004, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1018, 980, 1017, 1029, 987, 1064, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1059, 997, 1022, 994, 1041, 976, 1021, 29, 12, 0, 0, 0, 0, 0, 0, 1003, 1078, 1009, 1005, 1059, 998, 987, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1024, 989, 992, 1057, 1060, 999, 1018, 14, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8222, 8214, 8140, 8097, 8207, 8177, 8202] avg=8192.0000 max=8277 max_rank=0 min=8097 min_rank=4 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1029, 1057, 1021, 1069, 1028, 1004, 1056, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1039, 1052, 1020, 940, 1053, 1046, 957, 6, 3, 0, 0, 0, 0, 0, 0, 1054, 1069, 967, 1048, 1082, 1032, 1001, 1007, 8, 0, 0, 0, 0, 0, 0, 0, 1055, 1061, 1043, 1050, 1045, 977, 1037, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1025, 1007, 1070, 1079, 1027, 1007, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1012, 1001, 1009, 984, 1006, 969, 1015, 30, 12, 0, 0, 0, 0, 0, 0, 998, 1025, 1009, 993, 977, 998, 1043, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1008, 992, 1038, 1026, 1083, 956, 1048, 14, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8334, 8106, 8268, 8272, 8213, 8076, 8077, 8190] avg=8192.0000 max=8334 max_rank=0 min=8076 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1042, 986, 1031, 1034, 980, 1038, 1022, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 985, 984, 981, 968, 1040, 965, 998, 6, 3, 0, 0, 0, 0, 0, 0, 994, 1037, 1061, 1036, 1037, 1013, 1004, 1051, 9, 0, 0, 0, 0, 0, 0, 0, 1055, 1068, 986, 1073, 1036, 1012, 1040, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1023, 1006, 1025, 1019, 1090, 989, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 998, 1035, 1016, 1040, 1042, 1069, 1035, 31, 12, 0, 0, 0, 0, 0, 0, 1051, 1056, 1005, 998, 1058, 1015, 1051, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1048, 1034, 970, 1011, 1050, 1021, 1057, 14, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 7951, 8242, 8241, 8145, 8299, 8226, 8296] avg=8192.0000 max=8299 max_rank=5 min=7951 min_rank=1 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32966 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1008, 986, 1025, 1000, 992, 999, 989, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1015, 1043, 1008, 1069, 1033, 1003, 1079, 5, 3, 0, 0, 0, 0, 0, 0, 1079, 960, 1006, 1019, 1075, 1018, 988, 1013, 8, 0, 0, 0, 0, 0, 0, 0, 1000, 1052, 1002, 1007, 1019, 1043, 1000, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 976, 1023, 1069, 1049, 1038, 1057, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1009, 1065, 1001, 1019, 1042, 1061, 1011, 30, 13, 0, 0, 0, 0, 0, 0, 1037, 1013, 1011, 1039, 1036, 1052, 1007, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1017, 1004, 1068, 1049, 1003, 999, 1005, 14, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8027, 8293, 8166, 8159, 8272, 8245, 8209, 8165] avg=8192.0000 max=8293 max_rank=1 min=8027 min_rank=0 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32891 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1028, 1072, 1002, 1016, 1008, 959, 1018, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1011, 1047, 1004, 1003, 992, 1030, 1035, 5, 3, 0, 0, 0, 0, 0, 0, 1064, 1011, 988, 992, 998, 1021, 1055, 1036, 8, 0, 0, 0, 0, 0, 0, 0, 1014, 1056, 1014, 965, 966, 1043, 1004, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1024, 1062, 1036, 1017, 991, 1044, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1003, 1061, 981, 1033, 970, 997, 1046, 30, 13, 0, 0, 0, 0, 0, 0, 1025, 1004, 1035, 1053, 1024, 1030, 1071, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 959, 1021, 1067, 1023, 1024, 1032, 997, 1062, 14, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8158, 8173, 8057, 8316, 8160, 8296, 8211] avg=8192.0000 max=8316 max_rank=4 min=8057 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32983 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1011, 1047, 1003, 1058, 1043, 971, 1004, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1048, 1072, 1029, 1038, 1041, 1034, 1033, 6, 3, 0, 0, 0, 0, 0, 0, 1046, 1032, 1035, 1019, 994, 1003, 967, 1020, 8, 0, 0, 0, 0, 0, 0, 0, 1028, 1034, 1041, 1035, 1007, 1064, 988, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1018, 1033, 1101, 1007, 1021, 1034, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1031, 991, 1011, 1063, 1029, 1059, 974, 30, 12, 0, 0, 0, 0, 0, 0, 994, 1049, 1048, 985, 1026, 1012, 946, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1034, 996, 1071, 1005, 1018, 1041, 960, 14, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8339, 8124, 8216, 8252, 8210, 8095, 8150] avg=8192.0000 max=8339 max_rank=1 min=8095 min_rank=6 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1034, 1019, 1012, 1045, 1024, 996, 992, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1010, 1019, 975, 1042, 1067, 1059, 1060, 6, 3, 0, 0, 0, 0, 0, 0, 1030, 1041, 1017, 1012, 1026, 1006, 1004, 1017, 8, 0, 0, 0, 0, 0, 0, 0, 1027, 971, 1024, 1033, 1013, 1043, 1053, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 999, 1037, 1024, 1025, 998, 1000, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1005, 1023, 1049, 975, 1001, 983, 1028, 30, 12, 0, 0, 0, 0, 0, 0, 1041, 1014, 1033, 1034, 1051, 1058, 1032, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 961, 1070, 1024, 1039, 1036, 1012, 1015, 981, 14, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8261, 8161, 8186, 8147, 8125, 8300, 8165] avg=8192.0000 max=8300 max_rank=6 min=8125 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1012, 1029, 1023, 1005, 1104, 1020, 1023, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1052, 996, 1035, 1058, 994, 1045, 986, 9, 5, 0, 0, 0, 0, 0, 0, 1007, 1008, 1073, 1027, 1006, 1035, 1019, 1001, 12, 0, 0, 0, 0, 0, 0, 0, 1042, 1001, 1006, 1047, 1017, 1041, 1083, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1012, 1008, 1018, 1048, 1011, 1021, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1002, 1023, 1002, 1074, 1000, 1010, 1000, 34, 15, 0, 0, 0, 0, 0, 0, 987, 1034, 1007, 1032, 998, 1021, 1024, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1008, 988, 987, 1009, 1041, 1032, 1086, 18, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8206, 8188, 8265, 8094, 8214, 8156, 8157] avg=8192.0000 max=8265 max_rank=3 min=8094 min_rank=4 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32621 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8192, 8320, 8320, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8448, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1028, 1039, 970, 1107, 1014, 1029, 1047, 1029, 1001, 1005, 1017, 930, 997, 981, 1041, 1049, 1030, 1017, 988, 1010, 1041, 974, 1050, 1077, 978, 1077, 1053, 996, 1025, 1002, 1001, 1032, 996, 1018, 1034, 1014, 1045, 1016, 1041, 1067, 978, 1031, 1027, 1073, 1008, 1073, 989, 992, 1005, 1018, 1043, 1038, 1068, 1053, 1019, 1015, 1021, 1032, 1031, 1022, 1027, 1020, 1038, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1008, 1001, 1007, 1059, 1026, 1054, 1022, 1058, 1087, 1045, 988, 1028, 995, 1007, 1024, 1021, 1015, 1043, 1059, 1035, 1018, 1009, 1047, 1009, 1011, 1052, 959, 1037, 999, 1022, 1019, 1030, 983, 1002, 1031, 1075, 1013, 1036, 1004, 1048, 1059, 959, 1053, 977, 987, 950, 1043, 1057, 1019, 1015, 974, 1015, 1050, 1014, 1033, 1060, 1010, 1042, 1012, 1025, 1062, 1056, 1012, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1041, 1074, 1016, 1012, 1019, 1062, 1044, 1015, 1014, 1002, 1024, 1026, 1016, 1010, 1085, 994, 1039, 1011, 1018, 991, 1029, 1007, 1008, 1022, 1059, 1054, 1023, 1062, 1059, 1002, 1009, 1017, 1009, 1056, 1037, 994, 1013, 1048, 951, 1042, 1050, 1041, 1000, 1062, 1013, 1023, 1051, 1035, 1072, 1023, 1010, 992, 1009, 1002, 1034, 997, 1035, 1007, 999, 1013, 995, 1023, 1022, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[987, 1012, 1019, 1038, 1005, 1034, 994, 1076, 1016, 1025, 1009, 1013, 1033, 980, 1006, 1012, 1011, 1056, 1048, 1071, 1006, 1019, 1053, 1043, 1037, 1029, 1041, 1038, 1022, 1046, 1084, 1009, 1055, 1071, 1028, 971, 996, 1021, 1035, 994, 1009, 1074, 1060, 972, 1050, 1029, 1027, 1011, 1026, 1022, 1078, 1034, 1066, 1022, 1013, 1011, 1018, 1020, 1031, 989, 990, 997, 964, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1051, 1002, 1052, 1012, 1034, 1024, 1067, 1116, 1024, 1033, 1094, 1010, 1047, 972, 1004, 1046, 998, 977, 1063, 1042, 1042, 995, 1039, 1015, 1008, 1024, 1006, 1015, 1008, 1010, 1083, 1001, 1009, 1064, 1032, 1012, 1060, 1015, 1014, 1028, 1036, 1001, 960, 1019, 962, 1049, 1007, 1016, 1047, 1012, 1016, 1019, 954, 976, 994, 1073, 1024, 1081, 1042, 1032, 1058, 978, 992, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1013, 994, 1012, 991, 1023, 1028, 1029, 954, 991, 1037, 1013, 1037, 990, 1045, 1030, 1050, 1026, 991, 1015, 1016, 1047, 988, 980, 1009, 1008, 1058, 1015, 1066, 1052, 1013, 1003, 1038, 1032, 1040, 1043, 1047, 1000, 1023, 1059, 1015, 1028, 1007, 986, 1072, 995, 1045, 1041, 1043, 998, 1038, 1086, 1061, 1070, 1030, 1039, 970, 995, 1058, 1034, 1021, 1049, 973, 1030, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1033, 950, 1009, 1031, 1013, 993, 1009, 1019, 1018, 1029, 1022, 987, 990, 1056, 1030, 1012, 987, 1000, 1057, 1052, 1043, 1003, 1043, 1034, 1031, 1013, 1032, 1023, 988, 1032, 999, 1021, 995, 1030, 990, 1049, 1034, 1082, 1029, 1043, 1003, 1040, 1002, 994, 1033, 1034, 1059, 1026, 1054, 1019, 1044, 976, 1040, 1024, 1030, 1073, 1042, 1045, 1064, 988, 1096, 984, 1037, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1033, 1040, 1040, 1019, 1045, 1051, 1002, 1002, 980, 1095, 1020, 996, 980, 1011, 1073, 1040, 1055, 994, 1039, 985, 1001, 1033, 1079, 1060, 978, 1020, 1068, 1050, 994, 1045, 1037, 1013, 1035, 1041, 1016, 1037, 1051, 999, 1044, 1066, 1013, 1030, 1027, 961, 1037, 1001, 1045, 1051, 991, 1036, 963, 994, 1050, 1040, 994, 1040, 986, 1044, 1011, 1002, 1029, 1011, 1018, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1008, 1001, 1007, 1059, 1026, 1042, 1022, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 1045, 988, 1028, 995, 1007, 1024, 1021, 29, 6, 0, 0, 0, 0, 0, 0, 1015, 1043, 1059, 1035, 1018, 1009, 1044, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1040, 959, 1037, 999, 1022, 1019, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 983, 973, 1031, 1075, 1013, 1036, 1004, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 959, 1053, 977, 987, 950, 1043, 1057, 12, 6, 0, 0, 0, 0, 0, 0, 1019, 1015, 974, 1015, 1042, 1014, 1033, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1042, 1012, 1025, 1062, 1056, 1012, 1066, 8, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8230, 8232, 8117, 8163, 8103, 8172, 8296] avg=8192.0000 max=8296 max_rank=7 min=8103 min_rank=5 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32734 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1051, 1002, 1052, 1012, 1034, 1012, 1067, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1033, 1094, 1010, 1047, 972, 1004, 1046, 30, 6, 0, 0, 0, 0, 0, 0, 998, 977, 1063, 1042, 1042, 995, 1036, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1012, 1006, 1015, 1008, 1010, 1083, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1034, 1032, 1012, 1060, 1015, 1014, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1001, 960, 1019, 962, 1049, 1007, 1016, 12, 6, 0, 0, 0, 0, 0, 0, 1047, 1012, 1016, 1019, 947, 976, 994, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1081, 1042, 1032, 1058, 978, 992, 1040, 7, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8346, 8266, 8168, 8143, 8204, 8068, 8084, 8257] avg=8192.0000 max=8346 max_rank=0 min=8068 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32613 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1028, 1039, 970, 1107, 1014, 1017, 1047, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1005, 1017, 930, 997, 981, 1041, 1049, 29, 6, 0, 0, 0, 0, 0, 0, 1030, 1017, 988, 1010, 1041, 974, 1047, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1065, 1053, 996, 1025, 1002, 1001, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 996, 989, 1034, 1014, 1045, 1016, 1041, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1031, 1027, 1073, 1008, 1073, 989, 992, 12, 6, 0, 0, 0, 0, 0, 0, 1005, 1018, 1043, 1038, 1060, 1053, 1019, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1032, 1031, 1022, 1027, 1020, 1038, 1049, 8, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8056, 8184, 8152, 8202, 8189, 8251, 8251] avg=8192.0000 max=8251 max_rank=0 min=8056 min_rank=1 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32893 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1013, 994, 1012, 991, 1023, 1016, 1029, 954, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1037, 1013, 1037, 990, 1045, 1030, 1050, 30, 6, 0, 0, 0, 0, 0, 0, 1026, 991, 1015, 1016, 1047, 988, 977, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1046, 1015, 1066, 1052, 1013, 1003, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1010, 1043, 1047, 1000, 1023, 1059, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1007, 986, 1072, 995, 1045, 1041, 1043, 12, 6, 0, 0, 0, 0, 0, 0, 998, 1038, 1086, 1061, 1062, 1030, 1039, 970, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1058, 1034, 1021, 1049, 973, 1030, 1046, 8, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8032, 8229, 8069, 8241, 8229, 8235, 8284, 8217] avg=8192.0000 max=8284 max_rank=6 min=8032 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32965 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[987, 1012, 1019, 1038, 1005, 1022, 994, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1025, 1009, 1013, 1033, 980, 1006, 1012, 31, 6, 0, 0, 0, 0, 0, 0, 1011, 1056, 1048, 1071, 1006, 1019, 1050, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1017, 1041, 1038, 1022, 1046, 1084, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1040, 1028, 971, 996, 1021, 1035, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1074, 1060, 972, 1050, 1029, 1027, 1011, 12, 6, 0, 0, 0, 0, 0, 0, 1026, 1022, 1078, 1034, 1058, 1022, 1013, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1020, 1031, 989, 990, 997, 964, 980, 8, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8131, 8304, 8294, 8140, 8250, 8264, 8000] avg=8192.0000 max=8304 max_rank=2 min=8000 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32654 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1041, 1074, 1016, 1012, 1019, 1050, 1044, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1002, 1024, 1026, 1016, 1010, 1085, 994, 30, 6, 0, 0, 0, 0, 0, 0, 1039, 1011, 1018, 991, 1029, 1007, 1005, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1042, 1023, 1062, 1059, 1002, 1009, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1026, 1037, 994, 1013, 1048, 951, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1041, 1000, 1062, 1013, 1023, 1051, 1035, 12, 6, 0, 0, 0, 0, 0, 0, 1072, 1023, 1010, 992, 1002, 1002, 1034, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1007, 999, 1013, 995, 1023, 1022, 1014, 7, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8207, 8122, 8273, 8120, 8293, 8132, 8118] avg=8192.0000 max=8293 max_rank=5 min=8118 min_rank=7 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32663 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1033, 950, 1009, 1031, 1013, 981, 1009, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1029, 1022, 987, 990, 1056, 1030, 1012, 29, 6, 0, 0, 0, 0, 0, 0, 987, 1000, 1057, 1052, 1043, 1003, 1040, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1001, 1032, 1023, 988, 1032, 999, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1001, 990, 1049, 1034, 1082, 1029, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1040, 1002, 994, 1033, 1034, 1059, 1026, 12, 6, 0, 0, 0, 0, 0, 0, 1054, 1019, 1044, 976, 1032, 1024, 1030, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1045, 1064, 988, 1096, 984, 1037, 1018, 8, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8045, 8179, 8216, 8127, 8223, 8209, 8252, 8285] avg=8192.0000 max=8285 max_rank=7 min=8045 min_rank=0 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32969 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1033, 1040, 1040, 1019, 1045, 1034, 1002, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1095, 1020, 996, 980, 1011, 1073, 1040, 34, 9, 0, 0, 0, 0, 0, 0, 1055, 994, 1039, 985, 1001, 1033, 1074, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1004, 1068, 1050, 994, 1045, 1037, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1007, 1016, 1037, 1051, 999, 1044, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1030, 1027, 961, 1037, 1001, 1045, 1051, 16, 8, 0, 0, 0, 0, 0, 0, 991, 1036, 963, 994, 1039, 1040, 994, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1044, 1011, 1002, 1029, 1011, 1018, 995, 11, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8238, 8241, 8189, 8255, 8189, 8097, 8112] avg=8192.0000 max=8255 max_rank=4 min=8097 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32653 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 8064, 8064, 8320, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8448, 8320, 8192, 8320, 8064, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[996, 996, 996, 1013, 1023, 984, 1010, 1039, 1008, 1032, 1011, 1028, 1043, 1012, 1009, 1047, 1057, 1041, 1005, 1059, 1053, 1059, 1007, 996, 1041, 1021, 1031, 1004, 1033, 998, 1009, 1006, 1065, 1028, 1039, 980, 1035, 990, 1025, 1029, 1050, 1035, 996, 1027, 1018, 983, 1026, 994, 1052, 1052, 1004, 1014, 1012, 1047, 1064, 1068, 1021, 999, 1055, 1001, 1046, 1043, 1067, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[975, 994, 1008, 1032, 1015, 974, 1009, 992, 1043, 1001, 1075, 1065, 1050, 1002, 1025, 1078, 1044, 1045, 1005, 1053, 1051, 1037, 1063, 1031, 1052, 1022, 1017, 1009, 1028, 1000, 1004, 1034, 1063, 1029, 958, 975, 1014, 1022, 1021, 993, 1041, 1049, 1027, 1040, 1006, 1021, 1038, 1030, 1049, 1035, 1022, 983, 991, 1045, 1024, 962, 1006, 1035, 1009, 1042, 1038, 1032, 1041, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1017, 994, 1021, 1051, 1061, 1027, 1029, 998, 994, 1041, 1053, 1000, 1039, 1059, 1018, 991, 1028, 1059, 1033, 1029, 1021, 1052, 1079, 1012, 1004, 997, 988, 1053, 984, 1003, 1014, 998, 1025, 1033, 1041, 1004, 1018, 1002, 1002, 1010, 981, 1025, 1017, 1003, 980, 1026, 1006, 1062, 1040, 1049, 1006, 989, 1011, 1051, 1051, 1013, 1027, 1019, 1119, 1041, 1070, 1024, 1033, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1065, 1044, 1060, 1037, 1059, 1045, 1010, 983, 1006, 973, 1018, 1048, 1034, 1008, 1003, 1006, 1013, 1062, 1038, 990, 1029, 1063, 1050, 1011, 1020, 1039, 1005, 1053, 1052, 1020, 1053, 1095, 988, 1032, 1034, 1020, 996, 1022, 988, 1001, 994, 1016, 998, 1025, 1022, 990, 994, 1034, 1047, 990, 1015, 1027, 1027, 963, 1026, 1039, 1026, 1003, 1032, 1047, 1046, 1027, 1013, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1031, 1054, 1035, 1028, 1074, 1027, 1060, 1000, 1010, 1073, 1024, 1026, 1019, 1051, 1048, 1037, 982, 1068, 1032, 1033, 1023, 1082, 1010, 1011, 1009, 1061, 994, 1010, 1032, 1057, 989, 1052, 1014, 1034, 1024, 1017, 1025, 1056, 1037, 943, 1052, 1010, 1017, 1027, 1009, 1043, 1023, 994, 1017, 993, 1026, 954, 1000, 991, 1010, 1032, 975, 1072, 981, 1025, 1036, 1060, 1004, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1043, 1024, 1014, 971, 1049, 1027, 1033, 985, 964, 1079, 979, 1003, 1045, 998, 1051, 1082, 1048, 1051, 1045, 1052, 1021, 1033, 1045, 1003, 995, 1065, 1008, 1058, 989, 1027, 1012, 975, 1028, 1071, 1018, 977, 1015, 1031, 1008, 1017, 989, 1032, 1013, 1012, 1059, 1026, 1070, 1003, 1050, 980, 1034, 994, 957, 1068, 1018, 1042, 1009, 990, 1001, 1052, 1028, 1068, 1045, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[997, 1030, 1036, 1022, 987, 1002, 1075, 1035, 1020, 1003, 1038, 1046, 1030, 1028, 1005, 1019, 999, 1060, 1025, 1058, 1029, 1005, 1090, 1072, 1028, 1017, 1019, 1041, 1044, 989, 1042, 1044, 988, 1036, 988, 1007, 978, 1040, 977, 991, 992, 1039, 1087, 1048, 1016, 1013, 1003, 1013, 1021, 1017, 1049, 1029, 1036, 986, 1067, 995, 1070, 1053, 964, 1041, 984, 1042, 972, 1059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1019, 1049, 1040, 987, 1001, 987, 1054, 964, 1010, 1031, 1013, 1053, 1061, 1021, 1022, 1060, 1094, 1014, 1018, 1061, 1009, 1025, 963, 1049, 1036, 985, 1035, 1052, 990, 1063, 1032, 1069, 1037, 1064, 974, 1038, 1014, 985, 1009, 1035, 1004, 1037, 1003, 1029, 1055, 1008, 1039, 1018, 1019, 1001, 974, 1021, 1059, 989, 1021, 1010, 1069, 1051, 993, 1047, 1036, 1016, 1024, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[996, 996, 996, 1013, 1023, 984, 1010, 1039, 29, 0, 0, 0, 0, 0, 0, 0, 1008, 1032, 1011, 1028, 1016, 1012, 1009, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 949, 1005, 1059, 1053, 1059, 1007, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1021, 1031, 1004, 1033, 998, 1009, 1006, 3, 0, 0, 0, 0, 0, 0, 0, 1065, 1028, 1039, 980, 1035, 990, 1025, 1029, 71, 0, 0, 0, 0, 0, 0, 0, 1050, 1035, 996, 1027, 1018, 983, 1026, 994, 24, 0, 0, 0, 0, 0, 0, 0, 1052, 1052, 1004, 1014, 1012, 1047, 1064, 1068, 21, 17, 0, 0, 0, 0, 0, 0, 1021, 999, 1055, 1001, 1046, 997, 1067, 1004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 8163, 8185, 8146, 8262, 8153, 8351, 8190] avg=8192.0000 max=8351 max_rank=6 min=8086 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32956 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1031, 1054, 1035, 1028, 1074, 1027, 1060, 1000, 29, 0, 0, 0, 0, 0, 0, 0, 1010, 1073, 1024, 1026, 992, 1051, 1048, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 982, 974, 1032, 1033, 1023, 1082, 1010, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1061, 994, 1010, 1032, 1057, 989, 1052, 3, 0, 0, 0, 0, 0, 0, 0, 1014, 1034, 1024, 1017, 1025, 1056, 1037, 943, 73, 0, 0, 0, 0, 0, 0, 0, 1052, 1010, 1017, 1027, 1009, 1043, 1023, 994, 24, 0, 0, 0, 0, 0, 0, 0, 1017, 993, 1026, 954, 1000, 991, 1010, 1032, 21, 17, 0, 0, 0, 0, 0, 0, 975, 1072, 981, 1025, 1036, 1014, 1004, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8338, 8261, 8147, 8207, 8223, 8199, 8061, 8100] avg=8192.0000 max=8338 max_rank=0 min=8061 min_rank=6 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32583 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[997, 1030, 1036, 1022, 987, 1002, 1075, 1035, 29, 0, 0, 0, 0, 0, 0, 0, 1020, 1003, 1038, 1046, 1003, 1028, 1005, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 999, 967, 1025, 1058, 1029, 1005, 1090, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1017, 1019, 1041, 1044, 989, 1042, 1044, 3, 0, 0, 0, 0, 0, 0, 0, 988, 1036, 988, 1007, 978, 1040, 977, 991, 72, 0, 0, 0, 0, 0, 0, 0, 992, 1039, 1087, 1048, 1016, 1013, 1003, 1013, 24, 0, 0, 0, 0, 0, 0, 0, 1021, 1017, 1049, 1029, 1036, 986, 1067, 995, 21, 17, 0, 0, 0, 0, 0, 0, 1070, 1053, 964, 1041, 984, 996, 972, 1059, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8162, 8245, 8227, 8077, 8235, 8238, 8139] avg=8192.0000 max=8245 max_rank=2 min=8077 min_rank=4 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[975, 994, 1008, 1032, 1015, 974, 1009, 992, 29, 0, 0, 0, 0, 0, 0, 0, 1043, 1001, 1075, 1065, 1022, 1002, 1025, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 953, 1005, 1053, 1051, 1037, 1063, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1022, 1017, 1009, 1028, 1000, 1004, 1034, 3, 0, 0, 0, 0, 0, 0, 0, 1063, 1029, 958, 975, 1014, 1022, 1021, 993, 71, 0, 0, 0, 0, 0, 0, 0, 1041, 1049, 1027, 1040, 1006, 1021, 1038, 1030, 25, 0, 0, 0, 0, 0, 0, 0, 1049, 1035, 1022, 983, 991, 1045, 1024, 962, 21, 17, 0, 0, 0, 0, 0, 0, 1006, 1035, 1009, 1042, 1038, 986, 1041, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8028, 8311, 8237, 8169, 8146, 8277, 8149, 8219] avg=8192.0000 max=8311 max_rank=1 min=8028 min_rank=0 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1065, 1044, 1060, 1037, 1059, 1045, 1010, 983, 29, 0, 0, 0, 0, 0, 0, 0, 1006, 973, 1018, 1048, 1007, 1008, 1003, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 968, 1038, 990, 1029, 1063, 1050, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1039, 1005, 1053, 1052, 1020, 1053, 1095, 3, 0, 0, 0, 0, 0, 0, 0, 988, 1032, 1034, 1020, 996, 1022, 988, 1001, 73, 0, 0, 0, 0, 0, 0, 0, 994, 1016, 998, 1025, 1022, 990, 994, 1034, 24, 0, 0, 0, 0, 0, 0, 0, 1047, 990, 1015, 1027, 1027, 963, 1026, 1039, 21, 16, 0, 0, 0, 0, 0, 0, 1026, 1003, 1032, 1047, 1046, 982, 1013, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8332, 8069, 8162, 8340, 8154, 8097, 8171, 8211] avg=8192.0000 max=8340 max_rank=3 min=8069 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32633 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1017, 994, 1021, 1051, 1061, 1027, 1029, 998, 28, 0, 0, 0, 0, 0, 0, 0, 994, 1041, 1053, 1000, 1012, 1059, 1018, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 966, 1033, 1029, 1021, 1052, 1079, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 997, 988, 1053, 984, 1003, 1014, 998, 3, 0, 0, 0, 0, 0, 0, 0, 1025, 1033, 1041, 1004, 1018, 1002, 1002, 1010, 72, 0, 0, 0, 0, 0, 0, 0, 981, 1025, 1017, 1003, 980, 1026, 1006, 1062, 24, 0, 0, 0, 0, 0, 0, 0, 1040, 1049, 1006, 989, 1011, 1051, 1051, 1013, 21, 16, 0, 0, 0, 0, 0, 0, 1027, 1019, 1119, 1041, 1070, 980, 1033, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8168, 8220, 8044, 8207, 8124, 8247, 8300] avg=8192.0000 max=8300 max_rank=7 min=8044 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1043, 1024, 1014, 971, 1049, 1027, 1033, 985, 30, 0, 0, 0, 0, 0, 0, 0, 964, 1079, 979, 1003, 1018, 998, 1051, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 958, 1045, 1052, 1021, 1033, 1045, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1065, 1008, 1058, 989, 1027, 1012, 975, 3, 0, 0, 0, 0, 0, 0, 0, 1028, 1071, 1018, 977, 1015, 1031, 1008, 1017, 72, 0, 0, 0, 0, 0, 0, 0, 989, 1032, 1013, 1012, 1059, 1026, 1070, 1003, 24, 0, 0, 0, 0, 0, 0, 0, 1050, 980, 1034, 994, 957, 1068, 1018, 1042, 21, 17, 0, 0, 0, 0, 0, 0, 1009, 990, 1001, 1052, 1028, 1021, 1045, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8174, 8205, 8132, 8237, 8228, 8181, 8203] avg=8192.0000 max=8237 max_rank=4 min=8132 min_rank=3 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1019, 1049, 1040, 987, 1001, 987, 1054, 964, 32, 4, 0, 0, 0, 0, 0, 0, 1010, 1031, 1013, 1053, 1018, 1021, 1022, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 916, 1018, 1061, 1009, 1025, 963, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 985, 1035, 1052, 990, 1063, 1032, 1069, 9, 0, 0, 0, 0, 0, 0, 0, 1037, 1064, 974, 1038, 1014, 985, 1009, 1035, 74, 0, 0, 0, 0, 0, 0, 0, 1004, 1037, 1003, 1029, 1055, 1008, 1039, 1018, 30, 0, 0, 0, 0, 0, 0, 0, 1019, 1001, 974, 1021, 1059, 989, 1021, 1010, 24, 20, 0, 0, 0, 0, 0, 0, 1069, 1051, 993, 1047, 1036, 964, 1024, 990, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8228, 8135, 8271, 8230, 8223, 8138, 8174] avg=8192.0000 max=8271 max_rank=3 min=8135 min_rank=2 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8448, 8192, 8064, 8192, 8192, 8192, 8064, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 7680, 8320, 8448, 8320, 8448, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8064, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8064, 8192, 8192, 8320, 8192, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1019, 1026, 1040, 971, 972, 1003, 1051, 1055, 1028, 993, 1021, 1069, 1061, 1044, 981, 1051, 1020, 1011, 995, 1066, 1098, 1032, 1006, 1056, 1034, 1012, 982, 1006, 1015, 1046, 1015, 1025, 983, 1002, 995, 1028, 1038, 1029, 990, 1044, 1013, 1052, 981, 979, 1041, 1066, 997, 1073, 1060, 994, 1052, 1030, 1051, 1056, 1049, 1050, 1008, 1008, 1024, 994, 993, 1019, 1020, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1061, 1028, 978, 1011, 1045, 1050, 1057, 1017, 1055, 976, 990, 1069, 1047, 1014, 1078, 1071, 992, 1042, 1011, 1016, 1008, 1069, 1038, 1078, 1039, 986, 1015, 1068, 1044, 1053, 998, 957, 1014, 996, 983, 1023, 1011, 1045, 1032, 1052, 1011, 974, 1009, 1023, 1032, 997, 993, 961, 1021, 1030, 1005, 1059, 1043, 1047, 1035, 1024, 1054, 1005, 1018, 1013, 1036, 988, 1012, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1028, 1029, 972, 986, 1116, 1005, 1027, 1009, 1057, 996, 1058, 1065, 1031, 1031, 1058, 1029, 1043, 1002, 1004, 1027, 1032, 999, 1042, 1086, 1005, 1031, 1051, 1002, 1009, 1024, 978, 1016, 1031, 941, 1007, 1007, 1048, 1016, 1041, 1019, 1008, 1050, 1046, 958, 1021, 1042, 1004, 1018, 1000, 983, 1036, 1024, 1025, 1075, 1073, 1023, 1029, 1034, 1024, 1009, 1046, 1088, 990, 972] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1043, 1067, 1055, 984, 1043, 1025, 962, 1067, 1003, 1026, 985, 1018, 1047, 1046, 1015, 1083, 982, 1043, 1012, 1052, 1047, 1029, 1041, 1000, 985, 1003, 1041, 1041, 1085, 1027, 973, 996, 1059, 1065, 1027, 1008, 1008, 992, 996, 1036, 959, 1008, 1075, 1025, 1073, 1057, 1000, 1033, 1024, 1012, 1035, 1017, 1023, 1022, 1025, 1003, 990, 1045, 987, 1019, 1001, 1011, 1030, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1020, 1010, 1007, 979, 1060, 1043, 1071, 1013, 1031, 1019, 1013, 1002, 1068, 1021, 999, 1028, 1048, 1011, 1046, 1000, 1028, 998, 1043, 1018, 1051, 1049, 1058, 1056, 1017, 987, 1040, 1024, 1009, 1050, 1044, 989, 1029, 1027, 1001, 1048, 1035, 1020, 986, 1017, 1000, 1019, 1043, 1062, 1016, 983, 1046, 1076, 1035, 988, 1028, 954, 1036, 1027, 1070, 1026, 977, 994, 1014, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[998, 1049, 1028, 1038, 1019, 1016, 964, 999, 1064, 942, 974, 1008, 972, 1024, 969, 1069, 1004, 1045, 1023, 972, 986, 1073, 996, 1037, 1035, 1055, 1034, 1057, 1089, 1039, 1055, 994, 1020, 1032, 1051, 1018, 984, 1000, 1038, 1056, 1028, 1006, 994, 1034, 1032, 1026, 983, 1021, 1022, 1009, 1049, 1035, 995, 1090, 1040, 1055, 1010, 1017, 1082, 1018, 1005, 1039, 1054, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1044, 1046, 1038, 996, 1029, 1027, 1052, 1033, 1026, 997, 997, 1032, 1048, 954, 1051, 1009, 1006, 1032, 1029, 1027, 1024, 1012, 1027, 1004, 993, 1051, 1033, 1009, 1008, 1014, 1044, 1042, 1032, 1045, 1052, 1023, 1050, 1040, 1033, 1050, 992, 993, 1014, 966, 1039, 998, 1052, 1002, 1037, 981, 1014, 1039, 1042, 1030, 1039, 1042, 1045, 1024, 942, 1030, 1012, 1070, 1054, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1040, 998, 1021, 1008, 1017, 1034, 1038, 1044, 1027, 1058, 1073, 1040, 1016, 1000, 1020, 1087, 1025, 1038, 992, 999, 1063, 1033, 1015, 1036, 972, 1012, 1023, 999, 990, 1015, 977, 998, 1052, 1000, 968, 1031, 1043, 1013, 1022, 997, 1030, 1074, 1005, 1009, 1036, 1011, 1050, 1052, 1059, 1043, 1009, 1044, 997, 1036, 1016, 1021, 1045, 1047, 1007, 996, 1044, 997, 1054, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1020, 1010, 1007, 979, 1055, 1043, 1071, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1019, 1013, 1002, 1068, 1021, 999, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1011, 1046, 1000, 1028, 998, 1043, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1049, 1058, 1056, 1017, 987, 1040, 1024, 17, 1, 0, 0, 0, 0, 0, 0, 1009, 1050, 1044, 989, 1029, 1027, 1001, 1048, 8, 6, 0, 0, 0, 0, 0, 0, 1035, 1020, 986, 1017, 1000, 1019, 1043, 1062, 38, 0, 0, 0, 0, 0, 0, 0, 1016, 983, 1046, 1076, 1035, 949, 1028, 954, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1027, 1070, 1026, 977, 994, 1014, 1029, 8, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8158, 8176, 8300, 8211, 8220, 8087, 8186] avg=8192.0000 max=8300 max_rank=3 min=8087 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32704 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1019, 1026, 1040, 971, 967, 1003, 1051, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 993, 1021, 1069, 1061, 1044, 981, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1011, 995, 1066, 1098, 1032, 1006, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1012, 982, 1006, 1015, 1046, 1015, 1025, 17, 1, 0, 0, 0, 0, 0, 0, 983, 1002, 995, 1028, 1038, 1029, 990, 1044, 8, 6, 0, 0, 0, 0, 0, 0, 1013, 1052, 981, 979, 1041, 1066, 997, 1073, 41, 0, 0, 0, 0, 0, 0, 0, 1060, 994, 1052, 1030, 1051, 1014, 1049, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1008, 1024, 994, 993, 1019, 1020, 1013, 8, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8225, 8268, 8153, 8123, 8243, 8300, 8092] avg=8192.0000 max=8300 max_rank=6 min=8092 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1061, 1028, 978, 1011, 1040, 1050, 1057, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 976, 990, 1069, 1047, 1014, 1078, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1042, 1011, 1016, 1008, 1069, 1038, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 986, 1015, 1068, 1044, 1053, 998, 957, 18, 1, 0, 0, 0, 0, 0, 0, 1014, 996, 983, 1023, 1011, 1045, 1032, 1052, 8, 7, 0, 0, 0, 0, 0, 0, 1011, 974, 1009, 1023, 1032, 997, 993, 961, 41, 0, 0, 0, 0, 0, 0, 0, 1021, 1030, 1005, 1059, 1043, 1005, 1035, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1005, 1018, 1013, 1036, 988, 1012, 1029, 8, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8275, 8238, 8179, 8171, 8041, 8222, 8168] avg=8192.0000 max=8275 max_rank=1 min=8041 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32602 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1043, 1067, 1055, 984, 1038, 1025, 962, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1026, 985, 1018, 1047, 1046, 1015, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1043, 1012, 1052, 1047, 1029, 1041, 985, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1003, 1041, 1041, 1085, 1027, 973, 996, 18, 1, 0, 0, 0, 0, 0, 0, 1059, 1065, 1027, 1008, 1008, 992, 996, 1036, 8, 7, 0, 0, 0, 0, 0, 0, 959, 1008, 1075, 1025, 1073, 1057, 1000, 1033, 40, 0, 0, 0, 0, 0, 0, 0, 1024, 1012, 1035, 1017, 1023, 981, 1025, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1045, 987, 1019, 1001, 1011, 1030, 1045, 7, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8198, 8191, 8170, 8206, 8270, 8120, 8140] avg=8192.0000 max=8270 max_rank=5 min=8120 min_rank=6 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1028, 1029, 972, 986, 1110, 1005, 1027, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 996, 1058, 1065, 1031, 1031, 1058, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1002, 1004, 1027, 1032, 999, 1042, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1031, 1051, 1002, 1009, 1024, 978, 1016, 17, 1, 0, 0, 0, 0, 0, 0, 1031, 941, 1007, 1007, 1048, 1016, 1041, 1019, 8, 6, 0, 0, 0, 0, 0, 0, 1008, 1050, 1046, 958, 1021, 1042, 1004, 1018, 42, 0, 0, 0, 0, 0, 0, 0, 1000, 983, 1036, 1024, 1025, 1032, 1073, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1034, 1024, 1009, 1046, 1088, 990, 972, 8, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8302, 8219, 8134, 8124, 8189, 8196, 8206] avg=8192.0000 max=8302 max_rank=1 min=8124 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[998, 1049, 1028, 1038, 1014, 1016, 964, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 942, 974, 1008, 972, 1024, 969, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1045, 1023, 972, 986, 1073, 996, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1055, 1034, 1057, 1089, 1039, 1055, 994, 18, 1, 0, 0, 0, 0, 0, 0, 1020, 1032, 1051, 1018, 984, 1000, 1038, 1056, 8, 7, 0, 0, 0, 0, 0, 0, 1028, 1006, 994, 1034, 1032, 1026, 983, 1021, 42, 0, 0, 0, 0, 0, 0, 0, 1022, 1009, 1049, 1035, 995, 1047, 1040, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1017, 1082, 1018, 1005, 1039, 1054, 1066, 8, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8106, 7997, 8120, 8377, 8214, 8166, 8252, 8304] avg=8192.0000 max=8377 max_rank=3 min=7997 min_rank=1 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1044, 1046, 1038, 996, 1024, 1027, 1052, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 997, 997, 1032, 1048, 954, 1051, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1032, 1029, 1027, 1024, 1012, 1027, 989, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1051, 1033, 1009, 1008, 1014, 1044, 1042, 17, 1, 0, 0, 0, 0, 0, 0, 1032, 1045, 1052, 1023, 1050, 1040, 1033, 1050, 8, 6, 0, 0, 0, 0, 0, 0, 992, 993, 1014, 966, 1039, 998, 1052, 1002, 40, 0, 0, 0, 0, 0, 0, 0, 1037, 981, 1014, 1039, 1042, 989, 1039, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1024, 942, 1030, 1012, 1070, 1054, 1020, 7, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8091, 8146, 8212, 8339, 8096, 8183, 8209] avg=8192.0000 max=8339 max_rank=4 min=8091 min_rank=1 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1040, 998, 1021, 1008, 1008, 1034, 1038, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1058, 1073, 1040, 1016, 1000, 1020, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1038, 992, 999, 1063, 1033, 1015, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1012, 1023, 999, 990, 1015, 977, 998, 20, 5, 0, 0, 0, 0, 0, 0, 1052, 1000, 968, 1031, 1043, 1013, 1022, 997, 11, 11, 0, 0, 0, 0, 0, 0, 1030, 1074, 1005, 1009, 1036, 1011, 1050, 1052, 44, 0, 0, 0, 0, 0, 0, 0, 1059, 1043, 1009, 1044, 997, 987, 1016, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1047, 1007, 996, 1044, 997, 1054, 1020, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8290, 8178, 8011, 8148, 8311, 8176, 8231] avg=8192.0000 max=8311 max_rank=5 min=8011 min_rank=3 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32866 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8064, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8064, 8320, 8320, 8192, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8448, 8320, 8064, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8320, 8192, 8064, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1038, 1056, 1011, 1008, 1010, 995, 985, 1013, 1000, 1034, 995, 1037, 992, 1013, 1014, 1048, 980, 1042, 1032, 1045, 994, 1035, 1033, 1022, 1030, 986, 1048, 1022, 1034, 1045, 1061, 975, 1009, 1051, 952, 1019, 1019, 1046, 984, 1011, 1067, 993, 998, 986, 1025, 1007, 1060, 1048, 1071, 1014, 1088, 1044, 1030, 1011, 1074, 1046, 1058, 1007, 1022, 1017, 1029, 1061, 1013, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1046, 1066, 1035, 1010, 1021, 985, 1006, 1065, 1001, 1007, 1033, 1064, 1035, 1038, 993, 1069, 1024, 1042, 1001, 943, 994, 1038, 1005, 1011, 983, 995, 1048, 1043, 1029, 1019, 1063, 999, 1051, 1048, 1059, 1054, 1054, 1042, 1053, 1015, 1009, 988, 958, 1042, 1033, 996, 980, 1015, 978, 1032, 1047, 1039, 980, 1055, 1000, 1042, 1035, 1016, 1020, 1060, 1010, 1023, 1047, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1015, 964, 1058, 1046, 972, 1044, 1012, 995, 1062, 1046, 1042, 1032, 1020, 1042, 1042, 1026, 1086, 1073, 1017, 1020, 1011, 1037, 1064, 1039, 1021, 1006, 1058, 952, 1047, 1042, 1033, 980, 1057, 1013, 1007, 1058, 1023, 1030, 1030, 1057, 1006, 1036, 1001, 1068, 1030, 1021, 1045, 986, 1010, 1045, 997, 1040, 980, 1017, 1000, 998, 1028, 1004, 1024, 1016, 989, 1020, 1000, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1090, 1035, 1025, 1080, 1001, 1025, 1006, 1026, 1002, 1003, 1047, 1031, 995, 1014, 1036, 995, 1050, 1044, 1007, 999, 1050, 991, 1021, 1068, 985, 1040, 1031, 1017, 1063, 1010, 1016, 1027, 1028, 1042, 1021, 1031, 956, 983, 994, 1000, 1002, 1033, 1025, 1003, 1014, 1045, 1078, 1026, 1029, 1008, 1031, 1014, 1061, 1021, 1019, 1032, 1056, 991, 1081, 977, 1057, 1025, 1012, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1048, 1032, 1060, 1061, 1020, 1032, 1025, 1008, 1042, 1001, 1038, 1064, 1051, 1007, 1043, 1022, 1043, 1024, 1083, 1022, 1039, 994, 974, 980, 1014, 1017, 1019, 1030, 1017, 983, 1037, 981, 1009, 995, 1007, 992, 994, 1020, 1028, 1040, 1037, 981, 1008, 985, 1066, 1045, 1043, 1084, 1027, 1031, 1039, 980, 1026, 1045, 1057, 970, 1047, 1050, 1002, 996, 1036, 1044, 1006, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1028, 1007, 1066, 1061, 985, 1034, 1060, 1065, 1059, 1048, 1008, 974, 998, 970, 1038, 1005, 1023, 1042, 994, 1042, 1039, 1038, 989, 1023, 1037, 1006, 1032, 962, 1040, 1049, 1022, 993, 1034, 1016, 1062, 1045, 1003, 1000, 997, 983, 1017, 1031, 1056, 1024, 1040, 994, 1063, 1017, 992, 1033, 1020, 1004, 1008, 1030, 1053, 1066, 1027, 1036, 1052, 1036, 1032, 1003, 1031, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1022, 1014, 986, 1017, 1024, 1072, 1034, 995, 1066, 1003, 935, 1031, 973, 986, 1082, 1012, 1001, 993, 995, 1058, 1050, 1053, 993, 1044, 996, 1009, 1006, 1100, 975, 1067, 1066, 1031, 969, 1049, 1051, 1062, 1082, 993, 985, 977, 1030, 1055, 1074, 1001, 1035, 1084, 1061, 1051, 986, 996, 1016, 1033, 997, 1028, 1057, 996, 1031, 1001, 1023, 1107, 1019, 1028, 967, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1040, 1015, 974, 1004, 1007, 1028, 1057, 1046, 1035, 1048, 1005, 984, 1013, 1001, 1020, 1030, 1002, 1031, 988, 1055, 1031, 1031, 1089, 1052, 998, 1079, 991, 1005, 1035, 1029, 1007, 1012, 1066, 1011, 1011, 1013, 995, 1030, 997, 964, 1025, 1021, 1019, 1022, 1007, 1062, 1039, 1026, 1030, 1011, 1065, 1001, 1023, 1013, 993, 1040, 1027, 1016, 1010, 1013, 1073, 1085, 1030, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1023, 1056, 1011, 1008, 1010, 995, 985, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1034, 995, 1037, 992, 1013, 1014, 1048, 10, 6, 0, 0, 0, 0, 0, 0, 980, 1030, 1032, 1045, 994, 1035, 1033, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 986, 1048, 1022, 1034, 1045, 1061, 975, 13, 9, 0, 0, 0, 0, 0, 0, 1009, 1051, 952, 1019, 1019, 1046, 984, 1011, 24, 8, 0, 0, 0, 0, 0, 0, 1067, 993, 998, 986, 1025, 1007, 1036, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1014, 1088, 1044, 1030, 1011, 1074, 1046, 2, 0, 0, 0, 0, 0, 0, 0, 1037, 1007, 1022, 1017, 1029, 1061, 1013, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8149, 8171, 8223, 8123, 8160, 8380, 8229] avg=8192.0000 max=8380 max_rank=6 min=8101 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32892 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1000, 964, 1058, 1046, 972, 1044, 1012, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1046, 1042, 1032, 1020, 1042, 1042, 1026, 10, 6, 0, 0, 0, 0, 0, 0, 1086, 1061, 1017, 1020, 1011, 1037, 1064, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1006, 1058, 952, 1047, 1042, 1033, 980, 12, 9, 0, 0, 0, 0, 0, 0, 1057, 1013, 1007, 1058, 1023, 1030, 1030, 1057, 23, 8, 0, 0, 0, 0, 0, 0, 1006, 1036, 1001, 1068, 1030, 1021, 1022, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1045, 997, 1040, 980, 1017, 1000, 998, 2, 0, 0, 0, 0, 0, 0, 0, 1008, 1004, 1024, 1016, 989, 1020, 1000, 996, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8091, 8328, 8335, 8160, 8306, 8170, 8089, 8057] avg=8192.0000 max=8335 max_rank=2 min=8057 min_rank=7 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1030, 1066, 1035, 1010, 1021, 985, 1006, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1007, 1033, 1064, 1035, 1038, 993, 1069, 10, 7, 0, 0, 0, 0, 0, 0, 1024, 1030, 1001, 943, 994, 1038, 1005, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 983, 995, 1048, 1043, 1029, 1019, 1063, 999, 12, 9, 0, 0, 0, 0, 0, 0, 1051, 1048, 1059, 1054, 1054, 1042, 1053, 1015, 22, 8, 0, 0, 0, 0, 0, 0, 1009, 988, 958, 1042, 1033, 996, 958, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1032, 1047, 1039, 980, 1055, 1000, 1042, 2, 0, 0, 0, 0, 0, 0, 0, 1015, 1016, 1020, 1060, 1010, 1023, 1047, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8257, 8046, 8200, 8406, 7999, 8175, 8235] avg=8192.0000 max=8406 max_rank=4 min=7999 min_rank=5 imbalance_ratio=1.026123 max_load_reduction=57130 max_load_reduction_pct=0.871735 moved_to_echo=32815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1073, 1035, 1025, 1080, 1001, 1025, 1006, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1003, 1047, 1031, 995, 1014, 1036, 995, 10, 7, 0, 0, 0, 0, 0, 0, 1050, 1032, 1007, 999, 1050, 991, 1021, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1040, 1031, 1017, 1063, 1010, 1016, 1027, 13, 10, 0, 0, 0, 0, 0, 0, 1028, 1042, 1021, 1031, 956, 983, 994, 1000, 24, 8, 0, 0, 0, 0, 0, 0, 1002, 1033, 1025, 1003, 1014, 1045, 1054, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1008, 1031, 1014, 1061, 1021, 1019, 1032, 2, 0, 0, 0, 0, 0, 0, 0, 1035, 991, 1081, 977, 1057, 1025, 1012, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8140, 8218, 8212, 8087, 8202, 8217, 8189] avg=8192.0000 max=8271 max_rank=0 min=8087 min_rank=4 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1032, 1032, 1060, 1061, 1020, 1032, 1025, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1001, 1038, 1064, 1051, 1007, 1043, 1022, 9, 7, 0, 0, 0, 0, 0, 0, 1043, 1013, 1083, 1022, 1039, 994, 974, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1017, 1019, 1030, 1017, 983, 1037, 981, 13, 9, 0, 0, 0, 0, 0, 0, 1009, 995, 1007, 992, 994, 1020, 1028, 1040, 23, 8, 0, 0, 0, 0, 0, 0, 1037, 981, 1008, 985, 1066, 1045, 1020, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1031, 1039, 980, 1026, 1045, 1057, 970, 2, 0, 0, 0, 0, 0, 0, 0, 1026, 1050, 1002, 996, 1036, 1044, 1006, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8284, 8148, 8120, 8116, 8226, 8177, 8195] avg=8192.0000 max=8284 max_rank=1 min=8116 min_rank=4 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1007, 1014, 986, 1017, 1024, 1072, 1034, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1003, 935, 1031, 973, 986, 1082, 1012, 9, 6, 0, 0, 0, 0, 0, 0, 1001, 982, 995, 1058, 1050, 1053, 993, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1009, 1006, 1100, 975, 1067, 1066, 1031, 12, 9, 0, 0, 0, 0, 0, 0, 969, 1049, 1051, 1062, 1082, 993, 985, 977, 24, 8, 0, 0, 0, 0, 0, 0, 1030, 1055, 1074, 1001, 1035, 1084, 1037, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 986, 996, 1016, 1033, 997, 1028, 1057, 996, 2, 0, 0, 0, 0, 0, 0, 0, 1011, 1001, 1023, 1107, 1019, 1028, 967, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8103, 8176, 8271, 8200, 8367, 8111, 8159] avg=8192.0000 max=8367 max_rank=5 min=8103 min_rank=1 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1013, 1007, 1066, 1061, 985, 1034, 1060, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1048, 1008, 974, 998, 970, 1038, 1005, 10, 6, 0, 0, 0, 0, 0, 0, 1023, 1030, 994, 1042, 1039, 1038, 989, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1006, 1032, 962, 1040, 1049, 1022, 993, 12, 9, 0, 0, 0, 0, 0, 0, 1034, 1016, 1062, 1045, 1003, 1000, 997, 983, 24, 8, 0, 0, 0, 0, 0, 0, 1017, 1031, 1056, 1024, 1040, 994, 1039, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1033, 1020, 1004, 1008, 1030, 1053, 1066, 2, 0, 0, 0, 0, 0, 0, 0, 1007, 1036, 1052, 1036, 1032, 1003, 1031, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8116, 8178, 8162, 8172, 8218, 8208, 8191] avg=8192.0000 max=8291 max_rank=0 min=8116 min_rank=1 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1014, 1015, 974, 1004, 1007, 1028, 1057, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1048, 1005, 984, 1013, 1001, 1020, 1030, 12, 11, 0, 0, 0, 0, 0, 0, 1002, 1016, 988, 1055, 1031, 1031, 1089, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1079, 991, 1005, 1035, 1029, 1007, 1012, 17, 15, 0, 0, 0, 0, 0, 0, 1066, 1011, 1011, 1013, 995, 1030, 997, 964, 27, 12, 0, 0, 0, 0, 0, 0, 1025, 1021, 1019, 1022, 1007, 1062, 1012, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1011, 1065, 1001, 1023, 1013, 993, 1040, 3, 0, 0, 0, 0, 0, 0, 0, 998, 1016, 1010, 1013, 1073, 1085, 1030, 1056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8159, 8264, 8188, 8126, 8194, 8179, 8281] avg=8192.0000 max=8281 max_rank=7 min=8126 min_rank=4 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32780 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8064, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8064, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8192, 8320, 8320, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1048, 1047, 984, 1022, 1037, 1011, 999, 1046, 1011, 1012, 1026, 1045, 963, 1010, 1009, 997, 1011, 1010, 1003, 1018, 1009, 1020, 1030, 1031, 1035, 1035, 1027, 1043, 1006, 1043, 996, 1048, 1054, 984, 1080, 1053, 1032, 1025, 1046, 1023, 1055, 1028, 1038, 1016, 1029, 1045, 1038, 1033, 1007, 1017, 1004, 1026, 1008, 998, 1044, 1005, 1018, 1053, 1040, 1034, 999, 1002, 1062, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1008, 1027, 1064, 1044, 1023, 1002, 1027, 1055, 1017, 1004, 1070, 1007, 1032, 1019, 1009, 1025, 1060, 1012, 1088, 1068, 995, 1057, 1058, 992, 971, 1090, 977, 1015, 1001, 1049, 1050, 1054, 1068, 1033, 1023, 1025, 923, 1055, 1030, 1021, 1059, 965, 1015, 1003, 1002, 986, 1035, 1035, 1047, 1088, 970, 987, 1020, 1023, 1005, 1001, 1030, 998, 1014, 1021, 1022, 1039, 1029, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1018, 1020, 1038, 1049, 1021, 1088, 1060, 954, 1055, 1081, 1029, 946, 1084, 1043, 1038, 1062, 1010, 1028, 1016, 1015, 982, 1037, 1021, 1003, 1011, 993, 1003, 981, 1015, 1022, 1042, 1078, 1019, 985, 1039, 1001, 1017, 1006, 1013, 1059, 1011, 1014, 1034, 992, 1017, 1022, 1029, 1018, 985, 1081, 997, 1054, 984, 1033, 1021, 1032, 1064, 1026, 991, 1047, 1013, 1060, 1003, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1018, 986, 1030, 960, 990, 993, 999, 1033, 1034, 1037, 1013, 1032, 1015, 1079, 1026, 1036, 1074, 989, 1074, 1030, 961, 1031, 1038, 965, 1010, 1001, 1026, 1040, 1033, 1037, 978, 1073, 1024, 983, 1052, 1012, 997, 1033, 1040, 1008, 1043, 1002, 1002, 975, 1067, 1087, 1036, 1009, 1057, 1038, 1057, 987, 1081, 987, 1042, 1063, 1049, 1067, 977, 1003, 1056, 1011, 1016, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1061, 1030, 1020, 1018, 1031, 1009, 1034, 1002, 1058, 1029, 1027, 988, 994, 990, 995, 957, 1019, 1072, 1064, 1018, 1029, 1032, 1034, 1052, 1021, 1005, 1032, 1057, 1063, 1010, 1040, 1011, 1063, 988, 1019, 1026, 1010, 1034, 1000, 1036, 1032, 1027, 1041, 970, 1044, 1024, 999, 1035, 1080, 1012, 1025, 1048, 980, 1073, 995, 1038, 1035, 1016, 1010, 996, 1010, 1007, 1012, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1033, 1006, 972, 1043, 1011, 996, 1028, 1016, 1056, 1074, 1023, 1045, 1006, 1047, 1006, 1037, 979, 1000, 1039, 1011, 1079, 1005, 1001, 1014, 1044, 993, 1016, 1023, 1048, 1027, 1002, 1030, 1016, 1049, 1026, 1009, 992, 1020, 1006, 1050, 1021, 1067, 1017, 1054, 1023, 978, 1005, 993, 987, 1008, 1050, 995, 1021, 1026, 1040, 1019, 1061, 1062, 1011, 1001, 1033, 1072, 1040, 1074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1063, 1011, 1020, 1026, 1063, 1023, 1045, 1062, 1006, 1017, 999, 1051, 1004, 1013, 1026, 1009, 1013, 1027, 1014, 1031, 1017, 1032, 1059, 999, 1023, 1041, 1043, 1017, 1024, 1054, 1049, 1035, 967, 1039, 1006, 1025, 975, 1010, 999, 1029, 999, 994, 1016, 1056, 1019, 1017, 1006, 991, 1031, 1000, 979, 1007, 1028, 1048, 1026, 1051, 1084, 1054, 1006, 1022, 967, 1024, 1078, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1018, 1029, 1057, 1020, 1042, 1012, 1037, 1018, 993, 1090, 1028, 1050, 1001, 1014, 1036, 1008, 1018, 1054, 1002, 1024, 1038, 1031, 1042, 1016, 1039, 1038, 978, 1055, 1019, 1006, 1036, 1029, 1033, 1015, 1034, 1043, 1022, 1008, 1025, 1016, 1001, 1039, 1079, 1003, 991, 1002, 1026, 996, 1009, 1045, 1013, 1010, 1076, 1064, 985, 1003, 1035, 1018, 979, 983, 1024, 1008, 1044, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1046, 1047, 984, 1022, 1037, 1011, 999, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1012, 1026, 1045, 963, 1010, 1009, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1010, 996, 1018, 1009, 1020, 1030, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1035, 1027, 1043, 1006, 1043, 996, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 984, 1080, 1053, 1032, 1025, 1046, 1023, 13, 7, 1, 0, 0, 0, 0, 0, 1055, 1028, 1038, 1016, 1029, 1045, 1038, 1033, 21, 6, 0, 0, 0, 0, 0, 0, 1007, 1017, 1004, 1026, 1008, 998, 1044, 1005, 1, 0, 0, 0, 0, 0, 0, 0, 997, 1053, 1040, 1034, 999, 1002, 1062, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8073, 8125, 8214, 8318, 8309, 8110, 8195] avg=8192.0000 max=8318 max_rank=4 min=8073 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32932 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1007, 1027, 1064, 1044, 1023, 1002, 1027, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1004, 1070, 1007, 1032, 1019, 1009, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1012, 1080, 1068, 995, 1057, 1058, 992, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1090, 977, 1015, 1001, 1049, 1050, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1033, 1023, 1025, 923, 1055, 1030, 1021, 13, 8, 1, 0, 0, 0, 0, 0, 1059, 965, 1015, 1003, 1002, 986, 1035, 1035, 22, 6, 0, 0, 0, 0, 0, 0, 1047, 1088, 970, 987, 1020, 1023, 1005, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 998, 1014, 1021, 1022, 1039, 1029, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8183, 8322, 8188, 8200, 8128, 8141, 8125] avg=8192.0000 max=8322 max_rank=2 min=8125 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1032, 1006, 972, 1043, 1011, 996, 1028, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1074, 1023, 1045, 1006, 1047, 1006, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1000, 1031, 1011, 1079, 1005, 1001, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 993, 1016, 1023, 1048, 1027, 1002, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1049, 1026, 1009, 992, 1020, 1006, 1050, 12, 8, 1, 0, 0, 0, 0, 0, 1021, 1067, 1017, 1054, 1023, 978, 1005, 993, 22, 6, 0, 0, 0, 0, 0, 0, 987, 1008, 1050, 995, 1021, 1026, 1040, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1062, 1011, 1001, 1033, 1072, 1040, 1074, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8294, 8120, 8165, 8189, 8186, 8146, 8332] avg=8192.0000 max=8332 max_rank=7 min=8104 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1017, 986, 1030, 960, 990, 993, 999, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1037, 1013, 1032, 1015, 1079, 1026, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 989, 1066, 1030, 961, 1031, 1038, 965, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1001, 1026, 1040, 1033, 1037, 978, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 983, 1052, 1012, 997, 1033, 1040, 1008, 13, 8, 1, 0, 0, 0, 0, 0, 1043, 1002, 1002, 975, 1067, 1087, 1036, 1009, 22, 6, 0, 0, 0, 0, 0, 0, 1057, 1038, 1057, 987, 1081, 987, 1042, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1067, 977, 1003, 1056, 1011, 1016, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8008, 8272, 8154, 8179, 8171, 8249, 8312, 8191] avg=8192.0000 max=8312 max_rank=6 min=8008 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32923 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1017, 1020, 1038, 1049, 1021, 1088, 1060, 954, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1081, 1029, 946, 1084, 1043, 1038, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1028, 1009, 1015, 982, 1037, 1021, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 993, 1003, 981, 1015, 1022, 1042, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 985, 1039, 1001, 1017, 1006, 1013, 1059, 13, 7, 1, 0, 0, 0, 0, 0, 1011, 1014, 1034, 992, 1017, 1022, 1029, 1018, 22, 6, 0, 0, 0, 0, 0, 0, 985, 1081, 997, 1054, 984, 1033, 1021, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1026, 991, 1047, 1013, 1060, 1003, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8338, 8105, 8126, 8160, 8165, 8187, 8208] avg=8192.0000 max=8338 max_rank=1 min=8105 min_rank=2 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1059, 1030, 1020, 1018, 1031, 1009, 1034, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1029, 1027, 988, 994, 990, 995, 957, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1072, 1056, 1018, 1029, 1032, 1034, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1005, 1032, 1057, 1063, 1010, 1040, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 988, 1019, 1026, 1010, 1034, 1000, 1036, 12, 8, 1, 0, 0, 0, 0, 0, 1032, 1027, 1041, 970, 1044, 1024, 999, 1035, 22, 6, 0, 0, 0, 0, 0, 0, 1080, 1012, 1025, 1048, 980, 1073, 995, 1038, 1, 0, 0, 0, 0, 0, 0, 0, 1013, 1016, 1010, 996, 1010, 1007, 1012, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8038, 8312, 8221, 8197, 8200, 8252, 8113] avg=8192.0000 max=8312 max_rank=2 min=8038 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1061, 1011, 1020, 1026, 1063, 1023, 1045, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1017, 999, 1051, 1004, 1013, 1026, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1027, 1007, 1031, 1017, 1032, 1059, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1041, 1043, 1017, 1024, 1054, 1049, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1039, 1006, 1025, 975, 1010, 999, 1029, 13, 7, 1, 0, 0, 0, 0, 0, 999, 994, 1016, 1056, 1019, 1017, 1006, 991, 23, 6, 0, 0, 0, 0, 0, 0, 1031, 1000, 979, 1007, 1028, 1048, 1026, 1051, 1, 0, 0, 0, 0, 0, 0, 0, 1061, 1054, 1006, 1022, 967, 1024, 1078, 1067, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8125, 8185, 8267, 8071, 8127, 8171, 8279] avg=8192.0000 max=8311 max_rank=0 min=8071 min_rank=4 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1007, 1029, 1057, 1020, 1042, 1012, 1037, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1083, 1028, 1050, 1001, 1014, 1036, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1054, 990, 1024, 1038, 1031, 1042, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1038, 978, 1055, 1019, 1006, 1036, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1015, 1034, 1043, 1022, 1008, 1025, 1016, 16, 12, 6, 0, 0, 0, 0, 0, 1001, 1039, 1079, 1003, 991, 1002, 1026, 996, 27, 8, 0, 0, 0, 0, 0, 0, 1009, 1045, 1013, 1010, 1076, 1064, 985, 1003, 5, 7, 0, 0, 0, 0, 0, 0, 1008, 1018, 979, 983, 1024, 1008, 1044, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8213, 8213, 8176, 8230, 8172, 8217, 8093] avg=8192.0000 max=8230 max_rank=4 min=8093 min_rank=7 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8192, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8448, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1100, 1036, 1077, 1009, 1020, 1026, 1042, 1033, 1020, 1065, 1068, 1025, 996, 1010, 1054, 995, 987, 1013, 1054, 1006, 1019, 1049, 1011, 1022, 969, 1043, 1012, 991, 998, 1050, 1059, 985, 1010, 1036, 1008, 1012, 1020, 992, 971, 991, 1023, 996, 1043, 1045, 1023, 1054, 1044, 1031, 1027, 1003, 1011, 1028, 1076, 1046, 1033, 1063, 1008, 1031, 995, 1013, 1031, 1036, 991, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1032, 1032, 1058, 1014, 1028, 1012, 962, 1038, 1025, 1012, 1056, 1042, 1008, 1022, 1003, 1020, 1020, 974, 986, 1035, 1066, 1047, 1082, 1020, 1028, 1002, 957, 1076, 1039, 1050, 1042, 979, 1028, 1032, 1053, 1071, 979, 1072, 1028, 956, 1018, 1016, 1045, 994, 1009, 981, 1051, 1033, 991, 1029, 1070, 1007, 1088, 966, 1068, 1039, 1061, 1015, 988, 1013, 1013, 999, 1033, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1019, 1004, 1022, 1116, 1028, 992, 1033, 985, 1023, 987, 1022, 1025, 1043, 1018, 1018, 1054, 1038, 984, 1042, 1016, 1028, 1016, 1010, 1021, 1022, 1019, 996, 1063, 1075, 1029, 1044, 1012, 1085, 1091, 1028, 983, 1032, 1009, 1033, 1035, 1004, 1053, 993, 1034, 1010, 969, 1019, 1053, 1053, 1008, 977, 1037, 1025, 1058, 989, 1044, 966, 959, 1051, 1030, 1024, 1050, 1025, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1056, 1040, 1054, 1004, 1019, 1012, 1030, 992, 1038, 1008, 1051, 997, 1021, 1037, 1084, 991, 1047, 1081, 1045, 1036, 1014, 1062, 980, 996, 1004, 1006, 1011, 1003, 1050, 1041, 1052, 994, 976, 1038, 1066, 1009, 1024, 1003, 1021, 983, 1024, 1067, 1012, 966, 1023, 989, 1030, 994, 1015, 996, 1033, 984, 1085, 1044, 1020, 1014, 1095, 1048, 1014, 1005, 1026, 1032, 1032, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[980, 991, 1013, 1049, 1008, 1002, 982, 1058, 987, 1028, 1032, 1006, 1016, 1017, 1043, 1046, 1014, 970, 1049, 1003, 1013, 1012, 1032, 1044, 1037, 1068, 1062, 1047, 1026, 1013, 1069, 1045, 1038, 1018, 1020, 1030, 1046, 992, 1071, 1049, 988, 1021, 1005, 1037, 1050, 985, 1011, 1074, 1047, 1002, 985, 996, 1055, 1049, 1016, 1084, 1012, 1025, 1006, 1012, 1013, 1035, 1000, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1000, 1020, 1014, 1062, 1052, 1035, 1033, 1071, 1018, 964, 967, 1022, 999, 1072, 1044, 1062, 1017, 1041, 978, 1028, 1015, 1067, 1010, 985, 1040, 999, 1018, 973, 1004, 1013, 1050, 1038, 1027, 1037, 1016, 1013, 1020, 1009, 1033, 1023, 1036, 1011, 1022, 1030, 1052, 1043, 998, 1022, 1033, 1008, 1001, 1072, 1013, 982, 1018, 1080, 1004, 1026, 1033, 1052, 1070, 1049, 990, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1017, 1030, 1020, 1023, 957, 1018, 1037, 1054, 1017, 1027, 1012, 1060, 1021, 1063, 990, 980, 1050, 1070, 1064, 1010, 1004, 1017, 1004, 1024, 1001, 1044, 1017, 1044, 1051, 1039, 988, 983, 1036, 995, 1018, 1062, 995, 1016, 1050, 1009, 1030, 1035, 1040, 1009, 989, 1002, 1005, 1008, 1053, 986, 1035, 1062, 1043, 1000, 1080, 1046, 1047, 989, 1028, 1014, 1040, 1025, 1035, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1037, 969, 976, 1013, 1029, 1009, 1047, 997, 1061, 1036, 1045, 1030, 1011, 1032, 1005, 1054, 1034, 971, 1025, 1014, 1052, 1014, 1030, 1035, 1024, 1058, 1011, 1029, 1009, 1049, 988, 1060, 979, 1015, 1026, 1013, 1023, 1037, 1060, 1022, 1055, 1017, 1026, 990, 1054, 1047, 1011, 1075, 1025, 1012, 1037, 1004, 993, 1010, 1015, 999, 1023, 940, 1049, 985, 1068, 1026, 1077, 1069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1032, 1032, 1058, 1014, 1028, 1012, 962, 1038, 1, 0, 0, 0, 0, 0, 0, 0, 1025, 1012, 1056, 1042, 1008, 1014, 1003, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 974, 986, 1035, 1066, 1047, 1082, 1020, 4, 0, 0, 0, 0, 0, 0, 0, 1028, 1002, 957, 1076, 1039, 1050, 1035, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1032, 1053, 1071, 979, 1072, 1028, 956, 5, 2, 0, 0, 0, 0, 0, 0, 1018, 1016, 1045, 994, 1009, 981, 1051, 1033, 11, 3, 0, 0, 0, 0, 0, 0, 991, 1029, 1070, 1007, 1058, 966, 1068, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1015, 988, 1013, 1013, 999, 1033, 1023, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8180, 8234, 8166, 8226, 8161, 8228, 8164] avg=8192.0000 max=8234 max_rank=2 min=8161 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1056, 1040, 1054, 1004, 1019, 1012, 1030, 992, 1, 0, 0, 0, 0, 0, 0, 0, 1038, 1008, 1051, 997, 1021, 1029, 1084, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1081, 1045, 1036, 1014, 1062, 980, 996, 4, 0, 0, 0, 0, 0, 0, 0, 1004, 1006, 1011, 1003, 1050, 1041, 1045, 994, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1038, 1066, 1009, 1024, 1003, 1021, 983, 5, 2, 0, 0, 0, 0, 0, 0, 1024, 1067, 1012, 966, 1023, 989, 1030, 994, 11, 3, 0, 0, 0, 0, 0, 0, 1015, 996, 1033, 984, 1055, 1044, 1020, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 1048, 1014, 1005, 1026, 1032, 1032, 1012, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8219, 8265, 8154, 8127, 8119, 8161, 8283] avg=8192.0000 max=8283 max_rank=7 min=8119 min_rank=5 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1019, 1004, 1022, 1116, 1028, 992, 1033, 985, 1, 0, 0, 0, 0, 0, 0, 0, 1023, 987, 1022, 1025, 1043, 1010, 1018, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 984, 1042, 1016, 1028, 1016, 1010, 1021, 4, 0, 0, 0, 0, 0, 0, 0, 1022, 1019, 996, 1063, 1075, 1029, 1037, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 1091, 1028, 983, 1032, 1009, 1033, 1035, 5, 2, 0, 0, 0, 0, 0, 0, 1004, 1053, 993, 1034, 1010, 969, 1019, 1053, 10, 3, 0, 0, 0, 0, 0, 0, 1053, 1008, 977, 1037, 998, 1058, 989, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 966, 959, 1051, 1030, 1024, 1050, 1025, 1005, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8182, 8159, 8253, 8303, 8148, 8164, 8127] avg=8192.0000 max=8303 max_rank=4 min=8127 min_rank=7 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1037, 969, 976, 1013, 1029, 1009, 1047, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1036, 1045, 1030, 1011, 1024, 1005, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 971, 1025, 1014, 1052, 1014, 1030, 1035, 3, 0, 0, 0, 0, 0, 0, 0, 1024, 1058, 1011, 1029, 1009, 1049, 983, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1015, 1026, 1013, 1023, 1037, 1060, 1022, 5, 2, 0, 0, 0, 0, 0, 0, 1055, 1017, 1026, 990, 1054, 1047, 1011, 1075, 10, 3, 0, 0, 0, 0, 0, 0, 1025, 1012, 1037, 1004, 966, 1010, 1015, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 940, 1049, 985, 1068, 1026, 1077, 1069, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8077, 8266, 8178, 8223, 8182, 8288, 8068, 8254] avg=8192.0000 max=8288 max_rank=5 min=8068 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1100, 1036, 1077, 1009, 1020, 1026, 1042, 1033, 1, 0, 0, 0, 0, 0, 0, 0, 1020, 1065, 1068, 1025, 996, 1002, 1054, 995, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1013, 1054, 1006, 1019, 1049, 1011, 1022, 4, 0, 0, 0, 0, 0, 0, 0, 969, 1043, 1012, 991, 998, 1050, 1052, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1036, 1008, 1012, 1020, 992, 971, 991, 5, 2, 0, 0, 0, 0, 0, 0, 1023, 996, 1043, 1045, 1023, 1054, 1044, 1031, 10, 3, 0, 0, 0, 0, 0, 0, 1027, 1003, 1011, 1028, 1048, 1046, 1033, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1031, 995, 1013, 1031, 1036, 991, 1001, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8344, 8225, 8165, 8100, 8047, 8272, 8259, 8124] avg=8192.0000 max=8344 max_rank=0 min=8047 min_rank=4 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32702 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[980, 991, 1013, 1049, 1008, 1002, 982, 1058, 3, 0, 0, 0, 0, 0, 0, 0, 987, 1028, 1032, 1006, 1016, 1004, 1043, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 970, 1049, 1003, 1013, 1012, 1032, 1044, 7, 0, 0, 0, 0, 0, 0, 0, 1037, 1068, 1062, 1047, 1026, 1013, 1052, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1018, 1020, 1030, 1046, 992, 1071, 1049, 7, 7, 0, 0, 0, 0, 0, 0, 988, 1021, 1005, 1037, 1050, 985, 1011, 1074, 13, 6, 0, 0, 0, 0, 0, 0, 1047, 1002, 985, 996, 1020, 1049, 1016, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1025, 1006, 1012, 1013, 1035, 1000, 1002, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 8162, 8144, 8350, 8278, 8190, 8199, 8127] avg=8192.0000 max=8350 max_rank=3 min=8086 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1017, 1030, 1020, 1023, 957, 1018, 1037, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1027, 1012, 1060, 1021, 1055, 990, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1070, 1064, 1010, 1004, 1017, 1004, 1024, 3, 0, 0, 0, 0, 0, 0, 0, 1001, 1044, 1017, 1044, 1051, 1039, 983, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 995, 1018, 1062, 995, 1016, 1050, 1009, 5, 2, 0, 0, 0, 0, 0, 0, 1030, 1035, 1040, 1009, 989, 1002, 1005, 1008, 10, 3, 0, 0, 0, 0, 0, 0, 1053, 986, 1035, 1062, 1015, 1000, 1080, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 989, 1028, 1014, 1040, 1025, 1035, 1018, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8162, 8246, 8162, 8188, 8131, 8277, 8214] avg=8192.0000 max=8277 max_rank=6 min=8131 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1000, 1020, 1014, 1062, 1052, 1035, 1033, 1071, 1, 0, 0, 0, 0, 0, 0, 0, 1018, 964, 967, 1022, 999, 1064, 1044, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1041, 978, 1028, 1015, 1067, 1010, 985, 4, 0, 0, 0, 0, 0, 0, 0, 1040, 999, 1018, 973, 1004, 1013, 1043, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1037, 1016, 1013, 1020, 1009, 1033, 1023, 5, 2, 0, 0, 0, 0, 0, 0, 1036, 1011, 1022, 1030, 1052, 1043, 998, 1022, 10, 3, 0, 0, 0, 0, 0, 0, 1033, 1008, 1001, 1072, 986, 982, 1018, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1026, 1033, 1052, 1070, 1049, 990, 1002, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8140, 8145, 8128, 8185, 8227, 8180, 8243] avg=8192.0000 max=8288 max_rank=0 min=8128 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8192, 8192, 8192, 8320, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8192, 8320, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1009, 1059, 990, 989, 1070, 1043, 964, 1057, 1009, 990, 1005, 994, 1046, 1003, 1040, 1024, 1061, 1053, 1039, 1047, 1063, 1065, 1020, 1044, 1008, 1051, 1029, 1029, 1030, 1045, 1023, 989, 998, 1025, 998, 1019, 1052, 1046, 1012, 1031, 1074, 984, 1012, 1047, 1011, 994, 1006, 1042, 1029, 1016, 1013, 1031, 978, 1035, 1017, 1013, 1012, 1039, 1033, 1018, 1004, 1027, 1008, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1027, 1002, 1004, 1016, 1076, 1035, 1027, 1056, 1029, 1036, 1030, 1013, 1003, 1025, 1039, 1034, 1036, 1080, 964, 1031, 978, 1002, 1016, 982, 1019, 1044, 1010, 994, 1059, 1018, 1054, 1040, 972, 1012, 991, 1007, 1008, 974, 1038, 1019, 1014, 991, 1063, 985, 993, 1079, 1078, 1002, 1063, 1008, 1069, 985, 1008, 1012, 1051, 1043, 1072, 1059, 1059, 1018, 988, 972, 1065, 1059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1040, 1017, 1028, 1029, 1030, 1005, 985, 998, 1008, 1001, 1016, 1048, 1045, 1018, 1042, 1042, 1053, 1014, 999, 1048, 1001, 1046, 1031, 1024, 1017, 1039, 1005, 989, 1063, 1037, 1003, 1012, 1001, 1022, 1081, 1061, 1029, 1004, 1024, 1026, 1034, 1007, 1068, 1013, 1012, 1003, 1000, 1029, 1037, 1073, 987, 1019, 1036, 1004, 1046, 1045, 1013, 1033, 978, 1020, 1039, 1027, 1044, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1050, 971, 1052, 1034, 1073, 1072, 1033, 1034, 1005, 1049, 1054, 1033, 1050, 1016, 1044, 994, 1091, 999, 1014, 1065, 1056, 1016, 1016, 1042, 1024, 1009, 1007, 1049, 1022, 1004, 1014, 1034, 992, 1033, 1019, 1012, 1018, 1061, 959, 1008, 1014, 1036, 1050, 1019, 1025, 1029, 1014, 1010, 1030, 1008, 997, 1013, 1013, 1060, 1043, 1016, 987, 1023, 1010, 984, 974, 980, 1007, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1024, 1013, 1033, 997, 1031, 1025, 1004, 1001, 1002, 1068, 1000, 1004, 973, 1016, 985, 1023, 986, 1050, 1000, 1010, 1067, 1040, 1043, 1048, 1042, 1003, 1008, 1069, 1036, 990, 1015, 1008, 1010, 989, 1013, 1041, 991, 1019, 1014, 985, 1002, 1071, 1051, 996, 1021, 1016, 1077, 1023, 1002, 1040, 1031, 1045, 1007, 1040, 1033, 994, 1036, 995, 1061, 1053, 1073, 1062, 1051, 1080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1029, 978, 995, 1002, 1041, 1029, 1038, 1011, 1006, 983, 1028, 1069, 1031, 1026, 1028, 1006, 1022, 1042, 1015, 1019, 1054, 1001, 986, 992, 1035, 1034, 1021, 997, 1066, 1051, 974, 956, 1074, 996, 1040, 1056, 1064, 1109, 1047, 1005, 996, 1031, 1026, 986, 994, 1080, 1013, 1039, 1017, 1077, 1030, 993, 1002, 1015, 1000, 1035, 996, 1021, 1026, 1046, 1022, 1035, 1029, 1071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1026, 1028, 1079, 1069, 1032, 1016, 1064, 1032, 998, 1035, 1025, 1021, 1076, 1060, 1021, 994, 1001, 1036, 1046, 993, 1031, 1069, 1009, 1006, 984, 963, 981, 1018, 1044, 1043, 1040, 1042, 1010, 1041, 1043, 996, 1013, 1009, 1052, 996, 1033, 1018, 1018, 1015, 1027, 1029, 998, 1059, 1007, 1044, 1014, 1040, 1049, 1021, 987, 1020, 1004, 1048, 1029, 997, 965, 1015, 1059, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1073, 1065, 1000, 964, 985, 996, 999, 1009, 1035, 1042, 1017, 994, 1013, 991, 1015, 1078, 986, 1016, 1004, 1023, 1003, 1031, 997, 999, 995, 1026, 1063, 1008, 999, 1028, 995, 1012, 975, 1046, 1038, 1064, 1077, 1055, 1062, 1019, 1080, 1010, 999, 1014, 1008, 1005, 1017, 1069, 1003, 1018, 1042, 1053, 1009, 1055, 982, 996, 1042, 1025, 1050, 1057, 1060, 1040, 1043, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1009, 1059, 990, 989, 1068, 1043, 964, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 990, 1005, 994, 1046, 1003, 1040, 1024, 8, 3, 0, 0, 0, 0, 0, 0, 1061, 1043, 1039, 1047, 1063, 1065, 1020, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1051, 1029, 1029, 1030, 1045, 1023, 989, 26, 0, 0, 0, 0, 0, 0, 0, 998, 1025, 998, 1019, 1052, 1046, 1012, 1031, 2, 1, 0, 0, 0, 0, 0, 0, 1074, 984, 1010, 1047, 1011, 994, 1006, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1016, 1013, 1031, 978, 1035, 1017, 1013, 2, 1, 0, 0, 0, 0, 0, 0, 1012, 1039, 1033, 1018, 1004, 1027, 1008, 995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8122, 8382, 8230, 8184, 8168, 8135, 8136] avg=8192.0000 max=8382 max_rank=2 min=8122 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1027, 1002, 1004, 1016, 1074, 1035, 1027, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1036, 1030, 1013, 1003, 1025, 1039, 1034, 8, 3, 0, 0, 0, 0, 0, 0, 1036, 1070, 964, 1031, 978, 1002, 1016, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1044, 1010, 994, 1059, 1018, 1054, 1040, 27, 0, 0, 0, 0, 0, 0, 0, 972, 1012, 991, 1007, 1008, 974, 1038, 1019, 2, 1, 0, 0, 0, 0, 0, 0, 1014, 991, 1061, 985, 993, 1079, 1078, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1008, 1069, 985, 1008, 1012, 1051, 1043, 2, 1, 0, 0, 0, 0, 0, 0, 1072, 1059, 1059, 1018, 988, 972, 1065, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8220, 8079, 8265, 8024, 8203, 8242, 8262] avg=8192.0000 max=8265 max_rank=3 min=8024 min_rank=4 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1050, 971, 1052, 1034, 1071, 1072, 1033, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1049, 1054, 1033, 1050, 1016, 1044, 994, 7, 3, 0, 0, 0, 0, 0, 0, 1091, 990, 1014, 1065, 1056, 1016, 1016, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1009, 1007, 1049, 1022, 1004, 1014, 1034, 28, 0, 0, 0, 0, 0, 0, 0, 992, 1033, 1019, 1012, 1018, 1061, 959, 1008, 2, 1, 0, 0, 0, 0, 0, 0, 1014, 1036, 1048, 1019, 1025, 1029, 1014, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1008, 997, 1013, 1013, 1060, 1043, 1016, 2, 1, 0, 0, 0, 0, 0, 0, 987, 1023, 1010, 984, 974, 980, 1007, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 8255, 8290, 8191, 8105, 8195, 8183, 8000] avg=8192.0000 max=8317 max_rank=0 min=8000 min_rank=7 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32483 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1029, 978, 995, 1002, 1039, 1029, 1038, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 983, 1028, 1069, 1031, 1026, 1028, 1006, 7, 3, 0, 0, 0, 0, 0, 0, 1022, 1033, 1015, 1019, 1054, 1001, 986, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1034, 1021, 997, 1066, 1051, 974, 956, 28, 0, 0, 0, 0, 0, 0, 0, 1074, 996, 1040, 1056, 1064, 1109, 1047, 1005, 2, 1, 0, 0, 0, 0, 0, 0, 996, 1031, 1024, 986, 994, 1080, 1013, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1077, 1030, 993, 1002, 1015, 1000, 1035, 2, 1, 0, 0, 0, 0, 0, 0, 996, 1021, 1026, 1046, 1022, 1035, 1029, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8187, 8122, 8162, 8394, 8163, 8172, 8215] avg=8192.0000 max=8394 max_rank=4 min=8121 min_rank=0 imbalance_ratio=1.024658 max_load_reduction=57142 max_load_reduction_pct=0.871918 moved_to_echo=32944 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1024, 1013, 1033, 997, 1029, 1025, 1004, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1068, 1000, 1004, 973, 1016, 985, 1023, 7, 3, 0, 0, 0, 0, 0, 0, 986, 1041, 1000, 1010, 1067, 1040, 1043, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1003, 1008, 1069, 1036, 990, 1015, 1008, 28, 0, 0, 0, 0, 0, 0, 0, 1010, 989, 1013, 1041, 991, 1019, 1014, 985, 2, 1, 0, 0, 0, 0, 0, 0, 1002, 1071, 1049, 996, 1021, 1016, 1077, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1040, 1031, 1045, 1007, 1040, 1033, 994, 2, 1, 0, 0, 0, 0, 0, 0, 1036, 995, 1061, 1053, 1073, 1062, 1051, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8081, 8235, 8199, 8065, 8255, 8195, 8380] avg=8192.0000 max=8380 max_rank=7 min=8065 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32895 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1040, 1017, 1028, 1029, 1028, 1005, 985, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1001, 1016, 1048, 1045, 1018, 1042, 1042, 7, 2, 0, 0, 0, 0, 0, 0, 1053, 1005, 999, 1048, 1001, 1046, 1031, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1039, 1005, 989, 1063, 1037, 1003, 1012, 26, 0, 0, 0, 0, 0, 0, 0, 1001, 1022, 1081, 1061, 1029, 1004, 1024, 1026, 2, 1, 0, 0, 0, 0, 0, 0, 1034, 1007, 1066, 1013, 1012, 1003, 1000, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1073, 987, 1019, 1036, 1004, 1046, 1045, 2, 1, 0, 0, 0, 0, 0, 0, 1013, 1033, 978, 1020, 1039, 1027, 1044, 960, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8229, 8207, 8191, 8251, 8164, 8250, 8114] avg=8192.0000 max=8251 max_rank=4 min=8114 min_rank=7 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1026, 1028, 1079, 1069, 1030, 1016, 1064, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1035, 1025, 1021, 1076, 1060, 1021, 994, 7, 2, 0, 0, 0, 0, 0, 0, 1001, 1027, 1046, 993, 1031, 1069, 1009, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 984, 963, 981, 1018, 1044, 1043, 1040, 1042, 26, 0, 0, 0, 0, 0, 0, 0, 1010, 1041, 1043, 996, 1013, 1009, 1052, 996, 2, 1, 0, 0, 0, 0, 0, 0, 1033, 1018, 1016, 1015, 1027, 1029, 998, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1044, 1014, 1040, 1049, 1021, 987, 1020, 2, 1, 0, 0, 0, 0, 0, 0, 1004, 1048, 1029, 997, 965, 1015, 1059, 970, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8344, 8239, 8182, 8141, 8163, 8195, 8185, 8087] avg=8192.0000 max=8344 max_rank=0 min=8087 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32630 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1073, 1065, 1000, 964, 972, 996, 999, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1042, 1017, 994, 1013, 991, 1015, 1078, 12, 6, 0, 0, 0, 0, 0, 0, 986, 996, 1004, 1023, 1003, 1031, 997, 999, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1026, 1063, 1008, 999, 1028, 995, 1012, 31, 0, 0, 0, 0, 0, 0, 0, 975, 1046, 1038, 1064, 1077, 1055, 1062, 1019, 9, 5, 0, 0, 0, 0, 0, 0, 1080, 1010, 990, 1014, 1008, 1005, 1017, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1018, 1042, 1053, 1009, 1055, 982, 996, 8, 8, 0, 0, 0, 0, 0, 0, 1042, 1025, 1050, 1057, 1060, 1040, 1043, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8078, 8203, 8039, 8157, 8350, 8193, 8174, 8342] avg=8192.0000 max=8350 max_rank=4 min=8039 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=33059 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[981, 995, 1069, 1032, 1004, 1012, 1024, 973, 1061, 1004, 1025, 1030, 1056, 1036, 1017, 968, 997, 1050, 1075, 1053, 1020, 1001, 1030, 1015, 1060, 991, 1014, 1035, 1046, 1037, 945, 1046, 1022, 1012, 1009, 1025, 1013, 1057, 1024, 995, 1065, 1079, 1033, 970, 1002, 971, 1054, 1010, 1088, 1046, 1054, 1040, 1014, 1016, 996, 1066, 1051, 989, 1016, 1032, 1008, 994, 1040, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1008, 1005, 996, 1053, 1008, 1060, 1022, 1022, 989, 1048, 1027, 999, 1009, 993, 999, 1027, 1009, 999, 1060, 1094, 1039, 1015, 1060, 1020, 1004, 1029, 1058, 1012, 1055, 1062, 1005, 1046, 1018, 1024, 1023, 974, 1068, 1052, 992, 1022, 978, 1006, 1011, 1011, 1018, 1055, 1013, 1036, 1028, 1017, 985, 1064, 986, 992, 1016, 1031, 1008, 1011, 1031, 1064, 1035, 1093, 1014, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[999, 1034, 1041, 1030, 1048, 1044, 934, 1003, 952, 1034, 958, 1051, 986, 1041, 1013, 1018, 1031, 989, 1046, 1006, 1030, 1046, 1013, 938, 1084, 1047, 987, 1019, 1033, 1032, 1011, 1032, 937, 1044, 1043, 1020, 1011, 1025, 1095, 1043, 1028, 1013, 1005, 1028, 1040, 1090, 1030, 1015, 1068, 1058, 990, 1016, 1047, 1075, 1057, 1077, 964, 1001, 1047, 1022, 1040, 1029, 1008, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1038, 982, 1038, 1065, 1020, 984, 989, 1004, 1033, 1041, 995, 1039, 1019, 1034, 1056, 1021, 1036, 1055, 1054, 1035, 994, 1025, 1002, 1032, 1010, 1019, 1051, 1006, 963, 1046, 1051, 1017, 1007, 1007, 1038, 1062, 1019, 978, 983, 1023, 1050, 1030, 1019, 1037, 1039, 1019, 1001, 1065, 1050, 1000, 1009, 965, 1023, 1048, 1039, 981, 1037, 1055, 1004, 1034, 1033, 1025, 1029, 1073] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1023, 1071, 1008, 995, 1055, 1040, 1031, 1063, 1050, 1024, 995, 1057, 982, 984, 1033, 1042, 1004, 1060, 1005, 1078, 1024, 1011, 976, 1032, 978, 1023, 1004, 1070, 1032, 1057, 1003, 986, 1007, 1039, 1042, 1041, 1051, 1019, 1048, 1019, 1025, 1006, 1014, 1043, 1002, 996, 980, 1037, 1055, 1000, 1068, 1012, 1019, 1034, 1064, 1021, 1001, 1027, 1048, 998, 1015, 1011, 967, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1044, 1038, 1031, 1016, 1053, 1012, 1020, 1011, 1066, 1023, 997, 1024, 1003, 1032, 956, 990, 1036, 1044, 966, 988, 1002, 1034, 1060, 1062, 1003, 1019, 1021, 1080, 1060, 1039, 983, 1014, 997, 991, 1012, 996, 1017, 1021, 982, 1066, 1090, 1083, 1020, 981, 1049, 1027, 1019, 1007, 1043, 1015, 1072, 1065, 1038, 1000, 1059, 1020, 1057, 1057, 1020, 993, 1017, 1033, 986, 976] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1040, 1013, 1001, 1013, 1021, 1035, 1033, 989, 1071, 995, 969, 1012, 1009, 1037, 1024, 975, 1031, 1001, 1040, 1038, 1015, 1053, 1058, 1041, 1016, 1039, 995, 1010, 990, 1002, 1049, 1031, 1051, 1027, 1002, 1020, 1030, 1043, 979, 1037, 1037, 1024, 1054, 991, 1025, 1013, 985, 1047, 1045, 1022, 1026, 1018, 1062, 1009, 1054, 1071, 1020, 1020, 1055, 1024, 1077, 1009, 995, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1029, 1038, 1027, 1009, 1017, 1088, 1046, 1021, 992, 1008, 1071, 1012, 1074, 1052, 1026, 1010, 1004, 1029, 1040, 1068, 992, 1043, 997, 1018, 999, 1022, 1041, 989, 1047, 1043, 1017, 1020, 1001, 1072, 1013, 1018, 991, 1062, 1017, 1014, 1009, 1013, 1037, 1000, 1042, 1030, 990, 1035, 1008, 1016, 979, 1044, 1073, 1024, 995, 1013, 990, 996, 994, 1034, 1057, 1036, 1046, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1008, 1005, 996, 1053, 1008, 1060, 1022, 1022, 11, 0, 0, 0, 0, 0, 0, 0, 989, 1048, 1027, 999, 1009, 993, 999, 1027, 44, 0, 0, 0, 0, 0, 0, 0, 1009, 999, 1060, 1072, 1039, 1015, 1060, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1029, 1058, 1012, 1055, 1062, 1005, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1024, 1023, 974, 1068, 1052, 992, 1022, 10, 7, 0, 0, 0, 0, 0, 0, 978, 1006, 1011, 1011, 1018, 1055, 1013, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1017, 985, 1064, 986, 992, 1016, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1011, 1031, 1064, 1035, 1093, 1014, 1028, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8135, 8274, 8271, 8190, 8128, 8065, 8288] avg=8192.0000 max=8288 max_rank=7 min=8065 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1038, 982, 1038, 1065, 1020, 984, 989, 1004, 10, 0, 0, 0, 0, 0, 0, 0, 1033, 1041, 995, 1039, 1019, 1034, 1056, 1021, 45, 0, 0, 0, 0, 0, 0, 0, 1036, 1055, 1054, 1015, 994, 1025, 1002, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1019, 1051, 1006, 963, 1046, 1051, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1007, 1038, 1062, 1019, 978, 983, 1023, 11, 7, 0, 0, 0, 0, 0, 0, 1050, 1030, 1019, 1037, 1039, 1019, 1001, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1000, 1009, 965, 1023, 1048, 1039, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1055, 1004, 1034, 1033, 1025, 1029, 1073, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8283, 8213, 8163, 8135, 8260, 8059, 8293] avg=8192.0000 max=8293 max_rank=7 min=8059 min_rank=6 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[981, 995, 1069, 1032, 1004, 1012, 1024, 973, 11, 0, 0, 0, 0, 0, 0, 0, 1061, 1004, 1025, 1030, 1056, 1036, 1017, 968, 46, 0, 0, 0, 0, 0, 0, 0, 997, 1050, 1075, 1031, 1020, 1001, 1030, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 991, 1014, 1035, 1046, 1037, 945, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1012, 1009, 1025, 1013, 1057, 1024, 995, 11, 7, 0, 0, 0, 0, 0, 0, 1065, 1079, 1033, 970, 1002, 971, 1054, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1046, 1054, 1040, 1014, 1016, 996, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 989, 1016, 1032, 1008, 994, 1040, 1043, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8243, 8219, 8174, 8175, 8184, 8263, 8177] avg=8192.0000 max=8263 max_rank=6 min=8101 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1023, 1071, 1008, 995, 1055, 1040, 1031, 1063, 11, 0, 0, 0, 0, 0, 0, 0, 1050, 1024, 995, 1057, 982, 984, 1033, 1042, 45, 0, 0, 0, 0, 0, 0, 0, 1004, 1060, 1005, 1056, 1024, 1011, 976, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1023, 1004, 1070, 1032, 1057, 1003, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1039, 1042, 1041, 1051, 1019, 1048, 1019, 11, 7, 0, 0, 0, 0, 0, 0, 1025, 1006, 1014, 1043, 1002, 996, 980, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1000, 1068, 1012, 1019, 1034, 1064, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1027, 1048, 998, 1015, 1011, 967, 1031, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8212, 8168, 8153, 8284, 8103, 8217, 8102] avg=8192.0000 max=8297 max_rank=0 min=8102 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[999, 1034, 1041, 1030, 1048, 1044, 934, 1003, 10, 0, 0, 0, 0, 0, 0, 0, 952, 1034, 958, 1051, 986, 1041, 1013, 1018, 46, 0, 0, 0, 0, 0, 0, 0, 1031, 989, 1046, 987, 1030, 1046, 1013, 938, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 1047, 987, 1019, 1033, 1032, 1011, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 937, 1044, 1043, 1020, 1011, 1025, 1095, 1043, 11, 6, 0, 0, 0, 0, 0, 0, 1028, 1013, 1005, 1028, 1040, 1090, 1030, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1058, 990, 1016, 1047, 1075, 1057, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 964, 1001, 1047, 1022, 1040, 1029, 1008, 1040, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8099, 8080, 8245, 8235, 8249, 8331, 8154] avg=8192.0000 max=8331 max_rank=6 min=8080 min_rank=2 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32969 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1040, 1013, 1001, 1013, 1021, 1035, 1033, 989, 10, 0, 0, 0, 0, 0, 0, 0, 1071, 995, 969, 1012, 1009, 1037, 1024, 975, 45, 0, 0, 0, 0, 0, 0, 0, 1031, 1001, 1040, 1018, 1015, 1053, 1058, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1039, 995, 1010, 990, 1002, 1049, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1027, 1002, 1020, 1030, 1043, 979, 1037, 10, 7, 0, 0, 0, 0, 0, 0, 1037, 1024, 1054, 991, 1025, 1013, 985, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1022, 1026, 1018, 1062, 1009, 1054, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1020, 1055, 1024, 1077, 1009, 995, 1018, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8137, 8257, 8132, 8206, 8176, 8252, 8221] avg=8192.0000 max=8257 max_rank=2 min=8132 min_rank=3 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1044, 1038, 1031, 1016, 1053, 1012, 1020, 1011, 10, 0, 0, 0, 0, 0, 0, 0, 1066, 1023, 997, 1024, 1003, 1032, 956, 990, 45, 0, 0, 0, 0, 0, 0, 0, 1036, 1044, 966, 969, 1002, 1034, 1060, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1019, 1021, 1080, 1060, 1039, 983, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 997, 991, 1012, 996, 1017, 1021, 982, 1066, 10, 6, 0, 0, 0, 0, 0, 0, 1090, 1083, 1020, 981, 1049, 1027, 1019, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1015, 1072, 1065, 1038, 1000, 1059, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1057, 1020, 993, 1017, 1033, 986, 976, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8136, 8173, 8219, 8098, 8276, 8257, 8142] avg=8192.0000 max=8276 max_rank=5 min=8098 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1029, 1038, 1027, 1009, 1017, 1088, 1046, 1021, 15, 0, 0, 0, 0, 0, 0, 0, 992, 1008, 1071, 1012, 1074, 1052, 1026, 1010, 46, 0, 0, 0, 0, 0, 0, 0, 1004, 1029, 1040, 1029, 992, 1043, 997, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1022, 1041, 989, 1047, 1043, 1017, 1020, 1, 0, 0, 0, 0, 0, 0, 0, 1001, 1072, 1013, 1018, 991, 1062, 1017, 1014, 14, 11, 0, 0, 0, 0, 0, 0, 1009, 1013, 1037, 1000, 1042, 1030, 990, 1035, 4, 0, 0, 0, 0, 0, 0, 0, 948, 1016, 979, 1044, 1073, 1024, 995, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 990, 996, 994, 1034, 1057, 1036, 1046, 998, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 8291, 8152, 8179, 8213, 8160, 8092, 8159] avg=8192.0000 max=8291 max_rank=1 min=8092 min_rank=6 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7936, 8192, 8192, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8064, 8320, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8064, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8320, 8192, 8320, 8192, 8064, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1021, 1022, 1060, 1054, 1014, 1056, 992, 1022, 1009, 1046, 1008, 1067, 1027, 1028, 1055, 994, 1010, 999, 1006, 1016, 1042, 1034, 1051, 1015, 1011, 1023, 1000, 1040, 1024, 1089, 1040, 1002, 1050, 1054, 999, 1002, 985, 1010, 1020, 1001, 1013, 1039, 1030, 1021, 1021, 978, 1005, 1047, 1022, 1018, 1026, 1023, 1081, 994, 977, 1028, 1051, 1027, 1036, 1065, 988, 1003, 1039, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1041, 1020, 1038, 1007, 1064, 1040, 1029, 1022, 1023, 1041, 1017, 975, 973, 1014, 1035, 1046, 998, 1029, 984, 994, 992, 1015, 1013, 1031, 1008, 1028, 1023, 1053, 983, 1030, 1031, 1006, 1077, 989, 1032, 1044, 1053, 987, 1009, 1033, 1034, 1031, 1022, 1013, 1019, 998, 1067, 1032, 1030, 994, 1050, 1097, 1059, 987, 1012, 994, 1048, 1004, 1018, 1050, 981, 1040, 1097, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1037, 998, 1060, 1035, 987, 968, 1057, 996, 1018, 1014, 1003, 1035, 1047, 1023, 1031, 1038, 1086, 1005, 1074, 1066, 1016, 975, 1050, 1011, 988, 1048, 1022, 1015, 1031, 1016, 993, 1047, 1073, 1006, 950, 1026, 1032, 1011, 1022, 1026, 1034, 1029, 999, 1009, 1046, 1061, 1055, 1036, 1009, 990, 1023, 1020, 985, 1031, 1036, 1007, 1052, 1021, 1030, 1046, 998, 1033, 1014, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1069, 1013, 984, 1050, 1053, 1039, 1053, 1035, 1053, 1039, 1008, 1025, 999, 994, 1041, 1016, 987, 1021, 1027, 1027, 1010, 1019, 1010, 1022, 1028, 1003, 1017, 997, 1017, 1061, 1047, 1000, 991, 1001, 1078, 1075, 1046, 1011, 1011, 993, 1032, 987, 1005, 1018, 962, 1023, 1033, 1041, 1027, 1067, 1079, 1039, 996, 1082, 986, 1083, 1018, 1031, 1030, 1035, 1020, 1012, 990, 970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[988, 1007, 1041, 1046, 1004, 1033, 1056, 1020, 1025, 1002, 989, 1009, 994, 1010, 1020, 962, 1057, 979, 1016, 1062, 1105, 1003, 1014, 1049, 1003, 1011, 983, 1041, 1038, 996, 1094, 1075, 1054, 1036, 1037, 994, 1021, 1005, 1024, 996, 1019, 1036, 999, 1008, 1034, 1037, 1022, 1013, 1080, 1064, 1032, 1065, 1033, 1012, 981, 1031, 1008, 1001, 1036, 1058, 1065, 964, 1047, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1026, 1011, 985, 1039, 1003, 1010, 958, 1005, 1024, 1036, 1053, 954, 1039, 1094, 996, 1039, 1029, 1000, 1019, 967, 1044, 1021, 1047, 1002, 1063, 1065, 950, 976, 1014, 966, 1016, 1042, 1073, 1059, 990, 998, 1068, 1042, 998, 1060, 1006, 1081, 990, 1006, 986, 1065, 1072, 1003, 1039, 1080, 1024, 1009, 1051, 1007, 998, 1094, 1035, 1051, 999, 1052, 982, 1046, 1034, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1071, 1045, 1032, 994, 1021, 1021, 1041, 1008, 1056, 999, 1027, 1020, 1024, 1048, 1019, 985, 959, 996, 1025, 1002, 1052, 1083, 1023, 1034, 1001, 1054, 1033, 1033, 996, 1084, 998, 1036, 989, 996, 1045, 1062, 1026, 1005, 1051, 1026, 1021, 994, 1000, 1014, 997, 1034, 1019, 1013, 1029, 1043, 999, 1048, 1037, 1017, 1034, 1057, 1065, 994, 1028, 1023, 1011, 988, 1031, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1017, 1002, 1003, 1042, 1081, 1111, 996, 1077, 1039, 1018, 1051, 1056, 1092, 1026, 1004, 1020, 999, 994, 1095, 1025, 1027, 995, 1012, 1027, 999, 970, 951, 1047, 1028, 989, 1000, 1039, 1050, 1018, 1010, 997, 1006, 1028, 1020, 1005, 1033, 1028, 1050, 1063, 1022, 1014, 998, 1025, 1034, 1002, 1022, 1019, 1014, 1005, 970, 1051, 991, 1053, 1017, 991, 1057, 1040, 1051, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1021, 1022, 1060, 1054, 1014, 1033, 992, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1046, 1008, 1067, 1027, 1028, 1055, 994, 4, 3, 0, 0, 0, 0, 0, 0, 1010, 999, 1006, 1016, 1042, 1034, 1051, 1015, 8, 8, 0, 0, 0, 0, 0, 0, 1011, 1023, 1000, 1040, 1024, 1089, 1040, 1002, 27, 0, 0, 0, 0, 0, 0, 0, 1050, 1054, 999, 1002, 985, 1010, 1020, 1001, 6, 0, 0, 0, 0, 0, 0, 0, 1013, 1039, 1030, 1021, 1021, 978, 1005, 1047, 11, 0, 0, 0, 0, 0, 0, 0, 1022, 1018, 1026, 1023, 1081, 994, 977, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1027, 1036, 1056, 988, 1003, 1039, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8241, 8189, 8256, 8127, 8165, 8134, 8206] avg=8192.0000 max=8256 max_rank=3 min=8127 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1026, 1011, 985, 1039, 1003, 987, 958, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1036, 1053, 954, 1039, 1094, 996, 1039, 4, 3, 0, 0, 0, 0, 0, 0, 1029, 1000, 1019, 967, 1044, 1021, 1047, 1002, 9, 8, 0, 0, 0, 0, 0, 0, 1063, 1065, 950, 976, 1014, 966, 1016, 1042, 29, 0, 0, 0, 0, 0, 0, 0, 1073, 1059, 990, 998, 1068, 1042, 998, 1060, 6, 0, 0, 0, 0, 0, 0, 0, 1006, 1081, 990, 1006, 986, 1065, 1072, 1003, 11, 0, 0, 0, 0, 0, 0, 0, 1039, 1080, 1024, 1009, 1051, 1007, 998, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1051, 999, 1043, 982, 1046, 1034, 1045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8014, 8242, 8146, 8121, 8294, 8220, 8264, 8235] avg=8192.0000 max=8294 max_rank=4 min=8014 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=33013 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1041, 1020, 1038, 1007, 1064, 1017, 1029, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1041, 1017, 975, 973, 1014, 1035, 1046, 4, 3, 0, 0, 0, 0, 0, 0, 998, 1029, 984, 994, 992, 1015, 1013, 1031, 8, 8, 0, 0, 0, 0, 0, 0, 1008, 1028, 1023, 1053, 983, 1030, 1031, 1006, 26, 0, 0, 0, 0, 0, 0, 0, 1077, 989, 1032, 1044, 1053, 987, 1009, 1033, 6, 0, 0, 0, 0, 0, 0, 0, 1034, 1031, 1022, 1013, 1019, 998, 1067, 1032, 11, 0, 0, 0, 0, 0, 0, 0, 1030, 994, 1050, 1097, 1059, 987, 1012, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1004, 1018, 1041, 981, 1040, 1097, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8131, 8072, 8188, 8230, 8227, 8189, 8261] avg=8192.0000 max=8261 max_rank=7 min=8072 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1037, 998, 1060, 1035, 987, 948, 1057, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1014, 1003, 1035, 1047, 1023, 1031, 1038, 3, 3, 0, 0, 0, 0, 0, 0, 1086, 1005, 1074, 1066, 1016, 975, 1050, 1011, 8, 7, 0, 0, 0, 0, 0, 0, 988, 1048, 1022, 1015, 1031, 1016, 993, 1047, 27, 0, 0, 0, 0, 0, 0, 0, 1073, 1006, 950, 1026, 1032, 1011, 1022, 1026, 6, 0, 0, 0, 0, 0, 0, 0, 1034, 1029, 999, 1009, 1046, 1061, 1055, 1036, 10, 0, 0, 0, 0, 0, 0, 0, 1009, 990, 1023, 1020, 985, 1031, 1036, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1021, 1030, 1037, 998, 1033, 1014, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8215, 8298, 8187, 8152, 8279, 8066, 8221] avg=8192.0000 max=8298 max_rank=2 min=8066 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[988, 1007, 1041, 1046, 1004, 1010, 1056, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1002, 989, 1009, 994, 1010, 1020, 962, 4, 3, 0, 0, 0, 0, 0, 0, 1057, 979, 1016, 1062, 1105, 1003, 1014, 1049, 9, 8, 0, 0, 0, 0, 0, 0, 1003, 1011, 983, 1041, 1038, 996, 1094, 1075, 27, 0, 0, 0, 0, 0, 0, 0, 1054, 1036, 1037, 994, 1021, 1005, 1024, 996, 6, 0, 0, 0, 0, 0, 0, 0, 1019, 1036, 999, 1008, 1034, 1037, 1022, 1013, 11, 0, 0, 0, 0, 0, 0, 0, 1080, 1064, 1032, 1065, 1033, 1012, 981, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1001, 1036, 1049, 1065, 964, 1047, 992, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8018, 8302, 8268, 8173, 8179, 8262, 8162] avg=8192.0000 max=8302 max_rank=2 min=8018 min_rank=1 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1071, 1045, 1032, 994, 1021, 998, 1041, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 999, 1027, 1020, 1024, 1048, 1019, 985, 4, 3, 0, 0, 0, 0, 0, 0, 959, 996, 1025, 1002, 1052, 1083, 1023, 1034, 9, 8, 0, 0, 0, 0, 0, 0, 1001, 1054, 1033, 1033, 996, 1084, 998, 1036, 28, 0, 0, 0, 0, 0, 0, 0, 989, 996, 1045, 1062, 1026, 1005, 1051, 1026, 6, 0, 0, 0, 0, 0, 0, 0, 1021, 994, 1000, 1014, 997, 1034, 1019, 1013, 11, 0, 0, 0, 0, 0, 0, 0, 1029, 1043, 999, 1048, 1037, 1017, 1034, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 994, 1028, 1014, 1011, 988, 1031, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8185, 8191, 8263, 8206, 8103, 8227, 8151] avg=8192.0000 max=8263 max_rank=3 min=8103 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1069, 1013, 984, 1050, 1053, 1016, 1053, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1039, 1008, 1025, 999, 994, 1041, 1016, 4, 3, 0, 0, 0, 0, 0, 0, 987, 1021, 1027, 1027, 1010, 1019, 1010, 1022, 9, 8, 0, 0, 0, 0, 0, 0, 1028, 1003, 1017, 997, 1017, 1061, 1047, 1000, 29, 0, 0, 0, 0, 0, 0, 0, 991, 1001, 1078, 1075, 1046, 1011, 1011, 993, 6, 0, 0, 0, 0, 0, 0, 0, 1032, 987, 1005, 1018, 962, 1023, 1033, 1041, 11, 0, 0, 0, 0, 0, 0, 0, 1027, 1067, 1079, 1039, 996, 1082, 986, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1031, 1030, 1026, 1020, 1012, 990, 970, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8273, 8182, 8140, 8199, 8212, 8112, 8321, 8097] avg=8192.0000 max=8321 max_rank=6 min=8097 min_rank=7 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1017, 1002, 1003, 1042, 1081, 1075, 996, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1018, 1051, 1056, 1092, 1026, 1004, 1020, 7, 9, 0, 0, 0, 0, 0, 0, 999, 994, 1095, 1025, 1027, 995, 1012, 1027, 13, 11, 0, 0, 0, 0, 0, 0, 999, 970, 951, 1047, 1028, 989, 1000, 1039, 31, 0, 0, 0, 0, 0, 0, 0, 1050, 1018, 1010, 997, 1006, 1028, 1020, 1005, 8, 0, 0, 0, 0, 0, 0, 0, 1033, 1028, 1050, 1063, 1022, 1014, 998, 1025, 18, 0, 0, 0, 0, 0, 0, 0, 1034, 1002, 1022, 1019, 1014, 1005, 970, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1053, 1017, 974, 1057, 1040, 1051, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8322, 8198, 8054, 8142, 8251, 8073, 8203] avg=8192.0000 max=8322 max_rank=1 min=8054 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8192, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8064, 8320, 8192, 8320, 8320, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8064, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8192, 8320, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[983, 1016, 1003, 993, 1034, 956, 1044, 1025, 1033, 1043, 1066, 998, 1038, 1018, 1054, 1026, 1028, 1054, 1003, 1032, 1011, 1038, 1035, 1031, 996, 1020, 1012, 1015, 1034, 984, 1006, 1031, 1002, 1013, 1055, 1033, 1058, 1043, 1012, 1047, 1025, 1057, 1009, 1032, 1023, 1015, 1036, 1037, 1018, 1065, 1010, 1039, 1044, 999, 993, 994, 1007, 1004, 983, 1053, 1035, 1017, 1072, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1043, 1014, 1048, 1038, 1051, 985, 1028, 1025, 1002, 1084, 1058, 1030, 1015, 1020, 1050, 1005, 1004, 1002, 1016, 1064, 975, 1015, 1054, 1018, 1051, 994, 1051, 1050, 979, 1062, 1014, 1010, 1061, 1017, 1028, 990, 1019, 1037, 1008, 1006, 1026, 975, 997, 1040, 1018, 1021, 1040, 1046, 983, 1054, 995, 982, 999, 1015, 1063, 1077, 1076, 1012, 995, 997, 1047, 1089, 942, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1041, 1009, 1027, 1053, 1029, 1056, 1026, 1031, 933, 1066, 1044, 1009, 997, 1041, 1069, 998, 1056, 975, 1051, 1046, 1016, 1086, 1043, 1087, 1040, 1028, 991, 1001, 990, 998, 1005, 1028, 1011, 1070, 1008, 1026, 1011, 1055, 1007, 1017, 1009, 1043, 990, 1064, 1005, 992, 1015, 1037, 1027, 1017, 1026, 1014, 1018, 1015, 970, 1018, 993, 1035, 1037, 1063, 982, 1056, 1006, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1047, 1017, 1077, 1062, 1029, 1045, 999, 1075, 1032, 1026, 1077, 1029, 1040, 1013, 1015, 1010, 1039, 998, 1007, 992, 1024, 1062, 1017, 1058, 999, 996, 1029, 1058, 1030, 1023, 994, 1070, 1048, 1034, 986, 1017, 1017, 1042, 1016, 995, 1046, 1015, 1008, 984, 1040, 986, 1011, 1013, 1001, 979, 973, 1083, 1014, 1023, 1031, 1047, 1005, 995, 1016, 979, 983, 1033, 1050, 1077] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1051, 989, 1010, 1005, 1040, 1033, 1082, 964, 1005, 1040, 1045, 1049, 1014, 1025, 1030, 1049, 1042, 1005, 995, 1044, 1011, 1033, 1038, 1009, 1024, 1033, 1059, 1044, 991, 1054, 951, 1054, 1019, 1020, 1013, 1027, 1031, 978, 1010, 1031, 1071, 979, 972, 1012, 1022, 1023, 1097, 946, 1052, 1080, 1004, 1034, 992, 1011, 1062, 1026, 1037, 1015, 1016, 1015, 1054, 1030, 1002, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1056, 1063, 1013, 1044, 1022, 1043, 1047, 1005, 1043, 975, 1067, 1005, 1038, 1035, 971, 989, 1018, 1086, 1036, 1091, 1017, 1027, 1015, 1042, 1009, 1020, 978, 996, 1011, 998, 1004, 992, 1075, 1056, 1021, 1020, 1012, 992, 1011, 1048, 1032, 965, 1004, 968, 1021, 1056, 1039, 1042, 1002, 1079, 1013, 1035, 1049, 1011, 1041, 1034, 1075, 1007, 979, 978, 1002, 1081, 1027, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1080, 1002, 1029, 1035, 985, 993, 1020, 1028, 1031, 986, 1063, 1063, 1025, 1111, 1068, 1019, 986, 1037, 998, 1025, 1000, 1053, 1027, 1021, 992, 1004, 1002, 1039, 1050, 998, 1024, 1022, 963, 1048, 1044, 1003, 1008, 999, 1060, 1088, 1043, 1047, 992, 1068, 1023, 1034, 1011, 1040, 1015, 1016, 1041, 1003, 967, 1010, 998, 1028, 987, 1026, 974, 1023, 1045, 1020, 1036, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1000, 1020, 996, 1012, 1035, 1001, 1012, 1039, 1072, 995, 1017, 1021, 1025, 1081, 1023, 1040, 1000, 1049, 1003, 1034, 1054, 1022, 1055, 979, 1004, 1076, 1000, 999, 1048, 1008, 983, 1027, 998, 1022, 1049, 988, 1016, 987, 993, 1044, 1069, 995, 1027, 989, 1018, 1014, 1009, 1026, 1042, 1028, 1065, 985, 1045, 1065, 1040, 966, 1039, 1001, 1011, 1053, 1013, 1059, 1090, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[969, 1016, 1003, 993, 1034, 956, 1044, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1043, 1013, 998, 1038, 1018, 1054, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1054, 1003, 1032, 1011, 1008, 1035, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1020, 1012, 1015, 1034, 984, 1006, 1031, 53, 3, 0, 0, 0, 0, 0, 0, 1002, 1013, 1055, 1033, 1058, 1043, 1012, 1047, 7, 1, 0, 0, 0, 0, 0, 0, 1025, 1057, 1009, 1032, 1023, 1015, 1036, 1037, 27, 0, 0, 0, 0, 0, 0, 0, 1018, 1065, 1010, 1039, 1044, 999, 993, 994, 13, 0, 0, 0, 0, 0, 0, 0, 1007, 1004, 983, 1053, 1035, 1010, 1072, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8040, 8223, 8202, 8154, 8271, 8261, 8175, 8210] avg=8192.0000 max=8271 max_rank=4 min=8040 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1029, 1014, 1048, 1038, 1051, 985, 1028, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1084, 1005, 1030, 1015, 1020, 1050, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1002, 1016, 1064, 975, 985, 1054, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 994, 1051, 1050, 979, 1062, 1014, 1010, 53, 3, 0, 0, 0, 0, 0, 0, 1061, 1017, 1028, 990, 1019, 1037, 1008, 1006, 7, 1, 0, 0, 0, 0, 0, 0, 1026, 975, 997, 1040, 1018, 1021, 1040, 1046, 27, 0, 0, 0, 0, 0, 0, 0, 983, 1054, 995, 982, 999, 1015, 1063, 1077, 13, 0, 0, 0, 0, 0, 0, 0, 1076, 1012, 995, 997, 1047, 1082, 942, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8211, 8118, 8267, 8174, 8190, 8181, 8177] avg=8192.0000 max=8267 max_rank=3 min=8118 min_rank=2 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1041, 1063, 1013, 1044, 1022, 1043, 1047, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 975, 1014, 1005, 1038, 1035, 971, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1086, 1036, 1091, 1017, 997, 1015, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1020, 978, 996, 1011, 998, 1004, 992, 53, 3, 0, 0, 0, 0, 0, 0, 1075, 1056, 1021, 1020, 1012, 992, 1011, 1048, 7, 1, 0, 0, 0, 0, 0, 0, 1032, 965, 1004, 968, 1021, 1056, 1039, 1042, 27, 0, 0, 0, 0, 0, 0, 0, 1002, 1079, 1013, 1035, 1049, 1011, 1041, 1034, 14, 0, 0, 0, 0, 0, 0, 0, 1075, 1007, 979, 978, 1002, 1074, 1027, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 8070, 8302, 8064, 8243, 8154, 8278, 8147] avg=8192.0000 max=8302 max_rank=2 min=8064 min_rank=3 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1065, 1002, 1029, 1035, 985, 993, 1020, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 986, 1010, 1063, 1025, 1111, 1068, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1037, 998, 1025, 1000, 1022, 1027, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1004, 1002, 1039, 1050, 998, 1024, 1022, 53, 3, 0, 0, 0, 0, 0, 0, 963, 1048, 1044, 1003, 1008, 999, 1060, 1088, 7, 1, 0, 0, 0, 0, 0, 0, 1043, 1047, 992, 1068, 1023, 1034, 1011, 1040, 28, 0, 0, 0, 0, 0, 0, 0, 1015, 1016, 1041, 1003, 967, 1010, 998, 1028, 14, 0, 0, 0, 0, 0, 0, 0, 987, 1026, 974, 1023, 1045, 1013, 1036, 1060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8313, 8116, 8187, 8221, 8286, 8092, 8164] avg=8192.0000 max=8313 max_rank=1 min=8092 min_rank=6 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1027, 1009, 1027, 1053, 1029, 1056, 1026, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 933, 1066, 992, 1009, 997, 1041, 1069, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 975, 1051, 1046, 1016, 1054, 1043, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1028, 991, 1001, 990, 998, 1005, 1028, 52, 3, 0, 0, 0, 0, 0, 0, 1011, 1070, 1008, 1026, 1011, 1055, 1007, 1017, 7, 1, 0, 0, 0, 0, 0, 0, 1009, 1043, 990, 1064, 1005, 992, 1015, 1037, 29, 0, 0, 0, 0, 0, 0, 0, 1027, 1017, 1026, 1014, 1018, 1015, 970, 1018, 13, 0, 0, 0, 0, 0, 0, 0, 993, 1035, 1037, 1063, 982, 1049, 1006, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8105, 8328, 8136, 8213, 8184, 8118, 8194] avg=8192.0000 max=8328 max_rank=2 min=8105 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1032, 1017, 1077, 1062, 1029, 1045, 999, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1026, 1024, 1029, 1040, 1013, 1015, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 998, 1007, 992, 1024, 1031, 1017, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 999, 996, 1029, 1058, 1030, 1023, 994, 1070, 53, 3, 0, 0, 0, 0, 0, 0, 1048, 1034, 986, 1017, 1017, 1042, 1016, 995, 7, 1, 0, 0, 0, 0, 0, 0, 1046, 1015, 1008, 984, 1040, 986, 1011, 1013, 28, 0, 0, 0, 0, 0, 0, 0, 1001, 979, 973, 1083, 1014, 1023, 1031, 1047, 14, 0, 0, 0, 0, 0, 0, 0, 1005, 995, 1016, 979, 983, 1026, 1050, 1077, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8336, 8189, 8166, 8255, 8163, 8131, 8165, 8131] avg=8192.0000 max=8336 max_rank=0 min=8131 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32590 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1036, 989, 1010, 1005, 1040, 1033, 1082, 964, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1040, 993, 1049, 1014, 1025, 1030, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1005, 995, 1044, 1011, 1003, 1038, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1033, 1059, 1044, 991, 1054, 951, 1054, 52, 3, 0, 0, 0, 0, 0, 0, 1019, 1020, 1013, 1027, 1031, 978, 1010, 1031, 7, 1, 0, 0, 0, 0, 0, 0, 1071, 979, 972, 1012, 1022, 1023, 1097, 946, 27, 0, 0, 0, 0, 0, 0, 0, 1052, 1080, 1004, 1034, 992, 1011, 1062, 1026, 14, 0, 0, 0, 0, 0, 0, 0, 1037, 1015, 1016, 1015, 1054, 1023, 1002, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8205, 8147, 8265, 8137, 8149, 8275, 8199] avg=8192.0000 max=8275 max_rank=6 min=8137 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[975, 1020, 996, 1012, 1035, 1001, 1012, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 995, 963, 1021, 1025, 1081, 1023, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1049, 1003, 1034, 1054, 983, 1055, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1076, 1000, 999, 1048, 1008, 983, 1027, 54, 9, 0, 0, 0, 0, 0, 0, 998, 1022, 1049, 988, 1016, 987, 993, 1044, 12, 5, 0, 0, 0, 0, 0, 0, 1069, 995, 1027, 989, 1018, 1014, 1009, 1026, 30, 4, 0, 0, 0, 0, 0, 0, 1042, 1028, 1065, 985, 1045, 1065, 1040, 966, 16, 0, 0, 0, 0, 0, 0, 0, 1039, 1001, 1011, 1053, 1013, 1047, 1090, 1060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8090, 8220, 8157, 8208, 8114, 8181, 8252, 8314] avg=8192.0000 max=8314 max_rank=7 min=8090 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32861 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8320, 8192, 8448, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8448, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8064, 8320, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8064, 8192, 8192, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8064, 8192, 8192, 8448, 8320, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[982, 1016, 1010, 1016, 1029, 997, 1046, 1030, 1020, 1019, 1047, 1021, 983, 1027, 991, 1024, 1030, 1022, 1036, 1002, 1041, 1044, 994, 1029, 1022, 1033, 1009, 1062, 1005, 1027, 1062, 1029, 983, 1042, 1039, 1074, 1041, 1018, 996, 1046, 1027, 1065, 1054, 1022, 1009, 1061, 1032, 1003, 1023, 1028, 1050, 1010, 1013, 1071, 1007, 987, 1004, 995, 1019, 1005, 1013, 1040, 1012, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1046, 984, 1026, 1032, 1007, 1027, 1020, 1039, 1046, 1021, 1034, 995, 996, 1049, 1069, 1025, 1022, 1005, 1056, 1010, 1047, 1023, 1039, 1000, 1048, 1005, 983, 980, 982, 1017, 1026, 1030, 978, 1034, 1041, 1068, 1011, 1010, 1023, 1041, 976, 1049, 1028, 1052, 1012, 1041, 1013, 1017, 1016, 999, 1046, 1062, 1043, 1051, 988, 1054, 1038, 1058, 989, 1050, 993, 1019, 1035, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[991, 999, 1028, 980, 1025, 1045, 1036, 1021, 1021, 971, 1046, 1018, 1091, 1004, 1034, 1004, 1012, 1011, 1015, 1050, 1017, 983, 1022, 1022, 1044, 1041, 1019, 1082, 1033, 1035, 996, 1048, 1029, 1025, 1009, 1035, 1016, 1044, 1003, 1020, 1029, 1070, 1009, 1035, 1012, 1045, 994, 1037, 1057, 997, 1009, 1057, 1010, 1066, 1022, 991, 1042, 1032, 1024, 949, 1053, 990, 975, 1106] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1033, 990, 1064, 1026, 996, 1045, 970, 1048, 1007, 1034, 1032, 1017, 1043, 999, 1054, 1039, 1078, 1030, 956, 1041, 1020, 1029, 1091, 1054, 993, 1010, 1042, 987, 1045, 1036, 1018, 963, 991, 1032, 1036, 987, 1043, 1023, 953, 1073, 1032, 1020, 1017, 1048, 1047, 1021, 1012, 1002, 1016, 1050, 1054, 1003, 1015, 1009, 992, 1003, 1006, 1047, 1010, 1011, 1050, 1046, 1065, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1036, 991, 1012, 1002, 1027, 1041, 1038, 1033, 1023, 999, 996, 984, 1052, 1031, 1044, 1013, 1000, 1042, 981, 986, 1021, 1037, 1059, 1009, 1044, 1131, 1033, 1033, 1000, 1078, 1045, 1046, 990, 992, 1052, 1006, 988, 1049, 1030, 1039, 1068, 1018, 1003, 1033, 1064, 1025, 1035, 958, 1025, 1055, 978, 976, 996, 1037, 1039, 998, 1050, 1028, 998, 1051, 1002, 1038, 1010, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1005, 1007, 987, 1058, 1042, 991, 1007, 1023, 1023, 989, 988, 1034, 1049, 1006, 1011, 1018, 990, 1048, 1049, 997, 1019, 1012, 1071, 1034, 1016, 1010, 1000, 1024, 1007, 992, 1014, 1038, 999, 1052, 989, 1015, 1035, 1032, 1006, 1049, 1037, 1050, 980, 1017, 1022, 1052, 1044, 1086, 1054, 1076, 1014, 1023, 1059, 1058, 1003, 1047, 1018, 967, 1010, 1053, 1077, 1015, 1037, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1063, 1079, 1085, 1042, 988, 1015, 969, 1025, 994, 1017, 1013, 1033, 975, 1021, 1008, 1025, 1029, 1035, 1044, 1038, 1036, 1005, 1025, 1041, 1023, 1003, 1045, 1016, 1024, 1049, 1041, 1031, 1025, 1021, 1081, 997, 1016, 1031, 1033, 991, 1015, 1045, 1018, 1025, 1079, 1020, 985, 1038, 1028, 989, 993, 1021, 1038, 1062, 1061, 1007, 1041, 972, 1010, 1052, 1018, 996, 1006, 985] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1028, 1056, 1067, 986, 1056, 975, 1007, 1060, 1015, 1036, 980, 1025, 1073, 1069, 1028, 1022, 1029, 1053, 1006, 1001, 996, 1013, 1022, 992, 990, 1025, 999, 982, 1038, 1016, 1010, 982, 997, 1005, 1079, 1014, 1080, 1044, 1038, 1013, 1023, 990, 1042, 1040, 1003, 1038, 1055, 1018, 1038, 1024, 1069, 979, 1002, 1065, 1065, 1043, 1019, 988, 1000, 1008, 1040, 1028, 1049, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1046, 984, 1026, 1032, 1007, 1027, 1020, 1039, 16, 0, 0, 0, 0, 0, 0, 0, 1046, 1021, 1034, 995, 996, 1049, 1069, 1025, 14, 5, 0, 0, 0, 0, 0, 0, 1022, 1005, 1056, 1010, 1047, 1023, 1038, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1005, 983, 980, 982, 1017, 1026, 1030, 2, 1, 0, 0, 0, 0, 0, 0, 978, 1034, 1039, 1068, 1011, 1010, 1023, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1015, 1028, 1052, 1012, 1041, 1013, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 999, 1046, 1062, 1043, 1030, 988, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1058, 989, 1050, 993, 1019, 1035, 1012, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8254, 8201, 8074, 8204, 8154, 8238, 8214] avg=8192.0000 max=8254 max_rank=1 min=8074 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32810 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[982, 1016, 1010, 1016, 1029, 997, 1046, 1030, 16, 0, 0, 0, 0, 0, 0, 0, 1020, 1019, 1047, 1021, 983, 1027, 991, 1024, 14, 5, 0, 0, 0, 0, 0, 0, 1030, 1022, 1036, 1002, 1041, 1044, 993, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1033, 1009, 1062, 1005, 1027, 1062, 1029, 1, 1, 1, 0, 0, 0, 0, 0, 983, 1042, 1038, 1074, 1041, 1018, 996, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1030, 1054, 1022, 1009, 1061, 1032, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1028, 1050, 1010, 1013, 1049, 1007, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 995, 1019, 1005, 1013, 1040, 1012, 1042, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8151, 8197, 8252, 8238, 8238, 8167, 8151] avg=8192.0000 max=8252 max_rank=3 min=8142 min_rank=0 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32794 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1033, 990, 1064, 1026, 996, 1045, 970, 1048, 15, 0, 0, 0, 0, 0, 0, 0, 1007, 1034, 1032, 1017, 1043, 999, 1054, 1039, 14, 5, 0, 0, 0, 0, 0, 0, 1078, 1030, 956, 1041, 1020, 1029, 1090, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1010, 1042, 987, 1045, 1036, 1018, 963, 1, 1, 0, 0, 0, 0, 0, 0, 991, 1032, 1035, 987, 1043, 1023, 953, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 986, 1017, 1048, 1047, 1021, 1012, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1050, 1054, 1003, 1015, 989, 992, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1047, 1010, 1011, 1050, 1046, 1065, 1032, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8244, 8298, 8096, 8137, 8165, 8122, 8287] avg=8192.0000 max=8298 max_rank=2 min=8096 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[991, 999, 1028, 980, 1025, 1045, 1036, 1021, 16, 0, 0, 0, 0, 0, 0, 0, 1021, 971, 1046, 1018, 1091, 1004, 1034, 1004, 14, 5, 0, 0, 0, 0, 0, 0, 1012, 1011, 1015, 1050, 1017, 983, 1021, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1041, 1019, 1082, 1033, 1035, 996, 1048, 1, 1, 1, 0, 0, 0, 0, 0, 1029, 1025, 1008, 1035, 1016, 1044, 1003, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1035, 1009, 1035, 1012, 1045, 994, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 997, 1009, 1057, 1010, 1044, 1022, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1032, 1024, 949, 1053, 990, 975, 1106, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8208, 8131, 8301, 8180, 8196, 8187, 8192] avg=8192.0000 max=8301 max_rank=3 min=8131 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1036, 991, 1012, 1002, 1027, 1041, 1038, 1033, 16, 0, 0, 0, 0, 0, 0, 0, 1023, 999, 996, 984, 1052, 1031, 1044, 1013, 14, 5, 0, 0, 0, 0, 0, 0, 1000, 1042, 981, 986, 1021, 1037, 1058, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1131, 1033, 1033, 1000, 1078, 1045, 1046, 2, 1, 0, 0, 0, 0, 0, 0, 990, 992, 1050, 1006, 988, 1049, 1030, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 984, 1003, 1033, 1064, 1025, 1035, 958, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1055, 978, 976, 996, 1016, 1039, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1028, 998, 1051, 1002, 1038, 1010, 1038, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8161, 8134, 8413, 8144, 8170, 8083, 8235] avg=8192.0000 max=8413 max_rank=3 min=8083 min_rank=6 imbalance_ratio=1.026978 max_load_reduction=57123 max_load_reduction_pct=0.871628 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1005, 1007, 987, 1058, 1042, 991, 1007, 1023, 16, 0, 0, 0, 0, 0, 0, 0, 1023, 989, 988, 1034, 1049, 1006, 1011, 1018, 14, 5, 0, 0, 0, 0, 0, 0, 990, 1048, 1049, 997, 1019, 1012, 1070, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1010, 1000, 1024, 1007, 992, 1014, 1038, 1, 1, 1, 0, 0, 0, 0, 0, 999, 1052, 988, 1015, 1035, 1032, 1006, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1016, 980, 1017, 1022, 1052, 1044, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1076, 1014, 1023, 1059, 1036, 1003, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 967, 1010, 1053, 1077, 1015, 1037, 1001, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8137, 8219, 8104, 8176, 8254, 8312, 8198] avg=8192.0000 max=8312 max_rank=6 min=8104 min_rank=3 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32940 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1063, 1079, 1085, 1042, 988, 1015, 969, 1025, 16, 0, 0, 0, 0, 0, 0, 0, 994, 1017, 1013, 1033, 975, 1021, 1008, 1025, 14, 5, 0, 0, 0, 0, 0, 0, 1029, 1035, 1044, 1038, 1036, 1005, 1024, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1003, 1045, 1016, 1024, 1049, 1041, 1031, 2, 1, 1, 0, 0, 0, 0, 0, 1025, 1021, 1079, 997, 1016, 1031, 1033, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1011, 1018, 1025, 1079, 1020, 985, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 989, 993, 1021, 1038, 1040, 1061, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 972, 1010, 1052, 1018, 996, 1006, 985, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8105, 8252, 8236, 8193, 8191, 8177, 8100] avg=8192.0000 max=8282 max_rank=0 min=8100 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1028, 1056, 1067, 986, 1056, 975, 1007, 1060, 20, 0, 0, 0, 0, 0, 0, 0, 1015, 1036, 980, 1025, 1073, 1069, 1028, 1022, 17, 11, 0, 0, 0, 0, 0, 0, 1029, 1053, 1006, 1001, 996, 1013, 1014, 992, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1025, 999, 982, 1038, 1016, 1010, 982, 6, 8, 4, 0, 0, 0, 0, 0, 997, 1005, 1073, 1014, 1080, 1044, 1038, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 949, 1042, 1040, 1003, 1038, 1055, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1024, 1069, 979, 1002, 1030, 1065, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 988, 1000, 1008, 1040, 1028, 1049, 1003, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8276, 8104, 8060, 8264, 8168, 8250, 8159] avg=8192.0000 max=8276 max_rank=1 min=8060 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8192, 8320, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8320, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1017, 1047, 1064, 1025, 1001, 982, 1038, 1023, 975, 1063, 1016, 1019, 1014, 954, 1050, 966, 984, 1039, 1016, 1033, 1063, 1024, 983, 972, 1035, 1022, 1056, 1065, 1022, 985, 1083, 1023, 1013, 1019, 1024, 1021, 1060, 1036, 983, 1009, 1040, 993, 1031, 988, 1030, 961, 1023, 1046, 1032, 1020, 1017, 1041, 1080, 1036, 1043, 1068, 1038, 1054, 1003, 1042, 1031, 1010, 1025, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1014, 1082, 1012, 1025, 1007, 1060, 1042, 1026, 1074, 987, 1007, 1050, 1006, 1036, 1023, 991, 1007, 960, 1035, 1017, 1037, 982, 1058, 1033, 1008, 1014, 1034, 1032, 1057, 980, 1061, 984, 981, 1001, 1005, 982, 1047, 998, 975, 1013, 995, 1055, 1051, 1001, 1029, 1064, 1024, 1037, 1068, 1054, 1033, 1018, 1061, 1017, 1029, 1024, 1030, 1035, 1056, 1021, 1024, 1036, 1010, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1043, 1055, 1040, 1047, 1048, 1011, 1051, 1060, 1020, 976, 1029, 1034, 994, 1022, 1058, 1020, 1008, 978, 1061, 1030, 1009, 1035, 998, 1009, 978, 999, 993, 1040, 1029, 1031, 1040, 999, 1049, 1057, 1003, 1040, 1013, 1017, 1033, 993, 1021, 1035, 1027, 1110, 1027, 1025, 1040, 1004, 969, 1027, 974, 920, 1025, 986, 1064, 1030, 1030, 1047, 1042, 1080, 1033, 1019, 1017, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1024, 990, 1052, 1046, 1027, 1012, 1007, 1004, 1031, 993, 1010, 996, 1028, 982, 1050, 1052, 1018, 1023, 1017, 1043, 1092, 995, 1063, 1083, 1026, 1013, 1027, 1062, 1053, 996, 1026, 968, 1033, 1024, 999, 1021, 1040, 1048, 973, 1035, 964, 1052, 1043, 1006, 999, 1075, 1094, 1048, 978, 1023, 1021, 999, 1025, 1035, 1029, 1004, 977, 1064, 993, 1027, 1014, 1040, 1011, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1048, 1031, 1025, 1001, 1043, 1020, 1073, 1018, 1005, 1022, 1017, 986, 1015, 1066, 1005, 1003, 977, 981, 1054, 1035, 1051, 996, 992, 998, 1070, 1025, 1008, 1015, 1050, 1024, 1048, 995, 1044, 1048, 1007, 1035, 1065, 1030, 1037, 1084, 1082, 1003, 985, 1053, 1033, 1008, 994, 1054, 1080, 984, 1030, 1013, 953, 1048, 1020, 1003, 1054, 1041, 967, 1059, 1007, 1003, 1007, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1004, 1043, 1053, 1040, 1056, 996, 995, 1013, 1041, 1096, 990, 1046, 1051, 979, 962, 1011, 1052, 1014, 1060, 1034, 1006, 1028, 973, 1034, 1030, 1061, 1010, 1057, 1091, 1034, 1001, 980, 1001, 1008, 1018, 1007, 1054, 1044, 1075, 991, 995, 1004, 1012, 1054, 977, 1018, 1012, 1044, 1039, 1075, 1044, 989, 1054, 1052, 1005, 1000, 1027, 1049, 1005, 1019, 1009, 1018, 983, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1027, 1071, 1046, 1003, 1097, 990, 1015, 1039, 1066, 1029, 1073, 1040, 1039, 1053, 1004, 1024, 1050, 1008, 1028, 1079, 1006, 1061, 1021, 994, 1037, 1036, 1007, 1012, 1035, 981, 968, 1027, 1037, 1052, 1059, 1023, 1008, 1035, 1004, 1015, 1051, 1025, 1031, 1023, 1065, 990, 1013, 968, 1005, 976, 999, 997, 1051, 1017, 1029, 1014, 978, 983, 1050, 989, 1042, 1004, 994, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1062, 1008, 1042, 1040, 996, 992, 1050, 982, 1015, 1041, 1046, 1009, 1093, 967, 1009, 1004, 1072, 964, 1051, 1063, 1008, 1017, 1011, 1013, 1065, 1039, 1030, 1038, 1061, 1019, 1023, 1052, 1025, 1002, 1016, 1033, 1022, 1006, 1014, 1042, 1046, 996, 1029, 992, 988, 1005, 1004, 1046, 997, 1008, 1027, 1040, 1025, 1049, 1023, 1026, 1024, 1006, 983, 1071, 1013, 1026, 1047, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1043, 1055, 996, 1047, 1048, 1011, 1051, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 976, 1029, 1034, 994, 1022, 1058, 1020, 25, 0, 0, 0, 0, 0, 0, 0, 1008, 978, 1061, 1030, 1009, 1035, 998, 1009, 6, 5, 0, 0, 0, 0, 0, 0, 978, 999, 993, 1040, 1014, 1031, 1040, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1057, 1003, 1040, 1013, 1017, 1033, 993, 6, 0, 0, 0, 0, 0, 0, 0, 1021, 1035, 1027, 1110, 1027, 1025, 1040, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1027, 974, 920, 1025, 986, 1064, 1030, 14, 0, 0, 0, 0, 0, 0, 0, 1030, 1047, 1042, 1080, 1033, 1019, 1017, 1034, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8178, 8139, 8094, 8211, 8289, 8009, 8305] avg=8192.0000 max=8311 max_rank=0 min=8009 min_rank=6 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1017, 1047, 1019, 1025, 1001, 982, 1038, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1063, 1016, 1019, 1014, 954, 1050, 966, 25, 0, 0, 0, 0, 0, 0, 0, 984, 1039, 1016, 1033, 1063, 1024, 983, 972, 6, 6, 0, 0, 0, 0, 0, 0, 1035, 1022, 1056, 1065, 1007, 985, 1083, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1019, 1024, 1021, 1060, 1036, 983, 1009, 6, 0, 0, 0, 0, 0, 0, 0, 1040, 993, 1031, 988, 1030, 961, 1023, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1020, 1017, 1041, 1080, 1036, 1043, 1068, 14, 0, 0, 0, 0, 0, 0, 0, 1038, 1054, 1003, 1042, 1031, 1010, 1025, 1060, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8082, 8126, 8276, 8171, 8112, 8351, 8266] avg=8192.0000 max=8351 max_rank=6 min=8082 min_rank=1 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32900 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1024, 990, 1007, 1046, 1027, 1012, 1007, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 993, 1010, 996, 1028, 982, 1050, 1052, 25, 0, 0, 0, 0, 0, 0, 0, 1018, 1023, 1017, 1043, 1092, 995, 1063, 1083, 6, 6, 0, 0, 0, 0, 0, 0, 1026, 1013, 1027, 1062, 1037, 996, 1026, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1024, 999, 1021, 1040, 1048, 973, 1035, 6, 0, 0, 0, 0, 0, 0, 0, 964, 1052, 1043, 1006, 999, 1075, 1094, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1023, 1021, 999, 1025, 1035, 1029, 1004, 14, 0, 0, 0, 0, 0, 0, 0, 977, 1064, 993, 1027, 1014, 1040, 1011, 1033, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8167, 8346, 8155, 8179, 8281, 8128, 8163] avg=8192.0000 max=8346 max_rank=2 min=8117 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1014, 1082, 970, 1025, 1007, 1060, 1042, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 987, 1007, 1050, 1006, 1036, 1023, 991, 24, 0, 0, 0, 0, 0, 0, 0, 1007, 960, 1035, 1017, 1037, 982, 1058, 1033, 6, 5, 0, 0, 0, 0, 0, 0, 1008, 1014, 1034, 1032, 1041, 980, 1061, 984, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1001, 1005, 982, 1047, 998, 975, 1013, 6, 0, 0, 0, 0, 0, 0, 0, 995, 1055, 1051, 1001, 1029, 1064, 1024, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1054, 1033, 1018, 1061, 1017, 1029, 1024, 13, 0, 0, 0, 0, 0, 0, 0, 1030, 1035, 1056, 1021, 1024, 1036, 1010, 1021, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8198, 8140, 8154, 8008, 8256, 8317, 8237] avg=8192.0000 max=8317 max_rank=6 min=8008 min_rank=4 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1027, 1071, 1001, 1003, 1097, 990, 1015, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1029, 1073, 1040, 1039, 1053, 1004, 1024, 25, 0, 0, 0, 0, 0, 0, 0, 1050, 1008, 1028, 1079, 1006, 1061, 1021, 994, 6, 6, 0, 0, 0, 0, 0, 0, 1037, 1036, 1007, 1012, 1020, 981, 968, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1052, 1059, 1023, 1008, 1035, 1004, 1015, 6, 0, 0, 0, 0, 0, 0, 0, 1051, 1025, 1031, 1023, 1065, 990, 1013, 968, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 976, 999, 997, 1051, 1017, 1029, 1014, 14, 0, 0, 0, 0, 0, 0, 0, 978, 983, 1050, 989, 1042, 1004, 994, 1043, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8353, 8259, 8088, 8239, 8166, 8102, 8086] avg=8192.0000 max=8353 max_rank=1 min=8086 min_rank=7 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32593 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1048, 1031, 982, 1001, 1043, 1020, 1073, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1022, 1017, 986, 1015, 1066, 1005, 1003, 24, 0, 0, 0, 0, 0, 0, 0, 977, 981, 1054, 1035, 1051, 996, 992, 998, 6, 5, 0, 0, 0, 0, 0, 0, 1070, 1025, 1008, 1015, 1034, 1024, 1048, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1048, 1007, 1035, 1065, 1030, 1037, 1084, 6, 0, 0, 0, 0, 0, 0, 0, 1082, 1003, 985, 1053, 1033, 1008, 994, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 984, 1030, 1013, 953, 1048, 1020, 1003, 14, 0, 0, 0, 0, 0, 0, 0, 1054, 1041, 967, 1059, 1007, 1003, 1007, 1008, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8143, 8095, 8219, 8356, 8212, 8145, 8150] avg=8192.0000 max=8356 max_rank=4 min=8095 min_rank=2 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32863 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1004, 1043, 1008, 1040, 1056, 996, 995, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1096, 990, 1046, 1051, 979, 962, 1011, 25, 0, 0, 0, 0, 0, 0, 0, 1052, 1014, 1060, 1034, 1006, 1028, 973, 1034, 6, 6, 0, 0, 0, 0, 0, 0, 1030, 1061, 1010, 1057, 1075, 1034, 1001, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1008, 1018, 1007, 1054, 1044, 1075, 991, 6, 0, 0, 0, 0, 0, 0, 0, 995, 1004, 1012, 1054, 977, 1018, 1012, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1075, 1044, 989, 1054, 1052, 1005, 1000, 14, 0, 0, 0, 0, 0, 0, 0, 1027, 1049, 1005, 1019, 1009, 1018, 983, 1013, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8201, 8213, 8248, 8204, 8116, 8272, 8127] avg=8192.0000 max=8272 max_rank=6 min=8116 min_rank=5 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1062, 1008, 986, 1040, 996, 992, 1050, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1041, 1046, 1009, 1093, 967, 1009, 1004, 30, 0, 0, 0, 0, 0, 0, 0, 1072, 964, 1051, 1063, 1008, 1017, 1011, 1013, 10, 9, 0, 0, 0, 0, 0, 0, 1065, 1039, 1030, 1038, 1036, 1019, 1023, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1002, 1016, 1033, 1022, 1006, 1014, 1042, 8, 0, 0, 0, 0, 0, 0, 0, 1046, 996, 1029, 992, 988, 1005, 1004, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1008, 1027, 1040, 1025, 1049, 1023, 1026, 17, 0, 0, 0, 0, 0, 0, 0, 1024, 1006, 983, 1071, 1013, 1026, 1047, 1023, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8214, 8218, 8302, 8168, 8104, 8212, 8202] avg=8192.0000 max=8302 max_rank=3 min=8104 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8064, 8448, 8448, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8064, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8448, 8064, 8320, 8320, 8064, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8448, 8320, 8064, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8064, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1039, 1005, 1032, 1026, 1055, 1013, 1067, 976, 1031, 996, 985, 995, 1049, 1028, 1076, 1038, 989, 1031, 1033, 1039, 1040, 961, 1067, 1015, 1046, 1014, 1101, 1032, 995, 1026, 1001, 1083, 1007, 1034, 1001, 1023, 1050, 1020, 1025, 1031, 973, 1007, 1003, 1068, 997, 1004, 1040, 1005, 1059, 973, 1016, 1057, 1026, 1048, 1015, 1026, 1064, 995, 996, 1027, 1033, 1007, 1039, 983] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1066, 1033, 1008, 1059, 1015, 995, 1033, 1040, 1040, 1013, 1047, 1032, 984, 1042, 1052, 1018, 992, 1010, 1049, 1022, 1032, 1018, 988, 1011, 1042, 1079, 1034, 983, 1020, 1045, 1072, 1034, 958, 1010, 1038, 1031, 1018, 975, 994, 1029, 1039, 985, 1051, 1026, 976, 1007, 1067, 1001, 1017, 1052, 989, 1074, 1090, 1018, 1062, 1028, 1058, 1054, 990, 1016, 1034, 954, 973, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1027, 1021, 1046, 1027, 1041, 1010, 1052, 985, 1030, 1011, 1008, 1031, 977, 1044, 1045, 988, 1016, 1010, 1006, 989, 1003, 1047, 1028, 1033, 1032, 1027, 1065, 1027, 1054, 1013, 1015, 955, 1044, 977, 995, 1042, 1015, 1021, 1105, 1048, 1066, 1007, 973, 1057, 1015, 1007, 1032, 999, 1084, 1045, 1009, 1058, 1059, 1068, 1043, 962, 1015, 1023, 992, 1012, 1039, 1011, 1008, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1050, 1059, 1013, 951, 994, 1034, 1062, 1019, 1025, 980, 984, 1079, 1013, 998, 997, 985, 1010, 995, 1042, 1022, 1043, 1048, 1031, 1012, 1008, 1029, 1026, 1026, 986, 1090, 995, 990, 1026, 1047, 1103, 1033, 1038, 1029, 1031, 983, 1071, 1028, 1037, 1035, 1012, 1047, 1028, 1023, 988, 1057, 1017, 1036, 1007, 1046, 1045, 1010, 998, 1036, 1012, 1015, 1027, 1015, 1012, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[962, 1017, 937, 1030, 1000, 1008, 1020, 1043, 1055, 1050, 1040, 1065, 1027, 1048, 1059, 1017, 984, 982, 988, 1054, 1000, 994, 1015, 1000, 1013, 1007, 1001, 987, 1053, 1090, 1037, 1037, 1045, 999, 1037, 1017, 1049, 1091, 1039, 1030, 1019, 1062, 1040, 1026, 1081, 1059, 973, 1033, 1031, 1026, 974, 988, 956, 1049, 1047, 987, 1014, 1055, 1056, 985, 1034, 1028, 1002, 1084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1014, 1006, 1021, 1078, 1010, 1002, 1041, 1067, 1061, 1027, 1049, 1013, 1021, 1053, 1042, 1021, 1007, 997, 1005, 1060, 1016, 981, 1043, 1014, 1014, 1030, 1018, 1064, 1040, 1069, 1029, 998, 1003, 1087, 964, 1047, 1043, 1002, 978, 1088, 1027, 1040, 1042, 1001, 974, 1022, 1021, 1040, 1030, 993, 1002, 1039, 978, 1050, 1008, 1026, 1008, 967, 1028, 1021, 1032, 1005, 1024, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1029, 1002, 1053, 1032, 1059, 1056, 1059, 1013, 1049, 995, 1060, 1034, 1034, 1021, 1038, 1036, 1008, 1057, 1033, 1030, 984, 985, 998, 1020, 1056, 1012, 1039, 1047, 1019, 995, 1035, 1033, 1057, 1021, 1032, 1035, 990, 1033, 1053, 1002, 1023, 958, 1076, 1007, 958, 1043, 1024, 1063, 1011, 1016, 1018, 1045, 957, 1021, 1009, 1025, 997, 1006, 1057, 995, 1002, 1038, 1016, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1000, 1051, 1018, 1007, 1034, 1060, 1002, 1039, 1035, 1056, 1007, 1056, 1028, 1068, 1048, 1032, 1039, 1052, 1044, 984, 992, 1057, 990, 1013, 1006, 1016, 1011, 1019, 990, 1063, 1077, 1031, 1039, 981, 1043, 1023, 1035, 1033, 1026, 993, 1035, 991, 972, 1004, 983, 1002, 1021, 985, 987, 1037, 1065, 1054, 1053, 1013, 993, 959, 1003, 1038, 1053, 1022, 1042, 1037, 1045, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1039, 1005, 1032, 1026, 1055, 1013, 1057, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 996, 985, 995, 1049, 1028, 1034, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1031, 1033, 1039, 1040, 961, 1067, 1015, 42, 17, 0, 0, 0, 0, 0, 0, 1046, 1014, 1101, 1032, 995, 984, 1001, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1034, 1001, 1023, 1050, 1020, 1007, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1007, 1003, 1068, 997, 1004, 1040, 1005, 25, 5, 0, 0, 0, 0, 0, 0, 1059, 973, 1016, 1057, 1026, 1048, 1015, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 995, 996, 1027, 1033, 1007, 1039, 983, 13, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8156, 8234, 8256, 8173, 8127, 8220, 8167] avg=8192.0000 max=8256 max_rank=3 min=8127 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1027, 1021, 1046, 1027, 1041, 1010, 1042, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1011, 1008, 1031, 977, 1044, 1005, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1010, 1006, 989, 1003, 1047, 1028, 1033, 41, 16, 0, 0, 0, 0, 0, 0, 1032, 1027, 1065, 1027, 1054, 972, 1015, 955, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 977, 995, 1042, 1015, 1021, 1085, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 1007, 973, 1057, 1015, 1007, 1032, 999, 24, 6, 0, 0, 0, 0, 0, 0, 1084, 1045, 1009, 1058, 1059, 1068, 1043, 962, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1023, 992, 1012, 1039, 1011, 1008, 1042, 14, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8094, 8189, 8147, 8227, 8186, 8328, 8166] avg=8192.0000 max=8328 max_rank=6 min=8094 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1029, 1002, 1053, 1032, 1059, 1056, 1049, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 995, 1060, 1034, 1034, 1021, 998, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1057, 1033, 1030, 984, 985, 998, 1020, 40, 16, 0, 0, 0, 0, 0, 0, 1056, 1012, 1039, 1047, 1019, 955, 1035, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1021, 1032, 1035, 990, 1033, 1033, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 958, 1076, 1007, 958, 1043, 1024, 1063, 24, 6, 0, 0, 0, 0, 0, 0, 1011, 1016, 1018, 1045, 957, 1021, 1009, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1006, 1057, 995, 1002, 1038, 1016, 1027, 14, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8293, 8227, 8171, 8196, 8203, 8182, 8102, 8162] avg=8192.0000 max=8293 max_rank=0 min=8102 min_rank=6 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32649 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1066, 1033, 1008, 1059, 1015, 995, 1023, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1013, 1047, 1032, 984, 1042, 1012, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1010, 1049, 1022, 1032, 1018, 988, 1011, 42, 16, 0, 0, 0, 0, 0, 0, 1042, 1079, 1034, 983, 1020, 1003, 1072, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 958, 1010, 1038, 1031, 1018, 975, 976, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 985, 1051, 1026, 976, 1007, 1067, 1001, 24, 5, 0, 0, 0, 0, 0, 0, 1017, 1052, 989, 1074, 1090, 1018, 1062, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1054, 990, 1016, 1034, 954, 973, 1014, 13, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8188, 8180, 8267, 8035, 8181, 8330, 8116] avg=8192.0000 max=8330 max_rank=6 min=8035 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1050, 1059, 1013, 951, 994, 1034, 1052, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 980, 984, 1079, 1013, 998, 959, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 995, 1042, 1022, 1043, 1048, 1031, 1012, 44, 15, 0, 0, 0, 0, 0, 0, 1008, 1029, 1026, 1026, 986, 1046, 995, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1047, 1103, 1033, 1038, 1029, 1013, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1028, 1037, 1035, 1012, 1047, 1028, 1023, 23, 5, 0, 0, 0, 0, 0, 0, 988, 1057, 1017, 1036, 1007, 1046, 1045, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1036, 1012, 1015, 1027, 1015, 1012, 1048, 13, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8023, 8262, 8106, 8272, 8309, 8206, 8186] avg=8192.0000 max=8309 max_rank=5 min=8023 min_rank=1 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32973 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1014, 1006, 1021, 1078, 1010, 1002, 1031, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1027, 1049, 1013, 1021, 1053, 1002, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 997, 1005, 1060, 1016, 981, 1043, 1014, 43, 16, 0, 0, 0, 0, 0, 0, 1014, 1030, 1018, 1064, 1040, 1026, 1029, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1087, 964, 1047, 1043, 1002, 960, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1040, 1042, 1001, 974, 1022, 1021, 1040, 24, 5, 0, 0, 0, 0, 0, 0, 1030, 993, 1002, 1039, 978, 1050, 1008, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 967, 1028, 1021, 1032, 1005, 1024, 1035, 13, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8247, 8182, 8219, 8194, 8196, 8126, 8143] avg=8192.0000 max=8247 max_rank=1 min=8126 min_rank=6 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[962, 1017, 937, 1030, 1000, 1008, 1010, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1050, 1040, 1065, 1027, 1048, 1019, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 984, 982, 988, 1054, 1000, 994, 1015, 1000, 44, 16, 0, 0, 0, 0, 0, 0, 1013, 1007, 1001, 987, 1053, 1046, 1037, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 999, 1037, 1017, 1049, 1091, 1019, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1062, 1040, 1026, 1081, 1059, 973, 1033, 24, 6, 0, 0, 0, 0, 0, 0, 1031, 1026, 974, 988, 956, 1049, 1047, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1055, 1056, 985, 1034, 1028, 1002, 1084, 14, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8007, 8321, 8077, 8181, 8287, 8323, 8058, 8282] avg=8192.0000 max=8323 max_rank=5 min=8007 min_rank=0 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32950 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1000, 1051, 1018, 1007, 1034, 1060, 985, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1056, 1007, 1056, 1028, 1068, 998, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1052, 1044, 984, 992, 1057, 990, 1013, 49, 21, 0, 0, 0, 0, 0, 0, 1006, 1016, 1011, 1019, 990, 1014, 1077, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 981, 1043, 1023, 1035, 1033, 998, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 991, 972, 1004, 983, 1002, 1021, 985, 29, 10, 0, 0, 0, 0, 0, 0, 987, 1037, 1065, 1054, 1053, 1013, 993, 959, 5, 0, 0, 0, 0, 0, 0, 0, 1003, 1038, 1053, 1022, 1042, 1037, 1045, 1044, 18, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8280, 8241, 8164, 8145, 8032, 8166, 8314] avg=8192.0000 max=8314 max_rank=7 min=8032 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8448, 8192, 8192, 8320, 8192, 8320, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8064, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8448, 8192, 8320, 8320, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1008, 1032, 1024, 1029, 976, 1006, 984, 996, 1036, 977, 1016, 1023, 1032, 1044, 1004, 1069, 1014, 1012, 1029, 1003, 1067, 1008, 1010, 1020, 1023, 1058, 1041, 1023, 999, 1009, 1020, 1031, 1008, 1020, 1065, 1027, 995, 1035, 997, 1016, 1065, 1034, 1011, 1066, 1027, 992, 1008, 1055, 997, 1023, 1070, 1005, 1058, 1033, 1033, 1040, 1037, 1037, 1057, 1038, 1047, 1009, 996, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1034, 1033, 998, 1021, 1059, 1032, 1009, 1041, 1051, 991, 997, 994, 1061, 1012, 1062, 1044, 1051, 1009, 1033, 1090, 1003, 1044, 1019, 1020, 1005, 1040, 1043, 966, 997, 1020, 1063, 1009, 1058, 983, 1020, 1021, 1039, 1008, 1033, 976, 1059, 1016, 984, 995, 1043, 1034, 1043, 1000, 998, 1069, 1077, 985, 1053, 1041, 970, 983, 1011, 1053, 1002, 1017, 1052, 1030, 997, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1059, 1034, 984, 1026, 1026, 975, 1039, 1021, 976, 1066, 1007, 1033, 1025, 1026, 958, 983, 993, 1037, 1020, 1036, 1030, 1032, 1053, 1023, 1028, 1058, 1013, 1059, 1033, 1048, 1056, 1053, 1006, 1005, 1038, 1021, 998, 1011, 994, 1036, 1041, 1040, 1000, 1058, 998, 1032, 1046, 1029, 979, 1002, 1013, 1019, 1009, 1045, 1006, 1071, 1072, 1046, 996, 994, 1007, 1065, 1014, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1047, 946, 1015, 1081, 1064, 1012, 1023, 1022, 1022, 1028, 1047, 981, 1002, 1019, 1046, 1045, 1064, 1054, 1059, 1019, 1034, 1052, 1021, 1034, 1035, 1034, 1023, 1014, 1019, 1038, 1030, 996, 1015, 1024, 985, 1063, 1029, 1010, 996, 1040, 1051, 1002, 999, 1010, 1013, 1037, 1027, 981, 919, 1030, 1064, 996, 1016, 1033, 974, 1041, 1018, 994, 1046, 1002, 1052, 1032, 1088, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1032, 1021, 1049, 1081, 1041, 1015, 1009, 1005, 994, 1030, 1012, 1060, 1060, 1031, 1043, 1002, 1007, 1031, 1049, 1009, 1034, 1031, 1015, 955, 999, 1020, 1020, 1027, 1025, 1044, 1013, 1069, 1051, 1047, 1009, 1026, 1064, 1023, 1050, 1017, 990, 1019, 1025, 999, 1024, 1034, 1015, 1031, 1044, 1015, 1014, 1040, 1052, 960, 1022, 1001, 1062, 1025, 993, 1088, 990, 987, 963, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1042, 1004, 1043, 1011, 1045, 1063, 1036, 999, 1056, 1056, 1018, 998, 1062, 983, 1032, 1045, 1019, 1032, 1003, 1043, 1014, 1016, 1027, 1006, 1007, 997, 992, 1056, 1017, 1002, 1017, 1084, 981, 971, 1004, 1025, 1028, 1014, 1026, 979, 1009, 1060, 1008, 988, 1017, 986, 1035, 1068, 1056, 1037, 1047, 1011, 1100, 1048, 1004, 1045, 1000, 1035, 1051, 1059, 1012, 998, 1010, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1012, 1028, 1031, 1041, 993, 1009, 1065, 1021, 1011, 1015, 1021, 1026, 1043, 1023, 984, 1011, 994, 1021, 1038, 987, 1005, 1054, 1097, 1066, 1079, 1025, 1052, 1021, 994, 1039, 979, 991, 985, 990, 1004, 1032, 1033, 1055, 1020, 1088, 1069, 1077, 1035, 1048, 1005, 1009, 1016, 1044, 978, 1005, 1022, 1038, 1048, 985, 993, 993, 1010, 1043, 969, 1029, 1021, 1030, 1033, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1027, 989, 1013, 986, 1049, 1023, 1008, 1056, 969, 1008, 1024, 987, 1010, 1056, 1055, 1023, 1002, 999, 1016, 1061, 1029, 1035, 1033, 1056, 1013, 1072, 1047, 995, 1068, 1086, 1012, 988, 1006, 1026, 1038, 1022, 1076, 1019, 1038, 999, 984, 994, 1006, 997, 1023, 1040, 1020, 988, 1045, 993, 1016, 1011, 1030, 1029, 1011, 1053, 1031, 1018, 1020, 1058, 1052, 1051, 1014, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1008, 1032, 1024, 1029, 976, 1006, 984, 996, 4, 0, 0, 0, 0, 0, 0, 0, 1036, 977, 1016, 1023, 1032, 1044, 1004, 1069, 10, 3, 0, 0, 0, 0, 0, 0, 1014, 1012, 1029, 1003, 1067, 1008, 982, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1034, 1041, 1023, 999, 1009, 1020, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1020, 1065, 1027, 995, 1035, 997, 1016, 26, 0, 0, 0, 0, 0, 0, 0, 1065, 1034, 1011, 1066, 1027, 992, 1008, 1055, 9, 0, 0, 0, 0, 0, 0, 0, 997, 1023, 1070, 1005, 1058, 1033, 1033, 1040, 14, 2, 0, 0, 0, 0, 0, 0, 1037, 1037, 1057, 1022, 1047, 1009, 996, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8059, 8214, 8135, 8180, 8189, 8267, 8275, 8217] avg=8192.0000 max=8275 max_rank=6 min=8059 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32948 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1047, 946, 1015, 1081, 1064, 1012, 1023, 1022, 3, 0, 0, 0, 0, 0, 0, 0, 1022, 1028, 1047, 981, 1002, 1019, 1046, 1045, 10, 3, 0, 0, 0, 0, 0, 0, 1064, 1054, 1059, 1019, 1034, 1052, 993, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1010, 1023, 1014, 1019, 1038, 1030, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1024, 985, 1063, 1029, 1010, 996, 1040, 26, 0, 0, 0, 0, 0, 0, 0, 1051, 1002, 999, 1010, 1013, 1037, 1027, 981, 9, 0, 0, 0, 0, 0, 0, 0, 919, 1030, 1064, 996, 1016, 1033, 974, 1041, 14, 2, 0, 0, 0, 0, 0, 0, 1018, 994, 1046, 987, 1052, 1032, 1088, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8203, 8309, 8165, 8188, 8129, 8089, 8240] avg=8192.0000 max=8309 max_rank=2 min=8089 min_rank=6 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1034, 1033, 998, 1021, 1059, 1032, 1009, 1041, 4, 0, 0, 0, 0, 0, 0, 0, 1051, 991, 997, 994, 1061, 1012, 1062, 1044, 10, 3, 0, 0, 0, 0, 0, 0, 1051, 1009, 1033, 1090, 1003, 1044, 991, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1016, 1043, 966, 997, 1020, 1063, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 983, 1020, 1021, 1039, 1008, 1033, 976, 26, 0, 0, 0, 0, 0, 0, 0, 1059, 1016, 984, 995, 1043, 1034, 1043, 1000, 9, 0, 0, 0, 0, 0, 0, 0, 998, 1069, 1077, 985, 1053, 1041, 970, 983, 14, 2, 0, 0, 0, 0, 0, 0, 1011, 1053, 1002, 1001, 1052, 1030, 997, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8225, 8241, 8119, 8164, 8183, 8192, 8181] avg=8192.0000 max=8241 max_rank=2 min=8119 min_rank=3 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1059, 1034, 984, 1026, 1026, 975, 1039, 1021, 3, 0, 0, 0, 0, 0, 0, 0, 976, 1066, 1007, 1033, 1025, 1026, 958, 983, 10, 3, 0, 0, 0, 0, 0, 0, 993, 1037, 1020, 1036, 1030, 1032, 1024, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1034, 1013, 1059, 1033, 1048, 1056, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1005, 1038, 1021, 998, 1011, 994, 1036, 27, 0, 0, 0, 0, 0, 0, 0, 1041, 1040, 1000, 1058, 998, 1032, 1046, 1029, 9, 0, 0, 0, 0, 0, 0, 0, 979, 1002, 1013, 1019, 1009, 1045, 1006, 1071, 14, 2, 0, 0, 0, 0, 0, 0, 1072, 1046, 996, 979, 1007, 1065, 1014, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8087, 8195, 8324, 8136, 8253, 8160, 8214] avg=8192.0000 max=8324 max_rank=3 min=8087 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1042, 1004, 1043, 1011, 1045, 1063, 1036, 999, 4, 0, 0, 0, 0, 0, 0, 0, 1056, 1056, 1018, 998, 1062, 983, 1032, 1045, 9, 3, 0, 0, 0, 0, 0, 0, 1019, 1032, 1003, 1043, 1014, 1016, 998, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 975, 992, 1056, 1017, 1002, 1017, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 981, 971, 1004, 1025, 1028, 1014, 1026, 979, 27, 0, 0, 0, 0, 0, 0, 0, 1009, 1060, 1008, 988, 1017, 986, 1035, 1068, 9, 0, 0, 0, 0, 0, 0, 0, 1056, 1037, 1047, 1011, 1100, 1048, 1004, 1045, 13, 2, 0, 0, 0, 0, 0, 0, 1000, 1035, 1051, 1043, 1012, 998, 1010, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8262, 8131, 8150, 8055, 8180, 8363, 8148] avg=8192.0000 max=8363 max_rank=6 min=8055 min_rank=4 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1032, 1021, 1049, 1081, 1041, 1015, 1009, 1005, 4, 0, 0, 0, 0, 0, 0, 0, 994, 1030, 1012, 1060, 1060, 1031, 1043, 1002, 10, 3, 0, 0, 0, 0, 0, 0, 1007, 1031, 1049, 1009, 1034, 1031, 987, 955, 0, 0, 0, 0, 0, 0, 0, 0, 999, 996, 1020, 1027, 1025, 1044, 1013, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1047, 1009, 1026, 1064, 1023, 1050, 1017, 26, 0, 0, 0, 0, 0, 0, 0, 990, 1019, 1025, 999, 1024, 1034, 1015, 1031, 10, 0, 0, 0, 0, 0, 0, 0, 1044, 1015, 1014, 1040, 1052, 960, 1022, 1001, 14, 2, 0, 0, 0, 0, 0, 0, 1062, 1025, 993, 1071, 990, 987, 963, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 8245, 8103, 8193, 8313, 8147, 8164, 8114] avg=8192.0000 max=8313 max_rank=4 min=8103 min_rank=2 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1012, 1028, 1031, 1041, 993, 1009, 1065, 1021, 4, 0, 0, 0, 0, 0, 0, 0, 1011, 1015, 1021, 1026, 1043, 1023, 984, 1011, 10, 3, 0, 0, 0, 0, 0, 0, 994, 1021, 1038, 987, 1005, 1054, 1066, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1001, 1052, 1021, 994, 1039, 979, 991, 0, 0, 0, 0, 0, 0, 0, 0, 985, 990, 1004, 1032, 1033, 1055, 1020, 1088, 28, 0, 0, 0, 0, 0, 0, 0, 1069, 1077, 1035, 1048, 1005, 1009, 1016, 1044, 9, 0, 0, 0, 0, 0, 0, 0, 978, 1005, 1022, 1038, 1048, 985, 993, 993, 14, 3, 0, 0, 0, 0, 0, 0, 1010, 1043, 969, 1013, 1021, 1030, 1033, 1053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8147, 8231, 8156, 8235, 8312, 8079, 8172] avg=8192.0000 max=8312 max_rank=5 min=8079 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1027, 989, 1013, 986, 1049, 1023, 1008, 1056, 7, 0, 0, 0, 0, 0, 0, 0, 969, 1008, 1024, 987, 1010, 1056, 1055, 1023, 14, 7, 0, 0, 0, 0, 0, 0, 1002, 999, 1016, 1061, 1029, 1035, 993, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1040, 1047, 995, 1068, 1086, 1012, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1026, 1038, 1022, 1076, 1019, 1038, 999, 32, 0, 0, 0, 0, 0, 0, 0, 984, 994, 1006, 997, 1023, 1040, 1020, 988, 13, 0, 0, 0, 0, 0, 0, 0, 1045, 993, 1016, 1011, 1030, 1029, 1011, 1053, 18, 8, 0, 0, 0, 0, 0, 0, 1031, 1018, 1020, 1031, 1052, 1051, 1014, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8153, 8191, 8249, 8256, 8065, 8214, 8250] avg=8192.0000 max=8256 max_rank=4 min=8065 min_rank=5 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8448, 8192, 8448, 8192, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8320, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1051, 1039, 1043, 1024, 1017, 1052, 1070, 1039, 983, 1018, 1056, 962, 1065, 1029, 1032, 1089, 1035, 1035, 1002, 1027, 1023, 1016, 1016, 1029, 997, 1028, 999, 979, 1000, 1035, 996, 1051, 1037, 1015, 1012, 1071, 1011, 1041, 1000, 998, 1029, 1028, 1030, 1009, 995, 1047, 1008, 1080, 1045, 981, 994, 1009, 1002, 1014, 1003, 1041, 1020, 1016, 1069, 1034, 1062, 1008, 993, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1005, 1038, 1015, 1038, 984, 1030, 1003, 1034, 1006, 1015, 1023, 1011, 973, 995, 1072, 1044, 1091, 1058, 974, 1007, 998, 990, 1043, 1049, 1050, 980, 1074, 973, 1036, 1012, 1012, 1053, 1041, 1020, 1062, 1022, 1020, 1048, 1017, 1041, 1007, 999, 1047, 1041, 1018, 1064, 958, 995, 999, 992, 1034, 1058, 999, 1003, 1002, 1034, 1013, 1040, 1048, 1017, 1063, 1059, 1059, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1037, 1056, 1005, 985, 1063, 1034, 1095, 1057, 1012, 1036, 1031, 1014, 963, 1030, 1015, 996, 973, 983, 1030, 992, 1029, 1016, 1039, 1035, 1039, 1037, 999, 1051, 1025, 1035, 1009, 1066, 1065, 1006, 1054, 1025, 1008, 1005, 1024, 1037, 1017, 972, 1082, 1031, 1027, 1020, 1036, 954, 1022, 1047, 1027, 1033, 1040, 995, 1053, 986, 1048, 1062, 984, 993, 1019, 1047, 1005, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1033, 1033, 1005, 1016, 1052, 1043, 997, 1014, 991, 1039, 1036, 1032, 974, 1024, 1041, 1036, 1024, 1030, 1020, 1065, 1063, 1027, 1000, 991, 1053, 999, 995, 1049, 1012, 1037, 1067, 949, 1008, 1027, 992, 1002, 997, 1030, 1026, 1015, 1003, 1020, 1062, 1032, 1008, 1030, 1042, 1011, 1038, 1041, 1012, 955, 1023, 1014, 1051, 1003, 1010, 1022, 1061, 1061, 1087, 1052, 997, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1083, 1024, 1011, 1043, 1017, 1008, 1019, 975, 1027, 1034, 1032, 1061, 1003, 1037, 1002, 1007, 1036, 1013, 1007, 1037, 1002, 1039, 1027, 1075, 1026, 1005, 995, 978, 1043, 1023, 1060, 1021, 995, 1069, 1051, 1013, 1025, 999, 1050, 1055, 954, 945, 1048, 966, 1019, 1042, 1042, 1028, 1059, 996, 1046, 1069, 988, 1020, 1015, 990, 1056, 1013, 1061, 1055, 1033, 1022, 1041, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1042, 993, 1030, 1057, 1043, 999, 1021, 1002, 1016, 1045, 1024, 1024, 1025, 1004, 1017, 1012, 1097, 1022, 1043, 1033, 1000, 1029, 1006, 991, 1015, 1087, 1024, 1020, 1056, 978, 988, 1066, 994, 1015, 989, 1087, 1079, 977, 1004, 1057, 1026, 1064, 999, 1006, 992, 1071, 1024, 1025, 1036, 985, 1031, 1032, 1004, 1041, 1043, 1085, 1017, 1007, 991, 1007, 995, 1030, 993, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1077, 975, 981, 1024, 1041, 972, 1037, 1023, 1018, 1013, 981, 996, 1032, 1022, 1001, 994, 989, 996, 993, 1022, 1025, 1002, 1020, 1019, 1033, 1038, 1018, 1029, 1039, 1071, 1037, 982, 1083, 1007, 1059, 998, 1020, 991, 1034, 1039, 1073, 1020, 1044, 1047, 964, 1048, 1044, 1043, 1019, 1026, 1055, 1048, 1036, 1021, 1071, 1031, 1060, 1012, 1056, 997, 1001, 1004, 1067, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1012, 1085, 1018, 1064, 1059, 1039, 1055, 1003, 1017, 1052, 1020, 1032, 992, 991, 1015, 1039, 1024, 1022, 953, 1042, 1017, 1048, 995, 976, 982, 1027, 1028, 1033, 1047, 1053, 993, 1027, 1081, 993, 1038, 1054, 1027, 1004, 1026, 1039, 1009, 1040, 1023, 1018, 1067, 1072, 990, 1076, 1036, 1013, 1015, 1002, 969, 1020, 993, 1032, 996, 1038, 997, 982, 1032, 1059, 1013, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[969, 1038, 1015, 1038, 984, 1030, 1003, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1015, 1023, 1011, 973, 995, 1072, 1044, 36, 0, 0, 0, 0, 0, 0, 0, 1091, 1058, 974, 1007, 998, 990, 1043, 1049, 27, 1, 0, 0, 0, 0, 0, 0, 1050, 980, 1074, 973, 1036, 1012, 1012, 1053, 2, 0, 0, 0, 0, 0, 0, 0, 1019, 1020, 1062, 1022, 1020, 1048, 1017, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 999, 1047, 1041, 1018, 1064, 958, 995, 0, 0, 0, 0, 0, 0, 0, 0, 999, 992, 1034, 1058, 999, 1003, 1002, 1034, 19, 0, 0, 0, 0, 0, 0, 0, 1013, 1040, 1048, 1017, 1036, 1059, 1059, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8175, 8238, 8192, 8249, 8129, 8140, 8302] avg=8192.0000 max=8302 max_rank=7 min=8111 min_rank=0 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1013, 1039, 1043, 1024, 1017, 1052, 1070, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1018, 1056, 962, 1065, 1029, 1032, 1089, 38, 0, 0, 0, 0, 0, 0, 0, 1035, 1035, 1002, 1027, 1023, 1016, 1016, 1029, 27, 1, 0, 0, 0, 0, 0, 0, 997, 1028, 999, 979, 1000, 1035, 996, 1051, 2, 0, 0, 0, 0, 0, 0, 0, 1015, 1015, 1012, 1071, 1011, 1041, 1000, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1028, 1030, 1009, 995, 1047, 1008, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 981, 994, 1009, 1002, 1014, 1003, 1041, 19, 0, 0, 0, 0, 0, 0, 0, 1020, 1016, 1069, 1034, 1035, 1008, 993, 997, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8272, 8211, 8087, 8163, 8226, 8108, 8172] avg=8192.0000 max=8297 max_rank=0 min=8087 min_rank=3 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1005, 993, 1030, 1057, 1043, 999, 1021, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1045, 1024, 1024, 1025, 1004, 1017, 1012, 37, 0, 0, 0, 0, 0, 0, 0, 1097, 1022, 1043, 1033, 1000, 1029, 1006, 991, 25, 1, 0, 0, 0, 0, 0, 0, 1015, 1087, 1024, 1020, 1056, 978, 988, 1066, 2, 0, 0, 0, 0, 0, 0, 0, 972, 1015, 989, 1087, 1079, 977, 1004, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1064, 999, 1006, 992, 1071, 1024, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 985, 1031, 1032, 1004, 1041, 1043, 1085, 19, 0, 0, 0, 0, 0, 0, 0, 1017, 1007, 991, 1007, 970, 1030, 993, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8204, 8247, 8236, 8180, 8207, 8276, 8036] avg=8192.0000 max=8276 max_rank=6 min=8036 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[996, 1033, 1005, 1016, 1052, 1043, 997, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1039, 1036, 1032, 974, 1024, 1041, 1036, 37, 0, 0, 0, 0, 0, 0, 0, 1024, 1030, 1020, 1065, 1063, 1027, 1000, 991, 28, 1, 0, 0, 0, 0, 0, 0, 1053, 999, 995, 1049, 1012, 1037, 1067, 949, 2, 0, 0, 0, 0, 0, 0, 0, 986, 1027, 992, 1002, 997, 1030, 1026, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1020, 1062, 1032, 1008, 1030, 1042, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1041, 1012, 955, 1023, 1014, 1051, 1003, 19, 0, 0, 0, 0, 0, 0, 0, 1010, 1022, 1061, 1061, 1059, 1052, 997, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8210, 8249, 8163, 8075, 8208, 8156, 8319] avg=8192.0000 max=8319 max_rank=7 min=8075 min_rank=4 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1044, 1024, 1011, 1043, 1017, 1008, 1019, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1034, 1032, 1061, 1003, 1037, 1002, 1007, 39, 0, 0, 0, 0, 0, 0, 0, 1036, 1013, 1007, 1037, 1002, 1039, 1027, 1075, 26, 1, 0, 0, 0, 0, 0, 0, 1026, 1005, 995, 978, 1043, 1023, 1060, 1021, 2, 0, 0, 0, 0, 0, 0, 0, 973, 1069, 1051, 1013, 1025, 999, 1050, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 954, 945, 1048, 966, 1019, 1042, 1042, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 996, 1046, 1069, 988, 1020, 1015, 990, 19, 0, 0, 0, 0, 0, 0, 0, 1056, 1013, 1061, 1055, 1007, 1022, 1041, 1001, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8242, 8263, 8153, 8235, 8044, 8202, 8256] avg=8192.0000 max=8263 max_rank=2 min=8044 min_rank=5 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1000, 1056, 1005, 985, 1063, 1034, 1095, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1036, 1031, 1014, 963, 1030, 1015, 996, 37, 0, 0, 0, 0, 0, 0, 0, 973, 983, 1030, 992, 1029, 1016, 1039, 1035, 26, 1, 0, 0, 0, 0, 0, 0, 1039, 1037, 999, 1051, 1025, 1035, 1009, 1066, 2, 0, 0, 0, 0, 0, 0, 0, 1042, 1006, 1054, 1025, 1008, 1005, 1024, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 972, 1082, 1031, 1027, 1020, 1036, 954, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1047, 1027, 1033, 1040, 995, 1053, 986, 20, 0, 0, 0, 0, 0, 0, 0, 1048, 1062, 984, 993, 993, 1047, 1005, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8134, 8124, 8263, 8201, 8139, 8223, 8157] avg=8192.0000 max=8295 max_rank=0 min=8124 min_rank=2 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1038, 975, 981, 1024, 1041, 972, 1037, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1013, 981, 996, 1032, 1022, 1001, 994, 39, 0, 0, 0, 0, 0, 0, 0, 989, 996, 993, 1022, 1025, 1002, 1020, 1019, 25, 1, 0, 0, 0, 0, 0, 0, 1033, 1038, 1018, 1029, 1039, 1071, 1037, 982, 2, 0, 0, 0, 0, 0, 0, 0, 1060, 1007, 1059, 998, 1020, 991, 1034, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1020, 1044, 1047, 964, 1048, 1044, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1026, 1055, 1048, 1036, 1021, 1071, 1031, 20, 0, 0, 0, 0, 0, 0, 0, 1060, 1012, 1056, 997, 976, 1004, 1067, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8091, 8096, 8092, 8249, 8208, 8283, 8327, 8190] avg=8192.0000 max=8327 max_rank=6 min=8091 min_rank=0 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=33008 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[972, 1085, 1018, 1064, 1059, 1039, 1055, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1052, 1020, 1032, 992, 991, 1015, 1039, 40, 5, 0, 0, 0, 0, 0, 0, 1024, 1022, 953, 1042, 1017, 1048, 995, 976, 30, 5, 0, 0, 0, 0, 0, 0, 982, 1027, 1028, 1033, 1047, 1053, 993, 1027, 3, 0, 0, 0, 0, 0, 0, 0, 1044, 993, 1038, 1054, 1027, 1004, 1026, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1040, 1023, 1018, 1067, 1072, 990, 1076, 5, 0, 0, 0, 0, 0, 0, 0, 1036, 1013, 1015, 1002, 969, 1020, 993, 1032, 24, 0, 0, 0, 0, 0, 0, 0, 996, 1038, 997, 982, 997, 1059, 1013, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8203, 8112, 8193, 8225, 8300, 8104, 8104] avg=8192.0000 max=8300 max_rank=5 min=8104 min_rank=6 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8320, 8192, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8192, 8320, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8320, 8192, 8064, 8192, 8320, 8320, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8448, 8192, 8192, 8448, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1030, 1053, 1001, 998, 1024, 1024, 1104, 1010, 1081, 1029, 1048, 1043, 982, 1076, 1010, 1086, 1009, 1036, 1063, 999, 1020, 1012, 1031, 999, 994, 989, 1030, 1017, 986, 981, 1016, 998, 1018, 1054, 1072, 1025, 1030, 1012, 1001, 1040, 1045, 1050, 1022, 989, 985, 1020, 1057, 1042, 1041, 1027, 996, 1024, 992, 1015, 1024, 1006, 942, 1025, 974, 1047, 1051, 1006, 1070, 1055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1067, 1024, 1049, 1034, 1054, 1036, 1004, 1100, 1024, 1057, 979, 1004, 985, 1026, 996, 1045, 1006, 1023, 1011, 1024, 1023, 1051, 1021, 991, 1075, 1008, 1031, 1025, 971, 1003, 997, 1040, 1017, 990, 1018, 1035, 1032, 1007, 1012, 985, 983, 987, 1027, 1072, 1007, 1025, 1050, 1020, 1019, 1027, 1052, 990, 1031, 1037, 1033, 1032, 1042, 1029, 1066, 1071, 1032, 1007, 1016, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1044, 1028, 1029, 979, 1001, 1038, 1027, 1005, 1058, 1033, 1002, 964, 1054, 1082, 1042, 1019, 1016, 1020, 1008, 1026, 998, 1036, 1016, 1020, 1053, 998, 1012, 1073, 1019, 1028, 996, 966, 1009, 1028, 1019, 1065, 1002, 1015, 1035, 1023, 1038, 1061, 1029, 1020, 1014, 1051, 1056, 1075, 987, 1004, 1028, 1004, 1074, 986, 1054, 1033, 1009, 1028, 967, 1056, 1023, 1005, 1041, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1010, 1049, 1022, 963, 978, 1039, 1033, 1014, 1040, 998, 1043, 1016, 1035, 1050, 970, 1025, 1080, 1079, 989, 1008, 1013, 1000, 970, 1010, 1048, 1078, 1051, 1006, 1023, 1004, 1025, 1036, 1080, 1014, 1022, 979, 1030, 1001, 1012, 988, 1027, 1030, 1070, 1075, 1034, 1016, 1042, 1058, 1014, 1026, 999, 1022, 1005, 1079, 1028, 1037, 1040, 1057, 965, 1014, 1020, 1005, 1045, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1017, 1008, 1015, 1050, 1037, 1027, 1038, 1038, 1032, 1012, 960, 1013, 988, 976, 1042, 1023, 1046, 1062, 1022, 1042, 1041, 988, 995, 1016, 1029, 1025, 1032, 1072, 1037, 1080, 1067, 989, 1004, 995, 1032, 1024, 1037, 1090, 980, 1054, 1022, 1003, 1001, 963, 1053, 1013, 1031, 1015, 1065, 1030, 1024, 965, 1070, 1011, 1034, 1015, 1044, 1029, 1040, 1039, 1068, 1003, 971, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1009, 989, 1031, 1005, 1001, 1050, 1007, 1007, 1014, 1034, 979, 1085, 969, 1001, 1021, 1024, 996, 995, 1014, 1072, 1030, 991, 1023, 997, 1049, 1021, 1036, 1028, 1045, 1070, 986, 971, 1050, 988, 1039, 1054, 1043, 1008, 1027, 998, 1054, 1042, 1016, 1052, 1027, 1027, 1030, 1047, 1032, 1044, 1024, 1067, 1044, 1028, 1046, 1045, 1048, 1014, 1037, 1000, 1031, 989, 1018, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1066, 1035, 1021, 1046, 1003, 992, 992, 1020, 1004, 1026, 1032, 999, 1026, 1017, 1009, 1031, 1073, 1002, 1040, 1099, 1031, 1074, 983, 1044, 1024, 1032, 1031, 1002, 1044, 946, 994, 992, 1051, 1048, 1026, 1024, 1021, 1046, 1025, 1040, 1019, 1002, 1013, 1035, 1029, 1037, 1000, 977, 1025, 1022, 1035, 1044, 1019, 1066, 1040, 1041, 1038, 989, 1031, 1050, 997, 955, 1034, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1049, 1006, 1018, 1011, 1059, 1010, 1009, 963, 1035, 1007, 980, 1042, 1067, 1048, 1001, 1012, 1009, 1013, 1030, 1012, 1030, 1020, 1035, 1063, 981, 1052, 1023, 1011, 1000, 1012, 988, 1125, 1014, 1062, 1041, 1061, 986, 1032, 1046, 1061, 998, 1035, 1052, 1065, 1064, 949, 1011, 1006, 1017, 1074, 984, 1010, 1046, 1082, 1033, 985, 1003, 986, 1045, 1032, 994, 1032, 966, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1044, 1028, 1029, 979, 1001, 1038, 1027, 1005, 4, 0, 0, 0, 0, 0, 0, 0, 1058, 1033, 1002, 964, 1054, 1082, 1042, 1019, 8, 7, 0, 0, 0, 0, 0, 0, 1016, 1020, 1008, 1026, 998, 1036, 1016, 1020, 4, 0, 0, 0, 0, 0, 0, 0, 1053, 998, 1012, 1073, 1019, 1028, 996, 966, 21, 0, 0, 0, 0, 0, 0, 0, 1009, 1028, 1003, 1065, 1002, 1015, 1035, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1061, 1029, 1020, 1014, 1051, 1028, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1004, 1028, 1004, 1074, 956, 1054, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1028, 967, 1056, 1023, 1005, 1041, 1007, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8269, 8144, 8166, 8180, 8316, 8140, 8166] avg=8192.0000 max=8316 max_rank=5 min=8140 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1067, 1024, 1049, 1034, 1054, 1036, 1004, 1100, 4, 0, 0, 0, 0, 0, 0, 0, 1024, 1057, 979, 1004, 985, 1026, 996, 1045, 8, 7, 0, 0, 0, 0, 0, 0, 1006, 1023, 1011, 1024, 1023, 1051, 1021, 991, 4, 0, 0, 0, 0, 0, 0, 0, 1075, 1008, 1031, 1025, 971, 1003, 997, 1040, 20, 0, 0, 0, 0, 0, 0, 0, 1017, 990, 1002, 1035, 1032, 1007, 1012, 985, 0, 0, 0, 0, 0, 0, 0, 0, 983, 987, 1027, 1072, 1007, 1025, 1023, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1027, 1052, 990, 1031, 1005, 1033, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1029, 1066, 1071, 1032, 1007, 1016, 1001, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8372, 8131, 8154, 8170, 8080, 8144, 8189, 8296] avg=8192.0000 max=8372 max_rank=0 min=8080 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1030, 1053, 1001, 998, 1024, 1024, 1104, 1010, 4, 0, 0, 0, 0, 0, 0, 0, 1081, 1029, 1048, 1043, 982, 1076, 1010, 1086, 8, 7, 0, 0, 0, 0, 0, 0, 1009, 1036, 1063, 999, 1020, 1012, 1031, 999, 5, 0, 0, 0, 0, 0, 0, 0, 994, 989, 1030, 1017, 986, 981, 1016, 998, 21, 0, 0, 0, 0, 0, 0, 0, 1018, 1054, 1055, 1025, 1030, 1012, 1001, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1050, 1022, 989, 985, 1020, 1029, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1027, 996, 1024, 992, 984, 1024, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 942, 1025, 974, 1047, 1051, 1006, 1070, 1055, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8370, 8174, 8032, 8235, 8182, 8094, 8201] avg=8192.0000 max=8370 max_rank=1 min=8032 min_rank=3 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1010, 1049, 1022, 963, 978, 1039, 1033, 1014, 4, 0, 0, 0, 0, 0, 0, 0, 1040, 998, 1043, 1016, 1035, 1050, 970, 1025, 8, 7, 0, 0, 0, 0, 0, 0, 1080, 1079, 989, 1008, 1013, 1000, 970, 1010, 4, 0, 0, 0, 0, 0, 0, 0, 1048, 1078, 1051, 1006, 1023, 1004, 1025, 1036, 20, 0, 0, 0, 0, 0, 0, 0, 1080, 1014, 1006, 979, 1030, 1001, 1012, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1030, 1070, 1075, 1034, 1016, 1015, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1026, 999, 1022, 1005, 1046, 1028, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1057, 965, 1014, 1020, 1005, 1045, 997, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8192, 8153, 8291, 8110, 8325, 8177, 8176] avg=8192.0000 max=8325 max_rank=5 min=8110 min_rank=4 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32788 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1066, 1035, 1021, 1046, 1003, 992, 992, 1020, 4, 0, 0, 0, 0, 0, 0, 0, 1004, 1026, 1032, 999, 1026, 1017, 1009, 1031, 8, 7, 0, 0, 0, 0, 0, 0, 1073, 1002, 1040, 1099, 1031, 1074, 983, 1044, 4, 0, 0, 0, 0, 0, 0, 0, 1024, 1032, 1031, 1002, 1044, 946, 994, 992, 19, 0, 0, 0, 0, 0, 0, 0, 1051, 1048, 1010, 1024, 1021, 1046, 1025, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1002, 1013, 1035, 1029, 1037, 974, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1022, 1035, 1044, 1019, 1034, 1040, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 989, 1031, 1050, 997, 955, 1034, 1027, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8159, 8350, 8084, 8265, 8086, 8260, 8153] avg=8192.0000 max=8350 max_rank=2 min=8084 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1017, 1008, 1015, 1050, 1037, 1027, 1038, 1038, 4, 0, 0, 0, 0, 0, 0, 0, 1032, 1012, 960, 1013, 988, 976, 1042, 1023, 8, 7, 0, 0, 0, 0, 0, 0, 1046, 1062, 1022, 1042, 1041, 988, 995, 1016, 4, 0, 0, 0, 0, 0, 0, 0, 1029, 1025, 1032, 1072, 1037, 1080, 1067, 989, 20, 0, 0, 0, 0, 0, 0, 0, 1004, 995, 1016, 1024, 1037, 1090, 980, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1003, 1001, 963, 1053, 1013, 1004, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1030, 1024, 965, 1070, 980, 1034, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1029, 1040, 1039, 1068, 1003, 971, 992, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8061, 8216, 8351, 8200, 8074, 8183, 8217] avg=8192.0000 max=8351 max_rank=3 min=8061 min_rank=1 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1009, 989, 1031, 1005, 1001, 1050, 1007, 1007, 8, 0, 0, 0, 0, 0, 0, 0, 1014, 1034, 979, 1085, 969, 1001, 1021, 1024, 9, 11, 0, 0, 0, 0, 0, 0, 996, 995, 1014, 1072, 1030, 991, 1023, 997, 10, 0, 0, 0, 0, 0, 0, 0, 1049, 1021, 1036, 1028, 1045, 1070, 986, 971, 24, 0, 0, 0, 0, 0, 0, 0, 1050, 988, 1012, 1054, 1043, 1008, 1027, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1042, 1016, 1052, 1027, 1027, 991, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1044, 1024, 1067, 1044, 993, 1046, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1014, 1037, 1000, 1031, 989, 1018, 1017, 35, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8147, 8128, 8230, 8180, 8256, 8295, 8193] avg=8192.0000 max=8295 max_rank=6 min=8107 min_rank=0 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32924 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1049, 1006, 1018, 1011, 1059, 1010, 1009, 963, 4, 0, 0, 0, 0, 0, 0, 0, 1035, 1007, 980, 1042, 1067, 1048, 1001, 1012, 8, 7, 0, 0, 0, 0, 0, 0, 1009, 1013, 1030, 1012, 1030, 1020, 1035, 1063, 5, 0, 0, 0, 0, 0, 0, 0, 981, 1052, 1023, 1011, 1000, 1012, 988, 1125, 20, 0, 0, 0, 0, 0, 0, 0, 1014, 1062, 1024, 1061, 986, 1032, 1046, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1035, 1052, 1065, 1064, 949, 984, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1074, 984, 1010, 1046, 1049, 1033, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 986, 1045, 1032, 994, 1032, 966, 1043, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8207, 8217, 8212, 8286, 8153, 8198, 8134] avg=8192.0000 max=8286 max_rank=4 min=8129 min_rank=0 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8064, 8192, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8064, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1007, 1021, 1024, 1063, 1024, 1037, 1038, 1019, 995, 982, 1061, 1064, 1027, 992, 1022, 1014, 1040, 1035, 1046, 991, 1024, 1014, 1025, 999, 1076, 1045, 1020, 1008, 1063, 1042, 1011, 986, 1067, 1048, 985, 1018, 1032, 1046, 1043, 1029, 1035, 1009, 982, 982, 1035, 1077, 1021, 946, 980, 1035, 1024, 1050, 1037, 1016, 1019, 1053, 996, 1004, 1036, 1020, 1015, 1057, 1032, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1005, 1012, 1028, 1008, 1038, 1019, 1038, 984, 1023, 996, 1014, 1048, 1017, 1050, 1019, 1051, 1023, 978, 967, 1046, 1009, 1011, 1096, 1022, 1005, 1005, 1024, 1033, 1039, 1010, 997, 1048, 1025, 1049, 1008, 1040, 1007, 1028, 1048, 1037, 993, 1043, 1009, 1061, 1020, 1034, 1011, 1071, 1037, 1046, 992, 1014, 1033, 1043, 1003, 1054, 1054, 993, 1026, 1016, 1059, 960, 1012, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1042, 1046, 1025, 1058, 1002, 1016, 1034, 1017, 1014, 1035, 1049, 1015, 1023, 1011, 1016, 1044, 990, 1044, 975, 1035, 986, 1003, 999, 1058, 1013, 1008, 1031, 1062, 1054, 1009, 1054, 1032, 1001, 1039, 1035, 1014, 1056, 1010, 995, 1073, 1048, 1009, 1018, 1040, 1035, 996, 1038, 1009, 1020, 1029, 1046, 1021, 965, 1060, 1027, 969, 1017, 1016, 1005, 1058, 1006, 1016, 1030, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1037, 993, 1033, 1048, 983, 1013, 1004, 1010, 1024, 990, 1072, 1062, 1021, 1015, 975, 979, 1032, 1003, 1023, 1018, 1033, 987, 1012, 1009, 1045, 1057, 1093, 1048, 996, 1013, 1027, 1030, 1048, 996, 974, 1038, 1009, 1041, 1071, 1044, 1023, 1008, 1035, 1076, 1024, 1016, 1027, 996, 1035, 1010, 1052, 990, 1039, 995, 1008, 1009, 1023, 1026, 1096, 1038, 1012, 1051, 1030, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[974, 965, 1034, 1087, 1007, 1022, 1038, 1037, 1047, 984, 1086, 1043, 1040, 1029, 997, 1012, 1037, 1054, 1050, 1065, 958, 1036, 1016, 1006, 977, 1049, 1049, 1016, 1018, 1034, 1040, 1019, 1005, 991, 993, 1040, 1049, 969, 1027, 1052, 1026, 1047, 1045, 1038, 1043, 1079, 972, 1066, 1061, 975, 1029, 1012, 1036, 1015, 1015, 997, 956, 1024, 1028, 1024, 1022, 1033, 1014, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1021, 1003, 1077, 1048, 1039, 938, 999, 1028, 1068, 1032, 1017, 1006, 1088, 996, 1056, 1113, 984, 1036, 1053, 1066, 1037, 1018, 1016, 1061, 1066, 1013, 1036, 1040, 1018, 1040, 1016, 1046, 1015, 986, 1013, 1002, 1034, 979, 1009, 1065, 1054, 979, 998, 1018, 1036, 1039, 997, 1034, 1053, 1020, 1033, 1042, 970, 1011, 987, 960, 1013, 1012, 1043, 1006, 1052, 977, 1023, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1023, 1031, 992, 997, 1050, 1025, 1005, 998, 972, 1060, 1020, 1035, 1025, 1023, 1005, 972, 1028, 1030, 1051, 1020, 1037, 1003, 1060, 984, 1006, 1040, 1040, 1019, 1055, 1028, 1006, 1014, 1013, 1096, 1051, 1080, 1000, 1009, 994, 1047, 1035, 991, 977, 1036, 1103, 1024, 1019, 1076, 989, 1026, 1055, 1005, 995, 1011, 1044, 1000, 1083, 1031, 985, 1013, 991, 1052, 1007, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1037, 1036, 1026, 1007, 991, 976, 1009, 1046, 1043, 990, 1004, 1012, 1025, 1042, 1056, 1039, 993, 1013, 1034, 1014, 1006, 1015, 1013, 1024, 1068, 1045, 1023, 1019, 1011, 1009, 1001, 1016, 1024, 1052, 1000, 1014, 1015, 1058, 1066, 1048, 1095, 1029, 977, 1045, 1026, 1054, 1000, 998, 1024, 1040, 1025, 1005, 976, 1063, 1016, 1017, 1044, 1004, 1055, 1051, 991, 1038, 1014, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1037, 993, 1033, 1048, 983, 1013, 1004, 1010, 26, 2, 0, 0, 0, 0, 0, 0, 1024, 990, 1058, 1062, 1021, 1015, 975, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1003, 1023, 1018, 1033, 987, 1012, 1009, 15, 7, 0, 0, 0, 0, 0, 0, 1045, 1057, 1052, 1048, 996, 1013, 1027, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 996, 974, 1038, 1009, 1041, 1071, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1008, 1035, 1076, 1007, 1016, 1027, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1010, 1052, 990, 1039, 995, 1008, 1009, 41, 0, 0, 0, 0, 0, 0, 0, 1023, 1026, 1096, 1038, 1012, 1051, 1030, 1011, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8124, 8139, 8268, 8195, 8188, 8179, 8294] avg=8192.0000 max=8294 max_rank=7 min=8124 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1042, 1046, 1025, 1058, 1002, 1016, 1034, 1017, 27, 2, 0, 0, 0, 0, 0, 0, 1014, 1035, 1035, 1015, 1023, 1011, 1016, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1044, 975, 1035, 986, 1003, 999, 1058, 15, 7, 0, 0, 0, 0, 0, 0, 1013, 1008, 992, 1062, 1054, 1009, 1054, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1039, 1035, 1014, 1056, 1010, 995, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1009, 1018, 1040, 1018, 996, 1038, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1029, 1046, 1021, 965, 1060, 1027, 969, 39, 0, 0, 0, 0, 0, 0, 0, 1017, 1016, 1005, 1058, 1006, 1016, 1030, 1035, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8193, 8112, 8224, 8196, 8176, 8176, 8190] avg=8192.0000 max=8269 max_rank=0 min=8112 min_rank=2 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1007, 1021, 1024, 1063, 1024, 1037, 1038, 1019, 26, 2, 0, 0, 0, 0, 0, 0, 995, 982, 1047, 1064, 1027, 992, 1022, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1035, 1046, 991, 1024, 1014, 1025, 999, 15, 7, 0, 0, 0, 0, 0, 0, 1076, 1045, 981, 1008, 1063, 1042, 1011, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1048, 985, 1018, 1032, 1046, 1043, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1009, 982, 982, 1018, 1077, 1021, 946, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1035, 1024, 1050, 1037, 1016, 1019, 1053, 39, 0, 0, 0, 0, 0, 0, 0, 996, 1004, 1036, 1020, 1015, 1057, 1032, 992, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8143, 8196, 8212, 8242, 8070, 8253, 8159] avg=8192.0000 max=8261 max_rank=0 min=8070 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1021, 1003, 1077, 1048, 1039, 938, 999, 1028, 27, 2, 0, 0, 0, 0, 0, 0, 1068, 1032, 1003, 1006, 1088, 996, 1056, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1036, 1053, 1066, 1037, 1018, 1016, 1061, 15, 7, 0, 0, 0, 0, 0, 0, 1066, 1013, 997, 1040, 1018, 1040, 1016, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 986, 1013, 1002, 1034, 979, 1009, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 979, 998, 1018, 1019, 1039, 997, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1020, 1033, 1042, 970, 1011, 987, 960, 39, 0, 0, 0, 0, 0, 0, 0, 1013, 1012, 1043, 1006, 1052, 977, 1023, 1001, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8362, 8293, 8236, 8076, 8138, 8115, 8134] avg=8192.0000 max=8362 max_rank=1 min=8076 min_rank=4 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1005, 1012, 1028, 1008, 1038, 1019, 1038, 984, 26, 2, 0, 0, 0, 0, 0, 0, 1023, 996, 1000, 1048, 1017, 1050, 1019, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 978, 967, 1046, 1009, 1011, 1096, 1022, 15, 7, 0, 0, 0, 0, 0, 0, 1005, 1005, 985, 1033, 1039, 1010, 997, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1049, 1008, 1040, 1007, 1028, 1048, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1043, 1009, 1061, 1003, 1034, 1011, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1046, 992, 1014, 1033, 1043, 1003, 1054, 39, 0, 0, 0, 0, 0, 0, 0, 1054, 993, 1026, 1016, 1059, 960, 1012, 1047, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8204, 8174, 8122, 8216, 8225, 8261, 8174] avg=8192.0000 max=8261 max_rank=6 min=8122 min_rank=3 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[974, 965, 1034, 1087, 1007, 1022, 1038, 1037, 26, 2, 0, 0, 0, 0, 0, 0, 1047, 984, 1072, 1043, 1040, 1029, 997, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1054, 1050, 1065, 958, 1036, 1016, 1006, 15, 7, 0, 0, 0, 0, 0, 0, 977, 1049, 1009, 1016, 1018, 1034, 1040, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 991, 993, 1040, 1049, 969, 1027, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1047, 1045, 1038, 1026, 1079, 972, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 975, 1029, 1012, 1036, 1015, 1015, 997, 40, 0, 0, 0, 0, 0, 0, 0, 956, 1024, 1028, 1024, 1022, 1033, 1014, 1027, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8224, 8244, 8162, 8100, 8299, 8180, 8135] avg=8192.0000 max=8299 max_rank=5 min=8100 min_rank=4 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1023, 1031, 992, 997, 1050, 1025, 1005, 998, 26, 3, 0, 0, 0, 0, 0, 0, 972, 1060, 1006, 1035, 1025, 1023, 1005, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1030, 1051, 1020, 1037, 1003, 1060, 984, 16, 7, 0, 0, 0, 0, 0, 0, 1006, 1040, 1001, 1019, 1055, 1028, 1006, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1096, 1051, 1080, 1000, 1009, 994, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 991, 977, 1036, 1084, 1024, 1019, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1026, 1055, 1005, 995, 1011, 1044, 1000, 39, 0, 0, 0, 0, 0, 0, 0, 1083, 1031, 985, 1013, 991, 1052, 1007, 1044, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8098, 8236, 8169, 8264, 8242, 8164, 8213] avg=8192.0000 max=8264 max_rank=4 min=8098 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32883 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1037, 1036, 1026, 1007, 991, 976, 1009, 1046, 30, 8, 7, 0, 0, 0, 0, 0, 1043, 990, 981, 1012, 1025, 1042, 1056, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1013, 1034, 1014, 1006, 1015, 1013, 1024, 18, 12, 0, 0, 0, 0, 0, 0, 1068, 1045, 974, 1019, 1011, 1009, 1001, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1052, 1000, 1014, 1015, 1058, 1066, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 1029, 977, 1045, 1000, 1054, 1000, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1040, 1025, 1005, 976, 1063, 1016, 1017, 42, 0, 0, 0, 0, 0, 0, 0, 1044, 1004, 1055, 1051, 991, 1038, 1014, 1029, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8188, 8142, 8143, 8247, 8198, 8208, 8237] avg=8192.0000 max=8247 max_rank=4 min=8142 min_rank=2 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8064, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8064, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8064, 8320, 8192, 8064, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1049, 1081, 1078, 1061, 1067, 1020, 1004, 982, 1004, 990, 1044, 1074, 1015, 1032, 1036, 1048, 1055, 1015, 1019, 1026, 1059, 993, 981, 1051, 1007, 1047, 1013, 1022, 1036, 1021, 1023, 1033, 1030, 1012, 1019, 1041, 1013, 990, 1028, 1002, 1057, 1059, 994, 1061, 945, 1066, 991, 1041, 1021, 1013, 1000, 978, 1021, 991, 995, 1037, 1015, 1045, 1027, 1047, 1010, 1015, 997, 989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1031, 968, 1024, 1047, 1040, 994, 1063, 1054, 1043, 1018, 1000, 1016, 1040, 983, 1026, 1006, 994, 1060, 1006, 989, 1028, 1010, 1057, 993, 995, 1037, 1035, 999, 1063, 1021, 1039, 1057, 1018, 1027, 1038, 1036, 1014, 1026, 1024, 1053, 1050, 1041, 1045, 1023, 1031, 1074, 992, 1080, 970, 974, 1054, 1022, 1033, 1006, 1009, 946, 1054, 1033, 1021, 1047, 1011, 1005, 1025, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[989, 993, 1012, 1060, 1051, 1022, 1013, 1012, 988, 1008, 1052, 1037, 1032, 1014, 1045, 1089, 989, 1003, 1057, 1014, 1025, 981, 1008, 1032, 983, 1059, 1030, 1008, 1037, 1027, 1028, 1034, 1021, 1085, 1026, 1018, 1014, 1001, 1027, 1002, 1062, 1017, 1019, 992, 1027, 1036, 1042, 1026, 1031, 1052, 1005, 1048, 998, 1021, 996, 1001, 1025, 1024, 1057, 1016, 993, 1042, 1024, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1027, 1004, 1014, 1051, 1003, 1062, 1039, 1042, 981, 1056, 1082, 1053, 1075, 973, 1033, 986, 1012, 1037, 1078, 990, 1001, 1036, 1029, 1001, 1020, 972, 1024, 1037, 1001, 1071, 1034, 1064, 989, 1057, 1054, 1030, 1035, 1018, 1043, 1073, 1032, 993, 1063, 986, 974, 1050, 1030, 1047, 1007, 1016, 1002, 1039, 1012, 1055, 1017, 1009, 993, 990, 1023, 1043, 1004, 1001, 987, 976] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1068, 1035, 1022, 967, 1009, 990, 979, 1058, 1011, 1003, 1027, 1065, 1023, 1041, 1011, 1042, 1013, 1052, 1020, 1071, 1001, 992, 1006, 1025, 1028, 1069, 1037, 1031, 1053, 1054, 978, 975, 982, 1032, 1017, 1058, 1021, 1002, 1028, 1045, 1011, 1018, 1025, 1036, 999, 1003, 1039, 1008, 982, 1032, 1052, 1048, 1034, 1040, 1058, 999, 1067, 1029, 993, 1033, 1032, 1013, 1087, 957] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1030, 1018, 1004, 1005, 1097, 1056, 1075, 1002, 1044, 978, 1029, 1028, 1006, 981, 968, 987, 1047, 1008, 1042, 1023, 1024, 1021, 1019, 995, 1011, 1011, 1050, 981, 1020, 1033, 1067, 1029, 1005, 1052, 1016, 1015, 1038, 1039, 1015, 1029, 973, 1087, 1034, 1037, 1038, 1047, 1018, 1070, 991, 998, 999, 984, 1007, 1054, 980, 1034, 1029, 1086, 1053, 1068, 1002, 1029, 1019, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1022, 1055, 1014, 1054, 1021, 978, 1066, 1067, 1021, 1002, 1056, 1011, 1047, 997, 1096, 1062, 956, 1012, 996, 1011, 1055, 1010, 1062, 969, 1008, 1023, 1002, 1007, 1029, 998, 1026, 982, 1002, 1016, 1022, 1044, 980, 1088, 1081, 1066, 995, 1087, 988, 993, 1046, 1050, 1000, 1030, 975, 995, 1024, 1008, 1069, 1028, 992, 1022, 1048, 1028, 1007, 1010, 1040, 1058, 1007, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1034, 1018, 1025, 1017, 994, 1034, 1059, 1012, 1082, 1034, 986, 1049, 1026, 1044, 1027, 1031, 985, 1014, 1015, 1024, 1040, 995, 1032, 1016, 1026, 1020, 998, 1006, 1037, 1025, 1048, 982, 1065, 1065, 1014, 1029, 1026, 974, 1020, 1042, 993, 1036, 1019, 1061, 1026, 1024, 1019, 1025, 1035, 1018, 1031, 1042, 1053, 1060, 1020, 965, 1022, 1024, 1000, 1004, 1007, 1025, 1023, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1027, 1004, 1014, 1051, 1003, 1062, 1001, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1056, 1082, 1035, 1075, 973, 1033, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1037, 1078, 990, 1001, 1036, 1029, 1001, 30, 9, 4, 0, 0, 0, 0, 0, 1020, 972, 1024, 1037, 1001, 1071, 1034, 1064, 1, 0, 0, 0, 0, 0, 0, 0, 989, 1025, 1054, 1030, 1035, 1018, 1043, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 993, 1063, 986, 974, 1020, 1030, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1016, 1002, 1039, 1012, 1055, 1017, 1009, 38, 28, 0, 0, 0, 0, 0, 0, 993, 990, 1023, 1043, 1004, 1001, 987, 976, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8221, 8227, 8224, 8267, 8145, 8223, 8025] avg=8192.0000 max=8267 max_rank=4 min=8025 min_rank=7 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32660 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[989, 993, 1012, 1060, 1051, 1022, 976, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1008, 1052, 1019, 1032, 1014, 1045, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1003, 1057, 1014, 1025, 981, 1008, 1032, 30, 9, 4, 0, 0, 0, 0, 0, 983, 1059, 1030, 1008, 1037, 1027, 1028, 1034, 1, 0, 0, 0, 0, 0, 0, 0, 1021, 1053, 1026, 1018, 1014, 1001, 1027, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1017, 1019, 992, 1027, 1006, 1042, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1052, 1005, 1048, 998, 1021, 996, 1001, 37, 28, 0, 0, 0, 0, 0, 0, 1025, 1024, 1057, 1016, 993, 1042, 1024, 1056, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8247, 8152, 8207, 8162, 8191, 8217, 8245] avg=8192.0000 max=8247 max_rank=1 min=8115 min_rank=0 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1049, 1081, 1078, 1061, 1067, 1020, 967, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 990, 1044, 1056, 1015, 1032, 1036, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1015, 1019, 1026, 1059, 993, 981, 1051, 31, 9, 4, 0, 0, 0, 0, 0, 1007, 1047, 1013, 1022, 1036, 1021, 1023, 1033, 1, 0, 0, 0, 0, 0, 0, 0, 1030, 982, 1019, 1041, 1013, 990, 1028, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1059, 994, 1061, 945, 1035, 991, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1013, 1000, 978, 1021, 991, 995, 1037, 37, 26, 0, 0, 0, 0, 0, 0, 1015, 1045, 1027, 1047, 1010, 1015, 997, 989, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8305, 8225, 8243, 8203, 8105, 8183, 8119, 8153] avg=8192.0000 max=8305 max_rank=0 min=8105 min_rank=4 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32560 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1031, 968, 1024, 1047, 1040, 994, 1024, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1018, 1000, 998, 1040, 983, 1026, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1060, 1006, 989, 1028, 1010, 1057, 993, 31, 9, 4, 0, 0, 0, 0, 0, 995, 1037, 1035, 999, 1063, 1021, 1039, 1057, 1, 0, 0, 0, 0, 0, 0, 0, 1018, 996, 1038, 1036, 1014, 1026, 1024, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1041, 1045, 1023, 1031, 1043, 992, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 970, 974, 1054, 1022, 1033, 1006, 1009, 946, 39, 27, 0, 0, 0, 0, 0, 0, 1054, 1033, 1021, 1047, 1011, 1005, 1025, 1018, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8114, 8181, 8247, 8205, 8305, 8080, 8222] avg=8192.0000 max=8305 max_rank=5 min=8080 min_rank=6 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1068, 1035, 1022, 967, 1009, 990, 943, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1003, 1027, 1047, 1023, 1041, 1011, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1052, 1020, 1071, 1001, 992, 1006, 1025, 29, 9, 4, 0, 0, 0, 0, 0, 1028, 1069, 1037, 1031, 1053, 1054, 978, 975, 1, 0, 0, 0, 0, 0, 0, 0, 982, 1001, 1017, 1058, 1021, 1002, 1028, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1018, 1025, 1036, 999, 974, 1039, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1032, 1052, 1048, 1034, 1040, 1058, 999, 36, 27, 0, 0, 0, 0, 0, 0, 1067, 1029, 993, 1033, 1032, 1013, 1087, 957, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8092, 8205, 8222, 8226, 8154, 8110, 8308, 8219] avg=8192.0000 max=8308 max_rank=6 min=8092 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1030, 1018, 1004, 1005, 1097, 1056, 1036, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 978, 1029, 1010, 1006, 981, 968, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1008, 1042, 1023, 1024, 1021, 1019, 995, 30, 9, 4, 0, 0, 0, 0, 0, 1011, 1011, 1050, 981, 1020, 1033, 1067, 1029, 1, 0, 0, 0, 0, 0, 0, 0, 1005, 1021, 1016, 1015, 1038, 1039, 1015, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1087, 1034, 1037, 1038, 1017, 1018, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 991, 998, 999, 984, 1007, 1054, 980, 1034, 39, 27, 0, 0, 0, 0, 0, 0, 1029, 1086, 1053, 1068, 1002, 1029, 1019, 1001, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8003, 8222, 8203, 8178, 8274, 8113, 8295] avg=8192.0000 max=8295 max_rank=7 min=8003 min_rank=1 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1022, 1055, 1014, 1054, 1021, 978, 1027, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1002, 1056, 993, 1047, 997, 1096, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1012, 996, 1011, 1055, 1010, 1062, 969, 30, 9, 4, 0, 0, 0, 0, 0, 1008, 1023, 1002, 1007, 1029, 998, 1026, 982, 1, 0, 0, 0, 0, 0, 0, 0, 1002, 985, 1022, 1044, 980, 1088, 1081, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1087, 988, 993, 1046, 1020, 1000, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 975, 995, 1024, 1008, 1069, 1028, 992, 1022, 39, 27, 0, 0, 0, 0, 0, 0, 1048, 1028, 1007, 1010, 1040, 1058, 1007, 1022, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8274, 8114, 8076, 8268, 8159, 8179, 8228] avg=8192.0000 max=8274 max_rank=1 min=8076 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32834 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1034, 1018, 1025, 1017, 994, 1034, 1018, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 1034, 986, 1017, 1026, 1044, 1027, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1014, 1015, 1024, 1040, 995, 1032, 1016, 34, 14, 6, 0, 0, 0, 0, 0, 1026, 1020, 998, 1006, 1037, 1025, 1048, 982, 8, 0, 0, 0, 0, 0, 0, 0, 1065, 1027, 1014, 1029, 1026, 974, 1020, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1036, 1019, 1061, 1026, 990, 1019, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1018, 1031, 1042, 1053, 1060, 1020, 965, 41, 32, 0, 0, 0, 0, 0, 0, 1022, 1024, 1000, 1004, 1007, 1025, 1023, 1034, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8247, 8175, 8150, 8197, 8169, 8297, 8149] avg=8192.0000 max=8297 max_rank=6 min=8149 min_rank=7 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8448, 8192, 8192, 8192, 8192, 8192, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8192, 8192, 8192, 8320, 8320, 8192, 8064, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8320, 8192, 8320, 8064, 8320, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1027, 1034, 997, 1021, 1015, 997, 1018, 1000, 1050, 1019, 1117, 1006, 991, 956, 1059, 1050, 1046, 977, 1063, 996, 1000, 1045, 998, 1032, 986, 1060, 1067, 1064, 1021, 1024, 1069, 1021, 993, 1049, 1000, 1040, 1036, 986, 1032, 1022, 1094, 1008, 972, 1027, 1028, 1059, 1062, 1027, 960, 1009, 990, 980, 1043, 1020, 1044, 1041, 1068, 973, 1026, 1027, 1039, 990, 998, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1016, 1073, 1063, 1061, 965, 1077, 1012, 1020, 992, 980, 1020, 990, 1044, 1049, 1098, 1034, 1054, 1051, 1102, 972, 995, 991, 1010, 1027, 1043, 1012, 1034, 1003, 1039, 974, 1021, 1004, 1004, 1021, 1021, 999, 985, 1044, 1013, 989, 1032, 1020, 1014, 1014, 1025, 1018, 992, 1069, 994, 1029, 1047, 989, 1071, 952, 1035, 1029, 1041, 1040, 1040, 984, 1031, 1043, 1062, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1029, 1031, 984, 1015, 1010, 1026, 1004, 1006, 999, 988, 1030, 1023, 971, 1024, 1068, 1006, 975, 1014, 1058, 992, 1012, 1054, 990, 999, 1033, 1040, 1079, 1000, 993, 1043, 1006, 1024, 988, 1053, 1063, 987, 1115, 1067, 1053, 1020, 1018, 1073, 1029, 1032, 1061, 1005, 1032, 998, 1008, 1047, 1030, 1033, 1016, 1044, 1005, 991, 1024, 1046, 1061, 1002, 1022, 1064, 989, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1008, 1031, 1057, 979, 1001, 1004, 1058, 1041, 999, 1046, 992, 1065, 1029, 1069, 1023, 1030, 1012, 1003, 981, 1018, 1018, 1048, 993, 1028, 1047, 1056, 993, 1034, 1029, 1027, 1017, 1006, 1091, 1037, 1017, 1000, 1036, 1058, 1040, 1045, 977, 1048, 1067, 1017, 974, 1045, 1037, 1042, 1008, 965, 962, 1041, 1048, 1006, 1029, 1017, 1005, 994, 1041, 1020, 1006, 1022, 1041, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1016, 986, 1047, 1007, 1036, 996, 990, 1064, 1027, 1009, 996, 1055, 1056, 983, 1040, 989, 1036, 1039, 1070, 1024, 1033, 1043, 1046, 986, 1023, 1036, 1005, 1042, 1018, 1010, 1064, 1036, 1042, 1061, 1049, 1028, 970, 1034, 992, 1012, 1023, 1064, 1026, 1033, 1019, 1059, 1094, 976, 1067, 1031, 1008, 1033, 973, 1008, 996, 1014, 1007, 1029, 1009, 1013, 1069, 1014, 982, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1049, 1049, 1016, 1021, 991, 1039, 971, 1007, 1021, 1062, 1067, 1042, 1005, 1067, 1027, 1014, 975, 998, 1041, 1031, 1028, 1033, 1026, 1031, 989, 988, 989, 1011, 993, 1013, 1035, 1039, 1022, 998, 1023, 1043, 1061, 977, 980, 1027, 1058, 1050, 1041, 1022, 1027, 1063, 1023, 1036, 1019, 998, 1034, 1050, 1043, 1034, 1027, 1006, 971, 1071, 1004, 1038, 1025, 1027, 1056, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1019, 1089, 1007, 1025, 1057, 1032, 1071, 1038, 1055, 979, 1027, 983, 1071, 991, 1047, 1004, 1021, 991, 1029, 1076, 999, 1020, 1053, 984, 1026, 1002, 977, 1021, 1022, 1054, 1028, 1036, 1007, 1058, 1027, 1036, 1011, 984, 1044, 978, 1016, 1034, 999, 1006, 1041, 1013, 1040, 1007, 1053, 972, 1072, 1021, 1002, 1035, 966, 1025, 1070, 1029, 1028, 1051, 1042, 1002, 1018, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1037, 1007, 1064, 1011, 1028, 1022, 1017, 1050, 1008, 1068, 1022, 983, 1046, 965, 1067, 1014, 995, 1000, 1064, 1018, 1005, 1016, 1042, 1010, 1035, 1061, 1040, 1040, 971, 1011, 1022, 1005, 1009, 1012, 1059, 1041, 992, 1003, 1051, 1017, 1031, 1050, 995, 1005, 1015, 1036, 1068, 1056, 991, 1034, 988, 1030, 1019, 1041, 1062, 984, 1037, 1012, 1055, 1046, 1036, 1006, 1000, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1016, 1073, 1063, 1061, 965, 1077, 1012, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 992, 980, 1020, 990, 1044, 1049, 1090, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1051, 1102, 972, 995, 991, 1010, 1027, 19, 8, 0, 0, 0, 0, 0, 0, 1043, 1012, 1034, 1003, 1039, 974, 1021, 1004, 1, 0, 0, 0, 0, 0, 0, 0, 1004, 1020, 1021, 999, 985, 1044, 1013, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1020, 1014, 1014, 1025, 1018, 944, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1029, 1047, 989, 1071, 952, 1035, 1029, 48, 1, 0, 0, 0, 0, 0, 0, 1041, 1040, 1040, 984, 1011, 1043, 1062, 1063, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 8199, 8229, 8131, 8075, 8136, 8195, 8284] avg=8192.0000 max=8287 max_rank=0 min=8075 min_rank=4 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32690 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1029, 1031, 984, 1015, 1010, 1026, 1004, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 999, 988, 1030, 1023, 971, 1024, 1060, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1014, 1058, 992, 1012, 1054, 990, 999, 19, 8, 0, 0, 0, 0, 0, 0, 1033, 1040, 1079, 1000, 993, 1043, 1006, 1024, 2, 0, 0, 0, 0, 0, 0, 0, 988, 1051, 1063, 987, 1115, 1067, 1053, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1073, 1029, 1032, 1061, 1005, 982, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1047, 1030, 1033, 1016, 1044, 1005, 991, 50, 1, 0, 0, 0, 0, 0, 0, 1024, 1046, 1061, 1002, 1002, 1064, 989, 1034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8101, 8121, 8220, 8344, 8198, 8225, 8222] avg=8192.0000 max=8344 max_rank=4 min=8101 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32989 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1008, 1031, 1057, 979, 1001, 1004, 1058, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1046, 992, 1065, 1029, 1069, 1016, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1003, 981, 1018, 1018, 1048, 993, 1028, 18, 7, 0, 0, 0, 0, 0, 0, 1047, 1056, 993, 1034, 1029, 1027, 1017, 1006, 2, 0, 0, 0, 0, 0, 0, 0, 1091, 1035, 1017, 1000, 1036, 1058, 1040, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1048, 1067, 1017, 974, 1045, 987, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 965, 962, 1041, 1048, 1006, 1029, 1017, 50, 1, 0, 0, 0, 0, 0, 0, 1005, 994, 1041, 1020, 987, 1022, 1041, 1058, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8246, 8126, 8211, 8322, 8157, 8127, 8168] avg=8192.0000 max=8322 max_rank=4 min=8126 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1016, 986, 1047, 1007, 1036, 996, 990, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1009, 996, 1055, 1056, 983, 1032, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1039, 1070, 1024, 1033, 1043, 1046, 986, 19, 8, 0, 0, 0, 0, 0, 0, 1023, 1036, 1005, 1042, 1018, 1010, 1064, 1036, 2, 0, 0, 0, 0, 0, 0, 0, 1042, 1059, 1049, 1028, 970, 1034, 992, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1064, 1026, 1033, 1019, 1059, 1041, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1031, 1008, 1033, 973, 1008, 996, 1014, 53, 1, 0, 0, 0, 0, 0, 0, 1007, 1029, 1009, 1013, 1049, 1014, 982, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8147, 8304, 8236, 8186, 8241, 8184, 8096] avg=8192.0000 max=8304 max_rank=2 min=8096 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1049, 1049, 1016, 1021, 991, 1039, 971, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1062, 1067, 1042, 1005, 1067, 1020, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 975, 998, 1041, 1031, 1028, 1033, 1026, 1031, 19, 7, 0, 0, 0, 0, 0, 0, 989, 988, 989, 1011, 993, 1013, 1035, 1039, 1, 0, 0, 0, 0, 0, 0, 0, 1022, 997, 1023, 1043, 1061, 977, 980, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1050, 1041, 1022, 1027, 1063, 974, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 998, 1034, 1050, 1043, 1034, 1027, 1006, 49, 1, 0, 0, 0, 0, 0, 0, 971, 1071, 1004, 1038, 1005, 1027, 1056, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8298, 8189, 8058, 8130, 8271, 8261, 8186] avg=8192.0000 max=8298 max_rank=1 min=8058 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1019, 1089, 1007, 1025, 1057, 1032, 1071, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 979, 1027, 983, 1071, 991, 1039, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 991, 1029, 1076, 999, 1020, 1053, 984, 19, 8, 0, 0, 0, 0, 0, 0, 1026, 1002, 977, 1021, 1022, 1054, 1028, 1036, 2, 0, 0, 0, 0, 0, 0, 0, 1007, 1056, 1027, 1036, 1011, 984, 1044, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1034, 999, 1006, 1041, 1013, 990, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 972, 1072, 1021, 1002, 1035, 966, 1025, 50, 1, 0, 0, 0, 0, 0, 0, 1070, 1029, 1028, 1051, 1022, 1002, 1018, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8338, 8149, 8200, 8168, 8143, 8106, 8197, 8235] avg=8192.0000 max=8338 max_rank=0 min=8106 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1037, 1004, 1064, 1011, 1028, 1022, 1017, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1068, 1022, 983, 1046, 965, 1050, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1000, 1064, 1018, 1005, 1016, 1042, 1010, 22, 11, 0, 0, 0, 0, 0, 0, 1035, 1061, 1040, 1040, 971, 1011, 1022, 1005, 4, 6, 3, 0, 0, 0, 0, 0, 1009, 1008, 1059, 1041, 992, 1003, 1051, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1050, 995, 1005, 1015, 1036, 1013, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1034, 988, 1030, 1019, 1041, 1062, 984, 55, 4, 0, 0, 0, 0, 0, 0, 1037, 1012, 1055, 1046, 1010, 1006, 1000, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8156, 8183, 8198, 8180, 8201, 8208, 8177] avg=8192.0000 max=8233 max_rank=0 min=8156 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1027, 1034, 997, 1021, 1015, 997, 1018, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1019, 1117, 1006, 991, 956, 1051, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 977, 1063, 996, 1000, 1045, 998, 1032, 19, 8, 0, 0, 0, 0, 0, 0, 986, 1060, 1067, 1064, 1021, 1024, 1069, 1021, 2, 0, 0, 0, 0, 0, 0, 0, 993, 1047, 1000, 1040, 1036, 986, 1032, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 1008, 972, 1027, 1028, 1059, 1011, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 960, 1009, 990, 980, 1043, 1020, 1044, 1041, 51, 1, 0, 0, 0, 0, 0, 0, 1068, 973, 1026, 1027, 1019, 990, 998, 1067, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8240, 8184, 8314, 8156, 8226, 8139, 8168] avg=8192.0000 max=8314 max_rank=3 min=8109 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32689 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8448, 8192, 8192, 8192, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8192, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8448, 8192, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1054, 1018, 1001, 1027, 1036, 998, 1040, 1025, 1045, 1067, 1079, 1031, 1021, 996, 1026, 1021, 996, 1015, 1053, 1033, 1068, 1030, 1017, 1045, 1030, 1022, 1061, 992, 1007, 986, 1031, 997, 1058, 1011, 1031, 1015, 1034, 1019, 1022, 983, 1010, 1009, 1008, 1019, 1062, 1000, 1003, 1016, 1045, 1036, 1034, 1010, 1060, 1041, 1064, 990, 977, 1019, 1029, 1037, 965, 1023, 999, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[979, 1019, 1023, 1017, 1019, 1040, 1028, 1032, 1031, 1018, 968, 1000, 1033, 1000, 983, 1041, 1043, 1022, 1007, 1038, 1053, 1012, 1044, 1019, 1022, 998, 997, 1011, 1014, 1018, 1078, 1029, 1042, 1039, 1006, 1069, 1001, 996, 1016, 1069, 999, 1010, 980, 1044, 1006, 1072, 983, 1063, 1037, 1008, 1056, 1070, 1026, 995, 1052, 1052, 1040, 1046, 1012, 1002, 1023, 1050, 1009, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[998, 1019, 1048, 1030, 1042, 1007, 1046, 999, 1010, 1051, 994, 1079, 1017, 1033, 1067, 1077, 1035, 1012, 982, 1083, 1003, 1020, 994, 1057, 1003, 1054, 1067, 1019, 997, 1007, 1042, 1071, 1020, 1029, 1028, 1035, 1061, 1069, 978, 976, 1019, 1003, 986, 1014, 996, 998, 1038, 1020, 1041, 1022, 1012, 960, 1051, 1023, 1073, 1011, 1008, 1051, 1001, 993, 1016, 1016, 1007, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1073, 1070, 998, 1040, 1006, 1033, 1002, 1026, 1055, 1046, 1013, 1024, 1025, 1042, 1001, 1019, 1021, 983, 1049, 991, 984, 1026, 1025, 1055, 1020, 1021, 1022, 1001, 1000, 1050, 1018, 1008, 1068, 1068, 1054, 998, 1010, 994, 1014, 978, 974, 1030, 1078, 1013, 1023, 1044, 1024, 1012, 1048, 1060, 1030, 1041, 1017, 1046, 1042, 1026, 1034, 1004, 1013, 997, 1010, 1024, 1016, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[981, 1003, 1052, 993, 1030, 1061, 1015, 988, 1018, 1003, 991, 1014, 1006, 1074, 1057, 1011, 994, 1053, 991, 1002, 1047, 1065, 963, 1035, 1041, 1029, 1012, 1041, 1010, 1065, 1055, 1040, 1037, 1060, 1024, 1035, 990, 1049, 1040, 1067, 1062, 1036, 1045, 1044, 1032, 1041, 1026, 992, 975, 1040, 1058, 1015, 1042, 994, 1051, 1013, 1000, 1020, 967, 1005, 951, 1059, 1032, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1021, 1087, 1052, 1041, 988, 1067, 1050, 992, 1025, 1016, 982, 993, 990, 1023, 1034, 1026, 1002, 1004, 1003, 1000, 982, 1056, 969, 1040, 970, 1042, 1027, 1048, 1017, 1063, 1074, 1028, 1023, 1063, 983, 1038, 1008, 1023, 1046, 1001, 1041, 1022, 1044, 1040, 1045, 1031, 995, 1052, 968, 999, 1015, 1022, 1050, 1027, 1063, 1036, 1001, 989, 969, 1083, 1093, 1037, 1015, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1108, 1008, 1028, 971, 1004, 1011, 1057, 1009, 1017, 1039, 1005, 1052, 1027, 978, 1019, 1046, 981, 1010, 1041, 1028, 1028, 1087, 1054, 1030, 1059, 1019, 991, 997, 1000, 970, 1042, 1035, 1006, 1021, 1020, 993, 1025, 1058, 1006, 1045, 1014, 1023, 1003, 980, 1008, 1054, 1052, 1040, 1049, 996, 994, 1025, 1022, 1009, 1052, 1082, 1004, 1076, 1005, 990, 1036, 1020, 1057, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1008, 980, 1019, 995, 1008, 1031, 980, 1031, 1013, 995, 994, 1062, 1039, 1030, 1042, 1044, 1016, 1034, 1014, 1039, 1037, 1035, 1049, 1002, 1001, 1027, 1082, 1036, 1021, 1046, 1008, 1021, 1003, 1023, 985, 1057, 992, 1020, 1047, 1065, 1003, 1052, 989, 1012, 1055, 965, 1035, 1010, 1006, 1006, 1007, 1071, 1005, 956, 1033, 989, 1085, 1079, 977, 1053, 1096, 1042, 1046, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1073, 1070, 998, 1040, 1006, 1033, 1002, 1026, 7, 1, 0, 0, 0, 0, 0, 0, 1055, 1046, 1013, 1024, 1025, 1042, 1001, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 983, 1049, 991, 984, 1026, 1025, 1055, 3, 0, 0, 0, 0, 0, 0, 0, 1020, 1021, 1022, 1001, 1000, 1050, 1010, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1055, 1054, 998, 1010, 994, 1014, 978, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1030, 1078, 1013, 1023, 1044, 1024, 1012, 13, 2, 1, 0, 0, 0, 0, 0, 1048, 1060, 1030, 1041, 1017, 1046, 1016, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1004, 1013, 997, 1010, 1024, 1016, 999, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8221, 8137, 8132, 8171, 8214, 8284, 8121] avg=8192.0000 max=8284 max_rank=6 min=8121 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1054, 1018, 1001, 1027, 1036, 998, 1040, 1025, 7, 1, 0, 0, 0, 0, 0, 0, 1045, 1067, 1079, 1031, 1021, 996, 1026, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1015, 1053, 1033, 1068, 1030, 1017, 1045, 3, 0, 0, 0, 0, 0, 0, 0, 1030, 1022, 1061, 992, 1007, 986, 1023, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 998, 1031, 1015, 1034, 1019, 1022, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1009, 1008, 1019, 1062, 1000, 1003, 1016, 13, 2, 1, 0, 0, 0, 0, 0, 1045, 1036, 1034, 1010, 1060, 1041, 1038, 990, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1019, 1029, 1037, 965, 1023, 999, 1039, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8282, 8260, 8118, 8160, 8143, 8254, 8112] avg=8192.0000 max=8282 max_rank=1 min=8112 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[979, 1019, 1023, 1017, 1019, 1040, 1028, 1032, 8, 1, 0, 0, 0, 0, 0, 0, 1031, 1018, 968, 1000, 1033, 1000, 983, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1022, 1007, 1038, 1053, 1012, 1044, 1019, 3, 0, 0, 0, 0, 0, 0, 0, 1022, 998, 997, 1011, 1014, 1018, 1069, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1026, 1006, 1069, 1001, 996, 1016, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1010, 980, 1044, 1006, 1072, 983, 1063, 13, 2, 1, 0, 0, 0, 0, 0, 1037, 1008, 1056, 1070, 1026, 995, 1026, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1046, 1012, 1002, 1023, 1050, 1009, 1027, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8070, 8241, 8158, 8225, 8173, 8270, 8233] avg=8192.0000 max=8270 max_rank=6 min=8070 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32901 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[981, 1003, 1052, 993, 1030, 1061, 1015, 988, 7, 1, 0, 0, 0, 0, 0, 0, 1018, 1003, 991, 1014, 1006, 1074, 1057, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1053, 991, 1002, 1047, 1065, 963, 1035, 3, 0, 0, 0, 0, 0, 0, 0, 1041, 1029, 1012, 1041, 1010, 1065, 1047, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1047, 1024, 1035, 990, 1049, 1040, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1036, 1045, 1044, 1032, 1041, 1026, 992, 13, 2, 1, 0, 0, 0, 0, 0, 975, 1040, 1058, 1015, 1042, 994, 1025, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1020, 967, 1005, 951, 1059, 1032, 994, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8170, 8153, 8285, 8289, 8294, 8162, 8052] avg=8192.0000 max=8294 max_rank=5 min=8052 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1008, 980, 1019, 995, 1008, 1031, 980, 1031, 7, 1, 0, 0, 0, 0, 0, 0, 1013, 995, 994, 1062, 1039, 1030, 1042, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1034, 1014, 1039, 1037, 1035, 1049, 1002, 3, 0, 0, 0, 0, 0, 0, 0, 1001, 1027, 1082, 1036, 1021, 1046, 1000, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1010, 985, 1057, 992, 1020, 1047, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1052, 989, 1012, 1055, 965, 1035, 1010, 13, 2, 1, 0, 0, 0, 0, 0, 1006, 1006, 1007, 1071, 1005, 956, 1007, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 1079, 977, 1053, 1096, 1042, 1046, 1033, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8060, 8215, 8229, 8234, 8179, 8137, 8047, 8435] avg=8192.0000 max=8435 max_rank=7 min=8047 min_rank=6 imbalance_ratio=1.029663 max_load_reduction=57101 max_load_reduction_pct=0.871292 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[998, 1019, 1048, 1030, 1042, 1007, 1046, 999, 7, 1, 0, 0, 0, 0, 0, 0, 1010, 1051, 994, 1079, 1017, 1033, 1067, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1012, 982, 1083, 1003, 1020, 994, 1057, 3, 0, 0, 0, 0, 0, 0, 0, 1003, 1054, 1067, 1019, 997, 1007, 1034, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1016, 1028, 1035, 1061, 1069, 978, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1003, 986, 1014, 996, 998, 1038, 1020, 13, 2, 1, 0, 0, 0, 0, 0, 1041, 1022, 1012, 960, 1051, 1023, 1047, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1051, 1001, 993, 1016, 1016, 1007, 1018, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8324, 8189, 8252, 8183, 8090, 8167, 8134] avg=8192.0000 max=8324 max_rank=1 min=8090 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32574 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1021, 1087, 1052, 1041, 988, 1067, 1050, 992, 7, 1, 0, 0, 0, 0, 0, 0, 1025, 1016, 982, 993, 990, 1023, 1034, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1004, 1003, 1000, 982, 1056, 969, 1040, 3, 0, 0, 0, 0, 0, 0, 0, 970, 1042, 1027, 1048, 1017, 1063, 1066, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1050, 983, 1038, 1008, 1023, 1046, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1022, 1044, 1040, 1045, 1031, 995, 1052, 13, 2, 1, 0, 0, 0, 0, 0, 968, 999, 1015, 1022, 1050, 1027, 1037, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 989, 969, 1083, 1093, 1037, 1015, 1002, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 8085, 8059, 8261, 8172, 8286, 8154, 8213] avg=8192.0000 max=8306 max_rank=0 min=8059 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32825 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1108, 1008, 1028, 971, 1004, 1011, 1057, 1009, 12, 5, 0, 0, 0, 0, 0, 0, 1017, 1039, 1005, 1052, 1027, 978, 1019, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1010, 1041, 1028, 1028, 1087, 1054, 1030, 9, 0, 0, 0, 0, 0, 0, 0, 1059, 1019, 991, 997, 1000, 970, 1025, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1004, 1020, 993, 1025, 1058, 1006, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1023, 1003, 980, 1008, 1054, 1052, 1040, 17, 3, 5, 0, 0, 0, 0, 0, 1049, 996, 994, 1025, 1022, 1009, 1021, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1076, 1005, 990, 1036, 1020, 1057, 1020, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8169, 8268, 8096, 8157, 8199, 8198, 8236] avg=8192.0000 max=8268 max_rank=2 min=8096 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8064, 8192, 8192, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8448, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1025, 1021, 1047, 1059, 982, 1008, 1087, 1081, 987, 1027, 1022, 1050, 1021, 984, 1018, 1070, 1082, 999, 993, 1064, 1008, 1015, 1016, 1020, 982, 1010, 1028, 1016, 994, 1000, 1027, 985, 1028, 1015, 993, 993, 1029, 1053, 1030, 1014, 1049, 1005, 1027, 1022, 1052, 1023, 1000, 1008, 1037, 1051, 972, 997, 1064, 1071, 1007, 1028, 1034, 1055, 1009, 1050, 987, 1030, 1023, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1030, 1039, 1027, 1042, 1033, 976, 997, 1052, 1034, 1023, 1066, 996, 988, 1066, 1018, 1061, 1053, 949, 995, 968, 1002, 1043, 1044, 1007, 1026, 995, 1020, 1016, 1000, 1040, 1034, 1028, 1010, 1033, 1045, 1034, 1021, 1062, 991, 1090, 1019, 1006, 1040, 993, 997, 1063, 1000, 1057, 1026, 1067, 1043, 1058, 1073, 1048, 1000, 1047, 1024, 1016, 957, 1014, 986, 1021, 996, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1039, 1008, 1012, 978, 1078, 1003, 1022, 1015, 962, 995, 1016, 1009, 1047, 1011, 959, 1055, 1024, 1026, 1022, 1042, 1034, 1005, 995, 1029, 1021, 1030, 994, 1083, 984, 1043, 1024, 1004, 1065, 1025, 1059, 964, 988, 1030, 1040, 1023, 1031, 1067, 1028, 1025, 1054, 1035, 1002, 1035, 1042, 1009, 1013, 978, 1003, 1048, 995, 1033, 1067, 1067, 1053, 1067, 1051, 1010, 1050, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1003, 971, 1023, 998, 998, 1079, 1015, 1020, 1010, 1039, 1003, 1045, 990, 1016, 1004, 1040, 1060, 1053, 1007, 1044, 999, 991, 995, 1028, 1019, 1068, 1031, 1041, 1041, 1015, 1046, 987, 1040, 1035, 1024, 1030, 999, 1015, 1049, 1054, 1052, 1072, 1024, 1041, 1058, 1025, 1039, 1001, 1001, 1024, 1022, 1009, 990, 999, 1017, 1027, 1007, 1011, 997, 1051, 1047, 1047, 1061, 989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[976, 971, 1058, 1022, 1041, 1007, 1046, 1055, 1004, 1022, 1015, 1034, 1023, 1055, 1014, 1053, 1028, 1009, 1077, 1018, 1009, 995, 1022, 1065, 1047, 1013, 1030, 1041, 1006, 1027, 1009, 1078, 998, 1014, 989, 992, 1098, 1006, 1050, 993, 983, 1042, 981, 1020, 1008, 1005, 1049, 1069, 1091, 1024, 1000, 1019, 1009, 1007, 1043, 1023, 979, 961, 1065, 978, 1058, 1011, 1063, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1048, 987, 1005, 1051, 1068, 1000, 1018, 997, 1059, 1028, 1026, 1035, 1050, 1015, 1018, 1019, 991, 1017, 1030, 1045, 1028, 1024, 1008, 1002, 1012, 1035, 1058, 1045, 1009, 1046, 1085, 1054, 1014, 1052, 1009, 999, 1020, 982, 983, 993, 1072, 1049, 1025, 1053, 1009, 986, 1026, 1024, 1016, 988, 1036, 998, 967, 1027, 1030, 999, 1085, 1064, 983, 983, 1026, 1065, 1031, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1030, 1012, 1016, 1011, 981, 1001, 1026, 990, 1012, 1025, 1075, 988, 1028, 1047, 1000, 1036, 1047, 1010, 1026, 1050, 1043, 1024, 993, 1023, 1026, 1020, 1066, 978, 1030, 1047, 1056, 1003, 985, 1029, 1012, 1025, 1052, 976, 1068, 1046, 988, 1007, 1009, 1057, 1063, 999, 1029, 1016, 976, 1027, 1090, 1033, 1028, 1005, 1077, 983, 1049, 1047, 1004, 1007, 997, 1050, 1002, 1080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1050, 1067, 1026, 1020, 1032, 976, 984, 1060, 1009, 978, 1020, 987, 1065, 998, 971, 1030, 998, 1042, 967, 1046, 1030, 1048, 989, 1038, 1099, 1040, 1055, 1013, 1023, 979, 981, 1043, 1029, 1047, 1057, 1026, 1010, 1020, 1031, 1051, 1051, 1060, 976, 976, 984, 1053, 1062, 1065, 999, 994, 1038, 1031, 1056, 983, 1004, 1048, 997, 1004, 1035, 1001, 1066, 1038, 1032, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1025, 1021, 1047, 1059, 982, 1008, 1087, 1081, 9, 7, 0, 0, 0, 0, 0, 0, 987, 1027, 1022, 1050, 1021, 984, 1018, 1070, 17, 3, 0, 0, 0, 0, 0, 0, 1082, 999, 993, 1064, 1008, 1015, 1016, 1020, 22, 0, 0, 0, 0, 0, 0, 0, 982, 1010, 1011, 1016, 994, 1000, 1027, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1015, 993, 993, 1029, 1053, 1030, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 980, 1027, 1022, 1052, 1023, 1000, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1051, 972, 997, 1064, 1071, 1007, 1028, 10, 0, 0, 0, 0, 0, 0, 0, 1034, 1055, 1009, 1050, 987, 1030, 1023, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8326, 8199, 8219, 8025, 8155, 8161, 8237, 8214] avg=8192.0000 max=8326 max_rank=0 min=8025 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1030, 1039, 1027, 1042, 1033, 976, 997, 1052, 9, 7, 0, 0, 0, 0, 0, 0, 1034, 1023, 1066, 996, 988, 1066, 1018, 1061, 17, 3, 0, 0, 0, 0, 0, 0, 1053, 949, 995, 968, 1002, 1043, 1044, 1007, 22, 0, 0, 0, 0, 0, 0, 0, 1026, 995, 1003, 1016, 1000, 1040, 1034, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1033, 1045, 1034, 1021, 1062, 991, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 981, 1040, 993, 997, 1063, 1000, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1067, 1043, 1058, 1073, 1048, 1000, 1047, 10, 0, 0, 0, 0, 0, 0, 0, 1024, 1016, 957, 1014, 986, 1021, 996, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8272, 8083, 8142, 8286, 8150, 8372, 8019] avg=8192.0000 max=8372 max_rank=6 min=8019 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1003, 971, 1023, 998, 998, 1079, 1015, 1020, 8, 7, 0, 0, 0, 0, 0, 0, 1010, 1039, 1003, 1045, 990, 1016, 1004, 1040, 16, 3, 0, 0, 0, 0, 0, 0, 1060, 1053, 1007, 1044, 999, 991, 995, 1028, 23, 0, 0, 0, 0, 0, 0, 0, 1019, 1068, 1014, 1041, 1041, 1015, 1046, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1035, 1024, 1030, 999, 1015, 1049, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1046, 1024, 1041, 1058, 1025, 1039, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1024, 1022, 1009, 990, 999, 1017, 1027, 10, 0, 0, 0, 0, 0, 0, 0, 1007, 1011, 997, 1051, 1047, 1047, 1061, 965, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8166, 8200, 8231, 8246, 8286, 8099, 8186] avg=8192.0000 max=8286 max_rank=5 min=8099 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1039, 1008, 1012, 978, 1078, 1003, 1022, 1015, 8, 7, 0, 0, 0, 0, 0, 0, 962, 995, 1016, 1009, 1047, 1011, 959, 1055, 16, 3, 0, 0, 0, 0, 0, 0, 1024, 1026, 1022, 1042, 1034, 1005, 995, 1029, 23, 0, 0, 0, 0, 0, 0, 0, 1021, 1030, 977, 1083, 984, 1043, 1024, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1025, 1059, 964, 988, 1030, 1040, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1041, 1028, 1025, 1054, 1035, 1002, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1009, 1013, 978, 1003, 1048, 995, 1033, 10, 0, 0, 0, 0, 0, 0, 0, 1067, 1067, 1053, 1067, 1051, 1010, 1050, 986, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8073, 8200, 8166, 8194, 8251, 8131, 8351] avg=8192.0000 max=8351 max_rank=7 min=8073 min_rank=1 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32927 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[976, 971, 1058, 1022, 1041, 1007, 1046, 1055, 9, 7, 0, 0, 0, 0, 0, 0, 1004, 1022, 1015, 1034, 1023, 1055, 1014, 1053, 17, 3, 0, 0, 0, 0, 0, 0, 1028, 1009, 1077, 1018, 1009, 995, 1022, 1065, 22, 0, 0, 0, 0, 0, 0, 0, 1047, 1013, 1013, 1041, 1006, 1027, 1009, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1014, 989, 992, 1098, 1006, 1050, 993, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1017, 981, 1020, 1008, 1005, 1049, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 1024, 1000, 1019, 1009, 1007, 1043, 1023, 10, 0, 0, 0, 0, 0, 0, 0, 979, 961, 1065, 978, 1058, 1011, 1063, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8240, 8245, 8234, 8140, 8132, 8226, 8127] avg=8192.0000 max=8245 max_rank=2 min=8127 min_rank=7 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32625 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1048, 987, 1005, 1051, 1068, 1000, 1018, 997, 9, 8, 0, 0, 0, 0, 0, 0, 1059, 1028, 1026, 1035, 1050, 1015, 1018, 1019, 17, 3, 0, 0, 0, 0, 0, 0, 991, 1017, 1030, 1045, 1028, 1024, 1008, 1002, 22, 0, 0, 0, 0, 0, 0, 0, 1012, 1035, 1039, 1045, 1009, 1046, 1085, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1052, 1009, 999, 1020, 982, 983, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1024, 1025, 1053, 1009, 986, 1026, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 988, 1036, 998, 967, 1027, 1030, 999, 11, 0, 0, 0, 0, 0, 0, 0, 1085, 1064, 983, 983, 1026, 1065, 1031, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8270, 8167, 8325, 8052, 8219, 8072, 8240] avg=8192.0000 max=8325 max_rank=3 min=8052 min_rank=4 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32583 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1030, 1012, 1016, 1011, 981, 1001, 1026, 990, 9, 8, 0, 0, 0, 0, 0, 0, 1012, 1025, 1075, 988, 1028, 1047, 1000, 1036, 17, 3, 0, 0, 0, 0, 0, 0, 1047, 1010, 1026, 1050, 1043, 1024, 993, 1023, 22, 0, 0, 0, 0, 0, 0, 0, 1026, 1020, 1047, 978, 1030, 1047, 1056, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1029, 1012, 1025, 1052, 976, 1068, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 988, 982, 1009, 1057, 1063, 999, 1029, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1027, 1090, 1033, 1028, 1005, 1077, 983, 11, 0, 0, 0, 0, 0, 0, 0, 1049, 1047, 1004, 1007, 997, 1050, 1002, 1054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8084, 8231, 8238, 8207, 8193, 8143, 8230, 8210] avg=8192.0000 max=8238 max_rank=2 min=8084 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1050, 1067, 1026, 1020, 1032, 976, 984, 1060, 12, 12, 0, 0, 0, 0, 0, 0, 1009, 978, 1020, 987, 1065, 998, 971, 1030, 20, 7, 0, 0, 0, 0, 0, 0, 998, 1042, 967, 1046, 1030, 1048, 989, 1038, 26, 0, 0, 0, 0, 0, 0, 0, 1099, 1040, 1028, 1013, 1023, 979, 981, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1047, 1057, 1026, 1010, 1020, 1031, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1027, 976, 976, 984, 1053, 1062, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 999, 994, 1038, 1031, 1056, 983, 1004, 1048, 15, 1, 0, 0, 0, 0, 0, 0, 997, 1004, 1035, 1001, 1066, 1038, 1032, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8085, 8184, 8206, 8270, 8194, 8169, 8189] avg=8192.0000 max=8270 max_rank=4 min=8085 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8064, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8064, 8448, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8064, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8064, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[976, 1093, 1060, 1061, 1072, 1007, 1026, 1035, 1007, 1104, 1006, 1050, 1041, 1051, 1039, 1022, 1037, 1012, 993, 985, 979, 1051, 1032, 1013, 1026, 1042, 991, 1037, 1028, 1003, 1019, 1005, 1033, 981, 1057, 1009, 1010, 1028, 1028, 1045, 1011, 989, 1019, 973, 1024, 1035, 1015, 1048, 1011, 1004, 1061, 1037, 979, 998, 983, 1024, 1012, 1017, 1006, 1070, 1054, 1015, 1033, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[989, 1023, 1006, 1012, 1064, 1061, 991, 1031, 1022, 1057, 1045, 1047, 1027, 952, 1038, 992, 1045, 980, 1010, 977, 1026, 1025, 1015, 1033, 1013, 1019, 1033, 1017, 978, 1057, 1022, 998, 994, 1058, 1029, 1035, 1040, 1049, 1017, 1036, 993, 1052, 1058, 1058, 1007, 1021, 1069, 1044, 1056, 1053, 1029, 1014, 1044, 1016, 971, 1080, 991, 1037, 1042, 980, 1019, 1043, 984, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[983, 1052, 1041, 1030, 1059, 1004, 1047, 959, 1029, 1017, 964, 1030, 1012, 986, 1019, 1016, 1036, 1016, 1011, 967, 1002, 997, 1008, 1018, 1037, 1065, 1020, 1033, 1056, 1054, 1026, 1075, 1011, 1003, 1045, 1070, 1056, 1028, 1043, 1041, 980, 1036, 1019, 1058, 956, 1047, 1006, 1027, 1008, 1024, 1017, 1026, 1024, 978, 1028, 1008, 1050, 1008, 1047, 1063, 991, 1024, 1068, 1077] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1050, 965, 1051, 1046, 1065, 1019, 1067, 1033, 1054, 998, 1055, 1026, 1007, 1008, 1058, 975, 1045, 1022, 1051, 1061, 1028, 1014, 1011, 1046, 1029, 1045, 1053, 1018, 1064, 1042, 1002, 1041, 1041, 1036, 972, 999, 1006, 990, 1033, 1012, 1015, 1028, 1035, 1040, 977, 1038, 1036, 1060, 962, 1029, 1001, 997, 948, 1019, 1022, 1054, 1010, 1019, 1007, 975, 1032, 1007, 1067, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1031, 1054, 1014, 1025, 1064, 1011, 1054, 984, 984, 995, 1024, 1039, 1025, 1008, 1030, 1033, 1064, 1025, 1045, 1042, 993, 1028, 1021, 993, 987, 978, 1056, 1045, 1013, 1011, 1056, 1030, 1062, 1030, 997, 1023, 1011, 995, 1017, 1015, 1041, 1006, 1006, 1027, 1030, 1052, 999, 1007, 1060, 964, 1004, 1046, 1048, 1002, 1009, 1025, 1047, 1037, 1030, 1074, 1057, 1020, 1006, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1012, 1041, 1043, 1034, 1026, 1015, 1079, 1034, 1065, 1043, 1012, 1037, 999, 1046, 1034, 1048, 981, 1017, 1002, 1009, 1048, 1072, 1006, 1041, 987, 974, 987, 1059, 1016, 1063, 1054, 1025, 1043, 1045, 1008, 1015, 992, 1048, 1035, 1024, 1029, 1044, 1011, 997, 1056, 978, 964, 1008, 1006, 1021, 990, 999, 1092, 1013, 1021, 1041, 1010, 1042, 1012, 1004, 1057, 1013, 1024, 985] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1029, 1009, 1023, 1009, 1036, 1012, 967, 1013, 993, 981, 977, 1044, 1025, 1022, 1018, 1102, 1052, 1007, 1043, 1039, 1023, 1055, 1040, 1037, 981, 1009, 1005, 1014, 1068, 1025, 1010, 1075, 955, 1072, 1061, 989, 1048, 1032, 1020, 1036, 1014, 1063, 1050, 1027, 1003, 1001, 1075, 1015, 1013, 1073, 1007, 1019, 1009, 1031, 1019, 1066, 1007, 1058, 1046, 1024, 1003, 990, 983, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1005, 1016, 1049, 1012, 1097, 981, 1015, 1074, 996, 1022, 1049, 1014, 1000, 1013, 1027, 1011, 1014, 1052, 1010, 1043, 1026, 1025, 1046, 1048, 963, 1043, 1038, 1028, 1041, 1067, 990, 1079, 1024, 1020, 1008, 1029, 1018, 1072, 998, 973, 1009, 1039, 1010, 1036, 1007, 1024, 1043, 1009, 1035, 1012, 1009, 1078, 1010, 994, 1006, 1055, 1042, 1042, 926, 1022, 1041, 986, 1031, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[976, 1093, 1060, 1061, 1033, 1007, 1026, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1104, 1006, 1050, 1041, 1051, 1039, 1022, 7, 0, 0, 0, 0, 0, 0, 0, 1037, 1012, 993, 985, 979, 1051, 1032, 1013, 3, 1, 0, 0, 0, 0, 0, 0, 1026, 1042, 991, 1037, 1028, 1003, 1019, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 980, 1057, 1009, 1010, 1028, 1028, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 989, 1019, 973, 1024, 1035, 1015, 1048, 7, 3, 0, 0, 0, 0, 0, 0, 1011, 1004, 1061, 1037, 979, 998, 983, 1024, 32, 0, 0, 0, 0, 0, 0, 0, 1012, 1017, 1006, 1070, 1054, 1015, 1033, 1024, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8327, 8106, 8130, 8190, 8124, 8129, 8239] avg=8192.0000 max=8327 max_rank=1 min=8106 min_rank=2 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[983, 1052, 1041, 1030, 1022, 1004, 1047, 959, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1017, 964, 1030, 1012, 986, 1019, 1016, 8, 0, 0, 0, 0, 0, 0, 0, 1036, 1016, 1011, 967, 1002, 997, 1008, 1018, 3, 1, 0, 0, 0, 0, 0, 0, 1037, 1065, 1020, 1033, 1056, 1054, 1026, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1002, 1045, 1070, 1056, 1028, 1043, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1036, 1019, 1058, 956, 1047, 1006, 1027, 6, 3, 0, 0, 0, 0, 0, 0, 1008, 1024, 1017, 1026, 1024, 978, 1028, 1008, 31, 0, 0, 0, 0, 0, 0, 0, 1050, 1008, 1047, 1063, 991, 1024, 1068, 1077, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8081, 8059, 8344, 8296, 8138, 8144, 8336] avg=8192.0000 max=8344 max_rank=3 min=8059 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1029, 1009, 1023, 1009, 999, 1012, 967, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 993, 981, 977, 1044, 1025, 1022, 1018, 1102, 8, 0, 0, 0, 0, 0, 0, 0, 1052, 1007, 1043, 1039, 1023, 1055, 1040, 1037, 3, 1, 0, 0, 0, 0, 0, 0, 981, 1009, 1005, 1014, 1068, 1025, 1010, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 955, 1071, 1061, 989, 1048, 1032, 1020, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1063, 1050, 1027, 1003, 1001, 1075, 1015, 6, 3, 0, 0, 0, 0, 0, 0, 1013, 1073, 1007, 1019, 1009, 1031, 1019, 1066, 31, 0, 0, 0, 0, 0, 0, 0, 1007, 1058, 1046, 1024, 1003, 990, 983, 984, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8061, 8170, 8300, 8165, 8212, 8257, 8268, 8103] avg=8192.0000 max=8300 max_rank=2 min=8061 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[989, 1023, 1006, 1012, 1026, 1061, 991, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1057, 1045, 1047, 1027, 952, 1038, 992, 7, 0, 0, 0, 0, 0, 0, 0, 1045, 980, 1010, 977, 1026, 1025, 1015, 1033, 3, 1, 0, 0, 0, 0, 0, 0, 1013, 1019, 1033, 1017, 978, 1057, 1022, 977, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1057, 1029, 1035, 1040, 1049, 1017, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1052, 1058, 1058, 1007, 1021, 1069, 1044, 7, 3, 0, 0, 0, 0, 0, 0, 1056, 1053, 1029, 1014, 1044, 1016, 971, 1080, 31, 0, 0, 0, 0, 0, 0, 0, 991, 1037, 1042, 980, 1019, 1043, 984, 1012, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8187, 8115, 8116, 8257, 8312, 8294, 8116] avg=8192.0000 max=8312 max_rank=5 min=8115 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32979 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1050, 965, 1051, 1046, 1027, 1019, 1067, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 998, 1055, 1026, 1007, 1008, 1058, 975, 8, 0, 0, 0, 0, 0, 0, 0, 1045, 1022, 1051, 1061, 1028, 1014, 1011, 1046, 3, 1, 0, 0, 0, 0, 0, 0, 1029, 1045, 1053, 1018, 1064, 1042, 1002, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1035, 972, 999, 1006, 990, 1033, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1028, 1035, 1040, 977, 1038, 1036, 1060, 7, 3, 0, 0, 0, 0, 0, 0, 962, 1029, 1001, 997, 948, 1019, 1022, 1054, 31, 0, 0, 0, 0, 0, 0, 0, 1010, 1019, 1007, 975, 1032, 1007, 1067, 1020, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8189, 8282, 8272, 8088, 8239, 8063, 8145] avg=8192.0000 max=8282 max_rank=2 min=8063 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32535 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1031, 1054, 1014, 1025, 1026, 1011, 1054, 984, 0, 0, 0, 0, 0, 0, 0, 0, 984, 995, 1024, 1039, 1025, 1008, 1030, 1033, 8, 0, 0, 0, 0, 0, 0, 0, 1064, 1025, 1045, 1042, 993, 1028, 1021, 993, 3, 1, 0, 0, 0, 0, 0, 0, 987, 978, 1056, 1045, 1013, 1011, 1056, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1029, 997, 1023, 1011, 995, 1017, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1006, 1006, 1027, 1030, 1052, 999, 1007, 7, 3, 0, 0, 0, 0, 0, 0, 1060, 964, 1004, 1046, 1048, 1002, 1009, 1025, 31, 0, 0, 0, 0, 0, 0, 0, 1047, 1037, 1030, 1074, 1057, 1020, 1006, 1027, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8146, 8215, 8154, 8149, 8178, 8189, 8306] avg=8192.0000 max=8306 max_rank=7 min=8146 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1012, 1041, 1043, 1034, 990, 1015, 1079, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1043, 1012, 1037, 999, 1046, 1034, 1048, 8, 0, 0, 0, 0, 0, 0, 0, 981, 1017, 1002, 1009, 1048, 1072, 1006, 1041, 3, 1, 0, 0, 0, 0, 0, 0, 987, 974, 987, 1059, 1016, 1063, 1054, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1044, 1008, 1015, 992, 1048, 1035, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1044, 1011, 997, 1056, 978, 964, 1008, 6, 3, 0, 0, 0, 0, 0, 0, 1006, 1021, 990, 999, 1092, 1013, 1021, 1041, 30, 0, 0, 0, 0, 0, 0, 0, 1010, 1042, 1012, 1004, 1057, 1013, 1024, 985, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8292, 8180, 8143, 8209, 8096, 8213, 8155] avg=8192.0000 max=8292 max_rank=1 min=8096 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1005, 1016, 1049, 1012, 1050, 981, 1015, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1022, 1049, 1014, 1000, 1013, 1027, 1011, 12, 0, 0, 0, 0, 0, 0, 0, 1014, 1052, 1010, 1043, 1026, 1025, 1046, 1048, 8, 7, 0, 0, 0, 0, 0, 0, 963, 1043, 1038, 1028, 1041, 1067, 990, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1013, 1008, 1029, 1018, 1072, 998, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1039, 1010, 1036, 1007, 1024, 1043, 1009, 10, 5, 0, 0, 0, 0, 0, 0, 1035, 1012, 1009, 1078, 1010, 994, 1006, 1055, 37, 0, 0, 0, 0, 0, 0, 0, 1042, 1042, 926, 1022, 1041, 986, 1031, 1034, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8144, 8279, 8212, 8135, 8192, 8236, 8136] avg=8192.0000 max=8279 max_rank=2 min=8135 min_rank=4 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8064, 8064, 8448, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8320, 8320, 8064, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8320, 8320, 8448, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1020, 1010, 1110, 1040, 1002, 1011, 977, 1017, 1013, 1031, 1005, 1024, 1057, 1044, 1019, 1014, 1040, 1052, 1037, 1031, 1028, 1050, 1042, 1018, 1009, 989, 1047, 1045, 1001, 1065, 982, 1060, 1047, 1003, 986, 1053, 991, 1005, 1025, 1065, 1026, 1005, 999, 1060, 1046, 998, 999, 973, 1065, 1052, 998, 1033, 988, 1019, 1047, 993, 1024, 1049, 1027, 1023, 1005, 990, 998, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1012, 1036, 1033, 1030, 1087, 1057, 1017, 1031, 1061, 1032, 1007, 1040, 998, 1001, 1020, 1053, 1074, 1055, 1018, 1058, 1046, 1046, 1051, 1026, 1010, 1053, 1013, 971, 951, 1060, 965, 1046, 1045, 974, 1002, 1009, 1019, 1033, 1021, 1008, 1044, 1001, 1008, 1037, 1007, 1045, 1033, 1019, 976, 1039, 1086, 993, 1019, 1034, 1022, 996, 968, 1011, 1004, 966, 1049, 1034, 1049, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1084, 976, 1035, 1050, 993, 1008, 992, 1023, 999, 1051, 996, 1006, 1008, 1036, 1029, 985, 1038, 1038, 1040, 1035, 1055, 992, 1009, 1053, 1048, 1043, 1025, 1010, 1060, 1080, 997, 1009, 987, 1019, 1034, 1010, 1013, 1005, 1038, 1030, 1039, 1062, 1030, 1020, 1085, 1046, 997, 992, 1035, 1030, 1011, 1011, 1027, 989, 1035, 995, 1018, 1029, 1009, 1026, 1038, 1019, 1023, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1013, 986, 1033, 1031, 1024, 1046, 1005, 1020, 1046, 1055, 1041, 1062, 1012, 1038, 1016, 1072, 1033, 1040, 1065, 1025, 1030, 976, 993, 1052, 991, 1046, 991, 1044, 1023, 999, 1019, 1018, 1030, 1061, 988, 1057, 1054, 1054, 1011, 1017, 990, 978, 1019, 1014, 1016, 1032, 1006, 1002, 1033, 1033, 984, 998, 1021, 1068, 1026, 1017, 1015, 1025, 1055, 1034, 964, 1048, 1013, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1078, 1004, 993, 1026, 1022, 1016, 1057, 967, 1037, 1013, 1071, 1078, 1022, 1042, 1055, 993, 982, 1023, 1007, 1035, 1056, 1013, 1053, 979, 1092, 1048, 1010, 992, 999, 1012, 989, 1032, 1040, 1041, 1039, 1018, 1000, 1008, 1050, 1025, 1049, 1005, 1020, 1045, 1077, 994, 1028, 1038, 1039, 1031, 1007, 992, 996, 1010, 1022, 1048, 984, 1035, 1058, 980, 1015, 998, 1015, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1052, 1042, 978, 1044, 1044, 1003, 1005, 1056, 1038, 1012, 1000, 1054, 981, 1005, 1039, 965, 994, 1036, 1011, 1012, 992, 1060, 1015, 1033, 1038, 1048, 1037, 1002, 1011, 1011, 1051, 1026, 1084, 1017, 1046, 1089, 997, 1027, 1061, 1013, 996, 1017, 1006, 997, 964, 1017, 1033, 1003, 1029, 1018, 1039, 1027, 982, 1052, 1024, 1046, 1012, 1044, 1009, 1073, 1033, 1041, 1028, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[980, 1005, 976, 1021, 1022, 1016, 968, 1028, 1017, 1034, 998, 1053, 1043, 989, 1007, 997, 1042, 1031, 987, 1075, 1022, 1026, 1004, 1048, 1012, 1013, 1041, 1019, 1061, 997, 1018, 1064, 1056, 1043, 1038, 1024, 1075, 980, 993, 1006, 1012, 1011, 1034, 1063, 1046, 1096, 1035, 1039, 1012, 999, 1027, 1010, 1023, 1017, 1029, 1046, 1020, 1014, 1008, 1016, 1048, 982, 1025, 1095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1036, 1011, 1034, 1007, 1036, 1069, 1080, 1013, 1022, 1036, 1049, 1061, 1028, 1033, 1035, 1033, 989, 1004, 1043, 1023, 986, 1019, 988, 1027, 1022, 1043, 1022, 1068, 1022, 1012, 1015, 992, 1001, 1056, 988, 1043, 957, 1083, 998, 1026, 1001, 995, 1016, 1019, 1026, 1023, 1027, 1037, 1022, 982, 1051, 1049, 987, 1066, 991, 995, 1039, 1073, 1050, 1022, 1024, 1058, 995, 978] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1012, 1036, 1033, 1030, 1087, 1057, 1017, 1031, 4, 0, 0, 0, 0, 0, 0, 0, 1061, 1032, 1007, 1020, 998, 1001, 1020, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1055, 1018, 1031, 1046, 1046, 1051, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1053, 1013, 971, 951, 1060, 965, 1046, 5, 0, 0, 0, 0, 0, 0, 0, 1045, 974, 1002, 1000, 1019, 1033, 1021, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1001, 1008, 1037, 1007, 1045, 1033, 1019, 13, 3, 0, 0, 0, 0, 0, 0, 976, 1039, 1086, 993, 1019, 1034, 1022, 996, 24, 0, 0, 0, 0, 0, 0, 0, 968, 1011, 1004, 966, 1049, 1034, 1049, 1027, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 8192, 8347, 8074, 8102, 8210, 8189, 8115] avg=8192.0000 max=8347 max_rank=2 min=8074 min_rank=3 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32616 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1020, 1010, 1110, 1040, 1002, 1011, 977, 1017, 4, 0, 0, 0, 0, 0, 0, 0, 1013, 1031, 1005, 1004, 1057, 1044, 1019, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1052, 1037, 1004, 1028, 1050, 1042, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 989, 1047, 1045, 1001, 1065, 982, 1060, 5, 0, 0, 0, 0, 0, 0, 0, 1047, 1003, 986, 1044, 991, 1005, 1025, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1005, 999, 1060, 1046, 998, 999, 973, 13, 3, 0, 0, 0, 0, 0, 0, 1065, 1052, 998, 1033, 988, 1019, 1047, 993, 24, 0, 0, 0, 0, 0, 0, 0, 1024, 1049, 1027, 1023, 1005, 990, 998, 1054, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8187, 8271, 8203, 8166, 8122, 8219, 8177] avg=8192.0000 max=8271 max_rank=2 min=8122 min_rank=5 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1013, 986, 1033, 1031, 1024, 1046, 1005, 1020, 4, 0, 0, 0, 0, 0, 0, 0, 1046, 1055, 1041, 1042, 1012, 1038, 1016, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1040, 1065, 998, 1030, 976, 993, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1046, 991, 1044, 1023, 999, 1019, 1018, 5, 0, 0, 0, 0, 0, 0, 0, 1030, 1061, 988, 1048, 1054, 1054, 1011, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 990, 978, 1019, 1014, 1016, 1032, 1006, 1002, 13, 3, 0, 0, 0, 0, 0, 0, 1033, 1033, 984, 998, 1021, 1068, 1026, 1017, 24, 0, 0, 0, 0, 0, 0, 0, 1015, 1025, 1055, 1034, 964, 1048, 1013, 1028, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8322, 8187, 8136, 8263, 8073, 8204, 8189] avg=8192.0000 max=8322 max_rank=1 min=8073 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1052, 1042, 978, 1044, 1044, 1003, 1005, 1056, 4, 0, 0, 0, 0, 0, 0, 0, 1038, 1012, 1000, 1034, 981, 1005, 1039, 965, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1036, 1011, 986, 992, 1060, 1015, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1048, 1037, 1002, 1011, 1011, 1051, 1026, 5, 0, 0, 0, 0, 0, 0, 0, 1084, 1017, 1046, 1080, 997, 1027, 1061, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1017, 1006, 997, 964, 1017, 1033, 1003, 13, 3, 0, 0, 0, 0, 0, 0, 1029, 1018, 1039, 1027, 982, 1052, 1024, 1046, 23, 0, 0, 0, 0, 0, 0, 0, 1012, 1044, 1009, 1073, 1033, 1041, 1028, 1017, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8074, 8127, 8229, 8325, 8049, 8240, 8264] avg=8192.0000 max=8325 max_rank=4 min=8049 min_rank=5 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32878 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1084, 976, 1035, 1050, 993, 1008, 992, 1023, 4, 0, 0, 0, 0, 0, 0, 0, 999, 1051, 996, 986, 1008, 1036, 1029, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1038, 1040, 1008, 1055, 992, 1009, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1043, 1025, 1010, 1060, 1080, 997, 1009, 5, 0, 0, 0, 0, 0, 0, 0, 987, 1019, 1034, 1001, 1013, 1005, 1038, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1062, 1030, 1020, 1085, 1046, 997, 992, 13, 3, 0, 0, 0, 0, 0, 0, 1035, 1030, 1011, 1011, 1027, 989, 1035, 995, 24, 0, 0, 0, 0, 0, 0, 0, 1018, 1029, 1009, 1026, 1038, 1019, 1023, 1031, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8090, 8233, 8277, 8127, 8287, 8157, 8200] avg=8192.0000 max=8287 max_rank=5 min=8090 min_rank=1 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[980, 1005, 976, 1021, 1022, 1016, 968, 1028, 4, 0, 0, 0, 0, 0, 0, 0, 1017, 1034, 998, 1033, 1043, 989, 1007, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1031, 987, 1047, 1022, 1026, 1004, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1013, 1041, 1019, 1061, 997, 1018, 1064, 5, 0, 0, 0, 0, 0, 0, 0, 1056, 1043, 1038, 1015, 1075, 980, 993, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1011, 1034, 1063, 1046, 1096, 1035, 1039, 13, 3, 0, 0, 0, 0, 0, 0, 1012, 999, 1027, 1010, 1023, 1017, 1029, 1046, 25, 0, 0, 0, 0, 0, 0, 0, 1020, 1014, 1008, 1016, 1048, 982, 1025, 1095, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8020, 8118, 8207, 8230, 8206, 8352, 8188, 8215] avg=8192.0000 max=8352 max_rank=5 min=8020 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1078, 1004, 993, 1026, 1022, 1016, 1057, 967, 4, 0, 0, 0, 0, 0, 0, 0, 1037, 1013, 1071, 1057, 1022, 1042, 1055, 993, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1023, 1007, 1008, 1056, 1013, 1053, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 1048, 1010, 992, 999, 1012, 989, 1032, 5, 0, 0, 0, 0, 0, 0, 0, 1040, 1041, 1039, 1009, 1000, 1008, 1050, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1005, 1020, 1045, 1077, 994, 1028, 1038, 14, 3, 0, 0, 0, 0, 0, 0, 1039, 1031, 1007, 992, 996, 1010, 1022, 1048, 24, 0, 0, 0, 0, 0, 0, 0, 984, 1035, 1058, 980, 1015, 998, 1015, 1033, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8290, 8121, 8179, 8212, 8273, 8169, 8125] avg=8192.0000 max=8290 max_rank=1 min=8121 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1036, 1011, 1034, 1007, 1036, 1069, 1080, 1013, 10, 0, 0, 0, 0, 0, 0, 0, 1022, 1036, 1049, 1027, 1028, 1033, 1035, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1004, 1043, 987, 986, 1019, 988, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1043, 1022, 1068, 1022, 1012, 1015, 992, 7, 5, 0, 0, 0, 0, 0, 0, 1001, 1056, 988, 1026, 957, 1083, 998, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 995, 1016, 1019, 1026, 1023, 1027, 1037, 17, 9, 0, 0, 0, 0, 0, 0, 1022, 982, 1051, 1049, 987, 1066, 991, 995, 27, 0, 0, 0, 0, 0, 0, 0, 1039, 1073, 1050, 1022, 1024, 1058, 995, 978, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8263, 8043, 8208, 8135, 8170, 8170, 8251] avg=8192.0000 max=8296 max_rank=0 min=8043 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8064, 8320, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1061, 1040, 1074, 1003, 1052, 1054, 1023, 1032, 1006, 1035, 1007, 990, 1013, 1001, 968, 992, 1016, 1044, 990, 1018, 1037, 1056, 1061, 1044, 1058, 999, 1009, 1015, 1027, 1027, 1028, 1021, 979, 1051, 1008, 1020, 983, 1047, 1000, 1035, 993, 1030, 1015, 1009, 1005, 1039, 1019, 1022, 1041, 1018, 1019, 1016, 1051, 1035, 1059, 988, 1111, 1008, 1011, 1018, 1050, 1017, 1022, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[968, 1015, 1004, 1015, 1047, 1007, 986, 969, 987, 1001, 1021, 1017, 1028, 1031, 1097, 1069, 1009, 1054, 990, 977, 1030, 1019, 1101, 1013, 1051, 967, 987, 1023, 989, 1094, 1055, 1013, 1049, 1060, 1033, 1032, 1001, 1013, 986, 998, 1022, 1056, 1046, 993, 1017, 1036, 1042, 1022, 1063, 1059, 1013, 1026, 985, 1026, 1037, 1044, 1020, 1071, 1053, 1025, 1027, 959, 1015, 1073] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[999, 1042, 983, 1051, 1004, 990, 1023, 1021, 1059, 1017, 1022, 1013, 1023, 978, 1011, 999, 1022, 1034, 1045, 1020, 1002, 1039, 977, 1021, 974, 1031, 1047, 1006, 996, 1024, 1092, 1039, 1024, 1056, 974, 1023, 1017, 1007, 970, 1038, 1017, 991, 1084, 1003, 1011, 1028, 1059, 1079, 1021, 1009, 1059, 1017, 1034, 1067, 1090, 1040, 1057, 998, 1003, 1020, 1013, 1042, 1075, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1017, 1032, 1060, 1016, 1017, 1020, 1009, 1029, 1045, 966, 951, 1061, 988, 1037, 972, 1022, 1049, 995, 1019, 996, 1048, 1039, 1064, 991, 966, 1047, 1020, 1042, 1047, 1095, 1050, 1012, 997, 1045, 1020, 1050, 1046, 1027, 1046, 1032, 1017, 1043, 1027, 1024, 1043, 967, 995, 1021, 1044, 1046, 1024, 1050, 1024, 1031, 1015, 1035, 1038, 1018, 1007, 1028, 1058, 965, 992, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1053, 1073, 1005, 993, 1026, 1060, 1057, 1003, 1020, 1007, 984, 1023, 1065, 1044, 1069, 1024, 1024, 1050, 1031, 1029, 982, 1023, 1026, 1015, 1010, 1060, 1043, 992, 1039, 1071, 1014, 975, 1075, 1010, 1035, 1027, 1007, 1042, 1015, 1026, 1038, 1005, 998, 1013, 996, 992, 1064, 1030, 1033, 994, 1007, 970, 1028, 1014, 1049, 1060, 1016, 1030, 1030, 1034, 1001, 1019, 954, 1034] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1029, 1027, 1026, 1029, 1012, 969, 1006, 1037, 1090, 1030, 1007, 1018, 1013, 999, 1019, 1025, 1031, 1042, 993, 1042, 1005, 1055, 1036, 1059, 1010, 1036, 1021, 1045, 1039, 1030, 1003, 1013, 1002, 970, 988, 1025, 1020, 1036, 1008, 1028, 1065, 1085, 1029, 1074, 1001, 1001, 1074, 1001, 1012, 1062, 1042, 1006, 1044, 1009, 1064, 1018, 1009, 1001, 994, 1000, 1021, 1007, 1007, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[986, 1043, 1030, 1060, 1017, 1023, 996, 1030, 1018, 1044, 1056, 1029, 996, 1054, 998, 1026, 1057, 1001, 1008, 960, 1029, 1057, 1019, 1006, 997, 1016, 1032, 1037, 988, 1092, 1021, 1040, 1006, 1061, 1007, 1026, 1022, 1017, 1023, 1055, 1010, 1037, 1022, 1059, 980, 1050, 988, 1013, 1005, 994, 1033, 1030, 1002, 1037, 965, 1007, 1038, 989, 1061, 1043, 1029, 1037, 1082, 1042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[994, 1055, 964, 1017, 1007, 967, 1006, 1018, 1009, 1075, 1039, 1047, 1012, 1030, 1018, 1058, 1040, 1043, 1022, 1003, 1032, 1029, 1013, 1008, 1051, 1039, 999, 1036, 992, 1034, 1036, 1041, 1020, 1012, 998, 998, 1031, 1032, 997, 1020, 1052, 960, 979, 1087, 1016, 1069, 1031, 1014, 999, 1099, 1025, 995, 1013, 1038, 1056, 1032, 1041, 979, 1010, 1054, 1025, 994, 1041, 1085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1061, 1040, 1074, 1003, 1052, 1054, 1023, 1032, 32, 0, 0, 0, 0, 0, 0, 0, 1006, 1035, 1007, 990, 1013, 1001, 968, 992, 7, 6, 5, 0, 0, 0, 0, 0, 1016, 1044, 990, 1018, 1037, 1050, 1061, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 999, 1009, 1015, 1027, 1007, 1028, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1051, 1008, 1020, 983, 1047, 1000, 1035, 20, 4, 3, 0, 0, 0, 0, 0, 993, 1030, 1015, 1009, 1005, 1039, 1011, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1018, 1019, 1016, 1051, 1035, 1023, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 1008, 1011, 1018, 1050, 1017, 1022, 1009, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8371, 8030, 8260, 8164, 8150, 8124, 8191, 8246] avg=8192.0000 max=8371 max_rank=0 min=8030 min_rank=1 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1017, 1032, 1060, 1016, 1017, 1020, 1009, 1029, 31, 0, 0, 0, 0, 0, 0, 0, 1045, 966, 951, 1061, 988, 1037, 972, 1022, 7, 6, 5, 0, 0, 0, 0, 0, 1049, 995, 1019, 996, 1048, 1033, 1064, 991, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1047, 1020, 1042, 1047, 1074, 1050, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1045, 1020, 1050, 1046, 1027, 1046, 1032, 21, 4, 3, 0, 0, 0, 0, 0, 1017, 1043, 1027, 1024, 1043, 967, 987, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1046, 1024, 1050, 1024, 1031, 980, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1018, 1007, 1028, 1058, 965, 992, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8060, 8195, 8258, 8291, 8129, 8234, 8138] avg=8192.0000 max=8291 max_rank=4 min=8060 min_rank=1 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[999, 1042, 983, 1051, 1004, 990, 1023, 1021, 33, 0, 0, 0, 0, 0, 0, 0, 1059, 1017, 1022, 1013, 1023, 978, 1011, 999, 7, 6, 5, 0, 0, 0, 0, 0, 1022, 1034, 1045, 1020, 1002, 1033, 977, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1031, 1047, 1006, 996, 1004, 1092, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1056, 974, 1023, 1017, 1007, 970, 1038, 20, 4, 3, 0, 0, 0, 0, 0, 1017, 991, 1084, 1003, 1011, 1028, 1051, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1009, 1059, 1017, 1034, 1067, 1053, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 998, 1003, 1020, 1013, 1042, 1075, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8140, 8154, 8189, 8136, 8264, 8300, 8207] avg=8192.0000 max=8300 max_rank=6 min=8136 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[968, 1015, 1004, 1015, 1047, 1007, 986, 969, 31, 0, 0, 0, 0, 0, 0, 0, 987, 1001, 1021, 1017, 1028, 1031, 1097, 1069, 8, 6, 5, 0, 0, 0, 0, 0, 1009, 1054, 990, 977, 1030, 1013, 1101, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 967, 987, 1023, 989, 1073, 1055, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1060, 1033, 1032, 1001, 1013, 986, 998, 21, 4, 3, 0, 0, 0, 0, 0, 1022, 1056, 1046, 993, 1017, 1036, 1034, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1059, 1013, 1026, 985, 1026, 1002, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1071, 1053, 1025, 1027, 959, 1015, 1065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8042, 8270, 8187, 8158, 8200, 8226, 8218, 8235] avg=8192.0000 max=8270 max_rank=1 min=8042 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32879 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1053, 1073, 1005, 993, 1026, 1060, 1057, 1003, 32, 0, 0, 0, 0, 0, 0, 0, 1020, 1007, 984, 1023, 1065, 1044, 1069, 1024, 7, 6, 5, 0, 0, 0, 0, 0, 1024, 1050, 1031, 1029, 982, 1017, 1026, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1060, 1043, 992, 1039, 1050, 1014, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1010, 1035, 1027, 1007, 1042, 1015, 1026, 21, 4, 3, 0, 0, 0, 0, 0, 1038, 1005, 998, 1013, 996, 992, 1056, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 994, 1007, 970, 1028, 1014, 1013, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1030, 1030, 1034, 1001, 1019, 954, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8302, 8254, 8174, 8183, 8265, 8128, 8119, 8111] avg=8192.0000 max=8302 max_rank=0 min=8111 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32623 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[986, 1043, 1030, 1060, 1017, 1023, 996, 1030, 29, 0, 0, 0, 0, 0, 0, 0, 1018, 1044, 1056, 1029, 996, 1054, 998, 1026, 7, 6, 5, 0, 0, 0, 0, 0, 1057, 1001, 1008, 960, 1029, 1051, 1019, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1016, 1032, 1037, 988, 1071, 1021, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1061, 1007, 1026, 1022, 1017, 1023, 1055, 21, 4, 3, 0, 0, 0, 0, 0, 1010, 1037, 1022, 1059, 980, 1050, 980, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 994, 1033, 1030, 1002, 1037, 932, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 989, 1061, 1043, 1029, 1037, 1082, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8239, 8131, 8202, 8245, 8151, 8040, 8314] avg=8192.0000 max=8314 max_rank=7 min=8040 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1029, 1027, 1026, 1029, 1012, 969, 1006, 1037, 32, 0, 0, 0, 0, 0, 0, 0, 1090, 1030, 1007, 1018, 1013, 999, 1019, 1025, 7, 6, 5, 0, 0, 0, 0, 0, 1031, 1042, 993, 1042, 1005, 1049, 1036, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1036, 1021, 1045, 1039, 1010, 1003, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 970, 988, 1025, 1020, 1036, 1008, 1028, 20, 4, 3, 0, 0, 0, 0, 0, 1065, 1085, 1029, 1074, 1001, 1001, 1066, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1062, 1042, 1006, 1044, 1009, 1028, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1001, 994, 1000, 1021, 1007, 1007, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8219, 8257, 8177, 8104, 8322, 8221, 8069] avg=8192.0000 max=8322 max_rank=5 min=8069 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32716 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[994, 1055, 964, 1017, 1007, 967, 1006, 1018, 35, 0, 0, 0, 0, 0, 0, 0, 1009, 1075, 1039, 1047, 1012, 1030, 1018, 1058, 13, 12, 11, 0, 0, 0, 0, 0, 1040, 1043, 1022, 1003, 1032, 1017, 1013, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1039, 999, 1036, 992, 1011, 1036, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1012, 998, 998, 1031, 1032, 997, 1020, 23, 9, 5, 0, 0, 0, 0, 0, 1052, 960, 979, 1087, 1016, 1069, 1015, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1099, 1025, 995, 1013, 1038, 1012, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 979, 1010, 1054, 1025, 994, 1041, 1072, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8063, 8324, 8178, 8205, 8145, 8192, 8213, 8216] avg=8192.0000 max=8324 max_rank=1 min=8063 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8448, 8192, 8192, 8320, 8320, 8064, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8064, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8448, 8192, 8192, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8064, 8320, 8192, 8320, 8064, 8320, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[987, 1011, 1076, 1010, 1041, 1049, 1033, 1072, 1035, 1002, 1027, 1037, 1016, 1026, 995, 1005, 980, 1019, 1044, 1076, 1089, 984, 984, 1027, 964, 1045, 1009, 1030, 1032, 1030, 1025, 1013, 1035, 1033, 1061, 985, 1098, 1000, 1007, 1019, 1026, 1017, 1024, 1012, 1023, 1014, 1011, 1029, 993, 1004, 1044, 995, 1064, 1033, 1061, 981, 991, 1063, 1041, 1020, 1003, 1004, 992, 1080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1036, 1046, 979, 1030, 996, 1063, 996, 1025, 997, 1037, 1042, 948, 986, 976, 1024, 1022, 1004, 1059, 1033, 1024, 1023, 1037, 1041, 992, 1031, 1041, 1017, 1059, 972, 1013, 1034, 1030, 1045, 1035, 1015, 1038, 1013, 1066, 1061, 1021, 958, 1010, 1025, 998, 1029, 1092, 1057, 1003, 1041, 1016, 1056, 1024, 984, 1006, 1043, 1080, 1016, 1091, 1036, 992, 1035, 1021, 1010, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1018, 1027, 1068, 1044, 1065, 987, 1004, 1029, 1015, 1022, 1021, 1041, 1014, 1021, 952, 1039, 1012, 1032, 950, 1031, 1078, 1079, 1040, 1072, 1021, 1055, 1063, 976, 1035, 1040, 1024, 1041, 1071, 984, 1040, 1033, 1013, 1017, 1038, 1023, 1011, 1024, 1032, 1046, 1049, 1008, 1051, 993, 1005, 1032, 1060, 989, 954, 1006, 1036, 984, 1001, 1051, 1031, 996, 1025, 1018, 991, 1008] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1006, 1046, 993, 991, 1011, 1025, 1087, 1038, 1029, 1005, 997, 1030, 980, 1056, 1028, 1052, 996, 1018, 1005, 1007, 1032, 1046, 1031, 1114, 1020, 1008, 1041, 1078, 1094, 1039, 1026, 1042, 1014, 1011, 1097, 997, 1023, 1034, 973, 1027, 1076, 1025, 961, 1049, 1022, 1021, 1023, 972, 1082, 1059, 1007, 987, 1014, 1014, 963, 1074, 1024, 1003, 1004, 1056, 996, 962, 983, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1037, 1093, 1057, 1038, 1051, 1016, 1003, 1018, 1036, 1050, 1009, 1037, 1060, 1029, 1085, 1050, 1015, 1035, 1009, 1012, 997, 983, 1055, 989, 974, 1011, 1050, 1030, 1021, 1009, 1031, 1039, 985, 1072, 1000, 1032, 1018, 996, 1053, 993, 1005, 1056, 923, 1053, 1054, 1032, 1022, 1030, 999, 1029, 1021, 1019, 1022, 1037, 1029, 1050, 996, 993, 1005, 1015, 1005, 1062, 988, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1016, 1065, 1002, 1026, 1050, 1007, 993, 1015, 1021, 1090, 982, 973, 1020, 1069, 1027, 1033, 993, 1006, 1004, 994, 995, 1035, 992, 1080, 1056, 1017, 988, 985, 1073, 998, 988, 1036, 1001, 1004, 1010, 1066, 993, 1001, 995, 1063, 1060, 1039, 1035, 1042, 1023, 1028, 1020, 990, 1036, 1023, 1034, 1030, 1055, 1035, 1116, 1005, 1039, 1046, 992, 1077, 986, 1025, 995, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1038, 1047, 1040, 1058, 1063, 1052, 977, 964, 1047, 1046, 1038, 1017, 1030, 980, 1039, 1063, 1021, 1007, 1030, 1022, 1025, 1011, 1038, 1025, 997, 1003, 986, 1005, 970, 1019, 1066, 1051, 1020, 1005, 1067, 1020, 1037, 1048, 1053, 1005, 945, 1026, 1014, 976, 1044, 1058, 1056, 979, 1051, 1035, 1020, 1053, 1021, 1021, 1041, 1012, 998, 1027, 996, 1014, 1047, 1006, 1080, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[989, 1028, 1018, 963, 973, 1046, 1076, 1093, 998, 982, 995, 1007, 1023, 1025, 954, 1065, 1022, 1022, 1029, 1054, 1063, 999, 1030, 982, 1001, 985, 1004, 976, 1010, 1066, 1047, 1048, 1050, 1005, 986, 1051, 1011, 1070, 1018, 1058, 1083, 1036, 1002, 1063, 1013, 1037, 988, 1060, 1021, 1041, 1056, 1026, 1018, 1026, 1047, 1039, 993, 1032, 1001, 972, 1017, 1070, 1046, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1036, 1013, 979, 1030, 996, 1063, 996, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1037, 1042, 948, 986, 976, 1024, 1022, 13, 8, 0, 0, 0, 0, 0, 0, 1004, 1059, 1033, 1024, 1023, 1037, 1041, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1041, 1017, 1059, 972, 1013, 1034, 1030, 5, 0, 0, 0, 0, 0, 0, 0, 1045, 1035, 996, 1038, 1013, 1066, 1061, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 958, 1010, 1025, 998, 1029, 1092, 1057, 1003, 6, 0, 0, 0, 0, 0, 0, 0, 1041, 1016, 1056, 1024, 984, 1006, 1016, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1091, 1036, 992, 1035, 1021, 1010, 1006, 33, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8053, 8213, 8202, 8275, 8178, 8223, 8254] avg=8192.0000 max=8275 max_rank=4 min=8053 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32930 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[987, 979, 1076, 1010, 1041, 1049, 1033, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1002, 1027, 1037, 1016, 1026, 995, 1005, 13, 9, 0, 0, 0, 0, 0, 0, 980, 1019, 1044, 1076, 1089, 984, 984, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 964, 1045, 1009, 1030, 1032, 1030, 1025, 1013, 6, 0, 0, 0, 0, 0, 0, 0, 1035, 1033, 1040, 985, 1098, 1000, 1007, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1017, 1024, 1012, 1023, 1014, 1011, 1029, 6, 0, 0, 0, 0, 0, 0, 0, 993, 1004, 1044, 995, 1064, 1033, 1034, 981, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1063, 1041, 1020, 1003, 1004, 992, 1080, 32, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8165, 8203, 8154, 8217, 8162, 8148, 8240] avg=8192.0000 max=8247 max_rank=0 min=8148 min_rank=6 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[989, 996, 1018, 963, 973, 1046, 1076, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 998, 982, 995, 1007, 1023, 1025, 954, 1065, 13, 8, 0, 0, 0, 0, 0, 0, 1022, 1022, 1029, 1054, 1063, 999, 1030, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 985, 1004, 976, 1010, 1066, 1047, 1048, 5, 0, 0, 0, 0, 0, 0, 0, 1050, 1005, 967, 1051, 1011, 1070, 1018, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 1036, 1002, 1063, 1013, 1037, 988, 1060, 6, 0, 0, 0, 0, 0, 0, 0, 1021, 1041, 1056, 1026, 1018, 1026, 1020, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1032, 1001, 972, 1017, 1070, 1046, 1027, 32, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8070, 8201, 8142, 8230, 8288, 8247, 8204] avg=8192.0000 max=8288 max_rank=5 min=8070 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32969 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1018, 995, 1068, 1044, 1065, 987, 1004, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1022, 1021, 1041, 1014, 1021, 952, 1039, 13, 9, 0, 0, 0, 0, 0, 0, 1012, 1032, 950, 1031, 1078, 1079, 1040, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1055, 1063, 976, 1035, 1040, 1024, 1041, 6, 0, 0, 0, 0, 0, 0, 0, 1071, 984, 1019, 1033, 1013, 1017, 1038, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1024, 1032, 1046, 1049, 1008, 1051, 993, 6, 0, 0, 0, 0, 0, 0, 0, 1005, 1032, 1060, 989, 954, 1006, 1009, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1051, 1031, 996, 1025, 1018, 991, 1008, 32, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8147, 8294, 8261, 8198, 8220, 8039, 8167] avg=8192.0000 max=8294 max_rank=2 min=8039 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1037, 1059, 1057, 1038, 1051, 1016, 1003, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1050, 1009, 1037, 1060, 1029, 1085, 1050, 13, 8, 0, 0, 0, 0, 0, 0, 1015, 1035, 1009, 1012, 997, 983, 1055, 989, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1011, 1050, 1030, 1021, 1009, 1031, 1039, 5, 0, 0, 0, 0, 0, 0, 0, 985, 1072, 981, 1032, 1018, 996, 1053, 993, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1056, 923, 1053, 1054, 1032, 1022, 1030, 6, 0, 0, 0, 0, 0, 0, 0, 999, 1029, 1021, 1019, 1022, 1037, 1002, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 996, 993, 1005, 1015, 1005, 1062, 988, 1013, 34, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8377, 8095, 8170, 8130, 8181, 8179, 8125] avg=8192.0000 max=8377 max_rank=1 min=8095 min_rank=2 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1006, 1013, 993, 991, 1011, 1025, 1087, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1005, 997, 1030, 980, 1056, 1028, 1052, 12, 9, 0, 0, 0, 0, 0, 0, 996, 1018, 1005, 1007, 1032, 1046, 1031, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1008, 1041, 1078, 1094, 1039, 1026, 1042, 6, 0, 0, 0, 0, 0, 0, 0, 1014, 1011, 1075, 997, 1023, 1034, 973, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1025, 961, 1049, 1022, 1021, 1023, 972, 7, 0, 0, 0, 0, 0, 0, 0, 1082, 1059, 1007, 987, 1014, 1014, 938, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1003, 1004, 1056, 996, 962, 983, 1012, 33, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8198, 8249, 8354, 8154, 8156, 8175, 8086] avg=8192.0000 max=8354 max_rank=3 min=8086 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32571 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1016, 1032, 1002, 1026, 1050, 1007, 993, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1090, 982, 973, 1020, 1069, 1027, 1033, 14, 8, 0, 0, 0, 0, 0, 0, 993, 1006, 1004, 994, 995, 1035, 992, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1017, 988, 985, 1073, 998, 988, 1036, 5, 0, 0, 0, 0, 0, 0, 0, 1001, 1004, 991, 1066, 993, 1001, 995, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1039, 1035, 1042, 1023, 1028, 1020, 990, 6, 0, 0, 0, 0, 0, 0, 0, 1036, 1023, 1034, 1030, 1055, 1035, 1087, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1046, 992, 1077, 986, 1025, 995, 1043, 33, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8237, 8099, 8146, 8114, 8243, 8305, 8251] avg=8192.0000 max=8305 max_rank=6 min=8099 min_rank=2 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32913 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1038, 1011, 1040, 1058, 1063, 1052, 977, 964, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1046, 1038, 1017, 1030, 980, 1039, 1063, 15, 14, 0, 0, 0, 0, 0, 0, 1021, 1007, 1030, 1022, 1025, 1011, 1038, 1025, 3, 0, 0, 0, 0, 0, 0, 0, 997, 1003, 986, 1005, 970, 1019, 1066, 1051, 10, 0, 0, 0, 0, 0, 0, 0, 1020, 1005, 1030, 1020, 1037, 1048, 1053, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 945, 1026, 1014, 976, 1044, 1058, 1056, 979, 10, 0, 0, 0, 0, 0, 0, 0, 1051, 1035, 1020, 1053, 1021, 1021, 1007, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1027, 996, 1014, 1047, 1006, 1080, 986, 36, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8289, 8182, 8107, 8218, 8108, 8220, 8209] avg=8192.0000 max=8289 max_rank=1 min=8107 min_rank=3 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8192, 8192, 8192, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8320, 8320, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8320, 8320, 8320, 8320, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8192, 8448, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1006, 1039, 1024, 1031, 1035, 1039, 1074, 1074, 1043, 1036, 1041, 1018, 1034, 1055, 1008, 1010, 1001, 965, 1033, 1039, 1048, 1077, 1043, 1022, 994, 1038, 976, 1025, 1026, 1004, 1029, 1028, 1022, 1028, 1002, 1038, 1010, 1007, 1053, 1063, 1025, 1046, 1014, 1010, 1041, 961, 1002, 1014, 987, 1017, 1010, 1026, 1006, 1039, 1022, 1047, 1073, 992, 1027, 1001, 1028, 1011, 1018, 981] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1026, 1073, 1065, 988, 1032, 996, 995, 1013, 1028, 1014, 999, 1012, 1040, 990, 1030, 997, 1005, 1033, 1040, 1038, 990, 1050, 1058, 1047, 1008, 1004, 952, 1060, 1025, 1036, 1054, 1081, 992, 1053, 982, 1058, 1007, 1043, 1033, 1022, 1033, 968, 1031, 1008, 1011, 1089, 1048, 1015, 1024, 1026, 1031, 1036, 1030, 1017, 1039, 1046, 1036, 998, 1029, 1031, 980, 979, 1031, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1010, 1004, 1018, 998, 1018, 1038, 1043, 1033, 1062, 1063, 1020, 1039, 1058, 1018, 985, 955, 1003, 968, 1061, 1019, 1071, 1016, 940, 1035, 1003, 1013, 1028, 1021, 1023, 1041, 1036, 972, 1003, 969, 1044, 1049, 1018, 1025, 1027, 1066, 997, 1033, 1036, 971, 1044, 1042, 999, 1047, 1059, 1043, 1039, 1084, 1007, 1022, 1026, 1050, 1052, 1044, 1011, 1038, 1009, 1047, 1002, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[995, 1035, 1054, 1030, 1001, 1006, 985, 1032, 1015, 1052, 992, 1030, 1015, 981, 1025, 1049, 1054, 1017, 1077, 998, 1026, 1004, 1037, 1032, 1065, 1043, 1026, 1016, 1053, 978, 1014, 994, 999, 1004, 1021, 1019, 1031, 1015, 1006, 1083, 1036, 1012, 981, 1028, 993, 1025, 981, 1027, 1018, 1009, 1016, 1013, 1031, 1011, 1020, 1054, 1025, 1049, 1039, 1076, 1063, 1015, 1030, 1075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1020, 1009, 1034, 1058, 995, 1021, 1063, 1015, 1032, 1071, 1029, 1018, 1010, 1037, 1011, 1024, 987, 981, 1012, 984, 1008, 1051, 969, 978, 1026, 1044, 1022, 1036, 1019, 1040, 1017, 1057, 1046, 1029, 1020, 1012, 1029, 1021, 1082, 1020, 1037, 1029, 1068, 1032, 1001, 1040, 1060, 1020, 1067, 1068, 1008, 1000, 973, 1057, 1025, 1064, 1007, 986, 990, 978, 1045, 990, 994, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1049, 965, 981, 1032, 1043, 1016, 1092, 1048, 1033, 997, 1009, 1079, 1023, 1017, 1016, 992, 1013, 999, 1020, 1047, 1026, 1001, 1053, 983, 1006, 1008, 1017, 1041, 1011, 1015, 951, 1030, 1019, 1025, 1030, 1019, 1083, 1023, 1037, 1031, 1005, 1038, 1014, 1024, 983, 1015, 987, 1060, 997, 1067, 1009, 1056, 1010, 977, 1034, 1052, 1055, 991, 1011, 1079, 1063, 1070, 1042, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1012, 1056, 1051, 1010, 994, 1031, 1010, 1072, 1028, 1065, 1019, 1033, 1019, 1012, 1028, 1012, 978, 1003, 1012, 1057, 1033, 995, 1024, 1004, 981, 995, 955, 919, 1042, 999, 1058, 1038, 983, 1062, 1050, 1045, 1035, 1039, 985, 985, 1009, 994, 1054, 1042, 975, 1077, 1045, 1051, 1039, 1086, 1030, 1026, 1022, 1057, 1033, 1019, 1034, 1003, 1036, 1002, 1068, 1055, 1044, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1006, 1007, 1048, 1057, 1081, 1064, 1030, 1044, 1007, 1049, 1018, 1009, 1009, 972, 1049, 1028, 1088, 1023, 1046, 999, 1058, 991, 1020, 990, 1042, 1032, 1016, 1031, 1013, 1036, 1045, 973, 1037, 986, 1027, 1039, 977, 1003, 999, 992, 1039, 1023, 1067, 983, 1009, 1039, 1002, 1022, 1010, 1030, 1004, 1061, 1045, 1020, 992, 1006, 1020, 1000, 977, 1071, 1058, 1048, 1037, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1026, 1073, 1065, 988, 1032, 996, 995, 979, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1014, 999, 1012, 1040, 990, 1030, 997, 4, 3, 0, 0, 0, 0, 0, 0, 1005, 1033, 1040, 1038, 990, 1050, 1058, 1047, 29, 1, 0, 0, 0, 0, 0, 0, 1008, 1004, 952, 1060, 1025, 1036, 1054, 1081, 38, 5, 0, 0, 0, 0, 0, 0, 992, 1053, 982, 1055, 1007, 1043, 1033, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 968, 1031, 1008, 1011, 1089, 1048, 1015, 14, 0, 0, 0, 0, 0, 0, 0, 1024, 988, 1031, 1036, 1030, 1017, 1039, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 998, 1029, 1031, 961, 979, 1031, 1031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8117, 8291, 8263, 8187, 8217, 8211, 8096] avg=8192.0000 max=8291 max_rank=2 min=8096 min_rank=7 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1010, 1004, 1018, 998, 1018, 1038, 1043, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1063, 1020, 1039, 1058, 1018, 985, 955, 4, 3, 0, 0, 0, 0, 0, 0, 1003, 968, 1061, 1019, 1071, 1016, 940, 1035, 30, 1, 0, 0, 0, 0, 0, 0, 1003, 1013, 1028, 1021, 1023, 1041, 1036, 972, 39, 5, 0, 0, 0, 0, 0, 0, 1003, 969, 1044, 1046, 1018, 1025, 1027, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1033, 1036, 971, 1044, 1042, 999, 1047, 15, 0, 0, 0, 0, 0, 0, 0, 1059, 1004, 1039, 1084, 1007, 1022, 1026, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1044, 1011, 1038, 989, 1047, 1002, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8207, 8144, 8181, 8198, 8184, 8291, 8204] avg=8192.0000 max=8291 max_rank=6 min=8127 min_rank=0 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1049, 965, 981, 1032, 1043, 1016, 1092, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 997, 1009, 1079, 1023, 1017, 1016, 992, 5, 3, 0, 0, 0, 0, 0, 0, 1013, 999, 1020, 1047, 1026, 1001, 1053, 983, 30, 1, 0, 0, 0, 0, 0, 0, 1006, 1008, 1017, 1041, 1011, 1015, 951, 1030, 40, 5, 0, 0, 0, 0, 0, 0, 1019, 1025, 1030, 1016, 1083, 1023, 1037, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1038, 1014, 1024, 983, 1015, 987, 1060, 15, 0, 0, 0, 0, 0, 0, 0, 997, 1027, 1009, 1056, 1010, 977, 1034, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 991, 1011, 1079, 1042, 1070, 1042, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8174, 8173, 8124, 8264, 8141, 8162, 8307] avg=8192.0000 max=8307 max_rank=7 min=8124 min_rank=3 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32874 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[995, 1035, 1054, 1030, 1001, 1006, 985, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1052, 992, 1030, 1015, 981, 1025, 1049, 5, 3, 0, 0, 0, 0, 0, 0, 1054, 1017, 1077, 998, 1026, 1004, 1037, 1032, 30, 1, 0, 0, 0, 0, 0, 0, 1065, 1043, 1026, 1016, 1053, 978, 1014, 994, 37, 5, 0, 0, 0, 0, 0, 0, 999, 1004, 1021, 1016, 1031, 1015, 1006, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1012, 981, 1028, 993, 1025, 981, 1027, 15, 0, 0, 0, 0, 0, 0, 0, 1018, 972, 1016, 1013, 1031, 1011, 1020, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1049, 1039, 1076, 1042, 1015, 1030, 1075, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8167, 8276, 8231, 8175, 8098, 8135, 8351] avg=8192.0000 max=8351 max_rank=7 min=8098 min_rank=5 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32759 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1020, 1009, 1034, 1058, 995, 1021, 1063, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1071, 1029, 1018, 1010, 1037, 1011, 1024, 5, 3, 0, 0, 0, 0, 0, 0, 987, 981, 1012, 984, 1008, 1051, 969, 978, 29, 1, 0, 0, 0, 0, 0, 0, 1026, 1044, 1022, 1036, 1019, 1040, 1017, 1057, 40, 5, 0, 0, 0, 0, 0, 0, 1046, 1029, 1020, 1009, 1029, 1021, 1082, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1029, 1068, 1032, 1001, 1040, 1060, 1020, 15, 0, 0, 0, 0, 0, 0, 0, 1067, 1028, 1008, 1000, 973, 1057, 1025, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 986, 990, 978, 1024, 990, 994, 1060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8240, 8000, 8306, 8256, 8302, 8222, 8029] avg=8192.0000 max=8306 max_rank=3 min=8000 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1012, 1056, 1051, 1010, 994, 1031, 1010, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1065, 1019, 1033, 1019, 1012, 1028, 1012, 5, 3, 0, 0, 0, 0, 0, 0, 978, 1003, 1012, 1057, 1033, 995, 1024, 1004, 31, 1, 0, 0, 0, 0, 0, 0, 981, 995, 955, 919, 1042, 999, 1058, 1038, 40, 5, 0, 0, 0, 0, 0, 0, 983, 1062, 1050, 1042, 1035, 1039, 985, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 994, 1054, 1042, 975, 1077, 1045, 1051, 15, 0, 0, 0, 0, 0, 0, 0, 1039, 1046, 1030, 1026, 1022, 1057, 1033, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1003, 1036, 1002, 1047, 1055, 1044, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8224, 8138, 8032, 8181, 8262, 8272, 8227] avg=8192.0000 max=8272 max_rank=6 min=8032 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32942 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1006, 1007, 1048, 1057, 1081, 1064, 1030, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1049, 1018, 1009, 1009, 972, 1049, 1028, 8, 5, 0, 0, 0, 0, 0, 0, 1088, 1023, 1046, 999, 1058, 991, 1020, 990, 36, 3, 0, 0, 0, 0, 0, 0, 1042, 1032, 1016, 1031, 1013, 1036, 1045, 973, 41, 7, 0, 0, 0, 0, 0, 0, 1037, 986, 1027, 1034, 977, 1003, 999, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1023, 1067, 983, 1009, 1039, 1002, 1022, 20, 0, 0, 0, 0, 0, 0, 0, 1010, 989, 1004, 1061, 1045, 1020, 992, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1000, 977, 1071, 1027, 1048, 1037, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 8154, 8254, 8236, 8055, 8204, 8127, 8212] avg=8192.0000 max=8294 max_rank=0 min=8055 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32598 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1006, 1039, 1024, 1031, 1035, 1039, 1074, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1036, 1041, 1018, 1034, 1055, 1008, 1010, 5, 3, 0, 0, 0, 0, 0, 0, 1001, 965, 1033, 1039, 1048, 1077, 1043, 1022, 31, 1, 0, 0, 0, 0, 0, 0, 994, 1038, 976, 1025, 1026, 1004, 1029, 1028, 38, 5, 0, 0, 0, 0, 0, 0, 1022, 1028, 1002, 1035, 1010, 1007, 1053, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1046, 1014, 1010, 1041, 961, 1002, 1014, 15, 0, 0, 0, 0, 0, 0, 0, 987, 979, 1010, 1026, 1006, 1039, 1022, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 992, 1027, 1001, 1007, 1011, 1018, 981, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 8253, 8260, 8163, 8220, 8128, 8116, 8110] avg=8192.0000 max=8286 max_rank=0 min=8110 min_rank=7 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32574 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8320, 8192, 8320, 8192, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8192, 8320, 8192, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8064, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8192, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8064, 8192, 8320, 8192, 8320, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8448, 8192, 8320, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True + [2026-08-30 05:02:15] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 659.1 | throughput per GPU (TFLOP/s/GPU): 505.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.977938E+00 | loss scale: 1.0 | grad norm: 5.119 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[973, 1048, 1056, 982, 991, 994, 1018, 1088, 992, 968, 1021, 1019, 1005, 1005, 1040, 1011, 1054, 1038, 1029, 1031, 983, 1058, 1042, 1042, 1048, 1008, 1037, 988, 1031, 996, 1033, 1042, 1071, 1061, 982, 1084, 1020, 1009, 1028, 1055, 1012, 1078, 1040, 1072, 983, 1001, 1008, 966, 1036, 1041, 1065, 1037, 1021, 988, 1007, 1068, 1019, 1035, 986, 1032, 1024, 998, 1021, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1047, 986, 989, 1073, 990, 1022, 1038, 1059, 991, 1054, 1010, 1034, 1019, 1046, 1031, 982, 1049, 1039, 1018, 1050, 1014, 987, 1020, 1027, 1019, 994, 1057, 1009, 1056, 1022, 1033, 1030, 1012, 958, 1024, 1005, 1016, 999, 1070, 1064, 1028, 1036, 1040, 1073, 1047, 989, 1033, 1037, 1005, 1042, 1042, 1081, 1021, 1017, 968, 1072, 996, 959, 986, 990, 1060, 1042, 998, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1030, 980, 1005, 1087, 1010, 1018, 1049, 1020, 1007, 1025, 1066, 1023, 1011, 1054, 1009, 986, 1007, 1021, 1067, 1056, 1015, 1039, 1015, 1035, 1016, 1061, 1046, 1026, 1025, 1018, 1028, 1004, 1015, 1039, 1029, 1045, 1035, 1071, 1006, 1048, 1018, 1056, 997, 997, 1048, 1052, 1014, 972, 1003, 1027, 997, 1032, 988, 1043, 1014, 1044, 1004, 1001, 972, 1088, 1038, 1014, 984, 986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1045, 1034, 998, 1076, 1031, 1000, 984, 1043, 1004, 1012, 978, 1078, 1027, 1012, 1022, 1046, 1023, 1030, 1055, 1017, 1028, 1023, 1001, 980, 1071, 1043, 1062, 1032, 1044, 991, 1031, 1086, 1061, 1029, 1026, 1033, 1023, 990, 990, 997, 1004, 1008, 1023, 1041, 994, 1030, 1023, 1019, 1040, 1031, 973, 1020, 1051, 1001, 1027, 987, 971, 1033, 1060, 1042, 1022, 1014, 1025, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1018, 1094, 1040, 1063, 1048, 1051, 1057, 1037, 1029, 1061, 989, 1030, 1010, 1034, 1049, 1024, 1068, 1013, 1034, 970, 982, 990, 976, 1035, 982, 1011, 1032, 1050, 1006, 1010, 991, 1025, 1031, 1027, 1032, 996, 989, 1039, 991, 1043, 1004, 1048, 1020, 1016, 1007, 1047, 1047, 1068, 1018, 1024, 1002, 1021, 1038, 1035, 1084, 999, 1000, 1008, 1023, 1018, 1013, 1053, 994, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1057, 1011, 1055, 1052, 1015, 1027, 1005, 1002, 960, 1047, 1074, 1070, 1025, 1006, 1031, 1036, 1057, 1063, 1022, 945, 1036, 1025, 1018, 1053, 991, 1056, 1027, 991, 1015, 1049, 970, 1061, 1001, 1075, 1045, 1016, 999, 1027, 1012, 1027, 1029, 997, 1071, 1008, 1078, 972, 1056, 998, 1064, 978, 1007, 1027, 1036, 1025, 987, 1017, 1024, 1021, 1005, 1036, 1025, 1055, 953, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1014, 1020, 1012, 1047, 1030, 1010, 993, 1064, 1015, 1055, 1037, 974, 982, 999, 1042, 966, 1019, 1017, 1033, 1060, 1066, 1052, 1086, 980, 994, 1022, 1031, 1003, 1034, 1063, 1043, 1007, 1013, 1037, 1041, 1008, 1066, 1001, 1029, 1077, 1032, 1028, 961, 1060, 1016, 1083, 997, 992, 1010, 982, 986, 1029, 1049, 1067, 1034, 991, 991, 988, 1091, 1020, 1065, 1027, 985, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1011, 1057, 1024, 1009, 1014, 1031, 1058, 1023, 1012, 995, 1016, 1089, 1063, 977, 1017, 1030, 1016, 1003, 1072, 1002, 1030, 1028, 1087, 1033, 1045, 1034, 976, 1011, 1045, 977, 1022, 1042, 1036, 1064, 1058, 1030, 974, 986, 991, 1002, 1039, 983, 1020, 1027, 1003, 1050, 1033, 981, 1025, 1011, 974, 1063, 1051, 1065, 1046, 1001, 1029, 1051, 1007, 1010, 1014, 1062, 1000, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[973, 1048, 1056, 950, 991, 994, 1018, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 992, 968, 1021, 1019, 1005, 1005, 1040, 1011, 15, 5, 5, 0, 0, 0, 0, 0, 1054, 1038, 1002, 1031, 983, 1058, 1042, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1008, 1037, 988, 1031, 996, 1033, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1061, 982, 1084, 1020, 1009, 1028, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1078, 1040, 1072, 983, 1001, 1008, 966, 3, 0, 0, 0, 0, 0, 0, 0, 1036, 1041, 1065, 1037, 1021, 988, 1007, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1035, 986, 1032, 1024, 998, 1021, 1017, 32, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8086, 8250, 8175, 8295, 8163, 8263, 8186] avg=8192.0000 max=8295 max_rank=4 min=8086 min_rank=1 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32907 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1045, 1034, 998, 1041, 1031, 1000, 984, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1012, 978, 1078, 1027, 1012, 1022, 1046, 14, 5, 5, 0, 0, 0, 0, 0, 1023, 1030, 1027, 1017, 1028, 1023, 1001, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1043, 1062, 1032, 1044, 991, 1031, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1029, 1026, 1033, 1023, 990, 990, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1008, 1023, 1041, 994, 1030, 1023, 1019, 3, 0, 0, 0, 0, 0, 0, 0, 1040, 1031, 973, 1020, 1051, 1001, 1027, 987, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1033, 1060, 1042, 1022, 1014, 1025, 1041, 35, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8203, 8129, 8352, 8135, 8145, 8130, 8266] avg=8192.0000 max=8352 max_rank=3 min=8129 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1047, 986, 989, 1038, 990, 1022, 1038, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1054, 1010, 1034, 1019, 1046, 1031, 982, 15, 5, 4, 0, 0, 0, 0, 0, 1049, 1039, 991, 1050, 1014, 987, 1020, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 994, 1057, 1009, 1056, 1022, 1033, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 958, 1024, 1005, 1016, 999, 1070, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1036, 1040, 1073, 1047, 989, 1033, 1037, 3, 0, 0, 0, 0, 0, 0, 0, 1005, 1042, 1042, 1081, 1021, 1017, 968, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 996, 959, 986, 990, 1060, 1042, 998, 1031, 35, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8191, 8177, 8213, 8133, 8286, 8248, 8119] avg=8192.0000 max=8286 max_rank=5 min=8119 min_rank=7 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32786 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1018, 1094, 1040, 1028, 1048, 1051, 1057, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1061, 989, 1030, 1010, 1034, 1049, 1024, 15, 5, 4, 0, 0, 0, 0, 0, 1068, 1013, 1006, 970, 982, 990, 976, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1011, 1032, 1050, 1006, 1010, 991, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1027, 1032, 996, 989, 1039, 991, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1048, 1020, 1016, 1007, 1047, 1047, 1068, 3, 0, 0, 0, 0, 0, 0, 0, 1018, 1024, 1002, 1021, 1038, 1035, 1084, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1008, 1023, 1018, 1013, 1053, 994, 992, 35, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8373, 8250, 8040, 8100, 8133, 8260, 8221, 8159] avg=8192.0000 max=8373 max_rank=0 min=8040 min_rank=2 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1030, 980, 1005, 1052, 1010, 1018, 1049, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1025, 1066, 1023, 1011, 1054, 1009, 986, 15, 5, 4, 0, 0, 0, 0, 0, 1007, 1021, 1039, 1056, 1015, 1039, 1015, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1061, 1046, 1026, 1025, 1018, 1028, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1039, 1029, 1045, 1035, 1071, 1006, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1056, 997, 997, 1048, 1052, 1014, 972, 3, 0, 0, 0, 0, 0, 0, 0, 1003, 1027, 997, 1032, 988, 1043, 1014, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1001, 972, 1088, 1038, 1014, 984, 986, 35, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8205, 8227, 8217, 8273, 8157, 8148, 8145] avg=8192.0000 max=8273 max_rank=4 min=8145 min_rank=7 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1057, 1011, 1055, 1018, 1015, 1027, 1005, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 960, 1047, 1074, 1070, 1025, 1006, 1031, 1036, 15, 5, 5, 0, 0, 0, 0, 0, 1057, 1063, 995, 945, 1036, 1025, 1018, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1056, 1027, 991, 1015, 1049, 970, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1075, 1045, 1016, 999, 1027, 1012, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 997, 1071, 1008, 1078, 972, 1056, 998, 3, 0, 0, 0, 0, 0, 0, 0, 1064, 978, 1007, 1027, 1036, 1025, 987, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1021, 1005, 1036, 1025, 1055, 953, 1013, 34, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8274, 8192, 8152, 8187, 8212, 8141, 8188] avg=8192.0000 max=8274 max_rank=1 min=8141 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1014, 1020, 1012, 1013, 1030, 1010, 993, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1055, 1037, 974, 982, 999, 1042, 966, 15, 5, 4, 0, 0, 0, 0, 0, 1019, 1017, 1005, 1060, 1066, 1052, 1086, 980, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1022, 1031, 1003, 1034, 1063, 1043, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1037, 1041, 1008, 1066, 1001, 1029, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1028, 961, 1060, 1016, 1083, 997, 992, 3, 0, 0, 0, 0, 0, 0, 0, 1010, 982, 986, 1029, 1049, 1067, 1034, 991, 0, 0, 0, 0, 0, 0, 0, 0, 991, 988, 1091, 1020, 1065, 1027, 985, 1010, 34, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8094, 8285, 8190, 8257, 8172, 8148, 8234] avg=8192.0000 max=8285 max_rank=2 min=8094 min_rank=1 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32811 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1011, 1057, 1024, 972, 1014, 1031, 1058, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 995, 1016, 1089, 1063, 977, 1017, 1030, 18, 7, 9, 0, 0, 0, 0, 0, 1016, 1003, 1037, 1002, 1030, 1028, 1087, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1034, 976, 1011, 1045, 977, 1022, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1064, 1058, 1030, 974, 986, 991, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 983, 1020, 1027, 1003, 1050, 1033, 981, 5, 0, 0, 0, 0, 0, 0, 0, 1025, 1011, 974, 1063, 1051, 1065, 1046, 1001, 1, 0, 0, 0, 0, 0, 0, 0, 1029, 1051, 1007, 1010, 1014, 1062, 1000, 1001, 37, 28, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8233, 8236, 8137, 8123, 8141, 8237, 8239] avg=8192.0000 max=8239 max_rank=7 min=8123 min_rank=4 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32740 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8192, 8320, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8192, 8192, 8320, 8320, 8320, 8064, 8192, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8320, 8320, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8064, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1074, 982, 950, 1012, 1005, 1059, 1011, 1021, 1024, 1015, 1040, 1028, 1028, 1014, 956, 1012, 1038, 1012, 1049, 1010, 994, 993, 1007, 1022, 1025, 1010, 1040, 1003, 1051, 1017, 1044, 1099, 1045, 1023, 976, 1035, 1030, 1023, 968, 1040, 1011, 999, 1052, 1028, 1027, 1010, 1042, 1075, 1033, 1019, 1039, 1034, 989, 1055, 1031, 1013, 1054, 1017, 1047, 1045, 1067, 1035, 964, 1065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1000, 1027, 1027, 1040, 1055, 1058, 1042, 1048, 1031, 1027, 1011, 973, 1011, 1019, 1016, 1000, 1017, 1018, 1056, 1057, 985, 1064, 1039, 1001, 1034, 1030, 1015, 1014, 1052, 1014, 1054, 1012, 1012, 957, 1021, 1022, 988, 1025, 1013, 1064, 1010, 1036, 1018, 1010, 1033, 1008, 984, 1028, 1031, 1039, 1013, 1007, 1052, 1037, 1037, 1019, 1031, 1006, 1019, 994, 1041, 988, 1086, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1043, 1031, 1036, 1050, 1071, 987, 1026, 986, 1025, 1013, 981, 982, 1013, 1030, 1013, 1004, 998, 1057, 1003, 989, 1007, 1026, 1025, 996, 1025, 1008, 1056, 1017, 1009, 1035, 1013, 1061, 1009, 1036, 1004, 1068, 1048, 1044, 1043, 1039, 1071, 1078, 997, 976, 1072, 1024, 1005, 994, 1089, 1043, 1068, 1019, 988, 1002, 1026, 1029, 1011, 1019, 996, 1005, 1030, 1033, 986, 1068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[969, 1010, 1031, 1002, 1054, 1002, 1047, 1017, 1039, 1063, 1014, 995, 991, 1066, 1005, 1040, 1031, 1022, 985, 1008, 1027, 1020, 1066, 1019, 1002, 1000, 995, 1027, 1083, 1023, 975, 1012, 1031, 1026, 1035, 988, 1046, 1036, 1019, 1039, 1014, 1014, 1043, 1053, 1037, 1018, 1024, 1008, 1044, 1108, 1029, 1000, 1032, 1031, 1040, 948, 1017, 1024, 1057, 1026, 1008, 1026, 1035, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1026, 1062, 1051, 1040, 970, 1004, 1035, 1019, 1019, 1015, 1063, 1018, 1045, 1034, 988, 986, 1040, 1005, 977, 1080, 1024, 1052, 991, 999, 1006, 954, 1013, 1030, 1025, 1030, 1038, 1045, 1022, 1033, 1050, 1042, 1030, 966, 1021, 1018, 1004, 1013, 1034, 1042, 1032, 986, 1059, 1033, 982, 1040, 1011, 1076, 1048, 1034, 986, 1003, 998, 1042, 1022, 1035, 1023, 1048, 1031, 1088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[976, 1036, 1027, 979, 973, 1057, 1040, 1038, 1000, 1031, 1042, 1074, 1008, 1012, 1041, 1019, 1018, 1031, 1042, 1039, 990, 1096, 1048, 1029, 1027, 1008, 1027, 1080, 1051, 1033, 1004, 988, 1034, 1013, 1048, 995, 1001, 983, 1001, 1040, 979, 988, 1038, 1040, 1043, 1032, 1082, 1029, 1073, 1006, 1007, 1005, 1025, 1035, 994, 1025, 1019, 1017, 1069, 1008, 1026, 1000, 979, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1099, 1036, 1024, 1018, 1096, 1020, 991, 971, 1007, 1035, 996, 1010, 1083, 994, 1028, 1000, 1021, 1019, 1073, 1025, 983, 1047, 1062, 1030, 1021, 1004, 983, 1020, 1044, 1004, 1021, 1059, 987, 1021, 1029, 1064, 974, 1031, 1068, 1015, 1036, 1030, 1042, 1014, 999, 1017, 991, 1080, 1033, 1030, 1009, 1042, 1029, 1046, 1022, 1061, 1019, 990, 1024, 1001, 1018, 1020, 980, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[984, 1061, 1051, 998, 1058, 993, 1021, 1058, 1053, 1031, 1018, 1048, 1016, 1027, 1008, 1011, 1014, 980, 1015, 1053, 1019, 1044, 1027, 1032, 1068, 1049, 990, 988, 971, 997, 1046, 1029, 1047, 1006, 1034, 1034, 1042, 972, 1018, 1028, 1059, 1018, 995, 1037, 1002, 995, 980, 971, 1007, 1022, 1045, 993, 1022, 1013, 1042, 1090, 1024, 1010, 1063, 1088, 1067, 1018, 1018, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1000, 1027, 1027, 1040, 1050, 1058, 1042, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1027, 1011, 973, 1011, 1019, 1016, 1000, 33, 4, 0, 0, 0, 0, 0, 0, 1017, 1018, 1056, 1057, 985, 1063, 1039, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1030, 1015, 1014, 1052, 1014, 1054, 1012, 2, 1, 0, 0, 0, 0, 0, 0, 1012, 957, 1021, 1022, 988, 1025, 1013, 1064, 14, 0, 0, 0, 0, 0, 0, 0, 1010, 1036, 1018, 1010, 1033, 1008, 984, 1028, 3, 0, 0, 0, 0, 0, 0, 0, 1031, 1006, 1013, 1007, 1052, 1037, 1037, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1006, 1019, 994, 1041, 988, 1086, 1042, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8292, 8125, 8236, 8228, 8116, 8130, 8202, 8207] avg=8192.0000 max=8292 max_rank=0 min=8116 min_rank=4 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32655 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1074, 982, 950, 1012, 1000, 1059, 1011, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1015, 1040, 1028, 1028, 1014, 956, 1012, 32, 4, 0, 0, 0, 0, 0, 0, 1038, 1012, 1049, 1010, 994, 992, 1007, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1010, 1040, 1003, 1051, 1017, 1044, 1099, 2, 1, 0, 0, 0, 0, 0, 0, 1045, 1023, 976, 1035, 1030, 1023, 968, 1040, 14, 0, 0, 0, 0, 0, 0, 0, 1011, 999, 1052, 1028, 1027, 1010, 1042, 1075, 3, 0, 0, 0, 0, 0, 0, 0, 1033, 987, 1039, 1034, 989, 1055, 1031, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1017, 1047, 1045, 1067, 1035, 964, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8153, 8124, 8292, 8154, 8247, 8181, 8276] avg=8192.0000 max=8292 max_rank=3 min=8109 min_rank=0 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[969, 1010, 1031, 1002, 1049, 1002, 1047, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1063, 1014, 995, 991, 1066, 1005, 1040, 35, 4, 0, 0, 0, 0, 0, 0, 1031, 1022, 985, 1008, 1027, 1019, 1066, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1000, 995, 1027, 1083, 1023, 975, 1012, 2, 1, 0, 0, 0, 0, 0, 0, 1031, 1026, 1035, 988, 1046, 1036, 1019, 1039, 14, 0, 0, 0, 0, 0, 0, 0, 1014, 1014, 1043, 1053, 1037, 1018, 1024, 1008, 3, 0, 0, 0, 0, 0, 0, 0, 1044, 1073, 1029, 1000, 1032, 1031, 1040, 948, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1024, 1057, 1026, 1008, 1026, 1035, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8252, 8177, 8120, 8234, 8214, 8197, 8215] avg=8192.0000 max=8252 max_rank=1 min=8120 min_rank=3 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1043, 1031, 1036, 1050, 1065, 987, 1026, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1013, 981, 982, 1013, 1030, 1013, 1004, 33, 4, 0, 0, 0, 0, 0, 0, 998, 1057, 1003, 989, 1007, 1025, 1025, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1008, 1056, 1017, 1009, 1035, 1013, 1061, 2, 1, 0, 0, 0, 0, 0, 0, 1009, 1036, 1004, 1068, 1048, 1044, 1043, 1039, 14, 0, 0, 0, 0, 0, 0, 0, 1071, 1078, 997, 976, 1072, 1024, 1005, 994, 4, 0, 0, 0, 0, 0, 0, 0, 1089, 1010, 1068, 1019, 988, 1002, 1026, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1019, 996, 1005, 1030, 1033, 986, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8098, 8100, 8227, 8305, 8221, 8231, 8130] avg=8192.0000 max=8305 max_rank=4 min=8098 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32887 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1026, 1062, 1051, 1040, 965, 1004, 1035, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1015, 1063, 1018, 1045, 1034, 988, 986, 33, 4, 0, 0, 0, 0, 0, 0, 1040, 1005, 977, 1080, 1024, 1051, 991, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 954, 1013, 1030, 1025, 1030, 1038, 1045, 2, 1, 0, 0, 0, 0, 0, 0, 1022, 1033, 1050, 1042, 1030, 966, 1021, 1018, 15, 0, 0, 0, 0, 0, 0, 0, 1004, 1013, 1034, 1042, 1032, 986, 1059, 1033, 3, 0, 0, 0, 0, 0, 0, 0, 982, 1007, 1011, 1076, 1048, 1034, 986, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1042, 1022, 1035, 1023, 1048, 1031, 1069, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8205, 8167, 8144, 8197, 8206, 8147, 8268] avg=8192.0000 max=8268 max_rank=7 min=8144 min_rank=3 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32818 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1099, 1036, 1024, 1018, 1090, 1020, 991, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1035, 996, 1010, 1083, 994, 1028, 1000, 32, 3, 0, 0, 0, 0, 0, 0, 1021, 1019, 1073, 1025, 983, 1046, 1062, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1004, 983, 1020, 1044, 1004, 1021, 1059, 2, 1, 0, 0, 0, 0, 0, 0, 987, 1021, 1029, 1064, 974, 1031, 1068, 1015, 13, 0, 0, 0, 0, 0, 0, 0, 1036, 1030, 1042, 1014, 999, 1017, 991, 1080, 4, 0, 0, 0, 0, 0, 0, 0, 1033, 998, 1009, 1042, 1029, 1046, 1022, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 990, 1024, 1001, 1018, 1020, 980, 974, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8188, 8259, 8159, 8202, 8213, 8240, 8026] avg=8192.0000 max=8259 max_rank=2 min=8026 min_rank=7 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[976, 1036, 1027, 979, 961, 1057, 1040, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1031, 1042, 1074, 1008, 1012, 1041, 1019, 34, 6, 0, 0, 0, 0, 0, 0, 1018, 1031, 1042, 1039, 990, 1093, 1048, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1008, 1027, 1080, 1051, 1033, 1004, 988, 4, 3, 0, 0, 0, 0, 0, 0, 1034, 1013, 1048, 995, 1001, 983, 1001, 1040, 18, 0, 0, 0, 0, 0, 0, 0, 979, 988, 1038, 1040, 1043, 1032, 1082, 1029, 8, 6, 0, 0, 0, 0, 0, 0, 1073, 972, 1007, 1005, 1025, 1035, 994, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1017, 1069, 1008, 1026, 1000, 979, 1008, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8267, 8290, 8225, 8133, 8245, 8136, 8126] avg=8192.0000 max=8290 max_rank=2 min=8114 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32640 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[984, 1061, 1051, 998, 1053, 993, 1021, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1031, 1018, 1048, 1016, 1027, 1008, 1011, 32, 4, 0, 0, 0, 0, 0, 0, 1014, 980, 1015, 1053, 1019, 1043, 1027, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1049, 990, 988, 971, 997, 1046, 1029, 2, 1, 0, 0, 0, 0, 0, 0, 1047, 1006, 1034, 1034, 1042, 972, 1018, 1028, 14, 0, 0, 0, 0, 0, 0, 0, 1059, 1018, 995, 1037, 1002, 995, 980, 971, 3, 0, 0, 0, 0, 0, 0, 0, 1007, 990, 1045, 993, 1022, 1013, 1042, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1010, 1063, 1088, 1067, 1018, 1018, 1000, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8248, 8183, 8141, 8195, 8060, 8202, 8288] avg=8192.0000 max=8288 max_rank=7 min=8060 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8064, 8448, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8064, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8064, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8192, 8192, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1036, 1032, 1011, 1006, 997, 1033, 1011, 1002, 1008, 1041, 1027, 989, 1019, 1046, 1005, 1015, 1015, 1017, 1051, 968, 1009, 1026, 1003, 1040, 1006, 998, 1004, 1050, 1045, 1078, 1090, 1003, 1032, 1019, 1031, 1027, 1046, 997, 968, 1049, 1021, 1043, 996, 995, 1040, 1057, 1044, 1018, 1073, 1031, 1027, 984, 1009, 1022, 1017, 1003, 1040, 1030, 1033, 1053, 1090, 1015, 993, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1016, 1061, 1009, 1042, 1020, 1049, 1054, 1021, 1012, 1052, 1046, 988, 1021, 1052, 1057, 1032, 1033, 1002, 983, 1024, 1029, 1028, 1060, 1017, 1028, 1042, 1008, 1052, 996, 1032, 1007, 1042, 1058, 976, 1060, 1032, 963, 1014, 1019, 1045, 978, 1032, 1010, 983, 999, 1005, 999, 1014, 981, 1071, 1026, 1031, 1043, 1016, 964, 1020, 1022, 1000, 1048, 1032, 1051, 1074, 1052, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1058, 1043, 1007, 1002, 1048, 1082, 1011, 1059, 994, 974, 998, 1058, 1038, 1046, 1012, 1002, 1020, 1043, 1056, 1019, 1020, 1047, 1009, 1032, 1008, 1042, 1021, 1059, 1070, 1024, 973, 1006, 1039, 984, 1025, 995, 1001, 978, 1039, 1002, 1032, 980, 1013, 999, 1069, 1042, 992, 1044, 1023, 1048, 996, 1067, 1010, 1018, 1004, 1015, 1007, 1014, 1018, 1045, 965, 1042, 1061, 1088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1041, 980, 1015, 1040, 1064, 1023, 994, 994, 1004, 1025, 1040, 973, 1052, 1000, 1001, 1062, 999, 1040, 1047, 1027, 1030, 1049, 1080, 985, 1040, 1039, 977, 997, 989, 1021, 1009, 1049, 989, 1066, 1009, 1024, 1017, 1026, 1016, 1046, 998, 1072, 993, 1057, 1024, 1026, 1062, 1043, 1008, 992, 1071, 1009, 1046, 1041, 1029, 1032, 1043, 997, 1031, 1040, 1007, 1050, 982, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1049, 1040, 1002, 1034, 967, 1008, 1022, 1018, 992, 954, 1048, 944, 1052, 958, 1053, 1032, 1059, 1014, 1045, 1008, 1006, 1003, 1022, 1054, 996, 1010, 1054, 1011, 1055, 1008, 1046, 1023, 995, 1046, 1012, 958, 1052, 1023, 1000, 1035, 1013, 1009, 980, 1032, 998, 1062, 1055, 1065, 1030, 1015, 1050, 1013, 1052, 1035, 1043, 1054, 1045, 1037, 994, 1018, 1050, 1051, 1065, 1062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[996, 1004, 1028, 1040, 988, 1018, 989, 967, 1011, 1050, 1003, 1048, 1059, 1043, 1050, 1000, 1055, 992, 1017, 1042, 1049, 1072, 1037, 1046, 1040, 1005, 1108, 1041, 1032, 993, 1011, 976, 999, 1073, 995, 1031, 1029, 1029, 970, 1010, 971, 1024, 1023, 1016, 1022, 989, 1026, 1019, 1035, 1037, 1053, 1061, 975, 1069, 1056, 1037, 1029, 1067, 1044, 1037, 972, 1041, 966, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[997, 1025, 1015, 1015, 1049, 987, 1040, 1007, 1022, 991, 1016, 1043, 1010, 1011, 1002, 1026, 1042, 1070, 1014, 1015, 994, 1034, 1017, 1028, 1053, 992, 1052, 1015, 1047, 1060, 1029, 1030, 982, 1054, 1021, 1034, 1018, 1037, 1042, 969, 1022, 1053, 1033, 1031, 1052, 1063, 1066, 1002, 1018, 997, 1026, 1014, 1000, 1019, 954, 1011, 1064, 983, 1033, 1091, 1018, 1030, 1012, 1039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1046, 1007, 1016, 989, 1004, 1029, 1028, 1035, 1014, 998, 1031, 975, 1019, 1022, 978, 1011, 1056, 983, 1066, 1027, 1094, 1036, 1083, 994, 1069, 1059, 1041, 1050, 1001, 1042, 1008, 1046, 1032, 1010, 1044, 1022, 1037, 1003, 1005, 1027, 1051, 1032, 1009, 1046, 1064, 1061, 1030, 989, 1018, 1013, 1013, 1028, 1043, 994, 962, 1036, 1055, 1018, 998, 1006, 997, 1004, 1033, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1016, 1061, 1009, 1042, 1020, 1049, 1054, 1021, 26, 0, 0, 0, 0, 0, 0, 0, 1012, 1052, 1046, 988, 1021, 1052, 1057, 1032, 44, 4, 0, 0, 0, 0, 0, 0, 1033, 1002, 983, 1024, 1029, 1028, 1016, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1042, 1008, 1019, 996, 1032, 1007, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 976, 1060, 1032, 963, 1014, 1019, 1045, 35, 7, 0, 0, 0, 0, 0, 0, 978, 1032, 1010, 983, 999, 1000, 999, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1071, 1026, 1031, 1043, 1016, 964, 1020, 5, 0, 0, 0, 0, 0, 0, 0, 1022, 1000, 1048, 993, 1051, 1074, 1052, 1003, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8308, 8132, 8174, 8209, 8015, 8157, 8243] avg=8192.0000 max=8308 max_rank=1 min=8015 min_rank=5 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1036, 1032, 1011, 1006, 997, 1033, 1011, 1002, 26, 0, 0, 0, 0, 0, 0, 0, 1008, 1041, 1027, 989, 1019, 1046, 1005, 1015, 41, 4, 0, 0, 0, 0, 0, 0, 1015, 1017, 1051, 968, 1009, 1026, 962, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 998, 1004, 1017, 1045, 1078, 1090, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1019, 1031, 1027, 1046, 997, 968, 1049, 36, 7, 0, 0, 0, 0, 0, 0, 1021, 1043, 996, 995, 1040, 1051, 1044, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1031, 1027, 984, 1009, 1022, 1017, 1003, 6, 0, 0, 0, 0, 0, 0, 0, 1040, 1030, 1033, 1013, 1090, 1015, 993, 1052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8195, 8088, 8241, 8212, 8208, 8172, 8266] avg=8192.0000 max=8266 max_rank=7 min=8088 min_rank=2 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1058, 1043, 1007, 1002, 1048, 1082, 1011, 1059, 27, 0, 0, 0, 0, 0, 0, 0, 994, 974, 998, 1058, 1038, 1046, 1012, 1002, 42, 4, 0, 0, 0, 0, 0, 0, 1020, 1043, 1056, 1019, 1020, 1047, 967, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1042, 1021, 1025, 1070, 1024, 973, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 984, 1025, 995, 1001, 978, 1039, 1002, 36, 7, 0, 0, 0, 0, 0, 0, 1032, 980, 1013, 999, 1069, 1036, 992, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1048, 996, 1067, 1010, 1018, 1004, 1015, 6, 0, 0, 0, 0, 0, 0, 0, 1007, 1014, 1018, 1005, 965, 1042, 1061, 1088, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8337, 8168, 8204, 8169, 8106, 8165, 8187, 8200] avg=8192.0000 max=8337 max_rank=0 min=8106 min_rank=4 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32658 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1041, 980, 1015, 1040, 1064, 1023, 994, 994, 25, 0, 0, 0, 0, 0, 0, 0, 1004, 1025, 1040, 973, 1052, 1000, 1001, 1062, 44, 4, 0, 0, 0, 0, 0, 0, 999, 1040, 1047, 1027, 1030, 1049, 1036, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1039, 977, 965, 989, 1021, 1009, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1066, 1009, 1024, 1017, 1026, 1016, 1046, 36, 7, 0, 0, 0, 0, 0, 0, 998, 1072, 993, 1057, 1024, 1021, 1062, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 992, 1071, 1009, 1046, 1041, 1029, 1032, 5, 0, 0, 0, 0, 0, 0, 0, 1043, 997, 1031, 1000, 1007, 1050, 982, 1004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8205, 8213, 8089, 8236, 8270, 8233, 8114] avg=8192.0000 max=8270 max_rank=5 min=8089 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1049, 1040, 1002, 1034, 967, 1008, 1022, 1018, 25, 0, 0, 0, 0, 0, 0, 0, 992, 954, 1048, 944, 1052, 958, 1053, 1032, 42, 4, 0, 0, 0, 0, 0, 0, 1059, 1014, 1045, 1008, 1006, 1003, 980, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1010, 1054, 979, 1055, 1008, 1046, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1046, 1012, 958, 1052, 1023, 1000, 1035, 35, 7, 0, 0, 0, 0, 0, 0, 1013, 1009, 980, 1032, 998, 1056, 1055, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1015, 1050, 1013, 1052, 1035, 1043, 1054, 6, 0, 0, 0, 0, 0, 0, 0, 1045, 1037, 994, 979, 1050, 1051, 1065, 1062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8079, 8169, 8171, 8163, 8208, 8298, 8283] avg=8192.0000 max=8298 max_rank=6 min=8079 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32952 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[997, 1025, 1015, 1015, 1049, 987, 1040, 1007, 26, 0, 0, 0, 0, 0, 0, 0, 1022, 991, 1016, 1043, 1010, 1011, 1002, 1026, 42, 4, 0, 0, 0, 0, 0, 0, 1042, 1070, 1014, 1015, 994, 1034, 975, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 992, 1052, 982, 1047, 1060, 1029, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1054, 1021, 1034, 1018, 1037, 1042, 969, 38, 7, 0, 0, 0, 0, 0, 0, 1022, 1053, 1033, 1031, 1052, 1057, 1066, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 997, 1026, 1014, 1000, 1019, 954, 1011, 6, 0, 0, 0, 0, 0, 0, 0, 1064, 983, 1033, 1049, 1018, 1030, 1012, 1039, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8167, 8172, 8245, 8202, 8316, 8045, 8228] avg=8192.0000 max=8316 max_rank=5 min=8045 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[996, 1004, 1028, 1040, 988, 1018, 989, 967, 31, 4, 0, 0, 0, 0, 0, 0, 1011, 1050, 1003, 1048, 1059, 1043, 1050, 1000, 46, 7, 0, 0, 0, 0, 0, 0, 1055, 992, 1017, 1042, 1049, 1072, 991, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1005, 1108, 999, 1032, 993, 1011, 976, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1073, 995, 1031, 1029, 1029, 970, 1010, 39, 11, 0, 0, 0, 0, 0, 0, 971, 1024, 1023, 1016, 1022, 977, 1026, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1037, 1053, 1061, 975, 1069, 1056, 1037, 8, 0, 0, 0, 0, 0, 0, 0, 1029, 1067, 1044, 991, 972, 1041, 966, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8065, 8317, 8264, 8164, 8186, 8078, 8331, 8131] avg=8192.0000 max=8331 max_rank=6 min=8065 min_rank=0 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1046, 1007, 1016, 989, 1004, 1029, 1028, 1035, 26, 0, 0, 0, 0, 0, 0, 0, 1014, 998, 1031, 975, 1019, 1022, 978, 1011, 45, 4, 0, 0, 0, 0, 0, 0, 1056, 983, 1066, 1027, 1094, 1036, 1038, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1059, 1041, 1017, 1001, 1042, 1008, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1010, 1044, 1022, 1037, 1003, 1005, 1027, 35, 7, 0, 0, 0, 0, 0, 0, 1051, 1032, 1009, 1046, 1064, 1055, 1030, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1013, 1013, 1028, 1043, 994, 962, 1036, 6, 0, 0, 0, 0, 0, 0, 0, 1055, 1018, 998, 967, 997, 1004, 1033, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8097, 8294, 8283, 8222, 8276, 8113, 8071] avg=8192.0000 max=8294 max_rank=2 min=8071 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8064, 8320, 8192, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8064, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8064, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8064, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8064, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[992, 1048, 1012, 1034, 1015, 1031, 1056, 1027, 1004, 1079, 1055, 1051, 1038, 1002, 977, 1025, 1022, 1002, 980, 1025, 1009, 1035, 1051, 1031, 1011, 1013, 1023, 1047, 1011, 1038, 970, 1055, 995, 1038, 1020, 1028, 1021, 1031, 1027, 972, 1027, 986, 1060, 991, 1011, 1002, 1019, 1004, 1030, 970, 1035, 1042, 1032, 1009, 1043, 1023, 1066, 1043, 1049, 1009, 1032, 1055, 1025, 1072] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1056, 1003, 1047, 1064, 1030, 1012, 1035, 1025, 1047, 1008, 1026, 1000, 1018, 992, 991, 1020, 994, 1073, 1017, 969, 1033, 983, 1013, 1022, 1026, 1055, 1016, 1095, 990, 1024, 1030, 1043, 1086, 1033, 1048, 1068, 1002, 1005, 1018, 1015, 998, 994, 1052, 1054, 1018, 993, 1002, 1006, 1006, 1019, 1037, 1017, 1066, 1093, 1020, 1013, 1015, 1000, 1004, 1021, 1034, 978, 1013, 1051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[999, 1021, 1036, 985, 983, 997, 1077, 995, 1045, 1058, 1008, 1046, 1021, 1009, 1063, 1062, 1051, 1015, 1004, 1030, 1035, 977, 1016, 1058, 1005, 976, 1004, 1045, 1039, 1027, 1073, 1048, 1023, 1014, 1018, 1000, 1036, 1059, 985, 1025, 1067, 1032, 1020, 998, 1014, 1037, 1023, 967, 1060, 1010, 1012, 1019, 1018, 1019, 1029, 1045, 1023, 1022, 1049, 1004, 1040, 987, 1063, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1013, 1053, 979, 997, 1039, 1006, 1045, 966, 1049, 1040, 1055, 1037, 1006, 1049, 1060, 1050, 1052, 998, 985, 1038, 1024, 1032, 1010, 992, 1068, 1009, 1036, 1050, 1001, 985, 1013, 1051, 1072, 977, 1013, 987, 1004, 1040, 1070, 1030, 998, 1039, 1042, 1007, 1100, 1013, 1035, 1014, 1010, 973, 1020, 1009, 1023, 1006, 1057, 1011, 1021, 1019, 967, 1030, 1005, 1059, 1019, 1078] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[994, 1063, 1017, 1063, 1050, 1044, 1010, 1050, 1063, 1044, 1013, 1026, 1019, 991, 1012, 1032, 1001, 1034, 1071, 1029, 970, 1012, 987, 1000, 1034, 999, 967, 1069, 994, 996, 1060, 1010, 984, 1061, 987, 1045, 1050, 1040, 1007, 1030, 1034, 991, 1040, 1019, 1003, 1031, 1045, 1057, 1046, 1012, 1039, 988, 983, 1036, 1031, 1011, 990, 1001, 1036, 1076, 1048, 986, 1021, 1084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1009, 1021, 1041, 973, 1065, 1026, 1044, 996, 1039, 997, 1020, 1070, 1014, 1077, 1019, 1060, 1019, 1001, 1021, 1052, 1000, 955, 1005, 1070, 1004, 997, 915, 1030, 1043, 1069, 1028, 1062, 1019, 1052, 1034, 1029, 1043, 998, 1066, 997, 1038, 1051, 1023, 994, 974, 1007, 1008, 1042, 1027, 1029, 1025, 1025, 1044, 979, 1051, 1016, 999, 1024, 1016, 1018, 1082, 983, 1006, 1095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1068, 1023, 1031, 1024, 999, 1054, 1003, 1045, 1032, 992, 1004, 977, 1010, 1066, 1013, 1033, 978, 1004, 1057, 1016, 1039, 994, 1009, 1023, 981, 1000, 1009, 996, 1031, 1019, 1006, 1022, 1049, 1072, 1058, 1045, 1001, 1012, 1050, 1004, 1054, 1043, 1001, 1057, 1036, 1056, 1051, 994, 1037, 1011, 1029, 994, 1027, 1090, 1066, 1012, 1019, 1029, 1033, 1009, 1004, 1025, 1028, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1077, 1021, 1077, 1068, 1057, 997, 1023, 996, 973, 1029, 986, 990, 1022, 1011, 1000, 977, 1043, 1054, 1027, 1068, 1040, 992, 1025, 1023, 1043, 1029, 1068, 1060, 1032, 978, 1075, 1006, 993, 966, 1038, 1031, 1054, 1013, 1024, 1017, 1032, 1050, 1023, 1003, 1021, 1051, 1000, 1009, 1016, 981, 1038, 969, 1027, 1056, 1041, 1003, 1025, 1070, 982, 1016, 1009, 1019, 1064, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1056, 1003, 1047, 1064, 1030, 1012, 1014, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1008, 1026, 1000, 1018, 992, 991, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1073, 1017, 969, 1033, 983, 1013, 1022, 21, 16, 4, 0, 0, 0, 0, 0, 1026, 1055, 1016, 1095, 990, 1024, 1030, 1043, 3, 0, 0, 0, 0, 0, 0, 0, 1086, 1033, 1048, 1068, 1002, 1005, 1007, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 998, 994, 1052, 1054, 1018, 993, 1002, 1006, 7, 2, 0, 0, 0, 0, 0, 0, 1006, 1019, 1037, 1017, 1066, 1093, 1020, 1013, 2, 0, 0, 0, 0, 0, 0, 0, 1015, 1000, 1004, 1021, 1034, 978, 1013, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8095, 8145, 8282, 8264, 8126, 8273, 8100] avg=8192.0000 max=8282 max_rank=3 min=8095 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[992, 1048, 1012, 1034, 1015, 1031, 1035, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1079, 1055, 1051, 1038, 1002, 977, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1002, 980, 1025, 1009, 1035, 1051, 1031, 21, 17, 4, 0, 0, 0, 0, 0, 1011, 1013, 1023, 1047, 1011, 1038, 970, 1055, 3, 0, 0, 0, 0, 0, 0, 0, 995, 1038, 1020, 1028, 1021, 1031, 1016, 972, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 986, 1060, 991, 1011, 1002, 1019, 1004, 7, 2, 0, 0, 0, 0, 0, 0, 1030, 970, 1035, 1042, 1032, 1009, 1043, 1023, 2, 0, 0, 0, 0, 0, 0, 0, 1066, 1043, 1049, 1009, 1032, 1055, 1025, 1055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8224, 8197, 8171, 8121, 8109, 8186, 8334] avg=8192.0000 max=8334 max_rank=7 min=8109 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[994, 1063, 1017, 1063, 1050, 1044, 990, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1044, 1013, 1026, 1019, 991, 1012, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1034, 1071, 1029, 970, 1012, 987, 1000, 20, 17, 4, 0, 0, 0, 0, 0, 1034, 999, 967, 1069, 994, 996, 1060, 1010, 3, 0, 0, 0, 0, 0, 0, 0, 984, 1061, 987, 1045, 1050, 1040, 997, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 991, 1040, 1019, 1003, 1031, 1045, 1057, 6, 2, 0, 0, 0, 0, 0, 0, 1046, 1012, 1039, 988, 983, 1036, 1031, 1011, 2, 0, 0, 0, 0, 0, 0, 0, 990, 1001, 1036, 1076, 1048, 986, 1021, 1067, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 8193, 8145, 8132, 8194, 8228, 8148, 8225] avg=8192.0000 max=8271 max_rank=0 min=8132 min_rank=3 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[999, 1021, 1036, 985, 983, 997, 1055, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1058, 1008, 1046, 1021, 1009, 1063, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1015, 1004, 1030, 1035, 977, 1016, 1058, 22, 16, 4, 0, 0, 0, 0, 0, 1005, 976, 1004, 1045, 1039, 1027, 1073, 1048, 3, 0, 0, 0, 0, 0, 0, 0, 1023, 1014, 1018, 1000, 1036, 1059, 975, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1032, 1020, 998, 1014, 1037, 1023, 967, 6, 2, 0, 0, 0, 0, 0, 0, 1060, 1010, 1012, 1019, 1018, 1019, 1029, 1045, 2, 0, 0, 0, 0, 0, 0, 0, 1023, 1022, 1049, 1004, 1040, 987, 1063, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8305, 8228, 8220, 8150, 8166, 8214, 8182] avg=8192.0000 max=8305 max_rank=1 min=8071 min_rank=0 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1013, 1053, 979, 997, 1039, 1006, 1024, 966, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1040, 1055, 1037, 1006, 1049, 1060, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 998, 985, 1038, 1024, 1032, 1010, 992, 21, 17, 4, 0, 0, 0, 0, 0, 1068, 1009, 1036, 1050, 1001, 985, 1013, 1051, 3, 0, 0, 0, 0, 0, 0, 0, 1072, 977, 1013, 987, 1004, 1040, 1059, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1039, 1042, 1007, 1100, 1013, 1035, 1014, 7, 2, 0, 0, 0, 0, 0, 0, 1010, 973, 1020, 1009, 1023, 1006, 1057, 1011, 2, 0, 0, 0, 0, 0, 0, 0, 1021, 1019, 967, 1030, 1005, 1059, 1019, 1061, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8077, 8339, 8173, 8216, 8182, 8257, 8111, 8181] avg=8192.0000 max=8339 max_rank=1 min=8077 min_rank=0 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1009, 1021, 1041, 973, 1065, 1026, 1023, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 997, 1020, 1070, 1014, 1077, 1019, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1001, 1021, 1052, 1000, 955, 1005, 1070, 21, 17, 4, 0, 0, 0, 0, 0, 1004, 997, 915, 1030, 1043, 1069, 1028, 1062, 3, 0, 0, 0, 0, 0, 0, 0, 1019, 1052, 1034, 1029, 1043, 998, 1055, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1051, 1023, 994, 974, 1007, 1008, 1042, 7, 2, 0, 0, 0, 0, 0, 0, 1027, 1029, 1025, 1025, 1044, 979, 1051, 1016, 2, 0, 0, 0, 0, 0, 0, 0, 999, 1024, 1016, 1018, 1082, 983, 1006, 1078, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8289, 8165, 8151, 8227, 8146, 8198, 8206] avg=8192.0000 max=8289 max_rank=1 min=8146 min_rank=5 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1068, 1023, 1031, 1024, 999, 1054, 983, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 992, 1004, 977, 1010, 1066, 1013, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1004, 1057, 1016, 1039, 994, 1009, 1023, 20, 15, 4, 0, 0, 0, 0, 0, 981, 1000, 1009, 996, 1031, 1019, 1006, 1022, 3, 0, 0, 0, 0, 0, 0, 0, 1049, 1072, 1058, 1045, 1001, 1012, 1039, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1043, 1001, 1057, 1036, 1056, 1051, 994, 7, 2, 0, 0, 0, 0, 0, 0, 1037, 1011, 1029, 994, 1027, 1090, 1066, 1012, 2, 0, 0, 0, 0, 0, 0, 0, 1019, 1029, 1033, 1009, 1004, 1025, 1028, 967, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8120, 8159, 8067, 8280, 8301, 8268, 8114] avg=8192.0000 max=8301 max_rank=5 min=8067 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32963 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1077, 1021, 1077, 1068, 1057, 997, 998, 996, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1029, 986, 990, 1022, 1011, 1000, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1054, 1027, 1068, 1040, 992, 1025, 1023, 25, 19, 8, 0, 0, 0, 0, 0, 1043, 1029, 1068, 1060, 1032, 978, 1075, 1006, 6, 0, 0, 0, 0, 0, 0, 0, 993, 966, 1038, 1031, 1054, 1013, 1006, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1050, 1023, 1003, 1021, 1051, 1000, 1009, 10, 4, 0, 0, 0, 0, 0, 0, 1016, 981, 1038, 969, 1027, 1056, 1041, 1003, 7, 0, 0, 0, 0, 0, 0, 0, 1025, 1070, 982, 1016, 1009, 1019, 1064, 1009, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 7971, 8324, 8297, 8118, 8203, 8138, 8194] avg=8192.0000 max=8324 max_rank=2 min=7971 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32653 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8064, 8192, 8320, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8064, 8448, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8064, 8320, 8320, 8448, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1017, 1006, 1019, 1005, 988, 1011, 1049, 1040, 1007, 1011, 1020, 1023, 973, 1053, 1031, 1027, 989, 1011, 1039, 1113, 1040, 1001, 1027, 1005, 1004, 1059, 1050, 1073, 1040, 1021, 1024, 985, 1035, 1013, 1029, 1010, 1016, 1018, 985, 999, 1057, 1017, 1065, 1059, 1043, 1043, 996, 1036, 1035, 1013, 1019, 1035, 1042, 980, 1045, 1010, 1054, 1006, 982, 1025, 1015, 1039, 1031, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1066, 1050, 1057, 1028, 1026, 1035, 976, 1004, 1024, 1046, 983, 1032, 1031, 1023, 1037, 1011, 1048, 977, 1063, 1050, 1061, 996, 1024, 1002, 975, 1023, 985, 1013, 1059, 1026, 1030, 1015, 1000, 989, 1036, 1038, 1058, 998, 979, 1018, 979, 1040, 1030, 1027, 1026, 1036, 1025, 1033, 1036, 1042, 1010, 1042, 1020, 1070, 1055, 1055, 1056, 1055, 1047, 1033, 957, 990, 1005, 975] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1017, 1022, 1007, 1027, 1010, 1074, 1015, 1041, 981, 1036, 1040, 1009, 1069, 1040, 993, 1008, 984, 978, 928, 1001, 1014, 1042, 1053, 1052, 996, 1046, 995, 1022, 1066, 1043, 1070, 1013, 997, 1043, 1032, 1026, 989, 1040, 1019, 1059, 975, 1007, 1022, 1015, 1016, 1033, 1027, 1023, 1016, 1027, 1007, 1043, 1040, 1064, 1068, 987, 1011, 1016, 1019, 1016, 1038, 1072, 1066, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1034, 1009, 1018, 1021, 1034, 1064, 1073, 1000, 1041, 1004, 1018, 989, 1030, 997, 1002, 993, 977, 1018, 1028, 1056, 1039, 1021, 1032, 991, 1035, 1000, 1015, 1019, 1003, 1012, 1066, 1033, 1053, 990, 1021, 1035, 1076, 1084, 992, 1038, 1015, 1048, 1026, 984, 985, 1007, 1060, 1022, 1014, 1053, 1068, 1033, 1032, 1036, 1021, 987, 1037, 1017, 1048, 999, 1044, 1033, 1014, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1058, 998, 1049, 998, 1042, 1013, 1026, 1047, 973, 1028, 1054, 990, 996, 1099, 1055, 1021, 1071, 1008, 987, 995, 1069, 1068, 1030, 1023, 1027, 981, 1026, 1034, 1031, 967, 1050, 971, 1018, 1016, 1055, 1029, 968, 1020, 1017, 1033, 1036, 1022, 1025, 1009, 1007, 1022, 1009, 981, 1086, 1060, 1038, 1005, 1078, 980, 1033, 1051, 1011, 1072, 1014, 1004, 984, 1009, 1005, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1028, 1022, 1019, 1040, 1033, 1026, 1023, 1034, 1006, 1087, 1017, 1070, 1016, 992, 1011, 1026, 1036, 1018, 969, 1050, 1026, 1038, 1019, 1059, 1009, 992, 1031, 1017, 981, 1059, 1040, 1033, 1002, 1016, 993, 1021, 991, 1018, 1051, 1022, 1049, 1029, 1037, 990, 1023, 1035, 1009, 1029, 1014, 1049, 1047, 1065, 1049, 959, 1009, 968, 985, 1042, 1037, 1030, 1069, 1045, 1022, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1008, 1006, 1045, 991, 1023, 1040, 1050, 1026, 1055, 991, 1055, 973, 989, 1057, 1040, 1037, 1044, 1077, 992, 1030, 959, 1054, 1062, 979, 1026, 1000, 1018, 1004, 1087, 1063, 1043, 1022, 992, 1037, 1033, 1038, 1035, 1018, 1022, 1027, 1017, 972, 1026, 1036, 980, 1051, 1087, 1034, 1065, 1036, 1023, 1003, 1013, 1027, 1003, 1007, 994, 1041, 1010, 1003, 995, 1025, 1008, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1056, 1000, 1020, 1031, 995, 982, 1080, 1007, 998, 987, 1020, 1022, 1069, 973, 1025, 1027, 1010, 1038, 1012, 1017, 1015, 1027, 1030, 1049, 1046, 992, 964, 1045, 1002, 1007, 1014, 992, 1008, 1026, 1039, 998, 1029, 993, 1019, 1026, 1064, 1025, 1040, 1000, 1026, 1084, 997, 1035, 1049, 1013, 1068, 1062, 1048, 995, 1063, 1033, 1023, 1000, 1038, 1081, 1016, 1061, 1009, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1066, 1050, 1057, 1028, 1026, 1035, 963, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1046, 983, 1032, 1031, 1023, 1037, 1011, 23, 0, 0, 0, 0, 0, 0, 0, 1048, 977, 1063, 1050, 1061, 996, 1024, 1002, 1, 0, 0, 0, 0, 0, 0, 0, 975, 1023, 985, 1013, 1059, 1026, 1030, 1015, 9, 7, 0, 0, 0, 0, 0, 0, 1000, 989, 1036, 1038, 1058, 998, 979, 1018, 25, 0, 0, 0, 0, 0, 0, 0, 979, 1040, 1030, 1027, 1026, 1034, 1025, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1042, 1010, 1042, 965, 1070, 1055, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1055, 1047, 1033, 957, 990, 1005, 975, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8210, 8222, 8142, 8141, 8194, 8275, 8123] avg=8192.0000 max=8275 max_rank=6 min=8123 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1034, 1009, 1018, 1021, 1034, 1064, 1058, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1004, 1018, 989, 1030, 997, 1002, 993, 24, 0, 0, 0, 0, 0, 0, 0, 977, 1018, 1028, 1056, 1039, 1021, 1032, 991, 1, 0, 0, 0, 0, 0, 0, 0, 1035, 1000, 1015, 1019, 1003, 1012, 1066, 1033, 10, 7, 0, 0, 0, 0, 0, 0, 1053, 990, 1021, 1035, 1076, 1084, 992, 1038, 26, 0, 0, 0, 0, 0, 0, 0, 1015, 1048, 1026, 984, 985, 1005, 1060, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1053, 1068, 1033, 975, 1036, 1021, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1017, 1048, 999, 1044, 1033, 1014, 992, 5, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8098, 8163, 8200, 8315, 8145, 8187, 8190] avg=8192.0000 max=8315 max_rank=4 min=8098 min_rank=1 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1017, 1006, 1019, 1005, 988, 1011, 1035, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1011, 1020, 1023, 973, 1053, 1031, 1027, 24, 0, 0, 0, 0, 0, 0, 0, 989, 1011, 1039, 1113, 1040, 1001, 1027, 1005, 1, 0, 0, 0, 0, 0, 0, 0, 1004, 1059, 1050, 1073, 1040, 1021, 1024, 985, 10, 7, 0, 0, 0, 0, 0, 0, 1035, 1013, 1029, 1010, 1016, 1018, 985, 999, 26, 0, 0, 0, 0, 0, 0, 0, 1057, 1017, 1065, 1059, 1043, 1041, 996, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1013, 1019, 1035, 985, 980, 1045, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1006, 982, 1025, 1015, 1039, 1031, 1023, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8169, 8226, 8273, 8131, 8314, 8122, 8180] avg=8192.0000 max=8314 max_rank=5 min=8121 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1017, 1022, 1007, 1027, 1010, 1074, 1001, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1036, 1040, 1009, 1069, 1040, 993, 1008, 24, 0, 0, 0, 0, 0, 0, 0, 984, 978, 928, 1001, 1014, 1042, 1053, 1052, 1, 0, 0, 0, 0, 0, 0, 0, 996, 1046, 995, 1022, 1066, 1043, 1070, 1013, 10, 7, 0, 0, 0, 0, 0, 0, 997, 1043, 1032, 1026, 989, 1040, 1019, 1059, 26, 0, 0, 0, 0, 0, 0, 0, 975, 1007, 1022, 1015, 1016, 1031, 1027, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1027, 1007, 1043, 983, 1064, 1068, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1016, 1019, 1016, 1038, 1072, 1066, 1031, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8200, 8053, 8268, 8231, 8116, 8195, 8274] avg=8192.0000 max=8274 max_rank=7 min=8053 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1028, 1022, 1019, 1040, 1033, 1026, 1009, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1087, 1017, 1070, 1016, 992, 1011, 1026, 24, 0, 0, 0, 0, 0, 0, 0, 1036, 1018, 969, 1050, 1026, 1038, 1019, 1059, 1, 0, 0, 0, 0, 0, 0, 0, 1009, 992, 1031, 1017, 981, 1059, 1040, 1033, 10, 7, 0, 0, 0, 0, 0, 0, 1002, 1016, 993, 1021, 991, 1018, 1051, 1022, 26, 0, 0, 0, 0, 0, 0, 0, 1049, 1029, 1037, 990, 1023, 1033, 1009, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1049, 1047, 1065, 992, 959, 1009, 968, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1042, 1037, 1030, 1069, 1045, 1022, 1004, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8249, 8216, 8179, 8140, 8199, 8103, 8239] avg=8192.0000 max=8249 max_rank=1 min=8103 min_rank=6 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1058, 998, 1049, 998, 1042, 1013, 1012, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1028, 1054, 990, 996, 1099, 1055, 1021, 25, 0, 0, 0, 0, 0, 0, 0, 1071, 1008, 987, 995, 1069, 1068, 1030, 1023, 1, 0, 0, 0, 0, 0, 0, 0, 1027, 981, 1026, 1034, 1031, 967, 1050, 971, 10, 8, 0, 0, 0, 0, 0, 0, 1018, 1016, 1055, 1029, 968, 1020, 1017, 1033, 27, 0, 0, 0, 0, 0, 0, 0, 1036, 1022, 1025, 1009, 1007, 1020, 1009, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 1060, 1038, 1005, 1018, 980, 1033, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1072, 1014, 1004, 984, 1009, 1005, 1054, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8241, 8252, 8105, 8183, 8109, 8271, 8158] avg=8192.0000 max=8271 max_rank=6 min=8105 min_rank=3 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1008, 1006, 1045, 991, 1023, 1040, 1036, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 991, 1055, 973, 989, 1057, 1040, 1037, 23, 0, 0, 0, 0, 0, 0, 0, 1044, 1077, 992, 1030, 959, 1054, 1062, 979, 1, 0, 0, 0, 0, 0, 0, 0, 1026, 1000, 1018, 1004, 1087, 1063, 1043, 1022, 10, 7, 0, 0, 0, 0, 0, 0, 992, 1037, 1033, 1038, 1035, 1018, 1022, 1027, 25, 0, 0, 0, 0, 0, 0, 0, 1017, 972, 1026, 1036, 980, 1049, 1087, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1036, 1023, 1003, 958, 1027, 1003, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1041, 1010, 1003, 995, 1025, 1008, 1032, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8220, 8198, 8280, 8227, 8201, 8122, 8113] avg=8192.0000 max=8280 max_rank=3 min=8113 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32663 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1056, 1000, 1020, 1031, 995, 982, 1055, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 998, 987, 1020, 1022, 1069, 973, 1025, 1027, 28, 0, 0, 0, 0, 0, 0, 0, 1010, 1038, 1012, 1017, 1015, 1027, 1030, 1049, 8, 0, 0, 0, 0, 0, 0, 0, 1046, 992, 964, 1045, 1002, 1007, 1014, 992, 15, 12, 0, 0, 0, 0, 0, 0, 1008, 1026, 1039, 998, 1029, 993, 1019, 1026, 30, 0, 0, 0, 0, 0, 0, 0, 1064, 1025, 1040, 1000, 1026, 1071, 997, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1013, 1068, 1062, 978, 995, 1063, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1000, 1038, 1081, 1016, 1061, 1009, 1016, 10, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8149, 8206, 8089, 8168, 8258, 8261, 8259] avg=8192.0000 max=8261 max_rank=6 min=8089 min_rank=3 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32946 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8320, 7936, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8064, 8320, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8448, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1054, 1034, 997, 1034, 1017, 1007, 1064, 1009, 996, 1067, 1010, 1025, 1010, 1008, 1023, 1044, 1006, 1020, 1053, 1002, 1016, 962, 1036, 1028, 986, 1033, 1056, 1063, 1029, 1056, 1008, 1005, 1016, 1006, 1039, 981, 1015, 1014, 987, 1047, 1017, 1071, 1014, 1044, 1014, 1017, 1030, 1024, 1053, 968, 996, 1019, 1025, 1064, 1034, 1044, 1043, 979, 1073, 1048, 1022, 1008, 1039, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1039, 1090, 1081, 1042, 1015, 1060, 981, 1004, 985, 1026, 1023, 984, 1076, 1000, 1066, 1017, 1008, 1085, 1048, 1077, 1023, 1017, 998, 1034, 1050, 1019, 1047, 1076, 1013, 1020, 960, 1033, 1072, 1027, 1048, 968, 1029, 1036, 1018, 1031, 1006, 1024, 1037, 964, 995, 994, 1077, 1067, 989, 1010, 1006, 1004, 1034, 1008, 1062, 993, 1030, 975, 988, 960, 1022, 1037, 1015, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1005, 997, 997, 1043, 1012, 1024, 987, 997, 1019, 1030, 990, 1048, 1032, 1015, 1039, 1006, 1004, 1025, 1021, 1014, 1054, 1036, 1082, 1023, 1010, 993, 1074, 1058, 1058, 1049, 1018, 1050, 1040, 1024, 1011, 1080, 1052, 1037, 1018, 998, 1001, 1001, 1031, 1041, 984, 1019, 1023, 1013, 1017, 1016, 1059, 964, 1037, 1040, 1012, 1065, 975, 1029, 1030, 1018, 1027, 1009, 1044, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1079, 1038, 1050, 1055, 1004, 1009, 1022, 995, 1012, 1068, 1045, 1036, 983, 990, 1059, 1010, 1028, 1016, 1009, 986, 989, 1097, 1004, 1024, 1051, 991, 1008, 1016, 955, 1056, 1059, 1115, 1038, 1026, 1007, 1059, 991, 1049, 1073, 1007, 1017, 1033, 1028, 1014, 1033, 978, 992, 1067, 1039, 1014, 1071, 997, 1055, 989, 995, 1000, 1029, 1056, 980, 971, 1016, 1013, 1029, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1064, 980, 983, 1002, 1025, 1021, 982, 1019, 1025, 1010, 1036, 1014, 1040, 1034, 966, 1028, 993, 1028, 1066, 1019, 1011, 1038, 984, 1023, 1021, 953, 1030, 998, 1030, 1052, 1056, 1017, 1015, 1017, 1040, 1028, 1014, 1029, 1003, 1015, 995, 1059, 1026, 1042, 1045, 1051, 1084, 1039, 1023, 1016, 1045, 983, 1043, 1004, 1021, 1020, 1014, 1057, 1042, 1048, 1015, 1067, 1043, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1032, 1030, 1008, 1032, 1050, 1005, 1025, 997, 1011, 968, 973, 1022, 995, 1052, 1042, 1021, 1001, 985, 1032, 1036, 1065, 1015, 1049, 1016, 1037, 1026, 1032, 1040, 1022, 1017, 1025, 1018, 1030, 1093, 1013, 1019, 1007, 1034, 1019, 1062, 1040, 1040, 1027, 1013, 1054, 972, 995, 1039, 1014, 1073, 1057, 1045, 980, 1042, 1030, 1019, 995, 1005, 959, 976, 1044, 1020, 1083, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1058, 1050, 1015, 984, 1021, 1031, 1037, 1072, 1009, 1019, 1023, 1046, 988, 1005, 1058, 1041, 1037, 1026, 1011, 1025, 1004, 1020, 1055, 1006, 1002, 978, 1008, 1021, 1015, 990, 1017, 1015, 1070, 1000, 1039, 1045, 1067, 1017, 1037, 1070, 1034, 1019, 1007, 1000, 1023, 1031, 990, 1042, 1001, 1017, 972, 1045, 1008, 1070, 1009, 1011, 1012, 1056, 1048, 994, 1042, 1026, 1067, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1013, 1043, 1060, 990, 1068, 1047, 1021, 994, 1052, 1002, 1004, 1070, 1053, 1035, 1041, 1025, 1067, 1014, 994, 1008, 978, 1084, 1035, 989, 1063, 975, 1000, 1079, 1017, 1032, 936, 979, 1037, 988, 1036, 987, 1043, 1014, 1020, 958, 1046, 1017, 1022, 1024, 1012, 1006, 1065, 1068, 1068, 1053, 1044, 1036, 1008, 1026, 1029, 1004, 1038, 994, 1063, 1047, 1007, 1006, 985, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1046, 1034, 997, 1034, 1017, 1007, 1064, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1067, 1010, 1025, 1010, 1008, 1023, 1044, 8, 2, 0, 0, 0, 0, 0, 0, 1006, 1020, 1053, 1002, 1016, 962, 1036, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1033, 1056, 1063, 1029, 1056, 1008, 1005, 2, 0, 0, 0, 0, 0, 0, 0, 995, 1006, 1039, 981, 1015, 1014, 987, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1071, 1014, 1044, 1014, 1017, 1030, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 968, 996, 1019, 1025, 1064, 1034, 1044, 5, 0, 0, 0, 0, 0, 0, 0, 1043, 979, 1073, 1048, 1022, 1008, 1039, 1027, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8193, 8123, 8238, 8084, 8224, 8208, 8258] avg=8192.0000 max=8258 max_rank=7 min=8084 min_rank=4 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1031, 1090, 1081, 1042, 1015, 1060, 981, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1026, 1023, 984, 1076, 1000, 1066, 1017, 8, 2, 0, 0, 0, 0, 0, 0, 1008, 1085, 1048, 1077, 1023, 1017, 998, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1019, 1047, 1076, 1013, 1020, 960, 1033, 2, 0, 0, 0, 0, 0, 0, 0, 1050, 1027, 1048, 968, 1029, 1036, 1018, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1024, 1037, 964, 995, 994, 1077, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1010, 1006, 1004, 1034, 1008, 1062, 993, 5, 0, 0, 0, 0, 0, 0, 0, 1030, 975, 988, 960, 1022, 1037, 1015, 1013, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8304, 8187, 8290, 8220, 8207, 8157, 8111, 8060] avg=8192.0000 max=8304 max_rank=0 min=8060 min_rank=7 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32535 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1024, 1030, 1008, 1032, 1050, 1005, 1025, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 968, 973, 1022, 995, 1052, 1042, 1021, 8, 2, 0, 0, 0, 0, 0, 0, 1001, 985, 1032, 1036, 1065, 1015, 1049, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1026, 1032, 1040, 1022, 1017, 1025, 1018, 2, 0, 0, 0, 0, 0, 0, 0, 1009, 1093, 1013, 1019, 1007, 1034, 1019, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1040, 1027, 1013, 1054, 972, 995, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1073, 1057, 1045, 980, 1042, 1030, 1019, 5, 0, 0, 0, 0, 0, 0, 0, 995, 1005, 959, 976, 1044, 1020, 1083, 1058, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8094, 8199, 8219, 8256, 8173, 8265, 8159] avg=8192.0000 max=8265 max_rank=6 min=8094 min_rank=1 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1056, 980, 983, 1002, 1025, 1021, 982, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1010, 1036, 1014, 1040, 1034, 966, 1028, 8, 2, 0, 0, 0, 0, 0, 0, 993, 1028, 1066, 1019, 1011, 1038, 984, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 953, 1030, 998, 1030, 1052, 1056, 1017, 2, 0, 0, 0, 0, 0, 0, 0, 994, 1017, 1040, 1028, 1014, 1029, 1003, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1059, 1026, 1042, 1045, 1051, 1084, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1016, 1045, 983, 1043, 1004, 1021, 1020, 5, 0, 0, 0, 0, 0, 0, 0, 1014, 1057, 1042, 1048, 1015, 1067, 1043, 1045, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8068, 8163, 8162, 8159, 8140, 8334, 8160, 8350] avg=8192.0000 max=8350 max_rank=7 min=8068 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32984 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[998, 997, 997, 1043, 1012, 1024, 987, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1030, 990, 1048, 1032, 1015, 1039, 1006, 7, 2, 0, 0, 0, 0, 0, 0, 1004, 1025, 1021, 1014, 1054, 1036, 1082, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 993, 1074, 1058, 1058, 1049, 1018, 1050, 2, 0, 0, 0, 0, 0, 0, 0, 1019, 1024, 1011, 1080, 1052, 1037, 1018, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1001, 1031, 1041, 984, 1019, 1023, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1016, 1059, 964, 1037, 1040, 1012, 1065, 4, 0, 0, 0, 0, 0, 0, 0, 975, 1029, 1030, 1018, 1027, 1009, 1044, 1011, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8055, 8188, 8259, 8312, 8239, 8107, 8214, 8162] avg=8192.0000 max=8312 max_rank=3 min=8055 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1071, 1038, 1050, 1055, 1004, 1009, 1022, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1068, 1045, 1036, 983, 990, 1059, 1010, 8, 2, 0, 0, 0, 0, 0, 0, 1028, 1016, 1009, 986, 989, 1097, 1004, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 991, 1008, 1016, 955, 1056, 1059, 1115, 2, 0, 0, 0, 0, 0, 0, 0, 1017, 1026, 1007, 1059, 991, 1049, 1073, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1033, 1028, 1014, 1033, 978, 992, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1014, 1071, 997, 1055, 989, 995, 1000, 5, 0, 0, 0, 0, 0, 0, 0, 1029, 1056, 980, 971, 1016, 1013, 1029, 1011, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8244, 8213, 8153, 8253, 8229, 8155, 8165, 8124] avg=8192.0000 max=8253 max_rank=3 min=8124 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1050, 1050, 1015, 984, 1021, 1031, 1037, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1019, 1023, 1046, 988, 1005, 1058, 1041, 8, 2, 0, 0, 0, 0, 0, 0, 1037, 1026, 1011, 1025, 1004, 1020, 1055, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 978, 1008, 1021, 1015, 990, 1017, 1015, 2, 0, 0, 0, 0, 0, 0, 0, 1048, 1000, 1039, 1045, 1067, 1017, 1037, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1019, 1007, 1000, 1023, 1031, 990, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1017, 972, 1045, 1008, 1070, 1009, 1011, 5, 0, 0, 0, 0, 0, 0, 0, 1012, 1056, 1048, 994, 1042, 1026, 1067, 980, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8260, 8199, 8184, 8048, 8323, 8139, 8138, 8245] avg=8192.0000 max=8323 max_rank=4 min=8048 min_rank=3 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32845 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1003, 1043, 1060, 990, 1068, 1047, 1021, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1002, 1004, 1070, 1053, 1035, 1041, 1025, 10, 4, 3, 0, 0, 0, 0, 0, 1067, 1014, 994, 1008, 978, 1081, 1035, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 975, 1000, 1079, 1017, 1032, 936, 979, 3, 3, 0, 0, 0, 0, 0, 0, 1012, 988, 1036, 987, 1043, 1014, 1020, 958, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1017, 1022, 1024, 1012, 1006, 1065, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1053, 1044, 1036, 1008, 1026, 1029, 1004, 7, 0, 0, 0, 0, 0, 0, 0, 1038, 994, 1063, 1047, 1007, 1006, 985, 1017, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8299, 8166, 8087, 8058, 8247, 8275, 8178] avg=8192.0000 max=8299 max_rank=1 min=8058 min_rank=4 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8064, 8320, 8448, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8064, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1004, 1048, 1038, 1020, 1007, 1003, 1046, 1054, 1048, 1007, 1095, 1036, 995, 1026, 1048, 1103, 1028, 1000, 1024, 1037, 999, 1063, 1016, 1017, 1030, 1020, 1002, 1003, 1010, 1048, 995, 1047, 1012, 1041, 1070, 1003, 986, 1025, 1043, 1021, 1023, 1048, 1042, 1028, 1037, 999, 1036, 994, 980, 1037, 1024, 1032, 992, 1011, 930, 1017, 1024, 1040, 1047, 994, 989, 1022, 1026, 1046] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1007, 984, 991, 1072, 1053, 1004, 1077, 1014, 1063, 1014, 1017, 987, 1030, 1035, 1039, 982, 1001, 983, 1056, 993, 1057, 966, 976, 1040, 1027, 1005, 998, 1007, 1000, 1029, 1045, 994, 1041, 1005, 1073, 1021, 1003, 1056, 996, 1076, 1055, 1062, 1028, 1082, 983, 996, 997, 1053, 1058, 1010, 991, 1023, 1017, 1016, 1027, 1009, 1039, 1074, 1038, 1065, 1026, 1033, 1034, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[979, 1048, 1004, 984, 1048, 1002, 999, 1051, 1072, 995, 1033, 973, 1049, 1059, 1034, 1021, 968, 1060, 1032, 1049, 1021, 1036, 1003, 1006, 1022, 1057, 1017, 1019, 1064, 1033, 1024, 1024, 991, 1021, 993, 1022, 1068, 986, 1052, 1021, 1027, 1035, 1039, 1006, 1038, 1058, 1008, 1014, 1023, 1031, 987, 1023, 986, 1034, 973, 1073, 1012, 1033, 1045, 1028, 1029, 1032, 1024, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1025, 1071, 1009, 1041, 1023, 1005, 1045, 1026, 1050, 1079, 1025, 1057, 1065, 1031, 1041, 1027, 1070, 992, 1027, 1000, 1077, 1039, 1037, 1040, 966, 986, 1039, 1049, 1055, 963, 989, 1002, 1018, 1010, 993, 1005, 1018, 1050, 1019, 1018, 1038, 1047, 1038, 1013, 1068, 1026, 1034, 975, 1002, 1003, 958, 1033, 1015, 1001, 1020, 1031, 1061, 1052, 994, 969, 1039, 1019, 1011, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1054, 1022, 1040, 1023, 1042, 1053, 1063, 1035, 991, 1010, 1042, 1050, 1060, 1034, 989, 1052, 1035, 1000, 977, 1026, 1007, 1008, 990, 1043, 1003, 994, 993, 1025, 1004, 1036, 1028, 1065, 989, 1002, 1031, 1027, 1004, 1052, 1025, 1033, 996, 1015, 1027, 1010, 1038, 1041, 1031, 999, 1037, 1006, 1066, 1030, 1013, 1077, 1060, 1044, 979, 1035, 1005, 1003, 1021, 1035, 1012, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1030, 1025, 1002, 972, 1100, 1020, 1012, 1050, 1040, 987, 1017, 1038, 1031, 1061, 1057, 1030, 1081, 1025, 1075, 1014, 1053, 1011, 1017, 996, 1047, 1017, 969, 995, 1024, 1013, 1033, 959, 1010, 1012, 1004, 1053, 1072, 1051, 1003, 1062, 1034, 1056, 986, 1024, 1043, 1038, 1020, 1034, 1044, 993, 1035, 1005, 1012, 1016, 1003, 1034, 985, 1030, 1023, 1046, 1007, 1028, 991, 981] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1002, 1038, 1067, 1023, 971, 1009, 1020, 993, 1030, 1042, 1031, 1023, 1026, 1045, 1034, 1052, 1022, 1007, 1081, 1019, 1018, 948, 982, 1055, 1035, 1060, 1052, 1039, 1053, 1053, 1049, 1036, 1037, 1007, 1027, 971, 996, 1007, 1011, 1047, 1060, 1010, 1016, 1026, 1015, 1028, 1029, 1010, 1080, 977, 1046, 1039, 1010, 1045, 983, 1078, 1018, 1009, 1003, 1009, 997, 1031, 1004, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1113, 1003, 1019, 997, 998, 1046, 1064, 1018, 1002, 1041, 1015, 1018, 978, 974, 1021, 1004, 1027, 1012, 995, 1051, 1005, 1018, 1016, 1050, 978, 1009, 1038, 1066, 1011, 1026, 1014, 1056, 998, 1035, 1005, 1079, 1057, 1013, 1042, 1057, 1018, 1008, 1015, 1002, 1006, 1076, 1013, 983, 968, 1026, 1043, 1044, 1049, 1020, 1074, 1009, 1007, 1029, 1023, 1024, 992, 1050, 1056, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1007, 984, 991, 1072, 1053, 1004, 1056, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1014, 1017, 987, 1030, 1035, 1039, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 983, 1056, 993, 1057, 966, 976, 1040, 12, 5, 2, 0, 0, 0, 0, 0, 1027, 1005, 998, 1007, 1000, 1029, 1045, 994, 23, 0, 0, 0, 0, 0, 0, 0, 1041, 1005, 1073, 1021, 1003, 1056, 996, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1050, 1028, 1082, 983, 996, 997, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1010, 991, 1023, 1017, 1016, 1027, 1009, 26, 0, 0, 0, 0, 0, 0, 0, 1039, 1074, 1038, 1065, 1026, 1033, 1034, 1003, 19, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8114, 8091, 8128, 8266, 8244, 8177, 8335] avg=8192.0000 max=8335 max_rank=7 min=8091 min_rank=2 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=33022 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1004, 1048, 1038, 1020, 1007, 1003, 1026, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1007, 1095, 1036, 995, 1026, 1048, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1000, 1024, 1037, 999, 1063, 1016, 1017, 12, 4, 2, 0, 0, 0, 0, 0, 1030, 1020, 1002, 1003, 1010, 1048, 995, 1047, 23, 0, 0, 0, 0, 0, 0, 0, 1012, 1041, 1070, 1003, 986, 1025, 1043, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1036, 1042, 1028, 1037, 999, 1036, 994, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1037, 1024, 1032, 992, 1011, 930, 1017, 25, 0, 0, 0, 0, 0, 0, 0, 1024, 1040, 1047, 994, 989, 1022, 1026, 1046, 18, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8306, 8202, 8178, 8197, 8195, 8048, 8210] avg=8192.0000 max=8306 max_rank=1 min=8048 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1054, 1022, 1040, 1023, 1042, 1053, 1042, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 941, 1010, 1042, 1050, 1060, 1034, 989, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1000, 977, 1026, 1007, 1008, 990, 1043, 12, 4, 2, 0, 0, 0, 0, 0, 1003, 994, 993, 1025, 1004, 1036, 1028, 1065, 22, 0, 0, 0, 0, 0, 0, 0, 989, 1002, 1031, 1027, 1004, 1052, 1025, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1003, 1027, 1010, 1038, 1041, 1031, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1006, 1066, 1030, 1013, 1077, 1060, 1044, 24, 0, 0, 0, 0, 0, 0, 0, 979, 1035, 1005, 1003, 1021, 1035, 1012, 999, 19, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8178, 8104, 8170, 8159, 8145, 8357, 8112] avg=8192.0000 max=8357 max_rank=6 min=8104 min_rank=2 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1025, 1071, 1009, 1041, 1023, 1005, 1025, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1079, 1025, 1057, 1065, 1031, 1041, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 992, 1027, 1000, 1077, 1039, 1037, 1040, 12, 4, 2, 0, 0, 0, 0, 0, 966, 986, 1039, 1049, 1055, 963, 989, 1002, 23, 0, 0, 0, 0, 0, 0, 0, 1018, 1010, 993, 1005, 1018, 1050, 1019, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1035, 1038, 1013, 1068, 1026, 1034, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1003, 958, 1033, 1015, 1001, 1020, 1031, 25, 0, 0, 0, 0, 0, 0, 0, 1061, 1052, 994, 969, 1039, 1019, 1011, 1007, 18, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8323, 8300, 8072, 8127, 8227, 8088, 8174] avg=8192.0000 max=8323 max_rank=1 min=8072 min_rank=3 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32616 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1002, 1038, 1067, 1023, 971, 1009, 1000, 993, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1042, 1031, 1023, 1026, 1045, 1034, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1007, 1081, 1019, 1018, 948, 982, 1055, 11, 5, 2, 0, 0, 0, 0, 0, 1035, 1060, 1052, 1039, 1053, 1053, 1049, 1036, 23, 0, 0, 0, 0, 0, 0, 0, 1037, 1007, 1027, 971, 996, 1007, 1011, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 999, 1016, 1026, 1015, 1028, 1029, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 977, 1046, 1039, 1010, 1045, 983, 1078, 25, 0, 0, 0, 0, 0, 0, 0, 1018, 1009, 1003, 1009, 997, 1031, 1004, 995, 18, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8231, 8150, 8400, 8098, 8183, 8283, 8088] avg=8192.0000 max=8400 max_rank=3 min=8088 min_rank=7 imbalance_ratio=1.025391 max_load_reduction=57136 max_load_reduction_pct=0.871826 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1030, 1025, 1002, 972, 1100, 1020, 992, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 988, 987, 1017, 1038, 1031, 1061, 1057, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1025, 1075, 1014, 1053, 1011, 1017, 996, 12, 5, 2, 0, 0, 0, 0, 0, 1047, 1017, 969, 995, 1024, 1013, 1033, 959, 23, 0, 0, 0, 0, 0, 0, 0, 1010, 1012, 1004, 1053, 1072, 1051, 1003, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1044, 986, 1024, 1043, 1038, 1020, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 993, 1035, 1005, 1012, 1016, 1003, 1034, 25, 0, 0, 0, 0, 0, 0, 0, 985, 1030, 1023, 1046, 1007, 1028, 991, 981, 18, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8209, 8291, 8080, 8262, 8223, 8167, 8113] avg=8192.0000 max=8291 max_rank=2 min=8080 min_rank=3 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[979, 1048, 1004, 984, 1048, 1002, 980, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 995, 1033, 973, 1049, 1059, 1034, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1060, 1032, 1049, 1021, 1036, 1003, 1006, 12, 4, 2, 0, 0, 0, 0, 0, 1022, 1057, 1017, 1019, 1064, 1033, 1024, 1024, 24, 0, 0, 0, 0, 0, 0, 0, 991, 1021, 993, 1022, 1068, 986, 1052, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1023, 1039, 1006, 1038, 1058, 1008, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1031, 987, 1023, 986, 1034, 973, 1073, 26, 0, 0, 0, 0, 0, 0, 0, 1012, 1033, 1045, 1028, 1029, 1032, 1024, 1038, 17, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8182, 8193, 8284, 8150, 8213, 8156, 8262] avg=8192.0000 max=8284 max_rank=3 min=8096 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1113, 1003, 1019, 997, 998, 1046, 1035, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 942, 1041, 1015, 1018, 978, 974, 1021, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1012, 995, 1051, 1005, 1018, 1016, 1050, 15, 9, 7, 0, 0, 0, 0, 0, 978, 1009, 1038, 1066, 1011, 1026, 1014, 1056, 26, 0, 0, 0, 0, 0, 0, 0, 998, 1035, 1005, 1079, 1057, 1013, 1042, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 993, 1015, 1002, 1006, 1076, 1013, 983, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1026, 1043, 1044, 1049, 1020, 1074, 1009, 27, 0, 0, 0, 0, 0, 0, 0, 1007, 1029, 1023, 1024, 992, 1050, 1056, 1032, 22, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 7993, 8205, 8224, 8277, 8106, 8260, 8242] avg=8192.0000 max=8277 max_rank=4 min=7993 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7936, 8192, 8320, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8064, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65792 slack_slots=4480 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1074, 1041, 1035, 995, 1053, 1048, 1038, 1050, 1008, 1069, 1008, 1069, 1027, 1019, 977, 1043, 1007, 1019, 997, 1017, 1036, 1010, 1031, 1046, 1046, 1006, 1000, 1025, 1049, 1004, 1048, 1039, 1065, 1009, 1025, 987, 1047, 1021, 1008, 1022, 1000, 1018, 1006, 975, 968, 1022, 1045, 1016, 991, 1012, 1051, 1031, 1050, 1066, 994, 1042, 1000, 1009, 1036, 1033, 1032, 1016, 1032, 973] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1023, 998, 1024, 989, 1097, 995, 999, 989, 1037, 1039, 1008, 1063, 1017, 1018, 988, 1027, 1023, 1069, 1026, 1013, 1027, 1019, 998, 1016, 1032, 1041, 1045, 1001, 999, 1015, 994, 1023, 1048, 1025, 974, 1014, 1035, 1004, 981, 1016, 1103, 1069, 1057, 987, 1010, 1046, 1027, 1002, 1042, 1077, 1026, 1036, 1010, 1060, 1035, 1031, 1021, 1032, 1019, 1031, 1009, 1057, 1030, 970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1052, 1053, 989, 983, 998, 958, 974, 1021, 1038, 1041, 971, 1058, 1069, 1069, 1023, 1000, 958, 1050, 1011, 1012, 1104, 1026, 1043, 992, 947, 999, 1015, 1043, 996, 1022, 1037, 1014, 1071, 998, 1074, 1020, 1044, 998, 1013, 1041, 1039, 1066, 1017, 1053, 1030, 1017, 996, 1010, 1030, 1044, 1008, 1051, 1022, 1020, 1052, 995, 976, 1038, 1036, 1045, 1020, 1012, 1077, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[994, 1046, 1018, 1052, 1050, 1028, 1034, 1033, 1022, 1041, 1024, 1031, 978, 1030, 1031, 1019, 1006, 1041, 1065, 1040, 1043, 1025, 1100, 1007, 1021, 1054, 1024, 1006, 964, 1060, 1006, 1028, 993, 990, 1046, 1036, 1091, 1018, 1024, 1056, 1015, 973, 1052, 1048, 967, 1037, 987, 1019, 1019, 1060, 1024, 1016, 1016, 1013, 1064, 997, 994, 1001, 1071, 978, 1026, 1023, 956, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1025, 1033, 981, 1050, 1043, 1031, 1071, 1014, 1067, 1047, 976, 1049, 1023, 1016, 1064, 1039, 979, 1037, 1008, 1003, 1060, 1016, 1018, 978, 1000, 1023, 1015, 1050, 1054, 1030, 1029, 993, 1037, 1006, 1037, 976, 1015, 982, 1062, 996, 1000, 992, 997, 1060, 1062, 1053, 976, 1025, 1037, 986, 1037, 1002, 1005, 1038, 1061, 1074, 1018, 986, 1025, 1031, 1068, 1038, 1004, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[986, 1103, 1057, 1045, 1027, 1011, 1012, 1008, 1052, 977, 1052, 1082, 1017, 976, 1030, 1025, 1035, 1048, 1015, 1042, 1008, 1005, 1043, 1048, 979, 1059, 1024, 1024, 1031, 1003, 964, 1004, 1067, 1002, 1034, 1003, 1035, 995, 1003, 1016, 1016, 1066, 1108, 1075, 952, 1019, 1022, 1012, 1025, 1020, 1020, 1024, 1046, 986, 1033, 1036, 991, 985, 1036, 1011, 1019, 978, 1043, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1048, 1040, 963, 976, 1040, 1007, 1038, 1058, 1060, 1000, 1025, 1009, 1035, 1069, 1067, 952, 1000, 1007, 975, 1016, 998, 1032, 1016, 1022, 1012, 972, 1052, 1045, 1032, 1047, 1064, 1051, 1022, 1010, 1049, 1028, 989, 1076, 1006, 1081, 1026, 982, 1043, 1084, 1004, 1035, 1006, 1024, 1006, 1042, 1069, 1025, 1040, 1010, 1031, 1001, 991, 1026, 1018, 956, 1027, 1038, 1040, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1014, 987, 1021, 970, 1038, 1043, 1033, 1036, 1093, 1070, 1048, 1020, 1068, 1064, 1054, 981, 984, 1003, 1051, 989, 1011, 1038, 1063, 1079, 1065, 1039, 978, 1005, 1048, 1016, 995, 1058, 978, 1032, 1056, 1000, 1033, 1057, 1046, 1051, 1012, 1003, 1005, 1015, 1064, 1001, 1029, 977, 986, 1006, 1023, 1035, 1048, 1035, 1051, 1017, 1059, 1016, 1038, 977, 1001, 962, 1011, 950] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1025, 1033, 981, 1050, 1043, 1031, 1071, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1047, 976, 995, 1023, 1016, 1064, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1037, 1008, 1003, 1060, 1016, 1018, 978, 3, 0, 0, 0, 0, 0, 0, 0, 1000, 1023, 1015, 1050, 1054, 1030, 1029, 993, 21, 0, 0, 0, 0, 0, 0, 0, 1037, 1006, 1033, 976, 1015, 982, 1062, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 992, 997, 1060, 1062, 1053, 976, 1025, 13, 1, 0, 0, 0, 0, 0, 0, 1037, 986, 1037, 1002, 1005, 1038, 1024, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 986, 1025, 1031, 1068, 1038, 1004, 1028, 54, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8227, 8102, 8215, 8107, 8179, 8203, 8255] avg=8192.0000 max=8255 max_rank=7 min=8102 min_rank=2 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1023, 998, 1024, 989, 1097, 995, 999, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1039, 1008, 1009, 1017, 1018, 988, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1069, 1026, 1013, 1027, 1019, 998, 1016, 3, 0, 0, 0, 0, 0, 0, 0, 1032, 1041, 1045, 1001, 999, 1015, 994, 1023, 21, 0, 0, 0, 0, 0, 0, 0, 1048, 1025, 970, 1014, 1035, 1004, 981, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 1069, 1057, 987, 1010, 1046, 1027, 1002, 12, 1, 0, 0, 0, 0, 0, 0, 1042, 1077, 1026, 1036, 1010, 1060, 999, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1032, 1019, 1031, 1009, 1057, 1030, 970, 54, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8143, 8194, 8171, 8093, 8314, 8281, 8226] avg=8192.0000 max=8314 max_rank=5 min=8093 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1074, 1041, 1035, 995, 1053, 1048, 1038, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1069, 1008, 1014, 1027, 1019, 977, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1019, 997, 1017, 1036, 1010, 1031, 1046, 3, 0, 0, 0, 0, 0, 0, 0, 1046, 1006, 1000, 1025, 1049, 1004, 1048, 1039, 20, 0, 0, 0, 0, 0, 0, 0, 1065, 1009, 1021, 987, 1047, 1021, 1008, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1018, 1006, 975, 968, 1022, 1045, 1016, 12, 1, 0, 0, 0, 0, 0, 0, 991, 1012, 1051, 1031, 1050, 1066, 959, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1009, 1036, 1033, 1032, 1016, 1032, 973, 55, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8334, 8165, 8166, 8237, 8180, 8063, 8202, 8189] avg=8192.0000 max=8334 max_rank=0 min=8063 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[994, 1046, 1018, 1052, 1050, 1028, 1034, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1041, 1024, 978, 978, 1030, 1031, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1041, 1065, 1040, 1043, 1025, 1100, 1007, 3, 0, 0, 0, 0, 0, 0, 0, 1021, 1054, 1024, 1006, 964, 1060, 1006, 1028, 21, 0, 0, 0, 0, 0, 0, 0, 993, 990, 1042, 1036, 1091, 1018, 1024, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 973, 1052, 1048, 967, 1037, 987, 1019, 13, 1, 0, 0, 0, 0, 0, 0, 1019, 1060, 1024, 1016, 1016, 1013, 1027, 997, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1001, 1071, 978, 1026, 1023, 956, 1005, 53, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8123, 8330, 8184, 8250, 8112, 8172, 8110] avg=8192.0000 max=8330 max_rank=2 min=8110 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1052, 1053, 989, 983, 998, 958, 974, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1041, 971, 1004, 1069, 1069, 1023, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 958, 1050, 1011, 1012, 1104, 1026, 1043, 992, 3, 0, 0, 0, 0, 0, 0, 0, 947, 999, 1015, 1043, 996, 1022, 1037, 1014, 21, 0, 0, 0, 0, 0, 0, 0, 1071, 998, 1070, 1020, 1044, 998, 1013, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1066, 1017, 1053, 1030, 1017, 996, 1010, 13, 1, 0, 0, 0, 0, 0, 0, 1030, 1044, 1008, 1051, 1022, 1020, 1015, 995, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1038, 1036, 1045, 1020, 1012, 1077, 1057, 54, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8028, 8215, 8199, 8094, 8255, 8242, 8185, 8318] avg=8192.0000 max=8318 max_rank=7 min=8028 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=33000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1048, 1040, 963, 976, 1040, 1007, 1038, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1000, 1025, 957, 1035, 1069, 1067, 952, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1007, 975, 1016, 998, 1032, 1016, 1022, 3, 0, 0, 0, 0, 0, 0, 0, 1012, 972, 1052, 1045, 1032, 1047, 1064, 1051, 21, 0, 0, 0, 0, 0, 0, 0, 1022, 1010, 1045, 1028, 989, 1076, 1006, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 982, 1043, 1084, 1004, 1035, 1006, 1024, 12, 1, 0, 0, 0, 0, 0, 0, 1006, 1042, 1069, 1025, 1040, 1010, 995, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1026, 1018, 956, 1027, 1038, 1040, 1023, 52, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8165, 8069, 8296, 8257, 8217, 8188, 8174] avg=8192.0000 max=8296 max_rank=3 min=8069 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[986, 1103, 1057, 1045, 1027, 1011, 1012, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 977, 1052, 1027, 1017, 976, 1030, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1048, 1015, 1042, 1008, 1005, 1043, 1048, 3, 0, 0, 0, 0, 0, 0, 0, 979, 1059, 1024, 1024, 1031, 1003, 964, 1004, 21, 0, 0, 0, 0, 0, 0, 0, 1067, 1002, 1030, 1003, 1035, 995, 1003, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1066, 1108, 1075, 952, 1019, 1022, 1012, 12, 1, 0, 0, 0, 0, 0, 0, 1025, 1020, 1020, 1024, 1046, 986, 997, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 991, 985, 1036, 1011, 1019, 978, 1043, 1066, 55, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8156, 8247, 8109, 8151, 8283, 8154, 8187] avg=8192.0000 max=8283 max_rank=5 min=8109 min_rank=3 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1014, 987, 1021, 970, 1034, 1043, 1033, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 1070, 1048, 964, 1068, 1064, 1054, 981, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1003, 1051, 989, 1011, 1038, 1063, 1079, 7, 4, 0, 0, 0, 0, 0, 0, 1065, 1039, 978, 1005, 1048, 1016, 995, 1058, 26, 0, 0, 0, 0, 0, 0, 0, 978, 1032, 1046, 1000, 1033, 1057, 1046, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1003, 1005, 1015, 1064, 1001, 1029, 977, 17, 3, 0, 0, 0, 0, 0, 0, 986, 1006, 1023, 1035, 1048, 1035, 1001, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1016, 1038, 977, 1001, 962, 1011, 950, 56, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8342, 8229, 8230, 8243, 8126, 8151, 8077] avg=8192.0000 max=8342 max_rank=1 min=8077 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8064, 8448, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8064, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8320, 8192, 8064, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8192, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8192, 8320, 8064, 8320, 8192, 8320, 8192, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1059, 981, 1068, 1037, 1024, 1043, 1022, 967, 988, 983, 1044, 1021, 1000, 1007, 1049, 1051, 988, 1010, 1020, 1067, 1075, 1049, 945, 967, 1052, 1040, 1069, 1043, 1015, 1018, 1068, 988, 1046, 977, 999, 994, 1037, 986, 1051, 1019, 1074, 1021, 1040, 1039, 977, 1031, 999, 1018, 995, 1007, 1033, 1035, 1050, 985, 1066, 1028, 1015, 1038, 998, 1019, 1049, 1063, 1032, 1057] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[981, 1027, 969, 1037, 1048, 1022, 1018, 1029, 999, 1061, 1039, 999, 1046, 1023, 995, 1048, 995, 1048, 984, 1006, 1030, 1022, 1014, 1014, 1014, 1030, 1030, 1015, 1028, 998, 973, 1010, 1014, 1015, 1059, 1053, 1025, 1055, 1022, 1036, 1008, 1025, 1027, 1017, 1010, 957, 1018, 1054, 1074, 995, 1055, 1019, 1012, 990, 1065, 1042, 1068, 991, 1046, 1056, 1060, 1033, 1016, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[999, 1053, 1002, 1028, 1085, 1005, 1041, 1040, 1071, 1028, 1024, 1004, 975, 1024, 1043, 1040, 1021, 1008, 1041, 988, 1041, 1036, 1016, 1026, 1011, 1021, 1034, 1001, 1015, 1027, 1058, 1033, 994, 1047, 1044, 1010, 1037, 988, 1000, 1046, 1015, 975, 1035, 1042, 1002, 1009, 1058, 1080, 1024, 1055, 985, 1021, 1050, 1013, 1000, 1035, 1060, 981, 984, 1020, 1029, 1047, 998, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[995, 993, 997, 1041, 1026, 1060, 1056, 1034, 1021, 1023, 1058, 989, 977, 1002, 1024, 1018, 1066, 1060, 1030, 997, 1023, 1009, 1021, 994, 1012, 1035, 1024, 1054, 985, 1011, 1075, 997, 999, 1030, 1020, 1061, 1041, 1063, 1036, 973, 1022, 1065, 1057, 944, 1041, 1045, 974, 1038, 1045, 1049, 1030, 987, 1046, 999, 1010, 1005, 1022, 1062, 1041, 1043, 1030, 1002, 1005, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1010, 1005, 1046, 1033, 1018, 999, 1044, 1029, 1060, 1033, 995, 987, 995, 1014, 1015, 1025, 1039, 1039, 1056, 1081, 1038, 1068, 1015, 998, 998, 1053, 991, 1007, 1026, 1058, 1016, 1037, 1028, 1060, 980, 1026, 1064, 974, 981, 1034, 995, 1010, 1015, 1014, 1013, 1038, 1024, 1039, 1021, 1002, 1026, 970, 1020, 1081, 996, 984, 1056, 1051, 1024, 1018, 1064, 1044, 993, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1023, 995, 1036, 1000, 1062, 1019, 1038, 1023, 1079, 1019, 976, 1020, 1014, 1033, 998, 1027, 1036, 1018, 1055, 1015, 1045, 1051, 1054, 990, 1021, 988, 1039, 1015, 1089, 1015, 1021, 1035, 984, 1043, 1050, 1008, 1030, 1046, 987, 1063, 994, 1055, 978, 1004, 990, 994, 1012, 984, 1032, 987, 1027, 1017, 1010, 1041, 1023, 1015, 1071, 1026, 1025, 1023, 1035, 1031, 1035, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1034, 1068, 1024, 1040, 1007, 1038, 987, 1037, 1026, 1038, 1055, 1006, 1028, 1051, 1026, 1011, 1088, 1015, 1049, 988, 1037, 997, 1022, 1030, 1036, 988, 1003, 1005, 1041, 998, 1005, 1046, 1033, 1049, 989, 986, 1062, 1009, 960, 987, 1029, 1012, 1027, 990, 1059, 1081, 1044, 1003, 1042, 1026, 1024, 1064, 1063, 993, 1011, 988, 1019, 1009, 1038, 1008, 1005, 1032, 1074, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1044, 1033, 1011, 1012, 1033, 1022, 1007, 1041, 1037, 1046, 1015, 1006, 1061, 1059, 966, 1045, 1071, 997, 1002, 1014, 1060, 1032, 1017, 1064, 993, 1043, 1047, 983, 1018, 1081, 1031, 1013, 1048, 997, 1015, 1009, 1047, 1010, 971, 1022, 1031, 1015, 1018, 1038, 1004, 1029, 1019, 1000, 1066, 1045, 1037, 1003, 1061, 1043, 1006, 1001, 1047, 1017, 1011, 1050, 996, 992, 982, 1002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1059, 981, 1068, 1037, 1017, 1043, 1022, 967, 0, 0, 0, 0, 0, 0, 0, 0, 988, 983, 1044, 1021, 1000, 1007, 1049, 1051, 6, 5, 0, 0, 0, 0, 0, 0, 988, 1010, 1020, 1067, 1056, 1049, 945, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1040, 1069, 1043, 1015, 1018, 1068, 988, 1, 0, 0, 0, 0, 0, 0, 0, 1046, 977, 999, 994, 1037, 986, 1051, 1019, 14, 11, 0, 0, 0, 0, 0, 0, 1074, 1021, 1040, 1039, 977, 1031, 999, 1018, 31, 0, 0, 0, 0, 0, 0, 0, 995, 1007, 1033, 1035, 1050, 985, 1066, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1038, 998, 1019, 1049, 1063, 1032, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8154, 8102, 8294, 8134, 8230, 8199, 8229] avg=8192.0000 max=8294 max_rank=3 min=8102 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[981, 1027, 969, 1037, 1041, 1022, 1018, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1061, 1039, 999, 1046, 1023, 995, 1048, 6, 5, 0, 0, 0, 0, 0, 0, 995, 1048, 984, 1006, 1012, 1022, 1014, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1030, 1030, 1015, 1028, 998, 973, 1010, 1, 0, 0, 0, 0, 0, 0, 0, 1014, 1015, 1059, 1053, 1025, 1055, 1022, 1036, 13, 12, 0, 0, 0, 0, 0, 0, 1008, 1025, 1027, 1017, 1010, 957, 1018, 1054, 33, 0, 0, 0, 0, 0, 0, 0, 1074, 995, 1055, 1019, 1012, 990, 1065, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 991, 1046, 1056, 1060, 1033, 1016, 1067, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8221, 8095, 8099, 8304, 8149, 8252, 8292] avg=8192.0000 max=8304 max_rank=4 min=8095 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32997 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[995, 993, 997, 1041, 1019, 1060, 1056, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1023, 1058, 989, 977, 1002, 1024, 1018, 6, 5, 0, 0, 0, 0, 0, 0, 1066, 1060, 1030, 997, 1005, 1009, 1021, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1035, 1024, 1054, 985, 1011, 1075, 997, 1, 0, 0, 0, 0, 0, 0, 0, 999, 1030, 1020, 1061, 1041, 1063, 1036, 973, 13, 11, 0, 0, 0, 0, 0, 0, 1022, 1065, 1057, 944, 1041, 1045, 974, 1038, 31, 0, 0, 0, 0, 0, 0, 0, 1045, 1049, 1030, 987, 1046, 999, 1010, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1062, 1041, 1043, 1030, 1002, 1005, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8123, 8182, 8194, 8247, 8217, 8171, 8207] avg=8192.0000 max=8247 max_rank=4 min=8123 min_rank=1 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1034, 1068, 1024, 1040, 1000, 1038, 987, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1038, 1055, 1006, 1028, 1051, 1026, 1011, 6, 5, 0, 0, 0, 0, 0, 0, 1088, 1015, 1049, 988, 1019, 997, 1022, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 988, 1003, 1005, 1041, 998, 1005, 1046, 1, 0, 0, 0, 0, 0, 0, 0, 1033, 1049, 989, 986, 1062, 1009, 960, 987, 13, 11, 0, 0, 0, 0, 0, 0, 1029, 1012, 1027, 990, 1059, 1081, 1044, 1003, 31, 0, 0, 0, 0, 0, 0, 0, 1042, 1026, 1024, 1064, 1063, 993, 1011, 988, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1009, 1038, 1008, 1005, 1032, 1074, 996, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8252, 8208, 8123, 8099, 8276, 8211, 8139] avg=8192.0000 max=8276 max_rank=5 min=8099 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1023, 995, 1036, 1000, 1055, 1019, 1038, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 1019, 976, 1020, 1014, 1033, 998, 1027, 6, 5, 0, 0, 0, 0, 0, 0, 1036, 1018, 1055, 1015, 1027, 1051, 1054, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 988, 1039, 1015, 1089, 1015, 1021, 1035, 1, 0, 0, 0, 0, 0, 0, 0, 984, 1043, 1050, 1008, 1030, 1046, 987, 1063, 13, 12, 0, 0, 0, 0, 0, 0, 994, 1055, 978, 1004, 990, 994, 1012, 984, 33, 0, 0, 0, 0, 0, 0, 0, 1032, 987, 1027, 1017, 1010, 1041, 1023, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1026, 1025, 1023, 1035, 1031, 1035, 1067, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8177, 8246, 8224, 8236, 8044, 8152, 8268] avg=8192.0000 max=8268 max_rank=7 min=8044 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[999, 1053, 1002, 1028, 1078, 1005, 1041, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1028, 1024, 1004, 975, 1024, 1043, 1040, 6, 5, 0, 0, 0, 0, 0, 0, 1021, 1008, 1041, 988, 1023, 1036, 1016, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1021, 1034, 1001, 1015, 1027, 1058, 1033, 1, 0, 0, 0, 0, 0, 0, 0, 994, 1047, 1044, 1010, 1037, 988, 1000, 1046, 13, 12, 0, 0, 0, 0, 0, 0, 1015, 975, 1035, 1042, 1002, 1009, 1058, 1080, 33, 0, 0, 0, 0, 0, 0, 0, 1024, 1055, 985, 1021, 1050, 1013, 1000, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 981, 984, 1020, 1029, 1047, 998, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8220, 8159, 8201, 8191, 8249, 8183, 8087] avg=8192.0000 max=8249 max_rank=5 min=8087 min_rank=7 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1010, 1005, 1046, 1033, 998, 999, 1044, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1033, 995, 987, 995, 1014, 1015, 1025, 9, 10, 0, 0, 0, 0, 0, 0, 1039, 1039, 1056, 1081, 1010, 1068, 1015, 998, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1053, 991, 1007, 1026, 1058, 1016, 1037, 5, 0, 0, 0, 0, 0, 0, 0, 1028, 1060, 980, 1026, 1064, 974, 981, 1034, 18, 15, 0, 0, 0, 0, 0, 0, 995, 1010, 1015, 1014, 1013, 1038, 1024, 1039, 37, 0, 0, 0, 0, 0, 0, 0, 1021, 1002, 1026, 970, 1020, 1081, 996, 984, 6, 0, 0, 0, 0, 0, 0, 0, 1004, 1051, 1024, 1018, 1064, 1044, 993, 1063, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8143, 8306, 8191, 8180, 8185, 8106, 8261] avg=8192.0000 max=8306 max_rank=2 min=8106 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1044, 1033, 1011, 1012, 1026, 1022, 1007, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1046, 1015, 1006, 1061, 1059, 966, 1045, 6, 5, 0, 0, 0, 0, 0, 0, 1071, 997, 1002, 1014, 1041, 1032, 1017, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1043, 1047, 983, 1018, 1081, 1031, 1013, 1, 0, 0, 0, 0, 0, 0, 0, 1048, 997, 1015, 1009, 1047, 1010, 971, 1022, 14, 12, 0, 0, 0, 0, 0, 0, 1031, 1015, 1018, 1038, 1004, 1029, 1019, 1000, 32, 0, 0, 0, 0, 0, 0, 0, 1066, 1045, 1037, 1003, 1061, 1043, 1006, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1017, 1011, 1050, 996, 992, 982, 1002, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8246, 8238, 8210, 8145, 8186, 8262, 8053] avg=8192.0000 max=8262 max_rank=6 min=8053 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32646 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8448, 8192, 8064, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8320, 8064, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8320, 8320, 8192, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1006, 1030, 1063, 993, 994, 1016, 1028, 1037, 1046, 979, 1011, 1013, 983, 992, 1033, 1068, 1028, 1005, 1041, 1046, 1067, 1053, 1037, 1047, 1021, 1014, 1067, 1014, 1011, 1042, 1043, 1047, 1040, 999, 1027, 1036, 1000, 1058, 992, 999, 1045, 1002, 991, 1058, 1069, 1054, 958, 1015, 1060, 977, 1070, 1062, 1008, 964, 992, 1056, 976, 1034, 990, 1004, 994, 982, 1056, 1093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1044, 1022, 1059, 995, 963, 1076, 1025, 1009, 996, 997, 1059, 1037, 1018, 1071, 1033, 1004, 992, 1003, 1075, 1013, 981, 1004, 1027, 995, 1060, 1053, 1054, 1033, 1008, 1012, 1017, 989, 1040, 999, 1036, 1000, 1026, 1056, 1028, 1035, 1019, 996, 998, 1013, 1047, 1033, 1043, 1063, 1010, 1087, 1013, 1057, 1047, 983, 1046, 999, 965, 1040, 1029, 1047, 1004, 1024, 1000, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1019, 1032, 1031, 972, 1030, 976, 1069, 1029, 1056, 1043, 1034, 1013, 1030, 1054, 1037, 976, 1005, 1026, 1053, 1054, 1037, 1029, 1088, 1020, 1056, 996, 1011, 1048, 997, 1008, 1024, 999, 1037, 999, 1005, 966, 1033, 988, 1030, 1038, 1039, 970, 1001, 985, 1060, 1038, 1047, 1004, 1024, 1002, 1043, 977, 1029, 1066, 1062, 1048, 1024, 1044, 1003, 1026, 1047, 1019, 1021, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1031, 974, 1061, 996, 987, 1085, 1022, 996, 1019, 989, 1052, 1032, 1039, 1052, 980, 1016, 1069, 1026, 1044, 1006, 1064, 1006, 1007, 1061, 1036, 1044, 983, 1036, 1061, 1035, 1030, 1026, 1027, 1035, 1057, 1043, 1030, 1072, 1022, 1025, 1061, 989, 1089, 964, 985, 1013, 1039, 978, 974, 988, 986, 1039, 1028, 1047, 1026, 973, 998, 1041, 1019, 1024, 1038, 1002, 1034, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1090, 1017, 1025, 1028, 1051, 1031, 1068, 1038, 1007, 998, 988, 1029, 1006, 1065, 1021, 1042, 992, 1003, 1002, 1028, 1022, 1094, 990, 1032, 1043, 989, 1051, 1041, 1003, 1002, 1012, 1053, 1064, 1000, 1058, 964, 1027, 1043, 998, 974, 1001, 993, 1057, 1043, 990, 1035, 1006, 1010, 1051, 999, 1017, 1044, 984, 1054, 1005, 995, 1028, 1042, 1041, 1038, 1070, 1049, 1001, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1042, 1016, 1066, 1069, 1023, 997, 1032, 1013, 1049, 1040, 946, 1007, 993, 1034, 971, 1033, 1048, 1030, 1003, 1023, 980, 1016, 1041, 1073, 1044, 1018, 1039, 1037, 991, 1022, 1054, 1010, 985, 1021, 1000, 1069, 1009, 1044, 983, 1007, 1053, 1072, 1069, 1042, 1049, 1074, 989, 1000, 1029, 1029, 1013, 994, 990, 1031, 1017, 1019, 1033, 1015, 1040, 997, 1028, 1000, 1009, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1007, 1045, 1032, 1041, 1029, 1005, 1019, 1026, 1081, 1042, 1013, 998, 997, 1071, 1055, 982, 1039, 1002, 1000, 1072, 1027, 983, 977, 1028, 989, 999, 1050, 1046, 962, 1028, 1050, 1085, 1022, 1068, 1027, 983, 1027, 1060, 1057, 972, 1015, 989, 1036, 1004, 979, 1034, 1057, 1024, 1049, 1035, 995, 1018, 1059, 1015, 1018, 1040, 1034, 1041, 1035, 1062, 975, 1007, 1009, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1056, 1052, 1081, 1041, 1023, 1033, 1032, 1034, 1023, 984, 1054, 1019, 1052, 1053, 1001, 1000, 1008, 1028, 1030, 988, 986, 1067, 1026, 1001, 1062, 1010, 979, 965, 1033, 1063, 1005, 971, 1012, 1019, 1039, 1042, 1048, 1019, 1037, 1004, 983, 1004, 1037, 1024, 1060, 996, 1020, 1030, 1010, 1021, 1062, 1016, 1009, 988, 1005, 1039, 1071, 988, 1014, 1030, 1076, 1063, 990, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1006, 1030, 1026, 993, 994, 1016, 1028, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 979, 1011, 1013, 983, 992, 1033, 1068, 15, 0, 0, 0, 0, 0, 0, 0, 1028, 1005, 1041, 1046, 1067, 1053, 1037, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1014, 1067, 1014, 1011, 1042, 1043, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 999, 1027, 1036, 1000, 1058, 992, 999, 6, 3, 0, 0, 0, 0, 0, 0, 1045, 1002, 991, 1058, 1069, 1054, 958, 1015, 7, 4, 0, 0, 0, 0, 0, 0, 1060, 977, 1070, 1062, 1008, 964, 992, 1056, 18, 0, 0, 0, 0, 0, 0, 0, 976, 1034, 990, 1004, 994, 982, 1056, 1093, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8140, 8311, 8254, 8160, 8203, 8207, 8131] avg=8192.0000 max=8311 max_rank=2 min=8130 min_rank=0 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32701 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1044, 1022, 1023, 995, 963, 1076, 1025, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 996, 997, 1059, 1037, 1018, 1071, 1033, 1004, 15, 0, 0, 0, 0, 0, 0, 0, 992, 1003, 1075, 1013, 981, 1004, 1027, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1053, 1054, 1033, 1008, 1012, 1017, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 999, 1036, 1000, 1026, 1056, 1028, 1035, 6, 3, 0, 0, 0, 0, 0, 0, 1019, 996, 998, 1013, 1047, 1033, 1043, 1063, 7, 4, 0, 0, 0, 0, 0, 0, 1010, 1087, 1013, 1057, 1047, 983, 1046, 999, 17, 0, 0, 0, 0, 0, 0, 0, 965, 1040, 1029, 1047, 1004, 1024, 1000, 1029, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8230, 8077, 8221, 8229, 8223, 8259, 8140] avg=8192.0000 max=8259 max_rank=6 min=8077 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1031, 974, 1024, 996, 987, 1085, 1022, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 989, 1052, 1032, 1039, 1052, 980, 1016, 15, 0, 0, 0, 0, 0, 0, 0, 1069, 1026, 1044, 1006, 1064, 1006, 1007, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1044, 983, 1036, 1061, 1035, 1030, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1035, 1057, 1043, 1030, 1072, 1022, 1025, 6, 3, 0, 0, 0, 0, 0, 0, 1061, 989, 1089, 964, 985, 1013, 1039, 978, 7, 4, 0, 0, 0, 0, 0, 0, 974, 988, 986, 1039, 1028, 1047, 1026, 973, 18, 0, 0, 0, 0, 0, 0, 0, 998, 1041, 1019, 1024, 1038, 1002, 1034, 1025, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8115, 8194, 8270, 8246, 8320, 8129, 8079, 8183] avg=8192.0000 max=8320 max_rank=4 min=8079 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32711 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1019, 1032, 996, 972, 1030, 976, 1069, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1043, 1034, 1013, 1030, 1054, 1037, 976, 14, 0, 0, 0, 0, 0, 0, 0, 1005, 1026, 1053, 1054, 1037, 1029, 1088, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 996, 1011, 1048, 997, 1008, 1024, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 999, 1005, 966, 1033, 988, 1030, 1038, 6, 3, 0, 0, 0, 0, 0, 0, 1039, 970, 1001, 985, 1060, 1038, 1047, 1004, 7, 4, 0, 0, 0, 0, 0, 0, 1024, 1002, 1043, 977, 1029, 1066, 1062, 1048, 17, 0, 0, 0, 0, 0, 0, 0, 1024, 1044, 1003, 1026, 1047, 1019, 1021, 1009, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8257, 8299, 8134, 8105, 8155, 8268, 8195] avg=8192.0000 max=8299 max_rank=2 min=8105 min_rank=4 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1042, 1016, 1029, 1069, 1023, 997, 1032, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1040, 946, 1007, 993, 1034, 971, 1033, 15, 0, 0, 0, 0, 0, 0, 0, 1048, 1030, 1003, 1023, 980, 1016, 1041, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1018, 1039, 1037, 991, 1022, 1054, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1021, 1000, 1069, 1009, 1044, 983, 1007, 6, 3, 0, 0, 0, 0, 0, 0, 1053, 1072, 1069, 1042, 1049, 1074, 989, 1000, 8, 4, 0, 0, 0, 0, 0, 0, 1029, 1029, 1013, 994, 990, 1031, 1017, 1019, 18, 0, 0, 0, 0, 0, 0, 0, 1033, 1015, 1040, 997, 1028, 1000, 1009, 1066, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8088, 8200, 8210, 8127, 8360, 8140, 8190] avg=8192.0000 max=8360 max_rank=5 min=8088 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32817 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1090, 1017, 990, 1028, 1051, 1031, 1068, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 998, 988, 1029, 1006, 1065, 1021, 1042, 14, 0, 0, 0, 0, 0, 0, 0, 992, 1003, 1002, 1028, 1022, 1094, 990, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 989, 1051, 1041, 1003, 1002, 1012, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1000, 1058, 964, 1027, 1043, 998, 974, 6, 3, 0, 0, 0, 0, 0, 0, 1001, 993, 1057, 1043, 990, 1035, 1006, 1010, 7, 4, 0, 0, 0, 0, 0, 0, 1051, 999, 1017, 1044, 984, 1054, 1005, 995, 17, 0, 0, 0, 0, 0, 0, 0, 1028, 1042, 1041, 1038, 1070, 1049, 1001, 994, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8313, 8170, 8150, 8189, 8137, 8146, 8166, 8265] avg=8192.0000 max=8313 max_rank=0 min=8137 min_rank=4 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1007, 1045, 997, 1041, 1029, 1005, 1019, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1042, 1013, 998, 997, 1071, 1055, 982, 14, 0, 0, 0, 0, 0, 0, 0, 1039, 1002, 1000, 1072, 1027, 983, 977, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 984, 999, 1050, 1046, 962, 1028, 1050, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1068, 1027, 983, 1027, 1060, 1057, 972, 6, 3, 0, 0, 0, 0, 0, 0, 1015, 989, 1036, 1004, 979, 1034, 1057, 1024, 7, 4, 0, 0, 0, 0, 0, 0, 1049, 1035, 995, 1018, 1059, 1015, 1018, 1040, 17, 0, 0, 0, 0, 0, 0, 0, 1034, 1041, 1035, 1062, 975, 1007, 1009, 1010, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8253, 8115, 8204, 8225, 8149, 8246, 8175] avg=8192.0000 max=8253 max_rank=1 min=8115 min_rank=2 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1056, 1052, 1037, 1041, 1023, 1033, 1032, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 984, 1054, 1019, 1052, 1053, 1001, 1000, 18, 0, 0, 0, 0, 0, 0, 0, 1008, 1028, 1030, 988, 986, 1067, 1026, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1010, 979, 965, 1033, 1063, 1005, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1019, 1039, 1042, 1048, 1019, 1037, 1004, 8, 5, 0, 0, 0, 0, 0, 0, 983, 1004, 1037, 1024, 1060, 996, 1020, 1030, 12, 5, 0, 0, 0, 0, 0, 0, 1010, 1021, 1062, 1016, 1009, 988, 1005, 1039, 21, 0, 0, 0, 0, 0, 0, 0, 1071, 988, 1014, 1030, 1076, 1063, 990, 1020, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8308, 8204, 8114, 8078, 8233, 8171, 8171, 8257] avg=8192.0000 max=8308 max_rank=0 min=8078 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32832 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8192, 8320, 8320, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8448, 8192, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8448, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1016, 1046, 1033, 1048, 1027, 1026, 1020, 995, 1109, 986, 1048, 1049, 1016, 970, 979, 1003, 1000, 1067, 1007, 1033, 1032, 1042, 1028, 1007, 1004, 1044, 1080, 994, 996, 1000, 1077, 1008, 1021, 1085, 981, 1017, 977, 978, 1020, 991, 1083, 1016, 991, 1052, 1095, 1052, 1017, 996, 1033, 1032, 1016, 1064, 978, 1020, 1028, 1007, 1015, 1015, 1043, 1017, 1015, 1029, 1027, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[979, 1042, 978, 1033, 1016, 1019, 991, 1036, 1023, 1036, 982, 1020, 1095, 971, 1052, 963, 1022, 1053, 1007, 1066, 999, 1014, 1068, 1066, 983, 1015, 986, 991, 1060, 1008, 998, 1096, 1024, 999, 1020, 1034, 1044, 1047, 1047, 967, 1027, 1005, 1013, 987, 1040, 1031, 1060, 960, 1022, 1047, 1046, 1023, 1046, 999, 1107, 967, 1092, 1038, 1008, 1020, 1056, 1036, 1047, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1078, 1092, 1048, 1059, 1032, 1008, 1014, 1016, 1022, 1034, 1039, 1011, 1044, 996, 1042, 1034, 1017, 972, 1051, 1014, 1023, 1016, 1042, 1033, 957, 983, 1075, 988, 999, 1077, 969, 1036, 1040, 966, 1016, 996, 1037, 992, 1024, 1041, 1048, 1018, 1068, 1010, 1081, 1086, 1011, 980, 1027, 1033, 1042, 1000, 1052, 1016, 1039, 980, 1049, 1006, 1024, 1010, 986, 1018, 1004, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1010, 1003, 1017, 991, 1026, 1097, 1050, 984, 1005, 1009, 1033, 981, 1020, 1009, 1053, 1038, 1010, 1037, 1030, 1030, 1019, 1055, 1026, 997, 1000, 1068, 1023, 1080, 1001, 1052, 1022, 1049, 998, 1078, 1102, 988, 1003, 1053, 1066, 1052, 1016, 1027, 1044, 1041, 1019, 1018, 1014, 1023, 991, 1046, 962, 1027, 1017, 1052, 1009, 994, 989, 1017, 986, 966, 1048, 1011, 1041, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1006, 1057, 1049, 1073, 1032, 1012, 1035, 1017, 981, 1077, 1008, 1042, 1009, 1021, 1011, 1008, 1018, 1060, 1041, 1082, 1013, 1027, 991, 1029, 1042, 1012, 1006, 996, 1024, 1002, 1037, 1051, 1039, 998, 1012, 1026, 1050, 1011, 1034, 1013, 1002, 1065, 1008, 1096, 1014, 1034, 997, 1032, 1017, 993, 1023, 1017, 968, 1037, 1040, 998, 1005, 975, 1074, 1023, 1035, 1030, 989, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1057, 1045, 1059, 1017, 1013, 1019, 1015, 1001, 1011, 1023, 1017, 1015, 1019, 975, 1007, 1059, 1008, 1043, 1050, 1101, 1051, 992, 1030, 1079, 997, 1014, 1036, 1050, 1025, 1038, 1027, 1050, 1017, 1029, 1019, 1021, 1044, 979, 1018, 996, 1047, 962, 993, 1005, 1027, 1036, 1058, 1017, 1037, 1043, 1005, 948, 1011, 973, 1034, 993, 1060, 1025, 993, 1072, 1026, 1034, 1071, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[995, 1033, 1050, 1040, 1068, 1019, 1063, 997, 1009, 1038, 1033, 1061, 1066, 1043, 999, 1019, 1075, 1032, 1033, 1087, 1037, 1032, 996, 1021, 1005, 1041, 957, 972, 1032, 1030, 1028, 1038, 1021, 1015, 1054, 1025, 1035, 1052, 1022, 1048, 1032, 1006, 959, 1040, 1018, 1002, 1004, 1023, 1060, 977, 993, 1009, 956, 1065, 1023, 1003, 1021, 1037, 1022, 1036, 990, 1034, 978, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[985, 997, 1032, 1049, 1040, 987, 1039, 1029, 1070, 1033, 1021, 1047, 1020, 1020, 1000, 961, 999, 1000, 1035, 1055, 997, 1061, 1009, 983, 1050, 1042, 990, 1031, 1035, 1016, 1034, 1029, 1055, 1005, 1121, 1095, 972, 999, 1064, 992, 974, 1025, 1030, 1036, 1055, 1009, 1056, 1027, 1064, 1008, 1053, 998, 1043, 980, 1021, 1000, 1019, 982, 1049, 1042, 1016, 970, 1033, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1078, 1064, 1048, 1059, 1032, 1008, 1014, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1034, 1039, 1011, 1044, 996, 1042, 1034, 18, 0, 0, 0, 0, 0, 0, 0, 1017, 972, 1051, 965, 1023, 1016, 1042, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 957, 983, 1075, 988, 999, 1077, 969, 1036, 5, 4, 0, 0, 0, 0, 0, 0, 1040, 966, 1011, 996, 1037, 992, 1024, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1018, 1068, 1010, 1071, 1086, 1011, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1033, 1042, 1000, 1052, 1016, 1039, 980, 49, 1, 0, 0, 0, 0, 0, 0, 1049, 1006, 1024, 1010, 986, 1018, 1004, 1015, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8319, 8240, 8119, 8093, 8107, 8292, 8239, 8127] avg=8192.0000 max=8319 max_rank=0 min=8093 min_rank=3 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1006, 1030, 1049, 1073, 1032, 1012, 1035, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1077, 1008, 1042, 1009, 1021, 1011, 1008, 17, 0, 0, 0, 0, 0, 0, 0, 1018, 1060, 1041, 1030, 1013, 1027, 991, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1012, 1006, 996, 1024, 1002, 1037, 1051, 5, 3, 0, 0, 0, 0, 0, 0, 1039, 998, 1007, 1026, 1050, 1011, 1034, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1065, 1008, 1096, 1005, 1034, 997, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 993, 1023, 1017, 968, 1037, 1040, 998, 52, 1, 0, 0, 0, 0, 0, 0, 1005, 975, 1074, 1023, 1035, 1030, 989, 1012, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8174, 8209, 8178, 8178, 8239, 8146, 8158] avg=8192.0000 max=8254 max_rank=0 min=8146 min_rank=6 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1010, 977, 1017, 991, 1026, 1097, 1050, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1009, 1033, 981, 1020, 1009, 1053, 1038, 17, 0, 0, 0, 0, 0, 0, 0, 1010, 1037, 1030, 980, 1019, 1055, 1026, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1068, 1023, 1080, 1001, 1052, 1022, 1049, 6, 3, 0, 0, 0, 0, 0, 0, 998, 1078, 1096, 988, 1003, 1053, 1066, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1027, 1044, 1041, 1010, 1018, 1014, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1046, 962, 1027, 1017, 1052, 1009, 994, 50, 1, 0, 0, 0, 0, 0, 0, 989, 1017, 986, 966, 1048, 1011, 1041, 1013, 8, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8165, 8154, 8304, 8334, 8193, 8149, 8085] avg=8192.0000 max=8334 max_rank=4 min=8085 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[979, 1015, 978, 1033, 1016, 1019, 991, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1036, 982, 1020, 1095, 971, 1052, 963, 17, 0, 0, 0, 0, 0, 0, 0, 1022, 1053, 1007, 1014, 999, 1014, 1068, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1015, 986, 991, 1060, 1008, 998, 1096, 6, 3, 0, 0, 0, 0, 0, 0, 1024, 999, 1014, 1034, 1044, 1047, 1047, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1005, 1013, 987, 1031, 1031, 1060, 960, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1047, 1046, 1023, 1046, 999, 1107, 967, 52, 1, 0, 0, 0, 0, 0, 0, 1092, 1038, 1008, 1020, 1056, 1036, 1047, 1009, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8067, 8159, 8243, 8146, 8176, 8114, 8310, 8321] avg=8192.0000 max=8321 max_rank=7 min=8067 min_rank=0 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1016, 1019, 1033, 1048, 1027, 1026, 1020, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 986, 1048, 1049, 1016, 970, 979, 1003, 17, 0, 0, 0, 0, 0, 0, 0, 1000, 1067, 1007, 983, 1032, 1042, 1028, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1044, 1080, 994, 996, 1000, 1077, 1008, 5, 4, 0, 0, 0, 0, 0, 0, 1021, 1085, 976, 1017, 977, 978, 1020, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 1016, 991, 1052, 1085, 1052, 1017, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1032, 1016, 1064, 978, 1020, 1028, 1007, 50, 1, 0, 0, 0, 0, 0, 0, 1015, 1015, 1043, 1017, 1015, 1029, 1027, 1035, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8177, 8166, 8212, 8065, 8292, 8229, 8211] avg=8192.0000 max=8292 max_rank=5 min=8065 min_rank=4 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1057, 1018, 1059, 1017, 1013, 1019, 1015, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1023, 1017, 1015, 1019, 975, 1007, 1059, 17, 0, 0, 0, 0, 0, 0, 0, 1008, 1043, 1050, 1048, 1051, 992, 1030, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1014, 1036, 1050, 1025, 1038, 1027, 1050, 5, 3, 0, 0, 0, 0, 0, 0, 1017, 1029, 1014, 1021, 1044, 979, 1018, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 962, 993, 1005, 1018, 1036, 1058, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1043, 1005, 948, 1011, 973, 1034, 993, 53, 1, 0, 0, 0, 0, 0, 0, 1060, 1025, 993, 1072, 1026, 1034, 1071, 1000, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8143, 8301, 8245, 8118, 8136, 8098, 8296] avg=8192.0000 max=8301 max_rank=2 min=8098 min_rank=6 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[995, 1006, 1050, 1040, 1068, 1019, 1063, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1038, 1033, 1061, 1066, 1043, 999, 1019, 17, 0, 0, 0, 0, 0, 0, 0, 1075, 1032, 1033, 1034, 1037, 1032, 996, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1041, 957, 972, 1032, 1030, 1028, 1038, 6, 3, 0, 0, 0, 0, 0, 0, 1021, 1015, 1048, 1025, 1035, 1052, 1022, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1006, 959, 1040, 1009, 1002, 1004, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 977, 993, 1009, 956, 1065, 1023, 1003, 53, 1, 0, 0, 0, 0, 0, 0, 1021, 1037, 1022, 1036, 990, 1034, 978, 1027, 9, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8285, 8260, 8112, 8266, 8075, 8140, 8160] avg=8192.0000 max=8285 max_rank=1 min=8075 min_rank=5 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[985, 962, 1032, 1049, 1040, 987, 1039, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1033, 1021, 1047, 1020, 1020, 1000, 961, 21, 0, 0, 0, 0, 0, 0, 0, 999, 1000, 1035, 1000, 997, 1061, 1009, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1042, 990, 1031, 1035, 1016, 1034, 1029, 11, 8, 0, 0, 0, 0, 0, 0, 1055, 1005, 1110, 1095, 972, 999, 1064, 992, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1025, 1030, 1036, 1038, 1009, 1056, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1008, 1053, 998, 1043, 980, 1021, 1000, 55, 3, 0, 0, 0, 0, 0, 0, 1019, 982, 1049, 1042, 1016, 970, 1033, 1047, 11, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8193, 8084, 8246, 8292, 8195, 8225, 8178] avg=8192.0000 max=8292 max_rank=4 min=8084 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8064, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8192, 8192, 8320, 8192, 8448, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8064, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8320, 8064, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1009, 1044, 1027, 965, 1061, 965, 1003, 1033, 1036, 1015, 1049, 1011, 1074, 1000, 1056, 1016, 1087, 1012, 1026, 1068, 1009, 1003, 1017, 1019, 1017, 988, 982, 1026, 1025, 1025, 1033, 1032, 1034, 1021, 1019, 1075, 1032, 1046, 1028, 1065, 1019, 1041, 1023, 996, 1008, 987, 1062, 1002, 1016, 1015, 971, 1008, 975, 1055, 1017, 988, 1058, 1016, 1047, 1060, 1015, 1021, 1025, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[999, 1030, 964, 1011, 1015, 987, 1085, 1023, 1053, 994, 1075, 1027, 1083, 1086, 1019, 1020, 1014, 1014, 1060, 1035, 993, 1044, 1009, 1054, 991, 1006, 1049, 1012, 1008, 1008, 997, 1052, 1011, 963, 1045, 1042, 969, 976, 1051, 1061, 1010, 1051, 1021, 1024, 1041, 1020, 970, 1007, 1043, 993, 1015, 1042, 1006, 1085, 1048, 975, 1031, 1036, 1059, 1068, 1057, 1032, 971, 996] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[997, 986, 1020, 1022, 1042, 1003, 1002, 1059, 997, 984, 993, 994, 1043, 1015, 1035, 1038, 992, 1030, 1034, 997, 1018, 1053, 1044, 1032, 1070, 1007, 1016, 1027, 992, 1027, 966, 1017, 1016, 1043, 1042, 1014, 1013, 1001, 1006, 1034, 1014, 992, 1046, 1055, 1028, 1007, 1064, 1018, 1030, 990, 1022, 1046, 1081, 1060, 1004, 1092, 1009, 1017, 1060, 1034, 1051, 1059, 1044, 992] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1050, 1046, 1009, 1032, 1038, 1040, 1051, 976, 999, 1078, 1007, 1077, 1005, 1033, 1018, 1057, 1062, 1036, 991, 997, 983, 1002, 1023, 1061, 1015, 1008, 1007, 1019, 1069, 1031, 1023, 1010, 1008, 1019, 1067, 1013, 1036, 1024, 961, 1002, 1023, 1011, 1002, 1057, 1026, 1015, 1049, 988, 1029, 1024, 1035, 1047, 1076, 999, 1020, 1055, 989, 1013, 1001, 994, 1047, 1024, 1019, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[952, 1037, 1022, 1047, 1070, 1007, 1006, 1027, 1051, 1039, 983, 1048, 1037, 1018, 1012, 1038, 1050, 1060, 1023, 1005, 993, 1061, 981, 973, 1077, 1044, 994, 1007, 1004, 1015, 973, 1058, 1044, 1001, 1010, 1046, 1032, 1045, 1012, 1032, 1047, 1024, 1053, 1028, 991, 981, 988, 1038, 991, 1064, 1023, 1063, 994, 1058, 1022, 986, 1059, 1064, 1001, 997, 963, 1011, 1038, 1118] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[953, 1004, 1028, 1016, 994, 1063, 986, 1041, 1002, 1015, 970, 1010, 1028, 1051, 1039, 990, 1030, 1019, 1032, 999, 1052, 1043, 1054, 1011, 1040, 1071, 1035, 1040, 1016, 1001, 1009, 1067, 1068, 1028, 1007, 1084, 995, 960, 1015, 1024, 1002, 1055, 1062, 1055, 1021, 1064, 1068, 1030, 1029, 1017, 984, 1037, 1041, 980, 1038, 962, 1038, 1033, 1024, 1038, 1006, 1009, 1010, 1043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1025, 1039, 1043, 1067, 1059, 1111, 1016, 1042, 1039, 983, 1037, 1009, 1052, 987, 976, 993, 981, 1010, 1011, 1003, 1051, 997, 1087, 994, 1040, 1032, 1019, 1023, 1043, 1000, 1009, 1015, 1025, 1000, 1055, 1046, 1024, 992, 1030, 1041, 985, 1003, 1058, 993, 1006, 1005, 987, 999, 1042, 1044, 1034, 1035, 1033, 990, 1057, 1122, 980, 944, 1025, 1058, 1022, 1064, 1046, 998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[998, 1039, 981, 1010, 997, 993, 1025, 1018, 1034, 1022, 1011, 999, 1015, 1042, 1083, 985, 1041, 1067, 1058, 1060, 1010, 1032, 1034, 1050, 1024, 997, 1040, 1023, 1019, 1015, 1035, 1019, 977, 1065, 1014, 1018, 1062, 1087, 992, 981, 1002, 1007, 1039, 1029, 1087, 1017, 1028, 1025, 993, 992, 994, 1032, 1045, 1004, 1024, 1023, 1024, 1051, 1036, 1033, 1040, 1029, 1001, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1009, 1044, 1027, 965, 1061, 965, 1003, 1033, 24, 4, 0, 0, 0, 0, 0, 0, 1036, 1015, 1049, 1011, 1068, 1000, 1056, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1012, 1026, 1068, 1009, 1003, 1017, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 988, 982, 1026, 1025, 1025, 1033, 1032, 15, 7, 0, 0, 0, 0, 0, 0, 1034, 1021, 1019, 1074, 1032, 1046, 1028, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1041, 1023, 996, 1008, 987, 1062, 1002, 6, 2, 1, 0, 0, 0, 0, 0, 1016, 1015, 971, 999, 975, 1055, 1017, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1016, 1047, 1036, 1015, 1021, 1025, 1058, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8251, 8222, 8150, 8319, 8147, 8036, 8276] avg=8192.0000 max=8319 max_rank=4 min=8036 min_rank=6 imbalance_ratio=1.015503 max_load_reduction=57217 max_load_reduction_pct=0.873062 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[999, 1030, 964, 1011, 1015, 987, 1085, 1023, 24, 4, 0, 0, 0, 0, 0, 0, 1053, 994, 1075, 1027, 1077, 1086, 1019, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1014, 1060, 1035, 993, 1044, 1009, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1006, 1049, 1012, 1008, 1008, 997, 1052, 14, 7, 0, 0, 0, 0, 0, 0, 1011, 963, 1045, 1041, 969, 976, 1051, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1051, 1021, 1024, 1041, 1020, 970, 1007, 6, 2, 1, 0, 0, 0, 0, 0, 1043, 993, 1015, 1033, 1006, 1085, 1048, 975, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1036, 1059, 1044, 1057, 1032, 971, 996, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8351, 8205, 8144, 8117, 8153, 8198, 8226] avg=8192.0000 max=8351 max_rank=1 min=8117 min_rank=4 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[997, 986, 1020, 1022, 1042, 1003, 1002, 1059, 23, 4, 0, 0, 0, 0, 0, 0, 997, 984, 993, 994, 1037, 1015, 1035, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1030, 1034, 997, 1018, 1053, 1044, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1007, 1016, 1027, 992, 1027, 966, 1017, 14, 7, 0, 0, 0, 0, 0, 0, 1016, 1043, 1042, 1013, 1013, 1001, 1006, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 992, 1046, 1055, 1028, 1007, 1064, 1018, 6, 2, 1, 0, 0, 0, 0, 0, 1030, 990, 1022, 1037, 1081, 1060, 1004, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1017, 1060, 1011, 1051, 1059, 1044, 992, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8093, 8182, 8143, 8168, 8233, 8316, 8243] avg=8192.0000 max=8316 max_rank=6 min=8093 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32960 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1050, 1046, 1009, 1032, 1038, 1040, 1051, 976, 22, 4, 0, 0, 0, 0, 0, 0, 999, 1078, 1007, 1077, 1000, 1033, 1018, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1036, 991, 997, 983, 1002, 1023, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1008, 1007, 1019, 1069, 1031, 1023, 1010, 15, 7, 0, 0, 0, 0, 0, 0, 1008, 1019, 1067, 1012, 1036, 1024, 961, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1011, 1002, 1057, 1026, 1015, 1049, 988, 5, 2, 1, 0, 0, 0, 0, 0, 1029, 1024, 1035, 1038, 1076, 999, 1020, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1013, 1001, 972, 1047, 1024, 1019, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8269, 8136, 8204, 8129, 8179, 8276, 8075] avg=8192.0000 max=8276 max_rank=6 min=8075 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[953, 1004, 1028, 1016, 994, 1063, 986, 1041, 23, 4, 0, 0, 0, 0, 0, 0, 1002, 1015, 970, 1010, 1022, 1051, 1039, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1019, 1032, 999, 1052, 1043, 1054, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1071, 1035, 1040, 1016, 1001, 1009, 1067, 14, 7, 0, 0, 0, 0, 0, 0, 1068, 1028, 1007, 1083, 995, 960, 1015, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1055, 1062, 1055, 1021, 1064, 1068, 1030, 6, 2, 1, 0, 0, 0, 0, 0, 1029, 1017, 984, 1028, 1041, 980, 1038, 962, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1033, 1024, 1015, 1006, 1009, 1010, 1043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8099, 8222, 8300, 8180, 8366, 8079, 8178] avg=8192.0000 max=8366 max_rank=5 min=8079 min_rank=6 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[952, 1037, 1022, 1047, 1070, 1007, 1006, 1027, 22, 4, 0, 0, 0, 0, 0, 0, 1051, 1039, 983, 1048, 1031, 1018, 1012, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1060, 1023, 1005, 993, 1061, 981, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 1044, 994, 1007, 1004, 1015, 973, 1058, 15, 7, 0, 0, 0, 0, 0, 0, 1044, 1001, 1010, 1045, 1032, 1045, 1012, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1024, 1053, 1028, 991, 981, 988, 1038, 6, 2, 1, 0, 0, 0, 0, 0, 991, 1064, 1023, 1054, 994, 1058, 1022, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1064, 1001, 975, 963, 1011, 1038, 1118, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8220, 8127, 8194, 8221, 8159, 8192, 8229] avg=8192.0000 max=8229 max_rank=7 min=8127 min_rank=2 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1025, 1039, 1043, 1067, 1059, 1111, 1016, 1042, 24, 4, 0, 0, 0, 0, 0, 0, 1039, 983, 1037, 1009, 1046, 987, 976, 993, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1010, 1011, 1003, 1051, 997, 1087, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1032, 1019, 1023, 1043, 1000, 1009, 1015, 14, 7, 0, 0, 0, 0, 0, 0, 1025, 1000, 1055, 1045, 1024, 992, 1030, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1003, 1058, 993, 1006, 1005, 987, 999, 6, 2, 1, 0, 0, 0, 0, 0, 1042, 1044, 1034, 1026, 1033, 990, 1057, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 980, 944, 1025, 1034, 1022, 1064, 1046, 998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8430, 8070, 8116, 8202, 8212, 8045, 8348, 8113] avg=8192.0000 max=8430 max_rank=0 min=8045 min_rank=5 imbalance_ratio=1.029053 max_load_reduction=57106 max_load_reduction_pct=0.871368 moved_to_echo=32718 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[998, 1039, 981, 1010, 997, 993, 1025, 1018, 27, 9, 0, 0, 0, 0, 0, 0, 1034, 1022, 1011, 999, 1007, 1042, 1083, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1067, 1058, 1060, 1010, 1032, 1034, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 997, 1040, 1023, 1019, 1015, 1035, 1019, 17, 10, 0, 0, 0, 0, 0, 0, 977, 1065, 1014, 1012, 1062, 1087, 992, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1007, 1039, 1029, 1087, 1017, 1028, 1025, 8, 6, 6, 0, 0, 0, 0, 0, 993, 992, 994, 1016, 1045, 1004, 1024, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1051, 1036, 1006, 1040, 1029, 1001, 1009, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8183, 8326, 8199, 8190, 8254, 8091, 8196] avg=8192.0000 max=8326 max_rank=2 min=8091 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 8320, 8192, 8192, 8320, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8064, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8448, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1010, 1009, 1014, 1053, 1006, 951, 987, 1059, 987, 1031, 1024, 1050, 1030, 1086, 984, 1050, 1018, 1074, 1046, 1080, 981, 1027, 1043, 1044, 994, 1034, 1020, 1042, 1033, 993, 1046, 1036, 1011, 983, 1068, 1041, 1010, 1047, 1014, 1042, 1068, 1052, 1020, 974, 976, 957, 991, 1058, 1024, 1006, 1023, 993, 1052, 1023, 1030, 1009, 1064, 1026, 1045, 1006, 1057, 1042, 975, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1007, 1010, 994, 1053, 1035, 1020, 1051, 982, 1022, 1001, 1037, 1001, 1070, 1051, 1023, 1021, 1055, 1020, 1049, 1002, 1004, 975, 1078, 1011, 1032, 1024, 970, 983, 1077, 1032, 1061, 1028, 990, 1034, 1023, 1023, 1047, 989, 1025, 983, 1047, 1051, 1062, 1019, 1056, 1008, 1025, 1014, 1042, 1015, 1002, 1054, 995, 1049, 993, 995, 1025, 1024, 1031, 996, 1039, 1041, 1043, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1011, 1017, 1020, 1063, 1038, 1038, 959, 1026, 1024, 1011, 1000, 1050, 993, 1016, 1008, 1007, 1069, 989, 1028, 996, 1006, 1016, 1063, 1018, 991, 999, 988, 1064, 1000, 1007, 1055, 1006, 1005, 983, 1004, 1057, 998, 999, 1072, 1075, 1048, 1074, 994, 1018, 1055, 1078, 1039, 989, 1046, 1051, 996, 1060, 1032, 1024, 1083, 998, 1043, 1047, 1011, 1009, 960, 1071, 1025, 1016] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1005, 1009, 995, 1004, 964, 1015, 1027, 997, 1014, 1031, 1010, 1069, 1045, 1025, 1002, 1009, 966, 1035, 1061, 991, 1026, 1060, 1021, 1022, 1024, 1045, 981, 997, 986, 1062, 1025, 1042, 1046, 1008, 1013, 1051, 1020, 980, 1045, 1059, 1028, 1065, 1046, 1035, 1035, 1070, 1007, 1017, 1020, 1050, 1078, 1031, 1025, 997, 996, 1046, 1022, 1017, 1026, 1014, 1065, 1007, 1062, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1014, 1020, 1053, 1014, 982, 991, 1034, 1029, 1035, 1014, 991, 1015, 1022, 1009, 1044, 1016, 1041, 1034, 1048, 1036, 1013, 1046, 1033, 1046, 1017, 1052, 1061, 1051, 1015, 1007, 1073, 1049, 1027, 1017, 1075, 1028, 996, 1002, 1047, 1042, 1020, 1014, 1040, 968, 1010, 1037, 982, 1023, 1025, 1073, 1031, 1009, 1028, 1013, 1040, 967, 1023, 998, 1051, 1005, 1020, 968, 1067, 985] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1056, 1030, 1007, 998, 1008, 973, 1066, 974, 1036, 1030, 1039, 1030, 1016, 1031, 1006, 1021, 1028, 1011, 1003, 1012, 951, 1046, 1104, 1059, 1026, 1046, 1071, 1059, 1046, 1005, 1041, 987, 1091, 1013, 970, 1056, 1041, 1082, 1023, 1027, 1023, 990, 1011, 985, 996, 1046, 1024, 1030, 1047, 1014, 1020, 1038, 1008, 1001, 1022, 1003, 998, 975, 1043, 1070, 1000, 1030, 1023, 1020] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1007, 1024, 1033, 990, 1027, 1062, 1027, 1028, 997, 1017, 1006, 1031, 1000, 1034, 974, 1022, 1060, 1006, 1032, 1016, 1039, 1039, 1043, 1029, 1016, 1014, 1036, 1028, 1080, 1021, 999, 1074, 1028, 1031, 1052, 1044, 1017, 963, 1046, 1028, 1073, 1031, 988, 1047, 1027, 997, 1021, 1048, 1017, 1033, 1042, 1036, 993, 1015, 1014, 996, 1025, 1005, 993, 1058, 1055, 1027, 991, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[983, 1028, 1053, 1023, 1050, 1012, 1002, 970, 1010, 960, 1064, 1035, 1028, 1069, 1013, 1049, 1068, 1082, 1007, 1033, 1027, 1087, 1052, 970, 1039, 964, 1030, 1071, 1021, 985, 1058, 1030, 1047, 1033, 1001, 1045, 1034, 963, 1010, 1060, 962, 1065, 1025, 1007, 1021, 1028, 999, 1045, 1023, 1016, 1086, 1007, 1012, 1008, 1026, 1047, 987, 1030, 1092, 1029, 998, 952, 1008, 997] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1010, 1009, 1014, 1053, 1006, 951, 987, 1059, 54, 12, 0, 0, 0, 0, 0, 0, 987, 1031, 1024, 1050, 1030, 1086, 984, 1050, 6, 2, 1, 0, 0, 0, 0, 0, 1018, 1074, 1046, 1080, 981, 1027, 989, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1034, 1020, 1042, 1033, 993, 1020, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 983, 1068, 1024, 1010, 1047, 1014, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1050, 1020, 974, 976, 957, 991, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1006, 1022, 993, 1052, 1023, 1030, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1026, 1045, 1006, 1057, 1042, 975, 1007, 14, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8251, 8259, 8172, 8199, 8094, 8159, 8247] avg=8192.0000 max=8259 max_rank=2 min=8094 min_rank=5 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32699 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1007, 1010, 994, 1053, 1035, 1020, 1051, 982, 56, 12, 0, 0, 0, 0, 0, 0, 1022, 1001, 1037, 1001, 1070, 1051, 1023, 1021, 6, 2, 1, 0, 0, 0, 0, 0, 1055, 1020, 1049, 1002, 1004, 975, 1022, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1024, 970, 983, 1077, 1032, 1035, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1034, 1023, 1006, 1047, 989, 1025, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1049, 1062, 1019, 1056, 1008, 1025, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1015, 1001, 1054, 995, 1049, 993, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1024, 1031, 996, 1039, 1041, 1043, 1017, 14, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8235, 8138, 8181, 8097, 8280, 8144, 8241] avg=8192.0000 max=8280 max_rank=5 min=8097 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1056, 1030, 1007, 998, 1008, 973, 1066, 974, 57, 12, 0, 0, 0, 0, 0, 0, 1036, 1030, 1039, 1030, 1016, 1031, 1006, 1021, 6, 2, 1, 0, 0, 0, 0, 0, 1028, 1011, 1003, 1012, 951, 1046, 1047, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1046, 1071, 1059, 1046, 1005, 1015, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 1013, 970, 1039, 1041, 1082, 1023, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 988, 1011, 985, 996, 1046, 1024, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1014, 1019, 1038, 1008, 1001, 1022, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 998, 975, 1043, 1070, 1000, 1030, 1023, 1020, 14, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8218, 8157, 8255, 8286, 8103, 8152, 8184] avg=8192.0000 max=8286 max_rank=4 min=8103 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1005, 1009, 995, 1004, 964, 1015, 1027, 997, 53, 12, 0, 0, 0, 0, 0, 0, 1014, 1031, 1010, 1069, 1045, 1025, 1002, 1009, 6, 2, 1, 0, 0, 0, 0, 0, 966, 1035, 1061, 991, 1026, 1060, 968, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1045, 981, 997, 986, 1062, 1000, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1008, 1013, 1034, 1020, 980, 1045, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1063, 1046, 1035, 1035, 1070, 1007, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1050, 1077, 1031, 1025, 997, 996, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1017, 1026, 1014, 1065, 1007, 1062, 990, 13, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8081, 8214, 8129, 8137, 8205, 8301, 8242, 8227] avg=8192.0000 max=8301 max_rank=5 min=8081 min_rank=0 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32975 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1011, 1017, 1020, 1063, 1038, 1038, 959, 1026, 55, 12, 0, 0, 0, 0, 0, 0, 1024, 1011, 1000, 1050, 993, 1016, 1008, 1007, 6, 2, 1, 0, 0, 0, 0, 0, 1069, 989, 1028, 996, 1006, 1016, 1008, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 991, 999, 988, 1064, 1000, 1007, 1029, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 983, 1004, 1040, 998, 999, 1072, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1072, 994, 1018, 1055, 1078, 1039, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1051, 995, 1060, 1032, 1024, 1083, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1047, 1011, 1009, 960, 1071, 1025, 1016, 14, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8118, 8130, 8084, 8176, 8293, 8289, 8207] avg=8192.0000 max=8293 max_rank=5 min=8084 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32965 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1007, 1024, 1033, 990, 1027, 1062, 1027, 1028, 54, 11, 0, 0, 0, 0, 0, 0, 997, 1017, 1006, 1031, 1000, 1034, 974, 1022, 6, 2, 1, 0, 0, 0, 0, 0, 1060, 1006, 1032, 1016, 1039, 1039, 989, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1014, 1036, 1028, 1080, 1021, 975, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1031, 1052, 1027, 1017, 963, 1046, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1029, 988, 1047, 1027, 997, 1021, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1033, 1041, 1036, 993, 1015, 1014, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1005, 993, 1058, 1055, 1027, 991, 984, 13, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8263, 8090, 8210, 8244, 8192, 8230, 8145, 8162] avg=8192.0000 max=8263 max_rank=0 min=8090 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1014, 1020, 1053, 1014, 982, 991, 1034, 1029, 53, 12, 0, 0, 0, 0, 0, 0, 1035, 1014, 991, 1015, 1022, 1009, 1044, 1016, 6, 2, 1, 0, 0, 0, 0, 0, 1041, 1034, 1048, 1036, 1013, 1046, 980, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1052, 1061, 1051, 1015, 1007, 1047, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1017, 1075, 1011, 996, 1002, 1047, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1012, 1040, 968, 1010, 1037, 982, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1073, 1030, 1009, 1028, 1013, 1040, 967, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 998, 1051, 1005, 1020, 968, 1067, 985, 14, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8155, 8244, 8299, 8217, 8092, 8185, 8142] avg=8192.0000 max=8299 max_rank=3 min=8092 min_rank=5 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32636 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[983, 1028, 1053, 1023, 1050, 1012, 1002, 970, 57, 17, 0, 0, 0, 0, 0, 0, 1010, 960, 1064, 1035, 1028, 1069, 1013, 1049, 13, 9, 5, 0, 0, 0, 0, 0, 1068, 1082, 1007, 1033, 1027, 1087, 995, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 964, 1030, 1071, 1021, 985, 1024, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1033, 1001, 1016, 1034, 963, 1010, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 962, 1056, 1025, 1007, 1021, 1028, 999, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1016, 1081, 1007, 1012, 1008, 1026, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1030, 1092, 1029, 998, 952, 1008, 997, 17, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8255, 8269, 8164, 8164, 8143, 8220, 8126] avg=8192.0000 max=8269 max_rank=2 min=8126 min_rank=7 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32653 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8192, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8064, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8320, 8192, 8064, 8320, 8064, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8448, 8064, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8064, 8192, 8192, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1017, 1039, 999, 1006, 969, 1008, 1004, 1048, 1038, 1073, 1050, 1042, 1057, 1023, 1025, 1037, 985, 1027, 1013, 971, 1039, 1058, 956, 1003, 1054, 1074, 998, 1083, 1013, 1029, 959, 1008, 1012, 1018, 1036, 1021, 1014, 1033, 1008, 1084, 1000, 1060, 1016, 999, 1017, 978, 990, 1016, 1030, 1075, 1048, 1007, 1085, 1030, 972, 1064, 1024, 1021, 1060, 1020, 1016, 1022, 1038, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[995, 1069, 1002, 1035, 982, 1019, 1024, 1020, 1031, 1005, 1052, 1059, 1009, 1028, 1007, 1042, 996, 1078, 1046, 1051, 1013, 1055, 1016, 1006, 994, 1006, 1054, 1004, 1003, 1012, 1005, 1005, 983, 1044, 1055, 1061, 1015, 1045, 1042, 1041, 1044, 1011, 1030, 1027, 1013, 1031, 1023, 1014, 1025, 1036, 1032, 1021, 989, 1039, 1043, 1007, 1052, 997, 1010, 954, 1013, 1032, 1020, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1028, 1041, 1008, 1027, 984, 1035, 998, 1068, 1067, 995, 1019, 1053, 1000, 1018, 1028, 1018, 1002, 1006, 1028, 1058, 998, 1046, 1009, 1024, 1067, 1015, 1024, 1032, 1053, 1048, 1040, 1048, 1027, 1022, 1006, 1016, 996, 1006, 1011, 1073, 1061, 1027, 1020, 996, 1041, 1007, 996, 1106, 989, 1050, 1030, 1026, 1004, 1029, 1036, 977, 1025, 1037, 1031, 1000, 1019, 1007, 990, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1037, 971, 1011, 1063, 1045, 1042, 1027, 1027, 1025, 1020, 1017, 1045, 1044, 1074, 1030, 1013, 1003, 1017, 1009, 1013, 1038, 1022, 1065, 1071, 1036, 1041, 1014, 1027, 1042, 999, 1049, 993, 1014, 981, 1011, 1021, 1033, 1023, 1053, 980, 1011, 1003, 1004, 1007, 1045, 1022, 982, 1011, 1023, 1054, 1018, 1066, 1034, 1017, 1013, 987, 1012, 1043, 1027, 1100, 972, 1000, 1030, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1044, 1038, 1028, 996, 987, 1014, 1053, 1057, 1042, 1002, 1047, 1021, 982, 1043, 1084, 1053, 1020, 1032, 984, 1061, 1005, 1046, 1025, 1065, 1026, 1005, 1012, 1047, 1042, 993, 1018, 978, 1027, 1066, 1000, 1006, 998, 992, 997, 985, 1063, 996, 1038, 1032, 983, 1027, 1053, 1094, 1031, 1020, 1033, 994, 988, 1023, 992, 1067, 1002, 1031, 1067, 1033, 1013, 1019, 1025, 991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1021, 1038, 1010, 1037, 1004, 1033, 1005, 1062, 1048, 1010, 1051, 1002, 1047, 1046, 1039, 1069, 1076, 1029, 992, 1020, 998, 1011, 1036, 1022, 1003, 1006, 986, 1001, 1070, 1041, 997, 1011, 1009, 999, 990, 953, 1062, 1033, 1001, 1042, 1027, 1059, 1048, 1033, 1002, 1103, 1047, 1042, 1017, 993, 1080, 1018, 1031, 1019, 1049, 1006, 988, 989, 1007, 981, 1003, 1010, 1067, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1037, 1008, 1065, 990, 1009, 1040, 1063, 993, 1010, 1037, 1086, 1004, 999, 1056, 1055, 996, 1049, 969, 1003, 1006, 992, 1038, 1006, 1042, 1032, 943, 1056, 968, 1049, 1062, 996, 975, 966, 1009, 1026, 1006, 1040, 1023, 1024, 962, 1007, 1065, 1054, 1056, 1013, 1045, 1036, 1057, 1037, 982, 1032, 1043, 1093, 1017, 1043, 1055, 997, 1019, 1063, 1049, 1036, 1027, 1013, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1050, 1018, 1015, 1089, 1066, 1089, 998, 1034, 1025, 1014, 1073, 1015, 1030, 1004, 1017, 1050, 1005, 1019, 1051, 996, 1041, 1033, 999, 992, 1059, 1086, 1023, 988, 1001, 1020, 1037, 1022, 1082, 991, 1035, 993, 997, 1062, 1019, 1014, 1054, 1036, 1003, 1087, 1012, 1023, 1032, 1023, 1032, 1035, 1003, 1023, 1058, 978, 1032, 998, 994, 1017, 918, 1018, 985, 997, 1020, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1017, 1039, 999, 1006, 969, 1008, 1004, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1073, 980, 1042, 1057, 1023, 1025, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1027, 1013, 971, 1039, 1058, 956, 1003, 12, 6, 0, 0, 0, 0, 0, 0, 1054, 1074, 998, 1083, 1013, 1029, 959, 1008, 12, 7, 0, 0, 0, 0, 0, 0, 1012, 1018, 1036, 1021, 1014, 1033, 1008, 1084, 31, 10, 0, 0, 0, 0, 0, 0, 1000, 1060, 1016, 999, 1017, 978, 990, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1075, 1048, 1007, 1067, 1030, 972, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1021, 1060, 1020, 1016, 1022, 1038, 1017, 60, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8078, 8275, 8070, 8237, 8267, 8038, 8293, 8278] avg=8192.0000 max=8293 max_rank=6 min=8038 min_rank=5 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1028, 1041, 1008, 1027, 984, 1035, 998, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 995, 952, 1053, 1000, 1018, 1028, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1006, 1028, 1058, 998, 1046, 1009, 1024, 13, 5, 0, 0, 0, 0, 0, 0, 1067, 1015, 1024, 1032, 1053, 1048, 1040, 1048, 11, 7, 0, 0, 0, 0, 0, 0, 1027, 1022, 1006, 1016, 996, 1006, 1011, 1073, 34, 9, 0, 0, 0, 0, 0, 0, 1061, 1027, 1020, 996, 1041, 1007, 996, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1050, 1030, 1026, 988, 1029, 1036, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1037, 1031, 1000, 1019, 1007, 990, 990, 58, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8131, 8189, 8345, 8200, 8213, 8125, 8157] avg=8192.0000 max=8345 max_rank=3 min=8125 min_rank=6 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32695 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1021, 1038, 1010, 1037, 1004, 1033, 1005, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1010, 981, 1002, 1047, 1046, 1039, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1029, 992, 1020, 998, 1011, 1036, 1022, 13, 5, 0, 0, 0, 0, 0, 0, 1003, 1006, 986, 1001, 1070, 1041, 997, 1011, 11, 7, 0, 0, 0, 0, 0, 0, 1009, 999, 990, 953, 1062, 1033, 1001, 1042, 32, 10, 0, 0, 0, 0, 0, 0, 1027, 1059, 1048, 1033, 1002, 1103, 1047, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 993, 1080, 1018, 1015, 1019, 1049, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 988, 989, 1007, 981, 1003, 1010, 1067, 1007, 60, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8242, 8202, 8133, 8131, 8322, 8197, 8112] avg=8192.0000 max=8322 max_rank=5 min=8112 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[995, 1069, 1002, 1035, 982, 1019, 1024, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1005, 982, 1059, 1009, 1028, 1007, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1078, 1046, 1051, 1013, 1055, 1016, 1006, 12, 5, 0, 0, 0, 0, 0, 0, 994, 1006, 1054, 1004, 1003, 1012, 1005, 1005, 11, 7, 0, 0, 0, 0, 0, 0, 983, 1044, 1055, 1061, 1015, 1045, 1042, 1041, 31, 10, 0, 0, 0, 0, 0, 0, 1044, 1011, 1030, 1027, 1013, 1031, 1023, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1036, 1032, 1021, 973, 1039, 1043, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 997, 1010, 954, 1013, 1032, 1020, 1064, 60, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8163, 8278, 8101, 8327, 8155, 8176, 8202] avg=8192.0000 max=8327 max_rank=4 min=8101 min_rank=3 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1037, 1008, 1065, 990, 1009, 1040, 1063, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1037, 1014, 1004, 999, 1056, 1055, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 969, 1003, 1006, 992, 1038, 1006, 1042, 12, 6, 0, 0, 0, 0, 0, 0, 1032, 943, 1056, 968, 1049, 1062, 996, 975, 12, 7, 0, 0, 0, 0, 0, 0, 966, 1009, 1026, 1006, 1040, 1023, 1024, 962, 33, 10, 0, 0, 0, 0, 0, 0, 1007, 1065, 1054, 1056, 1013, 1045, 1036, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 982, 1032, 1043, 1075, 1017, 1043, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1019, 1063, 1049, 1036, 1027, 1013, 1007, 62, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8171, 8123, 8100, 8099, 8293, 8284, 8273] avg=8192.0000 max=8293 max_rank=5 min=8099 min_rank=4 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32949 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1037, 971, 1011, 1063, 1045, 1042, 1027, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1020, 950, 1045, 1044, 1074, 1030, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1017, 1009, 1013, 1038, 1022, 1065, 1071, 12, 5, 0, 0, 0, 0, 0, 0, 1036, 1041, 1014, 1027, 1042, 999, 1049, 993, 11, 7, 0, 0, 0, 0, 0, 0, 1014, 981, 1011, 1021, 1033, 1023, 1053, 980, 31, 9, 0, 0, 0, 0, 0, 0, 1011, 1003, 1004, 1007, 1045, 1022, 982, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1054, 1018, 1066, 1018, 1017, 1013, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1043, 1027, 1100, 972, 1000, 1030, 1009, 58, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8201, 8255, 8219, 8156, 8047, 8196, 8251] avg=8192.0000 max=8255 max_rank=2 min=8047 min_rank=5 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1044, 1038, 1028, 996, 987, 1014, 1053, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1002, 978, 1021, 982, 1043, 1084, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1032, 984, 1061, 1005, 1046, 1025, 1065, 13, 5, 0, 0, 0, 0, 0, 0, 1026, 1005, 1012, 1047, 1042, 993, 1018, 978, 11, 7, 0, 0, 0, 0, 0, 0, 1027, 1066, 1000, 1006, 998, 992, 997, 985, 34, 9, 0, 0, 0, 0, 0, 0, 1063, 996, 1038, 1032, 983, 1027, 1053, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1020, 1033, 994, 972, 1023, 992, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1031, 1067, 1033, 1013, 1019, 1025, 991, 60, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8205, 8256, 8139, 8114, 8245, 8132, 8241] avg=8192.0000 max=8256 max_rank=2 min=8114 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1050, 1018, 1015, 1089, 1066, 1089, 998, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1014, 993, 1015, 1030, 1004, 1017, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1019, 1051, 996, 1041, 1033, 999, 992, 16, 11, 0, 0, 0, 0, 0, 0, 1059, 1086, 1023, 988, 1001, 1020, 1037, 1022, 15, 11, 0, 0, 0, 0, 0, 0, 1082, 991, 1035, 993, 997, 1062, 1019, 1014, 36, 13, 0, 0, 0, 0, 0, 0, 1054, 1036, 1003, 1087, 1012, 1023, 1032, 976, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1035, 1003, 1023, 1032, 978, 1032, 998, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1017, 918, 1018, 985, 997, 1020, 1006, 67, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8343, 8148, 8163, 8262, 8242, 8223, 8133, 8022] avg=8192.0000 max=8343 max_rank=0 min=8022 min_rank=7 imbalance_ratio=1.018433 max_load_reduction=57193 max_load_reduction_pct=0.872696 moved_to_echo=32620 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8064, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8064, 8192, 8320, 8192, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8320, 8192, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8320, 8192, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 7936, 8320, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1023, 1010, 1021, 990, 1031, 1048, 1039, 1053, 1007, 998, 1053, 996, 1039, 1058, 1007, 991, 1016, 972, 1028, 1006, 1008, 1031, 1040, 1022, 1074, 1054, 1013, 989, 989, 1021, 1048, 1030, 1070, 980, 1011, 993, 1051, 1048, 1035, 1040, 1024, 1080, 1006, 1027, 1042, 977, 979, 994, 1015, 1027, 1055, 1067, 1028, 1032, 1054, 1059, 977, 985, 1026, 1010, 1019, 1042, 1042, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1004, 1023, 989, 1030, 1028, 1037, 1010, 989, 1064, 977, 964, 976, 1068, 1058, 1035, 1016, 998, 1010, 1005, 1043, 1101, 1086, 996, 1053, 1023, 1011, 985, 954, 1048, 1008, 1042, 1058, 989, 1016, 1011, 1030, 1036, 1027, 976, 1043, 1091, 1027, 985, 994, 1019, 1038, 1024, 1041, 994, 997, 1030, 1057, 1055, 1020, 1049, 988, 1040, 1071, 997, 1038, 986, 1091, 1043, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[988, 1005, 1034, 1016, 1070, 978, 1019, 1018, 1025, 1016, 994, 1056, 1083, 999, 1036, 1048, 992, 1066, 1007, 1040, 1007, 1008, 983, 1006, 1009, 994, 998, 1054, 964, 1042, 1058, 1095, 1059, 1061, 1015, 986, 1014, 1046, 1014, 1033, 971, 994, 1017, 987, 977, 1067, 1050, 989, 1030, 1010, 1053, 1039, 1031, 1067, 1006, 1050, 1102, 1063, 1043, 952, 1010, 1022, 1006, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1027, 984, 1001, 1027, 1021, 1049, 1074, 1036, 1047, 1043, 1008, 1021, 1090, 1071, 994, 1040, 995, 1023, 1021, 1014, 986, 1026, 1059, 987, 1038, 1061, 1024, 1056, 1002, 972, 1018, 1019, 1005, 1039, 1037, 1017, 1009, 983, 1063, 1053, 1097, 1024, 1019, 1054, 990, 1001, 989, 1066, 1018, 1004, 1015, 1001, 982, 996, 1026, 1017, 1008, 1055, 1035, 1020, 1035, 992, 1039, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1029, 1038, 1011, 1070, 1062, 1035, 1010, 1000, 1007, 1040, 1059, 1016, 1025, 1001, 1016, 1022, 1056, 996, 1024, 1031, 1053, 994, 1046, 1005, 1029, 1032, 992, 1032, 1015, 1037, 1002, 1036, 973, 1022, 1006, 1023, 1020, 969, 998, 1075, 1103, 1064, 1032, 1003, 1076, 986, 1084, 1024, 1027, 1032, 1003, 988, 1042, 992, 994, 971, 1029, 1042, 1044, 1016, 1049, 1022, 1012, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1026, 993, 1000, 983, 1005, 1037, 1067, 1015, 1024, 1055, 991, 993, 1026, 1031, 1063, 1021, 1031, 1060, 1005, 1053, 1021, 1037, 1056, 1037, 1039, 997, 1091, 1066, 1005, 1063, 1036, 1001, 1041, 1029, 1037, 1005, 1012, 1033, 1077, 1009, 949, 981, 1050, 1023, 1034, 1048, 1021, 1044, 1037, 1037, 1020, 1010, 997, 1033, 1016, 988, 1016, 966, 1044, 1004, 1007, 993, 1015, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1018, 1048, 976, 1018, 977, 1022, 1028, 1004, 1040, 1077, 990, 1036, 981, 1106, 1045, 1044, 1069, 1011, 1045, 1019, 994, 1036, 1016, 1012, 1012, 995, 1031, 1017, 1008, 1091, 975, 1034, 1019, 983, 1038, 1054, 1038, 939, 1024, 1038, 1020, 1009, 1014, 1032, 986, 1011, 1036, 1005, 1068, 1013, 1044, 1019, 1054, 1062, 998, 1042, 1040, 1039, 1025, 1011, 1007, 1031, 985, 1077] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[983, 1044, 1067, 998, 1035, 1008, 1007, 1025, 1008, 992, 1036, 1080, 1045, 985, 986, 977, 1012, 957, 1055, 1028, 1029, 1043, 1062, 1013, 1057, 984, 1005, 1018, 1045, 1029, 1058, 1019, 1008, 1003, 997, 1044, 1028, 1058, 1026, 1040, 1022, 1004, 999, 1102, 1017, 1025, 1014, 1074, 1041, 1013, 1030, 1049, 1031, 1058, 1006, 1023, 1022, 1068, 1007, 985, 1009, 1008, 1024, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1023, 1010, 1021, 990, 1031, 1048, 1039, 1053, 19, 8, 0, 0, 0, 0, 0, 0, 1007, 998, 1053, 996, 1020, 1058, 1007, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 972, 1028, 1006, 1008, 1031, 1040, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1054, 1013, 989, 989, 1021, 1048, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 980, 1011, 993, 1051, 1048, 1035, 1040, 8, 1, 0, 0, 0, 0, 0, 0, 1024, 1080, 1006, 1027, 1042, 977, 979, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1027, 1055, 1067, 1028, 1023, 1054, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 977, 985, 1026, 1010, 1019, 1042, 1042, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8130, 8123, 8210, 8237, 8129, 8328, 8137] avg=8192.0000 max=8328 max_rank=6 min=8123 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32831 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1026, 993, 1000, 983, 1005, 1037, 1067, 1015, 19, 8, 0, 0, 0, 0, 0, 0, 1024, 1055, 991, 993, 1007, 1031, 1063, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1060, 1005, 1053, 1021, 1037, 1056, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 997, 1091, 1066, 1005, 1063, 1036, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1029, 1037, 1005, 1012, 1033, 1077, 1009, 7, 1, 0, 0, 0, 0, 0, 0, 949, 981, 1050, 1023, 1034, 1048, 1021, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1037, 1020, 1010, 997, 1024, 1016, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 966, 1044, 1004, 1007, 993, 1015, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8185, 8300, 8291, 8251, 8150, 8129, 8077] avg=8192.0000 max=8300 max_rank=2 min=8077 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32607 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[988, 1005, 1034, 1016, 1070, 978, 1019, 1018, 20, 8, 0, 0, 0, 0, 0, 0, 1025, 1016, 994, 1056, 1063, 999, 1036, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1066, 1007, 1040, 1007, 1008, 983, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 994, 998, 1054, 964, 1042, 1058, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1061, 1015, 986, 1014, 1046, 1014, 1033, 8, 1, 0, 0, 0, 0, 0, 0, 971, 994, 1017, 987, 977, 1067, 1050, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1010, 1053, 1039, 1031, 1058, 1006, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 1063, 1043, 952, 1010, 1022, 1006, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8237, 8109, 8206, 8237, 8052, 8277, 8262] avg=8192.0000 max=8277 max_rank=6 min=8052 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32828 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1004, 1023, 989, 1030, 1028, 1037, 1010, 989, 20, 8, 0, 0, 0, 0, 0, 0, 1064, 977, 964, 976, 1048, 1058, 1035, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1010, 1005, 1043, 1101, 1086, 996, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1011, 985, 954, 1048, 1008, 1042, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1016, 1011, 1030, 1036, 1027, 976, 1043, 8, 1, 0, 0, 0, 0, 0, 0, 1091, 1027, 985, 994, 1019, 1038, 1024, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 994, 997, 1030, 1057, 1055, 1011, 1049, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1071, 997, 1038, 986, 1091, 1043, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8138, 8292, 8121, 8137, 8219, 8181, 8310] avg=8192.0000 max=8310 max_rank=7 min=8121 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32847 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1027, 984, 1001, 1027, 1021, 1049, 1074, 1036, 20, 7, 0, 0, 0, 0, 0, 0, 1047, 1043, 1008, 1021, 1070, 1071, 994, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1023, 1021, 1014, 986, 1026, 1059, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1061, 1024, 1056, 1002, 972, 1018, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1039, 1037, 1017, 1009, 983, 1063, 1053, 8, 1, 0, 0, 0, 0, 0, 0, 1097, 1024, 1019, 1054, 990, 1001, 989, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1004, 1015, 1001, 982, 988, 1026, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1055, 1035, 1020, 1035, 992, 1039, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8294, 8111, 8182, 8215, 8240, 8051, 8197] avg=8192.0000 max=8294 max_rank=1 min=8051 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32703 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1018, 1048, 976, 1018, 977, 1022, 1028, 1004, 18, 8, 0, 0, 0, 0, 0, 0, 1040, 1077, 990, 1036, 963, 1106, 1045, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 1011, 1045, 1019, 994, 1036, 1016, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 995, 1031, 1017, 1008, 1091, 975, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 983, 1038, 1054, 1038, 939, 1024, 1038, 8, 1, 0, 0, 0, 0, 0, 0, 1020, 1009, 1014, 1032, 986, 1011, 1036, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1013, 1044, 1019, 1054, 1053, 998, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1039, 1025, 1011, 1007, 1031, 985, 1077, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8301, 8202, 8155, 8142, 8113, 8291, 8215] avg=8192.0000 max=8301 max_rank=1 min=8113 min_rank=5 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32761 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1029, 1038, 1011, 1070, 1062, 1035, 1010, 1000, 19, 7, 0, 0, 0, 0, 0, 0, 1007, 1040, 1059, 1016, 1006, 1001, 1016, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 996, 1024, 1031, 1053, 994, 1046, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1032, 992, 1032, 1015, 1037, 1002, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1022, 1006, 1023, 1020, 969, 998, 1075, 8, 1, 0, 0, 0, 0, 0, 0, 1103, 1064, 1032, 1003, 1076, 986, 1084, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1032, 1003, 988, 1042, 984, 994, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1042, 1044, 1016, 1049, 1022, 1012, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8167, 8205, 8167, 8095, 8372, 8041, 8208] avg=8192.0000 max=8372 max_rank=5 min=8041 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32716 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[983, 1044, 1067, 998, 1035, 1008, 1007, 1025, 25, 11, 0, 0, 0, 0, 0, 0, 1008, 992, 1036, 1080, 1020, 985, 986, 977, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 957, 1055, 1028, 1029, 1038, 1062, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 984, 1005, 1018, 1045, 1029, 1058, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1003, 997, 1044, 1028, 1058, 1026, 1040, 11, 2, 5, 0, 0, 0, 0, 0, 1022, 1004, 999, 1102, 1017, 1025, 1014, 1074, 4, 0, 0, 0, 0, 0, 0, 0, 1041, 1013, 1030, 1049, 1031, 1045, 1006, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1064, 1007, 985, 1009, 1008, 1024, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8084, 8194, 8204, 8222, 8261, 8238, 8130] avg=8192.0000 max=8261 max_rank=5 min=8084 min_rank=1 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32851 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8064, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8448, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1037, 1003, 1056, 1065, 1013, 1037, 1029, 985, 995, 1006, 1011, 1025, 1009, 1028, 1065, 1020, 987, 1030, 977, 1017, 1053, 969, 1021, 1023, 993, 1029, 1037, 1038, 996, 974, 1055, 1102, 1051, 1037, 1021, 1037, 1007, 1031, 1050, 1019, 1059, 1063, 1073, 990, 1024, 1044, 1029, 1027, 1016, 1020, 1059, 1007, 1006, 1005, 1075, 985, 966, 1022, 996, 1026, 1035, 1050, 1009, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1086, 1041, 983, 986, 1026, 1044, 1048, 1069, 1004, 1035, 968, 988, 1004, 1057, 1028, 1025, 970, 1018, 1036, 999, 989, 1039, 1035, 1010, 1016, 1082, 1022, 1001, 1012, 1034, 1030, 1020, 1085, 1033, 1025, 1060, 1065, 1024, 988, 1034, 1005, 1015, 1044, 976, 1020, 1008, 1024, 1031, 1046, 1037, 973, 1049, 1018, 1046, 1006, 1004, 1030, 1060, 1017, 1044, 1001, 1023, 1005, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1039, 1017, 997, 1018, 1035, 1010, 1058, 998, 1036, 1030, 1037, 1043, 989, 1011, 1036, 1036, 1036, 1047, 1002, 1021, 1028, 1042, 985, 1002, 1023, 976, 1084, 1024, 1038, 1045, 976, 1069, 995, 1044, 1015, 995, 983, 1071, 1006, 1031, 1038, 1009, 1010, 992, 1053, 1044, 1027, 1038, 987, 1024, 1024, 967, 1024, 999, 1077, 1045, 1033, 1054, 1024, 988, 983, 1040, 1062, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[995, 1038, 1035, 1033, 1003, 1052, 1032, 1065, 1013, 1044, 1031, 1006, 1008, 1066, 1009, 1032, 1064, 982, 934, 1054, 1096, 979, 1018, 1019, 1011, 1021, 1040, 1035, 1020, 1011, 1038, 983, 1040, 1011, 1006, 1004, 1019, 965, 1008, 1003, 1017, 989, 1010, 1037, 1014, 1046, 1035, 1025, 1039, 1025, 1068, 1037, 1002, 988, 1060, 1018, 1068, 984, 1069, 1054, 1027, 976, 1069, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[978, 1028, 1054, 1025, 998, 959, 1033, 1037, 1034, 996, 1024, 1008, 1017, 1053, 1029, 1063, 1039, 1028, 978, 1073, 1005, 1007, 1020, 1016, 1048, 1042, 1013, 1067, 1075, 999, 1012, 988, 992, 1047, 1069, 1067, 988, 1020, 1008, 1027, 1057, 1005, 1019, 1057, 979, 1064, 955, 1077, 979, 1042, 1070, 989, 991, 1041, 1041, 1034, 1040, 1019, 1024, 946, 1048, 1068, 1058, 969] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1035, 1007, 1019, 1054, 1040, 1001, 1055, 1005, 1002, 1018, 1001, 1078, 1024, 1009, 1046, 973, 1020, 960, 1023, 999, 1016, 1049, 1011, 1005, 1026, 1053, 1015, 1021, 1037, 1046, 1018, 1027, 1029, 1074, 1004, 1004, 995, 1050, 1042, 1017, 1005, 1052, 1016, 1053, 1033, 1027, 1020, 1019, 993, 1029, 1055, 1043, 1031, 1036, 1032, 1051, 985, 994, 998, 1041, 1012, 1020, 994, 1089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1044, 1063, 1039, 988, 1024, 995, 998, 1026, 1008, 1037, 981, 1055, 1024, 995, 976, 993, 986, 1027, 1012, 1042, 1056, 988, 1003, 1032, 1036, 1053, 999, 1016, 1026, 1014, 1007, 1040, 1011, 1019, 1066, 1009, 1050, 1089, 1085, 997, 1014, 1005, 1005, 998, 1074, 1058, 1032, 1008, 1032, 1040, 1003, 1030, 1017, 1034, 997, 1007, 1022, 1043, 1047, 994, 1063, 993, 1024, 1087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1038, 1048, 1033, 1018, 991, 1071, 972, 1069, 1033, 1059, 1033, 1038, 1011, 1030, 1051, 1056, 1002, 998, 1048, 983, 990, 1010, 1032, 1020, 1029, 1035, 973, 1059, 1020, 1037, 991, 1027, 1043, 1032, 1032, 988, 1019, 1001, 1002, 1039, 1010, 1004, 1012, 1057, 1052, 1003, 1048, 1034, 1007, 1040, 1071, 992, 991, 1043, 961, 1015, 1024, 1005, 992, 980, 1032, 1067, 1054, 1081] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1037, 1003, 1056, 1065, 1013, 1037, 1029, 969, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1006, 1011, 1025, 1009, 1028, 1065, 1020, 8, 2, 0, 0, 0, 0, 0, 0, 987, 1030, 977, 1017, 1053, 969, 1021, 1023, 19, 18, 16, 15, 0, 0, 0, 0, 993, 1011, 1037, 1038, 996, 974, 1055, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1020, 1021, 1037, 1007, 1031, 1050, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1063, 1073, 990, 1024, 1029, 1029, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1020, 1059, 1007, 1006, 1005, 1075, 985, 7, 0, 0, 0, 0, 0, 0, 0, 966, 1022, 996, 1026, 1035, 1050, 1009, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8169, 8145, 8206, 8236, 8294, 8180, 8097] avg=8192.0000 max=8294 max_rank=5 min=8097 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32807 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[995, 1038, 1035, 1033, 1003, 1052, 1032, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1044, 1031, 1006, 1008, 1066, 1009, 1032, 8, 2, 0, 0, 0, 0, 0, 0, 1064, 982, 934, 1054, 1096, 979, 1018, 1019, 20, 18, 18, 15, 0, 0, 0, 0, 1011, 1003, 1040, 1035, 1020, 1011, 1038, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 994, 1006, 1004, 1019, 965, 1008, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 989, 1010, 1037, 1014, 1031, 1035, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1025, 1068, 1037, 1002, 988, 1060, 1018, 7, 0, 0, 0, 0, 0, 0, 0, 1068, 984, 1069, 1054, 1027, 976, 1069, 1036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8219, 8217, 8141, 8039, 8158, 8244, 8283] avg=8192.0000 max=8283 max_rank=7 min=8039 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1086, 1041, 983, 986, 1026, 1044, 1048, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1035, 968, 988, 1004, 1057, 1028, 1025, 8, 2, 0, 0, 0, 0, 0, 0, 970, 1018, 1036, 999, 989, 1039, 1035, 1010, 19, 19, 18, 15, 0, 0, 0, 0, 1016, 1063, 1022, 1001, 1012, 1034, 1030, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 1016, 1025, 1060, 1065, 1024, 988, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1015, 1044, 976, 1020, 993, 1024, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1037, 973, 1049, 1018, 1046, 1006, 1004, 7, 0, 0, 0, 0, 0, 0, 0, 1030, 1060, 1017, 1044, 1001, 1023, 1005, 1016, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8119, 8167, 8198, 8297, 8108, 8186, 8196] avg=8192.0000 max=8297 max_rank=4 min=8108 min_rank=5 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1035, 1007, 1019, 1054, 1040, 1001, 1055, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1018, 1001, 1078, 1024, 1009, 1046, 973, 8, 2, 0, 0, 0, 0, 0, 0, 1020, 960, 1023, 999, 1016, 1049, 1011, 1005, 20, 18, 17, 16, 0, 0, 0, 0, 1026, 1035, 1015, 1021, 1037, 1046, 1018, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1056, 1004, 1004, 995, 1050, 1042, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1052, 1016, 1053, 1033, 1012, 1020, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1029, 1055, 1043, 1031, 1036, 1032, 1051, 7, 0, 0, 0, 0, 0, 0, 0, 985, 994, 998, 1041, 1012, 1020, 994, 1069, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8161, 8154, 8225, 8197, 8210, 8277, 8113] avg=8192.0000 max=8277 max_rank=6 min=8113 min_rank=7 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1044, 1063, 1039, 988, 1024, 995, 998, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1037, 981, 1055, 1024, 995, 976, 993, 8, 2, 0, 0, 0, 0, 0, 0, 986, 1027, 1012, 1042, 1056, 988, 1003, 1032, 20, 18, 17, 15, 0, 0, 0, 0, 1036, 1035, 999, 1016, 1026, 1014, 1007, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1002, 1066, 1009, 1050, 1089, 1085, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1005, 1005, 998, 1074, 1042, 1032, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1040, 1003, 1030, 1017, 1034, 997, 1007, 8, 0, 0, 0, 0, 0, 0, 0, 1022, 1043, 1047, 994, 1063, 993, 1024, 1067, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8079, 8216, 8173, 8309, 8178, 8168, 8253] avg=8192.0000 max=8309 max_rank=4 min=8079 min_rank=1 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32908 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1039, 1017, 997, 1018, 1035, 1010, 1058, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1030, 1037, 1043, 989, 1011, 1036, 1036, 8, 2, 0, 0, 0, 0, 0, 0, 1036, 1047, 1002, 1021, 1028, 1042, 985, 1002, 20, 17, 17, 15, 0, 0, 0, 0, 1023, 959, 1084, 1024, 1038, 1045, 976, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1027, 1015, 995, 983, 1071, 1006, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1009, 1010, 992, 1053, 1029, 1027, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1024, 1024, 967, 1024, 999, 1077, 1045, 7, 0, 0, 0, 0, 0, 0, 0, 1033, 1054, 1024, 988, 983, 1040, 1062, 1046, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8228, 8232, 8218, 8123, 8196, 8154, 8230] avg=8192.0000 max=8232 max_rank=2 min=8123 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[978, 1028, 1054, 1025, 998, 959, 1033, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 996, 1024, 1008, 1017, 1053, 1029, 1063, 8, 2, 0, 0, 0, 0, 0, 0, 1039, 1028, 978, 1073, 1005, 1007, 1020, 1016, 18, 18, 17, 15, 0, 0, 0, 0, 1048, 1024, 1013, 1067, 1075, 999, 1012, 988, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1030, 1069, 1067, 988, 1020, 1008, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1005, 1019, 1057, 979, 1048, 955, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1042, 1070, 989, 991, 1041, 1041, 1034, 8, 0, 0, 0, 0, 0, 0, 0, 1040, 1019, 1024, 946, 1048, 1068, 1058, 951, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8095, 8234, 8234, 8226, 8201, 8197, 8195, 8154] avg=8192.0000 max=8234 max_rank=1 min=8095 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1038, 1048, 1033, 1018, 991, 1071, 972, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1059, 1033, 1038, 1011, 1030, 1051, 1056, 13, 3, 0, 0, 0, 0, 0, 0, 1002, 998, 1048, 983, 990, 1010, 1032, 1020, 25, 22, 22, 19, 0, 0, 0, 0, 1029, 1013, 973, 1059, 1020, 1037, 991, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1010, 1032, 988, 1019, 1001, 1002, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1004, 1012, 1057, 1052, 978, 1048, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1040, 1071, 992, 991, 1043, 961, 1015, 12, 0, 0, 0, 0, 0, 0, 0, 1024, 1005, 992, 980, 1032, 1067, 1054, 1056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8327, 8171, 8149, 8134, 8195, 8132, 8210] avg=8192.0000 max=8327 max_rank=1 min=8132 min_rank=6 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8064, 8192, 8320, 8192, 8192, 8192, 256, 256, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8448, 8192, 8192, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[962, 991, 1003, 995, 1053, 1023, 1008, 1022, 1008, 970, 1025, 968, 1080, 1029, 1028, 1019, 1021, 1025, 1025, 987, 1012, 1008, 1040, 1004, 1036, 1074, 1039, 1027, 1012, 1012, 1031, 1024, 1045, 1018, 1045, 1029, 1014, 986, 1011, 1049, 1044, 1072, 1057, 1036, 1037, 1025, 1060, 1014, 1027, 1014, 1023, 1036, 1018, 1051, 1016, 995, 1037, 1027, 1025, 1007, 1035, 1037, 1058, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1029, 949, 1023, 959, 1004, 1040, 1065, 1013, 1055, 1010, 1039, 1012, 1030, 1040, 998, 1032, 1016, 1012, 1020, 1049, 1033, 1044, 1011, 996, 1040, 1019, 1014, 1048, 1003, 1061, 1029, 982, 970, 986, 1006, 1032, 977, 1044, 1061, 989, 1011, 1030, 973, 1075, 987, 1065, 1025, 1060, 1076, 1061, 1025, 1037, 999, 1035, 1032, 1049, 1063, 1052, 1013, 999, 1024, 1044, 1038, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[978, 1035, 1016, 1045, 960, 1058, 1031, 971, 1014, 1049, 1026, 1008, 1017, 1011, 985, 1044, 1108, 1022, 1042, 972, 1010, 982, 966, 1011, 1002, 1053, 1028, 1051, 1048, 1051, 1033, 1024, 1017, 989, 1016, 1060, 1035, 990, 1016, 1008, 970, 1006, 1085, 993, 1012, 1017, 1028, 1008, 986, 1096, 1075, 1048, 1054, 1007, 1041, 1055, 1053, 995, 1020, 1059, 999, 1034, 1068, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1043, 1022, 1026, 1023, 1014, 1030, 987, 1079, 1004, 1077, 1030, 1031, 1002, 1020, 1052, 1017, 977, 979, 1049, 1025, 951, 1010, 1002, 1002, 1069, 996, 1034, 1060, 992, 1054, 1002, 994, 1036, 997, 1032, 977, 1063, 1054, 1012, 1065, 1044, 1038, 1034, 993, 1050, 1052, 1000, 1025, 1036, 1008, 999, 1066, 1074, 1012, 1075, 1059, 1005, 1009, 1029, 992, 1025, 981, 1001, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1027, 1055, 980, 991, 1004, 1063, 978, 1034, 994, 1021, 1053, 1005, 1001, 1018, 1032, 1015, 1050, 970, 1054, 1028, 1001, 1033, 998, 1005, 1012, 993, 1047, 1034, 1098, 1019, 997, 1038, 1032, 1021, 1068, 993, 1017, 1013, 1035, 1000, 1016, 1058, 1056, 1050, 1042, 971, 1043, 1057, 1021, 991, 1036, 1002, 1027, 995, 996, 978, 1054, 1074, 1023, 1066, 1005, 1030, 1029, 1089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1019, 1019, 1004, 996, 1090, 1073, 998, 1014, 1036, 1039, 1047, 1018, 998, 1032, 1036, 1027, 1040, 1044, 997, 1007, 1043, 964, 1043, 1010, 985, 1024, 1012, 1021, 978, 1028, 1054, 989, 1047, 1030, 1034, 1017, 1060, 1023, 1019, 1088, 1001, 1021, 984, 1010, 1033, 1062, 1042, 1029, 960, 1007, 1004, 1070, 1039, 1035, 1005, 1029, 998, 989, 1072, 1011, 1080, 1019, 1005, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1026, 1057, 985, 1006, 1075, 1027, 1033, 1043, 1029, 1051, 1046, 1046, 996, 1025, 965, 1051, 1043, 975, 988, 957, 993, 975, 1038, 1013, 1020, 1002, 1041, 995, 976, 1021, 1014, 1051, 1049, 1060, 1072, 1003, 1047, 980, 1005, 1034, 1028, 1042, 979, 1015, 981, 1043, 1019, 1019, 1092, 985, 1011, 1012, 1060, 1022, 1027, 1028, 986, 1055, 1050, 1062, 1080, 1025, 1058, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1033, 1030, 1001, 1025, 1000, 1015, 1039, 1009, 1047, 994, 1008, 1042, 994, 997, 1025, 1038, 1089, 1023, 996, 1019, 1078, 1002, 1064, 1038, 1008, 1012, 1010, 1043, 982, 1039, 1055, 1055, 1031, 1000, 1027, 1031, 1036, 1035, 1027, 1035, 1028, 1026, 986, 1019, 1056, 1005, 968, 983, 1053, 1039, 985, 1035, 1013, 1012, 1022, 998, 967, 1038, 1041, 1011, 1034, 1021, 1101, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[962, 991, 1003, 995, 1053, 1023, 1008, 1022, 28, 7, 5, 0, 0, 0, 0, 0, 1008, 970, 1025, 968, 1080, 1029, 1028, 1019, 7, 2, 0, 0, 0, 0, 0, 0, 1021, 1025, 1025, 987, 1012, 1008, 1040, 1004, 65, 1, 0, 0, 0, 0, 0, 0, 1036, 1074, 1039, 1027, 1012, 1005, 1031, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1018, 1038, 1029, 1014, 986, 1011, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1065, 1057, 1036, 1037, 1025, 1060, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1014, 1023, 1007, 1018, 1051, 1016, 995, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1027, 1025, 1007, 1035, 1037, 1058, 962, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8136, 8188, 8248, 8190, 8338, 8151, 8188] avg=8192.0000 max=8338 max_rank=5 min=8097 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=57198 max_load_reduction_pct=0.872772 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1029, 949, 1023, 959, 1004, 1040, 1065, 1013, 28, 7, 5, 0, 0, 0, 0, 0, 1055, 1010, 1039, 1012, 1030, 1040, 998, 1032, 7, 2, 0, 0, 0, 0, 0, 0, 1016, 1012, 1020, 1049, 1033, 1044, 1011, 996, 65, 1, 0, 0, 0, 0, 0, 0, 1040, 1019, 1014, 1048, 1003, 1054, 1029, 982, 0, 0, 0, 0, 0, 0, 0, 0, 970, 986, 999, 1032, 977, 1044, 1061, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1023, 973, 1075, 987, 1065, 1025, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1061, 1025, 1008, 999, 1035, 1032, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1052, 1013, 999, 1024, 1044, 1038, 958, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8225, 8247, 8189, 8058, 8219, 8285, 8191] avg=8192.0000 max=8285 max_rank=6 min=8058 min_rank=4 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32753 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1043, 1022, 1026, 1023, 1014, 1030, 987, 1079, 29, 7, 5, 0, 0, 0, 0, 0, 1004, 1077, 1030, 1031, 1002, 1020, 1052, 1017, 7, 2, 0, 0, 0, 0, 0, 0, 977, 979, 1049, 1025, 951, 1010, 1002, 1002, 66, 1, 0, 0, 0, 0, 0, 0, 1069, 996, 1034, 1060, 992, 1047, 1002, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 997, 1025, 977, 1063, 1054, 1012, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1031, 1034, 993, 1050, 1052, 1000, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1008, 999, 1036, 1074, 1012, 1075, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1009, 1029, 992, 1025, 981, 1001, 974, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 8242, 8062, 8194, 8229, 8229, 8299, 8016] avg=8192.0000 max=8299 max_rank=6 min=8016 min_rank=7 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[978, 1035, 1016, 1045, 960, 1058, 1031, 971, 28, 7, 4, 0, 0, 0, 0, 0, 1014, 1049, 1026, 1008, 1017, 1011, 985, 1044, 7, 2, 0, 0, 0, 0, 0, 0, 1108, 1022, 1042, 972, 1010, 982, 966, 1011, 67, 1, 0, 0, 0, 0, 0, 0, 1002, 1053, 1028, 1051, 1048, 1044, 1033, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 989, 1009, 1060, 1035, 990, 1016, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1000, 1085, 993, 1012, 1017, 1028, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1096, 1075, 1019, 1054, 1007, 1041, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 995, 1020, 1059, 999, 1034, 1068, 978, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8163, 8181, 8283, 8124, 8113, 8333, 8206] avg=8192.0000 max=8333 max_rank=6 min=8113 min_rank=5 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1027, 1055, 980, 991, 1004, 1063, 978, 1034, 27, 7, 5, 0, 0, 0, 0, 0, 994, 1021, 1053, 1005, 1001, 1018, 1032, 1015, 7, 2, 0, 0, 0, 0, 0, 0, 1050, 970, 1054, 1028, 1001, 1033, 998, 1005, 69, 1, 0, 0, 0, 0, 0, 0, 1012, 993, 1047, 1034, 1098, 1012, 997, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1021, 1061, 993, 1017, 1013, 1035, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1051, 1056, 1050, 1042, 971, 1043, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 991, 1036, 974, 1027, 995, 996, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1074, 1023, 1066, 1005, 1030, 1029, 1020, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8148, 8209, 8231, 8172, 8286, 8018, 8301] avg=8192.0000 max=8301 max_rank=7 min=8018 min_rank=6 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32777 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1019, 1019, 1004, 996, 1090, 1073, 998, 1014, 29, 7, 5, 0, 0, 0, 0, 0, 1036, 1039, 1047, 1018, 998, 1032, 1036, 1027, 7, 2, 0, 0, 0, 0, 0, 0, 1040, 1044, 997, 1007, 1043, 964, 1043, 1010, 65, 1, 0, 0, 0, 0, 0, 0, 985, 1024, 1012, 1021, 978, 1021, 1054, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1030, 1027, 1017, 1060, 1023, 1019, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1014, 984, 1010, 1033, 1062, 1042, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 960, 1007, 1004, 1040, 1039, 1035, 1005, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 998, 989, 1072, 1011, 1080, 1019, 1005, 963, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8242, 8214, 8084, 8311, 8175, 8119, 8137] avg=8192.0000 max=8311 max_rank=4 min=8084 min_rank=3 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1026, 1057, 985, 1006, 1075, 1027, 1033, 1043, 30, 13, 7, 0, 0, 0, 0, 0, 1029, 1051, 1046, 1046, 996, 1025, 965, 1051, 10, 7, 0, 0, 0, 0, 0, 0, 1043, 975, 988, 957, 993, 975, 1038, 1013, 71, 4, 0, 0, 0, 0, 0, 0, 1020, 1002, 1041, 995, 976, 1011, 1014, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1060, 1059, 1003, 1047, 980, 1005, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1028, 979, 1015, 981, 1043, 1019, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 985, 1011, 978, 1060, 1022, 1027, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1055, 1050, 1062, 1080, 1025, 1058, 973, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8302, 8226, 8057, 8110, 8237, 8112, 8203, 8289] avg=8192.0000 max=8302 max_rank=0 min=8057 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1033, 1030, 1001, 1025, 1000, 1015, 1039, 1009, 28, 7, 5, 0, 0, 0, 0, 0, 1047, 994, 1008, 1042, 994, 997, 1025, 1038, 7, 2, 0, 0, 0, 0, 0, 0, 1089, 1023, 996, 1019, 1078, 1002, 1064, 1038, 68, 1, 0, 0, 0, 0, 0, 0, 1008, 1012, 1010, 1043, 982, 1032, 1055, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1000, 1020, 1031, 1036, 1035, 1027, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1019, 986, 1019, 1056, 1005, 968, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1039, 985, 1006, 1013, 1012, 1022, 998, 0, 0, 0, 0, 0, 0, 0, 0, 967, 1038, 1041, 1011, 1034, 1021, 1101, 995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8154, 8378, 8197, 8215, 8064, 8128, 8208] avg=8192.0000 max=8378 max_rank=2 min=8064 min_rank=5 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8064, 8192, 8064, 8192, 8064, 8192, 8192, 640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8064, 8064, 8320, 8448, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8320, 8192, 8448, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8192, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1019, 1019, 1043, 999, 1024, 1006, 1029, 1003, 1024, 1019, 1013, 1052, 995, 986, 1032, 1019, 1089, 1054, 1028, 1036, 1072, 1008, 1020, 1037, 1046, 998, 1026, 986, 1066, 1015, 1037, 1028, 1008, 1057, 1042, 1019, 1044, 1044, 963, 1057, 1074, 1007, 1053, 986, 1028, 1003, 1039, 1002, 979, 1052, 1027, 959, 957, 1009, 1026, 1000, 1034, 1038, 1053, 1050, 1040, 1043, 997, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1040, 1019, 1016, 1019, 1029, 1043, 1051, 1023, 1010, 1010, 1057, 1040, 1019, 1027, 1008, 985, 1014, 1027, 985, 1035, 1056, 1048, 1052, 1034, 988, 985, 1031, 1022, 1064, 1012, 1016, 1024, 1048, 1031, 1020, 1035, 1032, 1024, 1032, 1043, 1030, 1013, 1052, 1053, 971, 1049, 1001, 1037, 938, 1072, 992, 1022, 1021, 1010, 1075, 1009, 1082, 1018, 1019, 999, 999, 1010, 1023, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1071, 1030, 1032, 1012, 979, 1047, 1011, 1036, 998, 1047, 1046, 1020, 1015, 1020, 1024, 1007, 1044, 1028, 1051, 1035, 948, 1016, 1015, 971, 1000, 1005, 1018, 1076, 994, 1033, 1032, 1046, 1016, 1006, 1055, 971, 1032, 1075, 1026, 1000, 1041, 1039, 1035, 1053, 1085, 1010, 1028, 1033, 1012, 998, 993, 1037, 1037, 1030, 1011, 1056, 1027, 989, 1056, 1038, 993, 1009, 1017, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[968, 1008, 1027, 1010, 1035, 1022, 1047, 1035, 1012, 997, 1065, 1055, 966, 1076, 969, 1019, 1071, 1048, 1010, 1042, 1042, 1050, 976, 987, 1026, 1029, 999, 1040, 1015, 1056, 1036, 1070, 1037, 1058, 1053, 1023, 1028, 1040, 976, 1022, 1000, 1023, 1012, 1052, 1026, 1081, 1020, 965, 1033, 971, 1002, 1022, 1025, 1060, 1019, 1034, 1035, 1003, 959, 1041, 1031, 1012, 1070, 995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1012, 993, 1027, 1012, 1006, 995, 1037, 1063, 1082, 1013, 1008, 990, 1013, 1016, 1056, 1025, 1030, 1008, 1019, 984, 1042, 985, 1015, 1037, 973, 1024, 1039, 1049, 1038, 989, 1078, 1044, 1046, 1082, 1019, 1049, 1065, 1027, 1054, 1002, 1010, 1039, 1021, 1011, 1006, 1061, 978, 1014, 1005, 1018, 1025, 1037, 974, 979, 981, 1030, 1006, 1036, 1045, 1048, 1101, 1052, 1034, 979] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[993, 991, 1046, 988, 1013, 1053, 1031, 1054, 1017, 968, 1055, 1024, 1011, 1020, 1010, 1012, 1018, 1027, 1007, 1000, 1047, 1009, 987, 1021, 993, 1007, 1045, 1023, 1008, 1014, 1003, 1029, 1009, 1040, 1044, 1062, 1031, 1024, 1006, 1048, 1051, 1062, 1040, 1031, 1023, 1041, 1025, 1003, 1030, 1027, 985, 1038, 1047, 1056, 1043, 1030, 1044, 1067, 1022, 986, 992, 1030, 1034, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1062, 1026, 977, 1070, 995, 1060, 1006, 1020, 967, 1047, 995, 1026, 998, 1012, 971, 1037, 1064, 1044, 1035, 1015, 1026, 978, 1066, 1045, 1017, 986, 969, 1072, 1041, 1017, 1063, 1043, 997, 1055, 1014, 1025, 1042, 1057, 1034, 1050, 1018, 1004, 1013, 1006, 1045, 999, 1023, 1002, 1051, 973, 1018, 1050, 1005, 1042, 1014, 1049, 1078, 1048, 976, 1030, 1012, 1027, 1060, 969] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[989, 962, 997, 1010, 1021, 1027, 940, 1010, 1058, 1065, 1030, 990, 998, 1005, 1017, 1020, 970, 1039, 1007, 1038, 1033, 1013, 1028, 998, 1050, 1029, 1012, 1073, 1033, 1000, 1020, 1052, 957, 995, 1012, 1014, 1010, 1050, 984, 1038, 1044, 1018, 1045, 1082, 1030, 1036, 1065, 1015, 1036, 1029, 1041, 1062, 1042, 1072, 1031, 1052, 1038, 991, 1013, 1010, 1065, 1047, 1061, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1019, 1019, 1043, 999, 1024, 1006, 1029, 1003, 24, 12, 0, 0, 0, 0, 0, 0, 1024, 1019, 1013, 1052, 995, 986, 1032, 1019, 40, 3, 0, 0, 0, 0, 0, 0, 1086, 1054, 1028, 1036, 1072, 1008, 1020, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 998, 1026, 973, 1066, 1015, 1037, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1057, 1042, 1019, 1044, 1004, 963, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1007, 1053, 986, 1028, 976, 1039, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1052, 1027, 959, 957, 1009, 1026, 1000, 13, 3, 3, 0, 0, 0, 0, 0, 1019, 1038, 1053, 1050, 1040, 1043, 997, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8183, 8341, 8189, 8194, 8165, 8028, 8258] avg=8192.0000 max=8341 max_rank=2 min=8028 min_rank=6 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1071, 1030, 1032, 1012, 979, 1047, 1011, 1036, 24, 11, 0, 0, 0, 0, 0, 0, 998, 1047, 1046, 1020, 1015, 1020, 1024, 1007, 41, 3, 0, 0, 0, 0, 0, 0, 1041, 1028, 1051, 1035, 948, 1016, 1015, 971, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1005, 1018, 1062, 994, 1033, 1032, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1006, 1055, 971, 1032, 1034, 1026, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1039, 1035, 1053, 1085, 983, 1028, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 998, 993, 1037, 1037, 1030, 1011, 1056, 14, 3, 3, 0, 0, 0, 0, 0, 1013, 989, 1056, 1038, 993, 1009, 1017, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8221, 8105, 8190, 8140, 8297, 8194, 8136] avg=8192.0000 max=8297 max_rank=5 min=8105 min_rank=2 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1040, 1019, 1016, 1019, 1029, 1043, 1051, 1023, 25, 12, 0, 0, 0, 0, 0, 0, 1010, 1010, 1057, 1040, 1019, 1027, 1008, 985, 39, 3, 0, 0, 0, 0, 0, 0, 1011, 1027, 985, 1035, 1056, 1048, 1052, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 988, 985, 1031, 1008, 1064, 1012, 1016, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1031, 1020, 1035, 1032, 985, 1032, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1013, 1052, 1053, 971, 1021, 1001, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 938, 1072, 992, 1022, 1021, 1010, 1075, 1009, 14, 4, 3, 0, 0, 0, 0, 0, 1066, 1018, 1019, 999, 999, 1010, 1023, 987, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8198, 8248, 8128, 8226, 8178, 8160, 8121] avg=8192.0000 max=8277 max_rank=0 min=8121 min_rank=7 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32685 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[968, 1008, 1027, 1010, 1035, 1022, 1047, 1035, 26, 12, 0, 0, 0, 0, 0, 0, 1012, 997, 1065, 1055, 966, 1076, 969, 1019, 40, 3, 0, 0, 0, 0, 0, 0, 1068, 1048, 1010, 1042, 1042, 1050, 976, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1029, 999, 1026, 1015, 1056, 1036, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1058, 1053, 1023, 1028, 1000, 976, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1023, 1012, 1052, 1026, 1052, 1020, 965, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 971, 1002, 1022, 1025, 1060, 1019, 1034, 14, 3, 3, 0, 0, 0, 0, 0, 1020, 1003, 959, 1041, 1031, 1012, 1070, 995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8202, 8223, 8257, 8197, 8150, 8186, 8131] avg=8192.0000 max=8257 max_rank=3 min=8131 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32664 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[993, 991, 1046, 988, 1013, 1053, 1031, 1054, 25, 12, 0, 0, 0, 0, 0, 0, 1017, 968, 1055, 1024, 1011, 1020, 1010, 1012, 39, 3, 0, 0, 0, 0, 0, 0, 1015, 1027, 1007, 1000, 1047, 1009, 987, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1007, 1045, 1009, 1008, 1014, 1003, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1040, 1044, 1062, 1031, 985, 1006, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1062, 1040, 1031, 1023, 1013, 1025, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1027, 985, 1038, 1047, 1056, 1043, 1030, 14, 4, 3, 0, 0, 0, 0, 0, 1028, 1067, 1022, 986, 992, 1030, 1034, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8159, 8113, 8108, 8225, 8248, 8277, 8200] avg=8192.0000 max=8277 max_rank=6 min=8108 min_rank=3 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32950 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1012, 993, 1027, 1012, 1006, 995, 1037, 1063, 25, 11, 0, 0, 0, 0, 0, 0, 1082, 1013, 1008, 990, 1013, 1016, 1056, 1025, 39, 3, 0, 0, 0, 0, 0, 0, 1027, 1008, 1019, 984, 1042, 985, 1015, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1024, 1039, 1035, 1038, 989, 1078, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1082, 1019, 1049, 1065, 988, 1054, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1039, 1021, 1011, 1006, 1033, 978, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1018, 1025, 1037, 974, 979, 981, 1030, 14, 3, 3, 0, 0, 0, 0, 0, 992, 1036, 1045, 1048, 1101, 1052, 1034, 979, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8245, 8117, 8220, 8305, 8112, 8069, 8287] avg=8192.0000 max=8305 max_rank=4 min=8069 min_rank=6 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[989, 962, 997, 1010, 1021, 1027, 940, 1010, 25, 12, 0, 0, 0, 0, 0, 0, 1058, 1065, 1030, 990, 998, 1005, 1017, 1020, 40, 3, 0, 0, 0, 0, 0, 0, 967, 1039, 1007, 1038, 1033, 1013, 1028, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1029, 1012, 1059, 1033, 1000, 1020, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 957, 995, 1012, 1014, 1010, 1010, 984, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1018, 1045, 1082, 1030, 1008, 1065, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1029, 1041, 1062, 1042, 1072, 1031, 1052, 14, 3, 3, 0, 0, 0, 0, 0, 1023, 991, 1013, 1010, 1065, 1047, 1061, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7993, 8226, 8123, 8255, 8020, 8307, 8385, 8227] avg=8192.0000 max=8385 max_rank=6 min=7993 min_rank=0 imbalance_ratio=1.023560 max_load_reduction=57151 max_load_reduction_pct=0.872055 moved_to_echo=32939 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1062, 1026, 977, 1070, 995, 1060, 1006, 1020, 27, 15, 0, 0, 0, 0, 0, 0, 967, 1047, 995, 1026, 998, 1012, 971, 1037, 45, 4, 0, 0, 0, 0, 0, 0, 1057, 1044, 1035, 1015, 1026, 978, 1066, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 986, 969, 1053, 1041, 1017, 1063, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1055, 1014, 1025, 1042, 1012, 1034, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 1004, 1013, 1006, 1045, 968, 1023, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 973, 1018, 1050, 1005, 1042, 1014, 1049, 19, 9, 7, 0, 0, 0, 0, 0, 1054, 1048, 976, 1030, 1012, 1027, 1060, 969, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8102, 8266, 8189, 8229, 8079, 8237, 8176] avg=8192.0000 max=8266 max_rank=2 min=8079 min_rank=5 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8064, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8320, 8320, 8320, 8064, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8064, 8192, 8192, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8320, 8320, 8192, 8320, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8192, 8192, 8192, 8320, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[995, 1047, 967, 1057, 1008, 1028, 981, 998, 1061, 1060, 1027, 1020, 1012, 1004, 1021, 1051, 1000, 1052, 1035, 1022, 1053, 982, 1036, 1017, 1022, 986, 1008, 1014, 1017, 1001, 972, 966, 1044, 1026, 1027, 1049, 1007, 1030, 984, 1079, 998, 985, 1061, 1048, 1079, 1017, 1059, 1053, 1054, 992, 998, 1024, 1033, 1041, 1047, 1073, 1028, 1082, 1032, 1009, 1001, 1053, 986, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1013, 991, 1046, 1029, 1065, 1041, 1013, 1050, 1009, 1012, 1060, 988, 1069, 1018, 1029, 1018, 1032, 1008, 1008, 1024, 1084, 1004, 1016, 1006, 993, 1081, 1037, 1026, 1003, 1040, 996, 996, 1033, 1038, 1085, 1017, 1004, 986, 1007, 1057, 992, 1062, 1032, 1048, 1072, 984, 1033, 1041, 1033, 1016, 977, 1004, 966, 1010, 986, 1065, 1013, 977, 1017, 1013, 1045, 1010, 1064, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1030, 964, 1064, 1018, 1007, 1051, 1031, 997, 1008, 1079, 1024, 1006, 1015, 993, 999, 1011, 1011, 1035, 1015, 1034, 1033, 1021, 1040, 986, 1022, 1025, 1053, 995, 1024, 1004, 1001, 1045, 1042, 1054, 1008, 1039, 1044, 984, 1020, 1026, 1028, 1000, 1037, 1008, 1057, 990, 1033, 1011, 1024, 1058, 1061, 1040, 1020, 1046, 999, 1043, 1028, 1003, 1072, 1012, 1026, 1031, 1039, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1025, 984, 990, 1032, 1035, 1042, 1011, 1048, 1017, 1014, 1037, 1055, 987, 1048, 996, 1016, 1035, 1036, 983, 1027, 1003, 1019, 1098, 1001, 1067, 993, 981, 1023, 999, 1056, 1014, 1029, 1044, 1013, 1001, 997, 1059, 974, 1081, 1051, 1037, 1004, 1011, 1034, 1022, 1040, 1041, 1007, 1017, 1036, 959, 1082, 1032, 973, 1038, 1055, 1001, 1064, 1015, 1052, 1021, 1002, 1050, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[992, 1044, 994, 1051, 1020, 1021, 1091, 1060, 997, 1034, 1047, 1022, 1010, 1030, 1045, 1043, 1047, 1027, 1010, 1043, 1060, 998, 1032, 1043, 1048, 992, 998, 1061, 994, 1020, 1029, 1069, 1036, 991, 1032, 1062, 1017, 1000, 1034, 984, 1029, 1014, 1033, 1001, 1035, 1019, 1037, 995, 989, 1045, 944, 1023, 1012, 1054, 934, 1038, 1041, 1017, 1022, 1010, 1052, 998, 1036, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1008, 983, 1029, 1035, 1017, 1014, 1025, 992, 1011, 975, 1052, 982, 1029, 982, 983, 1044, 1099, 1035, 1022, 1012, 1083, 1004, 1040, 1000, 993, 981, 1041, 1052, 1060, 1036, 1083, 1057, 1027, 995, 981, 1017, 1033, 995, 1025, 989, 1024, 1027, 1013, 961, 1077, 1017, 1029, 1046, 1046, 1008, 1015, 1013, 1032, 1061, 1019, 1008, 1028, 1024, 1088, 1051, 1024, 1019, 1040, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1016, 1037, 1043, 1033, 1064, 999, 1042, 1009, 1071, 1020, 1070, 1005, 1007, 1044, 1048, 1023, 1050, 1038, 989, 1030, 1048, 989, 1023, 965, 974, 1039, 1025, 1029, 949, 1032, 1043, 1006, 1026, 986, 1026, 1055, 1005, 1034, 989, 992, 1033, 1018, 1001, 1002, 981, 1052, 1054, 1032, 1024, 1064, 1018, 1016, 1046, 1011, 1054, 1055, 1031, 1055, 988, 1011, 1013, 1018, 1071, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1014, 1042, 1020, 986, 985, 1039, 1011, 1081, 1066, 1066, 1046, 1012, 1066, 994, 1030, 1042, 989, 1050, 1062, 980, 1017, 1058, 1029, 1023, 1017, 1043, 1045, 1015, 982, 1034, 1063, 986, 989, 1006, 984, 1007, 1043, 998, 997, 1005, 997, 1085, 1055, 1049, 1024, 1012, 1016, 1048, 1057, 997, 969, 1050, 1043, 1033, 1010, 1029, 1024, 975, 1013, 1044, 1005, 1044, 982, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1016, 1037, 1043, 1033, 1064, 999, 1042, 1009, 10, 0, 0, 0, 0, 0, 0, 0, 1071, 1020, 1055, 1005, 1007, 1044, 1048, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1038, 989, 1030, 1034, 989, 1023, 965, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1039, 1025, 1029, 949, 1032, 1043, 1006, 15, 14, 2, 0, 0, 0, 0, 0, 1026, 986, 1026, 1055, 1005, 1034, 989, 992, 23, 8, 0, 0, 0, 0, 0, 0, 1033, 1018, 1001, 1002, 958, 1052, 1054, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1064, 1018, 1016, 1046, 1011, 1054, 1055, 2, 0, 0, 0, 0, 0, 0, 0, 1031, 1055, 988, 1011, 1013, 1018, 1049, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8273, 8118, 8128, 8144, 8150, 8290, 8180] avg=8192.0000 max=8290 max_rank=6 min=8118 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[992, 1044, 994, 1051, 1020, 1021, 1091, 1060, 10, 0, 0, 0, 0, 0, 0, 0, 997, 1034, 1032, 1022, 1010, 1030, 1045, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1027, 1010, 1043, 1046, 998, 1032, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 992, 998, 1061, 994, 1020, 1029, 1069, 15, 13, 2, 0, 0, 0, 0, 0, 1036, 991, 1032, 1062, 1017, 1000, 1034, 984, 25, 7, 0, 0, 0, 0, 0, 0, 1029, 1014, 1033, 1001, 1010, 1019, 1037, 995, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1045, 944, 1023, 1012, 1054, 934, 1038, 2, 0, 0, 0, 0, 0, 0, 0, 1041, 1017, 1022, 1010, 1052, 998, 1016, 1030, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8213, 8246, 8241, 8188, 8138, 8041, 8186] avg=8192.0000 max=8283 max_rank=0 min=8041 min_rank=6 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1030, 964, 1064, 1018, 1007, 1051, 1031, 997, 9, 0, 0, 0, 0, 0, 0, 0, 1008, 1079, 1009, 1006, 1015, 993, 999, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1035, 1015, 1034, 1020, 1021, 1040, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1025, 1053, 995, 1024, 1004, 1001, 1045, 15, 13, 2, 0, 0, 0, 0, 0, 1042, 1054, 1008, 1039, 1044, 984, 1020, 1026, 25, 7, 0, 0, 0, 0, 0, 0, 1028, 1000, 1037, 1008, 1032, 990, 1033, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1058, 1061, 1040, 1020, 1046, 999, 1043, 2, 0, 0, 0, 0, 0, 0, 0, 1028, 1003, 1072, 1012, 1026, 1031, 1019, 1012, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8120, 8162, 8199, 8249, 8139, 8293, 8203] avg=8192.0000 max=8293 max_rank=6 min=8120 min_rank=1 imbalance_ratio=1.012329 max_load_reduction=57243 max_load_reduction_pct=0.873459 moved_to_echo=32884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[995, 1047, 967, 1057, 1008, 1028, 981, 998, 10, 0, 0, 0, 0, 0, 0, 0, 1061, 1060, 1012, 1020, 1012, 1004, 1021, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1052, 1035, 1022, 1039, 982, 1036, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 986, 1008, 1014, 1017, 1001, 972, 966, 15, 13, 2, 0, 0, 0, 0, 0, 1044, 1026, 1027, 1049, 1007, 1030, 984, 1079, 26, 7, 0, 0, 0, 0, 0, 0, 998, 985, 1061, 1048, 1053, 1017, 1059, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 992, 998, 1024, 1033, 1041, 1047, 1073, 2, 0, 0, 0, 0, 0, 0, 0, 1028, 1082, 1032, 1009, 1001, 1053, 966, 1017, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8091, 8241, 8183, 8016, 8279, 8274, 8264, 8188] avg=8192.0000 max=8279 max_rank=4 min=8016 min_rank=3 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=33005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1025, 984, 990, 1032, 1035, 1042, 1011, 1048, 9, 0, 0, 0, 0, 0, 0, 0, 1017, 1014, 1022, 1055, 987, 1048, 996, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1036, 983, 1027, 990, 1019, 1098, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 993, 981, 1023, 999, 1056, 1014, 1029, 15, 13, 2, 0, 0, 0, 0, 0, 1044, 1013, 1001, 997, 1059, 974, 1081, 1051, 24, 7, 0, 0, 0, 0, 0, 0, 1037, 1004, 1011, 1034, 998, 1040, 1041, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1036, 959, 1082, 1032, 973, 1038, 1055, 2, 0, 0, 0, 0, 0, 0, 0, 1001, 1064, 1015, 1052, 1021, 1002, 1030, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8155, 8189, 8192, 8251, 8172, 8194, 8207] avg=8192.0000 max=8251 max_rank=4 min=8155 min_rank=1 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32824 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1013, 991, 1046, 1029, 1065, 1041, 1013, 1050, 10, 0, 0, 0, 0, 0, 0, 0, 1009, 1012, 1045, 988, 1069, 1018, 1029, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1008, 1008, 1024, 1070, 1004, 1016, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1081, 1037, 1026, 1003, 1040, 996, 996, 15, 14, 2, 0, 0, 0, 0, 0, 1033, 1038, 1085, 1017, 1004, 986, 1007, 1057, 26, 7, 0, 0, 0, 0, 0, 0, 992, 1062, 1032, 1048, 1046, 984, 1033, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1016, 977, 1004, 966, 1010, 986, 1065, 2, 0, 0, 0, 0, 0, 0, 0, 1013, 977, 1017, 1013, 1045, 1010, 1043, 1044, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8188, 8168, 8203, 8260, 8238, 8059, 8162] avg=8192.0000 max=8260 max_rank=4 min=8059 min_rank=6 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32719 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1008, 983, 1029, 1035, 1017, 1014, 1025, 992, 10, 0, 0, 0, 0, 0, 0, 0, 1011, 975, 1037, 982, 1029, 982, 983, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 1035, 1022, 1012, 1069, 1004, 1040, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 993, 981, 1041, 1052, 1060, 1036, 1083, 1057, 15, 13, 2, 0, 0, 0, 0, 0, 1027, 995, 981, 1017, 1033, 995, 1025, 989, 26, 7, 0, 0, 0, 0, 0, 0, 1024, 1027, 1013, 961, 1051, 1017, 1029, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1008, 1015, 1013, 1032, 1061, 1019, 1008, 2, 0, 0, 0, 0, 0, 0, 0, 1028, 1024, 1088, 1051, 1024, 1019, 1020, 1045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8113, 8043, 8281, 8333, 8095, 8168, 8204, 8299] avg=8192.0000 max=8333 max_rank=3 min=8043 min_rank=1 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1014, 1042, 1020, 986, 985, 1039, 1011, 1081, 13, 0, 0, 0, 0, 0, 0, 0, 1066, 1066, 1027, 1012, 1066, 994, 1030, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1050, 1062, 980, 998, 1058, 1029, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1043, 1045, 1015, 982, 1034, 1063, 986, 19, 17, 3, 0, 0, 0, 0, 0, 989, 1006, 984, 1007, 1043, 998, 997, 1005, 29, 12, 0, 0, 0, 0, 0, 0, 997, 1085, 1055, 1049, 995, 1012, 1016, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 997, 969, 1050, 1043, 1033, 1010, 1029, 3, 0, 0, 0, 0, 0, 0, 0, 1024, 975, 1013, 1044, 1005, 1044, 953, 1053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8303, 8189, 8224, 8070, 8257, 8191, 8111] avg=8192.0000 max=8303 max_rank=1 min=8070 min_rank=4 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32629 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8064, 8192, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8320, 8192, 8448, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8064, 8320, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[999, 997, 1041, 1042, 1030, 1023, 1064, 1054, 1052, 1025, 1071, 1047, 1029, 990, 962, 995, 1055, 999, 997, 1051, 1034, 992, 1018, 1100, 1029, 970, 1055, 1033, 1017, 984, 1025, 999, 986, 1005, 1014, 1022, 1007, 1006, 1035, 1083, 991, 1026, 1064, 1052, 997, 1024, 1025, 1087, 1017, 971, 1016, 972, 1103, 1018, 1047, 1051, 1033, 1000, 995, 1078, 1015, 1044, 1007, 966] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1017, 1005, 1014, 977, 1051, 1038, 997, 1054, 1012, 988, 1003, 1031, 1001, 1000, 1014, 1004, 976, 1039, 1057, 1092, 1014, 1029, 982, 1026, 1023, 1043, 1007, 1038, 1063, 974, 1038, 1044, 1041, 1012, 1056, 996, 1057, 1024, 1018, 1068, 1000, 1035, 1019, 1046, 1027, 993, 995, 1011, 1012, 1002, 1010, 1017, 1071, 1065, 976, 1075, 1027, 1079, 1020, 1015, 1047, 1022, 1020, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1023, 1025, 992, 1046, 992, 1051, 1000, 1005, 1088, 1000, 1032, 1063, 961, 967, 1016, 1027, 1046, 996, 1047, 1087, 1004, 1057, 1054, 1012, 1061, 1014, 1047, 1006, 974, 989, 1067, 996, 992, 1032, 1032, 1034, 1048, 1007, 1060, 1022, 1012, 1062, 983, 1032, 988, 1029, 1058, 1016, 1010, 1087, 1036, 1015, 1057, 1057, 1015, 1041, 1012, 991, 1051, 1031, 968, 1005, 1005, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1042, 990, 1005, 1011, 999, 1019, 1052, 1048, 1037, 1034, 1009, 1042, 970, 1033, 1015, 1058, 1052, 1017, 1049, 954, 1053, 998, 1059, 1036, 1059, 1035, 1033, 999, 1047, 1030, 1010, 1010, 1017, 993, 997, 971, 1017, 1012, 1009, 1016, 1045, 1020, 987, 1059, 1026, 1026, 1041, 1015, 1020, 1065, 1023, 1020, 1039, 1022, 1009, 1045, 1003, 1052, 1024, 1009, 1054, 1041, 1024, 1030] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1014, 1048, 1036, 983, 1067, 989, 1028, 1009, 1058, 1057, 1076, 998, 1023, 1065, 992, 1034, 1011, 1004, 983, 1018, 1059, 1037, 1017, 1040, 1011, 1036, 1044, 1012, 1053, 1060, 1009, 1033, 1048, 948, 1018, 1008, 1003, 1039, 971, 1025, 992, 994, 1106, 1041, 1045, 1002, 999, 998, 1000, 1005, 994, 1040, 1044, 1010, 1018, 1036, 1037, 1020, 998, 1003, 1036, 1036, 1054, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1007, 1047, 1028, 1037, 1031, 1020, 1022, 1031, 1002, 1065, 1080, 1021, 1009, 1001, 986, 987, 1008, 1047, 1028, 1003, 1036, 1015, 1009, 1052, 1021, 1003, 998, 1000, 1052, 1047, 1019, 1023, 1046, 1013, 1047, 1022, 1018, 1024, 1001, 992, 1005, 1020, 1036, 1010, 1014, 1043, 1031, 1046, 1020, 1037, 1061, 1042, 1047, 1039, 1016, 1021, 988, 1001, 1030, 1051, 1050, 1008, 1042, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[1018, 973, 1032, 1015, 986, 1052, 1027, 966, 1032, 972, 987, 1036, 1063, 1036, 1052, 1043, 1026, 984, 998, 1042, 1017, 1043, 976, 1045, 1038, 1016, 1043, 1061, 1014, 1015, 1045, 1042, 1044, 1019, 1019, 1033, 980, 1060, 1032, 1033, 1076, 1037, 1010, 1007, 1060, 1009, 1000, 991, 1049, 1039, 1014, 1012, 1031, 1004, 1070, 1022, 1031, 1027, 1003, 1004, 1018, 1024, 1048, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1038, 1044, 1048, 1029, 989, 1027, 1025, 1005, 1011, 1066, 1018, 1042, 1055, 1050, 967, 991, 1005, 1078, 1006, 1069, 988, 1025, 1036, 1039, 999, 1024, 1028, 1024, 1032, 1034, 1063, 1071, 982, 1038, 1020, 991, 1006, 973, 1043, 1017, 1039, 1058, 1011, 1015, 1000, 1045, 1014, 1022, 1026, 1045, 1027, 1006, 1009, 1003, 997, 1005, 1057, 977, 1058, 1052, 1022, 1025, 1008, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[999, 997, 1041, 1042, 1030, 1023, 1064, 1054, 12, 8, 0, 0, 0, 0, 0, 0, 1052, 1025, 1071, 1047, 1029, 990, 962, 995, 10, 0, 0, 0, 0, 0, 0, 0, 1055, 999, 997, 1051, 1034, 992, 1018, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 970, 1055, 1033, 1017, 984, 1007, 999, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1005, 1014, 1022, 1007, 1006, 1035, 1083, 31, 12, 0, 0, 0, 0, 0, 0, 991, 1026, 1064, 1049, 997, 1024, 1025, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 971, 1016, 972, 1072, 1018, 1047, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1000, 995, 1078, 1015, 1044, 1007, 966, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8181, 8222, 8094, 8201, 8263, 8164, 8141] avg=8192.0000 max=8270 max_rank=0 min=8094 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1017, 1005, 1014, 977, 1051, 1038, 997, 1054, 11, 8, 0, 0, 0, 0, 0, 0, 1012, 988, 1003, 1031, 1001, 1000, 1014, 1004, 10, 0, 0, 0, 0, 0, 0, 0, 976, 1039, 1057, 1092, 1014, 1029, 982, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1043, 1007, 1038, 1063, 974, 1020, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1012, 1056, 996, 1057, 1024, 1018, 1068, 30, 11, 0, 0, 0, 0, 0, 0, 1000, 1035, 1019, 1043, 1027, 993, 995, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1002, 1010, 1017, 1041, 1065, 976, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1079, 1020, 1015, 1047, 1022, 1020, 1029, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8063, 8193, 8212, 8313, 8123, 8198, 8262] avg=8192.0000 max=8313 max_rank=4 min=8063 min_rank=1 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1023, 1025, 992, 1046, 992, 1051, 1000, 1005, 11, 8, 0, 0, 0, 0, 0, 0, 1088, 1000, 1032, 1063, 961, 967, 1016, 1027, 10, 0, 0, 0, 0, 0, 0, 0, 1046, 996, 1047, 1087, 1004, 1057, 1054, 990, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1014, 1047, 1006, 974, 989, 1049, 996, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1032, 1032, 1034, 1048, 1007, 1060, 1022, 29, 11, 0, 0, 0, 0, 0, 0, 1012, 1062, 983, 1029, 988, 1029, 1058, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1087, 1036, 1015, 1028, 1057, 1015, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 991, 1051, 1031, 968, 1005, 1005, 1003, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8164, 8281, 8136, 8267, 8177, 8289, 8069] avg=8192.0000 max=8289 max_rank=6 min=8069 min_rank=7 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32802 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1014, 1048, 1036, 983, 1067, 989, 1028, 1009, 11, 8, 0, 0, 0, 0, 0, 0, 1058, 1057, 1076, 998, 1023, 1065, 992, 1034, 10, 0, 0, 0, 0, 0, 0, 0, 1011, 1004, 983, 1018, 1059, 1037, 1017, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1036, 1044, 1012, 1053, 1060, 991, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 948, 1018, 1008, 1003, 1039, 971, 1025, 29, 12, 0, 0, 0, 0, 0, 0, 992, 994, 1106, 1038, 1045, 1002, 999, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1005, 994, 1040, 1015, 1010, 1018, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1020, 998, 1003, 1036, 1036, 1054, 1064, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8313, 8146, 8240, 8101, 8174, 8118, 8251] avg=8192.0000 max=8313 max_rank=1 min=8101 min_rank=4 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1018, 973, 1032, 1015, 986, 1052, 1027, 966, 11, 8, 0, 0, 0, 0, 0, 0, 1032, 972, 987, 1036, 1063, 1036, 1052, 1043, 10, 0, 0, 0, 0, 0, 0, 0, 1026, 984, 998, 1042, 1017, 1043, 976, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1016, 1043, 1061, 1014, 1015, 1027, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1019, 1019, 1033, 980, 1060, 1032, 1033, 29, 12, 0, 0, 0, 0, 0, 0, 1076, 1037, 1010, 1004, 1060, 1009, 1000, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1039, 1014, 1012, 1002, 1004, 1070, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1027, 1003, 1004, 1018, 1024, 1048, 1035, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8088, 8231, 8108, 8256, 8261, 8187, 8212, 8193] avg=8192.0000 max=8261 max_rank=4 min=8088 min_rank=0 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32853 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1007, 1047, 1028, 1037, 1031, 1020, 1022, 1031, 11, 8, 0, 0, 0, 0, 0, 0, 1002, 1065, 1080, 1021, 1009, 1001, 986, 987, 10, 0, 0, 0, 0, 0, 0, 0, 1008, 1047, 1028, 1003, 1036, 1015, 1009, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1003, 998, 1000, 1052, 1047, 1001, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1013, 1047, 1022, 1018, 1024, 1001, 992, 29, 12, 0, 0, 0, 0, 0, 0, 1005, 1020, 1036, 1007, 1014, 1043, 1031, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1037, 1061, 1042, 1018, 1039, 1016, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 988, 1001, 1030, 1051, 1050, 1008, 1042, 980, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8161, 8175, 8145, 8204, 8202, 8254, 8153] avg=8192.0000 max=8254 max_rank=6 min=8145 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1042, 990, 1005, 1011, 999, 1019, 1052, 1048, 11, 8, 0, 0, 0, 0, 0, 0, 1037, 1034, 1009, 1042, 970, 1033, 1015, 1058, 10, 0, 0, 0, 0, 0, 0, 0, 1052, 1017, 1049, 954, 1053, 998, 1059, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1035, 1033, 999, 1047, 1030, 992, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 993, 997, 971, 1017, 1012, 1009, 1016, 29, 12, 0, 0, 0, 0, 0, 0, 1045, 1020, 987, 1056, 1026, 1026, 1041, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1065, 1023, 1020, 1010, 1022, 1009, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1052, 1024, 1009, 1054, 1041, 1024, 1030, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8208, 8195, 8205, 8073, 8216, 8214, 8240] avg=8192.0000 max=8240 max_rank=7 min=8073 min_rank=4 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1038, 1044, 1048, 1029, 989, 1027, 1025, 1005, 15, 13, 0, 0, 0, 0, 0, 0, 1011, 1066, 1018, 1042, 1055, 1050, 967, 991, 14, 1, 0, 0, 0, 0, 0, 0, 1005, 1078, 1006, 1069, 988, 1025, 1036, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1024, 1028, 1024, 1032, 1034, 1036, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1038, 1020, 991, 1006, 973, 1043, 1017, 31, 15, 0, 0, 0, 0, 0, 0, 1039, 1058, 1011, 1005, 1000, 1045, 1014, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1045, 1027, 1006, 978, 1003, 997, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 977, 1058, 1052, 1022, 1025, 1008, 1019, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8215, 8216, 8248, 8116, 8194, 8087, 8227] avg=8192.0000 max=8248 max_rank=3 min=8087 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32624 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8064, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8064, 8320, 8192, 8192, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1007, 1023, 977, 986, 1028, 1017, 1054, 979, 989, 983, 1027, 1051, 1082, 1031, 1002, 1030, 1026, 1033, 1087, 1048, 1032, 998, 1007, 1042, 1003, 989, 1023, 1032, 1058, 1028, 1067, 1060, 1032, 1038, 1009, 1048, 1003, 1016, 1039, 1056, 1045, 1010, 1025, 1016, 999, 985, 1000, 1057, 1013, 1019, 1025, 975, 1042, 1025, 1040, 1023, 1023, 1020, 1009, 1023, 1027, 1026, 1019, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1047, 1083, 994, 1042, 985, 1072, 1042, 1055, 1033, 1075, 1023, 1010, 1040, 988, 1030, 969, 1023, 1089, 1024, 986, 1008, 1062, 1059, 1053, 1059, 1022, 987, 1045, 1006, 1013, 1069, 1091, 1047, 1013, 1014, 1042, 999, 967, 1049, 1016, 1046, 1033, 1014, 1008, 1030, 1010, 986, 1018, 1052, 1035, 993, 973, 997, 1004, 1015, 977, 990, 1015, 1041, 1022, 1046, 954, 1059, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1004, 1036, 1026, 1033, 1010, 1056, 1028, 1001, 1005, 1009, 992, 1104, 1053, 997, 1017, 1062, 970, 1002, 1047, 1017, 1011, 1013, 1008, 1048, 1074, 1006, 1031, 1044, 1042, 1020, 984, 1051, 997, 986, 1023, 1052, 1052, 1028, 1087, 1029, 987, 1010, 1019, 1102, 1080, 1038, 971, 1007, 1027, 1007, 1038, 990, 1042, 1046, 993, 980, 1027, 1025, 984, 1000, 1054, 1009, 1035, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1048, 1005, 1065, 996, 1037, 1051, 1041, 1013, 972, 1042, 1031, 1037, 1004, 1002, 1030, 1020, 984, 1018, 999, 998, 1063, 1070, 1018, 1042, 1061, 982, 1040, 989, 1033, 1009, 1015, 1020, 1031, 1027, 1019, 1015, 1013, 987, 1015, 1037, 1048, 1051, 1046, 1010, 1073, 1045, 1052, 1042, 1050, 969, 1054, 1000, 979, 1041, 1005, 1026, 1002, 1033, 1012, 1022, 1029, 1022, 1041, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1020, 1041, 997, 1059, 1031, 1047, 1061, 981, 1055, 1046, 990, 1018, 1050, 1039, 1015, 1019, 1003, 1017, 1041, 1033, 1017, 1079, 1028, 1007, 1043, 1038, 959, 996, 1009, 1015, 1053, 1063, 1011, 962, 1027, 1002, 997, 990, 1048, 998, 1021, 1060, 1022, 1011, 1019, 1044, 1046, 1040, 1017, 1054, 1016, 1032, 1044, 1020, 1001, 1025, 996, 1032, 1024, 985, 1063, 1027, 996, 1036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1053, 999, 990, 1028, 1033, 1063, 1009, 983, 1041, 1063, 1010, 1056, 1008, 992, 1035, 1033, 972, 1041, 1032, 958, 1036, 1047, 1097, 1035, 1006, 1019, 1013, 1036, 1014, 1003, 1051, 1009, 1032, 999, 1006, 1014, 1052, 1050, 1075, 1037, 987, 1086, 1010, 1006, 1051, 1005, 1023, 1001, 1006, 1027, 1094, 1039, 1004, 979, 1042, 1003, 1037, 1028, 993, 990, 1024, 1025, 993, 1053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[983, 1050, 1036, 1046, 1066, 1020, 992, 1016, 1025, 1049, 1045, 990, 1049, 1029, 1012, 1019, 1063, 1010, 1061, 1018, 1046, 1007, 1034, 990, 987, 1013, 989, 981, 983, 1007, 997, 1010, 1003, 1079, 1049, 987, 1036, 1057, 1050, 1045, 1037, 1026, 1007, 1036, 980, 1012, 1017, 1053, 991, 1004, 1076, 975, 1025, 1036, 1022, 1056, 1037, 1035, 1067, 1049, 1037, 970, 1040, 1019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1036, 1028, 1038, 1069, 1034, 1005, 1033, 1026, 978, 976, 1038, 1000, 1049, 1043, 1049, 1066, 1038, 980, 1005, 1007, 1053, 969, 1031, 1014, 1029, 1023, 1034, 1007, 981, 1049, 1020, 980, 1010, 1012, 1063, 1040, 1003, 1031, 1021, 1009, 1053, 1011, 1033, 1012, 1074, 1048, 1029, 974, 981, 1061, 1057, 1014, 989, 1021, 990, 1023, 1037, 1007, 1044, 1000, 1076, 1063, 1018, 1044] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1007, 1023, 977, 986, 1028, 996, 1054, 979, 0, 0, 0, 0, 0, 0, 0, 0, 989, 983, 1027, 1051, 1071, 1031, 1002, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1033, 1072, 1048, 1032, 998, 1007, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 989, 1023, 1032, 1058, 1028, 1067, 1060, 13, 8, 0, 0, 0, 0, 0, 0, 1032, 1038, 1009, 1048, 1003, 1016, 1034, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1010, 1025, 1016, 980, 985, 1000, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1019, 1025, 975, 1042, 1025, 1040, 1023, 21, 19, 2, 0, 0, 0, 0, 0, 1023, 1020, 1009, 1023, 1027, 1026, 1019, 1050, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8050, 8184, 8258, 8281, 8236, 8118, 8204, 8205] avg=8192.0000 max=8281 max_rank=3 min=8050 min_rank=0 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1047, 1083, 994, 1042, 985, 1050, 1042, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1075, 1023, 1010, 1029, 988, 1030, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1089, 1010, 986, 1008, 1062, 1059, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1022, 987, 1045, 1006, 1013, 1069, 1091, 12, 8, 0, 0, 0, 0, 0, 0, 1047, 1013, 1014, 1042, 999, 967, 1044, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1033, 1014, 1008, 1011, 1010, 986, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1035, 993, 973, 997, 1004, 1015, 977, 22, 19, 2, 0, 0, 0, 0, 0, 990, 1015, 1041, 1022, 1046, 954, 1059, 987, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8298, 8157, 8290, 8312, 8142, 8126, 8089, 8122] avg=8192.0000 max=8312 max_rank=3 min=8089 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1048, 1005, 1065, 996, 1037, 1029, 1041, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1042, 1031, 1037, 994, 1002, 1030, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1018, 985, 998, 1063, 1070, 1018, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 982, 1040, 989, 1033, 1009, 1015, 1020, 12, 7, 0, 0, 0, 0, 0, 0, 1031, 1027, 1019, 1015, 1013, 987, 1010, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1051, 1046, 1010, 1053, 1045, 1052, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 969, 1054, 1000, 979, 1041, 1005, 1026, 22, 20, 2, 0, 0, 0, 0, 0, 1002, 1033, 1012, 1022, 1029, 1022, 1041, 1005, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8128, 8178, 8168, 8139, 8347, 8168, 8174] avg=8192.0000 max=8347 max_rank=5 min=8128 min_rank=1 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1020, 1041, 997, 1059, 1031, 1025, 1061, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1046, 990, 1018, 1039, 1039, 1015, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1017, 1027, 1033, 1017, 1079, 1028, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1038, 959, 996, 1009, 1015, 1053, 1063, 12, 8, 0, 0, 0, 0, 0, 0, 1011, 962, 1027, 1002, 997, 990, 1043, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1060, 1022, 1011, 1000, 1044, 1046, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1054, 1016, 1032, 1044, 1020, 1001, 1025, 22, 19, 2, 0, 0, 0, 0, 0, 996, 1032, 1024, 985, 1063, 1027, 996, 1036, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8221, 8211, 8196, 8030, 8244, 8252, 8167] avg=8192.0000 max=8252 max_rank=6 min=8030 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[983, 1050, 1036, 1046, 1066, 999, 992, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1049, 1045, 990, 1038, 1029, 1012, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1010, 1047, 1018, 1046, 1007, 1034, 990, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1013, 989, 981, 983, 1007, 997, 1010, 12, 8, 0, 0, 0, 0, 0, 0, 1003, 1079, 1049, 987, 1036, 1057, 1045, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1026, 1007, 1036, 962, 1012, 1017, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1004, 1076, 975, 1025, 1036, 1022, 1056, 21, 18, 2, 0, 0, 0, 0, 0, 1037, 1035, 1067, 1049, 1037, 970, 1040, 1019, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8207, 8215, 7987, 8301, 8150, 8226, 8262] avg=8192.0000 max=8301 max_rank=4 min=7987 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32939 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1004, 1036, 1026, 1033, 1010, 1034, 1028, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1009, 992, 1104, 1042, 997, 1017, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1002, 1033, 1017, 1011, 1013, 1008, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1006, 1031, 1044, 1042, 1020, 984, 1051, 12, 8, 0, 0, 0, 0, 0, 0, 997, 986, 1023, 1052, 1052, 1028, 1082, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1010, 1019, 1102, 1060, 1038, 971, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1007, 1038, 990, 1042, 1046, 993, 980, 22, 20, 2, 0, 0, 0, 0, 0, 1027, 1025, 984, 1000, 1054, 1009, 1035, 1010, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8228, 8102, 8272, 8249, 8194, 8167, 8152] avg=8192.0000 max=8272 max_rank=3 min=8102 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1053, 999, 990, 1028, 1033, 1041, 1009, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1063, 1010, 1056, 998, 992, 1035, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1041, 1018, 958, 1036, 1047, 1097, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1019, 1013, 1036, 1014, 1003, 1051, 1009, 12, 7, 0, 0, 0, 0, 0, 0, 1032, 999, 1006, 1014, 1052, 1050, 1070, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1086, 1010, 1006, 1031, 1005, 1023, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1027, 1094, 1039, 1004, 979, 1042, 1003, 22, 20, 2, 0, 0, 0, 0, 0, 1037, 1028, 993, 990, 1024, 1025, 993, 1053, 5, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8228, 8204, 8170, 8260, 8149, 8238, 8151] avg=8192.0000 max=8260 max_rank=4 min=8136 min_rank=0 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1036, 1028, 1038, 1069, 1034, 979, 1033, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 978, 976, 1038, 1000, 1033, 1043, 1049, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 980, 986, 1007, 1053, 969, 1031, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1023, 1034, 1007, 981, 1049, 1020, 980, 15, 12, 0, 0, 0, 0, 0, 0, 1010, 1012, 1063, 1040, 1003, 1031, 1011, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 1011, 1033, 1012, 1048, 1048, 1029, 974, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1061, 1057, 1014, 989, 1021, 990, 1023, 26, 26, 4, 0, 0, 0, 0, 0, 1037, 1007, 1044, 1000, 1076, 1063, 1018, 1044, 10, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8183, 8078, 8150, 8179, 8208, 8192, 8303] avg=8192.0000 max=8303 max_rank=7 min=8078 min_rank=2 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8448, 8064, 8192, 8192, 8192, 8192, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8320, 8192, 8192, 8448, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8192, 8448, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8320, 8192, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[985, 1066, 994, 1031, 1035, 1062, 1016, 1050, 1036, 1063, 975, 1019, 1018, 1043, 1009, 1016, 1056, 1006, 1034, 1011, 1058, 1032, 1015, 1028, 1011, 995, 1067, 1001, 1021, 1017, 1032, 1022, 998, 1027, 1022, 1015, 1041, 1042, 1037, 1042, 997, 1037, 1036, 1021, 1056, 1025, 1030, 996, 1092, 978, 973, 995, 1014, 1049, 996, 1057, 985, 1042, 1020, 1027, 1069, 1022, 958, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1027, 1003, 1022, 1036, 1055, 1027, 993, 1014, 1064, 1030, 1016, 994, 1047, 1013, 971, 1035, 1049, 1020, 1000, 1011, 1004, 996, 1059, 1027, 1026, 1005, 1045, 1022, 1042, 1008, 1000, 1021, 1055, 1030, 1030, 1054, 984, 1058, 1003, 1011, 996, 1039, 1006, 1057, 1001, 992, 1037, 1040, 1040, 1045, 1056, 1060, 1035, 988, 1016, 1039, 985, 996, 1077, 1028, 1009, 1030, 1016, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[993, 1032, 967, 1017, 1042, 1053, 1062, 1031, 1029, 1044, 1065, 1051, 1043, 1006, 1007, 993, 998, 1002, 1018, 980, 1026, 1066, 1025, 1044, 1018, 1031, 1010, 1038, 1018, 942, 1023, 1018, 1036, 963, 1003, 1022, 1064, 1020, 1011, 1036, 1047, 992, 1067, 1051, 981, 1010, 1051, 1016, 1039, 1020, 1032, 1027, 986, 1053, 1007, 1015, 1037, 999, 1052, 1067, 1057, 1055, 993, 1035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1023, 1010, 1007, 1010, 1033, 1039, 1016, 1003, 1017, 1019, 1047, 991, 996, 1016, 1059, 1052, 1009, 997, 1035, 1032, 1052, 1029, 962, 993, 1053, 988, 1023, 1028, 1062, 1017, 1001, 1050, 1012, 1023, 1007, 1051, 1073, 1068, 981, 1009, 1043, 1033, 985, 1016, 1005, 1028, 986, 1029, 1021, 1048, 1047, 1043, 966, 1061, 1027, 1010, 1023, 1012, 1050, 1054, 1037, 990, 1050, 1079] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1035, 954, 1042, 960, 1068, 1002, 1054, 1033, 1001, 1003, 1026, 1037, 995, 974, 1009, 1048, 1013, 1034, 1002, 1055, 978, 1059, 1060, 1030, 986, 1017, 992, 1033, 1065, 1018, 1045, 965, 1013, 1092, 1048, 1048, 992, 1051, 1013, 984, 1023, 1050, 1016, 1014, 993, 1038, 998, 1067, 1017, 988, 1041, 1020, 1034, 1031, 1058, 1004, 1008, 1024, 1104, 1062, 1017, 1050, 1008, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1041, 1042, 1053, 1028, 1001, 1022, 1067, 1043, 994, 1034, 1093, 1032, 1019, 1010, 1011, 1060, 993, 1027, 957, 1011, 1045, 984, 1037, 1019, 981, 995, 1020, 1030, 1044, 1032, 1038, 1038, 948, 1011, 1048, 1058, 999, 1021, 1027, 1042, 1025, 1028, 982, 1047, 1068, 1033, 1009, 1023, 1022, 1011, 1034, 1054, 1009, 1010, 993, 1023, 1016, 1008, 1010, 984, 1040, 1008, 1071, 1073] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1003, 1001, 996, 1035, 1003, 979, 1060, 1006, 1005, 1022, 1000, 1034, 1012, 1031, 1080, 1036, 1075, 1032, 1050, 989, 1009, 1025, 1051, 966, 1062, 1004, 1030, 1058, 1058, 1005, 1027, 1009, 1043, 1012, 1050, 1039, 1036, 1007, 1021, 1034, 1012, 1063, 1000, 1004, 1009, 1043, 1054, 978, 1043, 1016, 1008, 1002, 1004, 1010, 1028, 1000, 1042, 974, 1038, 1089, 996, 997, 1045, 1086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[997, 1024, 1082, 1013, 1009, 1018, 970, 1030, 1033, 1024, 1035, 1014, 1024, 1061, 993, 1019, 1025, 1043, 965, 1037, 1079, 997, 1012, 1019, 990, 1049, 1049, 1016, 1047, 1000, 1004, 992, 1051, 962, 987, 980, 1024, 1006, 1028, 1038, 1025, 1028, 1066, 1084, 1056, 1010, 1028, 1042, 1025, 990, 996, 1041, 1026, 1024, 1081, 1009, 1028, 1058, 1073, 1008, 996, 1018, 1020, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[985, 1066, 994, 1031, 1035, 1062, 1016, 1050, 7, 6, 0, 0, 0, 0, 0, 0, 1036, 1063, 975, 1019, 1018, 1043, 1009, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1006, 1034, 1011, 1058, 1032, 1015, 1028, 22, 0, 0, 0, 0, 0, 0, 0, 1011, 995, 1067, 1001, 1021, 1017, 1032, 1022, 22, 0, 0, 0, 0, 0, 0, 0, 998, 1027, 1022, 1015, 1041, 1042, 1037, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1037, 1036, 1011, 1056, 1025, 1030, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 978, 973, 995, 1014, 1049, 996, 1057, 3, 2, 0, 0, 0, 0, 0, 0, 985, 1042, 970, 1027, 1069, 1022, 958, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8177, 8262, 8188, 8224, 8188, 8159, 8086] avg=8192.0000 max=8262 max_rank=2 min=8086 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1023, 1010, 1007, 1010, 1033, 1039, 1016, 1003, 7, 6, 0, 0, 0, 0, 0, 0, 1017, 1019, 1047, 991, 996, 1016, 1059, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 997, 1035, 1032, 1052, 1029, 962, 993, 22, 0, 0, 0, 0, 0, 0, 0, 1053, 988, 1023, 1028, 1062, 1017, 1001, 1050, 22, 0, 0, 0, 0, 0, 0, 0, 1012, 1023, 1007, 1051, 1073, 1068, 981, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1033, 985, 1006, 1005, 1028, 986, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1048, 1047, 1043, 966, 1061, 1027, 1010, 3, 2, 0, 0, 0, 0, 0, 0, 1023, 1012, 1000, 1054, 1037, 990, 1050, 1079, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8195, 8131, 8244, 8224, 8115, 8228, 8245] avg=8192.0000 max=8245 max_rank=7 min=8115 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32812 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1027, 1003, 1022, 1036, 1055, 1027, 993, 1014, 7, 6, 0, 0, 0, 0, 0, 0, 1064, 1030, 1016, 994, 1047, 1013, 971, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1020, 1000, 1011, 1004, 996, 1059, 1027, 23, 0, 0, 0, 0, 0, 0, 0, 1026, 1005, 1045, 1022, 1042, 1008, 1000, 1021, 23, 0, 0, 0, 0, 0, 0, 0, 1055, 1030, 1030, 1054, 984, 1058, 1003, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1039, 1006, 1047, 1001, 992, 1037, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1045, 1056, 1060, 1035, 988, 1016, 1039, 3, 2, 0, 0, 0, 0, 0, 0, 985, 996, 1025, 1028, 1009, 1030, 1016, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8168, 8189, 8192, 8225, 8158, 8284, 8130] avg=8192.0000 max=8284 max_rank=6 min=8130 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[993, 1032, 967, 1017, 1042, 1053, 1062, 1031, 7, 6, 0, 0, 0, 0, 0, 0, 1029, 1044, 1065, 1051, 1043, 1006, 1007, 991, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1002, 1018, 980, 1026, 1066, 1025, 1044, 22, 0, 0, 0, 0, 0, 0, 0, 1018, 1031, 1010, 1038, 1018, 942, 1023, 1018, 22, 0, 0, 0, 0, 0, 0, 0, 1036, 963, 1003, 1022, 1064, 1020, 1011, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 992, 1067, 1041, 981, 1010, 1051, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1020, 1032, 1027, 986, 1053, 1007, 1015, 3, 2, 0, 0, 0, 0, 0, 0, 1037, 999, 1002, 1067, 1057, 1055, 993, 1035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8236, 8181, 8120, 8155, 8205, 8184, 8245] avg=8192.0000 max=8245 max_rank=7 min=8120 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1041, 1042, 1053, 1028, 1001, 1022, 1067, 1043, 7, 6, 0, 0, 0, 0, 0, 0, 994, 1034, 1093, 1032, 1019, 1010, 1011, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1027, 957, 1011, 1045, 984, 1037, 1019, 22, 0, 0, 0, 0, 0, 0, 0, 981, 995, 1020, 1030, 1044, 1032, 1038, 1038, 21, 0, 0, 0, 0, 0, 0, 0, 948, 1011, 1048, 1058, 999, 1021, 1027, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1028, 982, 1037, 1068, 1033, 1009, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1011, 1034, 1054, 1009, 1010, 993, 1023, 3, 2, 0, 0, 0, 0, 0, 0, 1016, 1008, 961, 984, 1040, 1008, 1071, 1073, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8251, 8095, 8199, 8154, 8205, 8161, 8161] avg=8192.0000 max=8310 max_rank=0 min=8095 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[997, 1024, 1082, 1013, 1009, 1018, 970, 1030, 7, 6, 0, 0, 0, 0, 0, 0, 1033, 1024, 1035, 1014, 1024, 1061, 993, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1043, 965, 1037, 1079, 997, 1012, 1019, 23, 0, 0, 0, 0, 0, 0, 0, 990, 1049, 1049, 1016, 1047, 1000, 1004, 992, 23, 0, 0, 0, 0, 0, 0, 0, 1051, 962, 987, 980, 1024, 1006, 1028, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1028, 1066, 1074, 1056, 1010, 1028, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 990, 996, 1041, 1026, 1024, 1081, 1009, 3, 2, 0, 0, 0, 0, 0, 0, 1028, 1058, 1021, 1008, 996, 1018, 1020, 1058, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8201, 8200, 8170, 8076, 8329, 8197, 8207] avg=8192.0000 max=8329 max_rank=5 min=8076 min_rank=4 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1035, 954, 1042, 960, 1068, 1002, 1054, 1033, 10, 10, 0, 0, 0, 0, 0, 0, 1001, 1003, 1026, 1037, 995, 974, 1009, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1034, 1002, 1055, 978, 1059, 1060, 1030, 28, 0, 0, 0, 0, 0, 0, 0, 986, 1017, 992, 1033, 1065, 1018, 1045, 965, 27, 0, 0, 0, 0, 0, 0, 0, 1013, 1092, 1048, 1048, 992, 1046, 1013, 984, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1050, 1016, 998, 993, 1038, 998, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 988, 1041, 1020, 1034, 1031, 1058, 1004, 6, 3, 5, 0, 0, 0, 0, 0, 1008, 1024, 1039, 1062, 1017, 1050, 1008, 1037, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8090, 8259, 8148, 8236, 8183, 8207, 8245] avg=8192.0000 max=8259 max_rank=2 min=8090 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1003, 1001, 996, 1035, 1003, 979, 1060, 1006, 7, 6, 0, 0, 0, 0, 0, 0, 1005, 1022, 1000, 1034, 1012, 1031, 1080, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1032, 1050, 989, 1009, 1025, 1051, 966, 22, 0, 0, 0, 0, 0, 0, 0, 1062, 1004, 1030, 1058, 1058, 1005, 1027, 1009, 22, 0, 0, 0, 0, 0, 0, 0, 1043, 1012, 1050, 1039, 1036, 1007, 1021, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1063, 1000, 994, 1009, 1043, 1054, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1016, 1008, 1002, 1004, 1010, 1028, 1000, 3, 2, 0, 0, 0, 0, 0, 0, 1042, 974, 988, 1089, 996, 997, 1045, 1086, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8096, 8218, 8219, 8275, 8242, 8153, 8116, 8217] avg=8192.0000 max=8275 max_rank=3 min=8096 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8064, 8192, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8064, 8320, 8320, 8192, 8192, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8320, 8192, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8448, 8064, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[993, 1028, 1020, 1004, 1042, 1005, 1041, 1020, 1021, 1037, 1017, 961, 1017, 1026, 1055, 1023, 973, 1063, 1043, 1007, 1068, 988, 1045, 986, 1007, 1068, 1050, 1011, 1021, 1045, 1033, 997, 1111, 1041, 950, 1001, 1023, 918, 993, 1053, 1025, 967, 1023, 1009, 995, 1014, 999, 1066, 1023, 1048, 1084, 999, 1063, 950, 1042, 1020, 1081, 1040, 1100, 994, 1035, 1041, 1064, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[996, 1036, 980, 1039, 1003, 1016, 1036, 1093, 1009, 1012, 1021, 1032, 1014, 1013, 1065, 996, 1026, 1018, 994, 1010, 1039, 1042, 1039, 1061, 1013, 963, 1016, 1002, 1036, 1059, 1053, 999, 989, 1062, 1019, 1018, 1034, 1048, 1004, 1038, 1002, 983, 1053, 1018, 1007, 1034, 1028, 1034, 993, 1013, 1035, 1001, 1062, 1018, 1014, 1024, 1008, 999, 1011, 1064, 1015, 1080, 1016, 1081] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1041, 974, 1007, 1031, 1026, 992, 1010, 1055, 1089, 1049, 989, 1041, 1059, 1082, 1091, 1056, 1044, 1038, 958, 1065, 1031, 1013, 1036, 954, 1037, 1032, 969, 1008, 1032, 1030, 1056, 1052, 1035, 1061, 950, 1044, 1010, 991, 1049, 1009, 1013, 1028, 1033, 1023, 1022, 996, 1014, 1008, 1051, 959, 968, 1037, 1025, 1017, 1010, 990, 990, 1071, 1063, 1043, 1027, 1054, 1010, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1031, 1007, 1007, 1049, 982, 978, 1078, 1018, 990, 1033, 1029, 1017, 1018, 1026, 1003, 1050, 1002, 1008, 1013, 1002, 998, 991, 1017, 973, 1009, 1056, 1042, 1000, 1013, 1061, 1037, 1007, 1034, 1038, 1047, 1079, 1040, 1014, 996, 1022, 1066, 1026, 1062, 994, 990, 1052, 1034, 1044, 1049, 1068, 1059, 1040, 1053, 983, 954, 1047, 1048, 1058, 1033, 1029, 992, 1008, 983, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1039, 1067, 1022, 1008, 1012, 1027, 988, 1020, 1030, 1027, 1028, 1028, 1074, 1039, 1009, 1022, 937, 1017, 1027, 1065, 1079, 997, 979, 1081, 1000, 1042, 1031, 1033, 1028, 1021, 1035, 1056, 976, 992, 1050, 1028, 981, 1063, 1037, 1009, 1034, 1027, 998, 1021, 1032, 1048, 1056, 1034, 1000, 971, 1022, 1032, 969, 1013, 1048, 1077, 1025, 1046, 986, 1028, 1022, 1028, 997, 1018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1056, 1032, 1032, 1026, 1023, 1047, 994, 1042, 1025, 1015, 1018, 1026, 1052, 970, 1055, 1007, 1004, 972, 1022, 1010, 1018, 999, 1024, 1036, 1073, 986, 1019, 975, 1006, 1090, 1000, 984, 1029, 1037, 1022, 1018, 1025, 1036, 1034, 1039, 977, 1001, 1036, 1025, 1033, 1010, 1017, 1051, 1042, 1017, 1056, 1071, 998, 1044, 1012, 1063, 1055, 1060, 1033, 986, 997, 1006, 1043, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[986, 1002, 1022, 998, 964, 1076, 1054, 1054, 1011, 997, 1042, 1005, 1008, 1046, 1003, 1016, 1057, 1019, 1077, 1057, 1014, 987, 1031, 1041, 1002, 1043, 1010, 1007, 1025, 1015, 984, 1026, 1046, 1032, 970, 1023, 987, 1030, 1029, 977, 1009, 1056, 1000, 1019, 1079, 992, 1069, 1041, 992, 1040, 1015, 1034, 1068, 1011, 998, 1027, 1025, 1018, 1049, 1044, 1070, 1051, 1024, 1032] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1073, 1031, 996, 1012, 1048, 1052, 1019, 1060, 987, 994, 1016, 1020, 994, 989, 1016, 1066, 1031, 1033, 987, 1022, 1008, 1035, 1030, 1047, 1054, 1036, 1026, 995, 1017, 978, 1053, 1010, 1035, 1026, 1040, 991, 1016, 1026, 986, 1042, 1070, 1032, 999, 1000, 1016, 1054, 1011, 1016, 1059, 1024, 1012, 947, 983, 1069, 1067, 1014, 1013, 1017, 1075, 1047, 1056, 1014, 1023, 1021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[993, 1028, 1020, 1004, 1042, 1005, 1041, 1020, 2, 0, 0, 0, 0, 0, 0, 0, 1021, 1037, 1017, 961, 1017, 1026, 1051, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1063, 1043, 1007, 1068, 988, 1045, 986, 32, 0, 0, 0, 0, 0, 0, 0, 1007, 1068, 1050, 1011, 1021, 1045, 1033, 997, 8, 0, 0, 0, 0, 0, 0, 0, 1111, 1041, 950, 1001, 1023, 918, 993, 1053, 24, 0, 0, 0, 0, 0, 0, 0, 1025, 967, 1023, 1009, 995, 1014, 999, 1066, 1, 0, 0, 0, 0, 0, 0, 0, 1023, 1048, 1084, 999, 1063, 950, 1042, 1020, 3, 1, 0, 0, 0, 0, 0, 0, 1081, 1040, 1033, 994, 1035, 1041, 1064, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8153, 8205, 8240, 8114, 8099, 8233, 8337] avg=8192.0000 max=8337 max_rank=7 min=8099 min_rank=5 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32783 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1039, 1067, 1022, 1008, 1012, 1027, 988, 1020, 1, 0, 0, 0, 0, 0, 0, 0, 1030, 1027, 1028, 1028, 1074, 1039, 1006, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 937, 1017, 1027, 1065, 1079, 997, 979, 1081, 29, 0, 0, 0, 0, 0, 0, 0, 1000, 1042, 1031, 1033, 1028, 1021, 1035, 1056, 7, 0, 0, 0, 0, 0, 0, 0, 976, 992, 1050, 1028, 981, 1063, 1037, 1009, 21, 0, 0, 0, 0, 0, 0, 0, 1034, 1027, 998, 1021, 1032, 1048, 1056, 1034, 1, 0, 0, 0, 0, 0, 0, 0, 1000, 971, 1022, 1032, 969, 1013, 1048, 1077, 3, 1, 0, 0, 0, 0, 0, 0, 1025, 1046, 926, 1028, 1022, 1028, 997, 1018, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8254, 8211, 8253, 8157, 8251, 8136, 8090] avg=8192.0000 max=8254 max_rank=1 min=8090 min_rank=7 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32634 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1041, 974, 1007, 1031, 1026, 992, 1010, 1055, 2, 0, 0, 0, 0, 0, 0, 0, 1089, 1049, 989, 1041, 1059, 1082, 1087, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1038, 958, 1065, 1031, 1013, 1036, 954, 31, 0, 0, 0, 0, 0, 0, 0, 1037, 1032, 969, 1008, 1032, 1030, 1056, 1052, 7, 0, 0, 0, 0, 0, 0, 0, 1035, 1061, 950, 1044, 1010, 991, 1049, 1009, 23, 0, 0, 0, 0, 0, 0, 0, 1013, 1028, 1033, 1023, 1022, 996, 1014, 1008, 1, 0, 0, 0, 0, 0, 0, 0, 1051, 959, 968, 1037, 1025, 1017, 1010, 990, 3, 1, 0, 0, 0, 0, 0, 0, 990, 1071, 999, 1043, 1027, 1054, 1010, 988, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8452, 8170, 8223, 8172, 8138, 8061, 8182] avg=8192.0000 max=8452 max_rank=1 min=8061 min_rank=6 imbalance_ratio=1.031738 max_load_reduction=57084 max_load_reduction_pct=0.871033 moved_to_echo=32553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[986, 1002, 1022, 998, 964, 1076, 1054, 1054, 1, 0, 0, 0, 0, 0, 0, 0, 1011, 997, 1042, 1005, 1008, 1046, 1000, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 1019, 1077, 1057, 1014, 987, 1031, 1041, 31, 0, 0, 0, 0, 0, 0, 0, 1002, 1043, 1010, 1007, 1025, 1015, 984, 1026, 7, 0, 0, 0, 0, 0, 0, 0, 1046, 1032, 970, 1023, 987, 1030, 1029, 977, 22, 0, 0, 0, 0, 0, 0, 0, 1009, 1056, 1000, 1019, 1079, 992, 1069, 1041, 1, 0, 0, 0, 0, 0, 0, 0, 992, 1040, 1015, 1034, 1068, 1011, 998, 1027, 3, 1, 0, 0, 0, 0, 0, 0, 1025, 1018, 986, 1044, 1070, 1051, 1024, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8125, 8314, 8119, 8116, 8266, 8189, 8250] avg=8192.0000 max=8314 max_rank=2 min=8116 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[996, 1036, 980, 1039, 1003, 1016, 1036, 1093, 2, 0, 0, 0, 0, 0, 0, 0, 1009, 1012, 1021, 1032, 1014, 1013, 1061, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1018, 994, 1010, 1039, 1042, 1039, 1061, 30, 0, 0, 0, 0, 0, 0, 0, 1013, 963, 1016, 1002, 1036, 1059, 1053, 999, 7, 0, 0, 0, 0, 0, 0, 0, 989, 1062, 1019, 1018, 1034, 1048, 1004, 1038, 22, 0, 0, 0, 0, 0, 0, 0, 1002, 983, 1053, 1018, 1007, 1034, 1028, 1034, 1, 0, 0, 0, 0, 0, 0, 0, 993, 1013, 1035, 1001, 1062, 1018, 1014, 1024, 3, 1, 0, 0, 0, 0, 0, 0, 1008, 999, 949, 1064, 1015, 1080, 1016, 1081, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8158, 8259, 8148, 8234, 8160, 8164, 8212] avg=8192.0000 max=8259 max_rank=2 min=8148 min_rank=3 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1031, 1007, 1007, 1049, 982, 978, 1078, 1018, 1, 0, 0, 0, 0, 0, 0, 0, 990, 1033, 1029, 1017, 1018, 1026, 1000, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1008, 1013, 1002, 998, 991, 1017, 973, 30, 0, 0, 0, 0, 0, 0, 0, 1009, 1056, 1042, 1000, 1013, 1061, 1037, 1007, 7, 0, 0, 0, 0, 0, 0, 0, 1034, 1038, 1047, 1079, 1040, 1014, 996, 1022, 22, 0, 0, 0, 0, 0, 0, 0, 1066, 1026, 1062, 994, 990, 1052, 1034, 1044, 1, 0, 0, 0, 0, 0, 0, 0, 1049, 1068, 1059, 1040, 1053, 983, 954, 1047, 3, 1, 0, 0, 0, 0, 0, 0, 1048, 1058, 971, 1029, 992, 1008, 983, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8163, 8034, 8232, 8292, 8269, 8257, 8138] avg=8192.0000 max=8292 max_rank=4 min=8034 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32956 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1056, 1032, 1032, 1026, 1023, 1047, 994, 1042, 2, 0, 0, 0, 0, 0, 0, 0, 1025, 1015, 1018, 1026, 1052, 970, 1051, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 972, 1022, 1010, 1018, 999, 1024, 1036, 30, 0, 0, 0, 0, 0, 0, 0, 1073, 986, 1019, 975, 1006, 1090, 1000, 984, 7, 0, 0, 0, 0, 0, 0, 0, 1029, 1037, 1022, 1018, 1025, 1036, 1034, 1039, 22, 0, 0, 0, 0, 0, 0, 0, 977, 1001, 1036, 1025, 1033, 1010, 1017, 1051, 1, 0, 0, 0, 0, 0, 0, 0, 1042, 1017, 1056, 1071, 998, 1044, 1012, 1063, 3, 1, 0, 0, 0, 0, 0, 0, 1055, 1060, 971, 986, 997, 1006, 1043, 1025, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8164, 8115, 8140, 8262, 8151, 8307, 8143] avg=8192.0000 max=8307 max_rank=6 min=8115 min_rank=2 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32863 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1073, 1031, 996, 1012, 1048, 1052, 1019, 1060, 5, 0, 0, 0, 0, 0, 0, 0, 987, 994, 1016, 1020, 994, 989, 1001, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 1033, 987, 1022, 1008, 1035, 1030, 1047, 35, 0, 0, 0, 0, 0, 0, 0, 1054, 1036, 1026, 995, 1017, 978, 1053, 1010, 12, 0, 0, 0, 0, 0, 0, 0, 1035, 1026, 1040, 991, 1016, 1026, 986, 1042, 27, 0, 0, 0, 0, 0, 0, 0, 1070, 1032, 999, 1000, 1016, 1054, 1011, 1016, 4, 0, 0, 0, 0, 0, 0, 0, 1059, 1024, 1012, 947, 983, 1069, 1067, 1014, 8, 6, 0, 0, 0, 0, 0, 0, 1013, 1017, 993, 1047, 1056, 1014, 1023, 1021, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8067, 8228, 8181, 8189, 8202, 8189, 8184] avg=8192.0000 max=8296 max_rank=0 min=8067 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32764 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8320, 8448, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 7936, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8064, 8320, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8064, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8320, 8192, 8192, 8320, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8064, 8192, 8320, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1017, 1009, 1006, 1023, 1033, 1039, 1067, 1016, 965, 1057, 1059, 1034, 1010, 1016, 974, 1029, 984, 1066, 1041, 1034, 974, 1008, 1047, 1032, 981, 1027, 1045, 1043, 1064, 1003, 1040, 1004, 1016, 1053, 1033, 1045, 1043, 1066, 1042, 1025, 1050, 982, 1101, 993, 998, 1025, 1017, 1018, 1053, 993, 1023, 1010, 1041, 1033, 1034, 965, 1013, 1073, 1032, 967, 1025, 992, 1038, 990] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1035, 986, 1040, 1024, 1000, 1060, 1039, 1044, 974, 1006, 1009, 980, 1044, 1069, 1005, 1000, 1049, 1026, 1007, 997, 1033, 1013, 1021, 1003, 1011, 1033, 1043, 1018, 974, 1021, 1019, 1024, 1049, 1033, 1006, 1001, 1057, 1009, 1002, 1012, 1064, 1028, 1042, 1024, 1008, 1039, 1018, 1016, 1055, 1040, 1045, 1038, 983, 1040, 1006, 1055, 1042, 1009, 1016, 1034, 1038, 1043, 1018, 1059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1005, 1045, 1025, 1026, 1017, 1077, 1054, 1012, 1023, 1034, 1033, 1060, 1024, 1022, 1051, 1059, 1079, 1056, 1005, 979, 1015, 1009, 1015, 995, 972, 1057, 1065, 1071, 1044, 1022, 1040, 970, 1045, 1065, 1035, 1004, 1009, 1011, 1003, 1049, 980, 1023, 1014, 980, 1025, 1033, 1024, 1045, 1023, 1027, 1013, 1021, 1032, 950, 963, 1025, 1111, 1022, 966, 1014, 975, 1018, 1053, 1017] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1030, 1041, 1047, 1038, 1022, 1035, 1050, 1001, 1037, 1047, 1039, 1066, 992, 969, 1026, 1002, 1057, 1014, 1000, 1078, 1019, 1022, 1020, 1019, 992, 985, 1006, 1053, 1007, 1061, 1050, 1064, 1019, 1052, 1003, 1028, 991, 1040, 1042, 1040, 996, 1003, 1004, 1041, 1014, 997, 1008, 985, 995, 1016, 1004, 1006, 1037, 1014, 1018, 1001, 1018, 1082, 1026, 1040, 1070, 1033, 1031, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1064, 1043, 1053, 1031, 991, 1020, 1021, 994, 948, 1010, 1001, 1037, 1049, 1052, 1031, 972, 1032, 1030, 1035, 1042, 1027, 1038, 1020, 1003, 1047, 1025, 1015, 1024, 1015, 945, 1028, 1052, 1063, 1026, 1012, 1008, 1010, 971, 1014, 1057, 1025, 1010, 969, 1039, 1060, 1028, 1031, 1073, 1056, 1010, 1081, 1038, 1027, 999, 1051, 1008, 1002, 1036, 997, 1025, 1010, 1027, 1013, 1065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1052, 1014, 1102, 984, 1030, 1022, 1042, 1048, 1061, 1032, 1005, 1035, 1037, 1043, 1050, 1021, 973, 1042, 1062, 1040, 1004, 1035, 999, 1014, 1045, 1039, 1066, 998, 1030, 995, 1019, 1021, 989, 1028, 997, 1000, 1019, 1063, 1015, 1060, 1034, 1016, 1093, 985, 1015, 980, 1029, 972, 1038, 1025, 1008, 988, 1016, 981, 1044, 1014, 999, 1053, 1032, 1027, 1045, 1024, 1005, 982] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1033, 1009, 978, 1014, 1045, 1002, 1042, 1029, 1031, 997, 1023, 1042, 990, 1011, 1027, 1006, 1040, 1024, 1072, 1001, 1024, 978, 1029, 999, 1045, 994, 981, 990, 1119, 1035, 1071, 1013, 1029, 1038, 1024, 1029, 1059, 1043, 1009, 988, 1043, 975, 999, 1012, 1039, 1001, 1054, 1026, 1078, 1022, 1020, 999, 1037, 1063, 1047, 1041, 1052, 1006, 982, 1031, 1036, 1034, 978, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1022, 1028, 1072, 990, 1047, 1052, 1007, 1033, 1010, 1042, 1029, 1018, 1029, 1011, 1054, 1016, 1030, 1008, 1066, 978, 1014, 969, 1032, 1027, 1033, 1038, 1011, 1044, 1016, 1031, 991, 1039, 1048, 999, 1058, 1045, 987, 996, 1033, 1012, 1006, 1008, 1050, 1036, 1041, 1020, 1065, 999, 1031, 1033, 1040, 1017, 1016, 1045, 997, 1047, 1005, 1020, 1051, 980, 996, 1011, 1054, 1003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1005, 1045, 984, 1026, 1017, 1077, 1054, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1034, 1033, 1060, 1024, 1022, 1051, 1059, 12, 0, 0, 0, 0, 0, 0, 0, 1079, 1056, 1005, 979, 1015, 1009, 1015, 995, 16, 0, 0, 0, 0, 0, 0, 0, 972, 1057, 1065, 1071, 1034, 1022, 1040, 970, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1051, 1035, 1004, 1009, 1011, 1003, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1023, 1014, 980, 1025, 1033, 1024, 1045, 25, 0, 0, 0, 0, 0, 0, 0, 1023, 1027, 1013, 1021, 1032, 950, 963, 1025, 4, 2, 0, 0, 0, 0, 0, 0, 1111, 1022, 966, 1014, 975, 1018, 1053, 1017, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8318, 8169, 8231, 8207, 8149, 8060, 8182] avg=8192.0000 max=8318 max_rank=1 min=8060 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32598 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1017, 1009, 966, 1023, 1033, 1039, 1067, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 965, 1057, 1059, 1034, 1010, 1016, 974, 1029, 12, 0, 0, 0, 0, 0, 0, 0, 984, 1066, 1041, 1034, 974, 1008, 1047, 1032, 15, 0, 0, 0, 0, 0, 0, 0, 981, 1027, 1045, 1043, 1053, 1003, 1040, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1039, 1033, 1045, 1043, 1066, 1042, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 982, 1101, 993, 998, 1025, 1017, 1018, 25, 0, 0, 0, 0, 0, 0, 0, 1053, 993, 1023, 1010, 1041, 1033, 1034, 965, 5, 2, 0, 0, 0, 0, 0, 0, 1013, 1073, 1032, 967, 1025, 992, 1038, 990, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8156, 8201, 8196, 8309, 8209, 8159, 8136] avg=8192.0000 max=8309 max_rank=4 min=8136 min_rank=7 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32813 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1030, 1041, 1005, 1038, 1022, 1035, 1050, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1047, 1039, 1066, 992, 969, 1026, 1002, 12, 0, 0, 0, 0, 0, 0, 0, 1057, 1014, 1000, 1078, 1019, 1022, 1020, 1019, 16, 0, 0, 0, 0, 0, 0, 0, 992, 985, 1006, 1053, 998, 1061, 1050, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1038, 1003, 1028, 991, 1040, 1042, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 996, 1003, 1004, 1041, 1014, 997, 1008, 985, 26, 0, 0, 0, 0, 0, 0, 0, 995, 1016, 1004, 1006, 1037, 1014, 1018, 1001, 4, 2, 0, 0, 0, 0, 0, 0, 1018, 1082, 1026, 1040, 1070, 1033, 1031, 993, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8190, 8245, 8209, 8201, 8074, 8097, 8298] avg=8192.0000 max=8298 max_rank=7 min=8074 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1064, 1043, 1011, 1031, 991, 1020, 1021, 994, 0, 0, 0, 0, 0, 0, 0, 0, 948, 1010, 1001, 1037, 1049, 1052, 1031, 972, 12, 0, 0, 0, 0, 0, 0, 0, 1032, 1030, 1035, 1042, 1027, 1038, 1020, 1003, 16, 0, 0, 0, 0, 0, 0, 0, 1047, 1025, 1015, 1024, 1005, 945, 1028, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1012, 1012, 1008, 1010, 971, 1014, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1010, 969, 1039, 1060, 1028, 1031, 1073, 26, 0, 0, 0, 0, 0, 0, 0, 1056, 1010, 1081, 1038, 1027, 999, 1051, 1008, 4, 2, 0, 0, 0, 0, 0, 0, 1002, 1036, 997, 1025, 1010, 1027, 1013, 1065, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8112, 8243, 8141, 8147, 8261, 8276, 8181] avg=8192.0000 max=8276 max_rank=6 min=8112 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1052, 1014, 1058, 984, 1030, 1022, 1042, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1032, 1005, 1035, 1037, 1043, 1050, 1021, 12, 0, 0, 0, 0, 0, 0, 0, 973, 1042, 1062, 1040, 1004, 1035, 999, 1014, 17, 0, 0, 0, 0, 0, 0, 0, 1045, 1039, 1066, 998, 1020, 995, 1019, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 989, 1014, 997, 1000, 1019, 1063, 1015, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1016, 1093, 985, 1015, 980, 1029, 972, 27, 0, 0, 0, 0, 0, 0, 0, 1038, 1025, 1008, 988, 1016, 981, 1044, 1014, 4, 2, 0, 0, 0, 0, 0, 0, 999, 1053, 1032, 1027, 1045, 1024, 1005, 982, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8296, 8186, 8203, 8157, 8151, 8120, 8173] avg=8192.0000 max=8296 max_rank=1 min=8120 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32601 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1035, 986, 999, 1024, 1000, 1060, 1039, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1006, 1009, 980, 1044, 1069, 1005, 1000, 12, 0, 0, 0, 0, 0, 0, 0, 1049, 1026, 1007, 997, 1033, 1013, 1021, 1003, 16, 0, 0, 0, 0, 0, 0, 0, 1011, 1033, 1043, 1018, 965, 1021, 1019, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1019, 1006, 1001, 1057, 1009, 1002, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1028, 1042, 1024, 1008, 1039, 1018, 1016, 25, 0, 0, 0, 0, 0, 0, 0, 1055, 1040, 1045, 1038, 983, 1040, 1006, 1055, 4, 2, 0, 0, 0, 0, 0, 0, 1042, 1009, 1016, 1034, 1038, 1043, 1018, 1059, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8099, 8165, 8134, 8155, 8264, 8268, 8264] avg=8192.0000 max=8268 max_rank=6 min=8099 min_rank=1 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32951 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1033, 1009, 930, 1014, 1045, 1002, 1042, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 997, 1023, 1042, 990, 1011, 1027, 1006, 16, 1, 0, 0, 0, 0, 0, 0, 1040, 1024, 1072, 1001, 1024, 978, 1029, 999, 19, 0, 0, 0, 0, 0, 0, 0, 1045, 994, 981, 990, 1100, 1035, 1071, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1015, 1024, 1029, 1059, 1043, 1009, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 975, 999, 1012, 1039, 1001, 1054, 1026, 28, 0, 0, 0, 0, 0, 0, 0, 1078, 1022, 1020, 999, 1037, 1063, 1047, 1041, 10, 7, 0, 0, 0, 0, 0, 0, 1052, 1006, 982, 1031, 1036, 1034, 978, 1048, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8144, 8186, 8229, 8196, 8177, 8324, 8176] avg=8192.0000 max=8324 max_rank=6 min=8104 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1022, 1028, 1029, 990, 1047, 1052, 1007, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1042, 1029, 1018, 1029, 1011, 1054, 1016, 12, 0, 0, 0, 0, 0, 0, 0, 1030, 1008, 1066, 978, 1014, 969, 1032, 1027, 17, 0, 0, 0, 0, 0, 0, 0, 1033, 1038, 1011, 1044, 1006, 1031, 991, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 985, 1058, 1045, 987, 996, 1033, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1008, 1050, 1036, 1041, 1020, 1065, 999, 26, 0, 0, 0, 0, 0, 0, 0, 1031, 1033, 1040, 1017, 1016, 1045, 997, 1047, 4, 2, 0, 0, 0, 0, 0, 0, 1005, 1020, 1051, 980, 996, 1011, 1054, 1003, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8221, 8141, 8193, 8164, 8251, 8232, 8126] avg=8192.0000 max=8251 max_rank=5 min=8126 min_rank=7 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32773 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8192, 8320, 8192, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8320, 8320, 8320, 8192, 8320, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8064, 8192, 8192, 8320, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8192, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 8192, 8320, 8192, 8192, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8064, 8320, 8192, 8320, 8192, 8320, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1024, 988, 1018, 1086, 1124, 1007, 1049, 1079, 1023, 1034, 1060, 1022, 973, 1040, 1005, 1050, 1090, 1055, 1034, 1004, 1028, 1003, 1011, 1025, 1021, 995, 1012, 1015, 1030, 982, 1002, 1019, 1016, 999, 1034, 996, 1040, 1067, 1034, 1012, 1011, 1001, 973, 1048, 1017, 1010, 1035, 1061, 1023, 999, 1023, 1034, 983, 1036, 970, 1032, 997, 1006, 1080, 999, 1059, 973, 1045, 1015] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[980, 1031, 965, 1070, 1041, 1034, 1013, 1000, 1025, 1021, 1019, 1015, 1008, 997, 997, 1038, 1000, 1076, 1053, 1022, 1033, 977, 1020, 1063, 1043, 983, 1000, 1030, 1053, 1064, 1013, 1025, 1017, 1040, 1013, 1018, 1052, 1012, 1027, 1036, 1000, 996, 1028, 1022, 988, 1044, 1010, 1018, 1016, 1017, 1006, 1001, 1028, 1015, 1039, 1014, 1068, 1013, 1034, 1031, 1061, 1084, 1050, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1035, 1049, 1027, 1091, 1066, 1045, 1041, 1020, 1033, 1008, 1040, 1010, 997, 1086, 969, 1061, 1088, 1006, 1011, 992, 965, 1015, 1044, 1016, 1032, 995, 991, 1053, 1038, 1050, 965, 1012, 1038, 1112, 1002, 963, 1000, 998, 1025, 1038, 963, 1034, 1013, 1009, 980, 1013, 987, 1041, 967, 1047, 1042, 1052, 1003, 1065, 1027, 1028, 1049, 1036, 1053, 1004, 1006, 1051, 1052, 987] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1054, 1034, 1041, 968, 1003, 1027, 1015, 1017, 1029, 989, 1023, 1027, 1020, 1021, 1072, 1027, 1049, 1023, 1061, 1042, 1063, 1011, 970, 970, 1037, 1045, 1074, 1023, 991, 1025, 1056, 1022, 1012, 999, 1043, 1046, 1022, 1010, 995, 965, 1063, 1028, 1057, 1001, 1039, 1028, 1012, 1052, 1001, 1017, 1009, 1018, 1004, 1019, 1033, 1034, 1020, 1082, 1004, 1007, 1012, 1014, 1034, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[998, 978, 1002, 996, 1020, 1026, 1072, 1001, 994, 1052, 991, 1073, 970, 1017, 1017, 1035, 992, 983, 1004, 1019, 1064, 1047, 1012, 1019, 1054, 1050, 1066, 978, 1006, 1012, 1084, 1035, 1027, 1047, 1019, 1027, 1006, 1063, 1029, 1062, 1002, 1060, 1058, 985, 1029, 1006, 1021, 1005, 1027, 1012, 1036, 985, 1070, 1045, 1043, 1033, 1015, 998, 1016, 1047, 1041, 1049, 999, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1008, 1044, 1046, 1014, 1067, 1018, 1001, 1020, 1019, 1044, 1000, 1033, 1018, 1008, 1062, 1040, 1008, 1027, 997, 998, 992, 951, 990, 1010, 1052, 1018, 1040, 999, 1013, 1007, 1018, 1027, 1084, 1025, 1056, 1047, 1036, 1018, 1042, 1053, 1068, 1058, 1031, 1026, 1010, 982, 1016, 1002, 1026, 1014, 998, 1034, 1050, 976, 1010, 1041, 982, 971, 1015, 1058, 1049, 1040, 1084, 1045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1031, 971, 1027, 997, 1061, 1050, 1049, 998, 1003, 1013, 1030, 1047, 1030, 985, 1025, 1090, 951, 1028, 1011, 1080, 1046, 1021, 991, 1081, 1038, 1047, 1007, 1058, 991, 1029, 1065, 1012, 1010, 1028, 971, 1028, 1029, 982, 1027, 1076, 1043, 1008, 1031, 1122, 991, 987, 1041, 994, 1021, 999, 1039, 1038, 1020, 1027, 1011, 1049, 993, 1044, 999, 1075, 970, 1017, 1025, 978] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[965, 1050, 1036, 1027, 965, 1038, 1075, 1024, 1059, 1000, 1027, 1005, 1032, 1052, 1023, 995, 1041, 962, 1010, 1051, 1005, 1057, 990, 1040, 981, 968, 997, 1036, 1017, 1011, 1018, 1080, 1029, 1003, 1032, 1094, 1061, 1055, 1042, 1034, 1051, 990, 1046, 1055, 1045, 991, 1041, 1043, 996, 1023, 967, 1005, 1031, 1018, 1050, 1009, 1031, 1044, 997, 1068, 991, 1030, 1034, 993] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1054, 1034, 1041, 968, 982, 1027, 1015, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 989, 1023, 1027, 1020, 1021, 1072, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1023, 1061, 1042, 1063, 1011, 970, 970, 27, 0, 0, 0, 0, 0, 0, 0, 1037, 1045, 1074, 1023, 991, 1025, 1056, 1022, 3, 0, 0, 0, 0, 0, 0, 0, 1012, 999, 1043, 1046, 1022, 1010, 995, 933, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 1028, 1057, 1001, 1039, 1028, 1012, 1052, 14, 0, 0, 0, 0, 0, 0, 0, 1001, 1017, 1009, 1018, 1004, 1019, 1033, 1034, 21, 5, 0, 0, 0, 0, 0, 0, 1020, 1082, 1004, 1007, 1012, 1014, 1017, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8208, 8216, 8276, 8060, 8294, 8161, 8183] avg=8192.0000 max=8294 max_rank=5 min=8060 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1024, 988, 1018, 1086, 1100, 1007, 1049, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1034, 1060, 1022, 973, 1040, 1005, 1050, 1, 0, 0, 0, 0, 0, 0, 0, 1090, 1055, 1034, 1004, 1028, 1003, 1011, 1025, 29, 0, 0, 0, 0, 0, 0, 0, 1021, 995, 1012, 1015, 1030, 982, 1002, 1019, 3, 0, 0, 0, 0, 0, 0, 0, 1016, 999, 1034, 996, 1040, 1067, 1034, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1001, 973, 1048, 1017, 1010, 1035, 1061, 14, 0, 0, 0, 0, 0, 0, 0, 1023, 999, 1023, 1034, 983, 1036, 970, 1032, 24, 5, 0, 0, 0, 0, 0, 0, 997, 1006, 1080, 999, 1059, 973, 1027, 1015, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8351, 8208, 8279, 8079, 8164, 8170, 8129, 8156] avg=8192.0000 max=8351 max_rank=0 min=8079 min_rank=3 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32619 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1035, 1049, 1027, 1091, 1043, 1045, 1041, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1008, 1040, 1010, 997, 1086, 969, 1061, 1, 0, 0, 0, 0, 0, 0, 0, 1088, 1006, 1011, 992, 965, 1015, 1044, 1016, 29, 0, 0, 0, 0, 0, 0, 0, 1032, 995, 991, 1053, 1038, 1050, 965, 1012, 3, 0, 0, 0, 0, 0, 0, 0, 1038, 1112, 1002, 963, 1000, 998, 1025, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1034, 1013, 1009, 980, 1013, 987, 1041, 14, 0, 0, 0, 0, 0, 0, 0, 967, 1047, 1042, 1052, 1003, 1065, 1027, 1028, 23, 6, 0, 0, 0, 0, 0, 0, 1049, 1036, 1053, 1004, 1006, 1051, 1034, 987, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8351, 8205, 8166, 8139, 8141, 8054, 8260, 8220] avg=8192.0000 max=8351 max_rank=0 min=8054 min_rank=5 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32675 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[980, 1031, 965, 1070, 1019, 1034, 1013, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1021, 1019, 1015, 1008, 997, 997, 1038, 1, 0, 0, 0, 0, 0, 0, 0, 1000, 1076, 1053, 1022, 1033, 977, 1020, 1063, 29, 0, 0, 0, 0, 0, 0, 0, 1043, 983, 1000, 1030, 1053, 1064, 1013, 1025, 3, 0, 0, 0, 0, 0, 0, 0, 1017, 1040, 1013, 1018, 1052, 1012, 1027, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 996, 1028, 1022, 988, 1044, 1010, 1018, 14, 0, 0, 0, 0, 0, 0, 0, 1016, 1017, 1006, 1001, 1028, 1015, 1039, 1014, 22, 6, 0, 0, 0, 0, 0, 0, 1068, 1013, 1034, 1031, 1061, 1084, 1032, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8121, 8273, 8214, 8180, 8120, 8164, 8352] avg=8192.0000 max=8352 max_rank=7 min=8112 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[998, 978, 1002, 996, 998, 1026, 1072, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1052, 991, 1073, 970, 1017, 1017, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 992, 983, 1004, 1019, 1064, 1047, 1012, 1019, 30, 0, 0, 0, 0, 0, 0, 0, 1054, 1050, 1066, 978, 1006, 1012, 1084, 1035, 3, 0, 0, 0, 0, 0, 0, 0, 1027, 1047, 1019, 1027, 1006, 1063, 1029, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1060, 1058, 985, 1029, 1006, 1021, 1005, 13, 0, 0, 0, 0, 0, 0, 0, 1027, 1012, 1036, 985, 1070, 1045, 1043, 1033, 22, 6, 0, 0, 0, 0, 0, 0, 1015, 998, 1016, 1047, 1041, 1049, 983, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8149, 8170, 8288, 8244, 8179, 8279, 8156] avg=8192.0000 max=8288 max_rank=3 min=8071 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1008, 1044, 1046, 1014, 1044, 1018, 1001, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1044, 1000, 1033, 1018, 1008, 1062, 1040, 1, 0, 0, 0, 0, 0, 0, 0, 1008, 1027, 997, 998, 992, 951, 990, 1010, 30, 0, 0, 0, 0, 0, 0, 0, 1052, 1018, 1040, 999, 1013, 1007, 1018, 1027, 3, 0, 0, 0, 0, 0, 0, 0, 1084, 1025, 1056, 1047, 1036, 1018, 1042, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 1058, 1031, 1026, 1010, 982, 1016, 1002, 15, 0, 0, 0, 0, 0, 0, 0, 1026, 1014, 998, 1034, 1050, 976, 1010, 1041, 23, 6, 0, 0, 0, 0, 0, 0, 982, 971, 1015, 1058, 1049, 1040, 1065, 1045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8225, 8003, 8177, 8325, 8208, 8178, 8225] avg=8192.0000 max=8325 max_rank=4 min=8003 min_rank=2 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32936 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1031, 971, 1027, 997, 1038, 1050, 1049, 998, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1013, 1030, 1047, 1030, 985, 1025, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1028, 1011, 1080, 1046, 1021, 991, 1081, 30, 0, 0, 0, 0, 0, 0, 0, 1038, 1047, 1007, 1058, 991, 1029, 1065, 1012, 3, 0, 0, 0, 0, 0, 0, 0, 1010, 1028, 971, 1028, 1029, 982, 1027, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1008, 1031, 1122, 991, 987, 1041, 994, 14, 0, 0, 0, 0, 0, 0, 0, 1021, 999, 1039, 1038, 1020, 1027, 1011, 1049, 23, 6, 0, 0, 0, 0, 0, 0, 993, 1044, 999, 1075, 970, 1017, 1008, 978, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8223, 8239, 8250, 8115, 8231, 8233, 8084] avg=8192.0000 max=8250 max_rank=3 min=8084 min_rank=7 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32663 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[965, 1050, 1036, 1027, 942, 1038, 1075, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 1000, 1027, 1005, 1032, 1052, 1023, 995, 4, 0, 0, 0, 0, 0, 0, 0, 1041, 962, 1010, 1051, 1005, 1057, 990, 1040, 34, 0, 0, 0, 0, 0, 0, 0, 981, 968, 997, 1036, 1017, 1011, 1018, 1080, 5, 0, 0, 0, 0, 0, 0, 0, 1029, 1003, 1032, 1094, 1061, 1055, 1042, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 990, 1046, 1055, 1045, 991, 1041, 1043, 18, 0, 0, 0, 0, 0, 0, 0, 996, 1023, 967, 1005, 1031, 1018, 1050, 1009, 23, 9, 1, 0, 0, 0, 0, 0, 1031, 1044, 997, 1068, 991, 1030, 1006, 993, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8197, 8190, 8113, 8307, 8280, 8132, 8160] avg=8192.0000 max=8307 max_rank=4 min=8113 min_rank=3 imbalance_ratio=1.014038 max_load_reduction=57229 max_load_reduction_pct=0.873245 moved_to_echo=32879 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8192, 8064, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8064, 8320, 8192, 8448, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8320, 8320, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8192, 8320, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8320, 8192, 8064, 8320, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1061, 1034, 1063, 1008, 1049, 980, 1042, 1055, 987, 982, 975, 1055, 1041, 960, 1078, 1049, 1060, 1065, 1040, 1035, 1059, 1029, 1000, 1070, 1024, 993, 1028, 1054, 952, 983, 1012, 1022, 1036, 1018, 1024, 999, 1066, 1047, 1043, 999, 1077, 980, 1054, 1019, 1026, 1024, 1041, 1024, 1007, 1006, 1015, 1013, 1022, 990, 1003, 1006, 1048, 1002, 1024, 1068, 1014, 990, 1000, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1034, 1018, 1040, 990, 969, 998, 1015, 1034, 977, 1006, 1019, 1081, 1052, 1041, 1033, 1026, 1074, 993, 1009, 1020, 1052, 1045, 1005, 1032, 1008, 1030, 1018, 1061, 1015, 979, 986, 1058, 1032, 964, 1002, 1047, 1012, 1008, 1009, 1064, 1056, 1062, 1020, 1029, 1031, 991, 1071, 1000, 998, 1004, 1004, 1054, 1070, 1046, 988, 1007, 1027, 1032, 987, 1027, 1047, 1079, 1025, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1029, 1014, 1044, 1036, 1026, 1039, 1025, 1008, 1015, 1028, 999, 999, 990, 1027, 1067, 1001, 1029, 1031, 963, 1049, 1032, 1015, 1001, 997, 1013, 1012, 1033, 1006, 1042, 1057, 1060, 986, 1047, 1041, 1040, 1080, 1028, 1007, 1083, 1004, 1004, 1056, 1014, 1048, 1043, 945, 1016, 1050, 1015, 1064, 1057, 997, 988, 1004, 1001, 1027, 992, 1020, 1043, 1022, 995, 1080, 983, 1069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1053, 1021, 1065, 1053, 1010, 1020, 999, 1045, 985, 1034, 1073, 987, 1050, 1023, 1022, 1035, 1010, 953, 1004, 1000, 1036, 1037, 1016, 1036, 1014, 1044, 1003, 1030, 1103, 1062, 1012, 995, 998, 1039, 1011, 1024, 1014, 1071, 1002, 1019, 1043, 1054, 965, 1045, 1028, 1019, 1018, 1027, 1030, 1041, 1011, 1006, 997, 987, 1063, 999, 1004, 1002, 1022, 1028, 1030, 1016, 1044, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1004, 1000, 987, 1017, 1045, 990, 985, 1038, 1058, 1027, 1022, 990, 1068, 1054, 1060, 1042, 1026, 1037, 1014, 1007, 999, 1008, 1066, 1038, 1041, 966, 1024, 1033, 1029, 1013, 1027, 1054, 1035, 1070, 1058, 1017, 1007, 1053, 996, 1023, 1058, 1031, 1026, 1038, 1048, 1047, 1022, 981, 993, 980, 1018, 1019, 999, 1097, 1000, 1045, 981, 1019, 1041, 1006, 986, 1026, 1024, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1037, 1066, 1050, 1031, 987, 1080, 1022, 966, 1006, 1025, 1076, 1043, 1062, 983, 993, 1077, 1001, 1021, 996, 1026, 1083, 982, 1009, 1023, 1047, 1001, 1013, 1021, 1078, 979, 988, 1028, 980, 1012, 1021, 998, 1020, 1057, 987, 984, 1020, 1050, 1043, 1024, 1066, 1036, 1045, 1008, 1032, 995, 1015, 1045, 1023, 1009, 1010, 951, 1033, 1027, 1041, 1022, 1003, 1038, 1078, 1063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1009, 1031, 1109, 1035, 1019, 1036, 1003, 1014, 1078, 990, 986, 1066, 1057, 1026, 1002, 1061, 948, 1029, 1116, 1038, 1010, 980, 1004, 1039, 1029, 1025, 1026, 1004, 1016, 1031, 1017, 1007, 1009, 1008, 1005, 1038, 1048, 996, 1085, 1032, 1009, 1012, 1037, 1015, 999, 1043, 1024, 1032, 1006, 1031, 1051, 1019, 1030, 981, 1067, 1037, 982, 1038, 990, 1029, 1045, 994, 1015, 988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1035, 1034, 1036, 1101, 1034, 988, 1033, 983, 1052, 1013, 1018, 984, 1006, 1044, 999, 1026, 1032, 1025, 1060, 1024, 1043, 997, 1021, 1039, 992, 1000, 1080, 1019, 1028, 1035, 1007, 1038, 1029, 1046, 1012, 1026, 982, 1034, 1031, 987, 1036, 1014, 1015, 990, 1034, 1028, 998, 1037, 1019, 1029, 1052, 999, 1035, 1044, 1078, 1000, 960, 1014, 1041, 1052, 1028, 1010, 1065, 985] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1061, 1034, 1045, 1008, 1049, 980, 1042, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 987, 982, 975, 1055, 1022, 960, 1078, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 1065, 1040, 1035, 1059, 1029, 1000, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 993, 1028, 1054, 952, 983, 1012, 1022, 11, 7, 0, 0, 0, 0, 0, 0, 1036, 1018, 1024, 999, 1066, 1044, 1043, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 980, 1054, 1019, 1026, 1024, 1041, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1006, 1015, 1013, 1022, 990, 1003, 1006, 27, 12, 0, 0, 0, 0, 0, 0, 1048, 1002, 1024, 1068, 1014, 990, 1000, 1006, 7, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8108, 8358, 8086, 8229, 8218, 8101, 8162] avg=8192.0000 max=8358 max_rank=2 min=8086 min_rank=3 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1034, 1018, 1023, 990, 969, 998, 1015, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1006, 1019, 1081, 1033, 1041, 1033, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 993, 1009, 1020, 1052, 1045, 1005, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1030, 1018, 1061, 1015, 979, 986, 1058, 10, 7, 0, 0, 0, 0, 0, 0, 1032, 964, 1002, 1047, 1012, 1005, 1009, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1062, 1020, 1029, 1031, 991, 1071, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1004, 1004, 1054, 1070, 1046, 988, 1007, 26, 12, 0, 0, 0, 0, 0, 0, 1027, 1032, 987, 1027, 1047, 1079, 1025, 1025, 7, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8081, 8216, 8230, 8172, 8135, 8234, 8209, 8259] avg=8192.0000 max=8259 max_rank=7 min=8081 min_rank=0 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32837 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1053, 1021, 1047, 1053, 1010, 1020, 999, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 985, 1034, 1073, 987, 1031, 1023, 1022, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 953, 1004, 1000, 1036, 1037, 1016, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1044, 1003, 1030, 1103, 1062, 1012, 995, 11, 7, 0, 0, 0, 0, 0, 0, 998, 1039, 1011, 1024, 1014, 1068, 1002, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1054, 965, 1045, 1028, 1019, 1018, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1041, 1011, 1006, 997, 987, 1063, 999, 26, 12, 0, 0, 0, 0, 0, 0, 1004, 1002, 1022, 1028, 1030, 1016, 1044, 1049, 7, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8190, 8092, 8281, 8175, 8173, 8172, 8205] avg=8192.0000 max=8281 max_rank=3 min=8092 min_rank=2 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1004, 1000, 971, 1017, 1045, 990, 985, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1027, 1022, 990, 1049, 1054, 1060, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1037, 1014, 1007, 999, 1008, 1066, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 966, 1024, 1033, 1029, 1013, 1027, 1054, 10, 7, 0, 0, 0, 0, 0, 0, 1035, 1070, 1058, 1017, 1007, 1050, 996, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1031, 1026, 1038, 1048, 1047, 1022, 981, 0, 0, 0, 0, 0, 0, 0, 0, 993, 980, 1018, 1019, 999, 1097, 1000, 1045, 26, 12, 0, 0, 0, 0, 0, 0, 981, 1019, 1041, 1006, 986, 1026, 1024, 1023, 6, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8050, 8302, 8195, 8204, 8256, 8225, 8189, 8115] avg=8192.0000 max=8302 max_rank=1 min=8050 min_rank=0 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32785 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1009, 1031, 1091, 1035, 1019, 1036, 1003, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 990, 986, 1066, 1038, 1026, 1002, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 948, 1029, 1116, 1038, 1010, 980, 1004, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1025, 1026, 1004, 1016, 1031, 1017, 1007, 11, 7, 0, 0, 0, 0, 0, 0, 1009, 1008, 1005, 1038, 1048, 993, 1085, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 984, 1012, 1037, 1015, 999, 1043, 1024, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1031, 1051, 1019, 1030, 981, 1067, 1037, 25, 12, 0, 0, 0, 0, 0, 0, 982, 1038, 990, 1029, 1045, 994, 1015, 988, 7, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8247, 8164, 8173, 8218, 8146, 8259, 8091] avg=8192.0000 max=8259 max_rank=6 min=8091 min_rank=7 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1029, 1014, 1027, 1036, 1026, 1039, 1025, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1028, 999, 999, 973, 1027, 1067, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1031, 963, 1049, 1032, 1015, 1001, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1012, 1033, 1006, 1042, 1057, 1060, 986, 10, 6, 0, 0, 0, 0, 0, 0, 1047, 1041, 1040, 1080, 1028, 1004, 1083, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 979, 1056, 1014, 1048, 1043, 945, 1016, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1064, 1057, 997, 988, 1004, 1001, 1027, 25, 11, 0, 0, 0, 0, 0, 0, 992, 1020, 1043, 1022, 995, 1080, 983, 1069, 7, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8109, 8117, 8225, 8327, 8151, 8189, 8214] avg=8192.0000 max=8327 max_rank=4 min=8109 min_rank=1 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32881 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1037, 1066, 1033, 1031, 987, 1080, 1022, 966, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1025, 1076, 1043, 1043, 983, 993, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1021, 996, 1026, 1083, 982, 1009, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1001, 1013, 1021, 1078, 979, 988, 1028, 10, 7, 0, 0, 0, 0, 0, 0, 980, 1012, 1021, 998, 1020, 1054, 987, 984, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1050, 1043, 1024, 1066, 1036, 1045, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 995, 1015, 1045, 1023, 1009, 1010, 951, 25, 12, 0, 0, 0, 0, 0, 0, 1033, 1027, 1041, 1022, 1003, 1038, 1078, 1063, 7, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8246, 8141, 8172, 8056, 8267, 8117, 8315] avg=8192.0000 max=8315 max_rank=7 min=8056 min_rank=4 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1035, 1034, 1011, 1101, 1034, 988, 1033, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1013, 1018, 984, 982, 1044, 999, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1025, 1060, 1024, 1041, 997, 1021, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1000, 1080, 1019, 1028, 1035, 1007, 1038, 14, 10, 0, 0, 0, 0, 0, 0, 1029, 1046, 1012, 1026, 982, 1027, 1031, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1014, 1015, 990, 1034, 1028, 998, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1029, 1052, 999, 1035, 1044, 1078, 1000, 30, 14, 0, 0, 0, 0, 0, 0, 960, 1014, 1041, 1052, 1028, 1010, 1065, 985, 11, 7, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8118, 8239, 8223, 8140, 8122, 8300, 8175] avg=8192.0000 max=8300 max_rank=6 min=8118 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8320, 8192, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8192, 8192, 8320, 8192, 8320, 8320, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8192, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1028, 1026, 1034, 1078, 1004, 1048, 1015, 998, 1036, 992, 1029, 1028, 1022, 1029, 1069, 989, 1029, 996, 976, 1033, 1040, 994, 1009, 983, 1025, 1038, 1048, 985, 1021, 979, 1044, 1050, 970, 1028, 998, 1047, 1017, 1054, 1037, 1043, 1025, 1075, 1022, 1038, 1008, 1039, 1022, 1005, 1022, 972, 1054, 1049, 1011, 998, 1012, 1054, 1047, 1066, 977, 1045, 1038, 994, 1034, 1060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1018, 1042, 1007, 1005, 1034, 1050, 1035, 1066, 1008, 1025, 1051, 1040, 1017, 1030, 1014, 1040, 1040, 994, 1031, 1051, 968, 1069, 1057, 1080, 1050, 1042, 987, 1012, 1007, 1047, 1011, 1076, 990, 1071, 1027, 1023, 1055, 1003, 1018, 1051, 995, 990, 1026, 1069, 1011, 998, 1012, 999, 1011, 1017, 1020, 992, 1021, 1018, 1017, 1049, 990, 981, 1032, 967, 991, 1017, 1062, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[997, 993, 1017, 1002, 1020, 994, 1030, 990, 1081, 986, 1045, 1024, 1014, 1011, 994, 992, 985, 1014, 1003, 1005, 1006, 1009, 1015, 1109, 1053, 1046, 990, 993, 1032, 1043, 1011, 1060, 962, 982, 1044, 1023, 1043, 1014, 1032, 1039, 1037, 1002, 1016, 999, 1039, 1011, 1014, 1077, 1025, 1057, 1043, 1051, 1028, 1067, 1018, 1040, 1072, 1008, 996, 1025, 1069, 1051, 1030, 1058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[979, 1020, 1030, 1052, 1015, 1034, 1031, 1023, 1038, 978, 1033, 1035, 1015, 1013, 1045, 1059, 1006, 994, 1036, 990, 989, 1049, 1012, 997, 1008, 1014, 1009, 958, 1089, 1058, 1049, 983, 998, 971, 1006, 999, 1050, 1075, 1013, 1048, 1062, 1028, 1002, 1010, 1080, 1035, 1056, 1051, 1010, 1051, 1056, 1030, 1052, 1034, 1040, 1001, 1011, 982, 1007, 1045, 1077, 997, 981, 1037] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[991, 1071, 1017, 997, 1016, 994, 1036, 1039, 993, 1051, 1022, 1039, 1016, 1058, 1042, 978, 1035, 1012, 1035, 1021, 986, 1013, 1017, 1038, 1035, 1069, 1053, 1032, 992, 1013, 1060, 1019, 1026, 1031, 1030, 1037, 998, 1021, 1034, 1036, 993, 1029, 993, 1013, 1021, 1055, 1064, 1085, 1028, 1006, 986, 1011, 1060, 985, 1044, 1024, 1011, 1028, 1027, 997, 957, 1058, 1046, 1012] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1011, 990, 1008, 1027, 1036, 1058, 1011, 1096, 1006, 1006, 1007, 988, 993, 1077, 997, 1020, 1012, 1052, 1043, 1036, 982, 1057, 1009, 1030, 1016, 1037, 1047, 1006, 1013, 1023, 1048, 1081, 1023, 1030, 993, 1022, 1012, 1000, 1011, 1007, 1025, 1052, 1014, 1045, 1011, 949, 1053, 989, 1027, 1017, 1019, 1070, 1010, 984, 992, 1075, 1028, 1073, 1069, 1003, 993, 1064, 1044, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1010, 951, 1023, 978, 1022, 1037, 1006, 1043, 1052, 1037, 1014, 1052, 975, 1066, 991, 1062, 1032, 965, 1061, 998, 1073, 959, 986, 1053, 998, 1032, 1016, 1041, 1073, 1082, 1033, 1040, 1026, 1055, 1063, 983, 1014, 1008, 985, 1003, 1026, 1042, 1038, 1038, 1007, 1023, 1059, 1013, 1064, 1039, 1065, 1044, 1016, 998, 989, 1002, 1043, 1047, 1020, 946, 1014, 973, 1052, 1080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[997, 1037, 988, 1081, 1065, 1011, 1076, 1021, 1026, 990, 1054, 1029, 984, 988, 1036, 1054, 987, 1060, 1015, 1023, 1040, 1014, 1059, 1044, 1073, 1042, 1030, 1027, 1026, 1000, 1051, 988, 1007, 999, 1019, 981, 1047, 1011, 976, 1003, 1012, 983, 1065, 1010, 1007, 1062, 1024, 1041, 1033, 1002, 1026, 1048, 1003, 1036, 1058, 1045, 1024, 1029, 1019, 1025, 1021, 989, 1001, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1018, 1042, 1007, 1005, 1034, 1050, 1035, 1066, 5, 4, 0, 0, 0, 0, 0, 0, 1008, 1025, 1051, 1040, 1017, 1030, 1014, 1040, 6, 0, 0, 0, 0, 0, 0, 0, 1040, 994, 1031, 1051, 968, 1069, 1057, 1080, 18, 8, 0, 0, 0, 0, 0, 0, 1050, 995, 987, 1012, 1007, 1047, 1011, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1071, 1027, 1023, 1055, 1003, 1018, 1051, 39, 0, 0, 0, 0, 0, 0, 0, 995, 990, 1026, 1069, 1011, 998, 994, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1017, 1020, 970, 1021, 1018, 1017, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 990, 981, 1032, 967, 991, 1017, 1062, 1009, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8231, 8316, 8185, 8277, 8082, 8123, 8056] avg=8192.0000 max=8316 max_rank=2 min=8056 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32538 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1028, 1026, 1034, 1078, 1004, 1048, 1015, 998, 5, 4, 0, 0, 0, 0, 0, 0, 1036, 992, 1029, 1028, 1022, 1029, 1069, 989, 6, 0, 0, 0, 0, 0, 0, 0, 1029, 996, 976, 1033, 1040, 994, 1009, 983, 19, 7, 0, 0, 0, 0, 0, 0, 1025, 992, 1048, 985, 1021, 979, 1044, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1028, 998, 1047, 1017, 1054, 1037, 1043, 39, 0, 0, 0, 0, 0, 0, 0, 1025, 1075, 1022, 1038, 1008, 1039, 1004, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 972, 1054, 1026, 1011, 998, 1012, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1066, 977, 1045, 1038, 994, 1034, 1060, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8200, 8086, 8144, 8233, 8216, 8149, 8268] avg=8192.0000 max=8268 max_rank=7 min=8086 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32866 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[979, 1020, 1030, 1052, 1015, 1034, 1031, 1023, 5, 4, 0, 0, 0, 0, 0, 0, 1038, 978, 1033, 1035, 1015, 1013, 1045, 1059, 6, 0, 0, 0, 0, 0, 0, 0, 1006, 994, 1036, 990, 989, 1049, 1012, 997, 19, 7, 0, 0, 0, 0, 0, 0, 1008, 969, 1009, 958, 1089, 1058, 1049, 983, 0, 0, 0, 0, 0, 0, 0, 0, 998, 971, 1006, 999, 1050, 1075, 1013, 1048, 38, 0, 0, 0, 0, 0, 0, 0, 1062, 1028, 1002, 1010, 1080, 1035, 1037, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1051, 1056, 1007, 1052, 1034, 1040, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 982, 1007, 1045, 1077, 997, 981, 1037, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8222, 8099, 8123, 8198, 8305, 8251, 8145] avg=8192.0000 max=8305 max_rank=5 min=8099 min_rank=2 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[997, 993, 1017, 1002, 1020, 994, 1030, 990, 5, 4, 0, 0, 0, 0, 0, 0, 1081, 986, 1045, 1024, 1014, 1011, 994, 992, 6, 0, 0, 0, 0, 0, 0, 0, 985, 1014, 1003, 1005, 1006, 1009, 1015, 1109, 19, 8, 0, 0, 0, 0, 0, 0, 1053, 999, 990, 993, 1032, 1043, 1011, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 962, 982, 1044, 1023, 1043, 1014, 1032, 1039, 39, 0, 0, 0, 0, 0, 0, 0, 1037, 1002, 1016, 999, 1039, 1011, 996, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1057, 1043, 1028, 1028, 1067, 1018, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1008, 996, 1025, 1069, 1051, 1030, 1058, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8052, 8153, 8173, 8181, 8178, 8177, 8306, 8316] avg=8192.0000 max=8316 max_rank=7 min=8052 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32977 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1011, 990, 1008, 1027, 1036, 1058, 1011, 1096, 5, 4, 0, 0, 0, 0, 0, 0, 1006, 1006, 1007, 988, 993, 1077, 997, 1020, 6, 0, 0, 0, 0, 0, 0, 0, 1012, 1052, 1043, 1036, 982, 1057, 1009, 1030, 20, 7, 0, 0, 0, 0, 0, 0, 1016, 991, 1047, 1006, 1013, 1023, 1048, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1030, 993, 1022, 1012, 1000, 1011, 1007, 39, 0, 0, 0, 0, 0, 0, 0, 1025, 1052, 1014, 1045, 1011, 949, 1034, 989, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1017, 1019, 1046, 1010, 984, 992, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1073, 1069, 1003, 993, 1064, 1044, 1009, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8100, 8248, 8225, 8137, 8119, 8170, 8291] avg=8192.0000 max=8291 max_rank=7 min=8100 min_rank=1 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32717 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[991, 1071, 1017, 997, 1016, 994, 1036, 1039, 5, 4, 0, 0, 0, 0, 0, 0, 993, 1051, 1022, 1039, 1016, 1058, 1042, 978, 6, 0, 0, 0, 0, 0, 0, 0, 1035, 1012, 1035, 1021, 986, 1013, 1017, 1038, 19, 8, 0, 0, 0, 0, 0, 0, 1035, 1021, 1053, 1032, 992, 1013, 1060, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1031, 1030, 1037, 998, 1021, 1034, 1036, 40, 0, 0, 0, 0, 0, 0, 0, 993, 1029, 993, 1013, 1021, 1055, 1045, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1006, 986, 988, 1060, 985, 1044, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1028, 1027, 997, 957, 1058, 1046, 1012, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8205, 8184, 8225, 8253, 8234, 8121, 8144] avg=8192.0000 max=8253 max_rank=4 min=8121 min_rank=6 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1010, 951, 1023, 978, 1022, 1037, 1006, 1043, 5, 4, 0, 0, 0, 0, 0, 0, 1052, 1037, 1014, 1052, 975, 1066, 991, 1062, 6, 0, 0, 0, 0, 0, 0, 0, 1032, 965, 1061, 998, 1073, 959, 986, 1053, 19, 7, 0, 0, 0, 0, 0, 0, 998, 987, 1016, 1041, 1073, 1082, 1033, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1055, 1063, 983, 1014, 1008, 985, 1003, 38, 0, 0, 0, 0, 0, 0, 0, 1026, 1042, 1038, 1038, 1007, 1023, 1040, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 1039, 1065, 1021, 1016, 998, 989, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1047, 1020, 946, 1014, 973, 1052, 1080, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8079, 8255, 8153, 8270, 8175, 8227, 8194, 8183] avg=8192.0000 max=8270 max_rank=3 min=8079 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[997, 1037, 988, 1081, 1065, 1011, 1076, 1021, 8, 6, 0, 0, 0, 0, 0, 0, 1026, 990, 1054, 1029, 984, 988, 1036, 1054, 9, 0, 0, 0, 0, 0, 0, 0, 987, 1060, 1015, 1023, 1040, 1014, 1059, 1044, 23, 12, 0, 0, 0, 0, 0, 0, 1073, 988, 1030, 1027, 1026, 1000, 1051, 988, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 999, 1019, 981, 1047, 1011, 976, 1003, 42, 0, 0, 0, 0, 0, 0, 0, 1012, 983, 1065, 1010, 1007, 1062, 996, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1002, 1026, 1019, 1003, 1036, 1058, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1029, 1019, 1025, 1021, 989, 1001, 1014, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8290, 8170, 8277, 8183, 8085, 8176, 8222, 8133] avg=8192.0000 max=8290 max_rank=0 min=8085 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=57246 max_load_reduction_pct=0.873505 moved_to_echo=32616 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8064, 8192, 8192, 8320, 8320, 8320, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8320, 8192, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8192, 8192, 8448, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8064, 8192, 8064, 8320, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8192, 8320, 8192, 8192, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8064, 8192, 8192, 8320, 8320, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1022, 1042, 1043, 1036, 1027, 1031, 1005, 1054, 1004, 1031, 1004, 1006, 1005, 998, 1047, 1034, 981, 1022, 1000, 1026, 999, 1049, 973, 1041, 1048, 1010, 1027, 1030, 1056, 975, 998, 1031, 1027, 1057, 1014, 976, 1025, 1022, 1033, 965, 1029, 1039, 1023, 1057, 1041, 1031, 1047, 1045, 1069, 1049, 972, 987, 978, 1002, 1042, 1058, 1050, 1031, 990, 1061, 1061, 1046, 1014, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1052, 1016, 1032, 1005, 1021, 1039, 1022, 1013, 1058, 1052, 1034, 1066, 1019, 960, 986, 1046, 1049, 1040, 992, 990, 1063, 1009, 1025, 1015, 1013, 1095, 1028, 1031, 1015, 1021, 1018, 1022, 1033, 1043, 1008, 976, 951, 996, 1021, 1038, 1053, 1062, 1032, 993, 1050, 1076, 1004, 1001, 1035, 1032, 1008, 1047, 1011, 1016, 1010, 1034, 1045, 987, 1035, 1010, 1011, 1041, 1029, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1064, 989, 1008, 986, 1011, 1010, 975, 1048, 1014, 1037, 1065, 1065, 1026, 971, 1042, 1013, 1009, 1031, 1024, 995, 948, 1056, 989, 1006, 1000, 1067, 1026, 1050, 1008, 1019, 1014, 963, 1043, 1067, 1048, 985, 1026, 1010, 1069, 1025, 1019, 992, 1074, 987, 971, 1039, 1050, 984, 1049, 1029, 1029, 1081, 1015, 1061, 1061, 1056, 1064, 988, 1030, 1036, 1081, 1034, 991, 1013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1042, 1017, 1017, 1024, 1035, 1043, 1020, 1025, 1043, 1002, 1014, 1032, 1049, 1043, 1045, 1033, 973, 1018, 970, 1029, 1027, 1017, 1021, 998, 1001, 1004, 1028, 1069, 1007, 1025, 994, 1044, 1014, 1041, 1039, 1030, 1022, 1027, 981, 1054, 1024, 1013, 1017, 997, 1021, 1043, 1052, 991, 997, 1035, 1066, 970, 991, 1054, 1011, 1033, 1091, 986, 1050, 1019, 1018, 1067, 1033, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1048, 1024, 1028, 1074, 1039, 996, 1001, 985, 1057, 985, 1086, 1034, 994, 1015, 1035, 1040, 1017, 1035, 1014, 1070, 1045, 1078, 1014, 1010, 984, 1084, 1008, 999, 1022, 983, 1029, 1074, 1024, 1025, 1030, 1059, 1033, 994, 1069, 1008, 982, 1085, 1051, 1004, 1020, 985, 1008, 967, 1017, 1002, 1047, 1029, 995, 1044, 1010, 996, 1007, 1028, 1072, 1011, 1034, 963, 1036, 994] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1045, 1042, 1049, 1061, 1061, 1017, 1072, 997, 973, 1042, 1005, 1060, 1053, 1042, 1035, 1068, 1016, 1063, 1026, 1025, 1005, 985, 1019, 1069, 1063, 1021, 1011, 993, 1000, 1023, 1006, 1040, 988, 1031, 996, 1035, 1013, 1033, 992, 1067, 1013, 1012, 995, 980, 1018, 1052, 1030, 1005, 1022, 1027, 1021, 1026, 1031, 1074, 995, 999, 980, 1014, 995, 1008, 1002, 1031, 997, 1067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1049, 1055, 969, 1086, 1031, 1020, 1047, 1087, 1001, 1012, 1035, 1007, 1000, 992, 1021, 987, 1046, 1018, 1000, 1029, 994, 1048, 1004, 1002, 997, 1054, 1031, 973, 1066, 1013, 1021, 1055, 1026, 1050, 1024, 1017, 1066, 980, 995, 990, 1043, 1029, 1059, 992, 1040, 1049, 1037, 1015, 1005, 1071, 1021, 1030, 1021, 1007, 1024, 1009, 981, 1052, 972, 1004, 1053, 1028, 1028, 1068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[1015, 988, 994, 1042, 1045, 1062, 1005, 1014, 1020, 995, 1050, 1036, 1015, 967, 1001, 1039, 1027, 1083, 1010, 1035, 1021, 1027, 1066, 1027, 1091, 1026, 1011, 1050, 999, 990, 1038, 994, 1005, 1028, 1029, 1023, 1005, 1004, 1012, 1106, 1027, 964, 1020, 1033, 1000, 1023, 1023, 988, 1025, 980, 1052, 1012, 1063, 1057, 984, 1061, 1037, 1043, 1030, 1037, 1017, 1018, 1006, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1017, 1016, 1032, 1005, 1021, 1039, 1022, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1052, 1034, 1065, 1019, 960, 986, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1040, 992, 990, 1063, 1009, 1025, 1015, 28, 0, 0, 0, 0, 0, 0, 0, 1013, 1095, 1028, 1031, 1015, 1021, 1018, 1022, 4, 1, 0, 0, 0, 0, 0, 0, 1033, 1043, 1008, 976, 951, 996, 1021, 1038, 8, 5, 0, 0, 0, 0, 0, 0, 1053, 1062, 1032, 993, 1050, 1076, 1004, 1001, 8, 7, 0, 0, 0, 0, 0, 0, 1035, 1032, 1008, 1047, 1011, 1004, 1010, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 987, 1035, 1010, 998, 1041, 1029, 1001, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8220, 8211, 8248, 8079, 8286, 8181, 8146] avg=8192.0000 max=8286 max_rank=5 min=8079 min_rank=4 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32692 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[988, 1042, 1043, 1036, 1027, 1031, 1005, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 1031, 1004, 1005, 1005, 998, 1047, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1022, 1000, 1026, 999, 1049, 973, 1041, 27, 0, 0, 0, 0, 0, 0, 0, 1048, 1010, 1027, 1030, 1056, 975, 998, 1031, 4, 1, 0, 0, 0, 0, 0, 0, 1027, 1057, 1014, 976, 1025, 1022, 1033, 965, 8, 5, 0, 0, 0, 0, 0, 0, 1029, 1039, 1023, 1057, 1041, 1031, 1047, 1045, 8, 7, 0, 0, 0, 0, 0, 0, 1069, 1049, 972, 987, 978, 990, 1042, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1031, 990, 1061, 1048, 1046, 1014, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8128, 8118, 8180, 8132, 8327, 8145, 8280] avg=8192.0000 max=8327 max_rank=5 min=8118 min_rank=2 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1028, 989, 1008, 986, 1011, 1010, 975, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1037, 1065, 1064, 1026, 971, 1042, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1031, 1024, 995, 948, 1056, 989, 1006, 28, 0, 0, 0, 0, 0, 0, 0, 1000, 1067, 1026, 1050, 1008, 1019, 1014, 963, 5, 1, 0, 0, 0, 0, 0, 0, 1043, 1067, 1048, 985, 1026, 1010, 1069, 1025, 8, 5, 0, 0, 0, 0, 0, 0, 1019, 992, 1074, 987, 971, 1039, 1050, 984, 8, 8, 0, 0, 0, 0, 0, 0, 1049, 1029, 1029, 1081, 1015, 1048, 1061, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 988, 1030, 1036, 1068, 1034, 991, 1013, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8055, 8232, 8086, 8153, 8286, 8132, 8368, 8224] avg=8192.0000 max=8368 max_rank=6 min=8055 min_rank=0 imbalance_ratio=1.021484 max_load_reduction=57168 max_load_reduction_pct=0.872314 moved_to_echo=33010 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1008, 1017, 1017, 1024, 1035, 1043, 1020, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1043, 1002, 1014, 1031, 1049, 1043, 1045, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1018, 970, 1029, 1027, 1017, 1021, 998, 27, 0, 0, 0, 0, 0, 0, 0, 1001, 1004, 1028, 1069, 1007, 1025, 994, 1044, 5, 1, 0, 0, 0, 0, 0, 0, 1014, 1041, 1039, 1030, 1022, 1027, 981, 1054, 8, 5, 0, 0, 0, 0, 0, 0, 1024, 1013, 1017, 997, 1021, 1043, 1052, 991, 8, 7, 0, 0, 0, 0, 0, 0, 997, 1035, 1066, 970, 991, 1041, 1011, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 986, 1050, 1019, 1005, 1067, 1033, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8260, 8080, 8178, 8221, 8173, 8144, 8291] avg=8192.0000 max=8291 max_rank=7 min=8080 min_rank=2 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1013, 1024, 1028, 1074, 1039, 996, 1001, 985, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 985, 1086, 1033, 994, 1015, 1035, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1035, 1014, 1070, 1045, 1078, 1014, 1010, 28, 0, 0, 0, 0, 0, 0, 0, 984, 1084, 1008, 999, 1022, 983, 1029, 1074, 5, 1, 0, 0, 0, 0, 0, 0, 1024, 1025, 1030, 1059, 1033, 994, 1069, 1008, 8, 5, 0, 0, 0, 0, 0, 0, 982, 1085, 1051, 1004, 1020, 985, 1008, 967, 8, 7, 0, 0, 0, 0, 0, 0, 1017, 1002, 1047, 1029, 995, 1031, 1010, 996, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1028, 1072, 1011, 1021, 963, 1036, 994, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8245, 8311, 8189, 8255, 8117, 8127, 8132] avg=8192.0000 max=8311 max_rank=2 min=8117 min_rank=5 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32631 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[981, 988, 994, 1042, 1045, 1062, 1005, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 995, 1050, 1035, 1015, 967, 1001, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1083, 1010, 1035, 1021, 1027, 1066, 1027, 27, 0, 0, 0, 0, 0, 0, 0, 1091, 1026, 1011, 1050, 999, 990, 1038, 994, 5, 1, 0, 0, 0, 0, 0, 0, 1005, 1028, 1029, 1023, 1005, 1004, 1012, 1106, 8, 5, 0, 0, 0, 0, 0, 0, 1027, 964, 1020, 1033, 1000, 1023, 1023, 988, 8, 7, 0, 0, 0, 0, 0, 0, 1025, 980, 1052, 1012, 1063, 1044, 984, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1043, 1030, 1037, 1004, 1018, 1006, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8122, 8323, 8205, 8225, 8093, 8221, 8216] avg=8192.0000 max=8323 max_rank=2 min=8093 min_rank=5 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32755 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1011, 1042, 1049, 1061, 1061, 1017, 1072, 997, 0, 0, 0, 0, 0, 0, 0, 0, 973, 1042, 1005, 1059, 1053, 1042, 1035, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1063, 1026, 1025, 1005, 985, 1019, 1069, 27, 0, 0, 0, 0, 0, 0, 0, 1063, 1021, 1011, 993, 1000, 1023, 1006, 1040, 5, 1, 0, 0, 0, 0, 0, 0, 988, 1031, 996, 1035, 1013, 1033, 992, 1067, 8, 5, 0, 0, 0, 0, 0, 0, 1013, 1012, 995, 980, 1018, 1052, 1030, 1005, 8, 7, 0, 0, 0, 0, 0, 0, 1022, 1027, 1021, 1026, 1031, 1061, 995, 999, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1014, 995, 1008, 989, 1031, 997, 1067, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8277, 8235, 8163, 8168, 8120, 8182, 8081] avg=8192.0000 max=8310 max_rank=0 min=8081 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1005, 1055, 969, 1086, 1031, 1020, 1047, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1012, 1035, 1004, 1000, 992, 1021, 987, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 1018, 1000, 1029, 994, 1048, 1004, 1002, 31, 0, 0, 0, 0, 0, 0, 0, 997, 1054, 1031, 973, 1066, 1013, 1021, 1055, 7, 3, 0, 0, 0, 0, 0, 0, 1026, 1050, 1024, 1017, 1066, 980, 995, 990, 13, 9, 0, 0, 0, 0, 0, 0, 1043, 1029, 1059, 992, 1040, 1049, 1037, 1015, 11, 13, 0, 0, 0, 0, 0, 0, 1005, 1071, 1021, 1030, 1021, 987, 1024, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 981, 1052, 972, 1004, 1033, 1028, 1028, 1068, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8300, 8052, 8172, 8220, 8170, 8288, 8168, 8166] avg=8192.0000 max=8300 max_rank=0 min=8052 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8320, 8192, 8320, 8192, 8192, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8192, 8320, 8320, 8192, 8064, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8320, 8064, 8192, 8320, 8320, 8064, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8192, 8320, 8320, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8448, 8192, 8192, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8448, 8192, 8320, 8192, 8064, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[994, 1031, 985, 1044, 1040, 1088, 996, 1090, 1020, 1007, 1036, 988, 1037, 1086, 1022, 1047, 992, 1048, 1045, 1074, 995, 1028, 1066, 1034, 1031, 1017, 1053, 1021, 973, 1008, 997, 992, 1062, 1046, 991, 960, 991, 988, 1017, 1085, 1017, 1022, 992, 1027, 983, 1004, 1079, 1033, 1013, 1011, 1055, 987, 1054, 1035, 1042, 1014, 1039, 1007, 1042, 965, 1016, 988, 1059, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1017, 1040, 1010, 1024, 990, 1031, 1034, 1043, 1006, 1041, 1044, 1041, 1029, 1088, 1037, 972, 1036, 1016, 1023, 1035, 963, 993, 1025, 1030, 1043, 982, 995, 1023, 1076, 1044, 1070, 1033, 1012, 1007, 989, 1024, 1036, 1044, 1002, 1003, 963, 985, 1039, 1083, 991, 1041, 1044, 1014, 1004, 1068, 1017, 1022, 1044, 1075, 1015, 1071, 1012, 1026, 1026, 974, 1035, 998, 1016, 1022] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1012, 988, 1043, 1074, 1005, 1039, 1039, 1017, 996, 957, 1017, 1046, 1015, 1007, 1034, 1043, 1013, 1026, 980, 1028, 985, 1029, 1050, 1014, 1019, 1054, 1019, 1043, 1017, 1023, 1036, 1040, 956, 1008, 1016, 1008, 1040, 1036, 1073, 1040, 1037, 1026, 995, 1006, 1064, 1012, 1008, 1031, 1016, 1006, 1073, 1048, 1006, 1006, 1071, 1006, 1021, 1060, 998, 1031, 1029, 1039, 1051, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1046, 1013, 1034, 1048, 1013, 1019, 992, 986, 1020, 1023, 998, 997, 1046, 1064, 1030, 1051, 1035, 1023, 1003, 1023, 1069, 961, 937, 1011, 1031, 1010, 1071, 1027, 1029, 1031, 1043, 980, 1072, 1022, 978, 1047, 1029, 1026, 1018, 1013, 993, 1044, 975, 1054, 1017, 1057, 998, 1019, 1059, 1012, 1020, 1062, 1064, 1023, 1014, 1052, 1034, 1040, 1001, 1019, 1009, 1007, 1046, 1048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1004, 1064, 999, 1018, 1009, 1037, 1044, 1045, 1037, 1075, 1017, 1000, 992, 992, 1049, 1005, 999, 960, 992, 1052, 991, 1024, 1049, 985, 1064, 1029, 1041, 981, 993, 1057, 1029, 1026, 1006, 993, 1061, 1006, 1013, 1040, 986, 1048, 1054, 1037, 1039, 1046, 1038, 960, 988, 1011, 1008, 1034, 1044, 1055, 1078, 1056, 1017, 1060, 1013, 1082, 989, 1084, 1065, 973, 1013, 980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[973, 1043, 1024, 1000, 1033, 1041, 1022, 1033, 1005, 983, 1028, 1026, 1004, 1026, 1020, 961, 986, 1025, 1018, 1001, 1042, 1058, 1056, 1019, 958, 1053, 1069, 1006, 995, 1027, 1009, 1072, 1015, 1026, 1002, 1006, 1076, 1021, 1010, 1017, 1024, 1043, 988, 1032, 1007, 1034, 1043, 1052, 1061, 1050, 1044, 1000, 1044, 1009, 1067, 982, 1008, 1037, 999, 1076, 998, 1098, 1045, 1006] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1087, 989, 1076, 1020, 1009, 1050, 1048, 1018, 1051, 1041, 1008, 991, 1024, 1060, 1012, 1004, 972, 1010, 1021, 1023, 1033, 1055, 990, 1042, 1004, 1000, 1023, 1054, 973, 992, 1042, 1033, 1014, 1024, 1047, 967, 1025, 1026, 1026, 1049, 1069, 1026, 980, 989, 1000, 1053, 1023, 1040, 1039, 1058, 1055, 1014, 1004, 986, 990, 997, 1036, 1027, 1048, 1027, 1046, 1029, 1029, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1037, 1055, 1002, 939, 1028, 1005, 1058, 1028, 1080, 1032, 977, 996, 998, 1047, 1038, 1032, 984, 1031, 997, 1020, 1053, 1004, 1032, 1039, 1068, 985, 1026, 1092, 1008, 971, 1043, 1050, 943, 1010, 1054, 985, 1080, 1031, 1004, 1031, 1026, 1015, 1054, 1012, 1035, 1029, 981, 1029, 1043, 998, 993, 1005, 1026, 1040, 1025, 1022, 1045, 1002, 1019, 1034, 1042, 1052, 1087, 1029] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1012, 988, 1043, 1074, 1005, 1024, 1039, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 996, 957, 1017, 1046, 1015, 1007, 1034, 1043, 8, 1, 0, 0, 0, 0, 0, 0, 1013, 1026, 980, 1028, 985, 1029, 1050, 1014, 46, 0, 0, 0, 0, 0, 0, 0, 1019, 1054, 1011, 1043, 1017, 1023, 1036, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 956, 1008, 1016, 1008, 1040, 1036, 1073, 1040, 25, 6, 0, 0, 0, 0, 0, 0, 1037, 1026, 995, 1006, 1064, 1012, 1008, 1031, 14, 1, 0, 0, 0, 0, 0, 0, 1016, 1006, 1073, 1048, 954, 1006, 1071, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 1060, 998, 1031, 1029, 1039, 1025, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8124, 8171, 8243, 8208, 8194, 8180, 8214] avg=8192.0000 max=8243 max_rank=3 min=8124 min_rank=1 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1017, 1040, 1010, 1024, 990, 1016, 1034, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1041, 1044, 1041, 1029, 1088, 1037, 972, 8, 1, 0, 0, 0, 0, 0, 0, 1036, 1016, 1023, 1035, 963, 993, 1025, 1030, 48, 0, 0, 0, 0, 0, 0, 0, 1043, 982, 987, 1023, 1076, 1044, 1070, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1007, 989, 1024, 1036, 1044, 1002, 1003, 24, 6, 0, 0, 0, 0, 0, 0, 963, 985, 1039, 1083, 991, 1041, 1044, 1014, 14, 1, 0, 0, 0, 0, 0, 0, 1004, 1068, 1017, 1022, 990, 1075, 1015, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 1026, 1026, 974, 1035, 998, 991, 1022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8267, 8169, 8258, 8147, 8175, 8262, 8084] avg=8192.0000 max=8267 max_rank=1 min=8084 min_rank=7 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[994, 1031, 985, 1044, 1040, 1072, 996, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1007, 1036, 988, 1037, 1086, 1022, 1047, 8, 1, 0, 0, 0, 0, 0, 0, 992, 1048, 1045, 1074, 995, 1028, 1066, 1034, 48, 0, 0, 0, 0, 0, 0, 0, 1031, 1017, 1045, 1021, 973, 1008, 997, 992, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1046, 991, 960, 991, 988, 1017, 1085, 26, 6, 0, 0, 0, 0, 0, 0, 1017, 1022, 992, 1027, 983, 1004, 1079, 1033, 15, 1, 0, 0, 0, 0, 0, 0, 1013, 1011, 1055, 987, 1000, 1035, 1042, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1007, 1042, 965, 1016, 988, 1032, 1027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8252, 8330, 8084, 8172, 8173, 8157, 8116] avg=8192.0000 max=8330 max_rank=2 min=8084 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32618 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1046, 1013, 1034, 1048, 1013, 1004, 992, 986, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1023, 998, 997, 1046, 1064, 1030, 1051, 8, 1, 0, 0, 0, 0, 0, 0, 1035, 1023, 1003, 1023, 1069, 961, 937, 1011, 48, 0, 0, 0, 0, 0, 0, 0, 1031, 1010, 1063, 1027, 1029, 1031, 1043, 980, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1022, 978, 1047, 1029, 1026, 1018, 1013, 25, 6, 0, 0, 0, 0, 0, 0, 993, 1044, 975, 1054, 1017, 1057, 998, 1019, 14, 1, 0, 0, 0, 0, 0, 0, 1059, 1012, 1020, 1062, 1010, 1023, 1014, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1040, 1001, 1019, 1009, 1007, 1020, 1048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8238, 8110, 8214, 8236, 8172, 8252, 8178] avg=8192.0000 max=8252 max_rank=6 min=8110 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32838 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[973, 1043, 1024, 1000, 1033, 1026, 1022, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 983, 1028, 1026, 1004, 1026, 1020, 961, 8, 1, 0, 0, 0, 0, 0, 0, 986, 1025, 1018, 1001, 1042, 1058, 1056, 1019, 48, 0, 0, 0, 0, 0, 0, 0, 958, 1053, 1061, 1006, 995, 1027, 1009, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1026, 1002, 1006, 1076, 1021, 1010, 1017, 25, 6, 0, 0, 0, 0, 0, 0, 1024, 1043, 988, 1032, 1007, 1034, 1043, 1052, 14, 1, 0, 0, 0, 0, 0, 0, 1061, 1050, 1044, 1000, 990, 1009, 1067, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1037, 999, 1076, 998, 1098, 1019, 1006, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8062, 8253, 8181, 8204, 8238, 8203, 8241] avg=8192.0000 max=8253 max_rank=2 min=8062 min_rank=1 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32886 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1037, 1055, 1002, 939, 1028, 991, 1058, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 1032, 977, 996, 998, 1047, 1038, 1032, 8, 1, 0, 0, 0, 0, 0, 0, 984, 1031, 997, 1020, 1053, 1004, 1032, 1039, 47, 0, 0, 0, 0, 0, 0, 0, 1068, 985, 1018, 1092, 1008, 971, 1043, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 943, 1010, 1054, 985, 1080, 1031, 1004, 1031, 26, 6, 0, 0, 0, 0, 0, 0, 1026, 1015, 1054, 1012, 1035, 1029, 981, 1029, 13, 1, 0, 0, 0, 0, 0, 0, 1043, 998, 993, 1005, 973, 1040, 1025, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1002, 1019, 1034, 1042, 1052, 1060, 1029, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8209, 8207, 8235, 8170, 8195, 8099, 8283] avg=8192.0000 max=8283 max_rank=7 min=8099 min_rank=6 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1004, 1064, 999, 1018, 1009, 1022, 1044, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1075, 1017, 1000, 992, 992, 1049, 1005, 8, 1, 0, 0, 0, 0, 0, 0, 999, 960, 992, 1052, 991, 1024, 1049, 985, 49, 0, 0, 0, 0, 0, 0, 0, 1064, 1029, 1033, 981, 993, 1057, 1029, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 993, 1061, 1006, 1013, 1040, 986, 1048, 24, 6, 0, 0, 0, 0, 0, 0, 1054, 1037, 1039, 1046, 1038, 960, 988, 1011, 14, 1, 0, 0, 0, 0, 0, 0, 1008, 1034, 1044, 1055, 1023, 1056, 1017, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1082, 989, 1084, 1065, 973, 988, 980, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8176, 8101, 8212, 8183, 8188, 8297, 8174] avg=8192.0000 max=8297 max_rank=6 min=8101 min_rank=2 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32842 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1087, 989, 1076, 1020, 1009, 1028, 1048, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1041, 1008, 991, 1024, 1060, 1012, 1004, 12, 5, 0, 0, 0, 0, 0, 0, 972, 1010, 1021, 1023, 1033, 1055, 990, 1042, 49, 0, 0, 0, 0, 0, 0, 0, 1004, 1000, 1011, 1054, 973, 992, 1042, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1024, 1047, 967, 1025, 1026, 1026, 1049, 30, 8, 0, 0, 0, 0, 0, 0, 1069, 1026, 980, 989, 1000, 1053, 1023, 1040, 17, 4, 0, 0, 0, 0, 0, 0, 1039, 1058, 1055, 1014, 947, 986, 990, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1027, 1048, 1027, 1046, 1029, 995, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8208, 8195, 8109, 8216, 8201, 8086, 8246] avg=8192.0000 max=8275 max_rank=0 min=8086 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 8320, 8192, 8192, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8192, 8320, 8192, 8192, 8320, 8192, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8448, 8320, 8192, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8320, 8320, 8064, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8192, 8320, 8320, 8192, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8192, 8192, 8064, 8320, 8320, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8320, 8320, 8320, 7936, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8064, 8320, 8192, 8192, 8192, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1023, 1044, 1017, 1065, 1014, 1020, 1075, 1035, 1032, 1013, 1031, 1007, 1066, 1024, 1020, 1060, 1026, 1030, 1016, 1043, 1033, 1036, 963, 1064, 1040, 979, 1023, 1017, 1033, 1011, 1028, 1045, 1017, 994, 1056, 995, 1025, 1032, 1044, 1079, 1002, 1012, 994, 1031, 1020, 1000, 1020, 1025, 1015, 978, 992, 1042, 1002, 1026, 1017, 1062, 1010, 1017, 1016, 1031, 1017, 983, 1002, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1011, 989, 1050, 1064, 1039, 1045, 1105, 1028, 966, 1008, 1051, 997, 1044, 1009, 1037, 1023, 1065, 1007, 1024, 1027, 989, 1034, 1091, 1008, 1040, 1019, 1015, 1011, 1010, 978, 1046, 1061, 1085, 1011, 1062, 1008, 1047, 986, 993, 991, 1018, 979, 1007, 1029, 1019, 1041, 1047, 982, 1024, 1056, 1036, 1002, 1042, 982, 1024, 1042, 1033, 1031, 1068, 1007, 998, 997, 942, 1056] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1055, 1082, 1035, 989, 1004, 1015, 1041, 1030, 1011, 1015, 1047, 1048, 1045, 1030, 987, 994, 1003, 1039, 1060, 1000, 1024, 1025, 1030, 1033, 1015, 1033, 1023, 1001, 992, 1061, 1021, 1035, 989, 1039, 1073, 1032, 1059, 1001, 1004, 1041, 1007, 1060, 968, 1005, 1020, 1051, 982, 1033, 1049, 981, 1004, 1008, 1050, 1038, 1012, 1003, 1026, 1015, 1030, 1032, 985, 1083, 993, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[1084, 995, 1053, 1076, 1011, 1052, 1051, 1040, 996, 961, 1029, 1040, 1016, 1000, 1062, 1054, 1018, 1008, 962, 1066, 1046, 1022, 1029, 985, 1014, 1027, 1006, 973, 1039, 1056, 1013, 1002, 964, 973, 1007, 1032, 1048, 1040, 970, 1029, 1074, 1004, 1053, 989, 1041, 1070, 983, 1021, 1028, 1001, 1013, 981, 1072, 1044, 1022, 992, 1018, 1030, 1059, 1018, 1092, 990, 1081, 1011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1042, 1036, 1079, 1091, 1054, 977, 989, 983, 1013, 1015, 1034, 1045, 1047, 1009, 968, 1023, 930, 1032, 1024, 1041, 974, 1036, 992, 1080, 1019, 1011, 1082, 1038, 1001, 970, 1014, 1050, 1004, 1065, 1018, 987, 1043, 1019, 1030, 1038, 1000, 1051, 1038, 1043, 977, 1048, 976, 992, 1059, 999, 1024, 1038, 1024, 992, 1059, 995, 1023, 1056, 999, 1115, 1016, 1057, 1026, 1026] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1041, 1008, 1001, 1040, 970, 1014, 1021, 1041, 1052, 973, 981, 1000, 993, 1014, 1042, 1029, 1012, 1007, 1040, 1018, 1003, 1055, 1010, 1042, 1081, 1003, 997, 1018, 1043, 1009, 1030, 1020, 1032, 1049, 1018, 1021, 1084, 1098, 1071, 1033, 1029, 1044, 985, 991, 976, 1027, 1028, 1041, 1037, 1016, 1084, 1047, 1011, 1010, 1013, 1026, 1024, 975, 1062, 1006, 1028, 1053, 1010, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1060, 977, 1014, 1006, 1000, 1033, 1022, 1031, 1036, 1003, 1015, 985, 1026, 1052, 1023, 1003, 1044, 987, 1027, 1028, 1030, 1038, 1024, 994, 983, 1023, 1025, 995, 1040, 1047, 1035, 1050, 1023, 1034, 1043, 1036, 1027, 1044, 1065, 1059, 1039, 1006, 1000, 1054, 1075, 996, 1037, 951, 1013, 973, 1048, 1016, 1075, 1051, 1038, 1042, 1017, 976, 1027, 1014, 1010, 1007, 1030, 1054] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1029, 1006, 996, 993, 993, 1026, 1031, 991, 976, 1042, 1045, 1003, 1026, 1031, 1043, 1039, 1026, 1026, 1045, 960, 1049, 1092, 1043, 978, 1018, 1015, 995, 1029, 1058, 984, 1057, 1035, 929, 1027, 1038, 1070, 1051, 1016, 1033, 1068, 1059, 987, 1035, 1021, 1030, 1011, 1039, 1010, 1037, 1044, 1005, 1019, 1024, 991, 1031, 1025, 1003, 1063, 1049, 1027, 1059, 1009, 1022, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[988, 1044, 1017, 1065, 1014, 1020, 1075, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1013, 1031, 1007, 1066, 1024, 1020, 1060, 26, 1, 0, 0, 0, 0, 0, 0, 1026, 1030, 1016, 1043, 1033, 1036, 963, 1064, 5, 0, 0, 0, 0, 0, 0, 0, 1040, 979, 1023, 1017, 1033, 1011, 1028, 1045, 9, 1, 0, 0, 0, 0, 0, 0, 1017, 994, 1056, 995, 981, 1032, 1044, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1012, 994, 1031, 1020, 1000, 1020, 1025, 43, 0, 0, 0, 0, 0, 0, 0, 1015, 978, 992, 1042, 1002, 1026, 1017, 1062, 3, 0, 0, 0, 0, 0, 0, 0, 1010, 1017, 1007, 1031, 1017, 983, 1002, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8280, 8216, 8186, 8198, 8147, 8137, 8114] avg=8192.0000 max=8280 max_rank=1 min=8114 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[976, 989, 1050, 1064, 1039, 1045, 1105, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1008, 1051, 997, 1044, 1009, 1037, 1023, 26, 1, 0, 0, 0, 0, 0, 0, 1065, 1007, 1024, 1027, 989, 1034, 1091, 1008, 5, 0, 0, 0, 0, 0, 0, 0, 1040, 1019, 1015, 1011, 1010, 978, 1046, 1061, 9, 1, 0, 0, 0, 0, 0, 0, 1085, 1011, 1062, 1008, 1002, 986, 993, 991, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 979, 1007, 1029, 1019, 1041, 1047, 982, 44, 0, 0, 0, 0, 0, 0, 0, 1024, 1056, 1036, 1002, 1042, 982, 1024, 1042, 3, 0, 0, 0, 0, 0, 0, 0, 1033, 1031, 1059, 1007, 998, 997, 942, 1056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8162, 8250, 8190, 8138, 8166, 8211, 8123] avg=8192.0000 max=8296 max_rank=0 min=8123 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32638 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1019, 1082, 1035, 989, 1004, 1015, 1041, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 1015, 1047, 1048, 1045, 1030, 987, 994, 27, 1, 0, 0, 0, 0, 0, 0, 1003, 1039, 1060, 1000, 1024, 1025, 1030, 1033, 5, 0, 0, 0, 0, 0, 0, 0, 1015, 1033, 1023, 1001, 992, 1061, 1021, 1035, 9, 1, 0, 0, 0, 0, 0, 0, 989, 1039, 1073, 1032, 1014, 1001, 1004, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 1060, 968, 1005, 1020, 1051, 982, 1033, 44, 0, 0, 0, 0, 0, 0, 0, 1049, 981, 1004, 1008, 1050, 1038, 1012, 1003, 3, 0, 0, 0, 0, 0, 0, 0, 1026, 1015, 1021, 1032, 985, 1083, 993, 1040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8205, 8219, 8191, 8193, 8170, 8148, 8195] avg=8192.0000 max=8219 max_rank=2 min=8148 min_rank=6 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32706 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1005, 1008, 1001, 1040, 970, 1014, 1021, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 973, 981, 1000, 993, 1014, 1042, 1029, 27, 1, 0, 0, 0, 0, 0, 0, 1012, 1007, 1040, 1018, 1003, 1055, 1010, 1042, 5, 0, 0, 0, 0, 0, 0, 0, 1081, 1003, 997, 1018, 1043, 1009, 1030, 1020, 9, 1, 0, 0, 0, 0, 0, 0, 1032, 1049, 1018, 1021, 1038, 1098, 1071, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 1044, 985, 991, 976, 1027, 1028, 1041, 45, 0, 0, 0, 0, 0, 0, 0, 1037, 1016, 1084, 1047, 1011, 1010, 1013, 1026, 3, 0, 0, 0, 0, 0, 0, 0, 1024, 975, 1053, 1006, 1028, 1053, 1010, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8112, 8192, 8211, 8360, 8166, 8247, 8148] avg=8192.0000 max=8360 max_rank=4 min=8100 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32921 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1046, 995, 1053, 1076, 1011, 1052, 1051, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 996, 961, 1029, 1040, 1016, 1000, 1062, 1054, 28, 1, 0, 0, 0, 0, 0, 0, 1018, 1008, 962, 1066, 1046, 1022, 1029, 985, 5, 0, 0, 0, 0, 0, 0, 0, 1014, 1027, 1006, 973, 1039, 1056, 1013, 1002, 10, 1, 0, 0, 0, 0, 0, 0, 964, 973, 1007, 1032, 1003, 1040, 970, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1004, 1053, 989, 1041, 1070, 983, 1021, 44, 0, 0, 0, 0, 0, 0, 0, 1028, 1001, 1013, 981, 1072, 1044, 1022, 992, 3, 0, 0, 0, 0, 0, 0, 0, 1018, 1030, 1050, 1018, 1092, 990, 1081, 1011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 8187, 8141, 8141, 8018, 8279, 8156, 8290] avg=8192.0000 max=8324 max_rank=0 min=8018 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[994, 1006, 996, 993, 993, 1026, 1031, 991, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1042, 1045, 1003, 1026, 1031, 1043, 1039, 26, 1, 0, 0, 0, 0, 0, 0, 1026, 1026, 1045, 960, 1049, 1092, 1043, 978, 5, 0, 0, 0, 0, 0, 0, 0, 1018, 1015, 995, 1029, 1058, 984, 1057, 1035, 9, 1, 0, 0, 0, 0, 0, 0, 929, 1027, 1038, 1070, 1006, 1016, 1033, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 987, 1035, 1021, 1030, 1011, 1039, 1010, 44, 0, 0, 0, 0, 0, 0, 0, 1037, 1044, 1005, 1019, 1024, 991, 1031, 1025, 3, 0, 0, 0, 0, 0, 0, 0, 1003, 1063, 1040, 1027, 1059, 1009, 1022, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8030, 8232, 8224, 8201, 8187, 8236, 8179, 8247] avg=8192.0000 max=8247 max_rank=7 min=8030 min_rank=0 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1006, 1036, 1079, 1091, 1054, 977, 989, 983, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1015, 1034, 1045, 1047, 1009, 968, 1023, 27, 1, 0, 0, 0, 0, 0, 0, 930, 1032, 1024, 1041, 974, 1036, 992, 1080, 5, 0, 0, 0, 0, 0, 0, 0, 1019, 1011, 1082, 1038, 1001, 970, 1014, 1050, 9, 1, 0, 0, 0, 0, 0, 0, 1004, 1065, 1018, 987, 999, 1019, 1030, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1051, 1038, 1043, 977, 1048, 976, 992, 43, 0, 0, 0, 0, 0, 0, 0, 1059, 999, 1024, 1038, 1024, 992, 1059, 995, 3, 0, 0, 0, 0, 0, 0, 0, 1023, 1056, 990, 1115, 1016, 1057, 1026, 1026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8182, 8114, 8195, 8160, 8168, 8193, 8309] avg=8192.0000 max=8309 max_rank=7 min=8114 min_rank=2 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1015, 977, 1014, 1006, 1000, 1033, 1022, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1003, 1015, 985, 1026, 1052, 1023, 1003, 30, 3, 0, 0, 0, 0, 0, 0, 1044, 987, 1027, 1028, 1030, 1038, 1024, 994, 8, 0, 0, 0, 0, 0, 0, 0, 983, 1023, 1025, 995, 1040, 1047, 1035, 1050, 15, 8, 0, 0, 0, 0, 0, 0, 1023, 1034, 1043, 1036, 978, 1044, 1065, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1006, 1000, 1054, 1075, 996, 1037, 951, 46, 0, 0, 0, 0, 0, 0, 0, 1013, 973, 1048, 1016, 1075, 1051, 1038, 1042, 9, 0, 0, 0, 0, 0, 0, 0, 1017, 976, 1002, 1014, 1010, 1007, 1030, 1054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8176, 8180, 8221, 8282, 8204, 8265, 8110] avg=8192.0000 max=8282 max_rank=4 min=8098 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32861 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8192, 8320, 8192, 8064, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8320, 8320, 8192, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8320, 8192, 8192, 8448, 8192, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8192, 8320, 8192, 8320, 8192, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8192, 8064, 8320, 8320, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8064, 8192, 8320, 8448, 8192, 8192, 8448, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8064, 8320, 8192, 8320, 8192, 8192, 8320, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[972, 1019, 1045, 996, 1019, 1030, 949, 1000, 999, 1016, 1008, 1090, 1022, 1024, 1005, 1046, 1080, 1068, 1041, 1022, 988, 1040, 1023, 1020, 1025, 1031, 1019, 1049, 1029, 1019, 1051, 1004, 1058, 1010, 1055, 991, 1060, 1036, 999, 1022, 995, 998, 1096, 990, 971, 1035, 1021, 1037, 1038, 1020, 1001, 1026, 1003, 1039, 1038, 1017, 1036, 1049, 1030, 1042, 1026, 1024, 1025, 999] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[986, 1020, 1059, 992, 1040, 1042, 992, 991, 964, 1033, 1069, 985, 1031, 1000, 989, 1050, 1030, 1055, 998, 1017, 1032, 988, 1019, 1085, 1081, 999, 952, 1044, 1059, 1027, 1047, 1033, 1049, 1003, 996, 1027, 1030, 990, 1069, 1033, 1034, 1013, 1042, 1045, 980, 1026, 986, 1022, 1032, 1062, 1019, 1080, 1027, 974, 1017, 990, 1026, 994, 998, 1046, 1062, 1028, 1078, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1023, 1031, 969, 1008, 1037, 1045, 1060, 1005, 1020, 996, 1034, 1012, 1052, 1052, 1042, 1029, 1047, 1097, 1081, 1007, 1063, 1032, 996, 1025, 1061, 1031, 1053, 1043, 1011, 1020, 1047, 1048, 990, 1046, 1015, 1030, 1010, 1049, 1000, 962, 968, 982, 1042, 986, 1004, 1009, 1012, 1067, 961, 1024, 1036, 1010, 996, 1020, 1023, 1016, 1073, 978, 1114, 999, 1023, 1009, 1005, 1000] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1008, 987, 1027, 1006, 1059, 1052, 1028, 1034, 1046, 972, 996, 963, 1057, 1019, 1016, 1004, 1013, 1050, 1023, 1044, 1046, 1020, 1069, 1018, 995, 982, 1006, 1028, 1064, 1061, 975, 1015, 1017, 1015, 1036, 1023, 1022, 1029, 1031, 1025, 998, 1069, 1053, 1016, 999, 1010, 1040, 1001, 1090, 1002, 1004, 1045, 1020, 982, 1064, 1012, 1040, 1026, 990, 1020, 1100, 1028, 1018, 1028] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1024, 981, 1057, 1030, 1039, 1052, 1019, 1031, 1048, 1002, 987, 1011, 988, 1036, 1082, 1013, 1054, 1048, 1002, 990, 1010, 1034, 1034, 1060, 982, 1052, 1022, 1038, 1083, 983, 1032, 1032, 972, 1004, 1019, 1056, 1044, 1027, 1024, 1053, 1056, 1010, 986, 1060, 1023, 1010, 992, 1031, 1009, 1029, 1014, 1010, 1012, 1045, 994, 1018, 1018, 1008, 1020, 1000, 1015, 990, 1081, 1050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1003, 1051, 1019, 1038, 975, 1001, 1019, 1068, 1047, 1025, 1012, 1041, 1047, 1048, 1021, 1021, 1001, 1015, 1025, 1042, 1071, 1024, 1053, 1019, 974, 1062, 1002, 1008, 1029, 1003, 1050, 1016, 977, 1076, 1005, 1023, 1015, 1024, 1013, 1043, 1040, 1040, 1020, 994, 1022, 1044, 1021, 1011, 1031, 1013, 1039, 1043, 1038, 1022, 1000, 1009, 1051, 1046, 995, 1009, 1013, 1049, 1039, 941] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1066, 1020, 1036, 1052, 997, 992, 1054, 1038, 1034, 983, 1048, 1045, 1014, 1055, 1017, 1026, 994, 1044, 1017, 1005, 1017, 1023, 1023, 1026, 1033, 997, 1010, 996, 1000, 1030, 1020, 1013, 1004, 995, 1009, 1046, 1042, 1010, 1040, 1025, 1067, 1037, 1034, 999, 984, 1057, 1086, 1027, 1055, 1002, 1008, 997, 1037, 1024, 985, 1043, 1018, 1024, 1066, 975, 1056, 1026, 1000, 1033] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[999, 1002, 1015, 1005, 1054, 1038, 1011, 1040, 1074, 1059, 1019, 981, 1021, 1043, 1055, 1087, 1017, 1026, 1053, 1016, 1019, 987, 1041, 999, 1041, 998, 1010, 1051, 1025, 996, 1021, 1044, 1097, 1075, 985, 1076, 1052, 1009, 1025, 1032, 1039, 979, 1011, 982, 1002, 979, 1018, 1059, 1018, 999, 1054, 1025, 963, 1043, 1022, 997, 1006, 994, 1064, 1013, 1020, 1044, 1000, 1007] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[986, 1020, 1059, 992, 1040, 1042, 992, 991, 11, 7, 3, 0, 0, 0, 0, 0, 964, 1033, 1069, 985, 1031, 990, 989, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1007, 998, 1017, 1032, 988, 1019, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 999, 952, 1044, 1056, 1027, 1047, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 1003, 996, 1027, 1019, 990, 1069, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 1013, 1042, 1045, 980, 1026, 986, 1022, 33, 0, 0, 0, 0, 0, 0, 0, 1032, 1062, 1019, 1080, 1027, 974, 1017, 990, 15, 12, 3, 0, 0, 0, 0, 0, 1026, 994, 998, 1046, 1050, 1028, 1078, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8111, 8176, 8239, 8186, 8181, 8231, 8269] avg=8192.0000 max=8269 max_rank=7 min=8111 min_rank=1 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1023, 1031, 969, 1008, 1037, 1045, 1060, 1005, 10, 8, 3, 0, 0, 0, 0, 0, 1020, 996, 1034, 1012, 1052, 1041, 1042, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 1047, 1081, 1007, 1063, 1032, 996, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 1031, 1053, 1043, 1008, 1020, 1047, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1046, 1015, 1030, 1000, 1049, 1000, 962, 0, 0, 0, 0, 0, 0, 0, 0, 968, 982, 1042, 986, 1004, 1009, 1012, 1067, 35, 0, 0, 0, 0, 0, 0, 0, 961, 1024, 1036, 1010, 996, 1020, 1023, 1016, 15, 12, 3, 0, 0, 0, 0, 0, 1073, 978, 1114, 999, 1011, 1009, 1005, 1000, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8226, 8298, 8311, 8092, 8105, 8116, 8189] avg=8192.0000 max=8311 max_rank=3 min=8092 min_rank=4 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1008, 987, 1027, 1006, 1059, 1052, 1028, 1034, 10, 7, 3, 0, 0, 0, 0, 0, 1046, 972, 996, 963, 1057, 1009, 1016, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1002, 1023, 1044, 1046, 1020, 1069, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 995, 982, 1006, 1028, 1061, 1061, 975, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1015, 1036, 1023, 1012, 1029, 1031, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 998, 1069, 1053, 1016, 999, 1010, 1040, 1001, 33, 0, 0, 0, 0, 0, 0, 0, 1090, 1002, 1004, 1045, 1020, 982, 1064, 1012, 15, 12, 3, 0, 0, 0, 0, 0, 1040, 1026, 990, 1020, 1088, 1028, 1018, 1028, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8063, 8235, 8123, 8188, 8219, 8249, 8238] avg=8192.0000 max=8249 max_rank=6 min=8063 min_rank=1 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32894 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[972, 1019, 1045, 996, 1019, 1030, 949, 1000, 11, 7, 3, 0, 0, 0, 0, 0, 999, 1016, 1008, 1090, 1022, 1014, 1005, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 1019, 1041, 1022, 988, 1040, 1023, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1031, 1019, 1049, 1026, 1019, 1051, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1010, 1055, 991, 1049, 1036, 999, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 995, 998, 1096, 990, 971, 1035, 1021, 1037, 34, 0, 0, 0, 0, 0, 0, 0, 1038, 1020, 1001, 1026, 1003, 1039, 1038, 1017, 15, 12, 3, 0, 0, 0, 0, 0, 1036, 1049, 1030, 1042, 1014, 1024, 1025, 999, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8051, 8200, 8233, 8224, 8220, 8177, 8212, 8219] avg=8192.0000 max=8233 max_rank=2 min=8051 min_rank=0 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1066, 1020, 1036, 1052, 997, 992, 1054, 1038, 11, 8, 3, 0, 0, 0, 0, 0, 1034, 983, 1048, 1045, 1014, 1044, 1017, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 994, 996, 1017, 1005, 1017, 1023, 1023, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 997, 1010, 996, 997, 1030, 1020, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 995, 1009, 1046, 1031, 1010, 1040, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1037, 1034, 999, 984, 1057, 1086, 1027, 33, 0, 0, 0, 0, 0, 0, 0, 1055, 1002, 1008, 997, 1037, 1024, 985, 1043, 15, 12, 3, 0, 0, 0, 0, 0, 1018, 1024, 1066, 975, 1044, 1026, 1000, 1033, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8277, 8211, 8101, 8096, 8160, 8324, 8181, 8186] avg=8192.0000 max=8324 max_rank=5 min=8096 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32851 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1024, 981, 1057, 1030, 1039, 1052, 1019, 1031, 11, 8, 3, 0, 0, 0, 0, 0, 1048, 1002, 987, 1011, 988, 1025, 1082, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 1054, 1000, 1002, 990, 1010, 1034, 1034, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 982, 1052, 1022, 1038, 1080, 983, 1032, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1004, 1019, 1056, 1033, 1027, 1024, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1010, 986, 1060, 1023, 1010, 992, 1031, 33, 0, 0, 0, 0, 0, 0, 0, 1009, 1029, 1014, 1010, 1012, 1045, 994, 1018, 15, 11, 3, 0, 0, 0, 0, 0, 1018, 1008, 1020, 1000, 1004, 990, 1081, 1050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 8156, 8184, 8221, 8188, 8201, 8160, 8171] avg=8192.0000 max=8255 max_rank=0 min=8156 min_rank=1 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1003, 1051, 1019, 1038, 975, 1001, 1019, 1068, 10, 8, 3, 0, 0, 0, 0, 0, 1047, 1025, 1012, 1041, 1047, 1037, 1021, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 969, 1025, 1042, 1071, 1024, 1053, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 974, 1062, 1002, 1008, 1026, 1003, 1050, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 977, 1076, 1005, 1023, 1005, 1024, 1013, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1040, 1020, 994, 1022, 1044, 1021, 1011, 32, 0, 0, 0, 0, 0, 0, 0, 1031, 1013, 1039, 1043, 1038, 1022, 1000, 1009, 14, 11, 3, 0, 0, 0, 0, 0, 1051, 1046, 995, 1009, 1002, 1049, 1039, 941, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8251, 8204, 8141, 8166, 8224, 8223, 8132] avg=8192.0000 max=8251 max_rank=1 min=8132 min_rank=7 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[999, 1002, 1015, 1005, 1054, 1038, 1011, 1040, 15, 11, 5, 0, 0, 0, 0, 0, 1074, 1059, 1019, 981, 1021, 1022, 1055, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 973, 1053, 1016, 1019, 987, 1041, 999, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 998, 1010, 1051, 1020, 996, 1021, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 1075, 985, 1076, 1037, 1009, 1025, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 979, 1011, 982, 1002, 979, 1018, 1059, 36, 0, 0, 0, 0, 0, 0, 0, 1018, 999, 1054, 1025, 963, 1043, 1022, 997, 17, 16, 10, 0, 0, 0, 0, 0, 1006, 994, 1064, 1013, 1004, 1044, 1000, 1007, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8318, 8105, 8181, 8336, 8105, 8164, 8132] avg=8192.0000 max=8336 max_rank=4 min=8105 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8192, 8192, 8320, 8192, 8320, 8320, 8192, 8320, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8192, 8192, 8320, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8320, 8320, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8064, 8320, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8320, 8192, 8192, 8192, 8192, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8320, 8192, 8064, 8192, 8192, 8320, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66432 slack_slots=3840 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1026, 1036, 1003, 1033, 1013, 975, 1008, 982, 1010, 1072, 997, 1007, 1024, 1037, 1031, 1036, 1072, 1012, 1052, 1070, 1030, 959, 1055, 1005, 1052, 1042, 945, 1046, 1024, 978, 1031, 1014, 1030, 975, 1024, 940, 1080, 1018, 1023, 1008, 1002, 1037, 1046, 1023, 1028, 999, 1038, 1065, 997, 1057, 1051, 1045, 1059, 1040, 1006, 1059, 1089, 1010, 1050, 1025, 994, 962, 1030, 1049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1001, 984, 1019, 982, 1043, 1031, 984, 1028, 1053, 1002, 1033, 1058, 1033, 1065, 1023, 1020, 1044, 1036, 1028, 1063, 1000, 1061, 1049, 1024, 1015, 1043, 1006, 1050, 1068, 1029, 1000, 1038, 1027, 1027, 1013, 1062, 1010, 1009, 1025, 996, 997, 1062, 992, 1040, 1004, 1020, 1011, 1019, 1000, 988, 1028, 1019, 966, 1035, 1061, 1046, 1014, 1012, 1011, 1019, 1052, 1006, 1038, 1014] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1024, 1028, 1021, 1016, 1023, 1016, 964, 1011, 1028, 1026, 1007, 996, 1040, 998, 1026, 981, 1025, 1040, 1025, 1060, 1032, 1011, 1061, 1040, 1020, 1004, 1020, 984, 1013, 1033, 1049, 989, 1008, 1047, 994, 999, 963, 1062, 1026, 1015, 1023, 1043, 1104, 988, 1028, 1034, 986, 1051, 1050, 1023, 1052, 1004, 1034, 1055, 1039, 1031, 1041, 1036, 1072, 1077, 995, 991, 1049, 1005] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[997, 1055, 1048, 1033, 1001, 1017, 1038, 1012, 1015, 1043, 1015, 1045, 1021, 1043, 1067, 1056, 1024, 1025, 1013, 1046, 1016, 1015, 1029, 973, 1041, 1028, 1036, 1009, 994, 997, 997, 1038, 1018, 990, 1021, 1009, 1006, 1059, 1023, 1018, 994, 1014, 1050, 1095, 1035, 1014, 1065, 1033, 1032, 1028, 1043, 1028, 979, 1002, 1000, 991, 1038, 1022, 1032, 1017, 1026, 1029, 1015, 1023] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1029, 1047, 1003, 1025, 1061, 1041, 1046, 1007, 1008, 1066, 992, 1083, 1017, 1075, 1047, 1009, 1067, 959, 1018, 1002, 1065, 1003, 1040, 1035, 1035, 1030, 1025, 964, 1069, 1003, 1044, 978, 998, 1023, 1004, 1052, 1010, 1049, 1027, 1003, 1018, 1033, 1020, 1005, 1013, 1006, 996, 1024, 1026, 1054, 990, 998, 1017, 1063, 958, 983, 1038, 998, 1044, 1008, 1055, 1050, 1039, 1041] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[977, 1010, 1003, 984, 1020, 997, 1049, 1067, 1023, 1015, 1029, 1060, 1092, 994, 1017, 1008, 1014, 1021, 1013, 1037, 1032, 1015, 1058, 1022, 1040, 1026, 1007, 1067, 967, 986, 1018, 1029, 1041, 1024, 993, 1045, 1038, 1041, 1052, 974, 1040, 995, 1029, 999, 1036, 991, 1073, 1020, 992, 1042, 998, 1064, 977, 1015, 1030, 1028, 1024, 1032, 1031, 1052, 1034, 1019, 1086, 1024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1041, 1064, 979, 1033, 1040, 1059, 1045, 1039, 1011, 1056, 1038, 1032, 1079, 1005, 972, 1043, 1050, 1049, 1042, 1033, 997, 1054, 984, 1060, 1039, 1018, 1013, 1015, 1035, 990, 996, 1035, 982, 1031, 1005, 1005, 1012, 968, 1036, 1039, 1037, 1056, 1008, 1019, 998, 1015, 1060, 1043, 986, 1008, 1010, 1023, 1051, 1015, 957, 1064, 1053, 1001, 1005, 986, 1017, 1070, 1026, 1004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1019, 943, 991, 1018, 1031, 980, 1021, 1007, 1012, 1014, 993, 1004, 1033, 1018, 1029, 1051, 1023, 1090, 1022, 1006, 1059, 1021, 1063, 978, 1002, 975, 1014, 1028, 1048, 1016, 993, 1018, 1029, 975, 1021, 1045, 990, 1064, 988, 1040, 994, 1087, 1027, 1028, 975, 1003, 1007, 1035, 1067, 1029, 1070, 1033, 1029, 1076, 1044, 1022, 1041, 1032, 1065, 1015, 1081, 1050, 1023, 1031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1026, 1036, 1003, 1033, 1013, 975, 1008, 982, 49, 2, 0, 0, 0, 0, 0, 0, 1010, 1072, 997, 1007, 988, 1037, 1031, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1012, 1052, 1070, 1030, 959, 1006, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 1042, 945, 1046, 1024, 978, 1031, 1014, 31, 11, 3, 0, 0, 0, 0, 0, 1030, 975, 1024, 940, 1080, 1018, 1023, 1008, 47, 5, 0, 0, 0, 0, 0, 0, 1002, 1026, 1046, 1023, 1028, 999, 1038, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 997, 1057, 1051, 1045, 1059, 1037, 1006, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1010, 1050, 1025, 994, 962, 1030, 1049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8178, 8206, 8177, 8150, 8227, 8311, 8160] avg=8192.0000 max=8311 max_rank=6 min=8127 min_rank=0 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32848 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1029, 1047, 1003, 1025, 1061, 1041, 1046, 1007, 48, 2, 0, 0, 0, 0, 0, 0, 1008, 1066, 992, 1083, 981, 1075, 1047, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 959, 1018, 1002, 1065, 1003, 992, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1035, 1030, 1025, 964, 1069, 1003, 1044, 978, 31, 11, 3, 0, 0, 0, 0, 0, 998, 1023, 1004, 1052, 1010, 1049, 1027, 1003, 45, 5, 0, 0, 0, 0, 0, 0, 1018, 1022, 1020, 1005, 1013, 1006, 996, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1054, 990, 998, 1017, 1060, 958, 983, 0, 0, 0, 0, 0, 0, 0, 0, 991, 998, 1044, 1008, 1055, 1050, 1039, 1041, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8309, 8261, 8141, 8193, 8216, 8104, 8086, 8226] avg=8192.0000 max=8309 max_rank=0 min=8086 min_rank=6 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1001, 984, 1019, 982, 1043, 1031, 984, 1028, 49, 2, 0, 0, 0, 0, 0, 0, 1053, 1002, 1033, 1058, 997, 1065, 1023, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 1036, 1028, 1063, 1000, 1061, 1000, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 1043, 1006, 1050, 1068, 1029, 1000, 1038, 31, 11, 3, 0, 0, 0, 0, 0, 1027, 1027, 1013, 1062, 1010, 1009, 1025, 996, 44, 5, 0, 0, 0, 0, 0, 0, 997, 1051, 992, 1040, 1004, 1020, 1011, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 988, 1028, 1019, 966, 1032, 1061, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1012, 1011, 1019, 1052, 1006, 1038, 1014, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8251, 8256, 8294, 8218, 8134, 8140, 8120] avg=8192.0000 max=8294 max_rank=3 min=8120 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32612 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[977, 1010, 1003, 984, 1020, 997, 1049, 1067, 49, 2, 0, 0, 0, 0, 0, 0, 1023, 1015, 1029, 1060, 1054, 994, 1017, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 1021, 1013, 1037, 1032, 1015, 1009, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 1026, 1007, 1067, 967, 986, 1018, 1029, 33, 11, 3, 0, 0, 0, 0, 0, 1041, 1024, 993, 1045, 1038, 1041, 1052, 974, 44, 5, 0, 0, 0, 0, 0, 0, 1040, 984, 1029, 999, 1036, 991, 1073, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 992, 1042, 998, 1064, 977, 1012, 1030, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1032, 1031, 1052, 1034, 1019, 1086, 1024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8200, 8163, 8187, 8257, 8172, 8143, 8256] avg=8192.0000 max=8257 max_rank=4 min=8143 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1024, 1028, 1021, 1016, 1023, 1016, 964, 1011, 49, 2, 0, 0, 0, 0, 0, 0, 1028, 1026, 1007, 996, 1004, 998, 1026, 981, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 1040, 1025, 1060, 1032, 1011, 1012, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 1004, 1020, 984, 1013, 1033, 1049, 989, 31, 11, 3, 0, 0, 0, 0, 0, 1008, 1047, 994, 999, 963, 1062, 1026, 1015, 45, 5, 0, 0, 0, 0, 0, 0, 1023, 1032, 1104, 988, 1028, 1034, 986, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1023, 1052, 1004, 1034, 1052, 1039, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1036, 1072, 1077, 995, 991, 1049, 1005, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8066, 8245, 8157, 8164, 8246, 8285, 8219] avg=8192.0000 max=8285 max_rank=6 min=8066 min_rank=1 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32914 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[997, 1055, 1048, 1033, 1001, 1017, 1038, 1012, 52, 3, 0, 0, 0, 0, 0, 0, 1015, 1043, 1015, 1045, 979, 1043, 1067, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1025, 1013, 1046, 1016, 1015, 977, 973, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1028, 1036, 1009, 994, 997, 997, 1038, 35, 16, 9, 0, 0, 0, 0, 0, 1018, 990, 1021, 1009, 1006, 1059, 1023, 1018, 49, 7, 0, 0, 0, 0, 0, 0, 994, 998, 1050, 1095, 1035, 1014, 1065, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 1028, 1043, 1028, 979, 993, 1000, 991, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1022, 1032, 1017, 1026, 1029, 1015, 1023, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8256, 8263, 8089, 8200, 8200, 8284, 8094, 8150] avg=8192.0000 max=8284 max_rank=5 min=8089 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1041, 1064, 979, 1033, 1040, 1059, 1045, 1039, 46, 2, 0, 0, 0, 0, 0, 0, 1011, 1056, 1038, 1032, 1042, 1005, 972, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 1049, 1042, 1033, 997, 1054, 938, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 1018, 1013, 1015, 1035, 990, 996, 1035, 32, 11, 3, 0, 0, 0, 0, 0, 982, 1031, 1005, 1005, 1012, 968, 1036, 1039, 46, 5, 0, 0, 0, 0, 0, 0, 1037, 1045, 1008, 1019, 998, 1015, 1060, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1008, 1010, 1023, 1051, 1012, 957, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 1001, 1005, 986, 1017, 1070, 1026, 1004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8348, 8199, 8223, 8187, 8129, 8225, 8111, 8114] avg=8192.0000 max=8348 max_rank=0 min=8111 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32579 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1019, 943, 991, 1018, 1031, 980, 1021, 1007, 49, 2, 0, 0, 0, 0, 0, 0, 1012, 1014, 993, 1004, 997, 1018, 1029, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1090, 1022, 1006, 1059, 1021, 1014, 978, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 975, 1014, 1028, 1048, 1016, 993, 1018, 31, 12, 4, 0, 0, 0, 0, 0, 1029, 975, 1021, 1045, 990, 1064, 988, 1040, 45, 5, 0, 0, 0, 0, 0, 0, 994, 1075, 1027, 1028, 975, 1003, 1007, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 1067, 1029, 1070, 1033, 1029, 1072, 1044, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 994, 1032, 1065, 1015, 1081, 1050, 1023, 1031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8061, 8118, 8213, 8141, 8202, 8144, 8366, 8291] avg=8192.0000 max=8366 max_rank=6 min=8061 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=33003 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=70272 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8320, 8192, 8192, 8192, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=70272 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8064, 8192, 8320, 8320, 8320, 8192, 8320, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8320, 8192, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=70272 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 8192, 8192, 8192, 8192, 8320, 8320, 8192, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 8320, 8192, 8320, 8064, 8320, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=70272 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 8320, 8320, 8320, 8320, 8192, 8064, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 8320, 8320, 8320, 8192, 8192, 8320, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=70272 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8192, 8192, 8192, 8320, 8064, 8192, 8192, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=70272 valid_num_tokens=65920 slack_slots=4352 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=70272 valid_num_tokens=66304 slack_slots=3968 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=70272 valid_num_tokens=66048 slack_slots=4224 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=70272 valid_num_tokens=66176 slack_slots=4096 device_initiated_grouped_gemm=True + [2026-08-30 05:02:16] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 663.0 | throughput per GPU (TFLOP/s/GPU): 502.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.859732E+00 | loss scale: 1.0 | grad norm: 5.158 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 05:02:16 +[rank0]:[W830 05:02:17.780453468 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..81816ded0834119bae1258c6e9ae6bef226aba9f --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv @@ -0,0 +1,264 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 2 +0, 4 +1, 4 +2, 47 +3, 4 +4, 4 +5, 47 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1895 +1, 1895 +2, 1895 +3, 1895 +4, 1895 +5, 1895 +6, 1895 +7, 1895 +0, 38603 +1, 38595 +2, 38595 +3, 38595 +4, 38595 +5, 38595 +6, 38595 +7, 38595 +0, 39005 +1, 38993 +2, 38993 +3, 38993 +4, 38993 +5, 38993 +6, 38993 +7, 38993 +0, 39659 +1, 39643 +2, 39643 +3, 39643 +4, 39643 +5, 39643 +6, 39643 +7, 39643 +0, 41799 +1, 41783 +2, 41783 +3, 41783 +4, 41783 +5, 41783 +6, 41783 +7, 41783 +0, 42023 +1, 42007 +2, 42007 +3, 42007 +4, 42007 +5, 42007 +6, 42007 +7, 42007 +0, 42283 +1, 42257 +2, 42267 +3, 42267 +4, 42267 +5, 42267 +6, 42267 +7, 42267 +0, 42951 +1, 42935 +2, 42935 +3, 42935 +4, 42935 +5, 42935 +6, 42935 +7, 42935 +0, 43975 +1, 43959 +2, 43959 +3, 43959 +4, 43959 +5, 43959 +6, 43959 +7, 43959 +0, 45909 +1, 45893 +2, 45893 +3, 45893 +4, 45893 +5, 45893 +6, 45893 +7, 45893 +0, 47515 +1, 47499 +2, 47499 +3, 47499 +4, 47499 +5, 47499 +6, 47499 +7, 47499 +0, 91313 +1, 91383 +2, 91383 +3, 91383 +4, 91347 +5, 91383 +6, 91383 +7, 91383 +0, 91313 +1, 91383 +2, 91383 +3, 91383 +4, 91347 +5, 91383 +6, 91383 +7, 91383 +0, 92849 +1, 92919 +2, 92919 +3, 92919 +4, 92883 +5, 92919 +6, 92919 +7, 92919 +0, 92849 +1, 92919 +2, 92919 +3, 92919 +4, 92883 +5, 92919 +6, 92919 +7, 92919 +0, 92963 +1, 93033 +2, 93033 +3, 93033 +4, 92997 +5, 93033 +6, 93033 +7, 93033 +0, 109627 +1, 109609 +2, 109609 +3, 109613 +4, 109605 +5, 109613 +6, 109621 +7, 109639 +0, 110575 +1, 110519 +2, 110531 +3, 110493 +4, 110501 +5, 110551 +6, 110605 +7, 110629 +0, 111295 +1, 111277 +2, 111277 +3, 111277 +4, 111269 +5, 111277 +6, 111277 +7, 111277 +0, 111909 +1, 111853 +2, 111859 +3, 111885 +4, 111883 +5, 111913 +6, 111897 +7, 111921 +0, 135975 +1, 136111 +2, 136111 +3, 136091 +4, 135919 +5, 136091 +6, 136099 +7, 136099 +0, 135975 +1, 136111 +2, 136111 +3, 136091 +4, 135919 +5, 136091 +6, 136099 +7, 136099 +0, 135975 +1, 136111 +2, 136111 +3, 136091 +4, 135919 +5, 136091 +6, 136099 +7, 136099 +0, 135975 +1, 136111 +2, 136111 +3, 136091 +4, 135919 +5, 136091 +6, 136099 +7, 136099 +0, 135049 +1, 135185 +2, 135185 +3, 135165 +4, 134993 +5, 135165 +6, 135173 +7, 135173 +0, 2151 +1, 135185 +2, 3176 +3, 1896 +4, 1896 +5, 1896 +6, 1896 +7, 1896 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1/run.env b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..85e65076b3ea33847ec4086a21ea7e6e656139d0 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1/run.env @@ -0,0 +1,70 @@ +backend=echo +config=G3 +family=tierA +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=1792 +num_experts=64 +topk=16 +experts_per_rank=8 +expert_params=22020096 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=65536 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=2.00 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8d0dab45c0f62169d62a630d885b7ef06cfd87cf --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log @@ -0,0 +1,16635 @@ +[04:58:19] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G3 / echo (slots/rank=8 policy=matched) + E=64 K=16 H'=1792 m=8 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=2.00 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0830 04:58:20.427000 455382 torch/distributed/run.py:766] +W0830 04:58:20.427000 455382 torch/distributed/run.py:766] ***************************************** +W0830 04:58:20.427000 455382 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 04:58:20.427000 455382 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1792 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 2.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 16 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank [Gloo] Rank 0 is connected to 17 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 + +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : [Gloo] Rank 712 + is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W830 04:58:38.516128004 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 04:58:38.516523103 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 04:58:38.516523140 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 04:58:38.516600778 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 04:58:38.516632002 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 04:58:38.516727637 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 04:58:38.518570647 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 04:58:38.675336093 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.854 seconds +time to initialize megatron (seconds): 7.446 +[after megatron is initialized] datetime: 2026-08-30 04:58:44 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3906080768 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42217472 elements, 42217472 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight +Params for bucket 3 (42213376 elements, 42213376 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight +Params for bucket 4 (42213376 elements, 42213376 padded size): + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 5 (42213376 elements, 42213376 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.pre_mlp_layernorm.weight +Params for bucket 6 (42213376 elements, 42213376 padded size): + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.mlp.router.weight +Params for bucket 7 (42213376 elements, 42213376 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 8 (42213376 elements, 42213376 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight +Params for bucket 9 (42213376 elements, 42213376 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight +Params for bucket 10 (42213376 elements, 42213376 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 11 (42213376 elements, 42213376 padded size): + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.pre_mlp_layernorm.weight +Params for bucket 12 (42213376 elements, 42213376 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.weight +Params for bucket 13 (42213376 elements, 42213376 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 14 (42213376 elements, 42213376 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight +Params for bucket 15 (42213376 elements, 42213376 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight +Params for bucket 16 (42213376 elements, 42213376 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight +Params for bucket 17 (42213376 elements, 42213376 padded size): + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 64 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 +Params for bucket 6 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 +Params for bucket 10 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 +Params for bucket 17 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 +Params for bucket 18 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 24 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 28 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 +Params for bucket 30 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 +Params for bucket 34 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 35 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 +Params for bucket 42 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 +Params for bucket 43 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 47 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 48 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 49 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 +Params for bucket 55 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 56 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 61 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 +Params for bucket 62 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 63 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 04:58:44 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 04:58:44 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (506.90, 515.66) + train/valid/test-data-iterators-setup ..........: (64.66, 79.60) +[before the start of training step] datetime: 2026-08-30 04:58:44 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[133, 2389, 807, 497, 1358, 248, 1064, 879, 1318, 157, 1554, 721, 830, 707, 166, 471, 126, 756, 579, 1176, 1117, 178, 2861, 1928, 565, 889, 938, 820, 1189, 148, 1843, 1950, 2981, 489, 424, 493, 547, 2577, 1704, 738, 2678, 368, 1360, 1003, 1637, 1241, 1553, 851, 305, 612, 330, 2604, 926, 426, 615, 96, 669, 658, 1368, 1609, 2088, 326, 629, 1269] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[54, 3033, 551, 196, 1764, 593, 1155, 790, 1025, 198, 1617, 332, 994, 751, 226, 330, 78, 916, 610, 624, 1034, 266, 3312, 2027, 580, 1280, 925, 1225, 928, 339, 1562, 1948, 3153, 816, 287, 246, 96, 2396, 1717, 518, 2397, 606, 1591, 1216, 1468, 1787, 1365, 1544, 195, 624, 275, 1992, 725, 491, 432, 73, 925, 290, 1011, 1922, 1581, 523, 645, 1366] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[162, 3092, 875, 350, 1294, 649, 738, 1218, 788, 307, 1074, 482, 1187, 1028, 159, 416, 87, 555, 1541, 892, 908, 188, 2714, 1775, 843, 996, 1111, 1443, 1531, 337, 1836, 2115, 2817, 442, 333, 607, 276, 2134, 2028, 475, 2743, 288, 1602, 748, 1393, 1391, 935, 1328, 375, 441, 242, 1770, 1239, 598, 589, 105, 630, 306, 911, 1521, 1460, 705, 1200, 1213] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[90, 3010, 627, 240, 1266, 487, 833, 1247, 1199, 333, 1154, 185, 1014, 883, 327, 419, 130, 465, 352, 1330, 1101, 134, 2828, 1796, 326, 1157, 1184, 1121, 1515, 496, 1556, 2300, 3245, 522, 289, 1034, 182, 2576, 1882, 823, 2999, 432, 1242, 630, 1176, 1370, 1334, 1060, 190, 255, 248, 2332, 1494, 336, 337, 86, 675, 531, 1013, 1710, 1557, 424, 1529, 918] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[186, 2772, 1185, 281, 1241, 539, 1129, 1110, 1329, 200, 1003, 187, 1078, 657, 204, 480, 317, 403, 1303, 955, 1052, 166, 3023, 1644, 524, 886, 1327, 648, 1559, 648, 1597, 1693, 3163, 512, 181, 583, 213, 2202, 2457, 603, 3297, 347, 1183, 1200, 1570, 1392, 1291, 741, 257, 297, 316, 2146, 1230, 459, 741, 93, 672, 515, 1237, 1256, 1311, 784, 1153, 808] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[77, 3019, 747, 263, 1640, 686, 1102, 1055, 1108, 184, 1157, 214, 709, 611, 264, 572, 169, 764, 680, 1116, 944, 178, 3032, 1833, 523, 772, 1057, 808, 1383, 464, 1764, 2125, 3104, 403, 252, 495, 142, 2099, 2287, 663, 3129, 863, 1630, 495, 849, 1291, 1269, 1441, 258, 469, 369, 2170, 1015, 456, 543, 169, 422, 703, 799, 1770, 1923, 651, 1183, 1204] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[196, 3067, 638, 638, 1552, 621, 1102, 629, 1215, 145, 1974, 135, 1196, 1105, 216, 487, 138, 419, 1360, 802, 820, 502, 2978, 2015, 532, 880, 994, 1018, 1340, 290, 1644, 1491, 2993, 540, 268, 240, 251, 2830, 2070, 570, 2647, 229, 1356, 891, 1290, 1450, 1181, 1228, 349, 214, 450, 1723, 856, 946, 740, 71, 795, 572, 1322, 1690, 1600, 648, 360, 1027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[59, 2861, 979, 480, 1431, 556, 811, 937, 977, 167, 1101, 642, 1240, 612, 368, 516, 147, 481, 1240, 989, 1110, 126, 3048, 2161, 497, 1020, 856, 1326, 1337, 407, 2132, 2454, 2843, 851, 263, 524, 212, 2085, 1649, 339, 2467, 910, 1104, 1118, 1348, 1677, 1133, 1301, 277, 227, 624, 2183, 834, 394, 596, 98, 641, 191, 1132, 1641, 1450, 377, 797, 1182] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[54, 3033, 551, 196, 1764, 593, 1155, 790, 35, 0, 0, 0, 0, 0, 0, 0, 1025, 198, 1617, 332, 994, 751, 226, 330, 1259, 903, 552, 0, 0, 0, 0, 0, 78, 916, 610, 624, 1034, 266, 2651, 2027, 0, 0, 0, 0, 0, 0, 0, 0, 580, 1280, 925, 1225, 928, 339, 1562, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 1701, 816, 287, 246, 96, 2396, 1717, 518, 0, 0, 0, 0, 0, 0, 0, 0, 133, 606, 1591, 1216, 1468, 1787, 1365, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 195, 624, 275, 1992, 725, 491, 432, 73, 2264, 193, 0, 0, 0, 0, 0, 0, 925, 290, 1011, 1922, 1581, 523, 645, 1366, 74, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8187, 8206, 7884, 7777, 9710, 7264, 8337] avg=8192.0000 max=9710 max_rank=5 min=7264 min_rank=6 imbalance_ratio=1.185303 max_load_reduction=55826 max_load_reduction_pct=0.851837 moved_to_echo=33088 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[133, 2389, 807, 497, 1358, 248, 1064, 879, 31, 0, 0, 0, 0, 0, 0, 0, 1318, 157, 1554, 721, 830, 707, 166, 471, 1190, 904, 476, 0, 0, 0, 0, 0, 126, 756, 579, 1176, 1117, 178, 2290, 1928, 0, 0, 0, 0, 0, 0, 0, 0, 565, 889, 938, 820, 1189, 148, 1843, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 1609, 489, 424, 493, 547, 2577, 1704, 738, 0, 0, 0, 0, 0, 0, 0, 0, 149, 368, 1360, 1003, 1637, 1241, 1553, 851, 0, 0, 0, 0, 0, 0, 0, 0, 305, 612, 330, 2604, 926, 426, 615, 96, 2529, 182, 0, 0, 0, 0, 0, 0, 669, 658, 1368, 1609, 2088, 326, 629, 1269, 64, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7406, 8494, 8150, 7438, 8581, 8162, 8625, 8680] avg=8192.0000 max=8680 max_rank=7 min=7406 min_rank=0 imbalance_ratio=1.059570 max_load_reduction=56856 max_load_reduction_pct=0.867554 moved_to_echo=34048 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[186, 2772, 1185, 281, 1241, 539, 1129, 1110, 36, 0, 0, 0, 0, 0, 0, 0, 1329, 200, 1003, 187, 1078, 657, 204, 480, 1268, 789, 507, 0, 0, 0, 0, 0, 317, 403, 1303, 955, 1052, 166, 2407, 1644, 0, 0, 0, 0, 0, 0, 0, 0, 524, 886, 1327, 648, 1559, 648, 1597, 904, 0, 0, 0, 0, 0, 0, 0, 0, 1697, 512, 181, 583, 213, 2202, 2457, 603, 0, 0, 0, 0, 0, 0, 0, 0, 179, 347, 1183, 1200, 1570, 1392, 1291, 741, 0, 0, 0, 0, 0, 0, 0, 0, 257, 297, 316, 2146, 1230, 459, 741, 93, 3118, 198, 0, 0, 0, 0, 0, 0, 672, 515, 1237, 1256, 1311, 784, 1153, 808, 73, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8479, 7702, 8247, 8093, 8448, 7903, 8855, 7809] avg=8192.0000 max=8855 max_rank=6 min=7702 min_rank=1 imbalance_ratio=1.080933 max_load_reduction=56681 max_load_reduction_pct=0.864883 moved_to_echo=33015 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[77, 3019, 747, 263, 1640, 686, 1102, 1055, 32, 0, 0, 0, 0, 0, 0, 0, 1108, 184, 1157, 214, 709, 611, 264, 572, 1240, 985, 505, 0, 0, 0, 0, 0, 169, 764, 680, 1116, 944, 178, 2427, 1833, 0, 0, 0, 0, 0, 0, 0, 0, 523, 772, 1057, 808, 1383, 464, 1764, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 403, 252, 495, 142, 2099, 2287, 663, 0, 0, 0, 0, 0, 0, 0, 0, 174, 863, 1630, 495, 849, 1291, 1269, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 258, 469, 369, 2170, 1015, 456, 543, 169, 2955, 190, 0, 0, 0, 0, 0, 0, 422, 703, 799, 1770, 1923, 651, 1183, 1204, 68, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8621, 7549, 8111, 7911, 8015, 8012, 8594, 8723] avg=8192.0000 max=8723 max_rank=7 min=7549 min_rank=1 imbalance_ratio=1.064819 max_load_reduction=56813 max_load_reduction_pct=0.866898 moved_to_echo=33344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[196, 3067, 638, 638, 1552, 621, 1102, 629, 32, 0, 0, 0, 0, 0, 0, 0, 1215, 145, 1974, 135, 1196, 1105, 216, 487, 1195, 691, 496, 0, 0, 0, 0, 0, 138, 419, 1360, 802, 820, 502, 2383, 2015, 0, 0, 0, 0, 0, 0, 0, 0, 532, 880, 994, 1018, 1340, 290, 1644, 800, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 540, 268, 240, 251, 2830, 2070, 570, 0, 0, 0, 0, 0, 0, 0, 0, 147, 229, 1356, 891, 1290, 1450, 1181, 1228, 0, 0, 0, 0, 0, 0, 0, 0, 349, 214, 450, 1723, 856, 946, 740, 71, 2500, 183, 0, 0, 0, 0, 0, 0, 795, 572, 1322, 1690, 1600, 648, 360, 1027, 67, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8475, 8855, 8439, 7498, 8384, 7772, 8032, 8081] avg=8192.0000 max=8855 max_rank=1 min=7498 min_rank=3 imbalance_ratio=1.080933 max_load_reduction=56681 max_load_reduction_pct=0.864883 moved_to_echo=32269 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[90, 3010, 627, 240, 1266, 487, 833, 1247, 30, 0, 0, 0, 0, 0, 0, 0, 1199, 333, 1154, 185, 1014, 883, 327, 419, 1296, 1066, 471, 0, 0, 0, 0, 0, 130, 465, 352, 1330, 1101, 134, 2264, 1796, 0, 0, 0, 0, 0, 0, 0, 0, 326, 1157, 1184, 1121, 1515, 496, 1556, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 1751, 522, 289, 1034, 182, 2576, 1882, 823, 0, 0, 0, 0, 0, 0, 0, 0, 167, 432, 1242, 630, 1176, 1370, 1334, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 190, 255, 248, 2332, 1494, 336, 337, 86, 2832, 198, 0, 0, 0, 0, 0, 0, 675, 531, 1013, 1710, 1557, 424, 1529, 918, 63, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7830, 8347, 7572, 8589, 9059, 7411, 8308, 8420] avg=8192.0000 max=9059 max_rank=4 min=7411 min_rank=5 imbalance_ratio=1.105835 max_load_reduction=56477 max_load_reduction_pct=0.861771 moved_to_echo=33198 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[162, 3092, 875, 350, 1294, 649, 738, 1218, 29, 0, 0, 0, 0, 0, 0, 0, 788, 307, 1074, 482, 1187, 1028, 159, 416, 1125, 980, 452, 0, 0, 0, 0, 0, 87, 555, 1541, 892, 908, 188, 2172, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 843, 996, 1111, 1443, 1531, 337, 1836, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 1520, 442, 333, 607, 276, 2134, 2028, 475, 0, 0, 0, 0, 0, 0, 0, 0, 153, 288, 1602, 748, 1393, 1391, 935, 1328, 0, 0, 0, 0, 0, 0, 0, 0, 375, 441, 242, 1770, 1239, 598, 589, 105, 2590, 172, 0, 0, 0, 0, 0, 0, 630, 306, 911, 1521, 1460, 705, 1200, 1213, 61, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8407, 7998, 8118, 9232, 7815, 7838, 8121, 8007] avg=8192.0000 max=9232 max_rank=3 min=7815 min_rank=4 imbalance_ratio=1.126953 max_load_reduction=56304 max_load_reduction_pct=0.859131 moved_to_echo=31781 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[59, 2861, 979, 480, 1431, 556, 811, 937, 33, 0, 0, 0, 0, 0, 0, 0, 977, 167, 1101, 642, 1240, 612, 368, 516, 1135, 1138, 508, 0, 0, 0, 0, 0, 147, 481, 1240, 989, 1110, 126, 2439, 2161, 0, 0, 0, 0, 0, 0, 0, 0, 497, 1020, 856, 1326, 1337, 407, 2132, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 1534, 851, 263, 524, 212, 2085, 1649, 339, 0, 0, 0, 0, 0, 0, 0, 0, 137, 910, 1104, 1118, 1348, 1677, 1133, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 277, 227, 624, 2183, 834, 394, 596, 98, 2330, 174, 0, 0, 0, 0, 0, 0, 641, 191, 1132, 1641, 1450, 377, 797, 1182, 68, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8404, 8693, 8891, 7457, 8728, 7737, 7479] avg=8192.0000 max=8891 max_rank=3 min=7457 min_rank=4 imbalance_ratio=1.085327 max_load_reduction=56645 max_load_reduction_pct=0.864334 moved_to_echo=31401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2304, 3200, 2944, 17024, 8320, 4224, 4608, 896, 21120, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1280, 4096, 11136, 7424, 10752, 11648, 10112, 9600, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1024, 23296, 6528, 3072, 11648, 4480, 7936, 7936, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1280, 4864, 7680, 7936, 8192, 1792, 19072, 15232, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[13184, 4608, 2304, 4224, 1920, 18944, 15872, 4736, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5504, 3840, 8832, 13184, 13056, 4480, 7552, 9088, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4480, 7936, 8448, 8448, 10880, 3200, 13952, 8704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8960, 1792, 10752, 2944, 8320, 6400, 2048, 3712, 9728, 7552, 3968, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3380, 5, 5, 3675, 1, 234, 3820, 1, 4, 2719, 137, 41, 0, 241, 5, 24, 107, 3940, 391, 1574, 1, 1785, 0, 411, 3517, 334, 16, 1011, 4088, 1884, 0, 14, 3587, 1941, 1, 1855, 4059, 2806, 0, 508, 2177, 429, 117, 161, 0, 10, 97, 8, 2629, 995, 3729, 470, 183, 0, 0, 3, 6, 38, 0, 3, 0, 764, 3547, 2048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[3889, 4, 190, 2315, 849, 222, 4038, 9, 57, 2904, 2, 0, 3, 2, 81, 193, 9, 4077, 1475, 1083, 262, 193, 0, 732, 3611, 178, 0, 4024, 4016, 0, 0, 675, 3599, 56, 22, 3934, 2565, 680, 0, 16, 86, 0, 603, 3901, 5, 73, 67, 0, 2566, 1057, 4059, 4, 1493, 2, 0, 50, 0, 10, 437, 22, 1, 1025, 4090, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[3813, 187, 280, 3366, 239, 353, 4062, 1, 10, 3817, 0, 0, 1, 1, 0, 502, 29, 4023, 866, 940, 1837, 29, 12, 201, 3269, 42, 9, 3736, 3797, 41, 5, 13, 1420, 1818, 1, 4091, 1241, 3074, 0, 195, 25, 240, 186, 1476, 65, 102, 148, 148, 3647, 620, 1995, 40, 2, 0, 0, 11, 0, 0, 329, 274, 56, 3879, 3901, 1071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4075, 209, 306, 3404, 22, 437, 3773, 5, 78, 3665, 0, 222, 10, 540, 129, 0, 251, 4039, 506, 209, 437, 251, 0, 3307, 4017, 33, 0, 3224, 3030, 2650, 0, 0, 586, 1166, 0, 1752, 178, 1477, 0, 0, 245, 7, 58, 2481, 154, 103, 1955, 125, 3050, 1473, 3719, 95, 455, 0, 50, 0, 0, 268, 319, 197, 103, 871, 3817, 2003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3979, 36, 21, 2627, 396, 503, 3498, 7, 0, 2578, 0, 0, 0, 0, 0, 19, 2, 4032, 2743, 943, 307, 29, 0, 1292, 3088, 556, 2, 3417, 4076, 319, 0, 190, 2783, 228, 1, 4009, 2655, 103, 0, 5, 111, 1, 283, 3223, 160, 0, 178, 832, 3863, 3407, 4021, 9, 108, 0, 103, 0, 2, 0, 212, 171, 9, 293, 4086, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1725, 42, 13, 1599, 2286, 228, 4093, 0, 0, 2708, 0, 90, 1, 1, 22, 17, 7, 3793, 2338, 6, 66, 1685, 0, 1318, 2131, 36, 1, 4064, 4063, 0, 0, 2, 375, 5, 0, 3095, 3099, 1393, 234, 234, 242, 0, 2506, 3257, 0, 2286, 8, 7, 3141, 23, 3753, 0, 104, 1, 0, 0, 0, 0, 1755, 0, 0, 2794, 1353, 3536] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[3054, 1, 151, 3816, 5, 922, 4096, 0, 0, 3811, 0, 10, 0, 0, 8, 140, 0, 4026, 457, 409, 155, 2137, 3, 1004, 3127, 1, 596, 3275, 1646, 1, 0, 1018, 3324, 940, 38, 4089, 3131, 4043, 4, 3, 112, 16, 24, 3378, 97, 5, 31, 1081, 3530, 930, 2710, 26, 0, 0, 240, 0, 0, 8, 6, 183, 14, 178, 3491, 35] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[3963, 92, 298, 3564, 35, 95, 4074, 0, 0, 3158, 0, 16, 3, 5, 81, 0, 1, 4054, 2494, 909, 148, 1194, 0, 1791, 3126, 77, 0, 3159, 4034, 305, 0, 51, 2170, 453, 8, 3601, 2584, 2479, 0, 87, 623, 68, 26, 2679, 19, 37, 2234, 3, 3219, 518, 1540, 0, 12, 0, 1, 1, 25, 38, 519, 204, 0, 1518, 4092, 51] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4075, 209, 306, 3404, 22, 437, 553, 5, 0, 0, 0, 0, 0, 0, 0, 0, 78, 3665, 0, 222, 10, 540, 129, 0, 3220, 1120, 0, 0, 0, 0, 0, 0, 251, 3436, 506, 209, 437, 251, 0, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 33, 0, 3224, 495, 2650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 1166, 0, 277, 178, 1477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 7, 58, 2481, 154, 103, 1955, 125, 1415, 883, 0, 0, 0, 0, 0, 0, 3050, 1473, 3719, 95, 455, 0, 50, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 268, 319, 197, 103, 871, 3817, 2003, 592, 366, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9011, 8984, 8397, 10419, 3684, 7426, 9079, 8536] avg=8192.0000 max=10419 max_rank=3 min=3684 min_rank=4 imbalance_ratio=1.271851 max_load_reduction=55117 max_load_reduction_pct=0.841019 moved_to_echo=28725 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3380, 5, 5, 3675, 1, 234, 559, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2719, 137, 41, 0, 241, 5, 24, 3261, 1511, 0, 0, 0, 0, 0, 0, 107, 3352, 391, 1574, 1, 1785, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 3517, 334, 16, 1011, 668, 1884, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3587, 1941, 1, 293, 4059, 2806, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 2177, 429, 117, 161, 0, 10, 97, 8, 1909, 935, 0, 0, 0, 0, 0, 0, 2629, 995, 3729, 470, 183, 0, 0, 3, 231, 0, 0, 0, 0, 0, 0, 0, 6, 38, 0, 3, 0, 764, 3547, 2048, 627, 357, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7860, 7943, 7621, 7444, 13195, 5843, 8240, 7390] avg=8192.0000 max=13195 max_rank=4 min=5843 min_rank=5 imbalance_ratio=1.610718 max_load_reduction=52341 max_load_reduction_pct=0.798660 moved_to_echo=34668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3889, 4, 190, 2315, 849, 222, 591, 9, 0, 0, 0, 0, 0, 0, 0, 0, 57, 2904, 2, 0, 3, 2, 81, 193, 3447, 1484, 0, 0, 0, 0, 0, 0, 9, 3468, 1475, 1083, 262, 193, 0, 732, 0, 0, 0, 0, 0, 0, 0, 0, 3611, 178, 0, 4024, 656, 0, 0, 675, 0, 0, 0, 0, 0, 0, 0, 0, 3599, 56, 22, 621, 2565, 680, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 603, 3901, 5, 73, 67, 0, 1876, 1983, 0, 0, 0, 0, 0, 0, 2566, 1057, 4059, 4, 1493, 2, 0, 50, 239, 0, 0, 0, 0, 0, 0, 0, 0, 10, 437, 22, 1, 1025, 4090, 20, 1330, 370, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8069, 8173, 7222, 9144, 7559, 8594, 9470, 7305] avg=8192.0000 max=9470 max_rank=6 min=7222 min_rank=2 imbalance_ratio=1.156006 max_load_reduction=56066 max_load_reduction_pct=0.855499 moved_to_echo=32928 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1725, 42, 13, 1599, 2286, 228, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2708, 0, 90, 1, 1, 22, 17, 3498, 1505, 0, 0, 0, 0, 0, 0, 7, 3219, 2338, 6, 66, 1685, 0, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 2131, 36, 1, 4064, 656, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 375, 5, 0, 479, 3099, 1393, 234, 234, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 2506, 3257, 0, 2286, 8, 7, 1902, 1566, 0, 0, 0, 0, 0, 0, 3141, 23, 3753, 0, 104, 1, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1755, 0, 0, 2794, 1353, 3536, 1050, 348, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6488, 7842, 8639, 6890, 5819, 11774, 7248, 10836] avg=8192.0000 max=11774 max_rank=5 min=5819 min_rank=4 imbalance_ratio=1.437256 max_load_reduction=53762 max_load_reduction_pct=0.820343 moved_to_echo=35677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3979, 36, 21, 2627, 396, 503, 512, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2578, 0, 0, 0, 0, 0, 19, 2986, 1506, 0, 0, 0, 0, 0, 0, 2, 3430, 2743, 943, 307, 29, 0, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 3088, 556, 2, 3417, 666, 319, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 2783, 228, 1, 632, 2655, 103, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 111, 1, 283, 3223, 160, 0, 178, 832, 1904, 2021, 0, 0, 0, 0, 0, 0, 3863, 3407, 4021, 9, 108, 0, 103, 0, 236, 0, 0, 0, 0, 0, 0, 0, 2, 0, 212, 171, 9, 293, 4086, 20, 1356, 366, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8081, 7089, 8746, 8238, 6407, 8713, 11747, 6515] avg=8192.0000 max=11747 max_rank=6 min=6407 min_rank=4 imbalance_ratio=1.433960 max_load_reduction=53789 max_load_reduction_pct=0.820755 moved_to_echo=33382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3813, 187, 280, 3366, 239, 353, 595, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3817, 0, 0, 1, 1, 0, 502, 3467, 1403, 0, 0, 0, 0, 0, 0, 29, 3422, 866, 940, 1837, 29, 12, 201, 0, 0, 0, 0, 0, 0, 0, 0, 3269, 42, 9, 3736, 621, 41, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 1818, 1, 646, 1241, 3074, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 25, 240, 186, 1476, 65, 102, 148, 148, 1773, 2062, 0, 0, 0, 0, 0, 0, 3647, 620, 1995, 40, 2, 0, 0, 11, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 274, 56, 3879, 3901, 1071, 1383, 365, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8834, 9201, 7336, 7736, 8395, 6225, 6551, 11258] avg=8192.0000 max=11258 max_rank=7 min=6225 min_rank=5 imbalance_ratio=1.374268 max_load_reduction=54278 max_load_reduction_pct=0.828217 moved_to_echo=32429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3054, 1, 151, 3816, 5, 922, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3811, 0, 10, 0, 0, 8, 140, 3496, 608, 0, 0, 0, 0, 0, 0, 0, 3425, 457, 409, 155, 2137, 3, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 3127, 1, 596, 3275, 270, 1, 0, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 3324, 940, 38, 645, 3131, 4043, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 112, 16, 24, 3378, 97, 5, 31, 1081, 768, 2061, 0, 0, 0, 0, 0, 0, 3530, 930, 2710, 26, 0, 0, 240, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 183, 14, 178, 3491, 35, 1383, 365, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8549, 8073, 7590, 8288, 12128, 7573, 7672, 5663] avg=8192.0000 max=12128 max_rank=4 min=5663 min_rank=7 imbalance_ratio=1.480469 max_load_reduction=53408 max_load_reduction_pct=0.814941 moved_to_echo=33036 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3963, 92, 298, 3564, 35, 95, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3158, 0, 16, 3, 5, 81, 0, 3477, 1491, 0, 0, 0, 0, 0, 0, 1, 3448, 2494, 909, 148, 1194, 0, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 3126, 77, 0, 3159, 659, 305, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 2170, 453, 8, 568, 2584, 2479, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 623, 68, 26, 2679, 19, 37, 2234, 3, 1884, 1815, 0, 0, 0, 0, 0, 0, 3219, 518, 1540, 0, 12, 0, 1, 1, 238, 0, 0, 0, 0, 0, 0, 0, 25, 38, 519, 204, 0, 1518, 4092, 51, 1218, 368, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8644, 8231, 9985, 7377, 8349, 9388, 5529, 8033] avg=8192.0000 max=9985 max_rank=2 min=5529 min_rank=6 imbalance_ratio=1.218872 max_load_reduction=55551 max_load_reduction_pct=0.847641 moved_to_echo=31299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[17920, 6656, 128, 4224, 19584, 16128, 256, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25984, 1280, 640, 25984, 4736, 5248, 128, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[512, 27264, 11392, 6144, 3328, 7424, 128, 10112, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3712, 768, 3840, 20608, 512, 2688, 4736, 2304, 13440, 13440, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[27904, 640, 1280, 24448, 3840, 3072, 4608, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 25472, 256, 384, 128, 896, 384, 896, 26880, 10752, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 384, 3584, 1152, 256, 11392, 28416, 8832, 8960, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[25728, 9088, 25600, 768, 2432, 128, 512, 128, 1920, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[42, 7, 543, 163, 834, 1, 3679, 4095, 3984, 6, 885, 20, 0, 0, 108, 0, 16, 5, 0, 3407, 0, 396, 0, 3041, 524, 1194, 0, 470, 1013, 20, 4056, 80, 3998, 0, 3755, 3, 158, 1508, 0, 0, 0, 2667, 25, 3920, 0, 0, 3381, 1337, 0, 0, 6, 4058, 120, 3273, 0, 511, 0, 4080, 39, 897, 3159, 0, 52, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4074, 0, 4096, 204, 136, 39, 180, 3449, 14, 48, 3079, 0, 0, 615, 0, 70, 99, 0, 258, 2719, 6, 4069, 0, 4075, 0, 51, 1, 0, 1445, 3870, 4076, 332, 357, 5, 4065, 3, 2, 2931, 0, 44, 2397, 973, 52, 4076, 1, 0, 0, 283, 14, 0, 0, 49, 712, 3, 1, 0, 0, 4030, 0, 3165, 3619, 13, 50, 1686] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4002, 2814, 4096, 2, 18, 0, 37, 4096, 4096, 0, 0, 15, 0, 628, 0, 3961, 0, 11, 111, 1337, 85, 4051, 630, 1554, 9, 0, 1, 1, 1792, 361, 515, 0, 159, 849, 3025, 43, 285, 301, 0, 0, 3634, 0, 0, 3727, 0, 1, 2178, 1195, 0, 0, 0, 705, 3889, 0, 369, 0, 0, 4021, 0, 2, 4050, 0, 0, 2880] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3979, 1630, 2519, 103, 0, 0, 546, 3241, 343, 54, 1890, 0, 9, 60, 0, 49, 2, 709, 851, 606, 3, 58, 1, 1161, 0, 0, 0, 3586, 16, 136, 3707, 1162, 0, 3745, 3529, 16, 6, 524, 4, 129, 3983, 30, 0, 3440, 0, 0, 3118, 3995, 69, 0, 0, 25, 4089, 0, 50, 72, 0, 4087, 0, 4085, 4096, 8, 15, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3745, 0, 2498, 2, 28, 7, 2950, 4094, 62, 0, 3, 0, 5, 1, 22, 5, 1, 0, 792, 136, 2, 3728, 6, 3961, 126, 13, 3960, 77, 0, 1161, 1637, 2387, 0, 303, 4009, 3, 1, 83, 0, 2, 380, 4056, 713, 3641, 3, 0, 3352, 3270, 0, 0, 9, 50, 3653, 2, 22, 31, 0, 4093, 61, 2229, 4079, 51, 31, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1286, 7, 4088, 235, 99, 17, 4001, 4090, 3325, 5, 176, 0, 1, 0, 293, 0, 56, 4, 19, 8, 6, 249, 0, 4092, 0, 0, 0, 8, 770, 3305, 4089, 0, 1, 77, 4064, 704, 13, 4077, 1, 0, 2681, 3957, 57, 3935, 1, 1, 3982, 57, 2, 2, 60, 277, 2158, 23, 90, 0, 1, 3680, 367, 1022, 4017, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3378, 0, 3687, 37, 3149, 0, 3166, 4062, 3964, 5, 220, 999, 0, 2, 583, 85, 5, 0, 1935, 4, 8, 925, 8, 787, 0, 1, 375, 2340, 1192, 0, 3808, 41, 765, 0, 3996, 350, 0, 0, 0, 3, 1, 3342, 10, 4004, 74, 1, 4096, 2396, 0, 19, 25, 199, 788, 147, 0, 0, 0, 3110, 1067, 2472, 3522, 25, 54, 304] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4089, 0, 3726, 1, 100, 2, 0, 3468, 3171, 2, 19, 1, 8, 1, 0, 431, 3134, 194, 0, 603, 1, 4069, 0, 4074, 36, 0, 35, 0, 0, 16, 4088, 1, 155, 4, 4064, 1, 0, 3800, 0, 74, 4094, 367, 0, 4096, 0, 0, 4007, 2725, 109, 147, 231, 101, 1173, 7, 351, 33, 0, 2109, 0, 3273, 3345, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3378, 0, 2446, 37, 3149, 0, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 5, 220, 999, 0, 2, 583, 85, 3136, 1012, 0, 0, 0, 0, 0, 0, 5, 0, 1935, 4, 8, 925, 8, 787, 1246, 19, 0, 0, 0, 0, 0, 0, 0, 1, 375, 2340, 1192, 0, 3808, 41, 1117, 0, 0, 0, 0, 0, 0, 0, 765, 0, 3996, 350, 0, 0, 0, 3, 1039, 105, 0, 0, 0, 0, 0, 0, 1, 3342, 10, 0, 74, 1, 2850, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 25, 199, 788, 147, 0, 0, 4004, 926, 0, 0, 0, 0, 0, 0, 0, 3110, 1067, 2472, 1471, 25, 54, 304, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12176, 10006, 4937, 8874, 6258, 8674, 6108, 8503] avg=8192.0000 max=12176 max_rank=0 min=4937 min_rank=2 imbalance_ratio=1.486328 max_load_reduction=53360 max_load_reduction_pct=0.814209 moved_to_echo=29543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 2472, 1, 100, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3171, 2, 19, 1, 8, 1, 0, 431, 2678, 961, 0, 0, 0, 0, 0, 0, 3134, 194, 0, 603, 1, 4069, 0, 4074, 1219, 19, 0, 0, 0, 0, 0, 0, 36, 0, 35, 0, 0, 16, 4088, 1, 1129, 0, 0, 0, 0, 0, 0, 0, 155, 4, 4064, 1, 0, 3800, 0, 74, 987, 106, 0, 0, 0, 0, 0, 0, 4094, 367, 0, 0, 0, 0, 2788, 2725, 0, 0, 0, 0, 0, 0, 0, 0, 109, 147, 231, 101, 1173, 7, 351, 33, 4096, 790, 0, 0, 0, 0, 0, 0, 0, 2109, 0, 3273, 1397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6664, 7272, 13313, 5305, 9191, 9974, 7038, 6779] avg=8192.0000 max=13313 max_rank=2 min=5305 min_rank=3 imbalance_ratio=1.625122 max_load_reduction=52223 max_load_reduction_pct=0.796860 moved_to_echo=32982 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[42, 7, 362, 163, 834, 1, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 6, 885, 20, 0, 0, 108, 0, 3161, 908, 0, 0, 0, 0, 0, 0, 16, 5, 0, 3407, 0, 396, 0, 3041, 1028, 2, 0, 0, 0, 0, 0, 0, 524, 1194, 0, 470, 1013, 20, 4056, 80, 164, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 3755, 3, 158, 1508, 0, 0, 932, 15, 0, 0, 0, 0, 0, 0, 0, 2667, 25, 0, 0, 0, 2353, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4058, 120, 3273, 0, 511, 3920, 934, 0, 0, 0, 0, 0, 0, 0, 4080, 39, 897, 1319, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5088, 9072, 7895, 7521, 10369, 6382, 12822, 6387] avg=8192.0000 max=12822 max_rank=6 min=5088 min_rank=0 imbalance_ratio=1.565186 max_load_reduction=52714 max_load_reduction_pct=0.804352 moved_to_echo=35960 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4074, 0, 2717, 204, 136, 39, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 48, 3079, 0, 0, 615, 0, 70, 2663, 1040, 0, 0, 0, 0, 0, 0, 99, 0, 258, 2719, 6, 4069, 0, 4075, 0, 21, 0, 0, 0, 0, 0, 0, 0, 51, 1, 0, 1445, 3870, 4076, 332, 1241, 0, 0, 0, 0, 0, 0, 0, 357, 5, 4065, 3, 2, 2931, 0, 44, 1067, 117, 0, 0, 0, 0, 0, 0, 2397, 973, 52, 0, 1, 0, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 49, 712, 3, 1, 0, 4076, 786, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 3165, 1512, 13, 50, 1686, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7350, 7529, 11247, 11016, 8591, 3706, 5641, 10456] avg=8192.0000 max=11247 max_rank=2 min=3706 min_rank=5 imbalance_ratio=1.372925 max_load_reduction=54289 max_load_reduction_pct=0.828384 moved_to_echo=28394 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4002, 2814, 2717, 2, 18, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 628, 0, 3961, 3163, 1164, 0, 0, 0, 0, 0, 0, 0, 11, 111, 1337, 85, 4051, 630, 1554, 662, 21, 0, 0, 0, 0, 0, 0, 9, 0, 1, 1, 1792, 361, 515, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 159, 849, 3025, 43, 285, 301, 0, 0, 1195, 117, 0, 0, 0, 0, 0, 0, 3634, 0, 0, 0, 0, 1, 1516, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 705, 3889, 0, 369, 0, 3727, 933, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 2, 1691, 0, 0, 2880, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9590, 13027, 8462, 3920, 5974, 6346, 9623, 8594] avg=8192.0000 max=13027 max_rank=1 min=3920 min_rank=3 imbalance_ratio=1.590210 max_load_reduction=52509 max_load_reduction_pct=0.801224 moved_to_echo=30537 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3979, 1630, 1662, 103, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 54, 1890, 0, 9, 60, 0, 49, 2503, 1180, 0, 0, 0, 0, 0, 0, 2, 709, 851, 606, 3, 58, 1, 1161, 952, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3586, 16, 136, 3707, 1162, 766, 0, 0, 0, 0, 0, 0, 0, 0, 3745, 3529, 16, 6, 524, 4, 129, 1211, 75, 0, 0, 0, 0, 0, 0, 3983, 30, 0, 0, 0, 0, 2166, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 25, 4089, 0, 50, 72, 3440, 738, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4085, 1705, 8, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7920, 6088, 4359, 9373, 9239, 10174, 8483, 9900] avg=8192.0000 max=10174 max_rank=5 min=4359 min_rank=2 imbalance_ratio=1.241943 max_load_reduction=55362 max_load_reduction_pct=0.844757 moved_to_echo=37796 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3745, 0, 1659, 2, 28, 7, 2950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 3, 0, 5, 1, 22, 5, 3161, 1172, 0, 0, 0, 0, 0, 0, 1, 0, 792, 136, 2, 3728, 6, 3961, 1019, 12, 0, 0, 0, 0, 0, 0, 126, 13, 3960, 77, 0, 1161, 1637, 2387, 756, 0, 0, 0, 0, 0, 0, 0, 0, 303, 4009, 3, 1, 83, 0, 2, 1203, 71, 0, 0, 0, 0, 0, 0, 380, 4056, 713, 0, 3, 0, 2333, 3270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 50, 3653, 2, 22, 31, 3641, 933, 0, 0, 0, 0, 0, 0, 0, 4093, 61, 2229, 1704, 51, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8391, 4431, 9657, 10117, 5675, 10755, 8341, 8169] avg=8192.0000 max=10755 max_rank=5 min=4431 min_rank=1 imbalance_ratio=1.312866 max_load_reduction=54781 max_load_reduction_pct=0.835892 moved_to_echo=32940 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1286, 7, 2712, 235, 99, 17, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3325, 5, 176, 0, 1, 0, 293, 0, 3157, 1154, 0, 0, 0, 0, 0, 0, 56, 4, 19, 8, 6, 249, 0, 4092, 1211, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 770, 3305, 4089, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 1, 77, 4064, 704, 13, 4077, 1, 0, 1185, 117, 0, 0, 0, 0, 0, 0, 2681, 3957, 57, 0, 1, 1, 2771, 57, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 60, 277, 2158, 23, 90, 0, 3935, 933, 0, 0, 0, 0, 0, 0, 1, 3680, 367, 1022, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8357, 8111, 5666, 9410, 10239, 9525, 7480, 6748] avg=8192.0000 max=10239 max_rank=4 min=5666 min_rank=2 imbalance_ratio=1.249878 max_load_reduction=55297 max_load_reduction_pct=0.843765 moved_to_echo=33992 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[17280, 15488, 896, 0, 128, 128, 16896, 15360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3328, 1024, 3968, 8832, 128, 17664, 768, 22784, 7424, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[19072, 128, 6272, 1152, 128, 1408, 1024, 4608, 23680, 8704, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5504, 4992, 30592, 1152, 512, 13312, 128, 256, 8832, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[24704, 4480, 16768, 768, 4480, 128, 14592, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 29312, 1536, 17152, 12544, 128, 256, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[768, 1280, 4480, 6528, 6272, 8960, 25984, 4096, 7680, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 256, 384, 5504, 16640, 3456, 896, 768, 30848, 7040, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1952, 1, 8, 29, 0, 0, 649, 0, 26, 1468, 249, 0, 1, 3408, 3, 4060, 338, 0, 244, 3, 174, 0, 1136, 12, 0, 4096, 0, 0, 0, 15, 0, 0, 23, 3961, 11, 185, 4022, 134, 4092, 13, 4046, 4089, 3, 0, 2868, 219, 0, 0, 0, 1197, 3953, 0, 4095, 0, 4096, 3743, 2484, 0, 932, 0, 3445, 53] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[11, 0, 4037, 5, 11, 0, 0, 3770, 0, 3421, 1013, 0, 157, 0, 0, 4058, 8, 2937, 0, 214, 0, 622, 0, 2248, 0, 178, 1, 4050, 0, 1, 6, 2135, 20, 27, 0, 4096, 0, 0, 280, 0, 3283, 330, 135, 1181, 36, 11, 3357, 0, 4083, 54, 0, 680, 0, 0, 2777, 6, 4096, 3518, 137, 0, 1857, 0, 3453, 3236] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[62, 0, 4096, 28, 0, 1, 8, 4071, 3, 10, 2945, 973, 15, 68, 128, 4085, 88, 4086, 9, 3505, 0, 0, 3605, 2434, 1, 0, 0, 3286, 48, 0, 1, 0, 3549, 0, 2323, 0, 0, 0, 1257, 12, 4084, 0, 0, 727, 0, 8, 1, 8, 2646, 31, 3455, 4033, 1938, 133, 648, 4, 3729, 329, 44, 425, 116, 0, 134, 2346] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[725, 118, 3747, 5, 15, 47, 388, 0, 0, 3005, 57, 0, 262, 0, 6, 0, 17, 4062, 29, 0, 12, 2, 0, 3551, 0, 113, 1637, 4095, 1, 0, 0, 3827, 683, 3836, 94, 1145, 1, 0, 3835, 0, 1016, 267, 0, 3474, 3810, 0, 1308, 85, 0, 0, 0, 2, 47, 17, 2877, 3932, 3991, 4067, 0, 1082, 1, 0, 4060, 185] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[2315, 0, 3582, 3093, 10, 0, 18, 4049, 1097, 3477, 4000, 71, 3, 0, 275, 13, 13, 3764, 2, 0, 0, 0, 15, 125, 2, 0, 51, 3973, 0, 0, 0, 31, 16, 0, 0, 3897, 288, 0, 2542, 109, 3963, 0, 887, 4095, 3467, 0, 2032, 0, 4093, 27, 245, 12, 18, 48, 1248, 288, 4095, 0, 0, 0, 0, 56, 4086, 45] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 3690, 3896, 1, 1, 4, 3890, 4, 225, 2, 48, 2, 0, 46, 3645, 26, 4078, 12, 289, 320, 1, 437, 3539, 3, 0, 0, 4094, 0, 1, 3, 0, 27, 1264, 0, 1426, 117, 0, 4049, 560, 361, 953, 0, 4093, 4086, 0, 4066, 0, 607, 241, 1, 1190, 3, 1765, 3875, 423, 3897, 7, 0, 0, 404, 0, 3863, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1745, 0, 11, 351, 336, 3625, 0, 0, 306, 13, 0, 318, 46, 1365, 3837, 662, 208, 0, 0, 5, 704, 0, 0, 80, 3827, 3, 0, 5, 4, 163, 3904, 0, 3969, 215, 2, 4088, 0, 3711, 0, 3928, 3912, 5, 0, 25, 106, 3556, 0, 527, 3, 0, 18, 4055, 0, 4087, 3146, 0, 5, 51, 443, 4070, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4045, 21, 1378, 333, 0, 900, 4049, 0, 75, 47, 0, 1, 21, 2638, 31, 3527, 5, 0, 89, 0, 24, 34, 0, 0, 78, 4092, 22, 41, 0, 194, 1, 7, 17, 4016, 0, 0, 3998, 1, 4094, 0, 3283, 4089, 4096, 5, 4036, 51, 3958, 3, 0, 7, 20, 3, 4088, 204, 4092, 0, 2, 0, 0, 1, 3819, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1952, 1, 8, 29, 0, 0, 732, 40, 0, 0, 0, 0, 0, 0, 649, 0, 26, 1468, 249, 0, 1, 3408, 1814, 201, 0, 0, 0, 0, 0, 0, 3, 4060, 338, 0, 244, 3, 174, 0, 1018, 365, 0, 0, 0, 0, 0, 0, 1136, 12, 0, 4096, 0, 0, 0, 15, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3961, 11, 185, 4022, 134, 236, 0, 0, 0, 0, 0, 0, 0, 2995, 13, 4046, 0, 3, 0, 2868, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 3953, 0, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 3743, 2484, 0, 932, 0, 3445, 53, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2762, 7816, 6205, 9147, 8572, 10144, 8969, 11921] avg=8192.0000 max=11921 max_rank=7 min=2762 min_rank=0 imbalance_ratio=1.455200 max_load_reduction=53615 max_load_reduction_pct=0.818100 moved_to_echo=39606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[725, 118, 3747, 5, 15, 47, 388, 0, 181, 28, 0, 0, 0, 0, 0, 0, 0, 3005, 57, 0, 262, 0, 6, 0, 1768, 172, 0, 0, 0, 0, 0, 0, 17, 4062, 29, 0, 12, 2, 0, 3551, 992, 90, 0, 0, 0, 0, 0, 0, 0, 113, 1637, 4095, 1, 0, 0, 3827, 3302, 0, 0, 0, 0, 0, 0, 0, 683, 3836, 94, 1145, 1, 0, 3835, 0, 166, 0, 0, 0, 0, 0, 0, 0, 745, 267, 0, 0, 3810, 0, 1308, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 47, 17, 2683, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 4067, 0, 1082, 1, 0, 4060, 185, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5254, 5270, 8755, 12975, 9760, 6215, 6681, 10626] avg=8192.0000 max=12975 max_rank=3 min=5254 min_rank=0 imbalance_ratio=1.583862 max_load_reduction=52561 max_load_reduction_pct=0.802017 moved_to_echo=33282 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[62, 0, 4096, 28, 0, 1, 8, 4071, 730, 6, 0, 0, 0, 0, 0, 0, 3, 10, 2945, 973, 15, 68, 128, 4085, 1652, 35, 0, 0, 0, 0, 0, 0, 88, 4086, 9, 3505, 0, 0, 3605, 2434, 927, 364, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3286, 48, 0, 1, 0, 692, 0, 0, 0, 0, 0, 0, 0, 3549, 0, 2323, 0, 0, 0, 1257, 12, 37, 0, 0, 0, 0, 0, 0, 0, 2990, 0, 0, 0, 0, 8, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2646, 31, 3455, 4033, 1938, 133, 605, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 329, 44, 425, 116, 0, 134, 2346, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9002, 9914, 15018, 4028, 7178, 3007, 12845, 4544] avg=8192.0000 max=15018 max_rank=2 min=3007 min_rank=5 imbalance_ratio=1.833252 max_load_reduction=50518 max_load_reduction_pct=0.770844 moved_to_echo=27574 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 4037, 5, 11, 0, 0, 3770, 591, 30, 0, 0, 0, 0, 0, 0, 0, 3421, 1013, 0, 157, 0, 0, 4058, 1818, 59, 0, 0, 0, 0, 0, 0, 8, 2937, 0, 214, 0, 622, 0, 2248, 1021, 296, 0, 0, 0, 0, 0, 0, 0, 178, 1, 4050, 0, 1, 6, 2135, 1122, 0, 0, 0, 0, 0, 0, 0, 20, 27, 0, 4096, 0, 0, 280, 0, 163, 0, 0, 0, 0, 0, 0, 0, 2396, 330, 135, 0, 36, 11, 3357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 54, 0, 680, 0, 0, 2584, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 3518, 137, 0, 1857, 0, 3453, 3236, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8455, 10526, 7346, 7493, 4586, 6265, 7407, 13458] avg=8192.0000 max=13458 max_rank=7 min=4586 min_rank=4 imbalance_ratio=1.642822 max_load_reduction=52078 max_load_reduction_pct=0.794647 moved_to_echo=31716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2315, 0, 3582, 3093, 10, 0, 18, 4049, 709, 12, 0, 0, 0, 0, 0, 0, 1097, 3477, 4000, 71, 3, 0, 275, 13, 1814, 203, 0, 0, 0, 0, 0, 0, 13, 3764, 2, 0, 0, 0, 15, 125, 1018, 353, 0, 0, 0, 0, 0, 0, 2, 0, 51, 3973, 0, 0, 0, 31, 3892, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 3897, 288, 0, 2542, 109, 72, 0, 0, 0, 0, 0, 0, 0, 2901, 0, 887, 0, 3467, 0, 2032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 27, 245, 12, 18, 48, 1164, 288, 0, 0, 0, 0, 0, 0, 0, 0, 1263, 0, 0, 0, 0, 56, 4086, 45, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13788, 10953, 5290, 7949, 6924, 9287, 5895, 5450] avg=8192.0000 max=13788 max_rank=0 min=5290 min_rank=2 imbalance_ratio=1.683105 max_load_reduction=51748 max_load_reduction_pct=0.789612 moved_to_echo=27556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 1745, 0, 11, 351, 336, 664, 39, 0, 0, 0, 0, 0, 0, 3625, 0, 0, 306, 13, 0, 318, 46, 1810, 193, 0, 0, 0, 0, 0, 0, 1365, 3837, 662, 208, 0, 0, 5, 704, 1016, 331, 0, 0, 0, 0, 0, 0, 0, 0, 80, 3827, 3, 0, 5, 4, 3719, 0, 0, 0, 0, 0, 0, 0, 163, 3904, 0, 3969, 215, 2, 4088, 0, 234, 0, 0, 0, 0, 0, 0, 0, 2716, 0, 3928, 0, 5, 0, 25, 106, 0, 0, 0, 0, 0, 0, 0, 0, 3556, 0, 527, 3, 0, 18, 3782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 3146, 0, 5, 51, 443, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7242, 6311, 8128, 7638, 12575, 6780, 7886, 8976] avg=8192.0000 max=12575 max_rank=4 min=6311 min_rank=1 imbalance_ratio=1.535034 max_load_reduction=52961 max_load_reduction_pct=0.808121 moved_to_echo=36217 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3690, 3896, 1, 1, 4, 3890, 64, 37, 0, 0, 0, 0, 0, 0, 4, 225, 2, 48, 2, 0, 46, 3645, 1726, 203, 0, 0, 0, 0, 0, 0, 26, 4078, 12, 289, 320, 1, 437, 3539, 968, 32, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4094, 0, 1, 3, 0, 3890, 0, 0, 0, 0, 0, 0, 0, 27, 1264, 0, 1426, 117, 0, 4049, 560, 223, 0, 0, 0, 0, 0, 0, 0, 265, 953, 0, 0, 4086, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 241, 1, 1190, 3, 1765, 3615, 423, 0, 0, 0, 0, 0, 0, 0, 0, 1203, 7, 0, 0, 404, 0, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11584, 5901, 9702, 7991, 7666, 9370, 7845, 5477] avg=8192.0000 max=11584 max_rank=0 min=5477 min_rank=7 imbalance_ratio=1.414062 max_load_reduction=53952 max_load_reduction_pct=0.823242 moved_to_echo=30358 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4045, 21, 1378, 333, 0, 900, 732, 40, 0, 0, 0, 0, 0, 0, 4049, 0, 75, 47, 0, 1, 21, 2638, 1813, 201, 0, 0, 0, 0, 0, 0, 31, 3527, 5, 0, 89, 0, 24, 34, 1017, 365, 0, 0, 0, 0, 0, 0, 0, 0, 78, 4092, 22, 41, 0, 194, 3888, 0, 0, 0, 0, 0, 0, 0, 1, 7, 17, 4016, 0, 0, 3998, 1, 235, 0, 0, 0, 0, 0, 0, 0, 2997, 0, 3283, 0, 4096, 5, 4036, 51, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 3, 0, 7, 20, 3, 3813, 204, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 0, 2, 0, 0, 1, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7449, 8845, 5092, 8315, 8275, 14468, 8008, 5084] avg=8192.0000 max=14468 max_rank=5 min=5084 min_rank=7 imbalance_ratio=1.766113 max_load_reduction=51068 max_load_reduction_pct=0.779236 moved_to_echo=35835 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3200, 128, 29312, 8832, 1536, 512, 896, 17024, 4480, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[9984, 14848, 2688, 1536, 3456, 512, 27008, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1664, 30464, 1152, 4224, 768, 640, 4352, 12672, 8064, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1152, 384, 1920, 31616, 128, 128, 128, 6272, 24448, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[18048, 1664, 12288, 0, 15616, 128, 17792, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4480, 9088, 2560, 22528, 640, 256, 24192, 896, 1408, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[18944, 384, 4352, 7168, 6016, 2048, 22144, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9472, 10240, 8192, 2944, 768, 128, 896, 17920, 14336, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[227, 3, 0, 9, 4029, 6, 0, 3882, 0, 4017, 21, 3553, 369, 0, 19, 1658, 2914, 0, 0, 3719, 335, 0, 0, 0, 82, 3925, 0, 7, 0, 31, 2, 1368, 4048, 873, 25, 0, 0, 2572, 0, 24, 0, 4024, 3755, 0, 104, 19, 5, 3887, 4, 3699, 243, 3048, 0, 3941, 0, 0, 301, 61, 0, 0, 166, 455, 10, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[34, 0, 4081, 164, 3420, 747, 1, 4077, 0, 3739, 0, 4096, 11, 16, 35, 0, 3837, 3, 0, 293, 0, 0, 0, 31, 1102, 3970, 8, 134, 2, 4057, 0, 7, 2752, 0, 1569, 974, 0, 4095, 537, 0, 1076, 35, 13, 2064, 3, 0, 32, 0, 0, 4090, 0, 0, 0, 0, 4078, 0, 3587, 0, 27, 0, 0, 2951, 480, 3308] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 3495, 3393, 64, 4095, 0, 0, 3809, 458, 3880, 0, 2992, 53, 11, 0, 737, 4088, 0, 0, 0, 50, 786, 84, 2, 0, 3900, 0, 1, 73, 2800, 3, 221, 4096, 33, 0, 0, 6, 0, 125, 0, 4083, 79, 0, 1758, 3568, 492, 10, 3667, 7, 2987, 0, 0, 2, 7, 4, 0, 2150, 0, 0, 4001, 0, 2, 1967, 1497] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4094, 0, 3700, 0, 4039, 1511, 0, 4094, 0, 4010, 2492, 3238, 0, 126, 197, 0, 2, 0, 0, 628, 53, 0, 6, 129, 200, 4084, 0, 37, 10, 4096, 18, 3956, 3838, 30, 0, 5, 1, 0, 4021, 0, 0, 0, 10, 0, 4066, 0, 0, 79, 0, 3313, 0, 0, 0, 0, 236, 0, 3993, 4, 3, 54, 3, 1107, 18, 4035] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4094, 0, 2184, 103, 4082, 0, 0, 4021, 0, 4068, 86, 3965, 0, 2, 0, 0, 0, 68, 1, 222, 3772, 0, 0, 0, 2987, 2310, 177, 9, 3875, 25, 0, 315, 4082, 60, 73, 4, 16, 3996, 913, 13, 3445, 3769, 14, 1577, 48, 0, 0, 2961, 26, 252, 0, 17, 106, 1, 5, 12, 4096, 0, 0, 0, 0, 127, 0, 3557] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[2666, 0, 3905, 0, 4009, 0, 10, 3104, 337, 4082, 74, 3735, 22, 0, 40, 0, 2996, 0, 0, 0, 0, 1406, 68, 218, 2197, 3547, 0, 49, 471, 3487, 2, 143, 3474, 0, 83, 0, 0, 2843, 652, 0, 1059, 0, 0, 3442, 4077, 67, 0, 124, 0, 4028, 1, 0, 0, 556, 3684, 0, 992, 0, 0, 29, 0, 132, 0, 3725] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[676, 0, 110, 0, 4076, 994, 0, 3210, 0, 3793, 57, 2951, 0, 0, 0, 1, 985, 0, 0, 0, 321, 0, 398, 0, 525, 4075, 0, 63, 3460, 3945, 51, 65, 14, 3, 5, 0, 52, 3, 3780, 0, 4086, 1537, 1, 3084, 1138, 1, 13, 3996, 253, 3816, 0, 7, 8, 4056, 3894, 30, 3025, 0, 9, 1057, 98, 25, 6, 1783] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4025, 0, 2552, 87, 3840, 332, 0, 4095, 0, 3784, 1, 4089, 11, 0, 0, 0, 3932, 0, 0, 0, 9, 867, 0, 0, 3173, 4049, 0, 0, 0, 16, 2, 4, 1321, 1, 382, 0, 0, 39, 1, 0, 2523, 0, 0, 4091, 119, 3955, 218, 0, 4020, 3594, 0, 0, 21, 0, 0, 0, 2816, 141, 4054, 59, 1, 0, 12, 3300] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[227, 3, 0, 9, 0, 6, 0, 2855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3852, 21, 3553, 369, 0, 19, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 2914, 0, 0, 3719, 335, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 82, 2930, 0, 7, 0, 31, 2, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 873, 25, 0, 0, 2572, 0, 24, 0, 408, 0, 0, 0, 0, 0, 0, 0, 4024, 3755, 0, 104, 19, 5, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3699, 243, 3048, 0, 3941, 0, 0, 874, 619, 0, 0, 0, 0, 0, 0, 301, 61, 0, 0, 166, 455, 10, 4096, 165, 121, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3100, 9472, 10997, 4420, 7950, 11794, 12428, 5375] avg=8192.0000 max=12428 max_rank=6 min=3100 min_rank=0 imbalance_ratio=1.517090 max_load_reduction=53108 max_load_reduction_pct=0.810364 moved_to_echo=37547 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4025, 0, 2552, 87, 0, 332, 0, 3012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 1, 4089, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 0, 9, 867, 0, 0, 3840, 267, 0, 0, 0, 0, 0, 0, 3173, 3022, 0, 0, 0, 16, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1321, 1, 382, 0, 0, 39, 1, 0, 1692, 430, 0, 0, 0, 0, 0, 0, 564, 0, 0, 4091, 119, 3955, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 3594, 0, 0, 21, 0, 0, 0, 902, 653, 0, 0, 0, 0, 0, 0, 2816, 141, 4054, 59, 1, 0, 12, 3300, 155, 125, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10008, 7730, 8915, 6217, 3866, 8947, 9190, 10663] avg=8192.0000 max=10663 max_rank=7 min=3866 min_rank=4 imbalance_ratio=1.301636 max_load_reduction=54873 max_load_reduction_pct=0.837296 moved_to_echo=32666 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3495, 3393, 64, 0, 0, 0, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 458, 3721, 0, 2992, 53, 11, 0, 737, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 50, 786, 84, 2, 4095, 432, 0, 0, 0, 0, 0, 0, 0, 2911, 0, 1, 73, 2800, 3, 221, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 33, 0, 0, 6, 0, 125, 0, 2738, 400, 0, 0, 0, 0, 0, 0, 913, 79, 0, 1758, 3568, 492, 10, 3667, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2987, 0, 0, 2, 7, 4, 0, 869, 608, 0, 0, 0, 0, 0, 0, 2150, 0, 0, 4001, 0, 2, 1967, 1497, 159, 120, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9753, 7972, 9537, 6009, 7398, 10487, 4484, 9896] avg=8192.0000 max=10487 max_rank=5 min=4484 min_rank=6 imbalance_ratio=1.280151 max_load_reduction=55049 max_load_reduction_pct=0.839981 moved_to_echo=32265 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[676, 0, 110, 0, 0, 994, 0, 2361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3638, 57, 2951, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 321, 0, 398, 0, 4076, 432, 0, 0, 0, 0, 0, 0, 525, 3041, 0, 63, 3460, 3945, 51, 65, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3, 5, 0, 52, 3, 3780, 0, 2740, 337, 0, 0, 0, 0, 0, 0, 914, 1537, 1, 3084, 1138, 1, 13, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 253, 3816, 0, 7, 8, 4056, 3894, 30, 908, 512, 0, 0, 0, 0, 0, 0, 3025, 0, 9, 1057, 98, 25, 6, 1783, 155, 126, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 6647, 6212, 11150, 6934, 10684, 13484, 6284] avg=8192.0000 max=13484 max_rank=6 min=4141 min_rank=0 imbalance_ratio=1.645996 max_load_reduction=52052 max_load_reduction_pct=0.794250 moved_to_echo=37386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[34, 0, 4081, 164, 0, 747, 1, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 4096, 11, 16, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3837, 3, 0, 293, 0, 0, 0, 31, 3420, 116, 0, 0, 0, 0, 0, 0, 1102, 2955, 8, 134, 2, 4057, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2752, 0, 1569, 974, 0, 4095, 537, 0, 725, 433, 0, 0, 0, 0, 0, 0, 235, 35, 13, 2064, 3, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4078, 0, 887, 655, 0, 0, 0, 0, 0, 0, 3587, 0, 27, 0, 0, 2951, 480, 3308, 158, 128, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8016, 7739, 7700, 8265, 11085, 2382, 9710, 10639] avg=8192.0000 max=11085 max_rank=4 min=2382 min_rank=5 imbalance_ratio=1.353149 max_load_reduction=54451 max_load_reduction_pct=0.830856 moved_to_echo=33816 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2666, 0, 3905, 0, 0, 0, 10, 2283, 0, 0, 0, 0, 0, 0, 0, 0, 337, 3915, 74, 3735, 22, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2996, 0, 0, 0, 0, 1406, 68, 218, 4009, 112, 0, 0, 0, 0, 0, 0, 2197, 2648, 0, 49, 471, 3487, 2, 143, 0, 0, 0, 0, 0, 0, 0, 0, 3474, 0, 83, 0, 0, 2843, 652, 0, 710, 326, 0, 0, 0, 0, 0, 0, 237, 0, 0, 3442, 4077, 67, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 1, 0, 0, 556, 3684, 0, 790, 495, 0, 0, 0, 0, 0, 0, 992, 0, 0, 29, 0, 132, 0, 3725, 167, 109, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8864, 8123, 8809, 8997, 8088, 7947, 9554, 5154] avg=8192.0000 max=9554 max_rank=6 min=5154 min_rank=7 imbalance_ratio=1.166260 max_load_reduction=55982 max_load_reduction_pct=0.854218 moved_to_echo=30743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 3700, 0, 0, 1511, 0, 3011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 2492, 3238, 0, 126, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 628, 53, 0, 6, 129, 4039, 0, 0, 0, 0, 0, 0, 0, 200, 3048, 0, 37, 10, 4096, 18, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 30, 0, 5, 1, 0, 4021, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4066, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 0, 0, 0, 236, 0, 910, 653, 0, 0, 0, 0, 0, 0, 3993, 4, 3, 54, 3, 1107, 18, 4035, 164, 126, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12316, 9899, 4857, 11365, 8325, 4155, 5112, 9507] avg=8192.0000 max=12316 max_rank=0 min=4155 min_rank=5 imbalance_ratio=1.503418 max_load_reduction=53220 max_load_reduction_pct=0.812073 moved_to_echo=27099 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 2184, 103, 0, 0, 0, 2957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 86, 3965, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 1, 222, 3772, 0, 0, 0, 4082, 364, 0, 0, 0, 0, 0, 0, 2987, 1725, 177, 9, 3875, 25, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 60, 73, 4, 16, 3996, 913, 13, 2310, 423, 0, 0, 0, 0, 0, 0, 771, 3769, 14, 1577, 48, 0, 0, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 26, 252, 0, 17, 106, 1, 5, 12, 514, 641, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 127, 0, 3557, 167, 71, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9338, 7954, 8509, 9113, 11890, 9140, 1574, 8018] avg=8192.0000 max=11890 max_rank=4 min=1574 min_rank=6 imbalance_ratio=1.451416 max_load_reduction=53646 max_load_reduction_pct=0.818573 moved_to_echo=30622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4352, 25856, 256, 3072, 256, 8576, 11904, 128, 6656, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3712, 9472, 3840, 16128, 13184, 4608, 384, 14720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[15872, 3584, 19968, 512, 0, 3712, 128, 22272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[20992, 256, 4096, 5248, 384, 4864, 2560, 25344, 1408, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23680, 1024, 2176, 1024, 128, 13568, 10112, 128, 11008, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[896, 30208, 2816, 28672, 512, 256, 384, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[10368, 22400, 256, 384, 7936, 18560, 128, 6144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[18816, 128, 128, 4864, 4608, 3072, 640, 384, 31616, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 1, 0, 12, 231, 0, 103, 0, 1792, 0, 4096, 0, 0, 0, 104, 0, 1457, 4015, 4095, 12, 0, 2498, 4068, 4096, 2481, 0, 0, 97, 0, 5, 0, 3425, 4039, 0, 1217, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 767, 2907, 0, 4096, 3478, 4077, 2635, 459, 826, 41, 4, 0, 0, 300, 0, 19, 0, 3960, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[19, 0, 2490, 0, 2, 3668, 0, 550, 197, 0, 4046, 0, 0, 116, 0, 4, 375, 3860, 4095, 702, 0, 741, 1, 3770, 29, 3607, 764, 131, 8, 0, 3383, 4096, 1800, 0, 0, 77, 0, 1118, 2662, 0, 3729, 69, 0, 0, 361, 1, 1298, 0, 4096, 3292, 3691, 2692, 8, 443, 0, 0, 0, 0, 0, 0, 0, 345, 3084, 116] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 27, 0, 0, 1, 4092, 0, 4, 4094, 11, 3880, 4, 0, 8, 0, 0, 0, 4058, 3671, 64, 0, 215, 14, 4092, 2496, 3047, 55, 0, 0, 20, 910, 254, 1, 3236, 726, 4040, 72, 0, 1, 0, 51, 5, 22, 3854, 0, 170, 3556, 68, 8, 21, 4015, 3759, 4011, 2317, 0, 100, 1, 7, 0, 0, 3563, 1, 0, 914] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[2, 21, 3749, 0, 63, 3721, 0, 300, 21, 172, 3557, 1113, 0, 4093, 12, 0, 0, 3875, 4052, 0, 583, 4092, 985, 3929, 0, 4093, 103, 112, 2, 0, 3698, 1835, 8, 0, 2946, 0, 224, 0, 0, 0, 0, 0, 3469, 3471, 0, 79, 0, 0, 1855, 0, 4096, 0, 9, 0, 0, 724, 457, 5, 51, 0, 0, 0, 26, 3933] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[2689, 0, 233, 0, 0, 1784, 0, 60, 0, 872, 4060, 0, 0, 543, 154, 0, 0, 338, 4091, 4083, 16, 903, 4088, 2176, 1, 4074, 0, 0, 6, 7, 4077, 4096, 2263, 0, 2, 0, 44, 0, 0, 1, 0, 4068, 0, 3851, 5, 54, 0, 30, 3481, 250, 4096, 0, 4088, 3429, 0, 0, 1, 2, 0, 0, 64, 0, 1456, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[88, 0, 0, 49, 0, 3659, 0, 3830, 195, 2716, 4023, 0, 0, 700, 1, 86, 0, 4096, 4046, 0, 1, 4094, 999, 3870, 22, 4088, 131, 12, 0, 0, 4078, 3468, 442, 0, 0, 1, 3846, 0, 14, 1, 0, 110, 441, 0, 8, 35, 3911, 0, 846, 0, 4086, 0, 3204, 0, 5, 0, 37, 222, 4, 0, 2506, 5, 1560, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4094, 0, 396, 526, 4, 45, 0, 958, 2974, 49, 4096, 0, 32, 1, 1015, 0, 26, 327, 4021, 0, 0, 2430, 0, 3984, 2, 2950, 0, 1176, 2, 0, 4080, 3396, 1, 0, 991, 0, 4096, 21, 0, 1222, 68, 25, 3449, 879, 1, 0, 0, 0, 3785, 0, 3098, 154, 3583, 1, 0, 0, 22, 4094, 55, 0, 3400, 0, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4028, 0, 0, 1, 0, 2922, 0, 279, 2650, 1, 4090, 0, 0, 1118, 3, 0, 1, 4096, 4078, 0, 3540, 4085, 4027, 4083, 1589, 19, 0, 52, 0, 610, 58, 128, 3956, 184, 0, 318, 5, 1762, 0, 12, 2, 0, 3768, 4022, 0, 0, 1039, 0, 0, 0, 3628, 4, 4096, 34, 21, 0, 0, 0, 8, 1, 1044, 4, 157, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 1, 0, 12, 231, 0, 103, 0, 1704, 529, 0, 0, 0, 0, 0, 0, 1792, 0, 4096, 0, 0, 0, 104, 0, 0, 525, 0, 0, 0, 0, 0, 0, 1457, 3490, 0, 12, 0, 2498, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 97, 0, 5, 0, 3425, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 1217, 0, 0, 0, 0, 0, 3302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 767, 2907, 0, 1885, 0, 0, 0, 0, 0, 0, 0, 4096, 3478, 488, 2635, 459, 826, 41, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 19, 0, 3960, 0, 4095, 265, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6676, 6517, 11525, 6008, 8558, 5586, 12027, 8639] avg=8192.0000 max=12027 max_rank=6 min=5586 min_rank=5 imbalance_ratio=1.468140 max_load_reduction=53509 max_load_reduction_pct=0.816483 moved_to_echo=34810 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 21, 3749, 0, 63, 3721, 0, 300, 1712, 506, 0, 0, 0, 0, 0, 0, 21, 172, 3557, 1113, 0, 4093, 12, 0, 1049, 507, 0, 0, 0, 0, 0, 0, 0, 3368, 0, 0, 583, 4092, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2740, 103, 112, 2, 0, 3698, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2946, 0, 224, 0, 0, 0, 3167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3469, 3471, 0, 79, 0, 0, 1894, 304, 0, 0, 0, 0, 0, 0, 1855, 0, 490, 0, 9, 0, 0, 724, 0, 0, 0, 0, 0, 0, 0, 0, 457, 5, 51, 0, 0, 0, 26, 3933, 4052, 256, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10074, 10524, 9028, 8490, 6345, 9217, 3078, 8780] avg=8192.0000 max=10524 max_rank=1 min=3078 min_rank=6 imbalance_ratio=1.284668 max_load_reduction=55012 max_load_reduction_pct=0.839417 moved_to_echo=27420 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 2490, 0, 2, 3668, 0, 550, 1542, 486, 0, 0, 0, 0, 0, 0, 197, 0, 4046, 0, 0, 116, 0, 4, 924, 505, 0, 0, 0, 0, 0, 0, 375, 3355, 0, 702, 0, 741, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2415, 764, 131, 8, 0, 3383, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1800, 0, 0, 77, 0, 1118, 2662, 0, 3040, 0, 0, 0, 0, 0, 0, 0, 3729, 69, 0, 0, 361, 1, 1298, 0, 1706, 268, 0, 0, 0, 0, 0, 0, 4096, 3292, 443, 2692, 8, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 3084, 116, 4095, 244, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8757, 5792, 5174, 10826, 8697, 7432, 10974, 7884] avg=8192.0000 max=10974 max_rank=6 min=5174 min_rank=2 imbalance_ratio=1.339600 max_load_reduction=54562 max_load_reduction_pct=0.832550 moved_to_echo=34987 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 0, 0, 1, 4092, 0, 4, 1678, 527, 0, 0, 0, 0, 0, 0, 4094, 11, 3880, 4, 0, 8, 0, 0, 781, 531, 0, 0, 0, 0, 0, 0, 0, 3527, 0, 64, 0, 215, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2496, 2040, 55, 0, 0, 20, 910, 254, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3236, 726, 4040, 72, 0, 1, 0, 3300, 0, 0, 0, 0, 0, 0, 0, 51, 5, 22, 3854, 0, 170, 3556, 68, 1856, 226, 0, 0, 0, 0, 0, 0, 8, 21, 481, 3759, 4011, 2317, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 3563, 1, 0, 914, 3671, 265, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6329, 9309, 3820, 5775, 11376, 9808, 10697, 8422] avg=8192.0000 max=11376 max_rank=4 min=3820 min_rank=2 imbalance_ratio=1.388672 max_load_reduction=54160 max_load_reduction_pct=0.826416 moved_to_echo=40303 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2689, 0, 233, 0, 0, 1784, 0, 60, 1716, 282, 0, 0, 0, 0, 0, 0, 0, 872, 4060, 0, 0, 543, 154, 0, 1047, 47, 0, 0, 0, 0, 0, 0, 0, 291, 0, 4083, 16, 903, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2720, 0, 0, 6, 7, 4077, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2263, 0, 2, 0, 44, 0, 0, 1, 1754, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 3851, 5, 54, 0, 30, 1898, 307, 0, 0, 0, 0, 0, 0, 3481, 250, 482, 0, 4088, 3429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 64, 0, 1456, 0, 4091, 140, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6764, 6723, 9381, 10907, 4064, 10213, 11730, 5754] avg=8192.0000 max=11730 max_rank=6 min=4064 min_rank=4 imbalance_ratio=1.431885 max_load_reduction=53806 max_load_reduction_pct=0.821014 moved_to_echo=31761 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 396, 526, 4, 45, 0, 958, 1294, 514, 0, 0, 0, 0, 0, 0, 2974, 49, 4096, 0, 32, 1, 1015, 0, 756, 42, 0, 0, 0, 0, 0, 0, 26, 285, 0, 0, 0, 2430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1975, 0, 1176, 2, 0, 4080, 3396, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 991, 0, 4096, 21, 0, 1222, 3212, 0, 0, 0, 0, 0, 0, 0, 68, 25, 3449, 879, 1, 0, 0, 0, 1432, 219, 0, 0, 0, 0, 0, 0, 3785, 0, 372, 154, 3583, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4094, 55, 0, 3400, 0, 0, 7, 4021, 258, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7831, 8965, 2741, 10631, 9543, 6073, 7895, 11857] avg=8192.0000 max=11857 max_rank=7 min=2741 min_rank=2 imbalance_ratio=1.447388 max_load_reduction=53679 max_load_reduction_pct=0.819077 moved_to_echo=35368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[88, 0, 0, 49, 0, 3659, 0, 3830, 1707, 499, 0, 0, 0, 0, 0, 0, 195, 2716, 4023, 0, 0, 700, 1, 86, 1047, 536, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 1, 4094, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 2737, 131, 12, 0, 0, 4078, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 442, 0, 0, 1, 3846, 0, 14, 1, 3120, 0, 0, 0, 0, 0, 0, 0, 0, 110, 441, 0, 8, 35, 3911, 0, 1889, 304, 0, 0, 0, 0, 0, 0, 846, 0, 490, 0, 3204, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 222, 4, 0, 2506, 5, 1560, 0, 4046, 251, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9832, 9304, 8654, 10448, 7424, 6698, 4545, 8631] avg=8192.0000 max=10448 max_rank=3 min=4545 min_rank=6 imbalance_ratio=1.275391 max_load_reduction=55088 max_load_reduction_pct=0.840576 moved_to_echo=27298 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4028, 0, 0, 1, 0, 2922, 0, 279, 1516, 527, 0, 0, 0, 0, 0, 0, 2650, 1, 4090, 0, 0, 1118, 3, 0, 4, 536, 0, 0, 0, 0, 0, 0, 1, 3560, 0, 0, 3540, 4085, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1589, 14, 0, 52, 0, 610, 58, 128, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 184, 0, 318, 5, 1762, 0, 12, 3292, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3768, 4022, 0, 0, 1039, 0, 1677, 1, 0, 0, 0, 0, 0, 0, 0, 0, 435, 4, 4096, 34, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 1044, 4, 157, 13, 4078, 264, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9273, 8402, 15213, 2451, 9529, 10509, 4590, 5569] avg=8192.0000 max=15213 max_rank=2 min=2451 min_rank=3 imbalance_ratio=1.857056 max_load_reduction=50323 max_load_reduction_pct=0.767868 moved_to_echo=30197 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[1920, 21504, 0, 4864, 4224, 19072, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6656, 14720, 1152, 1664, 128, 768, 20352, 20736, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[12544, 3456, 5888, 4480, 8320, 2944, 2688, 1280, 24192, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[15104, 128, 6912, 640, 384, 19968, 128, 6016, 12928, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[18176, 7168, 3712, 9344, 19584, 7168, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3968, 4352, 11264, 16128, 384, 1152, 12800, 128, 14336, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 4352, 512, 128, 10624, 384, 10368, 4992, 32256, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12032, 3840, 31872, 1152, 128, 6656, 1408, 128, 5632, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3, 22, 91, 617, 1, 0, 0, 1074, 1277, 4075, 3485, 1095, 1032, 0, 0, 0, 3557, 0, 0, 3173, 1149, 4, 422, 0, 1, 2810, 9, 10, 3481, 2914, 0, 4073, 274, 1684, 0, 350, 0, 645, 287, 0, 1417, 1403, 2391, 4, 4096, 57, 45, 7, 0, 0, 0, 0, 902, 0, 0, 3750, 0, 3443, 0, 3019, 3952, 0, 3435, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 297, 0, 57, 4, 4060, 326, 0, 23, 0, 664, 25, 0, 0, 4044, 8, 1989, 4088, 1371, 1, 1288, 0, 0, 0, 0, 780, 77, 4084, 0, 3868, 3377, 0, 14, 1, 20, 4092, 573, 9, 0, 1, 34, 2721, 4078, 35, 0, 160, 1476, 97, 660, 3057, 8, 134, 0, 73, 1, 3216, 3820, 3280, 3946, 0, 3597] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 910, 0, 0, 0, 4096, 3385, 4023, 4096, 0, 4047, 141, 45, 4079, 0, 0, 4096, 0, 0, 2, 124, 0, 1457, 0, 2, 2999, 18, 96, 1216, 615, 4010, 3012, 14, 28, 0, 505, 0, 0, 2021, 0, 0, 4084, 3876, 0, 0, 2285, 330, 0, 985, 3050, 0, 143, 1725, 1, 0, 0, 3803, 209, 0, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[33, 0, 161, 6, 0, 0, 67, 4085, 0, 3476, 3054, 4051, 3463, 0, 2, 27, 3327, 3781, 427, 4075, 93, 0, 0, 0, 6, 0, 102, 1, 3622, 1001, 2618, 30, 340, 647, 0, 1466, 0, 0, 413, 0, 17, 3292, 2499, 0, 581, 22, 448, 0, 29, 156, 9, 0, 33, 0, 0, 4077, 0, 4065, 1338, 3994, 850, 20, 3732, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4095, 68, 0, 2, 0, 0, 0, 4069, 4091, 2021, 3823, 261, 1811, 2, 4094, 1793, 0, 1725, 3718, 0, 283, 3, 270, 36, 24, 2, 3491, 0, 4095, 0, 10, 4061, 3713, 0, 27, 125, 2441, 32, 0, 0, 336, 3, 3, 3121, 0, 12, 22, 0, 0, 1352, 0, 0, 90, 2020, 4066, 25, 0, 3753, 0, 546, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2146, 4, 0, 0, 84, 0, 4044, 2290, 3645, 268, 0, 0, 0, 0, 3132, 0, 5, 3797, 0, 259, 20, 110, 0, 477, 3, 0, 3487, 0, 103, 0, 3970, 4096, 4084, 0, 0, 0, 2854, 0, 0, 5, 3837, 0, 4096, 1465, 1, 0, 2722, 364, 45, 22, 203, 0, 213, 4086, 3298, 0, 616, 4010, 1512, 2, 0, 161] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 3689, 0, 0, 0, 0, 0, 0, 4073, 3739, 0, 4056, 3512, 3735, 0, 1451, 0, 142, 4035, 210, 0, 77, 46, 0, 0, 34, 671, 1104, 0, 125, 732, 0, 4044, 0, 11, 0, 1, 310, 1, 665, 961, 3844, 0, 2523, 1, 4, 0, 4017, 0, 0, 0, 4096, 1619, 3, 3054, 238, 4, 0, 258, 3848, 1, 506, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 33, 7, 2, 2560, 4096, 4042, 0, 763, 229, 0, 0, 3694, 51, 0, 4083, 588, 9, 7, 0, 76, 0, 0, 0, 4078, 0, 3942, 41, 7, 4092, 6, 11, 3, 191, 0, 0, 283, 237, 3164, 0, 6, 4073, 29, 0, 3953, 0, 0, 0, 4068, 0, 0, 2798, 0, 4096, 0, 2827, 3751, 0, 57, 3583] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[33, 0, 161, 6, 0, 0, 67, 4085, 3054, 3432, 0, 0, 0, 0, 0, 0, 0, 1391, 0, 4051, 3463, 0, 2, 27, 0, 0, 0, 0, 0, 0, 0, 0, 3327, 3781, 427, 2733, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 102, 1, 3622, 1001, 2618, 30, 1044, 562, 0, 0, 0, 0, 0, 0, 340, 647, 0, 1466, 0, 0, 413, 0, 785, 1, 0, 0, 0, 0, 0, 0, 17, 3292, 2499, 0, 581, 22, 448, 0, 557, 1, 0, 0, 0, 0, 0, 0, 29, 156, 9, 0, 33, 0, 0, 4077, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 1338, 0, 849, 20, 3732, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10838, 8934, 10361, 8986, 3652, 7417, 5344, 10004] avg=8192.0000 max=10838 max_rank=0 min=3652 min_rank=4 imbalance_ratio=1.322998 max_load_reduction=54698 max_load_reduction_pct=0.834625 moved_to_echo=26417 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 910, 0, 0, 0, 4023, 3267, 0, 0, 0, 0, 0, 0, 4096, 1354, 0, 4096, 0, 4047, 141, 45, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 2747, 0, 0, 2, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1457, 0, 2, 2999, 18, 96, 1216, 1017, 536, 0, 0, 0, 0, 0, 0, 615, 4010, 3012, 14, 28, 0, 505, 0, 789, 1, 0, 0, 0, 0, 0, 0, 0, 2021, 0, 0, 4084, 3876, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 2285, 330, 0, 985, 3050, 0, 143, 1725, 1013, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 209, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 13779, 6952, 7341, 8974, 10541, 9531, 218] avg=8192.0000 max=13779 max_rank=1 min=218 min_rank=7 imbalance_ratio=1.682007 max_load_reduction=51757 max_load_reduction_pct=0.789749 moved_to_echo=29264 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 297, 0, 57, 4, 0, 3282, 0, 0, 0, 0, 0, 0, 4060, 132, 0, 23, 0, 664, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 8, 1334, 4088, 1371, 1, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 780, 77, 4084, 0, 97, 538, 0, 0, 0, 0, 0, 0, 3868, 3377, 0, 14, 1, 20, 4092, 573, 383, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1, 34, 2721, 4078, 35, 0, 272, 6, 0, 0, 0, 0, 0, 0, 160, 1476, 97, 660, 3057, 8, 134, 0, 97, 0, 0, 0, 0, 0, 0, 0, 73, 1, 3216, 0, 3274, 3946, 0, 3597, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3642, 4904, 12134, 5576, 12328, 7156, 5689, 14107] avg=8192.0000 max=14107 max_rank=7 min=3642 min_rank=0 imbalance_ratio=1.722046 max_load_reduction=51429 max_load_reduction_pct=0.784744 moved_to_echo=39280 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 22, 91, 617, 1, 0, 0, 1074, 3485, 2594, 0, 0, 0, 0, 0, 0, 1277, 1630, 0, 1095, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3557, 0, 0, 2128, 1149, 4, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2810, 9, 10, 3481, 2914, 0, 4073, 1224, 425, 0, 0, 0, 0, 0, 0, 274, 1684, 0, 350, 0, 645, 287, 0, 611, 1, 0, 0, 0, 0, 0, 0, 1417, 1403, 2391, 4, 4096, 57, 45, 7, 434, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 902, 0, 0, 3750, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 3443, 0, 0, 3944, 0, 3435, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7887, 5034, 7260, 14947, 3852, 9862, 5872, 10822] avg=8192.0000 max=14947 max_rank=3 min=3852 min_rank=4 imbalance_ratio=1.824585 max_load_reduction=50589 max_load_reduction_pct=0.771927 moved_to_echo=30408 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 68, 0, 2, 0, 0, 4091, 3225, 0, 0, 0, 0, 0, 0, 0, 1614, 0, 2021, 3823, 261, 1811, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1793, 0, 1149, 3718, 0, 283, 3, 0, 0, 0, 0, 0, 0, 0, 0, 270, 36, 24, 2, 3491, 0, 4095, 0, 1227, 528, 0, 0, 0, 0, 0, 0, 10, 4061, 3713, 0, 27, 125, 2441, 32, 336, 5, 0, 0, 0, 0, 0, 0, 0, 0, 336, 3, 3, 3121, 0, 12, 240, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1352, 0, 0, 90, 2020, 1223, 0, 0, 0, 0, 0, 0, 0, 4066, 25, 0, 0, 0, 546, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11481, 9532, 11040, 9673, 10750, 3715, 4707, 4638] avg=8192.0000 max=11481 max_rank=0 min=3715 min_rank=5 imbalance_ratio=1.401489 max_load_reduction=54055 max_load_reduction_pct=0.824814 moved_to_echo=23810 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 33, 7, 2, 4042, 2429, 0, 0, 0, 0, 0, 0, 2560, 1639, 0, 0, 763, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3694, 51, 0, 2737, 588, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 4078, 0, 3942, 41, 1230, 398, 0, 0, 0, 0, 0, 0, 7, 4092, 6, 11, 3, 191, 0, 0, 787, 1, 0, 0, 0, 0, 0, 0, 283, 237, 3164, 0, 6, 4073, 29, 0, 559, 7, 0, 0, 0, 0, 0, 0, 3953, 0, 0, 0, 4068, 0, 0, 2798, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3744, 0, 57, 3583, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6513, 5191, 7086, 9765, 5098, 8358, 12045, 11480] avg=8192.0000 max=12045 max_rank=6 min=5098 min_rank=4 imbalance_ratio=1.470337 max_load_reduction=53491 max_load_reduction_pct=0.816208 moved_to_echo=36981 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2146, 4, 0, 0, 84, 0, 3645, 3445, 0, 0, 0, 0, 0, 0, 4044, 917, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3132, 0, 5, 2546, 0, 259, 20, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 3, 0, 3487, 0, 103, 0, 688, 565, 0, 0, 0, 0, 0, 0, 3970, 4096, 4084, 0, 0, 0, 2854, 0, 732, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3837, 0, 4096, 1465, 1, 0, 519, 3, 0, 0, 0, 0, 0, 0, 2722, 364, 45, 22, 203, 0, 213, 4086, 685, 0, 0, 0, 0, 0, 0, 0, 3298, 0, 616, 0, 1509, 2, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9324, 5229, 6072, 5323, 15736, 9926, 8340, 5586] avg=8192.0000 max=15736 max_rank=4 min=5229 min_rank=1 imbalance_ratio=1.920898 max_load_reduction=49800 max_load_reduction_pct=0.759888 moved_to_echo=39588 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3689, 0, 0, 0, 0, 0, 3739, 222, 0, 0, 0, 0, 0, 0, 0, 1630, 0, 0, 4056, 3512, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 0, 142, 2705, 210, 0, 77, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 671, 1104, 0, 125, 732, 1223, 36, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 11, 0, 1, 310, 1, 778, 1, 0, 0, 0, 0, 0, 0, 665, 961, 3844, 0, 2523, 1, 4, 0, 552, 11, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 0, 4096, 1619, 3, 3054, 1219, 0, 0, 0, 0, 0, 0, 0, 238, 4, 0, 0, 3837, 1, 506, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7651, 12933, 4631, 3925, 5146, 8561, 14008, 8681] avg=8192.0000 max=14008 max_rank=6 min=3925 min_rank=3 imbalance_ratio=1.709961 max_load_reduction=51528 max_load_reduction_pct=0.786255 moved_to_echo=36396 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7680, 11648, 5248, 0, 17408, 4608, 7808, 11520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9088, 26112, 10880, 1920, 128, 1024, 11008, 640, 5248, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[16128, 10368, 0, 11648, 13184, 8832, 5760, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[13312, 2432, 256, 3072, 15488, 1664, 640, 21632, 7808, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 4864, 256, 768, 23168, 4096, 15104, 6144, 7808, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 10240, 768, 1280, 128, 256, 5248, 26112, 22016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23424, 9728, 640, 18176, 9856, 1664, 896, 1664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2432, 7936, 16128, 128, 18176, 16768, 640, 128, 3712, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 68, 0, 0, 0, 1961, 0, 3417, 3911, 0, 0, 6, 0, 4081, 7, 0, 1, 10, 0, 4096, 3300, 0, 3974, 4049, 0, 299, 9, 279, 0, 3308, 0, 3075, 0, 0, 4095, 0, 9, 4063, 0, 1116, 38, 0, 10, 8, 381, 3230, 17, 1291, 0, 825, 0, 0, 4096, 1683, 0, 63, 0, 1197, 3501, 259, 4, 0, 3799, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 3591, 0, 0, 3943, 198, 2, 3960, 9, 2108, 0, 7, 0, 4053, 927, 4096, 0, 5, 0, 803, 0, 0, 0, 4096, 4096, 17, 53, 4087, 0, 0, 3682, 4096, 0, 1, 0, 52, 25, 4095, 0, 511, 0, 16, 0, 1, 0, 0, 90, 0, 258, 1712, 2265, 0, 3853, 30, 0, 0, 0, 668, 4068, 0, 0, 0, 4033, 29] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3945, 11, 3445, 0, 4086, 0, 142, 1, 0, 1488, 32, 285, 1, 4088, 0, 4079, 3982, 0, 85, 3855, 4096, 0, 29, 3786, 0, 0, 0, 0, 80, 490, 2, 0, 0, 0, 2, 4085, 234, 342, 87, 6, 0, 0, 4093, 3834, 0, 4, 0, 825, 296, 40, 4096, 163, 1112, 0, 0, 0, 0, 0, 4096, 0, 4035, 0, 178, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4094, 0, 2044, 1, 93, 0, 0, 1998, 0, 123, 29, 0, 4058, 4096, 1141, 3803, 7, 2013, 0, 4095, 246, 0, 56, 4096, 0, 0, 1372, 0, 0, 978, 0, 0, 0, 4096, 16, 0, 0, 4014, 0, 0, 75, 0, 84, 4081, 2, 1319, 2323, 3077, 5, 0, 0, 0, 4070, 45, 17, 0, 0, 0, 4090, 0, 3358, 47, 474, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 1959, 0, 4078, 3879, 0, 232, 143, 318, 3583, 4019, 7, 0, 0, 0, 239, 886, 650, 0, 0, 3649, 1, 0, 15, 90, 14, 4060, 795, 0, 0, 1509, 0, 0, 0, 4059, 11, 0, 4090, 0, 3, 0, 0, 0, 0, 4096, 0, 3004, 1143, 0, 4089, 3984, 0, 0, 15, 0, 2335, 1012, 3705, 59, 3770, 19] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4041, 0, 0, 0, 1768, 1, 16, 126, 19, 0, 4, 36, 356, 3922, 9, 7, 0, 0, 10, 3772, 17, 1, 227, 4073, 517, 114, 11, 0, 3, 4056, 0, 4092, 0, 56, 3418, 0, 2804, 3838, 0, 4096, 1018, 1006, 3, 3745, 0, 0, 216, 4025, 0, 0, 14, 0, 4087, 43, 0, 0, 2, 0, 4096, 11, 3735, 0, 2125, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 8, 9, 0, 1, 2203, 75, 4079, 2, 0, 17, 484, 4, 2819, 0, 3974, 0, 0, 0, 751, 0, 0, 4089, 3887, 57, 180, 31, 4096, 141, 0, 0, 4055, 0, 3877, 2, 2186, 72, 467, 330, 1212, 0, 1531, 5, 4027, 318, 0, 4, 0, 0, 0, 0, 0, 3812, 4023, 0, 682, 0, 3933, 4030, 2, 4059, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3980, 0, 0, 0, 0, 0, 2723, 4083, 3982, 58, 2990, 153, 0, 4096, 258, 2, 0, 2, 0, 1471, 0, 0, 1234, 4089, 3695, 0, 67, 254, 0, 4010, 7, 4066, 170, 104, 29, 0, 0, 4055, 0, 29, 0, 20, 1, 652, 0, 0, 0, 3337, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 6, 4095, 0, 4096, 0, 3624, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 68, 0, 0, 0, 1961, 0, 2702, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 0, 6, 0, 2985, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 4096, 3300, 0, 3974, 2749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 9, 279, 0, 3308, 0, 3075, 715, 88, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 9, 4063, 0, 1116, 832, 0, 0, 0, 0, 0, 0, 0, 38, 0, 10, 8, 381, 3230, 17, 1291, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 825, 0, 0, 4096, 1683, 0, 63, 1300, 176, 45, 0, 0, 0, 0, 0, 0, 1197, 1997, 259, 4, 0, 3799, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4731, 6909, 14130, 7773, 10115, 6434, 8188, 7256] avg=8192.0000 max=14130 max_rank=2 min=4731 min_rank=0 imbalance_ratio=1.724854 max_load_reduction=51406 max_load_reduction_pct=0.784393 moved_to_echo=31993 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3591, 0, 0, 3943, 198, 2, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2108, 0, 7, 0, 2964, 927, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 803, 0, 0, 0, 2781, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 53, 4087, 0, 0, 3682, 4096, 828, 88, 0, 0, 0, 0, 0, 0, 0, 1, 0, 52, 25, 4095, 0, 511, 827, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 1, 0, 0, 90, 0, 1696, 0, 0, 0, 0, 0, 0, 0, 258, 1712, 2265, 0, 3853, 30, 0, 0, 1315, 174, 52, 0, 0, 0, 0, 0, 0, 668, 2320, 0, 0, 0, 4033, 29, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10866, 10111, 3589, 16947, 5511, 1803, 9659, 7050] avg=8192.0000 max=16947 max_rank=3 min=1803 min_rank=5 imbalance_ratio=2.068726 max_load_reduction=48589 max_load_reduction_pct=0.741409 moved_to_echo=24023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 1959, 0, 4078, 3068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 143, 318, 3583, 2939, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 886, 650, 0, 0, 2477, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 90, 14, 4060, 795, 0, 811, 87, 0, 0, 0, 0, 0, 0, 0, 1509, 0, 0, 0, 4059, 11, 0, 820, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3, 0, 0, 0, 0, 4096, 973, 0, 0, 0, 0, 0, 0, 0, 0, 3004, 1143, 0, 4089, 3984, 0, 0, 1172, 173, 30, 0, 0, 0, 0, 0, 15, 0, 1332, 1012, 3705, 59, 3770, 19, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9121, 7222, 4252, 5873, 6399, 9162, 13595, 9912] avg=8192.0000 max=13595 max_rank=6 min=4252 min_rank=2 imbalance_ratio=1.659546 max_load_reduction=51941 max_load_reduction_pct=0.792557 moved_to_echo=39068 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3945, 11, 3445, 0, 4086, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1488, 32, 285, 1, 2978, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 85, 3855, 4096, 0, 29, 2566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 490, 2, 0, 1, 92, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4085, 234, 342, 87, 6, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3834, 0, 4, 0, 825, 1712, 0, 0, 0, 0, 0, 0, 0, 296, 40, 4096, 163, 1112, 0, 0, 0, 1220, 181, 57, 0, 0, 0, 0, 0, 0, 0, 2327, 0, 4035, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11629, 8863, 14613, 665, 5593, 10468, 7165, 6540] avg=8192.0000 max=14613 max_rank=2 min=665 min_rank=3 imbalance_ratio=1.783813 max_load_reduction=50923 max_load_reduction_pct=0.777023 moved_to_echo=29766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3980, 0, 0, 0, 0, 0, 2723, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 58, 2990, 153, 0, 2996, 258, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1471, 0, 0, 1234, 2776, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 0, 67, 254, 0, 4010, 7, 4066, 854, 88, 0, 0, 0, 0, 0, 0, 170, 104, 29, 0, 0, 4055, 0, 29, 836, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1, 652, 0, 0, 0, 3337, 1707, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 1313, 176, 52, 0, 0, 0, 0, 0, 0, 6, 2336, 0, 4096, 0, 3624, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9932, 10439, 5483, 13041, 5223, 5717, 5639, 10062] avg=8192.0000 max=13041 max_rank=3 min=5223 min_rank=4 imbalance_ratio=1.591919 max_load_reduction=52495 max_load_reduction_pct=0.801010 moved_to_echo=26641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 2044, 1, 93, 0, 0, 1577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 29, 0, 4058, 2996, 1141, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2013, 0, 4095, 246, 0, 56, 2781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1372, 0, 0, 978, 0, 0, 421, 88, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 4014, 0, 0, 836, 0, 0, 0, 0, 0, 0, 0, 75, 0, 84, 4081, 2, 1319, 2323, 3077, 1705, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4070, 45, 17, 0, 1315, 176, 52, 0, 0, 0, 0, 0, 0, 0, 2333, 0, 3358, 47, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7809, 12150, 9198, 2859, 8962, 12666, 5680, 6212] avg=8192.0000 max=12666 max_rank=5 min=2859 min_rank=3 imbalance_ratio=1.546143 max_load_reduction=52870 max_load_reduction_pct=0.806732 moved_to_echo=33520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 9, 0, 1, 2203, 75, 3226, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 17, 484, 4, 2062, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 751, 0, 0, 4089, 2639, 0, 0, 0, 0, 0, 0, 0, 0, 57, 180, 31, 4096, 141, 0, 0, 4055, 853, 61, 0, 0, 0, 0, 0, 0, 0, 3877, 2, 2186, 72, 467, 330, 1212, 575, 0, 0, 0, 0, 0, 0, 0, 0, 1531, 5, 4027, 318, 0, 4, 0, 1680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3812, 4023, 0, 682, 1248, 121, 51, 0, 0, 0, 0, 0, 0, 3933, 2299, 2, 4059, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5522, 6543, 7479, 9474, 8721, 7565, 9937, 10295] avg=8192.0000 max=10295 max_rank=7 min=5522 min_rank=0 imbalance_ratio=1.256714 max_load_reduction=55241 max_load_reduction_pct=0.842911 moved_to_echo=36518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4041, 0, 0, 0, 1768, 1, 16, 100, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4, 36, 356, 2868, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3772, 17, 1, 227, 2765, 0, 0, 0, 0, 0, 0, 0, 0, 517, 114, 11, 0, 3, 4056, 0, 4092, 26, 85, 0, 0, 0, 0, 0, 0, 0, 56, 3418, 0, 2804, 3838, 0, 4096, 800, 0, 0, 0, 0, 0, 0, 0, 1018, 1006, 3, 3745, 0, 0, 216, 4025, 1708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4087, 43, 0, 0, 1308, 169, 52, 0, 0, 0, 0, 0, 2, 0, 2336, 11, 3735, 0, 2125, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5926, 3299, 6792, 8904, 15012, 11721, 5673, 8209] avg=8192.0000 max=15012 max_rank=4 min=3299 min_rank=1 imbalance_ratio=1.832520 max_load_reduction=50524 max_load_reduction_pct=0.770935 moved_to_echo=40615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8448, 640, 1664, 8832, 256, 17024, 4608, 19456, 4608, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5248, 2688, 4224, 16384, 768, 4608, 2688, 16768, 12672, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 5888, 17280, 1408, 23040, 128, 18048, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 2048, 384, 19840, 8320, 128, 9728, 21632, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 9728, 7680, 6400, 3200, 24960, 512, 7040, 6400, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[16128, 3712, 5504, 128, 11904, 4480, 7040, 17152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7936, 4096, 3328, 1408, 8064, 22912, 2432, 16000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 5632, 7552, 256, 29312, 9856, 128, 768, 10240, 1408, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[3717, 0, 3920, 11, 0, 55, 429, 0, 6, 5, 6, 4002, 0, 3470, 1, 174, 0, 448, 1525, 4062, 18, 0, 843, 4084, 0, 2500, 2144, 716, 0, 0, 4096, 11, 0, 0, 393, 149, 1797, 980, 13, 0, 3, 3391, 3965, 0, 28, 0, 3409, 865, 4, 3831, 4036, 0, 4096, 0, 0, 0, 0, 31, 1608, 0, 107, 0, 71, 516] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[3504, 0, 4095, 0, 0, 0, 4063, 4, 0, 0, 3234, 2796, 0, 2, 0, 558, 0, 0, 885, 0, 3688, 0, 0, 4081, 0, 13, 10, 0, 78, 4032, 902, 1, 4, 3166, 3153, 461, 522, 0, 1975, 1231, 1040, 3668, 3927, 38, 35, 132, 1549, 0, 0, 1253, 9, 0, 4040, 0, 0, 2501, 400, 0, 0, 0, 384, 23, 0, 4079] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2545, 0, 4090, 917, 0, 0, 23, 0, 595, 78, 3497, 56, 0, 28, 725, 0, 0, 0, 0, 2, 1798, 0, 16, 4096, 0, 4055, 4093, 0, 0, 4027, 0, 2, 0, 4072, 4, 64, 0, 1, 4091, 3852, 0, 2908, 1055, 12, 0, 3234, 400, 2574, 0, 0, 1579, 0, 3332, 0, 0, 3700, 0, 0, 0, 6, 9, 66, 3934, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[7, 315, 152, 3, 361, 2, 0, 0, 31, 0, 163, 0, 91, 2237, 0, 227, 0, 0, 0, 63, 12, 294, 4056, 3707, 384, 4096, 3890, 903, 57, 1, 529, 0, 6, 3925, 1625, 0, 1648, 3672, 2737, 3564, 4093, 1465, 0, 0, 0, 228, 3849, 22, 0, 3181, 2816, 307, 4096, 0, 121, 3685, 1380, 8, 0, 0, 57, 18, 1452, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3955, 252, 4096, 0, 11, 646, 29, 1, 0, 451, 3340, 0, 0, 1028, 2, 11, 2795, 24, 1, 0, 4089, 2, 45, 4058, 1, 4070, 3083, 20, 155, 0, 5, 1, 0, 262, 2277, 1107, 0, 17, 4092, 4094, 0, 4066, 0, 52, 0, 0, 3590, 2844, 53, 0, 0, 0, 0, 0, 0, 3161, 0, 0, 68, 0, 3873, 1807, 1550, 452] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[7, 4088, 3835, 0, 0, 0, 37, 0, 0, 0, 1, 4078, 0, 4071, 3, 3579, 2, 1, 0, 0, 139, 0, 3702, 4031, 11, 4096, 929, 0, 1524, 10, 0, 0, 0, 194, 0, 0, 3, 366, 4091, 997, 0, 3924, 2673, 1556, 56, 19, 0, 3533, 50, 3484, 3, 47, 2440, 2, 0, 3469, 49, 0, 340, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3863, 64, 0, 2, 0, 0, 9, 0, 0, 56, 0, 1411, 1, 4065, 273, 0, 0, 202, 387, 9, 2446, 3140, 0, 4096, 1921, 0, 0, 4, 124, 0, 0, 0, 0, 0, 149, 0, 3942, 4033, 0, 3934, 4096, 4, 0, 0, 4096, 4096, 0, 0, 3021, 4096, 4095, 372, 10, 9, 0, 0, 0, 0, 0, 3718, 3792, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 3244, 7, 0, 0, 0, 0, 0, 1, 0, 144, 0, 0, 0, 0, 0, 0, 0, 3363, 2898, 0, 2, 4096, 1, 4095, 2125, 1691, 2412, 0, 10, 0, 0, 0, 412, 608, 0, 0, 3886, 2, 3963, 4088, 1256, 146, 5, 2, 4096, 3335, 0, 4025, 0, 0, 3954, 1153, 5, 2356, 0, 0, 0, 34, 71, 148, 3816, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3717, 0, 3920, 11, 0, 55, 429, 0, 704, 502, 1, 128, 0, 0, 0, 0, 6, 5, 6, 4002, 0, 3470, 1, 174, 2021, 2, 488, 0, 0, 0, 0, 0, 0, 448, 1525, 4062, 18, 0, 843, 3582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2372, 2144, 716, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 149, 1797, 980, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3965, 0, 28, 0, 2705, 865, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3831, 4036, 0, 2075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 1608, 0, 107, 0, 71, 516, 2903, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9467, 10175, 10478, 9339, 3329, 7566, 9946, 5236] avg=8192.0000 max=10478 max_rank=2 min=3329 min_rank=4 imbalance_ratio=1.279053 max_load_reduction=55058 max_load_reduction_pct=0.840118 moved_to_echo=26077 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2545, 0, 4090, 917, 0, 0, 23, 0, 82, 503, 594, 208, 0, 0, 0, 0, 595, 78, 3497, 56, 0, 28, 725, 0, 1644, 740, 420, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1798, 0, 16, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 4093, 0, 0, 4027, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4, 64, 0, 1, 2757, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 12, 0, 3234, 318, 2574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1579, 0, 1688, 0, 0, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 9, 66, 3934, 0, 2488, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8962, 7783, 5409, 11969, 10750, 7193, 6967, 6503] avg=8192.0000 max=11969 max_rank=3 min=5409 min_rank=2 imbalance_ratio=1.461060 max_load_reduction=53567 max_load_reduction_pct=0.817368 moved_to_echo=31413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3504, 0, 4095, 0, 0, 0, 4063, 4, 320, 501, 287, 0, 0, 0, 0, 0, 0, 0, 3234, 2796, 0, 2, 0, 558, 1993, 357, 528, 0, 0, 0, 0, 0, 0, 0, 885, 0, 3688, 0, 0, 3580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 10, 0, 78, 4032, 902, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3166, 3153, 461, 522, 0, 1331, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 0, 3927, 38, 35, 132, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 9, 0, 2047, 0, 0, 2501, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 384, 23, 0, 4079, 3140, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12774, 9468, 8153, 5036, 9868, 6401, 5810, 8026] avg=8192.0000 max=12774 max_rank=0 min=5036 min_rank=3 imbalance_ratio=1.559326 max_load_reduction=52762 max_load_reduction_pct=0.805084 moved_to_echo=30105 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[7, 315, 152, 3, 361, 2, 0, 0, 795, 455, 397, 210, 0, 0, 0, 0, 31, 0, 163, 0, 91, 2237, 0, 227, 2021, 495, 211, 0, 0, 0, 0, 0, 0, 0, 0, 63, 12, 294, 4056, 3252, 0, 0, 0, 0, 0, 0, 0, 0, 384, 3886, 3890, 903, 57, 1, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3925, 1625, 0, 1648, 3672, 1845, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 228, 3054, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3181, 2816, 307, 2075, 0, 121, 3685, 0, 0, 0, 0, 0, 0, 0, 0, 1380, 8, 0, 0, 57, 18, 1452, 0, 1254, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2697, 5476, 7677, 9650, 16285, 7397, 12185, 4169] avg=8192.0000 max=16285 max_rank=4 min=2697 min_rank=0 imbalance_ratio=1.987915 max_load_reduction=49251 max_load_reduction_pct=0.751511 moved_to_echo=40036 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3955, 252, 4096, 0, 11, 646, 29, 1, 742, 498, 595, 209, 0, 0, 0, 0, 0, 451, 3340, 0, 0, 1028, 2, 11, 0, 740, 586, 0, 0, 0, 0, 0, 2795, 24, 1, 0, 4089, 2, 45, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3861, 3083, 20, 155, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 2277, 1107, 0, 17, 2757, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 2848, 2844, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 3873, 1807, 1550, 452, 3480, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11034, 6158, 10516, 7126, 10514, 5744, 3214, 11230] avg=8192.0000 max=11230 max_rank=7 min=3214 min_rank=6 imbalance_ratio=1.370850 max_load_reduction=54306 max_load_reduction_pct=0.828644 moved_to_echo=30702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3863, 64, 0, 2, 0, 0, 851, 390, 577, 213, 0, 0, 0, 0, 9, 0, 0, 56, 0, 1411, 1, 4065, 2022, 715, 568, 0, 0, 0, 0, 0, 273, 0, 0, 202, 387, 9, 2446, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 1921, 0, 0, 4, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 2650, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 3245, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3021, 4096, 2073, 372, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3718, 3792, 0, 3366, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5960, 8847, 6067, 5932, 6832, 11441, 9581, 10876] avg=8192.0000 max=11441 max_rank=5 min=5932 min_rank=3 imbalance_ratio=1.396606 max_load_reduction=54095 max_load_reduction_pct=0.825424 moved_to_echo=38730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[7, 4088, 3835, 0, 0, 0, 37, 0, 0, 495, 594, 210, 0, 0, 0, 0, 0, 0, 1, 4078, 0, 4071, 3, 3579, 1204, 740, 566, 0, 0, 0, 0, 0, 2, 1, 0, 0, 139, 0, 3702, 3536, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3886, 929, 0, 1524, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 3, 366, 2757, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2673, 1556, 56, 19, 0, 3533, 0, 0, 0, 0, 0, 0, 0, 0, 50, 3484, 3, 47, 1236, 2, 0, 3469, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 340, 0, 4096, 0, 0, 0, 3358, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9266, 14242, 7380, 6360, 4317, 7837, 8291, 7843] avg=8192.0000 max=14242 max_rank=1 min=4317 min_rank=4 imbalance_ratio=1.738525 max_load_reduction=51294 max_load_reduction_pct=0.782684 moved_to_echo=28288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3244, 7, 0, 0, 0, 0, 846, 503, 565, 210, 0, 0, 0, 0, 0, 1, 0, 144, 0, 0, 0, 0, 1951, 702, 589, 0, 0, 0, 0, 0, 0, 0, 0, 3363, 2898, 0, 2, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3885, 2125, 1691, 2412, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 608, 0, 0, 2619, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 1256, 146, 5, 2, 3250, 3335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 2003, 1153, 5, 2356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 71, 148, 3816, 4085, 3499, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5376, 3387, 9856, 10124, 3641, 11957, 9542, 11653] avg=8192.0000 max=11957 max_rank=5 min=3387 min_rank=1 imbalance_ratio=1.459595 max_load_reduction=53579 max_load_reduction_pct=0.817551 moved_to_echo=36793 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 11648, 7936, 2432, 4224, 5120, 16768, 17792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 640, 10368, 11136, 128, 12288, 768, 8704, 12928, 4608, 3968, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[13824, 4736, 27392, 1024, 384, 768, 4608, 128, 4352, 3968, 3712, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 15872, 11520, 4480, 13312, 1536, 256, 18944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[9216, 0, 17024, 1920, 128, 3712, 16768, 17280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1920, 128, 2048, 128, 8704, 5888, 14720, 9216, 23552, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 25728, 18304, 3456, 4352, 8192, 5760, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3072, 512, 2432, 7808, 13056, 384, 11136, 27520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 3400, 8, 1, 0, 0, 2, 0, 0, 0, 3984, 4096, 0, 1258, 0, 51, 0, 2, 238, 0, 0, 0, 1522, 1829, 4039, 21, 2, 4018, 0, 15, 4092, 4072, 0, 0, 1878, 0, 2787, 959, 4091, 38, 269, 0, 0, 4, 0, 10, 0, 427, 1903, 4095, 0, 0, 12, 0, 4073, 0, 0, 4096, 0, 4068, 0, 8, 267, 3901] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 85, 0, 3819, 9, 0, 0, 27, 60, 0, 541, 4096, 0, 7, 439, 252, 3830, 1323, 0, 3898, 3, 0, 1, 0, 0, 3900, 15, 0, 0, 0, 5, 2765, 337, 0, 4090, 4091, 4062, 0, 3492, 23, 327, 0, 0, 0, 0, 3042, 202, 4096, 4055, 0, 980, 0, 0, 32, 4096, 0, 0, 4096, 0, 10, 3430, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 3351, 56, 3985, 1, 0, 0, 0, 4096, 0, 177, 2702, 149, 0, 3823, 0, 4052, 0, 0, 924, 3763, 3, 305, 0, 0, 4088, 0, 4096, 0, 0, 81, 0, 0, 0, 0, 4091, 4096, 12, 4082, 0, 8, 611, 0, 1, 0, 4020, 19, 0, 3250, 0, 0, 40, 309, 57, 4095, 0, 0, 0, 47, 721, 4076, 0, 349, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[22, 0, 696, 73, 2, 0, 102, 4096, 2438, 14, 0, 713, 0, 3, 0, 78, 3366, 0, 0, 1, 1, 1, 13, 1, 65, 4016, 0, 0, 0, 127, 4094, 0, 0, 80, 80, 4086, 4093, 2896, 3918, 10, 0, 2, 0, 1559, 1, 4073, 0, 3964, 34, 178, 229, 0, 17, 0, 3841, 51, 3299, 0, 1887, 0, 4090, 354, 2776, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 17, 0, 1, 0, 0, 104, 0, 4096, 0, 0, 3123, 99, 1, 0, 0, 112, 0, 781, 93, 2924, 0, 4, 0, 0, 79, 0, 2793, 163, 288, 4091, 3914, 0, 0, 16, 3915, 3944, 0, 4085, 78, 0, 8, 0, 0, 3, 431, 0, 0, 2956, 4075, 201, 0, 9, 20, 3993, 123, 1764, 4094, 1220, 3811, 4017, 0, 0, 4090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[207, 1, 3586, 17, 3, 6, 0, 41, 0, 0, 0, 3356, 0, 621, 0, 0, 0, 0, 0, 751, 0, 0, 0, 1865, 2002, 0, 0, 17, 9, 3620, 4090, 366, 2166, 1296, 165, 3964, 3587, 2383, 4021, 0, 0, 991, 1, 4, 0, 2071, 3668, 2627, 4096, 3562, 1192, 40, 492, 0, 3439, 1, 813, 2551, 0, 0, 1115, 703, 25, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 71, 0, 0, 3147, 2, 1369, 0, 42, 593, 1598, 0, 311, 379, 2790, 0, 0, 442, 4085, 0, 1890, 0, 5, 84, 3316, 8, 4096, 2187, 1, 1516, 0, 0, 221, 15, 0, 4096, 3929, 3848, 2, 0, 1378, 5, 2189, 0, 1, 47, 0, 418, 4089, 0, 0, 2820, 0, 4096, 2, 1092, 2807, 1, 4096, 2181, 0, 142, 129] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3, 4096, 78, 0, 0, 0, 4018, 1, 4, 0, 148, 1253, 0, 2045, 784, 0, 1330, 0, 674, 5, 1567, 1973, 4096, 3938, 1503, 3896, 1869, 2841, 1957, 0, 4094, 1, 3468, 0, 0, 0, 3262, 0, 2706, 0, 0, 198, 2493, 87, 0, 768, 0, 0, 1487, 1116, 0, 0, 1, 0, 3917, 0, 95, 3457, 0, 0, 241, 66, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3400, 8, 1, 0, 0, 2, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 4096, 0, 1258, 0, 51, 2308, 318, 0, 0, 0, 0, 0, 0, 0, 2, 238, 0, 0, 0, 1522, 1829, 1027, 489, 0, 0, 0, 0, 0, 0, 4039, 21, 2, 4018, 0, 15, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1878, 0, 1980, 959, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 269, 0, 0, 4, 0, 10, 0, 427, 4092, 515, 164, 0, 0, 0, 0, 0, 1903, 4095, 0, 0, 12, 0, 3046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1273, 0, 4068, 0, 8, 267, 3901, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7338, 12015, 5107, 12167, 4855, 5481, 9056, 9517] avg=8192.0000 max=12167 max_rank=3 min=4855 min_rank=4 imbalance_ratio=1.485229 max_load_reduction=53369 max_load_reduction_pct=0.814346 moved_to_echo=28909 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3351, 56, 3985, 1, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 177, 2702, 149, 0, 3823, 0, 0, 468, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 924, 3763, 3, 305, 0, 1032, 719, 150, 0, 0, 0, 0, 0, 0, 3938, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2909, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 611, 0, 1, 0, 4020, 19, 0, 81, 0, 163, 0, 0, 0, 0, 0, 3250, 0, 0, 40, 309, 57, 3063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 721, 4076, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11312, 11415, 10948, 8034, 7012, 4903, 6719, 5193] avg=8192.0000 max=11415 max_rank=1 min=4903 min_rank=5 imbalance_ratio=1.393433 max_load_reduction=54121 max_load_reduction_pct=0.825821 moved_to_echo=23827 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 4096, 78, 0, 0, 0, 4018, 1, 2597, 0, 0, 0, 0, 0, 0, 0, 4, 0, 148, 1253, 0, 2045, 784, 0, 1948, 373, 0, 0, 0, 0, 0, 0, 1330, 0, 674, 5, 1567, 1973, 4096, 3938, 987, 572, 143, 0, 0, 0, 0, 0, 1503, 3753, 1869, 2841, 1957, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 0, 0, 2317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 2493, 87, 0, 768, 0, 0, 4094, 435, 109, 0, 0, 0, 0, 0, 1487, 1116, 0, 0, 1, 0, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 1074, 0, 0, 241, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10793, 6555, 15285, 11924, 5785, 8184, 5534, 1476] avg=8192.0000 max=15285 max_rank=2 min=1476 min_rank=7 imbalance_ratio=1.865845 max_load_reduction=50251 max_load_reduction_pct=0.766769 moved_to_echo=20979 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 696, 73, 2, 0, 102, 4096, 3761, 0, 0, 0, 0, 0, 0, 0, 2438, 14, 0, 713, 0, 3, 0, 78, 0, 468, 0, 0, 0, 0, 0, 0, 3366, 0, 0, 1, 1, 1, 13, 1, 968, 718, 148, 0, 0, 0, 0, 0, 65, 3868, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 80, 4086, 2907, 2896, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1559, 1, 4073, 0, 3964, 4094, 0, 157, 0, 0, 0, 0, 0, 34, 178, 229, 0, 17, 0, 2873, 51, 0, 0, 0, 0, 0, 0, 0, 0, 3299, 0, 1887, 0, 4090, 354, 2776, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8752, 3714, 5217, 4060, 10059, 13850, 3382, 16502] avg=8192.0000 max=16502 max_rank=7 min=3382 min_rank=6 imbalance_ratio=2.014404 max_load_reduction=49034 max_load_reduction_pct=0.748199 moved_to_echo=43793 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 85, 0, 3819, 9, 0, 0, 27, 3353, 0, 0, 0, 0, 0, 0, 0, 60, 0, 541, 4096, 0, 7, 439, 252, 2311, 468, 0, 0, 0, 0, 0, 0, 3830, 1323, 0, 3898, 3, 0, 1, 0, 1037, 718, 148, 0, 0, 0, 0, 0, 0, 3752, 15, 0, 0, 0, 0, 2765, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 4090, 4091, 2876, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 0, 0, 0, 3042, 202, 4096, 5, 519, 139, 0, 0, 0, 0, 0, 4055, 0, 980, 0, 0, 32, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1266, 0, 10, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7293, 8174, 10958, 6532, 11417, 8330, 8126, 4706] avg=8192.0000 max=11417 max_rank=4 min=4706 min_rank=7 imbalance_ratio=1.393677 max_load_reduction=54119 max_load_reduction_pct=0.825790 moved_to_echo=32579 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[207, 1, 3586, 17, 3, 6, 0, 41, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 621, 0, 0, 1437, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 751, 0, 0, 0, 1865, 867, 629, 0, 0, 0, 0, 0, 0, 2002, 0, 0, 17, 9, 3620, 0, 366, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 1296, 165, 3964, 2548, 2383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 991, 1, 4, 0, 2071, 3668, 2627, 4090, 321, 161, 0, 0, 0, 0, 0, 4096, 3562, 1192, 40, 492, 0, 2572, 1, 0, 0, 0, 0, 0, 0, 0, 0, 813, 793, 0, 0, 1115, 703, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7721, 5824, 4112, 6014, 12522, 13934, 11955, 3454] avg=8192.0000 max=13934 max_rank=5 min=3454 min_rank=7 imbalance_ratio=1.700928 max_load_reduction=51602 max_load_reduction_pct=0.787384 moved_to_echo=41865 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 0, 1, 0, 0, 104, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3123, 99, 1, 0, 0, 2307, 451, 0, 0, 0, 0, 0, 0, 112, 0, 781, 93, 2924, 0, 4, 0, 1006, 692, 2, 0, 0, 0, 0, 0, 0, 77, 0, 2793, 163, 288, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3915, 2801, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3, 431, 0, 0, 4091, 515, 164, 0, 0, 0, 0, 0, 2956, 4075, 201, 0, 9, 20, 2987, 123, 0, 0, 0, 0, 0, 0, 0, 0, 1764, 1272, 1220, 3811, 4017, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 10077, 5614, 7235, 6810, 5212, 10371, 16174] avg=8192.0000 max=16174 max_rank=7 min=4043 min_rank=0 imbalance_ratio=1.974365 max_load_reduction=49362 max_load_reduction_pct=0.753204 moved_to_echo=38567 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 71, 0, 0, 3147, 2, 1369, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 42, 593, 1598, 0, 311, 379, 2790, 1581, 468, 0, 0, 0, 0, 0, 0, 0, 0, 442, 4085, 0, 1890, 0, 5, 1032, 719, 122, 0, 0, 0, 0, 0, 84, 3194, 8, 4096, 2187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 15, 0, 2909, 3929, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1378, 5, 2189, 0, 1, 47, 0, 1516, 353, 153, 0, 0, 0, 0, 0, 418, 4089, 0, 0, 2820, 0, 3064, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 873, 1, 4096, 2181, 0, 142, 129, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 7762, 8295, 9570, 7076, 5642, 10393, 8514] avg=8192.0000 max=10393 max_rank=6 min=5642 min_rank=5 imbalance_ratio=1.268677 max_load_reduction=55143 max_load_reduction_pct=0.841415 moved_to_echo=31625 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7168, 6656, 3200, 12800, 19200, 1152, 3584, 12288, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[18304, 17152, 2688, 128, 3712, 128, 23680, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 11008, 4608, 7936, 128, 3200, 4352, 5632, 29056, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[12800, 1408, 2176, 9856, 8320, 3968, 6016, 7680, 8064, 5376, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7808, 18688, 1920, 17920, 4352, 4096, 0, 11136, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[6016, 1664, 6272, 20224, 21248, 10240, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[10752, 128, 5504, 20992, 256, 4352, 5504, 3200, 11904, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 3200, 2560, 3968, 128, 14464, 3968, 11136, 22144, 2688, 1280, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 3050, 4083, 11, 0, 4094, 3996, 4096, 0, 3028, 65, 0, 0, 0, 9, 324, 0, 3337, 0, 3246, 3, 0, 0, 4082, 0, 19, 4086, 0, 3916, 0, 0, 0, 90, 4092, 1715, 0, 0, 0, 3145, 0, 0, 941, 0, 181, 896, 711, 15, 0, 0, 4, 0, 28, 0, 956, 50, 0, 60, 0, 3761, 1, 4086, 1, 0, 3358] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[986, 0, 0, 0, 0, 145, 3956, 3827, 154, 4043, 1, 0, 6, 0, 21, 4, 10, 0, 0, 0, 170, 3976, 1, 4093, 5, 0, 0, 4093, 346, 0, 4096, 0, 24, 33, 2949, 0, 73, 0, 0, 4080, 0, 22, 1073, 4096, 2, 4069, 0, 720, 0, 0, 0, 91, 2584, 3637, 81, 1818, 2964, 4017, 3, 0, 0, 0, 3267, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[26, 1, 3, 1939, 0, 4083, 4065, 3682, 20, 4096, 2486, 0, 0, 1, 0, 101, 1, 4092, 60, 3768, 421, 3652, 0, 4062, 0, 795, 12, 520, 4052, 0, 3270, 10, 0, 0, 19, 4, 46, 0, 4062, 2330, 1, 0, 0, 4096, 0, 1425, 0, 0, 0, 0, 21, 0, 0, 4067, 0, 178, 0, 0, 0, 14, 0, 0, 4038, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1866, 0, 0, 4096, 126, 3261, 370, 4091, 6, 0, 1337, 3, 0, 0, 0, 0, 232, 4091, 349, 4, 591, 4051, 0, 169, 3184, 0, 514, 0, 3864, 0, 0, 4096, 0, 1, 0, 0, 0, 3871, 25, 0, 3657, 3933, 7, 259, 0, 4096, 0, 0, 0, 0, 0, 2257, 3789, 600, 0, 456, 1, 0, 3038, 1, 3202, 42] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4057, 0, 45, 1, 4096, 4081, 820, 3762, 0, 0, 0, 114, 2, 552, 27, 7, 1962, 2, 0, 3994, 0, 4096, 0, 377, 0, 299, 3814, 0, 3078, 216, 3, 0, 0, 0, 90, 0, 1988, 3946, 299, 323, 0, 3705, 0, 3148, 0, 823, 0, 6, 0, 68, 0, 4096, 0, 0, 73, 3, 1, 0, 0, 3569, 4009, 3984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 156, 0, 2297, 0, 52, 3703, 4091, 0, 370, 2968, 1, 1417, 0, 0, 0, 0, 4096, 0, 0, 0, 3177, 0, 4080, 535, 3737, 0, 0, 0, 0, 4096, 0, 0, 4096, 3912, 82, 281, 31, 3605, 3271, 0, 0, 0, 229, 0, 4070, 0, 3859, 208, 610, 0, 0, 0, 1766, 0, 0, 1, 0, 20, 1481, 2360, 0, 0, 878] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1819, 1041, 0, 3939, 163, 3375, 0, 1693, 0, 0, 4086, 0, 0, 4095, 4055, 20, 332, 4092, 0, 0, 195, 111, 0, 0, 1, 4, 0, 4041, 4094, 4, 3, 3505, 4091, 1298, 0, 0, 16, 4063, 0, 0, 0, 4095, 3, 5, 0, 4096, 5, 19, 4, 0, 911, 2771, 3446, 5, 0, 0, 3, 37, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[785, 367, 21, 0, 0, 0, 0, 3640, 89, 0, 29, 1898, 3059, 0, 0, 4043, 0, 13, 1, 4041, 295, 1643, 0, 3987, 1, 9, 2122, 19, 0, 3646, 139, 0, 3935, 0, 0, 0, 6, 0, 548, 596, 3821, 445, 148, 1967, 0, 0, 0, 4071, 133, 0, 1247, 119, 43, 4096, 3693, 0, 0, 14, 3507, 2486, 3663, 1116, 0, 35] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3050, 4083, 11, 0, 4094, 3996, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3028, 65, 0, 0, 0, 9, 324, 2131, 198, 0, 0, 0, 0, 0, 0, 0, 3337, 0, 3246, 3, 0, 0, 1370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4086, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 4092, 1715, 0, 0, 0, 3145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 0, 181, 896, 711, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 28, 0, 956, 50, 0, 3016, 0, 0, 0, 0, 0, 0, 0, 60, 0, 3761, 1, 4086, 1, 0, 3358, 581, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16116, 5755, 7956, 8021, 9042, 2744, 4054, 11848] avg=8192.0000 max=16116 max_rank=0 min=2744 min_rank=5 imbalance_ratio=1.967285 max_load_reduction=49420 max_load_reduction_pct=0.754089 moved_to_echo=27688 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[986, 0, 0, 0, 0, 145, 3956, 825, 0, 0, 0, 0, 0, 0, 0, 0, 154, 4043, 1, 0, 6, 0, 21, 4, 2137, 185, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 170, 3976, 1, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4093, 346, 0, 4096, 0, 433, 0, 0, 0, 0, 0, 0, 0, 24, 33, 2949, 0, 73, 0, 0, 3260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 1073, 4096, 2, 4069, 0, 720, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 2584, 3637, 81, 1818, 2817, 0, 0, 0, 0, 0, 0, 0, 2964, 4017, 3, 0, 0, 0, 3267, 0, 582, 350, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5912, 6551, 5531, 8973, 6339, 10019, 11028, 11183] avg=8192.0000 max=11183 max_rank=7 min=5531 min_rank=2 imbalance_ratio=1.365112 max_load_reduction=54353 max_load_reduction_pct=0.829361 moved_to_echo=38569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1866, 0, 0, 4096, 126, 703, 0, 0, 0, 0, 0, 0, 0, 0, 370, 4091, 6, 0, 1337, 3, 0, 0, 2115, 157, 0, 0, 0, 0, 0, 0, 0, 0, 232, 4091, 349, 4, 591, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 3184, 0, 514, 0, 3864, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 3657, 3933, 7, 259, 0, 4096, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2257, 3789, 600, 2401, 0, 0, 0, 0, 0, 0, 0, 0, 456, 1, 0, 3038, 1, 3202, 42, 576, 332, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6791, 8079, 6627, 8142, 7190, 12012, 9047, 7648] avg=8192.0000 max=12012 max_rank=5 min=6627 min_rank=2 imbalance_ratio=1.466309 max_load_reduction=53524 max_load_reduction_pct=0.816711 moved_to_echo=35897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4057, 0, 45, 1, 4096, 880, 0, 0, 0, 0, 0, 0, 0, 0, 820, 3762, 0, 0, 0, 114, 2, 552, 2139, 197, 0, 0, 0, 0, 0, 0, 27, 7, 1962, 2, 0, 3994, 0, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 299, 3814, 0, 3078, 216, 419, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 90, 0, 1988, 3153, 0, 0, 0, 0, 0, 0, 0, 0, 299, 323, 0, 3705, 0, 3148, 0, 823, 36, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 68, 0, 4096, 0, 0, 3004, 0, 0, 0, 0, 0, 0, 0, 73, 3, 1, 0, 0, 3569, 4009, 3984, 583, 338, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9079, 7586, 7366, 8203, 5234, 8334, 7174, 12560] avg=8192.0000 max=12560 max_rank=7 min=5234 min_rank=4 imbalance_ratio=1.533203 max_load_reduction=52976 max_load_reduction_pct=0.808350 moved_to_echo=33302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 156, 0, 2297, 0, 52, 3703, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 2968, 1, 1417, 0, 0, 0, 2134, 202, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3177, 0, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 535, 3737, 0, 0, 0, 0, 4096, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3912, 82, 281, 31, 3605, 2605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 4070, 0, 3859, 33, 0, 0, 0, 0, 0, 0, 0, 208, 610, 0, 0, 0, 1766, 0, 0, 3015, 0, 0, 0, 0, 0, 0, 0, 1, 0, 20, 1481, 2360, 0, 0, 878, 584, 284, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7082, 7092, 8635, 8717, 14612, 8191, 5599, 5608] avg=8192.0000 max=14612 max_rank=4 min=5599 min_rank=6 imbalance_ratio=1.783691 max_load_reduction=50924 max_load_reduction_pct=0.777039 moved_to_echo=34010 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[785, 367, 21, 0, 0, 0, 0, 784, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 29, 1898, 3059, 0, 0, 4043, 2082, 176, 0, 0, 0, 0, 0, 0, 0, 13, 1, 4041, 295, 1643, 0, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 2122, 19, 0, 3646, 139, 0, 63, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 6, 0, 548, 477, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 445, 148, 1967, 0, 0, 0, 4071, 5, 0, 0, 0, 0, 0, 0, 0, 133, 0, 1247, 119, 43, 4096, 3693, 0, 2680, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3507, 2486, 3663, 1116, 0, 35, 567, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1957, 11376, 7331, 5999, 4966, 10457, 12011, 11439] avg=8192.0000 max=12011 max_rank=6 min=1957 min_rank=0 imbalance_ratio=1.466187 max_load_reduction=53525 max_load_reduction_pct=0.816727 moved_to_echo=38873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[26, 1, 3, 1939, 0, 4083, 4065, 793, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 2486, 0, 0, 1, 0, 101, 2121, 178, 0, 0, 0, 0, 0, 0, 1, 4092, 60, 3768, 421, 3652, 0, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 12, 520, 4052, 0, 3270, 10, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4, 46, 0, 4062, 1862, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 1425, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 4067, 0, 178, 2711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4038, 17, 578, 200, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10910, 9003, 13357, 8906, 5993, 5543, 6977, 4847] avg=8192.0000 max=13357 max_rank=2 min=4847 min_rank=7 imbalance_ratio=1.630493 max_load_reduction=52179 max_load_reduction_pct=0.796188 moved_to_echo=23360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1819, 1041, 0, 3939, 163, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 0, 0, 4086, 0, 0, 4095, 57, 163, 0, 0, 0, 0, 0, 0, 4055, 20, 332, 4092, 0, 0, 195, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4041, 4094, 4, 431, 0, 0, 0, 0, 0, 0, 0, 3, 3505, 4091, 1298, 0, 0, 16, 3247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 5, 0, 4096, 37, 0, 0, 0, 0, 0, 0, 0, 5, 19, 4, 0, 911, 2771, 3446, 5, 2485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 37, 0, 0, 0, 0, 15, 348, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7689, 10094, 8733, 8575, 12160, 8236, 9646, 403] avg=8192.0000 max=12160 max_rank=4 min=403 min_rank=7 imbalance_ratio=1.484375 max_load_reduction=53376 max_load_reduction_pct=0.814453 moved_to_echo=30445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3200, 4608, 7296, 4096, 13184, 4736, 14592, 8320, 4096, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4096, 15872, 12800, 1408, 512, 128, 13440, 17792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1920, 3584, 11904, 5376, 128, 16512, 20224, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 1792, 4992, 22400, 1024, 13696, 128, 17792, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[384, 640, 1280, 384, 3584, 23680, 11136, 2688, 22144, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 5120, 9472, 4992, 12672, 7808, 22656, 256, 2432, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4096, 11648, 2688, 19328, 1280, 16512, 896, 9600, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1536, 21120, 5632, 1920, 9984, 128, 128, 9216, 14976, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[17, 4071, 0, 2, 0, 3687, 0, 28, 1, 1368, 4096, 892, 0, 4022, 2384, 0, 2, 0, 4092, 0, 4076, 453, 11, 0, 7, 19, 3, 0, 474, 2358, 4094, 0, 427, 0, 0, 1, 4096, 0, 4062, 0, 0, 2467, 22, 0, 0, 4016, 2276, 3889, 5, 21, 176, 0, 0, 260, 3853, 15, 0, 0, 144, 3553, 0, 83, 12, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 2256, 0, 0, 96, 4096, 185, 0, 0, 0, 3850, 3015, 0, 35, 1, 0, 2909, 145, 0, 4095, 4093, 3, 716, 81, 143, 0, 3966, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1375, 3527, 646, 3828, 0, 835, 2974, 0, 0, 4056, 2154, 4044, 905, 0, 0, 0, 0, 0, 0, 43, 25, 3300, 0, 0, 4014, 26] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[30, 0, 14, 0, 2849, 4060, 2, 0, 305, 0, 2, 3646, 15, 0, 5, 6, 3063, 0, 0, 3248, 2739, 0, 200, 4096, 3, 0, 4095, 2466, 0, 4067, 553, 3961, 0, 0, 0, 0, 432, 768, 3656, 1552, 3994, 3, 135, 4058, 0, 2, 898, 2200, 1381, 1, 0, 1370, 0, 0, 3993, 0, 2, 178, 0, 33, 0, 1454, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[779, 0, 3700, 40, 4096, 0, 3987, 0, 0, 238, 0, 3084, 0, 3955, 117, 84, 2, 15, 2939, 0, 4026, 0, 3, 2, 0, 3813, 4093, 0, 4096, 3786, 451, 181, 3, 0, 0, 0, 3035, 0, 1251, 0, 0, 0, 83, 16, 3820, 0, 488, 0, 0, 52, 0, 4096, 0, 0, 1658, 0, 0, 0, 3995, 0, 2, 23, 3527, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3, 150, 3180, 0, 0, 4084, 0, 4071, 0, 9, 1869, 7, 273, 4096, 0, 193, 20, 12, 0, 0, 884, 4095, 4, 0, 4067, 103, 0, 0, 45, 4096, 23, 0, 0, 1879, 0, 2267, 4041, 0, 4096, 4040, 3285, 3919, 5, 0, 0, 2, 0, 2094, 4006, 234, 0, 398, 0, 0, 3400, 0, 0, 0, 469, 0, 0, 15, 102, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[2072, 793, 2241, 101, 5, 2715, 0, 0, 0, 4, 3921, 3689, 2, 4010, 363, 314, 131, 0, 1710, 1530, 3193, 1198, 0, 0, 2, 0, 4088, 7, 4007, 2223, 4096, 0, 0, 0, 1034, 0, 3151, 636, 4092, 24, 0, 4078, 4, 2, 72, 2874, 0, 0, 2397, 0, 0, 3331, 20, 0, 216, 1173, 0, 0, 2, 0, 15, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[2517, 0, 4095, 3369, 0, 2876, 0, 1508, 0, 0, 2, 0, 712, 3768, 1, 11, 1, 0, 231, 0, 3943, 1135, 0, 0, 2302, 4096, 4075, 0, 94, 124, 4091, 30, 23, 1287, 2, 3935, 2, 0, 2, 91, 0, 4096, 2, 4096, 0, 2658, 0, 124, 2, 1, 2, 19, 0, 42, 4076, 1, 0, 0, 2165, 0, 0, 0, 3929, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1837, 1087, 2136, 723, 18, 1495, 22, 72, 0, 0, 4, 0, 0, 178, 0, 3959, 0, 0, 4096, 4075, 4096, 1, 0, 2953, 0, 3882, 4061, 0, 3030, 0, 0, 0, 0, 246, 0, 150, 0, 0, 0, 2716, 0, 132, 339, 4096, 164, 3597, 89, 5, 4092, 36, 2, 3945, 0, 0, 2940, 0, 0, 7, 3965, 1289, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[17, 4071, 0, 2, 0, 2145, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1368, 4096, 892, 0, 4022, 2384, 0, 978, 85, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 0, 2754, 453, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 19, 0, 0, 474, 2180, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 0, 1, 4096, 0, 4062, 0, 344, 178, 0, 0, 0, 0, 0, 0, 0, 1016, 22, 0, 0, 4016, 2276, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 5, 21, 176, 0, 0, 260, 3853, 15, 1457, 282, 0, 0, 0, 0, 0, 0, 0, 0, 144, 3553, 0, 83, 12, 1, 3, 1169, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6263, 13826, 7312, 6774, 9108, 11219, 6069, 4965] avg=8192.0000 max=13826 max_rank=1 min=4965 min_rank=7 imbalance_ratio=1.687744 max_load_reduction=51710 max_load_reduction_pct=0.789032 moved_to_echo=31361 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[779, 0, 3700, 40, 4096, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 0, 3084, 0, 3955, 117, 84, 966, 0, 0, 0, 0, 0, 0, 0, 2, 15, 2939, 0, 2721, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3813, 0, 0, 4096, 3500, 451, 181, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3035, 0, 1251, 0, 339, 286, 0, 0, 0, 0, 0, 0, 0, 0, 83, 16, 3820, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4096, 0, 0, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 2, 23, 3527, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12602, 8444, 5682, 12041, 4914, 4407, 5806, 11640] avg=8192.0000 max=12602 max_rank=0 min=4407 min_rank=5 imbalance_ratio=1.538330 max_load_reduction=52934 max_load_reduction_pct=0.807709 moved_to_echo=26767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[30, 0, 14, 0, 2849, 2362, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 2, 3646, 15, 0, 5, 6, 657, 94, 0, 0, 0, 0, 0, 0, 3063, 0, 0, 3248, 1851, 0, 200, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2466, 0, 3760, 553, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 768, 3656, 1552, 231, 307, 0, 0, 0, 0, 0, 0, 3994, 2, 135, 4058, 0, 2, 898, 2200, 0, 0, 0, 0, 0, 0, 0, 0, 1381, 1, 0, 1370, 0, 0, 3993, 0, 1604, 0, 0, 0, 0, 0, 0, 0, 2, 178, 0, 33, 0, 1454, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5257, 4730, 12458, 10743, 6946, 11289, 8349, 5764] avg=8192.0000 max=12458 max_rank=2 min=4730 min_rank=1 imbalance_ratio=1.520752 max_load_reduction=53078 max_load_reduction_pct=0.809906 moved_to_echo=32348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2256, 0, 0, 96, 2384, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 3015, 0, 35, 1, 0, 983, 94, 0, 0, 0, 0, 0, 0, 2909, 145, 0, 4095, 2765, 3, 716, 81, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1375, 3527, 345, 0, 0, 0, 0, 0, 0, 0, 646, 1576, 0, 835, 2974, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 2154, 4044, 905, 0, 0, 0, 0, 0, 1618, 438, 0, 0, 0, 0, 0, 0, 0, 43, 25, 3300, 0, 0, 4014, 26, 3966, 1814, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4921, 7978, 10714, 149, 9340, 10087, 9159, 13188] avg=8192.0000 max=13188 max_rank=7 min=149 min_rank=3 imbalance_ratio=1.609863 max_load_reduction=52348 max_load_reduction_pct=0.798767 moved_to_echo=41774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1837, 1087, 2136, 723, 18, 871, 22, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 178, 0, 3959, 983, 34, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 2768, 1, 0, 2953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3882, 0, 0, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 150, 0, 0, 0, 2716, 345, 0, 0, 0, 0, 0, 0, 0, 0, 55, 339, 4096, 164, 3597, 89, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 36, 2, 3945, 0, 0, 2940, 0, 590, 15, 0, 0, 0, 0, 0, 0, 0, 7, 3965, 1289, 0, 0, 0, 1, 4061, 62, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6766, 5158, 13893, 6912, 3457, 8345, 11620, 9385] avg=8192.0000 max=13893 max_rank=2 min=3457 min_rank=4 imbalance_ratio=1.695923 max_load_reduction=51643 max_load_reduction_pct=0.788010 moved_to_echo=32807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2517, 0, 4095, 3369, 0, 1666, 0, 1508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 712, 3768, 1, 11, 952, 70, 0, 0, 0, 0, 0, 0, 1, 0, 231, 0, 2654, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2302, 4096, 0, 0, 94, 112, 4091, 30, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1287, 2, 3935, 2, 0, 2, 91, 337, 12, 0, 0, 0, 0, 0, 0, 0, 1679, 2, 4096, 0, 2658, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 19, 0, 42, 4076, 1, 1140, 472, 0, 0, 0, 0, 0, 0, 0, 0, 2165, 0, 0, 0, 3929, 0, 4075, 1945, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13155, 5516, 4021, 10725, 5691, 8559, 5755, 12114] avg=8192.0000 max=13155 max_rank=0 min=4021 min_rank=2 imbalance_ratio=1.605835 max_load_reduction=52381 max_load_reduction_pct=0.799271 moved_to_echo=32119 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 150, 3180, 0, 0, 2376, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1869, 7, 273, 4096, 0, 193, 212, 94, 0, 0, 0, 0, 0, 0, 20, 12, 0, 0, 598, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 103, 0, 0, 45, 3787, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1879, 0, 2267, 4041, 0, 4096, 4040, 74, 309, 0, 0, 0, 0, 0, 0, 3285, 1613, 5, 0, 0, 2, 0, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 234, 0, 398, 0, 0, 3400, 0, 1614, 449, 0, 0, 0, 0, 0, 0, 0, 0, 469, 0, 0, 15, 102, 0, 0, 1857, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9780, 6753, 4729, 8025, 16706, 6999, 10101, 2443] avg=8192.0000 max=16706 max_rank=4 min=2443 min_rank=7 imbalance_ratio=2.039307 max_load_reduction=48830 max_load_reduction_pct=0.745087 moved_to_echo=36249 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2072, 793, 2241, 101, 5, 1580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3921, 3689, 2, 4010, 363, 314, 766, 62, 0, 0, 0, 0, 0, 0, 131, 0, 1710, 1530, 2158, 1198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 4007, 2055, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 0, 3151, 636, 4092, 24, 269, 168, 0, 0, 0, 0, 0, 0, 0, 1679, 4, 2, 72, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2397, 0, 0, 3331, 20, 0, 216, 1173, 1073, 467, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 15, 0, 0, 0, 4088, 1932, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6792, 13131, 6727, 10167, 9374, 4631, 8677, 6037] avg=8192.0000 max=13131 max_rank=1 min=4631 min_rank=5 imbalance_ratio=1.602905 max_load_reduction=52405 max_load_reduction_pct=0.799637 moved_to_echo=28719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[6528, 12032, 0, 2560, 11776, 15488, 13312, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6144, 256, 13184, 13056, 18304, 6912, 1024, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 256, 10880, 8192, 128, 1664, 11648, 128, 24448, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7296, 8448, 15488, 4352, 7168, 13440, 4224, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7936, 7680, 640, 13184, 7040, 13184, 3840, 12416, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 3456, 1152, 6400, 18944, 1408, 18560, 12032, 2304, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[14080, 4480, 1152, 13184, 128, 384, 20224, 1280, 9216, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 1664, 13824, 14336, 1024, 20096, 2944, 4608, 6528, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4085, 0, 362, 30, 4060, 4096, 69, 0, 0, 0, 2426, 3668, 4, 4080, 256, 2, 0, 4087, 0, 0, 21, 0, 137, 2981, 3845, 3618, 0, 0, 4096, 1, 4095, 1575, 0, 467, 0, 71, 0, 1, 0, 0, 0, 0, 4088, 0, 42, 937, 0, 84, 4053, 0, 0, 0, 680, 0, 801, 64, 3906, 42, 2693, 0, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[3978, 0, 643, 4019, 4028, 0, 0, 63, 0, 0, 1216, 2877, 0, 2478, 174, 3293, 586, 0, 0, 0, 327, 149, 0, 294, 949, 0, 0, 2737, 52, 0, 23, 3569, 1446, 0, 1373, 84, 0, 4091, 1, 3824, 109, 991, 1, 0, 0, 3429, 0, 0, 4096, 0, 0, 3168, 716, 2947, 4, 0, 17, 45, 3700, 26, 0, 3965, 48, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[3390, 0, 0, 3576, 0, 671, 147, 0, 3974, 0, 0, 0, 0, 4076, 0, 4096, 4094, 50, 3556, 0, 15, 0, 0, 0, 0, 52, 0, 2651, 0, 0, 0, 4046, 0, 6, 0, 1940, 1362, 2949, 0, 0, 3932, 4095, 631, 4, 0, 0, 2, 17, 3221, 0, 1, 4, 147, 0, 5, 101, 4076, 0, 469, 4088, 0, 4090, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[3019, 0, 4094, 1, 3927, 124, 16, 1, 4050, 1, 0, 4091, 2, 0, 112, 0, 4048, 4039, 4060, 0, 1648, 0, 175, 22, 0, 1192, 3950, 4069, 2744, 3987, 1, 0, 0, 5, 1, 393, 1997, 203, 44, 4096, 0, 0, 0, 0, 0, 0, 838, 0, 4092, 10, 0, 497, 227, 0, 0, 0, 85, 10, 2686, 0, 31, 948, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[524, 0, 0, 4081, 1109, 0, 0, 1, 2563, 1090, 0, 4048, 527, 5, 550, 3382, 3890, 2915, 0, 0, 0, 31, 4096, 70, 0, 0, 3, 4096, 0, 2, 0, 4034, 0, 12, 131, 4090, 4096, 0, 40, 521, 0, 0, 0, 0, 0, 3991, 0, 13, 3229, 3719, 3225, 3044, 0, 0, 58, 2, 0, 0, 0, 74, 9, 1480, 0, 785] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1314, 0, 4077, 2729, 0, 3, 0, 4095, 1, 0, 0, 2232, 205, 2621, 0, 4095, 4096, 5, 3337, 2, 328, 0, 0, 30, 0, 0, 30, 4082, 0, 1344, 107, 2721, 0, 0, 0, 0, 4093, 3373, 0, 0, 180, 0, 0, 1, 39, 0, 0, 50, 3636, 346, 365, 4096, 0, 14, 2030, 4095, 0, 1, 20, 4095, 1567, 7, 72, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[3891, 1, 0, 0, 0, 3444, 11, 0, 0, 127, 0, 0, 0, 2974, 424, 3621, 1790, 4096, 651, 270, 0, 8, 601, 0, 0, 77, 3732, 4096, 0, 0, 4, 2768, 0, 0, 0, 0, 4096, 1, 3969, 35, 4066, 1692, 0, 0, 0, 4058, 2171, 288, 77, 0, 0, 3835, 635, 7, 1, 0, 12, 1236, 158, 2454, 0, 3728, 73, 358] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 3781, 4047, 8, 4, 435, 4090, 3903, 0, 0, 1, 404, 3034, 633, 4096, 3706, 6, 1788, 0, 130, 4, 0, 0, 0, 0, 3318, 4090, 0, 0, 2, 4090, 0, 9, 29, 0, 84, 3329, 0, 1058, 0, 0, 108, 0, 0, 1, 0, 1, 2254, 9, 0, 3968, 0, 298, 37, 3231, 41, 0, 4093, 3984, 62, 1370, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1850, 0, 362, 30, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 0, 0, 0, 2426, 3668, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 256, 2, 0, 4087, 0, 0, 21, 297, 92, 0, 0, 0, 0, 0, 0, 0, 137, 2981, 1544, 3618, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1575, 0, 467, 0, 71, 0, 1186, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4088, 0, 42, 2301, 2, 0, 0, 0, 0, 0, 0, 937, 0, 84, 3756, 0, 0, 0, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 64, 3906, 42, 2693, 0, 13, 957, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6302, 10261, 8835, 12376, 7395, 6434, 5457, 8476] avg=8192.0000 max=12376 max_rank=3 min=5457 min_rank=6 imbalance_ratio=1.510742 max_load_reduction=53160 max_load_reduction_pct=0.811157 moved_to_echo=27762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3978, 0, 643, 1819, 4028, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 2877, 0, 2478, 174, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 0, 327, 149, 0, 294, 232, 91, 0, 0, 0, 0, 0, 0, 949, 0, 0, 1099, 52, 0, 23, 3569, 0, 0, 0, 0, 0, 0, 0, 0, 1446, 0, 1373, 84, 0, 4091, 1, 3824, 1167, 0, 0, 0, 0, 0, 0, 0, 109, 991, 1, 0, 0, 3429, 0, 0, 1638, 1886, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2936, 716, 2947, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 45, 3700, 26, 0, 3965, 48, 0, 942, 403, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10531, 7749, 1679, 5692, 11986, 8054, 10699, 9146] avg=8192.0000 max=11986 max_rank=4 min=1679 min_rank=2 imbalance_ratio=1.463135 max_load_reduction=53550 max_load_reduction_pct=0.817108 moved_to_echo=39885 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3019, 0, 4094, 1, 3927, 124, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 1, 0, 4091, 2, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 4039, 4060, 0, 1648, 0, 175, 22, 36, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 3950, 1634, 2744, 3987, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 393, 1997, 203, 44, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 0, 2435, 0, 0, 0, 0, 0, 0, 0, 4092, 10, 0, 461, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 10, 2686, 0, 31, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11182, 8256, 14028, 13508, 6739, 3273, 4790, 3760] avg=8192.0000 max=14028 max_rank=2 min=3273 min_rank=5 imbalance_ratio=1.712402 max_load_reduction=51508 max_load_reduction_pct=0.785950 moved_to_echo=18562 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3390, 0, 0, 1619, 0, 671, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 4076, 0, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 50, 3556, 0, 15, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 52, 0, 1065, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1940, 1362, 2949, 0, 0, 1038, 0, 0, 0, 0, 0, 0, 0, 3932, 4095, 631, 4, 0, 0, 2, 17, 1586, 2346, 0, 0, 0, 0, 0, 0, 3221, 0, 1, 4, 147, 0, 5, 101, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 469, 4088, 0, 4090, 2, 0, 838, 501, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5827, 9299, 7796, 5163, 7295, 12613, 3479, 14064] avg=8192.0000 max=14064 max_rank=7 min=3479 min_rank=6 imbalance_ratio=1.716797 max_load_reduction=51472 max_load_reduction_pct=0.785400 moved_to_echo=37451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1314, 0, 4077, 1225, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2232, 205, 2621, 0, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 3337, 2, 328, 0, 0, 30, 305, 65, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1633, 0, 1344, 107, 2721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3373, 0, 0, 795, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 1, 39, 0, 0, 50, 2449, 2347, 0, 0, 0, 0, 0, 0, 3636, 346, 365, 3791, 0, 14, 2030, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 4095, 1567, 7, 72, 2, 644, 503, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10714, 6304, 8168, 5835, 8261, 5066, 14277, 6911] avg=8192.0000 max=14277 max_rank=6 min=5066 min_rank=5 imbalance_ratio=1.742798 max_load_reduction=51259 max_load_reduction_pct=0.782150 moved_to_echo=34515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3891, 1, 0, 0, 0, 3444, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 2974, 424, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 1790, 4096, 651, 270, 0, 8, 601, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 77, 3732, 1645, 0, 0, 4, 2768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3969, 35, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 1692, 0, 0, 0, 4058, 2171, 288, 2451, 2074, 0, 0, 0, 0, 0, 0, 77, 0, 0, 3554, 635, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1236, 158, 2454, 0, 3728, 73, 358, 0, 443, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7347, 4629, 7697, 8226, 8101, 16800, 4274, 8462] avg=8192.0000 max=16800 max_rank=5 min=4274 min_rank=6 imbalance_ratio=2.050781 max_load_reduction=48736 max_load_reduction_pct=0.743652 moved_to_echo=37637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[524, 0, 0, 1848, 1109, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2563, 1090, 0, 4048, 527, 5, 550, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 2915, 0, 0, 0, 31, 4096, 70, 223, 92, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1645, 0, 2, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 131, 4090, 4096, 0, 40, 521, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 13, 2451, 1937, 0, 0, 0, 0, 0, 0, 3229, 3719, 3225, 2821, 0, 0, 58, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 9, 1480, 0, 785, 956, 414, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3482, 9814, 11317, 5684, 10075, 8392, 13054, 3718] avg=8192.0000 max=13054 max_rank=6 min=3482 min_rank=0 imbalance_ratio=1.593506 max_load_reduction=52482 max_load_reduction_pct=0.800812 moved_to_echo=35239 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3781, 1833, 8, 4, 435, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 0, 0, 1, 404, 3034, 633, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 3706, 6, 1788, 0, 130, 4, 0, 0, 291, 91, 0, 0, 0, 0, 0, 0, 0, 0, 3318, 1642, 0, 0, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 29, 0, 84, 3329, 0, 1058, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 1, 0, 1, 2448, 2346, 0, 0, 0, 0, 0, 0, 2254, 9, 0, 3677, 0, 298, 37, 3231, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 4093, 3984, 62, 1370, 0, 0, 948, 501, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10151, 9224, 6016, 9052, 5684, 4904, 9506, 10999] avg=8192.0000 max=10999 max_rank=7 min=4904 min_rank=5 imbalance_ratio=1.342651 max_load_reduction=54537 max_load_reduction_pct=0.832169 moved_to_echo=31093 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 6784, 768, 128, 128, 15616, 3072, 512, 17792, 13056, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[16128, 128, 12672, 10240, 9088, 4608, 640, 12416, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4352, 2176, 11264, 18688, 1792, 18304, 256, 1280, 5376, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[18688, 1408, 1280, 13312, 1152, 17664, 5632, 6912, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[26368, 11392, 13440, 384, 6656, 256, 4992, 512, 1792, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1536, 4224, 3200, 6528, 16256, 13952, 4224, 9600, 6656, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 1536, 14080, 12032, 6528, 5376, 256, 25344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[21632, 4096, 3712, 21120, 1792, 3328, 2176, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[2574, 0, 4088, 0, 0, 1, 0, 0, 942, 0, 4, 2, 0, 0, 0, 281, 87, 0, 5, 10, 62, 2944, 0, 3789, 3821, 4092, 12, 0, 138, 0, 496, 8, 23, 4094, 0, 3, 110, 0, 3009, 3, 0, 4074, 193, 2324, 0, 0, 0, 0, 0, 1980, 4094, 1577, 4096, 0, 3616, 760, 0, 0, 4094, 0, 4071, 0, 4036, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 2, 4064, 19, 0, 2, 0, 0, 0, 0, 0, 0, 3654, 1935, 0, 6, 0, 3692, 0, 43, 0, 3904, 0, 19, 4046, 4089, 4095, 0, 0, 0, 1533, 1, 3716, 0, 4095, 0, 0, 3755, 55, 0, 443, 3469, 0, 558, 323, 281, 4096, 4068, 5, 167, 0, 0, 0, 152, 447, 112, 2400, 2191, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[99, 0, 0, 2588, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 1, 3995, 105, 0, 0, 4096, 4060, 1290, 4096, 4096, 0, 0, 3975, 3416, 0, 244, 3262, 0, 2534, 2, 46, 928, 881, 1607, 0, 1, 0, 0, 6, 0, 3225, 2446, 1, 70, 2337, 3962, 0, 6, 0, 0, 0, 0, 0, 3847, 0, 0, 4096, 0, 127] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[773, 0, 4092, 0, 4037, 772, 0, 0, 1, 23, 12, 3950, 0, 75, 0, 0, 4096, 0, 1, 1166, 90, 58, 0, 4062, 6, 0, 0, 0, 3598, 0, 0, 0, 4045, 0, 0, 0, 0, 3946, 3328, 3794, 0, 1155, 0, 0, 5, 4096, 71, 0, 0, 3971, 1066, 413, 79, 29, 4096, 507, 3, 0, 0, 27, 2, 3995, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4066, 0, 15, 3517, 0, 0, 303, 87, 0, 6, 0, 3353, 0, 28, 0, 132, 4087, 0, 0, 1, 3721, 3, 4050, 3, 0, 3, 0, 3064, 40, 362, 4079, 376, 3361, 2032, 38, 4095, 4, 4080, 0, 2557, 1118, 126, 0, 79, 7, 0, 0, 12, 0, 0, 12, 0, 4094, 1161, 0, 3944, 0, 180, 0, 4016, 523, 2801, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4043, 0, 0, 1986, 3, 0, 0, 0, 155, 0, 0, 41, 18, 0, 0, 12, 2473, 8, 0, 0, 4075, 2537, 4060, 3925, 0, 2195, 305, 0, 4047, 0, 0, 0, 1313, 4094, 4, 1, 2, 4096, 0, 22, 0, 3873, 15, 0, 4094, 52, 1236, 0, 1, 143, 4001, 0, 0, 0, 0, 0, 0, 69, 4088, 1, 4073, 379, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[3638, 0, 141, 2846, 356, 535, 74, 0, 246, 0, 3291, 4085, 0, 0, 0, 4005, 0, 0, 0, 24, 3781, 3489, 406, 1595, 3867, 0, 95, 1172, 0, 23, 0, 4089, 5, 0, 8, 16, 0, 4095, 6, 0, 13, 3087, 0, 1, 4096, 0, 0, 0, 1, 3177, 0, 0, 1036, 0, 4069, 3762, 0, 0, 0, 3, 0, 196, 4052, 155] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 1470, 2921, 0, 0, 17, 4095, 0, 0, 0, 4033, 0, 0, 0, 2160, 4096, 9, 0, 48, 4051, 3271, 4086, 484, 18, 773, 0, 3, 3972, 2510, 0, 4045, 3838, 1933, 1058, 0, 0, 0, 8, 0, 605, 4096, 121, 0, 0, 0, 0, 0, 260, 4096, 3079, 14, 165, 0, 0, 0, 0, 2, 0, 43, 18, 21, 10, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2574, 0, 4088, 0, 0, 1, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 942, 0, 4, 2, 0, 0, 0, 281, 62, 52, 0, 0, 0, 0, 0, 0, 67, 0, 5, 10, 0, 2944, 0, 3789, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 4092, 12, 0, 48, 0, 496, 8, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4094, 0, 3, 110, 0, 3009, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 193, 2324, 0, 0, 0, 0, 0, 38, 489, 0, 0, 0, 0, 0, 0, 1980, 2327, 1577, 4096, 0, 3616, 760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4071, 0, 4036, 23, 1278, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6680, 1343, 6815, 8477, 7242, 7118, 14356, 13505] avg=8192.0000 max=14356 max_rank=6 min=1343 min_rank=1 imbalance_ratio=1.752441 max_load_reduction=51180 max_load_reduction_pct=0.780945 moved_to_echo=42221 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[99, 0, 0, 2588, 0, 0, 0, 0, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 1, 4096, 1303, 0, 0, 0, 0, 0, 0, 3060, 105, 0, 0, 0, 4060, 1290, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3975, 1148, 0, 244, 3262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2534, 2, 46, 928, 515, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 3225, 2446, 1, 366, 965, 473, 0, 0, 0, 0, 0, 70, 2337, 2253, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 0, 0, 4096, 0, 127, 1236, 142, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3480, 9491, 12611, 12725, 5632, 7483, 4666, 9448] avg=8192.0000 max=12725 max_rank=3 min=3480 min_rank=0 imbalance_ratio=1.553345 max_load_reduction=52811 max_load_reduction_pct=0.805832 moved_to_echo=27229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 2, 4064, 19, 0, 384, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3654, 3692, 1560, 0, 0, 0, 0, 0, 0, 1482, 0, 6, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 19, 4046, 1374, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1533, 1, 3716, 0, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 55, 0, 443, 3469, 0, 558, 323, 1705, 1155, 485, 0, 0, 0, 0, 0, 281, 4096, 2313, 5, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 447, 112, 2400, 2191, 6, 0, 0, 1270, 69, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8562, 8908, 1531, 13438, 7640, 11948, 6862, 6647] avg=8192.0000 max=13438 max_rank=3 min=1531 min_rank=2 imbalance_ratio=1.640381 max_load_reduction=52098 max_load_reduction_pct=0.794952 moved_to_echo=33097 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[773, 0, 4092, 0, 4037, 772, 0, 0, 813, 0, 0, 0, 0, 0, 0, 0, 1, 23, 12, 3950, 0, 75, 0, 0, 90, 1373, 0, 0, 0, 0, 0, 0, 3137, 0, 1, 1166, 0, 58, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 2303, 3328, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 0, 5, 4096, 71, 0, 1643, 1016, 127, 0, 0, 0, 0, 0, 0, 3971, 607, 413, 79, 29, 4096, 507, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 27, 2, 3995, 4096, 0, 332, 146, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10487, 5524, 8424, 1215, 13470, 8113, 9702, 8601] avg=8192.0000 max=13470 max_rank=4 min=1215 min_rank=3 imbalance_ratio=1.644287 max_load_reduction=52066 max_load_reduction_pct=0.794464 moved_to_echo=39886 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3638, 0, 141, 2846, 356, 535, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 3291, 4085, 0, 0, 0, 4005, 3781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 3489, 406, 1595, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 0, 95, 1172, 0, 23, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8, 16, 0, 2390, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3087, 0, 1, 4096, 0, 0, 0, 1705, 0, 0, 0, 0, 0, 0, 0, 1, 3177, 0, 0, 1036, 0, 4069, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 196, 4052, 155, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7590, 15408, 5514, 9246, 2425, 8902, 12045, 4406] avg=8192.0000 max=15408 max_rank=1 min=2425 min_rank=4 imbalance_ratio=1.880859 max_load_reduction=50128 max_load_reduction_pct=0.764893 moved_to_echo=27778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1470, 2921, 0, 0, 17, 4095, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 2160, 4051, 1518, 0, 0, 0, 0, 0, 0, 3132, 9, 0, 48, 0, 3271, 4086, 484, 0, 0, 0, 0, 0, 0, 0, 0, 18, 773, 0, 3, 1328, 2510, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 1933, 1058, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, 4096, 121, 0, 0, 0, 0, 0, 0, 1126, 371, 0, 0, 0, 0, 0, 260, 4096, 1744, 14, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 43, 18, 21, 10, 11, 964, 148, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13415, 11762, 11030, 8677, 6837, 6319, 6279, 1217] avg=8192.0000 max=13415 max_rank=0 min=1217 min_rank=7 imbalance_ratio=1.637573 max_load_reduction=52121 max_load_reduction_pct=0.795303 moved_to_echo=20652 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4066, 0, 15, 3517, 0, 0, 303, 87, 811, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3353, 0, 28, 0, 132, 3721, 15, 0, 0, 0, 0, 0, 0, 3130, 0, 0, 1, 0, 3, 4050, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3064, 14, 362, 4079, 376, 0, 0, 0, 0, 0, 0, 0, 0, 3361, 2032, 38, 4095, 4, 2377, 0, 2557, 0, 0, 0, 0, 0, 0, 0, 0, 1118, 126, 0, 79, 7, 0, 0, 12, 1703, 11, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4094, 1161, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 4016, 523, 2801, 0, 0, 3, 146, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8799, 7255, 7187, 7898, 14464, 3057, 9207, 7669] avg=8192.0000 max=14464 max_rank=4 min=3057 min_rank=5 imbalance_ratio=1.765625 max_load_reduction=51072 max_load_reduction_pct=0.779297 moved_to_echo=34397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4043, 0, 0, 1986, 3, 0, 0, 0, 491, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 41, 18, 0, 0, 12, 4075, 1544, 0, 0, 0, 0, 0, 0, 1894, 8, 0, 0, 0, 2537, 4060, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2195, 305, 0, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1313, 4094, 4, 1, 2, 2390, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 15, 0, 4094, 52, 1236, 0, 1706, 1143, 477, 0, 0, 0, 0, 0, 1, 143, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 4088, 1, 4073, 379, 4096, 0, 1249, 88, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6523, 5845, 12424, 3860, 7826, 12596, 2419, 14043] avg=8192.0000 max=14043 max_rank=7 min=2419 min_rank=6 imbalance_ratio=1.714233 max_load_reduction=51493 max_load_reduction_pct=0.785721 moved_to_echo=36884 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[16000, 128, 128, 1280, 0, 16384, 13952, 18048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1408, 128, 7424, 15488, 128, 128, 0, 10368, 23680, 7424, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 768, 12160, 6528, 10880, 11520, 16384, 384, 6400, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12672, 16256, 1152, 7936, 1152, 12416, 8064, 6400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[15744, 7168, 512, 12288, 6528, 7040, 4864, 11904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5504, 16512, 384, 2944, 11776, 7424, 4352, 384, 8832, 5504, 2432, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 19840, 11648, 2048, 9728, 1280, 11904, 9088, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[19328, 0, 9856, 18048, 4480, 5376, 512, 4224, 4224, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 8, 3234, 14, 3124, 0, 1313, 0, 490, 0, 0, 0, 192, 3541, 0, 2875, 0, 0, 0, 3928, 4009, 98, 1775, 2096, 0, 0, 9, 3907, 0, 4045, 3845, 4096, 277, 1601, 0, 3832, 0, 28, 4096, 2, 4089, 3933, 10, 0, 149, 3, 0, 0, 4, 0, 0, 0, 66, 0, 0, 0, 199, 0, 552] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 74, 0, 0, 0, 4079, 2843, 4091, 220, 28, 0, 2, 0, 3941, 583, 4093, 1719, 0, 0, 72, 0, 0, 3820, 4096, 0, 4096, 4088, 0, 0, 13, 0, 0, 1211, 0, 3469, 4094, 0, 1066, 4074, 2674, 26, 0, 2042, 0, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 4096, 4014, 0, 0, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 232, 0, 3540, 1, 0, 0, 0, 0, 115, 255, 4092, 4070, 0, 11, 3727, 0, 96, 0, 3769, 37, 3959, 0, 872, 374, 4012, 0, 201, 207, 0, 4090, 0, 0, 3909, 0, 12, 3032, 1, 10, 0, 4073, 3967, 1572, 3886, 0, 190, 0, 797, 3, 0, 1214, 0, 38, 0, 4094, 0, 3727, 731, 603, 0, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 145, 0, 21, 0, 3854, 7, 0, 3994, 5, 1, 304, 0, 0, 2763, 0, 877, 3254, 1007, 26, 4052, 0, 3471, 17, 0, 0, 0, 0, 0, 12, 194, 0, 6, 4055, 4, 0, 4018, 0, 0, 34, 11, 3939, 0, 258, 3521, 3794, 3288, 1914, 387, 0, 87, 4085, 0, 0, 0, 33, 8, 3808, 4096, 395, 1736, 2055, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 3522, 0, 0, 567, 296, 0, 3078, 0, 0, 3, 17, 70, 0, 564, 1655, 0, 21, 1, 30, 0, 4096, 4050, 2, 3241, 0, 292, 0, 1, 4078, 2, 0, 571, 562, 41, 0, 3598, 3725, 329, 4096, 4096, 4068, 3851, 0, 3773, 3635, 3062, 49, 7, 50, 0, 0, 0, 0, 0, 0, 68, 3941, 425, 0, 0, 1, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 2010, 225, 0, 1062, 0, 3232, 2822, 0, 0, 4096, 0, 4082, 0, 0, 106, 4090, 17, 3534, 0, 0, 0, 0, 0, 0, 2818, 0, 0, 0, 3951, 0, 0, 115, 0, 2083, 344, 0, 433, 29, 161, 3810, 4096, 0, 0, 306, 4096, 4092, 3930, 0, 0, 0, 2065, 0, 2, 3, 55, 0, 4011, 3840, 0, 9, 11, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4001, 4096, 4092, 0, 2416, 4096, 0, 0, 161, 0, 1180, 0, 4084, 395, 4096, 311, 2862, 0, 3612, 4033, 0, 0, 0, 4096, 0, 0, 1, 3, 31, 99, 2, 0, 13, 0, 4096, 3211, 8, 0, 63, 4051, 4094, 19, 0, 0, 0, 17, 0, 1, 0, 0, 0, 2178, 4, 0, 0, 7, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[13, 0, 1, 0, 0, 1, 0, 4096, 44, 4054, 0, 0, 1702, 113, 319, 87, 35, 0, 0, 0, 2643, 1535, 1653, 0, 4001, 3, 1326, 587, 632, 4070, 1, 223, 279, 0, 3979, 445, 3536, 0, 3325, 1, 4095, 4059, 19, 3922, 0, 3920, 4096, 163, 0, 2401, 0, 3990, 0, 0, 24, 0, 0, 0, 0, 0, 65, 68, 10, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 74, 0, 0, 0, 4079, 1315, 821, 0, 0, 0, 0, 0, 0, 2843, 4091, 220, 28, 0, 2, 0, 3941, 563, 245, 0, 0, 0, 0, 0, 0, 583, 4093, 1474, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2248, 4096, 0, 4096, 4088, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1211, 0, 2085, 4094, 0, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 2674, 26, 0, 2042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 0, 0, 0, 0, 0, 0, 0, 110, 4096, 4014, 0, 0, 0, 5, 0, 2812, 0, 257, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6289, 11933, 6222, 14541, 8456, 6004, 797, 11294] avg=8192.0000 max=14541 max_rank=3 min=797 min_rank=6 imbalance_ratio=1.775024 max_load_reduction=50995 max_load_reduction_pct=0.778122 moved_to_echo=26551 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 145, 0, 21, 0, 3854, 7, 0, 0, 953, 0, 0, 0, 0, 0, 0, 3994, 5, 1, 304, 0, 0, 2763, 0, 655, 143, 0, 0, 0, 0, 0, 0, 877, 3254, 864, 26, 4052, 0, 3471, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4055, 4, 0, 2410, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3939, 0, 258, 3521, 0, 0, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 87, 4085, 0, 0, 0, 33, 3288, 3421, 0, 0, 0, 0, 0, 0, 8, 3808, 4096, 395, 1736, 2055, 0, 0, 7, 373, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4980, 7865, 12561, 206, 6509, 9636, 11301, 12478] avg=8192.0000 max=12561 max_rank=2 min=206 min_rank=3 imbalance_ratio=1.533325 max_load_reduction=52975 max_load_reduction_pct=0.808334 moved_to_echo=39924 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 232, 0, 3540, 1, 1362, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 255, 4092, 4070, 635, 532, 0, 0, 0, 0, 0, 0, 0, 11, 3195, 0, 96, 0, 3769, 37, 0, 0, 0, 0, 0, 0, 0, 0, 2330, 0, 872, 374, 4012, 0, 201, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 2349, 0, 12, 3032, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 4073, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 797, 3, 0, 1214, 0, 38, 3886, 1417, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3727, 731, 603, 0, 17, 0, 155, 267, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6060, 9699, 7108, 7996, 9483, 8051, 7545, 9594] avg=8192.0000 max=9699 max_rank=1 min=6060 min_rank=0 imbalance_ratio=1.183960 max_load_reduction=55837 max_load_reduction_pct=0.852005 moved_to_echo=34673 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 8, 3234, 1352, 970, 0, 0, 0, 0, 0, 0, 14, 3124, 0, 1313, 0, 490, 0, 0, 665, 506, 0, 0, 0, 0, 0, 0, 0, 192, 3035, 0, 2875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2311, 4009, 98, 1775, 2096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 0, 4045, 3845, 2461, 277, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 0, 28, 4096, 2, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 3, 0, 0, 4, 0, 0, 3933, 3686, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 199, 0, 552, 2644, 403, 265, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9660, 6112, 6102, 10298, 16136, 5324, 7775, 4129] avg=8192.0000 max=16136 max_rank=4 min=4129 min_rank=7 imbalance_ratio=1.969727 max_load_reduction=49400 max_load_reduction_pct=0.753784 moved_to_echo=33364 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4001, 4096, 1246, 2, 0, 0, 0, 0, 0, 0, 4092, 0, 2416, 4096, 0, 0, 161, 0, 0, 587, 0, 0, 0, 0, 0, 0, 1180, 0, 3497, 395, 4096, 311, 2862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2120, 4033, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 31, 99, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1265, 3211, 8, 0, 63, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1, 0, 0, 4094, 3653, 0, 0, 0, 0, 0, 0, 0, 2178, 4, 0, 0, 7, 11, 0, 2831, 398, 246, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13441, 11352, 12341, 10249, 147, 4566, 7765, 5675] avg=8192.0000 max=13441 max_rank=0 min=147 min_rank=4 imbalance_ratio=1.640747 max_load_reduction=52095 max_load_reduction_pct=0.794907 moved_to_echo=18153 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 1, 0, 0, 1, 0, 4096, 1377, 837, 0, 0, 0, 0, 0, 0, 44, 4054, 0, 0, 1702, 113, 319, 87, 574, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 2643, 1535, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2354, 3, 1326, 587, 632, 4070, 1, 223, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 3979, 445, 2125, 0, 3325, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 4059, 19, 3922, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2401, 0, 3990, 0, 0, 24, 0, 4096, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 68, 10, 0, 2826, 385, 270, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6325, 6893, 5866, 9196, 10154, 9432, 14046, 3624] avg=8192.0000 max=14046 max_rank=6 min=3624 min_rank=7 imbalance_ratio=1.714600 max_load_reduction=51490 max_load_reduction_pct=0.785675 moved_to_echo=37256 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3522, 0, 0, 567, 296, 0, 3078, 1115, 852, 0, 0, 0, 0, 0, 0, 0, 0, 3, 17, 70, 0, 564, 1655, 584, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 30, 0, 4096, 4050, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1908, 0, 292, 0, 1, 4078, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 562, 41, 0, 2162, 3725, 329, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 4068, 3851, 0, 3773, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 7, 50, 0, 0, 0, 0, 0, 0, 3062, 3277, 0, 0, 0, 0, 0, 0, 68, 3941, 425, 0, 0, 1, 2, 0, 2827, 358, 218, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9430, 2893, 8200, 6281, 11486, 13010, 6396, 7840] avg=8192.0000 max=13010 max_rank=5 min=2893 min_rank=1 imbalance_ratio=1.588135 max_load_reduction=52526 max_load_reduction_pct=0.801483 moved_to_echo=38732 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2010, 225, 0, 1062, 0, 3232, 2822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 106, 0, 505, 0, 0, 0, 0, 0, 0, 4090, 17, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2818, 0, 0, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 2083, 344, 0, 433, 29, 161, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 4096, 0, 0, 306, 0, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 0, 2, 3, 55, 4092, 3694, 0, 0, 0, 0, 0, 0, 0, 4011, 3840, 0, 9, 11, 0, 0, 2629, 402, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9351, 8789, 7136, 6769, 3165, 9513, 9911, 10902] avg=8192.0000 max=10902 max_rank=7 min=3165 min_rank=4 imbalance_ratio=1.330811 max_load_reduction=54634 max_load_reduction_pct=0.833649 moved_to_echo=33491 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4992, 8832, 11520, 4736, 13696, 8576, 5376, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 2688, 896, 10240, 0, 1280, 128, 128, 27264, 22784, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[6784, 7680, 15104, 512, 13824, 6016, 15872, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[13312, 14976, 2688, 6912, 10880, 16256, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7552, 22144, 3968, 12416, 13696, 0, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 9856, 256, 4224, 1920, 4224, 10880, 21504, 7808, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 22272, 12416, 4224, 2560, 2944, 128, 640, 16640, 2560, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11008, 11392, 6784, 5760, 6016, 896, 7936, 9984, 3712, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4088, 0, 0, 0, 1260, 3472, 0, 401, 0, 4096, 0, 0, 4096, 1, 1, 210, 3539, 11, 0, 48, 0, 0, 3351, 1462, 1, 3112, 84, 0, 358, 502, 0, 0, 3128, 0, 0, 4073, 0, 2, 0, 4096, 3658, 0, 4084, 0, 9, 0, 4009, 0, 3506, 879, 568, 454, 2381, 0, 126, 639, 0, 0, 0, 109, 3722, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 3625, 74, 0, 0, 201, 1404, 3635, 1005, 4055, 0, 538, 0, 2923, 12, 0, 82, 0, 151, 0, 3065, 1769, 0, 0, 11, 1, 0, 79, 0, 0, 0, 0, 45, 0, 0, 0, 2, 0, 4065, 2406, 0, 0, 2959, 2903, 0, 4088, 0, 4029, 0, 0, 2332, 16, 4095, 0, 3905, 0, 1423, 0, 0, 1, 3973, 13, 2555, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[2, 1486, 0, 0, 23, 4085, 0, 0, 1, 2, 7, 3943, 109, 0, 2255, 0, 0, 3991, 3769, 4091, 3737, 145, 4095, 0, 5, 22, 3, 0, 0, 0, 0, 2663, 12, 3, 0, 1, 0, 0, 1877, 4096, 3811, 0, 524, 0, 0, 299, 4096, 1, 0, 657, 0, 0, 0, 4096, 0, 0, 0, 4074, 0, 0, 4080, 2962, 0, 513] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 3, 3674, 36, 2, 0, 0, 0, 4093, 0, 4096, 4096, 0, 4090, 2651, 755, 0, 0, 4096, 0, 3459, 0, 3, 31, 0, 3, 0, 108, 607, 0, 0, 1263, 0, 0, 0, 0, 0, 3175, 1879, 3968, 2956, 0, 3140, 149, 300, 0, 743, 240, 258, 0, 3786, 15, 4052, 0, 31, 3, 0, 6, 0, 3688, 4081, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[3671, 0, 0, 36, 865, 4076, 0, 316, 0, 4096, 0, 289, 0, 0, 1006, 0, 0, 0, 3648, 648, 0, 8, 1003, 0, 4065, 2049, 85, 0, 0, 0, 0, 0, 11, 0, 0, 6, 98, 0, 0, 4085, 1, 0, 4063, 1, 13, 1125, 4069, 3892, 4094, 2549, 0, 0, 4096, 0, 3589, 0, 5, 4096, 222, 0, 3660, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[94, 4004, 41, 0, 0, 4096, 0, 0, 0, 2314, 2903, 0, 0, 4094, 0, 0, 15, 49, 465, 0, 3533, 0, 3145, 0, 0, 4096, 3, 0, 0, 0, 47, 2, 0, 0, 0, 3878, 0, 2, 0, 4086, 2806, 0, 720, 0, 159, 3635, 0, 20, 1769, 0, 3934, 0, 4079, 1, 1036, 0, 0, 2363, 4049, 2, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[78, 4, 1509, 2768, 0, 120, 2133, 0, 3, 1600, 4096, 436, 167, 4017, 0, 14, 0, 0, 4094, 24, 1721, 3829, 1, 2590, 0, 3547, 19, 5, 0, 0, 0, 0, 0, 4, 0, 4050, 0, 0, 413, 3940, 2459, 0, 3649, 27, 4096, 3762, 0, 20, 1, 0, 4, 687, 4086, 0, 2861, 0, 0, 176, 0, 438, 2086, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[3195, 0, 28, 0, 2124, 4081, 17, 0, 0, 2, 0, 7, 1604, 200, 3890, 4092, 0, 64, 0, 0, 0, 0, 0, 0, 0, 3868, 4033, 2, 8, 2082, 841, 3984, 541, 0, 1958, 3954, 0, 0, 186, 4036, 4096, 0, 4091, 5, 0, 0, 526, 1819, 1042, 0, 2562, 86, 4096, 263, 1831, 0, 19, 0, 0, 0, 303, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 0, 0, 1260, 3472, 0, 401, 45, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 4096, 1, 1, 210, 0, 0, 0, 0, 0, 0, 0, 0, 3539, 11, 0, 48, 0, 0, 3351, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3112, 84, 0, 358, 502, 0, 0, 3782, 1637, 0, 0, 0, 0, 0, 0, 3128, 0, 0, 4073, 0, 2, 0, 4096, 129, 0, 0, 0, 0, 0, 0, 0, 3658, 0, 302, 0, 9, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 879, 568, 454, 1592, 0, 126, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 3722, 0, 0, 0, 615, 1486, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9266, 5281, 8411, 9476, 11428, 7978, 7764, 5932] avg=8192.0000 max=11428 max_rank=4 min=5281 min_rank=1 imbalance_ratio=1.395020 max_load_reduction=54108 max_load_reduction_pct=0.825623 moved_to_echo=33102 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3674, 36, 2, 0, 0, 0, 77, 49, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 4096, 0, 4090, 2651, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 3459, 0, 3, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 108, 607, 0, 0, 1263, 2908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3175, 1879, 3968, 220, 0, 0, 0, 0, 0, 0, 0, 2956, 0, 232, 149, 300, 0, 743, 240, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 3786, 15, 2708, 0, 31, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3688, 4081, 0, 0, 0, 1047, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3841, 19781, 7540, 4889, 9242, 4620, 6801, 8822] avg=8192.0000 max=19781 max_rank=1 min=3841 min_rank=0 imbalance_ratio=2.414673 max_load_reduction=45755 max_load_reduction_pct=0.698166 moved_to_echo=29485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 3625, 74, 0, 0, 201, 1404, 3635, 78, 1, 0, 0, 0, 0, 0, 0, 1005, 964, 0, 538, 0, 2923, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 150, 0, 3065, 1769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 79, 0, 0, 0, 0, 2740, 1620, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 2, 0, 4065, 2406, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 2903, 0, 4088, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2332, 16, 2737, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1423, 0, 0, 1, 3973, 13, 2555, 0, 1058, 1471, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13114, 5442, 5066, 4451, 6740, 11239, 8990, 10494] avg=8192.0000 max=13114 max_rank=0 min=4451 min_rank=3 imbalance_ratio=1.600830 max_load_reduction=52422 max_load_reduction_pct=0.799896 moved_to_echo=37463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 1486, 0, 0, 23, 4085, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 1, 0, 7, 3943, 109, 0, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 3721, 4091, 3737, 145, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 22, 3, 0, 0, 0, 0, 2663, 489, 2, 0, 0, 0, 0, 0, 0, 12, 3, 0, 1, 0, 0, 1877, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3811, 0, 35, 0, 0, 299, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4080, 2962, 0, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5644, 6315, 19780, 3184, 5989, 8242, 4753, 11629] avg=8192.0000 max=19780 max_rank=2 min=3184 min_rank=3 imbalance_ratio=2.414551 max_load_reduction=45756 max_load_reduction_pct=0.698181 moved_to_echo=30613 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[94, 4004, 41, 0, 0, 4096, 0, 0, 78, 5, 0, 0, 0, 0, 0, 0, 0, 550, 2903, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 49, 460, 0, 3533, 0, 3145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 47, 2, 666, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3878, 0, 2, 0, 4086, 221, 0, 0, 0, 0, 0, 0, 0, 2806, 0, 54, 0, 159, 3635, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1769, 0, 3934, 0, 2726, 1, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2363, 4049, 2, 4096, 0, 0, 0, 1054, 839, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8318, 7547, 7202, 5739, 8187, 6674, 9466, 12403] avg=8192.0000 max=12403 max_rank=7 min=5739 min_rank=3 imbalance_ratio=1.514038 max_load_reduction=53133 max_load_reduction_pct=0.810745 moved_to_echo=36730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3195, 0, 28, 0, 2124, 4081, 17, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 1604, 200, 3890, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 4033, 2, 8, 2082, 841, 3984, 3789, 0, 0, 0, 0, 0, 0, 0, 541, 0, 1958, 3954, 0, 0, 186, 4036, 222, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 302, 5, 0, 0, 526, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 0, 2562, 86, 2737, 263, 1831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 303, 0, 0, 0, 1059, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9523, 9795, 64, 18607, 10897, 6748, 8521, 1381] avg=8192.0000 max=18607 max_rank=3 min=64 min_rank=2 imbalance_ratio=2.271362 max_load_reduction=46929 max_load_reduction_pct=0.716080 moved_to_echo=27547 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[78, 4, 1509, 2768, 0, 120, 2133, 0, 83, 49, 0, 0, 0, 0, 0, 0, 3, 374, 4096, 436, 167, 4017, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 24, 1721, 3829, 1, 2590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 19, 5, 0, 0, 0, 0, 3379, 641, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4050, 0, 0, 413, 3940, 224, 0, 0, 0, 0, 0, 0, 0, 2459, 0, 270, 27, 4096, 3762, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 687, 2719, 0, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 438, 2086, 0, 0, 2, 1060, 585, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6744, 9107, 12210, 7591, 8631, 10634, 6272, 4347] avg=8192.0000 max=12210 max_rank=2 min=4347 min_rank=7 imbalance_ratio=1.490479 max_load_reduction=53326 max_load_reduction_pct=0.813690 moved_to_echo=29884 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3671, 0, 0, 36, 865, 4076, 0, 316, 78, 44, 0, 0, 0, 0, 0, 0, 0, 973, 0, 289, 0, 0, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 648, 0, 8, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 2049, 85, 0, 0, 0, 0, 0, 3763, 1637, 0, 0, 0, 0, 0, 0, 11, 0, 0, 6, 98, 0, 0, 4085, 222, 0, 0, 0, 0, 0, 0, 0, 1, 0, 300, 1, 13, 1125, 4069, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2549, 0, 0, 2737, 0, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 222, 0, 3660, 0, 0, 0, 1059, 1486, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9086, 2268, 5263, 11599, 4422, 9401, 12969, 10528] avg=8192.0000 max=12969 max_rank=6 min=2268 min_rank=1 imbalance_ratio=1.583130 max_load_reduction=52567 max_load_reduction_pct=0.802109 moved_to_echo=37320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3712, 4224, 16128, 4864, 15616, 5760, 11648, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1536, 10752, 4352, 4352, 26112, 3072, 2560, 640, 7040, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[10752, 4096, 13312, 1280, 18048, 4480, 13440, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[19840, 0, 1792, 3200, 4608, 12928, 13568, 10112, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[15232, 9216, 5376, 2944, 4352, 20224, 3584, 4352, 640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 16768, 4352, 256, 1024, 2688, 896, 7936, 21632, 6528, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3840, 128, 2048, 16000, 128, 3200, 8448, 30720, 1536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[5120, 3840, 11136, 9344, 6016, 15360, 9856, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[253, 2976, 761, 401, 1203, 466, 779, 1046, 1504, 175, 1522, 396, 1144, 687, 125, 371, 177, 732, 1560, 681, 924, 197, 2760, 1732, 319, 894, 1127, 862, 1501, 536, 1774, 1775, 2824, 621, 369, 534, 133, 2272, 2092, 531, 2842, 398, 1505, 999, 1290, 1625, 1269, 1207, 317, 346, 277, 2288, 977, 595, 497, 126, 485, 855, 899, 1658, 1414, 791, 859, 1281] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[59, 2778, 1006, 380, 1835, 537, 1145, 922, 867, 143, 1763, 658, 1262, 526, 243, 423, 196, 418, 931, 839, 1014, 55, 3510, 2091, 655, 1068, 982, 820, 1069, 258, 1103, 1588, 3083, 599, 462, 653, 300, 1629, 1650, 709, 2886, 210, 1097, 769, 1215, 1752, 1601, 1036, 308, 468, 342, 1811, 1012, 425, 871, 121, 941, 859, 1616, 1884, 1753, 603, 717, 1010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[409, 2761, 378, 523, 1680, 306, 744, 1018, 1366, 146, 1489, 190, 1152, 935, 194, 420, 90, 653, 1025, 711, 758, 270, 3154, 2226, 649, 883, 850, 983, 1009, 524, 1857, 1611, 2719, 1136, 378, 810, 80, 1821, 1775, 751, 2524, 760, 1860, 879, 1766, 1469, 856, 1009, 333, 389, 540, 2354, 1129, 484, 344, 88, 649, 373, 1523, 1785, 1900, 454, 450, 1184] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[40, 2704, 516, 396, 1931, 407, 732, 756, 1063, 97, 1916, 249, 1057, 650, 178, 497, 273, 940, 560, 891, 708, 262, 2743, 2239, 525, 1469, 1027, 1129, 988, 332, 1382, 2318, 3316, 722, 472, 433, 190, 2255, 2071, 722, 2655, 372, 1341, 669, 1695, 1966, 1025, 1431, 327, 371, 430, 2166, 808, 696, 618, 83, 1182, 371, 959, 1913, 1344, 627, 421, 910] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[75, 2425, 818, 207, 1245, 286, 932, 726, 1465, 189, 1627, 275, 478, 1020, 332, 437, 122, 677, 1734, 869, 956, 396, 2901, 2108, 591, 1229, 908, 1099, 1422, 374, 2146, 1718, 2832, 478, 685, 445, 153, 2473, 1777, 1188, 2438, 286, 1974, 1060, 1629, 1380, 847, 954, 296, 415, 237, 1973, 740, 566, 644, 102, 548, 378, 874, 1887, 1847, 433, 1080, 1130] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[140, 2870, 879, 266, 1568, 430, 828, 1154, 853, 73, 1262, 285, 1416, 716, 213, 464, 275, 493, 1414, 1540, 789, 92, 2806, 1489, 478, 731, 951, 1073, 1665, 537, 1232, 1970, 3429, 775, 380, 442, 510, 2087, 2209, 455, 2653, 387, 1381, 988, 1574, 1448, 941, 1491, 415, 198, 412, 2051, 1141, 486, 850, 265, 672, 219, 852, 1783, 1440, 788, 746, 1116] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[63, 3149, 541, 265, 1592, 338, 877, 1015, 1534, 84, 1658, 144, 1121, 1196, 557, 402, 61, 569, 800, 550, 1155, 279, 2817, 2128, 887, 945, 948, 909, 1221, 435, 1963, 1965, 2895, 606, 434, 324, 77, 2590, 2036, 608, 3140, 249, 1583, 597, 1357, 1829, 1150, 1040, 307, 379, 266, 1744, 767, 519, 414, 92, 764, 390, 1166, 2071, 1538, 386, 735, 1315] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[55, 2485, 515, 250, 1518, 292, 1109, 1017, 1047, 141, 1224, 304, 1108, 778, 343, 305, 45, 402, 578, 885, 1244, 83, 3523, 1906, 677, 968, 1009, 1088, 1333, 221, 1460, 2231, 3526, 897, 265, 860, 329, 2027, 2712, 806, 2792, 196, 1200, 627, 1367, 1442, 1222, 682, 227, 450, 301, 2177, 836, 518, 722, 120, 742, 478, 617, 2476, 1967, 460, 1139, 1212] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[253, 2976, 761, 401, 1203, 466, 779, 1046, 277, 148, 0, 0, 0, 0, 0, 0, 1504, 175, 1522, 396, 1144, 687, 125, 371, 1292, 623, 273, 0, 0, 0, 0, 0, 177, 732, 1560, 681, 924, 197, 2137, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 319, 894, 1127, 862, 1501, 536, 1774, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 1232, 621, 369, 534, 133, 2272, 2092, 531, 0, 0, 0, 0, 0, 0, 0, 0, 206, 398, 1505, 999, 1290, 1625, 1269, 1207, 0, 0, 0, 0, 0, 0, 0, 0, 317, 346, 277, 2288, 977, 595, 497, 126, 2636, 300, 0, 0, 0, 0, 0, 0, 485, 855, 899, 1510, 1414, 791, 859, 1281, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8112, 8140, 8238, 7784, 8499, 8359, 8094] avg=8192.0000 max=8499 max_rank=5 min=7784 min_rank=4 imbalance_ratio=1.037476 max_load_reduction=57037 max_load_reduction_pct=0.870316 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[40, 2704, 516, 396, 1931, 407, 732, 756, 362, 171, 0, 0, 0, 0, 0, 0, 1063, 97, 1916, 249, 1057, 650, 178, 497, 1517, 619, 357, 0, 0, 0, 0, 0, 273, 940, 560, 891, 708, 262, 2124, 2239, 0, 0, 0, 0, 0, 0, 0, 0, 525, 1469, 1027, 1129, 988, 332, 1382, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 1447, 722, 472, 433, 190, 2255, 2071, 722, 0, 0, 0, 0, 0, 0, 0, 0, 192, 372, 1341, 669, 1695, 1966, 1025, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 327, 371, 430, 2166, 808, 696, 618, 83, 2463, 352, 0, 0, 0, 0, 0, 0, 1182, 371, 959, 1742, 1344, 627, 421, 910, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8015, 8200, 7997, 8451, 8312, 8691, 8314, 7556] avg=8192.0000 max=8691 max_rank=5 min=7556 min_rank=7 imbalance_ratio=1.060913 max_load_reduction=56845 max_load_reduction_pct=0.867386 moved_to_echo=32873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[409, 2761, 378, 523, 1680, 306, 744, 1018, 256, 163, 0, 0, 0, 0, 0, 0, 1366, 146, 1489, 190, 1152, 935, 194, 420, 1248, 717, 254, 0, 0, 0, 0, 0, 90, 653, 1025, 711, 758, 270, 2437, 2226, 0, 0, 0, 0, 0, 0, 0, 0, 649, 883, 850, 983, 1009, 524, 1857, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 1178, 1136, 378, 810, 80, 1821, 1775, 751, 0, 0, 0, 0, 0, 0, 0, 0, 179, 760, 1860, 879, 1766, 1469, 856, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 333, 389, 540, 2354, 1129, 484, 344, 88, 2345, 293, 0, 0, 0, 0, 0, 0, 649, 373, 1523, 1622, 1900, 454, 450, 1184, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8111, 8170, 7856, 7929, 8778, 8299, 8155] avg=8192.0000 max=8778 max_rank=5 min=7856 min_rank=3 imbalance_ratio=1.071533 max_load_reduction=56758 max_load_reduction_pct=0.866058 moved_to_echo=33161 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[59, 2778, 1006, 380, 1835, 537, 1145, 922, 248, 168, 0, 0, 0, 0, 0, 0, 867, 143, 1763, 658, 1262, 526, 243, 423, 1410, 792, 244, 0, 0, 0, 0, 0, 196, 418, 931, 839, 1014, 55, 2718, 2091, 0, 0, 0, 0, 0, 0, 0, 0, 655, 1068, 982, 820, 1069, 258, 1103, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 1346, 599, 462, 653, 300, 1629, 1650, 709, 0, 0, 0, 0, 0, 0, 0, 0, 209, 210, 1097, 769, 1215, 1752, 1601, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 308, 468, 342, 1811, 1012, 425, 871, 121, 2677, 327, 0, 0, 0, 0, 0, 0, 941, 859, 1616, 1716, 1753, 603, 717, 1010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9078, 8331, 8262, 7051, 7348, 7889, 8362, 9215] avg=8192.0000 max=9215 max_rank=7 min=7051 min_rank=3 imbalance_ratio=1.124878 max_load_reduction=56321 max_load_reduction_pct=0.859390 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[75, 2425, 818, 207, 1245, 286, 932, 726, 268, 168, 0, 0, 0, 0, 0, 0, 1465, 189, 1627, 275, 478, 1020, 332, 437, 1295, 655, 264, 0, 0, 0, 0, 0, 122, 677, 1734, 869, 956, 396, 2246, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 591, 1229, 908, 1099, 1422, 374, 2146, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 478, 685, 445, 153, 2473, 1777, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 177, 286, 1974, 1060, 1629, 1380, 847, 954, 0, 0, 0, 0, 0, 0, 0, 0, 296, 415, 237, 1973, 740, 566, 644, 102, 2261, 301, 0, 0, 0, 0, 0, 0, 548, 378, 874, 1719, 1847, 433, 1080, 1130, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7150, 8037, 9108, 8955, 8435, 8307, 7535, 8009] avg=8192.0000 max=9108 max_rank=2 min=7150 min_rank=0 imbalance_ratio=1.111816 max_load_reduction=56428 max_load_reduction_pct=0.861023 moved_to_echo=32286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[140, 2870, 879, 266, 1568, 430, 828, 1154, 308, 159, 0, 0, 0, 0, 0, 0, 853, 73, 1262, 285, 1416, 716, 213, 464, 1568, 633, 303, 0, 0, 0, 0, 0, 275, 493, 1414, 1540, 789, 92, 2173, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 478, 731, 951, 1073, 1665, 537, 1232, 1359, 0, 0, 0, 0, 0, 0, 0, 0, 1497, 775, 380, 442, 510, 2087, 2209, 455, 0, 0, 0, 0, 0, 0, 0, 0, 192, 387, 1381, 988, 1574, 1448, 941, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 415, 198, 412, 2051, 1141, 486, 850, 265, 2461, 364, 0, 0, 0, 0, 0, 0, 672, 219, 852, 1624, 1440, 788, 746, 1116, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8602, 7786, 8265, 8026, 8355, 8402, 8643, 7457] avg=8192.0000 max=8643 max_rank=6 min=7457 min_rank=7 imbalance_ratio=1.055054 max_load_reduction=56893 max_load_reduction_pct=0.868118 moved_to_echo=32857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[63, 3149, 541, 265, 1592, 338, 877, 1015, 307, 185, 0, 0, 0, 0, 0, 0, 1534, 84, 1658, 144, 1121, 1196, 557, 402, 1324, 636, 302, 0, 0, 0, 0, 0, 61, 569, 800, 550, 1155, 279, 2181, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 887, 945, 948, 909, 1221, 435, 1963, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 606, 434, 324, 77, 2590, 2036, 608, 0, 0, 0, 0, 0, 0, 0, 0, 227, 249, 1583, 597, 1357, 1829, 1150, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 307, 379, 266, 1744, 767, 519, 414, 92, 2913, 307, 0, 0, 0, 0, 0, 0, 764, 390, 1166, 1886, 1538, 386, 735, 1315, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8332, 8958, 7723, 8664, 7939, 8032, 7708, 8180] avg=8192.0000 max=8958 max_rank=1 min=7708 min_rank=6 imbalance_ratio=1.093506 max_load_reduction=56578 max_load_reduction_pct=0.863312 moved_to_echo=31859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[55, 2485, 515, 250, 1518, 292, 1109, 1017, 349, 221, 0, 0, 0, 0, 0, 0, 1047, 141, 1224, 304, 1108, 778, 343, 305, 1613, 795, 343, 0, 0, 0, 0, 0, 45, 402, 578, 885, 1244, 83, 2728, 1906, 0, 0, 0, 0, 0, 0, 0, 0, 677, 968, 1009, 1088, 1333, 221, 1460, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 1538, 897, 265, 860, 329, 2027, 2712, 806, 0, 0, 0, 0, 0, 0, 0, 0, 202, 196, 1200, 627, 1367, 1442, 1222, 682, 0, 0, 0, 0, 0, 0, 0, 0, 227, 450, 301, 2177, 836, 518, 722, 120, 2590, 375, 0, 0, 0, 0, 0, 0, 742, 478, 617, 2255, 1967, 460, 1139, 1212, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7811, 8001, 7871, 8295, 9434, 6938, 8316, 8870] avg=8192.0000 max=9434 max_rank=4 min=6938 min_rank=5 imbalance_ratio=1.151611 max_load_reduction=56102 max_load_reduction_pct=0.856049 moved_to_echo=33558 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1152, 22272, 5504, 2688, 12672, 3072, 7168, 7680, 2432, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1664, 2944, 12032, 6656, 11904, 12928, 8960, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[9728, 1152, 12544, 2560, 8832, 6528, 2304, 3328, 11392, 5504, 2432, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4864, 8192, 7808, 8064, 10240, 3328, 12928, 10496, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2560, 3072, 2816, 16640, 7424, 4352, 4992, 1024, 20352, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6016, 3968, 8576, 14080, 13312, 4608, 6272, 9216, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[10752, 5888, 3456, 4608, 1792, 17280, 16384, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1280, 4992, 8704, 7040, 7552, 1664, 18816, 16000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[3194, 10, 183, 4087, 23, 3084, 3998, 1, 13, 3926, 0, 2, 14, 0, 30, 0, 15, 4057, 1024, 305, 4, 974, 0, 1297, 3088, 0, 4, 4088, 3967, 5, 0, 0, 1322, 276, 0, 2691, 274, 3731, 0, 31, 699, 3, 26, 3779, 29, 7, 35, 1, 3993, 270, 3906, 0, 1178, 0, 0, 46, 0, 0, 19, 15, 3, 372, 3861, 1576] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4053, 0, 2356, 3602, 398, 131, 4046, 0, 30, 687, 9, 1, 0, 0, 484, 0, 31, 3642, 1924, 0, 57, 2002, 0, 43, 2248, 5, 0, 4055, 3891, 30, 47, 64, 878, 191, 0, 2974, 2898, 3421, 0, 2, 829, 19, 5, 2516, 9, 174, 143, 9, 2791, 36, 4032, 0, 2790, 0, 20, 288, 1, 28, 570, 1993, 3, 99, 4044, 937] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1224, 8, 0, 3299, 209, 398, 4068, 0, 0, 3144, 1, 0, 5, 69, 11, 63, 1, 4082, 1669, 1587, 0, 596, 1, 10, 2151, 0, 50, 2664, 3884, 429, 0, 3227, 2188, 2121, 12, 3997, 2561, 4086, 15, 7, 630, 0, 9, 1433, 4, 246, 6, 0, 2450, 2635, 1422, 84, 84, 0, 26, 133, 0, 1, 9, 39, 0, 670, 3983, 3835] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2509, 249, 1103, 3590, 29, 485, 4096, 0, 0, 3866, 0, 34, 38, 12, 33, 12, 13, 4096, 3895, 139, 89, 2593, 0, 3263, 4096, 0, 146, 3195, 3742, 442, 0, 1, 460, 1386, 0, 406, 3700, 3993, 0, 8, 34, 0, 21, 336, 0, 1, 250, 92, 3404, 255, 1759, 115, 8, 4, 9, 8, 0, 0, 22, 374, 119, 1575, 4007, 1424] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3259, 0, 112, 4061, 2163, 677, 3538, 42, 65, 4006, 0, 0, 0, 24, 89, 121, 21, 4051, 827, 1066, 0, 1018, 1, 86, 3885, 0, 0, 3766, 3955, 12, 9, 35, 4, 120, 1, 3950, 1410, 4066, 2, 104, 588, 0, 0, 1457, 209, 135, 1, 7, 2913, 615, 3026, 0, 585, 2, 8, 31, 1, 0, 9, 56, 0, 2219, 4022, 3106] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[3241, 308, 74, 2784, 4, 161, 4096, 164, 0, 1226, 2, 1, 6, 0, 35, 1, 2, 4055, 276, 26, 129, 1016, 0, 3677, 3982, 21, 19, 3783, 3950, 256, 0, 12, 406, 18, 2, 1487, 4062, 833, 1, 1205, 282, 0, 231, 4060, 14, 1827, 589, 45, 3258, 3045, 3852, 0, 1508, 124, 111, 0, 0, 35, 153, 0, 0, 1202, 3786, 93] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2358, 31, 1, 1114, 194, 2000, 4096, 2, 12, 4050, 0, 7, 0, 0, 7, 27, 3, 4004, 1730, 1113, 73, 1041, 15, 1856, 2429, 0, 17, 3609, 4001, 6, 7, 291, 1797, 121, 8, 4093, 2014, 3431, 1, 11, 173, 2, 32, 3864, 20, 1071, 600, 2, 3720, 1324, 3742, 15, 16, 0, 0, 217, 58, 3, 388, 149, 0, 392, 4087, 91] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4031, 10, 280, 2885, 659, 448, 4095, 0, 1, 1846, 0, 0, 0, 14, 0, 424, 0, 4077, 1, 2939, 59, 878, 0, 2814, 1370, 0, 1, 2094, 3415, 1, 13, 153, 231, 783, 0, 4087, 4006, 2887, 0, 747, 1440, 0, 81, 743, 0, 321, 2843, 37, 2881, 4062, 3294, 27, 2, 32, 0, 0, 0, 0, 4, 10, 0, 1446, 3064, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1224, 8, 0, 3299, 209, 398, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3144, 1, 0, 5, 69, 11, 63, 4036, 1172, 0, 0, 0, 0, 0, 0, 1, 2972, 1669, 1587, 0, 596, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 0, 50, 2664, 1228, 429, 0, 3227, 0, 0, 0, 0, 0, 0, 0, 0, 2188, 2121, 12, 3997, 2561, 1000, 15, 7, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 9, 1433, 4, 246, 6, 0, 3086, 1484, 224, 0, 0, 0, 0, 0, 2005, 2635, 1422, 84, 84, 0, 26, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 39, 0, 670, 3983, 3835, 886, 445, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5170, 8501, 6836, 9749, 11901, 7122, 6389, 9868] avg=8192.0000 max=11901 max_rank=4 min=5170 min_rank=0 imbalance_ratio=1.452759 max_load_reduction=53635 max_load_reduction_pct=0.818405 moved_to_echo=35280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4053, 0, 2356, 3602, 398, 131, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 687, 9, 1, 0, 0, 484, 0, 4014, 1175, 0, 0, 0, 0, 0, 0, 31, 2652, 1924, 0, 57, 2002, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 2248, 5, 0, 4055, 1230, 30, 47, 64, 0, 0, 0, 0, 0, 0, 0, 0, 878, 191, 0, 2974, 2898, 837, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 829, 19, 5, 2516, 9, 174, 143, 9, 2584, 1486, 200, 0, 0, 0, 0, 0, 2284, 36, 4032, 0, 2790, 0, 20, 288, 0, 0, 0, 0, 0, 0, 0, 0, 1, 28, 570, 1993, 3, 99, 4044, 937, 790, 507, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10572, 6400, 6709, 7679, 7780, 7974, 9450, 8972] avg=8192.0000 max=10572 max_rank=0 min=6400 min_rank=1 imbalance_ratio=1.290527 max_load_reduction=54964 max_load_reduction_pct=0.838684 moved_to_echo=34176 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3194, 10, 183, 4087, 23, 3084, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3926, 0, 2, 14, 0, 30, 0, 3967, 1198, 0, 0, 0, 0, 0, 0, 15, 2953, 1024, 305, 4, 974, 0, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 3088, 0, 4, 4088, 1254, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 276, 0, 2691, 274, 913, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 699, 3, 26, 3779, 29, 7, 35, 1, 2818, 1515, 223, 0, 0, 0, 0, 0, 3267, 270, 3906, 0, 1178, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 15, 3, 372, 3861, 1576, 881, 726, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10613, 9150, 6572, 8439, 5507, 9135, 8667, 7453] avg=8192.0000 max=10613 max_rank=0 min=5507 min_rank=4 imbalance_ratio=1.295532 max_load_reduction=54923 max_load_reduction_pct=0.838058 moved_to_echo=30762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3259, 0, 112, 4061, 2163, 677, 28, 42, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4006, 0, 0, 0, 24, 89, 121, 3510, 1194, 0, 0, 0, 0, 0, 0, 21, 2950, 827, 1066, 0, 1018, 1, 86, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 0, 3766, 1250, 12, 9, 35, 0, 0, 0, 0, 0, 0, 0, 0, 4, 120, 1, 3950, 1410, 995, 2, 104, 0, 0, 0, 0, 0, 0, 0, 0, 588, 0, 0, 1457, 209, 135, 1, 7, 3071, 1511, 222, 0, 0, 0, 0, 0, 2384, 615, 3026, 0, 585, 2, 8, 31, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 56, 0, 2219, 4022, 3106, 879, 529, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10342, 9009, 5969, 8957, 6586, 7201, 6651, 10821] avg=8192.0000 max=10821 max_rank=7 min=5969 min_rank=2 imbalance_ratio=1.320923 max_load_reduction=54715 max_load_reduction_pct=0.834885 moved_to_echo=31259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3241, 308, 74, 2784, 4, 161, 32, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1226, 2, 1, 6, 0, 35, 1, 4064, 1192, 0, 0, 0, 0, 0, 0, 2, 2952, 276, 26, 129, 1016, 0, 3677, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 21, 19, 3783, 1249, 256, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 406, 18, 2, 1487, 4062, 204, 1, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 231, 4060, 14, 1827, 589, 45, 629, 1509, 223, 0, 0, 0, 0, 0, 2666, 3045, 3852, 0, 1508, 124, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 153, 0, 0, 1202, 3786, 93, 880, 592, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6768, 6527, 8078, 9322, 7385, 9409, 11306, 6741] avg=8192.0000 max=11306 max_rank=6 min=6527 min_rank=1 imbalance_ratio=1.380127 max_load_reduction=54230 max_load_reduction_pct=0.827484 moved_to_echo=34841 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4031, 10, 280, 2885, 659, 448, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1846, 0, 0, 0, 14, 0, 424, 4066, 1034, 0, 0, 0, 0, 0, 0, 0, 2961, 1, 2939, 59, 878, 0, 2814, 0, 0, 0, 0, 0, 0, 0, 0, 1370, 0, 1, 2094, 1071, 1, 13, 153, 0, 0, 0, 0, 0, 0, 0, 0, 231, 783, 0, 4087, 4006, 702, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 1440, 0, 81, 743, 0, 321, 2843, 37, 2185, 1310, 227, 0, 0, 0, 0, 0, 2354, 4062, 3294, 27, 2, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 10, 0, 1446, 3064, 0, 889, 527, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8342, 7385, 9652, 4703, 10556, 9187, 9771, 5940] avg=8192.0000 max=10556 max_rank=4 min=4703 min_rank=3 imbalance_ratio=1.288574 max_load_reduction=54980 max_load_reduction_pct=0.838928 moved_to_echo=35454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2509, 249, 1103, 3590, 29, 485, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 0, 34, 38, 12, 33, 12, 4064, 1130, 0, 0, 0, 0, 0, 0, 13, 2982, 3895, 139, 89, 2593, 0, 3263, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 146, 3195, 1183, 442, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 460, 1386, 0, 406, 3700, 977, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 21, 336, 0, 1, 250, 92, 3016, 1429, 225, 0, 0, 0, 0, 0, 2786, 255, 1759, 115, 8, 4, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 374, 119, 1575, 4007, 1424, 889, 618, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7997, 9189, 12974, 9063, 6937, 5404, 4944, 9028] avg=8192.0000 max=12974 max_rank=2 min=4944 min_rank=6 imbalance_ratio=1.583740 max_load_reduction=52562 max_load_reduction_pct=0.802032 moved_to_echo=26313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2358, 31, 1, 1114, 194, 2000, 32, 2, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4050, 0, 7, 0, 0, 7, 27, 4064, 1208, 0, 0, 0, 0, 0, 0, 3, 2915, 1730, 1113, 73, 1041, 15, 1856, 0, 0, 0, 0, 0, 0, 0, 0, 2429, 0, 17, 3609, 1265, 6, 7, 291, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 121, 8, 4093, 2014, 839, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 173, 2, 32, 3864, 20, 1071, 600, 2, 2592, 1528, 220, 0, 0, 0, 0, 0, 3044, 1324, 3742, 15, 16, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 58, 3, 388, 149, 0, 392, 4087, 91, 869, 676, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5732, 9375, 8746, 7624, 8884, 10104, 8358, 6713] avg=8192.0000 max=10104 max_rank=5 min=5732 min_rank=0 imbalance_ratio=1.233398 max_load_reduction=55432 max_load_reduction_pct=0.845825 moved_to_echo=34059 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[23936, 640, 4224, 25472, 3712, 7424, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23296, 128, 256, 27264, 9856, 1280, 128, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 23424, 11392, 7296, 512, 10240, 128, 13056, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7296, 5120, 128, 23808, 20992, 6528, 128, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 1280, 2688, 128, 8064, 30976, 11136, 7040, 4736, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[20864, 12288, 25088, 256, 6272, 256, 256, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4736, 128, 512, 18304, 384, 3840, 4480, 256, 20096, 11776, 1792, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 22784, 128, 128, 128, 128, 768, 768, 31872, 9344, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4065, 915, 3922, 336, 0, 0, 232, 3481, 2858, 566, 0, 1, 4, 85, 524, 0, 551, 0, 17, 120, 19, 3144, 0, 4096, 32, 72, 1424, 1, 1, 1186, 3770, 75, 3, 85, 3926, 0, 2827, 1093, 3, 317, 3402, 194, 23, 2578, 0, 0, 4091, 2, 0, 0, 6, 21, 4093, 79, 6, 59, 0, 4082, 18, 3046, 4055, 6, 2, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4094, 59, 4093, 337, 3410, 0, 0, 3923, 14, 32, 314, 0, 0, 16, 41, 1124, 1222, 3, 4, 166, 41, 2776, 21, 4082, 3, 0, 45, 46, 3175, 500, 2562, 103, 0, 1, 4096, 13, 0, 3712, 1, 9, 3737, 3698, 0, 3854, 4, 0, 2292, 3539, 3, 0, 65, 157, 3711, 136, 0, 0, 0, 4057, 145, 0, 1, 99, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4049, 0, 3963, 119, 4029, 0, 3695, 4089, 3049, 4, 2776, 0, 0, 1, 984, 8, 13, 0, 0, 4, 0, 86, 16, 4083, 2, 124, 3, 0, 80, 0, 4096, 15, 68, 0, 4025, 38, 6, 3033, 6, 0, 623, 2604, 0, 3281, 324, 0, 4081, 942, 0, 0, 16, 921, 1817, 279, 0, 20, 3, 4047, 1, 14, 3896, 0, 133, 70] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3237, 0, 3901, 1, 0, 0, 2517, 3940, 1706, 0, 534, 168, 75, 75, 545, 1151, 113, 0, 187, 199, 0, 3789, 128, 3720, 0, 248, 513, 0, 0, 4034, 3793, 5, 9, 1585, 4070, 0, 13, 4025, 0, 215, 647, 38, 0, 2440, 25, 10, 1696, 117, 0, 8, 1, 41, 4036, 1306, 5, 167, 0, 2928, 37, 3763, 3761, 7, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[141, 292, 4074, 0, 17, 0, 3705, 4090, 3368, 39, 0, 0, 0, 518, 16, 224, 0, 0, 214, 0, 113, 4079, 0, 4026, 0, 0, 143, 0, 3, 4090, 3949, 870, 186, 0, 4092, 11, 0, 1626, 0, 81, 4080, 555, 0, 3977, 0, 0, 959, 3912, 0, 31, 0, 47, 1124, 0, 1372, 2, 0, 3535, 0, 0, 3989, 0, 0, 1986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3758, 246, 249, 0, 91, 2, 3665, 3750, 2566, 0, 4094, 2, 0, 12, 0, 0, 2, 0, 3247, 48, 0, 1134, 276, 3, 0, 0, 0, 932, 247, 61, 3817, 40, 424, 0, 4060, 0, 5, 1922, 3, 42, 21, 4052, 0, 2006, 27, 5, 4058, 1, 1, 189, 1, 3917, 3365, 14, 692, 22, 198, 4055, 1512, 2640, 4059, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3406, 3, 4093, 0, 302, 0, 3873, 3820, 4, 164, 4018, 2, 2, 0, 3852, 454, 0, 0, 18, 0, 90, 2, 0, 3726, 0, 211, 69, 0, 1748, 143, 4090, 1, 2, 283, 4014, 0, 48, 4095, 0, 0, 2, 4080, 116, 4074, 2, 0, 3543, 297, 44, 1, 0, 98, 19, 504, 225, 1, 7, 4089, 0, 4084, 1242, 570, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3998, 0, 4052, 10, 26, 5, 13, 4043, 0, 315, 77, 0, 0, 32, 278, 10, 52, 0, 11, 0, 85, 2810, 0, 4021, 0, 40, 0, 0, 223, 4095, 3822, 87, 0, 48, 4053, 794, 0, 1295, 22, 735, 2277, 1472, 35, 3034, 0, 0, 4096, 2945, 0, 3, 0, 8, 4028, 65, 1012, 2, 0, 3912, 50, 3573, 3972, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4065, 915, 1508, 336, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2858, 566, 0, 1, 4, 85, 524, 0, 2510, 588, 0, 0, 0, 0, 0, 0, 551, 0, 17, 120, 19, 3144, 0, 4096, 1795, 0, 0, 0, 0, 0, 0, 0, 32, 72, 1424, 1, 1, 1186, 3770, 75, 1424, 31, 0, 0, 0, 0, 0, 0, 3, 85, 3926, 0, 2827, 1093, 3, 317, 509, 189, 0, 0, 0, 0, 0, 0, 3402, 194, 23, 0, 0, 0, 3582, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 21, 4093, 79, 6, 59, 3481, 68, 0, 0, 0, 0, 0, 0, 0, 2469, 18, 3046, 4055, 6, 2, 22, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7056, 7136, 9742, 8016, 8952, 7203, 7813, 9618] avg=8192.0000 max=9742 max_rank=2 min=7056 min_rank=0 imbalance_ratio=1.189209 max_load_reduction=55794 max_load_reduction_pct=0.851349 moved_to_echo=33586 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4094, 59, 1573, 337, 3410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 32, 314, 0, 0, 16, 41, 1124, 3754, 613, 0, 0, 0, 0, 0, 0, 1222, 3, 4, 166, 41, 2776, 21, 4082, 1874, 0, 0, 0, 0, 0, 0, 0, 3, 0, 45, 46, 3175, 500, 2562, 103, 1416, 33, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 13, 0, 3712, 1, 9, 285, 188, 0, 0, 0, 0, 0, 0, 3737, 3698, 0, 0, 4, 0, 2007, 3539, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 65, 157, 3711, 136, 0, 0, 3923, 100, 0, 0, 0, 0, 0, 0, 0, 2453, 145, 0, 1, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9473, 5908, 10189, 7883, 8305, 12985, 8095, 2698] avg=8192.0000 max=12985 max_rank=5 min=2698 min_rank=7 imbalance_ratio=1.585083 max_load_reduction=52551 max_load_reduction_pct=0.801865 moved_to_echo=32083 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4049, 0, 1523, 119, 4029, 0, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3049, 4, 2776, 0, 0, 1, 984, 8, 3195, 594, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4, 0, 86, 16, 4083, 1814, 0, 0, 0, 0, 0, 0, 0, 2, 124, 3, 0, 80, 0, 4096, 15, 1412, 32, 0, 0, 0, 0, 0, 0, 68, 0, 4025, 38, 6, 3033, 6, 0, 507, 188, 0, 0, 0, 0, 0, 0, 623, 2604, 0, 0, 324, 0, 3574, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 921, 1817, 279, 0, 20, 4089, 86, 0, 0, 0, 0, 0, 0, 3, 2447, 1, 14, 3896, 0, 133, 70, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13415, 10611, 6016, 5764, 7871, 8067, 7228, 6564] avg=8192.0000 max=13415 max_rank=0 min=5764 min_rank=3 imbalance_ratio=1.637573 max_load_reduction=52121 max_load_reduction_pct=0.795303 moved_to_echo=29730 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3758, 246, 96, 0, 91, 2, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 4094, 2, 0, 12, 0, 0, 1953, 37, 0, 0, 0, 0, 0, 0, 2, 0, 3247, 48, 0, 1134, 276, 3, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 247, 61, 3817, 40, 1415, 2, 0, 0, 0, 0, 0, 0, 424, 0, 4060, 0, 5, 1922, 3, 42, 504, 188, 0, 0, 0, 0, 0, 0, 21, 4052, 0, 0, 27, 5, 3554, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 189, 1, 3917, 3365, 14, 692, 22, 3750, 53, 0, 0, 0, 0, 0, 0, 198, 2452, 1512, 2640, 4059, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7858, 8664, 4824, 6514, 7148, 7660, 12004, 10864] avg=8192.0000 max=12004 max_rank=6 min=4824 min_rank=2 imbalance_ratio=1.465332 max_load_reduction=53532 max_load_reduction_pct=0.816833 moved_to_echo=37676 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3237, 0, 1500, 1, 0, 0, 2517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1706, 0, 534, 168, 75, 75, 545, 1151, 2376, 584, 0, 0, 0, 0, 0, 0, 113, 0, 187, 199, 0, 3789, 128, 3720, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 248, 513, 0, 0, 4034, 3793, 5, 1021, 31, 0, 0, 0, 0, 0, 0, 9, 1585, 4070, 0, 13, 4025, 0, 215, 211, 136, 0, 0, 0, 0, 0, 0, 647, 38, 0, 0, 25, 10, 1485, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 41, 4036, 1306, 5, 167, 3940, 64, 0, 0, 0, 0, 0, 0, 0, 1771, 37, 3763, 3761, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7255, 7214, 9922, 9645, 10264, 2322, 9568, 9346] avg=8192.0000 max=10264 max_rank=4 min=2322 min_rank=5 imbalance_ratio=1.252930 max_load_reduction=55272 max_load_reduction_pct=0.843384 moved_to_echo=31500 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[141, 292, 1557, 0, 17, 0, 3705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3368, 39, 0, 0, 0, 518, 16, 224, 3874, 614, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 113, 4079, 0, 4026, 1867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 3, 4090, 3949, 870, 1237, 36, 0, 0, 0, 0, 0, 0, 186, 0, 4092, 11, 0, 1626, 0, 81, 123, 168, 0, 0, 0, 0, 0, 0, 4080, 555, 0, 0, 0, 0, 836, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 47, 1124, 0, 1372, 2, 4090, 103, 0, 0, 0, 0, 0, 0, 0, 2130, 0, 0, 3989, 0, 0, 1986, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5712, 8653, 10299, 10328, 6287, 9383, 6769, 8105] avg=8192.0000 max=10328 max_rank=3 min=5712 min_rank=0 imbalance_ratio=1.260742 max_load_reduction=55208 max_load_reduction_pct=0.842407 moved_to_echo=30544 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3406, 3, 1573, 0, 302, 0, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 164, 4018, 2, 2, 0, 3852, 454, 3968, 613, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 90, 2, 0, 3726, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 211, 69, 0, 1748, 143, 4090, 1, 1427, 33, 0, 0, 0, 0, 0, 0, 2, 283, 4014, 0, 48, 4095, 0, 0, 440, 190, 0, 0, 0, 0, 0, 0, 2, 4080, 116, 0, 2, 0, 3103, 297, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1, 0, 98, 19, 504, 225, 1, 3820, 106, 0, 0, 0, 0, 0, 0, 7, 2472, 0, 4084, 1242, 570, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9157, 13077, 5710, 7722, 9072, 7600, 4818, 8380] avg=8192.0000 max=13077 max_rank=1 min=4818 min_rank=6 imbalance_ratio=1.596313 max_load_reduction=52459 max_load_reduction_pct=0.800461 moved_to_echo=29870 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3998, 0, 1558, 10, 26, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 77, 0, 0, 32, 278, 10, 2954, 607, 0, 0, 0, 0, 0, 0, 52, 0, 11, 0, 85, 2810, 0, 4021, 1855, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 223, 4095, 3822, 87, 1365, 32, 0, 0, 0, 0, 0, 0, 0, 48, 4053, 794, 0, 1295, 22, 735, 509, 181, 0, 0, 0, 0, 0, 0, 2277, 1472, 35, 0, 0, 0, 3587, 2945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 4028, 65, 1012, 2, 4043, 80, 0, 0, 0, 0, 0, 0, 0, 2366, 50, 3573, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5610, 4273, 8834, 9664, 7637, 10316, 9241, 9961] avg=8192.0000 max=10316 max_rank=5 min=4273 min_rank=1 imbalance_ratio=1.259277 max_load_reduction=55220 max_load_reduction_pct=0.842590 moved_to_echo=37155 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[14848, 16768, 256, 0, 384, 128, 21760, 11776, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26752, 1536, 11008, 896, 7936, 128, 17792, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[13568, 1152, 11904, 256, 128, 768, 6272, 3072, 24704, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 18560, 1792, 17152, 25088, 768, 256, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2048, 128, 3712, 640, 384, 17920, 512, 27776, 13056, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 768, 2304, 1024, 5504, 14208, 29952, 1280, 10752, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 2048, 32384, 896, 2944, 20864, 128, 1408, 3200, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 256, 128, 5248, 22272, 2432, 3328, 384, 31232, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 35, 4094, 1898, 1161, 315, 51, 38, 26, 0, 4041, 0, 84, 0, 0, 2675, 0, 4086, 20, 1400, 0, 0, 8, 4096, 9, 163, 0, 3913, 0, 1, 0, 127, 779, 53, 1, 3803, 4, 641, 2070, 0, 3859, 3, 4096, 3257, 1843, 0, 4, 0, 3927, 0, 0, 722, 0, 0, 4075, 987, 4078, 469, 0, 0, 57, 2, 2553, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1214, 0, 3956, 3, 1944, 0, 168, 238, 339, 3761, 3518, 0, 0, 0, 94, 2389, 0, 2581, 1, 674, 1, 150, 0, 3169, 7, 30, 3285, 139, 12, 121, 0, 1, 1, 31, 0, 2685, 4050, 2, 2753, 0, 320, 0, 5, 3433, 4096, 0, 731, 10, 3821, 5, 0, 124, 0, 0, 3804, 0, 3869, 3771, 0, 71, 1, 1, 4094, 63] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[28, 0, 4063, 2323, 30, 13, 91, 802, 0, 1402, 7, 6, 117, 12, 1, 3034, 0, 161, 3040, 0, 0, 0, 2558, 80, 3285, 203, 0, 4096, 0, 1, 2, 0, 4, 3, 14, 4095, 4, 0, 1175, 0, 3756, 0, 385, 3917, 4005, 0, 4087, 10, 3942, 163, 3, 6, 3552, 0, 4080, 13, 4007, 253, 2, 0, 183, 21, 2476, 25] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[8, 0, 4062, 0, 4067, 1, 0, 143, 3673, 1, 0, 1, 416, 0, 0, 4086, 7, 803, 0, 0, 3, 0, 4, 3757, 191, 151, 1, 4096, 0, 0, 0, 3, 0, 769, 0, 4088, 35, 1, 3973, 8, 4091, 3761, 6, 4094, 1292, 0, 3143, 442, 189, 14, 1, 3273, 0, 0, 4093, 0, 4096, 0, 67, 0, 39, 0, 2587, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 3556, 1, 275, 0, 2, 3319, 0, 0, 4, 308, 1, 0, 0, 4005, 0, 4096, 0, 61, 17, 85, 783, 58, 2657, 696, 12, 4039, 0, 0, 0, 3, 0, 869, 74, 4093, 676, 0, 1193, 0, 3793, 4, 4003, 3007, 454, 14, 4094, 121, 4035, 2, 36, 553, 0, 12, 4055, 0, 3823, 135, 0, 9, 2062, 0, 3846, 594] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4078, 259, 6, 2, 2, 24, 2532, 8, 1, 0, 498, 0, 66, 3213, 0, 3989, 0, 6, 10, 0, 0, 1397, 465, 0, 0, 3647, 0, 1, 0, 25, 97, 0, 410, 4038, 285, 44, 229, 164, 2883, 4064, 512, 4058, 4094, 0, 4092, 0, 2345, 0, 0, 100, 433, 5, 4091, 4, 4091, 1728, 2581, 0, 839, 25, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4041, 1, 0, 4, 0, 1812, 164, 1366, 93, 374, 41, 0, 0, 3759, 0, 4089, 79, 4096, 301, 0, 0, 2, 2, 0, 1079, 4009, 0, 0, 75, 108, 3, 3, 0, 4093, 0, 37, 354, 35, 4091, 1884, 2075, 3934, 490, 6, 2926, 0, 4096, 16, 8, 3727, 1, 0, 4095, 0, 4093, 0, 0, 0, 0, 1, 4073, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[416, 139, 3988, 1076, 20, 1, 181, 4027, 3843, 0, 634, 51, 65, 13, 0, 3286, 0, 2117, 73, 2, 56, 7, 0, 3112, 70, 154, 0, 3884, 0, 0, 0, 0, 60, 0, 0, 4047, 3, 0, 4022, 0, 3073, 701, 25, 4091, 3383, 0, 260, 9, 1552, 568, 25, 32, 9, 24, 4059, 3880, 4079, 310, 0, 13, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4041, 1, 0, 4, 0, 1812, 962, 0, 0, 0, 0, 0, 0, 0, 164, 1366, 93, 374, 41, 0, 0, 3759, 1823, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 79, 4096, 301, 0, 0, 2, 1499, 662, 0, 0, 0, 0, 0, 0, 2, 0, 1079, 4009, 0, 0, 75, 108, 3272, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4093, 0, 37, 354, 35, 1141, 71, 24, 0, 0, 0, 0, 0, 698, 1884, 2075, 0, 490, 6, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 8, 3727, 1, 0, 2954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3107, 0, 0, 0, 0, 1, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6820, 7620, 10728, 8545, 5761, 8079, 10802, 7181] avg=8192.0000 max=10802 max_rank=6 min=5761 min_rank=4 imbalance_ratio=1.318604 max_load_reduction=54734 max_load_reduction_pct=0.835175 moved_to_echo=31823 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1214, 0, 3956, 3, 1944, 0, 168, 238, 910, 0, 0, 0, 0, 0, 0, 0, 339, 3761, 3518, 0, 0, 0, 94, 2389, 142, 0, 0, 0, 0, 0, 0, 0, 0, 2581, 1, 674, 1, 150, 0, 3169, 117, 579, 0, 0, 0, 0, 0, 0, 7, 30, 3285, 139, 12, 121, 0, 1, 2854, 0, 0, 0, 0, 0, 0, 0, 1, 31, 0, 2685, 4050, 2, 2753, 0, 1060, 5, 23, 0, 0, 0, 0, 0, 56, 0, 5, 0, 4096, 0, 731, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 5, 0, 124, 0, 0, 2744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2936, 3771, 0, 71, 1, 1, 4094, 63, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8433, 10243, 7272, 6449, 10610, 4898, 6694, 10937] avg=8192.0000 max=10937 max_rank=7 min=4898 min_rank=5 imbalance_ratio=1.335083 max_load_reduction=54599 max_load_reduction_pct=0.833115 moved_to_echo=33139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 4063, 2323, 30, 13, 91, 802, 942, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 7, 6, 117, 12, 1, 3034, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 161, 3040, 0, 0, 0, 2558, 80, 1377, 660, 0, 0, 0, 0, 0, 0, 3285, 203, 0, 4096, 0, 1, 2, 0, 3257, 0, 0, 0, 0, 0, 0, 0, 4, 3, 14, 4095, 4, 0, 1175, 0, 1137, 65, 24, 0, 0, 0, 0, 0, 641, 0, 385, 0, 4005, 0, 4087, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 163, 3, 6, 3552, 0, 2943, 13, 0, 0, 0, 0, 0, 0, 0, 0, 3041, 253, 2, 0, 183, 21, 2476, 25, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8292, 6252, 7876, 10844, 6521, 9128, 10622, 6001] avg=8192.0000 max=10844 max_rank=3 min=6001 min_rank=7 imbalance_ratio=1.323730 max_load_reduction=54692 max_load_reduction_pct=0.834534 moved_to_echo=32272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 35, 4094, 1898, 1161, 315, 51, 38, 962, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4041, 0, 84, 0, 0, 2675, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 20, 1400, 0, 0, 8, 4096, 1419, 549, 0, 0, 0, 0, 0, 0, 9, 163, 0, 3913, 0, 1, 0, 127, 2708, 0, 0, 0, 0, 0, 0, 0, 779, 53, 1, 3803, 4, 641, 2070, 0, 1141, 69, 29, 0, 0, 0, 0, 0, 647, 3, 4096, 0, 1843, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 722, 0, 0, 2934, 987, 0, 0, 0, 0, 0, 0, 0, 0, 3087, 469, 0, 0, 57, 2, 2553, 12, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8554, 8550, 11578, 6921, 8590, 6593, 8570, 6180] avg=8192.0000 max=11578 max_rank=2 min=6180 min_rank=7 imbalance_ratio=1.413330 max_load_reduction=53958 max_load_reduction_pct=0.823334 moved_to_echo=29933 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3556, 1, 275, 0, 2, 3319, 899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 308, 1, 0, 0, 4005, 1690, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 61, 17, 85, 783, 58, 1390, 506, 0, 0, 0, 0, 0, 0, 2657, 696, 12, 4039, 0, 0, 0, 3, 2501, 0, 0, 0, 0, 0, 0, 0, 0, 869, 74, 4093, 676, 0, 1193, 0, 1130, 65, 23, 0, 0, 0, 0, 0, 648, 4, 4003, 0, 454, 14, 4094, 121, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 2, 36, 553, 0, 12, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2901, 135, 0, 9, 2062, 0, 3846, 594, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8053, 6008, 6996, 9908, 8123, 9338, 7563, 9547] avg=8192.0000 max=9908 max_rank=3 min=6008 min_rank=1 imbalance_ratio=1.209473 max_load_reduction=55628 max_load_reduction_pct=0.848816 moved_to_echo=34571 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4062, 0, 4067, 1, 0, 143, 963, 0, 0, 0, 0, 0, 0, 0, 3673, 1, 0, 1, 416, 0, 0, 4086, 1823, 0, 0, 0, 0, 0, 0, 0, 7, 803, 0, 0, 3, 0, 4, 3757, 1499, 689, 0, 0, 0, 0, 0, 0, 191, 151, 1, 4096, 0, 0, 0, 3, 3405, 0, 0, 0, 0, 0, 0, 0, 0, 769, 0, 4088, 35, 1, 3973, 8, 1141, 71, 24, 0, 0, 0, 0, 0, 698, 3761, 6, 0, 1292, 0, 3143, 442, 0, 0, 0, 0, 0, 0, 0, 0, 189, 14, 1, 3273, 0, 0, 2952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 0, 67, 0, 39, 0, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9244, 10000, 6762, 7847, 10110, 9342, 6429, 5802] avg=8192.0000 max=10110 max_rank=4 min=5802 min_rank=7 imbalance_ratio=1.234131 max_load_reduction=55426 max_load_reduction_pct=0.845734 moved_to_echo=31683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[416, 139, 3988, 1076, 20, 1, 181, 4027, 959, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 634, 51, 65, 13, 0, 3286, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 2117, 73, 2, 56, 7, 0, 3112, 1126, 689, 0, 0, 0, 0, 0, 0, 70, 154, 0, 3884, 0, 0, 0, 0, 3402, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 4047, 3, 0, 4022, 0, 1131, 53, 24, 0, 0, 0, 0, 0, 525, 701, 25, 0, 3383, 0, 260, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1552, 568, 25, 32, 9, 24, 2928, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 3096, 310, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10807, 9261, 7182, 7510, 9340, 4903, 9018, 7515] avg=8192.0000 max=10807 max_rank=0 min=4903 min_rank=5 imbalance_ratio=1.319214 max_load_reduction=54729 max_load_reduction_pct=0.835098 moved_to_echo=30776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4078, 259, 6, 2, 2, 24, 962, 0, 0, 0, 0, 0, 0, 0, 2532, 8, 1, 0, 498, 0, 66, 3213, 1284, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 6, 10, 0, 0, 1397, 1056, 684, 0, 0, 0, 0, 0, 0, 465, 0, 0, 3647, 0, 1, 0, 25, 3374, 0, 0, 0, 0, 0, 0, 0, 97, 0, 410, 4038, 285, 44, 229, 164, 1140, 50, 24, 0, 0, 0, 0, 0, 493, 4064, 512, 0, 4094, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2345, 0, 0, 100, 433, 5, 2951, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3105, 1728, 2581, 0, 839, 25, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5333, 7602, 7142, 7512, 6481, 13255, 5838, 12373] avg=8192.0000 max=13255 max_rank=5 min=5333 min_rank=0 imbalance_ratio=1.618042 max_load_reduction=52281 max_load_reduction_pct=0.797745 moved_to_echo=37947 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[24448, 6784, 2688, 128, 3200, 128, 27904, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23936, 768, 128, 8576, 4096, 128, 23424, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6784, 1408, 4480, 27904, 128, 128, 128, 384, 24832, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 22016, 3328, 6272, 512, 256, 3456, 15744, 9600, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4480, 10496, 11136, 0, 19712, 128, 19456, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 1792, 512, 30976, 5120, 768, 15872, 256, 9088, 512, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[10624, 6656, 8320, 768, 1280, 128, 256, 26496, 11648, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1792, 256, 31872, 5632, 7552, 384, 512, 10496, 7680, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4096, 408, 8, 2194, 3254, 7, 0, 4064, 0, 3237, 0, 3265, 3387, 0, 6, 2, 0, 16, 0, 0, 313, 1, 10, 0, 2389, 3909, 107, 923, 3926, 4035, 118, 2018, 2043, 0, 0, 0, 0, 4053, 0, 922, 737, 9, 0, 4036, 0, 0, 31, 13, 275, 124, 3, 28, 0, 0, 4, 370, 4079, 1008, 0, 1376, 634, 5, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[3373, 0, 4095, 3, 3837, 1317, 43, 3407, 13, 4095, 14, 3980, 1508, 35, 0, 0, 3, 0, 17, 245, 1569, 0, 0, 0, 3, 3885, 0, 50, 3441, 3491, 2, 0, 1851, 3, 6, 2821, 50, 38, 1238, 0, 4080, 115, 0, 35, 4089, 0, 0, 11, 0, 3633, 0, 0, 0, 1000, 1, 0, 3472, 232, 71, 65, 62, 162, 0, 4075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[3802, 0, 350, 0, 3989, 252, 9, 4080, 0, 4006, 36, 1095, 3937, 13, 0, 0, 3, 1077, 0, 0, 0, 1, 0, 3724, 487, 1445, 0, 0, 3948, 3945, 1812, 43, 2772, 3911, 151, 189, 0, 10, 0, 1, 68, 36, 25, 3623, 6, 0, 0, 0, 37, 4083, 1, 0, 0, 0, 440, 4096, 3375, 0, 71, 1, 445, 68, 0, 4073] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4096, 4014, 0, 1313, 3451, 0, 0, 4081, 0, 2023, 0, 3965, 16, 0, 0, 14, 3699, 3, 0, 2000, 544, 0, 0, 8, 122, 4083, 99, 61, 5, 4096, 82, 3106, 563, 4009, 1522, 54, 0, 87, 4, 0, 4036, 42, 3, 4076, 54, 245, 0, 839, 41, 3561, 205, 0, 0, 0, 2, 56, 4009, 0, 18, 0, 72, 0, 0, 1157] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 334, 32, 0, 632, 0, 4065, 0, 2755, 48, 0, 1, 1, 3966, 70, 0, 15, 1, 14, 0, 2678, 0, 4053, 0, 4, 4090, 141, 305, 0, 4096, 1, 10, 2, 4, 3645, 2, 798, 4063, 654, 339, 327, 0, 542, 1226, 10, 0, 4092, 0, 1, 3, 229, 3819, 1753, 2747, 0, 4074, 4037, 0, 74, 812, 874] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4048, 1, 68, 0, 4083, 6, 0, 3820, 0, 4089, 0, 3991, 615, 0, 41, 2, 2282, 27, 0, 1, 3624, 0, 0, 0, 391, 3004, 0, 2, 25, 248, 213, 11, 0, 59, 405, 2, 1, 3553, 0, 16, 3956, 0, 4005, 4094, 885, 0, 137, 0, 1, 3672, 170, 1593, 0, 1202, 0, 0, 4050, 0, 15, 0, 1516, 1549, 1, 4062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4094, 93, 239, 0, 3657, 197, 96, 3894, 0, 3634, 1, 4095, 0, 425, 39, 0, 4075, 155, 0, 0, 4096, 54, 0, 0, 2776, 3951, 9, 179, 3488, 0, 49, 2225, 3608, 0, 1, 0, 23, 4079, 0, 58, 3638, 0, 0, 0, 0, 7, 0, 0, 5, 3961, 0, 0, 1, 0, 96, 21, 4094, 7, 203, 123, 3, 0, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 3653, 0, 0, 3230, 0, 3901, 4020, 3928, 0, 0, 17, 2, 135, 6, 0, 3002, 49, 560, 52, 1, 3925, 4079, 0, 17, 3707, 82, 0, 238, 84, 224, 1168, 0, 19, 3904, 0, 1, 26, 0, 496, 1056, 587, 13, 0, 2383, 0, 3465, 5, 0, 4, 0, 4095, 330, 4024, 0, 278, 579, 0, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 408, 8, 2194, 3254, 7, 0, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 0, 3265, 3387, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 313, 1, 10, 0, 3545, 0, 0, 0, 0, 0, 0, 0, 2389, 703, 107, 923, 3926, 4035, 118, 2018, 0, 0, 0, 0, 0, 0, 0, 0, 2043, 0, 0, 0, 0, 4053, 0, 922, 1170, 678, 0, 0, 0, 0, 0, 0, 737, 9, 0, 4036, 0, 0, 31, 13, 532, 468, 399, 0, 0, 0, 0, 0, 275, 124, 3, 28, 0, 0, 4, 370, 2036, 551, 0, 0, 0, 0, 0, 0, 2933, 1008, 0, 1376, 634, 5, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9536, 9365, 3885, 14219, 8866, 6225, 3391, 10049] avg=8192.0000 max=14219 max_rank=3 min=3391 min_rank=6 imbalance_ratio=1.735718 max_load_reduction=51317 max_load_reduction_pct=0.783035 moved_to_echo=28531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 350, 0, 3989, 252, 9, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3347, 36, 1095, 3937, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1077, 0, 0, 0, 1, 0, 3724, 3291, 0, 0, 0, 0, 0, 0, 0, 487, 261, 0, 0, 3948, 3945, 1812, 43, 0, 0, 0, 0, 0, 0, 0, 0, 2772, 3911, 151, 189, 0, 10, 0, 1, 432, 561, 0, 0, 0, 0, 0, 0, 68, 36, 25, 3623, 6, 0, 0, 0, 659, 387, 401, 0, 0, 0, 0, 0, 37, 4083, 1, 0, 0, 0, 440, 4096, 752, 511, 0, 0, 0, 0, 0, 0, 2427, 0, 71, 1, 445, 68, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8428, 8096, 10496, 8027, 5205, 9920, 7085] avg=8192.0000 max=10496 max_rank=3 min=5205 min_rank=5 imbalance_ratio=1.281250 max_load_reduction=55040 max_load_reduction_pct=0.839844 moved_to_echo=30237 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4014, 0, 1313, 3451, 0, 0, 3680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1691, 0, 3965, 16, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3699, 3, 0, 2000, 544, 0, 0, 8, 3546, 0, 0, 0, 0, 0, 0, 0, 122, 734, 99, 61, 5, 4096, 82, 3106, 0, 0, 0, 0, 0, 0, 0, 0, 563, 4009, 1522, 54, 0, 87, 4, 0, 1223, 666, 0, 0, 0, 0, 0, 0, 4036, 42, 3, 4076, 54, 245, 0, 839, 332, 459, 401, 0, 0, 0, 0, 0, 41, 3561, 205, 0, 0, 0, 2, 56, 2126, 550, 0, 0, 0, 0, 0, 0, 2884, 0, 18, 0, 72, 0, 0, 1157, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12458, 5686, 9800, 8305, 8128, 10487, 6541, 4131] avg=8192.0000 max=12458 max_rank=0 min=4131 min_rank=7 imbalance_ratio=1.520752 max_load_reduction=53078 max_load_reduction_pct=0.809906 moved_to_echo=29287 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 3, 3837, 1317, 43, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3422, 14, 3980, 1508, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 17, 245, 1569, 0, 0, 0, 2919, 0, 0, 0, 0, 0, 0, 0, 3, 699, 0, 50, 3441, 3491, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1851, 3, 6, 2821, 50, 38, 1238, 0, 1163, 577, 0, 0, 0, 0, 0, 0, 4080, 115, 0, 35, 4089, 0, 0, 11, 673, 398, 335, 0, 0, 0, 0, 0, 0, 3633, 0, 0, 0, 1000, 1, 0, 2023, 454, 0, 0, 0, 0, 0, 0, 2497, 232, 71, 65, 62, 162, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12367, 8972, 4753, 7686, 7747, 9736, 7111, 7164] avg=8192.0000 max=12367 max_rank=0 min=4753 min_rank=2 imbalance_ratio=1.509644 max_load_reduction=53169 max_load_reduction_pct=0.811295 moved_to_echo=31758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 334, 32, 0, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3392, 0, 2755, 48, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 70, 0, 15, 1, 14, 0, 2678, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 719, 0, 4, 4090, 141, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 10, 2, 4, 3645, 2, 798, 1219, 461, 0, 0, 0, 0, 0, 0, 4063, 654, 339, 327, 0, 542, 1226, 10, 673, 319, 66, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 3, 229, 3819, 1753, 2115, 550, 0, 0, 0, 0, 0, 0, 1967, 0, 4074, 4037, 0, 74, 812, 874, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[933, 6197, 10290, 5259, 10238, 8219, 12562, 11838] avg=8192.0000 max=12562 max_rank=6 min=933 min_rank=0 imbalance_ratio=1.533447 max_load_reduction=52974 max_load_reduction_pct=0.808319 moved_to_echo=42857 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 68, 0, 4083, 6, 0, 3445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3417, 0, 3991, 615, 0, 41, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2282, 27, 0, 1, 3624, 0, 0, 0, 3504, 0, 0, 0, 0, 0, 0, 0, 391, 541, 0, 2, 25, 248, 213, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 405, 2, 1, 3553, 0, 16, 899, 673, 0, 0, 0, 0, 0, 0, 3956, 0, 4005, 4094, 885, 0, 137, 0, 672, 464, 375, 0, 0, 0, 0, 0, 1, 3672, 170, 1593, 0, 1202, 0, 0, 1564, 544, 0, 0, 0, 0, 0, 0, 2913, 0, 15, 0, 1516, 1549, 1, 4062, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7603, 8066, 9438, 1431, 5608, 14588, 8746, 10056] avg=8192.0000 max=14588 max_rank=5 min=1431 min_rank=3 imbalance_ratio=1.780762 max_load_reduction=50948 max_load_reduction_pct=0.777405 moved_to_echo=38998 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 93, 239, 0, 3657, 197, 96, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3036, 1, 4095, 0, 425, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 155, 0, 0, 4096, 54, 0, 0, 3543, 0, 0, 0, 0, 0, 0, 0, 2776, 711, 9, 179, 3488, 0, 49, 2225, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 0, 1, 0, 23, 4079, 0, 58, 1183, 680, 0, 0, 0, 0, 0, 0, 3638, 0, 0, 0, 0, 7, 0, 0, 598, 469, 383, 0, 0, 0, 0, 0, 5, 3961, 0, 0, 1, 0, 96, 21, 2057, 551, 0, 0, 0, 0, 0, 0, 2945, 7, 203, 123, 3, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7793, 7596, 11923, 9437, 9632, 5095, 6692, 7368] avg=8192.0000 max=11923 max_rank=2 min=5095 min_rank=5 imbalance_ratio=1.455444 max_load_reduction=53613 max_load_reduction_pct=0.818069 moved_to_echo=28787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3653, 0, 0, 2913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3259, 4020, 3928, 0, 0, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 135, 6, 0, 3002, 49, 560, 52, 1, 3546, 0, 0, 0, 0, 0, 0, 0, 3925, 734, 0, 17, 3707, 82, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 84, 224, 1168, 0, 19, 3904, 0, 1, 1221, 669, 0, 0, 0, 0, 0, 0, 26, 0, 496, 1056, 587, 13, 0, 2383, 642, 461, 317, 0, 0, 0, 0, 0, 0, 3465, 5, 0, 4, 0, 4095, 330, 2124, 550, 0, 0, 0, 0, 0, 0, 2894, 0, 278, 579, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6567, 11226, 7351, 8703, 7290, 5981, 10573, 7845] avg=8192.0000 max=11226 max_rank=1 min=5981 min_rank=5 imbalance_ratio=1.370361 max_load_reduction=54310 max_load_reduction_pct=0.828705 moved_to_echo=31689 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 4608, 4864, 3584, 26368, 1920, 256, 24576, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[10112, 5120, 256, 1280, 22656, 16128, 2688, 7680, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 24320, 4096, 27136, 9600, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[14208, 1408, 128, 5376, 10240, 640, 128, 6528, 27520, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[21504, 1280, 4736, 6272, 2816, 1920, 896, 26624, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 26624, 384, 1664, 128, 2432, 8576, 6656, 14848, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[15104, 8320, 3328, 3072, 128, 19456, 1280, 1920, 8576, 4992, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[20608, 896, 4992, 17280, 5632, 896, 1408, 3328, 4864, 3456, 2688, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[3949, 16, 0, 388, 7, 1, 0, 187, 3890, 5, 4096, 8, 0, 9, 10, 0, 0, 3829, 3329, 1840, 42, 3965, 15, 3, 3948, 82, 0, 0, 0, 0, 4073, 4094, 0, 2, 1002, 438, 110, 253, 0, 147, 0, 2224, 0, 0, 79, 375, 1444, 0, 4089, 4096, 3437, 2252, 1024, 56, 0, 0, 0, 8, 0, 0, 2742, 0, 3914, 58] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4039, 0, 13, 0, 5, 4092, 0, 5, 25, 0, 4031, 76, 0, 8, 0, 150, 0, 1284, 4073, 100, 0, 2118, 4060, 3980, 3117, 2952, 0, 0, 0, 0, 4, 4095, 1918, 4, 43, 104, 4093, 0, 37, 5, 0, 0, 144, 2874, 0, 6, 60, 0, 97, 0, 4089, 2, 5, 4093, 8, 0, 0, 4094, 0, 0, 3231, 4, 2337, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[69, 0, 1, 0, 9, 3073, 0, 7, 565, 2500, 2722, 0, 0, 2512, 1, 0, 0, 2432, 3832, 57, 340, 4096, 4096, 3710, 383, 4092, 0, 476, 0, 0, 4096, 4057, 1269, 0, 75, 0, 0, 0, 0, 0, 0, 3791, 2, 0, 0, 4, 3396, 0, 4089, 35, 4096, 71, 3839, 1260, 0, 20, 108, 0, 0, 0, 0, 0, 298, 57] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[259, 0, 946, 1, 0, 2760, 39, 3241, 0, 72, 4004, 0, 64, 4091, 53, 96, 1, 4070, 4096, 0, 239, 3934, 1161, 1371, 4010, 1676, 2, 0, 201, 0, 4094, 2384, 3254, 0, 3292, 0, 22, 0, 0, 2, 9, 0, 4087, 0, 163, 11, 2910, 0, 1748, 0, 3617, 197, 59, 2, 1, 0, 0, 1658, 840, 0, 272, 0, 3, 524] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2036, 57, 864, 4, 259, 0, 7, 0, 3916, 0, 2198, 2, 1, 16, 0, 0, 4, 0, 3779, 8, 1763, 4095, 3997, 4039, 8, 3504, 0, 530, 15, 0, 4047, 3813, 83, 45, 1, 9, 0, 117, 0, 42, 0, 667, 0, 23, 0, 684, 2443, 1, 1437, 0, 4096, 4009, 4079, 350, 0, 0, 177, 2924, 1353, 2, 0, 0, 4031, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[2224, 0, 10, 0, 3323, 11, 0, 51, 270, 297, 4034, 1, 0, 3, 0, 27, 5, 4080, 112, 3693, 1539, 0, 3953, 4096, 944, 3, 0, 0, 0, 86, 4094, 4090, 0, 0, 4083, 3279, 0, 0, 0, 0, 0, 0, 229, 571, 0, 0, 4033, 0, 1028, 3243, 4069, 50, 1, 481, 0, 0, 3112, 3, 206, 0, 4005, 1, 196, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[3552, 19, 8, 0, 0, 1, 0, 8, 947, 3504, 3853, 3, 1, 3508, 0, 9, 130, 4096, 4052, 0, 0, 0, 753, 4088, 3959, 4061, 210, 0, 0, 0, 4070, 3596, 413, 35, 370, 0, 0, 238, 0, 0, 17, 0, 47, 873, 0, 0, 19, 0, 3878, 0, 4065, 4044, 4047, 0, 37, 0, 1413, 0, 1061, 0, 421, 0, 0, 130] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[3693, 0, 47, 0, 302, 1, 0, 1847, 3485, 3348, 2468, 80, 0, 2531, 0, 2, 130, 4023, 4043, 0, 0, 3865, 3870, 4096, 0, 2961, 0, 0, 0, 0, 4095, 3321, 0, 0, 1361, 3, 298, 0, 0, 2317, 0, 0, 3, 3950, 0, 0, 0, 0, 1, 1, 3001, 242, 4096, 1, 113, 0, 0, 2, 79, 0, 1026, 2, 832, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4039, 0, 13, 0, 5, 4092, 0, 5, 1844, 558, 0, 0, 0, 0, 0, 0, 25, 0, 4031, 76, 0, 8, 0, 150, 96, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 100, 0, 2118, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3117, 2952, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1918, 4, 43, 104, 4093, 0, 37, 5, 4095, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 144, 2874, 0, 6, 60, 0, 2982, 1631, 0, 0, 0, 0, 0, 0, 97, 0, 988, 2, 5, 4093, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3231, 4, 2337, 61, 2349, 1257, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10556, 4386, 6908, 6073, 11390, 7697, 5193, 13333] avg=8192.0000 max=13333 max_rank=7 min=4386 min_rank=1 imbalance_ratio=1.627563 max_load_reduction=52203 max_load_reduction_pct=0.796555 moved_to_echo=37613 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3949, 16, 0, 388, 7, 1, 0, 187, 1550, 1665, 0, 0, 0, 0, 0, 0, 3890, 5, 4096, 8, 0, 9, 10, 0, 287, 36, 0, 0, 0, 0, 0, 0, 0, 1877, 0, 1840, 42, 3965, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 82, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1002, 438, 110, 253, 0, 147, 4094, 893, 0, 0, 0, 0, 0, 0, 0, 2224, 0, 0, 79, 375, 1444, 0, 2436, 2, 0, 0, 0, 0, 0, 0, 4089, 4096, 831, 2252, 1024, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2742, 0, 3914, 58, 1, 1056, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7763, 8341, 7739, 8067, 6939, 6560, 12348, 7779] avg=8192.0000 max=12348 max_rank=6 min=6560 min_rank=5 imbalance_ratio=1.507324 max_load_reduction=53188 max_load_reduction_pct=0.811584 moved_to_echo=33626 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[259, 0, 946, 1, 0, 2760, 39, 3241, 1631, 1770, 0, 0, 0, 0, 0, 0, 0, 72, 4004, 0, 64, 4091, 53, 96, 305, 36, 0, 0, 0, 0, 0, 0, 1, 1995, 0, 0, 239, 3934, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 1676, 2, 0, 201, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3254, 0, 3292, 0, 22, 0, 0, 2, 2384, 1098, 0, 0, 0, 0, 0, 0, 9, 0, 4087, 0, 163, 11, 2910, 0, 2998, 562, 0, 0, 0, 0, 0, 0, 1748, 0, 874, 197, 59, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1658, 840, 0, 272, 0, 3, 524, 809, 1112, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10647, 8721, 7330, 9947, 10052, 10740, 2881, 5218] avg=8192.0000 max=10740 max_rank=5 min=2881 min_rank=6 imbalance_ratio=1.311035 max_load_reduction=54796 max_load_reduction_pct=0.836121 moved_to_echo=28891 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2036, 57, 864, 4, 259, 0, 7, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 2198, 2, 1, 16, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8, 1763, 4095, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3504, 0, 530, 15, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 45, 1, 9, 0, 117, 0, 42, 3813, 1012, 0, 0, 0, 0, 0, 0, 0, 667, 0, 23, 0, 684, 2443, 1, 2767, 1655, 0, 0, 0, 0, 0, 0, 1437, 0, 990, 4009, 4079, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 2924, 1353, 2, 0, 0, 4031, 1, 2384, 1259, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5074, 6169, 9867, 8068, 5122, 8240, 10865, 12131] avg=8192.0000 max=12131 max_rank=7 min=5074 min_rank=0 imbalance_ratio=1.480835 max_load_reduction=53405 max_load_reduction_pct=0.814896 moved_to_echo=36358 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2224, 0, 10, 0, 3323, 11, 0, 51, 1840, 1778, 0, 0, 0, 0, 0, 0, 270, 297, 4034, 1, 0, 3, 0, 27, 309, 41, 0, 0, 0, 0, 0, 0, 5, 1993, 0, 3693, 1539, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 3, 0, 0, 0, 86, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3279, 0, 0, 0, 0, 4090, 31, 0, 0, 0, 0, 0, 0, 0, 0, 229, 571, 0, 0, 4033, 0, 81, 1679, 0, 0, 0, 0, 0, 0, 1028, 3243, 974, 50, 1, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3112, 3, 206, 0, 4005, 1, 196, 0, 2417, 1255, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9237, 4982, 11183, 5086, 11483, 6593, 5777, 11195] avg=8192.0000 max=11483 max_rank=4 min=4982 min_rank=1 imbalance_ratio=1.401733 max_load_reduction=54053 max_load_reduction_pct=0.824783 moved_to_echo=35048 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[69, 0, 1, 0, 9, 3073, 0, 7, 1847, 1058, 0, 0, 0, 0, 0, 0, 565, 2500, 2722, 0, 0, 2512, 1, 0, 182, 36, 0, 0, 0, 0, 0, 0, 0, 1192, 0, 57, 340, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 4092, 0, 476, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 0, 75, 0, 0, 0, 0, 0, 4057, 1027, 0, 0, 0, 0, 0, 0, 0, 3791, 2, 0, 0, 4, 3396, 0, 2805, 1520, 0, 0, 0, 0, 0, 0, 4089, 35, 990, 71, 3839, 1260, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 298, 57, 2190, 1259, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6064, 8518, 9781, 9011, 6428, 11518, 10304, 3912] avg=8192.0000 max=11518 max_rank=5 min=3912 min_rank=7 imbalance_ratio=1.406006 max_load_reduction=54018 max_load_reduction_pct=0.824249 moved_to_echo=32162 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3552, 19, 8, 0, 0, 1, 0, 8, 1833, 1781, 0, 0, 0, 0, 0, 0, 947, 3504, 3853, 3, 1, 3508, 0, 9, 307, 36, 0, 0, 0, 0, 0, 0, 130, 2008, 0, 0, 0, 0, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 4061, 210, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 35, 370, 0, 0, 238, 0, 0, 3596, 1086, 0, 0, 0, 0, 0, 0, 17, 0, 47, 873, 0, 0, 19, 0, 2966, 1675, 0, 0, 0, 0, 0, 0, 3878, 0, 982, 4044, 4047, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1413, 0, 1061, 0, 421, 0, 0, 130, 2413, 1250, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7202, 12168, 2891, 12264, 5738, 5597, 12988, 6688] avg=8192.0000 max=12988 max_rank=6 min=2891 min_rank=2 imbalance_ratio=1.585449 max_load_reduction=52548 max_load_reduction_pct=0.801819 moved_to_echo=31011 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3693, 0, 47, 0, 302, 1, 0, 1847, 1353, 1750, 0, 0, 0, 0, 0, 0, 3485, 3348, 2468, 80, 0, 2531, 0, 2, 301, 36, 0, 0, 0, 0, 0, 0, 130, 1972, 0, 0, 0, 3865, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2961, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1361, 3, 298, 0, 0, 2317, 3321, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3950, 0, 0, 0, 0, 2959, 1679, 0, 0, 0, 0, 0, 0, 1, 1, 726, 242, 4096, 1, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 79, 0, 1026, 2, 832, 0, 2417, 922, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8993, 12251, 9837, 7020, 8384, 8591, 5180, 5280] avg=8192.0000 max=12251 max_rank=1 min=5180 min_rank=6 imbalance_ratio=1.495483 max_load_reduction=53285 max_load_reduction_pct=0.813065 moved_to_echo=27435 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4864, 8704, 3584, 128, 11776, 128, 11648, 896, 15104, 9472, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 11776, 0, 5760, 3968, 22144, 22016, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 6784, 4608, 8320, 256, 1152, 14336, 128, 20096, 10496, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[16384, 19456, 256, 1024, 256, 128, 28416, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[19840, 128, 1920, 512, 3968, 9984, 128, 5376, 13824, 10368, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[16384, 7424, 7424, 10880, 17152, 6272, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7040, 128, 10240, 3840, 4608, 640, 128, 2560, 29568, 7424, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[13184, 9728, 27520, 256, 128, 12800, 128, 384, 1792, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 27, 20, 219, 0, 20, 2161, 0, 14, 4096, 3617, 6, 4019, 164, 0, 0, 3634, 106, 0, 4095, 0, 87, 0, 0, 0, 45, 153, 2, 3981, 0, 0, 3375, 0, 4028, 53, 12, 0, 4093, 0, 120, 913, 6, 1541, 182, 468, 0, 0, 33, 3805, 0, 0, 1, 4096, 127, 102, 3999, 91, 482, 71, 4080, 258, 3141, 3992, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3308, 0, 2, 0, 2, 2282, 3, 850, 4089, 3539, 4060, 823, 0, 35, 4096, 3615, 10, 0, 373, 0, 3230, 0, 0, 0, 878, 0, 69, 3554, 12, 54, 3721, 0, 45, 0, 0, 0, 0, 347, 0, 0, 1577, 0, 0, 80, 1, 0, 4004, 3881, 0, 23, 118, 12, 0, 4088, 4074, 4024, 0, 0, 696, 3961, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4007, 0, 0, 391, 0, 2, 0, 2, 3683, 3992, 4096, 1805, 1814, 0, 74, 0, 0, 3938, 95, 4096, 0, 6, 4, 1, 0, 0, 4091, 0, 3765, 306, 57, 382, 0, 0, 0, 0, 4006, 103, 0, 1810, 4096, 0, 4096, 33, 1, 2448, 4096, 0, 0, 124, 4, 144, 724, 3294, 2616, 0, 12, 1319, 0, 2, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4049, 0, 3, 303, 0, 8, 2975, 2365, 4096, 3356, 4080, 2399, 1152, 26, 3946, 0, 0, 4096, 3, 0, 0, 5, 0, 67, 0, 91, 4094, 0, 60, 0, 0, 3827, 4041, 4, 2, 0, 4093, 2, 0, 2759, 0, 0, 3909, 214, 0, 0, 1686, 302, 0, 1028, 3, 0, 1423, 6, 0, 0, 0, 4053, 1, 0, 1005, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3237, 0, 4, 22, 0, 0, 1301, 3308, 3974, 0, 379, 4025, 0, 0, 4096, 137, 108, 4094, 3885, 1750, 709, 0, 0, 1, 0, 158, 4066, 0, 487, 7, 0, 2225, 3, 0, 0, 0, 3903, 0, 0, 805, 3585, 2, 27, 16, 52, 0, 2056, 0, 0, 551, 4093, 187, 2272, 4095, 13, 52, 0, 35, 1544, 175, 4085, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 1443, 0, 0, 0, 1654, 31, 4001, 3286, 3380, 0, 3702, 0, 5, 1, 2884, 0, 11, 4086, 63, 1135, 0, 115, 0, 408, 113, 3183, 331, 0, 0, 666, 759, 3915, 0, 23, 0, 0, 4075, 0, 1, 27, 1, 134, 4084, 3822, 0, 0, 895, 4096, 7, 7, 1, 2, 0, 4090, 490, 0, 1, 4093, 0, 506, 4009, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 175, 3393, 0, 0, 13, 1392, 0, 0, 0, 3928, 1100, 4068, 2724, 0, 355, 0, 19, 0, 3718, 102, 3115, 1, 0, 52, 727, 8, 0, 4010, 4007, 476, 0, 3320, 379, 4001, 0, 138, 0, 1349, 18, 23, 0, 783, 0, 4093, 1, 2, 0, 3695, 4094, 42, 0, 7, 0, 4067, 4093, 1471, 0, 88, 360, 0, 0, 28, 101] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 863, 1, 3127, 0, 25, 58, 3942, 3904, 4085, 1229, 3217, 32, 0, 4, 4086, 372, 163, 3979, 1047, 0, 0, 0, 76, 0, 9, 0, 3681, 0, 415, 2, 3269, 4033, 4001, 0, 1, 1, 0, 0, 0, 3368, 3643, 0, 3431, 0, 124, 0, 1698, 0, 0, 24, 445, 6, 0, 65, 0, 118, 0, 0, 3, 1170, 4082, 1736] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 20, 219, 0, 20, 2161, 0, 3617, 355, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 6, 1006, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 106, 0, 2687, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 153, 2, 3981, 0, 0, 3375, 2626, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 53, 12, 0, 4093, 0, 120, 173, 0, 0, 0, 0, 0, 0, 0, 913, 6, 1541, 182, 468, 0, 0, 33, 2046, 142, 32, 0, 0, 0, 0, 0, 3805, 0, 0, 1, 4096, 127, 102, 1953, 0, 0, 0, 0, 0, 0, 0, 0, 91, 482, 71, 4080, 258, 3141, 3992, 1, 1093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6419, 5286, 6514, 10182, 8479, 5363, 10084, 13209] avg=8192.0000 max=13209 max_rank=7 min=5286 min_rank=1 imbalance_ratio=1.612427 max_load_reduction=52327 max_load_reduction_pct=0.798447 moved_to_echo=37135 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4007, 0, 0, 391, 0, 2, 3683, 361, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3992, 1026, 1805, 1814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 2584, 95, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 4091, 0, 3765, 306, 2676, 0, 0, 0, 0, 0, 0, 0, 57, 382, 0, 0, 0, 0, 4006, 103, 166, 0, 0, 0, 0, 0, 0, 0, 0, 1810, 4096, 0, 4096, 33, 1, 2448, 1685, 137, 33, 0, 0, 0, 0, 0, 4096, 0, 0, 124, 4, 144, 724, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 2616, 0, 12, 1319, 0, 2, 1, 0, 1051, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8444, 8639, 6855, 10843, 4714, 14339, 6701, 5001] avg=8192.0000 max=14339 max_rank=5 min=4714 min_rank=4 imbalance_ratio=1.750366 max_load_reduction=51197 max_load_reduction_pct=0.781204 moved_to_echo=30755 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 175, 3393, 0, 0, 13, 1392, 0, 3928, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 1018, 2724, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2439, 102, 3115, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 727, 8, 0, 4010, 4007, 476, 0, 2658, 0, 0, 0, 0, 0, 0, 0, 3320, 379, 4001, 0, 138, 0, 1349, 18, 157, 0, 0, 0, 0, 0, 0, 0, 23, 0, 783, 0, 4093, 1, 2, 0, 2094, 129, 33, 0, 0, 0, 0, 0, 3695, 4094, 42, 0, 7, 0, 4067, 1999, 0, 0, 0, 0, 0, 0, 0, 0, 1471, 0, 88, 360, 0, 0, 28, 101, 993, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9260, 5197, 5676, 11938, 9362, 7158, 13904, 3041] avg=8192.0000 max=13904 max_rank=6 min=3041 min_rank=7 imbalance_ratio=1.697266 max_load_reduction=51632 max_load_reduction_pct=0.787842 moved_to_echo=33465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3308, 0, 2, 0, 2, 2282, 4089, 362, 0, 0, 0, 0, 0, 0, 3, 850, 0, 3539, 1005, 823, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3615, 10, 0, 373, 0, 3230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 0, 69, 3554, 12, 54, 2657, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 45, 0, 0, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 0, 0, 80, 1, 0, 2094, 0, 36, 0, 0, 0, 0, 0, 4004, 3881, 0, 23, 118, 12, 0, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4024, 0, 0, 696, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10045, 6255, 11324, 7224, 4113, 3788, 10032, 12755] avg=8192.0000 max=12755 max_rank=7 min=3788 min_rank=5 imbalance_ratio=1.557007 max_load_reduction=52781 max_load_reduction_pct=0.805374 moved_to_echo=30688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1443, 0, 0, 0, 1654, 31, 3380, 327, 0, 0, 0, 0, 0, 0, 4001, 3286, 0, 0, 926, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2884, 0, 11, 2680, 63, 1135, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 113, 3183, 331, 0, 0, 666, 2419, 0, 0, 0, 0, 0, 0, 0, 759, 3915, 0, 23, 0, 0, 4075, 0, 173, 0, 0, 0, 0, 0, 0, 0, 1, 27, 1, 134, 4084, 3822, 0, 0, 2092, 142, 30, 0, 0, 0, 0, 0, 895, 4096, 7, 7, 1, 2, 0, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 1, 4093, 0, 506, 4009, 0, 1091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6835, 8219, 6888, 7120, 8945, 10333, 7006, 10190] avg=8192.0000 max=10333 max_rank=5 min=6835 min_rank=0 imbalance_ratio=1.261353 max_load_reduction=55203 max_load_reduction_pct=0.842331 moved_to_echo=36474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4049, 0, 3, 303, 0, 8, 4096, 360, 0, 0, 0, 0, 0, 0, 2975, 2365, 0, 3356, 1021, 2399, 1152, 26, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 2688, 3, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 91, 4094, 0, 60, 0, 2666, 0, 0, 0, 0, 0, 0, 0, 0, 3827, 4041, 4, 2, 0, 4093, 2, 173, 0, 0, 0, 0, 0, 0, 0, 0, 2759, 0, 0, 3909, 214, 0, 0, 3, 142, 33, 0, 0, 0, 0, 0, 1686, 302, 0, 1028, 3, 0, 1423, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 1, 0, 1005, 4, 1093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8819, 13294, 6642, 6978, 12142, 7060, 4445, 6156] avg=8192.0000 max=13294 max_rank=1 min=4445 min_rank=6 imbalance_ratio=1.622803 max_load_reduction=52242 max_load_reduction_pct=0.797150 moved_to_echo=29803 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 863, 1, 3127, 0, 25, 58, 4085, 284, 0, 0, 0, 0, 0, 0, 3942, 3904, 0, 1229, 805, 32, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 372, 163, 2598, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 9, 0, 3681, 0, 415, 2, 2102, 0, 0, 0, 0, 0, 0, 0, 3269, 4033, 4001, 0, 1, 1, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 3368, 3643, 0, 3431, 0, 124, 0, 33, 143, 26, 0, 0, 0, 0, 0, 1698, 0, 0, 24, 445, 6, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 3, 1170, 4082, 1736, 1066, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8444, 9916, 8266, 6285, 11477, 10768, 2205, 8175] avg=8192.0000 max=11477 max_rank=4 min=2205 min_rank=6 imbalance_ratio=1.401001 max_load_reduction=54059 max_load_reduction_pct=0.824875 moved_to_echo=32625 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3237, 0, 4, 22, 0, 0, 3974, 33, 0, 0, 0, 0, 0, 0, 1301, 3308, 0, 0, 96, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 137, 108, 2686, 3885, 1750, 709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 158, 4066, 0, 487, 7, 247, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 3, 0, 0, 0, 3903, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 805, 3585, 2, 27, 16, 52, 0, 2095, 142, 3, 0, 0, 0, 0, 0, 2056, 0, 0, 551, 4093, 187, 2272, 2000, 0, 0, 0, 0, 0, 0, 0, 0, 13, 52, 0, 35, 1544, 175, 4085, 12, 1093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7270, 8730, 13371, 4966, 6304, 6727, 11159, 7009] avg=8192.0000 max=13371 max_rank=2 min=4966 min_rank=3 imbalance_ratio=1.632202 max_load_reduction=52165 max_load_reduction_pct=0.795975 moved_to_echo=31199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8832, 4736, 256, 13952, 2560, 8960, 17280, 1920, 7552, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[22016, 12416, 128, 1792, 8832, 512, 8704, 11648, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 8832, 15232, 384, 20224, 4224, 256, 2560, 12160, 1024, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[22912, 4352, 384, 18432, 5632, 10240, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11136, 18816, 12160, 128, 256, 4096, 17536, 640, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 256, 20352, 256, 3200, 768, 5248, 2432, 30976, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12288, 17920, 0, 13312, 6912, 12032, 3072, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 1280, 1280, 3456, 24448, 7680, 5248, 4480, 18176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4087, 0, 3, 4070, 0, 1235, 8, 4086, 206, 3381, 0, 0, 0, 0, 0, 3977, 4026, 3, 2478, 2571, 37, 0, 70, 0, 38, 369, 0, 0, 0, 0, 3766, 47, 0, 163, 0, 4089, 945, 0, 4, 716, 5, 0, 0, 3889, 15, 3559, 7, 0, 4039, 2729, 4, 11, 2, 1023, 3988, 5, 1637, 2, 4065, 176] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[111, 3906, 47, 7, 738, 0, 6, 0, 0, 208, 3061, 0, 3515, 155, 0, 1594, 0, 18, 1, 4096, 3161, 89, 0, 4028, 0, 3366, 171, 0, 1468, 0, 20, 0, 0, 1, 3972, 3719, 0, 4093, 0, 0, 0, 18, 3940, 981, 0, 0, 57, 1296, 0, 0, 3793, 1, 3952, 1401, 10, 46, 95, 3, 4095, 5, 4096, 0, 196, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[2360, 1, 4088, 659, 3263, 0, 4042, 3138, 258, 89, 1491, 0, 4038, 3917, 2288, 16, 46, 3132, 1, 261, 0, 0, 2658, 213, 698, 32, 1, 17, 0, 1, 2, 3971, 3, 17, 62, 4085, 10, 0, 0, 62, 1, 0, 8, 163, 0, 0, 3044, 0, 134, 82, 4080, 0, 1480, 3409, 0, 11, 40, 0, 4055, 0, 4094, 14, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 2251, 1430, 0, 1707, 4046, 3630, 545, 0, 1349, 0, 3819, 0, 3920, 0, 3086, 28, 3933, 3948, 0, 879, 3649, 0, 0, 0, 0, 0, 800, 0, 4096, 0, 465, 4075, 1, 45, 3676, 0, 302, 0, 25, 2, 484, 0, 284, 0, 872, 0, 0, 2, 0, 4070, 15, 0, 95, 5, 14, 4094, 14, 3880, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3994, 0, 733, 0, 0, 3877, 68, 1136, 88, 0, 0, 1254, 2091, 3374, 0, 125, 2, 34, 460, 4038, 3452, 0, 3, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3510, 4003, 14, 2303, 98, 0, 69, 0, 0, 3961, 138, 1, 0, 0, 2057, 1, 3971, 2, 0, 4075, 4096, 0, 0, 4, 0, 4083, 220, 4092, 3, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4078, 291, 290, 0, 3894, 1102, 1124, 150, 4096, 252, 28, 932, 0, 2417, 0, 3, 4, 4, 0, 4096, 4092, 0, 72, 3830, 0, 20, 0, 1, 0, 0, 1, 643, 0, 0, 3847, 0, 3969, 1631, 0, 1443, 0, 1, 2780, 708, 171, 44, 0, 0, 0, 3371, 5, 4, 4035, 1, 4, 136, 0, 0, 4096, 1, 4096, 0, 3773, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[3889, 0, 0, 2228, 2378, 279, 0, 4026, 3623, 1, 238, 80, 524, 3952, 0, 0, 0, 0, 0, 3950, 3889, 0, 2665, 4095, 1873, 0, 12, 0, 0, 32, 7, 4061, 8, 4042, 3458, 1, 154, 0, 0, 46, 0, 1, 43, 0, 0, 15, 0, 0, 0, 3434, 58, 21, 4091, 165, 0, 0, 0, 0, 4089, 14, 4093, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1328, 1293, 3956, 423, 4038, 0, 4096, 4090, 0, 0, 0, 4080, 0, 21, 0, 3, 7, 549, 0, 4078, 0, 186, 2100, 1398, 21, 0, 0, 604, 0, 141, 0, 4096, 0, 55, 318, 4096, 9, 4075, 0, 4096, 8, 7, 0, 3928, 0, 0, 9, 476, 0, 0, 6, 0, 849, 3508, 0, 5, 396, 3083, 4095, 1, 8, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[111, 3906, 47, 7, 495, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 3061, 0, 3515, 123, 0, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 84, 3161, 89, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3366, 171, 0, 1468, 0, 20, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2559, 3719, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3940, 981, 0, 0, 57, 1296, 0, 1413, 243, 781, 27, 0, 0, 0, 0, 0, 3793, 1, 3952, 1401, 10, 46, 5, 0, 0, 0, 0, 0, 0, 0, 95, 3, 3314, 5, 4096, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4572, 8501, 7381, 9037, 10372, 8756, 9208, 7709] avg=8192.0000 max=10372 max_rank=4 min=4572 min_rank=0 imbalance_ratio=1.266113 max_load_reduction=55164 max_load_reduction_pct=0.841736 moved_to_echo=36045 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4078, 291, 290, 0, 2611, 1102, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 252, 28, 932, 0, 1906, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 84, 4092, 0, 72, 3830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 1, 0, 0, 1, 643, 4012, 80, 0, 0, 0, 0, 0, 0, 0, 0, 2479, 0, 3969, 1631, 0, 1443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2780, 708, 171, 44, 0, 0, 70, 1368, 1283, 781, 425, 0, 0, 0, 0, 3371, 5, 4, 4035, 1, 4, 136, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3315, 1, 4096, 0, 3773, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9496, 7217, 8086, 4757, 9522, 7631, 7642, 11185] avg=8192.0000 max=11185 max_rank=7 min=4757 min_rank=3 imbalance_ratio=1.365356 max_load_reduction=54351 max_load_reduction_pct=0.829330 moved_to_echo=35980 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2360, 1, 4088, 659, 2188, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 89, 1491, 0, 4038, 3088, 2288, 16, 0, 0, 0, 0, 0, 0, 0, 0, 46, 3132, 1, 6, 0, 0, 2658, 213, 0, 0, 0, 0, 0, 0, 0, 0, 698, 32, 1, 17, 0, 1, 2, 3971, 255, 1674, 0, 0, 0, 0, 0, 0, 3, 17, 40, 4085, 10, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 163, 0, 0, 3044, 0, 1464, 22, 1075, 773, 689, 0, 0, 0, 134, 82, 4080, 0, 1480, 3409, 0, 11, 140, 0, 0, 0, 0, 0, 0, 0, 40, 0, 3282, 0, 4094, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13338, 11268, 6056, 6651, 4217, 7239, 9336, 7431] avg=8192.0000 max=13338 max_rank=0 min=4217 min_rank=4 imbalance_ratio=1.628174 max_load_reduction=52198 max_load_reduction_pct=0.796478 moved_to_echo=28223 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1328, 1293, 3956, 423, 2705, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 11, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7, 549, 0, 81, 0, 186, 2100, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 604, 0, 141, 0, 4096, 3997, 2183, 0, 0, 0, 0, 0, 0, 0, 55, 201, 4096, 9, 4075, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7, 0, 3928, 0, 0, 9, 476, 1907, 117, 1333, 784, 6, 0, 0, 0, 0, 0, 6, 0, 849, 3508, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 396, 3083, 3311, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13801, 4094, 4321, 11042, 12532, 8575, 4372, 6799] avg=8192.0000 max=13801 max_rank=0 min=4094 min_rank=1 imbalance_ratio=1.684692 max_load_reduction=51735 max_load_reduction_pct=0.789413 moved_to_echo=32278 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 2741, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1235, 8, 4086, 206, 2665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 4026, 3, 2478, 2571, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 70, 0, 38, 369, 0, 0, 3895, 2171, 0, 0, 0, 0, 0, 0, 0, 0, 2427, 47, 0, 163, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 945, 0, 4, 716, 5, 0, 0, 3889, 1899, 1339, 1346, 761, 595, 0, 0, 0, 15, 3559, 7, 0, 4039, 2729, 4, 11, 121, 0, 0, 0, 0, 0, 0, 0, 2, 1023, 3227, 5, 1637, 2, 4065, 176, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2749, 8200, 9160, 6580, 6726, 11499, 10485, 10137] avg=8192.0000 max=11499 max_rank=5 min=2749 min_rank=0 imbalance_ratio=1.403687 max_load_reduction=54037 max_load_reduction_pct=0.824539 moved_to_echo=38847 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3994, 0, 733, 0, 0, 3877, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 1254, 2091, 2661, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 2, 34, 460, 83, 3452, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3955, 606, 0, 0, 0, 0, 0, 0, 0, 3510, 2579, 14, 2303, 98, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 138, 1, 0, 0, 2057, 530, 1424, 0, 779, 593, 0, 0, 0, 1, 3971, 2, 0, 4075, 4096, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3304, 220, 4092, 3, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8672, 6219, 8130, 4562, 8573, 9483, 12265, 7632] avg=8192.0000 max=12265 max_rank=6 min=4562 min_rank=3 imbalance_ratio=1.497192 max_load_reduction=53271 max_load_reduction_pct=0.812851 moved_to_echo=37953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2251, 959, 0, 1707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3630, 545, 0, 1349, 0, 3011, 0, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3086, 28, 81, 3948, 0, 879, 3649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 800, 0, 4096, 3852, 2159, 0, 0, 0, 0, 0, 0, 0, 465, 2626, 1, 45, 3676, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2, 484, 0, 284, 0, 872, 1887, 1449, 471, 781, 672, 0, 0, 0, 0, 0, 2, 0, 4070, 15, 0, 95, 136, 0, 0, 0, 0, 0, 0, 0, 5, 14, 3313, 14, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4917, 12455, 11671, 10907, 7115, 6927, 4318, 7226] avg=8192.0000 max=12455 max_rank=1 min=4318 min_rank=6 imbalance_ratio=1.520386 max_load_reduction=53081 max_load_reduction_pct=0.809952 moved_to_echo=25586 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3889, 0, 0, 2228, 1595, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3623, 1, 238, 80, 524, 3116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 3889, 0, 2665, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1873, 0, 12, 0, 0, 32, 7, 4061, 3868, 2147, 0, 0, 0, 0, 0, 0, 8, 4042, 2228, 1, 154, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 43, 0, 0, 15, 0, 0, 1879, 1230, 783, 780, 695, 0, 0, 0, 0, 3434, 58, 21, 4091, 165, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3309, 14, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7991, 7582, 10731, 12000, 6479, 5426, 7910, 7417] avg=8192.0000 max=12000 max_rank=3 min=5426 min_rank=5 imbalance_ratio=1.464844 max_load_reduction=53536 max_load_reduction_pct=0.816895 moved_to_echo=27232 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[15872, 5504, 9216, 5632, 13312, 5376, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[1024, 128, 10752, 7168, 256, 384, 3200, 8704, 9728, 8448, 6656, 6272, 3712, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2688, 3456, 256, 640, 1536, 1408, 128, 16896, 27904, 11136, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 6912, 512, 640, 22656, 384, 10880, 23936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 4224, 26496, 384, 26112, 128, 8064, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 8192, 15232, 12032, 6528, 13824, 0, 10112, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[11776, 2432, 4864, 11904, 10496, 16640, 2304, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 14464, 8064, 128, 26624, 15360, 128, 384, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 2574, 1371, 0, 695, 0, 0, 12, 460, 0, 2708, 0, 0, 484, 0, 0, 21, 0, 0, 65, 82, 0, 3722, 0, 4090, 4023, 2066, 653, 200, 0, 2585, 8, 528, 2902, 0, 1376, 0, 3980, 4029, 0, 4096, 613, 9, 1063, 0, 0, 2363, 90, 185, 84, 0, 3970, 30, 21, 3502, 0, 1, 8, 0, 2101, 624, 4046, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[1883, 665, 353, 0, 1, 255, 1, 0, 0, 3923, 3280, 15, 0, 1264, 0, 0, 759, 0, 15, 0, 80, 0, 4039, 3959, 1, 1756, 4055, 0, 287, 3306, 2209, 7, 0, 4088, 1696, 720, 0, 370, 3877, 1, 0, 4096, 31, 0, 0, 0, 4095, 0, 4, 0, 53, 233, 3938, 3075, 43, 581, 0, 0, 0, 0, 3549, 199, 0, 2774] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[2705, 0, 3842, 21, 126, 0, 273, 0, 0, 0, 0, 4055, 0, 743, 0, 6, 0, 0, 0, 3518, 0, 0, 0, 4095, 0, 3766, 244, 1, 0, 6, 3425, 0, 0, 0, 0, 1199, 102, 0, 4096, 0, 0, 3176, 4082, 128, 4086, 0, 4096, 590, 0, 4096, 3, 3930, 4070, 0, 43, 3911, 0, 2, 27, 1, 1, 0, 0, 1071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3513, 0, 4073, 122, 2, 38, 4093, 0, 1046, 0, 0, 0, 0, 3552, 0, 0, 244, 0, 73, 3915, 86, 0, 0, 4096, 0, 3141, 1506, 3290, 492, 16, 3981, 2, 0, 0, 951, 897, 10, 0, 4094, 13, 7, 2520, 4046, 31, 25, 0, 4091, 3800, 0, 344, 150, 558, 0, 0, 0, 39, 0, 0, 0, 30, 4069, 0, 0, 2580] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4, 1050, 4016, 0, 60, 0, 4054, 3, 0, 0, 0, 3237, 18, 0, 0, 1, 0, 0, 55, 4095, 1, 905, 44, 3936, 0, 0, 3, 3232, 3920, 0, 2693, 39, 0, 1012, 0, 6, 0, 0, 2523, 0, 1261, 520, 4010, 0, 204, 490, 4096, 3787, 61, 0, 0, 3566, 3986, 0, 0, 3006, 91, 1225, 227, 5, 1, 0, 4093, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1231, 0, 4082, 0, 0, 1175, 39, 0, 0, 0, 2086, 447, 0, 2630, 70, 116, 96, 111, 36, 3700, 4096, 49, 3, 4096, 9, 3638, 25, 0, 1417, 234, 10, 0, 0, 0, 3995, 0, 15, 0, 4003, 4, 196, 4077, 4096, 716, 0, 206, 582, 0, 0, 4086, 0, 1276, 3808, 29, 0, 3923, 0, 39, 0, 21, 3916, 4, 1148, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2267, 1374, 1979, 0, 796, 0, 2753, 0, 4001, 313, 0, 4092, 0, 428, 0, 40, 147, 0, 10, 0, 3619, 0, 3976, 2793, 1552, 3994, 0, 552, 3, 0, 0, 219, 0, 2149, 2328, 16, 0, 0, 3739, 1711, 184, 4095, 0, 1, 0, 0, 0, 3808, 110, 0, 5, 3, 3490, 76, 130, 2018, 0, 0, 0, 0, 3028, 24, 0, 3713] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[725, 0, 4054, 14, 5, 0, 4083, 0, 334, 125, 0, 2242, 0, 3780, 795, 3, 0, 107, 0, 4081, 41, 0, 1, 3515, 0, 3427, 0, 2, 0, 0, 4096, 4, 0, 193, 722, 77, 0, 41, 3983, 0, 95, 4053, 4030, 0, 19, 0, 3784, 3070, 19, 1247, 3, 3983, 4088, 0, 0, 4050, 398, 1, 0, 0, 0, 0, 186, 60] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2574, 1371, 0, 695, 0, 0, 729, 113, 0, 0, 0, 0, 0, 0, 12, 460, 0, 2708, 0, 0, 484, 0, 1506, 685, 754, 0, 0, 0, 0, 0, 0, 21, 0, 0, 65, 82, 0, 3384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 4023, 2066, 653, 200, 0, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 8, 528, 2902, 0, 1376, 0, 3980, 4029, 66, 225, 0, 0, 0, 0, 0, 0, 0, 0, 547, 9, 1063, 0, 0, 2363, 0, 0, 0, 0, 0, 0, 0, 0, 90, 185, 84, 0, 2464, 30, 21, 3502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 2101, 624, 4046, 4096, 3411, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5482, 6609, 3552, 12134, 13114, 3982, 6376, 14287] avg=8192.0000 max=14287 max_rank=7 min=3552 min_rank=2 imbalance_ratio=1.744019 max_load_reduction=51249 max_load_reduction_pct=0.781998 moved_to_echo=37759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2705, 0, 3842, 21, 126, 0, 273, 0, 671, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 743, 0, 6, 1544, 531, 694, 0, 0, 0, 0, 0, 0, 0, 0, 3518, 0, 0, 0, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2401, 244, 1, 0, 6, 3425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 102, 0, 4096, 0, 440, 247, 0, 0, 0, 0, 0, 0, 0, 0, 3642, 128, 4086, 0, 4096, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 3930, 2526, 0, 43, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 27, 1, 1, 0, 0, 1071, 2645, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7762, 7573, 7242, 6077, 6084, 12542, 14509, 3747] avg=8192.0000 max=14509 max_rank=6 min=3747 min_rank=7 imbalance_ratio=1.771118 max_load_reduction=51027 max_load_reduction_pct=0.778610 moved_to_echo=36882 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 1050, 4016, 0, 60, 0, 4054, 3, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 18, 0, 0, 1, 1512, 87, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4095, 1, 905, 44, 3579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3232, 3920, 0, 2693, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 0, 6, 0, 0, 2523, 0, 432, 238, 0, 0, 0, 0, 0, 0, 1261, 0, 3578, 0, 204, 490, 4096, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 3566, 2474, 0, 0, 3006, 0, 0, 0, 0, 0, 0, 0, 0, 91, 1225, 227, 5, 1, 0, 4093, 0, 433, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9306, 4855, 8679, 9887, 4211, 13416, 9107, 6075] avg=8192.0000 max=13416 max_rank=5 min=4211 min_rank=4 imbalance_ratio=1.637695 max_load_reduction=52120 max_load_reduction_pct=0.795288 moved_to_echo=32809 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3513, 0, 4073, 122, 2, 38, 4093, 0, 560, 124, 0, 0, 0, 0, 0, 0, 1046, 0, 0, 0, 0, 3552, 0, 0, 0, 421, 579, 0, 0, 0, 0, 0, 244, 0, 73, 3915, 86, 0, 0, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2002, 1506, 3290, 492, 16, 3981, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 897, 10, 0, 4094, 13, 436, 247, 0, 0, 0, 0, 0, 0, 7, 0, 3610, 31, 25, 0, 4091, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 150, 558, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4069, 0, 0, 2580, 2099, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12525, 5598, 8043, 11289, 6648, 11564, 1091, 8778] avg=8192.0000 max=12525 max_rank=0 min=1091 min_rank=6 imbalance_ratio=1.528931 max_load_reduction=53011 max_load_reduction_pct=0.808884 moved_to_echo=28081 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1883, 665, 353, 0, 1, 255, 1, 0, 316, 125, 0, 0, 0, 0, 0, 0, 0, 3923, 3280, 15, 0, 1264, 0, 0, 1498, 686, 327, 0, 0, 0, 0, 0, 759, 0, 15, 0, 80, 0, 4039, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1113, 4055, 0, 287, 3306, 2209, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1696, 720, 0, 370, 3877, 1, 6, 243, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 53, 233, 2440, 3075, 43, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3549, 199, 0, 2774, 3410, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3599, 10993, 8484, 10978, 11001, 4120, 6429, 9932] avg=8192.0000 max=11001 max_rank=4 min=3599 min_rank=0 imbalance_ratio=1.342896 max_load_reduction=54535 max_load_reduction_pct=0.832138 moved_to_echo=31482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[725, 0, 4054, 14, 5, 0, 4083, 0, 611, 106, 0, 0, 0, 0, 0, 0, 334, 125, 0, 2242, 0, 3780, 795, 3, 1551, 678, 632, 0, 0, 0, 0, 0, 0, 107, 0, 4081, 41, 0, 1, 3197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2184, 0, 2, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 722, 77, 0, 41, 3983, 0, 434, 212, 0, 0, 0, 0, 0, 0, 95, 0, 3596, 0, 19, 0, 3784, 3070, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1247, 3, 3983, 2537, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 398, 1, 0, 0, 0, 0, 186, 60, 3375, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9598, 10140, 7427, 6286, 5662, 10564, 11839, 4020] avg=8192.0000 max=11839 max_rank=6 min=4020 min_rank=7 imbalance_ratio=1.445190 max_load_reduction=53697 max_load_reduction_pct=0.819351 moved_to_echo=32085 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1231, 0, 4082, 0, 0, 1175, 39, 0, 648, 124, 0, 0, 0, 0, 0, 0, 0, 0, 2086, 447, 0, 2630, 70, 116, 1445, 682, 671, 0, 0, 0, 0, 0, 96, 111, 36, 3700, 4096, 49, 3, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2319, 25, 0, 1417, 234, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 15, 0, 4003, 4, 441, 247, 0, 0, 0, 0, 0, 0, 196, 0, 3655, 716, 0, 206, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 1276, 2363, 29, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 21, 3916, 4, 1148, 0, 3395, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7299, 8147, 11816, 4014, 8705, 5355, 11677, 8523] avg=8192.0000 max=11816 max_rank=2 min=4014 min_rank=3 imbalance_ratio=1.442383 max_load_reduction=53720 max_load_reduction_pct=0.819702 moved_to_echo=34260 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2267, 1374, 1979, 0, 796, 0, 2753, 0, 712, 84, 0, 0, 0, 0, 0, 0, 4001, 313, 0, 4092, 0, 428, 0, 40, 1324, 686, 737, 0, 0, 0, 0, 0, 147, 0, 10, 0, 3619, 0, 3976, 2541, 0, 0, 0, 0, 0, 0, 0, 0, 1552, 2545, 0, 552, 3, 0, 0, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2149, 2328, 16, 0, 0, 3739, 1711, 0, 168, 0, 0, 0, 0, 0, 0, 184, 0, 0, 1, 0, 0, 0, 3808, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 5, 3, 2166, 76, 130, 2018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3028, 24, 0, 3713, 3409, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9965, 11621, 10293, 4871, 10111, 3993, 4508, 10174] avg=8192.0000 max=11621 max_rank=1 min=3993 min_rank=5 imbalance_ratio=1.418579 max_load_reduction=53915 max_load_reduction_pct=0.822678 moved_to_echo=28786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 9984, 384, 13568, 17024, 3328, 256, 21120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[12416, 3200, 25088, 1536, 1024, 2176, 15360, 128, 4352, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1664, 15232, 9856, 9216, 6784, 3840, 16512, 2944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[5504, 4864, 5376, 16896, 128, 12416, 1408, 256, 10496, 4480, 4480, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1792, 0, 18688, 896, 5504, 768, 20864, 17536, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1280, 256, 256, 19328, 8064, 1152, 8064, 27520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 8064, 12672, 2944, 1536, 512, 30336, 5760, 2304, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 1280, 384, 128, 16768, 896, 9600, 14336, 22272, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[7, 0, 59, 1024, 0, 0, 352, 0, 3848, 0, 4086, 3079, 0, 0, 0, 7, 0, 3940, 0, 0, 2102, 10, 0, 0, 0, 4030, 0, 8, 0, 59, 0, 3882, 150, 0, 2, 4036, 4065, 0, 4086, 4096, 32, 12, 1, 55, 2636, 0, 990, 1353, 3670, 4055, 0, 2, 0, 34, 4086, 2, 0, 4096, 0, 146, 1437, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4069, 0, 3, 31, 0, 0, 0, 0, 165, 4003, 4096, 0, 67, 0, 4078, 1698, 0, 4, 0, 5, 0, 0, 17, 0, 4077, 448, 74, 2436, 0, 3956, 1820, 22, 7, 0, 3957, 4096, 0, 4065, 1839, 2, 0, 0, 2, 3906, 46, 0, 0, 4085, 966, 0, 0, 10, 57, 4041, 0, 4, 4096, 0, 86, 3200, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 0, 0, 9, 96, 0, 4002, 0, 6, 10, 666, 48, 3801, 0, 2023, 0, 0, 3967, 0, 0, 0, 0, 37, 3790, 4, 419, 2, 1293, 0, 1462, 288, 1, 108, 3510, 0, 4080, 0, 0, 3, 17, 3184, 8, 2818, 16, 4096, 3934, 1466, 0, 10, 73, 0, 4095, 0, 1652, 4075, 1074, 3698, 2087, 9, 52, 3523] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[10, 4096, 0, 177, 0, 212, 0, 0, 4085, 155, 11, 2168, 4, 0, 0, 83, 1048, 64, 0, 1057, 0, 0, 0, 0, 477, 2634, 0, 1, 0, 0, 4012, 4090, 22, 0, 0, 4083, 3913, 1181, 4095, 0, 9, 4096, 9, 0, 0, 17, 0, 0, 4093, 8, 3686, 0, 1, 0, 4070, 0, 18, 4096, 0, 3659, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[165, 0, 3, 0, 29, 0, 9, 0, 4046, 0, 0, 3569, 38, 0, 1, 4093, 3, 1612, 0, 4029, 524, 0, 0, 0, 3, 0, 0, 16, 0, 0, 3712, 1673, 0, 0, 4, 4074, 4096, 56, 2509, 0, 28, 3872, 0, 1, 0, 4090, 3, 0, 4063, 198, 4086, 2, 0, 25, 4096, 0, 21, 3883, 370, 0, 3926, 7, 22, 2579] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4086, 19, 506, 0, 0, 0, 0, 0, 1668, 3678, 4073, 0, 879, 45, 4096, 3836, 0, 15, 0, 0, 0, 20, 0, 0, 3961, 47, 1109, 2709, 0, 2848, 5, 0, 0, 0, 4096, 3415, 0, 1952, 0, 0, 0, 0, 0, 0, 3866, 404, 0, 44, 2193, 0, 0, 0, 0, 4096, 5, 265, 4092, 2874, 4055, 407, 62, 2, 108] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 16, 14, 3208, 0, 1415, 12, 4055, 2, 0, 0, 4002, 32, 0, 0, 221, 0, 0, 0, 0, 0, 4075, 0, 0, 508, 0, 2548, 2, 95, 29, 0, 1262, 3949, 0, 4091, 4072, 383, 4096, 9, 0, 0, 0, 0, 4068, 4094, 1946, 2663, 0, 25, 0, 3132, 0, 39, 4096, 4089, 2886, 398, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[507, 0, 0, 0, 106, 802, 314, 2, 3706, 6, 0, 0, 0, 69, 0, 3333, 3980, 0, 0, 3301, 502, 0, 0, 0, 0, 1714, 124, 3, 2375, 0, 4068, 0, 0, 0, 4088, 4096, 4095, 4, 739, 0, 0, 83, 22, 0, 0, 3964, 0, 2411, 3159, 0, 30, 0, 4061, 73, 4049, 0, 1380, 20, 194, 4094, 4025, 0, 0, 37] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 59, 1024, 0, 0, 352, 0, 4065, 2078, 0, 0, 0, 0, 0, 0, 3848, 0, 4086, 3079, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 2102, 10, 0, 0, 2445, 2, 728, 0, 0, 0, 0, 0, 0, 4030, 0, 8, 0, 59, 0, 3882, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 2, 3532, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 32, 12, 1, 55, 2636, 0, 990, 1353, 2, 0, 504, 0, 0, 0, 0, 0, 3670, 4055, 0, 2, 0, 34, 1641, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1290, 0, 146, 1437, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7585, 11016, 9227, 7979, 11866, 5585, 9404, 2874] avg=8192.0000 max=11866 max_rank=4 min=2874 min_rank=7 imbalance_ratio=1.448486 max_load_reduction=53670 max_load_reduction_pct=0.818939 moved_to_echo=29729 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 0, 0, 9, 96, 3510, 2067, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 6, 10, 666, 48, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2023, 0, 0, 3967, 0, 0, 0, 2450, 1215, 725, 0, 0, 0, 0, 0, 0, 37, 3790, 4, 419, 2, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1462, 288, 1, 95, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 17, 3184, 8, 2818, 16, 4096, 1327, 202, 13, 0, 0, 0, 0, 0, 3934, 1466, 0, 10, 73, 0, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1652, 1283, 1074, 3698, 2087, 9, 52, 3523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5706, 5991, 10380, 5343, 5926, 11684, 7128, 13378] avg=8192.0000 max=13378 max_rank=7 min=5343 min_rank=3 imbalance_ratio=1.633057 max_load_reduction=52158 max_load_reduction_pct=0.795868 moved_to_echo=38116 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4069, 0, 3, 31, 0, 0, 0, 4096, 2078, 0, 0, 0, 0, 0, 0, 0, 165, 4003, 4096, 0, 67, 0, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 1698, 0, 4, 0, 5, 0, 0, 17, 2418, 1304, 728, 0, 0, 0, 0, 0, 0, 4077, 448, 74, 2436, 0, 3337, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 22, 7, 0, 3463, 0, 0, 4065, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 3906, 46, 0, 0, 1424, 619, 494, 0, 0, 0, 0, 0, 4085, 966, 0, 0, 10, 57, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1290, 0, 86, 3200, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10277, 9681, 6174, 12192, 9396, 6493, 6741, 4582] avg=8192.0000 max=12192 max_rank=3 min=4582 min_rank=7 imbalance_ratio=1.488281 max_load_reduction=53344 max_load_reduction_pct=0.813965 moved_to_echo=27212 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[10, 4096, 0, 177, 0, 212, 0, 0, 3913, 2081, 0, 0, 0, 0, 0, 0, 4085, 155, 11, 2168, 4, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 64, 0, 1057, 0, 0, 0, 0, 2437, 29, 733, 0, 0, 0, 0, 0, 477, 2634, 0, 1, 0, 0, 3380, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 3570, 0, 1181, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 9, 0, 0, 17, 0, 0, 32, 632, 513, 0, 0, 0, 0, 0, 4093, 8, 3686, 0, 1, 0, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1282, 0, 3659, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10489, 6445, 5368, 10582, 8868, 5308, 9421, 9055] avg=8192.0000 max=10582 max_rank=3 min=5308 min_rank=5 imbalance_ratio=1.291748 max_load_reduction=54954 max_load_reduction_pct=0.838531 moved_to_echo=32652 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[507, 0, 0, 0, 106, 802, 314, 2, 4095, 10, 0, 0, 0, 0, 0, 0, 3706, 6, 0, 0, 0, 69, 0, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 3301, 502, 0, 0, 0, 2422, 1066, 3, 0, 0, 0, 0, 0, 0, 1714, 124, 3, 2375, 0, 3432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3585, 0, 4, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 22, 0, 0, 3964, 0, 2411, 1163, 636, 511, 0, 0, 0, 0, 0, 3159, 0, 30, 0, 4061, 73, 1627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1380, 7, 194, 4094, 4025, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5836, 4885, 11274, 7648, 8416, 8790, 8950, 9737] avg=8192.0000 max=11274 max_rank=2 min=4885 min_rank=1 imbalance_ratio=1.376221 max_load_reduction=54262 max_load_reduction_pct=0.827972 moved_to_echo=35893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 19, 506, 0, 0, 0, 0, 3415, 2076, 0, 0, 0, 0, 0, 0, 0, 1668, 3678, 4073, 0, 879, 45, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 15, 0, 0, 0, 20, 0, 2451, 1310, 728, 0, 0, 0, 0, 0, 0, 3961, 47, 1109, 2709, 0, 2403, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3585, 0, 0, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 404, 0, 1430, 445, 511, 0, 0, 0, 0, 0, 44, 2193, 0, 0, 0, 0, 1645, 5, 0, 0, 0, 0, 0, 0, 0, 0, 265, 1288, 2874, 4055, 407, 62, 2, 108, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10102, 11699, 8360, 10234, 5537, 6656, 3887, 9061] avg=8192.0000 max=11699 max_rank=1 min=3887 min_rank=6 imbalance_ratio=1.428101 max_load_reduction=53837 max_load_reduction_pct=0.821487 moved_to_echo=25141 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 16, 14, 3208, 3949, 2078, 0, 0, 0, 0, 0, 0, 0, 1415, 12, 4055, 2, 0, 0, 1326, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 221, 0, 0, 0, 0, 1874, 1279, 728, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 508, 0, 2150, 2, 0, 0, 0, 0, 0, 0, 0, 0, 95, 29, 0, 1105, 0, 0, 4091, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 383, 4096, 9, 0, 0, 0, 0, 4068, 1397, 398, 157, 0, 0, 0, 0, 0, 4094, 1946, 2663, 0, 25, 0, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1290, 4089, 2886, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9269, 6810, 4134, 6735, 9392, 10508, 9986, 8702] avg=8192.0000 max=10508 max_rank=5 min=4134 min_rank=2 imbalance_ratio=1.282715 max_load_reduction=55028 max_load_reduction_pct=0.839661 moved_to_echo=38588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[165, 0, 3, 0, 29, 0, 9, 0, 4096, 1970, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 3569, 38, 0, 1, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1612, 0, 4029, 524, 0, 0, 0, 2451, 1309, 691, 0, 0, 0, 0, 0, 3, 0, 0, 16, 0, 0, 3131, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3566, 0, 56, 2509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3872, 0, 1, 0, 4090, 3, 0, 1429, 581, 508, 0, 0, 0, 0, 0, 4063, 198, 4086, 2, 0, 25, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1222, 370, 0, 3926, 7, 22, 2579, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6272, 9009, 10619, 4823, 6135, 10512, 10019, 8147] avg=8192.0000 max=10619 max_rank=2 min=4823 min_rank=3 imbalance_ratio=1.296265 max_load_reduction=54917 max_load_reduction_pct=0.837967 moved_to_echo=34813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3456, 8960, 8704, 18688, 19584, 128, 128, 6272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[10624, 7680, 128, 8704, 7168, 128, 128, 128, 19072, 7552, 5120, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1792, 384, 4096, 22528, 0, 1280, 25728, 10112, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[512, 12288, 128, 3328, 6656, 14848, 1536, 12032, 8320, 3584, 3328, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[15744, 7424, 11904, 21120, 128, 1792, 128, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 12288, 128, 1792, 256, 1152, 768, 3328, 31232, 14464, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[512, 20608, 4480, 1280, 8448, 128, 18944, 11520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[27264, 10880, 10496, 128, 4224, 256, 12800, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 98, 1273, 0, 4043, 3315, 2472, 6, 3977, 4081, 1, 7, 0, 0, 0, 1112, 3914, 0, 11, 0, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 29, 816, 0, 4088, 0, 222, 0, 0, 3011, 569, 2566, 0, 2, 1918, 0, 4096, 0, 3108, 34, 0, 0, 3833, 0, 4096, 24, 61, 671, 11, 1441, 0, 2456, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[17, 0, 1, 2271, 0, 1, 3698, 4096, 565, 3994, 0, 1569, 0, 0, 0, 0, 3109, 3053, 0, 2, 0, 2228, 0, 4089, 0, 4000, 0, 688, 1, 1487, 768, 0, 879, 1, 1612, 0, 2840, 0, 2310, 0, 20, 0, 53, 1, 0, 4096, 0, 5, 0, 6, 0, 164, 0, 4058, 332, 25, 4066, 0, 0, 4, 4036, 3, 4095, 1293] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[128, 0, 3428, 0, 0, 0, 3986, 4094, 0, 4000, 0, 3934, 21, 3, 0, 0, 0, 3951, 11, 48, 44, 0, 0, 76, 9, 500, 4012, 3239, 0, 0, 4096, 289, 4077, 0, 21, 5, 0, 0, 4078, 0, 100, 0, 0, 4018, 0, 4094, 51, 1901, 15, 74, 0, 1, 692, 3786, 66, 91, 0, 0, 750, 28, 3549, 0, 2270, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[611, 53, 63, 0, 0, 4086, 1, 3847, 0, 0, 509, 0, 4075, 0, 0, 0, 0, 1, 314, 0, 0, 4091, 3674, 0, 10, 0, 2, 2946, 0, 4, 4095, 0, 4, 2158, 0, 0, 5, 29, 2042, 3700, 0, 0, 3642, 2496, 0, 0, 0, 3389, 0, 1, 11, 278, 0, 4051, 0, 3994, 8, 963, 0, 0, 3879, 2, 3802, 2700] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 1115, 12, 522, 0, 3314, 961, 3951, 0, 3807, 349, 5, 0, 8, 0, 0, 1, 1, 1180, 0, 0, 4038, 0, 664, 18, 3617, 2, 3580, 3199, 0, 213, 2, 185, 2762, 2207, 2011, 0, 44, 4096, 871, 3, 10, 3233, 4096, 1050, 403, 16, 2621, 0, 60, 0, 12, 1502, 4094, 0, 1373, 0, 0, 4096, 4, 17, 208, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 394, 4094, 325, 4, 4064, 3809, 4096, 0, 3966, 4096, 0, 3787, 17, 2, 0, 1221, 3628, 0, 0, 0, 4085, 0, 4015, 0, 6, 743, 0, 13, 0, 4069, 98, 21, 464, 69, 25, 0, 0, 1, 3139, 0, 0, 0, 4095, 260, 2, 24, 0, 0, 6, 0, 626, 0, 1666, 0, 0, 3, 0, 4027, 4029, 537, 1, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[3, 14, 4088, 4093, 0, 3326, 31, 4026, 261, 3666, 0, 23, 3866, 0, 4, 0, 0, 1489, 0, 2282, 0, 0, 0, 1889, 0, 0, 0, 4089, 0, 279, 244, 0, 7, 4041, 2911, 2284, 0, 470, 33, 2297, 110, 0, 110, 3475, 689, 1016, 0, 1854, 0, 1, 0, 4, 0, 3768, 598, 10, 2, 0, 0, 2, 4084, 2, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[22, 0, 683, 4072, 1, 3100, 4096, 3441, 74, 3239, 4091, 6, 0, 0, 0, 637, 14, 0, 0, 1, 49, 4096, 0, 4092, 4, 15, 408, 3365, 8, 0, 4096, 0, 36, 1078, 5, 0, 0, 15, 24, 4051, 6, 0, 4036, 3995, 0, 1238, 0, 0, 0, 0, 0, 40, 5, 3873, 1, 0, 451, 1, 0, 3407, 2, 74, 3588, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 98, 1273, 0, 3365, 3315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3977, 4081, 1, 7, 0, 0, 0, 678, 443, 60, 0, 0, 0, 0, 0, 1112, 3914, 0, 11, 0, 3172, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 816, 643, 481, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 222, 0, 0, 3011, 569, 58, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 2, 1732, 0, 4096, 0, 3108, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 3833, 0, 4096, 24, 61, 1829, 0, 0, 0, 0, 0, 0, 0, 671, 11, 1441, 0, 2456, 2, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8052, 9253, 12284, 1969, 7948, 11504, 9877, 4649] avg=8192.0000 max=12284 max_rank=2 min=1969 min_rank=3 imbalance_ratio=1.499512 max_load_reduction=53252 max_load_reduction_pct=0.812561 moved_to_echo=33978 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[611, 53, 63, 0, 0, 3401, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 0, 4075, 0, 0, 0, 685, 442, 78, 0, 0, 0, 0, 0, 0, 1, 314, 0, 0, 3169, 3674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2, 2946, 0, 4, 4095, 0, 1002, 480, 0, 0, 0, 0, 0, 0, 4, 2158, 0, 0, 5, 29, 2042, 3700, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3642, 2253, 0, 0, 0, 3389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 278, 0, 4051, 0, 3994, 2845, 0, 0, 0, 0, 0, 0, 0, 8, 963, 0, 0, 3879, 2, 3802, 2700, 89, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 5789, 7158, 8539, 8014, 9284, 11180, 11443] avg=8192.0000 max=11443 max_rank=7 min=4129 min_rank=0 imbalance_ratio=1.396851 max_load_reduction=54093 max_load_reduction_pct=0.825394 moved_to_echo=39921 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[128, 0, 3428, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 3934, 21, 3, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 3951, 11, 48, 44, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 9, 500, 4012, 3239, 0, 0, 4096, 289, 1066, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 21, 5, 0, 0, 4078, 0, 123, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 3625, 0, 4094, 51, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 15, 74, 0, 1, 692, 3786, 66, 91, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 750, 28, 3549, 0, 2270, 0, 144, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7542, 8084, 4130, 13211, 8304, 9771, 7753, 6741] avg=8192.0000 max=13211 max_rank=3 min=4130 min_rank=2 imbalance_ratio=1.612671 max_load_reduction=52325 max_load_reduction_pct=0.798416 moved_to_echo=32569 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 1, 2271, 0, 0, 3698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 3994, 0, 1569, 0, 0, 0, 0, 1, 243, 1, 0, 0, 0, 0, 0, 3109, 3053, 0, 2, 0, 1722, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 688, 1, 1487, 768, 0, 1067, 263, 0, 0, 0, 0, 0, 0, 879, 1, 1612, 0, 2840, 0, 2310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 53, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 164, 0, 4058, 332, 25, 3029, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4, 4036, 3, 4095, 1293, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5987, 6373, 11975, 8274, 7642, 4174, 7614, 13497] avg=8192.0000 max=13497 max_rank=7 min=4174 min_rank=5 imbalance_ratio=1.647583 max_load_reduction=52039 max_load_reduction_pct=0.794052 moved_to_echo=32927 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1115, 12, 522, 0, 2758, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3807, 349, 5, 0, 8, 0, 0, 556, 437, 129, 0, 0, 0, 0, 0, 1, 1, 1180, 0, 0, 3127, 0, 664, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3617, 2, 3580, 3199, 0, 213, 2, 1029, 474, 0, 0, 0, 0, 0, 0, 185, 2762, 2207, 2011, 0, 44, 4096, 871, 125, 0, 0, 0, 0, 0, 0, 0, 3, 10, 3233, 3695, 1050, 403, 16, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 12, 1502, 4094, 0, 1373, 2922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 17, 208, 0, 3, 147, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5368, 5291, 4973, 12134, 12301, 11031, 9963, 4475] avg=8192.0000 max=12301 max_rank=4 min=4475 min_rank=7 imbalance_ratio=1.501587 max_load_reduction=53235 max_load_reduction_pct=0.812302 moved_to_echo=37770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 394, 4094, 325, 4, 3382, 3809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 4096, 0, 3787, 17, 2, 0, 682, 442, 128, 0, 0, 0, 0, 0, 1221, 3628, 0, 0, 0, 3164, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 743, 0, 13, 0, 4069, 98, 1067, 479, 0, 0, 0, 0, 0, 0, 21, 464, 69, 25, 0, 0, 1, 3139, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 260, 2, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 626, 0, 1666, 0, 0, 3029, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4027, 4029, 537, 1, 0, 9, 147, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12008, 13120, 12028, 6475, 3844, 3981, 5327, 8753] avg=8192.0000 max=13120 max_rank=1 min=3844 min_rank=4 imbalance_ratio=1.601562 max_load_reduction=52416 max_load_reduction_pct=0.799805 moved_to_echo=21905 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 14, 4088, 4093, 0, 2767, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 3666, 0, 23, 3866, 0, 4, 0, 559, 0, 112, 0, 0, 0, 0, 0, 0, 1489, 0, 2282, 0, 0, 0, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 279, 244, 0, 1049, 0, 0, 0, 0, 0, 0, 0, 7, 4041, 2911, 2284, 0, 470, 33, 2297, 109, 0, 0, 0, 0, 0, 0, 0, 110, 0, 110, 3127, 689, 1016, 0, 1854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 3768, 598, 10, 2977, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 4084, 2, 4095, 0, 127, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10996, 8491, 5660, 5661, 12152, 6906, 7358, 8312] avg=8192.0000 max=12152 max_rank=4 min=5660 min_rank=2 imbalance_ratio=1.483398 max_load_reduction=53384 max_load_reduction_pct=0.814575 moved_to_echo=34728 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 683, 4072, 1, 2580, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 3239, 4091, 6, 0, 0, 0, 637, 520, 443, 125, 0, 0, 0, 0, 0, 14, 0, 0, 1, 49, 3172, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4, 15, 408, 3365, 8, 0, 4096, 0, 896, 481, 0, 0, 0, 0, 0, 0, 36, 1078, 5, 0, 0, 15, 24, 4051, 122, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4036, 3605, 0, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 5, 3873, 1, 0, 2545, 0, 0, 0, 0, 0, 0, 0, 451, 1, 0, 3407, 2, 74, 3588, 0, 143, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11454, 9135, 7328, 9273, 5331, 8885, 6464, 7666] avg=8192.0000 max=11454 max_rank=0 min=5331 min_rank=4 imbalance_ratio=1.398193 max_load_reduction=54082 max_load_reduction_pct=0.825226 moved_to_echo=28346 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 8192, 5248, 17920, 3328, 1792, 17664, 1280, 7936, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[5504, 16128, 1536, 2432, 128, 17536, 3712, 18944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5248, 1024, 10368, 7552, 18560, 384, 17920, 4096, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[896, 1664, 12544, 12672, 128, 18304, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2816, 128, 11136, 21760, 2048, 14976, 128, 12928, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5248, 14592, 6912, 4608, 2944, 640, 15616, 14720, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 256, 128, 4992, 2304, 29440, 1024, 5632, 22272, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1024, 26752, 13184, 5632, 11776, 128, 128, 640, 3712, 2560, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4095, 0, 722, 0, 3185, 221, 0, 3, 4016, 0, 0, 0, 89, 3523, 286, 4096, 12, 4090, 0, 0, 4032, 4055, 2, 3132, 0, 4096, 0, 0, 33, 0, 4096, 1739, 0, 4096, 0, 19, 3116, 0, 17, 0, 66, 0, 1207, 560, 3, 1860, 0, 33, 561, 4000, 0, 3, 0, 0, 0, 46, 0, 326, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3993, 0, 0, 0, 316, 8, 58, 4093, 3914, 0, 0, 3983, 0, 0, 234, 2, 3995, 6, 3756, 3217, 75, 45, 708, 4094, 4086, 158, 992, 3397, 0, 0, 16, 0, 5, 0, 1371, 644, 1215, 6, 1, 1221, 2313, 4046, 627, 4061, 0, 1470, 4, 352, 355, 0, 0, 0, 3936, 12, 0, 0, 2735, 0, 4, 0, 12, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 3165, 0, 0, 0, 3885, 4089, 0, 11, 187, 0, 503, 2527, 0, 929, 57, 3, 3, 0, 0, 0, 336, 0, 4024, 4090, 3279, 2, 4052, 41, 245, 4069, 0, 0, 0, 338, 0, 0, 3543, 410, 795, 3960, 0, 0, 72, 3227, 1365, 0, 3157, 0, 1239, 1344, 0, 869, 4066, 1, 322, 0, 3422, 0, 0, 0, 1906, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3871, 0, 4079, 2791, 0, 0, 0, 1397, 0, 354, 4095, 3, 0, 3826, 2, 11, 0, 0, 3968, 3, 4096, 4077, 8, 0, 9, 4012, 0, 0, 2500, 2541, 2711, 3284, 0, 1, 0, 84, 2768, 0, 9, 0, 51, 3951, 65, 0, 0, 4086, 119, 1999, 7, 148, 128, 0, 0, 0, 4094, 0, 31, 0, 349, 0, 0, 7, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4095, 135, 607, 0, 45, 0, 0, 32, 0, 12, 3652, 359, 1, 3994, 0, 45, 1451, 0, 3939, 0, 3495, 0, 3872, 0, 0, 3796, 723, 0, 674, 3977, 131, 1723, 0, 0, 1, 459, 4091, 0, 4048, 152, 0, 3212, 3474, 0, 0, 747, 1, 0, 4096, 0, 0, 322, 4085, 0, 4043, 0, 0, 0, 0, 4, 5, 0, 1, 37] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 5, 4063, 0, 1, 774, 0, 266, 0, 0, 4089, 0, 1, 1232, 0, 10, 0, 89, 4083, 464, 4096, 1465, 0, 0, 0, 0, 3094, 0, 47, 3935, 0, 10, 0, 938, 0, 602, 4096, 2, 0, 0, 318, 3944, 0, 0, 0, 3841, 3773, 3483, 4094, 727, 2699, 4096, 0, 44, 3718, 11, 153, 0, 0, 0, 327, 0, 946, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 5, 4094, 0, 1, 3860, 0, 3, 0, 0, 2206, 0, 3589, 290, 0, 2, 297, 15, 596, 1, 4089, 4005, 0, 0, 3, 3763, 687, 0, 3904, 2, 0, 1485, 0, 0, 0, 0, 4096, 15, 329, 3931, 0, 4060, 0, 0, 0, 4096, 149, 1, 4094, 0, 98, 4096, 1182, 3, 4069, 14, 3, 0, 0, 0, 5, 0, 2398, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4095, 8, 3769, 13, 484, 0, 32, 28, 24, 1446, 4070, 0, 0, 4043, 0, 4083, 0, 29, 3142, 57, 0, 4090, 0, 55, 0, 0, 3462, 944, 4009, 0, 39, 0, 0, 30, 0, 32, 2615, 3585, 621, 6, 3303, 4064, 1214, 0, 4095, 250, 0, 0, 2283, 1391, 0, 47, 3, 0, 4055, 0, 2, 0, 18, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4095, 0, 722, 453, 0, 0, 0, 0, 0, 0, 0, 0, 3185, 221, 0, 3, 4016, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 89, 3523, 286, 2404, 12, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1818, 2, 3132, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4096, 1739, 0, 4096, 0, 1103, 1159, 0, 0, 0, 0, 0, 0, 19, 1267, 0, 17, 0, 66, 0, 1207, 0, 0, 0, 0, 0, 0, 0, 0, 560, 3, 1860, 0, 33, 561, 2692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 46, 0, 326, 4, 4032, 1849, 2237, 205, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9366, 7505, 10404, 9048, 12226, 2576, 5709, 8702] avg=8192.0000 max=12226 max_rank=4 min=2576 min_rank=5 imbalance_ratio=1.492432 max_load_reduction=53310 max_load_reduction_pct=0.813446 moved_to_echo=29213 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3165, 0, 0, 0, 3885, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 187, 0, 503, 2527, 0, 929, 0, 0, 0, 0, 0, 0, 0, 0, 57, 3, 3, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 1470, 2, 4052, 41, 245, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 3543, 410, 1121, 0, 0, 0, 0, 0, 0, 0, 795, 1610, 0, 0, 72, 3227, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3157, 0, 1239, 1344, 0, 869, 2737, 1, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 3422, 0, 0, 0, 1906, 3, 4090, 2350, 1809, 208, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11139, 4157, 399, 13903, 5412, 7069, 9347, 14110] avg=8192.0000 max=14110 max_rank=7 min=399 min_rank=2 imbalance_ratio=1.722412 max_load_reduction=51426 max_load_reduction_pct=0.784698 moved_to_echo=35938 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3993, 0, 0, 0, 316, 8, 415, 0, 0, 0, 0, 0, 0, 0, 58, 4093, 3914, 0, 0, 3983, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 234, 2, 3995, 6, 2204, 3217, 75, 45, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 1832, 158, 992, 3397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 5, 0, 1371, 644, 1215, 6, 1085, 1063, 0, 0, 0, 0, 0, 0, 1, 497, 2313, 4046, 627, 4061, 0, 1470, 0, 0, 0, 0, 0, 0, 0, 0, 4, 352, 355, 0, 0, 0, 2650, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 0, 4, 0, 12, 0, 4094, 724, 2254, 201, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4732, 12122, 9778, 7087, 5405, 13015, 3373, 10024] avg=8192.0000 max=13015 max_rank=5 min=3373 min_rank=6 imbalance_ratio=1.588745 max_load_reduction=52521 max_load_reduction_pct=0.801407 moved_to_echo=31817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 135, 607, 0, 45, 0, 0, 32, 386, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3652, 359, 1, 3994, 0, 45, 69, 0, 0, 0, 0, 0, 0, 0, 1451, 0, 3939, 0, 2051, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 674, 3977, 131, 1723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 459, 4091, 0, 4048, 152, 1115, 989, 0, 0, 0, 0, 0, 0, 0, 1306, 3474, 0, 0, 747, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 322, 4085, 0, 2721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 1, 37, 3796, 1906, 398, 207, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5300, 8132, 11313, 6830, 10855, 5528, 11224, 6354] avg=8192.0000 max=11313 max_rank=2 min=5300 min_rank=0 imbalance_ratio=1.380981 max_load_reduction=54223 max_load_reduction_pct=0.827377 moved_to_echo=33961 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3871, 0, 4079, 2791, 0, 0, 0, 1397, 453, 0, 0, 0, 0, 0, 0, 0, 0, 354, 4095, 3, 0, 3826, 2, 11, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 3, 2404, 4077, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2500, 2541, 2711, 3284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 84, 2768, 0, 9, 0, 1129, 1159, 0, 0, 0, 0, 0, 0, 51, 1606, 65, 0, 0, 4086, 119, 1999, 0, 0, 0, 0, 0, 0, 0, 0, 7, 148, 128, 0, 0, 0, 2756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 349, 0, 0, 7, 0, 1, 4012, 2345, 0, 209, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12591, 8371, 10460, 11045, 5150, 7926, 3039, 6954] avg=8192.0000 max=12591 max_rank=0 min=3039 min_rank=6 imbalance_ratio=1.536987 max_load_reduction=52945 max_load_reduction_pct=0.807877 moved_to_echo=23069 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4094, 0, 1, 3860, 0, 3, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2206, 0, 3589, 290, 0, 2, 84, 0, 0, 0, 0, 0, 0, 0, 297, 15, 596, 1, 2390, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 306, 0, 3904, 2, 0, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 329, 3931, 1127, 1161, 0, 0, 0, 0, 0, 0, 0, 1644, 0, 0, 0, 4096, 149, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 98, 4096, 1182, 3, 2729, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 5, 0, 2398, 0, 3763, 2416, 381, 213, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8417, 6171, 7304, 5700, 10659, 5890, 12216, 9179] avg=8192.0000 max=12216 max_rank=6 min=5700 min_rank=3 imbalance_ratio=1.491211 max_load_reduction=53320 max_load_reduction_pct=0.813599 moved_to_echo=37944 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4063, 0, 1, 774, 0, 266, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1, 1232, 0, 10, 80, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4083, 464, 2404, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 47, 3935, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 938, 0, 602, 4096, 2, 0, 0, 1025, 1159, 0, 0, 0, 0, 0, 0, 318, 1603, 0, 0, 0, 3841, 3773, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 727, 2699, 4096, 0, 44, 2503, 11, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 327, 0, 946, 0, 0, 2341, 1707, 190, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5562, 5412, 8505, 5379, 7822, 13018, 14174, 5664] avg=8192.0000 max=14174 max_rank=6 min=5379 min_rank=3 imbalance_ratio=1.730225 max_load_reduction=51362 max_load_reduction_pct=0.783722 moved_to_echo=40678 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 8, 3769, 13, 484, 0, 32, 28, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1446, 4070, 0, 0, 4043, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 3142, 57, 0, 4090, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1552, 944, 4009, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 32, 2615, 3585, 621, 6, 1118, 0, 0, 0, 0, 0, 0, 0, 3303, 1652, 1214, 0, 4095, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2283, 1391, 0, 47, 3, 0, 2730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 18, 0, 0, 0, 0, 0, 0, 2412, 1910, 207, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8429, 13666, 7373, 6544, 8007, 10514, 6454, 4549] avg=8192.0000 max=13666 max_rank=1 min=4549 min_rank=7 imbalance_ratio=1.668213 max_load_reduction=51870 max_load_reduction_pct=0.791473 moved_to_echo=29524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4608, 11264, 7168, 4096, 4864, 20480, 5504, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2048, 256, 23296, 896, 13952, 16896, 8448, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[18304, 2688, 6400, 9984, 5376, 1536, 21632, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 0, 6528, 128, 512, 128, 5632, 128, 23808, 16384, 10752, 1664, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[12160, 256, 27904, 2816, 640, 8832, 4352, 6656, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 1024, 128, 5632, 20864, 4352, 13952, 4608, 8832, 6784, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4864, 0, 8704, 1152, 19328, 13952, 7296, 10624, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 9216, 22528, 384, 4224, 23936, 128, 5120, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 3590, 26, 2165, 0, 3236, 4095, 0, 0, 0, 200, 4039, 0, 4070, 4096, 21, 0, 752, 0, 0, 19, 0, 0, 0, 1813, 4011, 0, 1, 0, 4059, 0, 956, 170, 7, 0, 4096, 100, 0, 0, 31, 3843, 0, 8, 0, 0, 0, 3926, 365, 0, 3728, 0, 0, 0, 0, 4, 0, 4096, 4094, 368, 3480, 65, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[15, 358, 0, 3097, 52, 3985, 7, 0, 3849, 0, 0, 126, 0, 618, 0, 4091, 2423, 0, 0, 1074, 2828, 0, 1227, 392, 0, 0, 4051, 4027, 204, 57, 0, 4085, 447, 0, 397, 2392, 4079, 2, 8, 4096, 57, 1248, 300, 0, 439, 0, 0, 0, 1518, 0, 3638, 3446, 0, 269, 54, 0, 0, 172, 3681, 2724, 2, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 2530, 1946, 0, 79, 0, 0, 4091, 0, 0, 22, 46, 49, 516, 3095, 4096, 4092, 1, 266, 520, 0, 263, 475, 0, 0, 4052, 3992, 0, 0, 0, 4096, 9, 0, 0, 0, 2644, 4075, 762, 11, 534, 0, 0, 7, 0, 0, 1466, 0, 2761, 43, 0, 3969, 0, 0, 0, 3638, 106, 0, 4028, 4096, 0, 1633, 106, 1419] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 30, 0, 1248, 0, 2787, 2, 3719, 4096, 21, 0, 0, 949, 0, 4082, 3630, 4041, 0, 0, 1, 4039, 0, 0, 302, 0, 44, 2, 3597, 122, 0, 0, 4094, 0, 4048, 440, 0, 2446, 54, 7, 0, 14, 0, 555, 222, 0, 0, 36, 2, 3799, 0, 3410, 4096, 0, 0, 0, 4096, 0, 0, 1952, 3544, 0, 9, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[3970, 55, 0, 4095, 6, 1195, 0, 2, 920, 3, 0, 2306, 81, 1446, 0, 4093, 3884, 888, 30, 0, 0, 9, 1, 37, 0, 1, 4096, 3368, 0, 20, 0, 4096, 403, 0, 0, 0, 4069, 9, 6, 1, 4084, 0, 1, 0, 0, 0, 12, 0, 4084, 0, 0, 3977, 1775, 1, 19, 4096, 2, 284, 4079, 4023, 4, 5, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[788, 0, 20, 3485, 0, 0, 0, 0, 3911, 12, 0, 4089, 174, 0, 0, 3987, 3956, 599, 73, 0, 0, 0, 1862, 1, 0, 18, 3483, 4096, 0, 4094, 0, 4096, 0, 0, 0, 112, 0, 772, 3, 0, 1856, 0, 0, 0, 0, 4067, 6, 0, 2724, 0, 0, 4096, 0, 1, 170, 4096, 3745, 0, 1038, 4096, 10, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4042, 0, 3636, 1173, 33, 0, 0, 12, 3, 0, 487, 3994, 0, 0, 0, 4095, 3889, 2, 29, 0, 5, 503, 3707, 1222, 241, 230, 0, 3985, 14, 0, 1, 4061, 1182, 0, 0, 1682, 2782, 4096, 30, 0, 3762, 530, 4085, 62, 3, 4096, 5, 0, 3345, 0, 0, 3928, 20, 263, 0, 300, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4024, 30, 145, 3567, 16, 130, 4, 0, 3687, 0, 0, 104, 4066, 0, 0, 3964, 4080, 4096, 2, 0, 0, 0, 239, 1861, 0, 0, 0, 4096, 1, 4085, 0, 4096, 0, 1, 1, 4093, 831, 1173, 0, 0, 3908, 0, 0, 11, 0, 0, 0, 0, 4079, 274, 0, 704, 0, 148, 0, 0, 456, 4, 10, 4087, 8, 525, 0, 2930] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 2530, 1946, 0, 79, 0, 0, 885, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 22, 46, 49, 516, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 1, 266, 520, 0, 263, 475, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 3992, 0, 0, 0, 970, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2644, 4075, 762, 11, 1220, 115, 0, 0, 0, 0, 0, 0, 534, 0, 0, 7, 0, 0, 1466, 0, 3126, 532, 0, 0, 0, 0, 0, 0, 2761, 43, 0, 1794, 0, 0, 0, 3638, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 4028, 4096, 0, 1633, 106, 1419, 659, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5442, 6067, 10229, 9014, 8836, 5665, 8236, 12047] avg=8192.0000 max=12047 max_rank=7 min=5442 min_rank=0 imbalance_ratio=1.470581 max_load_reduction=53489 max_load_reduction_pct=0.816177 moved_to_echo=34784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[15, 358, 0, 3097, 52, 3985, 7, 0, 768, 0, 0, 0, 0, 0, 0, 0, 3849, 0, 0, 126, 0, 618, 0, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 2423, 0, 0, 1074, 2828, 0, 1227, 392, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4027, 204, 57, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 397, 2392, 4079, 2, 8, 4096, 1613, 99, 0, 0, 0, 0, 0, 0, 57, 1248, 300, 0, 439, 0, 0, 0, 3118, 703, 0, 0, 0, 0, 0, 0, 1518, 0, 3638, 1559, 0, 269, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 3681, 2724, 2, 1, 0, 0, 572, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 6368, 8392, 9306, 13133, 5865, 7038, 7152] avg=8192.0000 max=13133 max_rank=4 min=5865 min_rank=5 imbalance_ratio=1.603149 max_load_reduction=52403 max_load_reduction_pct=0.799606 moved_to_echo=33188 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3590, 26, 2165, 0, 3236, 831, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 200, 4039, 0, 1766, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 752, 0, 0, 19, 0, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1813, 4011, 0, 1, 0, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 956, 170, 7, 0, 4096, 100, 0, 1604, 108, 0, 0, 0, 0, 0, 0, 0, 31, 3843, 0, 8, 0, 0, 0, 3098, 700, 0, 0, 0, 0, 0, 0, 3926, 365, 0, 1686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 4094, 368, 3480, 65, 6, 619, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9848, 10100, 5372, 6786, 7041, 7680, 5977, 12732] avg=8192.0000 max=12732 max_rank=7 min=5372 min_rank=2 imbalance_ratio=1.554199 max_load_reduction=52804 max_load_reduction_pct=0.805725 moved_to_echo=33430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4024, 30, 145, 3567, 16, 130, 4, 0, 157, 0, 0, 0, 0, 0, 0, 0, 3687, 0, 0, 104, 4066, 0, 0, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 2, 0, 0, 0, 239, 1861, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4085, 0, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4093, 831, 1173, 0, 0, 1563, 20, 0, 0, 0, 0, 0, 0, 3908, 0, 0, 11, 0, 0, 0, 0, 3126, 681, 0, 0, 0, 0, 0, 0, 4079, 274, 0, 320, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 4, 10, 4087, 8, 525, 0, 2930, 116, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8073, 9577, 10369, 9152, 7682, 7726, 4821, 8136] avg=8192.0000 max=10369 max_rank=2 min=4821 min_rank=6 imbalance_ratio=1.265747 max_load_reduction=55167 max_load_reduction_pct=0.841782 moved_to_echo=28365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 0, 1248, 0, 2787, 2, 3719, 913, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 949, 0, 4082, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 1, 4039, 0, 0, 302, 532, 0, 0, 0, 0, 0, 0, 0, 0, 44, 2, 3597, 122, 0, 0, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 440, 0, 2446, 54, 7, 0, 1431, 118, 0, 0, 0, 0, 0, 0, 14, 0, 555, 222, 0, 0, 36, 2, 3125, 624, 0, 0, 0, 0, 0, 0, 3799, 0, 3410, 1853, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1952, 3544, 0, 9, 0, 0, 680, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8699, 10723, 8915, 4734, 8544, 4578, 13158, 6185] avg=8192.0000 max=13158 max_rank=6 min=4578 min_rank=5 imbalance_ratio=1.606201 max_load_reduction=52378 max_load_reduction_pct=0.799225 moved_to_echo=32465 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3970, 55, 0, 4095, 6, 1195, 0, 2, 891, 0, 0, 0, 0, 0, 0, 0, 920, 3, 0, 2306, 81, 1446, 0, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 888, 30, 0, 0, 9, 1, 37, 520, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3368, 0, 20, 0, 966, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 4069, 9, 6, 1, 1617, 119, 0, 0, 0, 0, 0, 0, 4084, 0, 1, 0, 0, 0, 12, 0, 3130, 708, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 1782, 1775, 1, 19, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 284, 4079, 4023, 4, 5, 0, 0, 665, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10214, 6524, 5369, 8451, 6224, 7935, 11757, 9062] avg=8192.0000 max=11757 max_rank=6 min=5369 min_rank=2 imbalance_ratio=1.435181 max_load_reduction=53779 max_load_reduction_pct=0.820602 moved_to_echo=34978 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4042, 0, 3636, 1173, 33, 0, 0, 12, 876, 0, 0, 0, 0, 0, 0, 0, 3, 0, 487, 3994, 0, 0, 0, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 2, 29, 0, 5, 503, 3707, 1222, 510, 0, 0, 0, 0, 0, 0, 0, 241, 230, 0, 3985, 14, 0, 1, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1182, 0, 0, 1682, 2782, 4096, 30, 0, 1614, 113, 0, 0, 0, 0, 0, 0, 3762, 530, 4085, 62, 3, 4096, 5, 0, 3100, 704, 0, 0, 0, 0, 0, 0, 3345, 0, 0, 1777, 20, 263, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 652, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9772, 6261, 9867, 5432, 11499, 16347, 5705, 653] avg=8192.0000 max=16347 max_rank=5 min=653 min_rank=7 imbalance_ratio=1.995483 max_load_reduction=49189 max_load_reduction_pct=0.750565 moved_to_echo=34204 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[788, 0, 20, 3485, 0, 0, 0, 0, 913, 0, 0, 0, 0, 0, 0, 0, 3911, 12, 0, 4089, 174, 0, 0, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 599, 73, 0, 0, 0, 1862, 1, 532, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3483, 4096, 0, 4094, 0, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 772, 3, 0, 1572, 118, 0, 0, 0, 0, 0, 0, 1856, 0, 0, 0, 0, 4067, 6, 0, 3126, 685, 0, 0, 0, 0, 0, 0, 2724, 0, 0, 1853, 0, 1, 170, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3745, 0, 1038, 4096, 10, 0, 0, 0, 680, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5206, 9916, 7023, 12661, 2577, 9740, 8844, 9569] avg=8192.0000 max=12661 max_rank=3 min=2577 min_rank=4 imbalance_ratio=1.545532 max_load_reduction=52875 max_load_reduction_pct=0.806808 moved_to_echo=30730 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4352, 512, 18944, 26752, 512, 5760, 256, 4480, 4736, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12928, 512, 6400, 22272, 256, 10368, 128, 7040, 6272, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2048, 5120, 1024, 8320, 16896, 14336, 1024, 4224, 12288, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 384, 17536, 31232, 384, 8320, 128, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[24704, 128, 512, 10752, 5632, 6272, 4608, 13568, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[14336, 1920, 8832, 384, 512, 8192, 1536, 128, 24960, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26240, 768, 7168, 12672, 1920, 768, 256, 16256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[30592, 9728, 256, 2176, 7424, 512, 7424, 4352, 3712, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4073, 0, 0, 2540, 0, 0, 2, 2, 1, 0, 97, 4071, 0, 0, 2542, 169, 1, 0, 2952, 14, 40, 0, 4061, 1311, 0, 0, 0, 727, 38, 0, 1339, 4086, 1, 3939, 0, 0, 0, 4096, 0, 2099, 50, 3316, 0, 3723, 1, 3398, 2, 0, 0, 1397, 0, 10, 585, 3, 0, 437, 0, 465, 3477, 2, 3775, 2605, 4089, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3, 0, 0, 176, 0, 0, 90, 11, 2711, 6, 0, 0, 6, 14, 0, 0, 0, 2696, 0, 4086, 4096, 2686, 0, 0, 1233, 0, 2876, 52, 4096, 0, 4095, 20, 10, 4, 4025, 4050, 0, 0, 0, 0, 250, 0, 80, 14, 0, 1779, 3523, 4096, 0, 107, 0, 2380, 2779, 8, 0, 3576, 0, 1127, 4056, 0, 623] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4080, 6, 0, 4096, 107, 0, 3576, 9, 0, 746, 0, 3156, 0, 2, 0, 235, 0, 0, 3988, 0, 4061, 149, 21, 1337, 0, 0, 87, 20, 3644, 4092, 0, 1351, 64, 4079, 452, 83, 0, 3976, 3949, 176, 2127, 0, 1343, 5, 0, 3821, 0, 0, 14, 0, 937, 29, 3932, 0, 35, 0, 0, 0, 1660, 0, 4, 4085, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4064, 0, 364, 0, 0, 74, 13, 38, 12, 0, 3981, 0, 6, 108, 0, 0, 117, 4045, 35, 0, 7, 4091, 65, 4096, 1341, 4, 3, 0, 4096, 0, 0, 4077, 2, 0, 0, 0, 138, 4094, 3865, 0, 3436, 0, 1, 644, 3600, 0, 17, 0, 316, 282, 4, 0, 4096, 2255, 0, 0, 51, 0, 3948, 12, 0, 4026, 73, 4039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[2266, 3, 310, 4094, 713, 3630, 31, 47, 250, 0, 2209, 0, 26, 0, 0, 3870, 34, 0, 0, 0, 0, 16, 0, 2498, 12, 0, 0, 4088, 0, 153, 306, 27, 1390, 2045, 55, 3691, 0, 4072, 0, 4078, 1213, 4096, 0, 194, 372, 2, 0, 37, 2880, 4054, 3903, 7, 4040, 0, 2, 4044, 0, 0, 2, 0, 2, 769, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4077, 0, 1, 256, 0, 773, 0, 0, 349, 3236, 0, 4096, 0, 0, 23, 0, 19, 0, 0, 263, 1532, 4089, 3742, 0, 0, 0, 0, 1, 531, 0, 2126, 0, 439, 3999, 3, 978, 0, 3334, 3849, 0, 38, 4091, 0, 0, 0, 4, 0, 1, 0, 3628, 953, 0, 3700, 0, 4095, 1484, 0, 317, 596, 4094, 3195, 136, 1446, 42] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[3902, 0, 709, 1764, 142, 0, 0, 0, 0, 426, 0, 3966, 0, 10, 0, 4096, 0, 0, 74, 1090, 21, 0, 13, 4077, 3953, 0, 199, 3856, 3203, 3664, 0, 0, 97, 4083, 3157, 2627, 10, 95, 0, 0, 0, 989, 2, 0, 4092, 58, 7, 1542, 0, 4068, 2772, 412, 454, 0, 3443, 0, 0, 12, 0, 0, 97, 0, 562, 1792] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[2601, 4, 4096, 4094, 83, 0, 441, 0, 0, 0, 0, 2815, 94, 0, 0, 0, 7, 0, 0, 0, 7, 1101, 0, 3928, 3460, 0, 0, 8, 3585, 3112, 0, 0, 4, 921, 1, 0, 0, 0, 290, 1671, 0, 205, 0, 204, 157, 3698, 0, 0, 4088, 4094, 4017, 3878, 836, 0, 4084, 81, 0, 0, 4096, 0, 3613, 0, 162, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 0, 2540, 0, 0, 2, 2, 137, 0, 0, 0, 0, 0, 0, 0, 1, 0, 97, 4071, 0, 0, 2542, 169, 1397, 185, 0, 0, 0, 0, 0, 0, 1, 0, 2952, 14, 40, 0, 4061, 1311, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 32, 0, 1339, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3939, 0, 0, 0, 887, 0, 2099, 0, 0, 0, 0, 0, 0, 0, 0, 50, 3316, 0, 3723, 1, 3398, 2, 0, 3024, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 342, 3, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 3477, 2, 3775, 2605, 4089, 0, 67, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6754, 8462, 8383, 6184, 6926, 13553, 792, 14482] avg=8192.0000 max=14482 max_rank=7 min=792 min_rank=6 imbalance_ratio=1.767822 max_load_reduction=51054 max_load_reduction_pct=0.779022 moved_to_echo=35753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4080, 6, 0, 4096, 107, 0, 3576, 9, 925, 0, 0, 0, 0, 0, 0, 0, 0, 746, 0, 3156, 0, 2, 0, 235, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 4061, 149, 21, 1337, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 20, 3018, 4092, 0, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 64, 4079, 452, 83, 0, 860, 3949, 176, 0, 0, 0, 0, 0, 0, 0, 0, 2127, 0, 1343, 5, 0, 3821, 0, 0, 2936, 268, 0, 0, 0, 0, 0, 0, 14, 0, 937, 29, 2287, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 0, 4, 4085, 2, 0, 452, 206, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12799, 4319, 9976, 8568, 9663, 10500, 3302, 6409] avg=8192.0000 max=12799 max_rank=0 min=3302 min_rank=6 imbalance_ratio=1.562378 max_load_reduction=52737 max_load_reduction_pct=0.804703 moved_to_echo=29874 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4077, 0, 1, 256, 0, 773, 0, 0, 871, 0, 0, 0, 0, 0, 0, 0, 349, 3236, 0, 4096, 0, 0, 23, 0, 3628, 151, 0, 0, 0, 0, 0, 0, 19, 0, 0, 263, 1532, 4089, 3742, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 440, 0, 2126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 3999, 3, 978, 0, 721, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4091, 0, 0, 0, 4, 0, 1, 2462, 252, 0, 0, 0, 0, 0, 0, 0, 0, 953, 0, 2152, 0, 4095, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 596, 4094, 3195, 136, 1446, 42, 425, 30, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5978, 11483, 9706, 2567, 9989, 6848, 8684, 10281] avg=8192.0000 max=11483 max_rank=1 min=2567 min_rank=3 imbalance_ratio=1.401733 max_load_reduction=54053 max_load_reduction_pct=0.824783 moved_to_echo=35802 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 3, 0, 0, 176, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 90, 11, 2711, 6, 0, 0, 6, 3523, 182, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 2696, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2686, 0, 0, 1233, 0, 2876, 52, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 20, 10, 4, 871, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, 80, 14, 0, 2972, 7, 0, 0, 0, 0, 0, 0, 1779, 0, 4096, 0, 63, 0, 2380, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3576, 0, 1127, 4056, 0, 623, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4300, 6529, 10892, 10943, 9050, 3323, 11097, 9402] avg=8192.0000 max=11097 max_rank=6 min=3323 min_rank=5 imbalance_ratio=1.354614 max_load_reduction=54439 max_load_reduction_pct=0.830673 moved_to_echo=32872 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4064, 0, 364, 0, 0, 74, 13, 38, 964, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3981, 0, 6, 108, 0, 0, 282, 185, 0, 0, 0, 0, 0, 0, 117, 4045, 35, 0, 7, 4091, 65, 4096, 472, 0, 0, 0, 0, 0, 0, 0, 1341, 4, 3, 0, 3393, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 138, 886, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3436, 0, 1, 644, 3600, 0, 17, 0, 3023, 279, 0, 0, 0, 0, 0, 0, 316, 0, 4, 0, 2383, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 3948, 12, 0, 4026, 73, 4039, 470, 231, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5517, 4574, 12928, 8818, 4891, 11000, 4958, 12850] avg=8192.0000 max=12928 max_rank=2 min=4574 min_rank=1 imbalance_ratio=1.578125 max_load_reduction=52608 max_load_reduction_pct=0.802734 moved_to_echo=33699 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2601, 4, 4096, 4094, 83, 0, 441, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2815, 94, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 1101, 0, 3928, 416, 0, 0, 0, 0, 0, 0, 0, 3460, 0, 0, 8, 2965, 3112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 921, 1, 0, 0, 0, 290, 1671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 204, 157, 3698, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 4088, 0, 4017, 3878, 476, 0, 4084, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3613, 0, 162, 0, 99, 204, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11520, 7003, 5459, 9545, 2887, 4324, 16624, 8174] avg=8192.0000 max=16624 max_rank=6 min=2887 min_rank=4 imbalance_ratio=2.029297 max_load_reduction=48912 max_load_reduction_pct=0.746338 moved_to_echo=32009 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3902, 0, 709, 1764, 142, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 426, 0, 3966, 0, 10, 0, 4096, 4068, 4, 0, 0, 0, 0, 0, 0, 0, 0, 74, 1090, 21, 0, 13, 4077, 369, 0, 0, 0, 0, 0, 0, 0, 3953, 0, 199, 3856, 2653, 3664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 4083, 3157, 2627, 10, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 989, 2, 0, 4092, 58, 7, 1542, 70, 30, 0, 0, 0, 0, 0, 0, 0, 0, 2772, 412, 266, 0, 3443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 97, 0, 562, 1792, 52, 181, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6623, 12570, 5644, 14325, 9995, 6790, 6893, 2696] avg=8192.0000 max=14325 max_rank=3 min=2696 min_rank=7 imbalance_ratio=1.748657 max_load_reduction=51211 max_load_reduction_pct=0.781418 moved_to_echo=26374 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2266, 3, 310, 4094, 713, 3630, 31, 47, 951, 0, 0, 0, 0, 0, 0, 0, 250, 0, 2209, 0, 26, 0, 0, 3870, 4054, 187, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 16, 0, 2498, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4088, 0, 153, 306, 27, 0, 0, 0, 0, 0, 0, 0, 0, 1390, 2045, 55, 3691, 0, 876, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 1213, 4096, 0, 194, 372, 2, 0, 37, 3009, 275, 0, 0, 0, 0, 0, 0, 2880, 0, 3903, 7, 2350, 0, 2, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 769, 5, 0, 464, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12045, 10596, 2548, 4586, 12135, 9198, 13186, 1242] avg=8192.0000 max=13186 max_rank=6 min=1242 min_rank=7 imbalance_ratio=1.609619 max_load_reduction=52350 max_load_reduction_pct=0.798798 moved_to_echo=35761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[9088, 0, 16768, 4352, 10368, 2304, 14080, 8832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 896, 17408, 4224, 11904, 15744, 6400, 6528, 2048, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 4096, 7168, 1408, 8448, 9472, 12032, 21376, 1792, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[6912, 12800, 1408, 5120, 8320, 11136, 128, 1664, 17536, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[29184, 128, 5504, 16896, 1152, 4480, 4352, 128, 4224, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11520, 128, 384, 9984, 12544, 13952, 3840, 13696, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 4608, 6400, 20864, 256, 128, 2688, 8448, 21120, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2048, 23168, 3712, 7424, 256, 5248, 16128, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 339, 1, 0, 8, 3206, 9, 0, 165, 3719, 1, 0, 57, 0, 0, 845, 7, 15, 4075, 0, 0, 3, 0, 1685, 4096, 1, 4074, 3710, 3779, 3513, 4095, 0, 118, 65, 3875, 4093, 4053, 3044, 0, 0, 4096, 221, 0, 0, 270, 0, 0, 0, 89, 0, 0, 2, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 2018, 29, 129, 3672, 0, 0, 0, 2978, 0, 1470, 0, 2, 0, 1585, 0, 0, 3307, 0, 0, 0, 0, 3967, 0, 4014, 3965, 1, 4096, 0, 4095, 0, 0, 0, 0, 2458, 373, 2294, 0, 4057, 0, 586, 2549, 0, 3, 632, 1406, 4094, 817, 0, 0, 0, 0, 1275, 0, 0, 0, 49, 4092, 4096, 113, 22, 0, 1292, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[3743, 0, 0, 91, 1, 191, 0, 4092, 0, 208, 0, 147, 0, 0, 3281, 15, 0, 0, 4089, 2797, 0, 70, 0, 118, 3571, 2175, 15, 897, 0, 1896, 2, 0, 0, 0, 0, 0, 1499, 2, 206, 4010, 2597, 49, 3972, 4091, 4078, 3994, 3888, 0, 1, 28, 250, 0, 0, 0, 88, 619, 0, 4093, 3489, 0, 11, 0, 1170, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4000, 508, 9, 120, 0, 34, 4055, 0, 643, 4091, 0, 0, 533, 718, 0, 2178, 0, 89, 0, 276, 0, 3, 0, 0, 0, 0, 1616, 15, 4096, 5, 14, 3753, 0, 988, 24, 4007, 0, 0, 3134, 4081, 3581, 40, 554, 3990, 4068, 4090, 4089, 101, 0, 0, 1898, 4, 0, 4010, 0, 0, 0, 0, 85, 0, 2, 33, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[29, 2560, 0, 6, 0, 17, 676, 4066, 1239, 1402, 4089, 9, 0, 2, 23, 0, 3955, 1551, 99, 0, 3654, 0, 3959, 0, 0, 4096, 0, 51, 10, 3786, 0, 324, 0, 0, 0, 0, 45, 49, 0, 4096, 4070, 3079, 3448, 0, 1042, 0, 4059, 0, 257, 0, 0, 71, 0, 0, 0, 0, 1738, 4096, 3880, 0, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 6, 14, 20, 0, 4074, 766, 0, 0, 3, 4096, 0, 34, 20, 2, 0, 4083, 3767, 16, 58, 0, 3988, 3348, 1, 0, 4095, 0, 99, 371, 4035, 0, 0, 4096, 174, 3935, 0, 3111, 0, 0, 3885, 4020, 4080, 0, 0, 32, 3944, 0, 0, 0, 1, 6, 62, 0, 0, 144, 0, 0, 4096, 1028, 0, 0, 0, 26, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 8, 150, 7, 0, 4088, 1637, 0, 0, 427, 506, 0, 13, 1220, 0, 1407, 48, 36, 0, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 4095, 0, 3, 0, 0, 3574, 1, 4091, 15, 4094, 76, 32, 3860, 53, 3984, 2337, 4059, 3632, 218, 211, 1328, 0, 3955, 35, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4061, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[3513, 113, 148, 0, 4096, 4092, 0, 5, 0, 0, 0, 2870, 10, 619, 0, 3, 0, 4096, 2609, 3, 4071, 3579, 0, 1, 0, 2898, 0, 3, 0, 1514, 0, 29, 0, 0, 0, 0, 3903, 517, 189, 233, 4037, 4094, 3755, 2, 0, 4084, 2392, 24, 39, 0, 15, 0, 31, 15, 0, 0, 2469, 3982, 0, 196, 0, 206, 1077, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2018, 29, 129, 3672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2978, 0, 1470, 0, 2, 0, 1585, 0, 2070, 454, 0, 0, 0, 0, 0, 0, 0, 3307, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 3965, 1, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2458, 373, 2294, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 121, 0, 3, 632, 0, 0, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1275, 0, 0, 0, 4094, 952, 0, 0, 0, 0, 0, 0, 49, 4092, 4096, 113, 22, 0, 1292, 0, 0, 358, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5848, 8559, 7274, 16171, 9182, 2159, 6321, 10022] avg=8192.0000 max=16171 max_rank=3 min=2159 min_rank=5 imbalance_ratio=1.973999 max_load_reduction=49365 max_load_reduction_pct=0.753250 moved_to_echo=27684 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4096, 0, 1483, 461, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 339, 1, 0, 8, 3325, 1253, 0, 0, 0, 0, 0, 0, 3206, 9, 0, 165, 2089, 1, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 845, 7, 15, 4075, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1685, 4096, 1, 4074, 3710, 888, 0, 0, 0, 0, 0, 0, 0, 0, 3513, 195, 0, 118, 65, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 3044, 0, 0, 4096, 221, 0, 0, 270, 4093, 2622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 2, 14, 2430, 575, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6041, 9022, 5527, 5089, 14457, 7944, 14346, 3110] avg=8192.0000 max=14457 max_rank=4 min=3110 min_rank=7 imbalance_ratio=1.764771 max_load_reduction=51079 max_load_reduction_pct=0.779404 moved_to_echo=39857 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3743, 0, 0, 91, 1, 191, 0, 4092, 0, 489, 0, 0, 0, 0, 0, 0, 0, 208, 0, 147, 0, 0, 3281, 15, 39, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2797, 0, 70, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 2175, 15, 897, 0, 1896, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1499, 2, 206, 942, 0, 0, 0, 0, 0, 0, 0, 0, 2597, 4, 3972, 4091, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 28, 250, 0, 0, 0, 88, 619, 3888, 2704, 0, 0, 0, 0, 0, 0, 0, 4093, 3489, 0, 11, 0, 1170, 2, 2579, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8607, 4980, 7074, 8556, 2649, 14742, 7578, 11350] avg=8192.0000 max=14742 max_rank=5 min=2649 min_rank=4 imbalance_ratio=1.799561 max_load_reduction=50794 max_load_reduction_pct=0.775055 moved_to_echo=36319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[29, 2560, 0, 6, 0, 17, 676, 4066, 1458, 499, 0, 0, 0, 0, 0, 0, 1239, 1402, 4089, 9, 0, 2, 23, 0, 2500, 0, 0, 0, 0, 0, 0, 0, 3955, 1551, 99, 0, 2051, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 51, 10, 3786, 0, 324, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 49, 0, 963, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 147, 3448, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 71, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 1738, 4096, 3880, 0, 0, 0, 3, 0, 2634, 432, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9311, 9264, 11615, 8412, 1057, 8707, 4387, 12783] avg=8192.0000 max=12783 max_rank=7 min=1057 min_rank=4 imbalance_ratio=1.560425 max_load_reduction=52753 max_load_reduction_pct=0.804947 moved_to_echo=26934 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 8, 150, 7, 0, 4088, 1637, 1636, 12, 0, 0, 0, 0, 0, 0, 0, 0, 427, 506, 0, 13, 1220, 0, 3138, 1312, 0, 0, 0, 0, 0, 0, 1407, 48, 36, 0, 2294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4095, 0, 3, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3574, 1, 4091, 15, 4094, 12, 0, 0, 0, 0, 0, 0, 0, 0, 32, 177, 53, 3984, 2337, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 211, 1328, 0, 3955, 35, 0, 0, 0, 3632, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4061, 4093, 52, 545, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7539, 6616, 3785, 8347, 11787, 6801, 11908, 8753] avg=8192.0000 max=11908 max_rank=6 min=3785 min_rank=2 imbalance_ratio=1.453613 max_load_reduction=53628 max_load_reduction_pct=0.818298 moved_to_echo=39249 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4000, 508, 9, 120, 0, 34, 4055, 110, 382, 0, 0, 0, 0, 0, 0, 0, 643, 4091, 0, 0, 533, 718, 0, 2908, 1315, 0, 0, 0, 0, 0, 0, 2178, 0, 89, 0, 156, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1616, 15, 4096, 5, 14, 10, 0, 0, 0, 0, 0, 0, 0, 3753, 0, 988, 24, 4007, 0, 0, 737, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 170, 40, 554, 3990, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 1898, 4, 0, 4010, 0, 4090, 2753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 2, 33, 1, 2015, 503, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9218, 10208, 2426, 5756, 9509, 12924, 12856, 2639] avg=8192.0000 max=12924 max_rank=5 min=2426 min_rank=2 imbalance_ratio=1.577637 max_load_reduction=52612 max_load_reduction_pct=0.802795 moved_to_echo=37928 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 14, 20, 0, 4074, 766, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 34, 20, 2, 0, 3313, 1274, 0, 0, 0, 0, 0, 0, 4083, 3767, 16, 58, 0, 3988, 3348, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 99, 371, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 174, 3935, 0, 3111, 0, 0, 914, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 194, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 62, 0, 0, 144, 0, 0, 2670, 0, 0, 0, 0, 0, 0, 0, 4096, 1028, 0, 0, 0, 26, 0, 2498, 573, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5353, 8742, 15261, 8600, 12230, 4246, 2883, 8221] avg=8192.0000 max=15261 max_rank=2 min=2883 min_rank=6 imbalance_ratio=1.862915 max_load_reduction=50275 max_load_reduction_pct=0.767136 moved_to_echo=27580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3513, 113, 148, 0, 4096, 4092, 0, 5, 1624, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 10, 619, 0, 3, 3324, 1319, 0, 0, 0, 0, 0, 0, 0, 4096, 2609, 3, 2286, 3579, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2898, 0, 3, 0, 1514, 0, 29, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 517, 189, 56, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 195, 3755, 2, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 15, 0, 31, 15, 0, 0, 2392, 2765, 0, 0, 0, 0, 0, 0, 2469, 3982, 0, 196, 0, 206, 1077, 4, 149, 575, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13619, 8145, 12574, 4605, 4665, 8013, 5257, 8658] avg=8192.0000 max=13619 max_rank=0 min=4605 min_rank=3 imbalance_ratio=1.662476 max_load_reduction=51917 max_load_reduction_pct=0.792191 moved_to_echo=26593 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7936, 256, 12672, 4608, 19072, 4736, 12288, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4352, 20480, 12544, 512, 128, 256, 7680, 4224, 12416, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3712, 1408, 384, 10112, 1664, 128, 4352, 896, 26368, 17280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7296, 8704, 768, 512, 7936, 8448, 9728, 13952, 6400, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23040, 1280, 11392, 8832, 12288, 0, 0, 9216, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11776, 17280, 896, 6784, 512, 27648, 128, 384, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[14848, 12800, 7040, 3072, 8960, 7680, 11392, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 2304, 18304, 3584, 512, 1280, 6912, 128, 20736, 8320, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 1550, 0, 1, 0, 0, 1401, 0, 0, 0, 0, 0, 3, 3853, 1, 421, 0, 730, 4017, 2885, 4095, 212, 0, 0, 4094, 515, 0, 185, 0, 0, 64, 0, 6, 2609, 0, 4052, 0, 0, 354, 4096, 3717, 2, 0, 0, 3142, 4077, 0, 12, 0, 4059, 617, 654, 3726, 0, 0, 0, 2063, 3964, 1942, 0, 1359, 128, 0, 930] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[481, 0, 0, 0, 0, 1593, 0, 4096, 3847, 0, 0, 0, 4092, 0, 0, 2, 2, 1615, 459, 164, 0, 3953, 3967, 0, 0, 659, 4095, 134, 13, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 4041, 0, 0, 0, 3794, 3116, 4095, 4096, 2777, 95, 4096, 0, 0, 0, 0, 64, 0, 404, 3873, 0, 38, 1695] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3938, 113, 0, 47, 0, 0, 16, 4096, 2478, 27, 0, 533, 3576, 4096, 3991, 0, 0, 0, 0, 30, 3837, 0, 4086, 2, 0, 31, 0, 100, 369, 1783, 0, 0, 0, 202, 0, 0, 4, 2, 191, 3467, 4024, 7, 3720, 101, 346, 0, 3721, 3948, 4089, 3860, 0, 20, 4060, 0, 0, 0, 0, 4, 621, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 0, 3823, 0, 0, 0, 4091, 3981, 0, 1942, 4096, 5, 350, 48, 0, 0, 2714, 1311, 0, 836, 235, 16, 808, 290, 0, 0, 0, 51, 2111, 1, 0, 610, 4096, 0, 0, 0, 3875, 12, 52, 4096, 30, 3, 3058, 8, 4032, 3303, 0, 3731, 0, 0, 0, 3708, 0, 0, 0, 0, 20, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3114, 0, 7, 0, 2007, 4094, 0, 0, 0, 2974, 0, 0, 308, 16, 4095, 0, 0, 4, 0, 4096, 2, 263, 694, 0, 3947, 10, 0, 0, 864, 94, 0, 675, 1727, 9, 0, 4090, 1946, 0, 0, 4092, 0, 0, 4, 1, 0, 4095, 3920, 3697, 334, 4053, 1, 2, 3266, 0, 0, 311, 0, 0, 0, 12, 3055, 0, 0, 3657] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4085, 2820, 0, 0, 1221, 4096, 0, 7, 149, 4096, 910, 601, 4065, 8, 3994, 0, 0, 155, 0, 4079, 0, 546, 0, 0, 0, 265, 4, 0, 34, 77, 0, 17, 1745, 0, 0, 4052, 0, 0, 0, 4096, 0, 0, 3402, 0, 0, 0, 4092, 0, 4, 4096, 1309, 6, 3083, 0, 4090, 4, 0, 2613, 0, 4, 1000, 0, 0, 711] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 3837, 19, 847, 0, 3569, 0, 0, 500, 4096, 0, 0, 0, 0, 3679, 0, 0, 0, 4096, 36, 599, 146, 2231, 1773, 4015, 0, 171, 0, 1868, 0, 0, 1, 0, 4094, 245, 4049, 0, 160, 28, 4092, 3269, 0, 117, 0, 593, 2103, 0, 0, 19, 4000, 3294, 0, 3361, 53, 415, 8, 3, 0, 0, 0, 4056, 0, 0, 94] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[8, 4014, 507, 0, 0, 0, 0, 0, 0, 3773, 636, 0, 3991, 4096, 7, 3626, 0, 0, 3268, 4083, 4094, 0, 0, 60, 3, 1, 1316, 0, 0, 0, 2, 0, 1695, 2217, 0, 2019, 0, 0, 483, 3943, 3584, 7, 4096, 338, 33, 0, 4096, 823, 134, 0, 0, 0, 2586, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 1874, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1550, 0, 1, 0, 0, 1401, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3853, 1, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 4017, 2885, 4095, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 515, 0, 185, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2609, 0, 4052, 0, 0, 354, 3196, 0, 0, 0, 0, 0, 0, 0, 0, 3717, 2, 0, 0, 3142, 4077, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 617, 654, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 3964, 1942, 0, 1359, 128, 0, 930, 2447, 699, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3153, 4278, 11939, 4858, 10217, 10950, 6609, 13532] avg=8192.0000 max=13532 max_rank=7 min=3153 min_rank=0 imbalance_ratio=1.651855 max_load_reduction=52004 max_load_reduction_pct=0.793518 moved_to_echo=41308 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3114, 0, 7, 0, 2007, 4094, 0, 0, 41, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 16, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 2, 263, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 10, 0, 0, 864, 94, 0, 675, 3920, 2687, 0, 0, 0, 0, 0, 0, 1727, 9, 0, 4090, 1946, 0, 0, 3193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 4095, 0, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 334, 4053, 1, 2, 1122, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3055, 0, 0, 3657, 2144, 698, 287, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9464, 4378, 5059, 12197, 10965, 7797, 5823, 9853] avg=8192.0000 max=12197 max_rank=3 min=4378 min_rank=1 imbalance_ratio=1.488892 max_load_reduction=53339 max_load_reduction_pct=0.813889 moved_to_echo=34438 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[481, 0, 0, 0, 0, 1593, 0, 4096, 549, 427, 201, 0, 0, 0, 0, 0, 3847, 0, 0, 0, 3543, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1615, 459, 164, 0, 3953, 3967, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 659, 4095, 134, 13, 0, 0, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3591, 0, 0, 0, 0, 3116, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 2777, 95, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 404, 3873, 0, 38, 1695, 2690, 699, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7347, 7392, 10183, 8695, 3269, 6718, 12469, 9463] avg=8192.0000 max=12469 max_rank=6 min=3269 min_rank=4 imbalance_ratio=1.522095 max_load_reduction=53067 max_load_reduction_pct=0.809738 moved_to_echo=31919 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3938, 113, 0, 47, 0, 0, 16, 4096, 479, 393, 170, 0, 0, 0, 0, 0, 2478, 0, 0, 533, 3097, 4096, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3837, 0, 4086, 2, 21, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 100, 369, 1783, 0, 0, 3721, 25, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 4, 2, 191, 2706, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 7, 3306, 101, 346, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3860, 0, 20, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 621, 0, 0, 0, 0, 0, 2666, 591, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9252, 14195, 7976, 6029, 3105, 11732, 9363, 3884] avg=8192.0000 max=14195 max_rank=1 min=3105 min_rank=4 imbalance_ratio=1.732788 max_load_reduction=51341 max_load_reduction_pct=0.783401 moved_to_echo=28084 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4085, 2820, 0, 0, 1221, 4096, 0, 7, 545, 359, 201, 0, 0, 0, 0, 0, 149, 0, 910, 601, 3520, 8, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 4079, 0, 546, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 265, 4, 0, 34, 77, 0, 17, 4092, 3702, 0, 0, 0, 0, 0, 0, 1745, 0, 0, 4052, 0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 1309, 6, 1059, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 0, 4, 1000, 0, 0, 711, 2024, 699, 394, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13334, 9182, 4799, 8191, 8993, 3024, 10568, 7445] avg=8192.0000 max=13334 max_rank=0 min=3024 min_rank=5 imbalance_ratio=1.627686 max_load_reduction=52202 max_load_reduction_pct=0.796539 moved_to_echo=30030 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[8, 4014, 507, 0, 0, 0, 0, 0, 538, 436, 195, 0, 0, 0, 0, 0, 0, 0, 636, 0, 3453, 4096, 7, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3268, 4083, 4094, 0, 0, 60, 27, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1316, 0, 0, 0, 2, 0, 4096, 3410, 0, 0, 0, 0, 0, 0, 1695, 2217, 0, 2019, 0, 0, 483, 3073, 0, 0, 0, 0, 0, 0, 0, 0, 3584, 7, 3633, 338, 33, 0, 0, 823, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 1874, 0, 1701, 675, 363, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5698, 11818, 11532, 8828, 9487, 8418, 1019, 8736] avg=8192.0000 max=11818 max_rank=1 min=1019 min_rank=6 imbalance_ratio=1.442627 max_load_reduction=53718 max_load_reduction_pct=0.819672 moved_to_echo=27660 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 0, 3823, 0, 0, 260, 433, 190, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 1682, 4096, 5, 350, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 2714, 1311, 0, 836, 235, 23, 0, 0, 0, 0, 0, 0, 0, 16, 808, 290, 0, 0, 0, 51, 2111, 8, 3698, 0, 0, 0, 0, 0, 0, 1, 0, 610, 4096, 0, 0, 0, 3024, 0, 0, 0, 0, 0, 0, 0, 0, 12, 52, 3640, 30, 3, 3058, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 0, 3731, 0, 0, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 0, 0, 0, 0, 661, 393, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8803, 10114, 5167, 6982, 7731, 10827, 10742, 5170] avg=8192.0000 max=10827 max_rank=5 min=5167 min_rank=2 imbalance_ratio=1.321655 max_load_reduction=54709 max_load_reduction_pct=0.834793 moved_to_echo=34470 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3837, 19, 847, 0, 3569, 0, 0, 0, 12, 201, 0, 0, 0, 0, 0, 500, 0, 0, 0, 0, 0, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 36, 599, 146, 2231, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 171, 0, 1868, 0, 0, 1, 0, 3701, 0, 0, 0, 0, 0, 0, 0, 4094, 245, 4049, 0, 160, 28, 3193, 0, 0, 0, 0, 0, 0, 0, 0, 3269, 0, 105, 0, 593, 2103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4000, 3294, 0, 1154, 53, 415, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4056, 0, 0, 94, 2207, 698, 395, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8485, 4179, 8881, 9756, 11769, 6070, 8943, 7453] avg=8192.0000 max=11769 max_rank=4 min=4179 min_rank=1 imbalance_ratio=1.436646 max_load_reduction=53767 max_load_reduction_pct=0.820419 moved_to_echo=34235 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2176, 6656, 2688, 512, 21632, 128, 1920, 7168, 16000, 5504, 1920, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12032, 24192, 11776, 896, 8320, 128, 8320, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[14720, 128, 17408, 512, 4224, 13440, 0, 15744, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[15744, 12416, 640, 896, 3328, 17280, 1536, 8320, 2432, 2176, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12160, 2304, 5888, 512, 3200, 2048, 128, 2816, 19712, 17280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7040, 0, 5632, 1152, 15616, 16256, 15872, 4480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5248, 9216, 896, 22400, 2048, 256, 1152, 24832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 2560, 11904, 18176, 13952, 5120, 11904, 2176, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True + [2026-08-30 04:59:25] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 40651.8 | throughput per GPU (TFLOP/s/GPU): 8.2 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102348E+01 | loss scale: 1.0 | grad norm: 44.163 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 23.22 + +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 63755.8447265625 | max allocated: 101848.72216796875 | reserved: 103816.0 | max reserved: 103816.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[192, 3567, 792, 290, 1657, 376, 678, 1470, 1074, 310, 2243, 293, 731, 867, 237, 363, 172, 420, 834, 1151, 1065, 286, 2767, 2217, 672, 1449, 998, 1070, 858, 263, 1628, 1672, 2489, 460, 597, 639, 95, 2019, 1846, 314, 2589, 594, 1729, 1077, 1219, 1642, 930, 681, 306, 543, 497, 2182, 980, 726, 573, 113, 586, 337, 1051, 1848, 1255, 719, 661, 1577] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[92, 2934, 669, 292, 1315, 338, 829, 700, 1045, 242, 1223, 132, 1306, 624, 142, 368, 130, 490, 709, 738, 1370, 160, 3691, 1927, 235, 1372, 1072, 864, 831, 227, 1698, 2434, 3237, 1076, 318, 559, 123, 2726, 2162, 923, 2889, 131, 1087, 708, 1925, 1676, 1649, 1093, 299, 299, 279, 2491, 565, 337, 352, 86, 803, 461, 989, 1819, 1466, 669, 631, 1509] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[94, 2960, 494, 287, 1582, 186, 1189, 1193, 1175, 201, 1612, 172, 777, 754, 120, 347, 105, 907, 926, 790, 880, 104, 2764, 1560, 511, 1553, 859, 1076, 1614, 455, 1532, 2035, 2843, 444, 748, 558, 78, 2832, 2236, 810, 2557, 335, 1316, 826, 1575, 1841, 1073, 1027, 315, 567, 229, 2431, 1266, 549, 627, 92, 364, 195, 1051, 2228, 1293, 541, 792, 1083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[98, 2728, 762, 233, 2151, 681, 982, 1414, 1055, 167, 1177, 453, 1029, 783, 183, 451, 159, 559, 1336, 1233, 932, 228, 2389, 1626, 427, 1331, 1173, 1142, 942, 327, 1334, 1730, 3402, 601, 275, 793, 129, 2395, 2452, 608, 2171, 240, 1780, 871, 1318, 1182, 1297, 1559, 432, 222, 377, 2471, 853, 195, 704, 157, 723, 192, 1051, 1670, 1608, 649, 663, 1281] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[170, 2787, 479, 407, 998, 359, 916, 701, 1211, 245, 1622, 453, 1072, 594, 300, 350, 132, 613, 751, 1014, 1099, 664, 2947, 2039, 484, 1539, 1448, 794, 1458, 309, 1661, 1989, 3065, 1055, 446, 517, 208, 2165, 1931, 802, 2751, 540, 1367, 790, 1431, 1375, 1180, 842, 323, 638, 445, 2000, 695, 479, 589, 171, 464, 649, 1000, 1904, 1714, 470, 827, 1098] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[46, 2711, 349, 203, 1607, 290, 1038, 881, 1128, 164, 2110, 263, 522, 877, 241, 247, 86, 488, 1287, 489, 1222, 255, 3015, 2041, 529, 1102, 1053, 850, 1736, 418, 2547, 2291, 3109, 459, 591, 373, 79, 2499, 2175, 358, 2755, 570, 1512, 751, 1540, 1917, 828, 543, 216, 296, 229, 1724, 1041, 234, 353, 55, 508, 344, 1095, 2208, 1703, 365, 1795, 1225] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[114, 2567, 662, 333, 1497, 474, 856, 625, 843, 120, 1458, 214, 913, 862, 285, 377, 199, 599, 749, 1018, 897, 424, 2901, 1641, 322, 892, 1342, 884, 1087, 215, 1744, 2432, 3315, 639, 176, 452, 588, 2583, 2744, 420, 2608, 713, 1304, 929, 1228, 1641, 1084, 1799, 266, 144, 365, 2004, 801, 419, 715, 176, 1066, 397, 1399, 2364, 1463, 528, 592, 1068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[161, 2852, 586, 279, 1608, 631, 974, 1570, 1303, 130, 1693, 169, 787, 890, 302, 464, 192, 548, 582, 822, 1233, 339, 2869, 1544, 346, 1151, 1059, 1031, 957, 263, 1539, 1882, 2898, 606, 322, 550, 230, 2612, 2391, 629, 2608, 367, 1197, 570, 1391, 1596, 976, 896, 219, 316, 321, 2652, 979, 645, 763, 82, 798, 382, 1697, 1786, 1256, 662, 811, 1572] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[192, 3567, 792, 290, 1657, 376, 678, 1470, 168, 135, 0, 0, 0, 0, 0, 0, 1074, 310, 2243, 293, 731, 867, 237, 363, 1208, 731, 313, 0, 0, 0, 0, 0, 172, 420, 834, 1151, 1065, 286, 2319, 2217, 0, 0, 0, 0, 0, 0, 0, 0, 672, 1449, 998, 1070, 858, 263, 1628, 941, 0, 0, 0, 0, 0, 0, 0, 0, 830, 460, 597, 639, 95, 2019, 1846, 314, 0, 0, 0, 0, 0, 0, 0, 0, 282, 594, 1729, 1077, 1219, 1642, 930, 681, 0, 0, 0, 0, 0, 0, 0, 0, 306, 543, 497, 2182, 980, 726, 573, 113, 2307, 451, 0, 0, 0, 0, 0, 0, 586, 337, 1051, 1680, 1255, 719, 661, 1577, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9325, 8370, 8464, 7879, 6800, 8154, 8678, 7866] avg=8192.0000 max=9325 max_rank=0 min=6800 min_rank=4 imbalance_ratio=1.138306 max_load_reduction=56211 max_load_reduction_pct=0.857712 moved_to_echo=31498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[170, 2787, 479, 407, 998, 359, 916, 701, 173, 144, 0, 0, 0, 0, 0, 0, 1211, 245, 1622, 453, 1072, 594, 300, 350, 1487, 870, 334, 0, 0, 0, 0, 0, 132, 613, 751, 1014, 1099, 664, 2469, 2039, 0, 0, 0, 0, 0, 0, 0, 0, 484, 1539, 1448, 794, 1458, 309, 1661, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 1055, 446, 517, 208, 2165, 1931, 802, 0, 0, 0, 0, 0, 0, 0, 0, 300, 540, 1367, 790, 1431, 1375, 1180, 842, 0, 0, 0, 0, 0, 0, 0, 0, 323, 638, 445, 2000, 695, 479, 589, 171, 2451, 555, 0, 0, 0, 0, 0, 0, 464, 649, 1000, 1731, 1714, 470, 827, 1098, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7134, 8538, 8781, 8812, 8147, 7825, 8346, 7953] avg=8192.0000 max=8812 max_rank=3 min=7134 min_rank=0 imbalance_ratio=1.075684 max_load_reduction=56724 max_load_reduction_pct=0.865540 moved_to_echo=32271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[92, 2934, 669, 292, 1315, 338, 829, 700, 169, 183, 0, 0, 0, 0, 0, 0, 1045, 242, 1223, 132, 1306, 624, 142, 368, 1575, 1067, 423, 0, 0, 0, 0, 0, 130, 490, 709, 738, 1370, 160, 3085, 1927, 0, 0, 0, 0, 0, 0, 0, 0, 235, 1372, 1072, 864, 831, 227, 1698, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 1076, 318, 559, 123, 2726, 2162, 923, 0, 0, 0, 0, 0, 0, 0, 0, 312, 131, 1087, 708, 1925, 1676, 1649, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 299, 299, 279, 2491, 565, 337, 352, 86, 2577, 591, 0, 0, 0, 0, 0, 0, 803, 461, 989, 1650, 1466, 669, 631, 1509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7521, 8147, 8609, 7666, 8958, 8581, 7876, 8178] avg=8192.0000 max=8958 max_rank=4 min=7521 min_rank=0 imbalance_ratio=1.093506 max_load_reduction=56578 max_load_reduction_pct=0.863312 moved_to_echo=33593 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[98, 2728, 762, 233, 2151, 681, 982, 1414, 151, 116, 0, 0, 0, 0, 0, 0, 1055, 167, 1177, 453, 1029, 783, 183, 451, 1651, 756, 270, 0, 0, 0, 0, 0, 159, 559, 1336, 1233, 932, 228, 2003, 1626, 0, 0, 0, 0, 0, 0, 0, 0, 427, 1331, 1173, 1142, 942, 327, 1334, 974, 0, 0, 0, 0, 0, 0, 0, 0, 1135, 601, 275, 793, 129, 2395, 2452, 608, 0, 0, 0, 0, 0, 0, 0, 0, 237, 240, 1780, 871, 1318, 1182, 1297, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 432, 222, 377, 2471, 853, 195, 704, 157, 1934, 616, 0, 0, 0, 0, 0, 0, 723, 192, 1051, 1519, 1608, 649, 663, 1281, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9316, 7975, 8076, 7650, 8388, 8484, 7961, 7686] avg=8192.0000 max=9316 max_rank=0 min=7650 min_rank=3 imbalance_ratio=1.137207 max_load_reduction=56220 max_load_reduction_pct=0.857849 moved_to_echo=32519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[94, 2960, 494, 287, 1582, 186, 1189, 1193, 202, 135, 0, 0, 0, 0, 0, 0, 1175, 201, 1612, 172, 777, 754, 120, 347, 1379, 890, 313, 0, 0, 0, 0, 0, 105, 907, 926, 790, 880, 104, 2316, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 511, 1553, 859, 1076, 1614, 455, 1532, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 949, 444, 748, 558, 78, 2832, 2236, 810, 0, 0, 0, 0, 0, 0, 0, 0, 279, 335, 1316, 826, 1575, 1841, 1073, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 315, 567, 229, 2431, 1266, 549, 627, 92, 2278, 515, 0, 0, 0, 0, 0, 0, 364, 195, 1051, 2026, 1293, 541, 792, 1083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8322, 7740, 7588, 8745, 8655, 8272, 8869, 7345] avg=8192.0000 max=8869 max_rank=6 min=7345 min_rank=7 imbalance_ratio=1.082642 max_load_reduction=56667 max_load_reduction_pct=0.864670 moved_to_echo=33141 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[46, 2711, 349, 203, 1607, 290, 1038, 881, 200, 147, 0, 0, 0, 0, 0, 0, 1128, 164, 2110, 263, 522, 877, 241, 247, 1508, 1002, 341, 0, 0, 0, 0, 0, 86, 488, 1287, 489, 1222, 255, 2527, 2041, 0, 0, 0, 0, 0, 0, 0, 0, 529, 1102, 1053, 850, 1736, 418, 2547, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 459, 591, 373, 79, 2499, 2175, 358, 0, 0, 0, 0, 0, 0, 0, 0, 300, 570, 1512, 751, 1540, 1917, 828, 543, 0, 0, 0, 0, 0, 0, 0, 0, 216, 296, 229, 1724, 1041, 234, 353, 55, 2455, 563, 0, 0, 0, 0, 0, 0, 508, 344, 1095, 2008, 1703, 365, 1795, 1225, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7472, 8403, 8395, 9524, 7572, 7961, 7166, 9043] avg=8192.0000 max=9524 max_rank=3 min=7166 min_rank=6 imbalance_ratio=1.162598 max_load_reduction=56012 max_load_reduction_pct=0.854675 moved_to_echo=31742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[114, 2567, 662, 333, 1497, 474, 856, 625, 214, 141, 0, 0, 0, 0, 0, 0, 843, 120, 1458, 214, 913, 862, 285, 377, 1608, 1063, 328, 0, 0, 0, 0, 0, 199, 599, 749, 1018, 897, 424, 2432, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 322, 892, 1342, 884, 1087, 215, 1744, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 639, 176, 452, 588, 2583, 2744, 420, 0, 0, 0, 0, 0, 0, 0, 0, 284, 713, 1304, 929, 1228, 1641, 1084, 1799, 0, 0, 0, 0, 0, 0, 0, 0, 266, 144, 365, 2004, 801, 419, 715, 176, 2324, 600, 0, 0, 0, 0, 0, 0, 1066, 397, 1399, 2150, 1463, 528, 592, 1068, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7483, 8071, 7959, 7855, 8709, 8982, 7814, 8663] avg=8192.0000 max=8982 max_rank=5 min=7483 min_rank=0 imbalance_ratio=1.096436 max_load_reduction=56554 max_load_reduction_pct=0.862946 moved_to_echo=34168 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[161, 2852, 586, 279, 1608, 631, 974, 1570, 162, 140, 0, 0, 0, 0, 0, 0, 1303, 130, 1693, 169, 787, 890, 302, 464, 1406, 823, 325, 0, 0, 0, 0, 0, 192, 548, 582, 822, 1233, 339, 2404, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 346, 1151, 1059, 1031, 957, 263, 1539, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 967, 606, 322, 550, 230, 2612, 2391, 629, 0, 0, 0, 0, 0, 0, 0, 0, 284, 367, 1197, 570, 1391, 1596, 976, 896, 0, 0, 0, 0, 0, 0, 0, 0, 219, 316, 321, 2652, 979, 645, 763, 82, 2324, 525, 0, 0, 0, 0, 0, 0, 798, 382, 1697, 1624, 1256, 662, 811, 1572, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8963, 8292, 7664, 7405, 8307, 7277, 8826, 8802] avg=8192.0000 max=8963 max_rank=0 min=7277 min_rank=5 imbalance_ratio=1.094116 max_load_reduction=56573 max_load_reduction_pct=0.863235 moved_to_echo=33212 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 23168, 4864, 2432, 12416, 3456, 7552, 8576, 1536, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8192, 5376, 3584, 4480, 1536, 19840, 18048, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[5376, 3072, 9344, 14464, 11776, 4608, 6784, 10496, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8960, 1664, 13184, 2176, 7168, 6272, 1920, 3072, 11904, 7296, 2688, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3584, 10496, 9088, 7808, 9600, 2560, 13696, 9344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2304, 3584, 11392, 6528, 11648, 12928, 9088, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1280, 4736, 7296, 7296, 8704, 2560, 19584, 14720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2432, 3072, 2816, 18048, 7296, 3584, 4736, 1024, 18688, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4068, 2, 1224, 3860, 2, 18, 4012, 4, 0, 3625, 3, 10, 18, 68, 421, 4, 3, 4061, 1371, 22, 0, 2931, 0, 338, 4093, 14, 104, 4045, 4043, 4, 0, 228, 1829, 467, 0, 4064, 667, 1457, 0, 14, 0, 0, 0, 1409, 230, 5, 269, 4, 2455, 305, 4068, 0, 1718, 19, 82, 37, 0, 6, 17, 42, 4, 17, 4096, 3659] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[3676, 199, 701, 3104, 523, 1414, 3886, 0, 130, 657, 80, 0, 0, 4, 710, 12, 1, 4035, 286, 2381, 1053, 1330, 1, 1732, 3089, 14, 394, 4067, 4086, 0, 138, 199, 8, 972, 0, 4092, 544, 1561, 0, 55, 552, 0, 212, 3604, 0, 60, 126, 0, 3812, 1037, 3101, 18, 89, 0, 0, 18, 300, 7, 162, 1255, 0, 1586, 4068, 395] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3894, 0, 291, 2524, 65, 82, 4096, 2, 3, 3091, 0, 1, 2, 11, 404, 4, 0, 4053, 11, 37, 5, 2687, 13, 294, 2933, 134, 245, 2340, 4084, 19, 0, 201, 1372, 446, 0, 4020, 224, 3872, 18, 83, 2699, 3, 90, 2188, 84, 1798, 1124, 218, 3326, 3563, 4095, 8, 291, 0, 0, 25, 141, 2, 228, 3, 0, 29, 4042, 18] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4029, 16, 15, 3331, 6, 2935, 4086, 0, 3, 2118, 0, 1, 0, 0, 485, 14, 1, 4080, 1536, 213, 23, 171, 0, 1096, 3524, 39, 1, 3636, 4094, 2, 1, 40, 17, 51, 0, 4095, 2719, 4067, 0, 19, 802, 45, 2, 963, 0, 98, 1869, 10, 1727, 3998, 3415, 0, 1093, 0, 0, 31, 2, 0, 16, 20, 0, 713, 4096, 172] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[3381, 16, 20, 2003, 72, 42, 3175, 18, 477, 4054, 102, 19, 0, 579, 2, 184, 90, 4091, 1263, 3453, 145, 3, 0, 372, 627, 0, 0, 3570, 3572, 485, 0, 529, 2, 2393, 57, 4061, 32, 4066, 46, 30, 2975, 52, 0, 184, 360, 3173, 210, 0, 4048, 2835, 2628, 6, 4, 0, 0, 420, 36, 0, 5, 50, 0, 1317, 3897, 305] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[3890, 0, 0, 4089, 889, 1654, 3895, 0, 0, 4073, 0, 0, 0, 2, 5, 1, 15, 3810, 3884, 1404, 0, 2399, 0, 1678, 4062, 0, 0, 3307, 3981, 122, 0, 13, 56, 4, 0, 1309, 3830, 3817, 0, 35, 39, 0, 94, 1901, 0, 84, 71, 87, 3642, 260, 1558, 0, 11, 0, 0, 0, 0, 0, 115, 110, 0, 951, 4051, 338] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[3402, 74, 1340, 2784, 14, 816, 3473, 0, 0, 2792, 0, 1, 0, 114, 167, 1, 1, 4060, 457, 2174, 215, 799, 0, 921, 3801, 1, 4, 2704, 3988, 31, 37, 77, 4057, 692, 1, 1107, 3936, 1344, 0, 27, 388, 0, 404, 3832, 833, 104, 65, 6, 3244, 678, 4051, 8, 2050, 55, 0, 113, 3, 0, 53, 0, 41, 5, 3881, 310] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[3916, 4, 62, 3049, 6, 88, 3892, 0, 12, 3342, 0, 0, 111, 329, 129, 1, 3, 3741, 1845, 50, 0, 2116, 0, 75, 4032, 228, 4, 4061, 4057, 0, 0, 73, 2243, 265, 16, 3501, 158, 3685, 0, 87, 495, 0, 60, 738, 1, 723, 32, 0, 2689, 3634, 2490, 28, 474, 0, 2, 114, 0, 0, 1116, 227, 0, 3594, 3797, 141] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3424, 199, 701, 3104, 523, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 657, 80, 0, 0, 4, 710, 12, 3886, 851, 0, 0, 0, 0, 0, 0, 1, 3114, 286, 2381, 1053, 1330, 1, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 3089, 14, 394, 4067, 1059, 0, 138, 199, 0, 0, 0, 0, 0, 0, 0, 0, 8, 972, 0, 2213, 544, 1561, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 552, 0, 212, 3604, 0, 60, 126, 0, 2176, 1879, 135, 0, 0, 0, 0, 0, 3812, 1037, 2146, 18, 89, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 300, 7, 162, 1255, 0, 1586, 4068, 395, 921, 820, 252, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9365, 6330, 9898, 8960, 5353, 8744, 7120, 9766] avg=8192.0000 max=9898 max_rank=2 min=5353 min_rank=4 imbalance_ratio=1.208252 max_load_reduction=55638 max_load_reduction_pct=0.848969 moved_to_echo=30983 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3789, 2, 1224, 3860, 2, 18, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3625, 3, 10, 18, 68, 421, 4, 4012, 842, 0, 0, 0, 0, 0, 0, 3, 3134, 1371, 22, 0, 2931, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 14, 104, 4045, 1048, 4, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 467, 0, 2198, 667, 1457, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 230, 5, 269, 4, 2153, 1866, 177, 0, 0, 0, 0, 0, 2455, 305, 2815, 0, 1718, 19, 82, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 17, 42, 4, 17, 4096, 3659, 927, 1076, 279, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8899, 9003, 7799, 9536, 6632, 6113, 7431, 10123] avg=8192.0000 max=10123 max_rank=7 min=6113 min_rank=5 imbalance_ratio=1.235718 max_load_reduction=55413 max_load_reduction_pct=0.845535 moved_to_echo=30299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3627, 0, 291, 2524, 65, 82, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3091, 0, 1, 2, 11, 404, 4, 4096, 850, 0, 0, 0, 0, 0, 0, 0, 3128, 11, 37, 5, 2687, 13, 294, 0, 0, 0, 0, 0, 0, 0, 0, 2933, 134, 245, 2340, 1059, 19, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 1372, 446, 0, 2174, 224, 3872, 18, 83, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 3, 90, 2188, 84, 1798, 1124, 218, 2175, 1846, 179, 0, 0, 0, 0, 0, 3326, 3563, 2832, 8, 291, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 141, 2, 228, 3, 0, 29, 4042, 18, 925, 1084, 267, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6591, 8462, 6175, 6931, 8189, 12404, 10045, 6739] avg=8192.0000 max=12404 max_rank=5 min=6175 min_rank=2 imbalance_ratio=1.514160 max_load_reduction=53132 max_load_reduction_pct=0.810730 moved_to_echo=37377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3753, 16, 15, 3331, 6, 2935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2118, 0, 1, 0, 0, 485, 14, 4086, 852, 0, 0, 0, 0, 0, 0, 1, 3149, 1536, 213, 23, 171, 0, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 39, 1, 3636, 1062, 2, 1, 40, 0, 0, 0, 0, 0, 0, 0, 0, 17, 51, 0, 2215, 2719, 4067, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 802, 45, 2, 963, 0, 98, 1869, 10, 2180, 1880, 149, 0, 0, 0, 0, 0, 1727, 3998, 2362, 0, 1093, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 16, 20, 0, 713, 4096, 172, 931, 904, 276, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10056, 7559, 6189, 8305, 9088, 7998, 9211, 7130] avg=8192.0000 max=10056 max_rank=0 min=6189 min_rank=2 imbalance_ratio=1.227539 max_load_reduction=55480 max_load_reduction_pct=0.846558 moved_to_echo=33427 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3149, 16, 20, 2003, 72, 42, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 477, 4054, 102, 19, 0, 579, 2, 184, 3175, 743, 0, 0, 0, 0, 0, 0, 90, 3157, 1263, 3453, 145, 3, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 3570, 927, 485, 0, 529, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2393, 57, 2197, 32, 4066, 46, 30, 0, 0, 0, 0, 0, 0, 0, 0, 2975, 52, 0, 184, 360, 3173, 210, 0, 1902, 1864, 114, 0, 0, 0, 0, 0, 4048, 2835, 1819, 6, 4, 0, 0, 420, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 5, 50, 0, 1317, 3897, 305, 934, 695, 232, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5320, 9335, 8483, 6138, 8823, 10834, 9132, 7471] avg=8192.0000 max=10834 max_rank=5 min=5320 min_rank=0 imbalance_ratio=1.322510 max_load_reduction=54702 max_load_reduction_pct=0.834686 moved_to_echo=36260 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3623, 0, 0, 4089, 889, 1654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 2, 5, 1, 3895, 829, 0, 0, 0, 0, 0, 0, 15, 2940, 3884, 1404, 0, 2399, 0, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 3307, 1032, 122, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4, 0, 708, 3830, 3817, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 94, 1901, 0, 84, 71, 87, 2120, 601, 68, 0, 0, 0, 0, 0, 3642, 260, 1078, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 110, 0, 951, 4051, 338, 870, 412, 267, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10255, 8805, 12320, 8536, 8450, 5065, 4991, 7114] avg=8192.0000 max=12320 max_rank=2 min=4991 min_rank=6 imbalance_ratio=1.503906 max_load_reduction=53216 max_load_reduction_pct=0.812012 moved_to_echo=25620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3169, 74, 1340, 2784, 14, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2792, 0, 1, 0, 114, 167, 1, 3473, 830, 0, 0, 0, 0, 0, 0, 1, 3133, 457, 2174, 215, 799, 0, 921, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 1, 4, 2704, 1034, 31, 37, 77, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 692, 1, 599, 3936, 1344, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 388, 0, 404, 3832, 833, 104, 65, 6, 2124, 508, 177, 0, 0, 0, 0, 0, 3244, 678, 2802, 8, 2050, 55, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 53, 0, 41, 5, 3881, 310, 927, 1072, 233, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 7378, 7700, 7689, 10656, 8441, 8950, 6525] avg=8192.0000 max=10656 max_rank=4 min=6525 min_rank=7 imbalance_ratio=1.300781 max_load_reduction=54880 max_load_reduction_pct=0.837402 moved_to_echo=34572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3644, 4, 62, 3049, 6, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3342, 0, 0, 111, 329, 129, 1, 3892, 848, 0, 0, 0, 0, 0, 0, 3, 2883, 1845, 50, 0, 2116, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 228, 4, 4061, 1043, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 2243, 265, 16, 1891, 158, 3685, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 495, 0, 60, 738, 1, 723, 32, 0, 2166, 1610, 112, 0, 0, 0, 0, 0, 2689, 3634, 1715, 28, 474, 0, 2, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 227, 0, 3594, 3797, 141, 858, 663, 272, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6853, 8664, 6972, 9441, 8345, 5937, 8656, 10668] avg=8192.0000 max=10668 max_rank=7 min=5937 min_rank=5 imbalance_ratio=1.302246 max_load_reduction=54868 max_load_reduction_pct=0.837219 moved_to_echo=33606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 24704, 10752, 9856, 1536, 12544, 128, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8064, 128, 896, 14848, 1536, 6144, 3840, 384, 17024, 12160, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[9600, 5376, 128, 14208, 12160, 23936, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26240, 512, 768, 27776, 8320, 768, 256, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[24960, 16384, 17664, 128, 5760, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[28288, 384, 3712, 24832, 1664, 7168, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[512, 128, 1792, 1792, 128, 8320, 32000, 5376, 7296, 6784, 2176, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 23808, 256, 128, 256, 1152, 2432, 256, 30592, 6656, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4081, 0, 3929, 37, 1, 18, 2790, 3981, 1, 3, 3603, 1, 0, 0, 16, 2, 2434, 4, 13, 21, 0, 1165, 13, 3426, 10, 570, 514, 1601, 6, 376, 3999, 0, 47, 8, 4092, 113, 0, 3703, 0, 37, 596, 4088, 49, 159, 0, 1, 4096, 53, 13, 0, 0, 317, 3810, 0, 0, 0, 0, 4088, 0, 3641, 3966, 0, 0, 44] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4096, 601, 4084, 0, 73, 0, 1480, 4094, 18, 0, 328, 2, 8, 0, 14, 10, 55, 0, 0, 9, 1, 1109, 0, 3445, 18, 102, 0, 383, 31, 4063, 3985, 8, 2, 1, 4084, 1317, 420, 3118, 0, 312, 4096, 3989, 10, 1569, 3, 0, 698, 271, 595, 0, 30, 2723, 194, 511, 4052, 0, 0, 3476, 0, 2349, 3616, 0, 0, 83] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3406, 5, 4045, 447, 0, 2, 2465, 3956, 25, 0, 83, 0, 0, 52, 1, 4, 202, 0, 0, 13, 0, 4095, 0, 4061, 0, 71, 0, 6, 315, 4011, 4006, 3, 8, 0, 4090, 2, 0, 201, 113, 94, 1313, 3402, 12, 3914, 0, 2, 3987, 47, 0, 135, 0, 1178, 3638, 0, 127, 15, 96, 4096, 0, 3750, 4041, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3895, 342, 4095, 0, 277, 0, 1970, 4096, 4095, 0, 3901, 0, 556, 11, 0, 20, 2, 0, 0, 0, 1, 17, 31, 3529, 76, 0, 1, 1115, 1475, 1331, 3977, 18, 748, 23, 3719, 192, 0, 602, 0, 0, 3400, 1936, 0, 2388, 0, 0, 4052, 262, 0, 2, 0, 4020, 1, 0, 1436, 0, 0, 3717, 8, 62, 4095, 0, 0, 42] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3311, 732, 4096, 388, 0, 27, 4064, 3882, 7, 2, 1748, 52, 0, 79, 267, 0, 26, 7, 136, 117, 2, 3910, 0, 2938, 203, 129, 1, 158, 118, 57, 4015, 6, 3, 19, 4052, 31, 0, 13, 4, 13, 3889, 1894, 0, 4014, 0, 0, 4078, 908, 35, 0, 43, 623, 4061, 2, 3, 133, 9, 4092, 199, 2304, 4083, 273, 276, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4095, 2027, 4096, 11, 2, 0, 7, 4075, 31, 2, 754, 13, 0, 5, 0, 8, 847, 1, 1, 0, 0, 2943, 4, 4003, 0, 0, 0, 89, 148, 1049, 4067, 8, 0, 2, 4096, 292, 0, 626, 2, 17, 4092, 1723, 0, 4036, 0, 0, 2855, 1516, 0, 1, 0, 3984, 21, 0, 4049, 3, 0, 3976, 0, 1863, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3261, 1, 4081, 0, 48, 31, 1, 2010, 2281, 5, 0, 6, 66, 196, 2619, 2716, 157, 7, 0, 548, 10, 3445, 0, 2382, 20, 12, 1224, 10, 47, 4002, 3916, 1215, 85, 0, 4047, 3776, 0, 4067, 0, 34, 3900, 40, 12, 1633, 161, 605, 167, 4057, 0, 0, 0, 22, 4064, 36, 0, 2, 0, 4096, 15, 10, 211, 0, 175, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3474, 1263, 1668, 0, 2950, 0, 3859, 4089, 4067, 0, 4088, 0, 0, 0, 247, 3, 9, 0, 4020, 0, 65, 0, 1145, 992, 0, 0, 2613, 33, 520, 421, 1398, 2044, 0, 0, 3703, 0, 0, 1109, 0, 1311, 51, 3782, 51, 4, 0, 0, 4089, 0, 0, 31, 1, 418, 1633, 0, 2, 0, 0, 4095, 29, 1573, 4046, 608, 31, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4081, 0, 1307, 37, 1, 18, 2790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3603, 1, 0, 0, 16, 2, 3981, 568, 0, 0, 0, 0, 0, 0, 2434, 4, 13, 21, 0, 1165, 13, 3426, 1850, 0, 0, 0, 0, 0, 0, 0, 10, 570, 514, 1601, 6, 376, 3999, 0, 458, 66, 0, 0, 0, 0, 0, 0, 47, 8, 4092, 113, 0, 3703, 0, 37, 1053, 396, 0, 0, 0, 0, 0, 0, 530, 4088, 49, 159, 0, 1, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 317, 3810, 0, 0, 0, 3528, 376, 0, 0, 0, 0, 0, 0, 0, 2577, 0, 3641, 3966, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8175, 8926, 7600, 9449, 4880, 8044, 10228] avg=8192.0000 max=10228 max_rank=7 min=4880 min_rank=5 imbalance_ratio=1.248535 max_load_reduction=55308 max_load_reduction_pct=0.843933 moved_to_echo=32601 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 601, 1359, 0, 73, 0, 1480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 328, 2, 8, 0, 14, 10, 4094, 97, 0, 0, 0, 0, 0, 0, 55, 0, 0, 9, 1, 1109, 0, 3445, 1923, 0, 0, 0, 0, 0, 0, 0, 18, 102, 0, 383, 31, 4063, 3985, 8, 390, 459, 0, 0, 0, 0, 0, 0, 2, 1, 4084, 1317, 420, 3118, 0, 312, 895, 411, 0, 0, 0, 0, 0, 0, 3637, 3989, 10, 1569, 3, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 595, 0, 30, 2723, 194, 511, 4052, 0, 601, 391, 0, 0, 0, 0, 0, 0, 0, 2191, 0, 2349, 3616, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7609, 4571, 6542, 9439, 10560, 9479, 9097, 8239] avg=8192.0000 max=10560 max_rank=4 min=4571 min_rank=1 imbalance_ratio=1.289062 max_load_reduction=54976 max_load_reduction_pct=0.838867 moved_to_echo=37375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4095, 2027, 1363, 11, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 2, 754, 13, 0, 5, 0, 8, 4075, 397, 0, 0, 0, 0, 0, 0, 847, 1, 1, 0, 0, 2943, 4, 4003, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 148, 1049, 4067, 8, 446, 458, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 292, 0, 626, 2, 17, 1024, 412, 0, 0, 0, 0, 0, 0, 3634, 1723, 0, 4036, 0, 0, 0, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3984, 21, 0, 4049, 3, 2458, 392, 0, 0, 0, 0, 0, 0, 0, 2506, 0, 1863, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7505, 5285, 9728, 6265, 6471, 10909, 10908, 8465] avg=8192.0000 max=10909 max_rank=5 min=5285 min_rank=1 imbalance_ratio=1.331665 max_load_reduction=54627 max_load_reduction_pct=0.833542 moved_to_echo=36753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3406, 5, 1345, 447, 0, 2, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 83, 0, 0, 52, 1, 4, 3956, 553, 0, 0, 0, 0, 0, 0, 202, 0, 0, 13, 0, 4095, 0, 4061, 1905, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 6, 315, 4011, 4006, 3, 459, 147, 0, 0, 0, 0, 0, 0, 8, 0, 4090, 2, 0, 201, 113, 94, 1055, 407, 0, 0, 0, 0, 0, 0, 1166, 3402, 12, 3914, 0, 2, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 1178, 3638, 0, 127, 15, 3434, 388, 0, 0, 0, 0, 0, 0, 96, 2582, 0, 3750, 4041, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7670, 4674, 10276, 9018, 5970, 8543, 8915, 10470] avg=8192.0000 max=10470 max_rank=7 min=4674 min_rank=1 imbalance_ratio=1.278076 max_load_reduction=55066 max_load_reduction_pct=0.840240 moved_to_echo=33898 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3311, 732, 1363, 388, 0, 27, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 1748, 52, 0, 79, 267, 0, 3882, 566, 0, 0, 0, 0, 0, 0, 26, 7, 136, 117, 2, 3910, 0, 2938, 1929, 0, 0, 0, 0, 0, 0, 0, 203, 129, 1, 158, 118, 57, 4015, 6, 459, 435, 0, 0, 0, 0, 0, 0, 3, 19, 4052, 31, 0, 13, 4, 13, 1054, 412, 0, 0, 0, 0, 0, 0, 3454, 1894, 0, 4014, 0, 0, 0, 908, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 43, 623, 4061, 2, 3, 133, 3512, 392, 0, 0, 0, 0, 0, 0, 9, 2579, 199, 2304, 4083, 273, 276, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9885, 6603, 9065, 5581, 5601, 10270, 8804, 9727] avg=8192.0000 max=10270 max_rank=5 min=5581 min_rank=3 imbalance_ratio=1.253662 max_load_reduction=55266 max_load_reduction_pct=0.843292 moved_to_echo=34402 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3895, 342, 1349, 0, 277, 0, 1970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3901, 0, 556, 11, 0, 20, 4096, 563, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 17, 31, 3529, 1932, 0, 0, 0, 0, 0, 0, 0, 76, 0, 1, 1115, 1475, 1331, 3977, 18, 420, 385, 0, 0, 0, 0, 0, 0, 748, 23, 3719, 192, 0, 602, 0, 0, 963, 417, 0, 0, 0, 0, 0, 0, 3015, 1936, 0, 2388, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4020, 1, 0, 1436, 0, 3489, 397, 0, 0, 0, 0, 0, 0, 0, 2334, 8, 62, 4095, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7833, 13242, 5512, 8798, 6664, 7601, 9345, 6541] avg=8192.0000 max=13242 max_rank=1 min=5512 min_rank=2 imbalance_ratio=1.616455 max_load_reduction=52294 max_load_reduction_pct=0.797943 moved_to_echo=30151 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3474, 1263, 555, 0, 2950, 0, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 4088, 0, 0, 0, 247, 3, 4089, 569, 0, 0, 0, 0, 0, 0, 9, 0, 4020, 0, 65, 0, 1145, 992, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 33, 520, 421, 1398, 2044, 459, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3703, 0, 0, 1109, 0, 1311, 1055, 168, 0, 0, 0, 0, 0, 0, 46, 3782, 51, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 1, 418, 1633, 0, 2, 0, 3520, 160, 0, 0, 0, 0, 0, 0, 0, 2581, 29, 1573, 4046, 608, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12101, 13063, 7016, 7493, 7346, 3883, 5765, 8869] avg=8192.0000 max=13063 max_rank=1 min=3883 min_rank=5 imbalance_ratio=1.594604 max_load_reduction=52473 max_load_reduction_pct=0.800674 moved_to_echo=25863 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3261, 1, 1357, 0, 48, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2281, 5, 0, 6, 66, 196, 2619, 2716, 2010, 24, 0, 0, 0, 0, 0, 0, 157, 7, 0, 548, 10, 3445, 0, 2382, 1922, 0, 0, 0, 0, 0, 0, 0, 20, 12, 1224, 10, 47, 4002, 3916, 1215, 459, 437, 0, 0, 0, 0, 0, 0, 85, 0, 4047, 3776, 0, 4067, 0, 34, 1055, 411, 0, 0, 0, 0, 0, 0, 3463, 40, 12, 1633, 161, 605, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4064, 36, 0, 2, 143, 391, 0, 0, 0, 0, 0, 0, 0, 2582, 15, 10, 211, 0, 175, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4699, 9923, 8471, 11342, 13475, 9971, 4658, 2997] avg=8192.0000 max=13475 max_rank=4 min=2997 min_rank=7 imbalance_ratio=1.644897 max_load_reduction=52061 max_load_reduction_pct=0.794388 moved_to_echo=31101 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[29696, 4992, 10112, 896, 3456, 128, 16640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[896, 128, 32000, 5760, 512, 13440, 128, 1920, 8192, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[10624, 128, 14592, 128, 640, 384, 3200, 2816, 30208, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 19968, 256, 15616, 28160, 896, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[19072, 20864, 256, 17792, 256, 640, 0, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3840, 128, 4224, 768, 128, 16768, 1280, 24832, 14208, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 896, 4480, 3456, 2688, 15360, 29440, 3328, 3584, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 256, 128, 13312, 17536, 640, 9728, 256, 20736, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[353, 0, 3865, 983, 1937, 0, 0, 2688, 248, 682, 3570, 30, 6, 7, 0, 4075, 3, 950, 497, 24, 2, 0, 0, 0, 860, 0, 97, 4092, 0, 0, 61, 2, 0, 0, 0, 4058, 105, 0, 3943, 8, 4091, 0, 997, 4096, 2835, 0, 1235, 0, 3949, 0, 0, 0, 3248, 0, 3844, 58, 4047, 13, 0, 586, 239, 3, 3114, 35] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 4096, 0, 28, 1, 38, 1, 3586, 0, 0, 0, 0, 0, 0, 2145, 18, 4049, 144, 2, 0, 0, 986, 7, 617, 424, 0, 3520, 0, 0, 1, 3353, 0, 555, 0, 4096, 7, 0, 482, 2, 4093, 1413, 5, 4096, 2929, 0, 3996, 0, 0, 2, 0, 4055, 3, 0, 3942, 2, 4026, 571, 0, 0, 4090, 42, 3706, 403] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[168, 0, 4087, 7, 1339, 95, 0, 3563, 5, 213, 3420, 0, 33, 18, 0, 3062, 71, 0, 40, 86, 0, 0, 370, 13, 338, 453, 3510, 4080, 0, 0, 1, 53, 0, 12, 0, 4096, 4096, 35, 1434, 10, 209, 5, 54, 3889, 4095, 0, 3959, 0, 1543, 8, 0, 3037, 179, 0, 3991, 1791, 3981, 0, 0, 0, 0, 52, 4024, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1192, 0, 4092, 48, 0, 0, 0, 121, 2377, 3, 114, 0, 12, 0, 0, 523, 4054, 4066, 0, 2988, 0, 182, 0, 25, 0, 1, 27, 4093, 0, 1, 0, 121, 1735, 2198, 1, 3910, 0, 0, 4091, 55, 3738, 905, 79, 3620, 365, 0, 1515, 0, 2214, 0, 0, 603, 0, 0, 3799, 0, 4096, 3934, 7, 0, 0, 0, 4016, 615] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[325, 3, 3971, 4018, 0, 76, 1310, 893, 0, 61, 4079, 0, 891, 0, 21, 2285, 1, 3989, 3593, 431, 22, 171, 132, 3, 0, 172, 856, 4094, 5, 19, 0, 1, 1077, 0, 0, 1285, 0, 0, 1652, 1, 3694, 5, 1408, 3756, 155, 0, 2273, 24, 2303, 0, 452, 1, 213, 6, 4032, 4037, 3557, 0, 0, 19, 0, 0, 4095, 69] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[29, 0, 4010, 2, 2427, 3, 5, 24, 1788, 0, 4046, 0, 0, 26, 915, 2815, 4000, 0, 4039, 22, 1, 41, 0, 24, 0, 9, 0, 3987, 207, 1, 0, 0, 11, 4, 0, 3128, 272, 183, 3979, 0, 4084, 8, 70, 4092, 4096, 45, 0, 0, 1072, 10, 14, 31, 2048, 1723, 314, 0, 4087, 0, 0, 0, 1303, 3959, 2504, 78] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[2979, 0, 4033, 3, 0, 8, 263, 3748, 0, 5, 465, 0, 2681, 0, 0, 2708, 0, 4080, 0, 54, 0, 22, 0, 14, 0, 90, 7, 4041, 0, 2, 0, 0, 0, 3787, 26, 4096, 0, 0, 137, 0, 3861, 224, 10, 4093, 205, 0, 3111, 13, 735, 1008, 54, 1466, 3732, 37, 2108, 3023, 4096, 0, 3, 204, 208, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4096, 487, 0, 2, 0, 3828, 861, 0, 172, 0, 14, 0, 0, 0, 11, 3923, 680, 136, 0, 2, 229, 4, 40, 13, 0, 4091, 0, 0, 77, 8, 0, 3843, 1, 4094, 3686, 2, 4095, 102, 3821, 41, 217, 4092, 4018, 0, 75, 0, 3, 0, 0, 1872, 72, 84, 3405, 340, 3998, 1634, 0, 91, 1456, 233, 3724, 1863] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[353, 0, 3865, 983, 1937, 0, 0, 2688, 42, 0, 0, 0, 0, 0, 0, 0, 248, 682, 3570, 30, 6, 7, 0, 4075, 715, 744, 684, 319, 0, 0, 0, 0, 3, 950, 497, 24, 2, 0, 0, 0, 1486, 1242, 0, 0, 0, 0, 0, 0, 860, 0, 97, 4092, 0, 0, 61, 2, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3374, 105, 0, 3943, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3730, 0, 997, 0, 2835, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 3248, 0, 3100, 58, 0, 0, 0, 0, 0, 0, 0, 0, 1846, 13, 0, 586, 239, 3, 3114, 35, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9868, 11080, 4204, 7966, 7430, 8797, 10355, 5836] avg=8192.0000 max=11080 max_rank=1 min=4204 min_rank=2 imbalance_ratio=1.352539 max_load_reduction=54456 max_load_reduction_pct=0.830933 moved_to_echo=32418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[168, 0, 4087, 7, 1339, 95, 0, 3563, 2, 0, 0, 0, 0, 0, 0, 0, 5, 213, 3420, 0, 33, 18, 0, 3062, 704, 773, 691, 16, 0, 0, 0, 0, 71, 0, 40, 86, 0, 0, 370, 13, 1462, 1180, 0, 0, 0, 0, 0, 0, 338, 453, 3510, 4080, 0, 0, 1, 53, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3405, 4096, 35, 1434, 10, 0, 0, 0, 0, 0, 0, 0, 0, 191, 5, 54, 0, 4095, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1543, 8, 0, 3037, 179, 0, 3218, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 1815, 0, 0, 0, 0, 52, 4024, 11, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9261, 8935, 3222, 11144, 8992, 8304, 9776, 5902] avg=8192.0000 max=11144 max_rank=3 min=3222 min_rank=2 imbalance_ratio=1.360352 max_load_reduction=54392 max_load_reduction_pct=0.829956 moved_to_echo=32974 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[325, 3, 3971, 4018, 0, 76, 1310, 893, 38, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4079, 0, 891, 0, 21, 2285, 629, 781, 216, 288, 0, 0, 0, 0, 1, 3989, 3593, 431, 22, 171, 132, 3, 1306, 1138, 0, 0, 0, 0, 0, 0, 0, 172, 856, 4094, 5, 19, 0, 1, 2618, 0, 0, 0, 0, 0, 0, 0, 1077, 0, 0, 1069, 0, 0, 1652, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3368, 5, 1408, 0, 155, 0, 2273, 24, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 0, 452, 1, 213, 6, 3251, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 1622, 0, 0, 19, 0, 0, 4095, 69, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10634, 9251, 10786, 7765, 3799, 7233, 10263, 5805] avg=8192.0000 max=10786 max_rank=2 min=3799 min_rank=4 imbalance_ratio=1.316650 max_load_reduction=54750 max_load_reduction_pct=0.835419 moved_to_echo=27100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1192, 0, 4092, 48, 0, 0, 0, 121, 38, 0, 0, 0, 0, 0, 0, 0, 2377, 3, 114, 0, 12, 0, 0, 523, 724, 735, 659, 291, 0, 0, 0, 0, 4054, 4066, 0, 2988, 0, 182, 0, 25, 1504, 1098, 0, 0, 0, 0, 0, 0, 0, 1, 27, 4093, 0, 1, 0, 121, 2522, 0, 0, 0, 0, 0, 0, 0, 1735, 2198, 1, 3251, 0, 0, 4091, 55, 0, 0, 0, 0, 0, 0, 0, 0, 3409, 905, 79, 0, 365, 0, 1515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 0, 0, 603, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1868, 3934, 7, 0, 0, 0, 4016, 615, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5491, 5438, 13917, 6765, 11331, 6273, 5881, 10440] avg=8192.0000 max=13917 max_rank=2 min=5438 min_rank=1 imbalance_ratio=1.698853 max_load_reduction=51619 max_load_reduction_pct=0.787643 moved_to_echo=33925 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 0, 28, 1, 38, 1, 42, 0, 0, 0, 0, 0, 0, 0, 3586, 0, 0, 0, 0, 0, 0, 2145, 712, 763, 691, 319, 0, 0, 0, 0, 18, 4049, 144, 2, 0, 0, 986, 7, 1478, 1242, 0, 0, 0, 0, 0, 0, 617, 424, 0, 3520, 0, 0, 1, 3353, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 555, 0, 3405, 7, 0, 482, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3732, 1413, 5, 0, 2929, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4055, 3, 0, 3179, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 571, 0, 0, 4090, 42, 3706, 403, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4210, 8216, 7926, 10769, 4451, 12075, 7241, 10648] avg=8192.0000 max=12075 max_rank=5 min=4210 min_rank=0 imbalance_ratio=1.473999 max_load_reduction=53461 max_load_reduction_pct=0.815750 moved_to_echo=34415 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 4010, 2, 2427, 3, 5, 24, 42, 0, 0, 0, 0, 0, 0, 0, 1788, 0, 4046, 0, 0, 26, 915, 2815, 722, 60, 527, 318, 0, 0, 0, 0, 4000, 0, 4039, 22, 1, 41, 0, 24, 1501, 1241, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3987, 207, 1, 0, 0, 2851, 0, 0, 0, 0, 0, 0, 0, 11, 4, 0, 2601, 272, 183, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3724, 8, 70, 0, 4096, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 10, 14, 31, 2048, 1723, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1864, 0, 0, 0, 1303, 3959, 2504, 78, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6542, 11217, 10869, 7055, 7050, 7943, 5152, 9708] avg=8192.0000 max=11217 max_rank=1 min=5152 min_rank=6 imbalance_ratio=1.369263 max_load_reduction=54319 max_load_reduction_pct=0.828842 moved_to_echo=29853 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2979, 0, 4033, 3, 0, 8, 263, 3748, 40, 0, 0, 0, 0, 0, 0, 0, 0, 5, 465, 0, 2681, 0, 0, 2708, 724, 408, 691, 301, 0, 0, 0, 0, 0, 4080, 0, 54, 0, 22, 0, 14, 1504, 1241, 0, 0, 0, 0, 0, 0, 0, 90, 7, 4041, 0, 2, 0, 0, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 3787, 26, 3405, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3520, 224, 10, 0, 205, 0, 3111, 13, 0, 0, 0, 0, 0, 0, 0, 0, 735, 1008, 54, 1466, 3732, 37, 1700, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 1868, 0, 3, 204, 208, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11074, 7983, 6915, 6992, 7355, 7083, 11755, 6379] avg=8192.0000 max=11755 max_rank=6 min=6379 min_rank=7 imbalance_ratio=1.434937 max_load_reduction=53781 max_load_reduction_pct=0.820633 moved_to_echo=32572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 487, 0, 2, 0, 3828, 43, 0, 0, 0, 0, 0, 0, 0, 861, 0, 172, 0, 14, 0, 0, 0, 710, 663, 695, 301, 0, 0, 0, 0, 11, 3923, 680, 136, 0, 2, 229, 4, 1471, 1241, 0, 0, 0, 0, 0, 0, 40, 13, 0, 4091, 0, 0, 77, 8, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 1, 3399, 3686, 2, 4095, 102, 0, 0, 0, 0, 0, 0, 0, 0, 3477, 41, 217, 0, 4018, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1872, 72, 84, 2742, 340, 0, 0, 0, 0, 0, 0, 0, 0, 1817, 1634, 0, 91, 1456, 233, 3724, 1863, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8456, 3416, 7697, 7080, 15128, 7828, 5113, 10818] avg=8192.0000 max=15128 max_rank=4 min=3416 min_rank=1 imbalance_ratio=1.846680 max_load_reduction=50408 max_load_reduction_pct=0.769165 moved_to_echo=38887 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8192, 21120, 9088, 3840, 128, 512, 1792, 128, 11776, 9728, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11904, 1152, 640, 11136, 9600, 1920, 20608, 9344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[25216, 2688, 2944, 0, 18816, 128, 16256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8960, 1024, 15872, 128, 3712, 128, 1024, 17664, 5760, 4992, 4864, 2176, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2944, 10496, 128, 23936, 8192, 256, 19840, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[14592, 6272, 128, 1024, 7296, 4352, 29312, 3200, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5120, 128, 32256, 5632, 5760, 256, 1664, 14976, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1920, 1280, 4608, 32000, 256, 128, 256, 3584, 22144, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4090, 1, 4096, 0, 4024, 0, 0, 2735, 11, 4094, 0, 3919, 200, 21, 0, 570, 3221, 0, 0, 133, 48, 0, 0, 132, 0, 4078, 192, 0, 3914, 4094, 0, 0, 3908, 1352, 403, 2, 0, 1718, 0, 352, 1165, 3393, 65, 51, 28, 0, 4, 52, 0, 4013, 94, 0, 73, 0, 367, 538, 3804, 0, 614, 1, 2, 0, 0, 3964] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[523, 1, 785, 435, 3551, 0, 0, 96, 0, 4066, 0, 4094, 95, 0, 0, 7, 1366, 62, 1, 0, 4096, 88, 0, 0, 642, 4032, 61, 25, 3925, 2223, 1, 0, 3873, 2, 1, 0, 255, 3485, 0, 0, 1, 3084, 33, 2280, 9, 0, 201, 0, 3070, 4056, 0, 5, 11, 0, 802, 206, 3583, 3, 2464, 3695, 2003, 0, 0, 2239] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[2914, 1, 3296, 0, 4031, 0, 0, 2656, 1, 1142, 0, 4094, 2939, 849, 0, 20, 1063, 0, 0, 3, 1918, 559, 0, 4028, 0, 3947, 0, 0, 3, 4069, 813, 0, 4096, 18, 2465, 0, 0, 0, 0, 2775, 3610, 0, 0, 3418, 0, 157, 207, 3, 0, 3971, 1, 0, 731, 3, 0, 92, 2994, 2, 552, 2026, 2, 0, 67, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3895, 3009, 74, 2, 536, 0, 34, 3982, 15, 4032, 0, 1990, 3983, 0, 21, 0, 4096, 0, 47, 4087, 930, 0, 0, 0, 827, 2110, 0, 0, 0, 1649, 54, 679, 3447, 2468, 25, 0, 1823, 10, 0, 156, 3046, 0, 2106, 5, 7, 23, 0, 4031, 0, 3074, 0, 0, 3149, 6, 1, 0, 4096, 350, 399, 0, 1184, 0, 0, 78] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3931, 0, 0, 0, 279, 0, 0, 4092, 1, 4095, 0, 3928, 4091, 0, 0, 28, 3, 1, 0, 0, 0, 9, 0, 0, 2075, 4096, 0, 0, 4090, 3655, 0, 0, 89, 1042, 446, 0, 0, 4001, 0, 178, 5, 2123, 378, 3364, 0, 0, 133, 401, 0, 4090, 675, 443, 3, 1552, 4053, 0, 3959, 0, 0, 2, 11, 0, 120, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3717, 0, 3712, 1, 4096, 0, 2, 3847, 0, 162, 294, 4096, 0, 9, 0, 0, 930, 5, 0, 0, 1380, 0, 0, 0, 6, 4095, 0, 1, 1911, 4092, 3165, 20, 4076, 338, 0, 0, 0, 4092, 4, 0, 441, 79, 0, 2357, 1, 0, 48, 2345, 238, 4095, 0, 0, 0, 3, 2405, 4, 4090, 0, 308, 27, 710, 5, 233, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3917, 0, 4095, 101, 4096, 150, 1, 4095, 0, 3318, 479, 4096, 0, 55, 0, 0, 1319, 1, 0, 0, 42, 3, 0, 0, 5, 4004, 6, 1, 296, 4095, 4, 0, 4005, 3, 0, 0, 0, 4060, 3, 0, 0, 6, 6, 142, 768, 0, 6, 1045, 1022, 4029, 79, 339, 0, 3, 3883, 3494, 4007, 302, 2, 0, 4, 53, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[39, 2, 3, 0, 4014, 0, 0, 4081, 1, 4095, 0, 3812, 264, 0, 166, 26, 1403, 0, 0, 4, 0, 27, 0, 650, 0, 4096, 0, 0, 61, 4067, 72, 0, 4096, 235, 0, 449, 0, 3822, 0, 2594, 3569, 28, 0, 2002, 1806, 54, 0, 4096, 0, 4053, 0, 951, 1, 36, 2677, 0, 3359, 1, 0, 6, 66, 0, 959, 3793] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4090, 1, 4096, 0, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4094, 0, 3444, 200, 21, 0, 570, 0, 0, 0, 0, 0, 0, 0, 0, 3221, 0, 0, 133, 48, 0, 0, 132, 2735, 1110, 0, 0, 0, 0, 0, 0, 0, 1975, 192, 0, 3914, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3819, 1352, 403, 2, 0, 1718, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 3393, 65, 51, 28, 0, 4, 52, 993, 475, 291, 0, 0, 0, 0, 0, 0, 4013, 94, 0, 73, 0, 367, 538, 313, 89, 75, 0, 0, 0, 0, 0, 3438, 0, 614, 1, 2, 0, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11898, 8340, 7379, 10175, 7646, 6517, 5562, 8019] avg=8192.0000 max=11898 max_rank=0 min=5562 min_rank=6 imbalance_ratio=1.452393 max_load_reduction=53638 max_load_reduction_pct=0.818451 moved_to_echo=27744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[523, 1, 785, 435, 3275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 3598, 95, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1366, 62, 1, 0, 4096, 88, 0, 0, 96, 1098, 0, 0, 0, 0, 0, 0, 642, 1953, 61, 25, 3925, 2223, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 2, 1, 0, 255, 3485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3084, 33, 2280, 9, 0, 201, 0, 981, 496, 274, 0, 0, 0, 0, 0, 3070, 4056, 0, 5, 11, 0, 802, 206, 276, 88, 71, 0, 0, 0, 0, 0, 3238, 3, 2464, 3695, 2003, 0, 0, 2239, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5019, 7766, 6807, 8830, 7528, 7359, 8585, 13642] avg=8192.0000 max=13642 max_rank=7 min=5019 min_rank=0 imbalance_ratio=1.665283 max_load_reduction=51894 max_load_reduction_pct=0.791840 moved_to_echo=37114 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2914, 1, 3296, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1142, 0, 3598, 2939, 849, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 0, 0, 3, 1918, 559, 0, 4028, 2656, 1075, 0, 0, 0, 0, 0, 0, 0, 1911, 0, 0, 3, 4069, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 18, 2465, 0, 0, 0, 0, 2775, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 0, 0, 3418, 0, 157, 207, 3, 961, 496, 229, 0, 0, 0, 0, 0, 0, 3971, 1, 0, 731, 3, 0, 92, 313, 93, 59, 0, 0, 0, 0, 0, 2706, 2, 552, 2026, 2, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9929, 8549, 11302, 6796, 9261, 9081, 5263, 5355] avg=8192.0000 max=11302 max_rank=2 min=5263 min_rank=6 imbalance_ratio=1.379639 max_load_reduction=54234 max_load_reduction_pct=0.827545 moved_to_echo=28960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3717, 0, 3712, 1, 3777, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 294, 3600, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 5, 0, 0, 1380, 0, 0, 0, 3847, 1115, 0, 0, 0, 0, 0, 0, 6, 1983, 0, 1, 1911, 4092, 3165, 20, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 338, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 441, 79, 0, 2357, 1, 0, 48, 2345, 997, 496, 313, 0, 0, 0, 0, 0, 238, 4095, 0, 0, 0, 3, 2405, 4, 319, 93, 81, 0, 0, 0, 0, 0, 3696, 0, 308, 27, 710, 5, 233, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11209, 4065, 7277, 11178, 8417, 7077, 7238, 9075] avg=8192.0000 max=11209 max_rank=0 min=4065 min_rank=1 imbalance_ratio=1.368286 max_load_reduction=54327 max_load_reduction_pct=0.828964 moved_to_echo=31807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3895, 3009, 74, 2, 495, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4032, 0, 1749, 3983, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 47, 4087, 930, 0, 0, 0, 3982, 574, 0, 0, 0, 0, 0, 0, 827, 1023, 0, 0, 0, 1649, 54, 679, 0, 0, 0, 0, 0, 0, 0, 0, 3369, 2468, 25, 0, 1823, 10, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 3046, 0, 2106, 5, 7, 23, 0, 4031, 513, 241, 314, 0, 0, 0, 0, 0, 0, 3074, 0, 0, 3149, 6, 1, 0, 41, 78, 81, 0, 0, 0, 0, 0, 3701, 350, 399, 0, 1184, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7509, 9800, 13716, 4232, 7851, 10286, 6430, 5712] avg=8192.0000 max=13716 max_rank=2 min=4232 min_rank=3 imbalance_ratio=1.674316 max_load_reduction=51820 max_load_reduction_pct=0.790710 moved_to_echo=30279 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3917, 0, 4095, 101, 3777, 150, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3318, 479, 3600, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1319, 1, 0, 0, 42, 3, 0, 0, 4095, 1090, 0, 0, 0, 0, 0, 0, 5, 1939, 6, 1, 296, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 3, 0, 0, 0, 4060, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 142, 768, 0, 6, 1045, 975, 496, 307, 0, 0, 0, 0, 0, 1022, 4029, 79, 339, 0, 3, 3883, 3494, 319, 91, 79, 0, 0, 0, 0, 0, 3621, 302, 2, 0, 4, 53, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12041, 7452, 6550, 6346, 7980, 3751, 13338, 8078] avg=8192.0000 max=13338 max_rank=6 min=3751 min_rank=5 imbalance_ratio=1.628174 max_load_reduction=52198 max_load_reduction_pct=0.796478 moved_to_echo=33147 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3931, 0, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 3452, 4091, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 9, 0, 0, 4092, 1115, 0, 0, 0, 0, 0, 0, 2075, 1984, 0, 0, 4090, 3655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 1042, 446, 0, 0, 4001, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2123, 378, 3364, 0, 0, 133, 401, 997, 476, 303, 0, 0, 0, 0, 0, 0, 4090, 675, 443, 3, 1552, 4053, 0, 21, 2, 78, 0, 0, 0, 0, 0, 3578, 0, 0, 2, 11, 0, 120, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4189, 11667, 5220, 11804, 5754, 8180, 10917, 7805] avg=8192.0000 max=11804 max_rank=3 min=4189 min_rank=0 imbalance_ratio=1.440918 max_load_reduction=53732 max_load_reduction_pct=0.819885 moved_to_echo=32656 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[39, 2, 3, 0, 3698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 3345, 264, 0, 166, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1403, 0, 0, 4, 0, 27, 0, 650, 4081, 1120, 0, 0, 0, 0, 0, 0, 0, 1975, 0, 0, 61, 4067, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 235, 0, 449, 0, 3822, 0, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 28, 0, 2002, 1806, 54, 0, 4096, 1001, 467, 262, 0, 0, 0, 0, 0, 0, 4053, 0, 951, 1, 36, 2677, 0, 316, 97, 72, 0, 0, 0, 0, 0, 3025, 1, 0, 6, 66, 0, 959, 3793, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3742, 7897, 7285, 6175, 11099, 13285, 8203, 7850] avg=8192.0000 max=13285 max_rank=5 min=3742 min_rank=0 imbalance_ratio=1.621704 max_load_reduction=52251 max_load_reduction_pct=0.797287 moved_to_echo=40437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23040, 3072, 16128, 640, 22784, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[27008, 5504, 3456, 512, 2176, 21248, 128, 6144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[13440, 128, 128, 4352, 8448, 768, 0, 4864, 25600, 8320, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[11904, 8832, 2688, 13696, 2688, 256, 640, 12032, 7424, 3712, 2304, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3584, 14848, 384, 128, 14208, 28032, 4224, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[27008, 768, 4352, 5760, 4096, 128, 1408, 22400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4352, 31488, 896, 1792, 3968, 1664, 14208, 4352, 1920, 640, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 25088, 896, 26496, 11648, 1024, 256, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 148, 0, 0, 9, 36, 46, 3781, 0, 0, 205, 3662, 0, 114, 3881, 0, 4096, 4077, 2056, 2782, 3769, 2231, 885, 0, 0, 0, 0, 4046, 4096, 0, 0, 1854, 0, 361, 1100, 0, 3, 0, 221, 4032, 1, 0, 1, 3315, 0, 2603, 0, 3886, 1, 2, 42, 0, 0, 0, 0, 1, 0, 4090, 0, 17, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4042, 0, 28, 0, 24, 48, 0, 0, 10, 0, 4087, 0, 15, 157, 35, 0, 0, 4031, 41, 2602, 1679, 4094, 0, 3207, 0, 3874, 0, 0, 5, 9, 3981, 4094, 2480, 1115, 0, 0, 3981, 102, 0, 50, 1, 1220, 139, 1456, 44, 0, 1206, 2, 3871, 19, 4085, 46, 1607, 3120, 329, 0, 218, 1, 1380, 13, 0, 6, 2982, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[690, 0, 260, 0, 8, 4030, 0, 4, 0, 1, 4084, 0, 8, 0, 3923, 238, 0, 4063, 4071, 12, 2, 11, 11, 4095, 1409, 2710, 14, 1, 0, 9, 205, 4026, 4027, 15, 23, 0, 2, 641, 0, 8, 0, 0, 24, 3683, 0, 1006, 4088, 4, 80, 41, 4081, 0, 3974, 27, 1, 0, 3478, 1994, 0, 1, 7, 0, 3936, 510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[3150, 0, 0, 0, 0, 4, 58, 33, 2603, 54, 4088, 0, 0, 3457, 109, 0, 9, 3999, 20, 1227, 1, 469, 4062, 3943, 352, 0, 0, 0, 0, 54, 2804, 4096, 14, 0, 467, 0, 12, 0, 0, 3, 0, 4096, 109, 3555, 2, 1, 4034, 17, 4095, 0, 4096, 16, 4095, 584, 0, 0, 2, 35, 1775, 0, 2300, 0, 1636, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[29, 4096, 0, 0, 0, 1316, 0, 537, 1752, 14, 4029, 161, 0, 3878, 3993, 961, 13, 3947, 174, 0, 27, 4049, 15, 2461, 4094, 6, 1, 0, 86, 0, 4095, 3700, 45, 0, 701, 10, 0, 0, 0, 247, 8, 0, 3778, 0, 0, 435, 2781, 1, 2981, 1520, 4086, 4075, 0, 1, 2, 140, 33, 131, 0, 0, 0, 145, 946, 36] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4093, 0, 0, 4019, 0, 30, 0, 1064, 99, 3, 4091, 0, 0, 1842, 14, 0, 0, 3, 4095, 5, 5, 4038, 3699, 2277, 806, 32, 0, 0, 0, 351, 4074, 4096, 35, 0, 118, 0, 3845, 3905, 0, 4, 0, 71, 0, 0, 39, 945, 1959, 5, 4094, 0, 4074, 0, 3011, 417, 3, 0, 8, 129, 73, 0, 0, 0, 4043, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[346, 117, 0, 0, 252, 2874, 0, 3586, 1, 955, 1110, 0, 1, 4090, 0, 1, 1, 4072, 483, 20, 3936, 3679, 4093, 9, 0, 4090, 0, 0, 0, 0, 4041, 4096, 0, 3, 14, 0, 0, 0, 0, 0, 0, 161, 4089, 3600, 0, 2, 77, 0, 1726, 0, 4096, 24, 4094, 2, 2290, 3, 0, 0, 2992, 0, 0, 0, 6, 504] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[2454, 341, 0, 0, 128, 4065, 0, 10, 1, 2, 4096, 0, 100, 674, 79, 1698, 905, 3871, 44, 4088, 4020, 0, 1229, 4095, 0, 4094, 8, 0, 0, 0, 0, 3839, 0, 57, 20, 0, 1118, 1227, 0, 0, 0, 4, 437, 4025, 0, 1, 3041, 3, 24, 0, 3811, 5, 2771, 2475, 0, 0, 2, 0, 40, 4, 0, 0, 2542, 4088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 148, 0, 0, 9, 1314, 1118, 0, 0, 0, 0, 0, 0, 36, 46, 3781, 0, 0, 205, 3662, 0, 144, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 4096, 4077, 1552, 2782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2231, 885, 0, 0, 0, 0, 4046, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1854, 0, 361, 1100, 0, 3, 3881, 1590, 0, 0, 0, 0, 0, 0, 0, 221, 4032, 1, 0, 1, 3315, 0, 586, 360, 0, 0, 0, 0, 0, 0, 2603, 0, 1367, 1, 2, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 17, 1, 2179, 1401, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6674, 7874, 12621, 9358, 8789, 8516, 4015, 7689] avg=8192.0000 max=12621 max_rank=2 min=4015 min_rank=6 imbalance_ratio=1.540649 max_load_reduction=52915 max_load_reduction_pct=0.807419 moved_to_echo=29009 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4042, 0, 28, 0, 24, 48, 0, 0, 1313, 1176, 0, 0, 0, 0, 0, 0, 10, 0, 4087, 0, 15, 157, 35, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 2602, 1679, 3090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 0, 0, 5, 9, 3981, 2195, 0, 0, 0, 0, 0, 0, 0, 0, 2480, 1115, 0, 0, 3981, 102, 0, 50, 4031, 1352, 0, 0, 0, 0, 0, 0, 1, 1220, 139, 1456, 44, 0, 1206, 2, 586, 717, 0, 0, 0, 0, 0, 0, 3871, 19, 1437, 46, 1607, 3120, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 1, 1380, 13, 0, 6, 2982, 0, 1855, 1472, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6631, 4591, 7412, 10064, 13111, 5371, 10429, 7927] avg=8192.0000 max=13111 max_rank=4 min=4591 min_rank=1 imbalance_ratio=1.600464 max_load_reduction=52425 max_load_reduction_pct=0.799942 moved_to_echo=36838 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[690, 0, 260, 0, 8, 4030, 0, 4, 1291, 1174, 0, 0, 0, 0, 0, 0, 0, 1, 4084, 0, 8, 0, 3923, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 12, 2, 10, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 2710, 14, 1, 0, 9, 205, 2159, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 15, 23, 0, 2, 641, 0, 8, 4063, 1727, 0, 0, 0, 0, 0, 0, 0, 0, 24, 3683, 0, 1006, 4088, 4, 576, 1, 0, 0, 0, 0, 0, 0, 80, 41, 1436, 0, 3974, 27, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3478, 1994, 0, 1, 7, 0, 3936, 510, 2368, 1471, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7457, 8254, 4106, 6507, 10506, 9382, 5559, 13765] avg=8192.0000 max=13765 max_rank=7 min=4106 min_rank=2 imbalance_ratio=1.680298 max_load_reduction=51771 max_load_reduction_pct=0.789963 moved_to_echo=39212 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3150, 0, 0, 0, 0, 4, 58, 33, 1314, 1179, 0, 0, 0, 0, 0, 0, 2603, 54, 4088, 0, 0, 3457, 109, 0, 32, 0, 0, 0, 0, 0, 0, 0, 9, 0, 20, 1227, 1, 355, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 0, 0, 0, 54, 2804, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 467, 0, 12, 0, 0, 3, 3999, 1663, 0, 0, 0, 0, 0, 0, 0, 4096, 109, 3555, 2, 1, 4034, 17, 586, 82, 0, 0, 0, 0, 0, 0, 4095, 0, 1441, 16, 4095, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 35, 1775, 0, 2300, 0, 1636, 0, 2280, 1476, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5738, 10343, 5674, 5406, 6158, 12482, 10231, 9504] avg=8192.0000 max=12482 max_rank=5 min=5406 min_rank=3 imbalance_ratio=1.523682 max_load_reduction=53054 max_load_reduction_pct=0.809540 moved_to_echo=38375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[29, 4096, 0, 0, 0, 1316, 0, 537, 1187, 1176, 0, 0, 0, 0, 0, 0, 1752, 14, 4029, 161, 0, 3878, 3993, 961, 288, 0, 0, 0, 0, 0, 0, 0, 13, 0, 174, 0, 27, 3048, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 1, 0, 86, 0, 4095, 1983, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 701, 10, 0, 0, 0, 247, 3947, 1039, 0, 0, 0, 0, 0, 0, 8, 0, 3778, 0, 0, 435, 2781, 1, 530, 713, 0, 0, 0, 0, 0, 0, 2981, 1520, 1437, 4075, 0, 1, 2, 140, 0, 0, 0, 0, 0, 0, 0, 0, 33, 131, 0, 0, 0, 145, 946, 36, 1422, 1473, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8341, 15076, 3277, 10265, 5989, 8246, 10156, 4186] avg=8192.0000 max=15076 max_rank=1 min=3277 min_rank=2 imbalance_ratio=1.840332 max_load_reduction=50460 max_load_reduction_pct=0.769958 moved_to_echo=28577 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[346, 117, 0, 0, 252, 2874, 0, 3586, 1314, 1179, 0, 0, 0, 0, 0, 0, 1, 955, 1110, 0, 1, 4090, 0, 1, 258, 0, 0, 0, 0, 0, 0, 0, 1, 0, 483, 20, 3936, 2776, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4041, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 14, 0, 0, 0, 0, 0, 4072, 3, 0, 0, 0, 0, 0, 0, 0, 161, 4089, 3600, 0, 2, 77, 0, 586, 645, 0, 0, 0, 0, 0, 0, 1726, 0, 1441, 24, 4094, 2, 2290, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2992, 0, 0, 0, 6, 504, 6, 1476, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9668, 6416, 11309, 10327, 4092, 9160, 9580, 4984] avg=8192.0000 max=11309 max_rank=2 min=4092 min_rank=4 imbalance_ratio=1.380493 max_load_reduction=54227 max_load_reduction_pct=0.827438 moved_to_echo=27816 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2454, 341, 0, 0, 128, 4065, 0, 10, 1236, 1101, 0, 0, 0, 0, 0, 0, 1, 2, 4096, 0, 100, 674, 79, 1698, 0, 0, 0, 0, 0, 0, 0, 0, 905, 0, 44, 4088, 4020, 0, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 8, 0, 0, 0, 0, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 20, 0, 1118, 1227, 0, 0, 3871, 1728, 0, 0, 0, 0, 0, 0, 0, 4, 437, 4025, 0, 1, 3041, 3, 555, 0, 0, 0, 0, 0, 0, 0, 24, 0, 1332, 5, 2771, 2475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 40, 4, 0, 0, 2542, 4088, 2367, 1378, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9335, 6650, 10286, 6150, 8021, 8066, 6607, 10421] avg=8192.0000 max=10421 max_rank=7 min=6150 min_rank=3 imbalance_ratio=1.272095 max_load_reduction=55115 max_load_reduction_pct=0.840988 moved_to_echo=33115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 4019, 0, 30, 0, 1064, 1314, 1172, 0, 0, 0, 0, 0, 0, 99, 3, 4091, 0, 0, 1842, 14, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 5, 3047, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 806, 32, 0, 0, 0, 351, 4074, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 118, 0, 3845, 3905, 0, 4, 3, 960, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 39, 945, 1959, 5, 586, 708, 0, 0, 0, 0, 0, 0, 4094, 0, 1434, 0, 3011, 417, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 129, 73, 0, 0, 0, 4043, 22, 1317, 1468, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11692, 6332, 10851, 7459, 8870, 4313, 8959, 7060] avg=8192.0000 max=11692 max_rank=0 min=4313 min_rank=5 imbalance_ratio=1.427246 max_load_reduction=53844 max_load_reduction_pct=0.821594 moved_to_echo=29202 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 5888, 12672, 16384, 128, 2432, 20608, 128, 4608, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3840, 2304, 6272, 128, 6400, 256, 16128, 5248, 13824, 11648, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[19584, 1664, 11392, 4224, 19584, 6784, 2688, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8960, 15744, 128, 128, 128, 512, 23296, 17280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[18944, 4608, 384, 4096, 640, 12416, 128, 5248, 10368, 9344, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1152, 0, 8960, 12160, 13824, 13952, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4608, 1152, 29440, 256, 128, 14336, 11904, 2944, 1408, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6656, 1280, 3200, 128, 9344, 7040, 0, 384, 27904, 10112, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 2847, 0, 0, 0, 0, 3334, 3850, 4096, 3842, 1, 973, 0, 0, 0, 4081, 3933, 8, 4095, 7, 0, 0, 0, 0, 264, 2, 0, 2, 5, 0, 112, 680, 3812, 20, 0, 0, 1, 1211, 1, 0, 0, 4050, 7, 4087, 20, 0, 0, 3404, 4095, 0, 0, 685, 179, 3706, 3672, 0, 0, 0, 4086, 5, 12, 351, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[6, 0, 0, 23, 43, 23, 0, 1782, 27, 4086, 3923, 5, 3313, 728, 0, 5, 1716, 2565, 2, 4096, 0, 1800, 21, 0, 0, 0, 4091, 1494, 4096, 1, 4065, 25, 105, 4088, 325, 0, 0, 240, 4095, 0, 0, 38, 2093, 0, 4088, 25, 6, 12, 4096, 51, 0, 0, 0, 8, 34, 3986, 66, 0, 63, 4096, 0, 0, 0, 85] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 353, 4048, 8, 5, 0, 3685, 0, 113, 4094, 3922, 1, 291, 0, 0, 5, 7, 37, 0, 3330, 0, 37, 73, 132, 0, 327, 0, 121, 1, 0, 3573, 329, 139, 0, 0, 0, 0, 12, 3744, 0, 0, 4096, 1465, 62, 4018, 4088, 0, 0, 4096, 4094, 0, 23, 3039, 0, 1096, 4096, 0, 2899, 3134, 70, 0, 0, 873, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 0, 2, 3669, 0, 3887, 4096, 3994, 19, 0, 0, 2, 3426, 1350, 0, 4091, 0, 0, 73, 0, 0, 137, 161, 4, 1605, 0, 155, 0, 332, 4088, 0, 0, 0, 99, 2576, 0, 0, 81, 4060, 389, 4096, 3880, 0, 8, 2692, 871, 0, 22, 0, 0, 0, 4095, 0, 3205, 0, 4096, 0, 0, 184, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 27, 1275, 0, 6, 3020, 1610, 1941, 3565, 2302, 33, 4, 3084, 0, 4, 4083, 18, 0, 4034, 1177, 0, 0, 0, 1, 466, 0, 1851, 117, 343, 1, 3301, 0, 4088, 1042, 0, 0, 3209, 0, 698, 1, 125, 0, 0, 297, 0, 719, 0, 4018, 2589, 0, 0, 2208, 0, 10, 3151, 3614, 54, 0, 473, 4096, 2867, 6, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 49, 0, 776, 0, 0, 3403, 3189, 3797, 4065, 3591, 112, 2113, 5, 0, 4060, 61, 0, 4087, 4061, 0, 0, 0, 0, 2037, 67, 0, 4092, 1, 0, 4, 165, 189, 4059, 5, 0, 2, 15, 11, 1, 0, 4, 0, 0, 395, 0, 0, 4001, 0, 0, 28, 0, 0, 3291, 2519, 1446, 672, 0, 2661, 2421, 0, 5, 4074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4094, 2, 0, 0, 0, 0, 0, 4087, 4096, 399, 4093, 0, 0, 2, 4068, 0, 0, 4096, 454, 0, 315, 0, 4, 0, 164, 1, 4095, 0, 4096, 12, 3800, 2075, 123, 14, 0, 5, 7, 0, 0, 9, 2930, 1, 4, 0, 0, 21, 4021, 241, 0, 2, 4087, 0, 1246, 4095, 0, 3388, 25, 4026, 0, 0, 79, 1259] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[5, 0, 4091, 0, 719, 0, 0, 3548, 2859, 4078, 4046, 3635, 661, 37, 0, 0, 4083, 4095, 0, 3178, 2, 0, 0, 0, 0, 0, 1, 1491, 108, 61, 2, 12, 543, 159, 115, 2661, 643, 1, 54, 63, 414, 0, 4029, 0, 1174, 4096, 1, 0, 1574, 6, 0, 0, 4027, 0, 3768, 4096, 219, 0, 0, 827, 317, 37, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2847, 0, 0, 0, 0, 3334, 1229, 1170, 0, 0, 0, 0, 0, 0, 3850, 0, 3355, 1, 973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3933, 8, 2614, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 2, 0, 2, 5, 0, 112, 2926, 0, 0, 0, 0, 0, 0, 0, 680, 3812, 20, 0, 0, 1, 1211, 1, 2005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 7, 4087, 20, 0, 0, 937, 438, 0, 0, 0, 0, 0, 0, 3188, 4095, 0, 0, 685, 179, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 5, 12, 351, 0, 544, 487, 216, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8580, 8179, 10643, 3311, 7730, 9539, 11853, 5701] avg=8192.0000 max=11853 max_rank=6 min=3311 min_rank=3 imbalance_ratio=1.446899 max_load_reduction=53683 max_load_reduction_pct=0.819138 moved_to_echo=34823 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 23, 43, 23, 0, 1782, 1333, 1167, 0, 0, 0, 0, 0, 0, 27, 0, 3426, 5, 3313, 728, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1716, 2565, 2, 2615, 0, 1800, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1494, 4096, 1, 4065, 25, 2919, 0, 0, 0, 0, 0, 0, 0, 105, 4088, 325, 0, 0, 240, 4095, 0, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 38, 2093, 0, 4088, 25, 6, 12, 937, 477, 0, 0, 0, 0, 0, 0, 3836, 51, 0, 0, 0, 8, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 63, 4096, 0, 0, 0, 85, 544, 497, 260, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4377, 7504, 8719, 16691, 11029, 7676, 3929, 5611] avg=8192.0000 max=16691 max_rank=3 min=3929 min_rank=6 imbalance_ratio=2.037476 max_load_reduction=48845 max_load_reduction_pct=0.745316 moved_to_echo=28245 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 353, 4048, 8, 5, 0, 3685, 0, 1370, 1169, 0, 0, 0, 0, 0, 0, 113, 0, 3425, 1, 291, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 7, 37, 0, 2126, 0, 37, 73, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 0, 121, 1, 0, 3573, 329, 2925, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 12, 3744, 0, 2236, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1465, 62, 4018, 4088, 0, 0, 762, 490, 0, 0, 0, 0, 0, 0, 3836, 4094, 0, 23, 3039, 0, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2899, 3134, 70, 0, 0, 873, 0, 442, 497, 260, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10638, 3835, 2412, 7276, 6131, 14981, 12088, 8175] avg=8192.0000 max=14981 max_rank=5 min=2412 min_rank=2 imbalance_ratio=1.828735 max_load_reduction=50555 max_load_reduction_pct=0.771408 moved_to_echo=41375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 0, 2, 3669, 1369, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 3577, 3994, 19, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3426, 1350, 0, 2612, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 161, 4, 1605, 0, 155, 0, 2776, 0, 0, 0, 0, 0, 0, 0, 332, 4088, 0, 0, 0, 99, 2576, 0, 2237, 0, 0, 0, 0, 0, 0, 0, 0, 81, 4060, 389, 4096, 3880, 0, 8, 936, 489, 0, 0, 0, 0, 0, 0, 2522, 871, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3205, 0, 4096, 0, 0, 184, 0, 543, 519, 170, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10242, 7592, 7461, 4838, 9332, 13939, 3415, 8717] avg=8192.0000 max=13939 max_rank=5 min=3415 min_rank=6 imbalance_ratio=1.701538 max_load_reduction=51597 max_load_reduction_pct=0.787308 moved_to_echo=35403 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 49, 0, 776, 0, 0, 3403, 842, 1084, 0, 0, 0, 0, 0, 0, 3189, 0, 3550, 3591, 112, 2113, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 61, 0, 2609, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2037, 67, 0, 4092, 1, 0, 4, 2713, 0, 0, 0, 0, 0, 0, 0, 165, 189, 4059, 5, 0, 2, 15, 11, 1377, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 395, 0, 0, 935, 300, 0, 0, 0, 0, 0, 0, 3747, 0, 0, 28, 0, 0, 3291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1446, 672, 0, 2661, 2421, 0, 5, 4074, 543, 515, 254, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6156, 12560, 10791, 8914, 5823, 1635, 7066, 12591] avg=8192.0000 max=12591 max_rank=7 min=1635 min_rank=5 imbalance_ratio=1.536987 max_load_reduction=52945 max_load_reduction_pct=0.807877 moved_to_echo=27115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 2, 0, 0, 0, 0, 1369, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 3577, 399, 4093, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 2615, 454, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 164, 1, 4095, 0, 4096, 12, 2920, 0, 0, 0, 0, 0, 0, 0, 3800, 2075, 123, 14, 0, 5, 7, 0, 2235, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2930, 1, 4, 0, 0, 21, 937, 491, 0, 0, 0, 0, 0, 0, 3766, 241, 0, 2, 4087, 0, 1246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3388, 25, 4026, 0, 0, 79, 1259, 544, 519, 255, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6632, 8071, 7452, 11292, 8259, 4393, 9342, 10095] avg=8192.0000 max=11292 max_rank=3 min=4393 min_rank=5 imbalance_ratio=1.378418 max_load_reduction=54244 max_load_reduction_pct=0.827698 moved_to_echo=32089 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 27, 1275, 0, 6, 3020, 1610, 1055, 1019, 0, 0, 0, 0, 0, 0, 1941, 0, 2010, 33, 4, 3084, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 18, 0, 2575, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 466, 0, 1851, 117, 343, 1, 3301, 2546, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1042, 0, 0, 3209, 0, 698, 1720, 0, 0, 0, 0, 0, 0, 0, 1, 125, 0, 0, 297, 0, 719, 0, 923, 376, 0, 0, 0, 0, 0, 0, 3763, 2589, 0, 0, 2208, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3614, 54, 0, 473, 4096, 2867, 6, 8, 536, 292, 255, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8012, 7076, 7853, 8626, 10757, 2441, 8570, 12201] avg=8192.0000 max=12201 max_rank=7 min=2441 min_rank=5 imbalance_ratio=1.489380 max_load_reduction=53335 max_load_reduction_pct=0.813828 moved_to_echo=33969 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 4091, 0, 719, 0, 0, 3548, 1371, 1165, 0, 0, 0, 0, 0, 0, 2859, 0, 3527, 3635, 661, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4095, 0, 2025, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1491, 108, 61, 2, 12, 2913, 0, 0, 0, 0, 0, 0, 0, 543, 159, 115, 2661, 643, 1, 54, 63, 2236, 0, 0, 0, 0, 0, 0, 0, 414, 0, 4029, 0, 1174, 4096, 1, 0, 729, 489, 0, 0, 0, 0, 0, 0, 1472, 6, 0, 0, 4027, 0, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 827, 317, 37, 0, 0, 424, 519, 102, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10899, 10719, 10205, 4588, 6475, 10932, 9273, 2445] avg=8192.0000 max=10932 max_rank=5 min=2445 min_rank=7 imbalance_ratio=1.334473 max_load_reduction=54604 max_load_reduction_pct=0.833191 moved_to_echo=29125 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 4352, 18688, 512, 17792, 12544, 768, 128, 7168, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 384, 19328, 1408, 1664, 128, 6784, 17408, 9984, 9088, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5376, 10240, 3328, 20352, 6912, 2944, 1536, 5504, 4224, 3968, 1792, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12032, 0, 26496, 11776, 9472, 6016, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[25600, 12160, 128, 19840, 5760, 1920, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 3328, 4608, 4992, 14208, 512, 11904, 3840, 22656, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[26240, 12032, 0, 128, 14080, 256, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5888, 18560, 5760, 2688, 768, 3584, 11776, 896, 16256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[3761, 2, 24, 108, 49, 4091, 3956, 4096, 59, 363, 0, 3937, 1, 3059, 0, 0, 283, 2135, 0, 4079, 4095, 82, 0, 0, 0, 1, 0, 241, 0, 1, 151, 4002, 0, 4045, 3942, 2776, 2, 6, 0, 404, 0, 0, 0, 0, 18, 0, 2, 0, 0, 326, 0, 6, 3204, 0, 116, 0, 0, 23, 3943, 134, 4008, 0, 4001, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 563, 0, 0, 4072, 220, 3682, 350, 4075, 0, 4, 1, 0, 583, 0, 67, 0, 1, 84, 3499, 3133, 0, 78, 1559, 727, 194, 0, 0, 33, 80, 0, 4095, 0, 0, 761, 0, 0, 0, 341, 66, 193, 2289, 0, 4018, 3, 0, 1, 2852, 341, 2704, 156, 0, 3822, 1935, 2, 0, 3336, 0, 4088, 12, 3237, 0, 4096, 87] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4093, 449, 0, 7, 3847, 0, 0, 3434, 49, 0, 4, 488, 1339, 3303, 34, 6, 0, 0, 0, 201, 0, 0, 0, 2486, 646, 0, 307, 0, 106, 115, 538, 3950, 0, 0, 2695, 4083, 4090, 4030, 0, 2, 6, 0, 2858, 243, 0, 133, 56, 149, 0, 0, 4096, 0, 3743, 0, 20, 0, 12, 3629, 4096, 0, 4085, 0, 2108, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4090, 110, 3985, 0, 4048, 0, 1255, 25, 0, 2, 27, 3972, 45, 4087, 0, 21, 0, 34, 0, 4027, 4090, 0, 0, 3903, 0, 0, 0, 0, 3, 140, 0, 1578, 0, 76, 77, 4087, 165, 2, 26, 24, 0, 0, 463, 3620, 0, 3, 0, 0, 0, 55, 1709, 68, 3965, 2748, 0, 0, 0, 2382, 3874, 0, 4096, 0, 2654, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[7, 3, 219, 6, 89, 178, 0, 137, 4096, 4051, 0, 4021, 0, 3648, 0, 90, 3881, 0, 96, 4092, 4096, 0, 0, 4054, 0, 1, 0, 21, 0, 0, 22, 109, 0, 0, 4030, 3966, 3, 340, 1, 0, 0, 0, 0, 3799, 114, 0, 12, 0, 0, 4079, 11, 2956, 3955, 2338, 0, 0, 0, 1, 3938, 0, 2466, 0, 564, 46] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[2, 1, 1185, 181, 4096, 0, 0, 45, 27, 219, 0, 1, 0, 4034, 0, 87, 1, 0, 0, 4066, 4096, 101, 0, 4027, 0, 1, 0, 4081, 0, 0, 92, 1059, 0, 3876, 0, 4094, 39, 13, 74, 45, 0, 0, 0, 3609, 0, 0, 3, 44, 0, 4021, 2696, 135, 3296, 4086, 0, 0, 0, 4094, 3741, 0, 4045, 0, 2, 221] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 3688, 221, 0, 3479, 3842, 4085, 1806, 4060, 3, 754, 3029, 1, 3069, 0, 111, 0, 0, 0, 3754, 4081, 0, 2, 3359, 71, 0, 0, 0, 5, 4, 1, 4017, 0, 0, 2, 174, 6, 0, 9, 0, 3228, 0, 415, 39, 0, 2, 0, 0, 206, 346, 458, 868, 4094, 4038, 0, 0, 0, 0, 4093, 0, 23, 0, 4093, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[18, 229, 3768, 0, 0, 3718, 964, 3171, 4096, 68, 0, 6, 0, 3357, 0, 227, 5, 552, 0, 3660, 2859, 0, 902, 3723, 0, 36, 3750, 4, 37, 0, 606, 2854, 0, 2, 3215, 0, 117, 0, 45, 4015, 46, 273, 454, 0, 2405, 0, 25, 8, 0, 115, 0, 4, 4017, 4057, 0, 0, 85, 31, 4096, 0, 3879, 0, 0, 37] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2747, 2, 24, 108, 0, 4091, 3956, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 59, 363, 0, 3937, 1, 3059, 0, 0, 223, 0, 0, 0, 0, 0, 0, 0, 283, 2135, 0, 1153, 4095, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 241, 0, 1, 151, 4002, 2926, 24, 305, 0, 0, 0, 0, 0, 0, 4045, 3942, 2776, 2, 6, 0, 404, 680, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 2, 0, 2947, 25, 0, 0, 0, 0, 0, 0, 0, 326, 0, 6, 2219, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 996, 134, 4008, 0, 4001, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15024, 7642, 7748, 7651, 12646, 2992, 2667, 9166] avg=8192.0000 max=15024 max_rank=0 min=2667 min_rank=6 imbalance_ratio=1.833984 max_load_reduction=50512 max_load_reduction_pct=0.770752 moved_to_echo=27471 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2992, 563, 0, 0, 0, 220, 3682, 350, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4, 1, 0, 583, 0, 67, 243, 0, 0, 0, 0, 0, 0, 0, 0, 1, 84, 989, 3133, 0, 78, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 727, 194, 0, 0, 33, 80, 0, 4095, 2510, 1934, 363, 0, 0, 0, 0, 0, 0, 0, 761, 0, 0, 0, 341, 66, 811, 861, 0, 0, 0, 0, 0, 0, 193, 2289, 0, 4018, 3, 0, 1, 2852, 3056, 2138, 0, 0, 0, 0, 0, 0, 341, 2704, 156, 0, 2648, 1935, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3336, 0, 1032, 12, 3237, 0, 4096, 87, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7807, 4973, 5844, 9936, 2840, 14550, 7786, 11800] avg=8192.0000 max=14550 max_rank=5 min=2840 min_rank=4 imbalance_ratio=1.776123 max_load_reduction=50986 max_load_reduction_pct=0.777985 moved_to_echo=36976 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2989, 449, 0, 7, 0, 0, 0, 3434, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 4, 488, 1339, 3303, 34, 6, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 646, 0, 307, 0, 106, 115, 538, 3950, 144, 1827, 356, 0, 0, 0, 0, 0, 0, 0, 2695, 4083, 4090, 4030, 0, 2, 794, 861, 0, 0, 0, 0, 0, 0, 6, 0, 2858, 243, 0, 133, 56, 149, 3062, 2020, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2593, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3629, 1034, 0, 4085, 0, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6879, 5466, 2543, 7989, 16555, 8527, 6709, 10868] avg=8192.0000 max=16555 max_rank=4 min=2543 min_rank=2 imbalance_ratio=2.020874 max_load_reduction=48981 max_load_reduction_pct=0.747391 moved_to_echo=42659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2987, 110, 3985, 0, 0, 0, 1255, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 27, 3972, 45, 4087, 0, 21, 243, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 1139, 4090, 0, 0, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 140, 0, 1578, 2888, 1924, 377, 0, 0, 0, 0, 0, 0, 76, 77, 4087, 165, 2, 26, 24, 841, 860, 0, 0, 0, 0, 0, 0, 0, 0, 463, 3620, 0, 3, 0, 0, 2896, 2124, 0, 0, 0, 0, 0, 0, 0, 55, 1709, 68, 2747, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2382, 978, 0, 4096, 0, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8362, 8397, 9166, 6910, 6158, 9106, 7327, 10110] avg=8192.0000 max=10110 max_rank=7 min=6158 min_rank=4 imbalance_ratio=1.234131 max_load_reduction=55426 max_load_reduction_pct=0.845734 moved_to_echo=32701 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[6, 3, 219, 6, 0, 178, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4051, 0, 4021, 0, 3648, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 96, 1157, 4096, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 21, 0, 0, 22, 109, 2935, 42, 376, 0, 0, 0, 0, 0, 0, 0, 4030, 3966, 3, 340, 1, 0, 839, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3799, 114, 0, 12, 0, 2943, 47, 0, 0, 0, 0, 0, 0, 0, 4079, 11, 2956, 2740, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 995, 0, 2466, 0, 564, 46, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[549, 15906, 13284, 3506, 9180, 6915, 12124, 4072] avg=8192.0000 max=15906 max_rank=1 min=549 min_rank=0 imbalance_ratio=1.941650 max_load_reduction=49630 max_load_reduction_pct=0.757294 moved_to_echo=32291 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1185, 181, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 27, 219, 0, 1, 0, 4034, 0, 87, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1146, 4096, 101, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4081, 0, 0, 92, 1059, 2920, 1947, 318, 0, 0, 0, 0, 0, 0, 3876, 0, 4094, 39, 13, 74, 45, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3609, 0, 0, 3, 44, 2799, 2149, 0, 0, 0, 0, 0, 0, 0, 4021, 2696, 135, 2274, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 942, 0, 4045, 0, 2, 221, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1412, 4370, 9371, 10418, 8845, 8604, 13212, 9304] avg=8192.0000 max=13212 max_rank=6 min=1412 min_rank=0 imbalance_ratio=1.612793 max_load_reduction=52324 max_load_reduction_pct=0.798401 moved_to_echo=39965 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3688, 221, 0, 0, 3842, 4085, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 3, 754, 3029, 1, 3069, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 4081, 0, 2, 3359, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 5, 4, 1, 4017, 2692, 1653, 389, 0, 0, 0, 0, 0, 0, 0, 2, 174, 6, 0, 9, 0, 869, 0, 0, 0, 0, 0, 0, 0, 3228, 0, 415, 39, 0, 2, 0, 0, 3059, 1826, 0, 0, 0, 0, 0, 0, 206, 346, 458, 868, 2836, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 0, 23, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13642, 11027, 8504, 8832, 1060, 8569, 8752, 5150] avg=8192.0000 max=13642 max_rank=0 min=1060 min_rank=4 imbalance_ratio=1.665283 max_load_reduction=51894 max_load_reduction_pct=0.791840 moved_to_echo=23531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[11, 229, 3768, 0, 0, 3718, 964, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 68, 0, 6, 0, 3357, 0, 227, 1, 0, 0, 0, 0, 0, 0, 0, 5, 552, 0, 1035, 2859, 0, 902, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3750, 4, 37, 0, 606, 2854, 2625, 0, 382, 0, 0, 0, 0, 0, 0, 2, 3215, 0, 117, 0, 45, 4015, 852, 6, 0, 0, 0, 0, 0, 0, 46, 273, 454, 0, 2405, 0, 25, 8, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 4, 2783, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 31, 1034, 0, 3879, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11861, 7755, 9076, 10294, 8252, 6273, 6959, 5066] avg=8192.0000 max=11861 max_rank=0 min=5066 min_rank=7 imbalance_ratio=1.447876 max_load_reduction=53675 max_load_reduction_pct=0.819016 moved_to_echo=26550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11776, 5120, 9472, 384, 0, 12160, 13952, 13184, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 8064, 14848, 19200, 4480, 4480, 512, 4608, 6400, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3584, 2688, 4224, 15360, 2560, 256, 128, 3072, 23936, 10368, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4224, 2816, 256, 7808, 26496, 256, 1024, 23168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3456, 10240, 8064, 256, 25856, 0, 17536, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1536, 256, 4096, 4352, 256, 384, 1536, 21760, 19712, 9472, 2944, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 11648, 9216, 4096, 20864, 19328, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[16512, 4736, 896, 15488, 1408, 25216, 128, 640, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[914, 0, 6, 4083, 0, 716, 94, 13, 0, 5, 6, 0, 36, 23, 0, 68, 0, 0, 0, 3982, 1633, 0, 2, 3344, 12, 0, 4030, 0, 532, 3959, 2856, 0, 1, 4093, 9, 0, 0, 0, 2831, 0, 0, 4096, 3578, 0, 31, 4038, 4096, 4055, 0, 4092, 0, 0, 3, 7, 313, 3850, 0, 0, 0, 0, 20, 13, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 734, 0, 0, 0, 1, 0, 4, 4065, 44, 3860, 0, 8, 339, 2070, 2711, 0, 4039, 4083, 0, 4055, 2466, 2781, 0, 0, 20, 0, 0, 0, 3142, 17, 852, 0, 3806, 280, 491, 4095, 444, 42, 20, 2857, 4087, 4070, 0, 0, 4, 81, 3590, 0, 154, 51, 204, 139, 0, 8, 1616, 108, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[363, 0, 4096, 3487, 0, 6, 0, 0, 0, 402, 46, 0, 0, 3929, 0, 13, 0, 0, 2, 3771, 1645, 0, 3771, 4094, 0, 3921, 0, 137, 304, 0, 360, 0, 0, 11, 3985, 55, 0, 0, 1089, 2, 34, 1960, 4049, 48, 230, 0, 1923, 3994, 0, 0, 3684, 3850, 2970, 2383, 0, 2839, 0, 150, 0, 1, 143, 2, 0, 1787] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3895, 0, 3560, 77, 0, 0, 0, 0, 2, 0, 0, 4090, 0, 4079, 34, 2, 0, 0, 0, 2571, 0, 0, 1399, 4096, 0, 4096, 4095, 0, 3500, 199, 2122, 0, 0, 0, 6, 0, 6, 0, 4096, 0, 0, 4075, 4096, 3, 104, 0, 4096, 3991, 0, 0, 0, 7, 72, 49, 26, 4096, 0, 0, 0, 0, 0, 1, 2097, 898] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3235, 397, 0, 0, 3916, 0, 3327, 0, 3, 0, 3, 3840, 0, 0, 0, 0, 20, 74, 4096, 0, 4092, 3651, 3, 389, 13, 0, 127, 388, 1262, 0, 0, 3887, 394, 3165, 0, 0, 4095, 0, 18, 4096, 152, 0, 0, 42, 4096, 18, 0, 0, 0, 2787, 4093, 743, 33, 0, 0, 0, 0, 0, 4090, 2, 3836, 1153] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[101, 0, 3974, 165, 51, 0, 567, 30, 4045, 471, 0, 0, 0, 3531, 67, 0, 1, 0, 73, 8, 4096, 0, 11, 2864, 0, 3910, 4057, 0, 34, 0, 0, 0, 902, 1196, 0, 334, 310, 1, 1910, 0, 1337, 4096, 2979, 1, 81, 4096, 1302, 0, 0, 0, 4018, 0, 3760, 2903, 9, 1, 93, 0, 0, 0, 4092, 0, 0, 4059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4077, 0, 3739, 79, 0, 0, 3, 0, 1, 0, 3520, 3948, 0, 4096, 0, 303, 0, 0, 38, 259, 4015, 0, 4092, 4096, 0, 4096, 3879, 0, 0, 106, 0, 0, 0, 3849, 627, 61, 51, 0, 4096, 338, 0, 4045, 182, 11, 0, 0, 1, 4087, 0, 0, 0, 5, 4084, 0, 1, 12, 0, 0, 0, 0, 0, 8, 3731, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[3924, 0, 5, 0, 0, 2, 2264, 0, 1, 78, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 3877, 0, 4093, 3354, 907, 0, 3988, 0, 491, 89, 610, 0, 3, 4089, 2600, 0, 0, 0, 2316, 0, 182, 4096, 0, 0, 82, 0, 4096, 353, 0, 4033, 6, 0, 4093, 1469, 2814, 0, 0, 0, 0, 0, 173, 10, 3487, 66] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[914, 0, 6, 4083, 0, 716, 94, 13, 1382, 109, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 36, 23, 0, 68, 1412, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 1633, 0, 2, 550, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4030, 0, 532, 3959, 2856, 0, 0, 50, 0, 0, 0, 0, 0, 0, 1, 4093, 9, 0, 0, 0, 2831, 0, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 31, 4038, 2722, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3, 7, 313, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 13, 0, 4096, 3947, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7317, 1699, 6167, 11439, 8149, 14424, 8265, 8076] avg=8192.0000 max=14424 max_rank=5 min=1699 min_rank=1 imbalance_ratio=1.760742 max_load_reduction=51112 max_load_reduction_pct=0.779907 moved_to_echo=38914 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[363, 0, 4096, 3487, 0, 6, 0, 0, 1693, 51, 0, 0, 0, 0, 0, 0, 0, 402, 46, 0, 0, 3929, 0, 13, 1728, 71, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3771, 1645, 0, 3771, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 137, 304, 0, 360, 0, 189, 23, 0, 0, 0, 0, 0, 0, 0, 11, 3985, 55, 0, 0, 1089, 2, 570, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4049, 48, 230, 0, 1279, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3684, 3850, 2781, 2383, 0, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 1, 143, 2, 0, 1787, 1889, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9696, 6189, 9862, 4934, 5712, 9634, 15537, 3972] avg=8192.0000 max=15537 max_rank=6 min=3972 min_rank=7 imbalance_ratio=1.896606 max_load_reduction=49999 max_load_reduction_pct=0.762924 moved_to_echo=34855 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 734, 0, 1688, 109, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 4065, 44, 3860, 1724, 148, 0, 0, 0, 0, 0, 0, 0, 8, 339, 2070, 2711, 0, 4039, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 2466, 2781, 0, 0, 20, 0, 229, 50, 0, 0, 0, 0, 0, 0, 0, 0, 3142, 17, 852, 0, 3806, 280, 1212, 0, 0, 0, 0, 0, 0, 0, 491, 0, 444, 42, 20, 2857, 2716, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 81, 3361, 0, 154, 51, 0, 0, 0, 0, 0, 0, 0, 0, 204, 139, 0, 8, 1616, 108, 2, 0, 3947, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6627, 9846, 9838, 9601, 9309, 10640, 3651, 6024] avg=8192.0000 max=10640 max_rank=5 min=3651 min_rank=6 imbalance_ratio=1.298828 max_load_reduction=54896 max_load_reduction_pct=0.837646 moved_to_echo=29624 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3895, 0, 3560, 77, 0, 0, 0, 0, 1693, 109, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4090, 0, 4079, 34, 2, 1729, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2571, 0, 0, 1399, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 3500, 199, 2122, 0, 4, 50, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 4096, 0, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 104, 0, 2722, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 68, 49, 26, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2097, 898, 3927, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9334, 10084, 4644, 14066, 5323, 10916, 4246, 6923] avg=8192.0000 max=14066 max_rank=3 min=4246 min_rank=6 imbalance_ratio=1.717041 max_load_reduction=51470 max_load_reduction_pct=0.785370 moved_to_echo=27408 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3235, 397, 0, 0, 3916, 0, 1509, 109, 0, 0, 0, 0, 0, 0, 3327, 0, 3, 0, 3, 3840, 0, 0, 1541, 148, 0, 0, 0, 0, 0, 0, 0, 0, 20, 74, 4096, 0, 4092, 601, 0, 0, 0, 0, 0, 0, 0, 0, 3, 389, 13, 0, 127, 388, 1262, 0, 261, 50, 0, 0, 0, 0, 0, 0, 0, 3887, 394, 3165, 0, 0, 4095, 0, 1215, 0, 0, 0, 0, 0, 0, 0, 18, 0, 152, 0, 0, 42, 2722, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2787, 3832, 743, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 3836, 1153, 3948, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9166, 8862, 8883, 2493, 12756, 2952, 7395, 13029] avg=8192.0000 max=13029 max_rank=7 min=2493 min_rank=3 imbalance_ratio=1.590454 max_load_reduction=52507 max_load_reduction_pct=0.801193 moved_to_echo=36132 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4077, 0, 3739, 79, 0, 0, 3, 0, 1693, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3520, 3948, 0, 4096, 0, 303, 1729, 146, 0, 0, 0, 0, 0, 0, 0, 0, 38, 259, 4015, 0, 4092, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3879, 0, 0, 106, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 627, 61, 51, 0, 4096, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 11, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3823, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3731, 0, 3899, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9591, 13743, 9078, 8342, 9022, 4281, 3841, 7638] avg=8192.0000 max=13743 max_rank=1 min=3841 min_rank=6 imbalance_ratio=1.677612 max_load_reduction=51793 max_load_reduction_pct=0.790298 moved_to_echo=24782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3924, 0, 5, 0, 0, 2, 2264, 0, 1392, 112, 0, 0, 0, 0, 0, 0, 1, 78, 3993, 0, 0, 0, 0, 0, 1420, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 3877, 0, 4093, 542, 0, 0, 0, 0, 0, 0, 0, 0, 907, 0, 3988, 0, 491, 89, 610, 0, 265, 54, 0, 0, 0, 0, 0, 0, 3, 4089, 2600, 0, 0, 0, 2316, 0, 1218, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 82, 0, 2712, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 6, 0, 3828, 1469, 2814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 10, 3487, 66, 3947, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7699, 5641, 12404, 6404, 10226, 3329, 12150, 7683] avg=8192.0000 max=12404 max_rank=2 min=3329 min_rank=5 imbalance_ratio=1.514160 max_load_reduction=53132 max_load_reduction_pct=0.810730 moved_to_echo=33388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[101, 0, 3974, 165, 51, 0, 567, 30, 1184, 34, 0, 0, 0, 0, 0, 0, 4045, 471, 0, 0, 0, 3531, 67, 0, 1209, 149, 0, 0, 0, 0, 0, 0, 1, 0, 73, 8, 4096, 0, 11, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 4057, 0, 34, 0, 0, 0, 240, 16, 0, 0, 0, 0, 0, 0, 902, 1196, 0, 334, 310, 1, 1910, 0, 386, 0, 0, 0, 0, 0, 0, 0, 1337, 0, 2979, 1, 81, 4096, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 3520, 2903, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 4092, 0, 0, 4059, 3947, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6106, 9472, 4660, 8257, 5039, 9360, 10451, 12191] avg=8192.0000 max=12191 max_rank=7 min=4660 min_rank=2 imbalance_ratio=1.488159 max_load_reduction=53345 max_load_reduction_pct=0.813980 moved_to_echo=37041 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2176, 0, 15488, 128, 640, 11136, 15744, 20608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[7424, 1024, 7680, 8064, 128, 23680, 256, 4352, 12544, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 8192, 7808, 6784, 21248, 7680, 3456, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 512, 16640, 22144, 0, 21504, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1024, 20480, 22528, 2944, 4992, 4864, 7296, 0, 1536, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 17152, 10880, 3712, 1280, 128, 24320, 640, 7040, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[13312, 0, 22784, 8320, 128, 768, 7680, 128, 12288, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 384, 0, 128, 10240, 256, 13184, 12160, 29568, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[2940, 4096, 829, 0, 0, 6, 0, 0, 4096, 0, 0, 3544, 0, 0, 277, 4096, 24, 0, 40, 0, 853, 0, 0, 2013, 483, 3439, 0, 2, 0, 1693, 259, 556, 0, 0, 0, 4096, 3667, 9, 4031, 20, 0, 0, 0, 0, 18, 4033, 2753, 246, 3221, 179, 15, 2, 78, 0, 2713, 0, 0, 4019, 0, 2, 3149, 0, 0, 4039] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 579, 1304, 0, 8, 0, 4, 0, 131, 0, 3652, 9, 6, 5, 527, 3156, 3649, 152, 561, 0, 4090, 46, 1, 2630, 577, 468, 0, 3469, 0, 38, 1707, 4059, 0, 13, 0, 4086, 2604, 3515, 691, 68, 311, 0, 0, 17, 0, 159, 4096, 57, 3432, 45, 0, 0, 167, 0, 3693, 0, 0, 3928, 184, 0, 3548, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4094, 305, 0, 3, 0, 0, 0, 838, 10, 4, 1718, 0, 2, 293, 1803, 97, 4070, 0, 43, 5, 0, 3953, 0, 0, 1479, 98, 3983, 111, 298, 122, 744, 10, 0, 0, 4096, 3391, 0, 4095, 62, 1, 0, 0, 0, 0, 3733, 3717, 22, 4093, 71, 0, 0, 0, 29, 4096, 0, 2470, 4096, 185, 36, 3168, 0, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 30, 4081, 3750, 0, 3, 0, 0, 0, 1, 3278, 13, 1456, 0, 2614, 0, 3939, 1, 0, 715, 0, 901, 0, 45, 4003, 0, 0, 0, 0, 962, 4096, 0, 0, 0, 4031, 64, 0, 3217, 17, 10, 858, 0, 0, 0, 0, 2611, 1200, 4019, 19, 1, 0, 15, 693, 4056, 0, 2, 4094, 53, 2765, 3462, 0, 396, 4065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 59, 106, 0, 0, 1, 0, 0, 4091, 4064, 4071, 4096, 0, 3477, 17, 4095, 38, 3633, 4, 4, 0, 0, 0, 514, 160, 0, 0, 542, 0, 7, 3843, 4012, 28, 0, 0, 185, 4092, 2, 4096, 0, 0, 6, 0, 0, 0, 687, 207, 2433, 566, 3859, 0, 0, 17, 0, 4077, 0, 0, 3855, 0, 4096, 1, 0, 0, 495] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 1443, 580, 780, 1771, 0, 5, 0, 0, 0, 1, 3969, 11, 0, 0, 4083, 0, 0, 0, 0, 0, 205, 0, 0, 4034, 4, 0, 3876, 99, 580, 3295, 0, 0, 142, 223, 1647, 4093, 4096, 3979, 2, 4093, 5, 2, 0, 21, 4094, 0, 4, 1, 4090, 4056, 12, 1055, 0, 2863, 0, 106, 3007, 1, 0, 3203, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 0, 4043, 70, 0, 0, 207, 659, 0, 1996, 3772, 5, 82, 0, 423, 3973, 4050, 0, 643, 9, 0, 0, 0, 0, 4084, 377, 0, 3701, 0, 4070, 28, 1, 0, 0, 160, 4091, 0, 4084, 4066, 2, 0, 0, 0, 0, 0, 288, 9, 1636, 3740, 0, 0, 0, 76, 4096, 0, 0, 3931, 18, 4064, 19, 0, 63, 2998] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[24, 3849, 1722, 0, 17, 2, 0, 0, 150, 0, 0, 497, 0, 0, 4084, 165, 0, 3897, 474, 4094, 0, 0, 857, 4087, 0, 3578, 0, 3989, 0, 3, 2, 187, 2291, 0, 0, 4096, 4076, 65, 4096, 0, 0, 1193, 368, 0, 0, 28, 85, 0, 3337, 0, 825, 435, 5, 0, 3412, 0, 0, 3628, 1777, 60, 4081, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 579, 1304, 0, 8, 0, 4, 0, 631, 381, 0, 0, 0, 0, 0, 0, 131, 0, 3652, 5, 6, 5, 527, 3156, 0, 0, 0, 0, 0, 0, 0, 0, 3649, 152, 561, 0, 4090, 46, 1, 2630, 209, 414, 0, 0, 0, 0, 0, 0, 577, 259, 0, 3469, 0, 38, 1707, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4086, 2604, 3515, 60, 68, 0, 0, 0, 0, 0, 0, 0, 0, 311, 0, 0, 17, 0, 159, 4096, 57, 2410, 4, 0, 0, 0, 0, 0, 0, 3432, 45, 0, 0, 167, 0, 3279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 184, 0, 3548, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2907, 7482, 11752, 10109, 10346, 7054, 6923, 8963] avg=8192.0000 max=11752 max_rank=2 min=2907 min_rank=0 imbalance_ratio=1.434570 max_load_reduction=53784 max_load_reduction_pct=0.820679 moved_to_echo=33286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2940, 4096, 829, 0, 0, 6, 0, 0, 3684, 389, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1882, 0, 0, 277, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 40, 0, 853, 0, 0, 2013, 1537, 304, 65, 0, 0, 0, 0, 0, 483, 1902, 0, 2, 0, 1693, 259, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3667, 9, 347, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4033, 2753, 246, 2466, 1597, 0, 0, 0, 0, 0, 0, 3221, 179, 15, 2, 78, 0, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1164, 0, 2, 3149, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11944, 10351, 4836, 4895, 8139, 11113, 5904, 8354] avg=8192.0000 max=11944 max_rank=0 min=4836 min_rank=2 imbalance_ratio=1.458008 max_load_reduction=53592 max_load_reduction_pct=0.817749 moved_to_echo=33510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 30, 4081, 3750, 0, 3, 0, 2940, 397, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1741, 13, 1456, 0, 2614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 1, 0, 715, 0, 901, 0, 1789, 455, 60, 0, 0, 0, 0, 0, 45, 2214, 0, 0, 0, 0, 962, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 64, 0, 277, 17, 0, 0, 0, 0, 0, 0, 0, 0, 10, 858, 0, 0, 0, 0, 2611, 1200, 2512, 1477, 0, 0, 0, 0, 0, 0, 4019, 19, 1, 0, 15, 693, 3601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1185, 53, 2765, 3462, 0, 396, 4065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11201, 5825, 7860, 7317, 4389, 8668, 8348, 11928] avg=8192.0000 max=11928 max_rank=7 min=4389 min_rank=4 imbalance_ratio=1.456055 max_load_reduction=53608 max_load_reduction_pct=0.817993 moved_to_echo=33333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 305, 0, 3, 0, 0, 0, 3743, 397, 0, 0, 0, 0, 0, 0, 838, 10, 4, 913, 0, 2, 293, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 97, 4070, 0, 43, 5, 0, 3953, 0, 661, 460, 31, 0, 0, 0, 0, 0, 0, 818, 98, 3983, 111, 298, 122, 744, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 3391, 0, 352, 62, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3733, 3717, 22, 2513, 774, 0, 0, 0, 0, 0, 0, 4093, 71, 0, 0, 0, 29, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2470, 1186, 185, 36, 3168, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8542, 3863, 9320, 6174, 7911, 10760, 7829, 11137] avg=8192.0000 max=11137 max_rank=7 min=3863 min_rank=1 imbalance_ratio=1.359497 max_load_reduction=54399 max_load_reduction_pct=0.830063 moved_to_echo=37637 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 59, 106, 0, 0, 1, 0, 0, 3744, 374, 0, 0, 0, 0, 0, 0, 4091, 4064, 4071, 2175, 0, 3477, 17, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3633, 4, 4, 0, 0, 0, 514, 0, 458, 75, 0, 0, 0, 0, 0, 160, 0, 0, 542, 0, 7, 3843, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 185, 4092, 2, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 687, 207, 2433, 2365, 1846, 0, 0, 0, 0, 0, 0, 566, 3859, 0, 0, 17, 0, 3619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 0, 4096, 1, 0, 0, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4284, 21990, 4726, 8564, 4659, 7544, 8061, 5708] avg=8192.0000 max=21990 max_rank=1 min=4284 min_rank=0 imbalance_ratio=2.684326 max_load_reduction=43546 max_load_reduction_pct=0.664459 moved_to_echo=25972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1443, 580, 780, 1771, 0, 5, 0, 3637, 291, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2107, 11, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 1, 321, 73, 0, 0, 0, 0, 0, 4034, 3, 0, 3876, 99, 580, 3295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 223, 1647, 4093, 4096, 342, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 2, 0, 21, 4094, 0, 4, 1845, 1789, 0, 0, 0, 0, 0, 0, 1, 4090, 4056, 12, 1055, 0, 2542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 871, 1, 0, 3203, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8507, 6202, 600, 11887, 10545, 11853, 11756, 4186] avg=8192.0000 max=11887 max_rank=3 min=600 min_rank=2 imbalance_ratio=1.451050 max_load_reduction=53649 max_load_reduction_pct=0.818619 moved_to_echo=38340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4043, 70, 0, 0, 207, 3733, 381, 0, 0, 0, 0, 0, 0, 659, 0, 1996, 2003, 5, 82, 0, 423, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 4050, 0, 643, 9, 0, 0, 0, 1826, 460, 69, 0, 0, 0, 0, 0, 0, 2258, 377, 0, 3701, 0, 4070, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 160, 4091, 0, 351, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 288, 9, 2412, 1700, 0, 0, 0, 0, 0, 0, 1636, 3740, 0, 0, 0, 76, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1138, 18, 4064, 19, 0, 63, 2998, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8436, 5168, 11030, 10434, 8669, 4411, 9088, 8300] avg=8192.0000 max=11030 max_rank=2 min=4411 min_rank=5 imbalance_ratio=1.346436 max_load_reduction=54506 max_load_reduction_pct=0.831696 moved_to_echo=30468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[24, 3849, 1722, 0, 17, 2, 0, 0, 3746, 355, 0, 0, 0, 0, 0, 0, 150, 0, 0, 256, 0, 0, 4084, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 474, 4094, 0, 0, 857, 4087, 1603, 387, 13, 0, 0, 0, 0, 0, 0, 1975, 0, 3989, 0, 3, 2, 187, 0, 0, 0, 0, 0, 0, 0, 0, 2291, 0, 0, 4096, 4076, 65, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 368, 0, 0, 28, 85, 0, 2231, 228, 0, 0, 0, 0, 0, 0, 3337, 0, 825, 435, 5, 0, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1777, 60, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9715, 4655, 15412, 6156, 10878, 4133, 7627, 6960] avg=8192.0000 max=15412 max_rank=2 min=4133 min_rank=5 imbalance_ratio=1.881348 max_load_reduction=50124 max_load_reduction_pct=0.764832 moved_to_echo=29598 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[7808, 19840, 1152, 4864, 5760, 256, 5760, 9344, 7680, 3328, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4480, 2176, 384, 128, 128, 12800, 13824, 4096, 18816, 9472, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[3072, 14208, 4992, 8960, 5632, 128, 128, 256, 25984, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[20352, 12032, 4992, 512, 1408, 896, 25856, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2688, 8960, 2304, 11136, 20736, 0, 512, 19840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[9984, 4096, 9728, 11136, 128, 5120, 5248, 20480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[5376, 9472, 512, 15872, 3968, 2688, 14336, 13696, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2432, 256, 256, 22400, 26112, 7808, 2432, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4054, 24, 1823, 0, 4096, 7, 4096, 37, 4080, 1, 0, 0, 212, 156, 0, 1, 8, 1590, 3352, 0, 18, 360, 124, 576, 318, 4, 2831, 3978, 247, 8, 0, 4088, 0, 0, 0, 3521, 1, 1, 48, 4094, 3669, 0, 3368, 0, 2443, 5, 0, 0, 1, 87, 4016, 0, 118, 0, 0, 2226, 4049, 0, 0, 1800, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1193, 2, 0, 0, 0, 2732, 4030, 152, 3072, 0, 10, 0, 0, 0, 0, 0, 9, 1, 0, 2325, 0, 4049, 0, 4011, 0, 3666, 873, 2, 3975, 36, 4095, 189, 0, 4084, 103, 0, 24, 244, 0, 4082, 0, 7, 3941, 3851, 0, 0, 0, 4062, 0, 0, 0, 45, 0, 3924, 0, 14, 94, 73, 48, 207, 0, 2216, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[25, 1, 5, 0, 0, 3951, 0, 3145, 15, 941, 2801, 516, 4050, 0, 0, 428, 86, 3852, 0, 0, 3, 4063, 0, 2140, 0, 0, 52, 1, 2362, 0, 4059, 334, 4015, 3935, 0, 0, 3, 837, 2859, 4094, 0, 207, 741, 5, 24, 2904, 0, 7, 0, 14, 0, 0, 9, 3431, 0, 95, 0, 0, 4019, 0, 1486, 0, 4021, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 8, 0, 67, 0, 1011, 3910, 4086, 0, 4079, 575, 0, 488, 0, 0, 61, 60, 140, 3483, 0, 0, 4037, 0, 155, 232, 2787, 0, 1909, 0, 0, 4096, 0, 0, 12, 0, 10, 0, 1, 4072, 4075, 0, 0, 4069, 3709, 1, 207, 0, 3657, 3, 0, 0, 45, 0, 4066, 0, 0, 0, 0, 4094, 2221, 20, 0, 4090, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 3797, 2252, 417, 3595, 4054, 0, 4096, 4, 4096, 4080, 14, 2, 0, 0, 107, 75, 3961, 0, 2, 0, 598, 9, 4096, 0, 3306, 8, 34, 376, 4096, 29, 0, 14, 2134, 435, 0, 0, 0, 3942, 4093, 436, 0, 212, 4082, 0, 29, 0, 0, 0, 0, 0, 81, 41, 0, 172, 0, 0, 0, 2924, 2, 3823, 0, 12, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 231, 0, 1, 0, 4096, 0, 2196, 0, 2822, 16, 0, 34, 0, 0, 0, 0, 3392, 994, 0, 0, 2726, 98, 2871, 0, 566, 0, 3934, 789, 0, 4096, 0, 0, 2315, 39, 23, 0, 4045, 1, 4081, 0, 0, 4025, 4096, 811, 1662, 0, 3723, 2, 0, 0, 339, 1197, 269, 3898, 0, 79, 1, 2971, 0, 69, 318, 2710, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 1, 3545, 2119, 4078, 45, 4096, 0, 127, 10, 0, 0, 3867, 275, 0, 523, 66, 107, 3419, 544, 4089, 0, 1, 3052, 1010, 3892, 3, 2556, 1829, 1, 0, 4, 0, 630, 0, 3895, 2867, 0, 14, 4091, 1764, 0, 2455, 0, 1, 0, 1, 0, 42, 0, 171, 0, 29, 1802, 0, 0, 0, 1229, 0, 2592, 598] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 28, 0, 0, 4095, 8, 4096, 464, 3580, 1, 0, 0, 0, 8, 2184, 0, 2, 3587, 4033, 0, 4096, 0, 2924, 3, 13, 4096, 11, 1862, 0, 0, 0, 4006, 0, 199, 0, 4096, 4090, 0, 0, 3835, 0, 0, 4093, 0, 0, 0, 0, 0, 83, 70, 4087, 41, 0, 0, 0, 27, 0, 608, 0, 3996, 1211] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[25, 1, 5, 0, 0, 3951, 0, 1754, 0, 0, 0, 0, 0, 0, 0, 0, 15, 941, 2801, 516, 4050, 0, 0, 428, 1416, 611, 0, 0, 0, 0, 0, 0, 86, 3852, 0, 0, 3, 2555, 0, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1, 2362, 0, 2412, 334, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 3935, 0, 0, 3, 837, 2859, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 421, 5, 24, 2904, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 9, 3431, 0, 95, 2308, 1391, 320, 231, 0, 0, 0, 0, 0, 0, 4019, 0, 1486, 0, 4021, 0, 897, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5736, 10778, 8636, 5161, 13435, 3568, 7799, 10423] avg=8192.0000 max=13435 max_rank=4 min=3568 min_rank=5 imbalance_ratio=1.640015 max_load_reduction=52101 max_load_reduction_pct=0.794998 moved_to_echo=35225 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4054, 24, 1823, 0, 2284, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 37, 4080, 1, 0, 0, 212, 1388, 504, 0, 0, 0, 0, 0, 0, 156, 0, 1, 8, 1590, 2108, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 360, 124, 576, 318, 4, 2831, 2364, 247, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4088, 0, 0, 0, 3521, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 48, 2325, 3669, 0, 3368, 0, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 87, 4016, 0, 118, 0, 1812, 1769, 226, 0, 0, 0, 0, 0, 0, 2226, 4049, 0, 0, 1800, 0, 740, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 10325, 3881, 6824, 7618, 11854, 8034, 8815] avg=8192.0000 max=11854 max_rank=5 min=3881 min_rank=2 imbalance_ratio=1.447021 max_load_reduction=53682 max_load_reduction_pct=0.819122 moved_to_echo=36321 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1193, 2, 0, 0, 0, 2732, 4030, 85, 0, 0, 0, 0, 0, 0, 0, 0, 3072, 0, 10, 0, 0, 0, 0, 0, 1429, 609, 0, 0, 0, 0, 0, 0, 9, 1, 0, 2325, 0, 2546, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3666, 873, 2, 3975, 36, 2433, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 103, 0, 24, 244, 0, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2238, 3851, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 3924, 0, 14, 2301, 67, 1703, 233, 0, 0, 0, 0, 94, 73, 48, 207, 0, 2216, 4095, 0, 894, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8042, 5120, 8892, 11174, 6236, 10158, 8287, 7627] avg=8192.0000 max=11174 max_rank=3 min=5120 min_rank=1 imbalance_ratio=1.364014 max_load_reduction=54362 max_load_reduction_pct=0.829498 moved_to_echo=32308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 67, 0, 1011, 3910, 2279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 575, 0, 488, 0, 0, 61, 1429, 607, 0, 0, 0, 0, 0, 0, 60, 140, 3483, 0, 0, 2539, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 232, 2787, 0, 1909, 0, 0, 2434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 10, 0, 1, 4072, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2311, 3709, 1, 207, 0, 3657, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 45, 0, 4066, 0, 0, 2298, 1807, 1758, 233, 0, 0, 0, 0, 0, 0, 4094, 2221, 20, 0, 4090, 0, 891, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7275, 7239, 6377, 7362, 5872, 9885, 10210, 11316] avg=8192.0000 max=11316 max_rank=7 min=5872 min_rank=4 imbalance_ratio=1.381348 max_load_reduction=54220 max_load_reduction_pct=0.827332 moved_to_echo=37283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3797, 2252, 417, 3595, 4054, 0, 2284, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4080, 14, 2, 0, 0, 107, 10, 89, 0, 0, 0, 0, 0, 0, 75, 3961, 0, 2, 0, 377, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3306, 8, 34, 376, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2134, 435, 0, 0, 0, 3942, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 436, 0, 121, 4082, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 41, 0, 172, 0, 2308, 1812, 91, 1, 0, 0, 0, 0, 0, 0, 2924, 2, 3823, 0, 12, 0, 132, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 8402, 8520, 7838, 8310, 4668, 4506, 6893] avg=8192.0000 max=16399 max_rank=0 min=4506 min_rank=6 imbalance_ratio=2.001831 max_load_reduction=49137 max_load_reduction_pct=0.749771 moved_to_echo=24377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 231, 0, 1, 0, 4096, 0, 1225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2822, 16, 0, 34, 0, 0, 0, 1429, 410, 0, 0, 0, 0, 0, 0, 0, 3392, 994, 0, 0, 1714, 98, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 0, 3934, 789, 0, 2434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2315, 39, 23, 0, 4045, 1, 1780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2286, 4096, 811, 1662, 0, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 339, 1197, 269, 3898, 0, 2301, 971, 1739, 233, 0, 0, 0, 0, 79, 1, 2971, 0, 69, 318, 2710, 0, 602, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5553, 4711, 9069, 7723, 8203, 12578, 10949, 6750] avg=8192.0000 max=12578 max_rank=5 min=4711 min_rank=1 imbalance_ratio=1.535400 max_load_reduction=52958 max_load_reduction_pct=0.808075 moved_to_echo=38480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 1, 3545, 2119, 2274, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4096, 0, 127, 10, 0, 0, 3867, 892, 514, 0, 0, 0, 0, 0, 0, 275, 0, 523, 66, 107, 2150, 544, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3052, 1010, 3892, 3, 1519, 1829, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 630, 0, 3895, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2323, 1764, 0, 2455, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 42, 0, 171, 0, 29, 1616, 1804, 1768, 145, 0, 0, 0, 0, 1802, 0, 0, 0, 1229, 0, 2592, 598, 755, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12035, 9551, 7754, 11306, 5781, 6557, 5576, 6976] avg=8192.0000 max=12035 max_rank=0 min=5576 min_rank=6 imbalance_ratio=1.469116 max_load_reduction=53501 max_load_reduction_pct=0.816360 moved_to_echo=24890 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 28, 0, 0, 2280, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 464, 3580, 1, 0, 0, 0, 652, 609, 0, 0, 0, 0, 0, 0, 8, 2184, 0, 2, 3587, 2530, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2924, 3, 13, 4096, 11, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 199, 0, 4096, 1780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2175, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 70, 4087, 41, 0, 2310, 1815, 1660, 109, 0, 0, 0, 0, 0, 0, 27, 0, 608, 0, 3996, 1211, 894, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2311, 9410, 12407, 8148, 10081, 6268, 10175, 6736] avg=8192.0000 max=12407 max_rank=2 min=2311 min_rank=0 imbalance_ratio=1.514526 max_load_reduction=53129 max_load_reduction_pct=0.810684 moved_to_echo=33260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1280, 4096, 6400, 4608, 3712, 21248, 10112, 14592, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 640, 1408, 19968, 4224, 256, 15488, 11520, 10880, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[512, 384, 14208, 21248, 896, 14720, 0, 13952, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2048, 128, 16384, 6528, 7296, 2560, 23424, 1920, 5888, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 13568, 5120, 2432, 5376, 16640, 768, 21504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4096, 12544, 8704, 128, 896, 5248, 22400, 12032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 13440, 4608, 7296, 15616, 7040, 14720, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3200, 24320, 8064, 8320, 4608, 0, 0, 4736, 8704, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[10, 3, 4093, 0, 63, 0, 1538, 32, 1834, 1, 1836, 0, 4096, 0, 478, 0, 0, 0, 1848, 0, 4014, 19, 23, 549, 38, 3806, 4096, 3567, 10, 0, 0, 969, 1843, 0, 6, 0, 2750, 62, 46, 4089, 0, 3719, 4005, 4096, 2048, 0, 935, 0, 65, 0, 172, 4070, 7, 0, 41, 0, 0, 0, 3865, 0, 0, 14, 727, 53] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3477, 0, 3912, 0, 4095, 912, 0, 0, 0, 3, 94, 170, 1, 4096, 545, 2570, 0, 30, 3504, 4095, 38, 690, 0, 1139, 0, 4058, 0, 145, 186, 0, 107, 4095, 0, 0, 0, 4063, 0, 14, 0, 139, 1, 1, 0, 3772, 6, 0, 0, 0, 3800, 2486, 672, 0, 2982, 242, 3732, 3, 0, 1, 3914, 1, 0, 0, 1745, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 113, 1001, 0, 7, 4096, 4096, 3247, 296, 24, 491, 1252, 32, 4093, 94, 2723, 192, 0, 341, 0, 3805, 4095, 0, 4082, 0, 2689, 0, 475, 0, 19, 823, 38, 235, 3492, 30, 149, 3529, 29, 2313, 0, 0, 0, 0, 2599, 25, 0, 3862, 1318, 0, 4081, 1746, 24, 2, 3971, 0, 1, 0, 3, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 4095, 4080, 3444, 0, 1, 0, 0, 0, 4082, 0, 4092, 2135, 2690, 0, 3952, 2556, 20, 97, 96, 4068, 3824, 11, 0, 0, 0, 0, 4096, 435, 993, 0, 0, 4096, 34, 227, 12, 3981, 4062, 0, 4096, 0, 36, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 5, 22, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[3874, 170, 4081, 0, 0, 0, 0, 0, 0, 31, 3086, 0, 4008, 0, 1, 0, 2, 0, 1450, 0, 4095, 227, 19, 0, 0, 493, 4096, 0, 8, 0, 0, 4, 0, 0, 0, 0, 4096, 4059, 966, 2467, 0, 4096, 12, 4096, 6, 0, 0, 0, 4052, 0, 4095, 4087, 526, 0, 4094, 2679, 0, 0, 0, 0, 12, 0, 548, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1652, 6, 0, 0, 0, 0, 0, 0, 2080, 88, 0, 2289, 1191, 0, 30, 0, 4067, 0, 101, 4035, 4027, 0, 766, 4096, 0, 0, 4076, 0, 2, 1809, 68, 0, 354, 330, 1735, 266, 0, 2147, 0, 4060, 50, 2010, 0, 132, 44, 20, 3117, 4095, 653, 4095, 0, 0, 4096, 0, 0, 0, 3940, 0, 0, 0, 4009, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[16, 3991, 38, 3997, 4084, 0, 0, 0, 0, 1968, 4043, 0, 45, 82, 211, 0, 13, 373, 3153, 0, 2945, 2818, 0, 0, 144, 0, 3820, 7, 3607, 0, 0, 3819, 0, 77, 0, 0, 4096, 0, 3755, 0, 4063, 3906, 82, 0, 0, 0, 89, 0, 1, 30, 0, 0, 0, 4001, 0, 0, 0, 349, 26, 3952, 0, 888, 0, 1047] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3796, 0, 4095, 1, 373, 0, 0, 0, 0, 47, 1934, 1, 738, 1225, 959, 0, 629, 0, 10, 2, 3476, 4061, 4032, 0, 1, 4096, 0, 0, 304, 0, 4096, 80, 8, 0, 86, 37, 4080, 0, 470, 0, 0, 4096, 1035, 3052, 396, 40, 0, 1, 4039, 1, 61, 3888, 3076, 2, 4096, 0, 0, 0, 3022, 0, 0, 0, 94, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[10, 3, 4093, 0, 63, 0, 1538, 32, 1838, 419, 0, 0, 0, 0, 0, 0, 1834, 1, 1836, 0, 4096, 0, 478, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 875, 0, 4014, 19, 23, 549, 0, 0, 0, 0, 0, 0, 0, 0, 38, 458, 4096, 3567, 10, 0, 0, 969, 0, 0, 0, 0, 0, 0, 0, 0, 1843, 0, 6, 0, 2750, 62, 46, 4089, 554, 15, 133, 0, 0, 0, 0, 0, 0, 3346, 4005, 4096, 2048, 0, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 172, 4070, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 14, 727, 53, 41, 1510, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7996, 8485, 5480, 9138, 9498, 14430, 4299, 6210] avg=8192.0000 max=14430 max_rank=5 min=4299 min_rank=6 imbalance_ratio=1.761475 max_load_reduction=51106 max_load_reduction_pct=0.779816 moved_to_echo=34437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3477, 0, 3912, 0, 4095, 912, 0, 0, 1960, 795, 0, 0, 0, 0, 0, 0, 0, 3, 94, 170, 1, 4096, 545, 2570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1659, 4095, 38, 690, 0, 1139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 0, 145, 186, 0, 107, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 14, 0, 139, 1050, 897, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3772, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2903, 2486, 672, 0, 2982, 242, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3914, 1, 0, 0, 1745, 0, 3732, 1610, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15151, 7479, 7651, 5021, 6163, 3780, 9288, 11003] avg=8192.0000 max=15151 max_rank=0 min=3780 min_rank=5 imbalance_ratio=1.849487 max_load_reduction=50385 max_load_reduction_pct=0.768814 moved_to_echo=30234 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3796, 0, 4095, 1, 373, 0, 0, 0, 1978, 2, 0, 0, 0, 0, 0, 0, 0, 47, 1934, 1, 738, 1225, 959, 0, 264, 0, 0, 0, 0, 0, 0, 0, 629, 0, 6, 2, 3476, 4061, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 493, 0, 0, 304, 0, 4096, 80, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 86, 37, 4080, 0, 470, 0, 2, 954, 147, 0, 0, 0, 0, 0, 0, 3685, 1035, 3052, 396, 40, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3085, 1, 61, 3888, 3076, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3022, 0, 0, 0, 94, 0, 4096, 1625, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10245, 5168, 12206, 4974, 5784, 8209, 10113, 8837] avg=8192.0000 max=12206 max_rank=2 min=4974 min_rank=3 imbalance_ratio=1.489990 max_load_reduction=53330 max_load_reduction_pct=0.813751 moved_to_echo=32943 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 1234, 927, 0, 0, 0, 0, 0, 0, 131, 0, 4095, 4080, 3444, 0, 1, 0, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1931, 0, 4092, 2135, 2690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 308, 20, 97, 96, 4068, 3824, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 435, 993, 0, 1224, 967, 147, 0, 0, 0, 0, 0, 0, 3685, 34, 227, 12, 3981, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3129, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 22, 0, 4067, 1014, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2161, 12015, 10848, 12376, 7862, 12001, 3165, 5108] avg=8192.0000 max=12376 max_rank=3 min=2161 min_rank=0 imbalance_ratio=1.510742 max_load_reduction=53160 max_load_reduction_pct=0.811157 moved_to_echo=28136 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3874, 170, 4081, 0, 0, 0, 0, 0, 243, 333, 0, 0, 0, 0, 0, 0, 0, 31, 3086, 0, 4008, 0, 1, 0, 267, 0, 0, 0, 0, 0, 0, 0, 2, 0, 678, 0, 4095, 227, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4096, 0, 8, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4059, 966, 2467, 439, 960, 150, 0, 0, 0, 0, 0, 0, 3679, 12, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3092, 0, 4095, 4087, 526, 0, 0, 2679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 548, 0, 4094, 198, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8701, 7393, 5021, 4160, 13137, 7793, 14479, 4852] avg=8192.0000 max=14479 max_rank=6 min=4160 min_rank=3 imbalance_ratio=1.767456 max_load_reduction=51057 max_load_reduction_pct=0.779068 moved_to_echo=40261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1652, 6, 0, 0, 0, 0, 1978, 923, 0, 0, 0, 0, 0, 0, 0, 0, 2080, 88, 0, 2289, 1191, 0, 262, 0, 0, 0, 0, 0, 0, 0, 30, 0, 1925, 0, 101, 4035, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 766, 493, 0, 0, 4076, 0, 2, 1809, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 354, 330, 1735, 266, 0, 2147, 1219, 736, 145, 0, 0, 0, 0, 0, 0, 3653, 50, 2010, 0, 132, 44, 20, 0, 0, 0, 0, 0, 0, 0, 0, 2381, 4095, 653, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 0, 4009, 0, 4096, 1625, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4559, 5910, 10118, 7146, 7000, 5909, 11224, 13670] avg=8192.0000 max=13670 max_rank=7 min=4559 min_rank=0 imbalance_ratio=1.668701 max_load_reduction=51866 max_load_reduction_pct=0.791412 moved_to_echo=37803 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[16, 3991, 38, 3997, 4084, 0, 0, 0, 0, 716, 0, 0, 0, 0, 0, 0, 0, 1968, 4043, 0, 45, 82, 211, 0, 252, 0, 0, 0, 0, 0, 0, 0, 13, 373, 1492, 0, 2945, 2818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 3820, 7, 3607, 0, 0, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 4096, 0, 3755, 0, 945, 0, 140, 0, 0, 0, 0, 0, 4063, 3514, 82, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 30, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 26, 3952, 0, 888, 0, 1047, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12842, 6601, 7641, 11397, 9013, 7748, 4032, 6262] avg=8192.0000 max=12842 max_rank=0 min=4032 min_rank=6 imbalance_ratio=1.567627 max_load_reduction=52694 max_load_reduction_pct=0.804047 moved_to_echo=27055 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 113, 1001, 1838, 929, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4096, 3247, 296, 24, 491, 228, 0, 0, 0, 0, 0, 0, 0, 1252, 32, 1937, 94, 2723, 192, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 4095, 0, 4082, 0, 2689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 19, 823, 38, 235, 3492, 30, 1227, 614, 126, 0, 0, 0, 0, 0, 149, 3175, 29, 2313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1985, 25, 0, 3862, 1318, 0, 0, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 24, 2, 3971, 0, 1, 0, 3, 3, 4081, 1509, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3881, 12485, 6571, 11324, 7079, 5666, 8936, 9594] avg=8192.0000 max=12485 max_rank=1 min=3881 min_rank=0 imbalance_ratio=1.524048 max_load_reduction=53051 max_load_reduction_pct=0.809494 moved_to_echo=31275 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11264, 4224, 17920, 4096, 8704, 1024, 1664, 1152, 11136, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2432, 128, 512, 5376, 20992, 5120, 9728, 8960, 6784, 5248, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4224, 24832, 5248, 19584, 2560, 4224, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 384, 18816, 3968, 128, 1024, 7168, 1152, 24320, 9216, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[16640, 6656, 5760, 20096, 7936, 4352, 0, 4480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4992, 2816, 16128, 3840, 12416, 4096, 10752, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2048, 2176, 21376, 8448, 15616, 8064, 3456, 3072, 1792, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2048, 512, 10624, 4224, 21504, 14208, 10880, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[12, 351, 261, 4096, 0, 2168, 60, 0, 4060, 0, 0, 93, 0, 121, 1, 3989, 3694, 977, 0, 0, 27, 673, 9, 0, 0, 294, 3430, 1664, 0, 0, 0, 0, 69, 0, 2696, 0, 3883, 4093, 0, 1, 4096, 841, 4085, 0, 0, 0, 0, 0, 40, 4061, 0, 3563, 39, 121, 2, 0, 1200, 445, 3955, 4095, 309, 1962, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[3026, 0, 4096, 0, 13, 2599, 0, 10, 3701, 0, 157, 0, 279, 3725, 0, 2622, 4067, 0, 0, 45, 0, 0, 0, 252, 15, 29, 619, 3674, 29, 0, 3, 4041, 0, 0, 0, 4086, 4066, 3497, 0, 8, 18, 4096, 988, 8, 4096, 13, 0, 0, 4096, 2, 7, 4, 24, 2280, 0, 0, 333, 0, 1448, 1402, 449, 268, 1241, 104] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 1643, 3275, 84, 0, 551, 1533, 4096, 0, 32, 1, 33, 4093, 0, 3966, 4096, 0, 215, 0, 0, 0, 2, 4078, 1, 0, 768, 4065, 0, 0, 0, 2731, 0, 3841, 1036, 0, 3865, 1266, 0, 447, 178, 0, 3576, 0, 28, 26, 0, 110, 1, 0, 1262, 4003, 0, 4036, 0, 2320, 1, 7, 4046, 188, 36, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[3335, 0, 4094, 469, 0, 3145, 3466, 0, 0, 48, 143, 72, 0, 4075, 0, 11, 3005, 26, 0, 0, 2343, 0, 301, 658, 0, 0, 0, 1776, 0, 0, 364, 3968, 8, 4094, 0, 7, 4005, 0, 32, 4012, 4094, 0, 4093, 0, 0, 4, 0, 0, 4081, 134, 0, 1216, 3, 0, 126, 604, 0, 0, 173, 3749, 0, 3778, 24, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 0, 3816, 0, 55, 0, 374, 4054, 0, 0, 928, 0, 2303, 0, 4091, 4096, 6, 3, 0, 4096, 0, 0, 4095, 0, 0, 1, 3967, 4071, 1757, 149, 3595, 0, 0, 19, 0, 2956, 6, 0, 0, 825, 119, 239, 5, 0, 0, 0, 0, 4092, 0, 2112, 3807, 0, 3, 0, 1, 545, 1, 4038, 4090, 11, 1128, 19, 62] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1764, 46, 0, 3926, 0, 770, 0, 44, 2, 0, 0, 3, 0, 3782, 1174, 4024, 4096, 3929, 0, 132, 122, 0, 0, 69, 0, 0, 4060, 4074, 0, 0, 489, 2838, 0, 64, 2, 0, 4084, 40, 0, 259, 0, 182, 2090, 49, 0, 0, 0, 0, 3947, 3609, 10, 4096, 3, 41, 0, 1482, 0, 9, 3463, 4037, 131, 2593, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1, 168, 4066, 4096, 0, 1201, 0, 0, 4042, 0, 0, 2317, 0, 3815, 0, 4053, 3523, 8, 308, 0, 2795, 10, 89, 0, 1, 0, 965, 2776, 0, 0, 0, 3333, 48, 0, 0, 0, 3961, 21, 0, 2, 46, 16, 34, 0, 18, 3079, 952, 0, 2765, 0, 0, 4084, 35, 596, 1, 4084, 0, 0, 3841, 367, 0, 4019, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4078, 214, 118, 4096, 0, 4096, 0, 0, 1540, 0, 0, 26, 0, 4096, 127, 3886, 3601, 2694, 0, 1055, 0, 875, 110, 0, 3, 316, 4096, 4049, 0, 0, 0, 3508, 60, 0, 0, 1282, 4015, 32, 1185, 0, 12, 152, 12, 0, 5, 0, 528, 0, 49, 0, 0, 3389, 397, 4004, 4, 0, 0, 0, 3726, 1, 0, 4083, 16, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 351, 261, 3119, 0, 2168, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 93, 0, 121, 1, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 3694, 977, 0, 0, 27, 673, 9, 0, 525, 147, 0, 0, 0, 0, 0, 0, 0, 294, 3430, 1283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 2696, 0, 3883, 4093, 0, 1, 428, 0, 0, 0, 0, 0, 0, 0, 4096, 841, 4085, 0, 0, 0, 0, 0, 2102, 1341, 234, 0, 0, 0, 0, 0, 40, 4061, 0, 2222, 39, 121, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 445, 3955, 4095, 309, 1962, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5971, 6162, 6052, 5007, 11170, 12699, 6485, 11990] avg=8192.0000 max=12699 max_rank=5 min=5007 min_rank=3 imbalance_ratio=1.550171 max_load_reduction=52837 max_load_reduction_pct=0.806229 moved_to_echo=42344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3026, 0, 4096, 0, 13, 2599, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3701, 0, 157, 0, 279, 3725, 0, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 45, 0, 0, 0, 252, 0, 325, 0, 0, 0, 0, 0, 0, 15, 29, 619, 2831, 29, 0, 3, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4066, 3497, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 988, 8, 4096, 13, 0, 0, 1382, 1, 518, 0, 0, 0, 0, 0, 4096, 2, 7, 3, 24, 2280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 1448, 1402, 449, 268, 1241, 104, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9744, 9102, 4689, 7567, 11657, 11120, 6412, 5245] avg=8192.0000 max=11657 max_rank=4 min=4689 min_rank=2 imbalance_ratio=1.422974 max_load_reduction=53879 max_load_reduction_pct=0.822128 moved_to_echo=34434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1643, 2494, 84, 0, 551, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 32, 1, 33, 4093, 0, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 215, 0, 0, 0, 2, 4078, 420, 359, 0, 0, 0, 0, 0, 0, 1, 0, 768, 3132, 0, 0, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 1036, 0, 3865, 1266, 0, 447, 342, 0, 0, 0, 0, 0, 0, 0, 178, 0, 3576, 0, 28, 26, 0, 110, 2090, 1507, 574, 0, 0, 0, 0, 0, 1, 0, 1262, 2496, 0, 4036, 0, 2320, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 4046, 188, 36, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6305, 10131, 9170, 6632, 10797, 8089, 10115, 4297] avg=8192.0000 max=10797 max_rank=4 min=4297 min_rank=7 imbalance_ratio=1.317993 max_load_reduction=54739 max_load_reduction_pct=0.835251 moved_to_echo=33298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1764, 46, 0, 2990, 0, 770, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 3782, 1174, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3929, 0, 132, 122, 0, 0, 69, 503, 360, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 3139, 0, 0, 489, 2838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 2, 0, 4084, 40, 0, 259, 410, 0, 0, 0, 0, 0, 0, 0, 0, 182, 2090, 49, 0, 0, 0, 0, 2121, 1542, 575, 0, 0, 0, 0, 0, 3947, 3609, 10, 2554, 3, 41, 0, 1482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3463, 4037, 131, 2593, 0, 1, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5614, 6864, 9211, 10526, 4859, 6559, 11646, 10257] avg=8192.0000 max=11646 max_rank=6 min=4859 min_rank=4 imbalance_ratio=1.421631 max_load_reduction=53890 max_load_reduction_pct=0.822296 moved_to_echo=33321 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4078, 214, 118, 3107, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1540, 0, 0, 26, 0, 4096, 127, 1833, 0, 0, 0, 0, 0, 0, 0, 0, 3601, 2694, 0, 1055, 0, 875, 110, 0, 529, 361, 0, 0, 0, 0, 0, 0, 3, 316, 4096, 3113, 0, 0, 0, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 1282, 4015, 32, 1185, 0, 432, 0, 0, 0, 0, 0, 0, 0, 12, 152, 12, 0, 5, 0, 528, 0, 2053, 1281, 575, 0, 0, 0, 0, 0, 49, 0, 0, 2108, 397, 4004, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 1, 0, 4083, 16, 0, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11613, 7622, 9225, 11036, 7006, 4618, 6562, 7854] avg=8192.0000 max=11613 max_rank=0 min=4618 min_rank=5 imbalance_ratio=1.417603 max_load_reduction=53923 max_load_reduction_pct=0.822800 moved_to_echo=26040 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3335, 0, 4094, 358, 0, 3145, 3466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 143, 72, 0, 4075, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3005, 26, 0, 0, 2343, 0, 301, 658, 60, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 0, 0, 364, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4094, 0, 7, 4005, 0, 32, 4012, 49, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4093, 0, 0, 4, 0, 0, 5, 457, 250, 0, 0, 0, 0, 0, 4081, 134, 0, 759, 3, 0, 126, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 3749, 0, 3778, 24, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14398, 4344, 6550, 5701, 12207, 8903, 5707, 7726] avg=8192.0000 max=14398 max_rank=0 min=4344 min_rank=1 imbalance_ratio=1.757568 max_load_reduction=51138 max_load_reduction_pct=0.780304 moved_to_echo=34543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 168, 4066, 3119, 0, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 2317, 0, 3815, 0, 1917, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 8, 308, 0, 2795, 10, 89, 0, 525, 245, 0, 0, 0, 0, 0, 0, 1, 0, 965, 2139, 0, 0, 0, 3333, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 3961, 21, 0, 2, 428, 0, 0, 0, 0, 0, 0, 0, 46, 16, 34, 0, 18, 3079, 952, 0, 2136, 1538, 392, 0, 0, 0, 0, 0, 2765, 0, 0, 2546, 35, 596, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 367, 0, 4019, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8555, 12091, 7503, 6438, 4460, 8211, 10027, 8251] avg=8192.0000 max=12091 max_rank=1 min=4460 min_rank=4 imbalance_ratio=1.475952 max_load_reduction=53445 max_load_reduction_pct=0.815506 moved_to_echo=30949 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 2906, 0, 55, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 928, 0, 2303, 0, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 3, 0, 4096, 0, 0, 4095, 489, 351, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3056, 4071, 1757, 149, 3595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2956, 6, 0, 0, 399, 0, 0, 0, 0, 0, 0, 0, 825, 119, 239, 5, 0, 0, 0, 0, 2156, 1433, 560, 0, 0, 0, 0, 0, 4092, 0, 2112, 2374, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 545, 1, 4038, 4090, 11, 1128, 19, 62, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3336, 9220, 13136, 12629, 3380, 5337, 8582, 9916] avg=8192.0000 max=13136 max_rank=2 min=3336 min_rank=0 imbalance_ratio=1.603516 max_load_reduction=52400 max_load_reduction_pct=0.799561 moved_to_echo=27215 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[9344, 5504, 15232, 128, 4224, 3200, 1536, 128, 14080, 9216, 3712, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12288, 896, 14336, 18176, 128, 14080, 4096, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[19072, 7808, 3456, 15104, 512, 11136, 256, 8576, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[21504, 128, 384, 3456, 384, 26112, 1408, 12672, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 8064, 3840, 5376, 30848, 8960, 1280, 4736, 2560, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2176, 512, 24704, 18048, 1024, 17920, 1408, 256, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 640, 13952, 20096, 4224, 1792, 1024, 24064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[30208, 7680, 640, 1280, 9472, 1664, 512, 9216, 3072, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[3809, 0, 10, 38, 122, 0, 4095, 0, 1052, 0, 0, 2887, 0, 0, 0, 160, 0, 81, 584, 0, 4063, 2037, 0, 4094, 113, 2, 0, 30, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 2528, 1903, 1513, 69, 0, 359, 4063, 0, 219, 336, 78, 8, 2368, 2499, 3391, 0, 4094, 1, 0, 0, 3519, 6, 3745, 0, 3388, 69] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 0, 3249, 0, 0, 114, 0, 112, 2, 0, 3443, 0, 228, 0, 4093, 38, 0, 84, 0, 4001, 1476, 1317, 42, 268, 0, 3135, 0, 3845, 0, 0, 0, 0, 4041, 0, 1176, 8, 1478, 1209, 0, 7, 4051, 0, 67, 1185, 1326, 0, 179, 0, 705, 4094, 2, 3885, 0, 2491, 0, 22, 17, 2166, 3816, 0, 0, 4032, 39] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3318, 0, 0, 0, 0, 0, 109, 4092, 339, 0, 0, 0, 926, 2, 40, 0, 0, 0, 4086, 0, 4046, 4059, 0, 1116, 643, 4096, 0, 4, 0, 0, 1143, 36, 0, 5, 3636, 4020, 12, 2, 4096, 4, 186, 780, 1, 1917, 2070, 4078, 4093, 44, 0, 1, 0, 3750, 71, 0, 7, 443, 0, 0, 0, 659, 3510] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 4077, 1982, 0, 0, 339, 44, 0, 0, 0, 3411, 0, 0, 18, 0, 23, 26, 0, 0, 95, 3151, 0, 4096, 0, 4014, 10, 12, 4096, 431, 0, 455, 0, 3980, 0, 0, 0, 3641, 1281, 0, 0, 3272, 148, 158, 0, 0, 0, 216, 820, 4096, 2375, 3, 4073, 0, 4070, 0, 0, 0, 4069, 2776, 150, 0, 32, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 180, 0, 0, 4, 0, 4089, 0, 4, 0, 0, 3981, 3952, 7, 186, 0, 0, 0, 58, 0, 1456, 4085, 4018, 4090, 4096, 0, 235, 64, 507, 1, 3009, 2031, 0, 3810, 0, 0, 9, 0, 3791, 0, 0, 0, 0, 34, 3842, 4, 0, 0, 4091, 2353, 115, 2342, 0, 930, 1239, 233, 0, 1, 2979, 0, 378, 0, 3079, 253] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 14, 4094, 2454, 0, 795, 15, 3, 0, 0, 0, 0, 0, 29, 0, 1712, 2, 0, 0, 4083, 0, 2126, 4090, 4093, 1512, 0, 1938, 562, 4047, 0, 16, 0, 4096, 0, 21, 0, 0, 1812, 0, 0, 0, 0, 0, 0, 2678, 0, 0, 271, 4096, 4095, 141, 3328, 0, 5, 3, 0, 0, 0, 3, 197, 4065, 1486, 3560] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 105, 0, 0, 0, 0, 1995, 0, 0, 0, 4096, 0, 1, 849, 0, 181, 3, 554, 246, 1515, 3870, 2127, 2176, 379, 4095, 3965, 1, 2720, 0, 3672, 4093, 17, 131, 0, 0, 0, 4074, 3805, 230, 0, 4096, 15, 206, 772, 0, 0, 0, 0, 3597, 621, 1, 2565, 2641, 1392, 0, 0, 10, 274, 2, 51, 286, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1992, 0, 4096, 4094, 2029, 0, 42, 0, 6, 0, 0, 3435, 0, 0, 0, 0, 4063, 2615, 11, 62, 8, 0, 0, 3912, 216, 3328, 0, 0, 4096, 804, 0, 0, 0, 4096, 0, 0, 0, 0, 3193, 3824, 0, 0, 0, 410, 0, 0, 197, 0, 754, 4096, 3077, 101, 2739, 0, 3945, 1, 0, 0, 0, 267, 4009, 2, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3500, 0, 10, 38, 122, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 0, 0, 2887, 0, 0, 0, 160, 8, 782, 0, 0, 0, 0, 0, 0, 0, 81, 584, 0, 4063, 2037, 0, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 113, 2, 0, 30, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 0, 0, 11, 2528, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 1513, 69, 0, 359, 4063, 0, 219, 336, 2450, 1517, 556, 0, 0, 0, 0, 0, 78, 0, 2368, 2499, 3391, 0, 1894, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3519, 6, 3745, 0, 3388, 69, 1644, 309, 254, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7765, 4889, 8560, 1791, 8284, 11082, 10231, 12934] avg=8192.0000 max=12934 max_rank=7 min=1791 min_rank=3 imbalance_ratio=1.578857 max_load_reduction=52602 max_load_reduction_pct=0.802643 moved_to_echo=42531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3764, 0, 0, 3318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 4092, 339, 0, 0, 0, 926, 4093, 772, 0, 0, 0, 0, 0, 0, 2, 40, 0, 0, 0, 4086, 0, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 1116, 643, 1646, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 36, 0, 5, 3636, 4020, 12, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4, 186, 780, 1, 1917, 2070, 2450, 1499, 509, 0, 0, 0, 0, 0, 4078, 0, 44, 0, 1, 0, 1735, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 443, 0, 0, 0, 659, 3510, 1506, 332, 70, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7082, 10331, 5903, 7468, 8782, 13514, 5929, 6527] avg=8192.0000 max=13514 max_rank=5 min=5903 min_rank=2 imbalance_ratio=1.649658 max_load_reduction=52022 max_load_reduction_pct=0.793793 moved_to_echo=34752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3761, 0, 0, 3249, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 2, 0, 3443, 0, 228, 0, 4093, 705, 8, 0, 0, 0, 0, 0, 0, 38, 0, 84, 0, 4001, 1476, 1317, 19, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 3135, 0, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 1176, 8, 1478, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4051, 0, 67, 1185, 1326, 0, 179, 2300, 15, 338, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 3885, 0, 1153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 17, 2166, 3816, 0, 0, 4032, 39, 1000, 332, 250, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7124, 8591, 6935, 4948, 7662, 9468, 9134, 11674] avg=8192.0000 max=11674 max_rank=7 min=4948 min_rank=3 imbalance_ratio=1.425049 max_load_reduction=53862 max_load_reduction_pct=0.821869 moved_to_echo=37938 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3764, 0, 4077, 1982, 0, 0, 339, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3411, 0, 0, 18, 0, 4096, 782, 0, 0, 0, 0, 0, 0, 23, 26, 0, 0, 95, 3151, 0, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 10, 12, 1646, 431, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3733, 0, 0, 0, 3641, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3272, 148, 158, 0, 0, 0, 216, 2450, 1517, 552, 0, 0, 0, 0, 0, 820, 0, 2375, 3, 4073, 0, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 2776, 150, 0, 32, 0, 1635, 332, 247, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10206, 8307, 5092, 6568, 8655, 8313, 9154, 9241] avg=8192.0000 max=10206 max_rank=0 min=5092 min_rank=2 imbalance_ratio=1.245850 max_load_reduction=55330 max_load_reduction_pct=0.844269 moved_to_echo=35363 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3762, 0, 14, 4094, 2454, 0, 795, 15, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 29, 0, 4096, 781, 0, 0, 0, 0, 0, 0, 1712, 2, 0, 0, 4083, 0, 2126, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1512, 0, 1938, 226, 4047, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 21, 0, 0, 1812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2678, 0, 0, 336, 1515, 0, 0, 0, 0, 0, 0, 271, 0, 4095, 141, 3328, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 197, 4065, 1486, 3560, 2, 332, 254, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11134, 4909, 9717, 11832, 5675, 4529, 7841, 9899] avg=8192.0000 max=11832 max_rank=3 min=4529 min_rank=5 imbalance_ratio=1.444336 max_load_reduction=53704 max_load_reduction_pct=0.819458 moved_to_echo=27944 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 180, 0, 0, 4, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3981, 3952, 7, 186, 0, 2353, 781, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 1456, 4085, 4018, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 235, 64, 204, 1, 3009, 2031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3574, 0, 0, 9, 0, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 3842, 4, 0, 0, 303, 1515, 168, 0, 0, 0, 0, 0, 4091, 0, 115, 2342, 0, 930, 574, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2979, 0, 378, 0, 3079, 253, 497, 0, 236, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4273, 11264, 11411, 9640, 7374, 5866, 8285, 7423] avg=8192.0000 max=11411 max_rank=2 min=4273 min_rank=0 imbalance_ratio=1.392944 max_load_reduction=54125 max_load_reduction_pct=0.825882 moved_to_echo=28948 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1827, 0, 4096, 4094, 2029, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3435, 0, 0, 0, 0, 4096, 750, 0, 0, 0, 0, 0, 0, 4063, 2615, 11, 62, 8, 0, 0, 1708, 0, 0, 0, 0, 0, 0, 0, 0, 216, 3328, 0, 0, 1643, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 0, 0, 0, 0, 3193, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 0, 197, 0, 2453, 1454, 539, 0, 0, 0, 0, 0, 754, 0, 3077, 101, 2739, 0, 1818, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 4009, 2, 0, 16, 1588, 165, 258, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12088, 8287, 8467, 5991, 10855, 5053, 8490, 6305] avg=8192.0000 max=12088 max_rank=0 min=5053 min_rank=5 imbalance_ratio=1.475586 max_load_reduction=53448 max_load_reduction_pct=0.815552 moved_to_echo=30703 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3764, 0, 105, 0, 0, 0, 0, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 849, 0, 3597, 415, 0, 0, 0, 0, 0, 0, 181, 3, 554, 246, 1515, 3870, 2127, 955, 0, 0, 0, 0, 0, 0, 0, 0, 379, 4095, 3965, 1, 1093, 0, 3672, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 17, 123, 0, 0, 0, 4074, 3805, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 206, 772, 0, 0, 0, 1627, 806, 189, 0, 0, 0, 0, 0, 0, 0, 621, 1, 2565, 2641, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 274, 2, 51, 286, 11, 0, 559, 332, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5864, 8958, 9451, 17298, 8249, 7711, 6472, 1533] avg=8192.0000 max=17298 max_rank=3 min=1533 min_rank=7 imbalance_ratio=2.111572 max_load_reduction=48238 max_load_reduction_pct=0.736053 moved_to_echo=23965 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1536, 15616, 256, 1536, 10752, 4096, 2432, 2816, 14464, 9856, 2944, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[13312, 13056, 8576, 2688, 9728, 5376, 6784, 6656, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 13568, 6912, 8576, 4480, 12800, 7552, 8448, 2176, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 23936, 128, 1280, 128, 12928, 21760, 6016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6144, 2816, 1408, 384, 15232, 18816, 9600, 11648, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[24192, 256, 8320, 16896, 4736, 0, 9600, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[10112, 0, 16896, 5120, 20096, 3584, 9728, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1280, 128, 4096, 21632, 3968, 256, 1152, 5248, 23168, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 75, 0, 0, 905, 3835, 3858, 2109, 0, 147, 0, 0, 2468, 0, 0, 11, 0, 2534, 445, 1208, 4089, 0, 582, 133, 17, 623, 4061, 11, 0, 4, 0, 0, 3603, 0, 4087, 66, 4081, 360, 4063, 3395, 1301, 602, 3772, 2880, 3144, 0, 44, 0, 86, 4039, 0, 0, 0, 0, 0, 2140, 693, 10, 0, 55, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 7, 639, 4, 733, 2888, 1, 8, 3993, 5, 0, 0, 14, 0, 4089, 0, 0, 0, 4094, 17, 2008, 0, 812, 73, 0, 0, 0, 4094, 0, 0, 1, 1, 1844, 1, 4086, 0, 3075, 324, 98, 4089, 2, 3624, 4096, 3134, 3443, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2938, 4, 4, 0, 139, 999, 2024] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[90, 0, 0, 3931, 0, 257, 11, 3674, 0, 5, 4096, 1161, 0, 0, 0, 633, 36, 379, 1586, 3976, 46, 0, 2095, 0, 0, 3812, 0, 0, 0, 1266, 0, 0, 0, 0, 0, 0, 4096, 314, 65, 1962, 0, 47, 4047, 2565, 3461, 3425, 3577, 4051, 0, 0, 8, 2310, 0, 0, 4032, 58, 1, 7, 0, 0, 0, 0, 4041, 415] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[50, 0, 7, 0, 0, 4095, 320, 0, 550, 2, 0, 0, 0, 0, 3755, 0, 24, 1, 0, 2167, 4059, 11, 1717, 0, 2149, 34, 2257, 3551, 3084, 3874, 0, 0, 44, 0, 28, 0, 4094, 4081, 4009, 0, 4016, 0, 4076, 0, 573, 4096, 2406, 4096, 1028, 104, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 1171, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[28, 0, 0, 32, 0, 24, 93, 2930, 0, 1734, 15, 2, 0, 1764, 163, 1141, 281, 4095, 0, 0, 3770, 1173, 0, 0, 952, 3881, 0, 3369, 0, 1, 1507, 0, 3224, 0, 4032, 0, 4096, 4095, 423, 3626, 0, 4096, 170, 55, 4078, 1020, 3295, 9, 0, 0, 0, 345, 0, 1, 3934, 0, 0, 57, 0, 256, 0, 0, 241, 1528] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[231, 0, 0, 315, 0, 80, 2098, 2894, 0, 0, 4058, 2246, 0, 3686, 0, 4049, 4096, 34, 4051, 0, 0, 0, 8, 0, 0, 4060, 0, 4091, 0, 3411, 0, 0, 19, 0, 69, 0, 878, 122, 19, 2563, 767, 3378, 81, 50, 4096, 0, 4016, 4092, 0, 0, 0, 1420, 0, 0, 0, 468, 0, 4074, 0, 0, 0, 0, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4038, 2974, 2743, 0, 4072, 26, 0, 0, 10, 1, 724, 3588, 109, 1423, 2714, 18, 0, 0, 36, 4068, 0, 0, 18, 4086, 833, 0, 3, 178, 3565, 3, 2047, 4040, 0, 2, 50, 4076, 1, 3469, 33, 3802, 4096, 4070, 317, 0, 0, 4, 12, 0, 0, 0, 4, 22, 165, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4, 258, 11, 748, 0, 719, 1512, 0, 0, 0, 1096, 30, 0, 0, 1988, 0, 87, 3992, 260, 2975, 3865, 0, 2031, 4043, 51, 0, 0, 29, 0, 52, 53, 0, 4096, 0, 4073, 0, 45, 0, 2303, 4052, 0, 3588, 3982, 4094, 1766, 7, 0, 275, 1, 227, 0, 1947, 301, 0, 0, 4039, 3701, 76, 1, 0, 3158, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 75, 0, 0, 905, 3835, 1607, 968, 0, 0, 0, 0, 0, 0, 3858, 2109, 0, 147, 0, 0, 2468, 0, 240, 597, 9, 0, 0, 0, 0, 0, 0, 11, 0, 2534, 205, 1208, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 124, 17, 623, 4061, 11, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 0, 1713, 66, 4081, 360, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 1301, 602, 0, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 86, 4039, 0, 0, 0, 0, 3144, 2948, 0, 0, 0, 0, 0, 0, 0, 2140, 693, 10, 0, 55, 0, 0, 3395, 824, 767, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7390, 9428, 8047, 5422, 9823, 7281, 10261, 7884] avg=8192.0000 max=10261 max_rank=6 min=5422 min_rank=3 imbalance_ratio=1.252563 max_load_reduction=55275 max_load_reduction_pct=0.843430 moved_to_echo=35249 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[90, 0, 0, 3931, 0, 257, 11, 3674, 1610, 1151, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 1161, 0, 0, 0, 633, 24, 710, 269, 0, 0, 0, 0, 0, 36, 379, 1586, 3976, 22, 0, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3543, 0, 0, 0, 1266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1717, 314, 65, 1962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 2565, 0, 1564, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2310, 0, 0, 4032, 58, 3577, 2706, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 4041, 415, 47, 755, 769, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10724, 6898, 8094, 4809, 4058, 12227, 12691, 6035] avg=8192.0000 max=12691 max_rank=6 min=4058 min_rank=4 imbalance_ratio=1.549194 max_load_reduction=52845 max_load_reduction_pct=0.806351 moved_to_echo=35011 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 258, 11, 748, 0, 719, 1601, 1376, 0, 0, 0, 0, 0, 0, 1512, 0, 0, 0, 1096, 30, 0, 0, 140, 849, 285, 0, 0, 0, 0, 0, 1988, 0, 87, 3992, 120, 2975, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2031, 3758, 51, 0, 0, 29, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 4096, 0, 1707, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 0, 0, 3588, 0, 1869, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 1, 227, 0, 1947, 301, 0, 1766, 3112, 0, 0, 0, 0, 0, 0, 0, 4039, 3701, 76, 1, 0, 3158, 0, 4052, 870, 765, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4717, 3912, 13027, 5921, 5901, 7767, 7629, 16662] avg=8192.0000 max=16662 max_rank=7 min=3912 min_rank=1 imbalance_ratio=2.033936 max_load_reduction=48874 max_load_reduction_pct=0.745758 moved_to_echo=37959 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 7, 639, 4, 733, 2888, 1607, 1053, 0, 0, 0, 0, 0, 0, 1, 8, 3993, 5, 0, 0, 14, 0, 2210, 649, 5, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 1884, 17, 2008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 812, 68, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1844, 1, 1712, 0, 3075, 324, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 2, 3624, 0, 1432, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3443, 3201, 0, 0, 0, 0, 0, 0, 0, 2938, 4, 4, 0, 139, 999, 2024, 4089, 895, 767, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11027, 6885, 7998, 4974, 6958, 9191, 6644, 11859] avg=8192.0000 max=11859 max_rank=7 min=4974 min_rank=3 imbalance_ratio=1.447632 max_load_reduction=53677 max_load_reduction_pct=0.819046 moved_to_echo=34652 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[231, 0, 0, 315, 0, 80, 2098, 2894, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 2246, 0, 3686, 0, 4049, 0, 0, 286, 0, 0, 0, 0, 0, 4096, 34, 4051, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 0, 4091, 0, 3411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 69, 0, 369, 122, 19, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 767, 0, 81, 50, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 0, 0, 0, 468, 4016, 3201, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 16, 3378, 895, 164, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5963, 14325, 8189, 11276, 3161, 4990, 9105, 8527] avg=8192.0000 max=14325 max_rank=1 min=3161 min_rank=4 imbalance_ratio=1.748657 max_load_reduction=51211 max_load_reduction_pct=0.781418 moved_to_echo=25783 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 0, 32, 0, 24, 93, 2930, 1614, 345, 0, 0, 0, 0, 0, 0, 0, 1734, 15, 2, 0, 1764, 163, 1141, 2037, 214, 277, 0, 0, 0, 0, 0, 281, 4095, 0, 0, 1733, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 952, 3604, 0, 3369, 0, 1, 1507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 0, 4032, 0, 1709, 4095, 423, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 55, 0, 461, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 0, 1, 3934, 0, 3295, 3188, 0, 0, 0, 0, 0, 0, 0, 57, 0, 256, 0, 0, 241, 1528, 4096, 890, 773, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5066, 7347, 7282, 9433, 17109, 695, 10763, 7841] avg=8192.0000 max=17109 max_rank=4 min=695 min_rank=5 imbalance_ratio=2.088501 max_load_reduction=48427 max_load_reduction_pct=0.738937 moved_to_echo=36408 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[50, 0, 7, 0, 0, 4095, 320, 0, 1610, 1376, 0, 0, 0, 0, 0, 0, 550, 2, 0, 0, 0, 0, 3755, 0, 2191, 849, 2, 0, 0, 0, 0, 0, 24, 1, 0, 2167, 1868, 11, 1717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2149, 32, 2257, 3551, 3084, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 28, 0, 1715, 4081, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 4076, 0, 0, 1871, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 104, 0, 3, 0, 0, 0, 0, 2406, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1171, 0, 2, 0, 0, 125, 769, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7458, 7349, 5788, 14947, 9877, 14059, 3989, 2069] avg=8192.0000 max=14947 max_rank=3 min=2069 min_rank=7 imbalance_ratio=1.824585 max_load_reduction=50589 max_load_reduction_pct=0.771927 moved_to_echo=29994 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4038, 2974, 805, 1278, 0, 0, 0, 0, 0, 0, 2743, 0, 4072, 26, 0, 0, 10, 1, 1465, 788, 287, 0, 0, 0, 0, 0, 724, 3588, 109, 1423, 1249, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3781, 0, 0, 18, 4086, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 178, 3565, 3, 858, 4040, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 1, 3469, 0, 1736, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 4, 12, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 4, 22, 165, 0, 0, 0, 0, 0, 4076, 8, 384, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13191, 9392, 7111, 8754, 8649, 9326, 4454, 4659] avg=8192.0000 max=13191 max_rank=0 min=4454 min_rank=6 imbalance_ratio=1.610229 max_load_reduction=52345 max_load_reduction_pct=0.798721 moved_to_echo=27088 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11392, 0, 9728, 14080, 0, 10368, 0, 20480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[512, 8192, 128, 4736, 768, 5248, 8320, 19968, 10880, 7552, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11264, 8192, 5888, 14208, 7168, 5504, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3456, 256, 17280, 128, 11520, 12800, 11776, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 13312, 4608, 384, 1280, 256, 8448, 4096, 23168, 5376, 5248, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8704, 3968, 16256, 3712, 1152, 5504, 6528, 5888, 8320, 4736, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1408, 384, 128, 8448, 128, 2048, 8320, 640, 25856, 18944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6656, 18688, 2432, 11648, 7168, 16896, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1, 4096, 0, 1412, 0, 4096, 0, 0, 0, 2166, 0, 0, 1, 2275, 3950, 0, 0, 0, 3564, 194, 4095, 2145, 4048, 368, 0, 0, 0, 0, 149, 0, 0, 2458, 35, 4096, 0, 1567, 2, 4, 137, 4095, 3935, 0, 0, 0, 0, 3722, 0, 165, 0, 0, 0, 3, 2861, 0, 0, 36, 0, 1671, 0, 2, 4096, 0, 1, 4090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[249, 3845, 11, 0, 0, 4084, 0, 194, 191, 0, 12, 3483, 1198, 15, 3, 40, 0, 49, 4088, 4081, 1467, 1416, 4074, 13, 0, 0, 7, 301, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 3992, 1, 0, 3754, 0, 0, 4094, 0, 4095, 4096, 0, 4094, 65, 3309, 0, 0, 0, 0, 3439, 0, 0, 1352, 0, 3, 323] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[2259, 3195, 0, 0, 268, 4072, 0, 0, 0, 0, 0, 0, 0, 49, 341, 18, 0, 0, 3977, 1081, 4096, 133, 4096, 0, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 339, 1, 160, 0, 126, 94, 4096, 3673, 49, 4011, 93, 0, 3750, 2611, 3891, 0, 132, 2835, 0, 3563, 213, 0, 0, 1, 4094, 0, 0, 3799, 0, 0, 323] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[3478, 2, 91, 0, 0, 1, 0, 0, 0, 4019, 244, 28, 6, 224, 3653, 6, 0, 0, 139, 4087, 255, 3916, 4084, 9, 0, 1828, 29, 4060, 0, 0, 0, 0, 0, 1017, 3615, 47, 6, 15, 0, 3985, 18, 0, 4092, 0, 0, 4094, 3492, 3947, 3986, 0, 0, 33, 4096, 0, 0, 0, 0, 2918, 0, 0, 0, 0, 2, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[3578, 1, 3880, 0, 0, 47, 0, 224, 0, 4027, 48, 41, 4072, 4075, 4, 12, 4, 0, 1, 3373, 10, 0, 4064, 3, 0, 4039, 1, 0, 0, 0, 0, 0, 3328, 18, 0, 26, 25, 0, 0, 4056, 1, 0, 4095, 733, 0, 3634, 4051, 4095, 11, 3197, 0, 54, 1782, 0, 719, 0, 0, 2044, 0, 0, 1685, 0, 5, 473] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 184, 3406, 31, 0, 0, 1, 1264, 0, 0, 12, 3876, 66, 0, 115, 0, 510, 0, 0, 0, 2193, 9, 98, 3183, 0, 59, 1291, 0, 3572, 5, 4096, 0, 0, 771, 0, 0, 3820, 0, 0, 0, 0, 874, 2903, 1665, 0, 0, 3779, 3323, 434, 3894, 6, 23, 18, 0, 3850, 4008, 2, 4050, 0, 0, 4048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[8, 2774, 0, 2828, 4076, 0, 0, 0, 289, 35, 33, 4014, 4077, 4078, 2505, 2, 0, 0, 8, 44, 251, 3902, 4096, 1318, 1, 61, 4039, 0, 0, 0, 0, 0, 45, 21, 0, 4096, 0, 2, 59, 4096, 1259, 0, 3822, 1568, 0, 0, 358, 2, 0, 39, 0, 0, 1807, 47, 0, 57, 0, 4094, 0, 1728, 3995, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3399, 4083, 1, 2, 2001, 0, 0, 179, 0, 1332, 0, 0, 1049, 0, 1035, 0, 0, 93, 3962, 0, 331, 4083, 7, 3570, 1, 32, 3870, 0, 2, 5, 0, 0, 0, 0, 131, 28, 21, 0, 0, 4063, 0, 0, 0, 0, 0, 3849, 0, 0, 4095, 4078, 0, 3740, 4096, 0, 0, 231, 0, 4001, 2163, 314, 4096, 18, 0, 1575] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[249, 3738, 11, 0, 0, 4084, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 12, 3483, 1198, 15, 3, 40, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4088, 4081, 1467, 1416, 432, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 301, 0, 0, 0, 0, 3642, 1645, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 3992, 428, 339, 0, 0, 0, 0, 0, 0, 1, 0, 3754, 0, 0, 816, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 65, 3309, 0, 0, 0, 107, 21, 0, 0, 0, 0, 0, 0, 0, 3079, 0, 0, 1352, 0, 3, 323, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 6147, 11546, 5595, 8857, 8666, 11692, 4757] avg=8192.0000 max=11692 max_rank=6 min=4757 min_rank=7 imbalance_ratio=1.427246 max_load_reduction=53844 max_load_reduction_pct=0.821594 moved_to_echo=33972 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3399, 3969, 1, 2, 2001, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1332, 0, 0, 1049, 0, 1035, 0, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 93, 3962, 0, 331, 4083, 1, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 3870, 0, 2, 5, 0, 0, 6, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 131, 28, 21, 0, 0, 4063, 402, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4078, 0, 3740, 4096, 0, 0, 231, 114, 25, 0, 0, 0, 0, 0, 0, 0, 3582, 2163, 314, 4096, 18, 0, 1575, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9551, 4549, 12040, 5462, 5039, 768, 16379, 11748] avg=8192.0000 max=16379 max_rank=6 min=768 min_rank=5 imbalance_ratio=1.999390 max_load_reduction=49157 max_load_reduction_pct=0.750076 moved_to_echo=33934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 3982, 0, 1412, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 1, 2275, 3950, 0, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3564, 194, 4095, 2145, 429, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 2458, 3619, 1495, 0, 0, 0, 0, 0, 0, 35, 4096, 0, 1567, 2, 4, 137, 4095, 389, 164, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 743, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2861, 0, 0, 36, 114, 10, 0, 0, 0, 0, 0, 0, 0, 1497, 0, 2, 4096, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9491, 9487, 10795, 7721, 10489, 4843, 3024, 9686] avg=8192.0000 max=10795 max_rank=2 min=3024 min_rank=6 imbalance_ratio=1.317749 max_load_reduction=54741 max_load_reduction_pct=0.835281 moved_to_echo=28042 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2259, 3106, 0, 0, 268, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 341, 18, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 1081, 4096, 133, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4094, 0, 0, 0, 0, 0, 3662, 1506, 0, 0, 0, 0, 0, 0, 0, 339, 1, 160, 0, 126, 94, 4096, 392, 403, 0, 0, 0, 0, 0, 0, 3673, 49, 4011, 93, 0, 749, 2611, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 2835, 0, 3563, 213, 0, 0, 89, 26, 0, 0, 0, 0, 0, 0, 1, 3665, 0, 0, 3799, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9705, 1511, 9721, 9265, 5611, 15077, 6858, 7788] avg=8192.0000 max=15077 max_rank=5 min=1511 min_rank=1 imbalance_ratio=1.840454 max_load_reduction=50459 max_load_reduction_pct=0.769943 moved_to_echo=35334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3578, 1, 3880, 0, 0, 47, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 48, 41, 4072, 4075, 4, 12, 1069, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 3373, 10, 0, 431, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 1, 0, 0, 0, 0, 0, 3633, 1460, 0, 0, 0, 0, 0, 0, 3328, 18, 0, 26, 25, 0, 0, 4056, 380, 201, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 733, 0, 725, 4051, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3197, 0, 54, 1782, 0, 719, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1830, 0, 0, 1685, 0, 5, 473, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7730, 13348, 3822, 9133, 8034, 13700, 5776, 3993] avg=8192.0000 max=13700 max_rank=5 min=3822 min_rank=2 imbalance_ratio=1.672363 max_load_reduction=51836 max_load_reduction_pct=0.790955 moved_to_echo=31503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3478, 0, 91, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 244, 28, 6, 224, 3653, 6, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 4087, 255, 3916, 431, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1828, 29, 4060, 0, 0, 0, 0, 3653, 1649, 0, 0, 0, 0, 0, 0, 0, 1017, 3615, 47, 6, 15, 0, 3985, 432, 293, 0, 0, 0, 0, 0, 0, 18, 0, 4092, 0, 0, 805, 3492, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 0, 33, 4096, 0, 0, 0, 2, 22, 0, 0, 0, 0, 0, 0, 0, 2603, 0, 0, 0, 0, 2, 14, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3570, 9388, 8837, 11219, 9410, 12354, 8139, 2619] avg=8192.0000 max=12354 max_rank=5 min=2619 min_rank=7 imbalance_ratio=1.508057 max_load_reduction=53182 max_load_reduction_pct=0.811493 moved_to_echo=32522 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 3982, 0, 0, 184, 3406, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1264, 0, 0, 12, 3876, 66, 854, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 510, 0, 0, 0, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 9, 98, 3183, 0, 59, 1291, 0, 3572, 0, 1166, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 771, 0, 0, 3820, 303, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 580, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 3323, 434, 3894, 6, 23, 18, 114, 24, 0, 0, 0, 0, 0, 0, 0, 3447, 4008, 2, 4050, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7604, 6073, 2818, 9378, 9374, 3119, 11615, 15555] avg=8192.0000 max=15555 max_rank=7 min=2818 min_rank=2 imbalance_ratio=1.898804 max_load_reduction=49981 max_load_reduction_pct=0.762650 moved_to_echo=39663 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[8, 2697, 0, 2828, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 35, 33, 4014, 4077, 4078, 2505, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 44, 251, 3902, 434, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 1, 61, 4039, 0, 0, 0, 0, 0, 3662, 0, 0, 0, 0, 0, 0, 0, 45, 21, 0, 4096, 0, 2, 59, 4096, 0, 403, 0, 0, 0, 0, 0, 0, 1259, 0, 3822, 1568, 0, 0, 358, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 1807, 47, 0, 57, 77, 26, 0, 0, 0, 0, 0, 0, 0, 3665, 0, 1728, 3995, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9609, 15033, 5957, 7763, 8722, 7009, 2053, 9390] avg=8192.0000 max=15033 max_rank=1 min=2053 min_rank=6 imbalance_ratio=1.835083 max_load_reduction=50503 max_load_reduction_pct=0.770615 moved_to_echo=27174 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8960, 128, 19840, 2432, 896, 5248, 12288, 16256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 11648, 1664, 7680, 10496, 10752, 15488, 256, 7680, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 23424, 6272, 2048, 23168, 128, 128, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12288, 11264, 10368, 4352, 25472, 384, 768, 384, 640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[13056, 21504, 4096, 4352, 6656, 15744, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 384, 15744, 13440, 10624, 15616, 2688, 7552, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 6144, 15232, 4480, 256, 1408, 0, 6144, 21888, 10496, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3456, 13696, 3840, 6016, 896, 256, 384, 32256, 2816, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[131, 3302, 353, 287, 1377, 369, 692, 1349, 1133, 158, 1458, 179, 471, 770, 384, 432, 78, 660, 1097, 704, 948, 244, 2178, 2199, 465, 1108, 708, 1504, 1326, 455, 1777, 1948, 3103, 466, 657, 611, 47, 2455, 2411, 273, 3015, 715, 1364, 866, 1638, 1772, 800, 1140, 260, 519, 394, 2302, 1277, 541, 550, 63, 387, 406, 849, 2067, 1283, 478, 1173, 1410] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[101, 3081, 489, 469, 1104, 371, 1026, 827, 984, 97, 2055, 113, 909, 781, 296, 429, 172, 483, 1213, 440, 1007, 388, 3183, 2427, 627, 1581, 845, 1231, 1400, 420, 1803, 1914, 3252, 757, 587, 602, 26, 2431, 1622, 596, 2545, 426, 1654, 980, 1345, 1613, 1076, 1213, 369, 308, 390, 1928, 791, 658, 318, 86, 548, 158, 1201, 2252, 1224, 612, 492, 1210] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[85, 2816, 507, 404, 1510, 407, 765, 1223, 1139, 154, 1631, 334, 830, 723, 179, 610, 298, 432, 994, 1280, 851, 214, 3157, 2241, 261, 1158, 921, 1135, 1281, 469, 1580, 1934, 3045, 424, 297, 497, 127, 1980, 2355, 662, 2810, 474, 1484, 1028, 1672, 1457, 1176, 723, 298, 316, 348, 3103, 974, 393, 708, 47, 438, 234, 895, 1688, 1466, 509, 1301, 1084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[72, 2792, 422, 300, 1097, 488, 611, 1356, 1257, 242, 1270, 181, 1309, 1289, 219, 382, 126, 451, 1807, 718, 1021, 298, 2489, 1927, 375, 1192, 901, 1206, 1111, 378, 2245, 2465, 2630, 708, 549, 422, 223, 2538, 1997, 610, 2420, 251, 1681, 630, 1541, 1494, 1263, 1620, 295, 341, 469, 1312, 1100, 625, 604, 65, 790, 440, 776, 1833, 1933, 796, 558, 1025] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[194, 2873, 533, 416, 974, 205, 846, 968, 1227, 165, 1759, 606, 691, 833, 512, 405, 56, 600, 935, 596, 837, 354, 2864, 2019, 571, 1069, 932, 1204, 1358, 583, 1954, 1964, 2840, 559, 838, 540, 179, 2100, 1876, 916, 2543, 701, 1801, 952, 1180, 1960, 804, 1025, 220, 447, 333, 2166, 811, 687, 361, 76, 852, 555, 1197, 1848, 1595, 366, 1121, 984] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[148, 2766, 989, 219, 1506, 318, 921, 1286, 1191, 175, 1118, 320, 1027, 525, 156, 355, 133, 566, 879, 1207, 1120, 53, 3210, 1876, 587, 879, 1122, 1152, 1806, 236, 1581, 1907, 2886, 713, 324, 607, 178, 2234, 2283, 474, 2878, 441, 1493, 1075, 1467, 1461, 1121, 765, 414, 496, 382, 2893, 802, 475, 458, 152, 565, 225, 913, 1742, 1245, 451, 1315, 1274] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[95, 2843, 823, 402, 1062, 358, 760, 999, 1800, 147, 1305, 480, 1068, 628, 165, 414, 88, 385, 1389, 1140, 1024, 181, 2725, 2147, 572, 877, 763, 795, 2007, 433, 2311, 1887, 2838, 377, 627, 937, 91, 2003, 1556, 655, 2843, 749, 1558, 958, 1433, 1889, 828, 732, 257, 500, 250, 2268, 1092, 478, 622, 66, 573, 504, 1045, 1801, 1655, 373, 955, 950] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[96, 2728, 371, 324, 1537, 517, 622, 1288, 953, 183, 1252, 174, 1043, 1092, 137, 339, 67, 592, 662, 676, 1376, 107, 3356, 2079, 366, 1073, 717, 1485, 796, 528, 1465, 1885, 3212, 472, 192, 447, 223, 2732, 2901, 323, 2480, 316, 1572, 824, 1391, 1929, 1348, 1603, 386, 359, 283, 2636, 1009, 312, 423, 104, 998, 205, 1035, 2209, 1081, 657, 1211, 777] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[131, 3302, 353, 287, 1377, 369, 692, 1349, 440, 83, 0, 0, 0, 0, 0, 0, 1133, 158, 1458, 179, 471, 770, 384, 432, 1632, 1024, 0, 0, 0, 0, 0, 0, 78, 660, 1097, 704, 948, 244, 1663, 2199, 0, 0, 0, 0, 0, 0, 0, 0, 465, 1108, 708, 1504, 1326, 455, 1777, 841, 0, 0, 0, 0, 0, 0, 0, 0, 1467, 466, 657, 611, 47, 2455, 2411, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 715, 1364, 866, 1638, 1650, 800, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 260, 519, 394, 2302, 1277, 541, 550, 63, 3015, 4, 0, 0, 0, 0, 0, 0, 387, 406, 849, 2067, 1283, 478, 1173, 1410, 122, 75, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8383, 7641, 7593, 8184, 8387, 8173, 8925, 8250] avg=8192.0000 max=8925 max_rank=6 min=7593 min_rank=2 imbalance_ratio=1.089478 max_load_reduction=56611 max_load_reduction_pct=0.863815 moved_to_echo=33735 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[101, 3081, 489, 469, 1104, 371, 1026, 827, 643, 82, 0, 0, 0, 0, 0, 0, 984, 97, 2055, 113, 909, 781, 296, 429, 1710, 1007, 0, 0, 0, 0, 0, 0, 172, 483, 1213, 440, 1007, 388, 2430, 2427, 0, 0, 0, 0, 0, 0, 0, 0, 627, 1581, 845, 1231, 1400, 420, 1803, 825, 0, 0, 0, 0, 0, 0, 0, 0, 1538, 757, 587, 602, 26, 2431, 1622, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 1654, 980, 1345, 1502, 1076, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 369, 308, 390, 1928, 791, 658, 318, 86, 2545, 4, 0, 0, 0, 0, 0, 0, 548, 158, 1201, 2252, 1224, 612, 492, 1210, 111, 110, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8381, 8560, 8732, 8159, 8196, 7397, 7918] avg=8192.0000 max=8732 max_rank=3 min=7397 min_rank=6 imbalance_ratio=1.065918 max_load_reduction=56804 max_load_reduction_pct=0.866760 moved_to_echo=31670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[85, 2816, 507, 404, 1510, 407, 765, 1223, 638, 83, 0, 0, 0, 0, 0, 0, 1139, 154, 1631, 334, 830, 723, 179, 610, 1602, 1017, 0, 0, 0, 0, 0, 0, 298, 432, 994, 1280, 851, 214, 2410, 2241, 0, 0, 0, 0, 0, 0, 0, 0, 261, 1158, 921, 1135, 1281, 469, 1580, 834, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 424, 297, 497, 127, 1980, 2355, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 1484, 1028, 1672, 1357, 1176, 723, 0, 0, 0, 0, 0, 0, 0, 0, 298, 316, 348, 3103, 974, 393, 708, 47, 2810, 4, 0, 0, 0, 0, 0, 0, 438, 234, 895, 1688, 1466, 509, 1301, 1084, 100, 109, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8438, 8219, 8720, 7639, 7781, 7914, 9001, 7824] avg=8192.0000 max=9001 max_rank=6 min=7639 min_rank=3 imbalance_ratio=1.098755 max_load_reduction=56535 max_load_reduction_pct=0.862656 moved_to_echo=32520 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[72, 2792, 422, 300, 1097, 488, 611, 1356, 503, 106, 0, 0, 0, 0, 0, 0, 1257, 242, 1270, 181, 1309, 1289, 219, 382, 1383, 1296, 0, 0, 0, 0, 0, 0, 126, 451, 1807, 718, 1021, 298, 1900, 1927, 0, 0, 0, 0, 0, 0, 0, 0, 375, 1192, 901, 1206, 1111, 378, 2245, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 1244, 708, 549, 422, 223, 2538, 1997, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 1681, 630, 1541, 1391, 1263, 1620, 0, 0, 0, 0, 0, 0, 0, 0, 295, 341, 469, 1312, 1100, 625, 604, 65, 2420, 3, 0, 0, 0, 0, 0, 0, 790, 440, 776, 1833, 1933, 796, 558, 1025, 103, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7747, 8828, 8248, 8471, 8291, 8377, 7234, 8340] avg=8192.0000 max=8828 max_rank=1 min=7234 min_rank=6 imbalance_ratio=1.077637 max_load_reduction=56708 max_load_reduction_pct=0.865295 moved_to_echo=32242 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[96, 2728, 371, 324, 1537, 517, 622, 1288, 678, 81, 0, 0, 0, 0, 0, 0, 953, 183, 1252, 174, 1043, 1092, 137, 339, 1689, 991, 0, 0, 0, 0, 0, 0, 67, 592, 662, 676, 1376, 107, 2562, 2079, 0, 0, 0, 0, 0, 0, 0, 0, 366, 1073, 717, 1485, 796, 528, 1465, 813, 0, 0, 0, 0, 0, 0, 0, 0, 1519, 472, 192, 447, 223, 2732, 2901, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 1572, 824, 1391, 1796, 1348, 1603, 0, 0, 0, 0, 0, 0, 0, 0, 386, 359, 283, 2636, 1009, 312, 423, 104, 2480, 4, 0, 0, 0, 0, 0, 0, 998, 205, 1035, 2209, 1081, 657, 1211, 777, 133, 116, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 7853, 8121, 7243, 8809, 8850, 7996, 8422] avg=8192.0000 max=8850 max_rank=5 min=7243 min_rank=3 imbalance_ratio=1.080322 max_load_reduction=56686 max_load_reduction_pct=0.864960 moved_to_echo=34077 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[194, 2873, 533, 416, 974, 205, 846, 968, 578, 84, 0, 0, 0, 0, 0, 0, 1227, 165, 1759, 606, 691, 833, 512, 405, 1494, 1033, 0, 0, 0, 0, 0, 0, 56, 600, 935, 596, 837, 354, 2187, 2019, 0, 0, 0, 0, 0, 0, 0, 0, 571, 1069, 932, 1204, 1358, 583, 1954, 847, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 559, 838, 540, 179, 2100, 1876, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 1801, 952, 1180, 1825, 804, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 220, 447, 333, 2166, 811, 687, 361, 76, 2543, 3, 0, 0, 0, 0, 0, 0, 852, 555, 1197, 1848, 1595, 366, 1121, 984, 135, 99, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7671, 8725, 7584, 8518, 8351, 8288, 7647, 8752] avg=8192.0000 max=8752 max_rank=7 min=7584 min_rank=2 imbalance_ratio=1.068359 max_load_reduction=56784 max_load_reduction_pct=0.866455 moved_to_echo=33038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[95, 2843, 823, 402, 1062, 358, 760, 999, 550, 81, 0, 0, 0, 0, 0, 0, 1800, 147, 1305, 480, 1068, 628, 165, 414, 1493, 992, 0, 0, 0, 0, 0, 0, 88, 385, 1389, 1140, 1024, 181, 2081, 2147, 0, 0, 0, 0, 0, 0, 0, 0, 572, 877, 763, 795, 2007, 433, 2311, 814, 0, 0, 0, 0, 0, 0, 0, 0, 1342, 377, 627, 937, 91, 2003, 1556, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 749, 1558, 958, 1433, 1759, 828, 732, 0, 0, 0, 0, 0, 0, 0, 0, 257, 500, 250, 2268, 1092, 478, 622, 66, 2843, 3, 0, 0, 0, 0, 0, 0, 573, 504, 1045, 1801, 1655, 373, 955, 950, 130, 94, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7973, 8492, 8435, 8572, 7588, 8017, 8379, 8080] avg=8192.0000 max=8572 max_rank=3 min=7588 min_rank=4 imbalance_ratio=1.046387 max_load_reduction=56964 max_load_reduction_pct=0.869202 moved_to_echo=32064 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[148, 2766, 989, 219, 1506, 318, 921, 1286, 651, 85, 0, 0, 0, 0, 0, 0, 1191, 175, 1118, 320, 1027, 525, 156, 355, 1522, 1008, 0, 0, 0, 0, 0, 0, 133, 566, 879, 1207, 1120, 53, 2441, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 587, 879, 1122, 1152, 1806, 236, 1581, 814, 0, 0, 0, 0, 0, 0, 0, 0, 1357, 713, 324, 607, 178, 2234, 2283, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 441, 1493, 1075, 1467, 1359, 1121, 765, 0, 0, 0, 0, 0, 0, 0, 0, 414, 496, 382, 2893, 802, 475, 458, 152, 2878, 7, 0, 0, 0, 0, 0, 0, 565, 225, 913, 1742, 1245, 451, 1315, 1274, 102, 118, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8889, 7397, 8275, 8177, 8170, 7721, 8957, 7950] avg=8192.0000 max=8957 max_rank=6 min=7397 min_rank=1 imbalance_ratio=1.093384 max_load_reduction=56579 max_load_reduction_pct=0.863327 moved_to_echo=32798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 4096, 12672, 7424, 11776, 12672, 8448, 8832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[3840, 8960, 6912, 9728, 11136, 3584, 14720, 6912, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[5248, 2816, 7936, 15488, 11520, 4352, 8192, 8832, 1024, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[11264, 4480, 4096, 4736, 1152, 18560, 17024, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1024, 4224, 9088, 6784, 8192, 1920, 17792, 17024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[9728, 1408, 11904, 2432, 7424, 6656, 2048, 3456, 12544, 8448, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1024, 23296, 4608, 2944, 10240, 3072, 6272, 9344, 4736, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2560, 3328, 2944, 18688, 7936, 4224, 4096, 768, 21632, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4075, 10, 123, 3570, 15, 388, 3785, 11, 2, 3631, 12, 9, 13, 249, 65, 26, 24, 4095, 696, 370, 51, 479, 23, 2750, 3145, 2, 185, 3060, 4081, 1115, 16, 891, 614, 1317, 4, 4013, 397, 2812, 0, 1786, 2781, 0, 43, 672, 47, 122, 52, 246, 2973, 3244, 3242, 240, 26, 0, 4, 8, 13, 4, 1, 26, 0, 238, 3386, 258] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2317, 5, 31, 2613, 79, 961, 4069, 0, 1, 3824, 6, 2, 31, 1, 71, 268, 0, 3989, 623, 3962, 20, 157, 0, 2934, 1476, 1, 0, 3339, 3964, 2, 12, 332, 13, 3064, 0, 4070, 172, 2757, 4, 22, 132, 0, 194, 1862, 10, 638, 1018, 39, 2906, 2990, 3968, 0, 172, 0, 60, 6, 161, 0, 364, 48, 0, 191, 4095, 1490] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4032, 2, 408, 3963, 6, 28, 4093, 0, 26, 1375, 0, 1, 0, 13, 1, 48, 0, 3972, 161, 1950, 130, 838, 0, 1091, 1264, 492, 5, 3948, 3987, 10, 4, 217, 3386, 1407, 1, 4058, 119, 2914, 1, 32, 1508, 34, 624, 2573, 729, 540, 955, 113, 2986, 3337, 3711, 3, 22, 6, 0, 7, 1, 9, 492, 66, 219, 15, 3219, 384] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4012, 0, 391, 3931, 41, 89, 4080, 0, 9, 2415, 1, 2, 0, 61, 0, 0, 30, 2992, 3300, 66, 77, 3527, 12, 2329, 1430, 225, 0, 4077, 4075, 285, 0, 17, 15, 2194, 0, 4083, 764, 3749, 0, 40, 0, 0, 53, 333, 11, 0, 2796, 2, 683, 3987, 4024, 34, 8, 8, 208, 60, 11, 0, 14, 165, 2, 699, 4058, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[3031, 224, 29, 3790, 189, 504, 1018, 0, 0, 4050, 0, 19, 1, 36, 6, 9, 1, 4069, 1215, 1, 399, 972, 2, 443, 3993, 21, 1, 4007, 3831, 2, 5, 117, 16, 751, 0, 3668, 1006, 3853, 4, 1, 2962, 0, 151, 3497, 351, 1362, 83, 215, 3686, 217, 2539, 0, 346, 1, 16, 210, 5, 1, 218, 102, 40, 1232, 3951, 3067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2395, 2, 1425, 3479, 49, 24, 4011, 14, 31, 2398, 0, 0, 2, 46, 618, 40, 37, 4006, 2830, 1713, 35, 1198, 0, 481, 1158, 15, 108, 2143, 2393, 299, 38, 866, 894, 738, 0, 3572, 3874, 2847, 0, 64, 2806, 0, 3, 1145, 71, 107, 1443, 125, 4014, 2548, 3916, 0, 24, 0, 91, 0, 7, 18, 0, 153, 2, 883, 3825, 512] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[2974, 35, 1099, 3740, 61, 1673, 4090, 5, 0, 3987, 0, 31, 0, 134, 23, 69, 1, 4060, 1392, 1911, 6, 851, 0, 0, 3186, 11, 3, 4051, 4053, 23, 0, 125, 1108, 392, 18, 3995, 2138, 2421, 77, 373, 36, 0, 9, 2541, 33, 59, 4, 0, 4037, 564, 3301, 43, 9, 80, 4, 13, 0, 0, 622, 191, 4, 498, 3979, 1393] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[3651, 29, 41, 924, 262, 22, 4024, 0, 0, 3287, 0, 0, 1, 2, 0, 2, 12, 4063, 1338, 1415, 29, 682, 0, 378, 4089, 34, 8, 3791, 3961, 24, 0, 16, 138, 372, 7, 3540, 2974, 3983, 0, 248, 1575, 0, 20, 2498, 379, 1214, 62, 18, 2315, 3342, 790, 547, 474, 0, 0, 0, 227, 0, 1397, 19, 1, 2704, 2032, 2575] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4075, 10, 123, 3570, 15, 388, 619, 11, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3631, 12, 9, 13, 249, 65, 26, 3166, 1835, 0, 0, 0, 0, 0, 0, 24, 2962, 696, 370, 51, 479, 23, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 3145, 2, 185, 3060, 1595, 1115, 16, 891, 0, 0, 0, 0, 0, 0, 0, 0, 614, 1317, 4, 1250, 397, 2812, 0, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 2781, 0, 43, 672, 47, 122, 52, 246, 2349, 928, 0, 0, 0, 0, 0, 0, 2973, 3244, 2414, 240, 26, 0, 4, 8, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4, 1, 26, 0, 238, 3386, 258, 1133, 828, 137, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8811, 9008, 7355, 10009, 8180, 7240, 8909, 6024] avg=8192.0000 max=10009 max_rank=3 min=6024 min_rank=7 imbalance_ratio=1.221802 max_load_reduction=55527 max_load_reduction_pct=0.847275 moved_to_echo=30353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2317, 5, 31, 2613, 79, 961, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3824, 6, 2, 31, 1, 71, 268, 3404, 1861, 0, 0, 0, 0, 0, 0, 0, 2885, 623, 3962, 20, 157, 0, 2934, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 1, 0, 3339, 1549, 2, 12, 332, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3064, 0, 1268, 172, 2757, 4, 22, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 194, 1862, 10, 638, 1018, 39, 2282, 941, 0, 0, 0, 0, 0, 0, 2906, 2990, 2955, 0, 172, 0, 60, 6, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 364, 48, 0, 191, 4095, 1490, 1104, 1013, 133, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6671, 9469, 10581, 6711, 7300, 7116, 9089, 8599] avg=8192.0000 max=10581 max_rank=2 min=6671 min_rank=0 imbalance_ratio=1.291626 max_load_reduction=54955 max_load_reduction_pct=0.838547 moved_to_echo=32104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4012, 0, 391, 3931, 41, 89, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2415, 1, 2, 0, 61, 0, 0, 3413, 1867, 0, 0, 0, 0, 0, 0, 30, 2164, 3300, 66, 77, 3527, 12, 2329, 0, 0, 0, 0, 0, 0, 0, 0, 1430, 225, 0, 4077, 1592, 285, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2194, 0, 1272, 764, 3749, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 333, 11, 0, 2796, 2, 2346, 944, 0, 0, 0, 0, 0, 0, 683, 3987, 2996, 34, 8, 8, 208, 60, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 14, 165, 2, 699, 4058, 61, 828, 1028, 137, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9131, 7768, 11505, 7626, 8034, 6485, 7984, 7003] avg=8192.0000 max=11505 max_rank=2 min=6485 min_rank=5 imbalance_ratio=1.404419 max_load_reduction=54031 max_load_reduction_pct=0.824448 moved_to_echo=29506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4032, 2, 408, 3963, 6, 28, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 1375, 0, 1, 0, 13, 1, 48, 3424, 1855, 0, 0, 0, 0, 0, 0, 0, 2873, 161, 1950, 130, 838, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 492, 5, 3948, 1558, 10, 4, 217, 0, 0, 0, 0, 0, 0, 0, 0, 3386, 1407, 1, 1265, 119, 2914, 1, 32, 0, 0, 0, 0, 0, 0, 0, 0, 1508, 34, 624, 2573, 729, 540, 955, 113, 2295, 938, 0, 0, 0, 0, 0, 0, 2986, 3337, 2763, 3, 22, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 492, 66, 219, 15, 3219, 384, 1099, 948, 134, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9108, 6743, 7043, 7498, 9125, 10309, 9124, 6586] avg=8192.0000 max=10309 max_rank=5 min=6586 min_rank=7 imbalance_ratio=1.258423 max_load_reduction=55227 max_load_reduction_pct=0.842697 moved_to_echo=35144 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3031, 224, 29, 3790, 189, 504, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 19, 1, 36, 6, 9, 851, 1677, 0, 0, 0, 0, 0, 0, 1, 2943, 1215, 1, 399, 972, 2, 443, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 21, 1, 4007, 1497, 2, 5, 117, 0, 0, 0, 0, 0, 0, 0, 0, 16, 751, 0, 1143, 1006, 3853, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2962, 0, 151, 3497, 351, 1362, 83, 215, 2205, 848, 0, 0, 0, 0, 0, 0, 3686, 217, 1891, 0, 346, 1, 16, 210, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 218, 102, 40, 1232, 3951, 3067, 1126, 648, 129, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7934, 6649, 5976, 9643, 6774, 11674, 6367, 10519] avg=8192.0000 max=11674 max_rank=5 min=5976 min_rank=2 imbalance_ratio=1.425049 max_load_reduction=53862 max_load_reduction_pct=0.821869 moved_to_echo=35334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2395, 2, 1425, 3479, 49, 24, 656, 14, 0, 0, 0, 0, 0, 0, 0, 0, 31, 2398, 0, 0, 2, 46, 618, 40, 3355, 1633, 0, 0, 0, 0, 0, 0, 37, 2897, 2830, 1713, 35, 1198, 0, 481, 0, 0, 0, 0, 0, 0, 0, 0, 1158, 15, 108, 2143, 936, 299, 38, 866, 0, 0, 0, 0, 0, 0, 0, 0, 894, 738, 0, 1113, 3874, 2847, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 2806, 0, 3, 1145, 71, 107, 1443, 125, 1377, 826, 0, 0, 0, 0, 0, 0, 4014, 2548, 2916, 0, 24, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 18, 0, 153, 2, 883, 3825, 512, 1109, 1000, 80, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8044, 8123, 9191, 5563, 9530, 7903, 9593, 7589] avg=8192.0000 max=9593 max_rank=6 min=5563 min_rank=3 imbalance_ratio=1.171021 max_load_reduction=55943 max_load_reduction_pct=0.853622 moved_to_echo=34615 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2974, 35, 1099, 3740, 61, 1673, 668, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 31, 0, 134, 23, 69, 3422, 1826, 0, 0, 0, 0, 0, 0, 1, 2936, 1392, 1911, 6, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3186, 11, 3, 4051, 1584, 23, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 392, 18, 1246, 2138, 2421, 77, 373, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 9, 2541, 33, 59, 4, 0, 2333, 923, 0, 0, 0, 0, 0, 0, 4037, 564, 2458, 43, 9, 80, 4, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 191, 4, 498, 3979, 1393, 1124, 843, 136, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10255, 9492, 7097, 8983, 7773, 5938, 7208, 8790] avg=8192.0000 max=10255 max_rank=0 min=5938 min_rank=5 imbalance_ratio=1.251831 max_load_reduction=55281 max_load_reduction_pct=0.843521 moved_to_echo=29709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3651, 29, 41, 924, 262, 22, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3287, 0, 0, 1, 2, 0, 2, 3371, 1621, 0, 0, 0, 0, 0, 0, 12, 2934, 1338, 1415, 29, 682, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 34, 8, 3791, 1541, 24, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 138, 372, 7, 1098, 2974, 3983, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 1575, 0, 20, 2498, 379, 1214, 62, 18, 2284, 821, 0, 0, 0, 0, 0, 0, 2315, 3342, 584, 547, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 1397, 19, 1, 2704, 2032, 2575, 1129, 206, 136, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5582, 8284, 6788, 9503, 8820, 8871, 7262, 10426] avg=8192.0000 max=10426 max_rank=7 min=5582 min_rank=0 imbalance_ratio=1.272705 max_load_reduction=55110 max_load_reduction_pct=0.840912 moved_to_echo=35379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11904, 128, 1152, 15232, 1664, 4096, 6528, 768, 17536, 7296, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[19840, 896, 384, 28416, 11904, 1792, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23680, 20352, 19072, 896, 1152, 128, 384, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 25088, 128, 128, 128, 640, 896, 512, 24448, 14208, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26496, 384, 3584, 26112, 768, 3712, 4864, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6272, 10240, 128, 9728, 11520, 25344, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 22656, 11648, 11392, 768, 8704, 128, 10496, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 128, 3200, 896, 384, 6528, 28672, 9856, 8704, 6528, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4001, 0, 3997, 13, 0, 3, 4008, 4067, 3700, 76, 270, 102, 0, 12, 30, 443, 0, 1, 388, 38, 528, 64, 0, 1058, 41, 1006, 39, 2901, 4080, 52, 3806, 0, 63, 46, 4026, 0, 0, 860, 20, 8, 60, 2565, 932, 3070, 1, 0, 3697, 170, 9, 0, 38, 4096, 2827, 0, 2, 0, 0, 3037, 28, 1818, 3328, 19, 87, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1277, 0, 4023, 1, 2917, 22, 3257, 4096, 694, 4, 60, 10, 648, 4, 357, 0, 360, 0, 304, 0, 0, 2793, 10, 3473, 0, 0, 0, 0, 625, 1152, 4091, 4, 0, 375, 4083, 0, 8, 4018, 0, 531, 2320, 579, 0, 3727, 4, 0, 4095, 3511, 0, 2, 105, 3043, 549, 8, 125, 0, 0, 4080, 14, 1051, 3126, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3748, 0, 3742, 0, 16, 1, 3992, 3883, 4066, 0, 0, 15, 4, 0, 1, 12, 129, 56, 4, 0, 4, 267, 0, 3931, 6, 494, 158, 403, 1082, 0, 4063, 181, 10, 2124, 4092, 0, 92, 3961, 100, 8, 475, 1217, 9, 4096, 2, 117, 936, 611, 1, 39, 1, 4077, 3280, 433, 85, 3, 38, 4084, 1, 1804, 3575, 7, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4079, 3332, 4084, 0, 1922, 305, 0, 4088, 29, 0, 3699, 1, 0, 12, 0, 0, 0, 0, 22, 3, 107, 4093, 1, 3977, 0, 10, 0, 384, 1013, 15, 4094, 7, 211, 174, 3914, 0, 0, 0, 26, 111, 4016, 146, 0, 3849, 608, 0, 3832, 3713, 20, 5, 0, 2278, 3268, 120, 83, 2, 0, 3828, 19, 12, 21, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[620, 1, 4043, 10, 484, 0, 0, 3911, 3590, 9, 1, 0, 2, 9, 4017, 1151, 0, 0, 0, 184, 0, 2131, 339, 205, 0, 0, 173, 0, 0, 3309, 4045, 1763, 1, 50, 3938, 3671, 0, 3591, 17, 638, 280, 537, 0, 333, 0, 334, 4085, 3217, 39, 0, 0, 1317, 3687, 10, 37, 43, 9, 4092, 983, 63, 4056, 511, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[3348, 667, 4086, 113, 0, 0, 4, 3811, 3892, 111, 2220, 0, 1, 909, 0, 0, 0, 0, 316, 758, 28, 3921, 177, 1, 0, 0, 0, 436, 107, 3094, 4087, 37, 409, 201, 3828, 9, 1236, 389, 0, 1, 16, 3014, 6, 4000, 0, 0, 3465, 168, 0, 0, 4, 205, 225, 78, 3897, 20, 0, 3749, 0, 4000, 3729, 3, 0, 760] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3994, 0, 3992, 12, 3565, 12, 283, 4095, 148, 5, 3426, 7, 0, 0, 1439, 0, 14, 0, 44, 296, 56, 2611, 35, 4087, 0, 1742, 2885, 0, 23, 0, 4052, 13, 105, 26, 3986, 5, 0, 3274, 0, 0, 3062, 142, 1, 3948, 942, 143, 3548, 2796, 2, 0, 0, 86, 170, 151, 0, 3, 31, 4088, 926, 35, 1115, 0, 115, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4051, 0, 4096, 167, 10, 11, 3, 4095, 2970, 20, 3505, 27, 0, 0, 0, 1, 0, 31, 0, 369, 248, 304, 0, 3974, 0, 3661, 37, 2, 245, 4053, 3880, 0, 344, 0, 4082, 21, 7, 715, 8, 18, 206, 3343, 4, 3323, 0, 0, 3369, 1180, 66, 1, 0, 0, 2901, 39, 21, 0, 207, 4096, 0, 1141, 4086, 589, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4001, 0, 0, 13, 0, 3, 4008, 1941, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 76, 270, 102, 0, 12, 30, 443, 2708, 502, 0, 0, 0, 0, 0, 0, 0, 1, 388, 38, 528, 64, 0, 1058, 2102, 989, 0, 0, 0, 0, 0, 0, 41, 1006, 39, 2901, 4080, 52, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 46, 4026, 0, 0, 860, 20, 8, 277, 215, 146, 24, 0, 0, 0, 0, 60, 2565, 932, 2924, 1, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 38, 4096, 2827, 0, 2, 0, 3495, 0, 0, 0, 0, 0, 0, 0, 0, 2760, 28, 1818, 3328, 19, 87, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9966, 7843, 5168, 11710, 5685, 6652, 10467, 8045] avg=8192.0000 max=11710 max_rank=3 min=5168 min_rank=2 imbalance_ratio=1.429443 max_load_reduction=53826 max_load_reduction_pct=0.821320 moved_to_echo=30849 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1277, 0, 0, 1, 2917, 22, 3257, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 694, 4, 60, 10, 648, 4, 357, 0, 2999, 505, 0, 0, 0, 0, 0, 0, 360, 0, 304, 0, 0, 2793, 10, 3473, 2117, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 625, 1152, 3860, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 4083, 0, 8, 4018, 0, 531, 373, 231, 177, 24, 0, 0, 0, 0, 2320, 579, 0, 3550, 4, 0, 0, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 105, 3043, 549, 8, 125, 0, 3518, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 14, 1051, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9429, 5281, 10153, 5641, 9820, 9964, 7350, 7898] avg=8192.0000 max=10153 max_rank=2 min=5281 min_rank=1 imbalance_ratio=1.239380 max_load_reduction=55383 max_load_reduction_pct=0.845078 moved_to_echo=35032 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3748, 0, 0, 0, 16, 1, 3992, 1853, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 15, 4, 0, 1, 12, 686, 471, 0, 0, 0, 0, 0, 0, 129, 56, 4, 0, 4, 267, 0, 3931, 2007, 250, 0, 0, 0, 0, 0, 0, 6, 494, 158, 403, 1082, 0, 3834, 181, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2124, 4092, 0, 92, 3961, 100, 8, 373, 229, 195, 23, 0, 0, 0, 0, 475, 1217, 9, 3901, 2, 117, 0, 611, 0, 0, 0, 0, 0, 0, 0, 0, 1, 39, 1, 4077, 3280, 433, 85, 3, 3271, 0, 0, 0, 0, 0, 0, 0, 38, 3711, 1, 1804, 3575, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9610, 5255, 6648, 6158, 11207, 6332, 11190, 9136] avg=8192.0000 max=11207 max_rank=4 min=5255 min_rank=1 imbalance_ratio=1.368042 max_load_reduction=54329 max_load_reduction_pct=0.828995 moved_to_echo=37865 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4079, 3332, 0, 0, 1922, 305, 0, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 3699, 1, 0, 12, 0, 0, 2807, 513, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3, 107, 4093, 1, 3977, 2113, 1025, 0, 0, 0, 0, 0, 0, 0, 10, 0, 384, 1013, 15, 3863, 7, 0, 0, 0, 0, 0, 0, 0, 0, 211, 174, 3914, 0, 0, 0, 26, 111, 350, 231, 183, 24, 0, 0, 0, 0, 4016, 146, 0, 3666, 608, 0, 0, 3713, 0, 0, 0, 0, 0, 0, 0, 0, 20, 5, 0, 2278, 3268, 120, 83, 2, 3571, 0, 0, 0, 0, 0, 0, 0, 0, 3478, 19, 12, 21, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11589, 7061, 11341, 5292, 5224, 12149, 9347, 3533] avg=8192.0000 max=12149 max_rank=5 min=3533 min_rank=7 imbalance_ratio=1.483032 max_load_reduction=53387 max_load_reduction_pct=0.814621 moved_to_echo=30253 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[620, 1, 0, 10, 484, 0, 0, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 9, 1, 0, 2, 9, 4017, 1151, 2992, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 2131, 339, 205, 2022, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 3309, 3817, 1763, 0, 0, 0, 0, 0, 0, 0, 0, 1, 50, 3938, 3671, 0, 3591, 17, 638, 374, 228, 15, 23, 0, 0, 0, 0, 280, 537, 0, 318, 0, 334, 0, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 1317, 3687, 10, 37, 43, 3536, 0, 0, 0, 0, 0, 0, 0, 9, 3718, 983, 63, 4056, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2981, 12278, 5974, 9062, 12546, 4686, 8669, 9340] avg=8192.0000 max=12546 max_rank=4 min=2981 min_rank=0 imbalance_ratio=1.531494 max_load_reduction=52990 max_load_reduction_pct=0.808563 moved_to_echo=35241 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3348, 667, 0, 113, 0, 0, 4, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 111, 2220, 0, 1, 909, 0, 0, 2539, 514, 0, 0, 0, 0, 0, 0, 0, 0, 316, 758, 28, 3921, 177, 1, 1970, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 107, 3094, 3857, 37, 0, 0, 0, 0, 0, 0, 0, 0, 409, 201, 3828, 9, 1236, 389, 0, 1, 343, 230, 190, 22, 0, 0, 0, 0, 16, 3014, 6, 3810, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 205, 225, 78, 3897, 20, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 4000, 3729, 3, 0, 760, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5951, 10186, 8097, 7531, 6858, 7014, 8001, 11898] avg=8192.0000 max=11898 max_rank=7 min=5951 min_rank=0 imbalance_ratio=1.452393 max_load_reduction=53638 max_load_reduction_pct=0.818451 moved_to_echo=33771 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3994, 0, 0, 12, 3565, 12, 283, 1954, 0, 0, 0, 0, 0, 0, 0, 0, 148, 5, 3426, 7, 0, 0, 1439, 0, 2599, 501, 0, 0, 0, 0, 0, 0, 14, 0, 44, 296, 56, 2611, 35, 4087, 2117, 949, 0, 0, 0, 0, 0, 0, 0, 1742, 2885, 0, 23, 0, 3824, 13, 0, 0, 0, 0, 0, 0, 0, 0, 105, 26, 3986, 5, 0, 3274, 0, 0, 374, 228, 188, 24, 0, 0, 0, 0, 3062, 142, 1, 3760, 942, 143, 0, 2796, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 86, 170, 151, 0, 3, 3491, 0, 0, 0, 0, 0, 0, 0, 31, 3714, 926, 35, 1115, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9820, 8125, 10209, 8487, 8210, 10846, 3903, 5936] avg=8192.0000 max=10846 max_rank=5 min=3903 min_rank=6 imbalance_ratio=1.323975 max_load_reduction=54690 max_load_reduction_pct=0.834503 moved_to_echo=28895 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4051, 0, 0, 167, 10, 11, 3, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 2970, 20, 3505, 27, 0, 0, 0, 1, 2469, 515, 0, 0, 0, 0, 0, 0, 0, 31, 0, 369, 248, 304, 0, 3974, 2120, 900, 0, 0, 0, 0, 0, 0, 0, 3661, 37, 2, 245, 4053, 3657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 4082, 21, 7, 715, 8, 18, 378, 223, 163, 27, 0, 0, 0, 0, 206, 3343, 4, 3160, 0, 0, 0, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1, 0, 0, 2901, 39, 21, 0, 3581, 0, 0, 0, 0, 0, 0, 0, 207, 3718, 0, 1141, 4086, 589, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6190, 9507, 7946, 11655, 5986, 7893, 6609, 9750] avg=8192.0000 max=11655 max_rank=3 min=5986 min_rank=4 imbalance_ratio=1.422729 max_load_reduction=53881 max_load_reduction_pct=0.822159 moved_to_echo=30238 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25216, 4096, 0, 384, 8960, 384, 11648, 15360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 28288, 2048, 9984, 23040, 1152, 256, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 128, 1152, 1664, 1024, 16256, 640, 20736, 16640, 7296, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[10496, 11648, 1024, 25216, 1664, 640, 0, 15488, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[19200, 256, 13184, 256, 768, 1024, 5888, 1664, 19840, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 7040, 3328, 4224, 7296, 11776, 30336, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 128, 256, 15104, 17024, 896, 4352, 128, 28160, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1152, 3072, 32000, 3712, 1408, 16896, 256, 1408, 2944, 1920, 1280, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 3196, 0, 0, 0, 25, 136, 0, 2373, 94, 29, 65, 0, 0, 4084, 10, 3340, 54, 0, 1, 0, 0, 1, 696, 417, 3, 4056, 0, 0, 1, 58, 0, 3064, 1, 158, 2690, 0, 3856, 9, 4092, 2988, 679, 2673, 3560, 13, 270, 0, 4003, 182, 195, 3379, 2, 84, 3125, 1127, 4096, 2858, 43, 0, 2, 0, 3748, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[73, 28, 3993, 0, 248, 7, 0, 310, 5, 17, 2896, 41, 1108, 38, 0, 4052, 283, 2674, 0, 327, 0, 0, 24, 6, 605, 0, 0, 4073, 5, 0, 0, 1727, 14, 31, 0, 3433, 788, 1, 2689, 1, 1793, 1408, 3, 4047, 2245, 0, 961, 163, 2976, 0, 0, 2115, 3935, 0, 3927, 1962, 4095, 275, 13, 19, 83, 0, 3070, 2949] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4093, 5, 838, 1, 0, 0, 347, 1, 0, 0, 13, 0, 0, 0, 0, 2531, 0, 37, 127, 59, 0, 1182, 1171, 1063, 0, 4087, 190, 0, 0, 3073, 0, 548, 1, 4082, 2648, 0, 0, 6, 3401, 55, 4090, 4094, 4058, 0, 1528, 0, 498, 0, 265, 450, 782, 1, 759, 3955, 4064, 3814, 0, 0, 19, 8, 3547, 4045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 43, 3870, 0, 0, 0, 0, 4055, 3146, 1441, 889, 1526, 313, 0, 63, 486, 25, 2822, 2039, 3, 0, 33, 8, 56, 0, 2, 42, 4085, 2, 0, 0, 0, 3, 0, 0, 4080, 68, 0, 3555, 48, 20, 0, 4096, 3792, 3284, 0, 4023, 2, 1323, 0, 0, 684, 244, 0, 4031, 101, 4096, 73, 4, 8, 3165, 0, 3880, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[446, 0, 4048, 52, 8, 0, 17, 3966, 3125, 926, 3344, 82, 29, 0, 0, 3651, 0, 4094, 5, 406, 9, 0, 4095, 376, 90, 0, 868, 4095, 0, 0, 0, 2, 0, 0, 38, 1721, 20, 5, 280, 0, 4081, 96, 112, 4088, 3637, 10, 0, 1101, 4068, 1, 3, 0, 0, 158, 2144, 0, 3857, 54, 1, 0, 6, 4, 3947, 2370] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[126, 0, 4093, 7, 1, 0, 80, 9, 146, 344, 2, 0, 19, 1, 0, 4091, 0, 304, 76, 2, 0, 0, 2353, 5, 795, 0, 114, 3911, 0, 18, 0, 0, 0, 3471, 0, 3091, 1030, 1988, 4084, 0, 3595, 854, 0, 3559, 1051, 13, 4021, 0, 4075, 0, 0, 4047, 2854, 0, 2816, 29, 4095, 0, 7, 0, 0, 0, 4096, 263] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 17, 4082, 0, 149, 0, 99, 169, 1, 0, 0, 4, 799, 0, 0, 3163, 3241, 4074, 20, 11, 30, 0, 4040, 981, 20, 993, 0, 4071, 13, 0, 4, 3794, 3, 197, 0, 2907, 1219, 1, 3441, 0, 3727, 1947, 2, 3647, 257, 0, 1115, 0, 74, 225, 140, 3213, 0, 4, 3999, 37, 4075, 1634, 0, 2, 5, 3, 3887, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 3815, 4085, 146, 0, 4, 976, 3927, 141, 3695, 0, 84, 0, 315, 429, 614, 3381, 552, 0, 0, 0, 0, 5, 2, 0, 1391, 3724, 0, 0, 0, 467, 5, 0, 0, 2347, 2458, 0, 0, 30, 2038, 0, 0, 4094, 3999, 0, 4086, 0, 2717, 0, 436, 0, 2356, 0, 4096, 163, 4095, 0, 29, 4, 4, 1, 4090, 735] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3196, 0, 0, 0, 25, 136, 1191, 781, 0, 0, 0, 0, 0, 0, 0, 2373, 94, 29, 65, 0, 0, 4084, 1329, 853, 0, 0, 0, 0, 0, 0, 10, 3340, 54, 0, 1, 0, 0, 1, 1892, 0, 0, 0, 0, 0, 0, 0, 696, 417, 3, 4056, 0, 0, 1, 58, 1294, 997, 0, 0, 0, 0, 0, 0, 0, 3064, 1, 158, 2690, 0, 3856, 9, 681, 0, 0, 0, 0, 0, 0, 0, 1766, 2988, 679, 0, 3560, 13, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 182, 195, 3379, 2, 84, 1591, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 2858, 43, 0, 2, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5329, 8827, 5298, 7522, 10459, 9276, 10563, 8262] avg=8192.0000 max=10563 max_rank=6 min=5298 min_rank=2 imbalance_ratio=1.289429 max_load_reduction=54973 max_load_reduction_pct=0.838821 moved_to_echo=38560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[73, 28, 3993, 0, 248, 7, 0, 310, 1191, 1184, 0, 0, 0, 0, 0, 0, 5, 17, 2896, 41, 1108, 38, 0, 4052, 582, 1072, 0, 0, 0, 0, 0, 0, 283, 2674, 0, 327, 0, 0, 24, 6, 2863, 0, 0, 0, 0, 0, 0, 0, 605, 0, 0, 4073, 5, 0, 0, 1727, 1294, 437, 0, 0, 0, 0, 0, 0, 14, 31, 0, 3433, 788, 1, 2689, 1, 855, 0, 0, 0, 0, 0, 0, 0, 774, 1408, 3, 0, 2245, 0, 961, 163, 0, 0, 0, 0, 0, 0, 0, 0, 2976, 0, 0, 2115, 3935, 0, 2000, 1962, 0, 0, 0, 0, 0, 0, 0, 0, 1610, 275, 13, 19, 83, 0, 3070, 2949, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7034, 9811, 6177, 8141, 7812, 5554, 12988, 8019] avg=8192.0000 max=12988 max_rank=6 min=5554 min_rank=5 imbalance_ratio=1.585449 max_load_reduction=52548 max_load_reduction_pct=0.801819 moved_to_echo=34373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 5, 838, 1, 0, 0, 1182, 1197, 0, 0, 0, 0, 0, 0, 347, 1, 0, 0, 13, 0, 0, 0, 1105, 207, 0, 0, 0, 0, 0, 0, 0, 2531, 0, 37, 127, 59, 0, 1182, 2897, 0, 0, 0, 0, 0, 0, 0, 1171, 1063, 0, 4087, 190, 0, 0, 3073, 1284, 829, 0, 0, 0, 0, 0, 0, 0, 548, 1, 4082, 2648, 0, 0, 6, 165, 0, 0, 0, 0, 0, 0, 0, 1467, 55, 4090, 0, 4058, 0, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 0, 265, 450, 782, 1, 387, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 1598, 3814, 0, 0, 19, 8, 3547, 4045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7316, 1673, 6833, 11697, 7450, 11198, 6338, 13031] avg=8192.0000 max=13031 max_rank=7 min=1673 min_rank=1 imbalance_ratio=1.590698 max_load_reduction=52505 max_load_reduction_pct=0.801163 moved_to_echo=38017 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[446, 0, 4048, 52, 8, 0, 17, 3966, 1121, 1196, 0, 0, 0, 0, 0, 0, 3125, 926, 3344, 82, 29, 0, 0, 3651, 1326, 585, 0, 0, 0, 0, 0, 0, 0, 4094, 5, 406, 9, 0, 4095, 376, 2892, 0, 0, 0, 0, 0, 0, 0, 90, 0, 868, 4095, 0, 0, 0, 2, 1218, 994, 0, 0, 0, 0, 0, 0, 0, 0, 38, 1721, 20, 5, 280, 0, 467, 0, 0, 0, 0, 0, 0, 0, 1761, 96, 112, 0, 3637, 10, 0, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 1, 3, 0, 0, 158, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 54, 1, 0, 6, 4, 3947, 2370, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10854, 13068, 11877, 7267, 2531, 6717, 5322, 7900] avg=8192.0000 max=13068 max_rank=1 min=2531 min_rank=4 imbalance_ratio=1.595215 max_load_reduction=52468 max_load_reduction_pct=0.800598 moved_to_echo=22470 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 3870, 0, 0, 0, 0, 4055, 1191, 1110, 0, 0, 0, 0, 0, 0, 3146, 1441, 889, 1526, 313, 0, 63, 486, 6, 1100, 0, 0, 0, 0, 0, 0, 25, 2822, 2039, 3, 0, 33, 8, 56, 2682, 0, 0, 0, 0, 0, 0, 0, 0, 2, 42, 4085, 2, 0, 0, 0, 1294, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4080, 68, 0, 3555, 48, 878, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 3284, 0, 4023, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1323, 0, 0, 684, 244, 0, 2053, 101, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 73, 4, 8, 3165, 0, 3880, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10269, 8970, 7668, 5429, 8632, 11415, 4405, 8748] avg=8192.0000 max=11415 max_rank=5 min=4405 min_rank=6 imbalance_ratio=1.393433 max_load_reduction=54121 max_load_reduction_pct=0.825821 moved_to_echo=33200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[126, 0, 4093, 7, 1, 0, 80, 9, 1191, 1042, 0, 0, 0, 0, 0, 0, 146, 344, 2, 0, 19, 1, 0, 4091, 1168, 768, 0, 0, 0, 0, 0, 0, 0, 304, 76, 2, 0, 0, 2353, 5, 2517, 0, 0, 0, 0, 0, 0, 0, 795, 0, 114, 3911, 0, 18, 0, 0, 1294, 876, 0, 0, 0, 0, 0, 0, 0, 3471, 0, 3091, 1030, 1988, 4084, 0, 613, 0, 0, 0, 0, 0, 0, 0, 1551, 854, 0, 0, 1051, 13, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4047, 2854, 0, 1435, 29, 0, 0, 0, 0, 0, 0, 0, 0, 1610, 0, 7, 0, 0, 0, 4096, 263, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6549, 6539, 5257, 7008, 14277, 7490, 12440, 5976] avg=8192.0000 max=14277 max_rank=4 min=5257 min_rank=2 imbalance_ratio=1.742798 max_load_reduction=51259 max_load_reduction_pct=0.782150 moved_to_echo=40183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 4082, 0, 149, 0, 99, 169, 1185, 1066, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 799, 0, 0, 3163, 1211, 1091, 0, 0, 0, 0, 0, 0, 3241, 4074, 20, 11, 30, 0, 4040, 981, 2581, 0, 0, 0, 0, 0, 0, 0, 20, 993, 0, 4071, 13, 0, 4, 3794, 1287, 908, 0, 0, 0, 0, 0, 0, 3, 197, 0, 2907, 1219, 1, 3441, 0, 871, 0, 0, 0, 0, 0, 0, 0, 1608, 1947, 2, 0, 257, 0, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 225, 140, 3213, 0, 4, 2037, 37, 0, 0, 0, 0, 0, 0, 0, 0, 1603, 1634, 0, 2, 5, 3, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6767, 6269, 14978, 11090, 8639, 4929, 5730, 7134] avg=8192.0000 max=14978 max_rank=2 min=4929 min_rank=5 imbalance_ratio=1.828369 max_load_reduction=50558 max_load_reduction_pct=0.771454 moved_to_echo=26432 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3815, 4085, 146, 0, 4, 976, 1194, 1198, 0, 0, 0, 0, 0, 0, 3927, 141, 3695, 0, 84, 0, 315, 429, 666, 1122, 0, 0, 0, 0, 0, 0, 614, 3381, 552, 0, 0, 0, 0, 5, 2896, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1391, 3724, 0, 0, 0, 467, 1298, 500, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2347, 2458, 0, 0, 30, 896, 0, 0, 0, 0, 0, 0, 0, 872, 0, 0, 0, 3999, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2717, 0, 436, 0, 2356, 0, 2078, 163, 0, 0, 0, 0, 0, 0, 0, 0, 1603, 0, 29, 4, 4, 1, 4090, 735, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11418, 10379, 7448, 7382, 5736, 8957, 7750, 6466] avg=8192.0000 max=11418 max_rank=0 min=5736 min_rank=4 imbalance_ratio=1.393799 max_load_reduction=54118 max_load_reduction_pct=0.825775 moved_to_echo=28909 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 7424, 128, 21888, 11008, 2048, 17920, 128, 5504, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[10752, 5248, 11008, 1792, 2432, 128, 384, 19968, 7424, 6912, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[19840, 512, 1152, 13952, 10240, 256, 12800, 7424, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[9856, 7424, 9088, 0, 22144, 128, 16128, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4224, 23296, 2816, 896, 256, 128, 10624, 2688, 21248, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3456, 2560, 2432, 32128, 256, 128, 128, 9216, 10368, 5632, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12800, 8832, 128, 128, 3328, 128, 30336, 10496, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 128, 31232, 4224, 1408, 128, 256, 9728, 9472, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 126, 0, 0, 3858, 25, 4031, 219, 4095, 2827, 279, 0, 0, 2297, 54, 34, 45, 69, 1826, 0, 1555, 0, 4066, 0, 0, 4087, 3298, 0, 254, 3922, 0, 0, 11, 42, 0, 1, 0, 3791, 0, 139, 1594, 23, 5, 12, 2552, 0, 4020, 0, 6, 29, 0, 3987, 5, 4054, 80, 0, 1, 0, 84, 0, 4036] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[3725, 24, 0, 312, 3986, 157, 1, 4096, 0, 4041, 301, 4084, 31, 15, 1, 1618, 0, 3, 0, 0, 71, 3679, 0, 876, 0, 1508, 6, 66, 4086, 4096, 277, 0, 3919, 4, 225, 0, 0, 467, 0, 69, 273, 2, 8, 35, 0, 40, 0, 3911, 16, 4074, 112, 0, 0, 72, 44, 3333, 4096, 0, 7, 814, 0, 0, 2859, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 866, 10, 3963, 29, 0, 4063, 24, 4060, 0, 3171, 2524, 6, 71, 2413, 3398, 2, 5, 7, 4061, 172, 7, 1, 0, 2891, 0, 2, 53, 4061, 29, 3, 4033, 26, 27, 1, 302, 1575, 15, 0, 4019, 4081, 0, 1449, 1481, 1707, 0, 1969, 54, 2986, 0, 0, 21, 0, 48, 10, 1734, 4, 2, 5, 108, 0, 3949, 38] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4068, 18, 339, 731, 4096, 62, 0, 4096, 11, 3163, 7, 4071, 2, 67, 10, 0, 1742, 0, 0, 9, 0, 621, 0, 86, 2035, 221, 0, 0, 4070, 3688, 0, 0, 4095, 0, 0, 0, 529, 4080, 1730, 26, 3451, 0, 1, 2096, 0, 0, 0, 3439, 4, 4089, 0, 0, 0, 199, 0, 4013, 3285, 2, 33, 1, 0, 0, 1, 1249] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4093, 0, 3163, 0, 4093, 0, 0, 1112, 0, 4067, 0, 3798, 0, 0, 0, 814, 4092, 0, 1, 0, 2, 1, 0, 0, 244, 4043, 2, 8, 0, 3959, 0, 557, 4096, 134, 0, 0, 0, 1366, 356, 0, 1103, 46, 0, 2861, 346, 1278, 13, 2, 3928, 3703, 0, 203, 134, 152, 20, 0, 4093, 0, 74, 27, 30, 3438, 1, 4083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3462, 2350, 2, 0, 4096, 0, 1, 4077, 0, 4095, 0, 4096, 2367, 52, 820, 111, 11, 17, 0, 1129, 2791, 0, 2, 5, 0, 757, 0, 2, 1, 3889, 0, 1488, 297, 2, 0, 0, 0, 3909, 0, 0, 12, 7, 4063, 3970, 1317, 0, 0, 0, 0, 4096, 0, 11, 0, 100, 3100, 7, 4067, 18, 16, 0, 0, 827, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[2005, 0, 0, 1006, 3687, 349, 3, 2696, 0, 3999, 0, 3228, 0, 89, 0, 0, 3148, 146, 88, 3, 8, 23, 252, 0, 3624, 4096, 0, 0, 80, 3963, 0, 0, 3691, 0, 3, 105, 159, 1503, 2527, 0, 4094, 0, 0, 2197, 12, 53, 2425, 4087, 0, 3682, 178, 0, 0, 0, 193, 0, 4010, 0, 0, 6, 15, 0, 19, 4084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[24, 0, 327, 0, 636, 0, 308, 1510, 0, 3796, 341, 4096, 0, 0, 6, 0, 4083, 0, 0, 1037, 301, 177, 2, 0, 3553, 3799, 0, 0, 4086, 4081, 1212, 128, 3149, 108, 497, 0, 0, 3465, 344, 160, 90, 3965, 1, 3570, 57, 0, 0, 255, 12, 3996, 1, 0, 0, 4006, 0, 0, 4096, 0, 101, 3, 0, 64, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 126, 0, 0, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2561, 219, 4095, 2827, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2297, 54, 34, 45, 69, 1826, 0, 1555, 1788, 1628, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4087, 1510, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 0, 0, 11, 42, 0, 1, 0, 1179, 820, 0, 0, 0, 0, 0, 0, 3791, 0, 139, 1024, 23, 5, 12, 2552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 6, 29, 0, 3987, 5, 570, 311, 291, 0, 0, 0, 0, 0, 3743, 80, 0, 1, 0, 84, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5633, 10006, 9296, 9917, 5975, 7546, 9219, 7944] avg=8192.0000 max=10006 max_rank=1 min=5633 min_rank=0 imbalance_ratio=1.221436 max_load_reduction=55530 max_load_reduction_pct=0.847321 moved_to_echo=30684 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3725, 24, 0, 312, 3986, 157, 1, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2567, 301, 4084, 31, 15, 1, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 71, 3679, 0, 876, 2221, 1728, 0, 0, 0, 0, 0, 0, 0, 1508, 6, 66, 4086, 1875, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 4, 225, 0, 0, 467, 0, 69, 1182, 870, 0, 0, 0, 0, 0, 0, 273, 2, 8, 23, 0, 40, 0, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4074, 112, 0, 0, 72, 44, 3333, 12, 314, 292, 0, 0, 0, 0, 0, 3782, 0, 7, 814, 0, 0, 2859, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9703, 8617, 8578, 7818, 6736, 4257, 8269, 11558] avg=8192.0000 max=11558 max_rank=7 min=4257 min_rank=5 imbalance_ratio=1.410889 max_load_reduction=53978 max_load_reduction_pct=0.823639 moved_to_echo=30820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4068, 18, 339, 731, 4096, 62, 0, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2009, 7, 4071, 2, 67, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 0, 0, 9, 0, 621, 0, 86, 2000, 1728, 0, 0, 0, 0, 0, 0, 2035, 221, 0, 0, 4070, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 529, 4080, 1730, 26, 925, 870, 0, 0, 0, 0, 0, 0, 3451, 0, 1, 1346, 0, 0, 0, 3439, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4089, 0, 0, 0, 199, 0, 4013, 750, 252, 229, 0, 0, 0, 0, 0, 3033, 2, 33, 1, 0, 0, 1, 1249, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10812, 6177, 6186, 8014, 12255, 8237, 9536, 4319] avg=8192.0000 max=12255 max_rank=4 min=4319 min_rank=7 imbalance_ratio=1.495972 max_load_reduction=53281 max_load_reduction_pct=0.813004 moved_to_echo=34347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 3163, 0, 4093, 0, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2584, 0, 3798, 0, 0, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 2, 1, 0, 0, 2147, 469, 0, 0, 0, 0, 0, 0, 244, 4043, 2, 8, 0, 1812, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 134, 0, 0, 0, 1366, 356, 0, 1189, 236, 0, 0, 0, 0, 0, 0, 1103, 46, 0, 1837, 346, 1278, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3928, 3703, 0, 203, 134, 152, 20, 0, 1024, 314, 294, 0, 0, 0, 0, 0, 3779, 0, 74, 27, 30, 3438, 1, 4083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11756, 7196, 6712, 6666, 7377, 4625, 9772, 11432] avg=8192.0000 max=11756 max_rank=0 min=4625 min_rank=5 imbalance_ratio=1.435059 max_load_reduction=53780 max_load_reduction_pct=0.820618 moved_to_echo=33206 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 866, 10, 3963, 29, 0, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 24, 2580, 0, 3171, 2524, 6, 71, 2413, 0, 0, 0, 0, 0, 0, 0, 0, 3398, 2, 5, 7, 4061, 172, 7, 1, 2202, 1714, 0, 0, 0, 0, 0, 0, 0, 2891, 0, 2, 53, 1859, 29, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 26, 27, 1, 302, 1575, 15, 0, 1187, 863, 0, 0, 0, 0, 0, 0, 4019, 4081, 0, 930, 1481, 1707, 0, 1969, 0, 0, 0, 0, 0, 0, 0, 0, 54, 2986, 0, 0, 21, 0, 48, 10, 519, 133, 293, 0, 0, 0, 0, 0, 1601, 4, 2, 5, 108, 0, 3949, 38, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6354, 10789, 11569, 4837, 8029, 14187, 4064, 5707] avg=8192.0000 max=14187 max_rank=5 min=4064 min_rank=6 imbalance_ratio=1.731812 max_load_reduction=51349 max_load_reduction_pct=0.783524 moved_to_echo=31987 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3462, 2350, 2, 0, 4096, 0, 1, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2601, 0, 4096, 2367, 52, 820, 111, 0, 0, 0, 0, 0, 0, 0, 0, 11, 17, 0, 1129, 2791, 0, 2, 5, 2109, 1720, 0, 0, 0, 0, 0, 0, 0, 757, 0, 2, 1, 1780, 0, 1488, 0, 0, 0, 0, 0, 0, 0, 0, 297, 2, 0, 0, 0, 3909, 0, 0, 1198, 866, 0, 0, 0, 0, 0, 0, 12, 7, 4063, 2548, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 100, 3100, 7, 1422, 312, 296, 0, 0, 0, 0, 0, 3755, 18, 16, 0, 0, 827, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11402, 10047, 7784, 4028, 6272, 7947, 9344, 8712] avg=8192.0000 max=11402 max_rank=0 min=4028 min_rank=3 imbalance_ratio=1.391846 max_load_reduction=54134 max_load_reduction_pct=0.826019 moved_to_echo=32275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2005, 0, 0, 1006, 3687, 349, 3, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2541, 0, 3228, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3148, 146, 88, 3, 8, 23, 252, 0, 2149, 1137, 0, 0, 0, 0, 0, 0, 3624, 4096, 0, 0, 80, 1814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3691, 0, 3, 105, 159, 1503, 2527, 0, 1169, 573, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1411, 12, 53, 2425, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3682, 178, 0, 0, 0, 193, 0, 786, 308, 289, 0, 0, 0, 0, 0, 3702, 0, 0, 6, 15, 0, 19, 4084, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8036, 5858, 6954, 9614, 9730, 12082, 5436, 7826] avg=8192.0000 max=12082 max_rank=5 min=5436 min_rank=6 imbalance_ratio=1.474854 max_load_reduction=53454 max_load_reduction_pct=0.815643 moved_to_echo=35074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[24, 0, 327, 0, 636, 0, 308, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2403, 341, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 1037, 301, 177, 2, 0, 2217, 640, 0, 0, 0, 0, 0, 0, 3553, 3799, 0, 0, 4086, 1864, 1212, 128, 0, 0, 0, 0, 0, 0, 0, 0, 3149, 108, 497, 0, 0, 3465, 344, 160, 1114, 325, 0, 0, 0, 0, 0, 0, 90, 3965, 1, 2287, 57, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3996, 1, 0, 0, 4006, 0, 0, 1283, 319, 279, 0, 0, 0, 0, 0, 3777, 0, 101, 3, 0, 64, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[1840, 6846, 8457, 14642, 9162, 6655, 9896, 8038] avg=8192.0000 max=14642 max_rank=3 min=1840 min_rank=0 imbalance_ratio=1.787354 max_load_reduction=50894 max_load_reduction_pct=0.776581 moved_to_echo=33751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[21504, 2432, 4736, 2176, 24704, 640, 384, 9344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 19968, 896, 30720, 7808, 512, 1024, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[16896, 8192, 4224, 11520, 3328, 3200, 2560, 16256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 30720, 384, 256, 256, 4608, 7424, 7424, 6400, 2304, 2304, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[9472, 21504, 128, 128, 16512, 14208, 1536, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[27264, 128, 256, 896, 256, 4480, 6912, 25856, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[18816, 256, 128, 2304, 7424, 6528, 384, 2560, 16896, 10880, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[27264, 384, 768, 128, 1152, 16384, 4992, 256, 9216, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[3243, 0, 0, 0, 0, 2812, 19, 0, 13, 24, 3895, 0, 0, 167, 1, 0, 1040, 0, 4096, 7, 0, 3806, 227, 4084, 0, 3956, 2, 4, 97, 33, 4088, 4059, 3311, 0, 313, 0, 0, 3040, 0, 0, 12, 0, 0, 2753, 0, 0, 4053, 0, 0, 0, 4093, 214, 4035, 2, 329, 0, 2659, 1059, 10, 0, 0, 0, 3980, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1086, 0, 3412, 0, 0, 0, 0, 3, 0, 1781, 4038, 0, 0, 155, 8, 3, 0, 4096, 626, 0, 0, 420, 2242, 4096, 0, 0, 36, 0, 0, 0, 2988, 4069, 1, 0, 325, 0, 0, 2262, 0, 122, 1, 45, 4049, 3938, 46, 0, 3003, 0, 2976, 0, 3442, 1343, 3918, 208, 14, 10, 3567, 163, 3823, 1, 1, 42, 3177, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4073, 0, 4, 10, 0, 5, 3, 0, 8, 0, 3485, 0, 4, 363, 139, 385, 36, 3045, 4096, 0, 2, 3838, 57, 3944, 0, 674, 0, 1, 57, 248, 4095, 4070, 99, 2, 0, 1, 4007, 4, 0, 755, 0, 1627, 4087, 663, 0, 36, 3739, 2, 2705, 606, 3807, 4045, 3543, 450, 75, 0, 2, 657, 44, 0, 1938, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[2623, 0, 115, 51, 0, 2188, 0, 0, 2894, 3, 4069, 0, 0, 3271, 0, 11, 0, 1970, 3957, 2, 609, 3884, 550, 4096, 4016, 256, 0, 0, 0, 20, 4094, 4096, 4, 0, 0, 0, 85, 0, 0, 78, 0, 1043, 3471, 87, 2, 0, 3413, 75, 1072, 0, 4096, 3995, 4077, 48, 1, 0, 4, 454, 480, 0, 0, 0, 276, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4083, 0, 0, 4064, 360, 2458, 238, 4, 4010, 0, 4092, 4, 0, 3304, 78, 31, 17, 4096, 3727, 0, 0, 17, 1, 3871, 336, 4089, 0, 0, 105, 66, 4076, 2483, 11, 0, 0, 1, 3801, 470, 1, 0, 0, 108, 3509, 2, 0, 0, 4000, 782, 425, 0, 1378, 0, 3943, 0, 1, 0, 0, 1376, 36, 0, 25, 5, 0, 52] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[2496, 8, 0, 87, 1, 4036, 0, 180, 1154, 0, 3968, 0, 2, 0, 0, 0, 1300, 4095, 3364, 3072, 5, 1065, 2882, 4096, 102, 1508, 0, 37, 0, 3, 1081, 3929, 175, 0, 1076, 54, 3802, 18, 41, 0, 0, 0, 3934, 3675, 38, 144, 0, 0, 0, 0, 3486, 0, 0, 2231, 151, 7, 0, 7, 0, 0, 4095, 3, 173, 3955] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[2099, 3984, 0, 0, 139, 3945, 4, 3832, 0, 997, 4096, 0, 485, 1, 0, 0, 2695, 2087, 4096, 0, 0, 4096, 3966, 561, 1160, 696, 0, 29, 0, 0, 0, 1334, 1655, 0, 3718, 9, 1, 0, 627, 55, 0, 0, 151, 2409, 0, 379, 3726, 3, 0, 3708, 3985, 3647, 1047, 0, 0, 6, 3, 0, 5, 0, 0, 4, 96, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4019, 4, 0, 0, 390, 2861, 3835, 0, 0, 2895, 554, 0, 9, 4030, 31, 0, 1371, 3218, 3255, 4096, 3352, 274, 2, 0, 0, 0, 4096, 4067, 1175, 5, 0, 0, 177, 592, 0, 32, 1, 0, 42, 4082, 0, 0, 1451, 0, 0, 0, 4091, 4092, 4096, 0, 0, 0, 2413, 0, 1, 0, 924, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1086, 0, 3412, 0, 0, 0, 0, 3, 1096, 116, 0, 0, 0, 0, 0, 0, 0, 1781, 4038, 0, 0, 155, 8, 3, 1077, 107, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 420, 2242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 2988, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 325, 0, 0, 2262, 0, 122, 4096, 124, 0, 0, 0, 0, 0, 0, 1, 45, 4049, 3938, 46, 0, 3003, 0, 167, 0, 0, 0, 0, 0, 0, 0, 2976, 0, 1034, 1343, 3918, 208, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3567, 163, 3823, 1, 1, 42, 3177, 0, 502, 1312, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5713, 7169, 6484, 5900, 6930, 11249, 9503, 12588] avg=8192.0000 max=12588 max_rank=7 min=5713 min_rank=0 imbalance_ratio=1.536621 max_load_reduction=52948 max_load_reduction_pct=0.807922 moved_to_echo=40270 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3243, 0, 0, 0, 0, 2812, 19, 0, 1304, 115, 0, 0, 0, 0, 0, 0, 13, 24, 3895, 0, 0, 167, 1, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 1040, 0, 0, 7, 0, 3806, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 2, 4, 97, 33, 4088, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 3311, 0, 313, 0, 0, 3040, 0, 0, 4084, 818, 0, 0, 0, 0, 0, 0, 12, 0, 0, 2753, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1229, 214, 4035, 2, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2659, 1059, 10, 0, 0, 0, 3980, 0, 3278, 1560, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7493, 5175, 5080, 11049, 11566, 6818, 5809, 12546] avg=8192.0000 max=12546 max_rank=7 min=5080 min_rank=2 imbalance_ratio=1.531494 max_load_reduction=52990 max_load_reduction_pct=0.808563 moved_to_echo=36739 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 4, 10, 0, 5, 3, 0, 1213, 116, 0, 0, 0, 0, 0, 0, 8, 0, 3485, 0, 4, 363, 139, 385, 1078, 80, 0, 0, 0, 0, 0, 0, 36, 2841, 0, 0, 2, 3838, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 674, 0, 1, 57, 248, 4095, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 99, 2, 0, 1, 4007, 4, 0, 755, 3944, 817, 0, 0, 0, 0, 0, 0, 0, 1627, 4087, 663, 0, 36, 3739, 2, 124, 0, 0, 0, 0, 0, 0, 0, 2705, 606, 1143, 4045, 3543, 450, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 657, 44, 0, 1938, 0, 0, 0, 3279, 1451, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5424, 5542, 6774, 7951, 9629, 10278, 12567, 7371] avg=8192.0000 max=12567 max_rank=6 min=5424 min_rank=0 imbalance_ratio=1.534058 max_load_reduction=52969 max_load_reduction_pct=0.808243 moved_to_echo=39845 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 0, 4064, 360, 2458, 238, 4, 439, 70, 0, 0, 0, 0, 0, 0, 4010, 0, 4092, 4, 0, 3304, 78, 31, 657, 107, 0, 0, 0, 0, 0, 0, 17, 3822, 0, 0, 0, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 4089, 0, 0, 105, 66, 4076, 1756, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1, 3801, 470, 1, 0, 3871, 743, 0, 0, 0, 0, 0, 0, 0, 108, 3509, 2, 0, 0, 4000, 782, 167, 0, 0, 0, 0, 0, 0, 0, 425, 0, 414, 0, 3943, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1376, 36, 0, 25, 5, 0, 52, 2984, 525, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11716, 12283, 3857, 10428, 8898, 8568, 4783, 5003] avg=8192.0000 max=12283 max_rank=1 min=3857 min_rank=2 imbalance_ratio=1.499390 max_load_reduction=53253 max_load_reduction_pct=0.812576 moved_to_echo=27252 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2623, 0, 115, 51, 0, 2188, 0, 0, 1305, 117, 0, 0, 0, 0, 0, 0, 2894, 3, 4069, 0, 0, 3271, 0, 11, 1084, 51, 0, 0, 0, 0, 0, 0, 0, 1839, 0, 2, 609, 3884, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 256, 0, 0, 0, 20, 4094, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 85, 0, 0, 78, 4096, 789, 0, 0, 0, 0, 0, 0, 0, 1043, 3471, 87, 2, 0, 3413, 75, 80, 0, 0, 0, 0, 0, 0, 0, 1072, 0, 1230, 3995, 4077, 48, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 454, 480, 0, 0, 0, 276, 0, 3168, 1561, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6399, 11383, 6884, 11281, 5052, 8171, 10423, 5943] avg=8192.0000 max=11383 max_rank=1 min=5052 min_rank=4 imbalance_ratio=1.389526 max_load_reduction=54153 max_load_reduction_pct=0.826309 moved_to_echo=29589 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2496, 8, 0, 87, 1, 4036, 0, 180, 1110, 112, 0, 0, 0, 0, 0, 0, 1154, 0, 3968, 0, 2, 0, 0, 0, 1040, 107, 0, 0, 0, 0, 0, 0, 1300, 3822, 0, 3072, 5, 1065, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 1508, 0, 37, 0, 3, 1081, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 1076, 54, 3802, 18, 41, 0, 4096, 672, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 3675, 38, 144, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 0, 0, 2231, 151, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4095, 3, 173, 3955, 2692, 1329, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8030, 6271, 12146, 5508, 9934, 7957, 3436, 12254] avg=8192.0000 max=12254 max_rank=7 min=3436 min_rank=6 imbalance_ratio=1.495850 max_load_reduction=53282 max_load_reduction_pct=0.813019 moved_to_echo=33581 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2099, 3984, 0, 0, 139, 3945, 4, 3832, 1269, 38, 0, 0, 0, 0, 0, 0, 0, 997, 4096, 0, 485, 1, 0, 0, 353, 54, 0, 0, 0, 0, 0, 0, 2695, 1948, 0, 0, 0, 4096, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 696, 0, 29, 0, 0, 0, 943, 0, 0, 0, 0, 0, 0, 0, 0, 1655, 0, 3718, 9, 1, 0, 627, 55, 561, 817, 0, 0, 0, 0, 0, 0, 0, 0, 151, 2409, 0, 379, 3726, 3, 85, 0, 0, 0, 0, 0, 0, 0, 0, 3708, 1197, 3647, 1047, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 0, 0, 4, 96, 0, 3279, 1519, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15310, 5986, 12705, 2828, 7443, 6753, 9605, 4906] avg=8192.0000 max=15310 max_rank=0 min=2828 min_rank=3 imbalance_ratio=1.868896 max_load_reduction=50226 max_load_reduction_pct=0.766388 moved_to_echo=28707 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4019, 4, 0, 0, 1307, 119, 0, 0, 0, 0, 0, 0, 390, 2861, 3835, 0, 0, 2895, 554, 0, 1081, 111, 0, 0, 0, 0, 0, 0, 9, 3753, 0, 0, 1371, 3218, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3352, 274, 2, 0, 0, 0, 4096, 2867, 0, 0, 0, 0, 0, 0, 0, 0, 1175, 5, 0, 0, 177, 592, 0, 32, 4096, 7, 0, 0, 0, 0, 0, 0, 1, 0, 42, 4082, 0, 0, 1451, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1222, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2413, 0, 1, 0, 924, 0, 0, 1, 24, 1562, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5451, 11727, 11606, 10591, 6084, 5742, 9410, 4925] avg=8192.0000 max=11727 max_rank=1 min=4925 min_rank=7 imbalance_ratio=1.431519 max_load_reduction=53809 max_load_reduction_pct=0.821060 moved_to_echo=26161 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7296, 4352, 8576, 17408, 24704, 2944, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8704, 3840, 4480, 128, 7040, 128, 7808, 4096, 19328, 10880, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[5120, 21888, 0, 3200, 2048, 20352, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9088, 11520, 128, 128, 384, 384, 24576, 19968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 2944, 19328, 17664, 128, 640, 23424, 896, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[19712, 4096, 3584, 4224, 4608, 15488, 384, 4096, 9088, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[6528, 128, 5504, 128, 11904, 6400, 768, 1152, 28928, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8576, 5760, 31488, 128, 512, 10240, 896, 512, 7552, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 2, 4091, 0, 0, 21, 0, 0, 347, 3853, 4096, 0, 3993, 3900, 2600, 0, 4049, 0, 194, 4096, 358, 7, 0, 47, 0, 0, 1854, 0, 4096, 0, 0, 0, 3873, 602, 746, 0, 0, 0, 3576, 0, 0, 0, 0, 0, 3748, 0, 0, 6, 1423, 460, 1, 0, 3479, 45, 1266, 3754, 23, 2, 0, 713, 3358, 781, 6, 70] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[38, 183, 2341, 0, 0, 5, 3, 589, 0, 3605, 4009, 2722, 4080, 4085, 261, 0, 4095, 3918, 51, 4096, 0, 160, 12, 0, 4, 0, 1, 0, 0, 0, 776, 0, 0, 3442, 3373, 0, 81, 0, 0, 1, 0, 0, 33, 214, 262, 0, 9, 331, 3670, 0, 933, 0, 1442, 1975, 2560, 3849, 0, 4096, 23, 115, 0, 4069, 24, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[6, 0, 1698, 0, 0, 0, 0, 4027, 3982, 4096, 1020, 2972, 4079, 10, 1, 30, 4026, 1, 0, 3620, 0, 9, 0, 0, 0, 504, 63, 0, 37, 291, 0, 0, 4085, 0, 2976, 0, 0, 16, 3513, 1, 0, 4094, 4080, 0, 4010, 0, 76, 0, 3674, 0, 0, 1012, 5, 0, 2048, 1536, 468, 0, 28, 641, 130, 0, 0, 2671] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 465, 60, 2, 0, 9, 0, 0, 4054, 3870, 4096, 519, 4095, 1004, 2786, 1, 4018, 0, 0, 4096, 4013, 0, 0, 0, 0, 634, 8, 4, 4096, 0, 3361, 491, 0, 2276, 3, 38, 0, 5, 559, 0, 0, 634, 151, 2, 3763, 0, 0, 41, 3665, 318, 0, 15, 3214, 0, 154, 4094, 23, 0, 18, 13, 806, 0, 0, 4062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[18, 0, 1549, 902, 1, 0, 3962, 2, 104, 3791, 3675, 149, 2810, 18, 0, 0, 2929, 408, 1, 4081, 67, 1622, 0, 17, 0, 3, 0, 2644, 4060, 0, 3972, 0, 147, 4089, 261, 1, 0, 367, 0, 0, 133, 8, 58, 7, 4077, 4095, 6, 0, 3848, 0, 0, 0, 138, 0, 0, 4086, 0, 1785, 0, 3978, 2, 1486, 0, 179] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 216, 1714, 0, 0, 1, 125, 182, 4039, 2474, 162, 4082, 1942, 0, 22, 114, 2623, 0, 4095, 4091, 4076, 16, 1006, 115, 0, 87, 5, 0, 5, 0, 19, 0, 3857, 486, 174, 0, 1608, 974, 3534, 0, 0, 17, 3310, 0, 0, 0, 1, 3428, 0, 0, 1, 22, 0, 4093, 3699, 0, 248, 61, 3974, 2168, 2089, 581, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3542, 0, 0, 0, 2, 5, 4062, 3663, 4094, 4028, 31, 3792, 0, 0, 3764, 3192, 0, 1083, 3, 4, 935, 0, 0, 3700, 0, 0, 3932, 6, 103, 4085, 4095, 387, 2, 166, 38, 2, 7, 0, 799, 4087, 3026, 0, 181, 4, 0, 186, 0, 0, 0, 0, 3935, 5, 0, 4090, 5, 0, 0, 0, 76, 7, 1, 411] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4032, 9, 6, 0, 5, 2606, 4096, 4096, 4095, 0, 1377, 3, 0, 0, 2599, 4045, 0, 4096, 3782, 0, 87, 19, 0, 1111, 0, 0, 29, 0, 0, 52, 49, 4096, 3, 9, 0, 0, 574, 0, 0, 462, 0, 0, 4095, 1, 4, 0, 3637, 0, 0, 0, 4031, 0, 1292, 2813, 0, 2, 0, 4005, 0, 0, 222, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 1698, 0, 0, 0, 0, 4027, 4096, 84, 0, 0, 0, 0, 0, 0, 3982, 0, 0, 2972, 2942, 10, 1, 30, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 1, 0, 642, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 63, 0, 37, 291, 0, 0, 936, 14, 0, 0, 0, 0, 0, 0, 4085, 0, 2976, 0, 0, 16, 3513, 1, 1137, 253, 0, 0, 0, 0, 0, 0, 0, 4094, 4080, 0, 4010, 0, 76, 0, 2415, 0, 0, 0, 0, 0, 0, 0, 3674, 0, 0, 1012, 5, 0, 2048, 537, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 28, 641, 130, 0, 0, 2671, 746, 549, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9911, 9937, 4678, 1845, 11981, 14675, 7276, 5233] avg=8192.0000 max=14675 max_rank=5 min=1845 min_rank=3 imbalance_ratio=1.791382 max_load_reduction=50861 max_load_reduction_pct=0.776077 moved_to_echo=39165 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4091, 0, 0, 21, 0, 0, 3853, 339, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 2880, 3900, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 194, 726, 358, 7, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1854, 0, 4096, 0, 0, 0, 3757, 16, 0, 0, 0, 0, 0, 0, 3873, 602, 746, 0, 0, 0, 3576, 0, 1113, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 0, 0, 6, 2732, 0, 0, 0, 0, 0, 0, 0, 1423, 460, 1, 0, 3479, 45, 1266, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 0, 713, 3358, 781, 6, 70, 1825, 622, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8306, 9727, 5381, 9723, 10528, 6486, 7985, 7400] avg=8192.0000 max=10528 max_rank=4 min=5381 min_rank=2 imbalance_ratio=1.285156 max_load_reduction=55008 max_load_reduction_pct=0.839355 moved_to_echo=32399 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[38, 183, 2341, 0, 0, 5, 3, 589, 3605, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2722, 2943, 4085, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3918, 51, 726, 0, 160, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 776, 0, 3676, 16, 0, 0, 0, 0, 0, 0, 0, 3442, 3373, 0, 81, 0, 0, 1, 1137, 633, 0, 0, 0, 0, 0, 0, 0, 0, 33, 214, 262, 0, 9, 331, 2732, 0, 0, 0, 0, 0, 0, 0, 3670, 0, 933, 0, 1442, 1975, 2560, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 23, 115, 0, 4069, 24, 0, 1871, 622, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7097, 10011, 8962, 4473, 8667, 3581, 11925, 10820] avg=8192.0000 max=11925 max_rank=6 min=3581 min_rank=5 imbalance_ratio=1.455688 max_load_reduction=53611 max_load_reduction_pct=0.818039 moved_to_echo=34993 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 1549, 902, 1, 0, 3962, 2, 3791, 305, 0, 0, 0, 0, 0, 0, 104, 0, 0, 149, 2027, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2929, 408, 1, 724, 67, 1622, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2644, 4060, 0, 3972, 0, 3370, 16, 0, 0, 0, 0, 0, 0, 147, 4089, 261, 1, 0, 367, 0, 0, 783, 673, 0, 0, 0, 0, 0, 0, 133, 8, 58, 7, 4077, 4095, 6, 0, 2722, 0, 0, 0, 0, 0, 0, 0, 3848, 0, 0, 0, 138, 0, 0, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 0, 3978, 2, 1486, 0, 179, 1986, 619, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10530, 2298, 5768, 14065, 6321, 11106, 5413, 10035] avg=8192.0000 max=14065 max_rank=3 min=2298 min_rank=1 imbalance_ratio=1.716919 max_load_reduction=51471 max_load_reduction_pct=0.785385 moved_to_echo=32875 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 465, 60, 2, 0, 9, 0, 0, 3870, 340, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 519, 2954, 1004, 2786, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 726, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 8, 4, 4096, 0, 3361, 491, 3756, 16, 0, 0, 0, 0, 0, 0, 0, 2276, 3, 38, 0, 5, 559, 0, 1141, 674, 0, 0, 0, 0, 0, 0, 0, 634, 151, 2, 3763, 0, 0, 41, 2732, 0, 0, 0, 0, 0, 0, 0, 3665, 318, 0, 15, 3214, 0, 154, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 18, 13, 806, 0, 0, 4062, 1990, 622, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4746, 11318, 8757, 12366, 4696, 7323, 8796, 7534] avg=8192.0000 max=12366 max_rank=3 min=4696 min_rank=4 imbalance_ratio=1.509521 max_load_reduction=53170 max_load_reduction_pct=0.811310 moved_to_echo=28349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3542, 0, 0, 0, 2, 5, 3663, 340, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 4028, 23, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3764, 3192, 0, 193, 3, 4, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 3932, 6, 103, 4085, 3754, 4, 0, 0, 0, 0, 0, 0, 4095, 387, 2, 166, 38, 2, 7, 0, 8, 673, 0, 0, 0, 0, 0, 0, 799, 4087, 3026, 0, 181, 4, 0, 186, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 5, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 76, 7, 1, 411, 1988, 164, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7552, 11905, 8091, 15584, 5378, 9005, 5369, 2652] avg=8192.0000 max=15584 max_rank=3 min=2652 min_rank=7 imbalance_ratio=1.902344 max_load_reduction=49952 max_load_reduction_pct=0.762207 moved_to_echo=22404 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 216, 1714, 0, 0, 1, 125, 4039, 205, 0, 0, 0, 0, 0, 0, 182, 0, 0, 162, 2944, 1942, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 114, 2623, 0, 725, 4091, 4076, 16, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 87, 5, 0, 5, 0, 19, 2269, 16, 0, 0, 0, 0, 0, 0, 0, 3857, 486, 174, 0, 1608, 974, 3534, 1138, 609, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3310, 0, 0, 0, 1, 2732, 0, 0, 0, 0, 0, 0, 0, 3428, 0, 0, 1, 22, 0, 4093, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 61, 3974, 2168, 2089, 581, 0, 1798, 622, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6300, 5252, 12651, 2516, 12380, 6060, 8836, 11541] avg=8192.0000 max=12651 max_rank=2 min=2516 min_rank=3 imbalance_ratio=1.544312 max_load_reduction=52885 max_load_reduction_pct=0.806961 moved_to_echo=38817 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4032, 9, 6, 0, 5, 2606, 4096, 340, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 989, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2599, 4045, 0, 716, 3782, 0, 87, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 0, 0, 29, 0, 0, 52, 3755, 17, 0, 0, 0, 0, 0, 0, 49, 4096, 3, 9, 0, 0, 574, 0, 388, 466, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 4095, 1, 4, 0, 2738, 0, 0, 0, 0, 0, 0, 0, 3637, 0, 0, 0, 4031, 0, 1292, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4005, 0, 0, 222, 4096, 1371, 625, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11094, 5088, 11248, 4964, 5585, 7300, 9936, 10321] avg=8192.0000 max=11248 max_rank=2 min=4964 min_rank=3 imbalance_ratio=1.373047 max_load_reduction=54288 max_load_reduction_pct=0.828369 moved_to_echo=33142 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 6016, 2048, 2688, 16256, 384, 8320, 4736, 25344, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12288, 18816, 7936, 512, 128, 2048, 9216, 3584, 6912, 4608, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 6144, 256, 13440, 6656, 8448, 896, 11520, 13696, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 9344, 7424, 3584, 20224, 4224, 128, 640, 19584, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[16896, 0, 0, 10624, 17792, 14848, 5760, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[23424, 896, 1024, 1152, 16384, 2048, 11520, 9856, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[25600, 14208, 256, 5248, 12416, 5888, 1152, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 768, 17536, 2688, 128, 128, 4096, 7424, 31104, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[3064, 0, 0, 0, 3097, 4095, 800, 4096, 2222, 0, 0, 0, 0, 4073, 5, 0, 0, 0, 0, 3689, 0, 230, 1744, 4013, 0, 3, 8, 0, 0, 2828, 236, 4095, 0, 45, 190, 3101, 51, 57, 0, 4061, 15, 651, 0, 0, 4, 59, 488, 189, 80, 0, 3986, 0, 2, 358, 6, 0, 553, 3110, 3728, 0, 2495, 20, 3989, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2694, 238, 3367, 0, 0, 2614, 1654, 0, 0, 0, 1761, 3811, 6, 3710, 0, 3911, 209, 3178, 4049, 0, 0, 4004, 0, 1, 0, 36, 86, 2, 0, 3824, 0, 1, 3650, 50, 695, 13, 12, 623, 3, 0, 0, 3640, 4050, 0, 264, 1, 0, 0, 359, 2, 1630, 0, 0, 0, 47, 4012, 4087, 5, 55, 130, 3052, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3783, 0, 10, 0, 6, 0, 409, 0, 4091, 0, 2063, 0, 340, 4007, 510, 1, 0, 204, 5, 4095, 4089, 0, 1229, 4091, 1, 0, 0, 113, 38, 4035, 0, 4082, 35, 0, 2686, 143, 131, 135, 4, 0, 5, 12, 1120, 5, 0, 0, 1, 3952, 2, 28, 2771, 0, 4066, 20, 0, 1, 0, 179, 4095, 626, 4082, 0, 4018, 217] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[14, 1, 486, 4016, 4096, 4095, 4, 3303, 4076, 0, 151, 0, 1, 4092, 0, 136, 1814, 0, 0, 3943, 4090, 0, 0, 7, 0, 1, 0, 0, 156, 1, 0, 0, 0, 1300, 3591, 270, 85, 1205, 0, 815, 0, 0, 4070, 5, 0, 3, 393, 26, 0, 4012, 92, 0, 3998, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 2995, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3889, 0, 0, 3983, 3888, 4095, 0, 4095, 3981, 63, 1, 4096, 4024, 2878, 417, 1028, 6, 0, 0, 4088, 648, 0, 0, 2847, 3, 3, 0, 0, 0, 0, 14, 4095, 0, 0, 1, 0, 0, 0, 0, 4096, 294, 17, 436, 1, 2029, 0, 0, 3797, 0, 365, 0, 0, 926, 112, 13, 0, 34, 65, 4096, 0, 1, 0, 1111, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[805, 116, 0, 0, 110, 74, 0, 3838, 3967, 427, 6, 2, 0, 4094, 4091, 1111, 0, 0, 6, 3923, 1564, 228, 534, 4096, 0, 0, 0, 1045, 0, 0, 2867, 4088, 0, 0, 816, 3495, 0, 4024, 0, 0, 0, 444, 0, 76, 0, 70, 4096, 0, 50, 229, 0, 221, 3814, 1854, 0, 0, 0, 181, 3739, 0, 1665, 2, 3765, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 5, 74, 2532, 8, 580, 2753, 0, 0, 0, 121, 0, 3028, 0, 3926, 465, 0, 0, 4096, 0, 86, 2278, 3952, 0, 629, 1, 4096, 0, 347, 0, 3, 115, 2531, 188, 2895, 2075, 4011, 0, 3900, 0, 0, 1, 4083, 27, 0, 0, 1, 51, 0, 0, 0, 3727, 4093, 0, 0, 82, 0, 3229, 0, 4090, 0, 1457, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3349, 647, 1786, 2071, 674, 0, 0, 4096, 0, 4096, 80, 0, 71, 1, 218, 3864, 4019, 0, 12, 3664, 0, 403, 0, 0, 0, 19, 119, 4085, 3, 2495, 3167, 1, 732, 3347, 0, 10, 0, 0, 4052, 385, 0, 0, 0, 2264, 0, 3311, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 207, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2694, 238, 3367, 0, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 1761, 1609, 6, 3710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 209, 737, 4049, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 36, 86, 2, 0, 3824, 1913, 951, 0, 0, 0, 0, 0, 0, 0, 1, 3650, 50, 695, 13, 12, 623, 614, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3640, 4050, 0, 264, 1, 2441, 89, 0, 0, 0, 0, 0, 0, 0, 0, 359, 2, 1630, 0, 0, 0, 1492, 1251, 0, 0, 0, 0, 0, 0, 47, 4012, 1981, 5, 55, 130, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6911, 8740, 12910, 6813, 5658, 10488, 4734, 9282] avg=8192.0000 max=12910 max_rank=2 min=4734 min_rank=6 imbalance_ratio=1.575928 max_load_reduction=52626 max_load_reduction_pct=0.803009 moved_to_echo=30162 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3064, 0, 0, 0, 3097, 4095, 800, 958, 0, 0, 0, 0, 0, 0, 0, 0, 2222, 0, 0, 0, 0, 1720, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 855, 0, 230, 1744, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 0, 2828, 236, 4095, 2998, 1016, 0, 0, 0, 0, 0, 0, 0, 45, 190, 3101, 51, 57, 0, 4061, 560, 0, 0, 0, 0, 0, 0, 0, 15, 651, 0, 0, 4, 59, 488, 189, 2834, 140, 0, 0, 0, 0, 0, 0, 80, 0, 3986, 0, 2, 358, 6, 0, 1361, 1337, 0, 0, 0, 0, 0, 0, 553, 3110, 1807, 0, 2495, 20, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12014, 3947, 6842, 11184, 8065, 4380, 7130, 11974] avg=8192.0000 max=12014 max_rank=0 min=3947 min_rank=1 imbalance_ratio=1.466553 max_load_reduction=53522 max_load_reduction_pct=0.816681 moved_to_echo=31549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3889, 0, 0, 3983, 3888, 4095, 0, 958, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 63, 1, 4096, 4024, 1216, 417, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 947, 648, 0, 0, 2847, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 14, 4095, 2997, 718, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 615, 0, 0, 0, 0, 0, 0, 0, 294, 17, 436, 1, 2029, 0, 0, 3797, 3141, 140, 0, 0, 0, 0, 0, 0, 0, 365, 0, 0, 926, 112, 13, 0, 1496, 944, 0, 0, 0, 0, 0, 0, 34, 65, 1985, 0, 1, 0, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16813, 14826, 4448, 7830, 4712, 9855, 3856, 3196] avg=8192.0000 max=16813 max_rank=0 min=3196 min_rank=7 imbalance_ratio=2.052368 max_load_reduction=48723 max_load_reduction_pct=0.743454 moved_to_echo=21619 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3783, 0, 10, 0, 6, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2063, 0, 340, 1693, 510, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 5, 949, 4089, 0, 1229, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 113, 38, 4035, 0, 4082, 0, 999, 0, 0, 0, 0, 0, 0, 35, 0, 2686, 143, 131, 135, 4, 0, 615, 0, 0, 0, 0, 0, 0, 0, 5, 12, 1120, 5, 0, 0, 1, 3952, 3146, 0, 0, 0, 0, 0, 0, 0, 2, 28, 2771, 0, 4066, 20, 0, 1, 1495, 1315, 0, 0, 0, 0, 0, 0, 0, 179, 1985, 626, 4082, 0, 4018, 217, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4208, 8698, 10567, 9268, 3749, 8241, 9698, 11107] avg=8192.0000 max=11107 max_rank=7 min=3749 min_rank=4 imbalance_ratio=1.355835 max_load_reduction=54429 max_load_reduction_pct=0.830521 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[14, 1, 486, 4016, 4096, 4095, 4, 773, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 151, 0, 1, 1728, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 0, 0, 914, 4090, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 156, 1, 0, 0, 2417, 1021, 0, 0, 0, 0, 0, 0, 0, 1300, 3591, 270, 85, 1205, 0, 815, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 5, 0, 3, 393, 26, 3029, 113, 0, 0, 0, 0, 0, 0, 0, 4012, 92, 0, 3998, 0, 0, 1, 1496, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 1985, 0, 4096, 0, 2995, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13485, 6092, 6825, 3596, 7881, 7639, 10942, 9076] avg=8192.0000 max=13485 max_rank=0 min=3596 min_rank=3 imbalance_ratio=1.646118 max_load_reduction=52051 max_load_reduction_pct=0.794235 moved_to_echo=35538 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[805, 116, 0, 0, 110, 74, 0, 898, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 427, 6, 2, 0, 1729, 4091, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 909, 1564, 228, 534, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 0, 0, 2867, 4088, 2809, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 816, 3495, 0, 4024, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 444, 0, 76, 0, 70, 4096, 0, 3014, 131, 0, 0, 0, 0, 0, 0, 50, 229, 0, 221, 3814, 1854, 0, 0, 1365, 1344, 0, 0, 0, 0, 0, 0, 0, 181, 1812, 0, 1665, 2, 3765, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2003, 11333, 7337, 11830, 8897, 7831, 8877, 7428] avg=8192.0000 max=11830 max_rank=3 min=2003 min_rank=0 imbalance_ratio=1.444092 max_load_reduction=53706 max_load_reduction_pct=0.819489 moved_to_echo=33033 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 74, 2532, 8, 580, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 1279, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 949, 0, 86, 2278, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 1, 4096, 0, 347, 0, 3, 2015, 755, 0, 0, 0, 0, 0, 0, 115, 2531, 188, 2895, 2075, 4011, 0, 3900, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4083, 27, 0, 0, 1, 3147, 94, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 3727, 4093, 0, 0, 1179, 994, 0, 0, 0, 0, 0, 0, 82, 0, 1565, 0, 4090, 0, 1457, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3843, 5326, 7730, 7846, 16200, 7353, 10044, 7194] avg=8192.0000 max=16200 max_rank=4 min=3843 min_rank=0 imbalance_ratio=1.977539 max_load_reduction=49336 max_load_reduction_pct=0.752808 moved_to_echo=40791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3349, 647, 1786, 477, 0, 0, 0, 0, 0, 0, 0, 0, 674, 0, 0, 4096, 0, 1724, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 1, 218, 892, 4019, 0, 12, 3664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 0, 0, 0, 19, 119, 4085, 1518, 1025, 0, 0, 0, 0, 0, 0, 3, 2495, 3167, 1, 732, 3347, 0, 10, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 385, 0, 0, 0, 2264, 2972, 76, 0, 0, 0, 0, 0, 0, 0, 3311, 0, 0, 4096, 0, 0, 0, 1501, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 1976, 0, 4096, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6259, 6574, 8877, 7169, 10374, 9749, 10255, 6279] avg=8192.0000 max=10374 max_rank=4 min=6259 min_rank=0 imbalance_ratio=1.266357 max_load_reduction=55162 max_load_reduction_pct=0.841705 moved_to_echo=36657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[11648, 128, 3200, 8320, 20480, 13056, 3584, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[768, 7552, 15104, 640, 20608, 256, 20608, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2432, 4224, 512, 7168, 18560, 640, 5888, 26752, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 8064, 7296, 256, 22272, 6528, 128, 128, 11392, 9984, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 1152, 128, 5376, 384, 7296, 3328, 24320, 16768, 7552, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[20736, 512, 2304, 8320, 6144, 12800, 5120, 9984, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 6400, 14336, 9984, 3840, 12800, 128, 13568, 4736, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 1152, 9728, 8320, 6144, 256, 5248, 10240, 23808, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 4074, 0, 0, 109, 4091, 160, 0, 0, 3473, 1, 0, 0, 0, 248, 11, 2, 1, 1215, 4091, 0, 3283, 3864, 0, 22, 0, 0, 4087, 3870, 4095, 10, 0, 221, 2358, 4069, 0, 0, 4083, 44, 66, 3865, 0, 0, 1, 0, 4096, 3914, 0, 0, 0, 518, 651, 613, 0, 50, 15, 153, 0, 25, 4087, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 2626, 4095, 3, 0, 598, 0, 0, 0, 2634, 1341, 0, 3984, 0, 0, 34, 7, 58, 0, 4066, 0, 0, 4075, 9, 3287, 182, 182, 42, 96, 115, 0, 3, 1231, 0, 15, 480, 0, 4096, 0, 0, 3774, 1639, 0, 2363, 0, 4038, 4034, 0, 3923, 5, 0, 4095, 151, 0, 4, 0, 35, 20, 16, 4082, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3313, 22, 13, 0, 0, 0, 0, 0, 3949, 3779, 0, 3867, 1305, 4095, 0, 0, 0, 23, 2, 1, 3863, 582, 0, 3017, 4, 256, 0, 0, 0, 0, 0, 180, 0, 17, 44, 93, 4096, 3787, 0, 4061, 4096, 48, 2481, 0, 1694, 3103, 0, 4072, 0, 428, 3971, 0, 18, 3859, 0, 0, 0, 0, 1301, 1, 73, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[36, 0, 41, 65, 0, 0, 4013, 0, 16, 39, 3678, 496, 0, 4080, 12, 3104, 0, 0, 11, 2, 74, 24, 3710, 3962, 55, 3235, 4032, 18, 0, 3880, 105, 0, 0, 3976, 4056, 0, 149, 0, 3878, 13, 132, 3942, 291, 19, 0, 6, 1344, 4088, 0, 38, 3, 626, 4079, 0, 67, 22, 0, 0, 0, 0, 0, 23, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[3171, 0, 2112, 0, 1, 445, 110, 12, 610, 0, 52, 0, 0, 0, 4082, 4070, 145, 5, 227, 0, 4016, 0, 0, 4096, 253, 0, 3912, 0, 0, 4092, 2778, 0, 8, 4096, 0, 2136, 210, 0, 2154, 8, 1351, 3731, 0, 37, 23, 173, 1312, 154, 0, 3467, 0, 0, 4055, 6, 357, 1, 127, 0, 5, 2, 3858, 0, 2, 4074] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3015, 0, 3853, 0, 0, 19, 0, 3, 7, 0, 3, 3943, 3, 2392, 0, 2366, 0, 0, 0, 3846, 252, 0, 1170, 3052, 0, 4096, 4096, 0, 2300, 2537, 0, 37, 0, 0, 4096, 410, 0, 1182, 19, 1505, 4094, 4072, 26, 264, 0, 130, 0, 2115, 0, 4, 0, 1223, 5, 0, 157, 3533, 0, 51, 0, 0, 928, 0, 3610, 1122] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 42, 4008, 0, 0, 0, 4022, 11, 0, 9, 3883, 288, 0, 259, 65, 0, 0, 0, 4, 500, 294, 34, 4096, 0, 0, 4096, 4096, 15, 55, 2850, 0, 549, 0, 9, 4050, 6, 0, 1752, 4096, 300, 49, 2531, 4095, 169, 0, 95, 3640, 0, 70, 3544, 0, 3797, 3648, 0, 0, 0, 613, 5, 0, 0, 0, 0, 3891, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4084, 0, 2675, 0, 132, 3227, 4096, 1, 0, 0, 5, 3338, 0, 4085, 0, 15, 25, 10, 0, 0, 2303, 1, 4033, 4096, 0, 3759, 0, 0, 831, 0, 34, 8, 0, 0, 3689, 3405, 0, 2, 4081, 843, 4, 2928, 15, 0, 0, 0, 4091, 525, 57, 0, 0, 0, 294, 0, 4017, 4055, 0, 0, 0, 0, 0, 2, 770, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4074, 0, 0, 109, 4091, 160, 641, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3473, 1, 0, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 1, 1215, 4091, 0, 3283, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4087, 3870, 4095, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 2358, 4069, 0, 0, 2634, 44, 0, 0, 0, 0, 0, 0, 0, 0, 66, 766, 0, 0, 1, 0, 4096, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 651, 613, 0, 50, 0, 2, 0, 0, 0, 0, 0, 0, 15, 153, 0, 25, 4087, 0, 0, 0, 3099, 808, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9078, 3722, 12467, 12079, 9326, 8843, 1834, 8187] avg=8192.0000 max=12467 max_rank=2 min=1834 min_rank=6 imbalance_ratio=1.521851 max_load_reduction=53069 max_load_reduction_pct=0.809769 moved_to_echo=28190 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2626, 4095, 3, 0, 598, 0, 643, 469, 0, 0, 0, 0, 0, 0, 0, 0, 2634, 1341, 0, 3123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 7, 58, 0, 4066, 0, 0, 4075, 78, 0, 0, 0, 0, 0, 0, 0, 9, 2454, 182, 182, 42, 96, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1231, 0, 15, 480, 0, 2643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 1639, 0, 2363, 0, 4038, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 5, 0, 4095, 151, 0, 4, 783, 364, 0, 0, 0, 0, 0, 0, 0, 35, 20, 16, 4082, 1, 0, 4096, 3027, 810, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8435, 7098, 8318, 3080, 4372, 12821, 9325, 12087] avg=8192.0000 max=12821 max_rank=5 min=3080 min_rank=3 imbalance_ratio=1.565063 max_load_reduction=52715 max_load_reduction_pct=0.804367 moved_to_echo=38605 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[36, 0, 41, 65, 0, 0, 4013, 0, 609, 462, 0, 0, 0, 0, 0, 0, 16, 39, 3678, 496, 0, 3198, 12, 3104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 74, 24, 3710, 3962, 80, 0, 0, 0, 0, 0, 0, 0, 55, 2415, 4032, 18, 0, 3880, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 4056, 0, 149, 0, 2502, 13, 0, 0, 0, 0, 0, 0, 0, 0, 132, 781, 291, 19, 0, 6, 1344, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3, 626, 4079, 0, 67, 22, 802, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4096, 0, 3161, 767, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5226, 10543, 7863, 10505, 10696, 6661, 5995, 8047] avg=8192.0000 max=10696 max_rank=4 min=5226 min_rank=0 imbalance_ratio=1.305664 max_load_reduction=54840 max_load_reduction_pct=0.836792 moved_to_echo=31399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3015, 0, 3853, 0, 0, 19, 0, 3, 2, 585, 0, 0, 0, 0, 0, 0, 7, 0, 3, 3943, 3, 1875, 0, 2366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 252, 0, 1170, 3052, 47, 0, 0, 0, 0, 0, 0, 0, 0, 3058, 4096, 0, 2300, 2537, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 410, 0, 1182, 14, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 806, 26, 264, 0, 130, 0, 2115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1223, 5, 0, 157, 3533, 470, 453, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 928, 0, 3610, 1122, 3266, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7477, 8197, 8367, 12028, 7207, 7435, 5845, 8980] avg=8192.0000 max=12028 max_rank=3 min=5845 min_rank=6 imbalance_ratio=1.468262 max_load_reduction=53508 max_load_reduction_pct=0.816467 moved_to_echo=29467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3313, 22, 13, 0, 0, 0, 643, 431, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 3779, 0, 3031, 1305, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 1, 3863, 582, 76, 0, 0, 0, 0, 0, 0, 0, 0, 2252, 4, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 17, 44, 93, 2643, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 4096, 48, 2481, 0, 1694, 3103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 428, 3971, 0, 18, 3859, 760, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 1, 73, 22, 3257, 810, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4422, 16159, 4547, 2512, 6764, 12226, 13442, 5464] avg=8192.0000 max=16159 max_rank=1 min=2512 min_rank=3 imbalance_ratio=1.972534 max_load_reduction=49377 max_load_reduction_pct=0.753433 moved_to_echo=37896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3171, 0, 2112, 0, 1, 445, 110, 12, 338, 0, 0, 0, 0, 0, 0, 0, 610, 0, 52, 0, 0, 0, 4082, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 145, 5, 227, 0, 4016, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 3912, 0, 0, 4092, 2778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 2136, 210, 0, 1390, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 739, 0, 37, 23, 173, 1312, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3467, 0, 0, 4055, 6, 357, 1, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 5, 2, 3858, 0, 2, 4074, 2992, 426, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6189, 8814, 8489, 11035, 7848, 3789, 7886, 11486] avg=8192.0000 max=11486 max_rank=7 min=3789 min_rank=5 imbalance_ratio=1.402100 max_load_reduction=54050 max_load_reduction_pct=0.824738 moved_to_echo=31009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 42, 4008, 0, 0, 0, 4022, 11, 643, 585, 0, 0, 0, 0, 0, 0, 0, 9, 3883, 288, 0, 204, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 500, 294, 34, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3058, 4096, 15, 55, 2850, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4050, 6, 0, 1752, 2643, 300, 0, 0, 0, 0, 0, 0, 0, 0, 49, 501, 4095, 169, 0, 95, 3640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 3544, 0, 3797, 3648, 0, 0, 0, 50, 453, 0, 0, 0, 0, 0, 0, 613, 5, 0, 0, 0, 0, 3891, 0, 2030, 810, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9311, 4449, 4933, 10623, 8760, 8549, 11562, 7349] avg=8192.0000 max=11562 max_rank=6 min=4449 min_rank=1 imbalance_ratio=1.411377 max_load_reduction=53974 max_load_reduction_pct=0.823578 moved_to_echo=36220 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 2675, 0, 132, 3227, 4096, 1, 644, 539, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3338, 0, 3196, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 25, 10, 0, 0, 2303, 1, 4033, 4096, 84, 0, 0, 0, 0, 0, 0, 0, 0, 2801, 0, 0, 831, 0, 34, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 3405, 0, 2, 2624, 843, 0, 0, 0, 0, 0, 0, 0, 0, 4, 577, 15, 0, 0, 0, 4091, 525, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 294, 0, 4017, 4055, 805, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 770, 0, 2351, 813, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15398, 6554, 10552, 3674, 10563, 5212, 9647, 3936] avg=8192.0000 max=15398 max_rank=0 min=3674 min_rank=3 imbalance_ratio=1.879639 max_load_reduction=50138 max_load_reduction_pct=0.765045 moved_to_echo=29358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 16128, 16384, 512, 7424, 17408, 7168, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 15104, 128, 6656, 20864, 896, 4736, 11648, 3712, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[10368, 128, 22784, 4224, 256, 3840, 17024, 256, 4224, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 128, 384, 5632, 15104, 128, 20224, 23808, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 256, 128, 128, 14336, 128, 12544, 9344, 23296, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5760, 5760, 10240, 640, 4992, 512, 20224, 18048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 128, 17792, 13312, 128, 14720, 5504, 13952, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 9728, 18304, 10112, 896, 3072, 17152, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 1978, 0, 122, 4, 0, 4078, 3869, 0, 4090, 0, 0, 1, 3912, 3895, 0, 7, 0, 551, 2, 0, 0, 0, 123, 815, 2922, 0, 79, 1267, 775, 0, 30, 3076, 4070, 3486, 4, 4091, 2, 0, 3932, 0, 0, 19, 887, 2247, 154, 4053, 1252, 0, 0, 8, 0, 3361, 0, 0, 4094, 0, 365, 1915, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 74, 0, 62, 0, 7, 12, 13, 3986, 0, 374, 4096, 0, 0, 0, 0, 117, 0, 0, 3144, 632, 567, 3825, 91, 0, 3984, 0, 0, 0, 0, 3115, 102, 0, 894, 182, 3925, 3604, 66, 4096, 5, 0, 151, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4, 2, 396, 3917, 0, 17, 4095, 3920, 4042, 3794, 0, 0, 42] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 84, 3, 1542, 2, 0, 0, 0, 0, 762, 0, 3943, 0, 0, 0, 1823, 0, 4025, 0, 3479, 0, 0, 274, 0, 29, 5, 30, 4058, 0, 0, 152, 4096, 0, 0, 29, 3837, 3856, 250, 4091, 0, 84, 4089, 0, 0, 0, 551, 260, 0, 3222, 0, 2, 0, 1801, 0, 4096, 0, 30, 2937, 117, 3823, 16, 4003, 39, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[50, 10, 0, 0, 0, 0, 12, 0, 3139, 0, 72, 0, 2, 1135, 1823, 0, 171, 0, 38, 9, 1684, 0, 282, 404, 0, 1372, 150, 4092, 4060, 0, 4095, 18, 0, 0, 0, 3664, 19, 0, 3880, 0, 0, 5, 0, 0, 0, 3971, 4090, 1884, 4077, 18, 8, 195, 0, 0, 4095, 0, 66, 3438, 2355, 3909, 2885, 263, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 17, 0, 0, 32, 0, 0, 0, 0, 269, 0, 1984, 4096, 0, 0, 11, 0, 4059, 73, 0, 0, 17, 5, 592, 0, 1181, 0, 3593, 4018, 0, 3643, 762, 1, 4096, 4018, 94, 3471, 0, 6, 14, 39, 0, 41, 1493, 3398, 4057, 3454, 0, 778, 513, 127, 0, 4095, 0, 3080, 213, 0, 4089, 7, 68, 37, 3995] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 164, 0, 958, 3649, 0, 391, 0, 15, 0, 30, 4095, 0, 2, 0, 2960, 0, 3949, 0, 3054, 9, 0, 0, 129, 677, 2417, 60, 3194, 0, 0, 0, 1075, 15, 0, 3384, 3804, 4088, 0, 3941, 57, 1, 0, 0, 0, 0, 4015, 1, 21, 4033, 836, 0, 0, 0, 43, 4096, 0, 0, 4093, 0, 1201, 29, 830, 156, 4064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 651, 3282, 0, 1, 11, 220, 8, 0, 1998, 3988, 1, 0, 120, 2331, 4082, 0, 485, 3724, 2, 2805, 202, 0, 3, 699, 0, 67, 2, 40, 3833, 0, 0, 9, 9, 3483, 4082, 3965, 4092, 0, 1, 4087, 0, 2, 275, 4096, 123, 0, 1612, 114, 0, 0, 0, 0, 3979, 0, 0, 4036, 136, 2657, 92, 108, 0, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[7, 0, 3817, 124, 0, 0, 254, 16, 312, 0, 0, 157, 0, 122, 0, 3872, 0, 4070, 0, 1418, 0, 131, 12, 2, 3441, 3716, 0, 2, 7, 0, 2913, 0, 50, 3, 1, 4094, 4096, 0, 3717, 1321, 29, 1404, 0, 2, 58, 0, 232, 6, 4063, 0, 933, 0, 0, 0, 4022, 0, 2, 4075, 4096, 363, 2315, 2165, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1978, 0, 122, 4, 0, 4091, 2517, 0, 0, 0, 0, 0, 0, 4078, 3869, 0, 4090, 0, 0, 1, 3912, 87, 77, 0, 0, 0, 0, 0, 0, 3895, 0, 7, 0, 551, 2, 0, 0, 140, 685, 108, 0, 0, 0, 0, 0, 0, 123, 815, 2922, 0, 79, 1180, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3076, 2686, 3486, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 19, 887, 2247, 154, 1577, 699, 0, 0, 0, 0, 0, 0, 4053, 1252, 0, 0, 8, 0, 3176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8712, 16114, 5388, 5894, 9284, 9515, 8489, 2140] avg=8192.0000 max=16114 max_rank=1 min=2140 min_rank=7 imbalance_ratio=1.967041 max_load_reduction=49422 max_load_reduction_pct=0.754120 moved_to_echo=29428 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 84, 3, 1542, 2, 0, 0, 0, 4091, 1805, 0, 0, 0, 0, 0, 0, 0, 762, 0, 3943, 0, 0, 0, 1823, 10, 94, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 3479, 0, 0, 274, 0, 1470, 646, 132, 0, 0, 0, 0, 0, 29, 5, 30, 4058, 0, 0, 142, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2532, 3856, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 4089, 0, 0, 0, 551, 260, 0, 1132, 659, 0, 0, 0, 0, 0, 0, 3222, 0, 2, 0, 1801, 0, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 117, 2353, 16, 4003, 39, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7527, 6632, 10026, 8360, 6667, 6775, 8895, 10654] avg=8192.0000 max=10654 max_rank=7 min=6632 min_rank=1 imbalance_ratio=1.300537 max_load_reduction=54882 max_load_reduction_pct=0.837433 moved_to_echo=32991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 74, 0, 62, 0, 7, 12, 13, 4096, 2518, 0, 0, 0, 0, 0, 0, 3986, 0, 374, 4096, 0, 0, 0, 0, 214, 90, 0, 0, 0, 0, 0, 0, 117, 0, 0, 3144, 632, 567, 3825, 91, 1554, 660, 126, 0, 0, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 2901, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 894, 182, 2591, 3604, 66, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 4096, 1577, 674, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4, 2, 396, 3701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 3920, 2488, 3794, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6782, 8760, 10716, 6987, 7342, 6498, 8190, 10261] avg=8192.0000 max=10716 max_rank=2 min=6498 min_rank=5 imbalance_ratio=1.308105 max_load_reduction=54820 max_load_reduction_pct=0.836487 moved_to_echo=32291 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[50, 10, 0, 0, 0, 0, 12, 0, 3880, 2114, 0, 0, 0, 0, 0, 0, 3139, 0, 72, 0, 2, 1135, 1823, 0, 282, 94, 0, 0, 0, 0, 0, 0, 171, 0, 38, 9, 1684, 0, 282, 404, 1503, 617, 132, 0, 0, 0, 0, 0, 0, 1372, 150, 4092, 4060, 0, 3813, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2418, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3971, 4090, 1884, 1324, 629, 0, 0, 0, 0, 0, 0, 4077, 18, 8, 195, 0, 0, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 2355, 2406, 2885, 263, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6066, 6547, 4840, 13505, 2437, 11903, 8167, 12071] avg=8192.0000 max=13505 max_rank=3 min=2437 min_rank=4 imbalance_ratio=1.648560 max_load_reduction=52031 max_load_reduction_pct=0.793930 moved_to_echo=34578 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 164, 0, 958, 3649, 0, 391, 0, 3941, 2516, 0, 0, 0, 0, 0, 0, 15, 0, 30, 4095, 0, 2, 0, 2960, 0, 94, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 3054, 9, 0, 0, 129, 461, 640, 132, 0, 0, 0, 0, 0, 677, 2417, 60, 3194, 0, 0, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3384, 2511, 4088, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4015, 1, 21, 1577, 653, 0, 0, 0, 0, 0, 0, 4033, 836, 0, 0, 0, 43, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 740, 29, 830, 156, 4064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11619, 7196, 8374, 7423, 10055, 6268, 8782, 5819] avg=8192.0000 max=11619 max_rank=0 min=5819 min_rank=7 imbalance_ratio=1.418335 max_load_reduction=53917 max_load_reduction_pct=0.822708 moved_to_echo=30924 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 17, 0, 0, 32, 0, 3471, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 0, 1984, 4096, 0, 277, 94, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4059, 73, 0, 0, 17, 1572, 689, 132, 0, 0, 0, 0, 0, 5, 592, 0, 1181, 0, 3593, 3741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3643, 762, 1, 2704, 4018, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 14, 39, 0, 41, 1493, 3398, 4057, 82, 703, 0, 0, 0, 0, 0, 0, 3454, 0, 778, 513, 127, 0, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 0, 0, 2517, 7, 68, 37, 3995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3651, 6720, 6553, 9112, 11222, 9833, 8741, 9704] avg=8192.0000 max=11222 max_rank=4 min=3651 min_rank=0 imbalance_ratio=1.369873 max_load_reduction=54314 max_load_reduction_pct=0.828766 moved_to_echo=39500 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 651, 3282, 0, 1, 11, 220, 4092, 2481, 0, 0, 0, 0, 0, 0, 8, 0, 1998, 3988, 1, 0, 120, 2331, 264, 92, 0, 0, 0, 0, 0, 0, 4082, 0, 485, 3724, 2, 2805, 202, 0, 1022, 586, 128, 0, 0, 0, 0, 0, 3, 699, 0, 67, 2, 40, 3569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 2299, 4082, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 2, 275, 4096, 123, 0, 1555, 598, 0, 0, 0, 0, 0, 0, 1612, 114, 0, 0, 0, 0, 3759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 1635, 92, 108, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10738, 8802, 13036, 4380, 10364, 10737, 5485, 1994] avg=8192.0000 max=13036 max_rank=2 min=1994 min_rank=7 imbalance_ratio=1.591309 max_load_reduction=52500 max_load_reduction_pct=0.801086 moved_to_echo=28580 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 3817, 124, 0, 0, 254, 16, 3717, 2506, 0, 0, 0, 0, 0, 0, 312, 0, 0, 157, 0, 122, 0, 3872, 204, 98, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 1418, 0, 131, 12, 2, 144, 693, 133, 0, 0, 0, 0, 0, 3441, 3716, 0, 2, 7, 0, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 3, 1, 2694, 4096, 0, 0, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 29, 1404, 0, 2, 58, 0, 232, 6, 1569, 707, 0, 0, 0, 0, 0, 0, 4063, 0, 933, 0, 0, 0, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 219, 2315, 2165, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10441, 4765, 6603, 9875, 8165, 4007, 8787, 12893] avg=8192.0000 max=12893 max_rank=7 min=4007 min_rank=5 imbalance_ratio=1.573853 max_load_reduction=52643 max_load_reduction_pct=0.803268 moved_to_echo=33852 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3200, 0, 10624, 12672, 11136, 7552, 256, 20480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8320, 12160, 640, 18944, 3072, 3584, 4608, 768, 7936, 5248, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 384, 4480, 8064, 3712, 256, 768, 256, 31488, 16640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3712, 1792, 6784, 20480, 27264, 4480, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[11648, 4736, 2560, 20736, 128, 3328, 6144, 14976, 1408, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4224, 12928, 1152, 15616, 4096, 3712, 18176, 6144, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 13696, 128, 128, 512, 15104, 10368, 10240, 10496, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[28672, 2304, 1792, 768, 2048, 512, 29952, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1892, 699, 0, 0, 1318, 1417, 0, 4081, 0, 51, 160, 10, 3, 4, 0, 0, 19, 4095, 6, 4074, 0, 4096, 0, 2548, 1, 0, 1960, 3875, 0, 1681, 0, 0, 0, 32, 0, 0, 0, 3927, 0, 146, 4018, 3190, 0, 4070, 0, 2439, 0, 0, 0, 0, 0, 4091, 1, 79, 0, 0, 3172, 191, 4095, 0, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 165, 401, 3882, 0, 0, 4096, 4087, 0, 392, 0, 2352, 64, 0, 0, 0, 0, 3596, 0, 3544, 1, 4096, 0, 29, 58, 1809, 0, 0, 114, 806, 3097, 734, 89, 2253, 4094, 94, 1, 38, 4095, 0, 0, 0, 1479, 4096, 0, 3999, 0, 0, 3428, 0, 0, 0, 3916, 4096, 0, 85, 0, 7, 60, 0, 224, 29, 129, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[95, 1, 28, 0, 3, 4096, 0, 85, 0, 342, 131, 0, 0, 0, 0, 0, 42, 3463, 4096, 2908, 18, 0, 0, 181, 0, 201, 0, 0, 0, 0, 4092, 0, 302, 1, 59, 0, 1, 564, 3546, 2373, 0, 0, 3884, 3927, 3927, 4065, 0, 4096, 0, 0, 0, 0, 16, 4042, 3358, 4050, 0, 0, 5, 0, 3851, 2384, 1303, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[84, 22, 19, 3310, 0, 4085, 3868, 4091, 0, 4026, 1085, 4074, 1327, 3, 0, 0, 3644, 4096, 445, 571, 0, 2546, 1, 43, 0, 3995, 0, 0, 0, 1513, 3327, 0, 0, 111, 0, 149, 0, 1, 3867, 3601, 0, 0, 14, 4064, 0, 0, 0, 294, 3, 0, 0, 10, 0, 2558, 0, 0, 0, 0, 329, 260, 4096, 0, 1, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4, 26, 3995, 4044, 31, 1, 338, 4081, 7, 0, 0, 1, 4096, 7, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3921, 30, 0, 2728, 4061, 0, 0, 3818, 4096, 117, 781, 12, 4082, 3310, 0, 207, 310, 0, 962, 0, 0, 100, 67, 34, 0, 0, 8, 4093, 4095, 2, 0, 0, 0, 1, 4092, 0, 1171, 2708] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[5, 0, 16, 4066, 606, 3910, 0, 4096, 1, 4086, 1631, 359, 0, 0, 0, 3971, 10, 6, 0, 84, 0, 8, 1, 4081, 0, 3575, 0, 0, 4031, 2351, 3570, 0, 34, 0, 952, 0, 0, 77, 3362, 218, 0, 0, 0, 24, 0, 919, 0, 3544, 5, 0, 6, 0, 97, 4091, 963, 2, 259, 0, 0, 0, 3345, 219, 3730, 3225] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 275, 0, 1888, 3695, 3507, 1233, 3333, 3551, 0, 0, 6, 3, 0, 0, 486, 0, 1, 2356, 0, 0, 1, 0, 0, 18, 4084, 17, 4091, 1276, 0, 0, 509, 4080, 692, 2062, 0, 0, 4095, 0, 0, 3656, 3, 2990, 3944, 1, 1742, 0, 49, 4, 0, 4096, 1659, 4096, 0, 6, 2031, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 181, 4096, 2749, 0, 4042, 747, 3513, 0, 4096, 2, 0, 4093, 40, 0, 0, 0, 169, 5, 309, 33, 4073, 3942, 4041, 1089, 0, 14, 1297, 6, 0, 128, 1, 0, 407, 3283, 4096, 0, 0, 1814, 4093, 7, 0, 3870, 0, 3774, 1, 0, 19, 0, 0, 0, 0, 0, 3, 35, 0, 0, 0, 2, 314, 840, 117, 4096, 99] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1892, 699, 0, 0, 1318, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 51, 160, 10, 3, 4, 963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4095, 6, 3260, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2548, 1, 0, 1960, 3875, 0, 1681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 2174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 2096, 3190, 0, 4070, 0, 2439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 79, 1879, 814, 0, 0, 0, 0, 0, 0, 0, 0, 3172, 191, 4095, 0, 4095, 0, 1753, 296, 43, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 5272, 11476, 10065, 2206, 11941, 6864, 13645] avg=8192.0000 max=13645 max_rank=7 min=2206 min_rank=4 imbalance_ratio=1.665649 max_load_reduction=51891 max_load_reduction_pct=0.791794 moved_to_echo=34656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 165, 401, 3882, 0, 0, 4096, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 2352, 64, 0, 0, 0, 2778, 0, 0, 0, 0, 0, 0, 0, 0, 3596, 0, 3544, 1, 3278, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 58, 1809, 0, 0, 114, 806, 3008, 734, 0, 0, 0, 0, 0, 0, 0, 0, 89, 2253, 4094, 94, 1, 38, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 773, 4096, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3428, 0, 0, 0, 3916, 4096, 0, 85, 691, 818, 89, 0, 0, 0, 0, 0, 0, 7, 60, 0, 224, 29, 129, 0, 0, 855, 15, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8999, 5586, 10448, 6529, 10664, 8868, 13123, 1319] avg=8192.0000 max=13123 max_rank=6 min=1319 min_rank=7 imbalance_ratio=1.601929 max_load_reduction=52413 max_load_reduction_pct=0.799759 moved_to_echo=33974 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 26, 3995, 4044, 31, 1, 338, 453, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 4096, 7, 0, 3, 2774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 30, 0, 2728, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 4096, 117, 781, 12, 4082, 1833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 163, 0, 962, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 67, 34, 0, 0, 8, 4093, 4095, 2, 144, 818, 116, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 1171, 2708, 1477, 854, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8892, 6888, 3278, 10624, 14739, 1432, 9377, 10306] avg=8192.0000 max=14739 max_rank=4 min=1432 min_rank=5 imbalance_ratio=1.799194 max_load_reduction=50797 max_load_reduction_pct=0.775101 moved_to_echo=35854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[95, 1, 28, 0, 3, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 131, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 42, 3463, 4096, 2908, 18, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 1, 59, 0, 1, 564, 3546, 1314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2027, 3927, 3927, 4065, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4042, 3358, 4050, 1816, 0, 117, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3851, 2384, 1303, 0, 1059, 17, 41, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4234, 530, 10708, 4176, 5787, 18042, 13399, 8660] avg=8192.0000 max=18042 max_rank=5 min=530 min_rank=1 imbalance_ratio=2.202393 max_load_reduction=47494 max_load_reduction_pct=0.724701 moved_to_echo=45888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[84, 22, 19, 3310, 0, 4085, 3868, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 1085, 4074, 1327, 3, 0, 0, 2781, 0, 0, 0, 0, 0, 0, 0, 3644, 4096, 445, 571, 0, 2038, 1, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 0, 1513, 3232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 149, 0, 1, 3867, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4064, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 10, 0, 2558, 0, 0, 6, 508, 95, 0, 0, 0, 0, 0, 0, 0, 329, 260, 4096, 0, 1, 3, 1607, 856, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11842, 13296, 10838, 8740, 6122, 4366, 3180, 7152] avg=8192.0000 max=13296 max_rank=1 min=3180 min_rank=6 imbalance_ratio=1.623047 max_load_reduction=52240 max_load_reduction_pct=0.797119 moved_to_echo=20820 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 275, 0, 1888, 3695, 389, 0, 0, 0, 0, 0, 0, 0, 0, 1233, 3333, 3551, 0, 0, 6, 3, 0, 2384, 0, 0, 0, 0, 0, 0, 0, 0, 486, 0, 1, 2356, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4084, 17, 4091, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 4080, 692, 2062, 0, 0, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1908, 3, 2990, 3944, 1, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4, 0, 4096, 1659, 4096, 0, 1709, 0, 36, 0, 0, 0, 0, 0, 6, 2031, 0, 0, 0, 0, 0, 0, 1828, 734, 39, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6247, 10510, 2844, 9450, 9610, 10588, 11649, 4638] avg=8192.0000 max=11649 max_rank=6 min=2844 min_rank=2 imbalance_ratio=1.421997 max_load_reduction=53887 max_load_reduction_pct=0.822250 moved_to_echo=36485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 16, 4066, 606, 3910, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 1631, 359, 0, 0, 0, 3971, 2784, 0, 0, 0, 0, 0, 0, 0, 10, 6, 0, 84, 0, 7, 1, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3575, 0, 0, 4031, 2351, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 952, 0, 0, 77, 3362, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 919, 0, 3544, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 97, 4091, 963, 2, 0, 1, 102, 0, 0, 0, 0, 0, 259, 0, 0, 0, 3345, 219, 3730, 3225, 97, 857, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9058, 12832, 4189, 13425, 4546, 4487, 5267, 11732] avg=8192.0000 max=13425 max_rank=3 min=4189 min_rank=2 imbalance_ratio=1.638794 max_load_reduction=52111 max_load_reduction_pct=0.795151 moved_to_echo=26032 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 181, 4096, 2749, 0, 4042, 747, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4093, 40, 0, 0, 2391, 0, 0, 0, 0, 0, 0, 0, 0, 169, 5, 309, 33, 3256, 3942, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 0, 14, 1297, 6, 0, 120, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 3283, 4096, 0, 0, 1814, 2262, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2011, 0, 3774, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 35, 0, 1814, 817, 8, 0, 0, 0, 0, 0, 0, 0, 2, 314, 840, 117, 4096, 99, 1831, 740, 45, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12197, 10622, 11755, 2527, 11862, 5812, 2677, 8084] avg=8192.0000 max=12197 max_rank=0 min=2527 min_rank=3 imbalance_ratio=1.488892 max_load_reduction=53339 max_load_reduction_pct=0.813889 moved_to_echo=28435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3584, 128, 128, 128, 8192, 24704, 12672, 4224, 8064, 3840, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 384, 9088, 15360, 11776, 17024, 128, 12288, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 2048, 3584, 768, 20608, 2816, 14592, 6144, 9728, 5248, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1280, 20480, 6400, 6912, 9856, 128, 128, 4096, 17024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3712, 11904, 4608, 11520, 2432, 15232, 3968, 12544, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1152, 12160, 3968, 5504, 6144, 15488, 19072, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 512, 10496, 19072, 640, 18048, 14080, 2816, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 7168, 16640, 5248, 2944, 768, 20864, 12032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[541, 0, 3940, 0, 0, 9, 69, 0, 1, 1277, 4041, 335, 0, 4095, 0, 0, 0, 0, 8, 1, 4095, 2859, 0, 140, 0, 4095, 1325, 40, 0, 1, 0, 0, 1, 0, 88, 57, 4094, 3655, 3092, 4072, 100, 4065, 3056, 67, 4096, 3901, 12, 32, 4096, 0, 0, 0, 3647, 392, 3, 0, 1, 0, 0, 0, 137, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[511, 585, 3904, 0, 0, 0, 0, 62, 1, 0, 3644, 3466, 12, 4095, 0, 0, 0, 0, 4096, 0, 1000, 4086, 0, 729, 0, 4075, 0, 3564, 3989, 750, 0, 0, 58, 0, 22, 0, 4089, 4093, 0, 4091, 1, 4046, 0, 0, 754, 2, 0, 0, 4055, 612, 81, 8, 3932, 0, 0, 103, 0, 0, 131, 0, 0, 0, 771, 118] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[20, 3899, 4094, 0, 1219, 0, 0, 0, 59, 589, 4063, 51, 4062, 3553, 0, 0, 0, 0, 0, 47, 4036, 0, 6, 3727, 0, 0, 3409, 0, 0, 1, 4089, 80, 210, 186, 0, 689, 0, 0, 658, 20, 277, 3981, 0, 3, 33, 0, 1, 0, 2419, 5, 0, 4088, 1, 4088, 3586, 0, 0, 0, 761, 0, 3690, 0, 3836, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[232, 0, 271, 0, 0, 3987, 4089, 0, 0, 0, 0, 3994, 0, 4093, 0, 3805, 691, 0, 3954, 0, 0, 5, 10, 0, 0, 4094, 4061, 3972, 2575, 39, 3359, 1, 0, 0, 204, 0, 27, 827, 1540, 4083, 589, 2, 350, 759, 0, 3947, 12, 1876, 30, 186, 0, 19, 17, 0, 0, 0, 4, 0, 4066, 120, 227, 4, 3415, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 1106, 198, 6, 0, 112, 0, 0, 0, 339, 3647, 4075, 7, 3855, 1577, 0, 9, 0, 4096, 0, 1721, 4094, 0, 0, 1, 4095, 113, 0, 3102, 0, 4096, 0, 46, 1, 1, 352, 3317, 2053, 4085, 16, 0, 3861, 1, 253, 428, 11, 1, 3800, 0, 9, 479, 0, 0, 0, 4071, 0, 2, 0, 2466, 0, 4034, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1373, 638, 3, 0, 1, 4091, 16, 2, 0, 0, 3450, 0, 0, 0, 0, 0, 3, 126, 0, 0, 4083, 1288, 0, 42, 1103, 236, 4096, 0, 0, 0, 0, 0, 0, 0, 570, 7, 4091, 4077, 4096, 4094, 3, 2387, 3507, 927, 0, 453, 0, 4091, 1, 3843, 692, 2468, 363, 0, 4078, 2, 0, 0, 10, 1103, 0, 25, 4092, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[12, 1488, 2528, 3022, 3139, 0, 84, 0, 0, 0, 2197, 878, 0, 1714, 0, 0, 40, 0, 1351, 30, 0, 354, 3, 7, 0, 1, 4084, 47, 3655, 4096, 0, 2128, 21, 0, 73, 0, 3231, 22, 2024, 1, 18, 3790, 4096, 0, 3917, 4080, 39, 328, 2981, 0, 2095, 4030, 0, 0, 3866, 56, 0, 0, 0, 0, 0, 2, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 56, 0, 277, 214, 176, 0, 3560, 0, 0, 4033, 3481, 0, 0, 0, 4078, 87, 0, 4096, 1, 4093, 3914, 0, 552, 0, 0, 3, 0, 0, 0, 0, 0, 1084, 155, 0, 2779, 2680, 0, 462, 0, 0, 4096, 0, 813, 13, 3032, 0, 1055, 0, 972, 4078, 13, 1828, 0, 3816, 0, 0, 1, 3614, 2306, 0, 0, 0, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[541, 0, 3940, 0, 0, 9, 69, 0, 647, 675, 0, 0, 0, 0, 0, 0, 1, 1277, 1941, 335, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 4095, 2859, 0, 140, 0, 37, 0, 0, 0, 0, 0, 0, 0, 4095, 641, 40, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 88, 57, 2258, 3655, 3092, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 100, 2295, 3056, 67, 4096, 3901, 12, 32, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3647, 392, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 137, 0, 0, 0, 2100, 1770, 1161, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5881, 7649, 7140, 4777, 13223, 13559, 8138, 5169] avg=8192.0000 max=13559 max_rank=5 min=4777 min_rank=3 imbalance_ratio=1.655151 max_load_reduction=51977 max_load_reduction_pct=0.793106 moved_to_echo=40089 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[20, 3899, 4094, 0, 1219, 0, 0, 0, 1666, 0, 0, 0, 0, 0, 0, 0, 59, 589, 1952, 51, 4062, 3553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4036, 0, 6, 3727, 1029, 96, 0, 0, 0, 0, 0, 0, 0, 0, 1647, 0, 0, 1, 4089, 80, 0, 0, 0, 0, 0, 0, 0, 0, 210, 186, 0, 689, 0, 0, 658, 20, 0, 0, 0, 0, 0, 0, 0, 0, 277, 2247, 0, 3, 33, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2419, 5, 0, 4088, 1, 4088, 2557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 0, 3690, 0, 3836, 0, 2111, 1734, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10898, 10266, 8941, 5817, 1763, 2561, 13158, 12132] avg=8192.0000 max=13158 max_rank=6 min=1763 min_rank=4 imbalance_ratio=1.606201 max_load_reduction=52378 max_load_reduction_pct=0.799225 moved_to_echo=29614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[511, 585, 3904, 0, 0, 0, 0, 62, 0, 674, 0, 0, 0, 0, 0, 0, 1, 0, 1751, 3466, 12, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1000, 4086, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 3564, 3989, 750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 22, 0, 2255, 4093, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2284, 0, 0, 754, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 612, 81, 8, 3932, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 771, 118, 1893, 1762, 1160, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5736, 9325, 9911, 12378, 10519, 3041, 8791, 5835] avg=8192.0000 max=12378 max_rank=3 min=3041 min_rank=5 imbalance_ratio=1.510986 max_load_reduction=53158 max_load_reduction_pct=0.811127 moved_to_echo=28186 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[232, 0, 271, 0, 0, 3987, 4089, 0, 1985, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 4093, 0, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, 3954, 0, 0, 5, 10, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 4094, 1962, 3972, 2575, 39, 3359, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 16, 827, 1540, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 589, 2, 350, 759, 0, 3947, 12, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 30, 186, 0, 19, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4066, 120, 227, 4, 3415, 0, 0, 0, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10568, 11892, 4774, 16002, 6670, 7535, 252, 7843] avg=8192.0000 max=16002 max_rank=3 min=252 min_rank=6 imbalance_ratio=1.953369 max_load_reduction=49534 max_load_reduction_pct=0.755829 moved_to_echo=22300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1106, 198, 6, 0, 112, 0, 0, 55, 546, 0, 0, 0, 0, 0, 0, 0, 339, 1752, 4075, 7, 3855, 1577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 1721, 4094, 0, 0, 1168, 3, 0, 0, 0, 0, 0, 0, 1, 4095, 55, 0, 3102, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 1, 352, 1830, 2053, 4085, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2179, 1, 253, 428, 11, 1, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 479, 0, 0, 0, 2903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2466, 0, 4034, 0, 0, 0, 1895, 1682, 941, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2023, 11605, 11091, 11349, 8384, 6673, 3391, 11020] avg=8192.0000 max=11605 max_rank=1 min=2023 min_rank=0 imbalance_ratio=1.416626 max_load_reduction=53931 max_load_reduction_pct=0.822922 moved_to_echo=29468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1373, 638, 3, 0, 1, 4091, 16, 2, 2004, 674, 0, 0, 0, 0, 0, 0, 0, 0, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 126, 0, 0, 4083, 1288, 0, 42, 1170, 118, 0, 0, 0, 0, 0, 0, 1103, 236, 1974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 7, 2257, 4077, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1348, 3507, 927, 0, 453, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3843, 692, 2468, 363, 0, 2908, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1103, 0, 25, 4092, 5, 1792, 1039, 1160, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8802, 1658, 6830, 3313, 15101, 10329, 10277, 9226] avg=8192.0000 max=15101 max_rank=4 min=1658 min_rank=1 imbalance_ratio=1.843384 max_load_reduction=50435 max_load_reduction_pct=0.769577 moved_to_echo=44933 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 56, 0, 277, 214, 176, 0, 3560, 3, 445, 0, 0, 0, 0, 0, 0, 0, 0, 1934, 3481, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 4096, 1, 4093, 3914, 0, 552, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 155, 0, 2779, 1472, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2308, 0, 813, 13, 3032, 0, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 972, 4078, 13, 1828, 0, 2719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3614, 2306, 0, 0, 0, 22, 2099, 1788, 763, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8827, 9493, 13840, 0, 5952, 7221, 9610, 10593] avg=8192.0000 max=13840 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.689453 max_load_reduction=51696 max_load_reduction_pct=0.788818 moved_to_echo=33376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[12, 1488, 2528, 3022, 3139, 0, 84, 0, 1996, 532, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 878, 0, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 1351, 30, 0, 354, 3, 7, 1109, 115, 0, 0, 0, 0, 0, 0, 0, 1, 1973, 47, 3655, 4096, 0, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 73, 0, 1783, 22, 2024, 1, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2139, 4096, 0, 3917, 4080, 39, 328, 0, 0, 0, 0, 0, 0, 0, 0, 2981, 0, 2095, 4030, 0, 0, 2757, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 1141, 1651, 916, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12801, 3648, 3009, 11900, 3924, 14617, 11919, 3718] avg=8192.0000 max=14617 max_rank=5 min=3009 min_rank=2 imbalance_ratio=1.784302 max_load_reduction=50919 max_load_reduction_pct=0.776962 moved_to_echo=34178 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 11136, 3584, 8192, 128, 12160, 256, 13056, 11520, 6144, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1536, 384, 1024, 3968, 11904, 14848, 16000, 16384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[13696, 5632, 7552, 10752, 9856, 4480, 13952, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1152, 16640, 8320, 7680, 13440, 4992, 11648, 2304, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 14848, 11136, 2944, 9344, 15488, 128, 11264, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 2304, 12160, 16384, 4096, 21504, 1664, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6912, 7808, 14976, 3328, 4608, 8448, 4352, 3712, 8448, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[896, 128, 17664, 128, 19072, 16640, 128, 5248, 5632, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 3996, 4091, 0, 0, 67, 611, 3697, 4, 0, 4085, 342, 0, 26, 4058, 3569, 0, 68, 1199, 0, 0, 13, 121, 5, 0, 3008, 2037, 31, 367, 0, 3490, 0, 1132, 0, 2174, 3940, 1291, 0, 0, 0, 3748, 51, 0, 0, 3990, 17, 32, 0, 4074, 0, 970, 0, 600, 0, 113, 445, 262, 3871, 3939, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 2435, 3124, 0, 1, 0, 527, 4096, 8, 489, 3953, 1033, 3479, 0, 4025, 4050, 6, 0, 721, 0, 0, 1607, 4090, 0, 0, 4096, 2368, 0, 0, 1, 4095, 0, 1409, 0, 0, 0, 0, 53, 0, 0, 0, 1255, 0, 540, 70, 80, 0, 4096, 0, 0, 4094, 0, 2, 0, 28, 1, 12, 3716, 1887, 0, 4089, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4031, 0, 4094, 2784, 4096, 0, 0, 14, 3954, 0, 3, 0, 0, 7, 0, 4096, 3981, 0, 69, 0, 2272, 0, 543, 0, 0, 0, 0, 1899, 0, 106, 0, 3928, 0, 79, 0, 0, 3602, 0, 0, 0, 2752, 246, 4096, 0, 0, 2686, 471, 23, 3859, 967, 0, 214, 913, 132, 0, 0, 0, 227, 1687, 3470, 0, 3543, 660, 32] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1, 23, 0, 3855, 71, 1835, 0, 34, 3775, 0, 0, 4096, 0, 2024, 260, 3943, 1545, 2, 0, 0, 0, 6, 309, 6, 0, 0, 3165, 3971, 0, 0, 0, 4063, 220, 0, 0, 3573, 0, 418, 4, 0, 3235, 0, 0, 1, 1318, 0, 729, 0, 4096, 0, 4076, 3035, 10, 1858, 2, 98, 0, 0, 4096, 4096, 21, 84, 1474, 108] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[8, 0, 100, 3587, 0, 502, 0, 0, 2, 0, 0, 0, 0, 3994, 104, 4096, 4095, 4028, 0, 0, 1189, 0, 0, 0, 0, 12, 3509, 4095, 0, 0, 0, 4088, 0, 2033, 0, 467, 26, 4096, 0, 0, 879, 3, 1, 10, 0, 0, 0, 4093, 2862, 0, 14, 3893, 0, 0, 1, 4096, 1170, 24, 2, 4093, 0, 4067, 297, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[3930, 3, 4048, 3783, 0, 108, 0, 51, 1677, 0, 0, 0, 0, 0, 71, 2, 4096, 4072, 0, 0, 212, 0, 2331, 8, 0, 0, 3313, 1928, 0, 395, 0, 4096, 8, 4065, 0, 59, 4086, 1, 0, 2200, 1, 0, 90, 8, 192, 0, 375, 0, 3983, 0, 8, 4093, 0, 242, 0, 4094, 2, 4, 0, 3847, 13, 4040, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[3968, 0, 4092, 0, 4, 2035, 777, 4063, 0, 0, 0, 0, 0, 204, 0, 3559, 3962, 3990, 0, 0, 1273, 1428, 545, 109, 8, 26, 3987, 4096, 0, 2, 0, 752, 6, 0, 0, 0, 2145, 0, 2644, 64, 0, 0, 60, 257, 49, 4096, 7, 0, 9, 0, 0, 1408, 3389, 0, 178, 4092, 0, 5, 4093, 105, 18, 4031, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[2644, 1, 0, 3438, 204, 4074, 0, 1596, 4096, 0, 0, 0, 0, 4096, 0, 3670, 4087, 4096, 0, 77, 0, 24, 0, 36, 0, 6, 482, 4083, 307, 4096, 0, 0, 0, 2876, 0, 6, 4070, 0, 0, 205, 0, 3863, 0, 0, 0, 8, 0, 22, 4096, 0, 0, 12, 13, 3883, 48, 0, 7, 0, 1368, 0, 0, 3849, 0, 97] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2435, 1456, 0, 1, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 489, 3953, 1033, 3479, 0, 2341, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 6, 0, 721, 0, 0, 1607, 4090, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2368, 0, 0, 1, 2363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 0, 0, 0, 0, 53, 0, 1478, 1445, 278, 0, 0, 0, 0, 0, 0, 0, 1255, 0, 540, 70, 80, 0, 1668, 1684, 254, 0, 0, 0, 0, 0, 2651, 0, 0, 4094, 0, 2, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 3716, 1887, 0, 3518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4419, 15399, 10767, 8828, 4663, 5551, 6775, 9134] avg=8192.0000 max=15399 max_rank=1 min=4419 min_rank=0 imbalance_ratio=1.879761 max_load_reduction=50137 max_load_reduction_pct=0.765030 moved_to_echo=26123 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3996, 1906, 0, 0, 67, 611, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 4, 0, 4085, 342, 0, 26, 2360, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 0, 68, 1199, 0, 0, 13, 121, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3008, 2037, 31, 367, 0, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 0, 2174, 3940, 1291, 0, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 51, 0, 0, 3990, 17, 32, 2185, 1698, 216, 0, 0, 0, 0, 0, 0, 4074, 0, 970, 0, 600, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 445, 262, 3871, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6582, 10514, 4970, 7462, 9797, 11937, 5757, 8517] avg=8192.0000 max=11937 max_rank=5 min=4970 min_rank=2 imbalance_ratio=1.457153 max_load_reduction=53599 max_load_reduction_pct=0.817856 moved_to_echo=36008 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 23, 0, 1797, 71, 1835, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 4096, 0, 2024, 260, 2293, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 2, 0, 0, 0, 6, 309, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3165, 3971, 0, 0, 0, 2344, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 3573, 0, 418, 4, 0, 1467, 1445, 5, 0, 0, 0, 0, 0, 3235, 0, 0, 1, 1318, 0, 729, 0, 2058, 1650, 252, 0, 0, 0, 0, 0, 2651, 0, 4076, 3035, 10, 1858, 2, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 21, 73, 1474, 108, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3761, 12448, 1874, 9480, 7132, 9243, 11730, 9868] avg=8192.0000 max=12448 max_rank=1 min=1874 min_rank=2 imbalance_ratio=1.519531 max_load_reduction=53088 max_load_reduction_pct=0.810059 moved_to_echo=37973 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2644, 1, 0, 1602, 204, 4074, 0, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2135, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 77, 0, 24, 0, 36, 276, 0, 0, 0, 0, 0, 0, 0, 0, 6, 482, 4083, 307, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2876, 0, 6, 4070, 0, 0, 205, 0, 1445, 262, 0, 0, 0, 0, 0, 0, 3863, 0, 0, 0, 8, 0, 22, 1836, 1535, 0, 0, 0, 0, 0, 0, 2651, 0, 0, 12, 13, 3883, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1368, 0, 0, 3311, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10121, 10327, 8596, 8974, 8864, 7264, 6607, 4783] avg=8192.0000 max=10327 max_rank=1 min=4783 min_rank=7 imbalance_ratio=1.260620 max_load_reduction=55209 max_load_reduction_pct=0.842422 moved_to_echo=27518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3930, 3, 4048, 1763, 0, 108, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 1677, 0, 0, 0, 0, 0, 71, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 212, 0, 2331, 8, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 1928, 0, 395, 0, 2363, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4065, 0, 59, 4086, 1, 0, 2200, 1479, 1405, 275, 0, 0, 0, 0, 0, 1, 0, 90, 8, 192, 0, 375, 0, 2020, 0, 254, 0, 0, 0, 0, 0, 2578, 0, 8, 4093, 0, 242, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 3847, 13, 3476, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9903, 1750, 11008, 7999, 13578, 2940, 11015, 7343] avg=8192.0000 max=13578 max_rank=4 min=1750 min_rank=1 imbalance_ratio=1.657471 max_load_reduction=51958 max_load_reduction_pct=0.792816 moved_to_echo=34876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 100, 1668, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3994, 104, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4028, 0, 0, 1189, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3509, 4095, 0, 0, 0, 2351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2033, 0, 467, 26, 4096, 0, 0, 1480, 1011, 281, 0, 0, 0, 0, 0, 879, 3, 1, 10, 0, 0, 0, 4093, 1919, 1718, 257, 0, 0, 0, 0, 0, 1851, 0, 14, 3893, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 24, 2, 4093, 0, 3492, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2278, 6478, 9606, 9967, 9394, 8880, 9855, 9078] avg=8192.0000 max=9967 max_rank=3 min=2278 min_rank=0 imbalance_ratio=1.216675 max_load_reduction=55569 max_load_reduction_pct=0.847916 moved_to_echo=37207 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4031, 0, 4094, 1298, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 3, 0, 0, 7, 0, 2382, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 69, 0, 2272, 0, 543, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1899, 0, 106, 0, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 3602, 0, 0, 0, 1418, 1361, 241, 0, 0, 0, 0, 0, 2752, 246, 4096, 0, 0, 2686, 471, 23, 1486, 1714, 243, 0, 0, 0, 0, 0, 2498, 967, 0, 214, 913, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 1687, 3470, 0, 3048, 660, 32, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13533, 6346, 7119, 4272, 6701, 13717, 4724, 9124] avg=8192.0000 max=13717 max_rank=5 min=4272 min_rank=3 imbalance_ratio=1.674438 max_load_reduction=51819 max_load_reduction_pct=0.790695 moved_to_echo=34266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3968, 0, 4092, 0, 4, 2035, 777, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 3990, 0, 0, 1273, 1428, 545, 109, 289, 0, 0, 0, 0, 0, 0, 0, 8, 26, 3987, 4096, 0, 2, 0, 435, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2145, 0, 2644, 64, 271, 3, 274, 0, 0, 0, 0, 0, 0, 0, 60, 257, 49, 4096, 7, 0, 0, 1489, 46, 0, 0, 0, 0, 0, 6, 0, 0, 1408, 3389, 0, 178, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4093, 105, 18, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14939, 2274, 11596, 8554, 5407, 6004, 9073, 7689] avg=8192.0000 max=14939 max_rank=0 min=2274 min_rank=1 imbalance_ratio=1.823608 max_load_reduction=50597 max_load_reduction_pct=0.772049 moved_to_echo=28173 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[21376, 128, 512, 12160, 1408, 13824, 512, 16000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[14592, 128, 18816, 11520, 4480, 8576, 896, 6912, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 21632, 24576, 384, 4992, 128, 14208, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[29440, 16256, 256, 2048, 4992, 1536, 5376, 4480, 1792, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1664, 640, 18944, 21504, 128, 20480, 2432, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 11648, 0, 6400, 17920, 5888, 2816, 2560, 8960, 8192, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6912, 7936, 5632, 384, 2176, 10880, 1792, 4224, 13184, 11520, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[14976, 5120, 4224, 17792, 4352, 6784, 256, 12544, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[3697, 0, 0, 2050, 84, 2003, 0, 0, 0, 53, 1, 3985, 151, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 24, 234, 4088, 0, 0, 3753, 0, 0, 0, 147, 3366, 1, 0, 28, 10, 1953, 2070, 3301, 4038, 349, 371, 0, 4035, 4, 122, 0, 3763, 0, 0, 758, 0, 0, 4096, 0, 0, 712, 0, 4066, 0, 4038, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 329, 3951, 0, 0, 0, 0, 2, 0, 0, 3728, 163, 0, 0, 1, 0, 223, 4, 0, 4019, 4093, 1614, 3906, 18, 0, 1857, 825, 4086, 164, 19, 4067, 0, 0, 30, 3, 2, 38, 748, 0, 0, 0, 37, 0, 3884, 0, 0, 2460, 0, 4096, 209, 0, 0, 0, 3734, 0, 0, 0, 3512, 0, 3612, 3902, 2104, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4026, 23, 0, 1, 0, 3554, 0, 0, 70, 3887, 9, 3, 1568, 0, 4071, 1, 3521, 2909, 2960, 68, 351, 607, 1368, 0, 0, 4066, 371, 2956, 849, 3932, 0, 0, 3844, 0, 0, 428, 50, 0, 0, 0, 0, 0, 0, 4096, 4036, 1, 2588, 0, 98, 220, 0, 0, 0, 22, 201, 4083, 52, 550] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[3559, 0, 3091, 848, 53, 0, 0, 7, 0, 0, 0, 2697, 0, 1, 27, 0, 6, 0, 0, 0, 4094, 4043, 3, 4051, 557, 249, 0, 0, 1818, 0, 17, 175, 1, 4096, 0, 0, 1313, 3799, 4012, 62, 4043, 8, 0, 0, 0, 14, 0, 0, 3507, 243, 0, 1760, 4075, 0, 0, 4082, 1, 590, 4096, 0, 142, 368, 4028, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 8, 4005, 17, 0, 0, 0, 2844, 36, 0, 0, 4096, 1962, 1, 893, 2004, 4067, 0, 1, 2258, 89, 0, 3, 4064, 1598, 41, 0, 0, 4094, 1181, 13, 34, 4050, 116, 430, 10, 0, 4032, 0, 4096, 0, 0, 0, 0, 3610, 3382, 0, 0, 3, 4096, 4076, 13, 0, 6, 1496, 11, 0, 0, 0, 98, 85, 2613, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 2, 273, 0, 0, 4089, 3912, 0, 0, 84, 0, 2669, 194, 75, 0, 8, 264, 12, 0, 0, 0, 268, 0, 4007, 4096, 22, 0, 122, 3995, 686, 0, 3822, 0, 3662, 6, 28, 0, 1709, 140, 0, 66, 0, 1, 0, 3863, 4073, 4086, 0, 1238, 387, 21, 0, 4096, 0, 0, 4, 0, 0, 3741, 0, 3373, 2350, 4092, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4025, 0, 0, 1977, 186, 938, 305, 0, 105, 0, 4096, 4087, 0, 0, 0, 3717, 0, 64, 0, 120, 0, 0, 2383, 3769, 0, 0, 3, 1367, 0, 0, 0, 110, 1, 2125, 48, 4025, 0, 157, 3665, 2837, 14, 4096, 0, 1, 4096, 0, 0, 0, 457, 4093, 0, 0, 527, 4085, 10, 0, 0, 0, 6, 0, 574, 4082, 132, 3253] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4004, 0, 1748, 85, 62, 0, 0, 46, 3958, 464, 10, 0, 0, 2, 0, 3933, 0, 6, 0, 0, 0, 0, 712, 4088, 3629, 0, 398, 3826, 2191, 189, 0, 32, 0, 4070, 0, 9, 3481, 1666, 146, 2, 1843, 0, 11, 23, 474, 4073, 0, 3137, 84, 3403, 4073, 5, 4094, 0, 46, 1, 0, 0, 0, 3412, 1938, 0, 5, 157] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3697, 0, 0, 2050, 84, 2003, 0, 0, 739, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1, 3985, 151, 0, 0, 4089, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 234, 4088, 0, 0, 3753, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 147, 1947, 1, 0, 28, 10, 1953, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 3301, 4038, 349, 371, 0, 4035, 4, 122, 680, 220, 0, 0, 0, 0, 0, 0, 0, 1873, 0, 0, 758, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 0, 4066, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8573, 9949, 4118, 8077, 6156, 13120, 6727, 8816] avg=8192.0000 max=13120 max_rank=5 min=4118 min_rank=2 imbalance_ratio=1.601562 max_load_reduction=52416 max_load_reduction_pct=0.799805 moved_to_echo=34819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3559, 0, 3091, 848, 53, 0, 0, 7, 900, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2697, 0, 1, 27, 0, 107, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4094, 4043, 3, 3615, 0, 0, 0, 0, 0, 0, 0, 0, 557, 249, 0, 0, 1818, 0, 17, 175, 344, 9, 0, 0, 0, 0, 0, 0, 1, 2368, 0, 0, 1313, 3799, 4012, 62, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 8, 0, 0, 0, 14, 0, 0, 828, 14, 0, 0, 0, 0, 0, 0, 3507, 122, 0, 1760, 4075, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 1, 590, 4096, 0, 142, 359, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8550, 2832, 11761, 3169, 11555, 4907, 13546, 9216] avg=8192.0000 max=13546 max_rank=6 min=2832 min_rank=1 imbalance_ratio=1.653564 max_load_reduction=51990 max_load_reduction_pct=0.793304 moved_to_echo=39224 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 329, 3951, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3728, 163, 0, 0, 1, 1818, 0, 0, 0, 0, 0, 0, 0, 0, 223, 4, 0, 4019, 4093, 1614, 3485, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1857, 825, 4086, 164, 19, 4067, 332, 102, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3, 2, 38, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 3884, 0, 0, 2460, 0, 240, 0, 0, 0, 0, 0, 0, 0, 2038, 209, 0, 0, 0, 3734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3512, 0, 3612, 3800, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8465, 5712, 13438, 11470, 821, 6621, 5981, 13028] avg=8192.0000 max=13438 max_rank=2 min=821 min_rank=4 imbalance_ratio=1.640381 max_load_reduction=52098 max_load_reduction_pct=0.794952 moved_to_echo=26451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4004, 0, 1748, 85, 62, 0, 0, 46, 894, 93, 0, 0, 0, 0, 0, 0, 3958, 464, 10, 0, 0, 2, 0, 3933, 1510, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 712, 3648, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 0, 398, 3826, 2191, 189, 0, 32, 347, 0, 0, 0, 0, 0, 0, 0, 0, 2353, 0, 9, 3481, 1666, 146, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1843, 0, 11, 23, 474, 4073, 0, 3137, 823, 199, 0, 0, 0, 0, 0, 0, 84, 1694, 4073, 5, 4094, 0, 46, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 1938, 0, 5, 157, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6932, 9877, 4366, 10612, 7657, 10583, 9997, 5512] avg=8192.0000 max=10612 max_rank=3 min=4366 min_rank=2 imbalance_ratio=1.295410 max_load_reduction=54924 max_load_reduction_pct=0.838074 moved_to_echo=33749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4026, 23, 649, 66, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3554, 0, 0, 70, 3887, 1818, 0, 0, 0, 0, 0, 0, 0, 9, 3, 1568, 0, 4071, 1, 3521, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 2960, 68, 351, 607, 1368, 0, 0, 4066, 247, 107, 0, 0, 0, 0, 0, 0, 371, 1710, 849, 3932, 0, 0, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 50, 0, 0, 0, 0, 0, 597, 240, 0, 0, 0, 0, 0, 0, 0, 2038, 4036, 1, 2588, 0, 98, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 201, 3976, 52, 550, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8860, 9330, 11769, 9774, 10706, 1315, 8981, 4801] avg=8192.0000 max=11769 max_rank=2 min=1315 min_rank=5 imbalance_ratio=1.436646 max_load_reduction=53767 max_load_reduction_pct=0.820419 moved_to_echo=25803 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4005, 17, 0, 0, 0, 2844, 25, 93, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4096, 1962, 1, 893, 2004, 1818, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 1, 2258, 89, 0, 3, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 1598, 41, 0, 0, 4094, 1181, 13, 34, 345, 68, 0, 0, 0, 0, 0, 0, 4050, 68, 430, 10, 0, 4032, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 3382, 0, 0, 23, 240, 0, 0, 0, 0, 0, 0, 3, 2038, 4076, 13, 0, 6, 1496, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 85, 2545, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6992, 10810, 10044, 7374, 12686, 7255, 7643, 2732] avg=8192.0000 max=12686 max_rank=4 min=2732 min_rank=7 imbalance_ratio=1.548584 max_load_reduction=52850 max_load_reduction_pct=0.806427 moved_to_echo=30316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4025, 0, 0, 1977, 186, 938, 305, 0, 471, 91, 0, 0, 0, 0, 0, 0, 105, 0, 4096, 4087, 0, 0, 0, 3717, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 120, 0, 0, 2383, 3353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1367, 0, 0, 0, 110, 325, 111, 0, 0, 0, 0, 0, 0, 1, 1221, 48, 4025, 0, 157, 3665, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 1, 4096, 0, 0, 0, 433, 244, 0, 0, 0, 0, 0, 0, 457, 2029, 0, 0, 527, 4085, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 574, 3971, 132, 3253, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7993, 13825, 5920, 1916, 11954, 8884, 7108, 7936] avg=8192.0000 max=13825 max_rank=1 min=1916 min_rank=3 imbalance_ratio=1.687622 max_load_reduction=51711 max_load_reduction_pct=0.789047 moved_to_echo=35882 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 273, 0, 0, 4089, 3912, 0, 804, 91, 0, 0, 0, 0, 0, 0, 0, 84, 0, 2669, 194, 75, 0, 8, 171, 0, 0, 0, 0, 0, 0, 0, 264, 12, 0, 0, 0, 268, 0, 3576, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 0, 122, 3995, 686, 0, 3822, 340, 61, 0, 0, 0, 0, 0, 0, 0, 2118, 6, 28, 0, 1709, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 1, 0, 3863, 4073, 4086, 0, 740, 22, 0, 0, 0, 0, 0, 0, 1238, 194, 21, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3741, 0, 3373, 2289, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9171, 3201, 4120, 13144, 4001, 12851, 5553, 13495] avg=8192.0000 max=13495 max_rank=7 min=3201 min_rank=1 imbalance_ratio=1.647339 max_load_reduction=52041 max_load_reduction_pct=0.794083 moved_to_echo=35900 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[13184, 4480, 2688, 6784, 21376, 2304, 128, 12416, 2304, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9344, 8576, 512, 512, 16000, 15616, 4096, 5760, 4224, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4608, 11904, 1408, 8064, 4864, 11520, 14592, 9088, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4224, 640, 4224, 24832, 2560, 128, 1024, 17664, 10752, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[4352, 384, 5760, 2432, 12288, 8448, 8320, 23936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 640, 12160, 3584, 14080, 17024, 14464, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[23552, 128, 9472, 8960, 512, 7040, 8320, 2944, 4608, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[5376, 12032, 12416, 1792, 16256, 4096, 5504, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[3296, 0, 0, 0, 0, 0, 0, 4096, 673, 410, 4096, 6, 0, 0, 0, 280, 0, 0, 898, 4096, 0, 4094, 3459, 242, 0, 4085, 0, 147, 0, 3435, 0, 0, 23, 0, 2, 0, 3918, 0, 3055, 313, 0, 3975, 0, 0, 443, 4030, 3930, 614, 0, 9, 0, 3597, 0, 0, 0, 83, 0, 4096, 4096, 4, 22, 0, 13, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4, 4096, 818, 4003, 30, 0, 26, 4024, 641, 288, 4, 0, 0, 0, 271, 68, 0, 38, 0, 1944, 3281, 0, 3737, 0, 3261, 0, 0, 1, 172, 0, 14, 0, 0, 18, 0, 4056, 892, 0, 3736, 2546, 4096, 4057, 0, 4071, 4045, 3429, 3894, 0, 382, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 382, 0, 0, 3092, 31] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 41, 0, 2706, 37, 2488, 0, 4016, 4, 0, 92, 18, 0, 0, 3704, 0, 0, 4093, 482, 2331, 0, 203, 3938, 0, 0, 0, 4096, 0, 0, 608, 429, 3732, 189, 3814, 2933, 0, 0, 2432, 2731, 3892, 0, 15, 4068, 3913, 4060, 0, 0, 0, 0, 33, 0, 0, 0, 0, 4059, 14, 0, 0, 119, 66, 172] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 224, 160, 3929, 1002, 0, 190, 0, 0, 0, 1, 10, 3804, 0, 0, 0, 1, 0, 4096, 0, 8, 4084, 870, 3109, 0, 4096, 0, 0, 2, 1, 4084, 0, 4045, 105, 4083, 10, 4074, 3953, 2790, 175, 3687, 20, 4096, 0, 14, 0, 0, 8, 0, 3, 4095, 3, 0, 4090, 2, 0, 0, 0, 612, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 53, 0, 0, 0, 0, 0, 2125, 4080, 14, 3817, 0, 1, 0, 2280, 0, 31, 4089, 279, 0, 0, 3, 0, 4096, 3559, 11, 1659, 0, 4056, 511, 0, 0, 0, 4084, 4044, 3470, 3821, 653, 1, 266, 4061, 0, 1497, 0, 4094, 3981, 0, 0, 0, 12, 10, 17, 3, 0, 0, 4, 0, 4093, 0, 16, 0, 638, 107] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 101, 0, 11, 0, 4094, 1, 3841, 0, 0, 0, 4096, 2091, 0, 1036, 30, 15, 67, 0, 0, 4096, 2737, 0, 0, 2043, 1962, 0, 243, 133, 4096, 0, 0, 20, 3396, 0, 4041, 0, 4027, 268, 0, 4054, 0, 1600, 3581, 4095, 3985, 30, 0, 709, 0, 0, 6, 1, 4045, 0, 25, 0, 0, 7, 1, 949, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 289, 9, 3855, 0, 3, 4096, 372, 32, 0, 0, 0, 3917, 7, 4006, 2, 301, 128, 4080, 901, 0, 3, 0, 0, 4091, 0, 298, 0, 1058, 4083, 9, 0, 0, 3875, 0, 4096, 4073, 0, 0, 0, 2575, 2, 4096, 4095, 4058, 3395, 3698, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[3650, 0, 1, 0, 0, 1, 2621, 4096, 1219, 3933, 290, 0, 187, 0, 870, 426, 15, 0, 1848, 0, 0, 3777, 0, 0, 0, 3881, 0, 442, 2410, 4045, 5, 0, 3559, 161, 1772, 1224, 0, 0, 0, 3792, 4096, 2753, 687, 0, 2040, 425, 3104, 4071, 0, 0, 0, 0, 0, 0, 4046, 0, 35, 0, 0, 0, 0, 53, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3296, 0, 0, 0, 0, 0, 0, 4096, 2529, 0, 0, 0, 0, 0, 0, 0, 673, 410, 4096, 6, 0, 0, 0, 280, 1244, 366, 0, 0, 0, 0, 0, 0, 0, 0, 898, 4096, 0, 4094, 3459, 242, 38, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 147, 0, 2191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 2, 0, 0, 0, 3055, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 0, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3597, 0, 0, 0, 83, 3930, 2760, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 22, 0, 13, 0, 3918, 1215, 1135, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9921, 7075, 12827, 6423, 3393, 1019, 10379, 14499] avg=8192.0000 max=14499 max_rank=7 min=1019 min_rank=5 imbalance_ratio=1.769897 max_load_reduction=51037 max_load_reduction_pct=0.778763 moved_to_echo=29290 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 224, 160, 3929, 12, 0, 0, 0, 0, 0, 0, 0, 1002, 0, 190, 0, 0, 0, 1, 10, 1483, 1, 453, 0, 0, 0, 0, 0, 3804, 0, 0, 0, 1, 0, 4096, 0, 319, 307, 0, 0, 0, 0, 0, 0, 8, 4084, 870, 3109, 0, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3324, 0, 0, 105, 4083, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 2790, 175, 3368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 8, 0, 3, 4095, 3, 4096, 2744, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 612, 0, 4045, 1209, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4325, 3140, 8527, 10684, 7525, 10407, 10963, 9965] avg=8192.0000 max=10963 max_rank=6 min=3140 min_rank=1 imbalance_ratio=1.338257 max_load_reduction=54573 max_load_reduction_pct=0.832718 moved_to_echo=38860 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4, 4096, 818, 4003, 30, 0, 26, 4024, 2152, 0, 0, 0, 0, 0, 0, 0, 641, 288, 4, 0, 0, 0, 271, 68, 0, 312, 0, 0, 0, 0, 0, 0, 0, 38, 0, 1944, 3281, 0, 3737, 0, 350, 0, 0, 0, 0, 0, 0, 0, 3261, 0, 0, 1, 172, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4056, 0, 0, 3736, 2546, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4071, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 0, 0, 0, 0, 88, 3894, 2818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 3092, 31, 892, 1239, 965, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15153, 1584, 9350, 3448, 10356, 11862, 7182, 6601] avg=8192.0000 max=15153 max_rank=0 min=1584 min_rank=1 imbalance_ratio=1.849731 max_load_reduction=50383 max_load_reduction_pct=0.768784 moved_to_echo=36001 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 101, 0, 11, 0, 4094, 1, 2570, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 0, 0, 4096, 2091, 0, 1036, 48, 372, 377, 0, 0, 0, 0, 0, 30, 15, 67, 0, 0, 4096, 2737, 0, 310, 255, 0, 0, 0, 0, 0, 0, 0, 2043, 1962, 0, 243, 85, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2764, 0, 0, 0, 4027, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1600, 3271, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 0, 0, 6, 1, 4045, 0, 3985, 2816, 0, 0, 0, 0, 0, 0, 25, 0, 0, 7, 1, 949, 3, 0, 4041, 1238, 1153, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6777, 11861, 7510, 8429, 7079, 4901, 11562, 7417] avg=8192.0000 max=11861 max_rank=1 min=4901 min_rank=5 imbalance_ratio=1.447876 max_load_reduction=53675 max_load_reduction_pct=0.819016 moved_to_echo=30959 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 41, 0, 2706, 37, 2553, 0, 0, 0, 0, 0, 0, 0, 2488, 0, 4016, 4, 0, 92, 18, 0, 1483, 370, 414, 0, 0, 0, 0, 0, 0, 3704, 0, 0, 4093, 482, 2331, 0, 1, 281, 0, 0, 0, 0, 0, 0, 203, 3938, 0, 0, 0, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 608, 429, 3037, 189, 0, 2933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2432, 0, 3892, 0, 14, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 3913, 1896, 0, 0, 0, 0, 0, 0, 0, 4059, 14, 0, 0, 119, 66, 172, 3814, 835, 1145, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5345, 8885, 10892, 6754, 7196, 10398, 5842, 10224] avg=8192.0000 max=10892 max_rank=2 min=5345 min_rank=0 imbalance_ratio=1.329590 max_load_reduction=54644 max_load_reduction_pct=0.833801 moved_to_echo=33660 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 289, 9, 3855, 0, 3, 4096, 372, 2130, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 3917, 7, 4006, 2, 1479, 308, 0, 0, 0, 0, 0, 0, 301, 128, 4080, 901, 0, 3, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 298, 0, 1058, 2604, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2575, 0, 4096, 4095, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 3698, 1, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 1, 0, 4073, 1, 957, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10754, 9751, 5765, 8060, 7971, 14472, 3703, 5060] avg=8192.0000 max=14472 max_rank=5 min=3703 min_rank=6 imbalance_ratio=1.766602 max_load_reduction=51064 max_load_reduction_pct=0.779175 moved_to_echo=31206 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 53, 0, 0, 0, 0, 0, 2571, 0, 0, 0, 0, 0, 0, 0, 2125, 4080, 14, 3817, 0, 1, 0, 2280, 1469, 371, 453, 0, 0, 0, 0, 0, 0, 31, 4089, 279, 0, 0, 3, 0, 0, 307, 0, 0, 0, 0, 0, 0, 4096, 3559, 11, 1659, 0, 2587, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3324, 4044, 0, 3821, 653, 1, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 10, 17, 3, 0, 0, 3981, 2820, 0, 0, 0, 0, 0, 0, 4, 0, 4093, 0, 16, 0, 638, 107, 3470, 1241, 1152, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[2624, 14610, 4709, 12423, 11843, 1763, 6843, 10721] avg=8192.0000 max=14610 max_rank=1 min=1763 min_rank=5 imbalance_ratio=1.783447 max_load_reduction=50926 max_load_reduction_pct=0.777069 moved_to_echo=31170 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3650, 0, 1, 0, 0, 1, 2621, 4096, 268, 0, 0, 0, 0, 0, 0, 0, 1219, 3933, 290, 0, 187, 0, 870, 426, 1468, 38, 199, 0, 0, 0, 0, 0, 15, 0, 1848, 0, 0, 3777, 0, 0, 180, 136, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 442, 2410, 2577, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3559, 161, 1437, 1224, 0, 0, 0, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 687, 0, 1860, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 3104, 1912, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 53, 1, 0, 0, 841, 119, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10637, 8630, 5956, 9315, 10173, 10714, 9062, 1049] avg=8192.0000 max=10714 max_rank=5 min=1049 min_rank=7 imbalance_ratio=1.307861 max_load_reduction=54822 max_load_reduction_pct=0.836517 moved_to_echo=30998 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[17664, 0, 11520, 11520, 16384, 0, 0, 8832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[12032, 8832, 8704, 3840, 8320, 2304, 5248, 4224, 8704, 2176, 1920, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7040, 4480, 1024, 7936, 128, 256, 13824, 16640, 14848, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 12288, 8320, 512, 128, 1152, 4480, 384, 24320, 7936, 6656, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11776, 21632, 3200, 5376, 3968, 15360, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4224, 4608, 13952, 13696, 0, 6912, 15616, 7040, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4224, 3968, 11008, 7296, 7424, 12544, 16384, 256, 1664, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 768, 128, 3712, 128, 128, 12288, 256, 30720, 17792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 128, 4081, 0, 4084, 712, 2, 0, 0, 627, 84, 1401, 3778, 0, 0, 2069, 2323, 0, 4074, 2973, 0, 4, 1, 0, 0, 2, 0, 5, 1687, 0, 0, 0, 3886, 0, 0, 0, 4095, 3930, 0, 3576, 1, 0, 4012, 1023, 1, 2746, 0, 1768, 0, 4068, 0, 30, 0, 0, 4076, 0, 865, 3405, 19, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1, 46, 1057, 0, 77, 4095, 0, 4, 0, 3950, 4096, 0, 248, 4096, 0, 0, 0, 0, 0, 2834, 3937, 0, 3764, 0, 0, 0, 84, 0, 0, 79, 0, 1864, 0, 0, 0, 4063, 0, 0, 22, 2295, 3916, 0, 4092, 1, 2889, 35, 0, 0, 3863, 0, 171, 440, 4095, 42, 4041, 0, 0, 3195, 0, 0, 2144, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1894, 2328, 0, 246, 0, 0, 49, 0, 0, 4096, 99, 17, 0, 175, 0, 0, 4061, 0, 4085, 9, 3845, 695, 0, 0, 4078, 3845, 1799, 0, 0, 0, 0, 0, 36, 192, 0, 0, 0, 1, 1598, 4079, 17, 0, 4090, 35, 0, 0, 2, 0, 4023, 3169, 0, 0, 4093, 15, 6, 0, 167, 4066, 3931, 0, 2587, 0, 2108, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[3470, 0, 0, 0, 2709, 4063, 0, 0, 28, 1195, 3939, 0, 3334, 0, 4043, 4018, 0, 0, 0, 4096, 4050, 2, 4096, 0, 64, 317, 14, 0, 0, 0, 0, 130, 35, 29, 0, 2091, 0, 1, 33, 4096, 1116, 0, 4031, 15, 0, 1250, 126, 24, 0, 4095, 0, 0, 3179, 1690, 0, 0, 0, 4094, 0, 46, 13, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4075, 0, 0, 135, 81, 0, 3, 0, 1934, 0, 0, 1879, 1, 1, 3, 0, 208, 27, 4095, 0, 3980, 3945, 3810, 0, 0, 0, 3810, 0, 0, 0, 284, 13, 9, 0, 0, 2386, 3134, 0, 0, 4030, 396, 0, 4045, 1015, 0, 3745, 119, 1920, 3979, 0, 16, 0, 3194, 6, 1, 0, 0, 4096, 0, 799, 4096, 0, 266, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1421, 0, 3, 0, 0, 4073, 335, 0, 10, 3963, 3966, 68, 0, 0, 0, 3949, 0, 83, 436, 3340, 4096, 1903, 0, 0, 0, 4093, 4003, 0, 0, 0, 6, 0, 128, 1, 0, 4084, 0, 0, 0, 4078, 4096, 530, 4039, 0, 0, 57, 4045, 39, 0, 0, 19, 0, 4094, 160, 0, 0, 2, 0, 28, 3672, 716, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 3019, 4096, 0, 0, 0, 986, 4, 0, 0, 4020, 0, 2930, 0, 53, 0, 409, 0, 4096, 1923, 51, 4080, 2717, 1043, 0, 0, 88, 0, 0, 129, 1, 0, 0, 0, 0, 4017, 0, 0, 3, 4038, 34, 0, 3123, 113, 0, 3681, 1204, 3938, 99, 0, 3078, 111, 1342, 21, 70, 0, 0, 2923, 4037, 0, 4006, 0, 14, 39] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[322, 1869, 3777, 819, 993, 4096, 0, 0, 0, 3853, 4095, 0, 1, 718, 0, 0, 72, 0, 0, 0, 0, 0, 4096, 1402, 4058, 255, 33, 0, 0, 0, 0, 1465, 3009, 0, 0, 3840, 0, 1, 0, 3953, 1, 0, 3991, 0, 3738, 0, 1899, 4091, 11, 366, 313, 0, 3313, 5, 68, 0, 0, 0, 0, 0, 4096, 917, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 46, 1057, 0, 77, 4095, 0, 4, 876, 93, 0, 0, 0, 0, 0, 0, 0, 3950, 2913, 0, 248, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2834, 2807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 79, 0, 1864, 3764, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 22, 2295, 448, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 1649, 1, 2889, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 0, 171, 440, 4095, 42, 4041, 0, 682, 55, 0, 0, 0, 0, 0, 0, 0, 3195, 0, 0, 2144, 0, 0, 0, 1035, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6249, 11207, 5641, 7358, 6828, 8490, 13389, 6374] avg=8192.0000 max=13389 max_rank=6 min=5641 min_rank=2 imbalance_ratio=1.634399 max_load_reduction=52147 max_load_reduction_pct=0.795700 moved_to_echo=35081 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 128, 4081, 0, 4084, 765, 0, 0, 0, 0, 0, 0, 0, 712, 2, 0, 0, 627, 84, 1401, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2069, 2323, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 2, 0, 5, 1687, 2973, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 1442, 1, 0, 4012, 1023, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2746, 0, 1768, 0, 4068, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 865, 3405, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9058, 6604, 8466, 6041, 7981, 10409, 8612, 8365] avg=8192.0000 max=10409 max_rank=5 min=6041 min_rank=3 imbalance_ratio=1.270630 max_load_reduction=55127 max_load_reduction_pct=0.841171 moved_to_echo=35367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4075, 0, 0, 135, 81, 0, 3, 0, 866, 0, 0, 0, 0, 0, 0, 0, 1934, 0, 0, 1879, 1, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 27, 4095, 0, 2838, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 0, 0, 0, 284, 13, 3810, 1549, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2386, 3134, 0, 0, 4030, 453, 0, 0, 0, 0, 0, 0, 0, 396, 0, 1630, 1015, 0, 3745, 119, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 16, 0, 3194, 6, 1, 0, 689, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 799, 4096, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5160, 3818, 11113, 9466, 10012, 8825, 7885, 9257] avg=8192.0000 max=11113 max_rank=2 min=3818 min_rank=1 imbalance_ratio=1.356567 max_load_reduction=54423 max_load_reduction_pct=0.830429 moved_to_echo=35979 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1894, 2328, 0, 246, 0, 0, 49, 0, 875, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 71, 17, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4085, 9, 2742, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3845, 1799, 0, 0, 0, 0, 0, 0, 1566, 0, 0, 0, 0, 0, 0, 36, 192, 0, 0, 0, 1, 1598, 4079, 437, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1649, 35, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 3169, 0, 0, 4093, 15, 6, 0, 666, 1, 0, 0, 0, 0, 0, 0, 167, 4066, 3931, 0, 2587, 0, 2108, 0, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5394, 4359, 11592, 11288, 6343, 1703, 11973, 12884] avg=8192.0000 max=12884 max_rank=7 min=1703 min_rank=5 imbalance_ratio=1.572754 max_load_reduction=52652 max_load_reduction_pct=0.803406 moved_to_echo=32903 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3470, 0, 0, 0, 2709, 4063, 0, 0, 863, 89, 0, 0, 0, 0, 0, 0, 28, 1195, 2802, 0, 3334, 0, 4043, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2887, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 317, 14, 0, 0, 0, 0, 130, 4096, 1544, 0, 0, 0, 0, 0, 0, 35, 29, 0, 2091, 0, 1, 33, 4096, 461, 0, 0, 0, 0, 0, 0, 0, 1116, 0, 1624, 15, 0, 1250, 126, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3179, 1690, 0, 0, 702, 53, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 46, 13, 0, 0, 4, 995, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11194, 15420, 6985, 6165, 6746, 4155, 9719, 5152] avg=8192.0000 max=15420 max_rank=1 min=4155 min_rank=5 imbalance_ratio=1.882324 max_load_reduction=50116 max_load_reduction_pct=0.764709 moved_to_echo=25772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1421, 0, 3, 0, 0, 4073, 335, 0, 864, 90, 0, 0, 0, 0, 0, 0, 10, 3963, 2820, 68, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 436, 3340, 2914, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4003, 0, 0, 0, 6, 0, 0, 1547, 0, 0, 0, 0, 0, 0, 128, 1, 0, 4084, 0, 0, 0, 4078, 469, 0, 0, 0, 0, 0, 0, 0, 4096, 530, 1628, 0, 0, 57, 4045, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4094, 160, 0, 0, 713, 54, 0, 0, 0, 0, 0, 0, 2, 0, 28, 3672, 716, 0, 0, 0, 1002, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6786, 10810, 8676, 9649, 8760, 10395, 5040, 5420] avg=8192.0000 max=10810 max_rank=1 min=5040 min_rank=6 imbalance_ratio=1.319580 max_load_reduction=54726 max_load_reduction_pct=0.835052 moved_to_echo=29615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[322, 1869, 3777, 819, 993, 4096, 0, 0, 859, 96, 0, 0, 0, 0, 0, 0, 0, 3853, 2903, 0, 1, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 255, 33, 0, 0, 0, 0, 1465, 4096, 1531, 0, 0, 0, 0, 0, 0, 3009, 0, 0, 3840, 0, 1, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1601, 0, 3738, 0, 1899, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 11, 366, 313, 0, 3313, 5, 68, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 917, 0, 0, 1037, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12831, 7475, 1474, 11438, 10803, 11330, 4135, 6050] avg=8192.0000 max=12831 max_rank=0 min=1474 min_rank=2 imbalance_ratio=1.566284 max_load_reduction=52705 max_load_reduction_pct=0.804214 moved_to_echo=32318 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3019, 4096, 0, 0, 0, 986, 4, 668, 91, 0, 0, 0, 0, 0, 0, 0, 0, 2860, 0, 2930, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 4096, 1923, 38, 4080, 0, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 129, 1, 0, 2717, 1196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 3, 4038, 5, 0, 0, 0, 0, 0, 0, 0, 34, 0, 1259, 113, 0, 3681, 1204, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 3078, 111, 1342, 21, 70, 0, 8, 54, 0, 0, 0, 0, 0, 0, 0, 2923, 4037, 0, 4006, 0, 14, 39, 1015, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8864, 5843, 11589, 4131, 8063, 10229, 4783, 12034] avg=8192.0000 max=12034 max_rank=7 min=4131 min_rank=3 imbalance_ratio=1.468994 max_load_reduction=53502 max_load_reduction_pct=0.816376 moved_to_echo=35109 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[13568, 640, 12544, 1280, 6656, 12800, 8448, 10112, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4864, 128, 14848, 14592, 14336, 14720, 0, 2560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 22528, 8064, 5504, 21120, 1024, 2432, 128, 5120, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[14848, 7680, 5376, 640, 27392, 2048, 4224, 0, 3584, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8576, 9856, 0, 128, 256, 384, 5248, 21504, 11904, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11264, 7296, 8960, 1280, 4096, 20480, 1408, 4096, 6656, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3328, 256, 0, 24448, 3200, 128, 1664, 30720, 2304, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2688, 17152, 14464, 2048, 7168, 5120, 5504, 11776, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True + [2026-08-30 04:59:26] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 906.6 | throughput per GPU (TFLOP/s/GPU): 367.7 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101572E+01 | loss scale: 1.0 | grad norm: 45.914 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[55, 2702, 1471, 181, 1748, 80, 784, 540, 1105, 117, 636, 370, 1301, 558, 48, 513, 221, 795, 1609, 1050, 2220, 364, 2727, 1560, 84, 1530, 907, 1309, 1283, 415, 1330, 1923, 3499, 410, 202, 407, 132, 634, 2185, 1159, 3344, 1705, 1220, 396, 791, 2143, 1065, 725, 269, 608, 281, 1306, 555, 592, 2162, 135, 127, 852, 2102, 1777, 1005, 375, 1108, 729] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[63, 2351, 859, 201, 2240, 29, 608, 323, 1446, 82, 870, 836, 791, 723, 155, 550, 367, 993, 2064, 1395, 1735, 370, 3069, 1642, 93, 1569, 344, 1060, 872, 368, 1551, 1836, 3173, 459, 552, 207, 470, 1421, 1703, 1182, 2002, 1388, 2090, 725, 952, 1630, 858, 1398, 204, 532, 335, 997, 209, 309, 2014, 223, 219, 1402, 1662, 2562, 1000, 632, 886, 685] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[42, 2250, 1258, 137, 1935, 38, 1022, 294, 926, 80, 872, 604, 1003, 832, 66, 595, 133, 1284, 974, 624, 1747, 662, 2751, 1729, 95, 2087, 933, 1140, 779, 337, 1021, 2019, 3528, 500, 333, 699, 243, 1376, 1950, 1253, 2815, 1167, 1238, 356, 826, 2306, 1626, 1065, 305, 339, 308, 1095, 352, 381, 1913, 95, 435, 730, 2061, 2544, 1028, 631, 991, 778] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[162, 2347, 680, 110, 2199, 90, 871, 310, 1143, 61, 1175, 476, 849, 646, 83, 524, 149, 976, 1141, 1151, 1883, 563, 2819, 1785, 47, 1732, 470, 841, 651, 228, 1598, 1846, 3682, 527, 173, 606, 206, 1317, 2254, 946, 2398, 1533, 1375, 394, 587, 2383, 1377, 1098, 253, 364, 1017, 1057, 459, 499, 1781, 283, 427, 1127, 2643, 2579, 1093, 490, 460, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[115, 2666, 1108, 167, 1473, 18, 901, 386, 1417, 213, 1112, 536, 1271, 775, 91, 557, 190, 725, 1694, 634, 2288, 568, 2072, 1824, 114, 1592, 632, 1333, 1118, 413, 1769, 1769, 3279, 492, 588, 669, 289, 1488, 1751, 617, 2417, 1394, 1467, 375, 523, 2013, 1102, 671, 205, 352, 365, 1250, 650, 442, 1961, 117, 73, 1548, 2166, 1950, 1331, 422, 1172, 856] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[63, 2216, 1401, 196, 1707, 57, 808, 496, 1402, 112, 1555, 661, 572, 776, 97, 689, 187, 1201, 1355, 1176, 1827, 515, 2504, 1579, 54, 1563, 293, 1307, 951, 163, 1794, 1958, 3511, 419, 605, 545, 437, 1286, 1749, 716, 2518, 1491, 1700, 826, 719, 2168, 995, 753, 242, 566, 366, 1253, 431, 316, 2084, 155, 292, 794, 2115, 2327, 806, 464, 1070, 582] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[82, 2507, 1497, 248, 2050, 33, 821, 400, 1633, 128, 1078, 427, 708, 614, 77, 470, 157, 774, 1076, 1483, 1713, 697, 2479, 1880, 129, 1776, 604, 1029, 1261, 267, 1980, 1929, 3246, 256, 769, 508, 196, 1210, 1965, 705, 2364, 1963, 1956, 669, 432, 1950, 960, 757, 218, 523, 489, 1309, 268, 541, 1281, 93, 84, 1117, 2468, 1933, 952, 353, 1349, 645] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[201, 2226, 1567, 187, 1666, 31, 871, 505, 1403, 81, 1310, 824, 1068, 638, 90, 503, 197, 886, 1694, 972, 2278, 509, 2593, 1618, 115, 1742, 649, 1446, 995, 421, 1763, 1827, 3248, 426, 535, 441, 187, 1049, 2040, 655, 2642, 1167, 1425, 856, 580, 2204, 1023, 598, 378, 526, 370, 1188, 283, 277, 1417, 100, 95, 1330, 2042, 2098, 1218, 681, 1004, 577] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[55, 2702, 1471, 181, 1748, 80, 784, 540, 575, 206, 193, 0, 0, 0, 0, 0, 1105, 117, 636, 370, 1301, 558, 48, 513, 1891, 995, 31, 0, 0, 0, 0, 0, 221, 795, 1609, 1050, 2220, 364, 345, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 84, 1530, 907, 1309, 1283, 415, 1330, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 2504, 410, 202, 407, 132, 634, 2185, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1705, 1220, 396, 791, 1937, 1065, 725, 0, 0, 0, 0, 0, 0, 0, 0, 269, 608, 281, 1306, 555, 592, 2162, 135, 3344, 491, 0, 0, 0, 0, 0, 0, 127, 852, 2102, 1171, 1005, 375, 1108, 729, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8535, 7565, 8164, 8588, 7633, 7839, 9743, 7469] avg=8192.0000 max=9743 max_rank=6 min=7469 min_rank=7 imbalance_ratio=1.189331 max_load_reduction=55793 max_load_reduction_pct=0.851334 moved_to_echo=32684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[42, 2250, 1258, 137, 1935, 38, 1022, 294, 823, 222, 202, 0, 0, 0, 0, 0, 926, 80, 872, 604, 1003, 832, 66, 595, 1907, 1003, 44, 0, 0, 0, 0, 0, 133, 1284, 974, 624, 1747, 662, 349, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 95, 2087, 933, 1140, 779, 337, 1021, 1817, 0, 0, 0, 0, 0, 0, 0, 0, 2525, 500, 333, 699, 243, 1376, 1950, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 1238, 356, 826, 2084, 1626, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 305, 339, 308, 1095, 352, 381, 1913, 95, 2815, 495, 0, 0, 0, 0, 0, 0, 435, 730, 2061, 1677, 1028, 631, 991, 778, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 7932, 7502, 8209, 8879, 8362, 8098, 8331] avg=8192.0000 max=8879 max_rank=4 min=7502 min_rank=2 imbalance_ratio=1.083862 max_load_reduction=56657 max_load_reduction_pct=0.864517 moved_to_echo=33670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[63, 2351, 859, 201, 2240, 29, 608, 323, 829, 157, 184, 0, 0, 0, 0, 0, 1446, 82, 870, 836, 791, 723, 155, 550, 2128, 902, 44, 0, 0, 0, 0, 0, 367, 993, 2064, 1395, 1735, 370, 388, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 93, 1569, 344, 1060, 872, 368, 1551, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 2271, 459, 552, 207, 470, 1421, 1703, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1388, 2090, 725, 952, 1473, 858, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 204, 532, 335, 997, 209, 309, 2014, 223, 2002, 553, 0, 0, 0, 0, 0, 0, 219, 1402, 1662, 1689, 1000, 632, 886, 685, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7844, 8527, 8954, 7509, 8265, 8884, 7378, 8175] avg=8192.0000 max=8954 max_rank=2 min=7378 min_rank=6 imbalance_ratio=1.093018 max_load_reduction=56582 max_load_reduction_pct=0.863373 moved_to_echo=32702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[162, 2347, 680, 110, 2199, 90, 871, 310, 834, 229, 185, 0, 0, 0, 0, 0, 1143, 61, 1175, 476, 849, 646, 83, 524, 1954, 1047, 44, 0, 0, 0, 0, 0, 149, 976, 1141, 1151, 1883, 563, 357, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 47, 1732, 470, 841, 651, 228, 1598, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 2635, 527, 173, 606, 206, 1317, 2254, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1533, 1375, 394, 587, 2154, 1377, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 253, 364, 1017, 1057, 459, 499, 1781, 283, 2398, 508, 0, 0, 0, 0, 0, 0, 427, 1127, 2643, 1701, 1093, 490, 460, 542, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8017, 8002, 8005, 7228, 8664, 8518, 8619, 8483] avg=8192.0000 max=8664 max_rank=4 min=7228 min_rank=3 imbalance_ratio=1.057617 max_load_reduction=56872 max_load_reduction_pct=0.867798 moved_to_echo=34284 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[115, 2666, 1108, 167, 1473, 18, 901, 386, 630, 193, 177, 0, 0, 0, 0, 0, 1417, 213, 1112, 536, 1271, 775, 91, 557, 1436, 932, 34, 0, 0, 0, 0, 0, 190, 725, 1694, 634, 2288, 568, 263, 1824, 0, 0, 0, 0, 0, 0, 0, 0, 114, 1592, 632, 1333, 1118, 413, 1769, 1592, 0, 0, 0, 0, 0, 0, 0, 0, 2347, 492, 588, 669, 289, 1488, 1751, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1394, 1467, 375, 523, 1820, 1102, 671, 0, 0, 0, 0, 0, 0, 0, 0, 205, 352, 365, 1250, 650, 442, 1961, 117, 2417, 373, 0, 0, 0, 0, 0, 0, 73, 1548, 2166, 1286, 1331, 422, 1172, 856, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7834, 8374, 8186, 8563, 8241, 7352, 8132, 8854] avg=8192.0000 max=8854 max_rank=7 min=7352 min_rank=5 imbalance_ratio=1.080811 max_load_reduction=56682 max_load_reduction_pct=0.864899 moved_to_echo=32579 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[63, 2216, 1401, 196, 1707, 57, 808, 496, 752, 208, 196, 0, 0, 0, 0, 0, 1402, 112, 1555, 661, 572, 776, 97, 689, 1736, 998, 40, 0, 0, 0, 0, 0, 187, 1201, 1355, 1176, 1827, 515, 317, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 54, 1563, 293, 1307, 951, 163, 1794, 1762, 0, 0, 0, 0, 0, 0, 0, 0, 2513, 419, 605, 545, 437, 1286, 1749, 716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1491, 1700, 826, 719, 1960, 995, 753, 0, 0, 0, 0, 0, 0, 0, 0, 242, 566, 366, 1253, 431, 316, 2084, 155, 2518, 451, 0, 0, 0, 0, 0, 0, 292, 794, 2115, 1535, 806, 464, 1070, 582, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8638, 8157, 7887, 8270, 8444, 8382, 7658] avg=8192.0000 max=8638 max_rank=1 min=7658 min_rank=7 imbalance_ratio=1.054443 max_load_reduction=56898 max_load_reduction_pct=0.868195 moved_to_echo=32754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[82, 2507, 1497, 248, 2050, 33, 821, 400, 625, 187, 193, 0, 0, 0, 0, 0, 1633, 128, 1078, 427, 708, 614, 77, 470, 1719, 923, 33, 0, 0, 0, 0, 0, 157, 774, 1076, 1483, 1713, 697, 314, 1880, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1776, 604, 1029, 1261, 267, 1980, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 256, 769, 508, 196, 1210, 1965, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1963, 1956, 669, 432, 1763, 960, 757, 0, 0, 0, 0, 0, 0, 0, 0, 218, 523, 489, 1309, 268, 541, 1281, 93, 2364, 446, 0, 0, 0, 0, 0, 0, 84, 1117, 2468, 1275, 952, 353, 1349, 645, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8643, 7810, 8094, 8782, 7932, 8500, 7532, 8243] avg=8192.0000 max=8782 max_rank=3 min=7532 min_rank=6 imbalance_ratio=1.072021 max_load_reduction=56754 max_load_reduction_pct=0.865997 moved_to_echo=32207 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[201, 2226, 1567, 187, 1666, 31, 871, 505, 682, 216, 188, 0, 0, 0, 0, 0, 1403, 81, 1310, 824, 1068, 638, 90, 503, 1802, 929, 40, 0, 0, 0, 0, 0, 197, 886, 1694, 972, 2278, 509, 320, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 115, 1742, 649, 1446, 995, 421, 1763, 1639, 0, 0, 0, 0, 0, 0, 0, 0, 2319, 426, 535, 441, 187, 1049, 2040, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 1425, 856, 580, 1988, 1023, 598, 0, 0, 0, 0, 0, 0, 0, 0, 378, 526, 370, 1188, 283, 277, 1417, 100, 2642, 471, 0, 0, 0, 0, 0, 0, 95, 1330, 2042, 1376, 1218, 681, 1004, 577, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8340, 8688, 8474, 8770, 7652, 7637, 7652, 8323] avg=8192.0000 max=8770 max_rank=3 min=7637 min_rank=5 imbalance_ratio=1.070557 max_load_reduction=56766 max_load_reduction_pct=0.866180 moved_to_echo=31264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[896, 19328, 9856, 1536, 15104, 384, 6784, 3328, 5760, 1664, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1792, 8960, 17280, 11776, 8448, 4096, 8064, 5504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[10496, 896, 8704, 4736, 7680, 5632, 768, 4480, 14592, 7808, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1664, 7680, 11648, 8576, 15744, 4352, 2688, 13696, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 11904, 12544, 4608, 5504, 15232, 9088, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2176, 3840, 3584, 9472, 3328, 3456, 14720, 1280, 20608, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[768, 13696, 4864, 9472, 7936, 2688, 12928, 13696, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[19456, 3584, 3840, 4096, 2176, 9856, 15616, 7296, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4096, 3, 0, 458, 1, 2, 3, 16, 5, 4084, 781, 0, 3, 367, 161, 240, 0, 0, 3637, 4095, 4077, 3706, 0, 4065, 3946, 0, 0, 68, 2361, 0, 3, 0, 1967, 4096, 0, 4096, 0, 317, 11, 0, 17, 2, 1279, 3500, 6, 31, 54, 0, 0, 1777, 0, 1524, 1, 2459, 0, 0, 23, 1, 6, 13, 4093, 4028, 14, 43] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4089, 0, 69, 331, 0, 10, 0, 4, 32, 4059, 91, 5, 10, 1, 12, 8, 0, 0, 3837, 4071, 4050, 3444, 0, 4081, 3943, 1, 0, 0, 592, 0, 82, 53, 4073, 3885, 0, 4096, 0, 0, 0, 0, 52, 44, 3558, 3434, 78, 0, 63, 0, 0, 1039, 9, 2109, 0, 2450, 0, 0, 1, 27, 0, 0, 3815, 3911, 0, 17] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4087, 2, 65, 17, 17, 183, 76, 0, 0, 4079, 5, 0, 0, 1, 24, 9, 0, 0, 3221, 3984, 3930, 4094, 0, 4024, 3908, 0, 0, 1203, 221, 1, 10, 11, 1522, 3684, 0, 4083, 0, 13, 2, 0, 1, 42, 3059, 4095, 267, 5, 143, 2, 0, 741, 2, 3181, 1, 3618, 0, 0, 25, 0, 36, 0, 4090, 3694, 0, 58] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3784, 0, 0, 1038, 32, 0, 5, 19, 0, 4091, 10, 71, 0, 120, 6, 0, 15, 52, 2738, 4081, 3907, 4089, 0, 3982, 4089, 9, 2, 106, 136, 3, 5, 0, 3530, 4003, 20, 4056, 18, 6, 0, 0, 0, 553, 1291, 2027, 1, 2, 1087, 54, 0, 3938, 138, 2939, 3, 1002, 0, 0, 51, 0, 8, 180, 4077, 4069, 0, 93] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4069, 6, 0, 282, 1, 486, 0, 0, 15, 4080, 370, 0, 0, 0, 101, 164, 0, 0, 1382, 4055, 3951, 4038, 0, 4096, 4080, 1, 0, 4, 146, 0, 214, 3, 3975, 4075, 0, 4095, 0, 20, 0, 0, 984, 0, 302, 4056, 0, 0, 191, 1, 0, 2530, 31, 3114, 0, 2060, 0, 8, 13, 0, 1, 3, 3706, 3784, 3, 1040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4083, 0, 2, 1991, 0, 260, 33, 19, 0, 4062, 637, 6, 43, 6, 1038, 42, 0, 0, 2823, 4042, 4075, 4069, 4, 4086, 3886, 0, 0, 30, 1386, 0, 4, 64, 3014, 3785, 0, 4093, 0, 54, 34, 2, 376, 0, 1252, 3787, 0, 0, 581, 5, 14, 1877, 29, 1331, 52, 1087, 11, 2, 0, 0, 0, 116, 4086, 3107, 8, 142] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4093, 91, 3, 72, 37, 3, 1, 0, 0, 3899, 1, 1, 78, 0, 1, 48, 0, 1, 1741, 4035, 4095, 3947, 0, 4072, 4088, 0, 4, 5, 2659, 25, 0, 0, 2447, 4082, 4, 4094, 16, 9, 1, 10, 266, 1, 3940, 3358, 22, 0, 737, 5, 6, 2454, 0, 325, 0, 2224, 44, 3, 78, 0, 1, 107, 3778, 3827, 1, 696] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4094, 0, 0, 45, 0, 8, 89, 0, 0, 3980, 124, 0, 1, 1, 233, 20, 0, 0, 3197, 4068, 4020, 3355, 0, 4079, 4043, 16, 0, 111, 1718, 0, 162, 0, 2220, 3892, 4, 4095, 0, 1080, 0, 5, 21, 2, 1248, 3913, 4, 3, 235, 1, 4, 2331, 2, 2379, 0, 2541, 0, 0, 0, 0, 0, 92, 4021, 4032, 4, 43] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 3, 0, 458, 1, 2, 3, 16, 2927, 522, 0, 0, 0, 0, 0, 0, 5, 4084, 781, 0, 3, 367, 161, 240, 3543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3637, 0, 1544, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 68, 2361, 0, 3, 0, 1614, 1168, 0, 0, 0, 0, 0, 0, 1967, 4096, 0, 1210, 0, 317, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2, 1279, 3500, 6, 31, 54, 0, 1272, 677, 0, 0, 0, 0, 0, 0, 0, 1777, 0, 1524, 1, 2459, 0, 0, 1796, 0, 0, 0, 0, 0, 0, 0, 23, 1, 6, 13, 4093, 4028, 14, 43, 60, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8028, 9184, 8887, 9160, 7601, 6838, 7557, 8281] avg=8192.0000 max=9184 max_rank=1 min=6838 min_rank=5 imbalance_ratio=1.121094 max_load_reduction=56352 max_load_reduction_pct=0.859863 moved_to_echo=30277 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 69, 331, 0, 10, 0, 4, 2911, 525, 0, 0, 0, 0, 0, 0, 32, 4059, 91, 5, 10, 1, 12, 8, 3556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3837, 0, 1534, 3444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 1, 0, 0, 592, 0, 82, 53, 1614, 1160, 0, 0, 0, 0, 0, 0, 4073, 3885, 0, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 44, 3558, 3434, 78, 0, 63, 0, 1272, 673, 0, 0, 0, 0, 0, 0, 0, 1039, 9, 2109, 0, 2450, 0, 0, 1784, 0, 0, 0, 0, 0, 0, 0, 1, 27, 0, 0, 3815, 3911, 0, 17, 59, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7939, 7774, 8815, 7445, 9168, 9174, 7391, 7830] avg=8192.0000 max=9174 max_rank=5 min=7391 min_rank=6 imbalance_ratio=1.119873 max_load_reduction=56362 max_load_reduction_pct=0.860016 moved_to_echo=33563 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4087, 2, 65, 17, 17, 183, 76, 0, 2848, 517, 0, 0, 0, 0, 0, 0, 0, 4079, 5, 0, 0, 1, 24, 9, 3507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3221, 0, 1488, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 0, 0, 1203, 221, 1, 10, 11, 1608, 1136, 0, 0, 0, 0, 0, 0, 1522, 3684, 0, 1207, 0, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 42, 3059, 4095, 267, 5, 143, 2, 1268, 653, 0, 0, 0, 0, 0, 0, 0, 741, 2, 3181, 1, 3618, 0, 0, 1731, 0, 0, 0, 0, 0, 0, 0, 25, 0, 36, 0, 4090, 3694, 0, 58, 58, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7812, 7625, 8803, 8098, 6428, 9535, 9274, 7961] avg=8192.0000 max=9535 max_rank=5 min=6428 min_rank=4 imbalance_ratio=1.163940 max_load_reduction=56001 max_load_reduction_pct=0.854507 moved_to_echo=33198 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3784, 0, 0, 1038, 32, 0, 5, 19, 2918, 511, 0, 0, 0, 0, 0, 0, 0, 4091, 10, 71, 0, 120, 6, 0, 3471, 0, 0, 0, 0, 0, 0, 0, 15, 52, 2738, 0, 1480, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 9, 2, 106, 136, 3, 5, 0, 1598, 1163, 0, 0, 0, 0, 0, 0, 3530, 4003, 20, 1199, 18, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 1291, 2027, 1, 2, 1087, 54, 1259, 649, 0, 0, 0, 0, 0, 0, 0, 3938, 138, 2939, 3, 1002, 0, 0, 1721, 0, 0, 0, 0, 0, 0, 0, 51, 0, 8, 180, 4077, 4069, 0, 93, 57, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8307, 7769, 8374, 7111, 8776, 6923, 9741, 8535] avg=8192.0000 max=9741 max_rank=6 min=6923 min_rank=5 imbalance_ratio=1.189087 max_load_reduction=55795 max_load_reduction_pct=0.851364 moved_to_echo=33975 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4069, 6, 0, 282, 1, 486, 0, 0, 2899, 526, 0, 0, 0, 0, 0, 0, 15, 4080, 370, 0, 0, 0, 101, 164, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 0, 1496, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 1, 0, 4, 146, 0, 214, 3, 1613, 1156, 0, 0, 0, 0, 0, 0, 3975, 4075, 0, 1211, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 984, 0, 302, 4056, 0, 0, 191, 1, 1271, 657, 0, 0, 0, 0, 0, 0, 0, 2530, 31, 3114, 0, 2060, 0, 8, 1740, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 3, 3706, 3784, 3, 1040, 58, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8300, 6916, 7217, 9281, 7462, 9483, 8608] avg=8192.0000 max=9483 max_rank=6 min=6916 min_rank=2 imbalance_ratio=1.157593 max_load_reduction=56053 max_load_reduction_pct=0.855301 moved_to_echo=34834 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4093, 91, 3, 72, 37, 3, 1, 0, 2885, 524, 0, 0, 0, 0, 0, 0, 0, 3899, 1, 1, 78, 0, 1, 48, 3548, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1741, 0, 1551, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4, 5, 2659, 25, 0, 0, 1613, 1150, 0, 0, 0, 0, 0, 0, 2447, 4082, 4, 1210, 16, 9, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 266, 1, 3940, 3358, 22, 0, 737, 5, 1271, 680, 0, 0, 0, 0, 0, 0, 6, 2454, 0, 325, 0, 2224, 44, 3, 1804, 0, 0, 0, 0, 0, 0, 0, 78, 0, 1, 107, 3778, 3827, 1, 696, 60, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7709, 7576, 7240, 9544, 7779, 10280, 6860, 8548] avg=8192.0000 max=10280 max_rank=5 min=6860 min_rank=6 imbalance_ratio=1.254883 max_load_reduction=55256 max_load_reduction_pct=0.843140 moved_to_echo=33467 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 45, 0, 8, 89, 0, 2909, 525, 0, 0, 0, 0, 0, 0, 0, 3980, 124, 0, 1, 1, 233, 20, 3554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3197, 0, 1510, 3355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 16, 0, 111, 1718, 0, 162, 0, 1617, 1159, 0, 0, 0, 0, 0, 0, 2220, 3892, 4, 1203, 0, 1080, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2, 1248, 3913, 4, 3, 235, 1, 1275, 673, 0, 0, 0, 0, 0, 0, 4, 2331, 2, 2379, 0, 2541, 0, 0, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 4021, 4032, 4, 43, 62, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7670, 7913, 8062, 8826, 8404, 7375, 9032, 8254] avg=8192.0000 max=9032 max_rank=6 min=7375 min_rank=5 imbalance_ratio=1.102539 max_load_reduction=56504 max_load_reduction_pct=0.862183 moved_to_echo=33065 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 2, 1991, 0, 260, 33, 19, 2889, 525, 0, 0, 0, 0, 0, 0, 0, 4062, 637, 6, 43, 6, 1038, 42, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2823, 0, 1543, 4069, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 0, 30, 1386, 0, 4, 64, 1612, 1153, 0, 0, 0, 0, 0, 0, 3014, 3785, 0, 1210, 0, 54, 34, 2, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0, 1252, 3787, 0, 0, 581, 5, 1271, 677, 0, 0, 0, 0, 0, 0, 14, 1877, 29, 1331, 52, 1087, 11, 2, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 4086, 3107, 8, 142, 60, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9802, 9395, 8439, 8135, 8099, 7949, 6198, 7519] avg=8192.0000 max=9802 max_rank=0 min=6198 min_rank=6 imbalance_ratio=1.196533 max_load_reduction=55734 max_load_reduction_pct=0.850433 moved_to_echo=29765 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 22656, 0, 12160, 30848, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32384, 2048, 128, 256, 512, 1664, 640, 28416, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[22784, 31616, 128, 9728, 128, 1536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1792, 768, 16000, 28288, 384, 128, 3200, 128, 10240, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 128, 128, 512, 31744, 30464, 128, 2176, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32000, 128, 128, 1536, 9344, 128, 512, 256, 12928, 9344, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32512, 128, 256, 4352, 128, 1024, 256, 128, 23296, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 16768, 256, 17024, 128, 17536, 128, 128, 14208, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1, 182, 3039, 0, 3316, 5, 0, 63, 0, 0, 0, 3324, 4094, 4096, 3826, 0, 39, 38, 0, 2, 4022, 170, 0, 0, 198, 31, 917, 5, 0, 4095, 4031, 0, 4090, 255, 4, 2499, 0, 4048, 0, 0, 4092, 25, 0, 0, 0, 283, 715, 1532, 0, 4, 4030, 0, 3, 0, 2, 3, 4008, 291, 4, 0, 4096, 0, 0, 58] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 82, 968, 0, 3554, 133, 0, 0, 136, 2, 1, 3992, 4090, 3094, 3850, 0, 0, 7, 0, 0, 4076, 3046, 18, 0, 0, 0, 458, 0, 0, 4033, 4069, 0, 4094, 842, 46, 0, 68, 3526, 0, 80, 4029, 19, 0, 0, 18, 6, 1558, 449, 0, 0, 4011, 10, 0, 102, 13, 1, 3955, 2925, 0, 0, 4042, 0, 97, 35] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 3065, 3779, 0, 4079, 0, 0, 104, 391, 0, 0, 487, 4095, 4095, 4093, 0, 21, 358, 0, 1, 4089, 1, 0, 0, 0, 0, 3442, 0, 10, 4090, 44, 0, 1588, 148, 0, 180, 0, 4096, 6, 0, 4096, 0, 0, 2, 0, 10, 0, 2840, 148, 9, 4060, 0, 0, 0, 6, 10, 4094, 8, 0, 0, 4090, 0, 0, 3901] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 3318, 2121, 0, 3889, 0, 0, 131, 464, 0, 0, 2281, 4095, 3850, 4070, 6, 0, 27, 0, 0, 4062, 350, 0, 0, 9, 0, 3424, 0, 0, 4084, 0, 9, 4012, 2429, 630, 6, 0, 4096, 0, 0, 4096, 3, 9, 0, 0, 0, 56, 90, 67, 1, 4067, 17, 0, 0, 0, 0, 3992, 640, 0, 6, 4065, 0, 0, 1064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 60, 3817, 5, 2432, 6, 0, 28, 37, 5, 29, 3784, 4092, 3128, 3993, 9, 15, 3289, 0, 6, 4095, 700, 0, 4, 0, 82, 1523, 0, 10, 3982, 141, 0, 4066, 1341, 0, 7, 4, 4036, 99, 0, 2415, 2318, 0, 0, 0, 0, 883, 6, 1, 0, 3255, 0, 0, 11, 47, 0, 3901, 1730, 0, 0, 3886, 0, 341, 1917] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[21, 4085, 360, 0, 4017, 0, 5, 2, 0, 0, 1, 3138, 4020, 3770, 3599, 31, 228, 3682, 0, 24, 3901, 4, 11, 289, 0, 191, 3442, 1, 1, 4095, 27, 0, 4088, 748, 1, 0, 10, 3758, 0, 9, 4037, 1, 99, 0, 7, 0, 0, 8, 0, 983, 4086, 0, 2, 89, 0, 19, 4075, 40, 0, 38, 4091, 12, 1, 389] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 3038, 470, 38, 3935, 0, 0, 0, 8, 0, 1, 3599, 4067, 4073, 3910, 80, 43, 703, 2, 0, 4074, 7, 0, 0, 297, 0, 3237, 0, 0, 3999, 3621, 0, 4041, 19, 102, 0, 0, 4014, 1, 0, 4068, 74, 37, 256, 4, 0, 530, 47, 104, 188, 4093, 33, 0, 5, 0, 27, 4096, 326, 0, 190, 4062, 0, 1, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 4044, 4084, 1, 2591, 0, 0, 1, 51, 0, 0, 3045, 4093, 4023, 4059, 12, 0, 2476, 0, 0, 2505, 0, 0, 0, 63, 0, 2259, 0, 0, 4096, 980, 0, 4094, 2, 85, 532, 47, 4096, 2, 1, 4095, 0, 0, 0, 0, 0, 6, 227, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 115, 0, 1, 4096, 0, 0, 1560] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 182, 3039, 0, 3316, 5, 0, 63, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3324, 0, 4096, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 38, 0, 2, 4022, 170, 0, 0, 1761, 665, 0, 0, 0, 0, 0, 0, 198, 31, 917, 5, 0, 4095, 4031, 0, 69, 0, 0, 0, 0, 0, 0, 0, 4090, 255, 4, 2499, 0, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 25, 0, 0, 0, 283, 715, 1532, 2547, 109, 89, 0, 0, 0, 0, 0, 0, 4, 4030, 0, 3, 0, 2, 3, 4005, 0, 0, 0, 0, 0, 0, 0, 4008, 291, 4, 0, 2226, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6691, 8699, 6697, 9346, 10077, 9392, 8047, 6587] avg=8192.0000 max=10077 max_rank=4 min=6587 min_rank=7 imbalance_ratio=1.230103 max_load_reduction=55459 max_load_reduction_pct=0.846237 moved_to_echo=34103 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3318, 2121, 0, 3889, 0, 0, 131, 86, 0, 0, 0, 0, 0, 0, 0, 464, 0, 0, 2281, 0, 3850, 1361, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 4062, 350, 0, 0, 1748, 673, 0, 0, 0, 0, 0, 0, 9, 0, 3424, 0, 0, 4084, 0, 9, 69, 0, 0, 0, 0, 0, 0, 0, 4012, 2429, 630, 6, 0, 3268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 9, 0, 0, 0, 56, 90, 2709, 108, 90, 0, 0, 0, 0, 0, 67, 1, 4067, 17, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 3992, 640, 0, 6, 2209, 0, 0, 1064, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9545, 7962, 6860, 7595, 10345, 7161, 8157, 7911] avg=8192.0000 max=10345 max_rank=4 min=6860 min_rank=2 imbalance_ratio=1.262817 max_load_reduction=55191 max_load_reduction_pct=0.842148 moved_to_echo=33574 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 82, 968, 0, 3554, 133, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 136, 2, 1, 3992, 0, 3094, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4076, 3046, 18, 0, 1738, 579, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, 4033, 4069, 0, 60, 0, 0, 0, 0, 0, 0, 0, 4094, 842, 46, 0, 68, 2813, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 19, 0, 0, 18, 6, 1558, 449, 2563, 108, 89, 0, 0, 0, 0, 0, 0, 0, 4011, 10, 0, 102, 13, 1, 4001, 0, 0, 0, 0, 0, 0, 0, 3955, 2925, 0, 0, 2196, 0, 97, 35, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4812, 8512, 9464, 8620, 7943, 8839, 8138, 9208] avg=8192.0000 max=9464 max_rank=2 min=4812 min_rank=0 imbalance_ratio=1.155273 max_load_reduction=56072 max_load_reduction_pct=0.855591 moved_to_echo=34128 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 60, 3817, 5, 2432, 6, 0, 28, 84, 0, 0, 0, 0, 0, 0, 0, 37, 5, 29, 3784, 0, 3128, 1335, 9, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3289, 0, 6, 4095, 700, 0, 4, 1671, 663, 0, 0, 0, 0, 0, 0, 0, 82, 1523, 0, 10, 3982, 141, 0, 68, 0, 0, 0, 0, 0, 0, 0, 4066, 1341, 0, 7, 4, 3221, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2415, 2318, 0, 0, 0, 0, 883, 6, 2658, 104, 90, 0, 0, 0, 0, 0, 1, 0, 3255, 0, 0, 11, 47, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 3901, 1730, 0, 0, 2111, 0, 341, 1917, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6432, 8327, 10443, 5806, 8738, 8474, 7316, 10000] avg=8192.0000 max=10443 max_rank=2 min=5806 min_rank=3 imbalance_ratio=1.274780 max_load_reduction=55093 max_load_reduction_pct=0.840652 moved_to_echo=34528 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3065, 3779, 0, 4079, 0, 0, 104, 88, 0, 0, 0, 0, 0, 0, 0, 391, 0, 0, 487, 0, 4095, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 358, 0, 1, 4089, 1, 0, 0, 1762, 675, 0, 0, 0, 0, 0, 0, 0, 0, 3442, 0, 10, 4090, 44, 0, 73, 0, 0, 0, 0, 0, 0, 0, 1588, 148, 0, 180, 0, 3260, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 10, 0, 2840, 2729, 113, 90, 0, 0, 0, 0, 0, 148, 9, 4060, 0, 0, 0, 6, 10, 4005, 0, 0, 0, 0, 0, 0, 0, 4094, 8, 0, 0, 2215, 0, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11115, 6337, 6907, 7659, 5182, 9880, 8238, 10218] avg=8192.0000 max=11115 max_rank=0 min=5182 min_rank=4 imbalance_ratio=1.356812 max_load_reduction=54421 max_load_reduction_pct=0.830399 moved_to_echo=33518 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3038, 470, 38, 3935, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 3599, 0, 4073, 1307, 80, 0, 0, 0, 0, 0, 0, 0, 0, 43, 703, 2, 0, 4074, 7, 0, 0, 1747, 659, 0, 0, 0, 0, 0, 0, 297, 0, 3237, 0, 0, 3999, 3621, 0, 68, 0, 0, 0, 0, 0, 0, 0, 4041, 19, 102, 0, 0, 3203, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 74, 37, 256, 4, 0, 530, 47, 2603, 108, 89, 0, 0, 0, 0, 0, 104, 188, 4093, 33, 0, 5, 0, 27, 3978, 0, 0, 0, 0, 0, 0, 0, 4096, 326, 0, 190, 2207, 0, 1, 16, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7565, 9068, 7235, 11222, 7366, 7816, 8428, 6836] avg=8192.0000 max=11222 max_rank=3 min=6836 min_rank=7 imbalance_ratio=1.369873 max_load_reduction=54314 max_load_reduction_pct=0.828766 moved_to_echo=30446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[21, 4085, 360, 0, 4017, 0, 5, 2, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3138, 0, 3770, 1203, 31, 0, 0, 0, 0, 0, 0, 0, 0, 228, 3682, 0, 24, 3901, 4, 11, 289, 1759, 617, 0, 0, 0, 0, 0, 0, 0, 191, 3442, 1, 1, 4095, 27, 0, 64, 0, 0, 0, 0, 0, 0, 0, 4088, 748, 1, 0, 10, 2998, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 1, 99, 0, 7, 0, 0, 8, 2396, 109, 89, 0, 0, 0, 0, 0, 0, 983, 4086, 0, 2, 89, 0, 19, 3931, 0, 0, 0, 0, 0, 0, 0, 4075, 40, 0, 38, 2223, 12, 1, 389, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8569, 8143, 10515, 7821, 7854, 6746, 9110, 6778] avg=8192.0000 max=10515 max_rank=2 min=6746 min_rank=5 imbalance_ratio=1.283569 max_load_reduction=55021 max_load_reduction_pct=0.839554 moved_to_echo=30488 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4044, 4084, 1, 2591, 0, 0, 1, 86, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 3045, 0, 4023, 1357, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2476, 0, 0, 2505, 0, 0, 0, 1761, 673, 0, 0, 0, 0, 0, 0, 63, 0, 2259, 0, 0, 4096, 980, 0, 69, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 85, 532, 47, 3268, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 6, 227, 2702, 109, 89, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 4096, 115, 0, 1, 2226, 0, 0, 1560, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10807, 8488, 7415, 7467, 8031, 7228, 8102, 7998] avg=8192.0000 max=10807 max_rank=0 min=7228 min_rank=5 imbalance_ratio=1.319214 max_load_reduction=54729 max_load_reduction_pct=0.835098 moved_to_echo=31359 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32256, 6144, 128, 256, 17664, 128, 512, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 17920, 18688, 128, 27904, 256, 128, 384, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 1280, 31744, 128, 128, 256, 128, 128, 32000, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1152, 128, 128, 23680, 0, 30208, 10496, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 10624, 128, 128, 30848, 4352, 128, 384, 13952, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[30976, 2560, 256, 384, 128, 384, 3840, 5248, 20992, 896, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[30080, 5888, 896, 3328, 256, 25344, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 384, 18816, 128, 128, 32512, 12928, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[11, 0, 4017, 4086, 1, 1405, 38, 0, 1781, 0, 0, 4, 0, 2247, 0, 1, 1742, 3989, 0, 4095, 23, 175, 0, 0, 4095, 0, 5, 11, 0, 0, 3881, 2, 3897, 1, 0, 3110, 4095, 24, 0, 4096, 0, 0, 2452, 2, 5, 0, 53, 0, 3128, 0, 0, 0, 2, 0, 556, 4096, 566, 0, 0, 7, 0, 3753, 0, 4084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 2982, 4086, 0, 4035, 14, 0, 1142, 0, 0, 0, 0, 2297, 0, 0, 0, 3660, 0, 4096, 92, 3138, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 160, 0, 2696, 4096, 0, 0, 4096, 0, 0, 1022, 1, 630, 0, 510, 0, 4052, 0, 0, 0, 0, 0, 1520, 4095, 0, 24, 0, 0, 0, 711, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 3573, 4096, 0, 2545, 2, 0, 2334, 5, 0, 0, 7, 3, 1, 0, 1087, 3724, 0, 4096, 0, 2180, 0, 11, 4096, 0, 0, 0, 0, 3, 4052, 0, 4088, 0, 0, 3431, 4093, 8, 0, 4093, 0, 0, 287, 1, 0, 0, 245, 0, 4019, 0, 0, 0, 0, 0, 832, 4091, 279, 0, 1, 0, 0, 3827, 329, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 3738, 3967, 0, 3685, 0, 39, 2398, 18, 0, 0, 2, 0, 116, 0, 0, 4096, 3, 4077, 68, 4037, 4, 1, 4087, 0, 0, 0, 0, 0, 3999, 0, 1901, 1, 0, 3935, 4053, 0, 13, 4091, 0, 0, 40, 392, 0, 14, 204, 0, 4047, 27, 0, 4, 387, 32, 3756, 4054, 12, 0, 0, 55, 0, 13, 101, 4068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4020, 4096, 0, 3692, 0, 0, 0, 566, 0, 0, 9, 1142, 370, 0, 11, 3396, 0, 3932, 0, 3183, 0, 0, 4096, 0, 0, 0, 0, 0, 4064, 1, 4081, 0, 0, 3430, 3121, 0, 0, 4096, 0, 0, 281, 591, 63, 0, 1210, 0, 4096, 0, 0, 0, 128, 10, 1452, 4009, 1554, 0, 2, 0, 2, 418, 321, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[806, 0, 1592, 3958, 2, 4014, 2150, 0, 51, 1, 26, 43, 43, 4095, 116, 0, 339, 2646, 0, 4043, 1, 168, 0, 172, 4059, 11, 0, 0, 22, 2, 4038, 51, 4085, 28, 0, 1978, 4074, 0, 0, 3998, 6, 0, 5, 1, 19, 0, 555, 0, 3814, 0, 2, 0, 0, 26, 460, 3989, 1822, 20, 0, 33, 0, 4071, 14, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[11, 0, 3285, 4096, 0, 4025, 135, 0, 143, 0, 0, 0, 0, 21, 49, 1, 0, 3340, 33, 4096, 0, 4094, 0, 0, 4044, 0, 1, 0, 0, 1, 4090, 50, 567, 192, 2, 3766, 4095, 0, 41, 3904, 1, 3, 21, 0, 1, 0, 1163, 0, 4063, 14, 3, 0, 4, 0, 4029, 4070, 0, 0, 1, 1, 1, 3980, 4, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[67, 142, 2532, 4027, 34, 3985, 1, 0, 850, 2, 1, 1, 3, 51, 0, 6, 0, 4091, 4, 4033, 0, 404, 11, 20, 4073, 6, 1, 90, 10, 0, 4077, 13, 3973, 24, 0, 1642, 4089, 0, 0, 4027, 13, 0, 1559, 3, 72, 0, 3196, 0, 3787, 0, 7, 13, 99, 32, 1551, 4093, 623, 0, 1, 0, 44, 4094, 0, 4059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 4017, 1130, 1, 1405, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1781, 0, 0, 4, 0, 2247, 0, 1, 2240, 2128, 809, 623, 0, 0, 0, 0, 1742, 3989, 0, 1967, 23, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 5, 11, 0, 0, 3881, 2, 39, 0, 0, 0, 0, 0, 0, 0, 3897, 1, 0, 3110, 1855, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2452, 2, 5, 0, 53, 0, 4096, 2333, 0, 0, 0, 0, 0, 0, 3128, 0, 0, 0, 2, 0, 556, 2470, 0, 0, 0, 0, 0, 0, 0, 0, 566, 0, 0, 7, 0, 3753, 0, 4084, 778, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6602, 9833, 7896, 8033, 8887, 8941, 6156, 9188] avg=8192.0000 max=9833 max_rank=1 min=6156 min_rank=6 imbalance_ratio=1.200317 max_load_reduction=55703 max_load_reduction_pct=0.849960 moved_to_echo=33172 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2982, 1130, 0, 4035, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 0, 0, 0, 0, 2297, 0, 0, 2240, 2128, 809, 623, 0, 0, 0, 0, 0, 3660, 0, 1968, 92, 3138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 39, 0, 0, 0, 0, 0, 0, 0, 4096, 160, 0, 2696, 1856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 1, 630, 0, 510, 0, 4096, 2333, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 1520, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 711, 0, 4095, 778, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 9239, 8858, 8229, 8808, 8592, 8041, 5608] avg=8192.0000 max=9239 max_rank=1 min=5608 min_rank=7 imbalance_ratio=1.127808 max_load_reduction=56297 max_load_reduction_pct=0.859024 moved_to_echo=31049 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3738, 1098, 0, 3685, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 2398, 18, 0, 0, 2, 0, 116, 0, 2217, 2118, 801, 604, 0, 0, 0, 0, 0, 4096, 3, 1959, 68, 4037, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 3999, 0, 39, 0, 0, 0, 0, 0, 0, 0, 1901, 1, 0, 3935, 1836, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 392, 0, 14, 204, 0, 4091, 2265, 0, 0, 0, 0, 0, 0, 4047, 27, 0, 4, 387, 32, 3756, 2444, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 55, 0, 13, 101, 4068, 770, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8561, 8274, 10168, 8125, 7686, 7006, 10697, 5019] avg=8192.0000 max=10697 max_rank=6 min=5019 min_rank=7 imbalance_ratio=1.305786 max_load_reduction=54839 max_load_reduction_pct=0.836777 moved_to_echo=30408 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4020, 1133, 0, 3692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 0, 0, 9, 1142, 370, 0, 1707, 2043, 792, 624, 0, 0, 0, 0, 11, 3396, 0, 1889, 0, 3183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4064, 1, 38, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 3430, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 591, 63, 0, 1210, 0, 4096, 2339, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 128, 10, 1452, 2418, 0, 0, 0, 0, 0, 0, 0, 0, 1554, 0, 2, 0, 2, 418, 321, 4093, 761, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8845, 7253, 8479, 8199, 8925, 8580, 8104, 7151] avg=8192.0000 max=8925 max_rank=4 min=7151 min_rank=7 imbalance_ratio=1.089478 max_load_reduction=56611 max_load_reduction_pct=0.863815 moved_to_echo=32760 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3573, 1133, 0, 2545, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 5, 0, 0, 7, 3, 1, 0, 2239, 2128, 808, 624, 0, 0, 0, 0, 1087, 3724, 0, 1968, 0, 2180, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4052, 0, 39, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 3431, 1854, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 1, 0, 0, 245, 0, 4093, 2339, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 832, 2467, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 1, 0, 0, 3827, 329, 4095, 777, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7255, 8149, 8970, 8190, 9381, 6965, 7318, 9308] avg=8192.0000 max=9381 max_rank=4 min=6965 min_rank=5 imbalance_ratio=1.145142 max_load_reduction=56155 max_load_reduction_pct=0.856857 moved_to_echo=32972 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 3285, 1123, 0, 4025, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 0, 0, 21, 49, 1, 2243, 2133, 807, 628, 0, 0, 0, 0, 0, 3340, 33, 1963, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 1, 0, 0, 1, 4090, 50, 44, 0, 0, 0, 0, 0, 0, 0, 567, 192, 2, 3766, 1852, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 21, 0, 1, 0, 1163, 0, 3904, 2345, 0, 0, 0, 0, 0, 0, 4063, 14, 3, 0, 4, 0, 4029, 2442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 3980, 4, 4095, 777, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8579, 6025, 9430, 8230, 6420, 7438, 10555, 8859] avg=8192.0000 max=10555 max_rank=6 min=6025 min_rank=1 imbalance_ratio=1.288452 max_load_reduction=54981 max_load_reduction_pct=0.838943 moved_to_echo=33272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[806, 0, 1592, 1095, 2, 4014, 2150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 1, 26, 43, 43, 4095, 116, 0, 2228, 2101, 788, 603, 0, 0, 0, 0, 339, 2646, 0, 1942, 1, 168, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 11, 0, 0, 22, 2, 4038, 51, 38, 0, 0, 0, 0, 0, 0, 0, 4085, 28, 0, 1978, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 1, 19, 0, 555, 0, 3998, 2260, 0, 0, 0, 0, 0, 0, 3814, 0, 2, 0, 0, 26, 460, 2405, 0, 0, 0, 0, 0, 0, 0, 0, 1822, 20, 0, 33, 0, 4071, 14, 4087, 758, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9659, 10095, 5268, 8221, 7937, 6844, 6707, 10805] avg=8192.0000 max=10805 max_rank=7 min=5268 min_rank=2 imbalance_ratio=1.318970 max_load_reduction=54731 max_load_reduction_pct=0.835129 moved_to_echo=32293 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[67, 142, 2532, 1113, 34, 3985, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 850, 2, 1, 1, 3, 51, 0, 6, 2236, 2096, 808, 614, 0, 0, 0, 0, 0, 4091, 4, 1937, 0, 404, 11, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 6, 1, 90, 10, 0, 4077, 13, 39, 0, 0, 0, 0, 0, 0, 0, 3973, 24, 0, 1642, 1853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1559, 3, 72, 0, 3196, 0, 4027, 2300, 0, 0, 0, 0, 0, 0, 3787, 0, 7, 13, 99, 32, 1551, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 623, 0, 1, 0, 44, 4094, 0, 4059, 777, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7874, 6668, 6467, 8309, 7492, 11170, 7958, 9598] avg=8192.0000 max=11170 max_rank=5 min=6467 min_rank=2 imbalance_ratio=1.363525 max_load_reduction=54366 max_load_reduction_pct=0.829559 moved_to_echo=36218 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8704, 640, 128, 128, 128, 9856, 768, 128, 17408, 16896, 6528, 4992, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31104, 128, 128, 128, 640, 128, 14208, 19584, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[26752, 512, 128, 24064, 14464, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 256, 25856, 8960, 128, 27392, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3200, 29056, 128, 15616, 256, 17408, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32384, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4864, 128, 128, 128, 128, 20992, 896, 32768, 6272, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 5760, 1024, 896, 128, 7168, 0, 32512, 18560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 45, 8, 0, 1, 9, 0, 1232, 0, 11, 3, 3972, 3557, 0, 0, 3981, 350, 3935, 4, 0, 204, 4020, 4043, 78, 216, 143, 0, 0, 2, 4092, 118, 0, 3711, 3230, 3897, 1, 0, 0, 4096, 4096, 2, 1, 46, 14, 4091, 0, 0, 0, 2, 32, 0, 0, 4088, 4096, 19, 0, 0, 2, 0, 19, 4069, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2, 0, 3, 2, 1, 147, 145, 4038, 3620, 0, 1, 4062, 21, 4093, 0, 0, 3892, 4092, 4092, 0, 0, 0, 2, 0, 0, 4096, 0, 6, 4037, 3874, 3998, 163, 1, 1, 4072, 4084, 431, 0, 0, 0, 122, 0, 0, 11, 0, 9, 0, 31, 4091, 4051, 38, 0, 56, 53, 1, 0, 4095, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3, 0, 0, 1473, 0, 0, 2, 25, 0, 0, 637, 4096, 3564, 0, 223, 4032, 826, 0, 0, 0, 10, 4095, 2577, 0, 0, 0, 0, 0, 0, 4096, 12, 576, 3587, 4070, 2784, 1158, 0, 5, 4096, 4096, 2, 0, 0, 0, 4092, 0, 0, 0, 86, 0, 0, 0, 3010, 4089, 0, 0, 8, 84, 3738, 296, 4085, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1078, 0, 0, 115, 4091, 3267, 0, 0, 4096, 0, 3093, 0, 0, 2446, 4096, 4094, 0, 0, 1, 0, 0, 0, 4096, 0, 313, 4096, 4096, 2111, 6, 0, 0, 4088, 4096, 6, 0, 2, 0, 3231, 0, 0, 0, 0, 1, 0, 0, 3896, 4096, 0, 0, 167, 121, 0, 614, 4096, 0, 27, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[208, 0, 79, 0, 0, 135, 0, 262, 6, 0, 1, 4094, 1857, 0, 696, 4096, 0, 1320, 0, 268, 0, 3645, 2673, 0, 0, 344, 0, 0, 1, 3957, 1420, 310, 4072, 3216, 3704, 2293, 3, 0, 4038, 4041, 14, 0, 12, 3, 127, 0, 0, 10, 0, 3916, 0, 0, 3075, 2863, 0, 0, 175, 0, 105, 979, 4062, 0, 3455, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 426, 0, 0, 0, 255, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 2002, 0, 0, 2142, 4072, 2634, 0, 0, 0, 0, 0, 0, 4096, 1, 407, 4096, 4094, 251, 3061, 0, 0, 4095, 4096, 1, 1, 0, 0, 4070, 0, 0, 0, 0, 1043, 0, 0, 4095, 4096, 0, 0, 0, 2, 52, 16, 4096, 0, 48, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 51, 0, 124, 17, 15, 0, 2927, 120, 0, 0, 3863, 333, 41, 376, 4089, 12, 3204, 0, 2, 42, 3487, 4031, 1, 0, 82, 1, 33, 0, 4091, 1197, 3500, 3653, 3508, 148, 0, 0, 47, 4090, 4017, 0, 0, 3, 0, 1477, 0, 0, 1, 8, 3975, 23, 0, 4096, 3896, 0, 0, 286, 0, 6, 570, 4090, 1, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 1, 0, 3211, 1, 728, 276, 4064, 2117, 0, 7, 4094, 0, 58, 1, 0, 56, 3920, 3196, 0, 18, 108, 0, 0, 0, 4096, 129, 13, 3976, 4063, 3043, 3, 0, 4, 4090, 4093, 0, 0, 0, 0, 4095, 1, 11, 0, 80, 3425, 0, 0, 4096, 3966, 0, 0, 107, 142, 0, 64, 4095, 0, 84, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 45, 8, 0, 1, 9, 0, 1232, 4096, 2671, 0, 0, 0, 0, 0, 0, 0, 11, 3, 3972, 3557, 0, 0, 890, 0, 0, 0, 0, 0, 0, 0, 0, 350, 3935, 4, 0, 204, 1298, 4043, 78, 0, 0, 0, 0, 0, 0, 0, 0, 216, 143, 0, 0, 2, 4092, 118, 0, 1839, 1084, 0, 0, 0, 0, 0, 0, 1036, 3230, 3897, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 46, 14, 4091, 0, 0, 0, 3091, 1425, 836, 0, 0, 0, 0, 0, 2, 32, 0, 0, 4088, 2946, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 19, 4069, 0, 0, 0, 1638, 1150, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8062, 8433, 9912, 7494, 8164, 9506, 7087, 6878] avg=8192.0000 max=9912 max_rank=2 min=6878 min_rank=7 imbalance_ratio=1.209961 max_load_reduction=55624 max_load_reduction_pct=0.848755 moved_to_echo=31635 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1078, 4088, 2672, 0, 0, 0, 0, 0, 0, 0, 0, 115, 4091, 3267, 0, 0, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3093, 0, 0, 2446, 1322, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 313, 2030, 1105, 0, 0, 0, 0, 0, 0, 1143, 4096, 2111, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 3231, 0, 0, 0, 3180, 1424, 923, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3896, 2946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 121, 0, 614, 4096, 0, 27, 0, 1669, 1150, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7838, 8389, 10955, 7545, 7356, 8766, 6843, 7844] avg=8192.0000 max=10955 max_rank=2 min=6843 min_rank=6 imbalance_ratio=1.337280 max_load_reduction=54581 max_load_reduction_pct=0.832840 moved_to_echo=30809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 1, 0, 3211, 4090, 2669, 0, 0, 0, 0, 0, 0, 1, 728, 276, 4064, 2117, 0, 7, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 1, 0, 56, 1266, 3196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 108, 0, 0, 0, 4096, 129, 13, 1971, 1057, 0, 0, 0, 0, 0, 0, 1109, 4063, 3043, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 11, 0, 3178, 1424, 896, 0, 0, 0, 0, 0, 80, 3425, 0, 0, 4096, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 142, 0, 64, 4095, 0, 84, 2, 1597, 1114, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9973, 8109, 4577, 7392, 8222, 9605, 10453, 7205] avg=8192.0000 max=10453 max_rank=6 min=4577 min_rank=2 imbalance_ratio=1.276001 max_load_reduction=55083 max_load_reduction_pct=0.840500 moved_to_echo=35485 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 1473, 0, 0, 2, 25, 4096, 2671, 0, 0, 0, 0, 0, 0, 0, 0, 637, 4096, 3564, 0, 223, 902, 0, 0, 0, 0, 0, 0, 0, 0, 826, 0, 0, 0, 10, 1321, 2577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 576, 1778, 1105, 0, 0, 0, 0, 0, 0, 1001, 4070, 2784, 1158, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4092, 0, 0, 0, 3130, 1425, 808, 0, 0, 0, 0, 0, 86, 0, 0, 0, 3010, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 84, 3738, 296, 4085, 0, 3, 0, 1669, 1148, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 9422, 4734, 7567, 9018, 9457, 6037, 11031] avg=8192.0000 max=11031 max_rank=7 min=4734 min_rank=2 imbalance_ratio=1.346558 max_load_reduction=54505 max_load_reduction_pct=0.831680 moved_to_echo=35543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2, 0, 3, 2, 4072, 2664, 0, 0, 0, 0, 0, 0, 1, 147, 145, 4038, 3620, 0, 1, 905, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4093, 0, 0, 3892, 1312, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 6, 2005, 1108, 0, 0, 0, 0, 0, 0, 1116, 3874, 3998, 163, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 0, 0, 0, 122, 0, 0, 11, 3157, 1420, 916, 0, 0, 0, 0, 0, 0, 9, 0, 31, 4091, 2909, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 53, 1, 0, 4095, 0, 0, 0, 1672, 1142, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6745, 8857, 13410, 7217, 9153, 6057, 7078, 7019] avg=8192.0000 max=13410 max_rank=2 min=6057 min_rank=5 imbalance_ratio=1.636963 max_load_reduction=52126 max_load_reduction_pct=0.795380 moved_to_echo=29307 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[208, 0, 79, 0, 0, 135, 0, 262, 4038, 2635, 0, 0, 0, 0, 0, 0, 6, 0, 1, 4094, 1857, 0, 696, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1320, 0, 268, 0, 1177, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 1, 3957, 1420, 310, 2018, 983, 0, 0, 0, 0, 0, 0, 1136, 3216, 3704, 2293, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 12, 3, 127, 0, 0, 10, 3180, 1406, 918, 0, 0, 0, 0, 0, 0, 3916, 0, 0, 3075, 2059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 105, 979, 4062, 0, 3455, 1, 1485, 804, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7357, 7570, 5438, 9033, 10352, 5670, 9050, 11066] avg=8192.0000 max=11066 max_rank=7 min=5438 min_rank=2 imbalance_ratio=1.350830 max_load_reduction=54470 max_load_reduction_pct=0.831146 moved_to_echo=36138 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 426, 0, 0, 0, 255, 4095, 2671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2002, 0, 0, 2142, 1315, 2634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 407, 2030, 1098, 0, 0, 0, 0, 0, 0, 1143, 4094, 251, 3061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4070, 0, 0, 0, 3180, 1425, 923, 0, 0, 0, 0, 0, 0, 1043, 0, 0, 4095, 2946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 52, 16, 4096, 0, 48, 0, 1659, 1150, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7447, 9108, 8093, 7632, 8549, 9600, 8084, 7023] avg=8192.0000 max=9600 max_rank=5 min=7023 min_rank=7 imbalance_ratio=1.171875 max_load_reduction=55936 max_load_reduction_pct=0.853516 moved_to_echo=33256 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 0, 124, 17, 15, 0, 2927, 4090, 2620, 0, 0, 0, 0, 0, 0, 120, 0, 0, 3863, 333, 41, 376, 915, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3204, 0, 2, 42, 1125, 4031, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 1, 33, 0, 4091, 1197, 3500, 1811, 941, 0, 0, 0, 0, 0, 0, 1019, 3508, 148, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1477, 0, 0, 1, 3174, 1397, 823, 0, 0, 0, 0, 0, 8, 3975, 23, 0, 4096, 2802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 0, 6, 570, 4090, 1, 2, 0, 1421, 1094, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9844, 5648, 8417, 11656, 4722, 6875, 10904, 7470] avg=8192.0000 max=11656 max_rank=3 min=4722 min_rank=4 imbalance_ratio=1.422852 max_load_reduction=53880 max_load_reduction_pct=0.822144 moved_to_echo=29971 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1280, 17792, 128, 384, 8832, 10240, 27392, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[512, 128, 128, 128, 21376, 128, 128, 128, 25344, 11392, 7168, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 12416, 128, 128, 30464, 22528, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[896, 512, 3968, 2560, 32768, 128, 3712, 128, 12928, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[8704, 30208, 19968, 6784, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 896, 1280, 32384, 22528, 128, 1408, 7296, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 128, 128, 2048, 128, 256, 128, 9088, 32768, 21376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 768, 128, 128, 128, 32640, 2944, 5248, 15488, 8576, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 2, 63, 22, 0, 0, 4094, 4096, 0, 4096, 0, 2, 790, 4096, 0, 4095, 4061, 0, 0, 1, 0, 0, 3675, 0, 4096, 0, 1307, 0, 2726, 0, 0, 1667, 3, 0, 0, 1, 3558, 11, 0, 3998, 0, 0, 4091, 0, 127, 46, 0, 0, 0, 0, 4091, 0, 0, 2, 2533, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4089, 9, 0, 0, 0, 5, 0, 2316, 0, 8, 1, 3227, 4095, 0, 4094, 0, 1476, 0, 4096, 0, 4090, 4095, 0, 138, 1, 0, 2, 3896, 1, 4088, 0, 46, 0, 3059, 0, 0, 3408, 0, 0, 0, 16, 62, 1, 0, 4096, 0, 6, 541, 0, 2148, 485, 0, 0, 0, 0, 4063, 0, 0, 0, 4096, 0, 0, 3782] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 1, 0, 0, 4022, 2, 4, 4, 3844, 4093, 0, 4063, 0, 0, 471, 3849, 0, 4089, 3418, 0, 16, 0, 0, 15, 4096, 74, 4085, 0, 500, 1, 1729, 0, 25, 0, 3208, 0, 166, 0, 80, 680, 6, 3211, 1, 0, 3422, 0, 35, 2, 31, 0, 1, 0, 4020, 13, 0, 0, 4068, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4020, 0, 0, 0, 0, 0, 122, 69, 8, 0, 0, 4049, 4096, 1, 4043, 0, 37, 636, 3712, 0, 4088, 4088, 0, 4, 5, 0, 0, 4096, 246, 4096, 0, 4096, 7, 1886, 0, 1, 2957, 2033, 9, 4, 0, 0, 1117, 0, 3720, 4, 0, 0, 0, 0, 9, 3, 0, 0, 6, 4094, 0, 0, 3, 4096, 0, 0, 4075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[245, 4079, 0, 0, 0, 4, 1, 0, 1904, 31, 0, 0, 2656, 4093, 12, 4023, 3, 2, 0, 4096, 115, 4075, 4096, 0, 77, 0, 1, 1, 4092, 0, 4090, 0, 172, 0, 0, 0, 0, 3970, 0, 76, 0, 0, 19, 0, 0, 4018, 17, 0, 4030, 0, 3241, 2, 0, 34, 0, 0, 4070, 0, 0, 1, 4096, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 34, 1213, 32, 0, 0, 2759, 4096, 0, 4096, 0, 0, 7, 4096, 0, 4062, 3948, 0, 0, 0, 0, 0, 4015, 2, 4096, 0, 3749, 0, 1110, 0, 1, 2, 8, 17, 0, 2, 3391, 0, 0, 3603, 0, 4, 869, 0, 3968, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 4092, 1, 0, 4043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[3, 3962, 196, 0, 18, 0, 0, 0, 4009, 12, 0, 0, 768, 4076, 3, 4064, 13, 208, 3, 4089, 0, 3951, 4095, 0, 0, 0, 75, 25, 4029, 0, 3975, 1, 4091, 0, 4096, 0, 0, 0, 31, 2, 0, 97, 3855, 9, 5, 3125, 0, 37, 664, 0, 3, 34, 56, 34, 0, 0, 4096, 0, 113, 0, 3812, 9, 0, 3792] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4019, 0, 0, 0, 0, 0, 238, 19, 230, 2, 0, 2399, 4093, 1, 4096, 0, 3, 0, 4096, 19, 4096, 4055, 0, 0, 0, 0, 11, 3886, 0, 4083, 34, 0, 68, 2666, 22, 0, 1, 2396, 22, 0, 281, 2296, 1, 1, 4095, 0, 5, 1916, 0, 3974, 168, 0, 14, 0, 0, 4069, 0, 25, 0, 4003, 0, 40, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 2, 3738, 314, 0, 0, 0, 0, 0, 0, 63, 22, 0, 0, 4094, 0, 0, 3490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 790, 3812, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3675, 0, 4096, 0, 52, 0, 0, 0, 0, 0, 0, 0, 1307, 0, 2726, 0, 0, 1667, 3, 0, 969, 606, 232, 0, 0, 0, 0, 0, 0, 1, 3558, 11, 0, 3998, 0, 0, 2501, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 127, 46, 0, 0, 0, 0, 4096, 357, 0, 0, 0, 0, 0, 0, 307, 0, 0, 2, 2533, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 7669, 8665, 7824, 7510, 10069, 8717, 6938] avg=8192.0000 max=10069 max_rank=5 min=6938 min_rank=7 imbalance_ratio=1.229126 max_load_reduction=55467 max_load_reduction_pct=0.846359 moved_to_echo=33234 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 9, 0, 0, 0, 5, 0, 3732, 312, 0, 0, 0, 0, 0, 0, 2316, 0, 8, 1, 3227, 0, 0, 3488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 0, 3812, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 1, 0, 2, 3896, 1, 4088, 0, 52, 0, 0, 0, 0, 0, 0, 0, 46, 0, 3059, 0, 0, 3408, 0, 0, 962, 606, 232, 0, 0, 0, 0, 0, 0, 16, 62, 1, 0, 4096, 0, 6, 2484, 0, 0, 0, 0, 0, 0, 0, 541, 0, 2148, 485, 0, 0, 0, 0, 4095, 358, 0, 0, 0, 0, 0, 0, 305, 0, 0, 0, 4096, 0, 0, 3782, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 9040, 9383, 8178, 8313, 6665, 7627, 8183] avg=8192.0000 max=9383 max_rank=2 min=6665 min_rank=5 imbalance_ratio=1.145386 max_load_reduction=56153 max_load_reduction_pct=0.856827 moved_to_echo=30788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[245, 4079, 0, 0, 0, 4, 1, 0, 3718, 313, 0, 0, 0, 0, 0, 0, 1904, 31, 0, 0, 2656, 0, 12, 3428, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 3812, 115, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 1, 1, 4092, 0, 4090, 0, 52, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 3970, 0, 76, 964, 595, 232, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 4018, 17, 0, 2489, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 3241, 2, 0, 34, 0, 0, 4093, 357, 0, 0, 0, 0, 0, 0, 304, 0, 0, 1, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8360, 8031, 8028, 8313, 6009, 6543, 11757, 8495] avg=8192.0000 max=11757 max_rank=6 min=6009 min_rank=4 imbalance_ratio=1.435181 max_load_reduction=53779 max_load_reduction_pct=0.820602 moved_to_echo=32804 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4020, 0, 0, 0, 0, 0, 122, 3731, 314, 0, 0, 0, 0, 0, 0, 69, 8, 0, 0, 4049, 0, 1, 3445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 636, 3455, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 4096, 246, 4096, 0, 47, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 1886, 0, 1, 2957, 2033, 9, 969, 598, 210, 0, 0, 0, 0, 0, 4, 0, 0, 1117, 0, 3720, 4, 0, 2503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 6, 4096, 357, 0, 0, 0, 0, 0, 0, 308, 0, 0, 3, 4096, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 7572, 8216, 8494, 12766, 7348, 4471, 8482] avg=8192.0000 max=12766 max_rank=4 min=4471 min_rank=6 imbalance_ratio=1.558350 max_load_reduction=52770 max_load_reduction_pct=0.805206 moved_to_echo=33067 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 1, 0, 0, 3732, 309, 0, 0, 0, 0, 0, 0, 4022, 2, 4, 4, 3844, 0, 0, 3462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 3582, 0, 0, 3418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 15, 4096, 74, 4085, 0, 49, 0, 0, 0, 0, 0, 0, 0, 500, 1, 1729, 0, 25, 0, 3208, 0, 952, 601, 218, 0, 0, 0, 0, 0, 166, 0, 80, 680, 6, 3211, 1, 0, 2458, 0, 0, 0, 0, 0, 0, 0, 3422, 0, 35, 2, 31, 0, 1, 0, 4093, 357, 0, 0, 0, 0, 0, 0, 301, 13, 0, 0, 4068, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 11338, 7471, 8335, 7234, 6602, 7941, 8478] avg=8192.0000 max=11338 max_rank=1 min=6602 min_rank=5 imbalance_ratio=1.384033 max_load_reduction=54198 max_load_reduction_pct=0.826996 moved_to_echo=30255 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 34, 3707, 314, 0, 0, 0, 0, 0, 0, 1213, 32, 0, 0, 2759, 0, 0, 3490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3812, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 2, 4096, 0, 52, 0, 0, 0, 0, 0, 0, 0, 3749, 0, 1110, 0, 1, 2, 8, 17, 970, 606, 232, 0, 0, 0, 0, 0, 0, 2, 3391, 0, 0, 3603, 0, 4, 2505, 0, 0, 0, 0, 0, 0, 0, 869, 0, 3968, 0, 0, 0, 0, 0, 4096, 355, 0, 0, 0, 0, 0, 0, 307, 0, 0, 29, 4092, 1, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 7494, 7767, 8165, 6695, 9505, 9288, 8472] avg=8192.0000 max=9505 max_rank=5 min=6695 min_rank=4 imbalance_ratio=1.160278 max_load_reduction=56031 max_load_reduction_pct=0.854965 moved_to_echo=33960 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4019, 0, 0, 0, 0, 0, 238, 3739, 317, 0, 0, 0, 0, 0, 0, 19, 230, 2, 0, 2399, 0, 1, 3486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3805, 19, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3886, 0, 4083, 34, 56, 0, 0, 0, 0, 0, 0, 0, 0, 68, 2666, 22, 0, 1, 2396, 22, 966, 610, 235, 0, 0, 0, 0, 0, 0, 281, 2296, 1, 1, 4095, 0, 5, 2492, 0, 0, 0, 0, 0, 0, 0, 1916, 0, 3974, 168, 0, 14, 0, 0, 4093, 357, 0, 0, 0, 0, 0, 0, 294, 0, 25, 0, 4003, 0, 40, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8313, 6137, 7882, 8070, 6986, 9171, 10522, 8455] avg=8192.0000 max=10522 max_rank=6 min=6137 min_rank=1 imbalance_ratio=1.284424 max_load_reduction=55014 max_load_reduction_pct=0.839447 moved_to_echo=35134 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 3962, 196, 0, 18, 0, 0, 0, 3605, 314, 0, 0, 0, 0, 0, 0, 4009, 12, 0, 0, 768, 0, 3, 3463, 0, 0, 0, 0, 0, 0, 0, 0, 13, 208, 3, 3805, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 25, 4029, 0, 3975, 1, 52, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 31, 2, 970, 601, 232, 0, 0, 0, 0, 0, 0, 97, 3855, 9, 5, 3125, 0, 37, 2505, 0, 0, 0, 0, 0, 0, 0, 664, 0, 3, 34, 56, 34, 0, 0, 4076, 346, 0, 0, 0, 0, 0, 0, 307, 0, 113, 0, 3812, 9, 0, 3792, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8255, 8124, 8157, 10023, 9633, 5213, 8033] avg=8192.0000 max=10023 max_rank=4 min=5213 min_rank=6 imbalance_ratio=1.223511 max_load_reduction=55513 max_load_reduction_pct=0.847061 moved_to_echo=32902 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2560, 128, 256, 128, 30848, 128, 128, 32128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 1792, 1920, 29952, 256, 0, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 32512, 256, 0, 128, 128, 128, 512, 29824, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[14080, 128, 17280, 128, 128, 12032, 7680, 128, 7808, 4864, 1920, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[13696, 384, 128, 128, 23808, 0, 128, 27776, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 128, 128, 128, 31872, 384, 32640, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 512, 13312, 1920, 128, 29952, 128, 128, 19968, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[15616, 0, 13568, 768, 128, 128, 128, 128, 32768, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 3, 0, 4093, 0, 1932, 4096, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 458, 4096, 900, 3350, 0, 0, 37, 5, 0, 0, 0, 0, 0, 408, 4096, 0, 46, 4090, 4073, 0, 0, 0, 3607, 0, 4096, 4096, 0, 0, 0, 0, 4091, 4094, 0, 0, 4078, 1186, 456, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4086, 11, 0, 0, 3311, 31, 4095, 4052, 0, 0, 3, 0, 3, 0, 0, 0, 1, 13, 4, 0, 63, 0, 0, 3, 2, 575, 4002, 127, 3534, 69, 2, 527, 0, 94, 0, 0, 0, 5, 1231, 4096, 2, 450, 4069, 3782, 0, 0, 0, 3420, 0, 4096, 4045, 0, 670, 3, 102, 4080, 4008, 0, 0, 3847, 2, 3015, 2, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4092, 0, 0, 6, 4096, 0, 3360, 4096, 256, 0, 26, 0, 2312, 4, 0, 0, 0, 42, 0, 102, 0, 13, 0, 0, 0, 0, 4078, 0, 2572, 0, 0, 1, 18, 858, 0, 0, 0, 0, 4056, 3650, 0, 0, 1, 4039, 0, 1, 1, 3585, 0, 4096, 4089, 0, 2086, 0, 26, 4077, 3407, 0, 0, 3975, 47, 1906, 23, 539] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 4095, 2, 4092, 4088, 34, 0, 0, 0, 29, 0, 0, 0, 0, 0, 1, 7, 0, 0, 1, 0, 0, 393, 4074, 4014, 4017, 0, 0, 1307, 0, 59, 0, 0, 1, 0, 506, 4096, 0, 5, 2038, 4081, 3, 25, 0, 79, 0, 4094, 4091, 2, 0, 0, 0, 4078, 4059, 0, 0, 4059, 514, 3477, 18, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 730, 0, 4091, 0, 3014, 4081, 0, 0, 50, 0, 12, 0, 0, 1, 0, 2, 0, 5, 0, 0, 0, 0, 0, 3154, 4096, 6, 4092, 0, 0, 0, 1672, 2, 0, 0, 0, 1, 287, 4066, 0, 3, 4081, 1618, 1, 0, 0, 4030, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4092, 0, 0, 4096, 27, 1843, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4081, 0, 877, 0, 4056, 0, 4095, 3864, 0, 0, 0, 8, 120, 9, 14, 0, 335, 1950, 0, 21, 114, 0, 0, 2, 0, 3329, 4096, 3293, 3808, 0, 0, 0, 34, 1, 12, 0, 0, 0, 1019, 2387, 0, 21, 2461, 4093, 0, 0, 0, 429, 0, 4023, 4074, 0, 0, 162, 7, 4093, 3818, 44, 0, 4089, 1, 0, 694, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3555, 2, 0, 0, 3612, 0, 4080, 4072, 369, 85, 0, 0, 0, 0, 17, 0, 23, 21, 0, 456, 0, 194, 195, 17, 0, 0, 4074, 22, 3642, 0, 0, 262, 181, 1, 73, 50, 128, 0, 3684, 4096, 0, 0, 334, 3817, 167, 53, 2, 2286, 635, 4094, 4053, 0, 0, 0, 0, 4029, 4063, 1, 0, 2097, 3831, 2831, 290, 42] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 27, 0, 4095, 1, 67, 4004, 2, 0, 11, 3, 0, 0, 0, 0, 5, 649, 0, 0, 0, 312, 1, 22, 0, 4068, 4090, 1170, 4095, 11, 0, 0, 302, 0, 17, 3, 0, 0, 354, 3898, 0, 9, 4060, 3893, 0, 10, 0, 1858, 86, 4095, 4095, 0, 0, 1, 0, 4034, 4077, 0, 36, 4095, 3, 3836, 45, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[975, 0, 1, 0, 4095, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 29, 0, 0, 0, 3121, 2712, 1045, 516, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 1, 0, 4094, 3572, 0, 0, 0, 0, 0, 0, 0, 393, 1587, 4014, 4017, 0, 0, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 1, 0, 506, 4096, 1442, 706, 433, 0, 0, 0, 0, 0, 0, 5, 2038, 3375, 3, 25, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 2, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 1347, 0, 0, 4059, 514, 3477, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9165, 7457, 7675, 11318, 7243, 5525, 7738, 9415] avg=8192.0000 max=11318 max_rank=3 min=5525 min_rank=5 imbalance_ratio=1.381592 max_load_reduction=54218 max_load_reduction_pct=0.827301 moved_to_echo=29921 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[975, 0, 3, 0, 4093, 0, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 3121, 2736, 1050, 518, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 3578, 0, 0, 0, 0, 0, 0, 0, 458, 1597, 900, 3350, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 408, 4096, 1449, 704, 434, 0, 0, 0, 0, 0, 0, 46, 4090, 3369, 0, 0, 0, 3607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3662, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 0, 0, 4078, 1186, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7003, 11473, 7679, 6342, 7096, 11112, 7753, 7078] avg=8192.0000 max=11473 max_rank=1 min=6342 min_rank=3 imbalance_ratio=1.400513 max_load_reduction=54063 max_load_reduction_pct=0.824936 moved_to_echo=33039 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[973, 11, 0, 0, 3311, 31, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 3113, 2678, 1026, 511, 0, 0, 0, 0, 1, 13, 4, 0, 63, 0, 0, 3, 4096, 3541, 0, 0, 0, 0, 0, 0, 2, 575, 1560, 127, 3534, 69, 2, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 5, 1231, 4096, 1416, 654, 428, 0, 0, 0, 0, 0, 2, 450, 4069, 3128, 0, 0, 0, 3420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3617, 0, 670, 3, 102, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 1330, 0, 0, 3847, 2, 3015, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8421, 7334, 7721, 6396, 7924, 11069, 8472, 8199] avg=8192.0000 max=11069 max_rank=5 min=6396 min_rank=3 imbalance_ratio=1.351196 max_load_reduction=54467 max_load_reduction_pct=0.831100 moved_to_echo=35664 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[974, 0, 0, 6, 4096, 0, 3360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 26, 0, 2312, 4, 0, 0, 3118, 2277, 1046, 517, 0, 0, 0, 0, 0, 42, 0, 102, 0, 13, 0, 0, 4096, 3579, 0, 0, 0, 0, 0, 0, 0, 0, 1589, 0, 2572, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 18, 858, 0, 0, 0, 0, 4056, 3650, 1443, 698, 433, 0, 0, 0, 0, 0, 0, 0, 1, 3341, 0, 1, 1, 3585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 0, 2086, 0, 26, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 1130, 0, 0, 3975, 47, 1906, 23, 539, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8436, 9556, 7832, 4162, 11156, 6929, 9845, 7620] avg=8192.0000 max=11156 max_rank=4 min=4162 min_rank=3 imbalance_ratio=1.361816 max_load_reduction=54380 max_load_reduction_pct=0.829773 moved_to_echo=35550 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[972, 0, 877, 0, 4056, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 120, 9, 14, 0, 3109, 2551, 1050, 488, 0, 0, 0, 0, 335, 1950, 0, 21, 114, 0, 0, 2, 4023, 3376, 0, 0, 0, 0, 0, 0, 0, 3329, 1597, 3293, 3808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 1, 12, 0, 0, 0, 1019, 2387, 1449, 708, 432, 0, 0, 0, 0, 0, 0, 21, 2461, 3385, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3642, 0, 0, 162, 7, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1267, 44, 0, 4089, 1, 0, 694, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10000, 7349, 9821, 12027, 6042, 6296, 7904, 6097] avg=8192.0000 max=12027 max_rank=3 min=6042 min_rank=4 imbalance_ratio=1.468140 max_load_reduction=53509 max_load_reduction_pct=0.816483 moved_to_echo=26339 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[971, 0, 27, 0, 4095, 1, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 11, 3, 0, 0, 0, 0, 3124, 2729, 1054, 506, 0, 0, 0, 0, 5, 649, 0, 0, 0, 312, 1, 22, 4095, 3498, 0, 0, 0, 0, 0, 0, 0, 4068, 1585, 1170, 4095, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 17, 3, 0, 0, 354, 3898, 1451, 678, 438, 0, 0, 0, 0, 0, 0, 9, 4060, 3215, 0, 10, 0, 1858, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 3657, 0, 0, 1, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 1348, 0, 36, 4095, 3, 3836, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5161, 7429, 8582, 10929, 7141, 9152, 7778, 9364] avg=8192.0000 max=10929 max_rank=3 min=5161 min_rank=0 imbalance_ratio=1.334106 max_load_reduction=54607 max_load_reduction_pct=0.833237 moved_to_echo=33435 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[846, 2, 0, 0, 3612, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 85, 0, 0, 0, 0, 17, 0, 2709, 2715, 1045, 515, 0, 0, 0, 0, 23, 21, 0, 456, 0, 194, 195, 17, 4094, 3557, 0, 0, 0, 0, 0, 0, 0, 0, 1587, 22, 3642, 0, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 181, 1, 73, 50, 128, 0, 3684, 4096, 1442, 660, 429, 0, 0, 0, 0, 0, 0, 0, 334, 3157, 167, 53, 2, 2286, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 3624, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 1348, 1, 0, 2097, 3831, 2831, 290, 42, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8540, 7455, 8557, 5513, 10744, 5999, 8288, 10440] avg=8192.0000 max=10744 max_rank=4 min=5513 min_rank=3 imbalance_ratio=1.311523 max_load_reduction=54792 max_load_reduction_pct=0.836060 moved_to_echo=35471 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[975, 0, 730, 0, 4091, 0, 3014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 12, 0, 0, 1, 3120, 2735, 1050, 515, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 0, 4096, 3566, 0, 0, 0, 0, 0, 0, 0, 3154, 1597, 6, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1672, 2, 0, 0, 0, 1, 287, 4066, 1449, 279, 434, 0, 0, 0, 0, 0, 0, 3, 4081, 1339, 1, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3662, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1357, 0, 0, 4096, 27, 1843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8810, 7483, 7669, 8849, 8190, 9454, 7758, 7323] avg=8192.0000 max=9454 max_rank=5 min=7323 min_rank=7 imbalance_ratio=1.154053 max_load_reduction=56082 max_load_reduction_pct=0.855743 moved_to_echo=32725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[768, 128, 128, 128, 6528, 128, 128, 128, 24576, 21248, 8448, 4096, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 12032, 12800, 9600, 29184, 128, 128, 2176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 640, 21248, 24320, 256, 128, 128, 19328, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[768, 0, 29184, 128, 2816, 256, 256, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7680, 128, 1664, 128, 31488, 128, 24832, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[10496, 128, 128, 30336, 5632, 17408, 1152, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 2688, 128, 640, 256, 640, 256, 128, 32768, 28288, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[2304, 1024, 128, 128, 256, 128, 11648, 30464, 11648, 5120, 3584, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 1326, 13, 895, 0, 4089, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 275, 1, 3775, 0, 3964, 0, 0, 0, 3907, 2, 2914, 0, 0, 4096, 0, 0, 4095, 4092, 11, 90, 3167, 0, 4096, 15, 4096, 0, 0, 0, 0, 0, 2302, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1949, 4096, 4059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[20, 4096, 3, 1119, 1, 0, 4007, 27, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1211, 4029, 1901, 1, 401, 0, 0, 0, 4067, 6, 1956, 0, 0, 2273, 0, 0, 3999, 4037, 8, 2907, 1906, 6, 4090, 37, 3865, 0, 15, 0, 34, 27, 3574, 3873, 258, 0, 0, 0, 3, 73, 1, 4096, 0, 20, 0, 18, 472, 4034, 3063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[28, 4095, 370, 0, 1386, 0, 4087, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 3293, 4090, 61, 12, 1049, 0, 0, 2, 4083, 0, 525, 11, 0, 3586, 9, 0, 4047, 4062, 0, 2458, 2, 0, 4096, 0, 3949, 10, 0, 0, 0, 4, 3361, 474, 0, 0, 0, 22, 0, 0, 0, 4087, 0, 0, 0, 14, 4077, 4090, 4089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 3798, 23, 0, 3293, 0, 3702, 0, 304, 0, 4, 13, 0, 0, 27, 46, 0, 881, 4084, 3533, 58, 0, 40, 262, 0, 3345, 775, 0, 10, 749, 3303, 30, 36, 3415, 4030, 16, 4085, 0, 1, 4072, 132, 3864, 0, 0, 0, 0, 0, 414, 4094, 50, 0, 60, 0, 65, 1, 111, 4028, 1, 0, 1, 0, 974, 4046, 3760] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 4094, 0, 561, 1653, 0, 4096, 2, 0, 0, 0, 2, 0, 33, 3, 0, 0, 914, 74, 0, 40, 3730, 0, 0, 0, 4044, 0, 0, 0, 0, 589, 0, 3, 3771, 4096, 0, 4023, 3756, 0, 4096, 1, 4096, 409, 0, 0, 0, 0, 4090, 2823, 56, 0, 0, 339, 0, 0, 1, 4093, 0, 1, 0, 115, 3937, 4096, 1899] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 24, 1536, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 3825, 2524, 1, 598, 0, 0, 0, 4096, 0, 0, 1, 0, 4095, 0, 0, 3977, 4096, 128, 843, 4054, 0, 4096, 123, 4096, 0, 0, 0, 0, 0, 2693, 40, 87, 0, 0, 0, 0, 0, 0, 4096, 0, 101, 0, 0, 84, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 4092, 1, 3813, 137, 15, 3839, 3, 0, 0, 16, 0, 2, 7, 0, 0, 0, 4082, 20, 4010, 1, 4035, 0, 0, 0, 3270, 343, 930, 0, 0, 4096, 75, 0, 4095, 3965, 0, 0, 455, 0, 4017, 3, 4096, 8, 4, 0, 0, 1, 3505, 825, 0, 2, 26, 2, 3, 0, 4, 3861, 0, 0, 0, 66, 510, 4096, 3205] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4, 4066, 0, 317, 3498, 0, 4015, 0, 0, 5, 4, 2, 5, 0, 59, 0, 2, 726, 869, 3507, 9, 4071, 5, 26, 0, 4090, 0, 499, 0, 6, 4094, 0, 26, 4091, 4036, 24, 3932, 89, 1, 4061, 3, 4088, 0, 0, 0, 0, 0, 880, 0, 461, 2, 0, 0, 0, 0, 0, 4092, 2, 0, 2, 0, 1723, 4048, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[20, 1710, 3, 1119, 1, 0, 4007, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4090, 4004, 0, 0, 0, 0, 0, 0, 0, 1211, 4029, 1469, 1, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 6, 1956, 0, 0, 2273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 252, 8, 2907, 1906, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 3865, 0, 15, 0, 34, 27, 3574, 884, 432, 44, 0, 0, 0, 0, 0, 3873, 258, 0, 0, 0, 3, 73, 1, 3785, 2386, 282, 30, 0, 0, 0, 0, 2930, 0, 20, 0, 18, 472, 0, 3063, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6887, 8096, 7111, 8258, 9078, 8912, 10691, 6503] avg=8192.0000 max=10691 max_rank=6 min=6503 min_rank=7 imbalance_ratio=1.305054 max_load_reduction=54845 max_load_reduction_pct=0.836868 moved_to_echo=35184 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[28, 1710, 370, 0, 1386, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 4096, 4060, 0, 0, 0, 0, 0, 0, 0, 3293, 4090, 48, 12, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4038, 0, 525, 11, 0, 3586, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 254, 0, 2458, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 10, 0, 0, 0, 4, 3361, 882, 13, 45, 0, 0, 0, 0, 0, 474, 0, 0, 0, 22, 0, 0, 0, 3808, 2385, 282, 30, 0, 0, 0, 0, 2923, 0, 0, 0, 14, 4077, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7581, 8163, 8492, 8171, 6761, 8264, 7001, 11103] avg=8192.0000 max=11103 max_rank=7 min=6761 min_rank=4 imbalance_ratio=1.355347 max_load_reduction=54433 max_load_reduction_pct=0.830582 moved_to_echo=33129 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1710, 1326, 13, 895, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 275, 1, 2917, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 2, 2914, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 255, 11, 90, 3167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 2302, 884, 858, 43, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 3837, 2386, 282, 29, 0, 0, 0, 0, 2930, 0, 0, 0, 0, 1949, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8033, 8167, 7157, 10876, 7618, 8198, 6549, 8938] avg=8192.0000 max=10876 max_rank=3 min=6549 min_rank=6 imbalance_ratio=1.327637 max_load_reduction=54660 max_load_reduction_pct=0.834045 moved_to_echo=31303 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1709, 0, 561, 1653, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 33, 3, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 914, 74, 0, 40, 3730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 0, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3771, 256, 0, 4023, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 409, 0, 0, 0, 0, 4090, 883, 0, 44, 0, 0, 0, 0, 0, 2823, 56, 0, 0, 339, 0, 0, 1, 3840, 2385, 282, 29, 0, 0, 0, 0, 2928, 0, 1, 0, 115, 3937, 0, 1899, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8021, 8201, 4758, 4589, 11809, 9523, 9755, 8880] avg=8192.0000 max=11809 max_rank=4 min=4589 min_rank=3 imbalance_ratio=1.441528 max_load_reduction=53727 max_load_reduction_pct=0.819809 moved_to_echo=39967 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1708, 1, 3813, 137, 15, 3839, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 2, 7, 0, 0, 4017, 4067, 0, 0, 0, 0, 0, 0, 0, 4082, 20, 3098, 1, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3234, 343, 930, 0, 0, 4096, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 247, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 8, 4, 0, 0, 1, 3505, 833, 912, 36, 0, 0, 0, 0, 0, 825, 0, 2, 26, 2, 3, 0, 4, 3718, 2384, 266, 29, 0, 0, 0, 0, 2762, 0, 0, 0, 66, 510, 0, 3205, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9516, 8109, 11236, 8678, 4797, 9398, 7259, 6543] avg=8192.0000 max=11236 max_rank=2 min=4797 min_rank=4 imbalance_ratio=1.371582 max_load_reduction=54300 max_load_reduction_pct=0.828552 moved_to_echo=27997 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1710, 24, 1536, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 3938, 3825, 1950, 1, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 256, 128, 843, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 4096, 0, 0, 0, 0, 0, 2693, 884, 574, 45, 0, 0, 0, 0, 0, 40, 87, 0, 0, 0, 0, 0, 0, 3840, 2386, 282, 30, 0, 0, 0, 0, 2930, 0, 101, 0, 0, 84, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7366, 8162, 10312, 8147, 9258, 8415, 6665, 7211] avg=8192.0000 max=10312 max_rank=2 min=6665 min_rank=6 imbalance_ratio=1.258789 max_load_reduction=55224 max_load_reduction_pct=0.842651 moved_to_echo=31549 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1586, 23, 0, 3293, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 4, 13, 0, 0, 27, 46, 4072, 4017, 0, 0, 0, 0, 0, 0, 0, 881, 4084, 2730, 58, 0, 40, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3309, 775, 0, 10, 749, 3303, 30, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3415, 252, 16, 4085, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 3864, 0, 0, 0, 0, 0, 414, 869, 803, 36, 0, 0, 0, 0, 0, 4094, 50, 0, 60, 0, 65, 1, 111, 3778, 2212, 277, 29, 0, 0, 0, 0, 2882, 1, 0, 1, 0, 974, 0, 3760, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8604, 8483, 8055, 8176, 7805, 6118, 10677, 7618] avg=8192.0000 max=10677 max_rank=6 min=6118 min_rank=5 imbalance_ratio=1.303345 max_load_reduction=54859 max_load_reduction_pct=0.837082 moved_to_echo=32218 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4, 1694, 0, 317, 3498, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 2, 5, 0, 59, 0, 4061, 4019, 0, 0, 0, 0, 0, 0, 2, 726, 869, 2707, 9, 4071, 5, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 499, 0, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4091, 247, 24, 3932, 89, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4088, 0, 0, 0, 0, 0, 880, 889, 800, 48, 0, 0, 0, 0, 0, 0, 461, 2, 0, 0, 0, 0, 0, 3789, 2372, 286, 29, 0, 0, 0, 0, 2917, 2, 0, 2, 0, 1723, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9528, 8155, 8415, 8641, 8410, 6708, 6939, 8740] avg=8192.0000 max=9528 max_rank=0 min=6708 min_rank=5 imbalance_ratio=1.163086 max_load_reduction=56008 max_load_reduction_pct=0.854614 moved_to_echo=30797 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23296, 128, 128, 128, 256, 13824, 0, 28288, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 15360, 17024, 14976, 128, 17920, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12160, 1024, 128, 128, 384, 128, 128, 128, 30464, 18944, 2304, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 30592, 1152, 6912, 128, 768, 26240, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 32256, 512, 128, 0, 128, 128, 20864, 7040, 4480, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 13568, 1792, 7424, 10880, 128, 32000, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 31616, 2048, 256, 18432, 13440, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 128, 128, 128, 128, 128, 128, 128, 32640, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 2879, 4096, 4045, 0, 85, 8, 34, 20, 0, 0, 3821, 0, 0, 200, 0, 0, 0, 60, 3950, 0, 4096, 3754, 0, 3332, 0, 0, 0, 4094, 0, 3598, 0, 2, 4081, 0, 0, 1378, 0, 3695, 56, 4071, 0, 6, 1, 4017, 0, 4089, 5, 3, 0, 180, 0, 364, 0, 4029, 208, 70, 0, 0, 0, 1209, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[16, 4058, 4096, 4061, 1, 0, 0, 4, 0, 1, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 3039, 0, 4096, 3001, 0, 4073, 7, 0, 0, 4093, 0, 4091, 0, 0, 4093, 0, 0, 3812, 0, 2, 0, 4083, 0, 0, 0, 4096, 0, 4091, 3, 0, 0, 0, 0, 72, 0, 4094, 1663, 0, 3, 0, 0, 779, 1, 3, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3, 4093, 4096, 4090, 0, 17, 46, 1, 0, 0, 0, 4049, 0, 0, 0, 12, 0, 0, 0, 2106, 0, 4096, 2830, 46, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 3865, 0, 1, 4035, 0, 2953, 0, 4093, 0, 0, 0, 4090, 0, 3703, 384, 2, 0, 0, 0, 0, 0, 4096, 271, 0, 0, 271, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 3839, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 96, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 2097, 0, 4093, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3280, 0, 2954, 0, 4080, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 39, 4096, 5, 0, 0, 0, 0, 18, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 457, 0, 4093, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4095, 0, 3, 0, 4091, 0, 4096, 0, 0, 0, 0, 1, 6, 0, 4096, 131, 0, 0, 0, 0, 3518, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 3604, 4093, 4075, 1, 0, 0, 43, 0, 0, 5, 3762, 921, 0, 1, 0, 0, 0, 0, 4088, 13, 4001, 142, 24, 4095, 0, 0, 0, 4096, 0, 4041, 2, 0, 4093, 0, 0, 3077, 3, 168, 0, 3872, 5, 0, 3, 4043, 427, 4083, 0, 0, 7, 0, 1, 0, 441, 4095, 3875, 87, 0, 98, 0, 130, 0, 21, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 136, 4096, 3605, 1, 4, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4003, 4, 4059, 4048, 11, 4063, 39, 37, 0, 4096, 0, 4084, 7, 0, 4096, 0, 1, 3812, 26, 4082, 33, 3958, 0, 521, 0, 3894, 22, 4095, 18, 0, 1, 0, 5, 0, 0, 4084, 176, 0, 0, 0, 192, 100, 29, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 4063, 3630, 0, 0, 0, 73, 0, 23, 0, 4089, 25, 10, 0, 0, 0, 0, 0, 1238, 16, 4008, 2496, 0, 3968, 127, 1, 0, 4052, 0, 4096, 0, 0, 3910, 0, 232, 4079, 0, 4095, 32, 4075, 0, 0, 26, 4074, 31, 3930, 241, 1, 0, 4, 2, 4, 8, 4060, 123, 571, 0, 10, 0, 4090, 0, 0, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[16, 4058, 1864, 4061, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 2, 0, 0, 2232, 1189, 605, 0, 0, 0, 0, 0, 0, 0, 0, 3039, 0, 2510, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 7, 0, 0, 164, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1968, 0, 0, 3812, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 4094, 1663, 1586, 1520, 84, 0, 0, 0, 0, 0, 0, 3, 0, 0, 779, 1, 3, 6, 4096, 2740, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10004, 8125, 8550, 8335, 5782, 8093, 9019, 7628] avg=8192.0000 max=10004 max_rank=0 min=5782 min_rank=4 imbalance_ratio=1.221191 max_load_reduction=55532 max_load_reduction_pct=0.847351 moved_to_echo=30522 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2879, 1864, 4045, 0, 85, 8, 34, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 3821, 0, 0, 200, 0, 2232, 1189, 603, 0, 0, 0, 0, 0, 0, 0, 60, 3950, 0, 2510, 3754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3332, 0, 0, 0, 164, 0, 3598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1962, 0, 0, 1378, 0, 3695, 56, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 6, 1, 0, 0, 4089, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 180, 0, 364, 0, 4029, 208, 1586, 1516, 84, 0, 0, 0, 0, 0, 70, 0, 0, 0, 1209, 0, 0, 0, 4017, 2741, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8915, 8065, 10274, 7094, 7093, 8088, 7970, 8037] avg=8192.0000 max=10274 max_rank=2 min=7093 min_rank=4 imbalance_ratio=1.254150 max_load_reduction=55262 max_load_reduction_pct=0.843231 moved_to_echo=31188 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3839, 1864, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 96, 0, 0, 0, 2232, 1190, 605, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2510, 2097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 164, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 0, 0, 3280, 0, 2954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4096, 5, 1586, 1522, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4095, 2742, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9799, 8205, 8697, 8353, 8203, 8092, 7332, 6855] avg=8192.0000 max=9799 max_rank=0 min=6855 min_rank=7 imbalance_ratio=1.196167 max_load_reduction=55737 max_load_reduction_pct=0.850479 moved_to_echo=30482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 4093, 1864, 4090, 0, 17, 46, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 12, 2232, 1190, 571, 0, 0, 0, 0, 0, 0, 0, 0, 2106, 0, 2510, 2830, 46, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 164, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1858, 0, 1, 4035, 0, 2953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 3703, 384, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 271, 1586, 1436, 84, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 0, 0, 4090, 2742, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10114, 8054, 7492, 8355, 8847, 8096, 7475, 7103] avg=8192.0000 max=10114 max_rank=0 min=7103 min_rank=7 imbalance_ratio=1.234619 max_load_reduction=55422 max_load_reduction_pct=0.845673 moved_to_echo=31521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1864, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2232, 1190, 605, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 2510, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 164, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 4096, 131, 1586, 1522, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3518, 0, 0, 0, 4091, 2742, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5962, 8122, 7051, 8353, 10161, 8110, 7426, 10351] avg=8192.0000 max=10351 max_rank=7 min=5962 min_rank=0 imbalance_ratio=1.263550 max_load_reduction=55185 max_load_reduction_pct=0.842056 moved_to_echo=36048 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 136, 1864, 3605, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 2232, 1190, 605, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 4, 2487, 4048, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 39, 37, 0, 164, 0, 4084, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 0, 1, 3812, 26, 4082, 33, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 0, 521, 0, 0, 22, 4095, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 4084, 176, 1572, 1522, 82, 0, 0, 0, 0, 0, 0, 0, 0, 192, 100, 29, 0, 0, 3894, 2742, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5610, 8125, 10553, 8394, 9923, 8532, 7442, 6957] avg=8192.0000 max=10553 max_rank=2 min=5610 min_rank=0 imbalance_ratio=1.288208 max_load_reduction=54983 max_load_reduction_pct=0.838974 moved_to_echo=32854 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3604, 1857, 4075, 1, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3762, 921, 0, 1, 0, 2236, 1191, 608, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 13, 2446, 142, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 160, 0, 4041, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1961, 0, 0, 3077, 3, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3788, 5, 0, 3, 0, 427, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 441, 4095, 3875, 1555, 1524, 84, 0, 0, 0, 0, 0, 87, 0, 98, 0, 130, 0, 21, 0, 4043, 2745, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9580, 8724, 6713, 8298, 5209, 8306, 11582, 7124] avg=8192.0000 max=11582 max_rank=6 min=5209 min_rank=4 imbalance_ratio=1.413818 max_load_reduction=53954 max_load_reduction_pct=0.823273 moved_to_echo=32221 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1849, 3630, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4089, 25, 10, 0, 0, 2214, 1177, 578, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 16, 2456, 2496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 127, 1, 0, 162, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1880, 0, 232, 4079, 0, 4095, 32, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 26, 0, 31, 3930, 241, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 2, 4, 8, 4060, 123, 1552, 1452, 84, 0, 0, 0, 0, 0, 571, 0, 10, 0, 4090, 0, 0, 23, 4074, 2713, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5552, 8116, 6206, 8354, 10318, 8219, 7290, 11481] avg=8192.0000 max=11481 max_rank=7 min=5552 min_rank=0 imbalance_ratio=1.401489 max_load_reduction=54055 max_load_reduction_pct=0.824814 moved_to_echo=37308 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 18688, 14976, 31744, 128, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 128, 384, 256, 9856, 128, 128, 128, 32512, 22016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 15616, 0, 256, 27648, 128, 22144, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31744, 128, 640, 128, 0, 512, 32256, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32128, 1152, 128, 256, 128, 17920, 9600, 4864, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 26624, 128, 19968, 18944, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31872, 256, 128, 0, 1408, 0, 32256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 256, 128, 512, 512, 32768, 6528, 12672, 12032, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[44, 4096, 0, 0, 71, 339, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 39, 0, 0, 4095, 0, 0, 3653, 0, 0, 4096, 0, 0, 4096, 4096, 4003, 0, 0, 4096, 0, 0, 343, 4096, 4096, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 72, 4096, 4092, 4096, 0, 3711, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3719, 4088, 40, 0, 149, 0, 10, 541, 3913, 0, 0, 0, 0, 0, 5, 1, 2252, 0, 25, 0, 269, 4096, 0, 2, 3452, 0, 0, 4055, 0, 0, 3834, 4087, 4096, 0, 2, 4095, 0, 0, 2312, 4096, 4091, 0, 1, 0, 0, 0, 10, 0, 0, 29, 0, 0, 0, 0, 1, 4096, 4067, 4089, 0, 12, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[77, 4091, 0, 0, 1072, 0, 7, 445, 3780, 35, 0, 0, 0, 288, 0, 0, 2981, 8, 0, 0, 19, 4087, 5, 0, 1889, 340, 0, 4078, 0, 82, 4022, 3926, 3869, 0, 1, 4079, 0, 0, 3969, 4096, 4073, 0, 0, 0, 0, 39, 0, 2, 26, 1, 0, 13, 26, 1, 1572, 4096, 3966, 4074, 0, 384, 17, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1022, 4096, 0, 1, 84, 0, 3, 380, 3715, 0, 0, 0, 0, 1394, 0, 0, 3384, 0, 1, 0, 0, 4096, 0, 2, 4092, 0, 0, 4090, 0, 0, 4047, 4096, 3601, 0, 0, 4096, 15, 0, 2623, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 0, 0, 0, 9, 4096, 44, 4096, 0, 3876, 0, 43, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4024, 0, 0, 4, 21, 0, 3976, 2997, 0, 0, 1, 82, 0, 0, 0, 3854, 0, 703, 0, 0, 4063, 52, 0, 0, 334, 0, 3578, 0, 7, 3856, 4080, 3718, 3, 0, 4057, 175, 0, 178, 3903, 3968, 0, 0, 0, 14, 30, 0, 111, 306, 30, 0, 25, 0, 0, 4096, 4016, 4086, 4048, 78, 991, 0, 70, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[13, 4096, 0, 0, 2, 914, 0, 1472, 4096, 0, 1, 0, 0, 6, 0, 0, 4092, 0, 0, 0, 94, 4096, 0, 0, 3218, 0, 0, 4028, 0, 0, 4070, 4096, 4096, 0, 0, 4096, 0, 0, 2018, 4096, 4096, 0, 0, 0, 0, 0, 1, 1, 7, 0, 0, 0, 0, 0, 32, 4096, 4096, 4096, 0, 511, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[45, 4082, 0, 0, 162, 253, 0, 0, 3797, 0, 0, 1, 0, 305, 18, 12, 4067, 0, 0, 1, 0, 4054, 1, 18, 3647, 0, 0, 4094, 5, 9, 3915, 4095, 4027, 0, 2, 4095, 7, 0, 0, 4079, 4059, 0, 14, 2, 2, 14, 283, 0, 0, 0, 0, 0, 63, 0, 2008, 4023, 2379, 4083, 0, 3811, 0, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4092, 0, 130, 0, 75, 1, 578, 3941, 20, 14, 27, 0, 264, 0, 18, 4078, 0, 1, 1, 9, 4030, 0, 0, 3309, 0, 0, 4061, 0, 0, 2, 4096, 4071, 0, 19, 4095, 0, 0, 242, 4050, 4096, 0, 0, 37, 2, 0, 0, 1, 58, 0, 0, 2, 0, 0, 4096, 4004, 3831, 4096, 0, 4081, 0, 8, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3719, 4088, 40, 0, 149, 0, 10, 541, 1216, 945, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 5, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 2252, 0, 25, 0, 269, 4096, 0, 2, 360, 0, 0, 0, 0, 0, 0, 0, 3452, 0, 0, 1727, 0, 0, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 2312, 2791, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 0, 0, 0, 10, 0, 4062, 1, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 1, 4096, 2328, 197, 25, 0, 0, 0, 0, 0, 4067, 2676, 0, 12, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10708, 8013, 7004, 9013, 9201, 8165, 6676, 6756] avg=8192.0000 max=10708 max_rank=0 min=6676 min_rank=6 imbalance_ratio=1.307129 max_load_reduction=54828 max_load_reduction_pct=0.836609 moved_to_echo=30798 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[44, 4096, 0, 0, 71, 339, 0, 0, 1218, 945, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 39, 0, 0, 4095, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 3653, 0, 0, 1744, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 343, 2791, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 16, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 4096, 2352, 198, 24, 0, 0, 0, 0, 0, 4092, 2680, 0, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6713, 8194, 8590, 9493, 7137, 8184, 6742, 10483] avg=8192.0000 max=10483 max_rank=7 min=6713 min_rank=0 imbalance_ratio=1.279663 max_load_reduction=55053 max_load_reduction_pct=0.840042 moved_to_echo=32546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[77, 4091, 0, 0, 1072, 0, 7, 445, 1211, 945, 0, 0, 0, 0, 0, 0, 3780, 35, 0, 0, 0, 288, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 2981, 8, 0, 0, 19, 4087, 5, 0, 360, 0, 0, 0, 0, 0, 0, 0, 1889, 340, 0, 1737, 0, 82, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3869, 0, 1, 0, 0, 0, 3969, 2791, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 39, 0, 2, 3903, 0, 0, 0, 0, 0, 0, 0, 26, 1, 0, 13, 26, 1, 1572, 4096, 2341, 197, 23, 0, 0, 0, 0, 0, 3966, 2666, 0, 384, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7848, 8182, 7460, 8070, 10630, 8017, 8296, 7033] avg=8192.0000 max=10630 max_rank=4 min=7033 min_rank=7 imbalance_ratio=1.297607 max_load_reduction=54906 max_load_reduction_pct=0.837799 moved_to_echo=33976 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1022, 4096, 0, 1, 84, 0, 3, 380, 1218, 945, 0, 0, 0, 0, 0, 0, 3715, 0, 0, 0, 0, 1394, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3384, 0, 1, 0, 0, 4096, 0, 2, 360, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1742, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3601, 0, 0, 0, 15, 0, 2623, 2791, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 0, 0, 0, 9, 4096, 2348, 198, 24, 0, 0, 0, 0, 0, 44, 2680, 0, 3876, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7749, 9205, 7843, 9881, 9030, 8167, 7018, 6643] avg=8192.0000 max=9881 max_rank=3 min=6643 min_rank=7 imbalance_ratio=1.206177 max_load_reduction=55655 max_load_reduction_pct=0.849228 moved_to_echo=30858 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[45, 4082, 0, 0, 162, 253, 0, 0, 1217, 942, 0, 0, 0, 0, 0, 0, 3797, 0, 0, 1, 0, 305, 18, 12, 4095, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 1, 0, 4054, 1, 18, 366, 0, 0, 0, 0, 0, 0, 0, 3647, 0, 0, 1739, 5, 9, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 2, 0, 7, 0, 0, 2771, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 14, 2, 2, 14, 283, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 2008, 4023, 2355, 201, 24, 0, 0, 0, 0, 0, 2379, 2665, 0, 3811, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6701, 8228, 8507, 9315, 6807, 8445, 8674, 8859] avg=8192.0000 max=9315 max_rank=3 min=6701 min_rank=0 imbalance_ratio=1.137085 max_load_reduction=56221 max_load_reduction_pct=0.857864 moved_to_echo=32785 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 130, 0, 75, 1, 578, 1218, 934, 0, 0, 0, 0, 0, 0, 3941, 20, 14, 27, 0, 264, 0, 18, 4095, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 1, 1, 9, 4030, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 3309, 0, 0, 1729, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 19, 0, 0, 0, 242, 2760, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 37, 2, 0, 0, 1, 4072, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 2, 0, 0, 4096, 4004, 2332, 198, 24, 0, 0, 0, 0, 0, 3831, 2680, 0, 4081, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7028, 8379, 8475, 5040, 7092, 8208, 10714, 10600] avg=8192.0000 max=10714 max_rank=6 min=5040 min_rank=3 imbalance_ratio=1.307861 max_load_reduction=54822 max_load_reduction_pct=0.836517 moved_to_echo=36614 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4024, 0, 0, 4, 21, 0, 3976, 1204, 900, 0, 0, 0, 0, 0, 0, 2997, 0, 0, 1, 82, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 3854, 0, 703, 0, 0, 4063, 52, 0, 343, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 1524, 0, 7, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3718, 3, 0, 0, 175, 0, 178, 2660, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 14, 30, 0, 111, 4056, 0, 0, 0, 0, 0, 0, 0, 306, 30, 0, 25, 0, 0, 4096, 4016, 2054, 195, 24, 0, 0, 0, 0, 0, 4086, 2649, 78, 991, 0, 70, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10129, 7137, 9015, 5721, 6734, 8179, 10746, 7875] avg=8192.0000 max=10746 max_rank=6 min=5721 min_rank=3 imbalance_ratio=1.311768 max_load_reduction=54790 max_load_reduction_pct=0.836029 moved_to_echo=33534 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[13, 4096, 0, 0, 2, 914, 0, 1472, 1218, 945, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 6, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 94, 4096, 0, 0, 360, 0, 0, 0, 0, 0, 0, 0, 3218, 0, 0, 1715, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2018, 2791, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 1, 4072, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 32, 4096, 2313, 198, 24, 0, 0, 0, 0, 0, 4096, 2680, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8660, 8198, 8642, 9003, 8905, 8171, 6670, 7287] avg=8192.0000 max=9003 max_rank=3 min=6670 min_rank=6 imbalance_ratio=1.098999 max_load_reduction=56533 max_load_reduction_pct=0.862625 moved_to_echo=31033 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4992, 32768, 128, 256, 1664, 1664, 128, 7424, 9728, 7552, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31488, 128, 128, 0, 256, 0, 11776, 22272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[23296, 768, 0, 13696, 128, 128, 27904, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[28928, 128, 896, 128, 512, 32640, 128, 128, 2944, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26624, 21376, 128, 17408, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 0, 128, 128, 128, 128, 384, 128, 32384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[30336, 128, 128, 128, 128, 2304, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 128, 0, 128, 128, 128, 11904, 32640, 18432, 1664, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[86, 3408, 57, 0, 0, 15, 4096, 1014, 0, 0, 3448, 49, 375, 0, 0, 0, 3617, 0, 3396, 0, 0, 3164, 495, 4046, 0, 12, 0, 3819, 52, 493, 2194, 0, 0, 0, 0, 0, 3220, 2628, 3709, 0, 0, 0, 4084, 0, 0, 12, 0, 0, 0, 0, 41, 1502, 4060, 4093, 0, 4023, 4087, 0, 1, 235, 0, 0, 3, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4, 3443, 0, 0, 0, 0, 973, 3975, 0, 161, 432, 0, 4, 0, 0, 0, 564, 0, 3709, 0, 0, 3038, 1028, 4007, 0, 0, 0, 4096, 0, 0, 1, 0, 3667, 0, 0, 0, 3827, 4090, 4096, 0, 0, 7, 4026, 6, 0, 0, 0, 0, 0, 3, 10, 4093, 4095, 3918, 0, 4095, 4078, 0, 0, 15, 0, 7, 68, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 3368, 1, 0, 0, 0, 3675, 3758, 0, 1, 2139, 0, 17, 0, 0, 279, 141, 0, 4082, 0, 0, 3281, 0, 4096, 0, 0, 0, 4095, 4, 175, 0, 0, 21, 0, 0, 0, 3133, 4091, 4096, 0, 0, 0, 4095, 0, 0, 0, 87, 0, 0, 0, 0, 4096, 3902, 4019, 0, 4096, 4096, 0, 0, 640, 0, 0, 52, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3345, 32, 0, 0, 0, 8, 1768, 0, 40, 4095, 0, 2315, 0, 508, 0, 0, 0, 4096, 0, 0, 4094, 121, 3972, 0, 0, 0, 4096, 0, 0, 0, 0, 136, 0, 0, 1, 4059, 4089, 4091, 0, 0, 0, 4096, 0, 3, 0, 55, 18, 0, 0, 0, 3422, 4095, 4081, 0, 4096, 4066, 0, 0, 722, 0, 0, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 3911, 0, 49, 79, 0, 3229, 4085, 3, 1, 4012, 23, 0, 15, 3, 0, 0, 0, 3986, 0, 0, 229, 0, 4096, 0, 1, 0, 181, 60, 608, 0, 0, 4096, 0, 38, 0, 628, 4062, 3881, 0, 7, 82, 3448, 0, 0, 0, 2, 165, 0, 8, 0, 4095, 4087, 4075, 0, 4096, 4096, 1, 0, 4033, 11, 2, 0, 52] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 3635, 0, 0, 0, 0, 2231, 4096, 18, 4096, 4093, 0, 1, 9, 0, 0, 0, 1, 3468, 0, 0, 3451, 141, 4093, 0, 24, 0, 2714, 75, 116, 0, 9, 924, 3, 0, 0, 0, 4096, 4090, 0, 1, 0, 870, 0, 0, 0, 0, 0, 15, 0, 0, 4091, 4096, 3988, 0, 4092, 4071, 5, 22, 2759, 0, 0, 139, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 3893, 2569, 0, 1799, 4059, 0, 75, 0, 0, 0, 10, 3, 34, 0, 0, 4096, 60, 4096, 0, 0, 0, 137, 0, 30, 33, 0, 4094, 0, 0, 3, 1063, 4096, 4096, 0, 0, 1254, 4093, 0, 0, 0, 140, 0, 3, 0, 0, 4096, 3698, 4096, 0, 4096, 4096, 0, 0, 1622, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 395, 0, 18, 0, 21, 4015, 0, 0, 0, 0, 0, 0, 0, 862, 1358, 0, 212, 3, 0, 4093, 152, 4082, 0, 0, 0, 4096, 1, 168, 502, 103, 2074, 40, 243, 0, 0, 4019, 4001, 6, 0, 2665, 3898, 0, 55, 5, 1701, 43, 0, 0, 63, 3804, 2902, 3643, 45, 4096, 4038, 11, 1, 3450, 0, 69, 0, 487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 2514, 0, 0, 0, 0, 973, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 432, 0, 4, 0, 0, 0, 3068, 1053, 0, 0, 0, 0, 0, 0, 564, 0, 3709, 0, 0, 3038, 1028, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4095, 632, 0, 0, 0, 0, 0, 0, 3667, 0, 0, 0, 3827, 4090, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4026, 6, 0, 0, 0, 0, 2224, 1542, 0, 0, 0, 0, 0, 0, 0, 3, 10, 4093, 4095, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 15, 0, 7, 68, 0, 1303, 929, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7466, 4718, 9501, 8824, 11980, 7805, 8842, 6400] avg=8192.0000 max=11980 max_rank=4 min=4718 min_rank=1 imbalance_ratio=1.462402 max_load_reduction=53556 max_load_reduction_pct=0.817200 moved_to_echo=35027 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[86, 2489, 57, 0, 0, 15, 4096, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 49, 375, 0, 0, 0, 2778, 1100, 0, 0, 0, 0, 0, 0, 3617, 0, 3396, 0, 0, 3164, 495, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3819, 52, 493, 2194, 0, 4023, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3220, 2628, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 12, 0, 0, 2323, 1557, 0, 0, 0, 0, 0, 0, 0, 0, 41, 1502, 4060, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 1, 235, 0, 0, 3, 2, 1316, 919, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7757, 7750, 11845, 11165, 6207, 7976, 6273, 6563] avg=8192.0000 max=11845 max_rank=2 min=6207 min_rank=4 imbalance_ratio=1.445923 max_load_reduction=53691 max_load_reduction_pct=0.819260 moved_to_echo=27019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2460, 1, 0, 0, 0, 3675, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2139, 0, 17, 0, 0, 279, 3068, 1081, 0, 0, 0, 0, 0, 0, 141, 0, 4082, 0, 0, 3281, 0, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 175, 0, 0, 4096, 632, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 3133, 4091, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 87, 0, 2281, 1576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3902, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 640, 0, 0, 52, 0, 1332, 908, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9894, 6585, 8692, 9002, 7641, 8039, 8655, 7028] avg=8192.0000 max=9894 max_rank=0 min=6585 min_rank=1 imbalance_ratio=1.207764 max_load_reduction=55642 max_load_reduction_pct=0.849030 moved_to_echo=31363 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2443, 32, 0, 0, 0, 8, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4095, 0, 2315, 0, 508, 0, 3065, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 121, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 631, 0, 0, 0, 0, 0, 0, 136, 0, 0, 1, 4059, 4089, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 55, 18, 2316, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3422, 4095, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 722, 0, 0, 0, 16, 1292, 902, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4251, 11120, 9463, 8823, 8680, 8016, 8185, 6998] avg=8192.0000 max=11120 max_rank=1 min=4251 min_rank=0 imbalance_ratio=1.357422 max_load_reduction=54416 max_load_reduction_pct=0.830322 moved_to_echo=31879 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2650, 0, 0, 0, 0, 2231, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 4093, 0, 1, 9, 0, 0, 3069, 1077, 0, 0, 0, 0, 0, 0, 0, 1, 3468, 0, 0, 3451, 141, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 2714, 75, 116, 0, 9, 4092, 636, 0, 0, 0, 0, 0, 0, 924, 3, 0, 0, 0, 4096, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 870, 0, 0, 0, 0, 0, 2266, 1578, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4091, 4096, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 5, 22, 2759, 0, 0, 139, 3, 1337, 985, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8977, 12363, 8239, 7666, 5408, 4715, 8847, 9321] avg=8192.0000 max=12363 max_rank=1 min=4715 min_rank=5 imbalance_ratio=1.509155 max_load_reduction=53173 max_load_reduction_pct=0.811356 moved_to_echo=28291 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2856, 0, 49, 79, 0, 3229, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4012, 23, 0, 15, 3, 0, 2907, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 0, 229, 0, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 181, 60, 608, 0, 0, 4096, 599, 0, 0, 0, 0, 0, 0, 4096, 0, 38, 0, 628, 4062, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 82, 3448, 0, 0, 0, 2, 165, 2313, 1576, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4095, 4087, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4033, 11, 2, 0, 52, 1332, 1055, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10298, 8060, 5403, 5545, 9199, 7593, 8856, 10582] avg=8192.0000 max=10582 max_rank=7 min=5403 min_rank=2 imbalance_ratio=1.291748 max_load_reduction=54954 max_load_reduction_pct=0.838531 moved_to_echo=36230 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2991, 0, 0, 0, 0, 3893, 2569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1799, 4059, 0, 75, 0, 0, 0, 3068, 1101, 0, 0, 0, 0, 0, 0, 10, 3, 34, 0, 0, 4096, 60, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 30, 33, 0, 4096, 632, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3, 1063, 4096, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 4093, 0, 0, 0, 140, 0, 2325, 1576, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 3698, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1622, 0, 0, 0, 0, 1332, 1105, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9453, 10102, 5391, 4928, 9652, 9388, 8467, 8155] avg=8192.0000 max=10102 max_rank=1 min=4928 min_rank=3 imbalance_ratio=1.233154 max_load_reduction=55434 max_load_reduction_pct=0.845856 moved_to_echo=35662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2991, 395, 0, 18, 0, 21, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 2997, 979, 0, 0, 0, 0, 0, 0, 1358, 0, 212, 3, 0, 4093, 152, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 168, 502, 103, 4096, 617, 0, 0, 0, 0, 0, 0, 2074, 40, 243, 0, 0, 4019, 387, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 3898, 0, 55, 5, 1701, 43, 2067, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 63, 3804, 2902, 597, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 11, 1, 3450, 0, 69, 0, 487, 1328, 1105, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7440, 4838, 7002, 9583, 6769, 12004, 7411, 10489] avg=8192.0000 max=12004 max_rank=5 min=4838 min_rank=1 imbalance_ratio=1.465332 max_load_reduction=53532 max_load_reduction_pct=0.816833 moved_to_echo=36673 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 21504, 512, 128, 128, 128, 18176, 25344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 6144, 22400, 128, 2816, 128, 512, 1152, 24064, 8704, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5760, 128, 23040, 128, 0, 25472, 2048, 9472, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[15104, 128, 384, 128, 16000, 31232, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 4096, 28672, 128, 128, 128, 2048, 256, 18176, 12544, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 128, 128, 13568, 128, 128, 384, 640, 10624, 7936, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 23296, 256, 1664, 2816, 128, 32768, 4992, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 29312, 30976, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1352, 0, 4088, 28, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 2, 1, 3815, 4091, 0, 0, 0, 367, 4095, 0, 3132, 1, 1908, 0, 0, 4068, 1, 3982, 3897, 0, 0, 2685, 0, 4096, 0, 3, 0, 135, 0, 7, 4, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 32, 0, 3979, 3434, 0, 8, 4096, 0, 4041, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3829, 0, 4039, 233, 40, 0, 4096, 0, 0, 2, 0, 0, 68, 0, 0, 0, 15, 3962, 0, 0, 1757, 4087, 4095, 114, 4052, 1841, 335, 0, 0, 4070, 0, 2730, 4069, 6, 0, 3790, 17, 2990, 18, 0, 0, 138, 26, 0, 118, 0, 0, 4077, 1, 0, 10, 0, 0, 8, 0, 2216, 5, 455, 0, 0, 4096, 126, 4005, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[37, 0, 4096, 4, 0, 0, 4096, 0, 0, 86, 0, 0, 0, 0, 1, 0, 4032, 4096, 0, 0, 76, 3252, 4096, 59, 4026, 0, 1431, 0, 0, 4096, 200, 4096, 0, 0, 0, 4026, 0, 4096, 0, 0, 0, 3, 0, 27, 24, 35, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 3350, 3474, 444, 0, 0, 4094, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[714, 0, 4096, 2682, 4, 2, 4091, 0, 0, 0, 10, 0, 0, 0, 0, 0, 233, 4081, 739, 0, 1069, 1, 4078, 4092, 3486, 2, 3745, 0, 0, 4050, 0, 3164, 0, 0, 0, 4096, 0, 3817, 0, 0, 0, 0, 0, 3077, 106, 450, 0, 4096, 0, 30, 0, 0, 0, 1, 0, 510, 70, 664, 91, 0, 4096, 0, 4093, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 89, 0, 0, 4096, 0, 0, 3, 1, 0, 0, 0, 0, 0, 1600, 4096, 1, 0, 9, 3829, 4093, 48, 4096, 16, 0, 0, 0, 4096, 7, 4096, 4096, 0, 0, 4096, 0, 4092, 0, 0, 0, 4, 0, 210, 0, 248, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2552, 255, 3409, 0, 18, 4093, 0, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1092, 252, 3796, 3027, 0, 1, 4039, 61, 0, 3768, 0, 1, 0, 0, 0, 13, 4092, 4096, 0, 304, 0, 3371, 4088, 0, 4093, 0, 2, 0, 0, 3597, 15, 4096, 0, 0, 0, 2767, 28, 3841, 59, 4, 0, 21, 0, 410, 0, 631, 0, 4062, 0, 148, 0, 49, 8, 0, 41, 0, 2072, 1, 0, 0, 3674, 1, 3915, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[759, 1, 4083, 0, 0, 0, 4073, 0, 0, 455, 0, 30, 241, 0, 764, 8, 2007, 4087, 0, 0, 2, 45, 2590, 6, 3430, 2, 994, 0, 13, 4027, 1, 4084, 0, 0, 7, 3857, 2, 4095, 0, 0, 0, 0, 0, 3300, 0, 302, 5, 4066, 18, 0, 0, 3, 0, 0, 17, 3467, 2915, 3593, 0, 1, 4088, 23, 4075, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[509, 0, 4094, 15, 246, 1, 4055, 12, 0, 2606, 4, 4, 0, 406, 0, 0, 778, 4096, 3165, 0, 163, 32, 2835, 3246, 3241, 67, 0, 0, 0, 4072, 0, 4091, 0, 16, 0, 1045, 40, 4051, 0, 1, 0, 27, 0, 0, 0, 4095, 0, 3395, 1, 8, 0, 0, 43, 0, 0, 4096, 2397, 1, 0, 481, 4010, 3, 4086, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1352, 0, 4088, 28, 0, 0, 2279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 1, 4091, 3101, 0, 0, 0, 0, 0, 0, 3815, 0, 0, 0, 0, 367, 2614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3132, 1, 1908, 0, 0, 0, 1, 3334, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 0, 0, 2685, 0, 3547, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 7, 4, 0, 0, 4087, 1481, 648, 549, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 3372, 1789, 967, 0, 0, 0, 0, 0, 3979, 3434, 0, 8, 4096, 0, 669, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7747, 7196, 6796, 8376, 10132, 6938, 6160, 12191] avg=8192.0000 max=12191 max_rank=7 min=6160 min_rank=6 imbalance_ratio=1.488159 max_load_reduction=53345 max_load_reduction_pct=0.813980 moved_to_echo=35421 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[37, 0, 4096, 4, 0, 0, 2280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 1, 0, 4096, 3123, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 76, 3252, 2615, 59, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 1431, 0, 0, 0, 200, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 27, 24, 35, 0, 4087, 1481, 666, 549, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3350, 3418, 1789, 973, 0, 0, 0, 0, 0, 3474, 444, 0, 0, 4094, 0, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6417, 7306, 10034, 9087, 7573, 6899, 9530, 8690] avg=8192.0000 max=10034 max_rank=2 min=6417 min_rank=0 imbalance_ratio=1.224854 max_load_reduction=55502 max_load_reduction_pct=0.846893 moved_to_echo=32692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3829, 0, 4039, 233, 40, 0, 2280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 68, 0, 0, 0, 3962, 3103, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 1757, 4087, 2614, 114, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 1841, 335, 0, 0, 0, 0, 2286, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 6, 0, 3790, 17, 2590, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 26, 0, 118, 0, 0, 4077, 1481, 444, 400, 27, 0, 0, 0, 0, 1, 0, 10, 0, 0, 8, 0, 2216, 3342, 1789, 967, 0, 0, 0, 0, 0, 5, 455, 0, 0, 4096, 126, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10421, 7135, 8587, 8514, 10490, 6711, 8333, 5345] avg=8192.0000 max=10490 max_rank=4 min=5345 min_rank=7 imbalance_ratio=1.280518 max_load_reduction=55046 max_load_reduction_pct=0.839935 moved_to_echo=30879 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[714, 0, 4096, 2682, 4, 2, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4081, 3088, 0, 0, 0, 0, 0, 0, 233, 0, 739, 0, 1069, 1, 2604, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3486, 2, 3745, 0, 0, 0, 0, 2649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3077, 106, 450, 0, 4096, 1474, 515, 511, 27, 0, 0, 0, 0, 0, 30, 0, 0, 0, 1, 0, 510, 3415, 1787, 962, 0, 0, 0, 0, 0, 70, 664, 91, 0, 4096, 0, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9775, 7179, 8738, 9882, 7402, 10256, 6705, 5599] avg=8192.0000 max=10256 max_rank=5 min=5599 min_rank=7 imbalance_ratio=1.251953 max_load_reduction=55280 max_load_reduction_pct=0.843506 moved_to_echo=29962 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[509, 0, 4094, 15, 246, 1, 2257, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2606, 4, 4, 0, 406, 0, 0, 4096, 3105, 0, 0, 0, 0, 0, 0, 778, 0, 3165, 0, 163, 32, 1810, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 3241, 67, 0, 0, 0, 0, 0, 3425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 1045, 40, 3508, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4095, 0, 3395, 1025, 666, 543, 27, 0, 0, 0, 0, 1, 8, 0, 0, 43, 0, 0, 4096, 3409, 1771, 967, 0, 0, 0, 0, 0, 2397, 1, 0, 481, 4010, 3, 677, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7134, 10221, 9194, 6733, 4610, 9778, 10295, 7571] avg=8192.0000 max=10295 max_rank=6 min=4610 min_rank=4 imbalance_ratio=1.256714 max_load_reduction=55241 max_load_reduction_pct=0.842911 moved_to_echo=32254 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 89, 0, 0, 2280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 4096, 3122, 0, 0, 0, 0, 0, 0, 1600, 0, 1, 0, 9, 3829, 2613, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 7, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 210, 0, 248, 0, 4096, 1480, 666, 548, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 3417, 1789, 974, 0, 0, 0, 0, 0, 255, 3409, 0, 18, 4093, 0, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6465, 7222, 8100, 7549, 11736, 7279, 8732, 8453] avg=8192.0000 max=11736 max_rank=4 min=6465 min_rank=0 imbalance_ratio=1.432617 max_load_reduction=53800 max_load_reduction_pct=0.820923 moved_to_echo=36200 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[759, 1, 4083, 0, 0, 0, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 30, 241, 0, 764, 8, 4087, 3071, 0, 0, 0, 0, 0, 0, 2007, 0, 0, 0, 2, 45, 1650, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3430, 2, 994, 0, 13, 0, 1, 3415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3857, 2, 3543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3300, 0, 302, 5, 4066, 940, 669, 552, 31, 0, 0, 0, 0, 18, 0, 0, 3, 0, 0, 17, 3467, 3404, 1785, 956, 0, 0, 0, 0, 0, 2915, 3593, 0, 1, 4088, 23, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7100, 8656, 3710, 7855, 7409, 9865, 9650, 11291] avg=8192.0000 max=11291 max_rank=7 min=3710 min_rank=2 imbalance_ratio=1.378296 max_load_reduction=54245 max_load_reduction_pct=0.827713 moved_to_echo=38215 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1092, 252, 3796, 3027, 0, 1, 2248, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3768, 0, 1, 0, 0, 0, 13, 4096, 2743, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 304, 0, 3371, 2610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 0, 0, 0, 15, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2767, 28, 3326, 59, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 410, 0, 631, 0, 4062, 1478, 666, 515, 27, 0, 0, 0, 0, 0, 148, 0, 49, 8, 0, 41, 0, 3267, 1764, 854, 0, 0, 0, 0, 0, 2072, 1, 0, 0, 3674, 1, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10477, 10621, 10377, 7540, 6184, 7810, 6131, 6396] avg=8192.0000 max=10621 max_rank=1 min=6131 min_rank=6 imbalance_ratio=1.296509 max_load_reduction=54915 max_load_reduction_pct=0.837936 moved_to_echo=26521 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 256, 32512, 6144, 384, 128, 18176, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[16640, 0, 3968, 384, 3200, 15104, 19200, 7680, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[15232, 12032, 128, 512, 32256, 256, 5376, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[29568, 2048, 8448, 0, 128, 0, 256, 25472, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12160, 128, 128, 26368, 128, 27008, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 256, 128, 128, 128, 128, 128, 16256, 27136, 14336, 7680, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 384, 128, 7040, 256, 5888, 128, 32000, 10880, 4992, 4224, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 7040, 128, 128, 384, 512, 768, 128, 32640, 24576, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 21, 0, 3091, 0, 0, 5, 4096, 0, 4080, 4094, 4091, 231, 14, 0, 0, 0, 0, 0, 0, 260, 6, 1, 0, 0, 0, 2, 0, 0, 0, 4095, 134, 4096, 0, 1478, 0, 0, 0, 1, 4088, 232, 3814, 0, 0, 0, 0, 3370, 1, 0, 4094, 4096, 0, 3767, 0, 4091, 0, 3, 5, 1, 4080, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4070, 2454, 0, 0, 3571, 0, 0, 0, 3617, 0, 3844, 4076, 4058, 0, 26, 0, 2, 0, 0, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4062, 111, 3935, 0, 889, 0, 0, 0, 3783, 2676, 724, 2286, 0, 0, 0, 62, 3314, 0, 17, 3995, 4073, 0, 1450, 0, 4096, 0, 4, 28, 0, 4087, 53, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 55, 0, 0, 2049, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 280, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 39, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 14, 0, 0, 0, 1, 4093, 1960, 3791, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 4051, 0, 4096, 0, 0, 54, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 3658, 2, 0, 0, 21, 0, 0, 569, 4096, 0, 4094, 4095, 4093, 0, 136, 1971, 0, 0, 0, 16, 0, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1283, 1, 4096, 0, 0, 0, 0, 0, 3724, 4022, 1, 4096, 0, 0, 0, 216, 3753, 0, 0, 4096, 3281, 0, 3997, 0, 4096, 0, 4, 76, 0, 4095, 585, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 65, 0, 0, 1440, 0, 0, 1, 0, 0, 4064, 4096, 4095, 1, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 4095, 2875, 4096, 0, 0, 0, 0, 0, 4090, 3362, 61, 4096, 0, 0, 0, 0, 4093, 0, 0, 4096, 4096, 0, 4091, 0, 4096, 4, 5, 0, 0, 4096, 401, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[5, 4083, 499, 0, 4, 4096, 0, 0, 2, 2, 0, 4070, 4053, 4085, 0, 0, 0, 1, 0, 9, 951, 0, 65, 0, 100, 0, 933, 0, 3, 0, 43, 14, 4094, 7, 3945, 0, 0, 1, 474, 1, 2819, 4095, 0, 3063, 9, 5, 0, 0, 4092, 0, 0, 3526, 4090, 1, 4095, 0, 4082, 0, 15, 0, 14, 4090, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 3189, 0, 0, 396, 0, 7, 0, 248, 0, 3893, 4062, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 1930, 8, 33, 54, 59, 0, 0, 0, 0, 4086, 0, 4049, 68, 39, 2, 77, 0, 0, 2702, 40, 4094, 1, 0, 0, 20, 4047, 0, 0, 3249, 4096, 20, 3795, 434, 4023, 0, 38, 399, 2, 4084, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4075, 14, 0, 28, 4084, 0, 2, 0, 4096, 0, 4027, 4061, 3203, 44, 1079, 30, 16, 16, 0, 12, 0, 53, 0, 0, 0, 0, 0, 30, 0, 0, 33, 3672, 86, 4096, 65, 4065, 0, 0, 0, 1750, 1222, 1, 236, 0, 0, 2, 1102, 3992, 6, 0, 1390, 3997, 0, 3122, 0, 4061, 92, 784, 2752, 0, 4046, 94, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 21, 0, 3091, 0, 0, 648, 376, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 1406, 0, 4091, 231, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 6, 2477, 2674, 1059, 475, 38, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 4094, 4058, 0, 0, 0, 0, 0, 0, 4095, 134, 3245, 0, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3029, 232, 3814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 893, 1, 0, 4094, 0, 0, 3767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3, 5, 1, 3432, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 9843, 6989, 8155, 8952, 7076, 8755, 7534] avg=8192.0000 max=9843 max_rank=1 min=6989 min_rank=2 imbalance_ratio=1.201538 max_load_reduction=55693 max_load_reduction_pct=0.849808 moved_to_echo=32317 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4070, 2454, 0, 0, 3571, 0, 0, 649, 361, 0, 0, 0, 0, 0, 0, 0, 3617, 0, 1325, 0, 4058, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 164, 0, 2436, 2519, 693, 456, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4076, 4034, 0, 0, 0, 0, 0, 0, 4062, 111, 3118, 0, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3783, 1983, 724, 2286, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 878, 0, 17, 3995, 0, 0, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 28, 0, 3438, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11105, 9026, 6309, 8119, 8180, 8838, 6340, 7619] avg=8192.0000 max=11105 max_rank=0 min=6309 min_rank=2 imbalance_ratio=1.355591 max_load_reduction=54431 max_load_reduction_pct=0.830551 moved_to_echo=30977 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 55, 0, 0, 2049, 0, 0, 651, 376, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1411, 0, 4096, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3010, 2685, 1060, 474, 39, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 4096, 4057, 0, 0, 0, 0, 0, 0, 4096, 0, 3242, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3033, 1960, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 0, 0, 4096, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 54, 0, 3445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7227, 9883, 7270, 8192, 7352, 8785, 9232, 7595] avg=8192.0000 max=9883 max_rank=1 min=7227 min_rank=0 imbalance_ratio=1.206421 max_load_reduction=55653 max_load_reduction_pct=0.849197 moved_to_echo=32964 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3658, 2, 0, 0, 21, 0, 0, 651, 376, 0, 0, 0, 0, 0, 0, 569, 4096, 0, 1411, 0, 4093, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 1971, 0, 0, 0, 16, 0, 1363, 0, 2758, 2683, 1042, 475, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3249, 0, 0, 0, 0, 0, 0, 1283, 1, 3245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3724, 2980, 1, 4096, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 995, 0, 0, 4096, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 76, 0, 3444, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4708, 10305, 10340, 7344, 4529, 11017, 9088, 8205] avg=8192.0000 max=11017 max_rank=5 min=4529 min_rank=4 imbalance_ratio=1.344849 max_load_reduction=54519 max_load_reduction_pct=0.831894 moved_to_echo=32839 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3189, 0, 0, 396, 0, 7, 649, 372, 0, 0, 0, 0, 0, 0, 0, 248, 0, 1341, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 1930, 2974, 2552, 700, 469, 39, 0, 0, 0, 8, 33, 54, 59, 0, 0, 0, 0, 4062, 4057, 0, 0, 0, 0, 0, 0, 4086, 0, 3208, 68, 39, 2, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2002, 40, 4094, 1, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 0, 0, 3249, 0, 20, 3795, 434, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 38, 399, 2, 3435, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8709, 5685, 12763, 8273, 7480, 6157, 8571, 7898] avg=8192.0000 max=12763 max_rank=2 min=5685 min_rank=1 imbalance_ratio=1.557983 max_load_reduction=52773 max_load_reduction_pct=0.805252 moved_to_echo=30106 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 65, 0, 0, 1440, 0, 0, 651, 376, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1400, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3008, 2664, 871, 475, 38, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 4096, 4058, 0, 0, 0, 0, 0, 0, 4095, 2875, 3245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2491, 61, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 5, 0, 0, 3445, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6628, 5497, 7061, 8174, 10215, 10738, 9272, 7951] avg=8192.0000 max=10738 max_rank=5 min=5497 min_rank=1 imbalance_ratio=1.310791 max_load_reduction=54798 max_load_reduction_pct=0.836151 moved_to_echo=38176 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[5, 4083, 499, 0, 4, 4096, 0, 0, 653, 365, 0, 0, 0, 0, 0, 0, 2, 2, 0, 1398, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 951, 0, 65, 0, 3012, 2672, 1063, 462, 39, 0, 0, 0, 100, 0, 933, 0, 3, 0, 43, 14, 4053, 4051, 0, 0, 0, 0, 0, 0, 4094, 7, 3118, 0, 0, 1, 474, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2819, 3032, 0, 3063, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 0, 0, 3526, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 15, 0, 14, 3437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9705, 5487, 8274, 9197, 7695, 8928, 8702, 7548] avg=8192.0000 max=9705 max_rank=0 min=5487 min_rank=1 imbalance_ratio=1.184692 max_load_reduction=55831 max_load_reduction_pct=0.851913 moved_to_echo=32873 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4075, 14, 0, 28, 4084, 0, 2, 643, 376, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1388, 0, 3203, 44, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 30, 16, 16, 0, 12, 0, 53, 0, 2934, 2639, 316, 475, 39, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 33, 4061, 3958, 0, 0, 0, 0, 0, 0, 3672, 86, 3245, 65, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1750, 906, 1, 236, 0, 0, 2, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 6, 0, 1390, 0, 0, 3122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 92, 784, 2752, 0, 3403, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9222, 9810, 6530, 8082, 11133, 3997, 5576, 11186] avg=8192.0000 max=11186 max_rank=7 min=3997 min_rank=5 imbalance_ratio=1.365479 max_load_reduction=54350 max_load_reduction_pct=0.829315 moved_to_echo=31892 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[2048, 128, 128, 128, 1024, 0, 6016, 2048, 22656, 21120, 6912, 3840, 384, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[16256, 19456, 3072, 25600, 128, 128, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 128, 128, 28544, 0, 128, 28416, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[640, 20352, 0, 11136, 0, 31872, 640, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32384, 6400, 128, 128, 18816, 0, 128, 5248, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[29568, 3328, 25728, 256, 6528, 128, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 896, 3328, 128, 27520, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 1152, 128, 128, 0, 128, 128, 32640, 31616, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1940, 0, 0, 0, 4094, 1211, 4096, 161, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 6, 0, 4053, 4089, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 127, 3158, 0, 0, 0, 0, 0, 0, 56, 74, 0, 4096, 1, 4096, 2402, 4096, 0, 3929, 3431, 0, 4, 2, 0, 3942] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[2284, 0, 0, 0, 4094, 1468, 4096, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 59, 4081, 0, 572, 0, 4017, 4093, 0, 130, 1320, 0, 2, 3, 0, 0, 0, 0, 4096, 0, 4074, 1, 0, 0, 0, 27, 631, 0, 0, 0, 0, 0, 0, 0, 2906, 4, 4093, 0, 4077, 0, 3985, 13, 4095, 12, 0, 32, 3103, 0, 4069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[3747, 0, 0, 0, 4096, 2528, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 386, 1, 1195, 0, 0, 32, 4078, 3939, 0, 820, 0, 2, 0, 0, 0, 1, 2, 0, 4068, 0, 3369, 0, 0, 1, 0, 0, 1187, 22, 5, 14, 0, 0, 0, 483, 222, 1, 4089, 0, 4092, 4027, 3683, 0, 4075, 5, 0, 162, 2931, 0, 4081] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[927, 0, 0, 0, 1854, 4093, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 11, 0, 2015, 4096, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 12, 0, 4096, 0, 1, 0, 0, 3146, 4096, 0, 1, 42, 0, 0, 0, 1, 430, 0, 4049, 0, 4096, 1895, 3246, 0, 3884, 3704, 0, 0, 3393, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[3773, 3, 3, 27, 3891, 365, 4039, 3875, 4056, 2, 0, 0, 33, 0, 31, 47, 94, 4094, 0, 3026, 0, 4020, 3905, 0, 363, 747, 1, 9, 78, 87, 176, 1, 1, 4049, 14, 4057, 0, 0, 4, 0, 165, 214, 29, 259, 0, 0, 2, 0, 0, 8, 0, 4096, 163, 3700, 0, 4089, 0, 3891, 339, 0, 277, 18, 0, 3415] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[3994, 0, 168, 0, 3958, 4079, 3715, 0, 4056, 0, 105, 0, 0, 35, 0, 0, 67, 3849, 0, 350, 0, 4051, 4096, 0, 0, 0, 22, 0, 283, 1, 3, 39, 0, 3693, 37, 3983, 0, 10, 0, 0, 3411, 0, 0, 646, 0, 7, 0, 0, 11, 3041, 144, 504, 17, 4096, 782, 4090, 1, 4096, 5, 0, 8, 0, 0, 4083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[3964, 0, 0, 0, 4089, 2720, 4096, 116, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 357, 0, 4081, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 19, 9, 4069, 0, 4096, 0, 4096, 0, 4096, 1010, 0, 0, 62, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4085, 0, 0, 1, 3881, 0, 4067, 411, 4089, 0, 0, 0, 0, 2, 0, 320, 2, 3747, 0, 0, 0, 3159, 4096, 0, 0, 484, 11, 23, 0, 4, 0, 0, 0, 4067, 4, 4092, 1, 0, 2, 0, 0, 648, 33, 0, 7, 1, 0, 0, 499, 1198, 0, 3167, 1, 4095, 3046, 4092, 0, 4050, 0, 6, 3, 4050, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2284, 0, 0, 0, 2679, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 1961, 1471, 671, 0, 0, 0, 0, 0, 59, 4081, 0, 572, 0, 4017, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 1320, 0, 2, 3, 0, 0, 0, 4077, 3597, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4074, 1, 0, 0, 0, 744, 0, 0, 0, 0, 0, 0, 0, 27, 631, 0, 0, 0, 0, 0, 0, 3806, 1209, 499, 0, 0, 0, 0, 0, 0, 2906, 4, 4093, 0, 0, 0, 2024, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1415, 12, 0, 32, 3103, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6431, 8202, 9016, 9129, 8915, 6172, 9027, 8644] avg=8192.0000 max=9129 max_rank=3 min=6172 min_rank=5 imbalance_ratio=1.114380 max_load_reduction=56407 max_load_reduction_pct=0.860703 moved_to_echo=32758 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1940, 0, 0, 0, 2679, 1211, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2016, 1411, 671, 0, 0, 0, 0, 0, 0, 4078, 0, 6, 0, 4053, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 4096, 3597, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 744, 0, 0, 0, 0, 0, 0, 0, 127, 3158, 0, 0, 0, 0, 0, 0, 3802, 1160, 499, 0, 0, 0, 0, 0, 56, 74, 0, 4096, 1, 0, 2402, 2080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 3431, 0, 4, 2, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5991, 8194, 8424, 7800, 8935, 8746, 8709, 8737] avg=8192.0000 max=8935 max_rank=4 min=5991 min_rank=0 imbalance_ratio=1.090698 max_load_reduction=56601 max_load_reduction_pct=0.863663 moved_to_echo=35127 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3747, 0, 0, 0, 2679, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 386, 1812, 1464, 672, 0, 0, 0, 0, 0, 1, 1195, 0, 0, 32, 4078, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 0, 2, 0, 0, 0, 1, 2, 4092, 3597, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 3369, 0, 0, 1, 0, 745, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 22, 5, 14, 0, 0, 0, 3663, 1203, 499, 0, 0, 0, 0, 0, 483, 222, 1, 4089, 0, 0, 4027, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1408, 5, 0, 162, 2931, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8954, 8430, 5582, 8514, 8183, 6593, 10693, 8587] avg=8192.0000 max=10693 max_rank=6 min=5582 min_rank=2 imbalance_ratio=1.305298 max_load_reduction=54843 max_load_reduction_pct=0.836838 moved_to_echo=34056 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[927, 0, 0, 0, 1213, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 1597, 1395, 304, 0, 0, 0, 0, 0, 0, 4076, 0, 11, 0, 2015, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 4096, 3597, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 0, 1, 0, 0, 337, 0, 0, 0, 0, 0, 0, 0, 3146, 4096, 0, 1, 42, 0, 0, 0, 3809, 1147, 499, 0, 0, 0, 0, 0, 1, 430, 0, 4049, 0, 0, 1895, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1342, 3704, 0, 0, 3393, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6233, 7385, 6389, 7784, 4446, 12740, 8024, 12535] avg=8192.0000 max=12740 max_rank=5 min=4446 min_rank=4 imbalance_ratio=1.555176 max_load_reduction=52796 max_load_reduction_pct=0.805603 moved_to_echo=37745 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3773, 3, 3, 27, 2546, 365, 0, 3875, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 2, 0, 0, 33, 0, 31, 47, 2012, 1398, 638, 0, 0, 0, 0, 0, 94, 4094, 0, 3026, 0, 4020, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 747, 1, 9, 78, 87, 176, 1, 3700, 3546, 0, 0, 0, 0, 0, 0, 1, 4049, 14, 4057, 0, 0, 4, 0, 707, 0, 0, 0, 0, 0, 0, 0, 165, 214, 29, 259, 0, 0, 2, 0, 3631, 1149, 493, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 163, 0, 0, 2077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1344, 339, 0, 277, 18, 0, 3415, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10592, 8217, 11508, 8708, 8832, 5942, 6344, 5393] avg=8192.0000 max=11508 max_rank=2 min=5393 min_rank=7 imbalance_ratio=1.404785 max_load_reduction=54028 max_load_reduction_pct=0.824402 moved_to_echo=26511 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3994, 0, 168, 0, 2590, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 105, 0, 0, 35, 0, 0, 2013, 1471, 649, 0, 0, 0, 0, 0, 67, 3849, 0, 350, 0, 4051, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 283, 1, 3, 39, 4096, 3263, 0, 0, 0, 0, 0, 0, 0, 3693, 37, 3983, 0, 10, 0, 0, 719, 0, 0, 0, 0, 0, 0, 0, 3411, 0, 0, 646, 0, 7, 0, 0, 3809, 1209, 452, 0, 0, 0, 0, 0, 11, 3041, 144, 504, 17, 0, 782, 2077, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1416, 5, 0, 8, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10831, 8329, 8604, 7707, 8442, 9534, 6576, 5513] avg=8192.0000 max=10831 max_rank=0 min=5513 min_rank=7 imbalance_ratio=1.322144 max_load_reduction=54705 max_load_reduction_pct=0.834732 moved_to_echo=30065 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3964, 0, 0, 0, 2676, 2720, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2016, 1471, 670, 0, 0, 0, 0, 0, 0, 4096, 0, 357, 0, 4081, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 3597, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 4096, 0, 0, 0, 0, 743, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3809, 1209, 499, 0, 0, 0, 0, 0, 0, 19, 9, 4069, 0, 0, 0, 2080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1416, 1010, 0, 0, 62, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9476, 8253, 8821, 7705, 8907, 9613, 6177, 6584] avg=8192.0000 max=9613 max_rank=5 min=6177 min_rank=6 imbalance_ratio=1.173462 max_load_reduction=55923 max_load_reduction_pct=0.853317 moved_to_echo=31281 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 0, 1, 2531, 0, 0, 411, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 2, 0, 320, 2017, 1458, 640, 0, 0, 0, 0, 0, 2, 3747, 0, 0, 0, 3159, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 11, 23, 0, 4, 0, 0, 4095, 3572, 0, 0, 0, 0, 0, 0, 0, 4067, 4, 4092, 1, 0, 2, 0, 710, 0, 0, 0, 0, 0, 0, 0, 0, 648, 33, 0, 7, 1, 0, 0, 3812, 1200, 495, 0, 0, 0, 0, 0, 499, 1198, 0, 3167, 1, 0, 3046, 2075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1392, 0, 6, 3, 4050, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7028, 8526, 7192, 8189, 8876, 6196, 9986, 9543] avg=8192.0000 max=9986 max_rank=6 min=6196 min_rank=5 imbalance_ratio=1.218994 max_load_reduction=55550 max_load_reduction_pct=0.847626 moved_to_echo=34601 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[24832, 128, 256, 128, 19712, 16512, 0, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 28160, 128, 31872, 128, 128, 128, 0, 5504, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[6912, 14080, 128, 1024, 128, 128, 128, 0, 30208, 9600, 3968, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 29312, 0, 4352, 128, 29568, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1152, 7936, 256, 28288, 256, 0, 12160, 16000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 11136, 8576, 128, 512, 13568, 0, 32000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1536, 2688, 128, 128, 384, 128, 384, 128, 32384, 28416, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 128, 768, 15488, 11648, 4992, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3026, 0, 4096, 4096, 1, 413, 3903, 0, 0, 0, 1, 1, 1, 0, 0, 4075, 4055, 0, 0, 4092, 0, 0, 0, 626, 0, 124, 2, 4096, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 4096, 21, 0, 4084, 2, 0, 4066, 4096, 0, 98, 3627, 0, 4093, 0, 0, 3992, 210, 0, 58, 0, 362, 20, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3993, 21, 2814, 4070, 3433, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 851, 4073, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 4, 0, 0, 98, 0, 3950, 22, 797, 4096, 0, 0, 4096, 4089, 0, 3164, 1308, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 79, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 4059, 0, 20, 3931, 3986, 0, 4085, 0, 1, 0, 0, 169, 89, 0, 0, 699, 4093, 101, 0, 3976, 0, 0, 0, 43, 0, 0, 0, 4096, 19, 4096, 2, 0, 0, 0, 0, 33, 0, 4095, 0, 0, 4094, 0, 0, 4076, 4096, 0, 3493, 3981, 0, 4095, 0, 0, 4089, 0, 0, 1, 2, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1145, 2, 5, 4096, 3778, 550, 116, 0, 2, 0, 2, 3044, 3, 0, 0, 3845, 4066, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 15, 4086, 0, 4096, 0, 36, 0, 0, 0, 7, 20, 3942, 0, 0, 4096, 0, 1, 4091, 4040, 0, 3912, 4079, 0, 4096, 32, 0, 4096, 0, 0, 0, 0, 134, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 3827, 7, 4096, 4096, 367, 1, 4093, 0, 0, 0, 3, 108, 237, 0, 0, 3487, 4096, 74, 0, 4094, 0, 0, 0, 0, 0, 10, 33, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 473, 4096, 0, 0, 4094, 4096, 0, 246, 3400, 0, 4096, 0, 0, 4096, 2, 1, 0, 0, 5, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4076, 46, 4096, 4050, 2577, 373, 4065, 0, 0, 0, 0, 1726, 71, 19, 26, 4096, 3451, 0, 0, 4032, 0, 0, 1, 1, 38, 0, 66, 4061, 0, 3967, 0, 144, 0, 0, 0, 0, 3, 3921, 0, 1645, 4087, 0, 0, 2619, 4088, 49, 27, 0, 0, 3992, 5, 0, 4096, 0, 0, 2, 0, 20, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 15, 71, 0, 833, 0, 1329, 3873, 4094, 0, 3741, 0, 0, 5, 151, 206, 4077, 38, 0, 90, 3930, 307, 0, 3899, 0, 29, 2, 0, 0, 0, 0, 4061, 82, 4095, 0, 0, 0, 0, 0, 3869, 11, 2002, 0, 317, 4093, 0, 0, 3996, 4072, 206, 2, 3932, 5, 4036, 0, 1, 4034, 0, 0, 8, 1, 0, 0, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 186, 64, 4054, 3717, 7, 2, 4096, 13, 0, 218, 0, 0, 2798, 0, 1, 38, 4093, 163, 0, 3869, 11, 0, 0, 1, 23, 7, 0, 4087, 0, 4076, 4, 0, 220, 0, 11, 784, 0, 4096, 0, 31, 4085, 0, 0, 4057, 3979, 53, 1377, 4096, 0, 4073, 0, 0, 4094, 0, 0, 0, 4, 2946, 0, 100] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 0, 0, 4059, 0, 20, 2910, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 4085, 0, 1, 0, 0, 169, 984, 517, 0, 0, 0, 0, 0, 0, 89, 0, 0, 699, 4093, 101, 0, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 4096, 3999, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 2, 0, 0, 0, 0, 33, 2925, 454, 37, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 113, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 3579, 0, 3493, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 2, 6, 0, 0, 3527, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6999, 9742, 6033, 8138, 7566, 8284, 11053, 7721] avg=8192.0000 max=11053 max_rank=6 min=6033 min_rank=2 imbalance_ratio=1.349243 max_load_reduction=54483 max_load_reduction_pct=0.831345 moved_to_echo=34624 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3993, 21, 2814, 3013, 0, 0, 0, 0, 0, 0, 0, 0, 3433, 0, 4096, 0, 0, 0, 0, 0, 1019, 516, 0, 0, 0, 0, 0, 0, 1, 0, 0, 851, 4073, 0, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 98, 3014, 454, 38, 0, 0, 0, 0, 0, 0, 3950, 22, 797, 114, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 0, 3164, 1308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 79, 0, 0, 3528, 95, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9841, 9064, 6007, 8098, 7704, 8979, 8045, 7798] avg=8192.0000 max=9841 max_rank=0 min=6007 min_rank=2 imbalance_ratio=1.201294 max_load_reduction=55695 max_load_reduction_pct=0.849838 moved_to_echo=32526 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3026, 0, 4096, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 1, 413, 3903, 0, 0, 0, 1, 1, 1025, 517, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4075, 4055, 0, 0, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 124, 2, 4096, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 3011, 453, 38, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 113, 2, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 3579, 0, 98, 3627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 210, 0, 58, 0, 362, 20, 0, 3518, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10155, 5861, 9212, 8845, 7605, 8298, 7304, 8256] avg=8192.0000 max=10155 max_rank=0 min=5861 min_rank=1 imbalance_ratio=1.239624 max_load_reduction=55381 max_load_reduction_pct=0.845047 moved_to_echo=31463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1145, 2, 5, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 550, 116, 0, 2, 0, 2, 3044, 1025, 510, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3845, 4066, 0, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 15, 4086, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 36, 0, 0, 0, 7, 3014, 454, 38, 0, 0, 0, 0, 0, 20, 3942, 0, 0, 114, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 3530, 0, 3912, 4079, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 134, 0, 0, 3528, 95, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 9027, 8996, 8109, 7645, 8168, 11553, 7853] avg=8192.0000 max=11553 max_rank=6 min=4185 min_rank=0 imbalance_ratio=1.410278 max_load_reduction=53983 max_load_reduction_pct=0.823715 moved_to_echo=35219 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 71, 0, 833, 0, 1329, 2868, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3741, 0, 0, 5, 151, 206, 969, 514, 0, 0, 0, 0, 0, 0, 4077, 38, 0, 90, 3930, 307, 0, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2, 0, 0, 0, 0, 4061, 3942, 0, 0, 0, 0, 0, 0, 0, 82, 4095, 0, 0, 0, 0, 0, 3869, 2869, 454, 36, 0, 0, 0, 0, 0, 11, 2002, 0, 317, 113, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 206, 2, 3932, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 8, 1, 0, 0, 23, 3526, 94, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5116, 9680, 9472, 8034, 11405, 6439, 7704, 7686] avg=8192.0000 max=11405 max_rank=4 min=5116 min_rank=0 imbalance_ratio=1.392212 max_load_reduction=54131 max_load_reduction_pct=0.825974 moved_to_echo=33234 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 186, 64, 4054, 2741, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 4096, 13, 0, 218, 0, 0, 935, 506, 0, 0, 0, 0, 0, 0, 2798, 0, 1, 38, 4093, 163, 0, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1, 23, 7, 0, 4087, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4, 0, 220, 0, 11, 784, 2850, 456, 41, 0, 0, 0, 0, 0, 0, 4096, 0, 31, 106, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 3473, 53, 1377, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4, 2946, 0, 100, 3523, 95, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7047, 5777, 8112, 8107, 8442, 8290, 8999, 10762] avg=8192.0000 max=10762 max_rank=7 min=5777 min_rank=1 imbalance_ratio=1.313721 max_load_reduction=54774 max_load_reduction_pct=0.835785 moved_to_echo=36493 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4076, 46, 4096, 2998, 0, 0, 0, 0, 0, 0, 0, 0, 2577, 373, 4065, 0, 0, 0, 0, 1726, 1014, 516, 0, 0, 0, 0, 0, 0, 71, 19, 26, 4096, 3451, 0, 0, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 38, 0, 66, 4061, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 144, 0, 0, 0, 0, 2967, 453, 38, 0, 0, 0, 0, 0, 3, 3921, 0, 1645, 113, 0, 0, 2619, 0, 0, 0, 0, 0, 0, 0, 0, 3572, 49, 27, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 20, 0, 0, 3521, 93, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11216, 10271, 8728, 8066, 7569, 8301, 3653, 7732] avg=8192.0000 max=11216 max_rank=0 min=3653 min_rank=6 imbalance_ratio=1.369141 max_load_reduction=54320 max_load_reduction_pct=0.828857 moved_to_echo=27255 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 3827, 7, 4096, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 367, 1, 4093, 0, 0, 0, 3, 108, 1025, 517, 0, 0, 0, 0, 0, 0, 237, 0, 0, 3487, 4096, 74, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 33, 4096, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3012, 454, 38, 0, 0, 0, 0, 0, 0, 4096, 0, 473, 114, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3579, 0, 246, 3400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 0, 0, 5, 0, 0, 3528, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10977, 6114, 8976, 8139, 7600, 8777, 7225, 7728] avg=8192.0000 max=10977 max_rank=0 min=6114 min_rank=1 imbalance_ratio=1.339966 max_load_reduction=54559 max_load_reduction_pct=0.832504 moved_to_echo=31330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 21248, 256, 20608, 23680, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[28544, 384, 12416, 24448, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 256, 128, 128, 128, 3584, 128, 128, 28288, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 30208, 128, 3328, 1024, 128, 128, 31104, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[18304, 1408, 28288, 128, 128, 256, 256, 5376, 8064, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7296, 128, 128, 17280, 31872, 768, 0, 8576, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32640, 128, 256, 256, 128, 128, 4864, 23680, 3712, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 768, 128, 256, 128, 32768, 31872, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4094, 70, 0, 0, 0, 1, 1321, 0, 0, 0, 4089, 0, 4096, 2, 4096, 3965, 0, 4074, 0, 0, 2, 0, 6, 4096, 0, 0, 4089, 0, 167, 0, 0, 24, 4096, 0, 4073, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 104, 3090, 0, 90, 3941, 0, 3858, 4034, 3788, 2, 4041, 14, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[2, 0, 0, 20, 0, 4096, 1044, 0, 0, 0, 0, 2870, 0, 0, 0, 4062, 0, 3909, 1, 4095, 3786, 12, 4096, 0, 0, 0, 0, 2487, 4096, 0, 0, 2775, 0, 370, 0, 0, 0, 4096, 117, 241, 0, 2739, 0, 0, 0, 0, 2, 0, 0, 3727, 0, 0, 3986, 0, 0, 4094, 2, 4094, 4096, 450, 0, 168, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 441, 7, 0, 0, 0, 470, 3989, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 2622, 4095, 0, 0, 3965, 0, 233, 0, 0, 0, 3920, 55, 4087, 0, 570, 0, 0, 0, 0, 4075, 0, 0, 0, 19, 3, 4096, 0, 1, 4096, 0, 4096, 4096, 17, 0, 100, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1199, 1745, 48, 0, 0, 0, 3939, 0, 0, 0, 4085, 0, 4096, 0, 4095, 3581, 0, 4096, 0, 34, 63, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 3780, 4045, 17, 0, 1247, 0, 0, 1, 0, 0, 0, 0, 4084, 480, 9, 4027, 0, 164, 0, 0, 4080, 4096, 277, 9, 4047, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[3, 0, 96, 4, 0, 4073, 3508, 0, 179, 0, 222, 1702, 388, 2, 0, 3913, 25, 4074, 61, 3555, 4096, 0, 4035, 26, 1615, 0, 69, 0, 3973, 0, 0, 0, 1, 0, 0, 0, 0, 4038, 9, 4079, 7, 22, 0, 44, 8, 0, 533, 0, 0, 12, 0, 4081, 3740, 0, 0, 4094, 8, 1345, 4034, 0, 118, 3744, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1, 19, 6, 4, 2, 4006, 0, 89, 0, 3, 1523, 4093, 0, 0, 0, 4095, 0, 3702, 0, 4088, 4084, 0, 4095, 8, 0, 158, 29, 19, 961, 0, 0, 4050, 0, 303, 0, 0, 5, 3168, 356, 0, 5, 4089, 0, 0, 5, 0, 762, 1, 54, 4075, 21, 2539, 4093, 1, 1, 3570, 10, 3347, 4095, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3703, 0, 0, 0, 0, 91, 4003, 0, 0, 0, 4096, 0, 4095, 0, 4096, 4096, 1, 4096, 0, 10, 0, 0, 0, 711, 0, 0, 4040, 0, 590, 0, 0, 0, 4096, 144, 1414, 0, 3577, 0, 0, 0, 0, 2362, 0, 0, 4093, 0, 779, 3156, 0, 0, 4095, 0, 4093, 4096, 0, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[5, 0, 0, 1, 0, 3853, 28, 0, 112, 7, 1441, 4003, 55, 0, 0, 4091, 0, 4095, 0, 4030, 4096, 0, 4063, 18, 0, 0, 0, 2647, 3799, 4, 34, 427, 21, 337, 0, 0, 0, 4055, 0, 3971, 0, 3834, 0, 0, 69, 0, 3, 1, 36, 31, 238, 3592, 63, 0, 270, 3935, 7, 4096, 4094, 0, 3, 0, 71, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 20, 0, 4096, 1044, 0, 2675, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 0, 0, 0, 4062, 325, 0, 0, 0, 0, 0, 0, 0, 0, 175, 1, 4095, 3786, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2487, 4096, 0, 0, 2775, 811, 0, 0, 0, 0, 0, 0, 0, 0, 370, 0, 0, 0, 4096, 117, 241, 837, 494, 0, 0, 0, 0, 0, 0, 0, 2739, 0, 0, 0, 0, 2, 0, 4096, 1059, 0, 0, 0, 0, 0, 0, 0, 3727, 0, 0, 3986, 0, 0, 1587, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 2466, 450, 0, 168, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9507, 7257, 8069, 10169, 6155, 7896, 9300, 7183] avg=8192.0000 max=10169 max_rank=3 min=6155 min_rank=4 imbalance_ratio=1.241333 max_load_reduction=55367 max_load_reduction_pct=0.844833 moved_to_echo=30534 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4094, 70, 0, 2803, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1321, 0, 0, 0, 4089, 320, 0, 0, 0, 0, 0, 0, 0, 0, 183, 2, 4096, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 4096, 0, 0, 4089, 799, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 24, 4096, 0, 4073, 806, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 4074, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 3090, 0, 90, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3858, 2429, 3788, 2, 4041, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8574, 5731, 8246, 8992, 9652, 5396, 4812, 14133] avg=8192.0000 max=14133 max_rank=7 min=4812 min_rank=6 imbalance_ratio=1.725220 max_load_reduction=51403 max_load_reduction_pct=0.784348 moved_to_echo=33993 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 441, 7, 0, 2803, 1671, 0, 0, 0, 0, 0, 0, 0, 0, 470, 3989, 0, 0, 0, 4096, 325, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2622, 4095, 0, 0, 3965, 811, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 3920, 55, 4087, 837, 494, 0, 0, 0, 0, 0, 0, 0, 570, 0, 0, 0, 0, 4075, 0, 4096, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3, 4096, 0, 1, 1588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2466, 17, 0, 100, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4922, 8880, 8375, 11493, 9626, 9851, 5707, 6682] avg=8192.0000 max=11493 max_rank=3 min=4922 min_rank=0 imbalance_ratio=1.402954 max_load_reduction=54043 max_load_reduction_pct=0.824631 moved_to_echo=31866 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1199, 1745, 48, 2803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 0, 0, 0, 4085, 325, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 4095, 3581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 63, 0, 0, 4095, 0, 0, 4096, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 4045, 17, 0, 494, 0, 0, 0, 0, 0, 0, 0, 1247, 0, 0, 1, 0, 0, 0, 4096, 1110, 0, 0, 0, 0, 0, 0, 0, 4084, 480, 9, 4027, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 2466, 277, 9, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5796, 8349, 7859, 9099, 8336, 6454, 8764, 10879] avg=8192.0000 max=10879 max_rank=7 min=5796 min_rank=0 imbalance_ratio=1.328003 max_load_reduction=54657 max_load_reduction_pct=0.834000 moved_to_echo=34433 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 1, 0, 3853, 28, 0, 2802, 1605, 0, 0, 0, 0, 0, 0, 112, 7, 1441, 4003, 55, 0, 0, 4091, 325, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 4030, 4096, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2647, 3799, 4, 34, 427, 811, 0, 0, 0, 0, 0, 0, 0, 21, 337, 0, 0, 0, 4055, 0, 3971, 804, 493, 0, 0, 0, 0, 0, 0, 0, 3834, 0, 0, 69, 0, 3, 1, 4063, 1109, 0, 0, 0, 0, 0, 0, 36, 31, 238, 3592, 63, 0, 270, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 2465, 0, 3, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 10034, 8328, 7722, 9681, 9079, 5756, 6642] avg=8192.0000 max=10034 max_rank=1 min=5756 min_rank=6 imbalance_ratio=1.224854 max_load_reduction=55502 max_load_reduction_pct=0.846893 moved_to_echo=31158 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 19, 6, 4, 2, 4006, 0, 89, 2539, 1460, 0, 0, 0, 0, 0, 0, 0, 3, 1523, 4093, 0, 0, 0, 4095, 330, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 4088, 4084, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 29, 19, 961, 0, 0, 4050, 816, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 5, 3168, 356, 0, 733, 496, 0, 0, 0, 0, 0, 0, 5, 4089, 0, 0, 5, 0, 762, 1, 4095, 1006, 0, 0, 0, 0, 0, 0, 54, 4075, 21, 2539, 4093, 1, 1, 1377, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3347, 2453, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 10044, 8337, 6033, 5061, 9963, 12161, 5811] avg=8192.0000 max=12161 max_rank=6 min=5061 min_rank=4 imbalance_ratio=1.484497 max_load_reduction=53375 max_load_reduction_pct=0.814438 moved_to_echo=32996 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 96, 4, 0, 4073, 3508, 0, 2788, 1670, 0, 0, 0, 0, 0, 0, 179, 0, 222, 1702, 388, 2, 0, 3913, 320, 0, 0, 0, 0, 0, 0, 0, 25, 182, 61, 3555, 4096, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 0, 69, 0, 3973, 0, 0, 0, 799, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4038, 9, 4079, 837, 486, 0, 0, 0, 0, 0, 0, 7, 22, 0, 44, 8, 0, 533, 0, 4035, 1104, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4081, 3740, 0, 0, 1587, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1345, 2429, 0, 118, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12142, 6726, 7945, 6456, 9450, 5753, 9420, 7644] avg=8192.0000 max=12142 max_rank=0 min=5753 min_rank=5 imbalance_ratio=1.482178 max_load_reduction=53394 max_load_reduction_pct=0.814728 moved_to_echo=32267 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3703, 0, 0, 2802, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 91, 4003, 0, 0, 0, 4096, 325, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 711, 0, 0, 4040, 811, 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 0, 4096, 144, 1414, 837, 494, 0, 0, 0, 0, 0, 0, 0, 3577, 0, 0, 0, 0, 2362, 0, 4096, 1109, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 779, 3156, 0, 0, 1588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2466, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8515, 8377, 5572, 7575, 11144, 9616, 6562] avg=8192.0000 max=11144 max_rank=5 min=5572 min_rank=3 imbalance_ratio=1.360352 max_load_reduction=54392 max_load_reduction_pct=0.829956 moved_to_echo=34897 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 16128, 768, 11136, 26368, 128, 640, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 25472, 6528, 256, 22016, 11392, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 16128, 0, 128, 128, 0, 8064, 128, 32768, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 1536, 128, 32256, 31872, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 29056, 19712, 4608, 256, 12160, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1664, 256, 128, 7808, 25856, 128, 128, 23552, 6528, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 2048, 0, 0, 128, 31360, 4736, 17920, 5760, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 128, 3840, 25984, 512, 128, 0, 32640, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[200, 2817, 1324, 127, 1741, 30, 717, 288, 1551, 90, 1416, 273, 874, 769, 56, 514, 298, 1524, 1076, 1219, 1730, 458, 2306, 1549, 71, 1538, 538, 1423, 1103, 162, 1264, 2111, 3328, 401, 357, 286, 101, 1376, 2223, 1202, 2290, 1446, 1498, 646, 623, 2197, 1287, 682, 289, 709, 373, 1246, 240, 533, 1677, 170, 298, 592, 2651, 2416, 1224, 428, 1032, 558] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[152, 2110, 824, 118, 2174, 90, 818, 748, 1186, 100, 1290, 627, 635, 531, 47, 509, 275, 1289, 1376, 1525, 1922, 520, 2875, 1555, 63, 1550, 544, 736, 869, 298, 1414, 1859, 3708, 355, 429, 240, 583, 998, 2560, 619, 2539, 1165, 1538, 611, 654, 2172, 1283, 913, 480, 443, 465, 1278, 369, 213, 2308, 356, 296, 716, 2279, 2326, 857, 755, 983, 416] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[74, 2647, 1714, 41, 1896, 80, 931, 585, 902, 96, 718, 631, 797, 363, 139, 422, 182, 841, 1550, 1658, 1829, 441, 2518, 1751, 130, 1864, 500, 1196, 1103, 276, 1340, 1785, 3593, 425, 332, 862, 303, 897, 1953, 770, 2803, 1531, 1437, 692, 569, 2281, 1175, 681, 342, 442, 420, 1292, 464, 366, 1778, 105, 199, 757, 1954, 2569, 1139, 397, 1169, 839] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[124, 2253, 1016, 129, 2392, 52, 756, 525, 1070, 111, 1551, 589, 1015, 732, 149, 354, 180, 778, 1391, 1150, 1762, 671, 2480, 1873, 152, 1573, 463, 1241, 1073, 474, 2072, 1725, 2883, 274, 396, 606, 278, 1230, 1915, 425, 2530, 1741, 1531, 828, 859, 1825, 552, 1500, 426, 250, 371, 1243, 861, 323, 1516, 102, 80, 1160, 2415, 2234, 1112, 289, 1421, 484] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[100, 2495, 973, 275, 1988, 112, 1124, 540, 1306, 96, 1449, 273, 787, 600, 173, 639, 162, 852, 1728, 1278, 1699, 638, 2012, 1668, 226, 1609, 814, 591, 1089, 322, 1802, 1981, 2774, 269, 537, 331, 212, 1002, 2210, 558, 2861, 1570, 1744, 905, 496, 2017, 1126, 662, 365, 463, 939, 1673, 659, 537, 1613, 163, 129, 1097, 2537, 1680, 945, 687, 616, 758] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[61, 1797, 948, 59, 2068, 55, 1052, 245, 1442, 101, 1151, 874, 981, 1061, 28, 499, 159, 1438, 1282, 1428, 1993, 510, 2511, 2189, 56, 1789, 350, 1354, 1076, 351, 1861, 2354, 3547, 443, 309, 635, 131, 1000, 1269, 721, 2280, 1391, 1407, 364, 613, 2336, 1099, 759, 183, 428, 513, 882, 424, 309, 1853, 104, 440, 1113, 2110, 2647, 1064, 451, 862, 726] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[39, 2837, 793, 181, 1178, 131, 792, 212, 801, 266, 865, 778, 770, 717, 64, 657, 377, 836, 1758, 980, 2428, 287, 2426, 1710, 43, 2129, 475, 1066, 1033, 311, 1103, 1990, 3325, 847, 257, 854, 294, 1359, 2021, 702, 2440, 911, 1358, 549, 556, 2130, 1622, 999, 401, 332, 457, 1312, 761, 1073, 1624, 113, 799, 778, 2596, 1505, 855, 761, 1049, 863] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[62, 2699, 1206, 94, 1606, 45, 854, 437, 1559, 123, 1174, 392, 694, 701, 54, 405, 154, 716, 2105, 887, 2130, 661, 2204, 1847, 67, 2025, 468, 1110, 1449, 423, 1882, 2001, 3050, 291, 811, 955, 170, 1253, 2007, 713, 2778, 1351, 1204, 556, 422, 1790, 1126, 703, 353, 368, 515, 1665, 659, 419, 1147, 103, 237, 859, 2182, 1698, 1348, 480, 1530, 559] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[152, 2110, 824, 118, 2174, 90, 818, 748, 739, 255, 91, 84, 19, 0, 0, 0, 1186, 100, 1290, 627, 635, 531, 47, 509, 2539, 509, 0, 0, 0, 0, 0, 0, 275, 1289, 1376, 1525, 1831, 520, 0, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 63, 1550, 544, 736, 869, 298, 1414, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 2969, 355, 429, 240, 583, 998, 2560, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 1538, 611, 654, 2153, 1283, 913, 0, 0, 0, 0, 0, 0, 0, 0, 480, 443, 465, 1278, 369, 213, 2308, 356, 2366, 702, 0, 0, 0, 0, 0, 0, 296, 716, 1493, 2326, 857, 755, 983, 416, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 7973, 8371, 7078, 8753, 8317, 8980, 7842] avg=8192.0000 max=8980 max_rank=6 min=7078 min_rank=3 imbalance_ratio=1.096191 max_load_reduction=56556 max_load_reduction_pct=0.862976 moved_to_echo=33892 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[200, 2817, 1324, 127, 1741, 30, 717, 288, 664, 290, 82, 98, 19, 0, 0, 0, 1551, 90, 1416, 273, 874, 769, 56, 514, 2290, 409, 0, 0, 0, 0, 0, 0, 298, 1524, 1076, 1219, 1648, 458, 0, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 71, 1538, 538, 1423, 1103, 162, 1264, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 2664, 401, 357, 286, 101, 1376, 2223, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1446, 1498, 646, 623, 2178, 1287, 682, 0, 0, 0, 0, 0, 0, 0, 0, 289, 709, 373, 1246, 240, 533, 1677, 170, 1897, 817, 0, 0, 0, 0, 0, 0, 298, 592, 1736, 2416, 1224, 428, 1032, 558, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8397, 8242, 7772, 7920, 8610, 8360, 7951, 8284] avg=8192.0000 max=8610 max_rank=4 min=7772 min_rank=2 imbalance_ratio=1.051025 max_load_reduction=56926 max_load_reduction_pct=0.868622 moved_to_echo=33205 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[74, 2647, 1714, 41, 1896, 80, 931, 585, 716, 245, 87, 72, 20, 0, 0, 0, 902, 96, 718, 631, 797, 363, 139, 422, 2803, 446, 0, 0, 0, 0, 0, 0, 182, 841, 1550, 1658, 1742, 441, 0, 1751, 0, 0, 0, 0, 0, 0, 0, 0, 130, 1864, 500, 1196, 1103, 276, 1340, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 2877, 425, 332, 862, 303, 897, 1953, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1531, 1437, 692, 569, 2261, 1175, 681, 0, 0, 0, 0, 0, 0, 0, 0, 342, 442, 420, 1292, 464, 366, 1778, 105, 2072, 602, 0, 0, 0, 0, 0, 0, 199, 757, 1280, 2569, 1139, 397, 1169, 839, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9108, 7317, 8165, 7949, 8419, 8346, 7883, 8349] avg=8192.0000 max=9108 max_rank=0 min=7317 min_rank=1 imbalance_ratio=1.111816 max_load_reduction=56428 max_load_reduction_pct=0.861023 moved_to_echo=32997 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[124, 2253, 1016, 129, 2392, 52, 756, 525, 575, 237, 83, 90, 16, 0, 0, 0, 1070, 111, 1551, 589, 1015, 732, 149, 354, 2530, 440, 0, 0, 0, 0, 0, 0, 180, 778, 1391, 1150, 1679, 671, 0, 1873, 0, 0, 0, 0, 0, 0, 0, 0, 152, 1573, 463, 1241, 1073, 474, 2072, 1488, 0, 0, 0, 0, 0, 0, 0, 0, 2308, 274, 396, 606, 278, 1230, 1915, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1741, 1531, 828, 859, 1809, 552, 1500, 0, 0, 0, 0, 0, 0, 0, 0, 426, 250, 371, 1243, 861, 323, 1516, 102, 2040, 744, 0, 0, 0, 0, 0, 0, 80, 1160, 1581, 2234, 1112, 289, 1421, 484, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8541, 7722, 8536, 7432, 8820, 7876, 8361] avg=8192.0000 max=8820 max_rank=5 min=7432 min_rank=4 imbalance_ratio=1.076660 max_load_reduction=56716 max_load_reduction_pct=0.865417 moved_to_echo=32489 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[61, 1797, 948, 59, 2068, 55, 1052, 245, 707, 323, 94, 78, 20, 0, 0, 0, 1442, 101, 1151, 874, 981, 1061, 28, 499, 2280, 446, 0, 0, 0, 0, 0, 0, 159, 1438, 1282, 1428, 1899, 510, 0, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 56, 1789, 350, 1354, 1076, 351, 1861, 2031, 0, 0, 0, 0, 0, 0, 0, 0, 2840, 443, 309, 635, 131, 1000, 1269, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1391, 1407, 364, 613, 2316, 1099, 759, 0, 0, 0, 0, 0, 0, 0, 0, 183, 428, 513, 882, 424, 309, 1853, 104, 2065, 650, 0, 0, 0, 0, 0, 0, 440, 1113, 1382, 2647, 1064, 451, 862, 726, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7507, 8863, 8905, 8868, 7348, 7949, 7411, 8685] avg=8192.0000 max=8905 max_rank=2 min=7348 min_rank=4 imbalance_ratio=1.087036 max_load_reduction=56631 max_load_reduction_pct=0.864120 moved_to_echo=31393 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[39, 2837, 793, 181, 1178, 131, 792, 212, 667, 277, 119, 101, 21, 0, 0, 0, 801, 266, 865, 778, 770, 717, 64, 657, 2440, 431, 0, 0, 0, 0, 0, 0, 377, 836, 1758, 980, 2309, 287, 0, 1710, 0, 0, 0, 0, 0, 0, 0, 0, 43, 2129, 475, 1066, 1033, 311, 1103, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 847, 257, 854, 294, 1359, 2021, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 911, 1358, 549, 556, 2109, 1622, 999, 0, 0, 0, 0, 0, 0, 0, 0, 401, 332, 457, 1312, 761, 1073, 1624, 113, 1995, 803, 0, 0, 0, 0, 0, 0, 799, 778, 1692, 1505, 855, 761, 1049, 863, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7348, 7789, 8257, 7873, 8992, 8104, 8871, 8302] avg=8192.0000 max=8992 max_rank=4 min=7348 min_rank=0 imbalance_ratio=1.097656 max_load_reduction=56544 max_load_reduction_pct=0.862793 moved_to_echo=34269 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[100, 2495, 973, 275, 1988, 112, 1124, 540, 553, 272, 80, 94, 17, 0, 0, 0, 1306, 96, 1449, 273, 787, 600, 173, 639, 2861, 357, 0, 0, 0, 0, 0, 0, 162, 852, 1728, 1278, 1619, 638, 0, 1668, 0, 0, 0, 0, 0, 0, 0, 0, 226, 1609, 814, 591, 1089, 322, 1802, 1709, 0, 0, 0, 0, 0, 0, 0, 0, 2221, 269, 537, 331, 212, 1002, 2210, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1570, 1744, 905, 496, 2000, 1126, 662, 0, 0, 0, 0, 0, 0, 0, 0, 365, 463, 939, 1673, 659, 537, 1613, 163, 1655, 782, 0, 0, 0, 0, 0, 0, 129, 1097, 1661, 1680, 945, 687, 616, 758, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8623, 8541, 7945, 8162, 7340, 8503, 8849, 7573] avg=8192.0000 max=8849 max_rank=6 min=7340 min_rank=4 imbalance_ratio=1.080200 max_load_reduction=56687 max_load_reduction_pct=0.864975 moved_to_echo=32265 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[62, 2699, 1206, 94, 1606, 45, 854, 437, 608, 275, 101, 81, 15, 0, 0, 0, 1559, 123, 1174, 392, 694, 701, 54, 405, 2778, 390, 0, 0, 0, 0, 0, 0, 154, 716, 2105, 887, 2029, 661, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 0, 67, 2025, 468, 1110, 1449, 423, 1882, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 2442, 291, 811, 955, 170, 1253, 2007, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 1204, 556, 422, 1775, 1126, 703, 0, 0, 0, 0, 0, 0, 0, 0, 353, 368, 515, 1665, 659, 419, 1147, 103, 1814, 672, 0, 0, 0, 0, 0, 0, 237, 859, 1429, 1698, 1348, 480, 1530, 559, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8083, 8270, 8399, 9150, 8642, 7137, 7715, 8140] avg=8192.0000 max=9150 max_rank=3 min=7137 min_rank=5 imbalance_ratio=1.116943 max_load_reduction=56386 max_load_reduction_pct=0.860382 moved_to_echo=31634 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[896, 19712, 8832, 1024, 15104, 640, 7168, 3584, 5248, 2176, 768, 768, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 11136, 11776, 5248, 4864, 16640, 9344, 6912, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2560, 7168, 12288, 17152, 8576, 4352, 8704, 5248, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2944, 3456, 4096, 10624, 4480, 3840, 13568, 1280, 16000, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[896, 14080, 4224, 8832, 8832, 2688, 12800, 13696, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1792, 8320, 12288, 10240, 14848, 4224, 0, 14208, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9856, 1024, 9728, 4480, 6656, 5504, 768, 4096, 20608, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[20992, 3328, 3456, 4864, 2176, 9216, 16256, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4077, 0, 0, 445, 0, 26, 0, 3, 2, 3978, 97, 0, 3, 0, 57, 0, 1, 0, 1679, 4083, 4019, 3896, 0, 4029, 4087, 1, 7, 478, 1866, 1, 0, 8, 4089, 2665, 0, 4076, 113, 1998, 6, 0, 17, 6, 2401, 3451, 48, 1, 819, 101, 2, 511, 0, 3415, 16, 675, 1, 6, 0, 1, 4, 69, 4096, 4093, 0, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3409, 0, 0, 68, 0, 6, 0, 78, 66, 4096, 83, 72, 440, 109, 657, 41, 25, 33, 2421, 4003, 4006, 3670, 10, 4090, 4059, 0, 30, 1, 25, 0, 3, 73, 2651, 4095, 66, 4040, 0, 759, 1, 1, 2132, 1, 3314, 2944, 0, 8, 0, 0, 17, 244, 6, 2629, 0, 2795, 0, 1, 0, 0, 0, 32, 3983, 4022, 0, 221] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3790, 0, 0, 5, 0, 31, 3, 4, 0, 4038, 304, 4, 0, 4, 6, 649, 0, 0, 3568, 4038, 4090, 3760, 0, 4094, 3933, 0, 0, 1, 48, 158, 0, 9, 775, 4078, 0, 4096, 0, 202, 37, 139, 3265, 2, 266, 2248, 2958, 0, 106, 0, 0, 703, 1, 3556, 0, 2325, 0, 0, 1, 0, 0, 3, 3862, 3834, 0, 542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 13, 1, 4, 0, 3, 0, 48, 1, 3528, 67, 2, 98, 1, 963, 2, 0, 0, 2824, 4094, 3859, 4075, 0, 4092, 4086, 18, 3, 325, 908, 3, 267, 19, 1628, 3122, 0, 4096, 5, 11, 0, 2, 9, 5, 3490, 3965, 235, 0, 411, 52, 0, 3422, 27, 521, 2, 1653, 1, 0, 376, 1, 0, 4, 4074, 4067, 0, 957] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4030, 9, 1, 174, 0, 25, 2, 59, 3, 4087, 6, 0, 0, 5, 10, 10, 0, 12, 2262, 4096, 3992, 4036, 0, 4086, 4094, 1, 1, 4, 2341, 10, 5, 1, 3017, 4085, 0, 4078, 16, 91, 0, 0, 251, 28, 2465, 4029, 4, 2, 43, 0, 2, 770, 11, 939, 31, 3830, 1, 1, 2, 62, 0, 6, 3970, 3806, 37, 597] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[3884, 0, 278, 49, 0, 66, 6, 254, 1, 3393, 15, 0, 3, 0, 1793, 13, 0, 8, 1096, 2837, 3913, 4006, 0, 4060, 4093, 0, 41, 1863, 528, 342, 911, 1, 634, 3546, 8, 3930, 1, 15, 294, 3, 947, 115, 4085, 4083, 189, 9, 88, 5, 220, 2288, 0, 3092, 0, 329, 0, 288, 381, 0, 88, 0, 3698, 3678, 66, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4027, 0, 0, 675, 3, 50, 103, 4, 36, 3992, 10, 0, 6, 2, 289, 0, 42, 0, 719, 3553, 4069, 4005, 1, 3962, 4090, 1, 179, 460, 4, 0, 99, 71, 4044, 2771, 0, 4094, 563, 0, 2, 0, 893, 1, 4086, 4082, 0, 0, 1, 5, 6, 2291, 25, 3562, 2, 117, 2, 1, 9, 23, 0, 558, 3863, 4072, 0, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4075, 26, 0, 5, 0, 66, 43, 2, 10, 3557, 125, 0, 5, 0, 290, 5, 0, 1, 634, 4069, 4080, 4094, 0, 4079, 4094, 1, 70, 707, 40, 1, 820, 1, 4026, 2298, 4, 4084, 0, 2, 28, 1, 19, 6, 2766, 3976, 0, 0, 21, 17, 1, 2383, 10, 3476, 2, 3743, 16, 0, 224, 0, 0, 0, 4032, 3486, 15, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4077, 0, 0, 445, 0, 26, 0, 3, 3908, 0, 0, 0, 0, 0, 0, 0, 2, 3978, 97, 0, 3, 0, 57, 0, 3454, 121, 0, 0, 0, 0, 0, 0, 1, 0, 1679, 4083, 0, 2400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 7, 478, 1866, 1, 0, 8, 1974, 565, 0, 0, 0, 0, 0, 0, 4089, 2665, 0, 1712, 113, 1998, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 6, 2401, 3451, 48, 1, 819, 101, 138, 0, 0, 0, 0, 0, 0, 0, 2, 511, 0, 3415, 16, 675, 1, 6, 1496, 390, 44, 0, 0, 0, 0, 0, 0, 1, 4, 69, 3914, 4093, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8459, 7712, 8163, 8987, 10583, 6982, 6556, 8094] avg=8192.0000 max=10583 max_rank=4 min=6556 min_rank=6 imbalance_ratio=1.291870 max_load_reduction=54953 max_load_reduction_pct=0.838516 moved_to_echo=32215 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3409, 0, 0, 68, 0, 6, 0, 78, 3966, 0, 0, 0, 0, 0, 0, 0, 66, 4096, 83, 72, 440, 109, 657, 41, 3442, 124, 0, 0, 0, 0, 0, 0, 25, 33, 2421, 4003, 0, 2261, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 30, 1, 25, 0, 3, 73, 1956, 564, 0, 0, 0, 0, 0, 0, 2651, 4095, 66, 1697, 0, 759, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2132, 1, 3314, 2944, 0, 8, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 17, 244, 6, 2629, 0, 2795, 0, 1, 1409, 387, 42, 0, 0, 0, 0, 0, 0, 0, 0, 32, 3807, 4022, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7527, 9130, 8753, 6711, 9270, 8533, 7530, 8082] avg=8192.0000 max=9270 max_rank=4 min=6711 min_rank=3 imbalance_ratio=1.131592 max_load_reduction=56266 max_load_reduction_pct=0.858551 moved_to_echo=33415 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4075, 26, 0, 5, 0, 66, 43, 2, 3955, 0, 0, 0, 0, 0, 0, 0, 10, 3557, 125, 0, 5, 0, 290, 5, 3505, 124, 0, 0, 0, 0, 0, 0, 0, 1, 634, 4069, 0, 2522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 70, 707, 40, 1, 820, 1, 1977, 575, 0, 0, 0, 0, 0, 0, 4026, 2298, 4, 1716, 0, 2, 28, 1, 0, 0, 0, 0, 0, 0, 0, 0, 19, 6, 2766, 3976, 0, 0, 21, 17, 135, 0, 0, 0, 0, 0, 0, 0, 1, 2383, 10, 3476, 2, 3743, 16, 0, 1572, 391, 43, 0, 0, 0, 0, 0, 224, 0, 0, 0, 3854, 3486, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 7621, 7226, 8286, 8075, 6940, 11637, 7579] avg=8192.0000 max=11637 max_rank=6 min=6940 min_rank=5 imbalance_ratio=1.420532 max_load_reduction=53899 max_load_reduction_pct=0.822433 moved_to_echo=34231 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3790, 0, 0, 5, 0, 31, 3, 4, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 304, 4, 0, 4, 6, 649, 3514, 124, 0, 0, 0, 0, 0, 0, 0, 0, 3568, 4038, 0, 2316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 0, 1, 48, 158, 0, 9, 1983, 576, 0, 0, 0, 0, 0, 0, 775, 4078, 0, 1721, 0, 202, 37, 139, 0, 0, 0, 0, 0, 0, 0, 0, 3265, 2, 266, 2248, 2958, 0, 106, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 703, 1, 3556, 0, 2325, 0, 0, 1444, 392, 41, 0, 0, 0, 0, 0, 1, 0, 0, 3, 3691, 3834, 0, 542, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7803, 8643, 9922, 6708, 6952, 8975, 8462, 8071] avg=8192.0000 max=9922 max_rank=2 min=6708 min_rank=3 imbalance_ratio=1.211182 max_load_reduction=55614 max_load_reduction_pct=0.848602 moved_to_echo=32460 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 13, 1, 4, 0, 3, 0, 48, 3968, 0, 0, 0, 0, 0, 0, 0, 1, 3528, 67, 2, 98, 1, 963, 2, 3315, 124, 0, 0, 0, 0, 0, 0, 0, 0, 2824, 4094, 0, 2510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 18, 3, 325, 908, 3, 267, 19, 1983, 544, 0, 0, 0, 0, 0, 0, 1628, 3122, 0, 1721, 5, 11, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 9, 5, 3490, 3965, 235, 0, 411, 52, 137, 0, 0, 0, 0, 0, 0, 0, 0, 3422, 27, 521, 2, 1653, 1, 0, 1565, 392, 43, 0, 0, 0, 0, 0, 376, 1, 0, 4, 3894, 4067, 0, 957, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8101, 9428, 8156, 6489, 8304, 7626, 9299] avg=8192.0000 max=9428 max_rank=2 min=6489 min_rank=4 imbalance_ratio=1.150879 max_load_reduction=56108 max_load_reduction_pct=0.856140 moved_to_echo=31718 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3884, 0, 278, 49, 0, 66, 6, 254, 3938, 0, 0, 0, 0, 0, 0, 0, 1, 3393, 15, 0, 3, 0, 1793, 13, 3363, 122, 0, 0, 0, 0, 0, 0, 0, 8, 1096, 2837, 0, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 41, 1863, 528, 342, 911, 1, 1908, 550, 0, 0, 0, 0, 0, 0, 634, 3546, 8, 1643, 1, 15, 294, 3, 0, 0, 0, 0, 0, 0, 0, 0, 947, 115, 4085, 4083, 189, 9, 88, 5, 128, 0, 0, 0, 0, 0, 0, 0, 220, 2288, 0, 3092, 0, 329, 0, 288, 1541, 379, 44, 0, 0, 0, 0, 0, 381, 0, 88, 0, 3526, 3678, 66, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8475, 8703, 6406, 10237, 6144, 9649, 8181, 7741] avg=8192.0000 max=10237 max_rank=3 min=6144 min_rank=4 imbalance_ratio=1.249634 max_load_reduction=55299 max_load_reduction_pct=0.843796 moved_to_echo=31715 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4030, 9, 1, 174, 0, 25, 2, 59, 3962, 0, 0, 0, 0, 0, 0, 0, 3, 4087, 6, 0, 0, 5, 10, 10, 3430, 124, 0, 0, 0, 0, 0, 0, 0, 12, 2262, 4096, 0, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 4, 2341, 10, 5, 1, 1974, 562, 0, 0, 0, 0, 0, 0, 3017, 4085, 0, 1714, 16, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 28, 2465, 4029, 4, 2, 43, 0, 133, 0, 0, 0, 0, 0, 0, 0, 2, 770, 11, 939, 31, 3830, 1, 1, 1550, 390, 42, 0, 0, 0, 0, 0, 2, 62, 0, 6, 3795, 3806, 37, 597, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 7675, 8856, 8993, 8923, 6955, 7567, 8305] avg=8192.0000 max=8993 max_rank=3 min=6955 min_rank=5 imbalance_ratio=1.097778 max_load_reduction=56543 max_load_reduction_pct=0.862778 moved_to_echo=31750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4027, 0, 0, 675, 3, 50, 103, 4, 3843, 0, 0, 0, 0, 0, 0, 0, 36, 3992, 10, 0, 6, 2, 289, 0, 3497, 119, 0, 0, 0, 0, 0, 0, 42, 0, 719, 3553, 0, 2467, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 179, 460, 4, 0, 99, 71, 1982, 572, 0, 0, 0, 0, 0, 0, 4044, 2771, 0, 1720, 563, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 893, 1, 4086, 4082, 0, 0, 1, 5, 130, 0, 0, 0, 0, 0, 0, 0, 6, 2291, 25, 3562, 2, 117, 2, 1, 1538, 392, 41, 0, 0, 0, 0, 0, 9, 23, 0, 558, 3692, 4072, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8705, 7951, 6782, 7458, 9100, 9198, 7977, 8365] avg=8192.0000 max=9198 max_rank=5 min=6782 min_rank=2 imbalance_ratio=1.122803 max_load_reduction=56338 max_load_reduction_pct=0.859650 moved_to_echo=34640 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 15232, 30848, 0, 19456, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[20864, 26752, 128, 13696, 768, 3200, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1024, 128, 128, 768, 30208, 31104, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 12672, 128, 21248, 128, 15488, 128, 384, 12160, 3200, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[31488, 128, 384, 1536, 128, 384, 256, 512, 31616, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7552, 256, 22912, 28800, 3456, 128, 1536, 256, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 384, 3840, 5760, 640, 2176, 256, 15744, 4608, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 30720, 768, 128, 640, 128, 4096, 768, 27520, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[7, 644, 399, 0, 4094, 0, 25, 8, 3, 0, 3, 3941, 4075, 4010, 4038, 0, 0, 3994, 6, 0, 3373, 481, 8, 0, 94, 599, 3420, 67, 0, 4074, 238, 0, 4096, 36, 456, 1, 2, 3751, 1, 0, 4053, 10, 6, 0, 2, 294, 37, 0, 542, 0, 4076, 68, 0, 26, 4, 0, 4037, 2293, 5, 0, 4090, 5, 5, 39] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 3098, 4068, 0, 4092, 0, 0, 1107, 22, 0, 0, 4095, 4086, 1081, 1970, 27, 0, 36, 0, 7, 3841, 197, 1, 155, 142, 17, 3641, 0, 0, 4093, 3, 0, 3958, 1000, 37, 33, 6, 4086, 1, 1, 4090, 65, 2, 4, 13, 0, 598, 102, 19, 260, 4011, 0, 2, 0, 28, 0, 4068, 2946, 0, 0, 4095, 0, 71, 261] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 155, 3945, 0, 2623, 0, 0, 75, 885, 0, 0, 853, 4054, 4096, 4096, 234, 0, 3, 0, 0, 1739, 0, 0, 0, 78, 0, 4090, 0, 1, 4091, 228, 38, 3992, 3901, 1918, 1586, 0, 4096, 782, 0, 3770, 0, 0, 0, 0, 63, 213, 9, 0, 0, 3839, 6, 83, 0, 0, 0, 3796, 2249, 27, 0, 3911, 4, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 4033, 2713, 0, 4073, 0, 0, 193, 2320, 6, 0, 3724, 4096, 2400, 4096, 41, 1, 1255, 0, 0, 4095, 1, 0, 0, 6, 0, 4011, 0, 0, 4096, 294, 0, 201, 1105, 74, 20, 0, 4096, 0, 0, 4096, 0, 0, 49, 0, 369, 0, 178, 8, 3, 4091, 0, 0, 0, 0, 0, 3871, 25, 0, 0, 4096, 0, 0, 1800] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 345, 4085, 0, 167, 0, 0, 69, 355, 12, 5, 3948, 4091, 3026, 92, 258, 1, 2977, 0, 0, 4061, 278, 1, 0, 381, 0, 3222, 1, 0, 4087, 3755, 2, 4096, 545, 172, 228, 23, 4096, 11, 0, 4085, 5, 1, 72, 0, 0, 0, 6, 212, 66, 4090, 0, 0, 162, 0, 0, 3770, 689, 7, 0, 4070, 0, 0, 3911] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 3763, 3386, 0, 2569, 0, 3, 0, 22, 0, 0, 3030, 4093, 2882, 4082, 12, 47, 473, 4, 0, 3901, 0, 51, 0, 0, 2, 3780, 0, 0, 4057, 3320, 0, 4088, 2, 6, 639, 3, 4075, 0, 3, 4091, 1, 11, 0, 1, 0, 262, 2, 34, 0, 3995, 0, 1, 0, 8, 0, 4096, 0, 10, 1, 4031, 1, 0, 698] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1669, 958, 5, 4076, 0, 9, 188, 48, 1, 0, 1624, 4096, 3872, 4096, 39, 0, 3, 0, 2, 2633, 2958, 0, 2, 274, 0, 3983, 0, 33, 4077, 4074, 23, 3439, 90, 55, 1117, 0, 4016, 0, 0, 4093, 3, 80, 1, 4, 38, 0, 1098, 0, 0, 4049, 0, 0, 17, 0, 0, 4085, 249, 0, 0, 4060, 0, 1, 298] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[9, 666, 3539, 0, 2259, 213, 13, 0, 688, 0, 0, 4095, 4095, 306, 2400, 0, 29, 404, 2, 177, 1709, 1549, 123, 30, 0, 195, 3729, 0, 0, 3612, 922, 109, 3388, 1426, 0, 893, 54, 4005, 0, 1, 4094, 3, 0, 188, 2, 733, 839, 176, 2939, 0, 4066, 1, 3, 6, 38, 6, 3918, 4039, 0, 1, 3094, 84, 664, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[7, 644, 399, 0, 4094, 0, 25, 8, 255, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 2182, 0, 4010, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 6, 0, 3373, 481, 8, 0, 1759, 928, 715, 0, 0, 0, 0, 0, 94, 599, 3420, 67, 0, 2629, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 36, 456, 1, 2, 2568, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 10, 6, 0, 2, 294, 37, 0, 2289, 730, 487, 0, 0, 0, 0, 0, 542, 0, 4076, 68, 0, 26, 4, 0, 3588, 0, 0, 0, 0, 0, 0, 0, 1748, 2293, 5, 0, 4090, 5, 5, 39, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5432, 10236, 11264, 7047, 7160, 7908, 8304, 8185] avg=8192.0000 max=11264 max_rank=2 min=5432 min_rank=0 imbalance_ratio=1.375000 max_load_reduction=54272 max_load_reduction_pct=0.828125 moved_to_echo=31557 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3098, 4068, 0, 4092, 0, 0, 1107, 278, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 2268, 0, 1081, 1970, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 7, 3841, 197, 1, 155, 1827, 1011, 718, 0, 0, 0, 0, 0, 142, 17, 3641, 0, 0, 2642, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 1000, 37, 33, 6, 2797, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 65, 2, 4, 13, 0, 598, 102, 2307, 733, 489, 0, 0, 0, 0, 0, 19, 260, 4011, 0, 2, 0, 28, 0, 3597, 0, 0, 0, 0, 0, 0, 0, 1761, 2946, 0, 0, 4095, 0, 71, 261, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12643, 5368, 7793, 6445, 7833, 8403, 7917, 9134] avg=8192.0000 max=12643 max_rank=0 min=5368 min_rank=1 imbalance_ratio=1.543335 max_load_reduction=52893 max_load_reduction_pct=0.807083 moved_to_echo=33287 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 155, 3945, 0, 2623, 0, 0, 75, 278, 0, 0, 0, 0, 0, 0, 0, 885, 0, 0, 473, 0, 4096, 4096, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1739, 0, 0, 0, 380, 1014, 718, 0, 0, 0, 0, 0, 78, 0, 4090, 0, 1, 2640, 228, 38, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 3901, 1918, 1586, 0, 2804, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 0, 0, 0, 0, 63, 213, 9, 2152, 733, 484, 0, 0, 0, 0, 0, 0, 0, 3839, 6, 83, 0, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 1644, 2249, 27, 0, 3911, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7076, 9784, 3854, 7075, 14983, 7424, 7498, 7842] avg=8192.0000 max=14983 max_rank=4 min=3854 min_rank=2 imbalance_ratio=1.828979 max_load_reduction=50553 max_load_reduction_pct=0.771378 moved_to_echo=37747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4033, 2713, 0, 4073, 0, 0, 193, 278, 0, 0, 0, 0, 0, 0, 0, 2320, 6, 0, 2062, 0, 2400, 4096, 41, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1255, 0, 0, 4095, 1, 0, 0, 1662, 1014, 719, 0, 0, 0, 0, 0, 6, 0, 4011, 0, 0, 2643, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 1105, 74, 20, 0, 2804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 49, 0, 369, 0, 178, 2195, 734, 490, 0, 0, 0, 0, 0, 8, 3, 4091, 0, 0, 0, 0, 0, 3606, 0, 0, 0, 0, 0, 0, 0, 1676, 25, 0, 0, 4096, 0, 0, 1800, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11290, 10925, 8747, 6954, 4204, 8111, 7708, 7597] avg=8192.0000 max=11290 max_rank=0 min=4204 min_rank=4 imbalance_ratio=1.378174 max_load_reduction=54246 max_load_reduction_pct=0.827728 moved_to_echo=27620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3763, 3386, 0, 2569, 0, 3, 0, 277, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1678, 0, 2882, 4082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 47, 473, 4, 0, 3901, 0, 51, 0, 1352, 1008, 712, 0, 0, 0, 0, 0, 0, 2, 3780, 0, 0, 2618, 3320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 6, 639, 3, 2790, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 11, 0, 1, 0, 262, 2, 2322, 727, 490, 0, 0, 0, 0, 0, 34, 0, 3995, 0, 1, 0, 8, 0, 3603, 0, 0, 0, 0, 0, 0, 0, 1774, 0, 10, 1, 4031, 1, 0, 698, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9998, 8676, 7548, 9720, 7531, 7907, 7641, 6515] avg=8192.0000 max=9998 max_rank=0 min=6515 min_rank=7 imbalance_ratio=1.220459 max_load_reduction=55538 max_load_reduction_pct=0.847443 moved_to_echo=29594 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 345, 4085, 0, 167, 0, 0, 69, 278, 0, 0, 0, 0, 0, 0, 0, 355, 12, 5, 2186, 0, 3026, 92, 258, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2977, 0, 0, 4061, 278, 1, 0, 1762, 1014, 717, 0, 0, 0, 0, 0, 381, 0, 3222, 1, 0, 2638, 3755, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 545, 172, 228, 23, 2804, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 5, 1, 72, 0, 0, 0, 6, 2138, 732, 490, 0, 0, 0, 0, 0, 212, 66, 4090, 0, 0, 162, 0, 0, 3601, 0, 0, 0, 0, 0, 0, 0, 1632, 689, 7, 0, 4070, 0, 0, 3911, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4944, 5934, 10811, 9999, 7879, 7529, 8131, 10309] avg=8192.0000 max=10811 max_rank=2 min=4944 min_rank=0 imbalance_ratio=1.319702 max_load_reduction=54725 max_load_reduction_pct=0.835037 moved_to_echo=33848 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[9, 666, 3539, 0, 2259, 213, 13, 0, 276, 0, 0, 0, 0, 0, 0, 0, 688, 0, 0, 2263, 0, 306, 2400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 404, 2, 177, 1709, 1549, 123, 30, 1832, 995, 638, 0, 0, 0, 0, 0, 0, 195, 3729, 0, 0, 2322, 922, 109, 0, 0, 0, 0, 0, 0, 0, 0, 3388, 1426, 0, 893, 54, 2734, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 188, 2, 733, 839, 176, 2225, 652, 490, 0, 0, 0, 0, 0, 2939, 0, 4066, 1, 3, 6, 38, 6, 3605, 0, 0, 0, 0, 0, 0, 0, 1693, 4039, 0, 1, 3094, 84, 664, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6975, 5657, 7488, 7277, 8496, 9402, 10664, 9577] avg=8192.0000 max=10664 max_rank=6 min=5657 min_rank=1 imbalance_ratio=1.301758 max_load_reduction=54872 max_load_reduction_pct=0.837280 moved_to_echo=38139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1669, 958, 5, 4076, 0, 9, 188, 273, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 900, 0, 3872, 4096, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 2633, 2958, 0, 2, 724, 994, 715, 0, 0, 0, 0, 0, 274, 0, 3983, 0, 33, 2632, 4074, 23, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 90, 55, 1117, 0, 2749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 80, 1, 4, 38, 0, 1098, 2316, 730, 489, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 17, 0, 0, 3607, 0, 0, 0, 0, 0, 0, 0, 1769, 249, 0, 0, 4060, 0, 1, 298, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7178, 8956, 8031, 11019, 7450, 8852, 7673, 6377] avg=8192.0000 max=11019 max_rank=3 min=6377 min_rank=7 imbalance_ratio=1.345093 max_load_reduction=54517 max_load_reduction_pct=0.831863 moved_to_echo=30352 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 9216, 128, 256, 25472, 5504, 256, 256, 11392, 8064, 5760, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1024, 896, 29952, 128, 128, 20864, 12928, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4352, 128, 128, 14080, 0, 21760, 24960, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[27264, 8192, 2816, 4608, 128, 22144, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 14464, 23168, 128, 24064, 256, 128, 1664, 2304, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3840, 384, 32256, 128, 128, 256, 128, 128, 28800, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32384, 128, 128, 384, 128, 1536, 2048, 1664, 18048, 5888, 3968, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[13824, 12544, 128, 128, 31488, 128, 768, 7040, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4058, 4046, 46, 4035, 0, 0, 46, 0, 1, 0, 0, 1862, 6, 0, 7, 3925, 0, 4096, 10, 1964, 1, 193, 4009, 0, 1, 66, 0, 0, 4042, 35, 4003, 89, 0, 4009, 4026, 10, 2, 4066, 45, 0, 10, 84, 26, 51, 0, 14, 4080, 0, 13, 0, 0, 165, 3979, 4083, 35, 0, 2, 1, 5, 2, 312, 3975] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 71, 879, 4041, 0, 489, 0, 0, 1543, 0, 0, 0, 0, 419, 38, 0, 2942, 3107, 204, 4055, 0, 1630, 0, 0, 4015, 0, 0, 0, 65, 555, 4037, 0, 4022, 0, 0, 166, 3589, 116, 0, 3812, 59, 0, 2, 7, 1, 2, 4003, 0, 3394, 0, 0, 0, 25, 0, 3869, 4016, 2059, 0, 0, 0, 112, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 3944, 4085, 4, 3202, 229, 0, 2, 901, 0, 307, 0, 1721, 37, 9, 4, 4051, 0, 4083, 2, 196, 0, 0, 4091, 0, 0, 0, 0, 0, 4095, 0, 3569, 124, 0, 3434, 4044, 0, 0, 4096, 0, 0, 0, 1, 46, 0, 3982, 0, 4086, 0, 0, 17, 225, 46, 1810, 3897, 11, 0, 3, 0, 0, 1028, 84, 4070] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1733, 3424, 28, 4094, 55, 0, 3840, 0, 0, 0, 0, 202, 0, 0, 94, 4093, 0, 4096, 0, 2, 0, 9, 4096, 0, 0, 0, 0, 0, 3841, 0, 2897, 12, 0, 429, 4096, 0, 0, 3988, 0, 0, 8, 0, 3794, 0, 180, 0, 4077, 0, 0, 0, 0, 169, 4036, 3932, 176, 0, 0, 0, 0, 4039, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[7, 1, 1061, 4095, 10, 372, 0, 10, 3792, 3, 0, 0, 0, 0, 0, 0, 1010, 4081, 0, 4095, 1, 889, 4, 286, 4084, 1, 86, 0, 7, 109, 4093, 0, 4096, 0, 0, 3945, 3326, 0, 2, 3750, 29, 0, 4057, 0, 8, 0, 3008, 0, 3862, 1, 0, 0, 0, 15, 3142, 4065, 158, 2, 0, 0, 5, 104, 74, 3790] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4072, 1687, 0, 4017, 7, 0, 45, 867, 358, 3362, 613, 4029, 0, 7, 6, 3316, 493, 3673, 32, 621, 0, 303, 3905, 0, 593, 0, 3, 0, 4048, 45, 1861, 1672, 0, 906, 3813, 931, 0, 3343, 0, 0, 830, 0, 1541, 5, 39, 0, 4096, 2, 0, 0, 127, 547, 0, 4095, 4, 0, 0, 0, 30, 2522, 51, 3019] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[454, 0, 4093, 3908, 2, 2667, 1, 3, 76, 0, 0, 0, 35, 3295, 68, 0, 3887, 4023, 0, 4080, 0, 4090, 1, 0, 4094, 0, 0, 10, 0, 7, 4094, 0, 44, 0, 20, 1969, 4073, 0, 0, 3504, 0, 0, 2388, 2, 89, 0, 50, 1, 4038, 9, 0, 0, 123, 0, 2058, 4075, 0, 32, 0, 0, 2, 78, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 8, 1899, 4063, 86, 1947, 333, 0, 4032, 0, 0, 0, 0, 3212, 0, 0, 280, 4096, 47, 4064, 2, 1376, 48, 6, 4083, 10, 11, 0, 0, 0, 4015, 0, 4076, 17, 10, 759, 3606, 1663, 15, 4049, 0, 0, 0, 0, 1, 0, 867, 0, 3924, 0, 0, 28, 46, 157, 22, 4048, 412, 11, 0, 0, 9, 3954, 184, 4060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4058, 2981, 46, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 1, 0, 0, 1862, 6, 0, 2291, 1161, 474, 0, 0, 0, 0, 0, 7, 3925, 0, 1805, 10, 1964, 1, 193, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 0, 1, 66, 0, 0, 4042, 35, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 89, 0, 4009, 2723, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 10, 84, 26, 51, 0, 14, 4066, 1229, 0, 0, 0, 0, 0, 0, 4080, 0, 13, 0, 0, 165, 3979, 1693, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 2, 1, 5, 2, 312, 3975, 1065, 829, 87, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11120, 5841, 7905, 8066, 10836, 5525, 9930, 6313] avg=8192.0000 max=11120 max_rank=0 min=5525 min_rank=5 imbalance_ratio=1.357422 max_load_reduction=54416 max_load_reduction_pct=0.830322 moved_to_echo=32604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 71, 879, 2977, 0, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1543, 0, 0, 0, 0, 419, 38, 0, 2268, 1142, 423, 0, 0, 0, 0, 0, 2942, 3107, 204, 1787, 0, 1630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 65, 555, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 166, 2427, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 2, 7, 1, 2, 4003, 0, 3812, 1209, 0, 0, 0, 0, 0, 0, 3394, 0, 0, 0, 25, 0, 3869, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 2059, 0, 0, 0, 112, 4096, 0, 4096, 1064, 739, 88, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4416, 5833, 9670, 8584, 6731, 9095, 8953, 12254] avg=8192.0000 max=12254 max_rank=7 min=4416 min_rank=0 imbalance_ratio=1.495850 max_load_reduction=53282 max_load_reduction_pct=0.813019 moved_to_echo=37033 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3944, 3010, 4, 3202, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 901, 0, 307, 0, 1721, 37, 9, 2284, 1108, 476, 0, 0, 0, 0, 0, 4, 4051, 0, 1799, 2, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 124, 0, 3434, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 46, 0, 3982, 0, 4096, 1173, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 17, 225, 46, 1810, 1616, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3, 0, 0, 1028, 84, 4070, 1075, 833, 89, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10389, 6845, 6052, 8097, 9862, 9298, 7800, 7193] avg=8192.0000 max=10389 max_rank=0 min=6052 min_rank=2 imbalance_ratio=1.268188 max_load_reduction=55147 max_load_reduction_pct=0.841476 moved_to_echo=34153 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1733, 2523, 28, 4094, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 0, 0, 202, 0, 0, 2291, 1118, 482, 0, 0, 0, 0, 0, 94, 4093, 0, 1805, 0, 2, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 3841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2897, 12, 0, 429, 2770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3794, 0, 180, 0, 3988, 1183, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 169, 4036, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 4039, 0, 4096, 901, 844, 89, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8433, 7933, 6003, 7848, 6108, 9153, 9913, 10145] avg=8192.0000 max=10145 max_rank=7 min=6003 min_rank=2 imbalance_ratio=1.238403 max_load_reduction=55391 max_load_reduction_pct=0.845200 moved_to_echo=35319 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[454, 0, 4093, 2879, 2, 2667, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 35, 3295, 68, 0, 2282, 1159, 480, 0, 0, 0, 0, 0, 3887, 4023, 0, 1798, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 10, 0, 7, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 20, 1969, 2754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2388, 2, 89, 0, 50, 1, 3504, 1226, 0, 0, 0, 0, 0, 0, 4038, 9, 0, 0, 123, 0, 2058, 1690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 2, 78, 0, 4093, 1029, 839, 89, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10099, 7395, 13799, 8116, 4787, 7260, 7918, 6162] avg=8192.0000 max=13799 max_rank=2 min=4787 min_rank=4 imbalance_ratio=1.684448 max_load_reduction=51737 max_load_reduction_pct=0.789444 moved_to_echo=26127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4072, 1239, 0, 4017, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 867, 358, 3362, 613, 4029, 0, 7, 2059, 1170, 453, 0, 0, 0, 0, 0, 6, 3316, 493, 1614, 32, 621, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 3815, 0, 593, 0, 3, 0, 4048, 45, 0, 0, 0, 0, 0, 0, 0, 0, 1861, 1672, 0, 906, 2572, 931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 0, 1541, 5, 39, 0, 3343, 1237, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 127, 547, 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 30, 2522, 51, 3019, 448, 788, 90, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9335, 12963, 6385, 8504, 7942, 6995, 6460, 6952] avg=8192.0000 max=12963 max_rank=1 min=6385 min_rank=2 imbalance_ratio=1.582397 max_load_reduction=52573 max_load_reduction_pct=0.802200 moved_to_echo=28349 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[7, 1, 1061, 3017, 10, 372, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3792, 3, 0, 0, 0, 0, 0, 0, 2290, 1156, 392, 0, 0, 0, 0, 0, 1010, 4081, 0, 1805, 1, 889, 4, 286, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 1, 86, 0, 7, 109, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3945, 2249, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4057, 0, 8, 0, 3008, 0, 3750, 1223, 0, 0, 0, 0, 0, 0, 3862, 1, 0, 0, 0, 15, 3142, 1686, 0, 0, 0, 0, 0, 0, 0, 0, 158, 2, 0, 0, 5, 104, 74, 3790, 1078, 685, 89, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4478, 7633, 8076, 8291, 10292, 12075, 8706, 5985] avg=8192.0000 max=12075 max_rank=5 min=4478 min_rank=0 imbalance_ratio=1.473999 max_load_reduction=53461 max_load_reduction_pct=0.815750 moved_to_echo=37058 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 1899, 2993, 86, 1947, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 3212, 0, 0, 2273, 1151, 425, 0, 0, 0, 0, 0, 280, 4096, 47, 1791, 2, 1376, 48, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 10, 11, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 17, 10, 759, 2438, 1663, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 867, 0, 4049, 1218, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 28, 46, 157, 22, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 412, 11, 0, 0, 9, 3954, 184, 4060, 1070, 743, 89, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7266, 11093, 7646, 8030, 8978, 6135, 5856, 10532] avg=8192.0000 max=11093 max_rank=1 min=5856 min_rank=6 imbalance_ratio=1.354126 max_load_reduction=54443 max_load_reduction_pct=0.830734 moved_to_echo=31501 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[512, 128, 21760, 21632, 256, 20864, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[2944, 128, 128, 128, 256, 15872, 768, 31232, 7808, 6400, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[24576, 1920, 128, 15744, 20736, 2816, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31744, 128, 768, 128, 128, 768, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[13440, 1792, 384, 3712, 768, 14848, 256, 128, 18048, 9216, 3712, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 7296, 128, 5632, 128, 12160, 128, 30720, 9728, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31616, 128, 128, 128, 640, 1152, 18944, 13440, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8320, 30720, 768, 14208, 128, 10880, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 441, 0, 0, 8, 0, 0, 4096, 0, 294, 0, 3575, 3857, 0, 0, 3765, 0, 101, 0, 273, 3651, 4030, 3901, 0, 0, 0, 0, 0, 0, 4095, 6, 184, 3847, 4096, 3111, 187, 276, 0, 4014, 3820, 0, 114, 713, 1, 4047, 0, 0, 8, 0, 113, 0, 0, 4081, 489, 0, 23, 0, 1, 0, 0, 4096, 0, 222, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 11, 3, 3845, 47, 115, 2, 3852, 4057, 0, 0, 4088, 0, 123, 12, 3, 3, 4094, 4059, 1, 13, 12, 3, 0, 0, 3953, 56, 5, 3469, 4059, 3537, 0, 1, 0, 4087, 4096, 0, 0, 314, 37, 4054, 0, 0, 1, 24, 4, 1, 0, 4037, 4095, 0, 34, 2, 193, 564, 0, 4037, 0, 522, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[354, 0, 0, 1, 0, 0, 1, 2934, 0, 0, 407, 3893, 3277, 7, 11, 4087, 0, 2, 38, 119, 1, 3937, 4079, 0, 0, 0, 25, 0, 3, 4076, 73, 0, 4082, 3975, 4088, 252, 0, 0, 4096, 4086, 486, 0, 8, 5, 3944, 0, 103, 0, 2, 0, 11, 2, 4041, 3509, 0, 12, 255, 662, 1, 500, 4088, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 4056, 4030, 0, 0, 4096, 0, 353, 0, 0, 2682, 3210, 2825, 0, 0, 129, 0, 0, 0, 3743, 0, 356, 4096, 4082, 12, 776, 0, 0, 3959, 4080, 2, 0, 976, 0, 3527, 0, 0, 0, 0, 3845, 0, 0, 4085, 2433, 0, 0, 8, 2, 0, 43, 4096, 0, 67, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 26, 29, 0, 0, 1, 478, 0, 181, 0, 4011, 3786, 0, 0, 4080, 21, 6, 1, 9, 3811, 1588, 3449, 0, 0, 40, 71, 0, 0, 3248, 0, 3664, 4096, 4066, 327, 89, 0, 0, 4096, 4082, 1377, 8, 0, 24, 4095, 0, 0, 5, 0, 0, 0, 2, 4023, 4052, 0, 2, 0, 0, 219, 2363, 4096, 0, 12, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[13, 1004, 25, 2, 14, 65, 0, 3849, 49, 0, 0, 3265, 155, 0, 0, 2022, 769, 2687, 0, 0, 4093, 287, 3882, 13, 29, 6, 0, 286, 18, 2261, 1614, 1, 74, 1550, 4060, 1, 0, 564, 3742, 4081, 0, 0, 3564, 0, 3976, 0, 1, 21, 0, 300, 0, 217, 3822, 3366, 0, 0, 896, 136, 1360, 32, 4084, 0, 3280, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1, 1, 2, 0, 10, 0, 21, 4007, 9, 0, 150, 3403, 1318, 1, 0, 4043, 304, 260, 4, 9, 174, 3864, 4082, 0, 0, 446, 37, 0, 0, 4010, 0, 2135, 4080, 4096, 0, 314, 0, 0, 4076, 4049, 15, 0, 3899, 0, 4094, 0, 0, 0, 8, 261, 0, 12, 4095, 4082, 0, 28, 0, 19, 16, 0, 4089, 0, 0, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 134, 304, 1229, 0, 0, 0, 2, 0, 0, 67, 4095, 4092, 0, 9, 4091, 0, 0, 0, 0, 6, 4086, 3882, 0, 12, 0, 0, 0, 0, 4018, 0, 3, 4052, 4010, 4046, 0, 24, 1, 4094, 4096, 2503, 0, 2, 0, 3255, 0, 69, 0, 0, 160, 0, 0, 4096, 3622, 0, 0, 0, 72, 2, 1246, 4096, 3, 57, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 11, 3, 3845, 4096, 794, 0, 0, 0, 0, 0, 0, 47, 115, 2, 3852, 4057, 0, 0, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 12, 3, 3, 4094, 2816, 1, 0, 0, 0, 0, 0, 0, 0, 0, 13, 12, 3, 0, 0, 3953, 56, 5, 3293, 134, 0, 0, 0, 0, 0, 0, 3469, 2181, 3537, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 37, 4054, 0, 0, 1, 2675, 64, 0, 0, 0, 0, 0, 0, 24, 4, 1, 0, 4037, 4095, 0, 34, 342, 0, 0, 0, 0, 0, 0, 0, 2, 193, 564, 0, 4037, 0, 522, 0, 1814, 901, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8760, 9352, 7052, 7469, 9188, 7145, 8537, 8033] avg=8192.0000 max=9352 max_rank=1 min=7052 min_rank=2 imbalance_ratio=1.141602 max_load_reduction=56184 max_load_reduction_pct=0.857300 moved_to_echo=32903 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 441, 0, 0, 8, 0, 0, 4096, 3820, 780, 0, 0, 0, 0, 0, 0, 0, 294, 0, 3575, 3857, 0, 0, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 273, 3651, 4030, 2706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 184, 3234, 123, 0, 0, 0, 0, 0, 0, 3847, 2200, 3111, 187, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 713, 1, 4047, 0, 0, 8, 2463, 65, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 4081, 489, 0, 23, 329, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 222, 0, 1831, 866, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9145, 8905, 10761, 7642, 9621, 7411, 5035, 7016] avg=8192.0000 max=10761 max_rank=2 min=5035 min_rank=6 imbalance_ratio=1.313599 max_load_reduction=54775 max_load_reduction_pct=0.835800 moved_to_echo=29083 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[354, 0, 0, 1, 0, 0, 1, 2934, 4086, 797, 0, 0, 0, 0, 0, 0, 0, 0, 407, 3893, 3277, 7, 11, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 38, 119, 1, 3937, 2829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 3, 4076, 73, 0, 3299, 134, 0, 0, 0, 0, 0, 0, 4082, 2135, 4088, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 0, 8, 5, 3944, 0, 103, 0, 2674, 63, 0, 0, 0, 0, 0, 0, 2, 0, 11, 2, 4041, 3509, 0, 12, 344, 0, 0, 0, 0, 0, 0, 0, 255, 662, 1, 500, 4088, 3, 0, 0, 1777, 906, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8874, 6926, 7610, 10557, 7283, 7921, 8192] avg=8192.0000 max=10557 max_rank=4 min=6926 min_rank=2 imbalance_ratio=1.288696 max_load_reduction=54979 max_load_reduction_pct=0.838913 moved_to_echo=33953 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3967, 4080, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 4030, 0, 0, 1282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 2682, 3210, 1960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 3743, 0, 356, 3189, 134, 0, 0, 0, 0, 0, 0, 4096, 2193, 12, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 976, 0, 3527, 0, 0, 0, 2680, 65, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 4085, 2433, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 43, 4096, 0, 67, 0, 1824, 627, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8817, 9368, 8205, 7551, 7077, 7250, 10601, 6667] avg=8192.0000 max=10601 max_rank=6 min=6667 min_rank=7 imbalance_ratio=1.294067 max_load_reduction=54935 max_load_reduction_pct=0.838242 moved_to_echo=31595 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[13, 1004, 25, 2, 14, 65, 0, 3849, 4081, 728, 0, 0, 0, 0, 0, 0, 49, 0, 0, 3265, 155, 0, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 769, 2687, 0, 0, 4093, 287, 2683, 13, 0, 0, 0, 0, 0, 0, 0, 0, 29, 6, 0, 286, 18, 2261, 1614, 1, 3014, 70, 0, 0, 0, 0, 0, 0, 74, 825, 4060, 1, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3564, 0, 3976, 0, 1, 21, 1327, 28, 0, 0, 0, 0, 0, 0, 0, 300, 0, 217, 3822, 3366, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 896, 136, 1360, 32, 4084, 0, 3280, 0, 697, 867, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9781, 4094, 10532, 7299, 5524, 8917, 8037, 11352] avg=8192.0000 max=11352 max_rank=7 min=4094 min_rank=1 imbalance_ratio=1.385742 max_load_reduction=54184 max_load_reduction_pct=0.826782 moved_to_echo=33830 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 134, 304, 1229, 0, 0, 0, 2, 4096, 797, 0, 0, 0, 0, 0, 0, 0, 0, 67, 4095, 4092, 0, 9, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4086, 2693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4018, 0, 3, 3297, 134, 0, 0, 0, 0, 0, 0, 4052, 2155, 4046, 0, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2503, 0, 2, 0, 3255, 0, 69, 0, 2677, 63, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 4096, 3622, 0, 0, 327, 0, 0, 0, 0, 0, 0, 0, 0, 72, 2, 1246, 4096, 3, 57, 0, 1792, 862, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6562, 9543, 6785, 7464, 10278, 8569, 8205, 8130] avg=8192.0000 max=10278 max_rank=4 min=6562 min_rank=0 imbalance_ratio=1.254639 max_load_reduction=55258 max_load_reduction_pct=0.843170 moved_to_echo=35182 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 26, 29, 0, 0, 1, 478, 4082, 797, 0, 0, 0, 0, 0, 0, 0, 181, 0, 4011, 3786, 0, 0, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 21, 6, 1, 9, 3811, 1588, 2392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 71, 0, 0, 3248, 0, 3664, 3299, 134, 0, 0, 0, 0, 0, 0, 4096, 2185, 327, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 8, 0, 24, 4095, 0, 0, 5, 2670, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4023, 4052, 0, 2, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 2363, 4096, 0, 12, 0, 1817, 766, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5415, 9254, 7828, 10456, 6697, 8243, 8370, 9273] avg=8192.0000 max=10456 max_rank=3 min=5415 min_rank=0 imbalance_ratio=1.276367 max_load_reduction=55080 max_load_reduction_pct=0.840454 moved_to_echo=32583 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 2, 0, 10, 0, 21, 4007, 4049, 792, 0, 0, 0, 0, 0, 0, 9, 0, 150, 3403, 1318, 1, 0, 1265, 0, 0, 0, 0, 0, 0, 0, 0, 304, 260, 4, 9, 174, 3864, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 37, 0, 0, 4010, 0, 2135, 3284, 133, 0, 0, 0, 0, 0, 0, 4080, 2200, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3899, 0, 4094, 0, 0, 0, 2645, 65, 0, 0, 0, 0, 0, 0, 8, 261, 0, 12, 4095, 4082, 0, 28, 344, 0, 0, 0, 0, 0, 0, 0, 0, 19, 16, 0, 4089, 0, 0, 12, 1831, 906, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8883, 6146, 7447, 10045, 6594, 10718, 8830, 6873] avg=8192.0000 max=10718 max_rank=5 min=6146 min_rank=1 imbalance_ratio=1.308350 max_load_reduction=54818 max_load_reduction_pct=0.836456 moved_to_echo=33015 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 640, 640, 30208, 24576, 128, 128, 9472, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1280, 1152, 2176, 4224, 32768, 128, 4224, 128, 13440, 6784, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[27904, 16128, 19200, 1664, 384, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4480, 128, 9600, 128, 31104, 0, 256, 128, 19840, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 4736, 128, 256, 32384, 25728, 0, 128, 2560, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1152, 3584, 128, 512, 14464, 25216, 20992, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 1664, 384, 1280, 128, 128, 128, 23296, 32512, 6272, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 640, 256, 384, 128, 29440, 1792, 6400, 25984, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 1, 1, 4083, 0, 30, 0, 0, 4089, 25, 4096, 0, 0, 1226, 3220, 0, 4096, 1904, 301, 379, 19, 0, 0, 948, 0, 4049, 0, 418, 58, 4064, 20, 0, 0, 4068, 3428, 371, 0, 3370, 0, 58, 4096, 4, 0, 615, 170, 0, 655, 3, 0, 0, 9, 3906, 0, 0, 11, 3603, 3, 57, 3986] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[48, 3742, 0, 0, 23, 15, 0, 13, 29, 0, 0, 34, 0, 4055, 0, 4077, 0, 0, 4080, 4088, 0, 4085, 3944, 3, 0, 0, 1, 1, 4050, 64, 4094, 0, 4055, 0, 4091, 0, 13, 4036, 4, 5, 1, 0, 3, 0, 0, 4050, 0, 40, 723, 0, 46, 299, 0, 26, 144, 0, 4048, 0, 41, 0, 3367, 118, 4, 3976] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[63, 4081, 57, 7, 20, 0, 0, 332, 2219, 944, 0, 0, 3, 4081, 0, 4093, 0, 2705, 1, 4096, 0, 4084, 4042, 1, 0, 0, 0, 0, 4072, 0, 4086, 0, 3525, 15, 3952, 0, 5, 9, 17, 0, 0, 0, 2734, 51, 0, 4050, 2, 0, 0, 1, 42, 188, 0, 17, 0, 1, 4079, 0, 0, 0, 3982, 0, 0, 3879] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 2, 4086, 14, 0, 0, 123, 4096, 0, 4096, 0, 1, 18, 4096, 0, 4001, 4092, 327, 0, 1, 0, 0, 3530, 0, 4096, 0, 46, 0, 2727, 0, 0, 294, 4096, 0, 0, 23, 2735, 0, 0, 4067, 0, 0, 2337, 0, 79, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 173, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4083, 0, 0, 0, 0, 5, 0, 4094, 159, 0, 0, 710, 4040, 0, 4070, 0, 249, 0, 4042, 0, 3965, 3911, 90, 0, 37, 0, 0, 4092, 0, 4094, 0, 543, 0, 2614, 0, 0, 1823, 3520, 16, 0, 0, 2636, 9, 0, 4094, 1, 68, 3416, 0, 200, 22, 20, 146, 18, 0, 3963, 0, 15, 1, 927, 3, 0, 3840] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 44, 1, 2, 1376, 237, 1, 59, 43, 4094, 6, 4096, 8, 0, 5, 4096, 0, 3818, 4079, 28, 0, 0, 0, 0, 3350, 20, 4091, 0, 4076, 1, 4096, 0, 0, 4094, 576, 80, 0, 0, 6, 0, 1, 3789, 4, 0, 0, 0, 3003, 0, 14, 0, 0, 0, 4086, 0, 0, 6, 4018, 40, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4078, 55, 0, 3, 0, 0, 805, 428, 2, 0, 0, 4068, 4090, 0, 4096, 0, 0, 0, 4093, 112, 4025, 3041, 0, 0, 0, 0, 0, 4036, 2, 4095, 1, 3664, 0, 1365, 1, 0, 137, 2, 60, 0, 1, 2611, 0, 0, 4089, 1, 0, 187, 2, 4096, 0, 0, 5, 0, 0, 4084, 0, 0, 0, 4093, 0, 12, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4091, 139, 0, 0, 0, 113, 5, 3662, 0, 0, 0, 8, 4096, 32, 4010, 0, 3369, 0, 3876, 5, 4056, 2262, 4, 1214, 0, 117, 0, 1143, 0, 4087, 11, 5, 0, 12, 0, 0, 4080, 1804, 1936, 0, 0, 601, 0, 0, 2723, 1877, 0, 3696, 0, 1, 0, 0, 142, 0, 0, 3981, 27, 0, 184, 3792, 4073, 0, 302] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 1, 1, 2114, 1761, 0, 0, 0, 0, 0, 0, 4083, 0, 30, 0, 0, 806, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1226, 2622, 0, 0, 1904, 301, 0, 0, 0, 0, 0, 0, 0, 0, 379, 19, 0, 0, 948, 0, 4049, 0, 586, 0, 0, 0, 0, 0, 0, 0, 418, 58, 2661, 20, 0, 0, 4068, 3428, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 3370, 0, 58, 4096, 4, 0, 1169, 1403, 12, 0, 0, 0, 0, 0, 615, 170, 0, 655, 3, 0, 0, 9, 4096, 1585, 0, 0, 0, 0, 0, 0, 560, 0, 0, 11, 3603, 3, 57, 3986, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7973, 9040, 6053, 5981, 10653, 10483, 7133, 8220] avg=8192.0000 max=10653 max_rank=4 min=5981 min_rank=3 imbalance_ratio=1.300415 max_load_reduction=54883 max_load_reduction_pct=0.837448 moved_to_echo=36489 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[48, 3742, 0, 0, 23, 15, 0, 13, 2097, 1825, 0, 0, 0, 0, 0, 0, 29, 0, 0, 34, 0, 799, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 3329, 0, 0, 3944, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4050, 64, 4094, 0, 744, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 2679, 0, 13, 4036, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 4050, 0, 40, 1159, 1412, 15, 0, 0, 0, 0, 0, 723, 0, 46, 299, 0, 26, 144, 0, 4085, 1642, 0, 0, 0, 0, 0, 0, 581, 0, 41, 0, 3367, 118, 4, 3976, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7763, 4939, 11356, 8954, 10792, 6680, 6965, 8087] avg=8192.0000 max=11356 max_rank=2 min=4939 min_rank=1 imbalance_ratio=1.386230 max_load_reduction=54180 max_load_reduction_pct=0.826721 moved_to_echo=32524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[63, 4081, 57, 7, 20, 0, 0, 332, 2110, 1839, 0, 0, 0, 0, 0, 0, 2219, 944, 0, 0, 3, 805, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 1, 3336, 0, 0, 4042, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4086, 0, 745, 0, 0, 0, 0, 0, 0, 0, 3525, 15, 2588, 0, 5, 9, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 51, 0, 4050, 2, 0, 1166, 1364, 15, 0, 0, 0, 0, 0, 0, 1, 42, 188, 0, 17, 0, 1, 4084, 1655, 0, 0, 0, 0, 0, 0, 585, 0, 0, 0, 3982, 0, 0, 3879, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8509, 8064, 10085, 8903, 6159, 9382, 5988, 8446] avg=8192.0000 max=10085 max_rank=2 min=5988 min_rank=6 imbalance_ratio=1.231079 max_load_reduction=55451 max_load_reduction_pct=0.846115 moved_to_echo=29975 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 2, 2118, 1846, 0, 0, 0, 0, 0, 0, 4086, 14, 0, 0, 123, 807, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 18, 3336, 0, 0, 4092, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3530, 0, 4096, 0, 745, 0, 0, 0, 0, 0, 0, 0, 46, 0, 1786, 0, 0, 294, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2735, 0, 0, 4067, 0, 0, 1171, 941, 15, 0, 0, 0, 0, 0, 2337, 0, 79, 0, 0, 0, 0, 0, 4001, 1661, 0, 0, 0, 0, 0, 0, 587, 0, 0, 0, 4096, 173, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8060, 9126, 7774, 8372, 6222, 8952, 8078, 8952] avg=8192.0000 max=9126 max_rank=1 min=6222 min_rank=4 imbalance_ratio=1.114014 max_load_reduction=56410 max_load_reduction_pct=0.860748 moved_to_echo=32204 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4078, 55, 0, 3, 0, 0, 805, 2115, 1841, 0, 0, 0, 0, 0, 0, 428, 2, 0, 0, 4068, 806, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3333, 112, 0, 3041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 2, 4095, 1, 745, 0, 0, 0, 0, 0, 0, 0, 3664, 0, 894, 1, 0, 137, 2, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2611, 0, 0, 4089, 1, 0, 1169, 471, 15, 0, 0, 0, 0, 0, 187, 2, 4096, 0, 0, 5, 0, 0, 4025, 1657, 0, 0, 0, 0, 0, 0, 586, 0, 0, 0, 4093, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8897, 9400, 6486, 8879, 4758, 8357, 9972, 8787] avg=8192.0000 max=9972 max_rank=6 min=4758 min_rank=4 imbalance_ratio=1.217285 max_load_reduction=55564 max_load_reduction_pct=0.847839 moved_to_echo=31874 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 44, 1, 2, 2117, 1842, 0, 0, 0, 0, 0, 0, 1376, 237, 1, 59, 43, 807, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 5, 3336, 0, 0, 4079, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3350, 20, 4091, 0, 745, 0, 0, 0, 0, 0, 0, 0, 4076, 1, 2682, 0, 0, 4094, 576, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 3789, 4, 0, 1170, 1414, 15, 0, 0, 0, 0, 0, 0, 0, 3003, 0, 14, 0, 0, 0, 3818, 1658, 0, 0, 0, 0, 0, 0, 586, 0, 0, 6, 4018, 40, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 6625, 7456, 8206, 11509, 6399, 8493, 8746] avg=8192.0000 max=11509 max_rank=4 min=6399 min_rank=5 imbalance_ratio=1.404907 max_load_reduction=54027 max_load_reduction_pct=0.824387 moved_to_echo=35147 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 139, 0, 0, 0, 113, 5, 2122, 1798, 0, 0, 0, 0, 0, 0, 3662, 0, 0, 0, 8, 801, 32, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3369, 0, 3145, 5, 0, 2262, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 0, 117, 0, 1143, 0, 4087, 11, 710, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 0, 0, 4080, 1804, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, 0, 2723, 1877, 0, 1173, 8, 21, 0, 0, 0, 0, 0, 3696, 0, 1, 0, 0, 142, 0, 0, 4056, 1617, 0, 0, 0, 0, 0, 0, 566, 27, 0, 184, 3792, 4073, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8268, 8513, 8785, 7282, 7829, 6403, 9512, 8944] avg=8192.0000 max=9512 max_rank=6 min=6403 min_rank=5 imbalance_ratio=1.161133 max_load_reduction=56024 max_load_reduction_pct=0.854858 moved_to_echo=32688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4083, 0, 0, 0, 0, 5, 0, 2089, 1787, 0, 0, 0, 0, 0, 0, 4094, 159, 0, 0, 710, 796, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 3291, 0, 0, 3911, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 4092, 0, 4094, 0, 736, 0, 0, 0, 0, 0, 0, 0, 543, 0, 1712, 0, 0, 1823, 3520, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2636, 9, 0, 4094, 1, 68, 1155, 902, 15, 0, 0, 0, 0, 0, 3416, 0, 200, 22, 20, 146, 18, 0, 3965, 1608, 0, 0, 0, 0, 0, 0, 568, 0, 15, 1, 927, 3, 0, 3840, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7964, 9829, 7541, 8959, 7614, 8880, 9395, 5354] avg=8192.0000 max=9829 max_rank=1 min=5354 min_rank=7 imbalance_ratio=1.199829 max_load_reduction=55707 max_load_reduction_pct=0.850021 moved_to_echo=31243 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 6400, 5376, 25728, 128, 0, 27392, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4736, 128, 128, 256, 27904, 4480, 128, 28288, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32384, 256, 128, 128, 128, 128, 1280, 16896, 14592, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[384, 128, 14720, 128, 128, 30976, 1920, 128, 9344, 7936, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1664, 128, 128, 128, 25344, 128, 32768, 128, 5760, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[20096, 1408, 128, 128, 4992, 6528, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[16384, 128, 15104, 128, 128, 14592, 14208, 5632, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[11008, 256, 7552, 1280, 128, 384, 256, 128, 32256, 13184, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4071, 0, 5, 0, 4006, 0, 3910, 4080, 41, 0, 56, 0, 19, 0, 0, 0, 0, 765, 0, 390, 10, 1041, 0, 0, 0, 3397, 4073, 0, 4057, 0, 2, 56, 101, 21, 17, 3, 22, 0, 2222, 1074, 2, 0, 4095, 3945, 9, 1, 0, 87, 0, 4060, 4069, 0, 23, 1, 94, 4042, 3514, 0, 0, 4025, 6, 4079, 0, 45] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 4054, 56, 3982, 1, 554, 4096, 0, 71, 0, 0, 258, 63, 63, 0, 7, 487, 0, 2, 5, 0, 0, 17, 5, 0, 3922, 857, 4070, 0, 27, 40, 0, 0, 0, 1, 2, 1, 2970, 4094, 17, 0, 1830, 4034, 0, 18, 2, 4045, 0, 4057, 4094, 0, 2, 7, 107, 3732, 2833, 0, 0, 3752, 0, 3103, 7, 95] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3919, 55, 0, 6, 4096, 0, 238, 4056, 0, 37, 0, 0, 382, 212, 0, 27, 3, 0, 0, 0, 0, 113, 0, 0, 0, 0, 4094, 197, 826, 48, 0, 1164, 0, 0, 6, 0, 0, 123, 0, 3866, 1, 12, 3195, 3981, 2, 0, 0, 3074, 0, 4075, 4087, 0, 3440, 0, 0, 4072, 4096, 0, 14, 4095, 3883, 4041, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 215, 0, 4096, 0, 3904, 4096, 0, 0, 0, 0, 199, 0, 0, 0, 67, 4, 0, 0, 0, 0, 0, 0, 0, 8, 4092, 0, 2649, 0, 0, 35, 739, 0, 0, 0, 0, 0, 2, 3900, 0, 194, 4096, 3821, 0, 0, 0, 3344, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 3448, 4089, 2061, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4073, 0, 7, 0, 4095, 0, 3565, 4095, 0, 0, 0, 365, 43, 0, 0, 34, 0, 102, 17, 1, 12, 0, 0, 0, 0, 2, 4095, 0, 4092, 134, 16, 12, 3, 0, 0, 0, 0, 0, 5, 4087, 0, 18, 4078, 4027, 36, 0, 0, 272, 7, 4096, 4056, 0, 0, 0, 0, 3987, 4090, 14, 0, 4095, 3954, 3948, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4096, 5, 0, 0, 4089, 0, 1661, 3563, 0, 2, 0, 342, 0, 0, 0, 0, 0, 249, 0, 10, 0, 29, 0, 0, 0, 0, 4090, 361, 2124, 1, 0, 0, 2069, 0, 1, 0, 4, 0, 2102, 349, 0, 1, 1633, 3659, 40, 664, 0, 4096, 0, 4095, 4095, 0, 4096, 1, 3, 3934, 4090, 0, 1, 4080, 3541, 2360, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4087, 1123, 0, 0, 2443, 0, 991, 4096, 0, 24, 327, 587, 244, 0, 0, 2, 7, 0, 311, 1307, 8, 0, 0, 0, 0, 4, 3, 0, 3019, 3, 0, 3925, 0, 47, 0, 0, 0, 0, 8, 4066, 24, 31, 637, 3987, 0, 0, 1, 4096, 218, 4096, 3740, 0, 4096, 0, 2502, 3337, 4096, 0, 0, 3971, 0, 4057, 0, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4090, 3, 41, 339, 4096, 0, 3055, 4091, 4, 0, 0, 2, 3, 0, 0, 14, 5, 35, 0, 52, 34, 3, 3, 0, 0, 4096, 4076, 1, 3868, 0, 0, 560, 219, 1, 25, 0, 18, 0, 63, 4094, 0, 4, 1107, 4044, 0, 0, 0, 4093, 1, 4095, 4091, 0, 90, 0, 10, 3877, 4096, 0, 0, 4095, 219, 2822, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4054, 56, 3982, 1, 554, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 258, 63, 63, 0, 4057, 3707, 0, 0, 0, 0, 0, 0, 7, 487, 0, 2, 5, 0, 0, 17, 3752, 2715, 565, 389, 0, 0, 0, 0, 5, 0, 3569, 857, 4070, 0, 27, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 2970, 4094, 1385, 1025, 708, 353, 0, 0, 0, 0, 17, 0, 1830, 2649, 0, 18, 2, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2504, 0, 2, 7, 107, 3732, 0, 0, 0, 0, 0, 0, 0, 0, 2125, 0, 0, 0, 0, 3103, 7, 95, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10028, 8219, 7939, 8568, 10539, 8561, 6352, 5330] avg=8192.0000 max=10539 max_rank=4 min=5330 min_rank=7 imbalance_ratio=1.286499 max_load_reduction=54997 max_load_reduction_pct=0.839188 moved_to_echo=30782 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 0, 4006, 0, 3910, 1376, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 56, 0, 19, 0, 0, 0, 4060, 3686, 0, 0, 0, 0, 0, 0, 0, 765, 0, 390, 10, 1041, 0, 0, 4025, 2704, 562, 385, 0, 0, 0, 0, 0, 3397, 3706, 0, 4057, 0, 2, 56, 0, 0, 0, 0, 0, 0, 0, 0, 101, 21, 17, 3, 22, 0, 2222, 1074, 1354, 1019, 878, 367, 0, 0, 0, 0, 2, 0, 4095, 2591, 9, 1, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2488, 0, 23, 1, 94, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 2636, 0, 0, 0, 6, 4079, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9297, 7862, 9882, 11218, 7078, 6785, 6648, 6766] avg=8192.0000 max=11218 max_rank=3 min=6648 min_rank=6 imbalance_ratio=1.369385 max_load_reduction=54318 max_load_reduction_pct=0.828827 moved_to_echo=27277 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 55, 0, 6, 4096, 0, 238, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 382, 212, 0, 27, 4075, 3547, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 113, 0, 0, 4095, 2688, 564, 372, 0, 0, 0, 0, 0, 0, 3725, 197, 826, 48, 0, 1164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 123, 0, 3866, 1367, 1023, 1024, 369, 0, 0, 0, 0, 1, 12, 3195, 2614, 2, 0, 0, 3074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2500, 0, 3440, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 3072, 0, 14, 0, 3883, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5763, 8280, 7835, 5960, 7778, 8898, 10012, 11010] avg=8192.0000 max=11010 max_rank=7 min=5763 min_rank=0 imbalance_ratio=1.343994 max_load_reduction=54526 max_load_reduction_pct=0.832001 moved_to_echo=37698 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 215, 0, 4096, 0, 3904, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 4096, 3708, 0, 0, 0, 0, 0, 0, 67, 4, 0, 0, 0, 0, 0, 0, 3448, 2715, 566, 388, 0, 0, 0, 0, 0, 8, 3724, 0, 2649, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 739, 0, 0, 0, 0, 0, 2, 3900, 1312, 1026, 1022, 368, 0, 0, 0, 0, 0, 194, 4096, 2509, 0, 0, 0, 3344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2504, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3066, 0, 0, 0, 4089, 2061, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9596, 8003, 7188, 6416, 8369, 10143, 6600, 9221] avg=8192.0000 max=10143 max_rank=5 min=6416 min_rank=3 imbalance_ratio=1.238159 max_load_reduction=55393 max_load_reduction_pct=0.845230 moved_to_echo=34333 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 41, 339, 4096, 0, 3055, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 3, 0, 0, 14, 4095, 3703, 0, 0, 0, 0, 0, 0, 5, 35, 0, 52, 34, 3, 3, 0, 4095, 2711, 565, 387, 0, 0, 0, 0, 0, 4096, 3709, 1, 3868, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 219, 1, 25, 0, 18, 0, 63, 4094, 1388, 1024, 1024, 367, 0, 0, 0, 0, 0, 4, 1107, 2656, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2502, 0, 90, 0, 10, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 3072, 0, 0, 0, 219, 2822, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8914, 7821, 7890, 12234, 8223, 7860, 6480, 6114] avg=8192.0000 max=12234 max_rank=3 min=6114 min_rank=7 imbalance_ratio=1.493408 max_load_reduction=53302 max_load_reduction_pct=0.813324 moved_to_echo=28677 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1123, 0, 0, 2443, 0, 991, 1376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 327, 587, 244, 0, 0, 2, 4096, 3700, 0, 0, 0, 0, 0, 0, 7, 0, 311, 1307, 8, 0, 0, 0, 3971, 2720, 521, 387, 0, 0, 0, 0, 0, 4, 0, 0, 3019, 3, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 8, 4066, 1373, 941, 1028, 3, 0, 0, 0, 0, 24, 31, 637, 2614, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 2278, 0, 4096, 0, 2502, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 3068, 0, 0, 0, 0, 4057, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5933, 8980, 9232, 6951, 7466, 7403, 12431, 7140] avg=8192.0000 max=12431 max_rank=6 min=5933 min_rank=0 imbalance_ratio=1.517456 max_load_reduction=53105 max_load_reduction_pct=0.810318 moved_to_echo=34440 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 4089, 0, 1661, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 342, 0, 0, 0, 0, 4095, 3708, 0, 0, 0, 0, 0, 0, 0, 249, 0, 10, 0, 29, 0, 0, 4080, 2361, 565, 388, 0, 0, 0, 0, 0, 0, 3721, 361, 2124, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2069, 0, 1, 0, 4, 0, 2102, 349, 1256, 1025, 1023, 369, 0, 0, 0, 0, 0, 1, 1633, 2403, 40, 664, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2505, 0, 4096, 1, 3, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 0, 1, 0, 3541, 2360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6957, 8147, 7682, 6207, 8198, 8837, 10539, 8969] avg=8192.0000 max=10539 max_rank=6 min=6207 min_rank=3 imbalance_ratio=1.286499 max_load_reduction=54997 max_load_reduction_pct=0.839188 moved_to_echo=36543 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 4095, 0, 3565, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 43, 0, 0, 34, 4096, 3686, 0, 0, 0, 0, 0, 0, 0, 102, 17, 1, 12, 0, 0, 0, 4095, 2714, 560, 387, 0, 0, 0, 0, 0, 2, 3726, 0, 4092, 134, 16, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 5, 4087, 1382, 1016, 1023, 369, 0, 0, 0, 0, 0, 18, 4078, 2645, 36, 0, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2480, 0, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 14, 0, 0, 3954, 3948, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9048, 8224, 7888, 7982, 7885, 7049, 6474, 10986] avg=8192.0000 max=10986 max_rank=7 min=6474 min_rank=6 imbalance_ratio=1.341064 max_load_reduction=54550 max_load_reduction_pct=0.832367 moved_to_echo=32394 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[23296, 128, 128, 0, 15744, 26496, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 384, 20736, 20736, 128, 768, 128, 23168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 0, 19840, 0, 11776, 128, 2816, 31104, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 7552, 25984, 1536, 24832, 256, 128, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[3200, 128, 128, 128, 128, 128, 7424, 25600, 10880, 8192, 7808, 2688, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 1664, 384, 1792, 128, 1280, 128, 128, 31616, 21376, 4480, 3200, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 1280, 4352, 512, 30976, 128, 17920, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 256, 384, 1408, 1152, 384, 128, 128, 32768, 29568, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 3827, 35, 2844, 3523, 0, 4096, 1, 71, 0, 0, 32, 0, 0, 97, 0, 0, 3603, 4077, 0, 141, 4036, 117, 0, 0, 1356, 0, 0, 1, 3, 4088, 0, 0, 3461, 4096, 0, 0, 0, 17, 4096, 0, 4066, 0, 0, 1, 0, 21, 1578, 0, 59, 0, 0, 50, 7, 7, 31, 4096, 0, 1474, 0, 4, 2623, 3971, 3930] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[20, 3830, 0, 65, 3885, 10, 4084, 38, 1071, 0, 11, 3, 0, 0, 0, 0, 8, 3119, 1486, 39, 0, 3738, 25, 0, 0, 4066, 0, 0, 0, 0, 2724, 0, 0, 4065, 4084, 0, 4048, 1652, 0, 4025, 485, 4096, 0, 0, 0, 0, 54, 2029, 203, 46, 139, 12, 0, 0, 0, 11, 4010, 0, 235, 0, 6, 2, 4045, 4067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 4046, 32, 1, 4051, 0, 3997, 2, 0, 35, 1, 5, 0, 0, 47, 0, 2, 1995, 3661, 0, 448, 3948, 0, 0, 72, 4014, 0, 0, 0, 8, 4093, 1, 0, 3251, 3951, 97, 3681, 0, 0, 4085, 0, 4096, 845, 0, 0, 18, 0, 258, 2203, 69, 0, 11, 156, 0, 0, 13, 4080, 0, 35, 0, 43, 0, 4096, 4089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1, 4087, 90, 3948, 534, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3427, 540, 16, 0, 20, 0, 3, 0, 4035, 4, 0, 4, 95, 990, 0, 0, 3414, 4096, 422, 4022, 4096, 0, 4094, 0, 3033, 0, 0, 0, 1, 0, 3805, 130, 22, 8, 0, 0, 5, 53, 519, 4087, 0, 0, 3, 0, 3811, 4095, 4004] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[30, 3819, 4096, 3768, 2537, 80, 3181, 0, 163, 0, 0, 497, 0, 0, 0, 0, 0, 2522, 4081, 375, 0, 0, 0, 0, 0, 4057, 447, 28, 21, 0, 2367, 982, 0, 3734, 4044, 0, 2832, 834, 11, 4021, 0, 3533, 1152, 0, 0, 17, 33, 381, 0, 0, 0, 0, 55, 11, 13, 0, 4096, 0, 668, 0, 0, 0, 3650, 3400] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 4096, 248, 3977, 6, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 2822, 3684, 0, 4019, 0, 0, 0, 1479, 0, 1047, 0, 0, 4096, 0, 0, 4088, 4096, 0, 4036, 1238, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1319, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 131, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[33, 4095, 44, 1725, 16, 0, 4054, 0, 562, 0, 0, 0, 0, 0, 0, 0, 25, 3493, 3876, 2944, 1, 1, 0, 0, 1, 2186, 2, 38, 0, 3479, 4096, 4, 0, 4089, 3964, 0, 0, 4088, 1, 4095, 1, 4073, 0, 19, 127, 0, 1, 599, 808, 0, 0, 0, 0, 5, 2, 0, 4073, 13, 0, 2, 0, 741, 4068, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[19, 4063, 63, 1, 2686, 0, 4086, 14, 126, 1, 0, 0, 12, 0, 0, 116, 0, 3458, 4088, 0, 0, 4048, 0, 0, 3, 4092, 22, 762, 0, 0, 3935, 25, 0, 2981, 4096, 0, 101, 0, 0, 4089, 1, 4092, 5, 2, 0, 0, 0, 3463, 1249, 0, 0, 4, 0, 0, 0, 6, 4096, 9, 0, 0, 0, 1599, 4083, 4040] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 0, 65, 3885, 10, 3504, 38, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 0, 11, 3, 0, 0, 0, 0, 4010, 3709, 0, 0, 0, 0, 0, 0, 8, 3119, 948, 39, 0, 3738, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 2724, 0, 580, 140, 0, 0, 0, 0, 0, 0, 0, 4065, 666, 0, 4048, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 4096, 0, 0, 0, 0, 54, 2029, 1221, 398, 244, 0, 0, 0, 0, 0, 203, 46, 139, 12, 0, 0, 0, 11, 3830, 3174, 316, 0, 0, 0, 0, 0, 0, 0, 235, 0, 6, 2, 2824, 4067, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7522, 8804, 7877, 7510, 10431, 8527, 7731, 7134] avg=8192.0000 max=10431 max_rank=4 min=7134 min_rank=7 imbalance_ratio=1.273315 max_load_reduction=55105 max_load_reduction_pct=0.840836 moved_to_echo=33823 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 35, 2844, 3523, 0, 3515, 1, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 32, 0, 0, 97, 0, 4096, 3774, 0, 0, 0, 0, 0, 0, 0, 3603, 2601, 0, 141, 4036, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1356, 0, 0, 1, 3, 4088, 0, 581, 384, 0, 0, 0, 0, 0, 0, 0, 3461, 667, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 1, 0, 21, 1578, 1198, 1092, 245, 0, 0, 0, 0, 0, 0, 59, 0, 0, 50, 7, 7, 31, 3827, 3184, 322, 0, 0, 0, 0, 0, 0, 0, 1474, 0, 4, 2623, 2773, 3930, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9918, 8070, 10498, 6413, 4145, 8201, 7487, 10804] avg=8192.0000 max=10804 max_rank=7 min=4145 min_rank=4 imbalance_ratio=1.318848 max_load_reduction=54732 max_load_reduction_pct=0.835144 moved_to_echo=30637 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 32, 1, 4051, 0, 3430, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1, 5, 0, 0, 47, 0, 4080, 3765, 0, 0, 0, 0, 0, 0, 2, 1995, 2335, 0, 448, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 4014, 0, 0, 0, 8, 4093, 1, 567, 345, 0, 0, 0, 0, 0, 0, 0, 3251, 644, 97, 3681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 845, 0, 0, 18, 0, 258, 1236, 981, 236, 0, 0, 0, 0, 0, 2203, 69, 0, 11, 156, 0, 0, 13, 4046, 3071, 320, 0, 0, 0, 0, 0, 0, 0, 35, 0, 43, 0, 2860, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7516, 7933, 8728, 9100, 7673, 7670, 9889, 7027] avg=8192.0000 max=9889 max_rank=6 min=7027 min_rank=7 imbalance_ratio=1.207153 max_load_reduction=55647 max_load_reduction_pct=0.849106 moved_to_echo=32259 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 248, 3977, 6, 0, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3775, 0, 0, 0, 0, 0, 0, 0, 574, 1800, 3684, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1479, 0, 1047, 0, 0, 4096, 0, 581, 266, 0, 0, 0, 0, 0, 0, 0, 4088, 667, 0, 4036, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1319, 1236, 756, 245, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 3184, 321, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 131, 2860, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7742, 7871, 10077, 7469, 10029, 7652, 7602, 7094] avg=8192.0000 max=10077 max_rank=2 min=7094 min_rank=7 imbalance_ratio=1.230103 max_load_reduction=55459 max_load_reduction_pct=0.846237 moved_to_echo=32377 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[30, 0, 4096, 3768, 2537, 80, 2725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 497, 0, 0, 0, 0, 4096, 3706, 0, 0, 0, 0, 0, 0, 0, 2522, 2596, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 447, 28, 21, 0, 2367, 982, 456, 388, 0, 0, 0, 0, 0, 0, 0, 3734, 653, 0, 2832, 834, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3533, 1152, 0, 0, 17, 33, 381, 1105, 1097, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 11, 13, 0, 3819, 3145, 315, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 2545, 3400, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13236, 8462, 5493, 8746, 8064, 7564, 7358, 6613] avg=8192.0000 max=13236 max_rank=0 min=5493 min_rank=2 imbalance_ratio=1.615723 max_load_reduction=52300 max_load_reduction_pct=0.798035 moved_to_echo=29599 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 90, 3948, 534, 0, 3443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4087, 3773, 0, 0, 0, 0, 0, 0, 0, 3427, 346, 16, 0, 20, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 4, 0, 4, 95, 990, 0, 570, 50, 0, 0, 0, 0, 0, 0, 0, 3414, 667, 422, 4022, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3033, 0, 0, 0, 1, 0, 3805, 1236, 144, 245, 0, 0, 0, 0, 0, 130, 22, 8, 0, 0, 5, 53, 519, 4087, 3184, 321, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3811, 2859, 4004, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8016, 7869, 3812, 5748, 12621, 8464, 8329, 10677] avg=8192.0000 max=12621 max_rank=4 min=3812 min_rank=2 imbalance_ratio=1.540649 max_load_reduction=52915 max_load_reduction_pct=0.807419 moved_to_echo=40091 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 63, 1, 2686, 0, 3506, 14, 0, 0, 0, 0, 0, 0, 0, 0, 126, 1, 0, 0, 12, 0, 0, 116, 4096, 3768, 0, 0, 0, 0, 0, 0, 0, 3458, 2608, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 22, 762, 0, 0, 3935, 25, 580, 385, 0, 0, 0, 0, 0, 0, 0, 2981, 667, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 5, 2, 0, 0, 0, 3463, 1232, 1095, 245, 0, 0, 0, 0, 0, 1249, 0, 0, 4, 0, 0, 0, 6, 4063, 3184, 321, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1599, 2851, 4040, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6289, 8119, 10114, 9804, 3749, 10135, 8827, 8499] avg=8192.0000 max=10135 max_rank=5 min=3749 min_rank=4 imbalance_ratio=1.237183 max_load_reduction=55401 max_load_reduction_pct=0.845352 moved_to_echo=31210 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[33, 0, 44, 1725, 16, 0, 3479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 4073, 3773, 0, 0, 0, 0, 0, 0, 25, 3493, 2473, 2944, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2186, 2, 38, 0, 3479, 4096, 4, 575, 365, 0, 0, 0, 0, 0, 0, 0, 4089, 646, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4073, 0, 19, 127, 0, 1, 599, 1228, 1038, 237, 0, 0, 0, 0, 0, 808, 0, 0, 0, 0, 5, 2, 0, 4095, 3081, 322, 0, 0, 0, 0, 0, 0, 13, 0, 2, 0, 741, 2840, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5297, 8408, 8937, 10746, 8824, 7323, 8313, 7688] avg=8192.0000 max=10746 max_rank=3 min=5297 min_rank=0 imbalance_ratio=1.311768 max_load_reduction=54790 max_load_reduction_pct=0.836029 moved_to_echo=32148 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 29184, 5376, 640, 18816, 12032, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 4608, 16384, 17280, 128, 27136, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 31104, 2048, 128, 128, 128, 128, 13440, 9728, 6656, 2048, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4608, 256, 256, 128, 384, 128, 128, 640, 31872, 25216, 2560, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 2432, 128, 128, 8960, 22528, 31744, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 22272, 15744, 7168, 640, 19840, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2048, 128, 128, 640, 128, 0, 256, 128, 32640, 30080, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 25344, 512, 1920, 128, 3712, 26496, 1024, 4608, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 2942, 4096, 4035, 0, 343, 8, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 51, 0, 4096, 58, 4069, 304, 0, 4084, 0, 0, 0, 4094, 0, 4056, 0, 13, 4027, 7, 27, 4096, 0, 0, 0, 3516, 0, 60, 13, 4077, 60, 4041, 22, 0, 3, 0, 1163, 0, 8, 4089, 3969, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 4040, 4094, 4096, 0, 0, 0, 0, 188, 0, 0, 2734, 48, 0, 0, 0, 1, 0, 0, 3826, 16, 3983, 1397, 7, 3871, 1, 0, 0, 3881, 189, 3116, 33, 263, 4096, 7, 0, 4058, 28, 0, 0, 3945, 5, 0, 0, 3553, 0, 3700, 0, 1, 0, 343, 0, 532, 939, 4096, 0, 0, 0, 1, 95, 4096, 0, 0, 257] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4034, 3651, 4084, 0, 0, 11, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 4048, 0, 4096, 0, 1, 4048, 3, 0, 0, 4088, 50, 4085, 59, 0, 4068, 0, 0, 4086, 0, 210, 0, 4069, 0, 0, 103, 4096, 2, 3979, 0, 2, 0, 4, 97, 0, 819, 4094, 3359, 10, 1, 0, 0, 0, 136, 31, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 3825, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 114, 0, 0, 0, 0, 0, 136, 4077, 0, 4096, 493, 0, 4096, 0, 0, 0, 4096, 18, 4096, 0, 2, 3931, 0, 0, 3111, 0, 1889, 0, 4092, 0, 0, 0, 2420, 0, 2198, 0, 0, 0, 0, 0, 1, 3, 4096, 2427, 0, 0, 0, 0, 4034, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 3022, 4096, 4088, 0, 0, 0, 0, 3, 0, 0, 3947, 0, 0, 2, 4, 1, 0, 76, 4095, 0, 4095, 4006, 0, 4096, 0, 0, 0, 3652, 121, 4081, 2, 0, 3641, 0, 0, 4096, 16, 4095, 0, 607, 0, 19, 1, 3496, 6, 4096, 19, 0, 3, 176, 481, 200, 0, 4094, 991, 1, 1, 0, 97, 5, 0, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 697, 4090, 4096, 0, 34, 4, 0, 26, 0, 0, 3863, 0, 0, 0, 0, 0, 23, 1, 4086, 0, 4094, 4085, 0, 3333, 0, 0, 0, 4095, 15, 4096, 0, 0, 4095, 3, 3, 3827, 0, 3960, 0, 3869, 0, 0, 0, 4005, 195, 4096, 198, 10, 0, 120, 8, 0, 0, 4093, 206, 0, 0, 0, 96, 110, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[9, 4002, 4094, 3963, 1, 0, 0, 0, 0, 0, 2, 4088, 0, 0, 0, 11, 0, 0, 0, 4066, 79, 4029, 3611, 4, 4096, 0, 6, 0, 4063, 60, 4064, 0, 36, 4048, 23, 0, 3205, 31, 835, 0, 1763, 0, 140, 0, 4096, 0, 4030, 0, 0, 116, 50, 3, 0, 1, 4096, 2635, 33, 4, 0, 8, 106, 0, 4, 25] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[6, 0, 3797, 2063, 0, 4, 0, 0, 0, 2, 0, 2615, 0, 67, 0, 1020, 0, 13, 1, 4078, 0, 4094, 3879, 9, 383, 0, 36, 49, 3981, 0, 3825, 105, 1660, 3704, 0, 401, 2965, 12, 4096, 624, 0, 0, 4057, 0, 1143, 0, 4088, 226, 0, 14, 2556, 3115, 0, 0, 4061, 9, 0, 283, 1634, 243, 270, 0, 347, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4034, 1371, 4084, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 2219, 1522, 682, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 1877, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 3, 0, 0, 687, 50, 4085, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2299, 0, 0, 4086, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 103, 4096, 2, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 97, 0, 819, 4094, 3359, 1087, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 136, 31, 14, 3401, 2280, 921, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9500, 8521, 5926, 8932, 6595, 9806, 9462, 6794] avg=8192.0000 max=9806 max_rank=5 min=5926 min_rank=2 imbalance_ratio=1.197021 max_load_reduction=55730 max_load_reduction_pct=0.850372 moved_to_echo=32657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2942, 1539, 4035, 0, 343, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2205, 1546, 675, 0, 0, 0, 0, 0, 0, 51, 0, 4096, 58, 1864, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 688, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2276, 7, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3516, 0, 60, 13, 4077, 60, 1560, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1163, 0, 8, 4089, 3969, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3406, 2557, 935, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8879, 8522, 6373, 8828, 6419, 9308, 10308, 6899] avg=8192.0000 max=10308 max_rank=6 min=6373 min_rank=2 imbalance_ratio=1.258301 max_load_reduction=55228 max_load_reduction_pct=0.842712 moved_to_echo=32934 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4040, 1538, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 2734, 48, 0, 0, 0, 2158, 1415, 687, 0, 0, 0, 0, 0, 1, 0, 0, 3826, 16, 1825, 1397, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 1, 0, 0, 652, 189, 3116, 33, 0, 0, 0, 0, 0, 0, 0, 0, 263, 2314, 7, 0, 4058, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 5, 0, 0, 3553, 0, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 343, 0, 532, 939, 4096, 0, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 95, 4096, 0, 0, 257, 3229, 2556, 856, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9674, 7230, 7072, 7862, 6670, 8932, 7006, 11090] avg=8192.0000 max=11090 max_rank=7 min=6670 min_rank=4 imbalance_ratio=1.353760 max_load_reduction=54446 max_load_reduction_pct=0.830780 moved_to_echo=33698 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3825, 1539, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 114, 0, 0, 0, 2219, 840, 659, 0, 0, 0, 0, 0, 0, 0, 136, 4077, 0, 1877, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 688, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2221, 0, 0, 3111, 0, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 2420, 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4096, 2427, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 3408, 2557, 508, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9460, 7925, 6583, 8898, 7223, 7362, 7578, 10507] avg=8192.0000 max=10507 max_rank=7 min=6583 min_rank=2 imbalance_ratio=1.282593 max_load_reduction=55029 max_load_reduction_pct=0.839676 moved_to_echo=32670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 697, 1536, 4096, 0, 34, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 3863, 0, 0, 0, 0, 2218, 1567, 687, 0, 0, 0, 0, 0, 0, 23, 1, 4086, 0, 1876, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3333, 0, 0, 0, 688, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2313, 3, 3, 3827, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3869, 0, 0, 0, 4005, 195, 1581, 198, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 120, 8, 0, 0, 4093, 206, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 110, 0, 4, 0, 3407, 2554, 948, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6367, 8361, 10071, 8132, 10106, 9848, 5532, 7119] avg=8192.0000 max=10106 max_rank=4 min=5532 min_rank=6 imbalance_ratio=1.233643 max_load_reduction=55430 max_load_reduction_pct=0.845795 moved_to_echo=32605 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3022, 1539, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3947, 0, 0, 2, 4, 2219, 1567, 611, 0, 0, 0, 0, 0, 1, 0, 76, 4095, 0, 1876, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 614, 121, 4081, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2057, 0, 0, 4096, 16, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 19, 1, 3496, 6, 1581, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 176, 481, 200, 0, 4094, 991, 973, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 97, 5, 0, 8, 0, 3038, 2557, 948, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8649, 8353, 10054, 8914, 10264, 5729, 6918, 6655] avg=8192.0000 max=10264 max_rank=4 min=5729 min_rank=5 imbalance_ratio=1.252930 max_load_reduction=55272 max_load_reduction_pct=0.843384 moved_to_echo=29566 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 1421, 2063, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2615, 0, 67, 0, 1020, 2223, 1567, 626, 0, 0, 0, 0, 0, 0, 13, 1, 4078, 0, 1871, 3879, 9, 0, 0, 0, 0, 0, 0, 0, 0, 383, 0, 36, 49, 664, 0, 3825, 105, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 2084, 0, 401, 2965, 12, 4096, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 1143, 0, 1570, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2556, 3115, 0, 0, 4061, 9, 994, 0, 0, 0, 0, 0, 0, 0, 0, 283, 1634, 243, 270, 0, 347, 1, 3317, 2376, 951, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3494, 8120, 9851, 5062, 11842, 6996, 10749, 9422] avg=8192.0000 max=11842 max_rank=4 min=3494 min_rank=0 imbalance_ratio=1.445557 max_load_reduction=53694 max_load_reduction_pct=0.819305 moved_to_echo=39009 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[9, 4002, 1538, 3963, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4088, 0, 0, 0, 11, 2183, 1541, 679, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 79, 1846, 3611, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 682, 60, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 2287, 23, 0, 3205, 31, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1763, 0, 140, 0, 4096, 0, 1556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 50, 3, 0, 1, 4096, 2635, 1082, 0, 0, 0, 0, 0, 0, 0, 33, 4, 0, 8, 106, 0, 4, 25, 3381, 2556, 933, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9513, 8504, 9606, 8908, 6417, 7555, 7983, 7050] avg=8192.0000 max=9606 max_rank=2 min=6417 min_rank=4 imbalance_ratio=1.172607 max_load_reduction=55930 max_load_reduction_pct=0.853424 moved_to_echo=29005 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 22656, 12032, 30592, 128, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 384, 1664, 640, 8704, 256, 512, 384, 26624, 20096, 7040, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[21888, 128, 4352, 128, 27008, 384, 11776, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2048, 17920, 128, 512, 29568, 128, 15104, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[28032, 128, 128, 128, 5376, 512, 31488, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 256, 32384, 256, 14976, 17792, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 256, 3328, 4992, 768, 1792, 32768, 13696, 8576, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 128, 128, 29568, 256, 128, 128, 1152, 17664, 11648, 5376, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[101, 4093, 0, 0, 0, 1097, 0, 61, 4050, 0, 0, 0, 1, 22, 0, 0, 4096, 2, 0, 0, 41, 4079, 7, 1, 4089, 3, 0, 3941, 0, 7, 4095, 3968, 3649, 0, 0, 4090, 0, 5, 2770, 4090, 4057, 135, 124, 8, 0, 0, 0, 0, 0, 4, 201, 0, 0, 0, 2, 4039, 4056, 4092, 0, 442, 0, 18, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 3955, 119, 0, 2860, 1268, 61, 4095, 4092, 44, 30, 0, 0, 1359, 13, 0, 4096, 0, 72, 4, 251, 4078, 0, 10, 0, 46, 0, 3741, 0, 0, 2714, 4056, 4059, 0, 27, 4047, 7, 0, 3521, 4067, 4065, 1, 4, 0, 0, 0, 0, 15, 177, 0, 94, 0, 0, 0, 8, 4037, 206, 4031, 0, 164, 0, 0, 40, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[271, 4096, 0, 0, 533, 2244, 0, 29, 1711, 4, 0, 4, 107, 0, 0, 76, 3876, 0, 0, 0, 0, 4089, 0, 0, 4095, 2, 0, 4077, 11, 41, 4082, 3799, 3877, 0, 5, 4085, 0, 225, 4054, 4028, 3961, 54, 2, 0, 0, 0, 0, 118, 0, 0, 184, 0, 0, 0, 82, 3492, 4096, 4085, 36, 5, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4000, 4096, 0, 0, 0, 3, 0, 1731, 3881, 0, 0, 0, 0, 111, 1, 0, 2727, 0, 3, 0, 0, 4095, 0, 0, 3988, 0, 0, 4096, 0, 0, 4065, 4096, 4095, 0, 0, 4096, 0, 0, 15, 4096, 4096, 0, 0, 0, 0, 2, 3, 0, 0, 0, 30, 0, 0, 0, 0, 4096, 3356, 4096, 0, 662, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[2218, 4082, 2, 0, 2, 0, 0, 1954, 2880, 0, 15, 0, 92, 6, 1, 0, 1706, 0, 0, 7, 4, 4096, 0, 0, 3834, 0, 0, 4096, 4, 0, 4094, 4094, 4041, 0, 2, 4062, 105, 28, 542, 4089, 4096, 19, 0, 8, 0, 1, 0, 0, 5, 95, 0, 0, 0, 3, 153, 4088, 2952, 4063, 0, 3997, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1053, 3748, 8, 0, 1695, 0, 0, 2, 3952, 14, 0, 0, 3181, 1446, 45, 0, 2942, 25, 0, 0, 0, 4096, 0, 0, 3850, 299, 0, 2554, 266, 0, 3729, 4040, 1673, 7, 0, 4051, 4, 27, 4096, 4096, 4096, 0, 0, 2, 0, 6, 0, 186, 46, 227, 235, 1, 0, 14, 18, 4096, 825, 4096, 0, 788, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[2559, 4091, 0, 0, 332, 627, 0, 4, 4071, 1, 4, 0, 0, 109, 0, 0, 4096, 0, 28, 0, 32, 4083, 0, 0, 4096, 1, 0, 4039, 1, 0, 4096, 4094, 3660, 1, 1, 4090, 0, 11, 0, 4093, 4073, 0, 0, 0, 0, 30, 618, 0, 0, 0, 0, 0, 0, 0, 18, 4079, 3840, 4095, 0, 557, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4077, 4093, 0, 0, 0, 0, 0, 779, 4089, 0, 186, 0, 164, 0, 0, 0, 3695, 0, 0, 0, 629, 4022, 9, 0, 1, 15, 0, 4084, 0, 0, 3902, 4020, 3331, 0, 1, 4095, 0, 1, 58, 4095, 4095, 0, 1, 0, 3, 7, 0, 0, 1, 0, 0, 0, 0, 1, 173, 4096, 3999, 4096, 0, 3718, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3894, 119, 0, 2860, 1268, 61, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 44, 30, 0, 0, 1359, 13, 0, 1809, 971, 180, 0, 0, 0, 0, 0, 4096, 0, 72, 4, 251, 4078, 0, 10, 391, 109, 61, 0, 0, 0, 0, 0, 0, 46, 0, 3741, 0, 0, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 27, 2685, 7, 0, 3521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 1, 4, 0, 0, 0, 0, 15, 3876, 113, 0, 0, 0, 0, 0, 0, 177, 0, 94, 0, 0, 0, 8, 4037, 3954, 0, 0, 0, 0, 0, 0, 0, 206, 3922, 0, 164, 0, 0, 40, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12297, 8498, 9072, 4692, 10299, 8074, 8270, 4334] avg=8192.0000 max=12297 max_rank=0 min=4334 min_rank=7 imbalance_ratio=1.501099 max_load_reduction=53239 max_load_reduction_pct=0.812363 moved_to_echo=30977 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[101, 4030, 0, 0, 0, 1097, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 1, 22, 0, 0, 2730, 981, 177, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 41, 4079, 7, 1, 395, 110, 63, 0, 0, 0, 0, 0, 4089, 3, 0, 3941, 0, 7, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3649, 0, 0, 2714, 0, 5, 2770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 135, 124, 8, 0, 0, 0, 0, 3791, 115, 0, 0, 0, 0, 0, 0, 0, 4, 201, 0, 0, 0, 2, 4039, 3975, 0, 0, 0, 0, 0, 0, 0, 4056, 3982, 0, 442, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5289, 7961, 8794, 9405, 9138, 8230, 8221, 8498] avg=8192.0000 max=9405 max_rank=3 min=5289 min_rank=0 imbalance_ratio=1.148071 max_load_reduction=56131 max_load_reduction_pct=0.856491 moved_to_echo=34087 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[271, 4033, 0, 0, 533, 2244, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 1711, 4, 0, 4, 107, 0, 0, 76, 2721, 980, 168, 0, 0, 0, 0, 0, 3876, 0, 0, 0, 0, 4089, 0, 0, 394, 110, 63, 0, 0, 0, 0, 0, 4095, 2, 0, 4077, 11, 41, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 5, 2711, 0, 225, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 54, 2, 0, 0, 0, 0, 118, 3631, 112, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 82, 3492, 3916, 0, 0, 0, 0, 0, 0, 0, 4096, 3975, 36, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7110, 5771, 8532, 9587, 10872, 7878, 7674, 8112] avg=8192.0000 max=10872 max_rank=4 min=5771 min_rank=1 imbalance_ratio=1.327148 max_load_reduction=54664 max_load_reduction_pct=0.834106 moved_to_echo=34536 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4000, 4033, 0, 0, 0, 3, 0, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 0, 0, 111, 1, 0, 2710, 982, 182, 0, 0, 0, 0, 0, 2727, 0, 3, 0, 0, 4095, 0, 0, 395, 110, 63, 0, 0, 0, 0, 0, 3988, 0, 0, 4096, 0, 0, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2719, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 3, 0, 3914, 115, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 4096, 3981, 0, 0, 0, 0, 0, 0, 0, 3356, 3986, 0, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9767, 7867, 7393, 9439, 6829, 8130, 8107, 8004] avg=8192.0000 max=9767 max_rank=0 min=6829 min_rank=4 imbalance_ratio=1.192261 max_load_reduction=55769 max_load_reduction_pct=0.850967 moved_to_echo=31070 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1053, 3684, 8, 0, 1695, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 14, 0, 0, 3181, 1446, 45, 0, 2488, 975, 180, 0, 0, 0, 0, 0, 2942, 25, 0, 0, 0, 4096, 0, 0, 395, 116, 64, 0, 0, 0, 0, 0, 3850, 299, 0, 2554, 266, 0, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1673, 7, 0, 2681, 4, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 6, 0, 186, 3860, 115, 0, 0, 0, 0, 0, 0, 46, 227, 235, 1, 0, 14, 18, 4096, 3981, 0, 0, 0, 0, 0, 0, 0, 825, 3980, 0, 788, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6442, 12281, 7638, 8210, 8488, 8265, 8618, 5594] avg=8192.0000 max=12281 max_rank=1 min=5594 min_rank=7 imbalance_ratio=1.499146 max_load_reduction=53255 max_load_reduction_pct=0.812607 moved_to_echo=30965 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2218, 4019, 2, 0, 2, 0, 0, 1954, 0, 0, 0, 0, 0, 0, 0, 0, 2880, 0, 15, 0, 92, 6, 1, 0, 2729, 974, 182, 0, 0, 0, 0, 0, 1706, 0, 0, 7, 4, 4096, 0, 0, 392, 109, 63, 0, 0, 0, 0, 0, 3834, 0, 0, 4096, 4, 0, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 2, 2696, 105, 28, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 8, 0, 1, 0, 0, 3912, 114, 0, 0, 0, 0, 0, 0, 5, 95, 0, 0, 0, 3, 153, 4088, 3975, 0, 0, 0, 0, 0, 0, 0, 2952, 3954, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 6879, 6377, 9299, 7414, 8150, 8319, 10903] avg=8192.0000 max=10903 max_rank=7 min=6377 min_rank=2 imbalance_ratio=1.330933 max_load_reduction=54633 max_load_reduction_pct=0.833633 moved_to_echo=34786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2559, 4028, 0, 0, 332, 627, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 1, 4, 0, 0, 109, 0, 0, 2730, 981, 182, 0, 0, 0, 0, 0, 4096, 0, 28, 0, 32, 4083, 0, 0, 395, 110, 63, 0, 0, 0, 0, 0, 4096, 1, 0, 4039, 1, 0, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3660, 1, 1, 2714, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 30, 618, 0, 3912, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4079, 3979, 0, 0, 0, 0, 0, 0, 0, 3840, 3985, 0, 557, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7550, 8078, 8807, 9503, 6387, 8747, 8076, 8388] avg=8192.0000 max=9503 max_rank=3 min=6387 min_rank=4 imbalance_ratio=1.160034 max_load_reduction=56033 max_load_reduction_pct=0.854996 moved_to_echo=31598 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4077, 4030, 0, 0, 0, 0, 0, 779, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 186, 0, 164, 0, 0, 0, 2601, 982, 179, 0, 0, 0, 0, 0, 3695, 0, 0, 0, 629, 4022, 9, 0, 395, 110, 63, 0, 0, 0, 0, 0, 1, 15, 0, 4084, 0, 0, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3331, 0, 1, 2718, 0, 1, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 3, 7, 0, 0, 3841, 115, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 173, 4096, 3980, 0, 0, 0, 0, 0, 0, 0, 3999, 3986, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8886, 8201, 8923, 5401, 6109, 8062, 8251, 11703] avg=8192.0000 max=11703 max_rank=7 min=5401 min_rank=3 imbalance_ratio=1.428589 max_load_reduction=53833 max_load_reduction_pct=0.821426 moved_to_echo=34125 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[14336, 31872, 256, 0, 5504, 5248, 128, 8704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[28416, 128, 128, 21760, 128, 384, 15104, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[24064, 384, 0, 30720, 384, 128, 10368, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[23424, 31872, 128, 10368, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32640, 256, 256, 128, 128, 128, 640, 384, 30848, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[27264, 128, 128, 128, 1024, 32640, 128, 128, 3200, 896, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 384, 768, 128, 0, 128, 512, 32128, 31744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[28800, 128, 256, 128, 3584, 3072, 128, 128, 20608, 7936, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 734, 0, 336, 4089, 0, 0, 37, 0, 2795, 0, 0, 2539, 34, 0, 4096, 0, 0, 2568, 1057, 4096, 0, 0, 24, 3844, 0, 0, 66, 0, 0, 0, 0, 0, 3502, 3537, 4082, 30, 0, 0, 4047, 0, 247, 0, 0, 0, 0, 0, 2, 2956, 3776, 4073, 230, 4096, 4096, 0, 0, 12, 128, 0, 0, 311] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[259, 4094, 263, 0, 40, 1, 2102, 4023, 0, 24, 3900, 0, 0, 17, 853, 0, 0, 85, 3657, 1, 0, 94, 915, 4049, 0, 0, 61, 19, 0, 1125, 5, 0, 4096, 58, 0, 0, 3989, 3983, 4065, 8, 2, 81, 4053, 0, 0, 40, 2, 0, 1, 0, 122, 3443, 3219, 3974, 27, 4089, 4056, 3, 0, 593, 45, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4014, 0, 0, 0, 0, 2304, 4046, 0, 242, 145, 0, 1, 0, 0, 0, 58, 0, 3994, 0, 0, 6, 592, 4096, 0, 0, 0, 4095, 3, 52, 6, 0, 4096, 0, 0, 0, 3716, 3731, 4096, 0, 0, 1632, 4096, 0, 5, 0, 116, 0, 0, 0, 0, 4096, 3926, 4084, 0, 4096, 4096, 0, 0, 96, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[6, 3347, 0, 0, 0, 0, 256, 1405, 31, 374, 3879, 51, 2, 0, 0, 0, 0, 0, 3741, 0, 0, 3852, 36, 4088, 0, 0, 0, 4066, 802, 1473, 22, 0, 3465, 0, 3, 0, 0, 4056, 4096, 0, 8, 0, 4083, 25, 0, 0, 1607, 0, 0, 0, 0, 4075, 3916, 4089, 5, 4054, 4095, 0, 0, 270, 5, 44, 209, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4095, 3, 0, 0, 0, 4094, 4096, 0, 19, 4092, 0, 0, 0, 0, 3412, 0, 57, 12, 0, 0, 4088, 514, 4095, 0, 28, 0, 402, 0, 0, 25, 0, 3898, 0, 3, 0, 42, 4096, 3620, 0, 0, 3912, 2007, 0, 0, 1, 0, 182, 1, 0, 6, 4078, 1261, 4095, 1, 4095, 4096, 1, 0, 157, 0, 952, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 3942, 0, 0, 0, 16, 1785, 4096, 0, 161, 36, 6, 0, 0, 0, 442, 3604, 270, 4004, 0, 31, 3677, 31, 3969, 8, 0, 0, 4040, 0, 76, 13, 93, 4082, 0, 3, 0, 185, 3680, 3882, 0, 0, 0, 4089, 0, 0, 0, 2, 0, 0, 0, 0, 4056, 1494, 4095, 0, 4073, 4070, 0, 6, 1389, 0, 1, 129, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 2364, 31, 0, 1, 0, 4037, 179, 0, 1516, 0, 0, 832, 7, 3, 0, 320, 0, 3185, 0, 0, 4096, 0, 3904, 0, 0, 0, 4089, 0, 14, 0, 25, 4094, 0, 0, 0, 213, 4060, 4093, 0, 0, 0, 4018, 1, 0, 0, 13, 0, 6, 0, 29, 3943, 4082, 4035, 0, 4084, 4075, 0, 0, 3926, 0, 0, 261, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[2, 179, 309, 0, 0, 0, 1681, 3815, 0, 4079, 2382, 10, 0, 0, 0, 17, 4, 761, 47, 0, 0, 3998, 5, 3404, 0, 0, 0, 21, 1593, 4089, 1194, 49, 3633, 0, 0, 0, 4096, 3211, 4077, 0, 0, 0, 4065, 772, 0, 37, 282, 2, 0, 0, 2335, 2918, 1, 4094, 0, 3792, 4068, 4, 0, 507, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[259, 3373, 263, 0, 40, 1, 2102, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 3900, 0, 0, 17, 853, 0, 3864, 403, 0, 0, 0, 0, 0, 0, 0, 85, 3657, 1, 0, 94, 915, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 19, 0, 1125, 5, 0, 3974, 201, 0, 0, 0, 0, 0, 0, 4096, 58, 0, 0, 3989, 2930, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2, 81, 4053, 0, 0, 40, 2, 0, 2229, 1080, 0, 0, 0, 0, 0, 0, 1, 0, 122, 3443, 3219, 0, 27, 1457, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 3, 0, 593, 45, 0, 0, 0, 1430, 1053, 721, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10061, 9061, 6291, 5385, 11081, 7487, 8269, 7901] avg=8192.0000 max=11081 max_rank=4 min=5385 min_rank=3 imbalance_ratio=1.352661 max_load_reduction=54455 max_load_reduction_pct=0.830917 moved_to_echo=34738 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3375, 0, 0, 734, 0, 336, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 2795, 0, 0, 2539, 3881, 404, 0, 0, 0, 0, 0, 0, 34, 0, 4096, 0, 0, 2568, 1057, 1558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 3844, 0, 0, 66, 0, 4073, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 2602, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 247, 0, 0, 0, 2233, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2956, 3776, 0, 230, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 128, 0, 0, 311, 1446, 935, 721, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8534, 9656, 9313, 8208, 6134, 7619, 8423, 7649] avg=8192.0000 max=9656 max_rank=1 min=6134 min_rank=4 imbalance_ratio=1.178711 max_load_reduction=55880 max_load_reduction_pct=0.852661 moved_to_echo=29825 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 2758, 0, 0, 0, 0, 256, 1405, 0, 0, 0, 0, 0, 0, 0, 0, 31, 374, 3879, 51, 2, 0, 0, 0, 3895, 400, 0, 0, 0, 0, 0, 0, 0, 0, 3741, 0, 0, 3852, 36, 1554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 802, 1473, 22, 0, 4089, 201, 0, 0, 0, 0, 0, 0, 3465, 0, 3, 0, 0, 2984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4083, 25, 0, 0, 1607, 0, 2210, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 3916, 0, 5, 1444, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 270, 5, 44, 209, 0, 1444, 1072, 589, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4425, 8632, 9183, 10653, 6452, 9023, 9440, 7728] avg=8192.0000 max=10653 max_rank=3 min=4425 min_rank=0 imbalance_ratio=1.300415 max_load_reduction=54883 max_load_reduction_pct=0.837448 moved_to_echo=32643 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3307, 0, 0, 0, 0, 2304, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 145, 0, 1, 0, 0, 0, 3895, 404, 0, 0, 0, 0, 0, 0, 58, 0, 3994, 0, 0, 6, 592, 1558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 52, 6, 0, 4084, 201, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3716, 2744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1632, 4096, 0, 5, 0, 116, 0, 2233, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3926, 0, 0, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 96, 0, 0, 0, 0, 1446, 987, 707, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9657, 4687, 6208, 8441, 10556, 9174, 9481, 7332] avg=8192.0000 max=10556 max_rank=4 min=4687 min_rank=1 imbalance_ratio=1.288574 max_load_reduction=54980 max_load_reduction_pct=0.838928 moved_to_echo=36543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1948, 31, 0, 1, 0, 4037, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1516, 0, 0, 832, 7, 3, 0, 3892, 403, 0, 0, 0, 0, 0, 0, 320, 0, 3185, 0, 0, 4096, 0, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 14, 0, 25, 4035, 201, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 213, 2986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 1, 0, 0, 13, 0, 2226, 1041, 0, 0, 0, 0, 0, 0, 6, 0, 29, 3943, 4082, 0, 0, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 3926, 0, 0, 261, 0, 1379, 1074, 416, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6196, 6653, 9085, 8364, 7293, 7299, 9515, 11131] avg=8192.0000 max=11131 max_rank=7 min=6196 min_rank=0 imbalance_ratio=1.358765 max_load_reduction=54405 max_load_reduction_pct=0.830154 moved_to_echo=35238 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 144, 309, 0, 0, 0, 1681, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 2382, 10, 0, 0, 0, 17, 3877, 376, 0, 0, 0, 0, 0, 0, 4, 761, 47, 0, 0, 3998, 5, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1593, 4089, 1194, 49, 4094, 200, 0, 0, 0, 0, 0, 0, 3633, 0, 0, 0, 4096, 2358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 772, 0, 37, 282, 2, 2069, 912, 0, 0, 0, 0, 0, 0, 0, 0, 2335, 2918, 1, 0, 0, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4, 0, 507, 3, 0, 0, 0, 1205, 853, 35, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5951, 10741, 6102, 11240, 10087, 8139, 6601, 6675] avg=8192.0000 max=11240 max_rank=3 min=5951 min_rank=0 imbalance_ratio=1.372070 max_load_reduction=54296 max_load_reduction_pct=0.828491 moved_to_echo=31502 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3374, 3, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4092, 0, 0, 0, 0, 3412, 3442, 404, 0, 0, 0, 0, 0, 0, 0, 57, 12, 0, 0, 4088, 514, 1557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 402, 0, 0, 25, 0, 4095, 178, 0, 0, 0, 0, 0, 0, 3898, 0, 3, 0, 42, 3013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 2007, 0, 0, 1, 0, 182, 2232, 1092, 0, 0, 0, 0, 0, 0, 1, 0, 6, 4078, 1261, 0, 1, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 157, 0, 952, 0, 0, 1446, 1083, 721, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11567, 11369, 6228, 4728, 6956, 9426, 6806, 8456] avg=8192.0000 max=11567 max_rank=0 min=4728 min_rank=3 imbalance_ratio=1.411987 max_load_reduction=53969 max_load_reduction_pct=0.823502 moved_to_echo=31644 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3248, 0, 0, 0, 16, 1785, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 36, 6, 0, 0, 0, 442, 3690, 402, 0, 0, 0, 0, 0, 0, 3604, 270, 4004, 0, 31, 3677, 31, 1509, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4040, 0, 76, 13, 93, 4095, 192, 0, 0, 0, 0, 0, 0, 4082, 0, 3, 0, 185, 2707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 2, 0, 2220, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 1494, 0, 0, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 6, 1389, 0, 1, 129, 0, 1402, 973, 694, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9145, 4737, 13126, 8517, 6977, 7369, 7001, 8664] avg=8192.0000 max=13126 max_rank=2 min=4737 min_rank=1 imbalance_ratio=1.602295 max_load_reduction=52410 max_load_reduction_pct=0.799713 moved_to_echo=30011 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 21632, 640, 0, 896, 128, 16640, 25856, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 5632, 30464, 896, 256, 128, 2048, 256, 17664, 8576, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[27392, 128, 128, 0, 15744, 22400, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4096, 1280, 22784, 128, 128, 22400, 3200, 12160, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 7040, 256, 1024, 640, 384, 11264, 8064, 4608, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 6528, 14592, 128, 3712, 128, 896, 6528, 30464, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 2560, 29568, 21760, 0, 384, 11648, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 20608, 2432, 6912, 1408, 256, 32640, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[3844, 0, 4091, 8, 334, 0, 4054, 4, 62, 0, 110, 0, 391, 0, 0, 1, 1708, 3981, 1, 0, 226, 0, 3714, 2124, 4095, 2566, 0, 6, 8, 4096, 1, 3903, 35, 23, 1, 3703, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 4089, 45, 3963, 0, 0, 0, 39, 7, 0, 111, 600, 920, 376, 82, 0, 4049, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 34, 3994, 4041, 813, 0, 3714, 0, 0, 3815, 0, 0, 35, 0, 317, 1, 4096, 4096, 0, 0, 30, 93, 0, 0, 0, 0, 0, 6, 0, 3498, 635, 4092, 3823, 0, 0, 565, 0, 4096, 34, 0, 1, 0, 0, 37, 0, 0, 0, 3254, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 4037, 0, 4079, 4096, 0, 4037, 63] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[3912, 0, 4096, 0, 0, 0, 4061, 5, 0, 16, 0, 0, 1, 26, 48, 0, 18, 4044, 0, 124, 0, 2302, 3484, 3839, 4096, 187, 249, 0, 0, 4087, 1, 4095, 0, 0, 2, 0, 0, 4073, 0, 0, 0, 0, 0, 3544, 67, 67, 0, 4057, 0, 21, 6, 1, 0, 40, 0, 281, 0, 3966, 2213, 286, 4095, 0, 4089, 37] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[5, 3, 4093, 0, 0, 2, 3803, 0, 1, 0, 0, 0, 0, 110, 40, 249, 652, 4093, 32, 1, 3862, 0, 2317, 143, 921, 0, 17, 0, 0, 4094, 90, 3951, 1167, 0, 0, 4095, 70, 4090, 1, 0, 3, 0, 0, 3607, 0, 214, 0, 3991, 2, 0, 0, 2, 0, 34, 0, 3552, 3820, 712, 547, 2977, 4089, 0, 4084, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 0, 0, 36, 0, 0, 0, 0, 1, 0, 4070, 4096, 0, 0, 4088, 568, 13, 0, 4094, 0, 32, 0, 0, 4096, 0, 4096, 208, 0, 0, 3713, 0, 4096, 0, 0, 0, 263, 0, 14, 85, 3803, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 3594, 4060, 50, 0, 4087, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 27, 2377, 123, 3857, 0, 0, 1170, 1557, 0, 4093, 0, 3169, 1245, 0, 4090, 1998, 0, 233, 36, 6, 10, 4069, 88, 1, 0, 0, 3911, 2, 2629, 4061, 0, 403, 3918, 0, 2754, 104, 88, 1, 6, 26, 1101, 221, 20, 0, 1316, 0, 0, 0, 0, 0, 1233, 0, 0, 3613, 3989, 67, 0, 972, 53, 2800, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[28, 0, 4092, 4, 1413, 18, 4081, 0, 0, 2809, 0, 0, 164, 0, 0, 10, 3071, 4085, 956, 0, 0, 52, 3750, 3863, 4006, 2, 94, 5, 0, 4002, 6, 2971, 11, 0, 19, 3682, 0, 4092, 0, 2, 0, 370, 1, 103, 0, 1, 0, 4080, 0, 0, 31, 0, 0, 0, 0, 1181, 1, 4092, 1, 197, 4081, 7, 4096, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4040, 0, 2, 0, 4096, 0, 0, 14, 0, 0, 0, 37, 112, 520, 4055, 4096, 0, 0, 2, 2958, 4027, 0, 2830, 0, 1, 1, 14, 4052, 163, 3963, 4091, 0, 124, 4075, 0, 4093, 58, 0, 0, 0, 26, 0, 227, 100, 0, 4096, 10, 2, 1, 0, 0, 0, 0, 694, 1250, 2580, 1197, 0, 4096, 35, 3798, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3844, 0, 2107, 8, 334, 0, 4054, 4, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 110, 0, 391, 0, 0, 1, 3124, 2332, 364, 134, 0, 0, 0, 0, 1708, 369, 1, 0, 226, 0, 3714, 2124, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2566, 0, 6, 8, 972, 1, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 35, 23, 1, 3703, 0, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 45, 3963, 1850, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 7, 0, 111, 600, 4096, 3248, 0, 0, 0, 0, 0, 0, 920, 376, 82, 0, 1717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10351, 6518, 8142, 11551, 7325, 10453, 8101, 3095] avg=8192.0000 max=11551 max_rank=3 min=3095 min_rank=7 imbalance_ratio=1.410034 max_load_reduction=53985 max_load_reduction_pct=0.823746 moved_to_echo=28974 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3912, 0, 2109, 0, 0, 0, 4061, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 1, 26, 48, 0, 3117, 2359, 370, 134, 0, 0, 0, 0, 18, 375, 0, 124, 0, 2302, 3484, 3839, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 187, 249, 0, 0, 970, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3544, 67, 67, 0, 4057, 1853, 506, 0, 0, 0, 0, 0, 0, 0, 21, 6, 1, 0, 40, 0, 281, 4089, 3299, 0, 0, 0, 0, 0, 0, 0, 3966, 2213, 286, 1736, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10087, 6071, 10142, 9598, 3569, 10094, 7737, 8238] avg=8192.0000 max=10142 max_rank=2 min=3569 min_rank=4 imbalance_ratio=1.238037 max_load_reduction=55394 max_load_reduction_pct=0.845245 moved_to_echo=29638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 34, 2056, 4041, 813, 0, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3815, 0, 0, 35, 0, 317, 1, 2668, 2360, 375, 131, 0, 0, 0, 0, 4096, 379, 0, 0, 30, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 830, 635, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 0, 0, 565, 0, 3587, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 37, 0, 0, 0, 3254, 1807, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4037, 3342, 0, 0, 0, 0, 0, 0, 4096, 4037, 0, 4079, 1736, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10658, 9702, 4598, 5563, 8009, 5608, 7387, 14011] avg=8192.0000 max=14011 max_rank=7 min=4598 min_rank=2 imbalance_ratio=1.710327 max_load_reduction=51525 max_load_reduction_pct=0.786209 moved_to_echo=35015 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2109, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 1, 0, 3124, 2354, 375, 134, 0, 0, 0, 0, 4070, 379, 0, 0, 4088, 568, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 32, 0, 0, 972, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 3713, 0, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 0, 14, 85, 3803, 0, 4085, 1853, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3342, 0, 0, 0, 0, 0, 0, 3594, 4060, 50, 0, 1733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6205, 6024, 9118, 9194, 7508, 10612, 7438, 9437] avg=8192.0000 max=10612 max_rank=5 min=6024 min_rank=1 imbalance_ratio=1.295410 max_load_reduction=54924 max_load_reduction_pct=0.838074 moved_to_echo=34995 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 2107, 4, 1413, 18, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2809, 0, 0, 164, 0, 0, 10, 3052, 2351, 374, 134, 0, 0, 0, 0, 3071, 378, 956, 0, 0, 52, 3750, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 2, 94, 5, 0, 950, 6, 2971, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 19, 3682, 0, 3583, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 1, 103, 0, 1, 0, 4080, 1851, 509, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 1181, 4096, 3333, 0, 0, 0, 0, 0, 0, 1, 4092, 1, 197, 1730, 7, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7651, 8894, 12070, 8034, 7297, 6915, 8641, 6034] avg=8192.0000 max=12070 max_rank=2 min=6034 min_rank=7 imbalance_ratio=1.473389 max_load_reduction=53466 max_load_reduction_pct=0.815826 moved_to_echo=28887 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[5, 3, 2108, 0, 0, 2, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 110, 40, 249, 3123, 2356, 374, 134, 0, 0, 0, 0, 652, 380, 32, 1, 3862, 0, 2317, 143, 0, 0, 0, 0, 0, 0, 0, 0, 921, 0, 17, 0, 0, 971, 90, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 0, 0, 4095, 70, 3582, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3607, 0, 214, 0, 3991, 1851, 508, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 34, 0, 3552, 4084, 3339, 0, 0, 0, 0, 0, 0, 3820, 712, 547, 2977, 1733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5921, 6387, 7387, 5950, 8915, 10174, 11013, 9789] avg=8192.0000 max=11013 max_rank=6 min=5921 min_rank=0 imbalance_ratio=1.344360 max_load_reduction=54523 max_load_reduction_pct=0.831955 moved_to_echo=39891 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2100, 27, 2377, 123, 3857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 1557, 0, 4093, 0, 3169, 1245, 2987, 563, 376, 139, 0, 0, 0, 0, 0, 375, 1998, 0, 233, 36, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 88, 1, 0, 0, 924, 2, 2629, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 403, 3918, 0, 2407, 104, 88, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 26, 1101, 221, 20, 0, 1316, 1857, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1233, 0, 0, 2800, 3339, 0, 0, 0, 0, 0, 0, 3613, 3989, 67, 0, 409, 53, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8484, 15299, 2658, 7713, 10981, 4895, 7372, 8134] avg=8192.0000 max=15299 max_rank=1 min=2658 min_rank=2 imbalance_ratio=1.867554 max_load_reduction=50237 max_load_reduction_pct=0.766556 moved_to_echo=31382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2081, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 37, 112, 520, 3091, 2360, 375, 132, 0, 0, 0, 0, 4055, 379, 0, 0, 2, 2958, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2830, 0, 1, 1, 14, 961, 163, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 124, 4075, 0, 3584, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 227, 100, 0, 4096, 1827, 509, 0, 0, 0, 0, 0, 0, 10, 2, 1, 0, 0, 0, 0, 694, 3798, 3342, 0, 0, 0, 0, 0, 0, 1250, 2580, 1197, 0, 1736, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6179, 6641, 11421, 7933, 11932, 6785, 7847, 6798] avg=8192.0000 max=11932 max_rank=4 min=6179 min_rank=0 imbalance_ratio=1.456543 max_load_reduction=53604 max_load_reduction_pct=0.817932 moved_to_echo=33362 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 7936, 1792, 0, 4736, 256, 3712, 2048, 24320, 17152, 3072, 1152, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[17408, 23936, 4224, 7552, 12544, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7808, 128, 16896, 4096, 4992, 256, 31872, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[17792, 3072, 3072, 128, 8448, 6016, 17408, 9984, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 640, 128, 8448, 640, 8320, 128, 28928, 14848, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[13440, 128, 640, 23808, 128, 27520, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[24192, 2944, 512, 128, 128, 7552, 1024, 29824, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 1408, 128, 6400, 31104, 26624, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[67, 4023, 0, 0, 0, 1902, 134, 0, 0, 0, 74, 4071, 4093, 3957, 0, 0, 0, 0, 1, 0, 216, 0, 0, 0, 0, 1, 0, 0, 0, 0, 34, 0, 2545, 4096, 4043, 318, 4, 0, 0, 142, 4094, 4089, 70, 3466, 0, 0, 0, 464, 2173, 0, 0, 3985, 4002, 13, 4096, 47, 4085, 1136, 0, 8, 0, 4087, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4095, 1580, 63, 11, 0, 12, 3, 0, 2639, 0, 4081, 4093, 3999, 2365, 13, 272, 0, 102, 0, 0, 0, 15, 0, 0, 8, 89, 0, 21, 79, 25, 0, 3782, 0, 4076, 0, 0, 43, 0, 1, 414, 4040, 0, 4082, 17, 0, 0, 0, 4071, 14, 0, 4046, 4056, 1, 3579, 0, 4054, 471, 0, 1128, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4082, 1453, 0, 0, 4076, 0, 3, 118, 0, 0, 3923, 4089, 4086, 0, 28, 0, 0, 0, 0, 0, 0, 3458, 0, 0, 2, 170, 9, 0, 0, 0, 0, 4075, 380, 3962, 0, 0, 0, 0, 0, 255, 3754, 0, 4096, 2, 7, 0, 0, 4030, 16, 0, 4096, 3365, 0, 2312, 221, 4072, 0, 134, 0, 5, 4095, 1157, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 3554, 0, 0, 235, 0, 0, 0, 4091, 0, 4096, 4096, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 0, 80, 0, 0, 0, 0, 0, 1833, 13, 4081, 0, 406, 0, 0, 0, 1372, 4081, 11, 3715, 0, 0, 0, 0, 4096, 0, 0, 4091, 4096, 0, 4096, 0, 4096, 0, 435, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4095, 81, 0, 0, 357, 1, 0, 15, 2140, 1, 3999, 4096, 4035, 0, 0, 566, 0, 0, 0, 0, 0, 38, 2, 0, 0, 13, 0, 2, 0, 0, 0, 4096, 0, 4093, 0, 105, 0, 0, 6, 4062, 3250, 0, 4084, 0, 1, 0, 98, 3029, 0, 0, 3964, 4077, 0, 4082, 17, 4088, 140, 2800, 0, 8, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4072, 26, 48, 0, 4084, 0, 0, 0, 3176, 22, 4056, 4084, 4096, 64, 0, 0, 34, 32, 0, 6, 0, 871, 37, 0, 0, 0, 70, 0, 13, 0, 47, 3998, 3, 3903, 0, 278, 0, 0, 67, 3304, 2655, 162, 1912, 20, 12, 0, 0, 4071, 301, 0, 3924, 3962, 6, 4016, 2, 4032, 0, 0, 0, 0, 4062, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[167, 23, 3641, 0, 0, 4096, 0, 0, 964, 22, 0, 2949, 4096, 4096, 0, 2005, 0, 0, 10, 3, 0, 0, 237, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 4096, 0, 3825, 0, 0, 0, 4096, 0, 3, 1980, 0, 33, 0, 9, 23, 0, 0, 4070, 1165, 0, 39, 291, 3913, 1, 186, 3500, 0, 4095, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[3, 4089, 0, 0, 0, 4096, 0, 50, 1, 263, 0, 3665, 4094, 4084, 2521, 0, 12, 0, 13, 1, 0, 0, 3856, 0, 0, 0, 183, 0, 7, 6, 0, 0, 4096, 0, 4054, 0, 0, 0, 51, 0, 72, 1633, 341, 4091, 1, 5, 0, 0, 4044, 4, 13, 3929, 3955, 4, 4093, 1, 4094, 0, 36, 0, 0, 4075, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1580, 63, 11, 0, 12, 3, 642, 365, 0, 0, 0, 0, 0, 0, 0, 2639, 0, 4081, 0, 1744, 2365, 13, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 102, 0, 0, 0, 15, 0, 2255, 2254, 1348, 783, 21, 0, 0, 0, 0, 8, 89, 0, 21, 79, 25, 0, 4093, 4025, 0, 0, 0, 0, 0, 0, 3782, 0, 2928, 0, 0, 43, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 414, 4040, 0, 3440, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 14, 0, 0, 1802, 1, 3579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 471, 0, 1128, 0, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6771, 10842, 7050, 8340, 6754, 7911, 9467, 8401] avg=8192.0000 max=10842 max_rank=1 min=6754 min_rank=4 imbalance_ratio=1.323486 max_load_reduction=54694 max_load_reduction_pct=0.834564 moved_to_echo=32533 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[67, 4023, 0, 0, 0, 1902, 134, 0, 545, 363, 0, 0, 0, 0, 0, 0, 0, 0, 74, 4071, 0, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 216, 0, 0, 0, 2231, 2224, 1345, 776, 20, 0, 0, 0, 0, 1, 0, 0, 0, 0, 34, 0, 4093, 3965, 0, 0, 0, 0, 0, 0, 2545, 4096, 2904, 318, 4, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4089, 70, 2921, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 2173, 0, 0, 0, 1778, 13, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1136, 0, 8, 0, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7034, 5871, 6813, 8093, 10009, 11638, 8107, 7971] avg=8192.0000 max=11638 max_rank=5 min=5871 min_rank=1 imbalance_ratio=1.420654 max_load_reduction=53898 max_load_reduction_pct=0.822418 moved_to_echo=37725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 1453, 0, 0, 4076, 0, 3, 644, 355, 0, 0, 0, 0, 0, 0, 118, 0, 0, 3923, 0, 1782, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3458, 0, 2304, 1870, 1348, 761, 22, 0, 0, 0, 0, 2, 170, 9, 0, 0, 0, 0, 4089, 4074, 0, 0, 0, 0, 0, 0, 4075, 380, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 3754, 0, 3452, 2, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 16, 0, 0, 1495, 0, 2312, 221, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 134, 0, 5, 2747, 1157, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10613, 5851, 9763, 8344, 7301, 7470, 8074, 8120] avg=8192.0000 max=10613 max_rank=0 min=5851 min_rank=1 imbalance_ratio=1.295532 max_load_reduction=54923 max_load_reduction_pct=0.838058 moved_to_echo=30965 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3554, 0, 0, 235, 0, 0, 584, 366, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 1787, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 2309, 2277, 1348, 784, 22, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 4096, 4069, 0, 0, 0, 0, 0, 0, 1833, 13, 2931, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1372, 4081, 11, 3131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1819, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 435, 0, 0, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8835, 9997, 7391, 8245, 5183, 8595, 10011, 7279] avg=8192.0000 max=10011 max_rank=6 min=5183 min_rank=4 imbalance_ratio=1.222046 max_load_reduction=55525 max_load_reduction_pct=0.847244 moved_to_echo=31068 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 81, 0, 0, 357, 1, 0, 642, 367, 0, 0, 0, 0, 0, 0, 15, 2140, 1, 3999, 0, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 0, 0, 0, 0, 0, 38, 2, 2275, 2266, 1348, 786, 21, 0, 0, 0, 0, 0, 13, 0, 2, 0, 0, 0, 4096, 3943, 0, 0, 0, 0, 0, 0, 4096, 0, 2940, 0, 105, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 3250, 0, 3442, 0, 1, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 0, 0, 0, 1811, 0, 4082, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 140, 2800, 0, 8, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5543, 7915, 7302, 8054, 7147, 10853, 8939, 9783] avg=8192.0000 max=10853 max_rank=5 min=5543 min_rank=0 imbalance_ratio=1.324829 max_load_reduction=54683 max_load_reduction_pct=0.834396 moved_to_echo=36722 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 4089, 0, 0, 0, 4096, 0, 50, 643, 364, 0, 0, 0, 0, 0, 0, 1, 263, 0, 3665, 0, 1782, 2521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 13, 1, 0, 0, 3856, 0, 2302, 2198, 1341, 779, 21, 0, 0, 0, 0, 0, 183, 0, 7, 6, 0, 0, 4094, 3908, 0, 0, 0, 0, 0, 0, 4096, 0, 2911, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1633, 341, 3448, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 4, 13, 0, 1757, 4, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 36, 0, 0, 2734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9245, 8232, 10523, 8198, 7058, 5500, 9916, 6864] avg=8192.0000 max=10523 max_rank=2 min=5500 min_rank=5 imbalance_ratio=1.284546 max_load_reduction=55013 max_load_reduction_pct=0.839432 moved_to_echo=29338 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4072, 26, 48, 0, 4084, 0, 0, 300, 350, 0, 0, 0, 0, 0, 0, 0, 3176, 22, 4056, 0, 1787, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 32, 0, 6, 0, 871, 37, 2309, 2202, 1337, 750, 20, 0, 0, 0, 0, 0, 0, 70, 0, 13, 0, 47, 4084, 3904, 0, 0, 0, 0, 0, 0, 3998, 3, 2803, 0, 278, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 3304, 2655, 162, 1612, 20, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 301, 0, 0, 1760, 6, 4016, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 2725, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8880, 9105, 7598, 8118, 7149, 7765, 10156, 6765] avg=8192.0000 max=10156 max_rank=6 min=6765 min_rank=7 imbalance_ratio=1.239746 max_load_reduction=55380 max_load_reduction_pct=0.845032 moved_to_echo=31835 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[167, 23, 3641, 0, 0, 4096, 0, 0, 317, 371, 0, 0, 0, 0, 0, 0, 964, 22, 0, 2949, 0, 1783, 0, 2005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3, 0, 0, 237, 3718, 2313, 652, 1351, 790, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4048, 0, 0, 0, 0, 0, 0, 4096, 4079, 2935, 0, 3825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 1663, 0, 33, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 513, 0, 39, 291, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 1, 186, 3500, 0, 2744, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8615, 7723, 9096, 8144, 14935, 5804, 866, 10353] avg=8192.0000 max=14935 max_rank=4 min=866 min_rank=6 imbalance_ratio=1.823120 max_load_reduction=50601 max_load_reduction_pct=0.772110 moved_to_echo=31958 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[896, 128, 256, 128, 256, 0, 9216, 3840, 18304, 16000, 10880, 6272, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25600, 384, 128, 0, 12800, 128, 26368, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32512, 1792, 3712, 4736, 128, 22016, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[28544, 8576, 23296, 384, 4736, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[17792, 23552, 640, 23168, 128, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[1152, 12416, 128, 30848, 0, 14208, 4992, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 640, 128, 128, 128, 128, 128, 32768, 32000, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 28672, 10368, 128, 128, 18944, 256, 128, 4352, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 416, 3726, 4066, 1232, 3978, 0, 33, 10, 0, 0, 171, 0, 0, 4095, 79, 2349, 0, 4096, 2773, 0, 583, 23, 1, 0, 0, 0, 4, 0, 0, 4090, 0, 4096, 0, 20, 114, 0, 846, 3973, 3, 0, 296, 0, 0, 0, 0, 4085, 0, 3832, 0, 4096, 40, 4075, 0, 4031, 0, 207, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[3167, 21, 0, 0, 2002, 3447, 4013, 0, 4054, 85, 0, 0, 1, 2, 0, 64, 0, 628, 55, 0, 16, 4083, 4049, 7, 1044, 588, 4, 6, 0, 0, 0, 0, 43, 4096, 1, 4070, 13, 0, 0, 0, 3448, 1458, 0, 1, 0, 4, 163, 45, 0, 0, 0, 4078, 0, 4089, 3884, 4091, 0, 4025, 642, 20, 0, 0, 96, 3933] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[1755, 0, 0, 3, 3692, 1929, 4082, 0, 4091, 0, 0, 18, 0, 0, 0, 0, 0, 3544, 0, 151, 0, 4096, 4069, 0, 242, 0, 0, 0, 7, 0, 0, 0, 0, 4065, 50, 3977, 3, 0, 26, 0, 4054, 3287, 5, 0, 1, 0, 0, 0, 5, 255, 79, 4040, 3, 4076, 1287, 4079, 6, 3964, 0, 0, 0, 1013, 16, 3566] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[60, 0, 0, 0, 4092, 2407, 4096, 607, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4029, 0, 4095, 0, 3941, 3583, 0, 13, 791, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 1, 4046, 0, 0, 0, 0, 0, 0, 18, 10, 1, 4096, 0, 4096, 0, 4096, 0, 4095, 0, 0, 1, 977, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[27, 0, 0, 2, 101, 1655, 4094, 72, 4089, 0, 0, 0, 0, 0, 0, 30, 7, 1920, 97, 1033, 0, 4089, 4032, 0, 0, 5, 0, 7, 55, 1, 0, 0, 0, 4087, 3, 4087, 0, 0, 0, 0, 858, 4089, 1, 0, 0, 0, 0, 3, 0, 1, 1726, 3147, 2, 3983, 3634, 3408, 9, 4092, 3439, 0, 84, 3728, 7, 3832] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4037, 571, 4096, 40, 4060, 0, 0, 0, 0, 8, 98, 0, 67, 3648, 252, 3986, 0, 4013, 3640, 4, 441, 0, 0, 0, 41, 0, 22, 0, 1, 4070, 0, 3987, 0, 0, 0, 0, 547, 1889, 0, 0, 1, 12, 3, 0, 0, 0, 12, 4096, 14, 4088, 3, 4083, 151, 3791, 2232, 0, 0, 653, 0, 2783] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 266, 0, 0, 2665, 87, 3782, 3761, 4096, 2, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 437, 4096, 4011, 0, 3355, 0, 18, 0, 0, 0, 3897, 0, 0, 4078, 0, 3212, 0, 1, 0, 441, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 173, 0, 4094, 0, 4096, 3863, 3977, 7, 3966, 67, 9, 37, 2941, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1602, 29, 8, 0, 4090, 4061, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 3588, 0, 0, 0, 3958, 4096, 3, 694, 1, 2, 0, 0, 0, 0, 4, 3, 3390, 0, 4030, 0, 22, 0, 32, 297, 69, 0, 0, 0, 0, 0, 1, 0, 9, 6, 4089, 0, 4096, 315, 4088, 0, 4072, 4088, 0, 42, 2463, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3167, 21, 0, 0, 2002, 3447, 1218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 85, 0, 0, 1, 2, 0, 64, 2771, 1276, 0, 0, 0, 0, 0, 0, 0, 628, 55, 0, 16, 150, 4049, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 588, 4, 6, 0, 0, 0, 0, 4089, 2657, 0, 0, 0, 0, 0, 0, 43, 4096, 1, 4070, 13, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 3448, 1458, 0, 1, 0, 4, 163, 45, 2213, 1833, 24, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 3884, 1878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2037, 642, 20, 0, 0, 96, 3933, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9855, 8253, 4905, 8388, 8378, 9189, 9840, 6728] avg=8192.0000 max=9855 max_rank=0 min=4905 min_rank=2 imbalance_ratio=1.203003 max_load_reduction=55681 max_load_reduction_pct=0.849625 moved_to_echo=34135 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 416, 3726, 1235, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 33, 10, 0, 0, 171, 0, 2807, 1280, 0, 0, 0, 0, 0, 0, 0, 4095, 79, 2349, 0, 151, 2773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 23, 1, 0, 0, 0, 4, 0, 4096, 2665, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 20, 114, 0, 156, 0, 0, 0, 0, 0, 0, 0, 846, 3973, 3, 0, 296, 0, 0, 0, 2204, 1836, 24, 0, 0, 0, 0, 0, 0, 4085, 0, 3832, 0, 0, 40, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2039, 0, 207, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6610, 8279, 9447, 7372, 8476, 9182, 9828, 6342] avg=8192.0000 max=9828 max_rank=6 min=6342 min_rank=7 imbalance_ratio=1.199707 max_load_reduction=55708 max_load_reduction_pct=0.850037 moved_to_echo=33828 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1755, 0, 0, 3, 3692, 1929, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 18, 0, 0, 0, 0, 2818, 1280, 0, 0, 0, 0, 0, 0, 0, 3544, 0, 151, 0, 151, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 0, 0, 7, 0, 0, 0, 4076, 2665, 0, 0, 0, 0, 0, 0, 0, 4065, 50, 3977, 3, 0, 26, 0, 153, 0, 0, 0, 0, 0, 0, 0, 4054, 3287, 5, 0, 1, 0, 0, 0, 2206, 1805, 24, 0, 0, 0, 0, 0, 5, 255, 79, 4040, 3, 0, 1287, 1873, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2006, 0, 0, 0, 1013, 16, 3566, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8619, 8207, 7915, 6990, 8274, 11382, 7542, 6607] avg=8192.0000 max=11382 max_rank=5 min=6607 min_rank=7 imbalance_ratio=1.389404 max_load_reduction=54154 max_load_reduction_pct=0.826324 moved_to_echo=33805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[60, 0, 0, 0, 4092, 2407, 1244, 607, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2828, 1232, 0, 0, 0, 0, 0, 0, 6, 4029, 0, 4095, 0, 145, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 791, 0, 0, 0, 0, 0, 0, 4096, 2564, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 1, 4046, 0, 0, 0, 0, 0, 0, 2216, 1865, 24, 0, 0, 0, 0, 0, 18, 10, 1, 4096, 0, 0, 0, 1880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2072, 0, 0, 1, 977, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8410, 8156, 11858, 7464, 8345, 8152, 6005, 7146] avg=8192.0000 max=11858 max_rank=2 min=6005 min_rank=6 imbalance_ratio=1.447510 max_load_reduction=53678 max_load_reduction_pct=0.819061 moved_to_echo=29648 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1602, 29, 8, 0, 4090, 4061, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2826, 1237, 0, 0, 0, 0, 0, 0, 4, 3588, 0, 0, 0, 146, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 694, 1, 2, 0, 0, 0, 0, 4, 4096, 2575, 0, 0, 0, 0, 0, 0, 3, 3390, 0, 4030, 0, 22, 0, 32, 157, 0, 0, 0, 0, 0, 0, 0, 297, 69, 0, 0, 0, 0, 0, 1, 2211, 1855, 24, 0, 0, 0, 0, 0, 0, 9, 6, 4089, 0, 0, 315, 1877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2060, 4088, 0, 42, 2463, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11033, 8159, 7837, 7372, 7634, 4457, 6296, 12748] avg=8192.0000 max=12748 max_rank=7 min=4457 min_rank=5 imbalance_ratio=1.556152 max_load_reduction=52788 max_load_reduction_pct=0.805481 moved_to_echo=31135 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[27, 0, 0, 2, 101, 1655, 1244, 72, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 30, 2826, 1278, 0, 0, 0, 0, 0, 0, 7, 1920, 97, 1033, 0, 150, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7, 55, 1, 0, 0, 3983, 2661, 0, 0, 0, 0, 0, 0, 0, 4087, 3, 4087, 0, 0, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 858, 4089, 1, 0, 0, 0, 0, 3, 1843, 1864, 24, 0, 0, 0, 0, 0, 0, 1, 1726, 3147, 2, 0, 3634, 1565, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2070, 3439, 0, 84, 3728, 7, 3832, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3101, 8223, 7239, 6712, 8335, 8682, 10075, 13169] avg=8192.0000 max=13169 max_rank=7 min=3101 min_rank=0 imbalance_ratio=1.607544 max_load_reduction=52367 max_load_reduction_pct=0.799057 moved_to_echo=40261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 266, 0, 0, 2665, 87, 1141, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 13, 2615, 1285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 142, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3355, 0, 18, 0, 0, 0, 3897, 0, 4096, 2669, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 3212, 0, 1, 0, 441, 158, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 2156, 1810, 26, 0, 0, 0, 0, 0, 0, 173, 0, 4094, 0, 0, 3863, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1998, 67, 9, 37, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7920, 8011, 4590, 14035, 7890, 8080, 9951, 5059] avg=8192.0000 max=14035 max_rank=3 min=4590 min_rank=2 imbalance_ratio=1.713257 max_load_reduction=51501 max_load_reduction_pct=0.785843 moved_to_echo=30980 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4037, 571, 1244, 40, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 8, 98, 0, 2828, 1254, 0, 0, 0, 0, 0, 0, 67, 3648, 252, 3986, 0, 148, 3640, 4, 0, 0, 0, 0, 0, 0, 0, 0, 441, 0, 0, 0, 41, 0, 22, 0, 4088, 2611, 0, 0, 0, 0, 0, 0, 1, 4070, 0, 3987, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 547, 1889, 0, 0, 1, 12, 3, 0, 2209, 1727, 24, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 14, 0, 3, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 151, 1918, 2232, 0, 0, 653, 0, 2783, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9988, 8248, 11745, 7203, 8204, 6412, 5999, 7737] avg=8192.0000 max=11745 max_rank=2 min=5999 min_rank=6 imbalance_ratio=1.433716 max_load_reduction=53791 max_load_reduction_pct=0.820786 moved_to_echo=28352 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[14208, 18816, 128, 128, 384, 128, 256, 128, 17280, 14720, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[10752, 384, 128, 128, 21120, 17920, 9856, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 21504, 512, 11648, 512, 1280, 30336, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32000, 128, 31616, 128, 128, 256, 512, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 16256, 10496, 256, 256, 11776, 128, 26496, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32640, 128, 128, 128, 128, 128, 384, 128, 22400, 10240, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 4608, 1920, 31488, 128, 0, 13056, 14720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6400, 1408, 128, 128, 128, 128, 3968, 128, 32640, 21120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 49, 200, 0, 4042, 0, 1254, 4096, 4096, 0, 0, 0, 6, 0, 106, 3688, 31, 0, 0, 3518, 2813, 940, 11, 3982, 0, 0, 9, 0, 14, 0, 0, 4077, 72, 4022, 0, 15, 0, 0, 0, 0, 206, 4082, 99, 0, 4069, 0, 30, 3901, 3878, 1, 0, 4040, 0, 4038, 0, 47, 4080, 1, 0, 21, 0, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 149, 0, 4078, 4096, 13, 0, 4072, 7, 0, 106, 0, 3294, 463, 0, 0, 3874, 4096, 0, 0, 3872, 0, 0, 0, 0, 1, 0, 0, 3875, 0, 4096, 23, 14, 29, 0, 0, 3174, 0, 412, 280, 54, 4057, 0, 0, 3846, 4096, 180, 820, 4089, 1, 4026, 0, 0, 4096, 0, 1, 0, 220, 18, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4054, 75, 10, 3868, 4010, 0, 3433, 0, 10, 0, 1249, 0, 0, 0, 0, 4096, 2238, 311, 0, 4060, 47, 0, 153, 2495, 0, 0, 16, 3668, 0, 4076, 0, 0, 0, 1083, 0, 0, 74, 3606, 210, 0, 3991, 21, 55, 3734, 4096, 0, 0, 2808, 0, 3962, 0, 0, 4022, 1, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 32, 4044, 2, 24, 4096, 3954, 0, 4096, 0, 0, 8, 0, 1, 1407, 0, 0, 2617, 4060, 0, 0, 4096, 0, 0, 0, 1168, 0, 0, 0, 4096, 0, 2643, 0, 0, 1, 0, 0, 385, 0, 4096, 135, 2, 4096, 0, 0, 4096, 4084, 0, 4084, 24, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1, 81, 0, 0, 3473, 163, 180, 3977, 35, 0, 4039, 41, 0, 0, 22, 0, 3548, 0, 1, 509, 3200, 3978, 0, 3748, 0, 11, 0, 24, 0, 71, 24, 4087, 0, 4065, 2, 0, 2, 0, 1, 1145, 0, 4051, 3, 441, 4083, 0, 0, 4096, 4096, 0, 4094, 78, 2, 3962, 0, 20, 4076, 18, 62, 0, 0, 26, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 4095, 1, 3364, 3797, 136, 0, 3227, 518, 0, 0, 0, 0, 3884, 0, 1, 2484, 3546, 0, 0, 4094, 0, 629, 2, 95, 0, 0, 6, 4088, 0, 4096, 0, 18, 339, 248, 1, 0, 0, 2309, 0, 0, 4094, 121, 1649, 9, 4095, 31, 2471, 2505, 0, 2856, 3, 0, 3031, 679, 1667, 0, 1339, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 5, 3, 4096, 0, 3831, 4072, 15, 0, 3762, 0, 2, 0, 0, 2331, 134, 0, 136, 0, 4096, 468, 0, 4096, 0, 1, 0, 5, 0, 24, 0, 4093, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 2820, 0, 0, 4094, 0, 0, 3373, 3990, 0, 4096, 3460, 0, 4051, 0, 0, 4088, 37, 0, 0, 0, 0, 1, 245] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 3506, 0, 1363, 3874, 0, 0, 4087, 0, 0, 0, 0, 163, 0, 0, 0, 2, 4094, 0, 51, 4096, 2, 0, 21, 0, 3, 8, 0, 4095, 0, 4094, 166, 0, 3, 4, 0, 0, 14, 4024, 0, 183, 4087, 0, 2, 4073, 4006, 463, 3882, 3609, 0, 4093, 1, 0, 4083, 22, 0, 0, 5, 3327, 28, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 49, 200, 0, 4042, 0, 1254, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 106, 3688, 1096, 825, 14, 0, 0, 0, 0, 0, 31, 0, 0, 3518, 2813, 940, 11, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 14, 0, 0, 4077, 3432, 0, 0, 0, 0, 0, 0, 0, 72, 4022, 0, 15, 0, 0, 0, 0, 2969, 446, 0, 0, 0, 0, 0, 0, 206, 4082, 99, 0, 1179, 0, 30, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4040, 0, 3213, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 1, 0, 21, 0, 1, 1, 0, 2876, 400, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8545, 9831, 7926, 7532, 7524, 9497, 7301, 7380] avg=8192.0000 max=9831 max_rank=1 min=7301 min_rank=6 imbalance_ratio=1.200073 max_load_reduction=55705 max_load_reduction_pct=0.849991 moved_to_echo=31702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4054, 75, 10, 2833, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 3433, 0, 10, 0, 1249, 0, 1035, 809, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2238, 311, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 153, 2495, 0, 0, 16, 3668, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 1083, 0, 0, 3027, 471, 0, 0, 0, 0, 0, 0, 74, 3606, 210, 0, 1156, 21, 55, 3734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2808, 0, 3153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 1, 0, 0, 0, 0, 0, 0, 2821, 407, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6976, 10560, 7271, 10004, 8657, 8856, 5961, 7251] avg=8192.0000 max=10560 max_rank=1 min=5961 min_rank=6 imbalance_ratio=1.289062 max_load_reduction=54976 max_load_reduction_pct=0.838867 moved_to_echo=30725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 149, 0, 4078, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4072, 7, 0, 106, 0, 3294, 1096, 823, 14, 0, 0, 0, 0, 0, 463, 0, 0, 3874, 4096, 0, 0, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3875, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 23, 14, 29, 0, 0, 3174, 2887, 470, 0, 0, 0, 0, 0, 0, 0, 412, 280, 54, 1176, 0, 0, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 820, 4089, 1, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 220, 18, 0, 0, 2867, 389, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7235, 9425, 9029, 7502, 10693, 5768, 8293, 7591] avg=8192.0000 max=10693 max_rank=4 min=5768 min_rank=5 imbalance_ratio=1.305298 max_load_reduction=54843 max_load_reduction_pct=0.836838 moved_to_echo=32345 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 32, 4044, 2, 24, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 4096, 0, 0, 8, 0, 1, 1096, 836, 14, 0, 0, 0, 0, 0, 1407, 0, 0, 2617, 4060, 0, 0, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 0, 0, 0, 4096, 3616, 0, 0, 0, 0, 0, 0, 0, 0, 2643, 0, 0, 1, 0, 0, 385, 3054, 468, 0, 0, 0, 0, 0, 0, 0, 4096, 135, 2, 1187, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 24, 0, 3257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2895, 411, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7102, 10005, 8715, 8880, 6551, 9516, 7365, 7402] avg=8192.0000 max=10005 max_rank=1 min=6551 min_rank=4 imbalance_ratio=1.221313 max_load_reduction=55531 max_load_reduction_pct=0.847336 moved_to_echo=30834 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 4095, 1, 3364, 2775, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 3227, 518, 0, 0, 0, 0, 1022, 588, 21, 0, 0, 0, 0, 0, 3884, 0, 1, 2484, 3546, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 2, 95, 0, 0, 6, 4088, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 339, 248, 1, 0, 3056, 470, 0, 0, 0, 0, 0, 0, 0, 2309, 0, 0, 1176, 121, 1649, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 2471, 2505, 0, 2268, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3031, 679, 1667, 0, 1339, 0, 1, 0, 2897, 415, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10242, 5512, 10538, 8445, 8228, 5264, 7278, 10029] avg=8192.0000 max=10538 max_rank=2 min=5264 min_rank=5 imbalance_ratio=1.286377 max_load_reduction=54998 max_load_reduction_pct=0.839203 moved_to_echo=30799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 81, 0, 0, 3473, 163, 180, 2913, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4039, 41, 0, 0, 22, 0, 1064, 809, 14, 0, 0, 0, 0, 0, 3548, 0, 1, 509, 3200, 3978, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 24, 0, 71, 24, 4087, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 2, 0, 2, 0, 1, 1145, 2794, 471, 0, 0, 0, 0, 0, 0, 0, 4051, 3, 441, 1183, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 78, 2, 3153, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 18, 62, 0, 0, 26, 0, 0, 2886, 376, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6811, 6024, 11814, 7842, 8480, 9774, 7347, 7444] avg=8192.0000 max=11814 max_rank=2 min=6024 min_rank=1 imbalance_ratio=1.442139 max_load_reduction=53722 max_load_reduction_pct=0.819733 moved_to_echo=33045 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 3506, 0, 1363, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 163, 1037, 836, 14, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 51, 631, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 21, 0, 3, 8, 0, 4095, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 166, 0, 3, 4, 0, 0, 3054, 459, 0, 0, 0, 0, 0, 0, 14, 4024, 0, 183, 1185, 0, 2, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 3882, 3609, 0, 3257, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 22, 0, 0, 5, 3327, 28, 0, 2888, 411, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7708, 6137, 4778, 7676, 7780, 9481, 11212, 10764] avg=8192.0000 max=11212 max_rank=6 min=4778 min_rank=2 imbalance_ratio=1.368652 max_load_reduction=54324 max_load_reduction_pct=0.828918 moved_to_echo=39237 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 3, 4096, 0, 3831, 2982, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3762, 0, 2, 0, 0, 2331, 1090, 828, 14, 0, 0, 0, 0, 0, 134, 0, 136, 0, 4096, 468, 0, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 24, 0, 4093, 3532, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 3054, 458, 0, 0, 0, 0, 0, 0, 0, 2820, 0, 0, 1187, 0, 0, 3373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3460, 0, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 37, 0, 0, 0, 0, 1, 245, 2893, 411, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10917, 8042, 5465, 7655, 7623, 7380, 10779, 7675] avg=8192.0000 max=10917 max_rank=0 min=5465 min_rank=2 imbalance_ratio=1.332642 max_load_reduction=54619 max_load_reduction_pct=0.833420 moved_to_echo=33457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 256, 256, 128, 27520, 256, 14208, 23424, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 768, 19456, 20736, 128, 24832, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[9472, 0, 256, 17152, 28160, 5760, 128, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 25472, 768, 768, 9472, 256, 1792, 27136, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 31232, 256, 128, 384, 1408, 128, 4736, 23936, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31616, 768, 1792, 128, 1664, 3456, 128, 256, 23040, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12288, 0, 26752, 640, 128, 128, 1408, 9600, 8576, 6400, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 768, 256, 3840, 128, 128, 128, 32128, 28672, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3399, 11, 7, 0, 0, 284, 2872, 15, 430, 0, 4096, 12, 3815, 0, 4089, 4046, 1141, 4064, 0, 1965, 0, 5, 112, 3697, 0, 0, 3438, 724, 0, 0, 212, 0, 4096, 2, 1204, 0, 4096, 0, 0, 0, 60, 783, 4, 183, 0, 89, 0, 1340, 0, 0, 4091, 76, 4082, 4023, 2961, 0, 3, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[25, 69, 0, 149, 0, 19, 116, 0, 0, 0, 372, 38, 0, 46, 0, 4096, 0, 4059, 0, 4096, 3949, 0, 4044, 0, 1461, 3, 0, 3783, 1527, 413, 0, 4096, 0, 3506, 0, 0, 113, 140, 0, 2588, 0, 18, 0, 0, 0, 0, 2647, 0, 0, 4024, 18, 0, 4096, 0, 2, 3779, 5, 4096, 4044, 0, 0, 4096, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 35, 0, 205, 0, 4086, 7, 0, 0, 0, 395, 1091, 0, 0, 0, 4096, 0, 4096, 0, 4084, 4093, 1654, 4022, 1, 9, 0, 8, 283, 3985, 0, 2, 4093, 0, 124, 0, 0, 0, 3643, 0, 2593, 0, 3989, 0, 1, 0, 21, 0, 3, 11, 4040, 16, 1846, 216, 0, 0, 4095, 0, 4037, 4075, 315, 0, 265, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 207, 231, 1146, 0, 19, 671, 0, 0, 0, 0, 3650, 2, 0, 50, 4058, 67, 3362, 0, 4042, 3866, 1, 4096, 0, 0, 0, 9, 4072, 3835, 0, 0, 4052, 0, 123, 0, 17, 30, 2802, 8, 3, 0, 3760, 27, 0, 120, 0, 3950, 0, 12, 0, 0, 5, 4089, 0, 0, 4096, 0, 3901, 4096, 0, 0, 1061, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3968, 243, 0, 0, 0, 1028, 2278, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 56, 3674, 0, 0, 3351, 0, 84, 0, 0, 0, 3972, 0, 818, 0, 4090, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 170, 3166, 0, 0, 4063, 0, 4096, 4096, 33, 0, 1715, 60, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[8, 0, 0, 2, 0, 1186, 52, 0, 9, 1, 83, 3728, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4071, 0, 4095, 0, 4055, 0, 0, 1, 3962, 0, 0, 173, 0, 81, 0, 1, 0, 4096, 4096, 2519, 0, 4096, 0, 0, 18, 0, 3909, 0, 0, 270, 0, 0, 535, 0, 0, 336, 0, 4088, 4092, 0, 2, 3676, 4, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 77, 1, 3396, 206, 1, 0, 0, 86, 0, 0, 0, 9, 4096, 0, 4090, 0, 4068, 3882, 490, 4096, 5, 27, 0, 0, 2268, 4096, 0, 0, 3717, 0, 3911, 0, 0, 2, 4096, 24, 1480, 0, 337, 0, 0, 0, 0, 677, 0, 0, 4042, 0, 0, 4068, 0, 0, 3932, 16, 4096, 4078, 3, 0, 149, 12, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 18, 3912, 0, 0, 0, 0, 0, 4095, 694, 1, 0, 4096, 0, 4096, 0, 4092, 3575, 3824, 4096, 0, 6, 0, 0, 4088, 4096, 1082, 0, 290, 22, 4087, 0, 0, 0, 0, 0, 3606, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 4012, 0, 0, 2195, 0, 4002, 3535, 0, 0, 1674, 0, 270] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3399, 11, 7, 4089, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 284, 2872, 15, 430, 0, 4096, 1229, 229, 0, 0, 0, 0, 0, 0, 12, 3221, 0, 0, 4046, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1965, 0, 5, 112, 2385, 0, 0, 3438, 0, 0, 0, 0, 0, 0, 0, 0, 724, 0, 0, 212, 0, 4096, 2, 1204, 1083, 270, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 60, 783, 4, 2893, 699, 0, 0, 0, 0, 0, 0, 183, 0, 89, 0, 1340, 0, 0, 4091, 324, 0, 0, 0, 0, 0, 0, 0, 76, 2154, 4023, 2961, 0, 3, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8677, 9155, 8420, 7905, 7591, 8535, 6027, 9226] avg=8192.0000 max=9226 max_rank=7 min=6027 min_rank=6 imbalance_ratio=1.126221 max_load_reduction=56310 max_load_reduction_pct=0.859222 moved_to_echo=31379 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[25, 69, 0, 149, 0, 19, 116, 0, 4096, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 372, 38, 0, 46, 0, 4096, 1233, 94, 0, 0, 0, 0, 0, 0, 0, 3427, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1461, 3, 0, 3783, 986, 413, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 0, 0, 113, 140, 0, 2588, 447, 287, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 2647, 0, 2879, 702, 0, 0, 0, 0, 0, 0, 0, 4024, 18, 0, 4096, 0, 2, 3779, 345, 0, 0, 0, 0, 0, 0, 0, 5, 2161, 4044, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5639, 5879, 7376, 10742, 7081, 6246, 12264, 10309] avg=8192.0000 max=12264 max_rank=6 min=5639 min_rank=0 imbalance_ratio=1.497070 max_load_reduction=53272 max_load_reduction_pct=0.812866 moved_to_echo=35900 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 35, 0, 205, 0, 4086, 7, 0, 4084, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 395, 1091, 0, 0, 0, 4096, 1216, 246, 0, 0, 0, 0, 0, 0, 0, 3458, 0, 0, 4093, 1654, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8, 283, 2571, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 3643, 0, 2593, 1168, 290, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 1, 0, 21, 0, 3, 2862, 692, 0, 0, 0, 0, 0, 0, 11, 4040, 16, 1846, 216, 0, 0, 4095, 348, 0, 0, 0, 0, 0, 0, 0, 0, 2129, 4075, 315, 0, 265, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9577, 7044, 9206, 6966, 7818, 7568, 10572, 6785] avg=8192.0000 max=10572 max_rank=6 min=6785 min_rank=7 imbalance_ratio=1.290527 max_load_reduction=54964 max_load_reduction_pct=0.838684 moved_to_echo=32743 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3968, 243, 0, 4096, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 2278, 0, 0, 0, 4096, 1233, 227, 0, 0, 0, 0, 0, 0, 0, 3458, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 2370, 0, 0, 3351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 3972, 0, 818, 1077, 290, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4094, 0, 2915, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 3166, 0, 0, 4063, 348, 0, 0, 0, 0, 0, 0, 0, 0, 2161, 4096, 33, 0, 1715, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9489, 8862, 7554, 5777, 6241, 11801, 7747, 8065] avg=8192.0000 max=11801 max_rank=5 min=5777 min_rank=3 imbalance_ratio=1.440552 max_load_reduction=53735 max_load_reduction_pct=0.819931 moved_to_echo=33854 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 77, 1, 3396, 206, 1, 4068, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 9, 4096, 1233, 253, 0, 0, 0, 0, 0, 0, 0, 3453, 0, 0, 3882, 490, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 2268, 2643, 0, 0, 3717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 0, 2, 4096, 24, 1480, 1200, 289, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 0, 0, 677, 0, 2916, 702, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 4068, 0, 0, 3932, 348, 0, 0, 0, 0, 0, 0, 0, 16, 2161, 4078, 3, 0, 149, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8929, 5677, 7830, 8655, 11002, 4632, 12390, 6421] avg=8192.0000 max=12390 max_rank=6 min=4632 min_rank=5 imbalance_ratio=1.512451 max_load_reduction=53146 max_load_reduction_pct=0.810944 moved_to_echo=34445 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 207, 231, 1146, 0, 19, 671, 0, 4042, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3650, 2, 0, 50, 4058, 1175, 237, 0, 0, 0, 0, 0, 0, 67, 2838, 0, 0, 3866, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4072, 2474, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 17, 30, 2802, 8, 3, 1124, 238, 0, 0, 0, 0, 0, 0, 0, 3760, 27, 0, 120, 0, 3950, 0, 2916, 668, 0, 0, 0, 0, 0, 0, 12, 0, 0, 5, 4089, 0, 0, 4096, 286, 0, 0, 0, 0, 0, 0, 0, 0, 2058, 4096, 0, 0, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7496, 9172, 6772, 10607, 4345, 11441, 8488, 7215] avg=8192.0000 max=11441 max_rank=5 min=4345 min_rank=4 imbalance_ratio=1.396606 max_load_reduction=54095 max_load_reduction_pct=0.825424 moved_to_echo=31489 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 2, 0, 1186, 52, 0, 4096, 1181, 0, 0, 0, 0, 0, 0, 9, 1, 83, 3728, 0, 0, 0, 4096, 1231, 245, 0, 0, 0, 0, 0, 0, 0, 3458, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 1, 2556, 0, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 1, 0, 4096, 4096, 2519, 1161, 290, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 3909, 0, 2914, 700, 0, 0, 0, 0, 0, 0, 0, 270, 0, 0, 535, 0, 0, 336, 348, 0, 0, 0, 0, 0, 0, 0, 0, 2157, 4092, 0, 2, 3676, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6525, 9393, 7529, 6785, 12244, 11637, 1489, 9934] avg=8192.0000 max=12244 max_rank=4 min=1489 min_rank=6 imbalance_ratio=1.494629 max_load_reduction=53292 max_load_reduction_pct=0.813171 moved_to_echo=35304 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 18, 3912, 0, 0, 4092, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 694, 1, 0, 4096, 1210, 258, 0, 0, 0, 0, 0, 0, 0, 3450, 0, 0, 3575, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4088, 2633, 1082, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4087, 0, 0, 0, 0, 0, 3606, 1205, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 2915, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 2195, 352, 0, 0, 0, 0, 0, 0, 0, 0, 2102, 3535, 0, 0, 1674, 0, 270, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9204, 10354, 10849, 8099, 9214, 3676, 6559, 7581] avg=8192.0000 max=10849 max_rank=2 min=3676 min_rank=5 imbalance_ratio=1.324341 max_load_reduction=54687 max_load_reduction_pct=0.834457 moved_to_echo=27030 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 12416, 128, 2048, 21632, 0, 128, 26624, 2816, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 17152, 32128, 3328, 128, 12672, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 20480, 128, 128, 256, 128, 16256, 128, 23296, 5632, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 384, 256, 1664, 128, 20096, 1408, 128, 32768, 9472, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7552, 128, 128, 14720, 18688, 1536, 128, 23296, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 2304, 17792, 768, 512, 128, 32768, 9856, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 26880, 0, 0, 31616, 7168, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 12032, 0, 256, 256, 22912, 4224, 14848, 8576, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True + [2026-08-30 04:59:27] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 717.1 | throughput per GPU (TFLOP/s/GPU): 464.9 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.054824E+01 | loss scale: 1.0 | grad norm: 216.524 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[84, 1153, 2068, 36, 2162, 10, 1185, 43, 1880, 36, 1710, 820, 552, 339, 70, 476, 333, 1722, 1864, 1513, 3575, 1328, 596, 1652, 15, 3196, 202, 787, 461, 135, 703, 1381, 2887, 196, 89, 917, 862, 201, 848, 1443, 2160, 2654, 941, 266, 260, 2341, 1588, 670, 342, 630, 835, 182, 203, 654, 3402, 222, 44, 1487, 3525, 1656, 490, 909, 385, 160] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[58, 1280, 2042, 27, 1881, 4, 736, 92, 1914, 60, 1098, 496, 664, 462, 12, 638, 323, 1932, 2413, 1598, 3301, 704, 563, 1628, 0, 3448, 181, 967, 545, 131, 726, 1236, 3331, 260, 104, 742, 1070, 51, 1418, 1113, 2016, 3230, 789, 309, 188, 1835, 1611, 1113, 344, 762, 871, 100, 45, 313, 3210, 367, 31, 1596, 3522, 1976, 573, 722, 670, 94] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[68, 1032, 2356, 54, 1590, 14, 866, 22, 1307, 93, 1603, 571, 422, 500, 15, 502, 212, 2058, 2187, 1829, 3469, 923, 747, 1443, 6, 3334, 150, 1312, 483, 275, 591, 1172, 3455, 103, 246, 646, 511, 189, 1259, 1480, 1842, 3264, 1089, 623, 155, 2603, 1372, 1830, 239, 565, 1022, 158, 158, 295, 2965, 167, 46, 1463, 2817, 1866, 157, 780, 845, 120] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[37, 751, 2325, 132, 1901, 13, 749, 48, 1684, 70, 1050, 734, 553, 284, 32, 493, 216, 2183, 2530, 1321, 3417, 1003, 800, 1720, 5, 3167, 156, 1154, 528, 94, 733, 1223, 3424, 171, 165, 1173, 815, 79, 1264, 1187, 1886, 3184, 682, 319, 173, 2556, 1053, 900, 315, 470, 1139, 145, 169, 630, 3737, 212, 16, 1920, 3475, 1569, 331, 640, 529, 102] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[12, 1378, 2300, 94, 2195, 3, 751, 39, 1424, 69, 991, 455, 481, 277, 19, 348, 250, 2023, 2132, 1251, 3641, 1235, 530, 1447, 8, 3150, 178, 1056, 587, 261, 624, 1307, 3338, 166, 212, 1090, 308, 67, 1370, 1295, 2142, 2977, 1264, 371, 252, 2350, 1572, 2167, 92, 573, 1267, 78, 82, 345, 3022, 429, 13, 1283, 2759, 2520, 221, 630, 719, 46] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[26, 1123, 2671, 129, 1152, 7, 753, 44, 1842, 121, 1404, 529, 612, 354, 42, 464, 271, 2028, 2202, 1363, 3490, 869, 543, 1350, 3, 3010, 170, 1239, 665, 201, 893, 1505, 3226, 218, 75, 1253, 757, 86, 1235, 1813, 2241, 3347, 1002, 411, 339, 2356, 1063, 1257, 274, 624, 609, 91, 82, 386, 3323, 168, 46, 1616, 3544, 1420, 332, 500, 716, 51] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[41, 583, 1885, 66, 1833, 5, 807, 54, 1615, 59, 1180, 727, 538, 406, 24, 562, 221, 2177, 2429, 1815, 3528, 923, 572, 1371, 2, 3272, 165, 753, 442, 79, 685, 1382, 3456, 255, 132, 699, 473, 145, 1268, 1486, 2104, 3310, 971, 196, 224, 2634, 1583, 1365, 292, 708, 933, 147, 133, 467, 3560, 189, 16, 1697, 3475, 1584, 188, 1010, 462, 173] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[29, 766, 2677, 23, 2345, 6, 821, 16, 1211, 94, 722, 281, 839, 189, 7, 706, 175, 2840, 2162, 2084, 3330, 615, 847, 1447, 1, 2740, 144, 909, 700, 43, 562, 1340, 3624, 456, 45, 848, 978, 28, 1553, 1380, 2360, 3044, 797, 176, 103, 2203, 1484, 2073, 102, 479, 1224, 89, 93, 168, 3374, 278, 96, 1515, 3103, 1884, 199, 638, 426, 45] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[58, 1280, 2042, 27, 1881, 4, 736, 92, 1288, 823, 0, 0, 0, 0, 0, 0, 1914, 60, 1098, 496, 664, 462, 12, 638, 3070, 0, 0, 0, 0, 0, 0, 0, 323, 1932, 1205, 1598, 0, 704, 563, 1628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 181, 967, 545, 131, 726, 1236, 456, 385, 188, 0, 0, 0, 0, 0, 3331, 260, 104, 742, 1070, 51, 1418, 1113, 67, 0, 0, 0, 0, 0, 0, 0, 2016, 0, 789, 309, 188, 1379, 1611, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 344, 762, 871, 100, 45, 313, 3210, 367, 1942, 231, 0, 0, 0, 0, 0, 0, 31, 1596, 3267, 1976, 573, 722, 670, 94, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8414, 7953, 8263, 8156, 7405, 8185, 8929] avg=8192.0000 max=8929 max_rank=7 min=7405 min_rank=5 imbalance_ratio=1.089966 max_load_reduction=56607 max_load_reduction_pct=0.863754 moved_to_echo=32675 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[68, 1032, 2356, 54, 1590, 14, 866, 22, 1301, 746, 0, 0, 0, 0, 0, 0, 1307, 93, 1603, 571, 422, 500, 15, 502, 3225, 0, 0, 0, 0, 0, 0, 0, 212, 2058, 1092, 1829, 0, 923, 747, 1443, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3334, 150, 1312, 483, 275, 591, 1172, 647, 349, 150, 0, 0, 0, 0, 0, 3455, 103, 246, 646, 511, 189, 1259, 1480, 53, 0, 0, 0, 0, 0, 0, 0, 1842, 0, 1089, 623, 155, 1956, 1372, 1830, 0, 0, 0, 0, 0, 0, 0, 0, 239, 565, 1022, 158, 158, 295, 2965, 167, 1963, 244, 0, 0, 0, 0, 0, 0, 46, 1463, 2614, 1866, 157, 780, 845, 120, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8049, 8238, 8304, 8469, 7942, 8867, 7776, 7891] avg=8192.0000 max=8867 max_rank=5 min=7776 min_rank=6 imbalance_ratio=1.082397 max_load_reduction=56669 max_load_reduction_pct=0.864700 moved_to_echo=32476 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[84, 1153, 2068, 36, 2162, 10, 1185, 43, 1057, 635, 0, 0, 0, 0, 0, 0, 1880, 36, 1710, 820, 552, 339, 70, 476, 3324, 0, 0, 0, 0, 0, 0, 0, 333, 1722, 932, 1513, 0, 1328, 596, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3196, 202, 787, 461, 135, 703, 1381, 582, 297, 188, 0, 0, 0, 0, 0, 2887, 196, 89, 917, 862, 201, 848, 1443, 67, 0, 0, 0, 0, 0, 0, 0, 2160, 0, 941, 266, 260, 1759, 1588, 670, 0, 0, 0, 0, 0, 0, 0, 0, 342, 630, 835, 182, 203, 654, 3402, 222, 1597, 251, 0, 0, 0, 0, 0, 0, 44, 1487, 3270, 1656, 490, 909, 385, 160, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8433, 9207, 8076, 7947, 7510, 7644, 8318, 8401] avg=8192.0000 max=9207 max_rank=1 min=7510 min_rank=4 imbalance_ratio=1.123901 max_load_reduction=56329 max_load_reduction_pct=0.859512 moved_to_echo=31873 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[37, 751, 2325, 132, 1901, 13, 749, 48, 1270, 863, 0, 0, 0, 0, 0, 0, 1684, 70, 1050, 734, 553, 284, 32, 493, 3177, 0, 0, 0, 0, 0, 0, 0, 216, 2183, 1263, 1321, 0, 1003, 800, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3167, 156, 1154, 528, 94, 733, 1223, 635, 404, 186, 0, 0, 0, 0, 0, 3424, 171, 165, 1173, 815, 79, 1264, 1187, 66, 0, 0, 0, 0, 0, 0, 0, 1886, 0, 682, 319, 173, 1921, 1053, 900, 0, 0, 0, 0, 0, 0, 0, 0, 315, 470, 1139, 145, 169, 630, 3737, 212, 1914, 240, 0, 0, 0, 0, 0, 0, 16, 1920, 3223, 1569, 331, 640, 529, 102, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 8077, 8506, 8285, 8344, 6934, 8971, 8330] avg=8192.0000 max=8971 max_rank=6 min=6934 min_rank=5 imbalance_ratio=1.095093 max_load_reduction=56565 max_load_reduction_pct=0.863113 moved_to_echo=32579 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[26, 1123, 2671, 129, 1152, 7, 753, 44, 1335, 754, 0, 0, 0, 0, 0, 0, 1842, 121, 1404, 529, 612, 354, 42, 464, 3246, 0, 0, 0, 0, 0, 0, 0, 271, 2028, 1092, 1363, 0, 869, 543, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3010, 170, 1239, 665, 201, 893, 1505, 589, 356, 194, 0, 0, 0, 0, 0, 3226, 218, 75, 1253, 757, 86, 1235, 1813, 72, 0, 0, 0, 0, 0, 0, 0, 2241, 0, 1002, 411, 339, 1767, 1063, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 274, 624, 609, 91, 82, 386, 3323, 168, 2012, 244, 0, 0, 0, 0, 0, 0, 46, 1616, 3278, 1420, 332, 500, 716, 51, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7994, 8614, 7516, 8825, 8735, 8080, 7813, 7959] avg=8192.0000 max=8825 max_rank=3 min=7516 min_rank=2 imbalance_ratio=1.077271 max_load_reduction=56711 max_load_reduction_pct=0.865341 moved_to_echo=32587 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[12, 1378, 2300, 94, 2195, 3, 751, 39, 1187, 727, 0, 0, 0, 0, 0, 0, 1424, 69, 991, 455, 481, 277, 19, 348, 3386, 0, 0, 0, 0, 0, 0, 0, 250, 2023, 1065, 1251, 0, 1235, 530, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3150, 178, 1056, 587, 261, 624, 1307, 584, 340, 147, 0, 0, 0, 0, 0, 3338, 166, 212, 1090, 308, 67, 1370, 1295, 52, 0, 0, 0, 0, 0, 0, 0, 2142, 0, 1264, 371, 252, 1766, 1572, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 92, 573, 1267, 78, 82, 345, 3022, 429, 1790, 255, 0, 0, 0, 0, 0, 0, 13, 1283, 2560, 2520, 221, 630, 719, 46, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8686, 7450, 7801, 8242, 7898, 9534, 7933, 7992] avg=8192.0000 max=9534 max_rank=5 min=7450 min_rank=1 imbalance_ratio=1.163818 max_load_reduction=56002 max_load_reduction_pct=0.854523 moved_to_echo=33357 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[29, 766, 2677, 23, 2345, 6, 821, 16, 1213, 737, 0, 0, 0, 0, 0, 0, 1211, 94, 722, 281, 839, 189, 7, 706, 3096, 0, 0, 0, 0, 0, 0, 0, 175, 2840, 1080, 2084, 0, 615, 847, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2740, 144, 909, 700, 43, 562, 1340, 547, 345, 166, 0, 0, 0, 0, 0, 3624, 456, 45, 848, 978, 28, 1553, 1380, 59, 0, 0, 0, 0, 0, 0, 0, 2360, 0, 797, 176, 103, 1656, 1484, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 102, 479, 1224, 89, 93, 168, 3374, 278, 1831, 234, 0, 0, 0, 0, 0, 0, 96, 1515, 2878, 1884, 199, 638, 426, 45, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8633, 7145, 9088, 7497, 8971, 8649, 7872, 7681] avg=8192.0000 max=9088 max_rank=2 min=7145 min_rank=1 imbalance_ratio=1.109375 max_load_reduction=56448 max_load_reduction_pct=0.861328 moved_to_echo=33173 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[41, 583, 1885, 66, 1833, 5, 807, 54, 1319, 828, 0, 0, 0, 0, 0, 0, 1615, 59, 1180, 727, 538, 406, 24, 562, 3280, 0, 0, 0, 0, 0, 0, 0, 221, 2177, 1213, 1815, 0, 923, 572, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3272, 165, 753, 442, 79, 685, 1382, 654, 388, 186, 0, 0, 0, 0, 0, 3456, 255, 132, 699, 473, 145, 1268, 1486, 66, 0, 0, 0, 0, 0, 0, 0, 2104, 0, 971, 196, 224, 1980, 1583, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 292, 708, 933, 147, 133, 467, 3560, 189, 1991, 248, 0, 0, 0, 0, 0, 0, 16, 1697, 3223, 1584, 188, 1010, 462, 173, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7421, 8391, 8292, 8008, 7980, 8423, 8668, 8353] avg=8192.0000 max=8668 max_rank=6 min=7421 min_rank=0 imbalance_ratio=1.058105 max_load_reduction=56868 max_load_reduction_pct=0.867737 moved_to_echo=33424 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 8192, 18432, 640, 15104, 128, 6784, 384, 9984, 6144, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[16768, 0, 7552, 2688, 1792, 14208, 11392, 11392, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2048, 4864, 7936, 1024, 1024, 3328, 26624, 2048, 15104, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2048, 17024, 8960, 12800, 0, 7680, 5248, 12160, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26752, 1920, 1152, 7424, 5888, 896, 10240, 11264, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12928, 640, 9856, 4736, 4736, 2816, 256, 4224, 25856, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 12672, 24320, 14592, 2560, 5888, 4864, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 25344, 1408, 8192, 4480, 1280, 5632, 10624, 4736, 2944, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 0, 12, 6, 3949, 2324, 2944, 2553, 0, 4035, 410, 4090, 2641, 0, 2, 10, 3990, 3264, 4043, 54, 3964, 726, 2761, 0, 5, 0, 0, 3612, 0, 0, 7, 0, 4082, 3, 0, 0, 56, 0, 0, 0, 0, 298, 6, 2973, 49, 0, 1, 3, 4080, 2, 3838, 61, 14, 36, 0, 0, 579, 2903, 1132, 8, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[20, 0, 0, 0, 7, 48, 0, 4070, 2872, 3101, 3713, 28, 4075, 2841, 4082, 3444, 6, 1, 7, 3846, 3128, 4090, 0, 3969, 3844, 647, 0, 0, 5, 0, 3681, 28, 8, 248, 1, 2956, 0, 0, 0, 0, 318, 0, 67, 26, 2, 0, 508, 0, 0, 66, 22, 3969, 0, 4019, 16, 0, 75, 0, 0, 330, 255, 1094, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 4, 0, 0, 2, 2, 4093, 3383, 3327, 4032, 0, 4096, 425, 3770, 3788, 0, 0, 0, 3765, 3570, 4092, 0, 4012, 38, 1613, 0, 0, 0, 0, 4075, 0, 0, 2, 6, 4086, 1, 0, 0, 0, 0, 109, 2, 145, 397, 0, 2890, 13, 0, 0, 0, 3823, 1, 3543, 4, 0, 0, 0, 3, 48, 943, 1431, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 6, 0, 0, 4096, 2788, 2326, 3636, 0, 4083, 73, 4096, 1591, 4, 3, 0, 2637, 2873, 4092, 39, 4051, 129, 3281, 7, 5, 0, 0, 3933, 0, 43, 5, 0, 4055, 0, 0, 0, 0, 640, 0, 6, 0, 10, 0, 2461, 25, 0, 0, 53, 4074, 0, 3965, 0, 0, 11, 4, 5, 534, 3903, 1992, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1, 1, 12, 0, 0, 8, 0, 4063, 2320, 3528, 4066, 1, 4050, 3200, 4094, 1472, 0, 0, 0, 4095, 279, 4017, 0, 4085, 261, 1344, 0, 0, 94, 0, 4082, 0, 116, 0, 2, 3913, 0, 0, 0, 0, 79, 0, 0, 0, 144, 0, 3374, 4, 0, 150, 0, 4096, 0, 4064, 6, 0, 0, 0, 6, 0, 442, 4067, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 6, 1, 0, 4092, 1431, 3373, 3424, 0, 3924, 410, 4093, 3080, 0, 0, 0, 4096, 3778, 4093, 0, 4080, 441, 3528, 1, 4, 0, 0, 4082, 0, 0, 1, 0, 2963, 2, 0, 0, 0, 7, 0, 1, 0, 1, 0, 880, 6, 0, 0, 2, 4042, 0, 4088, 6, 0, 0, 2, 0, 95, 1920, 3582, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[6, 0, 6, 8, 0, 1, 0, 4049, 2015, 843, 3626, 0, 4075, 155, 4091, 3496, 0, 1, 0, 4083, 791, 4096, 3, 4086, 485, 3990, 0, 0, 0, 0, 4095, 1, 0, 0, 3, 4088, 9, 0, 0, 14, 0, 3, 3, 6, 487, 0, 900, 2, 0, 0, 0, 4082, 1, 4086, 1, 0, 1, 1, 1, 684, 3786, 3372, 1, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4, 8, 6, 1, 7, 5, 16, 4076, 3466, 3159, 4080, 3, 3817, 1536, 4042, 1966, 4, 0, 46, 3972, 3810, 4087, 15, 3604, 460, 2187, 0, 10, 0, 14, 4079, 0, 0, 0, 8, 4089, 8, 3, 2, 1, 8, 5, 0, 3, 1028, 1, 61, 11, 7, 22, 0, 4045, 2, 3676, 0, 0, 0, 0, 0, 29, 572, 3467, 2, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 0, 12, 6, 3949, 1581, 1856, 0, 0, 0, 0, 0, 0, 2324, 1873, 0, 0, 0, 410, 0, 2641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 10, 3990, 3264, 0, 54, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 726, 2761, 0, 5, 0, 0, 3612, 0, 862, 170, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4082, 3, 0, 0, 56, 2198, 2179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 6, 2973, 49, 4043, 1892, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4080, 2, 3838, 61, 14, 209, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 579, 2903, 1132, 8, 2, 2130, 972, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7412, 7248, 8984, 8136, 8525, 9261, 8208, 7762] avg=8192.0000 max=9261 max_rank=5 min=7248 min_rank=1 imbalance_ratio=1.130493 max_load_reduction=56275 max_load_reduction_pct=0.858688 moved_to_echo=33756 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 0, 0, 7, 48, 0, 4070, 2300, 1875, 0, 0, 0, 0, 0, 0, 2872, 1973, 0, 28, 0, 2841, 0, 3444, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 7, 3846, 3128, 0, 0, 1667, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 647, 0, 0, 5, 0, 3681, 28, 908, 170, 0, 0, 0, 0, 0, 0, 8, 248, 1, 2956, 0, 0, 0, 0, 2194, 2200, 0, 0, 0, 0, 0, 0, 318, 0, 67, 26, 2, 0, 508, 0, 4090, 1888, 0, 0, 0, 0, 0, 0, 0, 66, 22, 3969, 0, 4019, 16, 0, 220, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 330, 255, 1094, 3, 0, 2132, 1413, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 11158, 8655, 9283, 7607, 6899, 8312, 5302] avg=8192.0000 max=11158 max_rank=1 min=5302 min_rank=7 imbalance_ratio=1.362061 max_load_reduction=54378 max_load_reduction_pct=0.829742 moved_to_echo=28120 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 6, 0, 0, 4096, 2251, 1878, 0, 0, 0, 0, 0, 0, 2788, 1480, 0, 0, 0, 73, 0, 1591, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 2637, 2873, 0, 39, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 129, 3281, 7, 5, 0, 0, 3933, 0, 681, 174, 0, 0, 0, 0, 0, 0, 43, 5, 0, 4055, 0, 0, 0, 0, 2201, 2205, 0, 0, 0, 0, 0, 0, 640, 0, 6, 0, 10, 0, 2461, 25, 4092, 1895, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4074, 0, 3965, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 11, 4, 5, 534, 3903, 1992, 0, 0, 2177, 1385, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 5932, 7256, 8210, 8509, 9129, 8257, 10011] avg=8192.0000 max=10011 max_rank=7 min=5932 min_rank=1 imbalance_ratio=1.222046 max_load_reduction=55525 max_load_reduction_pct=0.847244 moved_to_echo=35906 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4, 0, 0, 2, 2, 4093, 2498, 1884, 0, 0, 0, 0, 0, 0, 3383, 2117, 0, 0, 0, 425, 0, 3788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3765, 3570, 0, 0, 1684, 0, 0, 0, 0, 0, 0, 0, 0, 38, 1613, 0, 0, 0, 0, 4075, 0, 974, 172, 0, 0, 0, 0, 0, 0, 0, 2, 6, 4086, 1, 0, 0, 0, 2027, 2212, 0, 0, 0, 0, 0, 0, 0, 109, 2, 145, 397, 0, 2890, 13, 4092, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 1, 3543, 4, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 48, 943, 1431, 0, 0, 2156, 1534, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8485, 9713, 9019, 6872, 8334, 9391, 7607, 6115] avg=8192.0000 max=9713 max_rank=1 min=6115 min_rank=7 imbalance_ratio=1.185669 max_load_reduction=55823 max_load_reduction_pct=0.851791 moved_to_echo=31447 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 6, 8, 0, 1, 0, 4049, 2245, 1874, 0, 0, 0, 0, 0, 0, 2015, 538, 0, 0, 0, 155, 0, 3496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4083, 791, 0, 3, 1716, 0, 0, 0, 0, 0, 0, 0, 0, 485, 3990, 0, 0, 0, 0, 4095, 1, 246, 175, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4088, 9, 0, 0, 14, 2198, 2201, 0, 0, 0, 0, 0, 0, 0, 3, 3, 6, 487, 0, 900, 2, 4096, 1893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 1, 4086, 1, 0, 59, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 684, 3786, 3372, 1, 3, 2195, 1381, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 6204, 6594, 8992, 8513, 7390, 8229, 11425] avg=8192.0000 max=11425 max_rank=7 min=6204 min_rank=1 imbalance_ratio=1.394653 max_load_reduction=54111 max_load_reduction_pct=0.825668 moved_to_echo=35557 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 12, 0, 0, 8, 0, 4063, 2519, 1863, 0, 0, 0, 0, 0, 0, 2320, 2245, 0, 1, 0, 3200, 0, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 279, 0, 0, 1715, 0, 0, 0, 0, 0, 0, 0, 0, 261, 1344, 0, 0, 94, 0, 4082, 0, 1033, 175, 0, 0, 0, 0, 0, 0, 116, 0, 2, 3913, 0, 0, 0, 0, 2201, 2187, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 144, 0, 3374, 4, 4017, 1893, 0, 0, 0, 0, 0, 0, 0, 150, 0, 4096, 0, 4064, 6, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 442, 4067, 0, 0, 2195, 1547, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8467, 9238, 6089, 6989, 8419, 9511, 8566, 8257] avg=8192.0000 max=9511 max_rank=5 min=6089 min_rank=2 imbalance_ratio=1.161011 max_load_reduction=56025 max_load_reduction_pct=0.854874 moved_to_echo=34753 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 6, 1, 0, 4092, 2120, 1805, 0, 0, 0, 0, 0, 0, 1431, 2147, 0, 0, 0, 410, 0, 3080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3778, 0, 0, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 441, 3528, 1, 4, 0, 0, 4082, 0, 987, 175, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2963, 2, 0, 0, 0, 2199, 2119, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 1, 0, 880, 6, 4093, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4042, 0, 4088, 6, 0, 239, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 95, 1920, 3582, 0, 0, 2192, 1304, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8025, 7068, 9587, 9218, 7284, 6882, 8377, 9095] avg=8192.0000 max=9587 max_rank=2 min=6882 min_rank=5 imbalance_ratio=1.170288 max_load_reduction=55949 max_load_reduction_pct=0.853714 moved_to_echo=31638 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 8, 6, 1, 7, 5, 16, 4076, 2527, 1756, 0, 0, 0, 0, 0, 0, 3466, 2004, 0, 3, 0, 1536, 0, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 46, 3972, 3810, 0, 15, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 460, 2187, 0, 10, 0, 14, 4079, 0, 927, 159, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4089, 8, 3, 2, 1, 2173, 2061, 0, 0, 0, 0, 0, 0, 8, 5, 0, 3, 1028, 1, 61, 11, 4087, 1869, 0, 0, 0, 0, 0, 0, 7, 22, 0, 4045, 2, 3676, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 572, 3467, 2, 6, 1940, 1553, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8406, 8975, 9352, 7836, 8345, 7073, 7980, 7569] avg=8192.0000 max=9352 max_rank=2 min=7073 min_rank=5 imbalance_ratio=1.141602 max_load_reduction=56184 max_load_reduction_pct=0.857300 moved_to_echo=30967 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 2304, 14848, 20224, 128, 128, 17152, 11136, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[6400, 19456, 128, 128, 128, 128, 31744, 128, 6656, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 30592, 21504, 0, 128, 13440, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 32512, 18048, 14848, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 256, 128, 32256, 128, 31360, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 384, 128, 30336, 128, 128, 128, 128, 17408, 17408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[20608, 14464, 0, 128, 0, 9088, 0, 21504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1152, 128, 128, 256, 2432, 128, 14080, 128, 32640, 14976, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 7, 5, 0, 29, 2, 1, 35, 0, 4096, 7, 85, 465, 4072, 4096, 0, 1, 4092, 23, 1, 4051, 0, 0, 0, 4045, 0, 305, 0, 0, 4009, 63, 11, 4050, 37, 0, 0, 1344, 518, 0, 7, 4082, 3321, 160, 18, 0, 0, 14, 15, 444, 2794, 4079, 0, 0, 3603, 4, 18, 4090, 8, 0, 21, 3267, 0, 45, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 30, 1, 0, 0, 20, 0, 0, 4095, 2, 1, 305, 4092, 4096, 82, 30, 4091, 0, 1, 4020, 3, 2, 0, 4087, 3, 0, 0, 0, 4000, 12, 1, 4089, 0, 34, 0, 2753, 246, 0, 2, 4070, 3462, 0, 1, 6, 0, 0, 0, 9, 2552, 4095, 0, 0, 899, 0, 2, 4092, 14, 2043, 2, 4095, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 0, 4, 0, 0, 2, 0, 1, 4088, 0, 59, 1220, 4095, 4094, 7, 154, 4063, 0, 0, 4092, 0, 0, 2, 4093, 0, 1291, 0, 1, 4082, 1, 0, 4074, 0, 5, 0, 3891, 157, 10, 13, 4077, 3494, 1, 1, 2, 2, 0, 0, 0, 1760, 4096, 0, 0, 1084, 0, 0, 4095, 0, 91, 0, 3232, 3, 1, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 1, 4, 0, 180, 4090, 0, 0, 1766, 4064, 4064, 4, 125, 4093, 0, 0, 3905, 7, 0, 0, 4096, 0, 564, 0, 0, 4073, 1, 0, 4096, 0, 2, 0, 808, 380, 0, 8, 3088, 3170, 9, 48, 0, 1, 0, 0, 33, 4001, 4096, 0, 0, 4060, 0, 11, 4096, 1, 258, 0, 2342, 0, 0, 3989] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 13, 0, 0, 10, 0, 2, 4017, 0, 5, 2163, 4091, 4092, 0, 6, 4096, 0, 0, 4094, 0, 1, 0, 4087, 5, 18, 0, 0, 4095, 2, 0, 4080, 0, 0, 0, 1834, 885, 4, 0, 3629, 4076, 0, 0, 0, 8, 1, 0, 142, 1881, 4096, 0, 0, 4017, 0, 0, 4096, 0, 114, 0, 1775, 7, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 76, 19, 3, 1, 1, 3776, 0, 0, 3726, 3865, 4096, 0, 0, 4095, 0, 0, 4067, 1, 0, 0, 4096, 0, 458, 0, 0, 4094, 6, 1, 4095, 4, 0, 0, 44, 3836, 10, 0, 3971, 3634, 33, 0, 0, 0, 0, 0, 217, 3684, 4096, 0, 0, 709, 0, 0, 4096, 0, 77, 0, 729, 0, 43, 3871] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[2, 8, 37, 0, 63, 1, 3, 0, 0, 4064, 0, 0, 23, 4039, 4087, 0, 28, 4091, 0, 1, 4096, 3, 0, 0, 4060, 9, 6, 0, 1, 4095, 0, 0, 4096, 0, 2, 9, 2461, 523, 0, 0, 4052, 3354, 0, 15, 1, 0, 0, 0, 1, 3552, 4095, 0, 0, 3725, 0, 3, 4089, 8, 1530, 1, 1274, 0, 1, 4027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 24, 0, 7, 10, 0, 0, 63, 4091, 0, 0, 1451, 4082, 4085, 13, 1, 4068, 0, 1, 4091, 1, 25, 4, 4079, 0, 457, 0, 0, 4077, 13, 0, 4086, 0, 1, 0, 3795, 697, 0, 10, 903, 1926, 23, 0, 15, 27, 0, 0, 610, 4074, 4091, 2, 0, 3219, 3, 0, 4093, 7, 246, 0, 3135, 0, 7, 3923] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 5, 0, 29, 2, 1, 35, 4096, 2969, 1073, 0, 0, 0, 0, 0, 0, 4096, 7, 85, 465, 2715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 23, 1, 4051, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3691, 0, 305, 0, 0, 4009, 63, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 37, 0, 0, 1344, 518, 0, 7, 714, 353, 0, 0, 0, 0, 0, 0, 4082, 3321, 160, 18, 0, 0, 14, 15, 710, 643, 0, 0, 0, 0, 0, 0, 444, 2794, 2296, 0, 0, 3603, 4, 18, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 8, 0, 21, 3267, 0, 45, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 7368, 8169, 8079, 7023, 8963, 9159, 8558] avg=8192.0000 max=9159 max_rank=6 min=7023 min_rank=4 imbalance_ratio=1.118042 max_load_reduction=56377 max_load_reduction_pct=0.860245 moved_to_echo=33703 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 1, 4, 0, 4064, 2974, 1078, 0, 0, 0, 0, 0, 180, 4090, 0, 0, 1766, 2709, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 125, 4093, 0, 0, 3905, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3738, 0, 564, 0, 0, 4073, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 808, 380, 0, 8, 713, 357, 0, 0, 0, 0, 0, 0, 3088, 3170, 9, 48, 0, 1, 0, 0, 713, 642, 0, 0, 0, 0, 0, 0, 33, 4001, 2305, 0, 0, 4060, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 1, 258, 0, 2342, 0, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8749, 8131, 8376, 6364, 7671, 10410, 7712] avg=8192.0000 max=10410 max_rank=6 min=6364 min_rank=4 imbalance_ratio=1.270752 max_load_reduction=55126 max_load_reduction_pct=0.841156 moved_to_echo=32157 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 30, 1, 0, 0, 20, 0, 4096, 2971, 1077, 0, 0, 0, 0, 0, 0, 4095, 2, 1, 305, 2727, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4091, 0, 1, 4020, 3, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3730, 3, 0, 0, 0, 4000, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 34, 0, 2753, 246, 0, 2, 718, 356, 0, 0, 0, 0, 0, 0, 4070, 3462, 0, 1, 6, 0, 0, 0, 713, 647, 0, 0, 0, 0, 0, 0, 9, 2552, 2305, 0, 0, 899, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 14, 2043, 2, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 7212, 8148, 7746, 8198, 8899, 5767, 11371] avg=8192.0000 max=11371 max_rank=7 min=5767 min_rank=6 imbalance_ratio=1.388062 max_load_reduction=54165 max_load_reduction_pct=0.826492 moved_to_echo=34235 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 4, 0, 0, 2, 0, 4094, 2973, 1078, 0, 0, 0, 0, 0, 1, 4088, 0, 59, 1220, 2729, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 154, 4063, 0, 0, 4092, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 3735, 0, 1291, 0, 1, 4082, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 5, 0, 3891, 157, 10, 13, 719, 357, 0, 0, 0, 0, 0, 0, 4077, 3494, 1, 1, 2, 2, 0, 0, 713, 647, 0, 0, 0, 0, 0, 0, 0, 1760, 2305, 0, 0, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 91, 0, 3232, 3, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8104, 8312, 9110, 9226, 8937, 5149, 8544] avg=8192.0000 max=9226 max_rank=4 min=5149 min_rank=6 imbalance_ratio=1.126221 max_load_reduction=56310 max_load_reduction_pct=0.859222 moved_to_echo=31856 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 76, 19, 3, 1, 4096, 2974, 1078, 0, 0, 0, 0, 0, 1, 3776, 0, 0, 3726, 2576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4067, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3738, 0, 458, 0, 0, 4094, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 44, 3836, 10, 0, 678, 357, 0, 0, 0, 0, 0, 0, 3971, 3634, 33, 0, 0, 0, 0, 0, 713, 611, 0, 0, 0, 0, 0, 0, 217, 3684, 2305, 0, 0, 709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 77, 0, 729, 0, 43, 3871, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 10079, 8164, 8297, 9024, 8962, 6915, 5842] avg=8192.0000 max=10079 max_rank=1 min=5842 min_rank=7 imbalance_ratio=1.230347 max_load_reduction=55457 max_load_reduction_pct=0.846207 moved_to_echo=30743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 8, 37, 0, 63, 1, 3, 0, 4087, 2972, 1080, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 23, 2684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4091, 0, 1, 4096, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3698, 9, 6, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 9, 2461, 523, 0, 0, 713, 359, 0, 0, 0, 0, 0, 0, 4052, 3354, 0, 15, 1, 0, 0, 0, 716, 642, 0, 0, 0, 0, 0, 0, 1, 3552, 2299, 0, 0, 3725, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 8, 1530, 1, 1274, 0, 1, 4027, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 6771, 8222, 7809, 8163, 8780, 9580, 7958] avg=8192.0000 max=9580 max_rank=6 min=6771 min_rank=1 imbalance_ratio=1.169434 max_load_reduction=55956 max_load_reduction_pct=0.853821 moved_to_echo=34481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 13, 0, 0, 10, 0, 4092, 2974, 1078, 0, 0, 0, 0, 0, 2, 4017, 0, 5, 2163, 2727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 4094, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3730, 5, 18, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 1834, 885, 4, 0, 718, 356, 0, 0, 0, 0, 0, 0, 3629, 4076, 0, 0, 0, 8, 1, 0, 713, 646, 0, 0, 0, 0, 0, 0, 142, 1881, 2305, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 114, 0, 1775, 7, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8914, 8198, 7850, 7877, 9073, 8345, 7111] avg=8192.0000 max=9073 max_rank=5 min=7111 min_rank=7 imbalance_ratio=1.107544 max_load_reduction=56463 max_load_reduction_pct=0.861557 moved_to_echo=32406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 24, 0, 7, 10, 0, 0, 4085, 2971, 1076, 0, 0, 0, 0, 0, 63, 4091, 0, 0, 1451, 2721, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4068, 0, 1, 4091, 1, 25, 4, 1, 0, 0, 0, 0, 0, 0, 0, 3722, 0, 457, 0, 0, 4077, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 1, 0, 3795, 697, 0, 10, 716, 356, 0, 0, 0, 0, 0, 0, 903, 1926, 23, 0, 15, 27, 0, 0, 712, 645, 0, 0, 0, 0, 0, 0, 610, 4074, 2303, 2, 0, 3219, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 7, 246, 0, 3135, 0, 7, 3923, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8339, 8192, 8269, 9661, 4251, 10211, 8440] avg=8192.0000 max=10211 max_rank=6 min=4251 min_rank=5 imbalance_ratio=1.246460 max_load_reduction=55325 max_load_reduction_pct=0.844193 moved_to_echo=32563 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1536, 24320, 18432, 128, 0, 21376, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[27904, 26496, 256, 128, 128, 128, 128, 128, 5760, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 17024, 7296, 128, 128, 5760, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 256, 128, 128, 128, 32768, 23808, 8704, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[384, 32768, 128, 128, 32512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[9088, 128, 4480, 128, 19968, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 32384, 128, 256, 11136, 21632, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[29824, 128, 3200, 0, 128, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4041, 4081, 0, 0, 0, 4081, 1, 9, 2, 4077, 1, 337, 7, 0, 4027, 16, 0, 3711, 0, 0, 4006, 4088, 0, 15, 4077, 1, 10, 3, 0, 0, 4086, 6, 1, 3721, 1579, 0, 44, 0, 0, 0, 0, 11, 3606, 10, 4048, 0, 1, 1, 3525, 44, 13, 0, 0, 0, 69, 18, 0, 7, 4057, 0, 2, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4078, 4010, 0, 2, 0, 4096, 0, 19, 0, 4095, 0, 13, 3, 0, 4018, 2, 36, 3709, 0, 235, 3974, 4084, 1249, 3, 4095, 3, 4, 8, 6, 0, 4091, 0, 48, 3572, 2780, 14, 1, 7, 0, 0, 11, 0, 3806, 50, 4096, 309, 0, 0, 630, 3, 0, 5, 0, 157, 0, 0, 20, 0, 4076, 0, 0, 4087, 31, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4093, 3956, 1, 27, 2, 4095, 5, 9, 0, 4084, 0, 1, 0, 23, 4064, 0, 0, 2531, 13, 3849, 4042, 4071, 25, 0, 4096, 15, 0, 0, 0, 1, 4087, 0, 54, 4084, 1834, 12, 0, 9, 0, 0, 7, 2, 4096, 0, 3939, 149, 6, 0, 29, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4092, 0, 4, 4093, 29, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4095, 3266, 2, 0, 0, 4096, 1, 0, 0, 4091, 2, 33, 0, 1, 4091, 0, 0, 3890, 0, 1430, 4094, 4096, 503, 9, 4094, 0, 2, 0, 2, 2, 4093, 0, 0, 4094, 3849, 0, 4, 1, 0, 0, 0, 1, 2787, 0, 4067, 109, 0, 0, 529, 9, 2, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 4093, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4081, 4096, 0, 0, 0, 4096, 0, 43, 0, 4095, 0, 0, 0, 7, 4094, 0, 0, 3089, 1, 4, 4096, 4052, 3, 50, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 4020, 3032, 0, 0, 0, 0, 0, 0, 0, 3702, 4, 4088, 15, 0, 0, 2291, 0, 0, 0, 0, 182, 1, 0, 0, 0, 4094, 10, 0, 4095, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4093, 4058, 0, 22, 0, 4096, 2, 4, 9, 4095, 0, 0, 1, 2, 4096, 0, 0, 4023, 2, 10, 4089, 4090, 2, 0, 4094, 3, 1, 0, 0, 7, 4092, 0, 0, 4096, 4069, 7, 0, 0, 0, 1, 0, 0, 4067, 4, 4079, 25, 1, 0, 62, 0, 0, 0, 5, 3, 0, 0, 0, 0, 4088, 0, 0, 4095, 0, 43] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4083, 3802, 0, 0, 0, 4095, 1, 2, 13, 4070, 0, 0, 4, 0, 4089, 3, 0, 3859, 5, 1, 4093, 4084, 557, 26, 4084, 0, 0, 0, 0, 0, 4088, 0, 0, 4076, 4065, 4, 0, 0, 3, 0, 2, 6, 4087, 0, 4088, 2, 0, 1, 17, 8, 10, 0, 0, 0, 4, 5, 0, 0, 4086, 1, 7, 4094, 3, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4079, 3560, 0, 2, 0, 4069, 14, 0, 20, 4096, 0, 6, 0, 0, 4093, 1, 0, 4082, 0, 188, 4095, 4092, 54, 20, 4096, 28, 0, 0, 0, 7, 4084, 16, 0, 4071, 2515, 0, 224, 0, 9, 0, 0, 0, 4079, 0, 4080, 2, 0, 4, 1522, 0, 0, 0, 0, 0, 104, 0, 0, 0, 4087, 14, 7, 4093, 8, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4041, 4081, 0, 0, 0, 237, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4047, 1, 337, 7, 0, 4027, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 0, 0, 3497, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 10, 3, 0, 0, 4086, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3721, 1579, 0, 44, 0, 0, 0, 898, 299, 0, 0, 0, 0, 0, 0, 0, 11, 3606, 10, 4048, 0, 1, 1, 210, 30, 12, 4, 0, 0, 0, 0, 3525, 44, 13, 0, 0, 0, 69, 18, 4088, 2946, 0, 0, 0, 0, 0, 0, 0, 7, 4057, 0, 2, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8369, 8437, 7223, 8179, 6542, 7933, 10703, 8150] avg=8192.0000 max=10703 max_rank=6 min=6542 min_rank=4 imbalance_ratio=1.306519 max_load_reduction=54833 max_load_reduction_pct=0.836685 moved_to_echo=33328 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4078, 4010, 0, 2, 0, 238, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 13, 3, 0, 4018, 2, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3709, 0, 235, 3469, 0, 1249, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 4, 8, 6, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 3572, 2780, 14, 1, 7, 0, 0, 901, 297, 0, 0, 0, 0, 0, 0, 11, 0, 3806, 50, 4096, 309, 0, 0, 208, 30, 12, 4, 0, 0, 0, 0, 630, 3, 0, 5, 0, 157, 0, 0, 4084, 2957, 0, 0, 0, 0, 0, 0, 20, 0, 4076, 0, 0, 4075, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8347, 8101, 8701, 8203, 7620, 8526, 7836, 8202] avg=8192.0000 max=8701 max_rank=2 min=7620 min_rank=4 imbalance_ratio=1.062134 max_load_reduction=56835 max_load_reduction_pct=0.867233 moved_to_echo=32184 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4093, 3956, 1, 27, 2, 238, 5, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 1, 0, 23, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2531, 13, 3849, 3528, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 15, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 4084, 1834, 12, 0, 9, 0, 0, 901, 302, 0, 0, 0, 0, 0, 0, 7, 2, 4096, 0, 3939, 149, 6, 0, 212, 30, 12, 4, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 7, 0, 4071, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4, 4081, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8331, 8142, 9946, 8195, 7196, 8457, 7063, 8206] avg=8192.0000 max=9946 max_rank=2 min=7063 min_rank=6 imbalance_ratio=1.214111 max_load_reduction=55590 max_load_reduction_pct=0.848236 moved_to_echo=30922 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4095, 3266, 2, 0, 0, 238, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 2, 33, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 0, 1430, 3574, 0, 503, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2, 0, 2, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3849, 0, 4, 1, 0, 0, 901, 306, 0, 0, 0, 0, 0, 0, 0, 1, 2787, 0, 4067, 109, 0, 0, 214, 30, 12, 4, 0, 0, 0, 0, 529, 9, 2, 0, 0, 0, 0, 0, 4096, 2957, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 4081, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7602, 8188, 9406, 8189, 9155, 7224, 7593, 8179] avg=8192.0000 max=9406 max_rank=2 min=7224 min_rank=5 imbalance_ratio=1.148193 max_load_reduction=56130 max_load_reduction_pct=0.856476 moved_to_echo=32151 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4081, 4096, 0, 0, 0, 238, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 7, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3089, 1, 4, 3576, 0, 3, 50, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4020, 3032, 0, 0, 0, 0, 0, 901, 306, 0, 0, 0, 0, 0, 0, 0, 0, 3702, 4, 4088, 15, 0, 0, 214, 30, 12, 4, 0, 0, 0, 0, 2291, 0, 0, 0, 0, 182, 1, 0, 4052, 2957, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 10, 0, 4083, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8458, 8166, 6723, 8188, 8260, 8069, 9483, 8189] avg=8192.0000 max=9483 max_rank=6 min=6723 min_rank=2 imbalance_ratio=1.157593 max_load_reduction=56053 max_load_reduction_pct=0.855301 moved_to_echo=34001 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4083, 3802, 0, 0, 0, 238, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4040, 0, 0, 4, 0, 4089, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3859, 5, 1, 3573, 0, 557, 26, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4065, 4, 0, 0, 3, 0, 901, 306, 0, 0, 0, 0, 0, 0, 2, 6, 4087, 0, 4088, 2, 0, 1, 214, 30, 12, 4, 0, 0, 0, 0, 17, 8, 10, 0, 0, 0, 4, 5, 4084, 2956, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 7, 4082, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8126, 8149, 8021, 8168, 9355, 8446, 7084, 8187] avg=8192.0000 max=9355 max_rank=4 min=7084 min_rank=6 imbalance_ratio=1.141968 max_load_reduction=56181 max_load_reduction_pct=0.857254 moved_to_echo=33072 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4093, 4058, 0, 22, 0, 238, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4065, 0, 0, 1, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 2, 10, 3569, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 1, 0, 0, 7, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 7, 0, 0, 0, 1, 901, 306, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4, 4079, 25, 1, 0, 214, 30, 12, 4, 0, 0, 0, 0, 62, 0, 0, 0, 5, 3, 0, 0, 4090, 2957, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4083, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8417, 8173, 7606, 8193, 9380, 8436, 7117, 8214] avg=8192.0000 max=9380 max_rank=4 min=7117 min_rank=6 imbalance_ratio=1.145020 max_load_reduction=56156 max_load_reduction_pct=0.856873 moved_to_echo=33147 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4079, 3560, 0, 2, 0, 231, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4060, 0, 6, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 188, 3566, 0, 54, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 28, 0, 0, 0, 7, 4084, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 2515, 0, 224, 0, 9, 0, 899, 310, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4080, 2, 0, 4, 219, 36, 15, 10, 0, 0, 0, 0, 1522, 0, 0, 0, 0, 0, 104, 0, 4092, 2939, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 14, 7, 4078, 8, 15, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7886, 8180, 7910, 8221, 8028, 8445, 8657, 8209] avg=8192.0000 max=8657 max_rank=6 min=7886 min_rank=0 imbalance_ratio=1.056763 max_load_reduction=56879 max_load_reduction_pct=0.867905 moved_to_echo=33339 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 30848, 128, 128, 128, 1920, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 30336, 128, 32512, 640, 128, 128, 1792, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32512, 128, 512, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 28928, 128, 5760, 28416, 0, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 31744, 23808, 128, 384, 128, 128, 128, 7296, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[8704, 128, 128, 128, 128, 384, 256, 128, 32768, 23680, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[18, 0, 3794, 1430, 9, 20, 0, 0, 8, 0, 104, 1, 4079, 0, 0, 4092, 4078, 4079, 0, 0, 23, 116, 4096, 0, 0, 0, 0, 0, 0, 0, 3878, 1, 419, 4075, 3986, 4080, 0, 6, 541, 4083, 0, 4, 3153, 0, 0, 0, 1, 13, 0, 171, 0, 52, 0, 10, 2, 21, 36, 4082, 12, 4089, 2938, 0, 3912, 24] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[3, 85, 3846, 3027, 0, 0, 0, 14, 1, 0, 0, 0, 4088, 0, 0, 4094, 4078, 4096, 0, 0, 1, 65, 2979, 5, 0, 0, 730, 0, 0, 11, 4085, 0, 0, 3986, 4078, 4086, 4, 0, 31, 4094, 0, 0, 3632, 3, 7, 0, 2, 0, 0, 6, 0, 5, 503, 0, 0, 0, 0, 4030, 1, 1680, 4096, 0, 4084, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3596, 1560, 0, 2, 0, 0, 0, 1, 0, 47, 4093, 0, 0, 4056, 4068, 4075, 3, 0, 13, 84, 3600, 0, 0, 0, 0, 0, 43, 0, 4095, 0, 0, 4092, 4095, 4084, 2, 0, 0, 4096, 28, 0, 3886, 0, 223, 0, 0, 4, 0, 34, 0, 1, 0, 0, 0, 2, 0, 4094, 0, 3390, 4079, 0, 4090, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[1, 46, 2858, 2165, 0, 0, 6, 0, 1, 15, 13, 0, 4090, 0, 0, 4090, 4096, 4095, 1, 0, 0, 126, 4094, 0, 0, 0, 4, 0, 17, 0, 4095, 0, 0, 4078, 4090, 4076, 5, 0, 1, 4058, 0, 0, 4093, 3, 0, 1, 0, 0, 2, 38, 4, 26, 18, 0, 0, 1, 0, 4094, 3, 2956, 4091, 0, 4083, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 3682, 4096, 1, 0, 0, 0, 0, 0, 10, 12, 4096, 0, 0, 4090, 4094, 4096, 0, 4, 0, 0, 4075, 0, 0, 0, 28, 0, 32, 3, 4083, 0, 0, 4096, 4086, 4096, 0, 0, 334, 4096, 0, 5, 409, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 3733, 4096, 0, 4091, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 11, 3973, 3072, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4079, 0, 0, 0, 1, 0, 1, 0, 4096, 0, 0, 4096, 4096, 4094, 0, 0, 0, 4096, 0, 0, 2495, 0, 9, 0, 0, 0, 0, 75, 5, 23, 37, 0, 0, 1, 0, 3929, 0, 2803, 4057, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[11, 1186, 561, 3693, 0, 0, 3, 0, 7, 16, 8, 0, 4088, 0, 0, 4084, 4005, 4076, 0, 2, 5, 0, 4094, 0, 0, 0, 3, 0, 30, 0, 4087, 0, 0, 4075, 4091, 4092, 0, 0, 25, 4085, 14, 0, 3645, 5, 1, 0, 0, 0, 2, 0, 0, 0, 171, 0, 0, 1, 8, 4096, 2, 3098, 4090, 0, 4072, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 315, 2175, 4067, 0, 15, 14, 7, 0, 14, 2, 0, 4091, 0, 0, 4078, 3958, 4074, 5, 1, 0, 0, 3243, 0, 0, 0, 14, 0, 86, 21, 4086, 0, 0, 4085, 4096, 4068, 0, 0, 0, 4090, 4, 0, 4086, 0, 543, 6, 0, 0, 0, 49, 0, 0, 3, 0, 6, 0, 0, 4092, 2, 1948, 4095, 0, 4094, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 3794, 1430, 9, 20, 0, 0, 1884, 126, 15, 0, 0, 0, 0, 0, 8, 0, 104, 1, 4064, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 294, 0, 0, 23, 116, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3878, 1, 3083, 916, 0, 0, 0, 0, 0, 0, 419, 4075, 3860, 0, 0, 6, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3153, 0, 0, 0, 1, 13, 3912, 702, 138, 0, 0, 0, 0, 0, 0, 171, 0, 52, 0, 10, 2, 21, 4083, 3942, 0, 0, 0, 0, 0, 0, 36, 1282, 12, 4089, 2938, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7296, 8269, 8607, 7878, 8901, 7923, 8281, 8381] avg=8192.0000 max=8901 max_rank=4 min=7296 min_rank=0 imbalance_ratio=1.086548 max_load_reduction=56635 max_load_reduction_pct=0.864182 moved_to_echo=33486 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 85, 3846, 3027, 0, 0, 0, 14, 1860, 129, 15, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4073, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 295, 0, 0, 1, 65, 2979, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 0, 0, 11, 4085, 0, 3096, 904, 0, 0, 0, 0, 0, 0, 0, 3986, 3949, 0, 4, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3632, 3, 7, 0, 2, 0, 4084, 705, 139, 0, 0, 0, 0, 0, 0, 6, 0, 5, 503, 0, 0, 0, 4094, 3947, 0, 0, 0, 0, 0, 0, 0, 1266, 1, 1680, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8979, 8168, 7423, 8826, 7970, 8572, 8555, 7043] avg=8192.0000 max=8979 max_rank=0 min=7043 min_rank=7 imbalance_ratio=1.096069 max_load_reduction=56557 max_load_reduction_pct=0.862991 moved_to_echo=32140 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3596, 1560, 0, 2, 0, 0, 1890, 129, 15, 0, 0, 0, 0, 0, 0, 1, 0, 47, 4078, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 294, 3, 0, 13, 84, 3600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 4095, 0, 3080, 918, 0, 0, 0, 0, 0, 0, 0, 4092, 3966, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 3886, 0, 223, 0, 0, 4, 4090, 701, 138, 0, 0, 0, 0, 0, 0, 34, 0, 1, 0, 0, 0, 2, 4096, 3946, 0, 0, 0, 0, 0, 0, 0, 1286, 0, 3390, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7192, 8182, 8062, 8136, 8060, 9070, 8079, 8755] avg=8192.0000 max=9070 max_rank=5 min=7192 min_rank=0 imbalance_ratio=1.107178 max_load_reduction=56466 max_load_reduction_pct=0.861603 moved_to_echo=33964 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 46, 2858, 2165, 0, 0, 6, 0, 1890, 129, 15, 0, 0, 0, 0, 0, 1, 15, 13, 0, 4075, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 295, 1, 0, 0, 126, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 17, 0, 4095, 0, 3095, 918, 0, 0, 0, 0, 0, 0, 0, 4078, 3961, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 1, 0, 0, 4083, 705, 139, 0, 0, 0, 0, 0, 2, 38, 4, 26, 18, 0, 0, 1, 4058, 3937, 0, 0, 0, 0, 0, 0, 0, 1286, 3, 2956, 4091, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7110, 8194, 8612, 8129, 8045, 9024, 8084, 8338] avg=8192.0000 max=9024 max_rank=5 min=7110 min_rank=0 imbalance_ratio=1.101562 max_load_reduction=56512 max_load_reduction_pct=0.862305 moved_to_echo=33491 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[11, 1186, 561, 3693, 0, 0, 3, 0, 1894, 133, 20, 0, 0, 0, 0, 0, 7, 16, 8, 0, 4068, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 287, 0, 2, 5, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 30, 0, 4087, 0, 3085, 924, 0, 0, 0, 0, 0, 0, 0, 4075, 3958, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3645, 5, 1, 0, 0, 0, 4072, 704, 140, 0, 0, 0, 0, 0, 2, 0, 0, 0, 171, 0, 0, 1, 4085, 3952, 0, 0, 0, 0, 0, 0, 8, 1278, 2, 3098, 4090, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7501, 8183, 8393, 8129, 8058, 8581, 8211, 8480] avg=8192.0000 max=8581 max_rank=5 min=7501 min_rank=0 imbalance_ratio=1.047485 max_load_reduction=56955 max_load_reduction_pct=0.869064 moved_to_echo=33330 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3682, 4096, 1, 0, 0, 0, 1886, 129, 15, 0, 0, 0, 0, 0, 0, 0, 10, 12, 4081, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 295, 0, 4, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 32, 3, 4083, 0, 3096, 917, 0, 0, 0, 0, 0, 0, 0, 4096, 3957, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 409, 0, 0, 0, 0, 0, 4091, 705, 139, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 3957, 0, 0, 0, 0, 0, 0, 0, 1284, 0, 3733, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9810, 8193, 8468, 8159, 8387, 5349, 8057, 9113] avg=8192.0000 max=9810 max_rank=0 min=5349 min_rank=5 imbalance_ratio=1.197510 max_load_reduction=55726 max_load_reduction_pct=0.850311 moved_to_echo=30906 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 3973, 3072, 0, 0, 0, 7, 1814, 129, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 295, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4096, 0, 3096, 881, 0, 0, 0, 0, 0, 0, 0, 4096, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2495, 0, 9, 0, 0, 0, 4096, 705, 140, 0, 0, 0, 0, 0, 0, 75, 5, 23, 37, 0, 0, 1, 4096, 3954, 0, 0, 0, 0, 0, 0, 0, 1234, 0, 2803, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9021, 8177, 8470, 8075, 8063, 7445, 8191, 8094] avg=8192.0000 max=9021 max_rank=0 min=7445 min_rank=5 imbalance_ratio=1.101196 max_load_reduction=56515 max_load_reduction_pct=0.862350 moved_to_echo=31793 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 315, 2175, 4067, 0, 15, 14, 7, 1889, 129, 15, 0, 0, 0, 0, 0, 0, 14, 2, 0, 4076, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 294, 5, 1, 0, 0, 3243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 86, 21, 4086, 0, 3079, 918, 0, 0, 0, 0, 0, 0, 0, 4085, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4086, 0, 543, 6, 0, 0, 4094, 701, 138, 0, 0, 0, 0, 0, 0, 49, 0, 0, 3, 0, 6, 0, 4090, 3930, 0, 0, 0, 0, 0, 0, 0, 1285, 2, 1948, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8627, 8170, 7501, 8204, 8052, 9572, 8078, 7332] avg=8192.0000 max=9572 max_rank=5 min=7332 min_rank=7 imbalance_ratio=1.168457 max_load_reduction=55964 max_load_reduction_pct=0.853943 moved_to_echo=33034 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 1664, 24576, 23168, 128, 128, 128, 128, 15104, 1152, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32512, 2432, 128, 128, 128, 512, 30336, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 10240, 128, 23808, 31616, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 25472, 128, 896, 128, 128, 128, 32640, 5632, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 896, 0, 256, 128, 32512, 128, 24832, 7296, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 256, 128, 32640, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 384, 128, 128, 768, 128, 128, 128, 32768, 31616, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 32640, 31616, 0, 128, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[31, 4081, 0, 0, 0, 4, 10, 2, 4091, 3754, 0, 4096, 4068, 2, 0, 180, 3, 4, 0, 50, 4096, 0, 0, 0, 3999, 0, 23, 3803, 4096, 0, 4093, 0, 0, 1069, 0, 0, 0, 0, 0, 20, 0, 4, 4096, 0, 0, 4092, 0, 18, 4085, 0, 3997, 4060, 0, 2, 0, 0, 0, 0, 2, 179, 3426, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[56, 4074, 0, 0, 0, 1, 0, 0, 4081, 2149, 153, 4045, 4039, 106, 0, 0, 110, 13, 23, 0, 4048, 18, 0, 0, 1795, 0, 282, 3998, 4079, 1, 4079, 3, 0, 4003, 0, 0, 43, 2, 57, 0, 0, 89, 4070, 19, 0, 4043, 0, 9, 4077, 10, 4081, 3872, 0, 0, 19, 3, 0, 0, 0, 0, 3964, 1, 10, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[10, 3960, 3, 0, 182, 0, 3, 0, 4093, 31, 1, 4061, 4080, 0, 0, 0, 0, 0, 0, 16, 4092, 0, 1, 4, 3717, 4, 43, 4084, 4096, 0, 4096, 0, 10, 3443, 0, 0, 0, 0, 3, 0, 102, 119, 4091, 0, 0, 4096, 0, 54, 4055, 0, 4096, 3924, 0, 199, 8, 0, 0, 0, 0, 562, 4094, 102, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[6, 3616, 0, 0, 2126, 0, 875, 0, 3936, 54, 0, 4086, 4089, 245, 0, 8, 0, 2, 3, 0, 4096, 0, 0, 4, 3985, 0, 0, 4059, 4086, 0, 3656, 103, 0, 2996, 0, 0, 0, 0, 6, 15, 0, 0, 4090, 0, 2, 4094, 3, 11, 4095, 127, 4086, 4094, 1, 162, 0, 1, 0, 13, 0, 487, 2218, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 3847, 0, 0, 0, 0, 0, 0, 3265, 4091, 1, 4095, 4095, 7, 7, 0, 0, 0, 0, 15, 4096, 0, 0, 2, 1778, 5, 0, 4095, 4075, 8, 4093, 0, 0, 4087, 0, 0, 11, 0, 43, 0, 0, 0, 4096, 0, 0, 4093, 0, 36, 4092, 0, 4054, 4096, 0, 17, 0, 0, 0, 0, 1, 176, 3159, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 3030, 0, 0, 3, 0, 0, 0, 2265, 3414, 330, 4092, 4096, 22, 3, 192, 39, 0, 0, 405, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 2939, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4082, 0, 4074, 4096, 0, 12, 20, 0, 0, 0, 0, 0, 3721, 27, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4084, 17, 0, 45, 1, 1, 8, 3334, 115, 0, 3879, 4073, 3163, 0, 0, 0, 0, 23, 58, 4096, 15, 33, 0, 2332, 0, 0, 4096, 4093, 0, 4092, 0, 0, 2978, 0, 24, 0, 0, 0, 0, 17, 1, 4095, 0, 0, 4090, 0, 0, 4096, 5, 4096, 4090, 0, 30, 0, 0, 0, 0, 0, 761, 3694, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[30, 4063, 0, 0, 28, 0, 48, 11, 4091, 3597, 0, 4094, 4081, 1422, 0, 0, 35, 8, 0, 6, 4080, 0, 0, 0, 1470, 0, 1, 4090, 4009, 0, 4087, 16, 0, 2405, 0, 7, 0, 7, 12, 0, 0, 2, 4096, 0, 0, 4087, 0, 0, 4050, 9, 3900, 4034, 0, 0, 0, 0, 0, 45, 1, 134, 3474, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[56, 4074, 0, 0, 0, 1, 0, 0, 2382, 1458, 46, 0, 0, 0, 0, 0, 4081, 2149, 153, 1663, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 13, 23, 0, 4048, 18, 0, 0, 2623, 1353, 0, 0, 0, 0, 0, 0, 1795, 0, 282, 1187, 0, 1, 4079, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 43, 2, 57, 0, 4079, 1080, 0, 0, 0, 0, 0, 0, 0, 89, 4024, 19, 0, 4043, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 34, 10, 4081, 3872, 0, 0, 19, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 1, 10, 11, 2959, 1420, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8017, 8152, 8188, 7347, 9264, 8184, 8019, 8365] avg=8192.0000 max=9264 max_rank=4 min=7347 min_rank=3 imbalance_ratio=1.130859 max_load_reduction=56272 max_load_reduction_pct=0.858643 moved_to_echo=33832 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[31, 4081, 0, 0, 0, 4, 10, 2, 2412, 1387, 46, 0, 0, 0, 0, 0, 4091, 3754, 0, 1684, 0, 2, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 50, 4096, 0, 0, 0, 2629, 1287, 0, 0, 0, 0, 0, 0, 3999, 0, 23, 1129, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 0, 0, 0, 0, 0, 20, 4096, 1088, 0, 0, 0, 0, 0, 0, 0, 4, 4050, 0, 0, 4092, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 3997, 4060, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 179, 3426, 0, 0, 0, 2980, 1423, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7973, 9711, 8069, 9244, 6273, 8164, 8092, 8010] avg=8192.0000 max=9711 max_rank=1 min=6273 min_rank=4 imbalance_ratio=1.185425 max_load_reduction=55825 max_load_reduction_pct=0.851822 moved_to_echo=30539 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[10, 3960, 3, 0, 182, 0, 3, 0, 2392, 1490, 46, 0, 0, 0, 0, 0, 4093, 31, 1, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4092, 0, 1, 4, 2609, 1382, 0, 0, 0, 0, 0, 0, 3717, 4, 43, 1212, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3443, 0, 0, 0, 0, 3, 0, 4096, 1091, 0, 0, 0, 0, 0, 0, 102, 119, 4045, 0, 0, 4096, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4096, 3924, 0, 199, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 4094, 102, 1, 0, 2989, 1412, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 5794, 8104, 9072, 8643, 8416, 8261, 9160] avg=8192.0000 max=9160 max_rank=7 min=5794 min_rank=1 imbalance_ratio=1.118164 max_load_reduction=56376 max_load_reduction_pct=0.860229 moved_to_echo=34480 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[6, 3616, 0, 0, 2126, 0, 875, 0, 2407, 1481, 46, 0, 0, 0, 0, 0, 3936, 54, 0, 1679, 0, 245, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 4096, 0, 0, 4, 2635, 1374, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 1204, 0, 0, 3656, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2996, 0, 0, 0, 0, 6, 15, 4086, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 2, 4094, 3, 11, 0, 0, 0, 0, 0, 0, 0, 0, 34, 127, 4086, 4094, 1, 162, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 487, 2218, 0, 0, 0, 2996, 1426, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10557, 5922, 8114, 8948, 8196, 8154, 8505, 7140] avg=8192.0000 max=10557 max_rank=0 min=5922 min_rank=1 imbalance_ratio=1.288696 max_load_reduction=54979 max_load_reduction_pct=0.838913 moved_to_echo=31995 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 17, 0, 45, 1, 1, 8, 2289, 1498, 49, 0, 0, 0, 0, 0, 3334, 115, 0, 1590, 0, 3163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 58, 4096, 15, 33, 0, 2640, 1392, 0, 0, 0, 0, 0, 0, 2332, 0, 0, 1206, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2978, 0, 24, 0, 0, 0, 0, 4093, 1090, 0, 0, 0, 0, 0, 0, 17, 1, 4046, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 5, 4096, 4090, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 761, 3694, 0, 0, 1, 2983, 1430, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7992, 8202, 8257, 7630, 8185, 8154, 8247, 8869] avg=8192.0000 max=8869 max_rank=7 min=7630 min_rank=3 imbalance_ratio=1.082642 max_load_reduction=56667 max_load_reduction_pct=0.864670 moved_to_echo=33455 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3030, 0, 0, 3, 0, 0, 0, 2410, 1494, 46, 0, 0, 0, 0, 0, 2265, 3414, 330, 1682, 0, 22, 3, 192, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 405, 4096, 0, 0, 0, 2627, 1386, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1216, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2939, 0, 0, 0, 0, 2, 0, 4096, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4074, 4096, 0, 12, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 27, 0, 0, 3000, 1422, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6983, 7908, 8553, 9408, 8133, 8146, 8235, 8170] avg=8192.0000 max=9408 max_rank=3 min=6983 min_rank=0 imbalance_ratio=1.148438 max_load_reduction=56128 max_load_reduction_pct=0.856445 moved_to_echo=32684 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[30, 4063, 0, 0, 28, 0, 48, 11, 2411, 1492, 46, 0, 0, 0, 0, 0, 4091, 3597, 0, 1683, 0, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 8, 0, 6, 4080, 0, 0, 0, 2606, 1384, 0, 0, 0, 0, 0, 0, 1470, 0, 1, 1214, 0, 0, 4087, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2405, 0, 7, 0, 7, 12, 0, 4009, 1091, 0, 0, 0, 0, 0, 0, 0, 2, 4050, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 9, 3900, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 1, 134, 3474, 0, 6, 0, 2990, 1411, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 10793, 8119, 6788, 7531, 8139, 7976, 8061] avg=8192.0000 max=10793 max_rank=1 min=6788 min_rank=3 imbalance_ratio=1.317505 max_load_reduction=54743 max_load_reduction_pct=0.835312 moved_to_echo=31707 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3847, 0, 0, 0, 0, 0, 0, 2412, 1494, 46, 0, 0, 0, 0, 0, 3265, 4091, 1, 1683, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 2, 2633, 1386, 0, 0, 0, 0, 0, 0, 1778, 5, 0, 1215, 0, 8, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 11, 0, 43, 0, 4075, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 4093, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4054, 4096, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 176, 3159, 0, 0, 0, 3000, 1425, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7799, 9054, 8132, 7099, 9311, 8179, 8201, 7761] avg=8192.0000 max=9311 max_rank=4 min=7099 min_rank=3 imbalance_ratio=1.136597 max_load_reduction=56225 max_load_reduction_pct=0.857925 moved_to_echo=33452 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 256, 32384, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 256, 32384, 32384, 128, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 2304, 27776, 256, 128, 128, 23936, 11392, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[29184, 17280, 512, 13440, 0, 4992, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 30848, 128, 0, 2432, 128, 1024, 128, 19200, 11904, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[23296, 128, 384, 9600, 0, 128, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 128, 128, 640, 32768, 128, 128, 128, 21120, 11008, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 23936, 0, 128, 128, 128, 128, 128, 32640, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4077, 0, 0, 0, 4075, 0, 364, 4019, 0, 0, 38, 13, 0, 26, 0, 15, 4082, 4069, 31, 38, 0, 0, 12, 0, 1, 4069, 15, 0, 25, 13, 0, 4094, 87, 4, 0, 0, 4057, 0, 3995, 4013, 0, 11, 4041, 3611, 4015, 0, 4085, 6, 0, 0, 0, 0, 64, 4079, 0, 47, 0, 23, 0, 84, 141, 8, 4089, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4041, 0, 0, 0, 4078, 0, 129, 4085, 0, 446, 1, 0, 0, 0, 0, 17, 4094, 4096, 3, 2, 0, 0, 0, 0, 0, 2270, 0, 29, 0, 0, 0, 4061, 31, 0, 25, 0, 4096, 3, 4042, 2604, 105, 1, 4096, 3714, 4083, 0, 4096, 9, 0, 12, 7, 0, 3927, 3788, 6, 2, 0, 0, 0, 4, 0, 1720, 1813, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4088, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4092, 0, 26, 0, 7, 0, 0, 0, 1961, 10, 0, 0, 0, 0, 3963, 72, 0, 12, 2, 4045, 0, 4089, 3739, 109, 0, 4073, 3649, 4093, 0, 4090, 0, 0, 18, 5, 0, 3929, 3798, 0, 149, 93, 0, 0, 1, 0, 110, 3176, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4094, 2, 35, 0, 4082, 2, 0, 4095, 0, 34, 8, 0, 0, 2, 4, 8, 4083, 3996, 0, 0, 2, 7, 0, 13, 0, 2997, 0, 2, 6, 0, 0, 4088, 0, 0, 0, 0, 4094, 212, 4089, 2950, 0, 1, 4048, 4095, 4094, 0, 4085, 0, 13, 0, 0, 0, 2017, 3972, 0, 0, 0, 1, 7, 1, 0, 253, 4044, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4082, 2, 14, 0, 4096, 0, 0, 4096, 0, 61, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 2328, 0, 0, 0, 0, 0, 2942, 0, 0, 55, 0, 4096, 0, 4070, 4096, 36, 0, 4060, 4081, 4096, 0, 4096, 0, 0, 0, 1, 0, 3986, 4096, 0, 0, 0, 0, 0, 0, 0, 1758, 1199, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4089, 12, 0, 1, 4082, 2, 0, 4092, 0, 2, 0, 0, 1, 0, 9, 0, 4087, 4065, 0, 8, 0, 0, 12, 10, 0, 3765, 0, 696, 4, 0, 0, 4037, 0, 0, 22, 0, 4096, 0, 4088, 3562, 20, 0, 4070, 4096, 4089, 0, 4086, 0, 0, 1, 0, 0, 4034, 3248, 0, 16, 0, 0, 3, 1, 2, 117, 1011, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4094, 12, 2, 6, 4079, 0, 414, 4062, 2, 0, 44, 2, 0, 0, 0, 11, 4095, 4086, 8, 0, 0, 0, 2, 0, 11, 2797, 0, 1126, 0, 0, 1, 4096, 0, 0, 0, 1, 4004, 21, 4061, 3943, 3, 0, 4052, 3981, 4082, 0, 4096, 0, 19, 0, 0, 0, 4066, 296, 0, 205, 3, 1, 0, 36, 0, 171, 3545, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4085, 0, 22, 1, 4060, 0, 14, 4043, 0, 59, 1, 0, 0, 2, 0, 0, 3978, 4094, 0, 0, 0, 0, 3, 2, 9, 4016, 0, 102, 5, 0, 0, 4082, 15, 13, 0, 0, 4096, 0, 4095, 4063, 1, 6, 4096, 4080, 4096, 0, 4091, 5, 1, 0, 0, 45, 25, 4024, 0, 6, 0, 0, 0, 24, 100, 1, 4075, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4004, 0, 364, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 13, 0, 26, 0, 15, 4085, 3995, 0, 0, 0, 0, 0, 0, 4082, 4069, 31, 38, 0, 0, 12, 0, 19, 0, 0, 0, 0, 0, 0, 0, 1, 4069, 15, 0, 25, 13, 0, 4094, 932, 52, 0, 0, 0, 0, 0, 0, 87, 4, 0, 0, 433, 0, 3995, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4041, 3611, 170, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 4079, 0, 47, 1946, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 84, 141, 8, 4089, 0, 3845, 746, 82, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8387, 8172, 8251, 9201, 8532, 7839, 6136, 9018] avg=8192.0000 max=9201 max_rank=3 min=6136 min_rank=6 imbalance_ratio=1.123169 max_load_reduction=56335 max_load_reduction_pct=0.859604 moved_to_echo=31525 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4007, 0, 129, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 1, 0, 0, 0, 0, 17, 4096, 3961, 0, 0, 0, 0, 0, 0, 4094, 4096, 3, 2, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 2270, 0, 29, 0, 0, 0, 4061, 941, 52, 0, 0, 0, 0, 0, 0, 31, 0, 25, 0, 437, 3, 4042, 2604, 0, 0, 0, 0, 0, 0, 0, 0, 105, 1, 4096, 3714, 173, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 7, 0, 3927, 3788, 6, 2, 1965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1720, 1813, 0, 3910, 753, 80, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8521, 8214, 7353, 7142, 8098, 9707, 8280] avg=8192.0000 max=9707 max_rank=6 min=7142 min_rank=4 imbalance_ratio=1.184937 max_load_reduction=55829 max_load_reduction_pct=0.851883 moved_to_echo=33227 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4017, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4013, 0, 0, 0, 0, 0, 0, 4088, 4092, 0, 26, 0, 7, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 1961, 10, 0, 0, 0, 0, 3963, 930, 52, 0, 0, 0, 0, 0, 0, 72, 0, 12, 2, 431, 0, 4089, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 4073, 3649, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 5, 0, 3929, 3798, 0, 149, 1940, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 1, 0, 110, 3176, 6, 3919, 744, 82, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7965, 8103, 8232, 6916, 8345, 8005, 9839, 8131] avg=8192.0000 max=9839 max_rank=6 min=6916 min_rank=3 imbalance_ratio=1.201050 max_load_reduction=55697 max_load_reduction_pct=0.849869 moved_to_echo=34320 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 35, 0, 4011, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 8, 0, 0, 2, 4, 8, 4085, 4012, 0, 0, 0, 0, 0, 0, 4083, 3996, 0, 0, 2, 7, 0, 13, 19, 0, 0, 0, 0, 0, 0, 0, 0, 2997, 0, 2, 6, 0, 0, 4088, 941, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 212, 4089, 2950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4048, 4095, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 2017, 3972, 0, 0, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 1, 0, 253, 4044, 0, 3920, 753, 82, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8153, 8120, 8086, 7687, 8318, 7966, 9061] avg=8192.0000 max=9061 max_rank=7 min=7687 min_rank=4 imbalance_ratio=1.106079 max_load_reduction=56475 max_load_reduction_pct=0.861740 moved_to_echo=33032 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 2, 6, 4008, 0, 414, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 44, 2, 0, 0, 0, 11, 4096, 4012, 0, 0, 0, 0, 0, 0, 4095, 4086, 8, 0, 0, 0, 2, 0, 19, 0, 0, 0, 0, 0, 0, 0, 11, 2797, 0, 1126, 0, 0, 1, 4096, 920, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 427, 21, 4061, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4052, 3981, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4066, 296, 0, 205, 1921, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 36, 0, 171, 3545, 0, 3909, 736, 82, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8504, 8167, 8210, 9003, 8453, 8209, 6507, 8483] avg=8192.0000 max=9003 max_rank=3 min=6507 min_rank=6 imbalance_ratio=1.098999 max_load_reduction=56533 max_load_reduction_pct=0.862625 moved_to_echo=31652 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 22, 1, 3979, 0, 14, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 1, 0, 0, 2, 0, 0, 4091, 4004, 0, 0, 0, 0, 0, 0, 3978, 4094, 0, 0, 0, 0, 3, 2, 23, 0, 0, 0, 0, 0, 0, 0, 9, 4016, 0, 102, 5, 0, 0, 4082, 947, 58, 0, 0, 0, 0, 0, 0, 15, 13, 0, 0, 424, 0, 4095, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4096, 4080, 169, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 45, 25, 4024, 0, 6, 1968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 100, 1, 4075, 0, 3927, 757, 81, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8059, 8157, 8100, 9219, 8610, 8357, 6069, 8965] avg=8192.0000 max=9219 max_rank=3 min=6069 min_rank=6 imbalance_ratio=1.125366 max_load_reduction=56317 max_load_reduction_pct=0.859329 moved_to_echo=32001 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 0, 1, 4011, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 9, 0, 4086, 4007, 0, 0, 0, 0, 0, 0, 4087, 4065, 0, 8, 0, 0, 12, 10, 19, 0, 0, 0, 0, 0, 0, 0, 0, 3765, 0, 696, 4, 0, 0, 4037, 941, 52, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 437, 0, 4088, 3562, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4070, 4096, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4034, 3248, 0, 16, 1965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 2, 117, 1011, 0, 3916, 753, 82, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8105, 8201, 9495, 8109, 8359, 9264, 5885] avg=8192.0000 max=9495 max_rank=3 min=5885 min_rank=7 imbalance_ratio=1.159058 max_load_reduction=56041 max_load_reduction_pct=0.855118 moved_to_echo=31617 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 14, 0, 4025, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 4096, 4001, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 2328, 0, 0, 0, 0, 0, 2942, 941, 52, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 437, 0, 4070, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4060, 4081, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3986, 4096, 0, 0, 1965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1758, 1199, 0, 3922, 753, 81, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8158, 8208, 6263, 8658, 8351, 10048, 7713] avg=8192.0000 max=10048 max_rank=6 min=6263 min_rank=3 imbalance_ratio=1.226562 max_load_reduction=55488 max_load_reduction_pct=0.846680 moved_to_echo=34770 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32128, 128, 1024, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 32640, 128, 128, 128, 128, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 256, 256, 4224, 23040, 128, 31360, 6016, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 128, 32640, 31360, 1408, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 24320, 128, 2048, 128, 128, 128, 31488, 7552, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 128, 128, 128, 3584, 256, 32640, 29056, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 22144, 27392, 128, 512, 15744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 640, 128, 128, 128, 128, 128, 128, 32768, 32128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4086, 0, 0, 1408, 0, 186, 4054, 30, 6, 3997, 2478, 4, 0, 4055, 0, 18, 1, 4068, 4069, 13, 0, 0, 1, 1, 9, 0, 0, 0, 4066, 27, 41, 3, 1958, 2104, 44, 2, 4072, 7, 14, 20, 866, 4090, 0, 0, 4063, 3275, 0, 0, 4080, 39, 22, 13, 0, 0, 4054, 0, 29, 2, 0, 4069, 5, 0, 0, 87] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4096, 6, 0, 1381, 0, 164, 4086, 4, 0, 3634, 4030, 0, 0, 10, 0, 0, 0, 4094, 4096, 0, 0, 0, 192, 0, 0, 0, 0, 0, 4096, 3, 13, 0, 0, 1138, 10, 0, 4094, 0, 0, 2773, 3071, 4096, 0, 2, 4095, 4079, 7, 2, 4093, 0, 7, 0, 0, 4, 4090, 0, 0, 0, 0, 4060, 0, 0, 10, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4092, 0, 0, 2139, 6, 4, 4054, 1, 3, 4078, 4087, 0, 0, 153, 0, 2, 2, 4093, 4093, 0, 0, 117, 91, 0, 0, 0, 7, 4, 4087, 0, 0, 0, 4029, 1784, 2, 0, 4093, 0, 0, 0, 0, 4092, 16, 0, 4095, 4096, 0, 0, 3753, 2, 10, 4, 0, 0, 4091, 0, 0, 167, 96, 4089, 0, 0, 1, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4096, 6, 0, 3413, 0, 0, 4095, 0, 4, 3987, 4069, 0, 0, 94, 0, 0, 2, 4096, 4092, 0, 0, 0, 0, 0, 24, 4, 0, 0, 4094, 25, 109, 0, 40, 347, 0, 0, 4093, 5, 0, 1279, 3612, 4092, 0, 6, 4075, 3479, 1, 0, 4096, 0, 0, 1, 0, 0, 4090, 0, 0, 0, 0, 4091, 19, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4073, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 988, 0, 0, 3941, 0, 0, 0, 4096, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 245, 255, 0, 0, 4096, 0, 0, 4083, 2982, 4096, 0, 0, 4085, 3475, 17, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 423, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4086, 1, 14, 81, 7, 8, 4096, 0, 3, 4095, 1673, 0, 0, 4067, 2, 0, 1, 4093, 4096, 0, 0, 209, 0, 0, 0, 13, 0, 0, 4096, 0, 30, 0, 2, 2639, 0, 0, 4094, 0, 0, 1028, 1282, 4095, 0, 0, 4096, 3992, 0, 1, 4090, 0, 0, 0, 0, 0, 4092, 0, 4, 0, 2, 4083, 0, 0, 1365, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4080, 5, 0, 21, 0, 0, 4078, 0, 2, 4096, 584, 17, 0, 3082, 2, 3, 0, 4090, 4094, 5, 10, 18, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 958, 337, 0, 17, 4043, 0, 0, 4002, 2860, 4078, 4, 0, 4096, 3450, 0, 0, 4087, 0, 0, 0, 0, 41, 4090, 0, 14, 0, 0, 4095, 62, 1, 938, 98] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[3999, 8, 23, 7, 0, 0, 4089, 18, 0, 4096, 2382, 0, 0, 3673, 29, 12, 0, 4059, 4087, 0, 5, 864, 115, 0, 7, 0, 0, 4, 4083, 1, 6, 5, 458, 0, 0, 0, 4055, 0, 4, 691, 3679, 4077, 0, 0, 4095, 4096, 0, 0, 4075, 0, 0, 3, 0, 0, 4088, 0, 1, 1, 49, 4085, 4, 0, 502, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 0, 1408, 0, 186, 2979, 30, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3249, 2478, 4, 0, 4055, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4068, 3880, 13, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4066, 27, 41, 3, 4063, 0, 0, 0, 0, 0, 0, 0, 1958, 2104, 44, 2, 4072, 7, 14, 20, 176, 149, 0, 0, 0, 0, 0, 0, 866, 0, 0, 0, 2136, 3275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 39, 22, 13, 0, 0, 4054, 0, 40, 0, 0, 0, 0, 0, 0, 0, 29, 2, 0, 4069, 5, 0, 0, 87, 1927, 1075, 572, 27, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8689, 9810, 7964, 8209, 8546, 6277, 8248, 7793] avg=8192.0000 max=9810 max_rank=1 min=6277 min_rank=5 imbalance_ratio=1.197510 max_load_reduction=55726 max_load_reduction_pct=0.850311 moved_to_echo=30864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 6, 0, 1381, 0, 164, 3003, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2954, 4030, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3905, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 13, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 1138, 10, 0, 4094, 0, 0, 2773, 160, 150, 0, 0, 0, 0, 0, 0, 3071, 0, 0, 2, 2153, 4079, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 7, 0, 0, 4, 4090, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 10, 0, 1942, 1083, 520, 26, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8654, 6994, 8191, 8182, 8325, 9314, 8235, 7641] avg=8192.0000 max=9314 max_rank=5 min=6994 min_rank=1 imbalance_ratio=1.136963 max_load_reduction=56222 max_load_reduction_pct=0.857880 moved_to_echo=33515 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 6, 0, 3413, 0, 0, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3241, 4069, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4, 0, 0, 4094, 25, 109, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 40, 347, 0, 0, 4093, 5, 0, 1279, 175, 150, 0, 0, 0, 0, 0, 0, 3612, 0, 0, 6, 2142, 3479, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4090, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 19, 0, 0, 0, 1933, 1086, 571, 27, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10524, 7408, 7999, 8321, 6089, 9240, 8228, 7727] avg=8192.0000 max=10524 max_rank=0 min=6089 min_rank=4 imbalance_ratio=1.284668 max_load_reduction=55012 max_load_reduction_pct=0.839417 moved_to_echo=31284 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 2139, 6, 4, 2979, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3315, 4087, 0, 0, 153, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 3902, 0, 0, 117, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 4087, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 4029, 1784, 2, 0, 4093, 0, 0, 0, 179, 150, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 2153, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 2, 10, 4, 0, 0, 4091, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 167, 96, 4089, 0, 0, 1, 3, 1942, 1075, 584, 26, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9221, 7560, 8205, 8164, 10237, 6265, 7901, 7983] avg=8192.0000 max=10237 max_rank=4 min=6265 min_rank=5 imbalance_ratio=1.249634 max_load_reduction=55299 max_load_reduction_pct=0.843796 moved_to_echo=32386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 0, 0, 0, 0, 3010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3326, 988, 0, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3905, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 245, 255, 0, 0, 4096, 0, 0, 4083, 180, 150, 0, 0, 0, 0, 0, 0, 2982, 0, 0, 0, 2147, 3475, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 4096, 0, 0, 0, 0, 1938, 1086, 586, 26, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7083, 8255, 8013, 8167, 9009, 8621, 8233, 8155] avg=8192.0000 max=9009 max_rank=4 min=7083 min_rank=0 imbalance_ratio=1.099731 max_load_reduction=56527 max_load_reduction_pct=0.862534 moved_to_echo=34018 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4080, 5, 0, 21, 0, 0, 2992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3320, 584, 17, 0, 3082, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3894, 5, 10, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 958, 337, 0, 17, 4043, 0, 0, 4002, 185, 157, 0, 0, 0, 0, 0, 0, 2860, 0, 4, 0, 2149, 3450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 41, 4090, 0, 43, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4095, 62, 1, 938, 98, 1947, 1086, 591, 26, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7098, 7010, 8017, 8130, 9699, 8463, 8261, 8858] avg=8192.0000 max=9699 max_rank=4 min=7010 min_rank=1 imbalance_ratio=1.183960 max_load_reduction=55837 max_load_reduction_pct=0.852005 moved_to_echo=35281 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4086, 1, 14, 81, 7, 8, 3010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3329, 1673, 0, 0, 4067, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 3905, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4096, 0, 30, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 2, 2639, 0, 0, 4094, 0, 0, 1028, 180, 150, 0, 0, 0, 0, 0, 0, 1282, 0, 0, 0, 2153, 3992, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4092, 0, 41, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 4083, 0, 0, 1365, 0, 1943, 1086, 586, 27, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7207, 9074, 8208, 8207, 8093, 7428, 8223, 9096] avg=8192.0000 max=9096 max_rank=7 min=7207 min_rank=0 imbalance_ratio=1.110352 max_load_reduction=56440 max_load_reduction_pct=0.861206 moved_to_echo=32840 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3999, 8, 23, 7, 0, 0, 3005, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 2382, 0, 0, 3673, 29, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 3896, 0, 5, 864, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4, 4083, 1, 6, 5, 4050, 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, 0, 4055, 0, 4, 691, 180, 150, 0, 0, 0, 0, 0, 0, 3679, 0, 0, 0, 2153, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 3, 0, 0, 4088, 0, 41, 0, 0, 0, 0, 0, 0, 0, 1, 1, 49, 4085, 4, 0, 502, 1, 1942, 1084, 587, 27, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7060, 9425, 8939, 8156, 5538, 9928, 8207, 8283] avg=8192.0000 max=9928 max_rank=5 min=5538 min_rank=4 imbalance_ratio=1.211914 max_load_reduction=55608 max_load_reduction_pct=0.848511 moved_to_echo=31956 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 26112, 20352, 128, 0, 19200, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32384, 128, 128, 128, 0, 128, 32768, 0, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 8576, 128, 384, 24064, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 31232, 128, 128, 1280, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 256, 640, 32768, 128, 128, 2816, 256, 15616, 8704, 4608, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[7808, 8704, 128, 128, 32640, 128, 128, 13952, 1536, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[18432, 0, 128, 128, 17280, 29952, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 256, 128, 32512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 89, 3724, 13, 39, 0, 4096, 0, 48, 0, 21, 1821, 12, 110, 4067, 4046, 4061, 0, 0, 7, 0, 0, 23, 0, 3657, 3886, 0, 265, 0, 1729, 5, 0, 31, 4026, 8, 1033, 0, 2, 5, 172, 0, 70, 4070, 1, 8, 0, 0, 11, 13, 0, 4072, 7, 0, 0, 4052, 4043, 5, 4078, 0, 4083, 0, 4027, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[35, 0, 0, 4088, 0, 0, 136, 4063, 0, 0, 1, 0, 342, 0, 0, 4096, 4090, 4028, 0, 0, 0, 56, 8, 0, 20, 3638, 3654, 7, 26, 5, 4093, 0, 0, 0, 4089, 0, 1315, 1, 0, 0, 0, 0, 11, 4075, 0, 3, 0, 0, 2, 0, 1, 4080, 0, 0, 18, 4096, 4095, 0, 4092, 11, 4096, 0, 3165, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 26, 534, 0, 0, 110, 3725, 0, 0, 26, 0, 2502, 0, 0, 4095, 4094, 2150, 0, 0, 0, 50, 0, 0, 0, 4091, 3944, 0, 10, 0, 4096, 0, 0, 0, 4087, 4, 2965, 0, 0, 0, 399, 0, 40, 4096, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 4096, 4096, 17, 4076, 0, 4046, 0, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[27, 0, 0, 2397, 0, 69, 204, 4010, 0, 0, 0, 0, 4095, 1, 0, 4076, 4064, 2106, 19, 0, 0, 560, 0, 0, 0, 2, 4053, 0, 191, 5, 4086, 12, 0, 22, 4080, 0, 2744, 6, 0, 0, 365, 0, 0, 4095, 28, 16, 3, 0, 10, 1, 0, 4082, 0, 0, 81, 4058, 4094, 0, 3966, 0, 3852, 2, 4026, 28] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3753, 0, 0, 824, 4094, 1, 103, 0, 0, 3405, 0, 0, 4093, 4092, 4095, 2, 1, 6, 0, 12, 4, 0, 455, 4050, 1, 0, 0, 3008, 4, 88, 0, 4094, 0, 615, 1, 4, 6, 49, 0, 0, 4086, 0, 0, 1, 0, 2, 5, 0, 4089, 3, 0, 0, 4089, 4092, 8, 4096, 0, 4089, 6, 4095, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 4, 4090, 0, 0, 22, 4036, 0, 0, 6, 0, 1350, 1, 55, 4096, 4093, 3987, 0, 1, 0, 181, 0, 0, 0, 229, 2509, 0, 0, 0, 4095, 0, 0, 0, 4088, 2, 4096, 0, 0, 0, 7, 0, 6, 4089, 1, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 1, 4096, 4096, 0, 4094, 0, 4006, 0, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 1, 55, 4082, 1, 7, 0, 0, 1222, 0, 1, 4096, 4096, 3763, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 0, 0, 3481, 0, 0, 0, 4096, 0, 3825, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 47, 0, 4077, 4096, 0, 4096, 0, 4063, 0, 4046, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[3, 0, 0, 4092, 0, 6, 0, 4085, 1, 37, 5, 0, 3996, 11, 1, 4085, 4089, 4065, 0, 17, 0, 792, 0, 0, 0, 39, 4082, 0, 0, 35, 4083, 11, 1, 0, 4065, 1, 179, 0, 0, 0, 78, 0, 205, 4095, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 45, 0, 4096, 4071, 9, 4096, 54, 3252, 0, 3654, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 89, 3724, 13, 39, 0, 4096, 947, 202, 0, 0, 0, 0, 0, 0, 0, 48, 0, 21, 1821, 12, 110, 4067, 1707, 0, 0, 0, 0, 0, 0, 0, 4046, 4061, 0, 0, 7, 0, 0, 23, 361, 7, 0, 0, 0, 0, 0, 0, 0, 3657, 2578, 0, 265, 0, 1729, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4026, 8, 1033, 0, 2, 5, 1831, 163, 0, 0, 0, 0, 0, 0, 172, 0, 70, 4070, 1, 8, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 11, 13, 0, 4065, 7, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 338, 0, 4083, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9110, 7786, 8505, 8234, 7099, 8201, 8148, 8453] avg=8192.0000 max=9110 max_rank=0 min=7099 min_rank=4 imbalance_ratio=1.112061 max_load_reduction=56426 max_load_reduction_pct=0.860992 moved_to_echo=31901 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 26, 534, 0, 0, 110, 3725, 961, 202, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 2502, 0, 0, 4095, 1706, 0, 0, 0, 0, 0, 0, 0, 4094, 2150, 0, 0, 0, 50, 0, 0, 367, 7, 0, 0, 0, 0, 0, 0, 0, 4091, 2616, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4, 2965, 0, 0, 0, 1830, 166, 0, 0, 0, 0, 0, 0, 399, 0, 40, 4096, 0, 0, 0, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 338, 0, 4046, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5558, 8329, 6668, 10813, 9052, 8465, 8156, 8495] avg=8192.0000 max=10813 max_rank=3 min=5558 min_rank=0 imbalance_ratio=1.319946 max_load_reduction=54723 max_load_reduction_pct=0.835007 moved_to_echo=34168 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[35, 0, 0, 4088, 0, 0, 136, 4063, 890, 203, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 342, 0, 0, 4096, 1713, 0, 0, 0, 0, 0, 0, 0, 4090, 4028, 0, 0, 0, 56, 8, 0, 340, 7, 0, 0, 0, 0, 0, 0, 20, 3638, 2424, 7, 26, 5, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1315, 1, 0, 0, 1837, 165, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4075, 0, 3, 0, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4073, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 11, 4096, 0, 3165, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9415, 6152, 8529, 10213, 7407, 8019, 8190, 7611] avg=8192.0000 max=10213 max_rank=3 min=6152 min_rank=1 imbalance_ratio=1.246704 max_load_reduction=55323 max_load_reduction_pct=0.844162 moved_to_echo=31227 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3753, 0, 0, 824, 4094, 987, 203, 0, 0, 0, 0, 0, 0, 1, 103, 0, 0, 3405, 0, 0, 4093, 1715, 0, 0, 0, 0, 0, 0, 0, 4092, 4095, 2, 1, 6, 0, 12, 4, 377, 7, 0, 0, 0, 0, 0, 0, 0, 455, 2686, 1, 0, 0, 3008, 4, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 4094, 0, 615, 1, 4, 6, 1839, 166, 0, 0, 0, 0, 0, 0, 49, 0, 0, 4086, 0, 0, 1, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 4082, 3, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 339, 0, 4089, 6, 4095, 15, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9861, 9317, 8596, 6154, 6813, 8062, 8181, 8552] avg=8192.0000 max=9861 max_rank=0 min=6154 min_rank=3 imbalance_ratio=1.203735 max_load_reduction=55675 max_load_reduction_pct=0.849533 moved_to_echo=31608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[27, 0, 0, 2397, 0, 69, 204, 4010, 990, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4076, 1665, 0, 0, 0, 0, 0, 0, 0, 4064, 2106, 19, 0, 0, 560, 0, 0, 380, 10, 0, 0, 0, 0, 0, 0, 0, 2, 2683, 0, 191, 5, 4086, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4080, 0, 2744, 6, 0, 0, 1783, 166, 0, 0, 0, 0, 0, 0, 365, 0, 0, 4095, 28, 16, 3, 0, 3928, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 4072, 0, 0, 81, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 3852, 2, 4026, 28, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7895, 9837, 7139, 6979, 8801, 8435, 8222, 8228] avg=8192.0000 max=9837 max_rank=1 min=6979 min_rank=3 imbalance_ratio=1.200806 max_load_reduction=55699 max_load_reduction_pct=0.849899 moved_to_echo=33686 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 1, 55, 4082, 998, 203, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 1222, 0, 1, 4096, 1715, 0, 0, 0, 0, 0, 0, 0, 4096, 3763, 0, 0, 0, 0, 0, 0, 381, 7, 0, 0, 0, 0, 0, 0, 0, 4080, 2717, 0, 0, 0, 3481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3825, 0, 0, 0, 1839, 166, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 47, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 4063, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5347, 7042, 8247, 10278, 9926, 8035, 8213, 8448] avg=8192.0000 max=10278 max_rank=3 min=5347 min_rank=0 imbalance_ratio=1.254639 max_load_reduction=55258 max_load_reduction_pct=0.843170 moved_to_echo=34622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4092, 0, 6, 0, 4085, 994, 203, 0, 0, 0, 0, 0, 0, 1, 37, 5, 0, 3996, 11, 1, 4085, 1715, 0, 0, 0, 0, 0, 0, 0, 4089, 4065, 0, 17, 0, 792, 0, 0, 380, 7, 0, 0, 0, 0, 0, 0, 0, 39, 2708, 0, 0, 35, 4083, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4065, 1, 179, 0, 0, 0, 1839, 164, 0, 0, 0, 0, 0, 0, 78, 0, 205, 4095, 0, 0, 0, 0, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4089, 0, 45, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 339, 54, 3252, 0, 3654, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9383, 9851, 9350, 6876, 6249, 8285, 8233, 7309] avg=8192.0000 max=9851 max_rank=1 min=6249 min_rank=4 imbalance_ratio=1.202515 max_load_reduction=55685 max_load_reduction_pct=0.849686 moved_to_echo=30076 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4, 4090, 0, 0, 22, 4036, 611, 203, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1350, 1, 55, 4096, 1714, 0, 0, 0, 0, 0, 0, 0, 4093, 3987, 0, 1, 0, 181, 0, 0, 233, 7, 0, 0, 0, 0, 0, 0, 0, 229, 1665, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 4096, 0, 0, 0, 1838, 165, 0, 0, 0, 0, 0, 0, 7, 0, 6, 4089, 1, 0, 0, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 4006, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8967, 7222, 8502, 5989, 10189, 8034, 8193, 8440] avg=8192.0000 max=10189 max_rank=4 min=5989 min_rank=3 imbalance_ratio=1.243774 max_load_reduction=55347 max_load_reduction_pct=0.844528 moved_to_echo=34856 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 16256, 20096, 128, 512, 128, 28672, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 22784, 128, 128, 1408, 32256, 7424, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 2816, 128, 31488, 128, 31232, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32640, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32640, 128, 16896, 128, 128, 128, 14720, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 28288, 128, 128, 128, 1664, 128, 128, 2944, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 256, 128, 128, 18816, 128, 256, 32768, 13696, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1152, 0, 384, 32768, 128, 128, 128, 0, 31488, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[9, 2, 0, 0, 3965, 4096, 0, 4096, 4082, 0, 0, 0, 4071, 0, 0, 0, 1, 4096, 0, 1118, 0, 7, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 2, 0, 1, 4096, 0, 0, 4040, 4096, 0, 0, 2429, 4, 4096, 0, 0, 4056, 4096, 4087, 4041, 0, 6, 0, 0, 0, 4085, 0, 76, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1, 4, 0, 0, 4080, 4094, 1, 4096, 4093, 0, 0, 2, 4093, 0, 24, 222, 0, 4063, 0, 0, 0, 21, 0, 0, 2, 450, 0, 252, 3, 0, 19, 1, 1, 1, 0, 7, 0, 4, 0, 4096, 9, 1, 4081, 4094, 0, 0, 4072, 0, 4092, 0, 0, 3206, 4075, 4088, 4034, 0, 59, 0, 0, 0, 4092, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[30, 34, 0, 0, 3995, 4080, 0, 4071, 4096, 16, 39, 0, 4041, 0, 0, 73, 41, 3902, 0, 0, 2, 48, 0, 0, 0, 2405, 0, 294, 4, 0, 27, 0, 2, 0, 0, 0, 0, 0, 0, 4081, 26, 0, 4060, 4094, 0, 809, 673, 7, 4063, 5, 24, 3616, 4068, 4080, 4096, 20, 0, 529, 0, 4, 4067, 4, 7, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[52, 0, 6, 192, 4094, 4088, 0, 4087, 4089, 2, 4, 3, 4081, 0, 29, 0, 7, 4094, 0, 0, 1, 0, 8, 0, 0, 2375, 0, 14, 1, 0, 0, 0, 0, 220, 1, 2, 1, 4, 0, 3725, 0, 0, 4050, 4091, 0, 17, 1713, 43, 4092, 0, 1, 4096, 4091, 4094, 3973, 0, 0, 0, 9, 0, 4063, 0, 0, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 18, 29, 22, 4059, 4073, 0, 4095, 4071, 0, 0, 8, 4028, 0, 0, 0, 0, 4074, 0, 6, 0, 0, 2, 0, 8, 58, 0, 999, 14, 0, 6, 0, 0, 322, 14, 0, 0, 7, 0, 4069, 0, 0, 4096, 4096, 0, 0, 3845, 68, 4085, 11, 0, 2995, 4088, 4080, 4082, 0, 0, 2, 11, 0, 4090, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3, 0, 0, 13, 4076, 4096, 0, 4096, 4096, 0, 0, 0, 4066, 0, 7, 0, 0, 4096, 0, 2, 0, 4, 0, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 3928, 0, 259, 4094, 4096, 0, 0, 3488, 1, 4096, 0, 0, 4084, 4096, 4096, 4095, 0, 0, 7, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4085, 4096, 0, 4095, 4094, 0, 0, 0, 4094, 0, 0, 1, 0, 4096, 0, 0, 12, 0, 0, 0, 3, 3, 0, 2, 0, 10, 0, 2, 0, 28, 0, 0, 0, 0, 0, 4091, 3, 0, 4096, 4095, 0, 51, 4083, 12, 4094, 0, 0, 4004, 4085, 4096, 4086, 0, 0, 3, 22, 0, 4092, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[98, 0, 0, 0, 4095, 4096, 0, 4084, 4084, 0, 16, 0, 4008, 40, 685, 0, 19, 4029, 0, 0, 7, 0, 0, 0, 0, 98, 0, 740, 0, 3, 0, 0, 93, 7, 0, 0, 0, 0, 18, 4079, 0, 0, 3986, 4096, 12, 11, 4094, 0, 4084, 12, 0, 2464, 4096, 4050, 4096, 0, 0, 0, 52, 120, 4064, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[30, 34, 0, 0, 3995, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 16, 39, 0, 4041, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 41, 3902, 0, 0, 2, 48, 0, 0, 3181, 668, 102, 18, 0, 0, 0, 0, 0, 2405, 0, 294, 4, 0, 27, 0, 4071, 2968, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4081, 2940, 1095, 0, 0, 0, 0, 0, 0, 26, 0, 4060, 913, 0, 809, 673, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 24, 3616, 0, 556, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 0, 4, 4067, 4, 7, 3, 2856, 1128, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8163, 7962, 9769, 8118, 6488, 8317, 8598] avg=8192.0000 max=9769 max_rank=3 min=6488 min_rank=5 imbalance_ratio=1.192505 max_load_reduction=55767 max_load_reduction_pct=0.850937 moved_to_echo=31521 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[9, 2, 0, 0, 3965, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 1118, 0, 7, 0, 0, 3182, 669, 102, 18, 0, 0, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 4096, 2993, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 2, 0, 1, 4096, 2961, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 914, 0, 0, 2429, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 557, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4085, 0, 76, 0, 2861, 1135, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8054, 8051, 9193, 7706, 8328, 7387, 8654, 8163] avg=8192.0000 max=9193 max_rank=2 min=7387 min_rank=5 imbalance_ratio=1.122192 max_load_reduction=56343 max_load_reduction_pct=0.859726 moved_to_echo=32532 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 0, 0, 4080, 4076, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 2, 4093, 0, 24, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 21, 0, 0, 3181, 670, 102, 18, 0, 0, 0, 0, 2, 450, 0, 252, 3, 0, 19, 1, 4096, 2990, 0, 0, 0, 0, 0, 0, 1, 1, 0, 7, 0, 4, 0, 4096, 2945, 1102, 0, 0, 0, 0, 0, 0, 9, 1, 4081, 913, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 0, 556, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 4092, 3, 0, 0, 2862, 1130, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8332, 8055, 7813, 8156, 9076, 7796, 8146] avg=8192.0000 max=9076 max_rank=5 min=7796 min_rank=6 imbalance_ratio=1.107910 max_load_reduction=56460 max_load_reduction_pct=0.861511 moved_to_echo=33174 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[52, 0, 6, 192, 4094, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 2, 4, 3, 4081, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4094, 0, 0, 1, 0, 8, 0, 3178, 671, 102, 18, 0, 0, 0, 0, 0, 2375, 0, 14, 1, 0, 0, 0, 4087, 2990, 0, 0, 0, 0, 0, 0, 0, 220, 1, 2, 1, 4, 0, 3725, 2956, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 913, 0, 17, 1713, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 557, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4063, 0, 0, 23, 2866, 1135, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8414, 8106, 8079, 9467, 8011, 6736, 8627, 8096] avg=8192.0000 max=9467 max_rank=3 min=6736 min_rank=5 imbalance_ratio=1.155640 max_load_reduction=56069 max_load_reduction_pct=0.855545 moved_to_echo=31470 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 13, 4076, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 4066, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4, 0, 0, 3182, 671, 102, 18, 0, 0, 0, 0, 0, 511, 0, 0, 0, 0, 0, 0, 4096, 2993, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 3928, 2961, 1103, 0, 0, 0, 0, 0, 0, 0, 259, 4094, 914, 0, 0, 3488, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 557, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 2868, 1135, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8067, 8075, 7600, 8026, 8756, 8736, 8106] avg=8192.0000 max=8756 max_rank=5 min=7600 min_rank=3 imbalance_ratio=1.068848 max_load_reduction=56780 max_load_reduction_pct=0.866394 moved_to_echo=33624 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[98, 0, 0, 0, 4095, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 16, 0, 4008, 40, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4029, 0, 0, 7, 0, 0, 0, 3187, 667, 108, 22, 0, 0, 0, 0, 0, 98, 0, 740, 0, 3, 0, 0, 4084, 2985, 0, 0, 0, 0, 0, 0, 93, 7, 0, 0, 0, 0, 18, 4079, 2961, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 909, 12, 11, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2464, 0, 543, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 120, 4064, 0, 0, 0, 2840, 1135, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8267, 8725, 8039, 7910, 8257, 9012, 7115, 8211] avg=8192.0000 max=9012 max_rank=5 min=7115 min_rank=6 imbalance_ratio=1.100098 max_load_reduction=56524 max_load_reduction_pct=0.862488 moved_to_echo=32595 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 29, 22, 4059, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 0, 0, 8, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 6, 0, 0, 2, 0, 3182, 668, 102, 18, 0, 0, 0, 0, 8, 58, 0, 999, 14, 0, 6, 0, 4095, 2985, 0, 0, 0, 0, 0, 0, 0, 322, 14, 0, 0, 7, 0, 4069, 2954, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 914, 0, 0, 3845, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 2995, 0, 556, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 11, 0, 4090, 0, 5, 0, 2856, 1134, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8005, 8052, 8165, 8466, 8923, 7644, 8098] avg=8192.0000 max=8923 max_rank=5 min=7644 min_rank=6 imbalance_ratio=1.089233 max_load_reduction=56613 max_load_reduction_pct=0.863846 moved_to_echo=33131 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4085, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 3182, 671, 102, 18, 0, 0, 0, 0, 3, 3, 0, 2, 0, 10, 0, 2, 4095, 2991, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 4091, 2952, 1103, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 913, 0, 51, 4083, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 557, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 22, 0, 4092, 0, 0, 0, 2868, 1133, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8087, 8081, 7106, 8174, 9158, 8647, 8118] avg=8192.0000 max=9158 max_rank=5 min=7106 min_rank=3 imbalance_ratio=1.117920 max_load_reduction=56378 max_load_reduction_pct=0.860260 moved_to_echo=34097 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32000, 128, 128, 128, 32512, 128, 768, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 384, 32512, 7424, 128, 896, 24448, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 640, 128, 128, 32768, 128, 128, 128, 22912, 9088, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 128, 128, 256, 32512, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 896, 128, 128, 128, 128, 128, 32256, 23680, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 6528, 0, 2304, 128, 128, 128, 128, 32768, 23936, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32512, 0, 1152, 128, 128, 128, 0, 25472, 5376, 896, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 28544, 0, 4480, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[3609, 3387, 4048, 19, 2, 0, 15, 25, 0, 0, 0, 930, 4062, 4058, 4055, 0, 5, 0, 376, 4082, 0, 238, 0, 7, 0, 0, 15, 0, 6, 0, 0, 3895, 24, 4023, 46, 1342, 0, 34, 0, 0, 3788, 0, 0, 0, 27, 3318, 28, 298, 3674, 0, 0, 3642, 4079, 9, 0, 4068, 16, 0, 21, 3, 0, 78, 4094, 90] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4076, 160, 4044, 1, 5, 0, 0, 0, 0, 1, 0, 18, 4092, 4094, 4092, 53, 1686, 0, 27, 4096, 17, 575, 25, 2573, 6, 0, 4, 0, 0, 0, 1, 4091, 0, 4079, 0, 3101, 2, 0, 1, 0, 4043, 0, 15, 4, 0, 484, 0, 0, 3775, 17, 3, 4096, 4096, 0, 0, 3983, 6, 0, 0, 0, 0, 0, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[3943, 0, 4096, 1, 0, 0, 7, 12, 125, 0, 0, 0, 4096, 4095, 4093, 0, 1524, 2, 29, 4094, 0, 0, 0, 1168, 15, 0, 0, 1, 0, 1183, 0, 3758, 1, 4094, 0, 4028, 1, 0, 2, 5, 4081, 0, 0, 0, 1, 67, 0, 0, 4096, 0, 0, 3966, 4092, 2, 1, 4095, 0, 294, 2, 0, 1, 382, 4083, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4094, 90, 4096, 0, 0, 0, 2, 0, 12, 0, 0, 279, 3883, 4096, 4086, 0, 292, 0, 0, 4096, 0, 143, 128, 3448, 20, 0, 0, 0, 0, 27, 0, 4088, 0, 4005, 0, 3633, 24, 1, 0, 0, 3942, 0, 0, 21, 0, 384, 1, 7, 2566, 0, 0, 3644, 3974, 0, 0, 4096, 2, 642, 0, 0, 0, 1409, 4096, 209] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[3716, 0, 4096, 0, 0, 0, 0, 0, 523, 0, 0, 60, 4016, 4096, 4096, 0, 3558, 0, 0, 4096, 0, 0, 2, 448, 0, 0, 0, 0, 0, 7, 0, 3945, 0, 4096, 0, 4096, 127, 0, 0, 0, 4095, 0, 0, 0, 0, 63, 0, 3, 4014, 0, 0, 4096, 4063, 0, 0, 4096, 0, 32, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[3856, 11, 4084, 0, 0, 0, 0, 0, 0, 11, 0, 2904, 4096, 4091, 4096, 0, 0, 0, 1072, 4087, 7, 3, 5, 118, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 4081, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 88, 199, 0, 4027, 3, 2, 4096, 4094, 1, 0, 4096, 0, 7, 0, 0, 0, 0, 4094, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[2017, 468, 4001, 0, 0, 31, 0, 0, 4, 0, 0, 363, 4091, 4095, 4088, 26, 3744, 6, 811, 4084, 0, 0, 1, 891, 19, 0, 13, 14, 0, 66, 0, 4082, 0, 4059, 0, 3376, 0, 1, 0, 12, 3800, 2, 0, 0, 4, 741, 0, 161, 4094, 77, 0, 4036, 4096, 0, 0, 4087, 0, 0, 0, 7, 0, 0, 4068, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[75, 4027, 4067, 7, 26, 0, 0, 23, 28, 0, 0, 200, 4096, 4074, 4089, 0, 3, 32, 2241, 4092, 0, 0, 0, 333, 7, 0, 4, 1, 4, 30, 38, 4094, 9, 4064, 0, 4081, 6, 0, 0, 0, 4062, 0, 0, 0, 0, 1276, 0, 0, 4096, 0, 0, 4087, 4079, 0, 0, 4085, 0, 0, 10, 0, 13, 2, 4066, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3609, 3387, 3961, 19, 2, 0, 15, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 4062, 0, 3420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 376, 4082, 0, 238, 0, 7, 804, 104, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 6, 0, 0, 3895, 3972, 0, 0, 0, 0, 0, 0, 0, 24, 4023, 46, 1342, 0, 34, 0, 0, 531, 87, 0, 0, 0, 0, 0, 0, 3788, 0, 0, 0, 27, 3318, 28, 298, 2846, 462, 0, 0, 0, 0, 0, 0, 3674, 0, 0, 3642, 429, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 21, 3, 0, 78, 4094, 90, 3606, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11018, 8412, 5616, 7888, 6087, 10767, 7754, 7994] avg=8192.0000 max=11018 max_rank=0 min=5616 min_rank=2 imbalance_ratio=1.344971 max_load_reduction=54518 max_load_reduction_pct=0.831879 moved_to_echo=32602 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4094, 90, 4008, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 279, 3883, 0, 3446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 0, 4096, 0, 143, 128, 3448, 784, 105, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 27, 0, 4088, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 3633, 24, 1, 0, 0, 535, 88, 0, 0, 0, 0, 0, 0, 3942, 0, 0, 21, 0, 384, 1, 7, 2773, 466, 0, 0, 0, 0, 0, 0, 2566, 0, 0, 3644, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 642, 0, 0, 0, 1409, 4096, 209, 3630, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7620, 8996, 8145, 8286, 7594, 6627, 10074] avg=8192.0000 max=10074 max_rank=7 min=6627 min_rank=6 imbalance_ratio=1.229736 max_load_reduction=55462 max_load_reduction_pct=0.846283 moved_to_echo=32581 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4076, 160, 3957, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 4092, 0, 3451, 53, 0, 0, 0, 0, 0, 0, 0, 0, 1686, 0, 27, 4096, 17, 575, 25, 2573, 808, 105, 0, 0, 0, 0, 0, 0, 6, 0, 4, 0, 0, 0, 1, 4091, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 3101, 2, 0, 1, 0, 536, 87, 0, 0, 0, 0, 0, 0, 4043, 0, 15, 4, 0, 484, 0, 0, 2858, 453, 0, 0, 0, 0, 0, 0, 3775, 17, 3, 4096, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4094, 0, 3530, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 7615, 9912, 8109, 7806, 7857, 8321, 7717] avg=8192.0000 max=9912 max_rank=2 min=7615 min_rank=1 imbalance_ratio=1.209961 max_load_reduction=55624 max_load_reduction_pct=0.848755 moved_to_echo=31701 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3943, 0, 4008, 1, 0, 0, 7, 12, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 4096, 0, 3452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1524, 2, 29, 4094, 0, 0, 0, 1168, 807, 105, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1, 0, 1183, 0, 3758, 4009, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 4028, 1, 0, 2, 5, 536, 88, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 1, 67, 0, 0, 2855, 466, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3966, 430, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 2, 0, 1, 382, 4083, 0, 3629, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7971, 7673, 7729, 8966, 8755, 7470, 8495, 8477] avg=8192.0000 max=8966 max_rank=3 min=7470 min_rank=5 imbalance_ratio=1.094482 max_load_reduction=56570 max_load_reduction_pct=0.863190 moved_to_echo=33197 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3856, 11, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 2904, 4096, 0, 3455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 4087, 7, 3, 5, 118, 807, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4096, 0, 0, 0, 0, 536, 88, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 88, 199, 0, 2856, 465, 0, 0, 0, 0, 0, 0, 4027, 3, 2, 4096, 431, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4094, 22, 3631, 87, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7863, 10466, 6204, 8097, 8801, 7704, 8560, 7841] avg=8192.0000 max=10466 max_rank=1 min=6204 min_rank=2 imbalance_ratio=1.277588 max_load_reduction=55070 max_load_reduction_pct=0.840302 moved_to_echo=32906 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[75, 4027, 3980, 7, 26, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 200, 4096, 0, 3449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 32, 2241, 4092, 0, 0, 0, 333, 804, 105, 0, 0, 0, 0, 0, 0, 7, 0, 4, 1, 4, 30, 38, 4094, 3988, 0, 0, 0, 0, 0, 0, 0, 9, 4064, 0, 4081, 6, 0, 0, 0, 535, 87, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 1276, 0, 0, 2846, 465, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 13, 2, 4066, 9, 3620, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 7773, 7610, 8166, 8782, 8649, 8612, 7806] avg=8192.0000 max=8782 max_rank=4 min=7610 min_rank=2 imbalance_ratio=1.072021 max_load_reduction=56754 max_load_reduction_pct=0.865997 moved_to_echo=33849 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2017, 468, 3913, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 363, 4091, 0, 3439, 26, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 6, 811, 4084, 0, 0, 1, 891, 813, 109, 0, 0, 0, 0, 0, 0, 19, 0, 13, 14, 0, 66, 0, 4082, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 3376, 0, 1, 0, 12, 540, 88, 0, 0, 0, 0, 0, 0, 3800, 2, 0, 0, 4, 741, 0, 161, 2860, 465, 0, 0, 0, 0, 0, 0, 4094, 77, 0, 4036, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4068, 0, 3622, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6429, 7923, 10459, 8203, 8076, 8033, 8630, 7783] avg=8192.0000 max=10459 max_rank=2 min=6429 min_rank=0 imbalance_ratio=1.276733 max_load_reduction=55077 max_load_reduction_pct=0.840408 moved_to_echo=32522 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3716, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 0, 0, 60, 4016, 0, 3455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 0, 0, 4096, 0, 0, 2, 448, 801, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3945, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 127, 0, 0, 0, 536, 88, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 63, 0, 3, 2835, 466, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 4096, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 4096, 0, 3630, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7724, 8054, 9010, 7962, 8943, 7462, 8537, 7844] avg=8192.0000 max=9010 max_rank=2 min=7462 min_rank=5 imbalance_ratio=1.099854 max_load_reduction=56526 max_load_reduction_pct=0.862518 moved_to_echo=32786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32512, 128, 27776, 256, 128, 128, 128, 4352, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[25472, 8192, 31872, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[768, 128, 0, 4864, 32512, 0, 27648, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[30464, 128, 128, 31744, 3456, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32000, 128, 128, 128, 128, 6528, 256, 512, 22784, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[10880, 128, 4608, 32768, 128, 1024, 256, 9088, 6528, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 1024, 128, 128, 128, 1920, 32768, 384, 28928, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 1408, 128, 32128, 32128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[22, 4, 346, 0, 336, 3836, 0, 0, 0, 68, 4096, 57, 4076, 0, 32, 0, 0, 2, 0, 0, 1, 3993, 13, 0, 3994, 0, 0, 3, 1, 49, 11, 4065, 0, 0, 0, 66, 0, 4096, 4019, 93, 4036, 2, 0, 0, 15, 0, 4037, 4084, 68, 31, 4073, 4073, 0, 27, 3616, 8, 2, 0, 0, 4083, 0, 0, 4083, 19] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 3857, 4096, 0, 0, 3, 16, 4096, 0, 0, 0, 0, 11, 0, 2, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 4091, 4095, 240, 4096, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4070, 4096, 0, 0, 4096, 0, 0, 0, 0, 4011, 0, 86, 4090, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 2274, 3103, 0, 0, 0, 587, 4071, 0, 2849, 0, 0, 9, 0, 17, 1, 0, 0, 4087, 0, 10, 3824, 0, 0, 0, 0, 0, 0, 4062, 2, 0, 730, 0, 0, 4096, 4096, 76, 4096, 0, 0, 0, 0, 0, 2841, 4096, 0, 0, 4092, 4093, 0, 0, 4096, 0, 0, 0, 0, 4063, 0, 167, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4, 4, 0, 0, 4092, 4077, 0, 0, 10, 504, 4092, 4, 1632, 15, 17, 0, 0, 0, 0, 0, 2, 4061, 6, 1, 4096, 0, 0, 3, 1, 20, 0, 4087, 1, 2, 159, 0, 0, 3793, 4091, 112, 4041, 32, 6, 0, 0, 0, 3641, 4090, 0, 0, 4094, 4090, 6, 0, 3004, 1, 0, 5, 46, 3478, 0, 82, 4034, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 4095, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 4096, 43, 0, 0, 0, 5, 0, 4096, 0, 0, 71, 0, 0, 3951, 4096, 28, 4096, 0, 0, 0, 0, 0, 2755, 4096, 0, 0, 4096, 4096, 0, 0, 1749, 0, 0, 0, 1, 3722, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 4088, 3153, 0, 3, 9, 0, 4087, 44, 141, 5, 1, 0, 14, 0, 0, 0, 16, 4094, 0, 9, 4081, 0, 2, 0, 0, 725, 0, 4084, 0, 0, 0, 2, 0, 4095, 4092, 0, 4049, 0, 2, 9, 33, 0, 4002, 4094, 4, 0, 4081, 4028, 4, 0, 4082, 2, 0, 0, 0, 4095, 3, 199, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 94, 0, 4056, 3550, 0, 0, 0, 4, 4096, 0, 1119, 2, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 20, 4096, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 5, 0, 4094, 4085, 0, 4096, 0, 0, 0, 0, 0, 4076, 4095, 2, 0, 4096, 4095, 0, 0, 4096, 1, 12, 0, 0, 3541, 0, 18, 4087, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3, 15, 16, 0, 393, 3185, 3, 0, 0, 0, 4084, 0, 3958, 0, 14, 0, 10, 0, 0, 0, 0, 4083, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 4096, 0, 5, 868, 0, 8, 4085, 4040, 0, 4046, 0, 22, 0, 12, 0, 3925, 4093, 0, 11, 4080, 4091, 0, 0, 4081, 7, 0, 13, 6, 4096, 0, 0, 4079, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[22, 4, 346, 0, 336, 3836, 0, 0, 1446, 138, 0, 0, 0, 0, 0, 0, 0, 68, 4096, 57, 4076, 0, 32, 0, 1686, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 3993, 13, 0, 3650, 456, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 3, 1, 49, 11, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 4096, 3792, 93, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 2, 0, 0, 15, 0, 4037, 434, 0, 0, 0, 0, 0, 0, 0, 0, 68, 31, 485, 4073, 0, 27, 3616, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4083, 0, 0, 4083, 19, 89, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 10015, 8115, 8076, 8047, 8524, 8308, 8323] avg=8192.0000 max=10015 max_rank=1 min=6128 min_rank=0 imbalance_ratio=1.222534 max_load_reduction=55521 max_load_reduction_pct=0.847183 moved_to_echo=33202 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 3857, 4096, 0, 0, 1444, 141, 0, 0, 0, 0, 0, 0, 3, 16, 4096, 0, 0, 0, 0, 11, 1685, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 3659, 455, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4091, 3863, 240, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 86, 4090, 0, 91, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9543, 5811, 8211, 8144, 8196, 8627, 8678, 8326] avg=8192.0000 max=9543 max_rank=0 min=5811 min_rank=1 imbalance_ratio=1.164917 max_load_reduction=55993 max_load_reduction_pct=0.854385 moved_to_echo=33827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 2274, 3103, 0, 0, 1452, 141, 0, 0, 0, 0, 0, 0, 0, 587, 4071, 0, 2849, 0, 0, 9, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 4087, 0, 10, 3661, 458, 0, 0, 0, 0, 0, 0, 3824, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 730, 0, 0, 4096, 3864, 76, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2841, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 167, 4096, 0, 91, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6972, 9210, 8234, 7839, 8768, 7372, 8677, 8464] avg=8192.0000 max=9210 max_rank=1 min=6972 min_rank=0 imbalance_ratio=1.124268 max_load_reduction=56326 max_load_reduction_pct=0.859467 moved_to_echo=33281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4, 4, 0, 0, 4092, 4077, 0, 0, 1453, 140, 0, 0, 0, 0, 0, 0, 10, 504, 4092, 4, 1632, 15, 17, 0, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4061, 6, 1, 3656, 458, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 1, 20, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 159, 0, 0, 3793, 3860, 112, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 32, 6, 0, 0, 0, 3641, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 4090, 6, 0, 3004, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 46, 3478, 0, 82, 4034, 0, 91, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9770, 7969, 8184, 8159, 7927, 8154, 7589, 7784] avg=8192.0000 max=9770 max_rank=0 min=7589 min_rank=6 imbalance_ratio=1.192627 max_load_reduction=55766 max_load_reduction_pct=0.850922 moved_to_echo=31454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 4095, 0, 0, 1454, 141, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 3661, 458, 0, 0, 0, 0, 0, 0, 4096, 43, 0, 0, 0, 5, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 3951, 3864, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2755, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 4096, 0, 0, 1749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3722, 0, 0, 4096, 0, 91, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9779, 9887, 8187, 8192, 7914, 7286, 6333, 7958] avg=8192.0000 max=9887 max_rank=1 min=6333 min_rank=6 imbalance_ratio=1.206909 max_load_reduction=55649 max_load_reduction_pct=0.849136 moved_to_echo=29491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 15, 16, 0, 393, 3185, 3, 0, 1448, 139, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 3958, 0, 14, 0, 1689, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4083, 0, 0, 3658, 457, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 868, 0, 8, 4085, 3812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 22, 0, 12, 0, 3925, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 486, 4091, 0, 0, 4081, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6, 4096, 0, 0, 4079, 16, 89, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5202, 9745, 8208, 8140, 8778, 8440, 8676, 8347] avg=8192.0000 max=9745 max_rank=1 min=5202 min_rank=0 imbalance_ratio=1.189575 max_load_reduction=55791 max_load_reduction_pct=0.851303 moved_to_echo=34241 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 4088, 3153, 0, 3, 1452, 144, 0, 0, 0, 0, 0, 0, 9, 0, 4087, 44, 141, 5, 1, 0, 1695, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 16, 4094, 0, 9, 3663, 461, 0, 0, 0, 0, 0, 0, 4081, 0, 2, 0, 0, 725, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 2, 9, 33, 0, 4002, 431, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 473, 4028, 4, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 199, 4096, 0, 94, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8848, 5982, 8257, 8841, 7951, 8526, 8593, 8538] avg=8192.0000 max=8848 max_rank=0 min=5982 min_rank=1 imbalance_ratio=1.080078 max_load_reduction=56688 max_load_reduction_pct=0.864990 moved_to_echo=33608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 94, 0, 4056, 3550, 0, 0, 1454, 140, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 1119, 2, 0, 0, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 20, 3660, 458, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 4094, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4076, 435, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 488, 4095, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3541, 0, 18, 4087, 0, 90, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9294, 6917, 8140, 8145, 7955, 8607, 8682, 7796] avg=8192.0000 max=9294 max_rank=0 min=6917 min_rank=1 imbalance_ratio=1.134521 max_load_reduction=56242 max_load_reduction_pct=0.858185 moved_to_echo=33040 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 128, 128, 0, 29440, 3584, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 1280, 32768, 128, 17920, 128, 128, 128, 13568, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 31104, 128, 640, 32768, 128, 768, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 512, 128, 23296, 29184, 128, 128, 11648, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32384, 128, 128, 128, 128, 896, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 32512, 128, 128, 29312, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 1920, 128, 128, 32384, 30848, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 3968, 32768, 128, 128, 28928, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4083, 0, 822, 0, 4072, 0, 18, 706, 0, 0, 6, 3707, 22, 0, 0, 10, 0, 12, 4087, 0, 0, 4048, 12, 4077, 4058, 1, 0, 0, 4, 4075, 0, 4076, 167, 3969, 4004, 0, 33, 0, 4031, 0, 3388, 14, 20, 3592, 2, 0, 3, 25, 0, 10, 0, 106, 4062, 0, 84, 5, 30, 4010, 0, 1, 0, 17, 64, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 10, 879, 3, 4084, 0, 4077, 312, 2, 0, 0, 3332, 0, 0, 0, 0, 0, 0, 4096, 40, 0, 4086, 17, 4034, 4010, 0, 15, 0, 329, 4096, 0, 4096, 0, 92, 4096, 0, 0, 0, 4091, 0, 4096, 0, 0, 3261, 0, 0, 0, 0, 0, 0, 0, 160, 4075, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4077, 0, 1419, 1, 4080, 12, 4076, 9, 0, 0, 0, 3979, 0, 0, 28, 3, 0, 0, 4094, 314, 2, 4094, 13, 3753, 4047, 0, 2, 2, 0, 4096, 0, 4094, 339, 603, 4089, 0, 0, 103, 4091, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 1772, 4096, 0, 90, 0, 8, 4004, 2, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4089, 3, 3, 0, 4095, 26, 4085, 2, 1, 3, 9, 4090, 0, 0, 0, 0, 0, 2, 3777, 11, 2, 4088, 1, 4054, 4089, 0, 0, 0, 666, 4082, 0, 4094, 362, 3686, 3373, 17, 0, 1330, 4079, 44, 3148, 126, 4, 0, 1, 0, 0, 0, 0, 3, 46, 0, 3915, 35, 0, 0, 0, 4085, 0, 0, 0, 10, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4094, 0, 2650, 0, 4065, 7, 878, 7, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 4087, 0, 0, 4096, 0, 3959, 4090, 0, 5, 0, 0, 4096, 0, 4096, 233, 492, 4096, 0, 0, 0, 4092, 0, 4096, 0, 0, 3713, 0, 2, 0, 0, 0, 0, 0, 390, 4091, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 349, 108, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 4096, 0, 3707, 4096, 0, 4, 0, 127, 4096, 0, 4096, 0, 3771, 4096, 0, 65, 0, 3975, 0, 3177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4081, 0, 2872, 0, 4081, 0, 3636, 0, 0, 0, 15, 3985, 0, 33, 0, 0, 0, 1, 4064, 13, 33, 4082, 0, 4064, 4082, 16, 2, 14, 370, 4076, 12, 4080, 7, 734, 4074, 8, 49, 3, 4068, 1, 4044, 20, 5, 281, 11, 0, 0, 12, 1, 0, 0, 478, 4044, 1, 0, 0, 0, 4066, 0, 14, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4062, 0, 4083, 0, 3977, 37, 362, 9, 0, 0, 2, 4069, 18, 0, 43, 0, 5, 0, 3970, 0, 0, 4086, 0, 3689, 4084, 2, 0, 12, 0, 4093, 8, 4073, 315, 342, 4091, 0, 0, 10, 4096, 0, 4093, 0, 9, 2226, 20, 0, 0, 0, 13, 0, 0, 1265, 4096, 21, 114, 0, 0, 4096, 0, 24, 21, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 822, 0, 3751, 0, 18, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3707, 22, 0, 0, 10, 4075, 159, 0, 0, 0, 0, 0, 0, 0, 12, 4087, 0, 0, 179, 12, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 1, 0, 0, 4, 0, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 167, 3969, 4004, 0, 33, 0, 2088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3388, 14, 20, 3592, 2, 0, 3, 25, 1943, 321, 308, 164, 0, 0, 0, 0, 0, 10, 0, 106, 4062, 0, 84, 5, 3385, 0, 0, 0, 0, 0, 0, 0, 30, 4010, 0, 1, 0, 17, 64, 3, 3924, 176, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5297, 7979, 8367, 7975, 10261, 9780, 7652, 8225] avg=8192.0000 max=10261 max_rank=4 min=5297 min_rank=0 imbalance_ratio=1.252563 max_load_reduction=55275 max_load_reduction_pct=0.843430 moved_to_echo=35918 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 879, 3, 3762, 0, 4077, 312, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3332, 0, 0, 0, 0, 4096, 161, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 40, 0, 181, 17, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 15, 0, 329, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 4096, 0, 0, 0, 2119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3261, 0, 0, 0, 0, 1972, 322, 311, 164, 0, 0, 0, 0, 0, 0, 0, 160, 4075, 0, 0, 0, 3417, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 6, 0, 3935, 177, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9043, 7591, 8368, 8286, 6307, 10126, 7652, 8163] avg=8192.0000 max=10126 max_rank=5 min=6307 min_rank=4 imbalance_ratio=1.236084 max_load_reduction=55410 max_load_reduction_pct=0.845490 moved_to_echo=32248 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1419, 1, 3758, 12, 4076, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 28, 3, 4096, 159, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 314, 2, 182, 13, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 2, 2, 0, 0, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 339, 603, 4089, 0, 0, 103, 2119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 1972, 322, 311, 164, 0, 0, 0, 0, 49, 0, 0, 1772, 4096, 0, 90, 0, 3423, 0, 0, 0, 0, 0, 0, 0, 8, 4004, 2, 0, 0, 0, 6, 0, 3918, 178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9275, 8265, 8358, 7981, 7253, 6858, 9430, 8116] avg=8192.0000 max=9430 max_rank=6 min=6858 min_rank=5 imbalance_ratio=1.151123 max_load_reduction=56106 max_load_reduction_pct=0.856110 moved_to_echo=31657 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3, 0, 3772, 26, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 9, 4090, 0, 0, 0, 0, 4082, 161, 0, 0, 0, 0, 0, 0, 0, 2, 3777, 11, 2, 182, 1, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 666, 0, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 362, 3686, 3373, 17, 0, 1330, 2113, 44, 0, 0, 0, 0, 0, 0, 0, 0, 3148, 126, 4, 0, 1, 0, 0, 0, 1966, 323, 311, 164, 0, 0, 0, 0, 0, 3, 46, 0, 3915, 35, 0, 0, 3418, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 10, 0, 0, 3928, 177, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7891, 8346, 8029, 8685, 10925, 6043, 7417, 8200] avg=8192.0000 max=10925 max_rank=4 min=6043 min_rank=5 imbalance_ratio=1.333618 max_load_reduction=54611 max_load_reduction_pct=0.833298 moved_to_echo=32585 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2872, 0, 3759, 0, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3985, 0, 33, 0, 0, 4076, 159, 0, 0, 0, 0, 0, 0, 0, 1, 4064, 13, 33, 181, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 16, 2, 14, 370, 0, 12, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 7, 734, 4074, 8, 49, 3, 2107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 20, 5, 281, 11, 0, 0, 12, 1961, 322, 311, 164, 0, 0, 0, 0, 1, 0, 0, 478, 4044, 1, 0, 0, 3413, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 14, 0, 0, 3, 0, 3922, 177, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10267, 8268, 8356, 8412, 6983, 7131, 7937, 8182] avg=8192.0000 max=10267 max_rank=0 min=6983 min_rank=4 imbalance_ratio=1.253296 max_load_reduction=55269 max_load_reduction_pct=0.843338 moved_to_echo=30233 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 349, 108, 3773, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 161, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 181, 0, 3707, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 127, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3771, 4096, 0, 65, 0, 2059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3177, 0, 0, 0, 0, 0, 0, 0, 1916, 323, 312, 164, 0, 0, 0, 0, 0, 0, 0, 1100, 4087, 0, 0, 0, 3425, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3935, 178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8326, 8353, 7994, 8159, 9991, 5892, 8612, 8209] avg=8192.0000 max=9991 max_rank=4 min=5892 min_rank=5 imbalance_ratio=1.219604 max_load_reduction=55545 max_load_reduction_pct=0.847549 moved_to_echo=32704 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2650, 0, 3744, 7, 878, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 160, 0, 0, 0, 0, 0, 0, 2, 0, 4087, 0, 0, 181, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 5, 0, 0, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 233, 492, 4096, 0, 0, 0, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3713, 0, 2, 0, 0, 1972, 321, 312, 164, 0, 0, 0, 0, 0, 0, 0, 390, 4091, 0, 0, 0, 3425, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4, 4, 3934, 178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7286, 8350, 8229, 8027, 6941, 10580, 7906, 8217] avg=8192.0000 max=10580 max_rank=5 min=6941 min_rank=4 imbalance_ratio=1.291504 max_load_reduction=54956 max_load_reduction_pct=0.838562 moved_to_echo=33644 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4083, 0, 3660, 37, 362, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4069, 18, 0, 43, 0, 4093, 159, 0, 0, 0, 0, 0, 0, 5, 0, 3970, 0, 0, 171, 0, 3689, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 2, 0, 12, 0, 0, 8, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 315, 342, 4091, 0, 0, 10, 2117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 9, 2226, 20, 0, 0, 0, 1979, 317, 314, 168, 0, 0, 0, 0, 13, 0, 0, 1265, 4096, 21, 114, 0, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 21, 0, 0, 0, 3903, 180, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8384, 7835, 8011, 6875, 9126, 8930, 8224] avg=8192.0000 max=9126 max_rank=5 min=6875 min_rank=4 imbalance_ratio=1.114014 max_load_reduction=56410 max_load_reduction_pct=0.860748 moved_to_echo=33155 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32512, 128, 128, 128, 128, 128, 128, 31488, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 13184, 128, 30080, 128, 21248, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 128, 1536, 0, 128, 31488, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1536, 13696, 32000, 128, 256, 1536, 16896, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32384, 512, 128, 1536, 128, 31360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 31360, 128, 128, 128, 128, 32768, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[30208, 256, 128, 13184, 128, 128, 128, 128, 15744, 2688, 2560, 1408, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 5376, 32512, 128, 384, 128, 27392, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4054, 0, 0, 0, 4088, 3836, 4037, 0, 4096, 0, 0, 54, 0, 0, 0, 0, 402, 0, 1301, 0, 47, 0, 3987, 0, 0, 0, 4093, 0, 20, 0, 4096, 0, 32, 0, 0, 0, 4096, 0, 0, 1, 2679, 0, 0, 4094, 0, 3565, 95, 0, 0, 0, 4096, 0, 329, 153, 0, 0, 0, 8, 4085, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1, 4073, 0, 2, 4088, 0, 5, 0, 4096, 3956, 4090, 0, 4079, 0, 16, 33, 4, 0, 0, 1, 0, 453, 3102, 0, 1, 0, 2584, 0, 2, 0, 3806, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 3804, 0, 0, 0, 541, 2, 0, 2447, 175, 4084, 133, 5, 0, 13, 4094, 0, 3515, 23, 9, 0, 0, 11, 4095, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 2934, 0, 0, 4047, 26, 40, 1, 4055, 2835, 4046, 0, 4086, 0, 565, 12, 69, 0, 0, 29, 63, 741, 3318, 0, 443, 8, 2648, 18, 0, 2, 3906, 0, 21, 0, 4085, 0, 0, 0, 0, 0, 3960, 26, 11, 34, 3955, 0, 19, 2465, 226, 3985, 97, 11, 14, 0, 4089, 157, 331, 0, 1, 0, 0, 26, 4062, 4069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[3, 4046, 0, 0, 4092, 8, 11, 5, 4031, 4096, 4084, 20, 4082, 0, 322, 0, 0, 0, 0, 0, 42, 91, 0, 0, 0, 0, 0, 0, 0, 11, 4087, 1, 0, 4, 4089, 0, 0, 0, 1, 0, 4094, 0, 3, 6, 4010, 5, 0, 3818, 0, 4090, 0, 2, 2, 3, 4091, 10, 4073, 0, 0, 0, 31, 0, 4087, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 4082, 0, 0, 4096, 0, 0, 0, 4095, 4096, 4096, 0, 4096, 1, 0, 2, 0, 0, 0, 0, 0, 132, 2020, 0, 3, 0, 2634, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 1905, 0, 0, 3972, 0, 4076, 0, 0, 0, 0, 4096, 2, 1638, 2, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 3870, 19, 0, 4066, 0, 0, 25, 4063, 4096, 4084, 39, 4082, 7, 214, 0, 53, 8, 0, 9, 190, 0, 3910, 6, 0, 16, 19, 0, 0, 24, 3936, 17, 0, 15, 4088, 0, 0, 1, 2, 0, 4089, 0, 12, 0, 2, 0, 0, 4075, 0, 4087, 0, 0, 7, 0, 4078, 40, 3947, 214, 3, 0, 0, 0, 4073, 4050] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 3963, 0, 41, 4060, 0, 2, 0, 4095, 3938, 4096, 0, 4046, 0, 191, 0, 28, 0, 0, 1, 0, 18, 4067, 0, 0, 1, 4090, 0, 0, 0, 4033, 0, 3, 0, 4077, 0, 0, 0, 0, 0, 4079, 0, 0, 2, 23, 7, 9, 67, 31, 3558, 422, 1, 0, 0, 4084, 151, 4090, 69, 13, 0, 0, 3, 4096, 4081] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[5, 4095, 0, 1, 4096, 0, 2, 3, 4096, 4015, 4092, 0, 4092, 0, 7, 33, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 571, 0, 0, 0, 4096, 0, 0, 0, 4092, 10, 0, 0, 0, 0, 4068, 0, 0, 0, 101, 0, 0, 3421, 73, 4084, 3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4, 1, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4054, 0, 0, 0, 150, 48, 0, 0, 0, 0, 0, 0, 4088, 3836, 67, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 1301, 0, 4096, 1079, 0, 0, 0, 0, 0, 0, 47, 0, 3987, 0, 0, 0, 4093, 0, 1635, 415, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 0, 32, 0, 0, 0, 2891, 1165, 0, 0, 0, 0, 0, 0, 3531, 0, 0, 1, 2679, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3565, 95, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 153, 0, 0, 0, 8, 1285, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8348, 8045, 6878, 10177, 8204, 10305, 7708, 5871] avg=8192.0000 max=10305 max_rank=5 min=5871 min_rank=7 imbalance_ratio=1.257935 max_load_reduction=55231 max_load_reduction_pct=0.842758 moved_to_echo=32088 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 2934, 0, 0, 4047, 26, 40, 1, 145, 48, 0, 0, 0, 0, 0, 0, 4055, 2835, 67, 0, 0, 0, 565, 12, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 29, 63, 741, 3318, 0, 4086, 1082, 0, 0, 0, 0, 0, 0, 443, 8, 2648, 18, 0, 2, 3906, 0, 1625, 401, 0, 0, 0, 0, 0, 0, 21, 0, 4085, 0, 0, 0, 0, 0, 2897, 1159, 0, 0, 0, 0, 0, 0, 3414, 26, 11, 34, 3955, 0, 19, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 226, 3985, 97, 11, 14, 0, 4041, 157, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 1, 0, 0, 26, 1278, 4069, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7241, 7534, 9388, 9051, 8162, 9924, 8531, 5705] avg=8192.0000 max=9924 max_rank=5 min=5705 min_rank=7 imbalance_ratio=1.211426 max_load_reduction=55612 max_load_reduction_pct=0.848572 moved_to_echo=32322 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 4073, 0, 2, 4088, 0, 5, 0, 139, 48, 0, 0, 0, 0, 0, 0, 4096, 3956, 67, 0, 0, 0, 16, 33, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 453, 3102, 0, 4079, 1094, 0, 0, 0, 0, 0, 0, 1, 0, 2584, 0, 2, 0, 3806, 0, 1639, 385, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 2929, 1168, 0, 0, 0, 0, 0, 0, 3280, 0, 0, 0, 541, 2, 0, 2447, 0, 0, 0, 0, 0, 0, 0, 0, 175, 4084, 133, 5, 0, 13, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3515, 23, 9, 0, 0, 11, 1288, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8356, 8168, 8733, 8417, 8194, 6270, 8456, 8942] avg=8192.0000 max=8942 max_rank=7 min=6270 min_rank=5 imbalance_ratio=1.091553 max_load_reduction=56594 max_load_reduction_pct=0.863556 moved_to_echo=31862 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 4046, 0, 0, 4092, 8, 11, 5, 150, 48, 0, 0, 0, 0, 0, 0, 4031, 4096, 68, 20, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 91, 0, 0, 4082, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4087, 1, 1635, 415, 0, 0, 0, 0, 0, 0, 0, 4, 4089, 0, 0, 0, 1, 0, 2924, 1166, 0, 0, 0, 0, 0, 0, 3529, 0, 3, 6, 4010, 5, 0, 3818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2, 2, 3, 4043, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 31, 0, 1286, 4085, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8363, 8537, 5307, 6149, 8184, 11371, 8150, 9475] avg=8192.0000 max=11371 max_rank=5 min=5307 min_rank=2 imbalance_ratio=1.388062 max_load_reduction=54165 max_load_reduction_pct=0.826492 moved_to_echo=37180 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 3870, 19, 0, 4066, 0, 0, 25, 150, 48, 0, 0, 0, 0, 0, 0, 4063, 4096, 68, 39, 0, 7, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 8, 0, 9, 190, 0, 3910, 6, 4082, 1092, 0, 0, 0, 0, 0, 0, 0, 16, 19, 0, 0, 24, 3936, 17, 1630, 414, 0, 0, 0, 0, 0, 0, 0, 15, 4088, 0, 0, 1, 2, 0, 2924, 1162, 0, 0, 0, 0, 0, 0, 3525, 0, 12, 0, 2, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 7, 0, 4030, 40, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 214, 3, 0, 0, 0, 1281, 4050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8487, 9350, 6056, 8192, 7614, 8164, 9495] avg=8192.0000 max=9495 max_rank=7 min=6056 min_rank=3 imbalance_ratio=1.159058 max_load_reduction=56041 max_load_reduction_pct=0.855118 moved_to_echo=33465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 0, 0, 4096, 0, 0, 0, 150, 48, 0, 0, 0, 0, 0, 0, 4095, 4096, 68, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 2020, 0, 4096, 1095, 0, 0, 0, 0, 0, 0, 3, 0, 2634, 0, 0, 0, 4093, 0, 1639, 415, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 2933, 1168, 0, 0, 0, 0, 0, 0, 3531, 0, 0, 0, 1905, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4048, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1638, 2, 0, 0, 0, 0, 1289, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8376, 8262, 7343, 8784, 8212, 9408, 8126, 7025] avg=8192.0000 max=9408 max_rank=5 min=7025 min_rank=7 imbalance_ratio=1.148438 max_load_reduction=56128 max_load_reduction_pct=0.856445 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 3963, 0, 41, 4060, 0, 2, 0, 155, 54, 0, 0, 0, 0, 0, 0, 4095, 3938, 60, 0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 1, 0, 18, 4067, 0, 4046, 1100, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 4033, 0, 1643, 417, 0, 0, 0, 0, 0, 0, 3, 0, 4077, 0, 0, 0, 0, 0, 2936, 1172, 0, 0, 0, 0, 0, 0, 3507, 0, 0, 2, 23, 7, 9, 67, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3558, 422, 1, 0, 0, 4030, 151, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 69, 13, 0, 0, 3, 1281, 4081, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8284, 9260, 10184, 8188, 3615, 8193, 9537] avg=8192.0000 max=10184 max_rank=3 min=3615 min_rank=5 imbalance_ratio=1.243164 max_load_reduction=55352 max_load_reduction_pct=0.844604 moved_to_echo=29533 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5, 4095, 0, 1, 4096, 0, 2, 3, 149, 48, 0, 0, 0, 0, 0, 0, 4096, 4015, 68, 0, 0, 0, 7, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4092, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 571, 0, 0, 0, 4096, 0, 1639, 412, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 10, 0, 0, 0, 0, 2930, 1168, 0, 0, 0, 0, 0, 0, 3507, 0, 0, 0, 101, 0, 0, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 73, 4084, 3, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 1, 1289, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8399, 8219, 9277, 6718, 8200, 7029, 8208, 9486] avg=8192.0000 max=9486 max_rank=7 min=6718 min_rank=3 imbalance_ratio=1.157959 max_load_reduction=56050 max_load_reduction_pct=0.855255 moved_to_echo=32923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[27904, 128, 128, 128, 13312, 128, 128, 24448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[22144, 512, 128, 0, 128, 128, 10368, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 31232, 128, 128, 32640, 128, 128, 128, 1280, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 30976, 640, 128, 0, 128, 1408, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 128, 23424, 9344, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 128, 0, 128, 768, 1536, 21888, 128, 32768, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 31616, 768, 128, 128, 128, 32384, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 128, 16640, 128, 128, 128, 32128, 128, 13184, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 3, 4096, 0, 2, 24, 1, 0, 1886, 0, 0, 4095, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 81, 0, 4096, 5, 4093, 0, 2, 4094, 4096, 4022, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 0, 410, 0, 0, 11, 0, 10, 4095, 0, 7, 1, 3464, 0, 4063, 2618, 4027] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 3, 0, 4055, 5, 0, 0, 392, 18, 482, 0, 0, 4087, 0, 42, 0, 55, 4047, 3654, 52, 4, 0, 0, 14, 0, 0, 0, 50, 4077, 0, 0, 4075, 0, 4095, 0, 0, 3965, 4031, 3706, 0, 0, 31, 0, 0, 0, 4065, 0, 1, 23, 0, 0, 5, 131, 61, 9, 3993, 102, 4004, 19, 4089, 0, 4076, 0, 4018] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 213, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 3, 51, 4096, 0, 4013, 0, 0, 3419, 4096, 4096, 0, 2, 0, 0, 0, 0, 4096, 1, 0, 0, 209, 0, 0, 0, 1, 0, 4096, 0, 3015, 0, 804, 0, 4092, 614, 4042] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 4, 8, 0, 240, 0, 4092, 0, 0, 4084, 0, 1, 14, 4, 4095, 3349, 0, 0, 0, 0, 24, 0, 0, 0, 0, 4096, 66, 126, 4092, 2, 4092, 0, 0, 3642, 4094, 3979, 0, 0, 0, 0, 0, 37, 4096, 0, 5, 0, 0, 4, 0, 0, 0, 0, 4054, 30, 3616, 0, 460, 0, 4033, 939, 4064] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[5, 3, 0, 4088, 1, 0, 23, 0, 0, 862, 0, 2, 4092, 0, 12, 0, 1, 4072, 13, 4, 0, 0, 15, 0, 0, 0, 1, 6, 4093, 0, 0, 4073, 0, 4093, 0, 0, 4071, 4087, 4089, 0, 0, 7, 18, 1, 0, 4095, 0, 3, 2, 2558, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 3277, 3, 4088, 1498, 4090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[15, 14, 0, 4081, 11, 0, 40, 60, 0, 4096, 0, 0, 4072, 13, 35, 0, 1, 3768, 4050, 0, 0, 0, 1, 0, 1, 11, 2, 0, 4088, 0, 10, 4078, 0, 4071, 0, 0, 4085, 4073, 4038, 23, 6, 0, 23, 0, 0, 4096, 8, 26, 0, 74, 1, 0, 0, 0, 0, 4096, 0, 3566, 0, 1427, 2, 4095, 1408, 1971] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 1, 4096, 0, 12, 0, 277, 0, 4019, 0, 0, 4096, 0, 13, 0, 0, 4096, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 28, 0, 4096, 10, 4096, 0, 0, 4081, 4092, 4040, 0, 0, 0, 0, 0, 2, 4072, 0, 0, 0, 74, 0, 1, 0, 6, 0, 4095, 28, 2689, 0, 4071, 0, 4059, 5, 4072] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 8, 2019, 5, 0, 4079, 0, 0, 86, 17, 4047, 4086, 0, 21, 12, 0, 120, 0, 0, 0, 0, 4066, 198, 401, 4092, 40, 4095, 0, 18, 3978, 4077, 4095, 23, 0, 0, 10, 0, 6, 4096, 0, 0, 1, 0, 40, 3, 0, 0, 0, 3767, 0, 2897, 0, 3948, 0, 3910, 948, 2233] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 4055, 5, 0, 0, 392, 3958, 0, 0, 0, 0, 0, 0, 0, 18, 482, 0, 0, 4087, 0, 42, 0, 1296, 226, 0, 0, 0, 0, 0, 0, 55, 4047, 3654, 52, 4, 0, 0, 14, 1045, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 3963, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 250, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 4065, 0, 1, 4095, 15, 0, 0, 0, 0, 0, 0, 23, 0, 0, 5, 131, 61, 9, 3993, 3555, 103, 0, 0, 0, 0, 0, 0, 102, 4004, 19, 4089, 0, 0, 0, 1677, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8413, 6151, 8985, 8088, 7921, 8207, 7880, 9891] avg=8192.0000 max=9891 max_rank=7 min=6151 min_rank=1 imbalance_ratio=1.207397 max_load_reduction=55645 max_load_reduction_pct=0.849075 moved_to_echo=33899 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 4096, 0, 2, 24, 1, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 1886, 0, 0, 4095, 0, 0, 0, 1299, 230, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 1047, 114, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3982, 81, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 4094, 253, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 4093, 16, 0, 0, 0, 0, 0, 0, 0, 410, 0, 0, 11, 0, 10, 4095, 3613, 101, 0, 0, 0, 0, 0, 0, 0, 7, 1, 3464, 0, 0, 2618, 1681, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8072, 7510, 9337, 8162, 8376, 8068, 8240, 7771] avg=8192.0000 max=9337 max_rank=2 min=7510 min_rank=1 imbalance_ratio=1.139771 max_load_reduction=56199 max_load_reduction_pct=0.857529 moved_to_echo=32455 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 4, 8, 0, 240, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4084, 0, 1, 14, 1311, 229, 0, 0, 0, 0, 0, 0, 4, 4095, 3349, 0, 0, 0, 0, 24, 1057, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 66, 126, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3642, 254, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4096, 0, 5, 4092, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4054, 3611, 101, 0, 0, 0, 0, 0, 0, 30, 3616, 0, 460, 0, 0, 939, 1696, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 9731, 8643, 8266, 7877, 8246, 7770, 6741] avg=8192.0000 max=9731 max_rank=1 min=6741 min_rank=7 imbalance_ratio=1.187866 max_load_reduction=55805 max_load_reduction_pct=0.851517 moved_to_echo=30634 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[5, 3, 0, 4088, 1, 0, 23, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 862, 0, 2, 4092, 0, 12, 0, 1320, 229, 0, 0, 0, 0, 0, 0, 1, 4072, 13, 4, 0, 0, 15, 0, 1064, 114, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 3979, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 253, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 18, 1, 0, 4095, 0, 3, 4093, 17, 0, 0, 0, 0, 0, 0, 2, 2558, 0, 0, 0, 0, 0, 4096, 3605, 102, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3277, 3, 0, 1498, 1706, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 6517, 5283, 8059, 8413, 8234, 10363, 10578] avg=8192.0000 max=10578 max_rank=7 min=5283 min_rank=2 imbalance_ratio=1.291260 max_load_reduction=54958 max_load_reduction_pct=0.838593 moved_to_echo=37588 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[15, 14, 0, 4081, 11, 0, 40, 60, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4072, 13, 35, 0, 636, 228, 0, 0, 0, 0, 0, 0, 1, 3768, 4050, 0, 0, 0, 1, 0, 512, 114, 0, 0, 0, 0, 0, 0, 1, 11, 2, 0, 3974, 0, 10, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 252, 4038, 23, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 23, 0, 0, 4096, 8, 26, 4071, 17, 0, 0, 0, 0, 0, 0, 0, 74, 1, 0, 0, 0, 0, 4096, 3593, 102, 0, 0, 0, 0, 0, 0, 0, 3566, 0, 1427, 2, 0, 1408, 823, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 9080, 8446, 8076, 8398, 8247, 7866, 7226] avg=8192.0000 max=9080 max_rank=1 min=7226 min_rank=7 imbalance_ratio=1.108398 max_load_reduction=56456 max_load_reduction_pct=0.861450 moved_to_echo=31737 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 213, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 1304, 230, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 1051, 114, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3982, 3, 51, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 253, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 1, 0, 4013, 15, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 1, 0, 4096, 3613, 102, 0, 0, 0, 0, 0, 0, 0, 3015, 0, 804, 0, 0, 614, 1687, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8284, 9722, 9355, 8139, 7768, 8127, 8021, 6120] avg=8192.0000 max=9722 max_rank=1 min=6120 min_rank=7 imbalance_ratio=1.186768 max_load_reduction=55814 max_load_reduction_pct=0.851654 moved_to_echo=30036 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 3798, 0, 0, 0, 0, 0, 0, 0, 8, 2019, 5, 0, 4079, 0, 0, 86, 724, 233, 0, 0, 0, 0, 0, 0, 17, 4047, 4086, 0, 21, 12, 0, 120, 585, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 198, 401, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 18, 3978, 244, 4095, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 6, 4096, 0, 0, 4095, 15, 0, 0, 0, 0, 0, 0, 1, 0, 40, 3, 0, 0, 0, 3767, 3600, 97, 0, 0, 0, 0, 0, 0, 0, 2897, 0, 3948, 0, 0, 948, 924, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7892, 7154, 9005, 8640, 8398, 8222, 7508, 8717] avg=8192.0000 max=9005 max_rank=2 min=7154 min_rank=1 imbalance_ratio=1.099243 max_load_reduction=56531 max_load_reduction_pct=0.862595 moved_to_echo=32845 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4096, 0, 12, 0, 277, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 4096, 0, 13, 0, 1314, 229, 0, 0, 0, 0, 0, 0, 0, 4096, 1213, 0, 0, 0, 0, 0, 1059, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 28, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4081, 254, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4072, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 74, 0, 1, 0, 6, 0, 4095, 3609, 103, 0, 0, 0, 0, 0, 0, 28, 2689, 0, 4071, 0, 0, 5, 1699, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8327, 9671, 6482, 8106, 8385, 8185, 7888, 8492] avg=8192.0000 max=9671 max_rank=1 min=6482 min_rank=2 imbalance_ratio=1.180542 max_load_reduction=55865 max_load_reduction_pct=0.852432 moved_to_echo=32950 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 23936, 128, 21632, 128, 0, 8064, 11904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 21632, 128, 128, 32768, 128, 128, 128, 9216, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 1280, 31488, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 31872, 384, 640, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 31360, 2048, 32128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32384, 24576, 128, 128, 128, 128, 256, 7424, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 3328, 128, 128, 256, 128, 128, 32384, 28800, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 32640, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 4022, 1902, 1, 0, 4046, 4093, 0, 0, 1, 0, 0, 380, 4096, 0, 4, 15, 0, 0, 4094, 0, 4096, 21, 184, 0, 283, 18, 1978, 82, 0, 0, 4096, 0, 3286, 10, 4088, 870, 0, 0, 3229, 0, 3, 4096, 0, 39, 0, 0, 4096, 0, 0, 0, 4016, 14, 292, 0, 0, 0, 4096, 0, 0, 0, 3988] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[18, 0, 3, 3522, 472, 0, 13, 208, 4070, 0, 0, 24, 0, 0, 3330, 4083, 0, 30, 2, 0, 9, 4056, 10, 4084, 21, 3331, 0, 1184, 209, 0, 4, 9, 0, 4090, 0, 3189, 0, 4045, 141, 112, 0, 2068, 0, 1833, 4096, 0, 14, 5, 0, 3962, 64, 4, 0, 3935, 57, 867, 0, 0, 2, 4012, 0, 0, 345, 4003] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 2894, 0, 4096, 1353, 0, 0, 3107, 4096, 0, 0, 0, 0, 0, 3583, 4016, 0, 0, 83, 0, 0, 4095, 0, 4096, 0, 37, 43, 3497, 0, 1, 252, 0, 0, 3997, 278, 3621, 0, 4096, 0, 0, 0, 1268, 0, 2, 4096, 668, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4064, 5, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 3656, 4005, 27, 0, 0, 4092, 0, 0, 0, 0, 0, 1322, 4096, 0, 1, 8, 0, 0, 4066, 0, 4074, 0, 0, 0, 4053, 2, 0, 191, 0, 0, 4096, 0, 4096, 0, 4096, 107, 13, 0, 4032, 0, 253, 4096, 0, 0, 0, 0, 3215, 0, 0, 0, 4083, 0, 53, 0, 0, 0, 3797, 0, 0, 26, 3980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 8, 4031, 0, 2, 1, 2452, 4095, 0, 17, 2, 1, 0, 0, 4017, 63, 0, 0, 27, 9, 4082, 0, 4080, 66, 432, 0, 4040, 117, 152, 1, 0, 0, 4087, 0, 2650, 0, 4073, 787, 1, 0, 3709, 0, 565, 4063, 0, 86, 0, 3, 4095, 0, 0, 0, 3654, 1, 2249, 0, 0, 0, 3943, 13, 0, 0, 3862] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 3632, 4096, 0, 0, 1, 0, 0, 4086, 0, 4096, 0, 4016, 0, 3939, 0, 0, 54, 0, 0, 4096, 0, 3872, 0, 4094, 232, 0, 0, 671, 0, 0, 4096, 6, 5, 0, 0, 4096, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[7, 2632, 0, 4093, 4096, 80, 5, 3162, 4092, 4, 0, 0, 6, 0, 48, 3987, 0, 0, 0, 0, 2, 4092, 4, 1232, 4, 0, 65, 4086, 0, 1056, 0, 0, 8, 4093, 0, 4096, 0, 4071, 0, 1, 3, 4091, 97, 0, 3963, 2306, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 25, 3, 3, 1795, 42, 0, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[24, 29, 0, 4024, 283, 4, 0, 264, 4085, 0, 47, 17, 0, 16, 0, 3968, 0, 0, 0, 0, 0, 4080, 1, 3108, 0, 3979, 0, 4058, 0, 213, 105, 38, 0, 4070, 0, 4096, 0, 4087, 429, 0, 18, 4079, 9, 0, 4053, 75, 117, 0, 10, 3871, 1, 13, 0, 4096, 0, 0, 0, 8, 2, 4062, 0, 19, 0, 4078] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4022, 1902, 1, 0, 4046, 363, 0, 0, 0, 0, 0, 0, 0, 2597, 0, 0, 1, 0, 0, 380, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 15, 0, 0, 4094, 0, 4096, 294, 174, 0, 0, 0, 0, 0, 0, 21, 184, 0, 283, 18, 1978, 82, 0, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3286, 10, 129, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3229, 0, 3, 4096, 0, 39, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4016, 14, 292, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3988, 373, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10335, 7074, 8677, 5535, 8391, 7833, 9234, 8457] avg=8192.0000 max=10335 max_rank=0 min=5535 min_rank=3 imbalance_ratio=1.261597 max_load_reduction=55201 max_load_reduction_pct=0.842300 moved_to_echo=33915 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3656, 4005, 27, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 2597, 0, 0, 0, 0, 0, 1322, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 4066, 0, 4074, 294, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 2, 0, 191, 0, 2975, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 129, 107, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 253, 4096, 0, 0, 0, 465, 0, 0, 0, 0, 0, 0, 0, 0, 3215, 0, 0, 0, 4083, 0, 53, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3797, 0, 0, 26, 3980, 373, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8051, 8015, 8617, 7221, 8441, 8846, 8169, 8176] avg=8192.0000 max=8846 max_rank=5 min=7221 min_rank=3 imbalance_ratio=1.079834 max_load_reduction=56690 max_load_reduction_pct=0.865021 moved_to_echo=33632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[7, 2632, 0, 4093, 4096, 80, 5, 3162, 363, 0, 0, 0, 0, 0, 0, 0, 2597, 4, 0, 0, 6, 0, 48, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 4, 1232, 294, 173, 0, 0, 0, 0, 0, 0, 4, 0, 65, 4086, 0, 1056, 0, 0, 2956, 0, 0, 0, 0, 0, 0, 0, 8, 4093, 0, 4096, 0, 129, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 97, 0, 3963, 2306, 0, 0, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 813, 0, 0, 0, 0, 0, 0, 0, 25, 3, 3, 1795, 42, 0, 0, 4093, 373, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14438, 6642, 5797, 8167, 8327, 10925, 4906, 6334] avg=8192.0000 max=14438 max_rank=0 min=4906 min_rank=6 imbalance_ratio=1.762451 max_load_reduction=51098 max_load_reduction_pct=0.779694 moved_to_echo=30492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 3, 3522, 472, 0, 13, 208, 361, 0, 0, 0, 0, 0, 0, 0, 2582, 0, 0, 24, 0, 0, 3330, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 2, 0, 9, 4056, 10, 4084, 293, 172, 0, 0, 0, 0, 0, 0, 21, 3331, 0, 1184, 209, 0, 4, 9, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3189, 0, 127, 141, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2068, 0, 1833, 4096, 0, 14, 5, 463, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 64, 4, 0, 3935, 57, 867, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4012, 0, 0, 345, 4003, 371, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4597, 10019, 8656, 7696, 7659, 8479, 9697, 8733] avg=8192.0000 max=10019 max_rank=1 min=4597 min_rank=0 imbalance_ratio=1.223022 max_load_reduction=55517 max_load_reduction_pct=0.847122 moved_to_echo=34568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2894, 0, 4096, 1353, 0, 0, 3107, 363, 0, 0, 0, 0, 0, 0, 0, 2600, 0, 0, 0, 0, 0, 3583, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 4095, 0, 4096, 294, 174, 0, 0, 0, 0, 0, 0, 0, 37, 43, 3497, 0, 1, 252, 0, 2975, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 278, 3621, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1268, 0, 2, 4096, 668, 0, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 5, 0, 0, 4096, 373, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11813, 10199, 8742, 6805, 8025, 6500, 4914, 8538] avg=8192.0000 max=11813 max_rank=0 min=4914 min_rank=6 imbalance_ratio=1.442017 max_load_reduction=53723 max_load_reduction_pct=0.819748 moved_to_echo=27977 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4086, 0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 2600, 0, 0, 0, 0, 1, 3632, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4086, 0, 4096, 294, 174, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 3939, 0, 0, 54, 0, 2973, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3872, 0, 129, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 0, 4096, 6, 5, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4075, 0, 0, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4092, 373, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4449, 10329, 8651, 10982, 8329, 5244, 8989, 8563] avg=8192.0000 max=10982 max_rank=3 min=4449 min_rank=0 imbalance_ratio=1.340576 max_load_reduction=54554 max_load_reduction_pct=0.832428 moved_to_echo=31125 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[24, 29, 0, 4024, 283, 4, 0, 264, 362, 0, 0, 0, 0, 0, 0, 0, 2592, 0, 47, 17, 0, 16, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 1, 3108, 294, 174, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 4058, 0, 213, 105, 38, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4096, 0, 129, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4079, 9, 0, 4053, 75, 117, 0, 465, 0, 0, 0, 0, 0, 0, 0, 10, 3871, 1, 13, 0, 4096, 0, 0, 816, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 4062, 0, 19, 0, 4078, 372, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4990, 6640, 7657, 11361, 8724, 8816, 8807, 8541] avg=8192.0000 max=11361 max_rank=3 min=4990 min_rank=0 imbalance_ratio=1.386841 max_load_reduction=54175 max_load_reduction_pct=0.826645 moved_to_echo=34888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 4031, 0, 2, 1, 2452, 369, 0, 0, 0, 0, 0, 0, 0, 2581, 0, 17, 2, 1, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 27, 9, 4082, 0, 4080, 299, 179, 0, 0, 0, 0, 0, 0, 66, 432, 0, 4040, 117, 152, 1, 0, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 2650, 0, 115, 787, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 0, 565, 4063, 0, 86, 0, 470, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 3654, 1, 2249, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 13, 0, 0, 3862, 376, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6863, 6618, 8739, 7769, 7640, 8893, 10820, 8194] avg=8192.0000 max=10820 max_rank=6 min=6618 min_rank=1 imbalance_ratio=1.320801 max_load_reduction=54716 max_load_reduction_pct=0.834900 moved_to_echo=35547 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 23424, 128, 128, 0, 32128, 128, 3584, 6528, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 12032, 128, 25216, 384, 3456, 768, 128, 23808, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 28928, 2432, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 5632, 128, 31616, 12160, 128, 128, 13312, 2944, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[20864, 128, 128, 128, 128, 128, 12416, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32640, 384, 28928, 128, 1024, 2688, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 29952, 128, 128, 384, 32256, 3072, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 23168, 128, 2688, 32640, 3072, 384, 128, 3840, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[35, 1183, 2623, 34, 1543, 0, 803, 11, 1523, 48, 1213, 471, 565, 445, 13, 412, 149, 2061, 2100, 1606, 3661, 874, 809, 1456, 4, 3607, 146, 1202, 446, 165, 600, 1421, 3347, 186, 122, 346, 689, 127, 1052, 1231, 2399, 3214, 757, 281, 183, 1816, 1484, 2054, 129, 675, 1062, 183, 120, 585, 3033, 198, 57, 1588, 3438, 2296, 283, 866, 381, 125] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[15, 746, 2119, 40, 1800, 0, 886, 30, 1086, 104, 940, 942, 705, 256, 28, 468, 260, 1793, 2093, 1579, 3567, 1186, 537, 1558, 1, 3343, 186, 859, 598, 109, 743, 1421, 3546, 190, 156, 729, 505, 155, 1249, 1134, 2315, 3046, 945, 217, 250, 2463, 1641, 1898, 160, 516, 954, 166, 92, 274, 3690, 273, 17, 1831, 3257, 1750, 266, 933, 870, 50] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[23, 783, 1997, 34, 1337, 1, 863, 23, 2455, 128, 1579, 670, 609, 563, 18, 418, 168, 1430, 2323, 1110, 3523, 1486, 532, 1887, 1, 3149, 156, 984, 427, 87, 993, 1230, 3156, 209, 129, 967, 884, 64, 746, 1322, 2102, 3499, 1203, 421, 199, 2407, 982, 551, 192, 548, 1154, 172, 274, 440, 3234, 275, 12, 2315, 3661, 1467, 657, 387, 828, 92] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[22, 1217, 2038, 88, 1931, 0, 837, 31, 1865, 49, 1200, 717, 1244, 258, 15, 518, 355, 1761, 2208, 1407, 3525, 1395, 639, 1596, 12, 3385, 165, 757, 392, 65, 735, 886, 3263, 184, 146, 691, 625, 120, 1217, 1384, 2240, 3015, 761, 233, 245, 2321, 1169, 1168, 143, 830, 1322, 93, 180, 443, 3508, 206, 71, 1780, 3561, 1850, 327, 683, 304, 140] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[33, 992, 2629, 71, 2035, 4, 790, 46, 1059, 102, 987, 693, 652, 304, 46, 446, 245, 1974, 2018, 1520, 3421, 809, 607, 1318, 36, 3033, 253, 951, 429, 198, 515, 1574, 3540, 201, 132, 805, 861, 151, 1198, 1417, 2421, 2781, 1082, 177, 132, 2331, 2000, 1497, 182, 751, 829, 128, 124, 718, 3452, 260, 19, 1588, 3095, 2060, 325, 1040, 316, 133] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[42, 939, 2317, 45, 1708, 23, 888, 27, 1077, 138, 1311, 631, 500, 301, 49, 387, 207, 1940, 2174, 1477, 3397, 879, 543, 1577, 6, 2821, 158, 956, 649, 159, 715, 1958, 3300, 298, 277, 1117, 944, 165, 850, 1618, 2393, 2901, 807, 233, 310, 2437, 1698, 1463, 169, 817, 772, 159, 103, 462, 3099, 134, 27, 1621, 3133, 1830, 367, 932, 924, 177] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[34, 1026, 2301, 65, 1972, 0, 1135, 14, 1839, 112, 1299, 570, 478, 347, 15, 352, 198, 2324, 1674, 1170, 3679, 1204, 701, 1572, 10, 3448, 139, 910, 236, 56, 696, 1588, 3249, 264, 89, 1122, 442, 118, 1101, 1209, 2273, 3174, 632, 192, 217, 2182, 1695, 873, 92, 769, 1287, 130, 182, 696, 3069, 118, 87, 1666, 3636, 2418, 203, 739, 277, 171] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[25, 960, 2102, 130, 1995, 2, 1021, 22, 1701, 28, 989, 379, 542, 306, 40, 403, 267, 2024, 2013, 1376, 3627, 1037, 614, 1782, 5, 3288, 190, 742, 167, 57, 638, 1425, 3486, 434, 201, 959, 339, 231, 1226, 1430, 2441, 2592, 606, 77, 272, 2322, 1674, 2001, 229, 564, 1140, 135, 74, 444, 3427, 247, 63, 1358, 3695, 2340, 320, 639, 544, 129] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[15, 746, 2119, 40, 1800, 0, 886, 30, 1833, 310, 0, 0, 0, 0, 0, 0, 1086, 104, 940, 942, 705, 256, 28, 468, 3257, 0, 0, 0, 0, 0, 0, 0, 260, 1793, 1329, 1579, 0, 1186, 537, 1558, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3343, 186, 859, 598, 109, 743, 1421, 764, 338, 56, 0, 0, 0, 0, 0, 3546, 190, 156, 729, 505, 155, 1249, 1134, 360, 0, 0, 0, 0, 0, 0, 0, 1617, 0, 945, 217, 250, 2463, 1641, 1898, 0, 0, 0, 0, 0, 0, 0, 0, 160, 516, 954, 166, 92, 274, 3690, 273, 1213, 711, 0, 0, 0, 0, 0, 0, 17, 1831, 2490, 1750, 266, 933, 870, 50, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7779, 7786, 8242, 8418, 8024, 9031, 8049, 8207] avg=8192.0000 max=9031 max_rank=5 min=7779 min_rank=0 imbalance_ratio=1.102417 max_load_reduction=56505 max_load_reduction_pct=0.862198 moved_to_echo=33311 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[35, 1183, 2623, 34, 1543, 0, 803, 11, 1935, 319, 0, 0, 0, 0, 0, 0, 1523, 48, 1213, 471, 565, 445, 13, 412, 3342, 0, 0, 0, 0, 0, 0, 0, 149, 2061, 1333, 1606, 0, 874, 809, 1456, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3607, 146, 1202, 446, 165, 600, 1421, 767, 350, 59, 0, 0, 0, 0, 0, 3347, 186, 122, 346, 689, 127, 1052, 1231, 373, 0, 0, 0, 0, 0, 0, 0, 1676, 0, 757, 281, 183, 1816, 1484, 2054, 0, 0, 0, 0, 0, 0, 0, 0, 129, 675, 1062, 183, 120, 585, 3033, 198, 1279, 751, 0, 0, 0, 0, 0, 0, 57, 1588, 2628, 2296, 283, 866, 381, 125, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8486, 8032, 8288, 8767, 7473, 8251, 8015, 8224] avg=8192.0000 max=8767 max_rank=3 min=7473 min_rank=4 imbalance_ratio=1.070190 max_load_reduction=56769 max_load_reduction_pct=0.866226 moved_to_echo=31963 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[22, 1217, 2038, 88, 1931, 0, 837, 31, 1815, 307, 0, 0, 0, 0, 0, 0, 1865, 49, 1200, 717, 1244, 258, 15, 518, 3218, 0, 0, 0, 0, 0, 0, 0, 355, 1761, 1402, 1407, 0, 1395, 639, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3385, 165, 757, 392, 65, 735, 886, 806, 327, 61, 0, 0, 0, 0, 0, 3263, 184, 146, 691, 625, 120, 1217, 1384, 348, 0, 0, 0, 0, 0, 0, 0, 1565, 0, 761, 233, 245, 2321, 1169, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 143, 830, 1322, 93, 180, 443, 3508, 206, 1200, 778, 0, 0, 0, 0, 0, 0, 71, 1780, 2722, 1850, 327, 683, 304, 140, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8286, 9084, 8555, 7591, 7978, 7462, 8703, 7877] avg=8192.0000 max=9084 max_rank=1 min=7462 min_rank=5 imbalance_ratio=1.108887 max_load_reduction=56452 max_load_reduction_pct=0.861389 moved_to_echo=32020 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[23, 783, 1997, 34, 1337, 1, 863, 23, 2105, 306, 0, 0, 0, 0, 0, 0, 2455, 128, 1579, 670, 609, 563, 18, 418, 3217, 0, 0, 0, 0, 0, 0, 0, 168, 1430, 1475, 1110, 0, 1486, 532, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3149, 156, 984, 427, 87, 993, 1230, 848, 307, 63, 0, 0, 0, 0, 0, 3156, 209, 129, 967, 884, 64, 746, 1322, 326, 0, 0, 0, 0, 0, 0, 0, 1469, 0, 1203, 421, 199, 2407, 982, 551, 0, 0, 0, 0, 0, 0, 0, 0, 192, 548, 1154, 172, 274, 440, 3234, 275, 1394, 800, 0, 0, 0, 0, 0, 0, 12, 2315, 2798, 1467, 657, 387, 828, 92, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7472, 9657, 8088, 8245, 7803, 7232, 8483, 8556] avg=8192.0000 max=9657 max_rank=1 min=7232 min_rank=5 imbalance_ratio=1.178833 max_load_reduction=55879 max_load_reduction_pct=0.852646 moved_to_echo=32074 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[42, 939, 2317, 45, 1708, 23, 888, 27, 1746, 296, 0, 0, 0, 0, 0, 0, 1077, 138, 1311, 631, 500, 301, 49, 387, 3101, 0, 0, 0, 0, 0, 0, 0, 207, 1940, 1376, 1477, 0, 879, 543, 1577, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2821, 158, 956, 649, 159, 715, 1958, 798, 353, 56, 0, 0, 0, 0, 0, 3300, 298, 277, 1117, 944, 165, 850, 1618, 376, 0, 0, 0, 0, 0, 0, 0, 1664, 0, 807, 233, 310, 2437, 1698, 1463, 0, 0, 0, 0, 0, 0, 0, 0, 169, 817, 772, 159, 103, 462, 3099, 134, 1155, 689, 0, 0, 0, 0, 0, 0, 27, 1621, 2388, 1830, 367, 932, 924, 177, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8031, 7495, 7999, 8629, 8945, 8612, 7559, 8266] avg=8192.0000 max=8945 max_rank=4 min=7495 min_rank=1 imbalance_ratio=1.091919 max_load_reduction=56591 max_load_reduction_pct=0.863510 moved_to_echo=33382 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[33, 992, 2629, 71, 2035, 4, 790, 46, 1674, 298, 0, 0, 0, 0, 0, 0, 1059, 102, 987, 693, 652, 304, 46, 446, 3123, 0, 0, 0, 0, 0, 0, 0, 245, 1974, 1281, 1520, 0, 809, 607, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3033, 253, 951, 429, 198, 515, 1574, 737, 353, 53, 0, 0, 0, 0, 0, 3540, 201, 132, 805, 861, 151, 1198, 1417, 376, 0, 0, 0, 0, 0, 0, 0, 1692, 0, 1082, 177, 132, 2331, 2000, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 182, 751, 829, 128, 124, 718, 3452, 260, 1107, 676, 0, 0, 0, 0, 0, 0, 19, 1588, 2366, 2060, 325, 1040, 316, 133, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8572, 7412, 7754, 8132, 8681, 8911, 8227, 7847] avg=8192.0000 max=8911 max_rank=5 min=7412 min_rank=1 imbalance_ratio=1.087769 max_load_reduction=56625 max_load_reduction_pct=0.864029 moved_to_echo=33666 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[34, 1026, 2301, 65, 1972, 0, 1135, 14, 1910, 320, 0, 0, 0, 0, 0, 0, 1839, 112, 1299, 570, 478, 347, 15, 352, 3359, 0, 0, 0, 0, 0, 0, 0, 198, 2324, 1063, 1170, 0, 1204, 701, 1572, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3448, 139, 910, 236, 56, 696, 1588, 611, 332, 62, 0, 0, 0, 0, 0, 3249, 264, 89, 1122, 442, 118, 1101, 1209, 353, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 632, 192, 217, 2182, 1695, 873, 0, 0, 0, 0, 0, 0, 0, 0, 92, 769, 1287, 130, 182, 696, 3069, 118, 1264, 794, 0, 0, 0, 0, 0, 0, 87, 1666, 2780, 2418, 203, 739, 277, 171, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8777, 8371, 8232, 8088, 7947, 7379, 8401, 8341] avg=8192.0000 max=8777 max_rank=0 min=7379 min_rank=5 imbalance_ratio=1.071411 max_load_reduction=56759 max_load_reduction_pct=0.866074 moved_to_echo=32068 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[25, 960, 2102, 130, 1995, 2, 1021, 22, 1560, 316, 0, 0, 0, 0, 0, 0, 1701, 28, 989, 379, 542, 306, 40, 403, 3311, 0, 0, 0, 0, 0, 0, 0, 267, 2024, 1278, 1376, 0, 1037, 614, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3288, 190, 742, 167, 57, 638, 1425, 735, 356, 63, 0, 0, 0, 0, 0, 3486, 434, 201, 959, 339, 231, 1226, 1430, 379, 0, 0, 0, 0, 0, 0, 0, 1706, 0, 606, 77, 272, 2322, 1674, 2001, 0, 0, 0, 0, 0, 0, 0, 0, 229, 564, 1140, 135, 74, 444, 3427, 247, 1032, 807, 0, 0, 0, 0, 0, 0, 63, 1358, 2825, 2340, 320, 639, 544, 129, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 7699, 8378, 7666, 8685, 8658, 8099, 8218] avg=8192.0000 max=8685 max_rank=4 min=7666 min_rank=3 imbalance_ratio=1.060181 max_load_reduction=56851 max_load_reduction_pct=0.867477 moved_to_echo=33660 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 7936, 18176, 512, 14336, 128, 7296, 256, 14592, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1920, 15360, 10624, 11264, 0, 8960, 4992, 12800, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 13824, 21120, 16128, 2816, 6272, 4480, 1024, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 26112, 1408, 7424, 3456, 896, 5760, 11520, 6144, 2816, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[27008, 2048, 1280, 6784, 5376, 1152, 8704, 10752, 2944, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[13056, 0, 6912, 1920, 1920, 18304, 12416, 11520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1408, 5504, 8576, 1280, 1152, 4096, 26624, 1792, 9728, 6016, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[12672, 768, 9600, 5120, 5376, 2816, 256, 3456, 25984, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 0, 2, 0, 4086, 3975, 2935, 4092, 0, 3856, 296, 3539, 2913, 0, 0, 3, 3965, 3126, 4096, 0, 4042, 682, 3185, 0, 0, 0, 0, 4072, 0, 0, 16, 0, 3961, 1, 0, 0, 0, 14, 384, 6, 5, 200, 0, 637, 0, 0, 0, 0, 4082, 0, 3794, 0, 0, 0, 1, 0, 81, 196, 3285, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4094, 3605, 909, 3781, 0, 4009, 953, 4095, 1082, 0, 0, 0, 4096, 2144, 4095, 0, 3892, 2874, 3822, 1, 0, 6, 0, 3999, 3, 17, 0, 13, 3430, 0, 0, 1, 0, 61, 21, 3, 2, 333, 0, 1361, 1, 0, 0, 3, 4096, 2, 4090, 0, 0, 0, 16, 0, 3, 640, 3983, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 2, 0, 3926, 467, 3536, 2365, 1, 4091, 299, 4092, 3514, 0, 1, 77, 4091, 2853, 4056, 0, 2385, 176, 3578, 2, 0, 2, 0, 3014, 0, 5, 1, 0, 4095, 0, 0, 1, 0, 0, 0, 3, 181, 1386, 0, 2408, 0, 0, 10, 3, 4074, 0, 4092, 1, 4, 0, 4, 0, 407, 2987, 3344, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4, 0, 0, 1, 0, 8, 0, 4092, 1896, 2296, 4010, 0, 4059, 477, 4083, 688, 1, 0, 0, 4070, 3508, 4094, 0, 3571, 465, 3793, 0, 0, 1, 0, 4047, 0, 24, 0, 0, 4089, 7, 4, 0, 0, 17, 4, 0, 21, 1469, 0, 2744, 5, 0, 3, 4, 4082, 0, 4002, 0, 0, 1, 23, 6, 247, 2685, 933, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 1, 68, 0, 4003, 958, 2981, 4049, 1, 3864, 218, 4084, 3095, 0, 0, 14, 3956, 3511, 4093, 27, 4031, 925, 3463, 15, 0, 1, 0, 3887, 0, 7, 0, 0, 4022, 5, 0, 2, 0, 83, 47, 0, 14, 365, 3, 421, 100, 0, 5, 0, 3927, 0, 3989, 8, 1, 0, 5, 0, 167, 3797, 1318, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4092, 3096, 3630, 4088, 0, 4095, 22, 4096, 1858, 0, 0, 2, 4092, 2892, 4096, 0, 4078, 0, 3137, 0, 0, 6, 0, 4059, 0, 4, 1, 2, 4096, 0, 0, 0, 0, 251, 8, 0, 0, 40, 0, 1642, 0, 0, 5, 0, 4096, 0, 3976, 13, 0, 0, 1, 0, 50, 2043, 1944, 0, 24] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[56, 0, 1, 0, 0, 0, 1, 4054, 3351, 1615, 4090, 0, 3992, 2, 4067, 1974, 0, 15, 1, 3901, 3848, 4058, 17, 4059, 73, 3172, 0, 3, 29, 0, 3694, 27, 300, 0, 0, 4081, 0, 1, 28, 0, 133, 18, 140, 50, 49, 16, 1038, 164, 21, 5, 2, 3355, 0, 4025, 1, 6, 19, 0, 0, 100, 2811, 3073, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 4095, 1273, 2990, 4020, 7, 4096, 79, 4096, 2999, 0, 0, 3, 4090, 1510, 4092, 0, 3974, 6, 1679, 0, 0, 0, 0, 4090, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 3, 3, 0, 16, 309, 0, 3338, 0, 0, 15, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 251, 3486, 2715, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 2, 0, 3926, 2253, 1184, 0, 0, 0, 0, 0, 0, 467, 3536, 431, 1, 0, 299, 0, 3514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 77, 1517, 2853, 0, 0, 2385, 0, 0, 0, 0, 0, 0, 0, 0, 176, 3578, 2, 0, 2, 0, 3014, 0, 449, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 4095, 0, 0, 1, 0, 2394, 1838, 0, 0, 0, 0, 0, 0, 0, 0, 3, 181, 1386, 0, 2408, 0, 4056, 1698, 0, 0, 0, 0, 0, 0, 0, 10, 3, 4074, 0, 4092, 1, 4, 197, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 407, 2987, 3344, 0, 0, 1928, 750, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7367, 8248, 6833, 7221, 8334, 9732, 8381, 9420] avg=8192.0000 max=9732 max_rank=5 min=6833 min_rank=2 imbalance_ratio=1.187988 max_load_reduction=55804 max_load_reduction_pct=0.851501 moved_to_echo=35867 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4094, 2209, 1894, 0, 0, 0, 0, 0, 0, 3605, 909, 688, 0, 0, 953, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1519, 2144, 0, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 2874, 3822, 1, 0, 6, 0, 3999, 3, 450, 0, 0, 0, 0, 0, 0, 0, 17, 0, 13, 3430, 0, 0, 1, 0, 2397, 1800, 0, 0, 0, 0, 0, 0, 61, 21, 3, 2, 333, 0, 1361, 1, 4095, 1698, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 2, 4090, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3, 640, 3983, 0, 0, 1930, 1199, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 7237, 7555, 11155, 7658, 7575, 8388, 7771] avg=8192.0000 max=11155 max_rank=3 min=7237 min_rank=1 imbalance_ratio=1.361694 max_load_reduction=54381 max_load_reduction_pct=0.829788 moved_to_echo=31392 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 0, 2, 0, 4086, 2124, 2049, 0, 0, 0, 0, 0, 0, 3975, 2935, 745, 0, 0, 296, 0, 2913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1470, 3126, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 682, 3185, 0, 0, 0, 0, 4072, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3961, 1, 0, 0, 0, 2070, 1732, 0, 0, 0, 0, 0, 0, 14, 384, 6, 5, 200, 0, 637, 0, 4096, 1469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 3794, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 81, 196, 3285, 0, 4, 1869, 1298, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 10864, 8641, 8375, 7780, 6811, 8066, 6734] avg=8192.0000 max=10864 max_rank=1 min=6734 min_rank=7 imbalance_ratio=1.326172 max_load_reduction=54672 max_load_reduction_pct=0.834229 moved_to_echo=29391 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 1, 0, 8, 0, 4092, 2237, 2008, 0, 0, 0, 0, 0, 0, 1896, 2296, 730, 0, 0, 477, 0, 688, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1509, 3508, 0, 0, 3571, 0, 0, 0, 0, 0, 0, 0, 0, 465, 3793, 0, 0, 1, 0, 4047, 0, 447, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4089, 7, 4, 0, 0, 2390, 1822, 0, 0, 0, 0, 0, 0, 17, 4, 0, 21, 1469, 0, 2744, 5, 4094, 1693, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4082, 0, 4002, 0, 0, 196, 0, 0, 0, 0, 0, 0, 0, 1, 23, 6, 247, 2685, 933, 0, 2, 1918, 1272, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8350, 6087, 8589, 8753, 8336, 10047, 8287, 7087] avg=8192.0000 max=10047 max_rank=5 min=6087 min_rank=1 imbalance_ratio=1.226440 max_load_reduction=55489 max_load_reduction_pct=0.846695 moved_to_echo=33757 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 4095, 2256, 2013, 0, 0, 0, 0, 0, 0, 1273, 2990, 732, 7, 0, 79, 0, 2999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1516, 1510, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1679, 0, 0, 0, 0, 4090, 0, 449, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 2396, 1840, 0, 0, 0, 0, 0, 0, 3, 3, 0, 16, 309, 0, 3338, 0, 4092, 1700, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4096, 0, 4094, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 3486, 2715, 0, 0, 1928, 1275, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8365, 8080, 7003, 6224, 8346, 9461, 8402, 9655] avg=8192.0000 max=9655 max_rank=7 min=6224 min_rank=3 imbalance_ratio=1.178589 max_load_reduction=55881 max_load_reduction_pct=0.852676 moved_to_echo=35864 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[56, 0, 1, 0, 0, 0, 1, 4054, 2200, 2053, 0, 0, 0, 0, 0, 0, 3351, 1615, 737, 0, 0, 2, 0, 1974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 1436, 3848, 0, 17, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 73, 3172, 0, 3, 29, 0, 3694, 27, 432, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 4081, 0, 1, 28, 0, 2380, 1792, 0, 0, 0, 0, 0, 0, 133, 18, 140, 50, 49, 16, 1038, 164, 4058, 1687, 0, 0, 0, 0, 0, 0, 21, 5, 2, 3355, 0, 4025, 1, 6, 190, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 100, 2811, 3073, 0, 0, 1843, 1300, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8365, 7679, 9376, 7430, 8582, 7353, 7605, 9146] avg=8192.0000 max=9376 max_rank=2 min=7353 min_rank=5 imbalance_ratio=1.144531 max_load_reduction=56160 max_load_reduction_pct=0.856934 moved_to_echo=32686 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 1, 68, 0, 4003, 2129, 2028, 0, 0, 0, 0, 0, 0, 958, 2981, 737, 1, 0, 218, 0, 3095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1467, 3511, 0, 27, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 925, 3463, 15, 0, 1, 0, 3887, 0, 435, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4022, 5, 0, 2, 0, 2390, 1735, 0, 0, 0, 0, 0, 0, 83, 47, 0, 14, 365, 3, 421, 100, 4093, 1694, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3927, 0, 3989, 8, 1, 190, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 167, 3797, 1318, 2, 0, 1864, 1284, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 7990, 9050, 8726, 8161, 6820, 8120, 8437] avg=8192.0000 max=9050 max_rank=2 min=6820 min_rank=5 imbalance_ratio=1.104736 max_load_reduction=56486 max_load_reduction_pct=0.861908 moved_to_echo=31538 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4092, 2255, 2047, 0, 0, 0, 0, 0, 0, 3096, 3630, 745, 0, 0, 22, 0, 1858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1517, 2892, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3137, 0, 0, 6, 0, 4059, 0, 450, 0, 0, 0, 0, 0, 0, 0, 4, 1, 2, 4096, 0, 0, 0, 0, 2396, 1840, 0, 0, 0, 0, 0, 0, 251, 8, 0, 0, 40, 0, 1642, 0, 4096, 1700, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 3976, 13, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 50, 2043, 1944, 0, 24, 1928, 1296, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8395, 9351, 8489, 7652, 8339, 7737, 8287, 7286] avg=8192.0000 max=9351 max_rank=1 min=7286 min_rank=7 imbalance_ratio=1.141479 max_load_reduction=56185 max_load_reduction_pct=0.857315 moved_to_echo=31649 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 32512, 17664, 15360, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 1408, 18688, 20608, 128, 128, 15232, 9728, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[18688, 20992, 5632, 128, 0, 2432, 0, 18176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5248, 25856, 128, 128, 128, 0, 30976, 128, 3584, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 12032, 23424, 0, 128, 30080, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 31872, 128, 32128, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[640, 512, 256, 384, 4224, 128, 13696, 384, 32768, 13440, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 128, 128, 31872, 128, 128, 128, 0, 18816, 14464, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[2, 6, 2, 0, 0, 1, 0, 0, 0, 4096, 0, 14, 28, 4094, 4089, 0, 11, 4095, 0, 1, 4093, 1, 0, 5, 4094, 0, 204, 0, 0, 4095, 0, 0, 4094, 0, 0, 0, 1370, 3936, 0, 0, 4096, 3886, 1, 0, 0, 0, 0, 0, 147, 2685, 4096, 1, 2, 393, 0, 2, 4094, 0, 5, 0, 3706, 0, 1, 4090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 28, 0, 558, 19, 0, 4095, 0, 0, 468, 3966, 4096, 0, 0, 4086, 2, 0, 4096, 0, 0, 0, 4096, 3, 3, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 2128, 305, 0, 1, 4071, 2488, 3, 0, 0, 0, 0, 0, 430, 3536, 4096, 0, 0, 1726, 0, 0, 4095, 9, 1897, 0, 3162, 0, 0, 3874] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 18, 3979, 0, 2, 1579, 4096, 4096, 0, 2, 4095, 0, 0, 4095, 0, 0, 0, 4096, 0, 11, 0, 0, 4091, 35, 3, 4096, 0, 0, 0, 4049, 607, 0, 0, 3717, 3123, 106, 0, 0, 0, 0, 1, 0, 2330, 4096, 0, 0, 3930, 0, 0, 4096, 0, 355, 0, 749, 2, 1, 4079] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 20, 0, 0, 1, 0, 6, 1, 0, 4093, 0, 0, 177, 3939, 4096, 4, 40, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 203, 0, 0, 4096, 0, 0, 4090, 0, 0, 11, 3633, 3468, 0, 0, 4096, 2980, 2, 0, 0, 17, 0, 16, 310, 3902, 4096, 0, 0, 418, 0, 0, 4096, 0, 14, 0, 1327, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 39, 2, 6, 6, 0, 0, 3, 39, 4096, 0, 12, 176, 4067, 4076, 0, 0, 4041, 0, 0, 4093, 0, 0, 0, 4095, 0, 564, 5, 0, 4086, 2, 0, 4096, 0, 1, 0, 3667, 93, 0, 0, 3345, 4060, 7, 0, 0, 0, 1, 0, 1027, 28, 4096, 0, 0, 3592, 0, 0, 4095, 3, 196, 0, 4088, 0, 0, 3733] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 730, 0, 3, 2, 3, 0, 44, 0, 4088, 0, 2, 597, 3965, 4094, 14, 4, 4076, 16, 8, 4090, 0, 0, 33, 4063, 0, 38, 0, 5, 4067, 34, 0, 4042, 99, 0, 0, 1994, 1725, 4, 6, 4083, 3971, 1, 43, 6, 36, 2, 6, 121, 3213, 4060, 3, 21, 251, 32, 0, 4086, 40, 115, 20, 3489, 0, 1, 4090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 0, 0, 4, 10, 0, 4076, 2, 2, 2078, 4053, 4092, 0, 0, 4088, 0, 2, 4092, 6, 0, 1, 4086, 0, 58, 27, 0, 3938, 0, 0, 4042, 0, 4, 63, 2084, 1776, 0, 1, 3837, 3900, 0, 2, 0, 106, 1, 16, 78, 1676, 4093, 0, 0, 2034, 0, 0, 4053, 0, 1260, 0, 1805, 0, 0, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2, 0, 0, 1, 0, 4095, 0, 0, 1810, 3391, 4096, 2, 5, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4041, 0, 0, 4085, 0, 0, 0, 3609, 3273, 0, 0, 4092, 4073, 288, 0, 0, 0, 0, 0, 169, 2997, 4096, 0, 0, 206, 0, 0, 4096, 0, 7, 0, 716, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 6, 2, 0, 0, 1, 0, 0, 4089, 2789, 712, 417, 0, 0, 0, 0, 0, 3384, 0, 14, 28, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4095, 0, 1, 4090, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 204, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3466, 0, 0, 0, 1370, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3886, 1, 0, 0, 0, 0, 0, 306, 211, 111, 3, 0, 0, 0, 0, 147, 2685, 3790, 1, 2, 393, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1305, 0, 5, 0, 3706, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8018, 7520, 8203, 8282, 8772, 8614, 7020, 9107] avg=8192.0000 max=9107 max_rank=7 min=7020 min_rank=6 imbalance_ratio=1.111694 max_load_reduction=56429 max_load_reduction_pct=0.861038 moved_to_echo=33513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 28, 0, 558, 19, 4096, 2790, 712, 417, 0, 0, 0, 0, 0, 3383, 0, 0, 468, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 3, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 1, 0, 2128, 305, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 2488, 3, 0, 0, 0, 0, 0, 306, 211, 111, 3, 0, 0, 0, 0, 430, 3536, 3790, 0, 0, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1305, 9, 1897, 0, 3162, 0, 0, 3874, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8626, 7817, 8181, 8087, 5903, 7193, 9482, 10247] avg=8192.0000 max=10247 max_rank=7 min=5903 min_rank=4 imbalance_ratio=1.250854 max_load_reduction=55289 max_load_reduction_pct=0.843643 moved_to_echo=32825 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4096, 2790, 692, 417, 0, 0, 0, 0, 18, 3287, 0, 2, 1579, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 11, 0, 0, 4091, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 0, 0, 4049, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3717, 3123, 106, 0, 0, 0, 0, 1, 306, 211, 111, 3, 0, 0, 0, 0, 0, 2330, 3790, 0, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1306, 0, 355, 0, 749, 2, 1, 4079, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7996, 8982, 8189, 8125, 8124, 7578, 10050, 6492] avg=8192.0000 max=10050 max_rank=6 min=6492 min_rank=7 imbalance_ratio=1.226807 max_load_reduction=55486 max_load_reduction_pct=0.846649 moved_to_echo=32244 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 39, 2, 6, 6, 0, 0, 3, 4076, 2790, 712, 417, 0, 0, 0, 0, 39, 3384, 0, 12, 176, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 564, 5, 0, 4086, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 1, 0, 3667, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3345, 4060, 7, 0, 0, 0, 1, 0, 306, 211, 111, 3, 0, 0, 0, 0, 1027, 28, 3790, 0, 0, 3592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1305, 3, 196, 0, 4088, 0, 0, 3733, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8051, 7678, 8131, 8641, 7229, 8044, 8437, 9325] avg=8192.0000 max=9325 max_rank=7 min=7229 min_rank=4 imbalance_ratio=1.138306 max_load_reduction=56211 max_load_reduction_pct=0.857712 moved_to_echo=33035 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 730, 0, 3, 2, 3, 0, 44, 4094, 2787, 716, 414, 0, 0, 0, 0, 0, 3372, 0, 2, 597, 3965, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4076, 16, 8, 4086, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 38, 0, 5, 4067, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3415, 99, 0, 0, 1994, 1725, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3971, 1, 43, 6, 36, 2, 6, 311, 213, 114, 4, 0, 0, 0, 0, 121, 3213, 3749, 3, 21, 251, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1299, 40, 115, 20, 3489, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8793, 7950, 8223, 8093, 7243, 8790, 7390, 9054] avg=8192.0000 max=9054 max_rank=7 min=7243 min_rank=4 imbalance_ratio=1.105225 max_load_reduction=56482 max_load_reduction_pct=0.861847 moved_to_echo=32477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 20, 0, 0, 1, 0, 6, 1, 4096, 2790, 712, 416, 0, 0, 0, 0, 0, 3381, 0, 0, 177, 3939, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 203, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3463, 0, 0, 11, 3633, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2980, 2, 0, 0, 17, 0, 16, 306, 211, 111, 3, 0, 0, 0, 0, 310, 3902, 3790, 0, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1306, 0, 14, 0, 1327, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8042, 7501, 8229, 8284, 10575, 7742, 8420, 6743] avg=8192.0000 max=10575 max_rank=4 min=6743 min_rank=7 imbalance_ratio=1.290894 max_load_reduction=54961 max_load_reduction_pct=0.838638 moved_to_echo=33480 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 0, 0, 4, 10, 4092, 2761, 709, 411, 0, 0, 0, 0, 0, 3367, 2, 2, 2078, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 2, 4089, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 58, 27, 0, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3422, 0, 4, 63, 2084, 1776, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3837, 3900, 0, 2, 0, 106, 1, 16, 306, 209, 111, 3, 0, 0, 0, 0, 78, 1676, 3787, 0, 0, 2034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1292, 0, 1260, 0, 1805, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7991, 9502, 8186, 7998, 7350, 8491, 7575, 8443] avg=8192.0000 max=9502 max_rank=1 min=7350 min_rank=4 imbalance_ratio=1.159912 max_load_reduction=56034 max_load_reduction_pct=0.855011 moved_to_echo=31859 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2, 0, 0, 1, 4096, 2790, 712, 416, 0, 0, 0, 0, 0, 3383, 0, 0, 1810, 3391, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3458, 0, 0, 0, 3609, 3273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4073, 288, 0, 0, 0, 0, 0, 306, 211, 111, 3, 0, 0, 0, 0, 169, 2997, 3790, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1306, 0, 7, 0, 716, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8019, 8586, 8194, 8026, 10340, 9084, 7162, 6125] avg=8192.0000 max=10340 max_rank=4 min=6125 min_rank=7 imbalance_ratio=1.262207 max_load_reduction=55196 max_load_reduction_pct=0.842224 moved_to_echo=32711 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[2304, 20480, 30336, 128, 128, 12672, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 896, 128, 128, 128, 128, 640, 128, 32768, 22400, 5760, 3328, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 27008, 128, 128, 7040, 31616, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[10496, 128, 3968, 128, 19072, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[31360, 28544, 512, 128, 128, 256, 128, 128, 2560, 1792, 896, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31872, 128, 1152, 128, 128, 32512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[27648, 128, 128, 128, 22656, 15232, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4093, 4093, 0, 0, 0, 4094, 2, 0, 0, 4096, 1, 6, 0, 0, 4086, 0, 11, 2126, 1, 51, 4095, 4094, 7, 0, 4095, 0, 1, 0, 0, 0, 4090, 0, 1, 4090, 2945, 0, 50, 1, 0, 2, 0, 0, 4095, 0, 4096, 53, 0, 2, 2954, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4095, 6, 0, 4094, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4079, 4084, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 4092, 0, 0, 4096, 6, 16, 4094, 4064, 448, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4078, 3138, 0, 3, 0, 0, 1, 0, 0, 4043, 0, 4093, 0, 0, 13, 371, 0, 0, 0, 0, 216, 0, 0, 0, 1, 4096, 23, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4071, 3877, 0, 0, 0, 4096, 0, 1, 0, 4083, 0, 288, 0, 0, 4095, 0, 0, 3113, 0, 3088, 4096, 4096, 1729, 0, 4096, 3, 0, 1, 0, 0, 4096, 0, 0, 2150, 2047, 0, 39, 0, 0, 1, 0, 0, 4091, 0, 4095, 0, 4, 0, 31, 0, 0, 0, 0, 49, 7, 0, 1, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4081, 3069, 0, 2, 0, 4096, 0, 0, 4, 4087, 0, 26, 0, 0, 4056, 0, 0, 2937, 0, 186, 4092, 4096, 210, 0, 4076, 0, 0, 0, 0, 0, 4095, 0, 0, 4088, 2451, 17, 15, 0, 0, 12, 0, 0, 3673, 22, 4070, 0, 9, 2, 3864, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 3973, 0, 607, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 3188, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4096, 3942, 0, 49, 0, 4090, 0, 5, 0, 4095, 1, 37, 0, 0, 4089, 1, 31, 4090, 32, 344, 4095, 4011, 1, 0, 4095, 0, 7, 1, 0, 0, 4087, 0, 15, 3960, 2993, 0, 0, 0, 9, 0, 0, 0, 4067, 10, 4096, 650, 0, 1, 150, 11, 0, 0, 0, 173, 1, 10, 0, 1, 4094, 0, 1, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4059, 3721, 1, 10, 5, 4073, 43, 13, 42, 4050, 17, 0, 25, 141, 3910, 10, 0, 2677, 0, 161, 4057, 4066, 0, 0, 4083, 1, 1, 1, 4, 1, 4079, 2, 0, 4045, 3239, 0, 27, 66, 0, 0, 0, 18, 3305, 0, 4073, 0, 0, 11, 2987, 16, 4, 0, 29, 308, 0, 0, 6, 0, 4020, 0, 13, 4079, 15, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4096, 4069, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2697, 0, 1661, 4096, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3622, 0, 1, 0, 0, 0, 0, 0, 4071, 0, 4095, 3, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4093, 4093, 0, 0, 0, 224, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 1, 6, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2126, 1, 51, 0, 3921, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 4090, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 2945, 0, 50, 1, 0, 2, 1252, 130, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 53, 0, 2, 43, 29, 0, 0, 0, 0, 0, 0, 2954, 0, 0, 0, 0, 7, 0, 0, 4095, 2618, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8412, 8155, 6117, 8191, 8471, 8318, 9674, 8198] avg=8192.0000 max=9674 max_rank=6 min=6117 min_rank=2 imbalance_ratio=1.180908 max_load_reduction=55862 max_load_reduction_pct=0.852386 moved_to_echo=34661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4079, 4084, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4062, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 16, 0, 3892, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3138, 0, 3, 0, 0, 1, 1252, 129, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 4093, 0, 0, 13, 43, 29, 0, 0, 0, 0, 0, 0, 371, 0, 0, 0, 0, 216, 0, 0, 4094, 2619, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 23, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8388, 8155, 8458, 8197, 8601, 8221, 7300, 8216] avg=8192.0000 max=8601 max_rank=4 min=7300 min_rank=6 imbalance_ratio=1.049927 max_load_reduction=56935 max_load_reduction_pct=0.868759 moved_to_echo=32338 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4071, 3877, 0, 0, 0, 225, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 288, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3113, 0, 3088, 0, 3923, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 1, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2150, 2047, 0, 39, 0, 0, 1, 1252, 130, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4095, 0, 4, 0, 43, 29, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 49, 7, 0, 4096, 2619, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8432, 11853, 8201, 5619, 8262, 6802, 8193] avg=8192.0000 max=11853 max_rank=2 min=5619 min_rank=4 imbalance_ratio=1.446899 max_load_reduction=53683 max_load_reduction_pct=0.819138 moved_to_echo=28876 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4081, 3069, 0, 2, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4053, 0, 26, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2937, 0, 186, 0, 3923, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4095, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 2451, 17, 15, 0, 0, 12, 1252, 130, 0, 0, 0, 0, 0, 0, 0, 0, 3673, 22, 4070, 0, 9, 2, 43, 29, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 0, 0, 4, 0, 0, 4092, 2619, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7377, 8139, 7256, 8176, 7965, 7848, 10579, 8196] avg=8192.0000 max=10579 max_rank=6 min=7256 min_rank=2 imbalance_ratio=1.291382 max_load_reduction=54957 max_load_reduction_pct=0.838577 moved_to_echo=34588 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 4069, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2697, 0, 1661, 0, 3923, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3622, 0, 1, 0, 0, 0, 1252, 130, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4095, 3, 0, 0, 43, 29, 0, 0, 0, 0, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 4096, 2619, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8390, 8158, 8283, 8197, 9101, 8241, 6974, 8192] avg=8192.0000 max=9101 max_rank=4 min=6974 min_rank=6 imbalance_ratio=1.110962 max_load_reduction=56435 max_load_reduction_pct=0.861130 moved_to_echo=32508 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 4095, 0, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 607, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3188, 0, 1, 0, 0, 0, 1252, 130, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 43, 29, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 4096, 2619, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8416, 8157, 8503, 8197, 8666, 8264, 7140, 8193] avg=8192.0000 max=8666 max_rank=4 min=7140 min_rank=6 imbalance_ratio=1.057861 max_load_reduction=56870 max_load_reduction_pct=0.867767 moved_to_echo=32263 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4059, 3721, 1, 10, 5, 212, 43, 13, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4006, 17, 0, 25, 141, 3910, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2677, 0, 161, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 1, 1, 4, 1, 4079, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 3239, 0, 27, 66, 0, 0, 1251, 130, 0, 0, 0, 0, 0, 0, 0, 18, 3305, 0, 4073, 0, 0, 11, 48, 32, 0, 0, 0, 0, 0, 0, 2987, 16, 4, 0, 29, 308, 0, 0, 4057, 2610, 0, 0, 0, 0, 0, 0, 6, 0, 4020, 0, 13, 4079, 15, 22, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8064, 8151, 6726, 8182, 8758, 7487, 10011, 8157] avg=8192.0000 max=10011 max_rank=6 min=6726 min_rank=2 imbalance_ratio=1.222046 max_load_reduction=55525 max_load_reduction_pct=0.847244 moved_to_echo=34413 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 3942, 0, 49, 0, 223, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 1, 37, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4090, 32, 344, 0, 3842, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 7, 1, 0, 0, 4087, 0, 5, 0, 0, 0, 0, 0, 0, 0, 15, 3960, 2993, 0, 0, 0, 9, 0, 1251, 127, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 10, 4096, 650, 0, 1, 42, 29, 0, 0, 0, 0, 0, 0, 150, 11, 0, 0, 0, 173, 1, 10, 4095, 2616, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8315, 8189, 8340, 8195, 8355, 8895, 7056, 8191] avg=8192.0000 max=8895 max_rank=5 min=7056 min_rank=6 imbalance_ratio=1.085815 max_load_reduction=56641 max_load_reduction_pct=0.864273 moved_to_echo=32497 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 31488, 128, 32768, 768, 128, 128, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 25728, 128, 6144, 0, 31360, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11136, 128, 128, 0, 128, 768, 128, 128, 32768, 20992, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 30976, 128, 128, 128, 1792, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 30720, 23680, 128, 256, 128, 128, 128, 10112, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32512, 128, 384, 128, 256, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1469, 3887, 2575, 0, 0, 0, 0, 0, 10, 117, 0, 4096, 0, 0, 4096, 4095, 4087, 0, 0, 31, 0, 4093, 0, 2, 0, 2, 0, 1, 0, 4096, 0, 0, 4096, 4096, 4095, 13, 0, 0, 4068, 24, 2, 3972, 0, 51, 0, 0, 0, 0, 0, 0, 21, 0, 1, 0, 0, 0, 4095, 0, 200, 4087, 0, 4058, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 3, 4095, 4016, 2, 2, 2, 0, 6, 0, 0, 0, 4096, 0, 0, 4091, 4072, 4094, 0, 3, 0, 9, 4019, 2, 0, 2, 0, 0, 0, 0, 4092, 0, 0, 4094, 4081, 4074, 0, 5, 13, 4090, 0, 0, 1816, 2, 50, 2, 2, 0, 0, 0, 0, 58, 13, 2, 0, 0, 0, 4093, 0, 2729, 3712, 1, 4093, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 1155, 4007, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4079, 4096, 0, 0, 0, 0, 3956, 0, 0, 2, 30, 0, 1, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 163, 4096, 0, 0, 4044, 0, 52, 0, 0, 0, 1, 0, 0, 0, 91, 0, 0, 0, 0, 4044, 0, 2546, 3468, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4094, 2446, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 3, 4095, 4095, 4096, 1, 0, 0, 2, 4088, 0, 0, 1, 14, 0, 13, 0, 4093, 0, 5, 4079, 4096, 4096, 1, 0, 0, 4096, 0, 2, 1645, 0, 58, 0, 0, 0, 0, 0, 2, 0, 61, 0, 0, 0, 0, 4042, 0, 4036, 4069, 7, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4075, 4013, 3, 0, 0, 1, 0, 4, 84, 117, 4093, 0, 0, 4086, 4096, 4089, 0, 11, 15, 0, 3741, 1, 0, 3, 0, 1, 3, 0, 4085, 0, 0, 4093, 4094, 4091, 0, 0, 16, 4096, 0, 1, 254, 2, 297, 0, 2, 3, 1, 6, 0, 0, 39, 0, 0, 0, 4, 4096, 0, 3730, 4082, 9, 4096, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 14, 4078, 2835, 0, 0, 0, 0, 0, 0, 8, 1, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 6, 3449, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 908, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1015, 0, 29, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 0, 4096, 0, 3648, 4094, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4073, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 102, 3808, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 16, 4096, 4096, 4096, 0, 0, 0, 4096, 2, 0, 1198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4095, 0, 4096, 3346, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[5, 138, 4069, 1999, 50, 0, 1, 0, 22, 17, 0, 0, 4071, 30, 0, 4046, 3876, 4068, 23, 26, 5, 0, 4059, 9, 0, 5, 0, 0, 0, 14, 4037, 21, 28, 4082, 4048, 3968, 0, 6, 156, 4087, 33, 0, 2853, 4, 0, 11, 0, 7, 0, 0, 1, 106, 509, 0, 16, 41, 110, 4084, 10, 2826, 4073, 0, 3882, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4095, 4016, 2, 2, 2, 0, 770, 138, 41, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4055, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 198, 0, 3, 0, 9, 4019, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4092, 0, 2124, 1982, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 4074, 0, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1816, 2, 50, 2, 2, 0, 4093, 1772, 161, 0, 0, 0, 0, 0, 0, 0, 0, 58, 13, 2, 0, 0, 4094, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2729, 3712, 1, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9069, 8152, 8303, 8200, 8035, 7898, 8096, 7783] avg=8192.0000 max=9069 max_rank=0 min=7783 min_rank=7 imbalance_ratio=1.107056 max_load_reduction=56467 max_load_reduction_pct=0.861618 moved_to_echo=31812 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1469, 3887, 2575, 0, 0, 0, 0, 763, 138, 41, 0, 0, 0, 0, 0, 0, 10, 117, 0, 4055, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 198, 0, 0, 31, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 1, 0, 4096, 0, 2120, 1965, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 4095, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 2, 3972, 0, 51, 0, 0, 0, 4095, 1769, 161, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 1, 0, 0, 4096, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 4087, 0, 1330, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8873, 8278, 8417, 8186, 8066, 10074, 8025, 5617] avg=8192.0000 max=10074 max_rank=5 min=5617 min_rank=7 imbalance_ratio=1.229736 max_load_reduction=55462 max_load_reduction_pct=0.846283 moved_to_echo=31782 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1155, 4007, 1032, 0, 0, 0, 0, 770, 138, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 198, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 30, 0, 1, 0, 4096, 0, 2125, 1983, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 4096, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 52, 0, 0, 0, 4044, 1773, 161, 0, 0, 0, 0, 0, 1, 0, 0, 0, 91, 0, 0, 0, 4096, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2546, 3468, 0, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7143, 8152, 8233, 8237, 8217, 10074, 8123, 7357] avg=8192.0000 max=10074 max_rank=5 min=7143 min_rank=0 imbalance_ratio=1.229736 max_load_reduction=55462 max_load_reduction_pct=0.846283 moved_to_echo=33771 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 2446, 0, 0, 0, 0, 770, 138, 41, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4055, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 198, 1, 0, 0, 2, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 13, 0, 4093, 0, 2125, 1983, 0, 0, 0, 0, 0, 0, 5, 0, 3958, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1645, 0, 58, 0, 0, 0, 4042, 1773, 161, 0, 0, 0, 0, 0, 0, 0, 2, 0, 61, 0, 0, 0, 4079, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 4069, 7, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7489, 8161, 8384, 8229, 8060, 7681, 8077, 9455] avg=8192.0000 max=9455 max_rank=7 min=7489 min_rank=0 imbalance_ratio=1.154175 max_load_reduction=56081 max_load_reduction_pct=0.855728 moved_to_echo=33273 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4073, 3835, 0, 0, 0, 0, 770, 138, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 198, 0, 0, 0, 102, 3808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 2125, 1983, 0, 0, 0, 0, 0, 0, 16, 0, 3958, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1198, 0, 0, 0, 0, 0, 4095, 1773, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3346, 0, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8857, 8151, 8204, 8205, 8070, 7229, 8035, 8785] avg=8192.0000 max=8857 max_rank=0 min=7229 min_rank=5 imbalance_ratio=1.081177 max_load_reduction=56679 max_load_reduction_pct=0.864853 moved_to_echo=32119 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 138, 4069, 1999, 50, 0, 1, 0, 736, 143, 47, 0, 0, 0, 0, 0, 22, 17, 0, 0, 4024, 30, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 186, 23, 26, 5, 0, 4059, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 14, 4037, 21, 2116, 1886, 0, 0, 0, 0, 0, 0, 28, 0, 3905, 3968, 0, 6, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 2853, 4, 0, 11, 0, 7, 4084, 1766, 161, 0, 0, 0, 0, 0, 0, 0, 1, 106, 509, 0, 16, 41, 4082, 3926, 0, 0, 0, 0, 0, 0, 110, 0, 10, 2826, 4073, 0, 1260, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7188, 8139, 8184, 8079, 8063, 8919, 8681, 8283] avg=8192.0000 max=8919 max_rank=5 min=7188 min_rank=0 imbalance_ratio=1.088745 max_load_reduction=56617 max_load_reduction_pct=0.863907 moved_to_echo=33946 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4075, 4013, 3, 0, 0, 1, 770, 138, 41, 0, 0, 0, 0, 0, 0, 4, 84, 117, 4052, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 198, 0, 11, 15, 0, 3741, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 3, 0, 4085, 0, 2121, 1983, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 4091, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 254, 2, 297, 0, 2, 3, 4096, 1770, 162, 0, 0, 0, 0, 0, 1, 6, 0, 0, 39, 0, 0, 0, 4093, 3934, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3730, 4082, 9, 1343, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9041, 8343, 8062, 8196, 8063, 6587, 8073, 9171] avg=8192.0000 max=9171 max_rank=7 min=6587 min_rank=5 imbalance_ratio=1.119507 max_load_reduction=56365 max_load_reduction_pct=0.860062 moved_to_echo=31894 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 14, 4078, 2835, 0, 0, 0, 0, 770, 138, 41, 0, 0, 0, 0, 0, 0, 0, 8, 1, 4055, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 198, 0, 0, 0, 6, 3449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2125, 1983, 0, 0, 0, 0, 0, 0, 908, 0, 3958, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 0, 29, 0, 0, 0, 4096, 1773, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 4096, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 4094, 0, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7876, 8160, 7749, 8204, 8962, 7074, 8426, 9085] avg=8192.0000 max=9085 max_rank=7 min=7074 min_rank=5 imbalance_ratio=1.109009 max_load_reduction=56451 max_load_reduction_pct=0.861374 moved_to_echo=33547 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32512, 1664, 128, 128, 128, 128, 31232, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 256, 128, 32512, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 2816, 32384, 22784, 128, 128, 128, 128, 6144, 1152, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 16896, 128, 640, 128, 128, 128, 32768, 14208, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 256, 1152, 128, 128, 128, 32768, 31488, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1024, 0, 31616, 32640, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 23936, 30976, 128, 10752, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 128, 17024, 15872, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 800, 0, 0, 0, 3742, 39, 0, 4095, 4096, 19, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 3970, 0, 314, 4096, 4095, 0, 4096, 0, 0, 3261, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 153, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4069, 0, 5, 4, 0, 10, 0, 4091, 3288, 24, 4089, 4089, 7, 0, 0, 0, 0, 3, 24, 4095, 6, 0, 0, 3392, 1, 0, 4076, 3901, 0, 4084, 0, 1, 1495, 0, 2, 0, 0, 6, 0, 16, 0, 4094, 0, 0, 4094, 1, 0, 4060, 1, 4072, 4091, 0, 7, 1, 0, 0, 6, 0, 304, 4023, 2, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 196, 0, 25, 0, 3967, 19, 720, 4046, 4096, 1633, 0, 94, 0, 0, 0, 142, 4086, 0, 0, 5, 4092, 0, 1, 4096, 4096, 0, 4096, 1, 0, 3902, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 4050, 4081, 0, 3, 0, 0, 0, 29, 0, 0, 1636, 38, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 3989, 4, 0, 418, 0, 0, 0, 3801, 837, 199, 4091, 4095, 160, 9, 0, 0, 0, 0, 59, 4095, 0, 0, 0, 4095, 0, 0, 4094, 4096, 0, 3988, 0, 0, 3104, 4, 0, 13, 0, 0, 0, 6, 0, 4096, 0, 0, 4089, 0, 0, 4088, 0, 4086, 3974, 0, 0, 6, 0, 0, 3, 0, 18, 4017, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 3203, 0, 5, 759, 0, 3, 0, 3974, 2667, 20, 4080, 4093, 14, 0, 24, 2, 5, 0, 0, 4094, 1, 2, 0, 4057, 0, 0, 4096, 4072, 0, 4093, 20, 1, 1699, 55, 0, 2, 0, 0, 0, 199, 1, 4094, 0, 0, 4095, 1, 0, 4096, 0, 4094, 3817, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4024, 0, 0, 487, 0, 0, 0, 3776, 477, 1, 4085, 4096, 83, 0, 1, 0, 0, 0, 36, 4096, 0, 0, 0, 3568, 0, 13, 4095, 4094, 0, 3859, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4095, 4079, 0, 5, 0, 0, 0, 0, 0, 85, 4095, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[59, 4095, 0, 0, 0, 25, 18, 8, 4049, 95, 338, 4071, 4077, 73, 0, 163, 9, 38, 13, 0, 4055, 43, 0, 0, 4089, 2, 0, 3912, 4083, 0, 4053, 17, 27, 3993, 0, 0, 0, 12, 0, 0, 0, 0, 4083, 1, 3, 4069, 19, 34, 4045, 0, 3740, 4002, 0, 0, 0, 23, 33, 0, 6, 0, 4048, 1, 12, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4062, 0, 0, 0, 0, 0, 0, 3748, 3512, 1, 4094, 4096, 0, 0, 1, 0, 0, 0, 92, 4096, 0, 0, 0, 2465, 0, 1, 4096, 4081, 0, 4036, 0, 0, 2322, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4094, 0, 4095, 4091, 0, 234, 0, 0, 0, 0, 0, 31, 4095, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4069, 0, 5, 4, 0, 10, 0, 2166, 1600, 29, 0, 0, 0, 0, 0, 4091, 3288, 24, 2489, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 24, 4095, 6, 0, 0, 2612, 1358, 0, 0, 0, 0, 0, 0, 3392, 1, 0, 0, 377, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1495, 0, 2, 0, 0, 6, 0, 4089, 1101, 0, 0, 0, 0, 0, 0, 16, 0, 4065, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 1, 4072, 4091, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 304, 4023, 2, 0, 2, 2975, 1360, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7883, 9899, 8098, 7854, 6694, 8176, 8260, 8672] avg=8192.0000 max=9899 max_rank=1 min=6694 min_rank=4 imbalance_ratio=1.208374 max_load_reduction=55637 max_load_reduction_pct=0.848953 moved_to_echo=31802 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 800, 0, 0, 0, 2274, 1602, 29, 0, 0, 0, 0, 0, 3742, 39, 0, 2493, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 2634, 1425, 0, 0, 0, 0, 0, 0, 3970, 0, 314, 0, 396, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3261, 1, 0, 0, 0, 0, 0, 4096, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 4096, 0, 0, 0, 2989, 1371, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8802, 6293, 8155, 8776, 8465, 8163, 8273, 8609] avg=8192.0000 max=8802 max_rank=0 min=6293 min_rank=1 imbalance_ratio=1.074463 max_load_reduction=56734 max_load_reduction_pct=0.865692 moved_to_echo=33510 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 196, 0, 25, 0, 2275, 1583, 29, 0, 0, 0, 0, 0, 3967, 19, 720, 2463, 0, 1633, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 4086, 0, 0, 5, 2634, 1426, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 395, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 0, 0, 0, 1, 0, 4096, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 4050, 4081, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 1636, 38, 0, 3, 2988, 1371, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8896, 8293, 8585, 9107, 8163, 8223, 6065] avg=8192.0000 max=9107 max_rank=4 min=6065 min_rank=7 imbalance_ratio=1.111694 max_load_reduction=56429 max_load_reduction_pct=0.861038 moved_to_echo=31558 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3989, 4, 0, 418, 0, 0, 0, 2275, 1601, 29, 0, 0, 0, 0, 0, 3801, 837, 199, 2490, 0, 160, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4095, 0, 0, 0, 2630, 1426, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 395, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3104, 4, 0, 13, 0, 0, 0, 4095, 1107, 0, 0, 0, 0, 0, 0, 6, 0, 4067, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 4086, 3974, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 18, 4017, 0, 2, 0, 2987, 1369, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8316, 7496, 8210, 8478, 8323, 8162, 8155, 8396] avg=8192.0000 max=8478 max_rank=3 min=7496 min_rank=1 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=33036 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4062, 0, 0, 0, 0, 0, 0, 2266, 1602, 29, 0, 0, 0, 0, 0, 3748, 3512, 1, 2492, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 4096, 0, 0, 0, 2634, 1421, 0, 0, 0, 0, 0, 0, 2465, 0, 1, 0, 394, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2322, 0, 0, 0, 0, 0, 0, 4096, 1107, 0, 0, 0, 0, 0, 0, 0, 1, 4067, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 4095, 4091, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4095, 0, 0, 0, 2989, 1371, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7959, 9754, 8243, 6896, 7525, 8164, 8509, 8486] avg=8192.0000 max=9754 max_rank=1 min=6896 min_rank=3 imbalance_ratio=1.190674 max_load_reduction=55782 max_load_reduction_pct=0.851166 moved_to_echo=32684 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[59, 4095, 0, 0, 0, 25, 18, 8, 2272, 1597, 29, 0, 0, 0, 0, 0, 4049, 95, 338, 2474, 0, 73, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 9, 38, 13, 0, 4055, 43, 0, 0, 2606, 1425, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 386, 0, 4053, 17, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3993, 0, 0, 0, 12, 0, 0, 4077, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 1, 3, 4069, 19, 34, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 3740, 4002, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 6, 0, 4048, 1, 12, 0, 2854, 1358, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 7192, 8189, 8547, 9167, 8180, 7846, 8312] avg=8192.0000 max=9167 max_rank=4 min=7192 min_rank=1 imbalance_ratio=1.119019 max_load_reduction=56369 max_load_reduction_pct=0.860123 moved_to_echo=33505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4024, 0, 0, 487, 0, 0, 0, 2273, 1599, 29, 0, 0, 0, 0, 0, 3776, 477, 1, 2486, 0, 83, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4096, 0, 0, 0, 2635, 1425, 0, 0, 0, 0, 0, 0, 3568, 0, 13, 0, 396, 0, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 1107, 0, 0, 0, 0, 0, 0, 0, 4, 4067, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 4095, 4079, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 4095, 0, 0, 0, 2988, 1372, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8412, 6824, 8192, 7836, 9297, 8167, 8268, 8540] avg=8192.0000 max=9297 max_rank=4 min=6824 min_rank=1 imbalance_ratio=1.134888 max_load_reduction=56239 max_load_reduction_pct=0.858139 moved_to_echo=34272 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3203, 0, 5, 759, 0, 3, 0, 2261, 1597, 29, 0, 0, 0, 0, 0, 3974, 2667, 20, 2483, 0, 14, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 4094, 1, 2, 0, 2635, 1417, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 394, 0, 4093, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1699, 55, 0, 2, 0, 0, 0, 4093, 1108, 0, 0, 0, 0, 0, 0, 199, 1, 4065, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 4094, 3817, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2988, 1372, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7857, 9182, 8156, 8564, 6958, 8361, 8002, 8456] avg=8192.0000 max=9182 max_rank=1 min=6958 min_rank=4 imbalance_ratio=1.120850 max_load_reduction=56354 max_load_reduction_pct=0.859894 moved_to_echo=31777 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 640, 30208, 128, 128, 128, 23808, 11008, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 128, 32640, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[29824, 128, 384, 0, 3200, 0, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 384, 32768, 128, 128, 128, 21120, 11392, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 31744, 128, 128, 2688, 128, 128, 128, 18176, 12800, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 23936, 128, 128, 128, 128, 128, 0, 32768, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31232, 11008, 1408, 19968, 0, 2048, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 128, 32384, 32256, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 4096, 4095, 0, 8, 0, 0, 0, 3, 0, 3561, 0, 21, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 4096, 4065, 0, 0, 4088, 2859, 4096, 0, 4082, 0, 0, 9, 0, 0, 4082, 2551, 0, 16, 0, 0, 0, 0, 58, 3, 3266, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4091, 0, 25, 2, 4093, 0, 5, 4075, 3, 0, 0, 0, 6, 0, 1, 0, 4088, 4096, 0, 1, 0, 0, 0, 0, 0, 1826, 0, 3555, 0, 0, 0, 4090, 0, 0, 0, 3, 4092, 0, 4096, 4060, 7, 10, 4096, 4095, 4096, 0, 4096, 0, 7, 0, 5, 0, 11, 2888, 0, 22, 0, 3, 0, 0, 0, 0, 3992, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 4096, 0, 3, 4096, 0, 49, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 32, 1, 0, 2220, 0, 5, 1, 0, 0, 4034, 1, 0, 12, 0, 4093, 5, 4077, 4095, 1, 0, 4091, 4050, 4095, 0, 4096, 0, 0, 0, 1, 0, 3958, 2792, 0, 0, 0, 0, 0, 2, 0, 61, 3286, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4092, 5, 1, 0, 4096, 0, 3, 4085, 0, 8, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 1747, 0, 529, 0, 0, 0, 4091, 0, 0, 2, 15, 4096, 0, 4096, 4077, 0, 0, 4095, 4088, 4093, 0, 4096, 0, 0, 0, 0, 10, 1405, 4010, 0, 3, 2, 0, 0, 0, 47, 477, 4074, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 680, 0, 47, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4095, 11, 0, 4096, 3603, 4096, 0, 4096, 0, 0, 0, 0, 0, 2509, 4087, 0, 0, 3, 0, 0, 0, 0, 1375, 4071, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4088, 36, 0, 4, 4096, 1, 0, 4096, 0, 0, 24, 1, 0, 0, 0, 9, 4091, 4069, 0, 0, 0, 0, 1, 0, 0, 2465, 0, 1027, 8, 3, 0, 4058, 0, 5, 3, 0, 4096, 0, 4094, 4095, 0, 0, 4095, 3846, 4093, 0, 4063, 0, 0, 23, 4, 0, 1451, 3256, 7, 0, 29, 0, 0, 0, 0, 215, 4084, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4094, 0, 0, 5, 3993, 1, 456, 4051, 13, 0, 1, 0, 0, 27, 0, 0, 4008, 4074, 63, 0, 5, 0, 86, 0, 0, 124, 28, 828, 47, 0, 2, 4068, 0, 21, 0, 0, 4036, 17, 4047, 3976, 162, 62, 4078, 3971, 3948, 0, 4064, 52, 44, 2, 0, 0, 3194, 4069, 0, 186, 0, 1, 0, 6, 0, 0, 3607, 19] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 4096, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 138, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4095, 1, 0, 4096, 4021, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 25, 2, 4093, 0, 5, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6, 0, 1, 0, 4091, 4084, 0, 0, 0, 0, 0, 0, 4088, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1826, 0, 3555, 0, 0, 0, 4090, 1201, 45, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 25, 0, 4096, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 7, 10, 0, 4095, 4096, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 5, 0, 11, 2888, 0, 22, 2611, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3992, 0, 4067, 22, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8185, 8181, 10721, 8184, 8470, 5544, 8096] avg=8192.0000 max=10721 max_rank=3 min=5544 min_rank=6 imbalance_ratio=1.308716 max_load_reduction=54815 max_load_reduction_pct=0.836411 moved_to_echo=30294 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 3, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 4091, 4080, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2220, 0, 5, 1, 0, 0, 4034, 1201, 45, 4, 0, 0, 0, 0, 0, 1, 0, 12, 0, 25, 5, 4077, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4050, 4095, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3958, 2792, 0, 0, 2611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 61, 3286, 0, 4068, 22, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8220, 8221, 7510, 8215, 8408, 9362, 7450] avg=8192.0000 max=9362 max_rank=6 min=7450 min_rank=7 imbalance_ratio=1.142822 max_load_reduction=56174 max_load_reduction_pct=0.857147 moved_to_echo=33435 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 4096, 4076, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 8, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3561, 0, 21, 0, 0, 0, 4096, 1197, 45, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2859, 4096, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4082, 2551, 0, 16, 2602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 3, 3266, 0, 4069, 22, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8175, 8198, 8924, 8186, 7216, 9260, 7430] avg=8192.0000 max=9260 max_rank=6 min=7216 min_rank=5 imbalance_ratio=1.130371 max_load_reduction=56276 max_load_reduction_pct=0.858704 moved_to_echo=32092 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 1, 0, 4096, 0, 3, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1, 0, 4092, 4084, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1747, 0, 529, 0, 0, 0, 4091, 1201, 45, 4, 0, 0, 0, 0, 0, 0, 0, 2, 15, 25, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4093, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1405, 4010, 0, 3, 2611, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 47, 477, 4074, 0, 4071, 22, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8185, 8188, 7617, 8215, 8443, 8039, 8704] avg=8192.0000 max=8704 max_rank=7 min=7617 min_rank=3 imbalance_ratio=1.062500 max_load_reduction=56832 max_load_reduction_pct=0.867188 moved_to_echo=33401 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 3993, 1, 456, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 0, 27, 0, 0, 4094, 4067, 0, 0, 0, 0, 0, 0, 4008, 4063, 63, 0, 5, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 28, 828, 47, 0, 2, 4068, 1196, 50, 11, 0, 0, 0, 0, 0, 0, 21, 0, 0, 21, 17, 4047, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 162, 62, 0, 3971, 3948, 0, 252, 52, 0, 0, 0, 0, 0, 0, 0, 0, 44, 2, 0, 0, 3194, 4069, 0, 186, 2592, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 3607, 19, 4015, 24, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8456, 8202, 8225, 6354, 8082, 8447, 10087, 7683] avg=8192.0000 max=10087 max_rank=6 min=6354 min_rank=3 imbalance_ratio=1.231323 max_load_reduction=55449 max_load_reduction_pct=0.846085 moved_to_echo=34299 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 138, 0, 0, 0, 4096, 1201, 45, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4021, 4096, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 2611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4071, 22, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8183, 8179, 9536, 8216, 8380, 6706, 8188] avg=8192.0000 max=9536 max_rank=3 min=6706 min_rank=6 imbalance_ratio=1.164062 max_load_reduction=56000 max_load_reduction_pct=0.854492 moved_to_echo=31490 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 680, 0, 47, 0, 0, 0, 4096, 1201, 45, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3603, 4096, 0, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2509, 4087, 0, 0, 2611, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1375, 4071, 0, 4071, 22, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8180, 8187, 6073, 8216, 7972, 9207, 9554] avg=8192.0000 max=9554 max_rank=7 min=6073 min_rank=3 imbalance_ratio=1.166260 max_load_reduction=55982 max_load_reduction_pct=0.854218 moved_to_echo=34949 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 36, 0, 4, 4096, 1, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1, 0, 0, 0, 9, 4088, 4084, 0, 0, 0, 0, 0, 0, 4091, 4065, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2465, 0, 1027, 8, 3, 0, 4058, 1191, 45, 4, 0, 0, 0, 0, 0, 0, 5, 3, 0, 25, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 4093, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4, 0, 1451, 3256, 7, 0, 2590, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 215, 4084, 0, 4071, 21, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8206, 8157, 8801, 8222, 8200, 7331, 8431] avg=8192.0000 max=8801 max_rank=3 min=7331 min_rank=6 imbalance_ratio=1.074341 max_load_reduction=56735 max_load_reduction_pct=0.865707 moved_to_echo=32184 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 512, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 16640, 27776, 128, 256, 20864, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 2176, 30464, 128, 32512, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 256, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 32768, 128, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 16768, 128, 6272, 128, 128, 128, 32640, 9600, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 128, 0, 30592, 32640, 0, 2176, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4073, 0, 4, 0, 1, 1, 4088, 0, 0, 4062, 3379, 27, 10, 3969, 0, 1, 0, 4087, 4096, 2, 0, 0, 0, 0, 0, 2, 0, 6, 4092, 0, 31, 0, 703, 1725, 0, 2, 4094, 6, 0, 2262, 390, 4094, 0, 0, 4093, 3855, 0, 4, 4094, 0, 0, 0, 0, 0, 4091, 0, 2, 0, 0, 4090, 5, 2, 93, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4094, 0, 3, 976, 0, 0, 4096, 0, 0, 4096, 1552, 0, 0, 4046, 0, 0, 0, 4096, 4096, 0, 2, 143, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 1729, 1, 0, 4095, 0, 0, 3826, 10, 4096, 0, 0, 4096, 3962, 1, 2, 4095, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4, 4095, 0, 0, 124, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4084, 0, 0, 400, 0, 4, 4095, 0, 0, 4096, 2548, 0, 0, 2235, 0, 0, 1, 4087, 4090, 0, 6, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 2166, 4016, 0, 6, 4095, 0, 0, 143, 545, 4096, 26, 0, 4096, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 4096, 0, 0, 15, 198] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4095, 0, 0, 974, 0, 0, 4096, 0, 0, 4085, 3592, 0, 0, 3977, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 12, 467, 5, 0, 4095, 0, 0, 2189, 1164, 4096, 6, 0, 4096, 3917, 0, 0, 4090, 0, 0, 0, 0, 19, 4096, 1, 0, 0, 0, 4096, 1, 0, 52, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 74, 3419, 0, 0, 4096, 0, 0, 4095, 4082, 0, 0, 1503, 0, 0, 0, 4096, 4096, 0, 0, 1239, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 167, 426, 0, 0, 4096, 0, 0, 133, 213, 4096, 0, 0, 4096, 3923, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1203, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4037, 18, 0, 463, 3, 0, 4088, 0, 36, 3823, 3721, 1, 0, 23, 11, 4, 0, 4078, 4067, 31, 0, 0, 0, 0, 0, 1, 36, 0, 4037, 38, 0, 0, 3, 3786, 0, 0, 4061, 20, 24, 4088, 2, 4033, 27, 1, 4062, 3970, 0, 0, 4083, 6, 5, 8, 0, 31, 4067, 46, 2, 123, 0, 4063, 68, 0, 404, 37] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 1, 888, 0, 45, 4075, 0, 4, 4092, 1089, 4, 2, 3031, 4, 0, 0, 4093, 4095, 2, 19, 352, 0, 0, 0, 0, 148, 0, 4096, 0, 32, 1, 457, 4096, 0, 0, 4093, 0, 0, 1735, 7, 4096, 0, 0, 4094, 3970, 13, 0, 4090, 0, 0, 1, 0, 1, 4095, 1, 0, 70, 2, 4095, 1, 0, 447, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 0, 0, 473, 0, 0, 4096, 0, 0, 4073, 4091, 0, 0, 371, 0, 0, 0, 4096, 4096, 0, 0, 101, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 478, 2425, 0, 0, 4096, 0, 0, 3681, 1272, 4096, 0, 0, 4096, 3507, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 6, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 4, 0, 1, 1, 3138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2785, 3379, 27, 10, 3969, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3868, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 4092, 0, 31, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 703, 1725, 0, 2, 3098, 6, 0, 2262, 0, 0, 0, 0, 0, 0, 0, 0, 390, 3839, 0, 0, 0, 3855, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4090, 5, 2, 93, 0, 1277, 996, 950, 255, 228, 25, 8, 0] rerouted_tokens_per_ep_rank=[7217, 10171, 7957, 8199, 7796, 8088, 8177, 7931] avg=8192.0000 max=10171 max_rank=1 min=7217 min_rank=0 imbalance_ratio=1.241577 max_load_reduction=55365 max_load_reduction_pct=0.844803 moved_to_echo=31992 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 3, 976, 0, 0, 3144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2808, 1552, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3868, 0, 2, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 3, 1729, 1, 0, 3098, 0, 0, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3840, 0, 0, 0, 3962, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 124, 3, 1288, 997, 952, 256, 228, 24, 8, 0] rerouted_tokens_per_ep_rank=[8217, 8406, 8109, 8168, 8657, 7815, 8185, 7979] avg=8192.0000 max=8657 max_rank=4 min=7815 min_rank=5 imbalance_ratio=1.056763 max_load_reduction=56879 max_load_reduction_pct=0.867905 moved_to_echo=32636 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 974, 0, 0, 3144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2800, 3592, 0, 0, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3868, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 12, 467, 5, 0, 3098, 0, 0, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 1164, 3840, 6, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 19, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 52, 22, 1285, 997, 952, 256, 228, 25, 8, 0] rerouted_tokens_per_ep_rank=[8213, 10369, 7965, 8171, 5771, 8927, 8198, 7922] avg=8192.0000 max=10369 max_rank=1 min=5771 min_rank=4 imbalance_ratio=1.265747 max_load_reduction=55167 max_load_reduction_pct=0.841782 moved_to_echo=30818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 0, 400, 0, 4, 3143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2808, 2548, 0, 0, 2235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 3862, 0, 6, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 2166, 4016, 0, 6, 3098, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 545, 3840, 26, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 198, 1288, 997, 952, 256, 228, 25, 8, 0] rerouted_tokens_per_ep_rank=[7631, 7591, 7959, 8167, 9429, 8501, 8195, 8063] avg=8192.0000 max=9429 max_rank=4 min=7591 min_rank=1 imbalance_ratio=1.151001 max_load_reduction=56107 max_load_reduction_pct=0.856125 moved_to_echo=34188 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 473, 0, 0, 3144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2792, 4091, 0, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3868, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 478, 2425, 0, 0, 3099, 0, 0, 3681, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 3840, 0, 0, 0, 3507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 2, 1281, 997, 952, 256, 228, 25, 8, 0] rerouted_tokens_per_ep_rank=[7713, 7254, 8065, 8167, 9683, 8619, 8184, 7851] avg=8192.0000 max=9683 max_rank=4 min=7254 min_rank=1 imbalance_ratio=1.182007 max_load_reduction=55853 max_load_reduction_pct=0.852249 moved_to_echo=34337 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 888, 0, 45, 3128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2805, 1089, 4, 2, 3031, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3867, 2, 19, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 4096, 0, 32, 1, 4070, 0, 0, 0, 0, 0, 0, 0, 457, 4096, 0, 0, 3097, 0, 0, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3840, 0, 0, 0, 3970, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 2, 4095, 1, 0, 447, 3, 1287, 996, 947, 256, 228, 24, 8, 0] rerouted_tokens_per_ep_rank=[8158, 6939, 8333, 8347, 9385, 7830, 8180, 8364] avg=8192.0000 max=9385 max_rank=4 min=6939 min_rank=1 imbalance_ratio=1.145630 max_load_reduction=56151 max_load_reduction_pct=0.856796 moved_to_echo=33759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4037, 18, 0, 463, 3, 0, 3133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 2618, 3721, 1, 0, 23, 11, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3836, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 36, 0, 4037, 38, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 3, 3786, 0, 0, 3069, 20, 24, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3780, 27, 1, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 6, 5, 8, 0, 31, 4067, 46, 0, 0, 0, 0, 0, 0, 0, 0, 2, 123, 0, 4063, 68, 0, 404, 37, 1205, 992, 955, 253, 231, 24, 13, 0] rerouted_tokens_per_ep_rank=[7654, 6414, 7945, 8150, 10990, 7780, 8233, 8370] avg=8192.0000 max=10990 max_rank=4 min=6414 min_rank=1 imbalance_ratio=1.341553 max_load_reduction=54546 max_load_reduction_pct=0.832306 moved_to_echo=35373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 74, 3419, 0, 0, 3144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2807, 4082, 0, 0, 1503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3868, 0, 0, 1239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 167, 426, 0, 0, 3099, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 213, 3840, 0, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1203, 3, 1288, 997, 952, 256, 228, 25, 8, 0] rerouted_tokens_per_ep_rank=[10733, 8392, 9203, 8167, 3825, 7976, 8184, 9056] avg=8192.0000 max=10733 max_rank=0 min=3825 min_rank=4 imbalance_ratio=1.310181 max_load_reduction=54803 max_load_reduction_pct=0.836227 moved_to_echo=29041 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 22272, 24064, 128, 128, 19200, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 30976, 128, 128, 1920, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 7680, 128, 128, 25216, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 18688, 128, 128, 24832, 128, 128, 18176, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3712, 30720, 128, 128, 0, 31232, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 256, 128, 32768, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 256, 128, 32768, 128, 128, 2432, 384, 10240, 8064, 7680, 2048, 1920, 256, 128, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 4015, 3, 86, 4096, 4088, 3892, 0, 0, 2, 8, 2, 0, 5, 25, 3985, 1, 118, 0, 4054, 0, 32, 0, 4094, 0, 1642, 0, 1, 0, 0, 0, 0, 4092, 6, 0, 4, 2, 0, 0, 0, 4090, 0, 0, 5, 4085, 4093, 0, 4086, 11, 4093, 20, 2613, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 2445, 0, 199, 389, 4087, 0, 0, 0, 0, 1734, 0, 21, 4096, 4094, 4077, 0, 0, 0, 30, 0, 0, 0, 2267, 4074, 0, 9, 0, 4096, 0, 0, 0, 4092, 0, 428, 0, 0, 0, 733, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4095, 0, 4096, 0, 4092, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[12, 0, 0, 1396, 0, 7, 113, 4094, 0, 0, 3, 0, 353, 0, 0, 4096, 4096, 3570, 0, 0, 0, 0, 0, 0, 0, 3104, 4095, 0, 4, 0, 3991, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 76, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 2, 4096, 0, 0, 0, 4093, 4096, 0, 4096, 2, 4096, 0, 3757, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 1041, 0, 3, 8, 4096, 0, 0, 23, 0, 2965, 0, 49, 4096, 4095, 4096, 0, 0, 0, 2, 0, 0, 1, 135, 4090, 0, 0, 0, 4050, 0, 5, 0, 4071, 0, 3420, 1, 0, 0, 626, 0, 3, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 30, 0, 4070, 4096, 0, 4096, 0, 4096, 0, 4078, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[32, 0, 0, 4033, 2, 0, 90, 3990, 0, 6, 2, 0, 20, 0, 0, 4095, 4064, 84, 0, 1, 0, 49, 0, 0, 1, 3375, 1304, 0, 153, 24, 4096, 0, 0, 0, 4041, 0, 4090, 16, 1, 4, 3415, 0, 1, 4072, 0, 1, 1, 0, 2, 0, 11, 4092, 6, 0, 5, 4094, 4093, 0, 4050, 0, 4048, 0, 4072, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 9, 360, 3299, 0, 0, 0, 4088, 0, 0, 107, 6, 4094, 0, 7, 4072, 4035, 3400, 49, 0, 48, 48, 48, 4, 0, 644, 3723, 0, 112, 4, 4063, 1, 0, 45, 4050, 3, 61, 0, 13, 31, 594, 0, 18, 4051, 0, 46, 0, 0, 1, 21, 0, 4043, 22, 103, 47, 3964, 4056, 16, 4010, 0, 4084, 31, 4000, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4020, 0, 10, 91, 4096, 0, 0, 0, 0, 205, 0, 18, 4096, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 171, 4096, 0, 10, 0, 4077, 0, 0, 0, 4096, 0, 3907, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 3895, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 856, 0, 33, 0, 4095, 0, 0, 0, 0, 3472, 0, 6, 4096, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 316, 4096, 0, 1, 0, 4069, 0, 0, 0, 4096, 0, 3400, 0, 0, 0, 196, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4067, 4096, 0, 4096, 0, 4096, 0, 4083, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4091, 1097, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 3, 86, 4096, 1250, 700, 0, 0, 0, 0, 0, 0, 4088, 3892, 0, 0, 2, 8, 2, 0, 662, 0, 0, 0, 0, 0, 0, 0, 5, 25, 3985, 1, 118, 0, 3194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 4094, 0, 1642, 0, 1, 0, 1464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 6, 0, 4, 2, 3393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 5, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 11, 282, 20, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9481, 10150, 8654, 7328, 7233, 7497, 8181, 7012] avg=8192.0000 max=10150 max_rank=1 min=7012 min_rank=7 imbalance_ratio=1.239014 max_load_reduction=55386 max_load_reduction_pct=0.845123 moved_to_echo=29923 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 1396, 0, 7, 113, 4094, 1098, 194, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 353, 0, 0, 4096, 1251, 702, 0, 0, 0, 0, 0, 0, 4096, 3570, 0, 0, 0, 0, 0, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 3104, 4095, 0, 4, 0, 3145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 1465, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 4096, 0, 0, 0, 0, 3394, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 4096, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 282, 0, 3757, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6914, 6405, 8317, 10348, 9656, 7566, 8193, 8137] avg=8192.0000 max=10348 max_rank=3 min=6405 min_rank=1 imbalance_ratio=1.263184 max_load_reduction=55188 max_load_reduction_pct=0.842102 moved_to_echo=33552 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2445, 0, 199, 389, 4087, 1098, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1734, 0, 21, 4096, 1251, 701, 0, 0, 0, 0, 0, 0, 4094, 4077, 0, 0, 0, 30, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 2267, 4074, 0, 9, 0, 3227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 428, 0, 0, 0, 1465, 0, 0, 0, 0, 0, 0, 0, 733, 0, 0, 4094, 0, 0, 0, 0, 3395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 282, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8417, 7803, 8870, 9577, 5985, 8222, 8193, 8469] avg=8192.0000 max=9577 max_rank=3 min=5985 min_rank=4 imbalance_ratio=1.169067 max_load_reduction=55959 max_load_reduction_pct=0.853867 moved_to_echo=30869 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1041, 0, 3, 8, 4096, 1098, 197, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 2965, 0, 49, 4096, 1251, 701, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 2, 0, 0, 661, 0, 0, 0, 0, 0, 0, 0, 1, 135, 4090, 0, 0, 0, 3191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4071, 0, 3420, 1, 0, 0, 1465, 0, 0, 0, 0, 0, 0, 0, 626, 0, 3, 4096, 0, 0, 1, 0, 3395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 30, 0, 4070, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 282, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6444, 9085, 8854, 7417, 8962, 8121, 8197, 8456] avg=8192.0000 max=9085 max_rank=1 min=6444 min_rank=0 imbalance_ratio=1.109009 max_load_reduction=56451 max_load_reduction_pct=0.861374 moved_to_echo=33736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 360, 3299, 0, 0, 0, 4088, 1096, 204, 0, 0, 0, 0, 0, 0, 0, 0, 107, 6, 4094, 0, 7, 4072, 1254, 695, 0, 0, 0, 0, 0, 0, 4035, 3400, 49, 0, 48, 48, 48, 4, 667, 0, 0, 0, 0, 0, 0, 0, 0, 644, 3723, 0, 112, 4, 3189, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4050, 3, 61, 0, 13, 31, 1468, 0, 0, 0, 0, 0, 0, 0, 594, 0, 18, 4051, 0, 46, 0, 0, 3361, 0, 0, 0, 0, 0, 0, 0, 1, 21, 0, 4043, 22, 103, 47, 3964, 3, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4010, 0, 266, 31, 4000, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9056, 10235, 8299, 7673, 5671, 8070, 8204, 8328] avg=8192.0000 max=10235 max_rank=1 min=5671 min_rank=4 imbalance_ratio=1.249390 max_load_reduction=55301 max_load_reduction_pct=0.843826 moved_to_echo=30273 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 0, 4033, 2, 0, 90, 3990, 1085, 199, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 20, 0, 0, 4095, 1237, 701, 0, 0, 0, 0, 0, 0, 4064, 84, 0, 1, 0, 49, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0, 1, 3375, 1304, 0, 153, 24, 3227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 4090, 16, 1, 4, 1448, 0, 0, 0, 0, 0, 0, 0, 3415, 0, 1, 4072, 0, 1, 1, 0, 3392, 0, 0, 0, 0, 0, 0, 0, 2, 0, 11, 4092, 6, 0, 5, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 278, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9431, 6061, 4867, 8084, 9600, 10882, 8211, 8400] avg=8192.0000 max=10882 max_rank=5 min=4867 min_rank=2 imbalance_ratio=1.328369 max_load_reduction=54654 max_load_reduction_pct=0.833954 moved_to_echo=37093 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 856, 0, 33, 0, 4095, 1098, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3472, 0, 6, 4096, 1251, 701, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 0, 0, 0, 0, 0, 664, 0, 0, 0, 0, 0, 0, 0, 0, 316, 4096, 0, 1, 0, 3206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3400, 0, 0, 0, 1465, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 4096, 0, 0, 0, 0, 3395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4067, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 282, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6280, 9526, 8838, 7619, 8961, 7687, 8164, 8461] avg=8192.0000 max=9526 max_rank=1 min=6280 min_rank=0 imbalance_ratio=1.162842 max_load_reduction=56010 max_load_reduction_pct=0.854645 moved_to_echo=33273 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4020, 0, 10, 91, 4096, 1098, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 18, 4096, 1251, 701, 0, 0, 0, 0, 0, 0, 4096, 4076, 0, 0, 0, 0, 0, 0, 665, 0, 0, 0, 0, 0, 0, 0, 0, 171, 4096, 0, 10, 0, 3213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3907, 0, 0, 0, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 282, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9513, 6271, 8837, 7490, 9468, 7491, 8193, 8273] avg=8192.0000 max=9513 max_rank=0 min=6271 min_rank=1 imbalance_ratio=1.161255 max_load_reduction=56023 max_load_reduction_pct=0.854843 moved_to_echo=33425 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 384, 21248, 128, 256, 768, 32640, 8832, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 10112, 29568, 128, 512, 128, 25600, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 27392, 128, 128, 128, 256, 128, 128, 5376, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32640, 128, 2304, 128, 30592, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 256, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32640, 128, 21120, 128, 128, 128, 11776, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[5760, 0, 128, 32768, 128, 128, 128, 128, 27136, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 256, 128, 16896, 128, 256, 32768, 10112, 5632, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[509, 0, 0, 2, 4092, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 9, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4082, 4096, 0, 0, 4085, 5, 4096, 0, 0, 3659, 4093, 4096, 4006, 0, 0, 0, 18, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4088, 4094, 0, 4094, 4096, 0, 0, 1, 4095, 0, 921, 3, 2, 4093, 0, 1, 0, 4, 0, 0, 2, 180, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 3, 0, 4096, 4089, 3, 7, 4080, 8, 4092, 0, 1, 3011, 4090, 4085, 4089, 0, 0, 0, 0, 0, 4095, 7, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4087, 0, 4096, 4096, 0, 0, 0, 4096, 0, 199, 0, 0, 4095, 4, 2, 0, 1, 0, 12, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4096, 4093, 0, 10, 3492, 0, 4096, 0, 0, 3964, 4085, 4096, 4081, 0, 1, 0, 6, 0, 4096, 0, 15, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4094, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 7, 0, 4081, 3, 0, 0, 41, 0, 0, 0, 43, 0, 43, 0, 0, 0, 0, 0, 202, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 4096, 0, 1, 3832, 0, 4096, 0, 0, 3905, 4096, 4095, 4096, 0, 0, 141, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 7, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 21, 4047, 0, 4096, 0, 0, 4074, 4096, 4096, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4083, 0, 4090, 4094, 0, 0, 0, 4096, 0, 218, 3, 0, 4095, 0, 23, 3, 0, 2, 0, 0, 967, 30, 13, 6, 0, 25, 0, 10, 89, 0, 0, 0, 0, 0, 4093, 0, 1, 4026, 4015, 1, 0, 3155, 0, 4072, 4, 0, 4015, 4046, 4096, 3868, 1, 51, 72, 0, 0, 4066, 12, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 840, 4, 0, 4096, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 4096, 0, 15, 3174, 0, 4096, 0, 0, 4091, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[239, 31, 20, 0, 4001, 4054, 19, 4070, 4010, 29, 51, 0, 4043, 0, 1101, 0, 17, 3955, 0, 1, 0, 0, 5, 1, 0, 0, 0, 72, 41, 0, 0, 11, 96, 193, 0, 2, 45, 0, 0, 4057, 96, 0, 3992, 4045, 0, 0, 4025, 1, 4058, 123, 0, 2736, 4041, 4055, 4077, 13, 102, 54, 0, 0, 3953, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4088, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 3689, 0, 921, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 1, 0, 4, 0, 0, 3853, 241, 0, 0, 0, 0, 0, 0, 2, 180, 0, 0, 0, 2, 0, 0, 4094, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 3542, 406, 81, 0, 0, 0, 0, 0, 3, 0, 4096, 365, 3, 7, 4080, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3011, 444, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 5, 0, 3724, 239, 104, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8103, 8710, 8194, 8122, 8126, 8562, 7545, 8174] avg=8192.0000 max=8710 max_rank=1 min=7545 min_rank=6 imbalance_ratio=1.063232 max_load_reduction=56826 max_load_reduction_pct=0.867096 moved_to_echo=32407 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 0, 0, 2, 4092, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 9, 0, 0, 3857, 242, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 3854, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 4096, 3545, 406, 81, 0, 0, 0, 0, 0, 0, 0, 4082, 366, 0, 0, 4085, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3659, 444, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 0, 0, 0, 3730, 239, 104, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8618, 7786, 8205, 7962, 8131, 8538, 8109, 8187] avg=8192.0000 max=8618 max_rank=0 min=7786 min_rank=1 imbalance_ratio=1.052002 max_load_reduction=56918 max_load_reduction_pct=0.868500 moved_to_echo=32965 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4094, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3690, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3, 0, 0, 41, 0, 0, 3856, 242, 0, 0, 0, 0, 0, 0, 0, 43, 0, 43, 0, 0, 0, 0, 4096, 3853, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 0, 0, 4096, 3547, 406, 81, 0, 0, 0, 0, 0, 0, 0, 4084, 366, 0, 1, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 445, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 4096, 0, 0, 0, 3730, 240, 104, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 7795, 8223, 8035, 8332, 8283, 8446, 8311] avg=8192.0000 max=8446 max_rank=6 min=7795 min_rank=1 imbalance_ratio=1.031006 max_load_reduction=57090 max_load_reduction_pct=0.871124 moved_to_echo=33372 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3690, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 2, 0, 1, 0, 12, 3857, 242, 0, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 4096, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3538, 406, 81, 0, 0, 0, 0, 0, 1, 0, 4096, 365, 0, 10, 3492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 444, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 4096, 0, 15, 0, 3728, 239, 103, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8102, 7985, 8213, 8476, 8119, 7964, 8489, 8188] avg=8192.0000 max=8489 max_rank=6 min=7964 min_rank=5 imbalance_ratio=1.036255 max_load_reduction=57047 max_load_reduction_pct=0.870468 moved_to_echo=32760 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3690, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3857, 242, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 4096, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3547, 406, 81, 0, 0, 0, 0, 0, 0, 0, 4096, 366, 0, 21, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 445, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3730, 239, 104, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 7794, 8195, 7996, 8130, 8530, 8608, 8169] avg=8192.0000 max=8608 max_rank=6 min=7794 min_rank=1 imbalance_ratio=1.050781 max_load_reduction=56928 max_load_reduction_pct=0.868652 moved_to_echo=33437 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3690, 0, 218, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 23, 3, 0, 2, 0, 3835, 242, 0, 0, 0, 0, 0, 0, 0, 967, 30, 13, 6, 0, 25, 0, 4090, 3854, 0, 0, 0, 0, 0, 0, 10, 89, 0, 0, 0, 0, 0, 4093, 3504, 406, 81, 0, 0, 0, 0, 0, 0, 1, 4026, 358, 1, 0, 3155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4015, 440, 0, 3868, 1, 0, 0, 0, 0, 0, 0, 0, 0, 51, 72, 0, 0, 4066, 12, 0, 0, 3657, 237, 102, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 8005, 8200, 8985, 8183, 7541, 8328, 8197] avg=8192.0000 max=8985 max_rank=3 min=7541 min_rank=5 imbalance_ratio=1.096802 max_load_reduction=56551 max_load_reduction_pct=0.862900 moved_to_echo=32249 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[239, 31, 20, 0, 4001, 3970, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 29, 51, 0, 3640, 0, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3955, 0, 1, 0, 0, 5, 1, 3822, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 41, 0, 0, 11, 4070, 3815, 0, 0, 0, 0, 0, 0, 96, 193, 0, 2, 45, 0, 0, 4057, 3505, 403, 84, 0, 0, 0, 0, 0, 96, 0, 3992, 357, 0, 0, 4025, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 2736, 430, 0, 4077, 13, 0, 0, 0, 0, 0, 0, 0, 0, 102, 54, 0, 0, 3953, 0, 0, 1, 3688, 236, 106, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8280, 8831, 8041, 8009, 8385, 8471, 7379, 8140] avg=8192.0000 max=8831 max_rank=1 min=7379 min_rank=6 imbalance_ratio=1.078003 max_load_reduction=56705 max_load_reduction_pct=0.865250 moved_to_echo=32375 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3690, 0, 840, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 72, 0, 0, 0, 0, 3856, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3547, 406, 81, 0, 0, 0, 0, 0, 0, 0, 4092, 366, 0, 15, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 445, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3730, 240, 104, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8111, 8630, 8265, 7951, 8130, 7647, 8632, 8170] avg=8192.0000 max=8632 max_rank=6 min=7647 min_rank=5 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=32579 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 128, 128, 128, 32768, 32128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 29568, 3584, 0, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32640, 2944, 128, 128, 29952, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 1792, 128, 128, 128, 128, 128, 128, 32768, 30848, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 384, 128, 0, 32640, 128, 128, 128, 29824, 1920, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 29568, 128, 3328, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32640, 128, 128, 128, 128, 128, 128, 30848, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 512, 0, 128, 128, 0, 128, 32768, 28288, 3328, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[3950, 9, 4096, 0, 0, 0, 7, 0, 0, 0, 1, 0, 4060, 4083, 4096, 0, 671, 0, 834, 4094, 0, 0, 0, 1211, 2, 38, 0, 0, 4, 0, 12, 4096, 7, 4066, 0, 4096, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 1544, 27, 0, 4092, 0, 8, 4096, 4088, 1, 0, 4091, 0, 4, 0, 15, 0, 0, 4093, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[3848, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1263, 4096, 4096, 4096, 0, 3568, 0, 3, 4096, 0, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4096, 0, 4085, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 425, 0, 0, 3088, 0, 0, 4076, 4096, 0, 0, 4095, 0, 90, 0, 0, 0, 43, 4096, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[2632, 66, 4096, 0, 0, 0, 0, 0, 0, 20, 0, 1474, 4059, 4067, 4096, 0, 4085, 0, 44, 4096, 245, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 4077, 0, 3854, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 962, 2, 11, 3158, 1, 30, 4096, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4090, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4076, 1, 4096, 2, 0, 0, 0, 0, 0, 40, 0, 14, 4054, 4096, 4096, 0, 3226, 0, 805, 4083, 14, 72, 0, 257, 27, 0, 0, 0, 0, 0, 0, 3408, 0, 4095, 14, 4037, 46, 0, 0, 1, 4096, 0, 2, 0, 0, 324, 2, 0, 4051, 0, 103, 4018, 4093, 0, 0, 4077, 0, 65, 0, 0, 0, 48, 4093, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[3233, 0, 4096, 0, 1, 0, 0, 0, 564, 0, 0, 804, 4096, 4096, 4096, 0, 1, 0, 60, 4096, 0, 0, 0, 223, 0, 0, 0, 0, 0, 2, 0, 3600, 0, 4096, 0, 4082, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3863, 3, 0, 4045, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4091, 164, 4088, 0, 12, 0, 0, 4, 4, 189, 0, 8, 3953, 4069, 4096, 1, 926, 14, 3940, 4092, 0, 0, 31, 2, 67, 0, 1, 0, 3, 68, 5, 4090, 6, 4092, 0, 2985, 0, 0, 0, 2, 4034, 0, 2, 7, 0, 556, 0, 0, 4078, 0, 66, 3541, 4093, 0, 0, 4093, 0, 0, 0, 0, 10, 0, 4053, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4058, 0, 4096, 0, 1, 0, 0, 0, 17, 0, 0, 16, 4096, 4096, 4096, 0, 2985, 0, 25, 4096, 0, 0, 0, 721, 0, 0, 0, 0, 0, 2, 0, 4081, 0, 4096, 0, 4078, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 16, 0, 0, 4085, 0, 0, 4096, 4096, 0, 0, 4096, 0, 395, 0, 0, 0, 1, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[3492, 502, 3956, 2, 4, 71, 50, 0, 1, 7, 52, 2, 3107, 4016, 4048, 39, 127, 218, 4061, 4050, 167, 78, 71, 179, 0, 0, 2, 0, 30, 0, 1, 4078, 19, 3925, 0, 3498, 0, 35, 6, 2, 3981, 12, 0, 0, 16, 919, 411, 0, 3816, 0, 0, 4028, 4084, 45, 18, 4081, 19, 0, 0, 161, 11, 0, 4038, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3950, 9, 4096, 0, 0, 0, 7, 0, 207, 122, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4060, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 834, 4094, 0, 0, 0, 1211, 494, 0, 0, 0, 0, 0, 0, 0, 2, 38, 0, 0, 4, 0, 12, 4096, 4088, 134, 0, 0, 0, 0, 0, 0, 7, 4066, 0, 4096, 0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 1544, 27, 0, 2992, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 8, 4096, 0, 1, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 15, 0, 0, 4093, 0, 3962, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8391, 7755, 7304, 8374, 8436, 8607, 8550, 8119] avg=8192.0000 max=8607 max_rank=5 min=7304 min_rank=2 imbalance_ratio=1.050659 max_load_reduction=56929 max_load_reduction_pct=0.868668 moved_to_echo=33712 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3848, 0, 4096, 0, 0, 0, 0, 0, 207, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1263, 4096, 3705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3568, 0, 3, 4096, 0, 3, 0, 7, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 135, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4085, 1, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 425, 0, 0, 2995, 0, 0, 0, 0, 0, 0, 0, 3088, 0, 0, 4076, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 43, 4096, 2, 3961, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 9064, 8172, 8306, 8450, 7516, 7517, 8237] avg=8192.0000 max=9064 max_rank=1 min=7516 min_rank=5 imbalance_ratio=1.106445 max_load_reduction=56472 max_load_reduction_pct=0.861694 moved_to_echo=31720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2632, 66, 4096, 0, 0, 0, 0, 0, 207, 122, 0, 0, 0, 0, 0, 0, 0, 20, 0, 1474, 4059, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 44, 4096, 245, 0, 0, 1, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 4094, 134, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 3854, 0, 0, 0, 0, 266, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 962, 2, 11, 2996, 0, 0, 0, 0, 0, 0, 0, 3158, 1, 30, 4096, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 3962, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7123, 9232, 8966, 8259, 8197, 8021, 7638, 8100] avg=8192.0000 max=9232 max_rank=1 min=7123 min_rank=0 imbalance_ratio=1.126953 max_load_reduction=56304 max_load_reduction_pct=0.859131 moved_to_echo=31956 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4076, 1, 4096, 2, 0, 0, 0, 0, 206, 123, 0, 0, 0, 0, 0, 0, 0, 40, 0, 14, 4054, 3705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3226, 0, 805, 4083, 14, 72, 0, 257, 493, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 3408, 4093, 134, 0, 0, 0, 0, 0, 0, 0, 4095, 14, 4037, 46, 0, 0, 1, 268, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 324, 2, 0, 2982, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 103, 4018, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 48, 4093, 4, 3962, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8504, 7813, 8950, 7662, 8461, 7406, 8523, 8217] avg=8192.0000 max=8950 max_rank=2 min=7406 min_rank=5 imbalance_ratio=1.092529 max_load_reduction=56586 max_load_reduction_pct=0.863434 moved_to_echo=32607 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4058, 0, 4096, 0, 1, 0, 0, 0, 207, 123, 0, 0, 0, 0, 0, 0, 17, 0, 0, 16, 4096, 3705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2985, 0, 25, 4096, 0, 0, 0, 721, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4081, 4096, 134, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4078, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 16, 0, 0, 2996, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 1, 4096, 0, 3962, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8485, 7834, 8322, 8313, 8442, 7107, 8534, 8499] avg=8192.0000 max=8534 max_rank=6 min=7107 min_rank=5 imbalance_ratio=1.041748 max_load_reduction=57002 max_load_reduction_pct=0.869781 moved_to_echo=32582 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4091, 164, 4088, 0, 12, 0, 0, 4, 207, 122, 0, 0, 0, 0, 0, 0, 4, 189, 0, 8, 3953, 3681, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 926, 14, 3940, 4092, 0, 0, 31, 2, 495, 0, 0, 0, 0, 0, 0, 0, 67, 0, 1, 0, 3, 68, 5, 4090, 4093, 135, 0, 0, 0, 0, 0, 0, 6, 4092, 0, 2985, 0, 0, 0, 2, 266, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 2, 7, 0, 556, 0, 0, 2994, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 66, 3541, 0, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4053, 0, 3961, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8688, 7836, 9500, 8462, 7351, 7593, 8037, 8069] avg=8192.0000 max=9500 max_rank=2 min=7351 min_rank=4 imbalance_ratio=1.159668 max_load_reduction=56036 max_load_reduction_pct=0.855042 moved_to_echo=31050 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3233, 0, 4096, 0, 1, 0, 0, 0, 207, 123, 0, 0, 0, 0, 0, 0, 564, 0, 0, 804, 4096, 3705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 60, 4096, 0, 0, 0, 223, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3600, 4096, 134, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3863, 3, 0, 2996, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 4096, 0, 0, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3962, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7660, 9169, 4875, 7832, 8446, 10957, 8494, 8103] avg=8192.0000 max=10957 max_rank=5 min=4875 min_rank=2 imbalance_ratio=1.337524 max_load_reduction=54579 max_load_reduction_pct=0.832809 moved_to_echo=36000 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3492, 502, 3956, 2, 4, 71, 50, 0, 211, 123, 0, 0, 0, 0, 0, 0, 1, 7, 52, 2, 3107, 3625, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 127, 218, 4061, 4050, 167, 78, 71, 179, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 30, 0, 1, 4078, 4084, 133, 0, 0, 0, 0, 0, 0, 19, 3925, 0, 3498, 0, 35, 6, 2, 268, 0, 0, 0, 0, 0, 0, 0, 3981, 12, 0, 0, 16, 919, 411, 0, 2990, 0, 0, 0, 0, 0, 0, 0, 3816, 0, 0, 4028, 0, 45, 18, 336, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 161, 11, 0, 4038, 0, 3915, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8411, 6833, 9447, 8328, 7753, 8329, 8243, 8192] avg=8192.0000 max=9447 max_rank=2 min=6833 min_rank=1 imbalance_ratio=1.153198 max_load_reduction=56089 max_load_reduction_pct=0.855850 moved_to_echo=32517 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[15616, 256, 9856, 32768, 512, 256, 128, 2688, 3968, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32512, 128, 128, 128, 128, 8704, 512, 128, 24064, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[30464, 128, 256, 32128, 0, 128, 128, 2816, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32640, 128, 30720, 128, 128, 128, 128, 2176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[640, 256, 128, 3584, 31616, 29568, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 31488, 32768, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[29440, 768, 32640, 128, 128, 128, 128, 128, 1664, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 640, 0, 256, 128, 128, 32768, 128, 31744, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 506, 4096, 0, 0, 1, 0, 4086, 0, 4023, 0, 9, 6, 0, 0, 0, 0, 3, 4090, 0, 3, 4094, 0, 0, 0, 0, 0, 3, 4088, 0, 0, 613, 26, 2, 4095, 4089, 28, 4033, 0, 0, 0, 0, 1, 4091, 4096, 0, 0, 4096, 4076, 1, 0, 4092, 0, 0, 0, 10, 3073, 0, 0, 4096, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3477, 4070, 0, 0, 0, 713, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 31, 0, 4015, 0, 0, 0, 0, 1, 4081, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4032, 1794, 0, 5, 0, 21, 4096, 2, 2387, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 37, 0, 3797, 0, 0, 122, 0, 0, 4094, 4090, 204, 4096, 4, 0, 0, 0, 0, 4064, 4095, 0, 0, 4096, 4096, 0, 0, 4021, 0, 0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 663, 4095, 0, 0, 0, 1, 4096, 0, 3495, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 12, 0, 0, 4096, 4096, 34, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 3991, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3906, 3296, 0, 0, 0, 42, 4096, 0, 933, 1, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 4045, 0, 0, 24, 0, 0, 4055, 4096, 1, 4096, 0, 0, 0, 0, 0, 4088, 4095, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 64, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 861, 4095, 0, 0, 0, 2, 4096, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4087, 0, 0, 0, 0, 3387, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 222, 4090, 0, 0, 11, 0, 4094, 0, 3978, 0, 2, 2, 0, 0, 0, 0, 95, 4093, 0, 0, 4091, 6, 8, 0, 1, 380, 0, 4077, 5, 0, 0, 0, 0, 3829, 4096, 0, 4056, 4, 15, 0, 0, 0, 4096, 4075, 0, 2, 4093, 4096, 2, 0, 4079, 0, 0, 3, 0, 3794, 0, 33, 4083, 24] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[10, 0, 466, 0, 1956, 2547, 28, 0, 0, 1045, 4061, 0, 817, 0, 0, 12, 7, 40, 25, 44, 0, 4037, 0, 140, 3911, 0, 50, 1, 25, 0, 7, 4066, 12, 4, 1080, 0, 0, 4082, 3751, 2063, 4052, 0, 0, 0, 59, 23, 3145, 4047, 0, 23, 4045, 4039, 18, 47, 3769, 25, 0, 0, 0, 3601, 0, 289, 4066, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 506, 4096, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 0, 4023, 0, 9, 6, 1276, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 3, 4046, 19, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3, 4088, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 613, 26, 2, 3650, 4089, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 1, 4091, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4076, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3073, 0, 0, 4096, 1, 285, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7280, 9547, 8161, 8199, 8408, 8257, 8219, 7465] avg=8192.0000 max=9547 max_rank=1 min=7280 min_rank=0 imbalance_ratio=1.165405 max_load_reduction=55989 max_load_reduction_pct=0.854324 moved_to_echo=32349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3906, 3296, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 933, 1, 0, 8, 1276, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4046, 19, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 4045, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 3614, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 64, 4096, 0, 282, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9871, 6501, 8161, 8176, 7735, 8315, 8241, 8536] avg=8192.0000 max=9871 max_rank=0 min=6501 min_rank=1 imbalance_ratio=1.204956 max_load_reduction=55665 max_load_reduction_pct=0.849380 moved_to_echo=32827 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3477, 4070, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 713, 4096, 0, 0, 0, 0, 0, 1276, 146, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 4046, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 31, 0, 4015, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3637, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 284, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10216, 6231, 8162, 8156, 7734, 8324, 8241, 8472] avg=8192.0000 max=10216 max_rank=0 min=6231 min_rank=1 imbalance_ratio=1.247070 max_load_reduction=55320 max_load_reduction_pct=0.844116 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4032, 1794, 0, 5, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 2, 2387, 0, 1, 0, 1276, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4046, 19, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 37, 0, 3797, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 3649, 4090, 204, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4064, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4096, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 285, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8500, 7929, 8161, 7942, 8065, 8295, 8167, 8477] avg=8192.0000 max=8500 max_rank=0 min=7929 min_rank=1 imbalance_ratio=1.037598 max_load_reduction=57036 max_load_reduction_pct=0.870300 moved_to_echo=33004 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 861, 4095, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 3950, 0, 0, 0, 1276, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4046, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3651, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3387, 0, 0, 4096, 0, 285, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7625, 9470, 8161, 8206, 7749, 8324, 8233, 7768] avg=8192.0000 max=9470 max_rank=1 min=7625 min_rank=0 imbalance_ratio=1.156006 max_load_reduction=56066 max_load_reduction_pct=0.855499 moved_to_echo=32074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 222, 4090, 0, 0, 2656, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4094, 0, 3978, 0, 2, 2, 1270, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 4093, 0, 0, 4043, 19, 0, 0, 0, 0, 0, 0, 4091, 6, 8, 0, 1, 380, 0, 4077, 13, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3413, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 4, 15, 0, 0, 0, 4096, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 50, 4096, 2, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3794, 0, 33, 4083, 24, 266, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6969, 9494, 8250, 8576, 7514, 8301, 8229, 8203] avg=8192.0000 max=9494 max_rank=1 min=6969 min_rank=0 imbalance_ratio=1.158936 max_load_reduction=56042 max_load_reduction_pct=0.855133 moved_to_echo=32247 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 466, 0, 1956, 2547, 28, 0, 2641, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 4061, 0, 817, 0, 0, 12, 1265, 150, 0, 0, 0, 0, 0, 0, 7, 40, 25, 44, 0, 4037, 0, 140, 4002, 24, 0, 0, 0, 0, 0, 0, 3911, 0, 50, 1, 25, 0, 7, 4066, 18, 0, 0, 0, 0, 0, 0, 0, 12, 4, 1080, 0, 0, 3628, 3751, 2063, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 59, 23, 3145, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 43, 4039, 18, 47, 3769, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3601, 0, 289, 4066, 1, 286, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7648, 7350, 8319, 8078, 10538, 7396, 7964, 8243] avg=8192.0000 max=10538 max_rank=4 min=7350 min_rank=1 imbalance_ratio=1.286377 max_load_reduction=54998 max_load_reduction_pct=0.839203 moved_to_echo=34141 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 663, 4095, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 3495, 0, 0, 0, 1276, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4046, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3651, 4096, 34, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 4096, 0, 285, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7427, 9014, 8161, 8203, 7793, 8324, 8242, 8372] avg=8192.0000 max=9014 max_rank=1 min=7427 min_rank=0 imbalance_ratio=1.100342 max_load_reduction=56522 max_load_reduction_pct=0.862457 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 30208, 0, 512, 32768, 128, 2304, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 128, 128, 128, 128, 512, 128, 32384, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 1920, 32768, 128, 19584, 128, 128, 128, 10240, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 512, 0, 15744, 28160, 128, 128, 21376, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 0, 128, 128, 31872, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 256, 32384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 1920, 128, 128, 28928, 32512, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 512, 32768, 128, 128, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4092, 0, 4096, 0, 4082, 20, 181, 22, 0, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 4074, 9, 2518, 4096, 0, 0, 0, 0, 4091, 7, 4087, 741, 3, 4095, 0, 2, 0, 4096, 0, 4087, 0, 0, 4093, 0, 0, 0, 0, 14, 0, 0, 572, 4096, 0, 90, 0, 0, 4071, 0, 4, 0, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 1, 8, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 4096, 1, 4096, 4095, 0, 0, 1, 3, 4096, 0, 4096, 702, 3397, 4096, 0, 3, 0, 4094, 6, 3651, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 387, 4096, 40, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1, 4018, 22, 3597, 387, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 3, 4096, 0, 4096, 0, 3799, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 228, 4096, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 2847, 0, 4094, 0, 1132, 51, 0, 0, 0, 4044, 0, 0, 0, 0, 1, 0, 4096, 34, 0, 4096, 0, 1139, 4096, 0, 1, 0, 0, 4067, 4, 4096, 729, 4095, 4096, 0, 0, 51, 4096, 8, 4068, 0, 0, 2342, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 18] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4091, 0, 4096, 0, 4074, 0, 38, 0, 1, 11, 0, 3987, 15, 0, 0, 0, 0, 0, 4087, 0, 6, 4090, 16, 3579, 4096, 0, 0, 0, 0, 4080, 2, 4094, 388, 325, 4091, 1, 0, 379, 4095, 211, 4053, 2, 1, 3394, 5, 0, 0, 0, 0, 3, 0, 20, 4084, 7, 20, 0, 0, 4094, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[3958, 0, 4057, 0, 3877, 130, 54, 2, 24, 0, 42, 4094, 50, 0, 0, 0, 0, 33, 4062, 235, 6, 4016, 0, 4095, 4030, 0, 0, 3, 0, 3919, 0, 3991, 397, 550, 4056, 0, 241, 0, 4062, 0, 4024, 0, 32, 3429, 19, 16, 13, 34, 9, 0, 104, 0, 4045, 0, 85, 0, 52, 3642, 10, 38, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 2961, 0, 4096, 0, 2830, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2322, 4096, 0, 0, 0, 0, 4096, 0, 4096, 60, 4055, 4096, 0, 0, 0, 4096, 38, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 3293, 0, 4095, 0, 3174, 0, 0, 0, 0, 2776, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4061, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2724, 271, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3580, 20, 181, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4094, 0, 0, 0, 0, 4092, 194, 0, 0, 0, 0, 0, 0, 0, 0, 830, 0, 2, 4074, 9, 2518, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 4091, 7, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 741, 3, 4095, 0, 2, 0, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4093, 0, 0, 0, 0, 1953, 502, 0, 0, 0, 0, 0, 0, 14, 0, 0, 572, 4096, 0, 90, 0, 2972, 935, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4, 0, 0, 5, 0, 3862, 291, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7899, 8383, 7433, 8225, 6049, 10635, 8679, 8233] avg=8192.0000 max=10635 max_rank=5 min=6049 min_rank=4 imbalance_ratio=1.298218 max_load_reduction=54901 max_load_reduction_pct=0.837723 moved_to_echo=33596 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 8, 0, 3592, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 193, 0, 0, 0, 0, 0, 0, 1, 0, 830, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 1, 3, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 702, 3397, 4096, 0, 3, 0, 1206, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3651, 0, 0, 0, 0, 0, 0, 0, 1953, 504, 0, 0, 0, 0, 0, 0, 1, 0, 0, 387, 4096, 40, 0, 0, 2972, 935, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3861, 291, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7697, 8381, 9024, 8237, 9410, 6108, 8431, 8248] avg=8192.0000 max=9410 max_rank=4 min=6108 min_rank=5 imbalance_ratio=1.148682 max_load_reduction=56126 max_load_reduction_pct=0.856415 moved_to_echo=32197 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 3524, 22, 3597, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 4096, 194, 0, 0, 0, 0, 0, 0, 0, 0, 830, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 3, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3799, 4096, 0, 0, 0, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 293, 0, 0, 0, 0, 1953, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 4096, 0, 3, 0, 2971, 935, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3862, 291, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7531, 8327, 9022, 8235, 9103, 6836, 8233, 8249] avg=8192.0000 max=9103 max_rank=4 min=6836 min_rank=5 imbalance_ratio=1.111206 max_load_reduction=56433 max_load_reduction_pct=0.861099 moved_to_echo=32421 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2847, 0, 3591, 0, 1132, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 4096, 194, 0, 0, 0, 0, 0, 0, 1, 0, 830, 34, 0, 4096, 0, 1139, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 1, 0, 0, 4067, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 729, 4095, 4096, 0, 0, 51, 1208, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 2342, 0, 0, 0, 0, 1953, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 2974, 935, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 18, 3862, 292, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7621, 8334, 6100, 8208, 10187, 8866, 7995, 8225] avg=8192.0000 max=10187 max_rank=4 min=6100 min_rank=2 imbalance_ratio=1.243530 max_load_reduction=55349 max_load_reduction_pct=0.844559 moved_to_echo=35273 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3573, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 3987, 15, 0, 0, 0, 4091, 194, 0, 0, 0, 0, 0, 0, 0, 0, 829, 0, 6, 4090, 16, 3579, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 4080, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 388, 325, 4091, 1, 0, 379, 1207, 211, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 2, 1, 3394, 5, 0, 0, 0, 1953, 501, 0, 0, 0, 0, 0, 0, 0, 3, 0, 20, 4084, 7, 20, 0, 2967, 935, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 3862, 291, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7707, 8299, 8520, 8216, 6602, 9909, 8036, 8247] avg=8192.0000 max=9909 max_rank=5 min=6602 min_rank=4 imbalance_ratio=1.209595 max_load_reduction=55627 max_load_reduction_pct=0.848801 moved_to_echo=32794 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4057, 0, 3397, 130, 54, 2, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 42, 4094, 50, 0, 0, 0, 3958, 192, 0, 0, 0, 0, 0, 0, 0, 33, 817, 235, 6, 4016, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 3, 0, 3919, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 397, 550, 4056, 0, 241, 0, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 32, 3429, 19, 16, 13, 34, 1943, 480, 0, 0, 0, 0, 0, 0, 9, 0, 104, 0, 4045, 0, 85, 0, 2952, 934, 0, 0, 0, 0, 0, 0, 52, 3642, 10, 38, 0, 0, 0, 0, 3800, 293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7640, 8360, 9202, 7951, 6429, 9990, 8129, 7835] avg=8192.0000 max=9990 max_rank=5 min=6429 min_rank=4 imbalance_ratio=1.219482 max_load_reduction=55546 max_load_reduction_pct=0.847565 moved_to_echo=32383 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3293, 0, 3591, 0, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2776, 0, 0, 0, 0, 4096, 194, 0, 0, 0, 0, 0, 0, 0, 0, 830, 0, 0, 4096, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2724, 271, 4096, 0, 0, 0, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 82, 0, 0, 0, 0, 1953, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 1, 0, 2974, 935, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3862, 292, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10058, 7066, 8987, 8232, 8299, 6635, 8010, 8249] avg=8192.0000 max=10058 max_rank=0 min=6635 min_rank=5 imbalance_ratio=1.227783 max_load_reduction=55478 max_load_reduction_pct=0.846527 moved_to_echo=31193 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2961, 0, 3592, 0, 2830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 194, 0, 0, 0, 0, 0, 0, 0, 0, 830, 0, 0, 4096, 0, 2322, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 60, 4055, 4096, 0, 0, 0, 1208, 38, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 1953, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 4, 0, 2974, 935, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3862, 292, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9383, 8386, 7248, 8232, 9457, 6557, 8023, 8250] avg=8192.0000 max=9457 max_rank=4 min=6557 min_rank=5 imbalance_ratio=1.154419 max_load_reduction=56079 max_load_reduction_pct=0.855698 moved_to_echo=32287 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 6656, 384, 128, 32768, 128, 25984, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32256, 128, 128, 0, 0, 128, 128, 30848, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 0, 128, 128, 128, 32640, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 31232, 128, 0, 0, 0, 32640, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 21376, 128, 28544, 256, 15104, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32256, 128, 128, 13696, 128, 128, 128, 128, 15616, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 1280, 32768, 128, 256, 0, 23808, 7552, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5760, 16512, 32768, 128, 256, 512, 9728, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 4093, 0, 0, 0, 4094, 4091, 4080, 0, 4089, 2, 0, 0, 0, 0, 6, 6, 8, 0, 3719, 2, 0, 0, 625, 0, 0, 0, 3525, 2, 0, 0, 4088, 0, 0, 3, 0, 0, 4080, 0, 0, 0, 1749, 1, 0, 4058, 0, 4096, 0, 0, 0, 8, 4096, 0, 2703, 26, 13, 0, 1, 0, 4083, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 5, 4096, 0, 0, 0, 4077, 4096, 4080, 0, 4093, 0, 0, 28, 0, 0, 0, 0, 7, 0, 26, 0, 0, 0, 894, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4071, 0, 0, 1, 3111, 56, 0, 4046, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 82, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 4095, 4054, 4094, 0, 4092, 0, 0, 9, 0, 0, 0, 0, 19, 0, 1456, 0, 0, 0, 2265, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4092, 0, 0, 2, 3683, 0, 0, 2773, 0, 2069, 67, 0, 0, 0, 4096, 2, 4076, 0, 0, 0, 0, 15, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 4049, 4094, 0, 4096, 0, 355, 0, 0, 0, 0, 0, 6, 28, 1704, 0, 0, 0, 1771, 1, 0, 0, 4082, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 1999, 4, 0, 3315, 0, 4095, 0, 0, 0, 0, 4096, 0, 3177, 0, 0, 0, 2, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[6, 4094, 0, 0, 4096, 0, 1, 0, 4037, 4096, 4096, 0, 4075, 0, 0, 195, 0, 0, 1, 0, 0, 360, 4064, 0, 0, 0, 58, 131, 8, 2, 4090, 0, 0, 0, 4088, 30, 0, 0, 2, 0, 4017, 4, 3, 16, 3887, 1, 0, 126, 0, 3796, 71, 5, 0, 0, 4095, 0, 3790, 23, 10, 0, 0, 0, 4076, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4095, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1705, 0, 0, 0, 466, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 3718, 0, 0, 3559, 1, 4096, 0, 0, 0, 0, 4096, 1, 2903, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 4050, 3, 0, 4027, 0, 39, 101, 4079, 3373, 4027, 12, 3995, 0, 714, 378, 0, 8, 57, 130, 0, 0, 4057, 46, 23, 0, 3647, 0, 42, 45, 3939, 0, 0, 4, 3927, 0, 0, 0, 4, 0, 4082, 0, 22, 0, 4, 0, 48, 320, 49, 4040, 1, 28, 14, 30, 4057, 348, 3631, 0, 1, 0, 0, 0, 4065, 4069] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4095, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 1, 0, 778, 0, 0, 0, 2639, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1931, 0, 0, 4053, 0, 2966, 0, 0, 0, 0, 4096, 1, 4015, 0, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3997, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 8, 0, 3719, 2, 4094, 1832, 0, 0, 0, 0, 0, 0, 0, 0, 625, 0, 0, 0, 3525, 2, 1694, 922, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 3, 0, 0, 2248, 1866, 0, 0, 0, 0, 0, 0, 2889, 0, 0, 0, 1749, 1, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 4096, 0, 269, 92, 4, 0, 0, 0, 0, 0, 2703, 26, 13, 0, 1, 0, 4083, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8090, 9667, 6768, 8205, 8697, 8565, 7361] avg=8192.0000 max=9667 max_rank=2 min=6768 min_rank=3 imbalance_ratio=1.180054 max_load_reduction=55869 max_load_reduction_pct=0.852493 moved_to_echo=32828 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 4003, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 28, 1704, 0, 4096, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 1771, 1, 0, 0, 4082, 0, 1694, 922, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 2254, 1867, 0, 0, 0, 0, 0, 0, 2891, 0, 0, 0, 1999, 4, 0, 3315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 269, 93, 4, 0, 0, 0, 0, 0, 3177, 0, 0, 0, 2, 0, 4096, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8407, 7674, 8470, 8218, 8209, 8557, 7810] avg=8192.0000 max=8557 max_rank=6 min=7674 min_rank=2 imbalance_ratio=1.044556 max_load_reduction=56979 max_load_reduction_pct=0.869431 moved_to_echo=32794 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 3999, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 1456, 0, 4095, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 2265, 0, 0, 0, 4096, 0, 1694, 925, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 2255, 1867, 0, 0, 0, 0, 0, 0, 2897, 0, 0, 2, 3683, 0, 0, 2773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2069, 67, 0, 0, 0, 4096, 2, 270, 93, 4, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 15, 4096, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8062, 7409, 8980, 8219, 9355, 6601, 8722] avg=8192.0000 max=9355 max_rank=5 min=6601 min_rank=6 imbalance_ratio=1.141968 max_load_reduction=56181 max_load_reduction_pct=0.857254 moved_to_echo=32897 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4000, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 26, 0, 4077, 1832, 0, 0, 0, 0, 0, 0, 0, 0, 894, 0, 0, 0, 4095, 0, 1694, 920, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2248, 1867, 0, 0, 0, 0, 0, 0, 2883, 0, 0, 1, 3111, 56, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 268, 93, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 82, 4096, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8124, 5942, 7603, 8211, 10097, 8557, 8809] avg=8192.0000 max=10097 max_rank=5 min=5942 min_rank=2 imbalance_ratio=1.232544 max_load_reduction=55439 max_load_reduction_pct=0.845932 moved_to_echo=35674 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 4090, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3983, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 360, 4064, 0, 4037, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 58, 131, 8, 2, 4090, 0, 1690, 908, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 30, 0, 0, 2, 0, 2255, 1862, 0, 0, 0, 0, 0, 0, 2844, 4, 3, 16, 3887, 1, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 71, 5, 0, 0, 4095, 0, 265, 92, 4, 0, 0, 0, 0, 0, 3790, 23, 10, 0, 0, 0, 4076, 534, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8274, 10303, 6887, 8237, 6881, 8328, 8433] avg=8192.0000 max=10303 max_rank=2 min=6881 min_rank=5 imbalance_ratio=1.257690 max_load_reduction=55233 max_load_reduction_pct=0.842789 moved_to_echo=31879 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4003, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 778, 0, 4096, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 2639, 0, 0, 0, 4096, 0, 1694, 926, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2256, 1867, 0, 0, 0, 0, 0, 0, 2900, 0, 0, 0, 1931, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2966, 0, 0, 0, 0, 4096, 1, 270, 93, 4, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 4096, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8100, 6714, 9355, 8219, 8884, 7430, 8646] avg=8192.0000 max=9355 max_rank=3 min=6714 min_rank=2 imbalance_ratio=1.141968 max_load_reduction=56181 max_load_reduction_pct=0.857254 moved_to_echo=33179 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4042, 3, 0, 4027, 0, 39, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3373, 0, 12, 3902, 0, 714, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 57, 130, 0, 0, 4057, 46, 4079, 1810, 0, 0, 0, 0, 0, 0, 23, 0, 3647, 0, 42, 45, 3939, 0, 1689, 926, 0, 0, 0, 0, 0, 0, 0, 4, 3927, 0, 0, 0, 4, 0, 2217, 1856, 0, 0, 0, 0, 0, 0, 2883, 0, 22, 0, 4, 0, 48, 320, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4040, 1, 28, 14, 30, 4057, 348, 273, 93, 8, 0, 0, 0, 0, 0, 3631, 0, 1, 0, 0, 0, 4065, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8379, 10187, 10311, 8008, 3277, 8941, 8221] avg=8192.0000 max=10311 max_rank=3 min=3277 min_rank=5 imbalance_ratio=1.258667 max_load_reduction=55225 max_load_reduction_pct=0.842667 moved_to_echo=28447 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4003, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1705, 0, 4096, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 0, 4090, 0, 1694, 912, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2256, 1867, 0, 0, 0, 0, 0, 0, 2859, 0, 0, 0, 3718, 0, 0, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 1, 266, 93, 4, 0, 0, 0, 0, 0, 2903, 0, 0, 0, 0, 0, 4096, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8100, 7640, 7162, 8219, 10136, 8557, 7534] avg=8192.0000 max=10136 max_rank=5 min=7162 min_rank=3 imbalance_ratio=1.237305 max_load_reduction=55400 max_load_reduction_pct=0.845337 moved_to_echo=34446 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[23168, 128, 128, 128, 20096, 128, 128, 22272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32640, 128, 0, 128, 128, 0, 18048, 14976, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32000, 0, 128, 32000, 128, 1152, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[28416, 128, 128, 0, 128, 128, 32768, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 29312, 256, 128, 128, 128, 32768, 384, 2176, 768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 256, 128, 512, 17536, 128, 32768, 14720, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 12416, 256, 128, 128, 32128, 128, 13568, 7424, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 5, 25, 2, 2250, 3, 0, 4092, 0, 0, 0, 0, 4089, 3961, 3, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 32, 277, 4094, 0, 4094, 1, 0, 4094, 4093, 4077, 0, 0, 1, 0, 0, 0, 4077, 0, 0, 0, 0, 2, 0, 16, 4, 0, 4091, 0, 1956, 0, 4096, 3, 4062, 67, 3773] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 1, 0, 0, 0, 3263, 0, 0, 4096, 0, 0, 0, 0, 4096, 1916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 4096, 0, 4090, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 567, 0, 0, 0, 0, 0, 4096, 0, 2185, 0, 4093, 0, 4096, 268, 4090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 12, 0, 0, 3943, 0, 0, 4096, 0, 0, 0, 0, 4096, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 4084, 0, 7, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 14, 1, 4096, 0, 3994, 0, 61, 0, 4065, 716, 3669] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 10, 0, 682, 0, 0, 4095, 0, 0, 1, 0, 4096, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 101, 4096, 0, 4096, 0, 0, 4088, 4096, 4084, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 3684, 0, 2962, 0, 3868, 1294, 3934] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 7, 0, 4086, 12, 0, 0, 0, 6, 2192, 0, 8, 4088, 0, 2, 0, 1, 4000, 3808, 5, 9, 0, 0, 22, 0, 0, 2, 0, 4078, 0, 0, 4096, 0, 4096, 20, 0, 4079, 4089, 4087, 0, 0, 0, 0, 0, 0, 4084, 0, 11, 6, 43, 33, 0, 0, 0, 0, 4068, 0, 3112, 0, 3194, 17, 4091, 1, 4083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4095, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 2945, 0, 3948, 817, 4078] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 2667, 0, 0, 4096, 0, 0, 0, 0, 4095, 2571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3655, 0, 3394, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[8, 0, 7, 4046, 20, 0, 42, 40, 1, 3627, 51, 0, 4008, 0, 0, 0, 61, 3773, 4013, 0, 24, 24, 13, 204, 0, 0, 0, 0, 4028, 331, 637, 4075, 0, 4078, 388, 0, 4048, 4054, 3805, 1, 0, 18, 0, 0, 87, 4060, 0, 60, 1, 2, 0, 71, 0, 138, 22, 3646, 105, 1272, 39, 4062, 15, 3080, 35, 3416] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 10, 4078, 8, 0, 0, 0, 0, 0, 0, 0, 682, 0, 0, 4095, 0, 0, 1, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3942, 0, 0, 0, 0, 0, 862, 159, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 101, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3934, 129, 18, 0, 0, 0, 0, 0, 22, 3684, 0, 2962, 0, 1604, 1294, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6051, 9060, 8135, 8171, 8184, 8177, 9566] avg=8192.0000 max=9566 max_rank=7 min=6051 min_rank=1 imbalance_ratio=1.167725 max_load_reduction=55970 max_load_reduction_pct=0.854034 moved_to_echo=34098 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 5, 25, 4075, 8, 0, 0, 0, 0, 0, 0, 2, 2250, 3, 0, 4092, 0, 0, 0, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3961, 3, 0, 0, 7, 0, 906, 159, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 32, 277, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4077, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 16, 4, 0, 4091, 3773, 136, 18, 0, 0, 0, 0, 0, 0, 1956, 0, 4096, 3, 1683, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 7684, 9126, 8340, 8171, 8164, 8040, 7805] avg=8192.0000 max=9126 max_rank=2 min=7684 min_rank=1 imbalance_ratio=1.114014 max_load_reduction=56410 max_load_reduction_pct=0.860748 moved_to_echo=32180 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 12, 0, 4078, 9, 0, 0, 0, 0, 0, 0, 0, 3943, 0, 0, 4096, 0, 0, 0, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4023, 0, 0, 0, 0, 0, 906, 159, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 4096, 3669, 136, 18, 0, 0, 0, 0, 0, 0, 3994, 0, 61, 0, 1685, 716, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 9377, 9185, 8024, 8198, 8167, 7934, 6456] avg=8192.0000 max=9377 max_rank=1 min=6456 min_rank=7 imbalance_ratio=1.144653 max_load_reduction=56159 max_load_reduction_pct=0.856918 moved_to_echo=30755 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 4086, 12, 0, 0, 0, 4072, 9, 0, 0, 0, 0, 0, 0, 6, 2192, 0, 8, 4088, 0, 2, 0, 1346, 0, 0, 0, 0, 0, 0, 0, 1, 4000, 3808, 5, 9, 0, 0, 22, 912, 159, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4078, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4078, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 11, 4087, 0, 0, 0, 0, 0, 0, 0, 6, 43, 33, 0, 0, 0, 0, 4068, 4083, 137, 17, 0, 0, 0, 0, 0, 0, 3112, 0, 3194, 17, 1696, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 7642, 8917, 8017, 8185, 8182, 8387, 8020] avg=8192.0000 max=8917 max_rank=2 min=7642 min_rank=1 imbalance_ratio=1.088501 max_load_reduction=56619 max_load_reduction_pct=0.863937 moved_to_echo=32774 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 7, 4046, 20, 0, 42, 40, 4037, 9, 0, 0, 0, 0, 0, 0, 1, 3627, 51, 0, 4008, 0, 0, 0, 1018, 0, 0, 0, 0, 0, 0, 0, 61, 3773, 4013, 0, 24, 24, 13, 204, 691, 161, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 331, 637, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 0, 4046, 0, 3805, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 87, 4060, 0, 60, 4069, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 71, 0, 138, 22, 3646, 3416, 106, 17, 0, 0, 0, 0, 0, 105, 1272, 39, 4062, 15, 1265, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8705, 8966, 8910, 8240, 8294, 7419, 6793] avg=8192.0000 max=8966 max_rank=2 min=6793 min_rank=7 imbalance_ratio=1.094482 max_load_reduction=56570 max_load_reduction_pct=0.863190 moved_to_echo=30746 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4078, 8, 0, 0, 0, 0, 0, 0, 0, 2667, 0, 0, 4096, 0, 0, 0, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2571, 0, 0, 0, 0, 0, 757, 159, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 113, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 3655, 0, 1407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 7880, 7583, 8035, 8190, 8184, 8324, 9158] avg=8192.0000 max=9158 max_rank=7 min=7583 min_rank=2 imbalance_ratio=1.117920 max_load_reduction=56378 max_load_reduction_pct=0.860260 moved_to_echo=33856 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 1, 0, 0, 4078, 9, 0, 0, 0, 0, 0, 0, 0, 3263, 0, 0, 4096, 0, 0, 0, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1916, 0, 0, 0, 0, 0, 913, 159, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 567, 0, 0, 0, 0, 0, 4096, 4090, 137, 18, 0, 0, 0, 0, 0, 0, 2185, 0, 4093, 0, 1698, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8707, 7085, 8040, 8191, 8177, 8908, 8244] avg=8192.0000 max=8908 max_rank=6 min=7085 min_rank=2 imbalance_ratio=1.087402 max_load_reduction=56628 max_load_reduction_pct=0.864075 moved_to_echo=33520 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4078, 8, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 479, 0, 0, 0, 0, 0, 880, 159, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4096, 4078, 132, 18, 0, 0, 0, 0, 0, 0, 4095, 0, 2945, 0, 1637, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 9490, 5614, 8035, 8190, 8184, 8347, 9494] avg=8192.0000 max=9494 max_rank=7 min=5614 min_rank=2 imbalance_ratio=1.158936 max_load_reduction=56042 max_load_reduction_pct=0.855133 moved_to_echo=34215 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 32768, 384, 1152, 31488, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 512, 128, 32768, 0, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 22784, 128, 128, 32768, 0, 128, 128, 10112, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32384, 24832, 128, 128, 128, 128, 256, 6912, 1280, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 24448, 128, 25088, 128, 12800, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 640, 128, 128, 128, 256, 128, 32384, 31232, 1152, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4034, 0, 79, 4, 1, 4095, 0, 5, 3, 0, 0, 1910, 4093, 0, 0, 14, 0, 0, 4064, 0, 4089, 0, 3259, 0, 4093, 31, 97, 0, 0, 2, 4096, 0, 2343, 0, 4090, 432, 59, 0, 3043, 1, 0, 4096, 0, 0, 0, 0, 3999, 0, 0, 3, 4081, 0, 2027, 9, 0, 0, 3335, 0, 6, 0, 4043] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 1495, 0, 0, 3084, 4096, 0, 0, 0, 0, 0, 964, 4096, 0, 0, 14, 0, 0, 4096, 0, 3992, 0, 1, 0, 4053, 0, 756, 80, 0, 0, 4096, 11, 4096, 0, 4096, 14, 0, 0, 3419, 0, 0, 4090, 0, 1, 0, 0, 2625, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4082] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 38, 0, 4096, 3660, 0, 0, 3458, 4087, 0, 0, 0, 0, 0, 554, 4064, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3, 0, 4092, 0, 2479, 23, 0, 0, 4096, 0, 3844, 0, 4096, 0, 0, 0, 1642, 0, 0, 4096, 77, 11, 0, 0, 640, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 3924, 1951, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 3946, 4095, 0, 1, 0, 1, 0, 4094, 0, 4075, 0, 175, 65, 411, 7, 3803, 919, 0, 0, 4093, 0, 981, 0, 4096, 20, 88, 0, 4096, 0, 27, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4062, 0, 150, 0, 0, 0, 4096, 0, 0, 0, 4071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[3, 19, 0, 3965, 185, 282, 4, 0, 4085, 1, 0, 16, 0, 0, 4090, 4088, 0, 4, 0, 2, 0, 3940, 0, 4096, 10, 2437, 0, 3260, 52, 878, 0, 0, 1, 4093, 54, 4081, 0, 4094, 0, 7, 2, 1380, 0, 0, 4071, 0, 8, 0, 2, 4092, 0, 0, 0, 4096, 1, 8, 1, 6, 0, 4096, 0, 0, 5, 4021] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4, 0, 4096, 0, 0, 0, 2446, 4096, 0, 0, 0, 0, 0, 2669, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 37, 96, 4087, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 4096, 2, 0, 0, 4091, 0, 1, 4096, 0, 0, 0, 0, 3053, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 3978, 0, 0, 18, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 408, 0, 0, 1164, 4096, 0, 0, 0, 0, 0, 2470, 4096, 0, 0, 0, 0, 0, 4096, 0, 3670, 0, 617, 83, 3714, 0, 2024, 8, 0, 0, 4096, 0, 4092, 0, 4096, 1, 0, 0, 3918, 0, 0, 4096, 7, 0, 0, 0, 2663, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 3828, 0, 0, 4, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4, 0, 4, 3909, 413, 0, 43, 8, 4024, 0, 0, 27, 7, 0, 1131, 4050, 0, 49, 0, 0, 42, 4034, 61, 536, 0, 1171, 0, 4055, 505, 2370, 0, 0, 0, 4090, 190, 2607, 0, 4023, 2053, 632, 2, 1831, 0, 548, 4067, 0, 4, 0, 31, 4055, 18, 2, 7, 3438, 82, 3307, 0, 25, 45, 3945, 0, 36, 113, 3942] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4034, 0, 79, 4, 1, 1728, 94, 0, 0, 0, 0, 0, 0, 4095, 0, 5, 3, 0, 0, 1910, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4064, 0, 4089, 523, 0, 0, 0, 0, 0, 0, 0, 0, 3259, 0, 4093, 31, 97, 0, 0, 1973, 0, 0, 0, 0, 0, 0, 0, 2, 395, 0, 2343, 0, 4090, 432, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3043, 1, 0, 4096, 0, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 3, 4081, 0, 2027, 326, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3335, 0, 6, 0, 4043, 169, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5940, 7902, 8690, 9453, 7321, 8232, 10436, 7562] avg=8192.0000 max=10436 max_rank=6 min=5940 min_rank=0 imbalance_ratio=1.273926 max_load_reduction=55100 max_load_reduction_pct=0.840759 moved_to_echo=33551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 38, 0, 4096, 3660, 0, 0, 3458, 1728, 93, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 554, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 520, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 0, 2479, 23, 0, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 3844, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1642, 0, 0, 4096, 77, 11, 0, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 640, 0, 0, 0, 4096, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 168, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13073, 6516, 8712, 8570, 8335, 6910, 5060, 8360] avg=8192.0000 max=13073 max_rank=0 min=5060 min_rank=6 imbalance_ratio=1.595825 max_load_reduction=52463 max_load_reduction_pct=0.800522 moved_to_echo=28665 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3924, 1951, 0, 0, 0, 1726, 94, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 3946, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4094, 0, 4075, 524, 0, 0, 0, 0, 0, 0, 0, 0, 175, 65, 411, 7, 3803, 919, 0, 1972, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 981, 0, 4096, 20, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 27, 4096, 0, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4062, 0, 150, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 169, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7695, 9934, 8695, 7352, 5580, 9311, 8633, 8336] avg=8192.0000 max=9934 max_rank=1 min=5580 min_rank=4 imbalance_ratio=1.212646 max_load_reduction=55602 max_load_reduction_pct=0.848419 moved_to_echo=31860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 1495, 0, 0, 3084, 1728, 94, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 964, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 3992, 524, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4053, 0, 756, 80, 0, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 395, 11, 4096, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 0, 0, 4090, 0, 1, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 2625, 0, 0, 0, 4096, 0, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4082, 169, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10488, 6951, 8626, 6863, 8612, 8602, 7047, 8347] avg=8192.0000 max=10488 max_rank=0 min=6863 min_rank=3 imbalance_ratio=1.280273 max_load_reduction=55048 max_load_reduction_pct=0.839966 moved_to_echo=32608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 4096, 0, 0, 0, 2446, 1728, 94, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2669, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 524, 0, 0, 0, 0, 0, 0, 0, 0, 37, 96, 4087, 0, 0, 0, 0, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 4088, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 4096, 0, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 3053, 0, 0, 0, 4096, 0, 6, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 18, 4096, 169, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8368, 8656, 8716, 6193, 8581, 9280, 7481, 8261] avg=8192.0000 max=9280 max_rank=5 min=6193 min_rank=3 imbalance_ratio=1.132812 max_load_reduction=56256 max_load_reduction_pct=0.858398 moved_to_echo=33603 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 19, 0, 3965, 185, 282, 4, 0, 1726, 94, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 16, 0, 0, 4090, 1886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 3940, 0, 4096, 523, 0, 0, 0, 0, 0, 0, 0, 10, 2437, 0, 3260, 52, 878, 0, 0, 1972, 0, 0, 0, 0, 0, 0, 0, 1, 395, 54, 4081, 0, 4094, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1380, 0, 0, 4071, 0, 8, 0, 1090, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 0, 0, 0, 4096, 1, 8, 326, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 4096, 0, 0, 5, 4021, 169, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6278, 10078, 8565, 8609, 8632, 6551, 8525, 8298] avg=8192.0000 max=10078 max_rank=1 min=6278 min_rank=0 imbalance_ratio=1.230225 max_load_reduction=55458 max_load_reduction_pct=0.846222 moved_to_echo=32006 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4, 3909, 413, 0, 43, 8, 1728, 95, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 27, 7, 0, 1131, 1853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 42, 4034, 61, 536, 520, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 0, 4055, 505, 2370, 0, 0, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 386, 190, 2607, 0, 4023, 2053, 632, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1831, 0, 548, 4067, 0, 4, 0, 1085, 0, 0, 0, 0, 0, 0, 0, 31, 4055, 18, 2, 7, 3438, 82, 3307, 328, 0, 0, 0, 0, 0, 0, 0, 0, 25, 45, 3945, 0, 36, 113, 3942, 169, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6204, 7042, 5242, 10077, 9891, 7537, 11268, 8275] avg=8192.0000 max=11268 max_rank=6 min=5242 min_rank=2 imbalance_ratio=1.375488 max_load_reduction=54268 max_load_reduction_pct=0.828064 moved_to_echo=36971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 408, 0, 0, 1164, 1728, 94, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2470, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3670, 524, 0, 0, 0, 0, 0, 0, 0, 0, 617, 83, 3714, 0, 2024, 8, 0, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 4092, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 0, 4096, 7, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 0, 0, 4096, 0, 1, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 0, 0, 4, 4096, 169, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7490, 8457, 8290, 8419, 8584, 9113, 7086, 8097] avg=8192.0000 max=9113 max_rank=5 min=7086 min_rank=6 imbalance_ratio=1.112427 max_load_reduction=56423 max_load_reduction_pct=0.860947 moved_to_echo=32880 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 31488, 0, 128, 256, 32512, 1408, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 3200, 256, 26240, 0, 32768, 2560, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 23424, 128, 640, 32768, 128, 128, 0, 8832, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 32640, 128, 28672, 4224, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 17792, 15104, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 7808, 256, 27776, 640, 12416, 1152, 0, 15872, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 25344, 128, 128, 128, 32128, 128, 5504, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32256, 8192, 384, 128, 10240, 13824, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True + [2026-08-30 04:59:27] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 713.5 | throughput per GPU (TFLOP/s/GPU): 467.3 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.313443E+01 | loss scale: 1.0 | grad norm: 39.511 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[28, 234, 2611, 5, 821, 5, 1322, 22, 2566, 55, 282, 961, 905, 238, 2, 860, 205, 2415, 1637, 960, 3775, 862, 295, 1506, 2, 3385, 107, 734, 416, 19, 176, 1448, 4036, 124, 51, 778, 1154, 21, 820, 2238, 2695, 2382, 371, 221, 7, 2565, 1232, 2197, 133, 1398, 350, 11, 15, 59, 2833, 216, 5, 3391, 3915, 1847, 225, 1025, 358, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[23, 131, 1938, 15, 750, 2, 960, 11, 2951, 129, 508, 881, 581, 292, 18, 921, 367, 1504, 2149, 977, 3835, 1038, 293, 1794, 1, 3414, 109, 655, 921, 80, 511, 1110, 3852, 115, 98, 1389, 1123, 28, 1102, 2130, 3288, 2451, 960, 263, 0, 2296, 647, 1146, 230, 1388, 398, 34, 49, 58, 2944, 292, 2, 2883, 3563, 1485, 371, 891, 1179, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[16, 146, 2017, 12, 825, 0, 882, 4, 2608, 70, 404, 1117, 863, 229, 5, 806, 296, 1534, 2089, 820, 3846, 902, 343, 1657, 1, 3431, 54, 1168, 366, 20, 213, 1080, 4002, 61, 36, 972, 1182, 17, 568, 2079, 3205, 2390, 541, 204, 23, 2534, 1091, 2489, 114, 1702, 530, 2, 6, 38, 3027, 199, 2, 3279, 3753, 1593, 339, 1330, 394, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[18, 147, 2322, 6, 693, 0, 978, 5, 2648, 74, 266, 1306, 1445, 77, 8, 713, 358, 1873, 2187, 693, 3858, 1042, 425, 1921, 4, 3124, 162, 714, 342, 63, 210, 1627, 3822, 123, 67, 1118, 1140, 23, 588, 2028, 3325, 2675, 791, 467, 28, 1731, 931, 2068, 210, 1580, 424, 18, 3, 60, 2596, 167, 0, 3271, 3797, 1312, 184, 986, 622, 72] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[16, 54, 2285, 0, 524, 0, 1172, 3, 2783, 66, 283, 820, 350, 262, 1, 844, 388, 2210, 1951, 955, 3915, 781, 449, 1491, 0, 3698, 91, 620, 353, 10, 110, 1416, 3995, 108, 58, 1092, 1521, 17, 485, 2460, 3205, 2313, 231, 284, 7, 2792, 1224, 2101, 112, 1532, 508, 26, 5, 13, 2828, 151, 0, 3346, 3696, 1656, 207, 1249, 390, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[27, 97, 2169, 11, 469, 0, 815, 8, 2815, 185, 466, 1066, 907, 425, 6, 897, 290, 2165, 2237, 777, 3757, 1082, 312, 1890, 2, 3134, 122, 860, 482, 21, 224, 1557, 3883, 107, 66, 1585, 1353, 18, 498, 2343, 3031, 2233, 360, 167, 11, 2372, 1081, 1556, 178, 1540, 412, 14, 19, 45, 2553, 268, 1, 3306, 3722, 1594, 362, 1014, 545, 24] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[19, 91, 2184, 1, 579, 3, 964, 3, 2896, 80, 326, 668, 599, 237, 0, 727, 373, 2059, 2094, 636, 3864, 950, 261, 1857, 0, 3641, 122, 601, 407, 24, 224, 1158, 4018, 82, 65, 1019, 1365, 27, 769, 1864, 3014, 2793, 818, 242, 2, 2499, 963, 2351, 134, 1377, 415, 5, 13, 12, 2982, 310, 0, 2889, 3794, 1877, 228, 1288, 657, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[11, 68, 2419, 3, 935, 0, 927, 3, 2044, 86, 258, 832, 919, 275, 0, 791, 262, 2334, 1895, 1255, 3702, 876, 287, 1724, 1, 3533, 119, 836, 278, 22, 128, 1463, 3995, 165, 47, 1081, 1633, 18, 803, 2398, 2537, 1828, 566, 192, 3, 2399, 1298, 1881, 180, 1851, 440, 26, 3, 53, 3192, 459, 0, 3063, 3779, 1436, 157, 1058, 681, 28] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[23, 131, 1938, 15, 750, 2, 960, 11, 3835, 295, 0, 0, 0, 0, 0, 0, 2951, 129, 508, 881, 581, 292, 18, 921, 2116, 0, 0, 0, 0, 0, 0, 0, 367, 1504, 2090, 977, 0, 1038, 293, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3414, 109, 655, 921, 80, 511, 1110, 1253, 446, 59, 17, 0, 0, 0, 0, 2599, 115, 98, 1389, 1123, 28, 1102, 2130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2451, 960, 263, 0, 1850, 647, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 230, 1388, 398, 34, 49, 58, 2944, 292, 2993, 79, 0, 0, 0, 0, 0, 0, 2, 2883, 1351, 1485, 371, 891, 1179, 12, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7960, 8397, 8063, 8576, 8584, 7317, 8465, 8174] avg=8192.0000 max=8584 max_rank=4 min=7317 min_rank=5 imbalance_ratio=1.047852 max_load_reduction=56952 max_load_reduction_pct=0.869019 moved_to_echo=32540 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[28, 234, 2611, 5, 821, 5, 1322, 22, 3775, 241, 0, 0, 0, 0, 0, 0, 2566, 55, 282, 961, 905, 238, 2, 860, 2325, 0, 0, 0, 0, 0, 0, 0, 205, 2415, 1592, 960, 0, 862, 295, 1506, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3385, 107, 734, 416, 19, 176, 1448, 1313, 498, 45, 19, 0, 0, 0, 0, 2723, 124, 51, 778, 1154, 21, 820, 2238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2382, 371, 221, 7, 2067, 1232, 2197, 0, 0, 0, 0, 0, 0, 0, 0, 133, 1398, 350, 11, 15, 59, 2833, 216, 2454, 87, 0, 0, 0, 0, 0, 0, 5, 3391, 1484, 1847, 225, 1025, 358, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9064, 8194, 7835, 8162, 7909, 8477, 7556, 8339] avg=8192.0000 max=9064 max_rank=0 min=7556 min_rank=6 imbalance_ratio=1.106445 max_load_reduction=56472 max_load_reduction_pct=0.861694 moved_to_echo=32281 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[18, 147, 2322, 6, 693, 0, 978, 5, 3858, 297, 0, 0, 0, 0, 0, 0, 2648, 74, 266, 1306, 1445, 77, 8, 713, 2255, 0, 0, 0, 0, 0, 0, 0, 358, 1873, 2127, 693, 0, 1042, 425, 1921, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3124, 162, 714, 342, 63, 210, 1627, 1244, 336, 60, 18, 0, 0, 0, 0, 2578, 123, 67, 1118, 1140, 23, 588, 2028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2675, 791, 467, 28, 1395, 931, 2068, 0, 0, 0, 0, 0, 0, 0, 0, 210, 1580, 424, 18, 3, 60, 2596, 167, 3028, 84, 0, 0, 0, 0, 0, 0, 0, 3271, 1440, 1312, 184, 986, 622, 72, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 8792, 8439, 7904, 7665, 8355, 8170, 7887] avg=8192.0000 max=8792 max_rank=1 min=7665 min_rank=4 imbalance_ratio=1.073242 max_load_reduction=56744 max_load_reduction_pct=0.865845 moved_to_echo=32077 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[19, 91, 2184, 1, 579, 3, 964, 3, 3864, 270, 0, 0, 0, 0, 0, 0, 2896, 80, 326, 668, 599, 237, 0, 727, 2253, 0, 0, 0, 0, 0, 0, 0, 373, 2059, 2037, 636, 0, 950, 261, 1857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3641, 122, 601, 407, 24, 224, 1158, 1307, 486, 57, 18, 0, 0, 0, 0, 2711, 82, 65, 1019, 1365, 27, 769, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2793, 818, 242, 2, 2013, 963, 2351, 0, 0, 0, 0, 0, 0, 0, 0, 134, 1377, 415, 5, 13, 12, 2982, 310, 2744, 84, 0, 0, 0, 0, 0, 0, 0, 2889, 1439, 1877, 228, 1288, 657, 16, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7978, 7786, 8173, 8045, 7902, 9182, 8076, 8394] avg=8192.0000 max=9182 max_rank=5 min=7786 min_rank=1 imbalance_ratio=1.120850 max_load_reduction=56354 max_load_reduction_pct=0.859894 moved_to_echo=33554 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 146, 2017, 12, 825, 0, 882, 4, 3846, 287, 0, 0, 0, 0, 0, 0, 2608, 70, 404, 1117, 863, 229, 5, 806, 2229, 0, 0, 0, 0, 0, 0, 0, 296, 1534, 2032, 820, 0, 902, 343, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3431, 54, 1168, 366, 20, 213, 1080, 1302, 492, 57, 18, 0, 0, 0, 0, 2700, 61, 36, 972, 1182, 17, 568, 2079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2390, 541, 204, 23, 2042, 1091, 2489, 0, 0, 0, 0, 0, 0, 0, 0, 114, 1702, 530, 2, 6, 38, 3027, 199, 2918, 83, 0, 0, 0, 0, 0, 0, 2, 3279, 1423, 1593, 339, 1330, 394, 10, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8035, 8331, 7584, 8202, 7615, 8780, 8619, 8370] avg=8192.0000 max=8780 max_rank=5 min=7584 min_rank=2 imbalance_ratio=1.071777 max_load_reduction=56756 max_load_reduction_pct=0.866028 moved_to_echo=33384 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[27, 97, 2169, 11, 469, 0, 815, 8, 3757, 272, 0, 0, 0, 0, 0, 0, 2815, 185, 466, 1066, 907, 425, 6, 897, 2216, 0, 0, 0, 0, 0, 0, 0, 290, 2165, 2173, 777, 0, 1082, 312, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3134, 122, 860, 482, 21, 224, 1557, 1268, 465, 64, 21, 0, 0, 0, 0, 2615, 107, 66, 1585, 1353, 18, 498, 2343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2233, 360, 167, 11, 1907, 1081, 1556, 0, 0, 0, 0, 0, 0, 0, 0, 178, 1540, 412, 14, 19, 45, 2553, 268, 2759, 88, 0, 0, 0, 0, 0, 0, 1, 3306, 1397, 1594, 362, 1014, 545, 24, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7625, 8983, 8689, 8220, 8585, 7315, 7876, 8243] avg=8192.0000 max=8983 max_rank=1 min=7315 min_rank=5 imbalance_ratio=1.096558 max_load_reduction=56553 max_load_reduction_pct=0.862930 moved_to_echo=32019 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 68, 2419, 3, 935, 0, 927, 3, 3702, 227, 0, 0, 0, 0, 0, 0, 2044, 86, 258, 832, 919, 275, 0, 791, 2244, 0, 0, 0, 0, 0, 0, 0, 262, 2334, 1843, 1255, 0, 876, 287, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3533, 119, 836, 278, 22, 128, 1463, 1300, 466, 52, 18, 0, 0, 0, 0, 2695, 165, 47, 1081, 1633, 18, 803, 2398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1828, 566, 192, 3, 1933, 1298, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 180, 1851, 440, 26, 3, 53, 3192, 459, 2310, 84, 0, 0, 0, 0, 0, 0, 0, 3063, 1433, 1436, 157, 1058, 681, 28, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 7449, 8581, 8216, 8840, 7701, 8598, 7856] avg=8192.0000 max=8840 max_rank=4 min=7449 min_rank=1 imbalance_ratio=1.079102 max_load_reduction=56696 max_load_reduction_pct=0.865112 moved_to_echo=32995 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 54, 2285, 0, 524, 0, 1172, 3, 3915, 286, 0, 0, 0, 0, 0, 0, 2783, 66, 283, 820, 350, 262, 1, 844, 2195, 0, 0, 0, 0, 0, 0, 0, 388, 2210, 1898, 955, 0, 781, 449, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3698, 91, 620, 353, 10, 110, 1416, 1300, 543, 53, 17, 0, 0, 0, 0, 2695, 108, 58, 1092, 1521, 17, 485, 2460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2313, 231, 284, 7, 2249, 1224, 2101, 0, 0, 0, 0, 0, 0, 0, 0, 112, 1532, 508, 26, 5, 13, 2828, 151, 2919, 82, 0, 0, 0, 0, 0, 0, 0, 3346, 1402, 1656, 207, 1249, 390, 23, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8255, 7604, 8172, 8211, 8436, 8409, 8176, 8273] avg=8192.0000 max=8436 max_rank=4 min=7604 min_rank=1 imbalance_ratio=1.029785 max_load_reduction=57100 max_load_reduction_pct=0.871277 moved_to_echo=33294 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 19072, 4736, 2048, 128, 15488, 8576, 15872, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 27392, 896, 6272, 3584, 384, 1920, 10880, 10368, 3840, 512, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[21376, 896, 512, 9088, 10496, 256, 5760, 17664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1408, 12416, 3584, 256, 128, 384, 23040, 2176, 22144, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 25472, 11392, 12800, 2176, 8960, 4864, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2560, 16128, 15872, 7168, 0, 7552, 2688, 13952, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 1024, 18048, 128, 5632, 128, 8064, 128, 30592, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[21376, 768, 2816, 7680, 6656, 2048, 128, 6656, 17920, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 23, 0, 4081, 0, 4080, 4096, 4096, 1, 0, 0, 0, 4096, 4094, 4096, 0, 2, 1399, 1, 0, 0, 5, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2707, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 4096, 0, 2, 3958, 1, 4082, 4076, 4096, 1, 4, 1, 4, 4094, 4065, 4085, 0, 25, 3465, 21, 0, 2, 4, 0, 3, 0, 4094, 4033, 13, 0, 13, 8, 0, 3622, 0, 0, 1, 4096, 4094, 0, 0, 0, 0, 4093, 25, 0, 0, 4, 0, 0, 0, 1, 1, 1257, 0, 4093, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 2, 0, 0, 4, 4093, 6, 4, 4073, 0, 4092, 4086, 4093, 0, 0, 1, 1, 4096, 4065, 4093, 0, 0, 1287, 17, 0, 0, 119, 0, 0, 0, 4091, 4084, 0, 0, 0, 0, 2, 3939, 0, 0, 3, 4096, 4095, 0, 1, 0, 0, 4092, 1, 8, 0, 3, 29, 0, 0, 2, 0, 2856, 3, 4095, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2, 1, 10, 0, 0, 6, 2, 4086, 0, 0, 3087, 2, 3998, 4046, 4072, 0, 25, 0, 2, 4046, 4085, 4082, 3, 15, 4057, 53, 7, 4, 296, 0, 12, 2, 4020, 4058, 13, 0, 0, 2, 0, 4087, 14, 0, 6, 4083, 4079, 1, 11, 77, 0, 4055, 24, 34, 1, 1, 48, 0, 14, 0, 0, 847, 0, 4058, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 4096, 31, 2, 3935, 0, 3354, 4096, 4095, 0, 0, 0, 0, 4096, 4091, 4096, 0, 12, 2825, 0, 0, 0, 55, 12, 0, 0, 4068, 4094, 0, 1, 0, 2, 2, 3983, 0, 1, 0, 4091, 4096, 0, 0, 0, 0, 4094, 0, 57, 0, 0, 6, 0, 0, 0, 0, 2146, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 4096, 2, 47, 3728, 0, 4022, 4094, 4075, 0, 0, 0, 0, 4096, 4096, 4093, 0, 16, 3804, 6, 0, 0, 3, 27, 0, 0, 4096, 3892, 5, 0, 0, 1, 0, 4091, 0, 1, 0, 4086, 4094, 0, 0, 2, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 869, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 9, 0, 4053, 0, 3760, 4094, 4095, 0, 0, 1, 0, 4095, 4080, 4090, 1, 2, 4081, 7, 0, 0, 17, 1, 0, 1, 4096, 4089, 0, 0, 0, 2, 0, 4041, 1, 0, 0, 4096, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 1, 1, 1, 439, 0, 4094, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4, 0, 2, 0, 4, 0, 3, 4089, 50, 2, 4018, 0, 4027, 4094, 4093, 13, 0, 2, 0, 4094, 4084, 4096, 3, 7, 2487, 22, 11, 0, 495, 0, 0, 0, 4080, 4088, 0, 15, 0, 0, 0, 3895, 1, 3, 2, 4092, 4094, 0, 0, 27, 0, 4091, 0, 0, 1, 0, 1, 0, 1, 4, 0, 1361, 0, 4080, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 3063, 1036, 0, 0, 0, 0, 0, 0, 23, 0, 4081, 0, 4080, 376, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1399, 1, 0, 0, 5, 0, 0, 0, 4096, 1027, 0, 0, 0, 0, 0, 0, 161, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2899, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2707, 0, 4095, 0, 0, 2534, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8561, 8198, 6528, 8347, 8178, 8182, 9347] avg=8192.0000 max=9347 max_rank=7 min=6528 min_rank=3 imbalance_ratio=1.140991 max_load_reduction=56189 max_load_reduction_pct=0.857376 moved_to_echo=34054 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 2, 0, 0, 4, 4093, 3061, 1035, 0, 0, 0, 0, 0, 0, 6, 4, 4073, 0, 4092, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 4065, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1287, 17, 0, 0, 119, 0, 0, 0, 4093, 1027, 0, 0, 0, 0, 0, 0, 161, 4084, 0, 0, 0, 0, 2, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4084, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 8, 0, 3, 29, 0, 2895, 1184, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2856, 3, 4095, 3, 0, 2527, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8550, 8168, 6543, 8186, 8184, 8212, 9497] avg=8192.0000 max=9497 max_rank=7 min=6543 min_rank=3 imbalance_ratio=1.159302 max_load_reduction=56039 max_load_reduction_pct=0.855087 moved_to_echo=34079 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 10, 0, 0, 6, 2, 4086, 3053, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 3087, 2, 3998, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 2, 4046, 4085, 5, 3, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 53, 7, 4, 296, 0, 12, 2, 4072, 1024, 0, 0, 0, 0, 0, 0, 158, 4058, 13, 0, 0, 2, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 6, 4083, 4068, 1, 11, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 24, 34, 1, 1, 48, 0, 2845, 1172, 0, 0, 0, 0, 0, 0, 14, 0, 0, 847, 0, 4058, 2, 0, 2503, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 7458, 8181, 9527, 8318, 8260, 8180, 7435] avg=8192.0000 max=9527 max_rank=3 min=7435 min_rank=7 imbalance_ratio=1.162964 max_load_reduction=56009 max_load_reduction_pct=0.854630 moved_to_echo=32193 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 4096, 3063, 1029, 0, 0, 0, 0, 0, 0, 31, 2, 3935, 0, 3354, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 5, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 2825, 0, 0, 0, 55, 12, 0, 0, 4095, 1028, 0, 0, 0, 0, 0, 0, 160, 4094, 0, 1, 0, 2, 2, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 57, 0, 0, 6, 0, 2879, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2146, 0, 4096, 0, 0, 2534, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7698, 8204, 8015, 8242, 8177, 8222, 8787] avg=8192.0000 max=8787 max_rank=7 min=7698 min_rank=1 imbalance_ratio=1.072632 max_load_reduction=56749 max_load_reduction_pct=0.865921 moved_to_echo=33428 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 3059, 1036, 0, 0, 0, 0, 0, 0, 9, 0, 4053, 0, 3760, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4080, 5, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 7, 0, 0, 17, 1, 0, 1, 4095, 1026, 0, 0, 0, 0, 0, 0, 161, 4089, 0, 0, 0, 2, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 2899, 1186, 0, 0, 0, 0, 0, 0, 1, 1, 1, 439, 0, 4094, 0, 0, 2532, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8198, 8184, 9228, 8293, 8182, 8182, 7079] avg=8192.0000 max=9228 max_rank=3 min=7079 min_rank=7 imbalance_ratio=1.126465 max_load_reduction=56308 max_load_reduction_pct=0.859192 moved_to_echo=31736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 4096, 3061, 1036, 0, 0, 0, 0, 0, 0, 2, 47, 3728, 0, 4022, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 3804, 6, 0, 0, 3, 27, 0, 0, 4075, 1027, 0, 0, 0, 0, 0, 0, 161, 3892, 5, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 4083, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2899, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 0, 4096, 0, 0, 2532, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8175, 8213, 8942, 8150, 8172, 8182, 7508] avg=8192.0000 max=8942 max_rank=3 min=7508 min_rank=7 imbalance_ratio=1.091553 max_load_reduction=56594 max_load_reduction_pct=0.863556 moved_to_echo=32012 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 2, 0, 4, 0, 3, 4089, 3067, 1035, 0, 0, 0, 0, 0, 0, 50, 2, 4018, 0, 4027, 368, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 4084, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2487, 22, 11, 0, 495, 0, 0, 0, 4093, 1029, 0, 0, 0, 0, 0, 0, 153, 4088, 0, 15, 0, 0, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 2, 4092, 4078, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 1, 0, 2892, 1190, 0, 0, 0, 0, 0, 0, 1, 4, 0, 1361, 0, 4080, 0, 0, 2536, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8478, 8190, 8137, 8151, 8203, 8175, 7998] avg=8192.0000 max=8478 max_rank=1 min=7998 min_rank=7 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=32527 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 4096, 3055, 1036, 0, 0, 0, 0, 0, 0, 0, 2, 3958, 1, 4082, 374, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4, 4094, 4065, 5, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 3465, 21, 0, 2, 4, 0, 3, 0, 4096, 1025, 0, 0, 0, 0, 0, 0, 160, 4033, 13, 0, 13, 8, 0, 3622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 25, 0, 0, 4, 0, 0, 2898, 1181, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1257, 0, 4093, 0, 1, 2521, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8418, 8198, 8616, 7849, 8180, 8201, 7885] avg=8192.0000 max=8616 max_rank=3 min=7849 min_rank=4 imbalance_ratio=1.051758 max_load_reduction=56920 max_load_reduction_pct=0.868530 moved_to_echo=32115 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[23424, 128, 128, 128, 1024, 128, 128, 128, 32768, 8320, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 12544, 128, 32768, 128, 128, 20224, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1280, 32512, 128, 128, 128, 128, 128, 31872, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 30976, 128, 31488, 3072, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 0, 23168, 9472, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 32768, 24576, 8320, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 4094, 0, 0, 4024, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4039, 0, 0, 4096, 420, 48, 0, 1002, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 1469, 0, 22, 0, 1000, 4094, 0, 4096, 0, 0, 263, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4, 4095, 0, 4096, 0, 0, 4039, 4096, 4096, 0, 0, 1, 0, 4095, 0, 0, 2191, 0, 1, 4096, 3937, 1, 0, 691, 0, 1, 0, 0, 3951, 0, 0, 4096, 0, 0, 4094, 4, 34, 0, 0, 3, 0, 0, 1377, 0, 8, 0, 150, 4093, 0, 4092, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4094, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 55, 4066, 1, 4090, 0, 2, 3437, 4091, 4084, 1, 5, 3, 0, 3995, 0, 0, 4022, 24, 4, 4024, 2510, 145, 0, 3, 6, 1, 0, 15, 4090, 3, 0, 4045, 0, 0, 4070, 47, 0, 22, 0, 1, 52, 18, 1682, 14, 8, 13, 562, 4054, 0, 4072, 3, 5, 11, 5, 1, 4078, 1, 0, 0, 4079, 10, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 23, 4080, 0, 4096, 0, 7, 3688, 4096, 4078, 0, 0, 4, 0, 4096, 0, 0, 3953, 0, 0, 4096, 3751, 1, 0, 59, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 3, 0, 0, 0, 0, 0, 553, 0, 87, 0, 243, 4037, 0, 4096, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 21, 4080, 0, 4088, 0, 4, 4001, 4094, 4096, 1, 0, 5, 0, 4096, 0, 0, 4079, 0, 0, 4095, 3874, 2, 1, 65, 0, 17, 0, 0, 4092, 1, 0, 4094, 2, 0, 4093, 6, 0, 1, 0, 2, 0, 1, 179, 2, 0, 1, 50, 4090, 1, 4094, 1, 2, 17, 0, 0, 4095, 0, 0, 0, 4092, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 4089, 0, 4094, 0, 0, 3571, 4095, 4096, 0, 0, 1, 0, 4093, 0, 0, 4057, 0, 0, 4095, 3670, 214, 0, 53, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 4092, 3, 7, 0, 0, 0, 0, 1, 495, 0, 1, 1, 13, 4095, 1, 4095, 0, 0, 216, 0, 1, 4096, 0, 2, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4088, 0, 4096, 0, 0, 3155, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 3075, 0, 0, 4096, 3791, 202, 0, 899, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4084, 0, 1, 0, 0, 0, 0, 0, 970, 0, 44, 0, 202, 4056, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 23, 4096, 0, 4090, 0, 0, 2107, 4091, 4093, 4, 4, 0, 3, 4091, 0, 3, 4037, 5, 2, 4094, 3525, 115, 2, 1253, 0, 0, 0, 2, 4094, 0, 0, 4094, 0, 0, 4093, 0, 16, 0, 3, 0, 0, 0, 531, 2, 20, 0, 564, 4038, 4, 4093, 2, 2, 148, 1, 6, 4093, 0, 0, 0, 4092, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 587, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 1233, 420, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 0, 0, 0, 0, 4094, 0, 0, 2863, 688, 56, 4, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1469, 0, 22, 0, 1000, 4082, 2819, 0, 0, 0, 0, 0, 0, 4094, 0, 4040, 0, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8705, 8206, 5740, 8707, 8195, 9392, 8397, 8194] avg=8192.0000 max=9392 max_rank=5 min=5740 min_rank=2 imbalance_ratio=1.146484 max_load_reduction=56144 max_load_reduction_pct=0.856689 moved_to_echo=34178 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 55, 4066, 1, 586, 0, 2, 3437, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4084, 1, 5, 3, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 24, 4, 1211, 2510, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 1, 0, 15, 4090, 3, 0, 2813, 688, 56, 4, 0, 0, 0, 0, 4045, 0, 0, 4065, 47, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 52, 18, 1682, 14, 8, 13, 562, 4077, 2816, 0, 0, 0, 0, 0, 0, 4054, 0, 4016, 3, 5, 11, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 0, 0, 4079, 10, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8102, 7916, 7679, 8179, 9243, 8095, 8174] avg=8192.0000 max=9243 max_rank=5 min=7679 min_rank=3 imbalance_ratio=1.128296 max_load_reduction=56293 max_load_reduction_pct=0.858963 moved_to_echo=33691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4095, 0, 587, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2191, 0, 1, 1233, 3937, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, 1, 0, 0, 3951, 0, 0, 2863, 689, 56, 4, 0, 0, 0, 0, 4096, 0, 0, 4089, 4, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1377, 0, 8, 0, 150, 4082, 2820, 0, 0, 0, 0, 0, 0, 4093, 0, 4036, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8725, 8206, 7363, 8255, 8223, 8440, 8130, 8194] avg=8192.0000 max=8725 max_rank=0 min=7363 min_rank=2 imbalance_ratio=1.065063 max_load_reduction=56811 max_load_reduction_pct=0.866867 moved_to_echo=32987 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 21, 4080, 0, 586, 0, 4, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 1, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 1232, 3874, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 17, 0, 0, 4092, 1, 0, 2863, 687, 56, 4, 0, 0, 0, 0, 4094, 2, 0, 4088, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 179, 2, 0, 1, 50, 4080, 2815, 0, 0, 0, 0, 0, 0, 4090, 1, 4038, 1, 2, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4092, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8692, 8212, 9188, 7785, 8191, 7130, 8149, 8189] avg=8192.0000 max=9188 max_rank=2 min=7130 min_rank=5 imbalance_ratio=1.121582 max_load_reduction=56348 max_load_reduction_pct=0.859802 moved_to_echo=31659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4089, 0, 587, 0, 0, 3571, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 1232, 3670, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 4096, 0, 0, 2863, 688, 56, 4, 0, 0, 0, 0, 4095, 0, 1, 4087, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 495, 0, 1, 1, 13, 4081, 2819, 0, 0, 0, 0, 0, 0, 4095, 1, 4039, 0, 0, 216, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8204, 9173, 7760, 8193, 7411, 8352, 8195] avg=8192.0000 max=9173 max_rank=2 min=7411 min_rank=5 imbalance_ratio=1.119751 max_load_reduction=56363 max_load_reduction_pct=0.860031 moved_to_echo=32151 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 23, 4080, 0, 587, 0, 7, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4078, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 0, 0, 1233, 3751, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 4096, 0, 0, 2863, 689, 56, 4, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 0, 87, 0, 243, 4082, 2820, 0, 0, 0, 0, 0, 0, 4037, 0, 4040, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8385, 8192, 8938, 7767, 8189, 7785, 8083, 8197] avg=8192.0000 max=8938 max_rank=2 min=7767 min_rank=3 imbalance_ratio=1.091064 max_load_reduction=56598 max_load_reduction_pct=0.863617 moved_to_echo=32254 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4088, 0, 587, 0, 0, 3155, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3075, 0, 0, 1233, 3791, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 899, 0, 0, 0, 0, 4096, 0, 0, 2863, 689, 56, 4, 0, 0, 0, 0, 4096, 0, 0, 4079, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 0, 44, 0, 202, 4082, 2820, 0, 0, 0, 0, 0, 0, 4056, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7830, 8206, 8301, 8607, 8176, 8118, 8096, 8202] avg=8192.0000 max=8607 max_rank=3 min=7830 min_rank=0 imbalance_ratio=1.050659 max_load_reduction=56929 max_load_reduction_pct=0.868668 moved_to_echo=32592 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 23, 4096, 0, 577, 0, 0, 2107, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4093, 4, 4, 0, 3, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4037, 5, 2, 1228, 3525, 115, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 0, 0, 0, 2, 4094, 0, 0, 2866, 691, 60, 10, 0, 0, 0, 0, 4094, 0, 0, 4077, 0, 16, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 2, 20, 0, 564, 4078, 2822, 0, 0, 0, 0, 0, 0, 4038, 4, 4033, 2, 2, 148, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4092, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6803, 8208, 8917, 8976, 8190, 8017, 8234, 8191] avg=8192.0000 max=8976 max_rank=3 min=6803 min_rank=0 imbalance_ratio=1.095703 max_load_reduction=56560 max_load_reduction_pct=0.863037 moved_to_echo=32632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32640, 128, 32384, 128, 128, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 29568, 128, 128, 9856, 25600, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 4736, 0, 128, 28032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 7296, 128, 256, 128, 2816, 32768, 22656, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4096, 128, 128, 0, 128, 32640, 128, 0, 22912, 5632, 512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4094, 0, 3, 0, 4095, 1, 4096, 0, 4085, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 14, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 1, 4047, 2, 0, 0, 4085, 99, 4096, 0, 3954, 0, 0, 16, 0, 0, 4092, 0, 1, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 4096, 4093, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 4094, 0, 0, 20, 0, 39, 0, 0, 0, 4096, 4096, 0, 1, 0, 42, 0, 0, 4095, 3, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[3989, 1, 18, 4081, 0, 169, 9, 4075, 78, 4086, 14, 3827, 4, 0, 15, 5, 5, 4035, 0, 0, 7, 4093, 21, 0, 4082, 3, 1, 0, 1, 13, 4087, 28, 4084, 3945, 5, 0, 38, 0, 0, 0, 0, 13, 0, 7, 4073, 13, 0, 4, 0, 31, 15, 1, 0, 4087, 4081, 1, 0, 9, 197, 6, 0, 4083, 8, 4088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4090, 0, 0, 4094, 0, 5, 0, 4095, 3, 4094, 0, 4095, 0, 2, 11, 0, 2, 4072, 7, 21, 0, 4093, 0, 0, 4094, 2, 0, 0, 0, 0, 4094, 0, 4086, 4066, 1, 0, 0, 2, 0, 0, 2, 0, 2, 1, 4094, 1, 0, 0, 0, 1, 0, 2, 0, 4094, 4094, 0, 0, 3, 49, 0, 0, 4071, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4092, 0, 0, 4038, 0, 0, 0, 4096, 3, 4095, 0, 4052, 0, 0, 0, 0, 0, 4096, 0, 46, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4057, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 31, 0, 46, 0, 0, 0, 4096, 4096, 0, 0, 0, 20, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4091, 0, 11, 0, 4096, 3, 4096, 0, 4088, 0, 0, 0, 0, 1, 4094, 4, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4071, 4095, 0, 0, 6, 0, 1, 0, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 2, 0, 11, 0, 0, 4096, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4048, 0, 2, 0, 4022, 0, 4096, 0, 4085, 0, 0, 3, 0, 0, 4087, 0, 3, 0, 4078, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 4091, 0, 0, 85, 0, 9, 0, 0, 0, 4096, 4096, 0, 12, 0, 12, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4092, 2, 0, 4066, 0, 0, 2, 4093, 23, 4093, 0, 4093, 1, 0, 2, 2, 0, 4065, 0, 0, 5, 4096, 2, 0, 4093, 2, 2, 11, 3, 4, 4090, 1, 4080, 4064, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 27, 3, 4, 3, 0, 7, 4088, 4092, 0, 0, 3, 27, 2, 0, 4093, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4094, 0, 3, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4085, 0, 0, 0, 0, 15, 12, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 31, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8209, 8193, 8189, 8224, 8173, 8191, 8149] avg=8192.0000 max=8224 max_rank=4 min=8149 min_rank=7 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 1, 4047, 2, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 99, 4096, 0, 3954, 0, 0, 16, 0, 15, 12, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 4091, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 1, 2, 0, 31, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 20, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 42, 0, 0, 4095, 3, 4053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8192, 8188, 8181, 8226, 8181, 8215, 8194] avg=8192.0000 max=8226 max_rank=4 min=8159 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3989, 1, 18, 4081, 0, 169, 9, 15, 0, 0, 0, 0, 0, 0, 0, 0, 78, 4086, 14, 3827, 4, 0, 15, 5, 15, 12, 0, 0, 0, 0, 0, 0, 5, 4035, 0, 0, 7, 4093, 21, 0, 3, 1, 0, 0, 0, 0, 0, 0, 4079, 3, 1, 0, 1, 13, 4087, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 3945, 5, 0, 38, 0, 0, 0, 31, 3, 0, 0, 0, 0, 0, 0, 0, 13, 0, 7, 4073, 13, 0, 4, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 31, 15, 1, 0, 4071, 4081, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 197, 6, 0, 4083, 8, 4045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8056, 8165, 8212, 8106, 8167, 8200, 8348] avg=8192.0000 max=8348 max_rank=7 min=8056 min_rank=1 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 4094, 0, 5, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 4095, 0, 2, 11, 0, 15, 12, 0, 0, 0, 0, 0, 0, 2, 4072, 7, 21, 0, 4093, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4066, 1, 0, 0, 2, 0, 0, 31, 3, 0, 0, 0, 0, 0, 0, 2, 0, 2, 1, 4094, 1, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 4078, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 49, 0, 0, 4071, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8232, 8199, 8187, 8189, 8177, 8175, 8173] avg=8192.0000 max=8232 max_rank=1 min=8173 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32714 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4091, 0, 11, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4088, 0, 0, 0, 0, 15, 12, 0, 0, 0, 0, 0, 0, 1, 4094, 4, 0, 0, 4093, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4095, 0, 0, 6, 0, 1, 0, 31, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 11, 0, 0, 4096, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8214, 8196, 8189, 8207, 8182, 8175, 8160] avg=8192.0000 max=8214 max_rank=1 min=8160 min_rank=7 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32724 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4048, 0, 2, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4085, 0, 0, 3, 0, 15, 12, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 3, 0, 4078, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 31, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 4091, 0, 0, 85, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 12, 0, 0, 4096, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8211, 8172, 8189, 8197, 8249, 8185, 8173] avg=8192.0000 max=8249 max_rank=5 min=8160 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32804 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 4038, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 4052, 0, 0, 0, 0, 15, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 46, 0, 4095, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4057, 0, 0, 0, 0, 1, 0, 31, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 31, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4096, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8177, 8241, 8189, 8188, 8205, 8222, 8169] avg=8192.0000 max=8241 max_rank=2 min=8145 min_rank=0 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4092, 2, 0, 4066, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4093, 0, 4093, 1, 0, 2, 2, 16, 15, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 5, 4096, 2, 0, 6, 8, 0, 0, 0, 0, 0, 0, 4087, 2, 2, 11, 3, 4, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4064, 11, 0, 0, 0, 0, 0, 34, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 27, 4080, 0, 0, 0, 0, 0, 0, 0, 3, 4, 3, 0, 7, 4064, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 27, 2, 0, 4093, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8245, 8182, 8200, 8199, 8202, 8173, 8170] avg=8192.0000 max=8245 max_rank=1 min=8165 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 32640, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 32512, 128, 0, 128, 128, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 32768, 128, 32384, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 256, 128, 128, 128, 32640, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 256, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32640, 128, 128, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 384, 128, 0, 32768, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[15, 0, 0, 4078, 11, 22, 20, 5, 0, 14, 7, 4068, 4064, 13, 0, 4080, 0, 4059, 15, 11, 34, 4064, 0, 12, 0, 4041, 0, 15, 4051, 34, 0, 0, 1, 4074, 0, 0, 0, 7, 4080, 4081, 0, 9, 0, 12, 4072, 3, 3, 13, 29, 4056, 2, 68, 0, 0, 1, 15, 4066, 22, 19, 9, 4078, 5, 4075, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4093, 0, 0, 0, 4054, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 20, 4096, 35, 0, 0, 0, 4085, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 5, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 4094, 2, 4096, 0, 0, 12, 4054, 0, 0, 0, 4083, 0, 0, 4095, 0, 0, 0, 106, 4096, 2, 0, 2, 0, 4032, 4095, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 4093, 4055, 0, 0, 4096, 0, 4075, 0, 1, 0, 4096, 2, 0, 0, 4088, 8, 1, 4091, 0, 0, 2, 0, 4096, 25, 5, 2, 5, 4096, 4094, 33, 2, 0, 0, 4094, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4096, 0, 4045, 0, 0, 0, 3816, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 263, 4096, 92, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[3, 5, 0, 4093, 0, 2, 4, 3, 0, 3, 42, 4089, 4091, 3, 0, 4088, 0, 4050, 0, 10, 0, 4091, 0, 2, 0, 4091, 0, 2, 4091, 5, 0, 1, 0, 4092, 4, 0, 0, 14, 4093, 4092, 0, 4, 0, 4, 4092, 3, 2, 0, 0, 4075, 0, 0, 0, 0, 0, 1, 4093, 4, 7, 0, 4093, 2, 4087, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 4082, 0, 0, 4096, 0, 4082, 0, 0, 0, 4042, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 59, 3, 4, 0, 4086, 4096, 27, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 0, 0, 4096, 0, 4095, 0, 0, 0, 4091, 1, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 3, 4096, 1, 0, 1, 1, 4096, 4096, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[15, 0, 0, 4078, 11, 22, 20, 5, 4067, 4, 0, 0, 0, 0, 0, 0, 0, 14, 7, 4068, 4064, 13, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 15, 11, 34, 4064, 0, 12, 55, 10, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 15, 4051, 34, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 1, 4010, 0, 0, 0, 7, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 12, 4072, 3, 3, 13, 4061, 11, 0, 0, 0, 0, 0, 0, 29, 4056, 2, 68, 0, 0, 1, 15, 4077, 0, 0, 0, 0, 0, 0, 0, 4066, 22, 19, 9, 0, 5, 4074, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8175, 8260, 8151, 8098, 8184, 8248, 8198] avg=8192.0000 max=8260 max_rank=2 min=8098 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32728 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4085, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4054, 0, 0, 56, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 20, 4031, 35, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8202, 8213, 8202, 8171, 8183, 8189, 8191] avg=8192.0000 max=8213 max_rank=2 min=8171 min_rank=4 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32734 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 1, 4096, 0, 0, 0, 0, 4083, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 12, 4054, 0, 0, 56, 10, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 4095, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 106, 4031, 2, 0, 2, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4074, 11, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8198, 8230, 8188, 8173, 8183, 8183, 8191] avg=8192.0000 max=8230 max_rank=2 min=8173 min_rank=4 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32730 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 4083, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 4055, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 1, 0, 4096, 2, 0, 56, 10, 0, 0, 0, 0, 0, 0, 0, 4088, 8, 1, 4091, 0, 0, 2, 9, 1, 0, 0, 0, 0, 0, 0, 0, 4031, 25, 5, 2, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 2, 0, 0, 4094, 0, 0, 0, 4076, 11, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8159, 8240, 8200, 8164, 8216, 8184, 8191] avg=8192.0000 max=8240 max_rank=2 min=8159 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4096, 0, 0, 0, 0, 4086, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 3816, 0, 0, 56, 10, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 263, 4031, 92, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8193, 7927, 8183, 8481, 8182, 8190, 8191] avg=8192.0000 max=8481 max_rank=4 min=7927 min_rank=2 imbalance_ratio=1.035278 max_load_reduction=57055 max_load_reduction_pct=0.870590 moved_to_echo=33044 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 5, 0, 4093, 0, 2, 4, 3, 4083, 5, 0, 0, 0, 0, 0, 0, 0, 3, 42, 4089, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 10, 0, 4091, 0, 2, 57, 13, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2, 4091, 5, 0, 1, 13, 6, 0, 0, 0, 0, 0, 0, 0, 4022, 4, 0, 0, 14, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 4092, 3, 2, 0, 4075, 10, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 7, 0, 0, 2, 4081, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8228, 8223, 8209, 8133, 8190, 8163, 8192] avg=8192.0000 max=8228 max_rank=1 min=8133 min_rank=4 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32678 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4085, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4091, 1, 0, 56, 10, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 3, 4031, 1, 0, 1, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 4076, 11, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8197, 8253, 8201, 8133, 8188, 8188, 8191] avg=8192.0000 max=8253 max_rank=2 min=8133 min_rank=4 imbalance_ratio=1.007446 max_load_reduction=57283 max_load_reduction_pct=0.874069 moved_to_echo=32700 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4085, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4082, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4042, 0, 0, 56, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 4031, 59, 3, 4, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 0, 0, 4076, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8184, 8190, 8202, 8183, 8210, 8191, 8191] avg=8192.0000 max=8210 max_rank=5 min=8183 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=57326 max_load_reduction_pct=0.874725 moved_to_echo=32775 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32640, 128, 128, 128, 32384, 128, 128, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 32256, 256, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 32768, 128, 128, 32768, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 437, 0, 0, 0, 4096, 0, 0, 3647, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 2, 0, 4092, 0, 0, 0, 4091, 0, 0, 0, 4096, 10, 0, 4096, 0, 1, 1, 0, 4095, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 0, 0, 3707, 4096, 4096, 0, 28, 4096, 0, 3913, 0, 0, 0, 4096, 0, 0, 602, 0, 20, 0, 1, 0, 4073, 0, 4096, 0, 35, 7, 4067, 0, 0, 0, 4035, 1, 0, 0, 4096, 1, 0, 4094, 0, 4, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4081] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 1, 0, 2, 4072, 4094, 4096, 3, 25, 4087, 1, 2992, 1, 1, 0, 4092, 0, 2, 869, 0, 0, 0, 0, 0, 4081, 0, 4095, 1, 14, 9, 4084, 0, 0, 0, 4096, 0, 0, 0, 4095, 4, 0, 4096, 0, 1, 70, 2, 4096, 171, 1, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[13, 4038, 0, 16, 16, 4, 0, 26, 4082, 4016, 4060, 18, 0, 4079, 12, 845, 0, 2, 14, 4058, 0, 31, 2829, 13, 7, 38, 0, 0, 4082, 32, 4063, 25, 591, 0, 3984, 8, 0, 5, 4086, 0, 0, 0, 4067, 68, 1, 4079, 3, 44, 25, 20, 3911, 94, 0, 0, 8, 19, 0, 20, 17, 0, 4054, 0, 0, 4013] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3964, 4096, 4096, 0, 0, 4073, 0, 2265, 0, 0, 0, 4096, 0, 0, 2279, 0, 62, 0, 0, 0, 4092, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 3929, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3908] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 0, 0, 0, 4088, 4096, 4096, 0, 2, 4095, 0, 2990, 0, 0, 0, 4095, 0, 0, 692, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 367, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 4096, 0, 3, 1, 0, 4096, 85, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4044, 4096, 4096, 0, 33, 4096, 0, 4042, 0, 0, 0, 4096, 0, 0, 308, 0, 39, 0, 0, 0, 4096, 0, 4096, 0, 3, 0, 4071, 0, 0, 0, 3834, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[13, 4086, 0, 5, 3, 0, 0, 0, 4066, 4091, 4081, 0, 74, 4092, 0, 1638, 1, 4, 0, 4084, 7, 6, 2341, 1, 2, 7, 0, 0, 4092, 3, 4090, 6, 269, 9, 4041, 2, 0, 3, 3734, 0, 4, 1, 4090, 0, 0, 4093, 0, 14, 2, 2, 4090, 197, 0, 3, 0, 4, 0, 3, 8, 0, 4088, 0, 0, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 1768, 0, 0, 0, 0, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3647, 0, 2328, 58, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4065, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 4083, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 13, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 13, 12, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 6301, 10144, 8175, 8158, 8190, 8145, 8229] avg=8192.0000 max=10144 max_rank=2 min=6301 min_rank=1 imbalance_ratio=1.238281 max_load_reduction=55392 max_load_reduction_pct=0.845215 moved_to_echo=32722 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3707, 1768, 0, 0, 28, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 602, 0, 2328, 58, 15, 0, 0, 0, 0, 0, 20, 0, 1, 0, 4073, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 7, 4040, 0, 0, 0, 4035, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4081, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4081, 13, 12, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 9416, 7099, 8178, 8118, 8182, 8133, 8213] avg=8192.0000 max=9416 max_rank=1 min=7099 min_rank=2 imbalance_ratio=1.149414 max_load_reduction=56120 max_load_reduction_pct=0.856323 moved_to_echo=32646 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[13, 4038, 0, 16, 16, 4, 0, 26, 4060, 2, 0, 0, 0, 0, 0, 0, 4082, 1733, 0, 18, 0, 0, 12, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 14, 4058, 0, 31, 2829, 13, 2283, 58, 15, 0, 0, 0, 0, 0, 7, 38, 0, 0, 4082, 32, 4051, 25, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 3957, 8, 0, 5, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 68, 1, 4067, 3, 44, 0, 0, 0, 0, 0, 0, 0, 0, 25, 20, 3911, 94, 0, 0, 8, 19, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 20, 17, 0, 4054, 0, 0, 4013, 12, 12, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 6690, 9303, 8235, 8647, 8250, 8096, 8140] avg=8192.0000 max=9303 max_rank=2 min=6690 min_rank=1 imbalance_ratio=1.135620 max_load_reduction=56233 max_load_reduction_pct=0.858047 moved_to_echo=33133 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 1, 0, 2, 4096, 2, 0, 0, 0, 0, 0, 0, 4072, 1767, 0, 3, 25, 0, 1, 2992, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4092, 0, 2, 869, 0, 2327, 59, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 14, 9, 4057, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 4083, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 70, 2, 4096, 171, 1, 0, 0, 1, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4093, 13, 12, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8860, 7366, 8165, 8176, 8183, 8367, 8224] avg=8192.0000 max=8860 max_rank=1 min=7366 min_rank=2 imbalance_ratio=1.081543 max_load_reduction=56676 max_load_reduction_pct=0.864807 moved_to_echo=32950 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4088, 1768, 0, 0, 2, 0, 0, 2990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 692, 0, 2328, 59, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 2, 4069, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4083, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 85, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 13, 12, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8848, 7189, 8142, 8534, 8183, 8216, 8229] avg=8192.0000 max=8848 max_rank=1 min=7189 min_rank=2 imbalance_ratio=1.080078 max_load_reduction=56688 max_load_reduction_pct=0.864990 moved_to_echo=33162 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[13, 4086, 0, 5, 3, 0, 0, 0, 4081, 3, 0, 0, 0, 0, 0, 0, 4066, 1759, 0, 0, 74, 0, 0, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4084, 7, 6, 2341, 1, 2332, 58, 20, 0, 0, 0, 0, 0, 2, 7, 0, 0, 4092, 3, 4074, 6, 0, 0, 0, 0, 0, 0, 0, 0, 269, 9, 4005, 2, 0, 3, 3734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4090, 0, 0, 4079, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4090, 197, 0, 3, 0, 4, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 4088, 0, 0, 4086, 14, 16, 16, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7537, 8854, 8184, 8022, 8188, 8329, 8231] avg=8192.0000 max=8854 max_rank=2 min=7537 min_rank=1 imbalance_ratio=1.080811 max_load_reduction=56682 max_load_reduction_pct=0.864899 moved_to_echo=32770 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4044, 1768, 0, 0, 33, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 308, 0, 2328, 58, 15, 0, 0, 0, 0, 0, 39, 0, 0, 0, 4096, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4044, 0, 0, 0, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 13, 12, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 9887, 6805, 8219, 7881, 8181, 8140, 8229] avg=8192.0000 max=9887 max_rank=1 min=6805 min_rank=2 imbalance_ratio=1.206909 max_load_reduction=55649 max_load_reduction_pct=0.849136 moved_to_echo=32431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3964, 1768, 0, 0, 0, 0, 0, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2279, 0, 2328, 58, 15, 0, 0, 0, 0, 0, 62, 0, 0, 0, 4092, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4069, 0, 0, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3908, 13, 12, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 7997, 8776, 8238, 8000, 8179, 8110, 8041] avg=8192.0000 max=8776 max_rank=2 min=7997 min_rank=1 imbalance_ratio=1.071289 max_load_reduction=56760 max_load_reduction_pct=0.866089 moved_to_echo=32330 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 32512, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32128, 14208, 0, 128, 256, 0, 128, 19200, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1408, 128, 32384, 128, 0, 128, 32000, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 13568, 128, 18688, 512, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32640, 640, 128, 128, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 128, 128, 0, 32768, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1519, 17, 13, 13, 3577, 22, 4075, 0, 22, 0, 45, 24, 59, 24, 3889, 26, 0, 0, 17, 3967, 5, 27, 7, 0, 23, 4066, 0, 4070, 3122, 0, 21, 4071, 4033, 0, 34, 1, 23, 1, 4075, 0, 0, 1, 4059, 0, 199, 19, 36, 4080, 0, 4, 0, 13, 4053, 0, 0, 4074, 0, 11, 25, 0, 4073, 4001, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[803, 0, 0, 0, 714, 0, 4096, 0, 0, 0, 240, 0, 0, 0, 4085, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4094, 2747, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3814, 0, 0, 3748, 0, 157, 0, 0, 4096, 0, 0, 4096, 22, 0, 0, 26, 4096, 4096, 28, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[2323, 0, 0, 0, 2649, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 3833, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 2255, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 32, 0, 0, 4019, 0, 126, 0, 0, 4095, 0, 0, 4096, 1100, 0, 0, 15, 4096, 4096, 25, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[2774, 0, 0, 0, 255, 0, 4093, 0, 0, 0, 21, 2, 0, 0, 2690, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 2, 4095, 3535, 2, 0, 4096, 4096, 0, 1, 0, 0, 3, 4091, 2, 4, 0, 4096, 8, 2998, 0, 0, 4090, 0, 29, 1, 0, 4094, 0, 0, 4095, 0, 0, 0, 4, 4095, 4076, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[984, 0, 0, 0, 1420, 0, 4096, 0, 0, 0, 372, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 1678, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3488, 0, 0, 4069, 0, 247, 0, 0, 4096, 0, 0, 4096, 38, 0, 0, 0, 4096, 4096, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[2427, 3, 0, 3, 2341, 6, 4089, 2, 6, 0, 214, 0, 0, 4, 3984, 6, 0, 0, 2, 4077, 4, 5, 0, 0, 0, 4090, 0, 4091, 2233, 0, 0, 4091, 4089, 3, 13, 5, 4, 0, 4091, 0, 10, 13, 4078, 0, 917, 5, 6, 4078, 0, 0, 0, 1, 4087, 0, 6, 4090, 0, 4, 5, 241, 4086, 4026, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1215, 0, 0, 0, 1209, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 3287, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3740, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 2806, 0, 0, 4096, 0, 4, 0, 0, 4096, 0, 0, 4096, 6, 0, 0, 7, 4096, 4094, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3492, 0, 0, 0, 899, 0, 4096, 0, 0, 0, 112, 0, 0, 0, 4064, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3207, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 364, 0, 0, 4068, 0, 13, 0, 0, 4096, 0, 0, 4096, 31, 0, 0, 135, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2323, 0, 0, 0, 2649, 0, 4096, 0, 264, 188, 63, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3833, 0, 4096, 210, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 2604, 1506, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1282, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2326, 1, 32, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 4095, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 0, 0, 15, 3908, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9583, 8146, 8206, 7633, 8193, 6378, 8253, 9144] avg=8192.0000 max=9583 max_rank=0 min=6378 min_rank=5 imbalance_ratio=1.169800 max_load_reduction=55953 max_load_reduction_pct=0.853775 moved_to_echo=31968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[803, 0, 0, 0, 714, 0, 4096, 0, 264, 188, 63, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 4085, 0, 4096, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2603, 1506, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1281, 2747, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2326, 0, 3814, 0, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 4096, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 26, 3908, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6128, 8631, 8205, 8126, 8193, 9888, 8285, 8080] avg=8192.0000 max=9888 max_rank=5 min=6128 min_rank=0 imbalance_ratio=1.207031 max_load_reduction=55648 max_load_reduction_pct=0.849121 moved_to_echo=34446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1519, 17, 13, 13, 3577, 22, 4075, 0, 261, 187, 62, 0, 0, 0, 0, 0, 22, 0, 45, 24, 59, 24, 3889, 26, 4071, 209, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3967, 5, 27, 7, 0, 2587, 1493, 0, 0, 0, 0, 0, 0, 23, 4066, 0, 1274, 3122, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 34, 1, 23, 1, 4075, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2305, 0, 199, 19, 36, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 13, 4053, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 25, 0, 3886, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9746, 8369, 8103, 8506, 8168, 6640, 8081, 7923] avg=8192.0000 max=9746 max_rank=0 min=6640 min_rank=5 imbalance_ratio=1.189697 max_load_reduction=55790 max_load_reduction_pct=0.851288 moved_to_echo=30812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2774, 0, 0, 0, 255, 0, 4093, 0, 264, 188, 63, 0, 0, 0, 0, 0, 0, 0, 21, 2, 0, 0, 2690, 0, 4096, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2603, 1506, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 1282, 3535, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 3, 4091, 2, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2326, 8, 2998, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 1, 0, 4094, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3907, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7637, 7019, 8203, 8915, 8194, 9426, 8155, 7987] avg=8192.0000 max=9426 max_rank=5 min=7019 min_rank=1 imbalance_ratio=1.150635 max_load_reduction=56110 max_load_reduction_pct=0.856171 moved_to_echo=33762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1215, 0, 0, 0, 1209, 0, 4096, 0, 264, 188, 63, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 3287, 0, 4096, 210, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 2604, 1506, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1282, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2326, 0, 2806, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7, 3908, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7035, 7608, 8208, 9118, 8192, 9228, 8132, 8015] avg=8192.0000 max=9228 max_rank=5 min=7035 min_rank=0 imbalance_ratio=1.126465 max_load_reduction=56308 max_load_reduction_pct=0.859192 moved_to_echo=33567 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[984, 0, 0, 0, 1420, 0, 4096, 0, 264, 188, 63, 0, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 4087, 0, 4096, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2604, 1506, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1282, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2326, 0, 3488, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 4096, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 3908, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7015, 8765, 8206, 7056, 8193, 9883, 8375, 8043] avg=8192.0000 max=9883 max_rank=5 min=7015 min_rank=0 imbalance_ratio=1.206421 max_load_reduction=55653 max_load_reduction_pct=0.849197 moved_to_echo=34494 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2427, 3, 0, 3, 2341, 6, 4089, 2, 266, 188, 65, 0, 0, 0, 0, 0, 6, 0, 214, 0, 0, 4, 3984, 6, 4091, 212, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4077, 4, 5, 0, 0, 2605, 1506, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1274, 2233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 13, 5, 4, 0, 4091, 0, 5, 0, 0, 0, 0, 0, 0, 0, 10, 13, 2306, 0, 917, 5, 6, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 6, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 241, 3898, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9390, 8517, 8199, 7597, 8210, 7335, 8114, 8174] avg=8192.0000 max=9390 max_rank=0 min=7335 min_rank=5 imbalance_ratio=1.146240 max_load_reduction=56146 max_load_reduction_pct=0.856720 moved_to_echo=31833 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3492, 0, 0, 0, 899, 0, 4096, 0, 264, 188, 63, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 4064, 0, 4095, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2604, 1506, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1282, 3207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2326, 0, 364, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 135, 3908, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9002, 8481, 8206, 8585, 8193, 6758, 8141, 8170] avg=8192.0000 max=9002 max_rank=0 min=6758 min_rank=5 imbalance_ratio=1.098877 max_load_reduction=56534 max_load_reduction_pct=0.862640 moved_to_echo=31262 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 32640, 128, 128, 128, 0, 20864, 12160, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1280, 128, 128, 512, 31232, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 18688, 128, 14720, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 10240, 22528, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[15616, 128, 128, 128, 13184, 128, 32768, 128, 2176, 1536, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 640, 128, 128, 32768, 0, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 1152, 128, 128, 128, 29952, 128, 32768, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 21, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4051, 4039, 0, 0, 0, 0, 4070, 26, 24, 78, 25, 10, 4064, 4061, 20, 0, 21, 4066, 4077, 30, 0, 0, 0, 1, 5, 31, 2, 0, 3974, 2, 447, 13, 45, 103, 4065, 1, 4069, 0, 19, 4058, 0, 4055, 0, 0, 21, 29, 27, 29, 4065, 18, 14, 1, 0, 0, 18, 5, 17, 0, 3574, 0, 1, 34, 4064, 4067] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4061, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 48, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 4, 0, 1, 4084, 4094, 0, 0, 0, 4096, 4095, 0, 0, 0, 2, 0, 2, 0, 0, 0, 4086, 0, 1, 0, 0, 0, 4094, 0, 4096, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 61, 2, 7, 0, 0, 0, 4044, 1, 1, 0, 4095, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 101, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 4086, 4096, 0, 0, 0, 4096, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 2, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 4094, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4091, 4089, 0, 0, 0, 2, 4088, 4, 7, 7, 5, 7, 4082, 4089, 3, 0, 0, 4089, 4091, 5, 0, 0, 4, 0, 3, 0, 0, 14, 4059, 7, 62, 0, 10, 1, 4090, 0, 4090, 0, 6, 4089, 0, 4084, 0, 0, 6, 4, 2, 9, 4092, 2, 4, 0, 13, 0, 28, 0, 0, 1, 4010, 3, 3, 4, 4089, 4088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3997, 17, 12, 6, 2, 1, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4079, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 99, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8186, 8191, 8131, 8180, 8186, 8181, 8291] avg=8192.0000 max=8291 max_rank=7 min=8131 min_rank=3 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32838 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 21, 4096, 0, 0, 0, 3997, 18, 12, 6, 2, 1, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4, 0, 0, 0, 4078, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 99, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8181, 8191, 8155, 8180, 8187, 8171, 8282] avg=8192.0000 max=8282 max_rank=7 min=8155 min_rank=3 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32820 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4049, 4039, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 24, 78, 25, 10, 4064, 4055, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4066, 4076, 30, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 31, 2, 0, 3974, 2, 447, 13, 3969, 17, 12, 6, 2, 1, 0, 0, 45, 103, 4053, 1, 0, 0, 19, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 21, 29, 27, 29, 4063, 0, 0, 0, 0, 0, 0, 0, 4065, 18, 14, 1, 0, 0, 18, 5, 4053, 6, 0, 0, 0, 0, 0, 0, 17, 0, 3574, 0, 1, 34, 4064, 98, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8276, 8194, 8481, 8279, 8224, 8180, 7788] avg=8192.0000 max=8481 max_rank=3 min=7788 min_rank=7 imbalance_ratio=1.035278 max_load_reduction=57055 max_load_reduction_pct=0.870590 moved_to_echo=32471 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 4084, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4086, 0, 1, 0, 3995, 17, 12, 6, 2, 1, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 61, 2, 7, 0, 4077, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 1, 1, 0, 4095, 99, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8177, 8192, 8122, 8178, 8189, 8248, 8240] avg=8192.0000 max=8248 max_rank=6 min=8122 min_rank=3 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32855 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4094, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3997, 17, 12, 6, 2, 1, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 4079, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 99, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8195, 8192, 8131, 8180, 8187, 8181, 8278] avg=8192.0000 max=8278 max_rank=7 min=8131 min_rank=3 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32826 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 4095, 0, 0, 0, 3997, 17, 12, 6, 2, 1, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4079, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 99, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8151, 8191, 8178, 8180, 8186, 8182, 8278] avg=8192.0000 max=8278 max_rank=7 min=8151 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32826 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4094, 4096, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4086, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 1, 0, 3997, 18, 12, 6, 2, 1, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4078, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 4096, 99, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8179, 8203, 8143, 8180, 8187, 8179, 8174] avg=8192.0000 max=8291 max_rank=0 min=8143 min_rank=3 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4085, 4089, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 5, 7, 4082, 4080, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4084, 5, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 14, 4059, 7, 62, 0, 3993, 18, 17, 9, 6, 7, 0, 0, 10, 1, 4073, 0, 0, 0, 6, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 6, 4, 2, 9, 4085, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 4, 0, 13, 0, 28, 0, 4070, 5, 0, 0, 0, 0, 0, 0, 0, 1, 4010, 3, 3, 4, 4089, 95, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8191, 8182, 8195, 8179, 8190, 8214, 8205] avg=8192.0000 max=8214 max_rank=6 min=8179 min_rank=4 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 32768, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32640, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32000, 128, 128, 128, 32768, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 512, 128, 32000, 256, 128, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 200, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 59, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3996, 3547, 0, 16, 4096, 374, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 3, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 284, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 4093, 0, 5, 0, 0, 0, 0, 0, 3990, 3916, 0, 10, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4036, 12, 4069, 0, 19, 0, 0, 4080, 0, 18, 0, 49, 0, 23, 42, 4069, 65, 28, 0, 4075, 0, 4070, 0, 11, 4072, 0, 0, 2, 4060, 0, 4065, 36, 217, 24, 0, 31, 26, 8, 4043, 41, 4076, 27, 4081, 0, 65, 0, 0, 0, 18, 26, 3624, 4077, 34, 0, 4059, 63, 4062, 14, 0, 0, 0, 0, 0, 19] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4094, 0, 4095, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 12, 0, 0, 4091, 1, 4094, 0, 0, 4095, 0, 0, 1, 4095, 3, 4094, 0, 979, 0, 2, 0, 2, 0, 4088, 0, 4094, 2, 4094, 5, 0, 0, 2, 1, 1, 1, 2838, 4094, 0, 148, 4095, 128, 4095, 0, 0, 0, 2, 2, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4084, 0, 4091, 0, 0, 5, 2, 4091, 0, 1, 0, 0, 0, 5, 21, 4088, 1, 6, 0, 4091, 0, 4090, 8, 5, 4088, 4, 0, 3, 4090, 8, 4086, 8, 2917, 34, 0, 0, 0, 2, 4090, 6, 4091, 4, 4093, 0, 1, 0, 0, 0, 5, 6, 1556, 3662, 0, 0, 4087, 0, 4090, 3, 0, 0, 4, 3, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 95, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 997, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 3438, 3670, 0, 10, 4096, 29, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 2, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 198, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 3971, 4069, 0, 7, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 217, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 3923, 4046, 0, 2, 4096, 6, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 2, 0, 0, 0, 0, 0, 0, 200, 0, 0, 4050, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 4096, 0, 3312, 41, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 3547, 0, 16, 782, 374, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8180, 8346, 8195, 7508, 8197, 8715, 8194] avg=8192.0000 max=8715 max_rank=6 min=7508 min_rank=4 imbalance_ratio=1.063843 max_load_reduction=56821 max_load_reduction_pct=0.867020 moved_to_echo=32614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 9, 3, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 0, 0, 0, 0, 4094, 0, 3312, 41, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 3916, 0, 10, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8180, 8146, 8195, 7731, 8199, 8698, 8194] avg=8192.0000 max=8698 max_rank=6 min=7731 min_rank=4 imbalance_ratio=1.061768 max_load_reduction=56838 max_load_reduction_pct=0.867279 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4036, 12, 10, 0, 19, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 49, 0, 23, 42, 4069, 4055, 2, 0, 0, 0, 0, 0, 0, 65, 28, 0, 4030, 0, 4070, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 4060, 0, 4065, 36, 0, 0, 0, 0, 0, 0, 0, 0, 217, 24, 0, 31, 26, 8, 4043, 41, 3282, 40, 0, 0, 0, 0, 0, 0, 4076, 27, 4081, 0, 65, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 18, 26, 3624, 4077, 34, 0, 775, 63, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 14, 0, 0, 0, 0, 0, 19, 4069, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8258, 8204, 8166, 7712, 8254, 8617, 8168] avg=8192.0000 max=8617 max_rank=6 min=7712 min_rank=4 imbalance_ratio=1.051880 max_load_reduction=56919 max_load_reduction_pct=0.868515 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 9, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4081, 2, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4046, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 4095, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 979, 0, 2, 0, 2, 0, 4088, 0, 3312, 40, 0, 0, 0, 0, 0, 0, 4094, 2, 4094, 5, 0, 0, 2, 1, 5, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2838, 4094, 0, 148, 781, 128, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 2, 0, 1, 4092, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8177, 8153, 8196, 8423, 8203, 7991, 8197] avg=8192.0000 max=8423 max_rank=4 min=7991 min_rank=6 imbalance_ratio=1.028198 max_load_reduction=57113 max_load_reduction_pct=0.871475 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 9, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 2, 0, 0, 0, 0, 0, 0, 95, 0, 0, 4050, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 0, 0, 0, 0, 0, 4096, 0, 3312, 41, 0, 0, 0, 0, 0, 0, 4096, 0, 4050, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3438, 3670, 0, 10, 782, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8180, 8241, 8195, 8446, 8151, 7929, 8194] avg=8192.0000 max=8446 max_rank=4 min=7929 min_rank=6 imbalance_ratio=1.031006 max_load_reduction=57090 max_load_reduction_pct=0.871124 moved_to_echo=32720 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 6, 0, 0, 5, 2, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 21, 4088, 4079, 7, 0, 0, 0, 0, 0, 0, 1, 6, 0, 4037, 0, 4090, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 4090, 8, 4086, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2917, 34, 0, 0, 0, 2, 4090, 6, 3311, 43, 0, 0, 0, 0, 0, 0, 4091, 4, 4093, 0, 1, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 5, 6, 1556, 3662, 0, 0, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 4, 3, 6, 0, 4088, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8201, 8147, 8199, 10403, 8200, 5998, 8200] avg=8192.0000 max=10403 max_rank=4 min=5998 min_rank=6 imbalance_ratio=1.269897 max_load_reduction=55133 max_load_reduction_pct=0.841263 moved_to_echo=32801 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 9, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4050, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 4096, 0, 3312, 41, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 4046, 0, 2, 782, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8180, 8147, 8195, 7666, 8195, 8759, 8194] avg=8192.0000 max=8759 max_rank=6 min=7666 min_rank=4 imbalance_ratio=1.069214 max_load_reduction=56777 max_load_reduction_pct=0.866348 moved_to_echo=32814 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 2, 0, 0, 0, 0, 0, 0, 4, 0, 2, 4050, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 4096, 0, 3312, 41, 0, 0, 0, 0, 0, 0, 4096, 0, 4036, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 4069, 0, 7, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8180, 8152, 8195, 7647, 8137, 8829, 8195] avg=8192.0000 max=8829 max_rank=6 min=7647 min_rank=4 imbalance_ratio=1.077759 max_load_reduction=56707 max_load_reduction_pct=0.865280 moved_to_echo=32808 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 0, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32640, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 128, 128, 32384, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 32768, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 27392, 31104, 128, 256, 6272, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[5888, 128, 128, 128, 128, 128, 32768, 128, 26496, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 158, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 3101, 4096, 0, 0, 4096, 0, 0, 851, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 1, 7, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4006, 4096, 0, 0, 4083, 0, 0, 115, 0, 4059, 0, 0, 4091, 0, 0, 0, 4096, 0, 2, 0, 7, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4095, 0, 2, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 1219, 0, 0, 0, 4093, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 4096, 0, 0, 4096, 0, 2740, 4090, 0, 0, 4096, 0, 4, 275, 0, 4094, 0, 0, 4094, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 3939, 0, 4096, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 45, 115, 46, 4063, 0, 4030, 3994, 34, 0, 21, 0, 4066, 117, 0, 0, 22, 4042, 29, 3, 4057, 22, 4075, 4047, 5, 30, 4014, 1, 0, 0, 41, 4020, 3, 39, 4073, 1, 27, 0, 4051, 21, 15, 1, 45, 19, 0, 45, 40, 4048, 7, 0, 25, 54, 0, 4068, 0, 0, 3942, 1, 4058, 4, 0, 10, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4082, 4096, 0, 0, 4096, 0, 2, 14, 0, 4095, 0, 0, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 4096, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 49, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4095, 0, 0, 4096, 0, 3798, 4096, 0, 0, 4096, 0, 2, 298, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4041, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 915, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2896, 4096, 0, 0, 4096, 0, 1, 375, 0, 4044, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4057, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 1367, 17, 0, 17, 4091, 10, 4053, 4085, 11, 0, 4, 0, 4087, 57, 0, 10, 4, 4088, 5, 10, 4083, 7, 2713, 4082, 0, 8, 4089, 2, 0, 0, 0, 4085, 0, 7, 4093, 0, 5, 0, 4086, 9, 3, 3, 0, 0, 4, 0, 9, 4084, 0, 0, 7, 0, 0, 4089, 0, 0, 4063, 0, 4087, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 158, 0, 0, 0, 3619, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 665, 0, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 4096, 0, 0, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3431, 477, 187, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4096, 0, 0, 0, 0, 44, 13, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7873, 8180, 7862, 8812, 8211, 8172, 8191, 8235] avg=8192.0000 max=8812 max_rank=3 min=7862 min_rank=2 imbalance_ratio=1.075684 max_load_reduction=56724 max_load_reduction_pct=0.865540 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1219, 0, 0, 0, 3616, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 665, 0, 2740, 0, 0, 0, 0, 0, 0, 0, 0, 3859, 0, 0, 4096, 0, 4, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 19, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 3431, 477, 187, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 0, 4096, 0, 3, 0, 0, 44, 13, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8931, 8198, 7501, 8234, 8214, 8170, 8192, 8096] avg=8192.0000 max=8931 max_rank=0 min=7501 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=56605 max_load_reduction_pct=0.863724 moved_to_echo=32672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 7, 0, 3619, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 665, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 4083, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 19, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 0, 0, 0, 0, 4072, 3431, 477, 187, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 2, 0, 1, 44, 13, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7723, 8176, 8765, 8063, 8174, 8176, 8207, 8252] avg=8192.0000 max=8765 max_rank=2 min=7723 min_rank=0 imbalance_ratio=1.069946 max_load_reduction=56771 max_load_reduction_pct=0.866257 moved_to_echo=32809 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 45, 115, 46, 3590, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 34, 0, 21, 0, 4053, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4042, 29, 3, 659, 22, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 3819, 5, 30, 4014, 1, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 3, 39, 19, 1, 27, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 21, 15, 1, 45, 19, 0, 45, 40, 4054, 3398, 473, 185, 0, 0, 0, 0, 4048, 7, 0, 25, 54, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 1, 4058, 4, 0, 10, 0, 43, 13, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7826, 8219, 8852, 7910, 8160, 8296, 8201, 8072] avg=8192.0000 max=8852 max_rank=2 min=7826 min_rank=0 imbalance_ratio=1.080566 max_load_reduction=56684 max_load_reduction_pct=0.864929 moved_to_echo=32729 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3619, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 665, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 4096, 0, 2, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 19, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 3431, 477, 187, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 1, 0, 0, 44, 13, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7715, 8179, 8843, 7977, 8211, 8171, 8191, 8249] avg=8192.0000 max=8843 max_rank=2 min=7715 min_rank=0 imbalance_ratio=1.079468 max_load_reduction=56693 max_load_reduction_pct=0.865067 moved_to_echo=32822 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 49, 0, 0, 0, 3619, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 665, 0, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 4096, 0, 2, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4077, 3431, 477, 187, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 4096, 0, 0, 0, 0, 44, 13, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7764, 8181, 8558, 8261, 8211, 8175, 8191, 8195] avg=8192.0000 max=8558 max_rank=2 min=7764 min_rank=0 imbalance_ratio=1.044678 max_load_reduction=56978 max_load_reduction_pct=0.869415 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 915, 0, 0, 0, 3619, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 665, 0, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 4096, 0, 1, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 19, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3431, 477, 187, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 4096, 0, 0, 0, 0, 44, 13, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8630, 8179, 7657, 8337, 8159, 8172, 8191, 8211] avg=8192.0000 max=8630 max_rank=0 min=7657 min_rank=2 imbalance_ratio=1.053467 max_load_reduction=56906 max_load_reduction_pct=0.868317 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1367, 17, 0, 17, 3610, 10, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 11, 0, 4, 0, 4067, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4, 4088, 5, 10, 661, 7, 2713, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 0, 8, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 7, 13, 0, 5, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 3, 0, 0, 4, 0, 9, 4080, 3422, 481, 193, 0, 0, 0, 0, 4084, 0, 0, 7, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 4087, 0, 0, 2, 0, 46, 20, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9074, 8224, 7498, 7942, 8196, 8204, 8172, 8226] avg=8192.0000 max=9074 max_rank=0 min=7498 min_rank=2 imbalance_ratio=1.107666 max_load_reduction=56462 max_load_reduction_pct=0.861542 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 32384, 128, 32768, 128, 128, 128, 128, 384, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 5376, 128, 27520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 0, 32640, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 128, 128, 256, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[30848, 128, 128, 32768, 128, 128, 2048, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 3712, 128, 128, 128, 28928, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32640, 27520, 3840, 1536, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 4096, 4096, 0, 646, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3476, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 3994, 0, 0, 0, 0, 0, 0, 4096, 1, 5, 2, 0, 4096, 4096, 4096, 24, 122, 4096, 4096, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4050, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4094, 4019, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4095, 4096, 4096, 47, 413, 4095, 4096, 0, 4094, 4089, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 4096, 4092, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 8, 1, 0, 4096, 2, 0, 3802, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4064, 4072, 26, 22, 0, 28, 57, 0, 4033, 0, 0, 0, 24, 3994, 4041, 4049, 2, 3323, 4064, 4023, 28, 4060, 4074, 0, 5, 0, 2, 24, 1, 0, 87, 4, 32, 0, 0, 0, 3983, 4062, 26, 28, 6, 0, 56, 0, 96, 3993, 0, 23, 0, 0, 0, 773, 0, 45, 76, 0, 31, 37, 4052, 41, 44, 4022, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 3600, 0, 0, 0, 0, 0, 0, 4096, 0, 48, 0, 0, 4096, 4096, 4096, 0, 1109, 4096, 4096, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3535, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3998, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 4096, 4096, 0, 1385, 4096, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2741, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4069, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4086, 4091, 7, 5, 0, 10, 7, 0, 4077, 0, 0, 0, 4, 4081, 4084, 4089, 0, 87, 4091, 4086, 16, 4087, 4090, 2, 0, 8, 2, 6, 0, 15, 18, 8, 6, 0, 0, 0, 4076, 4088, 5, 3, 12, 0, 10, 0, 1, 4068, 5, 9, 0, 0, 0, 4000, 0, 2, 0, 0, 6, 7, 4085, 6, 8, 4082, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4081, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 4096, 4096, 4096, 0, 22, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4094, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3210, 0, 646, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3424, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 4096, 660, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3476, 0, 130, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8207, 7952, 8180, 8204, 8188, 8837, 7702] avg=8192.0000 max=8837 max_rank=6 min=7702 min_rank=7 imbalance_ratio=1.078735 max_load_reduction=56699 max_load_reduction_pct=0.865158 moved_to_echo=32931 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3994, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 5, 2, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3210, 24, 122, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3424, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 660, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4050, 0, 130, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8212, 7452, 8180, 8204, 8188, 8858, 8276] avg=8192.0000 max=8858 max_rank=6 min=7452 min_rank=2 imbalance_ratio=1.081299 max_load_reduction=56678 max_load_reduction_pct=0.864838 moved_to_echo=33526 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4064, 4072, 26, 22, 0, 28, 57, 75, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 24, 3994, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3173, 2, 3323, 0, 4023, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 24, 1, 0, 87, 4064, 4048, 0, 0, 0, 0, 0, 0, 4, 32, 0, 0, 0, 3983, 4062, 26, 12, 0, 0, 0, 0, 0, 0, 0, 28, 6, 0, 56, 0, 96, 3993, 0, 3378, 660, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 773, 0, 45, 76, 4074, 651, 12, 0, 0, 0, 0, 0, 0, 31, 37, 4052, 41, 44, 4022, 3, 129, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8344, 8063, 10549, 8231, 8119, 8217, 5654, 8359] avg=8192.0000 max=10549 max_rank=2 min=5654 min_rank=6 imbalance_ratio=1.287720 max_load_reduction=54987 max_load_reduction_pct=0.839035 moved_to_echo=30349 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 4019, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4095, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3210, 47, 413, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 4082, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4092, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3424, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 4089, 660, 12, 0, 0, 0, 0, 0, 8, 1, 0, 4096, 2, 0, 3802, 0, 130, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8204, 7766, 8180, 8201, 8189, 8768, 8039] avg=8192.0000 max=8768 max_rank=6 min=7766 min_rank=2 imbalance_ratio=1.070312 max_load_reduction=56768 max_load_reduction_pct=0.866211 moved_to_echo=33197 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4092, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3210, 0, 1385, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3424, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2741, 0, 0, 0, 4094, 660, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4069, 0, 130, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8207, 8691, 8180, 8204, 8188, 7507, 8295] avg=8192.0000 max=8691 max_rank=2 min=7507 min_rank=6 imbalance_ratio=1.060913 max_load_reduction=56845 max_load_reduction_pct=0.867386 moved_to_echo=32194 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 4091, 7, 5, 0, 10, 7, 76, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4, 4081, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3194, 0, 87, 0, 4086, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8, 2, 6, 0, 15, 18, 4091, 4076, 0, 0, 0, 0, 0, 0, 8, 6, 0, 0, 0, 4076, 4088, 5, 13, 0, 0, 0, 0, 0, 0, 0, 3, 12, 0, 10, 0, 1, 4068, 5, 3420, 671, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4000, 0, 2, 0, 4090, 663, 11, 0, 0, 0, 0, 0, 0, 6, 7, 4085, 6, 8, 4082, 0, 135, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8163, 7383, 8218, 8196, 8190, 8775, 8329] avg=8192.0000 max=8775 max_rank=6 min=7383 min_rank=2 imbalance_ratio=1.071167 max_load_reduction=56761 max_load_reduction_pct=0.866104 moved_to_echo=33490 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3600, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 48, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3210, 0, 1109, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3424, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3535, 0, 0, 0, 4088, 660, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3998, 0, 130, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7772, 8252, 8415, 8187, 8204, 8188, 8294, 8224] avg=8192.0000 max=8415 max_rank=2 min=7772 min_rank=0 imbalance_ratio=1.027222 max_load_reduction=57121 max_load_reduction_pct=0.871597 moved_to_echo=32910 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4081, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3210, 0, 22, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3424, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 4096, 660, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 0, 130, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8228, 7328, 8180, 8204, 8188, 8843, 8312] avg=8192.0000 max=8843 max_rank=6 min=7328 min_rank=2 imbalance_ratio=1.079468 max_load_reduction=56693 max_load_reduction_pct=0.865067 moved_to_echo=33547 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 31616, 128, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 32128, 128, 128, 0, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[25728, 128, 7168, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 0, 0, 32640, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 128, 32640, 128, 27392, 5376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 27392, 0, 128, 128, 32768, 5376, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4078, 0, 0, 0, 182, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 2383, 0, 0, 4088, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1558, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[28, 11, 56, 0, 0, 3726, 2, 0, 7, 732, 4046, 0, 13, 0, 34, 0, 31, 4056, 0, 36, 0, 4022, 4070, 0, 4064, 0, 3191, 22, 29, 4082, 0, 29, 29, 35, 4053, 4067, 12, 4054, 0, 0, 3743, 33, 20, 0, 19, 69, 0, 4067, 16, 59, 76, 7, 26, 0, 557, 36, 77, 30, 31, 4068, 4058, 3966, 15, 26] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 0, 3464, 0, 1, 0, 797, 4096, 1, 0, 2, 0, 0, 0, 4096, 0, 0, 1, 4093, 4096, 0, 4095, 1, 3946, 0, 0, 4037, 3, 0, 0, 0, 4095, 4094, 0, 4094, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 7, 0, 0, 0, 55, 0, 0, 0, 1, 4093, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 3620, 0, 0, 0, 268, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 4084, 0, 4096, 0, 4000, 0, 0, 4055, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 356, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2892, 0, 0, 0, 1207, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 2526, 0, 0, 4094, 0, 16, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1571, 0, 0, 0, 0, 4078, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4012, 0, 0, 0, 65, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4073, 0, 0, 4079, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 49, 0, 1, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3384, 0, 0, 0, 739, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 4096, 0, 4046, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[20, 3, 50, 11, 0, 2726, 0, 0, 12, 1932, 4086, 0, 0, 0, 6, 0, 0, 4083, 0, 3, 0, 4077, 4091, 0, 4089, 0, 3276, 7, 6, 4092, 0, 61, 8, 8, 4085, 4090, 4, 4088, 0, 0, 3786, 11, 4, 0, 0, 5, 5, 4088, 0, 10, 42, 0, 3, 0, 371, 9, 0, 10, 4, 4089, 4081, 4084, 14, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4078, 0, 0, 4086, 593, 7, 0, 0, 0, 0, 0, 0, 182, 4096, 0, 0, 0, 0, 0, 3357, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 0, 2383, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 67, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1558, 0, 4094, 3503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8764, 7635, 8202, 7136, 8194, 8260, 9155, 8190] avg=8192.0000 max=9155 max_rank=6 min=7136 min_rank=3 imbalance_ratio=1.117554 max_load_reduction=56381 max_load_reduction_pct=0.860306 moved_to_echo=33799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 3620, 0, 0, 4074, 594, 7, 0, 0, 0, 0, 0, 0, 268, 4096, 0, 0, 0, 0, 0, 3357, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 0, 4000, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 67, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 356, 0, 4096, 3502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 7721, 8203, 8720, 8192, 8261, 7954, 8189] avg=8192.0000 max=8720 max_rank=3 min=7721 min_rank=1 imbalance_ratio=1.064453 max_load_reduction=56816 max_load_reduction_pct=0.866943 moved_to_echo=32596 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[28, 11, 56, 0, 0, 3726, 2, 0, 4060, 589, 6, 0, 0, 0, 0, 0, 7, 732, 4046, 0, 13, 0, 34, 0, 3331, 0, 0, 0, 0, 0, 0, 0, 31, 4056, 0, 36, 0, 4022, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 0, 3191, 22, 29, 4082, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 29, 35, 4053, 2, 12, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3743, 33, 20, 0, 19, 69, 0, 4067, 66, 1, 0, 0, 0, 0, 0, 0, 16, 59, 76, 7, 26, 0, 557, 36, 4065, 3469, 0, 0, 0, 0, 0, 0, 77, 30, 31, 4067, 0, 3966, 15, 26, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8478, 8163, 8155, 8014, 8185, 8018, 8311, 8212] avg=8192.0000 max=8478 max_rank=0 min=8014 min_rank=3 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=32726 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 2, 0, 3464, 0, 1, 4086, 594, 7, 0, 0, 0, 0, 0, 0, 797, 4096, 1, 0, 2, 0, 0, 3357, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4093, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 664, 1, 3946, 0, 0, 4037, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4094, 67, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 55, 0, 4092, 3502, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8253, 8200, 8651, 8191, 8241, 7656, 8188] avg=8192.0000 max=8651 max_rank=3 min=7656 min_rank=6 imbalance_ratio=1.056030 max_load_reduction=56885 max_load_reduction_pct=0.867996 moved_to_echo=32276 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2892, 0, 0, 4086, 594, 7, 0, 0, 0, 0, 0, 0, 1207, 4096, 0, 0, 0, 0, 0, 3357, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 0, 2526, 0, 0, 4094, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 67, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1571, 0, 4096, 3502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7579, 8660, 8202, 7301, 8192, 8260, 9169, 8173] avg=8192.0000 max=9169 max_rank=6 min=7301 min_rank=3 imbalance_ratio=1.119263 max_load_reduction=56367 max_load_reduction_pct=0.860092 moved_to_echo=33794 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3384, 0, 0, 4085, 594, 7, 0, 0, 0, 0, 0, 0, 739, 4096, 0, 0, 0, 0, 0, 3357, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 0, 4046, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 67, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4094, 3502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8192, 8202, 8803, 8195, 8261, 7622, 8190] avg=8192.0000 max=8803 max_rank=3 min=7622 min_rank=6 imbalance_ratio=1.074585 max_load_reduction=56733 max_load_reduction_pct=0.865677 moved_to_echo=32268 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4012, 0, 0, 4086, 593, 7, 0, 0, 0, 0, 0, 0, 65, 4096, 0, 0, 0, 0, 0, 3357, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 0, 4073, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 67, 2, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 49, 0, 4094, 3503, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8698, 7518, 8202, 8817, 8194, 8261, 7655, 8191] avg=8192.0000 max=8817 max_rank=3 min=7518 min_rank=1 imbalance_ratio=1.076294 max_load_reduction=56719 max_load_reduction_pct=0.865463 moved_to_echo=32301 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[20, 3, 50, 11, 0, 2726, 0, 0, 4084, 592, 8, 0, 0, 0, 0, 0, 12, 1932, 4086, 0, 0, 0, 6, 0, 3358, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 3, 0, 4077, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 3276, 7, 6, 4092, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 4085, 0, 4, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3786, 11, 4, 0, 0, 5, 5, 4088, 71, 4, 0, 0, 0, 0, 0, 0, 0, 10, 42, 0, 3, 0, 371, 9, 4090, 3489, 0, 0, 0, 0, 0, 0, 0, 10, 4, 4085, 0, 4084, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7494, 9394, 8170, 8094, 8193, 7974, 8014, 8203] avg=8192.0000 max=9394 max_rank=1 min=7494 min_rank=0 imbalance_ratio=1.146729 max_load_reduction=56142 max_load_reduction_pct=0.856659 moved_to_echo=32384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 27904, 128, 128, 32768, 4864, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 0, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5376, 128, 27520, 128, 128, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32128, 128, 128, 0, 128, 128, 128, 32768, 640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 256, 128, 128, 0, 4608, 128, 32768, 28032, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 6016, 32768, 128, 128, 128, 128, 0, 26880, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4064, 0, 1, 25, 45, 0, 0, 0, 43, 4059, 4048, 51, 4053, 3935, 10, 17, 0, 229, 33, 0, 0, 39, 29, 48, 1, 32, 0, 39, 13, 4038, 0, 0, 4051, 34, 4046, 22, 0, 0, 3889, 50, 43, 4063, 38, 0, 0, 56, 37, 0, 4057, 59, 4043, 15, 4057, 0, 3980, 4056, 11, 0, 0, 4053, 0, 24] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 88, 0, 0, 0, 4096, 4096, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3778, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 146, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 57, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 242, 0, 8, 0, 0, 0, 4096, 4096, 0, 4096, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 21, 0, 3752, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 171, 4096, 0, 4096, 0, 4096, 4091, 0, 0, 0, 4096, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 4094, 0, 0, 514, 0, 45, 0, 0, 0, 4095, 4096, 0, 4096, 3737, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 2, 0, 4096, 0, 4096, 0, 0, 2, 3773, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 123, 4096, 0, 4096, 0, 4092, 4095, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 352, 0, 5, 0, 0, 0, 4096, 4096, 0, 4096, 3648, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3974, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 199, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 68, 0, 22, 0, 0, 0, 4096, 4096, 0, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3691, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 4096, 396, 4096, 0, 4096, 0, 4096, 4087, 0, 0, 0, 4096, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4088, 10, 0, 588, 11, 47, 2, 0, 10, 4084, 4086, 27, 4087, 3558, 9, 6, 0, 7, 5, 4, 19, 8, 15, 10, 0, 1, 0, 2, 8, 4085, 0, 0, 4086, 8, 4072, 4, 0, 0, 3743, 10, 12, 4088, 9, 0, 0, 12, 0, 0, 4085, 2, 4088, 0, 4085, 0, 4080, 4085, 16, 0, 0, 4086, 181, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 1410, 0, 1, 0, 0, 0, 4096, 4096, 0, 4096, 2151, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4073, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 40, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 414, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 242, 0, 8, 3606, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 445, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4031, 55, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 21, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3752, 0, 0, 4096, 0, 0, 0, 0, 134, 60, 45, 0, 0, 0, 0, 0, 0, 0, 3962, 171, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4027, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8017, 8542, 8137, 8182, 8217, 8087, 8229, 8125] avg=8192.0000 max=8542 max_rank=1 min=8017 min_rank=0 imbalance_ratio=1.042725 max_load_reduction=56994 max_load_reduction_pct=0.869659 moved_to_echo=32658 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4064, 0, 1, 25, 45, 0, 3568, 65, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 4048, 51, 441, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 10, 17, 0, 229, 33, 0, 0, 39, 4059, 4003, 0, 0, 0, 0, 0, 0, 29, 48, 1, 32, 0, 39, 13, 4038, 3988, 54, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 34, 4046, 22, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3889, 50, 43, 4063, 38, 0, 0, 56, 133, 59, 44, 0, 0, 0, 0, 0, 37, 0, 3924, 59, 4043, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 3993, 11, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7768, 8518, 8390, 8242, 8157, 8375, 8078, 8008] avg=8192.0000 max=8518 max_rank=1 min=7768 min_rank=0 imbalance_ratio=1.039795 max_load_reduction=57018 max_load_reduction_pct=0.870026 moved_to_echo=32618 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 88, 3606, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 445, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 56, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 0, 4096, 0, 0, 0, 0, 134, 60, 45, 0, 0, 0, 0, 0, 0, 0, 3962, 146, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7854, 8630, 8170, 8184, 8196, 8113, 8204, 8185] avg=8192.0000 max=8630 max_rank=1 min=7854 min_rank=0 imbalance_ratio=1.053467 max_load_reduction=56906 max_load_reduction_pct=0.868317 moved_to_echo=32698 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4094, 0, 0, 514, 0, 45, 3606, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 445, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4095, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4031, 55, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 3773, 0, 0, 4096, 0, 0, 0, 0, 134, 60, 45, 0, 0, 0, 0, 0, 0, 0, 3962, 123, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8325, 8278, 8138, 8183, 8200, 8108, 8181, 8123] avg=8192.0000 max=8325 max_rank=0 min=8108 min_rank=5 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32612 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 68, 0, 22, 3606, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 445, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4031, 55, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3691, 0, 0, 4084, 0, 0, 0, 0, 134, 59, 45, 0, 0, 0, 0, 0, 0, 0, 3962, 396, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4024, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7857, 8571, 8137, 8182, 8196, 8013, 8454, 8126] avg=8192.0000 max=8571 max_rank=1 min=7857 min_rank=0 imbalance_ratio=1.046265 max_load_reduction=56965 max_load_reduction_pct=0.869217 moved_to_echo=32789 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 352, 0, 5, 3606, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 445, 3648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 56, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 4096, 0, 0, 0, 0, 134, 60, 45, 0, 0, 0, 0, 0, 0, 0, 3961, 199, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8189, 8151, 8184, 8196, 8309, 8256, 8128] avg=8192.0000 max=8309 max_rank=5 min=8123 min_rank=0 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32889 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 1410, 0, 1, 3606, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 445, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4031, 55, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4096, 0, 0, 0, 0, 134, 60, 45, 0, 0, 0, 0, 0, 0, 0, 3961, 40, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9178, 6692, 8241, 8182, 8196, 8408, 8097, 8542] avg=8192.0000 max=9178 max_rank=0 min=6692 min_rank=1 imbalance_ratio=1.120361 max_load_reduction=56358 max_load_reduction_pct=0.859955 moved_to_echo=33243 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4088, 10, 0, 588, 11, 47, 3605, 65, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 4086, 27, 433, 3558, 0, 0, 0, 0, 0, 0, 0, 0, 9, 6, 0, 7, 5, 4, 19, 8, 4084, 4030, 0, 0, 0, 0, 0, 0, 15, 10, 0, 1, 0, 2, 8, 4085, 4021, 55, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 8, 4072, 4, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 3743, 10, 12, 4088, 9, 0, 0, 12, 138, 62, 49, 0, 0, 0, 0, 0, 0, 0, 3947, 2, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4015, 16, 0, 0, 0, 181, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8414, 8116, 8172, 8197, 8178, 8123, 8037, 8299] avg=8192.0000 max=8414 max_rank=0 min=8037 min_rank=6 imbalance_ratio=1.027100 max_load_reduction=57122 max_load_reduction_pct=0.871613 moved_to_echo=32637 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 3584, 29184, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 31744, 1152, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 32768, 128, 128, 3200, 128, 256, 28928, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 256, 128, 128, 256, 128, 32768, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32640, 32256, 128, 0, 0, 0, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[30720, 128, 128, 32768, 128, 0, 0, 128, 1152, 512, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 32768, 32256, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1156, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2225, 0, 0, 1942, 0, 3425, 4096, 0, 3532, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3604, 11, 4093, 0, 0, 0, 0, 0, 4096, 4095, 2546, 0, 0, 369, 0, 1670, 4095, 0, 4069, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 4094, 5, 0, 0, 0, 0, 0, 0, 4093, 1, 21, 2, 0, 2, 4096, 4091, 0, 4095, 1, 0, 0, 4096, 3, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1703, 8, 4096, 0, 0, 0, 0, 0, 4096, 4096, 742, 1, 0, 2302, 0, 3469, 4096, 0, 4074, 0, 3, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 3, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1916, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 1884, 0, 0, 1910, 0, 2519, 4096, 0, 4059, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3609, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 692, 0, 0, 3541, 0, 350, 4096, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[9, 9, 6, 0, 9, 0, 4058, 4, 4083, 13, 2, 18, 0, 5, 4086, 4081, 3738, 0, 12, 282, 0, 70, 4084, 0, 4072, 9, 0, 1, 0, 11, 4074, 8, 0, 5, 0, 7, 11, 0, 0, 4078, 4085, 0, 7, 0, 10, 0, 0, 50, 4085, 0, 12, 0, 8, 10, 4087, 4089, 1, 4088, 0, 4, 13, 4083, 0, 4059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3532, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2008, 0, 0, 986, 0, 1836, 4096, 0, 3926, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[46, 43, 30, 0, 45, 0, 147, 0, 4030, 54, 38, 100, 0, 18, 4048, 4001, 3997, 0, 52, 3438, 23, 522, 4051, 0, 4048, 0, 0, 0, 0, 32, 4042, 42, 0, 32, 0, 23, 42, 0, 1, 4034, 4038, 0, 36, 0, 64, 0, 0, 96, 4052, 0, 0, 0, 27, 0, 4053, 4066, 7, 4057, 0, 12, 58, 4030, 0, 3961] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1156, 0, 4096, 1612, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 0, 0, 1942, 0, 3425, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 0, 0, 0, 0, 0, 4096, 0, 103, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2481, 1614, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2462, 1615, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6864, 8186, 9970, 7737, 8191, 8173, 8203, 8212] avg=8192.0000 max=9970 max_rank=2 min=6864 min_rank=0 imbalance_ratio=1.217041 max_load_reduction=55566 max_load_reduction_pct=0.847870 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3604, 11, 4096, 1610, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2546, 0, 0, 369, 0, 1670, 2377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 1, 0, 0, 0, 4094, 0, 103, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 2479, 1614, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 0, 0, 0, 0, 2462, 1615, 0, 0, 0, 0, 0, 0, 4093, 1, 21, 2, 0, 2, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9321, 8182, 6962, 8273, 8190, 8176, 8217, 8215] avg=8192.0000 max=9321 max_rank=0 min=6962 min_rank=2 imbalance_ratio=1.137817 max_load_reduction=56215 max_load_reduction_pct=0.857773 moved_to_echo=32798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1703, 8, 4096, 1611, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 742, 1, 0, 2302, 0, 3469, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 3, 0, 0, 0, 4083, 0, 103, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2484, 1614, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2462, 1615, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7418, 8186, 8892, 8269, 8194, 8170, 8192, 8215] avg=8192.0000 max=8892 max_rank=2 min=7418 min_rank=0 imbalance_ratio=1.085449 max_load_reduction=56644 max_load_reduction_pct=0.864319 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1916, 0, 4096, 1613, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1884, 0, 0, 1910, 0, 2519, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 4096, 0, 103, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2483, 1614, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2462, 1615, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7625, 8186, 8691, 8264, 8193, 8173, 8192, 8212] avg=8192.0000 max=8691 max_rank=2 min=7625 min_rank=0 imbalance_ratio=1.060913 max_load_reduction=56845 max_load_reduction_pct=0.867386 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[9, 9, 6, 0, 9, 0, 4058, 4, 4086, 1612, 0, 0, 0, 0, 0, 0, 4070, 13, 2, 18, 0, 5, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 3738, 0, 12, 282, 0, 70, 2366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 9, 0, 1, 0, 11, 4074, 8, 104, 13, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7, 11, 0, 0, 4078, 2477, 1615, 0, 0, 0, 0, 0, 0, 4085, 0, 7, 0, 10, 0, 0, 50, 2462, 1614, 0, 0, 0, 0, 0, 0, 4085, 0, 12, 0, 8, 10, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 4, 13, 4083, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9793, 8189, 6468, 8292, 8193, 8228, 8202, 8171] avg=8192.0000 max=9793 max_rank=0 min=6468 min_rank=2 imbalance_ratio=1.195435 max_load_reduction=55743 max_load_reduction_pct=0.850571 moved_to_echo=32794 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[46, 43, 30, 0, 45, 0, 147, 0, 4048, 1600, 0, 0, 0, 0, 0, 0, 4024, 54, 38, 100, 0, 18, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 52, 3438, 23, 522, 2352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 32, 4042, 42, 102, 6, 0, 0, 0, 0, 0, 0, 0, 32, 0, 23, 42, 0, 1, 4034, 2463, 1599, 0, 0, 0, 0, 0, 0, 4038, 0, 36, 0, 64, 0, 0, 96, 2439, 1597, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 27, 0, 4053, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7, 19, 0, 12, 58, 4030, 0, 3961, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5959, 8235, 10384, 8272, 8194, 8270, 8135, 8087] avg=8192.0000 max=10384 max_rank=2 min=5959 min_rank=0 imbalance_ratio=1.267578 max_load_reduction=55152 max_load_reduction_pct=0.841553 moved_to_echo=32686 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3609, 0, 4096, 1611, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 692, 0, 0, 3541, 0, 350, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 103, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2480, 1614, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2462, 1615, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9316, 8186, 6961, 8298, 8190, 8173, 8200, 8212] avg=8192.0000 max=9316 max_rank=0 min=6961 min_rank=2 imbalance_ratio=1.137207 max_load_reduction=56220 max_load_reduction_pct=0.857849 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3532, 0, 4096, 1612, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2008, 0, 0, 986, 0, 1836, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 0, 0, 0, 0, 4096, 0, 103, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2481, 1614, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2462, 1615, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9240, 8186, 7208, 8131, 8191, 8173, 8195, 8212] avg=8192.0000 max=9240 max_rank=0 min=7208 min_rank=2 imbalance_ratio=1.127930 max_load_reduction=56296 max_load_reduction_pct=0.859009 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 256, 128, 128, 128, 32768, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 0, 256, 19712, 12928, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32000, 128, 128, 128, 0, 128, 32768, 128, 896, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 32768, 19840, 12928, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32640, 128, 128, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[17920, 128, 128, 14848, 128, 13952, 19072, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 0, 19840, 128, 32768, 12928, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 765, 0, 0, 4096, 0, 0, 0, 0, 4096, 5, 4096, 3306, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 27, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 2816, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1396, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 3981, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3264, 2, 2, 0, 4096, 0, 4078, 4096, 0, 0, 0, 4, 4089, 0, 0, 4096, 1, 862, 0, 0, 4094, 0, 0, 0, 2, 4094, 2, 4096, 4070, 0, 4094, 2, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4016, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 463, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 3713, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[30, 19, 23, 1061, 0, 0, 0, 4022, 45, 4013, 3979, 41, 0, 2, 0, 4072, 0, 0, 4035, 0, 2913, 0, 1, 4045, 22, 0, 54, 0, 4060, 0, 4044, 4048, 1, 4052, 0, 59, 4038, 0, 45, 0, 27, 43, 54, 0, 4035, 73, 45, 4056, 40, 167, 0, 0, 0, 4051, 0, 53, 4049, 2, 32, 0, 39, 0, 10, 36] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1801, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2415, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 3978, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3870, 0, 0, 0, 4096, 0, 4096, 4092, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1810, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 4096, 2429, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 92, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[8, 5, 17, 3470, 0, 0, 0, 4080, 8, 4078, 4054, 9, 0, 1, 0, 4090, 0, 0, 4084, 0, 814, 0, 4, 4084, 2, 3, 12, 0, 4087, 0, 4087, 3866, 9, 4088, 0, 10, 4087, 0, 12, 0, 5, 9, 0, 0, 4085, 20, 2, 4088, 9, 46, 0, 0, 0, 4084, 0, 9, 4086, 1, 8, 0, 5, 0, 1, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 4096, 1024, 19, 3, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 765, 0, 0, 2681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 5, 4096, 3306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3678, 391, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 27, 0, 4069, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9236, 8212, 7542, 7825, 8189, 8175, 8165, 8192] avg=8192.0000 max=9236 max_rank=0 min=7542 min_rank=2 imbalance_ratio=1.127441 max_load_reduction=56300 max_load_reduction_pct=0.859070 moved_to_echo=32721 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4016, 0, 0, 0, 4096, 1024, 19, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 463, 0, 0, 2681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 4096, 3713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3678, 391, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9158, 8214, 7240, 8227, 8189, 8173, 8165, 8170] avg=8192.0000 max=9158 max_rank=0 min=7240 min_rank=2 imbalance_ratio=1.117920 max_load_reduction=56378 max_load_reduction_pct=0.860260 moved_to_echo=32697 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[30, 19, 23, 1061, 0, 0, 0, 4022, 1011, 19, 2, 0, 0, 0, 0, 0, 45, 4013, 3979, 41, 0, 2, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 2913, 0, 1, 2648, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 54, 0, 415, 0, 4044, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4050, 0, 59, 4038, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 43, 54, 0, 4035, 73, 45, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 40, 167, 0, 0, 0, 4051, 0, 53, 3645, 386, 0, 0, 0, 0, 0, 0, 4049, 2, 32, 0, 39, 0, 10, 36, 4050, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6187, 8102, 9597, 8583, 8193, 8314, 8342, 8218] avg=8192.0000 max=9597 max_rank=2 min=6187 min_rank=0 imbalance_ratio=1.171509 max_load_reduction=55939 max_load_reduction_pct=0.853561 moved_to_echo=33067 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3264, 2, 2, 0, 4096, 1023, 19, 3, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 4, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 862, 0, 0, 2680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 418, 2, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3676, 391, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 0, 4067, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8409, 8200, 7639, 8588, 8194, 8170, 8163, 8173] avg=8192.0000 max=8588 max_rank=3 min=7639 min_rank=2 imbalance_ratio=1.048340 max_load_reduction=56948 max_load_reduction_pct=0.868958 moved_to_echo=32700 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2816, 0, 0, 0, 4096, 1024, 19, 3, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1396, 0, 0, 2681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 4096, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3675, 391, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4074, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7958, 8213, 8173, 8495, 8191, 8173, 8162, 8171] avg=8192.0000 max=8495 max_rank=3 min=7958 min_rank=0 imbalance_ratio=1.036987 max_load_reduction=57041 max_load_reduction_pct=0.870377 moved_to_echo=32697 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[8, 5, 17, 3470, 0, 0, 0, 4080, 1025, 24, 4, 0, 0, 0, 0, 0, 8, 4078, 4054, 9, 0, 1, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 814, 0, 4, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 12, 0, 413, 0, 4087, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4084, 0, 10, 4087, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 9, 0, 0, 4085, 20, 2, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 9, 46, 0, 0, 0, 4084, 0, 9, 3674, 395, 0, 0, 0, 0, 0, 0, 4086, 1, 8, 0, 5, 0, 1, 9, 4069, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8633, 8171, 7566, 8383, 8202, 8185, 8217, 8179] avg=8192.0000 max=8633 max_rank=0 min=7566 min_rank=2 imbalance_ratio=1.053833 max_load_reduction=56903 max_load_reduction_pct=0.868271 moved_to_echo=32783 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3870, 0, 0, 0, 4096, 1024, 19, 3, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1810, 0, 0, 2681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 4096, 2429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3673, 391, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 92, 0, 4074, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9012, 8210, 8587, 6943, 8189, 8173, 8160, 8262] avg=8192.0000 max=9012 max_rank=0 min=6943 min_rank=3 imbalance_ratio=1.100098 max_load_reduction=56524 max_load_reduction_pct=0.862488 moved_to_echo=32784 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1801, 0, 0, 0, 4096, 1024, 19, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2415, 0, 0, 2681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 4096, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3675, 391, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4074, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6943, 8214, 9192, 8492, 8189, 8173, 8162, 8171] avg=8192.0000 max=9192 max_rank=2 min=6943 min_rank=0 imbalance_ratio=1.122070 max_load_reduction=56344 max_load_reduction_pct=0.859741 moved_to_echo=32695 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 256, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 3456, 128, 32768, 29440, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 32640, 128, 0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 11520, 0, 128, 21504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 24448, 128, 128, 0, 32768, 8192, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 256, 0, 0, 0, 32768, 0, 128, 29440, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 3957, 4096, 4096, 0, 0, 4094, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3569, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 932, 0, 0, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4088, 4096, 4096, 0, 12, 4095, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4061, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 68, 0, 0, 4090, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[43, 58, 13, 35, 32, 305, 0, 0, 67, 0, 3, 3, 0, 13, 0, 4046, 0, 3702, 54, 0, 0, 15, 4033, 4051, 4060, 4052, 40, 108, 3852, 0, 3825, 0, 1, 0, 0, 87, 0, 545, 4005, 4048, 41, 0, 34, 0, 4010, 43, 4056, 27, 39, 2578, 3737, 45, 20, 50, 172, 3964, 15, 0, 1, 1515, 0, 0, 4053, 40] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 3989, 4096, 4096, 0, 112, 4096, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3982, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 243, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 4094, 4096, 4096, 0, 5, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4058, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 50, 0, 0, 4085, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 5, 4096, 0, 2, 0, 0, 4094, 4096, 4096, 4095, 0, 1, 4094, 3, 4027, 1, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 4052, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 110, 2, 0, 4092, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4, 14, 3, 13, 5, 70, 0, 0, 18, 0, 0, 1, 0, 3, 0, 4087, 0, 4079, 5, 0, 0, 0, 4081, 4087, 4088, 4086, 20, 50, 4002, 0, 4084, 0, 0, 0, 0, 13, 0, 8, 4075, 4086, 9, 4, 7, 0, 4080, 7, 4085, 10, 9, 4022, 3735, 8, 5, 11, 42, 4050, 0, 0, 1, 373, 0, 0, 4088, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4092, 4096, 4096, 0, 15, 4096, 0, 3806, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3769, 4074, 0, 0, 0, 0, 4096, 0, 0, 0, 629, 0, 0, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 4096, 4015, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3943, 153, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 81, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 4094, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 4096, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 0, 0, 4095, 0, 3562, 60, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8192, 8134, 8058, 8132, 8189, 8046, 8652] avg=8192.0000 max=8652 max_rank=7 min=8046 min_rank=6 imbalance_ratio=1.056152 max_load_reduction=56884 max_load_reduction_pct=0.867981 moved_to_echo=33019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4015, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3943, 153, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 81, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 12, 4095, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4096, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 4090, 0, 3562, 60, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8192, 8265, 8305, 8132, 8189, 8538, 7783] avg=8192.0000 max=8538 max_rank=6 min=7783 min_rank=7 imbalance_ratio=1.042236 max_load_reduction=56998 max_load_reduction_pct=0.869720 moved_to_echo=32642 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[43, 58, 13, 35, 32, 305, 0, 0, 4060, 3953, 21, 0, 0, 0, 0, 0, 67, 0, 3, 3, 0, 13, 0, 4046, 3901, 148, 0, 0, 0, 0, 0, 0, 0, 3702, 54, 0, 0, 15, 80, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 40, 108, 3852, 0, 3825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 87, 0, 545, 4005, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 34, 0, 4010, 43, 4053, 27, 0, 0, 0, 0, 0, 0, 0, 0, 39, 2578, 3737, 45, 20, 50, 172, 369, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 1515, 0, 0, 4053, 40, 3447, 60, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8520, 8181, 7902, 7955, 8626, 8208, 7010, 9134] avg=8192.0000 max=9134 max_rank=7 min=7010 min_rank=6 imbalance_ratio=1.114990 max_load_reduction=56402 max_load_reduction_pct=0.860626 moved_to_echo=32978 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4013, 21, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 3942, 153, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 2, 0, 0, 81, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 1, 4094, 3, 4027, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 4094, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 2, 0, 4092, 0, 3562, 60, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8193, 8280, 8258, 8134, 8185, 8527, 7829] avg=8192.0000 max=8527 max_rank=6 min=7829 min_rank=7 imbalance_ratio=1.040894 max_load_reduction=57009 max_load_reduction_pct=0.869888 moved_to_echo=32675 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4015, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3943, 153, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 81, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 5, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4094, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 4085, 0, 3562, 60, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8192, 8270, 8328, 8132, 8189, 8533, 7760] avg=8192.0000 max=8533 max_rank=6 min=7760 min_rank=7 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=32614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4015, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3943, 153, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 81, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 112, 4096, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 4093, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 4096, 0, 3562, 60, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8192, 8166, 8305, 8132, 8189, 8456, 7964] avg=8192.0000 max=8456 max_rank=6 min=7964 min_rank=7 imbalance_ratio=1.032227 max_load_reduction=57080 max_load_reduction_pct=0.870972 moved_to_echo=32741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4, 14, 3, 13, 5, 70, 0, 0, 4088, 4002, 26, 0, 0, 0, 0, 0, 18, 0, 0, 1, 0, 3, 0, 4087, 3938, 155, 0, 0, 0, 0, 0, 0, 0, 4079, 5, 0, 0, 0, 79, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 20, 50, 4002, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8, 4075, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 7, 0, 4080, 7, 4081, 10, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4022, 3735, 8, 5, 11, 42, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 373, 0, 0, 4088, 8, 3525, 67, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8202, 8250, 8278, 8115, 8198, 8202, 8066] avg=8192.0000 max=8278 max_rank=3 min=8066 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32581 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4015, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3943, 153, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 81, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 15, 4096, 0, 3806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3769, 4074, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 0, 4094, 0, 3562, 60, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8132, 8192, 8269, 8049, 8133, 8189, 8224, 8348] avg=8192.0000 max=8348 max_rank=7 min=8049 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=57188 max_load_reduction_pct=0.872620 moved_to_echo=32894 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32384, 128, 128, 0, 128, 768, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 128, 384, 0, 0, 32768, 32128, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 1152, 128, 384, 32512, 128, 31744, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 3968, 128, 0, 32768, 128, 28416, 512, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 0, 32768, 31616, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 30208, 32128, 128, 128, 128, 256, 3072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 0, 640, 32768, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3, 133, 2445, 13, 736, 1, 1074, 1, 2807, 48, 274, 832, 978, 142, 0, 1412, 297, 2377, 1911, 585, 3804, 734, 294, 1863, 0, 3575, 102, 1145, 240, 21, 132, 1292, 3846, 128, 20, 1072, 1118, 18, 612, 2452, 2812, 2368, 525, 259, 7, 2294, 1110, 1668, 53, 1565, 522, 3, 5, 89, 2630, 212, 0, 3376, 3751, 2099, 143, 1233, 261, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3, 79, 2807, 5, 438, 0, 1184, 1, 3000, 42, 437, 1413, 924, 243, 0, 825, 217, 2452, 1714, 916, 3927, 661, 312, 1767, 0, 3578, 11, 603, 318, 15, 100, 1390, 4051, 99, 33, 767, 1391, 19, 403, 2306, 2905, 2374, 269, 239, 11, 2580, 1087, 1975, 60, 1501, 425, 23, 31, 79, 2493, 236, 8, 3212, 3701, 1818, 266, 1479, 307, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[8, 265, 2550, 1, 400, 1, 661, 21, 2843, 119, 289, 846, 680, 125, 0, 859, 277, 1891, 2384, 700, 3689, 773, 306, 1605, 0, 3274, 143, 925, 401, 20, 270, 1548, 3976, 97, 101, 1002, 1615, 16, 767, 2307, 2761, 3014, 579, 255, 3, 2004, 1184, 2121, 187, 2007, 469, 29, 8, 35, 2476, 366, 1, 3033, 3554, 1693, 468, 915, 571, 48] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[11, 159, 2650, 14, 379, 0, 1110, 0, 2814, 154, 432, 1003, 1079, 284, 1, 960, 295, 2118, 2205, 751, 3883, 993, 224, 1363, 0, 3629, 70, 800, 289, 30, 184, 1445, 3925, 125, 42, 1223, 1063, 11, 600, 2388, 3190, 2209, 569, 425, 4, 2382, 1189, 2211, 57, 1479, 604, 12, 55, 58, 2740, 198, 6, 3044, 3903, 1410, 196, 620, 253, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2, 202, 1941, 9, 467, 0, 1135, 5, 3336, 43, 430, 1632, 1052, 332, 1, 1040, 238, 2188, 1823, 326, 3916, 1047, 184, 1601, 1, 3812, 4, 560, 218, 8, 157, 1430, 3952, 53, 32, 1570, 1105, 10, 412, 2020, 3394, 2248, 303, 124, 5, 2392, 1194, 1997, 45, 1972, 566, 4, 22, 27, 2480, 184, 2, 3597, 3912, 1470, 309, 676, 313, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[4, 76, 2394, 3, 438, 0, 932, 0, 2788, 77, 208, 818, 840, 192, 0, 1236, 267, 2249, 2200, 1000, 3673, 798, 351, 1535, 0, 3582, 144, 757, 312, 33, 103, 1411, 4073, 82, 70, 1123, 1683, 23, 592, 1664, 3377, 2333, 259, 163, 2, 2202, 1725, 1867, 73, 1182, 727, 17, 15, 22, 3460, 402, 1, 3181, 3790, 1289, 217, 1098, 382, 21] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[22, 63, 1701, 102, 631, 0, 821, 24, 2703, 346, 414, 411, 606, 302, 13, 1165, 328, 1796, 1622, 724, 3823, 1530, 362, 1797, 14, 3250, 117, 1198, 312, 20, 215, 1464, 3702, 135, 140, 1149, 1233, 22, 722, 1847, 2715, 2321, 564, 189, 63, 2329, 1463, 2571, 105, 1061, 901, 77, 58, 155, 2605, 261, 1, 3283, 3883, 2175, 323, 922, 558, 107] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3, 27, 2025, 0, 675, 1, 1146, 1, 2861, 71, 188, 1075, 412, 176, 1, 644, 185, 2648, 1828, 618, 3783, 1063, 178, 1877, 0, 3389, 39, 617, 392, 10, 224, 1780, 4029, 66, 23, 1022, 1369, 25, 669, 2829, 2996, 2701, 748, 203, 5, 2231, 1172, 2321, 61, 1715, 477, 9, 2, 17, 2339, 196, 5, 3405, 3723, 1336, 195, 1110, 594, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 79, 2807, 5, 438, 0, 1184, 1, 3774, 508, 0, 0, 0, 0, 0, 0, 3000, 42, 437, 1413, 924, 243, 0, 825, 1179, 615, 0, 0, 0, 0, 0, 0, 217, 2452, 1714, 916, 153, 661, 312, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 11, 603, 318, 15, 100, 1390, 1726, 40, 0, 0, 0, 0, 0, 0, 2832, 99, 33, 767, 1391, 19, 403, 2306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 269, 239, 11, 2580, 1087, 1975, 0, 0, 0, 0, 0, 0, 0, 0, 60, 1501, 425, 23, 31, 79, 2493, 236, 2397, 448, 0, 0, 0, 0, 0, 0, 8, 3212, 1527, 1818, 266, 1479, 307, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8799, 8678, 8192, 7781, 7850, 7920, 7693, 8623] avg=8192.0000 max=8799 max_rank=0 min=7693 min_rank=6 imbalance_ratio=1.074097 max_load_reduction=56737 max_load_reduction_pct=0.865738 moved_to_echo=32086 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 133, 2445, 13, 736, 1, 1074, 1, 3655, 492, 0, 0, 0, 0, 0, 0, 2807, 48, 274, 832, 978, 142, 0, 1412, 1119, 614, 0, 0, 0, 0, 0, 0, 297, 2377, 1911, 585, 149, 734, 294, 1863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3575, 102, 1145, 240, 21, 132, 1292, 1749, 38, 0, 0, 0, 0, 0, 0, 2689, 128, 20, 1072, 1118, 18, 612, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1754, 525, 259, 7, 2294, 1110, 1668, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1565, 522, 3, 5, 89, 2630, 212, 2320, 454, 0, 0, 0, 0, 0, 0, 0, 3376, 1548, 2099, 143, 1233, 261, 14, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8553, 8226, 8210, 8294, 8109, 7617, 7853, 8674] avg=8192.0000 max=8674 max_rank=7 min=7617 min_rank=5 imbalance_ratio=1.058838 max_load_reduction=56862 max_load_reduction_pct=0.867645 moved_to_echo=32253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[8, 265, 2550, 1, 400, 1, 661, 21, 3545, 482, 0, 0, 0, 0, 0, 0, 2843, 119, 289, 846, 680, 125, 0, 859, 1157, 781, 0, 0, 0, 0, 0, 0, 277, 1891, 2384, 700, 144, 773, 306, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3274, 143, 925, 401, 20, 270, 1548, 1657, 40, 0, 0, 0, 0, 0, 0, 2779, 97, 101, 1002, 1615, 16, 767, 2307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2233, 579, 255, 3, 2004, 1184, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 187, 2007, 469, 29, 8, 35, 2476, 366, 2279, 431, 0, 0, 0, 0, 0, 0, 1, 3033, 1466, 1693, 468, 915, 571, 48, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7934, 7699, 8080, 8278, 8684, 8379, 8287, 8195] avg=8192.0000 max=8684 max_rank=4 min=7699 min_rank=1 imbalance_ratio=1.060059 max_load_reduction=56852 max_load_reduction_pct=0.867493 moved_to_echo=33545 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 159, 2650, 14, 379, 0, 1110, 0, 3731, 558, 0, 0, 0, 0, 0, 0, 2814, 154, 432, 1003, 1079, 284, 1, 960, 1142, 572, 0, 0, 0, 0, 0, 0, 295, 2118, 2205, 751, 152, 993, 224, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 70, 800, 289, 30, 184, 1445, 1820, 39, 0, 0, 0, 0, 0, 0, 2744, 125, 42, 1223, 1063, 11, 600, 2388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1637, 569, 425, 4, 2382, 1189, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 57, 1479, 604, 12, 55, 58, 2740, 198, 2632, 473, 0, 0, 0, 0, 0, 0, 6, 3044, 1610, 1410, 196, 620, 253, 16, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8612, 8441, 8101, 8306, 8196, 8417, 8308, 7155] avg=8192.0000 max=8612 max_rank=0 min=7155 min_rank=7 imbalance_ratio=1.051270 max_load_reduction=56924 max_load_reduction_pct=0.868591 moved_to_echo=32076 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[22, 63, 1701, 102, 631, 0, 821, 24, 3674, 475, 0, 0, 0, 0, 0, 0, 2703, 346, 414, 411, 606, 302, 13, 1165, 1077, 601, 0, 0, 0, 0, 0, 0, 328, 1796, 1622, 724, 149, 1530, 362, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3250, 117, 1198, 312, 20, 215, 1464, 1811, 37, 0, 0, 0, 0, 0, 0, 2588, 135, 140, 1149, 1233, 22, 722, 1847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 564, 189, 63, 2329, 1463, 2571, 0, 0, 0, 0, 0, 0, 0, 0, 105, 1061, 901, 77, 58, 155, 2605, 261, 2240, 470, 0, 0, 0, 0, 0, 0, 1, 3283, 1602, 2175, 323, 922, 558, 107, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7513, 7638, 8308, 8438, 7836, 8899, 7933, 8971] avg=8192.0000 max=8971 max_rank=7 min=7513 min_rank=0 imbalance_ratio=1.095093 max_load_reduction=56565 max_load_reduction_pct=0.863113 moved_to_echo=33639 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 76, 2394, 3, 438, 0, 932, 0, 3533, 591, 0, 0, 0, 0, 0, 0, 2788, 77, 208, 818, 840, 192, 0, 1236, 1188, 609, 0, 0, 0, 0, 0, 0, 267, 2249, 2200, 1000, 140, 798, 351, 1535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3582, 144, 757, 312, 33, 103, 1411, 1771, 46, 0, 0, 0, 0, 0, 0, 2839, 82, 70, 1123, 1683, 23, 592, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1724, 259, 163, 2, 2202, 1725, 1867, 0, 0, 0, 0, 0, 0, 0, 0, 73, 1182, 727, 17, 15, 22, 3460, 402, 2786, 464, 0, 0, 0, 0, 0, 0, 1, 3181, 1555, 1289, 217, 1098, 382, 21, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7971, 7956, 8540, 8159, 8076, 7942, 9148, 7744] avg=8192.0000 max=9148 max_rank=6 min=7744 min_rank=7 imbalance_ratio=1.116699 max_load_reduction=56388 max_load_reduction_pct=0.860413 moved_to_echo=32910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 202, 1941, 9, 467, 0, 1135, 5, 3763, 593, 0, 0, 0, 0, 0, 0, 3336, 43, 430, 1632, 1052, 332, 1, 1040, 1150, 582, 0, 0, 0, 0, 0, 0, 238, 2188, 1823, 326, 153, 1047, 184, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3812, 4, 560, 218, 8, 157, 1430, 1824, 39, 0, 0, 0, 0, 0, 0, 2763, 53, 32, 1570, 1105, 10, 412, 2020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1666, 303, 124, 5, 2392, 1194, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 45, 1972, 566, 4, 22, 27, 2480, 184, 2801, 474, 0, 0, 0, 0, 0, 0, 2, 3597, 1614, 1470, 309, 676, 313, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 9598, 7560, 8053, 7965, 7681, 8575, 7987] avg=8192.0000 max=9598 max_rank=1 min=7560 min_rank=2 imbalance_ratio=1.171631 max_load_reduction=55938 max_load_reduction_pct=0.853546 moved_to_echo=32208 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 27, 2025, 0, 675, 1, 1146, 1, 3635, 524, 0, 0, 0, 0, 0, 0, 2861, 71, 188, 1075, 412, 176, 1, 644, 1172, 700, 0, 0, 0, 0, 0, 0, 185, 2648, 1828, 618, 148, 1063, 178, 1877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3389, 39, 617, 392, 10, 224, 1780, 1736, 40, 0, 0, 0, 0, 0, 0, 2817, 66, 23, 1022, 1369, 25, 669, 2829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2001, 748, 203, 5, 2231, 1172, 2321, 0, 0, 0, 0, 0, 0, 0, 0, 61, 1715, 477, 9, 2, 17, 2339, 196, 2472, 451, 0, 0, 0, 0, 0, 0, 5, 3405, 1536, 1336, 195, 1110, 594, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8037, 7300, 8545, 8227, 8820, 8681, 7739, 8187] avg=8192.0000 max=8820 max_rank=4 min=7300 min_rank=1 imbalance_ratio=1.076660 max_load_reduction=56716 max_load_reduction_pct=0.865417 moved_to_echo=33427 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 14592, 3840, 1920, 128, 18432, 10240, 16768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 12544, 4736, 256, 256, 512, 21248, 2176, 19968, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2176, 17792, 15744, 5632, 1280, 7680, 2304, 13440, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[23168, 1024, 2688, 8064, 6656, 1920, 128, 8192, 9216, 5120, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[22144, 896, 512, 8960, 10624, 256, 4864, 17920, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 26240, 12544, 13312, 2176, 8064, 3328, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 1024, 18560, 256, 4224, 128, 8064, 128, 29312, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 28160, 640, 6656, 2560, 256, 1408, 11776, 14208, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 3, 0, 8, 0, 1, 24, 4091, 9, 0, 3566, 28, 4062, 4070, 4095, 0, 0, 2, 0, 4084, 4091, 4089, 0, 0, 4053, 1, 12, 3, 124, 0, 25, 6, 4033, 4039, 0, 36, 0, 0, 0, 4079, 0, 0, 1, 4091, 4072, 0, 2, 12, 8, 4026, 0, 2, 2, 2, 1, 1, 23, 0, 0, 576, 0, 4078, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 6, 1, 2, 1, 5, 0, 4086, 12, 13, 3864, 6, 4055, 4096, 4094, 9, 0, 0, 0, 4090, 4086, 4096, 0, 1, 4059, 28, 13, 1, 4, 3, 0, 0, 4076, 4092, 0, 1, 0, 0, 2, 4089, 0, 0, 1, 4096, 4096, 1, 7, 0, 0, 4082, 23, 9, 0, 3, 0, 0, 0, 0, 4, 227, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 4096, 65, 0, 4049, 0, 4026, 4095, 4093, 0, 0, 0, 1, 4095, 4084, 4094, 0, 3, 3696, 1, 0, 0, 23, 6, 0, 1, 4089, 4096, 0, 0, 0, 1, 0, 3919, 4, 4, 0, 4091, 4093, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 3, 0, 618, 0, 4092, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4094, 19, 9, 4002, 3, 4056, 4093, 4096, 0, 1, 0, 1, 4094, 4069, 4083, 0, 1, 4064, 2, 3, 0, 6, 3, 0, 0, 4094, 4090, 5, 0, 1, 1, 4, 4090, 0, 0, 0, 4091, 4094, 1, 1, 0, 0, 4092, 0, 2, 0, 1, 4, 0, 0, 0, 0, 185, 2, 4078, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 6, 0, 0, 4096, 7, 1, 3986, 0, 3874, 4091, 4093, 3, 0, 0, 0, 4096, 4089, 4093, 0, 3, 3588, 0, 0, 0, 17, 0, 7, 0, 4096, 4090, 5, 2, 0, 0, 4, 2515, 0, 2, 0, 4096, 4096, 0, 2, 0, 0, 4092, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2381, 0, 4094, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[119, 0, 1, 0, 53, 63, 27, 4058, 76, 30, 4085, 0, 4081, 4065, 4094, 3, 34, 0, 20, 4093, 4082, 4066, 2, 0, 2686, 3, 0, 1, 270, 0, 23, 4, 4078, 3263, 7, 1, 2, 0, 67, 3992, 0, 0, 0, 4086, 4020, 1, 25, 0, 0, 4032, 1, 0, 1, 9, 0, 0, 3, 0, 0, 1820, 1, 4088, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2410, 0, 4063, 4094, 4096, 0, 0, 0, 0, 4096, 4096, 4095, 0, 4, 4005, 3, 0, 0, 8, 0, 0, 0, 4096, 4062, 1, 0, 0, 0, 0, 4094, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1832, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 3, 17, 3638, 1, 4045, 4093, 4081, 0, 4, 0, 0, 4080, 4067, 4094, 0, 19, 2841, 0, 0, 0, 4, 5, 0, 0, 4094, 4068, 19, 0, 0, 0, 0, 3779, 0, 3, 0, 4081, 4095, 0, 1, 9, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 2093, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 3, 0, 8, 0, 1, 24, 4091, 3243, 775, 0, 0, 0, 0, 0, 0, 9, 0, 3566, 28, 4062, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 9, 4091, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 1, 12, 3, 124, 0, 25, 6, 4095, 398, 0, 0, 0, 0, 0, 0, 374, 4039, 0, 36, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 4072, 0, 2, 12, 8, 0, 0, 0, 0, 0, 0, 0, 8, 4026, 0, 2, 2, 2, 1, 1, 3677, 416, 0, 0, 0, 0, 0, 0, 23, 0, 0, 576, 0, 4078, 4, 0, 2867, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8085, 8191, 8717, 8528, 8186, 8135, 7548] avg=8192.0000 max=8717 max_rank=3 min=7548 min_rank=7 imbalance_ratio=1.064087 max_load_reduction=56819 max_load_reduction_pct=0.866989 moved_to_echo=32397 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4094, 3292, 780, 0, 0, 0, 0, 0, 0, 19, 9, 4002, 3, 4056, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 9, 4069, 4083, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 2, 3, 0, 6, 3, 0, 0, 4096, 399, 0, 0, 0, 0, 0, 0, 380, 4090, 5, 0, 1, 1, 4, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4094, 1, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 1, 4, 0, 3686, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 2, 4078, 0, 0, 2883, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8511, 8164, 8573, 8571, 8195, 8207, 7148] avg=8192.0000 max=8573 max_rank=3 min=7148 min_rank=7 imbalance_ratio=1.046509 max_load_reduction=56963 max_load_reduction_pct=0.869186 moved_to_echo=32121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 1, 2, 1, 5, 0, 4086, 3278, 780, 0, 0, 0, 0, 0, 0, 12, 13, 3864, 6, 4055, 423, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4086, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 28, 13, 1, 4, 3, 0, 0, 4094, 398, 0, 0, 0, 0, 0, 0, 378, 4092, 0, 1, 0, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 1, 7, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 23, 9, 0, 3, 0, 0, 3682, 420, 0, 0, 0, 0, 0, 0, 0, 0, 4, 227, 0, 4096, 0, 0, 2885, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8382, 8193, 8600, 8562, 8209, 8219, 7212] avg=8192.0000 max=8600 max_rank=3 min=7212 min_rank=7 imbalance_ratio=1.049805 max_load_reduction=56936 max_load_reduction_pct=0.868774 moved_to_echo=32202 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 6, 0, 0, 4096, 3294, 779, 0, 0, 0, 0, 0, 0, 7, 1, 3986, 0, 3874, 422, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4089, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3588, 0, 0, 0, 17, 0, 7, 0, 4093, 399, 0, 0, 0, 0, 0, 0, 380, 4090, 5, 2, 0, 0, 4, 2515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 2, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 2, 3688, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2381, 0, 4094, 0, 8, 2882, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8293, 8194, 8104, 6996, 8204, 8204, 9365] avg=8192.0000 max=9365 max_rank=7 min=6996 min_rank=4 imbalance_ratio=1.143188 max_load_reduction=56171 max_load_reduction_pct=0.857101 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 4096, 3288, 780, 0, 0, 0, 0, 0, 0, 65, 0, 4049, 0, 4026, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 4084, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3696, 1, 0, 0, 23, 6, 0, 1, 4093, 399, 0, 0, 0, 0, 0, 0, 380, 4096, 0, 0, 0, 1, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 4091, 4093, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3687, 421, 0, 0, 0, 0, 0, 0, 1, 3, 0, 618, 0, 4092, 0, 1, 2885, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8562, 8191, 8219, 8396, 8200, 8203, 7600] avg=8192.0000 max=8562 max_rank=1 min=7600 min_rank=7 imbalance_ratio=1.045166 max_load_reduction=56974 max_load_reduction_pct=0.869354 moved_to_echo=32399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 3292, 780, 0, 0, 0, 0, 0, 0, 3, 17, 3638, 1, 4045, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 10, 4067, 4094, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 2841, 0, 0, 0, 4, 5, 0, 0, 4081, 397, 0, 0, 0, 0, 0, 0, 380, 4068, 19, 0, 0, 0, 0, 3779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4081, 4095, 0, 1, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 14, 0, 3673, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2093, 0, 4096, 0, 0, 2883, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8126, 8194, 7328, 8246, 8197, 8205, 9072] avg=8192.0000 max=9072 max_rank=7 min=7328 min_rank=3 imbalance_ratio=1.107422 max_load_reduction=56464 max_load_reduction_pct=0.861572 moved_to_echo=33720 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 3299, 785, 0, 0, 0, 0, 0, 0, 0, 0, 2410, 0, 4063, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4095, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 3, 0, 0, 8, 0, 0, 0, 4096, 403, 0, 0, 0, 0, 0, 0, 370, 4062, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3692, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1832, 0, 4096, 0, 0, 2888, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 6882, 8196, 8515, 8527, 8205, 8215, 8816] avg=8192.0000 max=8816 max_rank=7 min=6882 min_rank=1 imbalance_ratio=1.076172 max_load_reduction=56720 max_load_reduction_pct=0.865479 moved_to_echo=33763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[119, 0, 1, 0, 53, 63, 27, 4058, 3280, 774, 0, 0, 0, 0, 0, 0, 76, 30, 4085, 0, 4081, 420, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 20, 9, 4082, 4066, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2686, 3, 0, 1, 270, 0, 23, 4, 4094, 399, 0, 0, 0, 0, 0, 0, 378, 3263, 7, 1, 2, 0, 67, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4020, 1, 25, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 1, 0, 1, 9, 0, 0, 3685, 420, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1820, 1, 4088, 0, 0, 2863, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8375, 8695, 8213, 7480, 7710, 8140, 8148, 8775] avg=8192.0000 max=8775 max_rank=7 min=7480 min_rank=3 imbalance_ratio=1.071167 max_load_reduction=56761 max_load_reduction_pct=0.866104 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3072, 31872, 128, 128, 128, 128, 128, 30592, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 128, 29696, 128, 32384, 3456, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 32768, 26368, 6272, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 9856, 128, 32768, 128, 128, 23040, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[29056, 128, 128, 128, 512, 128, 128, 128, 32768, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 32640, 128, 128, 128, 128, 128, 128, 29568, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 4079, 0, 4094, 0, 0, 3971, 4091, 4093, 0, 0, 0, 0, 4094, 0, 0, 3549, 0, 0, 4089, 3339, 9, 0, 98, 1, 2, 1, 0, 4094, 1, 0, 4095, 0, 0, 4095, 0, 333, 0, 0, 2, 4, 0, 990, 0, 6, 0, 16, 4092, 0, 4095, 0, 0, 7, 1, 0, 4096, 0, 1, 2, 4095, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 6, 4024, 25, 4061, 9, 0, 3405, 4096, 4031, 0, 0, 0, 13, 4092, 3, 0, 4090, 2, 1, 4080, 4011, 215, 0, 118, 1, 0, 23, 0, 4083, 0, 0, 4088, 11, 0, 4049, 66, 153, 0, 0, 16, 0, 1, 256, 0, 70, 0, 49, 4093, 0, 4079, 0, 4, 4, 0, 9, 4095, 0, 0, 0, 4096, 0, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 15, 4078, 0, 4058, 1, 0, 4059, 4096, 4096, 0, 0, 11, 0, 4081, 0, 0, 4067, 0, 0, 4072, 3860, 35, 0, 6, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 2, 4095, 14, 30, 0, 0, 1, 0, 0, 110, 0, 12, 0, 20, 4085, 0, 4096, 0, 0, 151, 0, 3, 4096, 0, 0, 0, 4092, 0, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 4, 4094, 0, 4096, 5, 3, 4034, 4096, 4093, 0, 0, 1, 1, 4084, 0, 0, 2646, 0, 0, 4096, 4081, 10, 0, 121, 1, 0, 1, 0, 4095, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 0, 2, 0, 780, 0, 10, 0, 62, 4088, 0, 4096, 0, 0, 536, 0, 5, 4096, 0, 0, 1, 4096, 0, 9, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3993, 0, 4079, 28, 136, 3371, 4096, 4096, 0, 0, 0, 1, 4096, 0, 1, 3122, 2, 0, 4061, 710, 419, 4, 34, 0, 0, 0, 0, 3757, 0, 0, 4078, 13, 1, 4096, 0, 547, 1, 0, 0, 0, 0, 2571, 3, 158, 0, 1585, 3895, 0, 4095, 0, 0, 246, 0, 0, 4088, 1, 4, 0, 4094, 0, 51, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4052, 0, 4096, 0, 0, 4077, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 3465, 0, 0, 4096, 3061, 232, 0, 538, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 160, 0, 0, 0, 0, 0, 441, 0, 54, 0, 310, 4079, 0, 4096, 0, 0, 3, 0, 0, 4096, 0, 3, 0, 4096, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4058, 0, 4093, 3, 0, 4077, 4094, 4093, 0, 0, 0, 0, 4096, 0, 0, 3536, 0, 0, 4086, 2610, 0, 0, 592, 1, 1, 0, 0, 4087, 0, 0, 4095, 0, 1, 4082, 0, 14, 0, 0, 1, 0, 1, 1303, 1, 0, 0, 177, 4087, 2, 4095, 0, 1, 50, 0, 0, 4096, 0, 0, 0, 4095, 1, 7, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4095, 0, 0, 2961, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4083, 0, 0, 4096, 3704, 1, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 4096, 0, 6, 0, 0, 0, 0, 0, 1397, 0, 0, 0, 90, 4096, 0, 4096, 0, 0, 42, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4079, 0, 361, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4093, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3549, 0, 0, 1336, 3339, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 1, 2, 1, 0, 4094, 1, 0, 2753, 933, 157, 98, 10, 0, 0, 0, 3938, 0, 0, 4095, 0, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 990, 0, 6, 0, 16, 4081, 2800, 0, 0, 0, 0, 0, 0, 4092, 0, 3997, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 2, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8412, 8197, 8233, 8148, 8366, 7899, 8097, 8184] avg=8192.0000 max=8412 max_rank=0 min=7899 min_rank=5 imbalance_ratio=1.026855 max_load_reduction=57124 max_load_reduction_pct=0.871643 moved_to_echo=32546 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4024, 25, 358, 9, 0, 3405, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4031, 0, 0, 0, 13, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 1, 1333, 4011, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 1, 0, 23, 0, 4083, 0, 0, 2747, 925, 157, 98, 10, 0, 0, 0, 3931, 11, 0, 4049, 66, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 1, 256, 0, 70, 0, 49, 4086, 2778, 0, 0, 0, 0, 0, 0, 4093, 0, 3981, 0, 4, 4, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4086, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7827, 8149, 9652, 8162, 8210, 7256, 8091, 8189] avg=8192.0000 max=9652 max_rank=2 min=7256 min_rank=5 imbalance_ratio=1.178223 max_load_reduction=55884 max_load_reduction_pct=0.852722 moved_to_echo=31746 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 4078, 0, 357, 1, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 11, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 1330, 3860, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4091, 0, 0, 2742, 925, 157, 98, 10, 0, 0, 0, 3939, 0, 2, 4095, 14, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 110, 0, 12, 0, 20, 4086, 2776, 0, 0, 0, 0, 0, 0, 4085, 0, 3998, 0, 0, 151, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4082, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8510, 8198, 9292, 8029, 8080, 7005, 8237, 8185] avg=8192.0000 max=9292 max_rank=2 min=7005 min_rank=5 imbalance_ratio=1.134277 max_load_reduction=56244 max_load_reduction_pct=0.858215 moved_to_echo=31507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4094, 0, 361, 5, 3, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4093, 0, 0, 1, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2646, 0, 0, 1338, 4081, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 1, 0, 1, 0, 4095, 0, 0, 2758, 933, 157, 98, 10, 0, 0, 0, 3939, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 780, 0, 10, 0, 62, 4086, 2802, 0, 0, 0, 0, 0, 0, 4088, 0, 3998, 0, 0, 536, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4086, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8501, 8189, 8075, 8174, 8035, 7742, 8627, 8193] avg=8192.0000 max=8627 max_rank=6 min=7742 min_rank=5 imbalance_ratio=1.053101 max_load_reduction=56909 max_load_reduction_pct=0.868362 moved_to_echo=32597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4052, 0, 361, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3465, 0, 0, 1338, 3061, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 0, 0, 0, 0, 4096, 0, 0, 2758, 933, 157, 98, 10, 0, 0, 0, 3939, 0, 0, 4096, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 441, 0, 54, 0, 310, 4086, 2802, 0, 0, 0, 0, 0, 0, 4079, 0, 3998, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4086, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8490, 8202, 8096, 8590, 8195, 7693, 8080, 8190] avg=8192.0000 max=8590 max_rank=3 min=7693 min_rank=5 imbalance_ratio=1.048584 max_load_reduction=56946 max_load_reduction_pct=0.868927 moved_to_echo=32158 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3993, 0, 360, 28, 136, 3371, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3122, 2, 0, 1327, 710, 419, 4, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 3757, 0, 0, 2734, 929, 157, 98, 10, 0, 0, 0, 3921, 13, 1, 4096, 0, 547, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2571, 3, 158, 0, 1585, 4086, 2790, 0, 0, 0, 0, 0, 0, 3895, 0, 3997, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 4, 0, 4084, 0, 51, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7889, 8203, 5585, 7719, 8579, 11193, 8138, 8230] avg=8192.0000 max=11193 max_rank=5 min=5585 min_rank=2 imbalance_ratio=1.366333 max_load_reduction=54343 max_load_reduction_pct=0.829208 moved_to_echo=36140 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 352, 0, 0, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 1334, 3704, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 2762, 938, 163, 105, 17, 0, 0, 0, 3933, 0, 1, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 0, 0, 0, 90, 4089, 2805, 0, 0, 0, 0, 0, 0, 4096, 0, 3991, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7409, 8199, 9122, 8084, 8036, 8381, 8129, 8176] avg=8192.0000 max=9122 max_rank=2 min=7409 min_rank=0 imbalance_ratio=1.113525 max_load_reduction=56414 max_load_reduction_pct=0.860809 moved_to_echo=32722 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4058, 0, 360, 3, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3536, 0, 0, 1335, 2610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 1, 1, 0, 0, 4087, 0, 0, 2751, 933, 157, 98, 10, 0, 0, 0, 3938, 0, 1, 4082, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1303, 1, 0, 0, 177, 4084, 2800, 0, 0, 0, 0, 0, 0, 4087, 2, 3997, 0, 1, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8498, 8199, 7481, 8630, 8035, 8367, 8137, 8189] avg=8192.0000 max=8630 max_rank=3 min=7481 min_rank=2 imbalance_ratio=1.053467 max_load_reduction=56906 max_load_reduction_pct=0.868317 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 7936, 128, 384, 0, 2432, 32768, 22400, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32512, 128, 2944, 128, 256, 30080, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 28672, 128, 128, 10752, 25472, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31488, 128, 128, 32768, 128, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[1536, 128, 128, 128, 0, 32512, 128, 0, 22016, 7552, 1280, 896, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 128, 32000, 0, 128, 1152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4079, 0, 4, 0, 4092, 8, 4095, 2, 4030, 0, 0, 60, 0, 2, 4091, 0, 16, 0, 4088, 1, 0, 4094, 0, 0, 6, 0, 0, 4093, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 2, 0, 3, 0, 0, 3, 4095, 4096, 2, 0, 0, 10, 0, 0, 4093, 1, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4053, 63, 108, 0, 4096, 0, 4087, 0, 4021, 0, 0, 0, 0, 0, 4046, 0, 60, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3827, 4, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4085, 0, 2, 48, 1, 51, 0, 0, 3, 4096, 4096, 0, 0, 0, 111, 0, 1, 4095, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4086, 0, 0, 3962, 2, 1, 0, 4079, 0, 4094, 0, 3509, 0, 7, 180, 0, 2, 4096, 0, 22, 0, 4048, 0, 38, 4096, 0, 0, 0, 0, 0, 4063, 0, 3935, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 11, 18, 4029, 10, 3, 58, 25, 682, 0, 10, 0, 4089, 4096, 0, 59, 0, 0, 6, 0, 4093, 0, 4031] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4048, 4, 59, 0, 4082, 0, 4090, 0, 4082, 0, 0, 0, 0, 0, 4088, 13, 18, 0, 4085, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 2, 0, 0, 6, 1, 0, 0, 8, 0, 4094, 0, 0, 9, 0, 0, 0, 0, 3, 4092, 4096, 0, 3, 0, 0, 0, 0, 4085, 0, 4089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4040, 2, 0, 0, 4096, 0, 4096, 0, 4067, 0, 1, 1, 0, 0, 4076, 0, 46, 0, 4087, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 4095, 3910, 0, 0, 2, 0, 0, 0, 0, 0, 0, 45, 4096, 0, 0, 1, 10, 185, 0, 0, 1, 4096, 4096, 0, 5, 0, 19, 0, 0, 4090, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4090, 0, 0, 0, 4096, 0, 4096, 0, 4037, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 67, 0, 0, 0, 4096, 4096, 0, 0, 0, 7, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4084, 0, 0, 4086, 3, 0, 1, 4095, 12, 4095, 0, 4075, 0, 0, 27, 1, 2, 4095, 0, 0, 2, 4094, 0, 0, 4094, 0, 1, 0, 0, 0, 4096, 0, 4096, 4091, 1, 1, 0, 0, 1, 0, 1, 2, 0, 0, 4095, 0, 0, 2, 3, 2, 1, 1, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 4093, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 0, 4042, 0, 0, 0, 4087, 275, 4095, 0, 3753, 0, 0, 3, 0, 0, 4096, 0, 28, 0, 4081, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4084, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 13, 0, 138, 0, 0, 4, 4096, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 4079, 0, 4, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 2, 4030, 0, 0, 60, 0, 60, 14, 4, 0, 0, 0, 0, 0, 2, 4082, 0, 16, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 6, 0, 0, 4093, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 2, 4075, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 4095, 3951, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4079, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8273, 8189, 8198, 8266, 8174, 8054, 8186] avg=8192.0000 max=8273 max_rank=1 min=8054 min_rank=6 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32680 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 3962, 2, 1, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3509, 0, 7, 180, 0, 60, 14, 4, 0, 0, 0, 0, 0, 2, 4087, 0, 22, 0, 4048, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4063, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3935, 4096, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 18, 4029, 10, 3, 58, 4065, 3, 0, 0, 0, 0, 0, 0, 25, 682, 0, 10, 0, 4089, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 6, 0, 4079, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8065, 7868, 8197, 8164, 8113, 8197, 8757, 8175] avg=8192.0000 max=8757 max_rank=6 min=7868 min_rank=1 imbalance_ratio=1.068970 max_load_reduction=56779 max_load_reduction_pct=0.866379 moved_to_echo=33242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 4048, 4, 59, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4082, 0, 0, 0, 0, 60, 14, 4, 0, 0, 0, 0, 0, 0, 4079, 13, 18, 0, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 2, 0, 0, 6, 1, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4094, 0, 0, 9, 4075, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4071, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8250, 8196, 8197, 8281, 8189, 8046, 8163] avg=8192.0000 max=8281 max_rank=4 min=8046 min_rank=6 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32679 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 4040, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4067, 0, 1, 1, 0, 60, 14, 3, 0, 0, 0, 0, 0, 0, 4068, 0, 46, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4095, 3910, 0, 0, 2, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4096, 0, 0, 1, 4075, 3, 0, 0, 0, 0, 0, 0, 10, 185, 0, 0, 1, 4096, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 19, 0, 0, 4076, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8159, 8242, 8201, 8189, 8089, 8220, 8243, 8193] avg=8192.0000 max=8243 max_rank=6 min=8089 min_rank=4 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 4053, 63, 108, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4021, 0, 0, 0, 0, 60, 14, 3, 0, 0, 0, 0, 0, 0, 4038, 0, 60, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 3827, 4, 0, 0, 0, 7, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 2, 48, 4075, 3, 0, 0, 0, 0, 0, 0, 1, 51, 0, 0, 3, 4096, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 1, 4081, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8341, 8185, 8194, 8197, 8016, 8213, 8102, 8288] avg=8192.0000 max=8341 max_rank=0 min=8016 min_rank=4 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32619 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4037, 0, 0, 0, 0, 67, 20, 6, 0, 0, 0, 0, 0, 0, 4081, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4077, 9, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 4096, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4076, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8226, 8179, 8199, 8271, 8183, 8094, 8179] avg=8192.0000 max=8271 max_rank=4 min=8094 min_rank=6 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 4086, 3, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4095, 0, 4075, 0, 0, 27, 1, 60, 14, 4, 0, 0, 0, 0, 0, 2, 4086, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 1, 1, 0, 0, 1, 0, 82, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 4095, 0, 0, 2, 4063, 3, 0, 0, 0, 0, 0, 0, 3, 2, 1, 1, 0, 4094, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8288, 8184, 8196, 8272, 8166, 8051, 8173] avg=8192.0000 max=8288 max_rank=1 min=8051 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32662 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 4042, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 275, 4095, 0, 3753, 0, 0, 3, 0, 60, 14, 4, 0, 0, 0, 0, 0, 0, 4087, 0, 28, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4084, 4062, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 13, 4073, 3, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 4, 4096, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4082, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8204, 8196, 8196, 8228, 8194, 8189, 8179] avg=8192.0000 max=8228 max_rank=4 min=8150 min_rank=0 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32790 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 256, 128, 128, 32640, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32640, 128, 256, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 0, 0, 32512, 128, 256, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 256, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 1152, 128, 128, 128, 32768, 31616, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 32768, 128, 31616, 0, 128, 384, 128, 512, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32640, 32384, 128, 128, 128, 0, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 1, 4093, 4096, 0, 0, 4071, 0, 4090, 0, 0, 0, 4094, 0, 0, 0, 4059, 0, 0, 4096, 0, 0, 0, 39, 4096, 26, 12, 2, 0, 4081, 4096, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 4095, 0, 4095, 0, 0, 0, 4020, 0, 0, 0, 4072, 0, 0, 4096, 0, 0, 0, 26, 4096, 101, 0, 1, 0, 4031, 4096, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 15, 0, 4096, 0, 0, 0, 0, 87, 0, 0, 3960, 4069, 0, 0, 4056, 0, 4086, 0, 0, 0, 3894, 0, 0, 0, 4071, 0, 0, 4085, 0, 0, 1, 3, 4096, 66, 0, 13, 11, 4096, 4096, 47, 0, 0, 0, 4096, 0, 9, 0, 0, 4096, 0, 0, 228, 0, 0, 0, 4077, 0, 3, 0, 4094, 1, 4084, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 6, 2, 4096, 0, 0, 0, 1, 0, 1, 0, 4096, 4093, 0, 0, 4082, 0, 4081, 0, 0, 0, 4092, 0, 0, 0, 4095, 2, 0, 4094, 0, 0, 0, 14, 4096, 11, 3, 0, 0, 4093, 4096, 8, 1, 4, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 4095, 0, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4081, 4094, 0, 0, 4096, 1, 4074, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 4, 4096, 9, 0, 2, 0, 4096, 4096, 27, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 1, 0, 0, 0, 4090, 4092, 0, 0, 4076, 4, 4095, 0, 0, 14, 3990, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 2, 54, 4095, 22, 0, 3, 0, 4096, 4095, 33, 0, 0, 0, 4095, 0, 1, 0, 1, 4091, 0, 1, 8, 0, 0, 0, 4096, 1, 0, 1, 4096, 0, 4096, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 43, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 4045, 0, 4001, 0, 0, 0, 3803, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 383, 6, 0, 1, 3959, 4096, 114, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 3, 28, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 2, 4096, 0, 0, 0, 1, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 29, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 4092, 0, 0, 61, 40, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 4094, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 14, 4023, 11, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 4, 0, 4096, 0, 0, 0, 4051, 11, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 2, 4024, 39, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 5, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8208, 8274, 8203, 8144, 8171, 8152, 8194] avg=8192.0000 max=8274 max_rank=2 min=8144 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 3960, 28, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 3894, 0, 0, 61, 40, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 4085, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 3, 4023, 66, 0, 13, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 4096, 0, 9, 0, 4050, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 228, 0, 0, 0, 4001, 39, 0, 0, 0, 0, 0, 0, 4077, 0, 3, 0, 5, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8131, 8081, 8169, 8212, 8212, 8364, 8170] avg=8192.0000 max=8364 max_rank=6 min=8081 min_rank=2 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 29, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4094, 0, 0, 61, 40, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 39, 4023, 26, 12, 2, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4052, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4027, 39, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8194, 8285, 8167, 8183, 8163, 8162, 8196] avg=8192.0000 max=8285 max_rank=2 min=8162 min_rank=6 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 29, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4020, 0, 0, 61, 40, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 26, 4023, 101, 0, 1, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 4052, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 4027, 39, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8221, 8216, 8180, 8182, 8181, 8177, 8197] avg=8192.0000 max=8221 max_rank=1 min=8177 min_rank=6 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4081, 29, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4074, 0, 0, 0, 4096, 0, 0, 61, 40, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4, 4023, 9, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 0, 0, 4052, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4025, 39, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8207, 8272, 8199, 8134, 8185, 8160, 8197] avg=8192.0000 max=8272 max_rank=2 min=8134 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32676 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 65, 44, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 2, 4014, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4055, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4033, 41, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8211, 8299, 8209, 8112, 8161, 8170, 8192] avg=8192.0000 max=8299 max_rank=2 min=8112 min_rank=4 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32635 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 29, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 14, 3990, 0, 0, 61, 40, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 54, 4022, 22, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 4095, 0, 1, 0, 4052, 11, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 1, 8, 0, 0, 0, 4023, 39, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 5, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8195, 8204, 8203, 8197, 8192, 8163, 8200] avg=8192.0000 max=8204 max_rank=2 min=8163 min_rank=6 imbalance_ratio=1.001465 max_load_reduction=57332 max_load_reduction_pct=0.874817 moved_to_echo=32752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 43, 10, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 28, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 3803, 0, 0, 61, 40, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 383, 6, 0, 1, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 0, 4095, 0, 0, 0, 4052, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 28, 0, 0, 0, 4018, 43, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8169, 7905, 8206, 8372, 8271, 8188, 8190] avg=8192.0000 max=8372 max_rank=4 min=7905 min_rank=2 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=33021 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32640, 0, 0, 128, 32128, 0, 0, 512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32640, 256, 0, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 32256, 640, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 128, 384, 0, 0, 128, 32256, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 32768, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 128, 128, 0, 32768, 0, 128, 0, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4088, 1, 0, 0, 0, 0, 0, 3950, 4096, 4096, 0, 65, 4047, 9, 3243, 0, 0, 0, 4095, 0, 0, 623, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 330, 2, 4069, 0, 0, 0, 3715, 0, 0, 0, 4096, 140, 0, 4085, 0, 14, 0, 0, 4096, 292, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 3, 0, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 0, 0, 3705, 4094, 4096, 0, 67, 4096, 0, 2883, 0, 0, 0, 4094, 1, 0, 1683, 0, 38, 0, 0, 0, 4029, 0, 4096, 0, 0, 1, 4096, 2, 0, 0, 4019, 0, 0, 0, 4096, 1, 0, 4094, 2, 0, 0, 0, 4095, 8, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 1, 0, 4096, 4096, 4096, 0, 13, 4091, 10, 2849, 0, 0, 0, 4096, 0, 0, 1067, 0, 1, 0, 0, 0, 4085, 0, 4096, 0, 102, 0, 4096, 0, 0, 0, 3997, 0, 0, 0, 4096, 17, 0, 4093, 0, 35, 11, 0, 4096, 124, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4078] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3993, 4091, 4096, 0, 6, 4096, 0, 3328, 0, 0, 0, 4096, 0, 0, 847, 0, 15, 0, 0, 0, 4096, 0, 4096, 0, 3, 0, 4096, 0, 0, 0, 4059, 0, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 0, 4094, 38, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 12, 0, 4055, 4096, 4096, 0, 10, 4096, 0, 3956, 0, 0, 0, 4096, 0, 0, 457, 0, 37, 0, 0, 0, 3939, 0, 4096, 0, 93, 0, 4072, 0, 0, 0, 3845, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4073, 4096, 4096, 0, 0, 4096, 0, 3994, 0, 0, 0, 4096, 0, 0, 280, 0, 0, 0, 0, 0, 4076, 0, 4096, 0, 70, 0, 4093, 0, 0, 0, 3882, 0, 0, 0, 4096, 0, 0, 4096, 0, 3, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 0, 0, 0, 0, 4094, 4094, 4095, 0, 5, 4093, 0, 3856, 0, 1, 0, 4095, 0, 1, 211, 1, 0, 0, 0, 0, 4096, 1, 4095, 0, 1, 0, 4096, 0, 0, 0, 4085, 1, 0, 0, 4095, 2, 0, 4096, 0, 2, 0, 0, 4095, 79, 1, 0, 1, 1, 1, 1, 1, 0, 4095, 0, 0, 4053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 1, 0, 3568, 4096, 4096, 0, 106, 4096, 0, 2534, 0, 0, 0, 4096, 2, 0, 2188, 0, 144, 0, 0, 0, 3935, 0, 4096, 0, 50, 3, 4096, 0, 0, 0, 3857, 0, 0, 0, 4096, 7, 0, 4094, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 4081] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4088, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3950, 0, 0, 0, 65, 875, 9, 3243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 623, 0, 3074, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 12, 11, 0, 0, 0, 0, 0, 0, 330, 2, 4069, 0, 0, 0, 3715, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 140, 0, 4085, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 292, 0, 0, 0, 0, 4026, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 3, 0, 4086, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8142, 7862, 8206, 8203, 8310, 8415, 8214] avg=8192.0000 max=8415 max_rank=6 min=7862 min_rank=2 imbalance_ratio=1.027222 max_load_reduction=57121 max_load_reduction_pct=0.871597 moved_to_echo=33142 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3705, 0, 0, 0, 67, 886, 0, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 1683, 0, 3111, 70, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4029, 0, 4096, 0, 12, 11, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 2, 0, 0, 4019, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 1, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 8, 0, 1, 0, 0, 4024, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4045, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7541, 8959, 8186, 8206, 8168, 8129, 8154] avg=8192.0000 max=8959 max_rank=2 min=7541 min_rank=1 imbalance_ratio=1.093628 max_load_reduction=56577 max_load_reduction_pct=0.863297 moved_to_echo=32657 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 2, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 885, 10, 2849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1067, 0, 3107, 70, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4085, 0, 4096, 0, 12, 11, 0, 0, 0, 0, 0, 0, 102, 0, 4096, 0, 0, 0, 3997, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 17, 0, 4093, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 124, 0, 0, 0, 0, 4026, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4078, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7853, 8340, 8205, 8283, 8216, 8258, 8187] avg=8192.0000 max=8340 max_rank=2 min=7853 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32944 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 12, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 10, 886, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 457, 0, 3111, 70, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 3939, 0, 4096, 0, 12, 11, 0, 0, 0, 0, 0, 0, 93, 0, 4072, 0, 0, 0, 3845, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4026, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4094, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8907, 7734, 8095, 8098, 8167, 8127, 8205] avg=8192.0000 max=8907 max_rank=1 min=7734 min_rank=2 imbalance_ratio=1.087280 max_load_reduction=56629 max_load_reduction_pct=0.864090 moved_to_echo=32597 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 6, 886, 0, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 847, 0, 3111, 70, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 0, 4096, 0, 12, 11, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 4059, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 38, 0, 0, 0, 0, 4021, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8213, 8124, 8230, 8246, 8173, 8154, 8205] avg=8192.0000 max=8246 max_rank=4 min=8124 min_rank=2 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 879, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 280, 0, 3114, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 0, 16, 13, 0, 0, 0, 0, 0, 0, 70, 0, 4093, 0, 0, 0, 3882, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8946, 7561, 8201, 8135, 8160, 8132, 8209] avg=8192.0000 max=8946 max_rank=1 min=7561 min_rank=2 imbalance_ratio=1.092041 max_load_reduction=56590 max_load_reduction_pct=0.863495 moved_to_echo=32636 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 5, 885, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 211, 1, 3109, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4095, 0, 12, 11, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4085, 1, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 2, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 79, 1, 0, 1, 1, 4024, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 4095, 0, 0, 4053, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8840, 7488, 8215, 8271, 8170, 8202, 8164] avg=8192.0000 max=8840 max_rank=1 min=7488 min_rank=2 imbalance_ratio=1.079102 max_load_reduction=56696 max_load_reduction_pct=0.865112 moved_to_echo=32807 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3568, 0, 0, 0, 106, 886, 0, 2534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 2188, 0, 3111, 71, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 3935, 0, 4096, 0, 12, 11, 0, 0, 0, 0, 0, 0, 50, 3, 4096, 0, 0, 0, 3857, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 7, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4025, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 4081, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7094, 9468, 8198, 8094, 8172, 8119, 8198] avg=8192.0000 max=9468 max_rank=2 min=7094 min_rank=1 imbalance_ratio=1.155762 max_load_reduction=56068 max_load_reduction_pct=0.855530 moved_to_echo=32583 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[256, 0, 0, 0, 32384, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[768, 128, 32768, 128, 0, 0, 31488, 128, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 32768, 128, 128, 7424, 128, 24960, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31616, 0, 0, 0, 384, 7168, 128, 26752, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 32640, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 32768, 640, 128, 128, 128, 128, 32256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32640, 256, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[2273, 5, 0, 0, 469, 0, 4095, 0, 0, 18, 310, 0, 5, 0, 2815, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 4096, 3815, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 4083, 12, 0, 0, 4096, 0, 1942, 0, 0, 4027, 0, 561, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 150, 4096, 4088, 2, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[495, 0, 0, 0, 568, 0, 4096, 0, 0, 0, 148, 0, 1, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4094, 2922, 1, 0, 4096, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 8, 3599, 0, 0, 4073, 0, 407, 1, 0, 4096, 2, 0, 4096, 59, 0, 0, 11, 4096, 4096, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3428, 0, 0, 0, 77, 0, 4096, 0, 0, 0, 70, 0, 0, 0, 3286, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3232, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 2118, 0, 0, 4022, 0, 33, 0, 0, 4096, 0, 0, 4096, 101, 0, 0, 0, 4096, 4096, 17, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[3415, 0, 0, 0, 19, 0, 4096, 0, 0, 0, 347, 0, 0, 0, 4025, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3713, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 681, 0, 0, 3981, 0, 5, 0, 0, 4096, 0, 0, 4096, 91, 0, 0, 58, 4096, 4096, 49, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3194, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 39, 0, 0, 0, 3830, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3825, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1208, 0, 0, 4077, 0, 116, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 88, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[2644, 1, 0, 0, 55, 1, 4095, 2, 0, 2, 61, 1, 0, 0, 3734, 1, 0, 1, 0, 4092, 1, 0, 0, 0, 2, 4095, 0, 4094, 3285, 0, 0, 4095, 4092, 0, 0, 0, 1, 0, 4094, 4, 0, 0, 4092, 5, 2306, 0, 0, 4096, 2, 196, 3, 0, 4095, 0, 0, 4095, 2, 1, 1, 0, 4094, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[2869, 0, 0, 0, 23, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 3992, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2139, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 3248, 0, 0, 4096, 0, 9, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 1, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1224, 0, 0, 0, 3314, 0, 4096, 0, 0, 0, 2141, 0, 0, 0, 3716, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 1248, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 28, 0, 0, 4067, 0, 0, 0, 0, 4096, 0, 0, 4096, 523, 0, 0, 122, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[495, 0, 0, 0, 568, 0, 4096, 0, 760, 166, 159, 0, 0, 0, 0, 0, 0, 0, 148, 0, 1, 0, 4093, 0, 4094, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3005, 1091, 0, 0, 0, 0, 0, 0, 0, 1075, 0, 0, 2922, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2245, 8, 3599, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 1, 0, 3930, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 11, 3937, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6244, 8352, 8192, 8094, 8188, 9925, 8436, 8105] avg=8192.0000 max=9925 max_rank=5 min=6244 min_rank=0 imbalance_ratio=1.211548 max_load_reduction=55611 max_load_reduction_pct=0.848557 moved_to_echo=34654 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2273, 5, 0, 0, 469, 0, 4095, 0, 760, 166, 159, 0, 0, 0, 0, 0, 0, 18, 310, 0, 5, 0, 2815, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 3001, 1091, 0, 0, 0, 0, 0, 0, 0, 1074, 0, 0, 3815, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4083, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2245, 0, 1942, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 0, 0, 3930, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 3937, 4088, 2, 14, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7927, 7260, 8193, 8978, 8191, 8214, 8582, 8191] avg=8192.0000 max=8978 max_rank=3 min=7260 min_rank=1 imbalance_ratio=1.095947 max_load_reduction=56558 max_load_reduction_pct=0.863007 moved_to_echo=33178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3428, 0, 0, 0, 77, 0, 4096, 0, 760, 166, 159, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 3286, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3005, 1090, 0, 0, 0, 0, 0, 0, 0, 1075, 0, 0, 3232, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2244, 0, 2118, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 3930, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 3937, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8686, 7468, 8191, 8405, 8192, 8384, 8059, 8151] avg=8192.0000 max=8686 max_rank=0 min=7468 min_rank=1 imbalance_ratio=1.060303 max_load_reduction=56850 max_load_reduction_pct=0.867462 moved_to_echo=32786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3415, 0, 0, 0, 19, 0, 4096, 0, 760, 166, 159, 0, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 4025, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3005, 1091, 0, 0, 0, 0, 0, 0, 0, 1075, 0, 0, 3713, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2245, 0, 681, 0, 0, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3930, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 58, 3937, 4096, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8615, 8484, 8192, 8884, 8192, 6907, 8031, 8231] avg=8192.0000 max=8884 max_rank=3 min=6907 min_rank=5 imbalance_ratio=1.084473 max_load_reduction=56652 max_load_reduction_pct=0.864441 moved_to_echo=31361 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3194, 0, 0, 0, 7, 0, 4096, 0, 760, 166, 159, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 3830, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3005, 1091, 0, 0, 0, 0, 0, 0, 0, 1075, 0, 0, 3825, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2245, 0, 1208, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 3930, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 88, 3937, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8382, 7981, 8192, 8995, 8192, 7530, 8142, 8122] avg=8192.0000 max=8995 max_rank=3 min=7530 min_rank=5 imbalance_ratio=1.098022 max_load_reduction=56541 max_load_reduction_pct=0.862747 moved_to_echo=31986 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2869, 0, 0, 0, 23, 0, 4096, 0, 764, 167, 163, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3992, 0, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3007, 1093, 0, 0, 0, 0, 0, 0, 0, 1069, 0, 0, 2139, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2239, 2, 3248, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3929, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3926, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8082, 8113, 8196, 7304, 8198, 9585, 8034, 8024] avg=8192.0000 max=9585 max_rank=5 min=7304 min_rank=3 imbalance_ratio=1.170044 max_load_reduction=55951 max_load_reduction_pct=0.853745 moved_to_echo=33841 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2644, 1, 0, 0, 55, 1, 4095, 2, 759, 165, 159, 0, 0, 0, 0, 0, 0, 2, 61, 1, 0, 0, 3734, 1, 4094, 16, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 1, 0, 0, 0, 3004, 1090, 0, 0, 0, 0, 0, 0, 2, 1075, 0, 0, 3285, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 1, 0, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2243, 5, 2306, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 196, 3, 0, 3930, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 3935, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7881, 7909, 8188, 8457, 8191, 8650, 8226, 8034] avg=8192.0000 max=8650 max_rank=5 min=7881 min_rank=0 imbalance_ratio=1.055908 max_load_reduction=56886 max_load_reduction_pct=0.868011 moved_to_echo=33101 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1224, 0, 0, 0, 3314, 0, 4096, 0, 760, 166, 159, 0, 0, 0, 0, 0, 0, 0, 2141, 0, 0, 0, 3716, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3005, 1091, 0, 0, 0, 0, 0, 0, 0, 1075, 0, 0, 1248, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2245, 1, 28, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 523, 0, 0, 122, 3937, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9719, 9969, 8192, 6419, 8192, 6341, 8026, 8678] avg=8192.0000 max=9969 max_rank=1 min=6341 min_rank=5 imbalance_ratio=1.216919 max_load_reduction=55567 max_load_reduction_pct=0.847885 moved_to_echo=31237 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 18048, 128, 15232, 0, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 8704, 0, 0, 24192, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[896, 128, 128, 512, 31488, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[19584, 128, 0, 0, 4608, 128, 32768, 128, 6144, 1408, 1280, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 1408, 128, 0, 31488, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 32768, 128, 0, 0, 0, 24064, 8832, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 3200, 128, 128, 0, 29568, 128, 32768, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 4096, 4096, 0, 8, 5, 0, 0, 0, 0, 0, 8, 4078, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4095, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4076, 4096, 0, 0, 5, 4096, 4096, 0, 20, 0, 0, 0, 0, 7, 0, 4, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 21, 7, 7, 0, 4096, 0, 0, 0, 0, 0, 4074, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 5, 0, 18, 4071, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 49, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4056, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4092, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 90, 0, 12, 0, 0, 0, 4016, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4094, 4093, 6, 1, 1, 1, 4094, 0, 0, 9, 0, 0, 4085, 4095, 0, 0, 0, 4095, 4096, 1, 0, 0, 0, 0, 1, 0, 2, 8, 4090, 0, 0, 1, 0, 1, 4095, 0, 4095, 0, 1, 4095, 0, 4095, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 1, 12, 0, 0, 0, 0, 2, 4078, 0, 0, 1, 4095, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 4096, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 29, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 3976, 4096, 0, 0, 2, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 123, 4072, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 9, 0, 1, 0, 9, 1, 4092, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 15, 5, 4, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4078, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4056, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8214, 8200, 8162, 8193, 8192, 8173, 8214] avg=8192.0000 max=8214 max_rank=1 min=8162 min_rank=3 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4092, 4096, 21, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 0, 0, 15, 5, 4, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 18, 4071, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 49, 0, 0, 0, 4056, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8194, 8188, 8171, 8193, 8192, 8218, 8157] avg=8192.0000 max=8223 max_rank=0 min=8157 min_rank=7 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4091, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 0, 0, 15, 5, 4, 0, 0, 0, 0, 0, 5, 4096, 4091, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 4096, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 2, 0, 0, 0, 4056, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8196, 8212, 8184, 8192, 8192, 8165, 8211] avg=8192.0000 max=8212 max_rank=2 min=8165 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=57324 max_load_reduction_pct=0.874695 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 4096, 0, 0, 15, 5, 4, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4092, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 90, 0, 12, 0, 4056, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8176, 8187, 8195, 8192, 8192, 8269, 8137] avg=8192.0000 max=8269 max_rank=6 min=8137 min_rank=7 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32790 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 4096, 0, 0, 18, 12, 7, 2, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4077, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4060, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8237, 8180, 8174, 8190, 8191, 8175, 8204] avg=8192.0000 max=8237 max_rank=1 min=8174 min_rank=3 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 15, 5, 4, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4095, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4056, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8209, 8191, 8201, 8192, 8192, 8174, 8189] avg=8192.0000 max=8209 max_rank=1 min=8174 min_rank=6 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32747 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4090, 4093, 6, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4085, 4095, 0, 0, 15, 5, 4, 0, 0, 0, 0, 0, 0, 4095, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 8, 4090, 0, 0, 1, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 12, 0, 0, 0, 4055, 19, 0, 0, 0, 0, 0, 0, 0, 2, 4078, 0, 0, 1, 25, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8213, 8187, 8177, 8192, 8192, 8182, 8201] avg=8192.0000 max=8213 max_rank=1 min=8177 min_rank=3 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3976, 4096, 0, 0, 15, 5, 4, 0, 0, 0, 0, 0, 2, 4096, 4091, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 4072, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 9, 0, 1, 0, 4056, 19, 0, 0, 0, 0, 0, 0, 9, 1, 4092, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8097, 8191, 8272, 8192, 8193, 8180, 8223] avg=8192.0000 max=8272 max_rank=3 min=8097 min_rank=1 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 32640, 32768, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32640, 0, 0, 128, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 256, 32768, 0, 0, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 128, 256, 0, 128, 0, 32512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 4, 0, 0, 4096, 3, 0, 5, 4094, 0, 4096, 0, 0, 4091, 0, 0, 0, 4096, 0, 4096, 0, 105, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 3909, 4086, 0, 0, 4096, 97, 4096, 0, 0, 10, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 11, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 24, 4078, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 58, 0, 3, 0, 0, 0, 4091, 0, 4096, 0, 4037, 0, 0, 0, 0, 35, 0, 0, 4096, 4024, 0, 59, 4096, 0, 4096, 0, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 1, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 4096, 2, 0, 0, 4093, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 279, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 3651, 4095, 0, 34, 4096, 139, 4094, 0, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 18, 0, 0, 4096, 15, 0, 0, 4092, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4095, 1, 4094, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 4095, 2, 1, 0, 4096, 0, 4095, 0, 1, 4096, 0, 0, 0, 4095, 0, 4095, 1, 157, 0, 1, 0, 0, 0, 4093, 1, 4095, 0, 4089, 1, 0, 2, 0, 0, 1, 0, 3938, 4055, 0, 29, 4095, 19, 4095, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 29, 0, 2, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 1008, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 3177, 3988, 0, 5, 4096, 25, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 10, 0, 0, 4096, 764, 0, 45, 4095, 0, 4096, 0, 0, 4077, 0, 0, 0, 4096, 0, 4096, 0, 722, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 4089, 2777, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 177, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3766, 4096, 0, 0, 4096, 153, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 1, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4083, 7, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 4094, 0, 3707, 75, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 2, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3651, 4095, 0, 34, 382, 139, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8187, 8084, 8195, 8155, 8224, 8301, 8190] avg=8192.0000 max=8301 max_rank=6 min=8084 min_rank=2 imbalance_ratio=1.013306 max_load_reduction=57235 max_load_reduction_pct=0.873337 moved_to_echo=32870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 11, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 7, 0, 0, 0, 0, 0, 0, 1, 0, 24, 4078, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 3, 0, 0, 0, 4091, 0, 3707, 75, 0, 0, 0, 0, 0, 0, 4096, 0, 4037, 0, 0, 0, 0, 35, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4024, 0, 59, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8186, 8092, 8195, 7934, 8200, 8561, 8193] avg=8192.0000 max=8561 max_rank=6 min=7934 min_rank=4 imbalance_ratio=1.045044 max_load_reduction=56975 max_load_reduction_pct=0.869370 moved_to_echo=32888 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 4096, 4083, 7, 0, 0, 0, 0, 0, 0, 3, 0, 5, 4094, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 4096, 0, 3707, 75, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 4086, 0, 0, 382, 97, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8192, 8091, 8191, 7983, 8204, 8474, 8199] avg=8192.0000 max=8474 max_rank=6 min=7983 min_rank=4 imbalance_ratio=1.034424 max_load_reduction=57062 max_load_reduction_pct=0.870697 moved_to_echo=32860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 4083, 7, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4092, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 3707, 75, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8204, 8096, 8196, 7883, 8199, 8565, 8189] avg=8192.0000 max=8565 max_rank=6 min=7883 min_rank=4 imbalance_ratio=1.045532 max_load_reduction=56971 max_load_reduction_pct=0.869308 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 7, 0, 0, 0, 0, 0, 0, 29, 0, 2, 4096, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 0, 0, 0, 0, 0, 4096, 0, 3707, 75, 0, 0, 0, 0, 0, 0, 4096, 0, 4059, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3177, 3988, 0, 5, 382, 25, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8186, 8116, 8196, 8886, 8187, 7577, 8189] avg=8192.0000 max=8886 max_rank=4 min=7577 min_rank=6 imbalance_ratio=1.084717 max_load_reduction=56650 max_load_reduction_pct=0.864410 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 4096, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 4083, 7, 0, 0, 0, 0, 0, 0, 764, 0, 45, 4095, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 722, 0, 0, 0, 0, 0, 4093, 0, 3707, 75, 0, 0, 0, 0, 0, 0, 4096, 0, 3946, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2777, 0, 2, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8196, 8893, 8175, 8597, 8074, 7250, 8194] avg=8192.0000 max=8893 max_rank=2 min=7250 min_rank=6 imbalance_ratio=1.085571 max_load_reduction=56643 max_load_reduction_pct=0.864304 moved_to_echo=32115 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 4096, 0, 3713, 79, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 4096, 0, 0, 369, 153, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8200, 8076, 8192, 8065, 8229, 8384, 8194] avg=8192.0000 max=8384 max_rank=6 min=8065 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=57152 max_load_reduction_pct=0.872070 moved_to_echo=32872 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 1, 4094, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 4082, 7, 0, 0, 0, 0, 0, 0, 2, 1, 0, 4096, 0, 3988, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 1, 0, 0, 0, 4093, 1, 3706, 75, 0, 0, 0, 0, 0, 0, 4095, 0, 4089, 1, 0, 2, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3938, 4055, 0, 29, 382, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 4091, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8185, 8088, 8196, 8033, 8219, 8424, 8188] avg=8192.0000 max=8424 max_rank=6 min=8033 min_rank=4 imbalance_ratio=1.028320 max_load_reduction=57112 max_load_reduction_pct=0.871460 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 32512, 128, 0, 128, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[896, 128, 128, 32768, 0, 32000, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 30720, 31232, 0, 256, 3072, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 128, 0, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2560, 0, 128, 0, 0, 0, 32768, 128, 29696, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 4083, 0, 4096, 4074, 0, 13, 0, 0, 4096, 0, 27, 0, 0, 4084, 0, 0, 4096, 0, 3650, 4096, 0, 0, 4096, 0, 42, 443, 0, 4003, 0, 0, 4057, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 93, 0, 4087, 0, 4096, 11, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 4092, 0, 0, 4096, 0, 3857, 4095, 0, 0, 4095, 0, 31, 230, 0, 4065, 0, 0, 4090, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 4096, 0, 10, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 528, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 4090, 0, 0, 4096, 0, 3328, 4096, 0, 0, 4096, 0, 8, 240, 0, 4089, 0, 0, 4092, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 450, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 4096, 0, 0, 4096, 0, 3384, 4096, 0, 0, 4096, 0, 6, 404, 0, 4078, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3959, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 4095, 0, 4095, 4095, 0, 0, 1, 0, 4095, 11, 0, 0, 1, 4094, 1, 0, 4095, 0, 4086, 4083, 1, 1, 4096, 0, 0, 10, 0, 4095, 1, 0, 4093, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 3, 1, 1, 0, 0, 4095, 0, 0, 4096, 0, 4095, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 199, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3820, 4096, 0, 0, 4096, 0, 0, 76, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1, 1661, 0, 10, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 41, 0, 0, 4095, 0, 0, 4096, 0, 1117, 4096, 0, 0, 4096, 0, 2, 1509, 0, 3880, 0, 0, 4074, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 173, 0, 0, 0, 4096, 0, 4096, 4087, 0, 0, 0, 0, 4096, 0, 27, 0, 0, 4095, 0, 0, 4096, 0, 3814, 4096, 0, 0, 4096, 0, 9, 115, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4063, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 528, 0, 0, 0, 4096, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 718, 0, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3707, 0, 8, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4092, 0, 0, 4, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 3378, 389, 378, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 12, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8342, 8184, 8136, 8051, 8240, 8190, 8180, 8213] avg=8192.0000 max=8342 max_rank=0 min=8051 min_rank=3 imbalance_ratio=1.018311 max_load_reduction=57194 max_load_reduction_pct=0.872711 moved_to_echo=32823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 4083, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 13, 0, 0, 4082, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 718, 0, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3707, 0, 42, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 4057, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4041, 3378, 389, 378, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 11, 0, 0, 0, 12, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7805, 8196, 8452, 8288, 8115, 8191, 8273, 8216] avg=8192.0000 max=8452 max_rank=2 min=7805 min_rank=0 imbalance_ratio=1.031738 max_load_reduction=57084 max_load_reduction_pct=0.871033 moved_to_echo=32795 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 0, 0, 4096, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 718, 0, 3857, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3707, 0, 31, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 4090, 0, 0, 2, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 3378, 388, 378, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 10, 0, 0, 12, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7823, 8194, 8667, 8063, 8212, 8189, 8180, 8208] avg=8192.0000 max=8667 max_rank=2 min=7823 min_rank=0 imbalance_ratio=1.057983 max_load_reduction=56869 max_load_reduction_pct=0.867752 moved_to_echo=32789 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 450, 0, 0, 0, 4096, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 718, 0, 3384, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3707, 0, 6, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 4091, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 3378, 389, 378, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 4096, 0, 0, 0, 0, 12, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8190, 8198, 8213, 8224, 8190, 8180, 8077] avg=8192.0000 max=8264 max_rank=0 min=8077 min_rank=7 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 199, 0, 0, 0, 4096, 0, 3717, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 711, 0, 3820, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3706, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 3385, 390, 379, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 13, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8012, 8170, 8627, 7878, 8245, 8206, 8180, 8218] avg=8192.0000 max=8627 max_rank=2 min=7878 min_rank=3 imbalance_ratio=1.053101 max_load_reduction=56909 max_load_reduction_pct=0.868362 moved_to_echo=32849 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 173, 0, 0, 0, 4096, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4082, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 718, 0, 3814, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3707, 0, 9, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4041, 3378, 389, 378, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 4096, 0, 0, 0, 0, 12, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7987, 8196, 8627, 7927, 8247, 8191, 8180, 8181] avg=8192.0000 max=8627 max_rank=2 min=7927 min_rank=3 imbalance_ratio=1.053101 max_load_reduction=56909 max_load_reduction_pct=0.868362 moved_to_echo=32799 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 4095, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4081, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 717, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 1, 3707, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4093, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4040, 3378, 389, 377, 4, 0, 0, 0, 4095, 3, 1, 1, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 1, 12, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7817, 8188, 8899, 7802, 8244, 8189, 8183, 8214] avg=8192.0000 max=8899 max_rank=2 min=7802 min_rank=3 imbalance_ratio=1.086304 max_load_reduction=56637 max_load_reduction_pct=0.864212 moved_to_echo=32830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 1661, 0, 10, 0, 4096, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4082, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 718, 0, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3707, 0, 2, 1509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 4074, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 3378, 389, 378, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 12, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9486, 8218, 5930, 9314, 8009, 8190, 8180, 8209] avg=8192.0000 max=9486 max_rank=0 min=5930 min_rank=2 imbalance_ratio=1.157959 max_load_reduction=56050 max_load_reduction_pct=0.855255 moved_to_echo=32588 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 0, 5760, 0, 27136, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 3072, 0, 128, 0, 32768, 0, 29824, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32640, 0, 32768, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 0, 32384, 27136, 3200, 3072, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32512, 128, 0, 32768, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 128, 0, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 128, 29696, 0, 128, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 10, 0, 4096, 4096, 4096, 0, 157, 4096, 4094, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3972, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4064, 0, 0, 0, 0, 0, 0, 4096, 0, 38, 0, 0, 4096, 4096, 4096, 9, 149, 4096, 4096, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 3947, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4085, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 27, 0, 4096, 4096, 4096, 0, 103, 4096, 4096, 0, 4096, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 43, 0, 0, 0, 4096, 3, 0, 0, 2, 0, 3954, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 4096, 4096, 4095, 0, 2, 4096, 4096, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4095, 4077, 1, 0, 0, 0, 0, 2, 4095, 0, 2, 0, 0, 4095, 4095, 4092, 0, 55, 4095, 4095, 0, 4095, 4083, 0, 0, 0, 1, 0, 1, 7, 0, 0, 0, 0, 0, 0, 4093, 4094, 1, 1, 0, 4, 1, 0, 3, 4089, 1, 1, 0, 0, 0, 4076, 0, 1, 12, 0, 1, 1, 4095, 1, 1, 4074, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 956, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3756, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3486, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 3843, 0, 0, 0, 0, 0, 0, 4096, 0, 208, 0, 0, 4096, 4096, 4096, 54, 255, 4096, 4096, 0, 4096, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3966, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 4096, 4096, 4096, 0, 44, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4094, 0, 0, 0, 0, 0, 40, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 10, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 157, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4008, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 4096, 86, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3972, 0, 113, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8165, 8153, 8192, 8192, 8183, 8238, 8181] avg=8192.0000 max=8238 max_rank=6 min=8153 min_rank=2 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32794 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4064, 0, 0, 0, 0, 0, 40, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 9, 149, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4010, 79, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3947, 0, 0, 0, 4096, 86, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 113, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8190, 8139, 8191, 8192, 8185, 8132, 8305] avg=8192.0000 max=8305 max_rank=7 min=8132 min_rank=6 imbalance_ratio=1.013794 max_load_reduction=57231 max_load_reduction_pct=0.873276 moved_to_echo=32814 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4085, 0, 0, 0, 0, 0, 40, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 103, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 4096, 3, 4010, 79, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3954, 8, 0, 0, 4096, 86, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 113, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8179, 8069, 8191, 8192, 8231, 8147, 8304] avg=8192.0000 max=8304 max_rank=7 min=8069 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=57232 max_load_reduction_pct=0.873291 moved_to_echo=32874 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4094, 0, 0, 0, 0, 0, 40, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 0, 2, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4010, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 86, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 113, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8177, 7982, 8191, 8192, 8187, 8279, 8296] avg=8192.0000 max=8296 max_rank=7 min=7982 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32954 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4090, 0, 0, 0, 0, 0, 45, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 0, 956, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4011, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 0, 0, 0, 4096, 85, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3486, 0, 118, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8143, 8941, 8191, 8196, 8193, 7938, 7700] avg=8192.0000 max=8941 max_rank=2 min=7700 min_rank=7 imbalance_ratio=1.091431 max_load_reduction=56595 max_load_reduction_pct=0.863571 moved_to_echo=32027 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 4077, 1, 0, 0, 0, 0, 40, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 4055, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 0, 55, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 7, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 1, 0, 3, 4089, 1, 4009, 79, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4076, 0, 1, 12, 4095, 86, 1, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 1, 1, 4074, 0, 113, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8154, 8036, 8198, 8188, 8187, 8272, 8286] avg=8192.0000 max=8286 max_rank=7 min=8036 min_rank=2 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32933 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3843, 0, 0, 0, 0, 0, 40, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 54, 255, 0, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4010, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 86, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3966, 0, 113, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7981, 8360, 8174, 8191, 8192, 8185, 8278, 8175] avg=8192.0000 max=8360 max_rank=1 min=7981 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32830 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4079, 0, 0, 0, 0, 0, 40, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 44, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4010, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 4096, 86, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 113, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8168, 8042, 8191, 8192, 8185, 8252, 8289] avg=8192.0000 max=8289 max_rank=7 min=8042 min_rank=2 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32918 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 32512, 128, 0, 0, 0, 0, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 384, 128, 0, 32512, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 31872, 0, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[31232, 128, 1792, 0, 0, 0, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 32128, 128, 128, 128, 32768, 768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 0, 128, 32768, 128, 32128, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 4079, 0, 0, 0, 302, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 4069, 0, 4096, 0, 3782, 0, 0, 4017, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 128, 0, 2, 0, 0, 4087, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 2145, 0, 0, 0, 1155, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4069, 6, 4096, 0, 3743, 0, 0, 4053, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1216, 0, 0, 0, 0, 4090, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 42, 0, 2, 1210, 0, 0, 0, 291, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4083, 86, 4096, 0, 3969, 0, 0, 4034, 0, 1, 0, 0, 4096, 4096, 0, 4096, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 2731, 0, 1, 0, 0, 4022, 4095, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2361, 0, 0, 0, 938, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4026, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 867, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3952, 0, 0, 0, 53, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 2, 4096, 0, 3904, 0, 0, 4049, 0, 1, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 318, 0, 1, 0, 0, 4093, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 76, 0, 0, 4086, 0, 0, 0, 131, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4094, 0, 4096, 0, 4090, 0, 0, 3946, 0, 21, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4035, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 5, 1225, 1, 0, 1, 3065, 4094, 0, 0, 0, 1, 0, 0, 4094, 2, 0, 0, 4095, 4095, 0, 4095, 0, 3762, 1, 1, 4082, 0, 9, 0, 1, 4095, 4095, 1, 4095, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 4095, 0, 1, 0, 1, 1, 0, 138, 0, 0, 0, 2, 4095, 4095, 4095, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 0, 3945, 0, 0, 0, 42, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4010, 0, 0, 4077, 0, 2, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 199, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 2145, 0, 0, 4077, 701, 519, 0, 0, 0, 0, 0, 0, 1155, 4096, 0, 0, 0, 0, 0, 3343, 3, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4069, 6, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 3743, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 0, 4096, 3395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7444, 8598, 8168, 8030, 8192, 8192, 8707, 8205] avg=8192.0000 max=8707 max_rank=6 min=7444 min_rank=0 imbalance_ratio=1.062866 max_load_reduction=56829 max_load_reduction_pct=0.867142 moved_to_echo=33296 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 4079, 0, 0, 4077, 702, 519, 0, 0, 0, 0, 0, 0, 302, 4096, 0, 0, 0, 0, 0, 3343, 3, 1, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 3782, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 4096, 3394, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4087, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9379, 7745, 8165, 8033, 8200, 8192, 7618, 8204] avg=8192.0000 max=9379 max_rank=0 min=7618 min_rank=6 imbalance_ratio=1.144897 max_load_reduction=56157 max_load_reduction_pct=0.856888 moved_to_echo=32214 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 42, 0, 2, 1210, 0, 0, 4077, 702, 519, 0, 0, 0, 0, 0, 0, 291, 4096, 0, 0, 0, 0, 0, 3343, 3, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4083, 86, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 3969, 0, 0, 4034, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2731, 0, 4096, 3394, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4022, 4095, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6552, 7734, 8262, 8238, 8198, 8192, 10223, 8137] avg=8192.0000 max=10223 max_rank=6 min=6552 min_rank=0 imbalance_ratio=1.247925 max_load_reduction=55313 max_load_reduction_pct=0.844009 moved_to_echo=34750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 3952, 0, 0, 4077, 702, 519, 0, 0, 0, 0, 0, 0, 53, 4096, 0, 0, 0, 0, 0, 3343, 3, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 3904, 0, 0, 4049, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 318, 0, 4096, 3394, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9251, 7496, 8190, 8188, 8191, 8191, 7820, 8209] avg=8192.0000 max=9251 max_rank=0 min=7496 min_rank=1 imbalance_ratio=1.129272 max_load_reduction=56285 max_load_reduction_pct=0.858841 moved_to_echo=32411 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 0, 3945, 0, 0, 4077, 702, 519, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 0, 0, 0, 0, 3343, 3, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 4010, 0, 0, 4077, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 199, 0, 4096, 3394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9254, 7485, 8189, 8323, 8191, 8192, 7691, 8211] avg=8192.0000 max=9254 max_rank=0 min=7485 min_rank=1 imbalance_ratio=1.129639 max_load_reduction=56282 max_load_reduction_pct=0.858795 moved_to_echo=32285 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 2361, 0, 0, 4080, 702, 523, 0, 0, 0, 0, 0, 0, 938, 4096, 0, 0, 0, 0, 0, 3350, 5, 7, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 4026, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 867, 0, 4096, 3394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7666, 8396, 8187, 8345, 8182, 8195, 8357, 8208] avg=8192.0000 max=8396 max_rank=1 min=7666 min_rank=0 imbalance_ratio=1.024902 max_load_reduction=57140 max_load_reduction_pct=0.871887 moved_to_echo=32942 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 76, 0, 0, 4086, 0, 0, 4077, 702, 519, 0, 0, 0, 0, 0, 0, 131, 4096, 0, 0, 0, 0, 0, 3343, 3, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 4090, 0, 0, 3946, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 3394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9460, 7574, 8187, 8291, 8191, 8192, 7491, 8150] avg=8192.0000 max=9460 max_rank=0 min=7491 min_rank=6 imbalance_ratio=1.154785 max_load_reduction=56076 max_load_reduction_pct=0.855652 moved_to_echo=32024 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 5, 1225, 1, 0, 4076, 702, 519, 0, 0, 0, 0, 0, 1, 3065, 4094, 0, 0, 0, 1, 0, 3343, 3, 1, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 3762, 1, 1, 4082, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 138, 0, 4095, 3393, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 4095, 19, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6530, 10508, 8188, 8088, 8191, 8190, 7629, 8212] avg=8192.0000 max=10508 max_rank=1 min=6530 min_rank=0 imbalance_ratio=1.282715 max_load_reduction=55028 max_load_reduction_pct=0.839661 moved_to_echo=32222 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 0, 5632, 0, 32768, 27264, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32640, 32768, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 256, 0, 128, 23040, 128, 0, 32640, 5632, 4224, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 0, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1920, 0, 31360, 128, 128, 32384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 6016, 32768, 0, 0, 0, 128, 0, 26752, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 5, 0, 35, 0, 0, 0, 4096, 4096, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3953, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4081, 129, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 48, 0, 82, 0, 0, 0, 4086, 4096, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3846, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4096, 134, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 881, 0, 12, 0, 0, 0, 4096, 4096, 0, 4096, 3244, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3962, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 91, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 65, 0, 42, 0, 0, 0, 4096, 4096, 0, 4096, 3607, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3968, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4093, 516, 4095, 0, 4096, 0, 4096, 4092, 0, 0, 0, 4096, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 3303, 1, 0, 0, 1, 1, 4095, 4095, 4, 4095, 575, 0, 1, 0, 0, 1, 1, 16, 0, 0, 1, 2, 0, 0, 0, 1, 4092, 0, 0, 4095, 0, 4095, 1, 32, 0, 4083, 1, 1, 4094, 1, 1, 0, 1, 0, 1, 4095, 181, 4095, 0, 4095, 0, 4093, 4095, 0, 0, 0, 4095, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 0, 11, 0, 381, 0, 0, 0, 4096, 4096, 0, 4096, 4019, 0, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3611, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 289, 4096, 0, 4096, 0, 4088, 3963, 0, 0, 0, 4096, 19, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 161, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3940, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3419, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 454, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 32, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 43, 0, 5, 0, 0, 0, 4096, 4096, 0, 4096, 3991, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 3934, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 182, 4096, 0, 4096, 0, 4096, 4091, 0, 0, 0, 4096, 37, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 48, 0, 82, 3439, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 653, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 0, 4086, 0, 0, 0, 0, 243, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 134, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 6, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7688, 8825, 8171, 8192, 8189, 8182, 8083, 8206] avg=8192.0000 max=8825 max_rank=1 min=7688 min_rank=0 imbalance_ratio=1.077271 max_load_reduction=56711 max_load_reduction_pct=0.865341 moved_to_echo=32660 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 65, 0, 42, 3439, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 653, 0, 0, 3607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 30, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 4092, 0, 0, 0, 0, 243, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 516, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7665, 8356, 8163, 8193, 8189, 8310, 8461, 8199] avg=8192.0000 max=8461 max_rank=6 min=7665 min_rank=0 imbalance_ratio=1.032837 max_load_reduction=57075 max_load_reduction_pct=0.870895 moved_to_echo=33159 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 5, 0, 35, 3439, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 653, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 0, 0, 4089, 0, 0, 0, 0, 243, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4081, 129, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7598, 8838, 8168, 8192, 8189, 8292, 8063, 8196] avg=8192.0000 max=8838 max_rank=1 min=7598 min_rank=0 imbalance_ratio=1.078857 max_load_reduction=56698 max_load_reduction_pct=0.865143 moved_to_echo=32740 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 881, 0, 12, 3439, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 653, 0, 0, 3244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 0, 4096, 0, 0, 0, 0, 243, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4087, 91, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8451, 7993, 8166, 8192, 8189, 8308, 8031, 8206] avg=8192.0000 max=8451 max_rank=0 min=7993 min_rank=1 imbalance_ratio=1.031616 max_load_reduction=57085 max_load_reduction_pct=0.871048 moved_to_echo=32734 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 43, 0, 5, 3439, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 653, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 4096, 0, 0, 0, 0, 243, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 182, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 6, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7606, 8740, 8168, 8192, 8189, 8280, 8131, 8230] avg=8192.0000 max=8740 max_rank=1 min=7606 min_rank=0 imbalance_ratio=1.066895 max_load_reduction=56796 max_load_reduction_pct=0.866638 moved_to_echo=32830 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 161, 0, 0, 3441, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 650, 0, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 0, 0, 4096, 0, 0, 0, 0, 246, 5, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 454, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7722, 8686, 8349, 8193, 8182, 7776, 8400, 8228] avg=8192.0000 max=8686 max_rank=1 min=7722 min_rank=0 imbalance_ratio=1.060303 max_load_reduction=56850 max_load_reduction_pct=0.867462 moved_to_echo=32586 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 0, 11, 0, 381, 3439, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 653, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 5, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3611, 0, 0, 4096, 0, 0, 0, 0, 243, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 289, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3963, 0, 0, 0, 6, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7946, 8768, 8170, 8192, 8189, 7957, 8238, 8076] avg=8192.0000 max=8768 max_rank=1 min=7946 min_rank=0 imbalance_ratio=1.070312 max_load_reduction=56768 max_load_reduction_pct=0.866211 moved_to_echo=32460 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 3303, 1, 0, 3438, 23, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 654, 4, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 16, 0, 4095, 4067, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 1, 4092, 4066, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4092, 1, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 1, 4094, 1, 1, 0, 1, 243, 3, 3, 0, 0, 0, 0, 0, 0, 1, 4095, 181, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4095, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10860, 5330, 8181, 8190, 8220, 8431, 8129, 8195] avg=8192.0000 max=10860 max_rank=0 min=5330 min_rank=1 imbalance_ratio=1.325684 max_load_reduction=54676 max_load_reduction_pct=0.834290 moved_to_echo=32975 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 2048, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 0, 0, 4608, 128, 640, 27520, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 32768, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 32640, 0, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 128, 256, 0, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 5248, 128, 0, 27648, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[30848, 128, 128, 32768, 128, 128, 0, 128, 2048, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2957, 1, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2173, 0, 0, 821, 0, 2241, 4096, 0, 4096, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 4096, 2, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2879, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2028, 0, 0, 1110, 0, 2196, 4096, 0, 4079, 0, 6, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4089, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3624, 225, 4095, 0, 0, 0, 0, 0, 4086, 4096, 1111, 1, 0, 1636, 0, 1701, 4096, 0, 4066, 0, 20, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4052, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4042, 0, 4096, 0, 0, 0, 4096, 14, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1297, 4, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2799, 0, 0, 442, 0, 3650, 4096, 0, 4091, 0, 40, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 30, 0, 0, 0, 0, 4096, 4040, 0, 4096, 0, 0, 0, 4096, 8, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3997, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2496, 0, 0, 1148, 0, 650, 4096, 0, 3997, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3867, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3012, 0, 0, 383, 0, 1167, 4096, 0, 3864, 0, 2, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 1, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1, 1, 1, 0, 0, 1, 3870, 34, 4095, 1, 0, 0, 0, 1, 4095, 4094, 1649, 0, 1, 2346, 1, 416, 4095, 2, 3964, 0, 0, 0, 0, 1, 4095, 1, 0, 1, 0, 1, 1, 0, 0, 4095, 4094, 0, 0, 1, 1, 0, 0, 7, 4095, 0, 0, 0, 1, 0, 4095, 4095, 0, 4095, 0, 0, 0, 4094, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3238, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 839, 0, 0, 3116, 0, 1021, 4096, 0, 4074, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2957, 1, 4096, 846, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2173, 0, 0, 821, 0, 2241, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4094, 0, 58, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3231, 864, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3248, 853, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 12, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7900, 8194, 8409, 8252, 8191, 8197, 8201, 8192] avg=8192.0000 max=8409 max_rank=2 min=7900 min_rank=0 imbalance_ratio=1.026489 max_load_reduction=57127 max_load_reduction_pct=0.871689 moved_to_echo=32781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 2879, 0, 4096, 846, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2028, 0, 0, 1110, 0, 2196, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 6, 0, 0, 0, 4093, 0, 58, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3231, 864, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3248, 853, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 12, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7821, 8194, 8508, 8238, 8191, 8197, 8197, 8190] avg=8192.0000 max=8508 max_rank=2 min=7821 min_rank=0 imbalance_ratio=1.038574 max_load_reduction=57028 max_load_reduction_pct=0.870178 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3867, 0, 4096, 846, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 0, 383, 0, 1167, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 2, 0, 0, 0, 4091, 0, 58, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3231, 864, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 3248, 853, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 12, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8809, 8194, 7736, 8017, 8191, 8195, 8203, 8191] avg=8192.0000 max=8809 max_rank=0 min=7736 min_rank=2 imbalance_ratio=1.075317 max_load_reduction=56727 max_load_reduction_pct=0.865585 moved_to_echo=32780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3238, 0, 4096, 846, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 839, 0, 0, 3116, 0, 1021, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 4096, 0, 58, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3231, 864, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3248, 853, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8194, 8150, 8230, 8191, 8197, 8204, 8190] avg=8192.0000 max=8230 max_rank=3 min=8150 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1297, 4, 4096, 848, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2799, 0, 0, 442, 0, 3650, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 40, 0, 0, 0, 4080, 0, 58, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3231, 864, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3248, 853, 0, 0, 0, 0, 0, 0, 4095, 30, 0, 0, 0, 0, 12, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6245, 8192, 10065, 8271, 8191, 8197, 8177, 8198] avg=8192.0000 max=10065 max_rank=2 min=6245 min_rank=0 imbalance_ratio=1.228638 max_load_reduction=55471 max_load_reduction_pct=0.846420 moved_to_echo=32763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3997, 0, 4096, 848, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2496, 0, 0, 1148, 0, 650, 3163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 4096, 0, 65, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3234, 868, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3248, 854, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8941, 8192, 7457, 8166, 8198, 8198, 8200, 8184] avg=8192.0000 max=8941 max_rank=0 min=7457 min_rank=2 imbalance_ratio=1.091431 max_load_reduction=56595 max_load_reduction_pct=0.863571 moved_to_echo=32780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 0, 0, 1, 3870, 34, 4095, 846, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 1, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1649, 0, 1, 2346, 1, 416, 3173, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 0, 0, 0, 1, 4095, 1, 58, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4095, 3230, 864, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 1, 0, 0, 7, 3246, 852, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 13, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8849, 8194, 7588, 8121, 8192, 8201, 8204, 8187] avg=8192.0000 max=8849 max_rank=0 min=7588 min_rank=2 imbalance_ratio=1.080200 max_load_reduction=56687 max_load_reduction_pct=0.864975 moved_to_echo=32784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3624, 225, 4096, 846, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 1, 0, 1636, 0, 1701, 3174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 20, 0, 0, 0, 4095, 0, 58, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3231, 864, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 3249, 853, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 12, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8791, 8182, 7623, 8241, 8191, 8154, 8150, 8204] avg=8192.0000 max=8791 max_rank=0 min=7623 min_rank=2 imbalance_ratio=1.073120 max_load_reduction=56745 max_load_reduction_pct=0.865860 moved_to_echo=32699 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 0, 128, 25984, 6912, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 128, 25856, 384, 32768, 6784, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32256, 0, 128, 0, 0, 128, 32768, 128, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[16128, 128, 128, 11008, 128, 13056, 25472, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 32768, 25856, 7040, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3874, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 248, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 4096, 4066, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3840, 0, 0, 0, 4096, 0, 4093, 4089, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1967, 0, 0, 4096, 0, 0, 0, 0, 4096, 9, 4096, 2383, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 4086, 0, 0, 0, 4096, 0, 4094, 4095, 0, 2, 0, 0, 4093, 0, 0, 4096, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 4096, 8, 4096, 4094, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 417, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3800, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 3975, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 3212, 0, 0, 0, 4096, 0, 4090, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 976, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4009, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1, 1, 0, 4065, 0, 0, 0, 4095, 1, 4094, 4095, 1, 0, 1, 0, 4095, 0, 0, 4095, 0, 94, 0, 1, 4095, 0, 1, 1, 0, 4093, 1, 4095, 4004, 0, 4095, 0, 1, 4095, 0, 1, 0, 0, 0, 0, 2, 4095, 1, 1, 4095, 1, 0, 0, 0, 0, 4095, 0, 1, 4095, 0, 1, 0, 0, 1, 26, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 2888, 0, 0, 0, 4096, 0, 4064, 3914, 0, 0, 0, 0, 4024, 0, 0, 4096, 0, 1171, 0, 0, 4096, 0, 0, 0, 0, 4069, 30, 4096, 4022, 0, 4096, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 27, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 361, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1283, 0, 0, 0, 4096, 0, 4096, 4055, 0, 0, 0, 4, 4071, 0, 0, 4096, 0, 3122, 0, 0, 4096, 0, 0, 0, 0, 4048, 35, 4096, 3827, 0, 4096, 0, 0, 4094, 14, 0, 0, 0, 0, 0, 23, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4086, 0, 0, 0, 4096, 1128, 6, 2, 0, 0, 0, 0, 0, 0, 41, 4095, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 299, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4053, 45, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3752, 293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9318, 8231, 6775, 8497, 8190, 8187, 8197, 8141] avg=8192.0000 max=9318 max_rank=0 min=6775 min_rank=2 imbalance_ratio=1.137451 max_load_reduction=56218 max_load_reduction_pct=0.857819 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3840, 0, 0, 0, 4096, 1128, 6, 2, 0, 0, 0, 0, 0, 0, 41, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1967, 0, 0, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 299, 2383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4052, 45, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 3752, 293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9072, 8226, 8738, 6787, 8190, 8186, 8193, 8144] avg=8192.0000 max=9072 max_rank=0 min=6787 min_rank=3 imbalance_ratio=1.107422 max_load_reduction=56464 max_load_reduction_pct=0.861572 moved_to_echo=32713 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1283, 0, 0, 0, 4096, 1128, 6, 2, 0, 0, 0, 0, 0, 0, 41, 4055, 0, 0, 0, 4, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3122, 0, 0, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 35, 299, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4055, 45, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3752, 293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6515, 8171, 9893, 8209, 8202, 8209, 8196, 8141] avg=8192.0000 max=9893 max_rank=2 min=6515 min_rank=0 imbalance_ratio=1.207642 max_load_reduction=55643 max_load_reduction_pct=0.849045 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3874, 0, 0, 0, 4096, 1128, 6, 2, 0, 0, 0, 0, 0, 0, 41, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 248, 0, 0, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 299, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4055, 45, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 0, 3752, 293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9106, 8233, 7019, 8457, 8190, 8186, 8196, 8149] avg=8192.0000 max=9106 max_rank=0 min=7019 min_rank=2 imbalance_ratio=1.111572 max_load_reduction=56430 max_load_reduction_pct=0.861053 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3212, 0, 0, 0, 4096, 1128, 6, 2, 0, 0, 0, 0, 0, 0, 41, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 976, 0, 0, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 299, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4049, 45, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 3752, 293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8444, 8231, 7747, 8404, 8190, 8187, 8190, 8143] avg=8192.0000 max=8444 max_rank=0 min=7747 min_rank=2 imbalance_ratio=1.030762 max_load_reduction=57092 max_load_reduction_pct=0.871155 moved_to_echo=32710 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 417, 0, 0, 0, 4096, 1135, 12, 3, 0, 0, 0, 0, 0, 0, 39, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3800, 0, 0, 2662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 292, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4057, 51, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3753, 299, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5663, 8231, 10558, 8363, 8189, 8180, 8204, 8148] avg=8192.0000 max=10558 max_rank=2 min=5663 min_rank=0 imbalance_ratio=1.288818 max_load_reduction=54978 max_load_reduction_pct=0.838898 moved_to_echo=32721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 4065, 0, 0, 0, 4095, 1128, 6, 2, 0, 0, 0, 0, 0, 1, 41, 4095, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 94, 0, 1, 2674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4093, 1, 299, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 1, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 1, 4053, 45, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 1, 26, 1, 3751, 293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9298, 8234, 6864, 8399, 8190, 8188, 8195, 8168] avg=8192.0000 max=9298 max_rank=0 min=6864 min_rank=2 imbalance_ratio=1.135010 max_load_reduction=56238 max_load_reduction_pct=0.858124 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2888, 0, 0, 0, 4096, 1128, 6, 2, 0, 0, 0, 0, 0, 0, 41, 3914, 0, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1171, 0, 0, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 30, 299, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4023, 45, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 361, 0, 3752, 293, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8120, 7979, 7942, 8420, 8195, 8213, 8165, 8502] avg=8192.0000 max=8502 max_rank=7 min=7942 min_rank=2 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=33075 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 23680, 0, 0, 0, 32768, 9088, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 0, 0, 128, 512, 128, 30080, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 32768, 0, 128, 32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 2432, 30464, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 384, 32640, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 0, 11392, 0, 128, 21504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 11, 4070, 2, 3824, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3768, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 615, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 4096, 4074, 4096, 4096, 0, 102, 4096, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2420, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 1703, 0, 0, 4010, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4068, 4096, 4096, 0, 22, 4096, 0, 3853, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3814, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 596, 0, 0, 4031, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 0, 0, 0, 1, 4095, 4095, 4094, 4095, 0, 20, 4092, 0, 3684, 0, 0, 0, 0, 2, 0, 3, 4093, 4095, 1, 0, 1, 0, 4095, 1, 4095, 1, 1, 3117, 4090, 1, 1, 1, 0, 4095, 0, 3, 1, 1364, 0, 1, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 3053, 4096, 4096, 0, 28, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4066, 4060, 0, 0, 0, 0, 4096, 0, 0, 0, 1087, 0, 0, 4090, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 4094, 4096, 4096, 0, 18, 4096, 0, 3879, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3892, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 405, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 48, 4096, 0, 3884, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4014, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 377, 0, 0, 3966, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4035, 4096, 4096, 0, 27, 4096, 0, 3769, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3740, 4073, 0, 0, 0, 0, 4096, 0, 0, 0, 759, 0, 0, 4077, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3951, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3770, 325, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 145, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 11, 4070, 2, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3768, 4096, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 615, 0, 0, 4096, 0, 3269, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8191, 8335, 8090, 8192, 8192, 8366, 7980] avg=8192.0000 max=8366 max_rank=6 min=7980 min_rank=7 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3951, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3770, 325, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 145, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 102, 4096, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2420, 4094, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1703, 0, 0, 4010, 0, 3269, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8191, 8318, 8455, 8192, 8192, 7016, 8982] avg=8192.0000 max=8982 max_rank=7 min=7016 min_rank=6 imbalance_ratio=1.096436 max_load_reduction=56554 max_load_reduction_pct=0.862946 moved_to_echo=32382 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3951, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3770, 325, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 145, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 22, 4096, 0, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3814, 4096, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 596, 0, 0, 4031, 0, 3269, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8191, 8309, 8154, 8192, 8192, 8412, 7896] avg=8192.0000 max=8412 max_rank=6 min=7896 min_rank=7 imbalance_ratio=1.026855 max_load_reduction=57124 max_load_reduction_pct=0.871643 moved_to_echo=32692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3951, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3770, 325, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 145, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 48, 4096, 0, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 4095, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 0, 3966, 0, 3269, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8191, 8337, 8211, 8192, 8192, 8611, 7612] avg=8192.0000 max=8611 max_rank=6 min=7612 min_rank=7 imbalance_ratio=1.051147 max_load_reduction=56925 max_load_reduction_pct=0.868607 moved_to_echo=32607 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3951, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3770, 325, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 145, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 27, 4096, 0, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3740, 4073, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 4077, 0, 3269, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8191, 8276, 8075, 8192, 8192, 8315, 8105] avg=8192.0000 max=8315 max_rank=6 min=8075 min_rank=3 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3953, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3774, 332, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 143, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 18, 4096, 0, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 4096, 0, 0, 0, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 0, 4096, 0, 3276, 2, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8202, 8332, 8169, 8191, 8190, 8476, 7781] avg=8192.0000 max=8476 max_rank=6 min=7781 min_rank=7 imbalance_ratio=1.034668 max_load_reduction=57060 max_load_reduction_pct=0.870667 moved_to_echo=32638 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3951, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3770, 325, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 145, 3053, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 28, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4060, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 0, 0, 4090, 0, 3269, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8191, 7294, 8403, 8192, 8192, 8628, 8446] avg=8192.0000 max=8628 max_rank=6 min=7294 min_rank=2 imbalance_ratio=1.053223 max_load_reduction=56908 max_load_reduction_pct=0.868347 moved_to_echo=33458 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 1, 10, 0, 0, 4095, 3950, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3768, 325, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 145, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 20, 4092, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4095, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3117, 4090, 1, 1, 1, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1364, 0, 1, 4095, 1, 3269, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8188, 8335, 7979, 8193, 8194, 7712, 8734] avg=8192.0000 max=8734 max_rank=7 min=7712 min_rank=6 imbalance_ratio=1.066162 max_load_reduction=56802 max_load_reduction_pct=0.866730 moved_to_echo=32833 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 0, 0, 128, 1280, 31616, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 0, 32768, 31616, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1536, 0, 0, 384, 32768, 128, 31104, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 6912, 0, 128, 32512, 128, 26240, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 28928, 32768, 128, 128, 128, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 32768, 30208, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True + [2026-08-30 04:59:28] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 732.5 | throughput per GPU (TFLOP/s/GPU): 455.1 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.037744E+01 | loss scale: 1.0 | grad norm: 16.758 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[16, 13, 713, 0, 170, 0, 1045, 1, 2930, 47, 46, 2601, 1606, 285, 6, 3159, 977, 815, 432, 328, 2926, 1738, 159, 1442, 1, 2679, 18, 847, 356, 10, 229, 1801, 4095, 36, 67, 1552, 3074, 4, 467, 1964, 3694, 323, 522, 512, 6, 955, 1178, 2864, 86, 3830, 644, 9, 0, 7, 915, 563, 0, 3597, 3173, 1190, 271, 1728, 812, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[7, 48, 931, 1, 291, 4, 1155, 7, 2791, 23, 103, 2114, 1293, 639, 0, 3365, 1208, 1366, 740, 801, 2025, 1308, 312, 1659, 11, 2634, 164, 1040, 442, 17, 178, 1856, 4020, 10, 67, 1386, 2407, 1, 1012, 1908, 3874, 585, 737, 510, 1, 1230, 1203, 2424, 178, 3465, 416, 7, 4, 5, 1012, 624, 0, 3022, 2986, 761, 539, 1341, 1233, 35] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[11, 40, 1084, 0, 175, 0, 1569, 4, 2655, 110, 134, 2648, 1221, 623, 2, 2898, 696, 1616, 471, 403, 2384, 1085, 237, 1458, 6, 2330, 26, 1572, 654, 12, 152, 1576, 4081, 5, 91, 2050, 2603, 2, 467, 2844, 3419, 431, 729, 858, 0, 703, 1103, 1978, 152, 3835, 562, 0, 1, 7, 811, 855, 2, 3371, 2771, 1126, 466, 1337, 1021, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3, 2, 715, 1, 227, 0, 1506, 0, 2523, 72, 48, 2331, 721, 1068, 0, 3583, 1275, 960, 243, 200, 2559, 1733, 196, 1880, 0, 2933, 9, 1312, 400, 0, 158, 1590, 4096, 23, 73, 1414, 2776, 7, 517, 2438, 3600, 244, 722, 384, 0, 1121, 979, 2732, 135, 3878, 746, 0, 0, 1, 916, 415, 0, 3589, 3142, 914, 139, 1467, 808, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 10, 715, 5, 152, 0, 1099, 1, 2811, 136, 69, 2571, 867, 726, 0, 3655, 1211, 1638, 473, 335, 2572, 1447, 158, 1209, 0, 2657, 15, 1060, 480, 8, 133, 1861, 4073, 25, 38, 1726, 2857, 9, 368, 2316, 3359, 342, 398, 423, 0, 1284, 1300, 1659, 102, 3847, 488, 7, 5, 7, 1493, 623, 0, 3484, 3163, 1269, 511, 1203, 1068, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[19, 9, 505, 2, 147, 0, 1399, 0, 2926, 77, 136, 2726, 901, 420, 1, 2829, 1113, 1053, 367, 206, 2629, 1341, 167, 1240, 1, 2669, 57, 1405, 759, 15, 202, 1702, 4083, 25, 54, 2567, 2635, 0, 465, 2406, 3884, 272, 657, 490, 4, 1081, 1146, 2701, 176, 3798, 668, 0, 4, 2, 662, 501, 0, 3405, 3117, 754, 430, 1213, 1297, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[23, 16, 979, 1, 190, 2, 1143, 0, 2924, 31, 82, 2002, 1091, 394, 3, 3249, 1251, 935, 672, 270, 2513, 1035, 218, 1617, 0, 2407, 10, 1314, 812, 23, 173, 1667, 3981, 10, 66, 1623, 3222, 1, 549, 2552, 3812, 415, 555, 570, 7, 1067, 1192, 2616, 157, 3751, 463, 1, 1, 12, 1174, 473, 0, 3379, 3092, 890, 538, 1151, 1162, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[12, 11, 567, 0, 128, 0, 1504, 0, 3132, 74, 87, 2261, 1420, 758, 0, 3405, 764, 872, 336, 492, 2649, 1663, 210, 1616, 0, 2607, 18, 1062, 531, 2, 278, 1636, 4082, 12, 58, 2206, 2710, 0, 543, 2173, 3533, 218, 761, 489, 0, 1093, 940, 1487, 146, 3866, 921, 0, 0, 5, 1021, 559, 0, 3400, 3395, 927, 540, 1218, 1135, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 13, 713, 0, 170, 0, 1045, 1, 3458, 2159, 241, 0, 0, 0, 0, 0, 2930, 47, 46, 2601, 1606, 285, 6, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 977, 815, 432, 328, 2296, 1738, 159, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2679, 18, 847, 356, 10, 229, 1801, 1340, 630, 0, 0, 0, 0, 0, 0, 637, 36, 67, 1552, 3074, 4, 467, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 2196, 323, 522, 512, 6, 955, 1178, 2864, 0, 0, 0, 0, 0, 0, 0, 0, 86, 3830, 644, 9, 0, 7, 915, 563, 2016, 158, 0, 0, 0, 0, 0, 0, 0, 1340, 3173, 1190, 271, 1728, 812, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7816, 8521, 8187, 7911, 7801, 8556, 8228, 8516] avg=8192.0000 max=8556 max_rank=5 min=7801 min_rank=4 imbalance_ratio=1.044434 max_load_reduction=56980 max_load_reduction_pct=0.869446 moved_to_echo=33101 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 48, 931, 1, 291, 4, 1155, 7, 3394, 2300, 202, 0, 0, 0, 0, 0, 2791, 23, 103, 2114, 1293, 639, 0, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 1208, 1366, 740, 801, 1589, 1308, 312, 1659, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2634, 164, 1040, 442, 17, 178, 1856, 1406, 436, 0, 0, 0, 0, 0, 0, 626, 10, 67, 1386, 2407, 1, 1012, 1908, 0, 0, 0, 0, 0, 0, 0, 0, 2303, 585, 737, 510, 1, 1230, 1203, 2424, 0, 0, 0, 0, 0, 0, 0, 0, 178, 3465, 416, 7, 4, 5, 1012, 624, 1694, 165, 0, 0, 0, 0, 0, 0, 0, 1126, 2986, 761, 539, 1341, 1233, 35, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8340, 8028, 8983, 8184, 7417, 8993, 7570, 8021] avg=8192.0000 max=8993 max_rank=5 min=7417 min_rank=4 imbalance_ratio=1.097778 max_load_reduction=56543 max_load_reduction_pct=0.862778 moved_to_echo=32001 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 2, 715, 1, 227, 0, 1506, 0, 3459, 2449, 240, 0, 0, 0, 0, 0, 2523, 72, 48, 2331, 721, 1068, 0, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 1275, 960, 243, 200, 2008, 1733, 196, 1880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2933, 9, 1312, 400, 0, 158, 1590, 1306, 551, 0, 0, 0, 0, 0, 0, 637, 23, 73, 1414, 2776, 7, 517, 2438, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 244, 722, 384, 0, 1121, 979, 2732, 0, 0, 0, 0, 0, 0, 0, 0, 135, 3878, 746, 0, 0, 1, 916, 415, 2012, 154, 0, 0, 0, 0, 0, 0, 0, 1337, 3142, 914, 139, 1467, 808, 12, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8602, 7897, 8495, 8259, 7885, 8322, 8257, 7819] avg=8192.0000 max=8602 max_rank=0 min=7819 min_rank=7 imbalance_ratio=1.050049 max_load_reduction=56934 max_load_reduction_pct=0.868744 moved_to_echo=32283 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 40, 1084, 0, 175, 0, 1569, 4, 3446, 1981, 226, 0, 0, 0, 0, 0, 2655, 110, 134, 2648, 1221, 623, 2, 917, 0, 0, 0, 0, 0, 0, 0, 0, 696, 1616, 471, 403, 1870, 1085, 237, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2330, 26, 1572, 654, 12, 152, 1576, 1241, 514, 0, 0, 0, 0, 0, 0, 635, 5, 91, 2050, 2603, 2, 467, 2844, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 431, 729, 858, 0, 703, 1103, 1978, 0, 0, 0, 0, 0, 0, 0, 0, 152, 3835, 562, 0, 1, 7, 811, 855, 1889, 146, 0, 0, 0, 0, 0, 0, 2, 1256, 2771, 1126, 466, 1337, 1021, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8536, 8310, 7836, 8083, 8697, 7834, 8258, 7982] avg=8192.0000 max=8697 max_rank=4 min=7834 min_rank=5 imbalance_ratio=1.061646 max_load_reduction=56839 max_load_reduction_pct=0.867294 moved_to_echo=32771 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[19, 9, 505, 2, 147, 0, 1399, 0, 3448, 1934, 228, 0, 0, 0, 0, 0, 2926, 77, 136, 2726, 901, 420, 1, 895, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 1053, 367, 206, 2063, 1341, 167, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2669, 57, 1405, 759, 15, 202, 1702, 1409, 566, 0, 0, 0, 0, 0, 0, 635, 25, 54, 2567, 2635, 0, 465, 2406, 0, 0, 0, 0, 0, 0, 0, 0, 2309, 272, 657, 490, 4, 1081, 1146, 2701, 0, 0, 0, 0, 0, 0, 0, 0, 176, 3798, 668, 0, 4, 2, 662, 501, 1908, 166, 0, 0, 0, 0, 0, 0, 0, 1269, 3117, 754, 430, 1213, 1297, 16, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7691, 8082, 7550, 8785, 8787, 8660, 7885, 8096] avg=8192.0000 max=8787 max_rank=4 min=7550 min_rank=2 imbalance_ratio=1.072632 max_load_reduction=56749 max_load_reduction_pct=0.865921 moved_to_echo=33428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 715, 5, 152, 0, 1099, 1, 3442, 2502, 238, 0, 0, 0, 0, 0, 2811, 136, 69, 2571, 867, 726, 0, 1153, 0, 0, 0, 0, 0, 0, 0, 0, 1211, 1638, 473, 335, 2013, 1447, 158, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2657, 15, 1060, 480, 8, 133, 1861, 1224, 559, 0, 0, 0, 0, 0, 0, 631, 25, 38, 1726, 2857, 9, 368, 2316, 0, 0, 0, 0, 0, 0, 0, 0, 1989, 342, 398, 423, 0, 1284, 1300, 1659, 0, 0, 0, 0, 0, 0, 0, 0, 102, 3847, 488, 7, 5, 7, 1493, 623, 1956, 146, 0, 0, 0, 0, 0, 0, 0, 1290, 3163, 1269, 511, 1203, 1068, 15, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8333, 8484, 7997, 7970, 7395, 8674, 8519] avg=8192.0000 max=8674 max_rank=6 min=7395 min_rank=5 imbalance_ratio=1.058838 max_load_reduction=56862 max_load_reduction_pct=0.867645 moved_to_echo=32558 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[23, 16, 979, 1, 190, 2, 1143, 0, 3362, 2221, 226, 0, 0, 0, 0, 0, 2924, 31, 82, 2002, 1091, 394, 3, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 935, 672, 270, 1972, 1035, 218, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2407, 10, 1314, 812, 23, 173, 1667, 1383, 541, 0, 0, 0, 0, 0, 0, 619, 10, 66, 1623, 3222, 1, 549, 2552, 0, 0, 0, 0, 0, 0, 0, 0, 2266, 415, 555, 570, 7, 1067, 1192, 2616, 0, 0, 0, 0, 0, 0, 0, 0, 157, 3751, 463, 1, 1, 12, 1174, 473, 1894, 163, 0, 0, 0, 0, 0, 0, 0, 1259, 3092, 890, 538, 1151, 1162, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 7555, 7970, 8330, 8642, 8688, 8089, 8099] avg=8192.0000 max=8688 max_rank=5 min=7555 min_rank=1 imbalance_ratio=1.060547 max_load_reduction=56848 max_load_reduction_pct=0.867432 moved_to_echo=33518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[12, 11, 567, 0, 128, 0, 1504, 0, 3447, 2327, 228, 0, 0, 0, 0, 0, 3132, 74, 87, 2261, 1420, 758, 0, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 764, 872, 336, 492, 2078, 1663, 210, 1616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 18, 1062, 531, 2, 278, 1636, 1282, 571, 0, 0, 0, 0, 0, 0, 635, 12, 58, 2206, 2710, 0, 543, 2173, 0, 0, 0, 0, 0, 0, 0, 0, 2100, 218, 761, 489, 0, 1093, 940, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 146, 3866, 921, 0, 0, 5, 1021, 559, 1906, 151, 0, 0, 0, 0, 0, 0, 0, 1266, 3395, 927, 540, 1218, 1135, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8810, 8031, 7987, 8337, 7088, 8575, 8484] avg=8192.0000 max=8810 max_rank=1 min=7088 min_rank=5 imbalance_ratio=1.075439 max_load_reduction=56726 max_load_reduction_pct=0.865570 moved_to_echo=32484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8576, 9344, 3840, 3072, 16000, 11392, 1664, 12160, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 20992, 384, 9728, 4480, 128, 1536, 13696, 10624, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[17408, 2944, 5120, 4352, 128, 8576, 9088, 18560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 256, 6272, 128, 1536, 128, 10496, 128, 27520, 17920, 1920, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[22784, 640, 768, 19328, 9216, 4992, 128, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1152, 30336, 4992, 128, 128, 128, 8064, 4736, 15360, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 10240, 24960, 7936, 3456, 10752, 8576, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[5120, 256, 640, 14592, 22400, 128, 4480, 18688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 5, 1, 0, 4095, 0, 652, 11, 4076, 0, 1, 3925, 2, 17, 0, 0, 4092, 0, 0, 0, 0, 0, 1, 4082, 6, 4092, 2, 0, 1, 4095, 3573, 0, 3, 0, 44, 4094, 4094, 6, 0, 0, 4089, 4083, 2, 1, 0, 0, 4096, 4095, 0, 1, 0, 0, 0, 3, 0, 0, 4096, 2, 4093, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[15, 0, 1, 0, 0, 0, 0, 4092, 0, 1516, 17, 4063, 0, 0, 3101, 0, 0, 2, 0, 4086, 2, 2, 4, 1, 31, 0, 4095, 2, 4093, 115, 15, 1, 4090, 3473, 4, 0, 0, 4, 4087, 4093, 0, 1, 0, 4081, 4090, 3, 3, 0, 0, 4071, 4072, 10, 0, 3, 4, 3, 1, 0, 0, 4091, 3, 4093, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8, 4096, 0, 2652, 0, 4083, 0, 0, 1545, 5, 15, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 4096, 46, 0, 2, 4092, 3955, 0, 0, 7, 0, 4087, 4096, 2, 0, 0, 4096, 4078, 0, 0, 3, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 4096, 0, 3520, 2, 4072, 0, 2, 1912, 0, 0, 0, 0, 4096, 4, 0, 0, 8, 25, 0, 4094, 0, 4094, 169, 0, 0, 4094, 2533, 25, 0, 1, 38, 4096, 4092, 0, 0, 0, 4090, 4094, 0, 0, 0, 0, 4096, 4094, 7, 0, 0, 3, 0, 1, 0, 0, 4087, 0, 4090, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4095, 0, 1564, 0, 4095, 0, 0, 3774, 0, 1, 0, 0, 4077, 3, 18, 0, 0, 6, 17, 4094, 2, 4095, 10, 0, 1, 4073, 2852, 1, 1, 0, 7, 4089, 4095, 4, 0, 0, 4094, 4094, 1, 0, 4, 0, 4096, 4079, 0, 0, 1, 0, 1, 0, 1, 0, 4095, 0, 4094, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 2486, 0, 4090, 0, 5, 3406, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 13, 0, 4096, 0, 4096, 56, 0, 0, 4096, 2229, 4, 0, 0, 1, 4096, 4096, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 2559, 0, 4093, 1, 15, 3800, 0, 4, 0, 0, 4094, 0, 0, 0, 1, 4, 0, 4096, 0, 4096, 10, 0, 0, 4095, 1811, 5, 0, 0, 0, 4089, 4095, 0, 0, 0, 4095, 4096, 0, 0, 1, 0, 4096, 4091, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 3, 1, 6, 4096, 0, 1817, 0, 4090, 6, 8, 2560, 0, 20, 2, 0, 4090, 0, 7, 2, 5, 28, 0, 4090, 0, 4092, 279, 0, 9, 4096, 3519, 0, 0, 2, 1, 4074, 4095, 0, 2, 0, 4093, 4081, 0, 0, 0, 1, 4085, 4092, 0, 0, 0, 1, 1, 0, 6, 0, 4076, 1, 4095, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2652, 0, 3085, 0, 0, 1545, 5, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 3000, 998, 83, 4, 0, 0, 0, 0, 7, 0, 4096, 0, 3996, 46, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 3955, 0, 0, 7, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4078, 0, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 7287, 8196, 8147, 9141, 8186, 8200, 8183] avg=8192.0000 max=9141 max_rank=4 min=7287 min_rank=1 imbalance_ratio=1.115845 max_load_reduction=56395 max_load_reduction_pct=0.860519 moved_to_echo=33710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 3520, 2, 3076, 0, 2, 1912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 8, 3001, 996, 83, 3, 0, 0, 0, 0, 25, 0, 4094, 0, 3995, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 2533, 25, 0, 1, 38, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4094, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 7, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4087, 0, 4090, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8512, 8191, 8283, 7786, 8190, 8206, 8182] avg=8192.0000 max=8512 max_rank=1 min=7786 min_rank=4 imbalance_ratio=1.039062 max_load_reduction=57024 max_load_reduction_pct=0.870117 moved_to_echo=32364 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 3, 1, 6, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1817, 0, 3090, 6, 8, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 0, 4090, 0, 7, 2, 5, 3003, 1000, 83, 3, 0, 0, 0, 0, 28, 0, 4090, 0, 3993, 279, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 3519, 0, 0, 2, 1, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 4081, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 4092, 0, 0, 0, 1, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4076, 1, 4095, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 7481, 8215, 8399, 8689, 8182, 8186, 8183] avg=8192.0000 max=8689 max_rank=4 min=7481 min_rank=1 imbalance_ratio=1.060669 max_load_reduction=56847 max_load_reduction_pct=0.867416 moved_to_echo=33240 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 0, 3094, 0, 0, 3774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4077, 3, 18, 0, 0, 2986, 1001, 83, 4, 0, 0, 0, 0, 6, 17, 4094, 2, 3995, 10, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 2852, 1, 1, 0, 7, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4094, 4094, 1, 0, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 1, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4094, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8432, 8173, 8125, 8037, 8204, 8183, 8195] avg=8192.0000 max=8432 max_rank=1 min=8037 min_rank=4 imbalance_ratio=1.029297 max_load_reduction=57104 max_load_reduction_pct=0.871338 moved_to_echo=32619 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 0, 3090, 0, 5, 3406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 3003, 1000, 83, 4, 0, 0, 0, 0, 13, 0, 4096, 0, 3996, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 2229, 4, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8987, 8187, 8161, 7423, 8200, 8194, 8196] avg=8192.0000 max=8987 max_rank=1 min=7423 min_rank=4 imbalance_ratio=1.097046 max_load_reduction=56549 max_load_reduction_pct=0.862869 moved_to_echo=32013 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 2559, 0, 3092, 1, 15, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4094, 0, 0, 0, 1, 3002, 1001, 83, 4, 0, 0, 0, 0, 4, 0, 4096, 0, 3996, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 1811, 5, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 9467, 8189, 8106, 6998, 8199, 8193, 8197] avg=8192.0000 max=9467 max_rank=1 min=6998 min_rank=4 imbalance_ratio=1.155640 max_load_reduction=56069 max_load_reduction_pct=0.855545 moved_to_echo=31587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[15, 0, 1, 0, 0, 0, 0, 4092, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1516, 17, 3066, 0, 0, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4086, 2, 2, 4, 1, 3003, 997, 90, 3, 0, 0, 0, 0, 31, 0, 4095, 2, 3971, 115, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 3473, 4, 0, 0, 4, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4081, 4090, 3, 3, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4072, 10, 0, 3, 4, 3, 13, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 3, 4093, 2, 0, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 7700, 8190, 8230, 8655, 8188, 8176, 8199] avg=8192.0000 max=8655 max_rank=4 min=7700 min_rank=1 imbalance_ratio=1.056519 max_load_reduction=56881 max_load_reduction_pct=0.867935 moved_to_echo=33218 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 5, 1, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 652, 11, 3079, 0, 1, 3925, 2, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4092, 0, 0, 0, 0, 3002, 997, 83, 4, 0, 0, 0, 0, 0, 1, 4082, 6, 3993, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 3573, 0, 3, 0, 44, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4089, 4083, 2, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 2, 4093, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7670, 8195, 8085, 8807, 8187, 8198, 8201] avg=8192.0000 max=8807 max_rank=4 min=7670 min_rank=1 imbalance_ratio=1.075073 max_load_reduction=56729 max_load_reduction_pct=0.865616 moved_to_echo=33393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8832, 24064, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 32000, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 16768, 128, 24704, 128, 128, 24064, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 128, 24064, 8064, 768, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 5, 4028, 4091, 0, 0, 18, 4021, 829, 4095, 0, 0, 0, 0, 0, 7, 0, 6, 0, 0, 0, 0, 4094, 0, 4090, 0, 4, 0, 0, 1, 0, 0, 54, 0, 4039, 3, 0, 4096, 4, 0, 0, 0, 4091, 4094, 0, 3517, 0, 2651, 0, 0, 4093, 0, 0, 4096, 0, 0, 4095, 0, 0, 1, 4091, 1, 1321, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4012, 4096, 0, 0, 0, 4093, 104, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 2, 11, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3852, 0, 3887, 0, 0, 4090, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 445, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 3514, 4093, 2, 5, 16, 4088, 1744, 4095, 0, 0, 0, 1, 7, 0, 0, 0, 0, 1, 5, 0, 4090, 2, 4093, 0, 0, 1, 0, 0, 0, 2, 29, 0, 3906, 1, 0, 4087, 0, 1, 0, 2, 4084, 4091, 0, 4045, 1, 2539, 0, 13, 4080, 0, 0, 4095, 1, 9, 4093, 3, 1, 5, 4095, 0, 594, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[1, 1, 3001, 4094, 5, 1, 9, 4095, 210, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4095, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 3, 0, 3944, 7, 2, 4094, 0, 1, 2, 0, 4095, 4095, 1, 4063, 0, 2957, 2, 0, 4096, 1, 1, 4090, 0, 0, 4095, 1, 4, 0, 4096, 0, 2182, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 3950, 4086, 1, 1, 30, 4065, 424, 4094, 3, 4, 0, 5, 0, 1, 0, 0, 0, 3, 0, 1, 4092, 0, 4094, 6, 0, 0, 0, 0, 1, 0, 164, 2, 4027, 16, 0, 4056, 1, 3, 1, 6, 4081, 4090, 0, 4086, 2, 1333, 0, 3, 4086, 0, 0, 4092, 2, 0, 4093, 2, 6, 0, 4092, 3, 2424, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3676, 4095, 0, 0, 1, 4094, 261, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 18, 34, 4096, 0, 0, 0, 6, 4093, 4096, 0, 4089, 0, 2709, 0, 0, 4084, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 0, 1570, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3689, 4091, 0, 0, 33, 4095, 1184, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 1, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 6, 14, 4096, 0, 0, 0, 1, 4095, 4088, 0, 4090, 0, 3244, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 475, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4076, 4096, 0, 0, 1, 4078, 137, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4091, 0, 5, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4093, 0, 3389, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 605, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4028, 354, 0, 0, 18, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 829, 4095, 0, 0, 0, 0, 0, 7, 2362, 1118, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4094, 0, 3735, 356, 0, 0, 0, 0, 0, 0, 4090, 0, 4, 0, 0, 1, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 54, 0, 4039, 3, 0, 4096, 4, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1373, 0, 0, 3517, 0, 2651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2892, 0, 0, 1, 4091, 1, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8426, 8411, 8191, 8191, 8278, 7541, 8192, 8306] avg=8192.0000 max=8426 max_rank=0 min=7541 min_rank=5 imbalance_ratio=1.028564 max_load_reduction=57110 max_load_reduction_pct=0.871429 moved_to_echo=32317 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3514, 354, 2, 5, 16, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 1744, 4095, 0, 0, 0, 1, 7, 0, 2358, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 4090, 2, 3737, 356, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 2, 4091, 2, 0, 0, 0, 0, 0, 0, 29, 0, 3906, 1, 0, 4087, 0, 1, 82, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1370, 0, 0, 4045, 1, 2539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4080, 0, 0, 4095, 1, 9, 3, 0, 0, 0, 0, 0, 0, 0, 2891, 3, 1, 5, 4095, 0, 594, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7980, 9322, 8191, 8189, 8106, 7957, 8201, 7590] avg=8192.0000 max=9322 max_rank=1 min=7590 min_rank=7 imbalance_ratio=1.137939 max_load_reduction=56214 max_load_reduction_pct=0.857758 moved_to_echo=31854 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4012, 354, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 104, 4096, 0, 0, 0, 0, 0, 0, 2365, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3740, 357, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 2, 11, 4096, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1374, 0, 0, 3852, 0, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2893, 0, 0, 0, 4096, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8459, 7683, 8194, 8194, 8270, 9113, 8189, 7434] avg=8192.0000 max=9113 max_rank=5 min=7434 min_rank=7 imbalance_ratio=1.112427 max_load_reduction=56423 max_load_reduction_pct=0.860947 moved_to_echo=33006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3676, 354, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 261, 4096, 0, 0, 0, 0, 0, 0, 2364, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3739, 357, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 18, 34, 4096, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1372, 0, 0, 4089, 0, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 2893, 0, 0, 0, 4096, 0, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8125, 7839, 8192, 8194, 8267, 8176, 8184, 8559] avg=8192.0000 max=8559 max_rank=7 min=7839 min_rank=1 imbalance_ratio=1.044800 max_load_reduction=56977 max_load_reduction_pct=0.869400 moved_to_echo=33186 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 3001, 354, 5, 1, 9, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 210, 4095, 0, 0, 0, 0, 0, 0, 2365, 1118, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4095, 0, 3738, 357, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 3, 0, 3944, 7, 2, 4094, 0, 1, 82, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1373, 0, 1, 4063, 0, 2957, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 1, 1, 4090, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2892, 1, 4, 0, 4096, 0, 2182, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7467, 7788, 8192, 8192, 8133, 8396, 8193, 9175] avg=8192.0000 max=9175 max_rank=7 min=7467 min_rank=0 imbalance_ratio=1.119995 max_load_reduction=56361 max_load_reduction_pct=0.860001 moved_to_echo=33897 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4076, 354, 0, 0, 1, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 137, 4096, 0, 0, 0, 0, 0, 0, 2365, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3740, 357, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 5, 0, 4091, 0, 5, 4096, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1374, 0, 0, 4093, 0, 3389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2893, 0, 0, 0, 4096, 0, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8509, 7716, 8193, 8194, 8279, 8856, 8195, 7594] avg=8192.0000 max=8856 max_rank=5 min=7594 min_rank=7 imbalance_ratio=1.081055 max_load_reduction=56680 max_load_reduction_pct=0.864868 moved_to_echo=32924 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3950, 350, 1, 1, 30, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 424, 4094, 3, 4, 0, 5, 0, 1, 2361, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 4092, 0, 3733, 359, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 0, 0, 1, 0, 4090, 3, 0, 0, 0, 0, 0, 0, 164, 2, 4027, 16, 0, 4056, 1, 3, 83, 0, 0, 0, 0, 0, 0, 0, 1, 6, 1361, 0, 0, 4086, 2, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4086, 0, 0, 4092, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2884, 2, 6, 0, 4092, 3, 2424, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8398, 8014, 8188, 8194, 8352, 6789, 8187, 9414] avg=8192.0000 max=9414 max_rank=7 min=6789 min_rank=5 imbalance_ratio=1.149170 max_load_reduction=56122 max_load_reduction_pct=0.856354 moved_to_echo=32742 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3689, 354, 0, 0, 33, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 4096, 0, 0, 0, 0, 0, 0, 2365, 1118, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 1, 0, 4096, 0, 3735, 357, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 6, 14, 4096, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1373, 0, 0, 4090, 0, 3244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2893, 0, 0, 0, 4096, 0, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8763, 8195, 8188, 7851, 8708, 8195, 7464] avg=8192.0000 max=8763 max_rank=1 min=7464 min_rank=7 imbalance_ratio=1.069702 max_load_reduction=56773 max_load_reduction_pct=0.866287 moved_to_echo=32218 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 29952, 2944, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 11008, 0, 128, 31872, 128, 22784, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[23168, 128, 128, 128, 32768, 128, 9728, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 128, 29952, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 128, 31872, 128, 128, 32768, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4992, 32768, 128, 128, 0, 128, 128, 128, 18944, 8960, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 0, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4066, 1, 0, 0, 3420, 14, 0, 0, 4059, 2, 4074, 0, 0, 4085, 0, 0, 2, 0, 9, 0, 4075, 0, 4094, 5, 3, 0, 0, 0, 0, 4092, 0, 0, 3456, 4096, 58, 4092, 0, 0, 15, 4095, 4096, 0, 1342, 2, 4096, 0, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 3, 1, 0, 4057, 0, 1, 0, 4094, 3, 0, 3, 1, 0, 4094, 0, 0, 1, 3583, 0, 1, 1, 4066, 3, 4053, 3, 2, 4094, 0, 4, 0, 0, 8, 0, 4096, 0, 4095, 7, 0, 1, 0, 0, 0, 4087, 4, 1, 2346, 4094, 165, 4088, 2, 4, 0, 4094, 4093, 0, 2135, 49, 4094, 2, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 2213, 12, 0, 0, 3993, 0, 4093, 0, 0, 4072, 0, 0, 0, 0, 12, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3474, 4096, 150, 4096, 0, 0, 13, 4096, 4096, 0, 2510, 1, 4096, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4021, 1, 0, 0, 3982, 0, 4092, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3502, 4096, 238, 4096, 0, 0, 2, 4096, 4096, 0, 397, 161, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4087, 0, 1, 0, 4096, 6, 0, 0, 0, 0, 4052, 1, 1, 0, 4008, 3, 1, 0, 4000, 1, 4094, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4075, 0, 4095, 1, 0, 0, 0, 0, 0, 4093, 0, 1, 2859, 4095, 97, 4095, 1, 0, 49, 4094, 4095, 0, 1300, 28, 4095, 0, 0, 1, 0, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2623, 19, 0, 0, 4078, 0, 4096, 0, 0, 4017, 0, 0, 0, 0, 9, 0, 4087, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3625, 4096, 51, 4096, 0, 0, 32, 4096, 4096, 0, 1942, 0, 4096, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 4091, 0, 3, 0, 4092, 1, 0, 6, 3, 0, 3990, 6, 0, 0, 3809, 10, 3, 0, 4060, 0, 4088, 4, 1, 4090, 0, 5, 4, 4, 0, 3, 4094, 0, 4093, 1, 1, 0, 0, 9, 5, 4081, 0, 2, 3985, 4090, 1, 4088, 3, 0, 0, 4092, 4092, 0, 371, 110, 4088, 5, 0, 43, 4, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3995, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4043, 0, 0, 0, 3200, 0, 0, 0, 3998, 0, 4037, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3954, 4096, 768, 4096, 0, 0, 0, 4096, 4096, 0, 578, 1, 4096, 0, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4066, 1, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 14, 0, 0, 4059, 2, 656, 137, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4085, 0, 0, 2, 0, 30, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4075, 0, 4094, 5, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3456, 699, 0, 0, 0, 0, 0, 0, 0, 0, 58, 4092, 0, 0, 15, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1342, 2, 4096, 0, 3, 0, 0, 0, 2722, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8188, 8288, 8191, 8187, 8247, 8059, 8165] avg=8192.0000 max=8288 max_rank=2 min=8059 min_rank=6 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32658 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 1, 0, 4057, 0, 1, 0, 4093, 19, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 3, 1, 0, 4094, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3583, 0, 1, 1, 4066, 3, 656, 136, 0, 0, 0, 0, 0, 0, 4053, 3, 2, 4094, 0, 4, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 4095, 7, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4, 1, 2346, 699, 0, 0, 0, 0, 0, 0, 0, 0, 165, 4088, 2, 4, 0, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2135, 49, 4094, 2, 0, 0, 0, 3, 2720, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8228, 8447, 8186, 8208, 7137, 8153, 9003] avg=8192.0000 max=9003 max_rank=7 min=7137 min_rank=5 imbalance_ratio=1.098999 max_load_reduction=56533 max_load_reduction_pct=0.862625 moved_to_echo=32501 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 1, 0, 0, 3982, 0, 656, 137, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4088, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3502, 699, 0, 0, 0, 0, 0, 0, 0, 0, 238, 4096, 0, 0, 2, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 161, 4096, 0, 0, 0, 0, 0, 2722, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8224, 8797, 8210, 8190, 8297, 8231, 7376] avg=8192.0000 max=8797 max_rank=2 min=7376 min_rank=7 imbalance_ratio=1.073853 max_load_reduction=56739 max_load_reduction_pct=0.865768 moved_to_echo=32094 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4047, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2213, 12, 0, 0, 3993, 0, 656, 137, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4072, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4085, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3474, 699, 0, 0, 0, 0, 0, 0, 0, 0, 150, 4096, 0, 0, 13, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2510, 1, 4096, 0, 0, 1, 0, 0, 2722, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8176, 7011, 8195, 8194, 8269, 8154, 9330] avg=8192.0000 max=9330 max_rank=7 min=7011 min_rank=2 imbalance_ratio=1.138916 max_load_reduction=56206 max_load_reduction_pct=0.857635 moved_to_echo=33947 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4087, 0, 1, 0, 4095, 19, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4052, 1, 33, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4008, 3, 1, 0, 4000, 1, 656, 136, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 1, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4095, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 2859, 699, 0, 0, 0, 0, 0, 0, 0, 0, 97, 4095, 1, 0, 49, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1300, 28, 4095, 0, 0, 1, 0, 13, 2721, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8188, 8806, 8221, 8172, 7652, 8136, 8158] avg=8192.0000 max=8806 max_rank=2 min=7652 min_rank=5 imbalance_ratio=1.074951 max_load_reduction=56730 max_load_reduction_pct=0.865631 moved_to_echo=32118 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3995, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4043, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3200, 0, 0, 0, 3998, 0, 656, 137, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 4095, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3954, 699, 0, 0, 0, 0, 0, 0, 0, 0, 768, 4096, 0, 0, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 1, 4096, 0, 0, 2, 0, 0, 2722, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8173, 7991, 8162, 8193, 8749, 8759, 7399] avg=8192.0000 max=8759 max_rank=6 min=7399 min_rank=7 imbalance_ratio=1.069214 max_load_reduction=56777 max_load_reduction_pct=0.866348 moved_to_echo=33100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 4091, 0, 3, 0, 4092, 24, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 6, 3, 0, 3990, 6, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3809, 10, 3, 0, 4060, 0, 662, 139, 0, 0, 0, 0, 0, 0, 4088, 4, 1, 4090, 0, 5, 4, 4, 32, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 4093, 1, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 9, 5, 4081, 0, 2, 3985, 683, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 3, 0, 0, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 110, 4088, 5, 0, 43, 4, 2, 2721, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8134, 8683, 8228, 8199, 8765, 7970, 7344] avg=8192.0000 max=8765 max_rank=5 min=7344 min_rank=7 imbalance_ratio=1.069946 max_load_reduction=56771 max_load_reduction_pct=0.866257 moved_to_echo=32278 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2623, 19, 0, 0, 4078, 0, 656, 137, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4017, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4087, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3625, 699, 0, 0, 0, 0, 0, 0, 0, 0, 51, 4096, 0, 0, 32, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1942, 0, 4096, 0, 0, 1, 0, 0, 2722, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8225, 7513, 8143, 8193, 8420, 8074, 8761] avg=8192.0000 max=8761 max_rank=7 min=7513 min_rank=2 imbalance_ratio=1.069458 max_load_reduction=56775 max_load_reduction_pct=0.866318 moved_to_echo=33448 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 32640, 0, 128, 0, 32768, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 27264, 5632, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 26880, 128, 128, 128, 32256, 128, 5376, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 32640, 0, 128, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[10624, 384, 32768, 128, 128, 128, 128, 128, 21888, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1536, 32768, 128, 128, 128, 31232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 32512, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 2974, 4096, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 1, 0, 4088, 0, 0, 4037, 0, 6, 0, 0, 4092, 0, 0, 4096, 0, 0, 4095, 4096, 4096, 0, 0, 4096, 0, 663, 4094, 171, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 1, 0, 268, 0, 81, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3401, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 3, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 227, 4096, 21, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 329, 0, 119, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[8, 0, 1657, 4092, 0, 0, 2, 2, 4094, 0, 4090, 0, 0, 2, 1, 2, 0, 4091, 1, 4, 4095, 0, 150, 1, 0, 4095, 0, 4, 4092, 1, 4, 4092, 4092, 4094, 0, 0, 4093, 3, 886, 4093, 21, 3, 0, 2, 4092, 2, 0, 3, 2, 0, 1, 0, 4094, 0, 0, 0, 1260, 2, 121, 4, 4083, 1, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2967, 4096, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 3968, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 2, 0, 4096, 0, 14, 4095, 626, 0, 0, 0, 4096, 0, 0, 0, 0, 112, 0, 0, 4096, 0, 0, 0, 510, 0, 4, 0, 4096, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 2881, 4096, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 118, 4093, 54, 0, 0, 0, 4096, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 0, 0, 1039, 0, 4, 0, 4096, 0, 17, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3782, 4095, 0, 0, 1, 1, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 4083, 1, 0, 0, 32, 4092, 0, 1, 4094, 0, 1, 4095, 4095, 4095, 2, 0, 4095, 1, 16, 4064, 41, 1, 0, 1, 4095, 1, 0, 0, 0, 50, 0, 1, 4095, 1, 0, 1, 140, 0, 84, 0, 4096, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3670, 4092, 2, 4, 4, 4, 4093, 7, 4088, 0, 0, 4, 0, 0, 0, 4069, 3, 8, 4012, 0, 6, 2, 5, 4092, 0, 0, 4091, 4, 0, 4089, 4091, 4091, 0, 0, 4092, 0, 71, 4092, 103, 4, 0, 3, 4090, 3, 0, 2, 1, 37, 4, 0, 4089, 0, 0, 5, 198, 0, 84, 14, 4091, 3, 14, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3652, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4, 4096, 14, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 428, 0, 3, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2974, 4096, 0, 0, 0, 0, 749, 221, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4037, 0, 6, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 0, 0, 0, 663, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 4096, 0, 0, 0, 3935, 27, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 1, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 268, 0, 81, 0, 4096, 0, 0, 0, 3347, 161, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8041, 8191, 8154, 8185, 8610, 8229, 8173, 7953] avg=8192.0000 max=8610 max_rank=4 min=7953 min_rank=7 imbalance_ratio=1.051025 max_load_reduction=56926 max_load_reduction_pct=0.868622 moved_to_echo=32965 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 1657, 4092, 0, 0, 2, 2, 749, 221, 0, 0, 0, 0, 0, 0, 4094, 0, 4090, 0, 0, 2, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 4, 4095, 0, 150, 1, 22, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 1, 4, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 0, 0, 0, 0, 3, 886, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 21, 3, 0, 2, 4092, 2, 0, 3, 3933, 28, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 4094, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 1260, 2, 121, 4, 4083, 1, 0, 4, 3343, 160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6731, 8190, 8365, 8194, 8831, 8084, 8163, 8978] avg=8192.0000 max=8978 max_rank=7 min=6731 min_rank=0 imbalance_ratio=1.095947 max_load_reduction=56558 max_load_reduction_pct=0.863007 moved_to_echo=34056 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3401, 4096, 0, 0, 0, 0, 749, 221, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 3, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 0, 0, 0, 227, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4096, 0, 0, 0, 3936, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 329, 0, 119, 0, 4096, 0, 0, 0, 3347, 160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8467, 8193, 8214, 8190, 8176, 8081, 8164, 8051] avg=8192.0000 max=8467 max_rank=0 min=8051 min_rank=7 imbalance_ratio=1.033569 max_load_reduction=57069 max_load_reduction_pct=0.870804 moved_to_echo=32472 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2881, 4096, 0, 0, 0, 0, 750, 221, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4084, 0, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 0, 0, 0, 118, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 4096, 0, 0, 0, 3936, 28, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 1039, 0, 4, 0, 4096, 0, 17, 0, 3346, 160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7948, 8190, 8186, 8190, 8064, 8114, 8182, 8662] avg=8192.0000 max=8662 max_rank=7 min=7948 min_rank=0 imbalance_ratio=1.057373 max_load_reduction=56874 max_load_reduction_pct=0.867828 moved_to_echo=33022 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3782, 4095, 0, 0, 1, 1, 749, 221, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 4083, 1, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0, 32, 4092, 0, 1, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3852, 0, 2, 0, 0, 1, 16, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 41, 1, 0, 1, 4095, 1, 0, 0, 3935, 28, 0, 0, 0, 0, 0, 0, 0, 50, 0, 1, 4095, 1, 0, 1, 4067, 0, 0, 0, 0, 0, 0, 0, 140, 0, 84, 0, 4096, 0, 0, 1, 3345, 160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8849, 8191, 8199, 8219, 7935, 8102, 8215, 7826] avg=8192.0000 max=8849 max_rank=0 min=7826 min_rank=7 imbalance_ratio=1.080200 max_load_reduction=56687 max_load_reduction_pct=0.864975 moved_to_echo=32078 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3652, 4096, 0, 0, 0, 0, 749, 221, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 4096, 0, 0, 0, 3936, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 428, 0, 3, 0, 4096, 0, 0, 0, 3347, 160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8718, 8193, 8209, 8190, 7953, 8075, 8164, 8034] avg=8192.0000 max=8718 max_rank=0 min=7953 min_rank=4 imbalance_ratio=1.064209 max_load_reduction=56818 max_load_reduction_pct=0.866974 moved_to_echo=32226 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3670, 4092, 2, 4, 4, 4, 749, 223, 0, 0, 0, 0, 0, 0, 4093, 7, 4088, 0, 0, 4, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 3, 8, 4012, 0, 6, 2, 28, 4, 0, 0, 0, 0, 0, 0, 5, 4092, 0, 0, 0, 4, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 0, 0, 0, 71, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 103, 4, 0, 3, 4090, 3, 0, 2, 3932, 28, 0, 0, 0, 0, 0, 0, 1, 37, 4, 0, 4089, 0, 0, 5, 4063, 0, 0, 0, 0, 0, 0, 0, 198, 0, 84, 14, 4091, 3, 14, 5, 3342, 160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8748, 8200, 8132, 8178, 8003, 8165, 8199, 7911] avg=8192.0000 max=8748 max_rank=0 min=7911 min_rank=7 imbalance_ratio=1.067871 max_load_reduction=56788 max_load_reduction_pct=0.866516 moved_to_echo=32278 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2967, 4096, 0, 0, 0, 0, 750, 221, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 3968, 0, 0, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 2, 0, 0, 0, 14, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 0, 0, 4096, 0, 0, 0, 3936, 28, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 4096, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 510, 0, 4, 0, 4096, 0, 5, 0, 3346, 160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8034, 8188, 8077, 8190, 7964, 8686, 8276, 8121] avg=8192.0000 max=8686 max_rank=5 min=7964 min_rank=4 imbalance_ratio=1.060303 max_load_reduction=56850 max_load_reduction_pct=0.867462 moved_to_echo=33047 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 25088, 32768, 128, 128, 128, 128, 6016, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32768, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4224, 128, 512, 128, 32768, 128, 128, 128, 26880, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1152, 128, 0, 128, 32768, 128, 0, 128, 31488, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 256, 128, 128, 32768, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[30848, 0, 128, 0, 0, 128, 2048, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 128, 32512, 128, 256, 128, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 17, 17, 0, 0, 0, 1, 3333, 4092, 0, 4096, 0, 42, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4080, 0, 4096, 0, 4096, 4091, 792, 0, 0, 0, 0, 0, 4061, 0, 0, 4096, 0, 4048, 0, 4096, 0, 0, 0, 4085, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 3431, 4096, 0, 4096, 0, 54, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 704, 0, 0, 0, 0, 0, 3998, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 3, 2, 0, 3647, 4093, 0, 4090, 2, 121, 0, 0, 1, 0, 4, 1, 4090, 0, 2, 4090, 2, 4092, 4, 0, 0, 4080, 1, 4087, 3, 4092, 4094, 461, 2, 0, 0, 5, 0, 3952, 1, 3, 4093, 0, 4094, 0, 4092, 2, 1, 2, 4091, 1, 3, 8, 3, 0, 3, 3, 0, 4092, 2, 1, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4, 4, 0, 0, 0, 0, 3751, 4096, 0, 4096, 0, 366, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 4093, 0, 4096, 0, 4096, 4094, 329, 0, 0, 0, 0, 0, 3733, 0, 0, 4096, 0, 4090, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 1, 0, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 3718, 4096, 2, 4096, 0, 96, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4078, 0, 4096, 0, 4096, 4096, 508, 0, 0, 0, 0, 0, 3896, 0, 0, 4096, 0, 4068, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[5, 0, 2, 0, 0, 2, 0, 683, 4092, 0, 4088, 6, 171, 0, 0, 0, 0, 7, 1, 4085, 0, 4, 4088, 4, 4089, 9, 0, 0, 4091, 5, 4086, 8, 4091, 4092, 3410, 4, 0, 0, 11, 0, 3924, 0, 0, 4091, 1, 4093, 0, 4091, 2, 4, 0, 4092, 0, 3, 3, 4, 0, 5, 1, 0, 4087, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2638, 4096, 0, 4096, 0, 329, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 1548, 0, 0, 0, 0, 0, 3675, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 1, 0, 3, 2586, 4095, 0, 4095, 0, 41, 0, 0, 3, 0, 1, 1, 4095, 0, 0, 4095, 1, 4095, 0, 0, 0, 4091, 1, 4094, 1, 4095, 4095, 1620, 1, 0, 0, 1, 0, 3895, 0, 8, 4095, 1, 4083, 0, 4095, 1, 1, 0, 4095, 0, 0, 0, 1, 46, 1, 1, 1, 4095, 0, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 17, 0, 0, 0, 1, 3333, 4096, 1115, 0, 0, 0, 0, 0, 0, 3941, 0, 4096, 0, 42, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4080, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 4091, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 4048, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 2974, 1121, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4, 2759, 1171, 151, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8579, 8087, 8192, 8183, 7808, 8325, 8180, 8182] avg=8192.0000 max=8579 max_rank=0 min=7808 min_rank=4 imbalance_ratio=1.047241 max_load_reduction=56957 max_load_reduction_pct=0.869095 moved_to_echo=32495 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 3, 2, 0, 3647, 4093, 1114, 0, 0, 0, 0, 0, 0, 3942, 0, 4090, 2, 121, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4090, 0, 2, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 4080, 1, 4087, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2922, 4094, 461, 2, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 1, 3, 0, 0, 4094, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 4091, 1, 3, 8, 3, 2971, 1120, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4092, 2, 1, 16, 2756, 1170, 151, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8863, 8156, 8189, 8182, 7484, 8266, 8202, 8194] avg=8192.0000 max=8863 max_rank=0 min=7484 min_rank=4 imbalance_ratio=1.081909 max_load_reduction=56673 max_load_reduction_pct=0.864761 moved_to_echo=32146 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 3431, 4096, 1115, 0, 0, 0, 0, 0, 0, 3945, 0, 4096, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 4096, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 4096, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2974, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 2759, 1171, 151, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8643, 8095, 8192, 8199, 7725, 8310, 8191, 8181] avg=8192.0000 max=8643 max_rank=0 min=7725 min_rank=4 imbalance_ratio=1.055054 max_load_reduction=56893 max_load_reduction_pct=0.868118 moved_to_echo=32407 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 3718, 4096, 1115, 0, 0, 0, 0, 0, 0, 3945, 2, 4096, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4078, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 4096, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 0, 0, 4068, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2974, 1121, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 16, 2759, 1171, 151, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8930, 8139, 8192, 8181, 7529, 8180, 8189, 8196] avg=8192.0000 max=8930 max_rank=0 min=7529 min_rank=4 imbalance_ratio=1.090088 max_load_reduction=56606 max_load_reduction_pct=0.863739 moved_to_echo=32094 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2638, 4096, 1115, 0, 0, 0, 0, 0, 0, 3945, 0, 4096, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 4096, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3675, 0, 0, 0, 0, 4096, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2974, 1121, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 2, 2759, 1171, 151, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7849, 8370, 8191, 8199, 8569, 7987, 8191, 8180] avg=8192.0000 max=8569 max_rank=4 min=7849 min_rank=0 imbalance_ratio=1.046021 max_load_reduction=56967 max_load_reduction_pct=0.869247 moved_to_echo=32927 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 1, 0, 3, 2586, 4095, 1115, 0, 0, 0, 0, 0, 0, 3944, 0, 4095, 0, 41, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4091, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2924, 4095, 1620, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 0, 8, 0, 1, 4083, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 0, 1, 2973, 1121, 0, 0, 0, 0, 0, 0, 46, 1, 1, 1, 4095, 0, 1, 1, 2758, 1171, 151, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7804, 8083, 8193, 8194, 8641, 8203, 8192, 8226] avg=8192.0000 max=8641 max_rank=4 min=7804 min_rank=0 imbalance_ratio=1.054810 max_load_reduction=56895 max_load_reduction_pct=0.868149 moved_to_echo=33262 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 2, 0, 0, 2, 0, 683, 4091, 1115, 0, 0, 0, 0, 0, 0, 3930, 0, 4088, 6, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4085, 0, 4, 4088, 4, 6, 0, 0, 0, 0, 0, 0, 0, 2, 9, 0, 0, 4091, 5, 4086, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2915, 4092, 3410, 4, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 0, 1, 4093, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4092, 0, 3, 3, 4, 2972, 1124, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 4087, 1, 0, 0, 2759, 1176, 156, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5898, 8195, 8195, 8201, 10432, 8226, 8204, 8185] avg=8192.0000 max=10432 max_rank=4 min=5898 min_rank=0 imbalance_ratio=1.273438 max_load_reduction=55104 max_load_reduction_pct=0.840820 moved_to_echo=35047 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4, 0, 0, 0, 0, 3751, 4096, 1115, 0, 0, 0, 0, 0, 0, 3945, 0, 4096, 0, 366, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 4094, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3733, 0, 0, 0, 0, 4090, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 2974, 1121, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 1, 0, 12, 2759, 1171, 151, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8970, 8411, 8192, 8197, 7348, 8039, 8187, 8192] avg=8192.0000 max=8970 max_rank=0 min=7348 min_rank=4 imbalance_ratio=1.094971 max_load_reduction=56566 max_load_reduction_pct=0.863129 moved_to_echo=31766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 23808, 32768, 8960, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32768, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[23424, 32768, 9472, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31616, 128, 32768, 128, 1280, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31232, 128, 128, 0, 128, 32768, 0, 1792, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 128, 23808, 9088, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 128, 22144, 9472, 1280, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 67, 0, 0, 0, 112, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 3982, 4027, 35, 4006, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 4096, 4096, 0, 0, 4092, 4087, 45, 4096, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4, 3, 0, 1, 1, 3, 4092, 0, 4, 0, 54, 6, 0, 4, 145, 4, 0, 4092, 4, 4086, 0, 0, 0, 0, 0, 4089, 0, 4093, 4, 0, 5, 4091, 0, 0, 0, 3, 2, 0, 4093, 0, 3828, 4077, 0, 4045, 4, 3, 4, 0, 0, 4092, 0, 3, 4, 0, 1, 4092, 4092, 3, 3, 4092, 4093, 136, 4081, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 12, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4079, 4072, 2, 4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 24, 4096, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 54, 0, 0, 0, 158, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 3939, 4065, 27, 4024, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 19, 4096, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 32, 0, 0, 0, 525, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 4096, 0, 3518, 4061, 88, 3833, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4095, 205, 4096, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 1, 4095, 0, 1, 2, 55, 1, 0, 1, 34, 1, 0, 4095, 1, 4090, 1, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 4095, 0, 1, 11, 1, 1, 0, 4095, 0, 3752, 4040, 1, 4077, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 4095, 0, 1, 4094, 4094, 318, 4095, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4, 3, 7, 5, 0, 7, 4090, 8, 0, 1, 34, 1, 7, 0, 0, 5, 0, 4069, 5, 4083, 4, 0, 0, 2, 2, 4088, 0, 4090, 6, 0, 8, 4090, 0, 0, 0, 5, 5, 0, 4091, 0, 4041, 4089, 0, 4060, 5, 5, 0, 0, 0, 4090, 1, 3, 6, 0, 0, 4091, 4091, 5, 4, 4088, 4087, 54, 4076, 20] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 120, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4005, 4079, 242, 3779, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 37, 4096, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3770, 182, 102, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 112, 0, 4096, 3913, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3910, 181, 0, 0, 0, 0, 0, 0, 3982, 257, 35, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 45, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8188, 8196, 8193, 8188, 8280, 8197, 8144] avg=8192.0000 max=8280 max_rank=5 min=8144 min_rank=7 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 3, 0, 1, 1, 3, 4092, 0, 3817, 183, 102, 0, 0, 0, 0, 0, 4, 0, 54, 6, 0, 4, 145, 4, 4092, 3912, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 4086, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4, 0, 5, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 4093, 0, 3909, 181, 0, 0, 0, 0, 0, 0, 3828, 260, 0, 4045, 4, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3, 4, 0, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 3986, 136, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8221, 8187, 8189, 8188, 8144, 8192, 8209] avg=8192.0000 max=8221 max_rank=1 min=8144 min_rank=5 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32733 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3813, 183, 102, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 12, 0, 4096, 3913, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3913, 183, 0, 0, 0, 0, 0, 0, 4079, 259, 2, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 24, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8033, 8197, 8192, 8192, 8424, 8192, 8112] avg=8192.0000 max=8424 max_rank=5 min=8033 min_rank=1 imbalance_ratio=1.028320 max_load_reduction=57112 max_load_reduction_pct=0.871460 moved_to_echo=32920 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 1, 4095, 0, 3783, 183, 102, 0, 0, 0, 0, 0, 1, 2, 55, 1, 0, 1, 34, 1, 4095, 3912, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4090, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 1, 1, 0, 4095, 0, 3911, 183, 0, 0, 0, 0, 0, 0, 3752, 257, 1, 4077, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3987, 318, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8102, 8192, 8190, 8203, 8088, 8190, 8404] avg=8192.0000 max=8404 max_rank=7 min=8088 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=57132 max_load_reduction_pct=0.871765 moved_to_echo=32885 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3806, 183, 102, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 158, 0, 4096, 3914, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3913, 182, 0, 0, 0, 0, 0, 0, 3939, 259, 27, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 19, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8222, 8197, 8192, 8192, 8249, 8192, 8105] avg=8192.0000 max=8249 max_rank=5 min=8105 min_rank=7 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32738 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3819, 183, 102, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 120, 0, 4096, 3913, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3913, 181, 0, 0, 0, 0, 0, 0, 4005, 260, 242, 3779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 37, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8154, 8197, 8194, 8190, 8286, 8192, 8123] avg=8192.0000 max=8286 max_rank=5 min=8123 min_rank=7 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32791 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 3, 7, 5, 0, 7, 4090, 8, 3833, 183, 108, 0, 0, 0, 0, 0, 0, 1, 34, 1, 7, 0, 0, 5, 4091, 3909, 0, 0, 0, 0, 0, 0, 0, 4069, 5, 4083, 4, 0, 0, 2, 8, 2, 0, 0, 0, 0, 0, 0, 2, 4088, 0, 0, 6, 0, 8, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 4091, 0, 3905, 181, 0, 0, 0, 0, 0, 0, 4041, 256, 0, 4060, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 3, 6, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 3971, 54, 4076, 20, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8048, 8173, 8194, 8187, 8367, 8189, 8130] avg=8192.0000 max=8367 max_rank=5 min=8048 min_rank=1 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32873 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3802, 184, 102, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 525, 0, 4096, 3915, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 3912, 181, 0, 0, 0, 0, 0, 0, 3518, 259, 88, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 205, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8568, 8197, 8192, 8196, 7698, 8192, 8309] avg=8192.0000 max=8568 max_rank=1 min=7698 min_rank=5 imbalance_ratio=1.045898 max_load_reduction=56968 max_load_reduction_pct=0.869263 moved_to_echo=32395 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[31232, 2176, 512, 32000, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 32000, 896, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 0, 32768, 0, 31360, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 32768, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 128, 30464, 1536, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 384, 128, 128, 128, 1152, 128, 32768, 31360, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 3943, 0, 0, 4096, 0, 0, 0, 24, 0, 0, 3803, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 569, 0, 0, 0, 0, 0, 4096, 0, 0, 3952, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 0, 2, 3982, 0, 4, 4092, 4, 4, 0, 30, 4, 0, 2854, 4090, 4092, 4, 0, 4091, 1, 0, 0, 4075, 0, 4092, 3, 0, 0, 3, 3, 4092, 4090, 0, 4092, 2, 2, 4092, 0, 0, 0, 4, 3, 3, 2, 6, 0, 1, 4, 0, 1350, 0, 0, 6, 0, 0, 4090, 4, 0, 4063, 4091, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 1, 4057, 0, 1, 4095, 1, 1, 0, 0, 1, 0, 3570, 4095, 4095, 1, 0, 4095, 0, 2, 0, 4093, 2, 4095, 0, 0, 1, 1, 2, 4094, 4095, 0, 4095, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 699, 0, 0, 0, 0, 0, 4095, 1, 0, 3955, 4095, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4016, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 4001, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3, 0, 6, 25, 0, 4, 4085, 0, 7, 4090, 7, 6, 6, 26, 6, 0, 3821, 4089, 4090, 7, 0, 4087, 0, 0, 0, 4079, 0, 4089, 0, 0, 6, 5, 6, 4091, 4085, 0, 4090, 0, 5, 4089, 0, 0, 0, 3, 5, 3, 3, 11, 0, 5, 6, 0, 242, 1, 0, 3, 0, 0, 4081, 5, 0, 4082, 4088, 4088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3961, 0, 0, 4096, 0, 0, 0, 34, 0, 0, 3224, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4092, 4096, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1232, 0, 0, 0, 0, 0, 4096, 0, 0, 3842, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 3762, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 0, 0, 0, 0, 0, 4096, 0, 0, 3936, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4007, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 3943, 0, 3548, 607, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 24, 0, 0, 3625, 449, 0, 0, 0, 0, 0, 0, 3803, 467, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 569, 0, 0, 0, 4096, 3486, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3952, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8194, 8366, 8196, 8191, 8189, 8154, 8147] avg=8192.0000 max=8366 max_rank=2 min=8099 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32681 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 0, 2, 3982, 0, 3544, 607, 0, 0, 0, 0, 0, 0, 4, 4092, 4, 4, 0, 30, 4, 0, 3620, 448, 0, 0, 0, 0, 0, 0, 2854, 466, 0, 4, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4092, 3, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4090, 0, 4092, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 3, 2, 6, 4092, 4089, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 1350, 0, 0, 6, 4091, 3485, 3, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 4063, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8206, 7416, 8177, 8189, 8199, 8940, 8256] avg=8192.0000 max=8940 max_rank=6 min=7416 min_rank=2 imbalance_ratio=1.091309 max_load_reduction=56596 max_load_reduction_pct=0.863586 moved_to_echo=33584 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4016, 0, 3548, 607, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 3625, 449, 0, 0, 0, 0, 0, 0, 4001, 467, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 4096, 3486, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8174, 8564, 8196, 8195, 8189, 7765, 8282] avg=8192.0000 max=8564 max_rank=2 min=7765 min_rank=6 imbalance_ratio=1.045410 max_load_reduction=56972 max_load_reduction_pct=0.869324 moved_to_echo=32431 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 1, 4057, 0, 3548, 607, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 0, 1, 0, 3624, 449, 0, 0, 0, 0, 0, 0, 3570, 467, 0, 1, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 4095, 0, 0, 1, 1, 4, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4095, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 699, 0, 0, 0, 4095, 3485, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 3955, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8172, 8135, 8196, 8195, 8189, 8284, 8149] avg=8192.0000 max=8284 max_rank=6 min=8135 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32817 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4095, 0, 3548, 607, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 3625, 449, 0, 0, 0, 0, 0, 0, 3762, 467, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 0, 0, 0, 4096, 3487, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3936, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8191, 8325, 8193, 8194, 8189, 8063, 8131] avg=8192.0000 max=8325 max_rank=2 min=8063 min_rank=6 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32577 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3548, 607, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 3625, 449, 0, 0, 0, 0, 0, 0, 4007, 467, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 4096, 3486, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8171, 8570, 8196, 8195, 8190, 7686, 8277] avg=8192.0000 max=8570 max_rank=2 min=7686 min_rank=6 imbalance_ratio=1.046143 max_load_reduction=56966 max_load_reduction_pct=0.869232 moved_to_echo=32348 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 6, 25, 0, 4, 4085, 0, 3547, 609, 0, 0, 0, 0, 0, 0, 7, 4090, 7, 6, 6, 26, 6, 0, 3624, 452, 0, 0, 0, 0, 0, 0, 3821, 458, 0, 7, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4089, 0, 0, 6, 5, 7, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4085, 0, 4090, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 3, 3, 11, 4090, 4087, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 242, 1, 0, 3, 4088, 3482, 2, 0, 0, 0, 0, 0, 0, 0, 4081, 5, 0, 4082, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8224, 8373, 8186, 8186, 8202, 7829, 8257] avg=8192.0000 max=8373 max_rank=2 min=7829 min_rank=6 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32474 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3961, 0, 3548, 608, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 34, 0, 0, 3625, 449, 0, 0, 0, 0, 0, 0, 3224, 467, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1232, 0, 0, 0, 4096, 3484, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3842, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8204, 7787, 8196, 8191, 8189, 8815, 8037] avg=8192.0000 max=8815 max_rank=6 min=7787 min_rank=2 imbalance_ratio=1.076050 max_load_reduction=56721 max_load_reduction_pct=0.865494 moved_to_echo=33232 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[29056, 3840, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 4864, 128, 0, 128, 32768, 27904, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 128, 0, 32000, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 128, 32256, 0, 28416, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 256, 128, 0, 29056, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 32768, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[642, 0, 4096, 18, 0, 0, 0, 0, 3454, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4075, 4092, 0, 0, 0, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[115, 0, 4090, 0, 3, 4, 0, 14, 3973, 0, 0, 4083, 0, 3, 5, 4091, 0, 4, 0, 1, 0, 0, 0, 3, 4092, 0, 0, 1, 4091, 4092, 4092, 5, 0, 4, 0, 4, 1, 3, 4093, 4, 4092, 4, 5, 0, 4090, 0, 0, 0, 1, 5, 4088, 7, 5, 4088, 4091, 4093, 4092, 2, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1598, 0, 4095, 0, 1, 1, 0, 2, 2496, 1, 0, 4092, 0, 0, 1, 4095, 0, 1, 0, 1, 0, 4, 0, 0, 4095, 0, 0, 0, 4095, 4095, 4095, 1, 0, 2, 0, 0, 0, 0, 4093, 1, 4095, 0, 1, 0, 4095, 0, 0, 0, 1, 1, 4094, 0, 1, 4095, 4095, 4092, 4095, 1, 1, 2, 0, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[322, 0, 4096, 0, 0, 0, 0, 0, 3774, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[259, 5, 4088, 0, 6, 2, 0, 1, 3797, 0, 0, 4081, 0, 6, 7, 4088, 0, 7, 0, 6, 0, 1, 2, 6, 4087, 0, 5, 3, 4088, 4091, 4089, 8, 0, 12, 0, 0, 0, 4, 4091, 6, 4090, 6, 27, 0, 4090, 0, 5, 0, 2, 7, 4089, 8, 7, 4086, 4087, 4090, 4090, 5, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[820, 0, 4096, 0, 0, 0, 0, 0, 3273, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1628, 0, 4096, 0, 0, 0, 0, 0, 2469, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 4096, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[504, 0, 4096, 1, 0, 0, 0, 0, 3572, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 19, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[115, 0, 4090, 0, 3, 4, 0, 14, 3345, 3, 3, 0, 0, 0, 0, 0, 3973, 0, 0, 4083, 0, 3, 5, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 3, 4092, 4090, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 4091, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 1, 3, 4093, 4, 4085, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 5, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 7, 5, 4088, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 2, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7577, 8810, 8190, 8189, 8194, 8188, 8199, 8189] avg=8192.0000 max=8810 max_rank=1 min=7577 min_rank=0 imbalance_ratio=1.075439 max_load_reduction=56726 max_load_reduction_pct=0.865570 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[642, 0, 4096, 18, 0, 0, 0, 0, 3349, 3, 2, 0, 0, 0, 0, 0, 3454, 0, 0, 4096, 0, 0, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8110, 8297, 8190, 8193, 8185, 8194, 8174, 8193] avg=8192.0000 max=8297 max_rank=1 min=8110 min_rank=0 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32746 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[322, 0, 4096, 0, 0, 0, 0, 0, 3349, 3, 2, 0, 0, 0, 0, 0, 3774, 0, 0, 4096, 0, 0, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7772, 8617, 8190, 8193, 8187, 8189, 8195, 8193] avg=8192.0000 max=8617 max_rank=1 min=7772 min_rank=0 imbalance_ratio=1.051880 max_load_reduction=56919 max_load_reduction_pct=0.868515 moved_to_echo=32764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1598, 0, 4095, 0, 1, 1, 0, 2, 3348, 3, 2, 0, 0, 0, 0, 0, 2496, 1, 0, 4092, 0, 0, 1, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4093, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 4095, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 2, 0, 1, 1, 0, 4094, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9050, 7337, 8195, 8192, 8187, 8188, 8191, 8196] avg=8192.0000 max=9050 max_rank=0 min=7337 min_rank=1 imbalance_ratio=1.104736 max_load_reduction=56486 max_load_reduction_pct=0.861908 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[820, 0, 4096, 0, 0, 0, 0, 0, 3349, 3, 2, 0, 0, 0, 0, 0, 3273, 0, 0, 4096, 0, 0, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8270, 8116, 8191, 8193, 8193, 8189, 8192, 8192] avg=8192.0000 max=8270 max_rank=0 min=8116 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[504, 0, 4096, 1, 0, 0, 0, 0, 3349, 3, 2, 0, 0, 0, 0, 0, 3572, 0, 0, 4096, 0, 0, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7955, 8415, 8190, 8193, 8206, 8189, 8195, 8193] avg=8192.0000 max=8415 max_rank=1 min=7955 min_rank=0 imbalance_ratio=1.027222 max_load_reduction=57121 max_load_reduction_pct=0.871597 moved_to_echo=32783 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[259, 5, 4088, 0, 6, 2, 0, 1, 3347, 8, 5, 0, 0, 0, 0, 0, 3797, 0, 0, 4081, 0, 6, 7, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 6, 0, 1, 2, 6, 4091, 4086, 0, 0, 0, 0, 0, 0, 4087, 0, 5, 3, 4088, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4, 4091, 6, 4084, 0, 0, 0, 0, 0, 0, 0, 4082, 6, 27, 0, 4090, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 0, 8, 7, 4086, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 5, 0, 0, 0, 0, 0, 1, 4089, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7721, 8632, 8199, 8191, 8197, 8210, 8200, 8186] avg=8192.0000 max=8632 max_rank=1 min=7721 min_rank=0 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=32793 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1628, 0, 4096, 0, 0, 0, 0, 0, 3349, 3, 5, 0, 0, 0, 0, 0, 2469, 0, 0, 4096, 0, 0, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9081, 7312, 8191, 8192, 8187, 8189, 8190, 8194] avg=8192.0000 max=9081 max_rank=0 min=7312 min_rank=1 imbalance_ratio=1.108521 max_load_reduction=56455 max_load_reduction_pct=0.861435 moved_to_echo=32760 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 0, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[5888, 128, 32768, 128, 128, 128, 0, 128, 26880, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26880, 128, 0, 32768, 0, 128, 128, 6016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4089, 7, 4093, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 65, 40, 3797, 0, 0, 0, 4096, 0, 0, 2746, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1547, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4091, 0, 4064, 5, 0, 0, 0, 4088, 7, 0, 4, 0, 4092, 4092, 0, 4091, 4, 4, 5, 5, 4089, 3, 31, 4087, 0, 4, 4088, 3, 0, 0, 4031, 3, 1, 5, 4092, 4, 4, 3806, 4090, 5, 0, 2, 0, 0, 1, 0, 0, 4091, 2, 0, 5, 4, 0, 4092, 333, 0, 0, 2, 4091, 15, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 4095, 0, 11, 4, 4079, 0, 0, 0, 4096, 0, 0, 274, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3827, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 4096, 0, 63, 58, 3775, 0, 0, 0, 4096, 0, 0, 2623, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1677, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4095, 3, 4095, 0, 5, 2, 0, 4093, 2, 0, 1, 0, 4093, 4095, 1, 4095, 0, 0, 1, 1, 4095, 1, 0, 4094, 1, 1, 4094, 1, 24, 51, 3971, 0, 1, 1, 4095, 1, 0, 2035, 4095, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 1, 0, 1, 4095, 2102, 0, 0, 0, 4095, 1, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4081, 0, 0, 4096, 0, 130, 25, 3754, 0, 0, 0, 4096, 0, 0, 2769, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1529, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4089, 14, 4075, 7, 78, 15, 0, 4084, 5, 0, 6, 0, 4089, 4091, 0, 4090, 2, 6, 7, 10, 4084, 6, 0, 4088, 1, 7, 4086, 6, 0, 0, 3814, 0, 3, 8, 4090, 4, 1, 1250, 4087, 0, 2, 0, 0, 0, 5, 0, 5, 4087, 7, 0, 7, 9, 0, 4090, 3035, 0, 0, 0, 4086, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 24, 0, 3884, 0, 0, 0, 4096, 0, 0, 778, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3508, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 148, 0, 65, 40, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2060, 0, 0, 2746, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3948, 150, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1900, 1547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2046, 2036, 7, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8194, 8183, 8146, 8902, 8192, 7543, 8191] avg=8192.0000 max=8902 max_rank=4 min=7543 min_rank=6 imbalance_ratio=1.086670 max_load_reduction=56634 max_load_reduction_pct=0.864166 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 0, 4064, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4, 4, 5, 5, 4089, 3, 31, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4, 148, 3, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 5, 2058, 4, 4, 3806, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 1, 0, 0, 4092, 3940, 150, 0, 0, 0, 0, 0, 4091, 2, 0, 5, 4, 0, 1898, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4091, 15, 0, 0, 0, 2044, 2034, 7, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8191, 8226, 8273, 9971, 8190, 6333, 8199] avg=8192.0000 max=9971 max_rank=4 min=6333 min_rank=6 imbalance_ratio=1.217163 max_load_reduction=55565 max_load_reduction_pct=0.847855 moved_to_echo=32693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 148, 0, 11, 4, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2060, 0, 0, 274, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3947, 150, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1900, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2046, 2036, 7, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8194, 8186, 8336, 6430, 8191, 9823, 8191] avg=8192.0000 max=9823 max_rank=6 min=6430 min_rank=4 imbalance_ratio=1.199097 max_load_reduction=55713 max_load_reduction_pct=0.850113 moved_to_echo=32635 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 148, 0, 63, 58, 3775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2060, 0, 0, 2623, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3948, 150, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1900, 1677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2046, 2036, 7, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8192, 8186, 8138, 8779, 8193, 7673, 8191] avg=8192.0000 max=8779 max_rank=4 min=7673 min_rank=6 imbalance_ratio=1.071655 max_load_reduction=56757 max_load_reduction_pct=0.866043 moved_to_echo=32836 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4088, 3, 4095, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 1, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 148, 1, 24, 51, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2059, 1, 0, 2035, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4092, 3946, 150, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 1, 1900, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 2045, 2036, 7, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8193, 8187, 8291, 8192, 8190, 8099, 8191] avg=8192.0000 max=8291 max_rank=3 min=8099 min_rank=6 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 148, 0, 24, 0, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2060, 0, 0, 778, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3946, 150, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1900, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2046, 2036, 7, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8194, 8186, 8152, 6934, 8190, 9504, 8191] avg=8192.0000 max=9504 max_rank=6 min=6934 min_rank=4 imbalance_ratio=1.160156 max_load_reduction=56032 max_load_reduction_pct=0.854980 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4077, 14, 4075, 7, 78, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 5, 0, 6, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 2, 6, 7, 10, 4084, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 7, 146, 6, 0, 0, 3814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 2050, 4, 1, 1250, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 0, 5, 4089, 3940, 155, 0, 0, 0, 0, 0, 4087, 7, 0, 7, 9, 0, 1891, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 2044, 2040, 12, 9, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8266, 8186, 8196, 8062, 7403, 8196, 9036, 8191] avg=8192.0000 max=9036 max_rank=6 min=7403 min_rank=4 imbalance_ratio=1.103027 max_load_reduction=56500 max_load_reduction_pct=0.862122 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 148, 0, 130, 25, 3754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2060, 0, 0, 2769, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3948, 150, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1900, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2046, 2036, 7, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8192, 8186, 8138, 8925, 8194, 7525, 8191] avg=8192.0000 max=8925 max_rank=4 min=7525 min_rank=6 imbalance_ratio=1.089478 max_load_reduction=56611 max_load_reduction_pct=0.863815 moved_to_echo=32835 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 128, 32768, 31616, 1280, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 0, 128, 128, 128, 15232, 17664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 32768, 128, 0, 128, 0, 16384, 16384, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 1280, 128, 384, 256, 31232, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 16512, 128, 128, 16384, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4059, 0, 0, 0, 0, 4089, 0, 4094, 0, 4096, 4093, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 35, 7, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4, 1, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4095, 4095, 1, 1, 0, 0, 4095, 1, 4095, 0, 4095, 4086, 1, 0, 0, 1, 4095, 4095, 4095, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 4092, 1, 3, 0, 0, 0, 0, 0, 4, 5, 1, 1, 0, 4093, 2, 1, 4095, 4091, 67, 11, 1, 0, 1, 0, 4019, 1, 0, 0, 1, 1, 0, 1, 1, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4091, 4092, 5, 4, 0, 8, 4094, 0, 4086, 0, 4091, 3616, 1, 0, 0, 0, 4090, 4091, 4092, 0, 8, 4090, 4091, 0, 0, 0, 0, 1, 4088, 2, 7, 3, 5, 0, 0, 0, 0, 13, 4, 5, 30, 4079, 5, 2, 4092, 4045, 643, 0, 0, 0, 5, 0, 3933, 4, 0, 0, 5, 5, 0, 6, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4090, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4088, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 11, 5, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4088, 4088, 7, 5, 1, 11, 4092, 0, 4088, 0, 4086, 4081, 4, 0, 0, 0, 4084, 4085, 4090, 0, 12, 4088, 4089, 0, 0, 0, 0, 1, 4089, 5, 21, 0, 8, 0, 0, 0, 0, 6, 7, 7, 0, 4078, 14, 0, 4089, 4066, 151, 17, 6, 0, 9, 0, 3925, 4, 0, 0, 7, 8, 0, 8, 7, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4074, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4056, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 22, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4068, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4010, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 120, 1, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4031, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 67, 3, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4059, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 91, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3967, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 7, 0, 4096, 4074, 4, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 4096, 0, 0, 4079, 38, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8278, 8195, 8191, 8216, 8068, 8235, 8188] avg=8192.0000 max=8278 max_rank=1 min=8068 min_rank=5 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32707 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 4092, 5, 4, 0, 8, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 91, 3616, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4091, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 2, 7, 3962, 127, 0, 0, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 13, 4, 4091, 4068, 5, 0, 0, 0, 0, 0, 5, 30, 3952, 5, 2, 0, 4045, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3933, 4, 0, 4084, 38, 22, 0, 0, 0, 0, 0, 0, 5, 5, 0, 6, 4, 0, 0, 4092, 4087, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 7794, 8189, 8187, 8189, 8682, 8086, 8199] avg=8192.0000 max=8682 max_rank=5 min=7794 min_rank=1 imbalance_ratio=1.059814 max_load_reduction=56854 max_load_reduction_pct=0.867523 moved_to_echo=33156 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 4095, 1, 1, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 91, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4095, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 3, 3966, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 1, 4095, 4073, 5, 0, 0, 0, 0, 0, 1, 0, 3965, 2, 1, 0, 4091, 67, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 1, 0, 4019, 1, 0, 4085, 38, 19, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 2, 0, 4095, 4090, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8274, 8194, 8190, 8183, 8127, 8175, 8191] avg=8192.0000 max=8274 max_rank=1 min=8127 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32676 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4090, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 91, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3967, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4074, 4, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4092, 0, 0, 4086, 38, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8275, 8195, 8191, 8176, 8075, 8240, 8188] avg=8192.0000 max=8275 max_rank=1 min=8075 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32679 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 91, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3967, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4074, 4, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 4095, 67, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4092, 0, 0, 4086, 38, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8218, 8195, 8191, 8174, 8130, 8238, 8188] avg=8192.0000 max=8238 max_rank=6 min=8130 min_rank=5 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32730 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4068, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 91, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3967, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 4096, 4074, 5, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 4096, 120, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4066, 0, 0, 4086, 38, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8197, 8195, 8191, 8198, 8184, 8210, 8187] avg=8192.0000 max=8210 max_rank=6 min=8174 min_rank=0 imbalance_ratio=1.002197 max_load_reduction=57326 max_load_reduction_pct=0.874725 moved_to_echo=32779 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4088, 4088, 7, 5, 1, 11, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 84, 4081, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4085, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 5, 21, 3961, 132, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 6, 7, 4089, 4068, 5, 0, 0, 0, 0, 0, 7, 0, 3946, 14, 0, 0, 4066, 151, 0, 0, 0, 0, 0, 0, 0, 0, 17, 6, 0, 9, 0, 3925, 4, 0, 4085, 41, 20, 0, 0, 0, 0, 0, 0, 7, 8, 0, 8, 7, 4, 0, 4090, 4084, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8257, 8181, 8209, 8183, 8184, 8107, 8208] avg=8192.0000 max=8257 max_rank=1 min=8107 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32682 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4074, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 91, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3967, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 4096, 4074, 5, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 4086, 38, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8243, 8192, 8186, 8217, 8086, 8245, 8187] avg=8192.0000 max=8245 max_rank=6 min=8086 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32735 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 31744, 128, 128, 0, 32768, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 32768, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 768, 32128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 128, 128, 128, 32768, 32640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 128, 32384, 128, 0, 32768, 384, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 32768, 128, 128, 31744, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 1236, 4096, 0, 5, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 8, 420, 0, 0, 0, 0, 4092, 0, 3245, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 35, 4085, 4096, 0, 3250, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 1, 2, 0, 4095, 0, 1, 2895, 4078, 1, 0, 3, 4094, 0, 0, 1, 1, 0, 4095, 0, 1, 4095, 4095, 1, 4095, 0, 0, 995, 1, 0, 1, 0, 4095, 1, 1750, 0, 1, 4092, 1, 1, 0, 0, 1, 0, 1, 10, 4095, 4095, 0, 2555, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4052, 0, 0, 3, 5, 0, 0, 0, 1, 4089, 4090, 4, 1, 0, 3, 20, 0, 4090, 1, 0, 1724, 4046, 11, 0, 0, 4090, 0, 3, 4, 7, 0, 4091, 4, 5, 4088, 4092, 3, 4091, 0, 0, 25, 0, 0, 0, 0, 4092, 4, 2590, 5, 6, 4082, 6, 2, 0, 0, 6, 0, 5, 419, 4091, 4089, 0, 3493, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1438, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 169, 0, 0, 0, 0, 4096, 0, 3618, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 42, 4094, 4096, 0, 2929, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 1937, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 201, 0, 0, 0, 0, 4096, 0, 2130, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 141, 4077, 4096, 0, 3783, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 3831, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 365, 0, 0, 0, 0, 4095, 0, 671, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 46, 4081, 4096, 0, 3281, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4087, 1, 0, 2, 8, 0, 0, 0, 0, 4088, 4086, 6, 0, 0, 8, 16, 7, 4088, 0, 2, 1087, 3998, 3, 0, 5, 4085, 0, 4, 7, 11, 0, 4088, 0, 8, 4084, 4089, 4, 4087, 0, 0, 1, 5, 0, 0, 1, 4084, 7, 4014, 9, 9, 4062, 10, 8, 0, 0, 8, 0, 7, 178, 4089, 4088, 0, 2992, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 745, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 415, 0, 0, 0, 0, 4096, 0, 3546, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 4096, 0, 3485, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 7, 0, 1236, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 0, 0, 0, 4092, 0, 3245, 1071, 3, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3318, 776, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4085, 772, 0, 3250, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8189, 7593, 8193, 8201, 8832, 8190, 8142] avg=8192.0000 max=8832 max_rank=5 min=7593 min_rank=2 imbalance_ratio=1.078125 max_load_reduction=56704 max_load_reduction_pct=0.865234 moved_to_echo=33365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4052, 0, 0, 3, 5, 0, 0, 0, 4092, 6, 0, 0, 0, 0, 0, 0, 1, 4086, 4090, 4, 1, 0, 3, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2245, 1, 0, 1724, 4046, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3, 4, 7, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 4088, 0, 3, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 4092, 4, 2590, 1070, 3, 1, 0, 0, 0, 0, 0, 5, 6, 4082, 6, 2, 0, 0, 6, 3313, 775, 0, 0, 0, 0, 0, 0, 0, 5, 419, 4091, 770, 0, 3493, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8158, 8205, 8027, 8194, 8191, 7785, 8195, 8781] avg=8192.0000 max=8781 max_rank=7 min=7785 min_rank=5 imbalance_ratio=1.071899 max_load_reduction=56755 max_load_reduction_pct=0.866013 moved_to_echo=32952 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 0, 0, 1438, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 0, 4096, 0, 3618, 1071, 3, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3318, 776, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4094, 772, 0, 2929, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8189, 7781, 8191, 8192, 8958, 8190, 7837] avg=8192.0000 max=8958 max_rank=5 min=7781 min_rank=2 imbalance_ratio=1.093506 max_load_reduction=56578 max_load_reduction_pct=0.863312 moved_to_echo=33177 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 1, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 4092, 4095, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2248, 0, 1, 2895, 4078, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 995, 1, 0, 1, 0, 4095, 1, 1750, 1071, 3, 1, 0, 0, 0, 0, 0, 0, 1, 4092, 1, 1, 0, 0, 1, 3318, 776, 0, 0, 0, 0, 0, 0, 0, 1, 10, 4095, 771, 0, 2555, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8190, 9223, 8193, 8192, 7918, 8190, 7433] avg=8192.0000 max=9223 max_rank=2 min=7433 min_rank=7 imbalance_ratio=1.125854 max_load_reduction=56313 max_load_reduction_pct=0.859268 moved_to_echo=31733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 19, 0, 1937, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 0, 4096, 0, 2130, 1071, 3, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3318, 776, 0, 0, 0, 0, 0, 0, 0, 0, 141, 4077, 772, 0, 3783, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8189, 8301, 8191, 8192, 7502, 8190, 8773] avg=8192.0000 max=8773 max_rank=7 min=7502 min_rank=5 imbalance_ratio=1.070923 max_load_reduction=56763 max_load_reduction_pct=0.866135 moved_to_echo=32657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 0, 0, 745, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 415, 0, 0, 0, 0, 4096, 0, 3546, 1071, 3, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3318, 776, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 772, 0, 3485, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8189, 7089, 8191, 8193, 9132, 8190, 8355] avg=8192.0000 max=9132 max_rank=5 min=7089 min_rank=2 imbalance_ratio=1.114746 max_load_reduction=56404 max_load_reduction_pct=0.860657 moved_to_echo=33870 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4087, 1, 0, 2, 8, 0, 0, 0, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 4080, 4086, 6, 0, 0, 8, 16, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2235, 0, 2, 1087, 3998, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4085, 0, 4, 7, 11, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4084, 0, 4, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 1, 4084, 7, 4014, 1075, 8, 8, 0, 0, 0, 0, 0, 9, 9, 4062, 10, 8, 0, 0, 8, 3317, 778, 0, 0, 0, 0, 0, 0, 0, 7, 178, 4089, 764, 0, 2992, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8196, 7332, 8192, 8183, 9203, 8201, 8035] avg=8192.0000 max=9203 max_rank=5 min=7332 min_rank=2 imbalance_ratio=1.123413 max_load_reduction=56333 max_load_reduction_pct=0.859573 moved_to_echo=33622 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 15, 0, 3831, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 365, 0, 0, 0, 0, 4095, 0, 671, 1071, 3, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3318, 776, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4081, 772, 0, 3281, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8189, 10190, 8191, 8192, 6206, 8190, 8180] avg=8192.0000 max=10190 max_rank=2 min=6206 min_rank=5 imbalance_ratio=1.243896 max_load_reduction=55346 max_load_reduction_pct=0.844513 moved_to_echo=30768 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[2688, 128, 0, 128, 128, 32768, 128, 21632, 8576, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 896, 32768, 6272, 0, 25856, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 18048, 128, 128, 14976, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 0, 128, 26624, 6272, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4089, 0, 13, 0, 0, 4091, 4096, 4085, 4, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 4095, 1, 1, 0, 1, 0, 0, 4094, 1, 0, 4095, 2, 4094, 4095, 1, 0, 1, 4095, 1, 1, 5, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 4092, 4095, 0, 4075, 1, 1, 0, 0, 2, 0, 1, 1, 4094, 0, 1, 0, 0, 15, 0, 4095, 0, 1, 0, 0, 4095, 4095, 4095, 0, 4095, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 136, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4078, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 19, 0, 4096, 0, 1, 0, 0, 4096, 4096, 4082, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[3, 0, 4092, 6, 5, 0, 5, 0, 0, 4089, 5, 1, 4090, 9, 4087, 4087, 3, 0, 2, 4091, 4, 6, 0, 6, 0, 6, 0, 0, 0, 1, 4, 2, 0, 4090, 4091, 0, 3965, 5, 0, 0, 3, 8, 5, 0, 5, 4085, 3, 6, 0, 0, 127, 0, 4091, 5, 0, 0, 0, 4091, 4091, 4083, 0, 4090, 4088, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4, 0, 4088, 6, 8, 0, 9, 0, 0, 4085, 7, 0, 4087, 18, 4084, 4082, 8, 9, 4, 4088, 6, 0, 0, 10, 5, 10, 0, 0, 0, 0, 6, 3, 0, 4085, 4087, 0, 4070, 4, 0, 0, 3, 14, 28, 0, 10, 4078, 3, 11, 0, 3, 1, 0, 4089, 7, 0, 0, 9, 4086, 4088, 4062, 0, 4088, 4083, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 4095, 0, 1, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 46, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4079, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 45, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4089, 0, 13, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4085, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8194, 8188, 8191, 8236, 8179, 8160, 8200] avg=8192.0000 max=8236 max_rank=4 min=8160 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32775 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 45, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 9, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 4096, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4078, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8192, 8187, 8191, 8119, 8186, 8283, 8189] avg=8192.0000 max=8283 max_rank=6 min=8119 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4092, 6, 5, 0, 5, 0, 4086, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 9, 4087, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 4091, 4, 6, 0, 6, 4081, 4, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 4, 2, 4091, 4088, 0, 0, 0, 0, 0, 0, 0, 4090, 45, 0, 3965, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 5, 0, 5, 4085, 3, 6, 4076, 8, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 4091, 5, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4083, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8189, 8197, 8192, 8105, 8199, 8269, 8186] avg=8192.0000 max=8269 max_rank=6 min=8105 min_rank=4 imbalance_ratio=1.009399 max_load_reduction=57267 max_load_reduction_pct=0.873825 moved_to_echo=32759 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 45, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 9, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 0, 1, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4082, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8192, 8187, 8191, 8231, 8186, 8167, 8193] avg=8192.0000 max=8231 max_rank=4 min=8167 min_rank=6 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4095, 1, 1, 0, 1, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1, 1, 5, 1, 4085, 4, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 4092, 45, 0, 4075, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 4094, 0, 1, 4080, 9, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4095, 0, 1, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4095, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8194, 8192, 8214, 8188, 8161, 8204] avg=8192.0000 max=8214 max_rank=4 min=8161 min_rank=6 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32767 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 1, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 45, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 7, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 4096, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8194, 8187, 8191, 8207, 8184, 8193, 8190] avg=8192.0000 max=8207 max_rank=4 min=8184 min_rank=5 imbalance_ratio=1.001831 max_load_reduction=57329 max_load_reduction_pct=0.874771 moved_to_echo=32774 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4088, 6, 8, 0, 9, 0, 4083, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 18, 4084, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 4, 4088, 6, 0, 0, 10, 4078, 4, 0, 0, 0, 0, 0, 0, 5, 10, 0, 0, 0, 0, 6, 3, 4088, 4086, 0, 0, 0, 0, 0, 0, 0, 4085, 37, 0, 4070, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 14, 28, 0, 10, 4078, 3, 11, 4073, 7, 3, 0, 0, 0, 0, 0, 0, 3, 1, 0, 4089, 7, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 9, 13, 0, 4062, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8191, 8207, 8198, 8196, 8230, 8147, 8167] avg=8192.0000 max=8230 max_rank=5 min=8147 min_rank=6 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32740 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 45, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 7, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4095, 0, 1, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8189, 8190, 8228, 8184, 8156, 8207] avg=8192.0000 max=8228 max_rank=4 min=8156 min_rank=6 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32775 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 384, 0, 32512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 32768, 128, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 128, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 384, 0, 32768, 128, 128, 0, 32512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3570, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4093, 0, 537, 0, 0, 0, 4096, 0, 0, 4085, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4, 4000, 9, 4090, 0, 4089, 0, 7, 0, 20, 0, 0, 4087, 0, 277, 4088, 5, 3277, 3, 4089, 4091, 0, 4078, 6, 4076, 0, 0, 5, 2, 5, 2, 7, 4, 14, 7, 0, 0, 0, 0, 0, 4, 5, 7, 0, 4086, 118, 4090, 0, 676, 0, 5, 2, 4090, 0, 6, 4091, 0, 6, 5, 4088, 0, 3915] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 20, 4096, 0, 4061, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 34, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 4096, 0, 4076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4035, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 61, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 6, 4062, 10, 4085, 0, 4082, 0, 12, 0, 38, 0, 0, 4084, 0, 119, 4073, 13, 3690, 0, 4086, 4086, 0, 4039, 8, 4070, 0, 0, 7, 43, 9, 0, 9, 5, 19, 11, 0, 0, 0, 0, 0, 7, 9, 11, 4, 4082, 167, 4078, 0, 294, 0, 9, 1, 4087, 14, 10, 4089, 0, 0, 11, 4084, 0, 3912] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4021, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 82, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 0, 4089] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 1, 4094, 0, 4095, 0, 4095, 0, 1, 0, 5, 0, 0, 4095, 0, 7, 4092, 0, 4017, 0, 4095, 4095, 0, 4092, 1, 4094, 2, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 4094, 1, 4095, 0, 85, 0, 1, 2, 4095, 1, 1, 4095, 0, 0, 2, 4094, 0, 4080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 4083, 0, 4096, 4096, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 16, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4037, 53, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3570, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 8, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 3842, 216, 0, 0, 0, 0, 0, 0, 4093, 0, 537, 0, 0, 0, 3875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 51, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8186, 7867, 8197, 8185, 8156, 8505, 8232] avg=8192.0000 max=8505 max_rank=6 min=7867 min_rank=2 imbalance_ratio=1.038208 max_load_reduction=57031 max_load_reduction_pct=0.870224 moved_to_echo=33078 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 4000, 9, 16, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 20, 0, 0, 4087, 0, 4029, 53, 0, 0, 0, 0, 0, 0, 277, 4088, 5, 3277, 3, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 6, 4076, 0, 0, 5, 2, 5, 5, 0, 0, 0, 0, 0, 0, 0, 2, 7, 4, 14, 7, 0, 0, 0, 4091, 4074, 8, 0, 0, 0, 0, 0, 0, 0, 4, 5, 7, 0, 4086, 118, 3835, 216, 0, 0, 0, 0, 0, 0, 4090, 0, 676, 0, 5, 2, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4091, 0, 6, 5, 51, 0, 3915, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8118, 8196, 7851, 8177, 8207, 8271, 8642, 8074] avg=8192.0000 max=8642 max_rank=6 min=7851 min_rank=2 imbalance_ratio=1.054932 max_load_reduction=56894 max_load_reduction_pct=0.868134 moved_to_echo=33194 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4037, 53, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4035, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3842, 216, 0, 0, 0, 0, 0, 0, 4096, 0, 61, 0, 0, 0, 3875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 51, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8186, 8332, 8197, 8184, 8154, 8032, 8243] avg=8192.0000 max=8332 max_rank=2 min=8032 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=57204 max_load_reduction_pct=0.872864 moved_to_echo=32613 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4037, 53, 0, 0, 0, 0, 0, 0, 20, 4096, 0, 4061, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3842, 216, 0, 0, 0, 0, 0, 0, 4096, 0, 34, 0, 0, 0, 3875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 51, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8186, 8378, 8197, 8192, 8154, 8005, 8216] avg=8192.0000 max=8378 max_rank=2 min=8005 min_rank=6 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32567 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 4094, 0, 16, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 4095, 0, 4035, 53, 0, 0, 0, 0, 0, 0, 7, 4092, 0, 4017, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 4094, 2, 0, 1, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 4095, 4079, 8, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 4094, 1, 3841, 216, 0, 0, 0, 0, 0, 0, 4095, 0, 85, 0, 1, 2, 3874, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 2, 51, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8189, 8317, 8196, 8185, 8155, 8058, 8229] avg=8192.0000 max=8317 max_rank=2 min=8058 min_rank=6 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32627 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4037, 53, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4083, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3842, 216, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 3875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 51, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8186, 8381, 8193, 8184, 8155, 7987, 8243] avg=8192.0000 max=8381 max_rank=2 min=7987 min_rank=6 imbalance_ratio=1.023071 max_load_reduction=57155 max_load_reduction_pct=0.872116 moved_to_echo=32569 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 6, 4062, 10, 11, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 38, 0, 0, 4084, 0, 4032, 55, 0, 0, 0, 0, 0, 0, 119, 4073, 13, 3690, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 8, 4070, 0, 0, 7, 43, 9, 6, 0, 0, 0, 0, 0, 0, 0, 0, 9, 5, 19, 11, 0, 0, 0, 4086, 4074, 9, 0, 0, 0, 0, 0, 0, 0, 7, 9, 11, 4, 4082, 167, 3834, 223, 0, 0, 0, 0, 0, 0, 4078, 0, 294, 0, 9, 1, 3858, 14, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4089, 0, 0, 11, 43, 0, 3912, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8221, 8092, 8182, 8213, 8337, 8254, 8065] avg=8192.0000 max=8337 max_rank=5 min=8065 min_rank=7 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32869 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4037, 53, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4021, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3842, 216, 0, 0, 0, 0, 0, 0, 4096, 0, 82, 0, 0, 0, 3875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 51, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8186, 8318, 8197, 8186, 8154, 8053, 8234] avg=8192.0000 max=8318 max_rank=2 min=8053 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32627 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 1792, 0, 128, 128, 30976, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 32768, 128, 30848, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32640, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 0, 32384, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 0, 32768, 32640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 32768, 384, 30720, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 512, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 32768, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 17, 4096, 0, 0, 0, 0, 0, 0, 0, 3936, 1, 0, 2347, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3668, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2319, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 8, 4094, 1, 1, 1, 1, 1, 1, 0, 4090, 0, 0, 194, 1, 4095, 4094, 0, 4095, 0, 0, 2, 1, 4069, 0, 4095, 0, 0, 0, 4088, 0, 0, 0, 1, 0, 0, 1, 3932, 0, 4095, 1, 0, 4095, 1, 4095, 4094, 0, 0, 4092, 0, 1, 0, 1, 2, 1, 1, 4095, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 611, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4066, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3615, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 646, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3838, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 940, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3335, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 3, 4085, 0, 0, 5, 3, 4085, 9, 3, 3, 8, 4, 7, 0, 4090, 0, 0, 318, 4, 4082, 4080, 0, 4082, 0, 0, 31, 9, 4083, 0, 4087, 0, 56, 12, 4032, 6, 14, 0, 10, 0, 0, 6, 3759, 0, 4085, 12, 1, 4086, 8, 4089, 4087, 0, 0, 4084, 0, 10, 0, 9, 7, 6, 6, 4061, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 3520, 0, 0, 715, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4076, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 4091, 0, 0, 0, 0, 4089, 5, 0, 2, 5, 2, 5, 0, 4091, 0, 0, 1302, 1, 4086, 4088, 0, 4088, 0, 2, 22, 6, 4091, 4, 4090, 0, 51, 8, 4041, 9, 0, 0, 0, 5, 0, 3, 2789, 0, 4090, 7, 4, 4090, 5, 4091, 4091, 0, 0, 4083, 0, 6, 0, 6, 7, 0, 3, 4071, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 17, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 4096, 155, 0, 0, 0, 0, 0, 0, 1, 0, 2347, 0, 3217, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3668, 0, 4096, 1, 89, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3302, 790, 0, 0, 0, 0, 0, 0, 0, 0, 2319, 0, 4096, 0, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3938, 159, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8187, 9661, 7855, 8188, 7050, 8195, 8192] avg=8192.0000 max=9661 max_rank=2 min=7050 min_rank=5 imbalance_ratio=1.179321 max_load_reduction=55875 max_load_reduction_pct=0.852585 moved_to_echo=31625 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 4096, 155, 0, 0, 0, 0, 0, 0, 0, 0, 611, 0, 3217, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 4096, 0, 89, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3302, 790, 0, 0, 0, 0, 0, 0, 0, 0, 3615, 0, 4096, 0, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3938, 159, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8248, 7924, 8252, 8188, 8346, 8195, 8192] avg=8192.0000 max=8346 max_rank=5 min=7924 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32921 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 4096, 155, 0, 0, 0, 0, 0, 0, 0, 0, 646, 0, 3217, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 0, 4096, 0, 89, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3302, 790, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 4096, 0, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3939, 159, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8091, 7959, 8024, 8188, 8690, 8194, 8194] avg=8192.0000 max=8690 max_rank=5 min=7959 min_rank=2 imbalance_ratio=1.060791 max_load_reduction=56846 max_load_reduction_pct=0.867401 moved_to_echo=33266 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 0, 8, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 4090, 4095, 155, 0, 0, 0, 0, 0, 0, 0, 0, 194, 1, 3217, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4069, 0, 4095, 0, 89, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 1, 0, 3301, 789, 0, 0, 0, 0, 0, 0, 0, 1, 3932, 0, 4095, 1, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4094, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 1, 4095, 0, 1, 3937, 159, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8346, 7506, 8257, 8179, 8664, 8191, 8197] avg=8192.0000 max=8664 max_rank=5 min=7506 min_rank=2 imbalance_ratio=1.057617 max_load_reduction=56872 max_load_reduction_pct=0.867798 moved_to_echo=33231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 4096, 155, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 3217, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 89, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3302, 790, 0, 0, 0, 0, 0, 0, 0, 0, 3335, 0, 4096, 0, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3938, 159, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8174, 8253, 8276, 8188, 8066, 8195, 8193] avg=8192.0000 max=8276 max_rank=3 min=8066 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32642 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4090, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 5, 2, 5, 0, 4091, 4088, 155, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 1, 3210, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 22, 6, 4091, 4, 4090, 0, 88, 1, 0, 0, 0, 0, 0, 0, 51, 8, 4041, 9, 0, 0, 0, 5, 3297, 788, 0, 0, 0, 0, 0, 0, 0, 3, 2789, 0, 4090, 7, 4, 634, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4091, 0, 0, 4083, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 3, 4071, 0, 5, 3936, 159, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8353, 8601, 8304, 8199, 7527, 8185, 8187] avg=8192.0000 max=8601 max_rank=2 min=7527 min_rank=5 imbalance_ratio=1.049927 max_load_reduction=56935 max_load_reduction_pct=0.868759 moved_to_echo=32098 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4080, 0, 0, 5, 3, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 3, 8, 4, 7, 0, 4090, 4082, 158, 0, 0, 0, 0, 0, 0, 0, 0, 318, 4, 3202, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 9, 4083, 0, 4087, 0, 91, 5, 0, 0, 0, 0, 0, 0, 56, 12, 4032, 6, 14, 0, 10, 0, 3299, 789, 0, 0, 0, 0, 0, 0, 0, 6, 3759, 0, 4085, 12, 1, 622, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4087, 0, 0, 4084, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 7, 6, 6, 4061, 0, 9, 3931, 165, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8364, 7604, 8306, 8218, 8485, 8189, 8194] avg=8192.0000 max=8485 max_rank=5 min=7604 min_rank=2 imbalance_ratio=1.035767 max_load_reduction=57051 max_load_reduction_pct=0.870529 moved_to_echo=33086 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3520, 4096, 157, 0, 0, 0, 0, 0, 0, 0, 0, 715, 0, 3217, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 0, 89, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3302, 790, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 4096, 0, 0, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 3939, 159, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 7773, 8028, 8262, 8188, 8708, 8192, 8187] avg=8192.0000 max=8708 max_rank=5 min=7773 min_rank=1 imbalance_ratio=1.062988 max_load_reduction=56828 max_load_reduction_pct=0.867126 moved_to_echo=33275 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 27776, 0, 32768, 128, 128, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 31488, 32768, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 7168, 128, 25728, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 128, 31616, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 128, 128, 26496, 6400, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 32000, 128, 32768, 128, 768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4092, 4096, 2712, 4096, 0, 4096, 32, 0, 324, 4096, 4092, 0, 0, 0, 0, 0, 4063, 0, 4096, 0, 4092, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1108, 0, 0, 13, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[6, 7, 4, 0, 4, 4091, 0, 0, 8, 2, 2, 0, 0, 5, 0, 4089, 0, 4090, 4, 4084, 1, 4086, 4087, 3136, 4091, 2, 4090, 17, 0, 21, 4087, 4087, 0, 2, 0, 34, 0, 4092, 0, 4089, 6, 4085, 5, 0, 4088, 5, 1, 5, 6, 0, 0, 4, 1, 6, 4090, 0, 0, 890, 0, 15, 0, 0, 6, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1, 1, 1, 0, 1, 4095, 0, 0, 0, 1, 2, 0, 0, 1, 1, 4095, 0, 4092, 1, 4095, 0, 4095, 4094, 2204, 4095, 1, 4095, 11, 0, 32, 4094, 4092, 0, 0, 0, 9, 0, 4082, 0, 4094, 1, 4094, 1, 0, 4094, 1, 5, 1, 1, 0, 0, 1, 1, 1, 4095, 0, 0, 1852, 0, 1, 0, 0, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 1919, 4096, 0, 4096, 0, 0, 18, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2147, 0, 0, 15, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4093, 4096, 1194, 4096, 0, 4096, 10, 0, 202, 4096, 4089, 0, 0, 0, 0, 0, 4071, 0, 4096, 0, 4096, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2745, 0, 0, 29, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[10, 12, 5, 0, 6, 4091, 0, 0, 14, 0, 2, 0, 1, 8, 0, 4084, 0, 4065, 5, 4080, 3, 4079, 4084, 2760, 4087, 4, 4086, 6, 0, 0, 4084, 4086, 0, 4, 0, 26, 0, 4090, 0, 4084, 8, 4079, 8, 0, 4082, 9, 7, 7, 9, 0, 0, 6, 5, 11, 4088, 0, 0, 1324, 0, 8, 0, 0, 11, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4091, 4096, 1753, 4096, 0, 4096, 60, 0, 90, 4096, 4090, 0, 0, 0, 0, 0, 4069, 0, 4096, 0, 4096, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2264, 0, 0, 31, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 710, 4096, 0, 4096, 0, 0, 391, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2988, 0, 0, 7, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4043, 0, 0, 2184, 1930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2184, 1912, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2052, 0, 2712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 324, 3952, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 4096, 3, 3, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4093, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2040, 1912, 141, 0, 0, 0, 0, 0, 0, 1108, 0, 0, 13, 0, 0, 0, 4096, 2166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8192, 8860, 8400, 8165, 8190, 8189, 7383] avg=8192.0000 max=8860 max_rank=2 min=7383 min_rank=7 imbalance_ratio=1.081543 max_load_reduction=56676 max_load_reduction_pct=0.864807 moved_to_echo=31927 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[6, 7, 4, 0, 4, 4091, 0, 0, 2180, 1928, 0, 0, 0, 0, 0, 0, 8, 2, 2, 0, 0, 5, 0, 4089, 2179, 1907, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 1, 2049, 0, 3136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 17, 0, 21, 3943, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 34, 0, 4092, 0, 4089, 3, 3, 0, 0, 0, 0, 0, 0, 6, 4085, 5, 0, 4085, 5, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4, 1, 6, 4090, 0, 2037, 1905, 141, 0, 0, 0, 0, 0, 0, 890, 0, 15, 0, 0, 6, 5, 4091, 2162, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8192, 9280, 8070, 8223, 8192, 8190, 7169] avg=8192.0000 max=9280 max_rank=2 min=7169 min_rank=7 imbalance_ratio=1.132812 max_load_reduction=56256 max_load_reduction_pct=0.858398 moved_to_echo=31774 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4094, 0, 0, 2185, 1930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2184, 1911, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2054, 0, 1919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3952, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 3, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2042, 1912, 141, 0, 0, 0, 0, 0, 0, 2147, 0, 0, 15, 0, 0, 0, 4096, 2166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8191, 8069, 8066, 8197, 8189, 8191, 8424] avg=8192.0000 max=8424 max_rank=7 min=8066 min_rank=3 imbalance_ratio=1.028320 max_load_reduction=57112 max_load_reduction_pct=0.871460 moved_to_echo=33001 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4039, 0, 0, 2185, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2185, 1911, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2052, 0, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 202, 3952, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4096, 3, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2041, 1911, 141, 0, 0, 0, 0, 0, 0, 2745, 0, 0, 29, 0, 0, 0, 4096, 2165, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8192, 7342, 8253, 8173, 8197, 8189, 9035] avg=8192.0000 max=9035 max_rank=7 min=7342 min_rank=2 imbalance_ratio=1.102905 max_load_reduction=56501 max_load_reduction_pct=0.862137 moved_to_echo=33594 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 0, 1, 4095, 0, 0, 2184, 1931, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 1, 1, 4095, 2184, 1910, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 2053, 0, 2204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 32, 3950, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4082, 0, 4094, 3, 3, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 4091, 1, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 4095, 0, 2042, 1911, 141, 0, 0, 0, 0, 0, 0, 1852, 0, 1, 0, 0, 1, 1, 4095, 2164, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8194, 8350, 8086, 8191, 8194, 8193, 8114] avg=8192.0000 max=8350 max_rank=2 min=8086 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32692 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 2184, 1930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2184, 1912, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2054, 0, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 3952, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2042, 1912, 141, 0, 0, 0, 0, 0, 0, 2988, 0, 0, 7, 0, 0, 0, 4096, 2166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8192, 6860, 8439, 8198, 8189, 8191, 9257] avg=8192.0000 max=9257 max_rank=7 min=6860 min_rank=2 imbalance_ratio=1.130005 max_load_reduction=56279 max_load_reduction_pct=0.858749 moved_to_echo=33835 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[10, 12, 5, 0, 6, 4091, 0, 0, 2179, 1926, 0, 0, 0, 0, 0, 0, 14, 0, 2, 0, 1, 8, 0, 4084, 2177, 1905, 0, 0, 0, 0, 0, 0, 0, 4065, 5, 0, 3, 2042, 0, 2760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 0, 0, 3934, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 26, 0, 4090, 0, 4084, 10, 4, 0, 0, 0, 0, 0, 0, 8, 4079, 8, 0, 4072, 9, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 6, 5, 11, 4088, 0, 2037, 1903, 146, 0, 0, 0, 0, 0, 0, 1324, 0, 8, 0, 0, 11, 8, 4087, 2160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8191, 8875, 8030, 8218, 8190, 8205, 7598] avg=8192.0000 max=8875 max_rank=2 min=7598 min_rank=7 imbalance_ratio=1.083374 max_load_reduction=56661 max_load_reduction_pct=0.864578 moved_to_echo=32211 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4026, 0, 0, 2185, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2185, 1911, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2051, 0, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 90, 3952, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4096, 3, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2040, 1911, 141, 0, 0, 0, 0, 0, 0, 2264, 0, 0, 31, 0, 0, 0, 4096, 2165, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8192, 7900, 8192, 8171, 8195, 8188, 8556] avg=8192.0000 max=8556 max_rank=7 min=7900 min_rank=2 imbalance_ratio=1.044434 max_load_reduction=56980 max_load_reduction_pct=0.869446 moved_to_echo=33110 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 0, 32768, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 32640, 0, 0, 17536, 15488, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 15360, 0, 128, 128, 0, 128, 128, 32768, 17408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 128, 128, 32768, 0, 16384, 15360, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 0, 128, 16512, 0, 16512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 32768, 17536, 15360, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 256, 0, 1152, 31616, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[9, 7, 1285, 2, 195, 0, 1495, 0, 3147, 83, 79, 2756, 1048, 379, 0, 3270, 1217, 921, 428, 590, 2534, 1315, 267, 1522, 2, 2599, 21, 1080, 611, 14, 314, 1941, 4032, 4, 49, 2027, 2588, 26, 634, 1728, 3403, 412, 1041, 791, 0, 861, 1243, 1965, 452, 3874, 420, 1, 1, 8, 690, 453, 0, 3484, 3101, 578, 279, 1033, 1198, 29] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[11, 6, 1075, 0, 220, 0, 1137, 0, 2233, 56, 96, 2613, 1027, 456, 0, 3128, 1570, 1201, 739, 427, 2462, 1479, 193, 1365, 0, 2551, 35, 1110, 463, 6, 67, 1457, 4069, 11, 60, 1912, 2057, 6, 639, 2071, 3636, 324, 496, 334, 0, 1023, 1302, 2917, 136, 3866, 469, 0, 0, 0, 1147, 1202, 1, 2963, 3380, 1199, 440, 1522, 1163, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[5, 27, 1042, 8, 243, 2, 1003, 9, 2533, 68, 41, 1863, 958, 509, 0, 3348, 1485, 1623, 650, 270, 2550, 1427, 213, 1492, 0, 1506, 55, 1145, 696, 13, 232, 1792, 4077, 14, 84, 1797, 3358, 10, 688, 2240, 3641, 332, 519, 468, 0, 895, 1161, 2641, 106, 3584, 371, 5, 1, 3, 1148, 629, 1, 3562, 3251, 1131, 375, 1104, 1493, 39] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3, 7, 1020, 5, 204, 2, 1447, 1, 2605, 71, 45, 1977, 853, 438, 0, 3557, 1277, 879, 857, 363, 2143, 1199, 245, 1561, 0, 2219, 6, 1266, 436, 8, 193, 1970, 4093, 3, 44, 1793, 3305, 11, 538, 2227, 3400, 469, 693, 435, 1, 1051, 1737, 2759, 207, 3811, 504, 0, 4, 7, 794, 574, 2, 3311, 3199, 1033, 503, 1227, 938, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[7, 19, 959, 0, 209, 0, 1209, 0, 2603, 39, 75, 1963, 1302, 574, 0, 3168, 1142, 1214, 340, 494, 2748, 1304, 250, 1398, 0, 2514, 35, 1203, 410, 20, 272, 1896, 4089, 60, 65, 1624, 2684, 0, 1237, 2136, 3736, 507, 542, 461, 5, 1624, 1103, 2323, 103, 3770, 529, 0, 1, 15, 894, 517, 0, 3521, 2710, 1110, 304, 1728, 753, 18] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[4, 21, 1120, 2, 141, 0, 1425, 2, 2769, 39, 58, 2233, 814, 579, 0, 2981, 1124, 1699, 320, 175, 2444, 1219, 236, 1595, 2, 2744, 3, 991, 754, 8, 159, 1737, 4070, 3, 103, 2040, 2307, 4, 405, 2543, 3333, 417, 445, 822, 3, 1074, 1091, 2573, 148, 3926, 514, 0, 1, 4, 910, 664, 2, 3538, 3308, 1054, 212, 1258, 1362, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[5, 26, 1243, 2, 168, 1, 1159, 24, 2839, 86, 53, 2680, 1049, 778, 3, 3355, 1699, 1215, 528, 515, 2666, 1279, 164, 1422, 5, 2340, 83, 1149, 743, 3, 196, 1796, 4091, 8, 60, 1497, 3260, 6, 622, 2075, 3425, 330, 609, 644, 0, 999, 1056, 2175, 125, 3553, 550, 0, 25, 6, 1187, 709, 0, 3584, 2736, 478, 299, 1368, 783, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[3, 1, 873, 3, 100, 0, 1657, 0, 2905, 106, 59, 2218, 1123, 396, 0, 3291, 968, 1530, 333, 229, 2497, 1135, 122, 1525, 0, 2907, 6, 925, 639, 1, 278, 1655, 4086, 20, 47, 2201, 2569, 5, 323, 2303, 3619, 165, 503, 574, 6, 1248, 1163, 2623, 61, 3869, 386, 0, 2, 0, 948, 589, 0, 3570, 3340, 1094, 251, 1639, 843, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 27, 1042, 8, 243, 2, 1003, 9, 3401, 2185, 75, 0, 0, 0, 0, 0, 2533, 68, 41, 1863, 958, 509, 0, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 1485, 1623, 650, 270, 1729, 1427, 213, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1506, 55, 1145, 696, 13, 232, 1792, 1265, 821, 0, 0, 0, 0, 0, 0, 676, 14, 84, 1797, 3358, 10, 688, 2240, 0, 0, 0, 0, 0, 0, 0, 0, 1884, 332, 519, 468, 0, 895, 1161, 2641, 0, 0, 0, 0, 0, 0, 0, 0, 106, 3584, 371, 5, 1, 3, 1148, 629, 1699, 492, 0, 0, 0, 0, 0, 0, 1, 1377, 3251, 1131, 375, 1104, 1493, 39, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8000, 7546, 8889, 7525, 8867, 7900, 8038, 8771] avg=8192.0000 max=8889 max_rank=2 min=7525 min_rank=3 imbalance_ratio=1.085083 max_load_reduction=56647 max_load_reduction_pct=0.864365 moved_to_echo=33576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[9, 7, 1285, 2, 195, 0, 1495, 0, 3363, 2137, 74, 0, 0, 0, 0, 0, 3147, 83, 79, 2756, 1048, 379, 0, 1537, 0, 0, 0, 0, 0, 0, 0, 0, 1217, 921, 428, 590, 1718, 1315, 267, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2599, 21, 1080, 611, 14, 314, 1941, 1182, 816, 0, 0, 0, 0, 0, 0, 669, 4, 49, 2027, 2588, 26, 634, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 1761, 412, 1041, 791, 0, 861, 1243, 1965, 0, 0, 0, 0, 0, 0, 0, 0, 452, 3874, 420, 1, 1, 8, 690, 453, 1659, 460, 0, 0, 0, 0, 0, 0, 0, 1347, 3101, 578, 279, 1033, 1198, 29, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8567, 9029, 7978, 8580, 7725, 8074, 8018, 7565] avg=8192.0000 max=9029 max_rank=1 min=7565 min_rank=7 imbalance_ratio=1.102173 max_load_reduction=56507 max_load_reduction_pct=0.862228 moved_to_echo=31382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 6, 1075, 0, 220, 0, 1137, 0, 3394, 1817, 70, 0, 0, 0, 0, 0, 2233, 56, 96, 2613, 1027, 456, 0, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 1570, 1201, 739, 427, 1669, 1479, 193, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2551, 35, 1110, 463, 6, 67, 1457, 1263, 793, 0, 0, 0, 0, 0, 0, 675, 11, 60, 1912, 2057, 6, 639, 2071, 0, 0, 0, 0, 0, 0, 0, 0, 1881, 324, 496, 334, 0, 1023, 1302, 2917, 0, 0, 0, 0, 0, 0, 0, 0, 136, 3866, 469, 0, 0, 0, 1147, 1202, 1587, 492, 0, 0, 0, 0, 0, 0, 1, 1146, 3380, 1199, 440, 1522, 1163, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7730, 7952, 8643, 7745, 7431, 8277, 8899, 8859] avg=8192.0000 max=8899 max_rank=6 min=7431 min_rank=4 imbalance_ratio=1.086304 max_load_reduction=56637 max_load_reduction_pct=0.864212 moved_to_echo=33466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 19, 959, 0, 209, 0, 1209, 0, 3411, 2159, 71, 0, 0, 0, 0, 0, 2603, 39, 75, 1963, 1302, 574, 0, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 1214, 340, 494, 1863, 1304, 250, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2514, 35, 1203, 410, 20, 272, 1896, 1298, 885, 0, 0, 0, 0, 0, 0, 678, 60, 65, 1624, 2684, 0, 1237, 2136, 0, 0, 0, 0, 0, 0, 0, 0, 1933, 507, 542, 461, 5, 1624, 1103, 2323, 0, 0, 0, 0, 0, 0, 0, 0, 103, 3770, 529, 0, 1, 15, 894, 517, 1608, 505, 0, 0, 0, 0, 0, 0, 0, 1362, 2710, 1110, 304, 1728, 753, 18, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8044, 8045, 8005, 8533, 8484, 8498, 7942, 7985] avg=8192.0000 max=8533 max_rank=3 min=7942 min_rank=6 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=32909 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 21, 1120, 2, 141, 0, 1425, 2, 3395, 2170, 67, 0, 0, 0, 0, 0, 2769, 39, 58, 2233, 814, 579, 0, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 1699, 320, 175, 1657, 1219, 236, 1595, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2744, 3, 991, 754, 8, 159, 1737, 1158, 787, 0, 0, 0, 0, 0, 0, 675, 3, 103, 2040, 2307, 4, 405, 2543, 0, 0, 0, 0, 0, 0, 0, 0, 1724, 417, 445, 822, 3, 1074, 1091, 2573, 0, 0, 0, 0, 0, 0, 0, 0, 148, 3926, 514, 0, 1, 4, 910, 664, 1513, 451, 0, 0, 0, 0, 0, 0, 2, 1368, 3308, 1054, 212, 1258, 1362, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8347, 7893, 8025, 8343, 8080, 8149, 8131, 8568] avg=8192.0000 max=8568 max_rank=7 min=7893 min_rank=1 imbalance_ratio=1.045898 max_load_reduction=56968 max_load_reduction_pct=0.869263 moved_to_echo=32928 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 7, 1020, 5, 204, 2, 1447, 1, 3414, 2031, 80, 0, 0, 0, 0, 0, 2605, 71, 45, 1977, 853, 438, 0, 1672, 0, 0, 0, 0, 0, 0, 0, 0, 1277, 879, 857, 363, 1453, 1199, 245, 1561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2219, 6, 1266, 436, 8, 193, 1970, 1181, 690, 0, 0, 0, 0, 0, 0, 679, 3, 44, 1793, 3305, 11, 538, 2227, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 469, 693, 435, 1, 1051, 1737, 2759, 0, 0, 0, 0, 0, 0, 0, 0, 207, 3811, 504, 0, 4, 7, 794, 574, 1805, 460, 0, 0, 0, 0, 0, 0, 2, 1280, 3199, 1033, 503, 1227, 938, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 7661, 7834, 7969, 8600, 8904, 8166, 8188] avg=8192.0000 max=8904 max_rank=5 min=7661 min_rank=1 imbalance_ratio=1.086914 max_load_reduction=56632 max_load_reduction_pct=0.864136 moved_to_echo=33858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 26, 1243, 2, 168, 1, 1159, 24, 3412, 2198, 75, 0, 0, 0, 0, 0, 2839, 86, 53, 2680, 1049, 778, 3, 1577, 0, 0, 0, 0, 0, 0, 0, 0, 1699, 1215, 528, 515, 1808, 1279, 164, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2340, 83, 1149, 743, 3, 196, 1796, 1190, 858, 0, 0, 0, 0, 0, 0, 679, 8, 60, 1497, 3260, 6, 622, 2075, 0, 0, 0, 0, 0, 0, 0, 0, 1772, 330, 609, 644, 0, 999, 1056, 2175, 0, 0, 0, 0, 0, 0, 0, 0, 125, 3553, 550, 0, 25, 6, 1187, 709, 1703, 463, 0, 0, 0, 0, 0, 0, 0, 1386, 2736, 478, 299, 1368, 783, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8313, 9065, 8630, 8363, 8207, 7585, 8321, 7052] avg=8192.0000 max=9065 max_rank=1 min=7052 min_rank=7 imbalance_ratio=1.106567 max_load_reduction=56471 max_load_reduction_pct=0.861679 moved_to_echo=31165 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 873, 3, 100, 0, 1657, 0, 3412, 2193, 79, 0, 0, 0, 0, 0, 2905, 106, 59, 2218, 1123, 396, 0, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 968, 1530, 333, 229, 1690, 1135, 122, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2907, 6, 925, 639, 1, 278, 1655, 1260, 807, 0, 0, 0, 0, 0, 0, 674, 20, 47, 2201, 2569, 5, 323, 2303, 0, 0, 0, 0, 0, 0, 0, 0, 1867, 165, 503, 574, 6, 1248, 1163, 2623, 0, 0, 0, 0, 0, 0, 0, 0, 61, 3869, 386, 0, 2, 0, 948, 589, 1674, 492, 0, 0, 0, 0, 0, 0, 0, 1377, 3340, 1094, 251, 1639, 843, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8321, 8345, 7532, 8478, 8142, 8149, 8021, 8548] avg=8192.0000 max=8548 max_rank=7 min=7532 min_rank=2 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=32860 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 8704, 128, 1536, 128, 10624, 128, 27264, 16896, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[5504, 128, 512, 14976, 22144, 128, 5120, 17408, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[14592, 3072, 4864, 4608, 128, 8832, 9856, 20096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 19456, 256, 8960, 4864, 128, 1792, 14336, 9856, 6528, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[21760, 640, 512, 18304, 8192, 4224, 128, 12288, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[10496, 10368, 4224, 3072, 13696, 10368, 1792, 11904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 10752, 25088, 7680, 2688, 10880, 8576, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1408, 30336, 3840, 128, 128, 128, 7808, 5376, 13312, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 2, 2, 6, 4091, 3, 3357, 23, 4068, 2, 46, 3101, 0, 2, 0, 2, 4078, 1, 0, 2, 0, 11, 7, 4091, 19, 4093, 2, 0, 2, 4058, 1720, 1, 0, 0, 0, 4095, 4085, 2, 0, 2, 4094, 4076, 0, 2, 20, 0, 4093, 4089, 0, 0, 0, 0, 0, 2, 1, 0, 4083, 2, 4096, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 0, 4096, 0, 2504, 0, 4076, 0, 1, 3271, 0, 4, 0, 0, 4090, 0, 10, 0, 1, 0, 0, 4096, 0, 4096, 3, 0, 1, 4096, 2420, 8, 0, 0, 15, 4095, 4094, 2, 0, 0, 4096, 4083, 0, 0, 0, 0, 4090, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4093, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 2207, 0, 4071, 0, 6, 3641, 5, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 39, 0, 0, 4096, 2320, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4, 4093, 3, 2619, 1, 4088, 5, 16, 2935, 1, 2, 0, 0, 4093, 2, 0, 0, 2, 2, 3, 4093, 0, 4092, 211, 1, 0, 4079, 2429, 0, 0, 3, 0, 4081, 4085, 0, 1, 0, 4091, 4092, 0, 2, 1, 1, 4090, 4094, 10, 4, 3, 0, 4, 3, 4, 3, 4095, 1, 4093, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 0, 1, 0, 4092, 0, 2606, 0, 4088, 0, 15, 3209, 0, 0, 0, 0, 4095, 0, 0, 0, 6, 3, 1, 4096, 0, 4094, 25, 1, 1, 4095, 2358, 3, 2, 0, 1, 4082, 4095, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 3, 0, 0, 4095, 0, 4086, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 3, 3477, 2, 4095, 0, 11, 3916, 1, 36, 0, 0, 4062, 0, 6, 0, 0, 4, 1, 4096, 0, 4096, 7, 0, 0, 4095, 756, 12, 0, 0, 0, 4089, 4096, 1, 0, 0, 4096, 4096, 0, 0, 8, 0, 4096, 4089, 0, 0, 0, 1, 0, 0, 0, 2, 4096, 0, 4094, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 1, 0, 0, 4096, 1, 3397, 0, 4064, 0, 0, 3915, 0, 0, 0, 0, 4096, 0, 3, 0, 3, 9, 0, 4096, 0, 4096, 221, 0, 0, 4089, 689, 3, 0, 0, 1, 4096, 4096, 2, 0, 0, 4096, 4096, 0, 0, 0, 0, 4085, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 4, 0, 0, 0, 0, 4092, 0, 3270, 0, 4091, 6, 6, 2944, 2, 2, 0, 2, 4078, 3, 0, 0, 0, 0, 0, 4090, 5, 4094, 13, 3, 7, 4090, 1964, 11, 0, 0, 16, 4082, 4082, 12, 0, 0, 4096, 4083, 0, 5, 0, 2, 4082, 4095, 4, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 4094, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 2, 2, 6, 4091, 2193, 1828, 66, 0, 0, 0, 0, 0, 3, 3357, 23, 1872, 2, 46, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4078, 1, 0, 2, 0, 4085, 3, 0, 0, 0, 0, 0, 0, 11, 7, 4091, 19, 4023, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 1720, 1, 0, 0, 0, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4094, 4076, 0, 2, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4089, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 4083, 2, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8404, 8173, 8155, 8046, 8196, 8186, 8186] avg=8192.0000 max=8404 max_rank=1 min=8046 min_rank=4 imbalance_ratio=1.025879 max_load_reduction=57132 max_load_reduction_pct=0.871765 moved_to_echo=32614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 1, 0, 4096, 2198, 1828, 67, 0, 0, 0, 0, 0, 0, 2504, 0, 1875, 0, 1, 3271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4090, 0, 10, 0, 1, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4025, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2420, 8, 0, 0, 15, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 3, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7651, 8202, 8125, 8806, 8181, 8193, 8187] avg=8192.0000 max=8806 max_rank=4 min=7651 min_rank=1 imbalance_ratio=1.074951 max_load_reduction=56730 max_load_reduction_pct=0.865631 moved_to_echo=33367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 2195, 1829, 67, 0, 0, 0, 0, 0, 0, 2207, 0, 1873, 0, 6, 3641, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4025, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2320, 1, 0, 0, 0, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 7732, 8196, 8160, 8684, 8191, 8195, 8191] avg=8192.0000 max=8684 max_rank=4 min=7732 min_rank=1 imbalance_ratio=1.060059 max_load_reduction=56852 max_load_reduction_pct=0.867493 moved_to_echo=33261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 0, 0, 0, 1, 0, 4092, 2204, 1823, 66, 0, 0, 0, 0, 0, 0, 2606, 0, 1881, 0, 15, 3209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 6, 4095, 3, 0, 0, 0, 0, 0, 0, 3, 1, 4096, 0, 4024, 25, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2358, 3, 2, 0, 1, 2259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4095, 0, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 7711, 8199, 8151, 8718, 8189, 8194, 8185] avg=8192.0000 max=8718 max_rank=4 min=7711 min_rank=1 imbalance_ratio=1.064209 max_load_reduction=56818 max_load_reduction_pct=0.866974 moved_to_echo=33286 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4, 4093, 2204, 1822, 66, 0, 0, 0, 0, 0, 3, 2619, 1, 1881, 5, 16, 2935, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 2, 0, 0, 2, 4085, 3, 0, 0, 0, 0, 0, 0, 2, 3, 4093, 0, 4022, 211, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 2429, 0, 0, 3, 0, 2259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 4092, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 4094, 10, 4, 3, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 3, 4, 3, 4095, 1, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 7461, 8187, 8332, 8770, 8187, 8210, 8200] avg=8192.0000 max=8770 max_rank=4 min=7461 min_rank=1 imbalance_ratio=1.070557 max_load_reduction=56766 max_load_reduction_pct=0.866180 moved_to_echo=33367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 2208, 1826, 67, 0, 0, 0, 0, 0, 3, 3477, 2, 1884, 0, 11, 3916, 1, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4062, 0, 6, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4, 1, 4096, 0, 4025, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 756, 12, 0, 0, 0, 2263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 9294, 8203, 8133, 7126, 8201, 8190, 8192] avg=8192.0000 max=9294 max_rank=1 min=7126 min_rank=4 imbalance_ratio=1.134521 max_load_reduction=56242 max_load_reduction_pct=0.858185 moved_to_echo=31709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 1, 0, 0, 4096, 2196, 1832, 71, 0, 0, 0, 0, 0, 1, 3397, 0, 1860, 0, 0, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 3, 4096, 8, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 4012, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 689, 3, 0, 0, 1, 2264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4090, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 9173, 8206, 8338, 7046, 8195, 8181, 8197] avg=8192.0000 max=9173 max_rank=1 min=7046 min_rank=4 imbalance_ratio=1.119751 max_load_reduction=56363 max_load_reduction_pct=0.860031 moved_to_echo=31619 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4, 0, 0, 0, 0, 4092, 2206, 1823, 66, 0, 0, 0, 0, 0, 0, 3270, 0, 1882, 6, 6, 2944, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4078, 3, 0, 0, 0, 4082, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 5, 4024, 13, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1964, 11, 0, 0, 16, 2259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 4083, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4082, 4095, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8110, 8170, 8142, 8340, 8196, 8187, 8198] avg=8192.0000 max=8340 max_rank=4 min=8110 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32921 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 14720, 128, 128, 128, 128, 18176, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 32256, 640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 32768, 17664, 14720, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 23552, 128, 15104, 128, 128, 27008, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3935, 4096, 0, 0, 0, 4087, 597, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4080, 5, 1, 4093, 0, 0, 0, 0, 4096, 4095, 0, 4088, 0, 2279, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 1394, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4028, 4096, 0, 0, 4, 4095, 1497, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 1, 4086, 0, 0, 0, 0, 4096, 4096, 0, 3931, 0, 2177, 0, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 665, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 3574, 4091, 0, 1, 16, 4050, 589, 4091, 0, 6, 4, 5, 0, 0, 0, 0, 0, 4, 2, 0, 4093, 0, 4092, 0, 0, 0, 4, 1, 0, 0, 3, 0, 3985, 28, 0, 4088, 0, 2, 0, 0, 4087, 4092, 1, 3897, 5, 1330, 0, 8, 4093, 2, 0, 4094, 4, 7, 4092, 3, 0, 8, 4089, 1, 2993, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4050, 4096, 0, 0, 15, 4095, 611, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 1, 4032, 21, 3, 4094, 0, 0, 0, 2, 4093, 4096, 0, 4095, 0, 2801, 0, 0, 4095, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 758, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4072, 4096, 1, 0, 7, 4086, 1281, 4095, 0, 2, 0, 0, 1, 0, 0, 0, 15, 0, 0, 1, 4090, 0, 4094, 2, 0, 0, 0, 0, 1, 0, 1, 0, 4012, 8, 33, 4091, 0, 0, 3, 0, 4092, 4096, 0, 3754, 1, 2940, 0, 3, 4083, 0, 0, 4092, 0, 4, 4095, 0, 0, 0, 4093, 0, 291, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4054, 4096, 0, 0, 3, 4096, 311, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 16, 43, 4094, 1, 0, 0, 0, 4096, 4096, 0, 3763, 0, 3387, 0, 0, 4084, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 0, 747, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2948, 4093, 0, 0, 1, 4066, 365, 4095, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4045, 9, 0, 4095, 1, 0, 0, 2, 4096, 4092, 0, 3817, 0, 3173, 1, 8, 4085, 0, 0, 4095, 0, 0, 4095, 0, 0, 1, 4086, 0, 2070, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4090, 4096, 0, 0, 0, 4095, 425, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4043, 12, 1, 4085, 0, 0, 0, 0, 4096, 4096, 0, 4072, 0, 3146, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 4096, 0, 597, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3935, 259, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 597, 4094, 0, 0, 0, 0, 0, 0, 2227, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3837, 258, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4080, 5, 1, 4093, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 0, 0, 4088, 0, 2279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2909, 0, 0, 0, 4096, 0, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8077, 8191, 8191, 8226, 7978, 8193, 8399] avg=8192.0000 max=8399 max_rank=7 min=7978 min_rank=5 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3574, 259, 0, 1, 16, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 589, 4091, 0, 6, 4, 5, 0, 0, 2223, 1158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 4093, 0, 3832, 257, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3985, 28, 0, 4088, 0, 2, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1607, 0, 1, 3897, 5, 1330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4093, 2, 0, 4094, 4, 7, 1, 0, 0, 0, 0, 0, 0, 0, 2906, 3, 0, 8, 4089, 1, 2993, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7901, 8076, 8188, 8189, 8133, 6840, 8209, 10000] avg=8192.0000 max=10000 max_rank=7 min=6840 min_rank=5 imbalance_ratio=1.220703 max_load_reduction=55536 max_load_reduction_pct=0.847412 moved_to_echo=33182 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4028, 259, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1497, 4093, 0, 0, 0, 0, 0, 0, 2227, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3837, 258, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 1, 4086, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 0, 0, 3931, 0, 2177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2909, 0, 0, 0, 4096, 0, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8386, 8976, 8191, 8192, 8211, 7719, 8191, 7670] avg=8192.0000 max=8976 max_rank=1 min=7670 min_rank=7 imbalance_ratio=1.095703 max_load_reduction=56560 max_load_reduction_pct=0.863037 moved_to_echo=31791 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4050, 259, 0, 0, 15, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 611, 4095, 0, 0, 0, 0, 0, 0, 2226, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3837, 258, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4032, 21, 3, 4094, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1609, 0, 0, 4095, 0, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2908, 0, 0, 0, 4096, 0, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8419, 8091, 8191, 8196, 8178, 8507, 8192, 7762] avg=8192.0000 max=8507 max_rank=5 min=7762 min_rank=7 imbalance_ratio=1.038452 max_load_reduction=57029 max_load_reduction_pct=0.870193 moved_to_echo=32639 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4072, 259, 1, 0, 7, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 1281, 4095, 0, 2, 0, 0, 1, 0, 2225, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 1, 4090, 0, 3837, 257, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4012, 8, 33, 4091, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1610, 0, 0, 3754, 1, 2940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4083, 0, 0, 4092, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 2908, 0, 0, 0, 4093, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8425, 8763, 8200, 8193, 8172, 8308, 8183, 7292] avg=8192.0000 max=8763 max_rank=1 min=7292 min_rank=7 imbalance_ratio=1.069702 max_load_reduction=56773 max_load_reduction_pct=0.866287 moved_to_echo=31955 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4054, 259, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 311, 4093, 0, 0, 0, 0, 0, 0, 2227, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3837, 258, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 16, 43, 4094, 1, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 0, 0, 3763, 0, 3387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 2909, 0, 0, 0, 4096, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8412, 7790, 8191, 8190, 8256, 8761, 8184, 7752] avg=8192.0000 max=8761 max_rank=5 min=7752 min_rank=7 imbalance_ratio=1.069458 max_load_reduction=56775 max_load_reduction_pct=0.866318 moved_to_echo=32953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2948, 259, 0, 0, 1, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 365, 4095, 1, 1, 1, 0, 0, 0, 2227, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3834, 258, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4045, 9, 0, 4095, 1, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1611, 0, 0, 3817, 0, 3173, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 4085, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2908, 0, 0, 1, 4086, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7274, 7849, 8188, 8189, 8178, 8603, 8190, 9065] avg=8192.0000 max=9065 max_rank=7 min=7274 min_rank=0 imbalance_ratio=1.106567 max_load_reduction=56471 max_load_reduction_pct=0.861679 moved_to_echo=34036 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 253, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 425, 4095, 0, 0, 0, 0, 0, 0, 2233, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3839, 261, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 10, 0, 4043, 12, 1, 4085, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1602, 0, 0, 4072, 0, 3146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2902, 0, 1, 0, 4096, 0, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8438, 7914, 8196, 8196, 8182, 8820, 8194, 7596] avg=8192.0000 max=8820 max_rank=5 min=7596 min_rank=7 imbalance_ratio=1.076660 max_load_reduction=56716 max_load_reduction_pct=0.865417 moved_to_echo=32792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 30848, 2176, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32384, 128, 128, 32768, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[23296, 128, 128, 128, 32768, 128, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 128, 32768, 0, 30720, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[5760, 32768, 128, 128, 128, 128, 128, 0, 17920, 9344, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 12928, 0, 128, 31488, 128, 21248, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 3559, 1, 0, 0, 4090, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3353, 4096, 47, 4096, 0, 0, 1, 4096, 4096, 0, 1234, 31, 4096, 0, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4088, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 3984, 0, 0, 4, 3745, 2, 0, 0, 4096, 0, 4090, 0, 1, 4067, 0, 0, 0, 0, 8, 0, 4087, 1, 4089, 1, 0, 0, 0, 0, 0, 4093, 0, 0, 3763, 4096, 0, 4096, 0, 0, 8, 4095, 4095, 2, 816, 16, 4094, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 4, 0, 8, 4032, 0, 5, 16, 4092, 22, 1, 3, 4, 0, 4091, 0, 0, 0, 3319, 0, 6, 16, 3943, 6, 4018, 0, 0, 4063, 2, 4, 5, 0, 0, 5, 4088, 6, 4091, 0, 0, 0, 0, 0, 4, 4088, 0, 0, 3765, 4090, 112, 4081, 4, 5, 0, 4091, 4091, 3, 1240, 1, 4091, 8, 0, 12, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4079, 0, 0, 0, 3663, 21, 0, 0, 4022, 0, 4095, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 4069, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2979, 4096, 178, 4096, 0, 0, 26, 4096, 4096, 0, 1377, 64, 4096, 0, 0, 44, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 3318, 2, 0, 0, 4093, 0, 4090, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3629, 4096, 244, 4096, 0, 0, 5, 4096, 4096, 0, 1045, 33, 4096, 0, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3751, 7, 0, 0, 3995, 0, 4095, 0, 0, 4076, 0, 0, 5, 0, 16, 0, 4079, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 3960, 4096, 49, 4095, 0, 0, 24, 4096, 4096, 0, 493, 33, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4054, 11, 0, 0, 4070, 0, 4064, 0, 0, 4072, 0, 0, 0, 0, 10, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3419, 4096, 63, 4096, 0, 0, 4, 4096, 4096, 0, 664, 25, 4096, 0, 0, 34, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 3777, 15, 0, 0, 4067, 0, 4094, 0, 0, 4080, 0, 0, 0, 0, 11, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2510, 4096, 270, 4096, 0, 0, 24, 4096, 4096, 0, 1240, 398, 4096, 0, 0, 1, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 8, 4032, 0, 5, 16, 4091, 5, 0, 0, 0, 0, 0, 0, 4092, 22, 1, 3, 4, 0, 4091, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3319, 0, 6, 16, 3943, 6, 419, 64, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 4063, 2, 4, 5, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4088, 6, 4091, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4088, 0, 0, 3765, 676, 0, 0, 0, 0, 0, 0, 0, 0, 112, 4081, 4, 5, 0, 3962, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1240, 1, 4091, 8, 0, 12, 0, 0, 2990, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8239, 7773, 8127, 8194, 8533, 8167, 8342] avg=8192.0000 max=8533 max_rank=5 min=7773 min_rank=2 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=33236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4073, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3559, 1, 0, 0, 4090, 0, 420, 64, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3353, 677, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4096, 0, 0, 1, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 31, 4096, 0, 0, 4, 0, 0, 2994, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8195, 8134, 8222, 8196, 8126, 8111, 8359] avg=8192.0000 max=8359 max_rank=7 min=8111 min_rank=6 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4025, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3318, 2, 0, 0, 4093, 0, 420, 64, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4094, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4091, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3629, 677, 0, 0, 0, 0, 0, 0, 0, 0, 244, 4096, 0, 0, 5, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 33, 4096, 0, 0, 2, 0, 0, 2994, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8147, 7897, 8219, 8193, 8402, 8312, 8170] avg=8192.0000 max=8402 max_rank=5 min=7897 min_rank=2 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=33077 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4088, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 3984, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3745, 2, 0, 0, 4096, 0, 420, 64, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 4067, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4087, 1, 4089, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3763, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 3966, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 816, 16, 4094, 0, 0, 0, 0, 0, 2994, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8108, 8331, 8193, 8190, 8533, 8072, 7920] avg=8192.0000 max=8533 max_rank=5 min=7920 min_rank=7 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=32715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4079, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3663, 21, 0, 0, 4022, 0, 420, 64, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4051, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2979, 677, 0, 0, 0, 0, 0, 0, 0, 0, 178, 4096, 0, 0, 26, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 64, 4096, 0, 0, 44, 0, 3, 2994, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8202, 8190, 8181, 8169, 7752, 8267, 8578] avg=8192.0000 max=8578 max_rank=7 min=7752 min_rank=5 imbalance_ratio=1.047119 max_load_reduction=56958 max_load_reduction_pct=0.869110 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 11, 0, 0, 4070, 0, 425, 70, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4072, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4085, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3419, 659, 0, 0, 0, 0, 0, 0, 0, 0, 63, 4096, 0, 0, 4, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 664, 25, 4096, 0, 0, 34, 0, 1, 3000, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8220, 8630, 8174, 8199, 8174, 8115, 7820] avg=8192.0000 max=8630 max_rank=2 min=7820 min_rank=7 imbalance_ratio=1.053467 max_load_reduction=56906 max_load_reduction_pct=0.868317 moved_to_echo=32308 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4092, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3751, 7, 0, 0, 3995, 0, 420, 64, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4076, 0, 0, 5, 0, 35, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4079, 0, 4096, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3960, 677, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4095, 0, 0, 24, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 33, 4096, 0, 0, 0, 0, 0, 2994, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8211, 8237, 8211, 8196, 8733, 8135, 7616] avg=8192.0000 max=8733 max_rank=5 min=7616 min_rank=7 imbalance_ratio=1.066040 max_load_reduction=56803 max_load_reduction_pct=0.866745 moved_to_echo=32680 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3777, 15, 0, 0, 4067, 0, 420, 64, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4080, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4088, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2510, 677, 0, 0, 0, 0, 0, 0, 0, 0, 270, 4096, 0, 0, 24, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1240, 398, 4096, 0, 0, 1, 0, 2, 2994, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8214, 8344, 8209, 8199, 7283, 8357, 8731] avg=8192.0000 max=8731 max_rank=7 min=7283 min_rank=5 imbalance_ratio=1.065796 max_load_reduction=56805 max_load_reduction_pct=0.866776 moved_to_echo=32570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 27392, 5504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 32768, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1024, 32768, 128, 128, 128, 31744, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 0, 128, 32640, 128, 128, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 29312, 128, 128, 128, 32384, 128, 3456, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 32640, 0, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8192, 640, 32768, 128, 0, 128, 0, 128, 24064, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[9, 0, 2853, 4090, 0, 3, 6, 5, 4091, 1, 4086, 3, 0, 5, 3, 0, 0, 4091, 2, 8, 3980, 0, 7, 0, 9, 4091, 0, 3, 4090, 6, 0, 4087, 4087, 4089, 0, 0, 4089, 15, 109, 4092, 8, 4, 0, 6, 4089, 4, 6, 1, 0, 2, 5, 0, 4079, 0, 0, 7, 1198, 5, 6, 0, 4091, 4, 6, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3538, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4010, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 50, 4096, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 559, 0, 18, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3653, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4090, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 4, 0, 4096, 0, 121, 4096, 37, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 221, 0, 54, 0, 4096, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3523, 4096, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 2, 4092, 0, 0, 4086, 0, 2, 0, 0, 4093, 0, 0, 4096, 1, 0, 4096, 4096, 4096, 2, 3, 4094, 0, 33, 4088, 176, 0, 0, 0, 4096, 0, 0, 0, 0, 13, 0, 0, 4095, 0, 0, 0, 182, 0, 188, 0, 4096, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3184, 4096, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4092, 0, 2, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 3, 0, 4095, 1, 132, 4094, 45, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 735, 0, 5, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3240, 4096, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4083, 0, 4, 0, 0, 4092, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 11, 0, 4096, 0, 257, 4083, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 578, 0, 23, 0, 4096, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3989, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3950, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 1, 39, 4094, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 154, 0, 54, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3934, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 4088, 0, 0, 4092, 0, 12, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 1, 4096, 22, 0, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 106, 0, 20, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 2853, 4090, 0, 3, 6, 5, 561, 42, 0, 0, 0, 0, 0, 0, 4091, 1, 4086, 3, 0, 5, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 8, 3980, 0, 7, 0, 47, 0, 0, 0, 0, 0, 0, 0, 9, 4091, 0, 3, 0, 6, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 15, 109, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 6, 4089, 4, 6, 1, 4041, 6, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 4079, 0, 0, 7, 4084, 0, 0, 0, 0, 0, 0, 0, 1198, 5, 6, 0, 4091, 4, 6, 5, 3526, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7569, 8191, 8135, 8196, 8214, 8165, 8177, 8889] avg=8192.0000 max=8889 max_rank=7 min=7569 min_rank=0 imbalance_ratio=1.085083 max_load_reduction=56647 max_load_reduction_pct=0.864365 moved_to_echo=33445 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3653, 4096, 0, 0, 0, 0, 563, 42, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4090, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4004, 0, 121, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 4096, 0, 0, 0, 4049, 7, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 221, 0, 54, 0, 4096, 0, 6, 0, 3533, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8354, 8195, 8227, 8193, 8225, 8189, 8196, 7957] avg=8192.0000 max=8354 max_rank=0 min=7957 min_rank=7 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32567 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3538, 4096, 0, 0, 0, 0, 562, 42, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4010, 0, 1, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 50, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 4048, 7, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 559, 0, 18, 0, 4096, 0, 0, 0, 3534, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8195, 8154, 8192, 8150, 8162, 8190, 8255] avg=8192.0000 max=8255 max_rank=7 min=8150 min_rank=4 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32757 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3523, 4096, 0, 0, 0, 0, 562, 42, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 0, 0, 4086, 0, 2, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4003, 0, 33, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 4096, 0, 0, 0, 4049, 7, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4095, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 182, 0, 188, 0, 4096, 0, 4, 0, 3534, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8189, 8229, 8190, 8129, 8328, 8197, 8051] avg=8192.0000 max=8328 max_rank=5 min=8051 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=32705 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3184, 4096, 0, 0, 0, 0, 563, 42, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4092, 0, 2, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4003, 1, 132, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 4096, 0, 0, 0, 4049, 7, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 735, 0, 5, 0, 4096, 0, 0, 0, 3533, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7885, 8188, 8234, 8192, 8233, 8197, 8191, 8416] avg=8192.0000 max=8416 max_rank=7 min=7885 min_rank=0 imbalance_ratio=1.027344 max_load_reduction=57120 max_load_reduction_pct=0.871582 moved_to_echo=33037 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3989, 4096, 0, 0, 0, 0, 562, 42, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3950, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 1, 39, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 4048, 7, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 154, 0, 54, 0, 4096, 0, 0, 0, 3534, 48, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8689, 8195, 8092, 8193, 8138, 8156, 8187, 7886] avg=8192.0000 max=8689 max_rank=0 min=7886 min_rank=7 imbalance_ratio=1.060669 max_load_reduction=56847 max_load_reduction_pct=0.867416 moved_to_echo=32367 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3240, 4096, 0, 0, 0, 0, 563, 42, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4083, 0, 4, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4004, 0, 257, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 0, 4049, 7, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 578, 0, 23, 0, 4096, 0, 3, 0, 3533, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7941, 8185, 8221, 8188, 8355, 8165, 8201, 8280] avg=8192.0000 max=8355 max_rank=4 min=7941 min_rank=0 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=33001 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3934, 4096, 0, 0, 0, 0, 563, 44, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4092, 0, 12, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 4049, 7, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 106, 0, 20, 0, 4096, 0, 0, 0, 3533, 47, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8637, 8198, 8244, 8192, 8092, 8174, 8197, 7802] avg=8192.0000 max=8637 max_rank=0 min=7802 min_rank=7 imbalance_ratio=1.054321 max_load_reduction=56899 max_load_reduction_pct=0.868210 moved_to_echo=32265 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 32384, 0, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 28032, 32768, 0, 128, 128, 128, 4608, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32768, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 128, 0, 128, 32768, 128, 128, 128, 32384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3840, 128, 384, 0, 32768, 128, 128, 128, 28288, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 32128, 128, 768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[6, 0, 0, 1, 6, 4, 0, 2259, 4091, 0, 4083, 5, 0, 0, 0, 0, 0, 7, 0, 4088, 0, 9, 4090, 7, 4090, 12, 0, 0, 4091, 2, 4087, 8, 4089, 4090, 1826, 4, 0, 0, 11, 0, 4088, 2, 0, 4090, 0, 4092, 0, 4087, 2, 4, 0, 4089, 0, 5, 2, 6, 11, 6, 7, 0, 4077, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 2, 0, 0, 0, 0, 1972, 4096, 0, 4096, 0, 192, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 0, 4096, 0, 4096, 4095, 2108, 0, 0, 0, 0, 0, 3933, 0, 0, 4096, 0, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2567, 4096, 0, 4096, 0, 92, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 1530, 0, 0, 0, 0, 0, 4002, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 5, 3083, 4096, 0, 4096, 0, 93, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 1000, 0, 0, 0, 0, 0, 4017, 0, 0, 4096, 0, 4085, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 5, 5, 0, 0, 0, 0, 2250, 4096, 0, 4096, 0, 111, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 5, 3, 4088, 0, 4096, 0, 4096, 4094, 1791, 0, 0, 0, 0, 0, 4027, 0, 0, 4096, 0, 4090, 0, 4093, 0, 0, 0, 4092, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 17, 2, 0, 0, 0, 7, 2846, 4092, 0, 4096, 0, 209, 0, 0, 8, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 0, 4096, 0, 4096, 4096, 1303, 0, 0, 0, 0, 0, 3836, 0, 0, 4096, 0, 4078, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2761, 4096, 0, 4096, 0, 66, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4095, 1359, 0, 0, 0, 0, 0, 4008, 0, 0, 4096, 0, 4088, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 22, 3323, 4096, 0, 4096, 0, 140, 0, 0, 2, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 0, 0, 0, 4090, 0, 4096, 0, 4096, 4094, 800, 0, 0, 0, 0, 0, 3899, 0, 0, 4096, 0, 4076, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 1, 6, 4, 0, 2259, 4090, 1451, 0, 0, 0, 0, 0, 0, 3980, 0, 4083, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4088, 0, 9, 4090, 7, 1, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0, 0, 4091, 2, 4087, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 4090, 1826, 4, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 4092, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4089, 0, 5, 2, 6, 2638, 1452, 0, 0, 0, 0, 0, 0, 11, 6, 7, 0, 4077, 0, 2, 0, 2506, 1461, 110, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7817, 8068, 8202, 8201, 8559, 8311, 8198, 8180] avg=8192.0000 max=8559 max_rank=4 min=7817 min_rank=0 imbalance_ratio=1.044800 max_load_reduction=56977 max_load_reduction_pct=0.869400 moved_to_echo=33248 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 2, 0, 0, 0, 0, 1972, 4096, 1454, 0, 0, 0, 0, 0, 0, 3985, 0, 4096, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2632, 4095, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 0, 0, 0, 4094, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2642, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 2512, 1464, 110, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7529, 8273, 8177, 8190, 8835, 8156, 8193, 8183] avg=8192.0000 max=8835 max_rank=4 min=7529 min_rank=0 imbalance_ratio=1.078491 max_load_reduction=56701 max_load_reduction_pct=0.865189 moved_to_echo=33367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2567, 4096, 1453, 0, 0, 0, 0, 0, 0, 3985, 0, 4096, 0, 92, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2632, 4096, 1530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 4096, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2643, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 2512, 1464, 110, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8116, 8174, 8190, 8192, 8258, 8227, 8194, 8185] avg=8192.0000 max=8258 max_rank=4 min=8116 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 5, 3083, 4096, 1453, 0, 0, 0, 0, 0, 0, 3985, 0, 4096, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2632, 4094, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 0, 0, 4085, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 2643, 1455, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 9, 2512, 1464, 110, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8639, 8174, 8193, 8192, 7726, 8231, 8189, 8192] avg=8192.0000 max=8639 max_rank=0 min=7726 min_rank=4 imbalance_ratio=1.054565 max_load_reduction=56897 max_load_reduction_pct=0.868179 moved_to_echo=32338 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 5, 0, 0, 0, 0, 2250, 4096, 1452, 0, 0, 0, 0, 0, 0, 3985, 0, 4096, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2632, 4094, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 4090, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 2643, 1454, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 14, 2510, 1464, 110, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7808, 8192, 8195, 8192, 8517, 8246, 8189, 8197] avg=8192.0000 max=8517 max_rank=4 min=7808 min_rank=0 imbalance_ratio=1.039673 max_load_reduction=57019 max_load_reduction_pct=0.870041 moved_to_echo=33149 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2761, 4096, 1454, 0, 0, 0, 0, 0, 0, 3976, 0, 4096, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2630, 4095, 1359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 4088, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2642, 1460, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 5, 2514, 1466, 113, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8311, 8138, 8199, 8192, 8084, 8218, 8198, 8196] avg=8192.0000 max=8311 max_rank=0 min=8084 min_rank=4 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32696 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 2, 0, 0, 0, 7, 2846, 4096, 1454, 0, 0, 0, 0, 0, 0, 3981, 0, 4096, 0, 209, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2632, 4096, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 0, 0, 4078, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 2642, 1455, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 1, 2512, 1464, 110, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8422, 8294, 8190, 8191, 8031, 8043, 8181, 8184] avg=8192.0000 max=8422 max_rank=0 min=8031 min_rank=4 imbalance_ratio=1.028076 max_load_reduction=57114 max_load_reduction_pct=0.871490 moved_to_echo=32439 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 22, 3323, 4096, 1453, 0, 0, 0, 0, 0, 0, 3985, 0, 4096, 0, 140, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2632, 4094, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 0, 0, 4076, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2643, 1455, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 9, 2512, 1464, 110, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8894, 8223, 8190, 8186, 7526, 8104, 8194, 8219] avg=8192.0000 max=8894 max_rank=0 min=7526 min_rank=4 imbalance_ratio=1.085693 max_load_reduction=56642 max_load_reduction_pct=0.864288 moved_to_echo=32043 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31872, 128, 0, 0, 0, 32768, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 21120, 32768, 11648, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[21120, 32768, 11776, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31872, 0, 32768, 128, 1024, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 128, 20096, 11776, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 0, 128, 128, 128, 21248, 11648, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[8, 0, 1, 5, 4, 6, 4084, 7, 1, 0, 22, 12, 0, 9, 0, 7, 0, 4084, 6, 4078, 1, 9, 0, 0, 1, 4085, 0, 4088, 8, 0, 5, 4086, 0, 2, 0, 8, 7, 0, 4090, 0, 3960, 4074, 0, 4085, 2, 7, 0, 4, 0, 4080, 8, 4, 0, 0, 3, 4089, 4089, 6, 7, 4088, 4089, 133, 4084, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4055, 4073, 39, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 1, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 453, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 3984, 4040, 226, 3561, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4092, 1, 4096, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 27, 0, 0, 0, 178, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 3960, 4055, 32, 4032, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 4096, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 56, 0, 0, 0, 71, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 3994, 4075, 10, 4065, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 17, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 62, 0, 0, 0, 234, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 3945, 4039, 38, 3950, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 19, 4096, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 60, 0, 0, 0, 116, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 3905, 4066, 97, 3965, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4086, 89, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 79, 0, 0, 0, 5, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 3, 0, 4096, 0, 0, 3, 0, 0, 0, 4096, 0, 4015, 4045, 9, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 4095, 0, 0, 4093, 4090, 0, 4096, 42] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 1, 5, 4, 6, 4084, 7, 3896, 178, 32, 0, 0, 0, 0, 0, 1, 0, 22, 12, 0, 9, 0, 7, 4089, 3909, 0, 0, 0, 0, 0, 0, 0, 4084, 6, 4078, 1, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 0, 2, 8, 0, 5, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8, 7, 0, 4090, 0, 3908, 179, 0, 0, 0, 0, 0, 0, 3960, 178, 0, 4085, 2, 7, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 8, 4, 0, 0, 3, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 4089, 133, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8049, 8180, 8187, 8194, 8236, 8184, 8285] avg=8192.0000 max=8285 max_rank=7 min=8049 min_rank=1 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32899 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3878, 180, 33, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 178, 0, 4096, 3918, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3916, 178, 0, 0, 0, 0, 0, 0, 3960, 177, 32, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4061, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8219, 8194, 8192, 8190, 8201, 8192, 8161] avg=8192.0000 max=8219 max_rank=1 min=8161 min_rank=7 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3895, 178, 33, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 17, 0, 4096, 3917, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3916, 179, 0, 0, 0, 0, 0, 0, 4055, 178, 39, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8046, 8193, 8194, 8191, 8360, 8192, 8158] avg=8192.0000 max=8360 max_rank=5 min=8046 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32901 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3864, 178, 33, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 453, 0, 4096, 3917, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 3918, 179, 0, 0, 0, 0, 0, 0, 3984, 176, 226, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 4061, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8489, 8193, 8192, 8195, 7947, 8192, 8157] avg=8192.0000 max=8489 max_rank=1 min=7947 min_rank=5 imbalance_ratio=1.036255 max_load_reduction=57047 max_load_reduction_pct=0.870468 moved_to_echo=32491 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3901, 180, 34, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 71, 0, 4096, 3918, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3916, 178, 0, 0, 0, 0, 0, 0, 3994, 174, 10, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8141, 8195, 8192, 8190, 8243, 8196, 8168] avg=8192.0000 max=8243 max_rank=5 min=8141 min_rank=1 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32797 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3863, 180, 33, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 234, 0, 4096, 3918, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3916, 178, 0, 0, 0, 0, 0, 0, 3945, 176, 38, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 19, 4061, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8310, 8194, 8191, 8190, 8109, 8192, 8178] avg=8192.0000 max=8310 max_rank=1 min=8109 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=57226 max_load_reduction_pct=0.873199 moved_to_echo=32669 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3889, 178, 33, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 116, 0, 4096, 3917, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3916, 179, 0, 0, 0, 0, 0, 0, 3905, 177, 97, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 89, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8189, 8194, 8194, 8191, 8144, 8192, 8236] avg=8192.0000 max=8236 max_rank=7 min=8144 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3869, 178, 33, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 5, 0, 4095, 3914, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 3917, 179, 0, 0, 0, 0, 0, 0, 4015, 176, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4061, 42, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8093, 8193, 8194, 8195, 8296, 8196, 8193] avg=8192.0000 max=8296 max_rank=5 min=8093 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32880 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 128, 31104, 1536, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31872, 1536, 512, 31872, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 32768, 384, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 384, 128, 0, 128, 1152, 128, 32768, 31360, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 0, 32768, 0, 31360, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[6, 0, 0, 14, 0, 6, 4031, 0, 8, 4089, 8, 7, 0, 0, 8, 0, 3935, 4085, 4090, 7, 0, 4081, 0, 3, 0, 4087, 0, 4088, 4, 0, 7, 7, 5, 4089, 4087, 0, 4090, 0, 6, 4084, 0, 0, 2, 4, 6, 5, 0, 10, 0, 5, 8, 0, 202, 0, 0, 4, 3, 0, 4083, 7, 4, 4091, 4086, 4084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4065, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 2985, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179, 0, 0, 0, 0, 0, 4096, 0, 0, 4056, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 3916, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 0, 4091, 0, 0, 4050, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 4011, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3541, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 4094, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3994, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 4093, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 3, 4096, 0, 0, 4056, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4053, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 3746, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 0, 0, 0, 0, 0, 4096, 0, 0, 4059, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 32, 0, 0, 3888, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 4096, 0, 0, 4041, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3986, 0, 0, 4096, 0, 0, 0, 45, 0, 0, 3884, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 0, 0, 0, 4096, 0, 0, 3933, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 14, 0, 6, 4031, 0, 3690, 434, 0, 0, 0, 0, 0, 0, 8, 4089, 8, 7, 0, 0, 8, 0, 3725, 345, 0, 0, 0, 0, 0, 0, 3935, 360, 0, 7, 0, 4081, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4088, 4, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4089, 2, 0, 0, 0, 6, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 6, 5, 0, 10, 4090, 4088, 0, 0, 0, 0, 0, 0, 0, 5, 8, 0, 202, 0, 0, 4, 4086, 3651, 2, 0, 0, 0, 0, 0, 3, 0, 4083, 7, 4, 4091, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8190, 8386, 8193, 8186, 8205, 7958, 8237] avg=8192.0000 max=8386 max_rank=2 min=7958 min_rank=6 imbalance_ratio=1.023682 max_load_reduction=57150 max_load_reduction_pct=0.872040 moved_to_echo=32586 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4065, 0, 3700, 435, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 3735, 346, 0, 0, 0, 0, 0, 0, 2985, 361, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179, 0, 0, 0, 4096, 3659, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4056, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8180, 7442, 8192, 8194, 8190, 8936, 8202] avg=8192.0000 max=8936 max_rank=6 min=7442 min_rank=2 imbalance_ratio=1.090820 max_load_reduction=56600 max_load_reduction_pct=0.863647 moved_to_echo=33522 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 4011, 0, 3700, 435, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3735, 346, 0, 0, 0, 0, 0, 0, 3541, 361, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 0, 0, 0, 4096, 3659, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8177, 7998, 8192, 8194, 8190, 8396, 8242] avg=8192.0000 max=8396 max_rank=6 min=7998 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=57140 max_load_reduction_pct=0.871887 moved_to_echo=33022 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4094, 0, 3700, 435, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 3735, 346, 0, 0, 0, 0, 0, 0, 3916, 361, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 4096, 3659, 2, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4050, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8198, 8373, 8192, 8194, 8190, 7969, 8191] avg=8192.0000 max=8373 max_rank=2 min=7969 min_rank=6 imbalance_ratio=1.022095 max_load_reduction=57163 max_load_reduction_pct=0.872238 moved_to_echo=32544 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 4094, 0, 3700, 435, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3735, 346, 0, 0, 0, 0, 0, 0, 3994, 361, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 4096, 3659, 2, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4056, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8173, 8451, 8192, 8193, 8190, 7901, 8205] avg=8192.0000 max=8451 max_rank=2 min=7901 min_rank=6 imbalance_ratio=1.031616 max_load_reduction=57085 max_load_reduction_pct=0.871048 moved_to_echo=32489 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3700, 435, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 32, 0, 0, 3735, 346, 0, 0, 0, 0, 0, 0, 3888, 361, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 4096, 3659, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4041, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8209, 8345, 8192, 8194, 8190, 7988, 8187] avg=8192.0000 max=8345 max_rank=2 min=7988 min_rank=6 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32559 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4053, 0, 3707, 436, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 3738, 351, 0, 0, 0, 0, 0, 0, 3746, 358, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 0, 0, 0, 4096, 3660, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4059, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8187, 8200, 8191, 8192, 8191, 8186, 8193] avg=8192.0000 max=8200 max_rank=2 min=8186 min_rank=6 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3986, 0, 3700, 435, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 45, 0, 0, 3735, 346, 0, 0, 0, 0, 0, 0, 3884, 361, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 0, 4096, 3659, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3933, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8121, 8222, 8341, 8192, 8189, 8190, 8202, 8079] avg=8192.0000 max=8341 max_rank=2 min=8079 min_rank=7 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32660 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 3584, 0, 0, 128, 32768, 29312, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 32384, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 128, 32512, 0, 29696, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 0, 29952, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[29952, 2944, 0, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[204, 0, 4096, 0, 0, 0, 0, 0, 3892, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 4096, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[863, 0, 4096, 0, 0, 0, 0, 0, 3232, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1359, 0, 4096, 0, 0, 0, 0, 0, 2737, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4092, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4, 0, 4087, 0, 7, 6, 0, 26, 3975, 4, 0, 4077, 0, 7, 10, 4087, 0, 9, 0, 11, 0, 0, 0, 3, 4089, 0, 3, 8, 4085, 4088, 4088, 10, 0, 2, 0, 0, 0, 1, 4089, 9, 4089, 7, 116, 0, 4087, 0, 0, 0, 7, 9, 4060, 14, 8, 4081, 4086, 4089, 4089, 7, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1149, 0, 4096, 5, 0, 0, 0, 0, 2947, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4091, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[517, 0, 4096, 0, 0, 0, 0, 0, 3580, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[225, 0, 4096, 2, 0, 0, 0, 0, 3881, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4083, 4092, 0, 0, 5, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[14, 0, 4096, 2, 0, 0, 0, 0, 4050, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4091, 4096, 0, 0, 30, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 4085, 4096, 0, 0, 11, 0, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[204, 0, 4096, 0, 0, 0, 0, 0, 3536, 13, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 4096, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7849, 8548, 8191, 8193, 8188, 8180, 8196, 8191] avg=8192.0000 max=8548 max_rank=1 min=7849 min_rank=0 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=32755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4087, 0, 7, 6, 0, 26, 3528, 13, 0, 0, 0, 0, 0, 0, 3975, 4, 0, 4077, 0, 7, 10, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 11, 0, 0, 0, 3, 4089, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 4085, 4088, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 4089, 9, 4075, 1, 0, 0, 0, 0, 0, 0, 4076, 7, 116, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 4060, 14, 8, 6, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 7, 2, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7671, 8632, 8197, 8196, 8177, 8286, 8193, 8184] avg=8192.0000 max=8632 max_rank=1 min=7671 min_rank=0 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=32840 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[863, 0, 4096, 0, 0, 0, 0, 0, 3536, 13, 0, 0, 0, 0, 0, 0, 3232, 0, 0, 4096, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8508, 7888, 8191, 8193, 8188, 8180, 8198, 8190] avg=8192.0000 max=8508 max_rank=0 min=7888 min_rank=1 imbalance_ratio=1.038574 max_load_reduction=57028 max_load_reduction_pct=0.870178 moved_to_echo=32756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1359, 0, 4096, 0, 0, 0, 0, 0, 3536, 13, 0, 0, 0, 0, 0, 0, 2737, 0, 0, 4096, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9004, 7393, 8191, 8193, 8192, 8179, 8194, 8190] avg=8192.0000 max=9004 max_rank=0 min=7393 min_rank=1 imbalance_ratio=1.099121 max_load_reduction=56532 max_load_reduction_pct=0.862610 moved_to_echo=32755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1149, 0, 4096, 5, 0, 0, 0, 0, 3536, 13, 0, 0, 0, 0, 0, 0, 2947, 0, 0, 4096, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8799, 7603, 8191, 8193, 8188, 8179, 8193, 8190] avg=8192.0000 max=8799 max_rank=0 min=7603 min_rank=1 imbalance_ratio=1.074097 max_load_reduction=56737 max_load_reduction_pct=0.865738 moved_to_echo=32750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 4096, 2, 0, 0, 0, 0, 3536, 13, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 4096, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 4096, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 6, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 6, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7661, 8706, 8191, 8187, 8218, 8179, 8186, 8208] avg=8192.0000 max=8706 max_rank=1 min=7661 min_rank=0 imbalance_ratio=1.062744 max_load_reduction=56830 max_load_reduction_pct=0.867157 moved_to_echo=32791 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[225, 0, 4096, 2, 0, 0, 0, 0, 3536, 13, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 4096, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 5, 0, 3, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7872, 8537, 8192, 8189, 8188, 8179, 8185, 8194] avg=8192.0000 max=8537 max_rank=1 min=7872 min_rank=0 imbalance_ratio=1.042114 max_load_reduction=56999 max_load_reduction_pct=0.869736 moved_to_echo=32746 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[517, 0, 4096, 0, 0, 0, 0, 0, 3541, 18, 0, 0, 0, 0, 0, 0, 3580, 0, 0, 4096, 0, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 2, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8229, 8192, 8192, 8197, 8174, 8191, 8189] avg=8192.0000 max=8229 max_rank=1 min=8172 min_rank=0 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[28416, 128, 0, 32768, 0, 128, 128, 4480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4352, 0, 32768, 128, 128, 128, 0, 128, 28288, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 0, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 128, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4087, 13, 4081, 13, 54, 2, 0, 4085, 1, 0, 8, 0, 4088, 4090, 0, 4087, 3, 9, 9, 1, 4083, 7, 11, 4083, 3, 10, 4040, 7, 0, 0, 3620, 0, 5, 8, 4089, 6, 4, 3258, 4084, 13, 1, 8, 6, 0, 0, 0, 0, 4086, 11, 0, 9, 9, 0, 4088, 1275, 0, 0, 0, 4081, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4082, 0, 51, 0, 4009, 0, 0, 0, 4096, 0, 0, 2138, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2008, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4088, 0, 16, 41, 3566, 0, 0, 0, 4096, 0, 0, 2840, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1737, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4095, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4017, 0, 0, 7, 3689, 0, 0, 0, 4096, 0, 0, 3111, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1464, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4095, 11, 4096, 0, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 4091, 0, 68, 13, 2714, 0, 0, 0, 4096, 0, 0, 2530, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2869, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4083, 0, 0, 4096, 0, 19, 19, 3894, 0, 0, 0, 4096, 0, 0, 1444, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2830, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0, 12, 25, 3619, 0, 0, 0, 4096, 0, 0, 2153, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2386, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4091, 4089, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 4086, 0, 0, 4093, 0, 24, 19, 3671, 0, 0, 0, 4096, 0, 0, 829, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3673, 0, 0, 0, 4096, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 13, 4081, 13, 54, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 8, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 3, 9, 9, 1, 4083, 7, 11, 0, 0, 0, 0, 0, 0, 0, 0, 477, 3, 10, 4040, 7, 0, 0, 3620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 1805, 6, 4, 3258, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 8, 6, 0, 0, 0, 0, 4087, 3606, 481, 0, 0, 0, 0, 0, 4086, 11, 0, 9, 9, 0, 1811, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 2277, 1803, 7, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8243, 8185, 8208, 8157, 9170, 8202, 7201, 8170] avg=8192.0000 max=9170 max_rank=4 min=7201 min_rank=6 imbalance_ratio=1.119385 max_load_reduction=56366 max_load_reduction_pct=0.860077 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 0, 4088, 0, 16, 41, 3566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1808, 0, 0, 2840, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3617, 481, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1814, 1737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2282, 1807, 7, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8193, 8190, 8190, 8744, 8193, 7647, 8194] avg=8192.0000 max=8744 max_rank=4 min=7647 min_rank=6 imbalance_ratio=1.067383 max_load_reduction=56792 max_load_reduction_pct=0.866577 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 0, 4017, 0, 0, 7, 3689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1808, 0, 0, 3111, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3617, 481, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1814, 1464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2282, 1807, 7, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8193, 8190, 8192, 9015, 8193, 7374, 8194] avg=8192.0000 max=9015 max_rank=4 min=7374 min_rank=6 imbalance_ratio=1.100464 max_load_reduction=56521 max_load_reduction_pct=0.862442 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 0, 4082, 0, 51, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1808, 0, 0, 2138, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3617, 481, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1814, 2008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2282, 1807, 7, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8193, 8190, 8621, 8042, 8193, 7918, 8194] avg=8192.0000 max=8621 max_rank=3 min=7918 min_rank=6 imbalance_ratio=1.052368 max_load_reduction=56915 max_load_reduction_pct=0.868454 moved_to_echo=32347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 0, 0, 4096, 0, 19, 19, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1808, 0, 0, 1444, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3606, 481, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1814, 2830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2282, 1807, 7, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8192, 8190, 8505, 7348, 8182, 8740, 8194] avg=8192.0000 max=8740 max_rank=6 min=7348 min_rank=4 imbalance_ratio=1.066895 max_load_reduction=56796 max_load_reduction_pct=0.866638 moved_to_echo=32464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 4091, 0, 68, 13, 2714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1808, 0, 0, 2530, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3611, 481, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1814, 2869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2282, 1807, 7, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8202, 8190, 7364, 8434, 8188, 8779, 8194] avg=8192.0000 max=8779 max_rank=6 min=7364 min_rank=3 imbalance_ratio=1.071655 max_load_reduction=56757 max_load_reduction_pct=0.866043 moved_to_echo=33595 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 1, 4089, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 0, 0, 4093, 0, 24, 19, 3671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1808, 0, 0, 829, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3609, 481, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1814, 3673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 2282, 1807, 7, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8186, 8190, 8284, 6733, 8180, 9583, 8195] avg=8192.0000 max=9583 max_rank=6 min=6733 min_rank=4 imbalance_ratio=1.169800 max_load_reduction=55953 max_load_reduction_pct=0.853775 moved_to_echo=32691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4087, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 4093, 0, 12, 25, 3619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1801, 0, 0, 2153, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3622, 487, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1812, 2386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2284, 1808, 9, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8192, 8188, 8223, 8050, 8205, 8294, 8201] avg=8192.0000 max=8294 max_rank=6 min=8050 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32750 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 14464, 128, 128, 18304, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 32768, 0, 128, 0, 0, 18304, 14464, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 14592, 18304, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 0, 0, 0, 32768, 28928, 3968, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3840, 128, 128, 32640, 128, 256, 128, 28800, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4048, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 30, 21, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4079, 0, 0, 0, 0, 4091, 0, 4096, 0, 4096, 4016, 0, 0, 0, 2, 4096, 4096, 4093, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 1, 0, 23, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 79, 3, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4085, 4088, 9, 8, 3, 13, 4091, 0, 4086, 0, 4082, 4023, 5, 0, 0, 0, 4084, 4087, 4087, 0, 41, 4086, 4087, 0, 1, 0, 0, 5, 4070, 2, 26, 9, 8, 0, 0, 0, 0, 9, 9, 10, 26, 4069, 26, 0, 4088, 4058, 157, 129, 0, 0, 0, 0, 3823, 7, 0, 0, 9, 10, 0, 9, 11, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 4096, 0, 0, 4096, 4096, 10, 1, 0, 1, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4072, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4091, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 40, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4085, 0, 0, 0, 0, 4094, 0, 4096, 0, 4096, 4066, 0, 0, 0, 1, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 29, 5, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4077, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4084, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 67, 3, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 4066, 0, 0, 0, 0, 4089, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 30, 2, 0, 0, 0, 4096, 0, 0, 4096, 4096, 109, 0, 1, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 4088, 9, 8, 3, 13, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4082, 4023, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4087, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 4070, 2, 26, 4022, 64, 2, 0, 0, 0, 0, 0, 9, 8, 0, 0, 0, 0, 9, 9, 4087, 4065, 3, 0, 0, 0, 0, 0, 10, 26, 4005, 26, 0, 0, 4058, 157, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 0, 3823, 7, 0, 4073, 31, 21, 0, 0, 0, 0, 0, 0, 9, 10, 0, 9, 11, 0, 0, 4088, 4084, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8143, 8210, 8192, 8190, 8282, 8084, 8211] avg=8192.0000 max=8282 max_rank=5 min=8084 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32767 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4096, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4031, 64, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 3, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4093, 0, 0, 4083, 31, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8178, 8190, 8193, 8174, 8158, 8249, 8189] avg=8192.0000 max=8249 max_rank=6 min=8158 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32770 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4031, 64, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 4096, 4075, 4, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4083, 0, 0, 4083, 31, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8225, 8190, 8184, 8187, 8138, 8220, 8188] avg=8192.0000 max=8225 max_rank=1 min=8138 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32733 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 4072, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4031, 64, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 4096, 4075, 3, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 4096, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 4083, 31, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8221, 8190, 8193, 8199, 8168, 8195, 8189] avg=8192.0000 max=8221 max_rank=1 min=8168 min_rank=5 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32751 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 4085, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4031, 64, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4096, 4075, 4, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4089, 0, 0, 4083, 31, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8196, 8191, 8192, 8191, 8157, 8229, 8188] avg=8192.0000 max=8229 max_rank=6 min=8157 min_rank=5 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32765 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 4079, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4096, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4031, 64, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 23, 0, 0, 4096, 4075, 3, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 4096, 79, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 4096, 0, 0, 4083, 31, 21, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8146, 8190, 8184, 8198, 8207, 8238, 8190] avg=8192.0000 max=8238 max_rank=6 min=8146 min_rank=1 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32833 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4077, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4037, 70, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 4076, 4, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 4096, 67, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4047, 0, 0, 4090, 38, 20, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8201, 8185, 8205, 8190, 8189, 8198, 8189] avg=8192.0000 max=8205 max_rank=3 min=8179 min_rank=0 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32766 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 4066, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4031, 64, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 2, 0, 4096, 4075, 4, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 4096, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3987, 0, 0, 4083, 31, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8226, 8190, 8193, 8207, 8237, 8123, 8192] avg=8192.0000 max=8237 max_rank=5 min=8123 min_rank=6 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32759 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32256, 128, 0, 0, 32768, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 128, 32768, 128, 128, 32256, 640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 128, 128, 128, 32768, 32640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 384, 0, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 128, 128, 0, 0, 32384, 128, 0, 32768, 256, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4063, 0, 0, 17, 10, 0, 4, 0, 0, 4082, 4086, 8, 0, 0, 11, 6, 0, 4085, 0, 1, 980, 3923, 22, 0, 2, 4085, 0, 7, 9, 11, 0, 4085, 0, 11, 4079, 4087, 5, 4085, 0, 0, 35, 0, 0, 0, 0, 4088, 0, 3377, 12, 11, 4077, 13, 5, 0, 0, 10, 0, 10, 347, 4087, 4083, 0, 3609, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 2836, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 661, 0, 0, 0, 0, 4096, 0, 968, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 419, 4089, 4096, 0, 3309, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 3081, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 217, 0, 0, 0, 0, 4096, 0, 1286, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 47, 4080, 4096, 0, 3563, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2824, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 198, 0, 0, 0, 0, 4096, 0, 1439, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 19, 4096, 4096, 0, 3712, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 1780, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 835, 0, 0, 0, 0, 4096, 0, 1784, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 50, 4083, 4096, 0, 3742, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 2875, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 440, 0, 0, 0, 0, 4096, 0, 1254, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 4083, 4096, 0, 3615, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 1120, 4096, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 133, 0, 0, 0, 0, 4096, 0, 3191, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 4090, 4096, 0, 3715, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 3710, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 809, 0, 0, 0, 0, 4096, 0, 1342, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 28, 4078, 4096, 0, 2303, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4063, 0, 0, 17, 10, 0, 4, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 4082, 4086, 8, 0, 0, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1700, 0, 1, 980, 3923, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4085, 0, 7, 9, 11, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4079, 2, 5, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 4088, 0, 3377, 1845, 0, 0, 0, 0, 0, 0, 0, 12, 11, 4077, 13, 5, 0, 0, 10, 3540, 540, 0, 0, 0, 0, 0, 0, 0, 10, 347, 4087, 541, 0, 3609, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8193, 6626, 8199, 8182, 9345, 8208, 8602] avg=8192.0000 max=9345 max_rank=5 min=6626 min_rank=2 imbalance_ratio=1.140747 max_load_reduction=56191 max_load_reduction_pct=0.857407 moved_to_echo=34337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1705, 6, 0, 2836, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 0, 0, 0, 0, 4096, 0, 968, 1850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3551, 541, 0, 0, 0, 0, 0, 0, 0, 0, 419, 4089, 542, 0, 3309, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8643, 8192, 8194, 7575, 8188, 8359] avg=8192.0000 max=8643 max_rank=2 min=7575 min_rank=5 imbalance_ratio=1.055054 max_load_reduction=56893 max_load_reduction_pct=0.868118 moved_to_echo=32316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1705, 0, 0, 2824, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 4096, 0, 1439, 1850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 3551, 541, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 542, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8625, 8192, 8194, 7583, 8189, 8369] avg=8192.0000 max=8625 max_rank=2 min=7583 min_rank=5 imbalance_ratio=1.052856 max_load_reduction=56911 max_load_reduction_pct=0.868393 moved_to_echo=32335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1705, 14, 0, 3081, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 0, 0, 0, 4096, 0, 1286, 1850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3551, 541, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4080, 542, 0, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8896, 8192, 8194, 7449, 8188, 8232] avg=8192.0000 max=8896 max_rank=2 min=7449 min_rank=5 imbalance_ratio=1.085938 max_load_reduction=56640 max_load_reduction_pct=0.864258 moved_to_echo=32063 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1705, 14, 0, 1780, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 835, 0, 0, 0, 0, 4096, 0, 1784, 1850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3551, 541, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4083, 542, 0, 3742, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 7595, 8192, 8194, 8565, 8188, 8417] avg=8192.0000 max=8565 max_rank=5 min=7595 min_rank=2 imbalance_ratio=1.045532 max_load_reduction=56971 max_load_reduction_pct=0.869308 moved_to_echo=33364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1705, 16, 0, 1120, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 4096, 0, 3191, 1850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3551, 541, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4090, 542, 0, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 6943, 8192, 8194, 9270, 8188, 8364] avg=8192.0000 max=9270 max_rank=5 min=6943 min_rank=2 imbalance_ratio=1.131592 max_load_reduction=56266 max_load_reduction_pct=0.858551 moved_to_echo=34016 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1694, 14, 0, 2875, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 4096, 0, 1254, 1854, 7, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3555, 548, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4083, 537, 0, 3615, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8191, 8679, 8185, 8192, 7652, 8199, 8242] avg=8192.0000 max=8679 max_rank=2 min=7652 min_rank=5 imbalance_ratio=1.059448 max_load_reduction=56857 max_load_reduction_pct=0.867569 moved_to_echo=32285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1705, 16, 0, 3710, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 809, 0, 0, 0, 0, 4096, 0, 1342, 1850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3551, 541, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4078, 542, 0, 2303, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 9529, 8192, 8192, 8097, 8188, 6951] avg=8192.0000 max=9529 max_rank=2 min=6951 min_rank=7 imbalance_ratio=1.163208 max_load_reduction=56007 max_load_reduction_pct=0.854599 moved_to_echo=31428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 1024, 32768, 4352, 0, 27648, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 32768, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 128, 128, 28416, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 13696, 128, 128, 19328, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[3328, 0, 0, 0, 0, 32768, 0, 14720, 14848, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 2, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 118, 0, 4095, 0, 1, 0, 0, 4084, 4096, 4089, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 126, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[7, 0, 4086, 11, 8, 4, 12, 0, 0, 4084, 10, 0, 4084, 14, 4084, 4081, 13, 0, 6, 4085, 10, 11, 0, 15, 15, 12, 0, 0, 0, 1, 8, 0, 0, 4081, 4085, 0, 3979, 8, 0, 0, 13, 5, 29, 0, 11, 4071, 4, 12, 0, 1, 128, 0, 4086, 6, 0, 0, 0, 4072, 4086, 4034, 0, 4085, 4079, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 87, 0, 4096, 0, 0, 0, 0, 4090, 4096, 4094, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 37, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 77, 0, 4096, 0, 0, 0, 0, 4089, 4096, 4054, 9, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4091, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 78, 0, 4096, 0, 4, 0, 0, 4094, 4096, 4059, 3, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 4095, 0, 1, 0, 0, 4096, 4096, 4088, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 4093, 0, 0, 0, 0, 4091, 4096, 4094, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 2, 0, 0, 4070, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 6, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 4095, 0, 1, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4089, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8194, 8188, 8190, 8158, 8187, 8232, 8199] avg=8192.0000 max=8232 max_rank=6 min=8158 min_rank=4 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32776 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 4086, 11, 8, 4, 12, 0, 4053, 9, 8, 0, 0, 0, 0, 0, 0, 2, 10, 0, 0, 14, 4084, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 6, 4085, 10, 11, 0, 15, 4078, 1, 0, 0, 0, 0, 0, 0, 15, 12, 0, 0, 0, 1, 8, 0, 4086, 4084, 0, 0, 0, 0, 0, 0, 0, 4081, 70, 0, 3979, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 5, 29, 0, 11, 4071, 4, 12, 4073, 6, 0, 0, 0, 0, 0, 0, 0, 1, 128, 0, 4086, 6, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4034, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8191, 8219, 8206, 8138, 8224, 8228, 8132] avg=8192.0000 max=8228 max_rank=6 min=8132 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32722 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4070, 9, 9, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 6, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 4096, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8194, 8188, 8190, 8135, 8187, 8240, 8218] avg=8192.0000 max=8240 max_rank=6 min=8135 min_rank=4 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4070, 9, 9, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 6, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 4096, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4094, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8194, 8188, 8190, 8182, 8187, 8201, 8210] avg=8192.0000 max=8210 max_rank=7 min=8182 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=57326 max_load_reduction_pct=0.874725 moved_to_echo=32780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4070, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4095, 0, 1, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8192, 8188, 8190, 8261, 8189, 8122, 8210] avg=8192.0000 max=8261 max_rank=4 min=8122 min_rank=6 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32782 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4070, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 6, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 4096, 0, 4, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4059, 3, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8187, 8188, 8190, 8220, 8189, 8196, 8182] avg=8192.0000 max=8220 max_rank=4 min=8182 min_rank=7 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32787 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4074, 11, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 64, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 5, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4093, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8189, 8190, 8255, 8186, 8126, 8207] avg=8192.0000 max=8255 max_rank=4 min=8126 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32774 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 37, 0, 0, 4070, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 6, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 4096, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4054, 9, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8192, 8188, 8190, 8187, 8187, 8191, 8178] avg=8192.0000 max=8223 max_rank=0 min=8178 min_rank=7 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32743 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 32768, 128, 128, 128, 128, 0, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 640, 0, 32256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 640, 0, 32768, 128, 128, 0, 32256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 32640, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 8, 3998, 12, 4081, 0, 4076, 0, 16, 0, 44, 0, 0, 4079, 0, 127, 4047, 11, 2177, 0, 4082, 4085, 0, 4067, 7, 4050, 0, 0, 8, 40, 10, 30, 13, 6, 20, 14, 0, 1, 0, 0, 0, 9, 11, 15, 0, 4070, 161, 4084, 0, 1930, 4, 9, 0, 4087, 20, 12, 4087, 0, 10, 9, 4079, 0, 3829] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4096, 0, 4069, 0, 4096, 4096, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 27, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4096, 0, 3848, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 249, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3999, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 100, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 4056, 0, 4096, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 37, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 68, 4096, 0, 2781, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 1300, 0, 0, 0, 4096, 0, 0, 4085, 0, 0, 0, 4096, 0, 4049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4095, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3390, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 3, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 713, 0, 0, 0, 4093, 0, 0, 4088, 0, 0, 0, 4096, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 4070, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 27, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 8, 3998, 12, 12, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 44, 0, 0, 4079, 0, 4036, 39, 0, 0, 0, 0, 0, 0, 127, 4047, 11, 2177, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 7, 4050, 0, 0, 8, 40, 10, 1, 0, 0, 0, 0, 0, 0, 0, 30, 13, 6, 20, 14, 0, 1, 0, 4085, 4069, 1, 0, 0, 0, 0, 0, 0, 0, 9, 11, 15, 0, 4070, 161, 3517, 545, 0, 0, 0, 0, 0, 0, 4084, 0, 1930, 4, 9, 0, 3541, 20, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4087, 0, 10, 9, 42, 0, 3829, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8107, 8214, 6888, 8183, 8239, 8328, 9588, 7989] avg=8192.0000 max=9588 max_rank=6 min=6888 min_rank=2 imbalance_ratio=1.170410 max_load_reduction=55948 max_load_reduction_pct=0.853699 moved_to_echo=34144 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 13, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4052, 39, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 3848, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3529, 546, 0, 0, 0, 0, 0, 0, 4096, 0, 249, 0, 0, 0, 3549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 43, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8187, 8477, 8193, 8185, 8171, 7894, 8224] avg=8192.0000 max=8477 max_rank=2 min=7894 min_rank=6 imbalance_ratio=1.034790 max_load_reduction=57059 max_load_reduction_pct=0.870651 moved_to_echo=32474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 13, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4052, 39, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3999, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3529, 546, 0, 0, 0, 0, 0, 0, 4095, 0, 100, 0, 0, 0, 3549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 43, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8187, 8623, 8193, 8181, 8171, 7744, 8232] avg=8192.0000 max=8623 max_rank=2 min=7744 min_rank=6 imbalance_ratio=1.052612 max_load_reduction=56913 max_load_reduction_pct=0.868423 moved_to_echo=32328 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 13, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4052, 39, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4069, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3529, 546, 0, 0, 0, 0, 0, 0, 4095, 0, 27, 0, 0, 0, 3549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 43, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8187, 8696, 8186, 8191, 8171, 7671, 8229] avg=8192.0000 max=8696 max_rank=2 min=7671 min_rank=6 imbalance_ratio=1.061523 max_load_reduction=56840 max_load_reduction_pct=0.867310 moved_to_echo=32262 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 13, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4052, 39, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4056, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3529, 546, 0, 0, 0, 0, 0, 0, 4096, 0, 37, 0, 0, 0, 3549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 43, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8187, 8681, 8192, 8183, 8171, 7682, 8235] avg=8192.0000 max=8681 max_rank=2 min=7682 min_rank=6 imbalance_ratio=1.059692 max_load_reduction=56855 max_load_reduction_pct=0.867538 moved_to_echo=32271 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 13, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4052, 39, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3390, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3529, 546, 0, 0, 0, 0, 0, 0, 4092, 0, 713, 0, 0, 0, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 43, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8187, 8014, 8196, 8186, 8171, 8351, 8225] avg=8192.0000 max=8351 max_rank=6 min=8014 min_rank=2 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32933 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 13, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4052, 39, 0, 0, 0, 0, 0, 0, 68, 4096, 0, 2781, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 3529, 546, 0, 0, 0, 0, 0, 0, 4096, 0, 1300, 0, 0, 0, 3549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 43, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8187, 7473, 8193, 8183, 8173, 8945, 8177] avg=8192.0000 max=8945 max_rank=6 min=7473 min_rank=2 imbalance_ratio=1.091919 max_load_reduction=56591 max_load_reduction_pct=0.863510 moved_to_echo=33478 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4058, 46, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4070, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3534, 550, 0, 0, 0, 0, 0, 0, 4096, 0, 27, 0, 0, 0, 3538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 36, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8200, 8684, 8200, 8188, 8180, 7661, 8225] avg=8192.0000 max=8684 max_rank=2 min=7661 min_rank=6 imbalance_ratio=1.060059 max_load_reduction=56852 max_load_reduction_pct=0.867493 moved_to_echo=32254 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 4480, 128, 128, 0, 28416, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 384, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[256, 32768, 128, 28416, 0, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 32768, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 0, 32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 0, 32768, 256, 28288, 4480, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 400, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4065, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 1028, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3944, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3224, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3991, 5, 0, 512, 0, 4096, 4096, 3, 4096, 0, 0, 0, 0, 4084, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3692, 0, 4096, 0, 0, 4096, 0, 4096, 4095, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4, 4084, 0, 0, 5, 0, 4074, 12, 3, 5, 11, 6, 7, 0, 4088, 0, 0, 115, 5, 4077, 4083, 0, 4078, 0, 0, 35, 12, 4052, 0, 4084, 0, 74, 17, 4009, 17, 12, 0, 9, 0, 0, 12, 4002, 0, 4082, 15, 0, 4081, 11, 4086, 4085, 0, 0, 4076, 0, 10, 0, 10, 13, 0, 7, 4056, 0, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 471, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4082, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3657, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 635, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3515, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 3933, 5, 0, 1132, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3676, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3528, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 670, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3877, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3646, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 41, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 0, 4096, 0, 0, 3479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3944, 0, 4096, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3581, 511, 0, 0, 0, 0, 0, 0, 0, 0, 3224, 0, 472, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4055, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8226, 8603, 8146, 8187, 7792, 8192, 8194] avg=8192.0000 max=8603 max_rank=2 min=7792 min_rank=5 imbalance_ratio=1.050171 max_load_reduction=56933 max_load_reduction_pct=0.868729 moved_to_echo=32365 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4084, 0, 0, 5, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3, 5, 11, 6, 7, 0, 4088, 4083, 41, 0, 0, 0, 0, 0, 0, 0, 0, 115, 5, 4077, 0, 0, 3464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 12, 4052, 0, 4084, 0, 106, 0, 0, 0, 0, 0, 0, 0, 74, 17, 4009, 17, 12, 0, 9, 0, 3569, 508, 0, 0, 0, 0, 0, 0, 0, 12, 4002, 0, 470, 15, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 4085, 0, 0, 4076, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 13, 0, 7, 4056, 0, 12, 4043, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8256, 7661, 8289, 8215, 8580, 8184, 8184] avg=8192.0000 max=8580 max_rank=5 min=7661 min_rank=2 imbalance_ratio=1.047363 max_load_reduction=56956 max_load_reduction_pct=0.869080 moved_to_echo=33163 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 41, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 4096, 0, 0, 3479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 4096, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3581, 511, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 472, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4053, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8232, 7975, 8267, 8188, 8296, 8194, 8192] avg=8192.0000 max=8296 max_rank=5 min=7975 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32870 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4096, 41, 0, 0, 0, 0, 0, 0, 0, 0, 471, 0, 4096, 0, 0, 3479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3581, 511, 0, 0, 0, 0, 0, 0, 0, 0, 3657, 0, 472, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4053, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8214, 8046, 8284, 8188, 8225, 8194, 8191] avg=8192.0000 max=8284 max_rank=3 min=8046 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 4096, 41, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 4096, 0, 0, 3479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3581, 511, 0, 0, 0, 0, 0, 0, 0, 0, 3515, 0, 472, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4055, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8177, 8210, 8297, 8188, 8083, 8192, 8191] avg=8192.0000 max=8297 max_rank=3 min=8083 min_rank=5 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32654 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 4096, 41, 0, 0, 0, 0, 0, 0, 5, 0, 512, 0, 4096, 0, 3, 3479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 2, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3581, 511, 0, 0, 0, 0, 0, 0, 0, 0, 3692, 0, 472, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4054, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8128, 8095, 8288, 8188, 8260, 8196, 8193] avg=8192.0000 max=8288 max_rank=3 min=8095 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32837 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 4096, 41, 0, 0, 0, 0, 0, 0, 5, 0, 1132, 0, 4096, 0, 0, 3479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3676, 0, 4096, 1, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3581, 511, 0, 0, 0, 0, 0, 0, 0, 0, 3528, 0, 472, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4055, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8070, 8712, 7879, 8188, 8096, 8192, 8193] avg=8192.0000 max=8712 max_rank=2 min=7879 min_rank=3 imbalance_ratio=1.063477 max_load_reduction=56824 max_load_reduction_pct=0.867065 moved_to_echo=32669 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 43, 0, 0, 0, 0, 0, 0, 0, 0, 670, 0, 4096, 0, 0, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 4096, 0, 113, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 3585, 512, 0, 0, 0, 0, 0, 0, 0, 0, 3646, 0, 462, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4053, 49, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8233, 8234, 8086, 8194, 8204, 8192, 8198] avg=8192.0000 max=8234 max_rank=2 min=8086 min_rank=3 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32788 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 0, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 29056, 0, 3840, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 4992, 128, 32768, 0, 128, 27904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 32768, 0, 128, 32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 128, 32000, 0, 32768, 128, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 32512, 32768, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 128, 0, 28672, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[13, 16, 8, 0, 7, 4090, 0, 0, 18, 2, 2, 0, 2, 11, 0, 4082, 0, 4078, 8, 4074, 4, 4083, 4078, 2482, 4085, 6, 4086, 49, 0, 13, 4073, 4083, 0, 6, 0, 63, 0, 4090, 0, 4081, 12, 4074, 10, 0, 4076, 11, 4, 8, 13, 0, 0, 6, 0, 14, 4085, 0, 0, 1496, 1, 11, 0, 0, 11, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 1535, 4096, 0, 4096, 4, 0, 254, 4096, 4095, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2322, 0, 0, 5, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 1002, 4096, 0, 4096, 0, 0, 103, 4096, 4096, 0, 0, 0, 0, 0, 4061, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3021, 0, 0, 27, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4094, 4096, 1914, 4096, 0, 4094, 0, 0, 104, 4096, 4095, 0, 0, 0, 0, 1, 4075, 1, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 2107, 0, 0, 22, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 4096, 4096, 1694, 4096, 0, 4096, 1, 0, 263, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2138, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 4096, 0, 4096, 4096, 1118, 4096, 0, 4096, 6, 0, 135, 4096, 4095, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2874, 0, 0, 16, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4096, 0, 4096, 4096, 2106, 4096, 0, 4096, 17, 0, 383, 4096, 4091, 0, 0, 0, 0, 0, 4071, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1675, 0, 0, 12, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 1360, 4096, 0, 4096, 8, 0, 176, 4096, 4096, 0, 0, 0, 0, 0, 4086, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2585, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[13, 16, 8, 0, 7, 4090, 0, 0, 2284, 1821, 0, 0, 0, 0, 0, 0, 18, 2, 2, 0, 2, 11, 0, 4082, 2262, 1818, 0, 0, 0, 0, 0, 0, 0, 4078, 8, 2435, 4, 0, 0, 2482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 49, 0, 13, 4073, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 63, 0, 4090, 0, 4081, 6, 0, 0, 0, 0, 0, 0, 0, 12, 4074, 10, 0, 4076, 11, 4, 8, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 6, 0, 14, 4085, 0, 2260, 1639, 177, 0, 0, 0, 0, 0, 0, 1496, 1, 11, 0, 0, 11, 11, 4085, 1802, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8197, 9007, 8041, 8246, 8195, 8194, 7417] avg=8192.0000 max=9007 max_rank=2 min=7417 min_rank=7 imbalance_ratio=1.099487 max_load_reduction=56529 max_load_reduction_pct=0.862564 moved_to_echo=32052 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4076, 0, 0, 2290, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2268, 1827, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2449, 0, 0, 0, 1535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 254, 4096, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2269, 1647, 178, 0, 0, 0, 0, 0, 0, 2322, 0, 0, 5, 0, 0, 0, 4096, 1806, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8191, 8080, 8265, 8194, 8193, 8190, 8229] avg=8192.0000 max=8265 max_rank=3 min=8080 min_rank=2 imbalance_ratio=1.008911 max_load_reduction=57271 max_load_reduction_pct=0.873886 moved_to_echo=32806 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4074, 0, 0, 2289, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2269, 1826, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2449, 0, 0, 0, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 4096, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4096, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2270, 1647, 178, 0, 0, 0, 0, 0, 0, 3021, 0, 0, 27, 0, 0, 0, 4096, 1807, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8191, 7547, 8111, 8164, 8191, 8191, 8951] avg=8192.0000 max=8951 max_rank=7 min=7547 min_rank=2 imbalance_ratio=1.092651 max_load_reduction=56585 max_load_reduction_pct=0.863419 moved_to_echo=33497 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 2290, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2269, 1827, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2449, 0, 0, 0, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 263, 4096, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2269, 1647, 178, 0, 0, 0, 0, 0, 0, 2138, 0, 0, 1, 0, 0, 0, 4096, 1806, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8192, 8238, 8272, 8199, 8191, 8190, 8041] avg=8192.0000 max=8272 max_rank=3 min=8041 min_rank=7 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32621 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4065, 0, 0, 2290, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2269, 1827, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2449, 0, 0, 0, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 135, 4096, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2269, 1647, 178, 0, 0, 0, 0, 0, 0, 2874, 0, 0, 16, 0, 0, 0, 4096, 1806, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8192, 7658, 8148, 8183, 8191, 8190, 8792] avg=8192.0000 max=8792 max_rank=7 min=7658 min_rank=2 imbalance_ratio=1.073242 max_load_reduction=56744 max_load_reduction_pct=0.865845 moved_to_echo=33356 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4066, 0, 0, 2289, 1826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2268, 1827, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2449, 0, 0, 0, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 4096, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4075, 1, 4096, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 2269, 1647, 178, 0, 0, 0, 0, 0, 0, 2107, 0, 0, 22, 0, 0, 0, 4096, 1805, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8191, 8459, 8111, 8180, 8191, 8193, 8030] avg=8192.0000 max=8459 max_rank=2 min=8030 min_rank=7 imbalance_ratio=1.032593 max_load_reduction=57077 max_load_reduction_pct=0.870926 moved_to_echo=32594 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4030, 0, 0, 2290, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2268, 1827, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2449, 0, 0, 0, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 383, 4096, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4096, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2269, 1647, 177, 0, 0, 0, 0, 0, 0, 1675, 0, 0, 12, 0, 0, 0, 4096, 1806, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8191, 8649, 8404, 8174, 8192, 8189, 7589] avg=8192.0000 max=8649 max_rank=2 min=7589 min_rank=7 imbalance_ratio=1.055786 max_load_reduction=56887 max_load_reduction_pct=0.868027 moved_to_echo=32144 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4071, 0, 0, 2290, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2268, 1827, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2442, 0, 0, 0, 1360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 176, 4096, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 12, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2269, 1654, 180, 0, 0, 0, 0, 0, 0, 2585, 0, 0, 0, 0, 0, 0, 4096, 1806, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8191, 7898, 8184, 8196, 8192, 8199, 8487] avg=8192.0000 max=8487 max_rank=7 min=7898 min_rank=2 imbalance_ratio=1.036011 max_load_reduction=57049 max_load_reduction_pct=0.870499 moved_to_echo=33074 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 32640, 0, 0, 18432, 14720, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 0, 1536, 32768, 31360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 18304, 128, 128, 128, 0, 128, 128, 32768, 14464, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 19584, 128, 0, 0, 13312, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 0, 128, 32768, 128, 18176, 13184, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 32768, 18176, 14720, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True + [2026-08-30 04:59:29] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 726.0 | throughput per GPU (TFLOP/s/GPU): 459.2 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.033628E+01 | loss scale: 1.0 | grad norm: 29.287 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[24, 1, 66, 0, 9, 0, 1654, 3, 1113, 226, 16, 2199, 1779, 2320, 0, 3171, 2579, 81, 8, 593, 7, 2581, 154, 1203, 1, 63, 148, 1899, 1917, 1, 670, 2069, 4051, 13, 204, 3314, 2953, 0, 945, 266, 4056, 5, 1335, 2575, 3, 96, 1090, 829, 812, 3570, 958, 0, 1, 6, 1, 3001, 0, 1328, 470, 888, 1366, 2063, 2748, 34] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[10, 0, 58, 1, 9, 0, 1857, 0, 687, 216, 11, 1524, 1654, 1899, 2, 3543, 3611, 201, 13, 615, 11, 2820, 183, 1085, 0, 65, 168, 1883, 1805, 1, 685, 2519, 4076, 9, 280, 2987, 2097, 1, 1119, 220, 4063, 0, 1202, 2272, 1, 112, 1820, 1334, 631, 3817, 863, 0, 0, 1, 0, 2973, 0, 1250, 628, 529, 1530, 1733, 2802, 50] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[51, 1, 26, 2, 11, 0, 1568, 2, 869, 200, 3, 1940, 1840, 2198, 2, 3470, 3241, 117, 25, 648, 2, 3178, 138, 1263, 3, 23, 197, 1896, 1600, 7, 652, 2321, 4090, 31, 198, 3038, 2134, 4, 1475, 465, 4050, 0, 1100, 2299, 1, 155, 1457, 784, 818, 3542, 729, 1, 3, 5, 7, 3171, 1, 836, 715, 785, 1865, 1861, 2299, 123] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[38, 5, 26, 12, 9, 3, 1545, 0, 815, 118, 33, 1670, 2333, 1901, 2, 3553, 2835, 197, 13, 535, 4, 2526, 176, 1354, 25, 16, 146, 2095, 1995, 5, 541, 2175, 4073, 25, 272, 2251, 1870, 2, 1563, 1014, 4033, 4, 1457, 2148, 8, 157, 1335, 1270, 855, 3640, 1444, 0, 16, 15, 8, 3026, 1, 1429, 762, 551, 1422, 1633, 2475, 76] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[44, 1, 86, 10, 20, 3, 1672, 1, 555, 111, 27, 1362, 2573, 2129, 3, 3516, 3185, 91, 24, 1209, 1, 2183, 222, 982, 0, 26, 213, 2387, 1609, 4, 654, 1794, 4091, 63, 144, 2778, 1787, 6, 1709, 771, 4048, 3, 1517, 2123, 5, 169, 1778, 1428, 954, 3457, 1087, 5, 2, 5, 27, 2776, 0, 956, 535, 396, 1692, 1763, 2627, 137] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[45, 2, 59, 4, 13, 1, 1469, 3, 835, 170, 13, 1836, 1732, 2138, 3, 3624, 3521, 90, 19, 515, 2, 2594, 205, 1310, 5, 64, 176, 2269, 1464, 2, 590, 2006, 4091, 5, 165, 2690, 2287, 4, 1313, 409, 4066, 1, 1369, 2590, 3, 275, 1084, 1400, 1250, 3220, 1058, 0, 2, 1, 23, 2715, 1, 722, 592, 706, 1541, 2365, 2737, 72] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[37, 1, 48, 2, 23, 0, 1208, 0, 440, 209, 6, 1676, 2197, 2366, 0, 3648, 3699, 129, 13, 653, 1, 2628, 169, 1366, 1, 25, 175, 1969, 1943, 5, 572, 1810, 4092, 27, 204, 2723, 2302, 1, 1565, 957, 4080, 0, 1180, 2422, 0, 90, 1324, 1584, 608, 3531, 780, 1, 1, 2, 10, 3072, 0, 865, 519, 403, 1519, 1725, 2867, 63] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[127, 6, 61, 0, 132, 0, 1342, 8, 1201, 142, 9, 957, 2025, 1842, 0, 3417, 3417, 200, 96, 517, 1, 2232, 255, 1032, 4, 138, 200, 1821, 1398, 9, 740, 1727, 4079, 27, 572, 2850, 1502, 7, 1671, 388, 4050, 0, 1818, 3033, 1, 166, 1581, 1754, 939, 3796, 1237, 14, 4, 5, 6, 2850, 2, 550, 707, 635, 1703, 1689, 2626, 218] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[24, 1, 66, 0, 9, 0, 1654, 3, 3075, 2557, 780, 0, 0, 0, 0, 0, 1113, 226, 16, 2199, 1779, 2320, 0, 1296, 0, 0, 0, 0, 0, 0, 0, 0, 2579, 81, 8, 593, 7, 2581, 154, 1203, 126, 0, 0, 0, 0, 0, 0, 0, 1, 63, 148, 1899, 1917, 1, 670, 2069, 1095, 226, 87, 0, 0, 0, 0, 0, 976, 13, 204, 3314, 2953, 0, 945, 266, 0, 0, 0, 0, 0, 0, 0, 0, 1499, 5, 1335, 2575, 3, 96, 1090, 829, 0, 0, 0, 0, 0, 0, 0, 0, 812, 3344, 958, 0, 1, 6, 1, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1328, 470, 888, 1366, 2063, 2535, 34, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 8949, 7332, 8176, 8671, 7432, 8123, 8684] avg=8192.0000 max=8949 max_rank=1 min=7332 min_rank=2 imbalance_ratio=1.092407 max_load_reduction=56587 max_load_reduction_pct=0.863449 moved_to_echo=32910 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 58, 1, 9, 0, 1857, 0, 3094, 2561, 871, 0, 0, 0, 0, 0, 687, 216, 11, 1524, 1654, 1899, 2, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 3611, 201, 13, 615, 11, 2820, 183, 1085, 129, 0, 0, 0, 0, 0, 0, 0, 0, 65, 168, 1883, 1805, 1, 685, 2519, 1224, 242, 88, 0, 0, 0, 0, 0, 982, 9, 280, 2987, 2097, 1, 1119, 220, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 0, 1202, 2272, 1, 112, 1820, 1334, 0, 0, 0, 0, 0, 0, 0, 0, 631, 3575, 863, 0, 0, 1, 0, 2973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 628, 529, 1530, 1733, 2585, 50, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8461, 7441, 8668, 8680, 7695, 8243, 8043, 8305] avg=8192.0000 max=8680 max_rank=3 min=7441 min_rank=1 imbalance_ratio=1.059570 max_load_reduction=56856 max_load_reduction_pct=0.867554 moved_to_echo=32286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[51, 1, 26, 2, 11, 0, 1568, 2, 3105, 2553, 853, 0, 0, 0, 0, 0, 869, 200, 3, 1940, 1840, 2198, 2, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 3241, 117, 25, 648, 2, 3178, 138, 1263, 106, 0, 0, 0, 0, 0, 0, 0, 3, 23, 197, 1896, 1600, 7, 652, 2321, 1199, 225, 72, 0, 0, 0, 0, 0, 985, 31, 198, 3038, 2134, 4, 1475, 465, 0, 0, 0, 0, 0, 0, 0, 0, 1497, 0, 1100, 2299, 1, 155, 1457, 784, 0, 0, 0, 0, 0, 0, 0, 0, 818, 3317, 729, 1, 3, 5, 7, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 1, 836, 715, 785, 1865, 1861, 2121, 123, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8470, 8718, 8195, 8330, 7293, 8051, 8307] avg=8192.0000 max=8718 max_rank=2 min=7293 min_rank=5 imbalance_ratio=1.064209 max_load_reduction=56818 max_load_reduction_pct=0.866974 moved_to_echo=31981 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[44, 1, 86, 10, 20, 3, 1672, 1, 3106, 2552, 865, 0, 0, 0, 0, 0, 555, 111, 27, 1362, 2573, 2129, 3, 1436, 0, 0, 0, 0, 0, 0, 0, 0, 3185, 91, 24, 1209, 1, 2183, 222, 982, 121, 0, 0, 0, 0, 0, 0, 0, 0, 26, 213, 2387, 1609, 4, 654, 1794, 1215, 219, 83, 0, 0, 0, 0, 0, 985, 63, 144, 2778, 1787, 6, 1709, 771, 0, 0, 0, 0, 0, 0, 0, 0, 1496, 3, 1517, 2123, 5, 169, 1778, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 954, 3238, 1087, 5, 2, 5, 27, 2776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 956, 535, 396, 1692, 1763, 2423, 137, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8360, 8196, 8018, 8204, 8243, 8519, 8094, 7902] avg=8192.0000 max=8519 max_rank=5 min=7902 min_rank=7 imbalance_ratio=1.039917 max_load_reduction=57017 max_load_reduction_pct=0.870010 moved_to_echo=32758 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[45, 2, 59, 4, 13, 1, 1469, 3, 3106, 2563, 891, 0, 0, 0, 0, 0, 835, 170, 13, 1836, 1732, 2138, 3, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 3521, 90, 19, 515, 2, 2594, 205, 1310, 126, 0, 0, 0, 0, 0, 0, 0, 5, 64, 176, 2269, 1464, 2, 590, 2006, 1252, 204, 86, 0, 0, 0, 0, 0, 985, 5, 165, 2690, 2287, 4, 1313, 409, 0, 0, 0, 0, 0, 0, 0, 0, 1503, 1, 1369, 2590, 3, 275, 1084, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 3016, 1058, 0, 2, 1, 23, 2715, 0, 0, 0, 0, 0, 0, 0, 0, 1, 722, 592, 706, 1541, 2365, 2525, 72, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8208, 8382, 8118, 7858, 8225, 8065, 8524] avg=8192.0000 max=8524 max_rank=7 min=7858 min_rank=4 imbalance_ratio=1.040527 max_load_reduction=57012 max_load_reduction_pct=0.869934 moved_to_echo=32672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[38, 5, 26, 12, 9, 3, 1545, 0, 3092, 2542, 874, 0, 0, 0, 0, 0, 815, 118, 33, 1670, 2333, 1901, 2, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 197, 13, 535, 4, 2526, 176, 1354, 114, 0, 0, 0, 0, 0, 0, 0, 25, 16, 146, 2095, 1995, 5, 541, 2175, 1227, 231, 78, 0, 0, 0, 0, 0, 981, 25, 272, 2251, 1870, 2, 1563, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 1491, 4, 1457, 2148, 8, 157, 1335, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 855, 3409, 1444, 0, 16, 15, 8, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1429, 762, 551, 1422, 1633, 2283, 76, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8324, 7754, 8534, 7978, 7870, 8773, 8157] avg=8192.0000 max=8773 max_rank=6 min=7754 min_rank=2 imbalance_ratio=1.070923 max_load_reduction=56763 max_load_reduction_pct=0.866135 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[127, 6, 61, 0, 132, 0, 1342, 8, 3100, 2557, 845, 0, 0, 0, 0, 0, 1201, 142, 9, 957, 2025, 1842, 0, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 3417, 200, 96, 517, 1, 2232, 255, 1032, 124, 0, 0, 0, 0, 0, 0, 0, 4, 138, 200, 1821, 1398, 9, 740, 1727, 1185, 245, 88, 0, 0, 0, 0, 0, 979, 27, 572, 2850, 1502, 7, 1671, 388, 0, 0, 0, 0, 0, 0, 0, 0, 1493, 0, 1818, 3033, 1, 166, 1581, 1754, 0, 0, 0, 0, 0, 0, 0, 0, 939, 3551, 1237, 14, 4, 5, 6, 2850, 0, 0, 0, 0, 0, 0, 0, 0, 2, 550, 707, 635, 1703, 1689, 2414, 218, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 7563, 7874, 7555, 7996, 9846, 8606, 7918] avg=8192.0000 max=9846 max_rank=5 min=7555 min_rank=3 imbalance_ratio=1.201904 max_load_reduction=55690 max_load_reduction_pct=0.849762 moved_to_echo=34366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[37, 1, 48, 2, 23, 0, 1208, 0, 3106, 2572, 897, 0, 0, 0, 0, 0, 440, 209, 6, 1676, 2197, 2366, 0, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 3699, 129, 13, 653, 1, 2628, 169, 1366, 132, 0, 0, 0, 0, 0, 0, 0, 1, 25, 175, 1969, 1943, 5, 572, 1810, 1260, 224, 90, 0, 0, 0, 0, 0, 986, 27, 204, 2723, 2302, 1, 1565, 957, 0, 0, 0, 0, 0, 0, 0, 0, 1508, 0, 1180, 2422, 0, 90, 1324, 1584, 0, 0, 0, 0, 0, 0, 0, 0, 608, 3307, 780, 1, 1, 2, 10, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 865, 519, 403, 1519, 1725, 2645, 63, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7894, 8385, 8790, 8074, 8765, 8108, 7781, 7739] avg=8192.0000 max=8790 max_rank=2 min=7739 min_rank=7 imbalance_ratio=1.072998 max_load_reduction=56746 max_load_reduction_pct=0.865875 moved_to_echo=32393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 128, 512, 128, 256, 128, 12416, 128, 24832, 20480, 6912, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26112, 1152, 256, 5376, 128, 20864, 1536, 9600, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12032, 128, 11008, 19584, 128, 1280, 11520, 10496, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 7936, 4992, 4992, 12672, 14848, 19584, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 512, 1536, 16256, 13824, 128, 5120, 16512, 9728, 1920, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6528, 1408, 128, 13184, 16256, 16896, 128, 11520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7936, 256, 2048, 22656, 17024, 128, 11392, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6912, 26880, 8192, 128, 128, 128, 128, 23680, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4086, 1, 0, 0, 25, 4079, 2, 4095, 3, 757, 0, 4, 0, 0, 0, 4093, 3590, 1, 2, 0, 1, 0, 0, 4030, 4, 1, 4093, 0, 4096, 4080, 0, 7, 0, 4, 4095, 0, 2, 4090, 4092, 694, 50, 2, 1, 4091, 0, 1, 0, 20, 0, 0, 3140, 0, 4, 0, 6, 0, 0, 1, 3, 4096, 3, 4089, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4087, 0, 0, 0, 8, 4079, 0, 4096, 0, 929, 2, 0, 0, 0, 0, 4096, 2648, 1, 0, 11, 1, 0, 1, 4094, 1, 0, 4096, 0, 4094, 4082, 0, 64, 0, 0, 4094, 2, 2, 4093, 4094, 684, 12, 2, 0, 4095, 0, 0, 1, 72, 0, 0, 3796, 2, 2, 1, 1, 0, 1, 5, 0, 4094, 0, 4093, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4084, 0, 1, 2, 37, 4087, 2, 4085, 6, 849, 1, 15, 0, 0, 7, 4049, 2445, 3, 1, 427, 0, 0, 11, 4094, 3, 1, 4088, 0, 4089, 4037, 1, 46, 7, 13, 4085, 4, 0, 4084, 4092, 905, 1, 0, 6, 4092, 0, 1, 0, 29, 7, 10, 3577, 8, 0, 3, 0, 0, 0, 0, 0, 4080, 0, 4055, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4084, 0, 0, 2, 24, 4069, 0, 4092, 0, 1176, 2, 0, 0, 0, 0, 4094, 2068, 2, 0, 66, 0, 0, 0, 4093, 2, 1, 4093, 0, 4094, 4092, 0, 82, 0, 0, 4093, 1, 0, 4093, 4090, 887, 8, 2, 2, 4091, 0, 0, 0, 41, 1, 1, 3889, 4, 3, 2, 3, 0, 5, 0, 3, 4090, 3, 4084, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4096, 3, 0, 0, 2, 4052, 0, 4096, 0, 942, 0, 11, 0, 0, 3, 3895, 2832, 0, 0, 51, 4, 0, 0, 4048, 0, 0, 4090, 59, 4096, 4078, 0, 204, 0, 1, 4077, 0, 31, 4090, 4093, 514, 1, 0, 0, 4096, 18, 0, 0, 1, 0, 0, 3919, 0, 0, 0, 0, 0, 22, 0, 3, 4015, 0, 4093, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4071, 3, 3, 0, 3, 4075, 0, 4090, 0, 538, 4, 4, 0, 1, 0, 4092, 3918, 1, 3, 3, 0, 0, 4, 4096, 1, 4, 4090, 0, 4091, 4093, 0, 9, 2, 0, 4092, 2, 0, 4090, 4089, 279, 15, 3, 0, 4091, 1, 3, 4, 7, 1, 1, 3438, 0, 0, 0, 0, 1, 19, 5, 3, 4095, 1, 4093, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 14, 4095, 0, 4095, 0, 991, 0, 1, 1, 1, 1, 4096, 3205, 0, 0, 60, 0, 0, 0, 4096, 0, 1, 4096, 1, 4095, 4085, 1, 178, 1, 0, 4095, 0, 0, 4095, 4096, 386, 2, 0, 0, 4096, 0, 0, 0, 37, 0, 0, 3368, 0, 1, 0, 0, 0, 0, 3, 0, 4096, 0, 4052, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4082, 0, 0, 0, 2, 4079, 0, 4092, 0, 1096, 2, 1, 0, 0, 0, 4086, 2355, 0, 0, 2, 0, 0, 0, 3969, 4, 2, 4096, 0, 4096, 4090, 0, 85, 0, 0, 4092, 0, 3, 4095, 4095, 1150, 4, 2, 0, 4094, 0, 0, 0, 25, 1, 6, 3643, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4090, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4086, 1, 0, 0, 25, 4079, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 3, 757, 0, 4, 0, 0, 0, 4093, 3212, 0, 0, 0, 0, 0, 0, 0, 3590, 1, 2, 0, 1, 0, 0, 4030, 686, 416, 59, 0, 0, 0, 0, 0, 4, 1, 4018, 0, 0, 4080, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 2, 3404, 0, 694, 0, 0, 0, 0, 0, 0, 0, 0, 50, 2, 1, 4091, 0, 1, 0, 20, 3599, 452, 0, 0, 0, 0, 0, 0, 0, 0, 3140, 0, 4, 0, 6, 0, 4096, 493, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4096, 3, 4089, 1, 1, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8069, 8785, 8110, 8199, 8216, 7739, 8210] avg=8192.0000 max=8785 max_rank=2 min=7739 min_rank=6 imbalance_ratio=1.072388 max_load_reduction=56751 max_load_reduction_pct=0.865952 moved_to_echo=32364 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 0, 0, 8, 4079, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 2, 0, 0, 0, 0, 4096, 3213, 0, 0, 0, 0, 0, 0, 0, 2648, 1, 0, 11, 1, 0, 1, 4094, 686, 416, 59, 0, 0, 0, 0, 0, 1, 0, 4021, 0, 0, 4082, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 2, 3407, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2, 0, 4095, 0, 0, 1, 72, 3600, 452, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 2, 2, 1, 1, 0, 4094, 494, 0, 0, 0, 0, 0, 0, 1, 5, 0, 4094, 0, 4093, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8240, 7917, 8168, 8189, 8234, 8390, 8209] avg=8192.0000 max=8390 max_rank=6 min=7917 min_rank=2 imbalance_ratio=1.024170 max_load_reduction=57146 max_load_reduction_pct=0.871979 moved_to_echo=33022 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 0, 2, 24, 4069, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 2, 0, 0, 0, 0, 4094, 3210, 0, 0, 0, 0, 0, 0, 0, 2068, 2, 0, 66, 0, 0, 0, 4093, 686, 416, 59, 0, 0, 0, 0, 0, 2, 1, 4018, 0, 0, 4092, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 3407, 0, 887, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 2, 4091, 0, 0, 0, 41, 3597, 452, 0, 0, 0, 0, 0, 0, 1, 1, 3889, 4, 3, 2, 3, 0, 4094, 493, 0, 0, 0, 0, 0, 0, 5, 0, 3, 4090, 3, 4084, 0, 4, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8482, 7390, 8195, 8388, 8193, 8490, 8205] avg=8192.0000 max=8490 max_rank=6 min=7390 min_rank=2 imbalance_ratio=1.036377 max_load_reduction=57046 max_load_reduction_pct=0.870453 moved_to_echo=33276 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 1, 2, 37, 4087, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 6, 849, 1, 15, 0, 0, 7, 4049, 3204, 0, 0, 0, 0, 0, 0, 0, 2445, 3, 1, 427, 0, 0, 11, 4094, 685, 415, 59, 0, 0, 0, 0, 0, 3, 1, 4013, 0, 0, 4037, 1, 46, 0, 0, 0, 0, 0, 0, 0, 0, 7, 13, 4085, 4, 0, 3399, 0, 905, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 4092, 0, 1, 0, 29, 3598, 451, 0, 0, 0, 0, 0, 0, 7, 10, 3577, 8, 0, 3, 0, 0, 4089, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4055, 0, 6, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8131, 8140, 8101, 8413, 8178, 8188, 8157] avg=8192.0000 max=8413 max_rank=4 min=8101 min_rank=3 imbalance_ratio=1.026978 max_load_reduction=57123 max_load_reduction_pct=0.871628 moved_to_echo=32936 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4071, 3, 3, 0, 3, 4075, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 4, 4, 0, 1, 0, 4092, 3208, 0, 0, 0, 0, 0, 0, 0, 3918, 1, 3, 3, 0, 0, 4, 4096, 686, 416, 59, 0, 0, 0, 0, 0, 1, 4, 4015, 0, 0, 4093, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 2, 0, 3404, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3, 0, 4091, 1, 3, 4, 7, 3597, 452, 0, 0, 0, 0, 0, 0, 1, 1, 3438, 0, 0, 0, 0, 1, 4091, 492, 0, 0, 0, 0, 0, 0, 19, 5, 3, 4095, 1, 4093, 0, 4, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8169, 7847, 9186, 8122, 7779, 8173, 8024, 8236] avg=8192.0000 max=9186 max_rank=2 min=7779 min_rank=4 imbalance_ratio=1.121338 max_load_reduction=56350 max_load_reduction_pct=0.859833 moved_to_echo=32212 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4096, 3, 0, 0, 2, 4052, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 942, 0, 11, 0, 0, 3, 3895, 3213, 0, 0, 0, 0, 0, 0, 0, 2832, 0, 0, 51, 4, 0, 0, 4048, 686, 416, 59, 0, 0, 0, 0, 0, 0, 0, 4015, 59, 0, 4078, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4077, 0, 31, 3404, 0, 514, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 18, 0, 0, 1, 3600, 452, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 4096, 493, 0, 0, 0, 0, 0, 0, 22, 0, 3, 4015, 0, 4093, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8064, 8096, 8356, 8027, 8168, 8508, 8149] avg=8192.0000 max=8508 max_rank=6 min=8027 min_rank=4 imbalance_ratio=1.038574 max_load_reduction=57028 max_load_reduction_pct=0.870178 moved_to_echo=32852 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 14, 4095, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 991, 0, 1, 1, 1, 1, 4096, 3212, 0, 0, 0, 0, 0, 0, 0, 3205, 0, 0, 60, 0, 0, 0, 4096, 687, 416, 59, 0, 0, 0, 0, 0, 0, 1, 4021, 1, 0, 4085, 1, 178, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 3408, 0, 386, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 37, 3603, 453, 0, 0, 0, 0, 0, 0, 0, 0, 3368, 0, 1, 0, 0, 0, 4095, 493, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4052, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8303, 8523, 8287, 7890, 8191, 7957, 8167] avg=8192.0000 max=8523 max_rank=2 min=7890 min_rank=4 imbalance_ratio=1.040405 max_load_reduction=57013 max_load_reduction_pct=0.869949 moved_to_echo=32205 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4082, 0, 0, 0, 2, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1096, 2, 1, 0, 0, 0, 4086, 3215, 0, 0, 0, 0, 0, 0, 0, 2355, 0, 0, 2, 0, 0, 0, 3969, 689, 421, 63, 0, 0, 0, 0, 0, 4, 2, 4016, 0, 0, 4090, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3, 3406, 0, 1150, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 4094, 0, 0, 0, 25, 3602, 456, 0, 0, 0, 0, 0, 0, 1, 6, 3643, 0, 1, 0, 0, 0, 4096, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4090, 0, 1, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8400, 7499, 8197, 8651, 8183, 8240, 8203] avg=8192.0000 max=8651 max_rank=4 min=7499 min_rank=2 imbalance_ratio=1.056030 max_load_reduction=56885 max_load_reduction_pct=0.867996 moved_to_echo=33277 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 128, 32768, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32256, 128, 0, 32640, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 128, 27264, 0, 5504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 28800, 128, 128, 128, 128, 128, 32768, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 7296, 128, 128, 128, 128, 128, 32512, 25728, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 256, 28800, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[23168, 128, 128, 640, 128, 0, 128, 32640, 5504, 3456, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 1, 4095, 0, 0, 1, 1799, 1, 3, 0, 0, 4, 0, 1, 4050, 1, 1, 0, 4095, 1, 26, 0, 1, 4092, 0, 0, 0, 2, 0, 4093, 0, 3, 4095, 0, 4095, 1, 1, 3221, 4092, 59, 1, 3578, 0, 4, 0, 0, 0, 0, 1, 4096, 0, 4095, 3644, 0, 0, 4091, 0, 1, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 2, 4093, 0, 0, 4094, 2, 0, 0, 2785, 0, 0, 2, 0, 0, 0, 1, 3986, 0, 1, 0, 4094, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 2, 4091, 3, 1, 4094, 3, 4094, 1, 1, 3865, 4096, 18, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 4095, 4060, 1, 0, 4094, 0, 2, 4094, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 3, 4096, 5, 0, 4095, 0, 1, 3, 2976, 0, 0, 0, 3, 7, 0, 1, 3944, 1, 2, 0, 4093, 0, 0, 3, 1, 4091, 3, 0, 0, 4, 1, 4091, 0, 0, 4094, 1, 4092, 0, 0, 3999, 4092, 0, 1, 1373, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 4095, 4083, 0, 0, 4090, 0, 1, 4091, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 16, 1515, 0, 0, 0, 0, 0, 0, 0, 3788, 0, 0, 0, 4096, 0, 26, 3, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 18, 4096, 0, 4096, 0, 0, 4086, 4077, 148, 0, 2923, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3886, 0, 3, 4091, 2, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4, 0, 0, 4092, 0, 0, 4092, 0, 0, 0, 3045, 5, 0, 4, 1, 0, 3, 0, 3807, 1, 2, 4, 4090, 0, 0, 0, 0, 4092, 4, 4, 0, 0, 1, 4091, 1, 4, 4092, 0, 4093, 4, 4, 3957, 4092, 11, 2, 1457, 2, 0, 0, 0, 0, 3, 4, 4091, 4, 4092, 4088, 1, 5, 4095, 0, 1, 4091, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[2, 7, 3, 4095, 0, 0, 4095, 0, 4, 1, 3685, 0, 4, 0, 0, 5, 0, 2, 4028, 3, 1, 0, 4095, 0, 0, 8, 0, 4091, 3, 3, 0, 1, 0, 4096, 0, 1, 4095, 1, 4092, 0, 0, 4086, 4088, 2, 0, 482, 3, 21, 0, 0, 5, 0, 0, 4095, 0, 4089, 4069, 0, 0, 4085, 0, 2, 4088, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 3644, 0, 1, 0, 0, 1, 0, 0, 3875, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 2, 4096, 0, 0, 3965, 4095, 3, 0, 802, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4096, 4092, 0, 0, 4095, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3430, 0, 0, 0, 0, 0, 0, 0, 3165, 0, 0, 0, 4096, 0, 5, 4, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 5, 4096, 0, 4096, 0, 0, 4042, 4095, 6, 0, 1867, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3873, 0, 0, 4089, 0, 0, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 4093, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2785, 0, 0, 2, 0, 0, 4094, 1229, 0, 0, 0, 0, 0, 0, 0, 1, 3986, 0, 1, 0, 4094, 0, 260, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 2741, 1347, 0, 0, 0, 0, 0, 0, 2, 4091, 3, 1, 0, 3, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3865, 2489, 18, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 1, 0, 4094, 0, 2, 124, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8110, 8345, 8186, 8193, 8037, 8192, 8282] avg=8192.0000 max=8345 max_rank=2 min=8037 min_rank=5 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1799, 1, 3, 0, 0, 4, 4095, 1229, 0, 0, 0, 0, 0, 0, 0, 1, 4050, 1, 1, 0, 4095, 1, 260, 2, 1, 0, 0, 0, 0, 0, 26, 0, 1, 4092, 0, 0, 0, 2, 2742, 1345, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3221, 2487, 59, 1, 3578, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3644, 0, 0, 4091, 0, 1, 124, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7132, 8412, 8208, 8190, 9351, 8192, 7861] avg=8192.0000 max=9351 max_rank=5 min=7132 min_rank=1 imbalance_ratio=1.141479 max_load_reduction=56185 max_load_reduction_pct=0.857315 moved_to_echo=33594 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 3, 4096, 5, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 2976, 0, 0, 0, 3, 7, 4094, 1228, 0, 0, 0, 0, 0, 0, 0, 1, 3944, 1, 2, 0, 4093, 0, 260, 2, 1, 0, 0, 0, 0, 0, 0, 3, 1, 4091, 3, 0, 0, 4, 2739, 1345, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 2487, 0, 1, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 4090, 0, 1, 124, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8312, 8304, 8186, 8183, 7860, 8190, 8301] avg=8192.0000 max=8312 max_rank=1 min=7860 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=57224 max_load_reduction_pct=0.873169 moved_to_echo=32534 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 7, 3, 4095, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 3685, 0, 4, 0, 0, 5, 4095, 1227, 0, 0, 0, 0, 0, 0, 0, 2, 4028, 3, 1, 0, 4095, 0, 259, 2, 1, 0, 0, 0, 0, 0, 0, 8, 0, 4091, 3, 3, 0, 1, 2737, 1344, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 2485, 2, 0, 482, 3, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4095, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 4085, 0, 2, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 9021, 8391, 8187, 8188, 7079, 8189, 8280] avg=8192.0000 max=9021 max_rank=1 min=7079 min_rank=5 imbalance_ratio=1.101196 max_load_reduction=56515 max_load_reduction_pct=0.862350 moved_to_echo=31736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 4092, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3045, 5, 0, 4, 1, 0, 4092, 1228, 0, 0, 0, 0, 0, 0, 3, 0, 3807, 1, 2, 4, 4090, 0, 260, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 4, 0, 0, 2739, 1345, 0, 0, 0, 0, 0, 0, 1, 4091, 1, 4, 0, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3957, 2487, 11, 2, 1457, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4091, 4, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 5, 4095, 0, 1, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8375, 8170, 8184, 8192, 7920, 8194, 8314] avg=8192.0000 max=8375 max_rank=1 min=7920 min_rank=5 imbalance_ratio=1.022339 max_load_reduction=57161 max_load_reduction_pct=0.872208 moved_to_echo=32620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1515, 0, 0, 0, 0, 0, 4096, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 3788, 0, 0, 0, 4096, 0, 259, 2, 1, 0, 0, 0, 0, 0, 26, 3, 0, 4090, 0, 0, 0, 0, 2742, 1341, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 2477, 148, 0, 2923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 3, 4091, 2, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 6856, 8146, 8202, 8208, 9634, 8192, 8107] avg=8192.0000 max=9634 max_rank=5 min=6856 min_rank=1 imbalance_ratio=1.176025 max_load_reduction=55902 max_load_reduction_pct=0.852997 moved_to_echo=34141 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3644, 0, 1, 0, 0, 1, 4096, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 0, 0, 0, 4096, 0, 260, 2, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 2742, 1346, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 2489, 3, 0, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4095, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8971, 8234, 8185, 8193, 7259, 8192, 8312] avg=8192.0000 max=8971 max_rank=1 min=7259 min_rank=5 imbalance_ratio=1.095093 max_load_reduction=56565 max_load_reduction_pct=0.863113 moved_to_echo=31956 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3430, 0, 0, 0, 0, 0, 4096, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 3165, 0, 0, 0, 4096, 0, 262, 8, 3, 0, 0, 0, 0, 0, 5, 4, 0, 4093, 0, 0, 0, 0, 2744, 1352, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 2481, 6, 0, 1867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 3873, 0, 0, 4089, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8759, 7534, 8198, 8189, 8396, 8195, 8079] avg=8192.0000 max=8759 max_rank=1 min=7534 min_rank=2 imbalance_ratio=1.069214 max_load_reduction=56777 max_load_reduction_pct=0.866348 moved_to_echo=32859 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[31872, 128, 128, 32768, 128, 128, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 128, 22016, 10880, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 22912, 128, 128, 128, 128, 128, 32768, 9856, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 31232, 19968, 256, 128, 14208, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 30720, 128, 128, 128, 32768, 128, 2176, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 3, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 4095, 0, 1, 1, 4084, 1, 0, 0, 0, 4093, 0, 1, 0, 0, 2, 1, 4091, 0, 4095, 4095, 0, 0, 0, 4095, 0, 145, 0, 0, 4092, 4095, 4095, 4095, 1, 0, 4095, 4095, 4095, 1, 3, 4095, 17, 3943, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[2, 2, 0, 1, 0, 1, 1, 0, 2, 0, 1, 1, 1, 0, 0, 0, 0, 2, 4094, 0, 2, 1, 4094, 1, 0, 0, 1, 4095, 2, 1, 0, 10, 0, 1, 4094, 0, 4094, 4094, 1, 0, 0, 4095, 1, 19, 0, 1, 4082, 4094, 4094, 4091, 2, 1, 4086, 4094, 4094, 0, 0, 4094, 0, 4089, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 4, 0, 1, 0, 1, 5, 8, 0, 2, 4, 1, 4, 0, 0, 0, 0, 6, 4085, 0, 5, 5, 4082, 0, 0, 0, 1, 4087, 0, 5, 2, 26, 0, 2, 4091, 8, 4091, 4087, 0, 4, 0, 4091, 1, 2, 0, 0, 4091, 4089, 4090, 4092, 5, 0, 4072, 4091, 4091, 5, 0, 4091, 14, 4089, 0, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4, 0, 0, 0, 0, 1, 0, 0, 1, 9, 0, 2, 0, 0, 4093, 5, 0, 0, 4091, 0, 0, 2, 0, 4085, 0, 1, 0, 0, 4, 0, 4090, 0, 4092, 4096, 1, 2, 0, 4092, 1, 11, 0, 0, 4089, 4096, 4094, 4095, 0, 0, 4095, 4094, 4095, 0, 4, 4094, 4, 4091, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 2, 0, 4096, 1, 0, 0, 4063, 0, 1, 6, 9, 4084, 0, 0, 0, 75, 8, 0, 4077, 3, 4092, 4096, 0, 2, 0, 4085, 0, 149, 0, 0, 4093, 4096, 4094, 4092, 0, 0, 4096, 4088, 4096, 2, 13, 4089, 9, 3895, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 4096, 1, 0, 0, 4006, 0, 0, 0, 0, 4086, 0, 0, 0, 45, 15, 0, 3988, 0, 4096, 4096, 0, 0, 0, 4096, 0, 609, 0, 0, 3994, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 64, 4096, 107, 3482, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 0, 0, 4086, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4095, 0, 4096, 4096, 0, 0, 0, 4095, 0, 76, 0, 0, 4094, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1, 4095, 9, 4022, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4095, 2, 0, 0, 4073, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 3, 0, 4087, 0, 4096, 4096, 0, 0, 0, 4095, 0, 171, 0, 0, 4090, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 9, 4096, 23, 3926, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 0, 1, 0, 1, 1, 0, 4094, 4089, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 1, 0, 0, 0, 4094, 4073, 5, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 2, 1, 4094, 1, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 2, 1, 0, 10, 4065, 12, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 4094, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 1, 19, 0, 1, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 1, 4086, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4089, 0, 0, 0, 0, 106, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8177, 8215, 8186, 8205, 8071, 8189, 8303] avg=8192.0000 max=8303 max_rank=7 min=8071 min_rank=5 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32768 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 3, 1, 1, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 4095, 4077, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 4084, 1, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 4066, 13, 0, 0, 0, 0, 0, 0, 2, 1, 4091, 0, 4095, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 145, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 4095, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 17, 3943, 0, 0, 0, 1, 106, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8182, 8203, 8173, 8204, 8205, 8197, 8179] avg=8192.0000 max=8205 max_rank=5 min=8173 min_rank=3 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32785 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 12, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 4094, 4074, 5, 0, 0, 0, 0, 0, 2, 0, 4096, 1, 0, 0, 4063, 0, 21, 0, 0, 0, 0, 0, 0, 0, 1, 6, 9, 4084, 0, 0, 0, 75, 4067, 12, 0, 0, 0, 0, 0, 0, 8, 0, 4077, 3, 4092, 15, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 149, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 4088, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4089, 9, 3895, 1, 0, 0, 0, 105, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8183, 8183, 8254, 8197, 8201, 8192, 8126] avg=8192.0000 max=8254 max_rank=3 min=8126 min_rank=7 imbalance_ratio=1.007568 max_load_reduction=57282 max_load_reduction_pct=0.874054 moved_to_echo=32716 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 9, 0, 2, 4094, 4077, 5, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 4091, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4085, 0, 1, 0, 0, 4067, 13, 0, 0, 0, 0, 0, 0, 4, 0, 4090, 0, 4092, 15, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 1, 11, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 4, 4091, 0, 1, 0, 0, 105, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8189, 8210, 8168, 8204, 8067, 8194, 8313] avg=8192.0000 max=8313 max_rank=7 min=8067 min_rank=5 imbalance_ratio=1.014771 max_load_reduction=57223 max_load_reduction_pct=0.873154 moved_to_echo=32778 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 1, 0, 1, 5, 8, 4091, 4085, 0, 0, 0, 0, 0, 0, 0, 2, 4, 1, 4, 0, 0, 0, 4090, 4074, 4, 0, 0, 0, 0, 0, 0, 6, 4085, 0, 5, 5, 4082, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 5, 2, 26, 4058, 12, 0, 0, 0, 0, 0, 0, 0, 2, 4091, 8, 4091, 15, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 1, 2, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 0, 4072, 4091, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 14, 4089, 0, 0, 0, 5, 105, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8179, 8204, 8191, 8211, 8059, 8179, 8318] avg=8192.0000 max=8318 max_rank=7 min=8059 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32767 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 4096, 4078, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4006, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 45, 4067, 13, 0, 0, 0, 0, 0, 0, 15, 0, 3988, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 0, 609, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 4096, 107, 3482, 0, 0, 0, 0, 106, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8261, 8124, 8211, 8114, 8572, 8197, 7869] avg=8192.0000 max=8572 max_rank=5 min=7869 min_rank=7 imbalance_ratio=1.046387 max_load_reduction=56964 max_load_reduction_pct=0.869202 moved_to_echo=32752 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4078, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4086, 0, 21, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4067, 14, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 76, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 9, 4022, 0, 0, 0, 0, 106, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8179, 8203, 8178, 8207, 8138, 8196, 8247] avg=8192.0000 max=8247 max_rank=7 min=8138 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32788 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4081, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 4073, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4070, 15, 0, 0, 0, 0, 0, 0, 3, 0, 4087, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 171, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 23, 3926, 0, 0, 0, 0, 109, 18, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8186, 8194, 8175, 8194, 8223, 8192, 8181] avg=8192.0000 max=8223 max_rank=5 min=8175 min_rank=3 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32790 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 32768, 32640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 31744, 128, 1280, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 256, 31616, 128, 128, 0, 128, 896, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 256, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 32640, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32640, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4095, 4095, 4038, 0, 0, 0, 1, 1, 4093, 0, 4094, 1, 1, 1, 0, 2, 0, 0, 1, 4093, 1, 0, 4, 0, 2, 4095, 1, 0, 4095, 0, 1, 4081, 4095, 4095, 0, 0, 4095, 4, 0, 2, 77, 4094, 0, 1, 4086, 1, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 1, 0, 1, 4092, 0, 0, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4045, 4096, 4094, 13, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 34, 0, 0, 0, 0, 4070, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 23, 0, 4096, 0, 63, 0, 62, 4096, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4058, 4096, 4078, 8, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 4067, 0, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 0, 4094, 4096, 4096, 12, 8, 4094, 0, 27, 0, 72, 4084, 1, 0, 4084, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4089, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 7, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 1, 0, 4096, 0, 0, 0, 5, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4079, 4095, 4086, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 4089, 0, 0, 0, 0, 0, 4092, 1, 0, 4096, 0, 0, 4096, 4095, 4096, 4, 3, 4096, 4, 0, 0, 117, 4091, 0, 0, 4005, 0, 0, 0, 0, 2, 0, 1, 4096, 2, 5, 0, 0, 0, 4089, 3, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4091, 4090, 4080, 0, 0, 0, 5, 3, 4092, 4, 4089, 5, 5, 6, 0, 3, 0, 5, 4, 4092, 114, 0, 8, 0, 2, 4091, 0, 0, 4084, 10, 0, 4073, 4089, 4090, 0, 0, 4090, 13, 0, 0, 2, 4091, 0, 5, 3985, 4, 5, 0, 0, 0, 5, 0, 4088, 0, 0, 6, 0, 4, 4092, 0, 0, 5, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4088, 4096, 4058, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4, 0, 66, 4096, 4, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4095, 4094, 4093, 1, 0, 1, 2, 1, 4094, 2, 4093, 1, 2, 2, 0, 2, 0, 0, 1, 4094, 13, 1, 4, 0, 2, 4095, 0, 0, 4092, 3, 0, 4077, 4094, 4094, 0, 0, 4094, 4, 0, 1, 2, 4094, 0, 2, 4090, 2, 0, 0, 0, 1, 1, 0, 4093, 0, 0, 1, 0, 0, 4094, 0, 0, 2, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4095, 31, 4038, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4094, 1, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 1, 0, 4, 0, 4052, 19, 18, 0, 0, 0, 0, 0, 2, 4095, 1, 0, 12, 0, 1, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4077, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 77, 4075, 0, 1, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 1, 4081, 12, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 1, 0, 1, 4095, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8192, 8188, 8192, 8178, 8240, 8188, 8192] avg=8192.0000 max=8240 max_rank=5 min=8166 min_rank=0 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4058, 31, 4078, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4067, 0, 0, 0, 0, 4053, 19, 18, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 12, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 8, 4076, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 4065, 1, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8182, 8160, 8190, 8219, 8222, 8202, 8186] avg=8192.0000 max=8222 max_rank=5 min=8160 min_rank=2 imbalance_ratio=1.003662 max_load_reduction=57314 max_load_reduction_pct=0.874542 moved_to_echo=32829 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4045, 31, 4094, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 34, 0, 0, 0, 4053, 19, 18, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 4078, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 4077, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8191, 8120, 8178, 8260, 8232, 8190, 8182] avg=8192.0000 max=8260 max_rank=4 min=8120 min_rank=2 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32864 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4079, 31, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4089, 0, 0, 0, 0, 4052, 19, 18, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 3, 4078, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 4072, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4096, 2, 5, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 4, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8187, 8180, 8201, 8184, 8194, 8200, 8194] avg=8192.0000 max=8201 max_rank=3 min=8180 min_rank=2 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4091, 30, 4080, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 4089, 5, 5, 6, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 4092, 114, 0, 8, 0, 4048, 19, 18, 0, 0, 0, 0, 0, 2, 4091, 0, 0, 12, 10, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4072, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4072, 0, 5, 3985, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4088, 0, 0, 6, 4070, 12, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 0, 0, 5, 0, 6, 4090, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8204, 8308, 8188, 8174, 8073, 8181, 8199] avg=8192.0000 max=8308 max_rank=2 min=8073 min_rank=5 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4095, 30, 4093, 1, 0, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 4093, 1, 2, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 13, 1, 4, 0, 4052, 19, 18, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 12, 3, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4076, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4075, 0, 2, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4093, 0, 0, 1, 4078, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 2, 4094, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8196, 8202, 8189, 8175, 8171, 8186, 8194] avg=8192.0000 max=8223 max_rank=0 min=8171 min_rank=5 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4089, 31, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 7, 0, 0, 0, 4053, 19, 18, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4076, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8192, 8188, 8203, 8175, 8177, 8191, 8194] avg=8192.0000 max=8216 max_rank=0 min=8175 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=57320 max_load_reduction_pct=0.874634 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4088, 22, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3, 0, 0, 0, 4059, 26, 25, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 4070, 4, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 8192, 8190, 8195, 8171, 8227, 8198, 8195] avg=8192.0000 max=8227 max_rank=5 min=8168 min_rank=0 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 256, 32640, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32640, 256, 128, 128, 0, 32512, 256, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32768, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[512, 32640, 128, 128, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 32640, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[3, 0, 4094, 0, 0, 2, 0, 1, 4093, 0, 0, 2, 4091, 0, 0, 0, 1409, 2, 1, 4091, 0, 2, 4092, 0, 4094, 4095, 0, 1, 3022, 0, 0, 2, 4094, 4094, 3, 2, 1, 0, 2, 234, 4096, 1, 0, 4094, 0, 4, 2736, 4094, 2, 1, 0, 790, 0, 0, 4093, 2, 4094, 2, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[2, 0, 4095, 0, 0, 1, 0, 0, 4091, 0, 0, 1, 4095, 0, 0, 3, 2301, 1, 1, 4094, 0, 1, 4095, 1, 4095, 4095, 2, 0, 2688, 0, 0, 1, 4095, 4095, 0, 1, 0, 3, 4, 273, 4077, 1, 0, 4095, 0, 1, 2088, 4095, 1, 0, 0, 837, 0, 0, 4093, 1, 4095, 1, 0, 0, 0, 16, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 33, 554, 0, 0, 4038, 0, 0, 4096, 0, 4096, 4096, 4, 0, 3939, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 53, 4049, 0, 0, 4096, 0, 0, 3467, 4096, 0, 0, 0, 32, 0, 0, 4096, 0, 4096, 0, 0, 3, 100, 118, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4096, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 4096, 0, 0, 18, 1343, 0, 0, 4095, 1, 0, 4096, 0, 4095, 4089, 6, 0, 3520, 0, 0, 0, 4096, 4088, 0, 0, 0, 4, 0, 619, 4086, 0, 0, 4096, 0, 0, 2285, 4094, 0, 1, 0, 409, 0, 0, 4096, 0, 4096, 0, 0, 1, 18, 3, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4, 0, 4089, 0, 0, 4, 0, 0, 4088, 0, 0, 5, 4084, 5, 0, 0, 2650, 6, 2, 4069, 0, 6, 4087, 6, 4090, 4092, 0, 0, 3595, 1, 0, 7, 4090, 4091, 0, 4, 0, 0, 7, 135, 4092, 4, 0, 4090, 0, 10, 1669, 4090, 5, 3, 0, 154, 0, 5, 4089, 6, 4089, 6, 4, 0, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 11, 159, 0, 0, 4087, 0, 0, 4096, 0, 4096, 4095, 0, 0, 3902, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 374, 4084, 0, 0, 4096, 0, 0, 3737, 4096, 0, 0, 0, 7, 0, 0, 4096, 0, 4096, 0, 0, 3, 5, 25, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 3, 1360, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4095, 0, 0, 3735, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 193, 4090, 0, 0, 4096, 0, 0, 2840, 4096, 0, 0, 0, 60, 0, 0, 4096, 0, 4096, 0, 0, 0, 7, 2, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 4096, 0, 0, 14, 971, 0, 0, 4087, 4, 0, 4096, 0, 4096, 4084, 10, 0, 3995, 0, 0, 0, 4096, 4089, 13, 0, 0, 4, 0, 461, 4073, 0, 0, 4096, 0, 0, 2705, 4092, 0, 0, 0, 15, 0, 0, 4096, 0, 4096, 0, 0, 0, 42, 26, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4094, 0, 0, 2, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 2, 1, 4091, 0, 2, 2761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 4095, 0, 1, 3022, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 4094, 3, 2, 1, 0, 2, 234, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4, 2736, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 790, 0, 0, 4093, 2, 2178, 1331, 294, 1, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 3548, 498, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8185, 8266, 7666, 8136, 8255, 8692, 8142] avg=8192.0000 max=8692 max_rank=6 min=7666 min_rank=3 imbalance_ratio=1.061035 max_load_reduction=56844 max_load_reduction_pct=0.867371 moved_to_echo=33225 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4095, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2301, 1, 1, 4094, 0, 1, 2763, 1, 0, 0, 0, 0, 0, 0, 0, 0, 546, 4095, 2, 0, 2688, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 4095, 0, 1, 0, 3, 4, 273, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 1, 0, 0, 0, 1, 2088, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 837, 0, 0, 4093, 1, 2178, 1332, 294, 1, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 16, 1, 0, 3549, 498, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8189, 9162, 7332, 8177, 7586, 8737, 8160] avg=8192.0000 max=9162 max_rank=2 min=7332 min_rank=3 imbalance_ratio=1.118408 max_load_reduction=56374 max_load_reduction_pct=0.860199 moved_to_echo=32660 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 4095, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 1343, 0, 0, 4095, 1, 0, 2764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 4089, 6, 0, 3520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3802, 4088, 0, 0, 0, 4, 0, 619, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 2285, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 409, 0, 0, 4096, 0, 2178, 1332, 294, 1, 0, 0, 0, 0, 4096, 0, 0, 1, 18, 3, 0, 8, 3549, 498, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8193, 8203, 8161, 8513, 7789, 8311, 8173] avg=8192.0000 max=8513 max_rank=4 min=7789 min_rank=5 imbalance_ratio=1.039185 max_load_reduction=57023 max_load_reduction_pct=0.870102 moved_to_echo=32786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4089, 0, 0, 4, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 5, 4083, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2650, 6, 2, 4069, 0, 6, 2758, 6, 0, 0, 0, 0, 0, 0, 0, 0, 546, 4092, 0, 0, 3595, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 4091, 0, 4, 0, 0, 7, 135, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 10, 1669, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 154, 0, 5, 4089, 6, 2175, 1329, 294, 1, 0, 0, 0, 0, 4089, 6, 4, 0, 0, 0, 3, 0, 3544, 497, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8181, 9497, 8241, 8033, 7193, 8061, 8143] avg=8192.0000 max=9497 max_rank=2 min=7193 min_rank=5 imbalance_ratio=1.159302 max_load_reduction=56039 max_load_reduction_pct=0.855087 moved_to_echo=31430 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4095, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 4038, 0, 0, 2764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 4096, 4, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3802, 4096, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 3467, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 4096, 0, 2179, 1332, 294, 1, 0, 0, 0, 0, 4096, 0, 0, 3, 100, 118, 0, 0, 3550, 498, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8218, 7356, 8585, 7951, 8935, 7934, 8365] avg=8192.0000 max=8935 max_rank=5 min=7356 min_rank=2 imbalance_ratio=1.090698 max_load_reduction=56601 max_load_reduction_pct=0.863663 moved_to_echo=33185 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4092, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 4087, 0, 0, 2760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 4095, 0, 0, 3902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 4096, 0, 0, 0, 0, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 3737, 1412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 2182, 1336, 300, 4, 0, 0, 0, 0, 4096, 0, 0, 3, 5, 25, 0, 0, 3552, 502, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8190, 7006, 8540, 8266, 9233, 7925, 8183] avg=8192.0000 max=9233 max_rank=5 min=7006 min_rank=2 imbalance_ratio=1.127075 max_load_reduction=56303 max_load_reduction_pct=0.859116 moved_to_echo=33607 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1360, 0, 0, 4096, 0, 0, 2764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 4095, 0, 0, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3802, 4096, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 2840, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 4096, 0, 2179, 1332, 294, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 2, 0, 1, 3550, 498, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8220, 8376, 8091, 8349, 7962, 8154] avg=8192.0000 max=8376 max_rank=3 min=7962 min_rank=6 imbalance_ratio=1.022461 max_load_reduction=57160 max_load_reduction_pct=0.872192 moved_to_echo=32556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 4095, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 971, 0, 0, 4087, 4, 0, 2764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 4084, 10, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3802, 4089, 13, 0, 0, 4, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 2705, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4096, 0, 2176, 1332, 294, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 42, 26, 0, 4, 3550, 498, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8188, 7826, 8635, 8369, 8196, 7914, 8216] avg=8192.0000 max=8635 max_rank=3 min=7826 min_rank=2 imbalance_ratio=1.054077 max_load_reduction=56901 max_load_reduction_pct=0.868240 moved_to_echo=32695 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[10752, 128, 128, 32768, 128, 128, 22144, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[30464, 32768, 128, 128, 128, 128, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 256, 256, 128, 128, 28416, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 0, 0, 128, 21632, 11392, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 2304, 0, 128, 32768, 128, 17536, 10752, 2432, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4480, 32768, 128, 128, 28416, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[3, 0, 3, 2, 2, 1, 4090, 4094, 1, 4094, 0, 0, 2, 2, 0, 2, 0, 4092, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 3746, 2, 3, 2, 0, 2, 0, 0, 3, 2, 0, 4094, 0, 4094, 4093, 4080, 4094, 1, 3, 15, 2, 0, 4092, 5, 2, 0, 0, 1, 4094, 4094, 2, 2, 4093, 0, 0, 4093, 376] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4088, 0, 4092, 4, 0, 0, 0, 10, 0, 0, 4093, 0, 7, 0, 0, 3, 4074, 0, 0, 0, 3638, 0, 16, 0, 0, 0, 0, 7, 0, 0, 0, 4094, 0, 4087, 4094, 4094, 4091, 0, 0, 0, 0, 1, 4094, 0, 0, 1, 3, 0, 4095, 4096, 0, 0, 4096, 0, 2, 4096, 464] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 1, 4094, 4091, 1, 4095, 3, 2, 1, 0, 0, 1, 1, 4092, 0, 0, 0, 0, 0, 3961, 0, 0, 0, 3953, 1, 0, 1, 1, 0, 0, 3, 1, 1, 0, 4095, 0, 4086, 4095, 4094, 4094, 0, 1, 9, 1, 0, 4091, 0, 1, 0, 9, 0, 4095, 4094, 1, 1, 4095, 1, 6, 4094, 266] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4085, 0, 4096, 21, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 2951, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 4054, 4096, 4096, 4096, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 130, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 1066] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[7, 0, 10, 6, 0, 8, 4083, 4090, 4, 4090, 0, 0, 3, 8, 0, 6, 0, 4086, 2, 0, 0, 8, 0, 3982, 0, 0, 0, 4061, 7, 0, 6, 0, 0, 0, 0, 8, 6, 0, 4090, 0, 4089, 4088, 3772, 4089, 1, 7, 0, 6, 0, 4088, 4, 5, 0, 0, 84, 4090, 4089, 6, 6, 4088, 0, 0, 4088, 365] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4095, 1, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 0, 0, 1, 4045, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4094, 4096, 4039, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 260] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4094, 4, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 2, 4080, 0, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 21, 0, 4096, 4096, 0, 0, 4096, 0, 5, 4096, 264] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4087, 0, 4089, 2, 0, 0, 0, 13, 0, 0, 4085, 0, 11, 0, 0, 4, 4069, 0, 0, 0, 3464, 0, 4, 0, 0, 0, 0, 2, 0, 0, 4, 4091, 0, 4080, 4093, 4073, 4094, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 25, 2, 4092, 4096, 0, 0, 4096, 0, 13, 4096, 655] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 2, 2, 1, 4090, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 2, 2, 0, 2, 3642, 443, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4058, 31, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3746, 2, 3, 2, 0, 4094, 394, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 2, 0, 4094, 0, 3699, 392, 0, 0, 0, 0, 0, 0, 4094, 0, 4080, 60, 1, 3, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 2, 0, 0, 1, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3628, 0, 0, 4093, 376, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8186, 8203, 8241, 8192, 8255, 8161, 8101] avg=8192.0000 max=8255 max_rank=5 min=8101 min_rank=7 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32709 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 1, 4094, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 3, 2, 1, 0, 0, 1, 3642, 444, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 0, 0, 0, 3961, 31, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 1, 0, 1, 1, 4094, 395, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1, 0, 4095, 0, 3700, 392, 0, 0, 0, 0, 0, 0, 4086, 0, 4094, 60, 0, 1, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 0, 9, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3628, 1, 6, 4094, 266, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8189, 8108, 8445, 8192, 8251, 8163, 7997] avg=8192.0000 max=8445 max_rank=3 min=7997 min_rank=7 imbalance_ratio=1.030884 max_load_reduction=57091 max_load_reduction_pct=0.871140 moved_to_echo=32603 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 10, 0, 3639, 444, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 7, 0, 0, 3, 4074, 31, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3638, 0, 16, 0, 0, 4096, 394, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4094, 0, 3700, 392, 0, 0, 0, 0, 0, 0, 4087, 0, 4094, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 1, 3, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 0, 2, 4096, 464, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8189, 8231, 8144, 8193, 8241, 8161, 8191] avg=8192.0000 max=8241 max_rank=5 min=8144 min_rank=3 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 10, 6, 0, 8, 4083, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 4, 4090, 0, 0, 3, 8, 0, 6, 3638, 443, 0, 0, 0, 0, 0, 0, 0, 4086, 2, 0, 0, 8, 0, 3982, 31, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 7, 0, 6, 0, 4089, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 0, 4090, 0, 3694, 392, 0, 0, 0, 0, 0, 0, 4089, 0, 3772, 59, 1, 7, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 5, 0, 0, 84, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 3623, 0, 0, 4088, 365, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8192, 8131, 8557, 8190, 7934, 8238, 8088] avg=8192.0000 max=8557 max_rank=3 min=7934 min_rank=5 imbalance_ratio=1.044556 max_load_reduction=56979 max_load_reduction_pct=0.869431 moved_to_echo=32450 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 0, 6, 0, 3644, 444, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4055, 31, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2951, 0, 0, 0, 0, 4096, 394, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 3702, 392, 0, 0, 0, 0, 0, 0, 4054, 0, 4096, 60, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 130, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 0, 0, 4096, 1066, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8211, 8205, 7441, 8196, 8220, 8289, 8791] avg=8192.0000 max=8791 max_rank=7 min=7441 min_rank=3 imbalance_ratio=1.073120 max_load_reduction=56745 max_load_reduction_pct=0.865860 moved_to_echo=33496 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 2, 0, 3647, 446, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4080, 37, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3833, 0, 0, 0, 0, 4096, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3701, 398, 0, 0, 0, 0, 0, 0, 4083, 0, 4095, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 21, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3625, 0, 5, 4096, 264, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8193, 8240, 8324, 8195, 8229, 8170, 7990] avg=8192.0000 max=8324 max_rank=3 min=7990 min_rank=7 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32584 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 2, 0, 3644, 444, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4045, 31, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 4096, 395, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3701, 392, 0, 0, 0, 0, 0, 0, 4094, 0, 4039, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 0, 0, 4096, 260, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8186, 8195, 8426, 8190, 8193, 8168, 7985] avg=8192.0000 max=8426 max_rank=3 min=7985 min_rank=7 imbalance_ratio=1.028564 max_load_reduction=57110 max_load_reduction_pct=0.871429 moved_to_echo=32536 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 13, 0, 3642, 444, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 11, 0, 0, 4, 4069, 31, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 0, 4, 0, 0, 4096, 394, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 4091, 0, 3699, 392, 0, 0, 0, 0, 0, 0, 4080, 0, 4073, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 25, 2, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 0, 13, 4096, 655, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8190, 8223, 7958, 8188, 8213, 8186, 8393] avg=8192.0000 max=8393 max_rank=7 min=7958 min_rank=3 imbalance_ratio=1.024536 max_load_reduction=57143 max_load_reduction_pct=0.871933 moved_to_echo=32980 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 32384, 512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 0, 29696, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 29056, 128, 128, 32768, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 32384, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 29696, 128, 128, 128, 128, 32768, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 256, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 29184, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 4032, 2, 0, 0, 1, 4094, 2, 2, 0, 9, 2, 0, 2528, 4093, 4094, 2, 0, 4093, 0, 4094, 0, 4094, 0, 4093, 1, 3, 10, 1, 1, 4094, 0, 0, 3, 0, 1, 1606, 0, 0, 0, 0, 2, 2, 0, 3, 1, 1, 2, 0, 4094, 2, 0, 2, 4094, 0, 4094, 2, 0, 2, 4093, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 3995, 1, 2, 1, 1, 4095, 1, 1, 0, 1, 1, 0, 1684, 4095, 4092, 1, 0, 4094, 0, 4092, 0, 4095, 0, 4095, 0, 0, 0, 1, 0, 4095, 0, 0, 1, 0, 0, 2511, 0, 0, 2, 0, 1, 1, 0, 1, 0, 0, 1, 0, 4095, 1, 0, 1, 4095, 0, 4095, 1, 0, 2, 4095, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3964, 0, 0, 0, 0, 4096, 0, 0, 0, 138, 0, 0, 3493, 4096, 4091, 0, 0, 4096, 0, 3999, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4018, 0, 1, 9, 0, 4091, 0, 0, 0, 50, 0, 0, 3281, 4096, 4089, 0, 0, 4096, 0, 4087, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4096, 1, 4095, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4071, 0, 1, 2, 0, 4095, 0, 0, 0, 29, 0, 0, 2776, 4096, 4095, 0, 0, 4096, 0, 4094, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2, 0, 0, 4088, 4071, 6, 0, 0, 7, 4090, 7, 6, 0, 0, 7, 0, 3372, 4088, 4090, 8, 0, 4088, 1, 4089, 0, 4088, 4, 4087, 0, 2, 23, 5, 14, 4090, 0, 0, 10, 0, 4, 701, 0, 0, 0, 0, 6, 6, 0, 3, 0, 1, 7, 0, 4087, 6, 0, 4, 4089, 0, 4089, 7, 1, 11, 4086, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4060, 0, 0, 0, 0, 4096, 0, 0, 0, 53, 0, 0, 3989, 4096, 4092, 0, 0, 4096, 0, 4090, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4086, 0, 1, 9, 0, 4083, 0, 0, 0, 23, 0, 0, 3684, 4096, 4089, 0, 0, 4096, 0, 4056, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4087, 0, 1, 0, 0, 0, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 4095, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 4032, 2, 0, 0, 51, 3, 0, 0, 0, 0, 0, 0, 1, 4094, 2, 2, 0, 9, 2, 0, 4053, 1, 0, 0, 0, 0, 0, 0, 2528, 0, 1017, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4093, 1, 3, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 3, 0, 1, 1606, 3026, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 3, 4094, 4092, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 4094, 2, 0, 2, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8164, 7641, 8199, 8771, 8193, 8196, 8190] avg=8192.0000 max=8771 max_rank=4 min=7641 min_rank=2 imbalance_ratio=1.070679 max_load_reduction=56765 max_load_reduction_pct=0.866165 moved_to_echo=33350 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 3995, 1, 2, 1, 51, 3, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 1, 1, 0, 4055, 2, 0, 0, 0, 0, 0, 0, 1684, 0, 1016, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 2511, 3025, 39, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 1, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8157, 6793, 8188, 9671, 8195, 8191, 8192] avg=8192.0000 max=9671 max_rank=4 min=6793 min_rank=2 imbalance_ratio=1.180542 max_load_reduction=55865 max_load_reduction_pct=0.852432 moved_to_echo=34249 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4018, 0, 1, 9, 51, 3, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 50, 0, 0, 4057, 1, 0, 0, 0, 0, 0, 0, 3281, 0, 1015, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 854, 3023, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8199, 8383, 8189, 8009, 8191, 8194, 8193] avg=8192.0000 max=8383 max_rank=2 min=8009 min_rank=4 imbalance_ratio=1.023315 max_load_reduction=57153 max_load_reduction_pct=0.872086 moved_to_echo=32587 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3964, 0, 0, 0, 51, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 138, 0, 0, 4057, 2, 0, 0, 0, 0, 0, 0, 3493, 0, 1016, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 699, 3024, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8293, 8508, 8189, 7858, 8191, 8191, 8192] avg=8192.0000 max=8508 max_rank=2 min=7858 min_rank=4 imbalance_ratio=1.038574 max_load_reduction=57028 max_load_reduction_pct=0.870178 moved_to_echo=32432 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4086, 0, 1, 9, 51, 3, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 23, 0, 0, 4057, 1, 0, 0, 0, 0, 0, 0, 3684, 0, 1015, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 1, 0, 0, 0, 452, 3023, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8164, 8755, 8189, 7602, 8191, 8191, 8198] avg=8192.0000 max=8755 max_rank=2 min=7602 min_rank=4 imbalance_ratio=1.068726 max_load_reduction=56781 max_load_reduction_pct=0.866409 moved_to_echo=32182 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4088, 4071, 6, 0, 0, 51, 3, 0, 0, 0, 0, 0, 0, 7, 4090, 7, 6, 0, 0, 7, 0, 4049, 2, 0, 0, 0, 0, 0, 0, 3372, 0, 1015, 8, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4, 4087, 0, 2, 23, 5, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4090, 0, 0, 10, 0, 4, 701, 3024, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 3, 4089, 4087, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 4087, 6, 0, 4, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 11, 4086, 4085, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8168, 8485, 8206, 7882, 8191, 8193, 8190] avg=8192.0000 max=8485 max_rank=2 min=7882 min_rank=4 imbalance_ratio=1.035767 max_load_reduction=57051 max_load_reduction_pct=0.870529 moved_to_echo=32456 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4071, 0, 1, 2, 51, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 29, 0, 0, 4057, 2, 0, 0, 0, 0, 0, 0, 2776, 0, 1017, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1318, 3027, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8183, 7887, 8189, 8479, 8192, 8190, 8192] avg=8192.0000 max=8479 max_rank=4 min=7887 min_rank=2 imbalance_ratio=1.035034 max_load_reduction=57057 max_load_reduction_pct=0.870621 moved_to_echo=33053 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4060, 0, 0, 0, 58, 5, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 53, 0, 0, 4057, 2, 0, 0, 0, 0, 0, 0, 3989, 0, 1005, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 100, 3029, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8208, 9084, 8187, 7264, 8192, 8190, 8189] avg=8192.0000 max=9084 max_rank=2 min=7264 min_rank=4 imbalance_ratio=1.108887 max_load_reduction=56452 max_load_reduction_pct=0.861389 moved_to_echo=31835 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 32768, 32384, 128, 128, 128, 512, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[24832, 0, 8192, 128, 0, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 128, 8320, 24320, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 0, 384, 128, 0, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[2191, 0, 4086, 0, 2, 0, 0, 4, 1, 0, 0, 4093, 0, 2, 3, 4076, 0, 2, 0, 3, 0, 2, 0, 1, 25, 1, 2, 2, 4092, 4094, 4094, 3, 0, 0, 0, 4092, 4092, 0, 3, 2, 4094, 0, 4, 0, 4094, 0, 1778, 2, 0, 2, 0, 2, 2, 124, 4093, 6, 4082, 2, 6, 4094, 4091, 0, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[3508, 0, 4083, 0, 0, 0, 1, 0, 0, 0, 1, 4096, 0, 0, 0, 4070, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 632, 0, 0, 0, 9, 0, 0, 218, 4096, 0, 3863, 0, 1, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2777, 0, 4066, 0, 0, 0, 1, 0, 0, 0, 1, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1185, 0, 0, 0, 4, 0, 0, 243, 4096, 0, 4015, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1441, 0, 4085, 0, 1, 0, 0, 2, 0, 0, 0, 4094, 0, 1, 1, 4095, 0, 1, 0, 1, 1, 1, 2, 1, 13, 0, 1, 1, 4095, 4094, 4094, 1, 0, 0, 0, 4094, 4093, 0, 0, 1, 4095, 0, 2, 0, 4095, 0, 2882, 1, 0, 1, 0, 3, 1, 6, 4095, 0, 3858, 1, 0, 4094, 4093, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1145, 4, 4089, 0, 7, 0, 0, 8, 0, 0, 0, 4085, 0, 7, 7, 4088, 0, 7, 0, 10, 0, 8, 7, 4, 56, 0, 5, 6, 4086, 4089, 4088, 10, 0, 0, 0, 4088, 4082, 0, 14, 7, 4089, 1, 6, 0, 4089, 0, 2343, 5, 0, 8, 0, 12, 7, 529, 4082, 9, 4090, 6, 16, 4077, 4077, 0, 0, 4083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1666, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1508, 0, 0, 0, 19, 68, 0, 1336, 4096, 0, 3848, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[2323, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1134, 0, 0, 0, 7, 3, 0, 837, 4096, 0, 4009, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[2040, 0, 3982, 0, 0, 0, 7, 0, 0, 0, 6, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1688, 0, 0, 0, 13, 0, 0, 544, 4096, 0, 4028, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2191, 0, 4086, 0, 2, 0, 0, 4, 1640, 378, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 2, 3, 4076, 5, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 2, 0, 1, 4094, 4089, 0, 0, 0, 0, 0, 0, 25, 1, 2, 2, 4087, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4092, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4, 0, 2451, 0, 1778, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 2, 124, 4093, 6, 3594, 3, 0, 0, 0, 0, 0, 0, 4082, 2, 6, 122, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8183, 8191, 8214, 8189, 8329, 7826, 8303] avg=8192.0000 max=8329 max_rank=5 min=7826 min_rank=6 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32647 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3508, 0, 4083, 0, 0, 0, 1, 0, 1640, 378, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4070, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2453, 0, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 218, 4096, 0, 3596, 3, 0, 0, 0, 0, 0, 0, 3863, 0, 1, 122, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9610, 8175, 8190, 8184, 8193, 7180, 7922, 8082] avg=8192.0000 max=9610 max_rank=0 min=7180 min_rank=5 imbalance_ratio=1.173096 max_load_reduction=55926 max_load_reduction_pct=0.853363 moved_to_echo=31377 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1441, 0, 4085, 0, 1, 0, 0, 2, 1640, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 1, 4095, 5, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 2, 1, 4094, 4092, 0, 0, 0, 0, 0, 0, 13, 0, 1, 1, 4089, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4093, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 2452, 0, 2882, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 1, 6, 4095, 0, 3594, 3, 0, 0, 0, 0, 0, 0, 3858, 1, 0, 122, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7547, 8199, 8193, 8199, 8189, 9432, 7703, 8074] avg=8192.0000 max=9432 max_rank=5 min=7547 min_rank=0 imbalance_ratio=1.151367 max_load_reduction=56104 max_load_reduction_pct=0.856079 moved_to_echo=33398 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1145, 4, 4089, 0, 7, 0, 0, 8, 1638, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 7, 7, 4088, 5, 3, 0, 0, 0, 0, 0, 0, 0, 7, 0, 10, 0, 8, 7, 4, 4088, 4080, 0, 0, 0, 0, 0, 0, 56, 0, 5, 6, 4081, 4089, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4082, 0, 14, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 6, 0, 2448, 0, 2343, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 12, 7, 529, 4082, 9, 3579, 3, 0, 0, 0, 0, 0, 0, 4090, 6, 16, 122, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7267, 8195, 8204, 8247, 8191, 8892, 8229, 8311] avg=8192.0000 max=8892 max_rank=5 min=7267 min_rank=0 imbalance_ratio=1.085449 max_load_reduction=56644 max_load_reduction_pct=0.864319 moved_to_echo=33623 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2040, 0, 3982, 0, 0, 0, 7, 0, 1640, 378, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 4089, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2453, 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 544, 4096, 0, 3596, 3, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 122, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8047, 8199, 8189, 8173, 8193, 8237, 8252, 8246] avg=8192.0000 max=8252 max_rank=6 min=8047 min_rank=0 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32928 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1666, 0, 3886, 0, 0, 0, 0, 0, 1640, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4081, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2453, 0, 1508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 68, 0, 1336, 4096, 0, 3596, 3, 0, 0, 0, 0, 0, 0, 3848, 0, 0, 122, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7570, 8185, 8189, 8158, 8193, 8057, 9118, 8066] avg=8192.0000 max=9118 max_rank=6 min=7570 min_rank=0 imbalance_ratio=1.113037 max_load_reduction=56418 max_load_reduction_pct=0.860870 moved_to_echo=33434 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2777, 0, 4066, 0, 0, 0, 1, 0, 1640, 378, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4093, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2453, 0, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 243, 4096, 0, 3596, 2, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 122, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8862, 8198, 8190, 8185, 8193, 7734, 7941, 8233] avg=8192.0000 max=8862 max_rank=0 min=7734 min_rank=5 imbalance_ratio=1.081787 max_load_reduction=56674 max_load_reduction_pct=0.864777 moved_to_echo=32101 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2323, 0, 3984, 0, 0, 0, 0, 0, 1645, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2445, 0, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 837, 4096, 0, 3600, 2, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 116, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8332, 8202, 8190, 8176, 8195, 7675, 8545, 8221] avg=8192.0000 max=8545 max_rank=6 min=7675 min_rank=5 imbalance_ratio=1.043091 max_load_reduction=56991 max_load_reduction_pct=0.869614 moved_to_echo=32636 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31872, 128, 128, 1024, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 19712, 0, 13184, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 128, 3840, 32768, 128, 28800, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 32768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[17152, 128, 32384, 0, 128, 0, 128, 128, 13184, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 32768, 0, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[472, 4093, 1, 0, 3, 3, 0, 2, 4093, 2, 0, 2, 0, 4094, 4094, 10, 4094, 0, 3, 2, 2, 0, 2, 2, 0, 2, 2, 0, 2, 4091, 0, 108, 2, 2, 0, 4094, 2, 0, 0, 4092, 4094, 0, 2, 4090, 0, 2, 0, 1, 4092, 4, 0, 3, 0, 4092, 4094, 3506, 0, 0, 0, 4092, 4093, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1477, 4087, 19, 0, 11, 9, 3, 6, 4083, 9, 0, 7, 0, 4089, 4090, 8, 4088, 3, 7, 7, 9, 0, 6, 5, 0, 5, 8, 0, 6, 4082, 0, 141, 3, 4, 0, 4090, 7, 0, 0, 4080, 4089, 0, 5, 4077, 0, 7, 1, 1, 4085, 13, 0, 13, 0, 4073, 4089, 2465, 0, 0, 0, 4081, 4088, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[924, 4094, 1, 3, 1, 1, 0, 1, 4095, 0, 0, 1, 0, 4093, 4095, 614, 4095, 0, 1, 1, 1, 0, 1, 0, 2, 1, 1, 0, 1, 4095, 0, 9, 0, 1, 1, 4095, 1, 0, 0, 4094, 4095, 0, 1, 4092, 0, 1, 0, 0, 4095, 2, 0, 0, 0, 4095, 4095, 2547, 0, 1, 0, 4094, 4095, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[131, 4096, 0, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4095, 4095, 69, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 5, 0, 0, 0, 4093, 0, 0, 0, 4096, 4095, 0, 0, 4067, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 4093, 3924, 0, 0, 0, 4096, 4096, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[175, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3913, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[122, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 76, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3911, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[21, 4096, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 4093, 0, 0, 4073, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 4073, 0, 0, 0, 4096, 4096, 7, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[319, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 19, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3759, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[472, 4093, 1, 0, 3, 3, 0, 2, 3509, 85, 37, 0, 0, 0, 0, 0, 4093, 2, 0, 2, 0, 3995, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 2, 2, 0, 2, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 2, 4091, 0, 108, 4057, 4, 0, 0, 0, 0, 0, 0, 2, 2, 0, 4094, 2, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 4090, 0, 2, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 583, 4, 0, 3, 0, 4092, 0, 3506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8102, 8195, 8266, 8194, 8199, 8188, 8187] avg=8192.0000 max=8266 max_rank=3 min=8102 min_rank=1 imbalance_ratio=1.009033 max_load_reduction=57270 max_load_reduction_pct=0.873871 moved_to_echo=32768 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[924, 4094, 1, 3, 1, 1, 0, 1, 3512, 85, 37, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 3994, 0, 614, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 1, 4095, 0, 9, 4058, 4, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 1, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 4092, 0, 1, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 583, 2, 0, 0, 0, 4095, 0, 2547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8659, 8704, 8190, 8171, 8194, 8199, 7227, 8192] avg=8192.0000 max=8704 max_rank=1 min=7227 min_rank=6 imbalance_ratio=1.062500 max_load_reduction=56832 max_load_reduction_pct=0.867188 moved_to_echo=31812 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[131, 4096, 0, 1, 0, 0, 0, 0, 3512, 85, 37, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 3995, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 5, 4056, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4067, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 584, 0, 1, 0, 0, 4096, 0, 3924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7862, 8156, 8187, 8164, 8191, 8173, 8605, 8198] avg=8192.0000 max=8605 max_rank=6 min=7862 min_rank=0 imbalance_ratio=1.050415 max_load_reduction=56931 max_load_reduction_pct=0.868698 moved_to_echo=33167 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1477, 4087, 19, 0, 11, 9, 3, 6, 3503, 85, 37, 0, 0, 0, 0, 0, 4083, 9, 0, 7, 0, 3990, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 7, 7, 9, 0, 6, 5, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 0, 6, 4082, 0, 141, 4052, 4, 0, 0, 0, 0, 0, 0, 3, 4, 0, 4090, 7, 0, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 5, 4077, 0, 7, 1, 1, 10, 0, 0, 0, 0, 0, 0, 0, 582, 13, 0, 13, 0, 4073, 0, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4088, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9237, 8097, 8211, 8298, 8186, 8190, 7146, 8171] avg=8192.0000 max=9237 max_rank=0 min=7146 min_rank=6 imbalance_ratio=1.127563 max_load_reduction=56299 max_load_reduction_pct=0.859055 moved_to_echo=31693 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[21, 4096, 0, 4, 0, 0, 0, 0, 3512, 85, 37, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3996, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4057, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4073, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 0, 0, 4096, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 7, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7755, 8117, 8188, 8158, 8187, 8177, 8753, 8201] avg=8192.0000 max=8753 max_rank=6 min=7755 min_rank=0 imbalance_ratio=1.068481 max_load_reduction=56783 max_load_reduction_pct=0.866440 moved_to_echo=33318 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[122, 4096, 0, 0, 0, 0, 0, 0, 3512, 85, 37, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3996, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4059, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 0, 0, 4096, 0, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7852, 8168, 8188, 8159, 8194, 8190, 8591, 8194] avg=8192.0000 max=8591 max_rank=6 min=7852 min_rank=0 imbalance_ratio=1.048706 max_load_reduction=56945 max_load_reduction_pct=0.868912 moved_to_echo=33169 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[175, 4096, 0, 0, 0, 0, 0, 0, 3512, 85, 37, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3995, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4059, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 0, 0, 4096, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7905, 8098, 8188, 8162, 8194, 8202, 8593, 8194] avg=8192.0000 max=8593 max_rank=6 min=7905 min_rank=0 imbalance_ratio=1.048950 max_load_reduction=56943 max_load_reduction_pct=0.868881 moved_to_echo=33183 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[319, 4096, 0, 0, 0, 0, 0, 0, 3518, 90, 38, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3981, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4058, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 0, 4096, 0, 3759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8061, 8094, 8189, 8158, 8196, 8206, 8433, 8199] avg=8192.0000 max=8433 max_rank=6 min=8061 min_rank=0 imbalance_ratio=1.029419 max_load_reduction=57103 max_load_reduction_pct=0.871323 moved_to_echo=33034 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 32768, 32768, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 0, 32000, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4736, 128, 128, 128, 0, 32768, 0, 28160, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[3712, 32768, 128, 128, 128, 128, 128, 128, 28160, 768, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 384, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4093, 3, 2, 2, 0, 6, 4094, 4, 3, 0, 4093, 0, 0, 2, 2, 0, 4090, 0, 2, 0, 31, 0, 4094, 0, 1, 0, 0, 0, 0, 2, 4094, 0, 0, 4088, 0, 4093, 0, 0, 2, 2, 4093, 0, 4094, 1, 4094, 0, 4092, 4093, 4094, 4063, 2, 2, 0, 2, 0, 0, 3, 3, 0, 2, 2, 2, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4094, 1, 1, 1, 0, 0, 4095, 2, 1, 0, 4095, 0, 0, 1, 1, 0, 4094, 1, 1, 1, 2, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 4095, 5, 0, 4093, 0, 4095, 0, 0, 1, 1, 4095, 0, 4095, 0, 4095, 0, 4095, 4094, 4094, 4091, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 28, 0, 40, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4068, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4, 4096, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 13, 0, 1, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 4096, 0, 4088, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4086, 12, 7, 9, 1, 19, 4091, 1, 9, 0, 4079, 0, 0, 6, 6, 0, 4083, 0, 7, 0, 97, 0, 4087, 0, 4, 0, 0, 5, 8, 7, 4089, 0, 2, 4071, 0, 4086, 0, 0, 8, 9, 4088, 0, 4090, 4, 4088, 0, 4086, 4087, 4087, 3994, 2, 2, 0, 6, 0, 0, 8, 9, 0, 8, 7, 0, 4081] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 6, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4090, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 23, 0, 16, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 4096, 0, 4080, 0, 4096, 0, 4096, 4096, 4096, 4095, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 3, 2, 2, 0, 6, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 4093, 0, 0, 2, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2, 0, 31, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 4094, 4072, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4093, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 2, 9, 0, 4094, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 17, 4063, 2, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 2, 2, 2, 4091, 4084, 5, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8197, 8205, 8174, 8186, 8198, 8179, 8193] avg=8192.0000 max=8205 max_rank=2 min=8174 min_rank=3 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 1, 1, 1, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 4095, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 1, 2, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4095, 4072, 5, 0, 0, 0, 0, 0, 0, 5, 0, 4093, 0, 4095, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 1, 9, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 17, 4091, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 4092, 4086, 5, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8194, 8182, 8174, 8197, 8200, 8205, 8188] avg=8192.0000 max=8205 max_rank=6 min=8174 min_rank=3 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 4096, 0, 0, 0, 0, 0, 4083, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 0, 0, 0, 4090, 4074, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 6, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8191, 8180, 8183, 8196, 8193, 8216, 8190] avg=8192.0000 max=8216 max_rank=6 min=8180 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=57320 max_load_reduction_pct=0.874634 moved_to_echo=32795 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4056, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 40, 0, 0, 0, 0, 4096, 4074, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4068, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 6, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8156, 8191, 8180, 8243, 8195, 8173, 8208, 8190] avg=8192.0000 max=8243 max_rank=3 min=8156 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 12, 7, 9, 1, 19, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 1, 9, 0, 4079, 0, 0, 6, 6, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 7, 0, 97, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5, 8, 7, 4089, 4065, 5, 0, 0, 0, 0, 0, 0, 0, 2, 4071, 0, 4086, 0, 0, 8, 3, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 4090, 4, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 17, 3994, 2, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 0, 8, 7, 0, 4081, 4079, 5, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8188, 8262, 8183, 8170, 8198, 8108, 8198] avg=8192.0000 max=8262 max_rank=2 min=8108 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32674 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 0, 4073, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 16, 0, 0, 0, 0, 4088, 4074, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 4095, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 6, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8191, 8180, 8206, 8197, 8185, 8212, 8190] avg=8192.0000 max=8212 max_rank=6 min=8175 min_rank=0 imbalance_ratio=1.002441 max_load_reduction=57324 max_load_reduction_pct=0.874695 moved_to_echo=32784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 4096, 4074, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8180, 8179, 8195, 8200, 8208, 8189] avg=8192.0000 max=8208 max_rank=6 min=8179 min_rank=3 imbalance_ratio=1.001953 max_load_reduction=57328 max_load_reduction_pct=0.874756 moved_to_echo=32792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4089, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7, 0, 0, 0, 0, 4096, 4079, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8192, 8167, 8194, 8200, 8189, 8200, 8198] avg=8192.0000 max=8200 max_rank=4 min=8167 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32640, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 128, 128, 256, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 32768, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 2, 0, 2, 0, 0, 282, 0, 2, 0, 0, 7, 4094, 3, 4093, 4083, 0, 0, 0, 2, 4089, 2, 4093, 4090, 2, 0, 3, 0, 4093, 4092, 3, 4089, 4094, 2, 4093, 0, 0, 0, 1, 0, 0, 4, 3896, 0, 1, 2, 0, 4093, 3, 2, 2, 4094, 2, 0, 2, 4093, 3, 2, 0, 4017, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 2, 0, 1, 0, 0, 24, 0, 1, 0, 0, 1, 4095, 1, 4094, 4095, 0, 0, 0, 2, 4091, 1, 4095, 4092, 1, 0, 1, 0, 4095, 4094, 1, 4094, 4093, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 2, 1, 4094, 1, 1, 1, 4095, 1, 0, 1, 4095, 0, 5, 0, 4091, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1881, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3695, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 27, 0, 2589, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 933, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 3421, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 3835, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4089, 0, 4096, 1, 0, 0, 0, 12, 0, 0, 3894, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 5, 4019, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3, 0, 0, 14, 8, 0, 6, 0, 0, 145, 0, 7, 0, 0, 3, 4087, 9, 4084, 4073, 0, 0, 0, 7, 4074, 9, 4087, 4075, 5, 0, 11, 0, 4086, 4085, 9, 4062, 4090, 4, 4087, 0, 0, 0, 1, 0, 0, 16, 3997, 3, 17, 3, 11, 4084, 11, 1, 2, 4087, 8, 0, 6, 4088, 14, 0, 0, 4052, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4075, 0, 4096, 8, 0, 0, 0, 21, 0, 0, 3721, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 0, 3436, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1699, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 8, 0, 2628, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 2, 0, 2, 0, 4093, 4090, 0, 0, 0, 0, 0, 0, 0, 282, 0, 2, 0, 0, 7, 4094, 3311, 0, 0, 0, 0, 0, 0, 0, 3, 6, 4083, 0, 0, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4090, 2, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 4089, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 3896, 0, 1, 4087, 273, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 3, 2, 2, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 3, 2, 0, 4017, 1, 499, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7696, 8183, 8193, 8197, 8262, 8195, 8620] avg=8192.0000 max=8620 max_rank=7 min=7696 min_rank=1 imbalance_ratio=1.052246 max_load_reduction=56916 max_load_reduction_pct=0.868469 moved_to_echo=33274 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 2, 0, 1, 0, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 24, 0, 1, 0, 0, 1, 4095, 3313, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4095, 0, 0, 0, 2, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 4094, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4088, 273, 2, 0, 0, 0, 0, 0, 2, 1, 4094, 1, 1, 1, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 5, 0, 4091, 1, 500, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7434, 8195, 8190, 8197, 8438, 8193, 8696] avg=8192.0000 max=8696 max_rank=7 min=7434 min_rank=1 imbalance_ratio=1.061523 max_load_reduction=56840 max_load_reduction_pct=0.867310 moved_to_echo=33524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 4094, 3315, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 4089, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3894, 0, 0, 4090, 273, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 5, 4019, 0, 500, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 7676, 8198, 8195, 8194, 8269, 8189, 8626] avg=8192.0000 max=8626 max_rank=7 min=7676 min_rank=1 imbalance_ratio=1.052979 max_load_reduction=56910 max_load_reduction_pct=0.868378 moved_to_echo=33278 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 14, 8, 0, 6, 0, 4087, 4084, 0, 0, 0, 0, 0, 0, 0, 145, 0, 7, 0, 0, 3, 4087, 3306, 0, 0, 0, 0, 0, 0, 0, 9, 6, 4073, 0, 0, 0, 7, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4075, 5, 0, 11, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 4062, 4090, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 16, 3997, 3, 17, 4078, 273, 3, 0, 0, 0, 0, 0, 3, 11, 4084, 11, 1, 2, 4084, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4088, 14, 0, 0, 4052, 5, 498, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 7548, 8169, 8186, 8173, 8388, 8204, 8666] avg=8192.0000 max=8666 max_rank=7 min=7548 min_rank=1 imbalance_ratio=1.057861 max_load_reduction=56870 max_load_reduction_pct=0.867767 moved_to_echo=33431 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 1881, 0, 0, 0, 0, 0, 4096, 3315, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 0, 0, 4090, 273, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 27, 0, 2589, 0, 500, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 9292, 8198, 8193, 8200, 8060, 8189, 7215] avg=8192.0000 max=9292 max_rank=1 min=7215 min_rank=7 imbalance_ratio=1.134277 max_load_reduction=56244 max_load_reduction_pct=0.858215 moved_to_echo=31664 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 1025, 0, 0, 0, 0, 0, 4094, 3315, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 4075, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 3721, 0, 0, 4090, 273, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 3436, 0, 500, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8434, 8198, 8195, 8187, 8105, 8189, 8039] avg=8192.0000 max=8434 max_rank=1 min=8039 min_rank=7 imbalance_ratio=1.029541 max_load_reduction=57102 max_load_reduction_pct=0.871307 moved_to_echo=32520 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 933, 0, 0, 0, 0, 0, 4096, 3315, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3421, 0, 0, 4090, 273, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 3835, 0, 503, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8344, 8198, 8192, 8196, 7787, 8189, 8438] avg=8192.0000 max=8438 max_rank=7 min=7787 min_rank=5 imbalance_ratio=1.030029 max_load_reduction=57098 max_load_reduction_pct=0.871246 moved_to_echo=32610 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 1699, 0, 0, 0, 0, 0, 4096, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 4095, 279, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 2628, 0, 500, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 9112, 8197, 8192, 8192, 8227, 8188, 7236] avg=8192.0000 max=9112 max_rank=1 min=7236 min_rank=7 imbalance_ratio=1.112305 max_load_reduction=56424 max_load_reduction_pct=0.860962 moved_to_echo=31843 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 128, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 28672, 128, 4096, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 30592, 128, 128, 32768, 2304, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 6272, 0, 128, 0, 0, 128, 32768, 26624, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 4093, 0, 2, 4085, 3, 0, 0, 4093, 3, 0, 4093, 9, 0, 630, 3, 0, 2, 4045, 2, 2, 4093, 3, 2, 3, 4092, 0, 0, 0, 2, 17, 0, 0, 4093, 0, 0, 1, 3510, 0, 3, 1, 4094, 0, 3, 4077, 3, 3, 0, 3, 0, 0, 6, 0, 0, 0, 2, 0, 4093, 4093, 4088, 4093, 4091, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 4095, 0, 1, 4093, 1, 0, 0, 4094, 1, 0, 4095, 2, 0, 811, 1, 0, 1, 3684, 1, 1, 4094, 2, 0, 2, 4095, 0, 2, 0, 1, 0, 0, 3, 4094, 0, 0, 1, 3694, 0, 2, 1, 4095, 0, 1, 4094, 0, 2, 0, 2, 0, 0, 2, 0, 0, 0, 1, 0, 4095, 4092, 4093, 4092, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 2, 4090, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 922, 0, 0, 0, 3745, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 23, 4096, 9, 0, 0, 3520, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4074, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1303, 0, 0, 0, 3576, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 21, 4096, 4, 0, 0, 3309, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 2, 4082, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2597, 0, 0, 0, 3264, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 21, 4096, 19, 0, 0, 2328, 0, 0, 0, 4080, 0, 0, 4096, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4073, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[7, 0, 4088, 7, 6, 4083, 11, 0, 0, 4083, 9, 0, 4079, 19, 0, 344, 12, 0, 4, 3957, 8, 8, 4084, 11, 19, 10, 4078, 0, 0, 0, 7, 70, 0, 0, 4084, 0, 0, 5, 3860, 0, 14, 6, 4087, 0, 10, 4018, 4, 9, 0, 0, 0, 0, 15, 0, 0, 0, 10, 0, 4086, 4086, 4069, 4085, 4079, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 943, 0, 0, 0, 3859, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 1, 0, 0, 3395, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4087, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2697, 0, 0, 0, 3911, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 4, 0, 0, 1583, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4093, 0, 2, 4085, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2814, 3, 0, 4093, 9, 0, 630, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 4045, 2, 2, 4093, 3, 335, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4092, 0, 0, 0, 2, 17, 4077, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 3510, 0, 930, 2, 0, 0, 0, 0, 0, 0, 3, 1, 4094, 0, 3, 4077, 3, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6, 0, 0, 0, 4093, 4091, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4088, 4093, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 7549, 8485, 8195, 8536, 8196, 8193, 8195] avg=8192.0000 max=8536 max_rank=4 min=7549 min_rank=1 imbalance_ratio=1.041992 max_load_reduction=57000 max_load_reduction_pct=0.869751 moved_to_echo=33120 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4095, 0, 1, 4093, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2814, 1, 0, 4095, 2, 0, 811, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3684, 1, 1, 4094, 2, 335, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 2, 0, 1, 0, 4079, 3, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 1, 3694, 0, 931, 2, 0, 0, 0, 0, 0, 0, 2, 1, 4095, 0, 1, 4094, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 4095, 4089, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 4092, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7723, 8119, 8182, 8725, 8207, 8188, 8199] avg=8192.0000 max=8725 max_rank=4 min=7723 min_rank=1 imbalance_ratio=1.065063 max_load_reduction=56811 max_load_reduction_pct=0.866867 moved_to_echo=33319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4090, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 0, 4096, 0, 0, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3745, 0, 0, 4096, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4081, 2, 0, 0, 0, 0, 0, 0, 0, 23, 4096, 9, 0, 0, 3520, 0, 931, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4074, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 7834, 8176, 8181, 8581, 8197, 8197, 8183] avg=8192.0000 max=8581 max_rank=4 min=7834 min_rank=1 imbalance_ratio=1.047485 max_load_reduction=56955 max_load_reduction_pct=0.869064 moved_to_echo=33158 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4082, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 0, 4096, 0, 0, 2597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 0, 0, 4096, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4081, 2, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 19, 0, 0, 2328, 0, 931, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4073, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 9509, 7695, 8180, 7397, 8188, 8203, 8182] avg=8192.0000 max=9509 max_rank=1 min=7397 min_rank=4 imbalance_ratio=1.160767 max_load_reduction=56027 max_load_reduction_pct=0.854904 moved_to_echo=31970 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 4088, 7, 6, 4083, 11, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2807, 9, 0, 4079, 19, 0, 344, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4, 3957, 8, 8, 4084, 11, 334, 0, 0, 0, 0, 0, 0, 0, 19, 10, 4078, 0, 0, 0, 7, 70, 4065, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 5, 3860, 0, 928, 2, 0, 0, 0, 0, 0, 0, 14, 6, 4087, 0, 10, 4018, 4, 9, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4086, 4083, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4069, 4085, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 7258, 8418, 8252, 8879, 8160, 8184, 8181] avg=8192.0000 max=8879 max_rank=4 min=7258 min_rank=1 imbalance_ratio=1.083862 max_load_reduction=56657 max_load_reduction_pct=0.864517 moved_to_echo=33404 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 0, 4096, 0, 0, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3576, 0, 0, 4096, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4081, 3, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 4, 0, 0, 3309, 0, 931, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8215, 8007, 8180, 8363, 8185, 8190, 8205] avg=8192.0000 max=8363 max_rank=4 min=8007 min_rank=2 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32943 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 0, 4096, 0, 0, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3859, 0, 0, 4096, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4081, 3, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 1, 0, 0, 3395, 0, 931, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7855, 8290, 8180, 8429, 8203, 8190, 8196] avg=8192.0000 max=8429 max_rank=4 min=7855 min_rank=1 imbalance_ratio=1.028931 max_load_reduction=57107 max_load_reduction_pct=0.871384 moved_to_echo=33018 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 4095, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2800, 0, 0, 4096, 0, 0, 2697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 0, 4096, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4087, 3, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4, 0, 0, 1583, 0, 935, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 9593, 8346, 8186, 6626, 8200, 8191, 8195] avg=8192.0000 max=9593 max_rank=1 min=6626 min_rank=4 imbalance_ratio=1.171021 max_load_reduction=55943 max_load_reduction_pct=0.853622 moved_to_echo=31212 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 25216, 0, 7552, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 0, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 22528, 128, 0, 32768, 128, 0, 10368, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 30080, 128, 128, 32768, 128, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 4093, 2, 4089, 38, 4093, 0, 3980, 0, 3, 0, 355, 3, 4090, 0, 0, 4093, 3819, 13, 0, 0, 0, 4093, 0, 0, 2, 4081, 0, 0, 3, 5, 2, 10, 2, 2, 4094, 6, 4084, 1, 2, 9, 4089, 2, 2, 3, 0, 0, 4094, 0, 0, 4092, 0, 2, 2, 3, 3926, 3, 191, 0, 0, 2, 4056, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 34, 0, 0, 4096, 0, 4096, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4046, 0, 9, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[2, 4095, 1, 4094, 12, 4094, 0, 4048, 0, 1, 0, 457, 1, 4092, 0, 0, 4094, 4006, 0, 0, 1, 0, 4095, 0, 0, 1, 4091, 0, 0, 1, 0, 1, 15, 1, 2, 4095, 2, 4093, 1, 1, 45, 4094, 1, 1, 2, 0, 0, 4095, 0, 0, 4095, 1, 1, 0, 1, 3528, 2, 179, 0, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 227, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3542, 0, 325, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 139, 0, 0, 4091, 0, 4096, 0, 0, 44, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3654, 0, 257, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 4086, 4, 4070, 104, 4085, 0, 3697, 0, 15, 0, 1708, 10, 4078, 0, 0, 4084, 3492, 35, 0, 0, 0, 4087, 0, 0, 9, 4053, 0, 0, 6, 18, 8, 13, 9, 8, 4088, 28, 4064, 5, 8, 55, 4067, 7, 9, 12, 0, 0, 4088, 2, 0, 4083, 4, 7, 6, 14, 2835, 10, 519, 0, 2, 0, 3943, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4, 0, 0, 4096, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 191, 0, 0, 4090, 0, 4096, 0, 0, 92, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 3310, 0, 496, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 30, 0, 0, 4096, 0, 4096, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2869, 0, 1191, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 4089, 38, 56, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 355, 3, 4090, 0, 0, 3778, 0, 0, 0, 0, 0, 0, 0, 4093, 3819, 13, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4081, 0, 0, 3, 5, 2, 4093, 1, 0, 0, 0, 0, 0, 0, 10, 2, 2, 4011, 6, 4084, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4089, 2, 2, 3, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 2, 3, 3926, 514, 83, 30, 0, 0, 0, 0, 0, 3, 191, 0, 0, 2, 4056, 2, 0, 3461, 258, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8229, 8043, 8187, 8118, 8169, 8652, 7973] avg=8192.0000 max=8652 max_rank=6 min=7973 min_rank=7 imbalance_ratio=1.056152 max_load_reduction=56884 max_load_reduction_pct=0.867981 moved_to_echo=32912 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1, 4094, 12, 56, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 457, 1, 4092, 0, 0, 3779, 0, 0, 0, 0, 0, 0, 0, 4094, 4006, 0, 0, 1, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 15, 1, 2, 4012, 2, 4093, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4094, 1, 1, 2, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 1, 3528, 514, 83, 30, 0, 0, 0, 0, 0, 2, 179, 0, 0, 0, 4094, 0, 0, 3463, 258, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8330, 8219, 8190, 8127, 8208, 8253, 7996] avg=8192.0000 max=8330 max_rank=1 min=7996 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32584 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 55, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 3781, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 139, 0, 0, 4008, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 4096, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3654, 514, 83, 30, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 4096, 0, 0, 3464, 259, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 7879, 8311, 8197, 8243, 8206, 8377, 8076] avg=8192.0000 max=8377 max_rank=6 min=7879 min_rank=1 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32902 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 55, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3781, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 191, 0, 0, 4007, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 4096, 0, 0, 0, 0, 1, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3310, 514, 83, 30, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 4096, 0, 0, 3464, 259, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 7877, 8314, 8201, 8294, 8255, 8033, 8315] avg=8192.0000 max=8315 max_rank=7 min=7877 min_rank=1 imbalance_ratio=1.015015 max_load_reduction=57221 max_load_reduction_pct=0.873123 moved_to_echo=32897 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 4070, 104, 55, 0, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1708, 10, 4078, 0, 0, 3771, 0, 0, 0, 0, 0, 0, 0, 4084, 3492, 35, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4053, 0, 0, 6, 18, 8, 4086, 1, 0, 0, 0, 0, 0, 0, 13, 9, 8, 4005, 28, 4064, 5, 8, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4067, 7, 9, 12, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4083, 4, 7, 6, 14, 2835, 513, 83, 30, 0, 0, 0, 0, 0, 10, 519, 0, 2, 0, 3943, 1, 0, 3456, 258, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7930, 9582, 7729, 8181, 8140, 8208, 7577, 8189] avg=8192.0000 max=9582 max_rank=1 min=7577 min_rank=6 imbalance_ratio=1.169678 max_load_reduction=55954 max_load_reduction_pct=0.853790 moved_to_echo=32114 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 55, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 3781, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 227, 0, 0, 4013, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3542, 514, 83, 30, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 4096, 0, 0, 3464, 259, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 7878, 8310, 8193, 8336, 8163, 8265, 8144] avg=8192.0000 max=8336 max_rank=4 min=7878 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 55, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 3781, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 34, 0, 0, 4013, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4046, 514, 83, 30, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 3464, 259, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 7878, 8310, 8193, 8143, 8168, 8769, 7828] avg=8192.0000 max=8769 max_rank=6 min=7828 min_rank=7 imbalance_ratio=1.070435 max_load_reduction=56767 max_load_reduction_pct=0.866196 moved_to_echo=32908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 48, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 3785, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 30, 0, 0, 4009, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2869, 521, 87, 37, 0, 0, 0, 0, 0, 0, 1191, 0, 0, 0, 4096, 0, 0, 3467, 261, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 7883, 8300, 8194, 8135, 8159, 7610, 9015] avg=8192.0000 max=9015 max_rank=7 min=7610 min_rank=6 imbalance_ratio=1.100464 max_load_reduction=56521 max_load_reduction_pct=0.862442 moved_to_echo=32919 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 32768, 128, 128, 128, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 128, 128, 32128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 3200, 0, 128, 128, 32640, 128, 0, 27776, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 2560, 128, 32768, 0, 0, 30336, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 32768, 128, 128, 128, 128, 27776, 4224, 768, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 31872, 128, 128, 128, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 256, 512, 0, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 0, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4054, 0, 0, 0, 0, 0, 4082, 4090, 3, 3, 2, 3, 2, 2, 0, 8, 4088, 0, 0, 2, 0, 40, 4092, 0, 0, 0, 4094, 3, 4093, 0, 4093, 0, 4093, 4, 0, 4093, 2, 3, 7, 0, 0, 12, 1, 0, 4092, 4, 0, 0, 3, 4093, 4092, 4085, 2, 0, 4087, 0, 2, 2, 3, 1, 2, 4091, 5, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4073, 0, 3, 0, 0, 0, 4094, 4094, 1, 1, 1, 1, 0, 1, 0, 2, 4094, 0, 0, 1, 0, 19, 4094, 0, 0, 1, 4095, 2, 4095, 0, 4094, 0, 4095, 1, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 1, 4095, 4094, 4093, 1, 0, 4092, 1, 2, 1, 0, 2, 1, 4093, 3, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 6, 4096, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4053, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 43, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[3817, 0, 15, 1, 0, 0, 4031, 4077, 9, 11, 10, 8, 4, 6, 0, 53, 4031, 0, 0, 7, 0, 272, 4081, 0, 0, 0, 4088, 11, 4084, 0, 4085, 0, 4087, 19, 0, 4088, 0, 9, 27, 0, 1, 68, 0, 0, 4086, 12, 0, 0, 9, 4087, 4086, 4058, 6, 0, 4064, 2, 8, 5, 10, 11, 5, 4077, 0, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 4096, 0, 4096, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4086, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 4096, 0, 4095, 0, 4096, 6, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 21, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4054, 0, 0, 0, 0, 0, 4082, 62, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 2, 3, 2, 2, 0, 8, 4093, 4080, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 2, 0, 40, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4, 0, 4093, 2, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 4092, 4, 0, 0, 4028, 38, 10, 6, 0, 0, 0, 0, 3, 0, 10, 4085, 2, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3, 1, 2, 4091, 5, 3, 4072, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8196, 8184, 8191, 8196, 8191, 8187, 8193] avg=8192.0000 max=8198 max_rank=0 min=8184 min_rank=2 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 3, 0, 0, 0, 4094, 62, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 2, 4095, 4082, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 19, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 4095, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 4032, 38, 10, 6, 0, 0, 0, 0, 1, 0, 10, 4093, 1, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 2, 1, 4093, 3, 1, 4074, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8184, 8170, 8192, 8187, 8183, 8198, 8190] avg=8192.0000 max=8232 max_rank=0 min=8170 min_rank=2 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4034, 38, 10, 6, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8178, 8154, 8198, 8186, 8184, 8202, 8186] avg=8192.0000 max=8248 max_rank=0 min=8154 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3817, 0, 15, 1, 0, 0, 4031, 62, 0, 0, 0, 0, 0, 0, 0, 0, 9, 11, 10, 8, 4, 6, 0, 53, 4087, 4076, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 7, 0, 272, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4084, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 19, 0, 4088, 0, 9, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 68, 0, 0, 4086, 12, 0, 0, 4015, 38, 10, 6, 0, 0, 0, 0, 9, 0, 10, 4058, 6, 0, 4064, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 10, 11, 5, 4077, 0, 10, 4066, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7926, 8264, 8353, 8180, 8224, 8236, 8149, 8204] avg=8192.0000 max=8353 max_rank=2 min=7926 min_rank=0 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32813 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4053, 0, 0, 0, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4058, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4034, 38, 10, 6, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8178, 8197, 8192, 8186, 8184, 8202, 8186] avg=8192.0000 max=8211 max_rank=0 min=8178 min_rank=1 imbalance_ratio=1.002319 max_load_reduction=57325 max_load_reduction_pct=0.874710 moved_to_echo=32758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4058, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4034, 38, 10, 6, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 4076, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8178, 8162, 8198, 8187, 8175, 8202, 8206] avg=8192.0000 max=8228 max_rank=0 min=8162 min_rank=2 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4058, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4034, 38, 10, 6, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8252, 8179, 8156, 8193, 8186, 8183, 8202, 8185] avg=8192.0000 max=8252 max_rank=0 min=8156 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 0, 0, 0, 0, 4096, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4054, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4037, 42, 17, 8, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8241, 8179, 8160, 8192, 8184, 8200, 8194, 8186] avg=8192.0000 max=8241 max_rank=0 min=8160 min_rank=2 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32384, 0, 128, 128, 0, 0, 32768, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 0, 0, 32256, 384, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 0, 384, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3, 4, 2, 0, 1, 16, 0, 0, 9, 1, 4, 0, 3, 2, 0, 4092, 4093, 0, 4093, 4091, 2, 4092, 0, 2, 4094, 2, 4093, 0, 0, 3871, 4078, 4093, 0, 2, 0, 223, 0, 4094, 0, 4087, 3, 0, 3, 0, 4087, 0, 2, 0, 4, 1, 0, 0, 1, 3, 4093, 0, 0, 4090, 3, 4093, 0, 0, 4, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 27, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 4095, 0, 4096, 0, 0, 4096, 4096, 4085, 0, 0, 0, 0, 11, 4090, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 4096, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[1, 1, 1, 0, 1, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 4094, 4080, 1, 4095, 4094, 1, 4094, 0, 1, 4094, 2, 4094, 0, 0, 3981, 4093, 4095, 0, 1, 0, 128, 0, 4094, 0, 4094, 1, 2, 1, 0, 4093, 1, 0, 0, 2, 0, 0, 0, 0, 2, 4094, 0, 0, 4094, 1, 4095, 0, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[10, 8, 7, 0, 5, 61, 0, 0, 52, 2, 11, 0, 11, 7, 0, 4083, 4083, 0, 4087, 4080, 6, 4085, 0, 14, 4085, 8, 4087, 0, 0, 3471, 4032, 4087, 0, 8, 0, 618, 0, 4088, 0, 4041, 11, 0, 9, 0, 4069, 0, 1, 1, 13, 3, 0, 0, 5, 11, 4086, 0, 0, 4075, 8, 4085, 0, 0, 13, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 6, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 4089, 0, 4096, 0, 0, 4096, 4096, 4076, 0, 0, 0, 0, 20, 4088, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 4096, 0, 4096, 8, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 4, 2, 0, 1, 16, 0, 0, 4093, 4081, 0, 0, 0, 0, 0, 0, 9, 1, 4, 0, 3, 2, 0, 4092, 3963, 119, 3, 1, 0, 0, 0, 0, 4093, 0, 0, 4091, 2, 9, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3871, 4078, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 223, 0, 3975, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 4087, 0, 2, 0, 4085, 12, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 1, 3, 4093, 0, 4080, 9, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 4092, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8197, 8197, 8081, 8287, 8192, 8191, 8191] avg=8192.0000 max=8287 max_rank=4 min=8081 min_rank=3 imbalance_ratio=1.011597 max_load_reduction=57249 max_load_reduction_pct=0.873550 moved_to_echo=32861 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 0, 1, 1, 0, 0, 4095, 4082, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 2, 1, 0, 4094, 3965, 119, 4, 1, 0, 0, 0, 0, 4080, 1, 0, 4094, 1, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3981, 4093, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 128, 0, 3975, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 4093, 1, 0, 0, 4086, 12, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 4094, 0, 4082, 8, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4094, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8189, 8185, 8206, 8198, 8196, 8188, 8192] avg=8192.0000 max=8206 max_rank=3 min=8182 min_rank=0 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32774 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3955, 118, 4, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3972, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 4084, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8174, 8200, 8322, 8079, 8194, 8195, 8192] avg=8192.0000 max=8322 max_rank=3 min=8079 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32660 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3966, 119, 4, 1, 0, 0, 0, 0, 4069, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3977, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8186, 8173, 8322, 8100, 8195, 8189, 8191] avg=8192.0000 max=8322 max_rank=3 min=8100 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32675 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[10, 8, 7, 0, 5, 61, 0, 0, 4087, 4075, 0, 0, 0, 0, 0, 0, 52, 2, 11, 0, 11, 7, 0, 4083, 3957, 118, 3, 1, 0, 0, 0, 0, 4083, 0, 0, 4080, 6, 9, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 3471, 4032, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 618, 0, 3970, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 9, 0, 4069, 0, 1, 1, 4077, 12, 0, 0, 0, 0, 0, 0, 13, 3, 0, 0, 5, 11, 4086, 0, 4073, 8, 0, 0, 0, 0, 0, 0, 0, 4075, 8, 4084, 0, 0, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8253, 8245, 8192, 7641, 8637, 8180, 8199, 8189] avg=8192.0000 max=8637 max_rank=4 min=7641 min_rank=3 imbalance_ratio=1.054321 max_load_reduction=56899 max_load_reduction_pct=0.868210 moved_to_echo=33205 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3946, 118, 4, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 3970, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4080, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 4084, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8165, 8200, 8322, 8086, 8188, 8196, 8199] avg=8192.0000 max=8322 max_rank=3 min=8086 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32669 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3966, 119, 4, 1, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3976, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8186, 8199, 8322, 8073, 8196, 8188, 8192] avg=8192.0000 max=8322 max_rank=3 min=8073 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32649 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3968, 122, 6, 2, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3974, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8194, 8190, 8320, 8076, 8195, 8190, 8190] avg=8192.0000 max=8320 max_rank=3 min=8076 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32651 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 1024, 128, 31872, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 32768, 31744, 1024, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 128, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 31872, 32768, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[83, 4, 98, 4, 55, 0, 1169, 0, 676, 152, 2, 1412, 2518, 1685, 0, 3720, 3730, 136, 17, 1190, 0, 2267, 170, 1221, 0, 20, 215, 2195, 1486, 1, 457, 1701, 4082, 32, 132, 2510, 2212, 2, 2044, 454, 4061, 0, 1288, 2260, 0, 111, 1358, 1596, 1081, 3633, 1129, 0, 0, 27, 5, 3158, 0, 992, 428, 526, 1788, 1455, 2707, 81] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[18, 0, 38, 0, 8, 1, 1851, 3, 483, 114, 5, 1321, 1513, 2003, 1, 3414, 3531, 132, 4, 631, 2, 3161, 147, 1281, 4, 116, 198, 1846, 1904, 3, 515, 2250, 4089, 4, 81, 2845, 1885, 3, 1424, 781, 4078, 0, 1074, 2482, 1, 139, 2229, 1226, 555, 3472, 936, 0, 0, 1, 0, 3199, 0, 792, 615, 685, 1269, 2245, 2864, 64] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[144, 2, 42, 2, 20, 1, 1552, 0, 1563, 211, 25, 2112, 2106, 2674, 0, 3592, 3430, 98, 12, 430, 16, 2669, 191, 1253, 2, 83, 140, 1977, 1250, 2, 886, 2279, 3955, 11, 125, 2618, 2330, 11, 856, 468, 4011, 1, 1310, 2334, 1, 280, 1078, 755, 901, 3422, 940, 3, 6, 10, 1, 2719, 0, 1368, 842, 867, 1505, 1319, 2511, 214] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[55, 1, 60, 0, 29, 0, 1721, 1, 690, 216, 3, 1769, 2143, 2755, 0, 3289, 3085, 105, 7, 854, 0, 2541, 160, 1245, 0, 147, 202, 2507, 2066, 2, 793, 1827, 4057, 25, 194, 2949, 2226, 0, 1314, 661, 4083, 0, 1472, 2001, 0, 216, 1178, 1128, 821, 3738, 989, 0, 1, 5, 2, 2803, 0, 894, 429, 511, 1590, 1448, 2456, 72] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[44, 9, 47, 1, 66, 0, 1775, 0, 690, 154, 3, 1385, 1484, 2199, 0, 3694, 3786, 129, 9, 558, 22, 2628, 162, 1215, 5, 100, 102, 1901, 1566, 3, 478, 2213, 4086, 11, 99, 2712, 2568, 6, 1255, 982, 4082, 0, 1352, 2013, 0, 154, 1796, 1228, 576, 3852, 989, 0, 6, 0, 4, 3119, 0, 910, 545, 767, 1419, 1812, 2653, 112] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[43, 0, 32, 2, 7, 0, 1452, 0, 1213, 181, 9, 1786, 1721, 1779, 0, 3258, 3180, 197, 23, 1324, 8, 3076, 131, 1166, 1, 65, 112, 1996, 2126, 0, 1000, 1870, 4089, 5, 297, 2952, 2498, 7, 1149, 434, 4041, 1, 1323, 2648, 2, 101, 991, 975, 780, 3658, 1172, 2, 0, 1, 8, 2997, 0, 1006, 533, 478, 1683, 1483, 2428, 36] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[17, 0, 100, 0, 6, 0, 1062, 0, 759, 134, 10, 1677, 2097, 2373, 0, 3436, 3363, 89, 39, 707, 0, 2317, 174, 1230, 0, 25, 185, 2246, 1499, 6, 661, 2721, 4093, 1, 268, 2742, 2339, 0, 1304, 508, 4040, 0, 1257, 2539, 0, 190, 1211, 1129, 777, 3340, 1186, 0, 0, 0, 23, 2915, 0, 818, 643, 539, 1454, 2488, 2728, 71] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[53, 1, 38, 8, 8, 2, 1416, 1, 740, 154, 10, 1522, 1607, 2570, 0, 3807, 3596, 168, 13, 550, 1, 2628, 178, 1411, 0, 69, 180, 1906, 1289, 9, 619, 2080, 4087, 10, 95, 2579, 1813, 5, 1237, 760, 4012, 1, 1623, 2024, 0, 178, 1755, 1145, 1268, 3713, 1195, 2, 0, 20, 14, 3067, 0, 815, 609, 647, 1628, 2068, 2403, 129] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[144, 2, 42, 2, 20, 1, 1552, 0, 2897, 2231, 1339, 0, 0, 0, 0, 0, 1563, 211, 25, 2112, 2106, 2674, 0, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 3152, 98, 12, 430, 16, 2669, 191, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 2, 83, 140, 1977, 1250, 2, 886, 2279, 835, 321, 267, 0, 0, 0, 0, 0, 1058, 11, 125, 2618, 2330, 11, 856, 468, 0, 0, 0, 0, 0, 0, 0, 0, 1780, 1, 1310, 2334, 1, 280, 1078, 755, 0, 0, 0, 0, 0, 0, 0, 0, 901, 3101, 940, 3, 6, 10, 1, 2719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1368, 842, 867, 1505, 1319, 2511, 214, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 10109, 7821, 8042, 7477, 7539, 7681, 8637] avg=8192.0000 max=10109 max_rank=1 min=7477 min_rank=4 imbalance_ratio=1.234009 max_load_reduction=55427 max_load_reduction_pct=0.845749 moved_to_echo=31334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 38, 0, 8, 1, 1851, 3, 2995, 2268, 1273, 0, 0, 0, 0, 0, 483, 114, 5, 1321, 1513, 2003, 1, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 3245, 132, 4, 631, 2, 3161, 147, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 4, 116, 198, 1846, 1904, 3, 515, 2250, 793, 325, 275, 0, 0, 0, 0, 0, 1094, 4, 81, 2845, 1885, 3, 1424, 781, 0, 0, 0, 0, 0, 0, 0, 0, 1810, 0, 1074, 2482, 1, 139, 2229, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 555, 3147, 936, 0, 0, 1, 0, 3199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 615, 685, 1269, 2245, 2864, 64, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8455, 6788, 8603, 8229, 8117, 8961, 7838, 8545] avg=8192.0000 max=8961 max_rank=5 min=6788 min_rank=1 imbalance_ratio=1.093872 max_load_reduction=56575 max_load_reduction_pct=0.863266 moved_to_echo=33461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[44, 9, 47, 1, 66, 0, 1775, 0, 2993, 2270, 1377, 0, 0, 0, 0, 0, 690, 154, 3, 1385, 1484, 2199, 0, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 3479, 129, 9, 558, 22, 2628, 162, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 5, 100, 102, 1901, 1566, 3, 478, 2213, 859, 361, 295, 0, 0, 0, 0, 0, 1093, 11, 99, 2712, 2568, 6, 1255, 982, 0, 0, 0, 0, 0, 0, 0, 0, 1812, 0, 1352, 2013, 0, 154, 1796, 1228, 0, 0, 0, 0, 0, 0, 0, 0, 576, 3491, 989, 0, 6, 0, 4, 3119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 910, 545, 767, 1419, 1812, 2653, 112, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8582, 7373, 8202, 7883, 8726, 8355, 8185, 8230] avg=8192.0000 max=8726 max_rank=4 min=7373 min_rank=1 imbalance_ratio=1.065186 max_load_reduction=56810 max_load_reduction_pct=0.866852 moved_to_echo=33496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[83, 4, 98, 4, 55, 0, 1169, 0, 2990, 2258, 1387, 0, 0, 0, 0, 0, 676, 152, 2, 1412, 2518, 1685, 0, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 3427, 136, 17, 1190, 0, 2267, 170, 1221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 215, 2195, 1486, 1, 457, 1701, 865, 340, 291, 0, 0, 0, 0, 0, 1092, 32, 132, 2510, 2212, 2, 2044, 454, 0, 0, 0, 0, 0, 0, 0, 0, 1803, 0, 1288, 2260, 0, 111, 1358, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 3293, 1129, 0, 0, 27, 5, 3158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 992, 428, 526, 1788, 1455, 2707, 81, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8048, 7913, 8428, 7571, 8478, 8416, 8693, 7989] avg=8192.0000 max=8693 max_rank=6 min=7571 min_rank=3 imbalance_ratio=1.061157 max_load_reduction=56843 max_load_reduction_pct=0.867355 moved_to_echo=33576 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[55, 1, 60, 0, 29, 0, 1721, 1, 2971, 2271, 1226, 0, 0, 0, 0, 0, 690, 216, 3, 1769, 2143, 2755, 0, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 105, 7, 854, 0, 2541, 160, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 202, 2507, 2066, 2, 793, 1827, 764, 350, 240, 0, 0, 0, 0, 0, 1086, 25, 194, 2949, 2226, 0, 1314, 661, 0, 0, 0, 0, 0, 0, 0, 0, 1812, 0, 1472, 2001, 0, 216, 1178, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 821, 3388, 989, 0, 1, 5, 2, 2803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 894, 429, 511, 1590, 1448, 2456, 72, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8335, 8875, 7747, 8898, 8455, 7807, 8009, 7410] avg=8192.0000 max=8898 max_rank=3 min=7410 min_rank=7 imbalance_ratio=1.086182 max_load_reduction=56638 max_load_reduction_pct=0.864227 moved_to_echo=31681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[53, 1, 38, 8, 8, 2, 1416, 1, 2993, 2231, 1420, 0, 0, 0, 0, 0, 740, 154, 10, 1522, 1607, 2570, 0, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 3304, 168, 13, 550, 1, 2628, 178, 1411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 180, 1906, 1289, 9, 619, 2080, 885, 348, 280, 0, 0, 0, 0, 0, 1094, 10, 95, 2579, 1813, 5, 1237, 760, 0, 0, 0, 0, 0, 0, 0, 0, 1781, 1, 1623, 2024, 0, 178, 1755, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 1268, 3365, 1195, 2, 0, 20, 14, 3067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 815, 609, 647, 1628, 2068, 2403, 129, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8105, 8253, 7665, 7593, 8507, 8931, 8311] avg=8192.0000 max=8931 max_rank=6 min=7593 min_rank=4 imbalance_ratio=1.090210 max_load_reduction=56605 max_load_reduction_pct=0.863724 moved_to_echo=33342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 100, 0, 6, 0, 1062, 0, 2998, 2247, 1281, 0, 0, 0, 0, 0, 759, 134, 10, 1677, 2097, 2373, 0, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 3090, 89, 39, 707, 0, 2317, 174, 1230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 185, 2246, 1499, 6, 661, 2721, 799, 313, 262, 0, 0, 0, 0, 0, 1095, 1, 268, 2742, 2339, 0, 1304, 508, 0, 0, 0, 0, 0, 0, 0, 0, 1793, 0, 1257, 2539, 0, 190, 1211, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 777, 3027, 1186, 0, 0, 0, 23, 2915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 643, 539, 1454, 2488, 2728, 71, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7711, 8406, 7646, 8717, 8257, 8119, 7928, 8752] avg=8192.0000 max=8752 max_rank=7 min=7646 min_rank=2 imbalance_ratio=1.068359 max_load_reduction=56784 max_load_reduction_pct=0.866455 moved_to_echo=33056 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[43, 0, 32, 2, 7, 0, 1452, 0, 2998, 2250, 1220, 0, 0, 0, 0, 0, 1213, 181, 9, 1786, 1721, 1779, 0, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 2911, 197, 23, 1324, 8, 3076, 131, 1166, 0, 0, 0, 0, 0, 0, 0, 0, 1, 65, 112, 1996, 2126, 0, 1000, 1870, 760, 347, 254, 0, 0, 0, 0, 0, 1091, 5, 297, 2952, 2498, 7, 1149, 434, 0, 0, 0, 0, 0, 0, 0, 0, 1791, 1, 1323, 2648, 2, 101, 991, 975, 0, 0, 0, 0, 0, 0, 0, 0, 780, 3311, 1172, 2, 0, 1, 8, 2997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 533, 478, 1683, 1483, 2428, 36, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8004, 7967, 8836, 8531, 8433, 7832, 8271, 7662] avg=8192.0000 max=8836 max_rank=2 min=7662 min_rank=7 imbalance_ratio=1.078613 max_load_reduction=56700 max_load_reduction_pct=0.865173 moved_to_echo=32198 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[512, 128, 512, 128, 256, 128, 12032, 128, 23936, 18048, 10624, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[14464, 128, 10752, 18304, 128, 1408, 11648, 9216, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25472, 1152, 128, 6272, 128, 21376, 1408, 10112, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 640, 1408, 16640, 13312, 128, 5504, 17024, 6656, 2816, 2176, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[8704, 128, 1408, 22016, 17920, 128, 10624, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6912, 1408, 128, 13056, 15232, 18048, 128, 11136, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6784, 26240, 8576, 128, 128, 128, 128, 24064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 7680, 4736, 5120, 12416, 14336, 20864, 896, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 29, 4088, 0, 4096, 0, 721, 0, 0, 2, 1, 0, 4095, 3312, 1, 0, 18, 2, 0, 0, 4087, 0, 0, 4094, 4, 4094, 4086, 1, 226, 5, 0, 4094, 2, 0, 4094, 4092, 75, 3, 0, 2, 4093, 1, 1, 1, 43, 0, 0, 3793, 1, 0, 2, 1, 2, 0, 3, 0, 4094, 0, 4082, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[4073, 0, 0, 0, 30, 4096, 1, 4096, 0, 434, 0, 9, 0, 0, 1, 4092, 3710, 0, 0, 43, 0, 0, 0, 4069, 1, 2, 4093, 9, 4096, 4075, 0, 74, 1, 3, 4086, 0, 15, 4092, 4096, 131, 0, 0, 0, 4096, 14, 0, 0, 4, 0, 0, 3818, 0, 1, 1, 0, 0, 2, 3, 0, 4083, 0, 4084, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 4095, 0, 4096, 0, 490, 0, 0, 0, 0, 0, 4088, 3514, 0, 0, 10, 0, 0, 0, 4072, 0, 0, 4096, 0, 4096, 4094, 0, 32, 1, 0, 4096, 0, 4, 4096, 4096, 231, 2, 0, 0, 4096, 0, 0, 0, 29, 0, 0, 3941, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4072, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 1, 4096, 0, 4096, 0, 1254, 0, 0, 0, 0, 0, 4085, 2543, 0, 0, 164, 0, 0, 0, 4093, 2, 0, 4096, 0, 4096, 4096, 0, 36, 0, 0, 4096, 0, 2, 4096, 4093, 538, 0, 0, 0, 4094, 0, 0, 0, 18, 0, 0, 3656, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 4093, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 1, 4096, 0, 4096, 0, 1371, 0, 0, 0, 0, 0, 4095, 2579, 0, 0, 10, 0, 0, 0, 4088, 0, 0, 4096, 0, 4096, 4092, 0, 60, 0, 0, 4096, 0, 0, 4096, 4096, 961, 1, 0, 0, 4096, 3, 0, 0, 3, 0, 0, 3223, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4094, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4094, 1, 1, 1, 14, 4087, 0, 4094, 0, 651, 0, 0, 0, 0, 0, 4094, 3153, 1, 0, 36, 2, 0, 0, 4085, 1, 0, 4095, 0, 4095, 4090, 0, 260, 4, 0, 4095, 1, 3, 4093, 4095, 399, 16, 1, 1, 4095, 1, 1, 1, 149, 0, 0, 3591, 0, 1, 1, 6, 1, 6, 0, 1, 4093, 1, 4024, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 13, 4089, 1, 4096, 0, 778, 0, 1, 0, 0, 0, 4095, 3204, 0, 0, 9, 0, 0, 0, 4094, 2, 0, 4096, 0, 4096, 4093, 0, 36, 1, 0, 4096, 0, 2, 4096, 4096, 307, 1, 0, 0, 4096, 1, 0, 0, 7, 0, 0, 3854, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4087, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 1, 4091, 0, 4096, 1, 646, 0, 9, 0, 0, 0, 4083, 2871, 0, 0, 2, 0, 0, 0, 4093, 1, 0, 4096, 0, 4096, 4090, 0, 54, 1, 0, 4095, 0, 2, 4093, 4094, 1228, 5, 0, 0, 4094, 0, 0, 0, 29, 0, 0, 3433, 0, 3, 0, 15, 0, 0, 0, 0, 4096, 0, 4023, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 29, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 721, 0, 0, 2, 1, 0, 4095, 3302, 0, 0, 0, 0, 0, 0, 0, 3312, 1, 0, 18, 2, 0, 0, 4087, 486, 402, 69, 0, 0, 0, 0, 0, 0, 0, 4003, 4, 0, 4086, 1, 226, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4094, 2, 0, 3608, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 4093, 1, 1, 1, 43, 3666, 388, 0, 0, 0, 0, 0, 0, 0, 0, 3793, 1, 0, 2, 1, 2, 4096, 428, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4094, 0, 4082, 2, 0, 22, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8121, 8377, 8320, 7784, 8198, 8323, 8205] avg=8192.0000 max=8377 max_rank=2 min=7784 min_rank=4 imbalance_ratio=1.022583 max_load_reduction=57159 max_load_reduction_pct=0.872177 moved_to_echo=32510 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 0, 0, 0, 0, 4088, 3306, 0, 0, 0, 0, 0, 0, 0, 3514, 0, 0, 10, 0, 0, 0, 4072, 486, 403, 70, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 0, 4094, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4, 3610, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 29, 3668, 387, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 1, 0, 0, 0, 4096, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4072, 0, 0, 22, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 7884, 8555, 8130, 7942, 8182, 8466, 8191] avg=8192.0000 max=8555 max_rank=2 min=7884 min_rank=1 imbalance_ratio=1.044312 max_load_reduction=56981 max_load_reduction_pct=0.869461 moved_to_echo=32781 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 0, 0, 0, 0, 0, 4085, 3303, 0, 0, 0, 0, 0, 0, 0, 2543, 0, 0, 164, 0, 0, 0, 4093, 486, 402, 70, 0, 0, 0, 0, 0, 2, 0, 4004, 0, 0, 4096, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 3610, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 18, 3667, 388, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 0, 0, 0, 0, 0, 4096, 429, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 4093, 0, 0, 22, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8642, 7758, 8138, 8246, 8167, 8181, 8218] avg=8192.0000 max=8642 max_rank=1 min=7758 min_rank=2 imbalance_ratio=1.054932 max_load_reduction=56894 max_load_reduction_pct=0.868134 moved_to_echo=32812 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4094, 1, 1, 1, 14, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 0, 0, 0, 4094, 3305, 0, 0, 0, 0, 0, 0, 0, 3153, 1, 0, 36, 2, 0, 0, 4085, 486, 403, 69, 0, 0, 0, 0, 0, 1, 0, 4004, 0, 0, 4090, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4095, 1, 3, 3607, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 1, 4095, 1, 1, 1, 149, 3667, 387, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 0, 1, 1, 6, 1, 4094, 428, 0, 0, 0, 0, 0, 0, 6, 0, 1, 4093, 1, 4024, 1, 0, 22, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8050, 8235, 8355, 8109, 8319, 8122, 8150] avg=8192.0000 max=8355 max_rank=3 min=8050 min_rank=1 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32700 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 0, 0, 30, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 0, 9, 0, 0, 1, 4092, 3306, 0, 0, 0, 0, 0, 0, 0, 3710, 0, 0, 43, 0, 0, 0, 4069, 486, 403, 69, 0, 0, 0, 0, 0, 1, 2, 4002, 9, 0, 4075, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4086, 0, 15, 3606, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 4, 3668, 387, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 0, 1, 1, 0, 0, 4096, 428, 0, 0, 0, 0, 0, 0, 2, 3, 0, 4083, 0, 4084, 2, 0, 22, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 7842, 8780, 8163, 7842, 8169, 8344, 8198] avg=8192.0000 max=8780 max_rank=2 min=7842 min_rank=1 imbalance_ratio=1.071777 max_load_reduction=56756 max_load_reduction_pct=0.866028 moved_to_echo=32553 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1371, 0, 0, 0, 0, 0, 4095, 3306, 0, 0, 0, 0, 0, 0, 0, 2579, 0, 0, 10, 0, 0, 0, 4088, 486, 403, 70, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 0, 4092, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3610, 0, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 3, 0, 0, 3, 3668, 387, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 0, 0, 0, 0, 0, 4096, 428, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4094, 0, 0, 22, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8772, 7636, 8156, 8667, 8158, 7747, 8215] avg=8192.0000 max=8772 max_rank=1 min=7636 min_rank=2 imbalance_ratio=1.070801 max_load_reduction=56764 max_load_reduction_pct=0.866150 moved_to_echo=32787 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 13, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 0, 1, 0, 0, 0, 4095, 3307, 0, 0, 0, 0, 0, 0, 0, 3204, 0, 0, 9, 0, 0, 0, 4094, 486, 402, 70, 0, 0, 0, 0, 0, 2, 0, 4004, 0, 0, 4093, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 2, 3610, 0, 307, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 1, 0, 0, 7, 3668, 387, 0, 0, 0, 0, 0, 0, 0, 0, 3854, 0, 0, 0, 0, 0, 4096, 428, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4087, 0, 0, 22, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8181, 8265, 8135, 8016, 8160, 8378, 8208] avg=8192.0000 max=8378 max_rank=6 min=8016 min_rank=4 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32762 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 646, 0, 9, 0, 0, 0, 4083, 3305, 0, 0, 0, 0, 0, 0, 0, 2871, 0, 0, 2, 0, 0, 0, 4093, 489, 402, 73, 0, 0, 0, 0, 0, 1, 0, 3994, 0, 0, 4090, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 2, 3604, 0, 1228, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4094, 0, 0, 0, 29, 3668, 387, 0, 0, 0, 0, 0, 0, 0, 0, 3433, 0, 3, 0, 15, 0, 4096, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4023, 0, 0, 29, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8044, 7930, 8139, 8930, 8183, 7975, 8151] avg=8192.0000 max=8930 max_rank=4 min=7930 min_rank=2 imbalance_ratio=1.090088 max_load_reduction=56606 max_load_reduction_pct=0.863739 moved_to_echo=33239 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 384, 29440, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 28928, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 32640, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32128, 128, 0, 32768, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 6400, 0, 128, 128, 128, 128, 32768, 26496, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[24960, 128, 0, 384, 128, 0, 0, 32768, 3968, 3328, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 29312, 128, 128, 128, 128, 128, 32768, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 0, 4091, 2, 0, 4094, 0, 0, 4, 2989, 0, 0, 0, 2, 0, 0, 0, 3847, 2, 0, 1, 4094, 0, 1, 0, 0, 4094, 0, 1, 2, 2, 0, 4091, 6, 0, 4093, 0, 4093, 0, 2, 3899, 4094, 0, 2, 1592, 0, 0, 0, 0, 4, 1, 2, 4094, 3, 4094, 4046, 2, 0, 4094, 0, 2, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3352, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 4096, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4079, 4096, 7, 0, 851, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4064, 0, 0, 4095, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1036, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 0, 4096, 0, 142, 0, 0, 4095, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 3900, 4096, 38, 0, 3488, 0, 1, 0, 1, 0, 0, 0, 4096, 0, 4096, 3924, 0, 0, 4096, 0, 0, 4089, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3022, 0, 0, 0, 0, 0, 0, 0, 3692, 0, 0, 0, 4096, 0, 51, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 3870, 4096, 1, 0, 1721, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4027, 0, 0, 4096, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3060, 0, 5, 0, 0, 0, 0, 0, 3817, 0, 0, 0, 4096, 0, 35, 5, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 4096, 0, 0, 3733, 4091, 55, 0, 1666, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4093, 4020, 0, 0, 4091, 0, 0, 4094, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 3404, 0, 0, 0, 0, 0, 0, 0, 3800, 0, 0, 0, 4096, 0, 3, 1, 1, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4009, 4096, 0, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4086, 0, 0, 4095, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 2, 1, 4094, 0, 0, 0, 3035, 1, 0, 0, 1, 0, 0, 0, 4045, 1, 1, 1, 4093, 1, 6, 0, 0, 4094, 1, 1, 2, 0, 1, 4092, 2, 0, 4092, 3, 4094, 1, 2, 3384, 4095, 0, 1, 1914, 0, 0, 1, 0, 2, 1, 1, 4094, 0, 4094, 3988, 1, 5, 4095, 0, 1, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1877, 0, 0, 0, 0, 2, 0, 0, 3711, 0, 0, 0, 4096, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 4092, 0, 2, 4096, 0, 4096, 0, 0, 3857, 4096, 4, 0, 2873, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4063, 0, 0, 4095, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4091, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2989, 0, 0, 0, 2, 0, 4092, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 2, 0, 1, 4094, 0, 246, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 1, 2, 2, 2652, 1410, 0, 0, 0, 0, 0, 0, 0, 4091, 6, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3899, 2438, 0, 2, 1592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 2, 4094, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 2, 0, 69, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8460, 8190, 8162, 8191, 7933, 8198, 8213] avg=8192.0000 max=8460 max_rank=1 min=7933 min_rank=5 imbalance_ratio=1.032715 max_load_reduction=57076 max_load_reduction_pct=0.870911 moved_to_echo=32535 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3352, 0, 0, 0, 0, 0, 4095, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 4096, 0, 246, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 2652, 1410, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 2440, 7, 0, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 70, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8820, 8361, 8171, 8193, 7377, 8192, 8230] avg=8192.0000 max=8820 max_rank=1 min=7377 min_rank=5 imbalance_ratio=1.076660 max_load_reduction=56716 max_load_reduction_pct=0.865417 moved_to_echo=31992 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 0, 0, 0, 0, 0, 4096, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 0, 4096, 0, 246, 0, 0, 0, 0, 0, 0, 0, 142, 0, 0, 4095, 0, 0, 0, 0, 2653, 1410, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3900, 2440, 38, 0, 3488, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 70, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 6505, 8206, 8300, 8190, 9867, 8193, 8083] avg=8192.0000 max=9867 max_rank=5 min=6505 min_rank=1 imbalance_ratio=1.204468 max_load_reduction=55669 max_load_reduction_pct=0.849442 moved_to_echo=34333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 2, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3035, 1, 0, 0, 1, 0, 4093, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 1, 1, 1, 4093, 1, 246, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4094, 1, 1, 2, 0, 2652, 1410, 0, 0, 0, 0, 0, 0, 1, 4092, 2, 0, 4092, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3384, 2439, 0, 1, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 1, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 1, 5, 70, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8503, 8388, 8166, 8192, 7740, 8193, 8161] avg=8192.0000 max=8503 max_rank=1 min=7740 min_rank=5 imbalance_ratio=1.037964 max_load_reduction=57033 max_load_reduction_pct=0.870255 moved_to_echo=32286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3022, 0, 0, 0, 0, 0, 4096, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 3692, 0, 0, 0, 4096, 0, 246, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 4096, 0, 0, 0, 0, 2653, 1410, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 2440, 1, 0, 1721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 70, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8491, 8034, 8210, 8192, 8032, 8192, 8193] avg=8192.0000 max=8491 max_rank=1 min=8032 min_rank=5 imbalance_ratio=1.036499 max_load_reduction=57045 max_load_reduction_pct=0.870438 moved_to_echo=32609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3060, 0, 5, 0, 0, 0, 4096, 1372, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 0, 0, 0, 4096, 0, 246, 0, 0, 0, 0, 0, 0, 0, 35, 5, 0, 4096, 0, 0, 0, 0, 2650, 1409, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3733, 2436, 55, 0, 1666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 69, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8533, 8159, 8195, 8196, 7890, 8188, 8183] avg=8192.0000 max=8533 max_rank=1 min=7890 min_rank=5 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=32457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3404, 0, 0, 0, 0, 0, 4096, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 0, 0, 0, 4096, 0, 246, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1, 4096, 0, 0, 0, 0, 2652, 1410, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 2440, 0, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 70, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8873, 8142, 8163, 8192, 7531, 8192, 8252] avg=8192.0000 max=8873 max_rank=1 min=7531 min_rank=5 imbalance_ratio=1.083130 max_load_reduction=56663 max_load_reduction_pct=0.864609 moved_to_echo=32167 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1877, 0, 0, 0, 0, 2, 4096, 1376, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 0, 0, 0, 4096, 0, 248, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 2657, 1416, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3857, 2432, 4, 0, 2873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 62, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 7351, 8056, 8169, 8190, 9166, 8188, 8221] avg=8192.0000 max=9166 max_rank=5 min=7351 min_rank=1 imbalance_ratio=1.118896 max_load_reduction=56370 max_load_reduction_pct=0.860138 moved_to_echo=33765 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 30848, 128, 128, 128, 32768, 128, 2048, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 30848, 19584, 128, 128, 15232, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32256, 128, 128, 640, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[256, 128, 128, 32768, 128, 128, 128, 128, 21248, 11392, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 21888, 128, 128, 0, 128, 128, 32768, 11008, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[2, 3, 0, 0, 0, 2, 2, 3, 0, 3, 2, 0, 2, 0, 0, 0, 0, 0, 4094, 0, 3, 1, 4080, 0, 0, 0, 0, 4094, 0, 2, 0, 22, 0, 2, 4096, 0, 4094, 4091, 0, 2, 0, 4094, 2, 0, 0, 1, 4094, 4093, 4093, 4086, 2, 0, 4092, 4094, 4093, 3, 0, 4094, 0, 4093, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[2, 2, 0, 2, 0, 2, 2, 0, 0, 3, 0, 1, 1, 0, 1, 0, 0, 0, 4094, 0, 2, 2, 4095, 1, 0, 0, 0, 4092, 1, 1, 0, 2, 0, 2, 4095, 2, 4094, 4094, 0, 1, 0, 4095, 2, 0, 0, 0, 4093, 4093, 4093, 4084, 2, 0, 4091, 4094, 4094, 0, 0, 4094, 10, 4095, 0, 0, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 4092, 0, 4096, 4096, 0, 0, 0, 4096, 0, 77, 0, 0, 4034, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 6, 4075, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4, 0, 0, 4035, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 6, 0, 4066, 0, 4094, 4096, 0, 0, 0, 4091, 0, 165, 0, 0, 4005, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 12, 4096, 38, 4039, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 100, 0, 0, 4077, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 15, 4008, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 4094, 0, 0, 0, 31, 0, 0, 4090, 0, 4096, 4096, 0, 0, 0, 4096, 0, 36, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 7, 4032, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 4078, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4067, 0, 4096, 4096, 0, 0, 0, 4096, 0, 143, 0, 0, 4091, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 29, 3968, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 1, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 4096, 0, 0, 0, 4096, 0, 49, 0, 0, 4090, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1, 4096, 15, 4044, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 0, 0, 0, 2, 2, 3, 4093, 4088, 2, 0, 0, 0, 0, 0, 0, 3, 2, 0, 2, 0, 0, 0, 4094, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 1, 4080, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 22, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 9, 4091, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 2, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4086, 2, 0, 4092, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4093, 0, 0, 0, 2, 33, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 8191, 8203, 8200, 8145, 8187, 8223] avg=8192.0000 max=8223 max_rank=7 min=8145 min_rank=5 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32755 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 0, 2, 0, 2, 2, 0, 4093, 4088, 2, 0, 0, 0, 0, 0, 0, 3, 0, 1, 1, 0, 1, 0, 4094, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 2, 4095, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 0, 2, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 2, 9, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4084, 2, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 10, 4095, 0, 0, 1, 1, 33, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8207, 8181, 8203, 8144, 8181, 8235] avg=8192.0000 max=8235 max_rank=7 min=8144 min_rank=5 imbalance_ratio=1.005249 max_load_reduction=57301 max_load_reduction_pct=0.874344 moved_to_echo=32763 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 77, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 4075, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8190, 8203, 8189, 8197, 8161, 8196, 8211] avg=8192.0000 max=8211 max_rank=7 min=8161 min_rank=5 imbalance_ratio=1.002319 max_load_reduction=57325 max_load_reduction_pct=0.874710 moved_to_echo=32765 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4078, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 143, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 29, 3968, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8197, 8187, 8184, 8172, 8284, 8196, 8127] avg=8192.0000 max=8284 max_rank=5 min=8127 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32779 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 0, 0, 4096, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4035, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 4084, 1, 0, 0, 0, 0, 0, 0, 6, 0, 4066, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 165, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 38, 4039, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8192, 8148, 8193, 8177, 8215, 8196, 8219] avg=8192.0000 max=8219 max_rank=7 min=8148 min_rank=2 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32807 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 100, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 4008, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8192, 8195, 8183, 8201, 8227, 8196, 8153] avg=8192.0000 max=8227 max_rank=5 min=8153 min_rank=7 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4096, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 31, 4086, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 36, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 4032, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8189, 8202, 8216, 8195, 8182, 8192, 8169] avg=8192.0000 max=8216 max_rank=3 min=8169 min_rank=7 imbalance_ratio=1.002930 max_load_reduction=57320 max_load_reduction_pct=0.874634 moved_to_echo=32738 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4088, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 49, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 15, 4044, 0, 0, 0, 0, 39, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8203, 8187, 8191, 8178, 8192, 8199] avg=8192.0000 max=8203 max_rank=2 min=8178 min_rank=5 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32760 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32384, 128, 640, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 32384, 0, 0, 128, 128, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 32768, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 32768, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4094, 4093, 4091, 2, 0, 0, 2, 3, 4094, 1, 4092, 2, 3, 2, 0, 0, 0, 1, 2, 4094, 0, 0, 3, 1, 0, 4094, 0, 0, 4091, 2, 0, 4093, 4093, 4093, 0, 0, 4093, 10, 0, 2, 6, 4093, 0, 3, 4077, 2, 1, 0, 3, 0, 2, 0, 4092, 0, 0, 3, 0, 2, 4094, 0, 2, 2, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4081, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 20, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 4092, 4096, 4096, 6, 0, 4096, 0, 7, 0, 17, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 4090, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4092, 4096, 4096, 1, 0, 4096, 0, 2, 0, 11, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4092, 4096, 4078, 4, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 26, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4094, 4094, 4094, 2, 0, 1, 2, 2, 4094, 0, 4094, 1, 2, 2, 0, 1, 0, 0, 2, 4094, 2, 0, 4, 1, 0, 4094, 0, 0, 4092, 1, 0, 4089, 4094, 4093, 0, 0, 4094, 0, 1, 1, 0, 4095, 0, 2, 4092, 2, 2, 0, 2, 0, 2, 0, 4094, 0, 0, 1, 0, 2, 4094, 0, 0, 1, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4096, 4096, 4041, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 19, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 46, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 4096, 4094, 2, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 1, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4094, 4096, 4087, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 13, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4094, 4096, 4096, 0, 0, 4096, 0, 1, 0, 5, 4096, 1, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 13, 4091, 2, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4092, 2, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4094, 0, 0, 3, 1, 4087, 3, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4093, 0, 0, 0, 10, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4083, 0, 3, 4077, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 4092, 0, 0, 3, 4077, 10, 2, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 2, 2, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8194, 8191, 8193, 8196, 8172, 8189, 8196] avg=8192.0000 max=8205 max_rank=0 min=8172 min_rank=5 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4086, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4080, 10, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8192, 8182, 8192, 8193, 8192, 8188, 8192] avg=8192.0000 max=8205 max_rank=0 min=8182 min_rank=2 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4081, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 20, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 6, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4086, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4080, 10, 2, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8185, 8186, 8203, 8199, 8195, 8186] avg=8192.0000 max=8203 max_rank=4 min=8185 min_rank=2 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32783 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 13, 4094, 2, 0, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 1, 2, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 2, 0, 4, 1, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4093, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 2, 4092, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 4094, 0, 0, 1, 4078, 10, 2, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 1, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8194, 8194, 8188, 8187, 8183, 8189, 8193] avg=8192.0000 max=8208 max_rank=0 min=8183 min_rank=5 imbalance_ratio=1.001953 max_load_reduction=57328 max_load_reduction_pct=0.874756 moved_to_echo=32752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4092, 13, 4078, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4086, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4080, 10, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8192, 8191, 8196, 8190, 8200, 8188, 8192] avg=8192.0000 max=8200 max_rank=5 min=8187 min_rank=0 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 13, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 19, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4086, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4080, 10, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8192, 8202, 8196, 8190, 8226, 8188, 8192] avg=8192.0000 max=8226 max_rank=5 min=8150 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 5, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 13, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4082, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4087, 14, 7, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8188, 8199, 8189, 8186, 8184, 8211, 8193] avg=8192.0000 max=8211 max_rank=6 min=8184 min_rank=5 imbalance_ratio=1.002319 max_load_reduction=57325 max_load_reduction_pct=0.874710 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 13, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4086, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4080, 10, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8192, 8192, 8196, 8191, 8180, 8188, 8192] avg=8192.0000 max=8205 max_rank=0 min=8180 min_rank=5 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 0, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32768, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32768, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 2, 0, 0, 4094, 0, 0, 2, 4092, 1, 0, 0, 877, 3, 2, 4092, 0, 3, 4090, 3, 4093, 4094, 0, 2, 3507, 1, 0, 2, 4093, 4094, 0, 2, 0, 0, 0, 156, 4094, 2, 0, 4093, 0, 4, 3375, 4093, 4, 1, 0, 274, 0, 2, 4093, 3, 4093, 3, 3, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 550, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 3952, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 389, 4093, 0, 0, 4096, 0, 0, 3247, 4096, 0, 0, 0, 59, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2, 0, 4087, 0, 0, 0, 4096, 0, 0, 6, 593, 0, 0, 4085, 0, 0, 4096, 0, 4096, 4093, 3, 0, 3960, 0, 0, 0, 4096, 4091, 0, 0, 0, 9, 0, 55, 4079, 0, 0, 4096, 0, 0, 3536, 4096, 0, 0, 0, 7, 0, 0, 4096, 0, 4096, 0, 0, 0, 16, 43, 0, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1419, 0, 0, 4096, 0, 0, 4095, 0, 4096, 4096, 0, 0, 3625, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 59, 4091, 0, 0, 4096, 0, 0, 3077, 4096, 0, 0, 0, 10, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 8, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 739, 0, 0, 4086, 0, 0, 4096, 0, 4096, 4096, 0, 0, 3330, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 119, 4065, 0, 0, 4096, 0, 0, 3405, 4096, 0, 0, 0, 600, 0, 0, 4096, 0, 4096, 0, 0, 0, 5, 35, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2, 0, 4094, 0, 0, 1, 0, 0, 4094, 0, 0, 2, 4093, 1, 0, 4, 1890, 2, 1, 4089, 0, 3, 4090, 2, 4094, 4094, 0, 1, 3003, 2, 0, 2, 4094, 4094, 0, 2, 0, 0, 0, 48, 4095, 1, 0, 4093, 0, 4, 2801, 4093, 2, 1, 1, 446, 0, 2, 4093, 2, 4094, 2, 3, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 2, 808, 0, 0, 4092, 0, 0, 4096, 0, 4096, 4096, 0, 0, 3473, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 758, 4094, 0, 0, 4096, 0, 0, 2738, 4096, 0, 0, 0, 417, 0, 0, 4096, 0, 4096, 0, 0, 0, 2, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 5, 909, 0, 0, 4090, 0, 0, 4096, 0, 4096, 4095, 0, 0, 3109, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 214, 4080, 0, 0, 4096, 0, 0, 3890, 4096, 0, 0, 0, 81, 0, 0, 4096, 0, 4096, 0, 0, 0, 2, 9, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 877, 3, 2, 4092, 0, 3, 3123, 3, 0, 0, 0, 0, 0, 0, 0, 0, 601, 4094, 0, 2, 3507, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 4094, 0, 2, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 846, 0, 4, 3375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 274, 0, 2, 4093, 3, 2663, 967, 223, 0, 0, 0, 0, 0, 4093, 3, 3, 0, 0, 0, 1, 0, 3492, 584, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8189, 8103, 8207, 8122, 8321, 8230, 8176] avg=8192.0000 max=8321 max_rank=5 min=8103 min_rank=2 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32849 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 0, 0, 4096, 0, 0, 3127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 4096, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 4096, 0, 0, 0, 0, 0, 389, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 847, 0, 0, 3247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 4096, 0, 2665, 969, 223, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3494, 584, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8190, 7773, 8650, 8358, 8187, 8012, 8174] avg=8192.0000 max=8650 max_rank=3 min=7773 min_rank=2 imbalance_ratio=1.055908 max_load_reduction=56886 max_load_reduction_pct=0.868011 moved_to_echo=32731 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 739, 0, 0, 4086, 0, 0, 3127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 4096, 0, 0, 3330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 4096, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 847, 0, 0, 3405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 600, 0, 0, 4096, 0, 2665, 969, 223, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 35, 0, 0, 3494, 584, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 7952, 8028, 8088, 8317, 8553, 8214] avg=8192.0000 max=8553 max_rank=6 min=7952 min_rank=2 imbalance_ratio=1.044067 max_load_reduction=56983 max_load_reduction_pct=0.869492 moved_to_echo=33172 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4094, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 4093, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1890, 2, 1, 4089, 0, 3, 3123, 2, 0, 0, 0, 0, 0, 0, 0, 0, 601, 4094, 0, 1, 3003, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 4094, 0, 2, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 846, 0, 4, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 446, 0, 2, 4093, 2, 2663, 967, 223, 0, 0, 0, 0, 0, 4094, 2, 3, 0, 0, 0, 1, 0, 3493, 584, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8194, 9110, 7703, 8015, 7747, 8400, 8177] avg=8192.0000 max=9110 max_rank=2 min=7703 min_rank=3 imbalance_ratio=1.112061 max_load_reduction=56426 max_load_reduction_pct=0.860992 moved_to_echo=32339 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 593, 0, 0, 4085, 0, 0, 3127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 4093, 3, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 4091, 0, 0, 0, 9, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 847, 0, 0, 3536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 2665, 969, 223, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 43, 0, 7, 3494, 584, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8189, 7805, 8658, 8028, 8462, 7960, 8240] avg=8192.0000 max=8658 max_rank=3 min=7805 min_rank=2 imbalance_ratio=1.056885 max_load_reduction=56878 max_load_reduction_pct=0.867889 moved_to_echo=32690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1419, 0, 0, 4096, 0, 0, 3127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 4096, 0, 0, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 4096, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 847, 0, 0, 3077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 2665, 968, 223, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 3494, 584, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8642, 8323, 8028, 8015, 7962, 8182] avg=8192.0000 max=8642 max_rank=2 min=7962 min_rank=6 imbalance_ratio=1.054932 max_load_reduction=56894 max_load_reduction_pct=0.868134 moved_to_echo=32187 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 808, 0, 0, 4092, 0, 0, 3127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 4096, 0, 0, 3473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 4095, 0, 0, 0, 0, 0, 758, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 847, 0, 0, 2738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 0, 4096, 0, 2665, 969, 223, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 3, 0, 0, 3494, 584, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8027, 8171, 8726, 7679, 8370, 8179] avg=8192.0000 max=8726 max_rank=4 min=7679 min_rank=5 imbalance_ratio=1.065186 max_load_reduction=56810 max_load_reduction_pct=0.866852 moved_to_echo=32954 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 909, 0, 0, 4090, 0, 0, 3125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 4095, 0, 0, 3109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3862, 4095, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 838, 0, 0, 3890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 4096, 0, 2671, 971, 230, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 9, 0, 0, 3500, 587, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8198, 8124, 7796, 8171, 8808, 8049, 8194] avg=8192.0000 max=8808 max_rank=5 min=7796 min_rank=3 imbalance_ratio=1.075195 max_load_reduction=56728 max_load_reduction_pct=0.865601 moved_to_echo=33222 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[30976, 32768, 0, 128, 0, 128, 0, 1920, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4864, 32768, 128, 128, 28032, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 6784, 0, 128, 26112, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 0, 0, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 128, 128, 28032, 4736, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7808, 128, 128, 32768, 0, 128, 25088, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 1920, 0, 128, 32768, 128, 21376, 7808, 1792, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4, 0, 5, 3, 0, 3, 4086, 4093, 0, 4093, 0, 4, 0, 3, 0, 3, 0, 4088, 3, 0, 0, 1, 0, 4071, 0, 1, 0, 3773, 3, 0, 3, 0, 2, 0, 0, 3, 3, 0, 4093, 0, 4093, 4092, 3942, 4093, 0, 0, 16, 3, 0, 4092, 5, 2, 0, 0, 19, 4093, 4091, 3, 3, 4092, 0, 0, 4093, 463] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 4095, 4096, 4096, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 24, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 1216] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4088, 0, 4090, 10, 0, 0, 0, 13, 0, 0, 4096, 0, 2, 0, 0, 0, 4064, 0, 0, 0, 2940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 29, 0, 4096, 4096, 0, 0, 4096, 0, 4, 4096, 1160] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 4090, 4096, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 334] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 35] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 4033, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 330] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3, 0, 3, 2, 0, 3, 4089, 4093, 1, 4094, 0, 1, 2, 3, 0, 3, 0, 4093, 2, 0, 0, 0, 0, 4044, 1, 0, 0, 3715, 2, 0, 2, 0, 1, 1, 0, 2, 2, 0, 4094, 0, 4094, 4094, 4041, 4093, 0, 0, 10, 2, 0, 4093, 3, 2, 0, 0, 24, 4093, 4093, 2, 2, 4093, 0, 0, 4093, 448] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 3809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 4095, 4096, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 8, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 288] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 5, 3, 0, 3, 4086, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4, 0, 3, 0, 3, 3563, 524, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 1, 0, 4071, 12, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3773, 3, 0, 3, 0, 4093, 493, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 3, 0, 4093, 0, 3600, 492, 0, 0, 0, 0, 0, 0, 4093, 37, 3942, 0, 0, 0, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 2, 0, 0, 19, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 3, 3559, 0, 0, 0, 463, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8190, 8184, 8366, 8193, 8091, 8199, 8119] avg=8192.0000 max=8366 max_rank=3 min=8091 min_rank=5 imbalance_ratio=1.021240 max_load_reduction=57170 max_load_reduction_pct=0.872345 moved_to_echo=32602 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 3566, 525, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2881, 0, 0, 0, 0, 4096, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3602, 492, 0, 0, 0, 0, 0, 0, 4076, 37, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 24, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3562, 0, 0, 0, 1216, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8188, 8205, 7471, 8190, 8212, 8204, 8874] avg=8192.0000 max=8874 max_rank=7 min=7471 min_rank=3 imbalance_ratio=1.083252 max_load_reduction=56662 max_load_reduction_pct=0.864594 moved_to_echo=33480 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 2, 0, 3, 4089, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 2, 3, 0, 3, 3565, 524, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 4044, 12, 9, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3715, 2, 0, 2, 0, 4093, 493, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2, 2, 0, 4094, 0, 3600, 492, 0, 0, 0, 0, 0, 0, 4094, 37, 4041, 0, 0, 0, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 2, 0, 0, 24, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 2, 3560, 0, 0, 0, 448, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8193, 8160, 8306, 8192, 8184, 8203, 8105] avg=8192.0000 max=8306 max_rank=3 min=8105 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32684 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3567, 525, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 0, 0, 0, 4096, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3603, 493, 0, 0, 0, 0, 0, 0, 4094, 36, 4090, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3562, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8188, 8201, 8367, 8192, 8223, 8181, 7992] avg=8192.0000 max=8367 max_rank=3 min=7992 min_rank=7 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32588 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 10, 0, 0, 0, 13, 0, 3567, 525, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4064, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2940, 0, 0, 0, 0, 4096, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3603, 493, 0, 0, 0, 0, 0, 0, 4081, 36, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 29, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3562, 0, 4, 0, 1160, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8205, 8183, 7529, 8192, 8212, 8209, 8822] avg=8192.0000 max=8822 max_rank=7 min=7529 min_rank=3 imbalance_ratio=1.076904 max_load_reduction=56714 max_load_reduction_pct=0.865387 moved_to_echo=33435 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3567, 525, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4069, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3603, 493, 0, 0, 0, 0, 0, 0, 4096, 36, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3562, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8188, 8186, 8677, 8192, 8228, 8180, 7693] avg=8192.0000 max=8677 max_rank=3 min=7693 min_rank=7 imbalance_ratio=1.059204 max_load_reduction=56859 max_load_reduction_pct=0.867599 moved_to_echo=32293 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 3567, 525, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3831, 0, 0, 0, 0, 4096, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3602, 493, 0, 0, 0, 0, 0, 0, 4095, 36, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3562, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8188, 8210, 8421, 8191, 8164, 8182, 7988] avg=8192.0000 max=8421 max_rank=3 min=7988 min_rank=7 imbalance_ratio=1.027954 max_load_reduction=57115 max_load_reduction_pct=0.871506 moved_to_echo=32525 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 3571, 527, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 17, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3809, 0, 0, 0, 0, 4096, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3602, 496, 0, 0, 0, 0, 0, 0, 4093, 29, 4095, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3559, 0, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8196, 8207, 8399, 8194, 8222, 8178, 7943] avg=8192.0000 max=8399 max_rank=3 min=7943 min_rank=7 imbalance_ratio=1.025269 max_load_reduction=57137 max_load_reduction_pct=0.871841 moved_to_echo=32537 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 0, 128, 0, 32640, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 28544, 0, 128, 0, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 384, 32512, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 28928, 128, 0, 128, 0, 32768, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 0, 32768, 0, 28928, 3968, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 28544, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4044, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 3433, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[2, 0, 0, 4093, 4050, 2, 0, 0, 2, 4094, 3, 2, 0, 12, 3, 0, 2731, 4093, 4094, 3, 1, 4092, 2, 4092, 0, 4093, 0, 4092, 2, 0, 12, 2, 2, 4094, 0, 0, 3, 0, 2, 1382, 0, 0, 0, 1, 2, 2, 0, 2, 0, 0, 3, 0, 4093, 2, 0, 1, 4093, 0, 4094, 2, 0, 2, 4092, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3, 0, 0, 4092, 4050, 3, 0, 0, 4, 4093, 3, 3, 1, 8, 3, 0, 3452, 4091, 4093, 4, 0, 4092, 0, 4093, 0, 4092, 0, 4092, 0, 0, 25, 3, 3, 4093, 0, 0, 0, 0, 3, 649, 0, 0, 1, 1, 3, 3, 0, 3, 0, 2, 4, 0, 4092, 2, 0, 2, 4093, 0, 4092, 3, 0, 5, 4091, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3944, 0, 0, 0, 0, 4096, 0, 0, 0, 37, 0, 0, 3564, 4096, 4095, 0, 0, 4096, 0, 4060, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4081, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 3270, 4096, 4096, 0, 0, 4096, 0, 4095, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3972, 0, 5, 3, 0, 4093, 0, 0, 0, 71, 0, 0, 3267, 4096, 4092, 0, 0, 4096, 0, 4069, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4073, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 3489, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4048, 0, 0, 0, 0, 4096, 0, 0, 0, 62, 0, 0, 2977, 4096, 4095, 0, 0, 4096, 0, 4082, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4092, 4050, 3, 0, 0, 58, 3, 0, 0, 0, 0, 0, 0, 4, 4093, 3, 3, 1, 8, 3, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 3452, 0, 833, 4, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4092, 0, 0, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 3, 649, 3202, 30, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 3, 0, 3, 4093, 4090, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4092, 2, 0, 2, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 5, 4091, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8175, 8382, 8209, 7980, 8194, 8195, 8192] avg=8192.0000 max=8382 max_rank=2 min=7980 min_rank=4 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32561 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3944, 0, 0, 0, 58, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 37, 0, 0, 4066, 1, 0, 0, 0, 0, 0, 0, 3564, 0, 833, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 684, 3204, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8200, 8457, 8189, 8014, 8192, 8191, 8192] avg=8192.0000 max=8457 max_rank=2 min=8014 min_rank=4 imbalance_ratio=1.032349 max_load_reduction=57079 max_load_reduction_pct=0.870956 moved_to_echo=32589 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4093, 4050, 2, 0, 0, 58, 3, 0, 0, 0, 0, 0, 0, 2, 4094, 3, 2, 0, 12, 3, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 2731, 0, 833, 3, 1, 0, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4092, 2, 0, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 3, 0, 2, 1382, 3203, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 2, 4093, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 2, 0, 1, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 2, 4092, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8178, 7662, 8198, 8716, 8192, 8192, 8190] avg=8192.0000 max=8716 max_rank=4 min=7662 min_rank=2 imbalance_ratio=1.063965 max_load_reduction=56820 max_load_reduction_pct=0.867004 moved_to_echo=33290 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4044, 0, 0, 0, 58, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 4064, 1, 0, 0, 0, 0, 0, 0, 3433, 0, 834, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 694, 3204, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8182, 8363, 8189, 8026, 8191, 8192, 8192] avg=8192.0000 max=8363 max_rank=2 min=8026 min_rank=4 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32601 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 3972, 0, 5, 3, 58, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 71, 0, 0, 4064, 1, 0, 0, 0, 0, 0, 0, 3267, 0, 833, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 909, 3201, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8229, 8169, 8189, 8237, 8192, 8191, 8192] avg=8192.0000 max=8237 max_rank=4 min=8137 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32812 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4073, 0, 0, 0, 58, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 3489, 0, 834, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 618, 3204, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8172, 8419, 8189, 7948, 8191, 8193, 8194] avg=8192.0000 max=8419 max_rank=2 min=7948 min_rank=4 imbalance_ratio=1.027710 max_load_reduction=57117 max_load_reduction_pct=0.871536 moved_to_echo=32526 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4081, 0, 0, 0, 58, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 4066, 1, 0, 0, 0, 0, 0, 0, 3270, 0, 834, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 830, 3204, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8175, 8199, 8189, 8160, 8192, 8191, 8192] avg=8192.0000 max=8238 max_rank=0 min=8160 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32735 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4048, 0, 0, 0, 60, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 62, 0, 0, 4066, 1, 0, 0, 0, 0, 0, 0, 2977, 0, 826, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1121, 3209, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8225, 7885, 8184, 8455, 8192, 8191, 8192] avg=8192.0000 max=8455 max_rank=4 min=7885 min_rank=2 imbalance_ratio=1.032104 max_load_reduction=57081 max_load_reduction_pct=0.870987 moved_to_echo=33030 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[26240, 0, 6784, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 0, 32768, 32384, 128, 128, 128, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 0, 128, 0, 128, 6912, 25728, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 256, 128, 0, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1651, 1, 4074, 0, 2, 0, 0, 7, 1, 0, 0, 4092, 1, 2, 3, 4090, 0, 3, 0, 3, 0, 0, 2, 2, 32, 0, 2, 2, 4093, 4094, 4093, 3, 0, 0, 0, 4093, 4089, 0, 2, 3, 4093, 0, 3, 0, 4094, 0, 2425, 2, 0, 3, 0, 12, 3, 35, 4093, 5, 4050, 2, 2, 4090, 4093, 0, 0, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1239, 0, 4083, 0, 3, 0, 0, 9, 0, 0, 0, 4091, 0, 3, 4, 4075, 0, 4, 0, 4, 0, 3, 2, 2, 18, 0, 2, 3, 4092, 4093, 4092, 4, 0, 0, 1, 4093, 4088, 0, 7, 3, 4093, 0, 8, 0, 4093, 0, 2695, 2, 0, 4, 0, 11, 3, 150, 4092, 1, 4093, 3, 0, 4091, 4087, 0, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[3451, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1079, 0, 0, 0, 1, 0, 0, 75, 4096, 0, 3593, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3107, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 542, 0, 0, 0, 24, 1, 0, 610, 4096, 0, 3993, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[2878, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1167, 0, 0, 0, 1, 0, 0, 157, 4096, 0, 3996, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2338, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1301, 0, 0, 0, 0, 0, 0, 592, 4096, 0, 4000, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[732, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3332, 0, 0, 0, 0, 12, 0, 168, 4096, 0, 3972, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[2683, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1203, 0, 0, 0, 0, 0, 0, 237, 4096, 0, 4072, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1239, 0, 4083, 0, 3, 0, 0, 9, 1717, 136, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3, 4, 4075, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 3, 2, 2, 4092, 4090, 0, 0, 0, 0, 0, 0, 18, 0, 2, 3, 4088, 4093, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 4088, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 8, 0, 2376, 0, 2695, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 11, 3, 150, 4092, 1, 3829, 2, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 126, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7189, 8175, 8197, 8208, 8192, 9174, 8092, 8309] avg=8192.0000 max=9174 max_rank=5 min=7189 min_rank=0 imbalance_ratio=1.119873 max_load_reduction=56362 max_load_reduction_pct=0.860016 moved_to_echo=33767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1651, 1, 4074, 0, 2, 0, 0, 7, 1717, 136, 2, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 1, 2, 3, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 2, 2, 4093, 4090, 0, 0, 0, 0, 0, 0, 32, 0, 2, 2, 4089, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4089, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 2377, 0, 2425, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 12, 3, 35, 4093, 5, 3828, 1, 0, 0, 0, 0, 0, 0, 4050, 2, 2, 126, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7590, 8191, 8193, 8222, 8187, 8900, 7980, 8273] avg=8192.0000 max=8900 max_rank=5 min=7590 min_rank=0 imbalance_ratio=1.086426 max_load_reduction=56636 max_load_reduction_pct=0.864197 moved_to_echo=33340 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3451, 0, 4090, 0, 0, 0, 0, 0, 1718, 136, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2378, 0, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 75, 4096, 0, 3833, 1, 0, 0, 0, 0, 0, 0, 3593, 0, 0, 127, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9397, 8194, 8191, 8187, 8192, 7553, 8006, 7816] avg=8192.0000 max=9397 max_rank=0 min=7553 min_rank=5 imbalance_ratio=1.147095 max_load_reduction=56139 max_load_reduction_pct=0.856613 moved_to_echo=31567 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3107, 0, 4019, 0, 0, 0, 0, 0, 1718, 136, 2, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2378, 0, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1, 0, 610, 4096, 0, 3833, 1, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 127, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8982, 8193, 8191, 8181, 8192, 7016, 8565, 8216] avg=8192.0000 max=8982 max_rank=0 min=7016 min_rank=5 imbalance_ratio=1.096436 max_load_reduction=56554 max_load_reduction_pct=0.862946 moved_to_echo=31989 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2878, 0, 4089, 0, 0, 0, 0, 0, 1718, 136, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2378, 0, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 157, 4096, 0, 3833, 1, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 127, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8823, 8194, 8191, 8188, 8192, 7641, 8088, 8219] avg=8192.0000 max=8823 max_rank=0 min=7641 min_rank=5 imbalance_ratio=1.077026 max_load_reduction=56713 max_load_reduction_pct=0.865372 moved_to_echo=32140 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2683, 0, 4093, 0, 0, 0, 0, 0, 1718, 136, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2378, 0, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 4096, 0, 3833, 1, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 127, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8632, 8194, 8191, 8188, 8192, 7677, 8167, 8295] avg=8192.0000 max=8632 max_rank=0 min=7677 min_rank=5 imbalance_ratio=1.053711 max_load_reduction=56904 max_load_reduction_pct=0.868286 moved_to_echo=32331 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[732, 0, 4072, 0, 0, 0, 0, 0, 1724, 140, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2372, 0, 3332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 168, 4096, 0, 3839, 1, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 117, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6672, 8201, 8191, 8174, 8197, 9800, 8116, 8185] avg=8192.0000 max=9800 max_rank=5 min=6672 min_rank=0 imbalance_ratio=1.196289 max_load_reduction=55736 max_load_reduction_pct=0.850464 moved_to_echo=34298 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2338, 0, 4057, 0, 0, 0, 0, 0, 1718, 136, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2378, 0, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 4096, 0, 3833, 1, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 127, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8251, 8194, 8191, 8188, 8192, 7775, 8522, 8223] avg=8192.0000 max=8522 max_rank=6 min=7775 min_rank=5 imbalance_ratio=1.040283 max_load_reduction=57014 max_load_reduction_pct=0.869965 moved_to_echo=32712 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 0, 19072, 0, 13824, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31872, 128, 128, 1024, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[18176, 128, 32640, 0, 128, 0, 0, 128, 13824, 1152, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 32768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 2048, 32768, 128, 30720, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[463, 4091, 7, 0, 5, 3, 0, 3, 4087, 3, 0, 3, 0, 4093, 4093, 16, 4092, 2, 4, 3, 5, 0, 3, 1, 0, 2, 4, 0, 3, 4089, 0, 32, 3, 2, 0, 4093, 3, 0, 0, 4091, 4093, 0, 3, 4087, 0, 3, 0, 1, 4090, 4, 0, 6, 2, 4090, 4093, 3583, 0, 1, 0, 4089, 4092, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[728, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3349, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[174, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 162, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3761, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[699, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3390, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[851, 4092, 5, 0, 3, 3, 0, 2, 4090, 1, 0, 3, 0, 4093, 4094, 32, 4093, 1, 3, 3, 3, 0, 2, 2, 1, 1, 3, 0, 2, 4089, 0, 5, 1, 2, 0, 4094, 2, 1, 0, 4092, 4093, 0, 2, 4093, 0, 2, 0, 3, 4092, 3, 0, 6, 1, 4091, 4094, 3203, 0, 0, 0, 4091, 4093, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[181, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 169, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3748, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[198, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3890, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[182, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 54, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3861, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[463, 4091, 7, 0, 5, 3, 0, 3, 3594, 53, 7, 0, 0, 0, 0, 0, 4087, 3, 0, 3, 0, 4037, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 4, 3, 5, 0, 3, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 3, 4089, 0, 32, 4086, 3, 0, 0, 0, 0, 0, 0, 3, 2, 0, 4093, 3, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 4087, 0, 3, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 496, 4, 0, 6, 2, 4090, 0, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 4092, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8146, 8200, 8219, 8192, 8188, 8181, 8184] avg=8192.0000 max=8226 max_rank=0 min=8146 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[174, 4096, 0, 0, 0, 0, 0, 0, 3599, 53, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4040, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 497, 0, 0, 0, 0, 4096, 0, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7930, 8298, 8190, 8187, 8192, 8191, 8354, 8194] avg=8192.0000 max=8354 max_rank=6 min=7930 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=32931 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[851, 4092, 5, 0, 3, 3, 0, 2, 3595, 53, 7, 0, 0, 0, 0, 0, 4090, 1, 0, 3, 0, 4037, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 3, 3, 3, 0, 2, 2, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 0, 2, 4089, 0, 5, 4087, 3, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4094, 2, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 4093, 0, 2, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 497, 3, 0, 6, 1, 4091, 0, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8611, 8163, 8198, 8191, 8192, 8194, 7801, 8186] avg=8192.0000 max=8611 max_rank=0 min=7801 min_rank=6 imbalance_ratio=1.051147 max_load_reduction=56925 max_load_reduction_pct=0.868607 moved_to_echo=32373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[728, 4096, 0, 0, 0, 0, 0, 0, 3599, 53, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4040, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4089, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 497, 0, 0, 0, 0, 4096, 0, 3349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8483, 8154, 8189, 8189, 8192, 8193, 7942, 8194] avg=8192.0000 max=8483 max_rank=0 min=7942 min_rank=6 imbalance_ratio=1.035522 max_load_reduction=57053 max_load_reduction_pct=0.870560 moved_to_echo=32521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[181, 4096, 0, 0, 0, 0, 0, 0, 3599, 53, 7, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4040, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 497, 0, 0, 0, 0, 4096, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7936, 8304, 8190, 8188, 8192, 8191, 8341, 8194] avg=8192.0000 max=8341 max_rank=6 min=7936 min_rank=0 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32918 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[182, 4096, 0, 0, 0, 0, 0, 0, 3599, 53, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4040, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 497, 0, 0, 0, 0, 4096, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7937, 8190, 8189, 8188, 8192, 8192, 8454, 8194] avg=8192.0000 max=8454 max_rank=6 min=7937 min_rank=0 imbalance_ratio=1.031982 max_load_reduction=57082 max_load_reduction_pct=0.871002 moved_to_echo=33032 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[699, 4096, 0, 0, 0, 0, 0, 0, 3599, 53, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4040, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 497, 0, 0, 0, 0, 4096, 0, 3390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8455, 8144, 8190, 8188, 8192, 8190, 7983, 8194] avg=8192.0000 max=8455 max_rank=0 min=7983 min_rank=6 imbalance_ratio=1.032104 max_load_reduction=57081 max_load_reduction_pct=0.870987 moved_to_echo=32559 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[198, 4096, 0, 0, 0, 0, 0, 0, 3602, 54, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4033, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 0, 0, 4096, 0, 3890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7958, 8137, 8190, 8186, 8192, 8197, 8480, 8196] avg=8192.0000 max=8480 max_rank=6 min=7958 min_rank=0 imbalance_ratio=1.035156 max_load_reduction=57056 max_load_reduction_pct=0.870605 moved_to_echo=33065 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 32768, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 32768, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 128, 0, 32384, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 32768, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4096, 128, 0, 128, 128, 32768, 0, 28800, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3584, 32768, 128, 0, 128, 128, 0, 128, 28800, 512, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4092, 6, 4, 3, 0, 5, 4094, 5, 4, 0, 4088, 0, 0, 3, 2, 0, 4087, 2, 3, 0, 37, 0, 4092, 0, 2, 0, 0, 2, 0, 1, 4093, 0, 3, 4087, 0, 4091, 0, 0, 3, 4, 4092, 0, 4093, 0, 4092, 0, 4091, 4092, 4092, 4056, 2, 1, 0, 3, 3, 0, 4, 4, 0, 4, 4, 0, 4090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 9, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4088, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4092, 5, 2, 2, 1, 4, 4094, 9, 1, 0, 4090, 0, 0, 2, 2, 0, 4091, 1, 2, 1, 34, 0, 4093, 0, 1, 0, 0, 2, 0, 0, 4093, 1, 3, 4085, 0, 4092, 0, 0, 3, 3, 4093, 0, 4094, 0, 4093, 0, 4092, 4092, 4093, 4060, 3, 1, 0, 2, 3, 0, 3, 3, 0, 2, 3, 0, 4090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 6, 4, 3, 0, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 4088, 0, 0, 3, 2, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 3, 0, 37, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 1, 4093, 4083, 5, 0, 0, 0, 0, 0, 0, 0, 3, 4087, 0, 4091, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 4093, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 9, 4056, 2, 1, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 4, 4, 0, 4090, 4089, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8193, 8215, 8186, 8185, 8191, 8166, 8196] avg=8192.0000 max=8215 max_rank=2 min=8166 min_rank=6 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32738 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8191, 8186, 8193, 8193, 8194, 8201, 8191] avg=8192.0000 max=8201 max_rank=6 min=8186 min_rank=2 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 5, 2, 2, 1, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 4090, 0, 0, 2, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 2, 1, 34, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4093, 4084, 5, 0, 0, 0, 0, 0, 0, 1, 3, 4085, 0, 4092, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4094, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 9, 4060, 3, 1, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 2, 3, 0, 4090, 4090, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8196, 8216, 8185, 8185, 8192, 8170, 8192] avg=8192.0000 max=8216 max_rank=2 min=8170 min_rank=6 imbalance_ratio=1.002930 max_load_reduction=57320 max_load_reduction_pct=0.874634 moved_to_echo=32739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8186, 8188, 8193, 8195, 8201, 8190] avg=8192.0000 max=8201 max_rank=6 min=8186 min_rank=2 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8, 0, 0, 0, 0, 4096, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8191, 8186, 8205, 8193, 8187, 8201, 8190] avg=8192.0000 max=8205 max_rank=3 min=8183 min_rank=0 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8186, 8188, 8193, 8195, 8201, 8190] avg=8192.0000 max=8201 max_rank=6 min=8186 min_rank=2 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4096, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8191, 8186, 8190, 8193, 8191, 8201, 8193] avg=8192.0000 max=8201 max_rank=6 min=8186 min_rank=2 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 4096, 4091, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8192, 8175, 8201, 8201, 8191, 8195, 8194] avg=8192.0000 max=8201 max_rank=3 min=8175 min_rank=2 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 32768, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 32768, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 756, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3520, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 5, 3, 0, 2, 1, 0, 256, 0, 3, 0, 0, 5, 4093, 3, 4092, 4085, 0, 0, 0, 0, 4089, 2, 4092, 4091, 1, 0, 4, 0, 4092, 4091, 3, 4086, 4094, 2, 4093, 0, 0, 0, 2, 0, 0, 0, 3896, 0, 2, 2, 2, 4092, 4, 2, 2, 4093, 3, 0, 2, 4093, 4, 0, 0, 4046, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3884, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[2, 0, 0, 6, 4, 0, 3, 1, 0, 321, 0, 4, 0, 0, 5, 4092, 4, 4091, 4073, 0, 0, 0, 2, 4088, 1, 4092, 4088, 2, 0, 5, 0, 4091, 4090, 4, 4084, 4093, 2, 4092, 0, 0, 0, 0, 0, 0, 9, 3939, 0, 6, 0, 5, 4090, 5, 0, 1, 4092, 4, 0, 3, 4092, 4, 1, 0, 3943, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3970, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 1, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 7, 0, 3828, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 4040, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 0, 4010, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 6, 4, 0, 3, 1, 4092, 4090, 0, 0, 0, 0, 0, 0, 0, 321, 0, 4, 0, 0, 5, 4092, 3752, 0, 0, 0, 0, 0, 0, 0, 4, 7, 4073, 0, 0, 0, 2, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4088, 2, 0, 5, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 4084, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3939, 0, 6, 4090, 146, 0, 0, 0, 0, 0, 0, 0, 5, 4090, 5, 0, 1, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 4, 1, 0, 3943, 2, 186, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8174, 8174, 8189, 8183, 8190, 8196, 8232] avg=8192.0000 max=8232 max_rank=7 min=8174 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32801 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 756, 0, 0, 0, 0, 0, 4096, 3757, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 0, 4093, 147, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3520, 0, 186, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8609, 8198, 8192, 8195, 8158, 8191, 7803] avg=8192.0000 max=8609 max_rank=1 min=7803 min_rank=7 imbalance_ratio=1.050903 max_load_reduction=56927 max_load_reduction_pct=0.868637 moved_to_echo=32347 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 5, 3, 0, 2, 1, 4093, 4090, 0, 0, 0, 0, 0, 0, 0, 256, 0, 3, 0, 0, 5, 4093, 3753, 0, 0, 0, 0, 0, 0, 0, 3, 6, 4085, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4091, 1, 0, 4, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 4086, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3896, 0, 2, 4091, 147, 0, 0, 0, 0, 0, 0, 2, 2, 4092, 4, 2, 2, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 4, 0, 0, 4046, 2, 186, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8110, 8183, 8192, 8185, 8138, 8199, 8334] avg=8192.0000 max=8334 max_rank=7 min=8110 min_rank=1 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32856 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 409, 0, 0, 0, 0, 0, 4096, 3757, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 4093, 147, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3884, 0, 186, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8262, 8198, 8194, 8195, 8139, 8191, 8167] avg=8192.0000 max=8262 max_rank=1 min=8139 min_rank=5 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 0, 0, 4096, 3757, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 4093, 147, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3970, 0, 186, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8274, 8198, 8193, 8195, 8042, 8191, 8253] avg=8192.0000 max=8274 max_rank=1 min=8042 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32681 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 4096, 3757, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 4091, 147, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 3828, 0, 186, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8175, 8199, 8192, 8195, 8276, 8191, 8118] avg=8192.0000 max=8276 max_rank=5 min=8118 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 4096, 3757, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 4093, 147, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4040, 0, 190, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7955, 8194, 8192, 8195, 8290, 8191, 8328] avg=8192.0000 max=8328 max_rank=7 min=7955 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=33004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 4096, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 4095, 149, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4010, 0, 186, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7977, 8192, 8192, 8193, 8303, 8186, 8301] avg=8192.0000 max=8303 max_rank=5 min=7977 min_rank=1 imbalance_ratio=1.013550 max_load_reduction=57233 max_load_reduction_pct=0.873306 moved_to_echo=32983 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 2816, 0, 128, 0, 0, 128, 32768, 30080, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 128, 31616, 0, 128, 32768, 1280, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 31360, 128, 1536, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 824, 0, 0, 0, 3975, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3393, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[3, 0, 4092, 2, 3, 4092, 5, 0, 0, 4091, 4, 0, 4089, 10, 0, 664, 4, 1, 2, 3702, 4, 3, 4091, 5, 5, 5, 4090, 0, 0, 0, 3, 11, 0, 0, 4090, 0, 0, 2, 3816, 1, 5, 2, 4092, 0, 4, 4074, 1, 5, 0, 4, 0, 0, 8, 0, 0, 0, 4, 0, 4092, 4091, 4084, 4091, 4088, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 4093, 1, 2, 4092, 4, 0, 0, 4092, 3, 0, 4091, 7, 0, 643, 4, 1, 2, 3707, 3, 3, 4092, 4, 3, 4, 4091, 0, 0, 0, 2, 19, 0, 0, 4092, 0, 0, 2, 3832, 0, 4, 2, 4093, 0, 3, 4074, 0, 3, 0, 3, 0, 0, 5, 1, 0, 0, 3, 0, 4093, 4092, 4086, 4093, 4090, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1046, 0, 0, 0, 3914, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3232, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 604, 0, 0, 0, 3700, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 3888, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1134, 0, 0, 0, 4015, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 19, 4096, 8, 0, 0, 3035, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4080, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2195, 0, 0, 0, 3738, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 2259, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4095, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1724, 0, 0, 0, 3696, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 2, 0, 0, 2772, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4095, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4092, 2, 3, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2989, 4, 0, 4089, 10, 0, 664, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 2, 3702, 4, 3, 4091, 5, 286, 0, 0, 0, 0, 0, 0, 0, 5, 5, 4090, 0, 0, 0, 3, 11, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 2, 3816, 1, 813, 0, 0, 0, 0, 0, 0, 0, 5, 2, 4092, 0, 4, 4074, 1, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8, 0, 0, 0, 4092, 4090, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4084, 4091, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 7756, 8098, 8198, 8722, 8186, 8194, 8185] avg=8192.0000 max=8722 max_rank=4 min=7756 min_rank=1 imbalance_ratio=1.064697 max_load_reduction=56814 max_load_reduction_pct=0.866913 moved_to_echo=33287 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2993, 0, 0, 4096, 0, 0, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 4096, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3393, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7913, 8357, 8187, 8303, 8195, 8191, 8198] avg=8192.0000 max=8357 max_rank=2 min=7913 min_rank=1 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=32887 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4093, 1, 2, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2990, 3, 0, 4091, 7, 0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 2, 3707, 3, 3, 4092, 4, 286, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4091, 0, 0, 0, 2, 19, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 2, 3832, 0, 813, 0, 0, 0, 0, 0, 0, 0, 4, 2, 4093, 0, 3, 4074, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5, 1, 0, 0, 4093, 4091, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4086, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7734, 8102, 8205, 8739, 8182, 8193, 8187] avg=8192.0000 max=8739 max_rank=4 min=7734 min_rank=1 imbalance_ratio=1.066772 max_load_reduction=56797 max_load_reduction_pct=0.866653 moved_to_echo=33301 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2993, 0, 0, 4096, 0, 0, 2195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3738, 0, 0, 4096, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 2259, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 9284, 8120, 8187, 7170, 8195, 8191, 8197] avg=8192.0000 max=9284 max_rank=1 min=7170 min_rank=4 imbalance_ratio=1.133301 max_load_reduction=56252 max_load_reduction_pct=0.858337 moved_to_echo=31753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2993, 0, 0, 4096, 0, 0, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 4096, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 8, 0, 0, 3035, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8223, 8397, 8187, 7972, 8188, 8196, 8182] avg=8192.0000 max=8397 max_rank=2 min=7972 min_rank=4 imbalance_ratio=1.025024 max_load_reduction=57139 max_load_reduction_pct=0.871872 moved_to_echo=32538 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2993, 0, 0, 4096, 0, 0, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 4096, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3232, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8135, 8296, 8187, 8142, 8195, 8191, 8198] avg=8192.0000 max=8296 max_rank=2 min=8135 min_rank=1 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32726 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2993, 0, 0, 4096, 0, 0, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 4096, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 3888, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7693, 8082, 8188, 8799, 8194, 8190, 8198] avg=8192.0000 max=8799 max_rank=4 min=7693 min_rank=1 imbalance_ratio=1.074097 max_load_reduction=56737 max_load_reduction_pct=0.865738 moved_to_echo=33381 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2978, 0, 0, 4096, 0, 0, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3696, 0, 0, 4096, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 3, 0, 0, 0, 0, 0, 0, 1, 4096, 2, 0, 0, 2772, 0, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8798, 8084, 8197, 7689, 8201, 8190, 8191] avg=8192.0000 max=8798 max_rank=1 min=7689 min_rank=4 imbalance_ratio=1.073975 max_load_reduction=56738 max_load_reduction_pct=0.865753 moved_to_echo=32271 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 0, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 23936, 128, 0, 32768, 128, 0, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 26240, 128, 6528, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 30464, 128, 128, 32768, 128, 2304, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 4091, 3, 4086, 54, 4091, 0, 3894, 0, 3, 0, 443, 4, 4089, 0, 0, 4091, 3844, 17, 0, 0, 0, 4092, 3, 0, 4, 4075, 0, 0, 4, 6, 4, 18, 5, 3, 4093, 12, 4082, 2, 4, 105, 4085, 3, 4, 6, 0, 0, 4092, 0, 0, 4090, 2, 4, 1, 7, 3568, 5, 408, 0, 0, 0, 4037, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 11, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 4096, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4057, 0, 11, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 67, 0, 0, 4096, 0, 4096, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3434, 0, 591, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 71, 0, 0, 4096, 0, 4096, 0, 0, 44, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3897, 0, 77, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 9, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 96, 0, 0, 4096, 0, 4096, 0, 0, 22, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3475, 0, 494, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 4092, 1, 4087, 51, 4092, 0, 3938, 0, 2, 0, 665, 4, 4089, 0, 0, 4092, 3843, 15, 0, 0, 0, 4093, 1, 0, 3, 4079, 0, 0, 3, 10, 3, 17, 3, 2, 4093, 8, 4082, 2, 3, 15, 4087, 3, 3, 4, 0, 0, 4093, 1, 0, 4092, 1, 3, 2, 5, 3391, 4, 417, 1, 0, 0, 4040, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 48, 0, 0, 4096, 0, 4096, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3997, 0, 42, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 45, 0, 0, 4096, 0, 4096, 0, 0, 89, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3948, 0, 13, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 35, 3, 4086, 54, 0, 0, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 443, 4, 4089, 0, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 4091, 3844, 17, 0, 0, 0, 61, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4075, 0, 0, 4, 6, 4, 4091, 0, 0, 0, 0, 0, 0, 0, 18, 5, 3, 4046, 12, 4082, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 105, 4085, 3, 4, 6, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 4, 1, 7, 3568, 288, 47, 36, 0, 0, 0, 0, 0, 5, 408, 0, 0, 0, 4037, 2, 0, 3743, 106, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8072, 8489, 8016, 8184, 8172, 8259, 8043, 8301] avg=8192.0000 max=8489 max_rank=1 min=8016 min_rank=2 imbalance_ratio=1.036255 max_load_reduction=57047 max_load_reduction_pct=0.870468 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 35, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4049, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4057, 289, 47, 36, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 3746, 106, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8061, 8253, 8193, 8156, 8162, 8525, 7959] avg=8192.0000 max=8525 max_rank=6 min=7959 min_rank=7 imbalance_ratio=1.040649 max_load_reduction=57011 max_load_reduction_pct=0.869919 moved_to_echo=32802 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 35, 1, 4087, 51, 0, 0, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 665, 4, 4089, 0, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 4092, 3843, 15, 0, 0, 0, 62, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4079, 0, 0, 3, 10, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 17, 3, 2, 4046, 8, 4082, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4087, 3, 3, 4, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 1, 3, 2, 5, 3391, 288, 47, 36, 0, 0, 0, 0, 0, 4, 417, 1, 0, 0, 4040, 1, 0, 3743, 106, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8112, 8711, 8013, 8190, 8163, 8169, 7866, 8312] avg=8192.0000 max=8711 max_rank=1 min=7866 min_rank=6 imbalance_ratio=1.063354 max_load_reduction=56825 max_load_reduction_pct=0.867081 moved_to_echo=32510 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 35, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 67, 0, 0, 4049, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3434, 289, 47, 36, 0, 0, 0, 0, 0, 0, 591, 0, 0, 0, 4096, 0, 0, 3746, 106, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8050, 8250, 8193, 8212, 8163, 7902, 8539] avg=8192.0000 max=8539 max_rank=7 min=7902 min_rank=6 imbalance_ratio=1.042358 max_load_reduction=56997 max_load_reduction_pct=0.869705 moved_to_echo=32816 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 35, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 71, 0, 0, 4049, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 4096, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3897, 289, 47, 36, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 4096, 0, 0, 3746, 106, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8055, 8253, 8195, 8216, 8200, 8365, 8025] avg=8192.0000 max=8365 max_rank=6 min=8025 min_rank=7 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32806 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 35, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 96, 0, 0, 4049, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3475, 289, 47, 36, 0, 0, 0, 0, 0, 0, 494, 0, 0, 0, 4096, 0, 0, 3746, 106, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8059, 8253, 8193, 8241, 8178, 7943, 8442] avg=8192.0000 max=8442 max_rank=7 min=7943 min_rank=6 imbalance_ratio=1.030518 max_load_reduction=57094 max_load_reduction_pct=0.871185 moved_to_echo=32804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 25, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4042, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4096, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3948, 292, 54, 37, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 3751, 108, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8057, 8245, 8195, 8183, 8244, 8427, 7968] avg=8192.0000 max=8427 max_rank=6 min=7968 min_rank=7 imbalance_ratio=1.028687 max_load_reduction=57109 max_load_reduction_pct=0.871414 moved_to_echo=32822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 35, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 48, 0, 0, 4049, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3997, 289, 47, 36, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 4096, 0, 0, 3746, 106, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8054, 8253, 8193, 8193, 8161, 8465, 7990] avg=8192.0000 max=8465 max_rank=6 min=7990 min_rank=7 imbalance_ratio=1.033325 max_load_reduction=57071 max_load_reduction_pct=0.870834 moved_to_echo=32809 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 32768, 128, 128, 128, 0, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 384, 128, 32768, 128, 0, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[384, 128, 128, 32384, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 32768, 128, 128, 128, 128, 29824, 2432, 384, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 0, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 2176, 128, 0, 0, 32768, 128, 0, 30080, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 32384, 128, 0, 0, 0, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 1152, 128, 32768, 0, 0, 31744, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[3983, 0, 8, 0, 0, 0, 4081, 4088, 4, 5, 5, 4, 2, 3, 0, 18, 4076, 0, 0, 3, 0, 91, 4087, 0, 2, 0, 4092, 5, 4090, 0, 4091, 0, 4092, 7, 0, 4092, 0, 4, 11, 0, 1, 39, 2, 0, 4091, 6, 0, 0, 4, 4092, 4091, 4077, 3, 0, 4079, 0, 4, 0, 6, 4, 2, 4087, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4022, 0, 2, 0, 0, 0, 4080, 4090, 3, 4, 3, 3, 1, 2, 0, 12, 4088, 0, 0, 2, 0, 66, 4091, 0, 1, 0, 4093, 4, 4092, 0, 4092, 0, 4093, 5, 0, 4093, 1, 3, 9, 0, 2, 25, 0, 0, 4092, 4, 0, 0, 3, 4093, 4092, 4076, 3, 0, 4084, 1, 3, 1, 4, 2, 1, 4090, 2, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4081, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 4096, 0, 4096, 0, 4096, 1, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4081, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4096, 2, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3983, 0, 8, 0, 0, 0, 4081, 32, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 5, 4, 2, 3, 0, 18, 4092, 4085, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 3, 0, 91, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 5, 4090, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 7, 0, 4092, 0, 4, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 39, 2, 0, 4091, 6, 0, 0, 4056, 19, 3, 1, 0, 0, 0, 0, 4, 0, 7, 4077, 3, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 4, 2, 4087, 0, 4, 4083, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8218, 8238, 8189, 8203, 8218, 8170, 8196] avg=8192.0000 max=8238 max_rank=2 min=8104 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=57298 max_load_reduction_pct=0.874298 moved_to_echo=32787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4022, 0, 2, 0, 0, 0, 4080, 32, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 3, 3, 1, 2, 0, 12, 4093, 4086, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 2, 0, 66, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4, 4092, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 5, 0, 4093, 1, 3, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 25, 0, 0, 4092, 4, 0, 0, 4058, 19, 3, 1, 0, 0, 0, 0, 3, 0, 7, 4076, 3, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4, 2, 1, 4090, 2, 3, 4084, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8207, 8228, 8190, 8201, 8204, 8174, 8196] avg=8192.0000 max=8228 max_rank=2 min=8136 min_rank=0 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4081, 0, 0, 0, 0, 0, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4077, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4064, 19, 3, 1, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8185, 8188, 8193, 8189, 8183, 8200, 8189] avg=8192.0000 max=8209 max_rank=0 min=8183 min_rank=5 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4081, 0, 0, 0, 0, 0, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4077, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4064, 19, 3, 1, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8185, 8186, 8194, 8190, 8183, 8200, 8189] avg=8192.0000 max=8209 max_rank=0 min=8183 min_rank=5 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4064, 19, 3, 1, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8185, 8173, 8193, 8189, 8183, 8200, 8189] avg=8192.0000 max=8224 max_rank=0 min=8173 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4064, 19, 3, 1, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4087, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8185, 8173, 8193, 8189, 8183, 8200, 8191] avg=8192.0000 max=8222 max_rank=0 min=8173 min_rank=2 imbalance_ratio=1.003662 max_load_reduction=57314 max_load_reduction_pct=0.874542 moved_to_echo=32763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 0, 0, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4077, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4064, 19, 3, 1, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8185, 8178, 8192, 8189, 8183, 8200, 8190] avg=8192.0000 max=8219 max_rank=0 min=8178 min_rank=2 imbalance_ratio=1.003296 max_load_reduction=57317 max_load_reduction_pct=0.874588 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4073, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 23, 6, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4093, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8186, 8172, 8192, 8186, 8199, 8192, 8196] avg=8192.0000 max=8213 max_rank=0 min=8172 min_rank=2 imbalance_ratio=1.002563 max_load_reduction=57323 max_load_reduction_pct=0.874680 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 0, 256, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 0, 0, 32512, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32640, 0, 128, 0, 0, 0, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 3, 2, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[5, 3, 3, 0, 2, 17, 0, 0, 28, 2, 4, 0, 5, 4, 0, 4090, 4090, 0, 4092, 4089, 3, 4091, 0, 5, 4092, 4, 4091, 0, 0, 3713, 4065, 4092, 0, 3, 0, 380, 0, 4092, 0, 4079, 5, 0, 4, 0, 4084, 3, 2, 1, 5, 0, 0, 0, 0, 5, 4091, 0, 0, 4088, 4, 4091, 0, 0, 6, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4, 3, 2, 0, 1, 18, 0, 0, 15, 2, 7, 0, 4, 3, 0, 4090, 4092, 0, 4093, 4090, 2, 4092, 0, 4, 4093, 3, 4093, 0, 0, 3799, 4075, 4092, 0, 3, 0, 294, 0, 4093, 0, 4079, 3, 0, 3, 0, 4086, 1, 1, 1, 4, 1, 0, 0, 0, 4, 4092, 0, 0, 4091, 3, 4092, 0, 0, 5, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 11, 4095, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 3, 3, 0, 2, 17, 0, 0, 4092, 4086, 0, 0, 0, 0, 0, 0, 28, 2, 4, 0, 5, 4, 0, 4090, 4000, 81, 1, 0, 0, 0, 0, 0, 4090, 0, 0, 4089, 3, 4, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 91, 0, 0, 3713, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 380, 0, 4011, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 0, 4084, 3, 2, 1, 4087, 6, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 5, 4091, 0, 4086, 5, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 4091, 0, 0, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8215, 8191, 7873, 8473, 8192, 8192, 8192] avg=8192.0000 max=8473 max_rank=4 min=7873 min_rank=3 imbalance_ratio=1.034302 max_load_reduction=57063 max_load_reduction_pct=0.870712 moved_to_echo=33049 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4005, 82, 1, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 4014, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8184, 8191, 8283, 8115, 8192, 8192, 8192] avg=8192.0000 max=8283 max_rank=3 min=8115 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32691 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4, 3, 2, 0, 1, 18, 0, 0, 4093, 4087, 0, 0, 0, 0, 0, 0, 15, 2, 7, 0, 4, 3, 0, 4090, 4002, 81, 1, 0, 0, 0, 0, 0, 4092, 0, 0, 4090, 2, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 91, 0, 0, 3799, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 294, 0, 4012, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 4086, 1, 1, 1, 4087, 6, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 4, 4092, 0, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 4092, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8205, 8192, 7968, 8388, 8188, 8193, 8194] avg=8192.0000 max=8388 max_rank=4 min=7968 min_rank=3 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32963 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4005, 82, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8184, 8196, 8283, 8110, 8193, 8192, 8192] avg=8192.0000 max=8283 max_rank=3 min=8110 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32687 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4005, 82, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4014, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8184, 8194, 8283, 8112, 8193, 8192, 8192] avg=8192.0000 max=8283 max_rank=3 min=8112 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32689 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4005, 82, 1, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4013, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8184, 8186, 8283, 8119, 8192, 8192, 8193] avg=8192.0000 max=8283 max_rank=3 min=8119 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32696 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4005, 82, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8184, 8196, 8283, 8110, 8193, 8191, 8192] avg=8192.0000 max=8283 max_rank=3 min=8110 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32686 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4008, 84, 4, 4, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4012, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8196, 8190, 8280, 8109, 8193, 8192, 8189] avg=8192.0000 max=8280 max_rank=3 min=8109 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=57256 max_load_reduction_pct=0.873657 moved_to_echo=32683 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 768, 0, 0, 32128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 768, 128, 32128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 32768, 32128, 768, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True + [2026-08-30 04:59:30] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 731.4 | throughput per GPU (TFLOP/s/GPU): 455.8 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 1.003443E+01 | loss scale: 1.0 | grad norm: 6.951 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[507, 0, 36, 3, 48, 3, 3152, 4, 783, 2028, 36, 252, 836, 1788, 0, 689, 1045, 708, 41, 2200, 0, 859, 451, 2279, 1, 4, 1200, 1472, 3276, 16, 2723, 434, 660, 215, 2235, 211, 396, 18, 3062, 1018, 3741, 0, 3623, 2024, 0, 206, 1755, 705, 3563, 452, 3169, 5, 0, 30, 0, 405, 0, 129, 521, 2226, 3699, 1289, 1388, 1917] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[949, 2, 53, 21, 49, 1, 3253, 1, 450, 2236, 19, 144, 800, 2666, 0, 403, 745, 305, 56, 2324, 0, 821, 509, 2339, 0, 3, 915, 1779, 3362, 35, 2792, 566, 695, 211, 1988, 194, 369, 11, 3284, 1019, 3757, 0, 3667, 2614, 0, 287, 1378, 1289, 3278, 399, 3066, 0, 0, 79, 0, 415, 1, 72, 439, 2137, 3489, 713, 1735, 1352] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[578, 3, 22, 26, 48, 0, 3002, 7, 431, 2173, 16, 133, 538, 2247, 0, 634, 493, 767, 42, 2771, 0, 683, 370, 2261, 0, 2, 863, 1871, 3403, 23, 2972, 646, 712, 219, 2355, 259, 404, 29, 3007, 1274, 3709, 0, 3626, 2431, 0, 137, 1574, 1200, 3591, 690, 3202, 0, 0, 143, 0, 622, 1, 60, 254, 1985, 3563, 1019, 1243, 1202] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[589, 4, 173, 9, 188, 0, 2793, 1, 416, 1671, 11, 207, 1048, 1804, 0, 648, 1248, 634, 62, 2198, 0, 687, 469, 2397, 2, 1, 735, 1807, 3389, 10, 2647, 765, 1336, 369, 2452, 262, 532, 17, 3246, 659, 3695, 0, 3190, 1937, 0, 228, 1713, 1330, 3505, 118, 2185, 1, 0, 13, 63, 580, 1, 150, 900, 1737, 3738, 1400, 1865, 1701] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[454, 4, 27, 11, 280, 0, 3078, 2, 643, 1136, 28, 121, 970, 1075, 0, 688, 1143, 337, 54, 2391, 0, 717, 587, 2183, 14, 5, 1353, 1497, 3153, 10, 2400, 745, 1863, 162, 2330, 296, 403, 27, 3125, 1017, 3594, 0, 3439, 2022, 0, 327, 2152, 1156, 3246, 239, 2848, 1, 1, 10, 1, 552, 0, 124, 936, 1987, 3362, 1239, 1848, 2123] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[667, 0, 79, 28, 106, 0, 2387, 7, 496, 1655, 70, 192, 799, 2892, 0, 591, 1343, 515, 37, 2630, 0, 972, 241, 2019, 11, 12, 661, 1793, 3333, 10, 2958, 617, 1114, 133, 2380, 237, 834, 39, 3254, 656, 3669, 0, 3153, 2540, 0, 152, 1851, 954, 3615, 300, 2743, 0, 4, 18, 0, 419, 0, 75, 425, 1585, 3522, 1813, 1595, 1335] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[454, 0, 76, 2, 49, 0, 3129, 2, 380, 1880, 20, 205, 800, 1937, 0, 403, 791, 495, 62, 2680, 0, 678, 330, 2182, 0, 4, 1304, 1631, 3591, 10, 2941, 775, 1235, 252, 2002, 259, 891, 14, 3005, 1042, 3720, 0, 3462, 2768, 0, 193, 2128, 867, 3358, 384, 2542, 1, 0, 7, 4, 729, 4, 58, 347, 1614, 3591, 1165, 1758, 1325] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[637, 7, 32, 11, 68, 0, 3037, 0, 256, 1839, 4, 207, 706, 2606, 0, 671, 1068, 445, 40, 1985, 0, 852, 322, 2226, 1, 3, 1171, 1692, 3278, 50, 2510, 672, 738, 145, 2142, 210, 430, 24, 3312, 1240, 3656, 0, 3693, 2211, 0, 321, 1332, 907, 3214, 439, 3018, 3, 1, 49, 1, 745, 0, 65, 482, 1894, 3579, 1611, 1808, 1870] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 0, 36, 3, 48, 3, 3152, 4, 3741, 756, 0, 0, 0, 0, 0, 0, 783, 2028, 36, 252, 836, 1788, 0, 689, 1774, 423, 0, 0, 0, 0, 0, 0, 1045, 708, 41, 2200, 0, 859, 451, 2279, 579, 241, 0, 0, 0, 0, 0, 0, 1, 4, 1200, 1472, 2007, 16, 2723, 434, 0, 0, 0, 0, 0, 0, 0, 0, 660, 215, 2235, 211, 396, 18, 2821, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2973, 2024, 0, 206, 1755, 705, 0, 0, 0, 0, 0, 0, 0, 0, 3563, 452, 3169, 5, 0, 30, 0, 405, 846, 71, 0, 0, 0, 0, 0, 0, 0, 129, 521, 2226, 1169, 1289, 1388, 1917, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 8609, 8403, 7857, 7574, 7663, 8541, 8639] avg=8192.0000 max=8639 max_rank=7 min=7574 min_rank=4 imbalance_ratio=1.054565 max_load_reduction=56897 max_load_reduction_pct=0.868179 moved_to_echo=32417 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[949, 2, 53, 21, 49, 1, 3253, 1, 3757, 713, 0, 0, 0, 0, 0, 0, 450, 2236, 19, 144, 800, 2666, 0, 403, 1673, 434, 0, 0, 0, 0, 0, 0, 745, 305, 56, 2324, 0, 821, 509, 2339, 586, 258, 0, 0, 0, 0, 0, 0, 0, 3, 915, 1779, 2060, 35, 2792, 566, 0, 0, 0, 0, 0, 0, 0, 0, 695, 211, 1988, 194, 369, 11, 3026, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3009, 2614, 0, 287, 1378, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 3278, 399, 3066, 0, 0, 79, 0, 415, 868, 72, 0, 0, 0, 0, 0, 0, 1, 72, 439, 2137, 1103, 713, 1735, 1352, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8799, 8825, 7943, 8150, 7513, 8577, 8177, 7552] avg=8192.0000 max=8825 max_rank=1 min=7513 min_rank=4 imbalance_ratio=1.077271 max_load_reduction=56711 max_load_reduction_pct=0.865341 moved_to_echo=31819 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[589, 4, 173, 9, 188, 0, 2793, 1, 3695, 764, 0, 0, 0, 0, 0, 0, 416, 1671, 11, 207, 1048, 1804, 0, 648, 1793, 438, 0, 0, 0, 0, 0, 0, 1248, 634, 62, 2198, 0, 687, 469, 2397, 510, 255, 0, 0, 0, 0, 0, 0, 2, 1, 735, 1807, 2076, 10, 2647, 765, 0, 0, 0, 0, 0, 0, 0, 0, 1336, 369, 2452, 262, 532, 17, 2991, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 1937, 0, 228, 1713, 1330, 0, 0, 0, 0, 0, 0, 0, 0, 3505, 118, 2185, 1, 0, 13, 63, 580, 875, 63, 0, 0, 0, 0, 0, 0, 1, 150, 900, 1737, 1181, 1400, 1865, 1701, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8036, 8460, 8043, 8618, 7825, 7403, 8935] avg=8192.0000 max=8935 max_rank=7 min=7403 min_rank=6 imbalance_ratio=1.090698 max_load_reduction=56601 max_load_reduction_pct=0.863663 moved_to_echo=32781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[578, 3, 22, 26, 48, 0, 3002, 7, 3709, 728, 0, 0, 0, 0, 0, 0, 431, 2173, 16, 133, 538, 2247, 0, 634, 1709, 440, 0, 0, 0, 0, 0, 0, 493, 767, 42, 2771, 0, 683, 370, 2261, 579, 236, 0, 0, 0, 0, 0, 0, 0, 2, 863, 1871, 2085, 23, 2972, 646, 0, 0, 0, 0, 0, 0, 0, 0, 712, 219, 2355, 259, 404, 29, 2771, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2976, 2431, 0, 137, 1574, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 690, 3202, 0, 0, 143, 0, 622, 878, 71, 0, 0, 0, 0, 0, 0, 1, 60, 254, 1985, 1126, 1019, 1243, 1202, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8123, 8321, 8202, 8462, 8023, 8318, 9197, 6890] avg=8192.0000 max=9197 max_rank=6 min=6890 min_rank=7 imbalance_ratio=1.122681 max_load_reduction=56339 max_load_reduction_pct=0.859665 moved_to_echo=32428 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[454, 4, 27, 11, 280, 0, 3078, 2, 3594, 687, 0, 0, 0, 0, 0, 0, 643, 1136, 28, 121, 970, 1075, 0, 688, 1612, 407, 0, 0, 0, 0, 0, 0, 1143, 337, 54, 2391, 0, 717, 587, 2183, 549, 246, 0, 0, 0, 0, 0, 0, 14, 5, 1353, 1497, 1932, 10, 2400, 745, 0, 0, 0, 0, 0, 0, 0, 0, 1863, 162, 2330, 296, 403, 27, 2879, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2822, 2022, 0, 327, 2152, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 3246, 239, 2848, 1, 1, 10, 1, 552, 814, 68, 0, 0, 0, 0, 0, 0, 0, 124, 936, 1987, 1063, 1239, 1848, 2123, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 6680, 8207, 7956, 8977, 8479, 7780, 9320] avg=8192.0000 max=9320 max_rank=7 min=6680 min_rank=1 imbalance_ratio=1.137695 max_load_reduction=56216 max_load_reduction_pct=0.857788 moved_to_echo=34556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[454, 0, 76, 2, 49, 0, 3129, 2, 3720, 734, 0, 0, 0, 0, 0, 0, 380, 1880, 20, 205, 800, 1937, 0, 403, 1722, 464, 0, 0, 0, 0, 0, 0, 791, 495, 62, 2680, 0, 678, 330, 2182, 553, 236, 0, 0, 0, 0, 0, 0, 0, 4, 1304, 1631, 2200, 10, 2941, 775, 0, 0, 0, 0, 0, 0, 0, 0, 1235, 252, 2002, 259, 891, 14, 2769, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2841, 2768, 0, 193, 2128, 867, 0, 0, 0, 0, 0, 0, 0, 0, 3358, 384, 2542, 1, 0, 7, 4, 729, 927, 68, 0, 0, 0, 0, 0, 0, 4, 58, 347, 1614, 1135, 1165, 1758, 1325, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 7811, 8007, 8865, 8464, 8797, 8020, 7406] avg=8192.0000 max=8865 max_rank=3 min=7406 min_rank=7 imbalance_ratio=1.082153 max_load_reduction=56671 max_load_reduction_pct=0.864731 moved_to_echo=32687 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[667, 0, 79, 28, 106, 0, 2387, 7, 3669, 720, 0, 0, 0, 0, 0, 0, 496, 1655, 70, 192, 799, 2892, 0, 591, 1689, 431, 0, 0, 0, 0, 0, 0, 1343, 515, 37, 2630, 0, 972, 241, 2019, 504, 256, 0, 0, 0, 0, 0, 0, 11, 12, 661, 1793, 2042, 10, 2958, 617, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 133, 2380, 237, 834, 39, 2998, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2587, 2540, 0, 152, 1851, 954, 0, 0, 0, 0, 0, 0, 0, 0, 3615, 300, 2743, 0, 4, 18, 0, 419, 860, 62, 0, 0, 0, 0, 0, 0, 0, 75, 425, 1585, 1113, 1813, 1595, 1335, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7663, 8815, 8517, 8104, 8391, 8084, 8021, 7941] avg=8192.0000 max=8815 max_rank=1 min=7663 min_rank=0 imbalance_ratio=1.076050 max_load_reduction=56721 max_load_reduction_pct=0.865494 moved_to_echo=32437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[637, 7, 32, 11, 68, 0, 3037, 0, 3656, 734, 0, 0, 0, 0, 0, 0, 256, 1839, 4, 207, 706, 2606, 0, 671, 1722, 428, 0, 0, 0, 0, 0, 0, 1068, 445, 40, 1985, 0, 852, 322, 2226, 594, 265, 0, 0, 0, 0, 0, 0, 1, 3, 1171, 1692, 2000, 50, 2510, 672, 0, 0, 0, 0, 0, 0, 0, 0, 738, 145, 2142, 210, 430, 24, 3047, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3022, 2211, 0, 321, 1332, 907, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 439, 3018, 3, 1, 49, 1, 745, 850, 77, 0, 0, 0, 0, 0, 0, 0, 65, 482, 1894, 1123, 1611, 1808, 1870, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8439, 7797, 8099, 7976, 7793, 8397, 8853] avg=8192.0000 max=8853 max_rank=7 min=7793 min_rank=5 imbalance_ratio=1.080688 max_load_reduction=56683 max_load_reduction_pct=0.864914 moved_to_echo=33019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 22912, 18560, 0, 1920, 13952, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3968, 14720, 256, 1536, 6528, 17024, 0, 4736, 13696, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 8320, 13568, 16512, 256, 22016, 5248, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 768, 4352, 15232, 9088, 10368, 13312, 12928, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8448, 1792, 17920, 2048, 4352, 256, 23424, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[7936, 4224, 512, 19200, 0, 6272, 3328, 17920, 4480, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[27392, 3072, 22784, 128, 128, 384, 128, 4480, 7040, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4864, 128, 512, 128, 896, 128, 23936, 128, 29568, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 10, 0, 0, 4096, 0, 3245, 0, 0, 0, 44, 7, 0, 0, 1, 0, 0, 0, 4, 1, 0, 56, 4087, 4096, 0, 0, 4096, 0, 0, 1, 0, 4096, 2667, 0, 2502, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 4095, 4090, 3798, 0, 0, 0, 0, 4068] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 3, 4, 0, 1, 4096, 0, 4042, 1, 0, 0, 52, 9, 0, 0, 0, 0, 0, 0, 14, 0, 0, 10, 4096, 4096, 7, 0, 4092, 0, 0, 4, 0, 4096, 2768, 1, 1437, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4091, 0, 4096, 4095, 3944, 4, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 6, 0, 1, 0, 4096, 0, 3816, 0, 0, 0, 30, 1, 0, 0, 0, 0, 0, 0, 1, 0, 2, 102, 4096, 4096, 0, 0, 4095, 0, 0, 10, 0, 4087, 1583, 0, 2724, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 4096, 0, 4096, 4096, 4039, 1, 0, 0, 0, 4077] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 2939, 0, 0, 0, 2, 10, 0, 0, 0, 0, 0, 0, 13, 0, 197, 17, 4096, 4096, 0, 0, 4095, 0, 0, 1, 0, 4095, 2134, 4, 2963, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 4096, 4096, 4020, 1, 0, 0, 0, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 0, 1, 4093, 0, 3599, 3, 0, 1, 27, 108, 0, 0, 0, 0, 0, 3, 7, 2, 1, 178, 4096, 4091, 0, 0, 4095, 0, 0, 22, 1, 4093, 1977, 0, 2640, 4096, 0, 0, 0, 0, 1, 0, 4095, 4, 0, 1, 0, 4095, 0, 4096, 0, 4092, 4095, 3740, 0, 0, 1, 0, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[3, 1, 0, 0, 4093, 2, 0, 1, 0, 4096, 0, 3037, 1, 0, 0, 41, 7, 0, 0, 1, 1, 0, 2, 3, 4, 4, 273, 4093, 4094, 0, 0, 4094, 0, 3, 4, 0, 4088, 2406, 25, 2677, 4094, 0, 0, 0, 1, 1, 3, 4096, 0, 0, 0, 0, 4093, 0, 4096, 1, 4094, 4092, 3842, 2, 1, 1, 0, 4065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 3124, 0, 0, 0, 72, 7, 0, 0, 0, 0, 0, 0, 3, 0, 11, 24, 4081, 4096, 0, 0, 4093, 0, 0, 1, 0, 4092, 2622, 0, 2396, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 4095, 0, 4096, 4096, 4050, 4, 0, 0, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 3877, 0, 0, 0, 46, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4, 16, 4094, 4096, 0, 0, 4096, 0, 0, 1, 0, 4094, 1968, 2, 2374, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4088, 0, 4096, 4096, 4026, 0, 0, 0, 0, 4079] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 10, 0, 3913, 150, 24, 0, 0, 0, 0, 0, 0, 4096, 0, 3245, 0, 0, 0, 44, 490, 106, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 4, 4096, 4071, 0, 0, 0, 0, 0, 0, 1, 0, 56, 4087, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3456, 2667, 0, 2502, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 177, 3798, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7981, 8179, 8132, 8626, 8192, 8190, 8043] avg=8192.0000 max=8626 max_rank=4 min=7981 min_rank=1 imbalance_ratio=1.052979 max_load_reduction=56910 max_load_reduction_pct=0.868378 moved_to_echo=33051 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 3, 4, 0, 3918, 150, 23, 0, 0, 0, 0, 0, 1, 4096, 0, 4042, 1, 0, 0, 52, 490, 106, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 14, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 7, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3456, 2768, 1, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 177, 3944, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8788, 8192, 8097, 7666, 8189, 8189, 8221] avg=8192.0000 max=8788 max_rank=1 min=7666 min_rank=4 imbalance_ratio=1.072754 max_load_reduction=56748 max_load_reduction_pct=0.865906 moved_to_echo=32265 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 0, 3918, 150, 24, 0, 0, 0, 0, 0, 1, 4093, 0, 3599, 3, 0, 1, 27, 490, 106, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 3, 7, 4091, 4068, 0, 0, 0, 0, 0, 0, 2, 1, 178, 4096, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 1, 3453, 1977, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 4095, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 177, 3740, 0, 0, 1, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8320, 8277, 8264, 8093, 8192, 8198, 8004] avg=8192.0000 max=8320 max_rank=1 min=8004 min_rank=7 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32487 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 2, 0, 3919, 150, 24, 0, 0, 0, 0, 0, 0, 4096, 0, 2939, 0, 0, 0, 2, 490, 106, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 13, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 197, 17, 4096, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3455, 2134, 4, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 177, 4020, 1, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 7633, 8191, 8297, 8557, 8192, 8191, 8284] avg=8192.0000 max=8557 max_rank=4 min=7633 min_rank=1 imbalance_ratio=1.044556 max_load_reduction=56979 max_load_reduction_pct=0.869431 moved_to_echo=33224 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 6, 0, 1, 3919, 150, 24, 0, 0, 0, 0, 0, 0, 4096, 0, 3816, 0, 0, 0, 30, 489, 106, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 2, 102, 4096, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3448, 1583, 0, 2724, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 177, 4039, 1, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8537, 8170, 8187, 7765, 8192, 8195, 8294] avg=8192.0000 max=8537 max_rank=1 min=7765 min_rank=4 imbalance_ratio=1.042114 max_load_reduction=56999 max_load_reduction_pct=0.869736 moved_to_echo=32446 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 0, 0, 4093, 2, 0, 1, 3915, 150, 23, 0, 0, 0, 0, 0, 0, 4096, 0, 3037, 1, 0, 0, 41, 489, 106, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 1, 0, 2, 3, 4094, 4071, 0, 0, 0, 0, 0, 0, 4, 4, 273, 4093, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 3449, 2406, 25, 2677, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 1, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 177, 3842, 2, 1, 1, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 7770, 8179, 8360, 8564, 8195, 8192, 8088] avg=8192.0000 max=8564 max_rank=4 min=7770 min_rank=1 imbalance_ratio=1.045410 max_load_reduction=56972 max_load_reduction_pct=0.869324 moved_to_echo=33039 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 3919, 150, 24, 0, 0, 0, 0, 0, 0, 4096, 0, 3124, 0, 0, 0, 72, 490, 106, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 3, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 11, 24, 4081, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3452, 2622, 0, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 177, 4050, 4, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7888, 8178, 8101, 8471, 8192, 8192, 8324] avg=8192.0000 max=8471 max_rank=4 min=7888 min_rank=1 imbalance_ratio=1.034058 max_load_reduction=57065 max_load_reduction_pct=0.870743 moved_to_echo=33179 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 0, 0, 3923, 152, 24, 0, 0, 0, 0, 0, 0, 4096, 0, 3877, 0, 0, 0, 46, 493, 107, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 4, 16, 4094, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3449, 1968, 2, 2374, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 173, 4026, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8619, 8170, 8098, 7794, 8192, 8189, 8278] avg=8192.0000 max=8619 max_rank=1 min=7794 min_rank=4 imbalance_ratio=1.052124 max_load_reduction=56917 max_load_reduction_pct=0.868484 moved_to_echo=32453 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 0, 128, 128, 0, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 27648, 18176, 128, 19840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 1536, 31488, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 128, 128, 31360, 1280, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 256, 768, 32768, 0, 128, 0, 32000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 27776, 128, 0, 128, 384, 3968, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 32768, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 46, 0, 0, 4082, 1109, 0, 4095, 1, 4096, 3861, 0, 0, 0, 0, 0, 376, 0, 0, 4096, 0, 4096, 0, 0, 0, 4093, 3, 0, 3888, 0, 3430, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 3695, 4084, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 47, 0, 0, 4071, 741, 0, 4096, 0, 4095, 4089, 0, 0, 0, 0, 0, 167, 0, 0, 4095, 0, 4096, 0, 0, 0, 4095, 6, 0, 3857, 0, 4010, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3529, 4061, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 23, 0, 0, 4096, 1292, 0, 4095, 0, 4095, 4042, 4, 0, 0, 0, 0, 51, 1, 0, 4096, 1, 4096, 0, 0, 0, 4096, 0, 0, 3160, 0, 3753, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4062, 4093, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 20, 0, 0, 4089, 1149, 0, 4096, 0, 4096, 4045, 0, 0, 0, 0, 0, 480, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4010, 0, 3551, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 3250, 3980, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 3, 4095, 0, 1, 3, 0, 0, 4096, 0, 0, 4096, 3, 0, 10, 0, 0, 4093, 803, 0, 4082, 3, 4092, 3989, 0, 0, 0, 0, 0, 104, 0, 3, 4095, 0, 4094, 0, 0, 0, 4092, 8, 0, 3581, 0, 4035, 0, 4093, 1, 1, 0, 7, 0, 4094, 0, 1, 3878, 4078, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 4095, 1, 0, 138, 0, 0, 4079, 1202, 0, 4088, 1, 4095, 3892, 1, 0, 0, 1, 0, 55, 1, 0, 4095, 1, 4096, 0, 0, 0, 4093, 2, 0, 3982, 0, 3787, 0, 4096, 1, 0, 0, 1, 0, 4096, 0, 0, 3383, 4060, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 22, 0, 0, 4082, 1197, 0, 4094, 0, 4094, 4081, 0, 0, 0, 0, 0, 200, 1, 0, 4096, 1, 4096, 0, 0, 0, 4096, 0, 0, 3847, 0, 3917, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 3159, 4071, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 0, 0, 45, 0, 0, 4095, 553, 0, 4096, 0, 4095, 4084, 0, 0, 0, 0, 0, 572, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 3973, 0, 4010, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3164, 4078, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 4008, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 4082, 1109, 0, 3054, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 88, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0, 0, 4096, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 3, 0, 3888, 0, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 4084, 0, 0, 0, 0, 359, 251, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8291, 8045, 8317, 7913, 8196, 8390] avg=8192.0000 max=8390 max_rank=7 min=7913 min_rank=5 imbalance_ratio=1.024170 max_load_reduction=57146 max_load_reduction_pct=0.871979 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 4008, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4089, 1149, 0, 3057, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 88, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 0, 0, 4096, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 0, 0, 4010, 0, 3551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3250, 3980, 0, 0, 0, 0, 359, 251, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8315, 8229, 8421, 8153, 8193, 7841] avg=8192.0000 max=8421 max_rank=4 min=7841 min_rank=7 imbalance_ratio=1.027954 max_load_reduction=57115 max_load_reduction_pct=0.871506 moved_to_echo=32608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 1, 4007, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 138, 0, 0, 4079, 1202, 0, 3054, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 88, 3892, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 1, 0, 4095, 1, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 2, 0, 3982, 0, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3383, 4060, 0, 0, 1, 0, 359, 251, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8474, 8071, 7997, 8363, 8193, 8055] avg=8192.0000 max=8474 max_rank=2 min=7997 min_rank=4 imbalance_ratio=1.034424 max_load_reduction=57062 max_load_reduction_pct=0.870697 moved_to_echo=32608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 4007, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 4096, 1292, 0, 3057, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 88, 4042, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 1, 0, 4096, 1, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 0, 0, 3160, 0, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4093, 0, 0, 0, 0, 359, 251, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8468, 8229, 7994, 7505, 8191, 8766] avg=8192.0000 max=8766 max_rank=7 min=7505 min_rank=5 imbalance_ratio=1.070068 max_load_reduction=56770 max_load_reduction_pct=0.866241 moved_to_echo=32456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 1, 0, 3, 4095, 0, 4004, 88, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 0, 0, 4093, 803, 0, 3054, 0, 0, 0, 0, 0, 0, 0, 4082, 3, 88, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 3, 4095, 0, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 8, 0, 3581, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 1, 0, 7, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3878, 4078, 0, 0, 0, 0, 359, 251, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8196, 7963, 8162, 8045, 8215, 8195, 8568] avg=8192.0000 max=8568 max_rank=7 min=7963 min_rank=2 imbalance_ratio=1.045898 max_load_reduction=56968 max_load_reduction_pct=0.869263 moved_to_echo=33023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 4007, 88, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 4071, 741, 0, 3056, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 88, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 4095, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 6, 0, 3857, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3529, 4061, 0, 0, 0, 0, 359, 251, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8193, 7915, 8273, 8107, 8465, 8191, 8201] avg=8192.0000 max=8465 max_rank=5 min=7915 min_rank=2 imbalance_ratio=1.033325 max_load_reduction=57071 max_load_reduction_pct=0.870834 moved_to_echo=32964 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 4011, 89, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4095, 553, 0, 3059, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 84, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 4096, 0, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 0, 0, 3973, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3164, 4078, 0, 0, 1, 0, 365, 252, 7, 6, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8188, 7752, 8264, 8512, 8566, 8185, 7873] avg=8192.0000 max=8566 max_rank=5 min=7752 min_rank=2 imbalance_ratio=1.045654 max_load_reduction=56970 max_load_reduction_pct=0.869293 moved_to_echo=33136 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 4006, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 4082, 1197, 0, 3057, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 88, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 1, 0, 4096, 1, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 0, 0, 3847, 0, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3159, 4071, 0, 0, 1, 0, 359, 251, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8358, 8263, 8143, 8356, 8192, 7842] avg=8192.0000 max=8358 max_rank=2 min=7842 min_rank=7 imbalance_ratio=1.020264 max_load_reduction=57178 max_load_reduction_pct=0.872467 moved_to_echo=32533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 4736, 128, 0, 30336, 0, 30592, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 32768, 128, 32128, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 768, 32128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 28160, 32512, 0, 0, 128, 0, 2944, 2048, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 2048, 128, 128, 32768, 128, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 384, 0, 0, 32768, 8064, 0, 24448, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 11, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 4096, 4089, 4096, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 4, 4096, 13, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4096, 0, 0, 0, 4096, 4087, 4091, 0, 0, 0, 4033, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 81, 4096, 3, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 45, 1, 1, 0, 0, 0, 4096, 0, 7, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 4083, 4095, 0, 0, 0, 3690, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4044, 4096, 4096, 0, 417, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 48, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4096, 0, 0, 0, 4096, 4070, 4096, 0, 0, 0, 4045, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4085, 4096, 4096, 4096, 0, 57, 4096, 5, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 15, 0, 1, 0, 0, 0, 4096, 0, 2, 4096, 0, 10, 7, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4094, 3, 0, 0, 4093, 4079, 4096, 2, 0, 0, 3862, 0, 4096, 0, 1, 0, 4096, 4096, 0, 0, 4089, 4096, 4094, 4094, 0, 216, 4091, 19, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 4096, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 1, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 4095, 0, 0, 0, 4095, 4089, 4096, 1, 0, 0, 4088, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4095, 4096, 0, 7, 4095, 6, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4096, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 4096, 4076, 4096, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4092, 4096, 4096, 4096, 0, 14, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 1, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 4081, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4085, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 13, 0, 0, 0, 0, 0, 3989, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8189, 8185, 8190, 8199, 8284, 8202, 8103] avg=8192.0000 max=8284 max_rank=5 min=8103 min_rank=7 imbalance_ratio=1.011230 max_load_reduction=57252 max_load_reduction_pct=0.873596 moved_to_echo=32788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 48, 4081, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 4085, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4070, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 4096, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4096, 5, 0, 0, 0, 0, 0, 3989, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8226, 8187, 8178, 8180, 8239, 8191, 8148] avg=8192.0000 max=8239 max_rank=5 min=8148 min_rank=7 imbalance_ratio=1.005737 max_load_reduction=57297 max_load_reduction_pct=0.874283 moved_to_echo=32758 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4081, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 4084, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 4095, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4089, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4095, 6, 0, 0, 0, 0, 0, 3989, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8179, 8194, 8195, 8199, 8282, 8202, 8098] avg=8192.0000 max=8282 max_rank=5 min=8098 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32781 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 4081, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4085, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4087, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 4096, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 4096, 3, 0, 0, 0, 0, 0, 3989, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8187, 8192, 8183, 8192, 8227, 8202, 8170] avg=8192.0000 max=8227 max_rank=5 min=8170 min_rank=7 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32791 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 15, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 2, 4079, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 7, 0, 0, 0, 0, 4083, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4094, 9, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 13, 4079, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3862, 0, 4096, 0, 1, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 4091, 19, 0, 0, 0, 1, 0, 3989, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8179, 8205, 8191, 8193, 8057, 8195, 8317] avg=8192.0000 max=8317 max_rank=7 min=8057 min_rank=5 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 45, 4096, 4087, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4096, 0, 7, 4081, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4085, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4083, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3690, 0, 4096, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4044, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 4096, 1, 0, 0, 0, 0, 0, 3989, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8187, 8191, 8200, 8192, 7884, 8150, 8504] avg=8192.0000 max=8504 max_rank=7 min=7884 min_rank=5 imbalance_ratio=1.038086 max_load_reduction=57032 max_load_reduction_pct=0.870239 moved_to_echo=32730 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4096, 4087, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4096, 0, 3, 4081, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4085, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 0, 0, 3989, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8183, 8189, 8200, 8202, 8288, 8202, 8088] avg=8192.0000 max=8288 max_rank=5 min=8088 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32780 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4096, 4087, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 18, 4087, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4076, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 3996, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8206, 8193, 8199, 8179, 8275, 8192, 8108] avg=8192.0000 max=8275 max_rank=5 min=8108 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32000, 0, 32768, 0, 128, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[896, 32768, 128, 0, 0, 0, 128, 0, 32000, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 128, 32768, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 4096, 4090, 0, 0, 0, 3664, 0, 0, 4, 4096, 5, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 401, 0, 0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4095, 4027, 0, 1, 0, 1870, 0, 0, 12, 4096, 1, 0, 4096, 4096, 78, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 144, 0, 0, 2, 0, 2067, 0, 0, 0, 4095, 4091, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 1, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4091, 0, 4096, 4037, 1, 4, 6, 1905, 0, 0, 13, 4096, 6, 0, 4092, 4096, 314, 0, 4096, 4093, 4096, 0, 0, 0, 0, 0, 0, 16, 0, 0, 29, 0, 1883, 0, 0, 0, 4093, 4092, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 4089, 0, 0, 0, 1097, 0, 0, 10, 4096, 0, 0, 4096, 4096, 971, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2024, 0, 0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4085, 0, 4096, 4050, 0, 0, 0, 1652, 0, 0, 15, 4096, 4, 0, 4096, 4096, 16, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 179, 0, 0, 1, 0, 2297, 0, 0, 0, 4096, 4087, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 4096, 4009, 0, 0, 0, 2400, 0, 0, 17, 4096, 0, 0, 4096, 4096, 88, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 149, 0, 0, 9, 0, 1535, 0, 0, 0, 4096, 4089, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 4054, 0, 0, 0, 2946, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 58, 0, 0, 1, 0, 1134, 0, 0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 4096, 4065, 0, 0, 0, 2216, 0, 0, 11, 4096, 0, 0, 4096, 4096, 26, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 192, 0, 0, 5, 0, 1683, 0, 0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2176, 1729, 189, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1920, 4090, 0, 0, 0, 3664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3900, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 401, 0, 4096, 2365, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8192, 9674, 8005, 8194, 6894, 8195] avg=8192.0000 max=9674 max_rank=3 min=6894 min_rank=6 imbalance_ratio=1.180908 max_load_reduction=55862 max_load_reduction_pct=0.852386 moved_to_echo=31288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2176, 1729, 189, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1919, 4027, 0, 1, 0, 1870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3900, 1, 0, 4096, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 2, 0, 2067, 0, 4096, 2365, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4091, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8190, 8194, 7817, 8087, 8194, 8674, 8190] avg=8192.0000 max=8674 max_rank=6 min=7817 min_rank=3 imbalance_ratio=1.058838 max_load_reduction=56862 max_load_reduction_pct=0.867645 moved_to_echo=33145 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2176, 1730, 189, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4085, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1920, 4050, 0, 0, 0, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3900, 4, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 1, 0, 2297, 0, 4096, 2366, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8185, 7622, 8031, 8190, 8939, 8186] avg=8192.0000 max=8939 max_rank=6 min=7622 min_rank=3 imbalance_ratio=1.091187 max_load_reduction=56597 max_load_reduction_pct=0.863602 moved_to_echo=33346 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 1, 0, 0, 0, 2176, 1729, 189, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4091, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1920, 4037, 1, 4, 6, 1905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3900, 6, 0, 4092, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 29, 0, 1883, 0, 4096, 2365, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4092, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8187, 8189, 7873, 8325, 8191, 8389, 8190] avg=8192.0000 max=8389 max_rank=6 min=7873 min_rank=3 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=33095 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2176, 1729, 189, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1919, 4089, 0, 0, 0, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3900, 0, 0, 4096, 0, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2024, 0, 4096, 2367, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8196, 7105, 8977, 8192, 8489, 8195] avg=8192.0000 max=8977 max_rank=4 min=7105 min_rank=3 imbalance_ratio=1.095825 max_load_reduction=56559 max_load_reduction_pct=0.863022 moved_to_echo=33853 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2176, 1729, 189, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1920, 4009, 0, 0, 0, 2400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3900, 0, 0, 4096, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 9, 0, 1535, 0, 4096, 2367, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8188, 8329, 8101, 8192, 8156, 8188] avg=8192.0000 max=8329 max_rank=3 min=8101 min_rank=4 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2176, 1731, 189, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1920, 4054, 0, 0, 0, 2946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3900, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 1, 0, 1134, 0, 4096, 2365, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8195, 8920, 7997, 8191, 7654, 8195] avg=8192.0000 max=8920 max_rank=3 min=7654 min_rank=6 imbalance_ratio=1.088867 max_load_reduction=56616 max_load_reduction_pct=0.863892 moved_to_echo=32037 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2181, 1731, 193, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 11, 0, 0, 0, 0, 0, 0, 0, 1915, 4065, 0, 0, 0, 2216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3880, 0, 0, 4096, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 5, 0, 1683, 0, 4096, 2365, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8199, 8197, 8196, 8013, 8192, 8341, 8197] avg=8192.0000 max=8341 max_rank=6 min=8013 min_rank=4 imbalance_ratio=1.018188 max_load_reduction=57195 max_load_reduction_pct=0.872726 moved_to_echo=32743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 32768, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 32768, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[15360, 32512, 128, 128, 128, 17792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 31232, 128, 0, 32768, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 896, 0, 0, 128, 0, 13056, 0, 32768, 18944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 0, 0, 17536, 13952, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 31, 0, 0, 4096, 4096, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 4096, 4079, 4096, 0, 0, 0, 4096, 1, 0, 4078, 4094, 0, 0, 0, 4095, 0, 0, 4096, 0, 2, 0, 0, 4, 1, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 68, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4032, 4086, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 68, 0, 0, 4096, 4096, 0, 0, 0, 4079, 0, 11, 0, 4096, 0, 12, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 4096, 4088, 4096, 0, 0, 0, 4096, 0, 0, 4033, 4093, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 206, 0, 0, 0, 0, 178, 0, 0, 4096, 4096, 0, 0, 0, 4086, 0, 3, 0, 4096, 0, 13, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 4096, 4057, 4096, 0, 0, 0, 4096, 0, 0, 3865, 3969, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 7, 4, 4096, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 67, 0, 0, 4096, 4096, 0, 0, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4070, 4069, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 231, 0, 0, 4094, 4096, 0, 0, 0, 4087, 0, 1, 0, 4096, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4069, 4096, 0, 0, 0, 4096, 0, 0, 3878, 4090, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 613, 0, 0, 0, 0, 415, 0, 0, 4096, 4096, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4074, 4096, 0, 0, 0, 4096, 0, 0, 3098, 4090, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4095, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 25, 0, 0, 0, 0, 197, 0, 0, 4093, 4096, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4090, 4096, 0, 0, 0, 4096, 0, 0, 3915, 4067, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 1, 3975, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 4096, 3841, 106, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 11, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 1, 0, 4078, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 1, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8077, 8074, 8188, 8197, 8190, 8425, 8191, 8194] avg=8192.0000 max=8425 max_rank=5 min=8074 min_rank=1 imbalance_ratio=1.028442 max_load_reduction=57111 max_load_reduction_pct=0.871445 moved_to_echo=33000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 1, 3975, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 4096, 3841, 106, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4079, 0, 11, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 12, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 4033, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8077, 8111, 8192, 8191, 8199, 8378, 8192, 8196] avg=8192.0000 max=8378 max_rank=5 min=8077 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32965 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 14, 3975, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 4094, 3841, 106, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 1, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 3878, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8089, 8272, 8190, 8199, 8180, 8220, 8192, 8194] avg=8192.0000 max=8272 max_rank=1 min=8089 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 68, 3975, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 3841, 106, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 4032, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8045, 8198, 8193, 8207, 8370, 8192, 8187] avg=8192.0000 max=8370 max_rank=5 min=8045 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32956 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 206, 3975, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 4096, 3841, 106, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 3, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 13, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4057, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 3865, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 4096, 0, 5, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8282, 8221, 8191, 8195, 8168, 8086, 8190, 8203] avg=8192.0000 max=8282 max_rank=0 min=8086 min_rank=5 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32647 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 1, 3975, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 4096, 3841, 106, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4081, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 4070, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8077, 8110, 8183, 8189, 8207, 8391, 8192, 8187] avg=8192.0000 max=8391 max_rank=5 min=8077 min_rank=0 imbalance_ratio=1.024292 max_load_reduction=57145 max_load_reduction_pct=0.871964 moved_to_echo=32977 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 613, 3975, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 415, 0, 0, 4096, 3841, 106, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4074, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 3098, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8689, 8458, 8192, 8192, 8185, 7440, 8192, 8188] avg=8192.0000 max=8689 max_rank=0 min=7440 min_rank=5 imbalance_ratio=1.060669 max_load_reduction=56847 max_load_reduction_pct=0.867416 moved_to_echo=32005 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 25, 3975, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 4093, 3848, 107, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 10, 4, 0, 0, 0, 0, 0, 0, 4083, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 0, 3915, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8101, 8245, 8202, 8180, 8200, 8226, 8195, 8187] avg=8192.0000 max=8245 max_rank=1 min=8101 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 0, 32768, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 1024, 31872, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 1280, 0, 0, 32768, 30848, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 2048, 128, 0, 30976, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3265, 0, 1, 4091, 0, 4093, 4096, 815, 0, 0, 23, 0, 3669, 0, 0, 1, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 4091, 0, 0, 4093, 2, 0, 0, 0, 4094, 0, 4096, 0, 4095, 0, 0, 4096, 4095, 3870, 0, 0, 0, 649, 0, 3, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2716, 0, 1, 4095, 0, 4089, 4096, 1180, 0, 0, 97, 0, 3937, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 4034, 0, 0, 0, 0, 4091, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 3620, 6, 0, 0, 804, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1638, 0, 0, 4095, 0, 4096, 4096, 3402, 0, 0, 28, 0, 3810, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4055, 0, 2, 0, 0, 4072, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 3120, 0, 0, 0, 353, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2374, 0, 0, 4093, 0, 4096, 4096, 1833, 0, 0, 37, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4006, 0, 0, 0, 0, 4059, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 3355, 0, 0, 0, 793, 0, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2338, 0, 0, 4096, 0, 4096, 4096, 2997, 0, 0, 33, 0, 3516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4058, 0, 1, 0, 0, 4037, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 3132, 0, 0, 0, 368, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2095, 0, 2, 4090, 0, 4090, 4096, 2652, 0, 0, 71, 0, 3853, 0, 0, 6, 0, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 4096, 4087, 0, 0, 4047, 3, 6, 0, 0, 4088, 0, 4096, 2, 4092, 0, 0, 4096, 4092, 3413, 0, 0, 0, 240, 0, 9, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3125, 0, 0, 4094, 0, 4096, 4096, 1142, 0, 0, 40, 0, 4064, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4055, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 3285, 0, 0, 0, 672, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2787, 0, 0, 4096, 0, 4096, 4096, 1534, 0, 0, 97, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 3632, 0, 0, 0, 215, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2374, 0, 0, 4093, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2102, 1833, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 4096, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3868, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 4059, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 4096, 3355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 0, 1, 0, 4096, 3199, 382, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8022, 8068, 8349, 8192, 8193, 8121, 8120, 8471] avg=8192.0000 max=8471 max_rank=7 min=8022 min_rank=0 imbalance_ratio=1.034058 max_load_reduction=57065 max_load_reduction_pct=0.870743 moved_to_echo=32905 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3265, 0, 1, 4091, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2102, 815, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3669, 0, 0, 1, 0, 0, 9, 0, 4096, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3868, 227, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 4094, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 4095, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 0, 3, 0, 4096, 3199, 382, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8912, 7033, 8003, 8191, 8189, 8245, 8634, 8329] avg=8192.0000 max=8912 max_rank=0 min=7033 min_rank=1 imbalance_ratio=1.087891 max_load_reduction=56624 max_load_reduction_pct=0.864014 moved_to_echo=33397 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2716, 0, 1, 4095, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2102, 1180, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 5, 0, 4096, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3868, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 4091, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 4096, 3620, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 804, 0, 0, 0, 4096, 3199, 382, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8367, 7468, 8266, 8192, 8190, 8181, 8385, 8487] avg=8192.0000 max=8487 max_rank=7 min=7468 min_rank=1 imbalance_ratio=1.036011 max_load_reduction=57049 max_load_reduction_pct=0.870499 moved_to_echo=33243 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2095, 0, 2, 4090, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2102, 2652, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 6, 0, 0, 18, 0, 4096, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3868, 227, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 4087, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 3, 6, 0, 0, 4088, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 2, 669, 0, 0, 0, 4092, 3413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 9, 0, 4096, 3196, 382, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7742, 8915, 8201, 8191, 8188, 8200, 8176, 7923] avg=8192.0000 max=8915 max_rank=1 min=7742 min_rank=0 imbalance_ratio=1.088257 max_load_reduction=56621 max_load_reduction_pct=0.863968 moved_to_echo=32487 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2338, 0, 0, 4096, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2102, 2997, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3516, 0, 0, 0, 0, 0, 0, 0, 4096, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3868, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 1, 0, 0, 4037, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 4096, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 4096, 3199, 382, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7989, 9228, 7840, 8192, 8193, 8152, 7897, 8045] avg=8192.0000 max=9228 max_rank=1 min=7840 min_rank=2 imbalance_ratio=1.126465 max_load_reduction=56308 max_load_reduction_pct=0.859192 moved_to_echo=32287 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3125, 0, 0, 4094, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2102, 1142, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 3, 0, 4096, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3868, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 4096, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 4096, 3285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 0, 0, 4096, 3199, 382, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8774, 7380, 8391, 8192, 8193, 8207, 8050, 8349] avg=8192.0000 max=8774 max_rank=0 min=7380 min_rank=1 imbalance_ratio=1.071045 max_load_reduction=56762 max_load_reduction_pct=0.866119 moved_to_echo=32799 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2787, 0, 0, 4096, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2089, 1534, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 4096, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3868, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 0, 0, 0, 4096, 3632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 0, 0, 4096, 3205, 386, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8442, 7816, 8351, 8194, 8197, 8245, 8389, 7902] avg=8192.0000 max=8442 max_rank=0 min=7816 min_rank=1 imbalance_ratio=1.030518 max_load_reduction=57094 max_load_reduction_pct=0.871185 moved_to_echo=32733 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1638, 0, 0, 4095, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2102, 3402, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 1, 0, 0, 0, 0, 0, 0, 4096, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3868, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 2, 0, 0, 4072, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 4096, 3120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 0, 0, 0, 4096, 3199, 382, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7288, 9628, 8135, 8192, 8193, 8185, 7885, 8030] avg=8192.0000 max=9628 max_rank=1 min=7288 min_rank=0 imbalance_ratio=1.175293 max_load_reduction=55908 max_load_reduction_pct=0.853088 moved_to_echo=32293 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 16896, 15616, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 5376, 0, 0, 0, 32768, 27520, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 20352, 0, 128, 32768, 12544, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32512, 128, 128, 0, 0, 32640, 0, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 4096, 0, 128, 0, 32768, 25600, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 32768, 0, 0, 30976, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[30976, 128, 0, 128, 0, 0, 128, 0, 32768, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 34, 0, 4084, 0, 0, 4096, 0, 4096, 5, 4096, 0, 4096, 0, 4089, 4096, 0, 0, 4031, 0, 4075, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 0, 12, 1, 4090, 0, 0, 4096, 0, 4095, 2, 4096, 0, 4096, 0, 4080, 4095, 0, 0, 3998, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 0, 50, 3, 4085, 0, 0, 4090, 0, 4095, 5, 4096, 0, 4096, 0, 3973, 4091, 0, 0, 3903, 0, 4095, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 22, 0, 4088, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 4092, 4096, 0, 0, 3791, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 36, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 4093, 4096, 0, 0, 4028, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 7, 0, 4096, 0, 0, 4095, 0, 4096, 0, 4096, 0, 4096, 0, 4050, 4096, 0, 0, 4080, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 3, 0, 4096, 0, 0, 4095, 0, 4096, 0, 4096, 0, 4096, 0, 4089, 4096, 0, 1, 4070, 0, 4089, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 40, 0, 4096, 0, 0, 4095, 0, 4096, 0, 4096, 0, 4096, 0, 4094, 4096, 0, 0, 4042, 0, 4086, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4084, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 27, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 4075, 0, 0, 0, 7, 85, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 4096, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8194, 8217, 8218, 8192, 8139, 8192] avg=8192.0000 max=8218 max_rank=4 min=8139 min_rank=6 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 4090, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 27, 0, 4080, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 4096, 0, 0, 0, 2, 85, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 4096, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8179, 8204, 8201, 8191, 8182, 8196] avg=8192.0000 max=8204 max_rank=3 min=8179 min_rank=2 imbalance_ratio=1.001465 max_load_reduction=57332 max_load_reduction_pct=0.874817 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4096, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 4096, 0, 0, 0, 0, 85, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 4096, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8208, 8216, 8229, 8192, 8115, 8192] avg=8192.0000 max=8229 max_rank=4 min=8115 min_rank=6 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32728 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 3, 4085, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 27, 0, 3973, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 0, 4095, 0, 0, 0, 8, 85, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 271, 0, 0, 0, 4096, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8189, 8208, 8096, 8111, 8191, 8351, 8198] avg=8192.0000 max=8351 max_rank=6 min=8096 min_rank=3 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4088, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 4083, 0, 0, 0, 0, 85, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 4096, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8186, 8215, 7979, 8192, 8388, 8192] avg=8192.0000 max=8388 max_rank=6 min=7979 min_rank=4 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4050, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 0, 0, 1, 85, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 4096, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8178, 8173, 8282, 8192, 8135, 8192] avg=8192.0000 max=8282 max_rank=4 min=8135 min_rank=6 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32801 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4070, 0, 4089, 0, 0, 0, 3, 85, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 4096, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8174, 8212, 8268, 8192, 8114, 8192] avg=8192.0000 max=8268 max_rank=4 min=8114 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4096, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 4086, 0, 0, 0, 1, 90, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4096, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8196, 8209, 8203, 8248, 8194, 8112, 8182] avg=8192.0000 max=8248 max_rank=4 min=8112 min_rank=6 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[256, 128, 32768, 0, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 0, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[896, 0, 0, 0, 32768, 0, 0, 0, 32000, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32000, 0, 32768, 0, 0, 0, 128, 768, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65536 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 256, 0, 32640, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65536 slack_slots=65536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 3930, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 166, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 3989, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 2, 0, 4096, 0, 0, 4096, 107, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 0, 0, 1, 0, 4095, 0, 4, 4096, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4095, 4091, 0, 0, 0, 4023, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 4095, 0, 0, 4096, 73, 0, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[2, 0, 0, 9, 0, 0, 2, 0, 4092, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4090, 4086, 0, 0, 0, 3971, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 1, 4095, 0, 0, 4096, 125, 0, 5, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 4069, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4, 0, 4096, 0, 0, 4096, 27, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 6, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 4072, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 30, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4058, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 38, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 4020, 72, 1, 1, 0, 0, 0, 4093, 0, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 76, 4091, 0, 0, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 107, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8193, 8192, 8156, 8192, 8193, 8194, 8227] avg=8192.0000 max=8227 max_rank=7 min=8156 min_rank=3 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 4020, 72, 1, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 76, 4092, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 27, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8191, 8192, 8237, 8192, 8194, 8196, 8147] avg=8192.0000 max=8237 max_rank=3 min=8147 min_rank=7 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32729 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 0, 0, 1, 0, 4094, 4019, 72, 1, 1, 0, 0, 0, 4095, 0, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 76, 4091, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 73, 0, 1, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8194, 8193, 8190, 8191, 8192, 8190, 8194] avg=8192.0000 max=8194 max_rank=1 min=8190 min_rank=3 imbalance_ratio=1.000244 max_load_reduction=57342 max_load_reduction_pct=0.874969 moved_to_echo=32767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 0, 0, 0, 4094, 4020, 72, 1, 1, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 76, 4089, 0, 0, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 166, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8191, 8191, 8095, 8192, 8194, 8192, 8286] avg=8192.0000 max=8286 max_rank=7 min=8095 min_rank=3 imbalance_ratio=1.011475 max_load_reduction=57250 max_load_reduction_pct=0.873566 moved_to_echo=32864 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 9, 0, 0, 2, 0, 4094, 4014, 72, 1, 1, 0, 0, 0, 4092, 0, 8, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 76, 4086, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 125, 0, 5, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8195, 8195, 8133, 8192, 8185, 8191, 8250] avg=8192.0000 max=8250 max_rank=7 min=8133 min_rank=3 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32818 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 4094, 4020, 72, 1, 1, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 76, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8191, 8191, 8260, 8192, 8194, 8192, 8126] avg=8192.0000 max=8260 max_rank=3 min=8126 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32704 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4022, 74, 2, 1, 3, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 74, 4096, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 38, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8190, 8191, 8228, 8193, 8192, 8188, 8156] avg=8192.0000 max=8228 max_rank=3 min=8156 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=57308 max_load_reduction_pct=0.874451 moved_to_echo=32729 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4020, 72, 1, 1, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 76, 4089, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8191, 8191, 8237, 8192, 8192, 8193, 8150] avg=8192.0000 max=8237 max_rank=3 min=8150 min_rank=7 imbalance_ratio=1.005493 max_load_reduction=57299 max_load_reduction_pct=0.874313 moved_to_echo=32727 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[640, 32768, 0, 0, 0, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 0, 0, 128, 0, 32768, 32256, 640, 128, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 640, 0, 128, 0, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 32768, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 4077, 0, 0, 4094, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4088, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 8, 185, 3985, 0, 0, 3987, 0, 0, 24, 0, 4093, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 3, 4093, 0, 11, 0, 0, 0, 0, 4096, 4096, 4096, 3, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 69, 4078, 0, 0, 4028, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4092, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2, 91, 4011, 0, 0, 4060, 0, 0, 21, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4096, 4096, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 4076, 0, 0, 3957, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 49, 4081, 0, 0, 4044, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 3955, 0, 0, 4085, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4094, 0, 0, 4083, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3998, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 4096, 3999, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 4094, 0, 0, 4, 0, 96, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3996, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8195, 8191, 8111, 8271, 8193, 8188, 8191] avg=8192.0000 max=8271 max_rank=4 min=8111 min_rank=3 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32843 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4000, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 69, 4096, 3999, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 4028, 0, 0, 4, 0, 96, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 3999, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8190, 8193, 8174, 8206, 8193, 8201, 8191] avg=8192.0000 max=8206 max_rank=4 min=8174 min_rank=3 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32791 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3998, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 8, 185, 4096, 3999, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 3987, 0, 0, 24, 0, 96, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 4093, 0, 11, 0, 0, 3996, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8183, 8191, 8300, 8092, 8186, 8202, 8198] avg=8192.0000 max=8300 max_rank=3 min=8092 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32678 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3998, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2, 91, 4096, 3999, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 4060, 0, 0, 21, 0, 96, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 3, 0, 0, 3996, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8191, 8198, 8188, 8191, 8192, 8192] avg=8192.0000 max=8198 max_rank=3 min=8188 min_rank=4 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3998, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 4096, 3999, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 3957, 0, 0, 1, 0, 96, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3996, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8196, 8191, 8251, 8130, 8193, 8188, 8191] avg=8192.0000 max=8251 max_rank=3 min=8130 min_rank=4 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32702 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3998, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 49, 4096, 3999, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4044, 0, 0, 11, 0, 96, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3996, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8196, 8191, 8149, 8232, 8193, 8188, 8191] avg=8192.0000 max=8232 max_rank=4 min=8149 min_rank=3 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32804 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4000, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 4096, 3998, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 4085, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3997, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8194, 8246, 8138, 8194, 8189, 8191] avg=8192.0000 max=8246 max_rank=3 min=8138 min_rank=4 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32712 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4000, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 3998, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4083, 0, 0, 2, 0, 100, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3996, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8194, 8107, 8279, 8193, 8188, 8191] avg=8192.0000 max=8279 max_rank=4 min=8107 min_rank=3 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32851 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32384, 0, 0, 32384, 0, 0, 128, 0, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 0, 128, 0, 0, 32000, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 0, 0, 32000, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 768, 32768, 32000, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 0, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 16, 0, 0, 0, 0, 4096, 4080, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 8, 0, 0, 0, 0, 4094, 4088, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 4090, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4089, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 4096, 18, 0, 0, 0, 0, 4093, 4080, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 4096, 4096, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 4095, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 10, 0, 0, 0, 0, 4096, 4087, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 7, 0, 0, 0, 0, 4096, 4089, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 10, 4080, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8191, 8199, 8186, 8192, 8190, 8193] avg=8192.0000 max=8199 max_rank=3 min=8186 min_rank=4 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 10, 4090, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8191, 8186, 8195, 8193, 8194, 8191] avg=8192.0000 max=8195 max_rank=4 min=8186 min_rank=3 imbalance_ratio=1.000366 max_load_reduction=57341 max_load_reduction_pct=0.874954 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 8, 0, 0, 0, 0, 4083, 2, 0, 0, 0, 0, 0, 0, 10, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8191, 8191, 8194, 8192, 8193, 8192] avg=8192.0000 max=8194 max_rank=4 min=8191 min_rank=0 imbalance_ratio=1.000244 max_load_reduction=57342 max_load_reduction_pct=0.874969 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 18, 0, 0, 0, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 10, 4080, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8190, 8191, 8203, 8185, 8191, 8197, 8192] avg=8192.0000 max=8203 max_rank=3 min=8185 min_rank=4 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8191, 8183, 8201, 8192, 8191, 8192] avg=8192.0000 max=8201 max_rank=4 min=8183 min_rank=3 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 4085, 2, 0, 0, 0, 0, 0, 0, 10, 4087, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 8191, 8193, 8192, 8192, 8190, 8192] avg=8192.0000 max=8194 max_rank=0 min=8190 min_rank=6 imbalance_ratio=1.000244 max_load_reduction=57342 max_load_reduction_pct=0.874969 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4086, 2, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8191, 8185, 8198, 8192, 8193, 8193] avg=8192.0000 max=8198 max_rank=4 min=8185 min_rank=3 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32776 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 4089, 4, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8199, 8196, 8185, 8192, 8188, 8191] avg=8192.0000 max=8199 max_rank=2 min=8185 min_rank=4 imbalance_ratio=1.000854 max_load_reduction=57337 max_load_reduction_pct=0.874893 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 32768, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 128, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 26, 0, 0, 0, 0, 4008, 0, 4096, 4068, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 72, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 109, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 96, 0, 0, 0, 0, 4074, 0, 4096, 3886, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 4090, 4089, 0, 4096, 184, 0, 0, 0, 0, 4074, 0, 4096, 4035, 0, 0, 0, 4096, 4096, 4093, 0, 3, 0, 0, 0, 1, 0, 0, 9, 4096, 3, 4093, 0, 0, 0, 4096, 0, 3992, 0, 0, 0, 4082, 0, 0, 0, 4096, 0, 4, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 64, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 4096, 0, 4096, 47, 0, 0, 0, 0, 4035, 0, 4096, 4045, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 14, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 4095, 0, 4096, 25, 0, 0, 0, 0, 4085, 0, 4096, 3976, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 94, 4096, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 44, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 179, 0, 0, 0, 0, 4091, 0, 4096, 3923, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4046, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 45, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 146, 0, 0, 0, 0, 4056, 0, 4096, 4014, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4096, 0, 4096, 0, 0, 0, 4096, 0, 3985, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 36, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 82, 0, 0, 0, 0, 4019, 0, 4096, 3952, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 105, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 0, 0, 0, 0, 0, 0, 4096, 4052, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 26, 0, 0, 0, 0, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 109, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4055, 40, 0, 0, 0, 0, 0, 0, 0, 72, 4052, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3987, 57, 44, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8196, 8073, 8273, 8191, 8220, 8228, 8189] avg=8192.0000 max=8273 max_rank=3 min=8073 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=57263 max_load_reduction_pct=0.873764 moved_to_echo=32828 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 64, 0, 0, 0, 0, 0, 0, 4096, 4052, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 47, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 109, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4055, 43, 0, 0, 0, 0, 0, 0, 0, 4, 4052, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 3987, 57, 44, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8189, 8121, 8250, 8194, 8152, 8228, 8190] avg=8192.0000 max=8250 max_rank=3 min=8121 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 14, 0, 0, 0, 1, 0, 0, 4096, 4052, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 25, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 0, 109, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4055, 40, 0, 0, 0, 0, 0, 0, 0, 94, 4052, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 3987, 57, 44, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8163, 8192, 8149, 8181, 8192, 8241, 8227, 8191] avg=8192.0000 max=8241 max_rank=5 min=8149 min_rank=2 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 0, 1, 0, 0, 4096, 4051, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4090, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 184, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 109, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 1, 0, 4055, 40, 0, 0, 0, 0, 0, 0, 0, 9, 4052, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3992, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 3987, 57, 44, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8183, 8297, 8240, 8192, 8157, 8110, 8193] avg=8192.0000 max=8297 max_rank=2 min=8110 min_rank=6 imbalance_ratio=1.012817 max_load_reduction=57239 max_load_reduction_pct=0.873398 moved_to_echo=32652 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 44, 0, 0, 0, 0, 0, 0, 4096, 4052, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 179, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 0, 109, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4055, 40, 0, 0, 0, 0, 0, 0, 0, 5, 4052, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4046, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3987, 57, 44, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8196, 8309, 8128, 8191, 8153, 8178, 8189] avg=8192.0000 max=8309 max_rank=2 min=8128 min_rank=3 imbalance_ratio=1.014282 max_load_reduction=57227 max_load_reduction_pct=0.873215 moved_to_echo=32711 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 45, 0, 0, 0, 0, 0, 0, 4096, 4052, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 146, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 0, 109, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4055, 40, 0, 0, 0, 0, 0, 0, 0, 43, 4052, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3985, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3987, 57, 44, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8196, 8241, 8219, 8190, 8191, 8117, 8189] avg=8192.0000 max=8241 max_rank=2 min=8117 min_rank=6 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32687 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 109, 0, 0, 0, 0, 0, 0, 4096, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 96, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 0, 0, 109, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4055, 43, 0, 0, 0, 0, 0, 0, 0, 27, 4052, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3987, 57, 44, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8258, 8192, 8209, 8091, 8194, 8175, 8228, 8189] avg=8192.0000 max=8258 max_rank=0 min=8091 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 36, 0, 0, 0, 0, 0, 0, 4096, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 82, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 0, 0, 0, 106, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4056, 40, 0, 0, 0, 0, 0, 0, 0, 105, 4046, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3990, 60, 50, 10, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8192, 8137, 8154, 8192, 8247, 8220, 8206] avg=8192.0000 max=8247 max_rank=5 min=8137 min_rank=2 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 384, 0, 0, 0, 128, 0, 0, 32768, 32512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 32000, 0, 0, 0, 896, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[384, 0, 32512, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 0, 128, 0, 0, 0, 32000, 512, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32384, 896, 0, 0, 0, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 128, 0, 32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 384, 32512, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4061, 0, 4096, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 206, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4074, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 3739, 0, 4085, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 15, 4096, 0, 0, 0, 4094, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 14, 0, 0, 588, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4081, 4096, 0, 4096, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4055, 0, 4093, 0, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 10, 0, 0, 136, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4092, 4096, 0, 4096, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 3652, 0, 4096, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 7, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 1070, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 4096, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 3846, 0, 4096, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 33, 0, 3, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 17, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3, 0, 0, 625, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4074, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 3437, 0, 4096, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 786, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4085, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4050, 0, 4096, 0, 3863, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 257, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4050, 0, 4096, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 210, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4090, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4061, 0, 502, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4076, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3594, 484, 8, 7, 0, 0, 0, 0, 206, 0, 4096, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8497, 8172, 8192, 8198, 8192, 8189, 7914, 8182] avg=8192.0000 max=8497 max_rank=0 min=7914 min_rank=6 imbalance_ratio=1.037231 max_load_reduction=57039 max_load_reduction_pct=0.870346 moved_to_echo=32477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3846, 0, 502, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 3, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4076, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3, 0, 0, 3594, 484, 8, 7, 0, 0, 0, 0, 625, 0, 4096, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8036, 8209, 8192, 8202, 8191, 8191, 8333, 8182] avg=8192.0000 max=8333 max_rank=6 min=8036 min_rank=0 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32897 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4055, 0, 501, 0, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4076, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 10, 0, 0, 3592, 484, 8, 7, 0, 0, 0, 0, 136, 0, 4096, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 12, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8548, 8176, 8192, 8191, 8191, 8196, 7844, 8198] avg=8192.0000 max=8548 max_rank=0 min=7844 min_rank=6 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=32429 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3739, 0, 500, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4089, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4076, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 14, 0, 0, 3585, 484, 8, 7, 0, 0, 0, 0, 588, 0, 4096, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 12, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 8185, 8192, 8198, 8193, 8187, 8296, 8185] avg=8192.0000 max=8296 max_rank=6 min=8100 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32861 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3652, 0, 502, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4076, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 3594, 484, 8, 7, 0, 0, 0, 0, 1070, 0, 4096, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 12, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7566, 8226, 8192, 8192, 8191, 8190, 8778, 8201] avg=8192.0000 max=8778 max_rank=6 min=7566 min_rank=0 imbalance_ratio=1.071533 max_load_reduction=56758 max_load_reduction_pct=0.866058 moved_to_echo=33360 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3437, 0, 502, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4076, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3594, 484, 8, 7, 0, 0, 0, 0, 786, 0, 4096, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7918, 8173, 8192, 8186, 8191, 8189, 8494, 8193] avg=8192.0000 max=8494 max_rank=6 min=7918 min_rank=0 imbalance_ratio=1.036865 max_load_reduction=57042 max_load_reduction_pct=0.870392 moved_to_echo=33067 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4050, 0, 502, 0, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4076, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 3594, 484, 8, 7, 0, 0, 0, 0, 257, 0, 4096, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8415, 8193, 8192, 8186, 8191, 8190, 7965, 8204] avg=8192.0000 max=8415 max_rank=0 min=7965 min_rank=6 imbalance_ratio=1.027222 max_load_reduction=57121 max_load_reduction_pct=0.871597 moved_to_echo=32550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4050, 0, 497, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3599, 490, 10, 9, 0, 0, 0, 0, 210, 0, 4096, 3606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8456, 8202, 8192, 8183, 8196, 8204, 7912, 8191] avg=8192.0000 max=8456 max_rank=0 min=7912 min_rank=6 imbalance_ratio=1.032227 max_load_reduction=57080 max_load_reduction_pct=0.870972 moved_to_echo=32503 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 30976, 0, 4096, 0, 30720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 0, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[3968, 0, 32768, 28928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65536 observed_loss=0 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 256, 0, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 0, 0, 128, 0, 0, 28800, 3968, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65536 slack_slots=65536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 4096, 0, 4088, 0, 0, 828, 4096, 0, 0, 0, 4096, 0, 4, 0, 3269, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 4093, 29, 0, 0, 0, 0, 1, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 38, 4096, 0, 0, 0, 4096, 0, 0, 0, 4077, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 107, 4096, 0, 0, 0, 4096, 0, 0, 0, 4008, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 68, 4096, 0, 0, 0, 4096, 0, 0, 0, 4045, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4096, 0, 4094, 0, 0, 58, 4096, 0, 0, 0, 4096, 0, 2, 0, 4047, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4095, 12, 0, 0, 0, 0, 1, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 126, 4096, 0, 0, 0, 4095, 0, 0, 0, 3971, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 52, 4096, 0, 0, 0, 4096, 0, 0, 0, 4044, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 31, 4096, 0, 0, 0, 4096, 0, 0, 0, 4065, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 15, 0, 0, 0, 0, 0, 0, 3935, 0, 4092, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4077, 0, 146, 8, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8065, 8327, 8191, 8189, 8192, 8188] avg=8192.0000 max=8327 max_rank=3 min=8065 min_rank=2 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 15, 0, 0, 0, 0, 0, 0, 3935, 0, 4096, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4008, 0, 146, 8, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8138, 8258, 8191, 8202, 8192, 8171] avg=8192.0000 max=8258 max_rank=3 min=8138 min_rank=2 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 4081, 15, 0, 0, 0, 0, 0, 0, 3935, 0, 4088, 0, 0, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 3269, 0, 146, 8, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 29, 0, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 3, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8189, 8851, 7523, 8190, 8211, 8191, 8182] avg=8192.0000 max=8851 max_rank=2 min=7523 min_rank=3 imbalance_ratio=1.080444 max_load_reduction=56685 max_load_reduction_pct=0.864944 moved_to_echo=32774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4081, 15, 0, 0, 0, 0, 0, 0, 3935, 0, 4094, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4047, 0, 146, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 12, 0, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 1, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8087, 8299, 8192, 8196, 8191, 8187] avg=8192.0000 max=8299 max_rank=3 min=8087 min_rank=2 imbalance_ratio=1.013062 max_load_reduction=57237 max_load_reduction_pct=0.873367 moved_to_echo=32766 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 15, 0, 0, 0, 0, 0, 0, 3935, 0, 4094, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4045, 0, 146, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8097, 8295, 8192, 8187, 8192, 8189] avg=8192.0000 max=8295 max_rank=3 min=8097 min_rank=2 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 15, 0, 0, 0, 0, 0, 0, 3935, 0, 4096, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3971, 0, 146, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8157, 8220, 8192, 8184, 8192, 8207] avg=8192.0000 max=8220 max_rank=3 min=8157 min_rank=2 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32775 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 15, 0, 0, 0, 0, 0, 0, 3935, 0, 4095, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4044, 0, 146, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8082, 8294, 8192, 8185, 8192, 8206] avg=8192.0000 max=8294 max_rank=3 min=8082 min_rank=2 imbalance_ratio=1.012451 max_load_reduction=57242 max_load_reduction_pct=0.873444 moved_to_echo=32775 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 16, 0, 0, 0, 0, 0, 0, 3932, 0, 4096, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4065, 0, 148, 11, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8194, 8059, 8320, 8196, 8182, 8194, 8206] avg=8192.0000 max=8320 max_rank=3 min=8059 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[31488, 0, 32768, 0, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 0, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[128, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 0, 32768, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 0, 128, 0, 31616, 0, 1280, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 32768, 32768, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4059, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4069, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4075, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 21, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4066, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 2, 2, 0, 0, 4096, 0, 0, 0, 4096, 4, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4093, 0, 2, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4063, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 32, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 4096, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 2, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4089, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4059, 0, 0, 0, 0, 0, 34, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 13, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4080, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8192, 8180, 8192, 8192, 8193, 8193, 8192] avg=8192.0000 max=8202 max_rank=0 min=8180 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=57334 max_load_reduction_pct=0.874847 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4075, 0, 0, 0, 0, 0, 8, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 13, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8198, 8192, 8192, 8190, 8188, 8192] avg=8192.0000 max=8198 max_rank=2 min=8188 min_rank=6 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4066, 0, 0, 0, 0, 0, 30, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 13, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4095, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 4079, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8192, 8180, 8196, 8192, 8189, 8190, 8192] avg=8192.0000 max=8205 max_rank=0 min=8180 min_rank=2 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4069, 0, 0, 0, 0, 0, 1, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 13, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8192, 8203, 8192, 8192, 8190, 8196, 8192] avg=8192.0000 max=8203 max_rank=2 min=8179 min_rank=0 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4063, 0, 0, 0, 0, 0, 6, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 13, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4095, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 1, 0, 4079, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8192, 8208, 8193, 8192, 8191, 8190, 8192] avg=8192.0000 max=8208 max_rank=2 min=8178 min_rank=0 imbalance_ratio=1.001953 max_load_reduction=57328 max_load_reduction_pct=0.874756 moved_to_echo=32765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 0, 0, 0, 0, 3, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 13, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8192, 8181, 8192, 8192, 8190, 8196, 8192] avg=8192.0000 max=8201 max_rank=0 min=8181 min_rank=2 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4069, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 13, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4083, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8191, 8203, 8192, 8194, 8194, 8191, 8192] avg=8192.0000 max=8203 max_rank=2 min=8179 min_rank=0 imbalance_ratio=1.001343 max_load_reduction=57333 max_load_reduction_pct=0.874832 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 15, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4081, 15, 5, 2, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8193, 8183, 8187, 8190, 8199, 8192, 8192] avg=8192.0000 max=8200 max_rank=0 min=8183 min_rank=2 imbalance_ratio=1.000977 max_load_reduction=57336 max_load_reduction_pct=0.874878 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65536 observed_loss=0 tokens_per_expert=[0, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 32640, 0, 0, 0, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 128, 0, 32640, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 32768, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65536 slack_slots=65536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 12, 0, 4058, 0, 4034, 1, 0, 4095, 0, 4096, 1, 0, 0, 1, 4095, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 95, 0, 4096, 0, 0, 0, 4096, 3, 4094, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 3, 0, 4096, 0, 4096, 4072, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 0, 4052, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 49, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4056, 0, 3912, 3, 0, 4094, 0, 4096, 1, 0, 0, 1, 4061, 0, 0, 4092, 4096, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 274, 4, 4096, 0, 0, 0, 4096, 7, 4090, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 28, 0, 0, 0, 4096, 17, 0, 4096, 0, 4096, 4023, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 5, 0, 4034, 0, 3957, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 253, 0, 4096, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4008, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 6, 0, 4075, 0, 3953, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 160, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 5, 0, 4096, 0, 4096, 4087, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4082, 0, 3947, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 155, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 4096, 0, 4096, 4094, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 8, 0, 4014, 0, 3800, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 376, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 4096, 4086, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4066, 0, 3941, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 165, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 2, 0, 4096, 0, 4096, 4093, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 0, 4058, 0, 4034, 1, 0, 178, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 5, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 95, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 3, 4069, 9, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8197, 8192, 8098, 8193, 8188, 8190, 8195] avg=8192.0000 max=8283 max_rank=0 min=8098 min_rank=3 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 4034, 0, 3957, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 253, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4096, 0, 4069, 9, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8195, 8192, 8252, 8193, 8204, 8195, 8131] avg=8192.0000 max=8252 max_rank=3 min=8131 min_rank=7 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 0, 4052, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 49, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 4069, 9, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 8197, 8193, 8053, 8192, 8187, 8178, 8219] avg=8192.0000 max=8317 max_rank=0 min=8053 min_rank=3 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32776 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4056, 0, 3912, 3, 0, 178, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 5, 1, 0, 0, 1, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 274, 4, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 4096, 17, 4069, 9, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 8162, 8189, 8275, 8193, 8203, 8219, 8146] avg=8192.0000 max=8275 max_rank=3 min=8146 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 4014, 0, 3800, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 376, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4069, 9, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8000, 8197, 8193, 8379, 8192, 8190, 8176, 8209] avg=8192.0000 max=8379 max_rank=3 min=8000 min_rank=0 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32767 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 4075, 0, 3953, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 160, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 5, 4069, 9, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8197, 8193, 8165, 8192, 8186, 8181, 8210] avg=8192.0000 max=8212 max_rank=0 min=8165 min_rank=3 imbalance_ratio=1.002441 max_load_reduction=57324 max_load_reduction_pct=0.874695 moved_to_echo=32769 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4082, 0, 3947, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 155, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 4069, 9, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8197, 8192, 8158, 8192, 8187, 8186, 8217] avg=8192.0000 max=8217 max_rank=7 min=8158 min_rank=3 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32782 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4066, 0, 3941, 0, 0, 184, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 165, 0, 3897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 2, 4075, 15, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8194, 8192, 8156, 8189, 8191, 8211, 8209] avg=8192.0000 max=8211 max_rank=6 min=8156 min_rank=3 imbalance_ratio=1.002319 max_load_reduction=57325 max_load_reduction_pct=0.874710 moved_to_echo=32800 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 256, 0, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 32512, 0, 31616, 128, 0, 1536, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 0, 0, 1536, 128, 31360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 32768, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[591, 0, 50, 0, 51, 0, 3267, 0, 685, 2340, 27, 125, 1168, 2448, 0, 394, 615, 654, 26, 2109, 0, 683, 403, 2230, 0, 1, 1174, 1907, 3570, 34, 3010, 768, 717, 157, 1654, 576, 326, 21, 3170, 747, 3528, 0, 3627, 2274, 0, 234, 1873, 769, 3386, 480, 2603, 0, 0, 56, 7, 381, 1, 108, 280, 2042, 3662, 833, 2021, 1673] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[695, 0, 61, 0, 63, 0, 2861, 3, 549, 1820, 30, 173, 1131, 1828, 0, 481, 1183, 539, 35, 2551, 0, 611, 253, 2012, 1, 2, 1338, 1893, 3353, 22, 2878, 539, 819, 264, 2158, 275, 270, 18, 3320, 976, 3790, 0, 3528, 2441, 0, 208, 1327, 798, 3478, 370, 2951, 0, 1, 22, 0, 657, 0, 281, 341, 2216, 3552, 704, 2081, 1785] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1100, 5, 51, 38, 69, 0, 3554, 3, 750, 1561, 46, 328, 911, 1596, 0, 477, 569, 429, 25, 2633, 0, 715, 557, 1919, 0, 16, 1433, 1702, 3591, 13, 2763, 503, 656, 308, 2259, 351, 493, 16, 2719, 1175, 3674, 0, 3501, 2212, 12, 252, 1313, 729, 3599, 551, 3255, 0, 11, 80, 0, 395, 0, 121, 421, 2471, 3594, 985, 1211, 1815] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[917, 22, 33, 15, 100, 2, 3260, 2, 373, 1431, 19, 147, 734, 2098, 4, 492, 942, 662, 19, 2848, 0, 767, 419, 2006, 7, 6, 803, 1347, 3201, 5, 2792, 749, 1260, 315, 2090, 182, 206, 38, 3057, 891, 3720, 0, 3528, 2281, 5, 237, 1593, 866, 3366, 314, 3092, 1, 2, 50, 4, 572, 2, 213, 455, 2166, 3144, 1451, 1943, 2270] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1125, 0, 12, 8, 16, 1, 3541, 3, 642, 2540, 12, 123, 646, 2190, 0, 446, 663, 184, 21, 2380, 0, 582, 390, 2180, 1, 1, 836, 1351, 3689, 35, 2685, 562, 515, 149, 2396, 564, 634, 14, 3171, 613, 3974, 0, 3665, 2797, 0, 254, 1552, 798, 3716, 373, 2890, 0, 0, 18, 2, 395, 1, 92, 201, 1991, 3524, 871, 2139, 1362] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1087, 5, 78, 10, 78, 4, 3343, 2, 711, 1928, 39, 167, 612, 1963, 5, 883, 775, 515, 90, 2633, 0, 695, 656, 2594, 3, 2, 940, 1587, 3154, 16, 2739, 507, 873, 167, 1939, 426, 290, 44, 3175, 399, 3515, 3, 3554, 2274, 0, 318, 1689, 474, 3579, 490, 3034, 0, 0, 15, 10, 340, 0, 197, 475, 1943, 3686, 1306, 1289, 2211] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[905, 14, 106, 6, 24, 0, 3457, 6, 544, 1964, 21, 137, 1050, 1756, 0, 690, 597, 509, 34, 2261, 0, 1073, 465, 1902, 1, 10, 1376, 1705, 3546, 19, 2723, 580, 658, 246, 1920, 687, 139, 12, 2939, 920, 3369, 0, 3758, 2305, 7, 170, 1840, 1029, 3510, 260, 3146, 4, 1, 32, 1, 453, 0, 99, 447, 1851, 3309, 1110, 1822, 2011] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[809, 15, 37, 2, 24, 0, 3631, 2, 383, 2349, 21, 125, 599, 2305, 0, 436, 1055, 545, 36, 2274, 0, 593, 418, 1931, 0, 27, 1098, 1552, 3658, 52, 2898, 785, 778, 286, 1541, 483, 463, 10, 3187, 763, 3453, 1, 3725, 2468, 2, 223, 1844, 856, 3731, 377, 2744, 0, 1, 42, 2, 335, 0, 164, 253, 2168, 3297, 1366, 1982, 1331] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[591, 0, 50, 0, 51, 0, 3267, 0, 3528, 177, 0, 0, 0, 0, 0, 0, 685, 2340, 27, 125, 1168, 2448, 0, 394, 2121, 0, 0, 0, 0, 0, 0, 0, 615, 654, 26, 2109, 0, 683, 403, 2230, 732, 444, 0, 0, 0, 0, 0, 0, 0, 1, 1174, 1907, 2025, 34, 3010, 768, 0, 0, 0, 0, 0, 0, 0, 0, 717, 157, 1654, 576, 326, 21, 3170, 747, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3166, 2274, 0, 234, 1873, 769, 0, 0, 0, 0, 0, 0, 0, 0, 3386, 480, 2603, 0, 0, 56, 7, 381, 813, 3, 0, 0, 0, 0, 0, 0, 1, 108, 280, 2042, 920, 833, 2021, 1673, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7664, 9308, 7896, 8919, 7826, 8316, 7729, 7878] avg=8192.0000 max=9308 max_rank=1 min=7664 min_rank=0 imbalance_ratio=1.136230 max_load_reduction=56228 max_load_reduction_pct=0.857971 moved_to_echo=31749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[695, 0, 61, 0, 63, 0, 2861, 3, 3790, 172, 0, 0, 0, 0, 0, 0, 549, 1820, 30, 173, 1131, 1828, 0, 481, 2057, 0, 0, 0, 0, 0, 0, 0, 1183, 539, 35, 2551, 0, 611, 253, 2012, 688, 431, 0, 0, 0, 0, 0, 0, 1, 2, 1338, 1893, 1902, 22, 2878, 539, 0, 0, 0, 0, 0, 0, 0, 0, 819, 264, 2158, 275, 270, 18, 3320, 976, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 2441, 0, 208, 1327, 798, 0, 0, 0, 0, 0, 0, 0, 0, 3478, 370, 2951, 0, 1, 22, 0, 657, 763, 3, 0, 0, 0, 0, 0, 0, 0, 281, 341, 2216, 892, 704, 2081, 1785, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7645, 8069, 8303, 8575, 8545, 7854, 8245, 8300] avg=8192.0000 max=8575 max_rank=3 min=7645 min_rank=0 imbalance_ratio=1.046753 max_load_reduction=56961 max_load_reduction_pct=0.869156 moved_to_echo=32944 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1100, 5, 51, 38, 69, 0, 3554, 3, 3674, 174, 0, 0, 0, 0, 0, 0, 750, 1561, 46, 328, 911, 1596, 0, 477, 2081, 0, 0, 0, 0, 0, 0, 0, 569, 429, 25, 2633, 0, 715, 557, 1919, 737, 436, 0, 0, 0, 0, 0, 0, 0, 16, 1433, 1702, 2036, 13, 2763, 503, 0, 0, 0, 0, 0, 0, 0, 0, 656, 308, 2259, 351, 493, 16, 2719, 1175, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 2212, 12, 252, 1313, 729, 0, 0, 0, 0, 0, 0, 0, 0, 3599, 551, 3255, 0, 11, 80, 0, 395, 818, 3, 0, 0, 0, 0, 0, 0, 0, 121, 421, 2471, 903, 985, 1211, 1815, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8668, 7750, 8020, 8466, 8419, 7574, 8712, 7927] avg=8192.0000 max=8712 max_rank=6 min=7574 min_rank=5 imbalance_ratio=1.063477 max_load_reduction=56824 max_load_reduction_pct=0.867065 moved_to_echo=32632 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[917, 22, 33, 15, 100, 2, 3260, 2, 3720, 152, 0, 0, 0, 0, 0, 0, 373, 1431, 19, 147, 734, 2098, 4, 492, 1820, 0, 0, 0, 0, 0, 0, 0, 942, 662, 19, 2848, 0, 767, 419, 2006, 657, 381, 0, 0, 0, 0, 0, 0, 7, 6, 803, 1347, 1815, 5, 2792, 749, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 315, 2090, 182, 206, 38, 3057, 891, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 2281, 5, 237, 1593, 866, 0, 0, 0, 0, 0, 0, 0, 0, 3366, 314, 3092, 1, 2, 50, 4, 572, 729, 3, 0, 0, 0, 0, 0, 0, 2, 213, 455, 2166, 791, 1451, 1943, 2270, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 7118, 8701, 7524, 8484, 8062, 8133, 9291] avg=8192.0000 max=9291 max_rank=7 min=7118 min_rank=1 imbalance_ratio=1.134155 max_load_reduction=56245 max_load_reduction_pct=0.858231 moved_to_echo=33970 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1125, 0, 12, 8, 16, 1, 3541, 3, 3974, 171, 0, 0, 0, 0, 0, 0, 642, 2540, 12, 123, 646, 2190, 0, 446, 2041, 0, 0, 0, 0, 0, 0, 0, 663, 184, 21, 2380, 0, 582, 390, 2180, 757, 427, 0, 0, 0, 0, 0, 0, 1, 1, 836, 1351, 2092, 35, 2685, 562, 0, 0, 0, 0, 0, 0, 0, 0, 515, 149, 2396, 564, 634, 14, 3171, 613, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3199, 2797, 0, 254, 1552, 798, 0, 0, 0, 0, 0, 0, 0, 0, 3716, 373, 2890, 0, 0, 18, 2, 395, 840, 3, 0, 0, 0, 0, 0, 0, 1, 92, 201, 1991, 885, 871, 2139, 1362, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8851, 8640, 7584, 7563, 8519, 8600, 8237, 7542] avg=8192.0000 max=8851 max_rank=0 min=7542 min_rank=7 imbalance_ratio=1.080444 max_load_reduction=56685 max_load_reduction_pct=0.864944 moved_to_echo=32898 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1087, 5, 78, 10, 78, 4, 3343, 2, 3515, 179, 0, 0, 0, 0, 0, 0, 711, 1928, 39, 167, 612, 1963, 5, 883, 2134, 0, 0, 0, 0, 0, 0, 0, 775, 515, 90, 2633, 0, 695, 656, 2594, 647, 447, 0, 0, 0, 0, 0, 0, 3, 2, 940, 1587, 1789, 16, 2739, 507, 0, 0, 0, 0, 0, 0, 0, 0, 873, 167, 1939, 426, 290, 44, 3175, 399, 449, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3102, 2274, 0, 318, 1689, 474, 0, 0, 0, 0, 0, 0, 0, 0, 3579, 490, 3034, 0, 0, 15, 10, 340, 718, 3, 0, 0, 0, 0, 0, 0, 0, 197, 475, 1943, 926, 1306, 1289, 2211, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 8442, 9052, 7583, 7762, 7860, 8189, 8347] avg=8192.0000 max=9052 max_rank=2 min=7583 min_rank=3 imbalance_ratio=1.104980 max_load_reduction=56484 max_load_reduction_pct=0.861877 moved_to_echo=32158 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[905, 14, 106, 6, 24, 0, 3457, 6, 3369, 164, 0, 0, 0, 0, 0, 0, 544, 1964, 21, 137, 1050, 1756, 0, 690, 1920, 0, 0, 0, 0, 0, 0, 0, 597, 509, 34, 2261, 0, 1073, 465, 1902, 731, 406, 0, 0, 0, 0, 0, 0, 1, 10, 1376, 1705, 2005, 19, 2723, 580, 0, 0, 0, 0, 0, 0, 0, 0, 658, 246, 1920, 687, 139, 12, 2939, 920, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3273, 2305, 7, 170, 1840, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 3510, 260, 3146, 4, 1, 32, 1, 453, 810, 6, 0, 0, 0, 0, 0, 0, 0, 99, 447, 1851, 819, 1110, 1822, 2011, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8051, 8082, 7978, 8419, 8000, 8624, 8223, 8159] avg=8192.0000 max=8624 max_rank=5 min=7978 min_rank=2 imbalance_ratio=1.052734 max_load_reduction=56912 max_load_reduction_pct=0.868408 moved_to_echo=33006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[809, 15, 37, 2, 24, 0, 3631, 2, 3453, 160, 0, 0, 0, 0, 0, 0, 383, 2349, 21, 125, 599, 2305, 0, 436, 1909, 0, 0, 0, 0, 0, 0, 0, 1055, 545, 36, 2274, 0, 593, 418, 1931, 750, 400, 0, 0, 0, 0, 0, 0, 0, 27, 1098, 1552, 2075, 52, 2898, 785, 0, 0, 0, 0, 0, 0, 0, 0, 778, 286, 1541, 483, 463, 10, 3187, 763, 470, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3252, 2468, 2, 223, 1844, 856, 0, 0, 0, 0, 0, 0, 0, 0, 3731, 377, 2744, 0, 1, 42, 2, 335, 833, 3, 0, 0, 0, 0, 0, 0, 0, 164, 253, 2168, 828, 1366, 1982, 1331, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8133, 8127, 8002, 8487, 7981, 8646, 8068, 8092] avg=8192.0000 max=8646 max_rank=5 min=7981 min_rank=4 imbalance_ratio=1.055420 max_load_reduction=56890 max_load_reduction_pct=0.868073 moved_to_echo=32787 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 1280, 2944, 16896, 7040, 8704, 14592, 14464, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6400, 4096, 384, 19712, 0, 5760, 3584, 16896, 5760, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4736, 16000, 256, 1408, 6912, 16256, 128, 4352, 16128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 9088, 13056, 15744, 256, 22528, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 25216, 19072, 128, 1920, 13056, 6400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6400, 1920, 16000, 3584, 2944, 256, 24832, 6528, 3712, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[28416, 3328, 23808, 128, 128, 384, 128, 3584, 6400, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[7296, 128, 512, 128, 512, 128, 27008, 128, 29056, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 10, 4, 0, 4096, 0, 3910, 0, 0, 0, 51, 2, 0, 0, 0, 0, 0, 0, 1, 0, 1, 10, 4094, 4096, 0, 0, 4092, 0, 0, 8, 0, 4095, 2246, 0, 2151, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 4096, 3902, 1, 0, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 3, 2, 0, 0, 4094, 0, 4072, 0, 0, 0, 50, 28, 1, 0, 0, 1, 0, 1, 8, 0, 0, 61, 4094, 4094, 4, 0, 4096, 3, 2, 17, 1, 4075, 1359, 0, 2783, 4095, 1, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4094, 1, 4095, 0, 4095, 4095, 3940, 0, 1, 0, 1, 4076] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 3643, 0, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 9, 0, 0, 32, 4087, 4096, 1, 0, 4094, 0, 0, 8, 0, 4091, 1673, 0, 2965, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 4095, 0, 4096, 4096, 3952, 2, 0, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 5, 0, 2, 4092, 13, 5, 11, 0, 4083, 0, 3291, 18, 0, 1, 40, 0, 2, 1, 2, 1, 4, 0, 0, 0, 1, 151, 4086, 4089, 0, 0, 4092, 0, 10, 8, 6, 4072, 2488, 46, 2409, 4094, 2, 0, 10, 2, 1, 0, 4084, 0, 0, 8, 0, 4090, 7, 4090, 0, 4088, 4085, 3889, 1, 1, 1, 2, 4052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 5, 0, 4096, 0, 3825, 0, 0, 0, 4, 21, 0, 0, 0, 0, 0, 0, 0, 0, 5, 107, 4094, 4096, 0, 0, 4093, 0, 0, 3, 0, 4080, 1081, 0, 3509, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4094, 4096, 3754, 0, 0, 0, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 5, 0, 3, 0, 4095, 0, 4004, 0, 0, 0, 235, 2, 0, 0, 0, 0, 0, 1, 5, 6, 0, 5, 4096, 4092, 1, 0, 4095, 1, 0, 1, 0, 4067, 742, 0, 3292, 4096, 2, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 0, 4094, 1, 4095, 4092, 4031, 0, 0, 0, 1, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4092, 1, 3, 11, 0, 4092, 1, 1851, 0, 2, 0, 20, 18, 0, 0, 2, 0, 0, 0, 2, 8, 2, 36, 4089, 4096, 3, 0, 4091, 0, 0, 5, 0, 4095, 3526, 41, 2813, 4083, 1, 2, 0, 1, 0, 0, 4092, 0, 1, 0, 0, 4096, 0, 4087, 0, 4096, 4094, 3991, 3, 0, 2, 0, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4091, 1, 9, 0, 11, 4094, 0, 3821, 0, 2, 0, 11, 7, 0, 0, 5, 0, 0, 0, 1, 0, 5, 61, 4090, 4095, 0, 0, 4089, 0, 0, 28, 3, 4093, 1980, 3, 2332, 4091, 0, 0, 5, 0, 0, 0, 4094, 0, 3, 0, 0, 4089, 1, 4089, 0, 4094, 4093, 4052, 0, 0, 4, 0, 4084] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 10, 4, 3928, 147, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 3910, 0, 0, 0, 51, 438, 50, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 1, 10, 4094, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3510, 2246, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 168, 3902, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8545, 8182, 8144, 7915, 8193, 8194, 8166] avg=8192.0000 max=8545 max_rank=1 min=7915 min_rank=4 imbalance_ratio=1.043091 max_load_reduction=56991 max_load_reduction_pct=0.869614 moved_to_echo=32468 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 3928, 146, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 3643, 0, 0, 0, 20, 438, 50, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 9, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4087, 0, 1, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3507, 1673, 0, 2965, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 168, 3952, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8247, 8191, 8161, 8153, 8193, 8192, 8216] avg=8192.0000 max=8247 max_rank=1 min=8153 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 3, 2, 0, 3927, 146, 12, 0, 0, 0, 0, 0, 0, 4094, 0, 4072, 0, 0, 0, 50, 436, 50, 0, 0, 0, 0, 0, 0, 28, 1, 0, 0, 1, 0, 1, 8, 4094, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4094, 0, 4, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 17, 1, 3493, 1359, 0, 2783, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 1, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 168, 3940, 0, 1, 0, 1, 4076, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8702, 8216, 8202, 7658, 8194, 8192, 8186] avg=8192.0000 max=8702 max_rank=1 min=7658 min_rank=4 imbalance_ratio=1.062256 max_load_reduction=56834 max_load_reduction_pct=0.867218 moved_to_echo=32230 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 2, 4092, 13, 5, 11, 3918, 146, 12, 0, 0, 0, 0, 0, 0, 4083, 0, 3291, 18, 0, 1, 40, 436, 50, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 1, 4, 0, 0, 4089, 4076, 0, 0, 0, 0, 0, 0, 0, 1, 151, 4086, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8, 6, 3490, 2488, 46, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 10, 2, 1, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4090, 7, 4090, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 167, 3889, 1, 1, 1, 2, 4052, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 7919, 8175, 8277, 8457, 8194, 8197, 8113] avg=8192.0000 max=8457 max_rank=4 min=7919 min_rank=1 imbalance_ratio=1.032349 max_load_reduction=57079 max_load_reduction_pct=0.870956 moved_to_echo=32961 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 5, 3928, 146, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 3825, 0, 0, 0, 4, 437, 50, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 5, 107, 4094, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3497, 1081, 0, 3509, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 168, 3754, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8412, 8199, 8246, 8090, 8193, 8194, 8015] avg=8192.0000 max=8412 max_rank=1 min=8015 min_rank=7 imbalance_ratio=1.026855 max_load_reduction=57124 max_load_reduction_pct=0.871643 moved_to_echo=32492 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 5, 0, 3, 3924, 146, 12, 0, 0, 0, 0, 0, 0, 4095, 0, 4004, 0, 0, 0, 235, 435, 50, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 5, 4092, 4083, 0, 0, 0, 0, 0, 0, 6, 0, 5, 4096, 0, 1, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3486, 742, 0, 3292, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 168, 4031, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8819, 8183, 8150, 7522, 8196, 8192, 8287] avg=8192.0000 max=8819 max_rank=1 min=7522 min_rank=4 imbalance_ratio=1.076538 max_load_reduction=56717 max_load_reduction_pct=0.865433 moved_to_echo=32197 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4092, 1, 3, 11, 3926, 147, 13, 0, 0, 0, 0, 0, 0, 4092, 1, 1851, 0, 2, 0, 20, 438, 50, 0, 0, 0, 0, 0, 0, 18, 0, 0, 2, 0, 0, 0, 2, 4096, 4083, 0, 0, 0, 0, 0, 0, 8, 2, 36, 4089, 0, 3, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3510, 3526, 41, 2813, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 2, 0, 1, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4087, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 168, 3991, 3, 0, 2, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 6454, 8201, 8176, 9895, 8180, 8186, 8250] avg=8192.0000 max=9895 max_rank=4 min=6454 min_rank=1 imbalance_ratio=1.207886 max_load_reduction=55641 max_load_reduction_pct=0.849014 moved_to_echo=34511 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4091, 1, 9, 0, 3930, 149, 13, 0, 0, 0, 0, 0, 11, 4094, 0, 3821, 0, 2, 0, 11, 444, 55, 0, 0, 0, 0, 0, 0, 7, 0, 0, 5, 0, 0, 0, 1, 4095, 4081, 0, 0, 0, 0, 0, 0, 0, 5, 61, 4090, 0, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3, 3500, 1980, 3, 2332, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 5, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4089, 1, 4089, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 163, 4052, 0, 0, 4, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8438, 8189, 8180, 7846, 8193, 8189, 8303] avg=8192.0000 max=8438 max_rank=1 min=7846 min_rank=4 imbalance_ratio=1.030029 max_load_reduction=57098 max_load_reduction_pct=0.871246 moved_to_echo=32531 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 1408, 31616, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 128, 31488, 1280, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 512, 32768, 0, 128, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 128, 28032, 15104, 128, 22272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 28544, 128, 128, 128, 512, 3584, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4, 0, 0, 4092, 2030, 0, 4095, 0, 4096, 4048, 0, 0, 0, 0, 0, 91, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4022, 0, 2422, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 3802, 4061, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 7, 0, 0, 4089, 1387, 0, 4096, 0, 4094, 4068, 0, 0, 0, 0, 0, 220, 1, 0, 4096, 1, 4096, 0, 0, 0, 4096, 0, 0, 3836, 0, 3459, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 3439, 4069, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 7, 0, 4093, 1, 1, 4094, 0, 1, 63, 1, 1, 4094, 1615, 1, 4076, 0, 4090, 4079, 0, 0, 0, 0, 0, 153, 0, 0, 4095, 3, 4095, 1, 1, 0, 4095, 0, 1, 4087, 0, 3306, 0, 4095, 1, 1, 1, 2, 1, 4095, 0, 1, 3094, 4095, 0, 0, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 73, 0, 0, 4051, 2150, 0, 4089, 0, 4096, 3938, 0, 0, 0, 0, 0, 80, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4026, 0, 3078, 0, 4096, 0, 0, 0, 12, 0, 4096, 0, 0, 3085, 4081, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 24, 0, 0, 4072, 1094, 0, 4090, 7, 4096, 4001, 0, 0, 0, 0, 0, 86, 0, 5, 4096, 0, 4096, 0, 0, 6, 4087, 12, 0, 3704, 0, 3775, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 3750, 4056, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4, 0, 5, 0, 0, 0, 4091, 0, 5, 0, 35, 0, 4092, 3, 3, 4085, 0, 1, 1, 3, 2, 4074, 1083, 0, 4082, 1, 4080, 3915, 6, 0, 0, 0, 0, 50, 12, 0, 4091, 10, 4093, 2, 3, 1, 4090, 13, 2, 3680, 0, 3660, 4, 4093, 2, 4, 3, 0, 3, 4092, 0, 5, 3947, 4088, 0, 2, 14, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 1, 4095, 1, 0, 14, 0, 0, 4090, 1880, 0, 4091, 0, 4096, 3849, 0, 0, 1, 1, 3, 427, 2, 1, 4096, 0, 4095, 0, 0, 0, 4095, 2, 0, 3933, 0, 3051, 0, 4095, 0, 1, 1, 5, 1, 4095, 0, 0, 3246, 4074, 1, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 5, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 63, 0, 0, 4085, 2058, 1, 3979, 11, 4094, 4008, 0, 0, 0, 0, 0, 396, 0, 6, 4096, 3, 4096, 0, 0, 0, 4088, 3, 0, 3914, 0, 2315, 0, 4091, 0, 0, 0, 30, 0, 4094, 0, 0, 3720, 4089, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 4096, 0, 3970, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4092, 2030, 0, 2418, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 126, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 4096, 0, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 0, 4022, 0, 2422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3802, 4061, 0, 0, 0, 0, 397, 192, 6, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8188, 8544, 8269, 8091, 7599, 8187, 8462] avg=8192.0000 max=8544 max_rank=2 min=7599 min_rank=5 imbalance_ratio=1.042969 max_load_reduction=56992 max_load_reduction_pct=0.869629 moved_to_echo=32339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3968, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4089, 1387, 0, 2418, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 126, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 1, 0, 4096, 1, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 0, 3836, 0, 3459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 4069, 0, 0, 1, 0, 397, 192, 6, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8188, 7901, 8290, 8222, 8450, 8187, 8108] avg=8192.0000 max=8450 max_rank=5 min=7901 min_rank=2 imbalance_ratio=1.031494 max_load_reduction=57086 max_load_reduction_pct=0.871063 moved_to_echo=32967 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4093, 0, 3964, 126, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4089, 1, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 1, 1, 4094, 1615, 1, 2417, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 126, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 4095, 3, 3903, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1155, 0, 1, 4087, 0, 3306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 2, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3094, 4095, 0, 0, 1, 1, 397, 192, 6, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8192, 8193, 8281, 8155, 8550, 8190, 7791] avg=8192.0000 max=8550 max_rank=5 min=7791 min_rank=7 imbalance_ratio=1.043701 max_load_reduction=56986 max_load_reduction_pct=0.869537 moved_to_echo=32686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 5, 0, 0, 0, 4091, 0, 3954, 126, 0, 0, 0, 0, 0, 0, 5, 0, 35, 0, 4088, 3, 3, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 2, 4074, 1083, 0, 2414, 0, 0, 0, 0, 0, 0, 0, 4082, 1, 126, 3915, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 12, 0, 4091, 10, 3901, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1154, 13, 2, 3680, 0, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4093, 2, 4, 3, 0, 3, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3947, 4088, 0, 2, 14, 1, 396, 192, 6, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8219, 7578, 8130, 8066, 8513, 8195, 8655] avg=8192.0000 max=8655 max_rank=7 min=7578 min_rank=2 imbalance_ratio=1.056519 max_load_reduction=56881 max_load_reduction_pct=0.867935 moved_to_echo=33429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 4096, 0, 3970, 126, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 4051, 2150, 0, 2418, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 126, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 4096, 0, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 0, 4026, 0, 3078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3085, 4081, 0, 0, 0, 0, 397, 192, 6, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8189, 8692, 8153, 8080, 8259, 8198, 7765] avg=8192.0000 max=8692 max_rank=2 min=7765 min_rank=7 imbalance_ratio=1.061035 max_load_reduction=56844 max_load_reduction_pct=0.867371 moved_to_echo=32302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 4095, 0, 3970, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 4090, 1880, 0, 2417, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 126, 3849, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 427, 2, 1, 4096, 0, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 2, 0, 3933, 0, 3051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 5, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3246, 4074, 1, 0, 2, 0, 397, 192, 6, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8187, 8402, 8068, 8432, 8141, 8192, 7922] avg=8192.0000 max=8432 max_rank=4 min=7922 min_rank=7 imbalance_ratio=1.029297 max_load_reduction=57104 max_load_reduction_pct=0.871338 moved_to_echo=32687 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 3970, 126, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4072, 1094, 0, 2412, 0, 0, 0, 0, 0, 0, 0, 4090, 7, 126, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 5, 4096, 0, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1153, 12, 0, 3704, 0, 3775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 4056, 0, 0, 0, 0, 396, 192, 6, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8189, 7602, 8224, 8091, 8650, 8184, 8404] avg=8192.0000 max=8650 max_rank=5 min=7602 min_rank=2 imbalance_ratio=1.055908 max_load_reduction=56886 max_load_reduction_pct=0.868011 moved_to_echo=33329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 5, 4096, 0, 3971, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 4085, 2058, 1, 2417, 0, 0, 0, 0, 0, 0, 0, 3979, 11, 123, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 6, 4096, 3, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 3, 0, 3914, 0, 2315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 30, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 4089, 0, 0, 3, 0, 400, 198, 12, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8184, 8624, 8121, 8399, 7374, 8203, 8429] avg=8192.0000 max=8624 max_rank=2 min=7374 min_rank=5 imbalance_ratio=1.052734 max_load_reduction=56912 max_load_reduction_pct=0.868408 moved_to_echo=32405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 9344, 128, 128, 31232, 0, 25088, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 1536, 128, 128, 32768, 128, 31232, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 256, 128, 128, 32768, 13312, 128, 19456, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 1024, 32000, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 0, 31744, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 28160, 32640, 128, 128, 128, 128, 3200, 1664, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 0, 0, 0, 4096, 0, 26, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 4077, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4088, 4096, 4095, 4096, 0, 2, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 1, 4096, 0, 20, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4079, 0, 0, 0, 4095, 1, 0, 0, 4095, 4085, 4095, 1, 0, 0, 4095, 0, 4096, 0, 1, 0, 4096, 4096, 0, 0, 4090, 4095, 4094, 4096, 0, 12, 4095, 1, 1, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 4096, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 4, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 2, 0, 1, 1, 15, 4, 3, 0, 1, 1, 4092, 0, 0, 4092, 0, 2, 0, 1, 1, 0, 0, 0, 1, 0, 4082, 0, 1, 0, 4095, 0, 1, 1, 4095, 4095, 4095, 0, 0, 0, 4095, 1, 4095, 0, 0, 0, 4095, 4095, 0, 1, 4091, 4093, 4095, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 4096, 0, 63, 4092, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 4033, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4050, 4096, 4090, 4096, 0, 89, 4096, 7, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4, 7, 0, 1, 0, 5, 7, 18, 0, 1, 0, 3, 0, 4077, 0, 0, 4088, 0, 0, 4, 6, 5, 4, 10, 0, 1, 0, 4071, 1, 5, 2, 4092, 0, 5, 6, 4091, 4092, 4085, 0, 1, 8, 4075, 4, 4089, 0, 0, 0, 4089, 4091, 0, 5, 4090, 4069, 4091, 4089, 2, 20, 4088, 27, 2, 0, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 16, 0, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 31, 4090, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4096, 5, 0, 0, 4096, 4059, 4096, 0, 0, 0, 3998, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4091, 4096, 4095, 4096, 0, 100, 4092, 5, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 47, 4088, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 4089, 6, 0, 0, 4096, 4019, 4096, 1, 0, 0, 3977, 0, 4096, 0, 0, 0, 4096, 4096, 5, 0, 4085, 4096, 4091, 4096, 0, 192, 4088, 8, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 11, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 26, 4073, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 4080, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 1, 0, 0, 0, 0, 0, 4031, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8197, 8190, 8203, 8206, 8214, 8198, 8138] avg=8192.0000 max=8214 max_rank=5 min=8138 min_rank=7 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 4096, 0, 1, 4074, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4080, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 1, 0, 0, 0, 0, 0, 4031, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8175, 8188, 8213, 8206, 8229, 8206, 8140] avg=8192.0000 max=8229 max_rank=5 min=8140 min_rank=7 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32781 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 2, 0, 1, 1, 15, 4095, 4082, 0, 0, 0, 0, 0, 0, 4, 3, 0, 1, 1, 4092, 0, 0, 4073, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 1, 1, 0, 0, 4079, 13, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4082, 0, 1, 0, 4095, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 15, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4095, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 4093, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4030, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8175, 8188, 8203, 8207, 8232, 8199, 8134] avg=8192.0000 max=8232 max_rank=5 min=8134 min_rank=7 imbalance_ratio=1.004883 max_load_reduction=57304 max_load_reduction_pct=0.874390 moved_to_echo=32772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 3, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 20, 4072, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4079, 13, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4079, 0, 0, 0, 4095, 24, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 15, 4085, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 1, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4095, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4095, 1, 1, 0, 1, 0, 0, 4031, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8190, 8186, 8199, 8197, 8233, 8199, 8149] avg=8192.0000 max=8233 max_rank=5 min=8149 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 7, 0, 1, 0, 5, 7, 18, 4091, 4076, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4077, 0, 0, 4069, 1, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4, 6, 5, 4, 10, 4075, 13, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4071, 1, 5, 2, 4092, 24, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 15, 4092, 4085, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4075, 4, 4089, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4090, 4069, 14, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4088, 27, 2, 0, 0, 0, 5, 4024, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 8151, 8205, 8196, 8204, 8217, 8180, 8174] avg=8192.0000 max=8217 max_rank=5 min=8151 min_rank=1 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32775 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 63, 4068, 1, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 9, 0, 0, 0, 0, 4080, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 4096, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4096, 7, 0, 0, 0, 0, 0, 4031, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8229, 8194, 8165, 8206, 8170, 8160, 8231] avg=8192.0000 max=8231 max_rank=7 min=8160 min_rank=6 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 16, 0, 0, 12, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 31, 4073, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 0, 0, 4080, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 15, 4059, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 4096, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 4092, 5, 0, 0, 0, 0, 0, 4031, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8201, 8188, 8192, 8175, 8135, 8201, 8236] avg=8192.0000 max=8236 max_rank=7 min=8135 min_rank=5 imbalance_ratio=1.005371 max_load_reduction=57300 max_load_reduction_pct=0.874329 moved_to_echo=32747 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 47, 4073, 2, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 16, 0, 0, 0, 0, 4081, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 4089, 28, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 13, 4019, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 4096, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4085, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 4088, 8, 0, 0, 0, 0, 0, 4035, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8218, 8197, 8165, 8135, 8106, 8193, 8334] avg=8192.0000 max=8334 max_rank=7 min=8106 min_rank=5 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32768 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32640, 128, 128, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32512, 128, 32768, 0, 128, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 32768, 128, 128, 0, 128, 0, 128, 32256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32640, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 32768, 0, 256, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4070, 0, 0, 0, 2819, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 1228, 0, 0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[2, 4095, 0, 0, 0, 0, 1, 1, 4095, 0, 4094, 1, 1, 1, 0, 0, 0, 0, 1, 0, 4095, 1, 4092, 1, 4094, 4089, 0, 0, 0, 3452, 0, 0, 2, 4095, 1, 0, 4095, 4095, 1, 1, 4092, 4095, 4095, 1, 0, 1, 0, 0, 0, 108, 1, 0, 0, 0, 543, 1, 0, 1, 4095, 4095, 0, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4064, 0, 0, 0, 2889, 0, 0, 0, 4096, 0, 0, 4096, 4096, 15, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 1013, 0, 0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4092, 0, 4096, 3974, 0, 1, 1, 2978, 0, 0, 3, 4096, 4, 0, 4095, 4096, 1, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 322, 0, 0, 7, 0, 918, 0, 0, 0, 4095, 4089, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 4096, 4020, 0, 0, 0, 1194, 0, 0, 20, 4096, 0, 0, 4096, 4096, 103, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 2801, 0, 0, 0, 4096, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[6, 4090, 3, 0, 0, 0, 6, 5, 4092, 0, 4089, 6, 6, 5, 0, 1, 0, 0, 5, 0, 4090, 4, 4079, 4, 4090, 4073, 0, 1, 8, 2376, 0, 0, 18, 4089, 0, 0, 4090, 4090, 18, 2, 4082, 4085, 4088, 6, 0, 5, 6, 0, 3, 9, 4, 0, 1, 0, 1700, 4, 0, 7, 4091, 4087, 0, 6, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4079, 0, 4096, 4050, 0, 0, 2, 2217, 0, 0, 25, 4096, 16, 0, 4096, 4096, 12, 0, 4096, 4091, 4096, 0, 0, 0, 0, 0, 0, 197, 0, 0, 10, 0, 1716, 0, 0, 0, 4094, 4072, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4052, 0, 4091, 4029, 0, 0, 1, 1885, 0, 0, 66, 4096, 14, 4, 4096, 4096, 24, 0, 4096, 4090, 4096, 0, 0, 0, 0, 0, 0, 143, 0, 0, 13, 0, 2085, 0, 0, 0, 4096, 4082, 0, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2427, 1642, 25, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1669, 4070, 0, 0, 0, 2819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 1228, 0, 4096, 2450, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8193, 8202, 8558, 8152, 8196, 7849, 8196] avg=8192.0000 max=8558 max_rank=3 min=7849 min_rank=6 imbalance_ratio=1.044678 max_load_reduction=56978 max_load_reduction_pct=0.869415 moved_to_echo=32393 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2427, 1641, 25, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1669, 4064, 0, 0, 0, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4056, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 0, 1013, 0, 4096, 2450, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8193, 8202, 8622, 8167, 8196, 7771, 8196] avg=8192.0000 max=8622 max_rank=3 min=7771 min_rank=6 imbalance_ratio=1.052490 max_load_reduction=56914 max_load_reduction_pct=0.868439 moved_to_echo=32330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 4095, 0, 0, 0, 0, 1, 1, 2426, 1641, 25, 0, 0, 0, 0, 0, 4095, 0, 4094, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 4092, 1, 10, 0, 0, 0, 0, 0, 0, 0, 1668, 4089, 0, 0, 0, 3452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 1, 0, 4055, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4095, 4, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 1, 0, 0, 0, 543, 1, 4095, 2450, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4095, 0, 1, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8193, 8200, 9209, 8155, 8193, 7198, 8197] avg=8192.0000 max=9209 max_rank=3 min=7198 min_rank=6 imbalance_ratio=1.124146 max_load_reduction=56327 max_load_reduction_pct=0.859482 moved_to_echo=31743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 0, 0, 0, 0, 2427, 1642, 25, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4092, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1669, 3974, 0, 1, 1, 2978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4, 0, 4055, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 7, 0, 918, 0, 4096, 2450, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4089, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8189, 8197, 8623, 8159, 8195, 7793, 8189] avg=8192.0000 max=8623 max_rank=3 min=7793 min_rank=6 imbalance_ratio=1.052612 max_load_reduction=56913 max_load_reduction_pct=0.868423 moved_to_echo=32336 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 4090, 3, 0, 0, 0, 6, 5, 2423, 1639, 25, 0, 0, 0, 0, 0, 4092, 0, 4089, 6, 6, 5, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4090, 4, 4079, 4, 10, 0, 0, 0, 0, 0, 0, 0, 1667, 4073, 0, 1, 8, 2376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4089, 0, 0, 4050, 0, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4085, 3, 6, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 4, 0, 1, 0, 1700, 4, 4090, 2446, 0, 0, 0, 0, 0, 0, 0, 7, 4091, 4087, 0, 6, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8200, 8192, 8125, 8177, 8187, 8257, 8201] avg=8192.0000 max=8257 max_rank=6 min=8125 min_rank=3 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2427, 1642, 25, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1669, 4020, 0, 0, 0, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 0, 0, 4056, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 2801, 0, 4096, 2450, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8193, 8197, 6883, 8275, 8196, 9407, 8195] avg=8192.0000 max=9407 max_rank=6 min=6883 min_rank=3 imbalance_ratio=1.148315 max_load_reduction=56129 max_load_reduction_pct=0.856461 moved_to_echo=34073 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 2427, 1642, 25, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4079, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1669, 4050, 0, 0, 2, 2217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 16, 0, 4056, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 10, 0, 1716, 0, 4096, 2454, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4072, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8187, 8185, 7938, 8205, 8187, 8473, 8171] avg=8192.0000 max=8473 max_rank=6 min=7938 min_rank=3 imbalance_ratio=1.034302 max_load_reduction=57063 max_load_reduction_pct=0.870712 moved_to_echo=33036 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 0, 0, 0, 2428, 1645, 28, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4052, 0, 14, 0, 0, 0, 0, 0, 0, 0, 1663, 4029, 0, 0, 1, 1885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 4096, 14, 4, 4042, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 13, 0, 2085, 0, 4096, 2451, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 5, 0, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8188, 8161, 7578, 8246, 8186, 8788, 8191] avg=8192.0000 max=8788 max_rank=6 min=7578 min_rank=3 imbalance_ratio=1.072754 max_load_reduction=56748 max_load_reduction_pct=0.865906 moved_to_echo=33411 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 128, 128, 19456, 13184, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 1152, 128, 0, 128, 0, 12032, 128, 32768, 19712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[13440, 32384, 0, 128, 128, 19840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 32768, 128, 128, 32512, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 32768, 128, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 164, 0, 0, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 3924, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4095, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 0, 0, 74, 0, 0, 4096, 4096, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4096, 4083, 4096, 0, 0, 0, 4096, 0, 0, 4022, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 0, 0, 1, 0, 5, 0, 0, 0, 1, 484, 0, 0, 4095, 4095, 1, 1, 0, 4078, 0, 0, 1, 4095, 0, 31, 1, 1, 1, 1, 4095, 4095, 0, 0, 1, 0, 4095, 4061, 4095, 1, 0, 0, 4095, 0, 1, 3612, 4094, 1, 1, 0, 4095, 0, 0, 4095, 1, 8, 1, 1, 8, 0, 4092, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 4096, 4096, 0, 0, 0, 4090, 0, 6, 0, 4096, 0, 8, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 4096, 4072, 4096, 0, 0, 0, 4096, 0, 0, 3928, 4094, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 1, 1, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 16, 0, 0, 0, 0, 89, 0, 0, 4096, 4096, 0, 0, 0, 4081, 0, 14, 0, 4096, 0, 11, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 4096, 4039, 4096, 0, 0, 0, 4096, 0, 0, 4068, 4046, 0, 0, 0, 4096, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 4088, 3, 0, 4, 0, 17, 2, 0, 0, 7, 30, 1, 0, 4086, 4088, 7, 2, 0, 4062, 6, 18, 7, 4089, 4, 30, 1, 6, 4, 0, 4081, 4089, 6, 1, 5, 0, 4087, 4060, 4087, 7, 4, 3, 4089, 0, 8, 4053, 4087, 8, 3, 0, 4089, 1, 3, 4089, 6, 0, 7, 7, 11, 0, 4081, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 155, 0, 0, 4096, 4096, 0, 0, 0, 4081, 0, 11, 0, 4096, 0, 22, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 4096, 4069, 4096, 0, 0, 0, 4096, 0, 0, 3953, 4090, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 9, 1, 4096, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 4096, 4096, 0, 0, 0, 4077, 0, 13, 0, 4096, 0, 22, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 4096, 4024, 4096, 0, 0, 0, 4096, 7, 0, 4077, 4079, 0, 0, 0, 4090, 0, 0, 4096, 0, 1, 0, 0, 13, 2, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 12, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 4096, 3930, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 4022, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4063, 17, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8107, 8192, 8184, 8212, 8267, 8192, 8185] avg=8192.0000 max=8267 max_rank=5 min=8107 min_rank=1 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 7, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 4096, 3930, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 3924, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 4063, 17, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8197, 8194, 8183, 8225, 8169, 8192, 8184] avg=8192.0000 max=8225 max_rank=4 min=8169 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4095, 0, 0, 1, 0, 5, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 484, 0, 0, 4095, 3929, 7, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 4078, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 31, 1, 1, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4061, 33, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 149, 0, 1, 3612, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 1, 8, 0, 4092, 0, 0, 4062, 17, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8516, 8180, 8214, 8190, 7857, 8193, 8196] avg=8192.0000 max=8516 max_rank=1 min=7857 min_rank=5 imbalance_ratio=1.039551 max_load_reduction=57020 max_load_reduction_pct=0.870056 moved_to_echo=32436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 4096, 3930, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 6, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 8, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 3928, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 0, 0, 4063, 17, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8226, 8196, 8181, 8201, 8171, 8191, 8185] avg=8192.0000 max=8226 max_rank=1 min=8171 min_rank=5 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32748 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4088, 3, 0, 4, 0, 17, 4082, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 30, 1, 0, 4086, 3923, 7, 0, 0, 0, 0, 0, 0, 4088, 7, 2, 0, 4062, 6, 18, 7, 4, 0, 0, 0, 0, 0, 0, 0, 4078, 4, 30, 1, 6, 4, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 5, 0, 4087, 4060, 33, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 3, 149, 0, 8, 4053, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 4089, 1, 3, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 11, 0, 4081, 0, 0, 4054, 17, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8056, 8194, 8204, 8192, 8311, 8199, 8184] avg=8192.0000 max=8311 max_rank=5 min=8056 min_rank=1 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32886 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 16, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 4096, 3930, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4081, 0, 14, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 11, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4039, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 4068, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 4096, 0, 0, 4063, 17, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8122, 8195, 8190, 8168, 8263, 8192, 8205] avg=8192.0000 max=8263 max_rank=5 min=8122 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 3, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 4096, 3930, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4081, 0, 11, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 22, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 3953, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 4096, 0, 1, 4063, 17, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8188, 8192, 8193, 8198, 8192, 8191, 8193] avg=8192.0000 max=8198 max_rank=4 min=8188 min_rank=1 imbalance_ratio=1.000732 max_load_reduction=57338 max_load_reduction_pct=0.874908 moved_to_echo=32774 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 4096, 3933, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4077, 0, 13, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 22, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4024, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 7, 0, 4077, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 13, 2, 4096, 0, 0, 4066, 20, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8124, 8193, 8187, 8150, 8306, 8186, 8204] avg=8192.0000 max=8306 max_rank=5 min=8124 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 0, 32768, 32512, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 128, 32512, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 256, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 1280, 128, 128, 31744, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 1280, 128, 0, 32768, 31488, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2664, 0, 0, 4096, 0, 4091, 4096, 980, 0, 0, 225, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 4094, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 3026, 0, 0, 0, 1339, 0, 2, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1869, 0, 0, 4095, 0, 4096, 4096, 2301, 0, 0, 146, 0, 3828, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4079, 0, 2, 0, 0, 4090, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 3484, 0, 0, 0, 681, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[8, 0, 10, 7, 2034, 8, 0, 4090, 0, 4089, 4088, 2047, 5, 0, 2, 7, 3347, 0, 6, 0, 0, 0, 2, 0, 0, 1, 0, 2, 6, 4087, 7, 0, 4, 5, 0, 7, 7, 4079, 4090, 0, 2, 4074, 2, 2, 3, 2, 4087, 7, 4082, 0, 4089, 6, 0, 4088, 4087, 3418, 0, 6, 6, 1433, 4, 8, 0, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1739, 0, 0, 4096, 0, 4096, 4096, 1802, 0, 0, 134, 0, 3860, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4071, 0, 0, 0, 0, 4082, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 2627, 0, 0, 0, 2157, 0, 7, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2618, 0, 5, 4084, 0, 4080, 4096, 1389, 0, 0, 64, 0, 3985, 0, 0, 2, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4094, 4088, 0, 0, 4085, 3, 0, 0, 0, 4035, 0, 4091, 0, 4091, 0, 0, 4096, 4096, 3741, 1, 0, 0, 576, 0, 13, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 2, 1, 2891, 0, 0, 4095, 0, 4095, 4095, 1112, 0, 1, 442, 1, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 1, 1, 0, 1, 1, 4095, 4095, 0, 0, 4088, 0, 1, 0, 0, 4076, 1, 4089, 0, 4095, 1, 0, 4095, 4095, 2566, 0, 1, 1, 1211, 0, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2866, 0, 1, 4088, 0, 4086, 4096, 792, 0, 0, 297, 0, 4033, 1, 0, 3, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4095, 4092, 0, 0, 4028, 1, 0, 0, 0, 4087, 0, 4096, 1, 4095, 0, 0, 4096, 4096, 2351, 0, 0, 0, 2033, 0, 4, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1676, 0, 5, 4086, 0, 4076, 4096, 801, 0, 0, 193, 0, 4021, 0, 0, 6, 0, 0, 21, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 4096, 4088, 0, 0, 4096, 5, 1, 0, 0, 4094, 0, 4091, 0, 4095, 0, 0, 4096, 4091, 2898, 1, 0, 0, 2698, 0, 5, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2664, 0, 0, 4096, 1552, 247, 0, 0, 0, 0, 0, 0, 0, 4091, 2513, 980, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 4096, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3910, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4094, 0, 30, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1087, 0, 0, 0, 0, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1339, 0, 2, 0, 4096, 2575, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8559, 7809, 8342, 8193, 8193, 8219, 8209, 8012] avg=8192.0000 max=8559 max_rank=0 min=7809 min_rank=1 imbalance_ratio=1.044800 max_load_reduction=56977 max_load_reduction_pct=0.869400 moved_to_echo=32633 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1869, 0, 0, 4095, 1552, 247, 0, 0, 0, 0, 0, 0, 0, 4096, 2513, 2301, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 1, 0, 0, 0, 0, 0, 0, 4096, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3909, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 2, 0, 0, 4090, 0, 30, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1087, 0, 0, 0, 0, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 681, 0, 0, 0, 4096, 2575, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7763, 9056, 8112, 8192, 8193, 8201, 8667, 7352] avg=8192.0000 max=9056 max_rank=1 min=7352 min_rank=7 imbalance_ratio=1.105469 max_load_reduction=56480 max_load_reduction_pct=0.861816 moved_to_echo=32413 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 1, 2891, 0, 0, 4095, 1552, 247, 0, 0, 0, 0, 0, 0, 0, 4095, 2512, 1112, 0, 1, 442, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 4095, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 3909, 187, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4095, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 1, 0, 0, 4076, 1, 30, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1087, 1, 0, 0, 0, 2566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1211, 0, 0, 0, 4095, 2574, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8789, 8163, 8376, 8192, 8195, 8196, 7743, 7882] avg=8192.0000 max=8789 max_rank=0 min=7743 min_rank=6 imbalance_ratio=1.072876 max_load_reduction=56747 max_load_reduction_pct=0.865891 moved_to_echo=32016 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 10, 7, 2034, 8, 0, 4090, 1549, 247, 0, 0, 0, 0, 0, 0, 0, 4089, 2508, 2047, 5, 0, 2, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3347, 0, 6, 0, 0, 0, 2, 0, 4088, 186, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 6, 4087, 7, 0, 3901, 187, 0, 0, 0, 0, 0, 0, 4, 5, 0, 7, 7, 4079, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 4074, 2, 2, 3, 2, 4087, 7, 30, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 1085, 6, 0, 0, 0, 3418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 1433, 4, 8, 0, 4085, 2570, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7953, 8658, 7629, 8191, 8193, 8209, 8591, 8112] avg=8192.0000 max=8658 max_rank=1 min=7629 min_rank=2 imbalance_ratio=1.056885 max_load_reduction=56878 max_load_reduction_pct=0.867889 moved_to_echo=33105 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1739, 0, 0, 4096, 1552, 247, 0, 0, 0, 0, 0, 0, 0, 4096, 2513, 1802, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 1, 0, 4096, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3910, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 4082, 0, 30, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1087, 0, 0, 0, 0, 2627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2157, 0, 7, 0, 4096, 2575, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7634, 8545, 8143, 8193, 8193, 8183, 7810, 8835] avg=8192.0000 max=8835 max_rank=7 min=7634 min_rank=0 imbalance_ratio=1.078491 max_load_reduction=56701 max_load_reduction_pct=0.865189 moved_to_echo=33021 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2866, 0, 1, 4088, 1552, 247, 0, 0, 0, 0, 0, 0, 0, 4086, 2513, 792, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 1, 0, 3, 0, 0, 5, 0, 4096, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3910, 187, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 1, 0, 0, 0, 4087, 0, 30, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1087, 0, 0, 0, 0, 2351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2033, 0, 4, 0, 4096, 2574, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8754, 7688, 8324, 8193, 8189, 8146, 7535, 8707] avg=8192.0000 max=8754 max_rank=0 min=7535 min_rank=6 imbalance_ratio=1.068604 max_load_reduction=56782 max_load_reduction_pct=0.866425 moved_to_echo=32577 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2618, 0, 5, 4084, 1552, 247, 0, 0, 0, 0, 0, 0, 0, 4080, 2513, 1389, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 2, 0, 0, 9, 0, 4096, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3909, 187, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 4088, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 3, 0, 0, 0, 4035, 0, 30, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1085, 0, 0, 0, 0, 3741, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 576, 0, 13, 0, 4096, 2572, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8506, 8046, 8279, 8192, 8185, 8153, 8917, 7258] avg=8192.0000 max=8917 max_rank=6 min=7258 min_rank=7 imbalance_ratio=1.088501 max_load_reduction=56619 max_load_reduction_pct=0.863937 moved_to_echo=32513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1676, 0, 5, 4086, 1559, 252, 0, 0, 0, 0, 0, 0, 0, 4076, 2501, 801, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 6, 0, 0, 21, 0, 4096, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3904, 190, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 4088, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 1, 0, 0, 4094, 0, 33, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1075, 0, 0, 0, 0, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2698, 0, 5, 0, 4096, 2578, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7578, 7571, 8331, 8190, 8195, 8229, 8064, 9378] avg=8192.0000 max=9378 max_rank=7 min=7571 min_rank=1 imbalance_ratio=1.144775 max_load_reduction=56158 max_load_reduction_pct=0.856903 moved_to_echo=33866 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 12160, 128, 128, 0, 32768, 20608, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 32768, 128, 0, 31360, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31232, 128, 128, 128, 0, 0, 128, 0, 32768, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 18432, 128, 128, 32768, 12544, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 8704, 128, 0, 0, 0, 24192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 20096, 11264, 128, 128, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32640, 128, 128, 128, 128, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 9, 0, 4096, 0, 0, 4094, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 1, 4071, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[9, 0, 4, 4089, 2, 7, 0, 4087, 8, 4089, 8, 7, 0, 4088, 7, 0, 40, 0, 4089, 6, 4, 4087, 0, 4083, 5, 4087, 4, 4087, 9, 3872, 4089, 5, 1, 4062, 0, 3999, 9, 0, 1, 0, 0, 0, 0, 0, 7, 6, 0, 5, 278, 2, 9, 0, 4086, 8, 0, 5, 0, 4, 4088, 1, 0, 14, 4079, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 20, 0, 4095, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4051, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 1, 0, 4095, 1, 4095, 1, 1, 0, 4095, 1, 0, 1, 0, 4088, 1, 1, 4095, 2, 4094, 0, 4095, 0, 4095, 1, 4092, 4095, 1, 22, 4090, 0, 4072, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 9, 0, 1, 0, 4095, 1, 0, 1, 0, 8, 4095, 0, 0, 1, 4092, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4096, 0, 0, 38, 1, 4083, 0, 0, 4096, 0, 4095, 7, 4096, 0, 4096, 0, 4090, 4095, 0, 0, 3954, 0, 4087, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 4096, 0, 0, 43, 0, 4084, 0, 0, 4096, 0, 4096, 14, 4096, 0, 4096, 0, 4082, 4096, 0, 0, 4030, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 37, 0, 4082, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 4086, 4096, 0, 0, 3929, 0, 4096, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 4096, 0, 0, 75, 0, 4066, 0, 0, 4096, 0, 4095, 5, 4096, 0, 4096, 0, 3908, 4094, 0, 0, 3926, 0, 4095, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 289, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4071, 0, 4095, 0, 0, 0, 2, 61, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8179, 8244, 8252, 8190, 8095, 8192] avg=8192.0000 max=8252 max_rank=4 min=8095 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=57284 max_load_reduction_pct=0.874084 moved_to_echo=32729 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4095, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 4096, 0, 0, 0, 7, 61, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8189, 8244, 8237, 8192, 8098, 8192] avg=8192.0000 max=8244 max_rank=3 min=8098 min_rank=6 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32719 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 1, 1, 0, 2, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1, 4092, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4090, 0, 4072, 1, 0, 0, 0, 61, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 9, 0, 1, 0, 4095, 1, 0, 1, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 0, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8194, 8165, 8241, 8268, 8192, 8089, 8196] avg=8192.0000 max=8268 max_rank=4 min=8089 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=57268 max_load_reduction_pct=0.873840 moved_to_echo=32745 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 4, 4089, 2, 7, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4089, 8, 7, 0, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4089, 6, 4, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 52, 9, 3872, 4089, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4062, 0, 3999, 9, 0, 1, 0, 61, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 5, 4087, 4087, 0, 0, 0, 0, 0, 0, 278, 2, 9, 0, 4086, 8, 0, 5, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4088, 1, 0, 14, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8207, 8201, 8036, 8154, 8192, 8362, 8186] avg=8192.0000 max=8362 max_rank=6 min=8036 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32894 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 4082, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 4096, 0, 0, 0, 14, 61, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 4096, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8193, 8234, 8122, 8192, 8219, 8192] avg=8192.0000 max=8234 max_rank=3 min=8122 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 1, 4083, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 52, 0, 4090, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 4087, 0, 0, 0, 13, 61, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 4096, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8190, 8195, 8244, 8137, 8192, 8192, 8194] avg=8192.0000 max=8244 max_rank=3 min=8137 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32715 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 4084, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 52, 0, 4082, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 4096, 0, 0, 0, 8, 61, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 4096, 0, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8187, 8201, 8244, 8217, 8192, 8110, 8193] avg=8192.0000 max=8244 max_rank=3 min=8110 min_rank=6 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32712 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 4066, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 42, 0, 3908, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 4095, 0, 0, 0, 24, 66, 23, 7, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 289, 0, 0, 0, 4096, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8182, 8213, 8049, 8149, 8194, 8371, 8191] avg=8192.0000 max=8371 max_rank=6 min=8049 min_rank=3 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32905 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[896, 128, 128, 0, 32768, 128, 0, 128, 31872, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 128, 32768, 128, 128, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32128, 0, 32640, 128, 0, 128, 128, 512, 256, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 512, 128, 32384, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4055, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 42, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4074, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 2, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 22, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 4002, 4096, 0, 0, 14, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 95, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 3, 4095, 0, 0, 4095, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 4095, 4093, 1, 1, 0, 4095, 4095, 1, 1, 0, 0, 4094, 4095, 0, 0, 1, 4095, 0, 0, 0, 4092, 0, 4094, 1, 0, 1, 4095, 4095, 1, 0, 0, 4095, 2, 1, 4095, 3, 0, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 9, 4096, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 4092, 4087, 0, 0, 0, 4005, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 4096, 91, 0, 4, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 5, 0, 0, 1, 0, 4095, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 4060, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4095, 0, 0, 4095, 36, 0, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 15, 4076, 0, 0, 4084, 0, 8, 9, 0, 0, 8, 0, 9, 0, 6, 1, 5, 4088, 4086, 8, 7, 0, 4084, 4087, 11, 1, 10, 1, 4084, 4082, 0, 6, 9, 4089, 0, 9, 0, 4088, 0, 4086, 4, 0, 9, 4077, 4089, 8, 0, 3, 4088, 14, 7, 4087, 0, 0, 0, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 4, 0, 4089, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4088, 4083, 0, 0, 0, 3909, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 2, 4092, 0, 0, 4096, 187, 0, 8, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4031, 61, 2, 1, 0, 0, 0, 65, 4094, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 42, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8192, 8187, 8214, 8192, 8196, 8194, 8173] avg=8192.0000 max=8214 max_rank=3 min=8173 min_rank=7 imbalance_ratio=1.002686 max_load_reduction=57322 max_load_reduction_pct=0.874664 moved_to_echo=32755 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4031, 61, 2, 1, 0, 0, 0, 65, 4094, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8187, 8233, 8191, 8196, 8194, 8153] avg=8192.0000 max=8233 max_rank=3 min=8153 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32734 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 3, 4095, 4091, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 4091, 4030, 61, 2, 1, 0, 0, 0, 65, 4093, 1, 1, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4092, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 2, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4095, 3, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8188, 8256, 8190, 8191, 8194, 8135] avg=8192.0000 max=8256 max_rank=3 min=8135 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4031, 61, 4, 1, 0, 0, 0, 65, 4081, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 95, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8192, 8189, 8148, 8205, 8196, 8192, 8226] avg=8192.0000 max=8226 max_rank=7 min=8148 min_rank=3 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32819 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 5, 0, 0, 1, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4092, 4030, 61, 2, 1, 0, 0, 0, 65, 4090, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 36, 0, 1, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 8189, 8215, 8191, 8194, 8193, 8167] avg=8192.0000 max=8215 max_rank=3 min=8167 min_rank=7 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 15, 4087, 4086, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4084, 0, 8, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 9, 0, 6, 1, 5, 4085, 4023, 61, 3, 0, 0, 0, 0, 65, 4086, 8, 7, 0, 4084, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 1, 4084, 4082, 0, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 9, 0, 4088, 0, 4086, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4077, 0, 8, 0, 3, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 14, 7, 4087, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8177, 8201, 8261, 8193, 8191, 8185, 8134] avg=8192.0000 max=8261 max_rank=3 min=8134 min_rank=7 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32703 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4090, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 4094, 4027, 61, 2, 1, 0, 0, 0, 65, 4087, 0, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 91, 0, 4, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8195, 8194, 8157, 8191, 8188, 8194, 8226] avg=8192.0000 max=8226 max_rank=7 min=8157 min_rank=3 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 4, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4089, 0, 13, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 4028, 65, 2, 2, 0, 0, 0, 60, 4083, 0, 0, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 187, 0, 8, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8204, 8201, 8052, 8183, 8184, 8190, 8322] avg=8192.0000 max=8322 max_rank=7 min=8052 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32879 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 32768, 128, 128, 0, 32384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 32768, 32256, 512, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 512, 0, 128, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 32768, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4077, 0, 0, 4095, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4082, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4020, 0, 0, 4070, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4095, 4090, 4095, 4094, 0, 0, 1, 1, 0, 72, 0, 1, 0, 4095, 4095, 4092, 4095, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 77, 3958, 1, 0, 4091, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 1, 0, 4095, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 1, 4095, 4090, 4095, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 15, 7, 4063, 0, 0, 4073, 0, 0, 24, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 4096, 4096, 4096, 1, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4089, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 5, 31, 4078, 0, 0, 4061, 0, 0, 6, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 1, 4095, 0, 6, 0, 0, 0, 0, 4096, 4096, 4096, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4087, 4067, 4088, 4083, 11, 10, 6, 7, 0, 247, 0, 8, 0, 4086, 4089, 4054, 4085, 3, 8, 9, 10, 1, 7, 3, 0, 8, 9, 0, 7, 0, 0, 25, 3820, 4, 0, 4091, 8, 0, 26, 0, 4087, 0, 7, 0, 0, 9, 0, 4083, 0, 12, 0, 4087, 3, 9, 36, 0, 0, 3, 4080, 4053, 4086, 0, 0, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 4096, 4085, 4096, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4086, 4088, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 14, 109, 3951, 0, 0, 4056, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 4096, 4096, 4096, 3, 0, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4059, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4034, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 4038, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 4095, 0, 0, 1, 0, 63, 34, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4028, 58, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8156, 8188, 8153, 8270, 8194, 8182, 8196] avg=8192.0000 max=8270 max_rank=4 min=8153 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32842 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4060, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4034, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4038, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 4091, 0, 0, 0, 0, 63, 34, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4028, 60, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8168, 8188, 8141, 8270, 8194, 8184, 8194] avg=8192.0000 max=8270 max_rank=4 min=8141 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=57266 max_load_reduction_pct=0.873810 moved_to_echo=32842 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 4095, 6, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 1, 0, 4059, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 1, 0, 1, 0, 4033, 58, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 77, 4095, 4037, 0, 0, 0, 0, 0, 0, 3958, 1, 0, 4091, 1, 0, 1, 0, 62, 34, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 1, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4026, 58, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4090, 4, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8224, 8191, 8213, 8148, 8194, 8180, 8193] avg=8192.0000 max=8224 max_rank=1 min=8148 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32715 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4059, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4034, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4096, 4038, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 4070, 0, 0, 0, 0, 63, 34, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 4028, 58, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8156, 8188, 8223, 8187, 8194, 8195, 8196] avg=8192.0000 max=8223 max_rank=3 min=8156 min_rank=1 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4089, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4059, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4034, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 5, 31, 4096, 4037, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 4061, 0, 0, 6, 0, 63, 34, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 6, 0, 0, 4028, 58, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8156, 8188, 8182, 8242, 8191, 8189, 8198] avg=8192.0000 max=8242 max_rank=4 min=8156 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32820 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4067, 4088, 6, 11, 10, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 8, 0, 4050, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 3, 8, 9, 10, 1, 7, 3, 4025, 58, 0, 0, 0, 0, 0, 0, 0, 8, 9, 0, 7, 0, 0, 25, 4087, 4031, 0, 0, 0, 0, 0, 0, 3820, 4, 0, 4091, 8, 0, 26, 0, 62, 34, 0, 0, 0, 0, 0, 0, 4087, 0, 7, 0, 0, 9, 0, 4083, 2, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4087, 3, 9, 36, 0, 4015, 58, 0, 0, 0, 0, 0, 0, 0, 3, 4080, 4053, 3, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8359, 8209, 8167, 8045, 8188, 8220, 8153] avg=8192.0000 max=8359 max_rank=1 min=8045 min_rank=4 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32606 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4083, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4036, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 15, 7, 4096, 4036, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 4073, 0, 0, 24, 0, 63, 34, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 6, 0, 0, 4028, 60, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8150, 8191, 8176, 8257, 8188, 8191, 8199] avg=8192.0000 max=8257 max_rank=4 min=8150 min_rank=1 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32835 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4047, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4038, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 14, 109, 4096, 4029, 0, 0, 0, 0, 0, 0, 3951, 0, 0, 4056, 0, 0, 10, 0, 65, 35, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 4029, 58, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8167, 8193, 8281, 8117, 8193, 8195, 8207] avg=8192.0000 max=8281 max_rank=3 min=8117 min_rank=4 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32712 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 384, 32768, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 384, 0, 128, 0, 32512, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 0, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 128, 32384, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 0, 32256, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32128, 128, 0, 32640, 128, 0, 128, 0, 512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2, 0, 0, 4096, 4096, 4095, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 4096, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4093, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 38, 0, 0, 0, 0, 4096, 4057, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4052, 3991, 3, 8, 6, 0, 4083, 459, 4087, 10, 1, 1, 0, 0, 8, 8, 0, 0, 0, 11, 6, 34, 0, 4088, 0, 4, 4079, 6, 0, 0, 0, 97, 4087, 4086, 4087, 0, 3628, 0, 0, 9, 10, 4086, 0, 36, 0, 4088, 0, 4085, 4080, 4086, 0, 9, 7, 0, 7, 0, 0, 9, 10, 0, 11, 10, 4, 4059] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4091, 4085, 2, 1, 1, 0, 4095, 119, 4095, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 4, 0, 4095, 0, 1, 4094, 0, 0, 0, 0, 11, 4095, 4095, 4087, 0, 3972, 0, 0, 1, 1, 4095, 0, 7, 0, 4095, 0, 4094, 4095, 4095, 0, 0, 1, 4, 1, 0, 0, 1, 1, 0, 1, 1, 1, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 10, 0, 0, 0, 0, 4096, 4086, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 4096, 40, 0, 0, 0, 0, 4096, 4056, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 4096, 0, 0, 0, 0, 4084, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 4096, 80, 0, 0, 0, 0, 4096, 4017, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 4096, 4096, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4042, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4000, 54, 3, 2, 3, 0, 0, 0, 0, 4093, 96, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8191, 8189, 8158, 8285, 8189, 8193, 8193] avg=8192.0000 max=8285 max_rank=4 min=8138 min_rank=0 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32860 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4042, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 3999, 54, 3, 2, 2, 0, 0, 0, 0, 4096, 96, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8190, 8191, 8158, 8285, 8189, 8193, 8192] avg=8192.0000 max=8285 max_rank=4 min=8138 min_rank=0 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32859 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4037, 4085, 2, 1, 1, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4, 0, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 11, 3991, 54, 3, 2, 3, 0, 0, 0, 0, 4095, 96, 0, 3972, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 7, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 0, 0, 1, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 4093, 4095, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8192, 8194, 8159, 8164, 8194, 8194, 8194] avg=8192.0000 max=8245 max_rank=0 min=8159 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32746 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4042, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 38, 0, 0, 0, 0, 4000, 54, 3, 2, 2, 0, 0, 0, 0, 4057, 96, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8191, 8191, 8195, 8249, 8189, 8191, 8192] avg=8192.0000 max=8249 max_rank=4 min=8138 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32821 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4042, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 10, 0, 0, 0, 0, 4000, 54, 3, 2, 3, 0, 0, 0, 0, 4086, 96, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8191, 8187, 8169, 8278, 8189, 8192, 8192] avg=8192.0000 max=8278 max_rank=4 min=8138 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32851 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3999, 3991, 3, 8, 6, 0, 0, 459, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 10, 1, 1, 0, 0, 8, 8, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 34, 0, 4088, 4081, 1, 0, 0, 0, 0, 0, 0, 0, 4, 4079, 6, 0, 0, 0, 97, 3991, 53, 3, 2, 2, 0, 0, 0, 0, 4086, 96, 0, 3628, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4083, 0, 36, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4084, 0, 9, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 10, 0, 11, 10, 4, 4059, 4087, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8466, 8202, 8221, 8237, 7819, 8214, 8187, 8190] avg=8192.0000 max=8466 max_rank=0 min=7819 min_rank=4 imbalance_ratio=1.033447 max_load_reduction=57070 max_load_reduction_pct=0.870819 moved_to_echo=32410 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4042, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 1, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 40, 0, 0, 0, 0, 4000, 54, 3, 2, 2, 0, 0, 0, 0, 4056, 96, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8191, 8185, 8203, 8248, 8189, 8190, 8192] avg=8192.0000 max=8248 max_rank=4 min=8138 min_rank=0 imbalance_ratio=1.006836 max_load_reduction=57288 max_load_reduction_pct=0.874146 moved_to_echo=32819 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4039, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 80, 0, 0, 0, 0, 4001, 57, 9, 6, 2, 0, 0, 0, 0, 4017, 95, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8188, 8178, 8257, 8208, 8183, 8196, 8191] avg=8192.0000 max=8257 max_rank=3 min=8135 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=57279 max_load_reduction_pct=0.874008 moved_to_echo=32778 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32640, 768, 0, 32256, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 32768, 0, 128, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32384, 32768, 128, 128, 128, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 256, 0, 128, 0, 128, 32000, 512, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 51, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 7, 0, 0, 0, 0, 4070, 0, 4096, 4005, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 75, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4080, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 68, 0, 2, 1, 0, 1, 0, 0, 4095, 0, 1, 0, 0, 4095, 4095, 2, 4094, 259, 0, 0, 0, 2, 3774, 1, 4095, 3551, 0, 1, 2, 4094, 4095, 4093, 1, 0, 1, 1, 3, 0, 0, 0, 537, 4092, 0, 4095, 0, 0, 0, 4095, 1, 4087, 1, 1, 1, 4094, 1, 0, 1, 4095, 1, 0, 0, 6, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 28, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3, 0, 0, 0, 0, 4076, 0, 4096, 4056, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4096, 0, 4096, 0, 0, 0, 4094, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 123, 0, 15, 9, 0, 6, 2, 0, 4085, 0, 9, 0, 0, 4086, 4089, 12, 4084, 1396, 0, 0, 0, 24, 2752, 11, 4085, 2247, 2, 5, 10, 4081, 4086, 4079, 10, 0, 38, 5, 24, 0, 0, 0, 1753, 4049, 0, 4086, 0, 5, 0, 4087, 4, 3917, 12, 8, 7, 4085, 9, 0, 4, 4087, 10, 0, 0, 33, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 17, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 4095, 0, 4096, 7, 0, 0, 0, 0, 3985, 0, 4096, 3728, 0, 0, 0, 4096, 4096, 4096, 0, 1, 0, 0, 0, 1, 0, 0, 464, 4096, 0, 4095, 0, 0, 0, 4094, 0, 4090, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 29, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 4096, 0, 4096, 129, 0, 0, 0, 0, 4033, 0, 4096, 3959, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 73, 4096, 0, 4095, 0, 0, 0, 4096, 0, 4067, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 7, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 109, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 68, 0, 0, 0, 0, 4028, 0, 4096, 3896, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 137, 4096, 0, 4096, 0, 0, 0, 4094, 0, 4052, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 201, 0, 0, 0, 6, 0, 0, 0, 4095, 0, 0, 0, 0, 4079, 4090, 0, 4096, 151, 0, 0, 0, 0, 4014, 0, 4096, 3823, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 66, 4096, 0, 4094, 0, 0, 0, 4096, 0, 4050, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 11, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 0, 0, 0, 0, 0, 0, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 4056, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4012, 75, 0, 0, 0, 0, 0, 0, 0, 33, 4096, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3659, 384, 42, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8198, 8174, 8589, 8183, 7841, 8230, 8182] avg=8192.0000 max=8589 max_rank=3 min=7841 min_rank=5 imbalance_ratio=1.048462 max_load_reduction=56947 max_load_reduction_pct=0.868942 moved_to_echo=32436 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 0, 0, 0, 0, 0, 0, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 4005, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4014, 75, 0, 0, 0, 0, 0, 0, 0, 75, 4096, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3659, 384, 42, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8198, 8172, 8538, 8185, 7883, 8216, 8182] avg=8192.0000 max=8538 max_rank=3 min=7883 min_rank=5 imbalance_ratio=1.042236 max_load_reduction=56998 max_load_reduction_pct=0.869720 moved_to_echo=32466 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 68, 0, 2, 1, 0, 1, 0, 4095, 4014, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 259, 0, 0, 0, 2, 3774, 0, 0, 0, 0, 0, 0, 0, 0, 1, 437, 3551, 0, 1, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 1, 1, 3, 0, 0, 4013, 75, 0, 0, 0, 0, 0, 0, 0, 537, 4092, 0, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 1, 4087, 1, 1, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 0, 6, 1, 3658, 384, 42, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8197, 8130, 8086, 8187, 8340, 8226, 8189] avg=8192.0000 max=8340 max_rank=5 min=8086 min_rank=3 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32942 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 0, 0, 0, 1, 0, 0, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 3728, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 1, 0, 4012, 75, 0, 0, 0, 0, 0, 0, 0, 464, 4096, 0, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4090, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 3659, 384, 42, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8194, 8087, 8261, 8185, 8271, 8225, 8184] avg=8192.0000 max=8271 max_rank=5 min=8087 min_rank=2 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32865 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 109, 0, 0, 0, 0, 0, 0, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 68, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 3896, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4012, 75, 0, 0, 0, 0, 0, 0, 0, 137, 4096, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4052, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3659, 384, 42, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8198, 8191, 8429, 8183, 7945, 8188, 8182] avg=8192.0000 max=8429 max_rank=3 min=7945 min_rank=5 imbalance_ratio=1.028931 max_load_reduction=57107 max_load_reduction_pct=0.871384 moved_to_echo=32498 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 123, 0, 15, 9, 0, 6, 2, 4086, 4004, 0, 0, 0, 0, 0, 0, 0, 6, 0, 9, 0, 0, 4086, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4083, 1396, 0, 0, 0, 24, 2752, 0, 0, 0, 0, 0, 0, 0, 0, 11, 435, 2247, 2, 5, 10, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 10, 0, 38, 5, 24, 0, 0, 4005, 75, 0, 0, 0, 0, 0, 0, 0, 1753, 4049, 0, 3703, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4, 3917, 12, 8, 7, 4085, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4087, 10, 0, 0, 33, 5, 3650, 383, 42, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 8190, 8267, 6791, 8236, 9510, 8082, 8215] avg=8192.0000 max=9510 max_rank=5 min=6791 min_rank=3 imbalance_ratio=1.160889 max_load_reduction=56026 max_load_reduction_pct=0.854889 moved_to_echo=34043 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 29, 0, 0, 0, 0, 0, 0, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 129, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 3959, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4014, 75, 0, 0, 0, 0, 0, 0, 0, 73, 4096, 0, 3711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4067, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 3659, 384, 42, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8191, 8257, 8492, 8185, 7880, 8202, 8189] avg=8192.0000 max=8492 max_rank=3 min=7880 min_rank=5 imbalance_ratio=1.036621 max_load_reduction=57044 max_load_reduction_pct=0.870422 moved_to_echo=32456 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 201, 0, 0, 0, 6, 0, 0, 4096, 4017, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4079, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 151, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 3823, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4019, 77, 0, 0, 0, 0, 0, 0, 0, 66, 4096, 0, 3704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4050, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 3665, 390, 48, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8170, 8258, 8350, 8192, 7866, 8167, 8213] avg=8192.0000 max=8350 max_rank=3 min=7866 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32438 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 2048, 0, 0, 0, 128, 30848, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 3200, 32768, 0, 29696, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 128, 128, 29312, 3200, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 640, 0, 128, 128, 128, 128, 128, 32768, 32128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 3584, 29312, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 0, 32128, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 128, 32512, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4079, 0, 4096, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 331, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 3460, 0, 4096, 0, 3383, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 1314, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1, 3758, 0, 4095, 1, 3699, 1, 0, 0, 0, 1, 0, 0, 294, 0, 0, 2, 0, 1, 4095, 1, 1, 4094, 1, 0, 1, 31, 4093, 0, 0, 1, 4095, 5, 0, 0, 4092, 0, 1, 0, 1, 3, 8, 4095, 0, 1, 0, 0, 1, 416, 0, 4093, 4086, 1, 23, 1, 0, 1, 0, 4095, 4074, 4084, 4, 4090, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 3920, 0, 4093, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 10, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3, 0, 0, 681, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4093, 4096, 0, 4096, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4029, 0, 4096, 0, 3583, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 577, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[9, 2599, 0, 4089, 5, 3850, 11, 0, 0, 0, 10, 0, 0, 1357, 9, 0, 27, 0, 6, 4086, 9, 10, 4080, 12, 12, 12, 109, 4071, 0, 0, 7, 4085, 58, 0, 0, 4063, 0, 4, 2, 16, 19, 29, 4074, 0, 10, 0, 0, 8, 254, 1, 4049, 3942, 16, 178, 3, 0, 11, 0, 4085, 4043, 3974, 97, 4055, 4080] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4010, 0, 4085, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 5, 0, 10, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 25, 4096, 0, 0, 0, 4096, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 498, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4072, 4096, 0, 4096, 4088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 3975, 0, 4084, 0, 3185, 0, 0, 0, 0, 0, 0, 0, 27, 0, 9, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 17, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 10, 0, 0, 1012, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4083, 4096, 0, 4096, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4079, 0, 870, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3878, 217, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3226, 637, 200, 23, 5, 0, 0, 0, 331, 0, 3459, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8721, 7985, 8187, 8169, 8191, 8187, 7886, 8210] avg=8192.0000 max=8721 max_rank=0 min=7886 min_rank=6 imbalance_ratio=1.064575 max_load_reduction=56815 max_load_reduction_pct=0.866928 moved_to_echo=32474 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3460, 0, 870, 0, 3383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 4096, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3878, 216, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 3226, 637, 200, 23, 5, 0, 0, 0, 1314, 0, 3459, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7713, 8005, 8187, 8173, 8190, 8189, 8869, 8210] avg=8192.0000 max=8869 max_rank=6 min=7713 min_rank=0 imbalance_ratio=1.082642 max_load_reduction=56667 max_load_reduction_pct=0.864670 moved_to_echo=33458 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 3758, 0, 870, 1, 3699, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 294, 0, 0, 4095, 3878, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4090, 1, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 31, 4093, 0, 0, 1, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4092, 0, 1, 0, 1, 3873, 217, 0, 0, 0, 0, 0, 0, 3, 8, 4095, 0, 1, 0, 0, 1, 3225, 637, 199, 23, 5, 0, 0, 0, 416, 0, 3456, 4086, 1, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4074, 4084, 4, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8330, 8268, 8190, 8198, 8189, 8197, 7983, 8181] avg=8192.0000 max=8330 max_rank=0 min=7983 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=57206 max_load_reduction_pct=0.872894 moved_to_echo=32550 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3920, 0, 870, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4, 4096, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 1, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3878, 217, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3, 0, 0, 3223, 637, 200, 23, 5, 0, 0, 0, 681, 0, 3459, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8362, 7987, 8187, 8180, 8191, 8186, 8236, 8207] avg=8192.0000 max=8362 max_rank=0 min=7987 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=57174 max_load_reduction_pct=0.872406 moved_to_echo=32820 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[9, 2599, 0, 869, 5, 3850, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1357, 9, 0, 4085, 3865, 0, 0, 0, 0, 0, 0, 27, 0, 6, 4081, 9, 10, 4080, 12, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 109, 4071, 0, 0, 7, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 4063, 0, 4, 2, 16, 3839, 215, 0, 0, 0, 0, 0, 0, 19, 29, 4074, 0, 10, 0, 0, 8, 3220, 630, 198, 23, 5, 0, 0, 0, 254, 1, 3419, 3942, 16, 178, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4043, 3974, 97, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7343, 9326, 8225, 8273, 8197, 8216, 7813, 8143] avg=8192.0000 max=9326 max_rank=1 min=7343 min_rank=0 imbalance_ratio=1.138428 max_load_reduction=56210 max_load_reduction_pct=0.857697 moved_to_echo=32369 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4029, 0, 870, 0, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3878, 217, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3226, 637, 200, 23, 5, 0, 0, 0, 577, 0, 3459, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8482, 7978, 8187, 8169, 8191, 8187, 8132, 8210] avg=8192.0000 max=8482 max_rank=0 min=7978 min_rank=1 imbalance_ratio=1.035400 max_load_reduction=57054 max_load_reduction_pct=0.870575 moved_to_echo=32720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4010, 0, 868, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 10, 4096, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 3878, 215, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 7, 0, 0, 3217, 637, 200, 23, 5, 0, 0, 0, 498, 0, 3459, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8562, 7984, 8187, 8194, 8192, 8178, 8053, 8186] avg=8192.0000 max=8562 max_rank=0 min=7984 min_rank=1 imbalance_ratio=1.045166 max_load_reduction=56974 max_load_reduction_pct=0.869354 moved_to_echo=32609 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3975, 0, 863, 0, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 9, 4096, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3884, 215, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 10, 0, 0, 3221, 640, 202, 28, 6, 0, 0, 0, 1012, 0, 3456, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8023, 8003, 8186, 8180, 8195, 8196, 8564, 8189] avg=8192.0000 max=8564 max_rank=6 min=8003 min_rank=1 imbalance_ratio=1.045410 max_load_reduction=56972 max_load_reduction_pct=0.869324 moved_to_echo=33144 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 32768, 32640, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[5120, 128, 27648, 32640, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 29952, 0, 7040, 128, 28800, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 1792, 128, 128, 32768, 31104, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 256, 32768, 128, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 0, 128, 25856, 5120, 1664, 256, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 32768, 0, 128, 128, 128, 31104, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 117, 4096, 0, 0, 0, 4096, 0, 0, 0, 3982, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 340, 4096, 0, 0, 0, 4096, 0, 0, 0, 3753, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 164, 4096, 0, 0, 0, 4096, 0, 0, 0, 3929, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 37, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[13, 0, 0, 0, 4088, 4083, 4047, 0, 2, 10, 0, 4089, 16, 0, 0, 10, 4085, 0, 4086, 0, 0, 460, 4085, 89, 0, 11, 3622, 1, 0, 9, 4084, 9, 4003, 11, 8, 54, 25, 4035, 4, 11, 4075, 0, 7, 10, 17, 0, 0, 4086, 10, 4082, 4078, 0, 9, 0, 4, 9, 8, 4087, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 4096, 1, 4094, 0, 0, 403, 4096, 0, 0, 0, 4096, 0, 1, 0, 3699, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4095, 4088, 0, 1, 0, 0, 4095, 2, 0, 0, 1, 4095, 0, 4095, 0, 0, 248, 4095, 6, 0, 1, 4012, 0, 0, 1, 3925, 1, 4089, 2, 1, 8, 3, 4092, 1, 1, 4094, 0, 1, 1, 2, 0, 0, 4095, 0, 4094, 4094, 0, 1, 3, 1, 0, 1, 4095, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 394, 4096, 0, 0, 0, 4096, 0, 6, 0, 3773, 0, 4047, 0, 0, 0, 0, 4096, 0, 0, 4090, 22, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 5, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 4096, 0, 4086, 0, 0, 715, 4096, 0, 0, 0, 4096, 0, 10, 0, 3419, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 4090, 31, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4070, 22, 0, 0, 0, 0, 0, 0, 3735, 0, 4096, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3753, 0, 339, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8188, 8171, 8188, 8203, 8191, 8192, 8206] avg=8192.0000 max=8206 max_rank=7 min=8171 min_rank=2 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4070, 22, 0, 0, 0, 0, 0, 0, 3735, 0, 4095, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3982, 0, 339, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8188, 7947, 8417, 8203, 8181, 8192, 8211] avg=8192.0000 max=8417 max_rank=3 min=7947 min_rank=2 imbalance_ratio=1.027466 max_load_reduction=57119 max_load_reduction_pct=0.871567 moved_to_echo=32787 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 5, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 4069, 22, 0, 0, 0, 0, 0, 0, 3735, 1, 4094, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 3699, 0, 339, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 1, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8188, 8233, 8135, 8202, 8183, 8191, 8202] avg=8192.0000 max=8233 max_rank=2 min=8135 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4070, 22, 0, 0, 0, 0, 0, 0, 3735, 0, 4096, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3929, 0, 339, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 37, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8188, 7995, 8364, 8203, 8218, 8192, 8179] avg=8192.0000 max=8364 max_rank=3 min=7995 min_rank=2 imbalance_ratio=1.020996 max_load_reduction=57172 max_load_reduction_pct=0.872375 moved_to_echo=32792 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 0, 5, 4083, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 10, 0, 4089, 16, 0, 0, 10, 4062, 22, 0, 0, 0, 0, 0, 0, 3725, 0, 4086, 0, 0, 460, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3622, 1, 0, 9, 4084, 9, 338, 0, 0, 0, 0, 0, 0, 0, 4003, 11, 8, 54, 25, 4035, 4, 11, 11, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 7, 10, 17, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4082, 4078, 0, 9, 0, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4087, 0, 0, 0, 0, 4, 0, 4085, 21, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8211, 8360, 8074, 8162, 8184, 8192, 8205] avg=8192.0000 max=8360 max_rank=2 min=8074 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=57176 max_load_reduction_pct=0.872437 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 4095, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 2, 0, 0, 1, 4069, 22, 0, 0, 0, 0, 0, 0, 3734, 0, 4095, 0, 0, 248, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4012, 0, 0, 1, 3925, 1, 339, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 1, 8, 3, 4092, 1, 1, 11, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 1, 2, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 1, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 4095, 21, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8190, 8083, 8279, 8208, 8182, 8193, 8213] avg=8192.0000 max=8279 max_rank=3 min=8083 min_rank=2 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 5, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4070, 22, 0, 0, 0, 0, 0, 0, 3735, 0, 4089, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 3773, 0, 339, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4090, 22, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 5, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8188, 8218, 8214, 8154, 8197, 8192, 8166] avg=8192.0000 max=8218 max_rank=2 min=8154 min_rank=4 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32709 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 4072, 27, 0, 0, 0, 0, 0, 0, 3728, 0, 4086, 0, 0, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 3419, 0, 340, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 17, 1, 0, 0, 0, 0, 0, 0, 4090, 31, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 6, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8195, 8529, 7865, 8201, 8200, 8192, 8154] avg=8192.0000 max=8529 max_rank=2 min=7865 min_rank=3 imbalance_ratio=1.041138 max_load_reduction=57007 max_load_reduction_pct=0.869858 moved_to_echo=32747 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 128, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[29952, 128, 32768, 0, 0, 2944, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32256, 128, 128, 128, 30592, 128, 2816, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32640, 0, 0, 0, 128, 128, 0, 32768, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32640, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4063, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 32, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 4096, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 67, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4008, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 4092, 16, 4041, 0, 0, 0, 0, 4095, 102, 0, 1, 0, 4095, 2, 3, 0, 4094, 0, 4095, 43, 0, 4095, 4, 1, 4095, 4095, 0, 4095, 0, 0, 0, 2, 0, 0, 1, 4095, 4073, 0, 1, 0, 4094, 0, 1, 1, 1, 1, 1, 4091, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 3996, 12, 0, 0, 4, 0, 0, 10, 11, 10, 10, 5, 2, 4076, 171, 3696, 0, 0, 5, 0, 4088, 238, 0, 2, 0, 4086, 13, 15, 0, 4085, 0, 4086, 45, 0, 4087, 32, 12, 4086, 4081, 10, 4085, 3, 0, 0, 15, 0, 0, 10, 4086, 4073, 0, 9, 0, 4077, 6, 11, 9, 15, 2, 2, 4065, 4084, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4061, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 5, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4016, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4091, 78, 0, 0, 0, 4096, 0, 0, 0, 4096, 5, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4092, 0, 0, 0, 0, 6, 0, 0, 4096, 4066, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4085, 0, 0, 0, 0, 0, 0, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4043, 14, 8, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4054, 41, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8163, 8195, 8190, 8178, 8191, 8204, 8192] avg=8192.0000 max=8223 max_rank=0 min=8163 min_rank=1 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32765 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 0, 0, 4096, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4043, 14, 8, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4053, 41, 0, 0, 0, 0, 0, 0, 0, 12, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8163, 8190, 8190, 8181, 8190, 8201, 8192] avg=8192.0000 max=8229 max_rank=0 min=8163 min_rank=1 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32764 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4008, 0, 0, 0, 1, 0, 0, 4095, 42, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4092, 16, 4042, 14, 8, 2, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 4087, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 2, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 43, 0, 0, 4, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 4053, 41, 0, 0, 0, 0, 0, 0, 1, 12, 4073, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 4091, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8179, 8230, 8193, 8224, 8191, 8181, 8192] avg=8192.0000 max=8230 max_rank=2 min=8146 min_rank=0 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4063, 0, 0, 0, 0, 0, 10, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4043, 14, 8, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 1, 0, 4054, 41, 0, 0, 0, 0, 0, 0, 0, 12, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8163, 8215, 8191, 8178, 8192, 8194, 8192] avg=8192.0000 max=8215 max_rank=2 min=8163 min_rank=1 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3996, 12, 0, 0, 4, 0, 0, 4087, 43, 0, 0, 0, 0, 0, 0, 10, 11, 10, 10, 5, 2, 4076, 171, 4033, 14, 8, 2, 0, 0, 0, 0, 3696, 0, 0, 5, 0, 4080, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4084, 13, 15, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 45, 0, 0, 32, 12, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4085, 3, 0, 0, 15, 0, 0, 4043, 41, 0, 0, 0, 0, 0, 0, 10, 12, 4073, 0, 9, 0, 4077, 6, 0, 0, 0, 0, 0, 0, 0, 0, 11, 9, 15, 2, 2, 4065, 4084, 10, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 8352, 8019, 8199, 8242, 8197, 8187, 8198] avg=8192.0000 max=8352 max_rank=1 min=8019 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32824 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4029, 0, 0, 0, 0, 0, 0, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4043, 14, 8, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4054, 41, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8163, 8251, 8190, 8178, 8191, 8204, 8192] avg=8192.0000 max=8251 max_rank=2 min=8163 min_rank=1 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32765 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4061, 0, 0, 0, 0, 0, 33, 4096, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4043, 14, 8, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4053, 41, 0, 0, 0, 0, 0, 0, 0, 12, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 8163, 8186, 8195, 8178, 8190, 8199, 8192] avg=8192.0000 max=8233 max_rank=0 min=8163 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4016, 0, 0, 0, 0, 0, 30, 4096, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4049, 15, 15, 9, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 4076, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 6, 0, 4053, 43, 0, 0, 0, 0, 0, 0, 0, 4, 4066, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8190, 8250, 8188, 8177, 8194, 8166, 8186] avg=8192.0000 max=8250 max_rank=2 min=8166 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=57286 max_load_reduction_pct=0.874115 moved_to_echo=32723 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32384, 128, 0, 0, 128, 0, 128, 32768, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 0, 32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32384, 0, 0, 128, 0, 32768, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 256, 32384, 128, 128, 128, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4086, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 12, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4096, 0, 4096, 4095, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 2, 0, 4084, 0, 3971, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 135, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 5, 0, 4096, 0, 4096, 4094, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 425, 9, 3753, 4, 4089, 0, 0, 4087, 8, 4084, 0, 15, 0, 0, 4068, 275, 0, 4084, 4073, 7, 4081, 7, 17, 13, 9, 4087, 0, 0, 34, 0, 4082, 0, 9, 2, 4088, 0, 4085, 0, 0, 12, 0, 11, 0, 0, 0, 14, 5, 0, 12, 10, 0, 0, 14, 4085, 0, 0, 4047, 7, 4082, 3716, 0, 16, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 67, 1, 4016, 1, 4095, 0, 0, 4095, 1, 4095, 0, 2, 0, 0, 4091, 17, 0, 4095, 4094, 1, 4094, 0, 1, 3, 1, 4095, 0, 0, 210, 0, 4095, 0, 1, 0, 4095, 0, 4095, 0, 0, 1, 0, 1, 0, 0, 2, 1, 1, 0, 1, 2, 0, 0, 2, 4095, 0, 0, 4091, 1, 4094, 3880, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 3, 0, 4084, 0, 3937, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 169, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 16, 0, 4096, 0, 4096, 4092, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 2, 0, 4048, 0, 3908, 0, 0, 4095, 0, 4096, 1, 0, 0, 0, 4092, 0, 0, 4094, 4096, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 231, 1, 4096, 0, 0, 0, 4096, 2, 4094, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 0, 4096, 0, 4096, 4080, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 28, 0, 4055, 0, 3911, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 252, 2, 4096, 0, 0, 0, 4096, 6, 4090, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 5, 0, 4096, 0, 4096, 4033, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 21, 0, 4045, 0, 3871, 3, 0, 4095, 0, 4096, 0, 0, 0, 1, 4082, 0, 0, 4091, 4096, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 237, 6, 4096, 0, 0, 0, 4096, 9, 4089, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 24, 0, 4096, 0, 4096, 4094, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4084, 0, 3971, 0, 0, 81, 34, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 135, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 5, 4008, 74, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 88, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8197, 8158, 8169, 8192, 8184, 8186, 8278] avg=8192.0000 max=8278 max_rank=7 min=8158 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=57258 max_load_reduction_pct=0.873688 moved_to_echo=32840 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4086, 0, 4095, 0, 0, 81, 34, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 12, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4008, 74, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 88, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8296, 8200, 8158, 8046, 8192, 8184, 8181, 8279] avg=8192.0000 max=8296 max_rank=0 min=8046 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=57240 max_load_reduction_pct=0.873413 moved_to_echo=32836 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 67, 1, 4016, 1, 4095, 0, 0, 81, 34, 0, 0, 0, 0, 0, 0, 8, 1, 4095, 0, 2, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4061, 4094, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4095, 0, 0, 210, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 2, 1, 1, 4094, 4087, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 2, 4095, 0, 4006, 74, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 88, 3880, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8295, 8197, 8174, 8249, 8191, 8187, 8180, 8063] avg=8192.0000 max=8295 max_rank=0 min=8063 min_rank=7 imbalance_ratio=1.012573 max_load_reduction=57241 max_load_reduction_pct=0.873428 moved_to_echo=32621 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 425, 9, 3753, 4, 4089, 0, 0, 81, 33, 0, 0, 0, 0, 0, 0, 8, 8, 4084, 0, 15, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 4051, 4073, 7, 0, 7, 17, 0, 0, 0, 0, 0, 0, 0, 0, 13, 9, 4087, 0, 0, 34, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 4088, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 11, 0, 0, 0, 14, 5, 4081, 4079, 0, 0, 0, 0, 0, 0, 0, 12, 10, 0, 0, 14, 4085, 0, 3994, 74, 0, 0, 0, 0, 0, 0, 0, 4047, 7, 88, 3716, 0, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8394, 8183, 8430, 8070, 8184, 8202, 8189, 7884] avg=8192.0000 max=8430 max_rank=2 min=7884 min_rank=7 imbalance_ratio=1.029053 max_load_reduction=57106 max_load_reduction_pct=0.871368 moved_to_echo=32459 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4048, 0, 3908, 0, 0, 81, 34, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 231, 1, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 4008, 74, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 88, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8073, 8197, 8156, 8264, 8192, 8190, 8200, 8264] avg=8192.0000 max=8264 max_rank=3 min=8073 min_rank=0 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32846 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 4084, 0, 3937, 0, 0, 81, 34, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 169, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 16, 4008, 74, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 88, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8196, 8158, 8195, 8192, 8184, 8196, 8276] avg=8192.0000 max=8276 max_rank=7 min=8139 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32848 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 21, 0, 4045, 0, 3871, 3, 0, 83, 35, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 237, 6, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 24, 4011, 76, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 85, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8058, 8179, 8145, 8264, 8201, 8203, 8211, 8275] avg=8192.0000 max=8275 max_rank=7 min=8058 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=57261 max_load_reduction_pct=0.873734 moved_to_echo=32890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 0, 4055, 0, 3911, 0, 0, 81, 34, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 252, 2, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 5, 4008, 74, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 88, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8187, 8157, 8279, 8192, 8202, 8193, 8217] avg=8192.0000 max=8279 max_rank=3 min=8109 min_rank=0 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 0, 32512, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 0, 1280, 128, 31616, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 768, 32128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 640, 128, 32256, 128, 32000, 128, 0, 768, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 0, 128, 32768, 128, 32128, 640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True + [2026-08-30 04:59:30] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 717.2 | throughput per GPU (TFLOP/s/GPU): 464.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.500163E+00 | loss scale: 1.0 | grad norm: 5.730 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2726, 30, 166, 53, 324, 1, 1835, 69, 835, 2898, 112, 257, 865, 1649, 1, 310, 510, 1358, 151, 1781, 0, 511, 1248, 1711, 28, 7, 2897, 1001, 2836, 85, 3136, 668, 26, 895, 2416, 105, 538, 150, 3430, 1053, 1954, 0, 3428, 1994, 1, 971, 1015, 1184, 3090, 26, 1683, 1, 3, 170, 2, 551, 0, 97, 740, 1304, 3411, 1249, 1172, 2818] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2571, 18, 107, 92, 550, 0, 2801, 49, 1091, 3374, 84, 449, 1274, 1269, 0, 214, 480, 816, 106, 1865, 0, 507, 880, 1375, 26, 18, 2697, 1058, 3392, 179, 3023, 581, 21, 1037, 2440, 416, 341, 197, 2577, 1067, 1841, 0, 3123, 2006, 9, 976, 1344, 757, 3386, 48, 1958, 1, 19, 242, 3, 535, 3, 155, 795, 1563, 3008, 826, 1093, 2803] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2707, 6, 201, 37, 430, 0, 2653, 8, 573, 3002, 76, 413, 1365, 1340, 0, 197, 306, 1136, 296, 1609, 0, 926, 1067, 1575, 0, 19, 2870, 1229, 3413, 175, 2756, 936, 24, 735, 3063, 108, 187, 134, 2736, 1553, 1554, 0, 2815, 1555, 0, 774, 1595, 849, 3250, 92, 1626, 0, 11, 317, 0, 419, 0, 61, 637, 1680, 3602, 1082, 876, 2880] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2736, 42, 231, 48, 239, 0, 2051, 74, 1027, 3174, 77, 188, 1371, 1135, 0, 115, 244, 1076, 67, 1838, 3, 696, 966, 1727, 8, 14, 2743, 1295, 3238, 206, 3162, 795, 10, 1330, 2837, 490, 496, 58, 2385, 1178, 2126, 0, 3045, 1666, 10, 871, 1227, 550, 3361, 24, 1658, 0, 24, 162, 0, 667, 0, 222, 810, 1786, 3474, 410, 1063, 3010] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2784, 24, 273, 44, 119, 0, 2551, 14, 832, 3449, 123, 278, 1145, 2249, 0, 243, 345, 1118, 86, 2012, 0, 690, 1231, 1484, 4, 4, 2694, 1418, 3477, 276, 3111, 300, 26, 721, 2193, 276, 155, 64, 2117, 1728, 1333, 0, 3107, 2350, 0, 916, 1030, 1088, 2842, 72, 1898, 2, 0, 259, 0, 355, 0, 169, 716, 1368, 3353, 1349, 1125, 2546] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2494, 22, 197, 78, 398, 1, 2456, 22, 769, 3284, 116, 182, 799, 1872, 0, 276, 477, 1383, 167, 1682, 0, 599, 973, 1454, 3, 9, 2942, 1405, 3337, 115, 2935, 562, 4, 814, 2826, 263, 150, 97, 2390, 1171, 2237, 0, 2817, 2122, 1, 1020, 1445, 892, 3375, 37, 2017, 1, 17, 297, 9, 300, 2, 392, 457, 1406, 3200, 904, 1118, 2746] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2622, 5, 101, 90, 373, 1, 2550, 9, 436, 3511, 177, 371, 895, 1748, 0, 280, 239, 1541, 143, 1523, 0, 771, 894, 1454, 0, 40, 3203, 1655, 3160, 81, 2453, 414, 26, 1254, 2775, 232, 188, 137, 2659, 1576, 1807, 0, 2905, 1624, 2, 762, 1336, 1142, 3569, 49, 1598, 5, 3, 244, 0, 483, 0, 177, 698, 1655, 2997, 1266, 1071, 2556] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2151, 32, 284, 49, 334, 0, 2335, 31, 270, 2934, 55, 314, 1434, 1071, 0, 391, 673, 1324, 114, 1942, 0, 499, 1421, 1763, 18, 7, 2821, 1211, 3201, 116, 2780, 792, 34, 1140, 2651, 216, 214, 78, 2758, 953, 2571, 1, 2849, 1925, 0, 693, 1565, 828, 3204, 54, 1774, 1, 4, 146, 0, 731, 0, 62, 565, 1767, 3382, 1024, 1147, 2832] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2736, 42, 231, 48, 239, 0, 2051, 74, 2515, 0, 0, 0, 0, 0, 0, 0, 1027, 3174, 77, 188, 1371, 1135, 0, 115, 873, 51, 0, 0, 0, 0, 0, 0, 244, 1076, 67, 1838, 3, 696, 966, 1727, 904, 466, 0, 0, 0, 0, 0, 0, 8, 14, 2743, 1295, 318, 206, 3162, 795, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1330, 2786, 490, 496, 58, 2385, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 2126, 0, 1268, 1666, 10, 871, 1227, 550, 0, 0, 0, 0, 0, 0, 0, 0, 3361, 24, 1658, 0, 24, 162, 0, 667, 2050, 405, 0, 0, 0, 0, 0, 0, 0, 222, 810, 1786, 958, 410, 1063, 3010, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7936, 8011, 7987, 8541, 8733, 7718, 8351, 8259] avg=8192.0000 max=8733 max_rank=4 min=7718 min_rank=5 imbalance_ratio=1.066040 max_load_reduction=56803 max_load_reduction_pct=0.866745 moved_to_echo=33061 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2726, 30, 166, 53, 324, 1, 1835, 69, 2202, 0, 0, 0, 0, 0, 0, 0, 835, 2898, 112, 257, 865, 1649, 1, 310, 983, 43, 0, 0, 0, 0, 0, 0, 510, 1358, 151, 1781, 0, 511, 1248, 1711, 1018, 457, 0, 0, 0, 0, 0, 0, 28, 7, 2897, 1001, 279, 85, 3136, 668, 0, 0, 0, 0, 0, 0, 0, 0, 26, 895, 2373, 105, 538, 150, 3430, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 1954, 0, 1427, 1994, 1, 971, 1015, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 3090, 26, 1683, 1, 3, 170, 2, 551, 2013, 355, 0, 0, 0, 0, 0, 0, 0, 97, 740, 1304, 941, 1249, 1172, 2818, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7406, 7953, 8745, 8101, 8570, 8546, 7894, 8321] avg=8192.0000 max=8745 max_rank=2 min=7406 min_rank=0 imbalance_ratio=1.067505 max_load_reduction=56791 max_load_reduction_pct=0.866562 moved_to_echo=33331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2571, 18, 107, 92, 550, 0, 2801, 49, 2634, 0, 0, 0, 0, 0, 0, 0, 1091, 3374, 84, 449, 1274, 1269, 0, 214, 896, 44, 0, 0, 0, 0, 0, 0, 480, 816, 106, 1865, 0, 507, 880, 1375, 927, 403, 0, 0, 0, 0, 0, 0, 26, 18, 2697, 1058, 334, 179, 3023, 581, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1037, 2396, 416, 341, 197, 2577, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 1841, 0, 1300, 2006, 9, 976, 1344, 757, 0, 0, 0, 0, 0, 0, 0, 0, 3386, 48, 1958, 1, 19, 242, 3, 535, 1775, 424, 0, 0, 0, 0, 0, 0, 3, 155, 795, 1563, 830, 826, 1093, 2803, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8822, 8695, 7359, 7916, 8052, 8233, 8391, 8068] avg=8192.0000 max=8822 max_rank=0 min=7359 min_rank=2 imbalance_ratio=1.076904 max_load_reduction=56714 max_load_reduction_pct=0.865387 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2784, 24, 273, 44, 119, 0, 2551, 14, 2700, 0, 0, 0, 0, 0, 0, 0, 832, 3449, 123, 278, 1145, 2249, 0, 243, 891, 39, 0, 0, 0, 0, 0, 0, 345, 1118, 86, 2012, 0, 690, 1231, 1484, 923, 449, 0, 0, 0, 0, 0, 0, 4, 4, 2694, 1418, 342, 276, 3111, 300, 0, 0, 0, 0, 0, 0, 0, 0, 26, 721, 2154, 276, 155, 64, 2117, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 0, 1293, 2350, 0, 916, 1030, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 2842, 72, 1898, 2, 0, 259, 0, 355, 1979, 435, 0, 0, 0, 0, 0, 0, 0, 169, 716, 1368, 925, 1349, 1125, 2546, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8509, 9249, 8338, 8149, 7241, 8010, 7842, 8198] avg=8192.0000 max=9249 max_rank=1 min=7241 min_rank=4 imbalance_ratio=1.129028 max_load_reduction=56287 max_load_reduction_pct=0.858871 moved_to_echo=31291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2494, 22, 197, 78, 398, 1, 2456, 22, 2591, 0, 0, 0, 0, 0, 0, 0, 769, 3284, 116, 182, 799, 1872, 0, 276, 808, 51, 0, 0, 0, 0, 0, 0, 477, 1383, 167, 1682, 0, 599, 973, 1454, 837, 429, 0, 0, 0, 0, 0, 0, 3, 9, 2942, 1405, 329, 115, 2935, 562, 0, 0, 0, 0, 0, 0, 0, 0, 4, 814, 2775, 263, 150, 97, 2390, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 2237, 0, 1172, 2122, 1, 1020, 1445, 892, 0, 0, 0, 0, 0, 0, 0, 0, 3375, 37, 2017, 1, 17, 297, 9, 300, 1888, 417, 0, 0, 0, 0, 0, 0, 2, 392, 457, 1406, 883, 904, 1118, 2746, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8157, 8001, 8300, 7664, 8889, 8358, 7908] avg=8192.0000 max=8889 max_rank=5 min=7664 min_rank=4 imbalance_ratio=1.085083 max_load_reduction=56647 max_load_reduction_pct=0.864365 moved_to_echo=32819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2707, 6, 201, 37, 430, 0, 2653, 8, 2650, 0, 0, 0, 0, 0, 0, 0, 573, 3002, 76, 413, 1365, 1340, 0, 197, 807, 55, 0, 0, 0, 0, 0, 0, 306, 1136, 296, 1609, 0, 926, 1067, 1575, 836, 483, 0, 0, 0, 0, 0, 0, 0, 19, 2870, 1229, 336, 175, 2756, 936, 0, 0, 0, 0, 0, 0, 0, 0, 24, 735, 3008, 108, 187, 134, 2736, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 1554, 0, 1172, 1555, 0, 774, 1595, 849, 0, 0, 0, 0, 0, 0, 0, 0, 3250, 92, 1626, 0, 11, 317, 0, 419, 2126, 427, 0, 0, 0, 0, 0, 0, 0, 61, 637, 1680, 993, 1082, 876, 2880, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8692, 7828, 8234, 8321, 8485, 7499, 8268, 8209] avg=8192.0000 max=8692 max_rank=0 min=7499 min_rank=5 imbalance_ratio=1.061035 max_load_reduction=56844 max_load_reduction_pct=0.867371 moved_to_echo=32461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2622, 5, 101, 90, 373, 1, 2550, 9, 2454, 0, 0, 0, 0, 0, 0, 0, 436, 3511, 177, 371, 895, 1748, 0, 280, 833, 50, 0, 0, 0, 0, 0, 0, 239, 1541, 143, 1523, 0, 771, 894, 1454, 863, 402, 0, 0, 0, 0, 0, 0, 0, 40, 3203, 1655, 311, 81, 2453, 414, 0, 0, 0, 0, 0, 0, 0, 0, 26, 1254, 2725, 232, 188, 137, 2659, 1576, 0, 0, 0, 0, 0, 0, 0, 0, 1807, 0, 1209, 1624, 2, 762, 1336, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 49, 1598, 5, 3, 244, 0, 483, 1769, 395, 0, 0, 0, 0, 0, 0, 0, 177, 698, 1655, 826, 1266, 1071, 2556, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8301, 7830, 8157, 8797, 7882, 8115, 8249] avg=8192.0000 max=8797 max_rank=4 min=7830 min_rank=2 imbalance_ratio=1.073853 max_load_reduction=56739 max_load_reduction_pct=0.865768 moved_to_echo=33043 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2151, 32, 284, 49, 334, 0, 2335, 31, 2491, 0, 0, 0, 0, 0, 0, 0, 270, 2934, 55, 314, 1434, 1071, 0, 391, 823, 50, 0, 0, 0, 0, 0, 0, 673, 1324, 114, 1942, 0, 499, 1421, 1763, 850, 456, 0, 0, 0, 0, 0, 0, 18, 7, 2821, 1211, 306, 116, 2780, 792, 0, 0, 0, 0, 0, 0, 0, 0, 34, 1140, 2601, 216, 214, 78, 2758, 953, 0, 0, 0, 0, 0, 0, 0, 0, 2571, 1, 1176, 1925, 0, 693, 1565, 828, 0, 0, 0, 0, 0, 0, 0, 0, 3204, 54, 1774, 1, 4, 146, 0, 731, 1999, 404, 0, 0, 0, 0, 0, 0, 0, 62, 565, 1767, 927, 1024, 1147, 2832, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7707, 7342, 9042, 8051, 7994, 8759, 8317, 8324] avg=8192.0000 max=9042 max_rank=2 min=7342 min_rank=1 imbalance_ratio=1.103760 max_load_reduction=56494 max_load_reduction_pct=0.862030 moved_to_echo=33394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[15488, 128, 10112, 15360, 128, 7040, 10624, 7296, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[20864, 256, 1664, 512, 2816, 128, 19328, 384, 20352, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 1408, 5504, 12544, 7296, 8192, 8704, 22272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[5888, 25728, 896, 2560, 9216, 12416, 128, 2048, 7040, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3328, 9856, 1152, 14336, 128, 5248, 8704, 12544, 7168, 3584, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 22912, 10368, 2560, 1280, 23424, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 7936, 20864, 2176, 2304, 1024, 21120, 10368, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[26112, 512, 14336, 128, 128, 1920, 128, 4096, 15616, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 0, 4037, 0, 3943, 4092, 1, 16, 387, 0, 1, 1431, 4094, 1, 0, 0, 0, 1, 6, 40, 832, 4061, 3, 4088, 0, 23, 0, 4084, 0, 0, 2, 0, 2200, 1, 0, 905, 4094, 2, 0, 0, 0, 1, 0, 4095, 0, 0, 1, 7, 4095, 0, 15, 0, 4096, 4095, 2715, 2, 0, 1, 1, 3970] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 3, 4085, 0, 4055, 4095, 0, 324, 443, 0, 0, 2665, 4068, 0, 1, 0, 0, 0, 0, 32, 693, 4057, 3, 4093, 3, 9, 0, 4053, 3, 0, 1, 0, 837, 2, 0, 158, 4092, 5, 0, 0, 0, 0, 0, 4095, 1, 4, 3, 1, 4095, 0, 38, 0, 4094, 4094, 3282, 1, 0, 0, 0, 3955] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 16, 4069, 0, 4028, 4096, 0, 128, 623, 0, 0, 1197, 4003, 0, 0, 0, 0, 0, 0, 68, 964, 4071, 0, 4093, 0, 35, 0, 4095, 0, 0, 0, 0, 2216, 12, 0, 798, 4082, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 4096, 0, 19, 0, 4094, 4071, 2415, 0, 0, 0, 0, 4049] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 141, 536, 0, 0, 1974, 4080, 0, 0, 0, 0, 0, 1, 34, 986, 4067, 0, 4091, 2, 3, 0, 4074, 2, 0, 1, 0, 1304, 0, 0, 424, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4095, 0, 44, 0, 4096, 4096, 2816, 0, 0, 0, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 2, 0, 4093, 2, 4069, 0, 4008, 4095, 0, 170, 435, 0, 2, 2417, 4090, 0, 0, 2, 0, 0, 0, 1, 533, 4069, 7, 4092, 3, 1, 7, 4053, 4, 0, 3, 0, 1274, 0, 0, 361, 4086, 0, 0, 0, 1, 0, 0, 4096, 3, 0, 0, 3, 4091, 0, 6, 0, 4096, 4092, 3406, 0, 0, 0, 0, 3862] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4081, 0, 4050, 4096, 0, 131, 332, 0, 0, 2560, 4093, 0, 0, 0, 0, 0, 0, 229, 620, 4055, 3, 4081, 0, 31, 0, 4036, 4, 0, 0, 0, 851, 1, 0, 362, 4094, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 105, 0, 4096, 4096, 3528, 0, 0, 0, 0, 3710] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4096, 4, 4096, 0, 4059, 4096, 0, 746, 85, 0, 0, 2529, 4092, 0, 0, 6, 0, 0, 0, 47, 1093, 4079, 0, 4091, 0, 1, 1, 3829, 3, 1, 0, 0, 725, 4, 2, 94, 4095, 21, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 6, 4096, 0, 22, 0, 4089, 4090, 3159, 0, 0, 0, 0, 4081] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4045, 0, 4061, 4092, 0, 77, 431, 2, 0, 1179, 4095, 0, 0, 0, 0, 0, 3, 7, 1052, 4085, 4, 4096, 1, 16, 0, 4089, 0, 1, 0, 0, 2772, 0, 0, 723, 4088, 0, 1, 0, 0, 0, 0, 4096, 0, 2, 0, 8, 4096, 0, 9, 0, 4079, 4096, 2302, 0, 0, 0, 0, 3833] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 16, 4069, 0, 20, 0, 0, 0, 0, 0, 0, 0, 4028, 1535, 0, 128, 623, 0, 0, 1197, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 68, 2561, 759, 731, 0, 0, 0, 0, 0, 964, 3291, 0, 0, 0, 35, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2216, 12, 0, 798, 4094, 2094, 0, 0, 0, 0, 0, 0, 4082, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 19, 0, 2051, 1999, 0, 0, 0, 0, 0, 0, 0, 1289, 2415, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 7511, 8122, 8385, 9214, 8180, 8170, 7753] avg=8192.0000 max=9214 max_rank=4 min=7511 min_rank=1 imbalance_ratio=1.124756 max_load_reduction=56322 max_load_reduction_pct=0.859406 moved_to_echo=33317 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4096, 0, 4037, 0, 20, 0, 0, 0, 0, 0, 0, 0, 3943, 1534, 1, 16, 387, 0, 1, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 1, 6, 40, 2558, 757, 736, 0, 0, 0, 0, 0, 832, 3283, 3, 0, 0, 23, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2200, 1, 0, 905, 4096, 2092, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 4095, 0, 15, 0, 2063, 1996, 0, 0, 0, 0, 0, 0, 0, 1296, 2715, 2, 0, 1, 1, 3970, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 7313, 8193, 8225, 9296, 8193, 8177, 7985] avg=8192.0000 max=9296 max_rank=4 min=7313 min_rank=1 imbalance_ratio=1.134766 max_load_reduction=56240 max_load_reduction_pct=0.858154 moved_to_echo=33651 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 0, 20, 0, 0, 0, 0, 0, 0, 0, 4095, 1535, 0, 141, 536, 0, 0, 1974, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 1, 34, 2561, 759, 736, 0, 0, 0, 0, 0, 986, 3287, 0, 0, 2, 3, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 1304, 0, 0, 424, 4096, 2093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 44, 0, 2063, 1998, 0, 0, 0, 0, 0, 0, 0, 1297, 2816, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8281, 8171, 8352, 7920, 8193, 8201, 8206] avg=8192.0000 max=8352 max_rank=3 min=7920 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4096, 4, 4096, 0, 20, 0, 0, 0, 0, 0, 0, 0, 4059, 1535, 0, 746, 85, 0, 0, 2529, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 6, 0, 0, 0, 47, 2561, 761, 735, 0, 0, 0, 0, 0, 1093, 3297, 0, 0, 0, 1, 1, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 725, 4, 2, 94, 4089, 2093, 0, 0, 0, 0, 0, 0, 4095, 21, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 4096, 0, 22, 0, 2060, 1998, 0, 0, 0, 0, 0, 0, 0, 1295, 3159, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8954, 8202, 8221, 7011, 8212, 8183, 8535] avg=8192.0000 max=8954 max_rank=1 min=7011 min_rank=4 imbalance_ratio=1.093018 max_load_reduction=56582 max_load_reduction_pct=0.863373 moved_to_echo=31941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2, 0, 4093, 2, 4069, 0, 20, 0, 0, 0, 0, 0, 0, 0, 4008, 1535, 0, 170, 435, 0, 2, 2417, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 2, 0, 0, 0, 1, 2560, 759, 735, 0, 0, 0, 0, 0, 533, 3289, 7, 0, 3, 1, 7, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 0, 1274, 0, 0, 361, 4096, 2094, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 1, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 4091, 0, 6, 0, 2061, 1998, 0, 0, 0, 0, 0, 0, 0, 1296, 3406, 0, 0, 0, 0, 3862, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8567, 8147, 7893, 7832, 8184, 8162, 8564] avg=8192.0000 max=8567 max_rank=1 min=7832 min_rank=4 imbalance_ratio=1.045776 max_load_reduction=56969 max_load_reduction_pct=0.869278 moved_to_echo=32742 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 3, 4085, 0, 20, 0, 0, 0, 0, 0, 0, 0, 4055, 1535, 0, 324, 443, 0, 0, 2665, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 1, 0, 0, 0, 0, 32, 2560, 757, 735, 0, 0, 0, 0, 0, 693, 3279, 3, 0, 3, 9, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 837, 2, 0, 158, 4094, 2094, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4, 3, 1, 4095, 0, 38, 0, 2062, 1999, 0, 0, 0, 0, 0, 0, 0, 1297, 3282, 1, 0, 0, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 9022, 8153, 8040, 7189, 8193, 8203, 8535] avg=8192.0000 max=9022 max_rank=1 min=7189 min_rank=4 imbalance_ratio=1.101318 max_load_reduction=56514 max_load_reduction_pct=0.862335 moved_to_echo=32120 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4081, 0, 20, 0, 0, 0, 0, 0, 0, 0, 4050, 1535, 0, 131, 332, 0, 0, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 229, 2561, 756, 736, 0, 0, 0, 0, 0, 620, 3278, 3, 0, 0, 31, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 851, 1, 0, 362, 4096, 2089, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 105, 0, 2063, 1992, 0, 0, 0, 0, 0, 0, 0, 1297, 3528, 0, 0, 0, 0, 3710, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8608, 8375, 7968, 7403, 8193, 8257, 8535] avg=8192.0000 max=8608 max_rank=1 min=7403 min_rank=4 imbalance_ratio=1.050781 max_load_reduction=56928 max_load_reduction_pct=0.868652 moved_to_echo=32388 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4045, 0, 26, 0, 0, 0, 0, 0, 0, 0, 4061, 1530, 0, 77, 431, 2, 0, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3, 7, 2562, 766, 740, 0, 0, 0, 0, 0, 1052, 3290, 4, 0, 1, 16, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2772, 0, 0, 723, 4079, 2096, 0, 0, 0, 0, 0, 0, 4088, 0, 1, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8, 4096, 0, 9, 0, 2068, 2000, 0, 0, 0, 0, 0, 0, 0, 1288, 2302, 0, 0, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 7280, 8173, 8452, 9671, 8188, 8183, 7423] avg=8192.0000 max=9671 max_rank=4 min=7280 min_rank=1 imbalance_ratio=1.180542 max_load_reduction=55865 max_load_reduction_pct=0.852432 moved_to_echo=33465 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 32640, 0, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 10368, 23680, 128, 0, 128, 128, 31616, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6784, 26368, 128, 0, 128, 128, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 384, 0, 16512, 16000, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32384, 12288, 128, 1792, 3328, 128, 128, 16000, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32640, 128, 128, 128, 0, 128, 128, 512, 20608, 6144, 5888, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 12288, 128, 128, 3840, 32768, 16768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 4095, 0, 4095, 0, 1, 4095, 0, 1, 4095, 1, 1, 4092, 0, 0, 375, 43, 4090, 0, 4095, 1, 0, 0, 3884, 4093, 1, 0, 4095, 7, 4094, 1, 1, 0, 580, 4095, 0, 748, 0, 16, 1, 4095, 1, 1, 0, 0, 1, 4095, 1, 1, 2545, 0, 0, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 2, 0, 0, 4096, 0, 0, 0, 4085, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 4088, 5, 0, 251, 36, 4090, 0, 4092, 0, 0, 0, 1747, 4096, 0, 1, 4096, 15, 4096, 0, 0, 0, 11, 4094, 0, 3453, 0, 7, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 2693, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4063, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3989, 72, 0, 403, 140, 4096, 0, 4088, 0, 0, 0, 3036, 4096, 0, 0, 4084, 0, 4096, 0, 0, 0, 18, 4095, 0, 1632, 0, 33, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3016, 0, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4084, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4087, 5, 0, 239, 97, 4094, 0, 4087, 0, 0, 0, 2810, 4096, 0, 4, 4096, 8, 4095, 0, 0, 0, 4, 4096, 0, 2418, 0, 12, 0, 4096, 0, 0, 0, 3, 0, 4094, 0, 0, 2623, 0, 2, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 5, 4096, 0, 0, 0, 4067, 0, 4096, 0, 0, 4096, 3, 0, 4096, 0, 0, 4084, 18, 0, 273, 70, 4093, 0, 4086, 0, 0, 0, 3200, 4096, 0, 4, 4096, 2, 4096, 0, 0, 0, 52, 4090, 0, 1377, 0, 46, 0, 4093, 0, 0, 0, 9, 0, 4092, 0, 0, 3196, 0, 1, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 2, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 263, 0, 4090, 0, 4096, 0, 0, 0, 3350, 4095, 1, 0, 4096, 11, 4095, 0, 0, 0, 8, 4093, 0, 1525, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3043, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4046, 4, 0, 310, 97, 4095, 0, 4090, 0, 0, 0, 3376, 4096, 0, 1, 4096, 0, 4096, 0, 0, 0, 34, 4096, 0, 2414, 0, 20, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2003, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 4078, 0, 4096, 0, 0, 4096, 1, 0, 4095, 0, 0, 4047, 17, 0, 201, 67, 4096, 0, 4081, 0, 0, 0, 3678, 4096, 0, 8, 4083, 2, 4096, 0, 0, 0, 71, 4087, 0, 872, 0, 17, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 3358, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 4095, 0, 2796, 1296, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4095, 0, 1, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4092, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 375, 43, 3742, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 956, 1, 0, 4095, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 580, 4095, 0, 748, 0, 16, 1841, 335, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2545, 0, 0, 0, 2, 1, 4094, 1284, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8207, 8203, 8256, 8944, 7616, 8194, 7928] avg=8192.0000 max=8944 max_rank=4 min=7616 min_rank=5 imbalance_ratio=1.091797 max_load_reduction=56592 max_load_reduction_pct=0.863525 moved_to_echo=32682 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 2796, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3989, 72, 0, 13, 1, 0, 0, 0, 0, 0, 0, 403, 140, 3748, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3036, 957, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4095, 0, 1632, 0, 33, 1842, 335, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3016, 0, 3, 0, 0, 0, 4096, 1285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8174, 8171, 8379, 8077, 7955, 8191, 8400] avg=8192.0000 max=8400 max_rank=7 min=7955 min_rank=5 imbalance_ratio=1.025391 max_load_reduction=57136 max_load_reduction_pct=0.871826 moved_to_echo=32623 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 2, 0, 0, 4096, 0, 2796, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4088, 5, 0, 13, 1, 0, 0, 0, 0, 0, 0, 251, 36, 3742, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1747, 957, 0, 1, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4094, 0, 3453, 0, 7, 1842, 335, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2693, 0, 0, 0, 0, 0, 4096, 1285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8196, 8202, 8121, 6816, 9742, 8192, 8074] avg=8192.0000 max=9742 max_rank=5 min=6816 min_rank=4 imbalance_ratio=1.189209 max_load_reduction=55794 max_load_reduction_pct=0.851349 moved_to_echo=32824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 2796, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 263, 0, 3742, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3350, 956, 1, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4093, 0, 1525, 0, 0, 1842, 335, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3043, 0, 0, 0, 1, 0, 4095, 1285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8208, 8206, 8101, 8414, 7803, 8191, 8424] avg=8192.0000 max=8424 max_rank=7 min=7803 min_rank=5 imbalance_ratio=1.028320 max_load_reduction=57112 max_load_reduction_pct=0.871460 moved_to_echo=32832 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 5, 4096, 0, 2796, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4084, 18, 0, 13, 0, 0, 0, 0, 0, 0, 0, 273, 70, 3745, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3200, 957, 0, 4, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4090, 0, 1377, 0, 46, 1842, 335, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 9, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3196, 0, 1, 0, 1, 0, 4096, 1285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8178, 8214, 8174, 8259, 7742, 8194, 8579] avg=8192.0000 max=8579 max_rank=7 min=7742 min_rank=5 imbalance_ratio=1.047241 max_load_reduction=56957 max_load_reduction_pct=0.869095 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 4096, 0, 2796, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 5, 0, 13, 1, 0, 0, 0, 0, 0, 0, 239, 97, 3746, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2810, 957, 0, 4, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 2418, 0, 12, 1842, 335, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2623, 0, 2, 0, 1, 0, 4095, 1285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8195, 8202, 8169, 7875, 8707, 8192, 8006] avg=8192.0000 max=8707 max_rank=5 min=7875 min_rank=4 imbalance_ratio=1.062866 max_load_reduction=56829 max_load_reduction_pct=0.867142 moved_to_echo=32780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 2796, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4046, 4, 0, 13, 1, 0, 0, 0, 0, 0, 0, 310, 97, 3747, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3376, 957, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4096, 0, 2414, 0, 20, 1842, 335, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 0, 0, 0, 0, 0, 4096, 1285, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8197, 8160, 8244, 8430, 8741, 8191, 7384] avg=8192.0000 max=8741 max_rank=5 min=7384 min_rank=7 imbalance_ratio=1.067017 max_load_reduction=56795 max_load_reduction_pct=0.866623 moved_to_echo=32746 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 0, 4096, 0, 2803, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4047, 17, 0, 15, 3, 0, 0, 0, 0, 0, 0, 201, 67, 3743, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 950, 0, 8, 4083, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 4087, 0, 872, 0, 17, 1845, 338, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3358, 0, 1, 0, 0, 0, 4096, 1286, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8181, 8178, 8092, 8721, 7230, 8191, 8741] avg=8192.0000 max=8741 max_rank=7 min=7230 min_rank=5 imbalance_ratio=1.067017 max_load_reduction=56795 max_load_reduction_pct=0.866623 moved_to_echo=32883 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 32640, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25088, 7680, 128, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2432, 640, 30080, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 0, 128, 32768, 0, 22400, 10496, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 896, 32768, 0, 14464, 0, 256, 14848, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 22528, 0, 128, 0, 128, 128, 32768, 10368, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4086, 1, 1, 4095, 4093, 4087, 0, 1, 0, 4089, 0, 4095, 4094, 0, 0, 4095, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 1, 1, 4095, 4, 0, 0, 0, 0, 7, 1, 4093, 0, 0, 0, 4092, 4095, 0, 1, 4095, 1, 0, 22, 0, 4088, 4095, 5, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4044, 3908, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 147, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4049, 86, 0, 42, 0, 4096, 4081, 27, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 4083, 4076, 0, 0, 0, 4096, 0, 4096, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 23, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 4096, 0, 0, 4065, 22, 1, 16, 0, 4096, 4093, 15, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4094, 4080, 4073, 0, 0, 0, 4096, 0, 4096, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 4096, 0, 9, 3, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4091, 28, 0, 13, 0, 4096, 4086, 9, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4092, 4079, 4025, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 4096, 0, 65, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4081, 26, 5, 13, 0, 4096, 4079, 18, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4095, 0, 4096, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 4067, 19, 0, 20, 0, 4092, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4080, 4059, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 10, 7, 0, 0, 3, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4084, 33, 0, 18, 0, 4096, 4085, 9, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4070, 4038, 0, 0, 0, 4095, 0, 4096, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 26, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4045, 42, 0, 73, 0, 4096, 4068, 22, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4044, 3908, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 147, 0, 0, 4025, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 4049, 86, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4081, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8022, 8194, 8193, 8300, 8192, 8239, 8203] avg=8192.0000 max=8300 max_rank=4 min=8022 min_rank=1 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32934 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4083, 4076, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 4025, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 4065, 22, 1, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8229, 8188, 8193, 8176, 8189, 8166, 8203] avg=8192.0000 max=8229 max_rank=1 min=8166 min_rank=6 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32734 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4086, 1, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4093, 4087, 0, 1, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 1, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4, 0, 0, 0, 4018, 32, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4093, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 1, 4095, 1, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4095, 5, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8251, 8194, 8193, 8151, 8193, 8180, 8189] avg=8192.0000 max=8251 max_rank=1 min=8151 min_rank=4 imbalance_ratio=1.007202 max_load_reduction=57285 max_load_reduction_pct=0.874100 moved_to_echo=32713 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4024, 32, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 4067, 19, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8262, 8191, 8193, 8152, 8194, 8168, 8188] avg=8192.0000 max=8262 max_rank=1 min=8152 min_rank=4 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 4079, 4025, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 1, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 65, 4, 0, 4025, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 4081, 26, 5, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4079, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8174, 8189, 8187, 8226, 8192, 8187, 8192] avg=8192.0000 max=8226 max_rank=4 min=8174 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=57310 max_load_reduction_pct=0.874481 moved_to_echo=32797 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4080, 4073, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 9, 3, 0, 4025, 32, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 4091, 28, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4086, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8223, 8190, 8189, 8167, 8193, 8194, 8190] avg=8192.0000 max=8223 max_rank=1 min=8167 min_rank=4 imbalance_ratio=1.003784 max_load_reduction=57313 max_load_reduction_pct=0.874527 moved_to_echo=32744 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4080, 4059, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 7, 0, 4025, 32, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 4084, 33, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4085, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8209, 8194, 8189, 8170, 8195, 8197, 8189] avg=8192.0000 max=8209 max_rank=1 min=8170 min_rank=4 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4, 6, 4, 0, 0, 0, 0, 0, 4070, 4038, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 26, 4, 0, 4030, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 4045, 42, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4068, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8166, 8196, 8199, 8194, 8188, 8205, 8182] avg=8192.0000 max=8206 max_rank=0 min=8166 min_rank=1 imbalance_ratio=1.001709 max_load_reduction=57330 max_load_reduction_pct=0.874786 moved_to_echo=32769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 32768, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32640, 32384, 0, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 32768, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 384, 128, 0, 32256, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32512, 0, 128, 32640, 384, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 32768, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 4055, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 0, 0, 0, 3787, 0, 0, 35, 4096, 375, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4034, 0, 4096, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 1, 0, 4096, 0, 4087, 1, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 1, 21, 0, 4095, 1, 0, 0, 0, 4095, 1, 0, 0, 4093, 22, 0, 12, 4095, 1, 1, 4095, 4095, 4095, 1, 0, 1, 1, 0, 0, 4094, 1, 0, 0, 0, 4093, 0, 4052, 1, 4095, 4095, 0, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 8, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 1, 0, 4076, 0, 0, 6, 4096, 30, 1, 8, 4096, 5, 0, 4096, 4088, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4075, 0, 4092, 4090, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 9, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 17, 0, 1, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4071, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 11, 0, 0, 0, 4085, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 9, 0, 4096, 0, 1, 0, 0, 4082, 0, 0, 8, 4096, 54, 0, 1, 4096, 11, 0, 4096, 4089, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 4050, 0, 4096, 4086, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 8, 0, 0, 0, 4089, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 4096, 0, 0, 1, 0, 4023, 0, 0, 1, 4096, 81, 0, 15, 4096, 5, 0, 4096, 4085, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4083, 0, 4096, 4087, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 11, 0, 0, 0, 4086, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 8, 0, 4096, 0, 0, 6, 0, 4019, 0, 0, 10, 4096, 316, 0, 0, 4096, 4, 0, 4096, 4089, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3930, 0, 4096, 4087, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 11, 0, 0, 0, 4067, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 4096, 0, 3, 0, 0, 4028, 0, 0, 15, 4096, 343, 0, 9, 4096, 27, 0, 4096, 4087, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4096, 0, 3815, 0, 4094, 4069, 0, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 4007, 82, 0, 0, 0, 0, 0, 0, 4055, 0, 4095, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3787, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 35, 4096, 375, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 89, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8175, 8189, 7951, 8427, 8198, 8192, 8217] avg=8192.0000 max=8427 max_rank=4 min=7951 min_rank=3 imbalance_ratio=1.028687 max_load_reduction=57109 max_load_reduction_pct=0.871414 moved_to_echo=33034 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 1, 0, 4006, 82, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 1, 1, 1, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 21, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4095, 1, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 22, 0, 12, 3920, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4095, 4095, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 1, 89, 4095, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8211, 8206, 8260, 8049, 8199, 8188, 8239] avg=8192.0000 max=8260 max_rank=3 min=8049 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32675 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 11, 0, 0, 0, 4007, 82, 0, 0, 0, 0, 0, 0, 4085, 0, 4092, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 9, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4082, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 54, 0, 1, 3921, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 89, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8202, 8191, 8247, 8091, 8191, 8193, 8225] avg=8192.0000 max=8247 max_rank=3 min=8091 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=57289 max_load_reduction_pct=0.874161 moved_to_echo=32700 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 0, 0, 0, 4007, 82, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 9, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 1, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 89, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8215, 8194, 8260, 8035, 8198, 8192, 8256] avg=8192.0000 max=8260 max_rank=3 min=8035 min_rank=4 imbalance_ratio=1.008301 max_load_reduction=57276 max_load_reduction_pct=0.873962 moved_to_echo=32681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 8, 0, 0, 0, 4003, 82, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4076, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 30, 1, 8, 3921, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 89, 4090, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8212, 8188, 8241, 8067, 8190, 8192, 8258] avg=8192.0000 max=8258 max_rank=7 min=8067 min_rank=4 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32707 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 11, 0, 0, 0, 4007, 82, 0, 0, 0, 0, 0, 0, 4086, 0, 4009, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 8, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4019, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 316, 0, 0, 3921, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 89, 4087, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8120, 8191, 8189, 8347, 8191, 8192, 8110] avg=8192.0000 max=8347 max_rank=4 min=8110 min_rank=7 imbalance_ratio=1.018921 max_load_reduction=57189 max_load_reduction_pct=0.872635 moved_to_echo=32840 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 8, 0, 0, 0, 4007, 82, 0, 0, 0, 0, 0, 0, 4089, 0, 4094, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4023, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 81, 0, 15, 3921, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4085, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 89, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8208, 8191, 8188, 8119, 8186, 8192, 8259] avg=8192.0000 max=8259 max_rank=7 min=8119 min_rank=4 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 11, 0, 0, 0, 4011, 84, 4, 0, 0, 0, 0, 0, 4067, 0, 4096, 0, 0, 0, 0, 0, 28, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 4028, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 343, 0, 9, 3911, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3815, 0, 83, 4069, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8193, 8186, 8200, 8401, 8183, 8195, 7972] avg=8192.0000 max=8401 max_rank=4 min=7972 min_rank=7 imbalance_ratio=1.025513 max_load_reduction=57135 max_load_reduction_pct=0.871811 moved_to_echo=32751 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 1280, 128, 128, 31360, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 0, 32256, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 0, 32768, 128, 128, 128, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 0, 0, 128, 0, 128, 0, 32128, 768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32128, 128, 768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 52, 0, 1, 11, 4096, 0, 0, 0, 4062, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 0, 4088, 4096, 0, 3426, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4095, 0, 0, 0, 0, 2416, 1, 2326, 0, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 325, 0, 0, 1, 4096, 0, 0, 0, 4005, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4075, 4096, 0, 3705, 0, 0, 4096, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1151, 0, 3111, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 61, 0, 0, 13, 4096, 0, 0, 0, 4066, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 0, 4089, 4096, 0, 2435, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 2497, 0, 3218, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 82, 0, 0, 6, 4096, 0, 0, 0, 4074, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 3947, 0, 1, 4096, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1380, 0, 2798, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 1, 0, 1, 0, 4095, 0, 0, 0, 1, 172, 0, 0, 1, 4095, 1, 0, 0, 4076, 0, 0, 1, 4095, 1, 4095, 1, 1, 0, 0, 4095, 4095, 0, 3462, 1, 4, 4095, 0, 4095, 1, 1, 0, 4095, 4094, 1, 0, 5, 1, 1, 0, 4095, 0, 0, 4095, 1, 0, 1, 0, 1824, 0, 2740, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 81, 0, 0, 45, 4096, 0, 0, 0, 4056, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 4096, 0, 932, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 3658, 0, 3523, 0, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 63, 0, 0, 14, 4096, 0, 0, 0, 4050, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 0, 4089, 4096, 0, 3316, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2961, 0, 1884, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3, 0, 0, 0, 4093, 0, 0, 0, 0, 93, 0, 1, 17, 4096, 0, 0, 0, 3992, 0, 8, 0, 4096, 0, 4096, 0, 0, 0, 0, 4084, 4096, 0, 2887, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 2956, 0, 2341, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 1, 4096, 3966, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4005, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3705, 0, 0, 4096, 1, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1151, 0, 3111, 0, 0, 2968, 121, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8388, 8146, 8180, 8885, 8192, 8192, 7351] avg=8192.0000 max=8885 max_rank=4 min=7351 min_rank=7 imbalance_ratio=1.084595 max_load_reduction=56651 max_load_reduction_pct=0.864426 moved_to_echo=32620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4095, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 172, 0, 0, 1, 4095, 3965, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4076, 0, 0, 1, 45, 0, 0, 0, 0, 0, 0, 0, 9, 1, 4095, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3462, 1, 4, 4095, 0, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 4094, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1824, 0, 2740, 0, 1, 2967, 121, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8234, 8218, 8202, 8645, 8197, 8193, 7654] avg=8192.0000 max=8645 max_rank=4 min=7654 min_rank=7 imbalance_ratio=1.055298 max_load_reduction=56891 max_load_reduction_pct=0.868088 moved_to_echo=32689 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 6, 4096, 3966, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4074, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 1, 4096, 1, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1380, 0, 2798, 0, 1, 2968, 121, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8150, 8215, 8200, 9128, 8192, 8192, 7268] avg=8192.0000 max=9128 max_rank=4 min=7268 min_rank=7 imbalance_ratio=1.114258 max_load_reduction=56408 max_load_reduction_pct=0.860718 moved_to_echo=32780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 1, 11, 4096, 3966, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4062, 0, 4, 0, 45, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3426, 0, 0, 4096, 0, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2416, 1, 2326, 0, 9, 2968, 121, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8126, 8207, 8193, 8605, 8192, 8187, 7841] avg=8192.0000 max=8605 max_rank=4 min=7841 min_rank=7 imbalance_ratio=1.050415 max_load_reduction=56931 max_load_reduction_pct=0.868698 moved_to_echo=32825 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 13, 4096, 3966, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4066, 0, 2, 0, 45, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2435, 0, 0, 4096, 0, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2497, 0, 3218, 0, 5, 2968, 121, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8136, 8209, 8194, 7614, 8192, 8190, 8809] avg=8192.0000 max=8809 max_rank=7 min=7614 min_rank=4 imbalance_ratio=1.075317 max_load_reduction=56727 max_load_reduction_pct=0.865585 moved_to_echo=32805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 45, 4096, 3966, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4056, 0, 3, 0, 45, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 0, 0, 4096, 0, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 0, 3523, 0, 10, 2968, 121, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8188, 8200, 8188, 6111, 8192, 8191, 10280] avg=8192.0000 max=10280 max_rank=7 min=6111 min_rank=4 imbalance_ratio=1.254883 max_load_reduction=55256 max_load_reduction_pct=0.843140 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 14, 4096, 3966, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4050, 0, 5, 0, 45, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3316, 0, 0, 4096, 0, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2961, 0, 1884, 0, 2, 2968, 121, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8139, 8196, 8194, 8495, 8192, 8192, 7936] avg=8192.0000 max=8495 max_rank=4 min=7936 min_rank=7 imbalance_ratio=1.036987 max_load_reduction=57041 max_load_reduction_pct=0.870377 moved_to_echo=32815 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 1, 17, 4096, 3968, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3992, 0, 8, 0, 49, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2887, 0, 0, 4096, 0, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2956, 0, 2341, 0, 6, 2971, 123, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8175, 8145, 8185, 8053, 8187, 8199, 8397] avg=8192.0000 max=8397 max_rank=7 min=8053 min_rank=4 imbalance_ratio=1.025024 max_load_reduction=57139 max_load_reduction_pct=0.871872 moved_to_echo=32836 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 24192, 128, 128, 32768, 128, 8704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 1024, 0, 128, 128, 32768, 31744, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 32384, 0, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 32768, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 18944, 128, 22016, 0, 128, 23808, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3875, 0, 0, 4032, 2343, 0, 4096, 4096, 0, 0, 4096, 0, 3923, 0, 0, 97, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 2109, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 9, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 3843, 0, 0, 4091, 3071, 4, 4095, 4094, 0, 0, 4094, 1, 4093, 0, 1, 15, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 1, 0, 0, 1, 1, 4092, 0, 0, 1, 1259, 0, 0, 0, 1, 3, 1, 4094, 2, 4095, 1, 0, 4095, 4095, 0, 0, 1, 1, 5, 0, 4095, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3942, 0, 3, 4079, 2865, 0, 4096, 4087, 0, 0, 4096, 0, 4054, 0, 0, 179, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4091, 0, 0, 0, 1259, 4, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 4096, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3536, 0, 0, 4079, 2649, 0, 4096, 4092, 0, 0, 4096, 0, 4067, 0, 0, 295, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 1730, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 14, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3991, 0, 1, 4083, 2500, 0, 4096, 4089, 0, 0, 4096, 0, 4031, 0, 0, 112, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4092, 0, 0, 0, 1655, 4, 0, 0, 0, 6, 0, 4096, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3275, 0, 0, 4093, 3838, 2, 4096, 4095, 0, 0, 4096, 0, 4096, 0, 0, 128, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 951, 0, 2, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3899, 0, 0, 4078, 3516, 0, 4096, 4096, 0, 0, 4096, 0, 4057, 0, 0, 80, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 740, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 12, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4066, 0, 1, 4066, 3199, 0, 4096, 4090, 0, 0, 4096, 0, 4016, 0, 0, 183, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 4088, 0, 0, 0, 824, 2, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 4096, 4095, 0, 0, 0, 0, 14, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3875, 0, 0, 4032, 221, 82, 8, 0, 0, 0, 0, 0, 2343, 0, 0, 4096, 0, 0, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 97, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2773, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 2776, 1318, 0, 0, 0, 0, 0, 0, 0, 2109, 0, 0, 0, 0, 0, 0, 4096, 2778, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 7541, 8034, 8187, 8186, 8983, 8194, 8193] avg=8192.0000 max=8983 max_rank=5 min=7541 min_rank=1 imbalance_ratio=1.096558 max_load_reduction=56553 max_load_reduction_pct=0.862930 moved_to_echo=33556 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 3843, 0, 0, 4091, 221, 82, 8, 0, 0, 0, 0, 0, 3071, 4, 0, 4094, 0, 0, 1102, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 15, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 2771, 1318, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 4092, 0, 0, 2777, 1319, 0, 0, 0, 0, 0, 0, 1, 1259, 0, 0, 0, 1, 3, 1, 4095, 2776, 0, 0, 0, 0, 0, 0, 4094, 2, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5, 0, 4095, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8248, 8272, 8122, 8186, 8191, 8136, 8192, 8189] avg=8192.0000 max=8272 max_rank=1 min=8122 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=57264 max_load_reduction_pct=0.873779 moved_to_echo=32708 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3536, 0, 0, 4079, 221, 82, 8, 0, 0, 0, 0, 0, 2649, 0, 0, 4092, 0, 0, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 295, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 2773, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 2776, 1318, 0, 0, 0, 0, 0, 0, 0, 1730, 0, 0, 0, 0, 0, 0, 4096, 2778, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7926, 7843, 8376, 8199, 8196, 8604, 8194, 8198] avg=8192.0000 max=8604 max_rank=5 min=7843 min_rank=1 imbalance_ratio=1.050293 max_load_reduction=56932 max_load_reduction_pct=0.868713 moved_to_echo=33192 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3275, 0, 0, 4093, 221, 82, 8, 0, 0, 0, 0, 0, 3838, 2, 0, 4095, 0, 0, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 128, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2773, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2776, 1318, 0, 0, 0, 0, 0, 0, 0, 951, 0, 2, 0, 0, 0, 0, 4096, 2778, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7679, 9037, 8238, 8187, 8190, 7827, 8193, 8185] avg=8192.0000 max=9037 max_rank=1 min=7679 min_rank=0 imbalance_ratio=1.103149 max_load_reduction=56499 max_load_reduction_pct=0.862106 moved_to_echo=32395 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3942, 0, 3, 4079, 221, 82, 8, 0, 0, 0, 0, 0, 2865, 0, 0, 4087, 0, 0, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 179, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 2773, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4091, 0, 0, 2777, 1319, 0, 0, 0, 0, 0, 0, 0, 1259, 4, 0, 0, 0, 0, 0, 4096, 2777, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8335, 8054, 8247, 8191, 8193, 8136, 8190, 8190] avg=8192.0000 max=8335 max_rank=0 min=8054 min_rank=1 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32709 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3991, 0, 1, 4083, 221, 82, 8, 0, 0, 0, 0, 0, 2500, 0, 0, 4089, 0, 0, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 112, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2773, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4092, 0, 0, 2778, 1319, 0, 0, 0, 0, 0, 0, 0, 1655, 4, 0, 0, 0, 6, 0, 4096, 2777, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8386, 7691, 8157, 8187, 8195, 8538, 8191, 8191] avg=8192.0000 max=8538 max_rank=5 min=7691 min_rank=1 imbalance_ratio=1.042236 max_load_reduction=56998 max_load_reduction_pct=0.869720 moved_to_echo=33115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3899, 0, 0, 4078, 221, 82, 8, 0, 0, 0, 0, 0, 3516, 0, 0, 4096, 0, 0, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 80, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2773, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 2777, 1319, 0, 0, 0, 0, 0, 0, 0, 740, 0, 0, 0, 0, 0, 0, 4096, 2777, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8714, 8151, 8188, 8194, 7613, 8192, 8196] avg=8192.0000 max=8714 max_rank=1 min=7613 min_rank=5 imbalance_ratio=1.063721 max_load_reduction=56822 max_load_reduction_pct=0.867035 moved_to_echo=32195 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4066, 0, 1, 4066, 227, 84, 12, 0, 0, 0, 0, 0, 3199, 0, 0, 4090, 0, 0, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 183, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 4096, 0, 0, 2774, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4088, 0, 0, 2775, 1319, 0, 0, 0, 0, 0, 0, 0, 824, 2, 0, 0, 0, 0, 0, 4096, 2777, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4096, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8456, 8384, 8211, 8211, 8191, 7699, 8190, 8194] avg=8192.0000 max=8456 max_rank=0 min=7699 min_rank=5 imbalance_ratio=1.032227 max_load_reduction=57080 max_load_reduction_pct=0.870972 moved_to_echo=32274 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 32768, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 128, 0, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[24064, 128, 0, 32768, 0, 0, 8832, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32384, 0, 128, 1152, 0, 0, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 0, 0, 22272, 10624, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 30464, 0, 128, 32640, 1792, 768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 10624, 128, 128, 0, 128, 128, 128, 32768, 22272, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 32768, 128, 0, 22272, 10624, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 1, 0, 4095, 1, 4095, 1, 1, 0, 3371, 1, 121, 4095, 0, 0, 2, 1, 0, 0, 4095, 4094, 4092, 4095, 4095, 1, 1306, 4095, 1, 4095, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4094, 1, 1, 0, 1320, 1, 0, 1, 0, 4095, 4095, 0, 1, 2073, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 3400, 0, 175, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 4037, 4096, 0, 130, 4096, 0, 4056, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2726, 0, 0, 0, 0, 4096, 4096, 0, 0, 1788, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 3340, 0, 117, 4096, 0, 0, 2, 0, 0, 0, 4093, 4096, 4096, 4087, 4096, 0, 1228, 4092, 0, 4078, 0, 0, 3, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1819, 0, 0, 0, 0, 4096, 4096, 0, 0, 1693, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 3214, 0, 34, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4088, 4096, 0, 2406, 4093, 0, 4081, 0, 4, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1259, 0, 0, 0, 0, 4096, 4096, 0, 0, 1284, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 3247, 0, 351, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4073, 4096, 0, 2242, 4096, 0, 4081, 0, 5, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1697, 0, 0, 0, 0, 4096, 4096, 0, 0, 667, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 3884, 0, 377, 4096, 0, 0, 0, 0, 0, 1, 4096, 4096, 4091, 4096, 4096, 0, 1035, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 558, 0, 0, 0, 0, 4096, 4096, 0, 0, 2332, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 3741, 0, 25, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 4079, 4096, 0, 2124, 4096, 0, 4074, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 863, 0, 0, 0, 0, 4096, 4096, 0, 0, 1453, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4082, 0, 0, 0, 2961, 0, 99, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4034, 4096, 0, 269, 4092, 0, 4066, 0, 4, 0, 0, 0, 0, 73, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2725, 0, 0, 0, 0, 4096, 4096, 0, 0, 2168, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 3400, 0, 175, 545, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 130, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 82, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2726, 0, 0, 0, 1683, 774, 19, 0, 0, 0, 0, 0, 0, 2413, 4096, 0, 0, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8207, 8192, 6944, 8214, 8192, 9298, 8297] avg=8192.0000 max=9298 max_rank=6 min=6944 min_rank=3 imbalance_ratio=1.135010 max_load_reduction=56238 max_load_reduction_pct=0.858124 moved_to_echo=34001 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 3371, 1, 121, 545, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1306, 2777, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 1320, 1, 0, 1, 1682, 773, 18, 0, 0, 0, 0, 0, 0, 2413, 4095, 0, 1, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8136, 8193, 8180, 8171, 8191, 7891, 8582] avg=8192.0000 max=8582 max_rank=7 min=7891 min_rank=6 imbalance_ratio=1.047607 max_load_reduction=56954 max_load_reduction_pct=0.869049 moved_to_echo=32835 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 3340, 0, 117, 544, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 1228, 2775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 3, 0, 0, 0, 27, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1819, 0, 0, 0, 1683, 773, 18, 0, 0, 0, 0, 0, 0, 2413, 4096, 0, 0, 1693, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8090, 8191, 8090, 8186, 8192, 8389, 8206] avg=8192.0000 max=8389 max_rank=6 min=8090 min_rank=1 imbalance_ratio=1.024048 max_load_reduction=57147 max_load_reduction_pct=0.871994 moved_to_echo=32973 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 3247, 0, 351, 545, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 2242, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 5, 0, 0, 0, 0, 28, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1697, 0, 0, 0, 1683, 774, 18, 0, 0, 0, 0, 0, 0, 2413, 4096, 0, 0, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8232, 8192, 9092, 8192, 8192, 8268, 7176] avg=8192.0000 max=9092 max_rank=3 min=7176 min_rank=7 imbalance_ratio=1.109863 max_load_reduction=56444 max_load_reduction_pct=0.861267 moved_to_echo=31828 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3884, 0, 377, 545, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1035, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 11, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 558, 0, 0, 0, 1683, 774, 18, 0, 0, 0, 0, 0, 0, 2413, 4096, 0, 0, 2332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8901, 8193, 7908, 8180, 8192, 7129, 8841] avg=8192.0000 max=8901 max_rank=1 min=7129 min_rank=6 imbalance_ratio=1.086548 max_load_reduction=56635 max_load_reduction_pct=0.864182 moved_to_echo=32342 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 3214, 0, 34, 544, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 2406, 2776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4, 0, 0, 0, 0, 23, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1259, 0, 0, 0, 1683, 773, 18, 0, 0, 0, 0, 0, 0, 2413, 4096, 0, 0, 1284, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7879, 8192, 9270, 8186, 8192, 7829, 7796] avg=8192.0000 max=9270 max_rank=3 min=7796 min_rank=7 imbalance_ratio=1.131592 max_load_reduction=56266 max_load_reduction_pct=0.858551 moved_to_echo=32003 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 3741, 0, 25, 545, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 2124, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 33, 4077, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 863, 0, 0, 0, 1683, 774, 18, 0, 0, 0, 0, 0, 0, 2413, 4096, 0, 0, 1453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8400, 8192, 8980, 8185, 8191, 7434, 7962] avg=8192.0000 max=8980 max_rank=3 min=7434 min_rank=6 imbalance_ratio=1.096191 max_load_reduction=56556 max_load_reduction_pct=0.862976 moved_to_echo=31772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 2961, 0, 99, 548, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 269, 2769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 4, 0, 0, 0, 0, 73, 4078, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2725, 0, 0, 0, 1684, 775, 18, 0, 0, 0, 0, 0, 0, 2412, 4096, 0, 0, 2168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7691, 8191, 7072, 8222, 8194, 9298, 8676] avg=8192.0000 max=9298 max_rank=6 min=7072 min_rank=3 imbalance_ratio=1.135010 max_load_reduction=56238 max_load_reduction_pct=0.858124 moved_to_echo=34390 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 19328, 32768, 0, 128, 13568, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32640, 0, 128, 10752, 22272, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 0, 32768, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 0, 128, 128, 128, 0, 0, 384, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 0, 13056, 128, 0, 128, 13568, 6272, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 128, 0, 27264, 128, 1408, 4480, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 34, 4096, 0, 0, 0, 0, 3, 0, 9, 0, 0, 0, 0, 0, 787, 4087, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3273, 4096, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 77, 4096, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 168, 4072, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3883, 4096, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4033, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 38, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 1, 0, 0, 4, 20, 0, 1, 4093, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 2333, 4095, 1, 1, 1, 5, 4095, 2, 1, 4095, 0, 1790, 4091, 0, 0, 1, 4095, 2, 0, 0, 0, 0, 4095, 0, 4083, 1, 4079, 4095, 1, 0, 0, 4095, 1, 1, 4095, 4069, 0, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1739, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2359, 4096, 1, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 32, 4096, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 648, 4088, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3410, 4096, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 4096, 4096, 0, 0, 0, 4094, 0, 0, 4095, 4096, 0, 6, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 37, 4096, 0, 0, 0, 0, 4, 0, 8, 0, 0, 0, 0, 0, 582, 4083, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3473, 4096, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4092, 4096, 0, 13, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1718, 4078, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2355, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 14, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 68, 4096, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 823, 4082, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3210, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4057, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 27, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 1, 0, 0, 4, 4074, 11, 8, 0, 0, 0, 0, 0, 20, 0, 1, 4093, 0, 1, 1, 0, 2959, 1089, 8, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 4095, 4087, 0, 0, 0, 0, 0, 0, 2333, 4095, 1, 1, 1, 5, 3006, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1125, 0, 1790, 4091, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 4079, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4087, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8172, 8185, 9444, 7008, 8192, 8185, 8158] avg=8192.0000 max=9444 max_rank=3 min=7008 min_rank=4 imbalance_ratio=1.152832 max_load_reduction=56092 max_load_reduction_pct=0.855896 moved_to_echo=31543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4075, 11, 9, 0, 0, 0, 0, 0, 0, 0, 77, 4096, 0, 0, 0, 0, 2960, 1089, 8, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 168, 4072, 0, 0, 0, 0, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 0, 3883, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8230, 8195, 7247, 9109, 8192, 8150, 8222] avg=8192.0000 max=9109 max_rank=4 min=7247 min_rank=3 imbalance_ratio=1.111938 max_load_reduction=56427 max_load_reduction_pct=0.861008 moved_to_echo=33673 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4075, 11, 9, 0, 0, 0, 0, 0, 0, 0, 34, 4096, 0, 0, 0, 0, 2960, 1089, 8, 0, 0, 0, 0, 0, 3, 0, 9, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 787, 4087, 0, 0, 0, 0, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 0, 3273, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8187, 8195, 7881, 8499, 8192, 8203, 8188] avg=8192.0000 max=8499 max_rank=4 min=7881 min_rank=3 imbalance_ratio=1.037476 max_load_reduction=57037 max_load_reduction_pct=0.870316 moved_to_echo=33082 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4073, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2960, 1089, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 1739, 4095, 0, 0, 0, 0, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 0, 2359, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8153, 8183, 8841, 7581, 8193, 8213, 8183] avg=8192.0000 max=8841 max_rank=3 min=7581 min_rank=4 imbalance_ratio=1.079224 max_load_reduction=56695 max_load_reduction_pct=0.865097 moved_to_echo=32170 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4075, 11, 8, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 0, 0, 0, 0, 2960, 1089, 8, 0, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 648, 4088, 0, 0, 0, 0, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 0, 3410, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8185, 8191, 7743, 8637, 8192, 8209, 8189] avg=8192.0000 max=8637 max_rank=4 min=7743 min_rank=3 imbalance_ratio=1.054321 max_load_reduction=56899 max_load_reduction_pct=0.868210 moved_to_echo=33227 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4075, 11, 9, 0, 0, 0, 0, 0, 0, 0, 37, 4096, 0, 0, 0, 0, 2960, 1089, 8, 0, 0, 0, 0, 0, 4, 0, 8, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 582, 4083, 0, 0, 0, 0, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 0, 3473, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8190, 8195, 7672, 8698, 8192, 8205, 8193] avg=8192.0000 max=8698 max_rank=4 min=7672 min_rank=3 imbalance_ratio=1.061768 max_load_reduction=56838 max_load_reduction_pct=0.867279 moved_to_echo=33288 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4075, 11, 10, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 0, 0, 0, 2960, 1089, 8, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 1718, 4078, 0, 0, 0, 0, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 0, 2355, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8182, 8189, 8803, 7577, 8192, 8203, 8198] avg=8192.0000 max=8803 max_rank=3 min=7577 min_rank=4 imbalance_ratio=1.074585 max_load_reduction=56733 max_load_reduction_pct=0.865677 moved_to_echo=32170 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4081, 13, 10, 0, 0, 0, 0, 0, 0, 0, 68, 4096, 0, 0, 0, 0, 2962, 1096, 14, 1, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 823, 4082, 0, 0, 0, 0, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 0, 3210, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8237, 8203, 7905, 8427, 8191, 8168, 8205] avg=8192.0000 max=8427 max_rank=4 min=7905 min_rank=3 imbalance_ratio=1.028687 max_load_reduction=57109 max_load_reduction_pct=0.871414 moved_to_echo=32991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32640, 128, 32768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 9088, 0, 23808, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 384, 32768, 0, 128, 128, 0, 23808, 8832, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 128, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8832, 32768, 128, 128, 128, 128, 24064, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 4095, 1, 0, 1, 1, 0, 4095, 0, 1, 0, 63, 2, 4093, 4095, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 4092, 4095, 0, 1, 157, 3, 1, 0, 4095, 0, 4094, 0, 1, 0, 0, 1, 0, 4095, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 1, 4094, 3875, 4095, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 16, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 3, 4090, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 6, 4093, 0, 4, 0, 0, 0, 0, 4096, 4096, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 4, 4087, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 5, 4095, 0, 7, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 3, 4090, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 6, 4092, 0, 4, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4092, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 4096, 0, 6, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4083, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 1, 0, 0, 0, 4088, 0, 0, 7, 4090, 0, 12, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4068, 4, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4082, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4094, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8185, 8191, 8190, 8171, 8184, 8199, 8220] avg=8192.0000 max=8220 max_rank=7 min=8171 min_rank=4 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4095, 4095, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 1, 0, 63, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 4092, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 157, 3, 1, 0, 4095, 0, 4067, 4, 3, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 1, 0, 4095, 4, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 3875, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8247, 8194, 8188, 8331, 8197, 8184, 7998] avg=8192.0000 max=8331 max_rank=4 min=7998 min_rank=7 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 4, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4068, 4, 3, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4089, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4095, 0, 7, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8185, 8192, 8197, 8171, 8185, 8195, 8220] avg=8192.0000 max=8220 max_rank=7 min=8171 min_rank=4 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 3, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4068, 4, 3, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4092, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4093, 0, 4, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 28, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8184, 8192, 8196, 8171, 8188, 8191, 8221] avg=8192.0000 max=8221 max_rank=7 min=8171 min_rank=4 imbalance_ratio=1.003540 max_load_reduction=57315 max_load_reduction_pct=0.874557 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 3, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4068, 4, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4092, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4092, 0, 4, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8185, 8192, 8194, 8171, 8191, 8190, 8220] avg=8192.0000 max=8220 max_rank=7 min=8171 min_rank=4 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4068, 4, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8185, 8191, 8190, 8171, 8198, 8186, 8220] avg=8192.0000 max=8220 max_rank=7 min=8171 min_rank=4 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4068, 4, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 6, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8185, 8192, 8190, 8171, 8195, 8191, 8220] avg=8192.0000 max=8220 max_rank=7 min=8171 min_rank=4 imbalance_ratio=1.003418 max_load_reduction=57316 max_load_reduction_pct=0.874573 moved_to_echo=32777 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 5, 7, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4088, 5, 9, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4090, 0, 12, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8180, 8192, 8191, 8179, 8198, 8200, 8217] avg=8192.0000 max=8217 max_rank=7 min=8179 min_rank=0 imbalance_ratio=1.003052 max_load_reduction=57319 max_load_reduction_pct=0.874619 moved_to_echo=32794 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 256, 128, 128, 0, 32768, 0, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 32640, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4068, 3667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 4096, 4096, 0, 286, 31, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4003, 0, 0, 0, 0, 243, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4062] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 2568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 1536, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4092, 4088, 3057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 4096, 4096, 0, 981, 4, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4067, 0, 0, 4, 0, 88, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4080, 1972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 4096, 4096, 0, 2115, 13, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 6, 0, 27, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4087, 0, 7, 1, 1, 0, 4095, 4095, 3394, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 2, 1, 12, 0, 4095, 0, 1, 4094, 4093, 0, 746, 1, 3, 4095, 4095, 4078, 0, 0, 0, 0, 1, 1, 4095, 4085, 3, 0, 1, 0, 5, 4094, 4095, 0, 0, 1, 4092, 1, 0, 0, 1, 1, 0, 1, 1, 0, 4058] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4092, 4072, 3633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 4096, 4096, 0, 448, 1, 0, 4096, 4096, 4096, 0, 1, 0, 0, 0, 0, 4096, 4055, 0, 0, 4, 0, 72, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4081, 3312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 4096, 4096, 0, 679, 18, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4043, 0, 0, 0, 0, 157, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4093, 4073, 3303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 4096, 4096, 0, 668, 23, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4062, 0, 0, 3, 0, 160, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 3667, 0, 0, 0, 0, 0, 0, 0, 4095, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3083, 949, 61, 0, 0, 0, 0, 0, 8, 0, 3147, 4096, 0, 286, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 4003, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4062, 3110, 997, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8744, 8189, 7568, 8196, 8281, 8193, 8185] avg=8192.0000 max=8744 max_rank=1 min=7568 min_rank=3 imbalance_ratio=1.067383 max_load_reduction=56792 max_load_reduction_pct=0.866577 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 7, 1, 1, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3394, 1, 1, 0, 0, 0, 1, 1, 4095, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 12, 0, 4095, 3076, 949, 61, 0, 0, 0, 0, 0, 0, 1, 3145, 4093, 0, 746, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 4078, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4034, 4085, 3, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 4058, 3110, 995, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8475, 8196, 7989, 8177, 8129, 8188, 8167] avg=8192.0000 max=8475 max_rank=1 min=7989 min_rank=3 imbalance_ratio=1.034546 max_load_reduction=57061 max_load_reduction_pct=0.870682 moved_to_echo=32661 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 1972, 0, 0, 0, 0, 0, 0, 0, 4095, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3083, 949, 61, 0, 0, 0, 0, 0, 9, 0, 3147, 4096, 0, 2115, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 4079, 0, 0, 6, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3110, 997, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7049, 8189, 9380, 8196, 8147, 8193, 8190] avg=8192.0000 max=9380 max_rank=3 min=7049 min_rank=1 imbalance_ratio=1.145020 max_load_reduction=56156 max_load_reduction_pct=0.856873 moved_to_echo=32726 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2568, 0, 0, 0, 0, 0, 0, 0, 4095, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3083, 949, 61, 0, 0, 0, 0, 0, 0, 0, 3147, 4096, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3110, 997, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 7645, 8189, 8779, 8190, 8131, 8193, 8201] avg=8192.0000 max=8779 max_rank=3 min=7645 min_rank=1 imbalance_ratio=1.071655 max_load_reduction=56757 max_load_reduction_pct=0.866043 moved_to_echo=32715 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 4092, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 3057, 0, 0, 0, 0, 0, 0, 0, 4095, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3083, 949, 61, 0, 0, 0, 0, 0, 5, 0, 3147, 4096, 0, 981, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 4067, 0, 0, 4, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4088, 3110, 997, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8134, 8189, 8233, 8196, 8194, 8193, 8201] avg=8192.0000 max=8233 max_rank=3 min=8134 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 4092, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 3633, 0, 0, 0, 0, 0, 0, 0, 4096, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3083, 949, 61, 0, 0, 0, 0, 0, 8, 0, 3147, 4096, 0, 448, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 4055, 0, 0, 4, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4086, 3110, 997, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8711, 8189, 7700, 8197, 8166, 8192, 8201] avg=8192.0000 max=8711 max_rank=1 min=7700 min_rank=3 imbalance_ratio=1.063354 max_load_reduction=56825 max_load_reduction_pct=0.867081 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 3312, 0, 0, 0, 0, 0, 0, 0, 4096, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3083, 949, 61, 0, 0, 0, 0, 0, 5, 0, 3147, 4096, 0, 679, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 4043, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4075, 3111, 997, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8393, 8189, 7945, 8192, 8235, 8192, 8197] avg=8192.0000 max=8393 max_rank=1 min=7945 min_rank=3 imbalance_ratio=1.024536 max_load_reduction=57143 max_load_reduction_pct=0.871933 moved_to_echo=32816 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 4093, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 0, 0, 0, 0, 0, 0, 0, 4096, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3087, 955, 68, 0, 0, 0, 0, 0, 14, 0, 3141, 4096, 0, 668, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 4062, 0, 0, 3, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4063, 3110, 1003, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8385, 8206, 7942, 8192, 8253, 8192, 8194] avg=8192.0000 max=8385 max_rank=1 min=7942 min_rank=3 imbalance_ratio=1.023560 max_load_reduction=57151 max_load_reduction_pct=0.872055 moved_to_echo=32831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 32640, 24960, 8064, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 25216, 32768, 0, 7552, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[24960, 128, 128, 0, 0, 0, 128, 128, 32768, 7936, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 32768, 24704, 7680, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 32768, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32384, 32512, 128, 0, 128, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1990, 0, 0, 0, 0, 4096, 4093, 45, 4096, 4096, 0, 0, 0, 21, 0, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3130, 0, 160, 2848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4093, 0, 7, 1, 4076, 1, 0, 1, 40, 0, 0, 0, 0, 4095, 4095, 14, 4090, 4090, 0, 0, 0, 477, 0, 1, 4095, 0, 0, 1, 2, 4095, 4095, 0, 1, 0, 1, 1, 0, 0, 0, 0, 4095, 4089, 0, 4095, 3624, 1, 796, 3268, 0, 0, 1, 1, 1, 2, 0, 0, 1, 4095, 2, 4088, 0, 4, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 789, 0, 0, 0, 0, 4092, 4091, 54, 4096, 4096, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 3943, 0, 1545, 1857, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 160, 0, 0, 0, 0, 4096, 4096, 195, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3917, 0, 1879, 2041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 638, 0, 0, 0, 0, 4095, 4084, 38, 4096, 4096, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4096, 4019, 0, 570, 2926, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1290, 0, 0, 0, 0, 4091, 4084, 223, 4096, 4096, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 4096, 3804, 0, 104, 2765, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 257, 0, 0, 0, 0, 4096, 4085, 11, 4096, 4096, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3892, 0, 93, 3942, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1393, 0, 0, 0, 0, 4092, 4068, 78, 4096, 4096, 0, 0, 0, 20, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 3465, 0, 462, 2798, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1990, 0, 0, 0, 0, 3287, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 45, 3947, 4096, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 0, 0, 3130, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 2848, 0, 0, 0, 0, 0, 0, 0, 4096, 809, 330, 149, 1, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 9370, 8109, 8193, 8191, 7056, 8233, 8191] avg=8192.0000 max=9370 max_rank=1 min=7056 min_rank=5 imbalance_ratio=1.143799 max_load_reduction=56166 max_load_reduction_pct=0.857025 moved_to_echo=31671 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 7, 1, 4076, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 40, 0, 0, 0, 0, 3286, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3941, 4090, 0, 0, 0, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3759, 0, 0, 3624, 1, 796, 0, 0, 0, 0, 0, 0, 0, 0, 3268, 0, 0, 1, 1, 1, 2, 0, 4095, 809, 330, 149, 0, 0, 0, 0, 0, 1, 4094, 2, 4088, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 7422, 8522, 8194, 8193, 8180, 8656, 8190] avg=8192.0000 max=8656 max_rank=6 min=7422 min_rank=1 imbalance_ratio=1.056641 max_load_reduction=56880 max_load_reduction_pct=0.867920 moved_to_echo=33219 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 638, 0, 0, 0, 0, 3286, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3947, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 2, 0, 4019, 0, 570, 0, 0, 0, 0, 0, 0, 0, 0, 2926, 0, 0, 0, 0, 0, 0, 0, 4096, 809, 330, 149, 1, 0, 0, 0, 3, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8008, 8090, 8192, 8191, 8357, 8311, 8194] avg=8192.0000 max=8357 max_rank=5 min=8008 min_rank=1 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=33053 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 3287, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 195, 3947, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 0, 0, 3917, 0, 1879, 0, 0, 0, 0, 0, 0, 0, 0, 2041, 0, 0, 0, 0, 0, 0, 0, 4096, 809, 330, 149, 1, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7543, 8238, 8192, 8191, 9562, 7426, 8191] avg=8192.0000 max=9562 max_rank=5 min=7426 min_rank=6 imbalance_ratio=1.167236 max_load_reduction=55974 max_load_reduction_pct=0.854095 moved_to_echo=33370 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1290, 0, 0, 0, 0, 3283, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 223, 3947, 4096, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 5, 0, 3804, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 2765, 0, 0, 0, 0, 0, 0, 0, 4096, 808, 330, 149, 0, 0, 0, 0, 3, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8657, 8280, 8192, 8193, 7679, 8148, 8194] avg=8192.0000 max=8657 max_rank=1 min=7679 min_rank=5 imbalance_ratio=1.056763 max_load_reduction=56879 max_load_reduction_pct=0.867905 moved_to_echo=32214 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 789, 0, 0, 0, 0, 3284, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 54, 3947, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 4, 0, 3943, 0, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 1857, 0, 0, 0, 0, 0, 0, 0, 4096, 808, 330, 149, 0, 0, 0, 0, 3, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8164, 8103, 8192, 8192, 9258, 7240, 8194] avg=8192.0000 max=9258 max_rank=5 min=7240 min_rank=6 imbalance_ratio=1.130127 max_load_reduction=56278 max_load_reduction_pct=0.858734 moved_to_echo=32884 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 3287, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3947, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3766, 0, 0, 3892, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 1, 0, 0, 0, 0, 0, 0, 4096, 809, 330, 149, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 7629, 8060, 8192, 8193, 7751, 9327, 8191] avg=8192.0000 max=9327 max_rank=6 min=7629 min_rank=1 imbalance_ratio=1.138550 max_load_reduction=56209 max_load_reduction_pct=0.857681 moved_to_echo=33462 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 1393, 0, 0, 0, 0, 3282, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 78, 3940, 4096, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 4, 0, 3465, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 2798, 0, 0, 0, 0, 0, 0, 0, 4096, 810, 334, 156, 0, 1, 0, 0, 4, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8743, 8134, 8189, 8192, 7693, 8195, 8191] avg=8192.0000 max=8743 max_rank=1 min=7693 min_rank=5 imbalance_ratio=1.067261 max_load_reduction=56793 max_load_reduction_pct=0.866592 moved_to_echo=32271 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[768, 31616, 32768, 0, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 128, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 30208, 128, 0, 29824, 128, 5632, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 0, 128, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 6656, 0, 0, 0, 0, 26368, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[22528, 128, 0, 128, 128, 128, 128, 0, 32768, 6528, 2688, 1280, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4074, 0, 0, 0, 0, 179, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 0, 3803, 0, 0, 4096, 0, 100, 0, 6, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 5, 4096, 0, 4096, 4095, 4096, 22, 0, 4093, 0, 0, 0, 4092, 4096, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4085, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 4069, 0, 0, 4096, 0, 10, 0, 5, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 6, 4096, 0, 4096, 4096, 4096, 15, 0, 4092, 0, 0, 0, 4093, 4096, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 1, 4095, 1, 0, 1, 0, 18, 0, 1, 0, 0, 4095, 0, 0, 2, 0, 1, 3068, 1, 1, 4089, 1, 1104, 1, 3, 4093, 2, 0, 1, 4095, 4095, 0, 0, 4092, 0, 1, 0, 5, 4, 34, 4091, 0, 1, 0, 4092, 0, 3971, 4092, 4093, 0, 1, 4095, 1, 0, 1, 4094, 4095, 0, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4073, 0, 0, 0, 0, 33, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 3923, 0, 0, 4096, 0, 123, 0, 10, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 4086, 1, 0, 8, 4096, 0, 4096, 4094, 4096, 18, 0, 4092, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4029, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 48, 0, 0, 0, 3824, 0, 0, 4096, 0, 219, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 58, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4050, 0, 0, 4096, 0, 22, 0, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4096, 0, 4095, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4067, 0, 0, 0, 0, 16, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3924, 0, 0, 4096, 0, 154, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 29, 0, 4096, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4037, 0, 0, 0, 0, 22, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 0, 3656, 0, 0, 4096, 0, 392, 0, 8, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4085, 0, 0, 5, 4096, 0, 4096, 4092, 4096, 58, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 4095, 1, 0, 1, 0, 4093, 27, 0, 0, 0, 0, 0, 0, 18, 0, 1, 0, 0, 4095, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 3068, 1, 1, 4089, 1, 269, 23, 8, 5, 0, 0, 0, 0, 1104, 1, 3, 4093, 2, 0, 1, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4092, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4, 34, 4091, 0, 1, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 0, 1, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4095, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8156, 7468, 9030, 8193, 8217, 8060, 8193] avg=8192.0000 max=9030 max_rank=3 min=7468 min_rank=2 imbalance_ratio=1.102295 max_load_reduction=56506 max_load_reduction_pct=0.862213 moved_to_echo=32663 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4029, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 48, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 0, 0, 4096, 0, 269, 25, 8, 5, 0, 0, 0, 0, 219, 0, 0, 4096, 0, 0, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 58, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8203, 8227, 8142, 8192, 8187, 8242, 8192] avg=8192.0000 max=8242 max_rank=6 min=8142 min_rank=3 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32813 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4073, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 4096, 0, 13, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 4096, 0, 269, 24, 7, 5, 0, 0, 0, 0, 123, 0, 10, 4096, 0, 0, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4086, 1, 0, 8, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8186, 8324, 8056, 8192, 8197, 8199, 8187] avg=8192.0000 max=8324 max_rank=2 min=8056 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=57212 max_load_reduction_pct=0.872986 moved_to_echo=32775 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4074, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 4096, 0, 14, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 0, 0, 4096, 0, 269, 23, 7, 5, 0, 0, 0, 0, 100, 0, 6, 4096, 0, 0, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 5, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8335, 8203, 8029, 8192, 8188, 8204, 8189] avg=8192.0000 max=8335 max_rank=1 min=8029 min_rank=3 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4085, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 9, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 4096, 0, 269, 23, 7, 5, 0, 0, 0, 0, 10, 0, 5, 4096, 0, 0, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 6, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8153, 8469, 7938, 8192, 8190, 8196, 8191] avg=8192.0000 max=8469 max_rank=2 min=7938 min_rank=3 imbalance_ratio=1.033813 max_load_reduction=57067 max_load_reduction_pct=0.870773 moved_to_echo=32769 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4096, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 4096, 0, 269, 25, 8, 5, 0, 0, 0, 0, 22, 0, 1, 4096, 0, 0, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8162, 8453, 7946, 8192, 8190, 8183, 8194] avg=8192.0000 max=8453 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.031860 max_load_reduction=57083 max_load_reduction_pct=0.871017 moved_to_echo=32759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4067, 0, 0, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 4096, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 4096, 0, 269, 23, 8, 5, 0, 0, 0, 0, 154, 0, 0, 4096, 0, 0, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8157, 8325, 8077, 8192, 8187, 8213, 8194] avg=8192.0000 max=8325 max_rank=2 min=8077 min_rank=3 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4037, 0, 0, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 4096, 0, 25, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 0, 0, 4096, 0, 274, 23, 11, 7, 0, 0, 0, 0, 392, 0, 8, 4096, 0, 0, 0, 3822, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4085, 0, 0, 5, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 58, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8184, 8067, 8318, 8191, 8180, 8239, 8196] avg=8192.0000 max=8318 max_rank=3 min=8067 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=57218 max_load_reduction_pct=0.873077 moved_to_echo=32806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 32768, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 256, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 32640, 128, 0, 128, 0, 32768, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2176, 128, 128, 32768, 128, 0, 128, 30720, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 32768, 0, 128, 0, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[384, 0, 128, 0, 0, 32768, 0, 128, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 30336, 128, 128, 32768, 128, 2176, 256, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4096, 0, 33, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 203, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3910, 0, 3997, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 4095, 4095, 4085, 0, 1, 1, 0, 4095, 1, 0, 0, 1, 4095, 0, 276, 0, 0, 4094, 4095, 4095, 0, 1, 0, 0, 0, 1, 89, 1, 0, 0, 1, 2, 2, 4009, 1, 1, 4080, 15, 1, 1, 2, 0, 0, 4095, 0, 4095, 4094, 0, 1, 25, 1, 0, 1, 4080, 0, 3814, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 4096, 0, 30, 0, 0, 4096, 4096, 4091, 0, 0, 3, 0, 12, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 243, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3849, 0, 4053, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 4096, 0, 7, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 460, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3638, 0, 4070, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 120, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3967, 0, 4096, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 206, 0, 0, 0, 0, 2, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3892, 0, 4084, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 308, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3800, 0, 4064, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 4096, 0, 40, 0, 0, 4096, 4096, 4092, 0, 0, 4, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4064, 347, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3778, 0, 3996, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4066, 19, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 4050, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 30, 0, 0, 0, 0, 0, 0, 4091, 203, 0, 0, 0, 0, 0, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 0, 3997, 0, 0, 0, 0, 227, 46, 28, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8186, 8179, 8211, 8203, 8163, 8189, 8211] avg=8192.0000 max=8211 max_rank=3 min=8163 min_rank=5 imbalance_ratio=1.002319 max_load_reduction=57325 max_load_reduction_pct=0.874710 moved_to_echo=32766 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 3, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 0, 0, 1, 4065, 19, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 0, 4048, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 89, 1, 4095, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 4009, 1, 1, 4076, 29, 0, 0, 0, 0, 0, 0, 4080, 15, 1, 1, 2, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 0, 1, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4080, 0, 3814, 0, 0, 1, 0, 227, 46, 27, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8183, 8419, 8253, 8121, 7967, 8214, 8198] avg=8192.0000 max=8419 max_rank=2 min=7967 min_rank=5 imbalance_ratio=1.027710 max_load_reduction=57117 max_load_reduction_pct=0.871536 moved_to_echo=32500 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 4066, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 30, 0, 0, 0, 0, 0, 0, 4082, 308, 0, 0, 0, 0, 0, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 0, 4064, 0, 0, 0, 0, 227, 46, 28, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8201, 8146, 8176, 8203, 8259, 8189, 8168] avg=8192.0000 max=8259 max_rank=5 min=8146 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32819 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4066, 19, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4050, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 30, 0, 0, 0, 0, 0, 0, 4096, 120, 0, 0, 0, 0, 0, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 4096, 3, 0, 0, 0, 227, 46, 28, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8181, 8151, 8163, 8203, 8085, 8189, 8370] avg=8192.0000 max=8370 max_rank=7 min=8085 min_rank=5 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32847 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4066, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 29, 0, 0, 0, 0, 0, 0, 4092, 206, 0, 0, 0, 0, 2, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 4084, 0, 1, 0, 0, 227, 46, 28, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8187, 8141, 8173, 8202, 8169, 8189, 8281] avg=8192.0000 max=8281 max_rank=7 min=8141 min_rank=2 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32841 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 4066, 20, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4050, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 29, 0, 0, 0, 0, 0, 0, 4089, 460, 0, 0, 0, 0, 0, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3638, 0, 4070, 0, 0, 0, 0, 227, 46, 30, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8189, 8153, 8180, 8201, 8418, 8189, 8014] avg=8192.0000 max=8418 max_rank=5 min=8014 min_rank=7 imbalance_ratio=1.027588 max_load_reduction=57118 max_load_reduction_pct=0.871552 moved_to_echo=32822 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 4065, 19, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 4050, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 12, 0, 1, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 29, 0, 0, 0, 0, 0, 0, 4084, 243, 0, 0, 0, 0, 0, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 0, 4053, 0, 3, 0, 0, 227, 46, 27, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8196, 8171, 8179, 8202, 8196, 8189, 8208] avg=8192.0000 max=8208 max_rank=7 min=8171 min_rank=2 imbalance_ratio=1.001953 max_load_reduction=57328 max_load_reduction_pct=0.874756 moved_to_echo=32795 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 4068, 20, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 4044, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 34, 0, 0, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 29, 0, 0, 0, 0, 0, 0, 4064, 347, 0, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 3996, 0, 0, 0, 0, 228, 52, 28, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8213, 8176, 8201, 8201, 8279, 8188, 8086] avg=8192.0000 max=8279 max_rank=5 min=8086 min_rank=7 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32754 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 512, 0, 0, 32512, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 1920, 128, 128, 128, 0, 128, 30976, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 30976, 0, 32256, 128, 128, 128, 0, 1920, 384, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 256, 128, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 128, 128, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 32768, 128, 128, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[639, 915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 338, 0, 0, 0, 0, 4096, 1, 109, 0, 0, 4096, 0, 0, 0, 2533, 0, 4096, 4096, 0, 4096, 4094, 0, 3659, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[634, 741, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 4092, 12, 0, 0, 0, 0, 0, 4095, 117, 0, 0, 0, 4095, 1, 1, 0, 3887, 0, 4095, 4094, 0, 4095, 4095, 1, 2859, 4094, 5, 4095, 0, 0, 0, 2, 4044, 0, 1, 4095, 0, 1, 1, 0, 4094, 0, 1, 1, 1, 0, 0, 4083, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1674, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 70, 0, 0, 0, 3, 4093, 0, 30, 0, 0, 4096, 0, 0, 0, 1775, 4, 4096, 4094, 0, 4096, 4077, 0, 3947, 4096, 0, 4091, 0, 2, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[1993, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 262, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 4096, 0, 0, 0, 2621, 0, 4096, 4096, 0, 4096, 4079, 0, 2975, 4096, 0, 4091, 0, 4, 0, 0, 4096, 10, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[789, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 318, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 4096, 0, 0, 0, 2790, 2, 4096, 4094, 0, 4096, 4079, 0, 3769, 4096, 0, 4092, 0, 3, 0, 0, 4096, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1483, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 317, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 4096, 0, 0, 0, 2928, 0, 4096, 4096, 0, 4096, 4084, 0, 3429, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[1216, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 257, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 0, 2489, 0, 4096, 4096, 0, 4096, 4095, 0, 3958, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1669, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 173, 0, 0, 0, 0, 4096, 0, 33, 0, 0, 4096, 0, 0, 0, 1935, 4, 4096, 4096, 0, 4096, 4052, 0, 3901, 4096, 0, 4088, 0, 3, 0, 0, 4096, 8, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[639, 915, 0, 0, 0, 0, 0, 0, 4096, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2326, 1769, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 0, 4096, 1, 109, 2068, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2533, 0, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 4094, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7977, 8191, 8382, 8104, 8302, 8195, 8192, 8193] avg=8192.0000 max=8382 max_rank=2 min=7977 min_rank=0 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32882 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[634, 741, 1, 1, 0, 1, 0, 0, 4095, 2328, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4092, 12, 2325, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 117, 0, 2067, 1769, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 3887, 0, 1474, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 4095, 1, 2859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4095, 0, 0, 0, 2, 4044, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 4083, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7801, 8201, 8048, 9458, 7504, 8149, 8192, 8183] avg=8192.0000 max=9458 max_rank=3 min=7504 min_rank=4 imbalance_ratio=1.154541 max_load_reduction=56078 max_load_reduction_pct=0.855682 moved_to_echo=32028 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1993, 347, 0, 0, 0, 0, 0, 0, 4096, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2326, 1769, 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0, 4096, 1, 1, 2068, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2621, 0, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 4079, 0, 2975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4, 0, 0, 4096, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8763, 8191, 8198, 8192, 7603, 8204, 8192, 8193] avg=8192.0000 max=8763 max_rank=0 min=7603 min_rank=4 imbalance_ratio=1.069702 max_load_reduction=56773 max_load_reduction_pct=0.866287 moved_to_echo=32192 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1674, 710, 0, 0, 0, 0, 0, 0, 4096, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2326, 1768, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 3, 4093, 0, 30, 2067, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1775, 4, 1474, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 4077, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2, 0, 0, 4096, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8808, 8190, 8033, 7349, 8573, 8198, 8192, 8193] avg=8192.0000 max=8808 max_rank=0 min=7349 min_rank=3 imbalance_ratio=1.075195 max_load_reduction=56728 max_load_reduction_pct=0.865601 moved_to_echo=33156 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[789, 519, 0, 0, 0, 0, 0, 0, 4096, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2326, 1769, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 0, 4096, 0, 21, 2067, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2790, 2, 1474, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 4079, 0, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3, 0, 0, 4096, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7731, 8191, 8272, 8362, 8397, 8198, 8192, 8193] avg=8192.0000 max=8397 max_rank=4 min=7731 min_rank=0 imbalance_ratio=1.025024 max_load_reduction=57139 max_load_reduction_pct=0.871872 moved_to_echo=32980 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1216, 271, 0, 0, 0, 0, 0, 0, 4096, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2326, 1769, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 4096, 0, 2, 2068, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2489, 0, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 4095, 0, 3958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7910, 8191, 8193, 8060, 8602, 8195, 8192, 8193] avg=8192.0000 max=8602 max_rank=4 min=7910 min_rank=0 imbalance_ratio=1.050049 max_load_reduction=56934 max_load_reduction_pct=0.868744 moved_to_echo=33182 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1483, 28, 0, 0, 0, 0, 0, 0, 4096, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2326, 1768, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 4096, 0, 19, 2068, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2928, 0, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 4084, 0, 3429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7935, 8190, 8270, 8499, 8062, 8195, 8192, 8193] avg=8192.0000 max=8499 max_rank=3 min=7935 min_rank=0 imbalance_ratio=1.037476 max_load_reduction=57037 max_load_reduction_pct=0.870316 moved_to_echo=32642 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1669, 518, 0, 0, 0, 0, 0, 0, 4096, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2327, 1768, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 4096, 0, 33, 2069, 1769, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1935, 4, 1477, 0, 0, 0, 0, 0, 0, 0, 0, 540, 0, 0, 4052, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 3, 0, 0, 4096, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8611, 8191, 8140, 7512, 8493, 8202, 8192, 8195] avg=8192.0000 max=8611 max_rank=0 min=7512 min_rank=3 imbalance_ratio=1.051147 max_load_reduction=56925 max_load_reduction_pct=0.868607 moved_to_echo=33082 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[10112, 4096, 128, 128, 0, 128, 0, 0, 32768, 18688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 128, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 20992, 128, 11904, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 4480, 0, 0, 32768, 128, 28544, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 18688, 14208, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1792, 0, 0, 0, 128, 32768, 128, 256, 16640, 14208, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 63, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 17, 0, 4096, 0, 4095, 2, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 1, 65, 0, 0, 4095, 1, 4095, 0, 2, 0, 0, 0, 4095, 0, 10, 4093, 1, 4094, 0, 3, 2, 1, 5, 0, 0, 4069, 0, 4095, 0, 1, 0, 4095, 0, 4095, 0, 0, 1, 0, 1, 0, 0, 46, 1, 1, 1, 1, 4086, 0, 0, 1, 4095, 4008, 0, 4090, 0, 4094, 4093, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 4093, 29, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 10, 0, 4096, 0, 4091, 18, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 25, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 15, 0, 4096, 0, 4096, 24, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 1, 65, 0, 0, 4066, 2, 0, 0, 0, 0, 0, 0, 4086, 1, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 10, 4093, 1, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 5, 0, 0, 4069, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4095, 0, 0, 16, 7, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 46, 1, 1, 4093, 4091, 0, 0, 0, 0, 0, 0, 1, 1, 4086, 0, 0, 1, 13, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 4093, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8184, 8203, 8174, 8214, 8234, 8110, 8187] avg=8192.0000 max=8234 max_rank=5 min=8110 min_rank=6 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32745 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 10, 0, 4067, 3, 0, 0, 0, 0, 0, 0, 4087, 0, 4091, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4076, 0, 0, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8196, 8191, 8194, 8196, 8185, 8205, 8193] avg=8192.0000 max=8205 max_rank=6 min=8176 min_rank=0 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32779 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 17, 0, 4067, 3, 0, 0, 0, 0, 0, 0, 4087, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4079, 0, 0, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8184, 8191, 8194, 8201, 8185, 8205, 8193] avg=8192.0000 max=8205 max_rank=6 min=8183 min_rank=0 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4067, 3, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8184, 8194, 8194, 8215, 8185, 8205, 8193] avg=8192.0000 max=8215 max_rank=4 min=8166 min_rank=0 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 7, 0, 4067, 3, 0, 0, 0, 0, 0, 0, 4087, 0, 4093, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4067, 0, 0, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8209, 8188, 8194, 8189, 8185, 8205, 8193] avg=8192.0000 max=8209 max_rank=1 min=8173 min_rank=0 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32772 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 25, 0, 4067, 3, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4070, 0, 0, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8188, 8188, 8194, 8189, 8189, 8205, 8192] avg=8192.0000 max=8205 max_rank=6 min=8188 min_rank=1 imbalance_ratio=1.001587 max_load_reduction=57331 max_load_reduction_pct=0.874802 moved_to_echo=32775 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 15, 0, 4073, 4, 0, 0, 0, 0, 0, 0, 4081, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4061, 0, 0, 17, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8201, 8186, 8198, 8187, 8188, 8196, 8192] avg=8192.0000 max=8201 max_rank=1 min=8186 min_rank=2 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 63, 0, 4067, 3, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4026, 0, 0, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8190, 8195, 8194, 8145, 8185, 8205, 8193] avg=8192.0000 max=8229 max_rank=0 min=8145 min_rank=4 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32728 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 0, 128, 128, 256, 0, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 0, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 32768, 128, 32640, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2706, 33, 129, 24, 304, 0, 2706, 30, 868, 3292, 114, 148, 1238, 1821, 1, 193, 286, 969, 146, 1845, 4, 741, 1002, 1399, 6, 6, 3234, 1143, 3438, 61, 2906, 464, 22, 606, 2749, 178, 160, 94, 2735, 1253, 2225, 3, 2863, 1810, 4, 859, 1427, 840, 3468, 53, 1802, 0, 1, 125, 0, 286, 2, 117, 1136, 1398, 3268, 787, 948, 3060] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2571, 16, 274, 75, 400, 0, 2114, 11, 597, 3151, 110, 221, 598, 2143, 0, 337, 608, 1296, 192, 1465, 0, 552, 992, 1274, 2, 21, 2912, 1096, 2706, 69, 2346, 632, 96, 825, 2522, 170, 185, 196, 3196, 1567, 1959, 0, 2975, 1686, 0, 901, 1551, 1045, 2914, 84, 2411, 0, 1, 294, 0, 679, 0, 145, 602, 2154, 3261, 1254, 1591, 2491] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1817, 94, 172, 53, 395, 11, 1997, 28, 826, 3136, 103, 140, 812, 901, 0, 230, 291, 1745, 42, 1933, 1, 505, 1233, 1565, 5, 24, 2719, 1182, 3545, 106, 2850, 694, 15, 1041, 2357, 176, 298, 65, 2606, 1142, 2027, 1, 3297, 1808, 6, 821, 1546, 1121, 3252, 100, 2132, 19, 20, 46, 7, 641, 2, 122, 633, 1992, 3150, 1640, 1245, 3053] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2464, 22, 451, 42, 378, 0, 2297, 55, 611, 3061, 131, 460, 1535, 1278, 0, 390, 261, 973, 186, 2334, 0, 391, 1282, 1598, 1, 23, 2720, 1403, 3201, 99, 2516, 572, 30, 1225, 2438, 193, 513, 153, 2718, 966, 2120, 0, 3005, 2037, 0, 592, 1337, 1006, 3390, 45, 1551, 5, 14, 266, 2, 627, 0, 142, 577, 1259, 3467, 1153, 1194, 2776] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2014, 25, 291, 90, 583, 2, 2146, 11, 728, 2937, 125, 305, 1726, 1813, 1, 351, 244, 1369, 142, 1946, 0, 528, 1246, 2081, 1, 3, 2685, 1519, 3303, 91, 2980, 656, 24, 1246, 2329, 106, 221, 75, 2407, 1748, 2218, 0, 2564, 1601, 3, 847, 1358, 826, 3249, 23, 1299, 0, 8, 245, 0, 410, 2, 123, 1104, 1229, 3427, 1255, 762, 2885] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2729, 37, 230, 72, 403, 0, 2698, 21, 711, 3027, 90, 332, 1231, 1665, 0, 230, 439, 1032, 87, 2000, 4, 540, 1439, 1432, 8, 3, 2797, 1553, 3218, 138, 2929, 500, 31, 889, 2318, 168, 306, 46, 2949, 1570, 2037, 3, 2687, 1687, 1, 1336, 1202, 939, 3068, 15, 1928, 2, 5, 202, 6, 279, 0, 157, 838, 1389, 3217, 659, 1231, 2776] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2421, 4, 109, 46, 302, 0, 2793, 49, 560, 3387, 186, 445, 1387, 1490, 0, 262, 297, 1332, 82, 1756, 1, 561, 955, 1529, 0, 27, 2841, 1074, 3248, 85, 2810, 855, 16, 983, 2408, 272, 125, 95, 2502, 1225, 1471, 0, 2888, 1776, 0, 1241, 1698, 796, 3356, 52, 1871, 0, 3, 239, 1, 452, 0, 114, 707, 1531, 3374, 1583, 1005, 2858] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2547, 4, 93, 51, 405, 0, 2424, 13, 1061, 3244, 142, 276, 902, 1486, 0, 218, 228, 1455, 200, 2134, 0, 831, 919, 1892, 5, 17, 2623, 1003, 3325, 100, 2937, 506, 14, 734, 2822, 335, 189, 128, 2506, 1139, 1912, 0, 3246, 2301, 0, 704, 1460, 883, 3277, 103, 1793, 1, 5, 226, 0, 389, 0, 97, 736, 1183, 3218, 1089, 1332, 2673] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2571, 16, 274, 75, 400, 0, 2114, 11, 2627, 0, 0, 0, 0, 0, 0, 0, 597, 3151, 110, 221, 598, 2143, 0, 337, 856, 0, 0, 0, 0, 0, 0, 0, 608, 1296, 192, 1465, 0, 552, 992, 1274, 901, 270, 0, 0, 0, 0, 0, 0, 2, 21, 2912, 1096, 519, 69, 2346, 632, 0, 0, 0, 0, 0, 0, 0, 0, 96, 825, 2522, 170, 185, 196, 3196, 1567, 141, 0, 0, 0, 0, 0, 0, 0, 1959, 0, 1077, 1686, 0, 901, 1551, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 2914, 84, 2411, 0, 1, 294, 0, 679, 1917, 47, 0, 0, 0, 0, 0, 0, 0, 145, 602, 2154, 587, 1254, 1591, 2491, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8088, 8013, 7550, 7597, 8898, 8219, 8347, 8824] avg=8192.0000 max=8898 max_rank=4 min=7550 min_rank=2 imbalance_ratio=1.086182 max_load_reduction=56638 max_load_reduction_pct=0.864227 moved_to_echo=34288 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2706, 33, 129, 24, 304, 0, 2706, 30, 2632, 0, 0, 0, 0, 0, 0, 0, 868, 3292, 114, 148, 1238, 1821, 1, 193, 824, 0, 0, 0, 0, 0, 0, 0, 286, 969, 146, 1845, 4, 741, 1002, 1399, 867, 343, 0, 0, 0, 0, 0, 0, 6, 6, 3234, 1143, 660, 61, 2906, 464, 0, 0, 0, 0, 0, 0, 0, 0, 22, 606, 2749, 178, 160, 94, 2735, 1253, 136, 0, 0, 0, 0, 0, 0, 0, 2225, 3, 1036, 1810, 4, 859, 1427, 840, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 53, 1802, 0, 1, 125, 0, 286, 2435, 47, 0, 0, 0, 0, 0, 0, 2, 117, 1136, 1398, 589, 787, 948, 3060, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8564, 8499, 7602, 8480, 7933, 8204, 8217, 8037] avg=8192.0000 max=8564 max_rank=0 min=7602 min_rank=2 imbalance_ratio=1.045410 max_load_reduction=56972 max_load_reduction_pct=0.869324 moved_to_echo=32391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1817, 94, 172, 53, 395, 11, 1997, 28, 2537, 0, 0, 0, 0, 0, 0, 0, 826, 3136, 103, 140, 812, 901, 0, 230, 949, 0, 0, 0, 0, 0, 0, 0, 291, 1745, 42, 1933, 1, 505, 1233, 1565, 999, 354, 0, 0, 0, 0, 0, 0, 5, 24, 2719, 1182, 680, 106, 2850, 694, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1041, 2357, 176, 298, 65, 2606, 1142, 156, 0, 0, 0, 0, 0, 0, 0, 2027, 1, 1193, 1808, 6, 821, 1546, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 3252, 100, 2132, 19, 20, 46, 7, 641, 2511, 45, 0, 0, 0, 0, 0, 0, 2, 122, 633, 1992, 568, 1640, 1245, 3053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7104, 7097, 8668, 8260, 7856, 8523, 8773, 9255] avg=8192.0000 max=9255 max_rank=7 min=7097 min_rank=1 imbalance_ratio=1.129761 max_load_reduction=56281 max_load_reduction_pct=0.858780 moved_to_echo=34407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2464, 22, 451, 42, 378, 0, 2297, 55, 2793, 0, 0, 0, 0, 0, 0, 0, 611, 3061, 131, 460, 1535, 1278, 0, 390, 865, 0, 0, 0, 0, 0, 0, 0, 261, 973, 186, 2334, 0, 391, 1282, 1598, 910, 319, 0, 0, 0, 0, 0, 0, 1, 23, 2720, 1403, 614, 99, 2516, 572, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1225, 2438, 193, 513, 153, 2718, 966, 143, 0, 0, 0, 0, 0, 0, 0, 2120, 0, 1087, 2037, 0, 592, 1337, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 3390, 45, 1551, 5, 14, 266, 2, 627, 2268, 50, 0, 0, 0, 0, 0, 0, 0, 142, 577, 1259, 624, 1153, 1194, 2776, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8502, 8331, 8254, 7948, 8379, 8179, 8218, 7725] avg=8192.0000 max=8502 max_rank=0 min=7725 min_rank=7 imbalance_ratio=1.037842 max_load_reduction=57034 max_load_reduction_pct=0.870270 moved_to_echo=32501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2014, 25, 291, 90, 583, 2, 2146, 11, 2760, 0, 0, 0, 0, 0, 0, 0, 728, 2937, 125, 305, 1726, 1813, 1, 351, 738, 0, 0, 0, 0, 0, 0, 0, 244, 1369, 142, 1946, 0, 528, 1246, 2081, 776, 329, 0, 0, 0, 0, 0, 0, 1, 3, 2685, 1519, 634, 91, 2980, 656, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1246, 2329, 106, 221, 75, 2407, 1748, 122, 0, 0, 0, 0, 0, 0, 0, 2218, 0, 928, 1601, 3, 847, 1358, 826, 0, 0, 0, 0, 0, 0, 0, 0, 3249, 23, 1299, 0, 8, 245, 0, 410, 2340, 49, 0, 0, 0, 0, 0, 0, 2, 123, 1104, 1229, 618, 1255, 762, 2885, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7922, 8724, 8661, 8569, 8278, 7781, 7623, 7978] avg=8192.0000 max=8724 max_rank=1 min=7623 min_rank=6 imbalance_ratio=1.064941 max_load_reduction=56812 max_load_reduction_pct=0.866882 moved_to_echo=31660 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2729, 37, 230, 72, 403, 0, 2698, 21, 2591, 0, 0, 0, 0, 0, 0, 0, 711, 3027, 90, 332, 1231, 1665, 0, 230, 773, 0, 0, 0, 0, 0, 0, 0, 439, 1032, 87, 2000, 4, 540, 1439, 1432, 814, 321, 0, 0, 0, 0, 0, 0, 8, 3, 2797, 1553, 617, 138, 2929, 500, 0, 0, 0, 0, 0, 0, 0, 0, 31, 889, 2318, 168, 306, 46, 2949, 1570, 127, 0, 0, 0, 0, 0, 0, 0, 2037, 3, 973, 1687, 1, 1336, 1202, 939, 0, 0, 0, 0, 0, 0, 0, 0, 3068, 15, 1928, 2, 5, 202, 6, 279, 2280, 46, 0, 0, 0, 0, 0, 0, 0, 157, 838, 1389, 580, 659, 1231, 2776, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8781, 8059, 8108, 8545, 8404, 8178, 7831, 7630] avg=8192.0000 max=8781 max_rank=0 min=7630 min_rank=7 imbalance_ratio=1.071899 max_load_reduction=56755 max_load_reduction_pct=0.866013 moved_to_echo=32043 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2421, 4, 109, 46, 302, 0, 2793, 49, 2722, 0, 0, 0, 0, 0, 0, 0, 560, 3387, 186, 445, 1387, 1490, 0, 262, 833, 0, 0, 0, 0, 0, 0, 0, 297, 1332, 82, 1756, 1, 561, 955, 1529, 879, 327, 0, 0, 0, 0, 0, 0, 0, 27, 2841, 1074, 616, 85, 2810, 855, 0, 0, 0, 0, 0, 0, 0, 0, 16, 983, 2408, 272, 125, 95, 2502, 1225, 141, 0, 0, 0, 0, 0, 0, 0, 1471, 0, 1035, 1776, 0, 1241, 1698, 796, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 52, 1871, 0, 3, 239, 1, 452, 2305, 53, 0, 0, 0, 0, 0, 0, 0, 114, 707, 1531, 599, 1583, 1005, 2858, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8446, 8550, 7719, 8308, 7767, 8017, 8332, 8397] avg=8192.0000 max=8550 max_rank=1 min=7719 min_rank=2 imbalance_ratio=1.043701 max_load_reduction=56986 max_load_reduction_pct=0.869537 moved_to_echo=32513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2547, 4, 93, 51, 405, 0, 2424, 13, 2592, 0, 0, 0, 0, 0, 0, 0, 1061, 3244, 142, 276, 902, 1486, 0, 218, 934, 0, 0, 0, 0, 0, 0, 0, 228, 1455, 200, 2134, 0, 831, 919, 1892, 983, 332, 0, 0, 0, 0, 0, 0, 5, 17, 2623, 1003, 638, 100, 2937, 506, 0, 0, 0, 0, 0, 0, 0, 0, 14, 734, 2822, 335, 189, 128, 2506, 1139, 154, 0, 0, 0, 0, 0, 0, 0, 1912, 0, 1175, 2301, 0, 704, 1460, 883, 0, 0, 0, 0, 0, 0, 0, 0, 3277, 103, 1793, 1, 5, 226, 0, 389, 2355, 46, 0, 0, 0, 0, 0, 0, 0, 97, 736, 1183, 580, 1089, 1332, 2673, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8263, 8974, 7829, 8021, 8435, 8195, 7690] avg=8192.0000 max=8974 max_rank=2 min=7690 min_rank=7 imbalance_ratio=1.095459 max_load_reduction=56562 max_load_reduction_pct=0.863068 moved_to_echo=32341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 1024, 6400, 12160, 4864, 9472, 9344, 22656, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[16000, 128, 8576, 14720, 128, 7424, 11648, 7552, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[2688, 10240, 1152, 15488, 128, 4736, 9088, 12800, 7168, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 22656, 9984, 4992, 768, 22400, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[256, 7552, 19968, 1664, 2048, 896, 21632, 10624, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[19328, 256, 1792, 512, 3200, 128, 19200, 256, 21376, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25984, 512, 14848, 128, 128, 1664, 128, 3840, 18432, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6016, 25344, 1024, 2432, 9472, 12672, 128, 2304, 6784, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 4087, 0, 4085, 4093, 0, 238, 192, 0, 0, 2144, 4096, 0, 3, 0, 0, 1, 8, 71, 1015, 4039, 1, 4055, 0, 12, 0, 3972, 6, 0, 1, 0, 1767, 7, 1, 292, 4092, 0, 0, 0, 0, 2, 0, 4092, 0, 4, 0, 0, 4094, 0, 16, 0, 4096, 4092, 2772, 0, 1, 2, 0, 3991] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 3999, 0, 4044, 4096, 0, 190, 132, 0, 0, 585, 4095, 0, 0, 0, 0, 0, 1, 4, 990, 4091, 0, 4095, 2, 0, 0, 4086, 0, 0, 0, 0, 2624, 0, 0, 602, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 4095, 4096, 3243, 0, 0, 0, 0, 4075] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 4096, 0, 4004, 4096, 0, 499, 574, 0, 0, 1345, 4094, 0, 0, 0, 0, 0, 0, 12, 446, 4087, 0, 4091, 0, 7, 0, 4057, 0, 0, 3, 0, 1683, 0, 0, 526, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 4096, 3288, 0, 0, 0, 0, 4048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[15, 6, 5, 0, 4077, 1, 3956, 2, 4075, 4082, 0, 176, 1249, 1, 0, 789, 4091, 1, 1, 23, 10, 12, 0, 21, 1151, 4095, 0, 4095, 8, 0, 0, 4093, 0, 5, 0, 12, 1318, 0, 17, 734, 4084, 2, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 2, 4084, 3, 1, 0, 4082, 4080, 3019, 0, 6, 2, 1, 3953] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 3, 4066, 1, 3995, 4093, 0, 38, 340, 0, 0, 1056, 4004, 0, 0, 0, 0, 0, 0, 3, 1444, 4089, 0, 4090, 0, 11, 1, 4059, 0, 0, 0, 0, 2095, 0, 0, 601, 4094, 3, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 105, 4096, 0, 53, 2, 4093, 4096, 2830, 3, 0, 0, 0, 3980] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4011, 0, 4090, 4096, 0, 142, 294, 0, 0, 2414, 4096, 0, 0, 0, 0, 0, 0, 1, 343, 4096, 0, 4087, 1, 18, 0, 4082, 0, 0, 1, 0, 1537, 0, 0, 240, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 103, 0, 4091, 4096, 3230, 0, 0, 0, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4064, 0, 4083, 4096, 0, 558, 341, 0, 0, 2865, 4079, 0, 0, 0, 0, 0, 3, 30, 230, 4076, 0, 4046, 3, 2, 0, 4084, 0, 0, 1, 0, 847, 0, 0, 339, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 4095, 0, 78, 0, 4096, 4096, 3074, 0, 0, 0, 0, 4045] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 0, 4079, 4096, 0, 491, 377, 0, 0, 2390, 4092, 0, 1, 0, 0, 0, 18, 18, 719, 4056, 0, 4072, 4, 6, 0, 4037, 1, 0, 3, 0, 497, 0, 0, 236, 4095, 2, 0, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 4094, 0, 27, 0, 4095, 4095, 3583, 0, 0, 0, 0, 4061] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 4096, 0, 47, 0, 0, 0, 0, 0, 0, 0, 4004, 1709, 0, 499, 574, 0, 0, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 12, 2102, 1985, 0, 0, 0, 0, 0, 0, 446, 3340, 0, 0, 0, 7, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1683, 0, 0, 526, 4096, 2106, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 2387, 962, 699, 0, 0, 0, 0, 0, 1032, 0, 3288, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8131, 8193, 7850, 8414, 8193, 8147, 8368] avg=8192.0000 max=8414 max_rank=4 min=7850 min_rank=3 imbalance_ratio=1.027100 max_load_reduction=57122 max_load_reduction_pct=0.871613 moved_to_echo=33122 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 4087, 0, 47, 0, 0, 0, 0, 0, 0, 0, 4085, 1708, 0, 238, 192, 0, 0, 2144, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 1, 8, 71, 2102, 1969, 0, 0, 0, 0, 0, 0, 1015, 3300, 1, 0, 0, 12, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 1767, 7, 1, 292, 4092, 2086, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 2, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4094, 0, 16, 0, 2385, 962, 691, 0, 0, 0, 0, 0, 1032, 0, 2772, 0, 1, 2, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8367, 8250, 8300, 8252, 8187, 8152, 7798] avg=8192.0000 max=8367 max_rank=1 min=7798 min_rank=7 imbalance_ratio=1.021362 max_load_reduction=57169 max_load_reduction_pct=0.872330 moved_to_echo=32389 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 3, 4066, 1, 47, 0, 0, 0, 0, 0, 0, 0, 3995, 1708, 0, 38, 340, 0, 0, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 3, 2101, 1985, 0, 0, 0, 0, 0, 0, 1444, 3341, 0, 0, 0, 11, 1, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2095, 0, 0, 601, 4096, 2105, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 105, 4096, 0, 53, 2, 2385, 961, 700, 0, 0, 0, 0, 0, 1031, 0, 2830, 3, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8209, 7137, 8093, 8856, 8897, 8194, 8306, 7844] avg=8192.0000 max=8897 max_rank=4 min=7137 min_rank=1 imbalance_ratio=1.086060 max_load_reduction=56639 max_load_reduction_pct=0.864243 moved_to_echo=33241 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[15, 6, 5, 0, 4077, 1, 3956, 2, 47, 0, 0, 0, 0, 0, 0, 0, 4075, 1703, 0, 176, 1249, 1, 0, 789, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 1, 23, 10, 12, 0, 21, 2095, 1987, 0, 0, 0, 0, 0, 0, 1151, 3346, 0, 0, 8, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 12, 1318, 0, 17, 734, 4080, 2108, 0, 0, 0, 0, 0, 0, 4084, 2, 0, 0, 4, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4084, 3, 1, 0, 2379, 959, 701, 0, 0, 0, 0, 0, 1028, 0, 3019, 0, 6, 2, 1, 3953, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 7993, 8241, 8598, 8274, 8183, 8129, 8009] avg=8192.0000 max=8598 max_rank=3 min=7993 min_rank=1 imbalance_ratio=1.049561 max_load_reduction=56938 max_load_reduction_pct=0.868805 moved_to_echo=32595 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 3999, 0, 47, 0, 0, 0, 0, 0, 0, 0, 4044, 1709, 0, 190, 132, 0, 0, 585, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4, 2102, 1987, 0, 0, 0, 0, 0, 0, 990, 3343, 0, 0, 2, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2624, 0, 0, 602, 4096, 2108, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 2387, 962, 700, 0, 0, 0, 0, 0, 1031, 0, 3243, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8142, 6660, 8189, 8421, 9430, 8199, 8146, 8349] avg=8192.0000 max=9430 max_rank=4 min=6660 min_rank=1 imbalance_ratio=1.151123 max_load_reduction=56106 max_load_reduction_pct=0.856110 moved_to_echo=34124 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4011, 0, 47, 0, 0, 0, 0, 0, 0, 0, 4090, 1709, 0, 142, 294, 0, 0, 2414, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 2100, 1984, 0, 0, 0, 0, 0, 0, 343, 3347, 0, 0, 1, 18, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1537, 0, 0, 240, 4096, 2103, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 103, 0, 2387, 961, 701, 0, 0, 0, 0, 0, 1030, 0, 3230, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8649, 8181, 7791, 7977, 8189, 8248, 8347] avg=8192.0000 max=8649 max_rank=1 min=7791 min_rank=3 imbalance_ratio=1.055786 max_load_reduction=56887 max_load_reduction_pct=0.868027 moved_to_echo=32761 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4064, 0, 47, 0, 0, 0, 0, 0, 0, 0, 4083, 1709, 0, 558, 341, 0, 0, 2865, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 3, 30, 2102, 1964, 0, 0, 0, 0, 0, 0, 230, 3331, 0, 0, 3, 2, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 847, 0, 0, 339, 4096, 2082, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4095, 0, 78, 0, 2387, 962, 697, 0, 0, 0, 0, 0, 1032, 0, 3074, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 9556, 8178, 7650, 7365, 8194, 8235, 8151] avg=8192.0000 max=9556 max_rank=1 min=7365 min_rank=4 imbalance_ratio=1.166504 max_load_reduction=55980 max_load_reduction_pct=0.854187 moved_to_echo=31945 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4096, 0, 53, 0, 0, 0, 0, 0, 0, 0, 4079, 1706, 0, 491, 377, 0, 0, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 18, 18, 2105, 1977, 0, 0, 0, 0, 0, 0, 719, 3304, 0, 0, 4, 6, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 497, 0, 0, 236, 4095, 2095, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 1, 1, 1, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 27, 0, 2390, 964, 697, 0, 0, 0, 0, 0, 1026, 0, 3583, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8245, 9043, 8211, 8070, 6927, 8197, 8173, 8670] avg=8192.0000 max=9043 max_rank=1 min=6927 min_rank=4 imbalance_ratio=1.103882 max_load_reduction=56493 max_load_reduction_pct=0.862015 moved_to_echo=31967 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32512, 13696, 0, 2432, 3584, 128, 0, 13696, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 256, 16896, 15872, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 32384, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 0, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 12416, 128, 128, 3584, 32768, 16896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 0, 25088, 128, 128, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6400, 26752, 128, 0, 128, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 384, 128, 19200, 7808, 5632, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 4, 4096, 0, 0, 0, 4088, 0, 4096, 0, 0, 4096, 3, 1, 4095, 0, 0, 4077, 10, 0, 164, 70, 4095, 0, 4089, 0, 0, 0, 3184, 4094, 0, 4, 4095, 0, 4092, 0, 0, 0, 55, 4096, 0, 2020, 1, 6, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 2709, 0, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 156, 31, 4094, 0, 4095, 0, 0, 0, 3596, 4096, 1, 0, 4096, 5, 4096, 0, 0, 0, 8, 4086, 0, 1201, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3208, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 4087, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4066, 0, 0, 56, 29, 4096, 0, 4093, 0, 0, 0, 3621, 4096, 0, 0, 4094, 0, 4096, 0, 0, 0, 4, 4096, 0, 1746, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2775, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 0, 20, 4092, 1, 0, 10, 4079, 0, 4092, 3, 0, 4088, 6, 4, 4076, 1, 4, 4040, 0, 5, 305, 252, 4091, 0, 4096, 0, 0, 0, 2849, 4088, 7, 4, 3962, 34, 4092, 0, 4, 3, 151, 4092, 4, 601, 0, 60, 3, 4091, 6, 4, 6, 12, 1, 4089, 0, 6, 4076, 8, 0, 0, 15, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 4086, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4068, 51, 0, 291, 96, 4095, 0, 4088, 0, 0, 0, 3060, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 21, 4046, 0, 1625, 0, 20, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 3109, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 4072, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 4052, 32, 0, 630, 153, 4096, 0, 4085, 0, 0, 0, 3435, 4096, 0, 9, 4094, 0, 4092, 0, 0, 0, 105, 4096, 0, 1472, 0, 43, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 2393, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4088, 4, 0, 664, 8, 4094, 0, 4096, 0, 0, 0, 3311, 4095, 1, 0, 4096, 8, 4095, 0, 0, 0, 7, 4095, 0, 2445, 0, 42, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1714, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4092, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4088, 4, 0, 53, 10, 4095, 0, 4095, 0, 0, 0, 3072, 4096, 0, 1, 4096, 4, 4095, 0, 0, 0, 100, 4096, 0, 1724, 0, 17, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 3220, 0, 1, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 2895, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 156, 31, 3730, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3596, 0, 1, 0, 4096, 5, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4086, 0, 1201, 0, 0, 2062, 356, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3208, 0, 0, 0, 0, 0, 4096, 1192, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8201, 8201, 8012, 8537, 7713, 8190, 8496] avg=8192.0000 max=8537 max_rank=4 min=7713 min_rank=5 imbalance_ratio=1.042114 max_load_reduction=56999 max_load_reduction_pct=0.869736 moved_to_echo=32936 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 4, 4096, 0, 2895, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4095, 0, 0, 4077, 10, 0, 9, 2, 0, 0, 0, 0, 0, 0, 164, 70, 3730, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3184, 0, 0, 4, 4095, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4096, 0, 2020, 1, 6, 2060, 356, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2709, 0, 0, 0, 3, 0, 4094, 1192, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8193, 8197, 8053, 8121, 8594, 8188, 7998] avg=8192.0000 max=8594 max_rank=5 min=7998 min_rank=7 imbalance_ratio=1.049072 max_load_reduction=56942 max_load_reduction_pct=0.868866 moved_to_echo=32901 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 4096, 0, 2895, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 9, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4052, 32, 0, 9, 2, 0, 0, 0, 0, 0, 0, 630, 153, 3731, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3435, 0, 0, 9, 4094, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 4096, 0, 1472, 0, 43, 2060, 356, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2393, 0, 0, 0, 0, 0, 4096, 1192, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8176, 8191, 8599, 8376, 8132, 8191, 7681] avg=8192.0000 max=8599 max_rank=3 min=7681 min_rank=7 imbalance_ratio=1.049683 max_load_reduction=56937 max_load_reduction_pct=0.868790 moved_to_echo=32380 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 0, 20, 4092, 1, 2892, 1194, 0, 0, 0, 0, 0, 0, 0, 10, 4079, 0, 9, 3, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 4076, 1, 4, 4040, 0, 5, 8, 2, 0, 0, 0, 0, 0, 0, 305, 252, 3727, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 0, 7, 4, 3962, 34, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 151, 4092, 4, 601, 0, 60, 2060, 356, 0, 0, 0, 0, 0, 0, 3, 4089, 6, 4, 6, 12, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4076, 8, 0, 0, 15, 0, 4088, 1191, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8189, 8146, 8380, 7694, 7331, 8210, 9384] avg=8192.0000 max=9384 max_rank=7 min=7331 min_rank=5 imbalance_ratio=1.145508 max_load_reduction=56152 max_load_reduction_pct=0.856812 moved_to_echo=32619 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 4096, 0, 2895, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4066, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 56, 29, 3731, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3621, 0, 0, 0, 4094, 0, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 1746, 0, 0, 2062, 356, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2775, 0, 0, 0, 0, 0, 4096, 1192, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8173, 7909, 8554, 8264, 8190, 8063] avg=8192.0000 max=8554 max_rank=4 min=7909 min_rank=3 imbalance_ratio=1.044189 max_load_reduction=56982 max_load_reduction_pct=0.869476 moved_to_echo=33071 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 0, 0, 4096, 0, 2895, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4068, 51, 0, 9, 2, 0, 0, 0, 0, 0, 0, 291, 96, 3730, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3060, 0, 0, 0, 4096, 0, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4046, 0, 1625, 0, 20, 2062, 356, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 0, 0, 0, 0, 0, 4096, 1192, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8192, 8226, 8205, 7995, 8130, 8191, 8397] avg=8192.0000 max=8397 max_rank=7 min=7995 min_rank=4 imbalance_ratio=1.025024 max_load_reduction=57139 max_load_reduction_pct=0.871872 moved_to_echo=32713 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 4096, 0, 2896, 1197, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 4, 0, 12, 4, 0, 0, 0, 0, 0, 0, 53, 10, 3722, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3072, 0, 0, 1, 4096, 4, 831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 4096, 0, 1724, 0, 17, 2067, 361, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3220, 0, 1, 0, 2, 0, 4096, 1196, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8192, 8204, 7880, 8004, 8365, 8186, 8515] avg=8192.0000 max=8515 max_rank=7 min=7880 min_rank=3 imbalance_ratio=1.039429 max_load_reduction=57021 max_load_reduction_pct=0.870071 moved_to_echo=33070 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 0, 2895, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 4, 0, 8, 2, 0, 0, 0, 0, 0, 0, 664, 8, 3730, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3311, 0, 1, 0, 4096, 8, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4095, 0, 2445, 0, 42, 2062, 356, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 0, 0, 0, 1, 0, 4095, 1192, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8201, 8198, 8498, 8255, 9007, 8190, 7002] avg=8192.0000 max=9007 max_rank=5 min=7002 min_rank=7 imbalance_ratio=1.099487 max_load_reduction=56529 max_load_reduction_pct=0.862564 moved_to_echo=32454 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 23296, 128, 128, 0, 128, 0, 32768, 9600, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2432, 768, 29952, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[26240, 0, 128, 128, 32640, 128, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 512, 32768, 128, 12928, 128, 256, 16512, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 32640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 32768, 128, 23168, 9600, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4096, 4096, 4095, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4085, 11, 0, 7, 0, 4095, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4092, 4075, 4076, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 4092, 0, 12, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4095, 23, 0, 18, 0, 4096, 4083, 10, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 4072, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4087, 19, 0, 5, 0, 4096, 4082, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 4091, 4094, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4074, 27, 0, 6, 0, 4093, 4095, 3, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 4090, 9, 8, 4076, 4039, 4054, 0, 4, 0, 4088, 4, 4087, 4085, 0, 0, 4085, 6, 5, 3, 8, 0, 0, 0, 14, 1, 4, 0, 4071, 2, 6, 7, 4082, 2, 108, 0, 0, 41, 16, 4, 4091, 0, 0, 0, 4089, 4092, 1, 5, 4086, 0, 7, 45, 1, 4092, 3981, 25, 3, 0, 0, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4087, 4091, 3953, 0, 0, 0, 4096, 0, 4096, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 6, 0, 0, 4096, 0, 60, 2, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 4096, 4, 0, 4064, 92, 0, 32, 0, 4096, 4080, 16, 0, 2, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4075, 4063, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 35, 3, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 4052, 41, 0, 12, 0, 4096, 4095, 17, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 4096, 4078, 0, 0, 0, 4095, 0, 4096, 4094, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4090, 23, 0, 5, 0, 4094, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4041, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4085, 11, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 1, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8241, 8193, 8196, 8159, 8186, 8169, 8202] avg=8192.0000 max=8241 max_rank=1 min=8159 min_rank=4 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 4075, 4076, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4095, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4092, 0, 12, 4, 0, 4042, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4095, 23, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 10, 0, 0, 0, 1, 0, 8, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8201, 8193, 8187, 8176, 8186, 8202, 8200] avg=8192.0000 max=8202 max_rank=6 min=8176 min_rank=4 imbalance_ratio=1.001221 max_load_reduction=57334 max_load_reduction_pct=0.874847 moved_to_echo=32764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4087, 3, 0, 0, 0, 0, 0, 0, 0, 4091, 3953, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 4, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 60, 2, 0, 4042, 22, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4, 0, 4064, 92, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 16, 0, 2, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8094, 8189, 8187, 8228, 8190, 8258, 8204] avg=8192.0000 max=8258 max_rank=6 min=8094 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=57278 max_load_reduction_pct=0.873993 moved_to_echo=32880 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 4090, 9, 8, 4076, 3, 0, 0, 0, 0, 0, 0, 0, 4039, 4054, 0, 4, 0, 50, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4085, 6, 5, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 4, 0, 4071, 4087, 4, 0, 0, 0, 0, 0, 0, 2, 6, 7, 4082, 2, 108, 0, 0, 4034, 22, 0, 0, 0, 0, 0, 0, 41, 16, 4, 4091, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 1, 5, 4086, 0, 7, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3981, 25, 3, 0, 0, 0, 4, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8151, 8192, 8181, 8263, 8236, 8208, 8114] avg=8192.0000 max=8263 max_rank=4 min=8114 min_rank=7 imbalance_ratio=1.008667 max_load_reduction=57273 max_load_reduction_pct=0.873917 moved_to_echo=32821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 4072, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 4042, 22, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4087, 19, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8215, 8193, 8196, 8179, 8193, 8177, 8188] avg=8192.0000 max=8215 max_rank=1 min=8177 min_rank=6 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4075, 4063, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 35, 3, 0, 4042, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4052, 41, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 17, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8188, 8192, 8190, 8198, 8184, 8171, 8218] avg=8192.0000 max=8218 max_rank=7 min=8171 min_rank=6 imbalance_ratio=1.003174 max_load_reduction=57318 max_load_reduction_pct=0.874603 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 4091, 4094, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 2, 0, 0, 4048, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 4074, 27, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4095, 3, 0, 0, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8222, 8192, 8203, 8174, 8174, 8167, 8209] avg=8192.0000 max=8222 max_rank=1 min=8167 min_rank=6 imbalance_ratio=1.003662 max_load_reduction=57314 max_load_reduction_pct=0.874542 moved_to_echo=32724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4041, 22, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4090, 23, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 1, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8224, 8192, 8196, 8159, 8187, 8184, 8201] avg=8192.0000 max=8224 max_rank=1 min=8159 min_rank=4 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32731 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 0, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 32640, 128, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32512, 0, 128, 0, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 0, 128, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 256, 128, 0, 32384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 128, 128, 32640, 256, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 0, 0, 0, 4086, 0, 4087, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 5, 0, 4086, 0, 0, 8, 4096, 66, 0, 0, 4096, 5, 0, 4096, 4092, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4059, 0, 4092, 4090, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3, 0, 4096, 0, 0, 0, 0, 4058, 0, 0, 0, 4096, 63, 0, 6, 4096, 8, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 4081, 0, 4096, 4089, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 26, 0, 5, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4070, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 20, 0, 0, 0, 4092, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 2, 0, 3994, 0, 0, 0, 4096, 166, 0, 1, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4026, 0, 4096, 4088, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 15, 0, 0, 0, 4076, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 25, 0, 4096, 0, 1, 3, 0, 3803, 0, 0, 15, 4096, 780, 0, 0, 4096, 0, 0, 4096, 4084, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4096, 0, 3582, 0, 4095, 4083, 0, 0, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[7, 4091, 0, 0, 0, 0, 5, 5, 4078, 0, 4080, 6, 5, 4, 0, 0, 0, 2, 3, 0, 4091, 1, 0, 3, 4090, 8, 0, 4, 6, 3804, 0, 0, 21, 4084, 371, 0, 0, 4092, 73, 4, 4090, 4088, 4089, 5, 0, 5, 4, 0, 0, 4087, 5, 0, 1, 0, 4089, 1, 4010, 4, 4092, 4018, 0, 4, 0, 6] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 4094, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 14, 0, 4096, 0, 0, 1, 0, 4095, 0, 0, 1, 4096, 7, 0, 5, 4096, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 4074, 0, 4095, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 42, 0, 3, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4052, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3983, 105, 2, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 0, 5, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 113, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8197, 8190, 8281, 8016, 8195, 8192, 8279] avg=8192.0000 max=8281 max_rank=3 min=8016 min_rank=4 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32682 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 0, 0, 0, 3979, 105, 2, 0, 0, 0, 0, 0, 4086, 0, 4087, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4084, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4086, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 66, 0, 0, 3889, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 113, 4090, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8186, 8179, 8276, 8064, 8191, 8192, 8265] avg=8192.0000 max=8276 max_rank=3 min=8064 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32712 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 15, 0, 0, 0, 3982, 105, 2, 0, 0, 0, 0, 0, 4076, 0, 4096, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 25, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 3, 0, 3803, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 780, 0, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3582, 0, 113, 4083, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8185, 8212, 7992, 8780, 8183, 8198, 7787] avg=8192.0000 max=8780 max_rank=4 min=7787 min_rank=7 imbalance_ratio=1.071777 max_load_reduction=56756 max_load_reduction_pct=0.866028 moved_to_echo=32948 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 20, 0, 0, 0, 3983, 105, 2, 0, 0, 0, 0, 0, 4092, 0, 4090, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 3994, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 166, 0, 1, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 113, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8195, 8188, 8181, 8152, 8195, 8192, 8227] avg=8192.0000 max=8227 max_rank=7 min=8152 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 3983, 105, 2, 0, 0, 0, 0, 0, 4088, 0, 4084, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4058, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 63, 0, 6, 3889, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 113, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8185, 8189, 8243, 8062, 8195, 8193, 8283] avg=8192.0000 max=8283 max_rank=7 min=8062 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32733 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[7, 4091, 0, 0, 0, 0, 5, 5, 3979, 105, 2, 0, 0, 0, 0, 0, 4078, 0, 4080, 6, 5, 4, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 4091, 1, 0, 3, 4085, 0, 0, 0, 0, 0, 0, 0, 4090, 8, 0, 4, 6, 3804, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 21, 4084, 371, 0, 0, 3886, 73, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4088, 4089, 5, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 1, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 4, 113, 4018, 0, 4, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8186, 8185, 8000, 8439, 8194, 8183, 8155] avg=8192.0000 max=8439 max_rank=4 min=8000 min_rank=3 imbalance_ratio=1.030151 max_load_reduction=57097 max_load_reduction_pct=0.871231 moved_to_echo=32971 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 3983, 105, 2, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 5, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 42, 0, 3, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 113, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8199, 8192, 8281, 8030, 8193, 8192, 8261] avg=8192.0000 max=8281 max_rank=3 min=8030 min_rank=4 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2, 0, 0, 0, 3985, 107, 4, 0, 0, 0, 0, 0, 4094, 0, 4095, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 14, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4095, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 7, 0, 5, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 110, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8203, 8201, 8282, 7993, 8190, 8194, 8279] avg=8192.0000 max=8282 max_rank=3 min=7993 min_rank=4 imbalance_ratio=1.010986 max_load_reduction=57254 max_load_reduction_pct=0.873627 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32000, 128, 1024, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 128, 128, 31872, 896, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 1536, 0, 128, 31104, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 32768, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 32128, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 29, 0, 0, 38, 4096, 0, 0, 0, 4075, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 0, 4085, 4096, 0, 2033, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 3444, 1, 2679, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 57, 0, 0, 9, 4096, 0, 0, 0, 3974, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4091, 4096, 0, 3490, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 3058, 0, 1703, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 152, 0, 0, 9, 4096, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 3502, 0, 2, 4096, 4, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2975, 0, 1554, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 439, 0, 1, 2, 4096, 0, 0, 0, 3963, 0, 8, 0, 4096, 0, 4096, 0, 0, 0, 0, 4080, 4096, 0, 3504, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 3108, 0, 1275, 0, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4089, 6, 0, 2, 0, 4088, 5, 1, 0, 5, 293, 5, 23, 21, 4089, 6, 0, 0, 4056, 7, 15, 5, 4090, 7, 4088, 0, 6, 0, 3, 4087, 4081, 4, 3925, 3, 0, 4087, 0, 4089, 6, 1, 9, 4089, 4090, 7, 0, 0, 9, 1, 0, 4069, 0, 2, 4090, 6, 0, 6, 3, 2160, 0, 1802, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 169, 0, 0, 3, 4096, 0, 0, 0, 4058, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 4096, 0, 3573, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1458, 0, 3027, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 176, 0, 0, 3, 4096, 0, 0, 0, 4054, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 2791, 0, 1, 4096, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2648, 0, 2614, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 88, 0, 0, 14, 4096, 0, 0, 0, 4057, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 4096, 0, 2408, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 2, 0, 0, 2990, 0, 2734, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 38, 4096, 3904, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4075, 0, 4, 0, 49, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2033, 0, 0, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3444, 1, 2679, 0, 5, 3096, 185, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8067, 8224, 8188, 7075, 8191, 8190, 9410] avg=8192.0000 max=9410 max_rank=7 min=7075 min_rank=4 imbalance_ratio=1.148682 max_load_reduction=56126 max_load_reduction_pct=0.856415 moved_to_echo=32866 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 9, 4096, 3904, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3974, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3490, 0, 0, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3058, 0, 1703, 0, 8, 3096, 185, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8066, 8119, 8194, 8532, 8191, 8194, 8052] avg=8192.0000 max=8532 max_rank=4 min=8052 min_rank=7 imbalance_ratio=1.041504 max_load_reduction=57004 max_load_reduction_pct=0.869812 moved_to_echo=32969 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 0, 1, 2, 4096, 3904, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3963, 0, 8, 0, 49, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3504, 0, 0, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3108, 0, 1275, 0, 15, 3096, 185, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8442, 8116, 8183, 8546, 8191, 8191, 7679] avg=8192.0000 max=8546 max_rank=4 min=7679 min_rank=7 imbalance_ratio=1.043213 max_load_reduction=56990 max_load_reduction_pct=0.869598 moved_to_echo=32607 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 6, 0, 2, 0, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 5, 293, 5, 23, 21, 4089, 3899, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 4056, 7, 15, 5, 49, 0, 0, 0, 0, 0, 0, 0, 6, 7, 4088, 0, 6, 0, 3, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4, 3925, 3, 0, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 9, 4089, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 4069, 0, 2, 4090, 6, 1, 1, 0, 0, 0, 0, 0, 0, 0, 6, 3, 2160, 0, 1802, 0, 0, 3090, 185, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8341, 8227, 8197, 8956, 8201, 8179, 7246] avg=8192.0000 max=8956 max_rank=4 min=7246 min_rank=7 imbalance_ratio=1.093262 max_load_reduction=56580 max_load_reduction_pct=0.863342 moved_to_echo=32582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 3, 4096, 3904, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4058, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3573, 0, 0, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 3027, 0, 4, 3096, 185, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8172, 8203, 8195, 8615, 8191, 8194, 7770] avg=8192.0000 max=8615 max_rank=4 min=7770 min_rank=7 imbalance_ratio=1.051636 max_load_reduction=56921 max_load_reduction_pct=0.868546 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 3, 4096, 3904, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4054, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2791, 0, 1, 946, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 0, 2614, 0, 2, 3096, 190, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8179, 8199, 8193, 7835, 8191, 8194, 8550] avg=8192.0000 max=8550 max_rank=7 min=7835 min_rank=4 imbalance_ratio=1.043701 max_load_reduction=56986 max_load_reduction_pct=0.869537 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4091, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 14, 4096, 3910, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4057, 0, 1, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2408, 0, 0, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 6, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2990, 0, 2734, 0, 4, 3102, 186, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8108, 8209, 8188, 7427, 8189, 8200, 9018] avg=8192.0000 max=9018 max_rank=7 min=7427 min_rank=4 imbalance_ratio=1.100830 max_load_reduction=56518 max_load_reduction_pct=0.862396 moved_to_echo=32834 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 9, 4096, 3904, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3502, 0, 2, 946, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2975, 0, 1554, 0, 1, 3096, 185, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8161, 8239, 8198, 8550, 8191, 8194, 7811] avg=8192.0000 max=8550 max_rank=4 min=7811 min_rank=7 imbalance_ratio=1.043701 max_load_reduction=56986 max_load_reduction_pct=0.869537 moved_to_echo=32746 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 0, 0, 32384, 128, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 21888, 128, 17408, 0, 128, 24832, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 0, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 25344, 128, 128, 7552, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 1408, 128, 128, 128, 32768, 31360, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3803, 0, 0, 4093, 3554, 1, 4096, 4095, 0, 0, 4096, 0, 4096, 0, 0, 111, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 725, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3814, 0, 1, 4061, 3188, 0, 4096, 4084, 0, 0, 4096, 0, 4081, 0, 0, 108, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 4095, 0, 0, 0, 1124, 5, 0, 0, 0, 0, 0, 4096, 4, 4091, 0, 0, 4096, 4092, 0, 0, 0, 0, 6, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3066, 0, 0, 4093, 3413, 2, 4096, 4095, 0, 0, 4096, 0, 4090, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1714, 0, 1, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 4096, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3675, 0, 0, 4080, 2618, 0, 4096, 4096, 0, 0, 4096, 0, 4076, 0, 0, 104, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 1823, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3956, 0, 0, 4096, 2558, 0, 4096, 4095, 0, 0, 4096, 0, 3995, 0, 0, 140, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1639, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3379, 0, 7, 4084, 3557, 0, 4096, 4088, 0, 0, 4096, 0, 3943, 0, 0, 123, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 4076, 0, 0, 0, 1286, 8, 2, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 4096, 4096, 0, 0, 0, 0, 17, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[7, 0, 8, 6, 3702, 2, 0, 4054, 3084, 4, 4090, 4091, 8, 0, 4085, 7, 3969, 0, 5, 42, 0, 3, 4088, 0, 0, 0, 24, 6, 5, 4088, 6, 0, 4, 0, 35, 5, 6, 4072, 1, 0, 2, 1448, 0, 0, 4, 1, 19, 6, 4071, 1, 4089, 5, 0, 4089, 4080, 0, 0, 6, 4, 18, 10, 4089, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2838, 0, 0, 4083, 3380, 0, 4096, 4092, 0, 0, 4096, 0, 4094, 0, 0, 319, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 1671, 1, 1, 0, 0, 0, 0, 4096, 1, 4094, 0, 0, 4096, 4095, 0, 0, 0, 0, 2, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3803, 0, 0, 4093, 506, 66, 6, 0, 0, 0, 0, 0, 3554, 1, 0, 4095, 0, 0, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 111, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2661, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2660, 1433, 0, 0, 0, 0, 0, 0, 0, 725, 0, 0, 0, 0, 0, 0, 4096, 2663, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8474, 8579, 8237, 8187, 8189, 7484, 8198, 8188] avg=8192.0000 max=8579 max_rank=1 min=7484 min_rank=5 imbalance_ratio=1.047241 max_load_reduction=56957 max_load_reduction_pct=0.869095 moved_to_echo=32059 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3814, 0, 1, 4061, 506, 66, 6, 0, 0, 0, 0, 0, 3188, 0, 0, 4084, 0, 0, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 108, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 2661, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4095, 0, 0, 2658, 1434, 0, 0, 0, 0, 0, 0, 0, 1124, 5, 0, 0, 0, 0, 0, 4096, 2662, 0, 0, 0, 0, 0, 0, 4096, 4, 4091, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8454, 8201, 8219, 8193, 8194, 7887, 8196, 8192] avg=8192.0000 max=8454 max_rank=0 min=7887 min_rank=5 imbalance_ratio=1.031982 max_load_reduction=57082 max_load_reduction_pct=0.871002 moved_to_echo=32469 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3379, 0, 7, 4084, 506, 66, 6, 0, 0, 0, 0, 0, 3557, 0, 0, 4088, 0, 0, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 0, 0, 123, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 2661, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4076, 0, 0, 2660, 1433, 0, 0, 0, 0, 0, 0, 0, 1286, 8, 2, 0, 0, 0, 0, 4096, 2663, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8048, 8574, 8096, 8188, 8181, 8055, 8191, 8203] avg=8192.0000 max=8574 max_rank=1 min=8048 min_rank=0 imbalance_ratio=1.046631 max_load_reduction=56962 max_load_reduction_pct=0.869171 moved_to_echo=32630 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3956, 0, 0, 4096, 506, 66, 6, 0, 0, 0, 0, 0, 2558, 0, 0, 4095, 0, 0, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 140, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2661, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2660, 1434, 0, 0, 0, 0, 0, 0, 0, 1639, 0, 0, 0, 0, 0, 0, 4096, 2662, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8630, 7582, 8165, 8187, 8190, 8397, 8198, 8187] avg=8192.0000 max=8630 max_rank=0 min=7582 min_rank=1 imbalance_ratio=1.053467 max_load_reduction=56906 max_load_reduction_pct=0.868317 moved_to_echo=32972 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3066, 0, 0, 4093, 506, 66, 6, 0, 0, 0, 0, 0, 3413, 2, 0, 4095, 0, 0, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 6, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2661, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2664, 1434, 0, 0, 0, 0, 0, 0, 0, 1714, 0, 1, 0, 0, 0, 0, 4096, 2662, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7737, 8439, 8126, 8187, 8194, 8473, 8192, 8188] avg=8192.0000 max=8473 max_rank=5 min=7737 min_rank=0 imbalance_ratio=1.034302 max_load_reduction=57063 max_load_reduction_pct=0.870712 moved_to_echo=33047 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3675, 0, 0, 4080, 506, 66, 6, 0, 0, 0, 0, 0, 2618, 0, 0, 4096, 0, 0, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 104, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 2661, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 2660, 1434, 0, 0, 0, 0, 0, 0, 0, 1823, 0, 0, 0, 0, 0, 0, 4096, 2662, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8333, 7643, 8210, 8194, 8191, 8581, 8198, 8186] avg=8192.0000 max=8581 max_rank=5 min=7643 min_rank=1 imbalance_ratio=1.047485 max_load_reduction=56955 max_load_reduction_pct=0.869064 moved_to_echo=33156 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 8, 6, 3702, 2, 0, 4054, 504, 66, 6, 0, 0, 0, 0, 0, 3084, 4, 0, 4091, 8, 0, 927, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 0, 5, 42, 0, 3, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 6, 5, 4088, 6, 0, 2654, 1424, 0, 0, 0, 0, 0, 0, 4, 0, 35, 5, 6, 4072, 1, 0, 2650, 1431, 0, 0, 0, 0, 0, 0, 2, 1448, 0, 0, 4, 1, 19, 6, 4090, 2658, 0, 0, 0, 0, 0, 0, 4071, 1, 4089, 5, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 18, 10, 4083, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8355, 8121, 8041, 8207, 8204, 8228, 8172, 8208] avg=8192.0000 max=8355 max_rank=0 min=8041 min_rank=2 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32812 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2838, 0, 0, 4083, 507, 67, 10, 0, 0, 0, 0, 0, 3380, 0, 0, 4092, 0, 0, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 319, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2664, 1433, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 2662, 1434, 0, 0, 0, 0, 0, 0, 0, 1671, 1, 1, 0, 0, 0, 0, 4096, 2662, 0, 0, 0, 0, 0, 0, 4096, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4086, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7505, 8397, 8442, 8193, 8193, 8431, 8191, 8184] avg=8192.0000 max=8442 max_rank=2 min=7505 min_rank=0 imbalance_ratio=1.030518 max_load_reduction=57094 max_load_reduction_pct=0.871185 moved_to_echo=32999 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32384, 0, 128, 1024, 0, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[25472, 128, 0, 32768, 128, 0, 7552, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 32768, 128, 0, 21376, 11520, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 11520, 128, 128, 128, 128, 128, 128, 32768, 21376, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 32768, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 28288, 128, 128, 32768, 4096, 640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 128, 0, 21376, 11520, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3777, 0, 112, 4095, 0, 1, 3, 0, 0, 0, 4096, 4096, 4094, 4095, 4096, 0, 701, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1596, 0, 0, 0, 0, 4096, 4096, 0, 0, 1988, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 2324, 0, 174, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 4078, 4096, 0, 555, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2404, 0, 0, 0, 0, 4096, 4096, 0, 0, 2748, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 3925, 0, 89, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 4078, 4096, 0, 1369, 4087, 0, 4084, 0, 6, 4, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 1181, 0, 0, 0, 0, 4096, 4096, 0, 0, 1637, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3581, 0, 64, 4096, 0, 0, 1, 0, 0, 1, 4096, 4096, 4094, 4096, 4096, 0, 2394, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1420, 0, 0, 0, 0, 4096, 4096, 0, 0, 725, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[8, 0, 2, 4089, 1, 6, 0, 4089, 7, 4092, 7, 7, 0, 2545, 7, 1, 4089, 0, 4, 0, 6, 0, 0, 4079, 4083, 4088, 3942, 4089, 7, 7, 4090, 3, 4067, 6, 9, 0, 6, 0, 0, 0, 0, 0, 29, 0, 6, 5, 0, 8, 4085, 0, 8, 0, 3527, 7, 0, 3, 0, 4088, 4089, 0, 6, 2239, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 3045, 0, 79, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4086, 4096, 0, 360, 4096, 0, 4067, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2049, 0, 0, 0, 0, 4096, 4096, 0, 0, 2671, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4076, 0, 0, 0, 2869, 0, 77, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4067, 4096, 0, 258, 4092, 0, 4042, 0, 5, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2676, 0, 0, 0, 0, 4096, 4096, 0, 0, 2333, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 3752, 0, 274, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 4095, 4096, 0, 2286, 4094, 0, 4095, 0, 1, 1, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 576, 0, 0, 0, 0, 4096, 4096, 0, 0, 1273, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3777, 0, 112, 763, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 3, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 701, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 18, 4077, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1596, 0, 0, 0, 1952, 197, 17, 0, 0, 0, 0, 0, 0, 4096, 2144, 0, 0, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8748, 8196, 7931, 8194, 8189, 7858, 8228] avg=8192.0000 max=8748 max_rank=1 min=7858 min_rank=6 imbalance_ratio=1.067871 max_load_reduction=56788 max_load_reduction_pct=0.866516 moved_to_echo=32469 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 3925, 0, 89, 761, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 1369, 3129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 6, 4, 0, 0, 0, 28, 4079, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 1181, 0, 0, 0, 1952, 197, 17, 0, 0, 0, 0, 0, 0, 4096, 2144, 0, 0, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8863, 8191, 8576, 8205, 8188, 7444, 7877] avg=8192.0000 max=8863 max_rank=1 min=7444 min_rank=6 imbalance_ratio=1.081909 max_load_reduction=56673 max_load_reduction_pct=0.864761 moved_to_echo=31714 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2324, 0, 174, 763, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 555, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 17, 4078, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2404, 0, 0, 0, 1952, 197, 17, 0, 0, 0, 0, 0, 0, 4096, 2144, 0, 0, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7356, 8193, 7768, 8184, 8189, 8666, 8988] avg=8192.0000 max=8988 max_rank=7 min=7356 min_rank=1 imbalance_ratio=1.097168 max_load_reduction=56548 max_load_reduction_pct=0.862854 moved_to_echo=34027 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 2869, 0, 77, 762, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 258, 3133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 5, 0, 0, 0, 0, 73, 4078, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2676, 0, 0, 0, 1952, 197, 18, 0, 0, 0, 0, 0, 0, 4096, 2144, 0, 0, 2333, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7784, 8192, 7458, 8201, 8189, 8939, 8581] avg=8192.0000 max=8939 max_rank=6 min=7458 min_rank=3 imbalance_ratio=1.091187 max_load_reduction=56597 max_load_reduction_pct=0.863602 moved_to_echo=33910 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 2, 4089, 1, 6, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4092, 7, 7, 0, 2545, 7, 1, 761, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4, 0, 6, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 0, 7, 7, 3132, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 6, 9, 0, 6, 0, 0, 0, 4071, 3, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 6, 5, 0, 8, 4089, 4080, 0, 0, 0, 0, 0, 0, 4085, 0, 8, 0, 3527, 7, 0, 3, 1949, 197, 17, 0, 0, 0, 0, 0, 0, 4088, 2140, 0, 6, 2239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 7427, 8178, 7091, 8162, 8217, 9793, 8473] avg=8192.0000 max=9793 max_rank=6 min=7091 min_rank=3 imbalance_ratio=1.195435 max_load_reduction=55743 max_load_reduction_pct=0.850571 moved_to_echo=34645 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3581, 0, 64, 763, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 1, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2394, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 4078, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1420, 0, 0, 0, 1952, 197, 16, 0, 0, 0, 0, 0, 0, 4096, 2144, 0, 0, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8504, 8195, 9625, 8186, 8188, 7681, 6965] avg=8192.0000 max=9625 max_rank=3 min=6965 min_rank=7 imbalance_ratio=1.174927 max_load_reduction=55911 max_load_reduction_pct=0.853134 moved_to_echo=31020 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 3045, 0, 79, 763, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 360, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 35, 4079, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2049, 0, 0, 0, 1952, 197, 17, 0, 0, 0, 0, 0, 0, 4096, 2144, 0, 0, 2671, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7973, 8193, 7581, 8185, 8188, 8311, 8913] avg=8192.0000 max=8913 max_rank=7 min=7581 min_rank=3 imbalance_ratio=1.088013 max_load_reduction=56623 max_load_reduction_pct=0.863998 moved_to_echo=33597 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 3752, 0, 274, 765, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2286, 3125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 0, 39, 4079, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 576, 0, 0, 0, 1955, 200, 17, 0, 0, 0, 0, 0, 0, 4096, 2141, 0, 0, 1273, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8881, 8198, 9506, 8219, 8188, 6844, 7511] avg=8192.0000 max=9506 max_rank=3 min=6844 min_rank=6 imbalance_ratio=1.160400 max_load_reduction=56030 max_load_reduction_pct=0.854950 moved_to_echo=30762 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 17152, 0, 128, 15616, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 0, 128, 128, 15488, 128, 0, 128, 15616, 1664, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 128, 128, 128, 0, 0, 256, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 0, 25856, 128, 896, 6144, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32640, 0, 128, 7936, 25088, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[2, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 26, 4096, 0, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 790, 4085, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3295, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 4096, 4094, 0, 0, 0, 4095, 0, 0, 4092, 4095, 0, 11, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3631, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4085, 1, 0, 0, 62, 0, 0, 35, 4077, 0, 6, 7, 0, 1, 7, 3, 9, 0, 1, 0, 5, 915, 4088, 7, 6, 0, 3, 4088, 10, 1, 4089, 0, 3241, 4067, 0, 8, 8, 4090, 0, 18, 2, 0, 0, 4087, 7, 3941, 7, 4081, 4090, 7, 0, 7, 4089, 11, 6, 4087, 4088, 0, 1, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 31, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 389, 4074, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3707, 4096, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4078, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 4086, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3784, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 8, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 50, 4096, 0, 0, 0, 0, 5, 0, 20, 0, 0, 0, 0, 0, 1663, 4075, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2384, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4071, 0, 4096, 4096, 0, 0, 0, 4095, 0, 0, 4092, 4096, 0, 25, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1162, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2934, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 546, 4090, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3549, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 4093, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 3, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4068, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3299, 772, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 470, 4096, 0, 0, 0, 0, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 3631, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8172, 8186, 7890, 8509, 8191, 8220, 8183] avg=8192.0000 max=8509 max_rank=4 min=7890 min_rank=3 imbalance_ratio=1.038696 max_load_reduction=57027 max_load_reduction_pct=0.870163 moved_to_echo=33103 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4096, 0, 0, 4, 0, 4066, 15, 6, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 0, 0, 0, 3299, 772, 4, 1, 0, 0, 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 790, 4085, 0, 0, 0, 0, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 3295, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4091, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8198, 8193, 8199, 8173, 8189, 8201, 8194] avg=8192.0000 max=8201 max_rank=6 min=8173 min_rank=4 imbalance_ratio=1.001099 max_load_reduction=57335 max_load_reduction_pct=0.874863 moved_to_echo=32757 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4068, 15, 6, 0, 0, 0, 0, 0, 0, 0, 50, 4096, 0, 0, 0, 0, 3299, 772, 4, 1, 0, 0, 0, 0, 5, 0, 20, 0, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 1663, 4075, 0, 0, 0, 0, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 2384, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4092, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8222, 8210, 9062, 7262, 8191, 8195, 8209] avg=8192.0000 max=9062 max_rank=3 min=7262 min_rank=4 imbalance_ratio=1.106201 max_load_reduction=56474 max_load_reduction_pct=0.861725 moved_to_echo=31857 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4085, 1, 0, 0, 62, 4062, 15, 6, 0, 0, 0, 0, 0, 0, 0, 35, 4077, 0, 6, 7, 0, 3293, 771, 4, 1, 0, 0, 0, 0, 1, 7, 3, 9, 0, 1, 0, 5, 4089, 4081, 0, 0, 0, 0, 0, 0, 915, 4088, 7, 6, 0, 3, 3317, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 781, 0, 3241, 4067, 0, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 18, 2, 0, 0, 4087, 7, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 7, 4081, 28, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 4087, 4084, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8194, 8196, 8346, 8106, 8203, 8071, 8189] avg=8192.0000 max=8346 max_rank=3 min=8071 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32569 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4068, 15, 6, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 0, 0, 0, 0, 3299, 772, 4, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 389, 4074, 0, 0, 0, 0, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 3707, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8203, 8192, 7787, 8588, 8191, 8202, 8188] avg=8192.0000 max=8588 max_rank=4 min=7787 min_rank=3 imbalance_ratio=1.048340 max_load_reduction=56948 max_load_reduction_pct=0.868958 moved_to_echo=33169 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4068, 15, 6, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 3299, 772, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 305, 4086, 0, 0, 0, 0, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 3784, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8187, 8186, 7715, 8664, 8191, 8212, 8196] avg=8192.0000 max=8664 max_rank=4 min=7715 min_rank=3 imbalance_ratio=1.057617 max_load_reduction=56872 max_load_reduction_pct=0.867798 moved_to_echo=33263 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4067, 15, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3299, 772, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 1162, 4096, 0, 0, 0, 0, 3324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 782, 0, 2934, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8172, 8186, 8582, 7812, 8192, 8220, 8188] avg=8192.0000 max=8582 max_rank=3 min=7812 min_rank=4 imbalance_ratio=1.047607 max_load_reduction=56954 max_load_reduction_pct=0.869049 moved_to_echo=32412 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 4069, 19, 7, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 3300, 777, 5, 4, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 546, 4090, 0, 0, 0, 0, 3319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 0, 3549, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4091, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8188, 8187, 7955, 8422, 8188, 8215, 8189] avg=8192.0000 max=8422 max_rank=4 min=7955 min_rank=3 imbalance_ratio=1.028076 max_load_reduction=57114 max_load_reduction_pct=0.871490 moved_to_echo=33014 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 128, 128, 32640, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6272, 32768, 128, 128, 0, 128, 26624, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 6272, 0, 26624, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 0, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 128, 32768, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 256, 32768, 0, 128, 128, 0, 26496, 6272, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 5, 4085, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 7, 4089, 0, 10, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 1, 4087, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 9, 4092, 0, 6, 0, 0, 0, 0, 4096, 4096, 4096, 3, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4089, 0, 4089, 4087, 4, 5, 16, 7, 7, 4089, 0, 7, 0, 177, 15, 4064, 4084, 0, 7, 7, 10, 1, 6, 0, 2, 4, 7, 0, 5, 0, 4078, 4085, 0, 4, 48, 22, 7, 0, 4086, 0, 4070, 0, 7, 8, 0, 8, 0, 4082, 0, 13, 0, 4082, 7, 20, 9, 0, 0, 4, 4077, 3854, 4088, 0, 0, 4088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 1, 4094, 4096, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 4095, 0, 1, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4064, 15, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8172, 8190, 8192, 8183, 8197, 8186, 8224] avg=8192.0000 max=8224 max_rank=7 min=8172 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4085, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4064, 15, 8, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4086, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 10, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8172, 8190, 8203, 8183, 8182, 8196, 8224] avg=8192.0000 max=8224 max_rank=7 min=8172 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4064, 15, 9, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4092, 0, 6, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 32, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8174, 8191, 8197, 8184, 8184, 8195, 8227] avg=8192.0000 max=8227 max_rank=7 min=8174 min_rank=1 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4064, 15, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8172, 8191, 8192, 8184, 8197, 8185, 8224] avg=8192.0000 max=8224 max_rank=7 min=8172 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 4087, 4, 5, 16, 7, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4069, 0, 7, 0, 177, 15, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 7, 7, 10, 1, 6, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 2, 4, 7, 0, 5, 0, 4078, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 48, 22, 7, 0, 4086, 0, 4056, 15, 8, 0, 0, 0, 0, 0, 4070, 0, 7, 8, 0, 8, 0, 4082, 5, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4082, 7, 20, 9, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 0, 4, 4077, 3854, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8339, 8202, 8181, 8246, 8180, 8213, 7967] avg=8192.0000 max=8339 max_rank=1 min=7967 min_rank=7 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4064, 15, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8172, 8190, 8192, 8183, 8197, 8186, 8224] avg=8192.0000 max=8224 max_rank=7 min=8172 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4064, 15, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8172, 8191, 8192, 8183, 8193, 8189, 8224] avg=8192.0000 max=8224 max_rank=7 min=8172 min_rank=1 imbalance_ratio=1.003906 max_load_reduction=57312 max_load_reduction_pct=0.874512 moved_to_echo=32789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4066, 19, 9, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 1, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8163, 8191, 8187, 8190, 8206, 8186, 8222] avg=8192.0000 max=8222 max_rank=7 min=8163 min_rank=1 imbalance_ratio=1.003662 max_load_reduction=57314 max_load_reduction_pct=0.874542 moved_to_echo=32804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 32640, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32640, 128, 128, 0, 256, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 128, 0, 32768, 0, 32512, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4091, 4081, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 4096, 4096, 0, 1156, 9, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 113, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4085] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 761, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 6, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4086, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 1160, 2, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4060, 0, 0, 0, 0, 147, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4091, 3520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 542, 3, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 69, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4071] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4089, 4061, 3799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 4096, 4096, 0, 199, 10, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 3995, 0, 0, 7, 0, 217, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4063] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4065, 0, 1, 7, 4, 0, 4079, 4092, 3191, 8, 0, 0, 0, 0, 7, 7, 0, 2, 0, 22, 6, 98, 0, 4089, 0, 3, 4086, 4088, 0, 918, 0, 33, 4087, 4089, 4074, 0, 0, 0, 0, 7, 8, 4089, 4022, 19, 0, 15, 0, 13, 4074, 4087, 0, 62, 6, 4058, 5, 0, 0, 8, 8, 0, 9, 7, 5, 3978] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 1034, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 49, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4091, 2880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 1227, 1, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4092] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3346, 0, 0, 0, 0, 0, 0, 0, 4096, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3150, 882, 49, 1, 0, 0, 0, 0, 0, 0, 4096, 3214, 0, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 4083, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3178, 934, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8356, 8178, 8071, 8193, 8136, 8191, 8207] avg=8192.0000 max=8356 max_rank=1 min=8071 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=57180 max_load_reduction_pct=0.872498 moved_to_echo=32727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 12, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 2854, 0, 0, 0, 0, 0, 0, 0, 4096, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3146, 882, 49, 1, 0, 0, 0, 0, 5, 0, 4096, 3214, 0, 1156, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 4079, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4085, 3178, 933, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 7864, 8174, 8480, 8197, 8239, 8191, 8202] avg=8192.0000 max=8480 max_rank=3 min=7864 min_rank=1 imbalance_ratio=1.035156 max_load_reduction=57056 max_load_reduction_pct=0.870605 moved_to_echo=32829 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 13, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 3799, 0, 0, 0, 0, 0, 0, 0, 4096, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3144, 882, 49, 1, 0, 0, 0, 0, 15, 0, 4096, 3214, 0, 199, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 3995, 0, 0, 7, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4063, 3178, 932, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8809, 8172, 7534, 8196, 8266, 8191, 8198] avg=8192.0000 max=8809 max_rank=1 min=7534 min_rank=3 imbalance_ratio=1.075317 max_load_reduction=56727 max_load_reduction_pct=0.865585 moved_to_echo=32851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 12, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 2836, 0, 0, 0, 0, 0, 0, 0, 4096, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3150, 882, 49, 1, 0, 0, 0, 0, 0, 0, 4096, 3214, 0, 1160, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 4060, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4086, 3178, 934, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 7846, 8178, 8472, 8196, 8254, 8191, 8205] avg=8192.0000 max=8472 max_rank=3 min=7846 min_rank=1 imbalance_ratio=1.034180 max_load_reduction=57064 max_load_reduction_pct=0.870728 moved_to_echo=32846 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 12, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 3520, 0, 0, 0, 0, 0, 0, 0, 4096, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3150, 882, 49, 1, 0, 0, 0, 0, 0, 0, 4096, 3214, 0, 542, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 4083, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4071, 3178, 934, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8530, 8178, 7855, 8196, 8199, 8191, 8188] avg=8192.0000 max=8530 max_rank=1 min=7855 min_rank=3 imbalance_ratio=1.041260 max_load_reduction=57006 max_load_reduction_pct=0.869843 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4065, 0, 1, 7, 4, 0, 12, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3191, 8, 0, 0, 0, 0, 7, 7, 4087, 912, 0, 0, 0, 0, 0, 0, 0, 2, 0, 22, 6, 98, 0, 4089, 3137, 881, 49, 1, 0, 0, 0, 0, 0, 3, 4086, 3207, 0, 918, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 5, 4074, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4040, 4022, 19, 0, 15, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 62, 6, 4058, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 0, 9, 7, 5, 3978, 3172, 930, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8212, 8285, 8247, 8173, 8117, 8204, 8117] avg=8192.0000 max=8285 max_rank=2 min=8117 min_rank=5 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3018, 0, 0, 0, 0, 0, 0, 0, 4096, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3150, 882, 49, 1, 0, 0, 0, 0, 0, 0, 4096, 3214, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 4094, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3179, 934, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8028, 8178, 8344, 8193, 8190, 8191, 8208] avg=8192.0000 max=8344 max_rank=3 min=8028 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=57192 max_load_reduction_pct=0.872681 moved_to_echo=32782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 8, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 2880, 0, 0, 0, 0, 0, 0, 0, 4096, 915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3151, 888, 52, 6, 0, 0, 0, 0, 1, 0, 4096, 3208, 0, 1227, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 4089, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4092, 3181, 936, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 7891, 8193, 8533, 8192, 8135, 8186, 8211] avg=8192.0000 max=8533 max_rank=3 min=7891 min_rank=1 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=32724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32384, 32512, 128, 0, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 32640, 25472, 7552, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 25728, 0, 7040, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 32768, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25472, 128, 0, 0, 0, 0, 128, 128, 32768, 7424, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 32768, 25216, 7168, 512, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 378, 0, 0, 0, 0, 4091, 4086, 53, 4096, 4096, 0, 0, 0, 1, 0, 0, 4096, 5, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 4, 4096, 4011, 0, 1122, 2628, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 102, 0, 0, 0, 0, 4096, 4095, 661, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3581, 0, 1001, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4047, 0, 23, 7, 3882, 4, 0, 0, 712, 1, 6, 0, 0, 4083, 4091, 57, 4087, 4088, 0, 2, 0, 1407, 0, 8, 4086, 0, 0, 6, 11, 4084, 4087, 0, 8, 0, 13, 6, 0, 0, 5, 0, 4088, 4049, 0, 4086, 2709, 6, 379, 3174, 0, 0, 9, 2, 2, 36, 0, 0, 3, 4089, 9, 4069, 0, 10, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 359, 0, 0, 0, 0, 4096, 4090, 103, 4096, 4096, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3793, 0, 1010, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1540, 0, 0, 0, 0, 4089, 4087, 19, 4096, 4096, 0, 0, 0, 47, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 4096, 3298, 0, 393, 2898, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1114, 0, 0, 0, 0, 4096, 4096, 107, 4096, 4096, 0, 0, 0, 55, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3737, 0, 291, 2887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1119, 0, 0, 0, 0, 4096, 4096, 41, 4096, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4075, 0, 205, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 285, 0, 0, 0, 0, 4095, 4094, 106, 4096, 4096, 0, 0, 0, 2, 0, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 4005, 0, 1277, 2517, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 3401, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 661, 4096, 3765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 3581, 0, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 2846, 0, 0, 0, 0, 0, 0, 0, 4096, 695, 331, 234, 3, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 7598, 8522, 8192, 8190, 8418, 8205, 8192] avg=8192.0000 max=8522 max_rank=2 min=7598 min_rank=1 imbalance_ratio=1.040283 max_load_reduction=57014 max_load_reduction_pct=0.869965 moved_to_echo=33005 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 3397, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4096, 3765, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 4, 0, 4011, 0, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 0, 0, 0, 0, 0, 0, 0, 4096, 694, 331, 234, 2, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 7861, 7915, 8196, 8191, 8973, 7985, 8196] avg=8192.0000 max=8973 max_rank=5 min=7861 min_rank=1 imbalance_ratio=1.095337 max_load_reduction=56563 max_load_reduction_pct=0.863083 moved_to_echo=33345 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 1540, 0, 0, 0, 0, 3396, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 3765, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 5, 0, 3298, 0, 393, 0, 0, 0, 0, 0, 0, 0, 0, 2898, 0, 0, 0, 0, 0, 0, 0, 4096, 693, 331, 234, 3, 0, 0, 0, 7, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 9023, 7927, 8191, 8190, 7532, 8255, 8199] avg=8192.0000 max=9023 max_rank=1 min=7532 min_rank=5 imbalance_ratio=1.101440 max_load_reduction=56513 max_load_reduction_pct=0.862320 moved_to_echo=32176 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 3401, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 103, 4096, 3765, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 3793, 0, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 2926, 0, 0, 0, 0, 0, 0, 0, 4096, 695, 331, 234, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 7850, 7971, 8191, 8190, 8639, 8284, 8192] avg=8192.0000 max=8639 max_rank=5 min=7850 min_rank=1 imbalance_ratio=1.054565 max_load_reduction=56897 max_load_reduction_pct=0.868179 moved_to_echo=33305 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4047, 0, 23, 7, 3882, 4, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 712, 1, 6, 0, 0, 3391, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4087, 3757, 0, 2, 0, 1407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4085, 0, 0, 6, 11, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 13, 6, 0, 0, 5, 4088, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 3792, 0, 0, 2709, 6, 379, 0, 0, 0, 0, 0, 0, 0, 0, 3174, 0, 0, 9, 2, 2, 36, 0, 4086, 692, 331, 231, 3, 0, 0, 0, 0, 3, 4089, 9, 4069, 0, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7990, 8201, 9310, 8194, 8204, 6886, 8566, 8185] avg=8192.0000 max=9310 max_rank=2 min=6886 min_rank=5 imbalance_ratio=1.136475 max_load_reduction=56226 max_load_reduction_pct=0.857941 moved_to_echo=31841 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 1114, 0, 0, 0, 0, 3401, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 107, 4096, 3765, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 3737, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 2887, 0, 0, 0, 0, 0, 0, 0, 4096, 695, 331, 234, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8611, 8023, 8191, 8191, 7864, 8245, 8192] avg=8192.0000 max=8611 max_rank=1 min=7864 min_rank=5 imbalance_ratio=1.051147 max_load_reduction=56925 max_load_reduction_pct=0.868607 moved_to_echo=32492 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 26, 1, 0, 0, 0, 0, 0, 0, 0, 1119, 0, 0, 0, 0, 3401, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 41, 4096, 3765, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 4075, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 2750, 0, 0, 0, 0, 0, 0, 0, 4096, 695, 331, 234, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8616, 7904, 8191, 8190, 8116, 8108, 8192] avg=8192.0000 max=8616 max_rank=1 min=7904 min_rank=2 imbalance_ratio=1.051758 max_load_reduction=56920 max_load_reduction_pct=0.868530 moved_to_echo=32606 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 32, 3, 5, 0, 0, 0, 0, 0, 0, 285, 0, 0, 0, 0, 3397, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 106, 4096, 3760, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 1, 0, 4005, 0, 1277, 0, 0, 0, 0, 0, 0, 0, 0, 2517, 0, 0, 0, 0, 0, 0, 0, 4096, 698, 336, 239, 2, 0, 0, 0, 1, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 7776, 7964, 8190, 8190, 9108, 7888, 8188] avg=8192.0000 max=9108 max_rank=5 min=7776 min_rank=1 imbalance_ratio=1.111816 max_load_reduction=56428 max_load_reduction_pct=0.861023 moved_to_echo=33374 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 32768, 0, 128, 128, 32640, 128, 0, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 30720, 128, 0, 29312, 128, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 5632, 128, 128, 0, 0, 27264, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[1152, 32768, 30208, 0, 128, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[22656, 0, 0, 128, 128, 128, 128, 0, 32768, 5632, 2688, 1920, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3767, 0, 0, 4096, 0, 306, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 1, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4076, 0, 0, 0, 0, 95, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 3695, 0, 0, 4096, 0, 287, 0, 8, 4096, 0, 0, 0, 4095, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 12, 4088, 1, 0, 9, 4092, 0, 4096, 4096, 4096, 23, 0, 4091, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4031, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 35, 0, 0, 0, 3632, 0, 0, 4096, 0, 428, 0, 8, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4082, 2, 0, 12, 4096, 0, 4096, 4089, 4096, 65, 0, 4094, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4061, 0, 0, 0, 0, 63, 0, 0, 0, 0, 4095, 0, 8, 0, 0, 0, 3973, 0, 0, 4096, 0, 68, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 3, 4096, 0, 4096, 4096, 4096, 15, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 54, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3943, 0, 0, 4096, 0, 92, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 4095, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 8, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[8, 0, 4, 4092, 1, 0, 8, 0, 5, 0, 8, 1, 0, 4089, 0, 0, 20, 0, 6, 3825, 8, 8, 4085, 10, 272, 11, 61, 4072, 0, 0, 6, 4087, 4090, 0, 0, 4060, 0, 3, 0, 35, 12, 365, 4055, 1, 7, 0, 4088, 1, 3711, 4079, 4078, 13, 4, 4089, 7, 0, 10, 4024, 4086, 0, 0, 31, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4080, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 3980, 0, 0, 4096, 0, 108, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 16, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 0, 0, 0, 0, 52, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 3887, 0, 0, 4096, 0, 155, 0, 2, 4096, 1, 0, 0, 4096, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 3, 4096, 0, 4096, 4096, 4096, 2, 0, 4095, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 4095, 18, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 4096, 0, 0, 4044, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 0, 0, 4096, 0, 213, 40, 0, 0, 0, 0, 0, 0, 306, 0, 0, 4096, 0, 0, 0, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 1, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8169, 8116, 8276, 8193, 8151, 8226, 8198] avg=8192.0000 max=8276 max_rank=3 min=8116 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4076, 0, 0, 0, 0, 4091, 18, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 0, 4096, 0, 10, 4044, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 0, 0, 4096, 0, 213, 39, 0, 0, 0, 0, 0, 0, 287, 0, 8, 4096, 0, 0, 0, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4088, 1, 0, 9, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 34, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8254, 8043, 8264, 8186, 8159, 8249, 8196] avg=8192.0000 max=8264 max_rank=3 min=8043 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32790 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4031, 0, 0, 0, 0, 4094, 18, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 35, 4044, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3632, 0, 0, 4096, 0, 213, 40, 0, 0, 0, 0, 0, 0, 428, 0, 8, 4096, 0, 0, 0, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4082, 2, 0, 12, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 34, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8186, 7981, 8406, 8193, 8149, 8284, 8194] avg=8192.0000 max=8406 max_rank=3 min=7981 min_rank=2 imbalance_ratio=1.026123 max_load_reduction=57130 max_load_reduction_pct=0.871735 moved_to_echo=32820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4061, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 4095, 0, 8, 4044, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 0, 4096, 0, 213, 40, 0, 0, 0, 0, 0, 0, 68, 0, 0, 4096, 0, 0, 0, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 3, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 34, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 8219, 8322, 8038, 8193, 8152, 8241, 8196] avg=8192.0000 max=8322 max_rank=2 min=8038 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4080, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 6, 4044, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 4096, 0, 213, 40, 0, 0, 0, 0, 0, 0, 108, 0, 0, 4096, 0, 0, 0, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 34, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8156, 8329, 8078, 8193, 8149, 8242, 8195] avg=8192.0000 max=8329 max_rank=2 min=8078 min_rank=3 imbalance_ratio=1.016724 max_load_reduction=57207 max_load_reduction_pct=0.872910 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 4096, 0, 0, 4044, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 0, 0, 4096, 0, 213, 40, 0, 0, 0, 0, 0, 0, 92, 0, 0, 4096, 0, 0, 0, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 8, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8203, 8292, 8062, 8193, 8150, 8225, 8203] avg=8192.0000 max=8292 max_rank=2 min=8062 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4, 4092, 1, 0, 8, 0, 4089, 17, 0, 0, 0, 0, 0, 0, 5, 0, 8, 1, 0, 4089, 0, 0, 4027, 9, 0, 0, 0, 0, 0, 0, 20, 0, 6, 3825, 8, 8, 4085, 10, 212, 39, 0, 0, 0, 0, 0, 0, 272, 11, 61, 4072, 0, 0, 6, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4060, 0, 3, 0, 35, 1, 0, 0, 0, 0, 0, 0, 0, 12, 365, 4055, 1, 7, 0, 4045, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3711, 4079, 34, 13, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4024, 4086, 0, 0, 31, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8139, 8213, 8288, 8189, 8486, 7848, 8154] avg=8192.0000 max=8486 max_rank=5 min=7848 min_rank=6 imbalance_ratio=1.035889 max_load_reduction=57050 max_load_reduction_pct=0.870514 moved_to_echo=32677 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 0, 0, 0, 0, 4095, 19, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 0, 4096, 0, 1, 4050, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 0, 4096, 0, 215, 42, 0, 0, 0, 0, 0, 0, 155, 0, 2, 4096, 1, 0, 0, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 3, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 27, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8210, 8240, 8124, 8196, 8140, 8221, 8200] avg=8192.0000 max=8240 max_rank=2 min=8124 min_rank=3 imbalance_ratio=1.005859 max_load_reduction=57296 max_load_reduction_pct=0.874268 moved_to_echo=32757 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 30720, 128, 128, 32768, 128, 1792, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 32640, 128, 0, 128, 0, 32768, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 0, 128, 128, 0, 32768, 0, 128, 32384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1792, 128, 128, 32768, 128, 0, 128, 31104, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 32768, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32384, 32768, 384, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 512, 32768, 128, 128, 128, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 9, 0, 4096, 1, 0, 0, 0, 4096, 4095, 4090, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 33, 0, 0, 0, 0, 6, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4070, 0, 4075, 0, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4096, 0, 63, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 104, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3954, 0, 4066, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 4057, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 13, 0, 4096, 0, 145, 0, 0, 4096, 4096, 4088, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 149, 0, 0, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3830, 0, 4036, 0, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4081, 0, 0, 0, 4088, 4084, 4049, 0, 3, 12, 0, 4090, 16, 0, 0, 9, 4087, 0, 1995, 0, 0, 4087, 4087, 4088, 0, 8, 0, 102, 0, 8, 388, 8, 0, 1, 7, 9, 13, 3678, 4, 9, 3978, 146, 5, 8, 9, 3, 0, 4088, 11, 4085, 4081, 0, 7, 32, 0, 8, 7, 4069, 0, 1985, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 175, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3954, 0, 4041, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 498, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3588, 0, 4096, 3, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 4, 0, 0, 4096, 4095, 4093, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 225, 0, 0, 0, 0, 1, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 3871, 0, 4090, 1, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4065, 24, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3823, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4072, 73, 0, 0, 0, 0, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 4096, 1, 0, 0, 0, 273, 154, 22, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8185, 7927, 8121, 8240, 8066, 8189, 8607] avg=8192.0000 max=8607 max_rank=7 min=7927 min_rank=2 imbalance_ratio=1.050659 max_load_reduction=56929 max_load_reduction_pct=0.868668 moved_to_echo=33102 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 9, 0, 4062, 24, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3822, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 4096, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4072, 72, 0, 0, 0, 0, 0, 0, 4087, 33, 0, 0, 0, 0, 6, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4075, 0, 4, 0, 0, 273, 154, 22, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8190, 7913, 8135, 8240, 8068, 8188, 8602] avg=8192.0000 max=8602 max_rank=7 min=7913 min_rank=2 imbalance_ratio=1.050049 max_load_reduction=56934 max_load_reduction_pct=0.868744 moved_to_echo=33098 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 13, 0, 4065, 24, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 3823, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 4096, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4072, 73, 0, 0, 0, 0, 0, 0, 4084, 149, 0, 0, 0, 0, 1, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 4036, 0, 6, 0, 0, 273, 154, 22, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8198, 8056, 8151, 8241, 8176, 8188, 8325] avg=8192.0000 max=8325 max_rank=7 min=8056 min_rank=2 imbalance_ratio=1.016235 max_load_reduction=57211 max_load_reduction_pct=0.872971 moved_to_echo=32930 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4081, 0, 0, 0, 8, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 12, 0, 4090, 16, 0, 0, 9, 4058, 23, 0, 0, 0, 0, 0, 0, 0, 0, 1995, 0, 0, 0, 3815, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 102, 0, 8, 388, 8, 4087, 4014, 0, 0, 0, 0, 0, 0, 0, 1, 7, 9, 13, 3678, 4, 9, 4061, 73, 0, 0, 0, 0, 0, 0, 3978, 146, 5, 8, 9, 3, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4082, 4081, 0, 7, 32, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4069, 0, 1985, 0, 0, 3, 0, 272, 154, 22, 3, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8211, 9898, 8615, 7855, 8083, 8221, 6515] avg=8192.0000 max=9898 max_rank=2 min=6515 min_rank=7 imbalance_ratio=1.208252 max_load_reduction=55638 max_load_reduction_pct=0.848969 moved_to_echo=30674 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4065, 24, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 3823, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4072, 73, 0, 0, 0, 0, 0, 0, 4090, 104, 0, 0, 0, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 4066, 0, 0, 0, 0, 273, 154, 22, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8189, 7982, 8126, 8241, 8136, 8188, 8473] avg=8192.0000 max=8473 max_rank=7 min=7982 min_rank=2 imbalance_ratio=1.034302 max_load_reduction=57063 max_load_reduction_pct=0.870712 moved_to_echo=33038 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4065, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3823, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4072, 72, 0, 0, 0, 0, 0, 0, 4096, 175, 0, 0, 0, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 4041, 0, 0, 0, 0, 273, 154, 22, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8185, 7918, 8143, 8240, 8213, 8188, 8448] avg=8192.0000 max=8448 max_rank=7 min=7918 min_rank=2 imbalance_ratio=1.031250 max_load_reduction=57088 max_load_reduction_pct=0.871094 moved_to_echo=33089 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4065, 24, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3823, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4072, 72, 0, 0, 0, 0, 0, 0, 4096, 498, 0, 0, 0, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3588, 0, 4096, 3, 0, 0, 0, 273, 154, 22, 4, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8185, 7925, 8121, 8240, 8536, 8188, 8140] avg=8192.0000 max=8536 max_rank=5 min=7925 min_rank=2 imbalance_ratio=1.041992 max_load_reduction=57000 max_load_reduction_pct=0.869751 moved_to_echo=33104 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4071, 25, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3820, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 4096, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4071, 72, 0, 0, 0, 0, 0, 0, 4095, 225, 0, 0, 0, 0, 1, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 0, 4090, 1, 1, 0, 0, 275, 159, 24, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8193, 7917, 8124, 8239, 8258, 8186, 8426] avg=8192.0000 max=8426 max_rank=7 min=7917 min_rank=2 imbalance_ratio=1.028564 max_load_reduction=57110 max_load_reduction_pct=0.871429 moved_to_echo=33109 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 1408, 128, 128, 128, 128, 128, 31616, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 31488, 0, 30592, 128, 128, 128, 0, 2304, 1280, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 2304, 0, 0, 0, 30592, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 128, 32384, 128, 128, 32640, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 512, 128, 32768, 32256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 128, 128, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[2588, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 41, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 0, 1384, 0, 4096, 4096, 0, 4096, 4087, 0, 3939, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[778, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 145, 0, 0, 0, 0, 4096, 0, 35, 0, 0, 4096, 0, 0, 0, 2326, 0, 4096, 4096, 0, 4096, 4084, 0, 3214, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1486, 279, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 113, 0, 0, 0, 0, 4092, 0, 32, 0, 0, 4096, 0, 0, 0, 2393, 5, 4096, 4091, 0, 4096, 4080, 0, 3898, 4096, 0, 4090, 0, 6, 0, 0, 4096, 7, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[469, 963, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 371, 0, 0, 0, 6, 4091, 2, 71, 0, 0, 4096, 0, 0, 0, 2921, 7, 4096, 4095, 0, 4096, 4073, 0, 3408, 4096, 0, 4089, 0, 1, 0, 0, 4096, 8, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[1846, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 240, 0, 0, 0, 0, 4096, 0, 41, 0, 0, 4096, 0, 0, 0, 1797, 0, 4096, 4096, 0, 4096, 4080, 0, 3467, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[2392, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 178, 0, 0, 0, 1, 4096, 0, 5, 0, 0, 4096, 0, 0, 0, 1897, 0, 4096, 4096, 0, 4096, 4095, 0, 3019, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[840, 199, 1, 0, 0, 4, 14, 0, 9, 8, 9, 7, 6, 0, 4071, 45, 0, 3, 0, 3, 0, 4090, 374, 0, 0, 0, 4088, 11, 0, 0, 3366, 0, 4089, 4072, 0, 4087, 4089, 10, 3816, 4083, 56, 4087, 0, 0, 0, 13, 3667, 1, 7, 4089, 0, 0, 7, 0, 4080, 7, 8, 8, 10, 0, 2, 4005, 4086, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[2171, 179, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 89, 0, 0, 0, 0, 4095, 0, 18, 0, 0, 4096, 0, 0, 0, 2478, 1, 4096, 4095, 0, 4096, 4050, 0, 3302, 4096, 0, 4095, 0, 1, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2588, 247, 0, 0, 0, 0, 0, 0, 4096, 1886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2210, 1880, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 4096, 0, 2, 2216, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1384, 0, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 4087, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8817, 8186, 8018, 7255, 8632, 8236, 8192, 8200] avg=8192.0000 max=8817 max_rank=0 min=7255 min_rank=3 imbalance_ratio=1.076294 max_load_reduction=56719 max_load_reduction_pct=0.865463 moved_to_echo=33260 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1486, 279, 0, 0, 0, 0, 0, 4, 4096, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2209, 1881, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 4092, 0, 32, 2215, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2393, 5, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 4080, 0, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 4096, 7, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7752, 8186, 8113, 8267, 8584, 8243, 8192, 8199] avg=8192.0000 max=8584 max_rank=4 min=7752 min_rank=0 imbalance_ratio=1.047852 max_load_reduction=56952 max_load_reduction_pct=0.869019 moved_to_echo=33218 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[469, 963, 0, 0, 0, 0, 0, 1, 4096, 1886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2210, 1880, 0, 0, 0, 0, 0, 0, 371, 0, 0, 0, 6, 4091, 2, 71, 2216, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2921, 7, 1774, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 4073, 0, 3408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1, 0, 0, 4096, 8, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7415, 8186, 8420, 8798, 8087, 8238, 8192, 8200] avg=8192.0000 max=8798 max_rank=3 min=7415 min_rank=0 imbalance_ratio=1.073975 max_load_reduction=56738 max_load_reduction_pct=0.865753 moved_to_echo=32717 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1846, 817, 0, 0, 0, 0, 0, 0, 4096, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2209, 1881, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 4096, 0, 41, 2215, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1797, 0, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 4080, 0, 3467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8646, 8186, 8255, 7668, 8153, 8236, 8192, 8200] avg=8192.0000 max=8646 max_rank=0 min=7668 min_rank=3 imbalance_ratio=1.055420 max_load_reduction=56890 max_load_reduction_pct=0.868073 moved_to_echo=32781 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[778, 1706, 0, 0, 0, 0, 0, 0, 4096, 1886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2210, 1880, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 4096, 0, 35, 2216, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2326, 0, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 4084, 0, 3214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8466, 8186, 8155, 8197, 7904, 8236, 8192, 8200] avg=8192.0000 max=8466 max_rank=0 min=7904 min_rank=4 imbalance_ratio=1.033447 max_load_reduction=57070 max_load_reduction_pct=0.870819 moved_to_echo=32532 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2392, 702, 0, 0, 0, 0, 0, 0, 4096, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2209, 1881, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 1, 4096, 0, 5, 2215, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1897, 0, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 4095, 0, 3019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9077, 8186, 8158, 7768, 7720, 8235, 8192, 8200] avg=8192.0000 max=9077 max_rank=0 min=7720 min_rank=4 imbalance_ratio=1.108032 max_load_reduction=56459 max_load_reduction_pct=0.861496 moved_to_echo=32347 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[840, 199, 1, 0, 0, 4, 14, 0, 4089, 1882, 0, 0, 0, 0, 0, 0, 9, 8, 9, 7, 6, 0, 4071, 45, 2205, 1874, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 4090, 374, 0, 2209, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 11, 0, 0, 3366, 0, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 603, 0, 0, 4089, 10, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4087, 0, 0, 0, 13, 3667, 1, 44, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 0, 7, 0, 4080, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 10, 0, 2, 4005, 4086, 9, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7029, 8234, 8332, 9230, 8518, 7868, 8190, 8135] avg=8192.0000 max=9230 max_rank=3 min=7029 min_rank=0 imbalance_ratio=1.126709 max_load_reduction=56306 max_load_reduction_pct=0.859161 moved_to_echo=32711 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2171, 179, 0, 0, 0, 0, 0, 1, 4096, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2209, 1881, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 4095, 0, 18, 2215, 1668, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2478, 1, 1778, 0, 0, 0, 0, 0, 0, 0, 0, 586, 0, 0, 4050, 0, 3302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 4096, 1, 51, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8334, 8186, 8085, 8353, 7938, 8244, 8194, 8202] avg=8192.0000 max=8353 max_rank=3 min=7938 min_rank=4 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32578 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 4864, 0, 0, 32640, 128, 28160, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12672, 5120, 128, 0, 0, 128, 128, 128, 32768, 15104, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 0, 128, 32384, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 18688, 128, 14208, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1280, 128, 0, 128, 128, 32768, 384, 256, 17792, 13312, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 17792, 15104, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4089, 7, 0, 3, 301, 0, 0, 4089, 7, 4085, 0, 13, 0, 0, 3, 4086, 0, 88, 4069, 5, 4083, 3, 15, 6, 6, 30, 17, 0, 3986, 0, 4082, 0, 7, 1, 4089, 0, 4082, 0, 0, 10, 0, 9, 0, 0, 117, 14, 8, 2, 10, 4011, 0, 0, 11, 4087, 3752, 0, 4059, 5, 4083, 4085, 0, 13, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 20, 0, 4094, 0, 4093, 11, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4095, 0, 0, 0, 4096, 5, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 26, 0, 4096, 0, 4096, 29, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 50, 0, 4095, 0, 4093, 10, 0, 0, 0, 0, 4085, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 5, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 4096, 10, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 2, 0, 4095, 0, 4096, 6, 0, 0, 1, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4032, 13, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 52, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8141, 8184, 8186, 8200, 8210, 8175, 8244, 8196] avg=8192.0000 max=8244 max_rank=6 min=8141 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32825 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 20, 0, 4032, 13, 0, 0, 0, 0, 0, 0, 4088, 0, 4093, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4095, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4072, 0, 0, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 52, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8192, 8180, 8199, 8190, 8175, 8244, 8196] avg=8192.0000 max=8244 max_rank=6 min=8160 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32805 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 7, 0, 3, 301, 0, 0, 4023, 13, 0, 0, 0, 0, 0, 0, 4083, 7, 4085, 0, 13, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 88, 4063, 5, 0, 3, 15, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 30, 17, 0, 3986, 0, 4082, 6, 1, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4089, 0, 4082, 0, 0, 12, 5, 0, 0, 0, 0, 0, 0, 10, 0, 9, 0, 0, 117, 14, 8, 4083, 4068, 0, 0, 0, 0, 0, 0, 2, 10, 4011, 0, 0, 11, 52, 3752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 5, 4083, 4, 0, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8436, 8191, 8260, 8134, 8196, 8309, 7838, 8172] avg=8192.0000 max=8436 max_rank=0 min=7838 min_rank=6 imbalance_ratio=1.029785 max_load_reduction=57100 max_load_reduction_pct=0.871277 moved_to_echo=32515 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 26, 0, 4032, 13, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4044, 0, 0, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 52, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8213, 8183, 8200, 8158, 8175, 8244, 8196] avg=8192.0000 max=8244 max_rank=6 min=8158 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 7, 0, 4032, 13, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 0, 0, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 52, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8148, 8194, 8184, 8200, 8195, 8175, 8244, 8196] avg=8192.0000 max=8244 max_rank=6 min=8148 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32810 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 50, 0, 4032, 13, 0, 0, 0, 0, 0, 0, 4087, 0, 4093, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4046, 0, 0, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 52, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8190, 8175, 8200, 8165, 8175, 8244, 8196] avg=8192.0000 max=8244 max_rank=6 min=8165 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32780 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4032, 15, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 52, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8143, 8186, 8185, 8200, 8209, 8177, 8244, 8192] avg=8192.0000 max=8244 max_rank=6 min=8143 min_rank=0 imbalance_ratio=1.006348 max_load_reduction=57292 max_load_reduction_pct=0.874207 moved_to_echo=32822 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 2, 0, 4036, 17, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 6, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4090, 0, 0, 18, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 42, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8186, 8183, 8203, 8213, 8175, 8234, 8192] avg=8192.0000 max=8234 max_rank=6 min=8150 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=57302 max_load_reduction_pct=0.874359 moved_to_echo=32814 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 512, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 0, 128, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 32640, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 128, 384, 128, 0, 32256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True + [2026-08-30 04:59:31] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 713.6 | throughput per GPU (TFLOP/s/GPU): 467.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.253225E+00 | loss scale: 1.0 | grad norm: 5.578 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[3254, 31, 221, 112, 606, 0, 2142, 118, 962, 3390, 364, 822, 1229, 1857, 0, 211, 453, 1158, 279, 1051, 1, 1076, 1488, 1102, 7, 17, 3066, 1129, 2734, 328, 2631, 359, 4, 1132, 2754, 434, 390, 278, 2526, 658, 1037, 0, 2190, 1798, 0, 1585, 1021, 970, 3231, 31, 1948, 2, 43, 402, 2, 625, 0, 317, 1111, 1389, 2458, 984, 1076, 2942] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2772, 38, 213, 76, 1067, 0, 2578, 35, 744, 3183, 174, 595, 1187, 1344, 0, 320, 203, 1087, 314, 1629, 0, 771, 1702, 1244, 12, 19, 2646, 1083, 3085, 329, 2840, 749, 11, 1536, 2345, 268, 538, 174, 2513, 1163, 1046, 0, 2513, 1628, 0, 1025, 1349, 839, 2879, 38, 1854, 1, 3, 594, 0, 583, 0, 171, 1065, 1472, 3147, 896, 1040, 2806] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2902, 38, 373, 121, 519, 0, 2454, 140, 920, 2830, 368, 773, 1085, 1009, 0, 211, 253, 1457, 318, 1820, 2, 647, 1722, 1172, 59, 25, 2923, 1201, 3091, 297, 2954, 563, 9, 1162, 2803, 581, 432, 259, 2245, 1671, 1285, 0, 2579, 1968, 0, 1059, 1215, 804, 2807, 44, 1899, 0, 66, 477, 1, 407, 1, 168, 806, 901, 2961, 780, 1152, 2747] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2470, 13, 199, 161, 714, 0, 2255, 36, 1303, 3298, 320, 528, 1212, 2018, 0, 231, 448, 1226, 252, 1234, 0, 604, 1708, 1563, 10, 103, 2781, 1446, 2966, 498, 2882, 549, 3, 1357, 2816, 264, 444, 201, 1798, 1447, 890, 0, 2575, 2279, 0, 1198, 918, 921, 2815, 34, 1723, 0, 39, 651, 0, 368, 0, 120, 961, 1375, 2703, 1229, 713, 2666] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3077, 63, 402, 54, 677, 1, 2776, 47, 324, 3424, 298, 489, 1461, 1262, 0, 186, 330, 1223, 224, 1737, 0, 730, 1816, 1458, 0, 34, 3014, 971, 2993, 194, 2425, 632, 14, 1602, 2698, 424, 309, 139, 2244, 1509, 1045, 2, 2197, 1746, 0, 1081, 1140, 684, 3290, 21, 1854, 0, 26, 369, 0, 872, 0, 111, 779, 1079, 3052, 1371, 961, 2595] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2984, 20, 325, 171, 1035, 1, 2147, 48, 919, 2945, 297, 386, 1030, 1608, 0, 212, 441, 2057, 308, 1284, 0, 576, 1322, 1552, 6, 12, 2706, 1216, 2849, 307, 2270, 439, 11, 1298, 2756, 247, 476, 382, 2462, 1236, 1642, 1, 2499, 2106, 1, 964, 1548, 1461, 2898, 45, 1400, 0, 12, 355, 1, 608, 0, 145, 1279, 1320, 2104, 1144, 1006, 2656] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2085, 19, 662, 126, 929, 0, 1800, 60, 780, 2907, 263, 646, 1551, 1385, 0, 316, 366, 1501, 298, 2029, 0, 773, 1325, 1943, 63, 32, 2603, 1104, 3211, 250, 2591, 882, 6, 1834, 2617, 145, 602, 321, 2521, 1074, 1717, 2, 2316, 1982, 2, 1033, 1309, 1270, 2859, 8, 1547, 1, 23, 357, 4, 582, 2, 170, 494, 1313, 2796, 1265, 816, 2048] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[3003, 76, 361, 113, 1015, 0, 2079, 65, 771, 3178, 389, 757, 1691, 952, 0, 238, 565, 1329, 335, 1753, 1, 576, 1693, 1480, 28, 7, 2695, 1065, 2732, 192, 2250, 620, 11, 1519, 2410, 209, 481, 218, 2321, 1220, 865, 2, 2261, 1790, 8, 1149, 1244, 941, 2870, 56, 1634, 9, 22, 447, 10, 725, 0, 123, 1226, 1600, 2961, 1365, 683, 3147] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2772, 38, 213, 76, 1067, 0, 2578, 35, 1902, 32, 0, 0, 0, 0, 0, 0, 744, 3183, 174, 595, 1187, 1344, 0, 320, 263, 0, 0, 0, 0, 0, 0, 0, 203, 1087, 314, 1629, 0, 771, 1702, 1244, 645, 178, 0, 0, 0, 0, 0, 0, 12, 19, 2646, 1083, 707, 329, 2840, 749, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1536, 1904, 268, 538, 174, 2513, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 1836, 1628, 0, 1025, 1349, 839, 0, 0, 0, 0, 0, 0, 0, 0, 2879, 38, 1854, 1, 3, 594, 0, 583, 2378, 117, 0, 0, 0, 0, 0, 0, 0, 171, 1065, 1472, 1128, 896, 1040, 2806, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8713, 7810, 7773, 8385, 8107, 7723, 8447, 8578] avg=8192.0000 max=8713 max_rank=0 min=7723 min_rank=5 imbalance_ratio=1.063599 max_load_reduction=56823 max_load_reduction_pct=0.867050 moved_to_echo=32855 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3254, 31, 221, 112, 606, 0, 2142, 118, 1485, 28, 0, 0, 0, 0, 0, 0, 962, 3390, 364, 822, 1229, 1857, 0, 211, 309, 0, 0, 0, 0, 0, 0, 0, 453, 1158, 279, 1051, 1, 1076, 1488, 1102, 562, 209, 0, 0, 0, 0, 0, 0, 7, 17, 3066, 1129, 627, 328, 2631, 359, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1132, 2236, 434, 390, 278, 2526, 658, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 0, 1600, 1798, 0, 1585, 1021, 970, 0, 0, 0, 0, 0, 0, 0, 0, 3231, 31, 1948, 2, 43, 402, 2, 625, 2107, 91, 0, 0, 0, 0, 0, 0, 0, 317, 1111, 1389, 882, 984, 1076, 2942, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7997, 9144, 7379, 8164, 7658, 8011, 8482, 8701] avg=8192.0000 max=9144 max_rank=1 min=7379 min_rank=2 imbalance_ratio=1.116211 max_load_reduction=56392 max_load_reduction_pct=0.860474 moved_to_echo=32852 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2470, 13, 199, 161, 714, 0, 2255, 36, 1633, 33, 0, 0, 0, 0, 0, 0, 1303, 3298, 320, 528, 1212, 2018, 0, 231, 316, 0, 0, 0, 0, 0, 0, 0, 448, 1226, 252, 1234, 0, 604, 1708, 1563, 661, 214, 0, 0, 0, 0, 0, 0, 10, 103, 2781, 1446, 680, 498, 2882, 549, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1357, 2286, 264, 444, 201, 1798, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 890, 0, 1881, 2279, 0, 1198, 918, 921, 0, 0, 0, 0, 0, 0, 0, 0, 2815, 34, 1723, 0, 39, 651, 0, 368, 2286, 100, 0, 0, 0, 0, 0, 0, 0, 120, 961, 1375, 970, 1229, 713, 2666, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7514, 9226, 7910, 8949, 7800, 8087, 8016, 8034] avg=8192.0000 max=9226 max_rank=1 min=7514 min_rank=0 imbalance_ratio=1.126221 max_load_reduction=56310 max_load_reduction_pct=0.859222 moved_to_echo=31937 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2984, 20, 325, 171, 1035, 1, 2147, 48, 1271, 32, 0, 0, 0, 0, 0, 0, 919, 2945, 297, 386, 1030, 1608, 0, 212, 309, 0, 0, 0, 0, 0, 0, 0, 441, 2057, 308, 1284, 0, 576, 1322, 1552, 641, 209, 0, 0, 0, 0, 0, 0, 6, 12, 2706, 1216, 653, 307, 2270, 439, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1298, 2238, 247, 476, 382, 2462, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 1642, 1, 1826, 2106, 1, 964, 1548, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 2898, 45, 1400, 0, 12, 355, 1, 608, 2196, 78, 0, 0, 0, 0, 0, 0, 0, 145, 1279, 1320, 755, 1144, 1006, 2656, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8034, 7706, 8390, 7609, 8350, 9549, 7593, 8305] avg=8192.0000 max=9549 max_rank=5 min=7593 min_rank=6 imbalance_ratio=1.165649 max_load_reduction=55987 max_load_reduction_pct=0.854294 moved_to_echo=33797 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2902, 38, 373, 121, 519, 0, 2454, 140, 1789, 33, 0, 0, 0, 0, 0, 0, 920, 2830, 368, 773, 1085, 1009, 0, 211, 315, 0, 0, 0, 0, 0, 0, 0, 253, 1457, 318, 1820, 2, 647, 1722, 1172, 662, 213, 0, 0, 0, 0, 0, 0, 59, 25, 2923, 1201, 709, 297, 2954, 563, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1162, 2275, 581, 432, 259, 2245, 1671, 0, 0, 0, 0, 0, 0, 0, 0, 1285, 0, 1884, 1968, 0, 1059, 1215, 804, 0, 0, 0, 0, 0, 0, 0, 0, 2807, 44, 1899, 0, 66, 477, 1, 407, 2382, 110, 0, 0, 0, 0, 0, 0, 1, 168, 806, 901, 1062, 780, 1152, 2747, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8369, 7511, 8266, 8731, 8634, 8215, 8193, 7617] avg=8192.0000 max=8731 max_rank=3 min=7511 min_rank=1 imbalance_ratio=1.065796 max_load_reduction=56805 max_load_reduction_pct=0.866776 moved_to_echo=32659 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2085, 19, 662, 126, 929, 0, 1800, 60, 1689, 30, 0, 0, 0, 0, 0, 0, 780, 2907, 263, 646, 1551, 1385, 0, 316, 294, 0, 0, 0, 0, 0, 0, 0, 366, 1501, 298, 2029, 0, 773, 1325, 1943, 594, 199, 0, 0, 0, 0, 0, 0, 63, 32, 2603, 1104, 736, 250, 2591, 882, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1834, 2124, 145, 602, 321, 2521, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 1717, 2, 1692, 1982, 2, 1033, 1309, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 2859, 8, 1547, 1, 23, 357, 4, 582, 2475, 103, 0, 0, 0, 0, 0, 0, 2, 170, 494, 1313, 1004, 1265, 816, 2048, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7400, 8142, 9028, 8261, 8627, 9007, 7959, 7112] avg=8192.0000 max=9028 max_rank=2 min=7112 min_rank=7 imbalance_ratio=1.102051 max_load_reduction=56508 max_load_reduction_pct=0.862244 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3077, 63, 402, 54, 677, 1, 2776, 47, 1844, 28, 0, 0, 0, 0, 0, 0, 324, 3424, 298, 489, 1461, 1262, 0, 186, 303, 0, 0, 0, 0, 0, 0, 0, 330, 1223, 224, 1737, 0, 730, 1816, 1458, 564, 205, 0, 0, 0, 0, 0, 0, 0, 34, 3014, 971, 686, 194, 2425, 632, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1602, 2190, 424, 309, 139, 2244, 1509, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 2, 1605, 1746, 0, 1081, 1140, 684, 0, 0, 0, 0, 0, 0, 0, 0, 3290, 21, 1854, 0, 26, 369, 0, 872, 2307, 113, 0, 0, 0, 0, 0, 0, 0, 111, 779, 1079, 1095, 1371, 961, 2595, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8969, 7747, 8287, 7956, 8431, 7303, 8852, 7991] avg=8192.0000 max=8969 max_rank=0 min=7303 min_rank=5 imbalance_ratio=1.094849 max_load_reduction=56567 max_load_reduction_pct=0.863144 moved_to_echo=32577 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3003, 76, 361, 113, 1015, 0, 2079, 65, 1794, 34, 0, 0, 0, 0, 0, 0, 771, 3178, 389, 757, 1691, 952, 0, 238, 274, 0, 0, 0, 0, 0, 0, 0, 565, 1329, 335, 1753, 1, 576, 1693, 1480, 585, 186, 0, 0, 0, 0, 0, 0, 28, 7, 2695, 1065, 624, 192, 2250, 620, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1519, 1950, 209, 481, 218, 2321, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 865, 2, 1642, 1790, 8, 1149, 1244, 941, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 56, 1634, 9, 22, 447, 10, 725, 2108, 113, 0, 0, 0, 0, 0, 0, 0, 123, 1226, 1600, 1054, 1365, 683, 3147, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8540, 8250, 8503, 7481, 7929, 7641, 7994, 9198] avg=8192.0000 max=9198 max_rank=7 min=7481 min_rank=3 imbalance_ratio=1.122803 max_load_reduction=56338 max_load_reduction_pct=0.859650 moved_to_echo=32762 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[9600, 128, 14080, 15360, 128, 9216, 9856, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3072, 11136, 2432, 12544, 128, 5760, 12800, 11520, 4992, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 1408, 7808, 10496, 8064, 9088, 7552, 21632, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6784, 25216, 2560, 5120, 10496, 11520, 0, 2048, 2432, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 256, 22528, 9216, 5504, 2432, 20864, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 11520, 17280, 2688, 3712, 2048, 18688, 9984, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[23680, 384, 13952, 128, 256, 3712, 128, 4864, 18304, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[22656, 384, 2816, 1024, 6656, 128, 18304, 640, 13440, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4077, 0, 4065, 4094, 1, 721, 4062, 1, 1, 1787, 4094, 0, 0, 1, 0, 0, 7, 116, 3284, 4083, 0, 4081, 0, 15, 0, 532, 1, 0, 2, 0, 265, 0, 1, 2908, 4095, 0, 1, 0, 0, 0, 0, 4096, 2, 0, 0, 100, 4094, 0, 0, 0, 4052, 4096, 403, 0, 0, 0, 0, 2303] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4029, 0, 4088, 4095, 0, 681, 4055, 0, 0, 874, 4095, 0, 0, 0, 0, 0, 1, 78, 3508, 4093, 0, 4087, 0, 79, 0, 743, 0, 0, 0, 0, 262, 1, 0, 3029, 4096, 5, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 26, 4095, 0, 4, 0, 4005, 4096, 312, 0, 0, 0, 0, 2908] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4096, 0, 4090, 0, 4094, 4096, 0, 1634, 4065, 0, 0, 2389, 4096, 0, 0, 0, 0, 0, 19, 281, 3840, 3940, 0, 3667, 0, 179, 0, 255, 1, 0, 4, 0, 50, 0, 0, 1411, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 4096, 0, 0, 0, 3950, 4096, 421, 0, 0, 0, 0, 2568] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 4085, 1, 4090, 4096, 1, 1584, 3997, 0, 3, 2255, 4096, 0, 0, 1, 0, 0, 13, 551, 3645, 4007, 8, 3833, 0, 64, 0, 296, 6, 0, 0, 0, 44, 0, 2, 1719, 4094, 4, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 3, 4095, 0, 18, 0, 3994, 4096, 417, 0, 0, 0, 0, 2223] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4091, 0, 4088, 2, 4093, 4088, 0, 1938, 4090, 3, 1, 1641, 4092, 3, 1, 0, 2, 0, 40, 143, 4027, 4013, 2, 3989, 0, 249, 0, 127, 11, 0, 5, 3, 139, 0, 0, 1176, 4089, 0, 1, 3, 0, 1, 1, 4093, 0, 0, 3, 2, 4093, 3, 0, 3, 3962, 4093, 733, 0, 3, 1, 3, 2389] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 4057, 0, 4085, 4091, 0, 2398, 3924, 0, 0, 900, 4095, 0, 2, 1, 0, 0, 45, 257, 3762, 3765, 4, 4009, 0, 122, 0, 223, 14, 1, 0, 0, 230, 1, 0, 1744, 4096, 0, 0, 0, 1, 0, 0, 4096, 2, 0, 0, 6, 4091, 0, 4, 0, 3992, 4094, 220, 0, 0, 1, 0, 3109] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 4032, 0, 4060, 4096, 0, 314, 4022, 0, 0, 2687, 4096, 0, 0, 0, 0, 0, 17, 494, 3947, 3986, 0, 3770, 0, 455, 0, 420, 1, 0, 0, 0, 84, 0, 0, 1881, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 4096, 0, 0, 0, 3998, 4096, 254, 0, 0, 0, 0, 2424] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 6, 4083, 0, 4072, 4096, 0, 693, 4069, 0, 1, 1341, 4077, 0, 0, 0, 0, 0, 10, 247, 3768, 4072, 4, 4050, 0, 315, 0, 389, 3, 0, 1, 0, 141, 0, 0, 2928, 4095, 32, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 235, 4096, 0, 5, 0, 4052, 4091, 273, 0, 0, 0, 0, 2098] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4077, 0, 23, 4, 0, 0, 0, 0, 0, 0, 1171, 0, 1, 721, 4062, 1, 1, 1787, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 7, 116, 2787, 1014, 4, 0, 0, 0, 0, 0, 3284, 0, 0, 4054, 0, 15, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 265, 0, 1, 2908, 4094, 1880, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 100, 4094, 0, 0, 0, 2203, 1880, 0, 0, 0, 0, 0, 0, 4052, 1309, 403, 0, 0, 0, 0, 2303, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 7744, 8023, 7885, 9151, 8188, 8279, 8067] avg=8192.0000 max=9151 max_rank=4 min=7744 min_rank=1 imbalance_ratio=1.117065 max_load_reduction=56385 max_load_reduction_pct=0.860367 moved_to_echo=33685 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 4029, 0, 23, 4, 0, 0, 0, 0, 0, 0, 1179, 0, 0, 681, 4055, 0, 0, 874, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 78, 2787, 1019, 4, 0, 0, 0, 0, 0, 3508, 0, 0, 4060, 0, 79, 0, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 1, 0, 3029, 4095, 1884, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 26, 4095, 0, 4, 0, 2209, 1890, 0, 0, 0, 0, 0, 0, 4005, 1309, 312, 0, 0, 0, 0, 2908, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 6789, 7984, 8390, 9271, 8192, 8225, 8534] avg=8192.0000 max=9271 max_rank=4 min=6789 min_rank=1 imbalance_ratio=1.131714 max_load_reduction=56265 max_load_reduction_pct=0.858536 moved_to_echo=34222 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 4085, 1, 22, 4, 0, 0, 0, 0, 0, 0, 1179, 0, 1, 1584, 3997, 0, 3, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 13, 551, 2787, 1020, 3, 0, 0, 0, 0, 0, 3645, 0, 8, 3808, 0, 64, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 44, 0, 2, 1719, 4096, 1845, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 18, 0, 2162, 1891, 0, 0, 0, 0, 0, 0, 3994, 1309, 417, 0, 0, 0, 0, 2223, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 9019, 8471, 7821, 7712, 8191, 8169, 7943] avg=8192.0000 max=9019 max_rank=1 min=7712 min_rank=4 imbalance_ratio=1.100952 max_load_reduction=56517 max_load_reduction_pct=0.862381 moved_to_echo=32015 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 1, 4032, 0, 21, 4, 0, 0, 0, 0, 0, 0, 1171, 0, 0, 314, 4022, 0, 0, 2687, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 17, 494, 2787, 1012, 3, 0, 0, 0, 0, 0, 3947, 0, 0, 3746, 0, 455, 0, 420, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 84, 0, 0, 1881, 4096, 1835, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 2151, 1877, 0, 0, 0, 0, 0, 0, 3998, 1309, 254, 0, 0, 0, 0, 2424, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8194, 8409, 8568, 7897, 8190, 8139, 7985] avg=8192.0000 max=8568 max_rank=3 min=7897 min_rank=4 imbalance_ratio=1.045898 max_load_reduction=56968 max_load_reduction_pct=0.869263 moved_to_echo=32211 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 4057, 0, 23, 4, 0, 0, 0, 0, 0, 0, 1177, 0, 0, 2398, 3924, 0, 0, 900, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 1, 0, 0, 45, 257, 2785, 1019, 3, 0, 0, 0, 0, 0, 3762, 0, 4, 3983, 0, 122, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 230, 1, 0, 1744, 4091, 1733, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 4091, 0, 4, 0, 2032, 1889, 0, 0, 0, 0, 0, 0, 3992, 1309, 220, 0, 0, 1, 0, 3109, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8399, 8207, 8094, 7814, 8189, 8024, 8631] avg=8192.0000 max=8631 max_rank=7 min=7814 min_rank=4 imbalance_ratio=1.053589 max_load_reduction=56905 max_load_reduction_pct=0.868301 moved_to_echo=32658 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4096, 0, 4090, 0, 21, 4, 0, 0, 0, 0, 0, 0, 1180, 0, 0, 1634, 4065, 0, 0, 2389, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 19, 281, 2787, 1021, 3, 0, 0, 0, 0, 0, 3840, 0, 0, 3643, 0, 179, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 50, 0, 0, 1411, 4096, 1815, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 2125, 1893, 0, 0, 0, 0, 0, 0, 3950, 1309, 421, 0, 0, 0, 0, 2568, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 9268, 8207, 7917, 7377, 8188, 8118, 8248] avg=8192.0000 max=9268 max_rank=1 min=7377 min_rank=4 imbalance_ratio=1.131348 max_load_reduction=56268 max_load_reduction_pct=0.858582 moved_to_echo=31931 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4091, 0, 4088, 2, 22, 4, 0, 0, 0, 0, 0, 0, 1179, 0, 0, 1938, 4090, 3, 1, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 1, 0, 2, 0, 40, 143, 2785, 1021, 3, 0, 0, 0, 0, 0, 4027, 0, 2, 3964, 0, 249, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 5, 3, 139, 0, 0, 1176, 4088, 1848, 0, 0, 0, 0, 0, 0, 4089, 0, 1, 3, 0, 1, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 4093, 3, 0, 3, 2165, 1893, 0, 0, 0, 0, 0, 0, 3962, 1308, 733, 0, 3, 1, 3, 2389, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8852, 8090, 8369, 7270, 8184, 8162, 8399] avg=8192.0000 max=8852 max_rank=1 min=7270 min_rank=4 imbalance_ratio=1.080566 max_load_reduction=56684 max_load_reduction_pct=0.864929 moved_to_echo=32015 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 6, 4083, 0, 26, 10, 0, 0, 0, 0, 0, 0, 1167, 0, 0, 693, 4069, 0, 1, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 10, 247, 2785, 1018, 8, 0, 0, 0, 0, 0, 3768, 0, 4, 4016, 0, 315, 0, 389, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 141, 0, 0, 2928, 4096, 1875, 0, 0, 0, 0, 0, 0, 4095, 32, 0, 0, 1, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 4096, 0, 5, 0, 2197, 1887, 0, 0, 0, 0, 0, 0, 4052, 1306, 273, 0, 0, 0, 0, 2098, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 7271, 8145, 8492, 9044, 8214, 8420, 7729] avg=8192.0000 max=9044 max_rank=4 min=7271 min_rank=1 imbalance_ratio=1.104004 max_load_reduction=56492 max_load_reduction_pct=0.862000 moved_to_echo=33407 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 256, 2176, 22400, 8192, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32128, 10496, 3072, 0, 128, 128, 128, 20096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9472, 0, 128, 9984, 32384, 128, 128, 13952, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 1280, 128, 128, 16896, 32768, 14720, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 32640, 128, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 512, 32768, 128, 128, 128, 17280, 15104, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[29824, 0, 128, 31360, 0, 1536, 0, 3072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 0, 4086, 0, 0, 0, 4090, 11, 4094, 0, 0, 4096, 0, 0, 4095, 0, 0, 4071, 6, 3, 26, 4083, 4096, 0, 4094, 0, 0, 1, 3600, 4096, 0, 2, 1700, 3706, 4096, 0, 0, 0, 0, 3193, 0, 33, 0, 31, 0, 4095, 0, 0, 0, 1, 0, 4095, 0, 0, 31, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 4081, 0, 0, 0, 4089, 8, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4074, 2, 37, 30, 4096, 4094, 0, 4092, 0, 0, 0, 3374, 4096, 0, 4, 2677, 3763, 4096, 0, 0, 0, 0, 2216, 0, 88, 0, 64, 0, 4095, 0, 0, 0, 2, 0, 4096, 0, 0, 66, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4095, 13, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 1, 9, 4081, 4090, 0, 4095, 0, 0, 0, 3442, 4096, 3, 0, 1481, 3634, 4096, 0, 0, 0, 0, 3548, 0, 174, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 25, 0, 3, 4096, 0, 0, 0, 4082, 6, 4096, 0, 0, 4096, 2, 0, 4096, 0, 0, 4063, 11, 0, 25, 4093, 4083, 0, 4090, 0, 0, 0, 2612, 4096, 0, 6, 2213, 3456, 4094, 0, 0, 2, 0, 3703, 0, 294, 0, 5, 0, 4094, 0, 0, 0, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4095, 8, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4080, 0, 0, 20, 4092, 4093, 0, 4096, 0, 0, 0, 2585, 4096, 1, 0, 2142, 3575, 4096, 0, 0, 0, 0, 3841, 0, 114, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 54, 0, 2, 4096, 0, 0, 0, 4084, 58, 4096, 0, 0, 4096, 0, 0, 4094, 0, 0, 4070, 1, 2, 30, 4088, 4094, 0, 4091, 0, 0, 0, 3245, 4096, 0, 4, 1798, 2448, 4093, 0, 0, 0, 0, 3864, 0, 807, 0, 5, 0, 4095, 0, 0, 0, 3, 0, 4095, 0, 0, 25, 0, 1, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3, 0, 0, 21, 0, 0, 4082, 1, 0, 1, 4093, 21, 4090, 0, 3, 4089, 0, 0, 4093, 3, 3, 4088, 0, 54, 8, 4071, 4090, 0, 4092, 3, 2, 0, 3627, 4091, 2, 0, 2375, 2515, 4092, 3, 3, 0, 4, 3061, 3, 555, 0, 0, 2, 4092, 3, 3, 1, 0, 3, 4092, 3, 3, 84, 0, 0, 3, 2, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4081, 2, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4050, 17, 2, 27, 4096, 4096, 0, 4085, 0, 0, 0, 3364, 4096, 0, 5, 1922, 3561, 4095, 0, 0, 0, 0, 3237, 0, 179, 0, 27, 0, 4096, 0, 0, 0, 14, 0, 4096, 0, 0, 2, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 0, 4086, 0, 3660, 460, 10, 4, 0, 0, 0, 0, 0, 0, 4090, 11, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4071, 6, 3, 2, 1, 0, 0, 0, 0, 0, 0, 26, 4083, 0, 0, 4084, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3600, 0, 0, 2, 1700, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3193, 0, 33, 0, 31, 4096, 436, 29, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 1, 0, 4096, 4067, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 8189, 8178, 8194, 8548, 7818, 8190, 8195] avg=8192.0000 max=8548 max_rank=4 min=7818 min_rank=5 imbalance_ratio=1.043457 max_load_reduction=56988 max_load_reduction_pct=0.869568 moved_to_echo=32751 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 4081, 0, 3658, 467, 10, 5, 0, 0, 0, 0, 0, 0, 4089, 8, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4074, 2, 37, 3, 1, 0, 0, 0, 0, 0, 0, 30, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3374, 0, 0, 4, 2677, 3296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2216, 0, 88, 0, 64, 4096, 436, 28, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8185, 8213, 8208, 9351, 6928, 8192, 8230] avg=8192.0000 max=9351 max_rank=4 min=6928 min_rank=5 imbalance_ratio=1.141479 max_load_reduction=56185 max_load_reduction_pct=0.857315 moved_to_echo=32701 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 4096, 0, 3657, 443, 10, 5, 0, 0, 0, 0, 0, 0, 4095, 8, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 20, 4092, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2585, 0, 1, 0, 2142, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 0, 114, 0, 10, 4096, 436, 29, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 1, 0, 4096, 4067, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8191, 8180, 8198, 7860, 8526, 8191, 8176] avg=8192.0000 max=8526 max_rank=5 min=7860 min_rank=4 imbalance_ratio=1.040771 max_load_reduction=57010 max_load_reduction_pct=0.869904 moved_to_echo=32753 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 25, 0, 3, 4096, 0, 3649, 428, 9, 5, 0, 0, 0, 0, 0, 0, 4082, 6, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4063, 11, 0, 3, 1, 0, 0, 0, 0, 0, 0, 25, 4093, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2612, 0, 0, 6, 2213, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3703, 0, 294, 0, 5, 4094, 434, 29, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8176, 8176, 8199, 7859, 8561, 8187, 8163] avg=8192.0000 max=8561 max_rank=5 min=7859 min_rank=4 imbalance_ratio=1.045044 max_load_reduction=56975 max_load_reduction_pct=0.869370 moved_to_echo=32770 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 21, 0, 0, 4082, 1, 3654, 312, 10, 4, 0, 0, 0, 0, 0, 1, 4093, 21, 4084, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 3, 4088, 0, 54, 2, 1, 0, 0, 0, 0, 0, 0, 8, 4071, 0, 0, 4082, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3627, 0, 2, 0, 2375, 2203, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 3061, 3, 555, 0, 0, 4092, 436, 29, 0, 0, 0, 0, 0, 2, 4091, 3, 3, 1, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 84, 0, 0, 3, 2, 3, 4091, 4060, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8087, 8202, 8244, 8166, 8210, 8183, 8195, 8249] avg=8192.0000 max=8249 max_rank=7 min=8087 min_rank=0 imbalance_ratio=1.006958 max_load_reduction=57287 max_load_reduction_pct=0.874130 moved_to_echo=32837 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 4096, 0, 3654, 451, 10, 5, 0, 0, 0, 0, 0, 0, 4095, 13, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 3, 1, 0, 0, 0, 0, 0, 0, 9, 4081, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3442, 0, 3, 0, 1481, 3183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3548, 0, 174, 0, 0, 4096, 436, 29, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 4067, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8196, 8196, 8175, 8109, 8283, 8191, 8167] avg=8192.0000 max=8283 max_rank=5 min=8109 min_rank=4 imbalance_ratio=1.011108 max_load_reduction=57253 max_load_reduction_pct=0.873611 moved_to_echo=32750 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 54, 0, 2, 4096, 0, 3658, 303, 9, 5, 0, 0, 0, 0, 0, 0, 4084, 58, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4070, 1, 2, 3, 1, 0, 0, 0, 0, 0, 0, 30, 4088, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3245, 0, 0, 4, 1798, 2145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 807, 0, 5, 4093, 436, 29, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 1, 0, 1, 0, 4096, 4067, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8230, 8171, 8200, 7192, 9234, 8192, 8190] avg=8192.0000 max=9234 max_rank=5 min=7192 min_rank=4 imbalance_ratio=1.127197 max_load_reduction=56302 max_load_reduction_pct=0.859100 moved_to_echo=32808 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 4096, 0, 3660, 445, 12, 7, 0, 0, 0, 0, 0, 0, 4081, 2, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4050, 17, 2, 5, 8, 0, 0, 0, 0, 0, 0, 27, 4096, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3364, 0, 0, 5, 1922, 3116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 0, 179, 0, 27, 4095, 436, 29, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 4096, 4067, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8167, 8178, 8196, 8407, 8003, 8198, 8166] avg=8192.0000 max=8407 max_rank=4 min=8003 min_rank=5 imbalance_ratio=1.026245 max_load_reduction=57129 max_load_reduction_pct=0.871719 moved_to_echo=32774 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 32768, 128, 128, 32640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 128, 32768, 128, 29312, 3328, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 32768, 0, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 256, 0, 128, 128, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25856, 0, 128, 128, 16384, 23424, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 26752, 128, 2304, 0, 256, 32768, 3584, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 4091, 4081, 0, 0, 0, 4094, 0, 4096, 4094, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4095, 0, 397, 122, 0, 1, 1, 0, 3673, 0, 0, 0, 4001, 4096, 0, 0, 4095, 22, 0, 4, 0, 4096, 4069, 12, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4092, 4091, 4091, 0, 0, 0, 4096, 0, 4096, 4094, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 0, 0, 4096, 0, 1811, 96, 0, 0, 0, 0, 2495, 0, 0, 0, 3757, 4096, 1, 0, 4080, 69, 0, 3, 0, 4096, 4080, 16, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 4095, 4094, 0, 0, 0, 4095, 0, 4096, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 285, 47, 0, 0, 3, 0, 3813, 0, 0, 0, 4036, 4096, 0, 0, 4090, 22, 0, 2, 0, 4095, 4092, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 4086, 4076, 0, 0, 0, 4096, 0, 4096, 4095, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 4096, 0, 1280, 55, 0, 0, 1, 0, 3038, 0, 0, 0, 3870, 4096, 0, 0, 4083, 42, 0, 2, 0, 4096, 4031, 11, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4091, 0, 4096, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 987, 244, 0, 0, 1, 0, 2872, 0, 0, 0, 4028, 4096, 0, 0, 4053, 105, 0, 0, 0, 4094, 4096, 7, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[3, 4, 0, 4, 4073, 3, 0, 4093, 4093, 4093, 0, 3, 0, 4087, 3, 4088, 4092, 0, 8, 4092, 4, 4, 5, 0, 0, 0, 0, 0, 3, 2, 3, 4093, 0, 3, 8, 4092, 4, 562, 266, 1, 1, 23, 3, 3302, 3, 0, 0, 4026, 4092, 0, 3, 4091, 1, 0, 19, 0, 4083, 4090, 6, 1, 0, 0, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 4084, 4083, 0, 0, 0, 4096, 0, 4096, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 240, 113, 0, 2, 0, 0, 3816, 0, 0, 0, 4004, 4096, 0, 0, 4063, 92, 0, 7, 0, 4096, 4069, 15, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4093, 4045, 4084, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 845, 106, 0, 3, 0, 0, 3359, 0, 0, 0, 4060, 4096, 0, 0, 4036, 42, 0, 11, 0, 4096, 3963, 33, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4091, 4081, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 13, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 397, 122, 0, 3162, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3673, 0, 0, 0, 4001, 908, 9, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4095, 22, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 12, 0, 0, 0, 1, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8192, 8200, 8195, 7777, 8593, 8185, 8201] avg=8192.0000 max=8593 max_rank=5 min=7777 min_rank=4 imbalance_ratio=1.048950 max_load_reduction=56943 max_load_reduction_pct=0.868881 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 285, 47, 0, 3163, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3813, 0, 0, 0, 4036, 908, 9, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4090, 22, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4092, 1, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8209, 8190, 8195, 7591, 8769, 8178, 8211] avg=8192.0000 max=8769 max_rank=5 min=7591 min_rank=4 imbalance_ratio=1.070435 max_load_reduction=56767 max_load_reduction_pct=0.866196 moved_to_echo=32749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 4091, 4091, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 1811, 96, 0, 3163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2495, 0, 0, 0, 3757, 909, 9, 0, 0, 0, 0, 0, 0, 4064, 1, 0, 4080, 69, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 16, 0, 0, 0, 1, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8202, 8188, 8186, 9167, 7170, 8217, 8216] avg=8192.0000 max=9167 max_rank=4 min=7170 min_rank=5 imbalance_ratio=1.119019 max_load_reduction=56369 max_load_reduction_pct=0.860123 moved_to_echo=32770 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 4, 0, 4, 4073, 3, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 3, 0, 20, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 8, 4089, 4, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 3, 4093, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 3, 8, 4092, 4, 562, 266, 1, 3156, 0, 0, 0, 0, 0, 0, 0, 1, 23, 3, 3302, 3, 0, 0, 4026, 907, 9, 0, 0, 0, 0, 0, 0, 4060, 0, 3, 4091, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4090, 6, 1, 0, 0, 0, 3, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8212, 8202, 8193, 8092, 8274, 8174, 8206] avg=8192.0000 max=8274 max_rank=5 min=8092 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57262 max_load_reduction_pct=0.873749 moved_to_echo=32746 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 4086, 4076, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 1280, 55, 0, 3163, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3038, 0, 0, 0, 3870, 909, 9, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4083, 42, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4031, 11, 0, 1, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8182, 8191, 8194, 8595, 7827, 8191, 8162] avg=8192.0000 max=8595 max_rank=4 min=7827 min_rank=5 imbalance_ratio=1.049194 max_load_reduction=56941 max_load_reduction_pct=0.868851 moved_to_echo=32775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4084, 4083, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 240, 113, 0, 3163, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3816, 0, 0, 0, 4004, 909, 9, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4063, 92, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 15, 0, 1, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8187, 8186, 8188, 7612, 8740, 8226, 8204] avg=8192.0000 max=8740 max_rank=5 min=7612 min_rank=4 imbalance_ratio=1.066895 max_load_reduction=56796 max_load_reduction_pct=0.866638 moved_to_echo=32782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 987, 244, 0, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2872, 0, 0, 0, 4028, 907, 9, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4053, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 7, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8212, 8191, 8196, 8487, 7817, 8222, 8220] avg=8192.0000 max=8487 max_rank=4 min=7817 min_rank=5 imbalance_ratio=1.036011 max_load_reduction=57049 max_load_reduction_pct=0.870499 moved_to_echo=32746 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4093, 4, 6, 0, 0, 0, 0, 0, 0, 4045, 4084, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 845, 106, 0, 3168, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3359, 0, 0, 0, 4060, 912, 12, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 4036, 42, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3963, 33, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8140, 8188, 8189, 8215, 8346, 8143, 8116] avg=8192.0000 max=8346 max_rank=5 min=8116 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=57190 max_load_reduction_pct=0.872650 moved_to_echo=32820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32512, 128, 128, 32640, 512, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 32512, 128, 128, 128, 0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 0, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 26368, 128, 0, 0, 31872, 7296, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 128, 6528, 1152, 128, 25344, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[2, 4093, 0, 0, 1, 0, 3, 4, 4086, 4081, 24, 3, 3, 4, 0, 2, 0, 1, 3, 8, 2179, 3, 657, 0, 4092, 2, 0, 3, 0, 4093, 1, 2, 0, 4090, 4053, 0, 15, 4092, 1, 1, 4088, 4093, 4091, 3, 0, 3, 4, 0, 0, 4092, 3, 0, 0, 0, 4090, 1, 1270, 3, 4092, 4092, 3, 3, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 24, 0, 0, 0, 4073, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 0, 174, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 4077, 4058, 0, 0, 4096, 1, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3554, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 35, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 773, 0, 538, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 4050, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2808, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 23, 0, 0, 0, 4068, 4090, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 479, 0, 4096, 0, 0, 1, 1, 4092, 0, 0, 5, 4093, 4091, 0, 0, 4096, 11, 0, 4096, 4089, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 3098, 0, 4095, 4095, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 76, 0, 0, 0, 4072, 4084, 9, 0, 0, 0, 0, 0, 2, 0, 0, 0, 943, 0, 264, 0, 4096, 0, 0, 3, 0, 4087, 0, 0, 4, 4095, 4072, 0, 3, 4095, 5, 0, 4096, 4091, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 2866, 0, 4095, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 36, 0, 0, 0, 4050, 4088, 25, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1072, 0, 171, 0, 4096, 0, 0, 7, 0, 4054, 0, 0, 3, 4088, 4091, 0, 2, 4096, 1, 0, 4096, 4090, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2898, 0, 4095, 4091, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 94, 0, 0, 0, 4059, 4079, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 0, 158, 0, 4096, 0, 0, 7, 0, 4087, 0, 0, 6, 4094, 4043, 0, 0, 4096, 2, 0, 4096, 4090, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2936, 0, 4092, 4092, 0, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 19, 0, 0, 0, 4022, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 107, 0, 4096, 0, 0, 8, 0, 4010, 0, 0, 7, 4092, 4096, 0, 0, 4096, 0, 0, 4096, 4081, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3803, 0, 4096, 4087, 0, 0, 10, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 36, 0, 0, 0, 2865, 1193, 0, 0, 0, 0, 0, 0, 4050, 4088, 25, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1072, 0, 171, 0, 4090, 2878, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 4054, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 3, 4088, 4091, 0, 2, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2898, 0, 1192, 4091, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8185, 8212, 8173, 8210, 8192, 8192, 8184] avg=8192.0000 max=8212 max_rank=2 min=8173 min_rank=3 imbalance_ratio=1.002441 max_load_reduction=57324 max_load_reduction_pct=0.874695 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 24, 0, 0, 0, 2865, 1193, 0, 0, 0, 0, 0, 0, 4073, 4095, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 0, 174, 0, 4090, 2878, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4058, 0, 0, 25, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3554, 0, 1193, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8190, 7573, 8201, 8161, 8198, 8192, 8843] avg=8192.0000 max=8843 max_rank=7 min=7573 min_rank=2 imbalance_ratio=1.079468 max_load_reduction=56693 max_load_reduction_pct=0.865067 moved_to_echo=33394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 23, 0, 0, 0, 2865, 1193, 0, 0, 0, 0, 0, 0, 4068, 4090, 13, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 479, 0, 4090, 2878, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 4092, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 5, 4093, 4091, 0, 0, 25, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3098, 0, 1192, 4095, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8193, 7964, 8206, 8225, 8191, 8193, 8388] avg=8192.0000 max=8388 max_rank=7 min=7964 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32997 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 4093, 0, 0, 1, 0, 3, 4, 2863, 1191, 0, 0, 0, 0, 0, 0, 4086, 4081, 24, 3, 3, 4, 0, 2, 21, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 8, 2179, 3, 657, 0, 4086, 2875, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 3, 0, 4093, 1, 2, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4053, 0, 15, 26, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4093, 5, 3, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1270, 3, 1192, 4092, 3, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8224, 9812, 8209, 8186, 8196, 8186, 6566] avg=8192.0000 max=9812 max_rank=2 min=6566 min_rank=7 imbalance_ratio=1.197754 max_load_reduction=55724 max_load_reduction_pct=0.850281 moved_to_echo=31134 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 76, 0, 0, 0, 2865, 1192, 0, 0, 0, 0, 0, 0, 4072, 4084, 9, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 943, 0, 264, 0, 4090, 2877, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4087, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 4072, 0, 3, 26, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2866, 0, 1192, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8187, 8176, 8202, 8205, 8193, 8193, 8150] avg=8192.0000 max=8230 max_rank=0 min=8150 min_rank=7 imbalance_ratio=1.004639 max_load_reduction=57306 max_load_reduction_pct=0.874420 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 35, 0, 0, 0, 2865, 1193, 0, 0, 0, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 773, 0, 538, 0, 4090, 2878, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4050, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2808, 0, 1193, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8208, 8279, 8208, 8165, 8198, 8192, 8097] avg=8192.0000 max=8279 max_rank=2 min=8097 min_rank=7 imbalance_ratio=1.010620 max_load_reduction=57257 max_load_reduction_pct=0.873672 moved_to_echo=32652 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 94, 0, 0, 0, 2863, 1193, 0, 0, 0, 0, 0, 0, 4059, 4079, 15, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 0, 158, 0, 4091, 2878, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 4087, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 6, 4094, 4043, 0, 0, 25, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2936, 0, 1192, 4092, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8246, 8174, 8132, 8206, 8170, 8191, 8192, 8225] avg=8192.0000 max=8246 max_rank=0 min=8132 min_rank=2 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32778 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 19, 0, 0, 0, 2868, 1195, 0, 0, 0, 0, 0, 0, 4022, 4096, 33, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 107, 0, 4096, 2882, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 4010, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 7, 4092, 4096, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3803, 0, 1183, 4087, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8175, 7388, 8131, 8214, 8177, 8196, 9083] avg=8192.0000 max=9083 max_rank=7 min=7388 min_rank=2 imbalance_ratio=1.108765 max_load_reduction=56453 max_load_reduction_pct=0.861404 moved_to_echo=33670 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[23296, 128, 9600, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 128, 128, 32640, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 32640, 0, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 7296, 128, 2560, 0, 32768, 23040, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32640, 32768, 128, 128, 128, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 0, 384, 0, 128, 128, 23040, 9600, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 4019, 0, 1, 10, 4096, 0, 0, 0, 3972, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 4084, 4096, 0, 1839, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 2397, 0, 36, 0, 38] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3271, 0, 0, 18, 4096, 0, 0, 0, 3863, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 2048, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3077, 0, 7, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4093, 0, 1, 4, 4096, 0, 0, 0, 4051, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 4094, 0, 448, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 0, 0, 0, 0, 3679, 0, 7, 0, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 3, 4092, 0, 0, 3, 1, 4092, 1, 0, 0, 4, 3881, 0, 1, 10, 4092, 4, 3, 0, 3970, 1, 0, 2, 4092, 1, 4092, 0, 4, 3, 3, 4089, 4091, 1, 865, 3, 0, 4091, 0, 4092, 4, 3, 8, 4092, 4092, 4, 0, 3, 5, 3, 0, 4092, 3, 1, 4089, 4, 0, 4, 4, 3471, 0, 0, 0, 67] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 3895, 0, 0, 31, 4096, 0, 0, 0, 3512, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 0, 4090, 4096, 0, 2121, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4093, 0, 0, 0, 0, 2734, 1, 5, 0, 25] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 4086, 0, 0, 0, 0, 3872, 0, 0, 30, 4096, 0, 0, 0, 3671, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4095, 0, 1003, 0, 3, 4096, 3, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3693, 0, 0, 0, 23] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4, 0, 0, 0, 4093, 0, 0, 0, 0, 4010, 0, 0, 8, 4096, 0, 0, 0, 3939, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4079, 4096, 0, 812, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 3433, 0, 89, 0, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3, 0, 0, 0, 4091, 0, 0, 0, 0, 3868, 0, 1, 30, 4096, 0, 0, 0, 3237, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 0, 4086, 4096, 0, 1380, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4087, 0, 0, 0, 0, 3736, 0, 10, 0, 47] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4085, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 1, 10, 3320, 990, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3972, 0, 1, 0, 317, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1839, 0, 0, 2783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2397, 0, 36, 0, 38, 4096, 769, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8340, 8386, 8187, 8718, 8191, 8191, 7336] avg=8192.0000 max=8718 max_rank=4 min=7336 min_rank=7 imbalance_ratio=1.064209 max_load_reduction=56818 max_load_reduction_pct=0.866974 moved_to_echo=32436 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 4, 3320, 990, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4051, 0, 1, 0, 317, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 448, 0, 0, 2783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3679, 0, 7, 0, 14, 4096, 769, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8408, 8465, 8195, 7325, 8191, 8190, 8565] avg=8192.0000 max=8565 max_rank=7 min=7325 min_rank=4 imbalance_ratio=1.045532 max_load_reduction=56971 max_load_reduction_pct=0.869308 moved_to_echo=32271 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3271, 0, 0, 18, 3320, 990, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3863, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2048, 0, 0, 2783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3077, 0, 7, 0, 4, 4096, 769, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 7599, 8276, 8199, 8927, 8191, 8193, 7953] avg=8192.0000 max=8927 max_rank=4 min=7599 min_rank=1 imbalance_ratio=1.089722 max_load_reduction=56609 max_load_reduction_pct=0.863785 moved_to_echo=33264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4069, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 0, 0, 31, 3320, 990, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3512, 0, 2, 0, 317, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2121, 0, 0, 2783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 1, 5, 0, 25, 4096, 769, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8236, 7927, 8193, 9000, 8191, 8188, 7630] avg=8192.0000 max=9000 max_rank=4 min=7630 min_rank=7 imbalance_ratio=1.098633 max_load_reduction=56536 max_load_reduction_pct=0.862671 moved_to_echo=33009 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4092, 0, 0, 3, 1, 4092, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 3881, 0, 1, 10, 3316, 988, 0, 0, 0, 0, 0, 0, 4092, 4, 3, 0, 3970, 1, 0, 2, 316, 0, 0, 0, 0, 0, 0, 0, 8, 1, 4092, 0, 4, 3, 3, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 865, 3, 0, 2781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 8, 4092, 4091, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 4092, 3, 1, 4089, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 3471, 0, 0, 0, 67, 4092, 768, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8201, 8388, 8200, 7741, 8205, 8198, 8406] avg=8192.0000 max=8406 max_rank=7 min=7741 min_rank=4 imbalance_ratio=1.026123 max_load_reduction=57130 max_load_reduction_pct=0.871735 moved_to_echo=32550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 4086, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 30, 3320, 990, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3671, 0, 0, 1, 317, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1003, 0, 3, 2783, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3693, 0, 0, 0, 23, 4096, 769, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8212, 8085, 8198, 7887, 8191, 8193, 8581] avg=8192.0000 max=8581 max_rank=7 min=7887 min_rank=4 imbalance_ratio=1.047485 max_load_reduction=56955 max_load_reduction_pct=0.869064 moved_to_echo=32852 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4, 0, 0, 0, 4093, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 8, 3321, 992, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3939, 0, 0, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 812, 0, 0, 2771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3433, 0, 89, 0, 14, 4096, 775, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8331, 8355, 8175, 7679, 8185, 8203, 8407] avg=8192.0000 max=8407 max_rank=7 min=7679 min_rank=4 imbalance_ratio=1.026245 max_load_reduction=57129 max_load_reduction_pct=0.871719 moved_to_echo=32474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3, 0, 0, 0, 4091, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 0, 1, 30, 3320, 990, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3237, 0, 4, 0, 317, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1380, 0, 0, 2783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4087, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3736, 0, 10, 0, 47, 4096, 769, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8209, 7654, 8189, 8259, 8191, 8180, 8658] avg=8192.0000 max=8658 max_rank=7 min=7654 min_rank=2 imbalance_ratio=1.056885 max_load_reduction=56878 max_load_reduction_pct=0.867889 moved_to_echo=33288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 26240, 128, 256, 0, 256, 32768, 6272, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 30336, 128, 128, 128, 2560, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 10624, 128, 128, 22272, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 30976, 0, 128, 256, 26624, 7936, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 57, 0, 0, 3894, 2021, 0, 4096, 4089, 0, 0, 4096, 0, 2434, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 611, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4084, 0, 0, 0, 3981, 1, 0, 8, 0, 0, 0, 4096, 8, 3375, 0, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 79, 0, 0, 4001, 2555, 0, 4096, 4089, 0, 0, 4096, 0, 2686, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 430, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 4094, 0, 0, 0, 3879, 0, 0, 23, 0, 0, 0, 4096, 2, 2731, 0, 0, 4096, 4095, 0, 0, 0, 0, 2, 1, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 43, 0, 0, 4056, 2854, 0, 4096, 4091, 0, 0, 4096, 0, 3121, 0, 0, 4095, 6, 0, 4096, 0, 0, 0, 160, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3561, 0, 0, 63, 0, 0, 0, 4096, 0, 2522, 0, 0, 4096, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 114, 0, 1, 3979, 1511, 0, 4096, 4073, 0, 0, 4096, 0, 3330, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 63, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 4093, 0, 0, 0, 3700, 3, 0, 1, 0, 0, 0, 4096, 4, 3694, 0, 0, 4096, 4095, 0, 0, 0, 0, 4, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 229, 0, 0, 4018, 1821, 0, 4096, 4096, 0, 0, 4096, 0, 3290, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 208, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4005, 0, 0, 46, 0, 0, 0, 4096, 0, 2767, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4, 0, 12, 4, 122, 2, 0, 4033, 1887, 6, 4090, 4083, 1, 0, 4087, 4, 2803, 0, 3, 4006, 0, 0, 4091, 0, 0, 2, 205, 4, 4, 4088, 4, 0, 3, 1, 0, 4, 4, 4066, 0, 0, 4, 4033, 0, 0, 41, 0, 3, 4, 4090, 0, 3348, 3, 0, 4089, 4089, 0, 0, 4, 3, 19, 0, 4092, 0, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 73, 0, 0, 3986, 2864, 0, 4096, 4094, 0, 0, 4096, 0, 2386, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 989, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 4081, 0, 0, 0, 3553, 0, 0, 54, 0, 0, 0, 4096, 0, 2486, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 82, 0, 1, 3993, 3481, 0, 4096, 4072, 0, 0, 4096, 0, 2771, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 123, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 4091, 0, 0, 0, 3707, 5, 0, 17, 0, 0, 0, 4096, 0, 2215, 0, 0, 4096, 4096, 0, 0, 0, 0, 8, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 57, 0, 0, 3894, 2629, 1465, 0, 0, 0, 0, 0, 0, 2021, 0, 0, 4089, 0, 0, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2434, 0, 0, 4096, 0, 0, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 611, 0, 0, 4096, 0, 0, 2365, 1376, 5, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4084, 0, 0, 3835, 263, 0, 0, 0, 0, 0, 0, 0, 3981, 1, 0, 8, 0, 0, 0, 4096, 261, 0, 0, 0, 0, 0, 0, 0, 8, 3375, 0, 0, 1204, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8045, 7841, 7785, 8453, 8188, 8347, 8683, 8194] avg=8192.0000 max=8683 max_rank=6 min=7785 min_rank=2 imbalance_ratio=1.059937 max_load_reduction=56853 max_load_reduction_pct=0.867508 moved_to_echo=33412 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 79, 0, 0, 4001, 2629, 1465, 0, 0, 0, 0, 0, 0, 2555, 0, 0, 4089, 0, 0, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2686, 0, 0, 4096, 0, 0, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 430, 0, 0, 4096, 0, 0, 2365, 1376, 5, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4094, 0, 0, 3836, 263, 0, 0, 0, 0, 0, 0, 0, 3879, 0, 0, 23, 0, 0, 0, 4096, 260, 0, 0, 0, 0, 0, 0, 0, 2, 2731, 0, 0, 1204, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8174, 8375, 8037, 8272, 8198, 8258, 8032, 8190] avg=8192.0000 max=8375 max_rank=1 min=8032 min_rank=6 imbalance_ratio=1.022339 max_load_reduction=57161 max_load_reduction_pct=0.872208 moved_to_echo=32678 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 229, 0, 0, 4018, 2629, 1465, 0, 0, 0, 0, 0, 0, 1821, 0, 0, 4096, 0, 0, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3290, 0, 0, 4096, 0, 0, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 4096, 0, 0, 2365, 1376, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3835, 263, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 46, 0, 0, 0, 4096, 261, 0, 0, 0, 0, 0, 0, 0, 0, 2767, 0, 0, 1204, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8341, 7648, 8641, 8050, 8194, 8408, 8067, 8187] avg=8192.0000 max=8641 max_rank=2 min=7648 min_rank=1 imbalance_ratio=1.054810 max_load_reduction=56895 max_load_reduction_pct=0.868149 moved_to_echo=32856 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 114, 0, 1, 3979, 2629, 1465, 0, 0, 0, 0, 0, 0, 1511, 0, 0, 4073, 0, 0, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3330, 0, 0, 4096, 0, 0, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 4096, 0, 0, 2365, 1376, 5, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4093, 0, 0, 3835, 263, 0, 0, 0, 0, 0, 0, 0, 3700, 3, 0, 1, 0, 0, 0, 4096, 261, 0, 0, 0, 0, 0, 0, 0, 4, 3694, 0, 0, 1204, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 7315, 8681, 7905, 8198, 8061, 8997, 8191] avg=8192.0000 max=8997 max_rank=6 min=7315 min_rank=1 imbalance_ratio=1.098267 max_load_reduction=56539 max_load_reduction_pct=0.862717 moved_to_echo=33447 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 43, 0, 0, 4056, 2629, 1465, 0, 0, 0, 0, 0, 0, 2854, 0, 0, 4091, 0, 0, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3121, 0, 0, 4095, 6, 0, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 4096, 0, 0, 2365, 1376, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3836, 263, 0, 0, 0, 0, 0, 0, 0, 3561, 0, 0, 63, 0, 0, 0, 4096, 260, 0, 0, 0, 0, 0, 0, 0, 0, 2522, 0, 0, 1204, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8676, 8477, 8002, 8195, 7980, 7822, 8191] avg=8192.0000 max=8676 max_rank=1 min=7822 min_rank=6 imbalance_ratio=1.059082 max_load_reduction=56860 max_load_reduction_pct=0.867615 moved_to_echo=32188 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 12, 4, 122, 2, 0, 4033, 2624, 1463, 0, 0, 0, 0, 0, 0, 1887, 6, 0, 4083, 1, 0, 1728, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2803, 0, 3, 4006, 0, 0, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 205, 4, 4, 4088, 4, 0, 2359, 1375, 5, 0, 0, 0, 0, 0, 3, 1, 0, 4, 4, 4066, 0, 0, 3831, 263, 0, 0, 0, 0, 0, 0, 4, 4033, 0, 0, 41, 0, 3, 4, 4090, 259, 0, 0, 0, 0, 0, 0, 0, 0, 3348, 3, 0, 1202, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 19, 0, 4087, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 7709, 8065, 8046, 8172, 8434, 8642, 8204] avg=8192.0000 max=8642 max_rank=6 min=7709 min_rank=1 imbalance_ratio=1.054932 max_load_reduction=56894 max_load_reduction_pct=0.868134 moved_to_echo=33452 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 73, 0, 0, 3986, 2633, 1469, 0, 0, 0, 0, 0, 0, 2864, 0, 0, 4094, 0, 0, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2386, 0, 0, 4096, 0, 0, 1246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 989, 0, 0, 4096, 0, 0, 2366, 1381, 11, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4081, 0, 0, 3836, 267, 0, 0, 0, 0, 0, 0, 0, 3553, 0, 0, 54, 0, 0, 0, 4096, 260, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 0, 0, 1196, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8161, 8688, 7728, 8843, 8191, 7963, 7778, 8184] avg=8192.0000 max=8843 max_rank=3 min=7728 min_rank=2 imbalance_ratio=1.079468 max_load_reduction=56693 max_load_reduction_pct=0.865067 moved_to_echo=32116 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 82, 0, 1, 3993, 2629, 1465, 0, 0, 0, 0, 0, 0, 3481, 0, 0, 4072, 0, 0, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2771, 0, 0, 4096, 0, 0, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 4096, 0, 0, 2365, 1376, 5, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4091, 0, 0, 3836, 263, 0, 0, 0, 0, 0, 0, 0, 3707, 5, 0, 17, 0, 0, 0, 4096, 260, 0, 0, 0, 0, 0, 0, 0, 0, 2215, 0, 0, 1204, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 9284, 8122, 7965, 8200, 8085, 7515, 8195] avg=8192.0000 max=9284 max_rank=1 min=7515 min_rank=6 imbalance_ratio=1.133301 max_load_reduction=56252 max_load_reduction_pct=0.858337 moved_to_echo=31995 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 896, 128, 128, 32000, 21120, 11776, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 30464, 128, 0, 256, 0, 128, 128, 32768, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[22912, 0, 128, 32768, 128, 0, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 23168, 128, 0, 9728, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 2816, 128, 128, 32768, 128, 0, 18944, 11136, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[19072, 128, 0, 32768, 128, 0, 13952, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 0, 30720, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 0, 0, 4, 8, 0, 4096, 4096, 0, 0, 4090, 0, 0, 2183, 4095, 4096, 8, 4073, 1731, 0, 1628, 4092, 0, 4087, 0, 3, 0, 0, 0, 0, 1813, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 432, 421, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 11, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 1032, 4096, 4096, 0, 4094, 1584, 0, 3589, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 1897, 0, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 41, 19, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4083, 0, 0, 4, 22, 0, 4096, 4096, 0, 0, 4088, 0, 0, 2031, 4096, 4096, 35, 4050, 3044, 0, 1468, 4093, 0, 4068, 0, 8, 0, 0, 0, 0, 831, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 567, 272, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 0, 0, 10, 8, 0, 4096, 4096, 0, 0, 4085, 0, 0, 1099, 4095, 4096, 7, 4076, 1138, 0, 2851, 4088, 0, 4091, 0, 10, 1, 0, 0, 0, 3034, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 4096, 0, 53, 23, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3, 0, 2, 4092, 0, 4, 0, 4091, 4, 4093, 4, 4, 14, 0, 4, 4060, 4088, 0, 2, 4092, 3, 0, 2132, 4090, 4079, 0, 4087, 2505, 4, 1276, 4092, 3, 4093, 1, 8, 0, 4, 0, 0, 2180, 0, 1, 80, 0, 4, 3, 0, 1, 4092, 3, 4, 0, 4, 4, 0, 3, 0, 4089, 4092, 0, 27, 12, 0, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 2, 0, 4096, 4096, 0, 0, 4096, 0, 0, 1408, 4096, 4096, 0, 4083, 1126, 0, 3109, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 2524, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 22, 7, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4077, 0, 0, 11, 21, 0, 4096, 4096, 0, 0, 4084, 0, 0, 1658, 4096, 4096, 17, 4008, 2305, 0, 2253, 4082, 0, 4068, 0, 9, 0, 0, 0, 0, 1823, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 4096, 0, 219, 25, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4096, 0, 4087, 0, 0, 2, 16, 0, 4096, 4096, 0, 0, 4079, 0, 0, 869, 4096, 4096, 1, 4050, 1514, 0, 2922, 4090, 0, 4084, 0, 13, 0, 0, 0, 0, 2916, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 4095, 4096, 0, 2, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 4, 22, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 0, 0, 4088, 0, 0, 2031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 3913, 3044, 0, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 8, 0, 0, 0, 0, 831, 1544, 267, 137, 16, 6, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 4078, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 4077, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 3829, 0, 567, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8199, 8671, 8460, 6877, 8176, 8197, 8764] avg=8192.0000 max=8764 max_rank=7 min=6877 min_rank=4 imbalance_ratio=1.069824 max_load_reduction=56772 max_load_reduction_pct=0.866272 moved_to_echo=32014 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4, 8, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 0, 0, 4090, 0, 0, 2183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3935, 1731, 0, 1628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 3, 0, 0, 0, 0, 1813, 1544, 267, 138, 17, 6, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4095, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4075, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 3829, 0, 432, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8190, 8825, 7302, 7875, 8185, 8189, 8778] avg=8192.0000 max=8825 max_rank=2 min=7302 min_rank=3 imbalance_ratio=1.077271 max_load_reduction=56711 max_load_reduction_pct=0.865341 moved_to_echo=33027 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 2, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 0, 0, 4096, 0, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 1126, 0, 3109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2524, 1544, 267, 138, 16, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4079, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 3829, 0, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8190, 8056, 8180, 8591, 8181, 8192, 7954] avg=8192.0000 max=8591 max_rank=4 min=7954 min_rank=7 imbalance_ratio=1.048706 max_load_reduction=56945 max_load_reduction_pct=0.868912 moved_to_echo=32918 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 0, 0, 4096, 0, 0, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 1584, 0, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1897, 1543, 267, 138, 16, 6, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4079, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 3829, 0, 41, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8197, 7680, 9129, 7963, 8198, 8192, 7985] avg=8192.0000 max=9129 max_rank=3 min=7680 min_rank=2 imbalance_ratio=1.114380 max_load_reduction=56407 max_load_reduction_pct=0.860703 moved_to_echo=32338 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 10, 8, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 0, 0, 4085, 0, 0, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3938, 1138, 0, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 10, 1, 0, 0, 0, 3034, 1544, 267, 138, 16, 6, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4095, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 4072, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 3829, 0, 53, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8196, 7736, 7934, 9107, 8180, 8190, 8001] avg=8192.0000 max=9107 max_rank=4 min=7736 min_rank=2 imbalance_ratio=1.111694 max_load_reduction=56429 max_load_reduction_pct=0.861038 moved_to_echo=33478 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 2, 4092, 0, 4, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4093, 4, 4, 14, 0, 4, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 2547, 0, 2, 4092, 3, 0, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 2505, 4, 1276, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 8, 0, 4, 0, 0, 2180, 1541, 266, 138, 17, 6, 0, 0, 0, 0, 1, 80, 0, 4, 3, 0, 1, 4090, 4063, 0, 0, 0, 0, 0, 0, 4092, 3, 4, 0, 4, 4, 0, 3, 4075, 16, 0, 0, 0, 0, 0, 0, 0, 4089, 3826, 0, 27, 12, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8177, 8776, 7737, 8254, 8242, 8201, 7957] avg=8192.0000 max=8776 max_rank=2 min=7737 min_rank=3 imbalance_ratio=1.071289 max_load_reduction=56760 max_load_reduction_pct=0.866089 moved_to_echo=32654 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 11, 21, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 2550, 0, 0, 4084, 0, 0, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3869, 2305, 0, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 9, 0, 0, 0, 0, 1823, 1546, 269, 139, 17, 13, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 4065, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 3827, 0, 219, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8192, 8292, 8444, 7885, 8182, 8183, 8167] avg=8192.0000 max=8444 max_rank=3 min=7885 min_rank=4 imbalance_ratio=1.030762 max_load_reduction=57092 max_load_reduction_pct=0.871155 moved_to_echo=32417 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 2, 16, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 0, 0, 4079, 0, 0, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3913, 1514, 0, 2922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 13, 0, 0, 0, 0, 2916, 1544, 267, 137, 17, 6, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 4073, 17, 0, 0, 0, 0, 0, 0, 0, 4095, 3829, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8195, 7500, 8350, 8984, 8192, 8192, 7930] avg=8192.0000 max=8984 max_rank=4 min=7500 min_rank=2 imbalance_ratio=1.096680 max_load_reduction=56552 max_load_reduction_pct=0.862915 moved_to_echo=33298 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[20480, 0, 128, 32768, 128, 0, 12416, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 17024, 12416, 2176, 1152, 256, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 31488, 14976, 128, 19200, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 128, 32640, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 256, 0, 128, 128, 0, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 30720, 0, 1408, 896, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 2645, 4096, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4067, 0, 0, 4003, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 1408, 4094, 0, 0, 1, 4095, 0, 0, 4094, 4096, 0, 179, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1664, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4078, 0, 0, 4044, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 2155, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 353, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2374, 4096, 0, 0, 0, 0, 2, 0, 17, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4037, 0, 0, 4081, 0, 0, 0, 0, 0, 4096, 0, 4022, 0, 1529, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 337, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 2244, 4096, 0, 0, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4041, 0, 0, 4048, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 1716, 4095, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 244, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 3, 0, 0, 15, 0, 0, 55, 4069, 0, 4, 4, 0, 0, 4, 0, 2, 5, 1, 1, 3, 110, 4091, 4, 4, 3, 36, 4090, 10, 3, 4092, 0, 0, 4082, 4057, 3, 4, 4088, 167, 75, 2, 0, 0, 4085, 3, 4088, 4, 3791, 4092, 4, 0, 0, 4092, 5, 4, 4092, 3991, 0, 17, 0, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2697, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4069, 0, 0, 4007, 0, 1, 1, 0, 0, 4096, 0, 4096, 0, 1397, 4094, 0, 1, 0, 4096, 0, 0, 4096, 4093, 0, 117, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3119, 4096, 0, 0, 0, 0, 4, 0, 9, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4040, 0, 0, 3979, 0, 0, 0, 0, 0, 4096, 0, 4072, 0, 972, 4096, 0, 0, 0, 4096, 0, 0, 4092, 4096, 0, 202, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 637, 4096, 0, 0, 0, 0, 2, 0, 11, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4048, 0, 0, 4013, 0, 0, 0, 0, 0, 4096, 0, 4061, 0, 3457, 4096, 0, 1, 0, 4096, 0, 0, 4095, 4096, 0, 166, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 2, 0, 4054, 31, 9, 0, 0, 0, 0, 0, 0, 0, 2645, 4096, 0, 0, 0, 0, 2000, 168, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 4095, 4087, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 4096, 0, 19, 14, 0, 0, 0, 0, 0, 0, 4076, 0, 1408, 2063, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 4096, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8909, 8194, 8174, 8205, 8132, 7548, 8182] avg=8192.0000 max=8909 max_rank=1 min=7548 min_rank=6 imbalance_ratio=1.087524 max_load_reduction=56627 max_load_reduction_pct=0.864059 moved_to_echo=32067 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4054, 31, 9, 0, 0, 0, 0, 0, 0, 0, 1664, 4096, 0, 0, 0, 0, 2001, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 4096, 0, 19, 14, 0, 0, 0, 0, 0, 0, 4091, 0, 2155, 2064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 4095, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7929, 8183, 8178, 8216, 8173, 8310, 8357] avg=8192.0000 max=8357 max_rank=7 min=7929 min_rank=1 imbalance_ratio=1.020142 max_load_reduction=57179 max_load_reduction_pct=0.872482 moved_to_echo=33056 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 4054, 31, 9, 0, 0, 0, 0, 0, 0, 0, 2244, 4096, 0, 0, 0, 0, 2000, 168, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 4096, 0, 19, 14, 0, 0, 0, 0, 0, 0, 4090, 0, 1716, 2064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 4096, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8508, 8193, 8171, 8179, 8177, 7870, 8247] avg=8192.0000 max=8508 max_rank=1 min=7870 min_rank=6 imbalance_ratio=1.038574 max_load_reduction=57028 max_load_reduction_pct=0.870178 moved_to_echo=32473 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 3, 0, 0, 15, 4050, 31, 9, 0, 0, 0, 0, 0, 0, 0, 55, 4069, 0, 4, 4, 0, 1999, 168, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 5, 1, 1, 3, 4092, 4082, 0, 0, 0, 0, 0, 0, 110, 4091, 4, 4, 3, 36, 4076, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3, 42, 0, 0, 4082, 4057, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 167, 75, 2, 0, 0, 4085, 3, 19, 14, 0, 0, 0, 0, 0, 0, 4088, 4, 3791, 2062, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 3905, 3991, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 6299, 8190, 8334, 8191, 8453, 9949, 7922] avg=8192.0000 max=9949 max_rank=6 min=6299 min_rank=1 imbalance_ratio=1.214478 max_load_reduction=55587 max_load_reduction_pct=0.848190 moved_to_echo=34515 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4054, 31, 9, 0, 0, 0, 0, 0, 0, 0, 2697, 4096, 0, 0, 0, 0, 2000, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 1, 1, 0, 0, 4096, 0, 19, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 1397, 2063, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 4093, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8961, 8183, 8181, 8207, 8138, 7557, 8119] avg=8192.0000 max=8961 max_rank=1 min=7557 min_rank=6 imbalance_ratio=1.093872 max_load_reduction=56575 max_load_reduction_pct=0.863266 moved_to_echo=32021 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4055, 34, 10, 0, 0, 0, 0, 0, 0, 0, 2374, 4096, 0, 0, 0, 0, 2002, 172, 0, 0, 0, 0, 0, 0, 2, 0, 17, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 4096, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 4096, 0, 21, 15, 0, 0, 0, 0, 0, 0, 4022, 0, 1529, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 4096, 0, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8644, 8201, 8162, 8174, 8213, 7611, 8336] avg=8192.0000 max=8644 max_rank=1 min=7611 min_rank=6 imbalance_ratio=1.055176 max_load_reduction=56892 max_load_reduction_pct=0.868103 moved_to_echo=32334 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4054, 31, 9, 0, 0, 0, 0, 0, 0, 0, 637, 4096, 0, 0, 0, 0, 2001, 168, 0, 0, 0, 0, 0, 0, 2, 0, 11, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 4096, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 4096, 0, 19, 14, 0, 0, 0, 0, 0, 0, 4061, 0, 3457, 2064, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 4096, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 6902, 8196, 8167, 8186, 8142, 9583, 8170] avg=8192.0000 max=9583 max_rank=6 min=6902 min_rank=1 imbalance_ratio=1.169800 max_load_reduction=55953 max_load_reduction_pct=0.853775 moved_to_echo=34081 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4054, 31, 9, 0, 0, 0, 0, 0, 0, 0, 3119, 4096, 0, 0, 0, 0, 2001, 168, 0, 0, 0, 0, 0, 0, 4, 0, 9, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 4096, 0, 19, 14, 0, 0, 0, 0, 0, 0, 4072, 0, 972, 2064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 4096, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 9384, 8196, 8169, 8178, 8108, 7108, 8203] avg=8192.0000 max=9384 max_rank=1 min=7108 min_rank=6 imbalance_ratio=1.145508 max_load_reduction=56152 max_load_reduction_pct=0.856812 moved_to_echo=31597 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32384, 256, 128, 128, 0, 0, 32768, 128, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 128, 16512, 16512, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 128, 128, 32512, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 384, 0, 0, 32768, 32512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 31360, 32768, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 15488, 32768, 0, 128, 128, 0, 16128, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 3, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4083, 4094, 0, 3, 0, 0, 0, 0, 4095, 5, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 3, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4083, 4092, 0, 2, 0, 0, 0, 0, 4096, 11, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4092, 0, 4092, 4049, 6, 3, 4, 4, 0, 4092, 0, 4, 0, 40, 8, 4066, 4087, 3, 4, 5, 5, 3, 4, 226, 0, 3, 4, 0, 3, 0, 4082, 4090, 0, 3, 1167, 17, 4, 0, 4089, 0, 4052, 0, 4, 11, 0, 4, 0, 4092, 0, 7, 2736, 4092, 0, 2, 2, 0, 0, 3, 4089, 0, 4092, 0, 0, 4091] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4090, 4096, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 24, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3949, 4092, 0, 4, 0, 0, 0, 0, 4096, 7, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 21, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3722, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3951, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 25, 0, 0, 0, 4095, 0, 4072, 0, 0, 2, 0, 0, 0, 4091, 0, 0, 3996, 4088, 0, 9, 0, 0, 0, 0, 4096, 21, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4090, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 16, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 3555, 4091, 0, 5, 0, 0, 0, 0, 4096, 13, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 12, 3750, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4087, 6, 3, 3, 0, 0, 0, 0, 0, 0, 0, 4083, 4092, 0, 2, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8189, 8017, 8196, 8036, 8189, 8512, 8198] avg=8192.0000 max=8512 max_rank=6 min=8017 min_rank=2 imbalance_ratio=1.039062 max_load_reduction=57024 max_load_reduction_pct=0.870117 moved_to_echo=32935 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 13, 3750, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4089, 6, 3, 3, 0, 0, 0, 0, 0, 0, 0, 4083, 4094, 0, 3, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8192, 8018, 8194, 8036, 8193, 8515, 8191] avg=8192.0000 max=8515 max_rank=6 min=8018 min_rank=2 imbalance_ratio=1.039429 max_load_reduction=57021 max_load_reduction_pct=0.870071 moved_to_echo=32935 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 126, 3750, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4096, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4091, 6, 3, 3, 0, 0, 0, 0, 0, 0, 0, 3949, 4092, 0, 4, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8196, 8131, 8194, 8057, 8191, 8380, 8193] avg=8192.0000 max=8380 max_rank=6 min=8057 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=57156 max_load_reduction_pct=0.872131 moved_to_echo=32821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 354, 3750, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4096, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 6, 3, 3, 0, 0, 0, 0, 0, 0, 0, 3722, 4096, 0, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8189, 8359, 8192, 8054, 8203, 8153, 8186] avg=8192.0000 max=8359 max_rank=2 min=8054 min_rank=4 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32596 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4092, 4049, 6, 3, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4, 0, 40, 8, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3, 4, 5, 5, 3, 4, 226, 3746, 158, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 3, 0, 4082, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1167, 17, 4, 0, 4089, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 4, 11, 0, 4, 0, 4092, 6, 3, 3, 0, 0, 0, 0, 0, 0, 7, 2736, 4092, 0, 2, 2, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4089, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8166, 8207, 8241, 8182, 9214, 8175, 7174, 8177] avg=8192.0000 max=9214 max_rank=4 min=7174 min_rank=6 imbalance_ratio=1.124756 max_load_reduction=56322 max_load_reduction_pct=0.859406 moved_to_echo=32740 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 144, 3750, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 6, 3, 3, 0, 0, 0, 0, 0, 0, 0, 3951, 4096, 0, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8189, 8149, 8192, 8034, 8204, 8382, 8186] avg=8192.0000 max=8382 max_rank=6 min=8034 min_rank=4 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32806 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 78, 3751, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4095, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 2, 0, 0, 0, 4091, 7, 10, 8, 5, 0, 0, 0, 0, 0, 0, 3996, 4088, 0, 9, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8184, 8085, 8190, 8056, 8195, 8428, 8206] avg=8192.0000 max=8428 max_rank=6 min=8056 min_rank=4 imbalance_ratio=1.028809 max_load_reduction=57108 max_load_reduction_pct=0.871399 moved_to_echo=32885 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 531, 3750, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4084, 6, 3, 3, 0, 0, 0, 0, 0, 0, 0, 3555, 4091, 0, 5, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8190, 8536, 8196, 8049, 8186, 7992, 8199] avg=8192.0000 max=8536 max_rank=2 min=7992 min_rank=6 imbalance_ratio=1.041992 max_load_reduction=57000 max_load_reduction_pct=0.869751 moved_to_echo=32426 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 32768, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 30080, 32768, 0, 128, 128, 0, 2688, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 1280, 128, 128, 0, 32768, 0, 31616, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 1536, 30080, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 128, 0, 128, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4096, 4096, 0, 2902, 4096, 0, 4096, 4096, 4078, 0, 0, 0, 0, 0, 0, 4096, 207, 0, 0, 1, 0, 5, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 1052] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 4096, 4096, 0, 2188, 4096, 0, 4096, 4096, 4055, 0, 0, 0, 0, 0, 0, 4096, 153, 0, 0, 1, 0, 19, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 1824] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 4096, 4096, 0, 2944, 4096, 0, 4096, 4096, 4077, 0, 0, 0, 0, 0, 0, 4096, 760, 0, 0, 0, 0, 8, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 413] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3166, 4096, 0, 4096, 4096, 4060, 0, 0, 0, 0, 0, 0, 4096, 794, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 173] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4036, 0, 18, 4, 4, 0, 4092, 4093, 110, 5, 3, 0, 0, 0, 4, 4, 0, 0, 0, 21, 4, 64, 0, 4092, 0, 3, 4076, 4090, 0, 3892, 4071, 15, 4091, 4092, 3845, 0, 0, 0, 0, 4, 5, 4091, 1280, 17, 0, 0, 0, 0, 4088, 4091, 0, 6, 4, 4076, 1, 0, 0, 5, 5, 2985, 5, 4, 2, 138] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3201, 4096, 0, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 4096, 410, 0, 0, 0, 0, 10, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 496] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3069, 4096, 0, 4096, 4096, 3968, 0, 0, 0, 0, 0, 0, 4096, 38, 0, 0, 0, 0, 16, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 1116] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4092, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 5, 0, 4096, 4096, 0, 3269, 4096, 0, 4096, 4096, 4085, 0, 0, 0, 0, 0, 0, 4096, 505, 0, 0, 0, 0, 8, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 327] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 9, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4073, 12, 0, 0, 0, 0, 0, 0, 4, 0, 1017, 0, 0, 2188, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 67, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 153, 0, 0, 1, 0, 19, 3553, 13, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 1824, 3079, 476, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8177, 8181, 7305, 8218, 7835, 8195, 9430] avg=8192.0000 max=9430 max_rank=7 min=7305 min_rank=3 imbalance_ratio=1.151123 max_load_reduction=56106 max_load_reduction_pct=0.856110 moved_to_echo=33678 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 9, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4073, 13, 0, 0, 0, 0, 0, 0, 3, 0, 1017, 0, 0, 2902, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 67, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 207, 0, 0, 1, 0, 5, 3553, 13, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 1052, 3079, 476, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8177, 8182, 8018, 8241, 7875, 8194, 8650] avg=8192.0000 max=8650 max_rank=7 min=7875 min_rank=5 imbalance_ratio=1.055908 max_load_reduction=56886 max_load_reduction_pct=0.868011 moved_to_echo=32960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4074, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 0, 0, 3201, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 67, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 410, 0, 0, 0, 0, 10, 3553, 13, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 496, 3079, 476, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8177, 8182, 8314, 8243, 8082, 8195, 8142] avg=8192.0000 max=8314 max_rank=3 min=8082 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32662 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 9, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4073, 12, 0, 0, 0, 0, 0, 0, 4, 0, 1017, 0, 0, 2944, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 67, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 760, 0, 0, 0, 0, 8, 3553, 13, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 413, 3079, 476, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8177, 8181, 8061, 8240, 8430, 8195, 8056] avg=8192.0000 max=8430 max_rank=5 min=8056 min_rank=7 imbalance_ratio=1.029053 max_load_reduction=57106 max_load_reduction_pct=0.871368 moved_to_echo=32921 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4074, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 0, 0, 3166, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 67, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 794, 0, 0, 0, 0, 0, 3553, 13, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 173, 3079, 476, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8177, 8182, 8279, 8223, 8456, 8195, 7823] avg=8192.0000 max=8456 max_rank=5 min=7823 min_rank=7 imbalance_ratio=1.032227 max_load_reduction=57080 max_load_reduction_pct=0.870972 moved_to_echo=32697 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4036, 0, 18, 4, 4, 0, 9, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 110, 5, 3, 0, 0, 0, 4, 4, 4090, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4, 64, 0, 4092, 4070, 12, 0, 0, 0, 0, 0, 0, 0, 3, 1012, 0, 0, 3892, 4071, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 66, 3845, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 1280, 17, 0, 0, 0, 0, 3550, 13, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 6, 4, 4076, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 2985, 5, 4, 2, 138, 3064, 476, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8164, 8292, 8263, 8993, 8006, 8956, 8178, 6684] avg=8192.0000 max=8993 max_rank=3 min=6684 min_rank=7 imbalance_ratio=1.097778 max_load_reduction=56543 max_load_reduction_pct=0.862778 moved_to_echo=31824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 12, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 0, 0, 3069, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 53, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 38, 0, 0, 0, 0, 16, 3560, 17, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 1116, 3082, 483, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8180, 8184, 8179, 8117, 7727, 8192, 8773] avg=8192.0000 max=8773 max_rank=7 min=7727 min_rank=5 imbalance_ratio=1.070923 max_load_reduction=56763 max_load_reduction_pct=0.866135 moved_to_echo=32809 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 9, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4070, 13, 0, 0, 0, 0, 0, 0, 5, 0, 1017, 0, 0, 3269, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 67, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 505, 0, 0, 0, 0, 8, 3553, 13, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 327, 3079, 476, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8179, 8181, 8387, 8248, 8175, 8192, 7978] avg=8192.0000 max=8387 max_rank=3 min=7978 min_rank=7 imbalance_ratio=1.023804 max_load_reduction=57149 max_load_reduction_pct=0.872025 moved_to_echo=32593 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 32768, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 640, 32256, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 8192, 0, 0, 24704, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 31616, 128, 128, 128, 5632, 24704, 3840, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 4224, 128, 0, 128, 0, 128, 28544, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 186, 4094, 4096, 4096, 0, 0, 0, 3827, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 42, 0, 216, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 4094, 7, 4089, 4096, 4096, 0, 0, 0, 4012, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 69, 0, 159, 0, 3941, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 57, 4091, 4096, 4096, 0, 0, 0, 4066, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 46, 0, 505, 0, 3518, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4079, 0, 58, 4, 3839, 3, 3, 0, 16, 1, 4, 0, 0, 4092, 460, 4004, 4036, 4085, 0, 0, 1, 4069, 0, 5, 4091, 0, 0, 4, 8, 4088, 4091, 0, 5, 0, 6, 3, 0, 0, 0, 0, 4091, 4068, 0, 4090, 363, 4, 2526, 71, 1056, 0, 6, 4, 1, 2, 0, 0, 3, 4092, 7, 4082, 0, 12, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 27, 4094, 4096, 4096, 0, 0, 0, 4007, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 12, 0, 208, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 58, 4080, 4096, 4096, 0, 0, 0, 3254, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 43, 0, 1079, 0, 3762, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4094, 51, 4087, 4096, 4096, 0, 0, 0, 3928, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 2, 4096, 42, 0, 459, 0, 3712, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 59, 4090, 4096, 4096, 0, 0, 0, 3801, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4096, 166, 0, 314, 0, 3848, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 4096, 0, 0, 21, 3, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4094, 7, 3227, 757, 0, 0, 0, 0, 0, 0, 4089, 245, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3318, 69, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 0, 0, 0, 0, 0, 4096, 624, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8094, 8346, 8192, 8192, 7643, 8661, 8191] avg=8192.0000 max=8661 max_rank=6 min=7643 min_rank=5 imbalance_ratio=1.057251 max_load_reduction=56875 max_load_reduction_pct=0.867844 moved_to_echo=32687 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 21, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 57, 3227, 757, 0, 0, 0, 0, 0, 0, 4091, 245, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3318, 46, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3518, 0, 0, 0, 0, 0, 0, 4096, 624, 1, 0, 0, 0, 0, 0, 4, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8137, 8402, 8192, 8191, 7966, 8239, 8193] avg=8192.0000 max=8402 max_rank=2 min=7966 min_rank=5 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32589 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 186, 3227, 757, 0, 0, 0, 0, 0, 0, 4094, 245, 0, 0, 0, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3318, 42, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 4096, 624, 1, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8266, 8166, 8192, 8191, 7672, 8644, 8189] avg=8192.0000 max=8644 max_rank=6 min=7672 min_rank=5 imbalance_ratio=1.055176 max_load_reduction=56892 max_load_reduction_pct=0.868103 moved_to_echo=32696 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 21, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 27, 3227, 757, 0, 0, 0, 0, 0, 0, 4094, 245, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3318, 12, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 0, 0, 0, 0, 4096, 624, 1, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8107, 8346, 8192, 8191, 7634, 8661, 8189] avg=8192.0000 max=8661 max_rank=6 min=7634 min_rank=5 imbalance_ratio=1.057251 max_load_reduction=56875 max_load_reduction_pct=0.867844 moved_to_echo=32675 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 21, 3, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4094, 51, 3227, 757, 0, 0, 0, 0, 0, 0, 4087, 245, 0, 0, 0, 0, 3928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 3318, 42, 0, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3712, 0, 0, 0, 0, 0, 0, 4096, 624, 1, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8134, 8260, 8192, 8192, 7917, 8433, 8192] avg=8192.0000 max=8433 max_rank=6 min=7917 min_rank=5 imbalance_ratio=1.029419 max_load_reduction=57103 max_load_reduction_pct=0.871323 moved_to_echo=32734 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4079, 0, 58, 4, 3839, 3, 3, 21, 3, 0, 0, 0, 0, 0, 0, 0, 16, 1, 4, 0, 0, 4092, 460, 3180, 755, 0, 0, 0, 0, 0, 0, 4004, 241, 0, 0, 0, 1, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 0, 0, 4, 8, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 3, 0, 0, 0, 4091, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 3314, 363, 4, 2526, 0, 0, 0, 0, 0, 0, 0, 0, 71, 1056, 0, 6, 4, 1, 2, 0, 4085, 615, 1, 0, 0, 0, 0, 0, 0, 3, 4089, 7, 4082, 0, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8010, 8508, 8315, 8196, 8195, 10275, 5841, 8196] avg=8192.0000 max=10275 max_rank=5 min=5841 min_rank=6 imbalance_ratio=1.254272 max_load_reduction=55261 max_load_reduction_pct=0.843216 moved_to_echo=32507 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 24, 9, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 58, 3234, 758, 0, 0, 0, 0, 0, 0, 4080, 231, 0, 0, 0, 0, 3254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3314, 43, 0, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 0, 0, 0, 0, 4096, 631, 0, 0, 0, 0, 0, 0, 8, 0, 4087, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8149, 7565, 8188, 8192, 8533, 8489, 8191] avg=8192.0000 max=8533 max_rank=5 min=7565 min_rank=2 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=33405 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 21, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 59, 3227, 757, 0, 0, 0, 0, 0, 0, 4090, 245, 0, 0, 0, 0, 3801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 3318, 166, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3848, 0, 0, 0, 0, 0, 0, 4096, 624, 0, 0, 0, 0, 0, 0, 6, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8141, 8136, 8192, 8192, 7896, 8568, 8195] avg=8192.0000 max=8568 max_rank=6 min=7896 min_rank=5 imbalance_ratio=1.045898 max_load_reduction=56968 max_load_reduction_pct=0.869263 moved_to_echo=32851 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 128, 128, 32512, 128, 128, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 26624, 896, 128, 5504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 27776, 0, 128, 128, 128, 128, 0, 32768, 4992, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32640, 2048, 0, 0, 0, 128, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 0, 0, 32768, 1024, 25856, 6144, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3801, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3679, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4087, 4096, 719, 0, 2, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3989, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4083, 4096, 523, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4, 0, 4, 326, 3, 1, 6, 0, 3786, 0, 5, 0, 0, 4052, 2, 0, 13, 0, 4, 46, 4, 4, 4043, 7, 3859, 5, 1, 4075, 1, 0, 4, 4092, 4067, 0, 0, 4066, 0, 3, 2, 39, 20, 4066, 4076, 294, 3, 0, 4088, 0, 0, 4089, 4074, 0, 6, 4092, 4, 0, 5, 4083, 4091, 0, 0, 19, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 3143, 0, 4, 0, 0, 0, 1, 0, 0, 4096, 0, 4012, 0, 2, 4096, 0, 0, 0, 4095, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 3806, 4093, 1320, 0, 3, 4096, 0, 8, 4095, 4096, 2, 0, 4095, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 3726, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 4001, 0, 1, 4096, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3752, 4091, 852, 0, 5, 4096, 0, 2, 4095, 4096, 2, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3019, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 3921, 0, 3, 4096, 0, 0, 0, 4096, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 4074, 4092, 1263, 0, 4, 4096, 0, 12, 4094, 4096, 4, 0, 4091, 0, 0, 0, 4094, 4096, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3602, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 3830, 0, 5, 4096, 0, 0, 0, 4096, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 3924, 4088, 896, 0, 7, 4096, 0, 27, 4096, 4096, 10, 0, 4082, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3685, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 3829, 0, 4, 4096, 0, 0, 0, 4096, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 4068, 4090, 690, 0, 5, 4096, 0, 15, 4096, 4096, 5, 0, 4088, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 3726, 0, 16, 544, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3889, 162, 41, 0, 0, 0, 0, 0, 4001, 0, 1, 4096, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3752, 3385, 852, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8150, 8323, 8188, 8304, 8195, 7994, 8190, 8192] avg=8192.0000 max=8323 max_rank=1 min=7994 min_rank=5 imbalance_ratio=1.015991 max_load_reduction=57213 max_load_reduction_pct=0.873001 moved_to_echo=32571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4055, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3801, 0, 0, 545, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3890, 162, 41, 0, 0, 0, 0, 0, 3679, 0, 0, 4096, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3389, 719, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8443, 8189, 7981, 8195, 8197, 8188, 8192] avg=8192.0000 max=8443 max_rank=1 min=7981 min_rank=3 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4055, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3695, 0, 0, 545, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3890, 162, 41, 0, 0, 0, 0, 0, 3989, 0, 0, 4096, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3389, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8337, 8189, 8291, 8195, 7995, 8186, 8192] avg=8192.0000 max=8337 max_rank=1 min=7995 min_rank=5 imbalance_ratio=1.017700 max_load_reduction=57199 max_load_reduction_pct=0.872787 moved_to_echo=32568 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4, 326, 3, 1, 6, 0, 4088, 4048, 0, 0, 0, 0, 0, 0, 3786, 0, 5, 0, 0, 4052, 2, 0, 542, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4, 46, 4, 4, 4043, 7, 3886, 161, 41, 0, 0, 0, 0, 0, 3859, 5, 1, 4075, 1, 0, 4, 206, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 4066, 0, 3, 2, 39, 3, 0, 0, 0, 0, 0, 0, 0, 20, 4066, 3373, 294, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 6, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4083, 4091, 0, 0, 19, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8480, 8387, 8209, 8151, 8180, 7756, 8173, 8200] avg=8192.0000 max=8480 max_rank=0 min=7756 min_rank=5 imbalance_ratio=1.035156 max_load_reduction=57056 max_load_reduction_pct=0.870605 moved_to_echo=32309 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4055, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 3143, 0, 4, 544, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 3889, 162, 40, 0, 0, 0, 0, 0, 4012, 0, 2, 4096, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3806, 3387, 1320, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4092, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 7784, 8188, 8316, 8193, 8516, 8197, 8191] avg=8192.0000 max=8516 max_rank=5 min=7784 min_rank=1 imbalance_ratio=1.039551 max_load_reduction=57020 max_load_reduction_pct=0.870056 moved_to_echo=33097 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4053, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3019, 0, 7, 544, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3890, 162, 41, 0, 0, 0, 0, 0, 3921, 0, 3, 4096, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 3386, 1263, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4092, 4, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8149, 7666, 8189, 8226, 8189, 8727, 8199, 8191] avg=8192.0000 max=8727 max_rank=5 min=7666 min_rank=1 imbalance_ratio=1.065308 max_load_reduction=56809 max_load_reduction_pct=0.866837 moved_to_echo=33306 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4056, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3602, 0, 16, 546, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3895, 165, 40, 0, 0, 0, 0, 0, 3830, 0, 5, 4096, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 10, 3, 0, 0, 0, 0, 0, 0, 0, 3924, 3377, 896, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4084, 10, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8262, 8196, 8132, 8198, 8204, 8203, 8189] avg=8192.0000 max=8262 max_rank=1 min=8132 min_rank=3 imbalance_ratio=1.008545 max_load_reduction=57274 max_load_reduction_pct=0.873932 moved_to_echo=32794 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4056, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3685, 0, 9, 544, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3890, 162, 40, 0, 0, 0, 0, 0, 3829, 0, 4, 4096, 0, 0, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 3384, 690, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4092, 5, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8334, 8188, 8135, 8191, 8147, 8200, 8189] avg=8192.0000 max=8334 max_rank=1 min=8135 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=57202 max_load_reduction_pct=0.872833 moved_to_echo=32727 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32512, 0, 128, 0, 0, 28800, 128, 128, 4480, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 384, 128, 128, 128, 0, 32768, 32512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 128, 31232, 1408, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 0, 32768, 0, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 31872, 27136, 6656, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31232, 128, 128, 32768, 128, 0, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 335, 4093, 0, 0, 0, 0, 5, 4096, 0, 4096, 4096, 0, 0, 4063, 0, 0, 0, 1084, 0, 2703, 66, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4096, 4095, 4090, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 1146, 4089, 0, 0, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 4087, 0, 0, 0, 510, 0, 2325, 141, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 4092, 4089, 4054, 0, 3, 4, 0, 4092, 6, 0, 0, 4, 4091, 0, 338, 0, 0, 4090, 4092, 4092, 0, 4, 0, 33, 0, 4, 277, 4, 0, 1, 4, 9, 11, 1507, 3, 5, 570, 3211, 3, 4, 4, 1, 0, 4092, 4, 4089, 4085, 0, 4, 4092, 5, 0, 4, 3077, 0, 1624, 1670, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 1492, 4089, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 652, 0, 1865, 156, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 3038, 4072, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 53, 0, 1040, 65, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 4096, 4095, 4085, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 1063, 4088, 0, 0, 0, 0, 5, 4096, 0, 4096, 4096, 0, 0, 4080, 0, 0, 0, 234, 0, 2560, 292, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 53, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 798, 4089, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4060, 0, 0, 0, 1154, 0, 1934, 328, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 4096, 0, 0, 0, 0, 4096, 4095, 4083, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 2552, 4093, 0, 0, 0, 0, 2, 4096, 0, 4096, 4096, 0, 0, 4067, 0, 0, 0, 241, 0, 1123, 248, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 2783, 1259, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4096, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 3185, 1297, 0, 0, 0, 0, 0, 0, 335, 4093, 0, 0, 0, 0, 5, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 0, 2703, 66, 1, 0, 0, 4096, 902, 51, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8200, 8151, 8162, 8476, 7270, 8175, 8903] avg=8192.0000 max=8903 max_rank=7 min=7270 min_rank=5 imbalance_ratio=1.086792 max_load_reduction=56633 max_load_reduction_pct=0.864151 moved_to_echo=32824 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 2783, 1259, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 3185, 1297, 0, 0, 0, 0, 0, 0, 1146, 4089, 0, 0, 0, 0, 1, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 510, 0, 2325, 141, 1, 0, 0, 4096, 902, 50, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8183, 8149, 8148, 8559, 8073, 8199, 8025] avg=8192.0000 max=8559 max_rank=4 min=8025 min_rank=7 imbalance_ratio=1.044800 max_load_reduction=56977 max_load_reduction_pct=0.869400 moved_to_echo=32856 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 0, 0, 8, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 4092, 6, 0, 0, 4, 2778, 1257, 36, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 4056, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 33, 0, 4, 277, 4, 4091, 4040, 0, 0, 0, 0, 0, 0, 0, 1, 4, 9, 11, 1507, 3, 5, 3183, 1294, 0, 0, 0, 0, 0, 0, 570, 3211, 3, 4, 4, 1, 0, 2835, 0, 0, 0, 0, 0, 0, 0, 0, 4, 17, 4085, 0, 4, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3077, 0, 1624, 1670, 0, 3, 0, 4089, 901, 50, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8180, 8486, 8453, 6017, 6628, 8207, 11418] avg=8192.0000 max=11418 max_rank=7 min=6017 min_rank=4 imbalance_ratio=1.393799 max_load_reduction=54118 max_load_reduction_pct=0.825775 moved_to_echo=32270 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 2783, 1259, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 3186, 1297, 0, 0, 0, 0, 0, 0, 1492, 4089, 0, 0, 0, 0, 0, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 1865, 156, 0, 0, 0, 4096, 902, 50, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8176, 8156, 8143, 8516, 8418, 8206, 7721] avg=8192.0000 max=8516 max_rank=4 min=7721 min_rank=7 imbalance_ratio=1.039551 max_load_reduction=57020 max_load_reduction_pct=0.870056 moved_to_echo=32861 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 2783, 1259, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 3183, 1297, 0, 0, 0, 0, 0, 0, 1063, 4088, 0, 0, 0, 0, 5, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 2560, 292, 1, 0, 0, 4096, 901, 50, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8190, 8144, 8156, 8527, 7993, 8192, 8134] avg=8192.0000 max=8527 max_rank=4 min=7993 min_rank=5 imbalance_ratio=1.040894 max_load_reduction=57009 max_load_reduction_pct=0.869888 moved_to_echo=32846 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2783, 1259, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 3186, 1297, 0, 0, 0, 0, 0, 0, 3038, 4072, 0, 0, 0, 0, 0, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 1040, 65, 0, 0, 0, 4096, 902, 50, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8174, 8156, 8143, 8502, 9947, 8208, 6206] avg=8192.0000 max=9947 max_rank=5 min=6206 min_rank=7 imbalance_ratio=1.214233 max_load_reduction=55589 max_load_reduction_pct=0.848221 moved_to_echo=32863 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 53, 0, 2784, 1260, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 3191, 1298, 0, 0, 0, 0, 0, 0, 798, 4089, 0, 0, 0, 0, 0, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 0, 1934, 328, 0, 0, 0, 4096, 905, 50, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8233, 8152, 8176, 8423, 7723, 8170, 8467] avg=8192.0000 max=8467 max_rank=7 min=7723 min_rank=5 imbalance_ratio=1.033569 max_load_reduction=57069 max_load_reduction_pct=0.870804 moved_to_echo=32783 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 2783, 1259, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 3186, 1297, 0, 0, 0, 0, 0, 0, 2552, 4093, 0, 0, 0, 0, 2, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 1123, 248, 0, 0, 0, 4096, 904, 50, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8200, 8142, 8155, 8516, 9484, 8179, 6662] avg=8192.0000 max=9484 max_rank=5 min=6662 min_rank=7 imbalance_ratio=1.157715 max_load_reduction=56052 max_load_reduction_pct=0.855286 moved_to_echo=32841 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11008, 31872, 128, 128, 128, 128, 128, 22784, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 256, 128, 22272, 10112, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 7040, 0, 15232, 3072, 128, 128, 0, 32768, 7296, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 384, 0, 0, 0, 32512, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 128, 128, 384, 128, 32768, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 29696, 128, 128, 25600, 10496, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4085, 67, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 1, 4095, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2, 4094, 4096, 0, 4096, 4084, 0, 0, 4096, 0, 4093, 0, 4, 0, 0, 4019, 17, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4065, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 28, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4087, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4088, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 2, 4096, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4059, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4047, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 35, 0, 0, 0, 1, 4095, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 1, 4095, 4096, 0, 4096, 4059, 0, 0, 4096, 0, 4092, 0, 4, 0, 0, 3935, 62, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4072, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 0, 2, 4094, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 5, 4093, 4095, 0, 4096, 4088, 0, 0, 4096, 0, 4090, 0, 6, 0, 0, 4093, 13, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[3855, 680, 5, 0, 0, 3, 0, 0, 5, 5, 5, 4, 4, 0, 4085, 94, 0, 0, 4, 0, 0, 4092, 580, 3873, 0, 0, 4092, 6, 3, 0, 2, 0, 4092, 4089, 0, 4092, 4092, 5, 113, 4087, 181, 4092, 0, 0, 0, 7, 3004, 0, 4, 4092, 1, 0, 4, 0, 4086, 0, 4, 4, 7, 0, 3, 3984, 4091, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4076, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 12, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4075, 0, 0, 4096, 0, 4087, 0, 8, 0, 0, 4078, 29, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4055, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 6, 4092, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 6, 4090, 4096, 0, 4096, 4083, 0, 0, 4096, 0, 4090, 0, 4, 0, 0, 4093, 36, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4085, 67, 0, 0, 0, 0, 0, 1, 4015, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 2, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 1, 4034, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4, 0, 0, 4019, 17, 81, 50, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8179, 8146, 8192, 8178, 8264, 8192, 8202] avg=8192.0000 max=8264 max_rank=5 min=8146 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=57272 max_load_reduction_pct=0.873901 moved_to_echo=32836 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4047, 151, 0, 0, 0, 0, 0, 0, 4015, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 1, 4034, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4, 0, 0, 3935, 62, 81, 50, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8180, 8165, 8191, 8154, 8224, 8192, 8203] avg=8192.0000 max=8227 max_rank=0 min=8154 min_rank=4 imbalance_ratio=1.004272 max_load_reduction=57309 max_load_reduction_pct=0.874466 moved_to_echo=32773 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4065, 5, 0, 0, 0, 0, 0, 0, 4015, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 1, 4035, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 6, 81, 50, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 8180, 8160, 8190, 8192, 8320, 8192, 8203] avg=8192.0000 max=8320 max_rank=5 min=8099 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32907 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4072, 3, 0, 0, 0, 0, 0, 1, 4015, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 2, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 2, 4033, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 4093, 13, 81, 50, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 8179, 8148, 8195, 8181, 8333, 8192, 8203] avg=8192.0000 max=8333 max_rank=5 min=8105 min_rank=0 imbalance_ratio=1.017212 max_load_reduction=57203 max_load_reduction_pct=0.872849 moved_to_echo=32909 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4088, 30, 0, 0, 0, 0, 0, 0, 4015, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 2, 4035, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4059, 2, 81, 50, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8147, 8180, 8145, 8190, 8191, 8288, 8192, 8203] avg=8192.0000 max=8288 max_rank=5 min=8145 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32874 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3855, 680, 5, 0, 0, 3, 0, 0, 4006, 14, 0, 0, 0, 0, 0, 0, 5, 5, 5, 4, 4, 0, 4085, 94, 4075, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4031, 580, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 6, 3, 0, 2, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4092, 5, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 4092, 0, 0, 0, 7, 3004, 0, 81, 50, 0, 0, 0, 0, 0, 0, 4, 4092, 1, 0, 4, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 7, 0, 3, 3984, 4091, 5, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8563, 8279, 8488, 8193, 8302, 7415, 8187, 8109] avg=8192.0000 max=8563 max_rank=0 min=7415 min_rank=5 imbalance_ratio=1.045288 max_load_reduction=56973 max_load_reduction_pct=0.869339 moved_to_echo=32013 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4076, 17, 0, 0, 0, 0, 0, 0, 4016, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 12, 4022, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 8, 0, 0, 4078, 29, 80, 57, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8124, 8180, 8132, 8189, 8165, 8339, 8197, 8210] avg=8192.0000 max=8339 max_rank=5 min=8124 min_rank=0 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=32911 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4055, 2, 0, 0, 0, 0, 0, 0, 4016, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 2, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 6, 4031, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4, 0, 0, 4093, 36, 80, 50, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8088, 8179, 8152, 8196, 8173, 8353, 8192, 8203] avg=8192.0000 max=8353 max_rank=5 min=8088 min_rank=0 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32921 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32384, 1024, 128, 0, 0, 128, 0, 128, 32128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 32768, 0, 128, 0, 128, 31488, 256, 768, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 32768, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 128, 32256, 640, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4082, 0, 0, 0, 0, 74, 0, 4094, 0, 4095, 4051, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4083, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4077, 0, 0, 0, 0, 188, 0, 4096, 0, 4096, 3928, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 168, 0, 4095, 0, 4087, 4041, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4002, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 0, 231, 0, 4096, 0, 4092, 3879, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4091, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 3892, 0, 0, 0, 0, 383, 0, 4095, 0, 4091, 3924, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[8, 4090, 4, 0, 3, 136, 1548, 0, 4092, 4, 4090, 2265, 8, 0, 0, 0, 4091, 0, 35, 4040, 3, 4088, 0, 7, 5, 1, 18, 0, 0, 4033, 0, 4087, 0, 4, 3, 4092, 0, 195, 0, 0, 6, 0, 5, 0, 0, 485, 5, 4, 10, 5, 4077, 0, 0, 6, 4091, 3631, 0, 4071, 3, 4087, 4086, 0, 9, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 0, 0, 70, 0, 4095, 0, 4091, 4066, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4084, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 164, 0, 4096, 0, 4092, 3964, 0, 0, 1, 0, 4089, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4075, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 168, 0, 3755, 1, 0, 0, 0, 0, 0, 0, 335, 0, 4087, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4017, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 78, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8019, 8463, 8188, 8198, 8167, 8130, 8176, 8195] avg=8192.0000 max=8463 max_rank=1 min=8019 min_rank=0 imbalance_ratio=1.033081 max_load_reduction=57073 max_load_reduction_pct=0.870865 moved_to_echo=32668 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 0, 0, 0, 0, 74, 0, 3754, 1, 0, 0, 0, 0, 0, 0, 335, 0, 4095, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4017, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 78, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7911, 8481, 8196, 8199, 8167, 8130, 8257, 8195] avg=8192.0000 max=8481 max_rank=1 min=7911 min_rank=0 imbalance_ratio=1.035278 max_load_reduction=57055 max_load_reduction_pct=0.870590 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 0, 231, 0, 3756, 1, 0, 0, 0, 0, 0, 0, 335, 0, 4092, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4017, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 78, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8079, 8306, 8198, 8197, 8166, 8130, 8265, 8195] avg=8192.0000 max=8306 max_rank=1 min=8079 min_rank=0 imbalance_ratio=1.013916 max_load_reduction=57230 max_load_reduction_pct=0.873260 moved_to_echo=32756 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 3892, 0, 0, 0, 0, 383, 0, 3755, 1, 0, 0, 0, 0, 0, 0, 335, 0, 4091, 3924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4017, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 78, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8031, 8350, 8198, 8197, 8168, 8130, 8267, 8195] avg=8192.0000 max=8350 max_rank=1 min=8031 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4077, 0, 0, 0, 0, 188, 0, 3756, 1, 0, 0, 0, 0, 0, 0, 335, 0, 4096, 3928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4017, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 78, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8022, 8359, 8198, 8197, 8166, 8130, 8269, 8195] avg=8192.0000 max=8359 max_rank=1 min=8022 min_rank=0 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32760 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[8, 4090, 4, 0, 3, 136, 1548, 0, 3752, 1, 0, 0, 0, 0, 0, 0, 335, 4, 4090, 2265, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 35, 4040, 3, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 18, 0, 0, 4033, 0, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 4092, 0, 195, 0, 0, 4012, 53, 0, 0, 0, 0, 0, 0, 6, 0, 5, 0, 0, 485, 5, 4, 4084, 4029, 0, 0, 0, 0, 0, 0, 10, 5, 4077, 0, 0, 6, 78, 3631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 3, 3, 4086, 0, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9542, 6702, 8182, 8149, 8359, 8618, 7807, 8177] avg=8192.0000 max=9542 max_rank=0 min=6702 min_rank=1 imbalance_ratio=1.164795 max_load_reduction=55994 max_load_reduction_pct=0.854401 moved_to_echo=32961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 0, 0, 0, 0, 70, 0, 3759, 7, 0, 0, 0, 0, 0, 0, 326, 0, 4091, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4023, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 66, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7916, 8483, 8186, 8202, 8177, 8134, 8246, 8192] avg=8192.0000 max=8483 max_rank=1 min=7916 min_rank=0 imbalance_ratio=1.035522 max_load_reduction=57053 max_load_reduction_pct=0.870560 moved_to_echo=32749 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 164, 0, 3756, 1, 0, 0, 0, 0, 0, 0, 335, 0, 4092, 3964, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4017, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 78, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8016, 8392, 8190, 8197, 8166, 8134, 8249, 8192] avg=8192.0000 max=8392 max_rank=1 min=8016 min_rank=0 imbalance_ratio=1.024414 max_load_reduction=57144 max_load_reduction_pct=0.871948 moved_to_echo=32741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 640, 32256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32768, 128, 256, 0, 0, 32256, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32512, 128, 0, 128, 256, 2944, 0, 30080, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 0, 512, 128, 128, 32768, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2688, 128, 32768, 30208, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2511, 57, 243, 125, 1108, 0, 1994, 125, 745, 2956, 296, 355, 1150, 1689, 0, 255, 411, 988, 285, 1501, 0, 1204, 1927, 1443, 46, 85, 2987, 1012, 2673, 200, 2526, 931, 10, 1276, 2867, 148, 255, 271, 2095, 1159, 1226, 0, 2320, 1910, 0, 1442, 1044, 1005, 2767, 61, 1903, 1, 18, 567, 1, 627, 0, 131, 1182, 1518, 2717, 1714, 865, 2608] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2256, 13, 389, 117, 347, 0, 2057, 14, 777, 3024, 228, 659, 1986, 2180, 0, 194, 258, 1538, 231, 1244, 0, 617, 1409, 1761, 5, 16, 2576, 1289, 2871, 227, 2614, 724, 14, 1962, 2999, 202, 475, 267, 1800, 1748, 1727, 0, 2006, 1536, 0, 1107, 1349, 1080, 3217, 27, 1372, 0, 18, 810, 0, 557, 0, 181, 801, 1356, 2832, 1135, 943, 2394] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2347, 50, 270, 207, 447, 0, 1677, 18, 729, 3424, 395, 495, 1025, 2781, 0, 321, 593, 1693, 319, 1593, 1, 818, 846, 1546, 18, 187, 2123, 986, 3297, 502, 2718, 669, 5, 1110, 3085, 269, 541, 307, 2214, 1916, 1060, 0, 1878, 2589, 1, 1309, 1104, 1108, 3125, 26, 1516, 0, 5, 625, 0, 525, 6, 126, 619, 1006, 2739, 1245, 1189, 2193] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3043, 48, 342, 173, 682, 0, 2179, 72, 865, 3434, 180, 647, 1370, 1417, 0, 245, 514, 1061, 245, 1773, 2, 742, 1395, 1327, 14, 26, 2914, 1351, 3108, 122, 2559, 671, 14, 1917, 2378, 158, 334, 152, 2407, 1187, 744, 2, 2212, 1576, 0, 1365, 894, 757, 3187, 9, 1890, 2, 22, 511, 15, 1140, 0, 201, 962, 1448, 2898, 1117, 875, 2641] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2079, 11, 443, 172, 858, 1, 1855, 98, 891, 2441, 254, 421, 1502, 1384, 0, 247, 438, 1368, 272, 1580, 0, 771, 1656, 1556, 68, 4, 2430, 874, 3062, 280, 2696, 589, 11, 1512, 2545, 163, 445, 205, 2492, 1375, 1629, 0, 2515, 1729, 0, 1425, 1465, 1604, 3298, 40, 1494, 0, 14, 646, 2, 507, 1, 264, 1204, 1456, 2388, 919, 1058, 2829] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2348, 70, 703, 61, 1021, 3, 1695, 24, 735, 2342, 326, 621, 1807, 1571, 0, 363, 576, 1464, 355, 1970, 0, 837, 1217, 1775, 18, 39, 2770, 854, 2518, 74, 2679, 641, 7, 1509, 2819, 114, 672, 229, 2383, 1419, 1683, 0, 1997, 1902, 0, 1086, 1328, 885, 3273, 7, 1401, 2, 10, 426, 4, 495, 0, 194, 1403, 1192, 2635, 1080, 902, 3002] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2545, 12, 236, 178, 609, 0, 2572, 30, 603, 3469, 278, 392, 1128, 2169, 2, 218, 329, 1080, 307, 1267, 0, 734, 1396, 1486, 19, 18, 2942, 1187, 3373, 264, 2681, 725, 3, 1541, 2589, 257, 412, 143, 2191, 1573, 985, 0, 2382, 1892, 0, 1601, 1316, 1368, 3232, 15, 1856, 2, 7, 417, 1, 308, 1, 124, 664, 1215, 2453, 917, 973, 2849] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[3189, 17, 371, 133, 535, 0, 1978, 151, 1008, 2997, 194, 254, 1551, 777, 0, 124, 234, 1197, 156, 1901, 0, 589, 2135, 1446, 60, 48, 3142, 1390, 3140, 163, 2641, 717, 4, 1803, 2352, 555, 280, 210, 2187, 1190, 1665, 0, 2398, 1594, 0, 787, 879, 827, 3100, 57, 1985, 7, 15, 272, 1, 479, 0, 345, 1291, 1028, 3018, 762, 1011, 3196] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3043, 48, 342, 173, 682, 0, 2179, 72, 2406, 1, 0, 0, 0, 0, 0, 0, 865, 3434, 180, 647, 1370, 1417, 0, 245, 180, 0, 0, 0, 0, 0, 0, 0, 514, 1061, 245, 1773, 2, 742, 1395, 1327, 390, 325, 0, 0, 0, 0, 0, 0, 14, 26, 2914, 1351, 702, 122, 2559, 671, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1917, 1808, 158, 334, 152, 2407, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 744, 2, 1494, 1576, 0, 1365, 894, 757, 0, 0, 0, 0, 0, 0, 0, 0, 3187, 9, 1890, 2, 22, 511, 15, 1140, 1760, 393, 0, 0, 0, 0, 0, 0, 0, 201, 962, 1448, 2898, 1117, 875, 880, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8946, 8338, 7774, 8359, 7977, 6832, 8929, 8381] avg=8192.0000 max=8946 max_rank=0 min=6832 min_rank=5 imbalance_ratio=1.092041 max_load_reduction=56590 max_load_reduction_pct=0.863495 moved_to_echo=32119 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2347, 50, 270, 207, 447, 0, 1677, 18, 2553, 1, 0, 0, 0, 0, 0, 0, 729, 3424, 395, 495, 1025, 2781, 0, 321, 234, 0, 0, 0, 0, 0, 0, 0, 593, 1693, 319, 1593, 1, 818, 846, 1546, 506, 276, 0, 0, 0, 0, 0, 0, 18, 187, 2123, 986, 744, 502, 2718, 669, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1110, 2345, 269, 541, 307, 2214, 1916, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 1269, 2589, 1, 1309, 1104, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 3125, 26, 1516, 0, 5, 625, 0, 525, 1461, 333, 0, 0, 0, 0, 0, 0, 6, 126, 619, 1006, 2739, 1245, 1189, 731, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7570, 9404, 8191, 7947, 8707, 8440, 7616, 7661] avg=8192.0000 max=9404 max_rank=1 min=7570 min_rank=0 imbalance_ratio=1.147949 max_load_reduction=56132 max_load_reduction_pct=0.856506 moved_to_echo=32424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2256, 13, 389, 117, 347, 0, 2057, 14, 2223, 1, 0, 0, 0, 0, 0, 0, 777, 3024, 228, 659, 1986, 2180, 0, 194, 228, 0, 0, 0, 0, 0, 0, 0, 258, 1538, 231, 1244, 0, 617, 1409, 1761, 492, 295, 0, 0, 0, 0, 0, 0, 5, 16, 2576, 1289, 648, 227, 2614, 724, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1962, 2279, 202, 475, 267, 1800, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 1727, 0, 1355, 1536, 0, 1107, 1349, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 27, 1372, 0, 18, 810, 0, 557, 1595, 356, 0, 0, 0, 0, 0, 0, 0, 181, 801, 1356, 2832, 1135, 943, 798, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7417, 9276, 7845, 8099, 8747, 8154, 7952, 8046] avg=8192.0000 max=9276 max_rank=1 min=7417 min_rank=0 imbalance_ratio=1.132324 max_load_reduction=56260 max_load_reduction_pct=0.858459 moved_to_echo=32899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2511, 57, 243, 125, 1108, 0, 1994, 125, 2069, 1, 0, 0, 0, 0, 0, 0, 745, 2956, 296, 355, 1150, 1689, 0, 255, 218, 0, 0, 0, 0, 0, 0, 0, 411, 988, 285, 1501, 0, 1204, 1927, 1443, 470, 341, 0, 0, 0, 0, 0, 0, 46, 85, 2987, 1012, 604, 200, 2526, 931, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1276, 2179, 148, 255, 271, 2095, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 1226, 0, 1567, 1910, 0, 1442, 1044, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 2767, 61, 1903, 1, 18, 567, 1, 627, 1738, 412, 0, 0, 0, 0, 0, 0, 0, 131, 1182, 1518, 2717, 1714, 865, 869, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8233, 7664, 8570, 8391, 7393, 8194, 8095, 8996] avg=8192.0000 max=8996 max_rank=7 min=7393 min_rank=4 imbalance_ratio=1.098145 max_load_reduction=56540 max_load_reduction_pct=0.862732 moved_to_echo=32678 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2079, 11, 443, 172, 858, 1, 1855, 98, 2371, 1, 0, 0, 0, 0, 0, 0, 891, 2441, 254, 421, 1502, 1384, 0, 247, 193, 0, 0, 0, 0, 0, 0, 0, 438, 1368, 272, 1580, 0, 771, 1656, 1556, 417, 370, 0, 0, 0, 0, 0, 0, 68, 4, 2430, 874, 691, 280, 2696, 589, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1512, 1935, 163, 445, 205, 2492, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 1629, 0, 1698, 1729, 0, 1425, 1465, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 3298, 40, 1494, 0, 14, 646, 2, 507, 1885, 447, 0, 0, 0, 0, 0, 0, 1, 264, 1204, 1456, 2388, 919, 1058, 943, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7889, 7333, 8428, 7632, 8138, 9550, 8333, 8233] avg=8192.0000 max=9550 max_rank=5 min=7333 min_rank=1 imbalance_ratio=1.165771 max_load_reduction=55986 max_load_reduction_pct=0.854279 moved_to_echo=34254 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2348, 70, 703, 61, 1021, 3, 1695, 24, 1953, 3, 0, 0, 0, 0, 0, 0, 735, 2342, 326, 621, 1807, 1571, 0, 363, 218, 0, 0, 0, 0, 0, 0, 0, 576, 1464, 355, 1970, 0, 837, 1217, 1775, 466, 298, 0, 0, 0, 0, 0, 0, 18, 39, 2770, 854, 565, 74, 2679, 641, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1509, 2135, 114, 672, 229, 2383, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 1683, 0, 1340, 1902, 0, 1086, 1328, 885, 0, 0, 0, 0, 0, 0, 0, 0, 3273, 7, 1401, 2, 10, 426, 4, 495, 2004, 359, 0, 0, 0, 0, 0, 0, 0, 194, 1403, 1192, 2635, 1080, 902, 995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7881, 7983, 8958, 7640, 8468, 8224, 7981, 8401] avg=8192.0000 max=8958 max_rank=2 min=7640 min_rank=3 imbalance_ratio=1.093506 max_load_reduction=56578 max_load_reduction_pct=0.863312 moved_to_echo=33074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2545, 12, 236, 178, 609, 0, 2572, 30, 2611, 1, 0, 0, 0, 0, 0, 0, 603, 3469, 278, 392, 1128, 2169, 2, 218, 196, 0, 0, 0, 0, 0, 0, 0, 329, 1080, 307, 1267, 0, 734, 1396, 1486, 424, 350, 0, 0, 0, 0, 0, 0, 19, 18, 2942, 1187, 762, 264, 2681, 725, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1541, 1969, 257, 412, 143, 2191, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 985, 0, 1609, 1892, 0, 1601, 1316, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 3232, 15, 1856, 2, 7, 417, 1, 308, 1898, 423, 0, 0, 0, 0, 0, 0, 1, 124, 664, 1215, 2453, 917, 973, 950, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8794, 8455, 7373, 8598, 8089, 8771, 8159, 7297] avg=8192.0000 max=8794 max_rank=0 min=7297 min_rank=7 imbalance_ratio=1.073486 max_load_reduction=56742 max_load_reduction_pct=0.865814 moved_to_echo=32316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3189, 17, 371, 133, 535, 0, 1978, 151, 2431, 1, 0, 0, 0, 0, 0, 0, 1008, 2997, 194, 254, 1551, 777, 0, 124, 178, 0, 0, 0, 0, 0, 0, 0, 234, 1197, 156, 1901, 0, 589, 2135, 1446, 386, 353, 0, 0, 0, 0, 0, 0, 60, 48, 3142, 1390, 709, 163, 2641, 717, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1803, 1788, 555, 280, 210, 2187, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 1665, 0, 1619, 1594, 0, 787, 879, 827, 0, 0, 0, 0, 0, 0, 0, 0, 3100, 57, 1985, 7, 15, 272, 1, 479, 2129, 426, 0, 0, 0, 0, 0, 0, 0, 345, 1291, 1028, 3018, 762, 1011, 1066, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8806, 7083, 8397, 8870, 8017, 7371, 8471, 8521] avg=8192.0000 max=8870 max_rank=3 min=7083 min_rank=1 imbalance_ratio=1.082764 max_load_reduction=56666 max_load_reduction_pct=0.864655 moved_to_echo=32380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[10752, 128, 12032, 14848, 128, 10240, 9472, 8704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 1664, 8192, 10240, 21760, 8960, 7936, 7296, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 12672, 16512, 1920, 3456, 1792, 17792, 11648, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 512, 21888, 8960, 5504, 1920, 21120, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[20352, 384, 3072, 1280, 5632, 128, 16128, 640, 18688, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[25216, 256, 13440, 128, 128, 4352, 128, 4736, 14592, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[6400, 24192, 2176, 3968, 11520, 14080, 128, 2048, 1664, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3456, 10496, 2176, 12928, 128, 6400, 12032, 12416, 3584, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 3984, 0, 4095, 4096, 0, 801, 3827, 0, 0, 1705, 4094, 0, 0, 0, 0, 0, 114, 281, 4007, 3799, 0, 3779, 0, 309, 2, 591, 7, 0, 2, 0, 490, 0, 0, 2545, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 29, 4096, 0, 1, 0, 3910, 4096, 257, 0, 0, 0, 0, 2332] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 0, 4077, 0, 3960, 4094, 0, 1419, 3869, 0, 0, 1944, 4095, 0, 0, 0, 0, 0, 0, 522, 3985, 3494, 1, 3905, 2, 330, 1, 230, 13, 0, 0, 0, 72, 1, 3, 2911, 4096, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 6, 0, 4017, 4096, 346, 0, 0, 0, 0, 1749] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4094, 0, 4088, 4096, 0, 1710, 4008, 0, 0, 2840, 4094, 0, 0, 0, 0, 0, 0, 24, 2653, 4056, 0, 4015, 1, 221, 0, 74, 0, 0, 1, 0, 4, 0, 0, 1877, 4096, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 24, 4095, 0, 0, 0, 4027, 4096, 657, 0, 0, 0, 0, 2488] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 4036, 0, 4059, 4096, 0, 366, 3997, 0, 1, 843, 4096, 0, 0, 0, 0, 0, 0, 149, 4020, 3977, 6, 4080, 0, 204, 0, 1400, 2, 1, 4, 0, 402, 2, 0, 3330, 4093, 11, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 258, 4096, 0, 1, 0, 3774, 4093, 161, 0, 0, 0, 0, 1783] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 0, 4046, 0, 4035, 4096, 0, 1051, 4033, 0, 1, 2119, 4088, 0, 1, 0, 1, 1, 1, 46, 3497, 4050, 4, 4022, 0, 75, 1, 501, 0, 0, 2, 0, 167, 0, 0, 3229, 4096, 5, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 103, 4095, 0, 0, 0, 3994, 4094, 379, 0, 0, 0, 0, 1511] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 4096, 1, 4083, 3, 4079, 4092, 0, 1648, 3912, 0, 0, 2650, 4089, 0, 0, 0, 0, 0, 3, 289, 3745, 3403, 0, 3999, 2, 372, 4, 153, 0, 5, 0, 0, 30, 2, 0, 2603, 4093, 2, 0, 0, 4, 0, 0, 4096, 2, 0, 0, 11, 4093, 0, 0, 0, 3914, 4093, 414, 2, 0, 1, 0, 1544] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 4089, 0, 4093, 4093, 0, 488, 4068, 0, 0, 1479, 4096, 0, 0, 0, 0, 0, 7, 349, 3916, 3945, 4, 4082, 0, 48, 0, 363, 0, 0, 1, 0, 100, 0, 0, 2846, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 80, 4095, 0, 2, 0, 4096, 4096, 613, 0, 0, 0, 0, 2201] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4091, 7, 4077, 0, 4091, 4074, 0, 621, 4094, 0, 5, 2014, 4093, 4, 0, 6, 4, 0, 17, 69, 3787, 4063, 0, 4077, 0, 111, 0, 857, 16, 0, 0, 5, 352, 0, 2, 2367, 4091, 4, 0, 0, 0, 0, 3, 4091, 0, 0, 2, 41, 4090, 4, 19, 5, 4038, 4091, 400, 0, 5, 0, 4, 1742] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 0, 4077, 0, 27, 3, 0, 0, 0, 0, 0, 0, 1222, 0, 0, 1419, 3869, 0, 0, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 522, 2198, 1564, 54, 0, 0, 0, 0, 0, 3985, 3413, 1, 0, 2, 330, 1, 230, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 72, 1, 3, 2911, 4094, 1149, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 6, 0, 2756, 1174, 0, 0, 0, 0, 0, 0, 4017, 1898, 346, 0, 0, 0, 0, 1749, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8454, 8433, 7962, 8243, 8196, 8034, 8010] avg=8192.0000 max=8454 max_rank=1 min=7962 min_rank=3 imbalance_ratio=1.031982 max_load_reduction=57082 max_load_reduction_pct=0.871002 moved_to_echo=32483 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 4094, 0, 32, 3, 0, 0, 0, 0, 0, 0, 1261, 0, 0, 1710, 4008, 0, 0, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 24, 2198, 1615, 63, 0, 0, 0, 0, 0, 2653, 3961, 0, 0, 1, 221, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 1877, 4096, 1181, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4095, 0, 0, 0, 2834, 1212, 0, 0, 0, 0, 0, 0, 4027, 1898, 657, 0, 0, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 9819, 7994, 6910, 7159, 8194, 8165, 9070] avg=8192.0000 max=9819 max_rank=1 min=6910 min_rank=3 imbalance_ratio=1.198608 max_load_reduction=55717 max_load_reduction_pct=0.850174 moved_to_echo=32588 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 3984, 0, 30, 3, 0, 0, 0, 0, 0, 0, 1264, 0, 0, 801, 3827, 0, 0, 1705, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 114, 281, 2198, 1617, 59, 0, 0, 0, 0, 0, 4007, 3710, 0, 0, 0, 309, 2, 591, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2, 0, 490, 0, 0, 2545, 4096, 1113, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 1, 0, 2666, 1214, 0, 0, 0, 0, 0, 0, 3910, 1898, 257, 0, 0, 0, 0, 2332, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 7597, 8363, 8619, 8253, 8187, 8006, 8397] avg=8192.0000 max=8619 max_rank=3 min=7597 min_rank=1 imbalance_ratio=1.052124 max_load_reduction=56917 max_load_reduction_pct=0.868484 moved_to_echo=32843 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 0, 4046, 0, 32, 2, 0, 0, 0, 0, 0, 0, 1245, 0, 0, 1051, 4033, 0, 1, 2119, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 1, 0, 1, 1, 1, 46, 2197, 1594, 63, 0, 0, 0, 0, 0, 3497, 3955, 4, 0, 0, 75, 1, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 167, 0, 0, 3229, 4096, 1184, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 4095, 0, 0, 0, 2838, 1196, 0, 0, 0, 0, 0, 0, 3994, 1897, 379, 0, 0, 0, 0, 1511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8176, 8449, 7992, 8033, 8678, 8195, 8232, 7781] avg=8192.0000 max=8678 max_rank=4 min=7781 min_rank=7 imbalance_ratio=1.059326 max_load_reduction=56858 max_load_reduction_pct=0.867584 moved_to_echo=32886 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 4089, 0, 31, 3, 0, 0, 0, 0, 0, 0, 1263, 0, 0, 488, 4068, 0, 0, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 7, 349, 2198, 1617, 62, 0, 0, 0, 0, 0, 3916, 3852, 4, 0, 0, 48, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 100, 0, 0, 2846, 4093, 1201, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 4095, 0, 2, 0, 2881, 1213, 0, 0, 0, 0, 0, 0, 4096, 1898, 613, 0, 0, 0, 0, 2201, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 7298, 8329, 8183, 8241, 8189, 8271, 8808] avg=8192.0000 max=8808 max_rank=7 min=7298 min_rank=1 imbalance_ratio=1.075195 max_load_reduction=56728 max_load_reduction_pct=0.865601 moved_to_echo=33509 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 2, 4036, 0, 31, 3, 0, 0, 0, 0, 0, 0, 1253, 0, 0, 366, 3997, 0, 1, 843, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 149, 2196, 1603, 62, 0, 0, 0, 0, 0, 4020, 3884, 6, 0, 0, 204, 0, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4, 0, 402, 2, 0, 3330, 4096, 1200, 0, 0, 0, 0, 0, 0, 4093, 11, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 4096, 0, 1, 0, 2880, 1203, 0, 0, 0, 0, 0, 0, 3774, 1897, 161, 0, 0, 0, 0, 1783, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8168, 6460, 8106, 9514, 9037, 8198, 8438, 7615] avg=8192.0000 max=9514 max_rank=3 min=6460 min_rank=1 imbalance_ratio=1.161377 max_load_reduction=56022 max_load_reduction_pct=0.854828 moved_to_echo=33288 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 4096, 1, 4083, 3, 26, 3, 0, 0, 0, 0, 0, 0, 1259, 0, 0, 1648, 3912, 0, 0, 2650, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 3, 289, 2196, 1611, 53, 0, 0, 0, 0, 0, 3745, 3324, 0, 0, 2, 372, 4, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 30, 2, 0, 2603, 4092, 1177, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 4, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 11, 4093, 0, 0, 0, 2822, 1209, 0, 0, 0, 0, 0, 0, 3914, 1897, 414, 2, 0, 1, 0, 1544, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 9469, 8241, 7600, 7909, 8192, 8137, 7772] avg=8192.0000 max=9469 max_rank=1 min=7600 min_rank=3 imbalance_ratio=1.155884 max_load_reduction=56067 max_load_reduction_pct=0.855515 moved_to_echo=32010 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 4091, 7, 4077, 0, 35, 4, 0, 0, 0, 0, 0, 0, 1256, 0, 0, 621, 4094, 0, 5, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 6, 4, 0, 17, 69, 2197, 1620, 68, 0, 0, 0, 0, 0, 3787, 3960, 0, 0, 0, 111, 0, 857, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 5, 352, 0, 2, 2367, 4074, 1200, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 3, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 41, 4090, 4, 19, 5, 2877, 1215, 0, 0, 0, 0, 0, 0, 4038, 1894, 400, 0, 5, 0, 4, 1742, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 7990, 8078, 8715, 8016, 8185, 8253, 8083] avg=8192.0000 max=8715 max_rank=3 min=7990 min_rank=1 imbalance_ratio=1.063843 max_load_reduction=56821 max_load_reduction_pct=0.867020 moved_to_echo=32537 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[31872, 15232, 3328, 128, 128, 128, 128, 15360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 32512, 128, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 256, 1792, 17664, 12928, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[29696, 30080, 128, 0, 128, 1792, 128, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 640, 32768, 128, 128, 128, 22656, 9728, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[10112, 0, 0, 8192, 31872, 0, 128, 15616, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 1664, 128, 128, 21760, 32768, 9472, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 47, 0, 0, 4096, 0, 0, 0, 4096, 4, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4034, 0, 1, 18, 4091, 4095, 0, 4095, 0, 0, 0, 3472, 4096, 1, 0, 1014, 3179, 4096, 0, 0, 0, 0, 4031, 0, 576, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 4095, 22, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4063, 7, 5, 148, 4096, 4094, 0, 4091, 0, 0, 0, 3431, 4095, 0, 0, 1960, 3559, 4096, 0, 0, 0, 0, 2709, 0, 478, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 2, 4096, 0, 0, 0, 4087, 15, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4077, 0, 0, 29, 4095, 4095, 0, 4092, 0, 0, 0, 3164, 4096, 0, 2, 1729, 3675, 4096, 0, 0, 0, 0, 3545, 0, 119, 0, 25, 0, 4096, 0, 0, 0, 4, 0, 4095, 0, 0, 3, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 4089, 75, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4074, 3, 5, 49, 4095, 4095, 0, 4094, 0, 0, 0, 3567, 4094, 0, 2, 1204, 3751, 4093, 0, 0, 0, 0, 3626, 0, 32, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 37, 0, 0, 4096, 0, 0, 0, 4088, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4087, 5, 0, 204, 4096, 4093, 0, 4091, 0, 0, 0, 3960, 4096, 0, 6, 419, 3979, 4096, 0, 0, 0, 0, 3569, 0, 101, 0, 24, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 6, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 1, 4084, 0, 0, 1, 4096, 59, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 4088, 0, 3, 117, 4087, 4090, 0, 4096, 0, 0, 0, 3360, 4096, 0, 0, 997, 3999, 4096, 0, 0, 0, 0, 3647, 0, 98, 0, 6, 0, 4095, 0, 1, 0, 0, 0, 4096, 0, 0, 33, 0, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 50, 0, 4, 4096, 0, 0, 0, 4089, 189, 4096, 0, 0, 4096, 0, 0, 4094, 0, 0, 4055, 6, 0, 29, 4096, 4096, 0, 4093, 0, 0, 0, 2669, 4096, 0, 3, 1508, 3410, 4094, 0, 0, 1, 0, 3706, 0, 751, 0, 2, 0, 4096, 0, 0, 0, 8, 0, 4095, 0, 0, 8, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[5, 0, 5, 61, 0, 0, 4024, 0, 0, 0, 4091, 7, 4087, 0, 5, 4089, 0, 5, 4090, 0, 4, 4089, 0, 286, 118, 4064, 4082, 3, 4090, 0, 0, 0, 3714, 4087, 8, 0, 1116, 4037, 4088, 4, 5, 0, 9, 2914, 4, 57, 0, 16, 5, 4091, 6, 5, 0, 0, 2, 4091, 5, 5, 51, 8, 0, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 2, 4096, 0, 3767, 169, 79, 41, 15, 2, 0, 0, 0, 0, 4087, 15, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4095, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3164, 0, 0, 2, 1729, 3506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 0, 119, 0, 25, 4096, 325, 15, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8157, 8158, 8140, 8401, 8125, 8193, 8181] avg=8192.0000 max=8401 max_rank=4 min=8125 min_rank=5 imbalance_ratio=1.025513 max_load_reduction=57135 max_load_reduction_pct=0.871811 moved_to_echo=32900 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 4096, 0, 3766, 163, 79, 41, 15, 2, 0, 0, 0, 0, 4095, 22, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4063, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 148, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3431, 0, 0, 0, 1960, 3396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2709, 0, 478, 0, 2, 4096, 325, 15, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8172, 8156, 8259, 8787, 7625, 8190, 8177] avg=8192.0000 max=8787 max_rank=4 min=7625 min_rank=5 imbalance_ratio=1.072632 max_load_reduction=56749 max_load_reduction_pct=0.865921 moved_to_echo=32779 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 47, 0, 0, 4096, 0, 3770, 146, 79, 41, 15, 2, 0, 0, 0, 0, 4096, 4, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4034, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4091, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3472, 0, 1, 0, 1014, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 576, 0, 0, 4096, 325, 16, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8155, 8116, 8125, 7520, 9044, 8190, 8190] avg=8192.0000 max=9044 max_rank=5 min=7520 min_rank=4 imbalance_ratio=1.104004 max_load_reduction=56492 max_load_reduction_pct=0.862000 moved_to_echo=32944 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 4096, 0, 3768, 172, 79, 41, 15, 2, 0, 0, 0, 0, 4089, 75, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4074, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4095, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3567, 0, 0, 2, 1204, 3579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3626, 0, 32, 0, 3, 4093, 326, 15, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 4096, 4079, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8219, 8163, 8160, 8352, 8095, 8190, 8179] avg=8192.0000 max=8352 max_rank=4 min=8095 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=57184 max_load_reduction_pct=0.872559 moved_to_echo=32816 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 1, 4084, 0, 3771, 184, 79, 41, 15, 2, 0, 0, 0, 1, 4096, 59, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4088, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 117, 4087, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3360, 0, 0, 0, 997, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3647, 0, 98, 0, 6, 4096, 325, 16, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 2, 0, 4095, 4080, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8211, 8172, 8215, 8172, 8188, 8190, 8210] avg=8192.0000 max=8215 max_rank=3 min=8172 min_rank=2 imbalance_ratio=1.002808 max_load_reduction=57321 max_load_reduction_pct=0.874649 moved_to_echo=32760 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 37, 0, 0, 4096, 0, 3766, 183, 79, 41, 15, 2, 0, 0, 0, 0, 4088, 1, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 4096, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 6, 419, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 0, 101, 0, 24, 4096, 325, 16, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 4096, 4080, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 8144, 8173, 8314, 8181, 8131, 8191, 8183] avg=8192.0000 max=8314 max_rank=3 min=8131 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=57222 max_load_reduction_pct=0.873138 moved_to_echo=32686 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 5, 61, 0, 0, 4024, 0, 3766, 189, 86, 45, 16, 5, 0, 0, 0, 0, 4091, 7, 4042, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4074, 0, 4, 4089, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 118, 4064, 3996, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3714, 0, 8, 0, 1116, 3848, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 9, 2914, 4, 57, 0, 16, 4088, 324, 16, 0, 0, 0, 0, 0, 5, 4086, 6, 5, 0, 0, 2, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 51, 8, 0, 3, 0, 0, 4089, 4071, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8145, 8458, 8181, 8690, 7433, 8195, 8232] avg=8192.0000 max=8690 max_rank=4 min=7433 min_rank=5 imbalance_ratio=1.060791 max_load_reduction=56846 max_load_reduction_pct=0.867401 moved_to_echo=32550 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 50, 0, 4, 4096, 0, 3768, 157, 79, 41, 15, 2, 0, 0, 0, 0, 4089, 189, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4055, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 0, 3, 1508, 3253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3706, 0, 751, 0, 2, 4094, 325, 16, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 8, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8333, 8140, 8142, 7433, 8895, 8197, 8184] avg=8192.0000 max=8895 max_rank=5 min=7433 min_rank=4 imbalance_ratio=1.085815 max_load_reduction=56641 max_load_reduction_pct=0.864273 moved_to_echo=32709 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 384, 32512, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[27392, 0, 128, 128, 9984, 28288, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32640, 0, 128, 32640, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[768, 32768, 32128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 256, 0, 128, 32768, 0, 30208, 1408, 640, 384, 128, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 27776, 128, 2304, 0, 128, 32768, 2688, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 4096, 4088, 4096, 0, 0, 0, 4095, 0, 4096, 4096, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1145, 60, 0, 0, 0, 0, 3259, 0, 0, 0, 3680, 4096, 0, 0, 4065, 80, 0, 0, 0, 4095, 4096, 7, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4092, 4090, 4087, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 696, 56, 0, 3, 0, 0, 3670, 0, 0, 0, 3829, 4096, 0, 0, 4065, 13, 0, 11, 0, 4096, 4063, 10, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4093, 4083, 4086, 0, 0, 0, 4096, 0, 4096, 4092, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3, 0, 0, 4095, 0, 200, 306, 0, 3, 0, 0, 3650, 0, 0, 0, 4069, 4096, 0, 0, 4055, 65, 0, 7, 0, 4096, 4047, 14, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4093, 4079, 4087, 0, 0, 0, 4096, 0, 4096, 4094, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4096, 0, 359, 91, 0, 0, 0, 0, 3623, 0, 0, 0, 4082, 4096, 0, 0, 4093, 65, 0, 9, 0, 4096, 4076, 15, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 4096, 0, 0, 0, 4096, 0, 4096, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 703, 390, 0, 0, 1, 0, 3093, 0, 0, 0, 4021, 4096, 0, 0, 4036, 46, 0, 2, 0, 4096, 4095, 3, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 4094, 4095, 0, 0, 0, 4096, 0, 4096, 4091, 0, 21, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 541, 74, 0, 0, 0, 0, 3607, 0, 0, 0, 3930, 4096, 0, 0, 4087, 26, 0, 5, 0, 4096, 4091, 12, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 4093, 4073, 0, 0, 0, 4093, 0, 4096, 4096, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 1, 0, 0, 4095, 0, 966, 170, 0, 1, 2, 0, 3377, 0, 0, 0, 3707, 4096, 0, 0, 4067, 30, 0, 0, 0, 4096, 4080, 7, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[5, 6, 0, 2, 4074, 6, 1, 4091, 4091, 4091, 0, 5, 1, 4080, 0, 4083, 4088, 0, 22, 4089, 6, 5, 7, 2, 0, 0, 0, 1, 4, 4, 2, 4091, 0, 6, 12, 4090, 9, 1436, 55, 0, 0, 27, 5, 2641, 1, 0, 2, 3995, 4091, 0, 5, 4088, 0, 2, 17, 1, 4087, 4090, 5, 9, 0, 0, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 4079, 4087, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 359, 91, 0, 3188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3623, 0, 0, 0, 4082, 889, 17, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4093, 65, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 15, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8182, 8186, 8194, 7735, 8611, 8243, 8193] avg=8192.0000 max=8611 max_rank=5 min=7735 min_rank=4 imbalance_ratio=1.051147 max_load_reduction=56925 max_load_reduction_pct=0.868607 moved_to_echo=32782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1145, 60, 0, 3188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3259, 0, 0, 0, 3680, 889, 17, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4065, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 7, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8199, 8191, 8195, 8489, 7845, 8221, 8204] avg=8192.0000 max=8489 max_rank=4 min=7845 min_rank=5 imbalance_ratio=1.036255 max_load_reduction=57047 max_load_reduction_pct=0.870468 moved_to_echo=32759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 703, 390, 0, 3188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3093, 0, 0, 0, 4021, 889, 17, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4036, 46, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 3, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8205, 8182, 8195, 8378, 8021, 8160, 8200] avg=8192.0000 max=8378 max_rank=4 min=8021 min_rank=5 imbalance_ratio=1.022705 max_load_reduction=57158 max_load_reduction_pct=0.872162 moved_to_echo=32759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 4090, 4087, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 696, 56, 0, 3188, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3670, 0, 0, 0, 3829, 889, 17, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4065, 13, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4063, 10, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8193, 8186, 8187, 8036, 8408, 8165, 8175] avg=8192.0000 max=8408 max_rank=5 min=8036 min_rank=4 imbalance_ratio=1.026367 max_load_reduction=57128 max_load_reduction_pct=0.871704 moved_to_echo=32784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 21, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 541, 74, 0, 3188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3607, 0, 0, 0, 3930, 889, 17, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4087, 26, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 12, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8205, 8202, 8195, 7899, 8443, 8194, 8205] avg=8192.0000 max=8443 max_rank=5 min=7899 min_rank=4 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 4093, 4073, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 4096, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 966, 170, 0, 3186, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 3377, 0, 0, 0, 3707, 889, 17, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4067, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 7, 0, 0, 0, 1, 0, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8181, 8193, 8194, 8418, 7993, 8173, 8190] avg=8192.0000 max=8418 max_rank=4 min=7993 min_rank=5 imbalance_ratio=1.027588 max_load_reduction=57118 max_load_reduction_pct=0.871552 moved_to_echo=32774 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 4083, 4086, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 8, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4095, 0, 200, 306, 0, 3188, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3650, 0, 0, 0, 4069, 891, 17, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4055, 65, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4047, 14, 0, 1, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8183, 8190, 8182, 7792, 8630, 8203, 8164] avg=8192.0000 max=8630 max_rank=5 min=7792 min_rank=4 imbalance_ratio=1.053467 max_load_reduction=56906 max_load_reduction_pct=0.868317 moved_to_echo=32789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[5, 6, 0, 2, 4074, 6, 1, 4091, 7, 0, 0, 0, 0, 0, 0, 0, 4091, 4091, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 22, 4076, 6, 5, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 4, 2, 4091, 4083, 9, 0, 0, 0, 0, 0, 0, 0, 6, 12, 4090, 9, 1436, 55, 0, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 27, 5, 2641, 1, 0, 2, 3995, 890, 24, 0, 0, 0, 0, 0, 0, 4064, 0, 5, 4088, 0, 2, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4090, 5, 9, 0, 0, 0, 5, 6, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8188, 8206, 8194, 8789, 7585, 8177, 8205] avg=8192.0000 max=8789 max_rank=4 min=7585 min_rank=5 imbalance_ratio=1.072876 max_load_reduction=56747 max_load_reduction_pct=0.865891 moved_to_echo=32756 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32768, 128, 6144, 1280, 0, 25600, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 27008, 128, 0, 128, 31360, 7168, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 32640, 128, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32640, 0, 128, 32640, 384, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 17, 0, 0, 0, 4073, 4083, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1218, 0, 344, 0, 4096, 0, 0, 4, 0, 4093, 0, 0, 0, 4083, 4028, 0, 1, 4096, 0, 0, 4096, 4092, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2632, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 19, 0, 0, 0, 4085, 4081, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1279, 0, 1108, 0, 4096, 0, 1, 1, 0, 4094, 0, 0, 2, 4009, 4059, 0, 0, 4096, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1837, 0, 4096, 4096, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 19, 0, 0, 0, 4086, 4093, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 685, 0, 4096, 0, 0, 7, 0, 4066, 0, 0, 3, 4093, 4094, 0, 0, 4096, 7, 0, 4096, 4091, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2694, 0, 4096, 4094, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 2, 0, 0, 0, 4080, 4088, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 587, 0, 683, 0, 4096, 0, 0, 0, 1, 4074, 0, 0, 2, 4094, 4063, 0, 1, 4095, 4, 0, 4096, 4093, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4096, 0, 2876, 0, 4095, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 72, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 0, 325, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 4095, 4095, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2647, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 27, 0, 0, 0, 4081, 4087, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 0, 652, 0, 4096, 0, 0, 8, 0, 4069, 0, 0, 0, 4072, 4094, 0, 1, 4096, 1, 0, 4096, 4091, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2835, 0, 4096, 4091, 0, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 42, 0, 0, 0, 4033, 4082, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 346, 0, 4096, 0, 1, 4, 0, 4065, 0, 0, 5, 4082, 3833, 0, 2, 4096, 0, 0, 4096, 4091, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2514, 0, 4096, 4093, 0, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[3, 4090, 0, 1, 3, 0, 6, 5, 4091, 4084, 0, 5, 6, 6, 0, 0, 0, 3, 0, 7, 2921, 5, 532, 4, 4090, 11, 0, 0, 0, 4091, 0, 0, 0, 4059, 4008, 0, 1, 4091, 8, 6, 4087, 4091, 4089, 6, 0, 5, 0, 0, 1, 4089, 5, 0, 0, 0, 4088, 6, 736, 5, 4091, 4090, 0, 5, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 19, 0, 0, 0, 2325, 1745, 0, 0, 0, 0, 0, 0, 4086, 4093, 14, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 685, 0, 4092, 2271, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 4066, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 4094, 0, 0, 80, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2694, 0, 1750, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8204, 7768, 8179, 8277, 8191, 8192, 8540] avg=8192.0000 max=8540 max_rank=7 min=7768 min_rank=2 imbalance_ratio=1.042480 max_load_reduction=56996 max_load_reduction_pct=0.869690 moved_to_echo=33200 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 17, 0, 0, 0, 2325, 1745, 0, 0, 0, 0, 0, 0, 4073, 4083, 4, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1218, 0, 344, 0, 4092, 2271, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4093, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4028, 0, 1, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2632, 0, 1750, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8171, 7925, 8203, 8192, 8192, 8192, 8478] avg=8192.0000 max=8478 max_rank=7 min=7925 min_rank=2 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=33054 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 72, 0, 0, 0, 2325, 1745, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 0, 325, 0, 4092, 2271, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2647, 0, 1750, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8238, 8200, 7744, 8203, 8270, 8196, 8192, 8493] avg=8192.0000 max=8493 max_rank=7 min=7744 min_rank=2 imbalance_ratio=1.036743 max_load_reduction=57043 max_load_reduction_pct=0.870407 moved_to_echo=33151 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 19, 0, 0, 0, 2325, 1745, 0, 0, 0, 0, 0, 0, 4085, 4081, 3, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1279, 0, 1108, 0, 4092, 2271, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 4094, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 2, 4009, 4059, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1837, 0, 1750, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8180, 8750, 8202, 8150, 8194, 8192, 7684] avg=8192.0000 max=8750 max_rank=2 min=7684 min_rank=7 imbalance_ratio=1.068115 max_load_reduction=56786 max_load_reduction_pct=0.866486 moved_to_echo=32220 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 2, 0, 0, 0, 2324, 1744, 0, 0, 0, 0, 0, 0, 4080, 4088, 24, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 587, 0, 683, 0, 4092, 2271, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4074, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 4063, 0, 1, 80, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2876, 0, 1751, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8202, 7633, 8181, 8244, 8193, 8195, 8721] avg=8192.0000 max=8721 max_rank=7 min=7633 min_rank=2 imbalance_ratio=1.064575 max_load_reduction=56815 max_load_reduction_pct=0.866928 moved_to_echo=33353 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 27, 0, 0, 0, 2325, 1745, 0, 0, 0, 0, 0, 0, 4081, 4087, 20, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 0, 652, 0, 4092, 2271, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 4069, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4094, 0, 1, 80, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 0, 1750, 4091, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8199, 7649, 8183, 8248, 8191, 8192, 8681] avg=8192.0000 max=8681 max_rank=7 min=7649 min_rank=2 imbalance_ratio=1.059692 max_load_reduction=56855 max_load_reduction_pct=0.867538 moved_to_echo=33312 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 42, 0, 0, 0, 2325, 1745, 0, 0, 0, 0, 0, 0, 4033, 4082, 50, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 346, 0, 4093, 2271, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4, 0, 4065, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 5, 4082, 3833, 0, 2, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2514, 0, 1750, 4093, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8176, 8231, 8176, 8002, 8190, 8192, 8361] avg=8192.0000 max=8361 max_rank=7 min=8002 min_rank=4 imbalance_ratio=1.020630 max_load_reduction=57175 max_load_reduction_pct=0.872421 moved_to_echo=32745 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 4090, 0, 1, 3, 0, 6, 5, 2325, 1745, 0, 0, 0, 0, 0, 0, 4091, 4084, 0, 5, 6, 6, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 2921, 5, 532, 4, 4089, 2275, 0, 0, 0, 0, 0, 0, 4090, 11, 0, 0, 0, 4091, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4008, 0, 1, 71, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4091, 0, 6, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 5, 0, 0, 0, 4088, 6, 0, 0, 0, 0, 0, 0, 0, 0, 736, 5, 1737, 4090, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8204, 9836, 8209, 8153, 8189, 8189, 6578] avg=8192.0000 max=9836 max_rank=2 min=6578 min_rank=7 imbalance_ratio=1.200684 max_load_reduction=55700 max_load_reduction_pct=0.849915 moved_to_echo=31109 active_clone_links=6 active_echo_slots_local=6 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[18816, 128, 14080, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 256, 0, 128, 128, 18688, 14080, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 128, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32640, 32384, 0, 128, 640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 32768, 128, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 9984, 128, 4736, 128, 32768, 18176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2, 0, 0, 0, 4088, 0, 0, 0, 0, 3850, 0, 2, 23, 4096, 0, 0, 0, 3756, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4089, 4096, 0, 706, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 0, 0, 3935, 0, 17, 0, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 1, 0, 0, 4085, 0, 0, 0, 0, 3955, 0, 0, 24, 4096, 0, 0, 0, 3925, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4093, 4095, 0, 1435, 0, 1, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 2907, 1, 4, 0, 54] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2, 0, 0, 0, 4080, 0, 0, 0, 0, 3929, 0, 0, 12, 4096, 0, 0, 0, 3962, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4095, 0, 2241, 0, 1, 4096, 3, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2140, 0, 3, 0, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 10, 0, 0, 0, 4084, 0, 0, 0, 0, 4026, 0, 0, 31, 4096, 0, 0, 0, 4007, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 4096, 0, 527, 0, 1, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 3680, 0, 2, 0, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4040, 0, 4, 32, 4096, 0, 0, 0, 4051, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 4089, 4096, 0, 2597, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4090, 0, 0, 0, 0, 1553, 0, 8, 0, 15] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2, 0, 0, 0, 4085, 0, 0, 0, 0, 3761, 0, 0, 28, 4096, 0, 0, 0, 3516, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 4096, 0, 1688, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 3292, 0, 6, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1, 5, 4090, 0, 0, 5, 0, 4080, 3, 0, 1, 6, 3942, 0, 0, 0, 4090, 6, 3, 0, 3905, 0, 0, 9, 4090, 7, 4089, 1, 6, 2, 3, 4091, 4088, 3, 2332, 5, 5, 4086, 0, 4089, 6, 0, 2, 4089, 4088, 7, 0, 14, 8, 5, 0, 4089, 0, 1, 4089, 6, 0, 6, 12, 2045, 0, 0, 0, 26] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4005, 0, 3, 54, 4096, 0, 0, 0, 3257, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 4096, 0, 1495, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4092, 0, 0, 0, 0, 3415, 0, 44, 0, 37] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 2, 0, 0, 0, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 0, 2, 23, 2902, 1324, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3756, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 706, 0, 0, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 17, 0, 15, 4096, 1189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8101, 8148, 8190, 7271, 8191, 8190, 9252] avg=8192.0000 max=9252 max_rank=7 min=7271 min_rank=4 imbalance_ratio=1.129395 max_load_reduction=56284 max_load_reduction_pct=0.858826 moved_to_echo=32904 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 2, 0, 0, 0, 4080, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 0, 12, 2902, 1324, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3962, 0, 0, 1, 296, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2241, 0, 1, 2469, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 0, 3, 0, 12, 4096, 1189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8167, 8355, 8196, 8809, 8191, 8193, 7440] avg=8192.0000 max=8809 max_rank=4 min=7440 min_rank=7 imbalance_ratio=1.075317 max_load_reduction=56727 max_load_reduction_pct=0.865585 moved_to_echo=32633 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 2, 0, 0, 0, 4085, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 0, 0, 28, 2902, 1324, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3516, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1688, 0, 0, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3292, 0, 6, 0, 8, 4096, 1189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8015, 7908, 8193, 8253, 8191, 8193, 8593] avg=8192.0000 max=8593 max_rank=7 min=7908 min_rank=2 imbalance_ratio=1.048950 max_load_reduction=56943 max_load_reduction_pct=0.868881 moved_to_echo=33230 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 1, 0, 0, 4085, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 24, 2902, 1324, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3925, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1435, 0, 1, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2907, 1, 4, 0, 54, 4096, 1189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8205, 8317, 8194, 8000, 8191, 8189, 8251] avg=8192.0000 max=8317 max_rank=2 min=8000 min_rank=4 imbalance_ratio=1.015259 max_load_reduction=57219 max_load_reduction_pct=0.873093 moved_to_echo=32631 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 10, 0, 0, 0, 4084, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 31, 2902, 1324, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4007, 0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 527, 0, 1, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3680, 0, 2, 0, 22, 4096, 1189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8283, 8399, 8193, 7093, 8191, 8191, 8989] avg=8192.0000 max=8989 max_rank=7 min=7093 min_rank=4 imbalance_ratio=1.097290 max_load_reduction=56547 max_load_reduction_pct=0.862839 moved_to_echo=32464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 1, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 4, 32, 2902, 1324, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4051, 0, 1, 0, 296, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2597, 0, 0, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1553, 0, 8, 0, 15, 4096, 1189, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8302, 8444, 8190, 9162, 8191, 8186, 6861] avg=8192.0000 max=9162 max_rank=4 min=6861 min_rank=7 imbalance_ratio=1.118408 max_load_reduction=56374 max_load_reduction_pct=0.860199 moved_to_echo=32400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 5, 4090, 0, 0, 5, 0, 4080, 7, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 6, 3942, 0, 0, 0, 2902, 1321, 0, 0, 0, 0, 0, 0, 4090, 6, 3, 0, 3905, 0, 0, 9, 300, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 1, 6, 2, 3, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 2332, 5, 5, 2455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 4082, 4088, 7, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 4089, 0, 1, 4089, 6, 7, 3, 0, 0, 0, 0, 0, 0, 0, 6, 12, 2045, 0, 0, 0, 26, 4089, 1188, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8175, 8313, 8199, 8888, 8199, 8208, 7366] avg=8192.0000 max=8888 max_rank=4 min=7366 min_rank=7 imbalance_ratio=1.084961 max_load_reduction=56648 max_load_reduction_pct=0.864380 moved_to_echo=32661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4091, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 3, 54, 2902, 1324, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3257, 0, 3, 0, 296, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1495, 0, 0, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3415, 0, 44, 0, 37, 4096, 1192, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8288, 7652, 8181, 8060, 8191, 8186, 8784] avg=8192.0000 max=8784 max_rank=7 min=7652 min_rank=2 imbalance_ratio=1.072266 max_load_reduction=56752 max_load_reduction_pct=0.865967 moved_to_echo=33221 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 13056, 128, 128, 19840, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 23040, 128, 128, 0, 256, 32768, 9600, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 0, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 0, 30464, 0, 128, 128, 2432, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 31616, 0, 128, 256, 23296, 10624, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 314, 0, 0, 4027, 1832, 0, 4096, 4094, 0, 0, 4096, 0, 2692, 0, 0, 4095, 2, 0, 4096, 0, 0, 0, 317, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3675, 0, 0, 119, 0, 0, 0, 4096, 0, 3408, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 317, 0, 0, 4038, 2324, 0, 4096, 4091, 0, 0, 4096, 0, 3498, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 189, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 4094, 0, 0, 0, 3163, 2, 0, 80, 0, 0, 0, 4096, 0, 2770, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 26, 0, 0, 4082, 1881, 0, 4096, 4093, 0, 0, 4096, 0, 3965, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 380, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4085, 0, 0, 0, 4050, 0, 0, 17, 0, 0, 0, 4096, 0, 1993, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 65, 0, 1, 3936, 2731, 0, 4096, 4091, 0, 0, 4096, 0, 2086, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 194, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4090, 0, 0, 0, 3859, 0, 0, 3, 0, 0, 0, 4096, 4, 3512, 0, 0, 4096, 4094, 0, 0, 0, 0, 4, 1, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 138, 0, 0, 4077, 2521, 0, 4096, 4089, 0, 0, 4096, 0, 2510, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 398, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4092, 0, 0, 0, 3795, 0, 0, 40, 0, 0, 0, 4096, 5, 2908, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 163, 0, 0, 4045, 2846, 0, 4096, 4078, 0, 0, 4096, 0, 2004, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1114, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 4012, 0, 0, 10, 0, 0, 0, 4096, 0, 2182, 0, 0, 4096, 4096, 0, 0, 0, 0, 17, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 103, 0, 3, 4056, 1553, 0, 4096, 4084, 0, 0, 4096, 0, 3520, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 204, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 4093, 0, 0, 0, 3639, 2, 0, 10, 0, 0, 0, 4096, 0, 3284, 0, 0, 4096, 4094, 0, 0, 0, 0, 17, 2, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[8, 0, 20, 6, 216, 4, 0, 4074, 1652, 8, 4087, 4090, 4, 1, 4080, 6, 2573, 0, 6, 3902, 0, 1, 4088, 0, 0, 3, 385, 3, 6, 4089, 6, 0, 5, 3, 0, 6, 6, 4020, 0, 0, 7, 3901, 0, 0, 8, 0, 4, 6, 4081, 0, 3798, 5, 0, 4083, 4088, 2, 0, 6, 6, 8, 0, 4088, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 314, 0, 0, 4027, 2676, 1302, 0, 0, 0, 0, 0, 0, 1832, 0, 0, 4094, 0, 0, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2692, 0, 0, 4095, 2, 0, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 4096, 0, 0, 2161, 1530, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3799, 305, 0, 0, 0, 0, 0, 0, 0, 3675, 0, 0, 119, 0, 0, 0, 4096, 297, 0, 0, 0, 0, 0, 0, 4096, 0, 3408, 0, 0, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8319, 7861, 8053, 8110, 8200, 8187, 8619, 8187] avg=8192.0000 max=8619 max_rank=6 min=7861 min_rank=1 imbalance_ratio=1.052124 max_load_reduction=56917 max_load_reduction_pct=0.868484 moved_to_echo=33193 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 317, 0, 0, 4038, 2676, 1302, 0, 0, 0, 0, 0, 0, 2324, 0, 0, 4091, 0, 0, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3498, 0, 0, 4096, 0, 0, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 4096, 0, 0, 2161, 1530, 6, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4094, 0, 0, 3799, 305, 0, 0, 0, 0, 0, 0, 0, 3163, 2, 0, 80, 0, 0, 0, 4096, 297, 0, 0, 0, 0, 0, 0, 4096, 0, 2770, 0, 0, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8333, 8350, 8858, 7982, 8205, 7638, 7981, 8189] avg=8192.0000 max=8858 max_rank=2 min=7638 min_rank=5 imbalance_ratio=1.081299 max_load_reduction=56678 max_load_reduction_pct=0.864838 moved_to_echo=32013 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 26, 0, 0, 4082, 2676, 1302, 0, 0, 0, 0, 0, 0, 1881, 0, 0, 4093, 0, 0, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 0, 0, 4096, 0, 0, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 4096, 0, 0, 2161, 1530, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4085, 0, 0, 3799, 305, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 17, 0, 0, 0, 4096, 297, 0, 0, 0, 0, 0, 0, 4096, 0, 1993, 0, 0, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 7909, 9325, 8173, 8190, 8460, 7204, 8189] avg=8192.0000 max=9325 max_rank=2 min=7204 min_rank=6 imbalance_ratio=1.138306 max_load_reduction=56211 max_load_reduction_pct=0.857712 moved_to_echo=32043 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 138, 0, 0, 4077, 2676, 1302, 0, 0, 0, 0, 0, 0, 2521, 0, 0, 4089, 0, 0, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2510, 0, 0, 4096, 0, 0, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 4096, 0, 0, 2161, 1530, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4092, 0, 0, 3799, 305, 0, 0, 0, 0, 0, 0, 0, 3795, 0, 0, 40, 0, 0, 0, 4096, 297, 0, 0, 0, 0, 0, 0, 4096, 5, 2908, 0, 0, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8545, 7870, 8191, 8198, 8228, 8124, 8187] avg=8192.0000 max=8545 max_rank=1 min=7870 min_rank=2 imbalance_ratio=1.043091 max_load_reduction=56991 max_load_reduction_pct=0.869614 moved_to_echo=32737 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 65, 0, 1, 3936, 2676, 1302, 0, 0, 0, 0, 0, 0, 2731, 0, 0, 4091, 0, 0, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2086, 0, 0, 4096, 0, 0, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 4096, 0, 0, 2161, 1530, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4090, 0, 0, 3797, 305, 0, 0, 0, 0, 0, 0, 0, 3859, 0, 0, 3, 0, 0, 0, 4096, 297, 0, 0, 0, 0, 0, 0, 4096, 4, 3512, 0, 0, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7980, 8757, 7446, 7987, 8193, 8255, 8727, 8191] avg=8192.0000 max=8757 max_rank=1 min=7446 min_rank=2 imbalance_ratio=1.068970 max_load_reduction=56779 max_load_reduction_pct=0.866379 moved_to_echo=33366 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 163, 0, 0, 4045, 2676, 1302, 0, 0, 0, 0, 0, 0, 2846, 0, 0, 4078, 0, 0, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2004, 0, 0, 4096, 0, 0, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 0, 0, 4096, 0, 0, 2161, 1530, 6, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 3799, 305, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 10, 0, 0, 0, 4096, 297, 0, 0, 0, 0, 0, 0, 4096, 0, 2182, 0, 0, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8859, 7364, 8907, 8209, 8415, 7393, 8203] avg=8192.0000 max=8907 max_rank=3 min=7364 min_rank=2 imbalance_ratio=1.087280 max_load_reduction=56629 max_load_reduction_pct=0.864090 moved_to_echo=32220 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 20, 6, 216, 4, 0, 4074, 2670, 1301, 0, 0, 0, 0, 0, 0, 1652, 8, 0, 4090, 4, 1, 1922, 6, 0, 0, 0, 0, 0, 0, 0, 0, 2573, 0, 6, 3902, 0, 1, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 385, 3, 6, 4089, 6, 0, 2158, 1529, 10, 0, 0, 0, 0, 0, 5, 3, 0, 6, 6, 4020, 0, 0, 3792, 306, 0, 0, 0, 0, 0, 0, 7, 3901, 0, 0, 8, 0, 4, 6, 4087, 296, 0, 0, 0, 0, 0, 0, 4081, 0, 3798, 5, 0, 1107, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 8, 0, 4078, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8299, 7683, 7740, 8189, 8138, 8309, 8993, 8185] avg=8192.0000 max=8993 max_rank=6 min=7683 min_rank=1 imbalance_ratio=1.097778 max_load_reduction=56543 max_load_reduction_pct=0.862778 moved_to_echo=33625 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 103, 0, 3, 4056, 2676, 1302, 0, 0, 0, 0, 0, 0, 1553, 0, 0, 4084, 0, 0, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3520, 0, 0, 4096, 0, 0, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 4096, 0, 0, 2161, 1530, 6, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4093, 0, 0, 3797, 305, 0, 0, 0, 0, 0, 0, 0, 3639, 2, 0, 10, 0, 0, 0, 4096, 297, 0, 0, 0, 0, 0, 0, 4096, 0, 3284, 0, 0, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 7572, 8880, 7997, 8203, 8044, 8495, 8205] avg=8192.0000 max=8880 max_rank=2 min=7572 min_rank=1 imbalance_ratio=1.083984 max_load_reduction=56656 max_load_reduction_pct=0.864502 moved_to_echo=32947 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[22912, 0, 128, 32640, 128, 128, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 1408, 128, 128, 32384, 21504, 10496, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 23936, 128, 0, 8960, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 30208, 128, 0, 384, 0, 128, 128, 32768, 2432, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[17408, 128, 0, 32768, 128, 128, 15488, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 0, 30464, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 3200, 128, 128, 32768, 128, 0, 17408, 12288, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 9, 0, 4096, 4096, 0, 0, 4095, 0, 0, 643, 4096, 4096, 0, 4092, 2138, 0, 2943, 4089, 0, 4091, 0, 1, 0, 0, 0, 0, 2465, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 8, 8, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 16, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 1305, 4096, 4096, 0, 4069, 998, 0, 2657, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 2881, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 339, 16, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4090, 0, 0, 16, 14, 0, 4096, 4096, 0, 0, 4086, 0, 0, 2270, 4096, 4096, 1, 4046, 626, 0, 3625, 4090, 0, 4086, 0, 8, 0, 0, 0, 0, 1667, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 4096, 0, 38, 5, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 0, 20, 5, 0, 4095, 4096, 0, 0, 4093, 0, 0, 2264, 4096, 4096, 1, 4092, 1072, 0, 1972, 4095, 0, 4093, 0, 3, 0, 0, 0, 0, 2244, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 401, 221, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 0, 0, 45, 19, 0, 4096, 4096, 0, 0, 4088, 0, 0, 1319, 4096, 4096, 16, 4005, 1790, 0, 878, 4094, 0, 4078, 0, 5, 0, 0, 0, 0, 1584, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 4096, 0, 2330, 321, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4096, 0, 4090, 0, 0, 21, 6, 0, 4096, 4096, 0, 0, 4094, 0, 0, 2351, 4095, 4096, 8, 4079, 748, 0, 2314, 4090, 0, 4086, 0, 0, 1, 0, 0, 0, 1881, 0, 0, 31, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 4096, 0, 233, 630, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 0, 0, 29, 21, 0, 4096, 4096, 0, 0, 4079, 0, 0, 992, 4096, 4096, 2, 4078, 1145, 0, 2677, 4086, 0, 4071, 0, 4, 0, 0, 0, 0, 3379, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[5, 0, 0, 4089, 0, 6, 0, 4089, 7, 4091, 6, 6, 12, 0, 6, 4041, 4086, 0, 6, 4090, 5, 0, 2987, 4089, 4047, 12, 4086, 1171, 6, 696, 4090, 0, 4090, 23, 12, 0, 7, 0, 0, 734, 0, 6, 125, 0, 6, 5, 0, 3, 4088, 0, 7, 0, 0, 6, 0, 5, 0, 4087, 4090, 1, 2236, 276, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2335, 0, 0, 4096, 0, 0, 1305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 0, 4069, 998, 0, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2881, 1093, 881, 18, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3408, 667, 21, 0, 0, 0, 0, 0, 0, 4096, 3215, 0, 339, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8190, 7736, 8412, 8969, 8179, 8192, 7666] avg=8192.0000 max=8969 max_rank=4 min=7666 min_rank=7 imbalance_ratio=1.094849 max_load_reduction=56567 max_load_reduction_pct=0.863144 moved_to_echo=33006 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 16, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2336, 0, 0, 4086, 0, 0, 2270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 1, 4046, 626, 0, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 8, 0, 0, 0, 0, 1667, 1093, 881, 18, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4069, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 3408, 667, 21, 0, 0, 0, 0, 0, 0, 4096, 3215, 0, 38, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8198, 8692, 8986, 7753, 8166, 8195, 7354] avg=8192.0000 max=8986 max_rank=3 min=7354 min_rank=7 imbalance_ratio=1.096924 max_load_reduction=56550 max_load_reduction_pct=0.862885 moved_to_echo=31468 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 20, 5, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2336, 0, 0, 4093, 0, 0, 2264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 1, 4092, 1072, 0, 1972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 0, 0, 2244, 1093, 881, 18, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3408, 667, 21, 0, 0, 0, 0, 0, 0, 4096, 3215, 0, 401, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8693, 7825, 8332, 8178, 8192, 7933] avg=8192.0000 max=8693 max_rank=2 min=7825 min_rank=3 imbalance_ratio=1.061157 max_load_reduction=56843 max_load_reduction_pct=0.867355 moved_to_echo=32635 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 21, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2336, 0, 0, 4094, 0, 0, 2351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 8, 4079, 748, 0, 2314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 1, 0, 0, 0, 1881, 1093, 881, 18, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 1, 0, 4095, 4069, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 3408, 667, 21, 0, 0, 0, 0, 0, 0, 4096, 3215, 0, 233, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8195, 8781, 7837, 7960, 8196, 8200, 8174] avg=8192.0000 max=8781 max_rank=2 min=7837 min_rank=3 imbalance_ratio=1.071899 max_load_reduction=56755 max_load_reduction_pct=0.866013 moved_to_echo=32530 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 45, 19, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2336, 0, 0, 4088, 0, 0, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 16, 4005, 1790, 0, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 5, 0, 0, 0, 0, 1584, 1093, 881, 18, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 3408, 667, 21, 0, 0, 0, 0, 0, 0, 4096, 3215, 0, 2330, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8229, 7743, 7377, 7659, 8173, 8201, 9962] avg=8192.0000 max=9962 max_rank=7 min=7377 min_rank=3 imbalance_ratio=1.216064 max_load_reduction=55574 max_load_reduction_pct=0.847992 moved_to_echo=33995 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 4089, 0, 6, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 7, 4065, 6, 6, 12, 0, 6, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 2322, 0, 6, 4090, 5, 0, 2987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 12, 4086, 1171, 6, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 23, 12, 0, 7, 0, 0, 734, 1095, 883, 24, 0, 0, 0, 0, 0, 0, 6, 125, 0, 6, 5, 0, 3, 4089, 4070, 0, 0, 0, 0, 0, 0, 4088, 0, 7, 0, 0, 6, 0, 5, 3370, 669, 20, 0, 0, 0, 0, 0, 0, 4087, 3207, 1, 2236, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8143, 9410, 6648, 6868, 8304, 8165, 9807] avg=8192.0000 max=9807 max_rank=7 min=6648 min_rank=3 imbalance_ratio=1.197144 max_load_reduction=55729 max_load_reduction_pct=0.850357 moved_to_echo=33144 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 29, 21, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2336, 0, 0, 4079, 0, 0, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 2, 4078, 1145, 0, 2677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4, 0, 0, 0, 0, 3379, 1093, 881, 18, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 3408, 667, 20, 0, 0, 0, 0, 0, 0, 4096, 3215, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8216, 7407, 8590, 9446, 8173, 8199, 7313] avg=8192.0000 max=9446 max_rank=4 min=7313 min_rank=7 imbalance_ratio=1.153076 max_load_reduction=56090 max_load_reduction_pct=0.855865 moved_to_echo=33131 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2336, 0, 0, 4095, 0, 0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 0, 4092, 2138, 0, 2943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 0, 0, 0, 0, 2465, 1093, 881, 18, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3408, 667, 21, 0, 0, 0, 0, 0, 0, 4096, 3215, 0, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8174, 7074, 9861, 8549, 8167, 8192, 7327] avg=8192.0000 max=9861 max_rank=3 min=7074 min_rank=2 imbalance_ratio=1.203735 max_load_reduction=55675 max_load_reduction_pct=0.849533 moved_to_echo=32235 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[18688, 0, 128, 32768, 128, 0, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 256, 0, 128, 128, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[5504, 128, 32640, 9728, 128, 17792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 0, 128, 128, 0, 128, 27264, 5376, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 32768, 25728, 128, 5632, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32640, 128, 128, 256, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 0, 16896, 8832, 7168, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2277, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4082, 0, 0, 4037, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 1790, 4095, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 104, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 2453, 4096, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4040, 0, 0, 3898, 0, 0, 0, 0, 0, 4096, 0, 4070, 0, 1807, 4093, 0, 0, 0, 4096, 0, 0, 4094, 4092, 0, 121, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2271, 4096, 0, 0, 0, 0, 5, 0, 9, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4049, 0, 0, 3778, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 1999, 4094, 0, 0, 0, 4096, 0, 0, 4091, 4096, 0, 213, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 1561, 4096, 0, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4009, 0, 0, 3948, 0, 0, 0, 0, 0, 4096, 0, 4074, 0, 2452, 4090, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 339, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2737, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4073, 0, 0, 4045, 0, 0, 0, 0, 8, 4096, 0, 4092, 0, 1071, 4092, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 366, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1667, 4096, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4075, 0, 0, 4062, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 2293, 4091, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 197, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 1, 0, 0, 22, 4, 0, 131, 4052, 0, 6, 6, 0, 0, 7, 0, 9, 3, 3, 7, 5, 354, 4089, 6, 6, 5, 38, 4088, 22, 5, 4089, 0, 0, 4076, 4023, 4, 7, 4088, 78, 57, 1, 0, 0, 4079, 3, 4088, 7, 3454, 4090, 6, 0, 0, 4090, 8, 6, 4089, 4008, 0, 143, 0, 4086] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 1904, 4096, 0, 0, 0, 0, 3, 0, 9, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 4009, 0, 0, 3890, 0, 0, 0, 0, 0, 4096, 0, 4069, 0, 2391, 4092, 0, 0, 0, 4095, 0, 0, 4094, 4096, 0, 125, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 4044, 41, 9, 0, 0, 0, 0, 0, 0, 0, 2453, 4096, 0, 0, 0, 0, 2100, 124, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 0, 0, 0, 0, 0, 4096, 0, 64, 46, 0, 0, 0, 0, 0, 0, 4070, 0, 1807, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 4092, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8773, 8195, 8137, 8188, 8104, 7829, 8119] avg=8192.0000 max=8773 max_rank=1 min=7829 min_rank=6 imbalance_ratio=1.070923 max_load_reduction=56763 max_load_reduction_pct=0.866135 moved_to_echo=32240 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4044, 41, 9, 0, 0, 0, 0, 0, 0, 0, 2277, 4096, 0, 0, 0, 0, 2101, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 1, 0, 0, 4096, 0, 64, 46, 0, 0, 0, 0, 0, 0, 4096, 0, 1790, 1953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 4093, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8598, 8183, 8147, 8230, 8244, 7839, 8105] avg=8192.0000 max=8598 max_rank=1 min=7839 min_rank=6 imbalance_ratio=1.049561 max_load_reduction=56938 max_load_reduction_pct=0.868805 moved_to_echo=32418 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4044, 41, 9, 0, 0, 0, 0, 0, 0, 0, 2737, 4096, 0, 0, 0, 0, 2100, 124, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 8, 4096, 0, 64, 46, 0, 0, 0, 0, 0, 0, 4092, 0, 1071, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 4096, 0, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 9057, 8186, 8139, 8221, 8259, 7114, 8370] avg=8192.0000 max=9057 max_rank=1 min=7114 min_rank=6 imbalance_ratio=1.105591 max_load_reduction=56479 max_load_reduction_pct=0.861801 moved_to_echo=31964 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4044, 41, 9, 0, 0, 0, 0, 0, 0, 0, 2271, 4096, 0, 0, 0, 0, 2101, 123, 0, 0, 0, 0, 0, 0, 5, 0, 9, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 4096, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 0, 0, 0, 0, 4096, 0, 64, 46, 0, 0, 0, 0, 0, 0, 4077, 0, 1999, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 4096, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8591, 8197, 8136, 8197, 7984, 8028, 8213] avg=8192.0000 max=8591 max_rank=1 min=7984 min_rank=5 imbalance_ratio=1.048706 max_load_reduction=56945 max_load_reduction_pct=0.868912 moved_to_echo=32422 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 1, 0, 4044, 41, 9, 0, 0, 0, 0, 0, 0, 0, 1561, 4096, 0, 0, 0, 0, 2099, 124, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 4096, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 0, 4096, 0, 64, 46, 0, 0, 0, 0, 0, 0, 4074, 0, 2452, 1950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 4096, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7880, 8193, 8142, 8157, 8154, 8476, 8342] avg=8192.0000 max=8476 max_rank=6 min=7880 min_rank=1 imbalance_ratio=1.034668 max_load_reduction=57060 max_load_reduction_pct=0.870667 moved_to_echo=33129 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4044, 41, 9, 0, 0, 0, 0, 0, 0, 0, 1667, 4096, 0, 0, 0, 0, 2099, 124, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 4096, 0, 64, 46, 0, 0, 0, 0, 0, 0, 4095, 0, 2293, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 4096, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7986, 8188, 8143, 8223, 8268, 8339, 8199] avg=8192.0000 max=8339 max_rank=6 min=7986 min_rank=1 imbalance_ratio=1.017944 max_load_reduction=57197 max_load_reduction_pct=0.872757 moved_to_echo=33029 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4087, 1, 0, 0, 22, 4040, 42, 10, 0, 0, 0, 0, 0, 4, 0, 131, 4052, 0, 6, 6, 0, 2103, 125, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 3, 3, 7, 5, 4090, 4076, 0, 0, 0, 0, 0, 0, 354, 4089, 6, 6, 5, 38, 4035, 22, 0, 0, 0, 0, 0, 0, 0, 0, 5, 49, 0, 0, 4076, 4023, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 78, 57, 1, 0, 0, 4079, 3, 68, 53, 0, 0, 0, 0, 0, 0, 4088, 7, 3454, 1945, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 3896, 4008, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 6427, 8200, 8555, 8164, 8427, 9500, 8061] avg=8192.0000 max=9500 max_rank=6 min=6427 min_rank=1 imbalance_ratio=1.159668 max_load_reduction=56036 max_load_reduction_pct=0.855042 moved_to_echo=34152 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 4044, 41, 9, 0, 0, 0, 0, 0, 0, 0, 1904, 4096, 0, 0, 0, 0, 2100, 124, 0, 0, 0, 0, 0, 0, 3, 0, 9, 0, 0, 0, 0, 0, 4095, 4087, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 4095, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 0, 0, 0, 0, 0, 4096, 0, 64, 46, 0, 0, 0, 0, 0, 0, 4069, 0, 2391, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 4096, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8224, 8194, 8137, 8156, 8096, 8411, 8127] avg=8192.0000 max=8411 max_rank=6 min=8096 min_rank=5 imbalance_ratio=1.026733 max_load_reduction=57125 max_load_reduction_pct=0.871658 moved_to_echo=32790 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 128, 128, 32384, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 31360, 32768, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 128, 17280, 15616, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[31872, 128, 128, 128, 0, 128, 32768, 128, 640, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[384, 32768, 128, 128, 128, 128, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 512, 0, 0, 32768, 32384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 15104, 32768, 0, 128, 128, 0, 16896, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 1, 4092, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 3, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 4069, 4094, 0, 5, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 777, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 28, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3291, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4057, 4096, 0, 0, 0, 0, 0, 0, 4096, 6, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 4096, 4095, 0, 0, 9, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4077, 4095, 0, 4, 0, 0, 0, 0, 4096, 7, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 4, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4063, 4095, 0, 1, 0, 0, 0, 0, 4096, 5, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 19, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 3991, 4092, 0, 4, 0, 0, 0, 0, 4096, 13, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4089, 0, 4090, 4076, 4, 3, 14, 6, 0, 4089, 0, 6, 0, 64, 8, 4026, 4079, 4, 7, 7, 8, 0, 6, 311, 0, 6, 7, 0, 5, 0, 4065, 4088, 0, 5, 836, 32, 6, 0, 4080, 0, 4059, 0, 6, 0, 0, 6, 0, 4090, 0, 18, 2968, 4090, 0, 0, 5, 0, 0, 5, 4086, 0, 4089, 0, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4090, 4096, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 0, 5, 0, 0, 0, 4096, 0, 4080, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4085, 4091, 0, 3, 0, 0, 0, 0, 4096, 15, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 24, 3818, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 3981, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 1, 4092, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4094, 0, 5, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8192, 8053, 8195, 8080, 8192, 8436, 8196] avg=8192.0000 max=8436 max_rank=6 min=8053 min_rank=2 imbalance_ratio=1.029785 max_load_reduction=57100 max_load_reduction_pct=0.871277 moved_to_echo=32904 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 777, 3818, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3291, 4096, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8191, 8807, 8193, 8104, 8206, 7656, 8187] avg=8192.0000 max=8807 max_rank=2 min=7656 min_rank=6 imbalance_ratio=1.075073 max_load_reduction=56729 max_load_reduction_pct=0.865616 moved_to_echo=32153 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 37, 3818, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4090, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 4096, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8192, 8067, 8193, 8078, 8197, 8421, 8193] avg=8192.0000 max=8421 max_rank=6 min=8067 min_rank=2 imbalance_ratio=1.027954 max_load_reduction=57115 max_load_reduction_pct=0.871506 moved_to_echo=32889 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 12, 3818, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4092, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4095, 0, 4, 0, 0, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8193, 8042, 8193, 8085, 8192, 8446, 8195] avg=8192.0000 max=8446 max_rank=6 min=8042 min_rank=2 imbalance_ratio=1.031006 max_load_reduction=57090 max_load_reduction_pct=0.871124 moved_to_echo=32918 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 31, 3818, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 3981, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4095, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4095, 0, 1, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8191, 8060, 8194, 8081, 8198, 8427, 8192] avg=8192.0000 max=8427 max_rank=6 min=8060 min_rank=2 imbalance_ratio=1.028687 max_load_reduction=57109 max_load_reduction_pct=0.871414 moved_to_echo=32898 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 86, 3818, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4087, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 4092, 0, 4, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8195, 8116, 8194, 8095, 8186, 8355, 8200] avg=8192.0000 max=8355 max_rank=6 min=8095 min_rank=4 imbalance_ratio=1.019897 max_load_reduction=57181 max_load_reduction_pct=0.872513 moved_to_echo=32836 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 4076, 4, 3, 14, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 6, 0, 64, 8, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4, 7, 7, 8, 0, 6, 311, 3815, 116, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 5, 0, 4065, 4088, 5, 0, 0, 0, 0, 0, 0, 0, 0, 5, 836, 32, 6, 0, 4080, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 6, 0, 0, 6, 0, 4090, 7, 7, 2, 0, 0, 0, 0, 0, 0, 18, 2968, 4090, 0, 0, 5, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4086, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8186, 8353, 8176, 8932, 8177, 7348, 8171] avg=8192.0000 max=8932 max_rank=4 min=7348 min_rank=6 imbalance_ratio=1.090332 max_load_reduction=56604 max_load_reduction_pct=0.863708 moved_to_echo=32628 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 3819, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 4094, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4091, 0, 3, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8196, 8038, 8198, 8081, 8188, 8447, 8202] avg=8192.0000 max=8447 max_rank=6 min=8038 min_rank=2 imbalance_ratio=1.031128 max_load_reduction=57089 max_load_reduction_pct=0.871109 moved_to_echo=32918 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 0, 128, 0, 0, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 128, 1408, 30592, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 30720, 32768, 0, 128, 128, 0, 2176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 1024, 128, 128, 0, 32768, 0, 31872, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 2920, 4096, 0, 4096, 4096, 4058, 0, 0, 0, 0, 0, 0, 4096, 1015, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 199] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 4096, 0, 2455, 4096, 0, 4096, 4096, 4074, 0, 0, 1, 0, 0, 0, 4096, 82, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 1624] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4096, 4096, 0, 1851, 4096, 0, 4096, 4096, 4072, 0, 0, 0, 0, 0, 0, 4096, 88, 0, 0, 0, 0, 8, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 2188] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 4096, 4096, 0, 3786, 4096, 0, 4096, 4096, 4071, 0, 0, 0, 0, 0, 0, 4096, 152, 0, 0, 0, 0, 10, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 173] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3433, 4096, 0, 4096, 4096, 4014, 0, 0, 0, 0, 0, 0, 4096, 339, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 430] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 3156, 4096, 0, 4096, 4096, 4040, 0, 0, 0, 0, 0, 0, 4096, 313, 0, 0, 0, 0, 5, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 682] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4004, 0, 37, 6, 7, 0, 4090, 4091, 115, 8, 5, 0, 0, 0, 6, 6, 0, 0, 0, 27, 6, 117, 0, 4089, 0, 5, 4067, 4085, 0, 3615, 4042, 19, 4088, 4089, 3808, 0, 0, 0, 0, 7, 8, 4088, 1167, 29, 0, 0, 0, 0, 4082, 4089, 0, 55, 6, 4063, 5, 0, 0, 7, 8, 1675, 9, 7, 0, 1799] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4093, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 5, 0, 4096, 4096, 0, 3265, 4096, 0, 4096, 4096, 4086, 1, 0, 0, 0, 0, 0, 4096, 254, 0, 0, 0, 0, 13, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 576] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 10, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4063, 16, 0, 0, 0, 0, 0, 0, 4, 0, 1039, 0, 0, 3786, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 152, 0, 0, 0, 0, 10, 3638, 23, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 173, 3057, 389, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8176, 8175, 8925, 8236, 7919, 8190, 7717] avg=8192.0000 max=8925 max_rank=3 min=7717 min_rank=7 imbalance_ratio=1.089478 max_load_reduction=56611 max_load_reduction_pct=0.863815 moved_to_echo=32062 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4063, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 0, 0, 2920, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1015, 0, 0, 0, 0, 0, 3638, 23, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 199, 3057, 389, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8177, 8174, 8055, 8223, 8772, 8190, 7743] avg=8192.0000 max=8772 max_rank=5 min=7743 min_rank=7 imbalance_ratio=1.070801 max_load_reduction=56764 max_load_reduction_pct=0.866150 moved_to_echo=32928 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4063, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1039, 0, 0, 3433, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 339, 0, 0, 0, 0, 0, 3638, 23, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 430, 3057, 389, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8177, 8174, 8568, 8179, 8096, 8190, 7950] avg=8192.0000 max=8568 max_rank=3 min=7950 min_rank=7 imbalance_ratio=1.045898 max_load_reduction=56968 max_load_reduction_pct=0.869263 moved_to_echo=32415 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 10, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4063, 16, 0, 0, 0, 0, 0, 0, 1, 0, 1039, 0, 0, 3156, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 313, 0, 0, 0, 0, 5, 3638, 23, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 682, 3057, 389, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8176, 8175, 8292, 8205, 8075, 8190, 8222] avg=8192.0000 max=8292 max_rank=3 min=8075 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=57244 max_load_reduction_pct=0.873474 moved_to_echo=32692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 10, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4063, 15, 0, 0, 0, 0, 0, 0, 3, 0, 1039, 0, 0, 1851, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 88, 0, 0, 0, 0, 8, 3638, 23, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 2188, 3057, 389, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8177, 8174, 6989, 8237, 7853, 8190, 9722] avg=8192.0000 max=9722 max_rank=7 min=6989 min_rank=3 imbalance_ratio=1.186768 max_load_reduction=55814 max_load_reduction_pct=0.851654 moved_to_echo=34002 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 10, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4062, 15, 0, 0, 0, 0, 0, 0, 2, 0, 1039, 0, 0, 2455, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 4074, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 82, 0, 0, 0, 0, 4, 3638, 23, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 1624, 3057, 389, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8177, 8173, 7592, 8240, 7843, 8190, 9123] avg=8192.0000 max=9123 max_rank=7 min=7592 min_rank=3 imbalance_ratio=1.113647 max_load_reduction=56413 max_load_reduction_pct=0.860794 moved_to_echo=33396 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4004, 0, 37, 6, 7, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 115, 8, 5, 0, 0, 0, 6, 6, 4089, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 6, 117, 0, 4089, 4064, 15, 0, 0, 0, 0, 0, 0, 0, 5, 1029, 0, 0, 3615, 4042, 19, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 61, 3808, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4088, 1167, 29, 0, 0, 0, 0, 3636, 26, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 55, 6, 4063, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 1675, 9, 7, 0, 1799, 3038, 392, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8145, 8299, 8318, 8710, 7964, 8954, 8206, 6940] avg=8192.0000 max=8954 max_rank=5 min=6940 min_rank=7 imbalance_ratio=1.093018 max_load_reduction=56582 max_load_reduction_pct=0.863373 moved_to_echo=32064 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 10, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4060, 15, 0, 0, 0, 0, 0, 0, 5, 0, 1039, 0, 0, 3265, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 69, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 254, 0, 0, 0, 0, 13, 3638, 23, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 576, 3057, 389, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8177, 8173, 8405, 8252, 8024, 8190, 8119] avg=8192.0000 max=8405 max_rank=3 min=8024 min_rank=5 imbalance_ratio=1.026001 max_load_reduction=57131 max_load_reduction_pct=0.871750 moved_to_echo=32585 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 640, 32256, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 3456, 128, 0, 0, 0, 128, 29184, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 8320, 0, 0, 24576, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 30336, 128, 128, 0, 7680, 24448, 3200, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 32768, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 39, 4096, 4096, 4096, 0, 0, 0, 3997, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 388, 0, 223, 0, 3545, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 4085, 4096, 4096, 0, 0, 0, 3753, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 26, 0, 689, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 17, 4093, 4096, 4096, 0, 0, 0, 3999, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 7, 0, 625, 0, 3545, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4095, 74, 4089, 4096, 4096, 0, 0, 0, 3838, 0, 0, 4096, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 1, 4095, 91, 0, 312, 0, 3875, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4095, 66, 4090, 4096, 4096, 0, 0, 0, 3839, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 9, 0, 1300, 0, 2975, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 4094, 41, 4091, 4096, 4096, 0, 0, 0, 3905, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 2, 4096, 14, 0, 363, 0, 3862, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4064, 0, 85, 7, 3726, 5, 4, 0, 16, 0, 3, 0, 0, 4090, 638, 4071, 4037, 4075, 0, 0, 0, 4074, 0, 7, 4088, 0, 0, 6, 17, 4083, 4088, 0, 8, 0, 11, 5, 0, 0, 0, 0, 4089, 4034, 0, 4088, 6, 7, 3752, 95, 84, 0, 9, 0, 6, 9, 0, 0, 5, 4089, 10, 4019, 0, 21, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4093, 33, 4090, 4096, 4096, 0, 0, 0, 3732, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 3, 4096, 225, 0, 716, 0, 3481, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4094, 41, 2932, 1047, 0, 0, 0, 0, 0, 0, 4091, 213, 0, 0, 0, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 3011, 14, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3862, 0, 0, 0, 0, 0, 0, 4096, 951, 2, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8120, 8209, 8191, 8191, 7486, 8911, 8199] avg=8192.0000 max=8911 max_rank=6 min=7486 min_rank=5 imbalance_ratio=1.087769 max_load_reduction=56625 max_load_reduction_pct=0.864029 moved_to_echo=32787 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 39, 2932, 1047, 0, 0, 0, 0, 0, 0, 4096, 213, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3011, 388, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 0, 0, 0, 0, 0, 0, 4096, 951, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8114, 8306, 8191, 8190, 7718, 8594, 8194] avg=8192.0000 max=8594 max_rank=6 min=7718 min_rank=5 imbalance_ratio=1.049072 max_load_reduction=56942 max_load_reduction_pct=0.868866 moved_to_echo=32696 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 2932, 1047, 0, 0, 0, 0, 0, 0, 4085, 213, 0, 0, 0, 0, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3011, 26, 0, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 0, 4096, 951, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8122, 8051, 8191, 8190, 7822, 8737, 8194] avg=8192.0000 max=8737 max_rank=6 min=7822 min_rank=5 imbalance_ratio=1.066528 max_load_reduction=56799 max_load_reduction_pct=0.866684 moved_to_echo=32943 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 17, 2932, 1047, 0, 0, 0, 0, 0, 0, 4093, 213, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3011, 7, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 0, 0, 0, 0, 0, 0, 4096, 951, 2, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8091, 8305, 8191, 8190, 7740, 8594, 8196] avg=8192.0000 max=8594 max_rank=6 min=7740 min_rank=5 imbalance_ratio=1.049072 max_load_reduction=56942 max_load_reduction_pct=0.868866 moved_to_echo=32720 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4095, 66, 2932, 1047, 0, 0, 0, 0, 0, 0, 4090, 213, 0, 0, 0, 0, 3839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3011, 9, 0, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2975, 0, 0, 0, 0, 0, 0, 4096, 951, 2, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8145, 8142, 8191, 8190, 8416, 8024, 8199] avg=8192.0000 max=8416 max_rank=5 min=8024 min_rank=6 imbalance_ratio=1.027344 max_load_reduction=57120 max_load_reduction_pct=0.871582 moved_to_echo=32829 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 4096, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4095, 74, 2932, 1047, 0, 0, 0, 0, 0, 0, 4089, 213, 0, 0, 0, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3010, 91, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 0, 0, 0, 0, 0, 0, 4096, 951, 2, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8230, 8153, 8140, 8193, 8191, 7510, 8924, 8195] avg=8192.0000 max=8924 max_rank=6 min=7510 min_rank=5 imbalance_ratio=1.089355 max_load_reduction=56612 max_load_reduction_pct=0.863831 moved_to_echo=32820 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4064, 0, 85, 7, 3726, 5, 4, 41, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3, 0, 0, 4090, 638, 2890, 1047, 0, 0, 0, 0, 0, 0, 4071, 203, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4084, 0, 0, 6, 17, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 11, 5, 0, 0, 0, 4089, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 2994, 6, 7, 3752, 0, 0, 0, 0, 0, 0, 0, 0, 95, 84, 0, 9, 0, 6, 9, 0, 4075, 944, 1, 0, 0, 0, 0, 0, 0, 5, 4089, 10, 4019, 0, 21, 5, 6, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7932, 8684, 8348, 8197, 8200, 10793, 5223, 8159] avg=8192.0000 max=10793 max_rank=5 min=5223 min_rank=6 imbalance_ratio=1.317505 max_load_reduction=54743 max_load_reduction_pct=0.835312 moved_to_echo=32375 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4093, 33, 2932, 1047, 0, 0, 0, 0, 0, 0, 4090, 213, 0, 0, 0, 0, 3732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 3011, 225, 0, 716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3481, 0, 0, 0, 0, 0, 0, 4096, 951, 1, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 4096, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8229, 8107, 8035, 8191, 8194, 8051, 8529, 8200] avg=8192.0000 max=8529 max_rank=6 min=8035 min_rank=2 imbalance_ratio=1.041138 max_load_reduction=57007 max_load_reduction_pct=0.869858 moved_to_echo=32974 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 1792, 0, 0, 0, 0, 31232, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 32768, 128, 128, 128, 32512, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 25088, 0, 128, 0, 128, 128, 0, 32768, 7680, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 24192, 768, 128, 8064, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 1024, 23424, 8448, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3085, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 3982, 0, 3, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 4093, 1116, 0, 2, 4096, 0, 8, 4095, 4096, 4, 0, 4090, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3364, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3822, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4043, 4096, 1068, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4087, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3439, 0, 16, 0, 0, 0, 2, 0, 0, 4096, 0, 3894, 0, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3890, 4090, 1044, 0, 6, 4096, 0, 9, 4096, 4096, 5, 0, 4087, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 1691, 0, 4, 0, 0, 0, 4, 0, 0, 4096, 0, 3949, 0, 1, 4096, 0, 0, 0, 4095, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 3971, 4093, 2665, 0, 2, 4096, 0, 11, 4096, 4096, 4, 0, 4094, 0, 0, 0, 4094, 4096, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3875, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 4035, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4026, 4096, 336, 0, 0, 4096, 0, 5, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2275, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4012, 0, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4038, 4095, 1957, 0, 5, 4096, 0, 3, 4094, 4096, 2, 0, 4093, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[7, 0, 7, 451, 4, 0, 9, 0, 3514, 0, 7, 0, 0, 3979, 0, 0, 18, 0, 6, 66, 6, 7, 4028, 12, 4039, 9, 0, 4064, 0, 0, 6, 4089, 4088, 0, 0, 4048, 0, 5, 0, 44, 24, 3896, 4081, 481, 7, 0, 4074, 0, 0, 4081, 4068, 0, 0, 4088, 7, 0, 8, 4071, 4086, 0, 0, 35, 0, 16] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 3367, 0, 9, 0, 0, 0, 1, 0, 0, 4096, 0, 3866, 0, 4, 4096, 0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 4090, 949, 0, 5, 4096, 0, 13, 4095, 4096, 6, 0, 4093, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 4041, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3085, 0, 10, 1027, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3948, 86, 55, 0, 0, 0, 0, 0, 3982, 0, 3, 4096, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2980, 1116, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4095, 4, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8136, 8217, 8185, 8229, 8193, 8190, 8197, 8189] avg=8192.0000 max=8229 max_rank=3 min=8136 min_rank=0 imbalance_ratio=1.004517 max_load_reduction=57307 max_load_reduction_pct=0.874435 moved_to_echo=32769 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4042, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3364, 0, 0, 1028, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3948, 86, 54, 0, 0, 0, 0, 0, 3822, 0, 0, 4096, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 2982, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8490, 8184, 8066, 8193, 8093, 8182, 8190] avg=8192.0000 max=8490 max_rank=1 min=8066 min_rank=3 imbalance_ratio=1.036377 max_load_reduction=57046 max_load_reduction_pct=0.870453 moved_to_echo=32658 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4041, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3439, 0, 16, 1026, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 3948, 86, 55, 0, 0, 0, 0, 0, 3894, 0, 1, 4096, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 2978, 1044, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4095, 5, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8577, 8187, 8139, 8193, 7918, 8196, 8189] avg=8192.0000 max=8577 max_rank=1 min=7918 min_rank=5 imbalance_ratio=1.046997 max_load_reduction=56959 max_load_reduction_pct=0.869125 moved_to_echo=32496 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4041, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 1691, 0, 4, 1027, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 3948, 86, 55, 0, 0, 0, 0, 0, 3949, 0, 1, 4096, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 2980, 2665, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4095, 4, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 6814, 8189, 8193, 8192, 9618, 8204, 8189] avg=8192.0000 max=9618 max_rank=5 min=6814 min_rank=1 imbalance_ratio=1.174072 max_load_reduction=55918 max_load_reduction_pct=0.853241 moved_to_echo=34203 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4041, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3875, 0, 11, 1028, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3948, 86, 55, 0, 0, 0, 0, 0, 4035, 0, 0, 4096, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 2982, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 9012, 8185, 8279, 8193, 7344, 8196, 8190] avg=8192.0000 max=9012 max_rank=1 min=7344 min_rank=5 imbalance_ratio=1.100098 max_load_reduction=56524 max_load_reduction_pct=0.862488 moved_to_echo=31923 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 4041, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2275, 0, 2, 1028, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3948, 86, 55, 0, 0, 0, 0, 0, 4012, 0, 1, 4096, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 2981, 1957, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 7403, 8185, 8257, 8193, 8981, 8193, 8189] avg=8192.0000 max=8981 max_rank=5 min=7403 min_rank=1 imbalance_ratio=1.096313 max_load_reduction=56555 max_load_reduction_pct=0.862961 moved_to_echo=33556 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 7, 451, 4, 0, 9, 0, 4081, 4020, 0, 0, 0, 0, 0, 0, 3514, 0, 7, 0, 0, 3979, 0, 0, 1029, 2, 0, 0, 0, 0, 0, 0, 18, 0, 6, 66, 6, 7, 4028, 12, 3948, 91, 54, 0, 0, 0, 0, 0, 4039, 9, 0, 4064, 0, 0, 6, 141, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4048, 0, 5, 0, 44, 2, 1, 0, 0, 0, 0, 0, 0, 24, 3896, 2961, 481, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4071, 4083, 0, 0, 35, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8579, 8531, 8236, 8259, 8188, 7369, 8161, 8213] avg=8192.0000 max=8579 max_rank=0 min=7369 min_rank=5 imbalance_ratio=1.047241 max_load_reduction=56957 max_load_reduction_pct=0.869095 moved_to_echo=31931 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 4042, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 3367, 0, 9, 1026, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 3948, 86, 54, 0, 0, 0, 0, 0, 3866, 0, 4, 4096, 0, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2978, 949, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4095, 6, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8137, 8492, 8185, 8114, 8191, 8023, 8207, 8187] avg=8192.0000 max=8492 max_rank=1 min=8023 min_rank=5 imbalance_ratio=1.036621 max_load_reduction=57044 max_load_reduction_pct=0.870422 moved_to_echo=32608 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32256, 0, 128, 0, 0, 25088, 0, 128, 8320, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 128, 31616, 768, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32128, 23936, 9728, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[31616, 128, 128, 32768, 0, 0, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 512, 128, 0, 128, 0, 32768, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 32768, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 13, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 1279, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1208, 0, 1284, 355, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 912, 4082, 0, 0, 0, 0, 5, 4096, 0, 4096, 4096, 0, 0, 4079, 0, 0, 0, 756, 0, 2263, 189, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 20, 0, 4096, 1, 0, 0, 0, 4096, 4094, 4095, 1, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 1692, 4065, 0, 0, 0, 0, 2, 4096, 0, 4095, 4096, 0, 0, 4079, 0, 0, 0, 390, 0, 2224, 81, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 32, 0, 4096, 0, 0, 0, 0, 4096, 4095, 4094, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 2787, 4093, 0, 0, 0, 0, 3, 4096, 0, 4096, 4096, 0, 0, 4056, 0, 0, 0, 435, 0, 923, 76, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 3383, 4082, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4082, 0, 0, 0, 179, 0, 699, 29, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1718, 4087, 0, 0, 0, 0, 2, 4096, 0, 4096, 4096, 0, 0, 4091, 0, 0, 0, 606, 0, 1617, 166, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 37, 0, 4096, 0, 0, 0, 0, 4096, 4095, 4091, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 2735, 4084, 0, 0, 0, 0, 11, 4096, 0, 4096, 4096, 0, 0, 4067, 0, 0, 0, 338, 0, 989, 108, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4076, 0, 0, 0, 4089, 4084, 4045, 0, 5, 9, 0, 4090, 13, 0, 0, 6, 4087, 0, 607, 0, 2, 4084, 4089, 4089, 0, 7, 0, 35, 0, 7, 317, 7, 0, 0, 7, 17, 20, 233, 5, 8, 363, 2749, 4, 7, 6, 2, 0, 4089, 6, 4085, 4077, 0, 6, 4088, 6, 0, 6, 3547, 0, 1955, 2497, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 13, 0, 2322, 1684, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 2910, 1757, 0, 0, 0, 0, 0, 0, 1279, 4095, 0, 0, 0, 0, 0, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1208, 0, 1284, 355, 0, 0, 0, 4096, 1176, 55, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8186, 8121, 8138, 8720, 7786, 8209, 8174] avg=8192.0000 max=8720 max_rank=4 min=7786 min_rank=5 imbalance_ratio=1.064453 max_load_reduction=56816 max_load_reduction_pct=0.866943 moved_to_echo=32889 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 32, 0, 2322, 1684, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 2910, 1757, 0, 0, 0, 0, 0, 0, 2787, 4093, 0, 0, 0, 0, 3, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 923, 76, 0, 0, 0, 4096, 1176, 56, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8205, 8119, 8150, 8634, 9295, 8169, 6762] avg=8192.0000 max=9295 max_rank=5 min=6762 min_rank=7 imbalance_ratio=1.134644 max_load_reduction=56241 max_load_reduction_pct=0.858170 moved_to_echo=32860 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 2322, 1684, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 2910, 1757, 0, 0, 0, 0, 0, 0, 3383, 4082, 0, 0, 0, 0, 0, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 699, 29, 0, 0, 0, 4096, 1176, 56, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8177, 8119, 8149, 8582, 9877, 8195, 6235] avg=8192.0000 max=9877 max_rank=5 min=6235 min_rank=7 imbalance_ratio=1.205688 max_load_reduction=55659 max_load_reduction_pct=0.849289 moved_to_echo=32889 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 2322, 1684, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 2910, 1757, 0, 0, 0, 0, 0, 0, 912, 4082, 0, 0, 0, 0, 5, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 756, 0, 2263, 189, 0, 0, 0, 4096, 1176, 55, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8187, 8114, 8154, 8741, 7411, 8192, 8535] avg=8192.0000 max=8741 max_rank=4 min=7411 min_rank=5 imbalance_ratio=1.067017 max_load_reduction=56795 max_load_reduction_pct=0.866623 moved_to_echo=32879 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 20, 0, 2322, 1684, 71, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4025, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 11, 0, 0, 0, 4096, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 2909, 1756, 0, 0, 0, 0, 0, 0, 1692, 4065, 0, 0, 0, 0, 2, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 2224, 81, 1, 0, 0, 4096, 1176, 55, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8192, 8121, 8148, 8487, 8171, 8192, 8023] avg=8192.0000 max=8487 max_rank=4 min=8023 min_rank=7 imbalance_ratio=1.036011 max_load_reduction=57049 max_load_reduction_pct=0.870499 moved_to_echo=32873 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 2322, 1684, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 2909, 1757, 0, 0, 0, 0, 0, 0, 1718, 4087, 0, 0, 0, 0, 2, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 1617, 166, 0, 0, 0, 4096, 1176, 56, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8178, 8117, 8140, 8759, 8219, 8204, 7717] avg=8192.0000 max=8759 max_rank=4 min=7717 min_rank=7 imbalance_ratio=1.069214 max_load_reduction=56777 max_load_reduction_pct=0.866348 moved_to_echo=32899 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4076, 0, 0, 0, 1, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 9, 0, 4090, 13, 0, 0, 6, 2320, 1685, 73, 0, 0, 0, 0, 0, 0, 0, 607, 0, 2, 4011, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 35, 0, 7, 317, 7, 4087, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 7, 17, 20, 233, 5, 8, 2909, 1753, 0, 0, 0, 0, 0, 0, 363, 2749, 4, 7, 6, 2, 0, 2404, 0, 0, 0, 0, 0, 0, 0, 0, 6, 12, 4077, 0, 6, 4088, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3547, 0, 1955, 2497, 0, 5, 0, 4084, 1179, 55, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8201, 8709, 8494, 4952, 5535, 8195, 13328] avg=8192.0000 max=13328 max_rank=7 min=4952 min_rank=4 imbalance_ratio=1.626953 max_load_reduction=52208 max_load_reduction_pct=0.796631 moved_to_echo=32010 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 37, 0, 2322, 1684, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 2907, 1757, 0, 0, 0, 0, 0, 0, 2735, 4084, 0, 0, 0, 0, 11, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 989, 108, 1, 0, 0, 4096, 1175, 55, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8210, 8116, 8163, 8661, 9242, 8180, 6762] avg=8192.0000 max=9242 max_rank=5 min=6762 min_rank=7 imbalance_ratio=1.128174 max_load_reduction=56294 max_load_reduction_pct=0.858978 moved_to_echo=32845 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[14976, 31360, 128, 128, 128, 128, 128, 19328, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 7552, 0, 12032, 3584, 128, 128, 0, 32768, 9472, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 640, 0, 128, 32256, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 384, 128, 32768, 32384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 128, 128, 18688, 13568, 640, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 256, 32768, 0, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 28160, 128, 128, 23296, 14080, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4053, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 3, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2, 4092, 4096, 0, 4096, 4089, 0, 0, 4096, 0, 4092, 0, 5, 0, 0, 4061, 42, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4074, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4073, 22, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4082, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 4080, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4075, 21, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4062, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 26, 0, 0, 0, 7, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 4096, 4067, 0, 0, 4096, 0, 4090, 0, 6, 0, 0, 3981, 51, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4083, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 28, 0, 0, 0, 5, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 4095, 4096, 0, 4096, 4080, 0, 0, 4096, 0, 4094, 0, 1, 0, 0, 3990, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4064, 57, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 85, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 2, 4095, 4095, 0, 4096, 4091, 0, 0, 4096, 0, 4093, 0, 5, 0, 0, 3983, 5, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4068, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 47, 0, 0, 0, 3, 4093, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 5, 4091, 4096, 0, 4096, 4079, 0, 0, 4096, 0, 4090, 0, 6, 0, 0, 4067, 17, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[3813, 2711, 13, 0, 0, 5, 0, 0, 9, 8, 8, 6, 6, 0, 4075, 150, 0, 0, 1, 0, 0, 4091, 786, 3637, 0, 0, 4089, 10, 0, 0, 0, 0, 4089, 4088, 0, 4088, 4088, 8, 150, 4083, 224, 4090, 0, 0, 2, 12, 957, 0, 7, 4089, 0, 5, 6, 0, 4081, 2, 7, 7, 9, 1, 3, 3927, 4087, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4053, 32, 0, 0, 0, 0, 0, 0, 3753, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 3, 4027, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 5, 0, 0, 4061, 42, 343, 55, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7858, 8172, 8137, 8194, 8177, 8598, 8192, 8208] avg=8192.0000 max=8598 max_rank=5 min=7858 min_rank=0 imbalance_ratio=1.049561 max_load_reduction=56938 max_load_reduction_pct=0.868805 moved_to_echo=33175 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4074, 21, 0, 0, 0, 0, 0, 0, 3754, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4075, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 4027, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4073, 22, 342, 55, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7870, 8172, 8126, 8191, 8187, 8588, 8192, 8210] avg=8192.0000 max=8588 max_rank=5 min=7870 min_rank=0 imbalance_ratio=1.048340 max_load_reduction=56948 max_load_reduction_pct=0.868958 moved_to_echo=33177 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4082, 11, 0, 0, 0, 0, 0, 0, 3753, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 1, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 1, 4027, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4075, 21, 343, 55, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7866, 8173, 8143, 8191, 8171, 8590, 8192, 8210] avg=8192.0000 max=8590 max_rank=5 min=7866 min_rank=0 imbalance_ratio=1.048584 max_load_reduction=56946 max_load_reduction_pct=0.868927 moved_to_echo=33163 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4062, 99, 0, 0, 0, 0, 0, 0, 3754, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 1, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 7, 4027, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 3981, 51, 342, 55, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7935, 8173, 8156, 8191, 8154, 8525, 8192, 8210] avg=8192.0000 max=8525 max_rank=5 min=7935 min_rank=0 imbalance_ratio=1.040649 max_load_reduction=57011 max_load_reduction_pct=0.869919 moved_to_echo=33081 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4083, 97, 0, 0, 0, 0, 0, 0, 3753, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 5, 4027, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 3990, 6, 343, 55, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7953, 8172, 8156, 8193, 8171, 8489, 8192, 8210] avg=8192.0000 max=8489 max_rank=5 min=7953 min_rank=0 imbalance_ratio=1.036255 max_load_reduction=57047 max_load_reduction_pct=0.870468 moved_to_echo=33062 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4064, 57, 0, 0, 0, 0, 0, 1, 3754, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4075, 1, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 4026, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 5, 0, 0, 3983, 5, 342, 55, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 14, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7896, 8172, 8207, 8194, 8182, 8483, 8192, 8210] avg=8192.0000 max=8483 max_rank=5 min=7896 min_rank=0 imbalance_ratio=1.035522 max_load_reduction=57053 max_load_reduction_pct=0.870560 moved_to_echo=33067 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3813, 2711, 13, 0, 0, 5, 0, 0, 3742, 21, 0, 0, 0, 0, 0, 0, 9, 8, 8, 6, 6, 0, 4075, 150, 4067, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4016, 786, 3637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 10, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4088, 8, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 4090, 0, 0, 2, 12, 957, 0, 341, 60, 0, 0, 0, 0, 0, 0, 7, 4089, 0, 5, 6, 0, 4081, 2, 2, 0, 0, 0, 0, 0, 0, 0, 7, 7, 9, 1, 3, 3927, 4087, 8, 15, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10305, 8330, 8440, 8186, 8328, 5686, 8192, 8069] avg=8192.0000 max=10305 max_rank=0 min=5686 min_rank=5 imbalance_ratio=1.257935 max_load_reduction=55231 max_load_reduction_pct=0.842758 moved_to_echo=30275 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4068, 10, 0, 0, 0, 0, 0, 0, 3754, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4075, 1, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 3, 4025, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 4067, 17, 342, 55, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 13, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7853, 8172, 8171, 8196, 8166, 8577, 8192, 8209] avg=8192.0000 max=8577 max_rank=5 min=7853 min_rank=0 imbalance_ratio=1.046997 max_load_reduction=56959 max_load_reduction_pct=0.869125 moved_to_echo=33144 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32384, 3072, 128, 0, 0, 128, 0, 128, 30080, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[256, 32768, 0, 128, 128, 128, 29312, 256, 2816, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 0, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 0, 128, 0, 128, 32256, 896, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 32640, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 256, 32640, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 4020, 0, 0, 0, 0, 240, 0, 4096, 0, 4096, 3934, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 4082, 0, 0, 0, 0, 400, 0, 4096, 0, 4081, 3729, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4092, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4033, 0, 0, 0, 0, 114, 0, 4096, 0, 4088, 4054, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 16, 0, 4095, 0, 4094, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4084, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 227, 0, 4095, 0, 4079, 3938, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4052, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 4051, 0, 0, 0, 0, 135, 0, 4095, 0, 4095, 4011, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4095, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[11, 4089, 6, 13, 5, 168, 1314, 0, 4089, 6, 4086, 2384, 12, 0, 0, 0, 4087, 0, 67, 3988, 5, 4085, 1, 17, 2, 3, 25, 9, 0, 3998, 0, 4083, 0, 7, 5, 4090, 0, 350, 0, 0, 8, 0, 8, 0, 0, 826, 8, 5, 19, 9, 4064, 0, 0, 11, 4088, 3250, 0, 4052, 0, 4084, 4079, 0, 13, 7] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4053, 0, 0, 0, 0, 296, 0, 4096, 0, 4071, 3872, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4092, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 0, 0, 0, 0, 400, 0, 3735, 25, 0, 0, 0, 0, 0, 0, 353, 0, 4081, 3729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 3957, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 114, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8242, 8163, 8195, 8200, 8149, 8087, 8302, 8198] avg=8192.0000 max=8302 max_rank=6 min=8087 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4020, 0, 0, 0, 0, 240, 0, 3735, 25, 0, 0, 0, 0, 0, 0, 353, 0, 4096, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3957, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 114, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8020, 8383, 8197, 8200, 8147, 8087, 8304, 8198] avg=8192.0000 max=8383 max_rank=1 min=8020 min_rank=0 imbalance_ratio=1.023315 max_load_reduction=57153 max_load_reduction_pct=0.872086 moved_to_echo=32736 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4033, 0, 0, 0, 0, 114, 0, 3735, 25, 0, 0, 0, 0, 0, 0, 353, 0, 4088, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3957, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 114, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7907, 8495, 8196, 8200, 8147, 8087, 8306, 8198] avg=8192.0000 max=8495 max_rank=1 min=7907 min_rank=0 imbalance_ratio=1.036987 max_load_reduction=57041 max_load_reduction_pct=0.870377 moved_to_echo=32738 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 16, 0, 3734, 25, 0, 0, 0, 0, 0, 0, 353, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3957, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 114, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7870, 8543, 8195, 8201, 8148, 8087, 8294, 8198] avg=8192.0000 max=8543 max_rank=1 min=7870 min_rank=0 imbalance_ratio=1.042847 max_load_reduction=56993 max_load_reduction_pct=0.869644 moved_to_echo=32727 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 0, 0, 0, 227, 0, 3734, 25, 0, 0, 0, 0, 0, 0, 353, 0, 4079, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3957, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 114, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8076, 8370, 8195, 8201, 8147, 8087, 8262, 8198] avg=8192.0000 max=8370 max_rank=1 min=8076 min_rank=0 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32694 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4051, 0, 0, 0, 0, 135, 0, 3734, 25, 0, 0, 0, 0, 0, 0, 353, 0, 4095, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4095, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3957, 94, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4091, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 114, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7945, 8459, 8195, 8200, 8148, 8088, 8303, 8198] avg=8192.0000 max=8459 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.032593 max_load_reduction=57077 max_load_reduction_pct=0.870926 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[11, 4089, 6, 13, 5, 168, 1314, 0, 3736, 25, 0, 0, 0, 0, 0, 0, 339, 6, 4086, 2384, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 67, 3988, 5, 4, 1, 17, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 25, 9, 0, 3998, 0, 4083, 14, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 4090, 0, 350, 0, 0, 3953, 95, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 826, 8, 5, 4081, 3989, 0, 0, 0, 0, 0, 0, 19, 9, 4064, 0, 0, 11, 110, 3250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 4079, 0, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9367, 6827, 8169, 8134, 8500, 8925, 7463, 8151] avg=8192.0000 max=9367 max_rank=0 min=6827 min_rank=1 imbalance_ratio=1.143433 max_load_reduction=56169 max_load_reduction_pct=0.857071 moved_to_echo=33039 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4053, 0, 0, 0, 0, 296, 0, 3735, 25, 0, 0, 0, 0, 0, 0, 353, 0, 4071, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 3957, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 114, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8109, 8296, 8194, 8200, 8150, 8088, 8302, 8197] avg=8192.0000 max=8302 max_rank=6 min=8088 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=57234 max_load_reduction_pct=0.873322 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 896, 128, 128, 32768, 32000, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 384, 0, 0, 31744, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 1024, 31872, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32640, 128, 128, 128, 256, 2816, 0, 29952, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[2816, 128, 32768, 30080, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True + [2026-08-30 04:59:32] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 716.0 | throughput per GPU (TFLOP/s/GPU): 465.6 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.093822E+00 | loss scale: 1.0 | grad norm: 5.401 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1998, 135, 786, 340, 1075, 1, 2148, 81, 665, 3329, 581, 757, 1882, 1954, 1, 226, 294, 1305, 555, 928, 0, 554, 1604, 1484, 64, 26, 3074, 1208, 2690, 613, 2104, 774, 5, 1708, 2007, 176, 520, 735, 2213, 1938, 473, 6, 1915, 1317, 1, 1627, 1384, 915, 2486, 33, 1309, 1, 63, 888, 1, 734, 0, 289, 864, 1363, 2295, 1267, 1282, 2485] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2401, 39, 997, 389, 1073, 0, 2107, 146, 1089, 3261, 638, 530, 1701, 1620, 1, 304, 375, 1367, 378, 1338, 0, 684, 1704, 1080, 80, 33, 2855, 1104, 2804, 486, 2749, 653, 3, 1900, 2339, 300, 386, 892, 2026, 1661, 660, 0, 1778, 1464, 0, 1451, 1347, 894, 2580, 23, 1457, 3, 35, 771, 4, 445, 0, 466, 1441, 587, 2419, 857, 944, 2417] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2838, 98, 524, 300, 1086, 0, 1912, 79, 1372, 2745, 833, 757, 1351, 1835, 0, 311, 517, 881, 751, 750, 0, 940, 1645, 1345, 153, 76, 2677, 1141, 1749, 644, 2340, 782, 1, 1126, 2440, 166, 501, 872, 1614, 1443, 890, 0, 1745, 2210, 0, 1933, 779, 830, 2539, 32, 1937, 20, 191, 879, 1, 478, 4, 367, 1618, 1279, 2395, 1243, 1257, 2314] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2183, 81, 596, 330, 1114, 0, 1861, 209, 567, 2830, 329, 1221, 2124, 1495, 0, 418, 332, 1353, 1003, 1559, 0, 600, 2309, 1463, 68, 99, 2668, 1349, 2980, 729, 2107, 660, 12, 1352, 2180, 202, 464, 732, 1968, 1455, 808, 0, 1996, 1270, 1, 1349, 992, 1210, 2886, 33, 1406, 2, 90, 954, 33, 618, 0, 192, 996, 664, 2553, 991, 823, 2667] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2683, 87, 397, 561, 1137, 0, 1555, 284, 878, 2740, 283, 919, 1400, 2072, 0, 343, 529, 1188, 712, 618, 0, 720, 1648, 1211, 71, 21, 2874, 1418, 2119, 736, 2413, 883, 3, 1373, 2508, 333, 754, 886, 2098, 1618, 608, 2, 1903, 1670, 1, 1460, 1144, 936, 2912, 26, 1401, 9, 105, 1203, 9, 1090, 0, 271, 1014, 938, 2411, 1005, 1433, 1912] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2482, 41, 408, 231, 827, 0, 1700, 95, 1420, 3422, 656, 440, 1324, 2186, 0, 242, 679, 1680, 354, 1184, 0, 998, 1900, 1698, 44, 53, 2489, 958, 2329, 679, 2195, 799, 2, 1950, 2229, 260, 510, 716, 2075, 1374, 724, 0, 1519, 1941, 0, 1577, 1024, 1232, 2656, 47, 1586, 6, 12, 984, 0, 391, 0, 331, 1016, 656, 2288, 934, 1283, 2700] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2289, 57, 633, 153, 1616, 1, 2463, 143, 654, 2658, 561, 563, 1844, 1163, 0, 175, 253, 1571, 1165, 1287, 1, 670, 2504, 1332, 53, 35, 2985, 1570, 2212, 522, 2476, 1152, 5, 1807, 1607, 169, 504, 566, 1745, 1730, 685, 6, 1895, 1048, 2, 1696, 1183, 876, 2622, 70, 1170, 5, 97, 668, 0, 648, 3, 256, 1239, 1119, 2710, 958, 1032, 2654] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2054, 129, 727, 510, 1006, 5, 1703, 136, 1038, 2819, 499, 496, 2002, 1271, 0, 419, 527, 1223, 941, 1277, 1, 727, 2072, 1624, 54, 31, 2723, 1016, 2377, 718, 2500, 747, 9, 1924, 2261, 203, 368, 663, 1707, 1380, 749, 4, 1904, 1617, 1, 1154, 1700, 795, 2324, 8, 1719, 29, 56, 781, 7, 900, 7, 404, 1453, 1219, 2345, 1049, 877, 2547] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2183, 81, 596, 330, 1114, 0, 1861, 209, 1226, 408, 0, 0, 0, 0, 0, 0, 567, 2156, 329, 1221, 2124, 1495, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 332, 1353, 1003, 1559, 0, 600, 2309, 1463, 266, 236, 0, 0, 0, 0, 0, 0, 68, 99, 608, 1349, 2980, 729, 2107, 660, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1352, 1543, 202, 464, 732, 1968, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 808, 0, 1996, 1270, 1, 1349, 992, 1210, 401, 0, 0, 0, 0, 0, 0, 0, 2886, 33, 1406, 2, 90, 954, 33, 618, 2060, 244, 0, 0, 0, 0, 0, 0, 0, 192, 996, 664, 2553, 991, 823, 1197, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8008, 8310, 9121, 8600, 7728, 8027, 8326, 7416] avg=8192.0000 max=9121 max_rank=2 min=7416 min_rank=7 imbalance_ratio=1.113403 max_load_reduction=56415 max_load_reduction_pct=0.860825 moved_to_echo=31497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2401, 39, 997, 389, 1073, 0, 2107, 146, 1111, 470, 0, 0, 0, 0, 0, 0, 1089, 2484, 638, 530, 1701, 1620, 1, 304, 0, 0, 0, 0, 0, 0, 0, 0, 375, 1367, 378, 1338, 0, 684, 1704, 1080, 307, 253, 0, 0, 0, 0, 0, 0, 80, 33, 651, 1104, 2804, 486, 2749, 653, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1900, 1656, 300, 386, 892, 2026, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 1778, 1464, 0, 1451, 1347, 894, 430, 0, 0, 0, 0, 0, 0, 0, 2580, 23, 1457, 3, 35, 771, 4, 445, 2204, 221, 0, 0, 0, 0, 0, 0, 0, 466, 1441, 587, 2419, 857, 944, 1085, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8733, 8367, 7486, 8560, 8824, 8024, 7743, 7799] avg=8192.0000 max=8824 max_rank=4 min=7486 min_rank=2 imbalance_ratio=1.077148 max_load_reduction=56712 max_load_reduction_pct=0.865356 moved_to_echo=32390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2838, 98, 524, 300, 1086, 0, 1912, 79, 1064, 396, 0, 0, 0, 0, 0, 0, 1372, 2091, 833, 757, 1351, 1835, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 517, 881, 751, 750, 0, 940, 1645, 1345, 258, 264, 0, 0, 0, 0, 0, 0, 153, 76, 610, 1141, 1749, 644, 2340, 782, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1126, 1727, 166, 501, 872, 1614, 1443, 0, 0, 0, 0, 0, 0, 0, 0, 890, 0, 1745, 2210, 0, 1933, 779, 830, 449, 0, 0, 0, 0, 0, 0, 0, 2539, 32, 1937, 20, 191, 879, 1, 478, 2067, 212, 0, 0, 0, 0, 0, 0, 4, 367, 1618, 1279, 2395, 1243, 1257, 1038, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8297, 8550, 7351, 7495, 7450, 8836, 8356, 9201] avg=8192.0000 max=9201 max_rank=7 min=7351 min_rank=2 imbalance_ratio=1.123169 max_load_reduction=56335 max_load_reduction_pct=0.859604 moved_to_echo=33843 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1998, 135, 786, 340, 1075, 1, 2148, 81, 1142, 480, 0, 0, 0, 0, 0, 0, 665, 2536, 581, 757, 1882, 1954, 1, 226, 0, 0, 0, 0, 0, 0, 0, 0, 294, 1305, 555, 928, 0, 554, 1604, 1484, 313, 217, 0, 0, 0, 0, 0, 0, 64, 26, 701, 1208, 2690, 613, 2104, 774, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1708, 1421, 176, 520, 735, 2213, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 473, 6, 1915, 1317, 1, 1627, 1384, 915, 369, 0, 0, 0, 0, 0, 0, 0, 2486, 33, 1309, 1, 63, 888, 1, 734, 2373, 228, 0, 0, 0, 0, 0, 0, 0, 289, 864, 1363, 2295, 1267, 1282, 1115, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8602, 7254, 8180, 8716, 8007, 8116, 8475] avg=8192.0000 max=8716 max_rank=4 min=7254 min_rank=2 imbalance_ratio=1.063965 max_load_reduction=56820 max_load_reduction_pct=0.867004 moved_to_echo=33314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2683, 87, 397, 561, 1137, 0, 1555, 284, 879, 395, 0, 0, 0, 0, 0, 0, 878, 2088, 283, 919, 1400, 2072, 0, 343, 0, 0, 0, 0, 0, 0, 0, 0, 529, 1188, 712, 618, 0, 720, 1648, 1211, 257, 271, 0, 0, 0, 0, 0, 0, 71, 21, 655, 1418, 2119, 736, 2413, 883, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1373, 1776, 333, 754, 886, 2098, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 608, 2, 1903, 1670, 1, 1460, 1144, 936, 461, 0, 0, 0, 0, 0, 0, 0, 2912, 26, 1401, 9, 105, 1203, 9, 1090, 2219, 175, 0, 0, 0, 0, 0, 0, 0, 271, 1014, 938, 2411, 1005, 1433, 858, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7978, 7983, 7154, 8316, 8841, 8185, 9149, 7930] avg=8192.0000 max=9149 max_rank=6 min=7154 min_rank=2 imbalance_ratio=1.116821 max_load_reduction=56387 max_load_reduction_pct=0.860397 moved_to_echo=34105 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2482, 41, 408, 231, 827, 0, 1700, 95, 1241, 494, 0, 0, 0, 0, 0, 0, 1420, 2606, 656, 440, 1324, 2186, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 679, 1680, 354, 1184, 0, 998, 1900, 1698, 322, 241, 0, 0, 0, 0, 0, 0, 44, 53, 567, 958, 2329, 679, 2195, 799, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1950, 1578, 260, 510, 716, 2075, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 724, 0, 1519, 1941, 0, 1577, 1024, 1232, 410, 0, 0, 0, 0, 0, 0, 0, 2656, 47, 1586, 6, 12, 984, 0, 391, 1922, 247, 0, 0, 0, 0, 0, 0, 0, 331, 1016, 656, 2288, 934, 1283, 1212, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7519, 8874, 9056, 7624, 8465, 8427, 7851, 7720] avg=8192.0000 max=9056 max_rank=2 min=7519 min_rank=0 imbalance_ratio=1.105469 max_load_reduction=56480 max_load_reduction_pct=0.861816 moved_to_echo=32463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2289, 57, 633, 153, 1616, 1, 2463, 143, 1220, 383, 0, 0, 0, 0, 0, 0, 654, 2025, 561, 563, 1844, 1163, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 253, 1571, 1165, 1287, 1, 670, 2504, 1332, 250, 174, 0, 0, 0, 0, 0, 0, 53, 35, 680, 1570, 2212, 522, 2476, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1807, 1138, 169, 504, 566, 1745, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 685, 6, 1895, 1048, 2, 1696, 1183, 876, 295, 0, 0, 0, 0, 0, 0, 0, 2622, 70, 1170, 5, 97, 668, 0, 648, 2305, 243, 0, 0, 0, 0, 0, 0, 3, 256, 1239, 1119, 2710, 958, 1032, 1191, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8958, 6985, 9207, 8700, 7664, 7686, 7828, 8508] avg=8192.0000 max=9207 max_rank=2 min=6985 min_rank=1 imbalance_ratio=1.123901 max_load_reduction=56329 max_load_reduction_pct=0.859512 moved_to_echo=31686 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2054, 129, 727, 510, 1006, 5, 1703, 136, 1176, 411, 0, 0, 0, 0, 0, 0, 1038, 2140, 499, 496, 2002, 1271, 0, 419, 0, 0, 0, 0, 0, 0, 0, 0, 527, 1223, 941, 1277, 1, 727, 2072, 1624, 268, 247, 0, 0, 0, 0, 0, 0, 54, 31, 618, 1016, 2377, 718, 2500, 747, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1924, 1594, 203, 368, 663, 1707, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 749, 4, 1904, 1617, 1, 1154, 1700, 795, 420, 0, 0, 0, 0, 0, 0, 0, 2324, 8, 1719, 29, 56, 781, 7, 900, 2105, 238, 0, 0, 0, 0, 0, 0, 7, 404, 1453, 1219, 2345, 1049, 877, 1133, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7857, 7865, 8907, 8061, 7848, 8344, 8167, 8487] avg=8192.0000 max=8907 max_rank=2 min=7848 min_rank=4 imbalance_ratio=1.087280 max_load_reduction=56629 max_load_reduction_pct=0.864090 moved_to_echo=32846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5632, 128, 14720, 12544, 128, 12288, 9600, 7808, 3328, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[18944, 768, 5120, 2816, 8960, 128, 15488, 1280, 9088, 3456, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 13184, 12544, 1920, 4096, 6144, 15488, 12672, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 2688, 9728, 7936, 19456, 8320, 8960, 8832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3584, 10624, 5888, 8960, 128, 6016, 15488, 11264, 2304, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[21120, 384, 12032, 128, 768, 7168, 128, 5376, 17280, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[7808, 18176, 4480, 5760, 13696, 13696, 128, 2560, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 384, 5120, 9856, 19328, 5248, 18944, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 2164, 0, 3376, 4019, 0, 907, 3321, 0, 0, 218, 4096, 0, 0, 0, 0, 0, 3906, 2832, 307, 1303, 0, 2333, 0, 2891, 2, 5, 1544, 0, 0, 0, 188, 0, 0, 3704, 4093, 3677, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 4095, 4096, 0, 0, 0, 0, 4090, 124, 0, 0, 0, 0, 42] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1273, 0, 3902, 4078, 0, 1728, 3226, 0, 0, 275, 4014, 0, 0, 0, 0, 0, 3993, 1361, 602, 1518, 0, 2598, 0, 2967, 20, 20, 1296, 0, 0, 0, 297, 0, 0, 3736, 4059, 4010, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4045, 4096, 0, 13, 0, 4, 4096, 19, 0, 0, 0, 0, 98] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4094, 0, 2928, 0, 3878, 4034, 1, 1213, 2646, 0, 2, 764, 3974, 1, 0, 0, 1, 0, 4048, 2469, 352, 323, 1, 1646, 1, 3115, 0, 59, 2988, 1, 0, 1, 139, 0, 0, 3243, 3836, 3445, 0, 0, 0, 1, 0, 4094, 0, 0, 1, 3990, 4094, 1, 9, 0, 9, 4094, 18, 0, 1, 2, 2, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 4093, 2, 2635, 1, 3814, 3947, 0, 2671, 3311, 0, 0, 732, 3861, 0, 3, 0, 0, 0, 4059, 3021, 1110, 89, 0, 233, 0, 3952, 0, 1, 1186, 0, 0, 0, 2, 0, 0, 3009, 3996, 3370, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4080, 4096, 0, 1, 0, 0, 4096, 27, 0, 0, 0, 0, 40] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4095, 2, 3267, 1, 3570, 4011, 0, 1614, 2455, 0, 0, 899, 3984, 1, 0, 4, 1, 1, 4088, 3312, 816, 199, 0, 1111, 0, 3671, 4, 41, 1043, 0, 2, 1, 48, 0, 0, 3020, 4056, 3595, 1, 0, 0, 0, 0, 4095, 1, 0, 1, 4072, 4092, 1, 1, 1, 69, 4095, 89, 1, 1, 0, 1, 102] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 2589, 0, 3837, 3912, 0, 2051, 3140, 0, 0, 1783, 4088, 0, 0, 0, 0, 0, 4093, 3402, 772, 115, 0, 567, 0, 3276, 0, 0, 2296, 0, 1, 0, 23, 0, 0, 2582, 3991, 2677, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3875, 4096, 0, 0, 0, 13, 4096, 40, 0, 0, 0, 0, 29] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4095, 0, 2329, 0, 2494, 3975, 0, 612, 2729, 0, 0, 102, 4062, 1, 0, 2, 1, 1, 4089, 3373, 684, 620, 0, 3052, 0, 3230, 0, 8, 2061, 0, 2, 1, 511, 1, 3, 3799, 3874, 3512, 0, 0, 0, 1, 0, 4095, 2, 0, 0, 3874, 4094, 1, 0, 1, 120, 4095, 15, 0, 1, 0, 1, 12] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 13, 2888, 0, 3414, 4060, 0, 872, 3034, 0, 0, 163, 4034, 0, 0, 2, 0, 0, 3813, 3620, 228, 54, 0, 1690, 6, 3083, 0, 43, 2231, 0, 6, 0, 375, 5, 3, 3444, 3756, 3930, 0, 0, 0, 0, 4, 4090, 1, 3, 0, 4038, 4096, 0, 18, 0, 25, 4092, 275, 0, 0, 6, 0, 25] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1273, 0, 1262, 355, 0, 0, 0, 0, 0, 0, 3459, 0, 0, 1728, 3226, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 1255, 0, 0, 0, 0, 0, 3993, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 602, 1518, 0, 2598, 0, 2967, 20, 20, 1497, 605, 0, 0, 0, 0, 0, 0, 1296, 0, 0, 0, 297, 0, 0, 3736, 2783, 60, 0, 0, 0, 0, 0, 0, 4059, 4010, 0, 0, 0, 0, 0, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 4096, 0, 13, 0, 88, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 19, 0, 0, 0, 0, 98, 4018, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6986, 8688, 6609, 9827, 8172, 8777, 8242, 8235] avg=8192.0000 max=9827 max_rank=3 min=6609 min_rank=2 imbalance_ratio=1.199585 max_load_reduction=55709 max_load_reduction_pct=0.850052 moved_to_echo=33426 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 2164, 0, 1287, 307, 0, 0, 0, 0, 0, 0, 2993, 0, 0, 907, 3321, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 1282, 0, 0, 0, 0, 0, 3906, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 307, 1303, 0, 2333, 0, 2891, 2, 5, 1527, 605, 0, 0, 0, 0, 0, 0, 1544, 0, 0, 0, 188, 0, 0, 3704, 2783, 61, 0, 0, 0, 0, 0, 0, 4093, 3677, 0, 0, 0, 0, 0, 708, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4095, 4096, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 124, 0, 0, 0, 0, 42, 3958, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7854, 7439, 8020, 8973, 8280, 8478, 8278, 8214] avg=8192.0000 max=8973 max_rank=3 min=7439 min_rank=1 imbalance_ratio=1.095337 max_load_reduction=56563 max_load_reduction_pct=0.863083 moved_to_echo=33250 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 4094, 0, 2928, 0, 1249, 353, 0, 0, 0, 0, 0, 0, 3438, 0, 1, 1213, 2646, 0, 2, 764, 0, 0, 0, 0, 0, 0, 0, 0, 1243, 1, 0, 0, 1, 0, 4048, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 352, 323, 1, 1646, 1, 3115, 0, 59, 1482, 605, 0, 0, 0, 0, 0, 0, 2988, 1, 0, 1, 139, 0, 0, 3243, 2782, 60, 0, 0, 0, 0, 0, 0, 3836, 3445, 0, 0, 0, 1, 0, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3990, 4094, 1, 9, 0, 87, 0, 0, 0, 0, 0, 0, 0, 9, 4094, 18, 0, 1, 2, 2, 14, 3974, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8627, 8064, 7762, 7584, 9214, 7989, 8182, 8114] avg=8192.0000 max=9214 max_rank=4 min=7584 min_rank=3 imbalance_ratio=1.124756 max_load_reduction=56322 max_load_reduction_pct=0.859406 moved_to_echo=33499 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 4093, 2, 2635, 1, 1214, 347, 0, 0, 0, 0, 0, 0, 3381, 0, 0, 2671, 3311, 0, 0, 732, 0, 0, 0, 0, 0, 0, 0, 0, 1207, 0, 3, 0, 0, 0, 4059, 3021, 0, 0, 0, 0, 0, 0, 0, 0, 1110, 89, 0, 233, 0, 3952, 0, 1, 1440, 605, 0, 0, 0, 0, 0, 0, 1186, 0, 0, 0, 2, 0, 0, 3009, 2783, 59, 0, 0, 0, 0, 0, 0, 3996, 3370, 0, 0, 0, 0, 0, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 1, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 27, 0, 0, 0, 0, 40, 3888, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8294, 10095, 8290, 7430, 7039, 8074, 8263, 8051] avg=8192.0000 max=10095 max_rank=1 min=7039 min_rank=4 imbalance_ratio=1.232300 max_load_reduction=55441 max_load_reduction_pct=0.845963 moved_to_echo=31427 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 2589, 0, 1285, 349, 0, 0, 0, 0, 0, 0, 3402, 0, 0, 2051, 3140, 0, 0, 1783, 0, 0, 0, 0, 0, 0, 0, 0, 1279, 0, 0, 0, 0, 0, 4093, 3402, 0, 0, 0, 0, 0, 0, 0, 0, 772, 115, 0, 567, 0, 3276, 0, 0, 1524, 605, 0, 0, 0, 0, 0, 0, 2296, 0, 1, 0, 23, 0, 0, 2582, 2783, 58, 0, 0, 0, 0, 0, 0, 3991, 2677, 0, 0, 0, 0, 0, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 4096, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 40, 0, 0, 0, 0, 29, 3854, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8319, 10376, 8774, 6859, 7743, 7376, 8057, 8032] avg=8192.0000 max=10376 max_rank=1 min=6859 min_rank=3 imbalance_ratio=1.266602 max_load_reduction=55160 max_load_reduction_pct=0.841675 moved_to_echo=31208 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4095, 2, 3267, 1, 1252, 325, 0, 0, 0, 0, 0, 0, 3165, 0, 0, 1614, 2455, 0, 0, 899, 0, 0, 0, 0, 0, 0, 0, 0, 1247, 1, 0, 4, 1, 1, 4088, 3312, 0, 0, 0, 0, 0, 0, 0, 0, 816, 199, 0, 1111, 0, 3671, 4, 41, 1485, 605, 0, 0, 0, 0, 0, 0, 1043, 0, 2, 1, 48, 0, 0, 3020, 2783, 60, 0, 0, 0, 0, 0, 0, 4056, 3595, 1, 0, 0, 0, 0, 707, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4072, 4092, 1, 1, 1, 80, 0, 0, 0, 0, 0, 0, 0, 69, 4095, 89, 1, 1, 0, 1, 102, 3951, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8943, 8133, 8654, 7932, 6957, 8359, 8249, 8309] avg=8192.0000 max=8943 max_rank=0 min=6957 min_rank=4 imbalance_ratio=1.091675 max_load_reduction=56593 max_load_reduction_pct=0.863541 moved_to_echo=31874 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4095, 0, 2329, 0, 1277, 227, 0, 0, 0, 0, 0, 0, 2211, 0, 0, 612, 2729, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 1, 0, 2, 1, 1, 4089, 3373, 0, 0, 0, 0, 0, 0, 0, 0, 684, 620, 0, 3052, 0, 3230, 0, 8, 1514, 605, 0, 0, 0, 0, 0, 0, 2061, 0, 2, 1, 511, 1, 3, 3799, 2783, 60, 0, 0, 0, 0, 0, 0, 3874, 3512, 0, 0, 0, 1, 0, 707, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3874, 4094, 1, 0, 1, 56, 0, 0, 0, 0, 0, 0, 0, 120, 4095, 15, 0, 1, 0, 1, 12, 3915, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7929, 5654, 8738, 9713, 9221, 8094, 8028, 8159] avg=8192.0000 max=9713 max_rank=3 min=5654 min_rank=1 imbalance_ratio=1.185669 max_load_reduction=55823 max_load_reduction_pct=0.851791 moved_to_echo=33502 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 13, 2888, 0, 1272, 315, 0, 0, 0, 0, 0, 0, 3018, 0, 0, 872, 3034, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 0, 0, 2, 0, 0, 3813, 3620, 0, 0, 0, 0, 0, 0, 0, 0, 228, 54, 0, 1690, 6, 3083, 0, 43, 1508, 606, 0, 0, 0, 0, 0, 0, 2231, 0, 6, 0, 375, 5, 3, 3444, 2785, 61, 0, 0, 0, 0, 0, 0, 3756, 3930, 0, 0, 0, 0, 4, 699, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4038, 4096, 0, 18, 0, 81, 0, 0, 0, 0, 0, 0, 0, 25, 4092, 275, 0, 0, 6, 0, 25, 3999, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8584, 7087, 8689, 7218, 8910, 8389, 8237, 8422] avg=8192.0000 max=8910 max_rank=4 min=7087 min_rank=1 imbalance_ratio=1.087646 max_load_reduction=56626 max_load_reduction_pct=0.864044 moved_to_echo=33958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25088, 0, 128, 11776, 23936, 0, 128, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 32128, 32768, 128, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 20096, 128, 10112, 2688, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[10112, 128, 128, 128, 128, 128, 32128, 23424, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4992, 4224, 128, 13312, 128, 26240, 128, 256, 12032, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 32768, 640, 128, 128, 128, 128, 384, 31616, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[31744, 28288, 128, 0, 0, 128, 128, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[14720, 128, 128, 128, 1664, 128, 128, 26624, 22272, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3796, 0, 0, 2, 0, 0, 0, 4095, 4074, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3629, 107, 4088, 179, 4096, 4093, 0, 4094, 0, 0, 0, 711, 4095, 0, 0, 2, 3590, 4096, 0, 0, 0, 0, 9, 0, 132, 0, 122, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 45, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 3793, 0, 0, 3, 0, 0, 0, 4084, 4087, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3852, 41, 4095, 288, 4081, 4045, 0, 4095, 0, 0, 0, 184, 4092, 0, 0, 1, 3771, 4096, 0, 0, 0, 0, 0, 0, 156, 0, 290, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 2, 3738, 0, 0, 3, 0, 2, 0, 4088, 4079, 4095, 0, 1, 4094, 0, 2, 4095, 1, 1, 4005, 4, 4082, 22, 4089, 4093, 0, 4096, 1, 0, 0, 300, 4092, 12, 0, 0, 4038, 4094, 1, 1, 0, 0, 9, 0, 14, 0, 165, 0, 4095, 2, 1, 2, 1, 1, 4095, 2, 0, 12, 0, 1, 0, 1, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 3938, 0, 0, 11, 0, 0, 0, 4088, 4091, 4092, 0, 1, 4095, 0, 1, 4095, 1, 1, 4003, 6, 4094, 18, 4095, 4093, 0, 4092, 1, 0, 0, 181, 4095, 2, 0, 6, 3946, 4094, 1, 1, 0, 0, 13, 1, 65, 0, 115, 1, 4094, 0, 1, 0, 1, 1, 4094, 1, 1, 2, 0, 0, 1, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 3882, 0, 0, 4, 0, 0, 0, 4095, 4093, 4095, 0, 1, 4095, 0, 1, 4095, 1, 1, 3960, 124, 4095, 68, 4095, 4095, 0, 4089, 1, 0, 0, 280, 4095, 3, 0, 35, 3900, 4093, 1, 1, 0, 0, 0, 1, 7, 0, 28, 1, 4095, 0, 1, 0, 0, 1, 4095, 1, 1, 1, 0, 3, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 3828, 0, 1, 35, 0, 0, 0, 4092, 4064, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 3815, 107, 4069, 32, 4096, 4090, 0, 4093, 0, 0, 0, 584, 4096, 0, 1, 2, 3607, 4095, 0, 0, 0, 0, 2, 0, 243, 0, 98, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 1, 0, 1, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3490, 0, 0, 98, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3893, 5, 4096, 205, 4093, 4048, 0, 4096, 0, 0, 0, 372, 4096, 4, 0, 10, 4057, 4095, 0, 0, 0, 0, 1, 0, 60, 0, 143, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 5, 0, 3486, 0, 6, 16, 0, 0, 0, 4058, 4064, 4096, 0, 0, 4087, 3, 0, 4095, 0, 0, 3875, 61, 4067, 232, 4096, 4084, 0, 4076, 0, 0, 0, 400, 4096, 0, 11, 11, 3936, 4091, 0, 0, 5, 0, 10, 0, 45, 0, 318, 0, 4089, 0, 0, 0, 10, 0, 4086, 0, 0, 0, 0, 20, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3796, 0, 0, 2, 0, 3927, 239, 147, 110, 0, 0, 0, 0, 0, 0, 4095, 4074, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 3629, 107, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 4093, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 711, 0, 0, 0, 2, 3590, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 132, 0, 122, 4096, 3833, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 1, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8193, 7993, 8256, 8252, 8203, 8192, 8226] avg=8192.0000 max=8256 max_rank=3 min=7993 min_rank=2 imbalance_ratio=1.007812 max_load_reduction=57280 max_load_reduction_pct=0.874023 moved_to_echo=32873 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 3793, 0, 0, 3, 0, 3927, 239, 147, 110, 0, 0, 0, 0, 0, 0, 4084, 4087, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 3852, 41, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 4045, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 1, 3771, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 290, 4081, 3833, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8195, 8157, 8318, 7905, 8370, 8192, 8178] avg=8192.0000 max=8370 max_rank=5 min=7905 min_rank=4 imbalance_ratio=1.021729 max_load_reduction=57166 max_load_reduction_pct=0.872284 moved_to_echo=32645 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 2, 3738, 0, 0, 3, 0, 3926, 239, 147, 110, 0, 0, 0, 0, 2, 0, 4088, 4079, 0, 0, 1, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 169, 1, 1, 4005, 4, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4093, 0, 3986, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 12, 0, 0, 4038, 3947, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 0, 14, 0, 165, 4089, 3831, 10, 0, 0, 0, 0, 0, 0, 4095, 2, 1, 2, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 12, 0, 1, 0, 1, 2, 4095, 4082, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8194, 8264, 8102, 8298, 8119, 8197, 8195] avg=8192.0000 max=8298 max_rank=4 min=8102 min_rank=3 imbalance_ratio=1.012939 max_load_reduction=57238 max_load_reduction_pct=0.873383 moved_to_echo=32809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3938, 0, 0, 11, 0, 3926, 239, 147, 110, 0, 0, 0, 0, 0, 0, 4088, 4091, 0, 0, 1, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 169, 1, 1, 4003, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4093, 0, 3982, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 2, 0, 6, 3946, 3947, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 13, 1, 65, 0, 115, 4095, 3832, 11, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 0, 1, 1, 1, 4092, 4084, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8372, 8204, 8275, 8094, 8083, 8133, 8192, 8183] avg=8192.0000 max=8372 max_rank=0 min=8083 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32591 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3490, 0, 0, 98, 0, 3927, 239, 147, 110, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 3893, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 4048, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 4, 0, 10, 4057, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 60, 0, 143, 4093, 3833, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 4096, 4085, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8011, 8216, 8163, 8239, 8391, 8141, 8192, 8183] avg=8192.0000 max=8391 max_rank=4 min=8011 min_rank=0 imbalance_ratio=1.024292 max_load_reduction=57145 max_load_reduction_pct=0.871964 moved_to_echo=32907 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 3828, 0, 1, 35, 0, 3927, 239, 147, 110, 0, 0, 0, 0, 0, 0, 4092, 4064, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 169, 0, 0, 3815, 107, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 4090, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 1, 2, 3607, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 243, 0, 98, 4096, 3833, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 4096, 4085, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8288, 8180, 8161, 8105, 8142, 8283, 8193, 8184] avg=8192.0000 max=8288 max_rank=0 min=8105 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=57248 max_load_reduction_pct=0.873535 moved_to_echo=32802 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 3486, 0, 6, 16, 0, 3929, 243, 149, 113, 0, 0, 0, 0, 0, 0, 4058, 4064, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 166, 0, 0, 3875, 61, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 4084, 0, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 11, 11, 3936, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 10, 0, 45, 0, 318, 4096, 3825, 12, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 10, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 1, 0, 4096, 4084, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7947, 8141, 8172, 8279, 8300, 8311, 8185, 8201] avg=8192.0000 max=8311 max_rank=5 min=7947 min_rank=0 imbalance_ratio=1.014526 max_load_reduction=57225 max_load_reduction_pct=0.873184 moved_to_echo=32997 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3882, 0, 0, 4, 0, 3926, 239, 147, 110, 0, 0, 0, 0, 0, 0, 4095, 4093, 0, 0, 1, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 169, 1, 1, 3960, 124, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 4095, 0, 3979, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 3, 0, 35, 3900, 3946, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7, 0, 28, 4095, 3832, 12, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 3, 1, 0, 1, 4095, 4083, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8309, 8213, 8351, 8143, 8165, 7976, 8193, 8186] avg=8192.0000 max=8351 max_rank=2 min=7976 min_rank=5 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1152, 0, 32768, 0, 31872, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[3072, 0, 128, 128, 128, 30848, 31616, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 29952, 0, 128, 256, 0, 31488, 1920, 1280, 896, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 1408, 128, 128, 31104, 512, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 32768, 0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 768, 0, 1280, 32768, 30720, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 4082, 0, 0, 0, 4088, 0, 4096, 4096, 0, 3610, 4096, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 0, 0, 4096, 0, 4095, 4096, 0, 2039, 0, 0, 1, 0, 0, 0, 7, 4096, 0, 0, 2125, 27, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4015, 4054, 0, 0, 0, 4080, 0, 4096, 4096, 0, 3569, 4096, 0, 0, 0, 781, 0, 0, 0, 0, 0, 0, 0, 3694, 0, 0, 0, 4096, 0, 4081, 4096, 0, 2594, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1726, 29, 0, 1, 0, 4095, 0, 42, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[2, 2, 0, 4, 4093, 1, 0, 4094, 4079, 4093, 0, 2, 0, 3996, 0, 4092, 4094, 0, 3615, 4094, 2, 2, 2, 257, 0, 0, 0, 0, 0, 2, 0, 3965, 0, 2, 4, 4094, 1, 4084, 4093, 0, 2098, 19, 2, 0, 0, 0, 0, 0, 4094, 0, 2, 2389, 43, 2, 0, 1, 4092, 8, 15, 0, 0, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4095, 4084, 4096, 0, 0, 0, 4082, 0, 4096, 4096, 0, 3907, 4096, 0, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 3763, 0, 0, 0, 4096, 0, 4019, 4096, 0, 1056, 56, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1839, 1591, 0, 0, 0, 4096, 1, 14, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 4019, 4073, 0, 0, 0, 4085, 0, 4096, 4094, 0, 3930, 4096, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 3440, 1, 0, 0, 4095, 0, 4072, 4096, 0, 1062, 17, 0, 1, 0, 0, 0, 3, 4096, 0, 0, 3199, 600, 0, 3, 0, 4096, 0, 30, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4088, 1, 1, 4095, 4095, 4094, 0, 1, 0, 4036, 1, 4093, 4095, 0, 3986, 4095, 1, 1, 1, 464, 0, 0, 0, 0, 1, 0, 1, 3866, 0, 1, 1, 4095, 0, 4061, 4093, 0, 1338, 38, 1, 4, 0, 0, 0, 0, 4095, 0, 1, 2361, 317, 0, 3, 0, 4094, 2, 11, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4092, 1, 0, 4095, 3999, 4095, 0, 1, 0, 3966, 0, 4093, 4095, 0, 3647, 4095, 1, 1, 1, 651, 0, 0, 0, 0, 1, 0, 0, 4003, 0, 1, 2, 4095, 0, 4089, 4094, 1, 1080, 0, 1, 0, 1, 0, 0, 0, 4095, 0, 1, 2955, 64, 1, 1, 0, 4092, 39, 83, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4088, 4066, 4034, 0, 0, 0, 4084, 0, 4096, 4090, 0, 3889, 4096, 0, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 3812, 9, 0, 0, 4096, 0, 4088, 4096, 0, 1673, 2, 0, 19, 0, 0, 0, 28, 4096, 0, 0, 2348, 236, 0, 19, 0, 4096, 0, 54, 0, 4, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4015, 84, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 3569, 0, 0, 0, 0, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3694, 2744, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2594, 7, 0, 0, 0, 0, 0, 0, 4096, 2454, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1726, 29, 0, 1, 0, 1261, 91, 0, 0, 0, 0, 0, 0, 4095, 0, 42, 0, 0, 0, 0, 0, 2709, 1331, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8179, 8354, 8080, 8198, 9151, 7204, 8177] avg=8192.0000 max=9151 max_rank=5 min=7204 min_rank=6 imbalance_ratio=1.117065 max_load_reduction=56385 max_load_reduction_pct=0.860367 moved_to_echo=32730 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4092, 84, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 3610, 0, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 2744, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2039, 0, 0, 1, 0, 0, 0, 7, 4096, 2453, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2125, 27, 0, 0, 0, 1270, 91, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 2728, 1331, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8264, 8200, 8291, 8212, 8596, 7609, 8171] avg=8192.0000 max=8596 max_rank=5 min=7609 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=56940 max_load_reduction_pct=0.868835 moved_to_echo=32588 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 0, 4, 4093, 1, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4079, 85, 0, 2, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 3615, 0, 2, 2, 2, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3965, 2742, 1640, 0, 0, 0, 0, 0, 0, 0, 2, 4, 22, 1, 4084, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2098, 19, 2, 0, 0, 0, 0, 0, 4094, 2452, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 2389, 43, 2, 0, 1, 1273, 91, 0, 0, 0, 0, 0, 0, 4092, 8, 15, 0, 0, 0, 0, 2, 2735, 1330, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8162, 7880, 8349, 8206, 8665, 7895, 8182] avg=8192.0000 max=8665 max_rank=5 min=7880 min_rank=2 imbalance_ratio=1.057739 max_load_reduction=56871 max_load_reduction_pct=0.867783 moved_to_echo=32948 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4084, 85, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 3907, 0, 0, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3763, 2744, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4019, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 56, 0, 0, 0, 0, 0, 0, 4096, 2454, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1839, 1591, 0, 0, 0, 1274, 91, 0, 0, 0, 0, 0, 0, 4096, 1, 14, 0, 0, 0, 0, 0, 2737, 1331, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8251, 8077, 8149, 8136, 7662, 8891, 8179] avg=8192.0000 max=8891 max_rank=6 min=7662 min_rank=5 imbalance_ratio=1.085327 max_load_reduction=56645 max_load_reduction_pct=0.864334 moved_to_echo=32868 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4088, 1, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 84, 0, 1, 0, 4036, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 3986, 0, 1, 1, 1, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3866, 2743, 1640, 0, 0, 0, 0, 0, 0, 0, 1, 1, 22, 0, 4061, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1338, 38, 1, 4, 0, 0, 0, 0, 4095, 2453, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 2361, 317, 0, 3, 0, 1274, 91, 0, 0, 0, 0, 0, 0, 4094, 2, 11, 1, 0, 0, 0, 1, 2736, 1330, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8217, 8456, 8251, 8178, 7929, 8142, 8175] avg=8192.0000 max=8456 max_rank=2 min=7929 min_rank=5 imbalance_ratio=1.032227 max_load_reduction=57080 max_load_reduction_pct=0.870972 moved_to_echo=32424 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4019, 84, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 3930, 0, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 2743, 1642, 0, 0, 0, 0, 0, 0, 1, 0, 0, 22, 0, 4072, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 17, 0, 1, 0, 0, 0, 3, 4096, 2454, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3199, 600, 0, 3, 0, 1267, 91, 0, 0, 0, 0, 0, 0, 4096, 0, 30, 0, 0, 0, 0, 0, 2722, 1330, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8188, 8073, 7825, 8191, 7633, 9256, 8178] avg=8192.0000 max=9256 max_rank=6 min=7633 min_rank=5 imbalance_ratio=1.129883 max_load_reduction=56280 max_load_reduction_pct=0.858765 moved_to_echo=33258 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4092, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 3999, 85, 0, 1, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 3647, 0, 1, 1, 1, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4003, 2743, 1641, 0, 0, 0, 0, 0, 0, 0, 1, 2, 22, 0, 4089, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 0, 1, 0, 1, 0, 0, 0, 4095, 2452, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 2955, 64, 1, 1, 0, 1274, 91, 0, 0, 0, 0, 0, 0, 4092, 39, 83, 1, 0, 0, 0, 1, 2736, 1330, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8051, 8304, 8388, 8209, 7629, 8482, 8282] avg=8192.0000 max=8482 max_rank=6 min=7629 min_rank=5 imbalance_ratio=1.035400 max_load_reduction=57054 max_load_reduction_pct=0.870575 moved_to_echo=32602 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 4066, 74, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 3889, 0, 0, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3812, 2748, 1643, 0, 0, 0, 0, 0, 0, 9, 0, 0, 13, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1673, 2, 0, 19, 0, 0, 0, 28, 4096, 2453, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2348, 236, 0, 19, 0, 1260, 98, 0, 0, 0, 0, 0, 0, 4096, 0, 54, 0, 4, 0, 3, 0, 2700, 1335, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8224, 8192, 8203, 8206, 8271, 8057, 8192] avg=8192.0000 max=8271 max_rank=5 min=8057 min_rank=6 imbalance_ratio=1.009644 max_load_reduction=57265 max_load_reduction_pct=0.873795 moved_to_echo=32726 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32128, 0, 30208, 0, 128, 128, 128, 3456, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[13056, 256, 128, 128, 128, 0, 0, 128, 32768, 19712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 384, 128, 128, 0, 128, 128, 21888, 10752, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32512, 768, 0, 128, 0, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 256, 128, 32640, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 128, 30464, 22016, 13184, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 18944, 2944, 128, 128, 128, 10240, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 2774, 0, 0, 4088, 0, 0, 0, 240, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 9, 0, 3964, 0, 0, 0, 16, 4096, 0, 0, 4096, 151, 0, 4096, 4044, 4096, 0, 0, 0, 0, 784, 4, 4096, 0, 0, 0, 0, 4096, 0, 309, 0, 4096, 4096, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 1617, 0, 0, 4096, 0, 0, 0, 1965, 4094, 13, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 0, 4092, 0, 4095, 0, 1, 44, 0, 4065, 0, 0, 1, 11, 4096, 0, 0, 4096, 104, 0, 4096, 3973, 4096, 0, 0, 0, 0, 258, 0, 4096, 0, 0, 1, 0, 4096, 0, 239, 0, 4096, 4092, 0, 0, 12, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[2, 1472, 0, 1, 4051, 0, 2, 2, 1313, 4093, 0, 2, 2, 2, 0, 4077, 0, 2, 0, 2, 2, 2, 4094, 0, 4094, 2, 0, 1, 0, 4064, 0, 0, 3, 0, 4091, 0, 2, 4094, 230, 0, 4093, 4036, 4093, 2, 0, 2, 0, 370, 25, 4094, 2, 0, 0, 0, 4092, 2, 829, 2, 4094, 4094, 1, 1, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 1103, 0, 0, 4092, 0, 1, 1, 1503, 4088, 0, 1, 1, 1, 0, 4073, 0, 1, 1, 0, 0, 1, 4095, 0, 4094, 0, 0, 15, 0, 4086, 0, 0, 6, 0, 4095, 0, 1, 4095, 55, 0, 4093, 4062, 4095, 1, 0, 1, 1, 915, 4, 4095, 1, 0, 0, 0, 4094, 0, 570, 1, 4095, 4095, 1, 1, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1369, 0, 1, 4096, 0, 0, 0, 1628, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 104, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 4096, 8, 0, 4096, 4077, 4096, 0, 0, 0, 0, 279, 0, 4096, 0, 0, 1, 0, 4096, 0, 729, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 1671, 0, 0, 4096, 0, 0, 0, 957, 4076, 27, 0, 0, 0, 0, 4065, 12, 0, 0, 0, 0, 0, 4068, 0, 4096, 0, 6, 32, 0, 4025, 0, 0, 22, 46, 4096, 0, 0, 4096, 473, 0, 4096, 3882, 4096, 0, 0, 0, 0, 771, 0, 4096, 0, 0, 1, 0, 4096, 0, 452, 0, 4089, 4073, 0, 0, 20, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 2037, 0, 0, 4094, 0, 1, 1, 774, 4091, 0, 1, 1, 1, 0, 4072, 0, 1, 0, 0, 0, 1, 4077, 0, 4095, 0, 4, 32, 0, 4094, 0, 0, 7, 0, 4093, 0, 0, 4095, 319, 0, 4095, 4089, 4095, 1, 1, 1, 1, 684, 0, 4095, 1, 0, 0, 0, 4093, 0, 295, 1, 4095, 4095, 1, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1670, 0, 0, 4096, 0, 0, 0, 455, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 3966, 0, 0, 1, 0, 4096, 0, 0, 4096, 1064, 0, 4096, 3904, 4096, 0, 0, 0, 0, 542, 0, 4096, 0, 0, 1, 0, 4096, 0, 686, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2774, 0, 0, 4088, 0, 0, 0, 1090, 514, 488, 296, 0, 0, 0, 0, 240, 3002, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 3964, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 3783, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 4044, 0, 0, 0, 0, 0, 784, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 3582, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9250, 7338, 8195, 8089, 8046, 8436, 8193, 7989] avg=8192.0000 max=9250 max_rank=0 min=7338 min_rank=1 imbalance_ratio=1.129150 max_load_reduction=56286 max_load_reduction_pct=0.858856 moved_to_echo=32664 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 1472, 0, 1, 4051, 0, 2, 2, 1089, 514, 487, 296, 0, 0, 0, 0, 1313, 3000, 0, 2, 2, 2, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 2, 2, 4094, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 1, 0, 4064, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 3, 0, 4091, 0, 2, 3781, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3606, 4036, 0, 2, 0, 2, 0, 370, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4091, 2, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 829, 2, 3580, 4094, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7916, 8396, 8199, 8181, 8107, 8016, 8212, 8509] avg=8192.0000 max=8509 max_rank=7 min=7916 min_rank=0 imbalance_ratio=1.038696 max_load_reduction=57027 max_load_reduction_pct=0.870163 moved_to_echo=32844 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1369, 0, 1, 4096, 0, 0, 0, 1090, 514, 488, 296, 0, 0, 0, 0, 1628, 3002, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 104, 0, 4094, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3783, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 4077, 0, 0, 0, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 729, 0, 3582, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7854, 8725, 8196, 8313, 7887, 7964, 8190, 8407] avg=8192.0000 max=8725 max_rank=1 min=7854 min_rank=0 imbalance_ratio=1.065063 max_load_reduction=56811 max_load_reduction_pct=0.866867 moved_to_echo=32448 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1670, 0, 0, 4096, 0, 0, 0, 1090, 514, 488, 296, 0, 0, 0, 0, 455, 3002, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3966, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 3783, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 3904, 0, 0, 0, 0, 0, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 0, 3582, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 7553, 8195, 8082, 8944, 8054, 8190, 8364] avg=8192.0000 max=8944 max_rank=4 min=7553 min_rank=1 imbalance_ratio=1.091797 max_load_reduction=56592 max_load_reduction_pct=0.863525 moved_to_echo=33552 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1103, 0, 0, 4092, 0, 1, 1, 1087, 514, 487, 296, 0, 0, 0, 0, 1503, 2997, 0, 1, 1, 1, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 4095, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 15, 0, 4086, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 6, 0, 4095, 0, 1, 3782, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3606, 4062, 0, 1, 0, 1, 1, 915, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 570, 1, 3581, 4095, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7581, 8576, 8197, 8215, 7939, 8586, 8191, 8251] avg=8192.0000 max=8586 max_rank=5 min=7581 min_rank=0 imbalance_ratio=1.048096 max_load_reduction=56950 max_load_reduction_pct=0.868988 moved_to_echo=32967 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1617, 0, 0, 4096, 0, 0, 0, 1089, 514, 488, 296, 0, 0, 0, 0, 1965, 3001, 13, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4092, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 44, 0, 4065, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 1, 11, 4096, 0, 0, 3783, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 3973, 0, 0, 0, 0, 0, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 3582, 4092, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8100, 9068, 8194, 8225, 7995, 7839, 8190, 7925] avg=8192.0000 max=9068 max_rank=1 min=7839 min_rank=5 imbalance_ratio=1.106934 max_load_reduction=56468 max_load_reduction_pct=0.861633 moved_to_echo=31949 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1671, 0, 0, 4096, 0, 0, 0, 1087, 518, 492, 298, 0, 0, 0, 0, 957, 2983, 27, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4068, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 32, 0, 4025, 0, 0, 18, 9, 0, 0, 0, 0, 0, 0, 22, 46, 4096, 0, 0, 3780, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 3882, 0, 0, 0, 0, 0, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 3571, 4073, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8162, 8032, 8182, 8186, 8417, 8257, 8184, 8116] avg=8192.0000 max=8417 max_rank=4 min=8032 min_rank=1 imbalance_ratio=1.027466 max_load_reduction=57119 max_load_reduction_pct=0.871567 moved_to_echo=32974 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2037, 0, 0, 4094, 0, 1, 1, 1088, 514, 488, 296, 0, 0, 0, 0, 774, 2999, 0, 1, 1, 1, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4077, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 32, 0, 4094, 0, 0, 17, 3, 0, 0, 0, 0, 0, 0, 7, 0, 4093, 0, 0, 3782, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3607, 4089, 0, 1, 1, 1, 1, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 1, 3581, 4095, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8519, 7848, 8178, 8245, 8201, 8384, 8186, 7975] avg=8192.0000 max=8519 max_rank=0 min=7848 min_rank=1 imbalance_ratio=1.039917 max_load_reduction=57017 max_load_reduction_pct=0.870010 moved_to_echo=32746 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[8960, 24064, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[28928, 32128, 0, 128, 128, 128, 128, 4608, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 30336, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 13824, 0, 128, 32768, 0, 128, 128, 8832, 4224, 3968, 2432, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 256, 0, 32384, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4224, 128, 28672, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 4094, 0, 1, 0, 3523, 0, 0, 176, 1, 4091, 0, 38, 409, 4095, 1, 1, 0, 18, 0, 0, 17, 4095, 1, 4095, 0, 1, 1, 1, 4091, 236, 0, 3844, 1, 171, 4095, 71, 4095, 1, 1, 1, 4095, 4095, 1, 0, 1, 0, 1, 0, 4095, 1, 0, 2409, 1, 559, 1, 752, 68, 0, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 2814, 0, 0, 40, 0, 4096, 0, 38, 401, 4096, 0, 0, 0, 3, 0, 1, 1, 4096, 0, 4096, 0, 0, 0, 0, 4092, 860, 0, 3651, 0, 310, 4096, 172, 4096, 0, 0, 187, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2513, 0, 937, 0, 326, 7, 0, 32, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4089, 0, 0, 0, 3947, 0, 0, 6, 0, 4096, 0, 1, 312, 4096, 0, 0, 0, 159, 0, 0, 93, 4096, 0, 4096, 0, 0, 0, 0, 4096, 152, 0, 3941, 0, 2, 4096, 143, 4096, 0, 0, 54, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1834, 0, 240, 0, 1395, 16, 0, 1, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 3946, 0, 0, 4, 0, 4096, 0, 0, 79, 4096, 0, 0, 0, 327, 0, 0, 55, 4096, 0, 4096, 0, 0, 0, 0, 4094, 293, 0, 4028, 0, 1, 4096, 592, 4096, 0, 0, 42, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1891, 0, 668, 0, 361, 8, 0, 13, 0, 4078] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4087, 0, 2, 0, 3134, 2, 0, 50, 2, 4084, 0, 62, 578, 4094, 2, 1, 0, 230, 1, 0, 350, 4094, 2, 4094, 0, 2, 0, 2, 4093, 293, 2, 3886, 2, 144, 4093, 85, 4093, 2, 0, 86, 4094, 4093, 2, 3, 0, 3, 1, 0, 4094, 0, 0, 2452, 2, 97, 2, 776, 81, 0, 0, 0, 4090] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 4095, 0, 1, 1, 3679, 0, 0, 5, 1, 4083, 0, 5, 33, 4095, 1, 1, 0, 173, 0, 0, 64, 4095, 1, 4095, 0, 1, 1, 1, 4092, 288, 0, 3524, 1, 60, 4095, 85, 4094, 1, 1, 5, 4094, 4095, 1, 0, 0, 0, 1, 0, 4095, 1, 0, 3492, 1, 94, 1, 733, 61, 0, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 3529, 0, 0, 211, 0, 4096, 0, 0, 453, 4096, 0, 0, 0, 17, 0, 0, 5, 4096, 0, 4096, 0, 0, 0, 0, 4094, 163, 0, 3876, 0, 555, 4096, 191, 4096, 0, 0, 37, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2314, 0, 185, 0, 705, 50, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 3737, 0, 0, 90, 0, 4095, 0, 108, 51, 4096, 0, 0, 0, 37, 0, 11, 11, 4096, 0, 4096, 0, 0, 0, 0, 4058, 97, 0, 3812, 0, 4, 4096, 18, 4096, 0, 0, 38, 4096, 4096, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 3250, 0, 172, 0, 844, 46, 0, 15, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 560, 4096, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 79, 3509, 196, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 327, 0, 0, 55, 3900, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 4028, 0, 1, 0, 592, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4040, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1891, 0, 1173, 347, 56, 0, 0, 0, 0, 0, 668, 0, 361, 8, 0, 13, 0, 4078, 2363, 580, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8602, 7884, 8378, 8197, 8663, 8178, 7563, 8071] avg=8192.0000 max=8663 max_rank=4 min=7563 min_rank=6 imbalance_ratio=1.057495 max_load_reduction=56873 max_load_reduction_pct=0.867813 moved_to_echo=32475 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 560, 4089, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 1, 312, 3509, 197, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 159, 0, 0, 93, 3899, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 3941, 0, 2, 0, 143, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 4040, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1834, 0, 1173, 347, 56, 0, 0, 0, 0, 0, 240, 0, 1395, 16, 0, 1, 0, 4095, 2363, 580, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8596, 8121, 8247, 8199, 7987, 8190, 7506, 8690] avg=8192.0000 max=8690 max_rank=7 min=7506 min_rank=6 imbalance_ratio=1.060791 max_load_reduction=56846 max_load_reduction_pct=0.867401 moved_to_echo=32373 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 560, 4094, 0, 1, 0, 3523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 1, 4091, 0, 38, 409, 3508, 196, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 18, 0, 0, 17, 3899, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4095, 0, 1, 1, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 3844, 1, 171, 0, 71, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4039, 4095, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 2409, 1, 1173, 347, 56, 0, 0, 0, 0, 0, 559, 1, 752, 68, 0, 0, 0, 4095, 2362, 580, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8419, 8031, 8197, 8071, 8139, 8083, 8417] avg=8192.0000 max=8419 max_rank=1 min=8031 min_rank=2 imbalance_ratio=1.027710 max_load_reduction=57117 max_load_reduction_pct=0.871536 moved_to_echo=32710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 560, 4087, 0, 2, 0, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 50, 2, 4084, 0, 62, 578, 3507, 196, 0, 0, 0, 0, 0, 0, 4094, 2, 1, 0, 230, 1, 0, 350, 3897, 0, 0, 0, 0, 0, 0, 0, 7, 2, 4094, 0, 2, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 293, 2, 3886, 2, 144, 0, 85, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 86, 4038, 4093, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 4094, 0, 0, 2452, 2, 1172, 347, 56, 0, 0, 0, 0, 0, 97, 2, 776, 81, 0, 0, 0, 4090, 2362, 580, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7783, 8481, 8575, 8200, 8158, 8224, 8127, 7988] avg=8192.0000 max=8575 max_rank=2 min=7783 min_rank=0 imbalance_ratio=1.046753 max_load_reduction=56961 max_load_reduction_pct=0.869156 moved_to_echo=32497 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 560, 4096, 0, 0, 0, 2814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4096, 0, 38, 401, 3509, 196, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 1, 1, 3900, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 860, 0, 3651, 0, 310, 0, 172, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 4040, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2513, 0, 1173, 347, 56, 0, 0, 0, 0, 0, 937, 0, 326, 7, 0, 32, 0, 4096, 2363, 580, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7470, 8280, 8001, 8195, 8742, 8323, 8184, 8341] avg=8192.0000 max=8742 max_rank=4 min=7470 min_rank=0 imbalance_ratio=1.067139 max_load_reduction=56794 max_load_reduction_pct=0.866608 moved_to_echo=33590 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 560, 4096, 0, 0, 0, 3529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 4096, 0, 0, 453, 3509, 196, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 5, 3900, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 3876, 0, 555, 0, 191, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4040, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2314, 0, 1173, 347, 56, 0, 0, 0, 0, 0, 185, 0, 705, 50, 0, 0, 0, 4096, 2363, 580, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8465, 8018, 8197, 8534, 8172, 7986, 7979] avg=8192.0000 max=8534 max_rank=4 min=7979 min_rank=7 imbalance_ratio=1.041748 max_load_reduction=57002 max_load_reduction_pct=0.869781 moved_to_echo=32671 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 560, 4095, 0, 1, 1, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 4083, 0, 5, 33, 3508, 197, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 173, 0, 0, 64, 3898, 0, 0, 0, 0, 0, 0, 0, 6, 1, 4095, 0, 1, 1, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 3524, 1, 60, 0, 85, 3747, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5, 4038, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 3492, 1, 1173, 347, 56, 0, 0, 0, 0, 0, 94, 1, 733, 61, 0, 0, 0, 4094, 2362, 581, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8337, 7832, 8232, 8197, 7705, 8141, 9166, 7926] avg=8192.0000 max=9166 max_rank=6 min=7705 min_rank=4 imbalance_ratio=1.118896 max_load_reduction=56370 max_load_reduction_pct=0.860138 moved_to_echo=32938 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 552, 4095, 0, 0, 0, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 4095, 0, 108, 51, 3513, 197, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 37, 0, 11, 11, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 3812, 0, 4, 0, 18, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4035, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 3250, 0, 1176, 351, 61, 0, 0, 0, 0, 0, 172, 0, 844, 46, 0, 15, 0, 4096, 2368, 583, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8384, 8054, 8054, 8154, 7676, 8169, 8921, 8124] avg=8192.0000 max=8921 max_rank=6 min=7676 min_rank=4 imbalance_ratio=1.088989 max_load_reduction=56615 max_load_reduction_pct=0.863876 moved_to_echo=32890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 4480, 32768, 0, 128, 128, 28416, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 512, 32384, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 0, 1024, 128, 128, 640, 31232, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 20224, 128, 9472, 2816, 512, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[2432, 128, 30592, 128, 1280, 0, 1408, 30080, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 640, 128, 32768, 0, 256, 2432, 28160, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3072, 128, 6016, 384, 0, 128, 0, 32768, 18944, 4736, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 14, 0, 4096, 4095, 0, 3815, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 2705, 4096, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 2797, 0, 0, 4093, 0, 0, 0, 4096, 2949, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 26, 0, 0, 0, 88, 0, 4096, 4069, 0, 4009, 4096, 0, 2, 0, 0, 4096, 4096, 0, 4096, 0, 0, 3344, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1604, 0, 0, 4096, 0, 0, 0, 4096, 3240, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 10, 2, 43, 1, 0, 2, 111, 3, 4092, 4089, 2, 3874, 4089, 2, 12, 0, 2, 4060, 4094, 0, 4093, 0, 0, 2395, 4094, 0, 2, 4090, 2, 0, 2, 0, 22, 3, 2, 2, 2, 0, 2, 2284, 0, 0, 4093, 0, 1, 2, 4090, 3539, 7, 1, 0, 4092, 35, 0, 0, 2, 2, 2, 0, 4092, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 2, 1, 37, 0, 0, 0, 72, 1, 4094, 4068, 0, 4024, 4094, 1, 2, 0, 1, 4079, 4094, 0, 4095, 0, 0, 2362, 4095, 1, 1, 4095, 1, 0, 1, 0, 32, 1, 1, 15, 0, 0, 1, 2418, 0, 1, 4095, 1, 1, 1, 4094, 3352, 1, 1, 0, 4094, 8, 0, 0, 1, 1, 5, 0, 4095, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 72, 0, 0, 0, 44, 0, 4096, 4006, 0, 3760, 4096, 0, 1, 0, 0, 4096, 4095, 0, 4096, 0, 0, 2488, 4096, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 2840, 1, 0, 4094, 0, 0, 0, 4096, 3168, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 1, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 0, 150, 0, 4096, 3979, 0, 3729, 4096, 0, 3, 0, 0, 4096, 4096, 0, 4096, 0, 0, 2713, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2067, 0, 0, 4096, 0, 0, 0, 4096, 3721, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 2, 1, 59, 0, 0, 0, 256, 0, 4095, 4087, 0, 3929, 4094, 1, 1, 0, 1, 4082, 4095, 0, 4095, 0, 0, 2090, 4095, 1, 1, 4094, 1, 0, 1, 0, 9, 1, 1, 0, 0, 0, 1, 2710, 0, 1, 4095, 0, 0, 1, 4094, 3223, 6, 1, 0, 4095, 21, 0, 0, 1, 1, 3, 0, 4095, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 29, 0, 8, 0, 45, 0, 4096, 4015, 0, 3968, 4096, 0, 0, 0, 0, 4096, 4081, 0, 4096, 0, 0, 1759, 4096, 0, 0, 4090, 0, 0, 0, 0, 31, 0, 0, 3, 0, 0, 0, 3205, 9, 0, 4086, 0, 0, 0, 4096, 3302, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 12, 3, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 3942, 3324, 878, 8, 0, 0, 0, 0, 14, 0, 0, 4095, 0, 3815, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 1616, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 2797, 0, 0, 4093, 0, 0, 0, 1602, 2, 0, 0, 0, 0, 0, 0, 4096, 2949, 0, 0, 0, 772, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8078, 8201, 8417, 8183, 8494, 7820, 8190] avg=8192.0000 max=8494 max_rank=5 min=7820 min_rank=6 imbalance_ratio=1.036865 max_load_reduction=57042 max_load_reduction_pct=0.870392 moved_to_echo=32687 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 26, 0, 0, 0, 3942, 3324, 878, 8, 0, 0, 0, 0, 88, 0, 0, 4069, 0, 4009, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3344, 1616, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 1604, 0, 0, 4096, 0, 0, 0, 1602, 2, 0, 0, 0, 0, 0, 0, 4096, 3240, 0, 0, 0, 772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8320, 8203, 9056, 8177, 7304, 8108, 8190] avg=8192.0000 max=9056 max_rank=3 min=7304 min_rank=5 imbalance_ratio=1.105469 max_load_reduction=56480 max_load_reduction_pct=0.861816 moved_to_echo=31779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 10, 2, 43, 1, 0, 2, 3935, 3320, 878, 8, 0, 0, 0, 0, 111, 3, 0, 4089, 2, 3874, 154, 2, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2, 4060, 4094, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2395, 1615, 0, 2, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 22, 3, 2, 2, 2, 0, 4092, 4076, 0, 0, 0, 0, 0, 0, 2, 2284, 0, 0, 4093, 0, 1, 2, 1601, 2, 0, 0, 0, 0, 0, 0, 4090, 3539, 7, 1, 0, 772, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 0, 4092, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8201, 8235, 8177, 8104, 8201, 7985, 8444, 8189] avg=8192.0000 max=8444 max_rank=6 min=7985 min_rank=5 imbalance_ratio=1.030762 max_load_reduction=57092 max_load_reduction_pct=0.871155 moved_to_echo=32819 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 72, 0, 0, 0, 3942, 3324, 878, 8, 0, 0, 0, 0, 44, 0, 0, 4006, 0, 3760, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4095, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2488, 1616, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 2840, 1, 0, 4094, 0, 0, 0, 1602, 2, 0, 0, 0, 0, 0, 0, 4096, 3168, 0, 0, 0, 772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8224, 7964, 8201, 8200, 8180, 8539, 8036, 8192] avg=8192.0000 max=8539 max_rank=5 min=7964 min_rank=1 imbalance_ratio=1.042358 max_load_reduction=56997 max_load_reduction_pct=0.869705 moved_to_echo=32947 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 21, 0, 0, 0, 3942, 3324, 878, 8, 0, 0, 0, 0, 150, 0, 0, 3979, 0, 3729, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2713, 1616, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 0, 4096, 0, 0, 0, 1602, 2, 0, 0, 0, 0, 0, 0, 4096, 3721, 0, 0, 0, 772, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8173, 8012, 8204, 8424, 8177, 7767, 8590, 8189] avg=8192.0000 max=8590 max_rank=6 min=7767 min_rank=5 imbalance_ratio=1.048584 max_load_reduction=56946 max_load_reduction_pct=0.868927 moved_to_echo=32723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 1, 59, 0, 0, 0, 3940, 3323, 878, 8, 0, 0, 0, 0, 256, 0, 0, 4087, 0, 3929, 154, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4082, 4095, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2090, 1616, 1, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 1, 1, 0, 0, 0, 4095, 4078, 0, 0, 0, 0, 0, 0, 1, 2710, 0, 1, 4095, 0, 0, 1, 1601, 2, 0, 0, 0, 0, 0, 0, 4094, 3223, 6, 1, 0, 772, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 0, 4095, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8212, 8427, 8188, 7803, 8185, 8411, 8117, 8193] avg=8192.0000 max=8427 max_rank=1 min=7803 min_rank=3 imbalance_ratio=1.028687 max_load_reduction=57109 max_load_reduction_pct=0.871414 moved_to_echo=32906 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 1, 37, 0, 0, 0, 3940, 3322, 878, 8, 0, 0, 0, 0, 72, 1, 0, 4068, 0, 4024, 154, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4079, 4094, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2362, 1616, 1, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 32, 1, 1, 15, 0, 0, 4094, 4078, 0, 0, 0, 0, 0, 0, 1, 2418, 0, 1, 4095, 1, 1, 1, 1601, 2, 0, 0, 0, 0, 0, 0, 4094, 3352, 1, 1, 0, 772, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5, 0, 4095, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8320, 8185, 8076, 8222, 8121, 8228, 8195] avg=8192.0000 max=8320 max_rank=1 min=8076 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32766 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 29, 0, 8, 0, 3944, 3327, 883, 15, 0, 0, 0, 0, 45, 0, 0, 4015, 0, 3968, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 1607, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 3, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 3205, 9, 0, 4086, 0, 0, 0, 1606, 9, 0, 0, 0, 0, 0, 0, 4096, 3302, 0, 0, 0, 769, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8180, 8177, 7456, 8211, 8915, 8193, 8198] avg=8192.0000 max=8915 max_rank=5 min=7456 min_rank=3 imbalance_ratio=1.088257 max_load_reduction=56621 max_load_reduction_pct=0.863968 moved_to_echo=33517 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 0, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 384, 128, 128, 128, 31616, 26624, 7040, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 19968, 128, 128, 32768, 128, 128, 128, 12928, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 26496, 128, 128, 0, 6272, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 19968, 12928, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[896, 128, 0, 32512, 128, 31232, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2, 0, 0, 4096, 0, 4018, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4048, 4096, 93, 0, 0, 4, 0, 16, 3213, 0, 4007, 0, 2721, 0, 0, 0, 0, 772, 0, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1596, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 1, 0, 4095, 1, 3591, 1, 1, 4094, 0, 1, 4087, 4094, 0, 0, 4095, 1, 0, 4088, 4095, 160, 0, 2, 5, 1, 1, 2350, 0, 4095, 1, 1921, 0, 1, 0, 0, 2842, 0, 0, 4094, 0, 1, 1, 0, 0, 4095, 1, 1, 0, 2, 1, 0, 1, 0, 4093, 4095, 1, 1429, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3, 0, 1, 4093, 0, 2, 0, 4094, 0, 3597, 2, 2, 4092, 0, 2, 4083, 4091, 0, 3, 4094, 2, 0, 4092, 4094, 34, 0, 12, 0, 2, 0, 2846, 0, 4094, 3, 1839, 0, 2, 0, 0, 1737, 0, 0, 4038, 0, 2, 2, 0, 0, 4093, 1, 2, 0, 0, 2, 0, 2, 0, 4093, 4094, 0, 2288, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4096, 0, 2761, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4089, 4096, 272, 0, 2, 1, 0, 151, 2014, 0, 4095, 0, 2095, 0, 0, 0, 0, 3766, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1233, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4029, 0, 0, 4090, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4039, 4096, 197, 0, 0, 3, 0, 1, 3013, 0, 4042, 0, 1429, 0, 0, 0, 0, 1869, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 1873, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3480, 0, 0, 4093, 3, 0, 4096, 4096, 0, 0, 4095, 0, 0, 4054, 4096, 141, 0, 0, 30, 0, 102, 1963, 0, 3997, 0, 3330, 0, 0, 0, 0, 3308, 0, 0, 4070, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 101, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 6, 1, 0, 4095, 1, 3838, 1, 1, 4092, 0, 1, 4089, 4093, 0, 0, 4095, 1, 0, 4095, 4095, 62, 0, 3, 1, 1, 5, 3031, 0, 4095, 3, 1251, 0, 1, 0, 0, 946, 0, 0, 4094, 0, 1, 1, 0, 0, 4094, 1, 1, 0, 2, 1, 0, 1, 0, 4095, 4095, 1, 3150, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 22, 0, 0, 4096, 0, 3429, 0, 0, 4074, 24, 0, 4096, 4096, 0, 0, 4084, 0, 0, 3996, 4090, 393, 1, 0, 35, 0, 40, 2351, 0, 3827, 0, 1706, 0, 0, 0, 0, 3379, 5, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 4096, 4096, 0, 1304, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 2, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 4095, 0, 0, 504, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4096, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 4, 0, 16, 3213, 0, 4096, 1278, 0, 0, 0, 0, 0, 0, 3804, 0, 2721, 0, 0, 0, 0, 772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 2782, 1327, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2265, 1621, 203, 3, 0, 0, 0, 0, 0, 4096, 2475, 0, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8617, 8180, 8700, 7297, 8196, 8188, 8167] avg=8192.0000 max=8700 max_rank=3 min=7297 min_rank=4 imbalance_ratio=1.062012 max_load_reduction=56836 max_load_reduction_pct=0.867249 moved_to_echo=31848 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 4090, 0, 0, 504, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4096, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 3, 0, 1, 3013, 0, 4096, 1278, 0, 0, 0, 0, 0, 0, 3837, 0, 1429, 0, 0, 0, 0, 1869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 2782, 1327, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2265, 1621, 205, 3, 0, 0, 0, 0, 0, 4095, 2475, 0, 1873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8623, 8171, 8588, 7135, 8197, 8190, 8443] avg=8192.0000 max=8623 max_rank=1 min=7135 min_rank=4 imbalance_ratio=1.052612 max_load_reduction=56913 max_load_reduction_pct=0.868423 moved_to_echo=31965 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 1, 4093, 0, 2, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3597, 2, 2, 4092, 0, 2, 503, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 3, 4094, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 12, 0, 2, 0, 2846, 0, 4094, 1277, 0, 0, 0, 0, 0, 0, 3887, 3, 1839, 0, 2, 0, 0, 1737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 2, 2, 0, 0, 2779, 1323, 0, 0, 0, 0, 0, 0, 4093, 1, 2, 0, 0, 2, 0, 2, 2257, 1620, 207, 3, 0, 0, 0, 0, 0, 4093, 2474, 0, 2288, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8198, 8226, 8265, 7468, 8144, 8187, 8858] avg=8192.0000 max=8858 max_rank=7 min=7468 min_rank=4 imbalance_ratio=1.081299 max_load_reduction=56678 max_load_reduction_pct=0.864838 moved_to_echo=32657 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2761, 0, 0, 4096, 0, 0, 504, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 2, 1, 0, 151, 2014, 0, 4096, 1278, 0, 0, 0, 0, 0, 0, 3888, 0, 2095, 0, 0, 0, 0, 3766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2782, 1327, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2265, 1621, 207, 3, 0, 0, 0, 0, 0, 4096, 2475, 0, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 7361, 8221, 7814, 9749, 8205, 8192, 7804] avg=8192.0000 max=9749 max_rank=4 min=7361 min_rank=1 imbalance_ratio=1.190063 max_load_reduction=55787 max_load_reduction_pct=0.851242 moved_to_echo=33950 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3591, 1, 1, 4094, 0, 1, 503, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 4095, 1, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 2, 5, 1, 1, 2350, 0, 4095, 1278, 0, 0, 0, 0, 0, 0, 3888, 1, 1921, 0, 1, 0, 0, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 1, 0, 0, 2781, 1324, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 2, 1, 0, 1, 2260, 1620, 207, 3, 0, 0, 0, 0, 0, 4093, 2475, 1, 1429, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8192, 8219, 7892, 8653, 8201, 8191, 7999] avg=8192.0000 max=8653 max_rank=4 min=7892 min_rank=3 imbalance_ratio=1.056274 max_load_reduction=56883 max_load_reduction_pct=0.867966 moved_to_echo=33044 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 0, 0, 4093, 3, 0, 504, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4095, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 30, 0, 102, 1963, 0, 4096, 1278, 0, 0, 0, 0, 0, 0, 3795, 0, 3330, 0, 0, 0, 0, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 2782, 1327, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 2265, 1621, 202, 3, 0, 0, 0, 0, 0, 4096, 2475, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8080, 8185, 7610, 10433, 8179, 8188, 6672] avg=8192.0000 max=10433 max_rank=4 min=6672 min_rank=7 imbalance_ratio=1.273560 max_load_reduction=55103 max_load_reduction_pct=0.840805 moved_to_echo=33472 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 22, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3429, 0, 0, 4074, 24, 0, 501, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 4084, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 1, 0, 35, 0, 40, 2351, 0, 4090, 1282, 0, 0, 0, 0, 0, 0, 3627, 0, 1706, 0, 0, 0, 0, 3379, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4092, 0, 0, 0, 0, 0, 2787, 1329, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 2266, 1625, 200, 10, 0, 0, 0, 0, 0, 4096, 2471, 0, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8028, 8107, 8192, 8712, 8213, 8209, 7871] avg=8192.0000 max=8712 max_rank=4 min=7871 min_rank=7 imbalance_ratio=1.063477 max_load_reduction=56824 max_load_reduction_pct=0.867065 moved_to_echo=33005 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 6, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3838, 1, 1, 4092, 0, 1, 503, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4095, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 3, 1, 1, 5, 3031, 0, 4095, 1277, 0, 0, 0, 0, 0, 0, 3888, 3, 1251, 0, 1, 0, 0, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 1, 0, 0, 2780, 1325, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 2, 1, 0, 1, 2261, 1620, 207, 3, 0, 0, 0, 0, 0, 4095, 2475, 1, 3150, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8437, 8227, 8475, 6089, 8201, 8191, 9722] avg=8192.0000 max=9722 max_rank=7 min=6089 min_rank=4 imbalance_ratio=1.186768 max_load_reduction=55814 max_load_reduction_pct=0.851654 moved_to_echo=32203 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[384, 0, 128, 32768, 128, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[30720, 128, 16384, 0, 128, 0, 0, 18688, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 28800, 128, 128, 32768, 128, 128, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32768, 19840, 128, 13056, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 0, 128, 128, 0, 0, 22272, 10624, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1408, 128, 128, 128, 128, 384, 20864, 0, 32768, 10240, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 128, 18176, 13056, 1664, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3931, 3970, 0, 0, 0, 0, 47, 0, 9, 0, 4087, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4024, 1117, 0, 0, 4088, 4096, 0, 28, 0, 0, 1, 0, 7, 0, 4096, 0, 0, 4090, 3274, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4081, 4017, 0, 0, 0, 0, 1, 0, 11, 0, 3983, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 286, 2, 0, 4096, 4096, 0, 1532, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4089, 2479, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3979, 3802, 0, 0, 0, 0, 95, 0, 19, 0, 4070, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4085, 676, 1, 0, 4096, 4096, 0, 17, 0, 1, 1, 0, 0, 0, 4096, 0, 0, 4094, 3736, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 0, 9, 0, 0, 3896, 3980, 0, 1, 1, 0, 0, 1, 55, 1, 4085, 0, 1, 1, 0, 4094, 1, 1, 1, 10, 4095, 3, 1, 4095, 0, 0, 4091, 1, 0, 1, 11, 4094, 655, 3, 0, 4094, 4093, 0, 183, 1, 0, 25, 1, 11, 0, 4095, 1, 1, 4095, 3511, 0, 4047, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 0, 26, 0, 0, 3397, 3907, 0, 2, 2, 1, 0, 2, 3, 4, 4091, 2, 5, 2, 3, 4094, 2, 2, 1, 27, 4094, 5, 1, 4091, 0, 0, 4087, 1, 0, 2, 92, 4059, 2079, 19, 2, 4082, 4091, 0, 170, 2, 0, 13, 2, 1, 0, 4094, 2, 2, 4094, 2749, 0, 3943, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4060, 3949, 0, 0, 0, 0, 87, 0, 21, 0, 4057, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 4076, 499, 9, 0, 4089, 4096, 0, 852, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 4086, 2899, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 9, 0, 0, 3738, 3964, 0, 1, 1, 0, 2, 1, 71, 1, 4077, 1, 0, 1, 1, 4094, 1, 1, 1, 1, 4095, 3, 1, 4095, 0, 0, 4087, 0, 0, 1, 18, 4071, 1039, 10, 0, 4094, 4093, 0, 42, 1, 0, 3, 1, 4, 0, 4095, 1, 1, 4095, 3505, 0, 4025, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 4038, 4069, 0, 0, 0, 0, 86, 0, 87, 0, 3968, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 4078, 579, 1, 0, 4096, 4096, 0, 383, 0, 2, 2, 0, 0, 0, 4088, 0, 0, 4085, 3224, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 3970, 0, 0, 0, 0, 343, 0, 0, 0, 0, 0, 0, 0, 47, 0, 9, 0, 4087, 0, 0, 0, 4058, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 1117, 0, 0, 3230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 1, 0, 7, 0, 4096, 3133, 514, 38, 0, 0, 0, 0, 0, 0, 0, 957, 3274, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8244, 8207, 8191, 8193, 8371, 7817, 8327] avg=8192.0000 max=8371 max_rank=5 min=7817 min_rank=6 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32708 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 3802, 0, 0, 0, 0, 343, 0, 0, 0, 0, 0, 0, 0, 95, 0, 19, 0, 4070, 0, 0, 0, 4058, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 676, 1, 0, 3237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1, 1, 0, 0, 0, 4096, 3136, 515, 38, 0, 0, 0, 0, 0, 0, 0, 958, 3736, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8124, 8248, 8192, 8193, 7999, 7804, 8790] avg=8192.0000 max=8790 max_rank=7 min=7804 min_rank=6 imbalance_ratio=1.072998 max_load_reduction=56746 max_load_reduction_pct=0.865875 moved_to_echo=32786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 0, 26, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3397, 3907, 0, 2, 2, 1, 342, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4, 4091, 2, 5, 2, 4053, 6, 0, 0, 0, 0, 0, 0, 3, 4094, 2, 2, 1, 27, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 4087, 1, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 92, 4059, 2079, 19, 2, 3226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 2, 0, 13, 2, 1, 0, 4094, 3136, 513, 38, 0, 0, 0, 0, 0, 2, 2, 958, 2749, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 7651, 8168, 8228, 8183, 9477, 7969, 7654] avg=8192.0000 max=9477 max_rank=5 min=7651 min_rank=1 imbalance_ratio=1.156860 max_load_reduction=56059 max_load_reduction_pct=0.855392 moved_to_echo=33283 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4017, 0, 0, 0, 0, 343, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 3983, 0, 0, 0, 4058, 6, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 286, 2, 0, 3237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1532, 0, 1, 0, 0, 0, 0, 4096, 3132, 515, 38, 0, 0, 0, 0, 0, 0, 0, 957, 2479, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8441, 8059, 8190, 8193, 7621, 9314, 7532] avg=8192.0000 max=9314 max_rank=6 min=7532 min_rank=7 imbalance_ratio=1.136963 max_load_reduction=56222 max_load_reduction_pct=0.857880 moved_to_echo=32660 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 0, 9, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 3980, 0, 1, 1, 0, 343, 0, 0, 0, 0, 0, 0, 0, 0, 1, 55, 1, 4085, 0, 1, 1, 4055, 6, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 1, 10, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4091, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 11, 4094, 655, 3, 0, 3235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 1, 0, 25, 1, 11, 0, 4095, 3137, 515, 38, 0, 0, 0, 0, 0, 1, 1, 958, 3511, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8221, 8205, 8205, 8190, 7998, 8006, 8518] avg=8192.0000 max=8518 max_rank=7 min=7998 min_rank=5 imbalance_ratio=1.039795 max_load_reduction=57018 max_load_reduction_pct=0.870026 moved_to_echo=32712 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 3949, 0, 0, 0, 0, 343, 0, 0, 0, 0, 0, 0, 0, 87, 0, 21, 0, 4057, 0, 0, 0, 4058, 6, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 499, 9, 0, 3231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 852, 0, 2, 0, 0, 0, 0, 4095, 3130, 514, 38, 0, 0, 0, 0, 0, 0, 0, 956, 2899, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8352, 8229, 8177, 8194, 7815, 8631, 7951] avg=8192.0000 max=8631 max_rank=6 min=7815 min_rank=5 imbalance_ratio=1.053589 max_load_reduction=56905 max_load_reduction_pct=0.868301 moved_to_echo=32591 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 9, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3738, 3964, 0, 1, 1, 0, 343, 0, 0, 0, 0, 0, 0, 0, 2, 1, 71, 1, 4077, 1, 0, 1, 4056, 5, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 1, 1, 1, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4087, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 18, 4071, 1039, 10, 0, 3235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1, 0, 3, 1, 4, 0, 4095, 3137, 515, 37, 0, 0, 0, 0, 0, 1, 1, 958, 3505, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8047, 8215, 8197, 8185, 8373, 7835, 8490] avg=8192.0000 max=8490 max_rank=7 min=7835 min_rank=6 imbalance_ratio=1.036377 max_load_reduction=57046 max_load_reduction_pct=0.870453 moved_to_echo=32883 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 11, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 4069, 0, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0, 0, 86, 0, 87, 0, 3968, 0, 0, 0, 4059, 5, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 4078, 579, 1, 0, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 0, 2, 2, 0, 0, 0, 4088, 3131, 517, 37, 0, 0, 0, 0, 0, 0, 0, 954, 3224, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8456, 8205, 8156, 8205, 7882, 8160, 8274] avg=8192.0000 max=8456 max_rank=1 min=7882 min_rank=5 imbalance_ratio=1.032227 max_load_reduction=57080 max_load_reduction_pct=0.870972 moved_to_echo=32521 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32640, 7040, 128, 128, 25856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 0, 32768, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 7680, 25472, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[384, 128, 384, 128, 32512, 128, 128, 128, 32512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[3328, 128, 128, 128, 128, 128, 0, 32768, 25088, 4224, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 31232, 31744, 0, 128, 128, 128, 2816, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 4072, 416, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4037, 0, 0, 0, 4087, 0, 0, 0, 0, 113, 0, 0, 0, 3661, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4086, 715, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3787, 0, 0, 0, 4091, 0, 0, 0, 0, 216, 0, 0, 0, 3531, 0, 0, 4096, 4052, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 4092, 1229, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3917, 0, 0, 0, 3993, 0, 0, 0, 0, 5, 0, 0, 0, 3127, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 3756, 0, 0, 2, 1, 0, 4095, 1, 1, 0, 16, 0, 4038, 849, 1, 1, 1, 1, 0, 1, 4095, 1, 1, 1, 0, 1, 0, 4092, 4095, 0, 1, 3932, 3, 1, 0, 4092, 0, 0, 0, 1, 266, 1, 1, 0, 3518, 0, 3, 4092, 4095, 0, 4, 0, 0, 0, 1, 4095, 0, 4095, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 4095, 11, 0, 0, 0, 0, 4066, 837, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 3816, 0, 0, 0, 4068, 0, 0, 0, 0, 83, 0, 0, 0, 3573, 0, 0, 4096, 4025, 0, 1, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4094, 0, 4094, 3915, 1, 0, 8, 2, 2, 4094, 0, 2, 0, 7, 4, 3978, 785, 0, 2, 2, 2, 0, 2, 4093, 2, 2, 2, 0, 2, 0, 4088, 4092, 0, 1, 3637, 5, 2, 0, 4077, 0, 0, 0, 2, 340, 0, 2, 1, 3725, 0, 6, 4085, 4094, 1, 0, 2, 0, 0, 1, 4092, 0, 4094, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 3938, 1, 0, 4, 1, 0, 4095, 1, 1, 0, 13, 0, 3982, 448, 1, 1, 1, 1, 1, 1, 4094, 0, 1, 1, 0, 1, 0, 4092, 4095, 0, 1, 3982, 3, 1, 0, 4092, 0, 0, 0, 1, 155, 0, 1, 0, 3859, 0, 2, 4090, 4095, 0, 4, 0, 0, 0, 1, 4095, 0, 4095, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4078, 4096, 0, 0, 0, 0, 0, 4094, 78, 0, 0, 0, 0, 4091, 256, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 4096, 4086, 0, 0, 3999, 0, 0, 0, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 3852, 0, 0, 4096, 4067, 0, 20, 0, 0, 0, 0, 4096, 16, 4096, 2, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[89, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4086, 24, 0, 0, 0, 0, 0, 0, 0, 715, 0, 0, 0, 0, 0, 0, 4096, 3402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3787, 0, 0, 0, 4091, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 3531, 4096, 339, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4052, 0, 0, 0, 0, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8208, 8213, 8192, 8104, 8182, 8165, 8191] avg=8192.0000 max=8281 max_rank=0 min=8104 min_rank=4 imbalance_ratio=1.010864 max_load_reduction=57255 max_load_reduction_pct=0.873642 moved_to_echo=32642 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[89, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 4072, 24, 0, 0, 0, 0, 0, 0, 0, 416, 0, 0, 0, 0, 0, 0, 4096, 3402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 4087, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 3661, 4096, 339, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8202, 7914, 8192, 8350, 8209, 8197, 8191] avg=8192.0000 max=8350 max_rank=4 min=7914 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=57186 max_load_reduction_pct=0.872589 moved_to_echo=32947 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[88, 0, 4094, 3915, 1, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 2, 0, 7, 4, 3978, 24, 0, 0, 0, 0, 0, 0, 0, 785, 0, 2, 2, 2, 0, 2, 4093, 3401, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 0, 2, 0, 4088, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3637, 5, 2, 0, 4077, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 340, 0, 2, 1, 3725, 4094, 339, 0, 0, 0, 0, 0, 0, 0, 6, 4085, 4094, 1, 0, 2, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8108, 8111, 8287, 8188, 7948, 8503, 8204, 8187] avg=8192.0000 max=8503 max_rank=5 min=7948 min_rank=4 imbalance_ratio=1.037964 max_load_reduction=57033 max_load_reduction_pct=0.870255 moved_to_echo=32842 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[89, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 4092, 24, 0, 0, 0, 0, 0, 0, 0, 1229, 0, 0, 0, 0, 0, 0, 4096, 3402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 0, 3993, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3127, 4096, 339, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8281, 8235, 8727, 8192, 8136, 7567, 8207, 8191] avg=8192.0000 max=8727 max_rank=2 min=7567 min_rank=5 imbalance_ratio=1.065308 max_load_reduction=56809 max_load_reduction_pct=0.866837 moved_to_echo=32101 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[88, 0, 4095, 3756, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 16, 0, 4038, 24, 0, 0, 0, 0, 0, 0, 0, 849, 1, 1, 1, 1, 0, 1, 4095, 3402, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 4092, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3932, 3, 1, 0, 4092, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 266, 1, 1, 0, 3518, 4095, 339, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 4095, 0, 4, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7942, 8175, 8351, 8191, 8255, 8221, 8210, 8191] avg=8192.0000 max=8351 max_rank=2 min=7942 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[89, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 11, 0, 0, 0, 0, 4066, 24, 0, 0, 0, 0, 0, 0, 0, 837, 0, 0, 0, 0, 0, 0, 4096, 3402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 0, 0, 0, 4068, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 3573, 4096, 339, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4025, 0, 1, 0, 0, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8196, 8335, 8193, 8110, 8091, 8139, 8193] avg=8192.0000 max=8335 max_rank=2 min=8091 min_rank=5 imbalance_ratio=1.017456 max_load_reduction=57201 max_load_reduction_pct=0.872818 moved_to_echo=32533 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[63, 0, 4078, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 78, 0, 0, 0, 0, 4091, 30, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 4096, 3407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 4086, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 4096, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 3852, 4096, 343, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 0, 20, 0, 0, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8237, 8293, 7759, 8198, 8328, 8313, 8207, 8201] avg=8192.0000 max=8328 max_rank=4 min=7759 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=57208 max_load_reduction_pct=0.872925 moved_to_echo=33049 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[88, 0, 4095, 3938, 1, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 13, 0, 3982, 24, 0, 0, 0, 0, 0, 0, 0, 448, 1, 1, 1, 1, 1, 1, 4094, 3402, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 4092, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3982, 3, 1, 0, 4092, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 155, 0, 1, 0, 3859, 4095, 339, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 4095, 0, 4, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8127, 8116, 7950, 8190, 8305, 8450, 8207, 8191] avg=8192.0000 max=8450 max_rank=5 min=7950 min_rank=2 imbalance_ratio=1.031494 max_load_reduction=57086 max_load_reduction_pct=0.871063 moved_to_echo=33153 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[768, 0, 32768, 32128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 31232, 128, 128, 0, 32640, 0, 1920, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 32512, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[5632, 128, 128, 128, 128, 128, 128, 32768, 27264, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 32640, 128, 128, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 1280, 128, 128, 128, 28928, 32768, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4032, 0, 19, 1, 2, 0, 4094, 4094, 11, 2, 1, 0, 0, 0, 2, 2, 0, 0, 676, 119, 2, 83, 2, 4094, 0, 2, 4085, 4093, 0, 285, 4094, 15, 4092, 4094, 3017, 0, 0, 0, 0, 2, 2, 4089, 4057, 11, 0, 0, 0, 1, 4093, 4094, 0, 147, 2, 4072, 0, 1, 10, 2, 2, 3934, 1, 2, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 3166, 0, 0, 0, 0, 0, 0, 4096, 3794, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 4096, 0, 0, 596, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 2950, 13, 0, 0, 0, 0, 0, 4096, 3743, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 753, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 78, 0, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 21, 4096, 0, 4096, 4096, 3423, 0, 0, 0, 0, 0, 0, 4096, 4047, 0, 0, 0, 0, 29, 4096, 4096, 0, 0, 0, 4096, 0, 0, 594, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 56, 4096, 0, 4096, 4096, 3887, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 154, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4076, 0, 12, 1, 1, 0, 4095, 4095, 4, 1, 1, 0, 0, 0, 1, 1, 0, 0, 510, 37, 1, 68, 0, 4095, 0, 1, 4090, 4094, 0, 110, 4095, 4, 4095, 4095, 3204, 0, 0, 0, 0, 1, 1, 4093, 4084, 4, 0, 1, 0, 0, 4095, 4095, 0, 6, 1, 4087, 0, 0, 243, 1, 1, 4033, 1, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4073, 0, 6, 1, 1, 0, 4095, 4095, 7, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1099, 40, 1, 35, 0, 4095, 0, 1, 4090, 4094, 0, 181, 4095, 5, 4094, 4095, 2933, 0, 0, 0, 0, 1, 1, 4095, 3890, 3, 0, 0, 0, 0, 4094, 4095, 0, 4, 1, 4089, 1, 0, 77, 1, 1, 4039, 0, 1, 1, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4089, 4081, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 658, 7, 0, 0, 0, 4094, 15, 0, 4096, 4096, 0, 4, 4096, 0, 4096, 4096, 2837, 0, 0, 0, 0, 0, 0, 4096, 3500, 0, 0, 0, 0, 39, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1154, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 634, 0, 0, 0, 0, 4096, 3092, 407, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 918, 3166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3794, 0, 0, 0, 0, 2, 86, 83, 14, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 596, 0, 0, 4096, 0, 0, 0, 0, 3678, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8187, 8229, 8109, 8180, 8075, 8178, 8375] avg=8192.0000 max=8375 max_rank=7 min=8075 min_rank=5 imbalance_ratio=1.022339 max_load_reduction=57161 max_load_reduction_pct=0.872208 moved_to_echo=32808 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 733, 0, 0, 0, 0, 4096, 3092, 407, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 918, 2950, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3743, 0, 0, 0, 0, 0, 86, 83, 14, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 0, 0, 4096, 0, 0, 0, 0, 3678, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8188, 8328, 8109, 7977, 8022, 8178, 8531] avg=8192.0000 max=8531 max_rank=7 min=7977 min_rank=4 imbalance_ratio=1.041382 max_load_reduction=57005 max_load_reduction_pct=0.869827 moved_to_echo=32708 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 3092, 407, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4013, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 918, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 86, 83, 14, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 4096, 0, 0, 0, 0, 3678, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8187, 7596, 8165, 8901, 8373, 8178, 7933] avg=8192.0000 max=8901 max_rank=4 min=7596 min_rank=2 imbalance_ratio=1.086548 max_load_reduction=56635 max_load_reduction_pct=0.864182 moved_to_echo=33385 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 12, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 4096, 3092, 406, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4013, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 918, 3423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4047, 0, 0, 0, 0, 29, 86, 83, 14, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 0, 0, 4096, 0, 0, 0, 0, 3677, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8189, 7672, 8131, 8437, 8355, 8178, 8371] avg=8192.0000 max=8437 max_rank=4 min=7672 min_rank=2 imbalance_ratio=1.029907 max_load_reduction=57099 max_load_reduction_pct=0.871262 moved_to_echo=33341 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4076, 0, 12, 1, 1, 0, 12, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 1, 0, 0, 0, 1, 1, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 510, 37, 1, 68, 0, 4095, 3091, 406, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 4011, 0, 110, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 919, 3204, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 4084, 4, 0, 1, 0, 0, 85, 83, 14, 0, 0, 0, 0, 0, 4081, 0, 0, 6, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 1, 1, 4033, 1, 1, 1, 0, 3677, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8195, 8208, 8216, 8219, 8365, 8175, 7961] avg=8192.0000 max=8365 max_rank=5 min=7961 min_rank=7 imbalance_ratio=1.021118 max_load_reduction=57171 max_load_reduction_pct=0.872360 moved_to_echo=32720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4032, 0, 19, 1, 2, 0, 12, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 1, 0, 0, 0, 2, 2, 4094, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 676, 119, 2, 83, 2, 4094, 3090, 406, 0, 0, 0, 0, 0, 0, 0, 2, 4085, 4010, 0, 285, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 919, 3017, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4089, 4057, 11, 0, 0, 0, 1, 85, 83, 14, 0, 0, 0, 0, 0, 4079, 0, 0, 147, 2, 4072, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2, 2, 3934, 1, 2, 1, 0, 3676, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8160, 8202, 8472, 8397, 8030, 8342, 8301, 7632] avg=8192.0000 max=8472 max_rank=2 min=7632 min_rank=7 imbalance_ratio=1.034180 max_load_reduction=57064 max_load_reduction_pct=0.870728 moved_to_echo=32305 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 6, 1, 1, 0, 12, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 1, 0, 0, 0, 1, 1, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 40, 1, 35, 0, 4095, 3091, 406, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 4011, 0, 181, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 919, 2933, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 3890, 3, 0, 0, 0, 0, 85, 83, 14, 0, 0, 0, 0, 0, 4080, 0, 0, 4, 1, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 1, 1, 4039, 0, 1, 1, 2, 3677, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8188, 8198, 8767, 8288, 7947, 8171, 8175, 7802] avg=8192.0000 max=8767 max_rank=2 min=7802 min_rank=7 imbalance_ratio=1.070190 max_load_reduction=56769 max_load_reduction_pct=0.866226 moved_to_echo=32095 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 2, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 658, 7, 0, 0, 0, 4094, 3095, 410, 0, 0, 0, 0, 0, 0, 15, 0, 4096, 4006, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 912, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3500, 0, 0, 0, 0, 39, 89, 90, 19, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 0, 0, 4096, 0, 0, 0, 0, 3677, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8179, 8190, 8264, 8121, 7845, 7833, 8173, 8931] avg=8192.0000 max=8931 max_rank=7 min=7833 min_rank=5 imbalance_ratio=1.090210 max_load_reduction=56605 max_load_reduction_pct=0.863724 moved_to_echo=32782 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[3584, 128, 128, 32512, 128, 128, 128, 128, 29440, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 7424, 25472, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 31232, 128, 0, 128, 0, 128, 768, 768, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 0, 256, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 32128, 0, 768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 4480, 256, 128, 256, 128, 32768, 24832, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 4096, 0, 0, 0, 156, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 902, 0, 4094, 0, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 520, 0, 4093, 0, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4089, 0, 90, 2, 3583, 1, 0, 0, 21, 0, 1, 0, 0, 4094, 279, 4094, 4012, 4090, 0, 0, 0, 1268, 0, 2, 4094, 0, 0, 2, 8, 4093, 4094, 0, 2, 0, 3, 2, 0, 0, 1, 0, 4094, 4075, 0, 4093, 721, 2, 4092, 23, 2335, 0, 3, 0, 2, 1, 0, 0, 2, 4094, 3, 4064, 0, 5, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 98, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 463, 0, 4095, 0, 3536, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4092, 0, 74, 1, 3690, 1, 1, 0, 2, 0, 0, 0, 0, 4095, 193, 4095, 4022, 4094, 0, 0, 0, 933, 0, 1, 4095, 0, 0, 1, 3, 4094, 4095, 0, 1, 0, 1, 1, 0, 0, 0, 0, 4095, 4090, 0, 4095, 1753, 1, 4095, 15, 1620, 0, 1, 1, 1, 1, 0, 0, 1, 4095, 2, 4081, 0, 3, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 4096, 4096, 0, 0, 0, 43, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 854, 0, 4070, 0, 3237, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4092, 0, 100, 1, 3663, 1, 1, 0, 3, 0, 1, 0, 0, 4095, 102, 4095, 4021, 4093, 0, 0, 0, 879, 0, 1, 4095, 0, 0, 1, 3, 4094, 4095, 0, 1, 0, 0, 1, 0, 0, 0, 1, 4095, 4092, 0, 4095, 755, 1, 4095, 15, 2765, 0, 1, 1, 0, 1, 0, 0, 1, 4095, 1, 4080, 0, 3, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 4084, 0, 4082, 4096, 4096, 0, 0, 0, 40, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 13, 4096, 166, 0, 4060, 0, 3905, 0, 0, 0, 0, 0, 0, 24, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4082, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3010, 756, 256, 0, 0, 0, 0, 0, 3699, 4096, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 0, 0, 902, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3055, 0, 0, 0, 0, 0, 0, 4096, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8318, 8118, 7951, 8191, 8192, 8336, 8237, 8193] avg=8192.0000 max=8336 max_rank=5 min=7951 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=57200 max_load_reduction_pct=0.872803 moved_to_echo=32958 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 90, 2, 3583, 1, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 1, 0, 0, 4094, 279, 3005, 752, 256, 0, 0, 0, 0, 0, 3698, 4012, 0, 0, 0, 0, 1268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 2, 8, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 2, 0, 0, 1, 4094, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3323, 0, 0, 721, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2335, 0, 3, 0, 2, 1, 0, 4093, 1085, 1, 0, 0, 0, 0, 0, 0, 2, 4094, 3, 4064, 0, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7905, 8408, 8978, 8198, 8195, 8138, 7543, 8171] avg=8192.0000 max=8978 max_rank=2 min=7543 min_rank=6 imbalance_ratio=1.095947 max_load_reduction=56558 max_load_reduction_pct=0.863007 moved_to_echo=32047 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3010, 756, 256, 0, 0, 0, 0, 0, 3700, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 0, 0, 520, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3564, 0, 0, 0, 0, 0, 0, 4096, 1086, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8332, 8118, 7811, 8191, 8191, 7953, 8747, 8193] avg=8192.0000 max=8747 max_rank=6 min=7811 min_rank=2 imbalance_ratio=1.067749 max_load_reduction=56789 max_load_reduction_pct=0.866531 moved_to_echo=33084 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 74, 1, 3690, 1, 1, 140, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 193, 3008, 755, 256, 0, 0, 0, 0, 0, 3699, 4022, 0, 0, 0, 0, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 1, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3335, 0, 0, 1753, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1620, 0, 1, 1, 1, 1, 0, 4095, 1086, 1, 0, 0, 0, 0, 0, 0, 1, 4095, 2, 4081, 0, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7999, 8309, 8654, 8193, 8192, 9184, 6821, 8184] avg=8192.0000 max=9184 max_rank=5 min=6821 min_rank=6 imbalance_ratio=1.121094 max_load_reduction=56352 max_load_reduction_pct=0.859863 moved_to_echo=32381 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4080, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3010, 756, 256, 0, 0, 0, 0, 0, 3698, 4096, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 1, 0, 854, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 0, 0, 0, 0, 0, 0, 4096, 1086, 1, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8316, 8117, 7837, 8191, 8191, 8265, 8420, 8199] avg=8192.0000 max=8420 max_rank=6 min=7837 min_rank=2 imbalance_ratio=1.027832 max_load_reduction=57116 max_load_reduction_pct=0.871521 moved_to_echo=33075 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3010, 756, 256, 0, 0, 0, 0, 0, 3700, 4096, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 0, 0, 463, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3536, 0, 0, 0, 0, 0, 0, 4096, 1086, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8332, 8118, 7894, 8191, 8191, 7898, 8719, 8193] avg=8192.0000 max=8719 max_rank=6 min=7894 min_rank=2 imbalance_ratio=1.064331 max_load_reduction=56817 max_load_reduction_pct=0.866959 moved_to_echo=33001 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 100, 1, 3663, 1, 1, 140, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 4095, 102, 3008, 755, 256, 0, 0, 0, 0, 0, 3699, 4021, 0, 0, 0, 0, 879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 1, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 3337, 0, 0, 755, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2765, 0, 1, 1, 0, 1, 0, 4095, 1085, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 4080, 0, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7998, 8220, 8599, 8193, 8192, 8189, 7963, 8182] avg=8192.0000 max=8599 max_rank=2 min=7963 min_rank=6 imbalance_ratio=1.049683 max_load_reduction=56937 max_load_reduction_pct=0.868790 moved_to_echo=32526 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4084, 0, 3011, 762, 261, 0, 0, 0, 0, 0, 3676, 4096, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3334, 13, 0, 166, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 0, 0, 0, 0, 0, 4096, 1085, 0, 0, 0, 0, 0, 0, 24, 0, 4096, 0, 4096, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8336, 8128, 7812, 8188, 8192, 7573, 9086, 8221] avg=8192.0000 max=9086 max_rank=6 min=7573 min_rank=5 imbalance_ratio=1.109131 max_load_reduction=56450 max_load_reduction_pct=0.861359 moved_to_echo=33072 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[29696, 32640, 0, 0, 0, 0, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 0, 384, 128, 31488, 128, 128, 1152, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 24064, 0, 128, 128, 128, 128, 0, 32768, 8704, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 26752, 128, 0, 6144, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 640, 24192, 6144, 2176, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 2, 136, 0, 1, 3, 0, 3957, 0, 2, 0, 0, 270, 0, 3483, 8, 0, 2, 40, 2, 2, 109, 3, 4077, 3, 0, 4085, 9, 0, 2, 4094, 3801, 0, 0, 4074, 0, 0, 3, 60, 6, 4094, 4086, 4070, 2, 1, 4088, 0, 0, 4084, 4091, 0, 0, 2162, 2, 0, 2, 4083, 4093, 1, 0, 2436, 0, 5] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 1, 97, 1, 0, 1, 0, 4034, 0, 1, 0, 0, 187, 0, 3489, 1, 0, 1, 15, 1, 1, 109, 1, 4092, 1, 6, 4092, 1, 0, 1, 4095, 3781, 0, 0, 4083, 0, 1, 9, 27, 4, 4095, 4090, 4083, 1, 0, 4093, 0, 0, 4087, 4094, 0, 0, 1854, 1, 0, 1, 4095, 4095, 0, 0, 2811, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 4096, 4081, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 14, 4096, 0, 0, 4096, 4096, 0, 0, 99, 0, 0, 0, 4096, 4096, 0, 0, 4036, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 70, 0, 4087, 0, 0, 4096, 0, 0, 0, 4096, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 11, 4096, 0, 0, 4096, 4096, 0, 0, 186, 0, 0, 0, 4096, 4096, 0, 0, 3847, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 48, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4075, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 124, 0, 0, 0, 4096, 4096, 0, 0, 3949, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 36, 0, 4074, 0, 1, 4096, 0, 0, 0, 4095, 4080, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 33, 4096, 0, 1, 4096, 4096, 3, 0, 109, 0, 0, 0, 4096, 4096, 0, 0, 3972, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 17, 0, 4059, 0, 12, 4096, 0, 0, 0, 4096, 4055, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4084, 4096, 0, 37, 4096, 0, 25, 4096, 4096, 27, 0, 188, 0, 0, 0, 4085, 4096, 0, 0, 3899, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 1, 82, 1, 0, 1, 0, 4048, 0, 1, 0, 0, 108, 0, 3816, 3, 0, 1, 14, 1, 1, 200, 1, 4087, 1, 7, 4092, 2, 0, 1, 4095, 3877, 0, 0, 4085, 0, 1, 49, 26, 3, 4095, 4089, 4091, 1, 0, 4093, 0, 0, 4088, 4093, 0, 0, 1537, 1, 0, 1, 4095, 4095, 0, 0, 2649, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4056, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 4086, 3448, 533, 40, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 0, 0, 0, 95, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3312, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 648, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8152, 8344, 8155, 8201, 8270, 8189, 7532, 8693] avg=8192.0000 max=8693 max_rank=7 min=7532 min_rank=6 imbalance_ratio=1.061157 max_load_reduction=56843 max_load_reduction_pct=0.867355 moved_to_echo=32684 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 2, 136, 0, 1, 3, 0, 4094, 4049, 0, 0, 0, 0, 0, 0, 3957, 0, 2, 0, 0, 270, 0, 3483, 155, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 40, 2, 2, 109, 3, 4084, 3445, 533, 39, 0, 0, 0, 0, 4077, 3, 0, 4085, 9, 0, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 4074, 0, 0, 3, 60, 95, 4, 0, 0, 0, 0, 0, 0, 6, 0, 4082, 4070, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 3308, 0, 0, 2162, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4083, 648, 1, 0, 2436, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8287, 7867, 8267, 8186, 8037, 8161, 9556, 7175] avg=8192.0000 max=9556 max_rank=6 min=7175 min_rank=7 imbalance_ratio=1.166504 max_load_reduction=55980 max_load_reduction_pct=0.854187 moved_to_echo=32929 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4057, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 4086, 3448, 533, 39, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 95, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3312, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 648, 0, 0, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8342, 8176, 8193, 8288, 8199, 7594, 8591] avg=8192.0000 max=8591 max_rank=7 min=7594 min_rank=6 imbalance_ratio=1.048706 max_load_reduction=56945 max_load_reduction_pct=0.868912 moved_to_echo=32672 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 97, 1, 0, 1, 0, 4095, 4054, 0, 0, 0, 0, 0, 0, 4034, 0, 1, 0, 0, 187, 0, 3489, 156, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 15, 1, 1, 109, 1, 4085, 3447, 533, 39, 0, 0, 0, 0, 4092, 1, 6, 4092, 1, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3781, 0, 0, 4083, 0, 1, 9, 27, 95, 4, 0, 0, 0, 0, 0, 0, 4, 0, 4086, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3310, 0, 0, 1854, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 648, 0, 0, 2811, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8250, 7867, 8233, 8203, 8000, 8174, 9252, 7557] avg=8192.0000 max=9252 max_rank=6 min=7557 min_rank=7 imbalance_ratio=1.129395 max_load_reduction=56284 max_load_reduction_pct=0.858826 moved_to_echo=32983 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4057, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4064, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 3448, 533, 39, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4096, 0, 0, 0, 0, 95, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3312, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 648, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8316, 8106, 8196, 8276, 8202, 7507, 8780] avg=8192.0000 max=8780 max_rank=7 min=7507 min_rank=6 imbalance_ratio=1.071777 max_load_reduction=56756 max_load_reduction_pct=0.866028 moved_to_echo=32765 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4057, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4077, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4085, 3448, 533, 39, 0, 0, 0, 0, 4074, 0, 1, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 0, 0, 0, 95, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3312, 3, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 648, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8329, 8141, 8181, 8275, 8220, 7521, 8716] avg=8192.0000 max=8716 max_rank=7 min=7521 min_rank=6 imbalance_ratio=1.063965 max_load_reduction=56820 max_load_reduction_pct=0.867004 moved_to_echo=32732 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4057, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4087, 3451, 539, 39, 0, 0, 0, 0, 4059, 0, 12, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 4096, 0, 0, 0, 0, 96, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 3303, 27, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 645, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8342, 8133, 8176, 8253, 8211, 7639, 8629] avg=8192.0000 max=8629 max_rank=7 min=7639 min_rank=6 imbalance_ratio=1.053345 max_load_reduction=56907 max_load_reduction_pct=0.868332 moved_to_echo=32732 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 82, 1, 0, 1, 0, 4095, 4054, 0, 0, 0, 0, 0, 0, 4048, 0, 1, 0, 0, 108, 0, 3816, 156, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 14, 1, 1, 200, 1, 4085, 3447, 533, 39, 0, 0, 0, 0, 4087, 1, 7, 4092, 2, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 4085, 0, 1, 49, 26, 95, 4, 0, 0, 0, 0, 0, 0, 3, 0, 4085, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3309, 0, 0, 1537, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 648, 0, 0, 2649, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8129, 8325, 8200, 8137, 8180, 8935, 7395] avg=8192.0000 max=8935 max_rank=6 min=7395 min_rank=7 imbalance_ratio=1.090698 max_load_reduction=56601 max_load_reduction_pct=0.863663 moved_to_echo=32647 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31872, 0, 0, 32768, 0, 128, 128, 128, 768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 5248, 128, 0, 27648, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 384, 128, 128, 128, 0, 32768, 32512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 26496, 128, 0, 6272, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 128, 640, 128, 32768, 27648, 4352, 384, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32640, 0, 128, 0, 0, 640, 0, 31232, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1581, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 131, 0, 2455, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 998, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 31, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 0, 0, 4095, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 93, 0, 3125, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 729, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 165, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3210, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 182, 0, 3905, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 824, 4093, 4071, 0, 2, 2, 0, 4094, 3, 0, 3941, 2, 4094, 0, 72, 0, 0, 4093, 285, 4094, 0, 2, 0, 22, 0, 2, 196, 2, 0, 0, 2, 5, 5, 0, 2, 2, 0, 3295, 0, 2, 1, 0, 3936, 4094, 2, 4092, 4091, 0, 2, 4094, 1, 0, 2, 1543, 0, 2362, 4016, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 705, 4095, 4083, 0, 1, 1, 0, 4095, 2, 0, 3992, 1, 4095, 0, 28, 0, 0, 4095, 180, 4095, 0, 1, 0, 10, 0, 1, 169, 1, 0, 0, 1, 2, 2, 0, 1, 1, 0, 2892, 1, 1, 0, 0, 4038, 4095, 0, 4094, 4094, 0, 1, 4095, 0, 0, 1, 1584, 0, 2864, 4024, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 170, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 4082, 4096, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 105, 0, 4013, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 1086, 4095, 4082, 0, 1, 2, 0, 4095, 2, 0, 3995, 1, 4095, 0, 15, 0, 0, 4094, 165, 4095, 0, 1, 0, 13, 0, 1, 246, 1, 0, 0, 1, 2, 2, 0, 1, 1, 0, 3105, 1, 1, 1, 0, 3994, 4095, 0, 4094, 4094, 0, 1, 4095, 0, 0, 1, 1424, 0, 2374, 4070, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1999, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4084, 0, 0, 0, 0, 103, 0, 6, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 2949, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4084, 0, 0, 0, 101, 0, 3040, 4096, 7, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3093, 0, 0, 0, 1581, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 4093, 3365, 596, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 0, 0, 4096, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 2455, 4096, 0, 0, 0, 407, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8770, 8235, 8102, 8113, 8188, 8799, 8195, 7134] avg=8192.0000 max=8799 max_rank=5 min=7134 min_rank=7 imbalance_ratio=1.074097 max_load_reduction=56737 max_load_reduction_pct=0.865738 moved_to_echo=32316 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3093, 0, 0, 0, 998, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 30, 0, 4093, 3365, 596, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 0, 0, 4095, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 3125, 4096, 0, 0, 0, 407, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8235, 8103, 8119, 8188, 8743, 8195, 7766] avg=8192.0000 max=8743 max_rank=5 min=7766 min_rank=7 imbalance_ratio=1.067261 max_load_reduction=56793 max_load_reduction_pct=0.866592 moved_to_echo=32892 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3089, 0, 0, 0, 824, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 4094, 3, 0, 3941, 2, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 4006, 285, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 22, 0, 2, 196, 2, 4089, 3364, 595, 4, 0, 0, 0, 0, 0, 0, 2, 5, 5, 0, 2, 2, 4094, 4089, 0, 0, 0, 0, 0, 0, 0, 3295, 0, 2, 1, 0, 3936, 730, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4091, 0, 2, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1543, 0, 2362, 4016, 0, 2, 0, 407, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7984, 8086, 8457, 8276, 8199, 7964, 8193, 8377] avg=8192.0000 max=8457 max_rank=2 min=7964 min_rank=5 imbalance_ratio=1.032349 max_load_reduction=57079 max_load_reduction_pct=0.870956 moved_to_echo=32733 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3092, 0, 0, 0, 705, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 2, 0, 3992, 1, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 4008, 180, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 0, 1, 169, 1, 4091, 3364, 595, 4, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 1, 1, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 2892, 1, 1, 0, 0, 4038, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1584, 0, 2864, 4024, 0, 1, 0, 407, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7880, 8134, 8311, 8236, 8193, 7663, 8193, 8926] avg=8192.0000 max=8926 max_rank=7 min=7663 min_rank=5 imbalance_ratio=1.089600 max_load_reduction=56610 max_load_reduction_pct=0.863800 moved_to_echo=32975 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3093, 0, 0, 0, 170, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4093, 3365, 596, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 0, 4082, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 4013, 4096, 0, 0, 0, 407, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7359, 8235, 8097, 8071, 8188, 8726, 8194, 8666] avg=8192.0000 max=8726 max_rank=5 min=7359 min_rank=0 imbalance_ratio=1.065186 max_load_reduction=56810 max_load_reduction_pct=0.866852 moved_to_echo=33774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3093, 0, 0, 0, 729, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 3, 0, 4093, 3365, 596, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 3210, 0, 0, 0, 0, 4096, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 3905, 4096, 1, 0, 0, 407, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7918, 8235, 8101, 8226, 8190, 8037, 8193, 8636] avg=8192.0000 max=8636 max_rank=7 min=7918 min_rank=0 imbalance_ratio=1.054199 max_load_reduction=56900 max_load_reduction_pct=0.868225 moved_to_echo=33056 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3091, 0, 0, 0, 1086, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4095, 2, 0, 3995, 1, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4007, 165, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 1, 246, 1, 4091, 3364, 595, 5, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 1, 1, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 3105, 1, 1, 1, 0, 3994, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1424, 0, 2374, 4070, 0, 1, 0, 407, 45, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8138, 8282, 8317, 8192, 7833, 8193, 8322] avg=8192.0000 max=8322 max_rank=7 min=7833 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=57214 max_load_reduction_pct=0.873016 moved_to_echo=32540 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3084, 0, 0, 0, 1999, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 6, 0, 4096, 3370, 598, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 2949, 0, 0, 0, 0, 4096, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 3040, 4096, 7, 0, 0, 414, 51, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9179, 8238, 8083, 8178, 8198, 7771, 8180, 7709] avg=8192.0000 max=9179 max_rank=0 min=7709 min_rank=7 imbalance_ratio=1.120483 max_load_reduction=56357 max_load_reduction_pct=0.859940 moved_to_echo=31858 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[0, 0, 128, 0, 0, 32128, 640, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 32512, 128, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 27264, 128, 128, 128, 0, 32512, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[24832, 0, 0, 0, 8192, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 384, 128, 768, 128, 32768, 27008, 4864, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 5248, 0, 24192, 32640, 128, 128, 0, 3328, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4083, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2130, 0, 0, 0, 10, 4096, 23, 906, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 15, 0, 0, 4096, 4091, 4070, 0, 4030, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3814, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4087, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2098, 0, 0, 0, 28, 4096, 8, 1254, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 75, 0, 0, 4096, 4095, 4069, 0, 4084, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3965, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4080, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1838, 0, 0, 0, 46, 4096, 3, 1609, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 11, 0, 0, 4096, 4096, 4075, 0, 4064, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3585, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4046, 2130, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 4093, 19, 534, 0, 1, 0, 28, 4095, 706, 828, 0, 0, 4095, 1, 1, 0, 0, 0, 50, 4095, 0, 4095, 1500, 1, 9, 4094, 4095, 4095, 0, 3836, 0, 1, 0, 4091, 1, 4095, 0, 1, 1, 0, 4094, 0, 1, 0, 1, 3, 1, 2697, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[3988, 1924, 35, 0, 0, 1, 0, 0, 3, 2, 2, 2, 2, 0, 4088, 28, 316, 0, 2, 1, 14, 4094, 1165, 719, 0, 0, 4094, 3, 1, 0, 0, 0, 100, 4093, 0, 4094, 1402, 2, 10, 4092, 4094, 4094, 0, 3762, 0, 3, 0, 4075, 2, 4094, 0, 2, 2, 0, 4091, 0, 2, 2, 3, 0, 0, 2937, 4094, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4074, 1378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2000, 0, 0, 0, 25, 4095, 19, 1669, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 218, 0, 0, 4096, 4095, 4019, 0, 4090, 0, 0, 9, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2980, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4051, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2461, 0, 0, 0, 74, 4086, 8, 1680, 0, 3, 4096, 0, 0, 0, 0, 24, 0, 4095, 0, 4096, 19, 0, 0, 4096, 4092, 4037, 0, 4088, 0, 0, 11, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3282, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4063, 2533, 3, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 4092, 21, 379, 0, 1, 0, 0, 4095, 738, 562, 0, 0, 4095, 1, 1, 0, 0, 0, 39, 4095, 0, 4095, 594, 1, 7, 4094, 4095, 4095, 0, 3821, 0, 1, 0, 4089, 1, 4095, 0, 1, 1, 0, 4094, 0, 1, 0, 1, 2, 1, 3627, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4087, 813, 0, 0, 0, 0, 0, 0, 2606, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4085, 3, 0, 0, 0, 0, 0, 0, 2098, 0, 0, 0, 28, 4096, 8, 1254, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 0, 0, 75, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4084, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 4096, 0, 504, 228, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7516, 8184, 8597, 8189, 7763, 8302, 8192, 8793] avg=8192.0000 max=8793 max_rank=7 min=7516 min_rank=0 imbalance_ratio=1.073364 max_load_reduction=56743 max_load_reduction_pct=0.865829 moved_to_echo=33050 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4083, 1404, 0, 0, 0, 0, 0, 0, 2606, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 4, 0, 0, 0, 0, 0, 0, 2130, 0, 0, 0, 10, 4096, 23, 906, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4030, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3814, 4096, 0, 504, 228, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8104, 8180, 8278, 8188, 7703, 8249, 8192, 8642] avg=8192.0000 max=8642 max_rank=7 min=7703 min_rank=4 imbalance_ratio=1.054932 max_load_reduction=56894 max_load_reduction_pct=0.868134 moved_to_echo=32786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3988, 1924, 35, 0, 0, 1, 0, 0, 2592, 11, 0, 0, 0, 0, 0, 0, 3, 2, 2, 2, 2, 0, 4088, 28, 4083, 3, 0, 0, 0, 0, 0, 0, 316, 0, 2, 1, 14, 4094, 1165, 719, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 1, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 100, 3589, 0, 0, 1402, 2, 10, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3762, 0, 3, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 2, 2, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3, 0, 0, 2937, 4094, 2, 504, 227, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8551, 8213, 7418, 8189, 9195, 8008, 8191, 7771] avg=8192.0000 max=9195 max_rank=4 min=7418 min_rank=2 imbalance_ratio=1.122437 max_load_reduction=56341 max_load_reduction_pct=0.859695 moved_to_echo=33165 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4080, 1169, 0, 0, 0, 0, 0, 0, 2606, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4085, 3, 0, 0, 0, 0, 0, 0, 1838, 0, 0, 0, 46, 4096, 3, 1609, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4064, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3585, 4096, 0, 504, 228, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7866, 8184, 8705, 8189, 7699, 8288, 8192, 8413] avg=8192.0000 max=8705 max_rank=2 min=7699 min_rank=4 imbalance_ratio=1.062622 max_load_reduction=56831 max_load_reduction_pct=0.867172 moved_to_echo=32592 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4046, 2130, 0, 0, 0, 1, 0, 0, 2602, 11, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4093, 19, 4084, 4, 0, 0, 0, 0, 0, 0, 534, 0, 1, 0, 28, 4095, 706, 828, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 50, 3591, 0, 0, 1500, 1, 9, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3836, 0, 1, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3, 1, 2697, 4095, 1, 504, 228, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8790, 8205, 7304, 8188, 9245, 8081, 8192, 7531] avg=8192.0000 max=9245 max_rank=4 min=7304 min_rank=2 imbalance_ratio=1.128540 max_load_reduction=56291 max_load_reduction_pct=0.858932 moved_to_echo=33049 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4074, 1378, 0, 0, 0, 0, 0, 0, 2606, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 3, 0, 0, 0, 0, 0, 0, 2000, 0, 0, 0, 25, 4095, 19, 1669, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 0, 0, 218, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 4090, 0, 0, 9, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2980, 4096, 0, 504, 228, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8069, 8183, 8921, 8190, 7906, 8267, 8192, 7808] avg=8192.0000 max=8921 max_rank=2 min=7808 min_rank=7 imbalance_ratio=1.088989 max_load_reduction=56615 max_load_reduction_pct=0.863876 moved_to_echo=32173 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4051, 757, 0, 0, 0, 0, 0, 0, 2609, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 4, 0, 0, 0, 0, 0, 0, 2461, 0, 0, 0, 74, 4086, 8, 1680, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 24, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3584, 0, 0, 19, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 4088, 0, 0, 11, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3282, 4096, 0, 510, 231, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7428, 8181, 9427, 8215, 7699, 8274, 8193, 8119] avg=8192.0000 max=9427 max_rank=2 min=7428 min_rank=0 imbalance_ratio=1.150757 max_load_reduction=56109 max_load_reduction_pct=0.856155 moved_to_echo=32285 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4063, 2533, 3, 0, 0, 1, 0, 0, 2601, 11, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4092, 21, 4084, 4, 0, 0, 0, 0, 0, 0, 379, 0, 1, 0, 0, 4095, 738, 562, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 39, 3591, 0, 0, 594, 1, 7, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3821, 0, 1, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2, 1, 3627, 4095, 1, 504, 227, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9212, 8206, 6886, 8188, 8326, 8067, 8192, 8459] avg=8192.0000 max=9212 max_rank=0 min=6886 min_rank=2 imbalance_ratio=1.124512 max_load_reduction=56324 max_load_reduction_pct=0.859436 moved_to_echo=33044 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11776, 0, 128, 128, 256, 32768, 2688, 9344, 8960, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32640, 0, 31872, 0, 128, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[256, 28800, 0, 0, 3840, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32512, 12160, 128, 0, 0, 128, 0, 0, 20864, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 27008, 32768, 128, 4096, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4074, 26, 0, 3937, 0, 0, 4096, 0, 4096, 0, 154, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4085, 166, 2, 2385, 2, 39, 3965, 0, 4094, 2, 1739, 3532, 3, 0, 0, 0, 4094, 0, 13, 0, 2, 4093, 0, 5, 0, 1, 4, 4080, 0, 4077, 0, 4089, 0, 2, 1, 4094, 0, 19, 0, 0, 2, 0, 2, 0, 0, 500, 2, 2, 6, 3, 4067, 0, 0, 3, 4094, 0, 0, 4079, 0, 4092, 4090, 0, 4, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4087, 65, 0, 3651, 0, 0, 4096, 0, 4096, 0, 389, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[3896, 25, 0, 3403, 0, 0, 4096, 0, 4096, 0, 863, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4073, 72, 1, 2942, 1, 29, 4030, 0, 4095, 1, 1201, 3751, 2, 0, 0, 0, 4095, 0, 5, 0, 1, 4094, 0, 2, 0, 0, 2, 4083, 0, 4082, 0, 4091, 0, 1, 1, 4095, 0, 7, 0, 0, 1, 0, 1, 0, 0, 309, 1, 1, 3, 1, 4087, 0, 0, 1, 4095, 0, 0, 4090, 0, 4094, 4092, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4090, 89, 0, 3798, 0, 0, 4096, 0, 4095, 0, 206, 4096, 0, 0, 1, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4086, 85, 0, 3659, 0, 0, 4096, 0, 4085, 0, 335, 4096, 0, 0, 2, 0, 4093, 0, 0, 20, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4093, 68, 1, 2615, 1, 22, 4031, 0, 4095, 1, 1404, 3829, 2, 0, 0, 0, 4095, 0, 6, 0, 1, 4094, 0, 1, 0, 0, 3, 4090, 0, 4090, 0, 4088, 0, 1, 1, 4095, 0, 3, 1, 0, 1, 0, 1, 0, 0, 340, 1, 1, 3, 1, 4084, 0, 0, 1, 4095, 0, 0, 4089, 0, 4091, 4094, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4087, 65, 0, 3651, 0, 0, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 0, 389, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3343, 639, 133, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8556, 7942, 8187, 8201, 8213, 8046, 8195, 8196] avg=8192.0000 max=8556 max_rank=0 min=7942 min_rank=1 imbalance_ratio=1.044434 max_load_reduction=56980 max_load_reduction_pct=0.869446 moved_to_echo=32650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4074, 26, 0, 3937, 0, 0, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 0, 154, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3343, 639, 133, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4092, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8790, 7707, 8187, 8201, 8213, 8047, 8195, 8196] avg=8192.0000 max=8790 max_rank=0 min=7707 min_rank=1 imbalance_ratio=1.072998 max_load_reduction=56746 max_load_reduction_pct=0.865875 moved_to_echo=32651 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4085, 166, 2, 2385, 2, 39, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 2, 1739, 3532, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 13, 0, 2, 4089, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 4080, 0, 4077, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4094, 0, 19, 0, 0, 3237, 639, 132, 1, 0, 0, 0, 0, 2, 0, 2, 0, 0, 500, 2, 2, 4086, 3948, 0, 0, 0, 0, 0, 0, 6, 3, 4067, 0, 0, 3, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4092, 4, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7407, 8731, 8203, 8171, 8125, 8542, 8176, 8181] avg=8192.0000 max=8731 max_rank=1 min=7407 min_rank=0 imbalance_ratio=1.065796 max_load_reduction=56805 max_load_reduction_pct=0.866776 moved_to_echo=33024 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3896, 25, 0, 3403, 0, 0, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 0, 863, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3343, 639, 133, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4092, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8077, 8416, 8187, 8200, 8213, 8048, 8199, 8196] avg=8192.0000 max=8416 max_rank=1 min=8048 min_rank=5 imbalance_ratio=1.027344 max_load_reduction=57120 max_load_reduction_pct=0.871582 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4073, 72, 1, 2942, 1, 29, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3456, 1, 1201, 3751, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 5, 0, 1, 4090, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4083, 0, 4082, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 7, 0, 0, 3290, 639, 132, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 309, 1, 1, 4088, 3949, 0, 0, 0, 0, 0, 0, 3, 1, 4087, 0, 0, 1, 4095, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4094, 4, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7858, 8411, 8193, 8177, 8166, 8350, 8190, 8191] avg=8192.0000 max=8411 max_rank=1 min=7858 min_rank=0 imbalance_ratio=1.026733 max_load_reduction=57125 max_load_reduction_pct=0.871658 moved_to_echo=32897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4090, 89, 0, 3798, 0, 0, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3456, 0, 206, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3343, 639, 133, 2, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4092, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8730, 7759, 8189, 8201, 8214, 8053, 8195, 8195] avg=8192.0000 max=8730 max_rank=0 min=7759 min_rank=1 imbalance_ratio=1.065674 max_load_reduction=56806 max_load_reduction_pct=0.866791 moved_to_echo=32657 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4086, 85, 0, 3659, 0, 0, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3445, 0, 335, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 20, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 3349, 640, 136, 5, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 4093, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8577, 7878, 8197, 8193, 8229, 8069, 8199, 8194] avg=8192.0000 max=8577 max_rank=0 min=7878 min_rank=1 imbalance_ratio=1.046997 max_load_reduction=56959 max_load_reduction_pct=0.869125 moved_to_echo=32691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4093, 68, 1, 2615, 1, 22, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3456, 1, 1404, 3829, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 0, 1, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 4090, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 3, 1, 0, 3290, 639, 132, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 340, 1, 1, 4090, 3947, 0, 0, 0, 0, 0, 0, 3, 1, 4084, 0, 0, 1, 4095, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4091, 4, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7541, 8692, 8193, 8192, 8163, 8381, 8187, 8187] avg=8192.0000 max=8692 max_rank=1 min=7541 min_rank=0 imbalance_ratio=1.061035 max_load_reduction=56844 max_load_reduction_pct=0.867371 moved_to_echo=32918 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32512, 640, 128, 26496, 128, 128, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 128, 32768, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[27648, 128, 6400, 31616, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 1152, 128, 128, 32768, 31744, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 128, 0, 26624, 5120, 1152, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2674, 74, 1095, 467, 998, 0, 2035, 337, 1218, 3031, 473, 530, 1558, 1169, 0, 161, 669, 1282, 732, 1772, 0, 846, 2060, 1053, 67, 27, 3107, 1020, 2809, 410, 2374, 715, 10, 1812, 2133, 172, 470, 438, 2043, 1457, 603, 1, 1482, 1541, 0, 1334, 1175, 971, 2661, 9, 1193, 4, 163, 563, 8, 766, 0, 465, 1425, 836, 2272, 1294, 698, 2774] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[3182, 73, 575, 285, 1279, 0, 1644, 147, 1114, 2802, 783, 780, 1289, 1345, 0, 279, 399, 1178, 494, 894, 0, 771, 1982, 1205, 60, 39, 2629, 1832, 2796, 734, 2337, 607, 1, 1692, 2418, 532, 730, 511, 1476, 951, 488, 0, 1609, 1954, 0, 1511, 1233, 1073, 2523, 36, 1489, 1, 194, 862, 1, 665, 0, 511, 1294, 1197, 2355, 820, 1111, 2764] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[3041, 42, 323, 398, 927, 0, 1896, 127, 1437, 3236, 765, 1221, 1348, 1639, 10, 257, 464, 926, 866, 732, 1, 791, 2452, 1057, 73, 38, 2704, 1076, 2224, 700, 2234, 819, 0, 1371, 2840, 323, 666, 799, 1664, 723, 698, 0, 1615, 1512, 1, 1686, 1070, 650, 2952, 54, 1465, 5, 175, 804, 0, 558, 3, 527, 1312, 1273, 2569, 751, 1159, 2487] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2299, 46, 536, 319, 1177, 0, 2558, 55, 650, 3439, 926, 493, 980, 2087, 0, 80, 369, 1762, 563, 934, 0, 1031, 1310, 927, 61, 80, 2830, 951, 2070, 586, 2458, 846, 3, 1282, 2466, 438, 460, 618, 1602, 1661, 400, 0, 2007, 1779, 0, 1787, 1359, 781, 2798, 41, 962, 1, 172, 1242, 0, 733, 0, 348, 1006, 1717, 2248, 1750, 884, 2568] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2719, 112, 601, 259, 1044, 0, 1543, 510, 1582, 3018, 873, 1139, 1517, 1818, 0, 371, 331, 1262, 800, 1216, 0, 728, 1965, 1288, 73, 131, 2538, 1268, 2838, 1222, 2324, 391, 2, 1560, 2562, 564, 755, 430, 1607, 1157, 455, 0, 2032, 1934, 0, 1279, 864, 1063, 2378, 34, 1650, 23, 152, 856, 19, 776, 0, 409, 1073, 663, 2064, 572, 841, 2281] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2260, 67, 867, 260, 1550, 0, 1793, 259, 914, 2927, 612, 705, 1507, 1242, 0, 178, 659, 1343, 1305, 1767, 0, 711, 1181, 1171, 71, 66, 2676, 1207, 2529, 422, 2404, 715, 2, 1868, 1748, 344, 810, 678, 2244, 1467, 738, 0, 1734, 1551, 0, 1617, 1210, 1102, 2939, 19, 1264, 0, 57, 486, 2, 634, 0, 328, 1067, 1116, 2216, 1152, 1182, 2593] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2788, 25, 1110, 287, 1071, 0, 1396, 128, 889, 2443, 549, 969, 1752, 730, 0, 113, 280, 1773, 874, 890, 3, 692, 2086, 1848, 79, 27, 2479, 1581, 2569, 399, 2565, 987, 44, 2307, 2305, 426, 564, 443, 1693, 917, 518, 0, 1703, 1830, 8, 1427, 1112, 725, 2530, 42, 1027, 1, 110, 736, 11, 648, 0, 599, 1650, 1380, 2420, 1059, 1271, 2648] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2785, 71, 661, 375, 1398, 1, 2039, 112, 1199, 3001, 676, 1077, 1172, 1437, 0, 193, 443, 1054, 437, 857, 2, 882, 1505, 1105, 110, 26, 2738, 1069, 3073, 626, 2087, 451, 1, 1397, 2275, 328, 520, 1159, 2052, 1522, 577, 2, 1884, 2132, 1, 1542, 840, 1350, 2812, 34, 1751, 1, 61, 896, 0, 527, 1, 767, 1185, 618, 2304, 736, 1251, 2348] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2674, 74, 1095, 467, 998, 0, 2035, 337, 1197, 0, 0, 0, 0, 0, 0, 0, 1218, 2503, 473, 530, 1558, 1169, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 669, 1282, 732, 1772, 0, 846, 2060, 1053, 527, 189, 0, 0, 0, 0, 0, 0, 67, 27, 599, 1020, 2809, 410, 2374, 715, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1812, 1681, 172, 470, 438, 2043, 1457, 0, 0, 0, 0, 0, 0, 0, 0, 603, 1, 1482, 1541, 0, 1334, 1175, 971, 452, 1, 0, 0, 0, 0, 0, 0, 2661, 9, 1193, 4, 163, 563, 8, 766, 2508, 331, 0, 0, 0, 0, 0, 0, 0, 465, 1425, 836, 2272, 1294, 698, 1057, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8877, 7612, 9130, 8021, 8083, 7560, 8206, 8047] avg=8192.0000 max=9130 max_rank=2 min=7560 min_rank=5 imbalance_ratio=1.114502 max_load_reduction=56406 max_load_reduction_pct=0.860687 moved_to_echo=31896 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3041, 42, 323, 398, 927, 0, 1896, 127, 1073, 0, 0, 0, 0, 0, 0, 0, 1437, 2672, 765, 1221, 1348, 1639, 10, 257, 0, 0, 0, 0, 0, 0, 0, 0, 464, 926, 866, 732, 1, 791, 2452, 1057, 563, 169, 0, 0, 0, 0, 0, 0, 73, 38, 522, 1076, 2224, 700, 2234, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1371, 2238, 323, 666, 799, 1664, 723, 0, 0, 0, 0, 0, 0, 0, 0, 698, 0, 1615, 1512, 1, 1686, 1070, 650, 602, 1, 0, 0, 0, 0, 0, 0, 2952, 54, 1465, 5, 175, 804, 0, 558, 2182, 296, 0, 0, 0, 0, 0, 0, 3, 527, 1312, 1273, 2569, 751, 1159, 949, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7827, 9349, 8021, 7686, 7784, 7835, 8491, 8543] avg=8192.0000 max=9349 max_rank=1 min=7686 min_rank=3 imbalance_ratio=1.141235 max_load_reduction=56187 max_load_reduction_pct=0.857346 moved_to_echo=32653 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3182, 73, 575, 285, 1279, 0, 1644, 147, 1193, 0, 0, 0, 0, 0, 0, 0, 1114, 2314, 783, 780, 1289, 1345, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 399, 1178, 494, 894, 0, 771, 1982, 1205, 487, 188, 0, 0, 0, 0, 0, 0, 60, 39, 507, 1832, 2796, 734, 2337, 607, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1692, 1906, 532, 730, 511, 1476, 951, 0, 0, 0, 0, 0, 0, 0, 0, 488, 0, 1609, 1954, 0, 1511, 1233, 1073, 512, 1, 0, 0, 0, 0, 0, 0, 2523, 36, 1489, 1, 194, 862, 1, 665, 2122, 329, 0, 0, 0, 0, 0, 0, 0, 511, 1294, 1197, 2355, 820, 1111, 1054, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8378, 7904, 7598, 8912, 7799, 8381, 8222, 8342] avg=8192.0000 max=8912 max_rank=3 min=7598 min_rank=2 imbalance_ratio=1.087891 max_load_reduction=56624 max_load_reduction_pct=0.864014 moved_to_echo=32744 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2260, 67, 867, 260, 1550, 0, 1793, 259, 1119, 0, 0, 0, 0, 0, 0, 0, 914, 2417, 612, 705, 1507, 1242, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 659, 1343, 1305, 1767, 0, 711, 1181, 1171, 509, 177, 0, 0, 0, 0, 0, 0, 71, 66, 516, 1207, 2529, 422, 2404, 715, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1868, 1378, 344, 810, 678, 2244, 1467, 0, 0, 0, 0, 0, 0, 0, 0, 738, 0, 1734, 1551, 0, 1617, 1210, 1102, 370, 1, 0, 0, 0, 0, 0, 0, 2939, 19, 1264, 0, 57, 486, 2, 634, 2160, 309, 0, 0, 0, 0, 0, 0, 0, 328, 1067, 1116, 2216, 1152, 1182, 988, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8175, 7575, 8823, 7930, 8791, 8323, 7870, 8049] avg=8192.0000 max=8823 max_rank=2 min=7575 min_rank=1 imbalance_ratio=1.077026 max_load_reduction=56713 max_load_reduction_pct=0.865372 moved_to_echo=33033 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2719, 112, 601, 259, 1044, 0, 1543, 510, 984, 0, 0, 0, 0, 0, 0, 0, 1582, 2492, 873, 1139, 1517, 1818, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 331, 1262, 800, 1216, 0, 728, 1965, 1288, 525, 155, 0, 0, 0, 0, 0, 0, 73, 131, 490, 1268, 2838, 1222, 2324, 391, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1560, 2019, 564, 755, 430, 1607, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 2032, 1934, 0, 1279, 864, 1063, 543, 1, 0, 0, 0, 0, 0, 0, 2378, 34, 1650, 23, 152, 856, 19, 776, 2048, 272, 0, 0, 0, 0, 0, 0, 0, 409, 1073, 663, 2064, 572, 841, 870, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7772, 9792, 8270, 8737, 8094, 8171, 8208, 6492] avg=8192.0000 max=9792 max_rank=1 min=6492 min_rank=7 imbalance_ratio=1.195312 max_load_reduction=55744 max_load_reduction_pct=0.850586 moved_to_echo=30965 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2299, 46, 536, 319, 1177, 0, 2558, 55, 1108, 0, 0, 0, 0, 0, 0, 0, 650, 2840, 926, 493, 980, 2087, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 369, 1762, 563, 934, 0, 1031, 1310, 927, 598, 175, 0, 0, 0, 0, 0, 0, 61, 80, 546, 951, 2070, 586, 2458, 846, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1282, 1943, 438, 460, 618, 1602, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 2007, 1779, 0, 1787, 1359, 781, 523, 1, 0, 0, 0, 0, 0, 0, 2798, 41, 962, 1, 172, 1242, 0, 733, 2284, 306, 0, 0, 0, 0, 0, 0, 0, 348, 1006, 1717, 2248, 1750, 884, 979, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8056, 7669, 7598, 8007, 8637, 8539, 8932] avg=8192.0000 max=8932 max_rank=7 min=7598 min_rank=3 imbalance_ratio=1.090332 max_load_reduction=56604 max_load_reduction_pct=0.863708 moved_to_echo=34115 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2788, 25, 1110, 287, 1071, 0, 1396, 128, 1149, 0, 0, 0, 0, 0, 0, 0, 889, 2014, 549, 969, 1752, 730, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 280, 1773, 874, 890, 3, 692, 2086, 1848, 427, 185, 0, 0, 0, 0, 0, 0, 79, 27, 475, 1581, 2569, 399, 2565, 987, 0, 0, 0, 0, 0, 0, 0, 0, 44, 2307, 1812, 426, 564, 443, 1693, 917, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 1703, 1830, 8, 1427, 1112, 725, 493, 2, 0, 0, 0, 0, 0, 0, 2530, 42, 1027, 1, 110, 736, 11, 648, 2004, 319, 0, 0, 0, 0, 0, 0, 0, 599, 1650, 1380, 2420, 1059, 1271, 995, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7954, 7016, 9058, 8682, 8206, 7818, 7428, 9374] avg=8192.0000 max=9374 max_rank=7 min=7016 min_rank=1 imbalance_ratio=1.144287 max_load_reduction=56162 max_load_reduction_pct=0.856964 moved_to_echo=32826 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2785, 71, 661, 375, 1398, 1, 2039, 112, 1013, 0, 0, 0, 0, 0, 0, 0, 1199, 2478, 676, 1077, 1172, 1437, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 443, 1054, 437, 857, 2, 882, 1505, 1105, 522, 160, 0, 0, 0, 0, 0, 0, 110, 26, 528, 1069, 3073, 626, 2087, 451, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1397, 1793, 328, 520, 1159, 2052, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 577, 2, 1884, 2132, 1, 1542, 840, 1350, 482, 1, 0, 0, 0, 0, 0, 0, 2812, 34, 1751, 1, 61, 896, 0, 527, 2210, 280, 0, 0, 0, 0, 0, 0, 1, 767, 1185, 618, 2304, 736, 1251, 895, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8455, 8232, 6967, 7970, 8772, 8811, 8572, 7757] avg=8192.0000 max=8811 max_rank=5 min=6967 min_rank=2 imbalance_ratio=1.075562 max_load_reduction=56725 max_load_reduction_pct=0.865555 moved_to_echo=33912 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[4480, 128, 14080, 14336, 128, 12288, 8960, 7808, 4096, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 13312, 14848, 3200, 4992, 5120, 14464, 9856, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 3968, 10112, 8832, 18560, 8192, 8448, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 512, 4224, 10112, 20992, 5120, 18816, 5632, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[9088, 19840, 5760, 7040, 11136, 11520, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[21760, 512, 5888, 2688, 9472, 128, 14976, 1792, 8960, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3712, 10624, 6144, 9088, 128, 6528, 14592, 9728, 4224, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[21632, 384, 10880, 128, 1152, 6528, 128, 5376, 17536, 2560, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 2, 3104, 0, 3606, 4041, 0, 1442, 2524, 0, 0, 367, 4069, 0, 0, 0, 0, 0, 4091, 2909, 1443, 117, 0, 1323, 0, 3157, 10, 1, 2179, 1, 0, 0, 392, 0, 0, 3602, 4030, 2609, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4039, 4096, 0, 0, 0, 26, 4096, 54, 0, 0, 0, 0, 13] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 2438, 0, 3723, 4036, 0, 1547, 2234, 0, 0, 1624, 3946, 0, 0, 0, 0, 0, 4074, 3476, 971, 128, 0, 999, 0, 2773, 0, 1, 1263, 0, 0, 0, 237, 0, 0, 3727, 4082, 3618, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4086, 4096, 0, 1, 0, 26, 4096, 135, 0, 0, 0, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 3565, 0, 3694, 4044, 0, 2476, 2647, 0, 0, 304, 3988, 0, 0, 0, 0, 0, 4060, 3509, 169, 73, 0, 319, 0, 3962, 2, 15, 2052, 0, 1, 0, 23, 0, 0, 3025, 4052, 3083, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3959, 4096, 0, 0, 0, 20, 4096, 52, 0, 0, 0, 0, 57] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1530, 0, 3709, 4050, 0, 2854, 3604, 0, 0, 441, 4051, 0, 0, 0, 0, 0, 4095, 2536, 784, 1021, 0, 1182, 0, 3418, 0, 0, 1128, 0, 0, 0, 50, 1, 0, 2983, 3926, 3593, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3908, 4096, 0, 1, 0, 118, 4096, 130, 0, 0, 0, 0, 39] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 1994, 0, 3380, 3945, 0, 2443, 2843, 0, 0, 1013, 4080, 0, 0, 0, 0, 0, 4084, 3180, 1232, 245, 0, 469, 2, 3895, 10, 7, 1026, 1, 3, 0, 68, 0, 0, 3532, 4077, 3283, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 3956, 4094, 0, 0, 0, 0, 4096, 365, 0, 0, 0, 0, 19] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 1, 3859, 0, 3977, 4059, 0, 3049, 2351, 0, 0, 665, 3958, 0, 0, 0, 0, 0, 4083, 2814, 638, 32, 0, 487, 0, 3691, 0, 0, 1673, 0, 1, 0, 11, 0, 0, 2918, 4031, 2585, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 3969, 4094, 0, 0, 0, 1, 4095, 263, 0, 0, 0, 0, 38] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 2, 3094, 0, 3467, 3944, 0, 1137, 2640, 0, 0, 1243, 4012, 0, 0, 0, 0, 0, 4085, 2805, 595, 863, 0, 617, 0, 3651, 0, 0, 1993, 1, 1, 0, 56, 0, 0, 3554, 3882, 3319, 0, 0, 0, 0, 2, 4096, 0, 1, 0, 3981, 4095, 0, 6, 0, 15, 4095, 156, 2, 0, 0, 0, 32] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 0, 1766, 0, 3643, 4044, 0, 1134, 3130, 1, 1, 145, 4057, 1, 4, 2, 1, 0, 4028, 3981, 1228, 60, 0, 1314, 0, 3470, 2, 8, 2392, 0, 2, 1, 104, 0, 0, 3227, 3775, 3428, 0, 0, 0, 1, 0, 4093, 8, 0, 0, 4038, 4090, 1, 0, 0, 126, 4082, 16, 1, 1, 0, 0, 34] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 2438, 0, 854, 589, 0, 0, 0, 0, 0, 0, 2765, 0, 0, 1547, 2234, 0, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 0, 0, 0, 1020, 3476, 0, 0, 0, 0, 0, 0, 0, 0, 971, 128, 0, 999, 0, 2773, 0, 1, 2465, 177, 0, 0, 0, 0, 0, 0, 1263, 0, 0, 0, 237, 0, 0, 3727, 2898, 141, 0, 0, 0, 0, 0, 0, 4082, 3618, 0, 0, 0, 0, 0, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4086, 4096, 0, 1, 0, 104, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 135, 0, 0, 0, 0, 8, 3895, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7975, 8170, 8442, 7514, 8266, 8721, 8288, 8160] avg=8192.0000 max=8721 max_rank=5 min=7514 min_rank=3 imbalance_ratio=1.064575 max_load_reduction=56815 max_load_reduction_pct=0.866928 moved_to_echo=33435 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 2, 3104, 0, 827, 591, 0, 0, 0, 0, 0, 0, 2678, 0, 0, 1442, 2524, 0, 0, 367, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 1025, 2909, 0, 0, 0, 0, 0, 0, 0, 0, 1443, 117, 0, 1323, 0, 3157, 10, 1, 2475, 177, 0, 0, 0, 0, 0, 0, 2179, 1, 0, 0, 392, 0, 0, 3602, 2898, 142, 0, 0, 0, 0, 0, 0, 4030, 2609, 0, 0, 0, 0, 0, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 4096, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 54, 0, 0, 0, 0, 13, 3899, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8621, 7011, 8003, 8703, 9214, 7660, 8236, 8088] avg=8192.0000 max=9214 max_rank=4 min=7011 min_rank=1 imbalance_ratio=1.124756 max_load_reduction=56322 max_load_reduction_pct=0.859406 moved_to_echo=33198 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 1994, 0, 775, 590, 0, 0, 0, 0, 0, 0, 2510, 0, 0, 2443, 2843, 0, 0, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 1023, 3180, 0, 0, 0, 0, 0, 0, 0, 0, 1232, 245, 0, 469, 2, 3895, 10, 7, 2471, 177, 0, 0, 0, 0, 0, 0, 1026, 1, 3, 0, 68, 0, 0, 3532, 2898, 138, 0, 0, 0, 0, 0, 0, 4077, 3283, 0, 0, 0, 0, 0, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3956, 4094, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 365, 0, 0, 0, 0, 19, 3807, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7455, 8809, 8283, 8508, 7666, 8381, 8147, 8287] avg=8192.0000 max=8809 max_rank=1 min=7455 min_rank=0 imbalance_ratio=1.075317 max_load_reduction=56727 max_load_reduction_pct=0.865585 moved_to_echo=32481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 3565, 0, 847, 587, 0, 0, 0, 0, 0, 0, 2744, 0, 0, 2476, 2647, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 0, 0, 0, 0, 1016, 3509, 0, 0, 0, 0, 0, 0, 0, 0, 169, 73, 0, 319, 0, 3962, 2, 15, 2457, 177, 0, 0, 0, 0, 0, 0, 2052, 0, 1, 0, 23, 0, 0, 3025, 2898, 141, 0, 0, 0, 0, 0, 0, 4052, 3083, 0, 0, 0, 0, 0, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 4096, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 52, 0, 0, 0, 0, 57, 3903, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9096, 8171, 8513, 7174, 8140, 8156, 8158, 8128] avg=8192.0000 max=9096 max_rank=0 min=7174 min_rank=3 imbalance_ratio=1.110352 max_load_reduction=56440 max_load_reduction_pct=0.861206 moved_to_echo=32582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1530, 0, 851, 592, 0, 0, 0, 0, 0, 0, 2754, 0, 0, 2854, 3604, 0, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 1025, 2536, 0, 0, 0, 0, 0, 0, 0, 0, 784, 1021, 0, 1182, 0, 3418, 0, 0, 2478, 177, 0, 0, 0, 0, 0, 0, 1128, 0, 0, 0, 50, 1, 0, 2983, 2898, 141, 0, 0, 0, 0, 0, 0, 3926, 3593, 0, 0, 0, 0, 0, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 4096, 0, 1, 0, 104, 0, 0, 0, 0, 0, 0, 0, 118, 4096, 130, 0, 0, 0, 0, 39, 3909, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7069, 9653, 7612, 9060, 7201, 8540, 8109, 8292] avg=8192.0000 max=9653 max_rank=1 min=7069 min_rank=0 imbalance_ratio=1.178345 max_load_reduction=55883 max_load_reduction_pct=0.852707 moved_to_echo=32142 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 1, 3859, 0, 912, 590, 0, 0, 0, 0, 0, 0, 2954, 0, 0, 3049, 2351, 0, 0, 665, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 0, 0, 0, 0, 1023, 2814, 0, 0, 0, 0, 0, 0, 0, 0, 638, 32, 0, 487, 0, 3691, 0, 0, 2470, 177, 0, 0, 0, 0, 0, 0, 1673, 0, 1, 0, 11, 0, 0, 2918, 2898, 142, 0, 0, 0, 0, 0, 0, 4031, 2585, 0, 0, 0, 0, 0, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3969, 4094, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 263, 0, 0, 0, 0, 38, 3917, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9458, 9019, 7795, 7495, 7643, 7637, 8175, 8314] avg=8192.0000 max=9458 max_rank=0 min=7495 min_rank=3 imbalance_ratio=1.154541 max_load_reduction=56078 max_load_reduction_pct=0.855682 moved_to_echo=31769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 0, 1766, 0, 840, 585, 0, 0, 0, 0, 0, 0, 2697, 0, 0, 1134, 3130, 1, 1, 145, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 1, 4, 2, 1, 0, 1002, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 60, 0, 1314, 0, 3470, 2, 8, 2441, 184, 0, 0, 0, 0, 0, 0, 2392, 0, 2, 1, 104, 0, 0, 3227, 2896, 142, 0, 0, 0, 0, 0, 0, 3775, 3428, 0, 0, 0, 1, 0, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4038, 4090, 1, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 126, 4082, 16, 1, 1, 0, 0, 34, 3902, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7287, 7108, 9048, 8707, 8764, 8217, 8243, 8162] avg=8192.0000 max=9048 max_rank=2 min=7108 min_rank=1 imbalance_ratio=1.104492 max_load_reduction=56488 max_load_reduction_pct=0.861938 moved_to_echo=33386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 2, 3094, 0, 795, 590, 0, 0, 0, 0, 0, 0, 2575, 0, 0, 1137, 2640, 0, 0, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 1023, 2805, 0, 0, 0, 0, 0, 0, 0, 0, 595, 863, 0, 617, 0, 3651, 0, 0, 2472, 177, 0, 0, 0, 0, 0, 0, 1993, 1, 1, 0, 56, 0, 0, 3554, 2898, 139, 0, 0, 0, 0, 0, 0, 3882, 3319, 0, 0, 0, 0, 2, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3981, 4095, 0, 6, 0, 97, 0, 0, 0, 0, 0, 0, 0, 15, 4095, 156, 2, 0, 0, 0, 32, 3805, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8575, 7595, 7840, 8375, 8642, 8224, 8180, 8105] avg=8192.0000 max=8642 max_rank=4 min=7595 min_rank=1 imbalance_ratio=1.054932 max_load_reduction=56894 max_load_reduction_pct=0.868134 moved_to_echo=33151 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 21376, 0, 6784, 4736, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[31872, 25600, 0, 0, 0, 128, 128, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32000, 32768, 128, 128, 0, 896, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32256, 128, 128, 128, 128, 0, 8192, 25216, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[21760, 0, 0, 16128, 22016, 128, 128, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[7168, 2560, 0, 6784, 128, 28032, 128, 128, 19840, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 32768, 1280, 128, 128, 0, 0, 256, 31104, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[13824, 128, 128, 128, 1024, 128, 0, 26624, 23296, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 2, 0, 3788, 0, 0, 3, 0, 0, 0, 4091, 4085, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3818, 19, 4093, 54, 4096, 4053, 0, 4096, 0, 0, 0, 488, 4094, 0, 2, 13, 4032, 4089, 0, 0, 0, 0, 36, 0, 53, 0, 51, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3863, 0, 0, 6, 0, 0, 0, 4095, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4016, 24, 4096, 42, 4093, 4091, 0, 4096, 0, 0, 0, 434, 4095, 8, 0, 1, 3824, 4095, 0, 0, 0, 0, 0, 0, 33, 0, 45, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 0, 3973, 0, 1, 63, 0, 0, 0, 4087, 4089, 4095, 0, 0, 4096, 0, 0, 4095, 0, 0, 3782, 24, 4090, 104, 4096, 4089, 0, 4090, 0, 0, 0, 447, 4082, 0, 1, 15, 3837, 4096, 0, 0, 1, 0, 3, 0, 68, 0, 10, 0, 4095, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 10, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 3851, 0, 1, 11, 0, 0, 0, 4087, 4093, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3102, 0, 4093, 209, 4096, 4090, 0, 4094, 0, 0, 0, 1544, 4095, 0, 0, 0, 3135, 4095, 0, 0, 0, 0, 4, 0, 359, 0, 92, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 1, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 3968, 0, 0, 153, 0, 0, 0, 4089, 4095, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3950, 35, 4094, 20, 4092, 4029, 0, 4090, 0, 0, 0, 136, 4092, 2, 1, 81, 3914, 4096, 0, 0, 0, 0, 0, 0, 74, 0, 35, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 1, 0, 0, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 1, 3651, 0, 2, 0, 0, 0, 0, 4025, 4070, 4095, 0, 1, 4093, 0, 0, 4095, 1, 0, 3865, 75, 4094, 73, 4095, 4083, 0, 4087, 1, 1, 1, 234, 4095, 7, 8, 45, 3937, 4084, 1, 1, 0, 0, 1, 1, 56, 0, 446, 1, 4094, 0, 1, 0, 13, 1, 4088, 1, 1, 8, 0, 0, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3993, 0, 0, 13, 0, 0, 0, 4085, 4083, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3756, 50, 4078, 98, 4096, 4087, 0, 4093, 0, 0, 1, 303, 4093, 0, 6, 18, 3917, 4096, 0, 0, 0, 0, 3, 0, 133, 0, 52, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4033, 0, 0, 123, 0, 0, 0, 4093, 4095, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 3751, 0, 4096, 34, 4095, 4039, 0, 4093, 0, 0, 0, 551, 4096, 1, 0, 13, 3726, 4094, 0, 0, 0, 0, 0, 0, 110, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 3788, 0, 0, 3, 0, 3779, 236, 187, 50, 1, 0, 0, 0, 0, 0, 4091, 4085, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 3818, 19, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 4053, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 3858, 0, 2, 13, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 53, 0, 51, 4089, 3886, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8046, 8199, 8247, 8153, 8393, 8118, 8191, 8189] avg=8192.0000 max=8393 max_rank=4 min=8046 min_rank=0 imbalance_ratio=1.024536 max_load_reduction=57143 max_load_reduction_pct=0.871933 moved_to_echo=32891 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3863, 0, 0, 6, 0, 3779, 236, 187, 50, 1, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 4016, 24, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 4091, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 3859, 8, 0, 1, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 45, 4095, 3886, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 4096, 4089, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8122, 8214, 8453, 8179, 8126, 8063, 8191, 8188] avg=8192.0000 max=8453 max_rank=2 min=8063 min_rank=5 imbalance_ratio=1.031860 max_load_reduction=57083 max_load_reduction_pct=0.871017 moved_to_echo=32568 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 3973, 0, 1, 63, 0, 3778, 235, 187, 50, 1, 0, 0, 0, 0, 0, 4087, 4089, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 3782, 24, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 4089, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 3847, 0, 1, 15, 3837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 68, 0, 10, 4096, 3886, 3, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4095, 4093, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8291, 8199, 8213, 8233, 8147, 8067, 8188, 8198] avg=8192.0000 max=8291 max_rank=0 min=8067 min_rank=5 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32600 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 3851, 0, 1, 11, 0, 3779, 236, 187, 50, 1, 0, 0, 0, 0, 0, 4087, 4093, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 3102, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 4090, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1544, 3859, 0, 0, 0, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 359, 0, 92, 4095, 3886, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 4096, 4093, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8203, 7512, 8343, 8538, 8439, 8193, 8191] avg=8192.0000 max=8538 max_rank=4 min=7512 min_rank=2 imbalance_ratio=1.042236 max_load_reduction=56998 max_load_reduction_pct=0.869720 moved_to_echo=33361 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 3968, 0, 0, 153, 0, 3779, 235, 187, 50, 1, 0, 0, 0, 0, 0, 4089, 4095, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 3950, 35, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4029, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 3857, 2, 1, 81, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 35, 4096, 3886, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 0, 4096, 4088, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8374, 8207, 8396, 8089, 7991, 8095, 8193, 8191] avg=8192.0000 max=8396 max_rank=2 min=7991 min_rank=4 imbalance_ratio=1.024902 max_load_reduction=57140 max_load_reduction_pct=0.871887 moved_to_echo=32470 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4033, 0, 0, 123, 0, 3779, 236, 187, 50, 1, 0, 0, 0, 0, 0, 4093, 4095, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 3751, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4039, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 3860, 1, 0, 13, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 7, 4094, 3886, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8409, 8211, 8164, 8116, 8151, 8101, 8191, 8193] avg=8192.0000 max=8409 max_rank=0 min=8101 min_rank=5 imbalance_ratio=1.026489 max_load_reduction=57127 max_load_reduction_pct=0.871689 moved_to_echo=32636 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 3651, 0, 2, 0, 0, 3780, 238, 191, 52, 2, 0, 0, 0, 0, 0, 4025, 4070, 0, 0, 1, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 1, 0, 3865, 75, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 4083, 0, 4035, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 234, 3857, 7, 8, 45, 3937, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 56, 0, 446, 4084, 3886, 4, 0, 0, 0, 0, 0, 1, 4092, 0, 1, 0, 13, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 8, 0, 0, 1, 0, 1, 4095, 4091, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7918, 8112, 8350, 8194, 8089, 8479, 8196, 8198] avg=8192.0000 max=8479 max_rank=5 min=7918 min_rank=0 imbalance_ratio=1.035034 max_load_reduction=57057 max_load_reduction_pct=0.870621 moved_to_echo=32962 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3993, 0, 0, 13, 0, 3779, 236, 187, 50, 1, 0, 0, 0, 0, 0, 4085, 4083, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 3756, 50, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 4087, 0, 4043, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 303, 3857, 0, 6, 18, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 133, 0, 52, 4096, 3886, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8259, 8191, 8201, 8229, 8101, 8174, 8193, 8188] avg=8192.0000 max=8259 max_rank=0 min=8101 min_rank=4 imbalance_ratio=1.008179 max_load_reduction=57277 max_load_reduction_pct=0.873978 moved_to_echo=32656 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 896, 0, 768, 32768, 31104, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 32768, 0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 2560, 128, 0, 30080, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 0, 32640, 0, 32384, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 31232, 0, 128, 384, 0, 30336, 1920, 1536, 512, 128, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[4224, 30976, 128, 128, 256, 30336, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4075, 4078, 0, 0, 0, 4068, 0, 4096, 4096, 0, 3925, 4096, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 0, 0, 4096, 0, 3989, 4096, 0, 786, 9, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 3152, 235, 0, 1, 0, 4096, 0, 30, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 4071, 4092, 0, 0, 0, 4095, 0, 4096, 4095, 0, 3986, 4096, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 3809, 1, 0, 0, 4095, 0, 3982, 4096, 0, 1083, 79, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 2924, 340, 0, 2, 0, 4096, 0, 15, 0, 1, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4084, 0, 0, 4095, 4090, 4094, 0, 0, 0, 4083, 0, 4096, 4092, 0, 4022, 4096, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 3612, 0, 0, 0, 4096, 0, 3967, 4096, 0, 1367, 152, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 2523, 450, 0, 4, 0, 4096, 51, 24, 0, 1, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4029, 4087, 0, 0, 0, 4066, 0, 4096, 4096, 0, 4023, 4096, 0, 0, 0, 936, 0, 0, 0, 0, 0, 0, 0, 3818, 0, 0, 0, 4096, 0, 4042, 4096, 0, 1797, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1787, 80, 0, 0, 0, 4095, 3, 3, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 4093, 4096, 0, 0, 0, 4062, 0, 4096, 4096, 0, 4054, 4096, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 3812, 0, 0, 0, 4096, 0, 4065, 4096, 0, 1194, 38, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 1823, 1043, 0, 0, 0, 4095, 2, 6, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4080, 3964, 4094, 0, 0, 0, 4087, 0, 4096, 4096, 0, 3815, 4096, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 4096, 0, 4081, 4095, 0, 2597, 13, 0, 1, 0, 0, 0, 2, 4096, 0, 0, 1443, 340, 0, 1, 0, 4096, 0, 25, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 4026, 4090, 0, 0, 0, 4093, 0, 4096, 4090, 0, 3993, 4096, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 0, 0, 3845, 2, 0, 0, 4096, 0, 4081, 4096, 0, 1321, 5, 0, 8, 0, 0, 0, 7, 4096, 0, 0, 1709, 279, 0, 8, 0, 4096, 0, 21, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4094, 1, 0, 4095, 3839, 4084, 0, 1, 0, 3962, 0, 4095, 4092, 0, 3978, 4095, 1, 1, 1, 273, 0, 0, 0, 0, 1, 1, 0, 3983, 0, 1, 1, 4095, 1, 4055, 4094, 0, 1417, 6, 1, 0, 0, 0, 0, 11, 4095, 0, 1, 2693, 206, 0, 11, 1, 4095, 0, 151, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4075, 105, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 0, 3925, 0, 0, 0, 0, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 2866, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 3989, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 786, 9, 0, 0, 0, 0, 0, 5, 4096, 2607, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3152, 235, 0, 1, 0, 1094, 366, 0, 0, 0, 0, 0, 0, 4096, 0, 30, 0, 0, 0, 0, 0, 2879, 1166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8248, 7986, 8339, 8149, 7503, 8944, 8171] avg=8192.0000 max=8944 max_rank=6 min=7503 min_rank=5 imbalance_ratio=1.091797 max_load_reduction=56592 max_load_reduction_pct=0.863525 moved_to_echo=32767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4029, 105, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 0, 4023, 0, 0, 0, 0, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 2866, 1490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 4042, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 2, 0, 0, 0, 0, 0, 0, 4096, 2606, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1787, 80, 0, 0, 0, 1096, 366, 0, 0, 0, 0, 0, 0, 4095, 3, 3, 0, 0, 0, 0, 0, 2886, 1166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8196, 8200, 8685, 8174, 8202, 8501, 7425, 8153] avg=8192.0000 max=8685 max_rank=2 min=7425 min_rank=6 imbalance_ratio=1.060181 max_load_reduction=56851 max_load_reduction_pct=0.867477 moved_to_echo=32281 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 4071, 105, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3725, 0, 3986, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3809, 2865, 1490, 0, 0, 0, 0, 0, 0, 1, 0, 0, 64, 0, 3982, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 79, 0, 0, 0, 0, 0, 5, 4096, 2606, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2924, 340, 0, 2, 0, 1098, 366, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 1, 0, 1, 0, 2889, 1166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8271, 7900, 8164, 8143, 7869, 8826, 8168] avg=8192.0000 max=8826 max_rank=6 min=7869 min_rank=5 imbalance_ratio=1.077393 max_load_reduction=56710 max_load_reduction_pct=0.865326 moved_to_echo=33006 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4084, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 4090, 105, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3722, 0, 4022, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 2866, 1490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 3967, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 152, 0, 0, 0, 0, 0, 8, 4096, 2606, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2523, 450, 0, 4, 0, 1098, 366, 0, 0, 0, 0, 0, 0, 4096, 51, 24, 0, 1, 0, 0, 0, 2891, 1166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8278, 7985, 7968, 8127, 8229, 8537, 8229] avg=8192.0000 max=8537 max_rank=6 min=7968 min_rank=3 imbalance_ratio=1.042114 max_load_reduction=56999 max_load_reduction_pct=0.869736 moved_to_echo=33122 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4093, 105, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 0, 4054, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3812, 2866, 1490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 4065, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 38, 0, 6, 0, 0, 0, 0, 4096, 2606, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1823, 1043, 0, 0, 0, 1099, 366, 0, 0, 0, 0, 0, 0, 4095, 2, 6, 0, 0, 0, 0, 0, 2892, 1166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8260, 8161, 8168, 8225, 7940, 8427, 8161] avg=8192.0000 max=8427 max_rank=6 min=7940 min_rank=5 imbalance_ratio=1.028687 max_load_reduction=57109 max_load_reduction_pct=0.871414 moved_to_echo=32753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4080, 4, 0, 0, 0, 0, 0, 0, 0, 3964, 105, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 0, 3815, 0, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 2866, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 4081, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2597, 13, 0, 1, 0, 0, 0, 2, 4096, 2607, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1443, 340, 0, 1, 0, 1098, 366, 0, 0, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 0, 0, 0, 2891, 1166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8180, 8156, 7945, 8177, 8240, 9316, 7344, 8178] avg=8192.0000 max=9316 max_rank=5 min=7344 min_rank=6 imbalance_ratio=1.137207 max_load_reduction=56220 max_load_reduction_pct=0.857849 moved_to_echo=33078 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 4026, 105, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 3993, 0, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3845, 2866, 1490, 0, 0, 0, 0, 0, 0, 2, 0, 0, 64, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1321, 5, 0, 8, 0, 0, 0, 7, 4096, 2606, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1709, 279, 0, 8, 0, 1097, 366, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 0, 2888, 1166, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8224, 8905, 8201, 8243, 8043, 7555, 8171] avg=8192.0000 max=8905 max_rank=2 min=7555 min_rank=6 imbalance_ratio=1.087036 max_load_reduction=56631 max_load_reduction_pct=0.864120 moved_to_echo=32012 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4094, 1, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 3839, 97, 0, 1, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 0, 3978, 0, 1, 1, 1, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3983, 2870, 1490, 0, 0, 0, 0, 0, 0, 0, 1, 1, 55, 1, 4055, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 6, 1, 0, 0, 0, 0, 11, 4095, 2605, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 2693, 206, 0, 11, 1, 1100, 371, 0, 0, 0, 0, 0, 0, 4095, 0, 151, 1, 0, 0, 0, 1, 2887, 1170, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 7899, 7969, 8345, 8207, 8135, 8478, 8305] avg=8192.0000 max=8478 max_rank=6 min=7899 min_rank=1 imbalance_ratio=1.034912 max_load_reduction=57058 max_load_reduction_pct=0.870636 moved_to_echo=33125 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[29824, 0, 31872, 0, 128, 128, 128, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 384, 128, 128, 0, 128, 128, 23168, 9344, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11648, 384, 128, 128, 0, 0, 0, 128, 32768, 20864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32256, 896, 0, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 128, 0, 30720, 23040, 12032, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 512, 128, 32384, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 18176, 3072, 0, 128, 128, 8832, 2944, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 2842, 0, 0, 4095, 0, 0, 0, 366, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 2, 61, 0, 4065, 0, 0, 1, 0, 4096, 0, 3, 4096, 4, 0, 4096, 3935, 4096, 0, 0, 0, 0, 488, 2, 4096, 0, 0, 1, 0, 4096, 0, 524, 0, 4096, 4093, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 1454, 0, 0, 4096, 0, 0, 0, 1415, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4083, 0, 4095, 0, 0, 34, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 4096, 3927, 4096, 0, 0, 0, 0, 363, 7, 4096, 0, 0, 0, 0, 4096, 0, 1000, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 2319, 0, 0, 4096, 0, 0, 0, 923, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 13, 0, 4048, 0, 0, 3, 0, 4096, 0, 0, 4096, 75, 0, 4096, 4053, 4096, 0, 0, 0, 0, 507, 1, 4096, 0, 0, 0, 0, 4096, 0, 354, 0, 4096, 4092, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 1857, 0, 0, 4096, 0, 0, 0, 1826, 4094, 7, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 1, 0, 4092, 0, 4095, 0, 1, 28, 0, 4087, 0, 0, 2, 6, 4096, 0, 0, 4096, 15, 0, 4096, 4076, 4096, 0, 0, 0, 0, 298, 8, 4096, 0, 0, 0, 0, 4096, 0, 89, 0, 4095, 4094, 0, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 2076, 0, 0, 4096, 0, 0, 0, 1472, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 16, 0, 4070, 0, 0, 0, 0, 4096, 0, 0, 4096, 122, 0, 4096, 4081, 4096, 0, 0, 0, 0, 373, 0, 4096, 0, 0, 0, 0, 4096, 0, 79, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 755, 0, 0, 4096, 0, 0, 0, 2395, 4093, 1, 0, 0, 0, 0, 4067, 1, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 209, 0, 4090, 0, 0, 2, 3, 4079, 0, 5, 4096, 5, 0, 4096, 4009, 4096, 0, 0, 0, 0, 577, 0, 4096, 0, 0, 0, 0, 4096, 0, 287, 0, 4096, 4094, 0, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 2161, 0, 0, 4096, 0, 0, 0, 679, 4088, 21, 0, 0, 0, 0, 4087, 2, 0, 0, 0, 3, 0, 4091, 0, 4096, 0, 5, 47, 0, 4036, 0, 0, 7, 9, 4096, 0, 0, 4096, 24, 0, 4096, 4018, 4096, 0, 0, 0, 0, 610, 0, 4096, 0, 0, 0, 0, 4096, 0, 683, 0, 4096, 4092, 0, 0, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1, 1416, 0, 1, 4093, 0, 1, 1, 2113, 4086, 1, 1, 1, 1, 0, 4076, 0, 1, 0, 0, 0, 1, 4090, 0, 4094, 1, 7, 10, 0, 4075, 0, 0, 16, 10, 4095, 0, 0, 4095, 117, 0, 4094, 4050, 4095, 1, 0, 1, 1, 411, 6, 4095, 1, 0, 0, 0, 4095, 0, 89, 1, 4095, 4082, 1, 1, 12, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2319, 0, 0, 4096, 0, 0, 0, 1389, 388, 376, 51, 28, 3, 0, 0, 923, 2704, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 13, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4045, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 4053, 0, 0, 0, 0, 0, 507, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 3708, 4092, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8650, 7725, 8187, 8129, 8219, 8280, 8190, 8156] avg=8192.0000 max=8650 max_rank=0 min=7725 min_rank=1 imbalance_ratio=1.055908 max_load_reduction=56886 max_load_reduction_pct=0.868011 moved_to_echo=32845 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2076, 0, 0, 4096, 0, 0, 0, 1389, 388, 376, 51, 28, 3, 0, 0, 1472, 2704, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 16, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4045, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 4081, 0, 0, 0, 0, 0, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 3708, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8407, 8271, 8195, 8154, 8263, 8174, 8189, 7883] avg=8192.0000 max=8407 max_rank=0 min=7883 min_rank=7 imbalance_ratio=1.026245 max_load_reduction=57129 max_load_reduction_pct=0.871719 moved_to_echo=32509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2842, 0, 0, 4095, 0, 0, 0, 1389, 388, 376, 51, 28, 3, 0, 0, 366, 2704, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4068, 0, 2, 61, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 3, 4045, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 3935, 0, 0, 0, 0, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 0, 3708, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9172, 7166, 8193, 8196, 8149, 8143, 8192, 8325] avg=8192.0000 max=9172 max_rank=0 min=7166 min_rank=1 imbalance_ratio=1.119629 max_load_reduction=56364 max_load_reduction_pct=0.860046 moved_to_echo=32809 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1857, 0, 0, 4096, 0, 0, 0, 1388, 388, 376, 51, 28, 3, 0, 0, 1826, 2703, 7, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4067, 0, 1, 28, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 4096, 0, 0, 4045, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 4076, 0, 0, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 3707, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8626, 8192, 8183, 8164, 8094, 8197, 7893] avg=8192.0000 max=8626 max_rank=1 min=7893 min_rank=7 imbalance_ratio=1.052979 max_load_reduction=56910 max_load_reduction_pct=0.868378 moved_to_echo=32348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1454, 0, 0, 4096, 0, 0, 0, 1389, 388, 376, 51, 28, 3, 0, 0, 1415, 2704, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4083, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4045, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 3927, 0, 0, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 0, 3708, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7785, 8215, 8186, 8197, 8143, 8010, 8196, 8804] avg=8192.0000 max=8804 max_rank=7 min=7785 min_rank=0 imbalance_ratio=1.074707 max_load_reduction=56732 max_load_reduction_pct=0.865662 moved_to_echo=33153 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 755, 0, 0, 4096, 0, 0, 0, 1388, 388, 376, 51, 28, 3, 0, 0, 2395, 2702, 1, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 209, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4079, 0, 5, 4045, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 4009, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 3708, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7085, 9165, 8194, 8367, 8139, 8306, 8189, 8091] avg=8192.0000 max=9165 max_rank=1 min=7085 min_rank=0 imbalance_ratio=1.118774 max_load_reduction=56371 max_load_reduction_pct=0.860153 moved_to_echo=32725 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2161, 0, 0, 4096, 0, 0, 0, 1386, 388, 376, 51, 28, 3, 0, 0, 679, 2699, 21, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 0, 4091, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4068, 0, 5, 47, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 4096, 0, 0, 4045, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 4018, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 0, 3708, 4092, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8489, 7486, 8195, 8156, 8181, 8348, 8189, 8492] avg=8192.0000 max=8492 max_rank=7 min=7486 min_rank=1 imbalance_ratio=1.036621 max_load_reduction=57044 max_load_reduction_pct=0.870422 moved_to_echo=33210 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 1416, 0, 1, 4093, 0, 1, 1, 1390, 390, 377, 55, 33, 3, 0, 0, 2113, 2689, 1, 1, 1, 1, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4090, 0, 4095, 7, 0, 0, 0, 0, 0, 0, 4061, 1, 7, 10, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 10, 4095, 0, 0, 4040, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3717, 4050, 0, 1, 0, 1, 1, 411, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4095, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 1, 3705, 4082, 1, 1, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7761, 8882, 8194, 8154, 8278, 8181, 8194, 7892] avg=8192.0000 max=8882 max_rank=1 min=7761 min_rank=0 imbalance_ratio=1.084229 max_load_reduction=56654 max_load_reduction_pct=0.864471 moved_to_echo=32545 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 128, 128, 512, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[3200, 128, 29696, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[11264, 21632, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 14976, 0, 128, 32768, 0, 128, 128, 11136, 3200, 3072, 512, 256, 128, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[29824, 32256, 0, 128, 0, 128, 128, 3712, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 128, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 128, 32768, 0, 128, 32384, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 3079, 0, 0, 37, 0, 4090, 0, 3, 465, 4096, 0, 0, 0, 64, 0, 1, 4, 4096, 0, 4096, 0, 0, 0, 0, 4090, 543, 0, 3679, 0, 173, 4096, 5, 4096, 0, 0, 63, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2590, 0, 819, 0, 715, 58, 0, 4, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 3194, 0, 0, 100, 0, 4095, 0, 7, 106, 4096, 0, 0, 0, 188, 0, 0, 176, 4096, 0, 4096, 0, 0, 0, 0, 4095, 467, 0, 3755, 0, 4, 4096, 164, 4096, 0, 0, 17, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2918, 0, 106, 0, 1079, 7, 0, 2, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 2787, 0, 0, 39, 0, 4094, 0, 0, 146, 4096, 0, 0, 0, 3, 0, 0, 4, 4096, 0, 4096, 0, 0, 0, 0, 4085, 815, 0, 3785, 0, 544, 4096, 146, 4096, 0, 0, 5, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2049, 0, 1137, 0, 726, 113, 0, 2, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 3555, 0, 0, 102, 0, 4096, 0, 4, 306, 4096, 0, 0, 0, 5, 0, 0, 24, 4096, 0, 4096, 0, 0, 0, 0, 4093, 1005, 0, 3475, 0, 90, 4096, 131, 4096, 0, 0, 17, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2380, 0, 795, 0, 388, 16, 0, 2, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 3286, 0, 0, 145, 0, 4094, 0, 0, 245, 4096, 0, 0, 0, 84, 0, 0, 64, 4096, 0, 4096, 0, 0, 0, 0, 4090, 1490, 0, 3434, 0, 3, 4096, 117, 4096, 0, 0, 13, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2268, 0, 407, 0, 719, 19, 0, 2, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4095, 0, 0, 0, 3212, 0, 0, 245, 0, 4091, 0, 4, 703, 4096, 0, 0, 0, 17, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 4095, 755, 0, 3725, 0, 542, 4096, 20, 4096, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1892, 0, 609, 0, 551, 17, 0, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 3, 3833, 0, 0, 7, 0, 4096, 0, 2, 275, 4096, 0, 0, 0, 19, 0, 1, 2, 4096, 0, 4096, 0, 0, 0, 0, 4082, 751, 0, 3426, 0, 83, 4096, 69, 4096, 0, 0, 18, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1779, 0, 893, 0, 1053, 78, 0, 11, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 4092, 0, 1, 1, 3378, 0, 0, 168, 1, 4089, 0, 44, 186, 4095, 1, 1, 0, 192, 0, 6, 79, 4095, 1, 4095, 0, 1, 0, 1, 4084, 332, 0, 3964, 1, 96, 4095, 415, 4095, 1, 0, 9, 4095, 4095, 1, 0, 0, 1, 1, 0, 4094, 1, 0, 2688, 1, 83, 1, 430, 231, 0, 4, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 807, 4095, 0, 0, 0, 3555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 4096, 0, 4, 306, 3564, 116, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 24, 3980, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 0, 3475, 0, 90, 3442, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4079, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2380, 0, 1103, 654, 17, 0, 0, 0, 0, 0, 795, 0, 388, 16, 0, 2, 0, 4094, 2186, 524, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8457, 8188, 8105, 8197, 8143, 8191, 8250, 8005] avg=8192.0000 max=8457 max_rank=0 min=8005 min_rank=7 imbalance_ratio=1.032349 max_load_reduction=57079 max_load_reduction_pct=0.870956 moved_to_echo=32589 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 807, 4096, 0, 0, 0, 3194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 4095, 0, 7, 106, 3564, 117, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 188, 0, 0, 176, 3979, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 467, 0, 3755, 0, 4, 3442, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2918, 0, 1103, 654, 17, 0, 0, 0, 0, 0, 106, 0, 1079, 7, 0, 2, 0, 4096, 2186, 524, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8097, 7989, 8439, 8199, 7832, 8192, 8788, 8000] avg=8192.0000 max=8788 max_rank=6 min=7832 min_rank=4 imbalance_ratio=1.072754 max_load_reduction=56748 max_load_reduction_pct=0.865906 moved_to_echo=32812 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 807, 4095, 0, 0, 0, 3079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 4090, 0, 3, 465, 3564, 117, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 64, 0, 1, 4, 3979, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 3679, 0, 173, 3442, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2590, 0, 1103, 654, 17, 0, 0, 0, 0, 0, 819, 0, 715, 58, 0, 4, 0, 4096, 2186, 524, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7981, 8276, 8144, 8194, 7842, 8238, 8459, 8402] avg=8192.0000 max=8459 max_rank=6 min=7842 min_rank=4 imbalance_ratio=1.032593 max_load_reduction=57077 max_load_reduction_pct=0.870926 moved_to_echo=32941 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 807, 4096, 0, 0, 0, 2787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 4094, 0, 0, 146, 3564, 117, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 4, 3979, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 815, 0, 3785, 0, 544, 3442, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2049, 0, 1103, 654, 17, 0, 0, 0, 0, 0, 1137, 0, 726, 113, 0, 2, 0, 4096, 2186, 524, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7690, 7960, 8082, 8189, 8732, 8180, 7919, 8784] avg=8192.0000 max=8784 max_rank=7 min=7690 min_rank=0 imbalance_ratio=1.072266 max_load_reduction=56752 max_load_reduction_pct=0.865967 moved_to_echo=33615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 807, 4096, 0, 0, 0, 3286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 4094, 0, 0, 245, 3564, 117, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 84, 0, 0, 64, 3979, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1490, 0, 3434, 0, 3, 3442, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2268, 0, 1103, 654, 17, 0, 0, 0, 0, 0, 407, 0, 719, 19, 0, 2, 0, 4096, 2186, 524, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8165, 8223, 8194, 8486, 8188, 8138, 7953] avg=8192.0000 max=8486 max_rank=4 min=7953 min_rank=7 imbalance_ratio=1.035889 max_load_reduction=57050 max_load_reduction_pct=0.870514 moved_to_echo=32765 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 807, 4095, 0, 0, 0, 3212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 4091, 0, 4, 703, 3564, 117, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 2, 3979, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 755, 0, 3725, 0, 542, 3442, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1892, 0, 1103, 654, 17, 0, 0, 0, 0, 0, 609, 0, 551, 17, 0, 0, 0, 4096, 2186, 524, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8114, 8724, 8094, 8199, 8484, 8176, 7762, 7983] avg=8192.0000 max=8724 max_rank=1 min=7762 min_rank=6 imbalance_ratio=1.064941 max_load_reduction=56812 max_load_reduction_pct=0.866882 moved_to_echo=32405 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 796, 4092, 0, 1, 1, 3378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 1, 4089, 0, 44, 186, 3568, 117, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 192, 0, 6, 79, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 332, 0, 3964, 1, 96, 3438, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 4072, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4094, 1, 0, 2688, 1, 1106, 657, 23, 0, 0, 0, 0, 0, 83, 1, 430, 231, 0, 4, 0, 4095, 2193, 527, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8269, 8173, 8352, 8182, 8246, 8178, 8572, 7564] avg=8192.0000 max=8572 max_rank=6 min=7564 min_rank=7 imbalance_ratio=1.046387 max_load_reduction=56964 max_load_reduction_pct=0.869202 moved_to_echo=32560 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 807, 4096, 0, 0, 3, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 2, 275, 3564, 117, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 1, 2, 3979, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 751, 0, 3426, 0, 83, 3442, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1779, 0, 1103, 654, 17, 0, 0, 0, 0, 0, 893, 0, 1053, 78, 0, 11, 0, 4096, 2186, 528, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8739, 8061, 8097, 8182, 7771, 8193, 7648, 8845] avg=8192.0000 max=8845 max_rank=7 min=7648 min_rank=6 imbalance_ratio=1.079712 max_load_reduction=56691 max_load_reduction_pct=0.865036 moved_to_echo=32457 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 6528, 32768, 0, 128, 128, 26368, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 128, 128, 0, 640, 0, 128, 384, 31872, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[6272, 0, 29312, 128, 1536, 27648, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 256, 32640, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 18688, 128, 8832, 5248, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 896, 128, 32768, 0, 128, 2432, 28544, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4864, 128, 5760, 640, 0, 128, 0, 32768, 17536, 4224, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 4096, 4085, 0, 3966, 4096, 0, 3, 0, 0, 4096, 4096, 0, 4096, 0, 0, 3227, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 1509, 0, 0, 4096, 0, 0, 0, 4096, 3522, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 47, 0, 0, 0, 127, 0, 4096, 4068, 0, 3900, 4096, 0, 1, 0, 0, 4096, 4093, 0, 4096, 0, 0, 2336, 4096, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 2058, 0, 0, 4096, 0, 0, 0, 4096, 3847, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 36, 0, 0, 0, 155, 0, 4096, 4034, 0, 3964, 4096, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 2763, 4096, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 19, 0, 0, 0, 2284, 1, 0, 4095, 0, 0, 0, 4096, 3110, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 9, 1, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 43, 0, 0, 1, 493, 0, 4096, 3997, 0, 3971, 4096, 0, 48, 0, 0, 4096, 4094, 0, 4096, 0, 0, 2883, 4096, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 2, 0, 0, 0, 2102, 0, 0, 4095, 0, 0, 0, 4096, 2829, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 1, 1, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 0, 528, 0, 4096, 4067, 0, 3971, 4096, 0, 6, 0, 0, 4095, 4096, 0, 4095, 0, 0, 2472, 4096, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 21, 0, 0, 0, 1484, 0, 0, 4096, 0, 0, 0, 4096, 3830, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 20, 0, 0, 0, 3, 0, 4096, 4083, 0, 3389, 4096, 0, 1, 0, 0, 4096, 4093, 0, 4096, 0, 0, 2384, 4096, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 28, 0, 0, 0, 3215, 0, 0, 4096, 0, 0, 0, 4096, 3256, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 19, 0, 0, 0, 57, 0, 4096, 4037, 0, 3743, 4096, 0, 0, 0, 0, 4096, 4089, 0, 4096, 0, 0, 3256, 4096, 0, 0, 4094, 0, 0, 0, 0, 8, 0, 0, 2, 0, 0, 0, 1544, 3, 0, 4092, 0, 0, 0, 4096, 3697, 0, 0, 0, 4096, 27, 0, 0, 0, 0, 0, 4, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 139, 0, 0, 0, 148, 0, 4095, 4060, 0, 4020, 4095, 1, 6, 0, 1, 4093, 4095, 0, 4095, 0, 0, 1618, 4095, 1, 1, 4093, 1, 0, 1, 0, 31, 1, 1, 82, 0, 0, 1, 2638, 0, 0, 4091, 0, 2, 1, 4091, 3635, 2, 1, 0, 4095, 4, 0, 0, 1, 1, 0, 7, 4095, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 47, 0, 0, 0, 4017, 3470, 633, 30, 0, 0, 0, 0, 127, 0, 79, 4068, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2336, 1480, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 2058, 0, 0, 4096, 0, 0, 0, 1983, 5, 3, 0, 0, 0, 0, 0, 4096, 3847, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8174, 8185, 7912, 8165, 8145, 8569, 8189] avg=8192.0000 max=8569 max_rank=6 min=7912 min_rank=3 imbalance_ratio=1.046021 max_load_reduction=56967 max_load_reduction_pct=0.869247 moved_to_echo=33068 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4017, 3470, 633, 31, 0, 0, 0, 0, 42, 0, 79, 4085, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3227, 1480, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 1509, 0, 0, 4096, 0, 0, 0, 1983, 5, 3, 0, 0, 0, 0, 0, 4096, 3522, 0, 0, 0, 626, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8172, 8190, 8803, 8187, 7596, 8248, 8189] avg=8192.0000 max=8803 max_rank=3 min=7596 min_rank=5 imbalance_ratio=1.074585 max_load_reduction=56733 max_load_reduction_pct=0.865677 moved_to_echo=32220 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 36, 0, 0, 0, 4017, 3470, 633, 30, 0, 0, 0, 0, 155, 0, 79, 4034, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2763, 1480, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 19, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 2284, 1, 0, 4095, 0, 0, 0, 1983, 5, 3, 0, 0, 0, 0, 0, 4096, 3110, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8232, 8186, 8339, 8191, 8371, 7832, 8199] avg=8192.0000 max=8371 max_rank=5 min=7832 min_rank=6 imbalance_ratio=1.021851 max_load_reduction=57165 max_load_reduction_pct=0.872269 moved_to_echo=32593 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 0, 4017, 3470, 633, 31, 0, 0, 0, 0, 528, 0, 79, 4067, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2472, 1480, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 21, 0, 0, 4096, 4064, 0, 0, 0, 0, 0, 0, 0, 1484, 0, 0, 4096, 0, 0, 0, 1983, 5, 3, 0, 0, 0, 0, 0, 4096, 3830, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8157, 8645, 8192, 8047, 8182, 7571, 8552, 8190] avg=8192.0000 max=8645 max_rank=1 min=7571 min_rank=5 imbalance_ratio=1.055298 max_load_reduction=56891 max_load_reduction_pct=0.868088 moved_to_echo=32495 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 43, 0, 0, 1, 4017, 3470, 633, 31, 0, 0, 0, 0, 493, 0, 79, 3997, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 4091, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2883, 1480, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 2102, 0, 0, 4095, 0, 0, 0, 1983, 5, 3, 0, 0, 0, 0, 0, 4096, 2829, 0, 0, 0, 626, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8540, 8233, 8459, 8170, 8188, 7560, 8191] avg=8192.0000 max=8540 max_rank=1 min=7560 min_rank=6 imbalance_ratio=1.042480 max_load_reduction=56996 max_load_reduction_pct=0.869690 moved_to_echo=32109 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 19, 0, 0, 0, 4017, 3470, 633, 31, 0, 0, 0, 0, 57, 0, 79, 4037, 0, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3256, 1480, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 1544, 3, 0, 4092, 0, 0, 0, 1983, 5, 3, 0, 0, 0, 0, 0, 4096, 3697, 0, 0, 0, 626, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 7916, 8180, 8830, 8171, 7630, 8446, 8193] avg=8192.0000 max=8830 max_rank=3 min=7630 min_rank=5 imbalance_ratio=1.077881 max_load_reduction=56706 max_load_reduction_pct=0.865265 moved_to_echo=32440 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 139, 0, 0, 0, 4021, 3477, 639, 31, 0, 0, 0, 0, 148, 0, 74, 4060, 0, 4020, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 4083, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1618, 1472, 1, 1, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 31, 1, 1, 82, 0, 0, 4095, 4064, 0, 0, 0, 0, 0, 0, 1, 2638, 0, 0, 4091, 0, 2, 1, 1984, 10, 6, 0, 0, 0, 0, 0, 4091, 3635, 2, 1, 0, 618, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 7, 4095, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8310, 8303, 8185, 7186, 8275, 8733, 8351, 8193] avg=8192.0000 max=8733 max_rank=5 min=7186 min_rank=3 imbalance_ratio=1.066040 max_load_reduction=56803 max_load_reduction_pct=0.866745 moved_to_echo=33552 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 20, 0, 0, 0, 4017, 3470, 633, 30, 0, 0, 0, 0, 3, 0, 79, 4083, 0, 3389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2384, 1480, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 28, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 3215, 0, 0, 4096, 0, 0, 0, 1983, 5, 3, 0, 0, 0, 0, 0, 4096, 3256, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 7554, 8185, 7960, 8195, 9302, 7978, 8192] avg=8192.0000 max=9302 max_rank=5 min=7554 min_rank=1 imbalance_ratio=1.135498 max_load_reduction=56234 max_load_reduction_pct=0.858063 moved_to_echo=33667 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1664, 0, 640, 32512, 0, 30976, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 20992, 11904, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 16896, 128, 0, 32768, 0, 128, 128, 15872, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 256, 0, 0, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 384, 0, 0, 128, 32256, 27776, 5120, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 27776, 128, 128, 0, 5120, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3904, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4087, 4096, 56, 0, 0, 3, 0, 0, 2233, 0, 4076, 0, 1068, 0, 0, 0, 0, 3231, 0, 0, 4069, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1849, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 5, 0, 0, 4096, 0, 3650, 0, 0, 4090, 3, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4071, 4096, 140, 0, 0, 9, 0, 48, 1160, 0, 4034, 0, 3041, 0, 0, 0, 0, 3005, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1323, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4096, 0, 3799, 0, 0, 4094, 2, 0, 4096, 4096, 0, 0, 4094, 0, 0, 4080, 4095, 46, 0, 0, 5, 0, 7, 2557, 0, 4081, 0, 2272, 0, 0, 0, 0, 3259, 0, 0, 4075, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 2, 0, 0, 0, 0, 4096, 4096, 0, 394, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3206, 0, 0, 4091, 2, 0, 4096, 4096, 0, 0, 4095, 0, 0, 4012, 4096, 230, 0, 1, 10, 0, 84, 1899, 0, 4088, 0, 2897, 0, 0, 0, 0, 3007, 0, 0, 4071, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1075, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2813, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4085, 4096, 445, 0, 0, 0, 0, 14, 1816, 0, 4084, 0, 2688, 0, 0, 0, 0, 3196, 0, 0, 4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1351, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 1, 0, 4095, 1, 3397, 1, 1, 4074, 6, 1, 4089, 4095, 0, 0, 4088, 1, 0, 4083, 4095, 119, 0, 8, 11, 1, 4, 2294, 0, 4091, 0, 1985, 0, 1, 0, 0, 2388, 0, 0, 4095, 0, 1, 1, 0, 0, 4094, 1, 1, 0, 3, 1, 0, 1, 0, 4095, 4095, 1, 2119, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3629, 0, 0, 4087, 10, 0, 4096, 4096, 0, 0, 4091, 0, 0, 4024, 4095, 218, 1, 0, 25, 0, 37, 2586, 0, 4053, 0, 2221, 0, 0, 0, 0, 3003, 0, 0, 4081, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 699, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3701, 0, 0, 4089, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4081, 4096, 61, 0, 0, 9, 0, 10, 1749, 0, 4034, 0, 3394, 0, 0, 0, 0, 2665, 0, 0, 4086, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 792, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3650, 0, 0, 4090, 3, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4071, 33, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 9, 0, 48, 1160, 0, 4096, 1859, 0, 0, 0, 0, 0, 0, 2754, 0, 3041, 0, 0, 0, 0, 3005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2204, 1906, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1602, 1280, 1200, 0, 0, 0, 0, 0, 0, 2896, 4096, 0, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8197, 8331, 8200, 7312, 8800, 8203, 8178, 8315] avg=8192.0000 max=8800 max_rank=4 min=7312 min_rank=3 imbalance_ratio=1.074219 max_load_reduction=56736 max_load_reduction_pct=0.865723 moved_to_echo=33496 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 0, 4096, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 33, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 3, 0, 0, 2233, 0, 4096, 1859, 0, 0, 0, 0, 0, 0, 2783, 0, 1068, 0, 0, 0, 0, 3231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 2204, 1906, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1602, 1293, 1200, 0, 0, 0, 0, 0, 0, 2896, 4096, 0, 1849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8588, 8216, 8247, 7082, 8179, 8191, 8841] avg=8192.0000 max=8841 max_rank=7 min=7082 min_rank=4 imbalance_ratio=1.079224 max_load_reduction=56695 max_load_reduction_pct=0.865097 moved_to_echo=32293 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3799, 0, 0, 4094, 2, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4080, 34, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 5, 0, 7, 2557, 0, 4096, 1858, 0, 0, 0, 0, 0, 0, 2787, 0, 2272, 0, 0, 0, 0, 3259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 2203, 1906, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 2, 0, 0, 0, 1602, 1294, 1200, 0, 0, 0, 0, 0, 0, 2896, 4096, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8483, 8208, 8569, 8318, 8184, 8195, 7386] avg=8192.0000 max=8569 max_rank=3 min=7386 min_rank=7 imbalance_ratio=1.046021 max_load_reduction=56967 max_load_reduction_pct=0.869247 moved_to_echo=32083 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 0, 0, 4091, 2, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4012, 33, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 1, 10, 0, 84, 1899, 0, 4096, 1859, 0, 0, 0, 0, 0, 0, 2791, 0, 2897, 0, 0, 0, 0, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 2204, 1906, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1602, 1297, 1200, 0, 0, 0, 0, 0, 0, 2896, 4096, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7887, 8140, 8179, 8695, 8181, 8195, 8067] avg=8192.0000 max=8695 max_rank=4 min=7887 min_rank=1 imbalance_ratio=1.061401 max_load_reduction=56841 max_load_reduction_pct=0.867325 moved_to_echo=33138 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2813, 0, 0, 4096, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 33, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 0, 0, 14, 1816, 0, 4096, 1859, 0, 0, 0, 0, 0, 0, 2789, 0, 2688, 0, 0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 2204, 1906, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1602, 1295, 1200, 0, 0, 0, 0, 0, 0, 2896, 4096, 0, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 7497, 8214, 8230, 8673, 8194, 8193, 8343] avg=8192.0000 max=8673 max_rank=4 min=7497 min_rank=1 imbalance_ratio=1.058716 max_load_reduction=56863 max_load_reduction_pct=0.867661 moved_to_echo=33403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3397, 1, 1, 4074, 6, 1, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 4083, 27, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 8, 11, 1, 4, 2294, 0, 4095, 1862, 0, 0, 0, 0, 0, 0, 2790, 0, 1985, 0, 1, 0, 0, 2388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 2206, 1906, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 3, 1, 0, 1, 1607, 1301, 1203, 6, 0, 0, 0, 0, 0, 2892, 4095, 1, 2119, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8057, 8199, 8394, 7164, 8209, 8218, 9109] avg=8192.0000 max=9109 max_rank=7 min=7164 min_rank=4 imbalance_ratio=1.111938 max_load_reduction=56427 max_load_reduction_pct=0.861008 moved_to_echo=32700 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 0, 0, 4087, 10, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4024, 34, 0, 0, 0, 0, 0, 0, 0, 0, 218, 1, 0, 25, 0, 37, 2586, 0, 4096, 1858, 0, 0, 0, 0, 0, 0, 2767, 0, 2221, 0, 0, 0, 0, 3003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 2203, 1906, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 1602, 1286, 1200, 0, 0, 0, 0, 0, 0, 2896, 4096, 0, 699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8314, 8149, 8821, 7991, 8190, 8188, 7691] avg=8192.0000 max=8821 max_rank=3 min=7691 min_rank=7 imbalance_ratio=1.076782 max_load_reduction=56715 max_load_reduction_pct=0.865402 moved_to_echo=32060 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3701, 0, 0, 4089, 1, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 33, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 9, 0, 10, 1749, 0, 4096, 1859, 0, 0, 0, 0, 0, 0, 2754, 0, 3394, 0, 0, 0, 0, 2665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 2204, 1906, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1602, 1280, 1200, 0, 0, 0, 0, 0, 0, 2896, 4096, 0, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8379, 8210, 7784, 8813, 8196, 8178, 7784] avg=8192.0000 max=8813 max_rank=4 min=7784 min_rank=3 imbalance_ratio=1.075806 max_load_reduction=56723 max_load_reduction_pct=0.865524 moved_to_echo=32971 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 32640, 384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[22272, 0, 19584, 0, 128, 0, 0, 23808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 23168, 32768, 128, 9728, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 128, 12928, 10368, 9728, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 0, 128, 128, 0, 0, 17664, 15360, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 28160, 128, 128, 32768, 128, 128, 4736, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1408, 128, 128, 128, 128, 256, 16384, 0, 32768, 14976, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3927, 3982, 0, 0, 0, 0, 0, 0, 15, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4094, 1251, 49, 0, 4096, 4096, 0, 708, 0, 1, 0, 0, 1, 0, 4096, 0, 0, 4095, 2363, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 15, 0, 0, 3867, 4032, 0, 1, 1, 1, 2, 1, 87, 1, 4078, 2, 0, 1, 0, 4086, 1, 1, 1, 0, 4095, 1, 0, 4093, 0, 0, 4091, 3, 0, 1, 28, 4095, 906, 2, 0, 4095, 4094, 0, 312, 1, 0, 8, 1, 4, 0, 4095, 1, 1, 4093, 3176, 0, 3972, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4029, 3945, 0, 0, 0, 0, 35, 0, 25, 0, 4020, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 4081, 555, 1, 0, 4096, 4096, 0, 257, 0, 2, 0, 0, 5, 0, 4096, 0, 0, 4090, 3535, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3975, 4015, 0, 0, 0, 0, 5, 0, 27, 0, 4055, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4094, 96, 1, 0, 4096, 4096, 0, 644, 0, 2, 1, 0, 4, 0, 4095, 0, 0, 4087, 3574, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4037, 4018, 0, 0, 0, 0, 2, 0, 23, 0, 4065, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4092, 1204, 4, 0, 4096, 4096, 0, 209, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4091, 2825, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3982, 4070, 0, 0, 0, 0, 2, 0, 33, 0, 3966, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 147, 0, 0, 4096, 4096, 0, 690, 0, 0, 1, 0, 4, 0, 4095, 0, 0, 4070, 3520, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 3965, 3841, 0, 0, 0, 0, 60, 0, 16, 0, 4057, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 493, 1, 0, 4096, 4096, 0, 252, 0, 2, 1, 0, 1, 0, 4094, 0, 0, 4090, 3715, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3866, 3967, 0, 0, 0, 0, 7, 0, 24, 0, 4070, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 4096, 1280, 0, 0, 4096, 4096, 0, 337, 0, 0, 1, 0, 1, 0, 4096, 0, 0, 4095, 2833, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 3982, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1251, 49, 0, 0, 3349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 0, 1, 0, 0, 1, 0, 4096, 3174, 492, 2, 0, 0, 0, 0, 0, 0, 0, 921, 2363, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8160, 8201, 8196, 8192, 8743, 8474, 7380] avg=8192.0000 max=8743 max_rank=5 min=7380 min_rank=7 imbalance_ratio=1.067261 max_load_reduction=56793 max_load_reduction_pct=0.866592 moved_to_echo=32789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 3945, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 35, 0, 25, 0, 4020, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 555, 1, 0, 0, 3349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 2, 0, 0, 5, 0, 4096, 3171, 492, 3, 0, 0, 0, 0, 0, 0, 0, 919, 3535, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8225, 8176, 8191, 8193, 7986, 8026, 8550] avg=8192.0000 max=8550 max_rank=7 min=7986 min_rank=5 imbalance_ratio=1.043701 max_load_reduction=56986 max_load_reduction_pct=0.869537 moved_to_echo=32755 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 4015, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 5, 0, 27, 0, 4055, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 96, 1, 0, 0, 3349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 644, 0, 2, 1, 0, 4, 0, 4095, 3168, 492, 2, 0, 0, 0, 0, 0, 0, 0, 919, 3574, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8241, 8183, 8192, 8192, 7540, 8408, 8589] avg=8192.0000 max=8589 max_rank=7 min=7540 min_rank=5 imbalance_ratio=1.048462 max_load_reduction=56947 max_load_reduction_pct=0.868942 moved_to_echo=32729 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 4018, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 2, 0, 23, 0, 4065, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1204, 4, 0, 0, 3349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 0, 7, 0, 4096, 3171, 492, 2, 0, 0, 0, 0, 0, 0, 0, 920, 2825, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8306, 8186, 8195, 8192, 8649, 7977, 7841] avg=8192.0000 max=8649 max_rank=5 min=7841 min_rank=7 imbalance_ratio=1.055786 max_load_reduction=56887 max_load_reduction_pct=0.868027 moved_to_echo=32659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 4070, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 2, 0, 33, 0, 3966, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 147, 0, 0, 0, 3349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 690, 0, 0, 1, 0, 4, 0, 4095, 3155, 492, 2, 0, 0, 0, 0, 0, 0, 0, 915, 3520, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8303, 8097, 8192, 8192, 7591, 8439, 8531] avg=8192.0000 max=8531 max_rank=7 min=7591 min_rank=5 imbalance_ratio=1.041382 max_load_reduction=57005 max_load_reduction_pct=0.869827 moved_to_echo=32753 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 15, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 4032, 0, 1, 1, 1, 258, 0, 0, 0, 0, 0, 0, 0, 2, 1, 87, 1, 4078, 2, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 1, 1, 0, 4095, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4091, 3, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 28, 4095, 906, 2, 0, 0, 3332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 1, 0, 8, 1, 4, 0, 4095, 3178, 492, 2, 0, 0, 0, 0, 0, 1, 1, 915, 3176, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8160, 8267, 8196, 8189, 8363, 8093, 8065] avg=8192.0000 max=8363 max_rank=5 min=8065 min_rank=7 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 3967, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 7, 0, 24, 0, 4070, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1280, 0, 0, 0, 3349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 1, 0, 1, 0, 4096, 3174, 492, 3, 0, 0, 0, 0, 0, 0, 0, 921, 2833, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8084, 8197, 8193, 8194, 8725, 8104, 7850] avg=8192.0000 max=8725 max_rank=5 min=7850 min_rank=7 imbalance_ratio=1.065063 max_load_reduction=56811 max_load_reduction_pct=0.866867 moved_to_echo=32873 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 3841, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 60, 0, 16, 0, 4057, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 493, 1, 0, 0, 3349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 2, 1, 0, 1, 0, 4094, 3171, 492, 2, 0, 0, 0, 0, 0, 0, 0, 919, 3715, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8057, 8229, 8181, 8192, 7939, 8015, 8730] avg=8192.0000 max=8730 max_rank=7 min=7939 min_rank=5 imbalance_ratio=1.065674 max_load_reduction=56806 max_load_reduction_pct=0.866791 moved_to_echo=32876 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 7424, 25600, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 6016, 128, 0, 0, 26880, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 256, 128, 32512, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[3456, 128, 128, 128, 128, 128, 0, 32768, 25472, 3968, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 31744, 31872, 0, 128, 128, 128, 2048, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 4071, 270, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3964, 0, 0, 0, 4089, 0, 0, 0, 0, 8, 0, 0, 0, 3980, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 4095, 723, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3892, 0, 0, 0, 4095, 0, 0, 0, 0, 11, 0, 0, 0, 3554, 0, 0, 4096, 4094, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 4095, 37, 0, 0, 0, 0, 4065, 723, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 3866, 0, 0, 0, 4079, 0, 0, 0, 0, 17, 0, 0, 0, 3608, 0, 0, 4096, 4083, 0, 2, 0, 0, 0, 0, 4096, 2, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 4095, 122, 0, 0, 0, 0, 4078, 843, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 4095, 0, 0, 3995, 0, 0, 0, 4082, 0, 0, 0, 0, 28, 0, 0, 0, 3240, 0, 0, 4096, 4091, 0, 1, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 3867, 1, 1, 7, 1, 0, 4095, 0, 1, 0, 1, 0, 4038, 796, 1, 1, 1, 1, 0, 1, 4095, 0, 1, 1, 0, 1, 0, 4094, 4088, 0, 1, 3865, 0, 1, 0, 4087, 0, 0, 0, 1, 217, 0, 1, 1, 3600, 1, 1, 4095, 4086, 1, 10, 0, 0, 0, 1, 4094, 0, 4095, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4090, 4096, 0, 0, 0, 0, 0, 4093, 28, 0, 0, 0, 0, 4095, 489, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 4016, 0, 0, 0, 4092, 0, 0, 0, 0, 4, 0, 0, 0, 3684, 0, 0, 4096, 4069, 0, 6, 0, 0, 0, 0, 4096, 5, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 49, 0, 0, 0, 0, 4086, 908, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3933, 0, 0, 0, 3950, 0, 0, 0, 0, 15, 0, 0, 0, 3458, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 32, 0, 0, 0, 0, 4089, 1755, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3691, 0, 0, 0, 4043, 0, 0, 0, 0, 25, 0, 0, 0, 2757, 0, 0, 4096, 4088, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[30, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 15, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 723, 0, 0, 0, 0, 0, 0, 4096, 3282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 0, 4095, 0, 214, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 3554, 4096, 570, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8190, 8101, 8192, 8210, 8231, 8198, 8192] avg=8192.0000 max=8231 max_rank=5 min=8101 min_rank=2 imbalance_ratio=1.004761 max_load_reduction=57305 max_load_reduction_pct=0.874405 moved_to_echo=32831 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[30, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 8, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 270, 0, 0, 0, 0, 0, 0, 4096, 3282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 0, 0, 4089, 0, 214, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 3980, 4096, 570, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8159, 7648, 8192, 8276, 8654, 8194, 8192] avg=8192.0000 max=8654 max_rank=5 min=7648 min_rank=2 imbalance_ratio=1.056396 max_load_reduction=56882 max_load_reduction_pct=0.867950 moved_to_echo=33316 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[30, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 37, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 723, 0, 0, 0, 0, 0, 0, 4094, 3282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 0, 0, 0, 4079, 0, 214, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 3608, 4096, 570, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 2, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8181, 8099, 8193, 8168, 8291, 8188, 8195] avg=8192.0000 max=8291 max_rank=5 min=8099 min_rank=2 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32842 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[30, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 122, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 843, 0, 0, 0, 0, 0, 0, 4096, 3282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 0, 4082, 0, 214, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 3240, 4096, 570, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8279, 8221, 8192, 8300, 7934, 8195, 8194] avg=8192.0000 max=8300 max_rank=4 min=7934 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=57236 max_load_reduction_pct=0.873352 moved_to_echo=32623 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[30, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 49, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 908, 0, 0, 0, 0, 0, 0, 4095, 3282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 0, 0, 3950, 0, 214, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 3458, 4096, 570, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8215, 8285, 8192, 8106, 8139, 8185, 8192] avg=8192.0000 max=8285 max_rank=2 min=8106 min_rank=4 imbalance_ratio=1.011353 max_load_reduction=57251 max_load_reduction_pct=0.873581 moved_to_echo=32622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[30, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 32, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 1755, 0, 0, 0, 0, 0, 0, 4096, 3282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3691, 0, 0, 0, 4043, 0, 214, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 2757, 4096, 570, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8201, 9133, 8192, 7957, 7448, 8192, 8192] avg=8192.0000 max=9133 max_rank=2 min=7448 min_rank=5 imbalance_ratio=1.114868 max_load_reduction=56403 max_load_reduction_pct=0.860641 moved_to_echo=31789 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[20, 0, 4095, 3867, 1, 1, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 1, 0, 1, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 796, 1, 1, 1, 1, 0, 1, 4095, 3286, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 4094, 4088, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3865, 0, 1, 0, 4087, 0, 219, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1, 217, 0, 1, 1, 3600, 4095, 570, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4086, 1, 10, 0, 0, 9, 3, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7992, 8111, 8182, 8190, 8188, 8485, 8206, 8182] avg=8192.0000 max=8485 max_rank=5 min=7992 min_rank=0 imbalance_ratio=1.035767 max_load_reduction=57051 max_load_reduction_pct=0.870529 moved_to_echo=33061 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[30, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 28, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 0, 4096, 3282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 4092, 0, 214, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3684, 4096, 570, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 0, 6, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8200, 7867, 8193, 8331, 8354, 8178, 8197] avg=8192.0000 max=8354 max_rank=5 min=7867 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=57182 max_load_reduction_pct=0.872528 moved_to_echo=33060 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 31232, 0, 128, 0, 32640, 0, 1792, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 128, 384, 0, 128, 128, 27904, 32768, 4608, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32640, 384, 128, 0, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 0, 32768, 32640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[6528, 128, 128, 128, 128, 0, 128, 32768, 26368, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 32768, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 17, 4096, 0, 4096, 4096, 2685, 0, 0, 0, 0, 0, 0, 4096, 3981, 0, 0, 0, 0, 17, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1422, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 706, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3, 4096, 0, 4096, 4096, 3276, 0, 0, 0, 0, 0, 0, 4096, 3936, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 0, 0, 270, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 207, 4096, 0, 4096, 4096, 3189, 0, 0, 0, 0, 0, 0, 4092, 4092, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 4096, 0, 0, 697, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 55, 4096, 0, 4096, 4096, 3674, 0, 0, 0, 0, 0, 0, 4096, 4044, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 4096, 0, 0, 354, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 1, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 5, 4096, 0, 4096, 4096, 3406, 0, 0, 0, 0, 0, 0, 4095, 4068, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 4096, 0, 0, 705, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4094, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 2, 0, 0, 0, 4096, 5, 0, 4096, 4096, 0, 34, 4096, 0, 4096, 4096, 3326, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 0, 0, 20, 4096, 4096, 0, 0, 0, 4096, 0, 0, 590, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4085, 0, 1, 1, 1, 0, 4095, 4095, 5, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1125, 53, 1, 38, 0, 4095, 0, 1, 4093, 4095, 0, 255, 4095, 7, 4095, 4095, 2581, 0, 0, 0, 0, 1, 1, 4094, 4024, 3, 0, 2, 0, 3, 4093, 4095, 0, 13, 1, 4088, 0, 0, 171, 1, 1, 4026, 1, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 13, 4096, 0, 4096, 4096, 3009, 0, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1012, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 4096, 3805, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4022, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 4096, 2685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3981, 0, 0, 0, 0, 17, 54, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1422, 0, 0, 4096, 0, 0, 0, 0, 3143, 288, 20, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8191, 7972, 8135, 7734, 8148, 8192, 8969] avg=8192.0000 max=8969 max_rank=7 min=7734 min_rank=4 imbalance_ratio=1.094849 max_load_reduction=56567 max_load_reduction_pct=0.863144 moved_to_echo=33043 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 706, 0, 0, 0, 0, 4096, 3805, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4022, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 4096, 3276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3936, 0, 0, 0, 0, 1, 54, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 0, 0, 4096, 0, 0, 0, 0, 3143, 290, 20, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8191, 8608, 8121, 8325, 8087, 8192, 7819] avg=8192.0000 max=8608 max_rank=2 min=7819 min_rank=7 imbalance_ratio=1.050781 max_load_reduction=56928 max_load_reduction_pct=0.868652 moved_to_echo=32423 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 4096, 3804, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4022, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 4096, 3406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4068, 0, 0, 0, 0, 4, 54, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 705, 0, 0, 4096, 0, 0, 0, 0, 3143, 288, 20, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8192, 7906, 8124, 8455, 8221, 8192, 8252] avg=8192.0000 max=8455 max_rank=4 min=7906 min_rank=2 imbalance_ratio=1.032104 max_load_reduction=57081 max_load_reduction_pct=0.870987 moved_to_echo=33120 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 4096, 3805, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4022, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 4096, 3189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4092, 0, 0, 0, 0, 3, 54, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 0, 0, 4096, 0, 0, 0, 0, 3143, 288, 20, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8191, 7910, 8326, 8238, 8241, 8192, 8244] avg=8192.0000 max=8326 max_rank=3 min=7910 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=57210 max_load_reduction_pct=0.872955 moved_to_echo=32915 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 4096, 3805, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4022, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 4096, 3674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4044, 0, 0, 0, 0, 4, 54, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 4096, 0, 0, 0, 0, 3143, 288, 20, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8191, 7963, 8173, 8723, 8198, 8192, 7901] avg=8192.0000 max=8723 max_rank=4 min=7901 min_rank=7 imbalance_ratio=1.064819 max_load_reduction=56813 max_load_reduction_pct=0.866898 moved_to_echo=33014 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 4096, 3805, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4022, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 4096, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 4, 54, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 0, 0, 4096, 0, 0, 0, 0, 3143, 288, 20, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8191, 8002, 8131, 8058, 8208, 8192, 8559] avg=8192.0000 max=8559 max_rank=7 min=8002 min_rank=2 imbalance_ratio=1.044800 max_load_reduction=56977 max_load_reduction_pct=0.869400 moved_to_echo=33017 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 1, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 1, 0, 0, 0, 1, 1, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 53, 1, 38, 0, 4095, 3807, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 4013, 0, 255, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 951, 4095, 2581, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4024, 3, 0, 2, 0, 3, 61, 3, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 13, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 1, 1, 4026, 1, 1, 1, 0, 3144, 288, 21, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8198, 9120, 8369, 7628, 8191, 8192, 7655] avg=8192.0000 max=9120 max_rank=2 min=7628 min_rank=4 imbalance_ratio=1.113281 max_load_reduction=56416 max_load_reduction_pct=0.860840 moved_to_echo=31666 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 150, 2, 0, 0, 0, 4096, 3806, 1, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 4022, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 4096, 3326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 0, 0, 20, 54, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 4096, 0, 0, 0, 0, 3143, 288, 20, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8187, 8191, 8055, 8157, 8375, 8242, 8192, 8137] avg=8192.0000 max=8375 max_rank=4 min=8055 min_rank=2 imbalance_ratio=1.022339 max_load_reduction=57161 max_load_reduction_pct=0.872208 moved_to_echo=32946 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[7680, 32768, 25216, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[5248, 128, 128, 32768, 128, 128, 128, 0, 25216, 2432, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 32384, 128, 0, 128, 0, 128, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 32256, 0, 640, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 2304, 128, 128, 128, 0, 32768, 30464, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4080, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4089, 4096, 4096, 0, 0, 0, 24, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 913, 0, 4091, 0, 3183, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4093, 4096, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 2056, 0, 4095, 0, 2095, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 86, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 385, 0, 4096, 0, 3627, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4093, 4096, 4096, 0, 0, 0, 65, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 672, 0, 4084, 0, 3371, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 28, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2198, 0, 4096, 0, 1885, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 4091, 0, 4091, 4096, 4096, 0, 0, 0, 124, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 1029, 0, 4072, 0, 2962, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4092, 0, 65, 1, 3742, 1, 0, 0, 7, 0, 0, 0, 0, 4092, 104, 4095, 4041, 4095, 0, 0, 1, 1016, 0, 1, 4095, 0, 0, 1, 1, 4095, 4095, 0, 1, 0, 1, 1, 0, 0, 0, 0, 4095, 4084, 0, 4095, 1407, 1, 4082, 17, 1913, 0, 1, 1, 1, 1, 0, 0, 1, 4095, 1, 4094, 0, 3, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 40, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 461, 0, 4090, 0, 3612, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4080, 0, 0, 46, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 2833, 1132, 117, 0, 0, 0, 0, 0, 3926, 4096, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2964, 0, 0, 913, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 0, 0, 0, 0, 0, 4096, 1263, 1, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8180, 8046, 8192, 8191, 7968, 8543, 8191] avg=8192.0000 max=8543 max_rank=6 min=7968 min_rank=5 imbalance_ratio=1.042847 max_load_reduction=56993 max_load_reduction_pct=0.869644 moved_to_echo=32893 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 0, 4039, 0, 0, 46, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2833, 1132, 117, 0, 0, 0, 0, 0, 3930, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2964, 1, 0, 2056, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2095, 0, 0, 0, 0, 0, 0, 4096, 1263, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8177, 8028, 8192, 8192, 9116, 7454, 8192] avg=8192.0000 max=9116 max_rank=5 min=7454 min_rank=6 imbalance_ratio=1.112793 max_load_reduction=56420 max_load_reduction_pct=0.860901 moved_to_echo=32954 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4094, 0, 0, 46, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2834, 1132, 117, 0, 0, 0, 0, 0, 3933, 4096, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2964, 0, 0, 385, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3627, 0, 0, 0, 0, 0, 0, 4096, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8179, 8115, 8192, 8192, 7445, 8985, 8189] avg=8192.0000 max=8985 max_rank=6 min=7445 min_rank=5 imbalance_ratio=1.096802 max_load_reduction=56551 max_load_reduction_pct=0.862900 moved_to_echo=32811 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4081, 0, 0, 46, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2834, 1132, 117, 0, 0, 0, 0, 0, 3933, 4096, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2964, 0, 0, 2198, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1885, 0, 0, 0, 0, 0, 0, 4096, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8179, 8057, 8192, 8192, 9258, 7243, 8189] avg=8192.0000 max=9258 max_rank=5 min=7243 min_rank=6 imbalance_ratio=1.130127 max_load_reduction=56278 max_load_reduction_pct=0.858734 moved_to_echo=32882 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4094, 0, 0, 46, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2834, 1132, 117, 0, 0, 0, 0, 0, 3930, 4096, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2964, 1, 0, 672, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3371, 0, 0, 0, 0, 0, 0, 4096, 1262, 0, 0, 0, 0, 0, 0, 5, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8178, 8091, 8192, 8192, 7721, 8729, 8194] avg=8192.0000 max=8729 max_rank=6 min=7721 min_rank=5 imbalance_ratio=1.065552 max_load_reduction=56807 max_load_reduction_pct=0.866806 moved_to_echo=32836 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4082, 0, 0, 46, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2833, 1132, 117, 0, 0, 0, 0, 0, 3933, 4096, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2964, 0, 0, 461, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 0, 0, 0, 0, 0, 0, 4096, 1263, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8227, 8178, 8069, 8192, 8192, 7515, 8971, 8192] avg=8192.0000 max=8971 max_rank=6 min=7515 min_rank=5 imbalance_ratio=1.095093 max_load_reduction=56565 max_load_reduction_pct=0.863113 moved_to_echo=32870 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 65, 1, 3742, 1, 0, 47, 7, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4092, 104, 2833, 1130, 124, 0, 0, 0, 0, 0, 3924, 4041, 0, 0, 0, 1, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2954, 0, 0, 1407, 1, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1913, 0, 1, 1, 1, 1, 0, 4095, 1262, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 1, 4094, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7956, 8290, 8982, 8192, 8193, 8444, 7291, 8188] avg=8192.0000 max=8982 max_rank=2 min=7291 min_rank=6 imbalance_ratio=1.096436 max_load_reduction=56554 max_load_reduction_pct=0.862946 moved_to_echo=32116 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4094, 0, 0, 46, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4091, 0, 2834, 1132, 117, 0, 0, 0, 0, 0, 3928, 4096, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2964, 4, 0, 1029, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2962, 0, 0, 0, 0, 0, 0, 4096, 1262, 0, 0, 0, 0, 0, 0, 12, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8239, 8175, 8148, 8192, 8192, 8069, 8320, 8201] avg=8192.0000 max=8320 max_rank=6 min=8069 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32782 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 23808, 128, 0, 9216, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 22656, 0, 128, 128, 128, 128, 0, 32768, 10112, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 32768, 128, 128, 128, 32384, 128, 0, 384, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[31488, 32768, 0, 0, 0, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 32768, 128, 22784, 9088, 1024, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 4077, 0, 0, 0, 0, 0, 0, 42, 0, 4084, 0, 0, 4096, 0, 0, 0, 4096, 4031, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 11, 4096, 0, 0, 4096, 4096, 0, 0, 236, 0, 0, 0, 4096, 4096, 0, 0, 3900, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 12, 0, 4083, 0, 0, 4096, 0, 0, 0, 4096, 4064, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 5, 4096, 0, 0, 4096, 4096, 0, 0, 521, 0, 0, 0, 4096, 4096, 0, 0, 3606, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4073, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4, 4096, 0, 0, 4096, 4096, 0, 0, 87, 0, 0, 0, 4096, 4096, 0, 0, 4057, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 1, 72, 1, 0, 1, 0, 4026, 0, 1, 0, 0, 124, 0, 3528, 4, 0, 1, 27, 1, 1, 175, 1, 4076, 1, 14, 4090, 7, 0, 1, 4095, 3910, 0, 0, 4088, 0, 1, 26, 20, 3, 4095, 4083, 4095, 0, 0, 4093, 0, 0, 4082, 4094, 0, 2, 1706, 1, 0, 1, 4086, 4095, 0, 0, 2806, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 9, 0, 4077, 0, 1, 4096, 0, 0, 0, 4096, 4083, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 61, 4096, 0, 1, 4096, 4096, 2, 0, 93, 0, 0, 0, 4095, 4096, 0, 0, 4034, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 3, 0, 4085, 0, 3, 4096, 0, 0, 0, 4095, 4073, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 14, 4096, 0, 1, 4096, 4096, 3, 0, 168, 0, 0, 0, 4095, 4096, 0, 0, 3988, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 20, 0, 4074, 0, 4, 4096, 0, 0, 0, 4096, 4084, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4093, 4096, 0, 20, 4096, 0, 5, 4093, 4096, 11, 0, 215, 0, 0, 0, 4096, 4096, 0, 0, 3862, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 12, 0, 4088, 0, 0, 4096, 0, 0, 0, 4096, 4088, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 4096, 0, 3, 4096, 0, 2, 4096, 4096, 0, 0, 65, 0, 0, 0, 4096, 4096, 0, 0, 4035, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 4077, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 4085, 3784, 274, 10, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 4096, 0, 0, 0, 0, 27, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3710, 0, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 312, 0, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8261, 8195, 8191, 8166, 8191, 8042, 8308] avg=8192.0000 max=8308 max_rank=7 min=8042 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=57228 max_load_reduction_pct=0.873230 moved_to_echo=32707 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4085, 3784, 274, 9, 0, 0, 0, 0, 4083, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4096, 0, 0, 0, 0, 27, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3710, 0, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 312, 0, 0, 3606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8274, 8164, 8190, 8199, 8185, 8327, 8014] avg=8192.0000 max=8327 max_rank=6 min=8014 min_rank=7 imbalance_ratio=1.016479 max_load_reduction=57209 max_load_reduction_pct=0.872940 moved_to_echo=32725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4049, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4084, 3784, 274, 9, 0, 0, 0, 0, 4085, 0, 3, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4096, 0, 0, 0, 0, 27, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 14, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3710, 3, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 312, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8230, 8154, 8195, 8208, 8193, 7978, 8395] avg=8192.0000 max=8395 max_rank=7 min=7978 min_rank=6 imbalance_ratio=1.024780 max_load_reduction=57141 max_load_reduction_pct=0.871902 moved_to_echo=32774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4025, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4085, 3784, 274, 9, 0, 0, 0, 0, 4077, 0, 1, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 4096, 0, 0, 0, 0, 27, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 61, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3710, 2, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 312, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8205, 8161, 8185, 8218, 8241, 7902, 8441] avg=8192.0000 max=8441 max_rank=7 min=7902 min_rank=6 imbalance_ratio=1.030396 max_load_reduction=57095 max_load_reduction_pct=0.871201 moved_to_echo=32802 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4067, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4085, 3784, 274, 9, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4096, 0, 0, 0, 0, 27, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3710, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 312, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8248, 8153, 8202, 8208, 8184, 7893, 8465] avg=8192.0000 max=8465 max_rank=7 min=7893 min_rank=6 imbalance_ratio=1.033325 max_load_reduction=57071 max_load_reduction_pct=0.870834 moved_to_echo=32750 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4085, 3784, 274, 9, 0, 0, 0, 0, 4074, 0, 4, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 0, 0, 0, 27, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 20, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4093, 3710, 11, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 312, 0, 0, 3862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8276, 8172, 8185, 8219, 8197, 8034, 8270] avg=8192.0000 max=8276 max_rank=1 min=8034 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=57260 max_load_reduction_pct=0.873718 moved_to_echo=32720 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 72, 1, 0, 1, 0, 4095, 4086, 0, 0, 0, 0, 0, 0, 4026, 0, 1, 0, 0, 124, 0, 3528, 92, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 27, 1, 1, 175, 1, 4091, 3788, 274, 9, 0, 0, 0, 0, 4076, 1, 14, 4090, 7, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 0, 0, 4088, 0, 1, 26, 20, 32, 17, 0, 0, 0, 0, 0, 0, 3, 0, 4083, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 3696, 0, 2, 1706, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 307, 0, 0, 2806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 7771, 8372, 8193, 8094, 8162, 9487, 7200] avg=8192.0000 max=9487 max_rank=6 min=7200 min_rank=7 imbalance_ratio=1.158081 max_load_reduction=56049 max_load_reduction_pct=0.855240 moved_to_echo=32943 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4089, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4085, 3784, 274, 10, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 1, 0, 27, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 3710, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 312, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8271, 8165, 8195, 8224, 8183, 7873, 8443] avg=8192.0000 max=8443 max_rank=7 min=7873 min_rank=6 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=32723 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 128, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 32768, 2560, 0, 0, 30336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 0, 0, 128, 0, 32128, 768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32512, 0, 0, 32768, 0, 128, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 29696, 128, 128, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 128, 384, 128, 32768, 30336, 2304, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 167, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 43, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 51, 0, 4001, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 537, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 21, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 44, 0, 3804, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 544, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 1, 0, 33, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3638, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 70, 0, 3910, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 124, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 0, 25, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3858, 0, 0, 0, 0, 4093, 4096, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 129, 0, 4064, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 74, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 1, 4092, 0, 0, 0, 0, 58, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 32, 0, 4027, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 1295, 4095, 4082, 0, 1, 1, 0, 4095, 2, 0, 3997, 1, 4095, 0, 6, 0, 0, 4094, 256, 4095, 0, 1, 0, 11, 0, 1, 279, 1, 0, 0, 1, 1, 2, 0, 1, 1, 0, 3237, 1, 1, 1, 0, 4086, 4095, 0, 4094, 4094, 0, 1, 4095, 0, 0, 1, 761, 0, 2525, 4035, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 710, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 16, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 53, 0, 3502, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 761, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 88, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3456, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 111, 0, 3778, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3573, 0, 0, 0, 167, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 3, 0, 4094, 3725, 295, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 0, 0, 0, 4096, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 4001, 4096, 0, 0, 0, 228, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7836, 8203, 8163, 8161, 8191, 8397, 8193, 8392] avg=8192.0000 max=8397 max_rank=5 min=7836 min_rank=0 imbalance_ratio=1.025024 max_load_reduction=57139 max_load_reduction_pct=0.871872 moved_to_echo=33173 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3573, 0, 0, 0, 537, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8, 0, 4096, 3725, 295, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 3779, 0, 0, 0, 0, 4096, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 3804, 4096, 0, 0, 0, 228, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8203, 8164, 8146, 8191, 8246, 8192, 8188] avg=8192.0000 max=8246 max_rank=5 min=8146 min_rank=3 imbalance_ratio=1.006592 max_load_reduction=57290 max_load_reduction_pct=0.874176 moved_to_echo=32817 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3573, 0, 0, 0, 544, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 33, 0, 1, 0, 4094, 3725, 295, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 3638, 0, 0, 0, 0, 4096, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 3910, 4096, 1, 0, 0, 228, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8213, 8203, 8160, 8150, 8191, 8105, 8193, 8321] avg=8192.0000 max=8321 max_rank=7 min=8105 min_rank=5 imbalance_ratio=1.015747 max_load_reduction=57215 max_load_reduction_pct=0.873032 moved_to_echo=32810 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3573, 0, 0, 0, 124, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 5, 0, 4095, 3725, 295, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 3858, 0, 0, 0, 0, 4093, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 4064, 4096, 0, 0, 0, 228, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7793, 8203, 8156, 8146, 8191, 8322, 8192, 8533] avg=8192.0000 max=8533 max_rank=7 min=7793 min_rank=0 imbalance_ratio=1.041626 max_load_reduction=57003 max_load_reduction_pct=0.869797 moved_to_echo=33238 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3573, 0, 0, 0, 74, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 3, 0, 4094, 3725, 295, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 4096, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 4027, 4096, 0, 0, 0, 228, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7743, 8203, 8162, 8176, 8191, 8468, 8194, 8399] avg=8192.0000 max=8468 max_rank=5 min=7743 min_rank=0 imbalance_ratio=1.033691 max_load_reduction=57068 max_load_reduction_pct=0.870789 moved_to_echo=33252 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3573, 0, 0, 0, 710, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 2, 0, 4094, 3725, 295, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 0, 0, 4096, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 3502, 4096, 0, 0, 0, 228, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8379, 8203, 8160, 8133, 8191, 8382, 8193, 7895] avg=8192.0000 max=8382 max_rank=5 min=7895 min_rank=7 imbalance_ratio=1.023193 max_load_reduction=57154 max_load_reduction_pct=0.872101 moved_to_echo=32661 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3559, 0, 0, 0, 1295, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 2, 0, 3997, 1, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4056, 256, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 1, 279, 1, 4094, 3728, 300, 2, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 1, 1, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 3237, 1, 1, 1, 0, 4086, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 761, 0, 2525, 4035, 0, 1, 0, 235, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8936, 8115, 8413, 8417, 8194, 7693, 8190, 7578] avg=8192.0000 max=8936 max_rank=0 min=7578 min_rank=7 imbalance_ratio=1.090820 max_load_reduction=56600 max_load_reduction_pct=0.863647 moved_to_echo=31655 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3573, 0, 0, 0, 761, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 1, 0, 4096, 3725, 295, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 3456, 0, 0, 0, 0, 4096, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 3778, 4096, 1, 0, 0, 228, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8430, 8203, 8158, 8207, 8196, 7923, 8189, 8230] avg=8192.0000 max=8430 max_rank=0 min=7923 min_rank=5 imbalance_ratio=1.029053 max_load_reduction=57106 max_load_reduction_pct=0.871368 moved_to_echo=32538 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 1280, 0, 29696, 32768, 128, 128, 0, 1920, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 29824, 128, 128, 128, 0, 32768, 3072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65664 observed_loss=-128 tokens_per_expert=[28672, 0, 0, 0, 4224, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 0, 32640, 384, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 384, 128, 384, 128, 32768, 29824, 2432, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65664 slack_slots=65408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[4067, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2381, 0, 0, 0, 38, 4095, 65, 1632, 0, 1, 4096, 0, 0, 0, 0, 4, 0, 4095, 1, 4096, 168, 0, 0, 4096, 4095, 4062, 0, 4070, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2693, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[4086, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1355, 0, 0, 0, 37, 4096, 37, 1413, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 14, 0, 0, 4096, 4096, 4080, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3588, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4058, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1464, 0, 0, 0, 55, 4095, 17, 2053, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 23, 0, 0, 4096, 4096, 4086, 0, 3982, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3509, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[4082, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1721, 0, 0, 0, 34, 4096, 81, 1349, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 38, 0, 0, 4096, 4096, 4084, 0, 4036, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3509, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[4084, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1427, 0, 0, 0, 17, 4096, 25, 1579, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 32, 0, 0, 4096, 4095, 4065, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3603, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[4072, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1444, 0, 0, 0, 36, 4093, 12, 2203, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 4096, 143, 0, 0, 4096, 4094, 4061, 0, 4086, 0, 0, 10, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3823, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[4084, 2026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1309, 0, 0, 0, 40, 4096, 3, 1264, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 17, 0, 0, 4096, 4096, 4086, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3556, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[4017, 2928, 27, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4091, 8, 430, 0, 1, 0, 3, 4095, 619, 351, 0, 0, 4095, 1, 2, 0, 0, 0, 55, 4093, 0, 4095, 937, 1, 7, 4094, 4095, 4095, 0, 3916, 0, 1, 0, 4091, 1, 4095, 0, 1, 1, 0, 4094, 0, 1, 1, 1, 3, 1, 3088, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4086, 1776, 0, 0, 0, 0, 0, 0, 2499, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1355, 0, 0, 0, 37, 4096, 37, 1413, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 67, 4080, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3588, 4096, 0, 495, 178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8363, 8191, 7973, 8191, 8028, 8241, 8192, 8357] avg=8192.0000 max=8363 max_rank=0 min=7973 min_rank=2 imbalance_ratio=1.020874 max_load_reduction=57173 max_load_reduction_pct=0.872391 moved_to_echo=32818 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4084, 1555, 0, 0, 0, 0, 0, 0, 2498, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1427, 0, 0, 0, 17, 4096, 25, 1579, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 32, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 68, 4065, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 4096, 0, 495, 178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8139, 8191, 8178, 8191, 8046, 8227, 8192, 8372] avg=8192.0000 max=8372 max_rank=7 min=8046 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=57164 max_load_reduction_pct=0.872253 moved_to_echo=32837 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4067, 1300, 0, 0, 0, 0, 0, 0, 2498, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2381, 0, 0, 0, 38, 4095, 65, 1632, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 168, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 68, 4062, 0, 4070, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2693, 4096, 0, 495, 178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7867, 8191, 9245, 8196, 8182, 8201, 8192, 7462] avg=8192.0000 max=9245 max_rank=2 min=7462 min_rank=7 imbalance_ratio=1.128540 max_load_reduction=56291 max_load_reduction_pct=0.858932 moved_to_echo=32037 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4058, 1232, 0, 0, 0, 0, 0, 0, 2499, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1464, 0, 0, 0, 55, 4095, 17, 2053, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 67, 4086, 0, 3982, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3509, 4096, 0, 495, 178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7791, 8191, 8719, 8192, 8037, 8136, 8192, 8278] avg=8192.0000 max=8719 max_rank=2 min=7791 min_rank=0 imbalance_ratio=1.064331 max_load_reduction=56817 max_load_reduction_pct=0.866959 moved_to_echo=32643 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4082, 1546, 0, 0, 0, 0, 0, 0, 2499, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1721, 0, 0, 0, 34, 4096, 81, 1349, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 38, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 67, 4084, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3509, 4096, 0, 495, 178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8129, 8191, 8316, 8191, 8052, 8187, 8192, 8278] avg=8192.0000 max=8316 max_rank=2 min=8052 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=57220 max_load_reduction_pct=0.873108 moved_to_echo=32709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4084, 2026, 0, 0, 0, 0, 0, 0, 2499, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1309, 0, 0, 0, 40, 4096, 3, 1264, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 67, 4086, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3556, 4096, 0, 495, 178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8611, 8191, 7747, 8191, 8031, 8248, 8192, 8325] avg=8192.0000 max=8611 max_rank=0 min=7747 min_rank=2 imbalance_ratio=1.051147 max_load_reduction=56925 max_load_reduction_pct=0.868607 moved_to_echo=32796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4017, 2928, 27, 0, 0, 0, 0, 0, 2501, 2, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4091, 8, 4095, 0, 0, 0, 0, 0, 0, 0, 430, 0, 1, 0, 3, 4095, 619, 351, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 2, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 55, 4093, 0, 0, 937, 1, 7, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 60, 4095, 0, 3916, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 3, 1, 3088, 4095, 1, 497, 184, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9475, 8199, 6536, 8187, 9003, 8072, 8192, 7872] avg=8192.0000 max=9475 max_rank=0 min=6536 min_rank=2 imbalance_ratio=1.156616 max_load_reduction=56061 max_load_reduction_pct=0.855423 moved_to_echo=33139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4072, 589, 0, 0, 0, 0, 0, 0, 2498, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1444, 0, 0, 0, 36, 4093, 12, 2203, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 143, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 67, 4061, 0, 4086, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 4096, 0, 495, 178, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7161, 8191, 8822, 8197, 8157, 8224, 8192, 8592] avg=8192.0000 max=8822 max_rank=2 min=7161 min_rank=0 imbalance_ratio=1.076904 max_load_reduction=56714 max_load_reduction_pct=0.865387 moved_to_echo=33165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 13056, 128, 0, 0, 0, 0, 0, 20096, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 1408, 128, 128, 31360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[11648, 0, 128, 0, 384, 32768, 896, 11904, 8320, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 27392, 32768, 128, 3968, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[640, 32640, 0, 32384, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4087, 45, 0, 3692, 0, 0, 4096, 0, 4096, 0, 364, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4094, 4, 0, 3768, 0, 0, 4096, 0, 4096, 0, 313, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4021, 5, 0, 3767, 0, 0, 4096, 0, 4095, 0, 389, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4077, 42, 0, 3723, 0, 0, 4096, 0, 4092, 0, 339, 4096, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4093, 117, 1, 2215, 1, 10, 4003, 0, 4095, 1, 1795, 3832, 2, 0, 2, 0, 4095, 0, 1, 0, 1, 4094, 0, 2, 0, 0, 3, 4094, 0, 4092, 0, 4085, 0, 1, 1, 4095, 0, 0, 0, 0, 1, 10, 1, 1, 0, 323, 1, 1, 2, 1, 4085, 0, 0, 2, 4095, 0, 0, 4090, 0, 4095, 4094, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4087, 24, 0, 3513, 0, 0, 4096, 0, 4096, 0, 566, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4072, 16, 0, 3858, 0, 0, 4096, 0, 4095, 0, 244, 4096, 0, 0, 1, 0, 4095, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4093, 37, 0, 3873, 0, 0, 4096, 0, 4094, 0, 187, 4096, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4094, 4, 0, 3768, 0, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 0, 313, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3569, 491, 43, 1, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4095, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8393, 8014, 8191, 8195, 8200, 8158, 8192, 8193] avg=8192.0000 max=8393 max_rank=0 min=8014 min_rank=1 imbalance_ratio=1.024536 max_load_reduction=57143 max_load_reduction_pct=0.871933 moved_to_echo=32743 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4087, 45, 0, 3692, 0, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 0, 364, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3569, 491, 44, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8351, 8065, 8191, 8192, 8201, 8151, 8193, 8192] avg=8192.0000 max=8351 max_rank=0 min=8065 min_rank=1 imbalance_ratio=1.019409 max_load_reduction=57185 max_load_reduction_pct=0.872574 moved_to_echo=32737 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4021, 5, 0, 3767, 0, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 0, 389, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3569, 491, 43, 1, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4095, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8089, 8191, 8194, 8201, 8156, 8192, 8193] avg=8192.0000 max=8320 max_rank=0 min=8089 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32742 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4087, 24, 0, 3513, 0, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 0, 566, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3569, 491, 43, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4095, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 8267, 8191, 8195, 8200, 8147, 8192, 8193] avg=8192.0000 max=8267 max_rank=1 min=8147 min_rank=5 imbalance_ratio=1.009155 max_load_reduction=57269 max_load_reduction_pct=0.873856 moved_to_echo=32732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4072, 16, 0, 3858, 0, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 0, 244, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3569, 491, 43, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8473, 7945, 8191, 8193, 8200, 8148, 8193, 8193] avg=8192.0000 max=8473 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.034302 max_load_reduction=57063 max_load_reduction_pct=0.870712 moved_to_echo=32734 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4093, 37, 0, 3873, 0, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 0, 187, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3569, 491, 43, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8530, 7886, 8194, 8195, 8200, 8146, 8192, 8193] avg=8192.0000 max=8530 max_rank=0 min=7886 min_rank=1 imbalance_ratio=1.041260 max_load_reduction=57006 max_load_reduction_pct=0.869843 moved_to_echo=32731 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4077, 42, 0, 3723, 0, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3602, 0, 339, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3569, 490, 46, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8369, 8037, 8203, 8192, 8202, 8149, 8192, 8192] avg=8192.0000 max=8369 max_rank=0 min=8037 min_rank=1 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32735 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4093, 117, 1, 2215, 1, 10, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3601, 1, 1795, 3832, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 4085, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4092, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 3491, 494, 46, 4, 0, 0, 0, 0, 1, 10, 1, 1, 0, 323, 1, 1, 4095, 4048, 0, 0, 0, 0, 0, 0, 2, 1, 4085, 0, 0, 2, 4095, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4094, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6949, 9233, 8184, 8180, 8132, 8481, 8190, 8187] avg=8192.0000 max=9233 max_rank=1 min=6949 min_rank=0 imbalance_ratio=1.127075 max_load_reduction=56303 max_load_reduction_pct=0.859116 moved_to_echo=32990 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32640, 384, 128, 28416, 128, 128, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[28928, 128, 4224, 32512, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 0, 28544, 3968, 384, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 128, 128, 0, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 384, 128, 128, 32768, 32512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True + [2026-08-30 04:59:33] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 732.2 | throughput per GPU (TFLOP/s/GPU): 455.3 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.976653E+00 | loss scale: 1.0 | grad norm: 5.199 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2349, 310, 1013, 540, 2096, 0, 1049, 411, 771, 2841, 1297, 1706, 1856, 1851, 5, 314, 258, 1398, 1051, 625, 0, 961, 1967, 1094, 257, 71, 2312, 868, 1407, 1335, 1609, 1298, 11, 2082, 1894, 497, 462, 1171, 1537, 1732, 418, 0, 1126, 1000, 2, 1465, 911, 640, 2111, 53, 1307, 14, 319, 1469, 8, 1138, 4, 1129, 1342, 876, 1577, 996, 1171, 2154] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2315, 124, 1129, 657, 1526, 2, 1415, 542, 1289, 3214, 1010, 1373, 1627, 1589, 0, 240, 473, 944, 1195, 601, 0, 970, 1745, 1342, 114, 238, 2367, 1388, 2253, 1216, 1936, 992, 2, 2367, 1737, 743, 719, 1241, 1408, 856, 384, 0, 1151, 1785, 0, 1407, 916, 662, 2152, 58, 1382, 6, 271, 1005, 2, 981, 0, 649, 1030, 625, 1579, 1184, 1116, 2292] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2266, 156, 1356, 657, 2155, 1, 1182, 416, 919, 2545, 1201, 813, 1985, 741, 1, 382, 365, 1187, 1683, 923, 0, 1015, 2018, 1305, 119, 67, 2562, 960, 1714, 1048, 1967, 1083, 11, 2646, 1656, 315, 790, 781, 2004, 496, 615, 2, 889, 1119, 0, 1645, 694, 840, 2312, 47, 1162, 4, 191, 1358, 1, 1549, 0, 1041, 1778, 666, 2048, 908, 883, 2293] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2154, 142, 1154, 628, 1520, 0, 1725, 547, 1297, 2402, 1644, 980, 2047, 1058, 0, 506, 351, 1223, 1393, 804, 0, 1027, 1770, 1088, 181, 68, 2275, 1078, 1743, 757, 2322, 1317, 3, 1944, 1966, 432, 488, 619, 1503, 883, 276, 0, 1086, 1371, 0, 1251, 837, 735, 1775, 26, 2344, 3, 434, 1834, 5, 1201, 1, 853, 1574, 993, 1728, 965, 853, 2352] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2714, 112, 1168, 759, 1661, 0, 1425, 685, 1062, 3007, 1389, 1239, 1202, 1606, 1, 190, 702, 1398, 1438, 605, 2, 1056, 1321, 898, 109, 137, 2319, 902, 1758, 747, 1522, 824, 1, 2090, 2009, 230, 816, 1864, 1918, 1082, 246, 0, 1055, 1519, 7, 1788, 1302, 937, 2005, 27, 1507, 6, 224, 1474, 1, 961, 1, 856, 1124, 1095, 1268, 938, 929, 2298] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2673, 139, 1180, 790, 1056, 0, 1086, 433, 1022, 2962, 1508, 736, 1091, 2120, 0, 313, 822, 1317, 1205, 619, 0, 956, 2018, 921, 108, 285, 2581, 997, 1490, 1536, 1545, 847, 1, 1920, 1539, 781, 757, 1093, 1537, 986, 307, 2, 1218, 1418, 0, 1503, 1207, 1177, 2501, 65, 1610, 1, 294, 1532, 1, 623, 1, 883, 1037, 1203, 1626, 1343, 959, 2055] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1978, 146, 1404, 572, 1834, 4, 1188, 867, 826, 2093, 1120, 1531, 1781, 857, 0, 516, 552, 1384, 1526, 1268, 2, 863, 2014, 1205, 140, 36, 2325, 929, 1724, 728, 1525, 1120, 6, 2385, 1936, 201, 950, 1224, 1569, 901, 488, 5, 1058, 1007, 4, 1762, 893, 602, 2346, 89, 787, 87, 446, 1338, 9, 1196, 0, 724, 1970, 815, 1887, 1465, 985, 2343] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2093, 112, 1070, 552, 2264, 0, 1416, 522, 958, 1855, 1120, 1133, 1595, 830, 1, 511, 503, 2035, 1281, 1013, 0, 963, 2207, 1173, 81, 53, 2583, 1166, 1852, 687, 1631, 890, 4, 2884, 1733, 116, 521, 1599, 1816, 687, 415, 0, 1052, 1786, 6, 1281, 1242, 1006, 2673, 67, 895, 6, 231, 1015, 3, 981, 1, 746, 1403, 982, 1743, 888, 798, 2806] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2315, 124, 1129, 657, 1526, 2, 1415, 542, 270, 0, 0, 0, 0, 0, 0, 0, 1289, 1656, 1010, 1373, 1627, 1589, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 473, 944, 1195, 601, 0, 970, 1745, 1342, 192, 0, 0, 0, 0, 0, 0, 0, 114, 238, 1319, 1388, 2253, 1216, 1936, 992, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1514, 1737, 743, 719, 1241, 1408, 856, 0, 0, 0, 0, 0, 0, 0, 0, 384, 0, 1151, 1785, 0, 1407, 916, 662, 1558, 719, 0, 0, 0, 0, 0, 0, 2152, 58, 1382, 6, 271, 1005, 2, 981, 1022, 391, 329, 0, 0, 0, 0, 0, 0, 649, 1030, 625, 1579, 1184, 1116, 1270, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7980, 8784, 7462, 9456, 8220, 8582, 7599, 7453] avg=8192.0000 max=9456 max_rank=3 min=7453 min_rank=7 imbalance_ratio=1.154297 max_load_reduction=56080 max_load_reduction_pct=0.855713 moved_to_echo=31854 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2349, 310, 1013, 540, 2096, 0, 1049, 411, 238, 0, 0, 0, 0, 0, 0, 0, 771, 1464, 1297, 1706, 1856, 1851, 5, 314, 0, 0, 0, 0, 0, 0, 0, 0, 258, 1398, 1051, 625, 0, 961, 1967, 1094, 169, 0, 0, 0, 0, 0, 0, 0, 257, 71, 1289, 868, 1407, 1335, 1609, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1331, 1894, 497, 462, 1171, 1537, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 1126, 1000, 2, 1465, 911, 640, 1377, 702, 0, 0, 0, 0, 0, 0, 2111, 53, 1307, 14, 319, 1469, 8, 1138, 961, 344, 321, 0, 0, 0, 0, 0, 4, 1129, 1342, 876, 1577, 996, 1171, 1193, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8006, 9264, 7523, 8134, 8635, 7641, 8045, 8288] avg=8192.0000 max=9264 max_rank=1 min=7523 min_rank=2 imbalance_ratio=1.130859 max_load_reduction=56272 max_load_reduction_pct=0.858643 moved_to_echo=32609 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2154, 142, 1154, 628, 1520, 0, 1725, 547, 222, 0, 0, 0, 0, 0, 0, 0, 1297, 1238, 1644, 980, 2047, 1058, 0, 506, 0, 0, 0, 0, 0, 0, 0, 0, 351, 1223, 1393, 804, 0, 1027, 1770, 1088, 158, 0, 0, 0, 0, 0, 0, 0, 181, 68, 1268, 1078, 1743, 757, 2322, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1243, 1966, 432, 488, 619, 1503, 883, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 1086, 1371, 0, 1251, 837, 735, 1164, 691, 0, 0, 0, 0, 0, 0, 1775, 26, 2344, 3, 434, 1834, 5, 1201, 1049, 321, 316, 0, 0, 0, 0, 0, 1, 853, 1574, 993, 1728, 965, 853, 1303, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8092, 8770, 7814, 8734, 7137, 7411, 9308, 8270] avg=8192.0000 max=9308 max_rank=6 min=7137 min_rank=4 imbalance_ratio=1.136230 max_load_reduction=56228 max_load_reduction_pct=0.857971 moved_to_echo=32126 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2714, 112, 1168, 759, 1661, 0, 1425, 685, 239, 0, 0, 0, 0, 0, 0, 0, 1062, 1549, 1389, 1239, 1202, 1606, 1, 190, 0, 0, 0, 0, 0, 0, 0, 0, 702, 1398, 1438, 605, 2, 1056, 1321, 898, 170, 0, 0, 0, 0, 0, 0, 0, 109, 137, 1293, 902, 1758, 747, 1522, 824, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1336, 2009, 230, 816, 1864, 1918, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 1055, 1519, 7, 1788, 1302, 937, 1458, 704, 0, 0, 0, 0, 0, 0, 2005, 27, 1507, 6, 224, 1474, 1, 961, 1025, 345, 322, 0, 0, 0, 0, 0, 1, 856, 1124, 1095, 1268, 938, 929, 1273, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8763, 8238, 7590, 7292, 9256, 9016, 7897, 7484] avg=8192.0000 max=9256 max_rank=4 min=7292 min_rank=3 imbalance_ratio=1.129883 max_load_reduction=56280 max_load_reduction_pct=0.858765 moved_to_echo=33653 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1978, 146, 1404, 572, 1834, 4, 1188, 867, 272, 0, 0, 0, 0, 0, 0, 0, 826, 1079, 1120, 1531, 1781, 857, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 552, 1384, 1526, 1268, 2, 863, 2014, 1205, 194, 0, 0, 0, 0, 0, 0, 0, 140, 36, 1296, 929, 1724, 728, 1525, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1525, 1936, 201, 950, 1224, 1569, 901, 0, 0, 0, 0, 0, 0, 0, 0, 488, 5, 1058, 1007, 4, 1762, 893, 602, 1014, 706, 0, 0, 0, 0, 0, 0, 2346, 89, 787, 87, 446, 1338, 9, 1196, 1045, 394, 323, 0, 0, 0, 0, 0, 0, 724, 1970, 815, 1887, 1465, 985, 1298, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8265, 7710, 9008, 7498, 8312, 7539, 8060, 9144] avg=8192.0000 max=9144 max_rank=7 min=7498 min_rank=3 imbalance_ratio=1.116211 max_load_reduction=56392 max_load_reduction_pct=0.860474 moved_to_echo=33055 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2266, 156, 1356, 657, 2155, 1, 1182, 416, 302, 0, 0, 0, 0, 0, 0, 0, 919, 1312, 1201, 813, 1985, 741, 1, 382, 0, 0, 0, 0, 0, 0, 0, 0, 365, 1187, 1683, 923, 0, 1015, 2018, 1305, 215, 0, 0, 0, 0, 0, 0, 0, 119, 67, 1428, 960, 1714, 1048, 1967, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1692, 1656, 315, 790, 781, 2004, 496, 0, 0, 0, 0, 0, 0, 0, 0, 615, 2, 889, 1119, 0, 1645, 694, 840, 1233, 778, 0, 0, 0, 0, 0, 0, 2312, 47, 1162, 4, 191, 1358, 1, 1549, 1023, 437, 356, 0, 0, 0, 0, 0, 0, 1041, 1778, 666, 2048, 908, 883, 1270, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8491, 7354, 8711, 8386, 7745, 7815, 8440, 8594] avg=8192.0000 max=8711 max_rank=2 min=7354 min_rank=1 imbalance_ratio=1.063354 max_load_reduction=56825 max_load_reduction_pct=0.867081 moved_to_echo=32594 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2673, 139, 1180, 790, 1056, 0, 1086, 433, 224, 0, 0, 0, 0, 0, 0, 0, 1022, 1522, 1508, 736, 1091, 2120, 0, 313, 0, 0, 0, 0, 0, 0, 0, 0, 822, 1317, 1205, 619, 0, 956, 2018, 921, 160, 0, 0, 0, 0, 0, 0, 0, 108, 285, 1429, 997, 1490, 1536, 1545, 847, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1217, 1539, 781, 757, 1093, 1537, 986, 0, 0, 0, 0, 0, 0, 0, 0, 307, 2, 1218, 1418, 0, 1503, 1207, 1177, 1440, 788, 0, 0, 0, 0, 0, 0, 2501, 65, 1610, 1, 294, 1532, 1, 623, 919, 319, 364, 0, 0, 0, 0, 0, 1, 883, 1037, 1203, 1626, 1343, 959, 1136, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7581, 8312, 8018, 8237, 7911, 9060, 8229, 8188] avg=8192.0000 max=9060 max_rank=5 min=7581 min_rank=0 imbalance_ratio=1.105957 max_load_reduction=56476 max_load_reduction_pct=0.861755 moved_to_echo=33388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2093, 112, 1070, 552, 2264, 0, 1416, 522, 329, 0, 0, 0, 0, 0, 0, 0, 958, 956, 1120, 1133, 1595, 830, 1, 511, 0, 0, 0, 0, 0, 0, 0, 0, 503, 2035, 1281, 1013, 0, 963, 2207, 1173, 235, 0, 0, 0, 0, 0, 0, 0, 81, 53, 1439, 1166, 1852, 687, 1631, 890, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1844, 1733, 116, 521, 1599, 1816, 687, 0, 0, 0, 0, 0, 0, 0, 0, 415, 0, 1052, 1786, 6, 1281, 1242, 1006, 899, 785, 0, 0, 0, 0, 0, 0, 2673, 67, 895, 6, 231, 1015, 3, 981, 1252, 476, 359, 0, 0, 0, 0, 0, 1, 746, 1403, 982, 1743, 888, 798, 1554, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8358, 7104, 9410, 7799, 8320, 8472, 7958, 8115] avg=8192.0000 max=9410 max_rank=2 min=7104 min_rank=1 imbalance_ratio=1.148682 max_load_reduction=56126 max_load_reduction_pct=0.856415 moved_to_echo=32865 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[18560, 1280, 9600, 5248, 14208, 128, 10496, 4480, 2176, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 11776, 14592, 3328, 5504, 9600, 13312, 7680, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[17920, 512, 11008, 128, 2432, 11136, 128, 8704, 8320, 3072, 2816, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1152, 1024, 10880, 8320, 13952, 8064, 14080, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 6912, 11264, 7296, 13568, 8704, 7808, 10368, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8192, 10880, 10368, 9600, 13184, 10752, 128, 3072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[3200, 128, 8704, 11008, 128, 12160, 8064, 6656, 10240, 5888, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[4096, 11008, 10880, 6528, 128, 7936, 15104, 9088, 1536, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 4084, 0, 520, 1, 2167, 11, 0, 3086, 1412, 0, 13, 1528, 3313, 5, 0, 21, 7, 0, 4075, 4065, 4, 47, 11, 380, 0, 3921, 3954, 7, 4086, 14, 1, 2, 2597, 6, 8, 4072, 39, 4090, 0, 9, 0, 0, 1, 4086, 1157, 0, 1, 4089, 4085, 6, 0, 0, 1, 4087, 315, 0, 6, 0, 2, 138] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1026, 0, 1339, 5, 0, 2900, 1055, 0, 0, 831, 3728, 0, 9, 1, 0, 0, 4030, 4085, 73, 57, 0, 572, 0, 4071, 3997, 6, 4096, 0, 3, 0, 2766, 0, 0, 3990, 53, 4092, 0, 0, 0, 0, 0, 4096, 2052, 0, 0, 4095, 4096, 0, 20, 0, 0, 4095, 138, 0, 0, 0, 0, 63] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 4095, 0, 959, 0, 2513, 1, 0, 3804, 202, 0, 6, 2349, 3626, 0, 8, 3, 1, 0, 4085, 4057, 3, 4, 3, 5, 0, 4069, 4001, 0, 4096, 0, 0, 0, 1704, 0, 2, 3770, 76, 4092, 3, 4, 4, 0, 0, 4092, 1222, 0, 1, 4063, 4093, 3, 8, 3, 0, 4092, 393, 0, 3, 0, 1, 14] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 1181, 0, 2839, 1, 0, 3809, 372, 0, 1, 2812, 3171, 0, 2, 0, 0, 0, 4096, 3989, 63, 0, 0, 3, 0, 4024, 3322, 0, 4096, 0, 0, 0, 927, 17, 0, 3685, 623, 4091, 0, 0, 0, 0, 0, 4096, 1086, 6, 0, 4095, 4094, 0, 18, 0, 0, 4074, 757, 1, 0, 0, 0, 90] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 510, 0, 2741, 0, 0, 3548, 1564, 0, 0, 2883, 2996, 0, 0, 0, 0, 0, 4095, 4094, 2, 0, 0, 7, 0, 4092, 3625, 0, 4096, 0, 1, 0, 1490, 0, 0, 3407, 149, 4073, 0, 0, 0, 0, 0, 4096, 1270, 0, 0, 4089, 4095, 0, 0, 0, 0, 4096, 292, 0, 0, 0, 0, 129] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 6, 1, 0, 4096, 1, 666, 0, 2043, 0, 1, 2252, 974, 0, 0, 1704, 3587, 0, 0, 0, 0, 0, 4095, 4096, 79, 0, 0, 6, 0, 4018, 4056, 1, 4096, 0, 0, 0, 2762, 2, 0, 3303, 24, 4094, 0, 0, 0, 1, 0, 4096, 3008, 4, 0, 4072, 4096, 0, 0, 0, 0, 4095, 140, 0, 0, 0, 0, 61] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 1967, 0, 2008, 2, 2, 3631, 576, 4, 0, 3034, 3509, 0, 0, 0, 0, 0, 4085, 4093, 42, 1, 0, 6, 0, 4081, 3484, 1, 4096, 2, 3, 1, 521, 0, 0, 3812, 238, 4091, 0, 1, 0, 0, 0, 4095, 1163, 1, 0, 4094, 4090, 0, 2, 0, 3, 4094, 533, 2, 0, 0, 0, 71] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 10, 667, 1, 1923, 2, 0, 3069, 692, 0, 0, 1446, 3780, 2, 0, 6, 2, 3, 4093, 4093, 132, 4, 0, 334, 0, 4054, 3917, 5, 4094, 0, 2, 0, 2446, 18, 4, 3927, 38, 4088, 2, 0, 3, 0, 1, 4093, 1956, 2, 0, 4091, 4090, 2, 3, 1, 3, 4079, 236, 1, 2, 1, 2, 22] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 4084, 0, 520, 1, 1557, 1366, 191, 25, 0, 0, 0, 0, 2167, 11, 0, 2895, 1412, 0, 13, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 5, 0, 21, 7, 0, 647, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 4, 47, 11, 380, 0, 3921, 3954, 7, 123, 0, 0, 0, 0, 0, 0, 0, 2529, 14, 1, 2, 2597, 6, 8, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4090, 0, 9, 0, 0, 1, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 0, 1, 4089, 2478, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 315, 0, 6, 0, 2, 138, 3428, 241, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7750, 8026, 8058, 8447, 9229, 8077, 7731, 8218] avg=8192.0000 max=9229 max_rank=4 min=7731 min_rank=6 imbalance_ratio=1.126587 max_load_reduction=56307 max_load_reduction_pct=0.859177 moved_to_echo=33255 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 1181, 0, 1561, 1369, 236, 25, 0, 0, 0, 0, 2839, 1, 0, 3573, 372, 0, 1, 2812, 0, 0, 0, 0, 0, 0, 0, 0, 3171, 0, 2, 0, 0, 0, 651, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 3, 0, 4024, 3322, 0, 124, 0, 0, 0, 0, 0, 0, 0, 2535, 0, 0, 0, 927, 17, 0, 3685, 0, 0, 0, 0, 0, 0, 0, 0, 623, 4091, 0, 0, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 6, 0, 4095, 2484, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 757, 1, 0, 0, 0, 90, 3445, 241, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8467, 9598, 7813, 7536, 7164, 8661, 7689, 8608] avg=8192.0000 max=9598 max_rank=1 min=7164 min_rank=4 imbalance_ratio=1.171631 max_load_reduction=55938 max_load_reduction_pct=0.853546 moved_to_echo=32122 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1026, 0, 1561, 1370, 179, 25, 0, 0, 0, 0, 1339, 5, 0, 2721, 1055, 0, 0, 831, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 9, 1, 0, 0, 640, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 73, 57, 0, 572, 0, 4071, 3997, 6, 124, 0, 0, 0, 0, 0, 0, 0, 2535, 0, 3, 0, 2766, 0, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4092, 0, 0, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 2052, 0, 0, 4095, 2485, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 138, 0, 0, 0, 0, 63, 3390, 241, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8257, 5951, 8463, 8900, 9294, 8092, 8652, 7927] avg=8192.0000 max=9294 max_rank=4 min=5951 min_rank=1 imbalance_ratio=1.134521 max_load_reduction=56242 max_load_reduction_pct=0.858185 moved_to_echo=33965 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 4095, 0, 959, 0, 1561, 1369, 235, 25, 0, 0, 0, 0, 2513, 1, 0, 3569, 202, 0, 6, 2349, 0, 0, 0, 0, 0, 0, 0, 0, 3626, 0, 8, 3, 1, 0, 649, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 3, 5, 0, 4069, 4001, 0, 123, 0, 0, 0, 0, 0, 0, 0, 2535, 0, 0, 0, 1704, 0, 2, 3770, 0, 0, 0, 0, 0, 0, 0, 0, 76, 4092, 3, 4, 4, 0, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 1222, 0, 1, 4063, 2483, 3, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 393, 0, 3, 0, 1, 14, 3436, 241, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8247, 8640, 8344, 8208, 8011, 8123, 7783, 8180] avg=8192.0000 max=8640 max_rank=1 min=7783 min_rank=6 imbalance_ratio=1.054688 max_load_reduction=56896 max_load_reduction_pct=0.868164 moved_to_echo=32097 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 510, 0, 1561, 1369, 220, 25, 0, 0, 0, 0, 2741, 0, 0, 3328, 1564, 0, 0, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 2996, 0, 0, 0, 0, 0, 651, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 0, 4092, 3625, 0, 124, 0, 0, 0, 0, 0, 0, 0, 2535, 0, 1, 0, 1490, 0, 0, 3407, 0, 0, 0, 0, 0, 0, 0, 0, 149, 4073, 0, 0, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 1270, 0, 0, 4089, 2485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 292, 0, 0, 0, 0, 129, 3444, 241, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7781, 10516, 7741, 7850, 7433, 8169, 7844, 8202] avg=8192.0000 max=10516 max_rank=1 min=7433 min_rank=4 imbalance_ratio=1.283691 max_load_reduction=55020 max_load_reduction_pct=0.839539 moved_to_echo=31648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 1, 0, 4096, 1, 666, 0, 1561, 1370, 139, 25, 0, 0, 0, 0, 2043, 0, 1, 2113, 974, 0, 0, 1704, 0, 0, 0, 0, 0, 0, 0, 0, 3587, 0, 0, 0, 0, 0, 651, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 6, 0, 4018, 4056, 1, 124, 0, 0, 0, 0, 0, 0, 0, 2535, 0, 0, 0, 2762, 2, 0, 3303, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4094, 0, 0, 0, 1, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 3008, 4, 0, 4072, 2485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 140, 0, 0, 0, 0, 61, 3444, 241, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7865, 6835, 8334, 8284, 8602, 8066, 9569, 7981] avg=8192.0000 max=9569 max_rank=6 min=6835 min_rank=1 imbalance_ratio=1.168091 max_load_reduction=55967 max_load_reduction_pct=0.853989 moved_to_echo=34218 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4096, 0, 1967, 0, 1563, 1371, 229, 27, 0, 0, 0, 0, 2008, 2, 2, 3402, 576, 4, 0, 3034, 0, 0, 0, 0, 0, 0, 0, 0, 3509, 0, 0, 0, 0, 0, 645, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1, 0, 6, 0, 4081, 3484, 1, 127, 0, 0, 0, 0, 0, 0, 0, 2533, 2, 3, 1, 521, 0, 0, 3812, 0, 0, 0, 0, 0, 0, 0, 0, 238, 4091, 0, 1, 0, 0, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 1, 0, 4094, 2472, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 533, 2, 0, 0, 0, 71, 3440, 247, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9254, 9028, 8247, 7742, 6872, 8271, 7732, 8390] avg=8192.0000 max=9254 max_rank=0 min=6872 min_rank=4 imbalance_ratio=1.129639 max_load_reduction=56282 max_load_reduction_pct=0.858795 moved_to_echo=31265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 10, 667, 1, 1560, 1368, 190, 25, 0, 0, 0, 0, 1923, 2, 0, 2879, 692, 0, 0, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 2, 0, 6, 2, 3, 650, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 132, 4, 0, 334, 0, 4054, 3917, 5, 123, 0, 0, 0, 0, 0, 0, 0, 2534, 0, 2, 0, 2446, 18, 4, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4088, 2, 0, 3, 0, 1, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 1956, 2, 0, 4091, 2481, 2, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4079, 236, 1, 2, 1, 2, 22, 3443, 241, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7915, 6942, 8536, 8569, 8931, 8077, 8536, 8030] avg=8192.0000 max=8931 max_rank=4 min=6942 min_rank=1 imbalance_ratio=1.090210 max_load_reduction=56605 max_load_reduction_pct=0.863724 moved_to_echo=33574 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[27776, 128, 128, 128, 128, 128, 5248, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 2816, 128, 128, 128, 128, 640, 27520, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[12928, 128, 128, 32768, 19968, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[20352, 128, 128, 128, 15232, 128, 128, 30080, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 128, 128, 1408, 0, 32384, 30464, 128, 1024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 7552, 128, 12544, 11008, 1664, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[17664, 128, 128, 24576, 6912, 128, 128, 16640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1280, 32768, 128, 128, 128, 128, 128, 31616, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[6, 12, 6, 4033, 0, 1, 0, 0, 1, 8, 4089, 4083, 4050, 0, 6, 4084, 2, 8, 4085, 2, 6, 87, 2822, 4096, 2549, 3818, 520, 3, 4092, 0, 0, 0, 309, 1348, 2555, 0, 0, 628, 1810, 6, 2, 1, 0, 0, 4, 1047, 0, 3047, 4, 4090, 12, 6, 10, 4, 2, 4066, 6, 8, 26, 39, 4037, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 4090, 0, 17, 36, 0, 0, 0, 3943, 4087, 3801, 0, 0, 4096, 0, 0, 4095, 0, 0, 1, 2143, 4071, 2028, 4087, 507, 0, 4049, 0, 0, 0, 44, 2247, 2388, 3, 0, 816, 2264, 0, 0, 0, 0, 5, 0, 2800, 0, 1638, 0, 4095, 0, 0, 0, 0, 0, 4090, 0, 0, 7, 0, 4082, 0, 5, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4078, 0, 0, 12, 0, 0, 3, 4094, 4096, 3966, 1, 0, 4095, 1, 0, 4094, 0, 0, 5, 3647, 4095, 1074, 3990, 353, 1, 4075, 0, 1, 1, 23, 2573, 2945, 1, 14, 1179, 1317, 0, 0, 0, 0, 1, 0, 1506, 0, 1994, 0, 4096, 0, 2, 3, 3, 0, 4093, 0, 1, 2, 0, 4088, 0, 13, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 0, 4069, 0, 0, 4, 0, 0, 0, 3984, 4096, 3962, 0, 0, 4096, 0, 0, 4096, 0, 0, 12, 3869, 4096, 2065, 3945, 628, 0, 4096, 0, 0, 0, 17, 1769, 1858, 0, 18, 512, 1918, 0, 0, 0, 0, 0, 0, 1397, 0, 2736, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4093, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 4, 0, 3909, 0, 0, 6, 0, 0, 0, 4056, 4095, 4020, 0, 0, 4092, 0, 0, 4096, 0, 0, 21, 3560, 4093, 2136, 4058, 852, 0, 4084, 1, 0, 1, 15, 2263, 2280, 1, 9, 998, 965, 0, 0, 1, 0, 0, 0, 1036, 1, 2595, 0, 4095, 0, 0, 0, 4, 0, 4095, 0, 0, 4, 0, 4086, 0, 4, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 112, 0, 4000, 0, 6, 1, 0, 0, 0, 4016, 4075, 4090, 0, 0, 4096, 0, 0, 4096, 0, 0, 18, 3555, 4092, 1711, 4092, 587, 0, 4091, 0, 0, 0, 66, 2261, 1063, 1, 3, 485, 2730, 0, 0, 0, 0, 0, 0, 985, 0, 3067, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4046, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 8, 0, 4029, 0, 7, 0, 0, 0, 0, 3877, 4085, 4094, 0, 0, 4094, 0, 0, 4096, 0, 0, 1, 3112, 4087, 1992, 4016, 107, 0, 4084, 0, 0, 0, 9, 2883, 2276, 14, 2, 1160, 1397, 0, 0, 2, 0, 0, 0, 1162, 0, 2700, 0, 4095, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4045, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[2, 73, 2, 3968, 0, 8, 0, 1, 0, 0, 4082, 4089, 4088, 1, 2, 4093, 0, 2, 4094, 2, 2, 3, 1836, 4091, 2437, 3873, 236, 1, 4092, 2, 0, 1, 64, 3518, 2248, 0, 1, 1288, 1986, 2, 2, 0, 0, 0, 2, 280, 2, 2796, 2, 4094, 0, 2, 0, 0, 0, 4089, 2, 3, 0, 0, 4070, 2, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4090, 0, 17, 36, 0, 3676, 451, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 164, 3801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 0, 0, 1, 2143, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 2028, 4087, 507, 0, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 2247, 2388, 3, 0, 816, 2264, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2800, 0, 1638, 4096, 247, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4082, 0, 5, 0, 2631, 1464, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8271, 7908, 7228, 8341, 8628, 8786, 8185, 8189] avg=8192.0000 max=8786 max_rank=5 min=7228 min_rank=2 imbalance_ratio=1.072510 max_load_reduction=56750 max_load_reduction_pct=0.865936 moved_to_echo=33788 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[6, 12, 6, 4033, 0, 1, 0, 0, 3673, 450, 0, 0, 0, 0, 0, 0, 1, 8, 4089, 163, 4050, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 1011, 2, 6, 87, 2822, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2549, 3818, 520, 3, 1738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 1348, 2555, 0, 0, 628, 1810, 6, 875, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 4, 1047, 0, 3047, 4084, 247, 0, 0, 0, 0, 0, 0, 4, 4090, 12, 6, 10, 4, 2, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8, 26, 39, 4037, 0, 0, 0, 2624, 1479, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8317, 8034, 8628, 7531, 8432, 8194, 8219] avg=8192.0000 max=8628 max_rank=3 min=7531 min_rank=4 imbalance_ratio=1.053223 max_load_reduction=56908 max_load_reduction_pct=0.868347 moved_to_echo=32376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 3909, 0, 0, 6, 0, 3683, 451, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 164, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 0, 0, 21, 3560, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 2136, 4058, 852, 0, 1734, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2263, 2280, 1, 9, 998, 965, 0, 873, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1036, 1, 2595, 4092, 248, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4086, 0, 4, 0, 2631, 1477, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8053, 8240, 8688, 8782, 7404, 7973, 8194, 8202] avg=8192.0000 max=8782 max_rank=3 min=7404 min_rank=4 imbalance_ratio=1.072021 max_load_reduction=56754 max_load_reduction_pct=0.865997 moved_to_echo=31773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4078, 0, 0, 12, 0, 3684, 451, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 164, 3966, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1013, 0, 0, 5, 3647, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 3990, 353, 1, 1731, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2573, 2945, 1, 14, 1179, 1317, 0, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1506, 0, 1994, 4095, 248, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 3, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4088, 0, 13, 0, 2630, 1473, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8225, 8228, 8761, 7151, 8923, 7844, 8197, 8207] avg=8192.0000 max=8923 max_rank=4 min=7151 min_rank=3 imbalance_ratio=1.089233 max_load_reduction=56613 max_load_reduction_pct=0.863846 moved_to_echo=33171 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 4069, 0, 0, 4, 0, 3684, 451, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 164, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 0, 0, 12, 3869, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 3945, 628, 0, 1739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1769, 1858, 0, 18, 512, 1918, 0, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 0, 2736, 4096, 248, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 3, 0, 2631, 1481, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8110, 8991, 8377, 6968, 8477, 8192, 8210] avg=8192.0000 max=8991 max_rank=2 min=6968 min_rank=4 imbalance_ratio=1.097534 max_load_reduction=56545 max_load_reduction_pct=0.862808 moved_to_echo=31847 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 112, 0, 4000, 0, 6, 1, 0, 3665, 451, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 164, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 0, 0, 18, 3555, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1711, 4092, 587, 0, 1738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 2261, 1063, 1, 3, 485, 2730, 0, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 985, 0, 3067, 4096, 246, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 1, 0, 2631, 1479, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 8270, 8679, 8128, 7483, 8394, 8190, 8157] avg=8192.0000 max=8679 max_rank=2 min=7483 min_rank=4 imbalance_ratio=1.059448 max_load_reduction=56857 max_load_reduction_pct=0.867569 moved_to_echo=32224 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 4029, 0, 7, 0, 0, 3680, 453, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 155, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 0, 0, 1, 3112, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 4016, 107, 0, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2883, 2276, 14, 2, 1160, 1397, 0, 876, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1162, 0, 2700, 4094, 250, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 5, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 1, 0, 2637, 1481, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8126, 8206, 7841, 8617, 8208, 8197, 8164] avg=8192.0000 max=8617 max_rank=4 min=7841 min_rank=3 imbalance_ratio=1.051880 max_load_reduction=56919 max_load_reduction_pct=0.868515 moved_to_echo=33186 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 73, 2, 3968, 0, 8, 0, 1, 3678, 451, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 164, 4088, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1013, 2, 2, 3, 1836, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 2437, 3873, 236, 1, 1738, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 64, 3518, 2248, 0, 1, 1288, 1986, 2, 875, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 280, 2, 2796, 4093, 247, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 2, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 4070, 2, 0, 2, 2630, 1479, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8183, 8337, 6949, 8288, 9982, 7422, 8187, 8188] avg=8192.0000 max=9982 max_rank=4 min=6949 min_rank=2 imbalance_ratio=1.218506 max_load_reduction=55554 max_load_reduction_pct=0.847687 moved_to_echo=33779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 18944, 17664, 128, 128, 7168, 14464, 128, 7040, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 128, 128, 21120, 11904, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32256, 1408, 32128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[16000, 32000, 3840, 128, 13952, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 256, 128, 32256, 0, 128, 128, 128, 29440, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 8192, 128, 128, 256, 24576, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 10240, 128, 20608, 32768, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3505, 0, 0, 3511, 6, 4080, 0, 0, 0, 22, 0, 3974, 4096, 0, 4074, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4093, 0, 1, 0, 0, 0, 4, 4096, 0, 0, 230, 4088, 0, 15, 0, 4096, 0, 1068, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[6, 9, 0, 0, 3394, 11, 5, 3469, 25, 4090, 0, 7, 0, 8, 0, 3530, 4085, 0, 4070, 4086, 9, 8, 7, 4090, 0, 6, 0, 20, 0, 7, 0, 1, 0, 9, 21, 4087, 8, 4082, 4085, 0, 4088, 4088, 8, 0, 0, 1, 0, 0, 4088, 0, 8, 1836, 3644, 7, 1, 0, 4085, 0, 434, 5, 0, 0, 0, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3957, 0, 0, 3290, 0, 4043, 0, 0, 0, 19, 0, 4080, 4096, 0, 4071, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4088, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 1506, 3257, 0, 0, 0, 4096, 0, 353, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2255, 0, 0, 3713, 10, 4094, 0, 0, 0, 4, 0, 3557, 4089, 0, 4061, 4096, 0, 0, 0, 4092, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 4096, 4096, 0, 4095, 4093, 0, 2, 0, 0, 0, 4, 4096, 2, 0, 1491, 4082, 0, 6, 0, 4096, 0, 1304, 0, 1, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3617, 0, 0, 3743, 4, 4096, 0, 0, 0, 8, 0, 3915, 4094, 0, 4093, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4095, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 439, 4072, 0, 0, 0, 4088, 2, 606, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3370, 0, 0, 3198, 3, 4089, 0, 0, 0, 4, 0, 3593, 4095, 0, 4083, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 4096, 4096, 0, 4095, 4066, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 1731, 4002, 0, 3, 0, 4091, 0, 537, 0, 1, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2, 2, 0, 5, 2599, 2, 0, 3639, 10, 4055, 0, 2, 0, 1, 1, 3480, 4092, 0, 4053, 4094, 2, 2, 2, 4094, 0, 0, 0, 0, 2, 0, 2, 0, 0, 2, 3, 4094, 2, 4094, 4093, 0, 4094, 4082, 2, 0, 2, 0, 0, 0, 4093, 0, 2, 2086, 3814, 0, 1, 0, 4090, 1, 833, 2, 3, 0, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3616, 0, 0, 3548, 32, 4060, 0, 0, 0, 40, 0, 3834, 4096, 0, 4093, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 4096, 4096, 0, 4096, 4076, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 862, 4084, 0, 4, 0, 4096, 0, 424, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3505, 0, 0, 3511, 711, 673, 0, 0, 0, 0, 0, 0, 6, 4080, 0, 0, 0, 22, 0, 3974, 345, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4074, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 1, 0, 0, 0, 4, 5, 2, 0, 0, 0, 0, 0, 0, 3035, 0, 0, 230, 4088, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1068, 0, 0, 0, 0, 0, 3401, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8400, 8427, 8192, 8190, 8191, 8201, 7368, 8567] avg=8192.0000 max=8567 max_rank=7 min=7368 min_rank=6 imbalance_ratio=1.045776 max_load_reduction=56969 max_load_reduction_pct=0.869278 moved_to_echo=32327 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[6, 9, 0, 0, 3394, 11, 5, 3469, 709, 671, 0, 0, 0, 0, 0, 0, 25, 4090, 0, 7, 0, 8, 0, 3530, 344, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4070, 0, 9, 8, 7, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 20, 0, 7, 0, 1, 4086, 4083, 0, 0, 0, 0, 0, 0, 0, 9, 21, 4087, 8, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4088, 8, 0, 0, 1, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 3030, 0, 8, 1836, 3644, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 434, 5, 0, 0, 0, 8, 3392, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8274, 8004, 8206, 8203, 8205, 8192, 8526, 7926] avg=8192.0000 max=8526 max_rank=6 min=7926 min_rank=7 imbalance_ratio=1.040771 max_load_reduction=57010 max_load_reduction_pct=0.869904 moved_to_echo=32849 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3370, 0, 0, 3198, 711, 672, 0, 0, 0, 0, 0, 0, 3, 4089, 0, 0, 0, 4, 0, 3593, 345, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4083, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4066, 0, 0, 0, 0, 0, 1, 5, 2, 0, 0, 0, 0, 0, 0, 3035, 0, 0, 1731, 4002, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 537, 0, 1, 0, 1, 0, 3401, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7951, 8034, 8198, 8190, 8190, 8169, 8771, 8033] avg=8192.0000 max=8771 max_rank=6 min=7951 min_rank=0 imbalance_ratio=1.070679 max_load_reduction=56765 max_load_reduction_pct=0.866165 moved_to_echo=33163 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3617, 0, 0, 3743, 711, 672, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 8, 0, 3915, 345, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4087, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 3035, 0, 0, 439, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 606, 1, 0, 0, 0, 0, 3400, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8743, 8368, 8211, 8190, 8190, 8189, 7546, 8099] avg=8192.0000 max=8743 max_rank=0 min=7546 min_rank=6 imbalance_ratio=1.067261 max_load_reduction=56793 max_load_reduction_pct=0.866592 moved_to_echo=32024 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2255, 0, 0, 3713, 711, 671, 0, 0, 0, 0, 0, 0, 10, 4094, 0, 0, 0, 4, 0, 3557, 345, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4061, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 0, 2, 0, 0, 0, 4, 5, 2, 0, 0, 0, 0, 0, 0, 3035, 2, 0, 1491, 4082, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1304, 0, 1, 0, 3, 0, 3396, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7350, 8010, 8175, 8192, 8190, 8201, 8616, 8802] avg=8192.0000 max=8802 max_rank=7 min=7350 min_rank=0 imbalance_ratio=1.074463 max_load_reduction=56734 max_load_reduction_pct=0.865692 moved_to_echo=33809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3957, 0, 0, 3290, 711, 673, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 19, 0, 4080, 345, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4071, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 8, 5, 2, 0, 0, 0, 0, 0, 0, 3035, 0, 0, 1506, 3257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 353, 0, 0, 0, 0, 0, 3401, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8631, 8487, 8189, 8190, 8190, 8199, 7798, 7852] avg=8192.0000 max=8631 max_rank=0 min=7798 min_rank=6 imbalance_ratio=1.053589 max_load_reduction=56905 max_load_reduction_pct=0.868301 moved_to_echo=32039 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 2, 0, 5, 2599, 2, 0, 3639, 710, 672, 0, 0, 0, 0, 0, 0, 10, 4055, 0, 2, 0, 1, 1, 3480, 344, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4053, 0, 2, 2, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 4094, 4092, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4094, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4082, 2, 0, 2, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 3034, 0, 2, 2086, 3814, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 833, 2, 3, 0, 0, 2, 3398, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7631, 7893, 8175, 8190, 8193, 8187, 8937, 8330] avg=8192.0000 max=8937 max_rank=6 min=7631 min_rank=0 imbalance_ratio=1.090942 max_load_reduction=56599 max_load_reduction_pct=0.863632 moved_to_echo=33647 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3616, 0, 0, 3548, 715, 677, 0, 0, 0, 0, 0, 0, 32, 4060, 0, 0, 0, 40, 0, 3834, 347, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4093, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 0, 0, 0, 0, 0, 11, 10, 5, 0, 0, 0, 0, 0, 0, 3024, 0, 0, 862, 4084, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 424, 0, 0, 0, 1, 0, 3405, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8556, 8313, 8190, 8191, 8187, 8198, 7974, 7927] avg=8192.0000 max=8556 max_rank=0 min=7927 min_rank=7 imbalance_ratio=1.044434 max_load_reduction=56980 max_load_reduction_pct=0.869446 moved_to_echo=32286 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 26368, 128, 128, 28160, 5760, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[24320, 128, 128, 10240, 31104, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 0, 32640, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 128, 5632, 128, 128, 0, 128, 128, 27264, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32640, 0, 128, 0, 128, 128, 30080, 2816, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 4094, 0, 0, 0, 0, 4087, 4, 0, 0, 0, 0, 4095, 3, 0, 0, 63, 0, 0, 4018, 0, 4091, 0, 4, 4096, 0, 3332, 0, 7, 0, 5, 35, 0, 0, 4094, 9, 0, 4096, 0, 4096, 0, 0, 0, 0, 4084, 4072, 4096, 0, 0, 2, 0, 4096, 0, 2, 0, 4092, 4083, 0, 0, 769, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 355, 0, 0, 4077, 0, 4095, 0, 67, 4096, 0, 2816, 0, 0, 1, 0, 159, 0, 2, 4096, 554, 0, 4096, 0, 4096, 0, 0, 0, 0, 4088, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4035, 0, 0, 236, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[15, 6, 0, 1, 4008, 0, 9, 1, 25, 4078, 0, 8, 8, 8, 0, 4087, 0, 8, 0, 55, 0, 7, 3999, 0, 4087, 10, 18, 4087, 0, 2555, 0, 0, 0, 0, 13, 0, 0, 4088, 1537, 9, 4074, 7, 4087, 9, 0, 8, 3, 4055, 4088, 4074, 8, 0, 0, 0, 4085, 4, 0, 7, 4088, 4085, 5, 7, 106, 9] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 4093, 6, 0, 0, 133, 0, 0, 3914, 0, 4092, 0, 1, 4096, 0, 3039, 0, 1, 2, 1, 29, 1, 0, 4095, 376, 0, 4096, 1, 4096, 0, 0, 0, 0, 4092, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4095, 0, 0, 704, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 4094, 5, 0, 0, 2, 0, 0, 3987, 0, 4096, 0, 10, 4096, 0, 3382, 0, 0, 5, 4, 41, 0, 3, 4096, 5, 0, 4096, 0, 4096, 0, 0, 0, 0, 4091, 4091, 4096, 0, 0, 0, 0, 4096, 0, 9, 0, 4092, 4051, 0, 0, 801, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 100, 0, 0, 4052, 0, 4095, 0, 0, 4096, 0, 2241, 0, 1, 0, 0, 7, 0, 1, 4096, 554, 0, 4095, 1, 4095, 0, 0, 0, 0, 4095, 4080, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 1253, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1, 4, 1, 0, 4093, 0, 2, 2, 0, 4085, 1, 2, 2, 2, 0, 4087, 3, 2, 0, 193, 0, 2, 4091, 1, 4093, 1, 11, 4085, 1, 2734, 1, 2, 16, 0, 116, 0, 5, 4094, 198, 0, 4093, 1, 4090, 2, 0, 2, 2, 4078, 4092, 4094, 2, 0, 0, 0, 4090, 0, 0, 2, 4094, 4080, 2, 2, 877, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 9, 0, 0, 0, 0, 4085, 17, 0, 0, 52, 0, 0, 3973, 0, 4093, 0, 1, 4096, 0, 2533, 0, 0, 3, 2, 122, 0, 0, 4095, 20, 0, 4096, 0, 4096, 0, 0, 0, 0, 4077, 4095, 4096, 0, 0, 5, 0, 4096, 0, 4, 0, 4094, 4066, 0, 0, 1520, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4088, 14, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 0, 0, 3987, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 1255, 0, 3382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 41, 0, 3, 4096, 5, 0, 2841, 762, 14, 0, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3329, 4051, 0, 0, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8190, 8048, 8743, 7771, 8201, 8195, 8190] avg=8192.0000 max=8743 max_rank=3 min=7771 min_rank=4 imbalance_ratio=1.067261 max_load_reduction=56793 max_load_reduction_pct=0.866592 moved_to_echo=32357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[15, 6, 0, 1, 4008, 0, 9, 1, 4077, 14, 0, 0, 0, 0, 0, 0, 25, 4078, 0, 8, 8, 8, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 55, 0, 7, 3999, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 4087, 10, 18, 1252, 0, 2555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4088, 1537, 9, 2835, 762, 14, 0, 0, 0, 0, 0, 4074, 7, 14, 9, 0, 8, 3, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4074, 8, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3325, 4085, 5, 7, 106, 9, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8131, 8215, 8114, 7922, 9258, 8170, 8182, 7544] avg=8192.0000 max=9258 max_rank=4 min=7544 min_rank=7 imbalance_ratio=1.130127 max_load_reduction=56278 max_load_reduction_pct=0.858734 moved_to_echo=33154 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4088, 14, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 133, 0, 0, 3914, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 1255, 0, 3039, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 29, 1, 0, 4095, 376, 0, 2841, 763, 14, 0, 0, 0, 0, 0, 4096, 1, 14, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3331, 4095, 0, 0, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8189, 8107, 8388, 8122, 8203, 8199, 8130] avg=8192.0000 max=8388 max_rank=3 min=8107 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32654 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 4094, 0, 0, 0, 4088, 14, 0, 0, 0, 0, 0, 0, 0, 4087, 4, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 63, 0, 0, 4018, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4, 1255, 0, 3332, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 35, 0, 0, 4094, 9, 0, 2841, 762, 14, 0, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4096, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3329, 4083, 0, 0, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8187, 8138, 8689, 7760, 8194, 8178, 8183] avg=8192.0000 max=8689 max_rank=3 min=7760 min_rank=4 imbalance_ratio=1.060669 max_load_reduction=56847 max_load_reduction_pct=0.867416 moved_to_echo=32315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4096, 0, 0, 0, 4088, 14, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 4052, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1255, 0, 2241, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 4096, 554, 0, 2841, 763, 14, 0, 0, 0, 0, 0, 4095, 1, 14, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3332, 4096, 0, 0, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8193, 8205, 7592, 8276, 8205, 8184, 8681] avg=8192.0000 max=8681 max_rank=7 min=7592 min_rank=3 imbalance_ratio=1.059692 max_load_reduction=56855 max_load_reduction_pct=0.867538 moved_to_echo=33346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4088, 14, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 355, 0, 0, 4077, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 67, 1255, 0, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 159, 0, 2, 4096, 554, 0, 2841, 763, 14, 0, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3332, 4035, 0, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8188, 8487, 8233, 8430, 8198, 8199, 7603] avg=8192.0000 max=8487 max_rank=2 min=7603 min_rank=7 imbalance_ratio=1.036011 max_load_reduction=57049 max_load_reduction_pct=0.870499 moved_to_echo=32430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 4089, 20, 0, 0, 0, 0, 0, 0, 0, 4094, 9, 0, 0, 0, 0, 4085, 6, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 52, 0, 0, 3973, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 1248, 0, 2533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 122, 0, 0, 4095, 20, 0, 2848, 768, 21, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3320, 4066, 0, 0, 1520, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 8194, 8097, 7875, 7879, 8173, 8203, 8910] avg=8192.0000 max=8910 max_rank=7 min=7875 min_rank=3 imbalance_ratio=1.087646 max_load_reduction=56626 max_load_reduction_pct=0.864044 moved_to_echo=33165 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 1, 0, 4093, 0, 2, 2, 4082, 14, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 2, 2, 2, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 193, 0, 2, 4091, 1, 4048, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 11, 1251, 1, 2734, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 116, 0, 5, 4094, 198, 0, 2834, 763, 14, 0, 0, 0, 0, 0, 4093, 1, 14, 2, 0, 2, 2, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4094, 2, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3330, 4080, 2, 2, 877, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8180, 8340, 8094, 8040, 8192, 8196, 8295] avg=8192.0000 max=8340 max_rank=2 min=8040 min_rank=4 imbalance_ratio=1.018066 max_load_reduction=57196 max_load_reduction_pct=0.872742 moved_to_echo=32723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 1024, 0, 128, 32128, 128, 32512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 10112, 128, 22656, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 26752, 32640, 128, 128, 6272, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 640, 128, 128, 32768, 3328, 128, 22784, 6144, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3829, 0, 0, 118, 0, 0, 0, 4085, 0, 531, 0, 4096, 178, 4096, 0, 0, 0, 0, 0, 0, 466, 4096, 0, 9, 0, 0, 0, 0, 3921, 1, 0, 0, 0, 4068, 4086, 187, 4096, 0, 0, 4082, 4092, 2656, 0, 0, 0, 0, 0, 0, 4093, 0, 50, 0, 0, 4032, 0, 3816, 27, 0, 3, 0, 726] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3442, 0, 0, 0, 0, 0, 0, 4083, 0, 421, 0, 4096, 16, 4096, 0, 0, 0, 0, 0, 0, 2726, 4096, 0, 148, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 4045, 4093, 952, 4096, 0, 0, 4092, 4058, 1775, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 2656, 0, 3006, 10, 0, 10, 0, 1367] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3759, 0, 0, 36, 0, 0, 0, 4093, 0, 522, 0, 4096, 21, 4096, 0, 0, 0, 0, 0, 1, 2217, 4096, 0, 64, 0, 0, 0, 0, 4078, 1, 0, 0, 0, 4094, 4096, 633, 4096, 0, 0, 4093, 4080, 1832, 0, 0, 0, 0, 0, 0, 4095, 0, 8, 0, 0, 2535, 0, 3753, 4, 0, 3, 0, 1038] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[3, 6, 4086, 3366, 0, 8, 40, 0, 3, 0, 4086, 9, 455, 0, 4087, 0, 4087, 10, 1, 0, 0, 5, 0, 1496, 4087, 10, 95, 0, 9, 1, 7, 4094, 0, 9, 2, 4, 4053, 3911, 377, 4076, 9, 0, 4087, 3960, 3054, 12, 2, 3, 16, 7, 0, 4070, 0, 231, 0, 9, 2712, 9, 4009, 0, 0, 0, 0, 863] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3580, 6, 0, 16, 0, 0, 0, 4094, 0, 265, 0, 4084, 199, 4096, 0, 0, 0, 1, 0, 1, 383, 4096, 0, 256, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 4092, 4096, 1210, 4096, 0, 0, 4095, 4081, 1408, 0, 0, 0, 0, 0, 0, 4082, 0, 83, 1, 0, 3852, 0, 3716, 2, 0, 2, 0, 1537] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2652, 0, 0, 19, 0, 0, 0, 4096, 0, 423, 0, 4093, 36, 4096, 0, 0, 0, 0, 0, 0, 2385, 4096, 0, 53, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 4087, 4081, 1181, 4095, 0, 0, 4096, 4078, 1342, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 3655, 0, 3766, 5, 0, 0, 0, 923] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3820, 1, 0, 8, 0, 0, 0, 4086, 0, 302, 0, 4096, 257, 4096, 0, 0, 0, 0, 0, 2, 415, 4096, 0, 94, 0, 0, 0, 0, 4019, 2, 0, 0, 0, 4088, 4096, 840, 4094, 0, 0, 4070, 4091, 2028, 0, 0, 0, 0, 0, 0, 4066, 0, 45, 0, 0, 3872, 0, 3631, 19, 0, 16, 0, 1190] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 4093, 3889, 0, 2, 8, 0, 0, 0, 4087, 2, 527, 0, 4094, 8, 4094, 2, 2, 0, 0, 1, 1, 2109, 4094, 2, 44, 0, 2, 1, 2, 4070, 0, 0, 0, 2, 4084, 4094, 1026, 4091, 2, 2, 4078, 4024, 1993, 2, 0, 2, 0, 2, 0, 4093, 2, 0, 1, 2, 2236, 2, 3974, 16, 0, 0, 0, 672] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3829, 0, 0, 118, 0, 321, 199, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 531, 0, 3585, 178, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 466, 1965, 555, 44, 0, 0, 0, 0, 0, 4053, 0, 9, 0, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3314, 4086, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2117, 4092, 2656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 50, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 3816, 27, 0, 3, 0, 726, 190, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8563, 8379, 7126, 7983, 7588, 8865, 8195, 8837] avg=8192.0000 max=8865 max_rank=5 min=7126 min_rank=2 imbalance_ratio=1.082153 max_load_reduction=56671 max_load_reduction_pct=0.864731 moved_to_echo=33485 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[3, 6, 4086, 3366, 0, 8, 40, 0, 320, 198, 0, 0, 0, 0, 0, 0, 3, 0, 4086, 9, 455, 0, 3578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 10, 1, 0, 0, 5, 0, 1496, 1968, 553, 45, 0, 0, 0, 0, 0, 4044, 10, 95, 0, 9, 1, 7, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 4, 3302, 3911, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2119, 3960, 3054, 12, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 16, 7, 0, 4070, 0, 231, 0, 9, 4031, 0, 0, 0, 0, 0, 0, 0, 2712, 9, 4009, 0, 0, 0, 0, 863, 189, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8027, 8131, 8165, 8260, 7605, 9159, 8364, 7825] avg=8192.0000 max=9159 max_rank=5 min=7605 min_rank=4 imbalance_ratio=1.118042 max_load_reduction=56377 max_load_reduction_pct=0.860245 moved_to_echo=32953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3759, 0, 0, 36, 0, 321, 200, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 522, 0, 3585, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 2217, 1970, 559, 45, 0, 0, 0, 0, 0, 4053, 0, 64, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3335, 4096, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2123, 4080, 1832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 8, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 2535, 0, 3753, 4, 0, 3, 0, 1038, 190, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8412, 8221, 8888, 8195, 8065, 8035, 8154, 7566] avg=8192.0000 max=8888 max_rank=2 min=7566 min_rank=7 imbalance_ratio=1.084961 max_load_reduction=56648 max_load_reduction_pct=0.864380 moved_to_echo=31820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3442, 0, 0, 0, 0, 321, 198, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 421, 0, 3585, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2726, 1970, 552, 45, 0, 0, 0, 0, 0, 4053, 0, 148, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3295, 4093, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2122, 4058, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 2656, 0, 3006, 10, 0, 10, 0, 1367, 190, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8057, 8105, 9389, 8266, 8340, 7955, 8142, 7282] avg=8192.0000 max=9389 max_rank=2 min=7282 min_rank=7 imbalance_ratio=1.146118 max_load_reduction=56147 max_load_reduction_pct=0.856735 moved_to_echo=31719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3580, 6, 0, 16, 0, 320, 200, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 265, 0, 3575, 199, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 1, 383, 1971, 558, 45, 0, 0, 0, 0, 0, 4053, 0, 256, 0, 0, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3334, 4096, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2124, 4081, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 83, 1, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 3852, 0, 3716, 2, 0, 2, 0, 1537, 189, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8133, 7055, 8319, 8640, 7613, 8217, 9341] avg=8192.0000 max=9341 max_rank=7 min=7055 min_rank=2 imbalance_ratio=1.140259 max_load_reduction=56195 max_load_reduction_pct=0.857468 moved_to_echo=33811 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 2652, 0, 0, 19, 0, 321, 200, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 423, 0, 3582, 36, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2385, 1972, 558, 44, 0, 0, 0, 0, 0, 4053, 0, 53, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 4081, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2124, 4078, 1342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 3655, 0, 3766, 5, 0, 0, 0, 923, 190, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7288, 8137, 9055, 8181, 8591, 7544, 8158, 8582] avg=8192.0000 max=9055 max_rank=2 min=7288 min_rank=0 imbalance_ratio=1.105347 max_load_reduction=56481 max_load_reduction_pct=0.861832 moved_to_echo=32875 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3820, 1, 0, 8, 0, 325, 206, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 302, 0, 3578, 257, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 415, 1964, 562, 45, 0, 0, 0, 0, 0, 4047, 0, 94, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3320, 4096, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2106, 4091, 2028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 45, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 3872, 0, 3631, 19, 0, 16, 0, 1190, 193, 49, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8456, 8223, 7084, 8160, 8258, 8225, 8160, 8970] avg=8192.0000 max=8970 max_rank=7 min=7084 min_rank=2 imbalance_ratio=1.094971 max_load_reduction=56566 max_load_reduction_pct=0.863129 moved_to_echo=33613 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4093, 3889, 0, 2, 8, 0, 321, 200, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 527, 0, 3583, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 2, 0, 0, 1, 1, 2109, 1963, 557, 45, 0, 0, 0, 0, 0, 4051, 2, 44, 0, 2, 1, 2, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3327, 4094, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2115, 4024, 1993, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 2, 0, 1, 2, 4046, 0, 0, 0, 0, 0, 0, 0, 2236, 2, 3974, 16, 0, 0, 0, 672, 190, 43, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8515, 8207, 8774, 8172, 8449, 8140, 8146, 7133] avg=8192.0000 max=8774 max_rank=2 min=7133 min_rank=7 imbalance_ratio=1.071045 max_load_reduction=56762 max_load_reduction_pct=0.866119 moved_to_echo=31868 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[25600, 128, 29696, 128, 0, 128, 0, 8320, 1536, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 32768, 128, 3456, 0, 28672, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32512, 128, 768, 0, 128, 128, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 26624, 32640, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 28416, 128, 128, 256, 0, 2688, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 128, 12288, 15744, 4480, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 17024, 32512, 16128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 512, 128, 128, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2124, 0, 1, 0, 2957, 0, 4096, 81, 0, 4037, 4096, 0, 89, 0, 0, 3539, 4092, 0, 1950, 0, 0, 4053, 4096, 0, 0, 4082, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 1590, 0, 0, 0, 0, 0, 9, 9, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1338, 0, 0, 0, 3469, 0, 4096, 70, 0, 4090, 4096, 0, 25, 0, 0, 3905, 4096, 0, 1271, 0, 0, 4075, 4096, 0, 0, 4094, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 4083, 1, 2, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 2221, 0, 0, 0, 0, 0, 1, 2, 4095, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[11, 0, 75, 9, 2247, 12, 0, 31, 3878, 12, 4085, 501, 12, 4079, 4019, 8, 298, 0, 9, 3383, 4081, 1, 288, 0, 0, 4056, 4085, 10, 9, 4074, 9, 0, 7, 0, 48, 7, 9, 0, 1, 0, 9, 4085, 0, 0, 4087, 0, 5, 9, 4071, 4082, 5, 8, 0, 1616, 0, 4, 0, 8, 5, 0, 27, 4084, 0, 4077] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2010, 0, 0, 0, 2692, 0, 4096, 676, 0, 4084, 4096, 0, 248, 0, 0, 4040, 4094, 0, 778, 0, 0, 4066, 4096, 0, 0, 4095, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4091, 0, 0, 0, 4096, 4096, 0, 0, 0, 1864, 0, 0, 0, 0, 0, 2, 27, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2884, 0, 0, 0, 2707, 1, 4096, 151, 0, 4095, 4096, 0, 48, 0, 0, 3581, 4096, 0, 1363, 0, 0, 4075, 4096, 0, 0, 4096, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 4089, 0, 3, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 1536, 0, 0, 0, 0, 0, 2, 4, 4093, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1230, 0, 0, 0, 3405, 0, 4096, 12, 0, 4071, 4096, 0, 126, 0, 0, 3443, 4092, 0, 962, 0, 0, 4070, 4096, 0, 0, 4095, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 4087, 4, 13, 4088, 0, 0, 0, 4096, 4096, 0, 0, 0, 3128, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3, 0, 6, 2, 2966, 0, 0, 0, 2624, 3, 4094, 264, 1, 4070, 4091, 1, 120, 0, 2, 3648, 4086, 0, 736, 0, 0, 4025, 4094, 3, 2, 4090, 2, 0, 2, 0, 71, 2, 2, 0, 0, 0, 2, 4079, 0, 3, 4093, 3, 1, 2, 4093, 4093, 1, 2, 1, 1938, 0, 0, 0, 2, 2, 8, 16, 4094, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1077, 0, 1, 0, 3080, 0, 4096, 15, 0, 4079, 4096, 0, 151, 0, 0, 3538, 4095, 0, 1327, 0, 0, 4036, 4096, 0, 0, 4087, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 4091, 0, 0, 0, 4096, 4096, 0, 0, 0, 3164, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2124, 0, 1, 0, 3304, 2130, 758, 5, 0, 0, 0, 0, 2957, 0, 0, 81, 0, 4037, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 3539, 3334, 0, 1950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 42, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 4096, 4054, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 1966, 0, 0, 0, 1590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 4084, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8322, 7866, 8912, 8177, 8212, 8197, 7652, 8198] avg=8192.0000 max=8912 max_rank=2 min=7652 min_rank=6 imbalance_ratio=1.087891 max_load_reduction=56624 max_load_reduction_pct=0.864014 moved_to_echo=32259 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 75, 9, 2247, 12, 0, 31, 3241, 2122, 756, 5, 0, 0, 0, 0, 3878, 12, 0, 501, 12, 4079, 777, 8, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 9, 3383, 3325, 1, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 42, 10, 9, 4074, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 48, 7, 9, 0, 1, 0, 4085, 4043, 1, 0, 0, 0, 0, 0, 9, 4085, 0, 0, 4087, 0, 5, 9, 6, 0, 0, 0, 0, 0, 0, 0, 4071, 1960, 5, 8, 0, 1616, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 5, 0, 27, 4073, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8509, 9267, 7304, 8200, 8201, 8201, 7664, 8190] avg=8192.0000 max=9267 max_rank=1 min=7304 min_rank=2 imbalance_ratio=1.131226 max_load_reduction=56269 max_load_reduction_pct=0.858597 moved_to_echo=32256 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1338, 0, 0, 0, 3304, 2130, 758, 5, 0, 0, 0, 0, 3469, 0, 0, 70, 0, 4090, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 3905, 3338, 0, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 42, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 4096, 4054, 1, 0, 0, 0, 0, 0, 0, 4083, 1, 2, 4095, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 1966, 0, 0, 0, 2221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4083, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7535, 8420, 8539, 8211, 8178, 8188, 8283, 8182] avg=8192.0000 max=8539 max_rank=2 min=7535 min_rank=0 imbalance_ratio=1.042358 max_load_reduction=56997 max_load_reduction_pct=0.869705 moved_to_echo=32831 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1230, 0, 0, 0, 3304, 2130, 758, 5, 0, 0, 0, 0, 3405, 0, 0, 12, 0, 4071, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 0, 3443, 3334, 0, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 42, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4096, 4054, 1, 0, 0, 0, 0, 0, 0, 4087, 4, 13, 4088, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 1966, 0, 0, 0, 3128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4084, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7427, 8279, 7865, 8207, 8183, 8199, 9190, 8186] avg=8192.0000 max=9190 max_rank=6 min=7427 min_rank=0 imbalance_ratio=1.121826 max_load_reduction=56346 max_load_reduction_pct=0.859772 moved_to_echo=33758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2010, 0, 0, 0, 3304, 2130, 758, 5, 0, 0, 0, 0, 2692, 0, 0, 676, 0, 4084, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 4040, 3336, 0, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 42, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 4096, 4054, 1, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4091, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 1966, 0, 0, 0, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 27, 4084, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8243, 8402, 8203, 8175, 8171, 7926, 8209] avg=8192.0000 max=8402 max_rank=2 min=7926 min_rank=6 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 6, 2, 2966, 0, 0, 0, 3299, 2128, 757, 5, 0, 0, 0, 0, 2624, 3, 0, 264, 1, 4070, 791, 1, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 2, 3648, 3329, 0, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 42, 3, 2, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 71, 2, 2, 0, 0, 0, 4094, 4052, 1, 0, 0, 0, 0, 0, 2, 4079, 0, 3, 4093, 3, 1, 2, 7, 0, 0, 0, 0, 0, 0, 0, 4093, 1965, 1, 2, 1, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 8, 16, 4082, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9166, 7754, 7835, 8164, 8224, 8190, 8000, 8203] avg=8192.0000 max=9166 max_rank=0 min=7754 min_rank=1 imbalance_ratio=1.118896 max_load_reduction=56370 max_load_reduction_pct=0.860138 moved_to_echo=32617 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1077, 0, 1, 0, 3306, 2133, 762, 10, 0, 0, 0, 0, 3080, 0, 0, 15, 0, 4079, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 3538, 3333, 0, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 38, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 4096, 4058, 2, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 4091, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 1963, 0, 0, 0, 3164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4077, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7289, 7962, 8349, 8161, 8172, 8196, 9223, 8184] avg=8192.0000 max=9223 max_rank=6 min=7289 min_rank=0 imbalance_ratio=1.125854 max_load_reduction=56313 max_load_reduction_pct=0.859268 moved_to_echo=33775 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2884, 0, 0, 0, 3304, 2130, 758, 5, 0, 0, 0, 0, 2707, 1, 0, 151, 0, 4095, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 3581, 3338, 0, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 42, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 4096, 4054, 1, 0, 0, 0, 0, 0, 0, 4089, 0, 3, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 1966, 0, 0, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4082, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9081, 7745, 8330, 8213, 8191, 8194, 7598, 8184] avg=8192.0000 max=9081 max_rank=0 min=7598 min_rank=6 imbalance_ratio=1.108521 max_load_reduction=56455 max_load_reduction_pct=0.861435 moved_to_echo=32167 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 16000, 128, 128, 128, 26368, 17152, 6144, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 384, 128, 128, 0, 128, 0, 32768, 32512, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[24832, 128, 0, 1792, 128, 32640, 6400, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32512, 384, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 15744, 128, 128, 128, 17152, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[1152, 0, 128, 29184, 26752, 128, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[13, 0, 0, 4086, 3675, 9, 0, 4085, 3, 21, 10, 9, 4075, 0, 9, 4014, 4084, 0, 10, 4087, 6, 0, 4025, 4084, 0, 0, 3, 0, 9, 3560, 75, 0, 684, 183, 4080, 0, 10, 0, 0, 109, 0, 7, 4082, 0, 9, 7, 7, 1, 4069, 0, 11, 0, 13, 9, 0, 8, 0, 3984, 4087, 0, 4086, 134, 0, 4] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4055, 0, 0, 4096, 0, 0, 0, 0, 4088, 5, 0, 4096, 4096, 0, 0, 4093, 0, 0, 3848, 4096, 26, 0, 0, 34, 0, 4085, 0, 0, 48, 0, 4096, 0, 0, 0, 0, 3475, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 895, 4096, 0, 4020, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4041, 0, 0, 4096, 0, 0, 0, 0, 4095, 2, 0, 4096, 4096, 0, 0, 4095, 0, 0, 3814, 4095, 12, 0, 0, 5, 0, 3890, 0, 0, 2, 0, 4096, 1, 0, 0, 0, 1682, 1, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 2969, 4096, 0, 4065, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 0, 0, 0, 4090, 5, 0, 4096, 4096, 0, 0, 4092, 0, 0, 3873, 4092, 11, 2, 0, 6, 0, 3940, 0, 0, 23, 0, 4096, 4, 0, 9, 0, 2857, 1, 0, 4090, 0, 0, 0, 3, 0, 4091, 0, 0, 0, 6, 0, 0, 0, 0, 1977, 4096, 0, 3694, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4080, 0, 0, 4096, 0, 0, 0, 0, 4094, 9, 0, 4096, 4096, 0, 0, 4096, 0, 0, 3970, 4096, 2, 0, 0, 15, 0, 3824, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 571, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 4096, 0, 4096, 6, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4083, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 3978, 4096, 0, 0, 0, 0, 0, 4048, 0, 0, 55, 1, 4096, 0, 0, 11, 0, 1274, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2949, 4096, 0, 4082, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2, 4093, 4025, 2, 0, 4093, 0, 0, 2, 2, 4084, 0, 2, 4090, 4094, 0, 2, 4094, 2, 0, 4017, 4092, 0, 0, 2, 8, 2, 3890, 13, 1, 210, 4, 4094, 0, 2, 0, 1, 698, 0, 3, 4090, 0, 2, 2, 0, 1, 4092, 2, 2, 0, 1, 2, 0, 2, 0, 3539, 4094, 0, 4078, 3, 0, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4033, 0, 0, 4096, 0, 0, 0, 0, 4093, 10, 0, 4096, 4096, 0, 0, 4094, 0, 0, 3890, 4093, 19, 0, 0, 11, 0, 4026, 0, 0, 65, 0, 4096, 0, 0, 0, 0, 3585, 2, 0, 4083, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 873, 4096, 0, 3985, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 85, 4055, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 5, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 3848, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 34, 0, 4085, 0, 0, 4096, 157, 0, 0, 0, 0, 0, 0, 48, 0, 4096, 0, 0, 0, 0, 3475, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3854, 242, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3680, 412, 0, 0, 0, 0, 0, 0, 0, 895, 1527, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 8195, 8115, 8398, 9770, 8192, 8188, 6442] avg=8192.0000 max=9770 max_rank=4 min=6442 min_rank=7 imbalance_ratio=1.192627 max_load_reduction=55766 max_load_reduction_pct=0.850922 moved_to_echo=32592 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 85, 3675, 9, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 3, 21, 10, 9, 4075, 0, 9, 4014, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 170, 6, 0, 4025, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 9, 3560, 75, 0, 4084, 157, 0, 0, 0, 0, 0, 0, 684, 183, 4080, 0, 10, 0, 0, 109, 2146, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4082, 0, 9, 7, 7, 1, 3844, 242, 0, 0, 0, 0, 0, 0, 4069, 0, 11, 0, 13, 9, 0, 8, 3675, 411, 0, 0, 0, 0, 0, 0, 0, 3984, 1524, 0, 4086, 134, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7867, 8147, 8295, 7888, 7212, 8199, 8196, 9732] avg=8192.0000 max=9732 max_rank=7 min=7212 min_rank=4 imbalance_ratio=1.187988 max_load_reduction=55804 max_load_reduction_pct=0.851501 moved_to_echo=33339 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 85, 4041, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 3814, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 5, 0, 3890, 0, 0, 4096, 157, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 1, 0, 0, 0, 1682, 2151, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 3854, 243, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 3682, 412, 0, 0, 0, 0, 0, 0, 0, 2969, 1527, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8199, 8079, 8160, 7932, 8192, 8191, 8561] avg=8192.0000 max=8561 max_rank=7 min=7932 min_rank=4 imbalance_ratio=1.045044 max_load_reduction=56975 max_load_reduction_pct=0.869370 moved_to_echo=32876 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 85, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 5, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 3873, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 0, 6, 0, 3940, 0, 0, 4096, 157, 0, 0, 0, 0, 0, 0, 23, 0, 4096, 4, 0, 9, 0, 2857, 2151, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 3, 0, 3854, 242, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 6, 0, 0, 0, 3679, 412, 0, 0, 0, 0, 0, 0, 0, 1977, 1527, 0, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8275, 8197, 8136, 8212, 9140, 8190, 8188, 7198] avg=8192.0000 max=9140 max_rank=4 min=7198 min_rank=7 imbalance_ratio=1.115723 max_load_reduction=56396 max_load_reduction_pct=0.860535 moved_to_echo=32716 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 85, 4080, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 9, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 3970, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 15, 0, 3824, 0, 0, 4096, 157, 0, 0, 0, 0, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 571, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3854, 243, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3683, 412, 0, 0, 0, 0, 0, 0, 0, 3897, 1527, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 8205, 8236, 8094, 6832, 8191, 8191, 9526] avg=8192.0000 max=9526 max_rank=7 min=6832 min_rank=4 imbalance_ratio=1.162842 max_load_reduction=56010 max_load_reduction_pct=0.854645 moved_to_echo=32740 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 85, 4083, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 3978, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 4096, 157, 0, 0, 0, 0, 0, 0, 55, 1, 4096, 0, 0, 11, 0, 1274, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3854, 243, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3683, 412, 0, 0, 0, 0, 0, 0, 0, 2949, 1527, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8264, 8198, 8244, 8301, 7588, 8192, 8191, 8558] avg=8192.0000 max=8558 max_rank=7 min=7588 min_rank=4 imbalance_ratio=1.044678 max_load_reduction=56978 max_load_reduction_pct=0.869415 moved_to_echo=32529 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 75, 4033, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 10, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 3890, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 11, 0, 4026, 0, 0, 4096, 164, 0, 0, 0, 0, 0, 0, 65, 0, 4096, 0, 0, 0, 0, 3585, 2157, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4083, 0, 0, 0, 0, 0, 3857, 246, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 3683, 416, 0, 0, 0, 0, 0, 0, 0, 873, 1513, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8204, 8209, 8148, 8316, 9903, 8188, 8197, 6371] avg=8192.0000 max=9903 max_rank=4 min=6371 min_rank=7 imbalance_ratio=1.208862 max_load_reduction=55633 max_load_reduction_pct=0.848892 moved_to_echo=32659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 85, 4025, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4084, 0, 2, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 170, 2, 0, 4017, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 2, 3890, 13, 1, 4094, 157, 0, 0, 0, 0, 0, 0, 210, 4, 4094, 0, 2, 0, 1, 698, 2150, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 2, 2, 0, 1, 3851, 243, 0, 0, 0, 0, 0, 0, 4092, 2, 2, 0, 1, 2, 0, 2, 3681, 412, 0, 0, 0, 0, 0, 0, 0, 3539, 1526, 0, 4078, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8186, 8283, 8167, 7159, 8192, 8194, 9148] avg=8192.0000 max=9148 max_rank=7 min=7159 min_rank=4 imbalance_ratio=1.116699 max_load_reduction=56388 max_load_reduction_pct=0.860413 moved_to_echo=32693 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 1408, 128, 0, 31488, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 128, 128, 30848, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 128, 29568, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 21120, 12288, 0, 32128, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 768, 32128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1152, 256, 32768, 128, 128, 128, 128, 14336, 17280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 31360, 128, 128, 32768, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[12, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 311, 7, 0, 0, 0, 0, 4069, 0, 4096, 0, 4054, 0, 830, 0, 0, 4068, 0, 0, 0, 0, 4042, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4094, 4054, 2978, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 103, 0, 4095, 0, 0, 0, 49, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 4089, 0, 21, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4052, 3996, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 45, 0, 4096, 0, 0, 0, 29, 0, 4096, 0, 4065] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 238, 2, 0, 0, 0, 0, 4091, 0, 4096, 0, 4077, 0, 4, 0, 0, 4090, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4087, 3824, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 62, 0, 4095, 0, 0, 0, 7, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 4067, 0, 4096, 0, 4069, 0, 137, 0, 1, 4094, 0, 0, 0, 0, 4069, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 3869, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 46, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[15, 0, 0, 4096, 0, 0, 9, 0, 0, 8, 110, 1, 0, 0, 0, 0, 3833, 0, 4096, 0, 4057, 0, 265, 0, 0, 4086, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4066, 4067, 3833, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 202, 0, 4093, 0, 0, 0, 32, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 4083, 0, 0, 0, 368, 20, 0, 0, 0, 0, 10, 10, 0, 3455, 10, 3974, 16, 4059, 11, 2089, 7, 0, 4084, 10, 9, 9, 235, 3301, 211, 5, 4085, 1, 0, 3967, 0, 11, 11, 41, 4082, 4090, 2987, 0, 4077, 4059, 0, 0, 10, 0, 16, 8, 107, 0, 4086, 13, 9, 501, 0, 0, 3347, 0, 4051] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 4093, 1, 1, 0, 117, 0, 0, 25, 0, 0, 2, 2, 1, 3643, 2, 4092, 2, 4089, 2, 1042, 2, 1, 4086, 2, 2, 2, 35, 3929, 18, 2, 4094, 1, 0, 4069, 0, 0, 2, 13, 4092, 4094, 3722, 0, 4091, 4091, 0, 0, 3, 0, 2, 2, 58, 0, 4094, 3, 2, 29, 2, 0, 3784, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 14, 0, 0, 13, 732, 5, 0, 0, 0, 0, 3744, 0, 4096, 0, 3974, 0, 131, 0, 0, 4081, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4069, 3644, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 46, 0, 4096, 0, 0, 0, 37, 0, 4096, 0, 4087] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 4096, 0, 0, 2, 0, 3836, 207, 0, 0, 0, 0, 0, 0, 0, 0, 311, 7, 0, 0, 0, 0, 4096, 3889, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 3744, 0, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 4042, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 504, 4054, 2978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 4095, 3590, 260, 162, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 4096, 0, 4096, 54, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8153, 8303, 8643, 8187, 8209, 7536, 8210, 8295] avg=8192.0000 max=8643 max_rank=2 min=7536 min_rank=5 imbalance_ratio=1.055054 max_load_reduction=56893 max_load_reduction_pct=0.868118 moved_to_echo=32250 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4, 0, 3836, 208, 0, 0, 0, 0, 0, 0, 0, 0, 238, 2, 0, 0, 0, 0, 4096, 3888, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 3766, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4091, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 504, 4087, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 4095, 3592, 260, 163, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 4096, 54, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8224, 7861, 8258, 8209, 8415, 8172, 8253] avg=8192.0000 max=8415 max_rank=5 min=7861 min_rank=2 imbalance_ratio=1.027222 max_load_reduction=57121 max_load_reduction_pct=0.871597 moved_to_echo=33049 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4083, 0, 0, 0, 368, 3723, 206, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 10, 10, 0, 4077, 3853, 0, 0, 0, 0, 0, 0, 3455, 10, 0, 16, 3749, 11, 2089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 10, 9, 9, 235, 3301, 211, 77, 0, 0, 0, 0, 0, 0, 0, 5, 4085, 1, 0, 3967, 0, 11, 11, 17, 0, 0, 0, 0, 0, 0, 0, 41, 502, 4090, 2987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 16, 8, 107, 0, 4086, 3580, 251, 162, 0, 0, 0, 0, 0, 13, 9, 501, 0, 0, 3347, 0, 4051, 54, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8381, 7970, 9337, 7936, 8097, 7620, 8220, 7975] avg=8192.0000 max=9337 max_rank=2 min=7620 min_rank=5 imbalance_ratio=1.139771 max_load_reduction=56199 max_load_reduction_pct=0.857529 moved_to_echo=31912 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 0, 0, 3836, 208, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 4096, 3888, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 3776, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 504, 4052, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 4096, 3592, 260, 164, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4096, 0, 4065, 54, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8144, 8088, 7887, 8255, 8209, 8552, 8157, 8244] avg=8192.0000 max=8552 max_rank=5 min=7887 min_rank=2 imbalance_ratio=1.043945 max_load_reduction=56984 max_load_reduction_pct=0.869507 moved_to_echo=33162 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[15, 0, 0, 4096, 0, 0, 9, 0, 3837, 208, 0, 0, 0, 0, 0, 0, 0, 8, 110, 1, 0, 0, 0, 0, 4096, 3888, 0, 0, 0, 0, 0, 0, 3833, 0, 0, 0, 3747, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4088, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 500, 4067, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 202, 0, 4093, 3566, 259, 162, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 4096, 0, 4096, 54, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8103, 7845, 8251, 8209, 8400, 8285, 8278] avg=8192.0000 max=8400 max_rank=5 min=7845 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=57136 max_load_reduction_pct=0.871826 moved_to_echo=33172 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 0, 0, 3836, 207, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 4096, 3889, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 3758, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 4069, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 504, 4096, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 46, 0, 4096, 3592, 260, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 54, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 8110, 7962, 8241, 8208, 8469, 8160, 8246] avg=8192.0000 max=8469 max_rank=5 min=7962 min_rank=2 imbalance_ratio=1.033813 max_load_reduction=57067 max_load_reduction_pct=0.870773 moved_to_echo=33083 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4093, 1, 1, 0, 117, 3833, 208, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 2, 2, 1, 4091, 3883, 0, 0, 0, 0, 0, 0, 3643, 2, 0, 2, 3776, 2, 1042, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 2, 2, 2, 35, 3929, 18, 78, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 1, 0, 4069, 0, 0, 2, 17, 0, 0, 0, 0, 0, 0, 0, 13, 503, 4094, 3722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 2, 58, 0, 4094, 3589, 259, 164, 0, 0, 0, 0, 0, 3, 2, 29, 2, 0, 3784, 0, 4094, 54, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8254, 8004, 8469, 8153, 8185, 8332, 8171, 7968] avg=8192.0000 max=8469 max_rank=2 min=7968 min_rank=7 imbalance_ratio=1.033813 max_load_reduction=57067 max_load_reduction_pct=0.870773 moved_to_echo=32656 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 14, 0, 3837, 208, 0, 0, 0, 0, 0, 0, 0, 13, 732, 5, 0, 0, 0, 0, 4096, 3888, 0, 0, 0, 0, 0, 0, 3744, 0, 0, 0, 3657, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4095, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 499, 4069, 3644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 4096, 3597, 259, 163, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 4096, 0, 4087, 57, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8155, 8734, 7532, 8255, 8210, 8212, 8161, 8277] avg=8192.0000 max=8734 max_rank=1 min=7532 min_rank=2 imbalance_ratio=1.066162 max_load_reduction=56802 max_load_reduction_pct=0.866730 moved_to_echo=32860 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 4096, 32640, 28928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 640, 256, 0, 31744, 0, 32768, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 512, 30592, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 32640, 0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 1664, 128, 0, 128, 128, 128, 32768, 31104, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 384, 31744, 256, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[31104, 128, 0, 128, 30080, 128, 4608, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 768, 0, 32768, 28800, 2176, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4085, 0, 4087, 3878, 8, 5, 79, 9, 204, 4086, 3921, 10, 0, 171, 13, 3815, 3682, 0, 10, 11, 9, 0, 9, 4084, 10, 9, 10, 0, 7, 0, 4030, 4073, 0, 7, 4086, 61, 9, 0, 25, 0, 0, 0, 10, 4076, 0, 10, 0, 340, 0, 76, 4050, 210, 0, 2, 6, 0, 0, 7, 4080, 0, 4085, 0, 0, 4081] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4090, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 4096, 3999, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4096, 76, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 0, 0, 4096, 4, 4096, 0, 1, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4096, 3999, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4095, 77, 0, 4089, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 1, 16, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4089, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3850, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 4096, 4093, 196, 0, 4083, 0, 0, 0, 10, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 49, 0, 0, 0, 0, 4095, 7, 4096, 3, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3935, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 138, 0, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 22, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4094, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4042, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 20, 0, 4091, 0, 0, 0, 23, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4086, 4096, 0, 0, 0, 0, 1, 4094, 4096, 0, 0, 0, 0, 4096, 3938, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4094, 101, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 1, 2, 0, 0, 4096, 0, 0, 53, 0, 0, 0, 0, 4096, 10, 4096, 2, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4093, 0, 4094, 3981, 3, 1, 21, 2, 217, 4093, 4062, 2, 0, 47, 4, 4059, 3822, 1, 3, 3, 3, 1, 2, 4093, 14, 2, 2, 0, 2, 0, 4089, 4091, 3, 2, 4093, 2, 2, 2, 57, 0, 0, 0, 2, 4092, 1, 2, 0, 31, 3, 7, 4091, 24, 0, 36, 0, 0, 0, 2, 4090, 0, 4093, 0, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 4096, 167, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 2, 2, 0, 0, 0, 0, 0, 0, 76, 0, 4095, 0, 0, 0, 0, 0, 3897, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 3991, 49, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8218, 8175, 8277, 8198, 8171, 8143, 8159, 8195] avg=8192.0000 max=8277 max_rank=2 min=8143 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32668 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 4096, 167, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4095, 2, 2, 0, 0, 0, 0, 0, 0, 77, 0, 4089, 0, 0, 0, 4, 0, 3897, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 16, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 3992, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8222, 8175, 8277, 8197, 8170, 8161, 8143, 8191] avg=8192.0000 max=8277 max_rank=2 min=8143 min_rank=6 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 4087, 3878, 8, 5, 79, 9, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 3905, 10, 0, 171, 13, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 3682, 0, 10, 11, 9, 0, 9, 4084, 167, 14, 0, 0, 0, 0, 0, 0, 10, 9, 10, 0, 7, 0, 4030, 4073, 2, 2, 0, 0, 0, 0, 0, 0, 0, 7, 4086, 61, 9, 0, 25, 0, 3886, 104, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4076, 0, 10, 0, 340, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 76, 4050, 210, 0, 2, 6, 0, 3982, 47, 0, 0, 0, 0, 0, 0, 0, 7, 4080, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8098, 8118, 7986, 8143, 8178, 8474, 8373, 8166] avg=8192.0000 max=8474 max_rank=5 min=7986 min_rank=2 imbalance_ratio=1.034424 max_load_reduction=57062 max_load_reduction_pct=0.870697 moved_to_echo=33191 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 4096, 167, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 2, 0, 0, 0, 0, 0, 0, 138, 0, 4093, 0, 0, 0, 2, 0, 3897, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 3993, 49, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8175, 8213, 8196, 8233, 8145, 8139, 8212] avg=8192.0000 max=8233 max_rank=4 min=8139 min_rank=6 imbalance_ratio=1.005005 max_load_reduction=57303 max_load_reduction_pct=0.874374 moved_to_echo=32729 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 0, 0, 0, 0, 0, 0, 4096, 167, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4093, 2, 2, 0, 0, 0, 0, 0, 0, 196, 0, 4083, 0, 0, 0, 10, 0, 3897, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 49, 0, 0, 3993, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8217, 8175, 8128, 8197, 8289, 8145, 8186, 8199] avg=8192.0000 max=8289 max_rank=4 min=8128 min_rank=2 imbalance_ratio=1.011841 max_load_reduction=57247 max_load_reduction_pct=0.873520 moved_to_echo=32819 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 4096, 167, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 2, 0, 0, 0, 0, 0, 0, 20, 0, 4091, 0, 0, 0, 23, 0, 3897, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 3993, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8221, 8175, 8320, 8196, 8134, 8147, 8152, 8191] avg=8192.0000 max=8320 max_rank=2 min=8134 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=57216 max_load_reduction_pct=0.873047 moved_to_echo=32624 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4072, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 0, 0, 0, 0, 0, 4096, 174, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4094, 7, 6, 0, 0, 0, 0, 0, 0, 101, 0, 4094, 0, 0, 0, 2, 0, 3901, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 2, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 53, 0, 0, 3990, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 2, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8203, 8169, 8225, 8205, 8202, 8147, 8187, 8198] avg=8192.0000 max=8225 max_rank=2 min=8147 min_rank=5 imbalance_ratio=1.004028 max_load_reduction=57311 max_load_reduction_pct=0.874496 moved_to_echo=32734 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 4094, 3981, 3, 1, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 4045, 2, 0, 47, 4, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 1, 3, 3, 3, 1, 2, 4093, 167, 15, 0, 0, 0, 0, 0, 0, 14, 2, 2, 0, 2, 0, 4089, 4091, 2, 2, 0, 0, 0, 0, 0, 0, 3, 2, 4093, 2, 2, 2, 57, 0, 3894, 104, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 1, 2, 0, 31, 4046, 0, 0, 0, 0, 0, 0, 0, 3, 7, 4091, 24, 0, 36, 0, 0, 3989, 47, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8134, 8374, 8110, 8204, 8159, 8174, 8197, 8184] avg=8192.0000 max=8374 max_rank=1 min=8110 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=57162 max_load_reduction_pct=0.872223 moved_to_echo=32714 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[256, 0, 32768, 32512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 128, 512, 32384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[640, 128, 32768, 128, 128, 128, 128, 0, 31232, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[31360, 128, 128, 128, 128, 128, 128, 32768, 1408, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 0, 32512, 128, 0, 256, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 256, 0, 256, 128, 0, 32000, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[3706, 0, 236, 8, 9, 0, 4085, 4089, 45, 12, 7, 0, 0, 0, 9, 9, 0, 1, 3875, 4012, 10, 618, 0, 3737, 0, 7, 4043, 4081, 0, 6, 4083, 48, 4084, 4086, 0, 0, 0, 0, 0, 11, 12, 1273, 47, 73, 0, 6, 0, 0, 4079, 4084, 0, 547, 9, 3880, 0, 0, 3410, 11, 12, 3151, 15, 10, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4093, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4025, 0, 0, 0, 4068, 5, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 4, 7, 0, 2, 0, 0, 0, 89, 0, 0, 0, 0, 2, 3, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3649, 0, 0, 0, 3981, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 8, 0, 0, 0, 0, 0, 554, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4055, 3978, 0, 0, 0, 4067, 1, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 39, 9, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4091, 3933, 0, 0, 0, 4061, 1, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 3, 6, 0, 1, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 3062, 0, 0, 0, 3446, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 5, 32, 0, 0, 0, 0, 0, 1659, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4094, 4092, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4086, 4068, 0, 0, 0, 4050, 3, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 27, 9, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 6, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[3999, 0, 69, 2, 2, 0, 4093, 4094, 18, 3, 2, 0, 0, 0, 2, 2, 0, 0, 4000, 3945, 3, 152, 2, 3905, 0, 2, 4085, 4093, 0, 0, 4093, 8, 4093, 4094, 0, 0, 0, 0, 1, 3, 3, 510, 58, 13, 0, 3, 0, 0, 4091, 4093, 0, 228, 2, 4067, 0, 0, 3932, 3, 3, 3761, 2, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3706, 0, 236, 8, 9, 0, 23, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 45, 12, 7, 0, 0, 0, 9, 9, 4084, 4074, 0, 0, 0, 0, 0, 0, 0, 1, 332, 4012, 10, 618, 0, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4043, 4081, 0, 6, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4069, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1273, 47, 73, 0, 6, 0, 0, 4062, 3374, 9, 0, 0, 0, 0, 0, 4015, 0, 0, 547, 9, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3410, 11, 12, 3151, 15, 10, 0, 0, 169, 64, 17, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8071, 8240, 8710, 8185, 8164, 8856, 8451, 6859] avg=8192.0000 max=8856 max_rank=5 min=6859 min_rank=7 imbalance_ratio=1.081055 max_load_reduction=56680 max_load_reduction_pct=0.864868 moved_to_echo=32330 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 24, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 1, 0, 347, 3978, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 39, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 3, 4070, 3531, 10, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 177, 64, 17, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8181, 8393, 8194, 8223, 7754, 8127, 8450] avg=8192.0000 max=8450 max_rank=7 min=7754 min_rank=5 imbalance_ratio=1.031494 max_load_reduction=57086 max_load_reduction_pct=0.871063 moved_to_echo=32554 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 24, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 350, 3933, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 3, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 0, 4071, 3562, 11, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 179, 64, 17, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8215, 8182, 8344, 8193, 8185, 7837, 8128, 8452] avg=8192.0000 max=8452 max_rank=7 min=7837 min_rank=5 imbalance_ratio=1.031738 max_load_reduction=57084 max_load_reduction_pct=0.871033 moved_to_echo=32602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 350, 3649, 0, 0, 0, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 1, 0, 0, 0, 0, 0, 4072, 3567, 11, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 179, 64, 17, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8216, 8181, 7980, 8192, 8183, 8205, 8128, 8451] avg=8192.0000 max=8451 max_rank=7 min=7980 min_rank=2 imbalance_ratio=1.031616 max_load_reduction=57085 max_load_reduction_pct=0.871048 moved_to_echo=32967 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 23, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 350, 4025, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 4, 7, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 2, 3, 4070, 3562, 9, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 179, 64, 17, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8210, 8183, 8443, 8197, 8188, 7735, 8128, 8452] avg=8192.0000 max=8452 max_rank=7 min=7735 min_rank=5 imbalance_ratio=1.031738 max_load_reduction=57084 max_load_reduction_pct=0.871033 moved_to_echo=32503 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 24, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 350, 3062, 0, 0, 0, 3446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 5, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1659, 0, 0, 0, 0, 0, 4, 4072, 3556, 11, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 178, 64, 17, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8214, 8181, 6858, 8192, 8212, 9302, 8126, 8451] avg=8192.0000 max=9302 max_rank=5 min=6858 min_rank=2 imbalance_ratio=1.135498 max_load_reduction=56234 max_load_reduction_pct=0.858063 moved_to_echo=34091 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 19, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 341, 4068, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 27, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 6, 4075, 3564, 9, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 181, 68, 21, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8207, 8184, 8459, 8195, 8207, 7698, 8124, 8462] avg=8192.0000 max=8462 max_rank=7 min=7698 min_rank=5 imbalance_ratio=1.032959 max_load_reduction=57074 max_load_reduction_pct=0.870880 moved_to_echo=32491 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3999, 0, 69, 2, 2, 0, 23, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3, 2, 0, 0, 0, 2, 2, 4093, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 342, 3945, 3, 152, 2, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4085, 4093, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4077, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 510, 58, 13, 0, 3, 0, 0, 4070, 3483, 9, 0, 0, 0, 0, 0, 4027, 0, 0, 228, 2, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 3, 3, 3761, 2, 2, 0, 0, 175, 64, 17, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8204, 8349, 8188, 8174, 8149, 8324, 7959] avg=8192.0000 max=8349 max_rank=2 min=7959 min_rank=7 imbalance_ratio=1.019165 max_load_reduction=57187 max_load_reduction_pct=0.872604 moved_to_echo=32606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32256, 0, 0, 896, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32000, 128, 128, 31488, 128, 128, 0, 0, 1536, 640, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32640, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32384, 0, 384, 128, 128, 0, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 4480, 128, 128, 0, 128, 128, 128, 32640, 28288, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 2816, 30720, 128, 896, 128, 31360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[931, 4096, 0, 2339, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 432, 4096, 0, 0, 0, 3555, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 4096, 4095, 0, 4047, 0, 1483, 0, 0, 0, 0, 0, 0, 184, 0, 4096, 0, 3413, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[109, 4014, 0, 4085, 10, 0, 6, 0, 0, 161, 0, 0, 0, 0, 4090, 1315, 4088, 1213, 4069, 0, 2, 0, 4085, 0, 11, 4084, 0, 0, 10, 72, 4077, 4083, 0, 11, 0, 19, 8, 1, 0, 2, 0, 4086, 3980, 0, 4083, 3798, 10, 4079, 92, 745, 0, 13, 4, 4, 5, 0, 0, 6, 4085, 14, 878, 0, 21, 8] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3481, 4096, 1, 1434, 0, 7, 0, 0, 2, 0, 0, 0, 0, 0, 4095, 0, 4091, 694, 4096, 0, 0, 0, 2820, 0, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4095, 4096, 0, 4091, 0, 1624, 0, 0, 0, 0, 0, 0, 91, 0, 4096, 0, 2144, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[2789, 4096, 0, 1291, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 891, 4096, 0, 0, 0, 3083, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4092, 0, 2245, 0, 0, 0, 0, 0, 0, 141, 0, 4096, 0, 1851, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[2029, 4096, 0, 1905, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4090, 2833, 4096, 0, 0, 0, 2197, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4096, 4096, 0, 4076, 0, 2567, 0, 0, 0, 0, 0, 0, 58, 0, 4096, 0, 720, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1478, 4096, 1, 2255, 0, 40, 0, 0, 1, 0, 0, 0, 0, 0, 4091, 1, 4088, 210, 4096, 0, 0, 0, 3219, 0, 0, 4096, 8, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 4096, 5, 4094, 4095, 0, 4034, 0, 1368, 0, 0, 0, 0, 0, 0, 126, 0, 4096, 0, 3654, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1624, 4096, 0, 1732, 0, 13, 0, 0, 0, 2, 0, 0, 0, 0, 4091, 0, 4082, 451, 4096, 0, 0, 0, 3372, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 4096, 4095, 0, 4063, 0, 1740, 0, 0, 0, 0, 0, 0, 98, 0, 4096, 0, 3304, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[570, 4087, 0, 3232, 2, 0, 2, 2, 0, 34, 0, 0, 0, 0, 4092, 303, 4094, 1191, 4092, 0, 0, 0, 4022, 0, 3, 4093, 0, 0, 2, 13, 4092, 4093, 0, 3, 0, 4, 2, 1, 0, 3, 1, 4093, 4075, 0, 4093, 3958, 3, 4054, 34, 536, 0, 4, 3, 0, 1, 0, 0, 2, 4093, 3, 2543, 0, 6, 2] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[931, 4096, 0, 2339, 0, 3, 0, 0, 175, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2326, 1556, 0, 0, 0, 0, 0, 0, 3913, 432, 0, 0, 0, 0, 3555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 81, 5, 0, 4095, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1483, 0, 0, 0, 0, 0, 0, 4096, 2540, 4, 0, 0, 0, 0, 0, 184, 0, 4096, 0, 3413, 0, 0, 0, 1689, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7554, 7973, 7900, 8182, 8188, 8228, 8123, 9388] avg=8192.0000 max=9388 max_rank=7 min=7554 min_rank=0 imbalance_ratio=1.145996 max_load_reduction=56148 max_load_reduction_pct=0.856750 moved_to_echo=33927 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[109, 4014, 0, 4085, 10, 0, 6, 0, 175, 10, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 4090, 1315, 2260, 1547, 0, 0, 0, 0, 0, 0, 3907, 1213, 0, 0, 2, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 10, 72, 4077, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 19, 8, 1, 0, 2, 4084, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 3798, 10, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 92, 745, 0, 13, 4, 4, 5, 0, 4083, 2522, 3, 0, 0, 0, 0, 0, 0, 6, 4085, 14, 878, 0, 21, 8, 1641, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8409, 9373, 9207, 8243, 8208, 7966, 7471, 6659] avg=8192.0000 max=9373 max_rank=1 min=6659 min_rank=7 imbalance_ratio=1.144165 max_load_reduction=56163 max_load_reduction_pct=0.856979 moved_to_echo=30304 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3481, 4096, 1, 1434, 0, 7, 0, 0, 175, 10, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4095, 0, 2326, 1557, 0, 0, 0, 0, 0, 0, 3910, 694, 0, 0, 0, 0, 2820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 81, 1, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1624, 0, 0, 0, 0, 0, 0, 4095, 2539, 3, 0, 0, 0, 0, 0, 91, 0, 4096, 0, 2144, 0, 0, 0, 1689, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9204, 7980, 7424, 8183, 8189, 8269, 8261, 8026] avg=8192.0000 max=9204 max_rank=0 min=7424 min_rank=2 imbalance_ratio=1.123535 max_load_reduction=56332 max_load_reduction_pct=0.859558 moved_to_echo=32745 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2789, 4096, 0, 1291, 0, 0, 0, 0, 175, 10, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 2326, 1557, 0, 0, 0, 0, 0, 0, 3915, 891, 0, 0, 0, 0, 3083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2245, 0, 0, 0, 0, 0, 0, 4096, 2539, 4, 0, 0, 0, 0, 0, 141, 0, 4096, 0, 1851, 0, 0, 0, 1689, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8361, 7980, 7889, 8182, 8188, 8269, 8884, 7783] avg=8192.0000 max=8884 max_rank=6 min=7783 min_rank=7 imbalance_ratio=1.084473 max_load_reduction=56652 max_load_reduction_pct=0.864441 moved_to_echo=33124 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1478, 4096, 1, 2255, 0, 40, 0, 0, 175, 10, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4091, 1, 2326, 1557, 0, 0, 0, 0, 0, 0, 3907, 210, 0, 0, 0, 0, 3219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 81, 5, 0, 4095, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1368, 0, 0, 0, 0, 0, 0, 4094, 2539, 3, 0, 0, 0, 0, 0, 126, 0, 4096, 0, 3654, 0, 0, 0, 1689, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8055, 7976, 7336, 8190, 8189, 8215, 8004, 9571] avg=8192.0000 max=9571 max_rank=7 min=7336 min_rank=2 imbalance_ratio=1.168335 max_load_reduction=55965 max_load_reduction_pct=0.853958 moved_to_echo=33979 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2029, 4096, 0, 1905, 0, 4, 0, 0, 175, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2326, 1557, 0, 0, 0, 0, 0, 0, 3909, 2833, 0, 0, 0, 0, 2197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 81, 2, 0, 4096, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2567, 0, 0, 0, 0, 0, 0, 4096, 2539, 3, 0, 0, 0, 0, 0, 58, 0, 4096, 0, 720, 0, 0, 0, 1689, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8219, 7978, 8939, 8182, 8189, 8255, 9205, 6569] avg=8192.0000 max=9205 max_rank=6 min=6569 min_rank=7 imbalance_ratio=1.123657 max_load_reduction=56331 max_load_reduction_pct=0.859543 moved_to_echo=32218 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1624, 4096, 0, 1732, 0, 13, 0, 0, 178, 11, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4091, 0, 2327, 1557, 0, 0, 0, 0, 0, 0, 3892, 451, 0, 0, 0, 0, 3372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 75, 5, 0, 4095, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1740, 0, 0, 0, 0, 0, 0, 4096, 2539, 3, 0, 0, 0, 0, 0, 98, 0, 4096, 0, 3304, 0, 0, 0, 1694, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7654, 7977, 7715, 8181, 8189, 8238, 8378, 9204] avg=8192.0000 max=9204 max_rank=7 min=7654 min_rank=0 imbalance_ratio=1.123535 max_load_reduction=56332 max_load_reduction_pct=0.859558 moved_to_echo=34009 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[570, 4087, 0, 3232, 2, 0, 2, 2, 175, 10, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 4092, 303, 2314, 1556, 0, 0, 0, 0, 0, 0, 3913, 1191, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 13, 4092, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 2, 1, 0, 3, 4093, 4090, 0, 0, 0, 0, 0, 0, 1, 0, 80, 0, 0, 3958, 3, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 34, 536, 0, 4, 3, 0, 1, 0, 4093, 2536, 3, 0, 0, 0, 0, 0, 0, 2, 4093, 3, 2543, 0, 6, 2, 1681, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8080, 8299, 9126, 8193, 8196, 8096, 7210, 8336] avg=8192.0000 max=9126 max_rank=2 min=7210 min_rank=6 imbalance_ratio=1.114014 max_load_reduction=56410 max_load_reduction_pct=0.860748 moved_to_echo=31838 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[31360, 7936, 0, 0, 128, 0, 26368, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[768, 128, 32768, 128, 18560, 0, 128, 128, 13568, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 12416, 0, 128, 128, 128, 128, 0, 32768, 20352, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[13056, 32768, 128, 18304, 128, 128, 128, 128, 1408, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 640, 128, 0, 32384, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 256, 0, 0, 0, 0, 32768, 1664, 18560, 12544, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[10, 0, 11, 315, 0, 0, 12, 0, 2195, 0, 11, 0, 0, 319, 0, 70, 26, 0, 9, 107, 9, 11, 0, 18, 2312, 14, 4, 4036, 9, 0, 9, 4086, 30, 0, 0, 3979, 0, 5, 4084, 3504, 50, 4085, 4067, 3934, 8, 3699, 4081, 0, 0, 4070, 4027, 0, 0, 57, 11, 0, 12, 4064, 4080, 1, 0, 4085, 0, 10] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 1469, 0, 0, 0, 1, 0, 0, 1, 0, 456, 0, 1, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4094, 2375, 0, 4095, 4096, 4096, 0, 4058, 4096, 0, 1, 4095, 4096, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4089, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 1096, 0, 0, 0, 2, 0, 0, 3, 0, 11, 0, 3, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4073, 3019, 0, 4096, 4096, 4096, 0, 4092, 4096, 0, 9, 4093, 4096, 5, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 2, 0, 0, 0, 856, 0, 2, 0, 1, 0, 0, 0, 0, 53, 0, 5, 4096, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 4079, 3274, 0, 4096, 4093, 4096, 0, 4084, 4096, 0, 4, 4094, 4096, 8, 0, 1, 0, 4, 0, 4092, 4096, 0, 4, 4089, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 2413, 0, 0, 0, 0, 0, 0, 1, 0, 1244, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4087, 709, 0, 4096, 4096, 4096, 0, 4001, 4096, 0, 0, 4095, 4096, 7, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 2999, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 865, 0, 4096, 4096, 4096, 0, 4076, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 1976, 0, 0, 0, 0, 0, 0, 2, 0, 31, 0, 2, 4096, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 4082, 2158, 0, 4096, 4095, 4096, 0, 4091, 4096, 0, 5, 4095, 4096, 12, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4091, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[3, 0, 3, 126, 2, 1, 3, 0, 3520, 0, 3, 0, 0, 78, 0, 36, 8, 0, 2, 37, 2, 2, 0, 4, 747, 3, 14, 4081, 8, 0, 2, 4093, 5, 0, 0, 4067, 0, 2, 4066, 3847, 11, 4094, 4084, 4087, 0, 3957, 4086, 0, 0, 4066, 4091, 0, 0, 15, 3, 0, 3, 4086, 4093, 2, 0, 4093, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 11, 315, 0, 0, 12, 0, 4073, 3966, 29, 26, 0, 0, 0, 0, 2195, 0, 11, 0, 0, 319, 0, 70, 2413, 634, 0, 0, 0, 0, 0, 0, 26, 0, 9, 107, 9, 11, 0, 18, 4085, 4055, 0, 0, 0, 0, 0, 0, 2312, 14, 4, 4036, 9, 0, 9, 3452, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 3979, 0, 5, 1642, 3504, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 101, 3934, 8, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4027, 0, 0, 57, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4064, 7, 1, 0, 4085, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8442, 5642, 8320, 9836, 9160, 7792, 8165, 8179] avg=8192.0000 max=9836 max_rank=3 min=5642 min_rank=1 imbalance_ratio=1.200684 max_load_reduction=55700 max_load_reduction_pct=0.849915 moved_to_echo=33296 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4089, 3994, 29, 26, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 1096, 2407, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 4096, 4070, 0, 0, 0, 0, 0, 0, 11, 0, 3, 4096, 0, 0, 0, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1637, 3019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4093, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 7, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8221, 8171, 7570, 8752, 8290, 8203, 8191] avg=8192.0000 max=8752 max_rank=4 min=7570 min_rank=3 imbalance_ratio=1.068359 max_load_reduction=56784 max_load_reduction_pct=0.866455 moved_to_echo=33436 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4089, 3994, 29, 26, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 1469, 2419, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4095, 4070, 0, 0, 0, 0, 0, 0, 456, 0, 1, 4096, 0, 0, 0, 3459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1646, 2375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 4096, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 8459, 8167, 8012, 8117, 8256, 8195, 8192] avg=8192.0000 max=8459 max_rank=1 min=8012 min_rank=3 imbalance_ratio=1.032593 max_load_reduction=57077 max_load_reduction_pct=0.870926 moved_to_echo=32760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4089, 3991, 29, 26, 0, 0, 0, 0, 4027, 0, 0, 2, 0, 0, 0, 856, 2410, 635, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 4096, 4070, 0, 0, 0, 0, 0, 0, 53, 0, 5, 4096, 0, 0, 0, 3457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1640, 3274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 4096, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 4096, 8, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 7, 0, 4, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 7930, 8169, 7611, 9010, 8282, 8207, 8192] avg=8192.0000 max=9010 max_rank=4 min=7611 min_rank=3 imbalance_ratio=1.099854 max_load_reduction=56526 max_load_reduction_pct=0.862518 moved_to_echo=33691 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4089, 3994, 29, 26, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 2999, 2420, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4070, 0, 0, 0, 0, 0, 0, 255, 0, 0, 4096, 0, 0, 0, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1647, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 4096, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 10148, 8166, 7811, 6608, 8274, 8192, 8199] avg=8192.0000 max=10148 max_rank=1 min=6608 min_rank=4 imbalance_ratio=1.238770 max_load_reduction=55388 max_load_reduction_pct=0.845154 moved_to_echo=31273 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4089, 3994, 29, 26, 0, 0, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 2413, 2415, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4070, 0, 0, 0, 0, 0, 0, 1244, 0, 0, 4096, 0, 0, 0, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1643, 709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 4096, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8138, 9387, 8167, 8800, 6448, 8199, 8198, 8199] avg=8192.0000 max=9387 max_rank=1 min=6448 min_rank=4 imbalance_ratio=1.145874 max_load_reduction=56149 max_load_reduction_pct=0.856766 moved_to_echo=31044 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4090, 3996, 33, 27, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 1976, 2417, 640, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4069, 0, 0, 0, 0, 0, 0, 31, 0, 2, 4096, 0, 0, 0, 3454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1632, 2158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4088, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 9074, 8167, 7583, 7886, 8286, 8201, 8193] avg=8192.0000 max=9074 max_rank=1 min=7583 min_rank=3 imbalance_ratio=1.107666 max_load_reduction=56462 max_load_reduction_pct=0.861542 moved_to_echo=32566 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 126, 2, 1, 3, 0, 4086, 3982, 29, 26, 0, 0, 0, 0, 3520, 0, 3, 0, 0, 78, 0, 36, 2403, 635, 0, 0, 0, 0, 0, 0, 8, 0, 2, 37, 2, 2, 0, 4, 4094, 4060, 0, 0, 0, 0, 0, 0, 747, 3, 14, 4081, 8, 0, 2, 3458, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4067, 0, 2, 1634, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 102, 4087, 0, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4091, 0, 0, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4086, 7, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8261, 6675, 8209, 8313, 9555, 8157, 8175, 8191] avg=8192.0000 max=9555 max_rank=4 min=6675 min_rank=1 imbalance_ratio=1.166382 max_load_reduction=55981 max_load_reduction_pct=0.854202 moved_to_echo=34078 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 512, 128, 128, 128, 0, 32768, 32000, 256, 256, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5120, 128, 128, 32768, 128, 0, 128, 27776, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 896, 32640, 128, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 32640, 0, 128, 13184, 19840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[29824, 0, 128, 128, 0, 512, 0, 11008, 19328, 5120, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 256, 128, 128, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[2584, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 236, 0, 0, 0, 0, 4089, 0, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4048, 0, 186, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[2931, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 4096, 0, 3295, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1434, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 4015, 0, 599, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[3942, 0, 0, 0, 274, 4077, 3991, 0, 8, 24, 0, 4087, 23, 0, 3741, 9, 4084, 0, 340, 0, 1, 4081, 102, 3842, 0, 10, 0, 98, 256, 11, 4071, 10, 0, 0, 10, 17, 25, 0, 7, 12, 0, 112, 0, 11, 13, 2, 3824, 4086, 13, 4079, 4071, 0, 10, 4083, 6, 0, 10, 4085, 0, 1, 3971, 0, 6, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[2992, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 4096, 0, 35, 0, 0, 0, 0, 4095, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1294, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4096, 0, 0, 4094, 0, 0, 0, 3985, 0, 44, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[3325, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 729, 0, 0, 0, 0, 4095, 0, 3770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 4048, 0, 47, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[2866, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 4096, 8, 0, 1, 0, 4096, 0, 10, 5, 0, 2, 0, 4096, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 0, 0, 0, 0, 4096, 4096, 0, 4091, 4096, 0, 0, 4092, 0, 0, 0, 4059, 0, 361, 4095, 4, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[3098, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 45, 0, 0, 0, 0, 4096, 0, 3865, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1198, 0, 0, 0, 0, 4096, 4096, 0, 4094, 4096, 0, 0, 4094, 0, 0, 0, 4008, 0, 74, 4096, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[3288, 2, 0, 0, 89, 4092, 4067, 0, 2, 5, 0, 4094, 5, 0, 3999, 3, 4093, 0, 76, 0, 2, 4092, 28, 3000, 0, 3, 0, 23, 1733, 3, 4053, 3, 0, 0, 2, 4, 4, 0, 2, 3, 0, 187, 0, 3, 3, 0, 4026, 4093, 1, 4092, 4091, 0, 3, 4093, 0, 0, 2, 4093, 0, 15, 4061, 0, 3, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2931, 0, 0, 0, 0, 940, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 3042, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3295, 0, 855, 117, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4090, 3156, 937, 0, 0, 0, 0, 0, 0, 1434, 0, 0, 0, 0, 4096, 3241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 599, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7967, 8240, 7153, 8410, 8184, 8771, 8196, 8615] avg=8192.0000 max=8771 max_rank=5 min=7153 min_rank=2 imbalance_ratio=1.070679 max_load_reduction=56765 max_load_reduction_pct=0.866165 moved_to_echo=33766 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2992, 0, 0, 0, 0, 940, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 3042, 2, 0, 0, 0, 4096, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3939, 0, 855, 117, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3156, 937, 0, 0, 0, 0, 0, 0, 1294, 0, 0, 0, 0, 4096, 3241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 44, 4001, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8028, 8240, 7175, 9053, 8183, 8631, 8195, 8031] avg=8192.0000 max=9053 max_rank=3 min=7175 min_rank=2 imbalance_ratio=1.105103 max_load_reduction=56483 max_load_reduction_pct=0.861862 moved_to_echo=33040 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3942, 0, 0, 0, 274, 936, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 24, 0, 4087, 23, 0, 3741, 9, 47, 0, 0, 0, 0, 0, 0, 0, 3032, 0, 340, 0, 1, 4081, 102, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 98, 256, 11, 4071, 10, 853, 117, 46, 0, 0, 0, 0, 0, 0, 0, 10, 17, 25, 0, 7, 12, 4079, 3141, 935, 0, 0, 0, 0, 0, 0, 112, 0, 11, 13, 2, 3824, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4079, 4071, 0, 10, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4085, 0, 1, 3878, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9143, 7939, 11398, 5472, 8226, 7195, 8183, 7980] avg=8192.0000 max=11398 max_rank=2 min=5472 min_rank=3 imbalance_ratio=1.391357 max_load_reduction=54138 max_load_reduction_pct=0.826080 moved_to_echo=31584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2584, 0, 0, 0, 0, 940, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 3042, 0, 0, 0, 0, 4096, 0, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3723, 0, 855, 117, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3156, 937, 0, 0, 0, 0, 0, 0, 1518, 0, 0, 0, 0, 4096, 3241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 186, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7620, 8240, 7374, 8831, 8185, 8855, 8196, 8235] avg=8192.0000 max=8855 max_rank=5 min=7374 min_rank=2 imbalance_ratio=1.080933 max_load_reduction=56681 max_load_reduction_pct=0.864883 moved_to_echo=33471 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2866, 0, 0, 0, 0, 940, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 3042, 8, 0, 1, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 4096, 0, 3937, 0, 855, 117, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3156, 937, 0, 0, 0, 0, 0, 0, 1047, 0, 0, 0, 0, 4096, 3241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 361, 4000, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7902, 8238, 7157, 9059, 8181, 8384, 8191, 8424] avg=8192.0000 max=9059 max_rank=3 min=7157 min_rank=2 imbalance_ratio=1.105835 max_load_reduction=56477 max_load_reduction_pct=0.861771 moved_to_echo=33180 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3325, 0, 0, 0, 0, 940, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 3042, 0, 0, 0, 0, 4096, 0, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3770, 0, 855, 117, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3156, 937, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 4096, 3241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 47, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8361, 8240, 7867, 8884, 8183, 7709, 8196, 8096] avg=8192.0000 max=8884 max_rank=3 min=7709 min_rank=5 imbalance_ratio=1.084473 max_load_reduction=56652 max_load_reduction_pct=0.864441 moved_to_echo=32184 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3098, 0, 0, 0, 0, 936, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 51, 0, 0, 0, 0, 0, 0, 0, 3034, 0, 0, 0, 0, 4096, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3865, 0, 857, 120, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 3160, 942, 0, 0, 0, 0, 0, 0, 1198, 0, 0, 0, 0, 4096, 3239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 74, 3992, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8130, 8243, 7175, 8991, 8198, 8533, 8190, 8076] avg=8192.0000 max=8991 max_rank=3 min=7175 min_rank=2 imbalance_ratio=1.097534 max_load_reduction=56545 max_load_reduction_pct=0.862808 moved_to_echo=32997 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3288, 2, 0, 0, 89, 939, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 4094, 5, 0, 3999, 3, 48, 0, 0, 0, 0, 0, 0, 0, 3039, 0, 76, 0, 2, 4092, 28, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 23, 1733, 3, 4053, 3, 854, 117, 47, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4, 0, 2, 3, 4091, 3153, 937, 0, 0, 0, 0, 0, 0, 187, 0, 3, 3, 0, 4026, 3239, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 4091, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 15, 3966, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8385, 8156, 10237, 6836, 8196, 7458, 8189, 8079] avg=8192.0000 max=10237 max_rank=2 min=6836 min_rank=3 imbalance_ratio=1.249634 max_load_reduction=55299 max_load_reduction_pct=0.843796 moved_to_echo=31922 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 128, 32768, 25344, 7552, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 7168, 0, 128, 128, 128, 32512, 25984, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[25088, 128, 0, 0, 384, 7552, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32384, 0, 1408, 31872, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 32384, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[24320, 128, 512, 128, 128, 32768, 256, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 26624, 128, 30720, 128, 6912, 1024, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[3377, 3864, 211, 0, 0, 7, 0, 0, 13, 10, 12, 9, 10, 0, 4049, 64, 869, 2, 8, 0, 3766, 4088, 3904, 2, 0, 0, 4085, 18, 0, 0, 0, 0, 78, 4079, 0, 4086, 708, 13, 37, 4078, 4085, 360, 0, 3307, 0, 17, 0, 4007, 9, 4087, 0, 7, 9, 0, 4075, 0, 9, 11, 12, 0, 0, 0, 4083, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1990, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4062, 0, 0, 0, 4096, 4091, 2194, 30, 0, 1, 4096, 0, 0, 0, 0, 13, 0, 4095, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[2508, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3453, 0, 0, 0, 4096, 4095, 2338, 8, 0, 1, 4096, 0, 0, 0, 0, 4, 0, 4095, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 1, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[2855, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3460, 0, 0, 0, 4096, 4096, 1866, 37, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[2137, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4046, 0, 0, 0, 4095, 4096, 2045, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[2417, 4001, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4022, 0, 0, 0, 4096, 4087, 1829, 11, 0, 4, 4096, 0, 0, 0, 0, 13, 0, 4092, 1, 4095, 0, 0, 0, 4096, 4092, 0, 0, 4096, 0, 0, 4, 4096, 0, 4093, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1118, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4065, 0, 0, 0, 4096, 4089, 3071, 56, 0, 2, 4096, 0, 0, 0, 0, 13, 0, 4094, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 2, 4096, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[3321, 3965, 53, 0, 0, 2, 0, 0, 3, 2, 3, 3, 2, 0, 4082, 12, 1423, 0, 3, 0, 4010, 4094, 3543, 2, 0, 0, 4093, 3, 4, 0, 0, 0, 25, 4088, 0, 4093, 196, 3, 10, 4092, 4093, 86, 0, 3842, 0, 4, 0, 4082, 2, 4094, 0, 2, 2, 0, 4091, 0, 3, 2, 6, 1, 0, 0, 4093, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1990, 4003, 0, 0, 0, 0, 0, 0, 1599, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4012, 68, 8, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 2429, 0, 2194, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 13, 4083, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7718, 8184, 8715, 8196, 8064, 8277, 8193, 8189] avg=8192.0000 max=8715 max_rank=2 min=7718 min_rank=0 imbalance_ratio=1.063843 max_load_reduction=56821 max_load_reduction_pct=0.867020 moved_to_echo=32723 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3377, 3864, 211, 0, 0, 7, 0, 0, 1470, 125, 0, 0, 0, 0, 0, 0, 13, 10, 12, 9, 10, 0, 4049, 64, 4001, 62, 9, 0, 0, 0, 0, 0, 869, 2, 8, 0, 2234, 0, 3904, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 18, 0, 0, 0, 0, 4079, 4, 0, 0, 0, 0, 0, 0, 78, 4079, 0, 0, 708, 13, 37, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 84, 360, 0, 3307, 0, 17, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4087, 0, 7, 9, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 11, 12, 0, 0, 0, 4083, 11, 4082, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9054, 8239, 7019, 8186, 8868, 7775, 8187, 8208] avg=8192.0000 max=9054 max_rank=0 min=7019 min_rank=2 imbalance_ratio=1.105225 max_load_reduction=56482 max_load_reduction_pct=0.861847 moved_to_echo=33038 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2508, 3978, 0, 0, 0, 0, 0, 0, 1599, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4012, 68, 8, 0, 0, 0, 0, 0, 3453, 0, 0, 0, 2429, 0, 2338, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4, 4087, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8211, 8184, 8228, 8191, 8064, 8277, 8192, 8189] avg=8192.0000 max=8277 max_rank=5 min=8064 min_rank=4 imbalance_ratio=1.010376 max_load_reduction=57259 max_load_reduction_pct=0.873703 moved_to_echo=32722 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2855, 4061, 0, 0, 0, 0, 0, 0, 1599, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4012, 68, 8, 0, 0, 0, 0, 0, 3460, 0, 0, 0, 2429, 0, 1866, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8641, 8184, 7792, 8196, 8065, 8276, 8193, 8189] avg=8192.0000 max=8641 max_rank=0 min=7792 min_rank=2 imbalance_ratio=1.054810 max_load_reduction=56895 max_load_reduction_pct=0.868149 moved_to_echo=32723 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2417, 4001, 0, 0, 0, 0, 0, 3, 1599, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4008, 68, 8, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 2429, 0, 1829, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 13, 4079, 4, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 4096, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8146, 8180, 8291, 8196, 8062, 8280, 8194, 8187] avg=8192.0000 max=8291 max_rank=2 min=8062 min_rank=4 imbalance_ratio=1.012085 max_load_reduction=57245 max_load_reduction_pct=0.873489 moved_to_echo=32723 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2137, 4052, 0, 0, 0, 0, 0, 0, 1598, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4012, 68, 8, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 2429, 0, 2045, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7913, 8184, 8528, 8187, 8066, 8276, 8193, 8189] avg=8192.0000 max=8528 max_rank=2 min=7913 min_rank=0 imbalance_ratio=1.041016 max_load_reduction=57008 max_load_reduction_pct=0.869873 moved_to_echo=32724 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1118, 3972, 0, 0, 0, 0, 0, 0, 1603, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4013, 72, 9, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 2421, 0, 3071, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 13, 4080, 4, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6821, 8190, 9613, 8195, 8059, 8277, 8193, 8188] avg=8192.0000 max=9613 max_rank=2 min=6821 min_rank=0 imbalance_ratio=1.173462 max_load_reduction=55923 max_load_reduction_pct=0.853317 moved_to_echo=32717 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[3321, 3965, 53, 0, 0, 2, 0, 0, 1565, 126, 0, 0, 0, 0, 0, 0, 3, 2, 3, 3, 2, 0, 4082, 12, 4009, 66, 9, 0, 0, 0, 0, 0, 1423, 0, 3, 0, 2379, 0, 3543, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 4, 0, 0, 0, 4085, 4, 0, 0, 0, 0, 0, 0, 25, 4088, 0, 0, 196, 3, 10, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 84, 86, 0, 3842, 0, 4, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 2, 2, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 6, 1, 0, 0, 4093, 3, 4089, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9032, 8191, 7350, 8189, 8288, 8098, 8191, 8197] avg=8192.0000 max=9032 max_rank=0 min=7350 min_rank=2 imbalance_ratio=1.102539 max_load_reduction=56504 max_load_reduction_pct=0.862183 moved_to_echo=32774 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25472, 128, 128, 0, 19200, 0, 20864, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[19840, 32000, 384, 0, 0, 128, 0, 128, 12672, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 0, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[768, 512, 0, 31744, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 1024, 128, 128, 31744, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 32128, 640, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3663, 0, 0, 4096, 0, 3440, 0, 0, 4089, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 9, 4096, 0, 4095, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3744, 0, 0, 4096, 0, 3468, 0, 1, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 982, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3554, 0, 0, 4096, 0, 3830, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 801, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 5, 4096, 0, 4095, 4096, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3402, 0, 0, 4096, 0, 3968, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 799, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 21, 4096, 0, 4096, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4077, 309, 9, 1424, 6, 259, 3718, 0, 4038, 9, 534, 1666, 17, 0, 0, 0, 4083, 0, 60, 0, 8, 4078, 0, 24, 15, 2, 16, 4070, 0, 4019, 0, 3202, 0, 10, 6, 4086, 0, 76, 0, 0, 11, 3827, 12, 0, 0, 1815, 13, 8, 134, 13, 4008, 0, 0, 18, 4084, 0, 0, 4024, 0, 3644, 4072, 0, 21, 11] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4093, 0, 0, 3512, 0, 0, 4096, 0, 3887, 0, 1, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 777, 0, 0, 0, 4094, 4, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 11, 13, 4096, 0, 4091, 4096, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3504, 0, 0, 4096, 0, 3746, 0, 0, 4093, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 934, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 4096, 0, 4094, 4096, 9, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4091, 120, 2, 2381, 2, 95, 3975, 0, 3950, 2, 238, 2605, 4, 0, 3, 0, 4093, 0, 9, 0, 2, 4092, 0, 4, 2, 1, 4, 4089, 0, 4087, 0, 2539, 0, 2, 2, 4094, 0, 6, 0, 0, 3, 4064, 3, 0, 0, 812, 4, 3, 21, 3, 4070, 0, 0, 4, 4093, 0, 0, 4081, 0, 3780, 4092, 0, 6, 3] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[912, 0, 0, 3663, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 0, 0, 4089, 0, 0, 2, 0, 679, 9, 8, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2715, 0, 4096, 0, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4001, 86, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3098, 702, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 4095, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8671, 8227, 8184, 7897, 8183, 7896, 8183, 8295] avg=8192.0000 max=8671 max_rank=0 min=7896 min_rank=5 imbalance_ratio=1.058472 max_load_reduction=56865 max_load_reduction_pct=0.867691 moved_to_echo=32557 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[907, 309, 9, 1424, 6, 259, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 9, 534, 1666, 17, 0, 0, 0, 674, 9, 8, 0, 0, 0, 0, 0, 4075, 0, 60, 0, 8, 4078, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2, 16, 2698, 0, 4019, 0, 3202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 6, 4086, 0, 76, 0, 0, 3977, 86, 0, 0, 0, 0, 0, 0, 11, 3827, 12, 0, 0, 1815, 13, 8, 3084, 698, 0, 0, 0, 0, 0, 0, 134, 13, 4008, 0, 0, 18, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 3644, 95, 0, 21, 11, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6632, 6955, 8245, 9952, 8241, 9468, 8248, 7795] avg=8192.0000 max=9952 max_rank=3 min=6632 min_rank=0 imbalance_ratio=1.214844 max_load_reduction=55584 max_load_reduction_pct=0.848145 moved_to_echo=33752 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[912, 0, 0, 3744, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 1, 4093, 0, 0, 0, 0, 679, 9, 8, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2715, 0, 4096, 0, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4001, 86, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3098, 702, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8752, 8258, 8184, 7793, 8183, 7896, 8183, 8287] avg=8192.0000 max=8752 max_rank=0 min=7793 min_rank=3 imbalance_ratio=1.068359 max_load_reduction=56784 max_load_reduction_pct=0.866455 moved_to_echo=32549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[912, 0, 0, 3554, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 0, 4096, 0, 0, 0, 0, 679, 9, 8, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2715, 0, 4096, 0, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4001, 86, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3098, 702, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 4095, 95, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8562, 8622, 8184, 7612, 8184, 7896, 8183, 8293] avg=8192.0000 max=8622 max_rank=1 min=7612 min_rank=3 imbalance_ratio=1.052490 max_load_reduction=56914 max_load_reduction_pct=0.868439 moved_to_echo=32556 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[911, 0, 0, 3512, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 1, 4087, 0, 0, 0, 0, 679, 9, 8, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2715, 0, 4096, 2, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 4001, 86, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 3096, 702, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 11, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 4091, 95, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8519, 8671, 8184, 7590, 8185, 7895, 8194, 8298] avg=8192.0000 max=8671 max_rank=1 min=7590 min_rank=3 imbalance_ratio=1.058472 max_load_reduction=56865 max_load_reduction_pct=0.867691 moved_to_echo=32572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[912, 0, 0, 3402, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 4096, 0, 0, 0, 0, 679, 9, 8, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2715, 0, 4096, 0, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4001, 86, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 3098, 702, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 0, 4096, 95, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8410, 8760, 8184, 7610, 8183, 7897, 8183, 8309] avg=8192.0000 max=8760 max_rank=1 min=7610 min_rank=3 imbalance_ratio=1.069336 max_load_reduction=56776 max_load_reduction_pct=0.866333 moved_to_echo=32572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[904, 0, 0, 3504, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3746, 0, 0, 4093, 0, 0, 2, 0, 681, 12, 13, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2708, 0, 4096, 0, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4002, 91, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3101, 707, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4094, 94, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8504, 8547, 8179, 7738, 8190, 7904, 8180, 8294] avg=8192.0000 max=8547 max_rank=1 min=7738 min_rank=3 imbalance_ratio=1.043335 max_load_reduction=56989 max_load_reduction_pct=0.869583 moved_to_echo=32568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[911, 120, 2, 2381, 2, 95, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 2, 238, 2605, 4, 0, 3, 0, 677, 9, 8, 0, 0, 0, 0, 0, 4085, 0, 9, 0, 2, 4092, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4, 2711, 0, 4087, 0, 2539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4094, 0, 6, 0, 0, 3997, 86, 0, 0, 0, 0, 0, 0, 3, 4064, 3, 0, 0, 812, 4, 3, 3094, 701, 0, 0, 0, 0, 0, 0, 21, 3, 4070, 0, 0, 4, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 3780, 95, 0, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7486, 7496, 8192, 9344, 8187, 8684, 8182, 7965] avg=8192.0000 max=9344 max_rank=3 min=7486 min_rank=0 imbalance_ratio=1.140625 max_load_reduction=56192 max_load_reduction_pct=0.857422 moved_to_echo=33018 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 32000, 768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[7296, 512, 128, 25216, 128, 384, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 21760, 0, 32768, 128, 11136, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[30336, 128, 896, 28928, 128, 0, 128, 0, 5504, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[32768, 0, 128, 0, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 0, 0, 32000, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32512, 128, 128, 0, 2688, 128, 128, 24832, 5632, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[256, 128, 32768, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2633, 244, 667, 476, 1493, 0, 1148, 1150, 1407, 2763, 1286, 676, 905, 1931, 0, 510, 481, 951, 1600, 783, 0, 1202, 2335, 1095, 168, 234, 2052, 865, 1617, 1367, 2547, 712, 4, 1821, 1746, 832, 655, 1361, 1590, 1270, 478, 1, 1881, 1341, 1, 1767, 965, 847, 1848, 37, 1042, 52, 241, 1018, 4, 762, 0, 592, 1264, 404, 2190, 589, 1120, 2515] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2368, 160, 934, 755, 1705, 0, 1656, 285, 711, 2607, 1254, 1352, 1633, 1467, 1, 433, 384, 1453, 970, 641, 0, 767, 2496, 1051, 234, 167, 2299, 918, 1174, 780, 1653, 1046, 12, 2796, 1484, 246, 615, 1345, 1655, 929, 356, 1, 1395, 1284, 1, 1995, 1067, 637, 2449, 93, 1228, 23, 267, 1404, 3, 1242, 0, 998, 1781, 1037, 2029, 726, 644, 2440] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1987, 148, 1175, 668, 1448, 0, 1723, 391, 975, 3004, 1172, 1150, 1638, 1512, 0, 254, 551, 1134, 1748, 730, 0, 762, 2254, 1094, 93, 145, 2396, 1241, 1982, 1450, 1435, 1114, 3, 1572, 1859, 234, 715, 1526, 1035, 772, 290, 0, 1269, 1463, 0, 1727, 1560, 835, 2280, 70, 1289, 3, 326, 1930, 2, 957, 1, 1075, 1051, 654, 1696, 1113, 773, 2082] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2371, 169, 1249, 849, 1711, 0, 1283, 521, 951, 2464, 1547, 1142, 1322, 1420, 0, 285, 527, 1279, 1605, 787, 1, 1015, 2527, 1191, 188, 91, 2676, 859, 1636, 1181, 1795, 614, 4, 2029, 1660, 583, 717, 1037, 1403, 413, 283, 0, 1315, 1238, 0, 1840, 1009, 1079, 2067, 86, 1456, 12, 224, 1134, 0, 1179, 0, 708, 1576, 1409, 1405, 1228, 883, 2303] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[2178, 164, 1228, 588, 1193, 0, 1217, 467, 1728, 2914, 1789, 1020, 1433, 1814, 0, 448, 400, 1276, 1769, 352, 0, 1216, 1872, 1107, 64, 106, 2467, 848, 1326, 804, 2042, 965, 1, 2072, 1795, 466, 571, 930, 1355, 1115, 253, 1, 1058, 1825, 0, 1843, 969, 357, 1825, 40, 1528, 12, 174, 1593, 1, 793, 1, 1314, 2157, 787, 1595, 1150, 946, 2214] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2201, 217, 1146, 405, 1701, 2, 1141, 853, 1243, 2728, 1510, 1580, 1649, 1376, 2, 236, 528, 1060, 1505, 900, 1, 782, 2451, 701, 174, 65, 2853, 877, 1693, 1171, 1815, 1074, 4, 1605, 1285, 417, 846, 1175, 1581, 241, 503, 2, 1211, 2172, 0, 2315, 999, 702, 2259, 26, 1340, 48, 406, 647, 4, 917, 3, 660, 1444, 744, 1542, 1118, 1598, 2082] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2224, 104, 1528, 983, 1433, 0, 1603, 442, 856, 2991, 1116, 1623, 1815, 1365, 3, 326, 433, 1307, 1462, 694, 1, 848, 2360, 1052, 82, 227, 2373, 1166, 2066, 862, 1683, 1223, 2, 2133, 2078, 668, 916, 987, 1156, 860, 356, 0, 897, 1598, 0, 1850, 1116, 899, 1750, 45, 873, 1, 376, 1539, 2, 730, 7, 366, 1380, 469, 2081, 1587, 680, 1883] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1597, 130, 2178, 759, 1928, 1, 773, 561, 444, 1801, 1091, 1724, 2123, 1347, 0, 592, 1349, 1947, 2247, 872, 0, 838, 2172, 1513, 56, 8, 2585, 1822, 468, 583, 1258, 1114, 4, 1456, 2425, 37, 1626, 995, 1844, 378, 475, 0, 684, 1113, 0, 1237, 908, 1130, 2028, 27, 861, 5, 177, 778, 34, 1605, 9, 1075, 1881, 722, 1877, 1486, 681, 2097] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2368, 160, 934, 755, 1705, 0, 1656, 285, 245, 254, 0, 0, 0, 0, 0, 0, 711, 1024, 1254, 1352, 1633, 1467, 1, 433, 0, 0, 0, 0, 0, 0, 0, 0, 384, 1453, 970, 641, 0, 767, 2106, 1051, 0, 0, 0, 0, 0, 0, 0, 0, 234, 167, 1469, 918, 1174, 780, 1653, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2542, 1484, 246, 615, 1345, 1655, 929, 0, 0, 0, 0, 0, 0, 0, 0, 356, 1, 1395, 1284, 1, 1995, 1067, 637, 830, 407, 145, 0, 0, 0, 0, 0, 2449, 93, 1228, 23, 267, 1404, 3, 1242, 1583, 753, 0, 0, 0, 0, 0, 0, 0, 998, 1781, 1037, 2029, 726, 644, 1280, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8362, 7875, 7372, 7441, 8828, 8118, 9045, 8495] avg=8192.0000 max=9045 max_rank=6 min=7372 min_rank=2 imbalance_ratio=1.104126 max_load_reduction=56491 max_load_reduction_pct=0.861984 moved_to_echo=34486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2633, 244, 667, 476, 1493, 0, 1148, 1150, 229, 165, 0, 0, 0, 0, 0, 0, 1407, 1085, 1286, 676, 905, 1931, 0, 510, 0, 0, 0, 0, 0, 0, 0, 0, 481, 951, 1600, 783, 0, 1202, 1970, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 168, 234, 1312, 865, 1617, 1367, 2547, 712, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1656, 1746, 832, 655, 1361, 1590, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 478, 1, 1881, 1341, 1, 1767, 965, 847, 740, 420, 136, 0, 0, 0, 0, 0, 1848, 37, 1042, 52, 241, 1018, 4, 762, 1678, 776, 0, 0, 0, 0, 0, 0, 0, 592, 1264, 404, 2190, 589, 1120, 1319, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8205, 7800, 8082, 8822, 9114, 8577, 7458, 7478] avg=8192.0000 max=9114 max_rank=4 min=7458 min_rank=6 imbalance_ratio=1.112549 max_load_reduction=56422 max_load_reduction_pct=0.860931 moved_to_echo=32627 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1987, 148, 1175, 668, 1448, 0, 1723, 391, 221, 143, 0, 0, 0, 0, 0, 0, 975, 1180, 1172, 1150, 1638, 1512, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 551, 1134, 1748, 730, 0, 762, 1902, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 93, 145, 1531, 1241, 1982, 1450, 1435, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1429, 1859, 234, 715, 1526, 1035, 772, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 1269, 1463, 0, 1727, 1560, 835, 865, 348, 131, 0, 0, 0, 0, 0, 2280, 70, 1289, 3, 326, 1930, 2, 957, 1824, 643, 0, 0, 0, 0, 0, 0, 1, 1075, 1051, 654, 1696, 1113, 773, 1091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7904, 7881, 7921, 8991, 7573, 8488, 9324, 7454] avg=8192.0000 max=9324 max_rank=6 min=7454 min_rank=7 imbalance_ratio=1.138184 max_load_reduction=56212 max_load_reduction_pct=0.857727 moved_to_echo=32839 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2201, 217, 1146, 405, 1701, 2, 1141, 853, 240, 146, 0, 0, 0, 0, 0, 0, 1243, 1071, 1510, 1580, 1649, 1376, 2, 236, 0, 0, 0, 0, 0, 0, 0, 0, 528, 1060, 1505, 900, 1, 782, 2068, 701, 0, 0, 0, 0, 0, 0, 0, 0, 174, 65, 1823, 877, 1693, 1171, 1815, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1459, 1285, 417, 846, 1175, 1581, 241, 0, 0, 0, 0, 0, 0, 0, 0, 503, 2, 1211, 2172, 0, 2315, 999, 702, 1030, 348, 143, 0, 0, 0, 0, 0, 2259, 26, 1340, 48, 406, 647, 4, 917, 1657, 643, 0, 0, 0, 0, 0, 0, 3, 660, 1444, 744, 1542, 1118, 1598, 1091, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8052, 8667, 7545, 8692, 7008, 9425, 7947, 8200] avg=8192.0000 max=9425 max_rank=5 min=7008 min_rank=4 imbalance_ratio=1.150513 max_load_reduction=56111 max_load_reduction_pct=0.856186 moved_to_echo=32580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2178, 164, 1228, 588, 1193, 0, 1217, 467, 183, 188, 0, 0, 0, 0, 0, 0, 1728, 1144, 1789, 1020, 1433, 1814, 0, 448, 0, 0, 0, 0, 0, 0, 0, 0, 400, 1276, 1769, 352, 0, 1216, 1580, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 64, 106, 1577, 848, 1326, 804, 2042, 965, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1884, 1795, 466, 571, 930, 1355, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 253, 1, 1058, 1825, 0, 1843, 969, 357, 890, 370, 109, 0, 0, 0, 0, 0, 1825, 40, 1528, 12, 174, 1593, 1, 793, 1770, 683, 0, 0, 0, 0, 0, 0, 1, 1314, 2157, 787, 1595, 1150, 946, 1161, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7406, 9376, 7700, 7732, 8117, 7675, 8419, 9111] avg=8192.0000 max=9376 max_rank=1 min=7406 min_rank=0 imbalance_ratio=1.144531 max_load_reduction=56160 max_load_reduction_pct=0.856934 moved_to_echo=33322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2371, 169, 1249, 849, 1711, 0, 1283, 521, 248, 184, 0, 0, 0, 0, 0, 0, 951, 968, 1547, 1142, 1322, 1420, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 527, 1279, 1605, 787, 1, 1015, 2132, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 188, 91, 1710, 859, 1636, 1181, 1795, 614, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1845, 1660, 583, 717, 1037, 1403, 413, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 1315, 1238, 0, 1840, 1009, 1079, 966, 385, 147, 0, 0, 0, 0, 0, 2067, 86, 1456, 12, 224, 1134, 0, 1179, 1496, 711, 0, 0, 0, 0, 0, 0, 0, 708, 1576, 1409, 1405, 1228, 883, 1207, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8585, 7635, 8537, 8074, 7662, 8262, 8365, 8416] avg=8192.0000 max=8585 max_rank=0 min=7635 min_rank=1 imbalance_ratio=1.047974 max_load_reduction=56951 max_load_reduction_pct=0.869003 moved_to_echo=32705 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2224, 104, 1528, 983, 1433, 0, 1603, 442, 231, 194, 0, 0, 0, 0, 0, 0, 856, 1174, 1116, 1623, 1815, 1365, 3, 326, 0, 0, 0, 0, 0, 0, 0, 0, 433, 1307, 1462, 694, 1, 848, 1991, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 82, 227, 1517, 1166, 2066, 862, 1683, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1939, 2078, 668, 916, 987, 1156, 860, 0, 0, 0, 0, 0, 0, 0, 0, 356, 0, 897, 1598, 0, 1850, 1116, 899, 856, 314, 138, 0, 0, 0, 0, 0, 1750, 45, 873, 1, 376, 1539, 2, 730, 1817, 581, 0, 0, 0, 0, 0, 0, 7, 366, 1380, 469, 2081, 1587, 680, 988, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8742, 8278, 7788, 8826, 8606, 8024, 7714, 7558] avg=8192.0000 max=8826 max_rank=3 min=7558 min_rank=7 imbalance_ratio=1.077393 max_load_reduction=56710 max_load_reduction_pct=0.865326 moved_to_echo=31902 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1597, 130, 2178, 759, 1928, 1, 773, 561, 216, 137, 0, 0, 0, 0, 0, 0, 444, 703, 1091, 1724, 2123, 1347, 0, 592, 0, 0, 0, 0, 0, 0, 0, 0, 1349, 1947, 2247, 872, 0, 838, 1825, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 56, 8, 1649, 1822, 468, 583, 1258, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1319, 2425, 37, 1626, 995, 1844, 378, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 684, 1113, 0, 1237, 908, 1130, 936, 353, 131, 0, 0, 0, 0, 0, 2028, 27, 861, 5, 177, 778, 34, 1605, 1098, 651, 0, 0, 0, 0, 0, 0, 9, 1075, 1881, 722, 1877, 1486, 681, 1093, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8280, 8024, 10591, 6958, 8628, 6967, 7264, 8824] avg=8192.0000 max=10591 max_rank=2 min=6958 min_rank=3 imbalance_ratio=1.292847 max_load_reduction=54945 max_load_reduction_pct=0.838394 moved_to_echo=31683 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[3072, 128, 9728, 12160, 128, 14592, 8704, 6528, 7168, 3072, 1152, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 14080, 14336, 3584, 6784, 9472, 11648, 6016, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[1152, 1152, 12672, 8704, 12032, 8320, 14336, 7936, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[17664, 1408, 10112, 5504, 12672, 128, 10624, 4736, 1920, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[16512, 512, 9728, 256, 2304, 10112, 128, 8192, 12928, 5504, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[8320, 8448, 10880, 10368, 12544, 12288, 128, 3200, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 6912, 12544, 6272, 14464, 9088, 7424, 9344, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[4736, 10496, 12928, 5760, 128, 7552, 15616, 8832, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4096, 2, 785, 0, 1855, 0, 0, 2166, 662, 0, 0, 2618, 3611, 0, 0, 2, 1, 0, 4092, 3955, 11, 5, 0, 728, 1, 4089, 4041, 70, 4091, 4, 0, 0, 2255, 0, 0, 3992, 65, 4096, 1, 1, 0, 0, 0, 4094, 1584, 0, 0, 4096, 4096, 0, 0, 0, 0, 4086, 261, 0, 0, 2, 0, 21] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 1, 178, 0, 2811, 18, 0, 3433, 682, 0, 0, 2594, 3352, 0, 6, 0, 0, 0, 4094, 4059, 92, 1, 0, 178, 0, 4087, 3109, 23, 4096, 0, 0, 0, 2621, 2, 0, 3964, 105, 4088, 1, 0, 0, 0, 0, 4096, 1032, 0, 1, 4095, 4094, 0, 1, 1, 2, 4083, 348, 0, 0, 0, 1, 90] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4096, 5, 1041, 0, 3038, 1, 0, 2508, 539, 0, 0, 2652, 3761, 0, 0, 0, 0, 1, 4094, 4084, 47, 0, 2, 3, 0, 3998, 3677, 0, 4095, 0, 0, 0, 1179, 0, 1, 3567, 444, 4095, 0, 1, 2, 0, 0, 4096, 1791, 0, 0, 4095, 4095, 0, 10, 1, 0, 4087, 367, 0, 0, 1, 0, 60] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4094, 0, 1089, 0, 2555, 3, 2, 3295, 739, 0, 0, 2328, 3802, 0, 6, 0, 0, 0, 4093, 4073, 183, 5, 3, 329, 0, 4037, 3537, 2, 4096, 0, 1, 0, 1698, 1, 2, 3905, 257, 4093, 0, 0, 0, 0, 0, 4096, 246, 0, 0, 4096, 4095, 0, 5, 0, 0, 4095, 500, 0, 0, 0, 0, 173] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4094, 0, 1255, 1, 3326, 1, 0, 3566, 396, 0, 0, 1940, 2972, 0, 0, 1, 0, 2, 4094, 4074, 15, 1, 1, 36, 2, 4051, 3696, 0, 4095, 1, 8, 0, 1487, 4, 0, 3315, 187, 4083, 0, 2, 0, 0, 0, 4096, 1808, 0, 0, 4079, 4089, 0, 5, 0, 12, 4094, 594, 0, 0, 0, 0, 52] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 0, 1, 4095, 1, 448, 1, 2433, 0, 0, 3810, 745, 0, 10, 2494, 3913, 1, 1, 1, 1, 0, 4091, 4088, 29, 3, 0, 71, 0, 4063, 3835, 1, 4095, 0, 2, 1, 1156, 2, 0, 3965, 415, 4088, 1, 0, 0, 0, 1, 4095, 523, 0, 0, 4085, 4095, 1, 7, 0, 3, 4095, 621, 0, 1, 0, 1, 145] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 0, 371, 0, 2060, 4, 0, 1891, 1451, 0, 0, 285, 3022, 0, 3, 1, 1, 0, 4049, 4054, 17, 167, 9, 1720, 0, 3921, 4081, 3, 4096, 1, 0, 0, 2710, 0, 0, 3741, 21, 4089, 0, 0, 0, 0, 0, 4095, 3043, 0, 1, 4093, 4096, 0, 6, 0, 2, 4095, 202, 1, 0, 1, 0, 36] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 4095, 0, 1361, 0, 2478, 1, 0, 3403, 683, 0, 1, 2125, 3654, 1, 0, 1, 1, 0, 4077, 4082, 7, 16, 3, 59, 0, 4038, 3849, 9, 4095, 0, 2, 1, 1414, 0, 0, 3693, 373, 4086, 1, 1, 0, 0, 2, 4095, 1084, 9, 0, 4066, 4091, 1, 15, 0, 7, 4095, 414, 0, 1, 0, 1, 37] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4096, 2, 785, 0, 1387, 1349, 185, 228, 53, 0, 0, 0, 1855, 0, 0, 1981, 662, 0, 0, 2618, 0, 0, 0, 0, 0, 0, 0, 0, 3611, 0, 0, 2, 1, 0, 620, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 11, 5, 0, 728, 1, 4036, 4041, 70, 0, 0, 0, 0, 0, 0, 0, 0, 2603, 4, 0, 0, 2255, 0, 0, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4096, 1, 1, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 1584, 0, 0, 4096, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 261, 0, 0, 2, 0, 21, 3472, 139, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8086, 7116, 8189, 8892, 8854, 8029, 8389, 7981] avg=8192.0000 max=8892 max_rank=3 min=7116 min_rank=1 imbalance_ratio=1.085449 max_load_reduction=56644 max_load_reduction_pct=0.864319 moved_to_echo=33253 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 1, 178, 0, 1386, 1350, 294, 228, 53, 0, 0, 0, 2811, 18, 0, 3139, 682, 0, 0, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 3352, 0, 6, 0, 0, 0, 620, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 92, 1, 0, 178, 0, 4034, 3109, 23, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 0, 0, 0, 2621, 2, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 105, 4088, 1, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 0, 1, 4095, 2708, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4083, 348, 0, 0, 0, 1, 90, 3474, 139, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7587, 9244, 8037, 7437, 9194, 8062, 7838, 8137] avg=8192.0000 max=9244 max_rank=1 min=7437 min_rank=3 imbalance_ratio=1.128418 max_load_reduction=56292 max_load_reduction_pct=0.858948 moved_to_echo=33231 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4094, 0, 1255, 1, 1385, 1350, 306, 228, 53, 0, 0, 0, 3326, 1, 0, 3260, 396, 0, 0, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 2972, 0, 0, 1, 0, 2, 620, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 1, 36, 2, 3998, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2606, 1, 8, 0, 1487, 4, 0, 3315, 0, 0, 0, 0, 0, 0, 0, 0, 187, 4083, 0, 2, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 1808, 0, 0, 4079, 2704, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4094, 594, 0, 0, 0, 0, 52, 3474, 139, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8673, 8923, 7669, 7749, 7421, 8140, 8596, 8365] avg=8192.0000 max=8923 max_rank=1 min=7421 min_rank=4 imbalance_ratio=1.089233 max_load_reduction=56613 max_load_reduction_pct=0.863846 moved_to_echo=32522 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 4094, 0, 1089, 0, 1387, 1350, 282, 228, 53, 0, 0, 0, 2555, 3, 2, 3013, 739, 0, 0, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 3802, 0, 6, 0, 0, 0, 620, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 183, 5, 3, 329, 0, 3984, 3537, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 0, 1, 0, 1698, 1, 2, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 257, 4093, 0, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 4096, 2708, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 500, 0, 0, 0, 0, 173, 3473, 139, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8485, 8640, 8501, 8043, 8214, 8218, 7055, 8380] avg=8192.0000 max=8640 max_rank=1 min=7055 min_rank=6 imbalance_ratio=1.054688 max_load_reduction=56896 max_load_reduction_pct=0.868164 moved_to_echo=31867 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4096, 5, 1041, 0, 1387, 1350, 215, 228, 52, 0, 0, 0, 3038, 1, 0, 2293, 539, 0, 0, 2652, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 0, 0, 0, 0, 1, 620, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 2, 3, 0, 3946, 3677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2606, 0, 0, 0, 1179, 0, 1, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 444, 4095, 0, 1, 2, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 1791, 0, 0, 4095, 2708, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 367, 0, 0, 1, 0, 60, 3474, 139, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8376, 8523, 8466, 7675, 7353, 8410, 8605, 8128] avg=8192.0000 max=8605 max_rank=6 min=7353 min_rank=4 imbalance_ratio=1.050415 max_load_reduction=56931 max_load_reduction_pct=0.868698 moved_to_echo=32496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 1, 4095, 1, 448, 1, 1387, 1350, 326, 228, 53, 0, 0, 0, 2433, 0, 0, 3484, 745, 0, 10, 2494, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 1, 1, 1, 1, 0, 620, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 29, 3, 0, 71, 0, 4010, 3835, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2606, 0, 2, 1, 1156, 2, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 415, 4088, 1, 0, 0, 0, 1, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 523, 0, 0, 4085, 2708, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 621, 0, 1, 0, 1, 145, 3471, 139, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7892, 9166, 8625, 7949, 7732, 8372, 7324, 8476] avg=8192.0000 max=9166 max_rank=1 min=7324 min_rank=6 imbalance_ratio=1.118896 max_load_reduction=56370 max_load_reduction_pct=0.860138 moved_to_echo=31904 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 0, 371, 0, 1391, 1355, 166, 230, 55, 0, 0, 0, 2060, 4, 0, 1725, 1451, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 3022, 0, 3, 1, 1, 0, 611, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 17, 167, 9, 1720, 0, 3866, 4081, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2600, 1, 0, 0, 2710, 0, 0, 3741, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4089, 0, 0, 0, 0, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 3043, 0, 1, 4093, 2705, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 202, 1, 0, 1, 0, 36, 3438, 141, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7665, 5525, 7692, 9863, 9052, 7975, 9848, 7916] avg=8192.0000 max=9863 max_rank=3 min=5525 min_rank=1 imbalance_ratio=1.203979 max_load_reduction=55673 max_load_reduction_pct=0.849503 moved_to_echo=34791 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 0, 4095, 0, 1361, 0, 1385, 1350, 292, 228, 53, 0, 0, 0, 2478, 1, 0, 3111, 683, 0, 1, 2125, 0, 0, 0, 0, 0, 0, 0, 0, 3654, 1, 0, 1, 1, 0, 618, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 7, 16, 3, 59, 0, 3985, 3849, 9, 0, 0, 0, 0, 0, 0, 0, 0, 2606, 0, 2, 1, 1414, 0, 0, 3693, 0, 0, 0, 0, 0, 0, 0, 0, 373, 4086, 1, 1, 0, 0, 2, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 9, 0, 4066, 2706, 1, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4095, 414, 0, 1, 0, 1, 37, 3459, 139, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8772, 8399, 8357, 7928, 7716, 8330, 7881, 8153] avg=8192.0000 max=8772 max_rank=0 min=7716 min_rank=4 imbalance_ratio=1.070801 max_load_reduction=56764 max_load_reduction_pct=0.866150 moved_to_echo=32080 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[20608, 128, 128, 22016, 6016, 0, 128, 17152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[20864, 128, 128, 128, 14592, 128, 128, 30208, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[28160, 128, 128, 128, 128, 128, 4992, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[512, 256, 128, 3200, 128, 31872, 29952, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[1920, 32768, 128, 128, 128, 0, 128, 30976, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11136, 128, 128, 32768, 21760, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 3328, 128, 128, 128, 128, 640, 27776, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 6528, 128, 11136, 10880, 2176, 1920, 512, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 13, 0, 3941, 0, 27, 0, 0, 1, 0, 3998, 4095, 4090, 0, 0, 4096, 0, 0, 4095, 0, 1, 65, 3178, 4082, 3175, 3924, 192, 0, 4064, 0, 0, 0, 33, 2037, 2057, 5, 1, 1435, 812, 0, 1, 3, 0, 8, 0, 351, 0, 3578, 1, 4095, 0, 0, 0, 7, 0, 4093, 0, 1, 0, 0, 3979, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 32, 0, 4079, 0, 2, 9, 0, 0, 0, 4073, 4093, 3962, 0, 0, 4096, 0, 0, 4096, 0, 0, 5, 3697, 4095, 1142, 4083, 318, 0, 4088, 0, 0, 0, 16, 2761, 1859, 3, 0, 1053, 1705, 0, 0, 2, 0, 1, 0, 1694, 0, 2261, 0, 4093, 0, 0, 0, 0, 0, 4080, 0, 0, 78, 0, 4049, 0, 11, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 50, 0, 4053, 1, 4, 1, 0, 0, 0, 3848, 4095, 3965, 0, 0, 4095, 1, 0, 4096, 0, 0, 1, 3173, 4087, 3293, 4089, 446, 0, 4093, 0, 0, 1, 73, 1779, 1578, 2, 3, 546, 1628, 0, 0, 1, 0, 0, 0, 2568, 0, 1681, 0, 4095, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 1, 4096, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1, 1, 1, 4087, 0, 0, 6, 0, 0, 1, 4062, 4094, 4052, 0, 1, 4095, 0, 1, 4095, 1, 0, 2, 3745, 4095, 1034, 4065, 301, 0, 4080, 1, 0, 0, 21, 1958, 2582, 0, 10, 1017, 1482, 1, 1, 0, 0, 0, 1, 2339, 0, 2024, 1, 4094, 0, 1, 0, 1, 1, 4093, 1, 1, 7, 0, 4076, 1, 1, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4036, 0, 6, 11, 0, 0, 0, 4043, 4095, 3942, 0, 0, 4095, 1, 0, 4094, 0, 0, 10, 3371, 4096, 2773, 3928, 501, 0, 4082, 0, 1, 0, 56, 2136, 2970, 1, 5, 995, 547, 0, 0, 0, 0, 0, 0, 1195, 0, 2253, 0, 4095, 1, 1, 1, 2, 0, 4094, 0, 1, 2, 0, 4087, 0, 9, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 8, 0, 4052, 0, 2, 7, 0, 0, 0, 4009, 4094, 4069, 0, 0, 4096, 2, 0, 4096, 0, 0, 1, 2919, 4090, 2829, 4066, 183, 0, 4079, 0, 0, 0, 261, 1526, 3055, 5, 5, 935, 896, 0, 0, 0, 0, 0, 0, 871, 0, 3154, 0, 4094, 0, 0, 0, 3, 0, 4092, 0, 0, 0, 0, 4035, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 160, 0, 3675, 1, 0, 1, 0, 0, 0, 4059, 4095, 4092, 0, 0, 4096, 0, 0, 4096, 0, 0, 113, 3616, 4096, 3040, 4068, 108, 0, 4089, 0, 1, 1, 23, 2861, 1899, 1, 0, 1198, 399, 0, 0, 1, 0, 4, 0, 120, 0, 3604, 0, 4095, 0, 0, 0, 2, 0, 4093, 0, 0, 3, 0, 3818, 0, 8, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 1, 4035, 0, 1, 0, 0, 0, 0, 4081, 4095, 3969, 0, 1, 4095, 0, 1, 4095, 1, 0, 23, 3869, 4092, 2106, 4066, 467, 1, 4090, 1, 0, 0, 48, 1406, 1671, 1, 17, 638, 1587, 1, 1, 0, 0, 0, 1, 1590, 0, 3193, 1, 4095, 0, 1, 0, 3, 1, 4093, 0, 1, 1, 0, 4092, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 13, 0, 3941, 0, 27, 0, 0, 4096, 58, 0, 0, 0, 0, 0, 0, 1, 0, 3998, 153, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 0, 1, 65, 3178, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 3175, 3924, 192, 0, 1412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 2037, 2057, 5, 1, 1435, 812, 0, 1733, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 8, 0, 351, 0, 3578, 3207, 917, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 7, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3979, 0, 2, 0, 3884, 262, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8135, 8242, 7952, 8703, 8113, 8065, 8198, 8128] avg=8192.0000 max=8703 max_rank=3 min=7952 min_rank=2 imbalance_ratio=1.062378 max_load_reduction=56833 max_load_reduction_pct=0.867203 moved_to_echo=32504 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 50, 0, 4053, 1, 4, 1, 0, 4095, 58, 0, 0, 0, 0, 0, 0, 0, 0, 3848, 153, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 627, 0, 0, 1, 3173, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 3293, 4089, 446, 0, 1421, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 73, 1779, 1578, 2, 3, 546, 1628, 0, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2568, 0, 1681, 3207, 924, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 1, 3884, 262, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8262, 7966, 7889, 9250, 7355, 8381, 8189, 8244] avg=8192.0000 max=9250 max_rank=3 min=7355 min_rank=4 imbalance_ratio=1.129150 max_load_reduction=56286 max_load_reduction_pct=0.858856 moved_to_echo=32169 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 32, 0, 4079, 0, 2, 9, 0, 4096, 58, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 153, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 5, 3697, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 4083, 318, 0, 1420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2761, 1859, 3, 0, 1053, 1705, 0, 1744, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 1694, 0, 2261, 3207, 922, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 4049, 0, 11, 0, 3882, 262, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8276, 8188, 8424, 6963, 9141, 8087, 8175, 8282] avg=8192.0000 max=9141 max_rank=4 min=6963 min_rank=3 imbalance_ratio=1.115845 max_load_reduction=56395 max_load_reduction_pct=0.860519 moved_to_echo=33685 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 4087, 0, 0, 6, 0, 4095, 58, 0, 0, 0, 0, 0, 0, 0, 1, 4062, 153, 4052, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 626, 1, 0, 2, 3745, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 4065, 301, 0, 1417, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1958, 2582, 0, 10, 1017, 1482, 1, 1740, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2339, 0, 2024, 3207, 921, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 1, 1, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 1, 1, 7, 0, 4076, 1, 1, 1, 3883, 262, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8249, 8269, 8470, 6818, 8811, 8493, 8193, 8233] avg=8192.0000 max=8811 max_rank=4 min=6818 min_rank=3 imbalance_ratio=1.075562 max_load_reduction=56725 max_load_reduction_pct=0.865555 moved_to_echo=33730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 4052, 0, 2, 7, 0, 4096, 58, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 153, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 627, 0, 0, 1, 2919, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2829, 4066, 183, 0, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 1526, 3055, 5, 5, 935, 896, 0, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 871, 0, 3154, 3207, 920, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 2, 0, 3883, 262, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8223, 8231, 7639, 8495, 8423, 8152, 8191, 8182] avg=8192.0000 max=8495 max_rank=3 min=7639 min_rank=2 imbalance_ratio=1.036987 max_load_reduction=57041 max_load_reduction_pct=0.870377 moved_to_echo=32948 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4036, 0, 6, 11, 0, 4095, 58, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 153, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 626, 0, 0, 10, 3371, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2773, 3928, 501, 0, 1418, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 2136, 2970, 1, 5, 995, 547, 0, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1195, 0, 2253, 3206, 921, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 2, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4087, 0, 9, 0, 3884, 262, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8138, 8104, 8621, 8451, 7575, 8196, 8245] avg=8192.0000 max=8621 max_rank=3 min=7575 min_rank=5 imbalance_ratio=1.052368 max_load_reduction=56915 max_load_reduction_pct=0.868454 moved_to_echo=32467 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 160, 0, 3675, 1, 0, 1, 0, 4096, 61, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 147, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 113, 3616, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3040, 4068, 108, 0, 1406, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2861, 1899, 1, 0, 1198, 399, 0, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 120, 0, 3604, 3212, 927, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 4093, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3818, 0, 8, 0, 3887, 263, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7994, 8298, 8446, 8624, 8129, 7868, 8198, 7979] avg=8192.0000 max=8624 max_rank=3 min=7868 min_rank=5 imbalance_ratio=1.052734 max_load_reduction=56912 max_load_reduction_pct=0.868408 moved_to_echo=32174 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 4035, 0, 1, 0, 0, 4095, 58, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 153, 3969, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 626, 1, 0, 23, 3869, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2106, 4066, 467, 1, 1421, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1406, 1671, 1, 17, 638, 1587, 1, 1744, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1590, 0, 3193, 3207, 923, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 3, 1, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4092, 0, 2, 1, 3884, 262, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8191, 8204, 8612, 8062, 7113, 8915, 8196, 8243] avg=8192.0000 max=8915 max_rank=5 min=7113 min_rank=4 imbalance_ratio=1.088257 max_load_reduction=56621 max_load_reduction_pct=0.863968 moved_to_echo=32467 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 10752, 0, 21760, 25728, 7424, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[640, 16512, 17792, 128, 128, 7936, 9088, 128, 13952, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 384, 128, 32000, 128, 128, 128, 0, 32768, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 32256, 128, 128, 128, 31104, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[19456, 32384, 2560, 128, 11392, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32256, 1280, 32256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 5120, 128, 128, 256, 27648, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3760, 0, 0, 3539, 4, 4039, 0, 0, 0, 31, 0, 4024, 4094, 0, 4062, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 0, 4096, 4096, 0, 4096, 4090, 0, 1, 0, 0, 0, 9, 4096, 0, 0, 276, 4077, 0, 3, 0, 4096, 0, 757, 0, 1, 2, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2831, 0, 0, 3239, 1, 4090, 0, 0, 0, 80, 0, 3823, 4091, 0, 4080, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 4096, 4096, 0, 4092, 4094, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 1684, 3953, 0, 5, 0, 4096, 0, 795, 0, 2, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 3787, 0, 0, 3449, 2, 4033, 0, 1, 0, 26, 0, 3385, 4093, 0, 4009, 4095, 1, 1, 1, 4085, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 4096, 0, 4095, 4095, 0, 4096, 4066, 0, 3, 0, 1, 1, 21, 4095, 0, 0, 1283, 4038, 1, 1, 0, 4095, 0, 576, 1, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2883, 0, 0, 3914, 2, 4091, 0, 0, 0, 15, 0, 3615, 4095, 0, 4059, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 4096, 4096, 0, 4091, 4096, 0, 1, 0, 1, 0, 5, 4096, 0, 0, 1041, 4076, 0, 1, 0, 4096, 0, 877, 0, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3360, 0, 0, 3168, 0, 4088, 0, 0, 0, 20, 0, 3725, 4093, 0, 4077, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 4096, 4096, 0, 4090, 4064, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 1455, 4071, 0, 7, 0, 4096, 0, 640, 0, 5, 0, 2, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 3203, 1, 0, 3525, 0, 4095, 0, 1, 0, 32, 4, 3683, 4094, 0, 4089, 4095, 1, 1, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 4095, 0, 4095, 4094, 1, 4094, 4094, 1, 0, 1, 0, 0, 0, 4095, 0, 1, 1352, 4044, 0, 4, 0, 4092, 0, 544, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 3641, 1, 3929, 0, 0, 0, 43, 0, 3436, 4094, 0, 4064, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 4096, 4096, 0, 4096, 4064, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 3388, 820, 0, 3, 0, 4092, 0, 1196, 0, 0, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 2971, 1, 1, 3604, 5, 4095, 0, 1, 0, 6, 0, 3773, 4093, 0, 4034, 4095, 1, 1, 1, 4093, 0, 0, 3, 0, 1, 0, 0, 1, 0, 1, 1, 4095, 3, 4095, 4095, 0, 4091, 4094, 1, 0, 1, 0, 0, 0, 4095, 0, 1, 833, 4030, 0, 6, 0, 4094, 0, 1216, 1, 0, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 3787, 0, 0, 3449, 785, 534, 0, 0, 0, 0, 0, 0, 2, 4033, 0, 1, 0, 26, 0, 3385, 417, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4009, 0, 1, 1, 1, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4066, 0, 3, 0, 1, 1, 21, 8, 0, 0, 0, 0, 0, 0, 0, 3136, 0, 0, 1283, 4038, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 576, 1, 0, 0, 0, 0, 3268, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8556, 7864, 8137, 8191, 8192, 8196, 8459, 7941] avg=8192.0000 max=8556 max_rank=0 min=7864 min_rank=1 imbalance_ratio=1.044434 max_load_reduction=56980 max_load_reduction_pct=0.869446 moved_to_echo=32788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3760, 0, 0, 3539, 785, 534, 0, 0, 0, 0, 0, 0, 4, 4039, 0, 0, 0, 31, 0, 4024, 417, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4062, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 1, 0, 0, 0, 9, 8, 0, 0, 0, 0, 0, 0, 0, 3137, 0, 0, 276, 4077, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 757, 0, 1, 2, 1, 0, 3269, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8618, 8515, 8195, 8193, 8192, 8204, 7493, 8126] avg=8192.0000 max=8618 max_rank=0 min=7493 min_rank=6 imbalance_ratio=1.052002 max_load_reduction=56918 max_load_reduction_pct=0.868500 moved_to_echo=32015 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3360, 0, 0, 3168, 785, 534, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 20, 0, 3725, 417, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4077, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4064, 0, 0, 0, 0, 0, 3, 8, 0, 0, 0, 0, 0, 0, 0, 3137, 0, 0, 1455, 4071, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 640, 0, 5, 0, 2, 0, 3268, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7847, 8250, 8209, 8191, 8192, 8165, 8670, 8012] avg=8192.0000 max=8670 max_rank=6 min=7847 min_rank=0 imbalance_ratio=1.058350 max_load_reduction=56866 max_load_reduction_pct=0.867706 moved_to_echo=33039 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2831, 0, 0, 3239, 785, 534, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 80, 0, 3823, 417, 0, 0, 0, 0, 0, 0, 0, 39, 0, 4080, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4094, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, 0, 0, 0, 3137, 0, 0, 1684, 3953, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 795, 0, 2, 0, 2, 0, 3267, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7389, 8411, 8213, 8191, 8192, 8198, 8779, 8163] avg=8192.0000 max=8779 max_rank=6 min=7389 min_rank=0 imbalance_ratio=1.071655 max_load_reduction=56757 max_load_reduction_pct=0.866043 moved_to_echo=33332 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 3203, 1, 0, 3525, 785, 534, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 32, 4, 3683, 417, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4089, 0, 1, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 1, 0, 1, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 3136, 0, 1, 1352, 4044, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 544, 1, 0, 0, 0, 1, 3269, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8051, 8232, 8227, 8191, 8192, 8198, 8537, 7908] avg=8192.0000 max=8537 max_rank=6 min=7908 min_rank=7 imbalance_ratio=1.042114 max_load_reduction=56999 max_load_reduction_pct=0.869736 moved_to_echo=32835 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2883, 0, 0, 3914, 786, 534, 0, 0, 0, 0, 0, 0, 2, 4091, 0, 0, 0, 15, 0, 3615, 417, 0, 0, 0, 0, 0, 0, 0, 39, 0, 4059, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 1, 0, 1, 0, 5, 8, 0, 0, 0, 0, 0, 0, 0, 3137, 0, 0, 1041, 4076, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 877, 0, 0, 0, 0, 0, 3270, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8117, 8140, 8194, 8192, 8192, 8202, 8255, 8244] avg=8192.0000 max=8255 max_rank=6 min=8117 min_rank=0 imbalance_ratio=1.007690 max_load_reduction=57281 max_load_reduction_pct=0.874039 moved_to_echo=32893 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 3641, 790, 539, 0, 0, 0, 0, 0, 0, 1, 3929, 0, 0, 0, 43, 0, 3436, 418, 0, 0, 0, 0, 0, 0, 0, 31, 0, 4064, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4064, 0, 0, 0, 0, 0, 2, 13, 3, 0, 0, 0, 0, 0, 0, 3126, 0, 0, 3388, 820, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1196, 0, 0, 0, 1, 0, 3273, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9060, 7827, 8191, 8192, 8189, 8178, 7337, 8562] avg=8192.0000 max=9060 max_rank=0 min=7337 min_rank=6 imbalance_ratio=1.105957 max_load_reduction=56476 max_load_reduction_pct=0.861755 moved_to_echo=32266 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 2971, 1, 1, 3604, 785, 534, 0, 0, 0, 0, 0, 0, 5, 4095, 0, 1, 0, 6, 0, 3773, 417, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4034, 0, 1, 1, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 1, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4094, 1, 0, 1, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 3136, 0, 1, 833, 4030, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1216, 1, 0, 0, 0, 1, 3268, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7898, 8297, 8170, 8195, 8195, 8195, 8006, 8580] avg=8192.0000 max=8580 max_rank=7 min=7898 min_rank=0 imbalance_ratio=1.047363 max_load_reduction=56956 max_load_reduction_pct=0.869080 moved_to_echo=32976 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[32768, 0, 6656, 128, 128, 128, 128, 128, 26240, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[384, 0, 32512, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25088, 0, 128, 11392, 29184, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32512, 0, 128, 0, 256, 128, 29568, 3456, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 27008, 128, 128, 28160, 6400, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4096, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 4091, 1, 0, 0, 97, 0, 0, 4076, 0, 4093, 0, 4, 4096, 0, 2770, 0, 0, 4, 3, 82, 0, 0, 4096, 670, 0, 4096, 0, 4096, 0, 0, 0, 0, 4091, 4093, 4096, 0, 0, 1, 0, 4096, 0, 3, 0, 4094, 4075, 0, 0, 516, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 4092, 3, 0, 0, 68, 0, 0, 3992, 0, 4093, 0, 14, 4096, 0, 3089, 0, 0, 4, 2, 44, 2, 0, 4095, 15, 0, 4096, 0, 4096, 0, 0, 0, 0, 4088, 4095, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 4093, 4085, 0, 0, 990, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4096, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 4093, 2, 0, 0, 40, 0, 0, 4067, 0, 4095, 0, 11, 4095, 1, 2369, 0, 0, 7, 1, 21, 1, 0, 4095, 594, 0, 4096, 0, 4095, 0, 0, 0, 0, 4085, 4095, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 4094, 4073, 0, 0, 1119, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 1, 0, 0, 4093, 1, 0, 1, 1, 0, 4094, 0, 0, 1, 77, 0, 1, 3988, 1, 4095, 1, 33, 4083, 0, 1794, 0, 0, 11, 0, 305, 2, 1, 4095, 1529, 1, 4095, 0, 4094, 1, 0, 1, 1, 4092, 4095, 4095, 1, 0, 0, 0, 4095, 0, 1, 0, 4095, 4022, 0, 1, 541, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 1, 1, 1, 4091, 2, 1, 1, 1, 0, 4095, 0, 1, 0, 174, 0, 1, 4088, 1, 4094, 0, 7, 4095, 0, 2878, 0, 0, 3, 0, 20, 0, 0, 4095, 37, 0, 4094, 1, 4094, 1, 0, 1, 1, 4089, 4090, 4095, 1, 0, 0, 0, 4094, 0, 1, 1, 4095, 4085, 1, 1, 1006, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 216, 0, 0, 4049, 0, 4096, 0, 4, 4096, 1, 3068, 0, 1, 1, 1, 52, 0, 0, 4095, 64, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4063, 0, 0, 774, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4056, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 4095, 2, 0, 0, 1454, 0, 0, 4053, 0, 4094, 0, 130, 4000, 1, 1161, 0, 0, 0, 1, 378, 0, 1, 4096, 1217, 0, 4096, 0, 4096, 0, 0, 0, 0, 4088, 4093, 4095, 0, 0, 1, 0, 4096, 0, 0, 0, 4095, 4026, 0, 0, 15, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 1, 1, 0, 4090, 4, 1, 1, 1, 0, 4089, 0, 1, 0, 35, 0, 1, 3999, 0, 4095, 1, 1, 4095, 0, 3003, 0, 1, 4, 0, 15, 0, 0, 4095, 134, 0, 4095, 0, 4095, 1, 0, 1, 1, 4091, 4094, 4095, 1, 0, 0, 0, 4095, 0, 0, 1, 4094, 4081, 1, 1, 1024, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 0, 1, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 1, 1, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 77, 0, 1, 3988, 1, 3880, 0, 0, 0, 0, 0, 0, 0, 1568, 1, 33, 4083, 0, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 305, 2, 1, 4095, 1529, 1, 2527, 713, 203, 0, 0, 0, 0, 0, 7, 0, 4094, 1, 0, 1, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3379, 4022, 0, 1, 541, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8193, 7948, 7479, 9387, 8196, 8193, 7945] avg=8192.0000 max=9387 max_rank=4 min=7479 min_rank=3 imbalance_ratio=1.145874 max_load_reduction=56149 max_load_reduction_pct=0.856766 moved_to_echo=33721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4096, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 40, 0, 0, 4067, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 1568, 0, 11, 4095, 1, 2369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 21, 1, 0, 4095, 594, 0, 2527, 712, 203, 0, 0, 0, 0, 0, 7, 0, 4095, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3379, 4073, 0, 0, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8190, 7990, 8044, 8161, 8187, 8195, 8571] avg=8192.0000 max=8571 max_rank=7 min=7990 min_rank=2 imbalance_ratio=1.046265 max_load_reduction=56965 max_load_reduction_pct=0.869217 moved_to_echo=33114 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 68, 0, 0, 3992, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 1567, 0, 14, 4096, 0, 3089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 44, 2, 0, 4095, 15, 0, 2526, 712, 203, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3378, 4085, 0, 0, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8190, 7944, 8766, 7603, 8191, 8194, 8455] avg=8192.0000 max=8766 max_rank=3 min=7603 min_rank=4 imbalance_ratio=1.070068 max_load_reduction=56770 max_load_reduction_pct=0.866241 moved_to_echo=32443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 4096, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 97, 0, 0, 4076, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 1567, 0, 4, 4096, 0, 2770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 82, 0, 0, 4096, 670, 0, 2526, 712, 203, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3379, 4075, 0, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8190, 8055, 8437, 8296, 8194, 8193, 7973] avg=8192.0000 max=8437 max_rank=3 min=7973 min_rank=7 imbalance_ratio=1.029907 max_load_reduction=57099 max_load_reduction_pct=0.871262 moved_to_echo=32656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 1, 1, 4092, 5, 0, 0, 0, 0, 0, 0, 1, 4091, 2, 1, 1, 1, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 174, 0, 1, 4088, 1, 3879, 0, 0, 0, 0, 0, 0, 0, 1567, 0, 7, 4095, 0, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 20, 0, 0, 4095, 37, 0, 2527, 713, 203, 0, 0, 0, 0, 0, 7, 1, 4094, 1, 0, 1, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4095, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3379, 4085, 1, 1, 1006, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8195, 8144, 8547, 7598, 8194, 8188, 8475] avg=8192.0000 max=8547 max_rank=3 min=7598 min_rank=4 imbalance_ratio=1.043335 max_load_reduction=56989 max_load_reduction_pct=0.869583 moved_to_echo=32455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 4049, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 1568, 0, 4, 4096, 1, 3068, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 52, 0, 0, 4095, 64, 0, 2528, 713, 203, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3379, 4063, 0, 0, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 8194, 8146, 8738, 7657, 8197, 8194, 8216] avg=8192.0000 max=8738 max_rank=3 min=7657 min_rank=4 imbalance_ratio=1.066650 max_load_reduction=56798 max_load_reduction_pct=0.866669 moved_to_echo=32264 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4056, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1454, 0, 0, 4053, 0, 3882, 0, 0, 0, 0, 0, 0, 0, 1564, 0, 130, 4000, 1, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 378, 0, 1, 4096, 1217, 0, 2530, 717, 203, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3372, 4026, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8165, 8195, 9391, 6856, 9143, 8184, 8189, 7413] avg=8192.0000 max=9391 max_rank=2 min=6856 min_rank=3 imbalance_ratio=1.146362 max_load_reduction=56145 max_load_reduction_pct=0.856705 moved_to_echo=32929 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 1, 1, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 1, 1, 1, 0, 4089, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 35, 0, 1, 3999, 0, 3882, 0, 0, 0, 0, 0, 0, 0, 1568, 1, 1, 4095, 0, 3003, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 15, 0, 0, 4095, 134, 0, 2527, 712, 204, 0, 0, 0, 0, 0, 4, 0, 4095, 1, 0, 1, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3379, 4081, 1, 1, 1024, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8189, 7918, 8669, 7691, 8193, 8190, 8488] avg=8192.0000 max=8669 max_rank=3 min=7691 min_rank=4 imbalance_ratio=1.058228 max_load_reduction=56867 max_load_reduction_pct=0.867722 moved_to_echo=32562 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 27136, 32512, 128, 128, 6016, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 2176, 0, 128, 32384, 128, 31104, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[12544, 128, 256, 32768, 128, 20224, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32768, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 1024, 128, 128, 32768, 4352, 128, 20224, 5760, 1664, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3806, 1, 0, 9, 0, 0, 0, 4070, 0, 606, 0, 4095, 80, 4096, 0, 0, 0, 0, 0, 1, 231, 4096, 0, 36, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 4049, 4096, 445, 4096, 0, 0, 4089, 4092, 3160, 0, 0, 0, 0, 0, 0, 4094, 0, 9, 0, 0, 3984, 0, 3774, 10, 1, 8, 0, 334] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 3592, 0, 1, 26, 0, 1, 0, 4091, 1, 1198, 1, 4095, 78, 4095, 0, 0, 0, 0, 0, 0, 2072, 4095, 1, 88, 0, 1, 0, 0, 4021, 0, 1, 0, 0, 3926, 4095, 559, 4095, 1, 0, 4095, 3965, 2493, 1, 0, 0, 1, 1, 0, 4061, 0, 16, 0, 1, 2421, 1, 3381, 42, 0, 5, 0, 822] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2904, 0, 0, 8, 0, 0, 0, 4076, 0, 633, 0, 4096, 67, 4096, 0, 0, 0, 0, 0, 2, 1857, 4096, 0, 34, 0, 0, 0, 0, 4077, 2, 0, 0, 0, 4021, 4083, 750, 4094, 0, 0, 4085, 4070, 2198, 0, 0, 0, 0, 0, 0, 4086, 0, 50, 0, 0, 3706, 0, 2795, 10, 0, 2, 0, 1542] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3867, 1, 0, 10, 0, 0, 0, 4093, 0, 1006, 0, 4096, 71, 4096, 0, 0, 0, 0, 0, 0, 1192, 4096, 0, 99, 0, 0, 0, 0, 3990, 2, 0, 0, 0, 4094, 4096, 47, 4096, 0, 0, 4072, 4096, 1695, 0, 0, 0, 0, 0, 0, 4080, 0, 100, 0, 0, 3993, 0, 3436, 5, 0, 2, 0, 1009] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 2963, 0, 1, 29, 0, 0, 0, 4087, 1, 452, 0, 4095, 15, 4095, 1, 1, 0, 0, 0, 0, 1917, 4095, 0, 8, 1, 1, 1, 1, 4082, 0, 0, 0, 1, 4091, 4088, 912, 4094, 1, 1, 4093, 4087, 1769, 1, 0, 0, 0, 1, 0, 4091, 1, 100, 1, 1, 3407, 1, 3842, 10, 0, 0, 0, 1001] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3510, 1, 0, 9, 0, 0, 0, 4090, 0, 839, 0, 4095, 180, 4096, 0, 0, 0, 0, 0, 1, 1806, 4096, 0, 20, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 4067, 4095, 328, 4096, 0, 0, 4095, 4079, 1675, 0, 0, 0, 0, 0, 0, 4095, 0, 31, 0, 0, 3938, 0, 3525, 1, 1, 0, 0, 633] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4096, 3088, 0, 0, 0, 0, 0, 0, 4095, 0, 1815, 0, 4092, 72, 4096, 0, 0, 0, 1, 0, 1, 3630, 4096, 0, 26, 0, 0, 0, 0, 4061, 0, 0, 1, 0, 3769, 4093, 1803, 4096, 0, 0, 4095, 4093, 855, 0, 0, 0, 0, 0, 0, 4087, 0, 26, 0, 0, 961, 0, 3256, 9, 1, 0, 0, 1222] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4095, 3408, 0, 1, 188, 0, 0, 0, 4080, 1, 128, 0, 4095, 69, 4095, 1, 1, 0, 0, 0, 0, 1685, 4095, 1, 29, 0, 1, 0, 1, 3995, 0, 0, 0, 1, 4051, 4083, 233, 4093, 1, 1, 4093, 4055, 1498, 1, 1, 0, 1, 1, 0, 4095, 0, 46, 0, 1, 3856, 1, 3666, 3, 0, 0, 0, 1785] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4095, 3592, 0, 1, 26, 0, 527, 140, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 1, 1198, 1, 3193, 78, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 2072, 1886, 375, 37, 0, 0, 0, 0, 0, 4095, 1, 88, 0, 1, 0, 0, 4021, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3926, 3553, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2209, 3965, 2493, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4061, 0, 16, 0, 1, 4058, 0, 0, 0, 0, 0, 0, 0, 2421, 1, 3381, 42, 0, 5, 0, 822, 392, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8382, 8563, 8465, 8216, 8039, 8669, 8138, 7064] avg=8192.0000 max=8669 max_rank=5 min=7064 min_rank=7 imbalance_ratio=1.058228 max_load_reduction=56867 max_load_reduction_pct=0.867722 moved_to_echo=31910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3806, 1, 0, 9, 0, 527, 140, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 606, 0, 3193, 80, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 231, 1883, 375, 38, 0, 0, 0, 0, 0, 4096, 0, 36, 0, 0, 0, 0, 4072, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 3554, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2206, 4092, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 9, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 3774, 10, 1, 8, 0, 334, 392, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8579, 7949, 6624, 8214, 8048, 9458, 8161, 8503] avg=8192.0000 max=9458 max_rank=5 min=6624 min_rank=2 imbalance_ratio=1.154541 max_load_reduction=56078 max_load_reduction_pct=0.855682 moved_to_echo=34170 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3867, 1, 0, 10, 0, 527, 140, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1006, 0, 3194, 71, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1192, 1875, 375, 37, 0, 0, 0, 0, 0, 4096, 0, 99, 0, 0, 0, 0, 3990, 10, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 3554, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2197, 4096, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 100, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 3436, 5, 0, 2, 0, 1009, 392, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8641, 8364, 7575, 8195, 7697, 7988, 8239, 8837] avg=8192.0000 max=8837 max_rank=7 min=7575 min_rank=2 imbalance_ratio=1.078735 max_load_reduction=56699 max_load_reduction_pct=0.865158 moved_to_echo=32761 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 2904, 0, 0, 8, 0, 527, 140, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 633, 0, 3194, 67, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 1857, 1881, 375, 38, 0, 0, 0, 0, 0, 4096, 0, 34, 0, 0, 0, 0, 4077, 10, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4021, 3543, 750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2204, 4070, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 50, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 3706, 0, 2795, 10, 0, 2, 0, 1542, 390, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7675, 7970, 8249, 8217, 8316, 8472, 8192, 8445] avg=8192.0000 max=8472 max_rank=5 min=7675 min_rank=0 imbalance_ratio=1.034180 max_load_reduction=57064 max_load_reduction_pct=0.870728 moved_to_echo=33425 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4095, 2963, 0, 1, 29, 0, 527, 140, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 452, 0, 3193, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 1917, 1885, 375, 37, 0, 0, 0, 0, 0, 4095, 0, 8, 1, 1, 1, 1, 4082, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 3547, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2208, 4087, 1769, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 1, 100, 1, 1, 4057, 0, 0, 0, 0, 0, 0, 0, 3407, 1, 3842, 10, 0, 0, 0, 1001, 391, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7756, 7748, 8311, 8199, 8551, 8067, 8252, 8652] avg=8192.0000 max=8652 max_rank=7 min=7748 min_rank=1 imbalance_ratio=1.056152 max_load_reduction=56884 max_load_reduction_pct=0.867981 moved_to_echo=33522 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3510, 1, 0, 9, 0, 527, 140, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 839, 0, 3193, 180, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 1806, 1886, 375, 38, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 0, 0, 4038, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 3553, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2209, 4079, 1675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 31, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 3525, 1, 1, 0, 0, 633, 392, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8283, 8302, 8202, 8164, 7948, 7963, 8184, 8490] avg=8192.0000 max=8490 max_rank=7 min=7948 min_rank=4 imbalance_ratio=1.036377 max_load_reduction=57046 max_load_reduction_pct=0.870453 moved_to_echo=32585 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 3088, 0, 0, 0, 0, 530, 146, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1815, 0, 3187, 72, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 1, 3630, 1889, 375, 38, 0, 0, 0, 0, 0, 4096, 0, 26, 0, 0, 0, 0, 4061, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3769, 3536, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2206, 4093, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 26, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 961, 0, 3256, 9, 1, 0, 0, 1222, 395, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7860, 9169, 10030, 8199, 9109, 7154, 8171, 5844] avg=8192.0000 max=10030 max_rank=2 min=5844 min_rank=7 imbalance_ratio=1.224365 max_load_reduction=55506 max_load_reduction_pct=0.846954 moved_to_echo=30278 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4095, 3408, 0, 1, 188, 0, 527, 140, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 1, 128, 0, 3193, 69, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 1685, 1885, 375, 38, 0, 0, 0, 0, 0, 4095, 1, 29, 0, 1, 0, 1, 3995, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4051, 3543, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2208, 4055, 1498, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 46, 0, 1, 4055, 0, 0, 0, 0, 0, 0, 0, 3856, 1, 3666, 3, 0, 0, 0, 1785, 390, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8360, 7471, 8080, 8132, 7828, 7765, 8199, 9701] avg=8192.0000 max=9701 max_rank=7 min=7471 min_rank=1 imbalance_ratio=1.184204 max_load_reduction=55835 max_load_reduction_pct=0.851974 moved_to_echo=33493 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 32128, 28416, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 0, 128, 0, 128, 14464, 15104, 3072, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[26368, 128, 27776, 128, 128, 128, 0, 8448, 3200, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 17664, 32640, 15360, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 27264, 128, 128, 384, 0, 4224, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 128, 384, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[32768, 128, 384, 128, 128, 128, 128, 32384, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 32768, 128, 6784, 128, 25600, 640, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2487, 0, 1, 0, 3075, 0, 4096, 102, 0, 4082, 4096, 0, 57, 0, 0, 3667, 4095, 0, 721, 0, 0, 4059, 4096, 0, 0, 4096, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 4083, 1, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 2216, 0, 0, 0, 0, 0, 1, 9, 4095, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2178, 0, 2, 0, 2868, 0, 4096, 323, 0, 4089, 4096, 0, 19, 0, 0, 3751, 4094, 0, 1435, 0, 0, 4079, 4096, 0, 0, 4081, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 1684, 0, 0, 0, 0, 0, 3, 17, 4094, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1787, 0, 1, 0, 3425, 0, 4096, 35, 0, 4077, 4096, 0, 535, 0, 0, 2767, 4095, 0, 1131, 0, 0, 4060, 4096, 0, 0, 4095, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 2634, 0, 0, 0, 0, 0, 3, 8, 4092, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1451, 0, 1, 0, 3256, 0, 4096, 42, 0, 4090, 4096, 0, 32, 0, 0, 3743, 4094, 0, 1566, 0, 0, 4004, 4096, 0, 0, 4092, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 4047, 1, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 2310, 0, 0, 0, 0, 0, 1, 17, 4095, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 2295, 0, 0, 0, 3363, 0, 4095, 315, 1, 4032, 4095, 0, 11, 0, 1, 3550, 4093, 0, 278, 0, 0, 4043, 4095, 0, 1, 4095, 1, 0, 1, 1, 223, 1, 1, 0, 1, 0, 1, 4087, 0, 0, 4080, 0, 0, 1, 4095, 4095, 0, 1, 0, 2379, 0, 0, 0, 1, 0, 3, 8, 4095, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 2, 1, 2398, 0, 0, 0, 2839, 2, 4095, 105, 1, 4070, 4093, 1, 117, 0, 1, 3880, 4095, 0, 1186, 0, 0, 4047, 4095, 1, 1, 4091, 1, 0, 1, 0, 55, 1, 1, 0, 0, 0, 1, 4092, 0, 3, 4095, 0, 1, 1, 4094, 4095, 0, 1, 0, 1767, 0, 0, 0, 1, 1, 6, 7, 4095, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1641, 0, 0, 0, 3533, 0, 4096, 166, 0, 4080, 4096, 0, 199, 0, 0, 3905, 4095, 0, 300, 0, 0, 4089, 4096, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4058, 1, 0, 4091, 0, 0, 0, 4096, 4096, 0, 0, 0, 2366, 0, 0, 0, 0, 0, 3, 231, 4095, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 2160, 1, 0, 1, 3168, 0, 4095, 159, 1, 4037, 4095, 1, 48, 0, 1, 3353, 4095, 0, 1481, 0, 0, 4031, 4095, 1, 1, 4087, 1, 0, 1, 0, 225, 0, 1, 0, 0, 0, 1, 4089, 0, 2, 4095, 0, 0, 1, 4095, 4095, 0, 1, 0, 1807, 0, 0, 0, 1, 1, 8, 9, 4094, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2487, 0, 1, 0, 3321, 2145, 619, 32, 23, 0, 0, 0, 3075, 0, 0, 102, 0, 4082, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 3667, 3476, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 49, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 0, 4094, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 1951, 0, 0, 0, 2216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 4095, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8628, 8034, 7921, 8204, 8130, 8195, 8263, 8161] avg=8192.0000 max=8628 max_rank=0 min=7921 min_rank=2 imbalance_ratio=1.053223 max_load_reduction=56908 max_load_reduction_pct=0.868347 moved_to_echo=32749 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 2295, 0, 0, 0, 3320, 2144, 619, 32, 23, 0, 0, 0, 3363, 0, 0, 315, 1, 4032, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 3550, 3474, 0, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 48, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 223, 1, 1, 0, 1, 0, 4095, 4015, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 4080, 0, 0, 1, 17, 0, 0, 0, 0, 0, 0, 0, 4095, 1951, 0, 1, 0, 2379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 8, 4095, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8436, 8486, 7314, 8188, 8338, 8186, 8426, 8162] avg=8192.0000 max=8486 max_rank=1 min=7314 min_rank=2 imbalance_ratio=1.035889 max_load_reduction=57050 max_load_reduction_pct=0.870514 moved_to_echo=33112 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1451, 0, 1, 0, 3321, 2145, 619, 32, 23, 0, 0, 0, 3256, 0, 0, 42, 0, 4090, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 3743, 3475, 0, 1566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 49, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 4047, 1, 0, 4094, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 1951, 0, 0, 0, 2310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 4095, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7592, 8163, 8816, 8145, 8135, 8159, 8357, 8169] avg=8192.0000 max=8816 max_rank=2 min=7592 min_rank=0 imbalance_ratio=1.076172 max_load_reduction=56720 max_load_reduction_pct=0.865479 moved_to_echo=32820 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1787, 0, 1, 0, 3321, 2145, 619, 32, 23, 0, 0, 0, 3425, 0, 0, 35, 0, 4077, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 0, 2767, 3476, 0, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 49, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4094, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 1951, 0, 0, 0, 2634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 4092, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7928, 8312, 7909, 8204, 8137, 8206, 8681, 8159] avg=8192.0000 max=8681 max_rank=6 min=7909 min_rank=2 imbalance_ratio=1.059692 max_load_reduction=56855 max_load_reduction_pct=0.867538 moved_to_echo=33183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2178, 0, 2, 0, 3321, 2145, 619, 32, 23, 0, 0, 0, 2868, 0, 0, 323, 0, 4089, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 3751, 3475, 0, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 49, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 4094, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 1951, 0, 0, 0, 1684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 17, 4094, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8320, 8055, 8680, 8209, 8164, 8207, 7731, 8170] avg=8192.0000 max=8680 max_rank=2 min=7731 min_rank=6 imbalance_ratio=1.059570 max_load_reduction=56856 max_load_reduction_pct=0.867554 moved_to_echo=32272 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 1, 2398, 0, 0, 0, 3318, 2144, 619, 32, 23, 0, 0, 0, 2839, 2, 0, 105, 1, 4070, 775, 1, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 1, 3880, 3476, 0, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 48, 1, 1, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 55, 1, 1, 0, 0, 0, 4095, 4015, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 3, 4095, 0, 1, 1, 17, 0, 0, 0, 0, 0, 0, 0, 4094, 1951, 0, 1, 0, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 6, 7, 4095, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8538, 7793, 8660, 8189, 8168, 8210, 7813, 8165] avg=8192.0000 max=8660 max_rank=2 min=7793 min_rank=1 imbalance_ratio=1.057129 max_load_reduction=56876 max_load_reduction_pct=0.867859 moved_to_echo=32356 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1641, 0, 0, 0, 3323, 2147, 622, 33, 25, 0, 0, 0, 3533, 0, 0, 166, 0, 4080, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 3905, 3473, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 43, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 4020, 0, 0, 0, 0, 0, 0, 0, 4058, 1, 0, 4091, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 1949, 0, 0, 0, 2366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 231, 4095, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7791, 8552, 7877, 8228, 8127, 8168, 8411, 8382] avg=8192.0000 max=8552 max_rank=1 min=7791 min_rank=0 imbalance_ratio=1.043945 max_load_reduction=56984 max_load_reduction_pct=0.869507 moved_to_echo=33088 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 2160, 1, 0, 1, 3320, 2144, 619, 32, 23, 0, 0, 0, 3168, 0, 0, 159, 1, 4037, 775, 1, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 1, 3353, 3476, 0, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 48, 1, 1, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 225, 0, 1, 0, 0, 0, 4095, 4015, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 2, 4095, 0, 0, 1, 17, 0, 0, 0, 0, 0, 0, 0, 4095, 1951, 0, 1, 0, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 8, 9, 4094, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8303, 8141, 8359, 8169, 8337, 8205, 7854, 8168] avg=8192.0000 max=8359 max_rank=2 min=7854 min_rank=6 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32564 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66816 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 16512, 128, 128, 128, 26624, 17280, 4992, 384, 256, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[1024, 0, 128, 28672, 27904, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 32512, 384, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 128, 128, 384, 32768, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 640, 128, 128, 0, 128, 0, 32768, 32128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32768, 15616, 0, 128, 0, 17280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 32768, 0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[25600, 128, 0, 1280, 128, 32640, 6272, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66816 slack_slots=64256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4060, 0, 0, 4096, 0, 0, 0, 0, 4092, 1, 0, 4096, 4096, 0, 0, 4095, 0, 0, 3766, 4095, 4, 0, 0, 4, 0, 4094, 0, 0, 54, 0, 4096, 1, 0, 0, 0, 1922, 1, 0, 4092, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 2555, 4096, 0, 4028, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4088, 0, 0, 4096, 0, 0, 0, 0, 4091, 4, 0, 4096, 4096, 0, 0, 4094, 0, 0, 3712, 4094, 9, 0, 0, 9, 0, 4046, 0, 0, 20, 0, 4096, 0, 0, 0, 0, 2594, 2, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 1951, 4096, 0, 4056, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4074, 0, 0, 4096, 0, 0, 0, 0, 4093, 3, 0, 4096, 4096, 0, 0, 4094, 0, 0, 3935, 4096, 18, 0, 0, 7, 0, 4065, 0, 0, 69, 0, 4096, 0, 0, 0, 0, 1193, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3035, 4096, 0, 4083, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4055, 1, 0, 4095, 1, 0, 1, 1, 4095, 7, 1, 4084, 4095, 0, 1, 4094, 1, 0, 4024, 4088, 50, 0, 0, 15, 1, 3995, 1, 0, 12, 1, 4095, 0, 1, 0, 0, 351, 0, 0, 4095, 0, 1, 0, 0, 2, 4095, 0, 1, 0, 4, 1, 0, 1, 0, 3882, 4095, 0, 4095, 3, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4007, 1, 0, 4095, 0, 2, 1, 1, 4094, 0, 1, 4090, 4094, 0, 0, 4095, 1, 0, 4030, 4095, 0, 0, 0, 3, 1, 3949, 13, 0, 127, 3, 4095, 0, 1, 0, 0, 1187, 0, 0, 4095, 0, 1, 1, 0, 0, 4095, 1, 1, 0, 3, 1, 0, 1, 0, 3074, 4095, 1, 4085, 0, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 4093, 2, 0, 4096, 4096, 0, 0, 4095, 0, 0, 4035, 4094, 8, 1, 0, 6, 0, 4087, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 100, 1, 0, 4094, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 4061, 4096, 0, 4091, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4086, 0, 0, 4096, 0, 0, 0, 0, 4093, 1, 0, 4096, 4096, 0, 0, 4095, 0, 0, 3877, 4095, 10, 0, 0, 15, 0, 3943, 0, 0, 58, 0, 4096, 1, 0, 0, 0, 3114, 0, 0, 4094, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1365, 4096, 0, 4016, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 3910, 1, 0, 4095, 1, 0, 1, 1, 4094, 3, 1, 4092, 4095, 0, 0, 4094, 1, 0, 3945, 4094, 22, 0, 2, 4, 1, 4067, 2, 0, 269, 7, 4095, 0, 1, 0, 0, 1167, 0, 0, 4093, 0, 1, 1, 0, 1, 4094, 1, 1, 0, 1, 1, 0, 1, 0, 2991, 4095, 0, 4093, 1, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4055, 1, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4095, 7, 1, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 183, 1, 0, 4024, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 15, 1, 3995, 1, 0, 4095, 39, 0, 0, 0, 0, 0, 0, 12, 1, 4095, 0, 1, 0, 0, 351, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 2, 4005, 89, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 4, 1, 0, 1, 3822, 271, 0, 0, 0, 0, 0, 0, 0, 3882, 1259, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8191, 8297, 8196, 7024, 8192, 8195, 9239] avg=8192.0000 max=9239 max_rank=7 min=7024 min_rank=4 imbalance_ratio=1.127808 max_load_reduction=56297 max_load_reduction_pct=0.859024 moved_to_echo=32650 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4060, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 3766, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 4094, 0, 0, 4096, 40, 0, 0, 0, 0, 0, 0, 54, 0, 4096, 1, 0, 0, 0, 1922, 2564, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 0, 0, 0, 4006, 89, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 3823, 271, 0, 0, 0, 0, 0, 0, 0, 2555, 1260, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8206, 8190, 8044, 8238, 8637, 8188, 8190, 7843] avg=8192.0000 max=8637 max_rank=4 min=7843 min_rank=7 imbalance_ratio=1.054321 max_load_reduction=56899 max_load_reduction_pct=0.868210 moved_to_echo=32858 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4088, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 3712, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 0, 4046, 0, 0, 4096, 40, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 0, 0, 0, 0, 2594, 2564, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 0, 0, 0, 0, 0, 4006, 89, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 3822, 271, 0, 0, 0, 0, 0, 0, 0, 1951, 1260, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8234, 8192, 7989, 8200, 9274, 8189, 8191, 7267] avg=8192.0000 max=9274 max_rank=4 min=7267 min_rank=7 imbalance_ratio=1.132080 max_load_reduction=56262 max_load_reduction_pct=0.858490 moved_to_echo=32921 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4074, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 3935, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 7, 0, 4065, 0, 0, 4096, 40, 0, 0, 0, 0, 0, 0, 69, 0, 4096, 0, 0, 0, 0, 1193, 2564, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4006, 89, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3822, 271, 0, 0, 0, 0, 0, 0, 0, 3035, 1260, 0, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8220, 8193, 8214, 8226, 7922, 8193, 8189, 8379] avg=8192.0000 max=8379 max_rank=7 min=7922 min_rank=4 imbalance_ratio=1.022827 max_load_reduction=57157 max_load_reduction_pct=0.872147 moved_to_echo=32683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4086, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 3877, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 15, 0, 3943, 0, 0, 4096, 40, 0, 0, 0, 0, 0, 0, 58, 0, 4096, 1, 0, 0, 0, 3114, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 4006, 89, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3823, 271, 0, 0, 0, 0, 0, 0, 0, 1365, 1260, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8232, 8191, 8155, 8104, 9833, 8190, 8190, 6641] avg=8192.0000 max=9833 max_rank=4 min=6641 min_rank=7 imbalance_ratio=1.200317 max_load_reduction=55703 max_load_reduction_pct=0.849960 moved_to_echo=32854 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4095, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 4035, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 6, 0, 4087, 0, 0, 4096, 43, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 100, 2569, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 4013, 95, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 3824, 273, 0, 0, 0, 0, 0, 0, 0, 4061, 1250, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8231, 8195, 8305, 8241, 6766, 8203, 8193, 9402] avg=8192.0000 max=9402 max_rank=7 min=6766 min_rank=4 imbalance_ratio=1.147705 max_load_reduction=56134 max_load_reduction_pct=0.856537 moved_to_echo=32564 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 3910, 1, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4094, 3, 1, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 1, 0, 3945, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 2, 4, 1, 4067, 2, 0, 4095, 39, 0, 0, 0, 0, 0, 0, 269, 7, 4095, 0, 1, 0, 0, 1167, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 1, 0, 1, 4005, 89, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 1, 1, 0, 1, 3822, 271, 0, 0, 0, 0, 0, 0, 0, 2991, 1259, 0, 4093, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8057, 8194, 8223, 8232, 8103, 8190, 8192, 8345] avg=8192.0000 max=8345 max_rank=7 min=8057 min_rank=0 imbalance_ratio=1.018677 max_load_reduction=57191 max_load_reduction_pct=0.872665 moved_to_echo=32830 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4007, 1, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 4094, 0, 1, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 1, 0, 4030, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3949, 13, 0, 4094, 39, 0, 0, 0, 0, 0, 0, 127, 3, 4095, 0, 1, 0, 0, 1187, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 4005, 89, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 3, 1, 0, 1, 3823, 271, 0, 0, 0, 0, 0, 0, 0, 3074, 1259, 1, 4085, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8154, 8190, 8309, 8099, 7977, 8191, 8196, 8420] avg=8192.0000 max=8420 max_rank=7 min=7977 min_rank=4 imbalance_ratio=1.027832 max_load_reduction=57116 max_load_reduction_pct=0.871521 moved_to_echo=32784 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 128, 30592, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[0, 0, 0, 32768, 32384, 128, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 0, 32768, 0, 128, 128, 128, 128, 32128, 768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 23040, 10112, 128, 32640, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[640, 128, 32768, 128, 128, 0, 0, 11648, 20608, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 128, 128, 32256, 128, 0, 32768, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 1536, 128, 0, 31360, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 451, 2, 0, 0, 0, 0, 4095, 0, 4096, 0, 4076, 0, 774, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4094, 4064, 2701, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 200, 0, 4095, 0, 0, 0, 38, 0, 4096, 0, 4082] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[3, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 261, 2, 0, 0, 0, 0, 4010, 0, 4096, 0, 4036, 0, 112, 0, 0, 4078, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 4094, 4090, 3821, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 57, 0, 4094, 0, 0, 0, 10, 0, 4096, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 12, 0, 0, 112, 0, 0, 1, 1, 0, 4061, 1, 4095, 1, 4094, 1, 710, 0, 0, 4086, 1, 1, 0, 0, 4089, 1, 0, 4095, 0, 0, 4090, 0, 1, 1, 9, 4094, 4055, 3360, 0, 4095, 4095, 0, 0, 1, 0, 3, 1, 142, 0, 4095, 1, 1, 0, 79, 0, 3868, 1, 4088] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 3, 0, 0, 1, 100, 5, 0, 0, 0, 0, 4077, 0, 4096, 0, 4082, 0, 58, 0, 0, 4077, 0, 0, 0, 0, 4057, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 4088, 3908, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 89, 0, 4094, 0, 0, 0, 36, 0, 4096, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 0, 59, 1, 0, 10, 0, 0, 1, 1, 0, 3793, 1, 4094, 0, 4084, 1, 846, 1, 0, 4093, 1, 1, 1, 11, 4021, 10, 1, 4095, 1, 0, 4073, 0, 0, 1, 2, 4095, 4095, 3592, 0, 4095, 4092, 0, 0, 1, 0, 4, 1, 82, 0, 4095, 1, 1, 76, 1, 0, 3912, 0, 4094] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 3, 0, 0, 4, 274, 1, 0, 0, 0, 0, 4053, 0, 4096, 0, 4061, 0, 116, 0, 0, 4082, 0, 0, 0, 0, 4081, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4091, 4058, 3812, 0, 4095, 4096, 0, 0, 0, 1, 0, 0, 11, 0, 4095, 0, 0, 0, 26, 0, 4096, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 3, 0, 0, 1, 42, 1, 0, 0, 0, 0, 4092, 0, 4096, 0, 4094, 0, 16, 0, 0, 4070, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 4079, 4077, 4023, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 67, 0, 4096, 0, 0, 0, 19, 0, 4096, 0, 4093] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[1, 0, 0, 4094, 0, 0, 0, 55, 0, 0, 43, 0, 0, 1, 1, 0, 3926, 1, 4095, 1, 4090, 1, 882, 1, 0, 4090, 1, 1, 1, 1, 3892, 3, 1, 4095, 0, 0, 4073, 0, 0, 1, 5, 4095, 4086, 3439, 0, 4095, 4093, 0, 3, 1, 0, 3, 1, 200, 0, 4095, 1, 1, 18, 22, 0, 3932, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 0, 12, 3913, 163, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 1, 1, 0, 4095, 3932, 0, 0, 0, 0, 0, 0, 4061, 1, 0, 1, 3756, 1, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 1, 0, 0, 4089, 1, 52, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4090, 0, 1, 1, 6, 0, 0, 0, 0, 0, 0, 0, 9, 532, 4055, 3360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 1, 142, 0, 4095, 3562, 246, 182, 0, 0, 0, 0, 0, 1, 1, 0, 79, 0, 3868, 1, 4088, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8182, 8141, 8530, 8230, 8193, 7956, 8232, 8072] avg=8192.0000 max=8530 max_rank=2 min=7956 min_rank=5 imbalance_ratio=1.041260 max_load_reduction=57006 max_load_reduction_pct=0.869843 moved_to_echo=32453 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 10, 0, 3915, 164, 0, 0, 0, 0, 0, 0, 0, 0, 451, 2, 0, 0, 0, 0, 4096, 3932, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3739, 0, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 532, 4064, 2701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 200, 0, 4095, 3562, 245, 181, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 4096, 0, 4082, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8186, 8481, 8608, 8232, 8198, 7297, 8284, 8250] avg=8192.0000 max=8608 max_rank=2 min=7297 min_rank=5 imbalance_ratio=1.050781 max_load_reduction=56928 max_load_reduction_pct=0.868652 moved_to_echo=32029 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4096, 0, 0, 5, 0, 3914, 163, 0, 0, 0, 0, 0, 0, 0, 0, 261, 2, 0, 0, 0, 0, 4096, 3933, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 3702, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 4096, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 532, 4090, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 57, 0, 4094, 3562, 243, 182, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 4096, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8292, 7824, 8225, 8196, 8443, 8139, 8236] avg=8192.0000 max=8443 max_rank=5 min=7824 min_rank=2 imbalance_ratio=1.030640 max_load_reduction=57093 max_load_reduction_pct=0.871170 moved_to_echo=33014 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 3, 0, 3914, 164, 0, 0, 0, 0, 0, 0, 0, 1, 100, 5, 0, 0, 0, 0, 4096, 3932, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 3744, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 4057, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 532, 4088, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 4094, 3564, 246, 182, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4096, 0, 4094, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8177, 8134, 7879, 8186, 8197, 8528, 8175, 8260] avg=8192.0000 max=8528 max_rank=5 min=7879 min_rank=2 imbalance_ratio=1.041016 max_load_reduction=57008 max_load_reduction_pct=0.869873 moved_to_echo=33160 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 1, 0, 0, 59, 3910, 163, 0, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 1, 1, 0, 4094, 3932, 0, 0, 0, 0, 0, 0, 3793, 1, 0, 0, 3746, 1, 846, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 1, 1, 11, 4021, 10, 52, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 4073, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 2, 532, 4095, 3592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 1, 82, 0, 4095, 3563, 246, 182, 0, 0, 0, 0, 0, 1, 1, 76, 1, 0, 3912, 0, 4094, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8228, 8039, 8388, 8190, 8177, 8221, 8174, 8119] avg=8192.0000 max=8388 max_rank=2 min=8039 min_rank=1 imbalance_ratio=1.023926 max_load_reduction=57148 max_load_reduction_pct=0.872009 moved_to_echo=32691 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 3, 0, 3914, 164, 0, 0, 0, 0, 0, 0, 0, 4, 274, 1, 0, 0, 0, 0, 4096, 3931, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 3726, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4081, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 532, 4058, 3812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 0, 4095, 3559, 244, 182, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4096, 0, 4095, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8306, 7895, 8214, 8198, 8402, 8092, 8251] avg=8192.0000 max=8402 max_rank=5 min=7895 min_rank=2 imbalance_ratio=1.025635 max_load_reduction=57134 max_load_reduction_pct=0.871796 moved_to_echo=32943 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 3, 0, 3915, 164, 0, 0, 0, 0, 0, 0, 0, 1, 42, 1, 0, 0, 0, 0, 4096, 3932, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3740, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 4093, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 527, 4077, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 67, 0, 4096, 3552, 250, 181, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 0, 4093, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 8072, 7848, 8218, 8201, 8627, 8147, 8245] avg=8192.0000 max=8627 max_rank=5 min=7848 min_rank=2 imbalance_ratio=1.053101 max_load_reduction=56909 max_load_reduction_pct=0.868362 moved_to_echo=33220 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4094, 0, 0, 0, 55, 3912, 164, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 1, 1, 0, 4095, 3931, 0, 0, 0, 0, 0, 0, 3926, 1, 0, 1, 3752, 1, 882, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 1, 1, 1, 3892, 3, 52, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4073, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 5, 532, 4086, 3439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 3, 1, 200, 0, 4095, 3563, 246, 181, 0, 0, 0, 0, 0, 1, 1, 18, 22, 0, 3932, 0, 4095, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8226, 8071, 8564, 8041, 8176, 8062, 8293, 8103] avg=8192.0000 max=8564 max_rank=2 min=8041 min_rank=3 imbalance_ratio=1.045410 max_load_reduction=56972 max_load_reduction_pct=0.869324 moved_to_echo=32634 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 128, 128, 31360, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[32128, 128, 0, 128, 29952, 128, 3584, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 256, 0, 32256, 128, 32768, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 1408, 128, 0, 128, 128, 0, 32768, 31488, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 32512, 128, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[128, 4352, 32640, 28672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 128, 896, 0, 32768, 28544, 2048, 1536, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 0, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4093, 4096, 0, 0, 0, 0, 1, 4095, 4096, 0, 0, 0, 0, 4096, 4029, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4096, 45, 0, 4082, 0, 0, 0, 17, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 0, 0, 4095, 1, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 4080, 0, 1, 20, 1, 0, 4095, 4083, 1, 0, 0, 1, 4073, 3960, 0, 1, 1, 1, 0, 1, 4095, 1, 1, 1, 0, 1, 0, 4094, 4095, 0, 0, 4095, 1, 0, 0, 47, 0, 0, 0, 1, 4095, 0, 1, 15, 1, 0, 4, 4094, 0, 2, 99, 1, 0, 0, 0, 4093, 0, 4095, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4092, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 4096, 3975, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4094, 78, 0, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 2, 13, 0, 0, 4096, 0, 0, 30, 0, 0, 0, 0, 4096, 4, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4091, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 4094, 42, 0, 4093, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 2, 1, 0, 0, 4096, 0, 0, 19, 0, 0, 0, 0, 4095, 5, 4096, 2, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4093, 4095, 0, 0, 0, 0, 4, 4095, 4096, 0, 0, 0, 0, 4096, 3845, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 4095, 229, 0, 4092, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 0, 4096, 1, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 4049, 1, 0, 13, 1, 65, 4095, 4084, 1, 0, 18, 1, 4070, 3973, 1, 1, 1, 1, 0, 1, 4094, 0, 1, 1, 0, 1, 0, 4093, 4095, 5, 1, 4091, 5, 1, 0, 2, 0, 0, 0, 1, 4094, 0, 1, 1, 5, 0, 4, 4094, 48, 0, 46, 0, 0, 0, 1, 4095, 0, 4095, 0, 0, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4095, 3980, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4096, 25, 0, 4092, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 68, 0, 0, 4096, 3, 0, 22, 0, 0, 0, 0, 4096, 2, 4096, 1, 0, 4096] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 4095, 4034, 4, 0, 22, 1, 19, 4095, 4078, 1, 0, 0, 1, 4065, 4017, 1, 1, 1, 1, 0, 1, 4095, 0, 1, 1, 0, 1, 0, 4094, 4095, 2, 1, 4092, 2, 1, 0, 1, 0, 0, 0, 1, 4095, 0, 1, 10, 0, 2, 2, 4094, 49, 1, 70, 0, 0, 0, 1, 4093, 0, 4095, 0, 9, 4095] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 4095, 4080, 0, 1, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4083, 1, 0, 0, 1, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 1, 1, 1, 0, 1, 4095, 115, 2, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 47, 0, 3969, 66, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 15, 1, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 0, 2, 99, 1, 0, 4027, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8208, 8160, 8176, 8193, 8178, 8192, 8243, 8186] avg=8192.0000 max=8243 max_rank=6 min=8160 min_rank=1 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 4096, 115, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 4082, 0, 0, 0, 17, 0, 3970, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 4028, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8200, 8194, 8242, 8194, 8180, 8176, 8159, 8191] avg=8192.0000 max=8242 max_rank=2 min=8159 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=57294 max_load_reduction_pct=0.874237 moved_to_echo=32706 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 4096, 115, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 4093, 0, 0, 0, 3, 0, 3970, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 1, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 4026, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8198, 8194, 8245, 8193, 8174, 8178, 8158, 8196] avg=8192.0000 max=8245 max_rank=2 min=8158 min_rank=6 imbalance_ratio=1.006470 max_load_reduction=57291 max_load_reduction_pct=0.874191 moved_to_echo=32706 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 0, 0, 4096, 115, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 4093, 0, 0, 0, 2, 0, 3970, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 13, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 30, 0, 0, 4026, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8194, 8188, 8192, 8209, 8191, 8168, 8195] avg=8192.0000 max=8209 max_rank=4 min=8168 min_rank=6 imbalance_ratio=1.002075 max_load_reduction=57327 max_load_reduction_pct=0.874741 moved_to_echo=32763 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 4096, 115, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 4092, 0, 0, 0, 2, 0, 3970, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 13, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 4027, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8199, 8198, 8058, 8194, 8359, 8188, 8148, 8192] avg=8192.0000 max=8359 max_rank=4 min=8058 min_rank=2 imbalance_ratio=1.020386 max_load_reduction=57177 max_load_reduction_pct=0.872452 moved_to_echo=32887 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 4095, 4049, 1, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 65, 2, 4084, 1, 0, 18, 1, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 1, 1, 1, 1, 0, 1, 4094, 115, 2, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 4091, 5, 1, 0, 2, 0, 3969, 66, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 1, 5, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 48, 0, 46, 0, 0, 4027, 16, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 8241, 8189, 8191, 8140, 8181, 8235, 8189] avg=8192.0000 max=8241 max_rank=1 min=8140 min_rank=4 imbalance_ratio=1.005981 max_load_reduction=57295 max_load_reduction_pct=0.874252 moved_to_echo=32745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 4096, 117, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4092, 0, 0, 0, 2, 0, 3974, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 68, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 22, 0, 0, 4027, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8195, 8191, 8204, 8187, 8161, 8243, 8164, 8191] avg=8192.0000 max=8243 max_rank=5 min=8161 min_rank=4 imbalance_ratio=1.006226 max_load_reduction=57293 max_load_reduction_pct=0.874222 moved_to_echo=32759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 4095, 4034, 4, 0, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4078, 1, 0, 0, 1, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 1, 1, 1, 1, 0, 1, 4095, 115, 2, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4092, 2, 1, 0, 1, 0, 3969, 67, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 10, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4094, 49, 1, 70, 0, 0, 4028, 15, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 4093, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8167, 8164, 8234, 8192, 8135, 8187, 8261, 8196] avg=8192.0000 max=8261 max_rank=6 min=8135 min_rank=4 imbalance_ratio=1.008423 max_load_reduction=57275 max_load_reduction_pct=0.873947 moved_to_echo=32779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[31872, 128, 128, 128, 128, 0, 128, 32768, 1024, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[512, 128, 32768, 128, 128, 0, 128, 0, 31872, 640, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 0, 128, 32768, 0, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 32768, 128, 128, 384, 128, 0, 32256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 0, 32768, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4085, 0, 3, 1, 1, 0, 4094, 4096, 0, 1, 0, 0, 0, 0, 1, 1, 0, 2, 4055, 3905, 1, 46, 0, 3818, 0, 1, 4094, 4095, 0, 5, 4095, 7, 4095, 4095, 18, 0, 0, 0, 0, 1, 1, 647, 0, 5, 0, 2, 0, 1, 4093, 4095, 0, 89, 1, 4074, 0, 0, 3956, 1, 1, 3952, 1, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4085, 3190, 0, 0, 0, 3659, 2, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 16, 110, 0, 1, 0, 0, 0, 1227, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3991, 0, 0, 0, 4067, 2, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 1, 12, 0, 1, 0, 0, 0, 120, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4075, 0, 0, 0, 3984, 3, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 3, 23, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4094, 4093, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4087, 4076, 0, 0, 0, 3980, 2, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 6, 2, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4075, 0, 27, 1, 1, 0, 4095, 4095, 7, 1, 1, 1, 0, 0, 1, 1, 0, 0, 4016, 4074, 1, 62, 0, 4029, 0, 1, 4091, 4095, 0, 13, 4095, 2, 4094, 4095, 0, 0, 0, 0, 1, 1, 1, 244, 20, 4, 0, 1, 0, 0, 4094, 4095, 0, 51, 1, 4077, 0, 0, 3968, 1, 1, 4001, 0, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 3558, 0, 0, 0, 3432, 1, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 9, 104, 0, 1, 0, 0, 0, 1086, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[4078, 0, 7, 1, 1, 0, 4095, 4095, 7, 1, 1, 1, 0, 0, 1, 1, 0, 0, 3993, 4050, 1, 43, 0, 4029, 0, 1, 4090, 4095, 0, 15, 4095, 3, 4094, 4095, 0, 0, 0, 0, 0, 1, 1, 198, 60, 2, 0, 1, 0, 0, 4094, 4095, 0, 52, 1, 4080, 0, 0, 3995, 1, 1, 4059, 0, 1, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 3, 1, 1, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 2, 432, 3905, 1, 46, 0, 3818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4090, 0, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4058, 18, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 647, 0, 5, 0, 2, 0, 1, 4091, 3603, 3, 0, 0, 0, 0, 0, 4077, 0, 0, 89, 1, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 1, 1, 3952, 1, 1, 0, 0, 37, 20, 16, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8189, 8190, 8204, 8197, 8172, 8353, 8241, 7990] avg=8192.0000 max=8353 max_rank=5 min=7990 min_rank=7 imbalance_ratio=1.019653 max_load_reduction=57183 max_load_reduction_pct=0.872543 moved_to_echo=32756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 436, 3190, 0, 0, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4059, 16, 110, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 0, 0, 0, 0, 0, 1, 4091, 3629, 3, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 37, 20, 16, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8190, 7285, 8189, 8282, 8951, 8176, 8270] avg=8192.0000 max=8951 max_rank=5 min=7285 min_rank=2 imbalance_ratio=1.092651 max_load_reduction=56585 max_load_reduction_pct=0.863419 moved_to_echo=33679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 436, 4076, 0, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4059, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 3, 4091, 3631, 4, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 37, 20, 16, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8192, 8189, 8492, 8189, 8163, 7867, 8176, 8268] avg=8192.0000 max=8492 max_rank=2 min=7867 min_rank=5 imbalance_ratio=1.036621 max_load_reduction=57044 max_load_reduction_pct=0.870422 moved_to_echo=32474 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 437, 3991, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4059, 1, 12, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 0, 0, 1, 4092, 3639, 3, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 37, 20, 16, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8189, 8495, 8189, 8169, 7855, 8176, 8270] avg=8192.0000 max=8495 max_rank=2 min=7855 min_rank=5 imbalance_ratio=1.036987 max_load_reduction=57041 max_load_reduction_pct=0.870377 moved_to_echo=32470 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4075, 0, 27, 1, 1, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 1, 1, 0, 0, 1, 1, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 429, 4074, 1, 62, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 4090, 0, 13, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4058, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 244, 20, 4, 0, 1, 0, 0, 4092, 3568, 3, 0, 0, 0, 0, 0, 4078, 0, 0, 51, 1, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 1, 1, 4001, 0, 1, 0, 0, 37, 19, 16, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 8199, 8595, 8197, 8154, 7933, 8207, 8049] avg=8192.0000 max=8595 max_rank=2 min=7933 min_rank=5 imbalance_ratio=1.049194 max_load_reduction=56941 max_load_reduction_pct=0.868851 moved_to_echo=32343 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 436, 4075, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4059, 3, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 2, 4091, 3631, 3, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 37, 20, 16, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8193, 8189, 8495, 8190, 8181, 7843, 8176, 8269] avg=8192.0000 max=8495 max_rank=2 min=7843 min_rank=5 imbalance_ratio=1.036987 max_load_reduction=57041 max_load_reduction_pct=0.870377 moved_to_echo=32469 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 430, 3558, 0, 0, 0, 3432, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4052, 9, 104, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 0, 0, 0, 0, 0, 4, 4095, 3642, 3, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 44, 23, 19, 7, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8190, 8190, 7420, 8186, 8262, 8830, 8173, 8285] avg=8192.0000 max=8830 max_rank=5 min=7420 min_rank=2 imbalance_ratio=1.077881 max_load_reduction=56706 max_load_reduction_pct=0.865265 moved_to_echo=33550 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4078, 0, 7, 1, 1, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 1, 1, 0, 0, 1, 1, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 427, 4050, 1, 43, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 4090, 0, 15, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4058, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 198, 60, 2, 0, 1, 0, 0, 4093, 3547, 2, 0, 0, 0, 0, 0, 4078, 0, 0, 52, 1, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 1, 1, 4059, 0, 1, 1, 0, 37, 19, 16, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8184, 8200, 8550, 8199, 8153, 7904, 8211, 8135] avg=8192.0000 max=8550 max_rank=2 min=7904 min_rank=5 imbalance_ratio=1.043701 max_load_reduction=56986 max_load_reduction_pct=0.869537 moved_to_echo=32403 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 3584, 30976, 128, 256, 0, 31104, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66688 observed_loss=-1152 tokens_per_expert=[32512, 128, 128, 32512, 128, 128, 128, 0, 384, 256, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[32640, 0, 0, 256, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 3840, 128, 128, 0, 128, 0, 128, 32768, 28928, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 32512, 128, 256, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66688 slack_slots=64384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[632, 4096, 0, 1506, 0, 21, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 4085, 2202, 4096, 0, 0, 0, 3018, 0, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 4095, 4095, 0, 4066, 0, 2551, 0, 0, 0, 0, 0, 0, 80, 0, 4096, 0, 2319, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[391, 4090, 0, 2238, 1, 0, 1, 1, 0, 3, 0, 0, 0, 0, 4095, 182, 4095, 1052, 4094, 0, 0, 1, 4078, 0, 1, 4095, 0, 0, 1, 5, 4094, 4095, 0, 1, 0, 1, 1, 0, 0, 0, 0, 4095, 4091, 0, 4095, 4010, 1, 4092, 17, 1473, 0, 1, 1, 0, 1, 0, 0, 1, 4095, 3, 2941, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[2698, 4096, 0, 2024, 0, 7, 0, 0, 0, 1, 0, 0, 0, 0, 4093, 0, 4089, 490, 4096, 0, 0, 0, 3959, 0, 0, 4096, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 2, 4095, 4083, 0, 4010, 0, 903, 0, 0, 0, 0, 0, 1, 156, 0, 4096, 0, 2156, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[411, 4091, 0, 2004, 1, 0, 0, 0, 0, 6, 0, 1, 0, 0, 4094, 137, 4095, 1042, 4095, 0, 0, 0, 3866, 0, 1, 4095, 0, 0, 1, 5, 4095, 4095, 0, 1, 0, 1, 1, 0, 0, 0, 0, 4095, 4086, 0, 4095, 4045, 1, 3997, 10, 1872, 2, 1, 1, 0, 1, 0, 30, 0, 4095, 1, 3065, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1203, 4096, 0, 2427, 0, 7, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 4091, 2196, 4096, 0, 0, 0, 2462, 0, 0, 4096, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 4096, 4094, 0, 4007, 0, 2030, 0, 0, 0, 0, 0, 0, 105, 0, 4096, 0, 1951, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1932, 4096, 0, 2206, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4088, 90, 4096, 0, 0, 0, 3657, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 4095, 4096, 0, 4087, 0, 2143, 0, 0, 0, 0, 0, 0, 88, 0, 4096, 0, 2185, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[3377, 4096, 0, 349, 0, 4, 0, 0, 1, 1, 0, 0, 0, 0, 4095, 63, 4094, 3578, 4096, 0, 0, 0, 2928, 0, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4095, 4094, 0, 4075, 0, 1522, 0, 0, 0, 0, 0, 0, 39, 0, 4096, 0, 450, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[697, 4090, 0, 2647, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 201, 4095, 383, 4095, 0, 0, 0, 4047, 0, 1, 4095, 0, 0, 1, 2, 4094, 4095, 0, 1, 0, 1, 1, 1, 0, 1, 0, 4095, 4091, 0, 4095, 4018, 1, 4087, 10, 1039, 0, 1, 1, 1, 0, 0, 2, 1, 4095, 1, 3349, 0, 3, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[632, 4096, 0, 1506, 0, 21, 0, 0, 746, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 2327, 1697, 0, 0, 0, 0, 0, 0, 3306, 2202, 0, 0, 0, 0, 3018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 72, 3, 0, 4095, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2551, 0, 0, 0, 0, 0, 0, 4096, 2398, 0, 0, 0, 0, 0, 0, 80, 0, 4096, 0, 2319, 0, 0, 0, 1697, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7003, 8118, 8526, 8191, 8192, 8236, 9045, 8225] avg=8192.0000 max=9045 max_rank=6 min=7003 min_rank=0 imbalance_ratio=1.104126 max_load_reduction=56491 max_load_reduction_pct=0.861984 moved_to_echo=33698 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[411, 4091, 0, 2004, 1, 0, 0, 0, 748, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 4094, 137, 2322, 1696, 0, 0, 0, 0, 0, 0, 3314, 1042, 0, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 1, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 4045, 1, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1872, 2, 1, 1, 0, 1, 0, 4095, 2399, 1, 0, 0, 0, 0, 0, 30, 0, 4095, 1, 3065, 0, 1, 0, 1693, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7256, 8256, 8222, 8196, 8192, 8114, 8382, 8918] avg=8192.0000 max=8918 max_rank=7 min=7256 min_rank=0 imbalance_ratio=1.088623 max_load_reduction=56618 max_load_reduction_pct=0.863922 moved_to_echo=33606 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1932, 4096, 0, 2206, 0, 3, 0, 0, 746, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2327, 1697, 0, 0, 0, 0, 0, 0, 3309, 90, 0, 0, 0, 0, 3657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 72, 5, 0, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2143, 0, 0, 0, 0, 0, 0, 4096, 2398, 1, 0, 0, 0, 0, 0, 88, 0, 4096, 0, 2185, 0, 0, 0, 1697, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8985, 8117, 7056, 8190, 8191, 8260, 8638, 8099] avg=8192.0000 max=8985 max_rank=0 min=7056 min_rank=2 imbalance_ratio=1.096802 max_load_reduction=56551 max_load_reduction_pct=0.862900 moved_to_echo=33188 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2698, 4096, 0, 2024, 0, 7, 0, 0, 746, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4093, 0, 2327, 1697, 0, 0, 0, 0, 0, 0, 3310, 490, 0, 0, 0, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 72, 2, 0, 4083, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 903, 0, 0, 0, 0, 0, 1, 4095, 2398, 1, 0, 0, 0, 0, 0, 156, 0, 4096, 0, 2156, 0, 0, 0, 1697, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9573, 8118, 7759, 8192, 8191, 8167, 7398, 8138] avg=8192.0000 max=9573 max_rank=0 min=7398 min_rank=6 imbalance_ratio=1.168579 max_load_reduction=55963 max_load_reduction_pct=0.853928 moved_to_echo=31894 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[391, 4090, 0, 2238, 1, 0, 1, 1, 748, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4095, 182, 2325, 1696, 0, 0, 0, 0, 0, 0, 3314, 1052, 0, 0, 0, 1, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 1, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 4010, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1473, 0, 1, 1, 0, 1, 0, 4095, 2399, 1, 0, 0, 0, 0, 0, 0, 1, 4095, 3, 2941, 0, 2, 1, 1695, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7471, 8301, 8445, 8195, 8191, 8174, 7988, 8771] avg=8192.0000 max=8771 max_rank=7 min=7471 min_rank=0 imbalance_ratio=1.070679 max_load_reduction=56765 max_load_reduction_pct=0.866165 moved_to_echo=33124 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1203, 4096, 0, 2427, 0, 7, 0, 0, 747, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 2327, 1698, 0, 0, 0, 0, 0, 0, 3311, 2196, 0, 0, 0, 0, 2462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 72, 3, 0, 4094, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2030, 0, 0, 0, 0, 0, 0, 4096, 2398, 0, 0, 0, 0, 0, 0, 105, 0, 4096, 0, 1951, 0, 0, 0, 1697, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8482, 8119, 7969, 8192, 8192, 8176, 8524, 7882] avg=8192.0000 max=8524 max_rank=6 min=7882 min_rank=7 imbalance_ratio=1.040527 max_load_reduction=57012 max_load_reduction_pct=0.869934 moved_to_echo=32774 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3377, 4096, 0, 349, 0, 4, 0, 0, 750, 5, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4095, 63, 2332, 1697, 0, 0, 0, 0, 0, 0, 3309, 3578, 0, 0, 0, 0, 2928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 61, 2, 0, 4094, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1522, 0, 0, 0, 0, 0, 0, 4096, 2398, 0, 0, 0, 0, 0, 0, 39, 0, 4096, 0, 450, 0, 0, 0, 1703, 35, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8581, 8189, 9815, 8188, 8192, 8232, 8016, 6323] avg=8192.0000 max=9815 max_rank=2 min=6323 min_rank=7 imbalance_ratio=1.198120 max_load_reduction=55721 max_load_reduction_pct=0.850235 moved_to_echo=30763 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[697, 4090, 0, 2647, 1, 0, 1, 0, 748, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 201, 2325, 1697, 0, 0, 0, 0, 0, 0, 3314, 383, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 1, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 4018, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1039, 0, 1, 1, 1, 0, 0, 4095, 2398, 0, 0, 0, 0, 0, 0, 2, 1, 4095, 1, 3349, 0, 3, 1, 1695, 33, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8185, 8318, 7744, 8192, 8195, 8177, 7545, 9180] avg=8192.0000 max=9180 max_rank=7 min=7545 min_rank=6 imbalance_ratio=1.120605 max_load_reduction=56356 max_load_reduction_pct=0.859924 moved_to_echo=33097 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65792 observed_loss=-256 tokens_per_expert=[26496, 11136, 0, 0, 0, 128, 28032, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[11392, 32768, 0, 15488, 128, 128, 128, 128, 6016, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[0, 0, 640, 128, 0, 32640, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[512, 128, 32768, 128, 18432, 0, 128, 128, 13696, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 0, 0, 32768, 640, 18688, 13696, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 13568, 128, 128, 128, 128, 128, 128, 32768, 19200, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65792 slack_slots=65280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 1717, 0, 0, 0, 0, 0, 0, 1, 0, 179, 0, 1, 4096, 1, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4092, 2232, 0, 4096, 4095, 4096, 0, 4086, 4096, 0, 1, 4095, 4096, 3, 0, 0, 0, 1, 0, 4095, 4096, 0, 1, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 2, 0, 1576, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 5, 4096, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 4074, 2525, 0, 4096, 4096, 4096, 0, 4087, 4096, 0, 2, 4094, 4096, 4, 0, 0, 0, 2, 0, 4094, 4096, 0, 0, 4090, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 1, 37, 0, 0, 1, 0, 3559, 0, 1, 0, 0, 73, 0, 250, 2, 0, 1, 21, 1, 1, 0, 1, 1279, 1, 18, 4092, 2, 0, 1, 4095, 5, 0, 0, 4086, 0, 0, 4032, 3096, 3, 4095, 4088, 4094, 0, 4042, 4093, 0, 0, 4075, 4094, 1, 1, 8, 1, 0, 1, 4093, 4095, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 2205, 0, 0, 0, 1, 0, 0, 0, 0, 494, 0, 2, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4092, 1544, 0, 4095, 4095, 4096, 0, 4092, 4095, 0, 3, 4096, 4096, 5, 0, 0, 0, 1, 0, 4095, 4096, 0, 1, 4088, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 1, 0, 0, 0, 1396, 0, 0, 0, 2, 0, 0, 0, 0, 728, 0, 3, 4096, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 4086, 2010, 0, 4096, 4096, 4096, 0, 4094, 4096, 0, 2, 4095, 4096, 5, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 1, 68, 1, 0, 1, 0, 3638, 0, 1, 0, 0, 39, 0, 456, 2, 0, 1, 22, 1, 1, 0, 1, 1012, 1, 4, 4090, 3, 0, 1, 4095, 6, 0, 0, 4084, 0, 1, 4094, 3086, 4, 4095, 4090, 4075, 1, 3994, 4093, 0, 0, 4086, 4092, 0, 2, 9, 1, 0, 1, 4091, 4095, 0, 0, 4095, 0, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 3247, 0, 0, 0, 0, 0, 0, 829, 0, 0, 0, 1, 0, 0, 0, 0, 2620, 0, 1, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4094, 1553, 0, 4096, 4095, 4096, 0, 4039, 4096, 0, 1, 4096, 4096, 2, 0, 0, 0, 1, 0, 4095, 4096, 0, 1, 4094, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 1, 68, 0, 0, 1, 0, 3962, 1, 1, 0, 0, 30, 0, 471, 4, 0, 1, 15, 1, 1, 0, 1, 716, 1, 4, 4090, 7, 0, 1, 4095, 3, 0, 0, 4085, 0, 1, 4091, 3021, 3, 4095, 4089, 4086, 0, 4021, 4092, 0, 0, 4087, 4093, 0, 3, 10, 1, 0, 1, 4093, 4095, 0, 0, 4093, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 37, 0, 0, 1, 0, 4090, 4046, 18, 8, 0, 0, 0, 0, 3559, 0, 1, 0, 0, 73, 0, 250, 2345, 892, 2, 0, 0, 0, 0, 0, 2, 0, 1, 21, 1, 1, 0, 1, 4095, 4085, 0, 0, 0, 0, 0, 0, 1279, 1, 18, 4092, 2, 0, 1, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1723, 0, 0, 4032, 3096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 42, 4094, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4092, 1, 1, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 5, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8202, 7122, 8207, 8596, 8856, 8181, 8178, 8194] avg=8192.0000 max=8856 max_rank=4 min=7122 min_rank=1 imbalance_ratio=1.081055 max_load_reduction=56680 max_load_reduction_pct=0.864868 moved_to_echo=33409 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 4054, 18, 8, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 2, 0, 1576, 2351, 891, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 78, 0, 5, 4096, 0, 0, 0, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1727, 0, 0, 4074, 2525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 4094, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8171, 8863, 8184, 7382, 8326, 8225, 8196, 8189] avg=8192.0000 max=8863 max_rank=1 min=7382 min_rank=3 imbalance_ratio=1.081909 max_load_reduction=56673 max_load_reduction_pct=0.864761 moved_to_echo=32936 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 0, 0, 4091, 4053, 18, 9, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 2205, 2351, 892, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 4086, 0, 0, 0, 0, 0, 0, 494, 0, 2, 4096, 0, 0, 0, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1727, 0, 0, 4092, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4094, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 9406, 8182, 7795, 7363, 8230, 8199, 8189] avg=8192.0000 max=9406 max_rank=1 min=7363 min_rank=4 imbalance_ratio=1.148193 max_load_reduction=56130 max_load_reduction_pct=0.856476 moved_to_echo=31981 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 4053, 18, 8, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 1717, 2351, 892, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 179, 0, 1, 4096, 1, 0, 0, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1727, 0, 0, 4092, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4094, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8170, 9036, 8185, 7480, 8051, 8224, 8194, 8196] avg=8192.0000 max=9036 max_rank=1 min=7480 min_rank=3 imbalance_ratio=1.103027 max_load_reduction=56500 max_load_reduction_pct=0.862122 moved_to_echo=32665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4091, 4054, 18, 9, 0, 0, 0, 0, 4065, 0, 0, 1, 0, 0, 0, 1396, 2351, 891, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 728, 0, 3, 4096, 0, 0, 0, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1727, 0, 0, 4086, 2010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8172, 8706, 8185, 8030, 7823, 8232, 8196, 8192] avg=8192.0000 max=8706 max_rank=1 min=7823 min_rank=4 imbalance_ratio=1.062744 max_load_reduction=56830 max_load_reduction_pct=0.867157 moved_to_echo=32443 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 68, 1, 0, 1, 0, 4090, 4048, 18, 8, 0, 0, 0, 0, 3638, 0, 1, 0, 0, 39, 0, 456, 2344, 892, 2, 0, 0, 0, 0, 0, 2, 0, 1, 22, 1, 1, 0, 1, 4095, 4085, 0, 0, 0, 0, 0, 0, 1012, 1, 4, 4090, 3, 0, 1, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1722, 0, 1, 4094, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 42, 4075, 1, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4090, 0, 2, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 5, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8236, 7372, 8208, 8314, 8909, 8116, 8188, 8193] avg=8192.0000 max=8909 max_rank=4 min=7372 min_rank=1 imbalance_ratio=1.087524 max_load_reduction=56627 max_load_reduction_pct=0.864059 moved_to_echo=33406 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 4054, 20, 9, 0, 0, 0, 0, 3247, 0, 0, 0, 0, 0, 0, 829, 2354, 895, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 2620, 0, 1, 4096, 0, 0, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1722, 0, 0, 4094, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 4096, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4093, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8178, 7328, 8184, 9917, 7369, 8176, 8193, 8191] avg=8192.0000 max=9917 max_rank=3 min=7328 min_rank=1 imbalance_ratio=1.210571 max_load_reduction=55619 max_load_reduction_pct=0.848679 moved_to_echo=31929 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 68, 0, 0, 1, 0, 4090, 4047, 18, 9, 0, 0, 0, 0, 3962, 1, 1, 0, 0, 30, 0, 471, 2344, 892, 2, 0, 0, 0, 0, 0, 4, 0, 1, 15, 1, 1, 0, 1, 4095, 4083, 0, 0, 0, 0, 0, 0, 716, 1, 4, 4090, 7, 0, 1, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1723, 0, 1, 4091, 3021, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 42, 4086, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4091, 0, 3, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 5, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8235, 7703, 8201, 8022, 8839, 8152, 8192, 8192] avg=8192.0000 max=8839 max_rank=4 min=7703 min_rank=1 imbalance_ratio=1.078979 max_load_reduction=56697 max_load_reduction_pct=0.865128 moved_to_echo=33375 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[7168, 128, 128, 32768, 128, 0, 128, 25728, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 256, 128, 0, 128, 0, 32768, 32512, 256, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 384, 32768, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 0, 13824, 0, 128, 32768, 19072, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[30592, 128, 128, 128, 0, 256, 0, 8960, 18816, 7168, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[2678, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 2, 0, 1, 0, 4096, 0, 199, 0, 0, 1, 0, 4096, 0, 3729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1625, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4096, 0, 0, 4094, 0, 0, 0, 4012, 0, 44, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[3118, 0, 0, 0, 21, 4095, 4083, 0, 0, 3, 0, 4095, 2, 0, 4078, 1, 4095, 0, 14, 0, 0, 4095, 7, 2230, 0, 1, 0, 21, 2515, 1, 4044, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 376, 0, 1, 3, 0, 4083, 4095, 1, 4092, 4094, 0, 1, 4095, 0, 0, 1, 4095, 0, 0, 4074, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1666, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 4096, 0, 39, 0, 0, 0, 0, 4096, 0, 3890, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2332, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4096, 0, 0, 4094, 0, 0, 0, 4006, 0, 353, 4096, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[3118, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 4096, 0, 375, 1, 0, 0, 0, 4093, 0, 3533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1426, 0, 0, 0, 0, 4096, 4096, 0, 4094, 4096, 0, 0, 4093, 0, 0, 0, 3833, 0, 6, 4096, 2, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[2971, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 2, 0, 1, 0, 4096, 0, 81, 1, 0, 1, 0, 4096, 0, 3736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4096, 0, 0, 4095, 0, 0, 0, 3987, 0, 419, 4094, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[3602, 1, 0, 0, 39, 4095, 4081, 0, 1, 2, 0, 4095, 2, 0, 4062, 1, 4095, 0, 29, 0, 0, 4094, 14, 1687, 0, 1, 0, 15, 2874, 1, 3895, 1, 0, 0, 1, 1, 2, 0, 1, 1, 0, 156, 0, 1, 1, 0, 4078, 4095, 0, 4094, 4094, 0, 1, 4095, 1, 0, 1, 4094, 0, 70, 4061, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[3816, 1, 0, 0, 47, 4095, 4085, 0, 1, 2, 0, 4095, 1, 0, 4049, 1, 4095, 0, 20, 0, 0, 4094, 11, 1269, 0, 1, 0, 15, 3064, 1, 3867, 1, 0, 0, 1, 1, 2, 0, 1, 1, 0, 251, 0, 1, 1, 0, 4080, 4095, 0, 4093, 4094, 0, 1, 4095, 1, 0, 1, 4092, 0, 20, 4073, 0, 1, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[2352, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 4096, 0, 1195, 1, 0, 1, 0, 4060, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 904, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4096, 0, 0, 4095, 0, 0, 0, 3884, 0, 16, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3118, 0, 0, 0, 21, 1172, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 2, 0, 4078, 1, 10, 0, 0, 0, 0, 0, 0, 0, 3199, 0, 14, 0, 0, 4095, 7, 2230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 21, 2515, 1, 4044, 1, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 4091, 2923, 1014, 158, 0, 0, 0, 0, 0, 376, 0, 1, 3, 0, 4083, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 3, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 4064, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8394, 8189, 9545, 7321, 8190, 7544, 8192, 8161] avg=8192.0000 max=9545 max_rank=2 min=7321 min_rank=3 imbalance_ratio=1.165161 max_load_reduction=55991 max_load_reduction_pct=0.854355 moved_to_echo=32087 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2678, 0, 0, 0, 0, 1173, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 3200, 2, 0, 1, 0, 4096, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 3729, 0, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2923, 1015, 158, 0, 0, 0, 0, 0, 1625, 0, 0, 0, 0, 4096, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 44, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7947, 8202, 7498, 8564, 8189, 8802, 8192, 8142] avg=8192.0000 max=8802 max_rank=5 min=7498 min_rank=2 imbalance_ratio=1.074463 max_load_reduction=56734 max_load_reduction_pct=0.865692 moved_to_echo=33325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1666, 0, 0, 0, 0, 1173, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 3200, 1, 0, 0, 0, 4096, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3890, 0, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 2923, 1015, 158, 0, 0, 0, 0, 0, 2332, 0, 0, 0, 0, 4096, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 353, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6935, 8203, 7336, 8724, 8191, 9509, 8192, 8446] avg=8192.0000 max=9509 max_rank=5 min=6935 min_rank=0 imbalance_ratio=1.160767 max_load_reduction=56027 max_load_reduction_pct=0.854904 moved_to_echo=34338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3118, 0, 0, 0, 0, 1173, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 3200, 2, 0, 0, 0, 4096, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 3533, 0, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2923, 1015, 158, 0, 0, 0, 0, 0, 1426, 0, 0, 0, 0, 4096, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3833, 0, 6, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8387, 8203, 7673, 8365, 8189, 8603, 8190, 7926] avg=8192.0000 max=8603 max_rank=5 min=7673 min_rank=2 imbalance_ratio=1.050171 max_load_reduction=56933 max_load_reduction_pct=0.868729 moved_to_echo=32908 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2971, 0, 0, 0, 0, 1173, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 3200, 2, 0, 1, 0, 4096, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4096, 0, 3736, 0, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2923, 1015, 158, 0, 0, 0, 0, 0, 1093, 0, 0, 0, 0, 4096, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 419, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8240, 8202, 7380, 8572, 8189, 8270, 8193, 8490] avg=8192.0000 max=8572 max_rank=3 min=7380 min_rank=2 imbalance_ratio=1.046387 max_load_reduction=56964 max_load_reduction_pct=0.869202 moved_to_echo=33142 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3602, 1, 0, 0, 39, 1172, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4095, 2, 0, 4062, 1, 10, 0, 0, 0, 0, 0, 0, 0, 3199, 0, 29, 0, 0, 4094, 14, 1687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 2874, 1, 3895, 1, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 1, 1, 4091, 2923, 1014, 158, 0, 0, 0, 0, 0, 156, 0, 1, 1, 0, 4078, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 70, 4051, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8895, 8173, 9023, 7525, 8192, 7317, 8194, 8217] avg=8192.0000 max=9023 max_rank=2 min=7317 min_rank=5 imbalance_ratio=1.101440 max_load_reduction=56513 max_load_reduction_pct=0.862320 moved_to_echo=31920 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3816, 1, 0, 0, 47, 1172, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4095, 1, 0, 4049, 1, 10, 0, 0, 0, 0, 0, 0, 0, 3199, 0, 20, 0, 0, 4094, 11, 1269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 3064, 1, 3867, 1, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 1, 1, 4091, 2923, 1014, 158, 0, 0, 0, 0, 0, 251, 0, 1, 1, 0, 4080, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 20, 4063, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9121, 8159, 8593, 7687, 8192, 7414, 8193, 8177] avg=8192.0000 max=9121 max_rank=0 min=7414 min_rank=5 imbalance_ratio=1.113403 max_load_reduction=56415 max_load_reduction_pct=0.860825 moved_to_echo=31976 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2352, 0, 0, 0, 0, 1169, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 0, 3195, 2, 0, 0, 0, 4096, 0, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4060, 0, 3971, 0, 739, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2927, 1019, 162, 0, 0, 0, 0, 0, 904, 0, 0, 0, 0, 4096, 3077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 16, 4076, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7617, 8205, 8488, 8778, 8204, 8077, 8190, 7977] avg=8192.0000 max=8778 max_rank=3 min=7617 min_rank=0 imbalance_ratio=1.071533 max_load_reduction=56758 max_load_reduction_pct=0.866058 moved_to_echo=32448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[25600, 128, 128, 128, 0, 32768, 128, 7168, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[23424, 128, 0, 0, 128, 9472, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32128, 0, 1024, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 28928, 128, 30720, 128, 6016, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 128, 32768, 23424, 8192, 1280, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[0, 8192, 0, 128, 128, 0, 32768, 24704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[2292, 3994, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 4096, 0, 3543, 0, 0, 0, 4096, 4093, 2441, 15, 0, 1, 4096, 0, 0, 0, 0, 5, 0, 4095, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[2871, 3926, 16, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 4092, 5, 2401, 0, 1, 0, 4075, 4095, 3129, 1, 0, 0, 4095, 1, 1, 0, 0, 0, 7, 4095, 0, 4095, 85, 1, 3, 4094, 4095, 29, 0, 3935, 0, 1, 0, 4089, 1, 4095, 0, 1, 1, 0, 4094, 0, 1, 1, 1, 1, 1, 0, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[1400, 3758, 1, 0, 0, 0, 4, 0, 1, 1, 1, 1, 1, 0, 4092, 2, 3413, 0, 1, 0, 4091, 4095, 3632, 55, 0, 0, 4095, 1, 1, 0, 0, 0, 11, 4093, 0, 4095, 100, 1, 3, 4095, 4095, 9, 0, 4000, 0, 1, 0, 4094, 1, 4095, 1, 0, 1, 0, 4094, 1, 1, 1, 1, 1, 0, 0, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[2571, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3753, 0, 0, 0, 4094, 4092, 1906, 16, 0, 2, 4096, 0, 0, 0, 0, 10, 0, 4094, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[2706, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3459, 0, 0, 0, 4096, 4094, 2082, 3, 0, 2, 4096, 0, 0, 0, 0, 8, 0, 4094, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[2685, 4050, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4066, 0, 0, 0, 4096, 4092, 1478, 19, 0, 1, 4096, 0, 0, 0, 0, 9, 0, 4095, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 4081, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[2664, 4030, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 3964, 0, 0, 0, 4096, 4093, 1612, 18, 0, 1, 4096, 0, 0, 0, 0, 3, 0, 4095, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[2754, 3954, 21, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 4090, 11, 2719, 0, 1, 0, 4085, 4095, 2870, 5, 0, 0, 4095, 1, 1, 0, 0, 0, 11, 4095, 0, 4095, 102, 1, 3, 4094, 4095, 22, 0, 3925, 0, 1, 0, 4091, 1, 4095, 0, 1, 1, 0, 4094, 0, 1, 1, 2, 0, 0, 0, 4095, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1400, 3758, 1, 0, 0, 0, 4, 0, 1681, 38, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4092, 2, 4093, 1, 0, 0, 0, 0, 0, 0, 3413, 0, 1, 0, 2367, 0, 3632, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 4045, 43, 2, 0, 0, 0, 0, 0, 11, 4093, 0, 0, 100, 1, 3, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 50, 9, 0, 4000, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 1, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 4095, 1, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6882, 8193, 9468, 8187, 8265, 8154, 8193, 8194] avg=8192.0000 max=9468 max_rank=2 min=6882 min_rank=0 imbalance_ratio=1.155762 max_load_reduction=56068 max_load_reduction_pct=0.855530 moved_to_echo=32806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2292, 3994, 0, 0, 0, 0, 0, 1, 1683, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 3543, 0, 0, 0, 2370, 0, 2441, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 5, 4045, 43, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8008, 8190, 8369, 8191, 8153, 8242, 8192, 8191] avg=8192.0000 max=8369 max_rank=2 min=8008 min_rank=0 imbalance_ratio=1.021606 max_load_reduction=57167 max_load_reduction_pct=0.872299 moved_to_echo=32778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2571, 4040, 0, 0, 0, 0, 0, 0, 1682, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 3753, 0, 0, 0, 2369, 0, 1906, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 10, 4046, 43, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8331, 8188, 8044, 8198, 8152, 8239, 8193, 8191] avg=8192.0000 max=8331 max_rank=0 min=8044 min_rank=2 imbalance_ratio=1.016968 max_load_reduction=57205 max_load_reduction_pct=0.872879 moved_to_echo=32775 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2871, 3926, 16, 0, 0, 1, 0, 0, 1674, 38, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4092, 5, 4093, 1, 0, 0, 0, 0, 0, 0, 2401, 0, 1, 0, 2358, 0, 3129, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 4045, 43, 2, 0, 0, 0, 0, 0, 7, 4095, 0, 0, 85, 1, 3, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 50, 29, 0, 3935, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4095, 1, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8526, 8195, 7890, 8187, 8247, 8104, 8192, 8195] avg=8192.0000 max=8526 max_rank=0 min=7890 min_rank=2 imbalance_ratio=1.040771 max_load_reduction=57010 max_load_reduction_pct=0.869904 moved_to_echo=32738 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2706, 4030, 0, 0, 0, 0, 0, 0, 1683, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 3459, 0, 0, 0, 2370, 0, 2082, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 8, 4046, 43, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8457, 8190, 7914, 8196, 8152, 8244, 8192, 8191] avg=8192.0000 max=8457 max_rank=0 min=7914 min_rank=2 imbalance_ratio=1.032349 max_load_reduction=57079 max_load_reduction_pct=0.870956 moved_to_echo=32779 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2685, 4050, 0, 0, 0, 0, 0, 1, 1683, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 2370, 0, 1478, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 9, 4045, 43, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8457, 8188, 7933, 8195, 8153, 8227, 8192, 8191] avg=8192.0000 max=8457 max_rank=0 min=7933 min_rank=2 imbalance_ratio=1.032349 max_load_reduction=57079 max_load_reduction_pct=0.870956 moved_to_echo=32763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2664, 4030, 0, 0, 0, 0, 0, 1, 1686, 45, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 3964, 0, 0, 0, 2365, 0, 1612, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3, 4050, 45, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8428, 8190, 7959, 8196, 8146, 8237, 8190, 8190] avg=8192.0000 max=8428 max_rank=0 min=7959 min_rank=2 imbalance_ratio=1.028809 max_load_reduction=57108 max_load_reduction_pct=0.871399 moved_to_echo=32763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2754, 3954, 21, 0, 0, 1, 1, 0, 1678, 38, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 4090, 11, 4094, 2, 0, 0, 0, 0, 0, 0, 2719, 0, 1, 0, 2364, 0, 2870, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 0, 4045, 43, 1, 0, 0, 0, 0, 0, 11, 4095, 0, 0, 102, 1, 3, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 50, 22, 0, 3925, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 0, 0, 4095, 1, 4093, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8447, 8202, 7959, 8186, 8268, 8089, 8192, 8193] avg=8192.0000 max=8447 max_rank=0 min=7959 min_rank=2 imbalance_ratio=1.031128 max_load_reduction=57089 max_load_reduction_pct=0.871109 moved_to_echo=32742 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[512, 128, 0, 32384, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[19968, 31872, 128, 0, 0, 128, 128, 128, 13568, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[27392, 0, 128, 0, 18944, 0, 19200, 256, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 0, 384, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 0, 128, 32384, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66560 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66560 slack_slots=64512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4095, 0, 0, 3178, 0, 0, 4096, 0, 3239, 0, 0, 4028, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1834, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 3, 4096, 0, 4094, 4096, 7, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3442, 0, 0, 4096, 0, 3706, 0, 0, 4087, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1042, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4, 5, 4096, 0, 4095, 4096, 0, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4094, 60, 1, 3257, 0, 6, 4063, 0, 3670, 1, 111, 3007, 1, 0, 1, 0, 4095, 0, 2, 0, 1, 4094, 0, 2, 1, 0, 1, 4094, 0, 4094, 0, 1776, 0, 1, 0, 4095, 0, 0, 0, 0, 1, 4082, 1, 9, 0, 540, 1, 1, 3, 1, 4082, 0, 0, 1, 4095, 0, 0, 4091, 0, 4002, 4095, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4095, 0, 0, 3540, 0, 0, 4096, 0, 3504, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1135, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 5, 4096, 0, 4096, 4096, 7, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 1, 0, 3584, 0, 0, 4096, 0, 3788, 0, 2, 4084, 0, 0, 2, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 819, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 7, 4096, 0, 4093, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4095, 40, 1, 3372, 1, 30, 4064, 0, 3276, 1, 100, 3244, 2, 0, 0, 0, 4095, 0, 4, 0, 1, 4094, 0, 1, 0, 0, 1, 4094, 0, 4091, 0, 1992, 0, 1, 1, 4095, 0, 2, 1, 0, 1, 4066, 1, 0, 0, 414, 1, 1, 9, 1, 4090, 0, 0, 1, 4095, 0, 0, 4089, 0, 3971, 4094, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3946, 0, 0, 4096, 0, 4059, 0, 4, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 165, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 4, 4096, 0, 4095, 4096, 1, 0, 0] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4094, 48, 1, 3189, 1, 22, 4070, 0, 3452, 1, 109, 3004, 2, 0, 1, 0, 4095, 0, 0, 0, 1, 4094, 0, 2, 1, 0, 1, 4094, 0, 4093, 0, 2172, 0, 1, 1, 4095, 0, 0, 0, 0, 1, 4078, 1, 0, 0, 435, 1, 1, 8, 1, 4087, 0, 0, 2, 4095, 0, 0, 4089, 0, 3996, 4094, 0, 2, 1] tokens_per_ep_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] avg=65536.0000 max=65536 max_rank=0 min=65536 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[643, 60, 1, 3257, 0, 6, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3670, 1, 111, 3007, 1, 0, 1, 0, 855, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 1, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2730, 0, 4094, 0, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4058, 36, 0, 0, 0, 0, 0, 0, 1, 4082, 1, 9, 0, 540, 1, 1, 3415, 509, 0, 0, 0, 0, 0, 0, 3, 1, 4082, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4002, 37, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8030, 7646, 8194, 8602, 8190, 8559, 8182, 8133] avg=8192.0000 max=8602 max_rank=3 min=7646 min_rank=1 imbalance_ratio=1.050049 max_load_reduction=56934 max_load_reduction_pct=0.868744 moved_to_echo=33064 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[643, 0, 0, 3178, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3239, 0, 0, 4028, 0, 0, 1, 0, 855, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2731, 0, 4096, 0, 1834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4058, 36, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3416, 510, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4094, 38, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7917, 8123, 8191, 8661, 8191, 8022, 8193, 8238] avg=8192.0000 max=8661 max_rank=3 min=7917 min_rank=0 imbalance_ratio=1.057251 max_load_reduction=56875 max_load_reduction_pct=0.867844 moved_to_echo=32644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[643, 1, 0, 3584, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3788, 0, 2, 4084, 0, 0, 2, 0, 855, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2731, 0, 4096, 0, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4058, 36, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3417, 510, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 4093, 38, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8324, 8731, 8193, 7646, 8192, 8023, 8192, 8235] avg=8192.0000 max=8731 max_rank=1 min=7646 min_rank=3 imbalance_ratio=1.065796 max_load_reduction=56805 max_load_reduction_pct=0.866776 moved_to_echo=32642 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[643, 0, 0, 3442, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3706, 0, 0, 4087, 0, 0, 1, 0, 855, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2731, 0, 4096, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4058, 36, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3417, 510, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 4095, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8181, 8649, 8192, 7869, 8192, 8023, 8196, 8234] avg=8192.0000 max=8649 max_rank=1 min=7869 min_rank=3 imbalance_ratio=1.055786 max_load_reduction=56887 max_load_reduction_pct=0.868027 moved_to_echo=32645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[643, 40, 1, 3372, 1, 30, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3276, 1, 100, 3244, 2, 0, 0, 0, 855, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2730, 0, 4091, 0, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 2, 1, 0, 4057, 36, 0, 0, 0, 0, 0, 0, 1, 4066, 1, 0, 0, 414, 1, 1, 3416, 509, 0, 0, 0, 0, 0, 0, 9, 1, 4090, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3971, 37, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8151, 7478, 8195, 8814, 8193, 8409, 8196, 8100] avg=8192.0000 max=8814 max_rank=3 min=7478 min_rank=1 imbalance_ratio=1.075928 max_load_reduction=56722 max_load_reduction_pct=0.865509 moved_to_echo=32898 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[643, 0, 0, 3540, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3504, 0, 0, 4096, 0, 0, 0, 0, 855, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2731, 0, 4096, 0, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4058, 36, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3416, 510, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 4096, 38, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8279, 8455, 8192, 7962, 8190, 8022, 8194, 8242] avg=8192.0000 max=8455 max_rank=1 min=7962 min_rank=3 imbalance_ratio=1.032104 max_load_reduction=57081 max_load_reduction_pct=0.870987 moved_to_echo=32648 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[638, 0, 0, 3946, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 4, 4096, 0, 0, 2, 0, 859, 5, 5, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2724, 0, 4096, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4063, 38, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 3420, 513, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4095, 33, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8680, 9030, 8187, 6985, 8198, 8037, 8190, 8229] avg=8192.0000 max=9030 max_rank=1 min=6985 min_rank=3 imbalance_ratio=1.102295 max_load_reduction=56506 max_load_reduction_pct=0.862213 moved_to_echo=32654 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[643, 48, 1, 3189, 1, 22, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3452, 1, 109, 3004, 2, 0, 1, 0, 855, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2730, 0, 4093, 0, 2172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 4057, 36, 0, 0, 0, 0, 0, 0, 1, 4078, 1, 0, 0, 435, 1, 1, 3415, 509, 0, 0, 0, 0, 0, 0, 8, 1, 4087, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3996, 37, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7974, 7424, 8192, 8997, 8190, 8441, 8193, 8125] avg=8192.0000 max=8997 max_rank=3 min=7424 min_rank=1 imbalance_ratio=1.098267 max_load_reduction=56539 max_load_reduction_pct=0.862717 moved_to_echo=32949 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=16 budget=131072 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[5248, 256, 128, 27520, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=131072 expected_recv=65536 actual_valid_recv=65920 observed_loss=-384 tokens_per_expert=[128, 0, 128, 21888, 0, 32768, 0, 11008, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 128, 0, 32512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=131072 expected_recv=65536 actual_valid_recv=66176 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 32512, 384, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=131072 expected_recv=65536 actual_valid_recv=66048 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=131072 expected_recv=65536 actual_valid_recv=66432 observed_loss=-896 tokens_per_expert=[28800, 128, 384, 29696, 128, 0, 128, 0, 6912, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=131072 expected_recv=65536 actual_valid_recv=66304 observed_loss=-768 tokens_per_expert=[128, 32768, 128, 128, 0, 1408, 128, 128, 27392, 4096, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=131072 valid_num_tokens=66176 slack_slots=64896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=131072 valid_num_tokens=65920 slack_slots=65152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=131072 valid_num_tokens=66048 slack_slots=65024 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=131072 valid_num_tokens=66432 slack_slots=64640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=131072 valid_num_tokens=66304 slack_slots=64768 device_initiated_grouped_gemm=True + [2026-08-30 04:59:33] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 715.9 | throughput per GPU (TFLOP/s/GPU): 465.7 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.854495E+00 | loss scale: 1.0 | grad norm: 5.277 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 04:59:33 +[rank0]:[W830 04:59:35.272017964 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 04:59:39.147000 455382 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 04:59:39.148000 455382 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 455476 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 455382 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..2cdae5f06275aecda9ee05125a454dc885514640 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv @@ -0,0 +1,40 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/run.env b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..7d4f10a04dcebc455ab2725d67435cfde40b2351 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/run.env @@ -0,0 +1,70 @@ +backend=echo +config=G4 +family=tierA +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=896 +num_experts=128 +topk=32 +experts_per_rank=16 +expert_params=11010048 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=131072 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=2.00 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..78dd598df00ef4e32eabf9f9bfc4268123423118 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log @@ -0,0 +1,13 @@ +[05:02:24] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G4 / echo (slots/rank=16 policy=matched) + E=128 K=32 H'=896 m=16 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=2.00 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0830 05:02:25.817000 466818 torch/distributed/run.py:766] +W0830 05:02:25.817000 466818 torch/distributed/run.py:766] ***************************************** +W0830 05:02:25.817000 466818 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 05:02:25.817000 466818 torch/distributed/run.py:766] ***************************************** diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9714c242d3c4a17cc3caedca625ecff6c99a5a51 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv @@ -0,0 +1,8 @@ +model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir +G1 echo measure 2.00 OK 125477 1 0.0 1.90625 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 +G2 echo measure 2.00 OK 130289 1 0.0 1.02734375 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1 +G2 echo validate 1.08203125 OK 119267 1 0.0 1.02734375 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 +G2 echo forced 1.08203125 OK 119267 1 0.0 1.0234375 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 +G3 echo measure 2.00 OK 136111 1 0.0 1.01953125 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1 +G3 echo validate 1.072265625 OK 122853 1 0.0 1.01953125 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1 +G3 echo forced 1.072265625 OK 122899 1 0.0 1.017578125 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d9006e93b451fc939d96f0643bd9bf2f2ebef3f6 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv @@ -0,0 +1,4 @@ +model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir +G1 echo REQUIRED_CAPACITY_ABOVE_2.00 2.00 1.90625 0.0 125477 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 +G2 echo SELECTED 1.08203125 1.02734375 0.0 119267 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 +G3 echo SELECTED 1.072265625 1.01953125 0.0 122853 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/results.tsv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/results.tsv new file mode 100644 index 0000000000000000000000000000000000000000..51b7bb1060d2e6628551a2d6059daa0a0d9dccb6 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/results.tsv @@ -0,0 +1,8 @@ +config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir +G1 echo matched 1 4096 16 2.00 none 1 OK 671.53 666.90 125477 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 +G2 echo matched 4 4096 16 2.00 none 1 OK 675.77 669.20 130289 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1 +G2 echo matched 4 4096 16 1.08203125 none 1 OK 660.43 642.00 119267 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 +G2 echo matched 4 4096 16 1.08203125 none+forcebal 1 OK 628.07 616.10 119267 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 +G3 echo matched 8 4096 16 2.00 none 1 OK 721.76 713.60 136111 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1 +G3 echo matched 8 4096 16 1.072265625 none 1 OK 681.37 670.20 122853 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1 +G3 echo matched 8 4096 16 1.072265625 none+forcebal 1 OK 664.27 659.10 122899 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1 diff --git a/tiera-rerun-fixed-moonep/hf_watcher.log b/tiera-rerun-fixed-moonep/hf_watcher.log index 9981b8fb422680bc2214a46cfcd1c806e8dbbfc2..1b9c2665da85258862098e967b09dd7cbd71e488 100644 --- a/tiera-rerun-fixed-moonep/hf_watcher.log +++ b/tiera-rerun-fixed-moonep/hf_watcher.log @@ -1,3 +1,6 @@ Already logged in to Hugging Face as: AbhishekGhosh1998 Watching /workspace/tiera-rerun-fixed-moonep -> HF every 600s (kill to stop) [2026-08-30T04:52:39+00:00] backup pass + uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep +https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main +[2026-08-30T05:02:40+00:00] backup pass diff --git a/tiera-rerun-fixed-moonep/study.log b/tiera-rerun-fixed-moonep/study.log index 988a1a5e1e97f72990cf96b09d30a83b0658703c..6ffa67840555ca52d3d1c0c73708f9bdcfaa701c 100644 --- a/tiera-rerun-fixed-moonep/study.log +++ b/tiera-rerun-fixed-moonep/study.log @@ -10,3 +10,45 @@ driver_repo: /workspace/blackwell_run_scripts @ 774e9e2 === STAGE 7 -- Tier B probe and zero-drop capacity selection === STEP echo__tierbprobe__gbs16 +============================================================== + CAMPAIGN: echo backend=echo phase=tierbprobe + Megatron: /workspace/Megatron-MoE-Echo @ token-drop-analysis + GBS=16 seq=4096 repeats=3 nsys=1 + results -> /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe +============================================================== +-------------------------------------------------------------------------- +[04:52:54] CELL G1__echo__matched__seq4096__gbs16__cap2p00 (slots=1) + [guard] iter 6/12 (70s) + [guard] iter 12/12 (80s) +-------------------------------------------------------------------------- +[04:54:14] CELL G2__echo__matched__seq4096__gbs16__cap2p00 (slots=4) + [guard] iter 6/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +[04:55:36] G2 echo: measured required=1.02734375 at C=2.00 -> validating C=1.08203125 (safety=+5%) +-------------------------------------------------------------------------- +[04:55:36] CELL G2__echo__matched__seq4096__gbs16__cap1p08203125 (slots=4) + [guard] iter 5/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[04:56:58] CELL G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal (slots=4) + [guard] iter 9/12 (70s) + [guard] iter 12/12 (80s) +-------------------------------------------------------------------------- +[04:58:18] CELL G3__echo__matched__seq4096__gbs16__cap2p00 (slots=8) + [guard] iter 5/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +[04:59:41] G3 echo: measured required=1.01953125 at C=2.00 -> validating C=1.072265625 (safety=+5%) +-------------------------------------------------------------------------- +[04:59:41] CELL G3__echo__matched__seq4096__gbs16__cap1p072265625 (slots=8) + [guard] iter 6/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[05:01:03] CELL G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal (slots=8) + [guard] iter 8/12 (70s) + [guard] iter 12/12 (80s) +-------------------------------------------------------------------------- +[05:02:24] CELL G4__echo__matched__seq4096__gbs16__cap2p00 (slots=16)